From cd9c1727633be2ed7a9843ba819023b6c474e3cd Mon Sep 17 00:00:00 2001 From: Yicong Hui Date: Fri, 21 Aug 2026 20:18:49 +0200 Subject: [PATCH 0001/1579] HID: dockchannel-hid: forward correct report type on SET_REPORT dchid_set_report() always sends HID_OUTPUT_REPORT regardless of the report type requested by the caller. Inline it into its sole caller and forward rtype directly, so SET_REPORT works correctly for feature reports too, not just output reports, and to simplify function calls. Signed-off-by: Yicong Hui --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 9cab638b30c32..68f6625313ea3 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -593,12 +593,6 @@ static int dchid_get_report_cmd(struct dchid_iface *iface, u8 reportnum, void *b return ret <= 0 ? ret : ret - 1; } -/* Note: buf includes report number! */ -static int dchid_set_report(struct dchid_iface *iface, void *buf, size_t len) -{ - return dchid_cmd(iface, HID_OUTPUT_REPORT, REQ_SET_REPORT, buf, len, NULL, 0); -} - static int dchid_raw_request(struct hid_device *hdev, unsigned char reportnum, __u8 *buf, size_t len, unsigned char rtype, int reqtype) @@ -610,7 +604,7 @@ static int dchid_raw_request(struct hid_device *hdev, buf[0] = reportnum; return dchid_cmd(iface, rtype, REQ_GET_REPORT, &reportnum, 1, buf + 1, len - 1); case HID_REQ_SET_REPORT: - return dchid_set_report(iface, buf, len); + return dchid_cmd(iface, rtype, REQ_SET_REPORT, buf, len, NULL, 0); default: return -EIO; } From afd737e4f907d7a28e216b517168272f317ecafd Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:22:24 +0200 Subject: [PATCH 0002/1579] HID: spi-hid-apple: enumerate the fourth HID interface The J313 trackpad firmware reports five HID interfaces, but the driver clamps the count to three and only ever queries mngt, kbd and tp. The fourth one is the trackpad's haptic actuator: iface 3: name 'Actuator', country 0x0, in 16 out 64 ctrl 1024 Its report descriptor declares output report 0x53, which is the report a waveform is played with, so dropping the interface puts the taptic engine out of reach entirely. Raise the cap to four and give interface 3 a spihid_interface of its own so that it is queried, registered as a HID device and torn down with the others. The "expecting at least 3" sanity check tested against the same constant, so give it its own SPIHID_MIN_DEVICES; otherwise raising the cap would reject a machine that reports exactly three interfaces. Interface 4 is still not enumerated. What it is has not been established. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/spi-hid/spi-hid-apple-core.c | 34 ++++++++++++++++-------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c index 2ed909895391c..ba4b4b23e6fbc 100644 --- a/drivers/hid/spi-hid/spi-hid-apple-core.c +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -35,12 +35,14 @@ #define SPIHID_MAX_INPUT_REPORT_SIZE 0x800 -/* support only keyboard, trackpad and management dev for now */ -#define SPIHID_MAX_DEVICES 3 +/* management, keyboard, trackpad and the trackpad's actuator */ +#define SPIHID_MIN_DEVICES 3 +#define SPIHID_MAX_DEVICES 4 #define SPIHID_DEVICE_ID_MNGT 0x0 #define SPIHID_DEVICE_ID_KBD 0x1 #define SPIHID_DEVICE_ID_TP 0x2 +#define SPIHID_DEVICE_ID_AUX 0x3 #define SPIHID_DEVICE_ID_INFO 0xd0 #define SPIHID_READ_PACKET 0x20 @@ -86,6 +88,7 @@ struct spihid_apple { struct spihid_interface mngt; struct spihid_interface kbd; struct spihid_interface tp; + struct spihid_interface aux; wait_queue_head_t wait; struct mutex tx_lock; //< protects against concurrent SPI writes @@ -279,6 +282,8 @@ static struct spihid_apple *spihid_get_data(struct spihid_interface *idev) return container_of(idev, struct spihid_apple, kbd); case SPIHID_DEVICE_ID_TP: return container_of(idev, struct spihid_apple, tp); + case SPIHID_DEVICE_ID_AUX: + return container_of(idev, struct spihid_apple, aux); default: return NULL; } @@ -408,6 +413,8 @@ static struct spihid_interface *spihid_get_iface(struct spihid_apple *spihid, return &spihid->kbd; case SPIHID_DEVICE_ID_TP: return &spihid->tp; + case SPIHID_DEVICE_ID_AUX: + return &spihid->aux; default: return NULL; } @@ -494,18 +501,17 @@ static bool spihid_process_device_info(struct spihid_apple *spihid, u32 iface, num_devices = __le16_to_cpu(info->num_devices); - if (num_devices < SPIHID_MAX_DEVICES) { + if (num_devices < SPIHID_MIN_DEVICES) { dev_err(dev, - "Device info reports %u devices, expecting at least 3", - num_devices); + "Device info reports %u devices, expecting at least %u", + num_devices, SPIHID_MIN_DEVICES); return false; } spihid->num_devices = num_devices; if (spihid->num_devices > SPIHID_MAX_DEVICES) { - dev_info( - dev, - "limiting the number of devices to mngt, kbd and mouse"); + dev_info(dev, "limiting the number of devices to %u", + SPIHID_MAX_DEVICES); spihid->num_devices = SPIHID_MAX_DEVICES; } @@ -592,8 +598,12 @@ static bool spihid_process_iface_info(struct spihid_apple *spihid, u32 num, iface->name[name_len] = '\0'; } - dev_dbg(&spihid->spidev->dev, "Info for %s, country code: 0x%x", - iface->name, iface->country); + dev_dbg(&spihid->spidev->dev, + "iface %u: name '%s', country 0x%x, in %u out %u ctrl %u", + num, iface->name, iface->country, + iface->max_input_report_len, + iface->max_output_report_len, + iface->max_control_report_len); wake_up_interruptible(&spihid->wait); } @@ -1004,9 +1014,10 @@ int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops spihid->kbd.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); spihid->tp.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + spihid->aux.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); if (!spihid->report.buf || !spihid->kbd.hid_desc || - !spihid->tp.hid_desc) + !spihid->tp.hid_desc || !spihid->aux.hid_desc) return -ENOMEM; init_waitqueue_head(&spihid->wait); @@ -1081,6 +1092,7 @@ void spihid_apple_core_remove(struct spi_device *spi) /* destroy input devices */ + spihid_destroy_hid_device(&spihid->aux); spihid_destroy_hid_device(&spihid->tp); spihid_destroy_hid_device(&spihid->kbd); From df74aa8960d94c810390b2f6cc5dc90be0491ee7 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:22:24 +0200 Subject: [PATCH 0003/1579] HID: spi-hid-apple: address reports by report id, not interface id apple_ll_raw_request() refused any SET_REPORT whose report number differed from the interface id, and apple_ll_output_report() tagged its request with the interface id rather than the report id in buf[0]. The comment on the latter already suspected it: "second idev->id should maybe be buf[0]?". Both survived only because the report number and the interface id happened to coincide for every report the driver sent: the keyboard's LED report is report 1 on interface 1, and the SPI trackpad's multitouch enable is report 2 on interface 2. The actuator on interface 3 takes reports 0x21 and 0x53, so neither held. Its mode switch was rejected before it was sent, spi-hid-apple-of spi2.0: device:3 reportnum:33 mismatch and its waveforms went out labelled report 3, which the firmware ignored silently: every write was accepted and nothing moved. With both corrected the actuator latches host-controlled mode, feature report 0x21 reads back 01, and waveforms play on a J313. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/spi-hid/spi-hid-apple-core.c | 25 ++++++++++++------------ 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c index ba4b4b23e6fbc..0f8b13994f349 100644 --- a/drivers/hid/spi-hid/spi-hid-apple-core.c +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -362,15 +362,12 @@ static int apple_ll_raw_request(struct hid_device *hdev, return idev->reply_len; case HID_REQ_SET_REPORT: - if (buf[0] != reportnum) + if (!len || buf[0] != reportnum) return -EINVAL; - if (reportnum != idev->id) { - dev_warn(&spihid->spidev->dev, - "device:%u reportnum:" - "%hhu mismatch", - idev->id, reportnum); - return -EINVAL; - } + /* + * The report number is not the interface id: the keyboard's LED + * report is 1 on interface 1, the actuator's is 0x21 on 3. + */ return spihid_apple_request(spihid, idev->id, 0x52, reportnum, 0x00, 2, buf, len); default: return -EIO; @@ -385,11 +382,13 @@ static int apple_ll_output_report(struct hid_device *hdev, __u8 *buf, if (!spihid) return -1; - dev_dbg(&spihid->spidev->dev, - "apple_ll_output_report: device:%u len:%zu:", - idev->id, len); - // second idev->id should maybe be buf[0]? - return spihid_apple_request(spihid, idev->id, 0x51, idev->id, 0x00, 0, buf, len); + if (!len) + return -EINVAL; + + dev_dbg(&spihid->spidev->dev, "device:%u report:%u len:%zu", + idev->id, buf[0], len); + /* The request carries the report number, not the interface id. */ + return spihid_apple_request(spihid, idev->id, 0x51, buf[0], 0x00, 0, buf, len); } static struct hid_ll_driver apple_hid_ll = { From 6dbaa7a929dd30cd1ab585f6a4b9c84faa209a2c Mon Sep 17 00:00:00 2001 From: Yicong Hui Date: Fri, 21 Aug 2026 20:29:06 +0200 Subject: [PATCH 0004/1579] HID: apple-mtp-haptic: add driver for the MTP haptic touchpad actuator Add a driver to control the actuator of the MTP trackpad, for M2 and later devices. It plays a waveform with output report 0x53 and switches the actuator between device-controlled and host-controlled mode with feature report 0x21. The specific hardware commands and their fields have been reverse engineered on my own machine (Macbook Pro 2023 M2 Max) using the m1n1 hypervisor. Signed-off-by: Yicong Hui Co-developed-by: Chromatischer Signed-off-by: Chromatischer --- arch/arm64/configs/asahi.config | 1 + drivers/hid/Kconfig | 10 +++ drivers/hid/Makefile | 1 + drivers/hid/hid-apple-mtp-haptic.c | 106 +++++++++++++++++++++++++++++ include/linux/soc/apple/actuator.h | 19 ++++++ 5 files changed, 137 insertions(+) create mode 100644 drivers/hid/hid-apple-mtp-haptic.c create mode 100644 include/linux/soc/apple/actuator.h diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 4aa7f499997e7..a5b19d3e40eec 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -40,6 +40,7 @@ CONFIG_SND_SOC_TAS2764=m CONFIG_SND_SOC_TAS2770=m CONFIG_HID_APPLE=m CONFIG_HID_MAGICMOUSE=m +CONFIG_HID_APPLE_MTP_HAPTIC=m CONFIG_SERIAL_SAMSUNG=y CONFIG_SERIAL_SAMSUNG_CONSOLE=y CONFIG_HID_DOCKCHANNEL=m diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index d44e73d0c3439..3371082e799fb 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -146,6 +146,16 @@ config HID_APPLE Say Y here if you want support for keyboards of Apple iBooks, PowerBooks, MacBooks, MacBook Pros and Apple Aluminum. +config HID_APPLE_MTP_HAPTIC + tristate "Apple MTP Haptic Actuator support" + depends on HID + help + Support for the haptic actuator of the internal trackpads of Apple + Silicon laptops. + + This driver can also be built as a module. If so, it will be called + hid-apple-mtp-haptic. + config HID_APPLEIR tristate "Apple infrared receiver" depends on (USB_HID) diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index d2579e299b5e5..97f6bb600c0ae 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -29,6 +29,7 @@ obj-$(CONFIG_HID_ACCUTOUCH) += hid-accutouch.o obj-$(CONFIG_HID_ALPS) += hid-alps.o obj-$(CONFIG_HID_ACRUX) += hid-axff.o obj-$(CONFIG_HID_APPLE) += hid-apple.o +obj-$(CONFIG_HID_APPLE_MTP_HAPTIC) += hid-apple-mtp-haptic.o obj-$(CONFIG_HID_APPLEIR) += hid-appleir.o obj-$(CONFIG_HID_APPLETB_BL) += hid-appletb-bl.o obj-$(CONFIG_HID_APPLETB_KBD) += hid-appletb-kbd.o diff --git a/drivers/hid/hid-apple-mtp-haptic.c b/drivers/hid/hid-apple-mtp-haptic.c new file mode 100644 index 0000000000000..e4dca10249df6 --- /dev/null +++ b/drivers/hid/hid-apple-mtp-haptic.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include +#include + +#include "hid-ids.h" + +int apple_taptic_send(struct hid_device *haptic_hdev, u16 effect_type, u8 strength, u8 softness) +{ + u8 event_click[] = {0x53, 0x01, strength, softness, 0x03, 0x02, 0x22, 0x48, 0x49, + 0x01, 0x04, 0x60, 0x17, 0x2D, 0x03, 0x02, 0x18, 0x32, 0x1E}; + u8 event_deep_click[] = {0x53, 0x01, strength, softness, 0x01, 0x07, 0x18, 0x18, 0x4F, + 0x02, 0x02, 0x60, 0x0C, 0x25}; + u8 event_release[] = {0x53, 0x01, strength, softness}; + u8 *msg; + int msg_length; + int ret; + + switch (effect_type) { + case HID_HP_WAVEFORMPRESS & HID_USAGE: + msg = event_click; + msg_length = sizeof(event_click); + break; + case APPLE_HP_WAVEFORMDEEPCLICK & HID_USAGE: + msg = event_deep_click; + msg_length = sizeof(event_deep_click); + break; + case HID_HP_WAVEFORMRELEASE & HID_USAGE: + msg = event_release; + msg_length = sizeof(event_release); + break; + default: + return -EINVAL; + } + + ret = hid_hw_raw_request(haptic_hdev, msg[0], msg, msg_length, + HID_OUTPUT_REPORT, HID_REQ_SET_REPORT); + + if (ret < 0) + return ret; + return 0; +} +EXPORT_SYMBOL_GPL(apple_taptic_send); + +int apple_taptic_switch_modes(struct hid_device *haptic_hdev, bool enable_host_controlled) +{ + u8 msg[] = { 0x21, enable_host_controlled }; + int ret; + + ret = hid_hw_raw_request(haptic_hdev, msg[0], msg, sizeof(msg), + HID_FEATURE_REPORT, HID_REQ_SET_REPORT); + + if (ret < 0) + return ret; + return 0; +} +EXPORT_SYMBOL_GPL(apple_taptic_switch_modes); + +static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid_device_id *id) +{ + int ret; + + ret = hid_parse(haptic_hdev); + if (ret) { + hid_err(haptic_hdev, "hid parse failed\n"); + return ret; + } + + ret = hid_hw_start(haptic_hdev, 0); + if (ret) { + hid_err(haptic_hdev, "hw start failed\n"); + return ret; + } + + return 0; +} + +static bool actuator_match(struct hid_device *hdev, bool ignore_special_drivers) +{ + return (strcmp(hdev->name, "Apple MTP actuator") == 0); +} + +static void actuator_remove(struct hid_device *haptic_hdev) +{ + apple_taptic_switch_modes(haptic_hdev, false); + hid_hw_stop(haptic_hdev); +} + +static const struct hid_device_id apple_haptic_devices[] = { + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, + HID_ANY_ID), .driver_data = 0 }, + { } +}; +MODULE_DEVICE_TABLE(hid, apple_haptic_devices); + +static struct hid_driver apple_actuator_driver = { + .name = "hid-apple-mtp-haptic", + .id_table = apple_haptic_devices, + .probe = apple_actuator_probe, + .match = actuator_match, + .remove = actuator_remove +}; +module_hid_driver(apple_actuator_driver); + +MODULE_DESCRIPTION("Apple MTP Haptic Actuator driver"); +MODULE_LICENSE("GPL"); diff --git a/include/linux/soc/apple/actuator.h b/include/linux/soc/apple/actuator.h new file mode 100644 index 0000000000000..614f50d952424 --- /dev/null +++ b/include/linux/soc/apple/actuator.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef __APPLE_MTP_ACTUATOR_H__ +#define __APPLE_MTP_ACTUATOR_H__ +#include + +#define APPLE_HP_WAVEFORMDEEPCLICK 0x000e2001 + +#if IS_ENABLED(CONFIG_HID_APPLE_MTP_HAPTIC) +int apple_taptic_send(struct hid_device *hdev, u16 effect_type, u8 strength, u8 softness); +int apple_taptic_switch_modes(struct hid_device *hdev, bool currently_host_controlled); + +#else +static inline int apple_taptic_send(struct hid_device *hdev, u16 effect_type, + u8 strength, u8 softness) { return -ENODEV; } +static inline int apple_taptic_switch_modes(struct hid_device *hdev, + bool enable_host_controlled) { return -ENODEV; } +#endif +#endif /* __APPLE_MTP_ACTUATOR_H__ */ From 6022709ebe215c87cc8540de19ba5460b5d1fb32 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:22:29 +0200 Subject: [PATCH 0005/1579] HID: apple-mtp-haptic: bind the actuator on SPI machines The actuator on the SPI machines takes the same output report 0x53 and the same mode-switch feature report 0x21 as the MTP one, so the driver needs no protocol change to drive it; it simply never matched. Its id table covers BUS_HOST only, and its .match insists on the name "Apple MTP actuator", while spi-hid names the interface "Apple SPI Actuator". Nothing else claims the device either: hid-apple's id table matches every Apple SPI device, which makes hid_generic_match() stand aside, and hid-apple's probe then rejects anything that is not HID_TYPE_SPI_KEYBOARD. The actuator is left bound to no driver at all. Match BUS_SPI Apple devices as well and accept either name, via a helper in the actuator header so hid-magicmouse can use the same test. Start with HID_CONNECT_HIDRAW so the waveform parameters can be exercised from userspace: the strength and softness that suit this engine are not known yet, and the MTP defaults feel harsh on a J313. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-apple-mtp-haptic.c | 6 ++++-- include/linux/soc/apple/actuator.h | 28 ++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/drivers/hid/hid-apple-mtp-haptic.c b/drivers/hid/hid-apple-mtp-haptic.c index e4dca10249df6..0678953acbb40 100644 --- a/drivers/hid/hid-apple-mtp-haptic.c +++ b/drivers/hid/hid-apple-mtp-haptic.c @@ -66,7 +66,7 @@ static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid return ret; } - ret = hid_hw_start(haptic_hdev, 0); + ret = hid_hw_start(haptic_hdev, HID_CONNECT_HIDRAW); if (ret) { hid_err(haptic_hdev, "hw start failed\n"); return ret; @@ -77,7 +77,7 @@ static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid static bool actuator_match(struct hid_device *hdev, bool ignore_special_drivers) { - return (strcmp(hdev->name, "Apple MTP actuator") == 0); + return apple_taptic_is_actuator(hdev); } static void actuator_remove(struct hid_device *haptic_hdev) @@ -89,6 +89,8 @@ static void actuator_remove(struct hid_device *haptic_hdev) static const struct hid_device_id apple_haptic_devices[] = { { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, HID_ANY_ID), .driver_data = 0 }, + { HID_DEVICE(BUS_SPI, HID_GROUP_ANY, SPI_VENDOR_ID_APPLE, + HID_ANY_ID), .driver_data = 0 }, { } }; MODULE_DEVICE_TABLE(hid, apple_haptic_devices); diff --git a/include/linux/soc/apple/actuator.h b/include/linux/soc/apple/actuator.h index 614f50d952424..0b565d5567eb0 100644 --- a/include/linux/soc/apple/actuator.h +++ b/include/linux/soc/apple/actuator.h @@ -2,18 +2,38 @@ #ifndef __APPLE_MTP_ACTUATOR_H__ #define __APPLE_MTP_ACTUATOR_H__ + #include +#include +#include #define APPLE_HP_WAVEFORMDEEPCLICK 0x000e2001 -#if IS_ENABLED(CONFIG_HID_APPLE_MTP_HAPTIC) +/* + * The actuator is a HID device of its own, named by the transport: "Apple + * MTP actuator" on the MTP machines, "Apple SPI Actuator" on the SPI ones. + */ +static inline bool apple_taptic_is_actuator(struct hid_device *hdev) +{ + return !strcmp(hdev->name, "Apple MTP actuator") || + !strcmp(hdev->name, "Apple SPI Actuator"); +} + +#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) int apple_taptic_send(struct hid_device *hdev, u16 effect_type, u8 strength, u8 softness); -int apple_taptic_switch_modes(struct hid_device *hdev, bool currently_host_controlled); +int apple_taptic_switch_modes(struct hid_device *hdev, bool host_controlled); #else static inline int apple_taptic_send(struct hid_device *hdev, u16 effect_type, - u8 strength, u8 softness) { return -ENODEV; } + u8 strength, u8 softness) +{ + return -ENODEV; +} + static inline int apple_taptic_switch_modes(struct hid_device *hdev, - bool enable_host_controlled) { return -ENODEV; } + bool host_controlled) +{ + return -ENODEV; +} #endif #endif /* __APPLE_MTP_ACTUATOR_H__ */ From 43b2e9d9ae1349388d297a25041f7bae629cca6f Mon Sep 17 00:00:00 2001 From: Yicong Hui Date: Fri, 21 Aug 2026 21:23:49 +0200 Subject: [PATCH 0006/1579] HID: magicmouse: add support for controlling MTP touchpad haptics Integrate the MTP actuator driver with the MTP trackpad driver to expose new functionality that allows userspace to control the haptic effects of the touchpad actuator as a force feedback device, following the convention for haptic touchpad functionality as specified in [1]. Set the INPUT_PROP_PRESSUREPAD property within the MTP trackpad device indicating that this device is a haptic touchpad, and set the FF_HAPTIC capability to indicate that force feedback haptic effects can be uploaded to the trackpad [2]. Userspace can thus choose to upload force feedback effects to the touchpad, which would trigger it to switch into "Host controlled" mode, where the touchpad will no longer click unless signalled to do so by userspace. This patch uses an unconventional means of doing so, by writing a separate loadable module for the haptic actuator. This is because the haptic actuator is considered a separate device on MTP trackpads, whereas haptics for HUTRR63 (and the other kernel drivers) considers the haptic actuator and the touchpad to be the very same device. Therefore, to conform to the standardized interface in [1] this patch creates a separate driver for the separate HID device, and is designed such that commands to the actuator are sent through the touchpad. This changeset is relevant because allowing userspace control of the haptics allows for user control of: 1. Click feedback intensity 2. Sensitivity of when clicks trigger 3. Configuring "Force Touch"-like deep click functionality The actuator is a sibling HID device that the transport can register after the trackpad, so it is resolved lazily on the first effect upload. The force feedback device is created before the input device is registered and destroyed explicitly before the driver state goes away, and the whole integration is compiled out when the actuator driver is not reachable, so this driver does not depend on it. [1] https://lore.kernel.org/all/20250818-support-forcepads-v3-0-e4f9ab0add84@google.com/T/#m12f9be6691015fb9b2e834248c368e5f90 [2] https://docs.kernel.org/input/event-codes.html#input-prop-pressurepad Signed-off-by: Yicong Hui Co-developed-by: Chromatischer Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 347 ++++++++++++++++++++++++++++++++++- 1 file changed, 346 insertions(+), 1 deletion(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index f2683800bca24..802067b877c6f 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -17,7 +17,9 @@ #include #include #include +#include +#include "hid-haptic.h" #include "hid-ids.h" static bool emulate_3button = true; @@ -138,6 +140,14 @@ struct magicmouse_input_ops { int (*setup_input)(struct input_dev *input, struct hid_device *hdev); }; +struct effect_job { + struct work_struct work; + struct hid_device *taptic_hdev; + u16 effect_type; + u8 strength; + u8 softness; +}; + /** * struct magicmouse_sc - Tracks Magic Mouse-specific data. * @input: Input device through which we report events. @@ -149,6 +159,9 @@ struct magicmouse_input_ops { * @pos: multi touch position data of the last report. * @touches: Most recent data for a touch, indexed by tracking ID. * @tracking_ids: Mapping of current touch input data to @touches. + * @haptics: Haptic device state of an MTP trackpad, NULL otherwise. + * @taptic_hdev: Actuator HID device of an MTP trackpad, if resolved. + * @haptic_effects: Uploaded haptic effects of an MTP trackpad. * @hdev: Pointer to the underlying HID device. * @work: Workqueue to handle initialization retry for quirky devices. * @battery_timer: Timer for obtaining battery level information. @@ -175,6 +188,10 @@ struct magicmouse_sc { } touches[MAX_CONTACTS]; int tracking_ids[MAX_CONTACTS]; + struct hid_haptic_device *haptics; + struct hid_device *taptic_hdev; + struct effect_job *haptic_effects; + struct hid_device *hdev; struct delayed_work work; struct timer_list battery_timer; @@ -1056,6 +1073,7 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, struct magicmouse_sc *msc = hid_get_drvdata(hdev); __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); + __set_bit(INPUT_PROP_PRESSUREPAD, input->propbit); __clear_bit(BTN_0, input->keybit); __clear_bit(BTN_RIGHT, input->keybit); __clear_bit(BTN_MIDDLE, input->keybit); @@ -1063,7 +1081,7 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, __clear_bit(REL_X, input->relbit); __clear_bit(REL_Y, input->relbit); - mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK; + mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK | INPUT_MT_TOTAL_FORCE; /* finger touch area */ input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 5000, 0, 0); @@ -1164,6 +1182,57 @@ static int magicmouse_input_mapping(struct hid_device *hdev, return 0; } +static int magicmouse_init_haptics(struct magicmouse_sc *msc, + struct hid_device *hdev); + +#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +static int match_actuator(struct device *dev, const void *data) +{ + struct hid_device *hdev; + + if (dev->bus != &hid_bus_type) + return 0; + + hdev = to_hid_device(dev); + + return !strcmp(hdev->name, "Apple MTP actuator"); +} + +static struct hid_device *magicmouse_get_actuator(struct magicmouse_sc *msc) +{ + struct device *dev; + + if (msc->taptic_hdev) + return msc->taptic_hdev; + + if (!msc->hdev->dev.parent) + return NULL; + + dev = device_find_child(msc->hdev->dev.parent, NULL, match_actuator); + if (!dev) + return NULL; + + msc->taptic_hdev = to_hid_device(dev); + hid_dbg(msc->hdev, "bound to the MTP actuator\n"); + + return msc->taptic_hdev; +} + +static int magicmouse_switch_mode(struct input_dev *trackpad_idev, int mode) +{ + struct hid_device *trackpad_hdev = input_get_drvdata(trackpad_idev); + struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); + struct hid_device *actuator = magicmouse_get_actuator(msc); + + if (!actuator) + return -ENODEV; + + hid_dbg(trackpad_hdev, "switching haptics to mode %d\n", mode); + + return apple_taptic_switch_modes(actuator, mode); +} +#endif + static int magicmouse_input_configured(struct hid_device *hdev, struct hid_input *hi) @@ -1184,6 +1253,16 @@ static int magicmouse_input_configured(struct hid_device *hdev, return ret; } + /* + * This runs before the input device is registered, and the force + * feedback device has to exist by then for evdev clients to see + * FF_HAPTIC. + */ + ret = magicmouse_init_haptics(msc, hdev); + if (ret) + hid_warn(hdev, "haptics unavailable (%d), continuing without force feedback\n", + ret); + return 0; } @@ -1309,6 +1388,262 @@ static void magicmouse_battery_timer_tick(struct timer_list *t) } } +#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +static int apple_upload_effects(struct input_dev *trackpad_idev, + struct ff_effect *effect, struct ff_effect *old) +{ + struct hid_device *trackpad_hdev = input_get_drvdata(trackpad_idev); + struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); + struct hid_haptic_device *haptics = msc->haptics; + struct hid_device *actuator; + int ret; + + switch (effect->u.haptic.hid_usage) { + case (HID_HP_WAVEFORMPRESS & HID_USAGE): + case (HID_HP_WAVEFORMRELEASE & HID_USAGE): + case (APPLE_HP_WAVEFORMDEEPCLICK & HID_USAGE): + break; + default: + return -EINVAL; + } + + actuator = magicmouse_get_actuator(msc); + if (!actuator) + return -ENODEV; + + msc->haptic_effects[effect->id].taptic_hdev = actuator; + msc->haptic_effects[effect->id].effect_type = (effect->u.haptic.hid_usage) & HID_USAGE; + msc->haptic_effects[effect->id].strength = + (min(100, (effect->u.haptic.intensity)) * 255) / 100; + msc->haptic_effects[effect->id].softness = 0x90; + /* A future extension could add configurability to softness. */ + + /* The actuator may have been rebound or reset since the last upload. */ + ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_HOST); + if (ret) { + dev_err(&msc->hdev->dev, + "failed to switch to host-controlled mode: %d\n", ret); + msc->haptic_effects[effect->id].effect_type = 0; + return ret; + } + + haptics->mode = HID_HAPTIC_MODE_HOST; + + hid_dbg(trackpad_hdev, "uploaded haptic effect %d\n", effect->id); + return 0; +} + +static void haptic_playback_worker(struct work_struct *ws) +{ + struct effect_job *job = container_of(ws, struct effect_job, work); + + if (job->effect_type && + apple_taptic_send(job->taptic_hdev, job->effect_type, + job->strength, job->softness)) + hid_warn(job->taptic_hdev, "cannot play the haptic effect\n"); +} + +static int apple_taptic_playback(struct input_dev *trackpad_idev, int effect_id, int value) +{ + struct hid_device *trackpad_hdev = input_get_drvdata(trackpad_idev); + struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); + + if (value) + queue_work(msc->haptics->wq, &msc->haptic_effects[effect_id].work); + + return 0; +} + +static int apple_taptic_erase(struct input_dev *trackpad_idev, int effect_id) +{ + struct hid_device *trackpad_hdev = input_get_drvdata(trackpad_idev); + struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); + int i, ret = 0; + + msc->haptic_effects[effect_id].effect_type = 0; + + for (i = 0; i < FF_MAX_EFFECTS; i++) { + if (msc->haptic_effects[i].effect_type != 0) + return 0; + } + + /* Return to device-controlled mode if there are no effects left */ + if (msc->haptics->mode == HID_HAPTIC_MODE_HOST) { + flush_workqueue(msc->haptics->wq); + ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_DEVICE); + + if (ret) { + dev_err(&msc->hdev->dev, + "failed to switch back to device-controlled mode: %d\n", + ret); + return ret; + } + + msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + } + + return 0; +} + +static void apple_taptic_destroy(struct ff_device *ff) +{ + struct hid_haptic_device *haptic_dev = ff->private; + struct magicmouse_sc *msc = hid_get_drvdata(haptic_dev->hdev); + int ret; + + /* ff->private is devm-allocated; do not let ff-core kfree() it. */ + ff->private = NULL; + + if (msc->haptics && msc->haptics->mode == HID_HAPTIC_MODE_HOST) { + flush_workqueue(msc->haptics->wq); + ret = magicmouse_switch_mode(msc->haptics->input_dev, HID_HAPTIC_MODE_DEVICE); + if (ret) + hid_warn(msc->hdev, + "failed to switch back to device-controlled mode: %d\n", + ret); + else + msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + } + + destroy_workqueue(haptic_dev->wq); + haptic_dev->wq = NULL; + + kfree(msc->haptic_effects); + msc->haptic_effects = NULL; +} + +static int apple_taptic_init_mtp(struct magicmouse_sc *msc, + struct hid_haptic_device *haptic_dev) +{ + struct hid_device *trackpad_hdev = msc->hdev; + struct ff_device *ff; + int ret, i; + + haptic_dev->hdev = trackpad_hdev; + haptic_dev->input_dev = msc->input; + + msc->haptic_effects = kzalloc_objs(struct effect_job, FF_MAX_EFFECTS); + if (!msc->haptic_effects) { + dev_err(&trackpad_hdev->dev, "Cannot allocate haptic effects\n"); + return -ENOMEM; + } + + haptic_dev->wq = create_singlethread_workqueue("Apple trackpad haptics workqueue"); + if (!haptic_dev->wq) { + dev_err(&trackpad_hdev->dev, "Cannot allocate haptic workqueue\n"); + kfree(msc->haptic_effects); + msc->haptic_effects = NULL; + return -ENOMEM; + } + + for (i = 0; i < FF_MAX_EFFECTS; i++) + INIT_WORK(&msc->haptic_effects[i].work, haptic_playback_worker); + + /* FF_HAPTIC has to be in ffbit before the FF device is created. */ + input_set_capability(msc->input, EV_FF, FF_HAPTIC); + + ret = input_ff_create(msc->input, FF_MAX_EFFECTS); + if (ret) { + /* undo the capability set above, no FF device was created */ + __clear_bit(EV_FF, msc->input->evbit); + __clear_bit(FF_HAPTIC, msc->input->ffbit); + + kfree(msc->haptic_effects); + msc->haptic_effects = NULL; + + destroy_workqueue(haptic_dev->wq); + haptic_dev->wq = NULL; + + dev_err(&trackpad_hdev->dev, "Failed to create force-feedback device.\n"); + return ret; + } + + ff = msc->input->ff; + ff->private = haptic_dev; + ff->upload = apple_upload_effects; + ff->playback = apple_taptic_playback; + ff->erase = apple_taptic_erase; + ff->destroy = apple_taptic_destroy; + + /* + * Hold the input device until magicmouse_teardown_haptics(), which + * destroys the force feedback device while msc is still valid. + */ + input_get_device(msc->input); + + hid_info(trackpad_hdev, "initialised MTP haptics\n"); + return 0; +} + +static int magicmouse_init_haptics(struct magicmouse_sc *msc, struct hid_device *trackpad_hdev) +{ + struct hid_haptic_device *haptics; + int ret; + + if (msc->haptics) + return 0; + + /* + * Only the internal trackpads have an actuator. It is a separate HID + * device that may be registered after the trackpad, so it is resolved + * lazily when the first effect is uploaded. + */ + if (trackpad_hdev->bus != BUS_HOST || + trackpad_hdev->type != HID_TYPE_SPI_MOUSE) + return 0; + + haptics = devm_kzalloc(&trackpad_hdev->dev, sizeof(*haptics), GFP_KERNEL); + if (!haptics) + return -ENOMEM; + + haptics->hdev = trackpad_hdev; + + ret = apple_taptic_init_mtp(msc, haptics); + if (ret) { + devm_kfree(&trackpad_hdev->dev, haptics); + return ret; + } + + msc->haptics = haptics; + return 0; +} + +static void magicmouse_teardown_haptics(struct magicmouse_sc *msc) +{ + if (!msc->haptics) + return; + + /* + * Tear the force feedback device down while msc is still valid: an + * open evdev fd would otherwise run it from input_dev_release() after + * devres has freed the driver state. + */ + if (msc->haptics->input_dev) { + input_ff_destroy(msc->haptics->input_dev); + input_put_device(msc->haptics->input_dev); + msc->haptics->input_dev = NULL; + } + + msc->haptics = NULL; + msc->haptic_effects = NULL; + + if (msc->taptic_hdev) { + put_device(&msc->taptic_hdev->dev); + msc->taptic_hdev = NULL; + } +} +#else +static int magicmouse_init_haptics(struct magicmouse_sc *msc, + struct hid_device *hdev) +{ + return 0; +} + +static void magicmouse_teardown_haptics(struct magicmouse_sc *msc) +{ +} +#endif + static int magicmouse_probe(struct hid_device *hdev, const struct hid_device_id *id) { @@ -1355,6 +1690,11 @@ static int magicmouse_probe(struct hid_device *hdev, ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT); if (ret) { hid_err(hdev, "magicmouse hw start failed\n"); + /* + * hid_connect() cannot fail once hidinput_connect() has + * succeeded, so no input device is registered here. + */ + magicmouse_teardown_haptics(msc); return ret; } @@ -1457,6 +1797,8 @@ static int magicmouse_probe(struct hid_device *hdev, timer_delete_sync(&msc->battery_timer); hid_hw_stop(hdev); + magicmouse_teardown_haptics(msc); + return ret; } @@ -1472,6 +1814,9 @@ static void magicmouse_remove(struct hid_device *hdev) } hid_hw_stop(hdev); + + if (msc) + magicmouse_teardown_haptics(msc); } #ifdef CONFIG_PM From 0e7167b512b4b520b14d9a39de1d23566bd323fe Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:22:29 +0200 Subject: [PATCH 0007/1579] HID: magicmouse: allow haptics on SPI trackpads magicmouse_init_haptics() requires BUS_HOST, and match_actuator() looks for the MTP actuator by name, so the SPI trackpads were left without haptics even though their machines have an actuator that speaks the same protocol. Accept BUS_SPI as well and use the shared name test. The lookup itself needs no change: the actuator is a sibling HID device under the same SPI parent, which is what magicmouse_get_actuator() already searches. This does not change how a click feels by default. The actuator stays in device-controlled mode, where its firmware provides the click feedback, and only moves to host-controlled mode when userspace uploads an effect. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 802067b877c6f..660ed3982b8af 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -1195,7 +1195,7 @@ static int match_actuator(struct device *dev, const void *data) hdev = to_hid_device(dev); - return !strcmp(hdev->name, "Apple MTP actuator"); + return apple_taptic_is_actuator(hdev); } static struct hid_device *magicmouse_get_actuator(struct magicmouse_sc *msc) @@ -1584,11 +1584,12 @@ static int magicmouse_init_haptics(struct magicmouse_sc *msc, struct hid_device return 0; /* - * Only the internal trackpads have an actuator. It is a separate HID - * device that may be registered after the trackpad, so it is resolved - * lazily when the first effect is uploaded. + * Only the internal trackpads have an actuator: the MTP ones on + * BUS_HOST and the SPI ones, whose firmware exposes it as a separate + * "Actuator" interface. It may be registered after the trackpad, so + * it is resolved lazily on the first effect upload. */ - if (trackpad_hdev->bus != BUS_HOST || + if ((trackpad_hdev->bus != BUS_HOST && trackpad_hdev->bus != BUS_SPI) || trackpad_hdev->type != HID_TYPE_SPI_MOUSE) return 0; From d52740d6db6e805d8d8690ebb64e20f51009804c Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:42:42 +0200 Subject: [PATCH 0008/1579] HID: magicmouse: make the haptic waveform shape configurable The waveform sent to the actuator carries a strength and a softness. Both were fixed: softness at 0x90, and strength at the full 255 scaled by the effect's intensity. Those suit the MTP actuator, but on a J313 the same numbers produce a hard knock rather than a click. Judged by feel there, a press wants roughly strength 0x28 at softness 0xf0, far weaker than the MTP default, while a deep click wants full strength 0xff at a softness of 0xd0. Add four parameters for the two waveforms, each defaulting per transport so the MTP path keeps the values its firmware was tuned with. Strength now names what 100% intensity maps to, so an effect asking for half intensity still gets half of it. They are writable at runtime and take effect on the next effect upload, which is what makes tuning by feel practical: the values that suit a given actuator cannot be derived, only tried. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 62 ++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 660ed3982b8af..902f5029f43fa 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -53,6 +53,27 @@ static bool report_undeciphered; module_param(report_undeciphered, bool, 0644); MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state field using a MSC_RAW event"); +/* + * Waveform shaping, writable at runtime and applied on the next effect + * upload. Zero means "use the per-transport default": the MTP machines keep + * the values their firmware was tuned with, the SPI actuator in a J313 needs + * a much softer, weaker waveform. The SPI numbers were judged by feel there. + */ +static u8 haptic_softness; +module_param(haptic_softness, byte, 0644); +MODULE_PARM_DESC(haptic_softness, "softness of the press and release waveforms, 1-255 (0: 0xf0 on SPI, 0x90 on MTP)"); + +static u8 haptic_deep_softness; +module_param(haptic_deep_softness, byte, 0644); +MODULE_PARM_DESC(haptic_deep_softness, "softness of the deep-click waveform, 1-255 (0: 0xd0 on SPI, 0x90 on MTP)"); + +static u8 haptic_strength; +module_param(haptic_strength, byte, 0644); +MODULE_PARM_DESC(haptic_strength, "strength at 100% intensity for press and release, 1-255 (0: 0x28 on SPI, 0xff on MTP)"); + +static u8 haptic_deep_strength; +module_param(haptic_deep_strength, byte, 0644); +MODULE_PARM_DESC(haptic_deep_strength, "strength at 100% intensity for a deep click, 1-255 (0: 0xff on SPI, 0xff on MTP)"); #define TRACKPAD2_2021_BT_VERSION 0x110 #define TRACKPAD_2024_BT_VERSION 0x314 @@ -1389,6 +1410,40 @@ static void magicmouse_battery_timer_tick(struct timer_list *t) } #if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +static bool magicmouse_effect_is_deep(const struct ff_effect *effect) +{ + return effect->u.haptic.hid_usage == + (APPLE_HP_WAVEFORMDEEPCLICK & HID_USAGE); +} + +static u8 magicmouse_haptic_softness(struct hid_device *hdev, + const struct ff_effect *effect) +{ + bool deep = magicmouse_effect_is_deep(effect); + u8 configured = deep ? haptic_deep_softness : haptic_softness; + + if (configured) + return configured; + if (hdev->bus != BUS_SPI) + return 0x90; + + return deep ? 0xd0 : 0xf0; +} + +static u8 magicmouse_haptic_strength(struct hid_device *hdev, + const struct ff_effect *effect) +{ + bool deep = magicmouse_effect_is_deep(effect); + u8 configured = deep ? haptic_deep_strength : haptic_strength; + + if (configured) + return configured; + if (hdev->bus != BUS_SPI) + return 0xff; + + return deep ? 0xff : 0x28; +} + static int apple_upload_effects(struct input_dev *trackpad_idev, struct ff_effect *effect, struct ff_effect *old) { @@ -1414,9 +1469,10 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, msc->haptic_effects[effect->id].taptic_hdev = actuator; msc->haptic_effects[effect->id].effect_type = (effect->u.haptic.hid_usage) & HID_USAGE; msc->haptic_effects[effect->id].strength = - (min(100, (effect->u.haptic.intensity)) * 255) / 100; - msc->haptic_effects[effect->id].softness = 0x90; - /* A future extension could add configurability to softness. */ + (min(100, (effect->u.haptic.intensity)) * + magicmouse_haptic_strength(trackpad_hdev, effect)) / 100; + msc->haptic_effects[effect->id].softness = + magicmouse_haptic_softness(trackpad_hdev, effect); /* The actuator may have been rebound or reset since the last upload. */ ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_HOST); From 66c49f6b06c72670e865fbab0b6aec3e0d8f9bbd Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 22:13:13 +0200 Subject: [PATCH 0009/1579] HID: magicmouse: drive click feedback from the host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The actuator's firmware provides the feedback for a physical press, and how that press feels cannot be changed while it does. Playing the waveform from the host instead makes it adjustable, but it cannot be done by reacting to the button the trackpad reports: in host-controlled mode the firmware stops detecting clicks altogether. It keeps reporting fingers and never sets the button again, because deciding that a press happened belongs to whoever owns the actuator. Hooking the button bit therefore works exactly once — for the press already in flight when the mode changed — and then the pad has no click feedback and no button at all. So derive the click here. Sum the pressure of the fingers in each frame, call a press when it rises past haptic_press_threshold and a release when it falls below haptic_release_threshold, report BTN_MOUSE from that, and play the waveform from the haptics workqueue, which is where it can sleep. The thresholds default to 120 and 75, measured on a J313 against its own firmware: it asserts the button between 130 and 243 and releases between 26 and 73. The gap between the two is what stops a click chattering. This is off by default. Turning it off returns the actuator to the firmware on the next frame, and any failure while playing does the same, so a pad that cannot play a waveform falls back to working feedback rather than none. What the parameters do on an SPI actuator is narrower than the report layout suggests. The fifteen waveform bytes are ignored: zeroing them feels identical. Strength is close to on or off, with 0x00 silent and the values above it differing little. Softness is what shapes the pulse; swept from 0x10 to 0xff at a fixed strength it runs from a faint tick to a hard knock. A click that wants more weight than softness gives is made of more pulses, which is what haptic_press_pulses is for. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 231 +++++++++++++++++++++++++++++++++-- 1 file changed, 222 insertions(+), 9 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 902f5029f43fa..e8dba12a0ea0b 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -11,6 +11,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -74,6 +75,42 @@ MODULE_PARM_DESC(haptic_strength, "strength at 100% intensity for press and rele static u8 haptic_deep_strength; module_param(haptic_deep_strength, byte, 0644); MODULE_PARM_DESC(haptic_deep_strength, "strength at 100% intensity for a deep click, 1-255 (0: 0xff on SPI, 0xff on MTP)"); + +/* + * Host-driven click feedback: hold the actuator in host-controlled mode and + * play the pulses from here, which is the only way to change how a press + * feels. With this off, the firmware provides the feedback itself. + */ +static bool haptic_press; +module_param(haptic_press, bool, 0644); +MODULE_PARM_DESC(haptic_press, "play click feedback from the host on physical press (default off: the actuator's firmware does it)"); + +static u8 haptic_press_pulses = 1; +module_param(haptic_press_pulses, byte, 0644); +MODULE_PARM_DESC(haptic_press_pulses, "pulses played on press (default 1)"); + +static u8 haptic_release_pulses = 1; +module_param(haptic_release_pulses, byte, 0644); +MODULE_PARM_DESC(haptic_release_pulses, "release waveforms played on release, minimum 1 (default 1)"); + +/* + * Click thresholds, in the units the trackpad reports per finger summed over + * the fingers in a frame. On a J313 the firmware asserts the button between + * 130 and 243 and releases between 26 and 73; these sit just inside that, with + * enough hysteresis that a click cannot chatter. + */ +static u16 haptic_press_threshold = 120; +module_param(haptic_press_threshold, ushort, 0644); +MODULE_PARM_DESC(haptic_press_threshold, "summed finger pressure at which a click starts (default 120)"); + +static u16 haptic_release_threshold = 75; +module_param(haptic_release_threshold, ushort, 0644); +MODULE_PARM_DESC(haptic_release_threshold, "summed finger pressure at which a click ends (default 75)"); + +static u8 haptic_pulse_gap_ms = 25; +module_param(haptic_pulse_gap_ms, byte, 0644); +MODULE_PARM_DESC(haptic_pulse_gap_ms, "milliseconds between pulses when more than one (default 25)"); + #define TRACKPAD2_2021_BT_VERSION 0x110 #define TRACKPAD_2024_BT_VERSION 0x314 @@ -212,6 +249,9 @@ struct magicmouse_sc { struct hid_haptic_device *haptics; struct hid_device *taptic_hdev; struct effect_job *haptic_effects; + struct work_struct haptic_press_work; + struct work_struct haptic_release_work; + bool haptic_button_down; struct hid_device *hdev; struct delayed_work work; @@ -778,6 +818,9 @@ struct tp_mouse_report { u8 padding[4]; }; +static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, + bool firmware_button); + static void report_finger_data(struct input_dev *input, int slot, const struct input_mt_pos *pos, const struct tp_finger *f) @@ -807,6 +850,7 @@ static int magicmouse_raw_event_mtp(struct hid_device *hdev, struct input_dev *input = msc->input; struct tp_header *tp_hdr; struct tp_finger *f; + u32 pressure = 0; int i, n; u32 npoints; const size_t hdr_sz = sizeof(struct tp_header); @@ -852,10 +896,13 @@ static int magicmouse_raw_event_mtp(struct hid_device *hdev, int idx = map_contacs[i]; f = (struct tp_finger *)(data + hdr_sz + idx * touch_sz); report_finger_data(input, msc->tracking_ids[i], &msc->pos[i], f); + pressure += le16_to_int(f->pressure); } input_mt_sync_frame(input); - input_report_key(input, BTN_MOUSE, tp_hdr->buttons & 1); + input_report_key(input, BTN_MOUSE, + magicmouse_haptic_click(msc, pressure, + tp_hdr->buttons & 1)); input_sync(input); return 1; @@ -1252,6 +1299,168 @@ static int magicmouse_switch_mode(struct input_dev *trackpad_idev, int mode) return apple_taptic_switch_modes(actuator, mode); } + +static u8 magicmouse_haptic_softness(struct hid_device *hdev, bool deep); +static u8 magicmouse_haptic_strength(struct hid_device *hdev, bool deep); +static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc); + +/* + * Play the waveform @pulses times, @haptic_pulse_gap_ms apart. A failure here + * would leave host-controlled mode without any click feedback, so hand the + * actuator back to the firmware and turn the feature off instead. + */ +static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, + u8 pulses) +{ + struct hid_device *actuator = magicmouse_get_actuator(msc); + u8 strength, softness; + unsigned int i; + int ret; + + if (!pulses) + return; + + if (!actuator) { + hid_warn(msc->hdev, + "no actuator for host click feedback; using the firmware's\n"); + haptic_press = false; + return; + } + + if (msc->haptics->mode != HID_HAPTIC_MODE_HOST) { + ret = apple_taptic_switch_modes(actuator, HID_HAPTIC_MODE_HOST); + if (ret) { + hid_warn(msc->hdev, + "cannot take host-controlled mode (%d); using the firmware's\n", + ret); + haptic_press = false; + return; + } + msc->haptics->mode = HID_HAPTIC_MODE_HOST; + hid_info(msc->hdev, "host-driven click feedback active\n"); + } + + strength = magicmouse_haptic_strength(msc->hdev, false); + softness = magicmouse_haptic_softness(msc->hdev, false); + + for (i = 0; i < pulses; i++) { + ret = apple_taptic_send(actuator, usage, strength, softness); + if (ret) { + hid_warn(msc->hdev, + "click waveform failed (%d); returning click feedback to the firmware\n", + ret); + haptic_press = false; + magicmouse_haptic_release_mode(msc); + return; + } + hid_dbg(msc->hdev, "played %s pulse %u/%u at strength 0x%02x\n", + usage == (HID_HP_WAVEFORMRELEASE & HID_USAGE) ? + "release" : "press", + i + 1, pulses, strength); + if (i + 1 < pulses) + msleep(haptic_pulse_gap_ms); + } +} + +/* + * Hand click feedback back to the actuator's firmware, unless userspace still + * has effects uploaded and so still wants host-controlled mode. + */ +static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc) +{ + struct hid_device *actuator; + unsigned int i; + + if (msc->haptics->mode != HID_HAPTIC_MODE_HOST) + return; + + if (msc->haptic_effects) { + for (i = 0; i < FF_MAX_EFFECTS; i++) + if (msc->haptic_effects[i].effect_type) + return; + } + + actuator = magicmouse_get_actuator(msc); + if (actuator && !apple_taptic_switch_modes(actuator, + HID_HAPTIC_MODE_DEVICE)) + msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; +} + +static void magicmouse_press_worker(struct work_struct *ws) +{ + struct magicmouse_sc *msc = + container_of(ws, struct magicmouse_sc, haptic_press_work); + + if (!haptic_press) { + magicmouse_haptic_release_mode(msc); + return; + } + + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMPRESS & HID_USAGE, + haptic_press_pulses); +} + +static void magicmouse_release_worker(struct work_struct *ws) +{ + struct magicmouse_sc *msc = + container_of(ws, struct magicmouse_sc, haptic_release_work); + + if (!haptic_press) { + magicmouse_haptic_release_mode(msc); + return; + } + + /* + * The bare header tells the firmware that the click gesture is over, so + * send it even when no extra pulses were asked for. + */ + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMRELEASE & HID_USAGE, + max_t(u8, haptic_release_pulses, 1)); +} + +/* + * Decide whether the pad is clicked from the summed finger pressure, and + * queue the feedback for it: in host-controlled mode the firmware no longer + * sets the button itself. The wide gap between the press and release + * thresholds keeps a single click from chattering. + * + * Returns the button state to report. Playing the waveform sleeps, so it is + * left to the workqueue. + */ +static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, + bool firmware_button) +{ + if (!msc->haptics || !msc->haptics->wq) + return firmware_button; + + if (!haptic_press) { + /* Hand the actuator back and report its button again. */ + if (msc->haptics->mode == HID_HAPTIC_MODE_HOST) { + msc->haptic_button_down = false; + queue_work(msc->haptics->wq, &msc->haptic_release_work); + } + return firmware_button; + } + + if (!msc->haptic_button_down && pressure >= haptic_press_threshold) { + msc->haptic_button_down = true; + hid_dbg(msc->hdev, "click at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_press_work); + } else if (msc->haptic_button_down && + pressure <= haptic_release_threshold) { + msc->haptic_button_down = false; + hid_dbg(msc->hdev, "release at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_release_work); + } + + return msc->haptic_button_down; +} +#else +static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, + bool firmware_button) +{ + return firmware_button; +} #endif static int magicmouse_input_configured(struct hid_device *hdev, @@ -1416,10 +1625,8 @@ static bool magicmouse_effect_is_deep(const struct ff_effect *effect) (APPLE_HP_WAVEFORMDEEPCLICK & HID_USAGE); } -static u8 magicmouse_haptic_softness(struct hid_device *hdev, - const struct ff_effect *effect) +static u8 magicmouse_haptic_softness(struct hid_device *hdev, bool deep) { - bool deep = magicmouse_effect_is_deep(effect); u8 configured = deep ? haptic_deep_softness : haptic_softness; if (configured) @@ -1430,10 +1637,8 @@ static u8 magicmouse_haptic_softness(struct hid_device *hdev, return deep ? 0xd0 : 0xf0; } -static u8 magicmouse_haptic_strength(struct hid_device *hdev, - const struct ff_effect *effect) +static u8 magicmouse_haptic_strength(struct hid_device *hdev, bool deep) { - bool deep = magicmouse_effect_is_deep(effect); u8 configured = deep ? haptic_deep_strength : haptic_strength; if (configured) @@ -1470,9 +1675,11 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, msc->haptic_effects[effect->id].effect_type = (effect->u.haptic.hid_usage) & HID_USAGE; msc->haptic_effects[effect->id].strength = (min(100, (effect->u.haptic.intensity)) * - magicmouse_haptic_strength(trackpad_hdev, effect)) / 100; + magicmouse_haptic_strength(trackpad_hdev, + magicmouse_effect_is_deep(effect))) / 100; msc->haptic_effects[effect->id].softness = - magicmouse_haptic_softness(trackpad_hdev, effect); + magicmouse_haptic_softness(trackpad_hdev, + magicmouse_effect_is_deep(effect)); /* The actuator may have been rebound or reset since the last upload. */ ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_HOST); @@ -1595,6 +1802,9 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, for (i = 0; i < FF_MAX_EFFECTS; i++) INIT_WORK(&msc->haptic_effects[i].work, haptic_playback_worker); + INIT_WORK(&msc->haptic_press_work, magicmouse_press_worker); + INIT_WORK(&msc->haptic_release_work, magicmouse_release_worker); + /* FF_HAPTIC has to be in ffbit before the FF device is created. */ input_set_capability(msc->input, EV_FF, FF_HAPTIC); @@ -1670,6 +1880,9 @@ static void magicmouse_teardown_haptics(struct magicmouse_sc *msc) if (!msc->haptics) return; + cancel_work_sync(&msc->haptic_press_work); + cancel_work_sync(&msc->haptic_release_work); + /* * Tear the force feedback device down while msc is still valid: an * open evdev fd would otherwise run it from input_dev_release() after From 57ea3abab4908b0de38fa758eb02ed2dfb8fc222 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 22:25:12 +0200 Subject: [PATCH 0010/1579] HID: magicmouse: add a deep click The pad reports finger pressure well past the point where a click is asserted, so a second, firmer press inside one click can be told apart from an ordinary one. That is what Force Touch does, and the hardware already carries the signal. Fire once per click at haptic_deep_threshold, with its own pulse count, gap and waveform shape so it can be made to feel distinct from the first click, and clear the flag only on release so holding at depth does not repeat it. Linux has no key for a force click, so haptic_deep_keycode names one and nothing is reported unless it is set; the capability has to be advertised at probe, so that parameter is read-only. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 75 ++++++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 7 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index e8dba12a0ea0b..f8ba0d4100050 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -107,6 +107,32 @@ static u16 haptic_release_threshold = 75; module_param(haptic_release_threshold, ushort, 0644); MODULE_PARM_DESC(haptic_release_threshold, "summed finger pressure at which a click ends (default 75)"); +/* + * A deep click is a second, firmer press within one click, the way Force Touch + * behaves. On a J313 an ordinary click peaks around 250 and a firm press + * reaches past 1000, so the threshold sits well clear of the first one. + */ +static u16 haptic_deep_threshold = 500; +module_param(haptic_deep_threshold, ushort, 0644); +MODULE_PARM_DESC(haptic_deep_threshold, "summed finger pressure for a deep click, 0 disables (default 500)"); + +static u8 haptic_deep_pulses = 1; +module_param(haptic_deep_pulses, byte, 0644); +MODULE_PARM_DESC(haptic_deep_pulses, "pulses played for a deep click (default 1)"); + +static u8 haptic_deep_gap_ms = 15; +module_param(haptic_deep_gap_ms, byte, 0644); +MODULE_PARM_DESC(haptic_deep_gap_ms, "milliseconds between deep-click pulses, 0 to use haptic_pulse_gap_ms (default 15)"); + +/* + * Linux has no key for a force click, so which one to send is the user's + * choice; nothing is sent unless one is named. It is read at probe because the + * capability has to be advertised when the input device is set up. + */ +static u16 haptic_deep_keycode; +module_param(haptic_deep_keycode, ushort, 0444); +MODULE_PARM_DESC(haptic_deep_keycode, "key code reported on a deep click, 0 for none (default 0)"); + static u8 haptic_pulse_gap_ms = 25; module_param(haptic_pulse_gap_ms, byte, 0644); MODULE_PARM_DESC(haptic_pulse_gap_ms, "milliseconds between pulses when more than one (default 25)"); @@ -251,7 +277,9 @@ struct magicmouse_sc { struct effect_job *haptic_effects; struct work_struct haptic_press_work; struct work_struct haptic_release_work; + struct work_struct haptic_deep_work; bool haptic_button_down; + bool haptic_deep_fired; struct hid_device *hdev; struct delayed_work work; @@ -1310,10 +1338,10 @@ static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc); * actuator back to the firmware and turn the feature off instead. */ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, - u8 pulses) + bool deep, u8 pulses) { struct hid_device *actuator = magicmouse_get_actuator(msc); - u8 strength, softness; + u8 strength, softness, gap; unsigned int i; int ret; @@ -1340,8 +1368,10 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, hid_info(msc->hdev, "host-driven click feedback active\n"); } - strength = magicmouse_haptic_strength(msc->hdev, false); - softness = magicmouse_haptic_softness(msc->hdev, false); + strength = magicmouse_haptic_strength(msc->hdev, deep); + softness = magicmouse_haptic_softness(msc->hdev, deep); + gap = (deep && haptic_deep_gap_ms) ? haptic_deep_gap_ms + : haptic_pulse_gap_ms; for (i = 0; i < pulses; i++) { ret = apple_taptic_send(actuator, usage, strength, softness); @@ -1358,7 +1388,7 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, "release" : "press", i + 1, pulses, strength); if (i + 1 < pulses) - msleep(haptic_pulse_gap_ms); + msleep(gap); } } @@ -1396,10 +1426,22 @@ static void magicmouse_press_worker(struct work_struct *ws) return; } - magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMPRESS & HID_USAGE, + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMPRESS & HID_USAGE, false, haptic_press_pulses); } +static void magicmouse_deep_worker(struct work_struct *ws) +{ + struct magicmouse_sc *msc = + container_of(ws, struct magicmouse_sc, haptic_deep_work); + + if (!haptic_press) + return; + + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMPRESS & HID_USAGE, true, + haptic_deep_pulses); +} + static void magicmouse_release_worker(struct work_struct *ws) { struct magicmouse_sc *msc = @@ -1414,7 +1456,7 @@ static void magicmouse_release_worker(struct work_struct *ws) * The bare header tells the firmware that the click gesture is over, so * send it even when no extra pulses were asked for. */ - magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMRELEASE & HID_USAGE, + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMRELEASE & HID_USAGE, false, max_t(u8, haptic_release_pulses, 1)); } @@ -1444,13 +1486,27 @@ static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, if (!msc->haptic_button_down && pressure >= haptic_press_threshold) { msc->haptic_button_down = true; + msc->haptic_deep_fired = false; hid_dbg(msc->hdev, "click at pressure %u\n", pressure); queue_work(msc->haptics->wq, &msc->haptic_press_work); } else if (msc->haptic_button_down && pressure <= haptic_release_threshold) { msc->haptic_button_down = false; + msc->haptic_deep_fired = false; hid_dbg(msc->hdev, "release at pressure %u\n", pressure); queue_work(msc->haptics->wq, &msc->haptic_release_work); + } else if (msc->haptic_button_down && !msc->haptic_deep_fired && + haptic_deep_threshold && + pressure >= haptic_deep_threshold) { + /* Pressing harder without lifting: once per click, not while held. */ + msc->haptic_deep_fired = true; + hid_dbg(msc->hdev, "deep click at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_deep_work); + + if (haptic_deep_keycode) { + input_report_key(msc->input, haptic_deep_keycode, 1); + input_report_key(msc->input, haptic_deep_keycode, 0); + } } return msc->haptic_button_down; @@ -1804,6 +1860,10 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, INIT_WORK(&msc->haptic_press_work, magicmouse_press_worker); INIT_WORK(&msc->haptic_release_work, magicmouse_release_worker); + INIT_WORK(&msc->haptic_deep_work, magicmouse_deep_worker); + + if (haptic_deep_keycode) + input_set_capability(msc->input, EV_KEY, haptic_deep_keycode); /* FF_HAPTIC has to be in ffbit before the FF device is created. */ input_set_capability(msc->input, EV_FF, FF_HAPTIC); @@ -1882,6 +1942,7 @@ static void magicmouse_teardown_haptics(struct magicmouse_sc *msc) cancel_work_sync(&msc->haptic_press_work); cancel_work_sync(&msc->haptic_release_work); + cancel_work_sync(&msc->haptic_deep_work); /* * Tear the force feedback device down while msc is still valid: an From e55ac836658486cdc9b31afcb94f269b71c85b28 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Thu, 17 Sep 2026 09:39:53 +0200 Subject: [PATCH 0011/1579] HID: magicmouse: derive the click while the host holds the actuator The firmware reports the button only while it owns the actuator. The derived click covered haptic_press, but a force-feedback effect uploaded by userspace also takes host-controlled mode, and the mode cannot be released while an effect is loaded. In that state the driver reported the firmware's button, which never changes again, so the pad stopped clicking until every effect was erased. Derive the button whenever the actuator is in host-controlled mode, whatever put it there, and keep asking for it back when the waveform is not ours to play. Found by a review bot on the pull request. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index f8ba0d4100050..32cb3072f06da 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -1472,18 +1472,31 @@ static void magicmouse_release_worker(struct work_struct *ws) static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, bool firmware_button) { + bool host_mode; + if (!msc->haptics || !msc->haptics->wq) return firmware_button; - if (!haptic_press) { - /* Hand the actuator back and report its button again. */ - if (msc->haptics->mode == HID_HAPTIC_MODE_HOST) { - msc->haptic_button_down = false; - queue_work(msc->haptics->wq, &msc->haptic_release_work); - } + host_mode = msc->haptics->mode == HID_HAPTIC_MODE_HOST; + + if (!haptic_press && !host_mode) { + /* The firmware owns the actuator and reports the button. */ + msc->haptic_button_down = false; + msc->haptic_deep_fired = false; return firmware_button; } + /* + * Host-controlled mode without host-driven feedback means userspace + * uploaded an effect and owns the waveform. Ask for the actuator back, + * but keep deriving the button until it is: the firmware reports no + * button while it does not hold the actuator, and the mode cannot be + * released while an effect is loaded, so trusting it here would leave + * the pad unable to click until every effect is erased. + */ + if (!haptic_press) + queue_work(msc->haptics->wq, &msc->haptic_release_work); + if (!msc->haptic_button_down && pressure >= haptic_press_threshold) { msc->haptic_button_down = true; msc->haptic_deep_fired = false; From c18e5189bc4f76cac01d0cac559889e89f8c8a6b Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Thu, 17 Sep 2026 09:39:57 +0200 Subject: [PATCH 0012/1579] HID: magicmouse: forget the actuator's mode across system sleep The actuator leaves system sleep in device-controlled mode. The transport resumes the keyboard and the trackpad, not the actuator, and nothing tells the driver the mode changed, so msc->haptics->mode still reads host-controlled. The next press then skips the switch and plays a pulse while the firmware plays its own, which is felt as two clicks per press. On a J313 across an s2idle cycle the driver says nothing between 'host-driven click feedback active' before the suspend and the first press after it: it never re-took the mode because it believed it still held it. Drop the cached mode in reset_resume, which the transport does call for the trackpad, and drop the click state with it. The next press takes host-controlled mode again if it needs it. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 32cb3072f06da..df76efb9c5917 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -1524,12 +1524,34 @@ static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, return msc->haptic_button_down; } + +/* + * The actuator leaves system sleep in device-controlled mode whatever it was + * in before: the transport resumes the keyboard and the trackpad, not the + * actuator, and nothing reports the change. A cached host-controlled mode then + * makes the next press skip the switch and play a pulse while the firmware is + * playing its own, which is felt as two clicks. Forget it instead; the next + * press takes the mode again if it needs it. + */ +static void magicmouse_haptic_forget_mode(struct magicmouse_sc *msc) +{ + if (!msc->haptics) + return; + + msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + msc->haptic_button_down = false; + msc->haptic_deep_fired = false; +} #else static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, bool firmware_button) { return firmware_button; } + +static void magicmouse_haptic_forget_mode(struct magicmouse_sc *msc) +{ +} #endif static int magicmouse_input_configured(struct hid_device *hdev, @@ -2175,6 +2197,9 @@ static int magicmouse_reset_resume(struct hid_device *hdev) if (msc && (hdev->type == HID_TYPE_USBMOUSE || hdev->bus == BUS_SPI)) schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + if (msc) + magicmouse_haptic_forget_mode(msc); + return 0; } #endif From 61cb1d810bfd98978abce0345936562a89e35e32 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Thu, 17 Sep 2026 10:06:10 +0200 Subject: [PATCH 0013/1579] HID: apple-mtp-haptic: put the actuator back in device-controlled mode The actuator keeps its mode across a warm reboot. Every user of it starts out believing it is device-controlled, so one left in host-controlled mode by a previous boot is silent: the firmware plays nothing because it no longer owns the actuator, and the host that did is gone. The trackpad then has no click feedback at all, with nothing in the log to say why. Only a cold boot clears it. Assert device-controlled mode at probe instead of inheriting whatever the last boot left. The switch is not fatal if it fails, since that leaves the actuator exactly where it already was. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-apple-mtp-haptic.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/hid/hid-apple-mtp-haptic.c b/drivers/hid/hid-apple-mtp-haptic.c index 0678953acbb40..42ab019f7a27a 100644 --- a/drivers/hid/hid-apple-mtp-haptic.c +++ b/drivers/hid/hid-apple-mtp-haptic.c @@ -72,6 +72,21 @@ static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid return ret; } + /* + * The actuator keeps its mode across a warm reboot, and every user of + * it starts out believing it is device-controlled. One left in + * host-controlled mode by a previous boot is silent: its firmware plays + * nothing, and the host that was driving it is gone. Assert the state + * the drivers assume rather than inheriting the last one. A failure + * here is not fatal; it only means the actuator keeps whatever mode it + * had, which is what would have happened anyway. + */ + ret = apple_taptic_switch_modes(haptic_hdev, false); + if (ret) + hid_warn(haptic_hdev, + "cannot return the actuator to device-controlled mode (%d)\n", + ret); + return 0; } From 41af75d3472d2978fe9ec0df5c6dbe6f9210a4c9 Mon Sep 17 00:00:00 2001 From: Justin Pfister <119864532+JJRPF@users.noreply.github.com> Date: Wed, 16 Sep 2026 19:43:09 -0400 Subject: [PATCH 0014/1579] Bluetooth: hci_bcm4377: Add hardware reset recovery and post-init restoration When Bluetooth is stopped during heavy traffic (such as active LE scanning or discovery), transfer ring destruction commands to the Broadcom PCIe controller can time out and fail. Subsequent opens fail because rings cannot be created, leaving the device wedged. Implement hardware reset recovery using PCI Function-Level Reset (FLR) to re-initialize the PCIe controller and restart the firmware. Introduce a synchronous hdev->post_init hook to reload calibration data, regulatory PTB tables, and restore the device BD_ADDR after hardware reset, ensuring the controller is fully restored to operational state. Assisted-by: Antigravity:gemini-3.8-flash Signed-off-by: Justin Pfister <119864532+JJRPF@users.noreply.github.com> --- drivers/bluetooth/hci_bcm4377.c | 259 ++++++++++++++++++++++++++++---- 1 file changed, 229 insertions(+), 30 deletions(-) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 0eb137a99add7..5fe775766e781 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -11,11 +11,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include @@ -583,6 +585,10 @@ struct bcm4377_data { char vendor[BCM4377_OTP_MAX_PARAM_LEN]; const char *board_type; + bool needs_reset; + bool cal_needed; + bdaddr_t bdaddr; + struct completion event; struct bcm4377_context *ctx; @@ -1304,9 +1310,86 @@ static int bcm4378_send_ptb(struct bcm4377_data *bcm4377, return 0; } -static int bcm4377_hci_open(struct hci_dev *hdev) +static void bcm4377_disable_aspm(struct bcm4377_data *bcm4377); +static int bcm4377_init_cfg(struct bcm4377_data *bcm4377); +static int bcm4377_boot(struct bcm4377_data *bcm4377); +static int bcm4377_setup_rti(struct bcm4377_data *bcm4377); + +static int bcm4377_pci_reset(struct bcm4377_data *bcm4377) +{ + int ret; + int irq; + + dev_info(&bcm4377->pdev->dev, "performing hardware reset recovery\n"); + + irq = pci_irq_vector(bcm4377->pdev, 0); + if (irq > 0) + disable_irq(irq); + + if (bcm4377->hw->disable_aspm) + bcm4377_disable_aspm(bcm4377); + + ret = pci_reset_function(bcm4377->pdev); + if (ret) + dev_warn(&bcm4377->pdev->dev, + "function level reset failed with %d; trying to continue anyway\n", + ret); + + msleep(100); + + pci_set_master(bcm4377->pdev); + + ret = bcm4377_init_cfg(bcm4377); + if (ret) { + dev_err(&bcm4377->pdev->dev, + "failed to reinitialize PCI config: %d\n", ret); + goto out_irq; + } + + bcm4377->bootstage = 0; + bcm4377->rti_status = 0; + reinit_completion(&bcm4377->event); + + memset(bcm4377->ring_state, 0, sizeof(*bcm4377->ring_state)); + bcm4377->control_h2d_ring.generation = 0; + bcm4377->control_ack_ring.enabled = false; + bitmap_zero(bcm4377->control_h2d_ring.msgids, + bcm4377->control_h2d_ring.n_entries); + if (bcm4377->control_h2d_ring.events) + memset(bcm4377->control_h2d_ring.events, 0, + bcm4377->control_h2d_ring.n_entries * + sizeof(*bcm4377->control_h2d_ring.events)); + + if (irq > 0) + enable_irq(irq); + + ret = bcm4377_boot(bcm4377); + if (ret) { + dev_err(&bcm4377->pdev->dev, "failed to reboot firmware: %d\n", + ret); + return ret; + } + + ret = bcm4377_setup_rti(bcm4377); + if (ret) { + dev_err(&bcm4377->pdev->dev, "failed to re-setup RTI: %d\n", + ret); + return ret; + } + + bcm4377->needs_reset = false; + dev_info(&bcm4377->pdev->dev, + "hardware reset recovery completed successfully\n"); + return 0; + +out_irq: + if (irq > 0) + enable_irq(irq); + return ret; +} + +static int bcm4377_hci_open_rings(struct bcm4377_data *bcm4377) { - struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); int ret; dev_dbg(&bcm4377->pdev->dev, "creating rings\n"); @@ -1374,23 +1457,92 @@ static int bcm4377_hci_open(struct hci_dev *hdev) return ret; } +static int bcm4377_hci_open(struct hci_dev *hdev) +{ + struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); + int ret; + + if (bcm4377->needs_reset) { + dev_info(&bcm4377->pdev->dev, + "device marked for reset, reinitializing hardware before open\n"); + ret = bcm4377_pci_reset(bcm4377); + if (ret) + return ret; + bcm4377->cal_needed = true; + } + + ret = bcm4377_hci_open_rings(bcm4377); + if (ret && !bcm4377->needs_reset) { + dev_warn(&bcm4377->pdev->dev, + "creating rings failed (%d); attempting hardware reset recovery\n", + ret); + ret = bcm4377_pci_reset(bcm4377); + if (ret) { + bcm4377->needs_reset = true; + return ret; + } + bcm4377->cal_needed = true; + ret = bcm4377_hci_open_rings(bcm4377); + if (ret) + bcm4377->needs_reset = true; + } + + return ret; +} + static int bcm4377_hci_close(struct hci_dev *hdev) { struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); dev_dbg(&bcm4377->pdev->dev, "destroying rings in hci_close\n"); - bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring); - bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring); - bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring); - bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring); - bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring); - bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring); + if (!bcm4377->needs_reset) { + if (bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring)) + bcm4377->needs_reset = true; + + if (!bcm4377->needs_reset && + bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring)) + bcm4377->needs_reset = true; + if (!bcm4377->needs_reset && + bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring)) + bcm4377->needs_reset = true; + } - bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring); - bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring); - bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring); - bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring); + if (bcm4377->needs_reset) { + dev_warn(&bcm4377->pdev->dev, + "ring teardown failed; marking device for reset on next open\n"); + bcm4377->acl_d2h_ring.enabled = false; + bcm4377->acl_h2d_ring.enabled = false; + bcm4377->sco_d2h_ring.enabled = false; + bcm4377->sco_h2d_ring.enabled = false; + bcm4377->hci_d2h_ring.enabled = false; + bcm4377->hci_h2d_ring.enabled = false; + bcm4377->sco_event_ring.enabled = false; + bcm4377->sco_ack_ring.enabled = false; + bcm4377->hci_acl_event_ring.enabled = false; + bcm4377->hci_acl_ack_ring.enabled = false; + } return 0; } @@ -1467,6 +1619,67 @@ static int bcm4377_hci_setup(struct hci_dev *hdev) return bcm4377_check_bdaddr(bcm4377); } +static int bcm4377_hci_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr) +{ + struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); + struct sk_buff *skb; + int err; + + skb = __hci_cmd_sync(hdev, 0xfc01, 6, bdaddr, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) { + err = PTR_ERR(skb); + dev_err(&bcm4377->pdev->dev, + "Change address command failed (%d)", err); + return err; + } + kfree_skb(skb); + + bacpy(&bcm4377->bdaddr, bdaddr); + return 0; +} + +static int bcm4377_hci_post_init(struct hci_dev *hdev) +{ + struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); + const bdaddr_t *bda = NULL; + int ret; + + if (!bcm4377->cal_needed) + return 0; + + dev_info(&bcm4377->pdev->dev, + "reloading calibration and PTB after hardware reset\n"); + + ret = bcm4377_hci_setup(hdev); + if (ret) { + dev_err(&bcm4377->pdev->dev, + "failed to reload calibration/PTB: %d\n", ret); + return ret; + } + + if (bacmp(&bcm4377->bdaddr, BDADDR_ANY)) + bda = &bcm4377->bdaddr; + else if (bacmp(&hdev->setup_addr, BDADDR_ANY)) + bda = &hdev->setup_addr; + else if (bacmp(&hdev->public_addr, BDADDR_ANY)) + bda = &hdev->public_addr; + + if (bda) { + dev_info(&bcm4377->pdev->dev, + "restoring BD_ADDR %pMR after hardware reset\n", bda); + ret = bcm4377_hci_set_bdaddr(hdev, bda); + if (ret) { + dev_err(&bcm4377->pdev->dev, + "failed to restore BD_ADDR: %d\n", ret); + return ret; + } + bacpy(&hdev->bdaddr, bda); + } + + bcm4377->cal_needed = false; + return 0; +} + static int bcm4377_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb) { struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); @@ -1504,24 +1717,6 @@ static int bcm4377_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb) return ret; } -static int bcm4377_hci_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr) -{ - struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); - struct sk_buff *skb; - int err; - - skb = __hci_cmd_sync(hdev, 0xfc01, 6, bdaddr, HCI_INIT_TIMEOUT); - if (IS_ERR(skb)) { - err = PTR_ERR(skb); - dev_err(&bcm4377->pdev->dev, - "Change address command failed (%d)", err); - return err; - } - kfree_skb(skb); - - return 0; -} - static int bcm4377_alloc_transfer_ring(struct bcm4377_data *bcm4377, struct bcm4377_transfer_ring *ring) { @@ -2241,6 +2436,9 @@ static int bcm4377_probe_of(struct bcm4377_data *bcm4377) return -ENOENT; } + device_property_read_u8_array(&bcm4377->pdev->dev, "local-bd-address", + (u8 *)&bcm4377->bdaddr, 6); + return 0; } @@ -2387,6 +2585,7 @@ static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id) hdev->send = bcm4377_hci_send_frame; hdev->set_bdaddr = bcm4377_hci_set_bdaddr; hdev->setup = bcm4377_hci_setup; + hdev->post_init = bcm4377_hci_post_init; if (bcm4377->hw->broken_mws_transport_config) hci_set_quirk(hdev, HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG); From eec15fc3c4782e9f6785eaa888e724ef9c3cd808 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:04:35 -0400 Subject: [PATCH 0015/1579] soc: apple: sep: add the J700 platform and Mesa revision Port the Neo warm SEP profile and 0x3356 sensor identity from linux-aurora 450f581707. Fold in the later hardware finding from 18165cf1de: this J700 accepted the T6020-style keybag encoding and refused the strict form. Keep the wider DMA capability described in the profile so older boards retain their masks. Co-Authored-By: Claude Opus 5 --- drivers/soc/apple/profile.rs | 44 +++++++++++++++++++++++++++++++++++- drivers/soc/apple/sbio.rs | 5 ++++ drivers/soc/apple/sensor.rs | 8 +++++-- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 61faf6b88736f..2c3c8c1e70db5 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -8,12 +8,14 @@ //! here once. The driver reads addresses from the device tree, never scans for //! them, and never patches properties at runtime. //! -//! Two bring-up targets are modelled: +//! Three bring-up targets are modelled: //! //! * `T8103` / J313 (MacBook Air, M1): the host boots the SEP with the boot //! endpoint handshake over a 0x30000 shared-memory window. //! * `T6020` / J414s (MacBook Pro 14", M2 Pro): the driver does the warm //! single-message registration over a 0x40000 window. +//! * `T8140` / J700 (MacBook Neo, A18 Pro): iBoot boots sepOS before the AP +//! OS, so the driver uses the warm registration path. // The boot handshake, identity source, sensor and DART fields are consumed by // the boot-endpoint, identity and sensor-transport paths. @@ -115,6 +117,8 @@ pub(crate) struct PlatformProfile { /// Require a static `apple,dma-range` on the SEP DART. The T6020 SEP only /// accepts IOVAs below 4 GiB; T8103 works with the stock DART aperture. pub(crate) dart_range_required: bool, + /// Whether the SEP DART IOVA window needs a DMA mask wider than 32 bits. + pub(crate) wide_dma_mask: bool, /// Reserved-memory region holding the SEP firmware image (cold-boot path). pub(crate) firmware_region: &'static CStr, /// Identity keybag CREATE_KEYBAG field encoding (per-SoC; see [`KeybagCreate`]). @@ -138,6 +142,7 @@ const T8103: PlatformProfile = PlatformProfile { capture_qualified: false, }, dart_range_required: false, + wide_dma_mask: false, firmware_region: c"sepfw", // Strict enclave: the type goes in the third word; the first word is 0. keybag_create: KeybagCreate { @@ -164,6 +169,7 @@ const T6020: PlatformProfile = PlatformProfile { capture_qualified: false, }, dart_range_required: true, + wide_dma_mask: false, firmware_region: c"sepfw", // Proven encoding: the lenient enclave takes the variant in the first word. keybag_create: KeybagCreate { @@ -173,10 +179,43 @@ const T6020: PlatformProfile = PlatformProfile { }, }; +/// MacBook Neo. The J700 ADT records a pre-booted SEP, a spi2 Mesa sensor +/// (0x3356), and a SEP DART aperture above 4 GiB. These values describe that +/// board; device-tree nodes remain authoritative for addresses and resources. +const T8140: PlatformProfile = PlatformProfile { + name: "T8140/J700", + shmem_capacity: 0x4_0000, + shmem_first_item: b"CINP", + bootstrap: Bootstrap::WarmRegister, + identity: IdentitySource::Chosen, + sensor: SensorProfile { + controller_base: 0x3_8510_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode2, + expected_id: 0x3356, + capture_qualified: false, + }, + dart_range_required: true, + wide_dma_mask: true, + firmware_region: c"sepfw", + // J700 hardware refused the strict encoding and accepted this T6020 form + // (linux-aurora commit 18165cf1de, 2026-09-19). + keybag_create: KeybagCreate { + variant: 5, + bag_type: 0, + arg: -1, + }, +}; + static_assert!(T8103.shmem_capacity == 0x30000); static_assert!(T6020.shmem_capacity == 0x40000); +static_assert!(T8140.shmem_capacity == 0x40000); static_assert!(T8103.sensor.controller_base == 0x235108000); static_assert!(T6020.sensor.controller_base == 0x39b108000); +static_assert!(T8140.sensor.controller_base == 0x385108000); static_assert!(T8103.sensor.expected_id == T6020.sensor.expected_id); /// Whether the machine root declares `compatible`. @@ -216,5 +255,8 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { if machine_has(b"apple,t6020") { return Ok(&T6020); } + if machine_has(b"apple,t8140") { + return Ok(&T8140); + } Err(ENODEV) } diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 8b885ed98753e..8c5d6b6a0f950 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1205,6 +1205,11 @@ impl SepData { if let sensor::Offset12::Identifier(id) = st.offset12() { if id != 0 && !sensor::KNOWN_IDENTIFIERS.contains(&id) { + dev_err!( + self.dev, + "sensor: unknown Mesa identifier {:#06x} at status offset 12; not registering it with the enclave\n", + id + ); return None; } } diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 6b6393dc28c04..f5f3ea6307c24 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -79,6 +79,8 @@ pub(crate) enum PowerSource { None, NodeProperty, ChipLine, + /// Rails switched by the SMC on J700. + SmcRails, } impl PowerSource { @@ -86,6 +88,7 @@ impl PowerSource { match v { 1 => PowerSource::NodeProperty, 2 => PowerSource::ChipLine, + 3 => PowerSource::SmcRails, _ => PowerSource::None, } } @@ -95,6 +98,7 @@ impl PowerSource { PowerSource::None => c"none — no power line was taken", PowerSource::NodeProperty => c"the sensor node's own gpios property", PowerSource::ChipLine => c"/soc/pinctrl@39b028000 line 122, taken by device-tree node", + PowerSource::SmcRails => c"the sensor node's SMC-switched rails (apple,smc-power-key)", } } } @@ -102,8 +106,8 @@ impl PowerSource { pub(crate) const POWER_ON_READ_DELAYS_MS: [u32; 4] = [0, 3, 10, 50]; pub(crate) const STATUS_IDENTIFIER: usize = 12; -// Known Mesa sensor revisions: 0x3352 through M4, 0x335e on newer parts. -pub(crate) const KNOWN_IDENTIFIERS: [u16; 2] = [0x3352, 0x335e]; +// Known Mesa sensor revisions, including the J700's 0x3356. +pub(crate) const KNOWN_IDENTIFIERS: [u16; 3] = [0x3352, 0x3356, 0x335e]; static_assert!(STATUS_IDENTIFIER + 2 <= STATUS_LEN); const CMD_LEN: usize = 7; From 889bb3af8e416a065f6c65e8cec574abc67564f4 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:04:55 -0400 Subject: [PATCH 0016/1579] soc: apple: sep: power J700 Mesa through SMC rails Port linux-aurora 29adacc631 with DT-described pcLD rail selectors and symmetric power sequencing. Normalize macsmc write replies so a successful response size is not mistaken for an error; retain a GPIO path for older boards. Co-Authored-By: Claude Opus 5 --- .../input/apple,mesa-fingerprint.yaml | 21 ++- drivers/soc/apple/Kconfig | 1 + drivers/soc/apple/sbio.rs | 4 +- drivers/soc/apple/sensor.rs | 1 + drivers/soc/apple/sensor_shim.c | 147 ++++++++++++++++++ 5 files changed, 172 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml index be9bbcf473e85..68531cf93ba0b 100644 --- a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml +++ b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml @@ -37,6 +37,19 @@ properties: maxItems: 1 description: Power line to the sensor, asserted to power it up. + apple,smc-power-key: + $ref: /schemas/types.yaml#/definitions/string + description: + Four-character SMC key used by boards with SMC-switched sensor rails. + The rail operation is an eight-byte little-endian pair of an on/off + value and a rail selector word. + + apple,smc-power-rails: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 4 + description: Rail selector words, in power-on order. + firmware-name: maxItems: 1 description: @@ -51,9 +64,15 @@ properties: required: - compatible - reg - - enable-gpios - firmware-name +oneOf: + - required: + - enable-gpios + - required: + - apple,smc-power-key + - apple,smc-power-rails + unevaluatedProperties: false examples: diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index c6062a8e0751f..ce4284781219e 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -107,6 +107,7 @@ config APPLE_SEP depends on HW_RANDOM depends on TRUSTED_KEYS depends on CRYPTO + depends on MFD_MACSMC || !MFD_MACSMC select CRYPTO_HMAC select CRYPTO_SHA256 select CRYPTO_AES diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 8c5d6b6a0f950..99665791198b6 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -249,12 +249,14 @@ impl SepData { let bound = sensor::is_bound(); self.sensor_present.store(bound, Relaxed); - if bound && sensor::power_line().is_none() { + if bound && sensor::power_source() == sensor::PowerSource::None { dev_warn!( self.dev, "sensor: no power line ({}); an unpowered sensor answers sixteen zero bytes like a dead bus\n", sensor::power_source().name() ); + } else if bound { + dev_info!(self.dev, "sensor: power via {}\n", sensor::power_source().name()); } } diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index f5f3ea6307c24..8fec6b71c8715 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -200,6 +200,7 @@ pub(crate) fn is_bound() -> bool { unsafe { sep_sensor_bound() != 0 } } +#[allow(dead_code)] pub(crate) fn power_line() -> Option { // SAFETY: no arguments; returns a negative errno when unbound. let n = unsafe { sep_sensor_power_line() }; diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index aff7da78fe49e..ec4d30581da67 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -10,10 +10,14 @@ #include #include #include +#include #include #include +#include +#include #include #include +#include #include "shim.h" @@ -50,9 +54,19 @@ static bool sep_registered; #define SEP_POWER_NONE 0 #define SEP_POWER_NODE_PROPERTY 1 #define SEP_POWER_CHIP_LINE 2 +#define SEP_POWER_SMC_RAILS 3 static int sep_power_source; +/* J700's Mesa power rails are selected by the pcLD SMC pKW8 operation. + * The DT gives the key and selectors; each write is { u32 on/off, u32 rail }. + */ +#define SEP_SMC_MAX_RAILS 4 +static struct apple_smc *sep_smc; +static smc_key sep_smc_key; +static u32 sep_smc_rails[SEP_SMC_MAX_RAILS]; +static int sep_smc_nrails; + /* * Data-ready line for interrupt-driven capture. On j414s it is the pin adjacent * to the power line (122) on the same controller; other boards describe it as @@ -63,6 +77,107 @@ static struct gpio_desc *sep_drdy; /* set only when we own the gpiochip line (j4 static int sep_drdy_irq = -1; static DECLARE_COMPLETION(sep_drdy_done); +#if IS_ENABLED(CONFIG_MFD_MACSMC) +static int sep_smc_write_rail(u32 rail, u16 value) +{ + u8 payload[8]; + int ret; + + put_unaligned_le32(value, payload); + put_unaligned_le32(rail, payload + 4); + ret = apple_smc_write(sep_smc, sep_smc_key, payload, sizeof(payload)); + /* macsmc forwards the SMC response size. Write replies may report no + * payload or echo the request length; neither is a failure status. + */ + if (ret == 0 || ret == (int)sizeof(payload)) + return 0; + return ret < 0 ? ret : -EIO; +} + +static int sep_smc_set_power(int on) +{ + int i, rc = 0; + + if (on) { + for (i = 0; i < sep_smc_nrails && !rc; i++) + rc = sep_smc_write_rail(sep_smc_rails[i], 1); + } else { + for (i = sep_smc_nrails - 1; i >= 0 && !rc; i--) + rc = sep_smc_write_rail(sep_smc_rails[i], 0); + } + if (rc) + pr_err("apple_sep: sensor: SMC rail write failed (%d)\n", rc); + return rc; +} + +/* Nothing is written until the expected eight-byte SMC key is verified. */ +static bool sep_acquire_smc_power(struct spi_device *spi) +{ + struct device_node *np = spi->dev.of_node, *smc_np; + struct platform_device *smc_pdev; + struct apple_smc_key_info info; + const char *key = NULL; + int n, rc; + + if (!np || of_property_read_string(np, "apple,smc-power-key", &key)) + return false; + if (!key || strlen(key) != 4) { + dev_warn(&spi->dev, "sep sensor: apple,smc-power-key must have four characters\n"); + return false; + } + n = of_property_count_u32_elems(np, "apple,smc-power-rails"); + if (n <= 0 || n > SEP_SMC_MAX_RAILS || + of_property_read_u32_array(np, "apple,smc-power-rails", sep_smc_rails, n)) { + dev_warn(&spi->dev, "sep sensor: expected 1..%d SMC rail words\n", + SEP_SMC_MAX_RAILS); + return false; + } + + smc_np = of_find_compatible_node(NULL, NULL, "apple,smc"); + if (!smc_np) + return false; + smc_pdev = of_find_device_by_node(smc_np); + of_node_put(smc_np); + if (!smc_pdev) + return false; + sep_smc = dev_get_drvdata(&smc_pdev->dev); + put_device(&smc_pdev->dev); + if (!sep_smc) + return false; + + sep_smc_key = __SMC_KEY(key[0], key[1], key[2], key[3]); + rc = apple_smc_get_key_info(sep_smc, sep_smc_key, &info); + if (rc || info.size != 8) { + dev_warn(&spi->dev, "sep sensor: SMC key %s absent or not eight bytes\n", key); + sep_smc = NULL; + return false; + } + sep_smc_nrails = n; + sep_power_source = SEP_POWER_SMC_RAILS; + dev_info(&spi->dev, "sep sensor: power from SMC key %s (%d rails)\n", key, n); + if (sep_smc_set_power(0)) { + sep_power_source = SEP_POWER_NONE; + sep_smc_nrails = 0; + sep_smc = NULL; + return false; + } + return true; +} +#else +static int sep_smc_set_power(int on) +{ + return -ENODEV; +} + +static bool sep_acquire_smc_power(struct spi_device *spi) +{ + if (spi->dev.of_node && + of_property_present(spi->dev.of_node, "apple,smc-power-key")) + dev_warn(&spi->dev, "sep sensor: SMC power needs CONFIG_MFD_MACSMC\n"); + return false; +} +#endif + /* * Takes the power line as an output driven low: the power cycle begins with an * off phase, so the line must be actively driven off, not merely read as low. @@ -73,6 +188,9 @@ static void sep_acquire_power(struct spi_device *spi) struct gpio_device *gdev; struct gpio_chip *gc; + if (sep_acquire_smc_power(spi)) + return; + sep_power = gpiod_get_index(&spi->dev, NULL, 0, GPIOD_OUT_LOW); if (!IS_ERR(sep_power)) { sep_power_source = SEP_POWER_NODE_PROPERTY; @@ -137,6 +255,13 @@ static void sep_acquire_power(struct spi_device *spi) static void sep_release_power(void) { + if (sep_power_source == SEP_POWER_SMC_RAILS) { + sep_smc_set_power(0); + sep_smc = NULL; + sep_smc_nrails = 0; + sep_power_source = SEP_POWER_NONE; + return; + } if (!sep_power) return; gpiod_set_value_cansleep(sep_power, 0); @@ -312,6 +437,18 @@ int sep_sensor_cs_timing_mode(void) /* Power cycle: off, 10 ms, on, 7 ms. -ENODEV if there is no power line. */ int sep_sensor_power_cycle(void) { + if (sep_power_source == SEP_POWER_SMC_RAILS) { + int rc = sep_smc_set_power(0); + + if (rc) + return rc; + msleep(SEP_SENSOR_OFF_DELAY_MS); + rc = sep_smc_set_power(1); + if (rc) + return rc; + msleep(SEP_SENSOR_ON_DELAY_MS); + return 0; + } if (!sep_power) return -ENODEV; @@ -330,6 +467,8 @@ int sep_sensor_power_source(void) int sep_sensor_power_line(void) { + if (sep_power_source == SEP_POWER_SMC_RAILS) + return -ENODEV; if (!sep_power) return -ENODEV; return desc_to_gpio(sep_power); @@ -354,6 +493,14 @@ const char *sep_sensor_firmware_name(void) /* Powers the sensor on/off, holding the hardware settling delay. */ int sep_sensor_power(int on) { + if (sep_power_source == SEP_POWER_SMC_RAILS) { + int rc = sep_smc_set_power(on ? 1 : 0); + + if (rc) + return rc; + msleep(on ? SEP_SENSOR_ON_DELAY_MS : SEP_SENSOR_OFF_DELAY_MS); + return 0; + } if (!sep_power) return -ENODEV; From b65f17cced0b25bda3bb40ae9a4b4d673c5da778 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:05:18 -0400 Subject: [PATCH 0017/1579] soc: apple: sep: widen the J700 DMA mask before allocation Port linux-aurora 11d4dff9a3. The Neo SEP DART aperture starts above 4 GiB, so select a 42-bit DMA mask for that profile before allocating shared memory. Keep the established M1/M2 DMA masks unchanged for regression testing. Co-Authored-By: Claude Opus 5 --- drivers/soc/apple/sep.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 61468bb8acd22..7f520ba87d44b 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -38,6 +38,7 @@ use kernel::{ bindings, device, dma, + dma::Device as _, driver, new_mutex, of, @@ -2096,6 +2097,13 @@ impl platform::Driver for SepDriver { } let sep_node = dt::DtNode::of_device(dev).ok_or(ENODEV)?; + // J700's SEP DART window starts above 4 GiB. Leave the proven masks + // on older platforms unchanged for the M1/M2 regression run. + if profile::detect()?.wide_dma_mask { + // SAFETY: probe has not allocated DMA memory for this device yet. + unsafe { pdev.dma_set_mask_and_coherent(dma::DmaMask::new::<42>())? }; + } + if dt::registration_already_sent(&sep_node) { dev_err!( dev, From f3a5cf1ead57b735ee8428f8895570b004c63388 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:05:32 -0400 Subject: [PATCH 0018/1579] soc: apple: sep: wait for late J700 key-store discovery Port linux-aurora 41017b63c1: the measured J700 application-endpoint burst arrives around 43 seconds after boot. Allow up to 90 seconds, returning as soon as the key store is advertised. Co-Authored-By: Claude Opus 5 --- drivers/soc/apple/sep.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 7f520ba87d44b..773c95cb92979 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -86,7 +86,10 @@ const VERIFY_WORK_ID: u64 = 3; static_assert!(VERIFY_WORK_ID != ENROL_WORK_ID && VERIFY_WORK_ID != SETTLE_WORK_ID); static_assert!(VERIFY_WORK_ID != 0); -const EXCHANGE_TIMEOUT_MS: time::Msecs = 15000; +// J700 traces show the key-store endpoint arriving around 43 s after boot, +// well after the first infrastructure endpoints. The wait exits promptly on +// other machines when their key store becomes ready. +const EXCHANGE_TIMEOUT_MS: time::Msecs = 90000; const PHASE_ATTACH: u32 = 0; const PHASE_EXCHANGE: u32 = 1; From fe3e8cf31b016761ecc8c1e25ac5256ea90b682d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 21 Sep 2026 23:05:45 -0400 Subject: [PATCH 0019/1579] soc: apple: sep: refuse unverified raw xART writes The raw-owner locator scores CRC-valid record signatures in a fixed 6 MiB window rather than resolving the current .gl APFS file extent. If both root records have moved past slot zero, multiple shifted windows can pass the root test; a higher-base tie can extend into unrelated APFS blocks. An explicit start sector does not prove ownership. Fail closed on xart_writes=1 before SEP attach, including the explicit-sector path; leave read-only inspection available. Disable the write-enabled loader recommendation. This is a safety interlock, not a macOS-parity repair: the APFS mapping and record update protocol still need hardware validation. --- drivers/soc/apple/Kconfig | 3 +- drivers/soc/apple/sep.rs | 9 +++++- drivers/soc/apple/xart_store.rs | 55 +++++++++++++++++---------------- tools/aurora-sep/README.md | 12 ++++--- 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index ce4284781219e..f80dacfdc9491 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -123,7 +123,8 @@ config APPLE_SEP The driver reads the existing machine-wide xART gigalocker directly from the iBoot system container, locating it by its root records. It validates the complete store and both root records before registering with SEP. - Writes remain disabled unless the xart_writes module parameter is set. + Raw APFS xART writes are refused until the extent and update protocol + can be verified; the module parameter cannot override this safety gate. Say Y here if you have an Apple silicon Mac. diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 773c95cb92979..9c67e55ff83ef 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2092,6 +2092,13 @@ impl platform::Driver for SepDriver { _info: Option<&()>, ) -> impl PinInit { let dev: &device::Device = pdev.as_ref(); + if *module_parameters::xart_writes.value() != 0 { + dev_err!( + dev, + "raw APFS xART writes are blocked until the .gl extent and update protocol are validated\n" + ); + return Err(ENOTSUPP); + } if *module_parameters::provision_keybag.value() != 0 && *module_parameters::xart_writes.value() == 0 { @@ -2183,7 +2190,7 @@ module! { params: { xart_writes: u8 { default: 0, - description: "Allow writes to the validated shared xART store", + description: "Request shared xART writes (refused by the raw APFS owner until its extent is verified)", }, xart_start_sector: u64 { default: 0, diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index ac9e0bf6c51e2..54d289d8a2946 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -5,8 +5,9 @@ //! //! The APFS locator exposes the existing `.gl` file as a block device. This //! module implements the record validation, duplicate repair, lookup and -//! copy-on-write ordering the SEP requires. It never creates storage -//! and it can be opened read-only for safe inspection and bring-up. +//! copy-on-write ordering the SEP requires. It never creates storage. +//! The raw block owner is restricted to read-only until APFS metadata can +//! establish the current physical extent of the `.gl` file. use crate::shim; use kernel::prelude::*; @@ -132,13 +133,22 @@ impl Store { /// system container directly and serves only the gigalocker extent, with no /// external helper and no hand-supplied sector. /// - /// `start_sector == 0` (the default) *locates the gigalocker automatically* - /// inside the container by its CRC-checked root records; a non-zero - /// `start_sector` is an explicit override for the rare case the scan should - /// be skipped. + /// `start_sector == 0` (the default) searches for CRC-checked root records; + /// a non-zero sector skips the search. Neither method proves that the + /// window belongs to the current APFS `.gl` file, so writes are refused. pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { + // A CRC-valid record signature does not establish the APFS file's + // physical extent. Once both roots move away from slot zero, several + // shifted windows pass the two-root test, and a shifted window can + // extend into APFS-owned blocks. Never write through this raw owner + // until the mapping has been checked against APFS file metadata. + // An explicit sector is not proof of ownership either. + if writes_enabled { + return Err(ENOTSUPP); + } if start_sector != 0 { - return Self::open_based(OWNER_PATH, writes_enabled, start_sector << 9); + let base = start_sector.checked_mul(512).ok_or(EINVAL)?; + return Self::open_based(OWNER_PATH, false, base); } Self::open_located(writes_enabled) } @@ -150,10 +160,9 @@ impl Store { /// is probed read-only with the full CRC-checked [`open_based`]; because a /// base shifted a few slots off the true origin still keeps both roots in its /// window (passing the two-root test while dropping records that fall - /// outside), the base is pinned by the candidate that recovers the MOST live - /// records — the exact discriminator, since only the true origin captures the - /// whole set and container noise never forges a CRC-valid record. Repair is - /// never armed during discovery; only the pinned base is re-opened writable. + /// outside), the candidate with the most live records is used for read-only + /// diagnostics. This heuristic cannot establish the APFS file's origin or + /// prove it is safe to write. Repair is never armed during discovery. /// Bounded by [`MAX_LOCATE_ATTEMPTS`] probes; returns `ENODATA` if not found. fn open_located(writes_enabled: bool) -> Result { // The container is a writable block device; the block layer only grants @@ -187,11 +196,9 @@ impl Store { // shifted off the true origin by a few slots still keeps both // roots inside its 6 MiB window, so it passes yet silently // drops the live records that fall outside the shifted window. - // Probe every candidate READ-ONLY (writes_enabled forced false, - // so repair never fires at an unconfirmed base) and pin the one - // that recovers the MOST live records: only the true origin - // captures the whole record set, and container noise never - // forges a CRC-valid record, so max live records is exact. + // Probe candidates without writes and prefer the largest + // recovered record set for diagnostics. An equally valid + // shifted window is possible; this is not an APFS lookup. let hit = off + p as u64; let mut best: Option<(usize, usize, u64)> = None; // (valid, malformed, base) let mut k: u64 = 0; @@ -214,13 +221,9 @@ impl Store { let better = match best { None => true, // More live records wins; ties break to fewer - // malformed, then to the HIGHER base. An up-shift - // drops the lowest occupied slots (and any root - // there, which fails the two-root test), so it - // never ties on live count; the only bases that - // can tie are down-shifts, which are all lower - // than the true origin — so the highest surviving - // candidate is the true origin. + // malformed, then to the higher base. The tie + // rule is a heuristic and may choose an + // up-shifted base when both roots moved. Some(b) => { cand.0 > b.0 || (cand.0 == b.0 && cand.1 < b.1) @@ -232,10 +235,8 @@ impl Store { } } } - // The gigalocker is unique, so the first hit that confirms any - // base has pinned it. Re-open the winner through a writable - // handle and arm repair per `writes_enabled` — repair runs - // now, and only at this confirmed origin. + // Re-open the diagnostic candidate. open_owner refuses + // writes, including automatic repair, on raw APFS extents. if let Some((_, _, base)) = best { return Self::open_based_ext(OWNER_PATH, true, writes_enabled, base); } diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 6c28a65403f51..c3f17ad3c1171 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -12,10 +12,14 @@ systemctl daemon-reload systemctl enable aurora-sep.service ``` -The kernel driver locates the xART gigalocker itself inside the iBoot System -Container (`modprobe apple_sep xart_writes=1 provision_keybag=1`, with no -`xart_start_sector`): it opens the container directly and serves only the -gigalocker extent, needing no external helper and no hand-supplied sector. +The raw xART owner is currently read-only. Do not enable the installed +`aurora-sep.service` or run `load-driver` on a shared macOS/Linux installation: +that script requests `xart_writes=1`, which the kernel now rejects while the +physical `.gl` extent is selected by scanning record signatures rather than +validated against APFS file metadata. The scan is ambiguous when live root +records have moved away from the start of the file. Read-only inspection does +not establish write safety. Re-enable provisioning only after the APFS mapping +and macOS-compatible record update protocol are verified on hardware. For fingerprint support, apply `patches/libfprint-1.94.100-apple-sep.patch` to libfprint 1.94.100, build and From 9c0186e307f425f12c7d114e31dbc439ba55cc31 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 21 Sep 2026 23:56:38 -0400 Subject: [PATCH 0020/1579] soc: apple: sep: resolve read-only xART extent through APFS metadata Replace the ambiguous root-record signature locator with a narrow, checksum-validated APFS checkpoint and .gl file-extent lookup. Open the shared container read-only and make an explicit sector an assertion rather than an override. Keep writes refused while the macOS ownership and update protocol are verified. --- drivers/soc/apple/sep.rs | 9 +- drivers/soc/apple/shim.rs | 4 +- drivers/soc/apple/store_shim.c | 2 +- drivers/soc/apple/xart_apfs.rs | 342 ++++++++++++++++++++ drivers/soc/apple/xart_store.rs | 188 ++--------- tools/aurora-sep/README.md | 27 +- tools/aurora-sep/XART-SAFETY.md | 41 +++ tools/aurora-sep/apfs-xart-inspect.py | 320 ++++++++++++++++++ tools/aurora-sep/tests/test_xart_locator.py | 49 +++ 9 files changed, 805 insertions(+), 177 deletions(-) create mode 100644 drivers/soc/apple/xart_apfs.rs create mode 100644 tools/aurora-sep/XART-SAFETY.md create mode 100644 tools/aurora-sep/apfs-xart-inspect.py create mode 100644 tools/aurora-sep/tests/test_xart_locator.py diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 9c67e55ff83ef..5f128c7fa005d 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -32,6 +32,7 @@ mod store; mod transfer; mod trusted; mod xarm; +mod xart_apfs; mod xart_store; use kernel::{ @@ -726,7 +727,7 @@ impl SepData { let base = store.base(); dev_info!( dev, - "xART: in-kernel raw-extent owner (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", + "xART: APFS-resolved read-only .gl extent (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", base, slots, records, @@ -2095,7 +2096,7 @@ impl platform::Driver for SepDriver { if *module_parameters::xart_writes.value() != 0 { dev_err!( dev, - "raw APFS xART writes are blocked until the .gl extent and update protocol are validated\n" + "raw APFS xART writes are blocked until the .gl update protocol and ownership invariants are validated\n" ); return Err(ENOTSUPP); } @@ -2190,11 +2191,11 @@ module! { params: { xart_writes: u8 { default: 0, - description: "Request shared xART writes (refused by the raw APFS owner until its extent is verified)", + description: "Request shared xART writes (refused until the macOS update protocol is verified)", }, xart_start_sector: u64 { default: 0, - description: "Explicit 512-byte start sector of the xART gigalocker extent within the iBoot system container. Zero (default) locates the gigalocker automatically in the container by its root records; a non-zero value overrides the search. Either way the driver opens the container directly", + description: "Expected 512-byte start sector of the APFS .gl extent in the iBoot container. Zero accepts the APFS result; nonzero must match it and cannot override it", }, provision_keybag: u8 { default: 0, diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index f02a9b86e32ef..4e25060d15d25 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -75,8 +75,8 @@ impl StoreFile { } pub(crate) fn open_block(path: &CStr, writable: bool) -> Result { - // SAFETY: `path` is NUL-terminated; the shim accepts only a block - // device whose global read-only state matches `writable`. + // SAFETY: `path` is NUL-terminated; the shim opens read-only unless + // explicitly asked to write, and rejects writes to a read-only device. let handle = unsafe { sep_store_open_block(path.as_char_ptr(), c_int::from(writable)) }; if handle.is_null() { return Err(ENODEV); diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c index b067c39372fa7..bdfd5dfbb6391 100644 --- a/drivers/soc/apple/store_shim.c +++ b/drivers/soc/apple/store_shim.c @@ -82,7 +82,7 @@ void *sep_store_open_block(const char *path, int writable) if (IS_ERR(f)) return NULL; if (!S_ISBLK(file_inode(f)->i_mode) || - bdev_read_only(file_bdev(f)) == !!writable) { + (writable && bdev_read_only(file_bdev(f)))) { filp_close(f, NULL); return NULL; } diff --git a/drivers/soc/apple/xart_apfs.rs b/drivers/soc/apple/xart_apfs.rs new file mode 100644 index 0000000000000..9eb833baca82d --- /dev/null +++ b/drivers/soc/apple/xart_apfs.rs @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Narrow, read-only APFS lookup for the existing xART `.gl` file. +//! +//! This is deliberately not a general APFS implementation. Unsupported tree +//! shapes, encryption, fragmented files, and bad metadata fail closed. In +//! particular, finding an extent does not authorize raw writes to that file. + +use crate::shim; +use kernel::prelude::*; + +const BLOCK: usize = 4096; +const ROOT_INFO_SIZE: usize = 40; +const XART_ROLE: u16 = 0x100; +const FILE_SIZE: u64 = 0x600000; +const FS_UNENCRYPTED: u64 = 1; +const MOD: u64 = 0xffff_ffff; + +fn le16(data: &[u8], off: usize) -> Result { + Ok(u16::from_le_bytes( + data.get(off..off + 2) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + )) +} + +fn le32(data: &[u8], off: usize) -> Result { + Ok(u32::from_le_bytes( + data.get(off..off + 4) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + )) +} + +fn le64(data: &[u8], off: usize) -> Result { + Ok(u64::from_le_bytes( + data.get(off..off + 8) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + )) +} + +/// APFS's Fletcher-64 over the little-endian words after the checksum field. +fn checksum(data: &[u8]) -> Result { + if data.len() != BLOCK { + return Err(EINVAL); + } + let mut s1 = 0u64; + let mut s2 = 0u64; + for word in data[8..].chunks_exact(4) { + let value = u32::from_le_bytes(word.try_into().map_err(|_| EINVAL)?) as u64; + s1 = (s1 + value) % MOD; + s2 = (s2 + s1) % MOD; + } + let c1 = (MOD - (s1 + s2) % MOD) % MOD; + let c2 = (MOD - (s1 + c1) % MOD) % MOD; + Ok((c2 << 32) | c1) +} + +fn read_object(file: &shim::StoreFile, size: u64, paddr: u64) -> Result> { + let start = paddr.checked_mul(BLOCK as u64).ok_or(EINVAL)?; + if start.checked_add(BLOCK as u64).ok_or(EINVAL)? > size { + return Err(EINVAL); + } + let mut data = KVec::new(); + data.resize(BLOCK, 0, GFP_KERNEL)?; + file.read_block_exact(start, &mut data)?; + if le64(&data, 0)? != checksum(&data)? { + return Err(EIO); + } + Ok(data) +} + +/// The descriptor ring holds mapping blocks followed by its superblock. A +/// checksummed superblock alone is not a complete checkpoint: verify every +/// mapping and its ephemeral object before accepting that transaction. +fn valid_checkpoint( + file: &shim::StoreFile, + size: u64, + sb: &[u8], + sb_paddr: u64, + desc_base: u64, + desc_count: u64, +) -> Result<()> { + if le64(sb, 112)? != desc_base || le32(sb, 104)? as u64 != desc_count { + return Err(ENOTSUPP); + } + let index = le32(sb, 136)? as u64; + let len = le32(sb, 140)? as u64; + if index >= desc_count || len < 2 || len > desc_count { + return Err(EINVAL); + } + let last = desc_base + (index + len - 1) % desc_count; + if last != sb_paddr { + return Err(EINVAL); + } + let xid = le64(sb, 16)?; + for j in 0..len - 1 { + let paddr = desc_base + (index + j) % desc_count; + let map = read_object(file, size, paddr)?; + if le32(&map, 24)? & 0xffff != 12 || le64(&map, 16)? != xid { + return Err(EINVAL); + } + let flags = le32(&map, 32)?; + if flags & 1 != u32::from(j == len - 2) { + return Err(EINVAL); + } + let count = le32(&map, 36)? as usize; + if count > (BLOCK - 40) / 40 { + return Err(EINVAL); + } + for i in 0..count { + let at = 40 + i * 40; + if le32(&map, at + 8)? != BLOCK as u32 { + return Err(ENOTSUPP); + } + let object = read_object(file, size, le64(&map, at + 32)?)?; + if le64(&object, 8)? != le64(&map, at + 24)? + || le64(&object, 16)? != xid + || le32(&object, 24)? != le32(&map, at)? + { + return Err(EINVAL); + } + } + } + Ok(()) +} + +fn node_entry(node: &[u8], index: usize, fixed: bool) -> Result<(&[u8], &[u8])> { + if node.len() != BLOCK || le16(node, 34)? != 0 { + return Err(ENOTSUPP); + } + let flags = le16(node, 32)?; + if (flags & 4 != 0) != fixed { + return Err(ENOTSUPP); + } + let count = le32(node, 36)? as usize; + if index >= count || count > 512 { + return Err(EINVAL); + } + let toc = 56usize + .checked_add(le16(node, 40)? as usize) + .ok_or(EINVAL)?; + let table_len = le16(node, 42)? as usize; + let key_start = toc.checked_add(table_len).ok_or(EINVAL)?; + let val_end = BLOCK - if flags & 1 != 0 { ROOT_INFO_SIZE } else { 0 }; + let stride = if fixed { 4 } else { 8 }; + if key_start > val_end || toc.checked_add(count * stride).ok_or(EINVAL)? > key_start { + return Err(EINVAL); + } + let item = toc.checked_add(index * stride).ok_or(EINVAL)?; + let (ko, kl, vo, vl) = if fixed { + ( + le16(node, item)? as usize, + 16, + le16(node, item + 2)? as usize, + 16, + ) + } else { + ( + le16(node, item)? as usize, + le16(node, item + 2)? as usize, + le16(node, item + 4)? as usize, + le16(node, item + 6)? as usize, + ) + }; + let ks = key_start.checked_add(ko).ok_or(EINVAL)?; + let vs = val_end.checked_sub(vo).ok_or(EINVAL)?; + let ke = ks.checked_add(kl).ok_or(EINVAL)?; + let ve = vs.checked_add(vl).ok_or(EINVAL)?; + if ks < key_start || ke > val_end || vs < key_start || ve > val_end { + return Err(EINVAL); + } + Ok((&node[ks..ke], &node[vs..ve])) +} + +fn omap_lookup( + file: &shim::StoreFile, + size: u64, + omap_paddr: u64, + oid: u64, + xid: u64, +) -> Result { + let omap = read_object(file, size, omap_paddr)?; + if le64(&omap, 8)? != omap_paddr { + return Err(EINVAL); + } + let tree_paddr = le64(&omap, 48)?; + let tree = read_object(file, size, tree_paddr)?; + if le64(&tree, 8)? != tree_paddr || le16(&tree, 34)? != 0 || le16(&tree, 32)? & 4 == 0 { + return Err(ENOTSUPP); + } + let mut selected: Option<(u64, u32, u32, u64)> = None; + for i in 0..le32(&tree, 36)? as usize { + let (key, val) = node_entry(&tree, i, true)?; + let key_xid = le64(key, 8)?; + if le64(key, 0)? == oid && key_xid <= xid && selected.is_none_or(|v| key_xid > v.0) { + selected = Some((key_xid, le32(val, 0)?, le32(val, 4)?, le64(val, 8)?)); + } + } + let (_, flags, object_size, paddr) = selected.ok_or(ENODATA)?; + if flags & 1 != 0 || object_size != BLOCK as u32 { + return Err(ENOTSUPP); + } + Ok(paddr) +} + +fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Result { + if le64(vol, 264)? & FS_UNENCRYPTED == 0 { + return Err(ENOTSUPP); + } + let omap = le64(vol, 128)?; + let root_oid = le64(vol, 136)?; + let root_paddr = omap_lookup(file, size, omap, root_oid, xid)?; + let root = read_object(file, size, root_paddr)?; + if le64(&root, 8)? != root_oid || le16(&root, 34)? != 0 || le16(&root, 32)? & 4 != 0 { + return Err(ENOTSUPP); + } + + let count = le32(&root, 36)? as usize; + let mut gl_file: Option = None; + for i in 0..count { + let (key, val) = node_entry(&root, i, false)?; + if key.len() < 12 || val.len() < 8 || le64(key, 0)? != ((9u64 << 60) | 2) { + continue; + } + let name_len = (le32(key, 8)? & 0x3ff) as usize; + if name_len == 0 || 12 + name_len > key.len() || key[12 + name_len - 1] != 0 { + return Err(EINVAL); + } + if &key[12..12 + name_len - 1] == b".gl" { + if gl_file.replace(le64(val, 0)?).is_some() { + return Err(ENOTSUPP); + } + } + } + let gl_file = gl_file.ok_or(ENODATA)?; + let mut private_id: Option = None; + for i in 0..count { + let (key, val) = node_entry(&root, i, false)?; + if key.len() >= 8 && le64(key, 0)? == ((3u64 << 60) | gl_file) { + if val.len() < 92 || private_id.replace(le64(val, 8)?).is_some() { + return Err(EINVAL); + } + } + } + let private_id = private_id.ok_or(ENODATA)?; + let mut extent: Option = None; + for i in 0..count { + let (key, val) = node_entry(&root, i, false)?; + if key.len() < 16 || le64(key, 0)? != ((8u64 << 60) | private_id) { + continue; + } + if val.len() < 24 || le64(key, 8)? != 0 || le64(val, 0)? != FILE_SIZE || le64(val, 16)? != 0 + { + return Err(ENOTSUPP); + } + let paddr = le64(val, 8)?; + let start = paddr.checked_mul(BLOCK as u64).ok_or(EINVAL)?; + if start.checked_add(FILE_SIZE).ok_or(EINVAL)? > size || extent.replace(start).is_some() { + return Err(EINVAL); + } + } + extent.ok_or(ENODATA) +} + +/// Return the byte offset of the sole contiguous 6 MiB `.gl` file extent. +pub(crate) fn locate(file: &shim::StoreFile) -> Result { + let device_size = file.size()?; + let base = read_object(file, device_size, 0)?; + if &base[32..36] != b"NXSB" || le32(&base, 36)? != BLOCK as u32 { + return Err(ENOTSUPP); + } + let blocks = le64(&base, 40)?; + if blocks == 0 + || blocks > device_size / BLOCK as u64 + || le64(&base, 56)? != 0 + || le64(&base, 64)? != 2 + { + return Err(ENOTSUPP); + } + let size = blocks.checked_mul(BLOCK as u64).ok_or(EINVAL)?; + let desc_count_raw = le32(&base, 104)?; + if desc_count_raw & (1 << 31) != 0 { + return Err(ENOTSUPP); + } + let desc_count = desc_count_raw as u64; + let desc_base = le64(&base, 112)?; + if desc_count == 0 + || desc_count > 1024 + || desc_base.checked_add(desc_count).ok_or(EINVAL)? > size / BLOCK as u64 + { + return Err(EINVAL); + } + let mut best: Option> = None; + let mut best_xid = 0u64; + for paddr in desc_base..desc_base + desc_count { + let Ok(candidate) = read_object(file, size, paddr) else { + continue; + }; + let candidate_xid = le64(&candidate, 16)?; + if &candidate[32..36] == b"NXSB" + && le32(&candidate, 36)? == BLOCK as u32 + && le64(&candidate, 40)? == blocks + && le64(&candidate, 56)? == 0 + && le64(&candidate, 64)? == 2 + && valid_checkpoint(file, size, &candidate, paddr, desc_base, desc_count).is_ok() + && candidate_xid > best_xid + { + best_xid = candidate_xid; + best = Some(candidate); + } + } + let best = best.ok_or(ENODATA)?; + let xid = le64(&best, 16)?; + let omap = le64(&best, 160)?; + let mut found: Option = None; + for i in 0..100usize { + let oid = le64(&best, 184 + i * 8)?; + if oid == 0 { + continue; + } + let paddr = omap_lookup(file, size, omap, oid, xid)?; + let vol = read_object(file, size, paddr)?; + if &vol[32..36] != b"APSB" || le64(&vol, 8)? != oid || le64(&vol, 16)? > xid { + return Err(EINVAL); + } + if le16(&vol, 964)? == XART_ROLE { + if found + .replace(xart_extent(file, size, &vol, le64(&vol, 16)?)?) + .is_some() + { + return Err(ENOTSUPP); + } + } + } + found.ok_or(ENODATA) +} diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index 54d289d8a2946..7c73107bc2ae8 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -3,11 +3,10 @@ //! Device-wide xART gigalocker record store. //! -//! The APFS locator exposes the existing `.gl` file as a block device. This -//! module implements the record validation, duplicate repair, lookup and -//! copy-on-write ordering the SEP requires. It never creates storage. -//! The raw block owner is restricted to read-only until APFS metadata can -//! establish the current physical extent of the `.gl` file. +//! The APFS locator identifies the existing `.gl` file's physical extent. +//! This module validates records and serves SEP reads from that extent. It +//! never creates storage. Raw writes remain blocked until the record update +//! protocol and APFS ownership rules are verified against macOS. use crate::shim; use kernel::prelude::*; @@ -20,28 +19,11 @@ const BLOCK_SIZE: usize = 0x1000; const SLOT_SIZE: usize = 0x9000; const HEADER_SIZE: usize = 0x22; const DELETE_SIZE: usize = BLOCK_SIZE; -/// Size of the APFS raw extent located on the target machine. -/// -/// The logical store is always exactly this many bytes. The raw-extent owner -/// opens a larger container and serves only the [`STORE_SIZE`] window at its -/// located base, so a wrong base cannot read past the extent. A wrong window -/// still fails closed: -/// the two root records will not validate and `open_based` returns `ENODATA` -/// rather than serving SEP an unrelated store. A machine with a different extent -/// size must pass that size through an explicit, reviewed interface. +/// The current `.gl` format occupies one contiguous six-MiB APFS file extent. +/// Unsupported layouts fail closed in `xart_apfs::locate`. const STORE_SIZE: u64 = 0x600000; const MAX_SLOTS: usize = 4096; -/// Sequential read window for the automatic gigalocker search (64 KiB — small -/// enough for a reliable kernel allocation, large enough to keep the scan of -/// the container to a modest number of reads). -const SCAN_CHUNK: usize = 1 << 16; -/// Upper bound on candidate bases confirmed during the automatic search, so a -/// container full of coincidental root-shaped headers cannot spin the scan. The -/// real store's roots sit within the first slots, so it is found long before -/// this; exceeding it falls back rather than looping. -const MAX_LOCATE_ATTEMPTS: u32 = 64; - pub(crate) const MAX_VALUE: usize = 0x8000; const KEY_KIND: usize = 0x01; @@ -95,8 +77,7 @@ impl Slot { pub(crate) struct Store { file: shim::StoreFile, - /// Byte offset of the gigalocker window within the opened container: the - /// located extent's offset (an explicit start-sector override sets it). + /// Byte offset of the APFS-resolved gigalocker extent in the container. base: u64, slots: KVec, revision: u64, @@ -129,131 +110,23 @@ fn valid_key(key: &Key) -> bool { } impl Store { - /// Opens the store as the in-kernel raw-extent owner: it opens the iBoot - /// system container directly and serves only the gigalocker extent, with no - /// external helper and no hand-supplied sector. - /// - /// `start_sector == 0` (the default) searches for CRC-checked root records; - /// a non-zero sector skips the search. Neither method proves that the - /// window belongs to the current APFS `.gl` file, so writes are refused. + /// Opens the iBoot system container read-only, resolves `.gl` through APFS + /// metadata, then validates the record store at exactly that extent. + /// A nonzero start sector is an assertion against the APFS result, never + /// an override of it. Neither lookup nor record validation authorizes + /// raw writes: macOS's update protocol is still unverified. pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { - // A CRC-valid record signature does not establish the APFS file's - // physical extent. Once both roots move away from slot zero, several - // shifted windows pass the two-root test, and a shifted window can - // extend into APFS-owned blocks. Never write through this raw owner - // until the mapping has been checked against APFS file metadata. - // An explicit sector is not proof of ownership either. if writes_enabled { return Err(ENOTSUPP); } + let file = shim::StoreFile::open_block(OWNER_PATH, false)?; + let base = crate::xart_apfs::locate(&file)?; if start_sector != 0 { - let base = start_sector.checked_mul(512).ok_or(EINVAL)?; - return Self::open_based(OWNER_PATH, false, base); - } - Self::open_located(writes_enabled) - } - - /// Locates the gigalocker inside the iBoot system container with no external - /// help. The container is read sequentially and searched, at block-aligned - /// offsets, for a root record's signature — a `1` or `2` key kind with an - /// all-zero UUID, the shape of [`Key::root`]. Each base implied by such a hit - /// is probed read-only with the full CRC-checked [`open_based`]; because a - /// base shifted a few slots off the true origin still keeps both roots in its - /// window (passing the two-root test while dropping records that fall - /// outside), the candidate with the most live records is used for read-only - /// diagnostics. This heuristic cannot establish the APFS file's origin or - /// prove it is safe to write. Repair is never armed during discovery. - /// Bounded by [`MAX_LOCATE_ATTEMPTS`] probes; returns `ENODATA` if not found. - fn open_located(writes_enabled: bool) -> Result { - // The container is a writable block device; the block layer only grants - // a read-only handle to a read-only device, so scan it through a writable - // handle. This does not arm any write — discovery never mutates the - // container (see open_based_ext, called below with arm_writes == false). - let probe = shim::StoreFile::open_block(OWNER_PATH, true)?; - let size = probe.size()?; - if size < STORE_SIZE { - return Err(ENODATA); - } - - let mut buf: KVec = KVec::new(); - buf.resize(SCAN_CHUNK, 0, GFP_KERNEL)?; - let mut attempts: u32 = 0; - let mut off: u64 = 0; - - while off + HEADER_SIZE as u64 <= size { - let want = core::cmp::min(SCAN_CHUNK as u64, size - off) as usize; - probe.read_exact(off, &mut buf[..want])?; - - let mut p = 0usize; - while p + HEADER_SIZE <= want { - let kind = buf[p + KEY_KIND]; - let uuid_zero = buf[p + KEY_UUID..p + KEY_UUID + 16].iter().all(|&b| b == 0); - if (kind == 1 || kind == 2) && uuid_zero { - // A root signature anchors the gigalocker: the true base is - // this hit minus a whole number of slots (slots and blocks are - // both 0x1000-aligned, so every candidate stays block-aligned). - // The two-root test alone does NOT pin the base — a base - // shifted off the true origin by a few slots still keeps both - // roots inside its 6 MiB window, so it passes yet silently - // drops the live records that fall outside the shifted window. - // Probe candidates without writes and prefer the largest - // recovered record set for diagnostics. An equally valid - // shifted window is possible; this is not an APFS lookup. - let hit = off + p as u64; - let mut best: Option<(usize, usize, u64)> = None; // (valid, malformed, base) - let mut k: u64 = 0; - while k as usize <= MAX_SLOTS { - let step = k * SLOT_SIZE as u64; - if step > hit { - break; - } - let base = hit - step; - k += 1; - if base + STORE_SIZE > size { - continue; - } - attempts += 1; - if attempts > MAX_LOCATE_ATTEMPTS { - break; - } - if let Ok(store) = Self::open_based_ext(OWNER_PATH, true, false, base) { - let cand = (store.valid_records, store.malformed_records, base); - let better = match best { - None => true, - // More live records wins; ties break to fewer - // malformed, then to the higher base. The tie - // rule is a heuristic and may choose an - // up-shifted base when both roots moved. - Some(b) => { - cand.0 > b.0 - || (cand.0 == b.0 && cand.1 < b.1) - || (cand.0 == b.0 && cand.1 == b.1 && cand.2 > b.2) - } - }; - if better { - best = Some(cand); - } - } - } - // Re-open the diagnostic candidate. open_owner refuses - // writes, including automatic repair, on raw APFS extents. - if let Some((_, _, base)) = best { - return Self::open_based_ext(OWNER_PATH, true, writes_enabled, base); - } - if attempts > MAX_LOCATE_ATTEMPTS { - return Err(ENODATA); - } - } - p += BLOCK_SIZE; - } - - if want < SCAN_CHUNK { - break; + if start_sector.checked_mul(512).ok_or(EINVAL)? != base { + return Err(EINVAL); } - // Overlap one block so a signature on the boundary is not missed. - off += (SCAN_CHUNK - BLOCK_SIZE) as u64; } - Err(ENODATA) + Self::open_file(file, false, base) } /// Opens a caller-selected block mapping at offset zero. @@ -272,27 +145,15 @@ impl Store { /// device (the raw container) serves only its gigalocker extent. /// /// The block handle's writability equals `writes_enabled`, and repair runs - /// when writes are enabled. For the raw-extent owner, discovery instead - /// needs a writable handle (the block layer only grants a read-only handle - /// to a read-only device, and the container is writable) *without* arming - /// repair at an unconfirmed base — see [`open_based_ext`]. + /// when writes are enabled. The APFS owner is always read-only. fn open_based(path: &CStr, writes_enabled: bool, base: u64) -> Result { - Self::open_based_ext(path, writes_enabled, writes_enabled, base) + let file = shim::StoreFile::open_block(path, writes_enabled)?; + Self::open_file(file, writes_enabled, base) } - /// As [`open_based`], but with the block-handle writability (`dev_writable`) - /// decoupled from whether repair writes are armed (`arm_writes`). Discovery - /// of the raw-extent owner opens the writable container with - /// `dev_writable == true` yet `arm_writes == false`, so a candidate base is - /// fully scanned and CRC-validated without a single write landing at an - /// origin that has not yet been confirmed as the true gigalocker. - fn open_based_ext( - path: &CStr, - dev_writable: bool, - arm_writes: bool, - base: u64, - ) -> Result { - let file = shim::StoreFile::open_block(path, dev_writable)?; + /// Complete record validation using the same read-only handle with which + /// APFS ownership was resolved, avoiding a close/reopen window. + fn open_file(file: shim::StoreFile, arm_writes: bool, base: u64) -> Result { let size = file.size()?; if base % BLOCK_SIZE as u64 != 0 || STORE_SIZE % BLOCK_SIZE as u64 != 0 { return Err(EINVAL); @@ -540,8 +401,7 @@ impl Store { Ok(true) } - /// Byte offset of the served gigalocker window within the opened container: - /// the located extent's offset (an explicit start-sector override sets it). + /// Byte offset of the APFS-resolved gigalocker extent in the container. pub(crate) fn base(&self) -> u64 { self.base } diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index c3f17ad3c1171..767b9043d2ad5 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -14,12 +14,27 @@ systemctl enable aurora-sep.service The raw xART owner is currently read-only. Do not enable the installed `aurora-sep.service` or run `load-driver` on a shared macOS/Linux installation: -that script requests `xart_writes=1`, which the kernel now rejects while the -physical `.gl` extent is selected by scanning record signatures rather than -validated against APFS file metadata. The scan is ambiguous when live root -records have moved away from the start of the file. Read-only inspection does -not establish write safety. Re-enable provisioning only after the APFS mapping -and macOS-compatible record update protocol are verified on hardware. +that script requests `xart_writes=1`, which the kernel rejects. The kernel now +resolves the physical `.gl` extent from checksum-validated APFS metadata for +read-only operation; unsupported layouts fail closed. This removes the raw +record-signature locator, which was ambiguous when roots moved away from the +start of the file. APFS lookup alone does not establish write safety. Re-enable +provisioning only after macOS-compatible record updates and APFS ownership +invariants are verified on hardware. + +For read-only structural diagnostics on Linux, run +`sudo python3 apfs-xart-inspect.py /dev/disk/by-partlabel/iBootSystemContainer`. +The inspector checks APFS metadata checksums, resolves the latest complete +checkpoint's xART `.gl` file extent, and compares it with the legacy root-record +heuristic. It prints physical addresses and record metadata, not payloads. +It currently supports only a single-node object map and file-system tree and +cannot authorize raw writes. Apple's APFS reference is the format authority: +https://developer.apple.com/support/apple-file-system/Apple-File-System-Reference.pdf. +`python3 -m unittest discover -s tests -v` runs a synthetic regression case: +when the first live root moves to slot 10, the legacy heuristic chooses a +window shifted 90 APFS blocks beyond the actual `.gl` extent. This proves the +locator can misidentify the extent; it does not identify the historical cause +of any particular damaged container. For fingerprint support, apply `patches/libfprint-1.94.100-apple-sep.patch` to libfprint 1.94.100, build and diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md new file mode 100644 index 0000000000000..1ea3708dc0a2e --- /dev/null +++ b/tools/aurora-sep/XART-SAFETY.md @@ -0,0 +1,41 @@ +# Shared xART gigalocker safety gate + +The iBoot APFS container and its XART `.gl` file are shared with macOS. The +kernel owner must remain read-only. `apple_sep.xart_writes=1` is rejected even +with an explicit `xart_start_sector`; the latter is now only an assertion +against the APFS-resolved file extent. Do not enable provisioning services on a +dual-boot installation while this gate is in place. + +## Evidence collected on 2026-09-21 + +- A read-only inspection of `m2`'s iBoot partition found a complete, valid + checkpoint at transaction 57, one XART volume (role `0x100`), and one + unencrypted `.gl` extent: 6 MiB beginning at APFS block 1236. All 11 live + gigalocker records in that extent passed their payload CRC checks. The raw + signature heuristic happened to select the same block on this snapshot. +- A synthetic image with the first root moved to slot 10 reproduces a legacy + locator failure: the highest-scoring window begins 90 APFS blocks after the + true file start and extends beyond `.gl`. This proves ambiguity in that + locator, **not** the cause of the previously damaged installation. +- Static inspection of the macOS 27.2 AppleSEPManager kernel collection found + `gl_rec_write` allocating a fresh 0x9000-byte slot, writing it, then clearing + 0x1000 bytes at the former slot. The low-level write path also calls a disk + cache-synchronization ioctl. `gl_fixup_rev_init` validates records and + handles duplicates during open. These observations do not prove that our + APFS ownership and failure ordering match macOS. + +## Before authorizing writes + +1. Confirm the exact macOS vnode/raw-device and extent-locking contract, + including whether any APFS copy-on-write, snapshot, encryption, or remap + state can invalidate the physical mapping during Linux's lifetime. +2. Verify record ordering, duplicate selection, failed-write recovery, + barriers/cache synchronization, and revision behavior against both macOS + code and controlled non-production tests. +3. Compare a pre/post macOS-and-Linux power-cycle trace, with APFS checksums, + `.gl` mapping and records checked before and after. Keep a recoverable image + of the iBoot partition before the first write experiment. + +The current APFS locator supports only a complete checkpoint, single-node +object maps and file-system tree, and a single unencrypted 6 MiB `.gl` extent. +Unsupported layouts fail closed; this is not a general APFS implementation. diff --git a/tools/aurora-sep/apfs-xart-inspect.py b/tools/aurora-sep/apfs-xart-inspect.py new file mode 100644 index 0000000000000..63cc276646f5e --- /dev/null +++ b/tools/aurora-sep/apfs-xart-inspect.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +"""Read-only, metadata-only inspector for the iBoot APFS xART volume. + +This intentionally never prints gigalocker contents or attempts a write. +It rejects metadata that it cannot validate instead of guessing an extent. +Only single-node object maps and a single-node xART file-system tree are +supported; this is a diagnostic, not an APFS mount or write authorization. +""" + +import os +import struct +import sys +import zlib + +BLOCK = 4096 +MOD = 0xffffffff + + +def u16(buf, off): + return struct.unpack_from("= self.count: + raise ValueError("physical address out of partition") + data = os.pread(self.fd, BLOCK, paddr * BLOCK) + if len(data) != BLOCK: + raise ValueError("short block read") + if verify and u64(data, 0) != checksum(data): + raise ValueError("APFS checksum mismatch at block %d" % paddr) + return data + + +def node_entries(buf): + level, flags, count = u16(buf, 34), u16(buf, 32), u32(buf, 36) + fixed = bool(flags & 4) + start = 56 + u16(buf, 40) + key_start = start + u16(buf, 42) + val_end = BLOCK - (40 if flags & 1 else 0) + stride = 4 if fixed else 8 + if count > 512 or start < 56 or start + stride * count > key_start or key_start > val_end: + raise ValueError("invalid B-tree table") + entries = [] + for i in range(count): + at = start + i * stride + if fixed: + koff, voff = u16(buf, at), u16(buf, at + 2) + klen, vlen = 16, 16 + else: + koff, klen, voff, vlen = struct.unpack_from("= desc_count or not 2 <= length <= desc_count: + return False + if desc_base + (index + length - 1) % desc_count != sb_paddr: + return False + xid = u64(sb, 16) + try: + for j in range(length - 1): + mapping = disk.read(desc_base + (index + j) % desc_count) + if u32(mapping, 24) & 0xffff != 12 or u64(mapping, 16) != xid: + return False + if bool(u32(mapping, 32) & 1) != (j == length - 2): + return False + count = u32(mapping, 36) + if count > (BLOCK - 40) // 40: + return False + for i in range(count): + off = 40 + i * 40 + if u32(mapping, off + 8) != BLOCK: + return False + item = disk.read(u64(mapping, off + 32)) + if (u64(item, 8) != u64(mapping, off + 24) or + u64(item, 16) != xid or + u32(item, 24) != u32(mapping, off)): + return False + except ValueError: + return False + return True + + +def omap_lookup(disk, omap_paddr, oid, max_xid): + omap = disk.read(omap_paddr) + if u64(omap, 8) != omap_paddr: + raise ValueError("object-map identity mismatch") + tree = u64(omap, 48) + node = disk.read(tree) + level, flags, entries = node_entries(node) + if level: + raise ValueError("multilevel object map needs further inspection") + if not flags & 4: + raise ValueError("object map is not fixed-size") + matches = [] + for key, val in entries: + if u64(key, 0) == oid and u64(key, 8) <= max_xid: + matches.append((u64(key, 8), u32(val, 0), u64(val, 8), u32(val, 4))) + if not matches: + raise ValueError("object %d not found in map" % oid) + selected = max(matches) + if selected[1] & 1: + raise ValueError("object %d deleted at selected transaction" % oid) + return selected[2:] + + +def inspect(disk): + base = disk.read(0) + if base[32:36] != b"NXSB" or u32(base, 36) != BLOCK: + raise ValueError("unsupported container superblock") + desc_base, desc_count = u64(base, 112), u32(base, 104) + if desc_count < 1 or desc_count > 1024: + raise ValueError("unsupported checkpoint descriptor geometry") + checkpoints = [] + for paddr in range(desc_base, desc_base + desc_count): + try: + block = disk.read(paddr) + except ValueError: + continue + if (block[32:36] == b"NXSB" and u32(block, 36) == BLOCK and + checkpoint_complete(disk, block, paddr, desc_base, desc_count)): + checkpoints.append((u64(block, 16), paddr, block)) + if not checkpoints: + raise ValueError("no valid container superblock") + xid, cpaddr, sb = max(checkpoints) + print("partition_bytes=%d blocks=%d" % (disk.size, disk.count)) + print("block_zero_xid=%d" % u64(base, 16)) + print("valid_superblocks=%s" % [(x, p) for x, p, _ in checkpoints]) + print("selected_checkpoint_xid=%d block=%d" % (xid, cpaddr)) + print("container_omap=%d" % u64(sb, 160)) + fs_oids = [u64(sb, 184 + i * 8) for i in range(100)] + fs_oids = [v for v in fs_oids if v] + print("volume_oids=%s" % fs_oids) + for oid in fs_oids: + paddr, size = omap_lookup(disk, u64(sb, 160), oid, xid) + vol = disk.read(paddr) + if vol[32:36] != b"APSB" or u64(vol, 8) != oid: + raise ValueError("bad volume superblock") + role = u16(vol, 964) + print("volume oid=%d block=%d xid=%d role=0x%x" % + (oid, paddr, u64(vol, 16), role)) + if role != 0x100: + continue + vol_omap = u64(vol, 128) + root_oid = u64(vol, 136) + root_paddr, root_size = omap_lookup(disk, vol_omap, root_oid, xid) + root = disk.read(root_paddr) + level, flags, entries = node_entries(root) + print("xart_root oid=%d block=%d level=%d records=%d" % + (root_oid, root_paddr, level, len(entries))) + if level: + raise ValueError("multilevel xART FS tree needs further inspection") + files = [] + for key, val in entries: + if len(key) < 12 or len(val) < 8: + continue + k = u64(key, 0) + if k == ((9 << 60) | 2): + name_len = u32(key, 8) & 0x3ff + if name_len and 12 + name_len <= len(key): + name = key[12:12 + name_len - 1] + if name == b".gl": + files.append(u64(val, 0)) + print("gigalocker_file_ids=%s" % files) + if len(files) != 1: + raise ValueError("expected exactly one .gl file") + inode_values = [value for key, value in entries + if len(key) >= 8 and u64(key, 0) == ((3 << 60) | files[0])] + if len(inode_values) != 1 or len(inode_values[0]) < 92: + raise ValueError(".gl inode missing or short") + inode = inode_values[0] + private_id, inode_flags = u64(inode, 8), u64(inode, 48) + print("gigalocker_inode_private_id=%d flags=0x%x links=%d write_generation=%d" % + (private_id, inode_flags, u32(inode, 56), u32(inode, 64))) + print("xart_volume_flags=0x%x snapshot_count=%d" % (u64(vol, 264), u32(disk.read(vol_omap), 36))) + extents = [] + for key, val in entries: + if len(key) >= 16 and len(val) >= 24 and u64(key, 0) == ((8 << 60) | private_id): + logical, length_flags, physical = u64(key, 8), u64(val, 0), u64(val, 8) + length = length_flags & ((1 << 56) - 1) + if logical % BLOCK or length % BLOCK or physical + length // BLOCK > disk.count: + raise ValueError("invalid .gl extent") + extents.append((logical, length, physical, length_flags >> 56, u64(val, 16))) + extents.sort() + print("gigalocker_extents_logical_bytes_physical_blocks_flags=%s" % extents) + if not extents: + raise ValueError("no .gl extents") + if len(extents) == 1 and extents[0][0] == 0 and extents[0][1] == 0x600000: + inspect_raw_locator(disk, extents[0][2]) + scan_other_roots(disk, extents[0][2], extents[0][1]) + + +def inspect_raw_locator(disk, gl_block): + """Mirror the removed first-root candidate scoring, without writes.""" + slot_size, store_size = 0x9000, 0x600000 + gl_base = gl_block * BLOCK + gl = os.pread(disk.fd, store_size, gl_base) + if len(gl) != store_size: + raise ValueError("short gigalocker extent read") + roots = [] + records = [] + for idx in range(store_size // slot_size): + off = idx * slot_size + if gl[off + 1] in (1, 2) and gl[off + 2:off + 18] == bytes(16): + roots.append(idx) + kind = gl[off + 1] + if not kind: + continue + length = u32(gl, off + 0x12) + valid = (1 <= kind <= 4 and (kind > 2 or gl[off + 2:off + 18] == bytes(16)) + and 1 <= length <= 0x8000 and + zlib.crc32(gl[off + 0x22:off + 0x22 + length]) == u32(gl, off + 0x16)) + records.append((idx, kind, length, u64(gl, off + 0x1a), valid)) + print("gigalocker_records_slot_kind_length_revision_crc_valid=%s" % records) + print("raw_root_signature_slots=%s" % roots) + if not roots: + return + hit = gl_base + roots[0] * slot_size + candidates = [] + attempts = 0 + for k in range(4097): + step = k * slot_size + if step > hit: + break + base = hit - step + if base + store_size > disk.size: + continue + attempts += 1 + if attempts > 64: + break + raw = os.pread(disk.fd, store_size, base) + live = {} + malformed = 0 + for idx in range(store_size // slot_size): + off = idx * slot_size + kind = raw[off + 1] + if kind == 0: + continue + uid = raw[off + 2:off + 18] + length = u32(raw, off + 0x12) + crc = u32(raw, off + 0x16) + revision = u64(raw, off + 0x1a) + if not (1 <= kind <= 4 and (kind > 2 or uid == bytes(16)) and + 1 <= length <= 0x8000 and + zlib.crc32(raw[off + 0x22:off + 0x22 + length]) == crc): + malformed += 1 + continue + key = (kind, uid) + if key not in live or revision >= live[key]: + live[key] = revision + if (1, bytes(16)) in live and (2, bytes(16)) in live: + candidates.append((len(live), -malformed, base)) + if not candidates: + print("raw_locator=no_valid_candidate") + return + best = max(candidates) + print("raw_locator_candidates=%d attempts=%d" % (len(candidates), attempts)) + print("raw_locator_best_block=%d apfs_extent_block=%d shift_blocks=%d live_records=%d malformed=%d" % + (best[2] // BLOCK, gl_block, (best[2] - gl_base) // BLOCK, best[0], -best[1])) + + +def scan_other_roots(disk, gl_block, gl_size): + """Count CRC-valid root signatures outside APFS's current file extent.""" + outside = [] + for block in range(disk.count): + off = block * BLOCK + header = os.pread(disk.fd, 0x22, off) + if len(header) != 0x22 or header[1] not in (1, 2) or header[2:18] != bytes(16): + continue + length = u32(header, 0x12) + if not 1 <= length <= 0x8000: + continue + payload = os.pread(disk.fd, length, off + 0x22) + if len(payload) != length or zlib.crc32(payload) != u32(header, 0x16): + continue + if not gl_block <= block < gl_block + gl_size // BLOCK: + outside.append((block, header[1], u64(header, 0x1a))) + print("crc_valid_root_signatures_outside_apfs_gl=%s" % outside[:32]) + if len(outside) > 32: + print("crc_valid_root_signatures_outside_apfs_gl_additional=%d" % (len(outside) - 32)) + + +if __name__ == "__main__": + try: + inspect(Disk(sys.argv[1])) + except (ValueError, OSError, IndexError) as exc: + print("INCOMPLETE: %s" % exc, file=sys.stderr) + sys.exit(1) diff --git a/tools/aurora-sep/tests/test_xart_locator.py b/tools/aurora-sep/tests/test_xart_locator.py new file mode 100644 index 0000000000000..167c977ec2bd0 --- /dev/null +++ b/tools/aurora-sep/tests/test_xart_locator.py @@ -0,0 +1,49 @@ +"""Regression proof for the ambiguity of the legacy raw-header locator.""" + +import contextlib +import importlib.util +import io +from pathlib import Path +import struct +import tempfile +import unittest +import zlib + + +SCRIPT = Path(__file__).resolve().parents[1] / "apfs-xart-inspect.py" +SPEC = importlib.util.spec_from_file_location("apfs_xart_inspect", SCRIPT) +INSPECTOR = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(INSPECTOR) + + +def record(kind, revision): + out = bytearray(0x9000) + payload = b"test" + out[1] = kind + struct.pack_into(" Date: Fri, 3 Jul 2026 13:13:26 +0200 Subject: [PATCH 0021/1579] arm64: dts: apple: add minimal MacBook Neo device tree Introduce the original T8140/J700 CPU, interrupt, power-management, watchdog and framebuffer description as the base for later SEP device-tree work. Drop an empty aliases node and excess whitespace from the imported commit. Signed-off-by: Yureka Lilian (cherry picked and cleaned from c8c58c8d3b1f5f4a44ef2419b2fb999c64491572) --- arch/arm64/boot/dts/apple/Makefile | 1 + arch/arm64/boot/dts/apple/t8140-j700.dts | 42 ++ arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 756 ++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8140.dtsi | 176 +++++ 4 files changed, 975 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8140-j700.dts create mode 100644 arch/arm64/boot/dts/apple/t8140-pmgr.dtsi create mode 100644 arch/arm64/boot/dts/apple/t8140.dtsi diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index 69d4f03152742..83ee9012405ab 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -109,3 +109,4 @@ dtb-$(CONFIG_ARCH_APPLE) += t8122-j434.dtb dtb-$(CONFIG_ARCH_APPLE) += t8122-j504.dtb dtb-$(CONFIG_ARCH_APPLE) += t8122-j613.dtb dtb-$(CONFIG_ARCH_APPLE) += t8122-j615.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8140-j700.dtb diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts new file mode 100644 index 0000000000000..76b1537428ee5 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Neo + * + * target-type: J700 + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t8140.dtsi" + +/ { + compatible = "apple,j700", "apple,t8140", "apple,arm-platform"; + model = "Apple MacBook Neo"; + + chosen { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + framebuffer0: framebuffer@0 { + compatible = "apple,simple-framebuffer", "simple-framebuffer"; + reg = <0 0 0 0>; /* To be filled by loader */ + /* Format properties will be added by loader */ + status = "disabled"; + }; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* To be filled by loader */ + }; + + memory@800000000 { + device_type = "memory"; + reg = <0x100 0 0x2 0>; /* To be filled by loader */ + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi new file mode 100644 index 0000000000000..73c09c0bce2a9 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -0,0 +1,756 @@ +&pmgr { + ps_sep_aon: power-controller@60 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sep_aon"; + apple,always-on; + }; + + ps_msg: power-controller@108 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msg"; + }; + + ps_aic: power-controller@110 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aic"; + apple,always-on; + }; + + ps_dwi: power-controller@118 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dwi"; + }; + + ps_gpio: power-controller@120 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gpio"; + }; + + ps_pmc: power-controller@138 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x138 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmc"; + }; + + ps_pms_fpwm0: power-controller@140 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm0"; + }; + + ps_pms_fpwm1: power-controller@148 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm1"; + }; + + ps_pms_fpwm2: power-controller@150 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm2"; + }; + + ps_pms_fpwm3: power-controller@158 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm3"; + }; + + ps_pms_fpwm4: power-controller@160 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x160 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm4"; + }; + + ps_pms_c1ppt: power-controller@168 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_c1ppt"; + apple,always-on; + }; + + ps_soc_rc: power-controller@170 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x170 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_rc"; + }; + + ps_soc_dpe: power-controller@178 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x178 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_dpe"; + apple,always-on; + }; + + ps_pmgr_soc_ocla: power-controller@180 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x180 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmgr_soc_ocla"; + }; + + ps_ispsens0: power-controller@188 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x188 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens0"; + }; + + ps_ispsens1: power-controller@190 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens1"; + }; + + ps_ap_tmm: power-controller@1a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ap_tmm"; + }; + + ps_afggfxni0: power-controller@1b8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "afggfxni0"; + apple,always-on; + }; + + ps_gfx: power-controller@1f0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gfx"; + }; + + ps_jpg: power-controller@230 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x230 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "jpg"; + }; + + ps_msr: power-controller@238 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x238 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr"; + }; + + ps_venc0_sys: power-controller@240 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x240 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_sys"; + }; + + ps_prores: power-controller@250 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x250 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "prores"; + }; + + ps_apcie_gp: power-controller@258 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x258 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_gp"; + }; + + ps_isp_sys: power-controller@268 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x268 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_sys"; + }; + + ps_ans: power-controller@278 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x278 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ans"; + }; + + ps_disp_sys: power-controller@280 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x280 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_sys"; + apple,always-on; /* TODO: figure out if we can enable PM here */ + }; + + ps_dispext0_sys: power-controller@288 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_sys"; + }; + + ps_ane_sys: power-controller@290 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x290 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_sys"; + }; + + ps_msr_ase_core: power-controller@298 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x298 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr_ase_core"; + power-domains = <&ps_msr>; + }; + + ps_apcie_sys_gp: power-controller@2a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_gp"; + power-domains = <&ps_apcie_gp>; + }; + + ps_sio_cpu: power-controller@2a8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sio_cpu"; + }; + + ps_fpwm0: power-controller@2b0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm0"; + }; + + ps_fpwm1: power-controller@2b8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm1"; + }; + + ps_fpwm2: power-controller@2c0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm2"; + }; + + ps_i2c0: power-controller@2c8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c0"; + }; + + ps_i2c1: power-controller@2d0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c1"; + }; + + ps_i2c2: power-controller@2d8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c2"; + }; + + ps_i2c3: power-controller@2e0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c3"; + }; + + ps_i2c4: power-controller@2e8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c4"; + }; + + ps_spi_p: power-controller@2f0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + }; + + ps_uart_p: power-controller@2f8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_p"; + }; + + ps_dpa_p: power-controller@300 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa_p"; + apple,always-on; + }; + + ps_aes: power-controller@308 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x308 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aes"; + }; + + ps_apcie_st: power-controller@310 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x310 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_st"; + power-domains = <&ps_ans>; + }; + + ps_disp_fe: power-controller@318 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x318 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_fe"; + power-domains = <&ps_disp_sys>; + apple,always-on; /* TODO: figure out if we can enable PM here */ + }; + + ps_dispext0_fe: power-controller@320 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x320 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_fe"; + power-domains = <&ps_dispext0_sys>; + }; + + ps_spi0: power-controller@330 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi0"; + power-domains = <&ps_spi_p>; + }; + + ps_spi1: power-controller@338 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x338 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi1"; + power-domains = <&ps_spi_p>; + }; + + ps_spi2: power-controller@340 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x340 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi2"; + power-domains = <&ps_spi_p>; + }; + + ps_spi3: power-controller@348 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x348 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi3"; + power-domains = <&ps_spi_p>; + }; + + ps_qspi: power-controller@350 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x350 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "qspi"; + power-domains = <&ps_spi_p>; + }; + + ps_uart_n: power-controller@358 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x358 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_n"; + power-domains = <&ps_uart_p>; + }; + + ps_uart0: power-controller@360 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x360 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart0"; + power-domains = <&ps_uart_p>; + }; + + ps_uart1: power-controller@368 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x368 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart1"; + power-domains = <&ps_uart_p>; + }; + + ps_uart2: power-controller@370 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x370 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart2"; + power-domains = <&ps_uart_p>; + }; + + ps_uart3: power-controller@378 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x378 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart3"; + power-domains = <&ps_uart_p>; + }; + + ps_uart4: power-controller@380 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x380 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart4"; + power-domains = <&ps_uart_p>; + }; + + ps_uart5: power-controller@388 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x388 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart5"; + power-domains = <&ps_uart_p>; + }; + + ps_dpa0: power-controller@390 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa0"; + power-domains = <&ps_dpa_p>; + }; + + ps_dpa1: power-controller@398 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x398 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa1"; + power-domains = <&ps_dpa_p>; + }; + + ps_apcie_sys_st: power-controller@3a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_st"; + power-domains = <&ps_ans>, <&ps_apcie_st>; + }; + + ps_apcie_phy_sw: power-controller@3c0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_phy_sw"; + power-domains = <&ps_apcie_sys_st>, <&ps_apcie_sys_gp>; + }; + + ps_pmp: power-controller@3d0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmp"; + apple,always-on; + }; + + ps_pms_sram: power-controller@3d8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_sram"; + apple,always-on; + }; + + ps_atc0_usb_aon: power-controller@3e0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_usb_aon"; + }; + + ps_atc0_common: power-controller@3e8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_common"; + power-domains = <&ps_atc0_usb_aon>; + }; + + ps_atc0_usb: power-controller@3f0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_usb"; + power-domains = <&ps_atc0_common>; + }; + + ps_sep: power-controller@c00 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc00 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sep"; + apple,always-on; + }; + + ps_venc0_dma: power-controller@8000 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_dma"; + power-domains = <&ps_venc0_sys>; + }; + + ps_venc0_pipe4: power-controller@8008 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_pipe4"; + power-domains = <&ps_venc0_dma>; + }; + + ps_venc0_pipe5: power-controller@8010 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_pipe5"; + power-domains = <&ps_venc0_dma>; + }; + + ps_venc0_me0: power-controller@8018 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_me0"; + power-domains = <&ps_venc0_dma>; + }; + + ps_venc0_me1: power-controller@8020 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_me1"; + power-domains = <&ps_venc0_me0>; + }; +}; + +&pmgr_mini { + ps_debug_gated: power-controller@0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_gated"; + apple,always-on; + }; + + ps_nub_spmi0: power-controller@58 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x58 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi0"; + apple,always-on; + }; + + ps_nub_spmi1: power-controller@60 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi1"; + apple,always-on; + }; + + ps_nub_spmi_a0: power-controller@68 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x68 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi_a0"; + apple,always-on; + }; + + ps_nub_aon_nogate: power-controller@78 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x78 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_aon_nogate"; + apple,always-on; + }; + + ps_nub_ocla: power-controller@80 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x80 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_ocla"; + apple,always-on; + }; + + ps_nub_gpio: power-controller@88 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x88 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_gpio"; + apple,always-on; + }; + + ps_nub_fabric: power-controller@a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_fabric"; + apple,always-on; + }; + + ps_nub_sram: power-controller@a8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_sram"; + apple,always-on; + }; + + ps_debug_switch: power-controller@b0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_switch"; + apple,always-on; + }; + + ps_smc_i2cm0: power-controller@8028 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "smc_i2cm0"; + }; + + ps_smc_i2cm1: power-controller@8030 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "smc_i2cm1"; + }; + + ps_smc_i2cm2: power-controller@8038 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "smc_i2cm2"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi new file mode 100644 index 0000000000000..f2199ded4b783 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple T8140 "A18 Pro" SoC + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include + +/ { + compatible = "apple,t8140", "apple,arm-platform"; + + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu_e0>; + }; + core1 { + cpu = <&cpu_e1>; + }; + core2 { + cpu = <&cpu_e2>; + }; + core3 { + cpu = <&cpu_e3>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu_p0>; + }; + core1 { + cpu = <&cpu_p1>; + }; + }; + }; + + cpu_e0: cpu@0 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x0>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; + next-level-cache = <&l2_cache_0>; + }; + + cpu_e1: cpu@1 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x1>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; + next-level-cache = <&l2_cache_0>; + }; + cpu_e2: cpu@2 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x2>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; + next-level-cache = <&l2_cache_0>; + }; + + cpu_e3: cpu@3 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x3>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; + next-level-cache = <&l2_cache_0>; + }; + + cpu_p0: cpu@10100 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10100>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; + next-level-cache = <&l2_cache_1>; + }; + + cpu_p1: cpu@10101 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10101>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; + next-level-cache = <&l2_cache_1>; + }; + + l2_cache_0: l2-cache-0 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x400000>; + }; + + l2_cache_1: l2-cache-1 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x1000000>; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupt-parent = <&aic>; + interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; + interrupts = , + , + , + ; + }; + + clkref: clock-ref { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + clock-output-names = "clkref"; + }; + + soc: soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + + nonposted-mmio; + ranges; + + pmgr: power-management@300700000 { + compatible = "apple,t8140-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x3 0x700000 0 0x14000>; + /* child nodes are added in t8140-pmgr.dtsi */ + }; + + aic: interrupt-controller@301000000 { + compatible = "apple,t8140-aic3", "apple,t8122-aic3"; + #interrupt-cells = <3>; + interrupt-controller; + reg = <0x00000003 0x01000000 0x00000000 0x1cc000>, + <0x00000003 0x01040000 0x00000000 0x4000>; + reg-names = "core", "event"; + power-domains = <&ps_aic>; + }; + + pmgr_mini: power-management@308280000 { + compatible = "apple,t8140-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x3 0x8280000 0 0xc000>; + /* child nodes are added in t8140-pmgr.dtsi */ + }; + + wdt: watchdog@3082b0000 { + compatible = "apple,t8140-wdt", "apple,wdt"; + reg = <0x3 0x082b0000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + }; + }; +}; + +#include "t8140-pmgr.dtsi" From e9e52e6dc932d6f878ef8289db71f428f18f4354 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 16:48:39 +0200 Subject: [PATCH 0022/1579] phy: apple: atc: add DisplayPort tunnel clock support for USB4/TBT On T602X (M2 Pro, j416s) Type-C PHYs, a monitor connected directly via DP alt-mode or a USB-C-to-HDMI adapter already works, because that path drives the AUSPLL through the PHY's normal per-mode configuration table. A monitor connected through a USB4/Thunderbolt hub instead carries DisplayPort as a tunneled stream over the USB4 link, and the DCP display coprocessor firmware expects the PHY driver to hand it a dedicated tunnel pixel clock on request, outside the normal mode-configure path. Nothing implements that request today, so a tunneled display never gets a clock and stays dark even though the USB4 link itself comes up. Add the tunnel clock state machine: atc_tunnel_start() brings up the fixed AUSPLL frequency descriptor and PCLK1 output for one of the four standard DisplayPort link rates, after a preflight read of the gate/output/lock registers confirms the PHY is in the specific idle state this sequence expects (refusing with -EBUSY otherwise rather than clobbering live state). Before touching anything it snapshots the registers it is about to modify; atc_tunnel_restore() replays that snapshot in reverse to undo the sequence, and is called unconditionally at the top of atcphy_configure() so any mode change, power-down, or unplug leaves the tunnel registers exactly as it found them instead of stuck mid-sequence, and clears the request-guard state so a later tunnel request is not permanently refused by a stale in-progress flag. Expose this through a new exported apple_atc_dp_tunnel_rate(phy, rate) -- the exact symbol name drivers/gpu/drm/apple/dcp.c resolves via symbol_get() to ask a PHY for a tunnel clock. It is gated to apple,j416s machines and to ATC cores that are both a t6020-atcphy and a recognized Type-C core address, so it is a correctly-typed no-op everywhere else, and only proceeds while the PHY is already in USB4 or TBT mode. Co-Authored-By: Claude Sonnet 5 --- drivers/phy/apple/atc.c | 247 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 247 insertions(+) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 2e822a3e37c3a..e4331cf72fe48 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -649,6 +649,9 @@ struct apple_atcphy { const struct atcphy_hw *hw; enum atcphy_mode mode; int dp_link_rate; + bool tunnel_attempted, tunnel_saved; + u8 tunnel_rate; + u32 tunnel_saved_regs[12]; bool swap_lanes; bool pipehandler_up; @@ -1894,12 +1897,204 @@ static int atcphy_power_on(struct apple_atcphy *atcphy) return 0; } +/* + * J416s DP tunnel clock. Native 13.5 uses a fixed AUSPLL descriptor and + * selects the requested rate at TX_DP_CTRL0. All addresses below belong + * to the existing ATC core mapping, not ACIO RC. + */ +struct atc_tunnel_saved_reg { + u32 reg; + u32 mask; +}; + +static const struct atc_tunnel_saved_reg atc_tunnel_regs[] = { + { ACIOPHY_CFG0, 0x0003ffff }, + { ACIOPHY_SLEEP_CTRL, 0x00000fff }, + { ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, 0x0000207c }, + { AUSPLL_FREQ_CFG, 0x00000003 }, + { AUSPLL_FREQ_DESC_A, 0xffffffff }, + { AUSPLL_FREQ_DESC_B, 0x0fffffff }, + { AUSPLL_FREQ_DESC_C, 0x007fffff }, + { AUSPLL_CLKOUT_DIV, 0x001f0000 }, + { AUSPLL_CLKOUT_DTC_VREG, 0x00000080 }, + { AUSPLL_BGR, 0x00000001 }, + { AUSPLL_CLKOUT_MASTER, 0x00000054 }, + { AUSPLL_APB_CMD_OVERRIDE, 0x1ffffff9 }, +}; + +static int atc_tunnel_command(struct apple_atcphy *atcphy, u32 command) +{ + u32 value; + int ret; + + core_mask32(atcphy, AUSPLL_APB_CMD_OVERRIDE, + AUSPLL_APB_CMD_OVERRIDE_CMD, + FIELD_PREP(AUSPLL_APB_CMD_OVERRIDE_CMD, command) | + AUSPLL_APB_CMD_OVERRIDE_REQ | AUSPLL_APB_CMD_OVERRIDE_UNK28); + ret = readl_poll_timeout(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE, + value, value & AUSPLL_APB_CMD_OVERRIDE_ACK, 10, 10000); + core_clear32(atcphy, AUSPLL_APB_CMD_OVERRIDE, AUSPLL_APB_CMD_OVERRIDE_REQ); + return ret; +} + +static void atc_tunnel_restore(struct apple_atcphy *atcphy) +{ + int i; + + lockdep_assert_held(&atcphy->lock); + if (!atcphy->tunnel_saved) + return; + /* Stop our output before restoring the unused PLL descriptor. */ + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + for (i = ARRAY_SIZE(atc_tunnel_regs) - 1; i >= 0; i--) + core_mask32(atcphy, atc_tunnel_regs[i].reg, atc_tunnel_regs[i].mask, + atcphy->tunnel_saved_regs[i] & atc_tunnel_regs[i].mask); + atcphy->tunnel_saved = false; + atcphy->tunnel_attempted = false; + atcphy->tunnel_rate = 0; +} + +static int atc_tunnel_start(struct apple_atcphy *atcphy, u8 rate) +{ + u32 selector, value, gates, outputs, command, status; + unsigned int i; + int ret; + + lockdep_assert_held(&atcphy->lock); + switch (rate) { + case 0x06: + selector = 4; + break; + case 0x0a: + selector = 3; + break; + case 0x14: + selector = 1; + break; + case 0x1e: + selector = 0; + break; + default: + return -EINVAL; + } + if (atcphy->tunnel_saved) { + if (atcphy->tunnel_rate != rate) + dev_info(atcphy->dev, "USB4 tunnel clock busy: active rate=0x%x requested=0x%x\n", + atcphy->tunnel_rate, rate); + return atcphy->tunnel_rate == rate ? 0 : -EBUSY; + } + if (atcphy->tunnel_attempted) + return -EALREADY; + /* Read every guard before deciding, so a refusal records the whole state. */ + gates = readl(atcphy->regs.core + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0); + outputs = readl(atcphy->regs.core + AUSPLL_CLKOUT_MASTER); + command = readl(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE); + status = readl(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT); + dev_info(atcphy->dev, + "USB4 tunnel clock preflight: +7000=%08x +2200=%08x +2000=%08x +7044=%08x\n", + gates, outputs, command, status); + /* + * Before the first rate request, TX_DP_CTRL0 gate bits read 0xe001: + * the enable gates are set, the byte-clock reset is asserted, and the + * selector and reset-release bits are clear. Native configureDPTunnelMode + * tracks tunnel clients in software rather than through these gates, so + * this exact state is the only additional gate configuration accepted + * here, together with no enabled PLL output, no outstanding command, + * and no lock. Any other gate state is refused. + */ + if (outputs & 0x54 || command & AUSPLL_APB_CMD_OVERRIDE_REQ || + status & ACIOPHY_AUSPLL_LOCK) + return -EBUSY; + if ((gates & (DPTX_PCLK1_ENABLE | DPTX_PCLK2_ENABLE | DPRX_PCLK_ENABLE)) && + gates != 0x0000e001) + return -EBUSY; + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_CMN_SHM_STS_REG0, + value, value & ACIOPHY_CMN_SHM_STS_REG0_CMD_READY, + 10, 10000); + if (ret) + return ret; + atcphy->tunnel_attempted = true; + for (i = 0; i < ARRAY_SIZE(atc_tunnel_regs); i++) + atcphy->tunnel_saved_regs[i] = readl(atcphy->regs.core + atc_tunnel_regs[i].reg); + atcphy->tunnel_saved = true; + + /* Native configureDPTunnelMode, first clock client; no lane mux writes. */ + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG_OV); + udelay(2); + core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x30); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0xc0); + udelay(2); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x03); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x0c); + udelay(2); + core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, 0x300); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0xc00); + udelay(2); + /* Additional override bit alongside RX_BIG_OV; exact meaning not independently confirmed. */ + core_set32(atcphy, ACIOPHY_CFG0, 0xc00); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG_OV); + udelay(2); + /* Additional override bit alongside RX_SMALL_OV; exact meaning not independently confirmed. */ + core_set32(atcphy, ACIOPHY_CFG0, 0xc0); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL_OV); + udelay(2); + /* Additional override bit alongside RX_CLAMP_OV; exact meaning not independently confirmed. */ + core_clear32(atcphy, ACIOPHY_CFG0, 0xc000); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(2)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(3)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, + FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + + /* Native fixed descriptor at cache VA 0xfffffe0007582418. */ + core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, 0xffffffff, + 0x21c | (8 << 14) | (3 << 18) | (8 << 22) | (7 << 26)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_B, 0x0fffffff, 0); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, 0x007fffff, + (5 << 9) | (1 << 14) | (5 << 16) | (2 << 20) | BIT(22)); + core_mask32(atcphy, AUSPLL_CLKOUT_DIV, AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI, + FIELD_PREP(AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI, 1)); + core_set32(atcphy, AUSPLL_CLKOUT_DTC_VREG, AUSPLL_DTC_VREG_BYPASS); + core_set32(atcphy, AUSPLL_BGR, AUSPLL_BGR_CTRL_AVAIL); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(2)); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(4)); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(6)); + ret = atc_tunnel_command(atcphy, 0); + if (ret) + goto restore; + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, + value, value & ACIOPHY_AUSPLL_LOCK, 10, 10000); + if (ret) + goto restore; + ret = atc_tunnel_command(atcphy, 0x2000); + if (ret) + goto restore; + atcphy->tunnel_rate = rate; + return 0; +restore: + atc_tunnel_restore(atcphy); + return ret; +} + static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) { int ret = 0; u32 reg; lockdep_assert_held(&atcphy->lock); + /* Restore before the existing mode path can power down the registers. */ + atc_tunnel_restore(atcphy); if (mode == APPLE_ATCPHY_MODE_OFF) { ret = atcphy_power_off(atcphy); @@ -2144,6 +2339,58 @@ static const struct phy_ops apple_atc_dp_phy_ops = { .set_mode = atcphy_dpphy_set_mode, }; +/* + * j416s has one ATC PHY core per Type-C port at 0x703000000 (left), + * 0xb03000000 (left), 0xf03000000 (right) -- the same top-byte pattern + * as the ACIO/DPIN0 and crossbar addresses in drivers/thunderbolt/apple.c + * and drivers/mux/apple-display-crossbar.c. Originally this only + * accepted the right port's core while this was a single-port proof + * of concept; generalized once a second port needed testing. + */ +static bool apple_atc_is_typec_core(u64 base) +{ + return base == 0x703000000ULL || base == 0xb03000000ULL || + base == 0xf03000000ULL; +} + +/* + * Ported from aurora-silicon/linux#8's apple_atc_dp_tunnel_rate() (its own + * comment: "Based on Oliver Lukschander's t6020 tunnel clock patch") -- + * renamed to the reference's cross-module symbol name so + * dcp_tunnel_set_rate() (drivers/gpu/drm/apple/dcp.c) can find it via + * symbol_get(), keeping this project's own T602X-specific implementation + * (the reference's own version hard-fails on anything but t8103's fixed + * AUSPLL descriptor/PCLK1 selectors). Also accepts TBT mode, not just + * USB4: a genuine dock may negotiate either. + */ +int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); +int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) +{ + struct apple_atcphy *atcphy; + int ret; + + if (!phy || phy->ops != &apple_atc_dp_phy_ops || + !of_machine_is_compatible("apple,j416s")) + return -EOPNOTSUPP; + atcphy = phy_get_drvdata(phy); + if (!of_device_is_compatible(atcphy->np, "apple,t6020-atcphy") || + !apple_atc_is_typec_core(atcphy->res.core->start) || + resource_size(atcphy->res.core) < 0x7048) + return -EINVAL; + guard(mutex)(&atcphy->lock); + if (!rate) { + atc_tunnel_restore(atcphy); + return 0; + } + if (atcphy->mode != APPLE_ATCPHY_MODE_USB4 && + atcphy->mode != APPLE_ATCPHY_MODE_TBT) + return -EBUSY; + ret = atc_tunnel_start(atcphy, rate); + dev_info(atcphy->dev, "DP tunnel clock: rate=0x%x result=%d\n", rate, ret); + return ret; +} +EXPORT_SYMBOL_GPL(apple_atc_dp_tunnel_rate); + static struct phy *atcphy_xlate(struct device *dev, const struct of_phandle_args *args) { struct apple_atcphy *atcphy = dev_get_drvdata(dev); From 95731b2390fc80aaee15ae676be6d31b9451a22b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 24 Sep 2026 22:28:31 -0400 Subject: [PATCH 0023/1579] arm64: preserve tested J414s display and SEP bring-up --- .../arm64/boot/dts/apple/t602x-j414-j416.dtsi | 13 + arch/arm64/configs/asahi.config | 13 + drivers/gpu/drm/apple/dcp-internal.h | 3 + drivers/gpu/drm/apple/dcp.c | 81 ++- drivers/gpu/drm/apple/iomfb_template.c | 17 +- drivers/soc/apple/Kconfig | 11 +- drivers/soc/apple/bio.rs | 27 +- drivers/soc/apple/sbio.rs | 550 ++++++++++++++---- drivers/soc/apple/sep-bio.h | 3 + drivers/soc/apple/sep.rs | 143 +++-- drivers/soc/apple/xart_apfs.rs | 99 +++- drivers/soc/apple/xart_store.rs | 26 +- tools/aurora-sep/README.md | 88 ++- tools/aurora-sep/XART-SAFETY.md | 48 +- tools/aurora-sep/apfs-xart-inspect.py | 65 ++- tools/aurora-sep/extract-mesa-calibration.py | 343 +++++++++++ tools/aurora-sep/load-driver | 8 +- .../libfprint-1.94.100-apple-sep.patch | 26 +- .../tests/test_extract_mesa_calibration.py | 72 +++ tools/aurora-sep/tests/test_xart_locator.py | 40 ++ 20 files changed, 1460 insertions(+), 216 deletions(-) create mode 100644 tools/aurora-sep/extract-mesa-calibration.py create mode 100644 tools/aurora-sep/tests/test_extract_mesa_calibration.py diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 3cff1b10057fe..97025fb0abec1 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -22,10 +22,23 @@ dcp = &dcp; dcpext0 = &dcpext0; dcpext1 = &dcpext1; + sep = &sep; disp0 = &display; disp0_piodma = &disp0_piodma; keyboard = &keyboard; sio = &sio; + usb4_0_rc = &usb4_0_acio; + usb4_0_pcie_adapter = &usb4_0_acio; + usb4_0_nhi = &usb4_0_nhi; + usb4_0_pcie = &apciec0; + usb4_1_rc = &usb4_1_acio; + usb4_1_pcie_adapter = &usb4_1_acio; + usb4_1_nhi = &usb4_1_nhi; + usb4_1_pcie = &apciec1; + usb4_2_rc = &usb4_2_acio; + usb4_2_pcie_adapter = &usb4_2_acio; + usb4_2_nhi = &usb4_2_nhi; + usb4_2_pcie = &apciec2; }; }; diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 4aa7f499997e7..c8158c632d5aa 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -7,6 +7,17 @@ CONFIG_ARM64_16K_PAGES=y CONFIG_ARM64_MEMORY_MODEL_CONTROL=y CONFIG_ARM_APPLE_CPUIDLE=y CONFIG_ARM_APPLE_SOC_CPUFREQ=m +CONFIG_ZRAM=m +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_NETFILTER=y +CONFIG_NF_TABLES=m +CONFIG_NETFILTER_XTABLES=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP6_NF_IPTABLES=m +CONFIG_BT_RFCOMM=m CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m CONFIG_PCIE_APPLE=m @@ -46,6 +57,8 @@ CONFIG_HID_DOCKCHANNEL=m CONFIG_SPI_HID_APPLE_OF=m CONFIG_SPI_HID_APPLE_CORE=m CONFIG_USB_DWC3_APPLE=m +CONFIG_USB4=m +CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_RTC_DRV_MACSMC=m CONFIG_APPLE_ADMAC=m diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 3a7732dfbfd96..1e7e8773ff1c4 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -149,6 +149,7 @@ struct apple_dcp { /* DCP has crashed */ bool crashed; + u32 crash_count; DECLARE_BITMAP(iomfb_surfaces, DCP_MAX_PLANES); @@ -285,6 +286,8 @@ struct apple_dcp { u32 nr_typec_routes; bool phy_managed_by_typec; bool typec_cable_connected; + /* CRTC is off while the physical Type-C cable may remain connected. */ + bool typec_crtc_off; struct delayed_work typec_reconnect_wq; struct delayed_work typec_fabric_retrain_wq; u32 typec_reconnect_tries; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 08156394b233f..b6c5e33b26c74 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -64,6 +64,10 @@ struct apple_dcp_typec_port { struct list_head routes; struct device_node *connector_np; struct apple_dcp_typec_route *owner; + /* Keep a port on its last DCP while that pipeline remains free. */ + struct apple_dcp_typec_route *preferred_route; + /* Ignore the USB4 fallback immediately following this port's DP teardown. */ + unsigned long dp_release_deadline; /* DRM connector for this physical port, driven by whichever DCP owns it */ struct apple_connector *connector; /* last mux state acted on, to collapse the per-candidate notifications */ @@ -119,10 +123,9 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); /* - * Pipelines are ranked by CRTC index so the fabric's choice is a pure function - * of the topology rather than of plug order. A pipeline whose fixed output is - * live is not a candidate at all, so a hybrid is only ever ranked here when it - * is genuinely free. + * For a port without a prior owner, rank pipelines by CRTC index. A pipeline + * whose fixed output is live is not a candidate at all, so a hybrid is only + * ever ranked here when it is genuinely free. */ static unsigned int dcp_typec_route_score(struct apple_dcp_typec_route *route) { @@ -327,6 +330,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (port->owner) { struct apple_dcp *dcp = port->owner->dcp; + port->preferred_route = port->owner; if (port->hpd || dcp->typec_cable_connected || (dcp->typec_connector && dcp->typec_connector->connected)) @@ -336,26 +340,43 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (ret) return ret; port->owner = NULL; + port->dp_release_deadline = jiffies + msecs_to_jiffies(10000); if (dcp->hdmi_hpd && dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd)) dcp_dptx_connect(dcp, 0); } - if (state->mode == TYPEC_MODE_USB4) - dcp_typec_retrain_active_routes(); + /* + * A port leaving DP can report SAFE/NONE before falling back to USB4. + * Resetting every other live CRTC for that same cable removal blanks + * unaffected displays. Keep the guard across the brief Type-C state + * sequence; a later, independent USB4 attach still gets recovery. + */ + if (state->mode == TYPEC_MODE_USB4) { + if (!port->dp_release_deadline || + time_after_eq(jiffies, port->dp_release_deadline)) + dcp_typec_retrain_active_routes(); + port->dp_release_deadline = 0; + } return 0; } if (!port->owner) { - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; - - if (!dcp_typec_route_available(candidate)) - continue; - score = dcp_typec_route_score(candidate); - if (score < best_score) { - best = candidate; - best_score = score; + if (port->preferred_route && + dcp_typec_route_available(port->preferred_route)) + best = port->preferred_route; + + if (!best) { + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (!dcp_typec_route_available(candidate)) + continue; + score = dcp_typec_route_score(candidate); + if (score < best_score) { + best = candidate; + best_score = score; + } } } @@ -366,6 +387,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (ret) return ret; port->owner = best; + port->dp_release_deadline = 0; } @@ -519,6 +541,8 @@ static void dcp_typec_route_unregister(void *data) typec_mux_unregister(route->typec_mux); guard(mutex)(&dcp_typec_fabric_lock); + if (port->preferred_route == route) + port->preferred_route = NULL; if (port->owner == route) { struct apple_dcp *dcp = route->dcp; @@ -851,8 +875,10 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ { struct apple_dcp *dcp = cookie; + dcp->crash_count++; dcp->crashed = true; - dev_err(dcp->dev, "DCP has crashed\n"); + dev_err(dcp->dev, "DCP has crashed (count=%u, log size=%zu)\n", + dcp->crash_count, crashlog_size); if (dcp->connector) { dcp->connector->connected = 0; drm_edid_free(dcp->connector->drm_edid); @@ -933,8 +959,12 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state) struct drm_crtc_state *crtc_state; bool needs_modeset; - if (dcp->crashed) + if (dcp->crashed) { + dev_err_ratelimited(dcp->dev, + "DCP diagnostic: rejecting CRTC %u atomic check: crash flag=%u count=%u\n", + crtc->base.id, dcp->crashed, dcp->crash_count); return -EINVAL; + } crtc_state = drm_atomic_get_new_crtc_state(state, crtc); @@ -1384,6 +1414,7 @@ void dcp_poweron(struct platform_device *pdev) int ret; if (dcp_is_typec_output(dcp)) { + WRITE_ONCE(dcp->typec_crtc_off, false); /* * A Type-C CRTC disable releases its DPTX session. Re-establish it * synchronously before IOMFB is powered back on. @@ -1426,6 +1457,15 @@ void dcp_poweroff(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + if (dcp_is_typec_output(dcp)) { + /* + * The DPTX disconnect below is part of DPMS, not a cable removal. + * Keep the DRM connector logically present so userspace can wake it. + */ + WRITE_ONCE(dcp->typec_crtc_off, true); + cancel_delayed_work(&dcp->typec_reconnect_wq); + } + if (dcp->avep) av_service_disconnect(dcp); @@ -1443,12 +1483,7 @@ void dcp_poweroff(struct platform_device *pdev) "failed to deassert Type-C DPTX HPD: %d\n", ret); dcp_dptx_disconnect(dcp, 0); - if (READ_ONCE(dcp->typec_cable_connected)) { - dcp->typec_reconnect_tries = 0; - mod_delayed_work(system_freezable_wq, - &dcp->typec_reconnect_wq, - DPTX_RECONNECT_DELAY); - } + /* dcp_poweron() reconnects the link on DPMS wake. */ } } else if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index c0b533c4f8a3c..e1450db8978a4 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -936,7 +936,11 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) swap_id = cookie->swap_id; kref_put(&cookie->refcount, release_swap_cookie); if (ret <= 0) { - dcp->crashed = true; + /* A disconnected external display can power down before it ACKs this + * clear swap. The deadline alone does not mean RTKit crashed; its + * crash callback owns the fatal flag. + */ + dev_warn(dcp->dev, "clear swap timed out during power-off\n"); return; } @@ -1026,6 +1030,17 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) if (dcp->main_display) return; + /* + * Deasserting synthetic Type-C HPD for DPMS is not a cable unplug. + * Suppress only the disconnect: a real cable can be removed while the + * CRTC is off and reinserted before it is re-enabled. Its connect event + * must reach DRM or the connector stays disconnected indefinitely. + */ + if (dcp_is_typec_output(dcp) && !(*connected) && + READ_ONCE(dcp->typec_crtc_off) && + READ_ONCE(dcp->typec_cable_connected)) + return; + if (dcp->during_modeset) { /* * Remember it rather than dropping it. Resume re-runs the diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index f80dacfdc9491..d3e5538c9d64d 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -121,10 +121,13 @@ config APPLE_SEP SEP-backed trusted key source so keyctl can seal keys to the enclave. The driver reads the existing machine-wide xART gigalocker directly from - the iBoot system container, locating it by its root records. It validates - the complete store and both root records before registering with SEP. - Raw APFS xART writes are refused until the extent and update protocol - can be verified; the module parameter cannot override this safety gate. + the iBoot system container, locating its .gl extent through read-only, + checksum-validated APFS metadata. It validates the complete store and + both root records before registering with SEP. + Raw APFS xART writes are disabled by default. An explicit xart_writes + opt-in is accepted only when the driver validates narrow physical + extent ownership invariants; this does not prove all macOS-compatible + update behavior. Keep it off until the target machine has been checked. Say Y here if you have an Apple silicon Mac. diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index e66710b6935cf..2bc1a492799c3 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -49,7 +49,9 @@ enum MatchResult { NotCompared = 2, } -pub(crate) const ENROL_STAGES: u32 = 8; +// A stage is an accepted capture, not a bucket of SEP's coverage percentage. +// Leave the final stage for the completed template. +pub(crate) const ENROL_STAGES: u32 = 16; const TOKEN_LIFETIME_NS: u64 = 10 * 1_000_000_000; @@ -180,7 +182,7 @@ ioctl_pod! { const MAGIC: u32 = 0xB1; -const IOC_GET_INFO: u32 = _IOR::(MAGIC, 0x01); +pub(crate) const IOC_GET_INFO: u32 = _IOR::(MAGIC, 0x01); const IOC_LIST: u32 = _IOR::(MAGIC, 0x02); const IOC_ENROL_START: u32 = _IOW::(MAGIC, 0x03); const IOC_ENROL_POLL: u32 = _IOR::(MAGIC, 0x04); @@ -414,6 +416,7 @@ pub(crate) struct Session { open: bool, op: Op, unseen: bool, + enrol_stage_unseen: bool, token: Option, } @@ -423,6 +426,7 @@ impl Session { open: false, op: Op::Idle, unseen: false, + enrol_stage_unseen: false, token: None, } } @@ -431,6 +435,7 @@ impl Session { self.open = false; self.op = Op::Idle; self.unseen = false; + self.enrol_stage_unseen = false; self.token = None; } @@ -443,6 +448,7 @@ pub(crate) struct Context<'a> { pub(crate) session: &'a mut Session, pub(crate) index: &'a IdentityIndex, pub(crate) sensor_present: bool, + pub(crate) live_identity_count: Option, } pub(crate) struct Handled { @@ -505,7 +511,14 @@ fn get_info(ctx: &mut Context<'_>, user: UserPtr) -> Result { enrolled: ctx.index.total() as u32, capacity: MAX_IDENTITIES as u32, enroll_stages: ENROL_STAGES, - reserved: [0; 3], + // reserved[0] is a validity flag. A zero count is only meaningful + // when SEP itself successfully answered LIST_IDENTITIES; older + // kernels and unavailable SEP contexts leave this flag clear. + reserved: [ + u32::from(ctx.live_identity_count.is_some()), + ctx.live_identity_count.unwrap_or(0) as u32, + 0, + ], }; UserSlice::new(user, core::mem::size_of::()) .writer() @@ -557,6 +570,8 @@ fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { guidance: Guidance::Place, percent: 0, }; + ctx.session.unseen = false; + ctx.session.enrol_stage_unseen = false; Ok(Handled { ret: 0, wake: false, @@ -586,6 +601,7 @@ pub(crate) fn enrol_advance( *percent = percent_now; *guidance = guide; session.unseen = true; + session.enrol_stage_unseen = true; return true; } } @@ -661,7 +677,9 @@ fn enrol_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { out.progress_percent = *percent; match terminal { None => { - out.state = if ctx.session.unseen { + // Guidance also wakes the poller, but it is not an image + // capture or a completed enrollment stage. + out.state = if ctx.session.enrol_stage_unseen { State::Progress } else { State::Pending @@ -683,6 +701,7 @@ fn enrol_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { .writer() .write(&out)?; ctx.session.unseen = false; + ctx.session.enrol_stage_unseen = false; ok() } diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 99665791198b6..91e5125b55acb 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -10,6 +10,24 @@ use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; impl SepData { + fn sbio_enrol_step(&self, op: &crate::sbio::SbioOp) -> Option> { + match self.sbio_call(op) { + SbioOutcome::Ok(payload) => Some(payload), + SbioOutcome::PrerequisiteMissing => { + dev_err!(self.dev, "enrol: {} prerequisite missing (status 0x01)\n", op.name()); + None + } + SbioOutcome::Status16 => { + dev_err!(self.dev, "enrol: {} refused (status 0x16)\n", op.name()); + None + } + SbioOutcome::Other => { + dev_err!(self.dev, "enrol: {} got no usable answer\n", op.name()); + None + } + } + } + fn sbio_expect_ok(&self, op: &crate::sbio::SbioOp) -> Option> { match self.sbio_call(op) { SbioOutcome::Ok(payload) => Some(payload), @@ -40,8 +58,14 @@ impl SepData { fn sbio_relay(&self, relay: &crate::sbio::SbioRelay<'_>) -> Option> { match self.sbio_transfer_raw(relay.opcode(), relay.name(), relay.payload()) { Ok(done) if done.status.is_ok() => Some(done.payload), - Ok(_) => None, - Err(_) => None, + Ok(done) => { + dev_err!(self.dev, "enrol: {} relay ({} bytes) refused with status {}\n", relay.name(), relay.payload().len(), done.status); + None + } + Err(e) => { + dev_err!(self.dev, "enrol: {} relay ({} bytes) failed ({:?})\n", relay.name(), relay.payload().len(), e); + None + } } } @@ -264,6 +288,7 @@ impl SepData { let listed = match self.enclave_identities_for(SBIO_PROBE_USER_ID) { Some(list) => list.len(), None => { + dev_warn!(self.dev, "enrol: identity enumeration failed; trying the existing context\n"); return self.enrol_into_existing_context(user); } }; @@ -288,10 +313,17 @@ impl SepData { proof: &crate::sbio::NoExistingCatacomb, ) -> bool { let system = crate::sbio::sbio_select_context(crate::sbio::ContextScope::SYSTEM, proof); - if self.sbio_expect_ok(&system).is_none() { + if self.sbio_enrol_step(&system).is_none() { return false; } + let Some(reply) = self.read_component_states() else { + dev_warn!(self.dev, "enrol: could not read component state after system SELECT_CONTEXT\n"); + return false; + }; + let states = crate::sbio::ComponentStates::new(&reply); + let owner_initial_state = states.state_for(crate::sbio::CatacombUser::OWNER.value()); + for (who, kind, what) in [ ( crate::sbio::CatacombUser::MASTER, @@ -304,14 +336,52 @@ impl SepData { c"owner catacomb", ), ] { + let Some(state) = states.state_for(who.value()) else { + dev_err!(self.dev, "enrol: {} is absent after system SELECT_CONTEXT\n", what); + return false; + }; + dev_info!(self.dev, "enrol: after system SELECT_CONTEXT {} state 0x{:x}\n", what, state); + if state & crate::sbio::COMPONENT_STATE_ACTIVE == 0 { + dev_err!(self.dev, "enrol: {} is not active after system SELECT_CONTEXT\n", what); + return false; + } + // On m2, state 0x3 refused SAVE_CATACOMB with status 0x6. The + // 0x04 save-pending bit is also used by the T2 Catacomb protocol; + // only persist a fresh component when SEP reports that bit. + if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING == 0 { + continue; + } if !self.save_catacomb(who, kind, what) { + dev_err!(self.dev, "enrol: could not save {} after selecting the system context\n", what); + return false; + } + } + + // The J414s owner exported successfully from state 0x3 in a bounded + // no-finger probe, although an earlier pre-UUID boot saw the master + // refuse an early save from the same state. Keep this path opt-in + // until a new enrollment and reboot prove that the owner blob belongs + // to the new user context. + if *module_parameters::j414s_persistent_enrol.value() != 0 { + if owner_initial_state == Some(0x3) + && !self.save_catacomb( + crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"fresh-context owner catacomb (user 501)", + ) + { + dev_err!(self.dev, "enrol: opt-in fresh-context owner export failed\n"); + return false; + } + if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() { + dev_err!(self.dev, "enrol: opt-in fresh context has no owner file\n"); return false; } } let per_user = crate::sbio::sbio_select_context(crate::sbio::ContextScope::user(user), proof); - if self.sbio_expect_ok(&per_user).is_none() { + if self.sbio_enrol_step(&per_user).is_none() { return false; } @@ -324,10 +394,11 @@ impl SepData { fn activate_protected_config(&self, user: crate::sbio::UserId) -> bool { let op = crate::sbio::sbio_protected_config(user); - let Some(config) = self.sbio_expect_ok(&op) else { + let Some(config) = self.sbio_enrol_step(&op) else { return false; }; if config.len() != crate::sbio::SBIO_PROTECTED_CONFIG_LEN { + dev_err!(self.dev, "enrol: protected configuration has {} bytes, expected {}\n", config.len(), crate::sbio::SBIO_PROTECTED_CONFIG_LEN); return false; } true @@ -339,19 +410,25 @@ impl SepData { let _ = self.sbio_call(&crate::sbio::sbio_cancel_operation()); } - let user = crate::sbio::UserId::new(SBIO_PROBE_USER_ID)?; + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + dev_err!(self.dev, "enrol: invalid probe user id\n"); + return None; + }; // 0x03 answers -3 until a user key bag is designated if !self.keybag_designated.load(Relaxed) { + dev_err!(self.dev, "enrol: no user keybag was designated\n"); return None; } if self.enrol_material.lock().is_none() { + dev_err!(self.dev, "enrol: no enrolment material is held\n"); return None; } // 0x03 answers 0x1 until an enrolment context exists if !self.open_enrolment_context(user) { + dev_err!(self.dev, "enrol: could not open the enclave enrolment context\n"); return None; } @@ -365,8 +442,8 @@ impl SepData { let op = crate::sbio::sbio_begin_enrol(user, crate::sbio::BE_AUTH_TYPE_ACM_CONTEXT, &acm_handle); - match self.sbio_call(&op) { - SbioOutcome::Ok(_payload) => { + match self.sbio_enrol_step(&op) { + Some(_payload) => { self.enrol_open.store(true, Relaxed); Some(OpenEnrolment { sep: self, @@ -377,6 +454,50 @@ impl SepData { } } + fn probe_owner_export(&self) { + if *module_parameters::probe_owner_export.value() == 0 { + return; + } + if self.templates_restored.load(Relaxed) + || crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_some() + || crate::catacomb::read(PRIVATE_TYPE_CATACOMB_MASTER).is_none() + || crate::catacomb::read(PRIVATE_TYPE_CATACOMB_USER).is_none() + { + dev_warn!(self.dev, "owner-probe: preconditions not met; no context selected\n"); + return; + } + let Some(identities) = self.enclave_identities_for(SBIO_PROBE_USER_ID) else { + dev_warn!(self.dev, "owner-probe: identity count unavailable; no context selected\n"); + return; + }; + let Some(proof) = crate::sbio::NoExistingCatacomb::from_zero_identities(identities.len()) else { + dev_warn!(self.dev, "owner-probe: enclave identity exists; no context selected\n"); + return; + }; + + // This is destructive to the enclave's *volatile* current context. + // It is opt-in and only reached after proving zero live identities. + let system = crate::sbio::sbio_select_context(crate::sbio::ContextScope::SYSTEM, &proof); + if self.sbio_expect_ok(&system).is_none() { + dev_warn!(self.dev, "owner-probe: system context selection refused\n"); + return; + } + let state = self.read_component_states().and_then(|reply| { + crate::sbio::ComponentStates::new(&reply) + .state_for(crate::sbio::CatacombUser::OWNER.value()) + }); + dev_info!(self.dev, "owner-probe: selected owner component state {:?}\n", state); + if !matches!(state, Some(value) if value & crate::sbio::COMPONENT_STATE_ACTIVE != 0) { + return; + } + let saved = self.save_catacomb( + crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"owner catacomb (user 501) diagnostic", + ); + dev_warn!(self.dev, "owner-probe: owner SAVE_CATACOMB and CONFIRM_SAVE completed {}\n", saved); + } + fn restore_all_components(&self) -> bool { // 0x6b is re-read before every component, not once up front. let user_id = SBIO_PROBE_USER_ID; @@ -384,27 +505,34 @@ impl SepData { let mut missing_files = false; let mut user_outcome = RestoreOutcome::NoStoredFile; - // 0x8002 (cold transition) is a success only on the owner component. - for (id, kind, what, cold_ok) in [ + // A 0x8002 reply is not sufficient evidence of restoration. Re-read + // the component state for every load, including the user component. + for (id, kind, what) in [ ( crate::sbio::CatacombUser::MASTER.value(), PRIVATE_TYPE_CATACOMB_MASTER, c"master catacomb", - false, ), ( crate::sbio::CatacombUser::OWNER.value(), PRIVATE_TYPE_CATACOMB_OWNER, c"owner catacomb", - true, ), - (user_id, PRIVATE_TYPE_CATACOMB_USER, c"user catacomb", false), + (user_id, PRIVATE_TYPE_CATACOMB_USER, c"user catacomb"), ] { let Some(reply) = self.read_component_states() else { return false; }; let states = crate::sbio::ComponentStates::new(&reply); - let outcome = self.restore_catacomb(states.state_for(id), id, kind, what, cold_ok); + let state = states.state_for(id); + let outcome = self.restore_catacomb(state, id, kind, what); + dev_info!( + self.dev, + "sbio: restore {} state {:?} outcome {:?}\n", + what, + state, + outcome + ); match outcome { RestoreOutcome::Restored | RestoreOutcome::AlreadyActive => any = true, RestoreOutcome::NoStoredFile => missing_files = true, @@ -419,7 +547,9 @@ impl SepData { user_outcome, RestoreOutcome::Restored | RestoreOutcome::AlreadyActive ); - let lockout = match self.restore_lockout() { + let lockout_outcome = self.restore_lockout(); + dev_info!(self.dev, "sbio: restore lockout outcome {:?}\n", lockout_outcome); + let lockout = match lockout_outcome { RestoreOutcome::Restored | RestoreOutcome::AlreadyActive => true, RestoreOutcome::EmptyTolerated => true, RestoreOutcome::NoStoredFile => { @@ -432,7 +562,7 @@ impl SepData { if missing_files { dev_warn!( self.dev, - "sbio: at least one of four artefacts has no file on disk; if enrolled before the four-artefact save, re-enrol once to write all four\n" + "sbio: at least one catacomb/lockout file is absent; match readiness depends on the user restore and device-view proof\n" ); } @@ -459,7 +589,6 @@ impl SepData { id: i32, kind: u8, what: &CStr, - cold_ok: bool, ) -> RestoreOutcome { let Some(state) = state else { return RestoreOutcome::Failed; @@ -490,18 +619,56 @@ impl SepData { Ok(done) if done.status.is_ok() => { self.confirm_active(id, what, LoadAnswer::StatusZero) } - Ok(done) - if done.status.answered() - == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) - && cold_ok => - { - self.confirm_active(id, what, LoadAnswer::ColdTransition) - } Ok(done) if done.status.answered() == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) => { - RestoreOutcome::Failed + let active = self.confirm_active(id, what, LoadAnswer::ColdTransition); + if id != SBIO_PROBE_USER_ID + || !matches!(&active, RestoreOutcome::Restored) + { + return active; + } + match self.log_identity_count(c"after first user cold-transition load") { + Some(n) if n > 0 => return active, + Some(0) => {} + _ => return RestoreOutcome::Failed, + } + + // Diagnostic only: one idempotent retry of the same + // host blob, with no context reset or save. An active + // state by itself did not prove a live identity on J414s. + let second = self.sbio_transfer_raw( + request.opcode(), request.name(), request.payload() + ); + let answer = match second { + Ok(done) if done.status.is_ok() => LoadAnswer::StatusZero, + Ok(done) + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) => + { + LoadAnswer::ColdTransition + } + Ok(done) + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_ALREADY_ACTIVE) => + { + LoadAnswer::AlreadyActive + } + Ok(done) => { + dev_warn!(self.dev, "sbio: second user LOAD_CATACOMB refused with status {}\n", done.status); + return RestoreOutcome::Failed; + } + Err(e) => { + dev_warn!(self.dev, "sbio: second user LOAD_CATACOMB transfer failed ({:?})\n", e); + return RestoreOutcome::Failed; + } + }; + let active = self.confirm_active(id, what, answer); + match (active, self.log_identity_count(c"after second user cold-transition load")) { + (RestoreOutcome::Restored, Some(n)) if n > 0 => RestoreOutcome::Restored, + _ => RestoreOutcome::Failed, + } } // 0x101 ALREADY_ACTIVE is a success Ok(done) @@ -510,7 +677,14 @@ impl SepData { { self.confirm_active(id, what, LoadAnswer::AlreadyActive) } - _ => RestoreOutcome::Failed, + Ok(done) => { + dev_warn!(self.dev, "sbio: {} LOAD_CATACOMB refused with status {}\n", what, done.status); + RestoreOutcome::Failed + } + Err(e) => { + dev_warn!(self.dev, "sbio: {} LOAD_CATACOMB transfer failed ({:?})\n", what, e); + RestoreOutcome::Failed + } } } } @@ -528,7 +702,9 @@ impl SepData { }; let states = crate::sbio::ComponentStates::new(&reply); - match states.state_for(id) { + let state = states.state_for(id); + dev_info!(self.dev, "sbio: {} LOAD_CATACOMB answered {} with post-load state {:?}\n", what, answer.describe(), state); + match state { Some(state) if state & crate::sbio::COMPONENT_STATE_ACTIVE != 0 => { RestoreOutcome::Restored } @@ -575,7 +751,14 @@ impl SepData { } fn save_all_components(&self, user: crate::sbio::UserId) -> bool { - let mut all = true; + if *module_parameters::j414s_persistent_enrol.value() != 0 { + return self.save_all_components_user_first(user); + } + // On m2fix11, a completed enrollment marked both master and user 0x7, + // but saving and confirming the user first was followed by master + // SAVE_CATACOMB status 0x6. Try the master before the user at this + // completion gate; post-match persistence keeps its separate order. + // Re-read state before each save and never send an unconditional one. for (who, kind, what) in [ ( crate::sbio::CatacombUser::MASTER, @@ -593,47 +776,144 @@ impl SepData { c"user catacomb", ), ] { - if !self.save_catacomb(who, kind, what) { - all = false; + let Some(reply) = self.read_component_states() else { + dev_err!(self.dev, "enrol: cannot read component state before saving {}\n", what); + return false; + }; + let states = crate::sbio::ComponentStates::new(&reply); + let Some(state) = states.state_for(who.value()) else { + dev_err!(self.dev, "enrol: {} is absent at enrollment completion\n", what); + return false; + }; + dev_info!(self.dev, "enrol: completed {} state 0x{:x}\n", what, state); + if state & crate::sbio::COMPONENT_STATE_ACTIVE == 0 { + dev_err!(self.dev, "enrol: {} is not active at enrollment completion\n", what); + return false; + } + if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING != 0 { + if !self.save_catacomb(who, kind, what) { + return false; + } + } else if who.value() == user.value() { + dev_err!(self.dev, "enrol: completed user catacomb is not marked for saving\n"); + return false; + } + if (who.value() == user.value() + || who.value() == crate::sbio::CatacombUser::MASTER.value()) + && crate::catacomb::read(kind).is_none() + { + dev_err!(self.dev, "enrol: {} has no durable host file\n", what); + return false; } } if !self.save_lockout() { - all = false; + dev_err!(self.dev, "enrol: lockout persistence failed\n"); + return false; } - all + true } - fn save_after_match(&self, user: crate::sbio::UserId) -> bool { + fn save_all_components_user_first(&self, user: crate::sbio::UserId) -> bool { + if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() { + dev_err!(self.dev, "enrol: opt-in completion has no owner Catacomb file\n"); + return false; + } + // Post-match persistence saves the user before the master because the + // user save can change system material. Test that order for a completed + // J414s enrollment, without changing the default path on other Macs. + for (who, kind, what) in [ + ( + crate::sbio::CatacombUser::enrolling(user), + PRIVATE_TYPE_CATACOMB_USER, + c"opt-in completed user catacomb", + ), + ( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"opt-in post-user master catacomb", + ), + ] { + let Some(reply) = self.read_component_states() else { + dev_err!(self.dev, "enrol: opt-in cannot read state before saving {}\n", what); + return false; + }; + let state = crate::sbio::ComponentStates::new(&reply).state_for(who.value()); + dev_info!(self.dev, "enrol: opt-in {} state {:?}\n", what, state); + let Some(state) = state else { + return false; + }; + if state & crate::sbio::COMPONENT_STATE_ACTIVE == 0 { + return false; + } + if who.value() == user.value() + && state & crate::sbio::COMPONENT_STATE_SAVE_PENDING == 0 + { + dev_err!(self.dev, "enrol: opt-in user catacomb is not save-pending\n"); + return false; + } + if !self.save_catacomb(who, kind, what) { + return false; + } + if crate::catacomb::read(kind).is_none() { + dev_err!(self.dev, "enrol: opt-in {} has no durable host file\n", what); + return false; + } + } if !self.save_lockout() { - dev_err!( - self.dev, - "verify: post-match lockout persistence failed; refusing to claim the AP and SEP anti-replay state are synchronized\n" - ); + dev_err!(self.dev, "enrol: opt-in lockout persistence failed\n"); return false; } + true + } - if !self.save_catacomb( - crate::sbio::CatacombUser::enrolling(user), - PRIVATE_TYPE_CATACOMB_USER, - c"post-match user catacomb", - ) { + fn save_after_match(&self, user: crate::sbio::UserId) -> bool { + if !self.save_lockout() { dev_err!( self.dev, - "verify: post-match user-catacomb persistence failed; the next boot may reject the older AP blob\n" + "verify: post-match lockout persistence failed; refusing to claim the AP and SEP anti-replay state are synchronized\n" ); return false; } - if !self.save_catacomb( - crate::sbio::CatacombUser::MASTER, - PRIVATE_TYPE_CATACOMB_MASTER, - c"post-match system (id -1) material snapshot", - ) { - dev_err!( - self.dev, - "verify: post-match system material snapshot failed; next boot may reinstall pre-match material and the user load could cold-transition (0x8002)\n" - ); - return false; + // Matching can update a template without making every component + // saveable. The user must be considered before the master, and 0x6b + // must be re-read after each save because saving the user can change + // the master's state. Never claim persistence without host copies. + for (who, kind, what) in [ + ( + crate::sbio::CatacombUser::enrolling(user), + PRIVATE_TYPE_CATACOMB_USER, + c"post-match user catacomb", + ), + ( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"post-match master catacomb", + ), + ] { + let Some(reply) = self.read_component_states() else { + dev_err!(self.dev, "verify: cannot read state before saving {}\n", what); + return false; + }; + let states = crate::sbio::ComponentStates::new(&reply); + let Some(state) = states.state_for(who.value()) else { + dev_err!(self.dev, "verify: {} is absent after matching\n", what); + return false; + }; + if state & crate::sbio::COMPONENT_STATE_ACTIVE == 0 { + dev_err!(self.dev, "verify: {} is not active after matching (state 0x{:x})\n", what, state); + return false; + } + if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING != 0 + && !self.save_catacomb(who, kind, what) + { + dev_err!(self.dev, "verify: {} persistence failed\n", what); + return false; + } + if crate::catacomb::read(kind).is_none() { + dev_err!(self.dev, "verify: {} has no durable host file\n", what); + return false; + } } if !self.resnapshot_identity_keybag() { @@ -647,33 +927,49 @@ impl SepData { true } - fn save_catacomb(&self, who: crate::sbio::CatacombUser, kind: u8, _what: &CStr) -> bool { + fn save_catacomb(&self, who: crate::sbio::CatacombUser, kind: u8, what: &CStr) -> bool { let selector = crate::sbio::SaveSelector::new(who); - let Some(blob) = self.sbio_expect_ok(&crate::sbio::sbio_save_catacomb(&selector)) else { - return false; + let save = match self.sbio_transfer(&crate::sbio::sbio_save_catacomb(&selector)) { + Ok(done) if done.status.is_ok() => done, + Ok(done) => { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB refused with status {}\n", what, done.status); + return false; + } + Err(e) => { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB transfer failed ({:?})\n", what, e); + return false; + } }; + let blob = save.payload; if blob.len() < crate::sbio::SBIO_SAVED_MIN || blob.len() > crate::sbio::SBIO_SAVED_MAX { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB returned invalid length {}\n", what, blob.len()); return false; } let at = crate::sbio::SBIO_SAVED_USER_ID_AT; let carried = i32::from_le_bytes([blob[at], blob[at + 1], blob[at + 2], blob[at + 3]]); if carried != who.value() { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB returned user id {}, expected {}\n", what, carried, who.value()); return false; } if crate::catacomb::write(kind, &blob).is_err() { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB host persistence failed\n", what); return false; } - if self - .sbio_expect_ok(&crate::sbio::sbio_confirm_save(&selector)) - .is_none() - { - return false; + match self.sbio_transfer(&crate::sbio::sbio_confirm_save(&selector)) { + Ok(done) if done.status.is_ok() => true, + Ok(done) => { + dev_err!(self.dev, "enrol: {} CONFIRM_SAVE refused with status {}\n", what, done.status); + false + } + Err(e) => { + dev_err!(self.dev, "enrol: {} CONFIRM_SAVE transfer failed ({:?})\n", what, e); + false + } } - true } fn save_lockout(&self) -> bool { @@ -855,8 +1151,9 @@ impl SepData { self.sks_designate_user_keybag(handle, stored.secret()); self.sks_machine_refkey(handle, stored.secret()); let prepared = self.cold_match_continue(handle, uuid); - self.ensure_restored_after(prepared); self.attach_bringup(); + self.ensure_restored_after(prepared); + self.probe_owner_export(); self.touchid_started.store(true, Relaxed); Ok(()) } @@ -878,7 +1175,7 @@ impl SepData { if !self.templates_restored.load(Relaxed) { dev_err!( self.dev, - "verify: refusing before touching the sensor; cold-match prep or restore did not complete, enclave holds no template (every match refused 0x1). A restore failure, not a non-matching finger\n" + "verify: refusing before touching the sensor; cold-match prep, restore, or device-view identity proof did not complete. This is not a non-matching finger\n" ); self.finish_verify( bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), @@ -1031,6 +1328,8 @@ impl SepData { } pub(crate) fn run_enrolment(&self) { + self.enrol_frames_accepted.store(0, Relaxed); + if !self.bring_sensor_online() { self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); let _ = sensor::idle(); @@ -1038,6 +1337,7 @@ impl SepData { } let Some(open) = self.begin_enrolment_on_enclave() else { + dev_err!(self.dev, "enrol: the enclave would not start an enrolment; see the preceding step\n"); self.finish_enrolment(Err(ENROL_STATUS_ENCLAVE)); let _ = sensor::idle(); return; @@ -1045,6 +1345,7 @@ impl SepData { let mut counter: u32 = 0; let mut enrolment_completed = false; + let mut enrolment_persisted = false; let mut identity: Option<[u8; bio::UUID_LEN]> = None; let outcome = loop { @@ -1104,18 +1405,18 @@ impl SepData { if let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) { if !self.save_all_components(user) { - dev_warn!( + dev_err!( + self.dev, + "enrol: enclave completed but Catacomb persistence failed; not reporting enrollment success\n" + ); + } else if !self.resnapshot_identity_keybag() { + dev_err!( self.dev, - "enrol: enrolment succeeded but at least one of four artefacts was not persisted; works until the next reboot (see which component above)\n" + "enrol: Catacombs saved but identity-bag snapshot failed; not reporting enrollment success\n" ); } else { self.templates_restored.store(true, Relaxed); - if !self.resnapshot_identity_keybag() { - dev_warn!( - self.dev, - "enrol: catacombs saved but post-commit identity-bag snapshot was not; template works this boot but is not reboot-persistent\n" - ); - } + enrolment_persisted = true; } } @@ -1129,10 +1430,10 @@ impl SepData { self.note_capture_end(); if let Some(result) = outcome { - let filed = match (result, identity) { - (Ok(()), Some(uuid)) => Ok(uuid), - (Ok(()), None) => Err(ENROL_STATUS_UNFILED), - (Err(status), _) => Err(status), + let filed = match (result, identity, enrolment_persisted) { + (Ok(()), Some(uuid), true) => Ok(uuid), + (Ok(()), _, _) => Err(ENROL_STATUS_UNFILED), + (Err(status), _, _) => Err(status), }; self.finish_enrolment(filed); } @@ -1140,14 +1441,17 @@ impl SepData { fn register_sensor(&self, id: &sensor::Identifier) -> bool { let stage = self.bringup.load(Relaxed); + dev_info!(self.dev, "sbio: registering sensor at bring-up stage {}\n", stage); if stage >= BRINGUP_ESTABLISHED { + let _ = self.log_identity_count(c"before CLEAR_STATE for sensor re-registration"); if self .sbio_expect_ok(&crate::sbio::sbio_clear_state()) .is_none() { return false; } + let _ = self.log_identity_count(c"after CLEAR_STATE for sensor re-registration"); } if stage == BRINGUP_FRESH { @@ -1173,6 +1477,7 @@ impl SepData { match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); + let _ = self.log_identity_count(c"after sensor serial registration"); true } _ => false, @@ -1330,6 +1635,7 @@ impl SepData { if listed && !self.device_view_synced.xchg(true, Relaxed) { let count = self.log_identity_count(c"after device-view synchronisation"); + dev_info!(self.dev, "sbio: device-view identity count {:?}, restore-ready before proof {}\n", count, self.templates_restored.load(Relaxed)); match count { Some(0) => { self.templates_restored.store(false, Relaxed); @@ -1348,9 +1654,12 @@ impl SepData { self.templates_restored.store(false, Relaxed); } } + dev_info!(self.dev, "sbio: restore-ready after device-view proof {}\n", self.templates_restored.load(Relaxed)); if self.templates_restored.load(Relaxed) { self.reconcile_identities(); } + } else if !listed { + dev_warn!(self.dev, "sbio: device-view synchronisation or match policy unavailable\n"); } true @@ -1613,10 +1922,11 @@ impl SepData { }; let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + dev_err!(self.dev, "enrol: invalid user id before image processing\n"); return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; if self - .sbio_expect_ok(&crate::sbio::sbio_prepare_image_processing()) + .sbio_enrol_step(&crate::sbio::sbio_prepare_image_processing()) .is_none() { return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); @@ -1631,7 +1941,7 @@ impl SepData { user, crate::shim::monotonic_ns(), ); - if self.sbio_expect_ok(&init).is_none() { + if self.sbio_enrol_step(&init).is_none() { return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); } @@ -1641,10 +1951,11 @@ impl SepData { } drop(capture); - let Some(assessment) = self.sbio_expect_ok(&crate::sbio::sbio_image_assessment()) else { + let Some(assessment) = self.sbio_enrol_step(&crate::sbio::sbio_image_assessment()) else { return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; if assessment.len() < crate::sbio::ASSESS_MIN_LEN { + dev_err!(self.dev, "enrol: image assessment short ({} bytes)\n", assessment.len()); return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); } let usable = assessment[crate::sbio::ASSESS_USABLE_ENROL] != 0; @@ -1652,10 +1963,11 @@ impl SepData { return ImageOutcome::Retry; } - let Some(result) = self.sbio_expect_ok(&crate::sbio::sbio_enrolment_result()) else { + let Some(result) = self.sbio_enrol_step(&crate::sbio::sbio_enrolment_result()) else { return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; let Some(parsed) = crate::sbio::EnrolmentResult::parse(&result) else { + dev_err!(self.dev, "enrol: malformed enrolment result ({} bytes)\n", result.len()); return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; @@ -1664,11 +1976,17 @@ impl SepData { } let percent = parsed.progress_percent(); + // Coverage can stop changing while SEP continues accepting frames. + // Count accepted captures so each one remains visible to libfprint. + let accepted = self.enrol_frames_accepted.fetch_add(1, Relaxed) + 1; + let stage = if parsed.complete() { + bio::ENROL_STAGES + } else { + accepted.min(bio::ENROL_STAGES - 1) + }; ImageOutcome::Progress { - stage: (percent * bio::ENROL_STAGES) - .div_ceil(100) - .min(bio::ENROL_STAGES), + stage, percent, complete: parsed.complete(), has_template: parsed.has_template(), @@ -2082,6 +2400,16 @@ impl SepData { if cmd == bio::IOC_ATTEST { return self.bio_attest(arg); } + // Query SEP outside the bio session/index locks. A host index can + // outlive its SEP identity after a failed cold restore; userspace + // must not mistake that index for proof of a live template. + let live_identity_count = if cmd == bio::IOC_GET_INFO + && self.touchid_started.load(Relaxed) + { + self.enclave_identity_count() + } else { + None + }; let handled = { let mut session = self.bio_session.lock(); let index = self.bio_index.lock(); @@ -2089,6 +2417,7 @@ impl SepData { session: &mut session, index: &index, sensor_present: self.sensor_present.load(Relaxed), + live_identity_count, }; bio::ioctl(&mut ctx, cmd, arg)? }; @@ -2146,32 +2475,42 @@ impl SepData { }) } - fn ensure_restored_after(&self, _prepared: bool) { - let restored = self.restore_all_components(); - self.templates_restored.store(restored, Relaxed); - if !restored { - dev_err!( - self.dev, - "matching unavailable this boot: restore did not complete, enclave holds no template (every match refused 0x1). Not the sensor or the finger; on-disk enrolments intact\n" - ); - return; - } - + fn ensure_restored_after(&self, prepared: bool) { + // Probe credential ordering without assuming it fixes cold restore. + // J414s still answered 0x8002 and listed zero identities with this + // credential established, so the identity proof below remains required. + let mut unlocked = false; if self.keybag_designated.load(Relaxed) { if let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) { let special = user.special_handle(); if let Ok(keybag::State::Present(stored)) = keybag::read(keybag::Slot::Identity) { - let _ = self.sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { + unlocked = self.sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { self.sks_req_unlock_special(special, stored.secret(), healthy) - }); + }).is_some(); } } } + let match_context = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) + .map(|user| self.establish_scrd_match_context(user)) + .unwrap_or(false); + dev_info!(self.dev, "sbio: pre-restore special unlock {}, SCRD match context {}\n", unlocked, match_context); - // enclave's match arm asserts a non-empty ACM context (<= 32 bytes) - if let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) { - self.establish_scrd_match_context(user); + let restored = self.restore_all_components(); + dev_info!(self.dev, "sbio: cold-match preparation {}, component restore {}\n", prepared, restored); + let identities = self.log_identity_count(c"after Catacomb restore with sensor registered"); + let identity_ready = matches!(identities, Some(n) if n > 0); + let ready = prepared && match_context && restored && identity_ready + && self.device_view_synced.load(Relaxed) && self.prove_restore(); + self.templates_restored.store(ready, Relaxed); + if !ready { + dev_err!( + self.dev, + "matching unavailable this boot: cold-match preparation, SCRD credential, Catacomb restore, device view, or identity proof did not complete; template state is unproven\n" + ); + return; } + self.reconcile_identities(); + } fn cold_match_continue( @@ -2260,7 +2599,7 @@ impl SepData { true } - fn log_identity_count(&self, _when: &CStr) -> Option { + fn enclave_identity_count(&self) -> Option { let op = crate::sbio::sbio_list_identities(); let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { return None; @@ -2269,6 +2608,15 @@ impl SepData { Some(records.count()) } + fn log_identity_count(&self, when: &CStr) -> Option { + let Some(count) = self.enclave_identity_count() else { + dev_warn!(self.dev, "sbio: identity list unavailable {}\n", when); + return None; + }; + dev_info!(self.dev, "sbio: identity count {} {}\n", count, when); + Some(count) + } + fn enclave_lists(&self, uuid: &[u8; bio::UUID_LEN]) -> Option { let op = crate::sbio::sbio_list_identities(); let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { @@ -2728,7 +3076,9 @@ static_assert!(SBIO_STATUS_ALREADY_ACTIVE != SBIO_STATUS_COLD_TRANSITION); pub(crate) const COMPONENT_STATE_COLD: u32 = 0x1; pub(crate) const COMPONENT_STATE_ACTIVE: u32 = 0x2; +pub(crate) const COMPONENT_STATE_SAVE_PENDING: u32 = 0x4; static_assert!(COMPONENT_STATE_COLD != COMPONENT_STATE_ACTIVE); +static_assert!(COMPONENT_STATE_SAVE_PENDING & (COMPONENT_STATE_COLD | COMPONENT_STATE_ACTIVE) == 0); pub(crate) const COMPONENT_PAIR_LEN: usize = 8; diff --git a/drivers/soc/apple/sep-bio.h b/drivers/soc/apple/sep-bio.h index 7ad0b814af648..2ec3949eaa665 100644 --- a/drivers/soc/apple/sep-bio.h +++ b/drivers/soc/apple/sep-bio.h @@ -53,6 +53,9 @@ struct sep_bio_info { __u32 reserved[3]; }; +/* reserved[0] flags; reserved[1] is a live SEP identity count only when valid. */ +#define SEP_BIO_INFO_LIVE_COUNT_VALID 1u + struct sep_bio_list { __u32 count; __u32 reserved; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 5f128c7fa005d..63e3654a290cf 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -78,6 +78,7 @@ use kernel::{ const SETTLE_MS: time::Msecs = 200; const FIRST_RESPONSE_MS: time::Msecs = 3000; +const RNG_SURVEY_ATTEMPTS: u32 = 5; const SETTLE_WORK_ID: u64 = 1; @@ -333,6 +334,7 @@ impl LoadAnswer { } } +#[derive(Debug)] enum RestoreOutcome { Restored, AlreadyActive, @@ -642,10 +644,14 @@ struct SepData { #[pin] fv_volumes: Mutex>, + enrol_frames_accepted: Atomic, + rng_shutdown: Atomic, rng_failures: Atomic, + rng_survey_attempts: Atomic, + rx: rxring::RxRing, rx_count: Atomic, @@ -727,7 +733,7 @@ impl SepData { let base = store.base(); dev_info!( dev, - "xART: APFS-resolved read-only .gl extent (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", + "xART: APFS-resolved .gl extent (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", base, slots, records, @@ -811,8 +817,10 @@ impl SepData { rng <- new_mutex!(None), machine_refkey <- new_mutex!(None), fv_volumes <- new_mutex!(KVec::new()), + enrol_frames_accepted: Atomic::new(0), rng_shutdown: Atomic::new(false), rng_failures: Atomic::new(0), + rng_survey_attempts: Atomic::new(0), rx: rxring::RxRing::new(), rx_count: Atomic::new(0), settle_mark: Atomic::new(0), @@ -1093,40 +1101,42 @@ impl SepData { } fn control_survey(&self) { - for param in [0x00u8, 0x01, 0xff] { let _ = self.control_request(&proto::op_nop(param)); } let _ = self.control_request(&proto::op_security_mode()); - let mut words = [0u32; 4]; - let mut got = 0; - for w in words.iter_mut() { - match self.get_entropy_word() { - Ok(v) => { - *w = v; - got += 1; - } - Err(_) => { - break; - } - } + let _state = self.control.lock(); + } + + fn survey_hwrng(&self) -> Result<()> { + // A working control endpoint must answer four consecutive draws before + // it is exposed to the kernel RNG core. Early attach can beat the SEP + // endpoint exchange, so this runs only after that exchange settles. + for _ in 0..4 { + let _ = self.get_entropy_word()?; } + self.register_hwrng() + } - if got == words.len() { - match self.register_hwrng() { - Ok(()) => {}, - Err(e) => dev_err!(self.dev, "hwrng: registration failed: {:?}\n", e), + fn tick_ready(this: &Arc) { + if this.rng.lock().is_some() { + return; + } + let attempt = this.rng_survey_attempts.load(Relaxed) + 1; + if attempt > RNG_SURVEY_ATTEMPTS { + return; + } + this.rng_survey_attempts.store(attempt, Relaxed); + match this.survey_hwrng() { + Ok(()) => dev_info!(this.dev, "hwrng: registered after ready-phase survey (attempt {})\n", attempt), + Err(e) if attempt < RNG_SURVEY_ATTEMPTS => { + dev_warn!(this.dev, "hwrng: survey attempt {} failed ({:?}); retrying\n", attempt, e); + Self::arm_settle(this); } - } else { - dev_err!( - self.dev, - "hwrng: not registering, the source did not answer during the survey\n" - ); + Err(e) => dev_err!(this.dev, "hwrng: survey failed after {} attempts ({:?})\n", attempt, e), } - - let _state = self.control.lock(); } @@ -1310,6 +1320,17 @@ impl SepData { return; }; + if *module_parameters::xarm_trace.value() != 0 { + dev_info!( + self.dev, + "xarm: op 0x{:02x} request_len {} reply_status 0x{:02x} reply_len {}\n", + req.opcode, + req.length, + done.reply.status, + done.reply.length + ); + } + if done.reply_bytes > 0 { if self.ool_write(&self.ool_xarm, 0, &staging[..done.reply_bytes]).is_err() { self.fail_xarm(req.tag); @@ -1551,6 +1572,7 @@ impl SepData { user: crate::sbio::UserId, ) -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { if !self.scrd_ready() { + dev_err!(self.dev, "enrol: SCRD endpoint is not ready\n"); return None; } @@ -1562,54 +1584,74 @@ impl SepData { (material.special, Secret(copy)) }; - let (status, _) = self.scrd_command(&crate::scrd::scrd_initialize())?; + let Some((status, _)) = self.scrd_command(&crate::scrd::scrd_initialize()) else { + dev_err!(self.dev, "enrol: SCRD initialize did not answer\n"); + return None; + }; if status != 0 { + dev_err!(self.dev, "enrol: SCRD initialize refused (status {})\n", status); return None; } - let (status, out) = self.scrd_command(&crate::scrd::scrd_context_create_tracked(user.value()))?; + let Some((status, out)) = self.scrd_command(&crate::scrd::scrd_context_create_tracked(user.value())) else { + dev_err!(self.dev, "enrol: SCRD context create did not answer\n"); + return None; + }; if status != 0 || out.len() < crate::scrd::SCRD_ACM_HANDLE_LEN { + dev_err!(self.dev, "enrol: SCRD context create failed (status {}, {} reply bytes)\n", status, out.len()); return None; } let mut acm_handle = [0u8; crate::scrd::SCRD_ACM_HANDLE_LEN]; acm_handle.copy_from_slice(&out[..crate::scrd::SCRD_ACM_HANDLE_LEN]); - let (status, _) = self.scrd_command(&crate::scrd::scrd_context_externalize(&acm_handle))?; + let Some((status, _)) = self.scrd_command(&crate::scrd::scrd_context_externalize(&acm_handle)) else { + dev_err!(self.dev, "enrol: SCRD context externalize did not answer\n"); + return None; + }; if status != 0 { + dev_err!(self.dev, "enrol: SCRD context externalize refused (status {})\n", status); return None; } - let out = self.sks_send(self.sks_req_verify_secret(special, &secret, &acm_handle))?; + let Some(out) = self.sks_send(self.sks_req_verify_secret(special, &secret, &acm_handle)) else { + dev_err!(self.dev, "enrol: SKS verify-secret did not answer\n"); + return None; + }; if out.reply.status != 0 { + dev_err!(self.dev, "enrol: SKS verify-secret refused (status {})\n", out.reply.status); return None; } - let (status, out) = self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle))?; + let Some((status, out)) = self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle)) else { + dev_err!(self.dev, "enrol: SCRD Touch ID policy check did not answer\n"); + return None; + }; let satisfied = out.len() >= 4 && u32::from_le_bytes([out[0], out[1], out[2], out[3]]) != 0; if status != 0 || !satisfied { + dev_err!(self.dev, "enrol: SCRD Touch ID policy rejected (status {}, satisfied {})\n", status, satisfied); return None; } Some(acm_handle) } - fn establish_scrd_match_context(&self, user: crate::sbio::UserId) { + fn establish_scrd_match_context(&self, user: crate::sbio::UserId) -> bool { if !self.scrd_ready() { - return; + return false; } let Some(du) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) else { - return; + return false; }; let special = du.special_handle(); let stored = match keybag::read(keybag::Slot::Identity) { Ok(keybag::State::Present(s)) => s, _ => { - return; + return false; } }; - let _ = (|| -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { + let established = (|| -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { let (status, _) = self.scrd_command(&crate::scrd::scrd_initialize())?; if status != 0 { return None; @@ -1631,7 +1673,9 @@ impl SepData { } self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle))?; Some(acm_handle) - })(); + })().is_some(); + dev_info!(self.dev, "scrd: cold-match credential established {}\n", established); + established } @@ -1730,7 +1774,8 @@ impl SepData { match this.phase.load(Relaxed) { PHASE_ATTACH => Self::tick_attach(this), PHASE_EXCHANGE => Self::tick_exchange(this), - _ => {} + PHASE_READY => Self::tick_ready(this), + _ => {}, } } @@ -1807,8 +1852,9 @@ impl SepData { } } - this.phase.store(PHASE_READY, Relaxed); this.run_bringup(); + this.phase.store(PHASE_READY, Relaxed); + Self::arm_settle(this); } fn endpoint_present(&self, id: u8) -> bool { @@ -2093,13 +2139,6 @@ impl platform::Driver for SepDriver { _info: Option<&()>, ) -> impl PinInit { let dev: &device::Device = pdev.as_ref(); - if *module_parameters::xart_writes.value() != 0 { - dev_err!( - dev, - "raw APFS xART writes are blocked until the .gl update protocol and ownership invariants are validated\n" - ); - return Err(ENOTSUPP); - } if *module_parameters::provision_keybag.value() != 0 && *module_parameters::xart_writes.value() == 0 { @@ -2191,7 +2230,7 @@ module! { params: { xart_writes: u8 { default: 0, - description: "Request shared xART writes (refused until the macOS update protocol is verified)", + description: "Opt in to shared xART writes only if APFS proves a single unsnapshotted .gl extent", }, xart_start_sector: u64 { default: 0, @@ -2209,5 +2248,17 @@ module! { default: 0, description: "Low 64 bits of an explicit xART OS UUID", }, + xarm_trace: u8 { + default: 0, + description: "Opt-in xART opcode/status trace without payloads or key material", + }, + probe_owner_export: u8 { + default: 0, + description: "Opt-in J414s diagnostic: with zero live identities, select the system context and try saving the missing owner Catacomb", + }, + j414s_persistent_enrol: u8 { + default: 0, + description: "Opt-in J414s enrollment test: export owner from the fresh context, then save user before master at completion", + }, }, } diff --git a/drivers/soc/apple/xart_apfs.rs b/drivers/soc/apple/xart_apfs.rs index 9eb833baca82d..1caca4aa1871c 100644 --- a/drivers/soc/apple/xart_apfs.rs +++ b/drivers/soc/apple/xart_apfs.rs @@ -15,6 +15,8 @@ const ROOT_INFO_SIZE: usize = 40; const XART_ROLE: u16 = 0x100; const FILE_SIZE: u64 = 0x600000; const FS_UNENCRYPTED: u64 = 1; +const INODE_WAS_CLONED: u64 = 0x10; +const INODE_WAS_EVER_CLONED: u64 = 0x400; const MOD: u64 = 0xffff_ffff; fn le16(data: &[u8], off: usize) -> Result { @@ -209,7 +211,74 @@ fn omap_lookup( Ok(paddr) } -fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Result { +fn is_gigalocker_name(name: &[u8]) -> bool { + if name == b".gl" { + return true; + } + if name.len() != 39 || &name[36..] != b".gl" { + return false; + } + name[..36].iter().enumerate().all(|(i, byte)| { + if matches!(i, 8 | 13 | 18 | 23) { + *byte == b'-' + } else { + byte.is_ascii_hexdigit() + } + }) +} + +fn validate_write_ownership( + file: &shim::StoreFile, + size: u64, + vol: &[u8], + xid: u64, + private_id: u64, + inode_flags: u64, + physical: u64, +) -> Result<()> { + // This is deliberately narrower than general APFS write support. On the + // tested J414s the .gl file has one unshared, unsnapshotted extent. Never + // raw-write a clone, snapshot, pending revert, or unknown extent layout. + if le64(vol, 264)? != FS_UNENCRYPTED + || le64(vol, 216)? != 0 + || le64(vol, 160)? != 0 + || le64(vol, 168)? != 0 + || inode_flags & (INODE_WAS_CLONED | INODE_WAS_EVER_CLONED) != 0 + { + return Err(ENOTSUPP); + } + let tree_paddr = le64(vol, 144)?; + let tree = read_object(file, size, tree_paddr)?; + if le64(&tree, 8)? != tree_paddr + || le64(&tree, 16)? > xid + || le32(&tree, 24)? != 0x4000_0002 + || le32(&tree, 28)? != 0x0f + || le16(&tree, 32)? != 3 + || le16(&tree, 34)? != 0 + || le32(&tree, 36)? != 1 + { + return Err(ENOTSUPP); + } + let (key, val) = node_entry(&tree, 0, false)?; + if key.len() != 8 + || val.len() < 20 + || le64(key, 0)? != ((2u64 << 60) | physical) + || le64(val, 0)? != ((1u64 << 60) | (FILE_SIZE / BLOCK as u64)) + || le64(val, 8)? != private_id + || le32(val, 16)? != 1 + { + return Err(ENOTSUPP); + } + Ok(()) +} + +fn xart_extent( + file: &shim::StoreFile, + size: u64, + vol: &[u8], + xid: u64, + writes_enabled: bool, +) -> Result { if le64(vol, 264)? & FS_UNENCRYPTED == 0 { return Err(ENOTSUPP); } @@ -232,7 +301,7 @@ fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Resul if name_len == 0 || 12 + name_len > key.len() || key[12 + name_len - 1] != 0 { return Err(EINVAL); } - if &key[12..12 + name_len - 1] == b".gl" { + if is_gigalocker_name(&key[12..12 + name_len - 1]) { if gl_file.replace(le64(val, 0)?).is_some() { return Err(ENOTSUPP); } @@ -240,12 +309,14 @@ fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Resul } let gl_file = gl_file.ok_or(ENODATA)?; let mut private_id: Option = None; + let mut inode_flags: Option = None; for i in 0..count { let (key, val) = node_entry(&root, i, false)?; if key.len() >= 8 && le64(key, 0)? == ((3u64 << 60) | gl_file) { if val.len() < 92 || private_id.replace(le64(val, 8)?).is_some() { return Err(EINVAL); } + inode_flags = Some(le64(val, 48)?); } } let private_id = private_id.ok_or(ENODATA)?; @@ -265,11 +336,23 @@ fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Resul return Err(EINVAL); } } - extent.ok_or(ENODATA) + let extent = extent.ok_or(ENODATA)?; + if writes_enabled { + validate_write_ownership( + file, + size, + vol, + xid, + private_id, + inode_flags.ok_or(ENODATA)?, + extent / BLOCK as u64, + )?; + } + Ok(extent) } /// Return the byte offset of the sole contiguous 6 MiB `.gl` file extent. -pub(crate) fn locate(file: &shim::StoreFile) -> Result { +pub(crate) fn locate(file: &shim::StoreFile, writes_enabled: bool) -> Result { let device_size = file.size()?; let base = read_object(file, device_size, 0)?; if &base[32..36] != b"NXSB" || le32(&base, 36)? != BLOCK as u32 { @@ -331,7 +414,13 @@ pub(crate) fn locate(file: &shim::StoreFile) -> Result { } if le16(&vol, 964)? == XART_ROLE { if found - .replace(xart_extent(file, size, &vol, le64(&vol, 16)?)?) + .replace(xart_extent( + file, + size, + &vol, + le64(&vol, 16)?, + writes_enabled, + )?) .is_some() { return Err(ENOTSUPP); diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index 7c73107bc2ae8..c487824f7d9a8 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -5,8 +5,8 @@ //! //! The APFS locator identifies the existing `.gl` file's physical extent. //! This module validates records and serves SEP reads from that extent. It -//! never creates storage. Raw writes remain blocked until the record update -//! protocol and APFS ownership rules are verified against macOS. +//! never creates storage. Writes are opt-in and require narrow APFS checks +//! for an unshared, unsnapshotted extent. use crate::shim; use kernel::prelude::*; @@ -110,23 +110,20 @@ fn valid_key(key: &Key) -> bool { } impl Store { - /// Opens the iBoot system container read-only, resolves `.gl` through APFS + /// Opens the iBoot system container, resolves `.gl` through APFS /// metadata, then validates the record store at exactly that extent. /// A nonzero start sector is an assertion against the APFS result, never - /// an override of it. Neither lookup nor record validation authorizes - /// raw writes: macOS's update protocol is still unverified. + /// an override of it. Writes additionally require a single-owner extent + /// with no snapshots or pending revert, checked on the same block handle. pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { - if writes_enabled { - return Err(ENOTSUPP); - } - let file = shim::StoreFile::open_block(OWNER_PATH, false)?; - let base = crate::xart_apfs::locate(&file)?; + let file = shim::StoreFile::open_block(OWNER_PATH, writes_enabled)?; + let base = crate::xart_apfs::locate(&file, writes_enabled)?; if start_sector != 0 { if start_sector.checked_mul(512).ok_or(EINVAL)? != base { return Err(EINVAL); } } - Self::open_file(file, false, base) + Self::open_file(file, writes_enabled, base) } /// Opens a caller-selected block mapping at offset zero. @@ -151,8 +148,8 @@ impl Store { Self::open_file(file, writes_enabled, base) } - /// Complete record validation using the same read-only handle with which - /// APFS ownership was resolved, avoiding a close/reopen window. + /// Complete record validation using the same handle with which APFS + /// ownership was resolved, avoiding a close/reopen window. fn open_file(file: shim::StoreFile, arm_writes: bool, base: u64) -> Result { let size = file.size()?; if base % BLOCK_SIZE as u64 != 0 || STORE_SIZE % BLOCK_SIZE as u64 != 0 { @@ -192,6 +189,9 @@ impl Store { if store.find(&Key::root(1)).is_none() || store.find(&Key::root(2)).is_none() { return Err(ENODATA); } + if arm_writes && (store.malformed_records != 0 || store.duplicate_records != 0) { + return Err(EIO); + } store.writes_enabled = arm_writes; if arm_writes { store.repair_disk()?; diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 767b9043d2ad5..b83ef5dea2e3d 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -12,15 +12,16 @@ systemctl daemon-reload systemctl enable aurora-sep.service ``` -The raw xART owner is currently read-only. Do not enable the installed -`aurora-sep.service` or run `load-driver` on a shared macOS/Linux installation: -that script requests `xart_writes=1`, which the kernel rejects. The kernel now -resolves the physical `.gl` extent from checksum-validated APFS metadata for -read-only operation; unsupported layouts fail closed. This removes the raw -record-signature locator, which was ambiguous when roots moved away from the -start of the file. APFS lookup alone does not establish write safety. Re-enable -provisioning only after macOS-compatible record updates and APFS ownership -invariants are verified on hardware. +`xart_writes=0` remains the default. `load-driver` explicitly opts into writes +and must only be enabled on a machine whose APFS `.gl` extent passes the +driver's narrow ownership checks: one physical extent, no snapshots or revert +state, and a single extent reference owned by the `.gl` inode. The kernel +resolves that extent from checksum-validated APFS metadata; unsupported layouts +fail closed. The opt-in has been tested on m2 with identity-keybag provisioning, +trusted-key seal/unseal, and reboot reload. The raw iBoot container hash did not +change in those tests, so an actual xART write was **not** validated. Do not +infer that all SEP or Touch ID operations work with a read-only GigaLocker. +Read `XART-SAFETY.md` before enabling writes on another dual-boot machine. For read-only structural diagnostics on Linux, run `sudo python3 apfs-xart-inspect.py /dev/disk/by-partlabel/iBootSystemContainer`. @@ -40,6 +41,75 @@ For fingerprint support, apply `patches/libfprint-1.94.100-apple-sep.patch` to libfprint 1.94.100, build and install libfprint, then install the fprintd device policy: +```sh +git clone --depth 1 --branch v1.94.100 \ + https://gitlab.freedesktop.org/libfprint/libfprint.git libfprint-aurora +cd libfprint-aurora +git apply ../linux/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch +meson setup build -Dprefix=/usr -Ddrivers=default -Dintrospection=false \ + -Dgtk-examples=false -Ddoc=false -Dinstalled-tests=false +ninja -C build +meson test -C build --print-errorlogs +``` + +Adjust the patch path for your checkout. The patch includes a fix for an +upstream Meson test loop when introspection is disabled. In the Aurora +`linux-build` guest, the additional packages needed for this build are +`libgudev libgusb pixman cairo glib2-devel`; `meson` and `ninja` are already +in the release image. The default driver set retains all normal libfprint +drivers and adds Aurora. This build was checked on aarch64: the shared library +and all default drivers compiled, with 0 test failures (most device tests +skip without their optional mocks/hardware). That does **not** prove Touch ID +works on m2. The patched library is installed privately under +`/opt/aurora-sep` there, with fprintd loading it via a service drop-in while +the distro libfprint remains untouched. fprintd discovers one Apple SEP sensor. +The Aurora driver now reports stage progress only when the kernel's stage +number increases; guidance and stage-zero events are not captured samples. +fprintd 1.94.5 itself emits an initial `enroll-stage-passed` after its +duplicate check, **before** starting enrollment. That first line does not +prove a finger was captured. +The device-specific `apple/mesa_calibration.bin` on m2 was recovered using +the bundled [`extract-mesa-calibration.py`](extract-mesa-calibration.py), copied +from [Gist revision `eb079a8007985d04e75182f20994f1a1c496f12e`](https://gist.github.com/DjDeveloperr/867a1961b861c570442724f48f770158/eb079a8007985d04e75182f20994f1a1c496f12e), +which scans the local iBoot System Container read-only for an FSCl/CALB record +with an IM4M manifest. It checks the container structure and markers, **not** +the manifest's cryptographic signature. Keep this blob private to the machine; +it is not a generic firmware package. On m2, a premature master +`SAVE_CATACOMB` returned SEP status `0x6` while its component state was `0x3` +(no save-pending bit). The +`m2fix10` fresh-context gate skips that unavailable save, and bounded +enrollment attempts reached the physical-finger wait without an immediate +enclave error. With the later UUID-forwarding stage2, a physical enrollment +completed on m2fix15 and master/user Catacombs were saved, but on each tested +reboot SEP listed zero live identities despite the host index still listing +the finger. m2fix16–m2fix20 narrowed this to the user Catacomb load: it +returns `0x8002` and becomes active, but reveals zero identities even with +the SCRD credential and sensor/device view prepared first. The saved files +remain backed up root-only on m2. A bounded repeated load still listed zero +identities. An opt-in owner export then produced a durable owner file, but +loading it did not recover the old user identity. `m2fix23` is a separate, +boot-tested candidate that saves owner in the fresh context and user before +master at completion; it has not been finger- or reboot-persistence-tested. +Login must stay password-based until a live post-reboot identity and +successful match are independently verified. + +On each Linux machine, extract its own calibration from its local iBoot System +Container (the input partition is discovered automatically): + +```sh +sudo install -d -m 0755 /usr/lib/firmware/apple +sudo python3 tools/aurora-sep/extract-mesa-calibration.py \ + -o /usr/lib/firmware/apple/mesa_calibration.bin +``` + +The extractor reads the iBoot partition but never writes to it. It creates the +output mode `0600`, rejects input/output aliases, and refuses ambiguous or +malformed calibration candidates. Do not commit or copy the resulting blob to +another machine. + +Once the hardware path is verified, install libfprint and then the fprintd +device policy: + ```sh install -Dm644 fprintd-aurora.conf \ /etc/systemd/system/fprintd.service.d/aurora.conf diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md index 1ea3708dc0a2e..52a976c63e5c4 100644 --- a/tools/aurora-sep/XART-SAFETY.md +++ b/tools/aurora-sep/XART-SAFETY.md @@ -1,10 +1,13 @@ # Shared xART gigalocker safety gate The iBoot APFS container and its XART `.gl` file are shared with macOS. The -kernel owner must remain read-only. `apple_sep.xart_writes=1` is rejected even -with an explicit `xart_start_sector`; the latter is now only an assertion -against the APFS-resolved file extent. Do not enable provisioning services on a -dual-boot installation while this gate is in place. +deployed `m2fix4` kernel remains read-only. The `m2fix5` source has an explicit +`xart_writes=1` opt-in; it accepts writes only when the APFS-resolved extent is +single-owner, unsnapshotted, un-cloned, and all existing records validate. +`xart_start_sector` remains only an assertion against the APFS result. The +opt-in is for the controlled J414s hardware test, not general APFS write +support or a promise that restoring an old raw image reverses SEP anti-replay +state. Do not enable it on other installations without equivalent checks. ## Evidence collected on 2026-09-21 @@ -39,3 +42,40 @@ dual-boot installation while this gate is in place. The current APFS locator supports only a complete checkpoint, single-node object maps and file-system tree, and a single unencrypted 6 MiB `.gl` extent. Unsupported layouts fail closed; this is not a general APFS implementation. + +## Read-only J414s recovery image (2026-09-23) + +After the minimal stage2 made the SEP mailbox and RNG functional, a fresh +read-only inspection still found one unencrypted 6 MiB `.gl` extent at APFS +block 1236, 11 live records with valid payload CRCs, and no malformed records. +The selected container checkpoint was xid 136. No xART writes or keybag +provisioning were attempted. + +The 524,288,000-byte `iBootSystemContainer` partition was copied through a +read-only block handle and compressed to +`work/m2-tb-sep-deploy/iboot-system-container-2026-09-22.img.zst` (7,857,330 +bytes, mode 0600). The local archive passed `zstd -t`; its decompressed +SHA-256 matched a separate read of the partition: +`09206fcf0500f5646ea04ec37cf1fc8a0493f1c5ee3271c2af562ed9b39a6d37`. +The compressed archive SHA-256 is +`8a885a2397c26404cf8395ca64b279efc793f9118eae408f2a6d84fe9af1ded8`. +The temporary m2 copy was removed after verification. This is a recovery +artifact, **not** evidence that Linux may safely write APFS-owned `.gl` blocks. + +## Narrow write-ownership evidence (2026-09-23) + +The live m2 partition SHA-256 still matches the independent backup hash above. +The updated read-only inspector selected checkpoint xid 136 and checked the +current xART volume superblock and checksum-validated extent-reference tree: +volume flags `0x1` (unencrypted only), zero snapshots, zero pending revert +fields, and an inode with no clone flag. The sole `.gl` extent is APFS blocks +1236–2771 (1536 blocks, 6 MiB); its only physical extent-reference record is +kind `APFS_KIND_NEW`, owner 16 (the inode's private ID), reference count 1. +The inspector now rejects missing, overlapping, shared, cloned/snapshotted, +or mismatched references, and four synthetic tests pass. The `m2fix5` kernel +independently checks the same narrow ownership conditions on its block handle +before arming writes and refuses a writable scan with malformed or duplicate +xART records. These checks establish the current physical mapping and lack of +APFS sharing; they do not by themselves establish macOS's full record-update +or SEP anti-replay recovery semantics. The APFS field meanings come from the +[Apple File System Reference](https://developer.apple.com/support/apple-file-system/Apple-File-System-Reference.pdf). diff --git a/tools/aurora-sep/apfs-xart-inspect.py b/tools/aurora-sep/apfs-xart-inspect.py index 63cc276646f5e..d11473f06d0fc 100644 --- a/tools/aurora-sep/apfs-xart-inspect.py +++ b/tools/aurora-sep/apfs-xart-inspect.py @@ -16,6 +16,16 @@ MOD = 0xffffffff +def is_gigalocker_name(name): + if name == b".gl": + return True + if len(name) != 39 or name[36:] != b".gl": + return False + return all(byte == ord("-") if i in (8, 13, 18, 23) + else byte in b"0123456789abcdefABCDEF" + for i, byte in enumerate(name[:36])) + + def u16(buf, off): return struct.unpack_from("> 60 != 2: + raise ValueError("unexpected extent-reference key type") + start = tagged & ((1 << 60) - 1) + length_and_kind = u64(val, 0) + length = length_and_kind & ((1 << 60) - 1) + kind = length_and_kind >> 60 + owner = u64(val, 8) + refcount = struct.unpack_from(" (1 << 60): + raise ValueError("invalid extent-reference length") + if start < physical + blocks and physical < start + length: + if (start != physical or length != blocks or kind != 1 or + owner != private_id or refcount != 1): + raise ValueError(".gl extent reference is shared or mismatched") + matches.append((start, length, kind, owner, refcount)) + if len(matches) != 1: + raise ValueError("expected exactly one unshared .gl extent reference") + return matches[0] + + +def inspect_extent_reference(disk, vol, physical, blocks, private_id): + snapshots = u64(vol, 216) + revert_xid, revert_sblock = u64(vol, 160), u64(vol, 168) + print("xart_volume_flags=0x%x snapshot_count=%d revert_xid=%d revert_sblock=%d" % + (u64(vol, 264), snapshots, revert_xid, revert_sblock)) + if snapshots or revert_xid or revert_sblock: + raise ValueError("xART volume has snapshot or pending revert state") + tree_paddr = u64(vol, 144) + tree = disk.read(tree_paddr) + if (u64(tree, 8) != tree_paddr or u32(tree, 24) != 0x40000002 or + u32(tree, 28) != 0x0f): + raise ValueError("unsupported extent-reference tree") + level, flags, entries = node_entries(tree) + if level or flags & 3 != 3: + raise ValueError("multilevel extent-reference tree needs inspection") + match = validate_extent_references(entries, physical, blocks, private_id) + print("gigalocker_extent_reference_block_length_kind_owner_refcount=%s" % + (match,)) + + def inspect(disk): base = disk.read(0) if base[32:36] != b"NXSB" or u32(base, 36) != BLOCK: @@ -183,6 +240,7 @@ def inspect(disk): if level: raise ValueError("multilevel xART FS tree needs further inspection") files = [] + names = [] for key, val in entries: if len(key) < 12 or len(val) < 8: continue @@ -191,8 +249,10 @@ def inspect(disk): name_len = u32(key, 8) & 0x3ff if name_len and 12 + name_len <= len(key): name = key[12:12 + name_len - 1] - if name == b".gl": + if is_gigalocker_name(name): files.append(u64(val, 0)) + names.append(name.decode("ascii")) + print("gigalocker_file_names=%s" % names) print("gigalocker_file_ids=%s" % files) if len(files) != 1: raise ValueError("expected exactly one .gl file") @@ -204,7 +264,6 @@ def inspect(disk): private_id, inode_flags = u64(inode, 8), u64(inode, 48) print("gigalocker_inode_private_id=%d flags=0x%x links=%d write_generation=%d" % (private_id, inode_flags, u32(inode, 56), u32(inode, 64))) - print("xart_volume_flags=0x%x snapshot_count=%d" % (u64(vol, 264), u32(disk.read(vol_omap), 36))) extents = [] for key, val in entries: if len(key) >= 16 and len(val) >= 24 and u64(key, 0) == ((8 << 60) | private_id): @@ -218,6 +277,8 @@ def inspect(disk): if not extents: raise ValueError("no .gl extents") if len(extents) == 1 and extents[0][0] == 0 and extents[0][1] == 0x600000: + inspect_extent_reference(disk, vol, extents[0][2], + extents[0][1] // BLOCK, private_id) inspect_raw_locator(disk, extents[0][2]) scan_other_roots(disk, extents[0][2], extents[0][1]) diff --git a/tools/aurora-sep/extract-mesa-calibration.py b/tools/aurora-sep/extract-mesa-calibration.py new file mode 100644 index 0000000000000..0485e2ef21b37 --- /dev/null +++ b/tools/aurora-sep/extract-mesa-calibration.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +"""Carve the per-device Mesa calibration from an Apple iBoot System Container.""" + +import argparse +import hashlib +import os +from pathlib import Path +import stat +import sys +import tempfile + + +ISC_PARTITION_NAME = "iBootSystemContainer" +SYS_BLOCK = Path("/sys/class/block") +DEV_ROOT = Path("/dev") +SCAN_CHUNK_SIZE = 4 * 1024 * 1024 +SCAN_OVERLAP = 4 * 1024 * 1024 + + +class DERError(ValueError): + pass + + +class TLV: + def __init__(self, data, start): + self.start = start + try: + self.tag = data[start] + first_length = data[start + 1] + except IndexError as error: + raise DERError("truncated DER header") from error + + if first_length < 0x80: + length = first_length + header_length = 2 + else: + length_bytes = first_length & 0x7f + if length_bytes == 0: + raise DERError("indefinite DER length") + if length_bytes > 4: + raise DERError("unreasonably large DER length") + length_start = start + 2 + length_end = length_start + length_bytes + if length_end > len(data): + raise DERError("truncated DER length") + encoded = data[length_start:length_end] + if encoded[0] == 0: + raise DERError("non-minimal DER length") + length = int.from_bytes(encoded, "big") + if length < 0x80: + raise DERError("non-minimal DER length") + header_length = 2 + length_bytes + + self.content_start = start + header_length + self.end = self.content_start + length + if self.end > len(data): + raise DERError("DER value extends beyond input") + + def content(self, data): + return data[self.content_start:self.end] + + +def child_tlvs(data, container): + offset = container.content_start + while offset < container.end: + child = TLV(data, offset) + if child.end > container.end: + raise DERError("DER child extends beyond its container") + yield child + offset = child.end + if offset != container.end: + raise DERError("malformed DER container") + + +def take_child(data, children, tag, value=None): + try: + child = next(children) + except StopIteration as error: + raise DERError("missing DER child") from error + if child.tag != tag: + raise DERError(f"unexpected DER tag 0x{child.tag:02x}") + if value is not None and child.content(data) != value: + raise DERError("unexpected DER value") + return child + +def validate_calibration(data, start): + """Return the end offset if data[start:] begins with a Mesa calibration.""" + outer = TLV(data, start) + if outer.tag != 0x30: + raise DERError("outer object is not a sequence") + outer_children = child_tlvs(data, outer) + take_child(data, outer_children, 0x16, b"comb") + + fdrd = take_child(data, outer_children, 0x30) + fdrd_children = child_tlvs(data, fdrd) + take_child(data, fdrd_children, 0x16, b"fdrd") + img4_octets = take_child(data, fdrd_children, 0x04) + try: + next(fdrd_children) + except StopIteration: + pass + else: + raise DERError("unexpected data in fdrd container") + + secb = take_child(data, outer_children, 0x30) + secb_children = child_tlvs(data, secb) + take_child(data, secb_children, 0x16, b"secb") + try: + next(outer_children) + except StopIteration: + pass + else: + raise DERError("unexpected data after secb container") + + img4 = TLV(data, img4_octets.content_start) + if img4.tag != 0x30 or img4.end != img4_octets.end: + raise DERError("fdrd is not one complete IMG4 sequence") + img4_children = child_tlvs(data, img4) + take_child(data, img4_children, 0x16, b"IMG4") + + im4p = take_child(data, img4_children, 0x30) + im4p_children = child_tlvs(data, im4p) + take_child(data, im4p_children, 0x16, b"IM4P") + take_child(data, im4p_children, 0x16, b"FSCl") + take_child(data, im4p_children, 0x16) + payload = take_child(data, im4p_children, 0x04).content(data) + if b"CALB" not in payload[:64]: + raise DERError("FSCl payload lacks its CALB header") + + # A signed calibration has an IM4M manifest following the IM4P. Requiring + # both markers avoids accepting an arbitrary, truncated FSCl object. + remainder = data[im4p.end:img4.end] + if b"IM4M" not in remainder or b"FSCl" not in remainder: + raise DERError("IMG4 lacks an FSCl manifest") + + return outer.end + + +def find_calibrations(data): + results = [] + signature = b"\x16\x04comb" + search_at = 0 + while True: + marker = data.find(signature, search_at) + if marker < 0: + break + # The outer SEQUENCE header immediately precedes the comb IA5String. + # Try every legal header width supported by TLV above. + for header_length in range(2, 7): + start = marker - header_length + if start < 0 or data[start] != 0x30: + continue + try: + end = validate_calibration(data, start) + except DERError: + continue + candidate = (start, end) + if candidate not in results: + results.append(candidate) + search_at = marker + 1 + return results + + +def discover_isc_inputs(sys_block=SYS_BLOCK, dev_root=DEV_ROOT): + """Find every iSC by GPT partition name, independent of disk numbering.""" + candidates = [] + try: + block_devices = sorted(sys_block.iterdir()) + except OSError as error: + block_devices = [] + sysfs_error = error + else: + sysfs_error = None + + for block_device in block_devices: + try: + properties = {} + for line in (block_device / "uevent").read_text().splitlines(): + key, separator, value = line.partition("=") + if separator: + properties[key] = value + except OSError: + continue + if properties.get("PARTNAME") != ISC_PARTITION_NAME: + continue + device_name = properties.get("DEVNAME", block_device.name) + if device_name.startswith("/dev/"): + device_name = device_name[5:] + device = dev_root / device_name + if device.exists(): + candidates.append(device) + + # This standard udev link also covers chroots where /sys is unavailable. + by_partlabel = dev_root / "disk/by-partlabel" / ISC_PARTITION_NAME + if by_partlabel.exists(): + candidates.append(by_partlabel) + + # Preserve order while removing aliases or duplicate sysfs entries. + unique = [] + seen = set() + for candidate in candidates: + try: + identity = candidate.resolve() + except OSError: + identity = candidate + if identity not in seen: + seen.add(identity) + unique.append(candidate) + if not unique and sysfs_error is not None: + raise RuntimeError( + f"cannot inspect {sys_block} and {by_partlabel} is unavailable: " + f"{sysfs_error}" + ) from sysfs_error + return unique + + +def scan_input(path): + """Scan a file or block device without retaining the whole iSC in memory.""" + mode = path.stat().st_mode + if not (stat.S_ISREG(mode) or stat.S_ISBLK(mode)): + raise RuntimeError(f"{path} is not a regular file or block device") + + candidates = {} + overlap = b"" + total = 0 + with path.open("rb", buffering=0) as source: + while True: + chunk = source.read(SCAN_CHUNK_SIZE) + if not chunk: + break + total += len(chunk) + window = overlap + chunk + window_start = total - len(window) + for start, end in find_calibrations(window): + absolute_start = window_start + start + candidates.setdefault(absolute_start, bytes(window[start:end])) + overlap = window[-SCAN_OVERLAP:] + return sorted(candidates.items()) + + +def write_private(path, blob): + path.parent.mkdir(parents=True, exist_ok=True) + old_umask = os.umask(0o077) + temporary = None + try: + fd, name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(name) + with os.fdopen(fd, "wb") as output: + output.write(blob) + output.flush() + os.fsync(output.fileno()) + os.chmod(temporary, stat.S_IRUSR | stat.S_IWUSR) + os.replace(temporary, path) + temporary = None + finally: + os.umask(old_umask) + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def validate_output(path, inputs): + """Reject output targets that could replace an input or special file.""" + if path.is_symlink(): + raise RuntimeError(f"output {path} is a symbolic link") + if not path.exists(): + return + for input_path in inputs: + try: + if os.path.samefile(path, input_path): + raise RuntimeError(f"output {path} aliases input {input_path}") + except FileNotFoundError: + pass + if not stat.S_ISREG(path.stat().st_mode): + raise RuntimeError(f"output {path} is not a regular file") + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "input", nargs="?", type=Path, + help=( + "raw iBoot System Container or dump; omit to discover all " + "iBootSystemContainer partitions" + ), + ) + parser.add_argument( + "-o", "--output", type=Path, default=Path("mesa_calibration.bin"), + help="output file (default: ./mesa_calibration.bin)", + ) + return parser.parse_args(argv) + + +def main(): + args = parse_args() + try: + inputs = [args.input] if args.input is not None else discover_isc_inputs() + if not inputs: + raise RuntimeError( + "no iBootSystemContainer partition found; pass its block device " + "or a raw partition dump explicitly" + ) + validate_output(args.output, inputs) + + candidates = [] + for input_path in inputs: + for offset, blob in scan_input(input_path): + candidates.append((input_path, offset, blob)) + if not candidates: + raise RuntimeError( + "no signed Mesa FSCl/CALB calibration found; expected the raw " + "iBoot System Container (Apple Silicon boot partition)" + ) + blobs = {} + for input_path, offset, blob in candidates: + blobs.setdefault(blob, []).append((input_path, offset)) + if len(blobs) != 1: + locations = ", ".join( + f"{input_path}@0x{offset:x}" + for input_path, offset, _ in candidates + ) + raise RuntimeError( + f"found different calibrations at {locations}; " + "pass the intended iSC explicitly" + ) + + blob, locations = next(iter(blobs.items())) + write_private(args.output, blob) + except (OSError, RuntimeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + digest = hashlib.sha256(blob).hexdigest() + formatted_locations = ", ".join( + f"{input_path}@0x{offset:x}" for input_path, offset in locations + ) + print(f"found Mesa calibration at {formatted_locations}") + print(f"wrote {len(blob)} bytes to {args.output}") + print(f"sha256 {digest}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-sep/load-driver b/tools/aurora-sep/load-driver index f5579fc01b360..a70db75e6b25b 100755 --- a/tools/aurora-sep/load-driver +++ b/tools/aurora-sep/load-driver @@ -4,6 +4,13 @@ set -eu keybag=/var/lib/aurora-sep-keybag.bin refkey=/var/lib/aurora-sep-refkey.bin +# An SSH/sudo caller cannot change its parent's session keyring with +# `keyctl new_session`. Run the whole probe in a child keyring instead; the +# marker prevents re-exec recursion and the keyring dies with this process. +if [ "${AURORA_SEP_SESSION:-0}" != 1 ]; then + exec keyctl session aurora-sep-load env AURORA_SEP_SESSION=1 "$0" "$@" +fi + modprobe apple_sep xart_writes=1 provision_keybag=1 work=$(mktemp -d /run/aurora-sep-load.XXXXXX) @@ -14,7 +21,6 @@ cleanup() { } trap cleanup EXIT HUP INT TERM -keyctl new_session aurora-sep-load >/dev/null attempts=300 verified=false while [ "$attempts" -gt 0 ]; do diff --git a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch index 37593f65ea05e..1947bd095b43b 100644 --- a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch +++ b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch @@ -135,10 +135,10 @@ index 0000000..b2d2f43 +#define AURORA_BIO_ATTEST _IOWR(AURORA_BIO_IOC_MAGIC, 0x0a, struct aurora_bio_attest) diff --git a/libfprint/drivers/aurora/aurora.c b/libfprint/drivers/aurora/aurora.c new file mode 100644 -index 0000000..55f3311 +index 0000000..80ace06 --- /dev/null +++ b/libfprint/drivers/aurora/aurora.c -@@ -0,0 +1,829 @@ +@@ -0,0 +1,837 @@ +/* + * Aurora SEP fingerprint driver + * @@ -170,6 +170,7 @@ index 0000000..55f3311 + + gint fd; + guint watch_id; ++ guint32 enroll_last_reported_stage; + + guint8 nonce[AURORA_BIO_NONCE_LEN]; + gboolean nonce_valid; @@ -516,7 +517,13 @@ index 0000000..55f3311 + return G_SOURCE_CONTINUE; + + case AURORA_BIO_STATE_PROGRESS: -+ fpi_device_enroll_progress (device, poll_res.stage, NULL, NULL); ++ /* fprintd calls this "enroll-stage-passed". A guidance wakeup or an ++ * initial stage-zero event must not masquerade as a captured sample. */ ++ if (poll_res.stage > self->enroll_last_reported_stage) ++ { ++ self->enroll_last_reported_stage = poll_res.stage; ++ fpi_device_enroll_progress (device, poll_res.stage, NULL, NULL); ++ } + return G_SOURCE_CONTINUE; + + case AURORA_BIO_STATE_DONE: @@ -543,7 +550,7 @@ index 0000000..55f3311 + self->watch_id = 0; + fpi_device_enroll_complete (device, NULL, + fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "enrolment failed, enclave status 0x%x", ++ "enrolment failed, driver status 0x%x", + poll_res.status)); + return G_SOURCE_REMOVE; + } @@ -571,6 +578,7 @@ index 0000000..55f3311 + return; + } + ++ self->enroll_last_reported_stage = 0; + aurora_stop_watch (self); + self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_enrol_ready, device); +} @@ -1177,3 +1185,13 @@ index ab09dc1..dfa7460 100644 # Virtual drivers (test-only, optional) 'virtual_image': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, 'virtual_device': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, +diff --git a/tests/meson.build b/tests/meson.build +index aad0675..2147ef3 100644 +--- a/tests/meson.build ++++ b/tests/meson.build +@@ -295,4 +295,4 @@ else +- foreach driver_test: drivers_tests ++ foreach driver_test, args: drivers_tests + test(driver_test, + sh, + args: ['-c', 'exit 77'] diff --git a/tools/aurora-sep/tests/test_extract_mesa_calibration.py b/tools/aurora-sep/tests/test_extract_mesa_calibration.py new file mode 100644 index 0000000000000..08884ba921e9b --- /dev/null +++ b/tools/aurora-sep/tests/test_extract_mesa_calibration.py @@ -0,0 +1,72 @@ +"""Offline safety and format tests for the bundled calibration extractor.""" + +import importlib.util +from pathlib import Path +import stat +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "extract-mesa-calibration.py" +SPEC = importlib.util.spec_from_file_location("extract_mesa_calibration", SCRIPT) +extractor = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(extractor) + + +def tlv(tag, content): + length = len(content) + if length < 128: + encoded_length = bytes([length]) + else: + width = (length.bit_length() + 7) // 8 + encoded_length = bytes([0x80 | width]) + length.to_bytes(width, "big") + return bytes([tag]) + encoded_length + content + + +def ia5(value): + return tlv(0x16, value) + + +def synthetic_record(with_manifest=True): + """Build a structural fixture, not a cryptographically signed IMG4.""" + im4p = tlv(0x30, ia5(b"IM4P") + ia5(b"FSCl") + ia5(b"test") + + tlv(0x04, b"CALB" + b"x" * 64)) + manifest = ia5(b"IM4M") + ia5(b"FSCl") if with_manifest else b"" + img4 = tlv(0x30, ia5(b"IMG4") + im4p + manifest) + fdrd = tlv(0x30, ia5(b"fdrd") + tlv(0x04, img4)) + secb = tlv(0x30, ia5(b"secb")) + return tlv(0x30, ia5(b"comb") + fdrd + secb) + + +class CalibrationExtractorTests(unittest.TestCase): + def test_scans_only_complete_manifest_marked_record(self): + record = synthetic_record() + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "isc.img" + source.write_bytes(b"prefix" + record + b"suffix") + self.assertEqual(extractor.scan_input(source), [(6, record)]) + + def test_rejects_record_without_manifest_markers(self): + self.assertEqual(extractor.find_calibrations(synthetic_record(False)), []) + + def test_rejects_input_output_aliases_and_symlinks(self): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "isc.img" + source.write_bytes(synthetic_record()) + with self.assertRaises(RuntimeError): + extractor.validate_output(source, [source]) + symlink = Path(directory) / "output.bin" + symlink.symlink_to(source) + with self.assertRaises(RuntimeError): + extractor.validate_output(symlink, [source]) + + def test_private_output_mode(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "mesa_calibration.bin" + extractor.write_private(output, b"sample") + self.assertEqual(output.read_bytes(), b"sample") + self.assertEqual(stat.S_IMODE(output.stat().st_mode), 0o600) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/aurora-sep/tests/test_xart_locator.py b/tools/aurora-sep/tests/test_xart_locator.py index 167c977ec2bd0..7493928bd6795 100644 --- a/tools/aurora-sep/tests/test_xart_locator.py +++ b/tools/aurora-sep/tests/test_xart_locator.py @@ -27,7 +27,47 @@ def record(kind, revision): return out +def extent_reference(start, blocks, owner, refcount=1, kind=1): + key = struct.pack(" Date: Thu, 24 Sep 2026 23:11:18 -0400 Subject: [PATCH 0024/1579] arm64: enable J414s Thunderbolt DP tunnel and rearm display route Extend the T6020 ATC tunnel clock gate and dcpext0 preference to J414s. Re-establish the DCP route when a new dock tunnel appears after a controller detach without a prior teardown callback. Tested on J414s m2fix42: dock DP tunnel enabled USB-3 at 1920x1280@60, direct USB-2 at 3840x2160@120 and eDP at 3024x1964@120 with active CRTCs and zero failed units. Controller-detach disconnect reporting remains under investigation. --- drivers/gpu/drm/apple/dcp.c | 18 +++++++++--------- drivers/phy/apple/atc.c | 5 +++-- drivers/thunderbolt/apple.c | 24 +++++++++++++++++++++--- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index b9774821d882f..698575b4ca88d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -561,7 +561,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, continue; score = dcp_typec_route_score(candidate); /* - * j416s-specific, not in the reference (t8103 has a single + * T6020-specific, not in the reference (t8103 has a single * dcpext): prefer a pipeline with a fixed output of its own * (dcpext0) for a Type-C tunnel route. dcpext1 (Type-C only, * no fixed output of its own) never completes link training @@ -569,15 +569,15 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, * accepts request_display but never issues another apcall, * on every attempt, even though the driver-issued connect * parameters are byte-identical between the two pipelines. - * Nothing in this driver's source explains the difference, - * so this is a firmware-internal decision on the dcpext1 - * coprocessor instance, not something fixable here. dcpext0, - * forced onto the same physical port and tunnel, reaches a - * full AUX/DPCD link (DPRX_DONE=1) and a working picture, so - * it is preferred unconditionally for a tunnel route on this - * hardware. + * Nothing in this driver's source explains the difference. + * dcpext0, forced onto the same physical port and tunnel, reaches + * a full AUX/DPCD link (DPRX_DONE=1) and a working picture on + * j416s. J414s uses the same T6020 display topology; its tunnel + * result is tested separately. */ - if (!candidate->dcp->fixed_phy) + if ((of_machine_is_compatible("apple,j414s") || + of_machine_is_compatible("apple,j416s")) && + !candidate->dcp->fixed_phy) score += 100; if (score < best_score) { best = candidate; diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index e4331cf72fe48..d8d0646c22e38 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2340,7 +2340,7 @@ static const struct phy_ops apple_atc_dp_phy_ops = { }; /* - * j416s has one ATC PHY core per Type-C port at 0x703000000 (left), + * j414s and j416s have one ATC PHY core per Type-C port at 0x703000000 (left), * 0xb03000000 (left), 0xf03000000 (right) -- the same top-byte pattern * as the ACIO/DPIN0 and crossbar addresses in drivers/thunderbolt/apple.c * and drivers/mux/apple-display-crossbar.c. Originally this only @@ -2370,7 +2370,8 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) int ret; if (!phy || phy->ops != &apple_atc_dp_phy_ops || - !of_machine_is_compatible("apple,j416s")) + (!of_machine_is_compatible("apple,j414s") && + !of_machine_is_compatible("apple,j416s"))) return -EOPNOTSUPP; atcphy = phy_get_drvdata(phy); if (!of_device_is_compatible(atcphy->np, "apple,t6020-atcphy") || diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 36ccfae7fdb50..fbe5930e9cfce 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -187,6 +187,7 @@ struct apple_cio { unsigned int idx; bool alive; /* tunnel up; cleared before it is torn down */ bool handed; /* appledrm has our callback; work only */ + bool rearm; /* a fresh tunnel needs a fresh DCP route */ } dpin[2]; }; @@ -1096,8 +1097,15 @@ static int apple_nhi_dp_tunnel_post_activate(struct tb_nhi *nhi, } else { struct apple_dpin_ctx *c = &anhi->acio->dpin[idx]; - scoped_guard(mutex, &c->lock) + /* + * A new tunnel can arrive without the old tunnel's teardown hook + * running on a controller detach. Force the DCP route through a + * down/up cycle instead of treating the old handoff as this one. + */ + scoped_guard(mutex, &c->lock) { c->alive = true; + c->rearm = true; + } queue_work(anhi->acio->dp_wq, &c->work); } apple_dp_dump_host_adapters(anhi); @@ -2041,12 +2049,15 @@ static void apple_dpin_down(struct apple_dpin_ctx *c) static void apple_dpin_work_fn(struct work_struct *work) { struct apple_dpin_ctx *c = container_of(work, struct apple_dpin_ctx, work); - bool want; + bool want, rearm; int ret; for (;;) { - scoped_guard(mutex, &c->lock) + scoped_guard(mutex, &c->lock) { want = c->alive; + rearm = c->rearm; + c->rearm = false; + } if (!want) { if (c->handed || c->regs) { @@ -2055,6 +2066,13 @@ static void apple_dpin_work_fn(struct work_struct *work) } return; } + if (rearm && c->handed) { + dev_info(c->acio->dev, + "dpin%u: replacing stale display route for new tunnel\n", + c->idx); + apple_dpin_down(c); + continue; + } if (c->handed) return; From 8de889b7b0c40b8f6f16b72143b8ccc1dc0ef00c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 24 Sep 2026 23:26:09 -0400 Subject: [PATCH 0025/1579] HID: apple-mtp-haptic: claim output-only actuator without hidraw The actuator has no input listener, and J414s Asahi does not enable CONFIG_HIDRAW. Claiming HID_CONNECT_HIDRAW left the actuator unbound. HID_CONNECT_DRIVER lets the kernel issue its output and feature reports without a userspace hidraw node. Tested on m2fix44: actuator bound, FF_HAPTIC effect upload/playback/erase succeeded, and MTP firmware reported the expected host-control and device-control handoffs. --- drivers/hid/hid-apple-mtp-haptic.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/hid/hid-apple-mtp-haptic.c b/drivers/hid/hid-apple-mtp-haptic.c index 42ab019f7a27a..4246d33462bb8 100644 --- a/drivers/hid/hid-apple-mtp-haptic.c +++ b/drivers/hid/hid-apple-mtp-haptic.c @@ -66,7 +66,12 @@ static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid return ret; } - ret = hid_hw_start(haptic_hdev, HID_CONNECT_HIDRAW); + /* + * The actuator has no input collection and does not need hidraw. + * Claim it for the driver itself so systems with HIDRAW disabled can + * still issue the output and feature reports used for haptics. + */ + ret = hid_hw_start(haptic_hdev, HID_CONNECT_DRIVER); if (ret) { hid_err(haptic_hdev, "hw start failed\n"); return ret; From c70d4063f4a907b638ffcb2375ef9d2264d647cc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 24 Sep 2026 23:39:53 -0400 Subject: [PATCH 0026/1579] drm/apple: reserve J414s hybrid DCP for Thunderbolt DP tunnel Route an available direct Type-C DP-alt-mode display to dcpext1 first so dcpext0 remains available for a dock DP tunnel. The tunnel still prefers dcpext0, which is the pipeline that completes link training on T6020. m2fix45 cold-booted with both displays attached: direct 3840x2160@120 on dcpext1 and dock 1920x1280@60 on dcpext0 came up without port resets, alongside eDP. SEP identity, haptic FF ioctl, and AVD H.264 decode remained functional. --- drivers/gpu/drm/apple/dcp.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 698575b4ca88d..a45803901af25 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -738,6 +738,14 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (!dcp_typec_route_available(candidate)) continue; score = dcp_typec_route_score(candidate); + /* + * On J414s, leave dcpext0 free for a dock's DP + * tunnel when a direct DP-alt-mode display can use + * the Type-C-only dcpext1 pipeline. + */ + if (of_machine_is_compatible("apple,j414s") && + candidate->dcp->fixed_phy) + score += 100; if (score < best_score) { best = candidate; best_score = score; From 7ada4fdbde54c7332aefed2beea9b3400f321c73 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 26 Sep 2026 01:36:20 -0400 Subject: [PATCH 0027/1579] iommu/apple-dart-x1n1: paravirt DART over the sptm2mmio transport A Linux guest under the x1n1 hypervisor on hardware SPTM cannot own the DART registers or tables; SPTM does. This driver forwards the guest's IOMMU intent to x1n1, which executes it against the SPTM DART endpoints. Transport: x1n1 exposes a synthetic MMIO page (from DT "reg") whose accesses trap to EL2. Identification registers (magic/version/features) are read directly. Each map/unmap becomes an entry in an operation batch the driver builds in a linear-map page; a single 64-bit write of the batch's guest physical address to the per-dart doorbell forwards the whole batch, so a range spanning many 32 MiB blocks and streams costs one trap and one world switch. The doorbell write world-switches through SPTM, which ties the resumed guest's SP_EL1 to the hypervisor SP_EL2; per the paravirt ABI the driver saves and restores the kernel SP around the store with interrupts masked. Every other register survives the switch. Binds "apple,dart-x1n1" with apple,x1n1-dart-id and #iommu-cells = <1>. Compiles under the asahi profile; the sptm2mmio round trip is validated in QEMU on real J700 SPTM via emuasi tests/x1n1-sptm2mmio-guest.S. A full Linux-guest DMA path is not yet exercised. Assisted-by: Claude Opus 4.8 --- drivers/iommu/Kconfig | 12 + drivers/iommu/Makefile | 1 + drivers/iommu/apple-dart-x1n1.c | 569 ++++++++++++++++++++++++++++++++ 3 files changed, 582 insertions(+) create mode 100644 drivers/iommu/apple-dart-x1n1.c diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 8b0e5505516a8..3a3ab24969f54 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -316,6 +316,18 @@ config APPLE_DART Say Y here if you are using an Apple SoC. +config APPLE_DART_X1N1 + tristate "Apple DART paravirt (x1n1 hypervisor) support" + depends on ARCH_APPLE || COMPILE_TEST + select IOMMU_API + help + Paravirtualised Apple DART for a Linux guest running under the x1n1 + hypervisor on hardware SPTM. The guest cannot program the DART + directly, so IOMMU map/unmap/flush are forwarded to SPTM through + x1n1 hypercalls. Bind against "apple,dart-x1n1" device-tree nodes. + + Say Y here only for a Linux guest hosted by x1n1. + config S390_IOMMU def_bool y if S390 && PCI depends on S390 && PCI diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 0275821f4ef98..b189696a0c325 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -36,4 +36,5 @@ obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o obj-$(CONFIG_APPLE_DART) += apple-dart.o +obj-$(CONFIG_APPLE_DART_X1N1) += apple-dart-x1n1.o obj-$(CONFIG_IOMMU_DEBUG_PAGEALLOC) += iommu-debug-pagealloc.o diff --git a/drivers/iommu/apple-dart-x1n1.c b/drivers/iommu/apple-dart-x1n1.c new file mode 100644 index 0000000000000..72f68fdac6f2f --- /dev/null +++ b/drivers/iommu/apple-dart-x1n1.c @@ -0,0 +1,569 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Paravirtualised Apple DART for the x1n1 hypervisor (sptm2mmio transport). + * + * On an Apple SoC running under hardware SPTM, a trusted Linux guest cannot + * touch the DART MMIO registers or own the DART translation tables: SPTM does. + * The x1n1 hypervisor (EL2) forwards the guest's IOMMU intent to the SPTM DART + * endpoints. This driver is the guest half: instead of programming an + * io-pgtable and TTBRs, each map/unmap becomes an entry in an operation batch + * that x1n1 executes against SPTM. + * + * Transport ("sptm2mmio"): x1n1 exposes a synthetic MMIO page whose every + * access traps to EL2. Reads of the identification registers are emulated + * inline. A 64-bit write to a per-device doorbell carries the guest-physical + * address of an operation batch that the guest built in ordinary RAM; x1n1 + * leaves the guest, runs the whole batch against SPTM, and re-enters. One + * doorbell write therefore costs one trap and one world switch for a batch of + * up to hundreds of ops. + * + * SP_EL1 ABI: SPTM's guest re-dispatch ties the resumed guest's SP_EL1 to the + * hypervisor's SP_EL2, so a doorbell write clobbers the caller's SP. Every + * other register survives, so the doorbell is issued with the kernel SP saved + * in a GPR and restored immediately after, under a local IRQ save. + * + * DT: x1n1 presents a paravirt DART as an "apple,dart-x1n1" node with the + * doorbell page in "reg", a u32 "apple,x1n1-dart-id" (the ADT dart-id SPTM + * knows) and #iommu-cells = <1> (the stream id). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* sptm2mmio register/doorbell map, shared with x1n1 src/sptm.c. */ +#define X1N1_MMIO_MAGIC 0x000 +#define X1N1_MMIO_VERSION 0x008 +#define X1N1_MMIO_MAX_OPS 0x010 +#define X1N1_MMIO_FEATURES 0x018 +#define X1N1_MMIO_DART_DB 0x800 /* + 8 * dart-id */ +#define X1N1_MMIO_SIZE 0x4000 + +#define X1N1_MAGIC 0x78316e31UL /* "x1n1" */ +#define X1N1_FEAT_DART BIT(0) + +/* DART unit ops. */ +#define X1N1_DART_INIT 1 /* () INIT + POWERUP */ +#define X1N1_DART_ATTACH 2 /* (sid, dva, level) */ +#define X1N1_DART_MAP 3 /* (sid, dva, ipa, size, prot) */ +#define X1N1_DART_UNMAP 4 /* (sid, dva, size) */ +#define X1N1_DART_ENABLE 5 /* (sid) */ +#define X1N1_DART_DISABLE 6 /* (sid) */ + +struct x1n1_op { + u32 op; + u32 flags; + u64 arg[6]; + u64 ret; +} __packed; + +struct x1n1_batch { + u32 count; + u32 flags; + s32 status; + u32 done; + struct x1n1_op op[]; +} __packed; + +#define DART_X1N1_PAGE_SHIFT 14 +#define DART_X1N1_PAGE_SIZE (1UL << DART_X1N1_PAGE_SHIFT) /* 16 KiB */ +/* x1n1's DART MAP forwards one contiguous run within a single 32 MiB leaf-table + * span, so every MAP op must stay inside one aligned block. */ +#define DART_X1N1_BLOCK_SHIFT 25 +#define DART_X1N1_BLOCK_SIZE (1UL << DART_X1N1_BLOCK_SHIFT) /* 32 MiB */ + +#define DART_X1N1_MAX_STREAMS 256 +#define DART_X1N1_APERTURE_END (SZ_4G - 1) + +/* One batch buffer per DART, sized to hold a full unmap/map of the aperture in + * per-block chunks; kept in the linear map so its guest PA is stable. A page + * holds (PAGE_SIZE-16)/64 ops; the callers below chunk to fit. */ +#define X1N1_BATCH_MAX_OPS ((PAGE_SIZE - sizeof(struct x1n1_batch)) / \ + sizeof(struct x1n1_op)) + +struct apple_dart_x1n1 { + struct device *dev; + struct iommu_device iommu; + void __iomem *mmio; + u32 dart_id; + u32 num_streams; + bool inited; + struct mutex init_lock; /* serialises one-time init */ + + struct x1n1_batch *batch; /* linear-map buffer, guest-PA stable */ + phys_addr_t batch_pa; + spinlock_t batch_lock; /* protects batch + doorbell */ +}; + +struct apple_dart_x1n1_master { + struct apple_dart_x1n1 *dart; + DECLARE_BITMAP(sids, DART_X1N1_MAX_STREAMS); +}; + +struct apple_dart_x1n1_domain { + struct iommu_domain domain; + struct apple_dart_x1n1 *dart; + DECLARE_BITMAP(sids, DART_X1N1_MAX_STREAMS); + struct mutex lock; /* protects sids / dart binding */ +}; + +static struct apple_dart_x1n1_domain *to_x1n1_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct apple_dart_x1n1_domain, domain); +} + +/* + * Ring a doorbell: publish the batch, write its guest-physical address to the + * doorbell, and return the batch status. The doorbell write world-switches to + * EL2, which clobbers SP_EL1 (see the file header), so save and restore the + * kernel SP around it with interrupts masked. The caller holds batch_lock. + */ +static int x1n1_ring(struct apple_dart_x1n1 *dart, u32 db_off) +{ + void __iomem *db = dart->mmio + db_off; + u64 batch_pa = dart->batch_pa; + unsigned long flags; + + dart->batch->status = 0; + dart->batch->done = 0; + /* Order the batch stores before the doorbell; x1n1 does dsb on its side. */ + wmb(); + + local_irq_save(flags); + asm volatile( + "mov x9, sp\n" + "str %x[pa], [%[db]]\n" + "mov sp, x9\n" + : + : [db] "r"(db), [pa] "r"(batch_pa) + : "x9", "memory"); + local_irq_restore(flags); + + return dart->batch->status; +} + +/* Bring the DART up in SPTM exactly once, before any stream is enabled. */ +static int apple_dart_x1n1_ensure_init(struct apple_dart_x1n1 *dart) +{ + int ret = 0; + + mutex_lock(&dart->init_lock); + if (!dart->inited) { + unsigned long flags; + + spin_lock_irqsave(&dart->batch_lock, flags); + dart->batch->count = 1; + dart->batch->op[0] = (struct x1n1_op){ .op = X1N1_DART_INIT }; + ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + spin_unlock_irqrestore(&dart->batch_lock, flags); + if (ret) + ret = -EIO; + else + dart->inited = true; + } + mutex_unlock(&dart->init_lock); + return ret; +} + +static int apple_dart_x1n1_attach_dev(struct iommu_domain *domain, + struct device *dev, + struct iommu_domain *old) +{ + struct apple_dart_x1n1_master *master = dev_iommu_priv_get(dev); + struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); + struct apple_dart_x1n1 *dart; + unsigned long flags; + unsigned int sid; + u32 n; + int ret; + + if (!master) + return -ENODEV; + dart = master->dart; + + ret = apple_dart_x1n1_ensure_init(dart); + if (ret) + return ret; + + mutex_lock(&dom->lock); + if (dom->dart && dom->dart != dart) { + mutex_unlock(&dom->lock); + return -EINVAL; + } + dom->dart = dart; + + /* For each stream: attach a level-2 table (the 32-bit aperture this + * driver advertises needs no level-0/1 root) and enable translation. + * x1n1 owns the tables. Build the ops in one batch. */ + spin_lock_irqsave(&dart->batch_lock, flags); + n = 0; + for_each_set_bit(sid, master->sids, dart->num_streams) { + if (test_bit(sid, dom->sids)) + continue; + if (n + 2 > X1N1_BATCH_MAX_OPS) + break; + dart->batch->op[n++] = (struct x1n1_op){ + .op = X1N1_DART_ATTACH, + .arg = { sid, 0, 2 }, + }; + dart->batch->op[n++] = (struct x1n1_op){ + .op = X1N1_DART_ENABLE, + .arg = { sid }, + }; + } + dart->batch->count = n; + ret = n ? x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id) : 0; + if (!ret) + for_each_set_bit(sid, master->sids, dart->num_streams) + set_bit(sid, dom->sids); + spin_unlock_irqrestore(&dart->batch_lock, flags); + + mutex_unlock(&dom->lock); + return ret ? -EIO : 0; +} + +static int apple_dart_x1n1_map_pages(struct iommu_domain *domain, + unsigned long iova, phys_addr_t paddr, + size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); + struct apple_dart_x1n1 *dart = dom->dart; + size_t total = pgsize * pgcount; + size_t done = 0; + unsigned long flags; + + if (!dart) + return -ENODEV; + + /* + * Forward the range in per-stream chunks that each stay inside one + * aligned 32 MiB block, matching x1n1's contiguous-map constraint. + * Pack as many MAP ops as fit into a batch, then ring once. + */ + spin_lock_irqsave(&dart->batch_lock, flags); + while (done < total) { + u32 n = 0; + int ret; + size_t batch_start = done; + + while (done < total && n < X1N1_BATCH_MAX_OPS) { + u64 cur_iova = iova + done; + u64 cur_pa = paddr + done; + size_t block_left = DART_X1N1_BLOCK_SIZE - + (cur_iova & (DART_X1N1_BLOCK_SIZE - 1)); + size_t chunk = min_t(size_t, total - done, block_left); + unsigned int sid; + bool full = false; + + for_each_set_bit(sid, dom->sids, dart->num_streams) { + if (n >= X1N1_BATCH_MAX_OPS) { + full = true; + break; + } + dart->batch->op[n++] = (struct x1n1_op){ + .op = X1N1_DART_MAP, + .arg = { sid, cur_iova, cur_pa, + chunk, prot }, + }; + } + if (full) + break; + done += chunk; + } + + dart->batch->count = n; + ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + if (ret) { + /* Report the bytes that fully mapped before this batch. */ + *mapped = batch_start; + spin_unlock_irqrestore(&dart->batch_lock, flags); + return -ENOMEM; + } + } + spin_unlock_irqrestore(&dart->batch_lock, flags); + + *mapped = done; + return 0; +} + +static size_t apple_dart_x1n1_unmap_pages(struct iommu_domain *domain, + unsigned long iova, size_t pgsize, + size_t pgcount, + struct iommu_iotlb_gather *gather) +{ + struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); + struct apple_dart_x1n1 *dart = dom->dart; + size_t total = pgsize * pgcount; + size_t done = 0; + unsigned long flags; + + if (!dart) + return 0; + + spin_lock_irqsave(&dart->batch_lock, flags); + while (done < total) { + u32 n = 0; + + while (done < total && n < X1N1_BATCH_MAX_OPS) { + u64 cur_iova = iova + done; + size_t block_left = DART_X1N1_BLOCK_SIZE - + (cur_iova & (DART_X1N1_BLOCK_SIZE - 1)); + size_t chunk = min_t(size_t, total - done, block_left); + unsigned int sid; + bool full = false; + + for_each_set_bit(sid, dom->sids, dart->num_streams) { + if (n >= X1N1_BATCH_MAX_OPS) { + full = true; + break; + } + dart->batch->op[n++] = (struct x1n1_op){ + .op = X1N1_DART_UNMAP, + .arg = { sid, cur_iova, chunk }, + }; + } + if (full) + break; + done += chunk; + } + + dart->batch->count = n; + x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + } + spin_unlock_irqrestore(&dart->batch_lock, flags); + + return total; +} + +static void apple_dart_x1n1_iotlb_sync(struct iommu_domain *domain, + struct iommu_iotlb_gather *gather) +{ + /* x1n1 invalidates inside every MAP/UNMAP it forwards to SPTM, so a + * separate flush is a no-op. */ +} + +static phys_addr_t apple_dart_x1n1_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova) +{ + /* SPTM owns the DART tables; the guest does not shadow them, so a + * hardware-accurate walk is not available here. The DMA path does not + * need it. */ + return 0; +} + +static void apple_dart_x1n1_domain_free(struct iommu_domain *domain) +{ + kfree(to_x1n1_domain(domain)); +} + +static struct iommu_domain * +apple_dart_x1n1_domain_alloc_paging(struct device *dev) +{ + struct apple_dart_x1n1_domain *dom; + + dom = kzalloc(sizeof(*dom), GFP_KERNEL); + if (!dom) + return NULL; + + mutex_init(&dom->lock); + dom->domain.pgsize_bitmap = DART_X1N1_PAGE_SIZE; + dom->domain.geometry.aperture_start = 0; + dom->domain.geometry.aperture_end = DART_X1N1_APERTURE_END; + dom->domain.geometry.force_aperture = true; + + if (dev) { + struct apple_dart_x1n1_master *master = dev_iommu_priv_get(dev); + + if (master) + dom->dart = master->dart; + } + + return &dom->domain; +} + +static struct iommu_device *apple_dart_x1n1_probe_device(struct device *dev) +{ + struct apple_dart_x1n1_master *master = dev_iommu_priv_get(dev); + + if (!dev_iommu_fwspec_get(dev) || !master) + return ERR_PTR(-ENODEV); + + device_link_add(dev, master->dart->dev, + DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); + + return &master->dart->iommu; +} + +static void apple_dart_x1n1_release_device(struct device *dev) +{ + kfree(dev_iommu_priv_get(dev)); +} + +static int apple_dart_x1n1_of_xlate(struct device *dev, + const struct of_phandle_args *args) +{ + struct apple_dart_x1n1_master *master = dev_iommu_priv_get(dev); + struct platform_device *iommu_pdev; + struct apple_dart_x1n1 *dart; + u32 sid; + + if (args->args_count != 1) + return -EINVAL; + sid = args->args[0]; + if (sid >= DART_X1N1_MAX_STREAMS) + return -EINVAL; + + iommu_pdev = of_find_device_by_node(args->np); + if (!iommu_pdev) + return -ENODEV; + dart = platform_get_drvdata(iommu_pdev); + put_device(&iommu_pdev->dev); + if (!dart) + return -ENODEV; + + if (!master) { + master = kzalloc(sizeof(*master), GFP_KERNEL); + if (!master) + return -ENOMEM; + master->dart = dart; + dev_iommu_priv_set(dev, master); + } else if (master->dart != dart) { + /* This driver forwards per (dart-id, sid); a device spanning + * two paravirt DARTs is not modelled. */ + return -EINVAL; + } + + if (sid >= dart->num_streams) + return -EINVAL; + + set_bit(sid, master->sids); + return 0; +} + +static bool apple_dart_x1n1_capable(struct device *dev, enum iommu_cap cap) +{ + return cap == IOMMU_CAP_CACHE_COHERENCY; +} + +static const struct iommu_ops apple_dart_x1n1_ops = { + .capable = apple_dart_x1n1_capable, + .domain_alloc_paging = apple_dart_x1n1_domain_alloc_paging, + .probe_device = apple_dart_x1n1_probe_device, + .release_device = apple_dart_x1n1_release_device, + .device_group = generic_device_group, + .of_xlate = apple_dart_x1n1_of_xlate, + .owner = THIS_MODULE, + .default_domain_ops = &(const struct iommu_domain_ops) { + .attach_dev = apple_dart_x1n1_attach_dev, + .map_pages = apple_dart_x1n1_map_pages, + .unmap_pages = apple_dart_x1n1_unmap_pages, + .iotlb_sync = apple_dart_x1n1_iotlb_sync, + .iova_to_phys = apple_dart_x1n1_iova_to_phys, + .free = apple_dart_x1n1_domain_free, + }, +}; + +static int apple_dart_x1n1_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_dart_x1n1 *dart; + u32 magic, features; + int ret; + + dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL); + if (!dart) + return -ENOMEM; + + dart->dev = dev; + mutex_init(&dart->init_lock); + spin_lock_init(&dart->batch_lock); + + dart->mmio = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(dart->mmio)) + return PTR_ERR(dart->mmio); + + if (of_property_read_u32(dev->of_node, "apple,x1n1-dart-id", + &dart->dart_id)) + return dev_err_probe(dev, -EINVAL, + "missing apple,x1n1-dart-id\n"); + if (dart->dart_id > 0xff) + return dev_err_probe(dev, -EINVAL, "dart-id out of range\n"); + + dart->num_streams = DART_X1N1_MAX_STREAMS; + of_property_read_u32(dev->of_node, "apple,x1n1-num-streams", + &dart->num_streams); + if (!dart->num_streams || dart->num_streams > DART_X1N1_MAX_STREAMS) + return dev_err_probe(dev, -EINVAL, "bad stream count\n"); + + /* Confirm we really are under x1n1: the emulated MAGIC register reads + * back "x1n1", and the DART feature bit is advertised. */ + magic = readl(dart->mmio + X1N1_MMIO_MAGIC); + features = readl(dart->mmio + X1N1_MMIO_FEATURES); + if (magic != (u32)X1N1_MAGIC || !(features & X1N1_FEAT_DART)) + return dev_err_probe(dev, -ENODEV, + "x1n1 paravirt IOMMU not present (magic 0x%x)\n", + magic); + + /* Batch buffer in the linear map so its guest PA is stable and x1n1 + * can read it by guest-physical address. */ + dart->batch = (void *)devm_get_free_pages(dev, GFP_KERNEL, 0); + if (!dart->batch) + return -ENOMEM; + dart->batch_pa = virt_to_phys(dart->batch); + + platform_set_drvdata(pdev, dart); + + ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart-x1n1.%s", + dev_name(dev)); + if (ret) + return ret; + + ret = iommu_device_register(&dart->iommu, &apple_dart_x1n1_ops, dev); + if (ret) + goto err_sysfs; + + dev_info(dev, "x1n1 paravirt DART id %u, %u streams (sptm2mmio)\n", + dart->dart_id, dart->num_streams); + return 0; + +err_sysfs: + iommu_device_sysfs_remove(&dart->iommu); + return ret; +} + +static void apple_dart_x1n1_remove(struct platform_device *pdev) +{ + struct apple_dart_x1n1 *dart = platform_get_drvdata(pdev); + + iommu_device_unregister(&dart->iommu); + iommu_device_sysfs_remove(&dart->iommu); +} + +static const struct of_device_id apple_dart_x1n1_of_match[] = { + { .compatible = "apple,dart-x1n1" }, + { }, +}; +MODULE_DEVICE_TABLE(of, apple_dart_x1n1_of_match); + +static struct platform_driver apple_dart_x1n1_driver = { + .driver = { + .name = "apple-dart-x1n1", + .of_match_table = apple_dart_x1n1_of_match, + }, + .probe = apple_dart_x1n1_probe, + .remove = apple_dart_x1n1_remove, +}; +module_platform_driver(apple_dart_x1n1_driver); + +MODULE_DESCRIPTION("Paravirtualised Apple DART for the x1n1 hypervisor"); +MODULE_LICENSE("GPL"); From 4d4bffc0fff90a2c7df8ae2039c87b577ffc1c89 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 26 Sep 2026 03:39:26 -0400 Subject: [PATCH 0028/1579] iommu/apple-dart-x1n1: attach L1 root + lazy L2, in-guest self-test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Installing a level-2 leaf table requires its level-1 parent present, so attach_dev now installs the level-1 root per stream (not a level-2 table) and enables translation; map_pages lazily attaches each 32 MiB window's level-2 table (tracked per-domain in an xarray, since re-attaching a table is fatal in SPTM) before mapping into it, in a phase that fully rings before any MAP ops are accumulated in the shared batch buffer. Add an optional DT-gated self-test (apple,x1n1-selftest) that drives the real batch path — INIT/POWERUP, attach L1+L2, ENABLE, MAP a freshly allocated guest page at the tested DVA, then UNMAP — and logs each batch status. Verified in QEMU on real J700 25G83 SPTM: booting this guest under x1n1, the self-test forwards the whole sequence to SPTM with no violation — DART table 3 endpoints INIT(6), POWERUP(5), MAP_TABLE(0) level 1 and 2, ENABLE(8), MAP(2) and UNMAP(3) all complete. A DMA-capable guest device and aligning the domain aperture to the DART's real DVA window (vm-base) remain for real DMA traffic. Assisted-by: Claude Opus 4.8 --- drivers/iommu/apple-dart-x1n1.c | 118 +++++++++++++++++++++++++++++--- 1 file changed, 110 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/apple-dart-x1n1.c b/drivers/iommu/apple-dart-x1n1.c index 72f68fdac6f2f..4f68c29b7a485 100644 --- a/drivers/iommu/apple-dart-x1n1.c +++ b/drivers/iommu/apple-dart-x1n1.c @@ -39,6 +39,7 @@ #include #include #include +#include /* sptm2mmio register/doorbell map, shared with x1n1 src/sptm.c. */ #define X1N1_MMIO_MAGIC 0x000 @@ -114,6 +115,10 @@ struct apple_dart_x1n1_domain { struct apple_dart_x1n1 *dart; DECLARE_BITMAP(sids, DART_X1N1_MAX_STREAMS); struct mutex lock; /* protects sids / dart binding */ + /* level-2 leaf tables already attached, keyed by (sid << 40 | block), + * block = dva >> 25. Installing a level-2 table twice is fatal in SPTM, + * so each 32 MiB window is attached at most once. */ + struct xarray l2_attached; }; static struct apple_dart_x1n1_domain *to_x1n1_domain(struct iommu_domain *dom) @@ -174,6 +179,60 @@ static int apple_dart_x1n1_ensure_init(struct apple_dart_x1n1 *dart) return ret; } +/* + * Optional in-guest self-test (DT "apple,x1n1-selftest"). Drives the real + * batch path — INIT/POWERUP, attach an L2 table + enable for one stream, MAP a + * freshly allocated guest page at a DVA, then UNMAP — and logs each batch's + * status. Proves the driver's forwarding reaches SPTM from within Linux. + */ +static void apple_dart_x1n1_selftest(struct apple_dart_x1n1 *dart) +{ + const u32 sid = 1; + const u64 dva = 0x10000004000ULL; /* the IOVA the transport was tested with */ + unsigned long flags; + void *page; + phys_addr_t pa; + int ret; + + if (apple_dart_x1n1_ensure_init(dart)) { + dev_err(dart->dev, "selftest: INIT failed\n"); + return; + } + + page = (void *)__get_free_page(GFP_KERNEL); + if (!page) + return; + pa = virt_to_phys(page); + + spin_lock_irqsave(&dart->batch_lock, flags); + /* Install the level-1 root, then the level-2 leaf table for this DVA's + * 32 MiB window (level 2 requires level 1 present), enable the stream, + * then map one page. */ + dart->batch->count = 4; + dart->batch->op[0] = (struct x1n1_op){ .op = X1N1_DART_ATTACH, + .arg = { sid, dva, 1 } }; + dart->batch->op[1] = (struct x1n1_op){ .op = X1N1_DART_ATTACH, + .arg = { sid, dva, 2 } }; + dart->batch->op[2] = (struct x1n1_op){ .op = X1N1_DART_ENABLE, + .arg = { sid } }; + dart->batch->op[3] = (struct x1n1_op){ .op = X1N1_DART_MAP, + .arg = { sid, dva, pa, + DART_X1N1_PAGE_SIZE, 3 } }; + ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + dev_info(dart->dev, "selftest: attach L1+L2+enable+map status %d done %u (dva %#llx pa %pa)\n", + ret, dart->batch->done, dva, &pa); + + dart->batch->count = 1; + dart->batch->op[0] = (struct x1n1_op){ .op = X1N1_DART_UNMAP, + .arg = { sid, dva, DART_X1N1_PAGE_SIZE } }; + ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + dev_info(dart->dev, "selftest: unmap status %d done %u\n", + ret, dart->batch->done); + spin_unlock_irqrestore(&dart->batch_lock, flags); + + free_page((unsigned long)page); +} + static int apple_dart_x1n1_attach_dev(struct iommu_domain *domain, struct device *dev, struct iommu_domain *old) @@ -201,9 +260,10 @@ static int apple_dart_x1n1_attach_dev(struct iommu_domain *domain, } dom->dart = dart; - /* For each stream: attach a level-2 table (the 32-bit aperture this - * driver advertises needs no level-0/1 root) and enable translation. - * x1n1 owns the tables. Build the ops in one batch. */ + /* For each stream: install the level-1 root table and enable translation. + * The level-2 leaf tables are attached lazily per 32 MiB window in + * map_pages (installing a level-2 table needs its level-1 parent + * present). x1n1 owns the tables. Build the ops in one batch. */ spin_lock_irqsave(&dart->batch_lock, flags); n = 0; for_each_set_bit(sid, master->sids, dart->num_streams) { @@ -213,7 +273,7 @@ static int apple_dart_x1n1_attach_dev(struct iommu_domain *domain, break; dart->batch->op[n++] = (struct x1n1_op){ .op = X1N1_DART_ATTACH, - .arg = { sid, 0, 2 }, + .arg = { sid, 0, 1 }, }; dart->batch->op[n++] = (struct x1n1_op){ .op = X1N1_DART_ENABLE, @@ -246,11 +306,46 @@ static int apple_dart_x1n1_map_pages(struct iommu_domain *domain, return -ENODEV; /* - * Forward the range in per-stream chunks that each stay inside one - * aligned 32 MiB block, matching x1n1's contiguous-map constraint. - * Pack as many MAP ops as fit into a batch, then ring once. + * A MAP needs the block's level-2 leaf table installed first, so first + * attach any not-yet-attached (sid, 32 MiB block) window, then map the + * range in per-stream chunks that each stay inside one aligned 32 MiB + * block (x1n1's contiguous-map constraint), packing MAP ops into a batch + * and ringing once per batch. Both phases hold batch_lock and reuse the + * one batch buffer, so the attaches are fully rung before any MAP ops + * are accumulated in it. */ spin_lock_irqsave(&dart->batch_lock, flags); + + /* Phase 1: attach missing level-2 tables (one op per ring). */ + for (u64 pos = 0; pos < total; ) { + u64 cur_iova = iova + pos; + u64 block = cur_iova >> DART_X1N1_BLOCK_SHIFT; + size_t block_left = DART_X1N1_BLOCK_SIZE - + (cur_iova & (DART_X1N1_BLOCK_SIZE - 1)); + unsigned int sid; + + for_each_set_bit(sid, dom->sids, dart->num_streams) { + unsigned long key = ((unsigned long)sid << 40) | block; + + if (xa_load(&dom->l2_attached, key)) + continue; + dart->batch->count = 1; + dart->batch->op[0] = (struct x1n1_op){ + .op = X1N1_DART_ATTACH, + .arg = { sid, cur_iova, 2 }, + }; + if (x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id)) { + *mapped = 0; + spin_unlock_irqrestore(&dart->batch_lock, flags); + return -ENOMEM; + } + xa_store(&dom->l2_attached, key, xa_mk_value(1), + GFP_ATOMIC); + } + pos += min_t(size_t, total - pos, block_left); + } + + /* Phase 2: map the range. */ while (done < total) { u32 n = 0; int ret; @@ -363,7 +458,10 @@ static phys_addr_t apple_dart_x1n1_iova_to_phys(struct iommu_domain *domain, static void apple_dart_x1n1_domain_free(struct iommu_domain *domain) { - kfree(to_x1n1_domain(domain)); + struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); + + xa_destroy(&dom->l2_attached); + kfree(dom); } static struct iommu_domain * @@ -376,6 +474,7 @@ apple_dart_x1n1_domain_alloc_paging(struct device *dev) return NULL; mutex_init(&dom->lock); + xa_init(&dom->l2_attached); dom->domain.pgsize_bitmap = DART_X1N1_PAGE_SIZE; dom->domain.geometry.aperture_start = 0; dom->domain.geometry.aperture_end = DART_X1N1_APERTURE_END; @@ -534,6 +633,9 @@ static int apple_dart_x1n1_probe(struct platform_device *pdev) dev_info(dev, "x1n1 paravirt DART id %u, %u streams (sptm2mmio)\n", dart->dart_id, dart->num_streams); + + if (of_property_read_bool(dev->of_node, "apple,x1n1-selftest")) + apple_dart_x1n1_selftest(dart); return 0; err_sysfs: From a7e6e4f7a1604918f45dc18e7abf6ab5913c4440 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 00:31:01 +1000 Subject: [PATCH 0029/1579] arm64: configs: asahi: enable the T8140 console, boot framebuffer and port controllers The MacBook Neo (J700, T8140) comes up from defconfig + asahi.config with no console, no display and no USB-C port controllers: - its serial console is a DockChannel UART (apple,dockchannel-uart, console=ttyDC0), so SERIAL_APPLE_DOCKCHANNEL and its early console have to be built in; - both USB-C ports are driven by SN201202x controllers on SPMI (apple,sn201202x); - the only display path on this tree is the simple-framebuffer the bootloader hands over, which needs simpledrm. All three were enabled in the kernel qualified on hardware on 2026-09-26 (source tree bac923ab, identical to this branch's base) and bound on that boot: ttyDC0 at 0x30812c000, Type-C port0 and port1 with PD contracts, simpledrm fb0. The other differences between that kernel's configuration and defconfig + asahi.config are lab and distribution choices -- drivers built in for the tethered initramfs, debug options, nftables, zram and Bluetooth profiles -- and stay out of this fragment. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 20a50b6fcb3a1..03b0a7cd2f075 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -32,6 +32,7 @@ CONFIG_DRM_ASAHI=m CONFIG_DRM_ADP=m CONFIG_DRM_APPLE=m CONFIG_DRM_APPLE_AUDIO=y +CONFIG_DRM_SIMPLEDRM=y CONFIG_SND_SOC_APPLE_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_MCA=m @@ -49,6 +50,7 @@ CONFIG_SPI_HID_APPLE_OF=m CONFIG_SPI_HID_APPLE_CORE=m CONFIG_USB_DWC3_APPLE=m CONFIG_USB_XHCI_PCI_ASMEDIA=y +CONFIG_TYPEC_SN201202X=m CONFIG_RTC_DRV_MACSMC=m CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m @@ -56,6 +58,8 @@ CONFIG_MFD_MACSMC=m CONFIG_COMMON_CLK_APPLE_NCO=m CONFIG_APPLE_DART=m CONFIG_APPLE_DOCKCHANNEL=m +CONFIG_SERIAL_APPLE_DOCKCHANNEL=y +CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y CONFIG_APPLE_MAILBOX=y CONFIG_APPLE_PMGR_MISC=y CONFIG_APPLE_RTKIT=y From bac0846c9087aa9016cabd58cf468963e2ebba83 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 9 Sep 2026 03:59:31 -0400 Subject: [PATCH 0030/1579] soc: apple: add Apple SEP driver (Touch ID and key sealing) Add a Rust driver for the Apple SEP (Secure Enclave Processor) on Apple silicon Macs, built as the CONFIG_APPLE_SEP module apple-sep. It attaches to a running SEP over the AP mailbox, drives the Touch ID fingerprint sensor, and exposes the enclave's key services to Linux. The driver is organised by SEP endpoint: - sep.rs transport, endpoint bring-up, mailbox and probe (crate root) - sbio.rs Touch ID: enrol, verify and match over /dev/sep-bio - sks.rs the key store: key-bag recovery and lock state - refkey.rs machine ref-key sealing, signing and attestation - fv.rs the device-bound FileVault key hierarchy - proto.rs the SEP wire protocol Touch ID enrol and verify are exposed through a /dev/sep-bio character device; the enclave matches and no biometric image ever crosses to userspace. A SEP-backed trusted key source lets keyctl seal keys to the enclave, which needs the trusted-keys core change included here to register a runtime trusted-key source. The C shims bridge kernel interfaces whose ABI is awkward to restate in Rust (struct hwrng, the file and crypto APIs, the trusted-key framework); no protocol logic lives in them. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/Kconfig | 24 +- drivers/soc/apple/Makefile | 4 +- drivers/soc/apple/bio.rs | 914 ++++++ drivers/soc/apple/bio_shim.c | 203 ++ drivers/soc/apple/catacomb.rs | 73 + drivers/soc/apple/control.rs | 183 ++ drivers/soc/apple/crypto_shim.c | 294 ++ drivers/soc/apple/der.rs | 158 + drivers/soc/apple/dt.rs | 407 +++ drivers/soc/apple/fv.rs | 158 + drivers/soc/apple/hwrng.rs | 81 + drivers/soc/apple/hwrng_shim.c | 72 + drivers/soc/apple/image.rs | 243 ++ drivers/soc/apple/keybag.rs | 293 ++ drivers/soc/apple/p256_shim.c | 164 + drivers/soc/apple/proto.rs | 301 ++ drivers/soc/apple/refkey.rs | 275 ++ drivers/soc/apple/refkey_seal.rs | 237 ++ drivers/soc/apple/rxring.rs | 78 + drivers/soc/apple/sbio.rs | 3316 +++++++++++++++++++++ drivers/soc/apple/scrd.rs | 144 + drivers/soc/apple/seed.rs | 113 + drivers/soc/apple/sensor.rs | 705 +++++ drivers/soc/apple/sensor_shim.c | 401 +++ drivers/soc/apple/sep-bio.h | 127 + drivers/soc/apple/sep.rs | 2413 +++++++++++++-- drivers/soc/apple/sha_shim.c | 44 + drivers/soc/apple/shim.h | 144 + drivers/soc/apple/shim.rs | 231 ++ drivers/soc/apple/shmem.rs | 235 ++ drivers/soc/apple/sks.rs | 1115 +++++++ drivers/soc/apple/store.rs | 334 +++ drivers/soc/apple/store_shim.c | 105 + drivers/soc/apple/transfer.rs | 389 +++ drivers/soc/apple/trusted.rs | 268 ++ drivers/soc/apple/trusted_shim.c | 176 ++ drivers/soc/apple/xarm.rs | 365 +++ include/keys/trusted-type.h | 3 + security/keys/trusted-keys/trusted_core.c | 140 +- 39 files changed, 14633 insertions(+), 297 deletions(-) create mode 100644 drivers/soc/apple/bio.rs create mode 100644 drivers/soc/apple/bio_shim.c create mode 100644 drivers/soc/apple/catacomb.rs create mode 100644 drivers/soc/apple/control.rs create mode 100644 drivers/soc/apple/crypto_shim.c create mode 100644 drivers/soc/apple/der.rs create mode 100644 drivers/soc/apple/dt.rs create mode 100644 drivers/soc/apple/fv.rs create mode 100644 drivers/soc/apple/hwrng.rs create mode 100644 drivers/soc/apple/hwrng_shim.c create mode 100644 drivers/soc/apple/image.rs create mode 100644 drivers/soc/apple/keybag.rs create mode 100644 drivers/soc/apple/p256_shim.c create mode 100644 drivers/soc/apple/proto.rs create mode 100644 drivers/soc/apple/refkey.rs create mode 100644 drivers/soc/apple/refkey_seal.rs create mode 100644 drivers/soc/apple/rxring.rs create mode 100644 drivers/soc/apple/sbio.rs create mode 100644 drivers/soc/apple/scrd.rs create mode 100644 drivers/soc/apple/seed.rs create mode 100644 drivers/soc/apple/sensor.rs create mode 100644 drivers/soc/apple/sensor_shim.c create mode 100644 drivers/soc/apple/sep-bio.h create mode 100644 drivers/soc/apple/sha_shim.c create mode 100644 drivers/soc/apple/shim.h create mode 100644 drivers/soc/apple/shim.rs create mode 100644 drivers/soc/apple/shmem.rs create mode 100644 drivers/soc/apple/sks.rs create mode 100644 drivers/soc/apple/store.rs create mode 100644 drivers/soc/apple/store_shim.c create mode 100644 drivers/soc/apple/transfer.rs create mode 100644 drivers/soc/apple/trusted.rs create mode 100644 drivers/soc/apple/trusted_shim.c create mode 100644 drivers/soc/apple/xarm.rs diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 8c920dca50ce5..22cb2ba19b0cf 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -83,6 +83,7 @@ config RUST_APPLE_RTKIT depends on PM depends on RUST select APPLE_RTKIT + select RUST_APPLE_MAILBOX config APPLE_AOP tristate "Apple \"Always-on\" Processor" @@ -99,17 +100,24 @@ config APPLE_AOP Say 'y' here if you have an Apple laptop. config APPLE_SEP - tristate "Apple Secure Element Processor" - depends on ARCH_APPLE || COMPILE_TEST - depends on PM + tristate "Apple SEP (Secure Enclave Processor)" + depends on ARCH_APPLE depends on RUST - select RUST_APPLE_RTKIT - select RUST_APPLE_MAILBOX + depends on HW_RANDOM + depends on TRUSTED_KEYS + depends on CRYPTO + select CRYPTO_HMAC + select CRYPTO_SHA256 + select CRYPTO_AES + select CRYPTO_GCM + select CRYPTO_ECDH help - A security co-processor persent on Apple SoCs, controlling transparent - disk encryption, secure boot, HDCP, biometric auth and probably more. + Driver for the Apple SEP (Secure Enclave Processor) on Apple silicon. + It drives the Touch ID fingerprint sensor over a /dev/sep-bio character + device (enrol and match), exposes the SEP hardware RNG, and registers a + SEP-backed trusted key source so keyctl can seal keys to the enclave. - Say 'y' here if you have an Apple SoC. + Say Y here if you have an Apple silicon Mac. config APPLE_PMP tristate "Apple Power Management Processor" diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 1baa6de944958..ab67304c059e6 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -22,6 +22,8 @@ apple-tunable-y = tunable.o obj-$(CONFIG_APPLE_AOP) += aop.o -obj-$(CONFIG_APPLE_SEP) += sep.o +obj-$(CONFIG_APPLE_SEP) += apple-sep.o +apple-sep-y := sep.o hwrng_shim.o store_shim.o bio_shim.o sha_shim.o \ + crypto_shim.o sensor_shim.o p256_shim.o trusted_shim.o obj-$(CONFIG_APPLE_PMP) += pmp.o diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs new file mode 100644 index 0000000000000..27d7bd25d1110 --- /dev/null +++ b/drivers/soc/apple/bio.rs @@ -0,0 +1,914 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +use crate::store::{Key, Store}; +use kernel::ioctl::{_IO, _IOR, _IOW, _IOWR}; +use kernel::prelude::*; +use kernel::uaccess::{UserPtr, UserSlice}; + +pub(crate) const IFACE_VERSION: u32 = 4; + +pub(crate) const ENROL_USER_ID: i32 = 1000; + +pub(crate) const UUID_LEN: usize = 16; +const LABEL_LEN: usize = 128; +const NONCE_LEN: usize = 32; +pub(crate) const TOKEN_LEN: usize = 32; +const MAX_IDENTITIES: usize = 32; + +pub(crate) const DEVICE_NAME: &CStr = c"sep-bio"; +pub(crate) const DEVICE_MODE: u16 = 0o600; + +#[derive(Clone, Copy, Default)] +#[repr(u32)] +enum State { + #[default] + Idle = 0, + Pending = 1, + Progress = 2, + Done = 3, + Failed = 4, +} + +#[derive(Clone, Copy, Default, PartialEq)] +#[repr(u32)] +pub(crate) enum Guidance { + #[default] + None = 0, + Place = 1, + LiftAndMove = 2, + HoldStill = 3, +} + +#[derive(Clone, Copy, Default)] +#[repr(u32)] +enum MatchResult { + #[default] + NoMatch = 0, + Match = 1, + NotCompared = 2, +} + +pub(crate) const ENROL_STAGES: u32 = 8; + +const TOKEN_LIFETIME_NS: u64 = 10 * 1_000_000_000; + +const SUSPEND_SLACK_NS: u64 = 1_000_000; + +const STATUS_LOCAL: u32 = 0; + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct Identity { + uuid: [u8; UUID_LEN], + label: [u8; LABEL_LEN], +} + +impl Identity { + const ZERO: Identity = Identity { + uuid: [0; UUID_LEN], + label: [0; LABEL_LEN], + }; +} + +#[repr(C)] +#[derive(Default)] +struct Info { + version: u32, + sensor_present: u32, + enrolled: u32, + capacity: u32, + enroll_stages: u32, + reserved: [u32; 3], +} + +#[repr(C)] +struct List { + count: u32, + reserved: u32, + id: [Identity; MAX_IDENTITIES], +} + +#[repr(C)] +struct EnrolStart { + flags: u32, + reserved: u32, + label: [u8; LABEL_LEN], +} + +#[repr(C)] +#[derive(Default)] +struct EnrolPoll { + state: State, + stage: u32, + stages_total: u32, + status: u32, + uuid: [u8; UUID_LEN], + guidance: Guidance, + progress_percent: u32, +} + +#[repr(C)] +struct VerifyStart { + flags: u32, + reserved: u32, + nonce: [u8; NONCE_LEN], +} + +#[repr(C)] +#[derive(Default)] +struct VerifyPoll { + state: State, + result: MatchResult, + status: u32, + reserved: u32, + uuid: [u8; UUID_LEN], + token: [u8; TOKEN_LEN], + deadline_ns: u64, +} + +#[repr(C)] +struct Delete { + uuid: [u8; UUID_LEN], +} + +pub(crate) const ATTEST_CHALLENGE_LEN: usize = 32; +/// Uncompressed P-256 public point (`04‖X‖Y`). +pub(crate) const ATTEST_PUB_LEN: usize = 65; +/// Max DER `SEQUENCE { INTEGER r, INTEGER s }` for P-256. +pub(crate) const ATTEST_SIG_MAX: usize = 72; + +#[repr(C)] +pub(crate) struct Attest { + pub(crate) sig_len: u32, + /// in: challenge to sign. + pub(crate) challenge: [u8; ATTEST_CHALLENGE_LEN], + /// out: ref-key public point. + pub(crate) public: [u8; ATTEST_PUB_LEN], + /// out: DER signature, zero-padded to `ATTEST_SIG_MAX`. + pub(crate) signature: [u8; ATTEST_SIG_MAX], + pub(crate) reserved: [u8; 3], +} + +// Sizes are baked into the ioctl numbers. +static_assert!(core::mem::size_of::() == 144); +static_assert!(core::mem::size_of::() == 32); +static_assert!(core::mem::size_of::() == 4616); +static_assert!(core::mem::size_of::() == 136); +static_assert!(core::mem::size_of::() == 40); +static_assert!(core::mem::size_of::() == 40); +static_assert!(core::mem::size_of::() == 72); +static_assert!(core::mem::size_of::() == 16); +static_assert!(core::mem::size_of::() == 176); + +macro_rules! ioctl_pod { + (to_user: $($t:ty),*; from_user: $($u:ty),*) => { + $( + // SAFETY: no padding, so the value is a faithful byte image. + unsafe impl kernel::transmute::AsBytes for $t {} + )* + $( + // SAFETY: every bit pattern of the members is a valid value. + unsafe impl kernel::transmute::FromBytes for $u {} + )* + }; +} +ioctl_pod! { + to_user: Info, List, EnrolPoll, VerifyPoll, Attest; + from_user: EnrolStart, VerifyStart, Delete, Attest +} + +const MAGIC: u32 = 0xB1; + +const IOC_GET_INFO: u32 = _IOR::(MAGIC, 0x01); +const IOC_LIST: u32 = _IOR::(MAGIC, 0x02); +const IOC_ENROL_START: u32 = _IOW::(MAGIC, 0x03); +const IOC_ENROL_POLL: u32 = _IOR::(MAGIC, 0x04); +const IOC_VERIFY_START: u32 = _IOW::(MAGIC, 0x05); +const IOC_VERIFY_POLL: u32 = _IOR::(MAGIC, 0x06); +const IOC_CANCEL: u32 = _IO(MAGIC, 0x07); +const IOC_DELETE: u32 = _IOW::(MAGIC, 0x08); +const IOC_DELETE_ALL: u32 = _IO(MAGIC, 0x09); +pub(crate) const IOC_ATTEST: u32 = _IOWR::(MAGIC, 0x0a); + +pub(crate) struct MatchEvidence { + identity: [u8; UUID_LEN], +} + +impl MatchEvidence { + pub(crate) fn from_enclave_reply(identity: [u8; UUID_LEN]) -> MatchEvidence { + MatchEvidence { identity } + } +} + +pub(crate) enum VerifyOutcome { + Failed(u32), + NoMatch, + Matched(MatchEvidence), +} + +struct ResultToken { + bytes: [u8; TOKEN_LEN], + nonce: [u8; NONCE_LEN], + identity: [u8; UUID_LEN], + deadline_ns: u64, + minted_mono_ns: u64, + minted_boot_ns: u64, +} + +impl ResultToken { + fn mint( + evidence: &MatchEvidence, + nonce: &[u8; NONCE_LEN], + bytes: [u8; TOKEN_LEN], + ) -> ResultToken { + let mono = crate::shim::monotonic_ns(); + ResultToken { + bytes, + nonce: *nonce, + identity: evidence.identity, + deadline_ns: mono.saturating_add(TOKEN_LIFETIME_NS), + minted_mono_ns: mono, + minted_boot_ns: crate::shim::boottime_ns(), + } + } + + fn usable(&self, nonce: &[u8; NONCE_LEN], identity: &[u8; UUID_LEN]) -> bool { + let mono = crate::shim::monotonic_ns(); + if mono > self.deadline_ns { + return false; + } + let suspended = crate::shim::boottime_ns().saturating_sub(self.minted_boot_ns); + let awake = mono.saturating_sub(self.minted_mono_ns); + if suspended.saturating_sub(awake) > SUSPEND_SLACK_NS { + return false; + } + self.nonce == *nonce && self.identity == *identity + } +} + +const PRIVATE_TYPE_IDENTITIES: u8 = 0xF1; + +fn identity_key() -> Key { + Key::root(PRIVATE_TYPE_IDENTITIES) +} + +const INDEX_FORMAT: u32 = 2; + +pub(crate) struct IdentityIndex { + entries: KVec, +} + +impl IdentityIndex { + pub(crate) fn new() -> IdentityIndex { + IdentityIndex { + entries: KVec::new(), + } + } + + pub(crate) fn load(store: &mut Store) -> Result { + let mut index = IdentityIndex::new(); + let Some(raw) = store.read(&identity_key())? else { + return Ok(index); + }; + if raw.len() < 4 { + return Ok(index); + } + if u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]]) != INDEX_FORMAT { + return Err(EINVAL); + } + if raw.len() < 8 { + return Ok(index); + } + let count = u32::from_le_bytes([raw[4], raw[5], raw[6], raw[7]]) as usize; + let stride = UUID_LEN + LABEL_LEN; + for i in 0..count.min(MAX_IDENTITIES) { + let at = 8 + i * stride; + if at + stride > raw.len() { + break; + } + let mut entry = Identity::ZERO; + entry.uuid.copy_from_slice(&raw[at..at + UUID_LEN]); + entry + .label + .copy_from_slice(&raw[at + UUID_LEN..at + stride]); + index.entries.push(entry, GFP_KERNEL)?; + } + Ok(index) + } + + fn save(&self, store: &mut Store) -> Result<()> { + let mut raw = KVec::new(); + raw.extend_from_slice(&INDEX_FORMAT.to_le_bytes(), GFP_KERNEL)?; + raw.extend_from_slice(&(self.entries.len() as u32).to_le_bytes(), GFP_KERNEL)?; + for entry in self.entries.iter() { + raw.extend_from_slice(&entry.uuid, GFP_KERNEL)?; + raw.extend_from_slice(&entry.label, GFP_KERNEL)?; + } + store.write(&identity_key(), &raw) + } + + pub(crate) fn total(&self) -> usize { + self.entries.len() + } + + pub(crate) fn insert(&mut self, id: Identity) -> Result<()> { + for existing in self.entries.iter_mut() { + if existing.uuid == id.uuid { + *existing = id; + return Ok(()); + } + } + if self.entries.len() >= MAX_IDENTITIES { + return Err(ENOSPC); + } + self.entries.push(id, GFP_KERNEL)?; + Ok(()) + } + + pub(crate) fn persist(&self, store: &mut Store) -> Result<()> { + self.save(store) + } + + pub(crate) fn identity_v1_for( + &self, + uuid: &[u8; UUID_LEN], + user_id: i32, + ) -> Option { + if !self.contains(uuid) { + return None; + } + Some(crate::sbio::IdentityV1::from_index(user_id, *uuid)) + } + + pub(crate) fn reconcile_to( + &mut self, + listed: &[[u8; UUID_LEN]], + ) -> Result> { + let mut dropped = KVec::new(); + for entry in self.entries.iter() { + if !listed.contains(&entry.uuid) { + dropped.push(entry.uuid, GFP_KERNEL)?; + } + } + self.entries.retain(|e| listed.contains(&e.uuid)); + for uuid in listed.iter() { + if !self.contains(uuid) { + self.insert(Identity { + uuid: *uuid, + label: [0u8; LABEL_LEN], + })?; + } + } + Ok(dropped) + } + + pub(crate) fn remove(&mut self, uuid: &[u8; UUID_LEN]) { + self.entries.retain(|e| e.uuid != *uuid); + } + + pub(crate) fn contains_uuid(&self, uuid: &[u8; UUID_LEN]) -> bool { + self.contains(uuid) + } + + fn contains(&self, uuid: &[u8; UUID_LEN]) -> bool { + self.entries.iter().any(|e| e.uuid == *uuid) + } +} + +enum Op { + Idle, + Enrol { + label: [u8; LABEL_LEN], + stage: u32, + terminal: Option, + guidance: Guidance, + percent: u32, + }, + Verify { + nonce: [u8; NONCE_LEN], + terminal: Option, + }, +} + +enum EnrolOutcome { + Failed(u32), + Done([u8; UUID_LEN]), +} + +impl Op { + fn is_terminal(&self) -> bool { + match self { + Op::Idle => false, + Op::Enrol { terminal, .. } => terminal.is_some(), + Op::Verify { terminal, .. } => terminal.is_some(), + } + } + + fn may_start(&self) -> bool { + matches!(self, Op::Idle) || self.is_terminal() + } +} + +pub(crate) struct Session { + open: bool, + op: Op, + unseen: bool, + token: Option, +} + +impl Session { + pub(crate) fn new() -> Session { + Session { + open: false, + op: Op::Idle, + unseen: false, + token: None, + } + } + + fn reset(&mut self) { + self.open = false; + self.op = Op::Idle; + self.unseen = false; + self.token = None; + } + + pub(crate) fn ready(&self) -> bool { + self.unseen + } +} + +pub(crate) struct Context<'a> { + pub(crate) session: &'a mut Session, + pub(crate) index: &'a IdentityIndex, + pub(crate) sensor_present: bool, +} + +pub(crate) struct Handled { + pub(crate) ret: isize, + pub(crate) wake: bool, + pub(crate) start_enrol: bool, + pub(crate) start_verify: bool, + pub(crate) delete_identity: Option, + pub(crate) delete_identities: KVec, +} + +fn ok() -> Result { + Ok(Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }) +} + +fn require_admin() -> Result<()> { + if crate::shim::capable_admin() { + Ok(()) + } else { + Err(EPERM) + } +} + +fn require_sensor(present: bool) -> Result<()> { + if present { + Ok(()) + } else { + Err(ENODEV) + } +} + +pub(crate) fn ioctl(ctx: &mut Context<'_>, cmd: u32, arg: usize) -> Result { + let user = UserPtr::from_addr(arg); + + match cmd { + IOC_GET_INFO => get_info(ctx, user), + IOC_LIST => list(ctx, user), + IOC_ENROL_START => enrol_start(ctx, user), + IOC_ENROL_POLL => enrol_poll(ctx, user), + IOC_VERIFY_START => verify_start(ctx, user), + IOC_VERIFY_POLL => verify_poll(ctx, user), + IOC_CANCEL => cancel(ctx), + IOC_DELETE => delete(ctx, user), + IOC_DELETE_ALL => delete_all(ctx), + _ => Err(ENOTTY), + } +} + +fn get_info(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let info = Info { + version: IFACE_VERSION, + sensor_present: u32::from(ctx.sensor_present), + enrolled: ctx.index.total() as u32, + capacity: MAX_IDENTITIES as u32, + enroll_stages: ENROL_STAGES, + reserved: [0; 3], + }; + UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&info)?; + ok() +} + +fn list(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let mut out = List { + count: 0, + reserved: 0, + id: [Identity::ZERO; MAX_IDENTITIES], + }; + for entry in ctx.index.entries.iter() { + if out.count as usize >= MAX_IDENTITIES { + break; + } + out.id[out.count as usize] = *entry; + out.count += 1; + } + + UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&out)?; + ok() +} + +fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { + require_admin()?; + let request: EnrolStart = UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; + if request.flags != 0 || request.reserved != 0 { + return Err(EINVAL); + } + require_sensor(ctx.sensor_present)?; + + if !ctx.session.op.may_start() { + return Err(EBUSY); + } + if ctx.index.total() >= MAX_IDENTITIES { + return Err(ENOSPC); + } + + ctx.session.op = Op::Enrol { + label: request.label, + stage: 0, + terminal: None, + guidance: Guidance::Place, + percent: 0, + }; + Ok(Handled { + ret: 0, + wake: false, + start_enrol: true, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }) +} + +pub(crate) fn enrol_advance( + session: &mut Session, + completed: u32, + percent_now: u32, + guide: Guidance, +) -> bool { + if let Op::Enrol { + stage, + terminal, + guidance, + percent, + .. + } = &mut session.op + { + if terminal.is_none() { + *stage = completed; + *percent = percent_now; + *guidance = guide; + session.unseen = true; + return true; + } + } + false +} + +pub(crate) fn enrol_guide(session: &mut Session, guide: Guidance) -> bool { + if let Op::Enrol { + terminal, guidance, .. + } = &mut session.op + { + if terminal.is_none() && *guidance != guide { + *guidance = guide; + session.unseen = true; + return true; + } + } + false +} + +pub(crate) fn enrol_finish( + session: &mut Session, + index: &mut IdentityIndex, + outcome: core::result::Result<[u8; UUID_LEN], u32>, +) -> bool { + let Op::Enrol { + label, terminal, .. + } = &mut session.op + else { + return false; + }; + if terminal.is_some() { + return false; + } + match outcome { + Ok(uuid) => { + match index.insert(Identity { + uuid, + label: *label, + }) { + Ok(()) => *terminal = Some(EnrolOutcome::Done(uuid)), + Err(_) => *terminal = Some(EnrolOutcome::Failed(STATUS_LOCAL)), + } + } + Err(status) => *terminal = Some(EnrolOutcome::Failed(status)), + } + session.unseen = true; + true +} + +pub(crate) fn enrol_is_live(session: &Session) -> bool { + matches!(&session.op, Op::Enrol { terminal: None, .. }) && session.open +} + +fn enrol_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let mut out = EnrolPoll { + stages_total: ENROL_STAGES, + ..EnrolPoll::default() + }; + + match &ctx.session.op { + Op::Idle => out.state = State::Idle, + Op::Verify { .. } => return Err(EBUSY), + Op::Enrol { + stage, + terminal, + guidance, + percent, + .. + } => { + out.stage = *stage; + out.guidance = *guidance; + out.progress_percent = *percent; + match terminal { + None => { + out.state = if ctx.session.unseen { + State::Progress + } else { + State::Pending + } + } + Some(EnrolOutcome::Failed(status)) => { + out.state = State::Failed; + out.status = *status; + } + Some(EnrolOutcome::Done(uuid)) => { + out.state = State::Done; + out.uuid = *uuid; + } + } + } + } + + UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&out)?; + ctx.session.unseen = false; + ok() +} + +fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let request: VerifyStart = UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; + if request.flags != 0 || request.reserved != 0 { + return Err(EINVAL); + } + require_sensor(ctx.sensor_present)?; + + if !ctx.session.op.may_start() { + return Err(EBUSY); + } + + if ctx.index.total() == 0 { + ctx.session.token = None; + ctx.session.op = Op::Verify { + nonce: request.nonce, + terminal: Some(VerifyOutcome::NoMatch), + }; + return Ok(Handled { + ret: 0, + wake: true, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }); + } + + ctx.session.token = None; + ctx.session.op = Op::Verify { + nonce: request.nonce, + terminal: None, + }; + Ok(Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: true, + delete_identity: None, + delete_identities: KVec::new(), + }) +} + +pub(crate) fn verify_finish( + session: &mut Session, + outcome: VerifyOutcome, + token_bytes: [u8; TOKEN_LEN], +) -> bool { + let Op::Verify { nonce, terminal } = &mut session.op else { + return false; + }; + if terminal.is_some() { + return false; + } + + let minted = match &outcome { + VerifyOutcome::Matched(evidence) => Some(ResultToken::mint(evidence, nonce, token_bytes)), + VerifyOutcome::NoMatch | VerifyOutcome::Failed(_) => None, + }; + + *terminal = Some(outcome); + session.token = minted; + session.unseen = true; + true +} + +pub(crate) fn verify_is_live(session: &Session) -> bool { + matches!(&session.op, Op::Verify { terminal: None, .. }) && session.open +} + +pub(crate) fn capture_is_live(session: &Session) -> bool { + enrol_is_live(session) || verify_is_live(session) +} + +fn verify_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let mut out = VerifyPoll::default(); + // Spend the token only after the copy to user succeeds; a fault must not lose a match. + let mut consume_token = false; + + match &ctx.session.op { + Op::Idle => out.state = State::Idle, + Op::Enrol { .. } => return Err(EBUSY), + Op::Verify { nonce, terminal } => match terminal { + None => { + out.state = if ctx.session.unseen { + State::Progress + } else { + State::Pending + } + } + Some(VerifyOutcome::Failed(status)) => { + out.state = State::Failed; + out.result = MatchResult::NotCompared; + out.status = *status; + } + Some(VerifyOutcome::NoMatch) => { + out.state = State::Done; + out.result = MatchResult::NoMatch; + } + Some(VerifyOutcome::Matched(evidence)) => { + out.state = State::Done; + match ctx.session.token.as_ref() { + Some(token) if token.usable(nonce, &evidence.identity) => { + out.result = MatchResult::Match; + out.uuid = evidence.identity; + out.token = token.bytes; + out.deadline_ns = token.deadline_ns; + consume_token = true; + } + _ => { + out.result = MatchResult::NotCompared; + } + } + } + }, + } + + UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&out)?; + if consume_token { + ctx.session.token = None; + } + ctx.session.unseen = false; + ok() +} + +fn cancel(ctx: &mut Context<'_>) -> Result { + if ctx.session.op.is_terminal() { + return Err(ENOENT); + } + + match &mut ctx.session.op { + Op::Idle => return Err(ENOENT), + Op::Enrol { terminal, .. } => *terminal = Some(EnrolOutcome::Failed(STATUS_LOCAL)), + Op::Verify { terminal, .. } => *terminal = Some(VerifyOutcome::Failed(STATUS_LOCAL)), + } + + ctx.session.token = None; + ctx.session.unseen = true; + + Ok(Handled { + ret: 0, + wake: true, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }) +} + +fn delete(ctx: &mut Context<'_>, user: UserPtr) -> Result { + require_admin()?; + let request: Delete = UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; + + if !ctx.index.contains(&request.uuid) { + pr_info!( + "sep_bio: DELETE of an identity not held; reporting success (nothing to delete)\n" + ); + return ok(); + } + + // `0x57` takes one `identity_v1_t` (signed user id + 16-byte UUID). + let Some(identity) = ctx.index.identity_v1_for(&request.uuid, ENROL_USER_ID) else { + pr_warn!( + "sep_bio: DELETE of a held identity could not be expressed as an identity_v1_t\n" + ); + return Err(ENOENT); + }; + + Ok(Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: false, + delete_identity: Some(identity), + delete_identities: KVec::new(), + }) +} + +fn delete_all(ctx: &mut Context<'_>) -> Result { + require_admin()?; + + if ctx.index.total() == 0 { + pr_info!( + "sep_bio: DELETE_ALL on empty index; reporting success (nothing to delete)\n" + ); + return ok(); + } + + let mut ids = KVec::new(); + for entry in ctx.index.entries.iter() { + let Some(identity) = ctx.index.identity_v1_for(&entry.uuid, ENROL_USER_ID) else { + pr_warn!( + "sep_bio: DELETE_ALL of a held identity could not be expressed as an identity_v1_t\n" + ); + return Err(ENOENT); + }; + ids.push(identity, GFP_KERNEL)?; + } + + Ok(Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: ids, + }) +} + +pub(crate) fn open(session: &mut Session) -> Result<()> { + if session.open { + return Err(EBUSY); + } + session.reset(); + session.open = true; + Ok(()) +} + +pub(crate) fn release(session: &mut Session) { + session.reset(); +} diff --git a/drivers/soc/apple/bio_shim.c b/drivers/soc/apple/bio_shim.c new file mode 100644 index 0000000000000..50f7e7ddf61ea --- /dev/null +++ b/drivers/soc/apple/bio_shim.c @@ -0,0 +1,203 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +struct sep_bio_chardev { + struct miscdevice misc; + wait_queue_head_t wq; + struct mutex lock; + void *ctx; /* NULL once detached */ + int (*f_open)(void *ctx); + void (*f_release)(void *ctx); + long (*f_ioctl)(void *ctx, unsigned int cmd, unsigned long arg); + int (*f_ready)(void *ctx); + bool registered; + int open_count; +}; + +static struct sep_bio_chardev *from_file(struct file *file) +{ + return container_of(file->private_data, struct sep_bio_chardev, misc); +} + +static int sep_bio_fop_open(struct inode *inode, struct file *file) +{ + struct sep_bio_chardev *d = from_file(file); + int ret; + + mutex_lock(&d->lock); + if (!d->ctx) { + mutex_unlock(&d->lock); + return -ENODEV; + } + ret = d->f_open(d->ctx); + if (!ret) + d->open_count++; + mutex_unlock(&d->lock); + + return ret; +} + +static int sep_bio_fop_release(struct inode *inode, struct file *file) +{ + struct sep_bio_chardev *d = from_file(file); + bool last; + + mutex_lock(&d->lock); + if (d->ctx) + d->f_release(d->ctx); + d->open_count--; + last = (d->open_count == 0) && !d->ctx; + mutex_unlock(&d->lock); + + /* Detached while this file was open: nobody else can reach it now. */ + if (last) { + mutex_destroy(&d->lock); + kfree(d); + } + + return 0; +} + +static long sep_bio_fop_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct sep_bio_chardev *d = from_file(file); + long ret; + + mutex_lock(&d->lock); + ret = d->ctx ? d->f_ioctl(d->ctx, cmd, arg) : -ENODEV; + mutex_unlock(&d->lock); + + return ret; +} + +/* + * Readable when a *_POLL ioctl has something the caller has not seen yet; quiet + * once that ioctl consumes it. + */ +static __poll_t sep_bio_fop_poll(struct file *file, + struct poll_table_struct *wait) +{ + struct sep_bio_chardev *d = from_file(file); + + __poll_t mask; + + poll_wait(file, &d->wq, wait); + + mutex_lock(&d->lock); + if (!d->ctx) + mask = EPOLLERR | EPOLLHUP; + else + mask = d->f_ready(d->ctx) ? (EPOLLIN | EPOLLRDNORM) : 0; + mutex_unlock(&d->lock); + + return mask; +} + +static const struct file_operations sep_bio_fops = { + .owner = THIS_MODULE, + .open = sep_bio_fop_open, + .release = sep_bio_fop_release, + .unlocked_ioctl = sep_bio_fop_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .poll = sep_bio_fop_poll, +}; + +/* @name must outlive the registration; the Rust side passes a &'static CStr. */ +void *sep_bio_register(const char *name, unsigned short mode, void *ctx, + int (*f_open)(void *), + void (*f_release)(void *), + long (*f_ioctl)(void *, unsigned int, unsigned long), + int (*f_ready)(void *)) +{ + struct sep_bio_chardev *d; + int ret; + + d = kzalloc(sizeof(*d), GFP_KERNEL); + if (!d) + return NULL; + + init_waitqueue_head(&d->wq); + mutex_init(&d->lock); + d->ctx = ctx; + d->f_open = f_open; + d->f_release = f_release; + d->f_ioctl = f_ioctl; + d->f_ready = f_ready; + + d->misc.minor = MISC_DYNAMIC_MINOR; + d->misc.name = name; + d->misc.fops = &sep_bio_fops; + d->misc.mode = mode; + + ret = misc_register(&d->misc); + if (ret) { + mutex_destroy(&d->lock); + kfree(d); + return NULL; + } + d->registered = true; + + return d; +} + +void sep_bio_unregister(void *dev) +{ + struct sep_bio_chardev *d = dev; + bool free_now; + + if (!d) + return; + + if (d->registered) + misc_deregister(&d->misc); + + mutex_lock(&d->lock); + d->ctx = NULL; + free_now = (d->open_count == 0); + mutex_unlock(&d->lock); + + wake_up_interruptible(&d->wq); + + if (free_now) { + mutex_destroy(&d->lock); + kfree(d); + } +} + +void sep_bio_wake(void *dev) +{ + struct sep_bio_chardev *d = dev; + + if (d) + wake_up_interruptible(&d->wq); +} + +/* In C so the CAP_SYS_ADMIN number stays a kernel header constant, not a Rust literal. */ +int sep_bio_capable_admin(void) +{ + return capable(CAP_SYS_ADMIN) ? 1 : 0; +} + +__u64 sep_bio_monotonic_ns(void) +{ + return ktime_get_ns(); +} + +__u64 sep_bio_boottime_ns(void) +{ + return ktime_get_boottime_ns(); +} diff --git a/drivers/soc/apple/catacomb.rs b/drivers/soc/apple/catacomb.rs new file mode 100644 index 0000000000000..8b18efb4ead77 --- /dev/null +++ b/drivers/soc/apple/catacomb.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! File-backed persistence for the SEP catacombs. +//! +//! A catacomb is the opaque identity blob the enclave returns from `0x6c`; each +//! lives in its own variable-length file, too large for the block store. + +use crate::shim; +use crate::store::crc16_ccitt_false; +use kernel::prelude::*; + +const MAGIC: [u8; 8] = *b"SEPCTMB1"; +/// magic(8) + len(4, LE) + crc(2, LE) + pad(2) +const HEADER: usize = 16; +const MAX_CATACOMB: usize = 1 << 20; + +fn path_for(kind: u8) -> Option<&'static CStr> { + Some(match kind { + crate::PRIVATE_TYPE_CATACOMB_MASTER => c"/var/lib/apple-sep-catacomb-master.bin", + crate::PRIVATE_TYPE_CATACOMB_OWNER => c"/var/lib/apple-sep-catacomb-owner.bin", + crate::PRIVATE_TYPE_CATACOMB_USER => c"/var/lib/apple-sep-catacomb-user.bin", + _ => return None, + }) +} + +pub(crate) fn is_kind(kind: u8) -> bool { + path_for(kind).is_some() +} + +pub(crate) fn write(kind: u8, blob: &[u8]) -> Result<()> { + let path = path_for(kind).ok_or(EINVAL)?; + if blob.len() > MAX_CATACOMB { + return Err(ENOSPC); + } + let mut head = [0u8; HEADER]; + head[..MAGIC.len()].copy_from_slice(&MAGIC); + head[8..12].copy_from_slice(&(blob.len() as u32).to_le_bytes()); + head[12..14].copy_from_slice(&crc16_ccitt_false(blob).to_le_bytes()); + + let file = shim::StoreFile::open_trunc(path)?; + file.write_all(0, &head)?; + if !blob.is_empty() { + file.write_all(HEADER as u64, blob)?; + } + file.sync() +} + +pub(crate) fn read(kind: u8) -> Option> { + let path = path_for(kind)?; + let file = shim::StoreFile::open_readonly(path).ok()?; + let size = file.size().ok()?; + if size < HEADER as u64 { + return None; + } + let mut head = [0u8; HEADER]; + file.read_exact(0, &mut head).ok()?; + if head[..MAGIC.len()] != MAGIC { + return None; + } + let len = u32::from_le_bytes([head[8], head[9], head[10], head[11]]) as usize; + let crc = u16::from_le_bytes([head[12], head[13]]); + if len == 0 || len > MAX_CATACOMB || size != (HEADER + len) as u64 { + return None; + } + let mut blob: KVec = KVec::with_capacity(len, GFP_KERNEL).ok()?; + blob.resize(len, 0, GFP_KERNEL).ok()?; + file.read_exact(HEADER as u64, &mut blob).ok()?; + if crc16_ccitt_false(&blob) != crc { + return None; + } + Some(blob) +} diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs new file mode 100644 index 0000000000000..be81aa1de38df --- /dev/null +++ b/drivers/soc/apple/control.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Control-endpoint bookkeeping: the in-flight request table, the tag +//! allocator, and the reserved-tag entropy sink. + +use crate::proto; +use kernel::prelude::*; + +const MAX_INFLIGHT: usize = 8; + +const POOL_LEN: usize = (proto::TAG_POOL_LAST - proto::TAG_POOL_FIRST + 1) as usize; + +#[derive(Clone, Copy)] +struct Slot { + used: bool, + tag: u8, + reply: Option, +} + +impl Slot { + const FREE: Slot = Slot { + used: false, + tag: 0, + reply: None, + }; +} + +struct EntropySink { + value: Option, + msg1: u32, + unclaimed: u32, + busy: bool, +} + +pub(crate) enum Delivery { + Entropy, + Matched, + Unmatched, +} + +pub(crate) struct ControlState { + slots: [Slot; MAX_INFLIGHT], + next_tag: u8, + retired: [u64; 4], + retired_count: u32, + // SEP-initiated messages (type != 0x01). + unsolicited: u32, + entropy: EntropySink, + // After the persistent-state exchange the SEP parks this endpoint; refuse rather than time out. + closed: bool, +} + +impl ControlState { + pub(crate) fn new() -> Self { + ControlState { + slots: [Slot::FREE; MAX_INFLIGHT], + next_tag: proto::TAG_POOL_FIRST, + retired: [0; 4], + retired_count: 0, + unsolicited: 0, + entropy: EntropySink { + value: None, + msg1: 0, + unclaimed: 0, + busy: false, + }, + closed: false, + } + } + + fn is_retired(&self, tag: u8) -> bool { + self.retired[(tag >> 6) as usize] & (1u64 << (tag & 0x3f)) != 0 + } + + fn retire(&mut self, tag: u8) { + if !self.is_retired(tag) { + self.retired[(tag >> 6) as usize] |= 1u64 << (tag & 0x3f); + self.retired_count += 1; + } + } + + fn tag_in_flight(&self, tag: u8) -> bool { + self.slots.iter().any(|s| s.used && s.tag == tag) + } + + pub(crate) fn alloc(&mut self) -> Result<(usize, u8)> { + if self.closed { + return Err(EPIPE); + } + let idx = self.slots.iter().position(|s| !s.used).ok_or(EBUSY)?; + + for _ in 0..POOL_LEN { + let tag = self.next_tag; + self.next_tag = if tag >= proto::TAG_POOL_LAST { + proto::TAG_POOL_FIRST + } else { + tag + 1 + }; + if !self.tag_in_flight(tag) && !self.is_retired(tag) { + self.slots[idx] = Slot { + used: true, + tag, + reply: None, + }; + return Ok((idx, tag)); + } + } + Err(EBUSY) + } + + pub(crate) fn take_reply(&mut self, idx: usize) -> Option { + self.slots[idx].reply.take() + } + + pub(crate) fn release(&mut self, idx: usize) { + self.slots[idx] = Slot::FREE; + } + + // Retire the tag so a late reply cannot be mistaken for a later request's answer. + pub(crate) fn abandon(&mut self, idx: usize) -> u8 { + let tag = self.slots[idx].tag; + self.retire(tag); + self.slots[idx] = Slot::FREE; + tag + } + + pub(crate) fn retired_count(&self) -> u32 { + self.retired_count + } + + pub(crate) fn deliver(&mut self, reply: proto::ControlReply) -> Delivery { + // The reserved entropy tag is claimed first, outstanding request or not. + if reply.tag == proto::TAG_ENTROPY { + if self.entropy.value.is_some() { + self.entropy.unclaimed = self.entropy.unclaimed.wrapping_add(1); + } + self.entropy.value = Some(reply.data_lo); + self.entropy.msg1 = reply.msg1; + return Delivery::Entropy; + } + + if let Some(slot) = self + .slots + .iter_mut() + .find(|s| s.used && s.tag == reply.tag && s.reply.is_none()) + { + slot.reply = Some(reply); + return Delivery::Matched; + } + + // No match: drop it, never hand it to another waiter. + Delivery::Unmatched + } + + pub(crate) fn note_unsolicited(&mut self) -> u32 { + self.unsolicited = self.unsolicited.wrapping_add(1); + self.unsolicited + } + + pub(crate) fn entropy_begin(&mut self) -> Result<()> { + if self.closed { + return Err(EPIPE); + } + if self.entropy.busy { + return Err(EBUSY); + } + self.entropy.busy = true; + if self.entropy.value.take().is_some() { + self.entropy.unclaimed = self.entropy.unclaimed.wrapping_add(1); + } + Ok(()) + } + + pub(crate) fn entropy_take(&mut self) -> Option<(u32, u32)> { + self.entropy.value.take().map(|v| (v, self.entropy.msg1)) + } + + pub(crate) fn entropy_end(&mut self) { + self.entropy.busy = false; + } + +} diff --git a/drivers/soc/apple/crypto_shim.c b/drivers/soc/apple/crypto_shim.c new file mode 100644 index 0000000000000..e4d0863cce01b --- /dev/null +++ b/drivers/soc/apple/crypto_shim.c @@ -0,0 +1,294 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +#define SEP_GCM_TAG_LEN 16 + +/* + * AES-GCM with a 16-byte (non-96-bit) IV, over the raw AES block cipher. The + * kernel's gcm(aes) only takes a 12-byte IV; the SEP's ECIES uses a 16-byte IV, + * whose J0 must be derived as GHASH_H(IV ‖ len-block), not IV‖0x00000001. + */ + +/* GF(2^128) multiply, GCM bit convention (big-endian block, R = 0xe1‖0^120). */ +static void sep_gf_mult(const u8 *X, const u8 *Y, u8 *out) +{ + u8 Z[16] = {0}; + u8 V[16]; + int i, j; + + memcpy(V, Y, 16); + for (i = 0; i < 128; i++) { + if ((X[i >> 3] >> (7 - (i & 7))) & 1) + for (j = 0; j < 16; j++) + Z[j] ^= V[j]; + { + int lsb = V[15] & 1; + + for (j = 15; j > 0; j--) + V[j] = (V[j] >> 1) | ((V[j - 1] & 1) << 7); + V[0] >>= 1; + if (lsb) + V[0] ^= 0xe1; + } + } + memcpy(out, Z, 16); +} + +/* GHASH: fold `len` bytes of `data` (zero-padded to 16) into accumulator Y. */ +static void sep_ghash(const u8 H[16], u8 Y[16], const u8 *data, size_t len) +{ + u8 blk[16], t[16]; + size_t i, n, j; + + for (i = 0; i < len; i += 16) { + n = (len - i >= 16) ? 16 : (len - i); + memset(blk, 0, 16); + memcpy(blk, data + i, n); + for (j = 0; j < 16; j++) + Y[j] ^= blk[j]; + sep_gf_mult(Y, H, t); + memcpy(Y, t, 16); + } +} + +/* Increment the low 32 bits (big-endian) of a counter block, in place. */ +static void sep_inc32(u8 cb[16]) +{ + u32 c = ((u32)cb[12] << 24) | ((u32)cb[13] << 16) | + ((u32)cb[14] << 8) | (u32)cb[15]; + c++; + cb[12] = c >> 24; + cb[13] = c >> 16; + cb[14] = c >> 8; + cb[15] = c; +} + +/* + * AES-GCM in place over `buf` = [aadlen AAD][datalen payload][16 tag], 16-byte + * IV, keylen 16 or 32. Returns 0, or -EBADMSG on a decrypt tag mismatch. + */ +static int sep_gcm16(int encrypt, const void *key, size_t keylen, + const u8 *iv, size_t aadlen, u8 *buf, size_t datalen) +{ + struct aes_enckey aes; + u8 H[16], J0[16], EJ0[16], S[16], cb[16], ks[16], lb[16], zero[16]; + u8 *ct; + u64 aadbits, ctbits; + size_t i, n, j; + int rc; + + rc = aes_prepareenckey(&aes, key, keylen); + if (rc) + return rc; + + /* H = AES_K(0^128) */ + memset(zero, 0, 16); + aes_encrypt(&aes, H, zero); + + /* J0 = GHASH_H(IV ‖ [0^64 ‖ len(IV)_64]) for a non-96-bit IV. */ + memset(J0, 0, 16); + sep_ghash(H, J0, iv, 16); + memset(lb, 0, 16); + lb[15] = 0x80; /* 128 bits, big-endian in the low 64 bits */ + for (j = 0; j < 16; j++) + J0[j] ^= lb[j]; + { + u8 t[16]; + + sep_gf_mult(J0, H, t); + memcpy(J0, t, 16); + } + aes_encrypt(&aes, EJ0, J0); + + ct = buf + aadlen; + + /* GHASH over AAD, then ciphertext (before CTR on decrypt). */ + memset(S, 0, 16); + sep_ghash(H, S, buf, aadlen); + if (!encrypt) + sep_ghash(H, S, ct, datalen); + + /* CTR keystream from inc32(J0). */ + memcpy(cb, J0, 16); + sep_inc32(cb); + for (i = 0; i < datalen; i += 16) { + aes_encrypt(&aes, ks, cb); + n = (datalen - i >= 16) ? 16 : (datalen - i); + for (j = 0; j < n; j++) + ct[i + j] ^= ks[j]; + sep_inc32(cb); + } + + if (encrypt) + sep_ghash(H, S, ct, datalen); + + /* lengths block: (aadbits)_64 ‖ (ctbits)_64, big-endian. */ + aadbits = (u64)aadlen * 8; + ctbits = (u64)datalen * 8; + memset(lb, 0, 16); + for (j = 0; j < 8; j++) + lb[7 - j] = (u8)(aadbits >> (8 * j)); + for (j = 0; j < 8; j++) + lb[15 - j] = (u8)(ctbits >> (8 * j)); + for (j = 0; j < 16; j++) + S[j] ^= lb[j]; + { + u8 t[16]; + + sep_gf_mult(S, H, t); + memcpy(S, t, 16); + } + for (j = 0; j < 16; j++) + S[j] ^= EJ0[j]; /* S is now the computed tag */ + + if (encrypt) { + memcpy(ct + datalen, S, 16); + rc = 0; + } else { + u8 diff = 0; + + for (j = 0; j < 16; j++) + diff |= S[j] ^ ct[datalen + j]; + rc = diff ? -EBADMSG : 0; + } + + memzero_explicit(ks, sizeof(ks)); + memzero_explicit(EJ0, sizeof(EJ0)); + memzero_explicit(H, sizeof(H)); + memzero_explicit(&aes, sizeof(aes)); + return rc; +} + +/* + * Fills `buf` from the kernel CSPRNG, waiting for the seed first so it never + * returns unseeded bytes. Host-side entropy for key-bag secrets the host must + * reproduce; anything measuring the enclave's own entropy stays on SEP. + */ +int sep_random_bytes(void *buf, size_t len) +{ + int ret = wait_for_random_bytes(); + + if (ret) + return ret; + get_random_bytes(buf, len); + return 0; +} + +/* HMAC-SHA256 over one message; writes 32 bytes to `out`, untouched on error. */ +int sep_hmac_sha256(const void *key, size_t keylen, + const void *data, size_t datalen, u8 *out) +{ + struct crypto_shash *tfm; + struct shash_desc *desc; + int rc; + + tfm = crypto_alloc_shash("hmac(sha256)", 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + rc = crypto_shash_setkey(tfm, key, keylen); + if (rc) + goto out_tfm; + + desc = kzalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL); + if (!desc) { + rc = -ENOMEM; + goto out_tfm; + } + desc->tfm = tfm; + + rc = crypto_shash_digest(desc, data, datalen, out); + + kfree_sensitive(desc); +out_tfm: + crypto_free_shash(tfm); + return rc; +} + +/* + * AES-256-GCM in place over `buf` = [aadlen AAD][payload][tag]. Encrypt appends + * the tag; decrypt expects it and leaves plaintext in the first `datalen` bytes + * after the AAD. keylen 16 or 32, ivlen 12 or 16 (a 16-byte IV is routed to the + * GHASH J0 derivation, not truncated). + */ +int sep_gcm(int encrypt, const void *key, size_t keylen, + const void *iv, size_t ivlen, size_t aadlen, + void *buf, size_t buflen, size_t datalen) +{ + struct crypto_aead *tfm; + struct aead_request *req; + struct scatterlist sg; + DECLARE_CRYPTO_WAIT(wait); + size_t cryptlen; + u8 ivcopy[16]; + int rc; + + if ((keylen != 16 && keylen != 32) || + (ivlen != 12 && ivlen != 16) || ivlen > sizeof(ivcopy)) + return -EINVAL; + + /* The whole AAD-plus-payload-plus-tag extent must be inside the buffer. */ + if (aadlen + datalen + SEP_GCM_TAG_LEN < aadlen || + aadlen + datalen + SEP_GCM_TAG_LEN > buflen) + return -EINVAL; + + /* A 16-byte IV needs the GHASH J0 derivation; gcm(aes) would silently + * use only its first 12 bytes. */ + if (ivlen == 16) + return sep_gcm16(encrypt, key, keylen, iv, aadlen, buf, + datalen); + + tfm = crypto_alloc_aead("gcm(aes)", 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + rc = crypto_aead_setkey(tfm, key, keylen); + if (rc) + goto out_tfm; + rc = crypto_aead_setauthsize(tfm, SEP_GCM_TAG_LEN); + if (rc) + goto out_tfm; + + req = aead_request_alloc(tfm, GFP_KERNEL); + if (!req) { + rc = -ENOMEM; + goto out_tfm; + } + + /* The API may modify the IV buffer, so it never sees the caller's. */ + memcpy(ivcopy, iv, ivlen); + + sg_init_one(&sg, buf, aadlen + datalen + SEP_GCM_TAG_LEN); + + /* AEAD API quirk: cryptlen includes the tag on decrypt but not on encrypt. */ + cryptlen = encrypt ? datalen : datalen + SEP_GCM_TAG_LEN; + + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + aead_request_set_ad(req, aadlen); + aead_request_set_crypt(req, &sg, &sg, cryptlen, ivcopy); + + rc = crypto_wait_req(encrypt ? crypto_aead_encrypt(req) + : crypto_aead_decrypt(req), &wait); + + aead_request_free(req); + memzero_explicit(ivcopy, sizeof(ivcopy)); +out_tfm: + crypto_free_aead(tfm); + return rc; +} diff --git a/drivers/soc/apple/der.rs b/drivers/soc/apple/der.rs new file mode 100644 index 0000000000000..bc3db32343eae --- /dev/null +++ b/drivers/soc/apple/der.rs @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! The key-store configuration blob, which is DER. + +use kernel::prelude::*; + +const TAG_SET: u8 = 0x31; +const TAG_SEQUENCE: u8 = 0x30; +const TAG_UTF8: u8 = 0x0c; +pub(crate) const TAG_INTEGER: u8 = 0x02; +pub(crate) const TAG_OCTET_STRING: u8 = 0x04; + +const MAX_ITEM: usize = 4096; + +fn put_header(out: &mut KVec, tag: u8, len: usize) -> Result<()> { + out.push(tag, GFP_KERNEL)?; + if len < 0x80 { + out.push(len as u8, GFP_KERNEL)?; + } else if len <= 0xff { + out.push(0x81, GFP_KERNEL)?; + out.push(len as u8, GFP_KERNEL)?; + } else if len <= 0xffff { + out.push(0x82, GFP_KERNEL)?; + out.push((len >> 8) as u8, GFP_KERNEL)?; + out.push((len & 0xff) as u8, GFP_KERNEL)?; + } else { + return Err(EINVAL); + } + Ok(()) +} + +fn integer_bytes(v: u32) -> Result> { + let mut raw = KVec::new(); + let be = v.to_be_bytes(); + let mut i = 0usize; + while i < 3 && be[i] == 0 { + i += 1; + } + if be[i] & 0x80 != 0 { + raw.push(0, GFP_KERNEL)?; + } + raw.extend_from_slice(&be[i..], GFP_KERNEL)?; + Ok(raw) +} + +#[derive(Clone, Copy)] +pub(crate) enum RefKeyValue<'a> { + /// op mnemonic: `oc`/`ow`/`ouw`. + Utf8(&'a [u8]), + Integer(u32), + Octets(&'a [u8]), + Der(&'a [u8]), +} + +/// Canonical DER `SET OF SEQUENCE`, members sorted by key — the shape the +/// enclave parser requires. +pub(crate) fn encode_refkey_set(items: &[(&[u8], RefKeyValue<'_>)]) -> Result> { + let mut members: KVec<(&[u8], KVec)> = KVec::new(); + for (key, value) in items { + if key.is_empty() || key.len() > MAX_ITEM { + return Err(EINVAL); + } + let mut inner = KVec::new(); + put_header(&mut inner, TAG_UTF8, key.len())?; + inner.extend_from_slice(key, GFP_KERNEL)?; + match value { + RefKeyValue::Utf8(s) => { + if s.len() > MAX_ITEM { + return Err(EINVAL); + } + put_header(&mut inner, TAG_UTF8, s.len())?; + inner.extend_from_slice(s, GFP_KERNEL)?; + } + RefKeyValue::Integer(v) => { + let raw = integer_bytes(*v)?; + put_header(&mut inner, TAG_INTEGER, raw.len())?; + inner.extend_from_slice(&raw, GFP_KERNEL)?; + } + RefKeyValue::Octets(b) => { + if b.len() > MAX_ITEM { + return Err(EINVAL); + } + put_header(&mut inner, TAG_OCTET_STRING, b.len())?; + inner.extend_from_slice(b, GFP_KERNEL)?; + } + RefKeyValue::Der(d) => inner.extend_from_slice(d, GFP_KERNEL)?, + } + let mut seq = KVec::new(); + put_header(&mut seq, TAG_SEQUENCE, inner.len())?; + seq.extend_from_slice(&inner, GFP_KERNEL)?; + members.push((key, seq), GFP_KERNEL)?; + } + members.sort_unstable_by(|a, b| a.0.cmp(b.0)); + let mut body = KVec::new(); + for (_, seq) in members.iter() { + body.extend_from_slice(seq, GFP_KERNEL)?; + } + let mut out = KVec::new(); + put_header(&mut out, TAG_SET, body.len())?; + out.extend_from_slice(&body, GFP_KERNEL)?; + Ok(out) +} + +fn take_tlv(buf: &[u8]) -> Option<(u8, &[u8], &[u8])> { + let tag = *buf.first()?; + let first = *buf.get(1)?; + let (len, off) = if first < 0x80 { + (first as usize, 2usize) + } else if first == 0x81 { + (*buf.get(2)? as usize, 3usize) + } else if first == 0x82 { + let hi = *buf.get(2)? as usize; + let lo = *buf.get(3)? as usize; + ((hi << 8) | lo, 4usize) + } else { + return None; + }; + if len > MAX_ITEM { + return None; + } + let end = off.checked_add(len)?; + if buf.len() < end { + return None; + } + Some((tag, &buf[off..end], &buf[end..])) +} + +pub(crate) fn refkey_find<'a>(set_blob: &'a [u8], key: &[u8]) -> Option<&'a [u8]> { + let mut body = match take_tlv(set_blob) { + Some((TAG_SET, body, [])) => body, + _ => return None, + }; + while !body.is_empty() { + let (seq_tag, seq, next) = take_tlv(body)?; + if seq_tag != TAG_SEQUENCE { + return None; + } + let (key_tag, this_key, after_key) = take_tlv(seq)?; + if key_tag != TAG_UTF8 { + return None; + } + let (_val_tag, _value, after_val) = take_tlv(after_key)?; + if this_key == key { + return Some(&after_key[..after_key.len() - after_val.len()]); + } + body = next; + } + None +} + +pub(crate) fn octet_string_body(tlv: &[u8]) -> Option<&[u8]> { + match take_tlv(tlv) { + Some((TAG_OCTET_STRING, body, [])) => Some(body), + _ => None, + } +} + diff --git a/drivers/soc/apple/dt.rs b/drivers/soc/apple/dt.rs new file mode 100644 index 0000000000000..7fc5be540c654 --- /dev/null +++ b/drivers/soc/apple/dt.rs @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Device-tree work: bringing the SEP and its DART out of `status = "disabled"` +//! at module init, and the boot-persistent marker that stops the one-shot +//! registration from being sent twice. + +use kernel::bindings; +use kernel::error::to_result; +use kernel::prelude::*; + +const SEP_COMPATIBLE: &CStr = c"apple,sep"; +const IOMMUS_PROP: &CStr = c"iommus"; + +const STATUS_PROP: &CStr = c"status"; +const STATUS_OKAY: &CStr = c"okay"; + +// SEP DMA is offset BIT(40) above the DART input; one 64-bit value = two cells. +const DMA_OFFSET_PROP: &CStr = c"apple,dma-offset"; +// [0x100, 0] = BIT(40); [0x1, 0] would be BIT(32) and miss the page tables. +const DMA_OFFSET_CELLS: [u32; 2] = [0x100, 0x0]; + +const DMA_RANGE_PROP: &CStr = c"apple,dma-range"; +const DMA_RANGE_CELLS: [u32; 4] = [0, 0, 1, 0]; + +const REGISTERED_PROP: &CStr = c"apple,sep-shmem-registered-iova"; + +pub(crate) struct DtNode(*mut bindings::device_node); + +impl DtNode { + fn find_compatible(compatible: &CStr) -> Option { + // SAFETY: `of_find_compatible_node` accepts NULL for `from` and `type`, + // takes a reference on the node it returns, and returns NULL when there + // is no match. + let np = unsafe { + bindings::of_find_compatible_node( + core::ptr::null_mut(), + core::ptr::null(), + compatible.as_char_ptr(), + ) + }; + (!np.is_null()).then_some(DtNode(np)) + } + + pub(crate) fn of_device(dev: &kernel::device::Device) -> Option { + // SAFETY: `dev` is valid for the duration of the borrow, and its + // `of_node` is either NULL or a live node the device holds a reference + // to. + let np = unsafe { (*dev.as_raw()).of_node }; + if np.is_null() { + return None; + } + // SAFETY: `np` is a live node; take our own reference for `DtNode`. + unsafe { bindings::of_node_get(np) }; + Some(DtNode(np)) + } + + fn parse_phandle(&self, name: &CStr, index: i32) -> Option { + // SAFETY: `self.0` is a valid node per the type invariant; + // `of_parse_phandle` takes a reference on what it returns and returns + // NULL when the property or the index is absent. + let np = unsafe { bindings::of_parse_phandle(self.0, name.as_char_ptr(), index) }; + (!np.is_null()).then_some(DtNode(np)) + } + + fn as_ptr(&self) -> *mut bindings::device_node { + self.0 + } + + fn is_available(&self) -> bool { + // SAFETY: `self.0` is a valid node per the type invariant. + unsafe { bindings::of_device_is_available(self.0) } + } + + pub(crate) fn has_property(&self, name: &CStr) -> bool { + // SAFETY: `self.0` is a valid node per the type invariant; a NULL + // length pointer is accepted. + let p = unsafe { + bindings::of_find_property(self.0, name.as_char_ptr(), core::ptr::null_mut()) + }; + !p.is_null() + } + +} + +impl Drop for DtNode { + fn drop(&mut self) { + // SAFETY: we own one reference, taken by the function that produced + // `self.0`. + unsafe { bindings::of_node_put(self.0) }; + } +} + +fn with_changeset(f: F) -> Result<()> +where + F: FnOnce(*mut bindings::of_changeset) -> Result, +{ + let mut cs = core::mem::MaybeUninit::::uninit(); + let csp = cs.as_mut_ptr(); + + // SAFETY: `csp` points at valid uninitialised storage of the right type and + // `of_changeset_init` initialises it in place. + unsafe { bindings::of_changeset_init(csp) }; + + let res = match f(csp) { + // SAFETY: `csp` is an initialised changeset. + Ok(true) => to_result(unsafe { bindings::of_changeset_apply(csp) }), + Ok(false) => Ok(()), + Err(e) => Err(e), + }; + + // SAFETY: `csp` is an initialised changeset. + unsafe { bindings::of_changeset_destroy(csp) }; + + res +} + +fn queue_u32_array( + cs: *mut bindings::of_changeset, + node: &DtNode, + name: &CStr, + cells: &[u32], +) -> Result<()> { + // SAFETY: `cs` is an initialised changeset, `node` is a valid node, `name` + // is NUL-terminated, and `cells` is valid for `cells.len()` reads. The + // callee copies both the name and the value. + to_result(unsafe { + bindings::of_changeset_add_prop_u32_array( + cs, + node.as_ptr(), + name.as_char_ptr(), + cells.as_ptr(), + cells.len(), + ) + }) +} + +fn queue_status_okay(cs: *mut bindings::of_changeset, node: &DtNode) -> Result<()> { + // SAFETY: as above; `of_changeset_update_prop_string` duplicates the string. + to_result(unsafe { + bindings::of_changeset_update_prop_string( + cs, + node.as_ptr(), + STATUS_PROP.as_char_ptr(), + STATUS_OKAY.as_char_ptr(), + ) + }) +} + +pub(crate) fn sep_node() -> Option { + DtNode::find_compatible(SEP_COMPATIBLE) +} + +pub(crate) fn enable_sep_and_dart() -> Result<()> { + let sep = sep_node().ok_or_else(|| { + pr_err!( + "apple_sep: no device-tree node with compatible '{}'\n", + SEP_COMPATIBLE + ); + ENODEV + })?; + + let dart = sep.parse_phandle(IOMMUS_PROP, 0).ok_or_else(|| { + pr_err!( + "apple_sep: SEP node has no '{}' phandle; cannot find its DART\n", + IOMMUS_PROP + ); + ENODEV + })?; + + let dart_available = dart.is_available(); + let need_offset = !dart.has_property(DMA_OFFSET_PROP); + let need_range = !dart.has_property(DMA_RANGE_PROP); + + pr_info!( + "apple_sep: SEP DART state: enabled={}, apple,dma-offset present={}, apple,dma-range present={}\n", + dart_available, + !need_offset, + !need_range + ); + + if dart_available && (need_offset || need_range) { + pr_err!( + "apple_sep: SEP DART already enabled but missing apple,dma-offset/apple,dma-range; the BIT(40) offset applies only at DART probe. Reboot and load this module first.\n" + ); + return Err(EBUSY); + } + + if need_offset || need_range || !dart_available { + with_changeset(|cs| { + let mut queued = false; + // Properties before status: the DART probes inside of_changeset_apply(). + if need_offset { + queue_u32_array(cs, &dart, DMA_OFFSET_PROP, &DMA_OFFSET_CELLS)?; + queued = true; + } + if need_range { + queue_u32_array(cs, &dart, DMA_RANGE_PROP, &DMA_RANGE_CELLS)?; + queued = true; + } + if !dart_available { + queue_status_okay(cs, &dart)?; + queued = true; + } + Ok(queued) + })?; + pr_info!( + "apple_sep: SEP DART changeset applied (offset BIT(40), range 0..4GiB, status okay)\n" + ); + } else { + pr_info!("apple_sep: SEP DART already enabled and configured\n"); + } + + if sep.is_available() { + pr_info!("apple_sep: SEP node already enabled\n"); + } else { + with_changeset(|cs| { + queue_status_okay(cs, &sep)?; + Ok(true) + })?; + pr_info!("apple_sep: SEP node enabled; platform device should now exist\n"); + } + + Ok(()) +} + +pub(crate) fn registration_already_sent(sep: &DtNode) -> bool { + sep.has_property(REGISTERED_PROP) +} + +pub(crate) fn mark_registration_sent(sep: &DtNode, iova: u64) -> Result<()> { + let cells = [(iova >> 32) as u32, iova as u32]; + with_changeset(|cs| { + queue_u32_array(cs, sep, REGISTERED_PROP, &cells)?; + Ok(true) + }) +} + +impl DtNode { + fn reg_base(&self) -> Option { + // SAFETY: `struct resource` is plain integers and pointers, so an + // all-zero value is valid. Zeroed rather than `default()` because + // bindgen's generated structs do not derive `Default`. + let mut res: bindings::resource = unsafe { core::mem::zeroed() }; + // SAFETY: `self.0` is a live node and `res` is a valid out-parameter. + let rc = unsafe { bindings::of_address_to_resource(self.0, 0, &mut res) }; + if rc != 0 { + None + } else { + Some(res.start) + } + } + + fn child_with_reg(&self, value: u32) -> Option { + let mut child: *mut bindings::device_node = core::ptr::null_mut(); + loop { + // SAFETY: `of_get_next_child` takes a live parent and the previous + // child, which it drops for us; NULL starts the iteration. + child = unsafe { bindings::of_get_next_child(self.0, child) }; + if child.is_null() { + return None; + } + let mut got: u32 = 0; + // SAFETY: `child` is live for this iteration and `got` is a valid + // out-parameter. + let rc = unsafe { + bindings::of_property_read_variable_u32_array( + child, + c"reg".as_char_ptr(), + &mut got, + 1, + 1, + ) + }; + if rc >= 0 && got == value { + return Some(DtNode(child)); + } + } + } +} + +fn node_at_address(base: u64) -> Option { + let mut np: *mut bindings::device_node = core::ptr::null_mut(); + loop { + // SAFETY: `of_find_node_with_property` accepts NULL to start and drops + // the reference to the previous node for us. + np = unsafe { bindings::of_find_node_with_property(np, c"reg".as_char_ptr()) }; + if np.is_null() { + return None; + } + let node = DtNode(np); + if node.reg_base() == Some(base) { + return Some(node); + } + } +} + +pub(crate) const SENSOR_COMPATIBLE: &CStr = c"apple,mesa-fingerprint"; + +const SENSOR_NODE_NAME: &CStr = c"mesa@0"; + +fn queue_sensor_node(cs_handle: *mut bindings::of_changeset, parent: &DtNode) -> Result<()> { + // SAFETY: `cs_handle` is a live changeset and `parent` a live node; the + // returned node belongs to the changeset, which owns it until applied. + let node = unsafe { + bindings::of_changeset_create_node(cs_handle, parent.0, SENSOR_NODE_NAME.as_char_ptr()) + }; + if node.is_null() { + pr_err!("apple_sep: could not create the sensor node\n"); + return Err(ENOMEM); + } + + // SAFETY: all four calls take the live changeset, the node just created and + // a NUL-terminated name; the u32 form takes a slice it copies. + let rc = unsafe { + let mut rc = bindings::of_changeset_add_prop_string( + cs_handle, + node, + c"compatible".as_char_ptr(), + SENSOR_COMPATIBLE.as_char_ptr(), + ); + // reg on an SPI child is the chip select. + if rc == 0 { + rc = add_u32(cs_handle, node, c"reg", 0); + } + if rc == 0 { + rc = add_u32(cs_handle, node, c"spi-max-frequency", 8_000_000); + } + // 20 ns setup/hold the controller does not apply itself. + if rc == 0 { + rc = add_u32(cs_handle, node, c"spi-cs-setup-delay-ns", 20); + } + if rc == 0 { + rc = add_u32(cs_handle, node, c"spi-cs-hold-delay-ns", 20); + } + // SPI mode 2 (CPOL=1, CPHA=0); absent spi-cpha is how CPHA=0 is spelled. + if rc == 0 { + rc = bindings::of_changeset_add_prop_bool(cs_handle, node, c"spi-cpol".as_char_ptr()); + } + rc + }; + if rc != 0 { + pr_err!("apple_sep: could not describe the sensor node: {}\n", rc); + return Err(Error::from_errno(rc)); + } + Ok(()) +} + +/// # Safety +unsafe fn add_u32( + cs_handle: *mut bindings::of_changeset, + node: *mut bindings::device_node, + name: &CStr, + value: u32, +) -> i32 { + // of_changeset_add_prop_u32 is a static inline missing from the bindings; wrap the array form. + // SAFETY: per this function's contract; `value` is read as one element. + unsafe { + bindings::of_changeset_add_prop_u32_array(cs_handle, node, name.as_char_ptr(), &value, 1) + } +} + +pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { + let controller = node_at_address(base).ok_or_else(|| { + pr_err!( + "apple_sep: no device-tree node with reg base 0x{:x}; the sensor's SPI bus is not in this tree\n", + base + ); + ENODEV + })?; + + let controller_ok = controller.is_available(); + let existing = controller.child_with_reg(cs); + pr_info!( + "apple_sep: sensor SPI bus at 0x{:x}: controller enabled={}, chip-select {} child present={}\n", + base, + controller_ok, + cs, + existing.is_some() + ); + + if controller_ok && existing.is_some() { + return Ok(()); + } + + if existing.is_some() && !controller_ok { + // Enable only the bus; a second child would be two devices at one chip select. + pr_info!("apple_sep: sensor node already present; enabling only the bus\n"); + return with_changeset(|cs_handle| { + queue_status_okay(cs_handle, &controller)?; + Ok(true) + }); + } + + pr_info!( + "apple_sep: creating the sensor node (absent from Linux's tree; Apple has it at /arm-io/spi2/mesa)\n" + ); + + with_changeset(|cs_handle| { + // Controller before child: the SPI core's notifier needs the bus to exist first. + if !controller_ok { + queue_status_okay(cs_handle, &controller)?; + } + queue_sensor_node(cs_handle, &controller)?; + Ok(true) + }) +} diff --git a/drivers/soc/apple/fv.rs b/drivers/soc/apple/fv.rs new file mode 100644 index 0000000000000..ba05aba7d854f --- /dev/null +++ b/drivers/soc/apple/fv.rs @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! Native FileVault key hierarchy: device-bound volume-key provisioning, +//! reverse-engineered and kept as capability. Not wired to a caller. +#![allow(dead_code)] + +use crate::{image, proto, shim}; +use crate::{LockState, SepData, SksRequest}; +use kernel::prelude::*; + +impl SepData { + const FV_LABEL: &'static [u8; 16] = b"AppleSEPvek00001"; + const FV_PKH: &'static [u8; 16] = b"AppleSEP-KEK-001"; + const FV_PARAM_LEN: usize = 0x130; + + fn fv_param(&self) -> Result> { + let mut p: KVec = KVec::new(); + p.resize(Self::FV_PARAM_LEN, 0u8, GFP_KERNEL)?; + p[0x10..0x20].copy_from_slice(Self::FV_LABEL); + Ok(p) + } + + fn sks_req_fv(&self, selector: u8, body: &image::Body) -> Result { + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_fv(selector, self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_PERFORM_OP_NAME, + msg, + img, + }) + } + + fn fv_send( + &self, + selector: u8, + body: image::Body, + _what: &str, + ) -> Option<(i32, Option, Option>)> { + let out = self.sks_send(self.sks_req_fv(selector, &body))?; + let mailbox: i32 = out.reply.status.into(); + let (mut opst, mut blob) = (None, None); + if mailbox == 0 { + if let Some(rbody) = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &out) { + let mut f = proto::FieldCursor::new(rbody); + opst = f.i32(); + if let Some(b) = f.blob() { + let mut owned: KVec = KVec::new(); + owned.extend_from_slice(b, GFP_KERNEL).ok()?; + blob = Some(owned); + } + } + } + Some((mailbox, opst, blob)) + } + + /// `0x42` mint the device-bound key-encryption key. + fn fv_new_kek(&self, handle: u64, param: &[u8]) -> Option> { + let mut b = image::Body::new(); + b.put_u32(0).ok()?; + b.put_u64(handle).ok()?; + b.put_blob(param).ok()?; + b.put_u32(0).ok()?; + b.put_blob(&[]).ok()?; + b.put_blob(Self::FV_PKH).ok()?; + let (mb, op, blob) = self.fv_send(crate::sks::OP_SKS_FV_NEW_KEK, b, "new_kek")?; + (mb == 0 && op == Some(0)).then_some(())?; + blob + } + + /// `0x40` mint the wrapped, device-bound volume key. + fn fv_new_vek(&self, handle: u64, param: &[u8]) -> Option> { + let mut b = image::Body::new(); + b.put_u32(0).ok()?; + b.put_u64(handle).ok()?; + b.put_blob(param).ok()?; + b.put_blob(&[]).ok()?; + b.put_blob(&[]).ok()?; + b.put_blob(Self::FV_PKH).ok()?; + let (mb, op, blob) = self.fv_send(crate::sks::OP_SKS_FV_NEW_VEK, b, "new_vek")?; + (mb == 0 && op == Some(0)).then_some(())?; + blob + } + + /// `0x41` install the volume key into our collection (empty KEK slot = self-derive). + fn fv_unwrap_vek(&self, handle: u64, param: &[u8], wrapped_vek: &[u8]) -> Option> { + let mut b = image::Body::new(); + b.put_u32(0).ok()?; + b.put_u64(handle).ok()?; + b.put_blob(param).ok()?; + b.put_u32(0).ok()?; + b.put_blob(&[]).ok()?; + b.put_blob(&[]).ok()?; + b.put_blob(wrapped_vek).ok()?; + b.put_blob(&[]).ok()?; + let (mb, op, blob) = self.fv_send(crate::sks::OP_SKS_FV_UNWRAP_VEK, b, "unwrap_vek")?; + (mb == 0 && op == Some(0)).then_some(())?; + blob.or_else(|| Some(KVec::new())) + } + + fn fv_dump(path: &CStr, data: &[u8]) { + if let Ok(f) = shim::StoreFile::open(path) { + let _ = f.write_all(0, data); + let _ = f.sync(); + } + } + + fn fv_load(path: &CStr) -> Option> { + let f = shim::StoreFile::open_readonly(path).ok()?; + let sz = f.size().unwrap_or(0); + if !(1..=8192).contains(&sz) { + return None; + } + let mut buf: KVec = KVec::new(); + buf.resize(sz as usize, 0u8, GFP_KERNEL).ok()?; + f.read_exact(0, &mut buf).ok()?; + Some(buf) + } + + /// Provisions the device-bound FileVault key hierarchy and installs the volume + /// key. The destructive clear (`0x47`) is never emitted; the installed VEK is + /// consumed by the storage inline-AES engine (ANS), so on its own this seals + /// nothing on Linux. + pub(crate) fn fv_provision(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) -> Option { + const KEK_PATH: &CStr = c"/var/lib/apple-sep-fv-kek.bin"; + const VEK_PATH: &CStr = c"/var/lib/apple-sep-fv-vek.bin"; + + if let Some(healthy) = self.sks_health_check(c"fv provision") { + let _ = self.sks_send(self.sks_req_change_lock_state( + handle, + LockState::Unlocked, + secret, + healthy, + )); + } + let param = self.fv_param().ok()?; + let h = handle.value() as u64; + + let (_kek, vek) = match (Self::fv_load(KEK_PATH), Self::fv_load(VEK_PATH)) { + (Some(kek), Some(vek)) => (kek, vek), + _ => { + let kek = self.fv_new_kek(h, ¶m)?; + let vek = self.fv_new_vek(h, ¶m)?; + Self::fv_dump(KEK_PATH, &kek); + Self::fv_dump(VEK_PATH, &vek); + (kek, vek) + } + }; + + let installed = self.fv_unwrap_vek(h, ¶m, &vek)?; + let vek_handle = if installed.len() >= 4 { + u64::from(u32::from_le_bytes([installed[0], installed[1], installed[2], installed[3]])) + } else { + 0 + }; + Some(vek_handle) + } +} diff --git a/drivers/soc/apple/hwrng.rs b/drivers/soc/apple/hwrng.rs new file mode 100644 index 0000000000000..92f3ac7043a78 --- /dev/null +++ b/drivers/soc/apple/hwrng.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Ownership of the hwrng-core registration. + +use kernel::prelude::*; + +pub(crate) type ReadFn = + unsafe extern "C" fn(ctx: *mut c_void, data: *mut c_void, max: usize, wait: bool) -> c_int; + +extern "C" { + fn sep_hwrng_alloc() -> *mut c_void; + fn sep_hwrng_free(mem: *mut c_void); + fn sep_hwrng_register( + mem: *mut c_void, + name: *const c_char, + quality: c_ushort, + ctx: *mut c_void, + read: Option, + ) -> c_int; + fn sep_hwrng_unregister(mem: *mut c_void); +} + +pub(crate) const QUALITY: c_ushort = 1024; + +pub(crate) const NAME: &CStr = c"apple-sep"; + +pub(crate) struct HwRngHandle { + mem: *mut c_void, + registered: bool, +} + +// SAFETY: `mem` is a plain heap allocation with no thread affinity; all access +// goes through the C shim, which locks inside the hwrng core. +unsafe impl Send for HwRngHandle {} + +impl HwRngHandle { + pub(crate) fn new() -> Result { + // SAFETY: no preconditions; returns NULL on allocation failure. + let mem = unsafe { sep_hwrng_alloc() }; + if mem.is_null() { + return Err(ENOMEM); + } + Ok(HwRngHandle { + mem, + registered: false, + }) + } + + /// # Safety + /// `ctx` must stay valid and safe to pass to `read` until [`Self::unregister`] + /// returns or this handle is dropped. + pub(crate) unsafe fn register(&mut self, ctx: *mut c_void, read: ReadFn) -> Result<()> { + if self.registered { + return Err(EBUSY); + } + // SAFETY: `mem` is live, `NAME` is a static NUL-terminated string, and + // the caller guarantees `ctx` outlives the registration. + let ret = + unsafe { sep_hwrng_register(self.mem, NAME.as_char_ptr(), QUALITY, ctx, Some(read)) }; + kernel::error::to_result(ret)?; + self.registered = true; + Ok(()) + } + + pub(crate) fn unregister(&mut self) { + if self.registered { + self.registered = false; + // SAFETY: `mem` is live and was registered. + unsafe { sep_hwrng_unregister(self.mem) }; + } + } +} + +impl Drop for HwRngHandle { + fn drop(&mut self) { + self.unregister(); + // SAFETY: `mem` is live and now unregistered, so nothing else refers to it. + unsafe { sep_hwrng_free(self.mem) }; + } +} diff --git a/drivers/soc/apple/hwrng_shim.c b/drivers/soc/apple/hwrng_shim.c new file mode 100644 index 0000000000000..e0507f935d0f7 --- /dev/null +++ b/drivers/soc/apple/hwrng_shim.c @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* + * hwrng core shim: struct hwrng lives here where the C compiler owns its + * layout, and Rust sees an opaque pointer plus four entry points. + */ + +#include +#include +#include + +#include "shim.h" + +struct sep_hwrng { + struct hwrng rng; + /* Opaque Rust-side context, handed back to the read callback. */ + void *ctx; + int (*read)(void *ctx, void *data, size_t max, bool wait); +}; + +static int sep_hwrng_read(struct hwrng *rng, void *data, size_t max, + bool wait) +{ + struct sep_hwrng *h = + container_of(rng, struct sep_hwrng, rng); + + return h->read(h->ctx, data, max, wait); +} + +void *sep_hwrng_alloc(void) +{ + return kzalloc(sizeof(struct sep_hwrng), GFP_KERNEL); +} + +/* Must not be called while registered. */ +void sep_hwrng_free(void *mem) +{ + kfree(mem); +} + +/* + * @name must outlive the registration; the Rust side passes a &'static CStr. + * @quality is bits of entropy per 1024 bits of input; 0 takes the core default, + * which is also 1024. + */ +int sep_hwrng_register(void *mem, const char *name, + unsigned short quality, void *ctx, + int (*read)(void *ctx, void *data, size_t max, + bool wait)) +{ + struct sep_hwrng *h = mem; + + h->ctx = ctx; + h->read = read; + h->rng.name = name; + h->rng.read = sep_hwrng_read; + h->rng.quality = quality; + + return hwrng_register(&h->rng); +} + +/* + * Blocks until the core is done with the device, so any read in flight must + * return promptly: the Rust side sets its shutdown flag and wakes its waiters + * before calling this. + */ +void sep_hwrng_unregister(void *mem) +{ + struct sep_hwrng *h = mem; + + hwrng_unregister(&h->rng); +} diff --git a/drivers/soc/apple/image.rs b/drivers/soc/apple/image.rs new file mode 100644 index 0000000000000..b81f8130fe602 --- /dev/null +++ b/drivers/soc/apple/image.rs @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Request and response images for the key-store endpoint. + +use kernel::prelude::*; + +extern "C" { + fn sep_sha256( + a: *const c_void, + alen: usize, + b: *const c_void, + blen: usize, + out: *mut u8, + ) -> c_int; +} + +pub(crate) const HEADER_SIZE: u32 = 0x50; + +pub(crate) const HEADER_WIRE: usize = 0x54; + +const DIGEST_OFF: usize = 0x00; +const DIGEST_LEN: usize = 16; + +const DIGEST_FROM: usize = 0x10; + +pub(crate) const SHA256_LEN: usize = 32; + +pub(crate) fn sha256(bytes: &[u8]) -> Result<[u8; SHA256_LEN]> { + let mut digest = [0u8; SHA256_LEN]; + // SAFETY: `bytes` is a live slice for the duration of the call and `digest` + // is exactly the 32 bytes the shim writes. The shim reads the second + // segment only when its length is nonzero, and it is zero here. + let rc = unsafe { + sep_sha256( + bytes.as_ptr().cast(), + bytes.len(), + core::ptr::null(), + 0, + digest.as_mut_ptr(), + ) + }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok(digest) +} + +const OFF_VERSION: usize = 0x10; +const OFF_TIMESTAMP: usize = 0x14; +const OFF_FLAGS: usize = 0x1c; +const OFF_RESERVED: usize = 0x20; +const OFF_PROC_ID: usize = 0x28; +const OFF_PID: usize = 0x30; +const OFF_CDHASH: usize = 0x34; +const OFF_TRAILER: usize = 0x48; + +static_assert!(OFF_VERSION == DIGEST_OFF + DIGEST_LEN); +static_assert!(OFF_TIMESTAMP == OFF_VERSION + 4); +static_assert!(OFF_FLAGS == OFF_TIMESTAMP + 8); +static_assert!(OFF_RESERVED == OFF_FLAGS + 4); +static_assert!(OFF_PROC_ID == OFF_RESERVED + 8); +static_assert!(OFF_PID == OFF_PROC_ID + 8); +static_assert!(OFF_CDHASH == OFF_PID + 4); +static_assert!(OFF_TRAILER == OFF_CDHASH + 20); +static_assert!(OFF_TRAILER + 8 == HEADER_SIZE as usize); +static_assert!(HEADER_WIRE == 4 + HEADER_SIZE as usize); + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Version { + V1, +} + +impl Version { + pub(crate) const fn wire(self) -> u32 { + match self { + Version::V1 => 1, + } + } + + const fn digest_end(self) -> usize { + match self { + Version::V1 => OFF_TRAILER, + } + } +} + +pub(crate) struct Body { + bytes: KVec, +} + +impl Body { + pub(crate) fn new() -> Body { + Body { bytes: KVec::new() } + } + + pub(crate) fn put_u32(&mut self, v: u32) -> Result<()> { + self.bytes.extend_from_slice(&v.to_le_bytes(), GFP_KERNEL)?; + Ok(()) + } + + pub(crate) fn put_i32(&mut self, v: i32) -> Result<()> { + self.bytes.extend_from_slice(&v.to_le_bytes(), GFP_KERNEL)?; + Ok(()) + } + + pub(crate) fn put_u64(&mut self, v: u64) -> Result<()> { + self.bytes.extend_from_slice(&v.to_le_bytes(), GFP_KERNEL)?; + Ok(()) + } + + pub(crate) fn put_blob(&mut self, bytes: &[u8]) -> Result<()> { + let len = u32::try_from(bytes.len()).map_err(|_| EINVAL)?; + self.put_u32(len)?; + self.bytes.extend_from_slice(bytes, GFP_KERNEL)?; + let pad = bytes.len().wrapping_neg() % 4; + for _ in 0..pad { + self.bytes.push(0, GFP_KERNEL)?; + } + Ok(()) + } + + fn as_slice(&self) -> &[u8] { + &self.bytes + } +} + +const fn pad_of(len: usize) -> usize { + len.wrapping_neg() % 4 +} +static_assert!(pad_of(0) == 0); +static_assert!(pad_of(1) == 3); +static_assert!(pad_of(2) == 2); +static_assert!(pad_of(3) == 1); +static_assert!(pad_of(4) == 0); + +pub(crate) struct RequestImage { + bytes: KVec, +} + +impl RequestImage { + pub(crate) fn as_slice(&self) -> &[u8] { + &self.bytes + } + + pub(crate) fn len(&self) -> usize { + self.bytes.len() + } +} + +pub(crate) fn build_request( + version: Version, + timestamp_us: u64, + body: &Body, +) -> Result { + let mut bytes = KVec::new(); + bytes.extend_from_slice(&HEADER_SIZE.to_le_bytes(), GFP_KERNEL)?; + + let header_at = bytes.len(); + for _ in 0..HEADER_SIZE { + bytes.push(0, GFP_KERNEL)?; + } + + let put = |bytes: &mut KVec, off: usize, src: &[u8]| { + bytes[header_at + off..header_at + off + src.len()].copy_from_slice(src); + }; + put(&mut bytes, OFF_VERSION, &version.wire().to_le_bytes()); + put(&mut bytes, OFF_TIMESTAMP, ×tamp_us.to_le_bytes()); + // Flags, reserved, proc_id, pid, cdhash and trailer stay zero; the enclave accepts that. + + bytes.extend_from_slice(body.as_slice(), GFP_KERNEL)?; + + static_assert!(OFF_VERSION == DIGEST_FROM); + let end = version.digest_end(); + let head = &bytes[header_at + DIGEST_FROM..header_at + end]; + let tail = &bytes[header_at + HEADER_SIZE as usize..]; + + let mut digest = [0u8; SHA256_LEN]; + // SAFETY: `head` and `tail` are live slices of `bytes` for the duration of + // the call, and `digest` is exactly the 32 bytes the shim writes. The shim + // reads only the two segments it is given and writes only the output. + let rc = unsafe { + sep_sha256( + head.as_ptr().cast(), + head.len(), + tail.as_ptr().cast(), + tail.len(), + digest.as_mut_ptr(), + ) + }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + + bytes[header_at + DIGEST_OFF..header_at + DIGEST_OFF + DIGEST_LEN] + .copy_from_slice(&digest[..DIGEST_LEN]); + + Ok(RequestImage { bytes }) +} + +pub(crate) struct ResponseImage<'a> { + pub(crate) body: &'a [u8], +} + +pub(crate) fn parse_response(bytes: &[u8]) -> Result> { + if bytes.len() < 4 { + return Err(EINVAL); + } + let header_size = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + + let header = header_size as usize; + if header < OFF_VERSION + 4 { + return Err(EINVAL); + } + let body_at = 4usize.checked_add(header).ok_or(EINVAL)?; + if bytes.len() < body_at { + return Err(EINVAL); + } + + Ok(ResponseImage { + body: &bytes[body_at..], + }) +} + +pub(crate) fn read_blob(body: &[u8], off: usize) -> Option<(&[u8], usize)> { + let len_end = off.checked_add(4)?; + if body.len() < len_end { + return None; + } + let len = u32::from_le_bytes([body[off], body[off + 1], body[off + 2], body[off + 3]]) as usize; + let end = len_end.checked_add(len)?; + if body.len() < end { + return None; + } + Some((&body[len_end..end], end + pad_of(len))) +} + +pub(crate) fn operation_status(body: &[u8]) -> Option { + if body.len() < 4 { + return None; + } + Some(i32::from_le_bytes([body[0], body[1], body[2], body[3]])) +} diff --git a/drivers/soc/apple/keybag.rs b/drivers/soc/apple/keybag.rs new file mode 100644 index 0000000000000..133a1ad5769db --- /dev/null +++ b/drivers/soc/apple/keybag.rs @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Host-side persistence for the key bag. +//! +//! `0x02` copies the bag out enclave-wrapped, `0x03` reloads it. NEVER create a +//! second bag while a stored blob exists — it orphans the previous one in the +//! enclave with no recovery; [`NoStoredKeyBag`] enforces this by type. + +use crate::shim; +use crate::store::crc16_ccitt_false; +use kernel::prelude::*; + +pub(crate) const KEYBAG_PATH: &CStr = c"/var/lib/apple-sep-keybag.bin"; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Slot { + Identity, +} + +impl Slot { + pub(crate) fn path(self) -> &'static CStr { + match self { + Slot::Identity => KEYBAG_PATH, + } + } + +} + +const MAGIC: [u8; 16] = *b"APPLE-SEP-KBAG01"; +/// v2 stores the bag secret; a v1 record is refused, not reinterpreted. +const VERSION: u32 = 2; + +const STATE_COMMITTED: u32 = 1; +/// Create refused by the enclave: nothing exists, retry allowed (unlike an +/// intent record, where a bag may exist unnamed — block retry). +const STATE_REFUSED: u32 = 2; +/// UUID field is the bag's own, read back via `0x06`; state 1 holds the +/// host-generated one, and for an identity bag these differ. +const STATE_COMMITTED_BAG_UUID: u32 = 3; + +/// Only an `AsGenerated` UUID may be repaired on a mismatch; adopting one for a +/// read-back record would point it at a different bag and strand the real one. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum UuidProvenance { + AsGenerated, + ReadBackFromBag, +} + +impl UuidProvenance { + fn state(self) -> u32 { + match self { + Self::AsGenerated => STATE_COMMITTED, + Self::ReadBackFromBag => STATE_COMMITTED_BAG_UUID, + } + } + + fn from_state(state: u32) -> Option { + match state { + STATE_COMMITTED => Some(Self::AsGenerated), + STATE_COMMITTED_BAG_UUID => Some(Self::ReadBackFromBag), + _ => None, + } + } +} + +const OFF_VERSION: usize = 0x10; +const OFF_STATE: usize = 0x14; +const OFF_LEN: usize = 0x18; +const OFF_CRC: usize = 0x1c; +const OFF_UUID: usize = 0x20; +const OFF_SECRET_LEN: usize = 0x30; +const HEADER: usize = 0x34; + +static_assert!(OFF_VERSION == MAGIC.len()); +static_assert!(OFF_STATE == OFF_VERSION + 4); +static_assert!(OFF_LEN == OFF_STATE + 4); +static_assert!(OFF_CRC == OFF_LEN + 4); +static_assert!(OFF_UUID == OFF_CRC + 4); +static_assert!(OFF_SECRET_LEN == OFF_UUID + UUID_LEN); +static_assert!(HEADER == OFF_SECRET_LEN + 4); + +pub(crate) const UUID_LEN: usize = 16; + +const MAX_WRAPPED: usize = crate::store::MAX_VALUE; + +/// Proof the host holds no bag; create takes one by value, so it is unreachable +/// without it and unrepeatable with it. +pub(crate) struct NoStoredKeyBag; + +pub(crate) struct StoredKeyBag { + wrapped: KVec, + uuid: [u8; UUID_LEN], + secret: crate::Secret, + provenance: UuidProvenance, +} + +impl StoredKeyBag { + pub(crate) fn wrapped(&self) -> &[u8] { + &self.wrapped + } + + pub(crate) fn uuid(&self) -> &[u8; UUID_LEN] { + &self.uuid + } + + pub(crate) fn uuid_provenance(&self) -> UuidProvenance { + self.provenance + } + + pub(crate) fn secret(&self) -> &[u8] { + &self.secret + } +} + +pub(crate) enum State { + Present(StoredKeyBag), + Absent(NoStoredKeyBag), +} + +/// CRC preimage; this byte order is the on-disk contract — changing it breaks +/// records written by an earlier build. +fn checksum_input( + state: u32, + wrapped: &[u8], + uuid: &[u8; UUID_LEN], + secret: &[u8], +) -> Result> { + let mut v = KVec::new(); + v.extend_from_slice(&VERSION.to_le_bytes(), GFP_KERNEL)?; + v.extend_from_slice(&state.to_le_bytes(), GFP_KERNEL)?; + v.extend_from_slice(&(wrapped.len() as u32).to_le_bytes(), GFP_KERNEL)?; + v.extend_from_slice(&(secret.len() as u32).to_le_bytes(), GFP_KERNEL)?; + v.extend_from_slice(uuid, GFP_KERNEL)?; + v.extend_from_slice(wrapped, GFP_KERNEL)?; + v.extend_from_slice(secret, GFP_KERNEL)?; + Ok(v) +} + +/// Absent only when the file is missing; every other unreadable state errors — +/// "cannot tell" must never read as "nothing here", which would orphan a bag. +pub(crate) fn read(slot: Slot) -> Result { + let file = match shim::StoreFile::open_readonly(slot.path()) { + Ok(f) => f, + Err(e) if e == ENOENT => { + return Ok(State::Absent(NoStoredKeyBag)); + } + Err(e) => return Err(e), + }; + + let size = file.size()?; + if size < HEADER as u64 { + return Err(EINVAL); + } + + let mut head = [0u8; HEADER]; + file.read_exact(0, &mut head)?; + if head[..MAGIC.len()] != MAGIC { + return Err(EINVAL); + } + let version = u32::from_le_bytes([ + head[OFF_VERSION], + head[OFF_VERSION + 1], + head[OFF_VERSION + 2], + head[OFF_VERSION + 3], + ]); + if version != VERSION { + return Err(ENOTSYNC); + } + let state = u32::from_le_bytes([ + head[OFF_STATE], + head[OFF_STATE + 1], + head[OFF_STATE + 2], + head[OFF_STATE + 3], + ]); + if state == STATE_REFUSED { + return Ok(State::Absent(NoStoredKeyBag)); + } + let Some(provenance) = UuidProvenance::from_state(state) else { + // Intent or unknown state: a bag may exist unnamed; creating again would + // orphan it permanently. + return Err(EEXIST); + }; + + let len = u32::from_le_bytes([ + head[OFF_LEN], + head[OFF_LEN + 1], + head[OFF_LEN + 2], + head[OFF_LEN + 3], + ]) as usize; + if len == 0 || len > MAX_WRAPPED || size < (HEADER + len) as u64 { + return Err(EINVAL); + } + let crc = u16::from_le_bytes([head[OFF_CRC], head[OFF_CRC + 1]]); + + let mut uuid = [0u8; UUID_LEN]; + uuid.copy_from_slice(&head[OFF_UUID..OFF_UUID + UUID_LEN]); + + let secret_len = u32::from_le_bytes([ + head[OFF_SECRET_LEN], + head[OFF_SECRET_LEN + 1], + head[OFF_SECRET_LEN + 2], + head[OFF_SECRET_LEN + 3], + ]) as usize; + if secret_len > MAX_WRAPPED || size < (HEADER + len + secret_len) as u64 { + return Err(EINVAL); + } + + let mut wrapped = KVec::with_capacity(len, GFP_KERNEL)?; + wrapped.resize(len, 0, GFP_KERNEL)?; + file.read_exact(HEADER as u64, &mut wrapped)?; + + let mut secret_bytes = KVec::with_capacity(secret_len, GFP_KERNEL)?; + secret_bytes.resize(secret_len, 0, GFP_KERNEL)?; + if secret_len > 0 { + file.read_exact((HEADER + len) as u64, &mut secret_bytes)?; + } + + if size != (HEADER + len + secret_len) as u64 { + return Err(EINVAL); + } + let secret = crate::Secret(secret_bytes); + + if crc16_ccitt_false(&checksum_input(state, &wrapped, &uuid, &secret)?) != crc { + return Err(EINVAL); + } + + Ok(State::Present(StoredKeyBag { + wrapped, + uuid, + secret, + provenance, + })) +} + +/// Replaces only the wrapped blob. The enclave ratchets bag material while a bag +/// is active, so this snapshot must be re-taken at the catacomb-save commit or +/// the enclave answers a later restore empty. +pub(crate) fn replace_wrapped( + slot: Slot, + fresh: &[u8], + snapshot_uuid: &[u8; UUID_LEN], +) -> Result<()> { + if fresh.is_empty() || fresh.len() > MAX_WRAPPED { + return Err(EINVAL); + } + let stored = match read(slot)? { + State::Present(stored) => stored, + State::Absent(_) => return Err(ENOENT), + }; + if stored.uuid() != snapshot_uuid { + // A snapshot of a different bag would strand the stored one. + return Err(EPERM); + } + write_record( + slot, + stored.uuid_provenance().state(), + fresh, + stored.uuid(), + stored.secret(), + ) +} + +fn write_record( + slot: Slot, + state: u32, + wrapped: &[u8], + uuid: &[u8; UUID_LEN], + secret: &[u8], +) -> Result<()> { + let crc = crc16_ccitt_false(&checksum_input(state, wrapped, uuid, secret)?); + + let mut head = [0u8; HEADER]; + head[..MAGIC.len()].copy_from_slice(&MAGIC); + head[OFF_VERSION..OFF_VERSION + 4].copy_from_slice(&VERSION.to_le_bytes()); + head[OFF_STATE..OFF_STATE + 4].copy_from_slice(&state.to_le_bytes()); + head[OFF_LEN..OFF_LEN + 4].copy_from_slice(&(wrapped.len() as u32).to_le_bytes()); + head[OFF_CRC..OFF_CRC + 2].copy_from_slice(&crc.to_le_bytes()); + head[OFF_UUID..OFF_UUID + UUID_LEN].copy_from_slice(uuid); + head[OFF_SECRET_LEN..OFF_SECRET_LEN + 4].copy_from_slice(&(secret.len() as u32).to_le_bytes()); + + let file = shim::StoreFile::open_trunc(slot.path())?; + // Body first, then the header that vouches for it: a crash between leaves a + // bad checksum, which `read` refuses — the safe side. + if !wrapped.is_empty() { + file.write_all(HEADER as u64, wrapped)?; + } + if !secret.is_empty() { + file.write_all((HEADER + wrapped.len()) as u64, secret)?; + } + file.write_all(0, &head)?; + file.sync() +} diff --git a/drivers/soc/apple/p256_shim.c b/drivers/soc/apple/p256_shim.c new file mode 100644 index 0000000000000..bcf1323e3c58f --- /dev/null +++ b/drivers/soc/apple/p256_shim.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Dj */ +/* + * Apple SEP driver — P-256 ECDH sender shim for the ref-key ECIES seal. + * + * The ref-key (op 0x22) SE-seal is an ECIES scheme: the AP encrypts to the + * ref-key's public point and the enclave decrypts (op "od"). The enclave holds + * the recipient private key and never exposes it; the encrypt half touches only + * the public key, so it runs here in the kernel with no secret crossing into + * the SEP request but the (public) ephemeral point. This shim does the sender's + * key agreement: generate an ephemeral P-256 keypair, return its public point + * for the envelope, and compute the ECDH shared secret against the recipient's. + * + * Byte order: the kernel's `ecdh-nist-p256` kpp is big-endian throughout + * (ecdh_set_secret decodes the key via ecc_digits_from_bytes; ecc_swap_digits + * reads/writes coordinates as __be64). So the private key, both public points + * (X||Y), and the shared secret are plain big-endian byte strings, matching the + * ECIES/SEP convention: this shim does no byte swapping. It only marshals + * through kmalloc'd buffers, since a scatterlist entry must live in the linear + * map and the caller's may not. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +#define P256_COORD_LEN 32 +#define P256_POINT_LEN 64 /* X || Y, big-endian, no 0x04 prefix */ + +/* + * Core key agreement. `priv_be` is a 32-byte big-endian private scalar, or NULL + * to generate a fresh ephemeral key. `pub_be_out` receives the 64-byte + * big-endian public point (X||Y), `shared_be_out` the 32-byte big-endian shared + * secret (X coordinate of priv*peer). Scatterlist buffers are kmalloc'd since + * sg_init_one requires the linear map. + */ +static int p256_agree(const u8 *priv_be, const u8 *peer_pub_be, + u8 *pub_be_out, u8 *shared_be_out) +{ + struct crypto_kpp *tfm; + struct kpp_request *req; + struct ecdh params; + struct scatterlist src, dst; + DECLARE_CRYPTO_WAIT(wait); + unsigned int enc_len; + char *enc = NULL; + u8 *pub_buf = NULL; + u8 *peer_buf = NULL; + u8 *shared_buf = NULL; + int rc; + + tfm = crypto_alloc_kpp("ecdh-nist-p256", 0, 0); + if (IS_ERR(tfm)) { + rc = PTR_ERR(tfm); + pr_err("apple_sep p256: alloc_kpp rc=%d\n", rc); + return rc; + } + + memset(¶ms, 0, sizeof(params)); + if (priv_be) { + params.key = (void *)priv_be; /* big-endian, as the kpp wants */ + params.key_size = P256_COORD_LEN; + } else { + params.key = NULL; + params.key_size = 0; /* kpp generates a valid random scalar */ + } + + enc_len = crypto_ecdh_key_len(¶ms); + enc = kmalloc(enc_len, GFP_KERNEL); + if (!enc) { + rc = -ENOMEM; + goto out_tfm; + } + rc = crypto_ecdh_encode_key(enc, enc_len, ¶ms); + if (rc) { + pr_err("apple_sep p256: encode_key rc=%d\n", rc); + goto out_enc; + } + rc = crypto_kpp_set_secret(tfm, enc, enc_len); + if (rc) { + pr_err("apple_sep p256: set_secret rc=%d\n", rc); + goto out_enc; + } + + req = kpp_request_alloc(tfm, GFP_KERNEL); + if (!req) { + rc = -ENOMEM; + goto out_enc; + } + + pub_buf = kmalloc(P256_POINT_LEN, GFP_KERNEL); + if (!pub_buf) { + rc = -ENOMEM; + goto out_req; + } + sg_init_one(&dst, pub_buf, P256_POINT_LEN); + kpp_request_set_input(req, NULL, 0); + kpp_request_set_output(req, &dst, P256_POINT_LEN); + kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + rc = crypto_wait_req(crypto_kpp_generate_public_key(req), &wait); + if (rc) { + pr_err("apple_sep p256: generate_public_key rc=%d\n", rc); + goto out_pub; + } + memcpy(pub_be_out, pub_buf, P256_POINT_LEN); + + peer_buf = kmalloc(P256_POINT_LEN, GFP_KERNEL); + shared_buf = kmalloc(P256_COORD_LEN, GFP_KERNEL); + if (!peer_buf || !shared_buf) { + rc = -ENOMEM; + goto out_pub; + } + memcpy(peer_buf, peer_pub_be, P256_POINT_LEN); + sg_init_one(&src, peer_buf, P256_POINT_LEN); + sg_init_one(&dst, shared_buf, P256_COORD_LEN); + kpp_request_set_input(req, &src, P256_POINT_LEN); + kpp_request_set_output(req, &dst, P256_COORD_LEN); + kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + rc = crypto_wait_req(crypto_kpp_compute_shared_secret(req), &wait); + if (rc) { + pr_err("apple_sep p256: compute_shared_secret rc=%d\n", rc); + goto out_pub; + } + memcpy(shared_be_out, shared_buf, P256_COORD_LEN); + rc = 0; + +out_pub: + if (shared_buf) { + memzero_explicit(shared_buf, P256_COORD_LEN); + kfree(shared_buf); + } + kfree(peer_buf); + kfree(pub_buf); +out_req: + kpp_request_free(req); +out_enc: + kfree_sensitive(enc); +out_tfm: + crypto_free_kpp(tfm); + return rc; +} + +/* + * ECIES sender key agreement with a fresh ephemeral key. `peer_pub_be` is the + * recipient's 64-byte big-endian public point (0x04 prefix already stripped by + * the caller). `eph_pub_be_out` receives the 64-byte big-endian ephemeral + * public point (caller prepends 0x04 for the envelope), `shared_be_out` the + * 32-byte big-endian shared secret for the KDF. + */ +int sep_p256_sender(const void *peer_pub_be, void *eph_pub_be_out, + void *shared_be_out) +{ + return p256_agree(NULL, peer_pub_be, eph_pub_be_out, shared_be_out); +} + diff --git a/drivers/soc/apple/proto.rs b/drivers/soc/apple/proto.rs new file mode 100644 index 0000000000000..d084a889109c5 --- /dev/null +++ b/drivers/soc/apple/proto.rs @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! The SEP wire protocol, reverse-engineered: opcode tables, request encoders +//! and reply decoders. +#![allow(dead_code)] + +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; + +pub(crate) const EP_CONTROL: u8 = 0x00; +pub(crate) const EP_DISCOVER: u8 = 0xFD; +pub(crate) const EP_SHMEM: u8 = 0xFE; +pub(crate) const EP_BOOT: u8 = 0xFF; +pub(crate) const EP_XARM: u8 = 0x13; +pub(crate) const EP_SBIO: u8 = 0x08; +pub(crate) const EP_XARS: u8 = 0x10; +pub(crate) const EP_SCRD: u8 = 0x0a; +pub(crate) const EP_SKS: u8 = 0x12; + +pub(crate) const DISCOVER_TYPE_DESCRIPTOR: u8 = 0x00; +pub(crate) const DISCOVER_TYPE_CONFIG: u8 = 0x01; + +pub(crate) const MSG_TAG_SHIFT: u32 = 8; +pub(crate) const MSG_TYPE_SHIFT: u32 = 16; +pub(crate) const MSG_PARAM_SHIFT: u32 = 24; +pub(crate) const MSG_DATA_SHIFT: u32 = 32; + +// 4 KiB units even though CPU pages are 16 KiB +pub(crate) const IOVA_SHIFT: u32 = 12; + +#[derive(Clone, Copy)] +pub(crate) struct Fields { + pub(crate) ep: u8, + pub(crate) tag: u8, + pub(crate) ty: u8, + pub(crate) param: u8, + pub(crate) data_lo: u32, +} + +pub(crate) fn decode(msg: &Message) -> Fields { + Fields { + ep: msg.msg0 as u8, + tag: (msg.msg0 >> MSG_TAG_SHIFT) as u8, + ty: (msg.msg0 >> MSG_TYPE_SHIFT) as u8, + param: (msg.msg0 >> MSG_PARAM_SHIFT) as u8, + data_lo: (msg.msg0 >> MSG_DATA_SHIFT) as u32, + } +} + +pub(crate) const fn encode_registration_msg0(iova: u64, size: usize) -> u64 { + (EP_SHMEM as u64) + | (((size as u64) >> IOVA_SHIFT) << MSG_TYPE_SHIFT) + | ((iova >> IOVA_SHIFT) << MSG_DATA_SHIFT) +} + +static_assert!(encode_registration_msg0(0xBEE0_0000, 0x4_0000) == 0x000b_ee00_0040_00fe); + +pub(crate) fn shmem_registration(iova: u64, size: usize) -> Result { + let unit = 1u64 << IOVA_SHIFT; + + if size == 0 || (size as u64) & (unit - 1) != 0 { + return Err(EINVAL); + } + if iova & (unit - 1) != 0 { + return Err(EINVAL); + } + + let size_field = (size as u64) >> IOVA_SHIFT; + if size_field > 0xFF { + return Err(EINVAL); + } + + let iova_field = iova >> IOVA_SHIFT; + if iova_field > u64::from(u32::MAX) { + return Err(EINVAL); + } + + Ok(Message { + msg0: encode_registration_msg0(iova, size), + msg1: 0, + }) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct Fourcc(pub(crate) [u8; 4]); + +impl Fourcc { + pub(crate) const ZERO: Fourcc = Fourcc([0; 4]); +} + +impl kernel::fmt::Display for Fourcc { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + use core::fmt::Write; + for c in self.0 { + let c = if (0x20..0x7f).contains(&c) { c } else { b'.' }; + f.write_char(c as char)?; + } + Ok(()) + } +} + +pub(crate) fn fourcc(msg: &Message) -> Fourcc { + Fourcc(((msg.msg0 >> MSG_DATA_SHIFT) as u32).to_be_bytes()) +} + +pub(crate) const CONTROL_REPLY_TYPE: u8 = 0x01; + +pub(crate) const TAG_UNSOLICITED: u8 = 0x00; + +pub(crate) const TAG_ENTROPY: u8 = 0xE7; + +pub(crate) const CONTROL_TIMEOUT_MS: u32 = 2000; + +pub(crate) const TAG_POOL_FIRST: u8 = 0x01; +pub(crate) const TAG_POOL_LAST: u8 = 0x7e; + +static_assert!(TAG_POOL_FIRST > TAG_UNSOLICITED); +static_assert!(TAG_POOL_LAST < TAG_ENTROPY); + +pub(crate) struct ControlOp { + ty: u8, + param: u8, + data: u32, + expect_reply: bool, + timeout_ms: u32, + reserved_tag: bool, + name: &'static CStr, +} + +impl ControlOp { + pub(crate) fn expects_reply(&self) -> bool { + self.expect_reply + } + pub(crate) fn timeout_ms(&self) -> u32 { + self.timeout_ms + } + pub(crate) fn uses_reserved_tag(&self) -> bool { + self.reserved_tag + } + pub(crate) fn name(&self) -> &'static CStr { + self.name + } +} + +pub(crate) fn op_nop(param: u8) -> ControlOp { + ControlOp { + ty: 0x00, + param, + data: 0, + expect_reply: true, + timeout_ms: CONTROL_TIMEOUT_MS, + reserved_tag: false, + name: c"NOP", + } +} + +pub(crate) fn op_security_mode() -> ControlOp { + ControlOp { + ty: 0x14, + param: 0x00, + data: 0, + expect_reply: true, + timeout_ms: CONTROL_TIMEOUT_MS, + reserved_tag: false, + name: c"SECMODE", + } +} + +// no op for control 0x18: it wedges the control endpoint + +pub(crate) fn op_get_entropy() -> ControlOp { + ControlOp { + ty: 0x36, + param: 0x00, + data: 0, + expect_reply: true, + timeout_ms: CONTROL_TIMEOUT_MS, + reserved_tag: true, + name: c"GET_ENTROPY", + } +} + +pub(crate) fn encode_control(op: &ControlOp, tag: u8) -> Message { + Message { + msg0: u64::from(EP_CONTROL) + | (u64::from(tag) << MSG_TAG_SHIFT) + | (u64::from(op.ty) << MSG_TYPE_SHIFT) + | (u64::from(op.param) << MSG_PARAM_SHIFT) + | (u64::from(op.data) << MSG_DATA_SHIFT), + msg1: 0, + } +} + +const OP_OOL_INBOUND_SIZE: u8 = 0x04; +const OP_OOL_INBOUND_ADDR: u8 = 0x02; +const OP_OOL_OUTBOUND_SIZE: u8 = 0x05; +const OP_OOL_OUTBOUND_ADDR: u8 = 0x03; + +fn op_ool(ty: u8, endpoint: u8, data: u32, name: &'static CStr) -> ControlOp { + ControlOp { + ty, + param: endpoint, + data, + expect_reply: true, + timeout_ms: CONTROL_TIMEOUT_MS, + reserved_tag: false, + name, + } +} + +const OP_DMA_RING_PAGES: u8 = 0x19; +const OP_DMA_RING_ADDR: u8 = 0x1a; + +pub(crate) const DMA_RING_PAGES: u32 = 4; + +pub(crate) fn op_dma_ring_pages(endpoint: u8, pages: u32) -> ControlOp { + op_ool(OP_DMA_RING_PAGES, endpoint, pages, c"RING_PAGES") +} + +pub(crate) fn op_dma_ring_addr(endpoint: u8, iova: u64) -> ControlOp { + op_ool( + OP_DMA_RING_ADDR, + endpoint, + (iova >> IOVA_SHIFT) as u32, + c"RING_ADDR", + ) +} + +pub(crate) fn op_ool_inbound_size(endpoint: u8, len: u32) -> ControlOp { + op_ool(OP_OOL_INBOUND_SIZE, endpoint, len, c"OOL_IN_SIZE") +} + +pub(crate) fn op_ool_inbound_addr(endpoint: u8, iova: u64) -> ControlOp { + op_ool( + OP_OOL_INBOUND_ADDR, + endpoint, + (iova >> IOVA_SHIFT) as u32, + c"OOL_IN_ADDR", + ) +} + +pub(crate) fn op_ool_outbound_size(endpoint: u8, len: u32) -> ControlOp { + op_ool(OP_OOL_OUTBOUND_SIZE, endpoint, len, c"OOL_OUT_SIZE") +} + +pub(crate) fn op_ool_outbound_addr(endpoint: u8, iova: u64) -> ControlOp { + op_ool( + OP_OOL_OUTBOUND_ADDR, + endpoint, + (iova >> IOVA_SHIFT) as u32, + c"OOL_OUT_ADDR", + ) +} + +pub(crate) struct FieldCursor<'a> { + body: &'a [u8], + at: usize, +} + +impl<'a> FieldCursor<'a> { + pub(crate) fn new(body: &'a [u8]) -> FieldCursor<'a> { + FieldCursor { body, at: 0 } + } + + pub(crate) fn i32(&mut self) -> Option { + let end = self.at.checked_add(4)?; + let v = i32::from_le_bytes(self.body.get(self.at..end)?.try_into().ok()?); + self.at = end; + Some(v) + } + + // blob = u32 len + bytes + zero pad to 4-byte boundary; 0x45 reply has two in a row + pub(crate) fn blob(&mut self) -> Option<&'a [u8]> { + let len_end = self.at.checked_add(4)?; + let len = u32::from_le_bytes(self.body.get(self.at..len_end)?.try_into().ok()?) as usize; + let end = len_end.checked_add(len)?; + let bytes = self.body.get(len_end..end)?; + let pad = len.wrapping_neg() % 4; + self.at = end.checked_add(pad)?; + Some(bytes) + } +} + +#[derive(Clone, Copy)] +pub(crate) struct ControlReply { + pub(crate) tag: u8, + pub(crate) data_lo: u32, + pub(crate) msg1: u32, +} + +impl ControlReply { + pub(crate) fn from_message(msg: &Message) -> ControlReply { + let f = decode(msg); + ControlReply { + tag: f.tag, + data_lo: f.data_lo, + msg1: msg.msg1, + } + } +} diff --git a/drivers/soc/apple/refkey.rs b/drivers/soc/apple/refkey.rs new file mode 100644 index 0000000000000..c0456a604b38d --- /dev/null +++ b/drivers/soc/apple/refkey.rs @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! Machine ref-key: Secure-Enclave key sealing, signing, and attestation. +//! +//! A ref-key (op `0x22`) is an EC keypair the enclave keeps; the private half +//! never leaves the SEP. The host-side ECIES encrypt lives in [`refkey_seal`]; +//! every half that needs the private key runs in the enclave. + +use crate::{image, keybag, proto, refkey_seal, shim}; +use crate::{LockState, MachineRefKey, SepData, SksRequest}; +use kernel::prelude::*; + +pub(crate) const REFKEY_ENVELOPE_VERSION: u32 = 2; + +impl SepData { + fn sks_req_refkey(&self, keybag: i32, der_set: &[u8]) -> Result { + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(keybag)?; + body.put_u32(crate::refkey::REFKEY_ENVELOPE_VERSION)?; + body.put_blob(der_set)?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_perform_operation(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_PERFORM_OP_NAME, + msg, + img, + }) + } + + fn sks_create_refkey( + &self, + handle: crate::sks::KeyBagHandle, + secret: &[u8], + ) -> Option> { + use crate::der::RefKeyValue as RV; + + // Class 9 = non-extractable (private stays enclave-resident); key type 5 = P-256. + const PROTECTION_CLASS: u32 = 9; + const KEY_TYPE: u32 = 5; + + // Unlock the bag first; an unbound create is refused. + if let Some(healthy) = self.sks_health_check(c"ref-key create") { + let _ = self.sks_send(self.sks_req_change_lock_state( + handle, + LockState::Unlocked, + secret, + healthy, + )); + } + + let create_der = { + let mut items: KVec<(&[u8], RV<'_>)> = KVec::new(); + if items.push((b"o", RV::Utf8(b"oc")), GFP_KERNEL).is_err() + || items.push((b"bc", RV::Integer(PROTECTION_CLASS)), GFP_KERNEL).is_err() + || items.push((b"kt", RV::Integer(KEY_TYPE)), GFP_KERNEL).is_err() + { + return None; + } + crate::der::encode_refkey_set(&items).ok()? + }; + let cout = self.sks_send(self.sks_req_refkey(handle.value(), &create_der))?; + if cout.reply.status != 0 { + return None; + } + let cbody = + self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &cout)?; + let mut cf = proto::FieldCursor::new(cbody); + let (Some(0), Some(cblob)) = (cf.i32(), cf.blob()) else { + return None; + }; + let mut refkey_blob: KVec = KVec::new(); + refkey_blob.extend_from_slice(cblob, GFP_KERNEL).ok()?; + Some(refkey_blob) + } + + fn refkey_pub(blob: &[u8]) -> Option<&[u8]> { + let rk_tlv = crate::der::refkey_find(blob, b"rk")?; + let pub_raw = crate::der::refkey_find(rk_tlv, b"pub") + .and_then(crate::der::octet_string_body)?; + (pub_raw.len() == refkey_seal::POINT_LEN).then_some(pub_raw) + } + + pub(crate) fn sks_machine_refkey(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) { + const MACHINE_REFKEY_PATH: &CStr = c"/var/lib/apple-sep-refkey.bin"; + + if self.machine_refkey.lock().is_some() { + return; + } + + if let Ok(f) = shim::StoreFile::open_readonly(MACHINE_REFKEY_PATH) { + let sz = f.size().unwrap_or(0); + if (1..=8192).contains(&sz) { + let mut blob: KVec = KVec::new(); + if blob.resize(sz as usize, 0u8, GFP_KERNEL).is_ok() + && f.read_exact(0, &mut blob).is_ok() + && self.cache_machine_refkey(blob) + { + return; + } + } + } + + let Some(blob) = self.sks_create_refkey(handle, secret) else { + dev_warn!( + self.dev, + "sks: could not create the machine ref-key; trusted-key sealing is unavailable this boot.\n" + ); + return; + }; + if let Ok(f) = shim::StoreFile::open(MACHINE_REFKEY_PATH) { + if f.write_all(0, &blob).is_ok() { + let _ = f.sync(); + } + } + let _ = self.cache_machine_refkey(blob); + } + + fn cache_machine_refkey(&self, blob: KVec) -> bool { + let Some(pubv) = Self::refkey_pub(&blob) else { + return false; + }; + let mut pub_raw: KVec = KVec::new(); + if pub_raw.extend_from_slice(pubv, GFP_KERNEL).is_err() { + return false; + } + *self.machine_refkey.lock() = Some(MachineRefKey { blob, pub_raw }); + true + } + + fn ensure_machine_refkey(&self) -> Result<()> { + if self.machine_refkey.lock().is_some() { + return Ok(()); + } + if !self.sks_ready() { + return Err(ENODEV); + } + let keybag::State::Present(stored) = keybag::read(keybag::Slot::Identity)? else { + return Err(ENODEV); + }; + let (handle, _uuid) = self.sks_recover(&stored).ok_or(EIO)?; + self.sks_machine_refkey(handle, stored.secret()); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + if self.machine_refkey.lock().is_some() { + Ok(()) + } else { + Err(EIO) + } + } + + pub(crate) fn refkey_seal_trusted(&self, key: &[u8]) -> Result> { + self.ensure_machine_refkey()?; + let guard = self.machine_refkey.lock(); + let mk = guard.as_ref().ok_or(ENODEV)?; + refkey_seal::ecies_seal(&mk.pub_raw, key) + } + + pub(crate) fn refkey_unseal_trusted(&self, sealed: &[u8]) -> Result> { + self.ensure_machine_refkey()?; + let blob = { + let guard = self.machine_refkey.lock(); + let mk = guard.as_ref().ok_or(ENODEV)?; + let mut b: KVec = KVec::new(); + b.extend_from_slice(&mk.blob, GFP_KERNEL)?; + b + }; + let keybag::State::Present(stored) = keybag::read(keybag::Slot::Identity)? else { + return Err(ENODEV); + }; + let (handle, _uuid) = self.sks_recover(&stored).ok_or(EIO)?; + if let Some(healthy) = self.sks_health_check(c"trusted-key unseal") { + let _ = self.sks_send(self.sks_req_change_lock_state( + handle, + LockState::Unlocked, + stored.secret(), + healthy, + )); + } + let recovered = self.sks_refkey_unseal(handle, &blob, sealed); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + recovered.ok_or(EIO) + } + + /// Op `osgn`: ECDSA-P256 over `challenge` as the pre-computed digest. + fn sks_refkey_sign( + &self, + handle: crate::sks::KeyBagHandle, + refkey_blob: &[u8], + challenge: &[u8], + ) -> Option> { + use crate::der::RefKeyValue as RV; + let mut items: KVec<(&[u8], RV<'_>)> = KVec::new(); + items.push((b"o", RV::Utf8(b"osgn")), GFP_KERNEL).ok()?; + items.push((b"d", RV::Octets(challenge)), GFP_KERNEL).ok()?; + items.push((b"rk", RV::Der(refkey_blob)), GFP_KERNEL).ok()?; + let der = crate::der::encode_refkey_set(&items).ok()?; + let out = self.sks_send(self.sks_req_refkey(handle.value(), &der))?; + if out.reply.status != 0 { + dev_warn!(self.dev, "sks: ref-key attest sign failed (status {})\n", out.reply.status); + return None; + } + let body = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &out)?; + let mut f = proto::FieldCursor::new(body); + let (Some(0), Some(sig)) = (f.i32(), f.blob()) else { + return None; + }; + // Enclave wraps the sig in OCTET STRING { SEQUENCE { r, s } }; strip to inner DER. + let der_sig = crate::der::octet_string_body(sig).unwrap_or(sig); + let mut owned: KVec = KVec::new(); + owned.extend_from_slice(der_sig, GFP_KERNEL).ok()?; + Some(owned) + } + + /// Attestation of key possession by signing proof; op `oa` (Apple-CA-chained) + /// needs the SEP device attestation key, absent on a Linux-attached SEP. + pub(crate) fn refkey_attest_sign(&self, challenge: &[u8]) -> Result<(KVec, KVec)> { + self.ensure_machine_refkey()?; + let (blob, pubk) = { + let guard = self.machine_refkey.lock(); + let mk = guard.as_ref().ok_or(ENODEV)?; + let mut b: KVec = KVec::new(); + b.extend_from_slice(&mk.blob, GFP_KERNEL)?; + let mut p: KVec = KVec::new(); + p.extend_from_slice(&mk.pub_raw, GFP_KERNEL)?; + (b, p) + }; + let keybag::State::Present(stored) = keybag::read(keybag::Slot::Identity)? else { + return Err(ENODEV); + }; + let (handle, _uuid) = self.sks_recover(&stored).ok_or(EIO)?; + if let Some(healthy) = self.sks_health_check(c"ref-key attest") { + let _ = self.sks_send(self.sks_req_change_lock_state( + handle, + LockState::Unlocked, + stored.secret(), + healthy, + )); + } + let sig = self.sks_refkey_sign(handle, &blob, challenge); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + Ok((sig.ok_or(EIO)?, pubk)) + } + + /// Enclave ECIES-decrypt (o = "oecd"); takes the ephemeral from the front of `d`. + fn sks_refkey_unseal( + &self, + handle: crate::sks::KeyBagHandle, + refkey_blob: &[u8], + sealed: &[u8], + ) -> Option> { + use crate::der::RefKeyValue as RV; + let mut items: KVec<(&[u8], RV<'_>)> = KVec::new(); + items.push((b"o", RV::Utf8(b"oecd")), GFP_KERNEL).ok()?; + items.push((b"d", RV::Octets(sealed)), GFP_KERNEL).ok()?; + items.push((b"rk", RV::Der(refkey_blob)), GFP_KERNEL).ok()?; + let der = crate::der::encode_refkey_set(&items).ok()?; + let out = self.sks_send(self.sks_req_refkey(handle.value(), &der))?; + if out.reply.status != 0 { + dev_warn!(self.dev, "trusted-keys: ref-key unseal failed (status {})\n", out.reply.status); + return None; + } + let body = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &out)?; + let mut f = proto::FieldCursor::new(body); + match (f.i32(), f.blob()) { + (Some(0), Some(rec)) => { + let mut owned = KVec::new(); + owned.extend_from_slice(rec, GFP_KERNEL).ok()?; + Some(owned) + } + _ => None, + } + } +} diff --git a/drivers/soc/apple/refkey_seal.rs b/drivers/soc/apple/refkey_seal.rs new file mode 100644 index 0000000000000..f1d95935690b5 --- /dev/null +++ b/drivers/soc/apple/refkey_seal.rs @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! Ref-key (op 0x22) ECIES **encrypt** — the AP half of the SE seal. +//! +//! The SEP has no scalar×G, so it cannot make the sender ephemeral (in-SEP `ow` +//! returns -10); `od` imports the one this module produces, ECDHs it against the +//! in-SEP private key and AES-GCM-decrypts. The encrypt runs here by construction. + +use kernel::prelude::*; + +extern "C" { + fn sep_p256_sender( + peer_pub_be: *const c_void, + eph_pub_be_out: *mut u8, + shared_be_out: *mut u8, + ) -> c_int; + + fn sep_gcm( + encrypt: c_int, + key: *const c_void, + keylen: usize, + iv: *const c_void, + ivlen: usize, + aadlen: usize, + buf: *mut c_void, + buflen: usize, + datalen: usize, + ) -> c_int; +} + +// The enclave's SE seal is an ECIES scheme: +// ECDH P-256 (shared = X), X9.63-SHA256 KDF, KDF-derived key‖IV, AES-GCM. + +pub(crate) const TAG_LEN: usize = 16; +/// 16-byte IV (KDF-derived "variable IV"), not the 12-byte GCM default. +pub(crate) const IV_LEN: usize = 16; +pub(crate) const AES256_KEY_LEN: usize = 32; +pub(crate) const AES128_KEY_LEN: usize = 16; +/// Uncompressed P-256 point: `0x04 ‖ X ‖ Y`. +pub(crate) const POINT_LEN: usize = 65; +/// Raw `X ‖ Y` (no prefix), as the ECDH shim exchanges. +const COORDS_LEN: usize = 64; + +fn wipe(bytes: &mut [u8]) { + for b in bytes.iter_mut() { + // SAFETY: `b` is a valid, uniquely borrowed byte for this write. + unsafe { core::ptr::write_volatile(b, 0) }; + } +} + +fn p256_sender(peer_pub64: &[u8]) -> Result<([u8; COORDS_LEN], [u8; 32])> { + if peer_pub64.len() != COORDS_LEN { + return Err(EINVAL); + } + let mut eph = [0u8; COORDS_LEN]; + let mut shared = [0u8; 32]; + // SAFETY: `peer_pub64` is 64 live bytes; `eph`/`shared` are exactly the 64 + // and 32 bytes the shim writes. + let rc = unsafe { + sep_p256_sender( + peer_pub64.as_ptr().cast(), + eph.as_mut_ptr(), + shared.as_mut_ptr(), + ) + }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok((eph, shared)) +} + +fn kdf_sha256_mode(z: &[u8], shared_info: &[u8], out_len: usize, mode: u8) -> Result> { + let mut out = KVec::new(); + let mut counter: u32 = 1; + while out.len() < out_len { + let ctr = counter.to_be_bytes(); + let mut buf = KVec::new(); + match mode { + 0 => { + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(&ctr, GFP_KERNEL)?; + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + } + 1 => { + buf.extend_from_slice(&ctr, GFP_KERNEL)?; + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + } + 2 => { + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + buf.extend_from_slice(&ctr, GFP_KERNEL)?; + } + 3 => { + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(&ctr, GFP_KERNEL)?; + } + _ => { + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + } + } + let digest = crate::image::sha256(&buf)?; + wipe(&mut buf); + let take = core::cmp::min(digest.len(), out_len - out.len()); + out.extend_from_slice(&digest[..take], GFP_KERNEL)?; + counter = counter.checked_add(1).ok_or(EINVAL)?; + } + Ok(out) +} + +fn gcm_encrypt(key: &[u8], iv: &[u8], aad: &[u8], plaintext: &[u8]) -> Result> { + let mut buf = KVec::new(); + buf.extend_from_slice(aad, GFP_KERNEL)?; + buf.extend_from_slice(plaintext, GFP_KERNEL)?; + buf.resize(aad.len() + plaintext.len() + TAG_LEN, 0u8, GFP_KERNEL)?; + // SAFETY: `key`/`iv` are live; `buf` holds aad‖plaintext‖tag-space and its + // length is passed as both the buffer size and the extent to authenticate. + let rc = unsafe { + sep_gcm( + 1, + key.as_ptr().cast(), + key.len(), + iv.as_ptr().cast(), + iv.len(), + aad.len(), + buf.as_mut_ptr().cast(), + buf.len(), + plaintext.len(), + ) + }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + let mut out = KVec::new(); + out.extend_from_slice(&buf[aad.len()..], GFP_KERNEL)?; + Ok(out) +} + +pub(crate) struct EciesParts { + /// Ephemeral point `0x04 ‖ X ‖ Y`; also the KDF sharedInfo. + pub(crate) eph_point: KVec, + pub(crate) ciphertext: KVec, + pub(crate) tag: KVec, +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn ecies_encrypt_parts( + recipient_pub: &[u8], + plaintext: &[u8], + key_len: usize, + variable_iv: bool, + iv_len: usize, + aad_eph: bool, + si_mode: u8, + kdf_mode: u8, +) -> Result { + if recipient_pub.len() != POINT_LEN || recipient_pub[0] != 0x04 { + return Err(EINVAL); + } + if key_len != AES256_KEY_LEN && key_len != AES128_KEY_LEN { + return Err(EINVAL); + } + if iv_len != 12 && iv_len != 16 { + return Err(EINVAL); + } + let (eph_coords, mut shared) = p256_sender(&recipient_pub[1..])?; + + let mut eph_point = KVec::new(); + eph_point.push(0x04, GFP_KERNEL)?; + eph_point.extend_from_slice(&eph_coords, GFP_KERNEL)?; + + let mut si = KVec::new(); + match si_mode { + 0 => si.extend_from_slice(&eph_point, GFP_KERNEL)?, + 1 => {} + 2 => si.extend_from_slice(&eph_point[1..], GFP_KERNEL)?, + 3 => si.extend_from_slice(&eph_point[1..33], GFP_KERNEL)?, + 4 => si.extend_from_slice(recipient_pub, GFP_KERNEL)?, + _ => { + si.extend_from_slice(&eph_point, GFP_KERNEL)?; + si.extend_from_slice(recipient_pub, GFP_KERNEL)?; + } + } + + let kdf_len = if variable_iv { + key_len + iv_len + } else { + key_len + }; + let mut km = kdf_sha256_mode(&shared, &si, kdf_len, kdf_mode)?; + wipe(&mut shared); + let mut iv = KVec::new(); + if variable_iv { + iv.extend_from_slice(&km[key_len..key_len + iv_len], GFP_KERNEL)?; + } else { + iv.resize(iv_len, 0u8, GFP_KERNEL)?; + } + let aad: &[u8] = if aad_eph { &eph_point } else { &[] }; + let ct_tag = gcm_encrypt(&km[..key_len], &iv, aad, plaintext); + wipe(&mut km); + let ct_tag = ct_tag?; + if ct_tag.len() != plaintext.len() + TAG_LEN { + return Err(EINVAL); + } + + let mut ciphertext = KVec::new(); + ciphertext.extend_from_slice(&ct_tag[..plaintext.len()], GFP_KERNEL)?; + let mut tag = KVec::new(); + tag.extend_from_slice(&ct_tag[plaintext.len()..], GFP_KERNEL)?; + + Ok(EciesParts { + eph_point, + ciphertext, + tag, + }) +} + +/// Returns the enclave-ready blob: `ephemeral_point ‖ ciphertext ‖ tag`. +pub(crate) fn ecies_seal(recipient_pub: &[u8], secret: &[u8]) -> Result> { + let parts = ecies_encrypt_parts( + recipient_pub, + secret, + AES256_KEY_LEN, + true, + IV_LEN, + false, + 0, + 0, + )?; + let mut blob = KVec::new(); + blob.extend_from_slice(&parts.eph_point, GFP_KERNEL)?; + blob.extend_from_slice(&parts.ciphertext, GFP_KERNEL)?; + blob.extend_from_slice(&parts.tag, GFP_KERNEL)?; + Ok(blob) +} diff --git a/drivers/soc/apple/rxring.rs b/drivers/soc/apple/rxring.rs new file mode 100644 index 0000000000000..4aee7f407fa76 --- /dev/null +++ b/drivers/soc/apple/rxring.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +use core::cell::UnsafeCell; +use kernel::soc::apple::mailbox::Message; +use kernel::sync::atomic::{Acquire, Atomic, Relaxed, Release}; + +const RING_LEN: u32 = 64; +const RING_MASK: u32 = RING_LEN - 1; + +pub(crate) struct RxRing { + slots: [UnsafeCell; RING_LEN as usize], + head: Atomic, + tail: Atomic, + dropped: Atomic, +} + +// SAFETY: ownership of each slot passes between the two sides via the +// release/acquire pairing on `head`/`tail`, so no slot is touched by both at +// once; single-producer/consumer are enforced by rx_lock and workqueue non-reentrancy. +unsafe impl Sync for RxRing {} +// SAFETY: `Message` is a plain data struct with no thread affinity. +unsafe impl Send for RxRing {} + +impl RxRing { + pub(crate) fn new() -> Self { + RxRing { + slots: core::array::from_fn(|_| UnsafeCell::new(Message { msg0: 0, msg1: 0 })), + head: Atomic::new(0), + tail: Atomic::new(0), + dropped: Atomic::new(0), + } + } + + pub(crate) fn push(&self, msg: Message) -> bool { + let head = self.head.load(Relaxed); + // Acquire pairs with the consumer's release of `tail`, so the reused slot is free. + let tail = self.tail.load(Acquire); + + if head.wrapping_sub(tail) >= RING_LEN { + self.dropped + .store(self.dropped.load(Relaxed).wrapping_add(1), Relaxed); + return false; + } + + let slot = &self.slots[(head & RING_MASK) as usize]; + // SAFETY: `head` is not published yet, so the consumer cannot be looking at + // this slot, and we are the only producer. + unsafe { slot.get().write(msg) }; + + // Release pairs with the consumer's acquire of `head`, publishing the slot write. + self.head.store(head.wrapping_add(1), Release); + true + } + + pub(crate) fn pop(&self) -> Option { + let tail = self.tail.load(Relaxed); + // Acquire pairs with the producer's release of `head`. + let head = self.head.load(Acquire); + + if head == tail { + return None; + } + + let slot = &self.slots[(tail & RING_MASK) as usize]; + // SAFETY: `head` is past `tail`, so the producer finished this slot and + // will not touch it again until we publish `tail`. + let msg = unsafe { slot.get().read() }; + + // Release pairs with the producer's acquire of `tail`, handing the slot back. + self.tail.store(tail.wrapping_add(1), Release); + Some(msg) + } + + pub(crate) fn dropped(&self) -> u32 { + self.dropped.load(Relaxed) + } +} diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs new file mode 100644 index 0000000000000..308f3b76c01b9 --- /dev/null +++ b/drivers/soc/apple/sbio.rs @@ -0,0 +1,3316 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! Touch ID: the biometric endpoint (SBIO) and its `/dev/sep-bio` interface. +//! The enclave matches; no biometric image ever crosses to userspace. + +#![allow(dead_code)] + +use super::*; +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; +use crate::proto::*; +use crate::sks::SKS_AUTH_TOKEN_LEN; + +impl SepData { + fn sbio_expect_ok(&self, op: &crate::sbio::SbioOp) -> Option> { + match self.sbio_call(op) { + SbioOutcome::Ok(payload) => Some(payload), + _ => None, + } + } + + pub(crate) fn sbio_call(&self, op: &crate::sbio::SbioOp) -> SbioOutcome { + let done = match self.sbio_transfer(op) { + Ok(done) => done, + Err(_) => { + return SbioOutcome::Other; + } + }; + + let Some(err) = done.status.answered() else { + return SbioOutcome::Other; + }; + + match err as u16 { + crate::sbio::SBIO_STATUS_OK => SbioOutcome::Ok(done.payload), + crate::sbio::SBIO_STATUS_PREREQUISITE => { + SbioOutcome::PrerequisiteMissing + } + crate::sbio::SBIO_STATUS_16 => { + SbioOutcome::Status16 + } + _ => { + SbioOutcome::Other + } + } + } + + fn sbio_relay(&self, relay: &crate::sbio::SbioRelay<'_>) -> Option> { + match self.sbio_transfer_raw(relay.opcode(), relay.name(), relay.payload()) { + Ok(done) if done.status.is_ok() => Some(done.payload), + Ok(_) => None, + Err(_) => None, + } + } + + fn sync_device_view(&self) -> bool { + let synced = matches!( + self.sbio_call(&crate::sbio::sbio_update_device_list()), + SbioOutcome::Ok(_) + ); + + let policy_ok = match self.sbio_call(&crate::sbio::sbio_match_policy()) { + SbioOutcome::Ok(policy) if policy.len() == crate::sbio::SBIO_MATCH_POLICY_LEN => { + true + } + SbioOutcome::Ok(_) => { + false + } + _ => { + false + } + }; + + synced && policy_ok + } + + fn note_capture_end(&self) { + self.last_capture_end_ns.store(shim::boottime_ns(), Relaxed); + } + + fn settle_before_capture(&self) { + let last = self.last_capture_end_ns.load(Relaxed); + if last == 0 { + return; + } + let elapsed_ms = shim::boottime_ns().saturating_sub(last) / 1_000_000; + if elapsed_ms >= u64::from(MATCH_SETTLE_MS) { + return; + } + let remaining = MATCH_SETTLE_MS - elapsed_ms as u32; + let _ = sensor::idle(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(remaining))); + } + + fn pace_between_captures(&self) { + let _ = sensor::idle(); + + if !self.await_sensor_state(sensor::STATE_IDLE, c"idle, between captures") { + dev_warn!( + self.dev, + "enrol: sensor did not idle within {} ms; continuing the reposition wait anyway (pause is for the person)\n", + ENROL_IDLE_TIMEOUT_MS + ); + } + + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ENROL_REPOSITION_MS))); + } + + fn stash_enrol_identity(&self, record: &[u8]) { + let mut found: KVec<([u8; bio::UUID_LEN], u32)> = KVec::new(); + crate::sbio::enrol_identity_candidates(record, SBIO_PROBE_USER_ID, |at, uuid| { + let _ = found.push((uuid, at as u32), GFP_KERNEL); + }); + for (_uuid, _at) in found.iter() { + } + *self.enrol_identity_candidates.lock() = found; + } + + fn identity_just_enrolled(&self) -> Option<[u8; bio::UUID_LEN]> { + let candidates = core::mem::take(&mut *self.enrol_identity_candidates.lock()); + let listed = self.enclave_identities_for(SBIO_PROBE_USER_ID)?; + + let mut confirmed: KVec<([u8; bio::UUID_LEN], u32)> = KVec::new(); + for (uuid, at) in candidates.iter() { + if listed.iter().any(|u| u == uuid) && !confirmed.iter().any(|(u, _)| u == uuid) { + if confirmed.push((*uuid, *at), GFP_KERNEL).is_err() { + break; + } + } + } + + match confirmed.len() { + 1 => { + let (uuid, _at) = confirmed[0]; + Some(uuid) + } + 0 => { + None + } + _ => { + None + } + } + } + + fn reconcile_identities(&self) { + let Some(listed) = self.enclave_identities_for(SBIO_PROBE_USER_ID) else { + return; + }; + + if listed.is_empty() { + return; + } + + let dropped = { + let mut index = self.bio_index.lock(); + match index.reconcile_to(&listed) { + Ok(dropped) => dropped, + Err(_) => { + return; + } + } + }; + + let held = { + let index = self.bio_index.lock(); + index.total() + }; + for _uuid in listed.iter() { + } + + if dropped.is_empty() && held == listed.len() { + return; + } + let persisted = { + let index = self.bio_index.lock(); + self.with_store(|store| index.persist(store)) + }; + match persisted { + Some(Ok(())) => {}, + _ => dev_warn!( + self.dev, + "sbio: the reconciled index could not be written; it is correct in memory for this boot and will be rebuilt at the next attach\n" + ), + } + } + + fn enclave_identities_for(&self, user_id: i32) -> Option> { + let op = crate::sbio::sbio_list_identities(); + let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { + return None; + }; + + let records = crate::sbio::IdentityRecords::new(&reply)?; + + let mut out = KVec::new(); + let mut failed = false; + records.each_built_in_for(user_id, |identity| { + if out.push(*identity.uuid(), GFP_KERNEL).is_err() { + failed = true; + } + }); + if failed { + return None; + } + Some(out) + } + + fn finish_enrolment(&self, outcome: core::result::Result<[u8; bio::UUID_LEN], u32>) { + let filed = { + let mut session = self.bio_session.lock(); + let mut index = self.bio_index.lock(); + bio::enrol_finish(&mut session, &mut index, outcome) + }; + if !filed { + return; + } + // lock ordering: store lock outside the session lock (UAF otherwise) + if outcome.is_ok() { + let index = self.bio_index.lock(); + let saved = self.with_store(|store| index.persist(store)); + match saved { + Some(Ok(())) => {}, + Some(Err(e)) => dev_err!( + self.dev, + "enrol: enclave holds a new template but the host index could not be written ({:?})\n", + e + ), + None => {} + } + } + self.bio_wake(); + } + + fn bio_wake(&self) { + if let Some(dev) = self.bio_dev.lock().as_ref() { + dev.wake(); + } + } + + pub(crate) fn attach_sensor(&self) { + if let Err(e) = sensor::register_driver() { + self.sensor_present.store(false, Relaxed); + dev_err!( + self.dev, + "sensor: could not register the SPI driver: {:?}\n", + e + ); + return; + } + + if let Err(e) = dt::enable_spi_sensor(sensor::CONTROLLER_BASE, sensor::CHIP_SELECT) { + self.sensor_present.store(false, Relaxed); + dev_warn!( + self.dev, + "sensor: could not enable the SPI bus at 0x{:x} or create the sensor node ({:?}); no sensor\n", + sensor::CONTROLLER_BASE, + e + ); + return; + } + + let bound = sensor::is_bound(); + self.sensor_present.store(bound, Relaxed); + if bound && sensor::power_line().is_none() { + dev_warn!( + self.dev, + "sensor: no power line ({}); an unpowered sensor answers sixteen zero bytes like a dead bus\n", + sensor::power_source().name() + ); + } + } + + fn open_enrolment_context(&self, user: crate::sbio::UserId) -> bool { + let listed = match self.enclave_identities_for(SBIO_PROBE_USER_ID) { + Some(list) => list.len(), + None => { + return self.enrol_into_existing_context(user); + } + }; + + let Some(proof) = crate::sbio::NoExistingCatacomb::from_zero_identities(listed) else { + return self.enrol_into_existing_context(user); + }; + + self.open_fresh_context(user, &proof) + } + + fn enrol_into_existing_context(&self, user: crate::sbio::UserId) -> bool { + if !self.activate_protected_config(user) { + return false; + } + true + } + + fn open_fresh_context(&self, user: crate::sbio::UserId, proof: &crate::sbio::NoExistingCatacomb) -> bool { + let system = crate::sbio::sbio_select_context(crate::sbio::ContextScope::SYSTEM, proof); + if self.sbio_expect_ok(&system).is_none() { + return false; + } + + for (who, kind, what) in [ + ( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"master catacomb", + ), + ( + crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"owner catacomb", + ), + ] { + if !self.save_catacomb(who, kind, what) { + return false; + } + } + + let per_user = crate::sbio::sbio_select_context(crate::sbio::ContextScope::user(user), proof); + if self.sbio_expect_ok(&per_user).is_none() { + return false; + } + + if !self.activate_protected_config(user) { + return false; + } + + true + } + + fn activate_protected_config(&self, user: crate::sbio::UserId) -> bool { + let op = crate::sbio::sbio_protected_config(user); + let Some(config) = self.sbio_expect_ok(&op) else { + return false; + }; + if config.len() != crate::sbio::SBIO_PROTECTED_CONFIG_LEN { + return false; + } + true + } + + fn begin_enrolment_on_enclave(&self) -> Option> { + if self.enrol_open.load(Relaxed) { + self.enrol_open.store(false, Relaxed); + let _ = self.sbio_call(&crate::sbio::sbio_cancel_operation()); + } + + let user = crate::sbio::UserId::new(SBIO_PROBE_USER_ID)?; + + // 0x03 answers -3 until a user key bag is designated + if !self.keybag_designated.load(Relaxed) { + return None; + } + + if self.enrol_material.lock().is_none() { + return None; + } + + // 0x03 answers 0x1 until an enrolment context exists + if !self.open_enrolment_context(user) { + return None; + } + + let Some(acm_handle) = self.establish_passcode_validated_context(user) else { + dev_err!( + self.dev, + "enrol: could not establish the PasscodeValidated SCRD credential; not falling back to a transient SKS token (a catacomb sealed under one does not survive a reboot)\n" + ); + return None; + }; + let op = crate::sbio::sbio_begin_enrol(user, crate::sbio::BE_AUTH_TYPE_ACM_CONTEXT, &acm_handle); + + match self.sbio_call(&op) { + SbioOutcome::Ok(_payload) => { + self.enrol_open.store(true, Relaxed); + Some(OpenEnrolment { + sep: self, + armed: true, + }) + } + _ => { + None + } + } + } + + fn restore_all_components(&self) -> bool { + // 0x6b is re-read before every component, not once up front. + let user_id = SBIO_PROBE_USER_ID; + let mut any = false; + let mut missing_files = false; + let mut user_outcome = RestoreOutcome::NoStoredFile; + + // 0x8002 (cold transition) is a success only on the owner component. + for (id, kind, what, cold_ok) in [ + ( + crate::sbio::CatacombUser::MASTER.value(), + PRIVATE_TYPE_CATACOMB_MASTER, + c"master catacomb", + false, + ), + ( + crate::sbio::CatacombUser::OWNER.value(), + PRIVATE_TYPE_CATACOMB_OWNER, + c"owner catacomb", + true, + ), + (user_id, PRIVATE_TYPE_CATACOMB_USER, c"user catacomb", false), + ] { + let Some(reply) = self.read_component_states() else { + return false; + }; + let states = crate::sbio::ComponentStates::new(&reply); + let outcome = self.restore_catacomb(states.state_for(id), id, kind, what, cold_ok); + match outcome { + RestoreOutcome::Restored | RestoreOutcome::AlreadyActive => any = true, + RestoreOutcome::NoStoredFile => missing_files = true, + RestoreOutcome::Failed | RestoreOutcome::EmptyTolerated => {} + } + if id == user_id { + user_outcome = outcome; + } + } + + let user_ok = matches!( + user_outcome, + RestoreOutcome::Restored | RestoreOutcome::AlreadyActive + ); + let lockout = match self.restore_lockout() { + RestoreOutcome::Restored | RestoreOutcome::AlreadyActive => true, + RestoreOutcome::EmptyTolerated => true, + RestoreOutcome::NoStoredFile => { + missing_files = true; + false + } + RestoreOutcome::Failed => false, + }; + + if missing_files { + dev_warn!( + self.dev, + "sbio: at least one of four artefacts has no file on disk; if enrolled before the four-artefact save, re-enrol once to write all four\n" + ); + } + + let _ = any; + user_ok && lockout + } + + fn read_component_states(&self) -> Option> { + let op = crate::sbio::sbio_context_state(); + let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { + return None; + }; + + let states = crate::sbio::ComponentStates::new(&reply); + if states.count() == 0 { + return None; + } + Some(reply) + } + + fn restore_catacomb( + &self, + state: Option, + id: i32, + kind: u8, + what: &CStr, + cold_ok: bool, + ) -> RestoreOutcome { + let Some(state) = state else { + return RestoreOutcome::Failed; + }; + match crate::sbio::component_action(state) { + crate::sbio::ComponentAction::AlreadyActive => { + RestoreOutcome::AlreadyActive + } + crate::sbio::ComponentAction::Unsupported(_) => { + RestoreOutcome::Failed + } + crate::sbio::ComponentAction::Load => { + let Some(blob) = self.read_stored(kind, what) else { + return RestoreOutcome::NoStoredFile; + }; + + let at = crate::sbio::SBIO_SAVED_USER_ID_AT; + if blob.len() < at + 4 { + return RestoreOutcome::Failed; + } + let carried = + i32::from_le_bytes([blob[at], blob[at + 1], blob[at + 2], blob[at + 3]]); + if carried != id { + return RestoreOutcome::Failed; + } + + let Some(request) = crate::sbio::SbioLoadCatacomb::new(&blob) else { + return RestoreOutcome::Failed; + }; + + match self.sbio_transfer_raw(request.opcode(), request.name(), request.payload()) { + Ok(done) if done.status.is_ok() => { + self.confirm_active(id, what, LoadAnswer::StatusZero) + } + Ok(done) + if done.status.answered() == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) + && cold_ok => + { + self.confirm_active(id, what, LoadAnswer::ColdTransition) + } + Ok(done) + if done.status.answered() == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) => + { + RestoreOutcome::Failed + } + // 0x101 ALREADY_ACTIVE is a success + Ok(done) + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_ALREADY_ACTIVE) => + { + self.confirm_active(id, what, LoadAnswer::AlreadyActive) + } + _ => { + RestoreOutcome::Failed + } + } + } + } + } + + fn confirm_active(&self, id: i32, what: &CStr, answer: LoadAnswer) -> RestoreOutcome { + let Some(reply) = self.read_component_states() else { + dev_err!( + self.dev, + "sbio: the {} answered {} and 0x6b could not be re-read; treating unknown as failure\n", + what, + answer.describe() + ); + return RestoreOutcome::Failed; + }; + let states = crate::sbio::ComponentStates::new(&reply); + + match states.state_for(id) { + Some(state) if state & crate::sbio::COMPONENT_STATE_ACTIVE != 0 => { + RestoreOutcome::Restored + } + Some(state) if state & crate::sbio::COMPONENT_STATE_COLD != 0 => { + RestoreOutcome::Failed + } + Some(_) => { + RestoreOutcome::Failed + } + None => { + RestoreOutcome::Failed + } + } + } + + fn restore_lockout(&self) -> RestoreOutcome { + let Some(blob) = self.read_stored(PRIVATE_TYPE_LOCKOUT, c"bio lockout record") else { + return RestoreOutcome::NoStoredFile; + }; + let Some(request) = crate::sbio::SbioLoadLockout::new(&blob) else { + return RestoreOutcome::Failed; + }; + match self.sbio_transfer_raw(request.opcode(), request.name(), request.payload()) { + Ok(done) if done.status.is_ok() => { + RestoreOutcome::Restored + } + Ok(done) + if matches!( + done.status, + transfer::DeviceStatus::Answered(crate::sbio::SBIO_STATUS_COLD_TRANSITION) + ) => + { + RestoreOutcome::EmptyTolerated + } + Ok(_) => { + RestoreOutcome::Failed + } + Err(_) => { + RestoreOutcome::Failed + } + } + } + + fn read_stored(&self, kind: u8, _what: &CStr) -> Option> { + if crate::catacomb::is_kind(kind) { + if let Some(blob) = crate::catacomb::read(kind) { + return Some(blob); + } + } + match self.with_store(|store| store.read(&store::Key::root(kind))) { + Some(Ok(Some(blob))) => Some(blob), + Some(Ok(None)) => None, + Some(Err(_)) => { + None + } + None => None, + } + } + + fn save_all_components(&self, user: crate::sbio::UserId) -> bool { + let mut all = true; + for (who, kind, what) in [ + ( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"master catacomb (user -1)", + ), + ( + crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"owner catacomb (user 501)", + ), + ( + crate::sbio::CatacombUser::enrolling(user), + PRIVATE_TYPE_CATACOMB_USER, + c"user catacomb", + ), + ] { + if !self.save_catacomb(who, kind, what) { + all = false; + } + } + if !self.save_lockout() { + all = false; + } + all + } + + fn save_after_match(&self, user: crate::sbio::UserId) -> bool { + + if !self.save_lockout() { + dev_err!( + self.dev, + "verify: post-match lockout persistence failed; refusing to claim the AP and SEP anti-replay state are synchronized\n" + ); + return false; + } + + if !self.save_catacomb( + crate::sbio::CatacombUser::enrolling(user), + PRIVATE_TYPE_CATACOMB_USER, + c"post-match user catacomb", + ) { + dev_err!( + self.dev, + "verify: post-match user-catacomb persistence failed; the next boot may reject the older AP blob\n" + ); + return false; + } + + if !self.save_catacomb( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"post-match system (id -1) material snapshot", + ) { + dev_err!( + self.dev, + "verify: post-match system material snapshot failed; next boot may reinstall pre-match material and the user load could cold-transition (0x8002)\n" + ); + return false; + } + + if !self.resnapshot_identity_keybag() { + dev_err!( + self.dev, + "verify: post-match identity-bag snapshot failed; the newly confirmed catacomb has no matching durable bag image\n" + ); + return false; + } + + true + } + + fn save_catacomb(&self, who: crate::sbio::CatacombUser, kind: u8, _what: &CStr) -> bool { + let selector = crate::sbio::SaveSelector::new(who); + + let Some(blob) = self.sbio_expect_ok(&crate::sbio::sbio_save_catacomb(&selector)) else { + return false; + }; + if blob.len() < crate::sbio::SBIO_SAVED_MIN || blob.len() > crate::sbio::SBIO_SAVED_MAX { + return false; + } + + let at = crate::sbio::SBIO_SAVED_USER_ID_AT; + let carried = i32::from_le_bytes([blob[at], blob[at + 1], blob[at + 2], blob[at + 3]]); + if carried != who.value() { + return false; + } + + if crate::catacomb::write(kind, &blob).is_err() { + return false; + } + + if self + .sbio_expect_ok(&crate::sbio::sbio_confirm_save(&selector)) + .is_none() + { + return false; + } + true + } + + fn save_lockout(&self) -> bool { + let Some(blob) = self.sbio_expect_ok(&crate::sbio::sbio_save_lockout()) else { + return false; + }; + if blob.is_empty() { + return false; + } + match self.with_store(|store| store.write(&store::Key::root(PRIVATE_TYPE_LOCKOUT), &blob)) { + Some(Ok(())) => { + true + } + Some(Err(_)) => { + false + } + None => false, + } + } + + fn enable_sbio(&self) -> Result<()> { + if self.sbio_ready.load(Relaxed) { + return Ok(()); + } + + self.register_ool(&self.ool_sbio)?; + + self.sbio_ready.store(true, Relaxed); + Ok(()) + } + + fn attach_bringup(&self) { + if !self.sensor_present.load(Relaxed) { + dev_warn!( + self.dev, + "sbio: no sensor bound at attach; catacomb restore ran but capture is unavailable\n" + ); + return; + } + let Some(patch) = self.wake_sensor() else { + return; + }; + if !self.complete_bringup(patch) { + dev_warn!( + self.dev, + "sbio: attach-time sensor bring-up did not complete; matching remains unavailable until a later bring-up succeeds.\n" + ); + } + } + + // ordering: the catacomb restore must complete before the sensor is woken + pub(crate) fn run_bringup(&self) { + if self.bringup_started.xchg(true, Relaxed) { + return; + } + if let Err(e) = self.enable_sbio() { + dev_err!(self.dev, "bringup: could not enable the biometric transport ({:?}); Touch ID is unavailable this boot\n", e); + return; + } + if let Err(e) = self.enable_sks() { + dev_warn!(self.dev, "bringup: could not enable the key store ({:?}); keybag and ref-key operations are unavailable\n", e); + } + if let Ok(keybag::State::Present(stored)) = keybag::read(keybag::Slot::Identity) { + if let Some((handle, uuid)) = self.sks_recover(&stored) { + self.sks_designate_user_keybag(handle, stored.secret()); + self.sks_machine_refkey(handle, stored.secret()); + let prepared = self.cold_match_continue(handle, uuid); + self.ensure_restored_after(prepared); + } + } + self.attach_bringup(); + } + + pub(crate) fn run_verify(&self) { + if !self.templates_restored.load(Relaxed) { + dev_err!( + self.dev, + "verify: refusing before touching the sensor; cold-match prep or restore did not complete, enclave holds no template (every match refused 0x1). A restore failure, not a non-matching finger\n" + ); + self.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), + [0u8; bio::TOKEN_LEN], + ); + return; + } + + let Some(token_bytes) = self.mint_token_bytes() else { + self.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), + [0u8; bio::TOKEN_LEN], + ); + return; + }; + + self.settle_before_capture(); + + let Some(patch) = self.wake_sensor() else { + self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); + let _ = sensor::idle(); + return; + }; + + // enclave refuses a capture from an uncalibrated sensor (0x65 answers 1) + let already_calibrated = self.sensor_calibrated.load(Relaxed); + self.sensor_calibrated.store(true, Relaxed); + let patch = if already_calibrated { + patch + } else { + if !self.calibrate_sensor() { + self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); + let _ = sensor::idle(); + return; + } + let Some(patch) = self.wake_sensor() else { + self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); + let _ = sensor::idle(); + return; + }; + patch + }; + if !self.complete_bringup(patch) { + self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); + let _ = sensor::idle(); + return; + } + + // The match uses the plain 0x23->0x06->0x65 pipeline, with no SCRD step. + + let outcome = self.verify_one_image(); + let _ = sensor::idle(); + sensor::power(false); + self.note_capture_end(); + + let definitive = matches!( + &outcome, + bio::VerifyOutcome::Matched(_) | bio::VerifyOutcome::NoMatch + ); + if definitive { + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + self.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), + token_bytes, + ); + return; + }; + if !self.save_after_match(user) { + self.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), + token_bytes, + ); + return; + } + } + self.finish_verify(outcome, token_bytes); + } + + fn verify_one_image(&self) -> bio::VerifyOutcome { + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + + if sensor::start_capture().is_err() { + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); + } + let advertised = match self.await_capture() { + CaptureWait::Ready(n) => n, + CaptureWait::Timeout => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_TIMEOUT); + } + CaptureWait::Fault(_state) => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); + } + CaptureWait::Abandon => return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), + }; + + let capture = match sensor::read_capture(advertised) { + Ok(capture) => capture, + Err(sensor::CaptureError::Checksum { .. }) | Err(sensor::CaptureError::Length(_)) => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_RETRY); + } + Err(sensor::CaptureError::Bus(_)) => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); + } + Err(sensor::CaptureError::NoMemory) => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR) + } + }; + + if self + .sbio_expect_ok(&crate::sbio::sbio_prepare_image_processing()) + .is_none() + { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + let _image = ImageContext { sep: self }; + + // MATCHING: byte 0x12 set, 0x13 clear; last-image flag at byte 9 (enrol byte 8) + let init = crate::sbio::sbio_image_processing_init( + crate::sbio::ImagePurpose::Matching, + true, + true, + 0, + user, + crate::shim::monotonic_ns(), + ); + if self.sbio_expect_ok(&init).is_none() { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + + let relay = crate::sbio::SbioRelay::new(&capture); + if self.sbio_relay(&relay).is_none() { + dev_err!( + self.dev, + "verify: enclave refused the capture (status above); no comparison performed, not a non-match. Reported as a failure so userspace does not tell the user their finger was rejected\n" + ); + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + drop(capture); + + let Some(assessment) = self.sbio_expect_ok(&crate::sbio::sbio_image_assessment()) else { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + if assessment.len() <= crate::sbio::ASSESS_USABLE_MATCH { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + + if assessment[crate::sbio::ASSESS_USABLE_MATCH] == 0 { + return bio::VerifyOutcome::Failed(ENROL_STATUS_RETRY); + } + + let Some(result) = self.sbio_expect_ok(&crate::sbio::sbio_match_result()) else { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + let Some(parsed) = crate::sbio::MatchResult::parse(&result) else { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + + if !parsed.matches(user) { + return bio::VerifyOutcome::NoMatch; + } + + // match result: user id groups templates, UUID at offset 0x04 identifies one + let identity = *parsed.identity_uuid(); + let known = self.bio_index.lock().contains_uuid(&identity); + + if !known { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + + bio::VerifyOutcome::Matched(bio::MatchEvidence::from_enclave_reply(identity)) + } + + pub(crate) fn run_enrolment(&self) { + let Some(patch) = self.wake_sensor() else { + self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + let _ = sensor::idle(); + return; + }; + + let already_calibrated = self.sensor_calibrated.load(Relaxed); + self.sensor_calibrated.store(true, Relaxed); + let patch = if already_calibrated { + patch + } else { + if !self.calibrate_sensor() { + self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + let _ = sensor::idle(); + return; + } + let Some(patch) = self.wake_sensor() else { + self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + let _ = sensor::idle(); + return; + }; + patch + }; + + if !self.complete_bringup(patch) { + self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + let _ = sensor::idle(); + return; + } + + let Some(open) = self.begin_enrolment_on_enclave() else { + self.finish_enrolment(Err(ENROL_STATUS_ENCLAVE)); + let _ = sensor::idle(); + return; + }; + + let mut counter: u32 = 0; + let mut enrolment_completed = false; + let mut identity: Option<[u8; bio::UUID_LEN]> = None; + + let outcome = loop { + if !bio::enrol_is_live(&self.bio_session.lock()) { + break None; + } + if counter >= ENROL_MAX_CAPTURES { + break Some(Err(ENROL_STATUS_TOO_MANY)); + } + + match self.enrol_one_image(counter) { + ImageOutcome::Progress { + stage, + percent, + complete, + has_template, + } => { + counter = counter.saturating_add(1); + let woke = bio::enrol_advance( + &mut self.bio_session.lock(), + stage, + percent, + bio::Guidance::LiftAndMove, + ); + if woke { + self.bio_wake(); + } + // completion is the flag at offset 0xbfe, not a derived stage count + if complete { + if !has_template { + break Some(Err(ENROL_STATUS_ENCLAVE)); + } + enrolment_completed = true; + break Some(Ok(())); + } + } + ImageOutcome::Retry => { + counter = counter.saturating_add(1); + } + ImageOutcome::NoFinger => { + break Some(Err(ENROL_STATUS_TIMEOUT)); + } + ImageOutcome::Failed(status) => break Some(Err(status)), + } + + self.pace_between_captures(); + }; + + if enrolment_completed { + open.completed(); + + if let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) { + if !self.save_all_components(user) { + dev_warn!( + self.dev, + "enrol: enrolment succeeded but at least one of four artefacts was not persisted; works until the next reboot (see which component above)\n" + ); + } else { + self.templates_restored.store(true, Relaxed); + if !self.resnapshot_identity_keybag() { + dev_warn!( + self.dev, + "enrol: catacombs saved but post-commit identity-bag snapshot was not; template works this boot but is not reboot-persistent\n" + ); + } + } + } + + identity = self.identity_just_enrolled(); + + self.reconcile_identities(); + } + + let _ = sensor::idle(); + sensor::power(false); + self.note_capture_end(); + + if let Some(result) = outcome { + let filed = match (result, identity) { + (Ok(()), Some(uuid)) => Ok(uuid), + (Ok(()), None) => Err(ENROL_STATUS_UNFILED), + (Err(status), _) => Err(status), + }; + self.finish_enrolment(filed); + } + } + + fn register_sensor(&self, id: &sensor::Identifier) -> bool { + let stage = self.bringup.load(Relaxed); + + if stage >= BRINGUP_ESTABLISHED { + if self.sbio_expect_ok(&crate::sbio::sbio_clear_state()).is_none() { + return false; + } + } + + if stage == BRINGUP_FRESH { + let op = crate::sbio::sbio_register_sensor(id); + match self.sbio_call(&op) { + SbioOutcome::Ok(_payload) => { + self.bringup.store(BRINGUP_IDENTIFIED, Relaxed); + } + _ => { + return false; + } + } + } + + let serial = match sensor::read_sensor_serial() { + Ok(serial) => serial, + Err(_) => { + return false; + } + }; + + let op = crate::sbio::sbio_register_sensor_serial(&serial); + match self.sbio_call(&op) { + SbioOutcome::Ok(_payload) => { + self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); + true + } + _ => { + false + } + } + } + + fn wake_sensor(&self) -> Option { + let _source = sensor::power_source(); + let cycled = sensor::power_cycle(); + if !sensor::cs_timing().is_hardware() { + dev_warn!( + self.dev, + "sensor: chip-select timing not reaching hardware; software emulation cannot time this sensor, silent status expected (mode 2 is the other half)\n" + ); + } + + let mut answered_without_identifier = false; + + for delay in sensor::POWER_ON_READ_DELAYS_MS { + if delay > 0 { + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(delay))); + } + let st = match sensor::status() { + Ok(st) => st, + Err(_) => { + return None; + } + }; + if st.is_silent() { + continue; + } + + if let sensor::Offset12::Identifier(id) = st.offset12() { + if id != 0 && id != sensor::EXPECTED_IDENTIFIER { + return None; + } + } + + let Some(id) = st.identifier_to_register() else { + answered_without_identifier = true; + continue; + }; + if !self.register_sensor(&id) { + return None; + } + + // The patch decision is made on status[8], not on state 9. + if st.patch_ack() == sensor::PATCH_ACCEPTED { + return Some(PatchLoaded(())); + } + return self.apply_sensor_patch(st.patch_ack()); + } + + dev_err!( + self.dev, + "sensor gated: all {} wake reads returned sixteen zero bytes. Power {}. CS timing: {}. Mode 2 (CPOL=1 CPHA=0)\n", + sensor::POWER_ON_READ_DELAYS_MS.len(), + if cycled { c"was cycled" } else { c"was NOT cycled" }, + sensor::cs_timing().name() + ); + if answered_without_identifier { + return None; + } + None + } + + fn calibrate_sensor(&self) -> bool { + let blob = match kernel::firmware::Firmware::request(CALIBRATION_FIRMWARE, &self.dev) { + Ok(fw) => CalibrationBlob::new(fw), + Err(e) => { + dev_err!( + self.dev, + "sensor: calibration blob {} could not be loaded ({:?}); per-device factory data, cannot be synthesised, stopping\n", + CALIBRATION_FIRMWARE, + e + ); + return false; + } + }; + + let Some(challenge) = self.sbio_expect_ok(&crate::sbio::sbio_module_challenge()) else { + return false; + }; + if challenge.len() != crate::sbio::SBIO_MODULE_CHALLENGE_LEN { + return false; + } + + let challenge = sensor::Params::new(challenge); + if sensor::send_module_challenge(&challenge).is_err() { + return false; + } + + let reply = match sensor::read_module_reply() { + Ok(reply) => reply, + Err(_) => { + return false; + } + }; + + // commit the sensor's reply, not the challenge; both are 64 bytes at count 0x4b + let Some(commit) = crate::sbio::sbio_module_commit(&reply) else { + return false; + }; + let Some(_serial) = self.sbio_expect_ok(&commit) else { + return false; + }; + + let request = match crate::sbio::SbioCalibration::new(&blob) { + Ok(request) => request, + Err(_) => { + return false; + } + }; + if self + .sbio_transfer_raw(request.opcode(), request.name(), request.payload()) + .ok() + .filter(|done| done.status.is_ok()) + .is_none() + { + dev_err!( + self.dev, + "sensor: 0x5b LOAD_CALIBRATION failed. Without it every capture is refused with status 1.\n" + ); + return false; + } + + let _ = self.sbio_call(&crate::sbio::sbio_diagnostics()); + + true + } + + fn complete_bringup(&self, patch: PatchLoaded) -> bool { + + let Some(params) = self.apply_sensor_parameters() else { + return false; + }; + + let op = crate::sbio::sbio_complete_init(patch, params); + if self.sbio_expect_ok(&op).is_none() { + dev_err!(self.dev, "sensor: 0x01 COMPLETE_INIT failed — status above\n"); + return false; + } + + let listed = self.sync_device_view(); + + if listed && !self.device_view_synced.xchg(true, Relaxed) { + let count = self.log_identity_count(c"after device-view synchronisation"); + match count { + Some(0) => { + self.templates_restored.store(false, Relaxed); + } + Some(n) if self.templates_restored.load(Relaxed) && self.prove_restore() => { + dev_warn!( + self.dev, + "Touch ID: restored {} enrolled identity/identities; enrolment survived reboot\n", + n + ); + } + Some(_) => { + self.templates_restored.store(false, Relaxed); + } + None => { + self.templates_restored.store(false, Relaxed); + } + } + if self.templates_restored.load(Relaxed) { + self.reconcile_identities(); + } + } + + true + } + + fn apply_sensor_parameters(&self) -> Option { + // 0x5d and 0x5c: an empty blob is a failure. + self.relay_parameters( + &crate::sbio::sbio_coverage_params(), + &sensor::Geometry::COVERAGE, + false, + )?; + self.relay_parameters( + &crate::sbio::sbio_operation_params(), + &sensor::Geometry::OPERATION, + false, + )?; + // 0x6a: an empty blob is success, and relaying nothing is correct. + self.relay_parameters( + &crate::sbio::sbio_transparent_channel(), + &sensor::Geometry::TRANSPARENT, + true, + )?; + + Some(ParametersApplied(())) + } + + fn relay_parameters( + &self, + op: &crate::sbio::SbioOp, + geom: &sensor::Geometry, + empty_is_success: bool, + ) -> Option { + let blob = match self.sbio_call(op) { + SbioOutcome::Ok(b) => sensor::Params::new(b), + _ => { + return None; + } + }; + + if blob.is_empty() { + if empty_is_success { + return Some(0); + } + return None; + } + + let relayed = blob.len(); + match sensor::send_encrypted_parameters(&blob, geom) { + Ok(()) => { + Some(relayed) + } + Err(sensor::ParamsError::Empty) => { + None + } + Err(sensor::ParamsError::TooLong(n, capacity)) => { + dev_err!( + self.dev, + "sensor: {}'s blob is {} bytes and this frame holds {} (declared 0x{:x} less the 9 bytes of header and CRC)\n", + op.name(), + n, + capacity, + geom.declared() + ); + None + } + Err(sensor::ParamsError::Transfer(_e)) => { + None + } + } + } + + fn establish_session(&self) -> bool { + + let share = match self.sbio_call(&crate::sbio::sbio_request_session_share()) { + SbioOutcome::Ok(sh) => sh, + SbioOutcome::Status16 => { + return false; + } + SbioOutcome::PrerequisiteMissing => { + return false; + } + SbioOutcome::Other => { + return false; + } + }; + + // 0x30 reserved for this reply; length is a floor, take the first 40 + if share.len() < sensor::SESSION_SHARE_LEN { + return false; + } + let mut out = [0u8; sensor::SESSION_SHARE_LEN]; + out.copy_from_slice(&share[..sensor::SESSION_SHARE_LEN]); + + // relay it: class 0x72, no CRC, no padding + if sensor::send_session_share(&out).is_err() { + return false; + } + + let reply = match sensor::read_session_reply() { + Ok(r) => r, + Err(_) => { + return false; + } + }; + + match self.sbio_transfer(&crate::sbio::sbio_commit_session_share(&reply)) { + Ok(done) if done.status.is_ok() => { + true + } + Ok(_) => { + false + } + Err(_) => { + false + } + } + } + + fn init_sequence_counter(&self) -> bool { + + let challenge = match self.sbio_call(&crate::sbio::sbio_request_challenge()) { + SbioOutcome::Ok(c) => c, + SbioOutcome::PrerequisiteMissing => { + if !self.establish_session() { + return false; + } + match self.sbio_call(&crate::sbio::sbio_request_challenge()) { + SbioOutcome::Ok(c) => c, + _ => { + return false; + } + } + } + SbioOutcome::Status16 => { + return false; + } + SbioOutcome::Other => { + return false; + } + }; + if challenge.len() < sensor::CHALLENGE_LEN { + return false; + } + let mut out = [0u8; sensor::CHALLENGE_LEN]; + out.copy_from_slice(&challenge[..sensor::CHALLENGE_LEN]); + + if sensor::send_challenge(&out).is_err() { + return false; + } + + let reply = match sensor::read_challenge_reply() { + Ok(r) => r, + Err(_) => { + return false; + } + }; + + match self.sbio_transfer(&crate::sbio::sbio_commit_challenge(&reply)) { + Ok(done) if done.status.is_ok() => { + true + } + Ok(_) => { + false + } + Err(_) => { + false + } + } + } + + fn apply_sensor_patch(&self, _ack_before: u8) -> Option { + let blob = match self.sbio_call(&crate::sbio::sbio_fetch_patch()) { + SbioOutcome::Ok(b) => b, + SbioOutcome::PrerequisiteMissing => { + if !self.init_sequence_counter() { + return None; + } + match self.sbio_call(&crate::sbio::sbio_fetch_patch()) { + SbioOutcome::Ok(b) => b, + _ => { + return None; + } + } + } + SbioOutcome::Status16 => { + return None; + } + SbioOutcome::Other => { + return None; + } + }; + if blob.is_empty() { + return None; + } + + // enable command is mandatory; skipping it leaves the sensor in state 9 + if sensor::setup_patch_enable().is_err() { + return None; + } + + if !self.await_sensor_state(sensor::STATE_IDLE, c"idle, before sending the patch") { + return None; + } + + if sensor::send_patch(&blob).is_err() { + return None; + } + + // acceptance is at status[8], not at the state + for _attempt in 0..PATCH_POLL_ATTEMPTS { + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(PATCH_POLL_MS))); + let st = match sensor::status() { + Ok(st) => st, + Err(_) => { + return None; + } + }; + if st.patch_ack() == sensor::PATCH_ACCEPTED { + + if let Ok(after) = sensor::status() { + if after.state == sensor::STATE_NEEDS_PATCH { + return None; + } + } + return Some(PatchLoaded(())); + } + } + + let _ = sensor::status(); + None + } + + fn await_sensor_state(&self, want: u8, _why: &CStr) -> bool { + for _attempt in 0..PATCH_POLL_ATTEMPTS { + match sensor::status() { + Ok(st) if st.state == want => { + return true; + } + Ok(_) => {} + Err(_) => { + return false; + } + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(PATCH_POLL_MS))); + } + false + } + + fn enrol_one_image(&self, counter: u32) -> ImageOutcome { + if sensor::start_capture().is_err() { + return ImageOutcome::Failed(ENROL_STATUS_SENSOR); + } + + let available = match self.await_capture() { + CaptureWait::Ready(n) => n, + CaptureWait::Timeout => return ImageOutcome::NoFinger, + CaptureWait::Fault(_) => { + return ImageOutcome::Failed(ENROL_STATUS_SENSOR); + } + CaptureWait::Abandon => return ImageOutcome::Failed(ENROL_STATUS_SENSOR), + }; + + let capture = match sensor::read_capture(available) { + Ok(c) => c, + Err(sensor::CaptureError::Checksum { advertised: _advertised, computed: _computed }) => { + return ImageOutcome::Retry; + } + Err(sensor::CaptureError::Length(_n)) => { + return ImageOutcome::Retry; + } + Err(sensor::CaptureError::Bus(_e)) => { + return ImageOutcome::Failed(ENROL_STATUS_SENSOR); + } + Err(sensor::CaptureError::NoMemory) => { + return ImageOutcome::Failed(ENROL_STATUS_SENSOR) + } + }; + + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + if self + .sbio_expect_ok(&crate::sbio::sbio_prepare_image_processing()) + .is_none() + { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + let _image = ImageContext { sep: self }; + + let init = crate::sbio::sbio_image_processing_init( + crate::sbio::ImagePurpose::Enrolment, + counter == 0, + false, + counter, + user, + crate::shim::monotonic_ns(), + ); + if self.sbio_expect_ok(&init).is_none() { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + + let relay = crate::sbio::SbioRelay::new(&capture); + if self.sbio_relay(&relay).is_none() { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + drop(capture); + + let Some(assessment) = self.sbio_expect_ok(&crate::sbio::sbio_image_assessment()) else { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + if assessment.len() < crate::sbio::ASSESS_MIN_LEN { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + let usable = assessment[crate::sbio::ASSESS_USABLE_ENROL] != 0; + if !usable { + return ImageOutcome::Retry; + } + + let Some(result) = self.sbio_expect_ok(&crate::sbio::sbio_enrolment_result()) else { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + let Some(parsed) = crate::sbio::EnrolmentResult::parse(&result) else { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + + if parsed.complete() { + self.stash_enrol_identity(&result); + } + + let percent = parsed.progress_percent(); + + ImageOutcome::Progress { + stage: (percent * bio::ENROL_STAGES) + .div_ceil(100) + .min(bio::ENROL_STAGES), + percent, + complete: parsed.complete(), + has_template: parsed.has_template(), + } + } + + fn await_capture(&self) -> CaptureWait { + let mut previous: Option<[u8; sensor::STATUS_LEN]> = None; + let mut armed_reported = false; + + for attempt in 0..ENROL_POLL_ATTEMPTS { + if !bio::capture_is_live(&self.bio_session.lock()) { + return CaptureWait::Abandon; + } + + let st = match sensor::status() { + Ok(st) => st, + Err(_) => { + return CaptureWait::Abandon; + } + }; + + if attempt == 0 || previous != Some(st.raw) { + previous = Some(st.raw); + } + + if st.state == sensor::STATE_NEEDS_PATCH { + return CaptureWait::Fault(st.state); + } + + let guide = match st.state { + sensor::STATE_ARMED => Some(bio::Guidance::Place), + sensor::STATE_READING => Some(bio::Guidance::HoldStill), + _ => None, + }; + if let Some(guide) = guide { + if bio::enrol_guide(&mut self.bio_session.lock(), guide) { + self.bio_wake(); + } + } + + if st.state == sensor::STATE_ARMED && !armed_reported { + armed_reported = true; + } + + if let sensor::Offset12::Available(count) = st.offset12() { + if count == 0 { + return CaptureWait::Timeout; + } + return CaptureWait::Ready(count); + } + + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ENROL_POLL_MS))); + } + let _ = sensor::status(); + CaptureWait::Timeout + } + + pub(crate) fn on_sbio(&self, msg: Message) { + let f = proto::decode(&msg); + let marker = f.tag; + + if !self.sbio_ready.load(Relaxed) { + return; + } + + // notification and the enclave's DMA are not ordered; buffer may still be poison + if !self.ool_await_written(&self.ool_sbio, 0, transfer::HEADER_LEN) { + self.sbio_fail_unwritten(); + return; + } + + let header = match self.ool_read(&self.ool_sbio, 0, transfer::HEADER_LEN) { + Ok(h) => h, + Err(_) => { + self.sbio_fail(); + return; + } + }; + let packet = match transfer::Packet::decode(&header) { + Ok(p) => p, + Err(_) => { + self.sbio_fail(); + return; + } + }; + + let _repeated_header = { + let mut bytes = [0u8; transfer::HEADER_LEN]; + bytes.copy_from_slice(&header[..transfer::HEADER_LEN]); + let mut last = self.sbio_last_header.lock(); + let same = last.as_ref().is_some_and(|prev| *prev == bytes); + *last = Some(bytes); + same + }; + + let chunk = packet.chunk as usize; + let payload = if chunk == 0 { + KVec::new() + } else if !self.ool_await_written(&self.ool_sbio, transfer::HEADER_LEN, chunk) { + self.sbio_fail_unwritten(); + return; + } else { + match self.ool_read(&self.ool_sbio, transfer::HEADER_LEN, chunk) { + Ok(p) => p, + Err(_) => { + self.sbio_fail(); + return; + } + } + }; + + let progress = self.sbio_rx.lock().on_chunk(marker, &packet, &payload); + + match progress { + transfer::Progress::NeedMore(cont) => { + self.sbio_continue(&cont); + } + transfer::Progress::Complete => { + // 0xFE requests the peer's next packet and acks the final one; nothing to send + self.sbio_wq.notify_all(); + } + transfer::Progress::Ignored(_why) => {}, + transfer::Progress::Grant => { + self.sbio_wq.notify_all(); + } + transfer::Progress::Notification { tag: _tag, opcode: _opcode } => { + let _ = self.sbio_rx.lock().awaiting(); + } + transfer::Progress::Failed(_why) => { + self.sbio_wq.notify_all(); + } + } + } + + fn sbio_continue(&self, cont: &transfer::Continuation) { + let mut header = [0u8; transfer::HEADER_LEN]; + if cont.packet().encode(&mut header).is_err() { + self.sbio_fail(); + return; + } + if self.ool_write(&self.ool_sbio, 0, &header).is_err() { + self.sbio_fail(); + return; + } + if self.send(crate::sbio::encode_sbio_continue(cont)).is_err() { + self.sbio_fail(); + } + } + + fn sbio_fail(&self) { + self.sbio_rx.lock().abort(); + self.sbio_wq.notify_all(); + } + + fn sbio_fail_unwritten(&self) { + self.sbio_rx + .lock() + .abort_with(transfer::DeviceStatus::BufferNeverWritten); + self.sbio_wq.notify_all(); + } + + fn sbio_transfer(&self, op: &crate::sbio::SbioOp) -> Result { + self.sbio_transfer_raw(op.opcode(), op.name(), op.payload()) + } + + fn sbio_transfer_raw( + &self, + opcode: u16, + name: &'static CStr, + payload: &[u8], + ) -> Result { + if !self.sbio_ready.load(Relaxed) { + return Err(ENODEV); + } + + self.sbio_rx.lock().begin(u32::from(opcode))?; + + if let Err(e) = self.sbio_start(opcode, name, payload) { + let mut guard = self.sbio_rx.lock(); + guard.abort(); + let _ = guard.take_done(); + return Err(e); + } + + let mut remaining = time::msecs_to_jiffies(SBIO_TIMEOUT_MS); + let mut guard = self.sbio_rx.lock(); + loop { + if let Some(done) = guard.take_done() { + let _ = done.opcode; + return Ok(done); + } + if remaining == 0 { + guard.abort(); + let _ = guard.take_done(); + drop(guard); + return Err(ETIMEDOUT); + } + match self + .sbio_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + + fn sbio_start(&self, opcode: u16, name: &'static CStr, payload: &[u8]) -> Result<()> { + if crate::sbio::encode_sbio_raw(opcode, 0).is_none() { + return Err(EPERM); + } + + let total = payload.len(); + if total > transfer::MAX_TRANSACTION as usize { + return Err(EINVAL); + } + + let capacity = OOL_SIZE_SBIO - transfer::HEADER_LEN; + + self.sbio_rx.lock().begin_send(u32::from(opcode)); + let result = self.sbio_send_chunks(opcode, name, payload, capacity); + self.sbio_rx.lock().end_send(); + result + } + + fn sbio_send_chunks( + &self, + opcode: u16, + name: &'static CStr, + payload: &[u8], + capacity: usize, + ) -> Result<()> { + let total = payload.len(); + let mut offset = 0usize; + let mut first = true; + let mut seq: u16 = 0; + let mut request = KVec::new(); + + loop { + let chunk = core::cmp::min(total - offset, capacity); + + let packet = transfer::Packet { + version: 1, + total: total as u32, + offset: offset as u32, + flags: 0, + err: 0, + opcode: u32::from(opcode), + chunk: chunk as u32, + }; + + request.clear(); + request.resize(transfer::HEADER_LEN + chunk, 0u8, GFP_KERNEL)?; + packet.encode(&mut request)?; + request[transfer::HEADER_LEN..].copy_from_slice(&payload[offset..offset + chunk]); + + self.ool_write(&self.ool_sbio, 0, &request)?; + + let msg = if first { + crate::sbio::encode_sbio_raw(opcode, seq) + } else { + crate::sbio::encode_sbio_next(opcode, seq) + } + .ok_or(EPERM)?; + self.send(msg)?; + + offset += chunk; + first = false; + seq = seq.wrapping_add(1); + if offset >= total { + return Ok(()); + } + + self.sbio_await_grant(name, offset, total)?; + } + } + + fn sbio_await_grant(&self, _name: &'static CStr, _sent: usize, _total: usize) -> Result<()> { + let mut remaining = time::msecs_to_jiffies(SBIO_TIMEOUT_MS); + let mut guard = self.sbio_rx.lock(); + loop { + if guard.take_grant() { + return Ok(()); + } + if remaining == 0 { + drop(guard); + return Err(ETIMEDOUT); + } + match self + .sbio_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + + pub(crate) fn register_bio(&self) -> Result<()> { + let mut guard = self.bio_dev.lock(); + if guard.is_some() { + return Ok(()); + } + + let ctx = core::ptr::from_ref(self).cast_mut().cast::(); + + // SAFETY: `ctx` is this `SepData`, kept alive by the `Arc` in the + // driver's private data. `remove()` drops the registration before that + // `Arc` can go, and the shim's unregister clears its stored context + // under the same lock every callback takes, so once it returns, no + // callback can observe this pointer again. + let dev = unsafe { + shim::BioChardev::register( + bio::DEVICE_NAME, + bio::DEVICE_MODE, + ctx, + bio_open_trampoline, + bio_release_trampoline, + bio_ioctl_trampoline, + bio_ready_trampoline, + ) + }?; + + *guard = Some(dev); + Ok(()) + } + + pub(crate) fn bio_open(&self) -> Result<()> { + bio::open(&mut self.bio_session.lock()) + } + + pub(crate) fn bio_release(&self) { + bio::release(&mut self.bio_session.lock()); + if let Some(dev) = self.bio_dev.lock().as_ref() { + dev.wake(); + } + } + + pub(crate) fn queue_enrolment(this: Arc) { + if workqueue::system() + .enqueue::, ENROL_WORK_ID>(this.clone()) + .is_err() + { + this.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + } + } + + pub(crate) fn queue_verify(this: Arc) { + if workqueue::system() + .enqueue::, VERIFY_WORK_ID>(this.clone()) + .is_err() + { + this.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), + [0u8; bio::TOKEN_LEN], + ); + } + } + + fn finish_verify(&self, outcome: bio::VerifyOutcome, token_bytes: [u8; bio::TOKEN_LEN]) { + let filed = bio::verify_finish(&mut self.bio_session.lock(), outcome, token_bytes); + if filed { + self.bio_wake(); + } + } + + fn mint_token_bytes(&self) -> Option<[u8; bio::TOKEN_LEN]> { + let mut bytes = [0u8; bio::TOKEN_LEN]; + for chunk in bytes.chunks_mut(4) { + match self.get_entropy_word() { + Ok(word) => chunk.copy_from_slice(&word.to_le_bytes()[..chunk.len()]), + Err(_) => { + return None; + } + } + } + Some(bytes) + } + + pub(crate) fn bio_ioctl(&self, cmd: u32, arg: usize) -> Result { + if cmd == bio::IOC_ATTEST { + return self.bio_attest(arg); + } + let handled = { + let mut session = self.bio_session.lock(); + let index = self.bio_index.lock(); + let mut ctx = bio::Context { + session: &mut session, + index: &index, + sensor_present: self.sensor_present.load(Relaxed), + }; + bio::ioctl(&mut ctx, cmd, arg)? + }; + + // 0x57 sent with the session lock released (it waits on a mailbox reply) + if let Some(identity) = handled.delete_identity { + self.delete_identity(&identity)?; + } + + let mut delete_all_err: Option = None; + for identity in handled.delete_identities.iter() { + if let Err(e) = self.delete_identity(identity) { + delete_all_err = delete_all_err.or(Some(e)); + } + } + if let Some(e) = delete_all_err { + return Err(e); + } + + if handled.wake { + self.bio_wake(); + } + Ok(handled) + } + + fn bio_attest(&self, arg: usize) -> Result { + let user = kernel::uaccess::UserPtr::from_addr(arg); + let req: bio::Attest = kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; + let (sig, pubk) = self.refkey_attest_sign(&req.challenge)?; + if sig.len() > bio::ATTEST_SIG_MAX || pubk.len() != bio::ATTEST_PUB_LEN { + return Err(EIO); + } + let mut out = bio::Attest { + sig_len: sig.len() as u32, + challenge: req.challenge, + public: [0u8; bio::ATTEST_PUB_LEN], + signature: [0u8; bio::ATTEST_SIG_MAX], + reserved: [0u8; 3], + }; + out.public.copy_from_slice(&pubk); + out.signature[..sig.len()].copy_from_slice(&sig); + kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&out)?; + Ok(bio::Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }) + } + + fn ensure_restored_after(&self, _prepared: bool) { + let restored = self.restore_all_components(); + self.templates_restored.store(restored, Relaxed); + if !restored { + dev_err!( + self.dev, + "matching unavailable this boot: restore did not complete, enclave holds no template (every match refused 0x1). Not the sensor or the finger; on-disk enrolments intact\n" + ); + return; + } + + if self.keybag_designated.load(Relaxed) { + if let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) { + let special = user.special_handle(); + if let Ok(keybag::State::Present(stored)) = keybag::read(keybag::Slot::Identity) { + let _ = self.sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { + self.sks_req_unlock_special(special, stored.secret(), healthy) + }); + } + } + } + + // enclave's match arm asserts a non-empty ACM context (<= 32 bytes) + if let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) { + self.establish_scrd_match_context(user); + } + } + + fn cold_match_continue( + &self, + source: crate::sks::KeyBagHandle, + uuid: [u8; keybag::UUID_LEN], + ) -> bool { + if !self.keybag_designated.load(Relaxed) { + return false; + } + + let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) else { + return false; + }; + let special = user.special_handle(); + self.log_identity_count(c"before the cold-match preparation"); + + let uuid_ok = self + .sks_send(self.sks_req_copy_uuid_special(special)) + .and_then(|out| self.sks_uuid_from_reply(&out)); + match uuid_ok { + Some(got) if got == uuid => {}, + Some(_) => { + return false; + } + None => { + return false; + } + } + + if self.sks_send(self.sks_req_unload_keybag(source)).is_none() { + return false; + } + + // unlock/0x18 is the destructive-template path (bag stays designated, locked) + self.cold_prepared.store(true, Relaxed); + true + } + + pub(crate) fn sep_random(&self, buf: &mut [u8]) -> Result<()> { + for chunk in buf.chunks_mut(4) { + let word = self.get_entropy_word()?; + let bytes = word.to_le_bytes(); + let take = chunk.len(); + chunk.copy_from_slice(&bytes[..take]); + } + Ok(()) + } + + pub(crate) fn wrapped_from_copy_reply(&self, out: &SksOutcome, _from: &CStr) -> Option> { + let body = self.sks_report_response(c"COPY_KEYBAG", out)?; + if out.reply.status != 0 || image::operation_status(body).unwrap_or(-1) != 0 { + return None; + } + let (blob, _) = image::read_blob(body, 4)?; + if blob.is_empty() { + return None; + } + let mut copy = KVec::new(); + if copy.extend_from_slice(blob, GFP_KERNEL).is_err() { + return None; + } + Some(copy) + } + + fn prove_restore(&self) -> bool { + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + return false; + }; + let op = crate::sbio::sbio_free_identity_count(user); + let Some(reply) = self.sbio_expect_ok(&op) else { + return false; + }; + if reply.len() != crate::sbio::SBIO_FREE_COUNT_REPLY_LEN { + return false; + } + let count = u32::from_le_bytes([reply[0], reply[1], reply[2], reply[3]]); + if count > crate::sbio::SBIO_FREE_COUNT_MAX { + return false; + } + if count == crate::sbio::SBIO_FREE_COUNT_MAX { + return true; + } + true + } + + fn log_identity_count(&self, _when: &CStr) -> Option { + let op = crate::sbio::sbio_list_identities(); + let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { + return None; + }; + let records = crate::sbio::IdentityRecords::new(&reply)?; + Some(records.count()) + } + + fn enclave_lists(&self, uuid: &[u8; bio::UUID_LEN]) -> Option { + let op = crate::sbio::sbio_list_identities(); + let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { + return None; + }; + let records = crate::sbio::IdentityRecords::new(&reply)?; + Some(records.lists_uuid(uuid)) + } + + fn delete_identity(&self, identity: &crate::sbio::IdentityV1) -> Result<()> { + if self.enclave_lists(identity.uuid()) == Some(false) { + self.forget_identity(identity.uuid(), c"a host entry the enclave never had"); + return Ok(()); + } + + let op = crate::sbio::sbio_delete_identity(identity); + + if self.sbio_expect_ok(&op).is_none() { + dev_err!( + self.dev, + "bio: 0x57 failed, status above; host index left unchanged (forgetting a template the enclave still holds would report a deletion that did not happen)\n" + ); + return Err(EIO); + } + + self.forget_identity(identity.uuid(), c"removed from the enclave"); + Ok(()) + } + + fn forget_identity(&self, uuid: &[u8; bio::UUID_LEN], why: &CStr) { + let persisted = { + let mut index = self.bio_index.lock(); + index.remove(uuid); + self.with_store(|store| index.persist(store)) + }; + match persisted { + Some(Ok(())) => {}, + _ => dev_warn!( + self.dev, + "bio: {} ({}) but the host index could not be written; correct in memory this boot, rebuilt at next attach\n", + Hex(uuid), + why + ), + } + } + + pub(crate) fn bio_ready(&self) -> bool { + self.bio_session.lock().ready() + } +} + +pub(crate) struct SbioOp { + opcode: u16, + payload: [u8; SBIO_MAX_PAYLOAD], + payload_len: usize, + name: &'static CStr, +} + +const SBIO_MAX_PAYLOAD: usize = 0x97; + +pub(crate) const SBIO_IMAGE_INIT_LEN: usize = 0x97; +static_assert!(SBIO_IMAGE_INIT_LEN <= SBIO_MAX_PAYLOAD); + +const fn pad_payload(src: &[u8]) -> [u8; SBIO_MAX_PAYLOAD] { + let mut out = [0u8; SBIO_MAX_PAYLOAD]; + let mut i = 0; + while i < src.len() { + out[i] = src[i]; + i += 1; + } + out +} + +impl SbioOp { + pub(crate) fn opcode(&self) -> u16 { + self.opcode + } + pub(crate) fn payload(&self) -> &[u8] { + &self.payload[..self.payload_len] + } + pub(crate) fn name(&self) -> &'static CStr { + self.name + } +} + +pub(crate) const SBIO_PROTOCOL_GENERATION: u32 = 1; + +const OP_SBIO_INIT_COMMS: u16 = 0x73; + +const OP_SBIO_REGISTER_SENSOR: u16 = 0x80; +pub(crate) fn sbio_register_sensor(id: &crate::sensor::Identifier) -> SbioOp { + SbioOp { + opcode: OP_SBIO_REGISTER_SENSOR, + payload: pad_payload(&id.value().to_le_bytes()), + payload_len: 2, + name: c"REGISTER_SENSOR", + } +} + +const OP_SBIO_SEND_SERIAL: u16 = 0x48; + +pub(crate) const SBIO_SIGNAL_QUALITY: u32 = 0; + +const OP_SBIO_COVERAGE_PARAMS: u16 = 0x5d; +pub(crate) fn sbio_coverage_params() -> SbioOp { + SbioOp { + opcode: OP_SBIO_COVERAGE_PARAMS, + payload: pad_payload(&SBIO_SIGNAL_QUALITY.to_le_bytes()), + payload_len: 4, + name: c"COVERAGE_PARAMS", + } +} + +const OP_SBIO_OPERATION_PARAMS: u16 = 0x5c; + +pub(crate) const SBIO_OPERATION_PARAMS_LEN: usize = match SBIO_PROTOCOL_GENERATION { + 1 => 4, + 6 => 8, + _ => 0, +}; +static_assert!(SBIO_OPERATION_PARAMS_LEN != 0); +static_assert!(SBIO_OPERATION_PARAMS_LEN <= SBIO_MAX_PAYLOAD); + +pub(crate) fn sbio_operation_params() -> SbioOp { + SbioOp { + opcode: OP_SBIO_OPERATION_PARAMS, + payload: pad_payload(&[]), + payload_len: SBIO_OPERATION_PARAMS_LEN, + name: c"OPERATION_PARAMS", + } +} + +const OP_SBIO_TRANSPARENT_CHANNEL: u16 = 0x6a; +pub(crate) fn sbio_transparent_channel() -> SbioOp { + SbioOp { + opcode: OP_SBIO_TRANSPARENT_CHANNEL, + payload: pad_payload(&[]), + payload_len: 0, + name: c"TRANSPARENT_CHANNEL", + } +} + +const OP_SBIO_COMPLETE_INIT: u16 = 0x01; +pub(crate) fn sbio_complete_init( + patch: crate::PatchLoaded, + params: crate::ParametersApplied, +) -> SbioOp { + let (_, _) = (patch, params); + let mut body = [0u8; 8]; + body[0..4].copy_from_slice(&1u32.to_le_bytes()); + body[4..8].copy_from_slice(&1u32.to_le_bytes()); + SbioOp { + opcode: OP_SBIO_COMPLETE_INIT, + payload: pad_payload(&body), + payload_len: 8, + name: c"COMPLETE_INIT", + } +} + +const CONTEXT_SCOPE_SYSTEM: i32 = -1; +static_assert!(CONTEXT_SCOPE_SYSTEM != 0); +static_assert!(CONTEXT_SCOPE_SYSTEM < 0); + +#[derive(Clone, Copy)] +pub(crate) struct ContextScope(i32); + +impl ContextScope { + pub(crate) const SYSTEM: ContextScope = ContextScope(CONTEXT_SCOPE_SYSTEM); + + pub(crate) fn user(id: UserId) -> ContextScope { + ContextScope(id.value()) + } + + pub(crate) fn value(&self) -> i32 { + self.0 + } +} + +const OP_SBIO_CONTEXT_STATE: u16 = 0x6b; +pub(crate) fn sbio_context_state() -> SbioOp { + SbioOp { + opcode: OP_SBIO_CONTEXT_STATE, + payload: pad_payload(&[]), + payload_len: 0, + name: c"CONTEXT_STATE", + } +} + +// 0x2d destroys the catacomb; only send with zero existing identities +pub(crate) struct NoExistingCatacomb { + _seal: (), +} + +impl NoExistingCatacomb { + pub(crate) fn from_zero_identities(count: usize) -> Option { + match count { + 0 => Some(NoExistingCatacomb { _seal: () }), + _ => None, + } + } +} + +const OP_SBIO_SELECT_CONTEXT: u16 = 0x2d; +pub(crate) fn sbio_select_context(scope: ContextScope, proof: &NoExistingCatacomb) -> SbioOp { + let NoExistingCatacomb { _seal: () } = proof; + SbioOp { + opcode: OP_SBIO_SELECT_CONTEXT, + payload: pad_payload(&scope.value().to_le_bytes()), + payload_len: 4, + name: c"SELECT_CONTEXT", + } +} + +pub(crate) const SBIO_PROTECTED_CONFIG_LEN: usize = 32; + +const OP_SBIO_PROTECTED_CONFIG: u16 = 0x2b; +pub(crate) fn sbio_protected_config(id: UserId) -> SbioOp { + SbioOp { + opcode: OP_SBIO_PROTECTED_CONFIG, + payload: pad_payload(&id.value().to_le_bytes()), + payload_len: 4, + name: c"PROTECTED_CONFIG", + } +} + +const OP_SBIO_BEGIN_ENROL: u16 = 0x03; + +// type 0 (ACM context) survives a reboot; type 1 (SKS token) does not +pub(crate) const BE_AUTH_TYPE_ACM_CONTEXT: u32 = 0; +pub(crate) const BE_AUTH_TYPE_SKS_TOKEN: u32 = 1; +static_assert!(BE_AUTH_TYPE_ACM_CONTEXT != BE_AUTH_TYPE_SKS_TOKEN); + +pub(crate) const SBIO_BEGIN_ENROL_LEN: usize = 0x44; +static_assert!(SBIO_BEGIN_ENROL_LEN <= SBIO_MAX_PAYLOAD); + +const BE_FLAGS: usize = 0; +const BE_USER_ID: usize = 4; +const BE_AUTH_TYPE: usize = 8; +const BE_TOKEN_LEN: usize = 12; +const BE_TOKEN: usize = 16; +const BE_SELECTOR: usize = 48; +static_assert!(BE_TOKEN + SKS_AUTH_TOKEN_LEN <= BE_SELECTOR); +const BE_SELECTOR_LEN: usize = 20; +static_assert!(BE_SELECTOR + BE_SELECTOR_LEN == SBIO_BEGIN_ENROL_LEN); + +pub(crate) const SBIO_BEGIN_ENROL_COPIED: usize = match SBIO_PROTOCOL_GENERATION { + 1 => 0x30, + 6 => SBIO_BEGIN_ENROL_LEN, + _ => 0, +}; +static_assert!(SBIO_BEGIN_ENROL_COPIED != 0); +static_assert!(SBIO_BEGIN_ENROL_COPIED <= SBIO_BEGIN_ENROL_LEN); + +const BE_SELECTOR_IN_RECORD: bool = SBIO_BEGIN_ENROL_COPIED >= BE_SELECTOR + BE_SELECTOR_LEN; + +static_assert!(!BE_SELECTOR_IN_RECORD); + +pub(crate) fn sbio_begin_enrol( + user: UserId, + auth_type: u32, + token: &[u8; SKS_AUTH_TOKEN_LEN], +) -> SbioOp { + let mut body = [0u8; SBIO_BEGIN_ENROL_LEN]; + body[BE_FLAGS..BE_FLAGS + 4].copy_from_slice(&0u32.to_le_bytes()); + body[BE_USER_ID..BE_USER_ID + 4].copy_from_slice(&user.value().to_le_bytes()); + body[BE_AUTH_TYPE..BE_AUTH_TYPE + 4].copy_from_slice(&auth_type.to_le_bytes()); + body[BE_TOKEN_LEN..BE_TOKEN_LEN + 4] + .copy_from_slice(&(SKS_AUTH_TOKEN_LEN as u32).to_le_bytes()); + body[BE_TOKEN..BE_TOKEN + SKS_AUTH_TOKEN_LEN].copy_from_slice(token); + SbioOp { + opcode: OP_SBIO_BEGIN_ENROL, + payload: pad_payload(&body), + payload_len: SBIO_BEGIN_ENROL_LEN, + name: c"BEGIN_ENROL", + } +} + +const OP_SBIO_MODULE_CHALLENGE: u16 = 0x31; +pub(crate) fn sbio_module_challenge() -> SbioOp { + SbioOp { + opcode: OP_SBIO_MODULE_CHALLENGE, + payload: pad_payload(&[]), + payload_len: 0, + name: c"MODULE_CHALLENGE", + } +} + +pub(crate) const SBIO_MODULE_CHALLENGE_LEN: usize = 0x40; + +const OP_SBIO_MODULE_COMMIT: u16 = 0x32; +pub(crate) fn sbio_module_commit(reply: &[u8]) -> Option { + if reply.is_empty() || reply.len() > SBIO_MAX_PAYLOAD { + return None; + } + Some(SbioOp { + opcode: OP_SBIO_MODULE_COMMIT, + payload: pad_payload(reply), + payload_len: reply.len(), + name: c"MODULE_COMMIT", + }) +} + +const OP_SBIO_LOAD_CALIBRATION: u16 = 0x5b; + +pub(crate) const SBIO_CALIBRATION_SOURCE: u32 = 3; + +pub(crate) struct SbioCalibration(KVec); + +impl SbioCalibration { + pub(crate) fn new(blob: &crate::CalibrationBlob) -> Result { + let bytes = blob.bytes(); + let mut body = KVec::new(); + body.extend_from_slice(&SBIO_CALIBRATION_SOURCE.to_le_bytes(), GFP_KERNEL)?; + body.extend_from_slice(bytes, GFP_KERNEL)?; + Ok(SbioCalibration(body)) + } + + pub(crate) fn opcode(&self) -> u16 { + OP_SBIO_LOAD_CALIBRATION + } + + pub(crate) fn payload(&self) -> &[u8] { + &self.0 + } + + pub(crate) fn name(&self) -> &'static CStr { + c"LOAD_CALIBRATION" + } +} + +const OP_SBIO_UPDATE_DEVICE_LIST: u16 = 0x7b; +pub(crate) fn sbio_update_device_list() -> SbioOp { + SbioOp { + opcode: OP_SBIO_UPDATE_DEVICE_LIST, + payload: pad_payload(&[]), + payload_len: 0, + name: c"UPDATE_DEVICE_LIST", + } +} + +pub(crate) const SBIO_MATCH_POLICY_LEN: usize = 2; + +const OP_SBIO_MATCH_POLICY: u16 = 0x1f; +pub(crate) fn sbio_match_policy() -> SbioOp { + SbioOp { + opcode: OP_SBIO_MATCH_POLICY, + payload: pad_payload(&[]), + payload_len: 0, + name: c"MATCH_POLICY", + } +} + +pub(crate) const SBIO_ENROL_RESULT_LEN: usize = 0xc98; + +const ER_STATUS: usize = 0x000; +const ER_ERROR: usize = 0x002; +const ER_PROGRESS: usize = 0x004; +const ER_HAS_TEMPLATE: usize = 0x006; +const ER_COMPLETE: usize = 0xbfe; + +static_assert!(ER_STATUS + 2 <= SBIO_ENROL_RESULT_LEN); +static_assert!(ER_ERROR + 2 <= SBIO_ENROL_RESULT_LEN); +static_assert!(ER_PROGRESS < SBIO_ENROL_RESULT_LEN); +static_assert!(ER_HAS_TEMPLATE + 4 <= SBIO_ENROL_RESULT_LEN); +static_assert!(ER_COMPLETE + 4 <= SBIO_ENROL_RESULT_LEN); +static_assert!(ER_COMPLETE > ER_HAS_TEMPLATE + 4); + +pub(crate) const ER_IDENTITY_LEN: usize = 4 + IDENTITY_UUID_LEN; +static_assert!(ER_IDENTITY_LEN == 20); +static_assert!(ER_IDENTITY_LEN <= SBIO_ENROL_RESULT_LEN); + +pub(crate) fn enrol_identity_candidates( + bytes: &[u8], + user_id: i32, + mut visit: impl FnMut(usize, [u8; IDENTITY_UUID_LEN]), +) { + if bytes.len() < ER_IDENTITY_LEN { + return; + } + let wanted = user_id.to_le_bytes(); + for at in 0..=bytes.len() - ER_IDENTITY_LEN { + if bytes[at..at + 4] != wanted { + continue; + } + let mut uuid = [0u8; IDENTITY_UUID_LEN]; + uuid.copy_from_slice(&bytes[at + 4..at + ER_IDENTITY_LEN]); + if uuid.iter().all(|&b| b == 0) { + continue; + } + visit(at, uuid); + } +} + +pub(crate) struct EnrolmentResult { + progress_raw: u8, + has_template: u32, + complete: u32, +} + +impl EnrolmentResult { + pub(crate) fn parse(bytes: &[u8]) -> Option { + if bytes.len() < SBIO_ENROL_RESULT_LEN { + return None; + } + Some(EnrolmentResult { + progress_raw: bytes[ER_PROGRESS], + has_template: u32::from_le_bytes([ + bytes[ER_HAS_TEMPLATE], + bytes[ER_HAS_TEMPLATE + 1], + bytes[ER_HAS_TEMPLATE + 2], + bytes[ER_HAS_TEMPLATE + 3], + ]), + complete: u32::from_le_bytes([ + bytes[ER_COMPLETE], + bytes[ER_COMPLETE + 1], + bytes[ER_COMPLETE + 2], + bytes[ER_COMPLETE + 3], + ]), + }) + } + + pub(crate) fn progress_percent(&self) -> u32 { + (u32::from(self.progress_raw) * 100 + 127) / 255 + } + + pub(crate) fn has_template(&self) -> bool { + self.has_template != 0 + } + + pub(crate) fn complete(&self) -> bool { + self.complete != 0 + } +} + +#[derive(Clone, Copy)] +pub(crate) struct CatacombUser(i32); + +impl CatacombUser { + pub(crate) const MASTER: CatacombUser = CatacombUser(-1); + pub(crate) const OWNER: CatacombUser = CatacombUser(501); + + pub(crate) fn enrolling(user: UserId) -> CatacombUser { + CatacombUser(user.value()) + } + + pub(crate) const fn value(&self) -> i32 { + self.0 + } +} +static_assert!(CatacombUser::MASTER.value() == -1); +static_assert!(CatacombUser::OWNER.value() == 501); +static_assert!(CatacombUser::MASTER.value() != CatacombUser::OWNER.value()); +static_assert!(CatacombUser::MASTER.value() < 0); + +pub(crate) const SBIO_SAVED_USER_ID_AT: usize = 8; +static_assert!(SBIO_SAVED_USER_ID_AT + 4 <= SBIO_SAVED_MIN); + +pub(crate) const SBIO_SAVE_SELECTOR_LEN: usize = 24; +const SS_USER_ID: usize = 0; +const SS_DEVICE: usize = 4; +const SS_DEVICE_LEN: usize = 20; +static_assert!(SS_DEVICE + SS_DEVICE_LEN == SBIO_SAVE_SELECTOR_LEN); +static_assert!(SBIO_SAVE_SELECTOR_LEN <= SBIO_MAX_PAYLOAD); + +pub(crate) struct SaveSelector([u8; SBIO_SAVE_SELECTOR_LEN]); + +impl SaveSelector { + pub(crate) fn new(user: CatacombUser) -> SaveSelector { + let mut out = [0u8; SBIO_SAVE_SELECTOR_LEN]; + out[SS_USER_ID..SS_USER_ID + 4].copy_from_slice(&user.value().to_le_bytes()); + SaveSelector(out) + } + + pub(crate) fn bytes(&self) -> &[u8; SBIO_SAVE_SELECTOR_LEN] { + &self.0 + } +} + +pub(crate) const SBIO_STATUS_COLD_TRANSITION: u32 = 0x8002; +static_assert!(SBIO_STATUS_COLD_TRANSITION != 0); + +// 0x101 from a 0x6d load = already active; a success answer, not an error +pub(crate) const SBIO_STATUS_ALREADY_ACTIVE: u32 = 0x101; +static_assert!(SBIO_STATUS_ALREADY_ACTIVE != 0); +static_assert!(SBIO_STATUS_ALREADY_ACTIVE != SBIO_STATUS_COLD_TRANSITION); + +pub(crate) const COMPONENT_STATE_COLD: u32 = 0x1; +pub(crate) const COMPONENT_STATE_ACTIVE: u32 = 0x2; +static_assert!(COMPONENT_STATE_COLD != COMPONENT_STATE_ACTIVE); + +pub(crate) const COMPONENT_PAIR_LEN: usize = 8; + +pub(crate) struct ComponentStates<'a>(&'a [u8]); + +impl<'a> ComponentStates<'a> { + pub(crate) fn new(reply: &'a [u8]) -> ComponentStates<'a> { + ComponentStates(reply) + } + + pub(crate) fn count(&self) -> usize { + self.0.len() / COMPONENT_PAIR_LEN + } + + pub(crate) fn trailing(&self) -> usize { + self.0.len() % COMPONENT_PAIR_LEN + } + + pub(crate) fn pair(&self, i: usize) -> Option<(i32, u32)> { + let at = i.checked_mul(COMPONENT_PAIR_LEN)?; + let bytes = self.0.get(at..at + COMPONENT_PAIR_LEN)?; + Some(( + i32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]), + u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]), + )) + } + + pub(crate) fn state_for(&self, id: i32) -> Option { + (0..self.count()) + .filter_map(|i| self.pair(i)) + .find(|(pair_id, _)| *pair_id == id) + .map(|(_, state)| state) + } +} + +pub(crate) enum ComponentAction { + AlreadyActive, + Load, + Unsupported(u32), +} + +pub(crate) fn component_action(state: u32) -> ComponentAction { + if state & COMPONENT_STATE_ACTIVE != 0 { + ComponentAction::AlreadyActive + } else if state & COMPONENT_STATE_COLD != 0 { + ComponentAction::Load + } else { + ComponentAction::Unsupported(state) + } +} + +const OP_SBIO_SAVE_LOCKOUT: u16 = 0x70; +pub(crate) fn sbio_save_lockout() -> SbioOp { + SbioOp { + opcode: OP_SBIO_SAVE_LOCKOUT, + payload: pad_payload(&[]), + payload_len: 0, + name: c"SAVE_LOCKOUT", + } +} + +pub(crate) struct SbioLoadLockout<'a>(&'a [u8]); + +impl<'a> SbioLoadLockout<'a> { + pub(crate) fn new(blob: &'a [u8]) -> Option> { + if blob.is_empty() || blob.len() > SBIO_SAVED_MAX { + return None; + } + Some(SbioLoadLockout(blob)) + } + + pub(crate) fn opcode(&self) -> u16 { + OP_SBIO_LOAD_LOCKOUT + } + + pub(crate) fn payload(&self) -> &[u8] { + self.0 + } + + pub(crate) fn name(&self) -> &'static CStr { + c"LOAD_LOCKOUT" + } +} +const OP_SBIO_LOAD_LOCKOUT: u16 = 0x71; + +const OP_SBIO_SAVE_CATACOMB: u16 = 0x6c; +pub(crate) fn sbio_save_catacomb(selector: &SaveSelector) -> SbioOp { + SbioOp { + opcode: OP_SBIO_SAVE_CATACOMB, + payload: pad_payload(selector.bytes()), + payload_len: SBIO_SAVE_SELECTOR_LEN, + name: c"SAVE_CATACOMB", + } +} + +const OP_SBIO_CONFIRM_SAVE: u16 = 0x37; +pub(crate) fn sbio_confirm_save(selector: &SaveSelector) -> SbioOp { + SbioOp { + opcode: OP_SBIO_CONFIRM_SAVE, + payload: pad_payload(selector.bytes()), + payload_len: SBIO_SAVE_SELECTOR_LEN, + name: c"CONFIRM_SAVE", + } +} + +pub(crate) const SBIO_SAVED_MIN: usize = 0x22; +pub(crate) const SBIO_SAVED_MAX: usize = 0x4b000; +static_assert!(SBIO_SAVED_MIN < SBIO_SAVED_MAX); +static_assert!(SBIO_SAVED_MAX as u32 <= transfer::MAX_TRANSACTION); + +pub(crate) struct SbioLoadCatacomb<'a>(&'a [u8]); + +impl<'a> SbioLoadCatacomb<'a> { + pub(crate) fn new(blob: &'a [u8]) -> Option> { + if blob.len() < SBIO_SAVED_MIN || blob.len() > SBIO_SAVED_MAX { + return None; + } + Some(SbioLoadCatacomb(blob)) + } + + pub(crate) fn opcode(&self) -> u16 { + OP_SBIO_LOAD_CATACOMB + } + + pub(crate) fn payload(&self) -> &[u8] { + self.0 + } + + pub(crate) fn name(&self) -> &'static CStr { + c"LOAD_CATACOMB" + } +} +const OP_SBIO_LOAD_CATACOMB: u16 = 0x6d; + +pub(crate) const IDENTITY_RECORD_LEN: usize = 40; +const IR_SELECTOR: usize = 20; +const IR_SELECTOR_BUILTIN: u32 = 1; +static_assert!(IR_SELECTOR == IDENTITY_V1_LEN); +static_assert!(IR_SELECTOR + 4 <= IDENTITY_RECORD_LEN); + +const OP_SBIO_LIST_IDENTITIES: u16 = 0x7a; +pub(crate) fn sbio_list_identities() -> SbioOp { + SbioOp { + opcode: OP_SBIO_LIST_IDENTITIES, + payload: pad_payload(&[]), + payload_len: 0, + name: c"LIST_IDENTITIES", + } +} + +const OP_SBIO_GROUP_STATE: u16 = 0x79; + +const OP_SBIO_LIST_IDENTITIES_SCOPED: u16 = 0x6e; + +pub(crate) struct IdentityRecords<'a>(&'a [u8]); + +impl<'a> IdentityRecords<'a> { + pub(crate) fn new(reply: &'a [u8]) -> Option> { + if reply.len() % IDENTITY_RECORD_LEN != 0 { + return None; + } + Some(IdentityRecords(reply)) + } + + pub(crate) fn count(&self) -> usize { + self.0.len() / IDENTITY_RECORD_LEN + } + + pub(crate) fn record(&self, i: usize) -> Option<(IdentityV1, bool)> { + let at = i.checked_mul(IDENTITY_RECORD_LEN)?; + let bytes = self.0.get(at..at + IDENTITY_RECORD_LEN)?; + let user_id = i32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + let mut uuid = [0u8; IDENTITY_UUID_LEN]; + uuid.copy_from_slice(&bytes[4..IDENTITY_V1_LEN]); + let selector = u32::from_le_bytes([ + bytes[IR_SELECTOR], + bytes[IR_SELECTOR + 1], + bytes[IR_SELECTOR + 2], + bytes[IR_SELECTOR + 3], + ]); + Some(( + IdentityV1 { user_id, uuid }, + selector == IR_SELECTOR_BUILTIN, + )) + } + + pub(crate) fn lists_uuid(&self, uuid: &[u8; IDENTITY_UUID_LEN]) -> bool { + (0..self.count()) + .filter_map(|i| self.record(i)) + .any(|(identity, _)| identity.uuid == *uuid) + } + + pub(crate) fn each_built_in_for(&self, user_id: i32, mut visit: impl FnMut(IdentityV1)) { + for i in 0..self.count() { + if let Some((identity, built_in)) = self.record(i) { + if built_in && identity.user_id == user_id { + visit(identity); + } + } + } + } +} + +const FIC_DEVICE_BUILTIN: u32 = 1; +pub(crate) const SBIO_FREE_COUNT_LEN: usize = 4 + 20; +static_assert!(SBIO_FREE_COUNT_LEN == SBIO_SAVE_SELECTOR_LEN); +pub(crate) const SBIO_FREE_COUNT_REPLY_LEN: usize = 4; +pub(crate) const SBIO_FREE_COUNT_MAX: u32 = 3; + +const OP_SBIO_FREE_IDENTITY_COUNT: u16 = 0x38; +pub(crate) fn sbio_free_identity_count(user: UserId) -> SbioOp { + let mut body = [0u8; SBIO_FREE_COUNT_LEN]; + body[0..4].copy_from_slice(&user.value().to_le_bytes()); + body[4..8].copy_from_slice(&FIC_DEVICE_BUILTIN.to_le_bytes()); + SbioOp { + opcode: OP_SBIO_FREE_IDENTITY_COUNT, + payload: pad_payload(&body), + payload_len: SBIO_FREE_COUNT_LEN, + name: c"FREE_IDENTITY_COUNT", + } +} + +const OP_SBIO_DELETE_IDENTITY: u16 = 0x57; +pub(crate) fn sbio_delete_identity(identity: &IdentityV1) -> SbioOp { + SbioOp { + opcode: OP_SBIO_DELETE_IDENTITY, + payload: pad_payload(&identity.wire()), + payload_len: IDENTITY_V1_LEN, + name: c"DELETE_IDENTITY", + } +} + +const OP_SBIO_MATCH_RESULT: u16 = 0x09; +pub(crate) fn sbio_match_result() -> SbioOp { + SbioOp { + opcode: OP_SBIO_MATCH_RESULT, + payload: pad_payload(&[]), + payload_len: 0, + name: c"MATCH_RESULT", + } +} + +pub(crate) const SBIO_MATCH_RESULT_LEN: usize = 0xca2; + +const MR_USER_ID: usize = 0x000; +const MR_IDENTITY: usize = 0x004; +const MR_CANDIDATES: usize = 0x014; +const MR_FLAGS: usize = 0xc8a; +const MR_SECOND_IDENTITY_UNUSED: usize = 0xc8e; +static_assert!(MR_IDENTITY != MR_SECOND_IDENTITY_UNUSED); +static_assert!(MR_USER_ID + 4 <= SBIO_MATCH_RESULT_LEN); +static_assert!(MR_IDENTITY + IDENTITY_UUID_LEN <= SBIO_MATCH_RESULT_LEN); +static_assert!(MR_CANDIDATES + 4 <= SBIO_MATCH_RESULT_LEN); +static_assert!(MR_FLAGS + 4 <= SBIO_MATCH_RESULT_LEN); +static_assert!(MR_SECOND_IDENTITY_UNUSED + IDENTITY_UUID_LEN <= SBIO_MATCH_RESULT_LEN); +static_assert!(MR_IDENTITY == MR_USER_ID + 4); +static_assert!(MR_IDENTITY + IDENTITY_UUID_LEN == MR_CANDIDATES); +static_assert!(MR_FLAGS > MR_USER_ID + 4); +static_assert!(SBIO_MATCH_RESULT_LEN != SBIO_ENROL_RESULT_LEN); + +pub(crate) const IDENTITY_UUID_LEN: usize = 16; +pub(crate) const IDENTITY_V1_LEN: usize = 4 + IDENTITY_UUID_LEN; +static_assert!(IDENTITY_V1_LEN == 0x14); + +#[derive(Clone, Copy)] +pub(crate) struct IdentityV1 { + user_id: i32, + uuid: [u8; IDENTITY_UUID_LEN], +} + +impl IdentityV1 { + pub(crate) fn from_index(user_id: i32, uuid: [u8; IDENTITY_UUID_LEN]) -> IdentityV1 { + IdentityV1 { user_id, uuid } + } + + pub(crate) fn uuid(&self) -> &[u8; IDENTITY_UUID_LEN] { + &self.uuid + } + + pub(crate) fn wire(&self) -> [u8; IDENTITY_V1_LEN] { + let mut out = [0u8; IDENTITY_V1_LEN]; + out[0..4].copy_from_slice(&self.user_id.to_le_bytes()); + out[4..IDENTITY_V1_LEN].copy_from_slice(&self.uuid); + out + } +} + +pub(crate) struct MatchResult { + user_id: i32, + identity: [u8; IDENTITY_UUID_LEN], +} + +impl MatchResult { + pub(crate) fn parse(bytes: &[u8]) -> Option { + if bytes.len() < SBIO_MATCH_RESULT_LEN { + return None; + } + let mut identity = [0u8; IDENTITY_UUID_LEN]; + identity.copy_from_slice(&bytes[MR_IDENTITY..MR_IDENTITY + IDENTITY_UUID_LEN]); + Some(MatchResult { + user_id: i32::from_le_bytes([ + bytes[MR_USER_ID], + bytes[MR_USER_ID + 1], + bytes[MR_USER_ID + 2], + bytes[MR_USER_ID + 3], + ]), + identity, + }) + } + + pub(crate) fn identity_uuid(&self) -> &[u8; IDENTITY_UUID_LEN] { + &self.identity + } + + pub(crate) fn matches(&self, user: UserId) -> bool { + self.user_id == user.value() + } + +} + +const OP_SBIO_IMAGE_CLEANUP: u16 = 0x22; +pub(crate) fn sbio_image_cleanup() -> SbioOp { + SbioOp { + opcode: OP_SBIO_IMAGE_CLEANUP, + payload: pad_payload(&[]), + payload_len: 0, + name: c"IMAGE_CLEANUP", + } +} + +const OP_SBIO_CANCEL: u16 = 0x05; +pub(crate) fn sbio_cancel_operation() -> SbioOp { + SbioOp { + opcode: OP_SBIO_CANCEL, + payload: pad_payload(&[]), + payload_len: 0, + name: c"CANCEL_OPERATION", + } +} + +const OP_SBIO_CLEAR_STATE: u16 = 0x19; +pub(crate) fn sbio_clear_state() -> SbioOp { + SbioOp { + opcode: OP_SBIO_CLEAR_STATE, + payload: pad_payload(&[]), + payload_len: 0, + name: c"CLEAR_STATE", + } +} + +const OP_SBIO_REGISTER_SERIAL: u16 = 0x17; +pub(crate) fn sbio_register_sensor_serial(serial: &crate::sensor::SensorSerial) -> SbioOp { + SbioOp { + opcode: OP_SBIO_REGISTER_SERIAL, + payload: pad_payload(serial.bytes()), + payload_len: crate::sensor::SENSOR_SERIAL_LEN, + name: c"REGISTER_SERIAL", + } +} + +const OP_SBIO_ENUMERATE: u16 = 0x7c; +pub(crate) const fn sbio_enumerate() -> SbioOp { + SbioOp { + opcode: OP_SBIO_ENUMERATE, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"ENUMERATE", + } +} + +const OP_SBIO_DIAGNOSTICS: u16 = 0x63; +pub(crate) const fn sbio_diagnostics() -> SbioOp { + SbioOp { + opcode: OP_SBIO_DIAGNOSTICS, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"DIAGNOSTICS", + } +} + +const SURVEY_ENUMERATE: SbioOp = sbio_enumerate(); +const SURVEY_DIAGNOSTICS: SbioOp = sbio_diagnostics(); + +static_assert!(SURVEY_ENUMERATE.payload_len == 0); +static_assert!(SURVEY_DIAGNOSTICS.payload_len == 0); + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum ImagePurpose { + Enrolment, + Matching, +} + +impl ImagePurpose { + const fn offset(self) -> usize { + match self { + ImagePurpose::Matching => 0x12, + ImagePurpose::Enrolment => 0x13, + } + } + + const fn last_image_offset(self) -> usize { + match self { + ImagePurpose::Enrolment => 0x08, + ImagePurpose::Matching => 0x09, + } + } + +} + +static_assert!(ImagePurpose::Enrolment.offset() != ImagePurpose::Matching.offset()); +static_assert!(ImagePurpose::Matching.offset() == 0x12); +static_assert!(ImagePurpose::Enrolment.offset() == 0x13); +static_assert!(ImagePurpose::Enrolment.last_image_offset() == 0x08); +static_assert!(ImagePurpose::Matching.last_image_offset() == 0x09); +static_assert!( + ImagePurpose::Enrolment.last_image_offset() != ImagePurpose::Matching.last_image_offset() +); +static_assert!(ImagePurpose::Matching.last_image_offset() < ImagePurpose::Matching.offset()); + +#[derive(Clone, Copy)] +pub(crate) struct UserId(i32); + +impl UserId { + pub(crate) fn new(value: i32) -> Option { + if value > 0 { + Some(UserId(value)) + } else { + None + } + } + + pub(crate) const fn value(&self) -> i32 { + self.0 + } +} + +const IPI_FIRST_IMAGE: usize = 0x07; +const IPI_TIMESTAMP: usize = 0x40; +const IPI_CAPTURE_COUNTER: usize = 0x48; +const IPI_DEVICE_KIND: usize = 0x4f; +const IPI_USER_ID: usize = 0x67; + +const IPI_DEVICE_BUILTIN: u32 = 1; + +static_assert!(IPI_USER_ID + 4 <= SBIO_MAX_PAYLOAD); +static_assert!(IPI_DEVICE_KIND + 4 <= SBIO_MAX_PAYLOAD); +static_assert!(IPI_TIMESTAMP + 8 <= SBIO_MAX_PAYLOAD); +static_assert!(IPI_CAPTURE_COUNTER + 4 <= SBIO_MAX_PAYLOAD); +static_assert!(IPI_FIRST_IMAGE < ImagePurpose::Matching.offset()); +static_assert!(ImagePurpose::Enrolment.offset() < IPI_TIMESTAMP); + +const OP_SBIO_PREPARE_IMAGE: u16 = 0x23; +pub(crate) fn sbio_prepare_image_processing() -> SbioOp { + SbioOp { + opcode: OP_SBIO_PREPARE_IMAGE, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"PREPARE_IMAGE_PROCESSING", + } +} + +const OP_SBIO_IMAGE_INIT: u16 = 0x06; +pub(crate) fn sbio_image_processing_init( + purpose: ImagePurpose, + first_image: bool, + last_image: bool, + capture_counter: u32, + user: UserId, + monotonic_ns: u64, +) -> SbioOp { + let mut payload = [0u8; SBIO_MAX_PAYLOAD]; + + if first_image { + payload[IPI_FIRST_IMAGE] = 1; + } + if last_image { + payload[purpose.last_image_offset()] = 1; + } + payload[purpose.offset()] = 1; + + // ns*24/1000 split to dodge u64 overflow and the kernel's absent __udivti3 + let scaled = (monotonic_ns / 1000) * 24 + ((monotonic_ns % 1000) * 24) / 1000; + payload[IPI_TIMESTAMP..IPI_TIMESTAMP + 8].copy_from_slice(&scaled.to_le_bytes()); + payload[IPI_CAPTURE_COUNTER..IPI_CAPTURE_COUNTER + 4] + .copy_from_slice(&capture_counter.to_le_bytes()); + payload[IPI_DEVICE_KIND..IPI_DEVICE_KIND + 4] + .copy_from_slice(&IPI_DEVICE_BUILTIN.to_le_bytes()); + payload[IPI_USER_ID..IPI_USER_ID + 4].copy_from_slice(&user.value().to_le_bytes()); + + SbioOp { + opcode: OP_SBIO_IMAGE_INIT, + payload, + payload_len: SBIO_IMAGE_INIT_LEN, + name: c"IMAGE_PROCESSING_INIT", + } +} + +const OP_SBIO_RELAY_CAPTURE: u16 = 0x65; + +pub(crate) struct SbioRelay<'a> { + capture: &'a crate::sensor::Capture, +} + +impl<'a> SbioRelay<'a> { + pub(crate) fn new(capture: &'a crate::sensor::Capture) -> SbioRelay<'a> { + SbioRelay { capture } + } + + pub(crate) fn opcode(&self) -> u16 { + OP_SBIO_RELAY_CAPTURE + } + + pub(crate) fn payload(&self) -> &[u8] { + self.capture.bytes() + } + + pub(crate) fn name(&self) -> &'static CStr { + c"RELAY_CAPTURE" + } +} + +const OP_SBIO_ASSESSMENT: u16 = 0x07; +pub(crate) fn sbio_image_assessment() -> SbioOp { + SbioOp { + opcode: OP_SBIO_ASSESSMENT, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"IMAGE_ASSESSMENT", + } +} + +const OP_SBIO_ENROL_RESULT: u16 = 0x08; +pub(crate) fn sbio_enrolment_result() -> SbioOp { + SbioOp { + opcode: OP_SBIO_ENROL_RESULT, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"ENROLMENT_RESULT", + } +} + +pub(crate) const ASSESS_MIN_LEN: usize = 0x89; +pub(crate) const ASSESS_ERROR: usize = 0x00; +pub(crate) const ASSESS_USABLE_MATCH: usize = 0x06; +pub(crate) const ASSESS_USABLE_ENROL: usize = 0x07; +pub(crate) const ASSESS_FEEDBACK: usize = 0x0e; +pub(crate) const ASSESS_DIRTY: usize = 0x51; + +static_assert!(ASSESS_ERROR + 2 <= ASSESS_MIN_LEN); +static_assert!(ASSESS_USABLE_MATCH < ASSESS_USABLE_ENROL); +static_assert!(ASSESS_FEEDBACK + 4 <= ASSESS_MIN_LEN); +static_assert!(ASSESS_DIRTY < ASSESS_MIN_LEN); + +pub(crate) const SBIO_SESSION_SHARE_LEN: usize = 40; + +pub(crate) const SBIO_STATUS_OK: u16 = 0x00; +pub(crate) const SBIO_STATUS_PREREQUISITE: u16 = 0x01; +pub(crate) const SBIO_STATUS_16: u16 = 0x16; +static_assert!(SBIO_STATUS_PREREQUISITE != SBIO_STATUS_16); +pub(crate) const SBIO_SESSION_MODE: u32 = 1; +static_assert!(SBIO_SESSION_SHARE_LEN <= SBIO_MAX_PAYLOAD); + +const OP_SBIO_SESSION_SHARE: u16 = 0x15; +pub(crate) fn sbio_request_session_share() -> SbioOp { + SbioOp { + opcode: OP_SBIO_SESSION_SHARE, + payload: pad_payload(&SBIO_SESSION_MODE.to_le_bytes()), + payload_len: 4, + name: c"SESSION_SHARE", + } +} + +const OP_SBIO_COMMIT_SESSION: u16 = 0x16; +pub(crate) fn sbio_commit_session_share(share: &[u8; SBIO_SESSION_SHARE_LEN]) -> SbioOp { + SbioOp { + opcode: OP_SBIO_COMMIT_SESSION, + payload: pad_payload(share), + payload_len: SBIO_SESSION_SHARE_LEN, + name: c"SESSION_COMMIT", + } +} +pub(crate) const SBIO_CHALLENGE_LEN: usize = 64; +static_assert!(SBIO_CHALLENGE_LEN <= SBIO_MAX_PAYLOAD); + +const OP_SBIO_CHALLENGE: u16 = 0x42; +pub(crate) fn sbio_request_challenge() -> SbioOp { + SbioOp { + opcode: OP_SBIO_CHALLENGE, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"SEQUENCE_CHALLENGE", + } +} + +const OP_SBIO_COMMIT_CHALLENGE: u16 = 0x18; +pub(crate) fn sbio_commit_challenge(reply: &[u8; SBIO_CHALLENGE_LEN]) -> SbioOp { + SbioOp { + opcode: OP_SBIO_COMMIT_CHALLENGE, + payload: pad_payload(reply), + payload_len: SBIO_CHALLENGE_LEN, + name: c"SEQUENCE_COMMIT", + } +} +const OP_SBIO_FETCH_PATCH: u16 = 0x5f; + +const SBIO_PATCH_SUBTYPE: u16 = 2; + +pub(crate) fn sbio_fetch_patch() -> SbioOp { + SbioOp { + opcode: OP_SBIO_FETCH_PATCH, + payload: pad_payload(&SBIO_PATCH_SUBTYPE.to_le_bytes()), + payload_len: 2, + name: c"FETCH_SENSOR_PATCH", + } +} + +const fn encode_sbio(opcode: u16, marker: u8, seq: u16) -> Message { + Message { + msg0: (EP_SBIO as u64) + | ((marker as u64) << MSG_TAG_SHIFT) + | ((opcode as u64) << MSG_TYPE_SHIFT) + | ((seq as u64) << 48), + msg1: 0, + } +} + +static_assert!(encode_sbio(OP_SBIO_INIT_COMMS, transfer::MARKER_FIRST, 0).msg0 == 0x0000_0073_fc08); + +pub(crate) fn encode_sbio_raw(opcode: u16, seq: u16) -> Option { + Some(encode_sbio(opcode, transfer::MARKER_FIRST, seq)) +} + +pub(crate) fn encode_sbio_next(opcode: u16, seq: u16) -> Option { + Some(encode_sbio(opcode, transfer::MARKER_NEXT, seq)) +} + +pub(crate) fn encode_sbio_continue(cont: &transfer::Continuation) -> Message { + encode_sbio(cont.opcode() as u16, transfer::MARKER_REQUEST, cont.seq()) +} diff --git a/drivers/soc/apple/scrd.rs b/drivers/soc/apple/scrd.rs new file mode 100644 index 0000000000000..0326308651ba4 --- /dev/null +++ b/drivers/soc/apple/scrd.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! SEP credential endpoint (SCRD, EP 0x0a) wire protocol. + +#![allow(dead_code)] + +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; +use crate::proto::*; +use crate::sks::SKS_AUTH_TOKEN_LEN; + +pub(crate) const SCRD_ACM_HANDLE_LEN: usize = SKS_AUTH_TOKEN_LEN; + +pub(crate) const SCRD_REQUEST_COMMAND: u8 = 1; + +const SCRD_MAGIC: [u8; 4] = *b"DRCS"; + +const SCRD_CMD_INITIALIZE: u8 = 0x0a; +const SCRD_CMD_CONTEXT_CREATE_TRACKED: u8 = 0x24; +const SCRD_CMD_CONTEXT_EXTERNALIZE: u8 = 0x13; +const SCRD_CMD_VERIFY_POLICY: u8 = 0x03; + +const SCRD_INIT_LOG_LEVEL: u8 = 0x28; + +const SCRD_POLICY_TOUCHID_ENROLLMENT: &[u8] = b"TouchIdEnrollment"; + +pub(crate) const SCRD_CONTEXT_CREATE_REPLY_LEN: usize = SCRD_ACM_HANDLE_LEN + 1 + 4; + +const SCRD_MAX_LOGICAL: usize = 64; +static_assert!( + 8 + SCRD_ACM_HANDLE_LEN + SCRD_POLICY_TOUCHID_ENROLLMENT.len() + 1 + 1 + 8 <= SCRD_MAX_LOGICAL +); + +pub(crate) struct ScrdCommand { + request: u8, + payload: [u8; SCRD_MAX_LOGICAL], + payload_len: usize, + name: &'static CStr, +} + +impl ScrdCommand { + pub(crate) fn request(&self) -> u8 { + self.request + } + pub(crate) fn payload(&self) -> &[u8] { + &self.payload[..self.payload_len] + } + pub(crate) fn name(&self) -> &'static CStr { + self.name + } +} + +fn scrd_header(out: &mut [u8; SCRD_MAX_LOGICAL], command: u8, byte5: u8, version: u8) { + out[0..4].copy_from_slice(&SCRD_MAGIC); + out[4] = command; + out[5] = byte5; + out[6] = 0; + out[7] = version; +} + +pub(crate) fn scrd_initialize() -> ScrdCommand { + let mut payload = [0u8; SCRD_MAX_LOGICAL]; + scrd_header(&mut payload, SCRD_CMD_INITIALIZE, SCRD_INIT_LOG_LEVEL, 0); + ScrdCommand { + request: SCRD_REQUEST_COMMAND, + payload, + payload_len: 8, + name: c"SCRD_INITIALIZE", + } +} + +pub(crate) fn scrd_context_create_tracked(session_uid: i32) -> ScrdCommand { + let mut payload = [0u8; SCRD_MAX_LOGICAL]; + scrd_header(&mut payload, SCRD_CMD_CONTEXT_CREATE_TRACKED, 0, 1); + payload[8..12].copy_from_slice(&session_uid.to_le_bytes()); + ScrdCommand { + request: SCRD_REQUEST_COMMAND, + payload, + payload_len: 12, + name: c"SCRD_CONTEXT_CREATE", + } +} + +pub(crate) fn scrd_context_externalize(handle: &[u8; SCRD_ACM_HANDLE_LEN]) -> ScrdCommand { + let mut payload = [0u8; SCRD_MAX_LOGICAL]; + scrd_header(&mut payload, SCRD_CMD_CONTEXT_EXTERNALIZE, 0, 1); + payload[8..8 + SCRD_ACM_HANDLE_LEN].copy_from_slice(handle); + ScrdCommand { + request: SCRD_REQUEST_COMMAND, + payload, + payload_len: 8 + SCRD_ACM_HANDLE_LEN, + name: c"SCRD_EXTERNALIZE", + } +} + +pub(crate) fn scrd_verify_touchid_enrollment(handle: &[u8; SCRD_ACM_HANDLE_LEN]) -> ScrdCommand { + let mut payload = [0u8; SCRD_MAX_LOGICAL]; + scrd_header(&mut payload, SCRD_CMD_VERIFY_POLICY, 0, 1); + let mut n = 8; + payload[n..n + SCRD_ACM_HANDLE_LEN].copy_from_slice(handle); + n += SCRD_ACM_HANDLE_LEN; + payload[n..n + SCRD_POLICY_TOUCHID_ENROLLMENT.len()] + .copy_from_slice(SCRD_POLICY_TOUCHID_ENROLLMENT); + n += SCRD_POLICY_TOUCHID_ENROLLMENT.len(); + // then NUL, preflight(1), u32 flags, u32 param count -- all zero + n += 1 + 1 + 4 + 4; + ScrdCommand { + request: SCRD_REQUEST_COMMAND, + payload, + payload_len: n, + name: c"SCRD_VERIFY_POLICY", + } +} + +pub(crate) struct ScrdReply { + pub(crate) request: u8, + pub(crate) response_size: u16, + pub(crate) status: i32, +} + +pub(crate) fn decode_scrd_reply(msg: &Message) -> ScrdReply { + let b = msg.msg0.to_le_bytes(); + ScrdReply { + request: b[1], + response_size: u16::from_le_bytes([b[2], b[3]]), + status: i32::from_le_bytes([b[4], b[5], b[6], b[7]]), + } +} + +pub(crate) fn encode_scrd(request: u8, len: usize) -> Message { + let len = (len & 0xffff) as u16; + let len = len.to_le_bytes(); + Message { + msg0: u64::from_le_bytes([EP_SCRD, request, len[0], len[1], 0, 0, 0, 0]), + msg1: 0, + } +} + +static_assert!(EP_SCRD == 0x0a); +static_assert!(SCRD_CMD_INITIALIZE == 0x0a); +static_assert!(SCRD_CMD_CONTEXT_CREATE_TRACKED == 0x24); +static_assert!(SCRD_CMD_CONTEXT_EXTERNALIZE == 0x13); +static_assert!(SCRD_CMD_VERIFY_POLICY == 0x03); diff --git a/drivers/soc/apple/seed.rs b/drivers/soc/apple/seed.rs new file mode 100644 index 0000000000000..d1793d8ebe361 --- /dev/null +++ b/drivers/soc/apple/seed.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Importing the enclave's existing records. + +use crate::shim; +use crate::store::{Key, Store}; +use kernel::prelude::*; + +pub(crate) const SEED_PATH: &CStr = c"/var/lib/apple-sep-seed.bin"; + +const MAGIC: [u8; 4] = *b"AXRT"; +const HEADER_LEN: usize = 8; +const RECORD_HEADER_LEN: usize = 1 + 16 + 4; + +const MIN_RECORD_LEN: u32 = 1; +const MAX_RECORD_LEN: u32 = 0x8000; + +const MAX_SEED_BYTES: u64 = 1 << 20; + +const MAX_RECORDS: u32 = 64; + +const TYPE_ROOT_LOW: u8 = 1; +const TYPE_ROOT_HIGH: u8 = 2; +const TYPE_SESSION_HIGH: u8 = 4; + +pub(crate) struct Imported { + pub(crate) records: usize, + pub(crate) roots: usize, + pub(crate) sessions: usize, + pub(crate) bytes: usize, +} + +fn le32(buf: &[u8], off: usize) -> u32 { + u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) +} + +fn read_seed() -> Result> { + let file = shim::StoreFile::open_readonly(SEED_PATH)?; + let size = file.size()?; + if size < HEADER_LEN as u64 || size > MAX_SEED_BYTES { + return Err(EINVAL); + } + let mut buf = KVec::with_capacity(size as usize, GFP_KERNEL)?; + buf.resize(size as usize, 0, GFP_KERNEL)?; + file.read_exact(0, &mut buf)?; + Ok(buf) +} + +pub(crate) fn import(store: &mut Store) -> Result { + let buf = read_seed()?; + + if buf[..4] != MAGIC { + return Err(EINVAL); + } + let count = le32(&buf, 4); + if count == 0 || count > MAX_RECORDS { + return Err(EINVAL); + } + + let mut offsets: KVec<(u8, [u8; 16], usize, usize)> = KVec::new(); + let mut pos = HEADER_LEN; + for _ in 0..count { + if pos + RECORD_HEADER_LEN > buf.len() { + return Err(EINVAL); + } + let kind = buf[pos]; + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&buf[pos + 1..pos + 17]); + let len = le32(&buf, pos + 17); + pos += RECORD_HEADER_LEN; + + if !(TYPE_ROOT_LOW..=TYPE_SESSION_HIGH).contains(&kind) { + return Err(EINVAL); + } + if !(MIN_RECORD_LEN..=MAX_RECORD_LEN).contains(&len) { + return Err(EINVAL); + } + // Root records carry an all-zero UUID; anything else would be unreachable. + let is_root = kind == TYPE_ROOT_LOW || kind == TYPE_ROOT_HIGH; + if is_root && uuid != [0u8; 16] { + return Err(EINVAL); + } + let len = len as usize; + if pos + len > buf.len() { + return Err(EINVAL); + } + offsets.push((kind, uuid, pos, len), GFP_KERNEL)?; + pos += len; + } + + // Writes are keyed, so re-running after a crash simply replaces. + let mut result = Imported { + records: 0, + roots: 0, + sessions: 0, + bytes: 0, + }; + for (kind, uuid, at, len) in offsets { + store.write(&Key::new(kind, uuid), &buf[at..at + len])?; + result.records += 1; + result.bytes += len; + if kind == TYPE_ROOT_LOW || kind == TYPE_ROOT_HIGH { + result.roots += 1; + } else { + result.sessions += 1; + } + } + + // Mark consumed last: a crash above leaves the flag clear and the import reruns. + store.mark_seeded()?; + Ok(result) +} diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs new file mode 100644 index 0000000000000..44eeb912868c7 --- /dev/null +++ b/drivers/soc/apple/sensor.rs @@ -0,0 +1,705 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! The fingerprint sensor over SPI. + +use kernel::prelude::*; +use kernel::time::delay::fsleep; +use kernel::time::Delta; + +extern "C" { + fn sep_sensor_register() -> c_int; + fn sep_sensor_unregister(); + fn sep_sensor_bound() -> c_int; + fn sep_sensor_power_line() -> c_int; + fn sep_sensor_cs_timing_mode() -> c_int; + fn sep_sensor_power_cycle() -> c_int; + fn sep_sensor_power_source() -> c_int; + fn sep_sensor_power(on: c_int) -> c_int; + fn sep_sensor_xfer(tx: *const c_void, rx: *mut c_void, len: usize) -> c_int; + fn sep_sensor_xfer_tx(tx: *const c_void, len: usize) -> c_int; + fn sep_sensor_xfer2( + tx: *const c_void, + tx_len: usize, + rx: *mut c_void, + rx_len: usize, + ) -> c_int; +} + +// Spi2. +pub(crate) const CONTROLLER_BASE: u64 = 0x3_9b10_8000; +pub(crate) const CHIP_SELECT: u32 = 0; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum CsTiming { + Software, + HookBypassed, + Hardware, +} + +impl CsTiming { + fn from_wire(v: c_int) -> CsTiming { + match v { + 2 => CsTiming::Hardware, + 1 => CsTiming::HookBypassed, + _ => CsTiming::Software, + } + } + + pub(crate) fn name(self) -> &'static CStr { + match self { + CsTiming::Hardware => c"HARDWARE — programmed into the controller, which is what this sensor needs", + CsTiming::HookBypassed => { + c"emulated — the controller has a set_cs_timing hook but a GPIO chip select bypasses it" + } + CsTiming::Software => c"emulated in software — no set_cs_timing hook, and the sensor does not accept this", + } + } + + pub(crate) fn is_hardware(self) -> bool { + matches!(self, CsTiming::Hardware) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum PowerSource { + None, + NodeProperty, + ChipLine, +} + +impl PowerSource { + fn from_wire(v: c_int) -> PowerSource { + match v { + 1 => PowerSource::NodeProperty, + 2 => PowerSource::ChipLine, + _ => PowerSource::None, + } + } + + pub(crate) fn name(self) -> &'static CStr { + match self { + PowerSource::None => c"none — no power line was taken", + PowerSource::NodeProperty => c"the sensor node's own gpios property", + PowerSource::ChipLine => c"/soc/pinctrl@39b028000 line 122, taken by device-tree node", + } + } +} + +pub(crate) const POWER_ON_READ_DELAYS_MS: [u32; 4] = [0, 3, 10, 50]; + +pub(crate) const STATUS_IDENTIFIER: usize = 12; +pub(crate) const EXPECTED_IDENTIFIER: u16 = 0x3352; +static_assert!(STATUS_IDENTIFIER + 2 <= STATUS_LEN); + +const CMD_LEN: usize = 7; +const CMD_GET_STATUS: [u8; CMD_LEN] = [0x80, 0x10, 0x00, 0x07, 0x00, 0x00, 0x00]; +const CMD_IDLE: [u8; CMD_LEN] = [0x80, 0x30, 0x00, 0x07, 0x00, 0x00, 0x00]; +const CMD_START_CAPTURE: [u8; CMD_LEN] = [0x80, 0x40, 0x00, 0x07, 0x00, 0x00, 0x00]; +// Precedes the patch blob. +const CMD_SETUP_PATCH_ENABLE: [u8; CMD_LEN] = [0x80, 0x60, 0x00, 0x07, 0x00, 0x00, 0x00]; +const CMD_GET_SENSOR_SERIAL: [u8; CMD_LEN] = [0x80, 0x70, 0x00, 0x07, 0x00, 0x00, 0x00]; + +const READ_CMD_LEN: usize = 11; +const CMD_READ_PREFIX: [u8; CMD_LEN] = [0x80, 0x13, 0x00, 0x0b, 0x00, 0x00, 0x00]; + +const STATUS_XFER_LEN: usize = 23; +const STATUS_AT: usize = 7; +pub(crate) const STATUS_LEN: usize = 16; +static_assert!(STATUS_AT + STATUS_LEN == STATUS_XFER_LEN); + +// Offsets index the 16-byte status, not the 23-byte transfer. +const STATUS_STATE: usize = STATUS_AT + 7; +const STATUS_COUNT: usize = STATUS_AT + 12; +const STATUS_STATE_TRANSFER_REL: usize = 7; +const STATUS_COUNT_TRANSFER_REL: usize = 12; + +static_assert!(STATUS_STATE == 14); +static_assert!(STATUS_COUNT == 19); +static_assert!(STATUS_COUNT + 4 == STATUS_XFER_LEN); +static_assert!(STATUS_STATE != STATUS_STATE_TRANSFER_REL); +static_assert!(STATUS_COUNT != STATUS_COUNT_TRANSFER_REL); +static_assert!(STATUS_STATE >= STATUS_AT && STATUS_STATE < STATUS_AT + STATUS_LEN); +static_assert!(STATUS_COUNT >= STATUS_AT && STATUS_COUNT + 4 <= STATUS_AT + STATUS_LEN); + +pub(crate) const STATE_DATA_READY: u8 = 7; +pub(crate) const STATE_READING: u8 = 19; +pub(crate) const STATE_IDLE: u8 = 0; +pub(crate) const STATE_ARMED: u8 = 17; +static_assert!(STATE_ARMED != STATE_READING); +static_assert!(STATE_ARMED != STATE_DATA_READY); +static_assert!(STATE_ARMED != STATE_IDLE); +pub(crate) const STATE_NEEDS_PATCH: u8 = 9; + +// status[8], not the state byte status[7]. +pub(crate) const STATUS_PATCH_ACK: usize = 8; +pub(crate) const PATCH_ACCEPTED: u8 = 0x29; +static_assert!(STATUS_PATCH_ACK != 7); +static_assert!(STATUS_PATCH_ACK < STATUS_LEN); + +pub(crate) const MAX_CAPTURE: usize = 0x10000; + +const CRC_LEN: usize = 2; + +const FRAME_HEADER: usize = 7; +const FRAME_LEN_AT: usize = 3; +static_assert!(FRAME_LEN_AT + 2 <= FRAME_HEADER); + +const TYPE_SESSION: u8 = 0x72; +const TYPE_SEQUENCE: u8 = 0x73; +const TYPE_ENCRYPTED: u8 = 0x55; + +pub(crate) const SESSION_SHARE_LEN: usize = 40; +pub(crate) const CHALLENGE_LEN: usize = 64; + +const MAX_FRAME_PAYLOAD: usize = CHALLENGE_LEN; +static_assert!(SESSION_SHARE_LEN <= MAX_FRAME_PAYLOAD); + +pub(crate) const SESSION_REPLY_LEN: usize = 0x33; +pub(crate) const SESSION_REPLY_AT: usize = 9; +pub(crate) const CHALLENGE_REPLY_LEN: usize = 0x4b; +pub(crate) const CHALLENGE_REPLY_AT: usize = 9; + +static_assert!(SESSION_REPLY_AT + SESSION_SHARE_LEN <= SESSION_REPLY_LEN); +static_assert!(CHALLENGE_REPLY_AT + CHALLENGE_LEN <= CHALLENGE_REPLY_LEN); + +fn check(rc: c_int) -> Result<()> { + if rc < 0 { + Err(Error::from_errno(rc)) + } else { + Ok(()) + } +} + +pub(crate) fn register_driver() -> Result<()> { + // SAFETY: takes no arguments and is idempotent. + check(unsafe { sep_sensor_register() }) +} + +pub(crate) fn unregister_driver() { + // SAFETY: idempotent, and safe when nothing was ever registered. + unsafe { sep_sensor_unregister() } +} + +pub(crate) fn is_bound() -> bool { + // SAFETY: reads one pointer for nullness. + unsafe { sep_sensor_bound() != 0 } +} + +pub(crate) fn power_line() -> Option { + // SAFETY: no arguments; returns a negative errno when unbound. + let n = unsafe { sep_sensor_power_line() }; + if n < 0 { + None + } else { + Some(n) + } +} + +pub(crate) fn power_cycle() -> bool { + // SAFETY: the shim holds the descriptor and applies its delays. + unsafe { sep_sensor_power_cycle() == 0 } +} + +pub(crate) fn cs_timing() -> CsTiming { + // SAFETY: reads one int. + CsTiming::from_wire(unsafe { sep_sensor_cs_timing_mode() }) +} + +pub(crate) fn power_source() -> PowerSource { + // SAFETY: reads one int. + PowerSource::from_wire(unsafe { sep_sensor_power_source() }) +} + +pub(crate) fn power(on: bool) -> bool { + // SAFETY: the shim holds the descriptor and applies its delays. + unsafe { sep_sensor_power(if on { 1 } else { 0 }) == 0 } +} + +fn command(cmd: &[u8; CMD_LEN]) -> Result<()> { + let mut rx = [0u8; CMD_LEN]; + // SAFETY: both buffers are `CMD_LEN` bytes and live across the call. + check(unsafe { sep_sensor_xfer(cmd.as_ptr().cast(), rx.as_mut_ptr().cast(), CMD_LEN) }) +} + +pub(crate) fn idle() -> Result<()> { + command(&CMD_IDLE) +} + +pub(crate) fn start_capture() -> Result<()> { + command(&CMD_START_CAPTURE) +} + +pub(crate) fn setup_patch_enable() -> Result<()> { + command(&CMD_SETUP_PATCH_ENABLE) +} + +fn send_framed(frame_type: u8, payload: &[u8]) -> Result<()> { + let total = FRAME_HEADER + payload.len(); + if payload.len() > MAX_FRAME_PAYLOAD || total > u16::MAX as usize { + return Err(EINVAL); + } + + let mut frame = [0u8; FRAME_HEADER + MAX_FRAME_PAYLOAD]; + frame[0] = 0x80; + frame[1] = frame_type; + frame[FRAME_LEN_AT..FRAME_LEN_AT + 2].copy_from_slice(&(total as u16).to_le_bytes()); + frame[FRAME_HEADER..total].copy_from_slice(payload); + + // SAFETY: `frame` is at least `total` bytes and lives across the call; the + // shim only reads from it and there is no receive buffer. + check(unsafe { sep_sensor_xfer_tx(frame.as_ptr().cast(), total) }) +} + +const ENCRYPTED_DECLARED_MAX: usize = 0x12c; +const ENCRYPTED_OVERHEAD: usize = FRAME_HEADER + CRC_LEN; +static_assert!(ENCRYPTED_OVERHEAD == 9); +const ENCRYPTED_TRANSFER_MAX: usize = 0x12b; +static_assert!(ENCRYPTED_TRANSFER_MAX < ENCRYPTED_DECLARED_MAX); + +pub(crate) struct Geometry { + declared: usize, + transfer: usize, +} + +impl Geometry { + // 0x5d. + pub(crate) const COVERAGE: Geometry = Geometry { + declared: 0x40, + transfer: 0x40, + }; + // 0x5c. + pub(crate) const OPERATION: Geometry = Geometry { + declared: 0x40, + transfer: 0x40, + }; + pub(crate) const MODULE_CHALLENGE: Geometry = Geometry { + declared: 0x49, + transfer: 0x49, + }; + + // 0x6a, transmitted zero-padded past declared. + pub(crate) const TRANSPARENT: Geometry = Geometry { + declared: 0x4d, + transfer: 0x12b, + }; + + pub(crate) const fn payload_capacity(&self) -> usize { + self.declared - ENCRYPTED_OVERHEAD + } + + pub(crate) fn declared(&self) -> usize { + self.declared + } +} + +static_assert!(Geometry::MODULE_CHALLENGE.declared < ENCRYPTED_DECLARED_MAX); +static_assert!(Geometry::MODULE_CHALLENGE.declared > ENCRYPTED_OVERHEAD); +static_assert!(Geometry::MODULE_CHALLENGE.transfer >= Geometry::MODULE_CHALLENGE.declared); +static_assert!(Geometry::MODULE_CHALLENGE.transfer <= ENCRYPTED_TRANSFER_MAX); +static_assert!(Geometry::MODULE_CHALLENGE.payload_capacity() == 0x40); +static_assert!(Geometry::COVERAGE.declared < ENCRYPTED_DECLARED_MAX); +static_assert!(Geometry::OPERATION.declared < ENCRYPTED_DECLARED_MAX); +static_assert!(Geometry::TRANSPARENT.declared < ENCRYPTED_DECLARED_MAX); +static_assert!(Geometry::COVERAGE.declared > ENCRYPTED_OVERHEAD); +static_assert!(Geometry::OPERATION.declared > ENCRYPTED_OVERHEAD); +static_assert!(Geometry::TRANSPARENT.declared > ENCRYPTED_OVERHEAD); +static_assert!(Geometry::COVERAGE.transfer >= Geometry::COVERAGE.declared); +static_assert!(Geometry::OPERATION.transfer >= Geometry::OPERATION.declared); +static_assert!(Geometry::TRANSPARENT.transfer >= Geometry::TRANSPARENT.declared); +static_assert!(Geometry::COVERAGE.transfer <= ENCRYPTED_TRANSFER_MAX); +static_assert!(Geometry::OPERATION.transfer <= ENCRYPTED_TRANSFER_MAX); +static_assert!(Geometry::TRANSPARENT.transfer <= ENCRYPTED_TRANSFER_MAX); + +fn crc16_ansi(data: &[u8]) -> u16 { + let mut crc: u16 = 0; + for &byte in data { + crc ^= u16::from(byte); + for _ in 0..8 { + if crc & 1 != 0 { + crc = (crc >> 1) ^ 0xA001; + } else { + crc >>= 1; + } + } + } + crc +} + +pub(crate) struct Params(KVec); + +impl Params { + pub(crate) fn new(blob: KVec) -> Params { + Params(blob) + } + + pub(crate) fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub(crate) fn len(&self) -> usize { + self.0.len() + } +} + +impl Drop for Params { + fn drop(&mut self) { + for b in self.0.iter_mut() { + // SAFETY: a valid, uniquely borrowed byte. + unsafe { core::ptr::write_volatile(b, 0) }; + } + } +} + +pub(crate) enum ParamsError { + Empty, + TooLong(usize, usize), + Transfer(Error), +} + +pub(crate) fn send_encrypted_parameters( + params: &Params, + geom: &Geometry, +) -> core::result::Result<(), ParamsError> { + if params.0.is_empty() { + return Err(ParamsError::Empty); + } + let capacity = geom.payload_capacity(); + if params.0.len() > capacity { + return Err(ParamsError::TooLong(params.0.len(), capacity)); + } + + let declared = geom.declared; + let transfer = geom.transfer; + let mut frame = [0u8; ENCRYPTED_TRANSFER_MAX]; + + frame[0] = 0x80; + frame[1] = TYPE_ENCRYPTED; + // Declared size, not the transferred count; the tail past it is zero padding. + frame[FRAME_LEN_AT..FRAME_LEN_AT + 2].copy_from_slice(&(declared as u16).to_le_bytes()); + frame[FRAME_HEADER..FRAME_HEADER + params.0.len()].copy_from_slice(¶ms.0); + + let crc_at = declared - CRC_LEN; + let crc = crc16_ansi(&frame[..crc_at]); + frame[crc_at..declared].copy_from_slice(&crc.to_le_bytes()); + + // SAFETY: `frame` is `ENCRYPTED_TRANSFER_MAX` bytes, `transfer` is no + // larger (asserted for every `Geometry`), and it lives across the call; the + // shim only reads from it and there is no receive buffer. + check(unsafe { sep_sensor_xfer_tx(frame.as_ptr().cast(), transfer) }) + .map_err(ParamsError::Transfer) +} + +pub(crate) const MODULE_REPLY_LENS: [usize; 2] = [0x4b, 0x4f]; +static_assert!(MODULE_REPLY_LENS[0] <= MAX_REPLY); +static_assert!(MODULE_REPLY_LENS[1] <= MAX_REPLY); +static_assert!(MODULE_REPLY_LENS[0] != MODULE_REPLY_LENS[1]); + +const MODULE_REPLY_ADJUST: usize = 11; +const MODULE_REPLY_AT: usize = 9; +static_assert!(MODULE_REPLY_AT == SESSION_REPLY_AT); +static_assert!(MODULE_REPLY_AT == CHALLENGE_REPLY_AT); +static_assert!(MODULE_REPLY_AT == SENSOR_SERIAL_AT); +static_assert!(MODULE_REPLY_ADJUST == MODULE_REPLY_AT + CRC_LEN); +static_assert!(MODULE_REPLY_LENS[0] > MODULE_REPLY_ADJUST); + +pub(crate) fn send_module_challenge(challenge: &Params) -> core::result::Result<(), ParamsError> { + send_encrypted_parameters(challenge, &Geometry::MODULE_CHALLENGE) +} + +pub(crate) fn read_module_reply() -> Result> { + let count = poll_until_ready_any(&MODULE_REPLY_LENS)?; + + let mut buf = [0xffu8; MAX_REPLY]; + read_reply_ready(count, &mut buf)?; + + // Relayed unparsed to the enclave, op 0x32. + let len = count - MODULE_REPLY_ADJUST; + let mut out = KVec::new(); + out.extend_from_slice(&buf[MODULE_REPLY_AT..MODULE_REPLY_AT + len], GFP_KERNEL)?; + Ok(out) +} + +fn poll_until_ready_any(accepted: &[usize]) -> Result { + for _ in 0..READ_POLL_ATTEMPTS { + let st = status()?; + if let Offset12::Available(count) = st.offset12() { + let count = count as usize; + if accepted.contains(&count) { + return Ok(count); + } + } + fsleep(Delta::from_millis(READ_POLL_MS)); + } + Err(ETIMEDOUT) +} + +pub(crate) fn send_session_share(share: &[u8; SESSION_SHARE_LEN]) -> Result<()> { + send_framed(TYPE_SESSION, share) +} + +pub(crate) fn send_challenge(challenge: &[u8; CHALLENGE_LEN]) -> Result<()> { + send_framed(TYPE_SEQUENCE, challenge) +} + +static_assert!(FRAME_HEADER + SESSION_SHARE_LEN == 0x2f); +static_assert!(FRAME_HEADER + CHALLENGE_LEN == 0x47); + +fn read_framed(total: usize, at: usize, out: &mut [u8]) -> Result<()> { + let mut buf = [0xffu8; MAX_REPLY]; + read_reply(total, &mut buf)?; + if at + out.len() > total { + return Err(EINVAL); + } + out.copy_from_slice(&buf[at..at + out.len()]); + Ok(()) +} + +fn poll_until_ready(expected: usize) -> Result<()> { + for _ in 0..READ_POLL_ATTEMPTS { + let st = status()?; + if let Offset12::Available(count) = st.offset12() { + if count as usize == expected { + return Ok(()); + } + } + fsleep(Delta::from_millis(READ_POLL_MS)); + } + Err(ETIMEDOUT) +} + +const READ_POLL_MS: i64 = 5; +const READ_POLL_ATTEMPTS: usize = 200; + +fn read_reply(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { + poll_until_ready(total)?; + read_reply_ready(total, buf) +} + +fn read_reply_ready(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { + if total > MAX_REPLY { + return Err(EINVAL); + } + + let mut cmd = [0u8; READ_CMD_LEN]; + cmd[..CMD_LEN].copy_from_slice(&CMD_READ_PREFIX); + cmd[CMD_LEN..].copy_from_slice(&(total as u32).to_le_bytes()); + + // SAFETY: `cmd` is `READ_CMD_LEN` bytes, `buf` is `MAX_REPLY` and `total` + // is bounded by it; both live across the call and the shim writes only into + // the receive buffer. + check(unsafe { + sep_sensor_xfer2( + cmd.as_ptr().cast(), + READ_CMD_LEN, + buf.as_mut_ptr().cast(), + total, + ) + }) +} + +fn read_framed_verified(total: usize, at: usize, out: &mut [u8]) -> Result<()> { + if total < CRC_LEN || at + out.len() > total - CRC_LEN { + return Err(EINVAL); + } + let mut buf = [0xffu8; MAX_REPLY]; + read_reply(total, &mut buf)?; + + let split = total - CRC_LEN; + let expected = u16::from_le_bytes([buf[split], buf[split + 1]]); + let computed = crc16_ansi(&buf[..split]); + if expected != computed { + return Err(EIO); + } + + out.copy_from_slice(&buf[at..at + out.len()]); + Ok(()) +} + +const MAX_REPLY: usize = 0x4f; +static_assert!(SESSION_REPLY_LEN <= MAX_REPLY); +static_assert!(CHALLENGE_REPLY_LEN <= MAX_REPLY); +static_assert!(SENSOR_SERIAL_REPLY_LEN <= MAX_REPLY); + +pub(crate) const SENSOR_SERIAL_REPLY_LEN: usize = 0x1b; +pub(crate) const SENSOR_SERIAL_LEN: usize = 16; +pub(crate) const SENSOR_SERIAL_AT: usize = 9; +static_assert!(SENSOR_SERIAL_AT == SESSION_REPLY_AT); +static_assert!(SENSOR_SERIAL_AT + SENSOR_SERIAL_LEN <= SENSOR_SERIAL_REPLY_LEN - CRC_LEN); + +pub(crate) struct SensorSerial([u8; SENSOR_SERIAL_LEN]); + +impl SensorSerial { + pub(crate) fn bytes(&self) -> &[u8; SENSOR_SERIAL_LEN] { + &self.0 + } +} + +pub(crate) fn read_sensor_serial() -> Result { + command(&CMD_GET_SENSOR_SERIAL)?; + let mut serial = [0u8; SENSOR_SERIAL_LEN]; + read_framed_verified(SENSOR_SERIAL_REPLY_LEN, SENSOR_SERIAL_AT, &mut serial)?; + Ok(SensorSerial(serial)) +} + +pub(crate) fn read_session_reply() -> Result<[u8; SESSION_SHARE_LEN]> { + let mut share = [0u8; SESSION_SHARE_LEN]; + read_framed(SESSION_REPLY_LEN, SESSION_REPLY_AT, &mut share)?; + Ok(share) +} + +pub(crate) fn read_challenge_reply() -> Result<[u8; CHALLENGE_LEN]> { + let mut reply = [0u8; CHALLENGE_LEN]; + read_framed(CHALLENGE_REPLY_LEN, CHALLENGE_REPLY_AT, &mut reply)?; + Ok(reply) +} + +pub(crate) fn send_patch(blob: &[u8]) -> Result<()> { + if blob.is_empty() { + return Err(EINVAL); + } + // SAFETY: `blob` is a live slice for the duration of the call and the shim + // only reads from it; there is no receive buffer to write into. + check(unsafe { sep_sensor_xfer_tx(blob.as_ptr().cast(), blob.len()) }) +} + +pub(crate) struct Status { + pub(crate) state: u8, + offset12: u32, + pub(crate) raw: [u8; STATUS_LEN], +} + +pub(crate) enum Offset12 { + Identifier(u16), + Available(u32), + Undefined, +} + +pub(crate) struct Identifier(u16); + +impl Identifier { + pub(crate) fn value(&self) -> u16 { + self.0 + } +} + +impl Status { + pub(crate) fn offset12(&self) -> Offset12 { + match self.state { + STATE_IDLE => Offset12::Identifier(u16::from_le_bytes([ + self.raw[STATUS_IDENTIFIER], + self.raw[STATUS_IDENTIFIER + 1], + ])), + STATE_DATA_READY => Offset12::Available(self.offset12), + _ => Offset12::Undefined, + } + } + + pub(crate) fn identifier_to_register(&self) -> Option { + match self.offset12() { + Offset12::Identifier(0) => None, + Offset12::Identifier(value) => Some(Identifier(value)), + Offset12::Available(_) | Offset12::Undefined => None, + } + } + + pub(crate) fn patch_ack(&self) -> u8 { + self.raw[STATUS_PATCH_ACK] + } + + pub(crate) fn is_silent(&self) -> bool { + self.raw.iter().all(|b| *b == 0) + } +} + +pub(crate) fn status() -> Result { + let mut tx = [0xffu8; STATUS_XFER_LEN]; + tx[..CMD_LEN].copy_from_slice(&CMD_GET_STATUS); + let mut rx = [0u8; STATUS_XFER_LEN]; + + // SAFETY: both buffers are `STATUS_XFER_LEN` bytes and live across the call. + check(unsafe { + sep_sensor_xfer(tx.as_ptr().cast(), rx.as_mut_ptr().cast(), STATUS_XFER_LEN) + })?; + + let mut raw = [0u8; STATUS_LEN]; + raw.copy_from_slice(&rx[STATUS_AT..STATUS_AT + STATUS_LEN]); + + Ok(Status { + state: rx[STATUS_STATE], + offset12: u32::from_le_bytes([ + rx[STATUS_COUNT], + rx[STATUS_COUNT + 1], + rx[STATUS_COUNT + 2], + rx[STATUS_COUNT + 3], + ]), + raw, + }) +} + +pub(crate) struct Capture(KVec); + +impl Capture { + pub(crate) fn bytes(&self) -> &[u8] { + &self.0 + } + +} + +impl Drop for Capture { + fn drop(&mut self) { + for b in self.0.iter_mut() { + // SAFETY: a valid, uniquely borrowed byte. + unsafe { core::ptr::write_volatile(b, 0) }; + } + } +} + +pub(crate) enum CaptureError { + Length(u32), + Bus(Error), + NoMemory, + Checksum { advertised: u16, computed: u16 }, +} + +pub(crate) fn read_capture(length: u32) -> core::result::Result { + let len = length as usize; + if length == 0 || len > MAX_CAPTURE || len <= CRC_LEN { + return Err(CaptureError::Length(length)); + } + + let mut cmd = [0u8; READ_CMD_LEN]; + cmd[..CMD_LEN].copy_from_slice(&CMD_READ_PREFIX); + cmd[CMD_LEN..].copy_from_slice(&length.to_le_bytes()); + + let mut buf = KVec::with_capacity(len, GFP_KERNEL).map_err(|_| CaptureError::NoMemory)?; + buf.resize(len, 0xff, GFP_KERNEL) + .map_err(|_| CaptureError::NoMemory)?; + + // SAFETY: `cmd` is `READ_CMD_LEN` bytes and `buf` is `len` bytes; both live + // across the call, and the shim writes only into the receive buffer. + let rc = unsafe { + sep_sensor_xfer2( + cmd.as_ptr().cast(), + READ_CMD_LEN, + buf.as_mut_ptr().cast(), + len, + ) + }; + if rc < 0 { + return Err(CaptureError::Bus(Error::from_errno(rc))); + } + + let split = len - CRC_LEN; + let advertised = u16::from_le_bytes([buf[split], buf[split + 1]]); + // CRC-16/ANSI, not CCITT-FALSE. + let computed = crc16_ansi(&buf[..split]); + if advertised != computed { + return Err(CaptureError::Checksum { + advertised, + computed, + }); + } + + Ok(Capture(buf)) +} diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c new file mode 100644 index 0000000000000..a182d6ac04817 --- /dev/null +++ b/drivers/soc/apple/sensor_shim.c @@ -0,0 +1,401 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* Fingerprint sensor SPI shim: moves bytes over the bus and toggles power. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +/* + * 8 MHz, SPI mode 2 (CPOL=1 CPHA=0), 8-bit words. Mode 3 returns sixteen zero + * bytes, else identical. Must agree with the node's spi-cpol / absent spi-cpha, + * which is what spi_setup() applies. + */ +#define SEP_SENSOR_HZ 8000000 +#define SEP_SENSOR_BITS 8 +static unsigned int sep_sensor_mode = SPI_MODE_2; +module_param_named(sensor_spi_mode, sep_sensor_mode, uint, 0444); +MODULE_PARM_DESC(sensor_spi_mode, + "SPI mode for the sensor: 2 (CPOL=1 CPHA=0, the default and the only mode this sensor answers under) or 3 (CPOL=1 CPHA=1, reads all-zero). For comparison only."); +#define SEP_SENSOR_MODE sep_sensor_mode + +/* Chip-select setup and hold, in ns. */ +#define SEP_SENSOR_CS_NS 20 + +#define SEP_SENSOR_OFF_DELAY_MS 10 +#define SEP_SENSOR_ON_DELAY_MS 7 + +static struct spi_device *sep_spi; +static struct gpio_desc *sep_power; +static bool sep_registered; + +/* + * Power line resolved by DT node path, not gpiochip index: registration order + * shifts, so an index would silently pick the wrong chip. + */ +#define SEP_SENSOR_GPIO_NODE "/soc/pinctrl@39b028000" +#define SEP_SENSOR_GPIO_LINE 122 + +#define SEP_POWER_NONE 0 +#define SEP_POWER_NODE_PROPERTY 1 +#define SEP_POWER_CHIP_LINE 2 + +static int sep_power_source; + +/* + * Takes the power line as an output driven low: the power cycle begins with an + * off phase, so the line must be actively driven off, not merely read as low. + */ +static void sep_acquire_power(struct spi_device *spi) +{ + struct device_node *np; + struct gpio_device *gdev; + struct gpio_chip *gc; + + sep_power = gpiod_get_index(&spi->dev, NULL, 0, GPIOD_OUT_LOW); + if (!IS_ERR(sep_power)) { + sep_power_source = SEP_POWER_NODE_PROPERTY; + dev_info(&spi->dev, + "sep sensor: power line from the device node, driven low\n"); + return; + } + sep_power = NULL; + + np = of_find_node_by_path(SEP_SENSOR_GPIO_NODE); + if (!np) { + dev_warn(&spi->dev, + "sep sensor: no DT node at %s, no power line\n", + SEP_SENSOR_GPIO_NODE); + return; + } + + gdev = gpio_device_find_by_fwnode(of_fwnode_handle(np)); + of_node_put(np); + if (!gdev) { + dev_warn(&spi->dev, + "sep sensor: %s has no registered GPIO device\n", + SEP_SENSOR_GPIO_NODE); + return; + } + + gc = gpio_device_get_chip(gdev); + if (gc) + sep_power = gpiochip_request_own_desc(gc, + SEP_SENSOR_GPIO_LINE, + "apple-mesa-power", + GPIO_LOOKUP_FLAGS_DEFAULT, + GPIOD_OUT_LOW); + if (!gc || IS_ERR(sep_power)) { + sep_power = NULL; + dev_warn(&spi->dev, + "sep sensor: could not take line %d on %s (%s)\n", + SEP_SENSOR_GPIO_LINE, SEP_SENSOR_GPIO_NODE, + gpio_device_get_label(gdev)); + gpio_device_put(gdev); + return; + } + + sep_power_source = SEP_POWER_CHIP_LINE; + dev_info(&spi->dev, + "sep sensor: power line %s line %d (%s), driven low\n", + SEP_SENSOR_GPIO_NODE, SEP_SENSOR_GPIO_LINE, + gpio_device_get_label(gdev)); + gpio_device_put(gdev); +} + +static void sep_release_power(void) +{ + if (!sep_power) + return; + gpiod_set_value_cansleep(sep_power, 0); + if (sep_power_source == SEP_POWER_CHIP_LINE) + gpiochip_free_own_desc(sep_power); + else + gpiod_put(sep_power); + sep_power = NULL; + sep_power_source = SEP_POWER_NONE; +} + +#define SEP_CS_TIMING_SOFTWARE 0 +#define SEP_CS_TIMING_HOOK_BYPASSED 1 +#define SEP_CS_TIMING_HARDWARE 2 + +static int sep_cs_timing_mode; + +/* + * Programs the 20 ns chip-select setup/hold and applies mode/speed via + * spi_setup(), which reaches the controller's set_cs_timing hook. + * + * This sensor needs hardware timing. The hook runs only for a native chip + * select; with a GPIO chip select the core silently emulates the delays in + * software, which this sensor rejects but which looks like success. + */ +static int sep_apply_cs_timing(struct spi_device *spi) +{ + struct spi_controller *ctlr = spi->controller; + int rc; + + if (!ctlr->set_cs_timing) + sep_cs_timing_mode = SEP_CS_TIMING_SOFTWARE; + else if (spi_get_csgpiod(spi, 0)) + sep_cs_timing_mode = SEP_CS_TIMING_HOOK_BYPASSED; + else + sep_cs_timing_mode = SEP_CS_TIMING_HARDWARE; + + /* + * Hold the bus lock: these fields are read when a transfer asserts chip + * select. spi_setup() takes the controller io_mutex, not the bus lock, + * so no deadlock. + */ + rc = spi_bus_lock(ctlr); + if (rc) + return rc; + + spi->mode = SEP_SENSOR_MODE; + spi->bits_per_word = SEP_SENSOR_BITS; + spi->max_speed_hz = SEP_SENSOR_HZ; + spi->cs_setup.value = SEP_SENSOR_CS_NS; + spi->cs_setup.unit = SPI_DELAY_UNIT_NSECS; + spi->cs_hold.value = SEP_SENSOR_CS_NS; + spi->cs_hold.unit = SPI_DELAY_UNIT_NSECS; + + rc = spi_setup(spi); + + spi_bus_unlock(ctlr); + + if (rc) + return rc; + + switch (sep_cs_timing_mode) { + case SEP_CS_TIMING_HARDWARE: + dev_info(&spi->dev, + "sep sensor: CS timing %u ns programmed in hardware\n", + SEP_SENSOR_CS_NS); + break; + case SEP_CS_TIMING_HOOK_BYPASSED: + dev_warn(&spi->dev, + "sep sensor: GPIO chip select, CS timing emulated in software (sensor needs hardware timing)\n"); + break; + default: + dev_warn(&spi->dev, + "sep sensor: no set_cs_timing hook, %u ns emulated in software (sensor needs hardware timing)\n", + SEP_SENSOR_CS_NS); + break; + } + return 0; +} + +static int sep_sensor_probe(struct spi_device *spi) +{ + int rc; + + rc = sep_apply_cs_timing(spi); + if (rc) + return rc; + + sep_spi = spi; + sep_acquire_power(spi); + /* Read the mode back: a mode that did not take must not be logged as + * the one requested. */ + dev_info(&spi->dev, + "sep sensor: bound, %u Hz, mode %u (CPOL=%d CPHA=%d)\n", + spi->max_speed_hz, + (unsigned int)(spi->mode & (SPI_CPOL | SPI_CPHA)), + !!(spi->mode & SPI_CPOL), !!(spi->mode & SPI_CPHA)); + return 0; +} + +static void sep_sensor_remove(struct spi_device *spi) +{ + sep_release_power(); + sep_spi = NULL; +} + +/* + * Its own compatible, deliberately not spidev's: capture must stay in the + * kernel, and a spidev node would leak an image to userspace. No + * MODULE_DEVICE_TABLE, so udev cannot auto-load this module. + */ +static const struct of_device_id sep_sensor_of_match[] = { + { .compatible = "apple,mesa-fingerprint" }, + { } +}; + +/* + * Legacy ID table only to silence the SPI core's "no spi_device_id" note. No + * MODULE_DEVICE_TABLE: it quiets a message, it does not advertise a binding. + */ +static const struct spi_device_id sep_sensor_spi_ids[] = { + { "mesa-fingerprint", 0 }, + { } +}; + +static struct spi_driver sep_sensor_driver = { + .driver = { + .name = "apple-mesa", + .of_match_table = sep_sensor_of_match, + }, + .id_table = sep_sensor_spi_ids, + .probe = sep_sensor_probe, + .remove = sep_sensor_remove, +}; + +/* + * Registered before the device tree gains the sensor node, so the bind happens + * when the SPI core's notifier creates the device. + */ +int sep_sensor_register(void) +{ + int rc; + + if (sep_registered) + return 0; + rc = spi_register_driver(&sep_sensor_driver); + if (rc) + return rc; + sep_registered = true; + return 0; +} + +void sep_sensor_unregister(void) +{ + if (!sep_registered) + return; + spi_unregister_driver(&sep_sensor_driver); + sep_registered = false; +} + +int sep_sensor_bound(void) +{ + return sep_spi != NULL; +} + +/* Whether those delays reach hardware, are emulated, or are silently bypassed. */ +int sep_sensor_cs_timing_mode(void) +{ + return sep_cs_timing_mode; +} + +/* Power cycle: off, 10 ms, on, 7 ms. -ENODEV if there is no power line. */ +int sep_sensor_power_cycle(void) +{ + if (!sep_power) + return -ENODEV; + + gpiod_set_value_cansleep(sep_power, 0); + msleep(SEP_SENSOR_OFF_DELAY_MS); + gpiod_set_value_cansleep(sep_power, 1); + msleep(SEP_SENSOR_ON_DELAY_MS); + return 0; +} + +/* How the power line was obtained: none, the node's property, or chip+line. */ +int sep_sensor_power_source(void) +{ + return sep_power_source; +} + +int sep_sensor_power_line(void) +{ + if (!sep_power) + return -ENODEV; + return desc_to_gpio(sep_power); +} + +/* Powers the sensor on/off, holding the hardware settling delay. */ +int sep_sensor_power(int on) +{ + if (!sep_power) + return -ENODEV; + + gpiod_set_value_cansleep(sep_power, on ? 1 : 0); + if (on) + msleep(SEP_SENSOR_ON_DELAY_MS); + else + msleep(SEP_SENSOR_OFF_DELAY_MS); + return 0; +} + +/* + * One chip-select assertion, `len` clocks, full duplex. The caller supplies all + * `len` transmit bytes (trailing ones as 0xff) rather than relying on the + * controller running its transmit buffer dry. + */ +int sep_sensor_xfer(const void *tx, void *rx, size_t len) +{ + struct spi_transfer xfer = { + .tx_buf = tx, + .rx_buf = rx, + .len = len, + .speed_hz = SEP_SENSOR_HZ, + .bits_per_word = SEP_SENSOR_BITS, + }; + + if (!sep_spi) + return -ENODEV; + if (!len) + return -EINVAL; + + return spi_sync_transfer(sep_spi, &xfer, 1); +} + +/* + * One chip-select assertion, transmit only, rx_buf NULL (not a discard buffer). + * A simultaneous receive changes the long transfer's pacing and the sensor + * rejects the patch blob, so the receive must be absent entirely. + */ +int sep_sensor_xfer_tx(const void *tx, size_t len) +{ + struct spi_transfer xfer = { + .tx_buf = tx, + .rx_buf = NULL, + .len = len, + .speed_hz = SEP_SENSOR_HZ, + .bits_per_word = SEP_SENSOR_BITS, + }; + + if (!sep_spi) + return -ENODEV; + if (!len) + return -EINVAL; + + return spi_sync_transfer(sep_spi, &xfer, 1); +} + +/* + * Two transfers in one chip-select assertion: command out, then read in. + * Chip-select stays asserted because neither transfer sets cs_change. + */ +int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) +{ + struct spi_transfer xfers[2] = { + { + .tx_buf = tx, + .len = tx_len, + .speed_hz = SEP_SENSOR_HZ, + .bits_per_word = SEP_SENSOR_BITS, + }, + { + .rx_buf = rx, + .len = rx_len, + .speed_hz = SEP_SENSOR_HZ, + .bits_per_word = SEP_SENSOR_BITS, + }, + }; + + if (!sep_spi) + return -ENODEV; + if (!tx_len || !rx_len) + return -EINVAL; + + return spi_sync_transfer(sep_spi, xfers, 2); +} diff --git a/drivers/soc/apple/sep-bio.h b/drivers/soc/apple/sep-bio.h new file mode 100644 index 0000000000000..7ad0b814af648 --- /dev/null +++ b/drivers/soc/apple/sep-bio.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* + * Userspace interface for the SEP biometric device. + * + * The enclave does the matching. Nothing biometric crosses this interface: only + * an operation, a stage, a status, an opaque identity UUID, and an opaque host + * label userspace chooses. + */ + +#pragma once + +#include +#include + +#define SEP_BIO_IFACE_VERSION 4 + +#define SEP_BIO_UUID_LEN 16 +#define SEP_BIO_LABEL_LEN 128 +#define SEP_BIO_NONCE_LEN 32 +#define SEP_BIO_TOKEN_LEN 32 +#define SEP_BIO_MAX_IDENTITIES 32 +#define SEP_BIO_CHALLENGE_LEN 32 +#define SEP_BIO_ATTEST_PUB_LEN 65 +#define SEP_BIO_ATTEST_SIG_MAX 72 + +enum { + SEP_BIO_STATE_IDLE = 0, + SEP_BIO_STATE_PENDING = 1, + SEP_BIO_STATE_PROGRESS = 2, + SEP_BIO_STATE_DONE = 3, + SEP_BIO_STATE_FAILED = 4, +}; + + +enum { + SEP_BIO_NO_MATCH = 0, + SEP_BIO_MATCH = 1, + SEP_BIO_NOT_COMPARED = 2, +}; + +struct sep_bio_identity { + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +struct sep_bio_info { + __u32 version; + __u32 sensor_present; + __u32 enrolled; + __u32 capacity; + __u32 enroll_stages; + __u32 reserved[3]; +}; + +struct sep_bio_list { + __u32 count; + __u32 reserved; + struct sep_bio_identity id[SEP_BIO_MAX_IDENTITIES]; +}; + +struct sep_bio_enrol_start { + __u32 flags; + __u32 reserved; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +#define SEP_BIO_GUIDANCE_NONE 0 +#define SEP_BIO_GUIDANCE_PLACE 1 +#define SEP_BIO_GUIDANCE_LIFT_AND_MOVE 2 +#define SEP_BIO_GUIDANCE_HOLD_STILL 3 + +struct sep_bio_enrol_poll { + __u32 state; + __u32 stage; + __u32 stages_total; + __u32 status; + __u8 uuid[SEP_BIO_UUID_LEN]; + __u32 guidance; + __u32 progress_percent; +}; +struct sep_bio_verify_start { + __u32 flags; + __u32 reserved; + __u8 nonce[SEP_BIO_NONCE_LEN]; +}; + +struct sep_bio_verify_poll { + __u32 state; + __u32 result; + __u32 status; + __u32 reserved; + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 token[SEP_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ + __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ +}; + +struct sep_bio_delete { + __u8 uuid[SEP_BIO_UUID_LEN]; +}; + +/* + * Device attestation of key possession: the enclave signs 'challenge' with the + * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and + * returns the DER signature and public point. The private key never leaves the + * enclave. + */ +struct sep_bio_attest { + __u32 sig_len; /* out: DER signature length */ + __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ + __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ + __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ + __u8 reserved[3]; +}; + +#define SEP_BIO_IOC_MAGIC 0xB1 + +#define SEP_BIO_GET_INFO _IOR (SEP_BIO_IOC_MAGIC, 0x01, struct sep_bio_info) +#define SEP_BIO_LIST _IOR (SEP_BIO_IOC_MAGIC, 0x02, struct sep_bio_list) +#define SEP_BIO_ENROL_START _IOW (SEP_BIO_IOC_MAGIC, 0x03, struct sep_bio_enrol_start) +#define SEP_BIO_ENROL_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x04, struct sep_bio_enrol_poll) +#define SEP_BIO_VERIFY_START _IOW (SEP_BIO_IOC_MAGIC, 0x05, struct sep_bio_verify_start) +#define SEP_BIO_VERIFY_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x06, struct sep_bio_verify_poll) +#define SEP_BIO_CANCEL _IO (SEP_BIO_IOC_MAGIC, 0x07) +#define SEP_BIO_DELETE _IOW (SEP_BIO_IOC_MAGIC, 0x08, struct sep_bio_delete) +#define SEP_BIO_DELETE_ALL _IO (SEP_BIO_IOC_MAGIC, 0x09) +#define SEP_BIO_ATTEST _IOWR(SEP_BIO_IOC_MAGIC, 0x0a, struct sep_bio_attest) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index bcceb7ed4a6c4..a270c7cec92a5 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1,15 +1,42 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! Driver for the Apple SEP (Secure Enclave Processor): attaches to a running +//! SEP over the AP mailbox, drives Touch ID enrol/verify via `/dev/sep-bio`, +//! and registers a hwrng and a SEP-backed `trusted` key source. #![recursion_limit = "2048"] -//! Apple SEP driver -//! -//! Copyright (C) The Asahi Linux Contributors +#[cfg(not(CONFIG_OF_DYNAMIC))] +compile_error!("apple_sep requires CONFIG_OF_DYNAMIC: the SEP and DART nodes ship disabled and are enabled with a device-tree changeset"); + +mod bio; +mod catacomb; +mod control; +mod der; +mod dt; +mod fv; +mod hwrng; +mod image; +mod keybag; +mod proto; +mod refkey; +mod refkey_seal; +mod rxring; +mod sbio; +mod scrd; +mod seed; +mod sensor; +mod shim; +mod shmem; +mod sks; +mod store; +mod transfer; +mod trusted; +mod xarm; use kernel::{ - bindings, device, dma, - module_platform_driver, + driver, new_mutex, of, platform, @@ -25,346 +52,2160 @@ use kernel::{ Atomic, Relaxed, // }, + new_condvar, Arc, + CondVar, + CondVarTimeoutResult, Mutex, // }, + time, types::ForeignOwnable, workqueue::{ self, + impl_has_delayed_work, impl_has_work, + new_delayed_work, new_work, + DelayedWork, Work, WorkItem, // }, // }; -const SHMEM_SIZE: usize = 0x30000; -const MSG_BOOT_TZ0: u64 = 0x5; -const MSG_BOOT_IMG4: u64 = 0x6; -const MSG_SET_SHMEM: u64 = 0x18; -const MSG_BOOT_TZ0_ACK1: u64 = 0x69; -const MSG_BOOT_TZ0_ACK2: u64 = 0xD2; -const MSG_BOOT_IMG4_ACK: u64 = 0x6A; -const MSG_ADVERTISE_EP: u64 = 0; -const EP_DISCOVER: u64 = 0xFD; -const EP_SHMEM: u64 = 0xFE; -const EP_BOOT: u64 = 0xFF; - -const MSG_TYPE_SHIFT: u32 = 16; -const MSG_TYPE_MASK: u64 = 0xFF; -//const MSG_PARAM_SHIFT: u32 = 24; -//const MSG_PARAM_MASK: u64 = 0xFF; - -const MSG_EP_MASK: u64 = 0xFF; -const MSG_DATA_SHIFT: u32 = 32; - -const IOVA_SHIFT: u32 = 0xC; - -type ShMem = dma::Coherent<[u8]>; - -fn align_up(v: usize, a: usize) -> usize { - (v + a - 1) & !(a - 1) -} - -fn memcpy_to_iomem(iomem: &mut ShMem, off: usize, src: &[u8]) -> Result<()> { - // SAFETY: - // as_slice_mut() checks that off and src.len() are whithin iomem's limits. - // memcpy_to_iomem is only called from within probe() ansuring there are no - // concurrent read and write accesses to the same region while the slice is - // alive per as_slice_mut()'s requiremnts. - unsafe { - let target = &mut iomem.as_mut()[off..off + src.len()]; - target.copy_from_slice(src); - } - Ok(()) -} - -fn build_shmem(dev: &platform::Device) -> Result { - let fwnode = dev.as_ref().fwnode().ok_or(EIO)?; - let mut iomem = dma::Coherent::::zeroed_slice(dev.as_ref(), SHMEM_SIZE, GFP_KERNEL)?; - - let panic_offset = 0x4000; - let panic_size = 0x8000; - memcpy_to_iomem(&mut iomem, panic_offset, &1u32.to_le_bytes())?; - - let lpol_offset = panic_offset + panic_size; - let lpol_prop_name = c"local-policy-manifest"; - let lpol_prop_size = fwnode.property_count_elem::(lpol_prop_name)?; - let lpol = fwnode - .property_read_array_vec(lpol_prop_name, lpol_prop_size)? - .required_by(dev.as_ref())?; - memcpy_to_iomem( - &mut iomem, - lpol_offset, - &(lpol_prop_size as u32).to_le_bytes(), - )?; - memcpy_to_iomem(&mut iomem, lpol_offset + 4, &lpol)?; - let lpol_size = align_up(lpol_prop_size + 4, 0x4000); - - let ibot_offset = lpol_offset + lpol_size; - let ibot_prop_name = c"iboot-manifest"; - let ibot_prop_size = fwnode.property_count_elem::(ibot_prop_name)?; - let ibot = fwnode - .property_read_array_vec(ibot_prop_name, ibot_prop_size)? - .required_by(dev.as_ref())?; - memcpy_to_iomem( - &mut iomem, - ibot_offset, - &(ibot_prop_size as u32).to_le_bytes(), - )?; - memcpy_to_iomem(&mut iomem, ibot_offset + 4, &ibot)?; - let ibot_size = align_up(ibot_prop_size + 4, 0x4000); - - memcpy_to_iomem(&mut iomem, 0, b"CNIP")?; - memcpy_to_iomem(&mut iomem, 4, &(panic_size as u32).to_le_bytes())?; - memcpy_to_iomem(&mut iomem, 8, &(panic_offset as u32).to_le_bytes())?; - - memcpy_to_iomem(&mut iomem, 16, b"OPLA")?; - memcpy_to_iomem(&mut iomem, 16 + 4, &(lpol_size as u32).to_le_bytes())?; - memcpy_to_iomem(&mut iomem, 16 + 8, &(lpol_offset as u32).to_le_bytes())?; - - memcpy_to_iomem(&mut iomem, 32, b"IPIS")?; - memcpy_to_iomem(&mut iomem, 32 + 4, &(ibot_size as u32).to_le_bytes())?; - memcpy_to_iomem(&mut iomem, 32 + 8, &(ibot_offset as u32).to_le_bytes())?; - - memcpy_to_iomem(&mut iomem, 48, b"llun")?; - Ok(iomem) -} +const SETTLE_MS: time::Msecs = 200; +const FIRST_RESPONSE_MS: time::Msecs = 3000; -#[pin_data] -struct SepReceiveWork { - data: Arc, - msg: Message, - #[pin] - work: Work, +const SETTLE_WORK_ID: u64 = 1; + +const ENROL_WORK_ID: u64 = 2; +static_assert!(ENROL_WORK_ID != SETTLE_WORK_ID && ENROL_WORK_ID != 0); +const VERIFY_WORK_ID: u64 = 3; +static_assert!(VERIFY_WORK_ID != ENROL_WORK_ID && VERIFY_WORK_ID != SETTLE_WORK_ID); +static_assert!(VERIFY_WORK_ID != 0); + +const EXCHANGE_TIMEOUT_MS: time::Msecs = 15000; + +const PHASE_ATTACH: u32 = 0; +const PHASE_EXCHANGE: u32 = 1; +const PHASE_READY: u32 = 2; + +const ENDPOINTS_BEFORE_EXCHANGE: usize = 7; + +const OOL_SIZE_XARM: usize = 0x8000; + +const OOL_SIZE_XARS: usize = 0x8000; + +const OOL_SIZE_SBIO: usize = 0x4000; + +const OOL_SIZE_SCRD: usize = 0x4000; + +const SBIO_TIMEOUT_MS: time::Msecs = 5000; + +const SKS_ALLOC: usize = 0x8000; + + +const SKS_MAX_CAPTURE: usize = 8; + +const XARS_MAX_CAPTURE: usize = 8; + +const SCRD_MAX_CAPTURE: usize = 8; + +const SCRD_TIMEOUT_MS: time::Msecs = 2000; + +const SKS_TIMEOUT_MS: time::Msecs = 2000; + +const SKS_TIMEOUT_PER_KIB_MS: time::Msecs = 6000; + +const SKS_TIMEOUT_MAX_MS: time::Msecs = 30_000; + +const SKS_MAX_ABANDONED: usize = 8; + +const fn sks_timeout_for(image_len: usize) -> time::Msecs { + let kib = image_len / 1024; + let kib = if kib > u16::MAX as usize { + u16::MAX as time::Msecs + } else { + kib as time::Msecs + }; + let scaled = SKS_TIMEOUT_MS.saturating_add(SKS_TIMEOUT_PER_KIB_MS.saturating_mul(kib)); + if scaled > SKS_TIMEOUT_MAX_MS { + SKS_TIMEOUT_MAX_MS + } else { + scaled + } } -impl_has_work! { - impl HasWork for SepReceiveWork { self.work } +static_assert!(sks_timeout_for(64) == SKS_TIMEOUT_MS); +static_assert!(sks_timeout_for(1508) > SKS_TIMEOUT_MS); +static_assert!(sks_timeout_for(usize::MAX) == SKS_TIMEOUT_MAX_MS); + +const SKS_MAX_SET_ASIDE: u32 = 16; + +const DMA_RING_SIZE: usize = 4 * (1 << 12); + +const OOL_WRITE_POLL_MS: u32 = 1; +const OOL_WRITE_POLL_ATTEMPTS: u32 = 200; + +const OOL_POISON_INBOUND: u8 = 0xA5; +const OOL_POISON_OUTBOUND: u8 = 0x5A; + +const PROTECTED_DATA_AVAILABLE: bool = true; + +const RNG_MAX_WORDS_PER_READ: usize = 16; + +const SECMODE_UNKNOWN: u32 = u32::MAX; + +const MAX_ENDPOINTS: usize = 64; + + +#[derive(Clone, Copy)] +struct Endpoint { + id: u8, + fourcc: proto::Fourcc, + have_descriptor: bool, + have_config: bool, + descriptor_msg0: u64, + descriptor_msg1: u32, + config_msg0: u64, + config_msg1: u32, } -impl SepReceiveWork { - fn new(data: Arc, msg: Message) -> Result> { - Arc::pin_init( - pin_init!(SepReceiveWork { - data, - msg, - work <- new_work!("SepReceiveWork::work"), - }), - GFP_ATOMIC, - ) +impl Endpoint { + fn new(id: u8) -> Self { + Endpoint { + id, + fourcc: proto::Fourcc::ZERO, + have_descriptor: false, + have_config: false, + descriptor_msg0: 0, + descriptor_msg1: 0, + config_msg0: 0, + config_msg1: 0, + } } } -impl WorkItem for SepReceiveWork { - type Pointer = Arc; +struct OolPair { + endpoint: u8, + allocated: usize, + declared_in: usize, + declared_out: usize, + inbound: shmem::ShMem, + outbound: shmem::ShMem, + registered: bool, +} + +fn sks_declared_sizes() -> (usize, usize) { + (0x8000, 0x4000) +} + +const SKS_MALFORMED: i8 = crate::sks::SKS_STATUS_MALFORMED; + +struct Hex<'a>(&'a [u8]); - fn run(this: Arc) { - this.data.process_message(this.msg); +impl kernel::fmt::Display for Hex<'_> { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + use core::fmt::Write; + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + for (i, byte) in self.0.iter().enumerate() { + if i > 0 { + f.write_char(' ')?; + } + f.write_char(DIGITS[(*byte >> 4) as usize] as char)?; + f.write_char(DIGITS[(*byte & 0x0f) as usize] as char)?; + } + Ok(()) } } -struct FwRegionParams { - addr: u64, - size: usize, +const SKS_SECRET_LEN: usize = 32; +static_assert!(SKS_SECRET_LEN % 4 == 0); + +const SKS_LOCK_STATE_VARIANT: u32 = 1; + +#[derive(Clone, Copy)] +enum LockState { + Unlocked, + Locked, } -#[pin_data] -struct SepData { - dev: ARef, - #[pin] - mbox: Mutex>>, - shmem: ShMem, - region_params: FwRegionParams, - fw_mapped: Atomic, +impl LockState { + const fn wire(self) -> i32 { + match self { + LockState::Unlocked => 0, + LockState::Locked => 1, + } + } } -impl SepData { - fn new( - dev: &platform::Device, - region_params: FwRegionParams, - ) -> Result> { - Arc::pin_init( - try_pin_init!(SepData { - shmem: build_shmem(dev)?, - dev: ARef::::from(dev.as_ref()), - mbox <- new_mutex!(None), - region_params, - fw_mapped: Atomic::new(false), - }), - GFP_KERNEL, - ) +static_assert!(LockState::Unlocked.wire() == 0); +static_assert!(LockState::Locked.wire() == 1); + +const SKS_LOCK_STATE_FLAGS: u64 = 0; + +enum SbioOutcome { + Ok(KVec), + // status 0x01 + PrerequisiteMissing, + // status 0x16; NOT "malformed" — that mapping is another service's status space + Status16, + Other, +} + +const BRINGUP_FRESH: u32 = 0; +const BRINGUP_IDENTIFIED: u32 = 1; +const BRINGUP_ESTABLISHED: u32 = 2; + +pub(crate) struct PatchLoaded(()); + +pub(crate) struct ParametersApplied(()); + +pub(crate) struct CalibrationBlob(kernel::firmware::Firmware); + +impl CalibrationBlob { + fn new(fw: kernel::firmware::Firmware) -> CalibrationBlob { + CalibrationBlob(fw) } - fn start(&self) -> Result<()> { - self.mbox.lock().as_ref().unwrap().send( - Message { - msg0: EP_BOOT | (MSG_BOOT_TZ0 << MSG_TYPE_SHIFT), - msg1: 0, - }, - false, - ) + + pub(crate) fn bytes(&self) -> &[u8] { + self.0.data() } - fn load_fw_and_shmem(&self) -> Result<()> { - let fw_addr = unsafe { - let res = bindings::dma_map_resource( - self.dev.as_raw(), - self.region_params.addr, - self.region_params.size, - bindings::dma_data_direction_DMA_TO_DEVICE, - 0, - ); - if bindings::dma_mapping_error(self.dev.as_raw(), res) != 0 { - dev_err!(self.dev, "Failed to map firmware"); - return Err(ENOMEM); - } - self.fw_mapped.store(true, Relaxed); - res >> IOVA_SHIFT - }; - let guard = self.mbox.lock(); - let mbox = guard.as_ref().unwrap(); - mbox.send( - Message { - msg0: EP_BOOT | (MSG_BOOT_IMG4 << MSG_TYPE_SHIFT) | (fw_addr << MSG_DATA_SHIFT), - msg1: 0, - }, - false, - )?; - let shm_addr = self.shmem.dma_handle() >> IOVA_SHIFT; - mbox.send( - Message { - msg0: EP_SHMEM | (MSG_SET_SHMEM << MSG_TYPE_SHIFT) | (shm_addr << MSG_DATA_SHIFT), - msg1: 0, - }, - false, - )?; - Ok(()) +} + +struct EnrolMaterial { + special: crate::sks::SpecialHandle, + secret: Secret, +} + +struct ImageContext<'a> { + sep: &'a SepData, +} + +impl Drop for ImageContext<'_> { + fn drop(&mut self) { + let op = crate::sbio::sbio_image_cleanup(); + let _ = self.sep.sbio_call(&op); } - fn process_boot_msg(&self, msg: Message) { - let ty = (msg.msg0 >> MSG_TYPE_SHIFT) & MSG_TYPE_MASK; - match ty { - MSG_BOOT_TZ0_ACK1 => {} - MSG_BOOT_TZ0_ACK2 => { - let res = self.load_fw_and_shmem(); - if let Err(e) = res { - dev_err!(self.dev, "Unable to load firmware: {:?}", e); - } - } - MSG_BOOT_IMG4_ACK => {} - _ => { - dev_err!(self.dev, "Unknown boot message type: {}", ty); - } +} + +struct OpenEnrolment<'a> { + sep: &'a SepData, + armed: bool, +} + +impl OpenEnrolment<'_> { + fn completed(mut self) { + self.armed = false; + self.sep.enrol_open.store(false, Relaxed); + } +} + +impl Drop for OpenEnrolment<'_> { + fn drop(&mut self) { + if !self.armed { + return; } + let _ = self.sep.sbio_call(&crate::sbio::sbio_cancel_operation()); + self.sep.enrol_open.store(false, Relaxed); } - fn process_discover_msg(&self, msg: Message) { - let ty = (msg.msg0 >> MSG_TYPE_SHIFT) & MSG_TYPE_MASK; - //let data = (msg.msg0 >> MSG_DATA_SHIFT) as u32; - //let param = (msg.msg0 >> MSG_PARAM_SHIFT) & MSG_PARAM_MASK; - match ty { - MSG_ADVERTISE_EP => { - /*dev_info!( - self.dev, - "Got endpoint {:?} at {}", - core::str::from_utf8(&data.to_be_bytes()), - param - );*/ - } - _ => { - //dev_warn!(self.dev, "Unknown discovery message type: {}", ty); - } +} + +enum ImageOutcome { + // `complete` flag from 0xbfe + Progress { + stage: u32, + percent: u32, + complete: bool, + has_template: bool, + }, + Retry, + NoFinger, + Failed(u32), +} + +enum CaptureWait { + Ready(u32), + Timeout, + Fault(u8), + Abandon, +} + +const ENROL_STATUS_SENSOR: u32 = 1; +const ENROL_STATUS_ENCLAVE: u32 = 2; +const ENROL_STATUS_TOO_MANY: u32 = 3; +const ENROL_STATUS_TIMEOUT: u32 = 4; +#[derive(Clone, Copy)] +enum LoadAnswer { + StatusZero, + ColdTransition, + // status 0x101: SEP re-activated the component from its own xART records + AlreadyActive, +} + +impl LoadAnswer { + fn describe(self) -> &'static CStr { + match self { + LoadAnswer::StatusZero => c"status 0", + LoadAnswer::ColdTransition => c"the cold-transition status 0x8002", + LoadAnswer::AlreadyActive => c"the already-active status 0x101", } } - fn process_message(&self, msg: Message) { - let ep = msg.msg0 & MSG_EP_MASK; - match ep { - EP_BOOT => self.process_boot_msg(msg), - EP_DISCOVER => self.process_discover_msg(msg), - _ => {} // dev_warn!(self.dev, "Message from unknown endpoint: {}", ep), +} + +enum RestoreOutcome { + Restored, + AlreadyActive, + NoStoredFile, + Failed, + // 0x8002 cold-transition; tolerated only for the lockout record, never a catacomb + EmptyTolerated, +} + +const PRIVATE_TYPE_CATACOMB_MASTER: u8 = 0xF2; +const PRIVATE_TYPE_CATACOMB_OWNER: u8 = 0xF3; +const PRIVATE_TYPE_CATACOMB_USER: u8 = 0xF4; +const PRIVATE_TYPE_LOCKOUT: u8 = 0xF5; +// 0xF1 is the identity index's type; a colliding type silently mis-restores +static_assert!(PRIVATE_TYPE_CATACOMB_MASTER != 0xF1); +static_assert!(PRIVATE_TYPE_CATACOMB_OWNER != PRIVATE_TYPE_CATACOMB_MASTER); +static_assert!(PRIVATE_TYPE_CATACOMB_USER != PRIVATE_TYPE_CATACOMB_OWNER); +static_assert!(PRIVATE_TYPE_CATACOMB_USER != PRIVATE_TYPE_CATACOMB_MASTER); +static_assert!(PRIVATE_TYPE_LOCKOUT != PRIVATE_TYPE_CATACOMB_USER); +static_assert!(PRIVATE_TYPE_LOCKOUT != PRIVATE_TYPE_CATACOMB_OWNER); +static_assert!(PRIVATE_TYPE_LOCKOUT != PRIVATE_TYPE_CATACOMB_MASTER); + +const ENROL_STATUS_UNFILED: u32 = 6; +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_RETRY); +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_ENCLAVE); +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_SENSOR); +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_TIMEOUT); +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_TOO_MANY); + +const ENROL_STATUS_RETRY: u32 = 5; +static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_TIMEOUT); +static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_ENCLAVE); +static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_SENSOR); + +const ENROL_MAX_CAPTURES: u32 = 12; + +const ENROL_POLL_MS: u32 = 2; + +const ENROL_CAPTURE_TIMEOUT_MS: u32 = 60_000; +const ENROL_POLL_ATTEMPTS: u32 = ENROL_CAPTURE_TIMEOUT_MS / ENROL_POLL_MS; +static_assert!(ENROL_POLL_ATTEMPTS * ENROL_POLL_MS == ENROL_CAPTURE_TIMEOUT_MS); + +const ENROL_REPOSITION_MS: u32 = 1800; + +const MATCH_SETTLE_MS: u32 = ENROL_REPOSITION_MS; + +const ENROL_IDLE_TIMEOUT_MS: u32 = 2000; + +const PATCH_POLL_MS: u32 = 20; +const PATCH_POLL_ATTEMPTS: u32 = 250; + +const SEAL_VECTOR: &[u8] = b"apple-sep seal round trip v1"; + +const SEAL_STATUS_SHAPE: i8 = SKS_MALFORMED; + +const SEAL_STATUS_NAMES_NOTHING: i8 = -11; + +const SEAL_STATUS_WRONG_KIND: i8 = -12; + +const SEAL_STATUS_MISSING_PREREQUISITE: i8 = -3; +static_assert!(SEAL_STATUS_MISSING_PREREQUISITE != SEAL_STATUS_WRONG_KIND); +static_assert!(SEAL_STATUS_MISSING_PREREQUISITE != SEAL_STATUS_NAMES_NOTHING); + +const SEAL_STATUS_BACKUP_WRAP: i8 = -14; + +static_assert!(SEAL_STATUS_WRONG_KIND != SEAL_STATUS_NAMES_NOTHING); +static_assert!(SEAL_STATUS_WRONG_KIND != SEAL_STATUS_SHAPE); +static_assert!(SEAL_STATUS_BACKUP_WRAP != SEAL_STATUS_WRONG_KIND); +static_assert!(SEAL_STATUS_BACKUP_WRAP != SEAL_STATUS_SHAPE); + +const SKS_WRAP_OVERHEAD: usize = 256; + +const SKS_WRAP_PRODUCTION_CAPACITY: u32 = SEAL_VECTOR.len() as u32 + SKS_WRAP_OVERHEAD as u32; + +const SKS_CAPACITY_MAP: [u32; 23] = [ + 128, 192, 255, 256, 257, 258, 300, 320, 384, 512, 513, 640, 768, 1023, 1024, 1025, 1026, 1280, + 1536, 2048, 2049, 3072, 4096, +]; + +const fn capacity_map_ascends() -> bool { + let mut i = 1; + while i < SKS_CAPACITY_MAP.len() { + if SKS_CAPACITY_MAP[i - 1] >= SKS_CAPACITY_MAP[i] { + return false; } + i += 1; } - fn remove(&self) { - *self.mbox.lock() = None; - if self.fw_mapped.load(Relaxed) { - unsafe { - bindings::dma_unmap_resource( - self.dev.as_raw(), - self.region_params.addr, - self.region_params.size, - bindings::dma_data_direction_DMA_TO_DEVICE, - 0, - ); - } + true +} +static_assert!(capacity_map_ascends()); +static_assert!(SKS_CAPACITY_MAP[0] > 0); +static_assert!(SKS_CAPACITY_MAP[5] < SKS_WRAP_PRODUCTION_CAPACITY); +static_assert!(SKS_WRAP_PRODUCTION_CAPACITY < SKS_CAPACITY_MAP[6]); +static_assert!(SKS_CAPACITY_MAP[2] + 1 == SKS_CAPACITY_MAP[3]); +static_assert!(SKS_CAPACITY_MAP[13] + 1 == SKS_CAPACITY_MAP[14]); + +const CALIBRATION_FIRMWARE: &CStr = c"apple/mesa_calibration.bin"; + +const SKS_CONFIG_MASKS: [u32; 17] = [ + 0xffff_ffff, + 0x1, + 0x3, + 0x5, + 0x9, + 0x11, + 0x21, + 0x41, + 0x81, + 0x101, + 0x201, + 0x401, + 0x801, + 0x1001, + 0x2001, + 0x4001, + 0x8001, +]; + +const fn config_masks_are_single_bit_probes() -> bool { + if SKS_CONFIG_MASKS[0] != u32::MAX || SKS_CONFIG_MASKS[1] != 1 { + return false; + } + let mut i = 2; + while i < SKS_CONFIG_MASKS.len() { + let m = SKS_CONFIG_MASKS[i]; + if m & 1 != 1 || (m & !1u32).count_ones() != 1 { + return false; + } + if i > 2 && SKS_CONFIG_MASKS[i - 1] >= m { + return false; } + i += 1; } + true } +static_assert!(config_masks_are_single_bit_probes()); -impl MailCallback for SepData { - type Data = Arc; - fn recv_message(data: ::Borrowed<'_>, msg: Message) { - let work = SepReceiveWork::new(data.into(), msg); - if let Ok(work) = work { - let res = workqueue::system().enqueue(work); - if res.is_err() { - dev_err!( - data.dev, - "Unable to schedule work item for message {}", - msg.msg0 - ); - } +const CONFIG_STAGE_NONE: u32 = 0; +const SBIO_PROBE_USER_ID: i32 = 1000; +static_assert!(SBIO_PROBE_USER_ID >= crate::sks::SKS_DESIGNATE_USER_MIN); +static_assert!(SBIO_PROBE_USER_ID == bio::ENROL_USER_ID); +static_assert!(SBIO_PROBE_USER_ID > 0); +static_assert!(SBIO_PROBE_USER_ID == crate::sks::SKS_IDENTITY_USER_ID); + +const SKS_LOAD_REPLY_LEN: usize = 8; + +// StoreType: the u32 at +0x60 of the create body +#[derive(Clone, Copy, PartialEq, Eq)] +struct StoreType(u32); + +const STORE_TYPE_MAX: u32 = 8; + +impl StoreType { + const IDENTITY: StoreType = StoreType(0); + const SEALING: StoreType = StoreType(1); + + const fn new(value: u32) -> Option { + if value <= STORE_TYPE_MAX { + Some(StoreType(value)) } else { - dev_err!( - data.dev, - "Unable to allocate work item for message {}", - msg.msg0 - ); + None } } + + const fn wire(self) -> u32 { + self.0 + } } -unsafe impl Send for SepData {} -unsafe impl Sync for SepData {} +static_assert!(StoreType::IDENTITY.wire() == 0); +static_assert!(StoreType::SEALING.wire() == 1); +static_assert!(StoreType::new(STORE_TYPE_MAX).is_some()); +static_assert!(StoreType::new(STORE_TYPE_MAX + 1).is_none()); -struct SepDriver(Arc); +#[derive(Clone, Copy)] -kernel::of_device_table!( - OF_TABLE, - MODULE_OF_TABLE, - (), - [(of::DeviceId::new(c"apple,sep"), ())] -); +#[must_use] +struct Healthy(()); -impl platform::Driver for SepDriver { - type IdInfo = (); +struct SksRequest { + name: &'static CStr, + msg: Message, + img: image::RequestImage, +} - const OF_ID_TABLE: Option> = Some(&OF_TABLE); +struct SksOutcome { + reply: crate::sks::SksReply, + response: Secret, +} - fn probe( - pdev: &platform::Device, - _info: Option<&()>, - ) -> impl PinInit { - let of = pdev.as_ref().of_node().ok_or(EIO)?; - let res = of.reserved_mem_region_to_resource_byname(c"sepfw")?; - let data = SepData::new( - pdev, - FwRegionParams { - addr: res.start(), - size: res.size().try_into()?, - }, - )?; - *data.mbox.lock() = Some(Mailbox::new_byname(pdev.as_ref(), c"mbox", data.clone())?); - data.start()?; - Ok(Self(data)) +pub(crate) struct Secret(pub(crate) KVec); + +impl Secret { + pub(crate) fn empty() -> Secret { + Secret(KVec::new()) } } -impl Drop for SepDriver { +impl core::ops::Deref for Secret { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + &self.0 + } +} + +impl Drop for Secret { fn drop(&mut self) { - self.0.remove(); + for b in self.0.iter_mut() { + // SAFETY: `b` is a valid, uniquely borrowed byte for this write, and + // a volatile store is what stops the compiler discarding a wipe of + // memory nothing reads afterwards. + unsafe { core::ptr::write_volatile(b, 0) }; + } + } +} + +#[derive(Clone, Copy)] +struct Abandoned { + selector: u8, + seq: u8, + label: &'static CStr, + waited_ms: u64, +} + +struct SksProbe { + active: bool, + captured: KVec, + label: Option<&'static CStr>, + unsolicited: u32, + abandoned: [Option; SKS_MAX_ABANDONED], + abandoned_next: usize, +} + +impl SksProbe { + fn new() -> Self { + SksProbe { + active: false, + captured: KVec::new(), + label: None, + unsolicited: 0, + abandoned: [None; SKS_MAX_ABANDONED], + abandoned_next: 0, + } + } +} + +struct XarsProbe { + active: bool, + captured: KVec, +} + +impl XarsProbe { + fn new() -> Self { + XarsProbe { + active: false, + captured: KVec::new(), + } + } +} + +struct ScrdProbe { + active: bool, + captured: KVec, +} + +impl ScrdProbe { + fn new() -> Self { + ScrdProbe { + active: false, + captured: KVec::new(), + } + } +} + +struct XarmState { + deferred_query: Option, + serviced: u32, + refused: u32, + os_uuid: Option<[u8; 16]>, +} + +impl XarmState { + fn new() -> Self { + XarmState { + deferred_query: None, + serviced: 0, + refused: 0, + os_uuid: None, + } + } +} + +struct EndpointTable { + eps: KVec, + discovery_msgs: u32, + unknown_types: u32, + dirty: bool, +} + +impl EndpointTable { + fn new() -> Self { + EndpointTable { + eps: KVec::new(), + discovery_msgs: 0, + unknown_types: 0, + dirty: false, + } + } + + fn slot(&mut self, id: u8) -> Result { + if let Some(i) = self.eps.iter().position(|e| e.id == id) { + return Ok(i); + } + if self.eps.len() >= MAX_ENDPOINTS { + return Err(ENOSPC); + } + self.eps.push(Endpoint::new(id), GFP_KERNEL)?; + self.dirty = true; + Ok(self.eps.len() - 1) + } +} + + +struct MachineRefKey { + blob: KVec, + pub_raw: KVec, +} + +#[pin_data] +struct SepData { + dev: ARef, + + #[pin] + mbox: Mutex>>, + + #[pin] + shmem: Mutex>, + + #[pin] + endpoints: Mutex, + + #[pin] + control: Mutex, + #[pin] + control_wq: CondVar, + + security_mode: Atomic, + + sks_wedged: Atomic, + + sks_seq: Atomic, + + // u32, not u8: the kernel's Rust atomics have no u8 AtomicType + phase: Atomic, + + #[pin] + ool_xarm: Mutex>, + + #[pin] + ool_sbio: Mutex>, + + #[pin] + sbio_last_header: Mutex>, + + #[pin] + dma_ring: Mutex>, + + #[pin] + sbio_rx: Mutex, + + #[pin] + sbio_wq: CondVar, + + sbio_ready: Atomic, + + templates_restored: Atomic, + + restore_attempted: Atomic, + + sensor_calibrated: Atomic, + + enrol_open: Atomic, + + #[pin] + enrol_material: Mutex>, + + #[pin] + enrol_identity_candidates: Mutex>, + + last_capture_end_ns: Atomic, + + device_view_synced: Atomic, + + backup_bag_other: Atomic, + + borrowed_sealing_handle: Atomic, + sealing_designations: Atomic, + config_stage: Atomic, + config_writes_sent: Atomic, + + keybag_designated: Atomic, + + cold_prepared: Atomic, + bringup_started: Atomic, + + bringup: Atomic, + + #[pin] + ool_sks: Mutex>, + + #[pin] + sks_probe: Mutex, + #[pin] + sks_wq: CondVar, + + #[pin] + ool_xars: Mutex>, + #[pin] + xars_probe: Mutex, + #[pin] + xars_wq: CondVar, + + #[pin] + ool_scrd: Mutex>, + #[pin] + scrd_probe: Mutex, + #[pin] + scrd_wq: CondVar, + + sensor_present: Atomic, + + #[pin] + bio_session: Mutex, + + #[pin] + bio_index: Mutex, + + #[pin] + bio_dev: Mutex>, + + #[pin] + store: Mutex>, + + #[pin] + xarm: Mutex, + + #[pin] + rng: Mutex>, + + #[pin] + machine_refkey: Mutex>, + + rng_shutdown: Atomic, + + rng_failures: Atomic, + + rx: rxring::RxRing, + + rx_count: Atomic, + settle_mark: Atomic, + settle_idle_ticks: Atomic, + + registered: Atomic, + + #[pin] + rx_work: Work, + + #[pin] + settle_work: DelayedWork, + + #[pin] + enrol_work: Work, + + #[pin] + verify_work: Work, +} + +impl_has_work! { + impl HasWork for SepData { self.rx_work } + impl HasWork for SepData { self.enrol_work } + impl HasWork for SepData { self.verify_work } +} + +impl_has_delayed_work! { + impl HasDelayedWork for SepData { self.settle_work } +} + +// SAFETY: every field is either internally synchronised (the mutexes, the +// atomics, the lock-free ring) or immutable after construction. The DMA buffer +// is only touched in probe, before the SEP has been told it exists, and at +// unbind, after the mailbox has been stopped. +unsafe impl Send for SepData {} +// SAFETY: see above. +unsafe impl Sync for SepData {} + +impl SepData { + fn new(pdev: &platform::Device) -> Result> { + let built = shmem::build(pdev)?; + let dev: &device::Device = pdev.as_ref(); + + let buf = built.buf; + + let ool_xarm = Self::alloc_ool( + dev, + xarm::EP_XARM, + OOL_SIZE_XARM, + (OOL_SIZE_XARM, OOL_SIZE_XARM), + )?; + let ool_sbio = Self::alloc_ool( + dev, + proto::EP_SBIO, + OOL_SIZE_SBIO, + (OOL_SIZE_SBIO, OOL_SIZE_SBIO), + )?; + let sks_declared = sks_declared_sizes(); + let ool_sks = Self::alloc_ool(dev, proto::EP_SKS, SKS_ALLOC, sks_declared)?; + let ool_xars = Self::alloc_ool( + dev, + xarm::EP_XARS, + OOL_SIZE_XARS, + (OOL_SIZE_XARS, OOL_SIZE_XARS), + )?; + let ool_scrd = Self::alloc_ool( + dev, + proto::EP_SCRD, + OOL_SIZE_SCRD, + (OOL_SIZE_SCRD, OOL_SIZE_SCRD), + )?; + let dma_ring = dma::Coherent::::zeroed_slice(dev, DMA_RING_SIZE, GFP_KERNEL)?; + + let mut store = match store::Store::open() { + Ok(mut store) => { + + if !store.seeded() { + match seed::import(&mut store) { + Ok(_imported) => {}, + Err(e) => dev_err!( + dev, + "seed '{}' could not be imported ({:?}); store was never seeded, so the exchange stops after the first ROOT_READ and endpoint count stays at {}\n", + seed::SEED_PATH, + e, + ENDPOINTS_BEFORE_EXCHANGE + ), + } + } + + Some(store) + } + Err(e) => { + dev_err!( + dev, + "could not open the backing store '{}': {:?}; persistent-state exchange cannot run\n", + store::STORE_PATH, + e + ); + None + } + }; + + let bio_index = match store.as_mut().map(bio::IdentityIndex::load) { + Some(Ok(index)) => index, + Some(Err(_)) => bio::IdentityIndex::new(), + None => bio::IdentityIndex::new(), + }; + + Arc::pin_init( + try_pin_init!(SepData { + dev: ARef::::from(dev), + mbox <- new_mutex!(None), + shmem <- new_mutex!(Some(buf)), + endpoints <- new_mutex!(EndpointTable::new()), + control <- new_mutex!(control::ControlState::new()), + control_wq <- new_condvar!("SepData::control_wq"), + security_mode: Atomic::new(SECMODE_UNKNOWN), + sks_seq: Atomic::new(0), + sks_wedged: Atomic::new(0), + bringup: Atomic::new(BRINGUP_FRESH), + keybag_designated: Atomic::new(false), + cold_prepared: Atomic::new(false), + bringup_started: Atomic::new(false), + enrol_open: Atomic::new(false), + sensor_calibrated: Atomic::new(false), + templates_restored: Atomic::new(false), + restore_attempted: Atomic::new(false), + enrol_material <- new_mutex!(None), + enrol_identity_candidates <- new_mutex!(KVec::new()), + last_capture_end_ns: Atomic::new(0), + device_view_synced: Atomic::new(false), + backup_bag_other: Atomic::new(0), + borrowed_sealing_handle: Atomic::new(0), + sealing_designations: Atomic::new(0), + config_writes_sent: Atomic::new(0), + config_stage: Atomic::new(CONFIG_STAGE_NONE), + phase: Atomic::new(PHASE_ATTACH), + ool_xarm <- new_mutex!(Some(ool_xarm)), + ool_sbio <- new_mutex!(Some(ool_sbio)), + sbio_last_header <- new_mutex!(None::<[u8; transfer::HEADER_LEN]>), + ool_sks <- new_mutex!(Some(ool_sks)), + sks_probe <- new_mutex!(SksProbe::new()), + sks_wq <- new_condvar!("SepData::sks_wq"), + ool_xars <- new_mutex!(Some(ool_xars)), + xars_probe <- new_mutex!(XarsProbe::new()), + xars_wq <- new_condvar!("SepData::xars_wq"), + ool_scrd <- new_mutex!(Some(ool_scrd)), + scrd_probe <- new_mutex!(ScrdProbe::new()), + scrd_wq <- new_condvar!("SepData::scrd_wq"), + dma_ring <- new_mutex!(Some(dma_ring)), + sbio_rx <- new_mutex!(transfer::Reassembly::new()), + sbio_wq <- new_condvar!("SepData::sbio_wq"), + sbio_ready: Atomic::new(false), + sensor_present: Atomic::new(false), + bio_session <- new_mutex!(bio::Session::new()), + bio_index <- new_mutex!(bio_index), + bio_dev <- new_mutex!(None), + store <- new_mutex!(store), + xarm <- new_mutex!(XarmState::new()), + rng <- new_mutex!(None), + machine_refkey <- new_mutex!(None), + rng_shutdown: Atomic::new(false), + rng_failures: Atomic::new(0), + rx: rxring::RxRing::new(), + rx_count: Atomic::new(0), + settle_mark: Atomic::new(0), + settle_idle_ticks: Atomic::new(0), + registered: Atomic::new(false), + rx_work <- new_work!("SepData::rx_work"), + enrol_work <- new_work!("SepData::enrol_work"), + verify_work <- new_work!("SepData::verify_work"), + settle_work <- new_delayed_work!("SepData::settle_work"), + }), + GFP_KERNEL, + ) + } + + fn attach(&self, sep_node: &dt::DtNode) -> Result<()> { + let (iova, size) = { + let guard = self.shmem.lock(); + let buf = guard.as_ref().ok_or(EINVAL)?; + (buf.dma_handle(), buf.len()) + }; + + let msg = proto::shmem_registration(iova, size)?; + + + if dt::registration_already_sent(sep_node) { + return Err(EBUSY); + } + + if let Err(e) = dt::mark_registration_sent(sep_node, iova) { + dev_err!( + self.dev, + "could not record the registration marker ({:?}); refusing to send, since without it an rmmod/insmod cycle could send it twice and park the SEP\n", + e + ); + return Err(e); + } + self.registered.store(true, Relaxed); + + self.mbox.lock().as_ref().ok_or(EINVAL)?.send(msg, false)?; + + // no ack on 0xFE; success is the discovery burst on 0xFD + Ok(()) + } + + fn send(&self, msg: Message) -> Result<()> { + self.mbox.lock().as_ref().ok_or(ENODEV)?.send(msg, false) + } + + + fn control_request(&self, op: &proto::ControlOp) -> Result> { + let (idx, tag) = self.control.lock().alloc()?; + let msg = proto::encode_control(op, tag); + + if let Err(e) = self.send(msg) { + self.control.lock().abandon(idx); + return Err(e); + } + + let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); + let mut guard = self.control.lock(); + loop { + if let Some(reply) = guard.take_reply(idx) { + guard.release(idx); + drop(guard); + if reply.tag != tag { + return Err(EIO); + } + return Ok(Some(reply)); + } + + if remaining == 0 { + let tag = guard.abandon(idx); + let retired = guard.retired_count(); + drop(guard); + if op.expects_reply() { + dev_warn!( + self.dev, + "control <- {}: no reply within {} ms; tag 0x{:02x} retired ({} retired in total)\n", + op.name(), + op.timeout_ms(), + tag, + retired + ); + } + return Ok(None); + } + + match self + .control_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + + fn get_entropy_word(&self) -> Result { + let op = proto::op_get_entropy(); + if !op.uses_reserved_tag() { + return Err(EINVAL); + } + + self.control.lock().entropy_begin()?; + + let msg = proto::encode_control(&op, proto::TAG_ENTROPY); + if let Err(e) = self.send(msg) { + self.control.lock().entropy_end(); + return Err(e); + } + + let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); + let mut guard = self.control.lock(); + let result = loop { + if let Some((value, _msg1)) = guard.entropy_take() { + break Ok(value); + } + if self.rng_shutdown.load(Relaxed) { + break Err(ECANCELED); + } + if remaining == 0 { + break Err(ETIMEDOUT); + } + match self + .control_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + }; + guard.entropy_end(); + result + } + + fn control_survey(&self) { + + for param in [0x00u8, 0x01, 0xff] { + let _ = self.control_request(&proto::op_nop(param)); + } + + match self.control_request(&proto::op_security_mode()) { + Ok(Some(reply)) => { + self.security_mode.store(reply.data_lo, Relaxed); + } + Ok(None) => {} + Err(_) => {} + } + + let mut words = [0u32; 4]; + let mut got = 0; + for w in words.iter_mut() { + match self.get_entropy_word() { + Ok(v) => { + *w = v; + got += 1; + } + Err(_) => { + break; + } + } + } + + if got == words.len() { + match self.register_hwrng() { + Ok(()) => {}, + Err(e) => dev_err!(self.dev, "hwrng: registration failed: {:?}\n", e), + } + } else { + dev_err!( + self.dev, + "hwrng: not registering, the source did not answer during the survey\n" + ); + } + + let _state = self.control.lock(); + } + + + fn alloc_ool( + dev: &device::Device, + endpoint: u8, + allocated: usize, + declared: (usize, usize), + ) -> Result { + if declared.0 > allocated || declared.1 > allocated { + return Err(EINVAL); + } + + let inbound = dma::Coherent::::zeroed_slice(dev, allocated, GFP_KERNEL)?; + let outbound = dma::Coherent::::zeroed_slice(dev, allocated, GFP_KERNEL)?; + + // SAFETY: nothing is registered yet, so the SEP does not know these + // buffers exist and cannot be touching them. + unsafe { + inbound.as_mut().fill(OOL_POISON_INBOUND); + outbound.as_mut().fill(OOL_POISON_OUTBOUND); + } + + Ok(OolPair { + endpoint, + allocated, + declared_in: declared.0, + declared_out: declared.1, + inbound, + outbound, + registered: false, + }) + } + + fn control_request_required(&self, op: &proto::ControlOp) -> Result { + match self.control_request(op)? { + Some(reply) => Ok(reply), + None => Err(ETIMEDOUT), + } + } + + fn register_ool(&self, slot: &Mutex>) -> Result<()> { + let guard = slot.lock(); + if guard.as_ref().is_some_and(|b| b.registered) { + return Ok(()); + } + + let (endpoint, allocated, declared_in, declared_out, in_iova, out_iova) = { + let b = guard.as_ref().ok_or(EINVAL)?; + ( + b.endpoint, + b.allocated, + b.declared_in, + b.declared_out, + b.inbound.dma_handle(), + b.outbound.dma_handle(), + ) + }; + + if declared_in > allocated || declared_out > allocated { + return Err(EINVAL); + } + + // drop the buffer lock before blocking control requests; the drain path takes it + drop(guard); + + self.control_request_required(&proto::op_ool_inbound_size(endpoint, declared_in as u32))?; + self.control_request_required(&proto::op_ool_inbound_addr(endpoint, in_iova))?; + self.control_request_required(&proto::op_ool_outbound_size(endpoint, declared_out as u32))?; + self.control_request_required(&proto::op_ool_outbound_addr(endpoint, out_iova))?; + + { + // deref first, or the guard's inherent as_mut wins over Option::as_mut + let mut guard = slot.lock(); + let pair: &mut Option = &mut guard; + if let Some(b) = pair.as_mut() { + b.registered = true; + } + } + Ok(()) + } + + fn with_store(&self, f: impl FnOnce(&mut store::Store) -> R) -> Option { + let mut guard = self.store.lock(); + let store: &mut Option = &mut guard; + store.as_mut().map(f) + } + + // pre-generate: an entropy draw on the drain path would deadlock on its own reply + fn prepare_os_uuid(&self) { + let key = xarm::os_uuid_key(); + + let existing = match self.with_store(|store| store.read(&key)) { + Some(result) => result, + None => return, + }; + + if let Ok(Some(value)) = existing { + if value.len() == 16 { + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&value); + self.xarm.lock().os_uuid = Some(uuid); + return; + } + } + + let mut uuid = [0u8; 16]; + for word in 0..4 { + match self.get_entropy_word() { + Ok(v) => uuid[word * 4..word * 4 + 4].copy_from_slice(&v.to_le_bytes()), + Err(_) => { + return; + } + } + } + xarm::make_uuid_v4(&mut uuid); + + match self.with_store(|store| store.write(&key, &uuid)) { + Some(Ok(())) => {} + Some(Err(_)) => { + return; + } + None => { + return; + } + } + self.xarm.lock().os_uuid = Some(uuid); + } + + fn on_xarm(&self, msg: Message) { + let req = crate::xarm::decode_xarm(&msg); + + if xarm::is_silent(req.opcode) { + self.xarm.lock().serviced += 1; + return; + } + + let ready = self.ool_xarm.lock().as_ref().is_some_and(|b| b.registered); + + if !ready { + if !xarm::needs_buffers(req.opcode) { + self.xarm.lock().deferred_query = Some(req.tag); + return; + } + + let mut state = self.xarm.lock(); + state.refused += 1; + drop(state); + self.fail_xarm(req.tag); + return; + } + + self.service_xarm(&req); + } + + fn service_xarm(&self, req: &crate::xarm::XarmRequest) { + let want = req.length as usize; + if want > OOL_SIZE_XARM { + self.fail_xarm(req.tag); + return; + } + + if want > 0 && !self.ool_await_written(&self.ool_xarm, 0, want) { + self.fail_xarm(req.tag); + return; + } + let payload = match self.ool_read(&self.ool_xarm, 0, want) { + Ok(p) => p, + Err(_) => { + self.fail_xarm(req.tag); + return; + } + }; + + let mut staging = KVec::new(); + if staging.resize(OOL_SIZE_XARM, 0u8, GFP_KERNEL).is_err() { + self.fail_xarm(req.tag); + return; + } + + let os_uuid = self.xarm.lock().os_uuid; + let serviced = self.with_store(|store| { + xarm::service( + req, + &payload, + &mut staging, + store, + PROTECTED_DATA_AVAILABLE, + os_uuid, + ) + }); + let Some(done) = serviced else { + self.fail_xarm(req.tag); + return; + }; + + if done.reply_bytes > 0 { + if self.ool_write(&self.ool_xarm, 0, &staging[..done.reply_bytes]).is_err() { + self.fail_xarm(req.tag); + return; + } + } + + self.xarm.lock().serviced += 1; + + self.send_xarm_reply(&done.reply); + } + + fn ool_read(&self, slot: &Mutex>, off: usize, len: usize) -> Result> { + let mut guard = slot.lock(); + let pair: &mut Option = &mut guard; + let buffers = pair.as_mut().ok_or(ENODEV)?; + if off + len > buffers.allocated { + return Err(EINVAL); + } + + let mut payload = KVec::new(); + if len > 0 { + // SAFETY: the SEP fills this buffer while preparing a request and + // then waits for the host's reply, so it is quiescent here, and the + // driver is the only other accessor. + let src = unsafe { &buffers.outbound.as_ref()[off..off + len] }; + payload.extend_from_slice(src, GFP_KERNEL)?; + } + + Ok(payload) + } + + fn ool_await_written(&self, slot: &Mutex>, off: usize, len: usize) -> bool { + for _ in 0..OOL_WRITE_POLL_ATTEMPTS { + { + let guard = slot.lock(); + let Some(buffers) = guard.as_ref() else { + return false; + }; + if off + len > buffers.allocated { + return true; + } + // SAFETY: the region is within the allocation as checked above, + // and the driver is the only host-side accessor. + let region = unsafe { &buffers.outbound.as_ref()[off..off + len] }; + if region.iter().any(|&b| b != OOL_POISON_OUTBOUND) { + return true; + } + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(OOL_WRITE_POLL_MS))); + } + false + } + + fn ool_write(&self, slot: &Mutex>, off: usize, bytes: &[u8]) -> Result<()> { + let guard = slot.lock(); + let buffers = guard.as_ref().ok_or(ENODEV)?; + if off + bytes.len() > buffers.allocated { + return Err(ENOSPC); + } + // SAFETY: the SEP reads this buffer only after the reply message is + // sent, which happens after this returns; the driver is the only writer. + unsafe { buffers.inbound.as_mut()[off..off + bytes.len()].copy_from_slice(bytes) }; + Ok(()) + } + + fn fail_xarm(&self, tag: u8) { + self.send_xarm_reply(&crate::xarm::XarmReply { + tag, + status: xarm::STATUS_FAILED, + length: 0, + args: [0; 3], + }); + } + + fn send_xarm_reply(&self, reply: &crate::xarm::XarmReply) { + let msg = crate::xarm::encode_xarm_reply(reply); + let _ = self.send(msg); + } + + fn release_deferred_query(&self) { + let Some(tag) = self.xarm.lock().deferred_query.take() else { + return; + }; + let mut reply = crate::xarm::XarmReply { + tag, + status: xarm::STATUS_OK, + length: 0, + args: [0; 3], + }; + reply.args[0] = u8::from(PROTECTED_DATA_AVAILABLE); + self.xarm.lock().serviced += 1; + self.send_xarm_reply(&reply); + } + + + fn on_xars(&self, msg: Message) { + let mut probe = self.xars_probe.lock(); + if probe.active { + if probe.captured.len() < XARS_MAX_CAPTURE { + let _ = probe.captured.push(msg, GFP_KERNEL); + } + drop(probe); + self.xars_wq.notify_all(); + } + } + + + fn on_scrd(&self, msg: Message) { + let mut probe = self.scrd_probe.lock(); + if probe.active { + if probe.captured.len() < SCRD_MAX_CAPTURE { + let _ = probe.captured.push(msg, GFP_KERNEL); + } + drop(probe); + self.scrd_wq.notify_all(); + } + } + + fn scrd_arm(&self) { + let mut probe = self.scrd_probe.lock(); + probe.captured.clear(); + probe.active = true; + } + + fn scrd_disarm(&self) -> KVec { + let mut probe = self.scrd_probe.lock(); + probe.active = false; + core::mem::take(&mut probe.captured) + } + + fn scrd_wait(&self, ms: time::Msecs, until: usize) { + let mut remaining = time::msecs_to_jiffies(ms); + let mut guard = self.scrd_probe.lock(); + loop { + if guard.captured.len() >= until || remaining == 0 { + return; + } + match self + .scrd_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + + fn scrd_zero_buffers(&self) -> Result<()> { + let mut guard = self.ool_scrd.lock(); + let pair: &mut Option = &mut guard; + let buffers = pair.as_mut().ok_or(ENODEV)?; + // SAFETY: nothing is outstanding on this endpoint here; the enclave + // touches these only between a request going out and its reply, and the + // driver is the only other accessor. + unsafe { + buffers.inbound.as_mut()[..buffers.allocated].fill(0); + buffers.outbound.as_mut()[..buffers.allocated].fill(0); + } + Ok(()) + } + + fn scrd_transact( + &self, + _name: &'static CStr, + request: u8, + payload: &[u8], + timeout_ms: time::Msecs, + ) -> Option<(crate::scrd::ScrdReply, KVec)> { + if !self.ool_registered(&self.ool_scrd) { + return None; + } + if self.scrd_zero_buffers().is_err() { + return None; + } + if !payload.is_empty() { + if self.ool_write(&self.ool_scrd, 0, payload).is_err() { + return None; + } + } + + let msg = crate::scrd::encode_scrd(request, payload.len()); + self.scrd_arm(); + if self.send(msg).is_err() { + let _ = self.scrd_disarm(); + return None; + } + + let mut examined = 0usize; + let correlating = loop { + self.scrd_wait(timeout_ms, examined + 1); + let next = { + let probe = self.scrd_probe.lock(); + let m = probe.captured.get(examined).copied(); + if m.is_some() { + examined += 1; + } + m + }; + let Some(candidate) = next else { break None }; + let decoded = crate::scrd::decode_scrd_reply(&candidate); + if decoded.request == request { + break Some(candidate); + } + if examined >= SCRD_MAX_CAPTURE { + break None; + } + }; + + let leftover = self.scrd_disarm(); + let correlating = match correlating { + Some(m) => Some(m), + None => leftover + .iter() + .skip(examined) + .copied() + .find(|m| crate::scrd::decode_scrd_reply(m).request == request), + }; + + let raw = correlating?; + + let reply = crate::scrd::decode_scrd_reply(&raw); + let mut response = KVec::new(); + if reply.response_size > 0 { + if let Ok(bytes) = self.ool_read(&self.ool_scrd, 0, reply.response_size as usize) { + response = bytes; + } + } + Some((reply, response)) + } + + fn scrd_ready(&self) -> bool { + if self.ool_registered(&self.ool_scrd) { + return true; + } + if !self.endpoint_present(proto::EP_SCRD) { + return false; + } + if self.register_ool(&self.ool_scrd).is_err() { + return false; + } + true + } + + fn scrd_command(&self, cmd: &crate::scrd::ScrdCommand) -> Option<(i32, KVec)> { + let (reply, out) = + self.scrd_transact(cmd.name(), cmd.request(), cmd.payload(), SCRD_TIMEOUT_MS)?; + Some((reply.status, out)) + } + + fn establish_passcode_validated_context( + &self, + user: crate::sbio::UserId, + ) -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { + if !self.scrd_ready() { + return None; + } + + let (special, secret) = { + let guard = self.enrol_material.lock(); + let material = guard.as_ref()?; + let mut copy = KVec::new(); + copy.extend_from_slice(&material.secret, GFP_KERNEL).ok()?; + (material.special, Secret(copy)) + }; + + let (status, _) = self.scrd_command(&crate::scrd::scrd_initialize())?; + if status != 0 { + return None; + } + + let (status, out) = self.scrd_command(&crate::scrd::scrd_context_create_tracked(user.value()))?; + if status != 0 || out.len() < crate::scrd::SCRD_ACM_HANDLE_LEN { + return None; + } + let mut acm_handle = [0u8; crate::scrd::SCRD_ACM_HANDLE_LEN]; + acm_handle.copy_from_slice(&out[..crate::scrd::SCRD_ACM_HANDLE_LEN]); + + let (status, _) = self.scrd_command(&crate::scrd::scrd_context_externalize(&acm_handle))?; + if status != 0 { + return None; + } + + let out = self.sks_send(self.sks_req_verify_secret(special, &secret, &acm_handle))?; + if out.reply.status != 0 { + return None; + } + + let (status, out) = self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle))?; + let satisfied = out.len() >= 4 + && u32::from_le_bytes([out[0], out[1], out[2], out[3]]) != 0; + if status != 0 || !satisfied { + return None; + } + + Some(acm_handle) + } + + fn establish_scrd_match_context(&self, user: crate::sbio::UserId) { + if !self.scrd_ready() { + return; + } + let Some(du) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) else { + return; + }; + let special = du.special_handle(); + let stored = match keybag::read(keybag::Slot::Identity) { + Ok(keybag::State::Present(s)) => s, + _ => { + return; + } + }; + + let _ = (|| -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { + let (status, _) = self.scrd_command(&crate::scrd::scrd_initialize())?; + if status != 0 { + return None; + } + let (status, out) = + self.scrd_command(&crate::scrd::scrd_context_create_tracked(user.value()))?; + if status != 0 || out.len() < crate::scrd::SCRD_ACM_HANDLE_LEN { + return None; + } + let mut acm_handle = [0u8; crate::scrd::SCRD_ACM_HANDLE_LEN]; + acm_handle.copy_from_slice(&out[..crate::scrd::SCRD_ACM_HANDLE_LEN]); + let (status, _) = self.scrd_command(&crate::scrd::scrd_context_externalize(&acm_handle))?; + if status != 0 { + return None; + } + let out = self.sks_send(self.sks_req_verify_secret(special, stored.secret(), &acm_handle))?; + if out.reply.status != 0 { + return None; + } + self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle))?; + Some(acm_handle) + })(); + } + + + fn hwrng_fill(&self, buf: *mut u8, max: usize, wait: bool) -> c_int { + if !wait { + return 0; + } + + let words = core::cmp::min(max / 4, RNG_MAX_WORDS_PER_READ); + let mut written: usize = 0; + let mut failure: Option = None; + + for i in 0..words { + if self.rng_shutdown.load(Relaxed) { + failure = Some(ENODEV); + break; + } + match self.get_entropy_word() { + Ok(value) => { + // SAFETY: the core guarantees `buf` is valid for `max` + // bytes and aligned for any type, and `i < max / 4`. + unsafe { buf.add(i * 4).cast::().write_unaligned(value) }; + written += 4; + } + Err(e) => { + failure = Some(e); + break; + } + } + } + + if written > 0 { + return written as c_int; + } + + // must return a negative errno, not 0: a blocking reader would spin on 0 + let e = failure.unwrap_or(EIO); + let n = self.rng_failures.load(Relaxed).wrapping_add(1); + self.rng_failures.store(n, Relaxed); + if n == 1 { + dev_err!( + self.dev, + "hwrng: read produced nothing ({:?}); further failures will not be logged\n", + e + ); + } + e.to_errno() + } + + fn register_hwrng(&self) -> Result<()> { + let mut guard = self.rng.lock(); + if guard.is_some() { + return Ok(()); + } + + let mut handle = hwrng::HwRngHandle::new()?; + let ctx = core::ptr::from_ref(self).cast_mut().cast::(); + + // SAFETY: `ctx` is this `SepData`, which is kept alive by the `Arc` held + // in the driver's private data. `remove()` sets `rng_shutdown`, wakes + // any blocked draw, and unregisters, and `hwrng_unregister()` blocks + // until the core has finished with the device, all before that `Arc` + // can be dropped. So the pointer cannot outlive the registration. + unsafe { handle.register(ctx, hwrng_read_trampoline) }?; + + *guard = Some(handle); + Ok(()) + } + + + fn drain(&self) -> u32 { + let mut n: u32 = 0; + while let Some(msg) = self.rx.pop() { + self.dispatch(msg); + n += 1; + } + if n > 0 { + // single writer, single instance: a load/store pair suffices + let total = self.rx_count.load(Relaxed).wrapping_add(u64::from(n)); + self.rx_count.store(total, Relaxed); + } + n + } + + + fn arm_settle(this: &Arc) { + let delay = time::msecs_to_jiffies(SETTLE_MS); + let _ = workqueue::system() + .enqueue_delayed::, SETTLE_WORK_ID>(this.clone(), delay); + } + + fn settle_tick(this: &Arc) { + match this.phase.load(Relaxed) { + PHASE_ATTACH => Self::tick_attach(this), + PHASE_EXCHANGE => Self::tick_exchange(this), + _ => {} + } + } + + fn tick_attach(this: &Arc) { + let now = this.rx_count.load(Relaxed); + + // HW: the SEP takes ~265 ms to answer a registration + if now == 0 { + let ticks = this.settle_idle_ticks.load(Relaxed).wrapping_add(1); + this.settle_idle_ticks.store(ticks, Relaxed); + if ticks.saturating_mul(u64::from(SETTLE_MS)) < u64::from(FIRST_RESPONSE_MS) { + Self::arm_settle(this); + return; + } + } + + if now != this.settle_mark.load(Relaxed) { + this.settle_mark.store(now, Relaxed); + Self::arm_settle(this); + return; + } + + if this.endpoint_count() == 0 { + this.phase.store(PHASE_READY, Relaxed); + return; + } + + this.prepare_os_uuid(); + + this.control_survey(); + + if let Err(e) = this.register_ool(&this.ool_xarm) { + dev_err!( + this.dev, + "xarm: out-of-line buffer registration failed ({:?}); the persistent-state exchange cannot run\n", + e + ); + this.phase.store(PHASE_READY, Relaxed); + return; + } + + this.phase.store(PHASE_EXCHANGE, Relaxed); + this.settle_mark.store(this.rx_count.load(Relaxed), Relaxed); + this.settle_idle_ticks.store(0, Relaxed); + this.release_deferred_query(); + Self::arm_settle(this); + } + + fn tick_exchange(this: &Arc) { + let now = this.rx_count.load(Relaxed); + + if now != this.settle_mark.load(Relaxed) { + this.settle_mark.store(now, Relaxed); + this.settle_idle_ticks.store(0, Relaxed); + Self::arm_settle(this); + return; + } + + let endpoints = this.endpoint_count(); + if endpoints <= ENDPOINTS_BEFORE_EXCHANGE { + let ticks = this.settle_idle_ticks.load(Relaxed).wrapping_add(1); + this.settle_idle_ticks.store(ticks, Relaxed); + if ticks.saturating_mul(u64::from(SETTLE_MS)) < u64::from(EXCHANGE_TIMEOUT_MS) { + Self::arm_settle(this); + return; + } + } + + this.phase.store(PHASE_READY, Relaxed); + this.run_bringup(); + } + + fn endpoint_present(&self, id: u8) -> bool { + self.endpoints.lock().eps.iter().any(|e| e.id == id) + } + + fn endpoint_count(&self) -> usize { + self.endpoints.lock().eps.len() + } + + fn dispatch(&self, msg: Message) { + let f = proto::decode(&msg); + + match f.ep { + proto::EP_DISCOVER => self.on_discovery(msg, f), + + proto::EP_SHMEM => {}, + + proto::EP_CONTROL => self.on_control(msg, f), + + xarm::EP_XARM => self.on_xarm(msg), + + proto::EP_SBIO => self.on_sbio(msg), + + proto::EP_SKS => self.on_sks(msg), + + // xars: only 0x08 and 0x04 are sent; other opcodes are destructive + xarm::EP_XARS => self.on_xars(msg), + + proto::EP_SCRD => self.on_scrd(msg), + + proto::EP_BOOT => dev_warn!( + self.dev, + "unexpected message from boot endpoint 0xff: type 0x{:02x} param 0x{:02x} msg0 {:#018x} msg1 {:#010x}\n", + f.ty, + f.param, + msg.msg0, + msg.msg1 + ), + + _ep => {}, + } + } + + fn on_control(&self, msg: Message, f: proto::Fields) { + if f.ty != proto::CONTROL_REPLY_TYPE { + self.control.lock().note_unsolicited(); + return; + } + + let reply = proto::ControlReply::from_message(&msg); + let disposition = self.control.lock().deliver(reply); + + match disposition { + control::Delivery::Entropy => { + self.control_wq.notify_all(); + } + control::Delivery::Matched => self.control_wq.notify_all(), + control::Delivery::Unmatched => {} + } + } + + fn on_discovery(&self, msg: Message, f: proto::Fields) { + let mut table = self.endpoints.lock(); + table.discovery_msgs += 1; + + let id = f.param; + + let idx = match table.slot(id) { + Ok(i) => i, + Err(_) => { + return; + } + }; + + match f.ty { + proto::DISCOVER_TYPE_DESCRIPTOR => { + let cc = proto::fourcc(&msg); + let e = &mut table.eps[idx]; + e.have_descriptor = true; + e.descriptor_msg0 = msg.msg0; + e.descriptor_msg1 = msg.msg1; + e.fourcc = cc; + table.dirty = true; + } + proto::DISCOVER_TYPE_CONFIG => { + let e = &mut table.eps[idx]; + e.have_config = true; + e.config_msg0 = msg.msg0; + e.config_msg1 = msg.msg1; + table.dirty = true; + } + _ => { + table.unknown_types += 1; + } + } + } + + + fn remove(&self) { + // first: resets the keyring static calls before freeing, so no keyctl op hits freed data + trusted::unregister(); + + let dev = self.bio_dev.lock().take(); + if dev.is_some() { + bio::release(&mut self.bio_session.lock()); + sensor::power(false); + sensor::unregister_driver(); + } + drop(dev); + + self.rng_shutdown.store(true, Relaxed); + self.control_wq.notify_all(); + if let Some(mut handle) = self.rng.lock().take() { + handle.unregister(); + } + + // stop the mailbox last: no callbacks after this + *self.mbox.lock() = None; + + let _ = self.store.lock().take(); + + for slot in [&self.ool_xarm, &self.ool_sbio, &self.ool_sks] { + if let Some(buffers) = slot.lock().take() { + if buffers.registered { + core::mem::forget(buffers); + } + } + } + + if let Some(ring) = self.dma_ring.lock().take() { + if self.sbio_ready.load(Relaxed) { + core::mem::forget(ring); + } + } + + let mut guard = self.shmem.lock(); + if let Some(buf) = guard.take() { + if self.registered.load(Relaxed) { + core::mem::forget(buf); + } else { + drop(buf); + } + } + } +} + +/// # Safety +/// +/// Each is called only by `bio_shim.c`, with the `*const SepData` handed to +/// `BioChardev::register`, which stays valid for the whole registered window. +unsafe extern "C" fn bio_open_trampoline(ctx: *mut c_void) -> c_int { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + match this.bio_open() { + Ok(()) => 0, + Err(e) => e.to_errno(), + } +} + +/// # Safety +unsafe extern "C" fn bio_release_trampoline(ctx: *mut c_void) { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + this.bio_release(); +} + +/// # Safety +unsafe extern "C" fn bio_ioctl_trampoline(ctx: *mut c_void, cmd: c_uint, arg: c_ulong) -> c_long { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + let handled = match this.bio_ioctl(cmd, arg) { + Ok(h) => h, + Err(e) => return e.to_errno() as c_long, + }; + + if handled.start_verify { + // SAFETY: as for `start_enrol` below: same pointer, same guarantee. + let borrow = unsafe { kernel::sync::ArcBorrow::::from_raw(ctx.cast::()) }; + SepData::queue_verify(borrow.into()); + } + + if handled.start_enrol { + // SAFETY: `ctx` is the pointer the driver's own `Arc` was made + // from and that `Arc` outlives every callback; see the contract on the + // registration. `ArcBorrow` is what turns that guarantee into an owned + // reference with the refcount incremented rather than a second owner of + // the same count. + let borrow = unsafe { kernel::sync::ArcBorrow::::from_raw(ctx.cast::()) }; + SepData::queue_enrolment(borrow.into()); + } + handled.ret as c_long +} + +/// # Safety +unsafe extern "C" fn bio_ready_trampoline(ctx: *mut c_void) -> c_int { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + c_int::from(this.bio_ready()) +} + +/// # Safety +/// +/// Only ever called by the hwrng core, through `hwrng_shim.c`. `ctx` is the +/// `*const SepData` handed to `HwRngHandle::register`, which stays valid for +/// the whole registered window (see the ordering note in `remove()`), and +/// `data` is valid for `max` bytes. +unsafe extern "C" fn hwrng_read_trampoline( + ctx: *mut c_void, + data: *mut c_void, + max: usize, + wait: bool, +) -> c_int { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + this.hwrng_fill(data.cast::(), max, wait) +} + +impl MailCallback for SepData { + type Data = Arc; + + fn recv_message(data: ::Borrowed<'_>, msg: Message) { + if !data.rx.push(msg) && data.rx.dropped() == 1 { + dev_err!( + data.dev, + "receive ring full; dropping messages (first was msg0 {:#018x})\n", + msg.msg0 + ); + } + + let this: Arc = data.into(); + let _ = workqueue::system().enqueue::, 0>(this); + } +} + +impl WorkItem for SepData { + type Pointer = Arc; + + fn run(this: Arc) { + if this.drain() > 0 { + SepData::arm_settle(&this); + } + } +} + +impl WorkItem for SepData { + type Pointer = Arc; + + fn run(this: Arc) { + SepData::settle_tick(&this); + } +} + +impl WorkItem for SepData { + type Pointer = Arc; + + fn run(this: Arc) { + this.run_enrolment(); + } +} + +impl WorkItem for SepData { + type Pointer = Arc; + + fn run(this: Arc) { + this.run_verify(); + } +} + + +struct SepDriver(Arc); + +const OF_TABLE: kernel::device_id::IdArray = + kernel::device_id::IdArray::new([(of::DeviceId::new(c"apple,sep"), ())]); + +impl platform::Driver for SepDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe( + pdev: &platform::Device, + _info: Option<&()>, + ) -> impl PinInit { + let dev: &device::Device = pdev.as_ref(); + let sep_node = dt::DtNode::of_device(dev).ok_or(ENODEV)?; + + if dt::registration_already_sent(&sep_node) { + dev_err!( + dev, + "the shared-memory registration was already sent on this boot (marker property present). It is one-shot per AP reset and cannot be repeated or withdrawn: reboot to attach again. Not probing.\n" + ); + return Err(EBUSY); + } + + let data = SepData::new(pdev)?; + + *data.mbox.lock() = Some(Mailbox::new_byname(dev, c"mbox", data.clone())?); + + if let Err(e) = data.register_bio() { + dev_err!( + dev, + "could not register /dev/{}: {:?}\n", + bio::DEVICE_NAME, + e + ); + } + + data.attach(&sep_node)?; + + data.attach_sensor(); + + if let Err(e) = trusted::register(data.clone()) { + dev_warn!(data.dev, "trusted-keys: registration failed ({:?})\n", e); + } + + if data.registered.load(Relaxed) { + SepData::arm_settle(&data); + } + + Ok(SepDriver(data)) + } +} + +impl Drop for SepDriver { + fn drop(&mut self) { + self.0.remove(); + } +} + + +#[pin_data] +struct SepModule { + // field-init order enables the DT nodes before the driver registers; a failure here aborts module init + _dt: (), + + #[pin] + _driver: driver::Registration>, +} + +impl kernel::InPlaceModule for SepModule { + fn init(module: &'static ThisModule) -> impl PinInit { + try_pin_init!(Self { + _dt: dt::enable_sep_and_dart()?, + + _driver <- driver::Registration::new( + ::NAME, + module, + ), + }) } } -module_platform_driver! { - type: SepDriver, +module! { + type: SepModule, name: "apple_sep", - description: "Secure enclave processor stub driver", + description: "Apple SEP coprocessor: warm attach, endpoint discovery and tag-correlated control endpoint", license: "Dual MIT/GPL", } diff --git a/drivers/soc/apple/sha_shim.c b/drivers/soc/apple/sha_shim.c new file mode 100644 index 0000000000000..2e823df021dde --- /dev/null +++ b/drivers/soc/apple/sha_shim.c @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* SHA-256 shim: hashes two caller-chosen segments; no protocol logic here. */ + +#include +#include +#include + +#include "shim.h" + +/* + * Hashes `a` then `b`, writes 32 bytes to `out`. Allocates a transform per call + * and may sleep. Returns 0 or a negative errno; `out` invalid on failure. + */ +int sep_sha256(const void *a, size_t alen, const void *b, size_t blen, + unsigned char *out) +{ + struct crypto_shash *tfm; + int rc; + + tfm = crypto_alloc_shash("sha256", 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + { + SHASH_DESC_ON_STACK(desc, tfm); + + desc->tfm = tfm; + + rc = crypto_shash_init(desc); + if (!rc && alen) + rc = crypto_shash_update(desc, a, alen); + if (!rc && blen) + rc = crypto_shash_update(desc, b, blen); + if (!rc) + rc = crypto_shash_final(desc, out); + + /* The digest is over a request that may contain a secret. */ + shash_desc_zero(desc); + } + + crypto_free_shash(tfm); + return rc; +} diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h new file mode 100644 index 0000000000000..2abb13765bad7 --- /dev/null +++ b/drivers/soc/apple/shim.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* + * Apple SEP driver — private declarations for the C shim. + * + * Glue between the Rust driver and kernel interfaces whose ABI is awkward or + * unsafe to restate in Rust: `struct hwrng`, whose layout moves with + * CONFIG_LOCKDEP; the file interface, whose O_* flags are macro-derived and so + * absent from the Rust bindings; and the crypto API, whose shash descriptor is + * sized by a macro at its call site. + * + * No protocol logic lives in the shim. Included only by the shim translation + * units. + */ + +#ifndef SEP_SHIM_H +#define SEP_SHIM_H + +#include + +/* -- hwrng_shim.c ------------------------------------------------------- */ + +void *sep_hwrng_alloc(void); +void sep_hwrng_free(void *mem); +int sep_hwrng_register(void *mem, const char *name, + unsigned short quality, void *ctx, + int (*read)(void *ctx, void *data, size_t max, + bool wait)); +void sep_hwrng_unregister(void *mem); + +/* -- store_shim.c ------------------------------------------------------- */ + +void *sep_store_open(const char *path); +void *sep_store_open_trunc(const char *path); +void *sep_store_open_ro(const char *path); +void sep_store_close(void *handle); +long long sep_store_size(void *handle); +long sep_store_read(void *handle, long long off, void *buf, size_t len); +long sep_store_write(void *handle, long long off, const void *buf, + size_t len); +int sep_store_sync(void *handle); + +/* -- sha_shim.c --------------------------------------------------------- */ + +int sep_sha256(const void *a, size_t alen, const void *b, size_t blen, + unsigned char *out); + +/* -- crypto_shim.c ------------------------------------------------------ */ + +/* HMAC-SHA256 over one message, 32 bytes out. */ +int sep_hmac_sha256(const void *key, size_t keylen, + const void *data, size_t datalen, u8 *out); + +/* + * AES-256-GCM in place over `buf` = `aadlen` bytes of AAD followed by the + * payload. Returns -EBADMSG on auth failure, which the caller must treat as + * fatal. + */ +int sep_gcm(int encrypt, const void *key, size_t keylen, + const void *iv, size_t ivlen, size_t aadlen, + void *buf, size_t buflen, size_t datalen); + +/* + * Fill `buf` from the kernel CSPRNG, seeded, or fail. For key-bag secrets only; + * anything measuring the enclave's own entropy (0x59 device-key probe, + * /dev/hwrng) stays on SEP. + */ +int sep_random_bytes(void *buf, size_t len); + +/* -- p256_shim.c ------------------------------------------------------- */ + +/* + * ECIES sender key agreement for the ref-key (op 0x22) SE-seal, with a fresh + * ephemeral P-256 keypair. `peer_pub_be` is the recipient's 64-byte public + * point (0x04 prefix stripped), `eph_pub_be_out` receives the 64-byte ephemeral + * public point, `shared_be_out` the 32-byte shared secret. All big-endian. + */ +int sep_p256_sender(const void *peer_pub_be, void *eph_pub_be_out, + void *shared_be_out); + +/* -- sensor_shim.c ------------------------------------------------------ */ + +int sep_sensor_register(void); +void sep_sensor_unregister(void); +int sep_sensor_bound(void); +int sep_sensor_power_line(void); +int sep_sensor_cs_timing_mode(void); +int sep_sensor_power_cycle(void); +int sep_sensor_power_source(void); +int sep_sensor_power(int on); +int sep_sensor_xfer(const void *tx, void *rx, size_t len); +int sep_sensor_xfer_tx(const void *tx, size_t len); +int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len); + +/* -- bio_shim.c --------------------------------------------------------- */ + +void *sep_bio_register(const char *name, unsigned short mode, void *ctx, + int (*f_open)(void *), + void (*f_release)(void *), + long (*f_ioctl)(void *, unsigned int, unsigned long), + int (*f_ready)(void *)); +void sep_bio_unregister(void *dev); +void sep_bio_wake(void *dev); +int sep_bio_capable_admin(void); +__u64 sep_bio_monotonic_ns(void); +__u64 sep_bio_boottime_ns(void); + +/* -- trusted_shim.c ---------------------------------------------------- */ + +/* + * The kernel trusted-key source (keyctl). The shim owns the framework structs + * and registration; the sealing policy is in Rust (trusted.rs), reached through + * the callbacks passed to sep_tk_register(). The payload struct is only + * forward-declared here since these prototypes take it by pointer; trusted_shim.c + * pulls in for the definition. + */ +struct trusted_key_payload; + +typedef int (*sep_tk_init_fn)(void); +typedef int (*sep_tk_seal_fn)(struct trusted_key_payload *p, char *datablob); +typedef int (*sep_tk_unseal_fn)(struct trusted_key_payload *p, char *datablob); +typedef int (*sep_tk_random_fn)(unsigned char *key, size_t key_len); +typedef void (*sep_tk_exit_fn)(void); + +int sep_tk_register(sep_tk_init_fn init, sep_tk_seal_fn seal, + sep_tk_unseal_fn unseal, sep_tk_random_fn random, + sep_tk_exit_fn exit); +void sep_tk_unregister(void); + +int sep_tk_register_key_type(void); +void sep_tk_unregister_key_type(void); + +size_t sep_tk_max_key_size(void); +size_t sep_tk_max_blob_size(void); + +unsigned char *sep_tk_key_ptr(struct trusted_key_payload *p); +unsigned int sep_tk_key_len(const struct trusted_key_payload *p); +void sep_tk_set_key_len(struct trusted_key_payload *p, unsigned int n); + +unsigned char *sep_tk_blob_ptr(struct trusted_key_payload *p); +unsigned int sep_tk_blob_len(const struct trusted_key_payload *p); +void sep_tk_set_blob_len(struct trusted_key_payload *p, unsigned int n); + +#endif /* SEP_SHIM_H */ diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs new file mode 100644 index 0000000000000..2968a04943e99 --- /dev/null +++ b/drivers/soc/apple/shim.rs @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Rust side of `store_shim.c`: the backing-store file. + +use kernel::prelude::*; + +extern "C" { + fn sep_store_open(path: *const c_char) -> *mut c_void; + fn sep_store_open_trunc(path: *const c_char) -> *mut c_void; + fn sep_store_open_ro(path: *const c_char) -> *mut c_void; + fn sep_store_close(handle: *mut c_void); + fn sep_store_size(handle: *mut c_void) -> i64; + fn sep_store_read(handle: *mut c_void, off: i64, buf: *mut c_void, len: usize) + -> c_long; + fn sep_store_write( + handle: *mut c_void, + off: i64, + buf: *const c_void, + len: usize, + ) -> c_long; + fn sep_store_sync(handle: *mut c_void) -> c_int; +} + +/// Backing-store file handle. +/// +/// # Invariants +/// `handle` is a live `struct file *` from `sep_store_open`. +pub(crate) struct StoreFile { + handle: *mut c_void, +} + +// SAFETY: a `struct file *` has no thread affinity; every access goes through +// the C shim, and the driver keeps this behind a mutex. +unsafe impl Send for StoreFile {} + +fn result_of(ret: c_long) -> Result { + if ret < 0 { + Err(Error::from_errno(ret as c_int)) + } else { + Ok(ret as usize) + } +} + +impl StoreFile { + pub(crate) fn open(path: &CStr) -> Result { + // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. + let handle = unsafe { sep_store_open(path.as_char_ptr()) }; + if handle.is_null() { + return Err(ENOENT); + } + Ok(StoreFile { handle }) + } + + pub(crate) fn open_trunc(path: &CStr) -> Result { + // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. + let handle = unsafe { sep_store_open_trunc(path.as_char_ptr()) }; + if handle.is_null() { + return Err(ENOENT); + } + Ok(StoreFile { handle }) + } + + pub(crate) fn open_readonly(path: &CStr) -> Result { + // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. + let handle = unsafe { sep_store_open_ro(path.as_char_ptr()) }; + if handle.is_null() { + return Err(ENOENT); + } + Ok(StoreFile { handle }) + } + + pub(crate) fn size(&self) -> Result { + // SAFETY: `handle` is live per the type invariant. + let n = unsafe { sep_store_size(self.handle) }; + if n < 0 { + return Err(EIO); + } + Ok(n as u64) + } + + pub(crate) fn read_exact(&self, off: u64, buf: &mut [u8]) -> Result<()> { + let mut done = 0usize; + while done < buf.len() { + // SAFETY: `handle` is live; the pointer and length describe the + // remaining tail of `buf`, which we own mutably. + let n = result_of(unsafe { + sep_store_read( + self.handle, + (off + done as u64) as i64, + buf[done..].as_mut_ptr().cast::(), + buf.len() - done, + ) + })?; + if n == 0 { + return Err(EIO); + } + done += n; + } + Ok(()) + } + + pub(crate) fn write_all(&self, off: u64, buf: &[u8]) -> Result<()> { + let mut done = 0usize; + while done < buf.len() { + // SAFETY: as above, for a shared borrow. + let n = result_of(unsafe { + sep_store_write( + self.handle, + (off + done as u64) as i64, + buf[done..].as_ptr().cast::(), + buf.len() - done, + ) + })?; + if n == 0 { + return Err(EIO); + } + done += n; + } + Ok(()) + } + + pub(crate) fn sync(&self) -> Result<()> { + // SAFETY: `handle` is live per the type invariant. + kernel::error::to_result(unsafe { sep_store_sync(self.handle) }) + } +} + +impl Drop for StoreFile { + fn drop(&mut self) { + // SAFETY: `handle` is live per the type invariant and is not used again. + unsafe { sep_store_close(self.handle) }; + } +} + +extern "C" { + fn sep_bio_register( + name: *const c_char, + mode: u16, + ctx: *mut c_void, + f_open: Option c_int>, + f_release: Option, + f_ioctl: Option c_long>, + f_ready: Option c_int>, + ) -> *mut c_void; + fn sep_bio_unregister(dev: *mut c_void); + fn sep_bio_wake(dev: *mut c_void); + fn sep_bio_capable_admin() -> c_int; + fn sep_bio_monotonic_ns() -> u64; + fn sep_bio_boottime_ns() -> u64; +} + +extern "C" { +} + +pub(crate) fn capable_admin() -> bool { + // SAFETY: no preconditions; reads the current task's credentials. + unsafe { sep_bio_capable_admin() != 0 } +} + +pub(crate) fn monotonic_ns() -> u64 { + // SAFETY: no preconditions. + unsafe { sep_bio_monotonic_ns() } +} + +/// `CLOCK_BOOTTIME`; differs from monotonic by suspend time — how "tokens don't +/// survive a suspend" is detected. +pub(crate) fn boottime_ns() -> u64 { + // SAFETY: no preconditions. + unsafe { sep_bio_boottime_ns() } +} + +/// The registered character device. +/// +/// # Invariants +/// `dev` is a live registration from `sep_bio_register`. +pub(crate) struct BioChardev { + dev: *mut c_void, +} + +// SAFETY: the registration has no thread affinity; every access goes through +// the C shim, which locks internally. +unsafe impl Send for BioChardev {} + +impl BioChardev { + /// Registers `/dev/`. + /// + /// # Safety + /// `ctx` must stay valid and safe to pass to the four callbacks until this + /// is dropped; the callbacks may run on any task at any time. + pub(crate) unsafe fn register( + name: &'static CStr, + mode: u16, + ctx: *mut c_void, + f_open: unsafe extern "C" fn(*mut c_void) -> c_int, + f_release: unsafe extern "C" fn(*mut c_void), + f_ioctl: unsafe extern "C" fn(*mut c_void, c_uint, c_ulong) -> c_long, + f_ready: unsafe extern "C" fn(*mut c_void) -> c_int, + ) -> Result { + // SAFETY: `name` is a static NUL-terminated string that outlives the + // registration, and the caller guarantees the same of `ctx`. + let dev = unsafe { + sep_bio_register( + name.as_char_ptr(), + mode, + ctx, + Some(f_open), + Some(f_release), + Some(f_ioctl), + Some(f_ready), + ) + }; + if dev.is_null() { + return Err(ENODEV); + } + Ok(BioChardev { dev }) + } + + pub(crate) fn wake(&self) { + // SAFETY: `dev` is live per the type invariant. + unsafe { sep_bio_wake(self.dev) }; + } +} + +impl Drop for BioChardev { + fn drop(&mut self) { + // SAFETY: `dev` is live per the type invariant and not used again. + // Deregistration waits for open files, so no callback runs once this returns. + unsafe { sep_bio_unregister(self.dev) }; + } +} diff --git a/drivers/soc/apple/shmem.rs b/drivers/soc/apple/shmem.rs new file mode 100644 index 0000000000000..134585398c8b4 --- /dev/null +++ b/drivers/soc/apple/shmem.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +use kernel::device; +use kernel::dma; +use kernel::platform; +use kernel::prelude::*; + +pub(crate) const SHMEM_SIZE: usize = 0x40000; + +const ENTRY_SIZE: usize = 16; +const ENTRY_OFF_FOURCC: usize = 0; +const ENTRY_OFF_SIZE: usize = 4; +const ENTRY_OFF_OFFSET: usize = 8; + +// Buffer-format constant, not PAGE_SIZE despite the coincidence. +const PAYLOAD_BASE: usize = 0x4000; +const PAYLOAD_ALIGN: usize = 0x4000; + +const CINP_MIN_SIZE: usize = 0x8000; + +const CINP_PAYLOAD: [u8; 1] = [0]; + +// Wire byte order, not byte-reversed; llun is the terminator spelling. +const FOURCC_CINP: &[u8; 4] = b"CINP"; +const FOURCC_OPLA: &[u8; 4] = b"OPLA"; +const FOURCC_IPIS: &[u8; 4] = b"IPIS"; +const FOURCC_TERM: &[u8; 4] = b"llun"; + +const PROP_LOCAL_POLICY: &CStr = c"local-policy-manifest"; +const PROP_IBOOT: &CStr = c"iboot-manifest"; + +pub(crate) type ShMem = dma::Coherent<[u8]>; + +#[derive(Clone, Copy)] +pub(crate) struct Region { + pub(crate) offset: usize, + // Entry size field = allocation size, not payload length. + pub(crate) size: usize, + pub(crate) payload_len: usize, +} + +impl Region { + fn place(offset: usize, payload_len: usize, min_size: usize) -> Region { + let mut size = align_up(payload_len + 4, PAYLOAD_ALIGN); + if size < min_size { + size = min_size; + } + Region { + offset, + size, + payload_len, + } + } + + fn end(&self) -> usize { + self.offset + self.size + } +} + +pub(crate) struct Shmem { + pub(crate) buf: ShMem, +} + +const fn align_up(v: usize, a: usize) -> usize { + (v + a - 1) & !(a - 1) +} + +fn write_at(buf: &mut ShMem, off: usize, src: &[u8]) -> Result<()> { + let end = off.checked_add(src.len()).ok_or(EINVAL)?; + if end > SHMEM_SIZE { + return Err(EINVAL); + } + // SAFETY: runs in probe before the SEP is told the buffer exists, and probe + // is the only writer, so nothing else accesses it. + unsafe { + buf.as_mut()[off..end].copy_from_slice(src); + } + Ok(()) +} + +fn write_entry( + buf: &mut ShMem, + index: usize, + fourcc: &[u8; 4], + size: usize, + offset: usize, +) -> Result<()> { + let base = index * ENTRY_SIZE; + if base + ENTRY_SIZE > PAYLOAD_BASE { + return Err(EINVAL); + } + write_at(buf, base + ENTRY_OFF_FOURCC, fourcc)?; + let size32: u32 = size.try_into().map_err(|_| EINVAL)?; + write_at(buf, base + ENTRY_OFF_SIZE, &size32.to_le_bytes())?; + // u32 offset at byte 8 (written as u64 with zero pad), never byte 12. + let offset64: u64 = offset.try_into().map_err(|_| EINVAL)?; + write_at(buf, base + ENTRY_OFF_OFFSET, &offset64.to_le_bytes())?; + Ok(()) +} + +fn read_blob(dev: &device::Device, name: &CStr) -> Result> { + let fwnode = dev.fwnode().ok_or(EIO)?; + let len = fwnode.property_count_elem::(name)?; + if len == 0 { + return Err(ENODATA); + } + fwnode + .property_read_array_vec::(name, len)? + .required_by(dev) +} + +fn read_manifest(dev: &device::Device, name: &CStr) -> Result> { + match read_blob(dev, name) { + Ok(v) => Ok(v), + Err(e) => { + dev_err!( + dev, + "refusing to attach: device-tree property '{}' missing or unreadable ({:?}); a CINP-only table would burn the one-shot registration and fault\n", + name, + e + ); + Err(e) + } + } +} + +fn verify_layout( + dev: &device::Device, + cinp: &Region, + opla: &Region, + ipis: &Region, + used: usize, +) -> Result<()> { + let regions = [ + (FOURCC_CINP, cinp), + (FOURCC_OPLA, opla), + (FOURCC_IPIS, ipis), + ]; + + for (name, r) in regions { + let bad = r.offset < PAYLOAD_BASE + || r.offset % PAYLOAD_ALIGN != 0 + || r.size % PAYLOAD_ALIGN != 0 + || r.size < r.payload_len + 4 + || r.end() > SHMEM_SIZE; + if bad { + dev_err!( + dev, + "refusing to attach: item '{}' region is malformed (offset 0x{:x}, size 0x{:x}, payload {} bytes)\n", + core::str::from_utf8(name).unwrap_or("????"), + r.offset, + r.size, + r.payload_len + ); + return Err(EINVAL); + } + } + + if cinp.offset != PAYLOAD_BASE + || opla.offset != cinp.end() + || ipis.offset != opla.end() + || used != ipis.end() + { + dev_err!( + dev, + "refusing to attach: regions do not tile from 0x{:x} (CINP 0x{:x}+0x{:x}, OPLA 0x{:x}+0x{:x}, IPIS 0x{:x}+0x{:x}, used 0x{:x})\n", + PAYLOAD_BASE, + cinp.offset, + cinp.size, + opla.offset, + opla.size, + ipis.offset, + ipis.size, + used + ); + return Err(EINVAL); + } + + if cinp.size < CINP_MIN_SIZE { + return Err(EINVAL); + } + + if 4 * ENTRY_SIZE > PAYLOAD_BASE || used > SHMEM_SIZE { + return Err(ENOSPC); + } + + Ok(()) +} + +pub(crate) fn build(pdev: &platform::Device) -> Result { + let dev: &device::Device = pdev.as_ref(); + + // Read manifests before allocating: a CINP-only registration would burn the one-shot and fault. + let opla_blob = read_manifest(dev, PROP_LOCAL_POLICY)?; + let ipis_blob = read_manifest(dev, PROP_IBOOT)?; + + let cinp = Region::place(PAYLOAD_BASE, CINP_PAYLOAD.len(), CINP_MIN_SIZE); + let opla = Region::place(cinp.end(), opla_blob.len(), 0); + let ipis = Region::place(opla.end(), ipis_blob.len(), 0); + let used = ipis.end(); + + verify_layout(dev, &cinp, &opla, &ipis, used)?; + + let mut buf = dma::Coherent::::zeroed_slice(dev, SHMEM_SIZE, GFP_KERNEL)?; + + // Payloads before entries: a failure leaves an all-zero table, not a valid-looking one. + write_at( + &mut buf, + cinp.offset, + &(cinp.payload_len as u32).to_le_bytes(), + )?; + write_at(&mut buf, cinp.offset + 4, &CINP_PAYLOAD)?; + + write_at( + &mut buf, + opla.offset, + &(opla.payload_len as u32).to_le_bytes(), + )?; + write_at(&mut buf, opla.offset + 4, &opla_blob)?; + + write_at( + &mut buf, + ipis.offset, + &(ipis.payload_len as u32).to_le_bytes(), + )?; + write_at(&mut buf, ipis.offset + 4, &ipis_blob)?; + + write_entry(&mut buf, 0, FOURCC_CINP, cinp.size, cinp.offset)?; + write_entry(&mut buf, 1, FOURCC_OPLA, opla.size, opla.offset)?; + write_entry(&mut buf, 2, FOURCC_IPIS, ipis.size, ipis.offset)?; + write_entry(&mut buf, 3, FOURCC_TERM, 0, 0)?; + + Ok(Shmem { buf }) +} diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs new file mode 100644 index 0000000000000..f30f4870db076 --- /dev/null +++ b/drivers/soc/apple/sks.rs @@ -0,0 +1,1115 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! SEP key store (SKS, endpoint `0x12`): key-bag and key-management request/reply +//! framing, DER-imaged request builders, and lock-state control. + +#![allow(dead_code)] + +use super::*; +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; +use crate::proto::*; + +impl SepData { + pub(crate) fn on_sks(&self, msg: Message) { + let r = crate::sks::decode_sks(&msg); + let mut probe = self.sks_probe.lock(); + if probe.active { + if probe.captured.len() < SKS_MAX_CAPTURE { + let _ = probe.captured.push(msg, GFP_KERNEL); + } + drop(probe); + self.sks_wq.notify_all(); + } else { + probe.unsolicited = probe.unsolicited.wrapping_add(1); + let matched = probe + .abandoned + .iter() + .flatten() + .find(|a| a.selector == r.selector && a.seq == r.seq) + .copied(); + drop(probe); + if let Some(a) = matched { + self.sks_wedged.store(0, Relaxed); + dev_warn!(self.dev, "sks: late answer to {} after {} ms; wedge lifted\n", a.label, a.waited_ms); + } + } + } + + fn sks_note_abandoned(&self, selector: u8, seq: u8, label: &'static CStr, waited_ms: u64) { + let mut probe = self.sks_probe.lock(); + let slot = probe.abandoned_next; + probe.abandoned[slot] = Some(Abandoned { + selector, + seq, + label, + waited_ms, + }); + probe.abandoned_next = (slot + 1) % SKS_MAX_ABANDONED; + } + + fn sks_read_buffer(&self, inbound: bool, len: usize) -> Result> { + let guard = self.ool_sks.lock(); + let pair = guard.as_ref().ok_or(ENODEV)?; + let src = if inbound { + &pair.inbound + } else { + &pair.outbound + }; + if len > pair.allocated { + return Err(EINVAL); + } + let mut out = KVec::new(); + // SAFETY: the enclave writes this buffer then signals the mailbox, so it is + // quiescent when we read it; the driver is the only other accessor. + out.extend_from_slice(unsafe { &src.as_ref()[..len] }, GFP_KERNEL)?; + Ok(out) + } + + fn sks_zero_buffers(&self) -> Result<()> { + let mut guard = self.ool_sks.lock(); + let pair: &mut Option = &mut guard; + let buffers = pair.as_mut().ok_or(ENODEV)?; + // SAFETY: the enclave touches these only between a request word and its + // reply, and none is outstanding here; the driver is the only other accessor. + unsafe { + buffers.inbound.as_mut()[..buffers.allocated].fill(0); + buffers.outbound.as_mut()[..buffers.allocated].fill(0); + } + Ok(()) + } + + pub(crate) fn ool_registered(&self, slot: &Mutex>) -> bool { + slot.lock().as_ref().is_some_and(|b| b.registered) + } + + pub(crate) fn sks_next_seq(&self) -> crate::sks::Sequence { + let n = self.sks_seq.load(Relaxed); + self.sks_seq.store(n.wrapping_add(1), Relaxed); + crate::sks::Sequence::from_counter(n as u8) + } + + fn sks_exchange( + &self, + label: &'static CStr, + msg: Message, + img: &image::RequestImage, + ) -> Option { + self.sks_exchange_patient(label, msg, img, 0, true) + } + + fn sks_exchange_patient( + &self, + label: &'static CStr, + msg: Message, + img: &image::RequestImage, + floor_ms: time::Msecs, + condemn: bool, + ) -> Option { + if !self.ool_registered(&self.ool_sks) { + return None; + } + + if self.sks_wedged.load(Relaxed) != 0 { + return None; + } + + if self.sks_zero_buffers().is_err() { + return None; + } + if self.ool_write(&self.ool_sks, 0, img.as_slice()).is_err() { + return None; + } + + let sized_ms = sks_timeout_for(img.len()); + let timeout_ms = if sized_ms < floor_ms { + floor_ms + } else { + sized_ms + }; + self.sks_arm(label); + if self.send(msg).is_err() { + self.sks_disarm(); + return None; + } + let started_ns = crate::shim::boottime_ns(); + + let sent = crate::sks::decode_sks(&msg); + let mut examined = 0usize; + let mut set_aside = 0u32; + let correlating = loop { + self.sks_wait(timeout_ms, examined + 1); + + let next = { + let probe = self.sks_probe.lock(); + let m = probe.captured.get(examined).copied(); + if m.is_some() { + examined += 1; + } + m + }; + + let Some(candidate) = next else { break None }; + + let decoded = crate::sks::decode_sks(&candidate); + if decoded.seq == sent.seq && decoded.selector == sent.selector { + break Some(candidate); + } + + set_aside = set_aside.saturating_add(1); + if set_aside >= SKS_MAX_SET_ASIDE { + break None; + } + }; + + let leftover = self.sks_disarm(); + let correlating = match correlating { + Some(m) => Some(m), + None => leftover.iter().skip(examined).copied().find(|m| { + let d = crate::sks::decode_sks(m); + d.seq == sent.seq && d.selector == sent.selector + }), + }; + + let Some(raw) = correlating else { + let waited_ms = crate::shim::boottime_ns().saturating_sub(started_ns) / 1_000_000; + self.sks_note_abandoned(sent.selector, sent.seq, label, waited_ms); + if condemn { + self.sks_wedged.store(1, Relaxed); + } + dev_err!(self.dev, "sks: {} got no reply after {} ms\n", label, waited_ms); + return None; + }; + + let reply = crate::sks::decode_sks(&raw); + // reply.status is signed here on (e.g. -13, not 0xf3). + + let mut response = Secret::empty(); + if reply.response_size > 0 { + if let Ok(bytes) = self.sks_read_buffer(false, reply.response_size as usize) { + response = Secret(bytes); + } + } + + Some(SksOutcome { reply, response }) + } + + pub(crate) fn sks_report_response<'a>( + &self, + _label: &CStr, + out: &'a SksOutcome, + ) -> Option<&'a [u8]> { + if out.response.is_empty() { + return None; + } + + let parsed = match image::parse_response(&out.response) { + Ok(p) => p, + Err(_) => { + return None; + } + }; + Some(parsed.body) + } + + pub(crate) fn sks_timestamp_us(&self) -> u64 { + crate::shim::boottime_ns() / 1000 + } + + pub(crate) fn sks_image_len(&self, img: &image::RequestImage) -> Result { + let declared_in = sks_declared_sizes().0; + if img.len() > declared_in { + return Err(ENOSPC); + } + crate::sks::ImageLen::of(img.len()).ok_or(EINVAL) + } + + fn sks_seal(&self, op: &crate::sks::SksOp, body: &image::Body) -> Result { + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: op.name(), + msg: crate::sks::encode_sks_read(op, self.sks_next_seq(), len), + img, + }) + } + + /// `0x4d` get capabilities. + fn sks_req_get_capabilities(&self) -> Result { + let op = crate::sks::sks_get_capabilities(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(1)?; + body.put_blob(&[])?; + self.sks_seal(&op, &body) + } + + /// `0x0d` designate. + fn sks_req_designate(&self, d: &crate::sks::Designation, secret: &[u8]) -> Result { + let mut body = image::Body::new(); + body.put_u32(crate::sks::SKS_DESIGNATE_VARIANT)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(d.source().value())?; + body.put_i32(d.user().special_handle().value())?; + body.put_blob(secret)?; + // Flags is a u64, not u32; a u32 leaves the body short -> enclave answers -13. + body.put_u64(crate::sks::SKS_DESIGNATE_FLAGS)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_designate(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_DESIGNATE_NAME, + msg, + img, + }) + } + + /// `0x05` unload the source handle. + pub(crate) fn sks_req_unload_keybag(&self, handle: crate::sks::KeyBagHandle) -> Result { + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_unload_keybag(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_UNLOAD_NAME, + msg, + img, + }) + } + + /// `0x06` UUID read, against the special handle. + pub(crate) fn sks_req_copy_uuid_special(&self, special: crate::sks::SpecialHandle) -> Result { + let op = crate::sks::sks_copy_keybag_uuid(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(special.value())?; + self.sks_seal(&op, &body) + } + + pub(crate) fn sks_req_unlock_special( + &self, + special: crate::sks::SpecialHandle, + secret: &[u8], + healthy: Healthy, + ) -> Result { + let Healthy(()) = healthy; + let mut body = image::Body::new(); + body.put_u32(SKS_LOCK_STATE_VARIANT)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(special.value())?; + body.put_i32(LockState::Unlocked.wire())?; + body.put_blob(secret)?; + body.put_u64(SKS_LOCK_STATE_FLAGS)?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_LOCK_STATE_NAME, + msg, + img, + }) + } + + /// `0x06` copy key-bag UUID. + fn sks_req_copy_keybag_uuid(&self, handle: crate::sks::KeyBagHandle) -> Result { + let op = crate::sks::sks_copy_keybag_uuid(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + self.sks_seal(&op, &body) + } + + /// `0x02` copy the designated biometric identity bag. + fn sks_req_copy_keybag_special(&self, handle: crate::sks::SpecialHandle) -> Result { + let op = crate::sks::sks_copy_keybag(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + self.sks_seal(&op, &body) + } + + /// `0x03` load key bag. + fn sks_req_load_keybag(&self, wrapped: &[u8]) -> Result { + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_blob(wrapped)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_load(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_LOAD_NAME, + msg, + img, + }) + } + + pub(crate) fn sks_send(&self, req: Result) -> Option { + match req { + Ok(r) => self.sks_exchange(r.name, r.msg, &r.img), + Err(_) => None, + } + } + + pub(crate) fn sks_designate_user_keybag(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) { + let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) else { + return; + }; + + let designation = crate::sks::Designation::new(handle, user); + + let Some(out) = self.sks_send(self.sks_req_designate(&designation, secret)) else { + dev_warn!(self.dev, "sks: DESIGNATE_KEYBAG did not complete; enrolment will refuse\n"); + return; + }; + let Some(body) = + self.sks_report_response(crate::sks::SKS_DESIGNATE_NAME, &out) + else { + return; + }; + + if body.len() != crate::sks::SKS_DESIGNATE_REPLY_LEN { + return; + } + let variant = u32::from_le_bytes([body[0], body[1], body[2], body[3]]); + if variant != crate::sks::SKS_DESIGNATE_VARIANT { + return; + } + + self.keybag_designated.store(true, Relaxed); + + self.sks_remember_enrolment_material( + designation.user().special_handle(), + secret, + ); + } + + fn sks_remember_enrolment_material( + &self, + special: crate::sks::SpecialHandle, + secret: &[u8], + ) { + let mut copy = KVec::new(); + if copy.extend_from_slice(secret, GFP_KERNEL).is_err() { + return; + } + *self.enrol_material.lock() = Some(EnrolMaterial { + special, + secret: Secret(copy), + }); + } + + /// `0x04` change lock state (v1). + pub(crate) fn sks_req_change_lock_state( + &self, + handle: crate::sks::KeyBagHandle, + state: LockState, + secret: &[u8], + healthy: Healthy, + ) -> Result { + let Healthy(()) = healthy; + let mut body = image::Body::new(); + body.put_u32(SKS_LOCK_STATE_VARIANT)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + body.put_i32(state.wire())?; + body.put_blob(secret)?; + // Trailing u64 goes after the blob (`0x18` puts it before). + body.put_u64(SKS_LOCK_STATE_FLAGS)?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_LOCK_STATE_NAME, + msg, + img, + }) + } + + /// `0x21` verify-secret (variant 1). + pub(crate) fn sks_req_verify_secret( + &self, + special: crate::sks::SpecialHandle, + secret: &[u8], + acm_context: &[u8; crate::scrd::SCRD_ACM_HANDLE_LEN], + ) -> Result { + let mut body = image::Body::new(); + body.put_u32(crate::sks::SKS_VERIFY_SECRET_VARIANT)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(special.value())?; + body.put_blob(secret)?; + body.put_blob(acm_context)?; + body.put_u64(0)?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_verify_secret(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_VERIFY_SECRET_NAME, + msg, + img, + }) + } + + pub(crate) fn sks_step( + &self, + label: &CStr, + build: impl FnOnce(Healthy) -> Result, + ) -> Option { + let healthy = self.sks_health_check(label)?; + let out = self.sks_send(build(healthy))?; + if out.reply.status != 0 { + return None; + } + Some(out) + } + + pub(crate) fn sks_health_check(&self, why: &CStr) -> Option { + let Some(out) = self.sks_send(self.sks_req_get_capabilities()) else { + dev_err!(self.dev, "sks: health check ({}) got no reply; endpoint gone for this boot\n", why); + return None; + }; + if out.reply.status != 0 { + dev_err!(self.dev, "sks: health check ({}) returned status {}\n", why, out.reply.status); + return None; + } + Some(Healthy(())) + } + + pub(crate) fn sks_recover( + &self, + stored: &keybag::StoredKeyBag, + ) -> Option<(crate::sks::KeyBagHandle, [u8; keybag::UUID_LEN])> { + let out = self.sks_send(self.sks_req_load_keybag(stored.wrapped()))?; + let body = self.sks_report_response(crate::sks::SKS_LOAD_NAME, &out)?; + + if out.reply.status != 0 { + dev_warn!( + self.dev, + "sks: LOAD_KEYBAG mailbox status {}; no handle\n", + out.reply.status + ); + return None; + } + if body.len() != SKS_LOAD_REPLY_LEN { + return None; + } + let status = i32::from_le_bytes([body[0], body[1], body[2], body[3]]); + let handle = i32::from_le_bytes([body[4], body[5], body[6], body[7]]); + if status != 0 || handle < 0 { + return None; + } + let handle = crate::sks::KeyBagHandle::from_load_reply(handle); + + let uuid = self.sks_read_uuid(handle)?; + + if uuid == *stored.uuid() { + Some((handle, uuid)) + } else { + dev_err!( + self.dev, + "sks: load succeeded but bag identity mismatch: stored UUID {}, loaded UUID {}\n", + Hex(stored.uuid()), + Hex(&uuid) + ); + None + } + } + + pub(crate) fn resnapshot_identity_keybag(&self) -> bool { + let special = { + let material = self.enrol_material.lock(); + let Some(material) = material.as_ref() else { + return false; + }; + material.special + }; + + let Some(out) = self.sks_send(self.sks_req_copy_keybag_special(special)) else { + return false; + }; + let Some(wrapped) = self.wrapped_from_copy_reply(&out, c"the designated identity bag") + else { + return false; + }; + + let Some(uuid) = self + .sks_send(self.sks_req_copy_uuid_special(special)) + .and_then(|out| self.sks_uuid_from_reply(&out)) + else { + return false; + }; + + match keybag::replace_wrapped(keybag::Slot::Identity, &wrapped, &uuid) { + Ok(()) => true, + Err(e) => { + dev_err!( + self.dev, + "enrol: could not commit the identity-bag snapshot ({:?}); may not survive reboot\n", + e + ); + false + } + } + } + + fn sks_read_uuid(&self, handle: crate::sks::KeyBagHandle) -> Option<[u8; keybag::UUID_LEN]> { + let out = self.sks_send(self.sks_req_copy_keybag_uuid(handle))?; + self.sks_uuid_from_reply(&out) + } + + pub(crate) fn sks_uuid_from_reply(&self, out: &SksOutcome) -> Option<[u8; keybag::UUID_LEN]> { + let body = self.sks_report_response(c"COPY_KEYBAG_UUID", out)?; + + if out.reply.status != 0 || image::operation_status(body).unwrap_or(-1) != 0 { + return None; + } + let (blob, _) = image::read_blob(body, 4)?; + if blob.len() != keybag::UUID_LEN { + return None; + } + let mut uuid = [0u8; keybag::UUID_LEN]; + uuid.copy_from_slice(blob); + Some(uuid) + } + + pub(crate) fn enable_sks(&self) -> Result<()> { + self.register_ool(&self.ool_sks) + } + + pub(crate) fn sks_ready(&self) -> bool { + if self.ool_registered(&self.ool_sks) { + return true; + } + if !self.endpoint_present(proto::EP_SKS) { + dev_err!( + self.dev, + "sks: endpoint 0x{:02x} (key store) was not advertised on this boot; keybag and ref-key operations cannot run\n", + proto::EP_SKS + ); + return false; + } + if let Err(e) = self.enable_sks() { + dev_err!(self.dev, "sks: could not register out-of-line buffers ({:?})\n", e); + return false; + } + true + } + + fn sks_arm(&self, label: &'static CStr) { + let mut probe = self.sks_probe.lock(); + probe.captured.clear(); + probe.label = Some(label); + probe.active = true; + } + + fn sks_disarm(&self) -> KVec { + let mut probe = self.sks_probe.lock(); + probe.active = false; + probe.label = None; + core::mem::take(&mut probe.captured) + } + + fn sks_wait(&self, ms: time::Msecs, until: usize) { + let mut remaining = time::msecs_to_jiffies(ms); + let mut guard = self.sks_probe.lock(); + loop { + if guard.captured.len() >= until || remaining == 0 { + return; + } + match self + .sks_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } +} + +const SKS_SELECTOR_MAX: u8 = 0x5f; + +pub(crate) const SKS_REPLY_BIT: u8 = 0x80; + +const OP_SKS_REWRAP_FORBIDDEN: u8 = 0x0f; + +pub(crate) struct SksOp { + opcode: u8, + name: &'static CStr, +} + +impl SksOp { + pub(crate) fn opcode(&self) -> u8 { + self.opcode + } + pub(crate) fn name(&self) -> &'static CStr { + self.name + } +} + +const OP_SKS_DEVICE_STATE: u8 = 0x19; +pub(crate) fn sks_get_device_state() -> SksOp { + SksOp { + opcode: OP_SKS_DEVICE_STATE, + name: c"GET_DEVICE_STATE", + } +} + +const OP_SKS_GET_CONFIGURATION: u8 = 0x23; + +const OP_SKS_SET_CONFIGURATION: u8 = 0x24; +static_assert!(OP_SKS_SET_CONFIGURATION != OP_SKS_GET_CONFIGURATION); + +const OP_SKS_NEW_PFK: u8 = 0x10; +static_assert!(OP_SKS_NEW_PFK != 0x0f); +static_assert!(OP_SKS_NEW_PFK != 0x09); + +// FileVault seal order: 0x42 KEK, then 0x40 VEK, then 0x41 install; 0x47 clear forbidden +pub(crate) const OP_SKS_FV_NEW_VEK: u8 = 0x40; +pub(crate) const OP_SKS_FV_UNWRAP_VEK: u8 = 0x41; +pub(crate) const OP_SKS_FV_NEW_KEK: u8 = 0x42; + +const OP_SKS_GENERIC_OPERATION: u8 = 0x1a; +const OP_SKS_PERFORM_OPERATION: u8 = 0x22; +const OP_SKS_IDENTITY_OPERATION: u8 = 0x51; +pub(crate) const SKS_PERFORM_OP_NAME: &CStr = c"PERFORM_OPERATION"; +static_assert!(OP_SKS_PERFORM_OPERATION != OP_SKS_GET_CONFIGURATION); +static_assert!(OP_SKS_PERFORM_OPERATION != OP_SKS_SET_CONFIGURATION); + +pub(crate) fn encode_sks_perform_operation(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_PERFORM_OPERATION, seq.value(), len.value()); + Some(msg) +} + +pub(crate) fn encode_sks_fv(selector: u8, seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(selector, seq.value(), len.value()); + Some(msg) +} +const OP_SKS_LAST_USER_OPERATION: u8 = 0x53; +static_assert!(OP_SKS_LAST_USER_OPERATION != 0x56); + +const OP_SKS_CAPABILITIES: u8 = 0x4d; +pub(crate) fn sks_get_capabilities() -> SksOp { + SksOp { + opcode: OP_SKS_CAPABILITIES, + name: c"GET_CAPABILITIES", + } +} + +const OP_SKS_COPY_UUID: u8 = 0x06; +pub(crate) fn sks_copy_keybag_uuid() -> SksOp { + SksOp { + opcode: OP_SKS_COPY_UUID, + name: c"COPY_KEYBAG_UUID", + } +} + +const OP_SKS_COPY_KEYBAG: u8 = 0x02; +pub(crate) fn sks_copy_keybag() -> SksOp { + SksOp { + opcode: OP_SKS_COPY_KEYBAG, + name: c"COPY_KEYBAG", + } +} + +const OP_SKS_CREATE_KEYBAG: u8 = 0x01; + +const OP_SKS_LOAD_KEYBAG: u8 = 0x03; + +pub(crate) const SKS_LOAD_NAME: &CStr = c"LOAD_KEYBAG"; + +const OP_SKS_CHANGE_LOCK_STATE: u8 = 0x04; + +pub(crate) const SKS_LOCK_STATE_NAME: &CStr = c"CHANGE_LOCK_STATE"; + +const OP_SKS_TOKEN_CREATE: u8 = 0x1c; + +pub(crate) const SKS_TOKEN_CREATE_NAME: &CStr = c"AUTH_TOKEN_CREATE"; + +const OP_SKS_TOKEN_VERIFY: u8 = 0x1d; + +static_assert!(OP_SKS_TOKEN_VERIFY == OP_SKS_TOKEN_CREATE + 1); + +#[derive(Clone, Copy)] +pub(crate) struct NewDeviceState(i32); + +impl NewDeviceState { + pub(crate) const UNLOCKED: NewDeviceState = NewDeviceState(0); + + pub(crate) const fn value(&self) -> i32 { + self.0 + } +} +static_assert!(NewDeviceState::UNLOCKED.value() == 0); + +const OP_SKS_DEVICE_STATE_TRANSITION: u8 = 0x18; +static_assert!(OP_SKS_DEVICE_STATE_TRANSITION + 1 == 0x19); + +pub(crate) const SKS_DEVICE_STATE_REPLY_LEN: usize = 20; +static_assert!(SKS_DEVICE_STATE_REPLY_LEN == 4 + 8 + 8); + +const OP_SKS_VERIFY_SECRET: u8 = 0x21; +pub(crate) const SKS_VERIFY_SECRET_NAME: &CStr = c"VERIFY_SECRET"; +pub(crate) const SKS_VERIFY_SECRET_VARIANT: u32 = 1; + +pub(crate) fn encode_sks_verify_secret(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_VERIFY_SECRET, seq.value(), len.value()); + Some(msg) +} + +pub(crate) const SKS_VERIFY_SECRET_REPLY_LEN: usize = 12; +static_assert!(SKS_VERIFY_SECRET_REPLY_LEN == 4 + 8); + +const SKS_SELECTOR_BITS: u8 = 0x7f; + +static_assert!(SKS_SELECTOR_MAX <= SKS_SELECTOR_BITS); + +pub(crate) const SKS_CLIENT_ID: u64 = u64::from_be_bytes(*b"LINUXSKS"); +static_assert!(SKS_CLIENT_ID == 0x4c49_4e55_5853_4b53); + +pub(crate) const SKS_CLIENT_ID_SEALING: u64 = u64::from_be_bytes(*b"LINUXSLS"); +static_assert!(SKS_CLIENT_ID_SEALING == 0x4c49_4e55_5853_4c53); +static_assert!(SKS_CLIENT_ID_SEALING != SKS_CLIENT_ID); + +#[derive(Clone, Copy)] +pub(crate) struct KeyBagHandle(i32); + +impl KeyBagHandle { + + pub(crate) const fn from_load_reply(v: i32) -> KeyBagHandle { + KeyBagHandle(v) + } + + pub(crate) const fn value(&self) -> i32 { + self.0 + } +} + +const OP_SKS_DESIGNATE_KEYBAG: u8 = 0x0d; + +pub(crate) const SKS_DESIGNATE_VARIANT: u32 = 1; + +pub(crate) const SKS_DESIGNATE_VARIANT_GENERIC: u32 = 0; + +static_assert!(SKS_DESIGNATE_VARIANT == 1); +static_assert!(SKS_DESIGNATE_VARIANT_GENERIC == 0); +static_assert!(SKS_DESIGNATE_VARIANT != SKS_DESIGNATE_VARIANT_GENERIC); + +pub(crate) const SKS_CREATE_VARIANT_IDENTITY: u32 = 5; +static_assert!(SKS_CREATE_VARIANT_IDENTITY != 1); + +pub(crate) const SKS_IDENTITY_UUID_LEN: usize = 16; +static_assert!(SKS_IDENTITY_UUID_LEN == crate::sbio::IDENTITY_UUID_LEN); + +pub(crate) const SKS_IDENTITY_USER_ID: i32 = 1000; + +pub(crate) const SKS_DESIGNATE_USER_MIN: i32 = 10; +static_assert!(SKS_DESIGNATE_USER_MIN > 0); + +#[derive(Clone, Copy)] +pub(crate) struct DesignateUser(i32); + +impl DesignateUser { + pub(crate) fn new(value: i32) -> Option { + if value >= SKS_DESIGNATE_USER_MIN { + Some(DesignateUser(value)) + } else { + None + } + } + + pub(crate) const fn special_handle(&self) -> SpecialHandle { + SpecialHandle(-self.0) + } +} + +#[derive(Clone, Copy)] +pub(crate) struct SpecialHandle(i32); + +impl SpecialHandle { +} + +impl SpecialHandle { + pub(crate) const fn value(&self) -> i32 { + self.0 + } +} + +pub(crate) const SKS_AUTH_TOKEN_LEN: usize = 16; + +pub(crate) struct AuthToken([u8; SKS_AUTH_TOKEN_LEN]); + +impl AuthToken { + pub(crate) fn from_reply(body: &[u8]) -> Option { + if body.len() < 8 { + return None; + } + let status = i32::from_le_bytes([body[0], body[1], body[2], body[3]]); + if status != 0 { + return None; + } + let len = u32::from_le_bytes([body[4], body[5], body[6], body[7]]) as usize; + if len != SKS_AUTH_TOKEN_LEN || body.len() < 8 + len { + return None; + } + let mut out = [0u8; SKS_AUTH_TOKEN_LEN]; + out.copy_from_slice(&body[8..8 + len]); + Some(AuthToken(out)) + } + +} + +impl Drop for AuthToken { + fn drop(&mut self) { + for b in self.0.iter_mut() { + // SAFETY: a valid, uniquely borrowed byte; volatile so the wipe is not elided. + unsafe { core::ptr::write_volatile(b, 0) }; + } + } +} + +pub(crate) struct Designation { + source: KeyBagHandle, + user: DesignateUser, +} + +impl Designation { + pub(crate) fn new(source: KeyBagHandle, user: DesignateUser) -> Designation { + Designation { source, user } + } + + pub(crate) const fn source(&self) -> KeyBagHandle { + self.source + } + + pub(crate) const fn user(&self) -> DesignateUser { + self.user + } +} + +pub(crate) const SKS_DESIGNATE_REPLY_LEN: usize = 4; + +pub(crate) const SKS_DESIGNATE_FLAGS: u64 = 0; +pub(crate) const SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG: u64 = 0x100; +static_assert!(SKS_DESIGNATE_FLAGS == 0); +static_assert!(SKS_DESIGNATE_FLAGS != SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG); + +// keybag_flags sits at +0x64 in a 0x01 body +pub(crate) struct CreateFlags(u32); + +pub(crate) const SKS_KEYBAG_FLAG_MAX: u32 = 0xff; +static_assert!((SKS_KEYBAG_FLAG_MAX as u64) < SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG); + +impl CreateFlags { + pub(crate) const fn new(bits: u32) -> Option { + if bits <= SKS_KEYBAG_FLAG_MAX { + Some(CreateFlags(bits)) + } else { + None + } + } + + pub(crate) const fn none() -> CreateFlags { + CreateFlags(0) + } + + pub(crate) const fn value(&self) -> u32 { + self.0 + } +} + +static_assert!(CreateFlags::new(SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG as u32).is_none()); +static_assert!(CreateFlags::new(0x100).is_none()); +static_assert!(CreateFlags::new(0x101).is_none()); +static_assert!(CreateFlags::new(0x1ff).is_none()); +static_assert!(CreateFlags::new(u32::MAX).is_none()); +static_assert!(CreateFlags::new(0x80).is_some()); +static_assert!(CreateFlags::none().value() == 0); + +// device keybag = 0x0d with a u64 flags of 0x100 at +0x78 + +pub(crate) fn encode_sks_designate(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_DESIGNATE_KEYBAG, seq.value(), len.value()); + Some(msg) +} + +pub(crate) const SKS_DESIGNATE_NAME: &CStr = c"DESIGNATE_KEYBAG"; + +const OP_SKS_WRAP: u8 = 0x08; +const OP_SKS_UNWRAP: u8 = 0x45; + +// 0x08/0x45 carry no keybag-handle field + +const OP_SKS_CHANGE_SECRET: u8 = 0x07; +const OP_SKS_DRAIN_BACKUP_KEYS: u8 = 0x17; +const OP_SKS_ESCROW_ENABLE: u8 = 0x29; +const OP_SKS_ESCROW_CREATE: u8 = 0x13; +const OP_SKS_ESCROW_PERSIST: u8 = 0x2b; +// 0x13 reply is status only, no handle + +const OP_SKS_PUBLIC_BACKUP_HANDLE: u8 = 0x2d; +const OP_SKS_UNLOAD_PUBLIC_BACKUP: u8 = 0x2e; + +static_assert!(SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG == 0x100); +static_assert!(SKS_DESIGNATE_FLAGS != SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG); + +const OP_SKS_MAKE_BACKUP_BAG: u8 = 0x11; +const OP_SKS_SET_BACKUP_BAG: u8 = 0x0e; +const OP_SKS_BACKUP_WRAP: u8 = 0x54; +const OP_SKS_BACKUP_UNWRAP: u8 = 0x55; + +pub(crate) const SKS_SET_BACKUP_NAME: &CStr = c"SET_BACKUP_BAG"; + +pub(crate) fn encode_sks_set_backup_bag(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_SET_BACKUP_BAG, seq.value(), len.value()); + Some(msg) +} + +const OP_SKS_SET_ENV: u8 = 0x2a; + +pub(crate) const SKS_CLIENT_ID_ALT: u64 = u64::from_be_bytes(*b"LINUXSKT"); +static_assert!(SKS_CLIENT_ID_ALT != SKS_CLIENT_ID); +static_assert!(SKS_CLIENT_ID_ALT != SKS_CLIENT_ID_SEALING); +const fn client_id_byte_distance(a: u64, b: u64) -> u32 { + let (x, y) = (a.to_be_bytes(), b.to_be_bytes()); + let mut differing = 0; + let mut i = 0; + while i < 8 { + if x[i] != y[i] { + differing += 1; + } + i += 1; + } + differing +} +static_assert!(client_id_byte_distance(SKS_CLIENT_ID, SKS_CLIENT_ID_ALT) == 1); + +const OP_SKS_UNLOAD_KEYBAG: u8 = 0x05; +pub(crate) const SKS_UNLOAD_NAME: &CStr = c"UNLOAD_KEYBAG"; + +pub(crate) fn encode_sks_unload_keybag(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_UNLOAD_KEYBAG, seq.value(), len.value()); + Some(msg) +} + +#[derive(Clone, Copy)] +pub(crate) struct Sequence(u8); + +const SEQ_MIN: u8 = 0x60; + +impl Sequence { + pub(crate) const fn from_counter(n: u8) -> Sequence { + Sequence(SEQ_MIN | (n & !SEQ_MIN)) + } + + pub(crate) const fn value(&self) -> u8 { + self.0 + } +} + +pub(crate) const SEQ_SPACE: usize = 64; + +pub(crate) const SEQ_FIRST_REUSE: u8 = 32; + +static_assert!(Sequence::from_counter(0).value() != Sequence::from_counter(1).value()); +static_assert!( + Sequence::from_counter(0).value() == Sequence::from_counter(SEQ_FIRST_REUSE).value() +); +static_assert!(SEQ_SPACE == 64); + +const fn every_sequence_is_above_the_selector_range() -> bool { + let mut n = 0u8; + loop { + if Sequence::from_counter(n).value() <= SKS_SELECTOR_MAX { + return false; + } + if n == 0xff { + return true; + } + n += 1; + } +} +static_assert!(every_sequence_is_above_the_selector_range()); + +#[derive(Clone, Copy)] +pub(crate) struct ImageLen(u16); + +impl ImageLen { + pub(crate) fn of(len: usize) -> Option { + if len < crate::image::HEADER_WIRE { + return None; + } + match u16::try_from(len) { + Ok(v) => Some(ImageLen(v)), + Err(_) => None, + } + } + + pub(crate) const fn value(&self) -> u16 { + self.0 + } +} + +const fn encode_sks_raw(selector: u8, seq: u8, len: u16) -> Message { + Message { + msg0: (EP_SKS as u64) + | ((selector as u64) << MSG_TAG_SHIFT) + | ((seq as u64) << MSG_TYPE_SHIFT) + | ((len as u64) << 48), + msg1: 0, + } +} + +pub(crate) fn encode_sks_read(op: &SksOp, seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(op.opcode(), seq.value(), len.value()) +} + +pub(crate) fn encode_sks_load(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_LOAD_KEYBAG, seq.value(), len.value()); + Some(msg) +} + +pub(crate) fn encode_sks_change_lock_state(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_CHANGE_LOCK_STATE, seq.value(), len.value()); + Some(msg) +} + +pub(crate) fn encode_sks_token_create(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_TOKEN_CREATE, seq.value(), len.value()); + Some(msg) +} + +pub(crate) struct SksReply { + pub(crate) selector: u8, + pub(crate) seq: u8, + pub(crate) status: i8, + pub(crate) flags: u16, + pub(crate) response_size: u16, +} + +pub(crate) fn decode_sks(msg: &Message) -> SksReply { + let b = msg.msg0.to_le_bytes(); + SksReply { + selector: b[1] & !SKS_REPLY_BIT, + seq: b[2], + status: b[3] as i8, + flags: u16::from_le_bytes([b[4], b[5]]), + response_size: u16::from_le_bytes([b[6], b[7]]), + } +} + +pub(crate) const SKS_STATUS_MALFORMED: i8 = -13; +static_assert!(SKS_STATUS_MALFORMED as u8 == 0xf3); + +pub(crate) const SKS_STATUS_REFUSED: i8 = -19; +static_assert!(SKS_STATUS_REFUSED != SKS_STATUS_MALFORMED); +static_assert!(SKS_STATUS_REFUSED != 0); diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs new file mode 100644 index 0000000000000..ef4140ec781dc --- /dev/null +++ b/drivers/soc/apple/store.rs @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Host-owned slotted backing store for the SEP's persistent records: a +//! compile-time path exclusive to this driver, re-initialised and reseeded on a +//! magic mismatch. Re-initialised is never served empty — an empty store halts the SEP. + +use crate::shim; +use kernel::prelude::*; + +pub(crate) const STORE_PATH: &CStr = c"/var/lib/apple-sep-state.bin"; + +const BLOCK_SIZE: usize = 0x8000; +const BLOCK_COUNT: usize = 72; +pub(crate) const STORE_SIZE: usize = BLOCK_SIZE * BLOCK_COUNT; +const SLOT_COUNT: usize = BLOCK_COUNT - 1; +pub(crate) const MAX_VALUE: usize = BLOCK_SIZE; + +const MAGIC: [u8; 16] = *b"APPLE-SEP-STOR01"; +const VERSION: u32 = 1; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct Key { + pub(crate) kind: u8, + pub(crate) uuid: [u8; 16], +} + +impl Key { + pub(crate) const fn new(kind: u8, uuid: [u8; 16]) -> Key { + Key { kind, uuid } + } + + pub(crate) const fn root(kind: u8) -> Key { + Key { + kind, + uuid: [0u8; 16], + } + } +} + +#[derive(Clone, Copy)] +struct Slot { + used: bool, + kind: u8, + uuid: [u8; 16], + len: u32, +} + +impl Slot { + const FREE: Slot = Slot { + used: false, + kind: 0, + uuid: [0; 16], + len: 0, + }; + + fn matches(&self, key: &Key) -> bool { + self.used && self.kind == key.kind && self.uuid == key.uuid + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Intent { + None, + Write { + slot: u16, + kind: u8, + }, + Delete { + slot: u16, + kind: u8, + }, +} + +const SB_MAGIC: usize = 0; +const SB_VERSION: usize = 16; +const SB_SLOT_COUNT: usize = 20; +const SB_BLOCK_SIZE: usize = 24; +const SB_GENERATION: usize = 28; +const SB_INTENT_KIND: usize = 36; +const SB_INTENT_SLOT: usize = 37; +const SB_INTENT_TYPE: usize = 39; +// In previously reserved space, so an older store reads it as zero (unseeded). +const SB_SEEDED: usize = 40; +const SB_SLOT_TABLE: usize = 64; +const SLOT_ENTRY_SIZE: usize = 24; + +const INTENT_NONE: u8 = 0; +const INTENT_WRITE: u8 = 1; +const INTENT_DELETE: u8 = 2; + +// Not internally locked; the caller holds a mutex around all access. +pub(crate) struct Store { + file: shim::StoreFile, + slots: [Slot; SLOT_COUNT], + generation: u64, + pub(crate) recovered: bool, + pub(crate) fresh: bool, + seeded: bool, +} + +fn le32(buf: &[u8], off: usize) -> u32 { + u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) +} + +fn le64(buf: &[u8], off: usize) -> u64 { + let mut v = [0u8; 8]; + v.copy_from_slice(&buf[off..off + 8]); + u64::from_le_bytes(v) +} + +impl Store { + pub(crate) fn open() -> Result { + let file = shim::StoreFile::open(STORE_PATH)?; + + let mut store = Store { + file, + slots: [Slot::FREE; SLOT_COUNT], + generation: 0, + recovered: false, + fresh: false, + seeded: false, + }; + + let size = store.file.size()?; + if size < STORE_SIZE as u64 { + store.create()?; + return Ok(store); + } + + let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; + sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; + store.file.read_exact(0, &mut sb)?; + + if sb[SB_MAGIC..SB_MAGIC + 16] != MAGIC + || le32(&sb, SB_VERSION) != VERSION + || le32(&sb, SB_SLOT_COUNT) as usize != SLOT_COUNT + || le32(&sb, SB_BLOCK_SIZE) as usize != BLOCK_SIZE + { + store.create()?; + return Ok(store); + } + + store.generation = le64(&sb, SB_GENERATION); + store.seeded = sb[SB_SEEDED] != 0; + + for i in 0..SLOT_COUNT { + let off = SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE; + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&sb[off + 2..off + 18]); + store.slots[i] = Slot { + used: sb[off] != 0, + kind: sb[off + 1], + uuid, + len: le32(&sb, off + 18), + }; + } + + // Crash recovery: writes are copy-on-write, so discard the intent, never complete it. + let intent_kind = sb[SB_INTENT_KIND]; + if intent_kind != INTENT_NONE { + store.recovered = true; + let slot = u16::from_le_bytes([sb[SB_INTENT_SLOT], sb[SB_INTENT_SLOT + 1]]) as usize; + if intent_kind == INTENT_WRITE && slot < SLOT_COUNT { + store.slots[slot] = Slot::FREE; + } + store.commit()?; + } + + Ok(store) + } + + fn create(&mut self) -> Result<()> { + self.slots = [Slot::FREE; SLOT_COUNT]; + self.generation = 1; + self.fresh = true; + self.seeded = false; + + let mut zero = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; + zero.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; + for b in 0..BLOCK_COUNT { + self.file.write_all((b * BLOCK_SIZE) as u64, &zero)?; + } + + self.commit()?; + Ok(()) + } + + fn commit(&mut self) -> Result<()> { + self.write_superblock(Intent::None)?; + self.file.sync() + } + + fn write_superblock(&mut self, intent: Intent) -> Result<()> { + let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; + sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; + + sb[SB_MAGIC..SB_MAGIC + 16].copy_from_slice(&MAGIC); + sb[SB_VERSION..SB_VERSION + 4].copy_from_slice(&VERSION.to_le_bytes()); + sb[SB_SLOT_COUNT..SB_SLOT_COUNT + 4].copy_from_slice(&(SLOT_COUNT as u32).to_le_bytes()); + sb[SB_BLOCK_SIZE..SB_BLOCK_SIZE + 4].copy_from_slice(&(BLOCK_SIZE as u32).to_le_bytes()); + sb[SB_GENERATION..SB_GENERATION + 8].copy_from_slice(&self.generation.to_le_bytes()); + + let (ikind, islot, itype) = match intent { + Intent::None => (INTENT_NONE, 0u16, 0u8), + Intent::Write { slot, kind } => (INTENT_WRITE, slot, kind), + Intent::Delete { slot, kind } => (INTENT_DELETE, slot, kind), + }; + sb[SB_SEEDED] = u8::from(self.seeded); + sb[SB_INTENT_KIND] = ikind; + sb[SB_INTENT_SLOT..SB_INTENT_SLOT + 2].copy_from_slice(&islot.to_le_bytes()); + sb[SB_INTENT_TYPE] = itype; + + for (i, slot) in self.slots.iter().enumerate() { + let off = SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE; + sb[off] = u8::from(slot.used); + sb[off + 1] = slot.kind; + sb[off + 2..off + 18].copy_from_slice(&slot.uuid); + sb[off + 18..off + 22].copy_from_slice(&slot.len.to_le_bytes()); + } + + self.file.write_all(0, &sb) + } + + fn find(&self, key: &Key) -> Option { + self.slots.iter().position(|s| s.matches(key)) + } + + fn find_free(&self) -> Option { + self.slots.iter().position(|s| !s.used) + } + + fn block_offset(slot: usize) -> u64 { + ((slot + 1) * BLOCK_SIZE) as u64 + } + + pub(crate) fn read(&mut self, key: &Key) -> Result>> { + let Some(idx) = self.find(key) else { + return Ok(None); + }; + let len = self.slots[idx].len as usize; + let mut value = KVec::with_capacity(len, GFP_KERNEL)?; + value.resize(len, 0, GFP_KERNEL)?; + if len > 0 { + self.file.read_exact(Self::block_offset(idx), &mut value)?; + } + Ok(Some(value)) + } + + pub(crate) fn write(&mut self, key: &Key, value: &[u8]) -> Result<()> { + if value.len() > MAX_VALUE { + return Err(ENOSPC); + } + + let old = self.find(key); + let fresh = self.find_free().ok_or(ENOSPC)?; + + // 1. Record the intent, durably, before touching any data block. + self.write_superblock(Intent::Write { + slot: fresh as u16, + kind: key.kind, + })?; + self.file.sync()?; + + // 2. Fill the new block and make it durable. + if !value.is_empty() { + self.file.write_all(Self::block_offset(fresh), value)?; + } + self.file.sync()?; + + // 3. Switch the slot table, release the old block, clear the intent. + self.slots[fresh] = Slot { + used: true, + kind: key.kind, + uuid: key.uuid, + len: value.len() as u32, + }; + if let Some(old) = old { + self.slots[old] = Slot::FREE; + } + self.generation = self.generation.wrapping_add(1); + self.commit() + } + + pub(crate) fn delete(&mut self, key: &Key) -> Result { + let Some(idx) = self.find(key) else { + return Ok(false); + }; + + self.write_superblock(Intent::Delete { + slot: idx as u16, + kind: key.kind, + })?; + self.file.sync()?; + + self.slots[idx] = Slot::FREE; + self.generation = self.generation.wrapping_add(1); + self.commit()?; + Ok(true) + } + + // Gates import, not store-emptiness: a store emptied after the SEP moved on must not be reseeded. + pub(crate) fn seeded(&self) -> bool { + self.seeded + } + + pub(crate) fn mark_seeded(&mut self) -> Result<()> { + self.seeded = true; + self.commit() + } + +} + +pub(crate) const fn crc16_ccitt_false(data: &[u8]) -> u16 { + let mut crc: u16 = 0xffff; + let mut i = 0; + while i < data.len() { + crc ^= (data[i] as u16) << 8; + let mut bit = 0; + while bit < 8 { + crc = if crc & 0x8000 != 0 { + (crc << 1) ^ 0x1021 + } else { + crc << 1 + }; + bit += 1; + } + i += 1; + } + crc +} + +// Canonical CCITT-FALSE check value; guards against a reflected/XOR variant. +static_assert!(crc16_ccitt_false(b"123456789") == 0x29B1); diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c new file mode 100644 index 0000000000000..33ed552f54798 --- /dev/null +++ b/drivers/soc/apple/store_shim.c @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* + * Backing-store file access. In C because the O_* flags are macro-derived and + * absent from the Rust bindings. + * + * These files are the driver's own. Opens run under kernel credentials so + * access never depends on which task triggered the operation: a keyctl(2) call + * from an unprivileged process reaches unseal in that process's context, and + * without this the driver could not read its own root-only store. + */ + +#include +#include +#include +#include +#include + +#include "shim.h" + +static struct file *open_as_kernel(const char *path, int flags, umode_t mode) +{ + const struct cred *old; + struct cred *kern; + struct file *f; + + kern = prepare_kernel_cred(&init_task); + if (!kern) + return ERR_PTR(-ENOMEM); + + old = override_creds(kern); + f = filp_open(path, flags, mode); + put_cred(revert_creds(old)); + + return f; +} + +/* Opens the store, creating if absent. Mode 0600: the file is the driver's own. */ +void *sep_store_open(const char *path) +{ + struct file *f = open_as_kernel(path, O_RDWR | O_CREAT | O_LARGEFILE, 0600); + + return IS_ERR(f) ? NULL : f; +} + +/* + * Opens truncated to empty, for callers that rewrite the whole file each time, + * so a shorter record cannot leave stale trailing bytes from a larger one. + */ +void *sep_store_open_trunc(const char *path) +{ + struct file *f = open_as_kernel(path, O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE, 0600); + + return IS_ERR(f) ? NULL : f; +} + +/* + * Opens an existing file read-only, NULL if absent. O_RDONLY and no O_CREAT so + * the seed cannot be created, truncated or written by mistake. + */ +void *sep_store_open_ro(const char *path) +{ + struct file *f = open_as_kernel(path, O_RDONLY | O_LARGEFILE, 0); + + return IS_ERR(f) ? NULL : f; +} + +void sep_store_close(void *handle) +{ + if (handle) + filp_close((struct file *)handle, NULL); +} + +/* Current length in bytes, or a negative errno. */ +long long sep_store_size(void *handle) +{ + struct file *f = handle; + + return i_size_read(file_inode(f)); +} + +/* Returns bytes read, 0 at end of file, or a negative errno. */ +long sep_store_read(void *handle, long long off, void *buf, size_t len) +{ + struct file *f = handle; + loff_t pos = off; + + return kernel_read(f, buf, len, &pos); +} + +/* Returns bytes written, or a negative errno. */ +long sep_store_write(void *handle, long long off, const void *buf, + size_t len) +{ + struct file *f = handle; + loff_t pos = off; + + return kernel_write(f, buf, len, &pos); +} + +/* Flushes data and metadata to durable storage. */ +int sep_store_sync(void *handle) +{ + return vfs_fsync((struct file *)handle, 0); +} diff --git a/drivers/soc/apple/transfer.rs b/drivers/soc/apple/transfer.rs new file mode 100644 index 0000000000000..67cf7ed13c312 --- /dev/null +++ b/drivers/soc/apple/transfer.rs @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! The generic transfer layer. + +use kernel::prelude::*; + +pub(crate) const MARKER_FIRST: u8 = 0xFC; +pub(crate) const MARKER_NEXT: u8 = 0xFD; +/// Requests the peer's next packet and acks the final one; same marker for both. +pub(crate) const MARKER_REQUEST: u8 = 0xFE; +pub(crate) const MARKER_ERROR: u8 = 0xFF; + +// A tag below MARKER_FIRST is not a marker; sound only while every marker >= it. +static_assert!(MARKER_NEXT >= MARKER_FIRST); +static_assert!(MARKER_REQUEST >= MARKER_FIRST); + +#[derive(Clone, Copy)] +pub(crate) enum DeviceStatus { + /// `0xffffffff` is a real refusal (signed −1), not the absence of an answer. + Answered(u32), + ReportedWithoutStatus, + NotAnswered, + BufferNeverWritten, +} + +impl DeviceStatus { + pub(crate) fn answered(&self) -> Option { + match self { + DeviceStatus::Answered(err) => Some(*err), + DeviceStatus::ReportedWithoutStatus + | DeviceStatus::NotAnswered + | DeviceStatus::BufferNeverWritten => None, + } + } + + pub(crate) fn is_ok(&self) -> bool { + matches!(self, DeviceStatus::Answered(0)) + } +} + +impl kernel::fmt::Display for DeviceStatus { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + match self { + DeviceStatus::Answered(err) => write!(f, "0x{:x}", err), + DeviceStatus::ReportedWithoutStatus => write!( + f, + "an error report from the device carrying no status word (the device DID answer)" + ), + DeviceStatus::NotAnswered => write!( + f, + "NONE — no message arrived at all (host sentinel, not an enclave value)" + ), + DeviceStatus::BufferNeverWritten => write!( + f, + "NONE — a payload notification arrived but the outbound buffer was never written (a race with the enclave's DMA, not a silent enclave)" + ), + } + } +} + +pub(crate) const HEADER_WORDS: usize = 7; +pub(crate) const HEADER_LEN: usize = HEADER_WORDS * 4; + +const VERSION: u32 = 1; + +pub(crate) const MAX_TRANSACTION: u32 = 0x4B000; + +#[derive(Clone, Copy)] +pub(crate) struct Packet { + pub(crate) version: u32, + pub(crate) total: u32, + pub(crate) offset: u32, + pub(crate) flags: u32, + pub(crate) err: u32, + pub(crate) opcode: u32, + pub(crate) chunk: u32, +} + +fn le32(buf: &[u8], word: usize) -> u32 { + let o = word * 4; + u32::from_le_bytes([buf[o], buf[o + 1], buf[o + 2], buf[o + 3]]) +} + +impl Packet { + pub(crate) fn decode(buf: &[u8]) -> Result { + if buf.len() < HEADER_LEN { + return Err(EINVAL); + } + Ok(Packet { + version: le32(buf, 0), + total: le32(buf, 1), + offset: le32(buf, 2), + flags: le32(buf, 3), + err: le32(buf, 4), + opcode: le32(buf, 5), + chunk: le32(buf, 6), + }) + } + + pub(crate) fn encode(&self, buf: &mut [u8]) -> Result<()> { + if buf.len() < HEADER_LEN { + return Err(EINVAL); + } + for (word, value) in [ + self.version, + self.total, + self.offset, + self.flags, + self.err, + self.opcode, + self.chunk, + ] + .into_iter() + .enumerate() + { + buf[word * 4..word * 4 + 4].copy_from_slice(&value.to_le_bytes()); + } + Ok(()) + } + + pub(crate) fn request_next(opcode: u32, received: u32, total: u32) -> Packet { + Packet { + version: VERSION, + total, + offset: received, + flags: 0, + err: 0, + opcode, + chunk: 0, + } + } +} + +struct Active { + opcode: u32, + total: u32, + payload: KVec, + seq: u16, +} + +pub(crate) struct Completed { + pub(crate) opcode: u32, + pub(crate) status: DeviceStatus, + pub(crate) payload: KVec, +} + +pub(crate) struct Continuation { + opcode: u32, + received: u32, + total: u32, + seq: u16, +} + +impl Continuation { + pub(crate) fn opcode(&self) -> u32 { + self.opcode + } + pub(crate) fn seq(&self) -> u16 { + self.seq + } + pub(crate) fn packet(&self) -> Packet { + Packet::request_next(self.opcode, self.received, self.total) + } +} + +pub(crate) enum Progress { + NeedMore(Continuation), + Complete, + /// A stray chunk, dropped without touching transfer state — distinct from + /// `Failed`, which kills the transfer. + Ignored(&'static CStr), + Grant, + Notification { tag: u8, opcode: u32 }, + Failed(&'static CStr), +} + +pub(crate) struct Reassembly { + active: Option, + done: Option, + sending: Option, + grants: u32, +} + +impl Reassembly { + pub(crate) fn new() -> Reassembly { + Reassembly { + active: None, + done: None, + sending: None, + grants: 0, + } + } + + pub(crate) fn begin(&mut self, opcode: u32) -> Result<()> { + if self.active.is_some() { + return Err(EBUSY); + } + self.done = None; + self.active = Some(Active { + opcode, + total: 0, + payload: KVec::new(), + seq: 0, + }); + Ok(()) + } + + pub(crate) fn awaiting(&self) -> Option { + self.active.as_ref().map(|a| a.opcode) + } + + pub(crate) fn begin_send(&mut self, opcode: u32) { + self.sending = Some(opcode); + self.grants = 0; + } + + pub(crate) fn end_send(&mut self) { + self.sending = None; + self.grants = 0; + } + + pub(crate) fn take_grant(&mut self) -> bool { + if self.grants > 0 { + self.grants -= 1; + return true; + } + false + } + + pub(crate) fn take_done(&mut self) -> Option { + self.done.take() + } + + pub(crate) fn abort(&mut self) { + self.abort_with(DeviceStatus::NotAnswered); + } + + pub(crate) fn abort_with(&mut self, status: DeviceStatus) { + if let Some(active) = self.active.take() { + self.done = Some(Completed { + opcode: active.opcode, + status, + payload: KVec::new(), + }); + } + } + + fn fail(&mut self) { + self.abort(); + } + + pub(crate) fn on_chunk(&mut self, marker: u8, packet: &Packet, payload: &[u8]) -> Progress { + if marker < MARKER_FIRST { + return Progress::Notification { + tag: marker, + opcode: packet.opcode, + }; + } + + // 0xFE is flow control for the request being sent, answered from `sending`. + if marker == MARKER_REQUEST { + return match self.sending { + Some(_) => { + // Counted under the caller's lock, closing the race with the waiter. + self.grants = self.grants.saturating_add(1); + Progress::Grant + } + None => Progress::Ignored(c"a 0xFE arrived with nothing being sent"), + }; + } + + let Some(active_opcode) = self.active.as_ref().map(|a| a.opcode) else { + return Progress::Ignored(c"no transfer outstanding"); + }; + + // Checked before the opcode test below, deliberately. + if marker == MARKER_ERROR { + let status = if packet.err != 0 { + DeviceStatus::Answered(packet.err) + } else { + DeviceStatus::ReportedWithoutStatus + }; + self.finish(status, KVec::new()); + return Progress::Complete; + } + + // The device echoes the opcode on a data chunk. + if packet.opcode != active_opcode { + return Progress::Ignored(c"chunk belongs to a different opcode"); + } + + if packet.version != VERSION { + self.fail(); + return Progress::Failed(c"header version is not 1"); + } + if packet.chunk as usize != payload.len() { + self.fail(); + return Progress::Failed(c"chunk length disagrees with the payload taken"); + } + if packet.total > MAX_TRANSACTION { + self.fail(); + return Progress::Failed(c"total length exceeds the maximum transaction size"); + } + + match marker { + MARKER_FIRST => { + let Some(active) = self.active.as_ref() else { + return Progress::Ignored(c"transfer vanished"); + }; + if !active.payload.is_empty() { + self.fail(); + return Progress::Failed(c"second first-chunk for one transfer"); + } + if packet.offset != 0 { + self.fail(); + return Progress::Failed(c"first chunk is not at offset zero"); + } + if let Some(active) = self.active.as_mut() { + active.total = packet.total; + } + } + MARKER_NEXT => { + let Some(active) = self.active.as_ref() else { + return Progress::Ignored(c"transfer vanished"); + }; + if packet.offset as usize != active.payload.len() { + self.fail(); + return Progress::Failed(c"continuation chunk is not at the expected offset"); + } + if packet.total != active.total { + self.fail(); + return Progress::Failed(c"continuation chunk changed the total length"); + } + } + _ => { + self.fail(); + return Progress::Failed(c"unexpected marker at or above 0xFC"); + } + } + + let Some(active) = self.active.as_mut() else { + return Progress::Ignored(c"transfer vanished"); + }; + + if packet.err != 0 { + let status = DeviceStatus::Answered(packet.err); + let collected = core::mem::take(&mut active.payload); + self.finish(status, collected); + return Progress::Complete; + } + + if active + .payload + .extend_from_slice(payload, GFP_KERNEL) + .is_err() + { + self.fail(); + return Progress::Failed(c"out of memory reassembling"); + } + + let received = active.payload.len() as u32; + let total = active.total; + + if received >= total { + let collected = core::mem::take(&mut active.payload); + self.finish(DeviceStatus::Answered(0), collected); + Progress::Complete + } else { + active.seq = active.seq.wrapping_add(1); + Progress::NeedMore(Continuation { + opcode: active_opcode, + received, + total, + seq: active.seq, + }) + } + } + + fn finish(&mut self, status: DeviceStatus, payload: KVec) { + let opcode = self.active.as_ref().map_or(0, |a| a.opcode); + self.active = None; + self.done = Some(Completed { + opcode, + status, + payload, + }); + } +} diff --git a/drivers/soc/apple/trusted.rs b/drivers/soc/apple/trusted.rs new file mode 100644 index 0000000000000..6162bc8af9d3c --- /dev/null +++ b/drivers/soc/apple/trusted.rs @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! A Linux trusted-key source backed by the SEP reference-key seal. +//! +//! Seal is a host-side ECIES encrypt to the machine ref-key's public point (no +//! enclave round trip). Unseal asks the enclave to ECIES-decrypt (op 0x22 +//! `"oecd"`) with the in-enclave private, which needs the login key bag reloaded +//! and unlocked first. The ref-key lifecycle lives on [`SepData`]; this is the +//! keyring glue. The config-dependent payload ABI is in `trusted_shim.c`. + +use core::sync::atomic::{AtomicPtr, Ordering}; + +use kernel::prelude::*; +use kernel::sync::Arc; +use kernel::types::ForeignOwnable; + +use crate::{Secret, SepData}; + +/// One opaque answer for every unseal rejection: telling a wrong blob, a foreign +/// machine's blob and a tampered blob apart would be an oracle. No name in the +/// Rust error set, hence the literal. +const EBADMSG: c_int = -74; + +/// The framework dispatches the ops with no context pointer, so `SepData` is +/// reached through this global. Holds the [`ForeignOwnable::into_foreign`] +/// pointer while registered; `null` otherwise, and the cmpxchg once-guard. +static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); + +extern "C" { + fn sep_tk_register( + init: unsafe extern "C" fn() -> c_int, + seal: unsafe extern "C" fn(*mut c_void, *mut c_char) -> c_int, + unseal: unsafe extern "C" fn(*mut c_void, *mut c_char) -> c_int, + random: unsafe extern "C" fn(*mut u8, usize) -> c_int, + exit: unsafe extern "C" fn(), + ) -> c_int; + fn sep_tk_unregister(); + + fn sep_tk_register_key_type() -> c_int; + fn sep_tk_unregister_key_type(); + + fn sep_tk_max_key_size() -> usize; + fn sep_tk_max_blob_size() -> usize; + + fn sep_tk_key_ptr(p: *mut c_void) -> *mut u8; + fn sep_tk_key_len(p: *const c_void) -> c_uint; + fn sep_tk_set_key_len(p: *mut c_void, n: c_uint); + + fn sep_tk_blob_ptr(p: *mut c_void) -> *mut u8; + fn sep_tk_blob_len(p: *const c_void) -> c_uint; + fn sep_tk_set_blob_len(p: *mut c_void, n: c_uint); +} + +/// Soundness rests on [`unregister`] resetting the framework's static calls +/// before it reclaims the `Arc`: no op runs after that, and while one runs the +/// pointer is live. +fn with_sep(f: impl FnOnce(&SepData) -> T) -> Option { + let ptr = SEP.load(Ordering::Acquire); + if ptr.is_null() { + return None; + } + // SAFETY: produced by `into_foreign` in `register`, reclaimed only by + // `unregister` after ops stop dispatching, so live for this borrow. + let borrow = unsafe { as ForeignOwnable>::borrow(ptr) }; + Some(f(&borrow)) +} + +/// # Safety +/// Called by the key-type core as the `init` op; takes no arguments. +unsafe extern "C" fn tk_init() -> c_int { + let rc = with_sep(|sep| { + dev_info!( + sep.dev, + "trusted-keys: registering \"trusted\" key type backed by the SEP ref-key seal\n" + ); + + // SAFETY: no preconditions; -EEXIST if already registered. + let krc = unsafe { sep_tk_register_key_type() }; + if krc != 0 { + dev_err!( + sep.dev, + "trusted-keys: register_key_type failed ({}); registration aborted\n", + krc + ); + } + krc + }); + rc.unwrap_or_else(|| ENODEV.to_errno()) +} + +/// `datablob` options (keyhandle/hash/policy) are ignored: no migration or +/// policy binding, so a key is bound only to the machine ref-key. +/// +/// # Safety +/// Called by the framework with a live `struct trusted_key_payload *p`. +unsafe extern "C" fn tk_seal(p: *mut c_void, _datablob: *mut c_char) -> c_int { + let rc = with_sep(|sep| { + // SAFETY: `p` is a live payload; key_len <= MAX_KEY_SIZE bytes at p->key. + let key_len = unsafe { sep_tk_key_len(p.cast_const()) } as usize; + // SAFETY: `p` is a live payload. + let key_ptr = unsafe { sep_tk_key_ptr(p) }; + // SAFETY: `key_ptr`/`key_len` describe `p->key`; read-only. + let key = unsafe { core::slice::from_raw_parts(key_ptr.cast_const(), key_len) }; + + let blob = match sep.refkey_seal_trusted(key) { + Ok(blob) => blob, + Err(e) => { + dev_warn!( + sep.dev, + "trusted-keys: seal failed ({:?}); the machine ref-key may not be established yet\n", + e + ); + return e.to_errno(); + } + }; + + // SAFETY: no preconditions. + let max_blob = unsafe { sep_tk_max_blob_size() }; + if blob.len() > max_blob { + dev_warn!( + sep.dev, + "trusted-keys: sealed blob {} byte(s) exceeds MAX_BLOB_SIZE {} byte(s); refusing\n", + blob.len(), + max_blob + ); + return E2BIG.to_errno(); + } + + // SAFETY: `p` is a live payload. + let dst = unsafe { sep_tk_blob_ptr(p) }; + // SAFETY: `dst` is `p->blob` (MAX_BLOB_SIZE bytes), blob.len() <= max_blob, + // no overlap. + unsafe { core::ptr::copy_nonoverlapping(blob.as_ptr(), dst, blob.len()) }; + // SAFETY: `p` is live. + unsafe { sep_tk_set_blob_len(p, blob.len() as c_uint) }; + + // Lengths only, never the key or blob bytes. + dev_info!( + sep.dev, + "trusted-keys: sealed {}-byte key into {}-byte enclave blob\n", + key_len, + blob.len() + ); + 0 + }); + rc.unwrap_or_else(|| ENODEV.to_errno()) +} + +/// # Safety +/// Called by the framework with a live `struct trusted_key_payload *p`. +unsafe extern "C" fn tk_unseal(p: *mut c_void, _datablob: *mut c_char) -> c_int { + let rc = with_sep(|sep| { + // SAFETY: `p` is a live payload; blob_len <= MAX_BLOB_SIZE bytes at p->blob. + let blob_len = unsafe { sep_tk_blob_len(p.cast_const()) } as usize; + // SAFETY: `p` is a live payload. + let blob_ptr = unsafe { sep_tk_blob_ptr(p) }; + // SAFETY: `blob_ptr`/`blob_len` describe `p->blob`; read-only. + let blob = unsafe { core::slice::from_raw_parts(blob_ptr.cast_const(), blob_len) }; + + // Every failure past here is one opaque EBADMSG. + let Ok(plain) = sep.refkey_unseal_trusted(blob) else { + return EBADMSG; + }; + let plain = Secret(plain); + + // SAFETY: no preconditions. + let max_key = unsafe { sep_tk_max_key_size() }; + if plain.is_empty() || plain.len() > max_key { + return EBADMSG; + } + + // SAFETY: `p` is a live payload. + let dst = unsafe { sep_tk_key_ptr(p) }; + // SAFETY: `dst` is `p->key` (MAX_KEY_SIZE + 1 bytes), plain.len() <= max_key, + // no overlap. + unsafe { core::ptr::copy_nonoverlapping(plain.as_ptr(), dst, plain.len()) }; + // SAFETY: `p` is live. + unsafe { sep_tk_set_key_len(p, plain.len() as c_uint) }; + + dev_info!( + sep.dev, + "trusted-keys: unsealed {}-byte enclave blob to {}-byte key\n", + blob_len, + plain.len() + ); + 0 + }); + rc.unwrap_or_else(|| ENODEV.to_errno()) +} + +/// Returns the byte count on success (not 0) per the framework contract, so the +/// caller's `ret != key_len` check passes. +/// +/// # Safety +/// Called by the framework with `key` writable for `key_len` bytes. +unsafe extern "C" fn tk_get_random(key: *mut u8, key_len: usize) -> c_int { + let rc = with_sep(|sep| { + if key_len == 0 { + return 0; + } + // SAFETY: the framework guarantees `key` is writable for `key_len` bytes. + let buf = unsafe { core::slice::from_raw_parts_mut(key, key_len) }; + match sep.sep_random(buf) { + Ok(()) => key_len as c_int, + Err(e) => e.to_errno(), + } + }); + rc.unwrap_or_else(|| ENODEV.to_errno()) +} + +/// # Safety +/// Called by the framework at most once per successful `init()`. +unsafe extern "C" fn tk_exit() { + // SAFETY: no preconditions; the key type was registered by `tk_init`. + unsafe { sep_tk_unregister_key_type() }; +} + +/// Idempotent: the cmpxchg claims the global slot once. On any failure the `Arc` +/// is reclaimed and the slot left clear, so a later call can retry. +pub(crate) fn register(sep: Arc) -> Result<()> { + let ptr = sep.into_foreign(); + + if SEP + .compare_exchange( + core::ptr::null_mut(), + ptr, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_err() + { + // SAFETY: `ptr` was just produced by `into_foreign` and never published. + drop(unsafe { as ForeignOwnable>::from_foreign(ptr) }); + return Err(EBUSY); + } + + // SAFETY: the shim stores these callbacks and calls + // `register_trusted_key_source`; valid for the module's lifetime. + let rc = unsafe { sep_tk_register(tk_init, tk_seal, tk_unseal, tk_get_random, tk_exit) }; + if rc != 0 { + // Nothing was wired, so no op can dispatch; clear the slot and reclaim. + let raw = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); + if !raw.is_null() { + // SAFETY: `raw` is the pointer we published; the ops are not wired. + drop(unsafe { as ForeignOwnable>::from_foreign(raw) }); + } + return Err(Error::from_errno(rc)); + } + Ok(()) +} + +/// Safe to call even if [`register`] never ran. Resets the framework's static +/// calls and runs `exit()` before reclaiming the `Arc`, so no op is in flight +/// against freed data once this returns. +pub(crate) fn unregister() { + if SEP.load(Ordering::Acquire).is_null() { + return; + } + // SAFETY: no preconditions; matched with the `sep_tk_register` above. + unsafe { sep_tk_unregister() }; + + let ptr = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); + if !ptr.is_null() { + // SAFETY: `ptr` came from `into_foreign` in `register`, and no op runs now. + drop(unsafe { as ForeignOwnable>::from_foreign(ptr) }); + } +} diff --git a/drivers/soc/apple/trusted_shim.c b/drivers/soc/apple/trusted_shim.c new file mode 100644 index 0000000000000..ed34794031f47 --- /dev/null +++ b/drivers/soc/apple/trusted_shim.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Dj */ +/* + * Apple SEP driver — trusted-key source shim (device-rooted sealing wired to + * the kernel trusted-key framework). + * + * An ABI the Rust bindings do not cover. 's struct + * trusted_key_payload is sized by MAX_KEY_SIZE / MAX_BLOB_SIZE and embeds an + * rcu_head, so its layout moves with the kernel config; and + * register_trusted_key_source / unregister_trusted_key_source, key_type_trusted + * and register_key_type() are C symbols absent from the Rust bindings. + * + * No sealing policy lives here: this file owns the struct handling and the + * registration only. init/seal/unseal/get_random/exit live in trusted.rs and + * are reached through the callbacks stored below. No key material is touched. + */ + +#include +#include +#include +#include + +#include "shim.h" + +/* + * register_trusted_key_source / unregister_trusted_key_source are declared by + * and exported by the trusted-key core. Not re-declared + * here: restating them risks a conflicting type (unregister returns void, not + * int). register validates src->{name,ops->{init,seal,unseal}}, honours + * trusted.source=, returns -EBUSY if another source is active, calls + * src->ops->init(), then wires the static calls; unregister resets those static + * calls, runs src->ops->exit(), and clears the active source. + */ + +/* Rust callbacks (trusted.rs); the sep_tk_*_fn types are declared in shim.h. */ +static sep_tk_init_fn rs_init; +static sep_tk_seal_fn rs_seal; +static sep_tk_unseal_fn rs_unseal; +static sep_tk_random_fn rs_random; +static sep_tk_exit_fn rs_exit; + +/* + * Trampolines for the framework's ops table, each forwarding to the stored Rust + * callback. Keeping these pointers in C means no Rust symbol is exported by + * name and the ops table stays a plain static initialiser. + */ +static int tk_init(void) +{ + return rs_init ? rs_init() : -ENODEV; +} + +static int tk_seal(struct trusted_key_payload *p, char *datablob) +{ + return rs_seal ? rs_seal(p, datablob) : -ENODEV; +} + +static int tk_unseal(struct trusted_key_payload *p, char *datablob) +{ + return rs_unseal ? rs_unseal(p, datablob) : -ENODEV; +} + +static int tk_get_random(unsigned char *key, size_t key_len) +{ + return rs_random ? rs_random(key, key_len) : -ENODEV; +} + +static void tk_exit(void) +{ + if (rs_exit) + rs_exit(); +} + +/* + * A writable array, not a string literal, so .name assigns cleanly whether the + * framework declares it char * or const char *. + */ +static char sep_tk_name[] = "applesep"; + +static struct trusted_key_ops sep_tk_ops = { + .migratable = 0, + .init = tk_init, + .seal = tk_seal, + .unseal = tk_unseal, + .get_random = tk_get_random, + .exit = tk_exit, +}; + +static struct trusted_key_source sep_tk_source = { + .name = sep_tk_name, + .ops = &sep_tk_ops, +}; + +/* + * Store the callbacks, then register. They must be in place first because + * register_trusted_key_source() calls init() synchronously. + */ +int sep_tk_register(sep_tk_init_fn init, sep_tk_seal_fn seal, + sep_tk_unseal_fn unseal, sep_tk_random_fn random, + sep_tk_exit_fn exit) +{ + rs_init = init; + rs_seal = seal; + rs_unseal = unseal; + rs_random = random; + rs_exit = exit; + return register_trusted_key_source(&sep_tk_source); +} + +void sep_tk_unregister(void) +{ + unregister_trusted_key_source(&sep_tk_source); + rs_init = NULL; + rs_seal = NULL; + rs_unseal = NULL; + rs_random = NULL; + rs_exit = NULL; +} + +/* + * Key-type lifecycle, owned by the source's init()/exit(), as the in-tree + * trusted-key sources (dcp, pkwm, tpm) do. + */ +int sep_tk_register_key_type(void) +{ + return register_key_type(&key_type_trusted); +} + +void sep_tk_unregister_key_type(void) +{ + unregister_key_type(&key_type_trusted); +} + +/* Compile-time facts the Rust side bounds-checks against. */ +size_t sep_tk_max_key_size(void) +{ + return MAX_KEY_SIZE; +} + +size_t sep_tk_max_blob_size(void) +{ + return MAX_BLOB_SIZE; +} + +/* + * Payload accessors. The struct layout tracks MAX_KEY_SIZE / MAX_BLOB_SIZE, so + * these wrappers keep the offsets in C where the header defines them. + */ +unsigned char *sep_tk_key_ptr(struct trusted_key_payload *p) +{ + return p->key; +} + +unsigned int sep_tk_key_len(const struct trusted_key_payload *p) +{ + return p->key_len; +} + +void sep_tk_set_key_len(struct trusted_key_payload *p, unsigned int n) +{ + p->key_len = n; +} + +unsigned char *sep_tk_blob_ptr(struct trusted_key_payload *p) +{ + return p->blob; +} + +unsigned int sep_tk_blob_len(const struct trusted_key_payload *p) +{ + return p->blob_len; +} + +void sep_tk_set_blob_len(struct trusted_key_payload *p, unsigned int n) +{ + p->blob_len = n; +} diff --git a/drivers/soc/apple/xarm.rs b/drivers/soc/apple/xarm.rs new file mode 100644 index 0000000000000..1c39467149b41 --- /dev/null +++ b/drivers/soc/apple/xarm.rs @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! The persistent-state service, `xarm`, endpoint `0x13`. + +#![allow(dead_code)] + +use crate::xarm::{XarmReply as Reply, XarmRequest as Request}; +use kernel::soc::apple::mailbox::Message; +use crate::store::{crc16_ccitt_false, Key, Store, MAX_VALUE}; +use kernel::prelude::*; + +pub(crate) use crate::proto::EP_XARM; +pub(crate) use crate::proto::EP_XARS; + +const OP_ROOT_READ: u8 = 0x00; +const OP_ROOT_WRITE: u8 = 0x01; +const OP_SESSION_READ: u8 = 0x05; +const OP_SESSION_WRITE: u8 = 0x06; +const OP_SESSION_DELETE: u8 = 0x07; +pub(crate) const OP_QUERY_PROTECTED: u8 = 0x0e; +const OP_GET_OS_UUID: u8 = 0x13; +const OP_NOTIFY_DISABLE_FIRST: u8 = 0x1d; +const OP_NOTIFY_DISABLE_LAST: u8 = 0x1f; + +pub(crate) const STATUS_OK: u8 = 0x00; +const STATUS_UNAVAILABLE: u8 = 0x02; +pub(crate) const STATUS_FAILED: u8 = 0x16; + +const ROOT_TYPE_BASE: u8 = 1; +const SESSION_TYPE_BASE: u8 = 3; + +/// Root read prefixes its value with seventeen zero bytes. +const ROOT_READ_PREFIX: usize = 17; + +/// `0x01` rejects when `(length >> 4) >= 0x7ff`. +const ROOT_WRITE_MAX: usize = 0x7fef; + +pub(crate) const PRIVATE_TYPE_OS_UUID: u8 = 0xF0; + +impl Reply { + fn ok(req: &Request, length: u16) -> Reply { + Reply { + tag: req.tag, + status: STATUS_OK, + length, + args: [0; 3], + } + } + + fn fail(req: &Request, status: u8) -> Reply { + Reply { + tag: req.tag, + status, + length: 0, + args: [0; 3], + } + } +} + +pub(crate) fn is_silent(opcode: u8) -> bool { + (OP_NOTIFY_DISABLE_FIRST..=OP_NOTIFY_DISABLE_LAST).contains(&opcode) +} + +pub(crate) fn needs_buffers(opcode: u8) -> bool { + opcode != OP_QUERY_PROTECTED +} + +pub(crate) fn opcode_name(opcode: u8) -> &'static CStr { + match opcode { + OP_ROOT_READ => c"ROOT_READ", + OP_ROOT_WRITE => c"ROOT_WRITE", + OP_SESSION_READ => c"SESSION_READ", + OP_SESSION_WRITE => c"SESSION_WRITE", + OP_SESSION_DELETE => c"SESSION_DELETE", + OP_QUERY_PROTECTED => c"QUERY_PROTECTED", + OP_GET_OS_UUID => c"GET_OS_UUID", + OP_NOTIFY_DISABLE_FIRST..=OP_NOTIFY_DISABLE_LAST => c"NOTIFY_DISABLE", + _ => c"UNKNOWN", + } +} + +fn root_key(args: &[u8; 3]) -> Key { + Key::root(ROOT_TYPE_BASE + (args[0] & 1)) +} + +fn session_type(args: &[u8; 3]) -> u8 { + SESSION_TYPE_BASE + (args[0] & 1) +} + +fn expected_crc(args: &[u8; 3]) -> u16 { + u16::from_le_bytes([args[1], args[2]]) +} + +fn uuid_from(bytes: &[u8]) -> [u8; 16] { + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&bytes[..16]); + uuid +} + +pub(crate) struct Serviced { + pub(crate) reply: Reply, + pub(crate) reply_bytes: usize, +} + +pub(crate) fn service( + req: &Request, + outbound: &[u8], + inbound: &mut [u8], + store: &mut Store, + protected_data: bool, + os_uuid: Option<[u8; 16]>, +) -> Serviced { + match req.opcode { + OP_ROOT_READ => { + let key = root_key(&req.args); + let value = match store.read(&key) { + Ok(v) => v, + Err(_) => { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + } + } + }; + + let Some(value) = value else { + return Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }; + }; + let total = ROOT_READ_PREFIX + value.len(); + if total > inbound.len() { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + inbound[..ROOT_READ_PREFIX].fill(0); + inbound[ROOT_READ_PREFIX..total].copy_from_slice(&value); + + Serviced { + reply: Reply::ok(req, total as u16), + reply_bytes: total, + } + } + + OP_ROOT_WRITE => { + if (req.length as usize) > ROOT_WRITE_MAX || outbound.len() > MAX_VALUE { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + if crc16_ccitt_false(outbound) != expected_crc(&req.args) { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + match store.write(&root_key(&req.args), outbound) { + Ok(()) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Err(_) => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } + } + + OP_SESSION_READ => { + if outbound.len() < 16 { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + let key = Key::new(session_type(&req.args), uuid_from(outbound)); + match store.read(&key) { + Ok(Some(value)) => { + if value.len() > inbound.len() { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + inbound[..value.len()].copy_from_slice(&value); + Serviced { + reply: Reply::ok(req, value.len() as u16), + reply_bytes: value.len(), + } + } + Ok(None) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Err(_) => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } + } + + OP_SESSION_WRITE => { + if outbound.len() < 16 { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + if crc16_ccitt_false(outbound) != expected_crc(&req.args) { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + let key = Key::new(session_type(&req.args), uuid_from(outbound)); + match store.write(&key, &outbound[16..]) { + Ok(()) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Err(_) => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } + } + + OP_SESSION_DELETE => { + if outbound.len() < 16 { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + let key = Key::new(session_type(&req.args), uuid_from(outbound)); + match store.delete(&key) { + Ok(true) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Ok(false) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Err(_) => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } + } + + OP_QUERY_PROTECTED => { + let mut reply = Reply::ok(req, 0); + reply.args[0] = u8::from(protected_data); + Serviced { + reply, + reply_bytes: 0, + } + } + + OP_GET_OS_UUID => match os_uuid { + Some(uuid) => { + if inbound.len() < 16 { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + inbound[..16].copy_from_slice(&uuid); + // Reply length must echo the request, not the 16 flushed bytes; setting 16 is wrong. + Serviced { + reply: Reply::ok(req, req.length), + reply_bytes: 16, + } + } + None => Serviced { + reply: Reply::fail(req, STATUS_UNAVAILABLE), + reply_bytes: 0, + }, + }, + + _ => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } +} + +pub(crate) fn make_uuid_v4(bytes: &mut [u8; 16]) { + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; +} + +pub(crate) fn os_uuid_key() -> Key { + Key::root(PRIVATE_TYPE_OS_UUID) +} + +pub(crate) struct XarmRequest { + pub(crate) tag: u8, + pub(crate) opcode: u8, + pub(crate) length: u16, + pub(crate) args: [u8; 3], +} + +pub(crate) struct XarmReply { + pub(crate) tag: u8, + pub(crate) status: u8, + pub(crate) length: u16, + pub(crate) args: [u8; 3], +} + +pub(crate) fn decode_xarm(msg: &Message) -> XarmRequest { + let b = msg.msg0.to_le_bytes(); + XarmRequest { + tag: b[1], + opcode: b[2], + length: u16::from_le_bytes([b[3], b[4]]), + args: [b[5], b[6], b[7]], + } +} + +pub(crate) fn encode_xarm_reply(reply: &XarmReply) -> Message { + let len = reply.length.to_le_bytes(); + Message { + msg0: u64::from_le_bytes([ + EP_XARM, + reply.tag, + reply.status, + len[0], + len[1], + reply.args[0], + reply.args[1], + reply.args[2], + ]), + msg1: 0, + } +} + +static_assert!(EP_XARM == 0x13); + +const OP_XARS_SETUP_OS_SESSION: u8 = 0x08; + +const OP_XARS_FETCH_KNOWN_SESSIONS: u8 = 0x04; + +pub(crate) struct XarsReply { + pub(crate) tag: u8, + pub(crate) status: u8, +} + +pub(crate) fn decode_xars_reply(msg: &Message) -> XarsReply { + let b = msg.msg0.to_le_bytes(); + XarsReply { + tag: b[1], + status: b[2], + } +} + +static_assert!(EP_XARS == 0x10); +static_assert!(OP_XARS_SETUP_OS_SESSION == 0x08); +static_assert!(OP_XARS_FETCH_KNOWN_SESSIONS == 0x04); diff --git a/include/keys/trusted-type.h b/include/keys/trusted-type.h index 03527162613f7..bd9d21f55f6cb 100644 --- a/include/keys/trusted-type.h +++ b/include/keys/trusted-type.h @@ -83,6 +83,9 @@ struct trusted_key_source { extern struct key_type key_type_trusted; +int register_trusted_key_source(struct trusted_key_source *src); +void unregister_trusted_key_source(struct trusted_key_source *src); + #define TRUSTED_DEBUG 0 #if TRUSTED_DEBUG diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c index 0b142d941cd2e..12fbef511bf69 100644 --- a/security/keys/trusted-keys/trusted_core.c +++ b/security/keys/trusted-keys/trusted_core.c @@ -61,6 +61,126 @@ DEFINE_STATIC_CALL_NULL(trusted_key_get_random, static void (*trusted_key_exit)(void); static unsigned char migratable; +static DEFINE_MUTEX(trusted_key_source_lock); +static bool trusted_key_source_active; +static struct trusted_key_source *trusted_key_source_registered; + +/* Defined below; registration needs it before its definition. */ +static int kernel_get_random(unsigned char *key, size_t key_len); + +/* + * Dispatch targets for when no source is registered. The call sites use + * static_call() directly, so the pointers must always be valid code; a NULL + * target would fault instead of returning an error. + */ +static int trusted_key_absent_seal(struct trusted_key_payload *p, char *datablob) +{ + return -ENODEV; +} + +static int trusted_key_absent_unseal(struct trusted_key_payload *p, char *datablob) +{ + return -ENODEV; +} + +static int trusted_key_absent_get_random(unsigned char *key, size_t key_len) +{ + return -ENODEV; +} + +/** + * register_trusted_key_source - offer a trust source from a module + * @src: the source. Must outlive its registration. + * + * Returns 0 on success, -EBUSY if a source is already active, -ENODEV if + * trusted.source= names a different one, or the source's own init() error. + */ +int register_trusted_key_source(struct trusted_key_source *src) +{ + int (*get_random)(unsigned char *key, size_t key_len); + int ret; + + if (!src || !src->name || !src->ops || !src->ops->init || + !src->ops->seal || !src->ops->unseal) + return -EINVAL; + + /* An explicit trusted.source= names one source; honour it. */ + if (trusted_key_source && strcmp(trusted_key_source, src->name)) + return -ENODEV; + + mutex_lock(&trusted_key_source_lock); + + if (trusted_key_source_active) { + mutex_unlock(&trusted_key_source_lock); + return -EBUSY; + } + + get_random = src->ops->get_random; + if (trusted_rng && strcmp(trusted_rng, "default")) { + if (!strcmp(trusted_rng, "kernel")) { + get_random = kernel_get_random; + } else if (strcmp(trusted_rng, src->name) || !get_random) { + mutex_unlock(&trusted_key_source_lock); + return -EINVAL; + } + } + if (!get_random) + get_random = kernel_get_random; + + ret = src->ops->init(); + if (ret) { + mutex_unlock(&trusted_key_source_lock); + return ret; + } + + static_call_update(trusted_key_seal, src->ops->seal); + static_call_update(trusted_key_unseal, src->ops->unseal); + static_call_update(trusted_key_get_random, get_random); + trusted_key_exit = src->ops->exit; + migratable = src->ops->migratable; + trusted_key_source_active = true; + trusted_key_source_registered = src; + + mutex_unlock(&trusted_key_source_lock); + pr_info("trusted_key: source '%s' registered\n", src->name); + return 0; +} +EXPORT_SYMBOL_GPL(register_trusted_key_source); + +/** + * unregister_trusted_key_source - withdraw a previously registered source + * @src: the source that was registered. + * + * Keys sealed by @src stay in the keyring and stop being usable, which is the + * intended outcome: the thing that could unseal them is gone. + */ +void unregister_trusted_key_source(struct trusted_key_source *src) +{ + mutex_lock(&trusted_key_source_lock); + + /* Only the source that registered may withdraw itself. */ + if (!trusted_key_source_active || src != trusted_key_source_registered) { + mutex_unlock(&trusted_key_source_lock); + return; + } + + static_call_update(trusted_key_seal, trusted_key_absent_seal); + static_call_update(trusted_key_unseal, trusted_key_absent_unseal); + static_call_update(trusted_key_get_random, trusted_key_absent_get_random); + + if (trusted_key_exit) + trusted_key_exit(); + trusted_key_exit = NULL; + migratable = 0; + trusted_key_source_active = false; + trusted_key_source_registered = NULL; + + mutex_unlock(&trusted_key_source_lock); + pr_info("trusted_key: source '%s' unregistered\n", + src && src->name ? src->name : "?"); +} +EXPORT_SYMBOL_GPL(unregister_trusted_key_source); + enum { Opt_err, Opt_new, Opt_load, Opt_update, @@ -337,7 +457,10 @@ static int kernel_get_random(unsigned char *key, size_t key_len) static int __init init_trusted(void) { int (*get_random)(unsigned char *key, size_t key_len); - int i, ret = 0; + /* -ENODEV when the built-in array is empty or nothing matches, so a + * later module source can still register instead of hitting -EBUSY. + */ + int i, ret = -ENODEV; for (i = 0; i < ARRAY_SIZE(trusted_key_sources); i++) { if (trusted_key_source && @@ -385,8 +508,21 @@ static int __init init_trusted(void) * encrypted_keys.ko depends on successful load of this module even if * trusted key implementation is not found. */ - if (ret == -ENODEV) + if (ret == -ENODEV) { + /* + * No built-in source. Leave the dispatch pointing at code that + * fails rather than at NULL, so a seal attempted before any + * module registers returns an error instead of faulting. + */ + static_call_update(trusted_key_seal, trusted_key_absent_seal); + static_call_update(trusted_key_unseal, trusted_key_absent_unseal); + static_call_update(trusted_key_get_random, + trusted_key_absent_get_random); return 0; + } + + if (!ret) + trusted_key_source_active = true; return ret; } From 7165ae9d98bf14713ba87d99d30d08b23e2a39c2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 12 Sep 2026 13:59:39 -0400 Subject: [PATCH 0031/1579] soc: apple: use shared xART gigalocker Serve SEP anti-replay records from the existing device-wide xART extent instead of a Linux-private seeded store. Validate the entire slot grid and both root records before SEP registration, preserve the store's copy-on-write ordering, and require both an explicit module parameter and a writable block mapping before any write. Keep Linux-only biometric metadata in a separate namespaced host store. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/Kconfig | 5 + drivers/soc/apple/keybag.rs | 2 +- drivers/soc/apple/refkey.rs | 34 ++- drivers/soc/apple/sbio.rs | 10 +- drivers/soc/apple/seed.rs | 113 --------- drivers/soc/apple/sep.rs | 84 ++++--- drivers/soc/apple/shim.h | 1 + drivers/soc/apple/shim.rs | 49 ++++ drivers/soc/apple/store.rs | 35 +-- drivers/soc/apple/store_shim.c | 19 ++ drivers/soc/apple/xarm.rs | 11 +- drivers/soc/apple/xart_store.rs | 402 ++++++++++++++++++++++++++++++++ 12 files changed, 572 insertions(+), 193 deletions(-) delete mode 100644 drivers/soc/apple/seed.rs create mode 100644 drivers/soc/apple/xart_store.rs diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 22cb2ba19b0cf..8977ecd5b863d 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -117,6 +117,11 @@ config APPLE_SEP device (enrol and match), exposes the SEP hardware RNG, and registers a SEP-backed trusted key source so keyctl can seal keys to the enclave. + The driver requires the existing machine-wide xART gigalocker at + /dev/mapper/sep-xart-gigalocker. It validates the complete store and both + root records before registering with SEP. Writes remain disabled unless + the xart_writes module parameter is set and the mapping is writable. + Say Y here if you have an Apple silicon Mac. config APPLE_PMP diff --git a/drivers/soc/apple/keybag.rs b/drivers/soc/apple/keybag.rs index 133a1ad5769db..90bed45619c50 100644 --- a/drivers/soc/apple/keybag.rs +++ b/drivers/soc/apple/keybag.rs @@ -11,7 +11,7 @@ use crate::shim; use crate::store::crc16_ccitt_false; use kernel::prelude::*; -pub(crate) const KEYBAG_PATH: &CStr = c"/var/lib/apple-sep-keybag.bin"; +pub(crate) const KEYBAG_PATH: &CStr = c"/var/lib/aurora-sep-keybag.bin"; #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum Slot { diff --git a/drivers/soc/apple/refkey.rs b/drivers/soc/apple/refkey.rs index c0456a604b38d..d2ffe586594ea 100644 --- a/drivers/soc/apple/refkey.rs +++ b/drivers/soc/apple/refkey.rs @@ -22,7 +22,8 @@ impl SepData { body.put_blob(der_set)?; let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; - let msg = crate::sks::encode_sks_perform_operation(self.sks_next_seq(), len).ok_or(EINVAL)?; + let msg = + crate::sks::encode_sks_perform_operation(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { name: crate::sks::SKS_PERFORM_OP_NAME, msg, @@ -54,8 +55,12 @@ impl SepData { let create_der = { let mut items: KVec<(&[u8], RV<'_>)> = KVec::new(); if items.push((b"o", RV::Utf8(b"oc")), GFP_KERNEL).is_err() - || items.push((b"bc", RV::Integer(PROTECTION_CLASS)), GFP_KERNEL).is_err() - || items.push((b"kt", RV::Integer(KEY_TYPE)), GFP_KERNEL).is_err() + || items + .push((b"bc", RV::Integer(PROTECTION_CLASS)), GFP_KERNEL) + .is_err() + || items + .push((b"kt", RV::Integer(KEY_TYPE)), GFP_KERNEL) + .is_err() { return None; } @@ -65,8 +70,7 @@ impl SepData { if cout.reply.status != 0 { return None; } - let cbody = - self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &cout)?; + let cbody = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &cout)?; let mut cf = proto::FieldCursor::new(cbody); let (Some(0), Some(cblob)) = (cf.i32(), cf.blob()) else { return None; @@ -78,13 +82,13 @@ impl SepData { fn refkey_pub(blob: &[u8]) -> Option<&[u8]> { let rk_tlv = crate::der::refkey_find(blob, b"rk")?; - let pub_raw = crate::der::refkey_find(rk_tlv, b"pub") - .and_then(crate::der::octet_string_body)?; + let pub_raw = + crate::der::refkey_find(rk_tlv, b"pub").and_then(crate::der::octet_string_body)?; (pub_raw.len() == refkey_seal::POINT_LEN).then_some(pub_raw) } pub(crate) fn sks_machine_refkey(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) { - const MACHINE_REFKEY_PATH: &CStr = c"/var/lib/apple-sep-refkey.bin"; + const MACHINE_REFKEY_PATH: &CStr = c"/var/lib/aurora-sep-refkey.bin"; if self.machine_refkey.lock().is_some() { return; @@ -198,7 +202,11 @@ impl SepData { let der = crate::der::encode_refkey_set(&items).ok()?; let out = self.sks_send(self.sks_req_refkey(handle.value(), &der))?; if out.reply.status != 0 { - dev_warn!(self.dev, "sks: ref-key attest sign failed (status {})\n", out.reply.status); + dev_warn!( + self.dev, + "sks: ref-key attest sign failed (status {})\n", + out.reply.status + ); return None; } let body = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &out)?; @@ -213,7 +221,7 @@ impl SepData { Some(owned) } - /// Attestation of key possession by signing proof; op `oa` (Apple-CA-chained) + /// Attestation of key possession by signing proof; op `oa` (attestation-chained) /// needs the SEP device attestation key, absent on a Linux-attached SEP. pub(crate) fn refkey_attest_sign(&self, challenge: &[u8]) -> Result<(KVec, KVec)> { self.ensure_machine_refkey()?; @@ -258,7 +266,11 @@ impl SepData { let der = crate::der::encode_refkey_set(&items).ok()?; let out = self.sks_send(self.sks_req_refkey(handle.value(), &der))?; if out.reply.status != 0 { - dev_warn!(self.dev, "trusted-keys: ref-key unseal failed (status {})\n", out.reply.status); + dev_warn!( + self.dev, + "trusted-keys: ref-key unseal failed (status {})\n", + out.reply.status + ); return None; } let body = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &out)?; diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 308f3b76c01b9..fff495601bb85 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -174,7 +174,7 @@ impl SepData { } let persisted = { let index = self.bio_index.lock(); - self.with_store(|store| index.persist(store)) + self.with_host_store(|store| index.persist(store)) }; match persisted { Some(Ok(())) => {}, @@ -218,7 +218,7 @@ impl SepData { // lock ordering: store lock outside the session lock (UAF otherwise) if outcome.is_ok() { let index = self.bio_index.lock(); - let saved = self.with_store(|store| index.persist(store)); + let saved = self.with_host_store(|store| index.persist(store)); match saved { Some(Ok(())) => {}, Some(Err(e)) => dev_err!( @@ -589,7 +589,7 @@ impl SepData { return Some(blob); } } - match self.with_store(|store| store.read(&store::Key::root(kind))) { + match self.with_host_store(|store| store.read(&store::Key::root(kind))) { Some(Ok(Some(blob))) => Some(blob), Some(Ok(None)) => None, Some(Err(_)) => { @@ -709,7 +709,7 @@ impl SepData { if blob.is_empty() { return false; } - match self.with_store(|store| store.write(&store::Key::root(PRIVATE_TYPE_LOCKOUT), &blob)) { + match self.with_host_store(|store| store.write(&store::Key::root(PRIVATE_TYPE_LOCKOUT), &blob)) { Some(Ok(())) => { true } @@ -2219,7 +2219,7 @@ impl SepData { let persisted = { let mut index = self.bio_index.lock(); index.remove(uuid); - self.with_store(|store| index.persist(store)) + self.with_host_store(|store| index.persist(store)) }; match persisted { Some(Ok(())) => {}, diff --git a/drivers/soc/apple/seed.rs b/drivers/soc/apple/seed.rs deleted file mode 100644 index d1793d8ebe361..0000000000000 --- a/drivers/soc/apple/seed.rs +++ /dev/null @@ -1,113 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -// Copyright 2026 Dj - -//! Importing the enclave's existing records. - -use crate::shim; -use crate::store::{Key, Store}; -use kernel::prelude::*; - -pub(crate) const SEED_PATH: &CStr = c"/var/lib/apple-sep-seed.bin"; - -const MAGIC: [u8; 4] = *b"AXRT"; -const HEADER_LEN: usize = 8; -const RECORD_HEADER_LEN: usize = 1 + 16 + 4; - -const MIN_RECORD_LEN: u32 = 1; -const MAX_RECORD_LEN: u32 = 0x8000; - -const MAX_SEED_BYTES: u64 = 1 << 20; - -const MAX_RECORDS: u32 = 64; - -const TYPE_ROOT_LOW: u8 = 1; -const TYPE_ROOT_HIGH: u8 = 2; -const TYPE_SESSION_HIGH: u8 = 4; - -pub(crate) struct Imported { - pub(crate) records: usize, - pub(crate) roots: usize, - pub(crate) sessions: usize, - pub(crate) bytes: usize, -} - -fn le32(buf: &[u8], off: usize) -> u32 { - u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) -} - -fn read_seed() -> Result> { - let file = shim::StoreFile::open_readonly(SEED_PATH)?; - let size = file.size()?; - if size < HEADER_LEN as u64 || size > MAX_SEED_BYTES { - return Err(EINVAL); - } - let mut buf = KVec::with_capacity(size as usize, GFP_KERNEL)?; - buf.resize(size as usize, 0, GFP_KERNEL)?; - file.read_exact(0, &mut buf)?; - Ok(buf) -} - -pub(crate) fn import(store: &mut Store) -> Result { - let buf = read_seed()?; - - if buf[..4] != MAGIC { - return Err(EINVAL); - } - let count = le32(&buf, 4); - if count == 0 || count > MAX_RECORDS { - return Err(EINVAL); - } - - let mut offsets: KVec<(u8, [u8; 16], usize, usize)> = KVec::new(); - let mut pos = HEADER_LEN; - for _ in 0..count { - if pos + RECORD_HEADER_LEN > buf.len() { - return Err(EINVAL); - } - let kind = buf[pos]; - let mut uuid = [0u8; 16]; - uuid.copy_from_slice(&buf[pos + 1..pos + 17]); - let len = le32(&buf, pos + 17); - pos += RECORD_HEADER_LEN; - - if !(TYPE_ROOT_LOW..=TYPE_SESSION_HIGH).contains(&kind) { - return Err(EINVAL); - } - if !(MIN_RECORD_LEN..=MAX_RECORD_LEN).contains(&len) { - return Err(EINVAL); - } - // Root records carry an all-zero UUID; anything else would be unreachable. - let is_root = kind == TYPE_ROOT_LOW || kind == TYPE_ROOT_HIGH; - if is_root && uuid != [0u8; 16] { - return Err(EINVAL); - } - let len = len as usize; - if pos + len > buf.len() { - return Err(EINVAL); - } - offsets.push((kind, uuid, pos, len), GFP_KERNEL)?; - pos += len; - } - - // Writes are keyed, so re-running after a crash simply replaces. - let mut result = Imported { - records: 0, - roots: 0, - sessions: 0, - bytes: 0, - }; - for (kind, uuid, at, len) in offsets { - store.write(&Key::new(kind, uuid), &buf[at..at + len])?; - result.records += 1; - result.bytes += len; - if kind == TYPE_ROOT_LOW || kind == TYPE_ROOT_HIGH { - result.roots += 1; - } else { - result.sessions += 1; - } - } - - // Mark consumed last: a crash above leaves the flag clear and the import reruns. - store.mark_seeded()?; - Ok(result) -} diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index a270c7cec92a5..68ddc2398a312 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -23,7 +23,6 @@ mod refkey_seal; mod rxring; mod sbio; mod scrd; -mod seed; mod sensor; mod shim; mod shmem; @@ -32,6 +31,7 @@ mod store; mod transfer; mod trusted; mod xarm; +mod xart_store; use kernel::{ device, @@ -794,7 +794,10 @@ struct SepData { bio_dev: Mutex>, #[pin] - store: Mutex>, + store: Mutex>, + + #[pin] + host_store: Mutex>, #[pin] xarm: Mutex, @@ -883,36 +886,44 @@ impl SepData { )?; let dma_ring = dma::Coherent::::zeroed_slice(dev, DMA_RING_SIZE, GFP_KERNEL)?; - let mut store = match store::Store::open() { - Ok(mut store) => { - - if !store.seeded() { - match seed::import(&mut store) { - Ok(_imported) => {}, - Err(e) => dev_err!( - dev, - "seed '{}' could not be imported ({:?}); store was never seeded, so the exchange stops after the first ROOT_READ and endpoint count stays at {}\n", - seed::SEED_PATH, - e, - ENDPOINTS_BEFORE_EXCHANGE - ), - } - } - + let xart_writes = *module_parameters::xart_writes.value() != 0; + let store = match xart_store::Store::open(xart_writes) { + Ok(store) => { + let (slots, records, revision, malformed, duplicates, repaired, writable) = + store.summary(); + dev_info!( + dev, + "xART: {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", + slots, + records, + revision, + malformed, + duplicates, + repaired, + if writable { "ENABLED" } else { "disabled" } + ); Some(store) } Err(e) => { dev_err!( dev, - "could not open the backing store '{}': {:?}; persistent-state exchange cannot run\n", - store::STORE_PATH, + "shared xART mapping '{}' is unavailable or invalid: {:?}\n", + xart_store::STORE_PATH, e ); + return Err(e); + } + }; + + let mut host_store = match store::Store::open() { + Ok(store) => Some(store), + Err(e) => { + dev_warn!(dev, "Linux host-state store unavailable: {:?}\n", e); None } }; - let bio_index = match store.as_mut().map(bio::IdentityIndex::load) { + let bio_index = match host_store.as_mut().map(bio::IdentityIndex::load) { Some(Ok(index)) => index, Some(Err(_)) => bio::IdentityIndex::new(), None => bio::IdentityIndex::new(), @@ -968,6 +979,7 @@ impl SepData { bio_index <- new_mutex!(bio_index), bio_dev <- new_mutex!(None), store <- new_mutex!(store), + host_store <- new_mutex!(host_store), xarm <- new_mutex!(XarmState::new()), rng <- new_mutex!(None), machine_refkey <- new_mutex!(None), @@ -1233,17 +1245,23 @@ impl SepData { Ok(()) } - fn with_store(&self, f: impl FnOnce(&mut store::Store) -> R) -> Option { + fn with_store(&self, f: impl FnOnce(&mut xart_store::Store) -> R) -> Option { let mut guard = self.store.lock(); + let store: &mut Option = &mut guard; + store.as_mut().map(f) + } + + fn with_host_store(&self, f: impl FnOnce(&mut store::Store) -> R) -> Option { + let mut guard = self.host_store.lock(); let store: &mut Option = &mut guard; store.as_mut().map(f) } // pre-generate: an entropy draw on the drain path would deadlock on its own reply fn prepare_os_uuid(&self) { - let key = xarm::os_uuid_key(); + let key = store::Key::root(0xf0); - let existing = match self.with_store(|store| store.read(&key)) { + let existing = match self.with_host_store(|store| store.read(&key)) { Some(result) => result, None => return, }; @@ -1258,17 +1276,12 @@ impl SepData { } let mut uuid = [0u8; 16]; - for word in 0..4 { - match self.get_entropy_word() { - Ok(v) => uuid[word * 4..word * 4 + 4].copy_from_slice(&v.to_le_bytes()), - Err(_) => { - return; - } - } + if shim::random_bytes(&mut uuid).is_err() { + return; } xarm::make_uuid_v4(&mut uuid); - match self.with_store(|store| store.write(&key, &uuid)) { + match self.with_host_store(|store| store.write(&key, &uuid)) { Some(Ok(())) => {} Some(Err(_)) => { return; @@ -1971,6 +1984,7 @@ impl SepData { *self.mbox.lock() = None; let _ = self.store.lock().take(); + let _ = self.host_store.lock().take(); for slot in [&self.ool_xarm, &self.ool_sbio, &self.ool_sks] { if let Some(buffers) = slot.lock().take() { @@ -2208,4 +2222,10 @@ module! { name: "apple_sep", description: "Apple SEP coprocessor: warm attach, endpoint discovery and tag-correlated control endpoint", license: "Dual MIT/GPL", + params: { + xart_writes: u8 { + default: 0, + description: "Allow writes to the validated shared xART mapping", + }, + }, } diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 2abb13765bad7..e39f036142825 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -33,6 +33,7 @@ void sep_hwrng_unregister(void *mem); void *sep_store_open(const char *path); void *sep_store_open_trunc(const char *path); void *sep_store_open_ro(const char *path); +void *sep_store_open_block(const char *path, int writable); void sep_store_close(void *handle); long long sep_store_size(void *handle); long sep_store_read(void *handle, long long off, void *buf, size_t len); diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index 2968a04943e99..a615b3aee0a96 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -9,6 +9,7 @@ extern "C" { fn sep_store_open(path: *const c_char) -> *mut c_void; fn sep_store_open_trunc(path: *const c_char) -> *mut c_void; fn sep_store_open_ro(path: *const c_char) -> *mut c_void; + fn sep_store_open_block(path: *const c_char, writable: c_int) -> *mut c_void; fn sep_store_close(handle: *mut c_void); fn sep_store_size(handle: *mut c_void) -> i64; fn sep_store_read(handle: *mut c_void, off: i64, buf: *mut c_void, len: usize) @@ -20,6 +21,12 @@ extern "C" { len: usize, ) -> c_long; fn sep_store_sync(handle: *mut c_void) -> c_int; + fn sep_random_bytes(buf: *mut c_void, len: usize) -> c_int; +} + +pub(crate) fn random_bytes(buf: &mut [u8]) -> Result<()> { + // SAFETY: `buf` is writable for exactly `buf.len()` bytes. + kernel::error::to_result(unsafe { sep_random_bytes(buf.as_mut_ptr().cast(), buf.len()) }) } /// Backing-store file handle. @@ -70,6 +77,16 @@ impl StoreFile { Ok(StoreFile { handle }) } + pub(crate) fn open_block(path: &CStr, writable: bool) -> Result { + // SAFETY: `path` is NUL-terminated; the shim accepts only a block + // device whose global read-only state matches `writable`. + let handle = unsafe { sep_store_open_block(path.as_char_ptr(), c_int::from(writable)) }; + if handle.is_null() { + return Err(ENODEV); + } + Ok(StoreFile { handle }) + } + pub(crate) fn size(&self) -> Result { // SAFETY: `handle` is live per the type invariant. let n = unsafe { sep_store_size(self.handle) }; @@ -100,6 +117,22 @@ impl StoreFile { Ok(()) } + pub(crate) fn read_block_exact(&self, off: u64, buf: &mut [u8]) -> Result<()> { + // SAFETY: `handle` remains live and `buf` is writable for its length. + let n = result_of(unsafe { + sep_store_read( + self.handle, + off as i64, + buf.as_mut_ptr().cast::(), + buf.len(), + ) + })?; + if n != buf.len() { + return Err(EIO); + } + Ok(()) + } + pub(crate) fn write_all(&self, off: u64, buf: &[u8]) -> Result<()> { let mut done = 0usize; while done < buf.len() { @@ -120,6 +153,22 @@ impl StoreFile { Ok(()) } + pub(crate) fn write_block_exact(&self, off: u64, buf: &[u8]) -> Result<()> { + // SAFETY: `handle` remains live and `buf` is readable for its length. + let n = result_of(unsafe { + sep_store_write( + self.handle, + off as i64, + buf.as_ptr().cast::(), + buf.len(), + ) + })?; + if n != buf.len() { + return Err(EIO); + } + Ok(()) + } + pub(crate) fn sync(&self) -> Result<()> { // SAFETY: `handle` is live per the type invariant. kernel::error::to_result(unsafe { sep_store_sync(self.handle) }) diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index ef4140ec781dc..26f6bf1dc8739 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -1,14 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj -//! Host-owned slotted backing store for the SEP's persistent records: a -//! compile-time path exclusive to this driver, re-initialised and reseeded on a -//! magic mismatch. Re-initialised is never served empty — an empty store halts the SEP. +//! Linux-only state. SEP anti-replay records live in `xart_store`. use crate::shim; use kernel::prelude::*; -pub(crate) const STORE_PATH: &CStr = c"/var/lib/apple-sep-state.bin"; +pub(crate) const STORE_PATH: &CStr = c"/var/lib/aurora-sep-host-state.bin"; const BLOCK_SIZE: usize = 0x8000; const BLOCK_COUNT: usize = 72; @@ -16,7 +14,7 @@ pub(crate) const STORE_SIZE: usize = BLOCK_SIZE * BLOCK_COUNT; const SLOT_COUNT: usize = BLOCK_COUNT - 1; pub(crate) const MAX_VALUE: usize = BLOCK_SIZE; -const MAGIC: [u8; 16] = *b"APPLE-SEP-STOR01"; +const MAGIC: [u8; 16] = *b"AURORA-SEP-STOR\x01"; const VERSION: u32 = 1; #[derive(Clone, Copy, PartialEq, Eq)] @@ -80,8 +78,6 @@ const SB_GENERATION: usize = 28; const SB_INTENT_KIND: usize = 36; const SB_INTENT_SLOT: usize = 37; const SB_INTENT_TYPE: usize = 39; -// In previously reserved space, so an older store reads it as zero (unseeded). -const SB_SEEDED: usize = 40; const SB_SLOT_TABLE: usize = 64; const SLOT_ENTRY_SIZE: usize = 24; @@ -96,7 +92,6 @@ pub(crate) struct Store { generation: u64, pub(crate) recovered: bool, pub(crate) fresh: bool, - seeded: bool, } fn le32(buf: &[u8], off: usize) -> u32 { @@ -119,7 +114,6 @@ impl Store { generation: 0, recovered: false, fresh: false, - seeded: false, }; let size = store.file.size()?; @@ -142,8 +136,6 @@ impl Store { } store.generation = le64(&sb, SB_GENERATION); - store.seeded = sb[SB_SEEDED] != 0; - for i in 0..SLOT_COUNT { let off = SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE; let mut uuid = [0u8; 16]; @@ -174,7 +166,6 @@ impl Store { self.slots = [Slot::FREE; SLOT_COUNT]; self.generation = 1; self.fresh = true; - self.seeded = false; let mut zero = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; zero.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; @@ -206,7 +197,6 @@ impl Store { Intent::Write { slot, kind } => (INTENT_WRITE, slot, kind), Intent::Delete { slot, kind } => (INTENT_DELETE, slot, kind), }; - sb[SB_SEEDED] = u8::from(self.seeded); sb[SB_INTENT_KIND] = ikind; sb[SB_INTENT_SLOT..SB_INTENT_SLOT + 2].copy_from_slice(&islot.to_le_bytes()); sb[SB_INTENT_TYPE] = itype; @@ -235,6 +225,9 @@ impl Store { } pub(crate) fn read(&mut self, key: &Key) -> Result>> { + if !(0xf0..=0xf5).contains(&key.kind) { + return Err(EINVAL); + } let Some(idx) = self.find(key) else { return Ok(None); }; @@ -248,6 +241,9 @@ impl Store { } pub(crate) fn write(&mut self, key: &Key, value: &[u8]) -> Result<()> { + if !(0xf0..=0xf5).contains(&key.kind) { + return Err(EINVAL); + } if value.len() > MAX_VALUE { return Err(ENOSPC); } @@ -283,6 +279,9 @@ impl Store { } pub(crate) fn delete(&mut self, key: &Key) -> Result { + if !(0xf0..=0xf5).contains(&key.kind) { + return Err(EINVAL); + } let Some(idx) = self.find(key) else { return Ok(false); }; @@ -299,16 +298,6 @@ impl Store { Ok(true) } - // Gates import, not store-emptiness: a store emptied after the SEP moved on must not be reseeded. - pub(crate) fn seeded(&self) -> bool { - self.seeded - } - - pub(crate) fn mark_seeded(&mut self) -> Result<()> { - self.seeded = true; - self.commit() - } - } pub(crate) const fn crc16_ccitt_false(data: &[u8]) -> u16 { diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c index 33ed552f54798..715c7c9c6f8db 100644 --- a/drivers/soc/apple/store_shim.c +++ b/drivers/soc/apple/store_shim.c @@ -11,6 +11,7 @@ */ #include +#include #include #include #include @@ -65,6 +66,22 @@ void *sep_store_open_ro(const char *path) return IS_ERR(f) ? NULL : f; } +void *sep_store_open_block(const char *path, int writable) +{ + struct file *f = open_as_kernel(path, + writable ? O_RDWR | O_LARGEFILE : O_RDONLY | O_LARGEFILE, + 0); + + if (IS_ERR(f)) + return NULL; + if (!S_ISBLK(file_inode(f)->i_mode) || + bdev_read_only(file_bdev(f)) == !!writable) { + filp_close(f, NULL); + return NULL; + } + return f; +} + void sep_store_close(void *handle) { if (handle) @@ -76,6 +93,8 @@ long long sep_store_size(void *handle) { struct file *f = handle; + if (S_ISBLK(file_inode(f)->i_mode)) + return bdev_nr_bytes(file_bdev(f)); return i_size_read(file_inode(f)); } diff --git a/drivers/soc/apple/xarm.rs b/drivers/soc/apple/xarm.rs index 1c39467149b41..acb912ea5d5e5 100644 --- a/drivers/soc/apple/xarm.rs +++ b/drivers/soc/apple/xarm.rs @@ -7,7 +7,8 @@ use crate::xarm::{XarmReply as Reply, XarmRequest as Request}; use kernel::soc::apple::mailbox::Message; -use crate::store::{crc16_ccitt_false, Key, Store, MAX_VALUE}; +use crate::store::crc16_ccitt_false; +use crate::xart_store::{Key, Store, MAX_VALUE}; use kernel::prelude::*; pub(crate) use crate::proto::EP_XARM; @@ -36,8 +37,6 @@ const ROOT_READ_PREFIX: usize = 17; /// `0x01` rejects when `(length >> 4) >= 0x7ff`. const ROOT_WRITE_MAX: usize = 0x7fef; -pub(crate) const PRIVATE_TYPE_OS_UUID: u8 = 0xF0; - impl Reply { fn ok(req: &Request, length: u16) -> Reply { Reply { @@ -172,7 +171,7 @@ pub(crate) fn service( } OP_SESSION_READ => { - if outbound.len() < 16 { + if outbound.len() != 16 { return Serviced { reply: Reply::fail(req, STATUS_FAILED), reply_bytes: 0, @@ -296,10 +295,6 @@ pub(crate) fn make_uuid_v4(bytes: &mut [u8; 16]) { bytes[8] = (bytes[8] & 0x3f) | 0x80; } -pub(crate) fn os_uuid_key() -> Key { - Key::root(PRIVATE_TYPE_OS_UUID) -} - pub(crate) struct XarmRequest { pub(crate) tag: u8, pub(crate) opcode: u8, diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs new file mode 100644 index 0000000000000..d4a7a3566ec13 --- /dev/null +++ b/drivers/soc/apple/xart_store.rs @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +// Copyright 2026 Aurora Silicon + +//! Apple's device-wide xART gigalocker record store. +//! +//! The APFS locator exposes the existing `.gl` file as a block device. This +//! module implements the record validation, duplicate repair, lookup and +//! copy-on-write ordering used by AppleSEPManager. It never creates storage +//! and it can be opened read-only for safe inspection and bring-up. + +use crate::shim; +use kernel::prelude::*; + +pub(crate) const STORE_PATH: &CStr = c"/dev/mapper/sep-xart-gigalocker"; + +const BLOCK_SIZE: usize = 0x1000; +const SLOT_SIZE: usize = 0x9000; +const HEADER_SIZE: usize = 0x22; +const DELETE_SIZE: usize = BLOCK_SIZE; +/// Size of the APFS raw extent located on the target machine. +/// +/// Accepting a larger block device would make a bad device-mapper table a +/// corruption hazard. Accepting a smaller one could silently truncate the +/// slot grid. A future locator for a machine with a different extent size must +/// pass that size through an explicit, reviewed interface instead of weakening +/// this check. +const STORE_SIZE: u64 = 0x600000; +const MAX_SLOTS: usize = 4096; + +pub(crate) const MAX_VALUE: usize = 0x8000; + +const KEY_KIND: usize = 0x01; +const KEY_UUID: usize = 0x02; +const LENGTH: usize = 0x12; +const CRC: usize = 0x16; +const REVISION: usize = 0x1a; +const PAYLOAD: usize = HEADER_SIZE; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct Key { + pub(crate) kind: u8, + pub(crate) uuid: [u8; 16], +} + +impl Key { + pub(crate) const fn new(kind: u8, uuid: [u8; 16]) -> Key { + Key { kind, uuid } + } + + pub(crate) const fn root(kind: u8) -> Key { + Key { + kind, + uuid: [0; 16], + } + } +} + +#[derive(Clone, Copy)] +struct Slot { + used: bool, + key: Key, + len: u32, + crc: u32, + revision: u64, +} + +impl Slot { + const FREE: Slot = Slot { + used: false, + key: Key::root(0), + len: 0, + crc: 0, + revision: 0, + }; + + fn matches(&self, key: &Key) -> bool { + self.used && self.key == *key + } +} + +pub(crate) struct Store { + file: shim::StoreFile, + slots: KVec, + revision: u64, + writes_enabled: bool, + valid_records: usize, + malformed_records: usize, + duplicate_records: usize, + repaired_records: usize, +} + +fn le32(bytes: &[u8], off: usize) -> u32 { + u32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]]) +} + +fn le64(bytes: &[u8], off: usize) -> u64 { + u64::from_le_bytes([ + bytes[off], + bytes[off + 1], + bytes[off + 2], + bytes[off + 3], + bytes[off + 4], + bytes[off + 5], + bytes[off + 6], + bytes[off + 7], + ]) +} + +fn valid_key(key: &Key) -> bool { + (1..=4).contains(&key.kind) && (key.kind > 2 || key.uuid == [0; 16]) +} + +impl Store { + pub(crate) fn open(writes_enabled: bool) -> Result { + Self::open_at(STORE_PATH, writes_enabled) + } + + /// Opens a caller-selected block mapping. + /// + /// Production always uses [`STORE_PATH`]. The separate xART self-test + /// module uses this entry point with its fixed loop-only mapper name, so it + /// can execute the real parser and write ordering without enabling SEP. + pub(crate) fn open_at(path: &CStr, writes_enabled: bool) -> Result { + let file = shim::StoreFile::open_block(path, writes_enabled)?; + let size = file.size()?; + if size != STORE_SIZE || size % BLOCK_SIZE as u64 != 0 { + return Err(EINVAL); + } + let count = (size / SLOT_SIZE as u64) as usize; + if count == 0 || count > MAX_SLOTS { + return Err(EINVAL); + } + + let mut slots = KVec::with_capacity(count, GFP_KERNEL)?; + for _ in 0..count { + slots.push(Slot::FREE, GFP_KERNEL)?; + } + let mut store = Store { + file, + slots, + revision: 0, + // Discovery is always read-only. Do not arm even repair writes + // until the required Apple root records have validated. + writes_enabled: false, + valid_records: 0, + malformed_records: 0, + duplicate_records: 0, + repaired_records: 0, + }; + store.scan()?; + // Serving an empty or unrelated 6 MiB mapping is the failure mode that + // originally desynchronised SEP from macOS. This Linux driver is never + // the authority that provisions a blank device-wide store. Require + // both existing root families before any mailbox registration can run. + if store.find(&Key::root(1)).is_none() || store.find(&Key::root(2)).is_none() { + return Err(ENODATA); + } + store.writes_enabled = writes_enabled; + if writes_enabled { + store.repair_disk()?; + } + Ok(store) + } + + fn slot_offset(slot: usize) -> u64 { + (slot * SLOT_SIZE) as u64 + } + + fn scan(&mut self) -> Result<()> { + let mut raw = KVec::with_capacity(SLOT_SIZE, GFP_KERNEL)?; + raw.resize(SLOT_SIZE, 0, GFP_KERNEL)?; + + for idx in 0..self.slots.len() { + self.file + .read_block_exact(Self::slot_offset(idx), &mut raw)?; + let kind = raw[KEY_KIND]; + if kind == 0 { + continue; + } + + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&raw[KEY_UUID..KEY_UUID + 16]); + let key = Key { kind, uuid }; + let len = le32(&raw, LENGTH) as usize; + let crc = le32(&raw, CRC); + let revision = le64(&raw, REVISION); + let valid = valid_key(&key) + && (1..=MAX_VALUE).contains(&len) + && crc32_ieee(&raw[PAYLOAD..PAYLOAD + len.min(MAX_VALUE)]) == crc; + + if !valid { + self.malformed_records += 1; + continue; + } + + self.revision = self.revision.max(revision); + let candidate = Slot { + used: true, + key, + len: len as u32, + crc, + revision, + }; + + if let Some(old) = self.find(&key) { + self.duplicate_records += 1; + // Equal revisions keep the later physical slot, matching the + // forward scan in AppleSEPManager's fixup pass. + if revision >= self.slots[old].revision { + self.slots[old] = Slot::FREE; + self.slots[idx] = candidate; + } + } else { + self.slots[idx] = candidate; + } + } + + self.valid_records = self.slots.iter().filter(|slot| slot.used).count(); + Ok(()) + } + + /// Removes malformed records and duplicate losers only after the mapping + /// has passed its complete read-only scan and both Apple roots exist. + fn repair_disk(&mut self) -> Result<()> { + let mut header = KVec::with_capacity(DELETE_SIZE, GFP_KERNEL)?; + header.resize(DELETE_SIZE, 0, GFP_KERNEL)?; + + for idx in 0..self.slots.len() { + if self.slots[idx].used { + continue; + } + self.file + .read_block_exact(Self::slot_offset(idx), &mut header)?; + if header[KEY_KIND] != 0 { + self.delete_slot(idx)?; + self.repaired_records += 1; + } + } + Ok(()) + } + + fn find(&self, key: &Key) -> Option { + let mut best: Option = None; + for (idx, slot) in self.slots.iter().enumerate() { + if !slot.matches(key) { + continue; + } + if best.is_none_or(|old| slot.revision >= self.slots[old].revision) { + best = Some(idx); + } + } + best + } + + fn find_free(&self, skip: usize) -> Option { + self.slots + .iter() + .enumerate() + .filter(|(_, slot)| !slot.used) + .nth(skip) + .map(|(idx, _)| idx) + } + + fn delete_slot(&self, slot: usize) -> Result<()> { + let mut zero = KVec::with_capacity(DELETE_SIZE, GFP_KERNEL)?; + zero.resize(DELETE_SIZE, 0, GFP_KERNEL)?; + self.file + .write_block_exact(Self::slot_offset(slot), &zero)?; + self.file.sync() + } + + pub(crate) fn read(&mut self, key: &Key) -> Result>> { + if !valid_key(key) { + return Err(EINVAL); + } + let Some(idx) = self.find(key) else { + return Ok(None); + }; + let slot = self.slots[idx]; + let mut raw = KVec::with_capacity(SLOT_SIZE, GFP_KERNEL)?; + raw.resize(SLOT_SIZE, 0, GFP_KERNEL)?; + self.file + .read_block_exact(Self::slot_offset(idx), &mut raw)?; + + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&raw[KEY_UUID..KEY_UUID + 16]); + let disk_key = Key { + kind: raw[KEY_KIND], + uuid, + }; + let len = le32(&raw, LENGTH) as usize; + if disk_key != *key + || len != slot.len as usize + || le32(&raw, CRC) != slot.crc + || le64(&raw, REVISION) != slot.revision + || crc32_ieee(&raw[PAYLOAD..PAYLOAD + len]) != slot.crc + { + return Err(EIO); + } + + let mut value = KVec::with_capacity(len, GFP_KERNEL)?; + value.extend_from_slice(&raw[PAYLOAD..PAYLOAD + len], GFP_KERNEL)?; + Ok(Some(value)) + } + + pub(crate) fn write(&mut self, key: &Key, value: &[u8]) -> Result<()> { + if !self.writes_enabled { + return Err(EROFS); + } + if !valid_key(key) || value.is_empty() || value.len() > MAX_VALUE { + return Err(EINVAL); + } + + let old = self.find(key); + // Apple skips the first free slot when creating a new key, but uses the + // first free slot for replacement. + let fresh = self.find_free(usize::from(old.is_none())).ok_or(ENOSPC)?; + let revision = self.revision.checked_add(1).ok_or(EINVAL)?; + let crc = crc32_ieee(value); + + let mut raw = KVec::with_capacity(SLOT_SIZE, GFP_KERNEL)?; + raw.resize(SLOT_SIZE, 0, GFP_KERNEL)?; + raw[KEY_KIND] = key.kind; + raw[KEY_UUID..KEY_UUID + 16].copy_from_slice(&key.uuid); + raw[LENGTH..LENGTH + 4].copy_from_slice(&(value.len() as u32).to_le_bytes()); + raw[CRC..CRC + 4].copy_from_slice(&crc.to_le_bytes()); + raw[REVISION..REVISION + 8].copy_from_slice(&revision.to_le_bytes()); + raw[PAYLOAD..PAYLOAD + value.len()].copy_from_slice(value); + + // The new record becomes authoritative only after its complete slot is + // durable. The old record is then removed and flushed separately. + self.file + .write_block_exact(Self::slot_offset(fresh), &raw)?; + self.file.sync()?; + self.slots[fresh] = Slot { + used: true, + key: *key, + len: value.len() as u32, + crc, + revision, + }; + self.revision = revision; + + if let Some(old) = old { + self.delete_slot(old)?; + self.slots[old] = Slot::FREE; + } + self.valid_records = self.slots.iter().filter(|slot| slot.used).count(); + Ok(()) + } + + pub(crate) fn delete(&mut self, key: &Key) -> Result { + if !valid_key(key) { + return Err(EINVAL); + } + let Some(idx) = self.find(key) else { + return Ok(false); + }; + if !self.writes_enabled { + return Err(EROFS); + } + self.delete_slot(idx)?; + self.slots[idx] = Slot::FREE; + self.valid_records -= 1; + Ok(true) + } + + pub(crate) fn summary(&self) -> (usize, usize, u64, usize, usize, usize, bool) { + ( + self.slots.len(), + self.valid_records, + self.revision, + self.malformed_records, + self.duplicate_records, + self.repaired_records, + self.writes_enabled, + ) + } +} + +/// Reflected CRC-32/ISO-HDLC (the IEEE CRC-32 used in gigalocker records). +pub(crate) const fn crc32_ieee(data: &[u8]) -> u32 { + let mut crc = 0xffff_ffffu32; + let mut i = 0; + while i < data.len() { + crc ^= data[i] as u32; + let mut bit = 0; + while bit < 8 { + crc = if crc & 1 != 0 { + (crc >> 1) ^ 0xedb8_8320 + } else { + crc >> 1 + }; + bit += 1; + } + i += 1; + } + crc ^ 0xffff_ffff +} + +static_assert!(crc32_ieee(b"123456789") == 0xcbf4_3926); From e7db648a80649eafa7d05ac01c8c35ad8e44cd17 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 14:57:22 -0400 Subject: [PATCH 0032/1579] soc: apple: add production SEP FileVault integration Unwrap a FileVault volume's key through the enclave and load the volume's class keys into the enclave's volatile state, then drive the storage controller's inline AES-XTS with the wrapped key so an encrypted APFS volume is read and written with no plaintext key material on the application processor. The volume and key-bag records come from plaintext APFS metadata; only wrapped keys cross to the enclave. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- block/blk-crypto.c | 4 + drivers/nvme/host/Kconfig | 1 + drivers/nvme/host/apple.c | 67 +- drivers/nvme/host/core.c | 5 + drivers/nvme/host/nvme.h | 3 + drivers/soc/apple/Kconfig | 4 + drivers/soc/apple/Makefile | 4 +- drivers/soc/apple/bio.rs | 9 +- drivers/soc/apple/control.rs | 46 +- drivers/soc/apple/crypto_shim.c | 29 +- drivers/soc/apple/der.rs | 6 +- drivers/soc/apple/dt.rs | 33 +- drivers/soc/apple/fv.rs | 1106 ++++++++++++++-- drivers/soc/apple/fv_shim.c | 315 +++++ drivers/soc/apple/image.rs | 47 +- drivers/soc/apple/keybag.rs | 40 +- drivers/soc/apple/proto.rs | 33 +- drivers/soc/apple/sbio.rs | 279 ++-- drivers/soc/apple/scrd.rs | 8 +- drivers/soc/apple/sensor.rs | 15 +- drivers/soc/apple/sep.rs | 345 ++--- drivers/soc/apple/shim.h | 47 +- drivers/soc/apple/shim.rs | 20 +- drivers/soc/apple/sks.rs | 450 +++---- drivers/soc/apple/store.rs | 36 +- drivers/soc/apple/transfer.rs | 58 +- drivers/soc/apple/work_shim.c | 14 + drivers/soc/apple/xarm.rs | 49 +- drivers/soc/apple/xart_store.rs | 22 +- drivers/spi/spi-apple.c | 87 ++ include/linux/apple-sep-fv.h | 54 + security/keys/trusted-keys/trusted_core.c | 21 +- tools/aurora-sep/README.md | 34 + tools/aurora-sep/aurora-sep.service | 13 + tools/aurora-sep/fprintd-aurora.conf | 2 + tools/aurora-sep/load-driver | 40 + .../libfprint-1.94.100-apple-sep.patch | 1179 +++++++++++++++++ 37 files changed, 3461 insertions(+), 1064 deletions(-) create mode 100644 drivers/soc/apple/fv_shim.c create mode 100644 drivers/soc/apple/work_shim.c create mode 100644 include/linux/apple-sep-fv.h create mode 100644 tools/aurora-sep/README.md create mode 100644 tools/aurora-sep/aurora-sep.service create mode 100644 tools/aurora-sep/fprintd-aurora.conf create mode 100755 tools/aurora-sep/load-driver create mode 100644 tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 856d3c5b1fa0d..8b19cd52ff3a3 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -116,6 +116,7 @@ void bio_crypt_set_ctx(struct bio *bio, const struct blk_crypto_key *key, bio->bi_crypt_context = bc; } +EXPORT_SYMBOL_GPL(bio_crypt_set_ctx); void __bio_crypt_free_ctx(struct bio *bio) { @@ -186,6 +187,7 @@ bool bio_crypt_dun_is_contiguous(const struct bio_crypt_ctx *bc, /* If the DUN wrapped through 0, don't treat it as contiguous. */ return carry == 0; } +EXPORT_SYMBOL_GPL(bio_crypt_dun_is_contiguous); /* * Checks that two bio crypt contexts are compatible - i.e. that @@ -349,6 +351,7 @@ int blk_crypto_init_key(struct blk_crypto_key *blk_key, return 0; } +EXPORT_SYMBOL_GPL(blk_crypto_init_key); bool blk_crypto_config_supported_natively(struct block_device *bdev, const struct blk_crypto_config *cfg) @@ -399,6 +402,7 @@ int blk_crypto_start_using_key(struct block_device *bdev, } return blk_crypto_fallback_start_using_mode(key->crypto_cfg.crypto_mode); } +EXPORT_SYMBOL_GPL(blk_crypto_start_using_key); /** * blk_crypto_evict_key() - Evict a blk_crypto_key from a block_device diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig index 31974c7dd20c9..de0fa14096286 100644 --- a/drivers/nvme/host/Kconfig +++ b/drivers/nvme/host/Kconfig @@ -127,6 +127,7 @@ config NVME_APPLE depends on OF && BLOCK depends on APPLE_RTKIT && APPLE_SART depends on ARCH_APPLE || COMPILE_TEST + select BLK_INLINE_ENCRYPTION select NVME_CORE help This provides support for the NVMe controller embedded in Apple SoCs diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index f2828beefd4e6..0513d120abffd 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -10,6 +10,8 @@ */ #include +#include +#include #include #include #include @@ -257,6 +259,7 @@ struct apple_nvme { unsigned long flush_interval; unsigned long last_flush; struct delayed_work flush_dwork; + struct blk_crypto_profile crypto_profile; }; /* Bit in recovery_flags: recovery_work owns the next controller stop. */ @@ -293,6 +296,9 @@ MODULE_PARM_DESC(flush_interval, static_assert(sizeof(struct nvme_command) == 64); static_assert(sizeof(struct apple_nvmmu_tcb) == 128); +#define APPLE_NVME_CRYPTO_KEY_SIZE 64 +#define APPLE_NVME_CRYPTO_DATA_UNIT_SIZE 4096 + static inline struct apple_nvme *ctrl_to_apple_nvme(struct nvme_ctrl *ctrl) { return container_of(ctrl, struct apple_nvme, ctrl); @@ -575,9 +581,12 @@ static void apple_nvme_submit_cmd_t8015(struct apple_nvme_queue *q, static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, - struct nvme_command *cmd) + struct nvme_command *cmd, + struct request *req) { struct apple_nvme *anv = queue_to_apple_nvme(q); + const u8 *key = NULL; + u64 dun = 0; u32 tag = nvme_tag_from_cid(cmd->common.command_id); struct apple_nvmmu_tcb *tcb = &q->tcbs[tag]; @@ -594,6 +603,22 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, else tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE; + if (unlikely(req->crypt_ctx)) { + const struct blk_crypto_key *blk_key = req->crypt_ctx->bc_key; + + key = blk_key->bytes; + dun = req->crypt_ctx->bc_dun[0]; + } + + if (key) { + __le64 dun_le = cpu_to_le64(dun); + + tcb->dma_flags |= BIT(2); + memcpy(tcb->aes_iv, &dun_le, sizeof(dun_le)); + memset(tcb->_aes_unk, 0, sizeof(tcb->_aes_unk)); + memcpy(tcb->_aes_unk, key, 4); + memcpy(tcb->_aes_unk + 16, key + 16, 48); + } memcpy(&q->sqes[tag], cmd, sizeof(*cmd)); /* @@ -846,9 +871,18 @@ static __always_inline void apple_nvme_unmap_rq(struct request *req) { struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); struct apple_nvme *anv = queue_to_apple_nvme(iod->q); + struct apple_nvmmu_tcb *tcb; + u32 tag; if (blk_rq_nr_phys_segments(req)) apple_nvme_unmap_data(anv, req); + tag = nvme_tag_from_cid(iod->cmd.common.command_id); + tcb = &iod->q->tcbs[tag]; + if (unlikely(tcb->dma_flags & BIT(2))) { + memzero_explicit(tcb->aes_iv, sizeof(tcb->aes_iv)); + memzero_explicit(tcb->_aes_unk, sizeof(tcb->_aes_unk)); + tcb->dma_flags &= ~BIT(2); + } } static void apple_nvme_complete_rq(struct request *req) @@ -1080,6 +1114,20 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, ret = nvme_setup_cmd(ns, req); if (ret) return ret; + if (unlikely(req->crypt_ctx)) { + const struct blk_crypto_key *key = req->crypt_ctx->bc_key; + + if (WARN_ON_ONCE(key->size != APPLE_NVME_CRYPTO_KEY_SIZE || + key->crypto_cfg.crypto_mode != + BLK_ENCRYPTION_MODE_AES_256_XTS || + key->crypto_cfg.key_type != + BLK_CRYPTO_KEY_TYPE_HW_WRAPPED || + key->crypto_cfg.data_unit_size != + APPLE_NVME_CRYPTO_DATA_UNIT_SIZE)) { + ret = BLK_STS_NOTSUPP; + goto out_free_cmd; + } + } if (blk_rq_nr_phys_segments(req)) { ret = apple_nvme_map_data(anv, req, cmnd); @@ -1095,7 +1143,7 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, } if (anv->hw->has_lsq_nvmmu) - apple_nvme_submit_cmd_t8103(q, cmnd); + apple_nvme_submit_cmd_t8103(q, cmnd, req); else apple_nvme_submit_cmd_t8015(q, cmnd); @@ -2374,6 +2422,18 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) if (ret) goto out_detach_genpd; + if (anv->hw->has_lsq_nvmmu) { + ret = devm_blk_crypto_profile_init(dev, &anv->crypto_profile, 0); + if (ret) + goto out_detach_genpd; + anv->crypto_profile.max_dun_bytes_supported = sizeof(u64); + anv->crypto_profile.key_types_supported = + BLK_CRYPTO_KEY_TYPE_HW_WRAPPED; + anv->crypto_profile.modes_supported[BLK_ENCRYPTION_MODE_AES_256_XTS] = + BIT(ilog2(APPLE_NVME_CRYPTO_DATA_UNIT_SIZE)); + anv->crypto_profile.dev = dev; + } + ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops, NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS | NVME_QUIRK_ADMIN_PAGE_ALIGN | anv->hw->quirks); @@ -2385,6 +2445,9 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) /* Released by apple_nvme_free_ctrl(), including asynchronous users. */ kref_get(&anv->ref); + if (anv->hw->has_lsq_nvmmu) + anv->ctrl.crypto_profile = &anv->crypto_profile; + return anv; out_detach_genpd: apple_nvme_detach_genpd(anv); diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 9e810008e151b..0ecc508d1a076 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -4176,6 +4176,11 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info) ns->ctrl = ctrl; kref_init(&ns->kref); +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + if (ctrl->crypto_profile) + blk_crypto_register(ctrl->crypto_profile, ns->queue); +#endif + if (nvme_init_ns_head(ns, info)) goto out_cleanup_disk; diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index ceb1bfe51a4b0..1999421533aa0 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -361,6 +361,9 @@ struct nvme_ctrl { int numa_node; struct blk_mq_tag_set *tagset; struct blk_mq_tag_set *admin_tagset; +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + struct blk_crypto_profile *crypto_profile; +#endif struct list_head namespaces; struct mutex namespaces_lock; struct srcu_struct srcu; diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 8977ecd5b863d..29df757134d40 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -99,6 +99,9 @@ config APPLE_AOP Say 'y' here if you have an Apple laptop. +config APPLE_SEP_FV_INTERFACE + bool + config APPLE_SEP tristate "Apple SEP (Secure Enclave Processor)" depends on ARCH_APPLE @@ -111,6 +114,7 @@ config APPLE_SEP select CRYPTO_AES select CRYPTO_GCM select CRYPTO_ECDH + select APPLE_SEP_FV_INTERFACE help Driver for the Apple SEP (Secure Enclave Processor) on Apple silicon. It drives the Touch ID fingerprint sensor over a /dev/sep-bio character diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index ab67304c059e6..76aa178f98f75 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -22,8 +22,10 @@ apple-tunable-y = tunable.o obj-$(CONFIG_APPLE_AOP) += aop.o +obj-$(CONFIG_APPLE_SEP_FV_INTERFACE) += fv_shim.o + obj-$(CONFIG_APPLE_SEP) += apple-sep.o apple-sep-y := sep.o hwrng_shim.o store_shim.o bio_shim.o sha_shim.o \ - crypto_shim.o sensor_shim.o p256_shim.o trusted_shim.o + crypto_shim.o sensor_shim.o p256_shim.o trusted_shim.o work_shim.o obj-$(CONFIG_APPLE_PMP) += pmp.o diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 27d7bd25d1110..e66710b6935cf 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -705,6 +705,7 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { nonce: request.nonce, terminal: Some(VerifyOutcome::NoMatch), }; + ctx.session.unseen = true; return Ok(Handled { ret: 0, wake: true, @@ -853,9 +854,7 @@ fn delete(ctx: &mut Context<'_>, user: UserPtr) -> Result { // `0x57` takes one `identity_v1_t` (signed user id + 16-byte UUID). let Some(identity) = ctx.index.identity_v1_for(&request.uuid, ENROL_USER_ID) else { - pr_warn!( - "sep_bio: DELETE of a held identity could not be expressed as an identity_v1_t\n" - ); + pr_warn!("sep_bio: DELETE of a held identity could not be expressed as an identity_v1_t\n"); return Err(ENOENT); }; @@ -873,9 +872,7 @@ fn delete_all(ctx: &mut Context<'_>) -> Result { require_admin()?; if ctx.index.total() == 0 { - pr_info!( - "sep_bio: DELETE_ALL on empty index; reporting success (nothing to delete)\n" - ); + pr_info!("sep_bio: DELETE_ALL on empty index; reporting success (nothing to delete)\n"); return ok(); } diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index be81aa1de38df..5e395dc6a0b80 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -28,8 +28,6 @@ impl Slot { struct EntropySink { value: Option, - msg1: u32, - unclaimed: u32, busy: bool, } @@ -44,11 +42,7 @@ pub(crate) struct ControlState { next_tag: u8, retired: [u64; 4], retired_count: u32, - // SEP-initiated messages (type != 0x01). - unsolicited: u32, entropy: EntropySink, - // After the persistent-state exchange the SEP parks this endpoint; refuse rather than time out. - closed: bool, } impl ControlState { @@ -58,14 +52,10 @@ impl ControlState { next_tag: proto::TAG_POOL_FIRST, retired: [0; 4], retired_count: 0, - unsolicited: 0, entropy: EntropySink { value: None, - msg1: 0, - unclaimed: 0, busy: false, }, - closed: false, } } @@ -80,14 +70,20 @@ impl ControlState { } } + fn reclaim_retired(&mut self, tag: u8) -> bool { + if !self.is_retired(tag) { + return false; + } + self.retired[(tag >> 6) as usize] &= !(1u64 << (tag & 0x3f)); + self.retired_count -= 1; + true + } + fn tag_in_flight(&self, tag: u8) -> bool { self.slots.iter().any(|s| s.used && s.tag == tag) } pub(crate) fn alloc(&mut self) -> Result<(usize, u8)> { - if self.closed { - return Err(EPIPE); - } let idx = self.slots.iter().position(|s| !s.used).ok_or(EBUSY)?; for _ in 0..POOL_LEN { @@ -132,11 +128,7 @@ impl ControlState { pub(crate) fn deliver(&mut self, reply: proto::ControlReply) -> Delivery { // The reserved entropy tag is claimed first, outstanding request or not. if reply.tag == proto::TAG_ENTROPY { - if self.entropy.value.is_some() { - self.entropy.unclaimed = self.entropy.unclaimed.wrapping_add(1); - } self.entropy.value = Some(reply.data_lo); - self.entropy.msg1 = reply.msg1; return Delivery::Entropy; } @@ -149,31 +141,25 @@ impl ControlState { return Delivery::Matched; } + if self.reclaim_retired(reply.tag) { + return Delivery::Unmatched; + } + // No match: drop it, never hand it to another waiter. Delivery::Unmatched } - pub(crate) fn note_unsolicited(&mut self) -> u32 { - self.unsolicited = self.unsolicited.wrapping_add(1); - self.unsolicited - } - pub(crate) fn entropy_begin(&mut self) -> Result<()> { - if self.closed { - return Err(EPIPE); - } if self.entropy.busy { return Err(EBUSY); } self.entropy.busy = true; - if self.entropy.value.take().is_some() { - self.entropy.unclaimed = self.entropy.unclaimed.wrapping_add(1); - } + self.entropy.value = None; Ok(()) } - pub(crate) fn entropy_take(&mut self) -> Option<(u32, u32)> { - self.entropy.value.take().map(|v| (v, self.entropy.msg1)) + pub(crate) fn entropy_take(&mut self) -> Option { + self.entropy.value.take() } pub(crate) fn entropy_end(&mut self) { diff --git a/drivers/soc/apple/crypto_shim.c b/drivers/soc/apple/crypto_shim.c index e4d0863cce01b..be7336ba00539 100644 --- a/drivers/soc/apple/crypto_shim.c +++ b/drivers/soc/apple/crypto_shim.c @@ -7,17 +7,26 @@ #include #include #include +#include #include #include #include #include -#include -#include #include "shim.h" #define SEP_GCM_TAG_LEN 16 +int sep_random_bytes(void *buf, size_t len) +{ + int ret = wait_for_random_bytes(); + + if (ret) + return ret; + get_random_bytes(buf, len); + return 0; +} + /* * AES-GCM with a 16-byte (non-96-bit) IV, over the raw AES block cipher. The * kernel's gcm(aes) only takes a 12-byte IV; the SEP's ECIES uses a 16-byte IV, @@ -174,21 +183,6 @@ static int sep_gcm16(int encrypt, const void *key, size_t keylen, return rc; } -/* - * Fills `buf` from the kernel CSPRNG, waiting for the seed first so it never - * returns unseeded bytes. Host-side entropy for key-bag secrets the host must - * reproduce; anything measuring the enclave's own entropy stays on SEP. - */ -int sep_random_bytes(void *buf, size_t len) -{ - int ret = wait_for_random_bytes(); - - if (ret) - return ret; - get_random_bytes(buf, len); - return 0; -} - /* HMAC-SHA256 over one message; writes 32 bytes to `out`, untouched on error. */ int sep_hmac_sha256(const void *key, size_t keylen, const void *data, size_t datalen, u8 *out) @@ -242,7 +236,6 @@ int sep_gcm(int encrypt, const void *key, size_t keylen, (ivlen != 12 && ivlen != 16) || ivlen > sizeof(ivcopy)) return -EINVAL; - /* The whole AAD-plus-payload-plus-tag extent must be inside the buffer. */ if (aadlen + datalen + SEP_GCM_TAG_LEN < aadlen || aadlen + datalen + SEP_GCM_TAG_LEN > buflen) return -EINVAL; diff --git a/drivers/soc/apple/der.rs b/drivers/soc/apple/der.rs index bc3db32343eae..fcd198561872f 100644 --- a/drivers/soc/apple/der.rs +++ b/drivers/soc/apple/der.rs @@ -140,8 +140,11 @@ pub(crate) fn refkey_find<'a>(set_blob: &'a [u8], key: &[u8]) -> Option<&'a [u8] if key_tag != TAG_UTF8 { return None; } - let (_val_tag, _value, after_val) = take_tlv(after_key)?; if this_key == key { + let (_val_tag, _value, after_val) = take_tlv(after_key)?; + if !after_val.is_empty() { + return None; + } return Some(&after_key[..after_key.len() - after_val.len()]); } body = next; @@ -155,4 +158,3 @@ pub(crate) fn octet_string_body(tlv: &[u8]) -> Option<&[u8]> { _ => None, } } - diff --git a/drivers/soc/apple/dt.rs b/drivers/soc/apple/dt.rs index 7fc5be540c654..ca3b9445e8d60 100644 --- a/drivers/soc/apple/dt.rs +++ b/drivers/soc/apple/dt.rs @@ -24,6 +24,8 @@ const DMA_RANGE_PROP: &CStr = c"apple,dma-range"; const DMA_RANGE_CELLS: [u32; 4] = [0, 0, 1, 0]; const REGISTERED_PROP: &CStr = c"apple,sep-shmem-registered-iova"; +const CHOSEN_PATH: &CStr = c"/chosen"; +const PREBOOT_UUID_PROP: &CStr = c"apfs-preboot-uuid"; pub(crate) struct DtNode(*mut bindings::device_node); @@ -151,6 +153,35 @@ pub(crate) fn sep_node() -> Option { DtNode::find_compatible(SEP_COMPATIBLE) } +pub(crate) fn preboot_uuid() -> Option<[u8; 16]> { + // SAFETY: the path is NUL-terminated; a non-NULL result owns one node + // reference, which `DtNode` releases. + let raw = unsafe { + bindings::of_find_node_opts_by_path(CHOSEN_PATH.as_char_ptr(), core::ptr::null_mut()) + }; + let chosen = (!raw.is_null()).then_some(DtNode(raw))?; + let mut text = core::ptr::null(); + // SAFETY: `chosen` is live, the property name is NUL-terminated, and + // `text` is a valid output pointer. + if unsafe { + bindings::of_property_read_string( + chosen.as_ptr(), + PREBOOT_UUID_PROP.as_char_ptr(), + &mut text, + ) + } != 0 + { + return None; + } + + let mut uuid = bindings::uuid_t { b: [0; 16] }; + // SAFETY: the property is NUL-terminated and `uuid` is a valid output. + if unsafe { bindings::uuid_parse(text, &mut uuid) } != 0 || uuid.b.iter().all(|&b| b == 0) { + return None; + } + Some(uuid.b) +} + pub(crate) fn enable_sep_and_dart() -> Result<()> { let sep = sep_node().ok_or_else(|| { pr_err!( @@ -393,7 +424,7 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { } pr_info!( - "apple_sep: creating the sensor node (absent from Linux's tree; Apple has it at /arm-io/spi2/mesa)\n" + "apple_sep: creating the sensor node (absent from Linux's tree; the firmware exposes it at /arm-io/spi2/mesa)\n" ); with_changeset(|cs_handle| { diff --git a/drivers/soc/apple/fv.rs b/drivers/soc/apple/fv.rs index ba05aba7d854f..9ba5156dee386 100644 --- a/drivers/soc/apple/fv.rs +++ b/drivers/soc/apple/fv.rs @@ -1,158 +1,996 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj -//! Native FileVault key hierarchy: device-bound volume-key provisioning, -//! reverse-engineered and kept as capability. Not wired to a caller. -#![allow(dead_code)] -use crate::{image, proto, shim}; -use crate::{LockState, SepData, SksRequest}; +use crate::{image, proto, SepData, SksRequest, PHASE_READY}; use kernel::prelude::*; +use kernel::sync::atomic::Relaxed; + +const WRAPPED_KEY_LEN: usize = 40; +const OPAQUE_KEY_LEN: usize = 64; +const IV_KEY_LEN: usize = 16; +const SYSTEM_CLIENT: u64 = 1; +const NO_KEYBAG_HANDLE: i32 = -1; +const WRAPPED_KEY_FLAG: i32 = 1 << 1; +const VOLUME_KEY_FLAG: i32 = 1; +const SECRET_MAX_LEN: usize = 1024; +const RECORD_MAX_LEN: usize = 528; +const FILE_KEY_MAX_LEN: usize = 168; +const EPHEMERAL_KEY_MAX_LEN: usize = 64; +const FV_UNWRAP_VERSION: u32 = 0; +const FV_UNWRAP_OPTIONS: u32 = 2; +const PFK_UNWRAP_VERSION: u32 = 2; +const PFK_SYSTEM_VOLUME_HANDLE: i32 = -5; +const PFK_UNWRAP_OPTIONS: u32 = 0x102; +const PFK_KEY_FLAG: i32 = 1 << 1; +const PFK_NEW_VERSION: u32 = 2; +const PFK_NEW_OPTIONS: u32 = 0x102; +const PFK_NEW_KEY_FLAG: i32 = 1 << 1; +const PFK_FS_CONTEXT_LEN: usize = 28; +const FV_LOAD_CLASS_KEYS: u32 = 0x12; +const FV_UNLOAD_CLASS_KEYS: u32 = 0x13; +const FV_SYSTEM_VOLUME_OPTION: u64 = 4; +const FV_MAX_VOLUME_MAPS: usize = 32; +const FV_STATE_UUID_KEY: &[u8] = b"kid"; + +pub(crate) struct VolumeMap { + apfs_uuid: [u8; 16], + bag_uuid: [u8; 16], + refs: u32, +} + +#[repr(C)] +struct KernelKey { + opaque: [u8; OPAQUE_KEY_LEN], + iv: [u8; IV_KEY_LEN], +} + +#[repr(C)] +struct KernelNewFileKey { + key: KernelKey, + wrapped_ekwk: [u8; FILE_KEY_MAX_LEN], + wrapped_ek: [u8; FILE_KEY_MAX_LEN], + wrapped_ekwk_len: usize, + wrapped_ek_len: usize, +} + +#[repr(C)] +struct KernelOps { + unwrap_media_key: + unsafe extern "C" fn(*mut c_void, *const u8, usize, u32, *mut KernelKey) -> c_int, + unwrap_volume_key: unsafe extern "C" fn( + *mut c_void, + *const u8, + usize, + *const u8, + usize, + *const u8, + usize, + *mut KernelKey, + ) -> c_int, + load_class_keys: unsafe extern "C" fn( + *mut c_void, + *const u8, + *const u8, + usize, + *const u8, + usize, + *const u8, + usize, + ) -> c_int, + unload_class_keys: unsafe extern "C" fn(*mut c_void, *const u8, *const u8, usize) -> c_int, + unwrap_file_key: unsafe extern "C" fn( + *mut c_void, + *const u8, + u32, + *const u8, + usize, + *const u8, + usize, + *mut KernelKey, + ) -> c_int, + new_file_key: unsafe extern "C" fn( + *mut c_void, + *const u8, + u32, + u64, + u16, + *mut KernelNewFileKey, + ) -> c_int, +} + +extern "C" { + fn sep_fv_register_v2(context: *mut c_void, ops: *const KernelOps) -> c_int; + fn sep_fv_unregister_v2(context: *mut c_void); +} + +static KERNEL_OPS: KernelOps = KernelOps { + unwrap_media_key: kernel_unwrap_media_key, + unwrap_volume_key: kernel_unwrap_volume_key, + load_class_keys: kernel_load_class_keys, + unload_class_keys: kernel_unload_class_keys, + unwrap_file_key: kernel_unwrap_file_key, + new_file_key: kernel_new_file_key, +}; + +const PFK_VOLUME_PARAMS_PREFIX: [u8; 12] = [ + 0x31, 0x1a, 0x30, 0x18, 0x0c, 0x04, b'v', b'u', b'i', b'd', 0x04, 0x10, +]; +const PFK_VOLUME_PARAMS_LEN: usize = PFK_VOLUME_PARAMS_PREFIX.len() + 16; + +const FV_PARAMS_DER: [u8; 35] = [ + 0x31, 0x21, 0x30, 0x0a, 0x0c, 0x02, b'k', b'c', 0x04, 0x04, 0, 0, 0, 0, 0x30, 0x13, 0x0c, 0x07, + b'o', b'p', b't', b'i', b'o', b'n', b's', 0x04, 0x08, 0, 0, 0, 0, 0, 0, 0, 0, +]; + +struct MediaKey { + opaque: [u8; OPAQUE_KEY_LEN], + iv_key: [u8; IV_KEY_LEN], +} + +struct VolumeKey { + opaque: [u8; OPAQUE_KEY_LEN], +} + +struct FileKey { + opaque: [u8; OPAQUE_KEY_LEN], + iv_key: [u8; IV_KEY_LEN], +} + +struct NewFileKey { + key: FileKey, + wrapped_ekwk: [u8; FILE_KEY_MAX_LEN], + wrapped_ek: [u8; FILE_KEY_MAX_LEN], + wrapped_ekwk_len: usize, + wrapped_ek_len: usize, +} + +impl Drop for NewFileKey { + fn drop(&mut self) { + image::wipe(&mut self.wrapped_ekwk); + image::wipe(&mut self.wrapped_ek); + } +} + +impl Drop for VolumeKey { + fn drop(&mut self) { + image::wipe(&mut self.opaque); + } +} + +impl Drop for MediaKey { + fn drop(&mut self) { + image::wipe(&mut self.opaque); + image::wipe(&mut self.iv_key); + } +} + +impl Drop for FileKey { + fn drop(&mut self) { + image::wipe(&mut self.opaque); + image::wipe(&mut self.iv_key); + } +} impl SepData { - const FV_LABEL: &'static [u8; 16] = b"AppleSEPvek00001"; - const FV_PKH: &'static [u8; 16] = b"AppleSEP-KEK-001"; - const FV_PARAM_LEN: usize = 0x130; + fn pfk_params(volume_uuid: &[u8; 16]) -> [u8; PFK_VOLUME_PARAMS_LEN] { + let mut params = [0u8; PFK_VOLUME_PARAMS_LEN]; + params[..PFK_VOLUME_PARAMS_PREFIX.len()].copy_from_slice(&PFK_VOLUME_PARAMS_PREFIX); + params[PFK_VOLUME_PARAMS_PREFIX.len()..].copy_from_slice(volume_uuid); + params + } + + fn pfk_class(protection_class: u32) -> Result { + match protection_class & 0x1f { + 1..=4 => Ok(protection_class), + 6 => Ok((protection_class & !0x1f) | 13), + 7 => Ok((protection_class & !0x1f) | 17), + _ => Err(EINVAL), + } + } + + fn fv_options(&self, volume_uuid: &[u8; 16]) -> u64 { + if self.xarm.lock().os_uuid == Some(*volume_uuid) { + FV_SYSTEM_VOLUME_OPTION + } else { + 0 + } + } + + fn fv_params(options: u64) -> [u8; FV_PARAMS_DER.len()] { + let mut params = FV_PARAMS_DER; + let options_at = params.len() - 8; + params[options_at..].copy_from_slice(&options.to_le_bytes()); + params + } + + fn fv_ready(&self) -> Result<()> { + if self.phase.load(Relaxed) != PHASE_READY { + return Err(EAGAIN); + } + if !self.sks_ready() { + return Err(ENODEV); + } + Ok(()) + } - fn fv_param(&self) -> Result> { - let mut p: KVec = KVec::new(); - p.resize(Self::FV_PARAM_LEN, 0u8, GFP_KERNEL)?; - p[0x10..0x20].copy_from_slice(Self::FV_LABEL); - Ok(p) + fn resolve_fv_uuid(&self, apfs_uuid: &[u8; 16]) -> [u8; 16] { + self.fv_volumes + .lock() + .iter() + .find(|entry| entry.apfs_uuid == *apfs_uuid) + .map_or(*apfs_uuid, |entry| entry.bag_uuid) } - fn sks_req_fv(&self, selector: u8, body: &image::Body) -> Result { - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), body)?; + fn record_fv_volume(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result<()> { + let mut volumes = self.fv_volumes.lock(); + if let Some(entry) = volumes + .iter_mut() + .find(|entry| entry.apfs_uuid == *apfs_uuid) + { + if entry.bag_uuid != *bag_uuid { + return Err(EINVAL); + } + entry.refs = entry.refs.checked_add(1).ok_or(EOVERFLOW)?; + return Ok(()); + } + if volumes.len() >= FV_MAX_VOLUME_MAPS { + return Err(ENOSPC); + } + volumes.push( + VolumeMap { + apfs_uuid: *apfs_uuid, + bag_uuid: *bag_uuid, + refs: 1, + }, + GFP_KERNEL, + )?; + Ok(()) + } + + fn unrecord_fv_volume(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result<()> { + let mut volumes = self.fv_volumes.lock(); + let index = volumes + .iter() + .position(|entry| entry.apfs_uuid == *apfs_uuid) + .ok_or(ENOENT)?; + if volumes[index].bag_uuid != *bag_uuid { + return Err(EINVAL); + } + if volumes[index].refs > 1 { + volumes[index].refs -= 1; + } else { + volumes.swap_remove(index); + } + Ok(()) + } + + fn sks_req_fv_blob_state( + &self, + volume_uuid: &[u8; 16], + volume_key: &[u8], + ) -> Result { + let params = Self::fv_params(self.fv_options(volume_uuid)); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(¶ms)?; + body.put_blob(volume_key)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_GET_BLOB_STATE_NAME, + msg: crate::sks::encode_sks_get_blob_state(self.sks_next_seq(), len), + img, + }) + } + + fn fv_blob_uuid(&self, volume_uuid: &[u8; 16], volume_key: &[u8]) -> Result<[u8; 16]> { + let out = self + .sks_send(self.sks_req_fv_blob_state(volume_uuid, volume_key)) + .ok_or(EIO)?; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!(self.dev, "fv: GET_BLOB_STATE failed with status {}\n", status); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_GET_BLOB_STATE_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + let version = fields.i32(); + if version != Some(0) { + dev_err!( + self.dev, + "fv: GET_BLOB_STATE response has version {:?}, {} body bytes\n", + version, + body.len() + ); + return Err(EMSGSIZE); + } + let state = match fields.blob() { + Some(state) => state, + None => { + dev_err!( + self.dev, + "fv: GET_BLOB_STATE has no complete state blob in {} body bytes\n", + body.len() + ); + return Err(EMSGSIZE); + } + }; + let uuid_tlv = match crate::der::refkey_find(state, FV_STATE_UUID_KEY) { + Some(uuid) => uuid, + None => { + let head0 = state + .get(..8) + .and_then(|bytes| bytes.try_into().ok()) + .map(u64::from_be_bytes) + .unwrap_or(0); + let head1 = state + .get(8..16) + .and_then(|bytes| bytes.try_into().ok()) + .map(u64::from_be_bytes) + .unwrap_or(0); + dev_err!( + self.dev, + "fv: GET_BLOB_STATE returned {} state bytes without a UUID (head {:016x}{:016x})\n", + state.len(), + head0, + head1 + ); + return Err(EMSGSIZE); + } + }; + let raw_uuid = crate::der::octet_string_body(uuid_tlv).ok_or(EMSGSIZE)?; + if raw_uuid.len() != 16 { + dev_err!( + self.dev, + "fv: GET_BLOB_STATE returned a {}-byte UUID\n", + raw_uuid.len() + ); + return Err(EMSGSIZE); + } + let mut bag_uuid = [0u8; 16]; + bag_uuid.copy_from_slice(raw_uuid); + Ok(bag_uuid) + } + + pub(crate) fn register_fv_kernel(&self) -> Result<()> { + let context = core::ptr::from_ref(self).cast_mut().cast::(); + // SAFETY: `KERNEL_OPS` is static and `remove()` unregisters this + // pointer before the driver's `Arc` can be dropped. + kernel::error::to_result(unsafe { sep_fv_register_v2(context, &KERNEL_OPS) }) + } + + pub(crate) fn unregister_fv_kernel(&self) { + let context = core::ptr::from_ref(self).cast_mut().cast::(); + // SAFETY: the pointer is the one passed to `sep_fv_register_v2`. + unsafe { sep_fv_unregister_v2(context) }; + } + + fn sks_req_unwrap_media_key_from_class( + &self, + wrapped: &[u8; WRAPPED_KEY_LEN], + protection_class: u32, + ) -> Result { + let mut body = image::Body::new(); + body.put_u32(1)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(wrapped)?; + body.put_i32(NO_KEYBAG_HANDLE)?; + body.put_u32(protection_class)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; - let msg = crate::sks::encode_sks_fv(selector, self.sks_next_seq(), len).ok_or(EINVAL)?; + let msg = + crate::sks::encode_sks_unwrap_media_key(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { - name: crate::sks::SKS_PERFORM_OP_NAME, + name: crate::sks::SKS_UNWRAP_MEDIA_KEY_NAME, msg, img, }) } - fn fv_send( + fn decode_media_key(&self, out: crate::SksOutcome) -> Result { + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!( + self.dev, + "fv: UNWRAP_MEDIA_KEY failed with status {}\n", + status + ); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_UNWRAP_MEDIA_KEY_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(1) { + return Err(EMSGSIZE); + } + let key = fields.blob().ok_or(EMSGSIZE)?; + let iv_key = fields.blob().ok_or(EMSGSIZE)?; + let flags = fields.i32().ok_or(EMSGSIZE)?; + + if key.len() != OPAQUE_KEY_LEN + || iv_key.len() != IV_KEY_LEN + || flags & WRAPPED_KEY_FLAG == 0 + { + return Err(EMSGSIZE); + } + + let mut opaque = [0; OPAQUE_KEY_LEN]; + opaque.copy_from_slice(key); + let mut iv = [0; IV_KEY_LEN]; + iv.copy_from_slice(iv_key); + Ok(MediaKey { opaque, iv_key: iv }) + } + + fn unwrap_media_key_from_class( &self, - selector: u8, - body: image::Body, - _what: &str, - ) -> Option<(i32, Option, Option>)> { - let out = self.sks_send(self.sks_req_fv(selector, &body))?; - let mailbox: i32 = out.reply.status.into(); - let (mut opst, mut blob) = (None, None); - if mailbox == 0 { - if let Some(rbody) = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &out) { - let mut f = proto::FieldCursor::new(rbody); - opst = f.i32(); - if let Some(b) = f.blob() { - let mut owned: KVec = KVec::new(); - owned.extend_from_slice(b, GFP_KERNEL).ok()?; - blob = Some(owned); - } + wrapped: &[u8; WRAPPED_KEY_LEN], + protection_class: u32, + ) -> Result { + self.fv_ready()?; + let out = self + .sks_send(self.sks_req_unwrap_media_key_from_class(wrapped, protection_class)) + .ok_or(EIO)?; + self.decode_media_key(out) + } + + fn sks_req_unwrap_vek(&self, secret: &[u8], kek: &[u8], vek: &[u8]) -> Result { + let mut body = image::Body::new(); + body.put_u32(FV_UNWRAP_VERSION)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(&FV_PARAMS_DER)?; + body.put_u32(FV_UNWRAP_OPTIONS)?; + body.put_blob(secret)?; + body.put_blob(kek)?; + body.put_blob(vek)?; + body.put_blob(&[])?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_unwrap_vek(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_UNWRAP_VEK_NAME, + msg, + img, + }) + } + + fn unwrap_vek(&self, secret: &[u8], kek: &[u8], vek: &[u8]) -> Result { + self.fv_ready()?; + let out = self + .sks_send(self.sks_req_unwrap_vek(secret, kek, vek)) + .ok_or(EIO)?; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!(self.dev, "fv: UNWRAP_VEK failed with status {}\n", status); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_UNWRAP_VEK_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(FV_UNWRAP_VERSION as i32) { + return Err(EMSGSIZE); + } + let key = fields.blob().ok_or(EMSGSIZE)?; + let flags = fields.i32().ok_or(EMSGSIZE)?; + if key.len() != OPAQUE_KEY_LEN || flags & VOLUME_KEY_FLAG == 0 { + return Err(EMSGSIZE); + } + + let mut opaque = [0; OPAQUE_KEY_LEN]; + opaque.copy_from_slice(key); + Ok(VolumeKey { opaque }) + } + + fn sks_req_unwrap_file_key( + &self, + volume_uuid: &[u8; 16], + protection_class: u32, + wrapped_ekwk: &[u8], + wrapped_ek: &[u8], + ) -> Result { + let class = Self::pfk_class(protection_class)?; + let bag_uuid = self.resolve_fv_uuid(volume_uuid); + let params = Self::pfk_params(&bag_uuid); + + let mut body = image::Body::new(); + body.put_u32(PFK_UNWRAP_VERSION)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_i32(PFK_SYSTEM_VOLUME_HANDLE)?; + body.put_blob(¶ms)?; + body.put_u32(class)?; + body.put_blob(wrapped_ek)?; + body.put_blob(wrapped_ekwk)?; + body.put_blob(&[])?; + body.put_u32(PFK_UNWRAP_OPTIONS)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_UNWRAP_PFK_NAME, + msg: crate::sks::encode_sks_unwrap_pfk(self.sks_next_seq(), len), + img, + }) + } + + fn unwrap_file_key( + &self, + volume_uuid: &[u8; 16], + protection_class: u32, + wrapped_ekwk: &[u8], + wrapped_ek: &[u8], + ) -> Result { + self.fv_ready()?; + let out = self + .sks_send(self.sks_req_unwrap_file_key( + volume_uuid, + protection_class, + wrapped_ekwk, + wrapped_ek, + )) + .ok_or(EIO)?; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!( + self.dev, + "fv: UNWRAP_PFK v2 class {} ekwk {} ek {} failed with status {}\n", + protection_class, + wrapped_ekwk.len(), + wrapped_ek.len(), + status + ); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_UNWRAP_PFK_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(PFK_UNWRAP_VERSION as i32) { + return Err(EMSGSIZE); + } + let key = fields.blob().ok_or(EMSGSIZE)?; + let iv_key = fields.blob().ok_or(EMSGSIZE)?; + let _ephemeral_key = fields.blob().ok_or(EMSGSIZE)?; + let flags = fields.i32().ok_or(EMSGSIZE)?; + if key.len() != OPAQUE_KEY_LEN || iv_key.len() != IV_KEY_LEN || flags & PFK_KEY_FLAG == 0 { + return Err(EMSGSIZE); + } + let mut opaque = [0; OPAQUE_KEY_LEN]; + opaque.copy_from_slice(key); + let mut iv = [0; IV_KEY_LEN]; + iv.copy_from_slice(iv_key); + Ok(FileKey { opaque, iv_key: iv }) + } + + fn sks_req_new_file_key( + &self, + volume_uuid: &[u8; 16], + protection_class: u32, + ) -> Result { + let class = Self::pfk_class(protection_class)?; + let bag_uuid = self.resolve_fv_uuid(volume_uuid); + let params = Self::pfk_params(&bag_uuid); + let mut context = [0u8; PFK_FS_CONTEXT_LEN]; + context[10..26].copy_from_slice(&bag_uuid); + let mut body = image::Body::new(); + body.put_u32(PFK_NEW_VERSION)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_i32(PFK_SYSTEM_VOLUME_HANDLE)?; + body.put_blob(¶ms)?; + body.put_u32(class)?; + body.put_u32(PFK_NEW_OPTIONS)?; + body.put_blob(&context)?; + body.put_blob(&[])?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_NEW_PFK_NAME, + msg: crate::sks::encode_sks_new_pfk(self.sks_next_seq(), len), + img, + }) + } + + fn new_file_key( + &self, + volume_uuid: &[u8; 16], + protection_class: u32, + ) -> Result { + self.fv_ready()?; + let class = Self::pfk_class(protection_class)?; + let out = self + .sks_send(self.sks_req_new_file_key(volume_uuid, protection_class)) + .ok_or(EIO)?; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!( + self.dev, + "fv: NEW_PFK class {} failed with status {}\n", + protection_class, + status + ); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_NEW_PFK_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(PFK_NEW_VERSION as i32) { + return Err(EMSGSIZE); + } + let opaque = fields.blob().ok_or(EMSGSIZE)?; + let iv = fields.blob().ok_or(EMSGSIZE)?; + let ephemeral = fields.blob().ok_or(EMSGSIZE)?; + let wrapped_ek = fields.blob().ok_or(EMSGSIZE)?; + let wrapped_ekwk = fields.blob().ok_or(EMSGSIZE)?; + let key_flags = fields.i32().ok_or(EMSGSIZE)?; + let wrapped_class = fields.i32().ok_or(EMSGSIZE)?; + if opaque.len() != OPAQUE_KEY_LEN + || iv.len() != IV_KEY_LEN + || ephemeral.is_empty() + || ephemeral.len() > EPHEMERAL_KEY_MAX_LEN + || wrapped_ek.is_empty() + || wrapped_ek.len() > FILE_KEY_MAX_LEN + || wrapped_ekwk.is_empty() + || wrapped_ekwk.len() > FILE_KEY_MAX_LEN + || key_flags & PFK_NEW_KEY_FLAG == 0 + || wrapped_class != class as i32 + { + dev_err!( + self.dev, + "fv: NEW_PFK malformed response: key {} iv {} eph {} ek {} ekwk {} flags {:#x} class {} expected {}\n", + opaque.len(), + iv.len(), + ephemeral.len(), + wrapped_ek.len(), + wrapped_ekwk.len(), + key_flags, + wrapped_class, + class + ); + return Err(EMSGSIZE); + } + + let mut key = FileKey { + opaque: [0; OPAQUE_KEY_LEN], + iv_key: [0; IV_KEY_LEN], + }; + key.opaque.copy_from_slice(opaque); + key.iv_key.copy_from_slice(iv); + let mut result = NewFileKey { + key, + wrapped_ekwk: [0; FILE_KEY_MAX_LEN], + wrapped_ek: [0; FILE_KEY_MAX_LEN], + wrapped_ekwk_len: wrapped_ekwk.len(), + wrapped_ek_len: wrapped_ek.len(), + }; + result.wrapped_ekwk[..wrapped_ekwk.len()].copy_from_slice(wrapped_ekwk); + result.wrapped_ek[..wrapped_ek.len()].copy_from_slice(wrapped_ek); + Ok(result) + } + + fn sks_req_load_class_keys( + &self, + volume_uuid: &[u8; 16], + secret: &[u8], + unlock_record: &[u8], + volume_key: &[u8], + ) -> Result { + let options = self.fv_options(volume_uuid); + let params = Self::fv_params(options); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(¶ms)?; + body.put_u32(FV_LOAD_CLASS_KEYS)?; + body.put_u64(options)?; + body.put_blob(secret)?; + body.put_blob(unlock_record)?; + body.put_blob(volume_key)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_SET_PROTECTION_NAME, + msg: crate::sks::encode_sks_set_protection(self.sks_next_seq(), len), + img, + }) + } + + fn load_class_keys( + &self, + volume_uuid: &[u8; 16], + secret: &[u8], + unlock_record: &[u8], + volume_key: &[u8], + ) -> Result<()> { + self.fv_ready()?; + let bag_uuid = self.fv_blob_uuid(volume_uuid, volume_key)?; + let request = self.sks_req_load_class_keys( + volume_uuid, + secret, + unlock_record, + volume_key, + )?; + self.record_fv_volume(volume_uuid, &bag_uuid)?; + let out = match self.sks_send(Ok(request)) { + Some(out) => out, + None => { + let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + return Err(EIO); } + }; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!( + self.dev, + "fv: LOAD_CLASS_KEYS failed with status {}\n", + status + ); + let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + return Err(EACCES); } - Some((mailbox, opst, blob)) - } - - /// `0x42` mint the device-bound key-encryption key. - fn fv_new_kek(&self, handle: u64, param: &[u8]) -> Option> { - let mut b = image::Body::new(); - b.put_u32(0).ok()?; - b.put_u64(handle).ok()?; - b.put_blob(param).ok()?; - b.put_u32(0).ok()?; - b.put_blob(&[]).ok()?; - b.put_blob(Self::FV_PKH).ok()?; - let (mb, op, blob) = self.fv_send(crate::sks::OP_SKS_FV_NEW_KEK, b, "new_kek")?; - (mb == 0 && op == Some(0)).then_some(())?; - blob - } - - /// `0x40` mint the wrapped, device-bound volume key. - fn fv_new_vek(&self, handle: u64, param: &[u8]) -> Option> { - let mut b = image::Body::new(); - b.put_u32(0).ok()?; - b.put_u64(handle).ok()?; - b.put_blob(param).ok()?; - b.put_blob(&[]).ok()?; - b.put_blob(&[]).ok()?; - b.put_blob(Self::FV_PKH).ok()?; - let (mb, op, blob) = self.fv_send(crate::sks::OP_SKS_FV_NEW_VEK, b, "new_vek")?; - (mb == 0 && op == Some(0)).then_some(())?; - blob - } - - /// `0x41` install the volume key into our collection (empty KEK slot = self-derive). - fn fv_unwrap_vek(&self, handle: u64, param: &[u8], wrapped_vek: &[u8]) -> Option> { - let mut b = image::Body::new(); - b.put_u32(0).ok()?; - b.put_u64(handle).ok()?; - b.put_blob(param).ok()?; - b.put_u32(0).ok()?; - b.put_blob(&[]).ok()?; - b.put_blob(&[]).ok()?; - b.put_blob(wrapped_vek).ok()?; - b.put_blob(&[]).ok()?; - let (mb, op, blob) = self.fv_send(crate::sks::OP_SKS_FV_UNWRAP_VEK, b, "unwrap_vek")?; - (mb == 0 && op == Some(0)).then_some(())?; - blob.or_else(|| Some(KVec::new())) - } - - fn fv_dump(path: &CStr, data: &[u8]) { - if let Ok(f) = shim::StoreFile::open(path) { - let _ = f.write_all(0, data); - let _ = f.sync(); - } - } - - fn fv_load(path: &CStr) -> Option> { - let f = shim::StoreFile::open_readonly(path).ok()?; - let sz = f.size().unwrap_or(0); - if !(1..=8192).contains(&sz) { - return None; - } - let mut buf: KVec = KVec::new(); - buf.resize(sz as usize, 0u8, GFP_KERNEL).ok()?; - f.read_exact(0, &mut buf).ok()?; - Some(buf) - } - - /// Provisions the device-bound FileVault key hierarchy and installs the volume - /// key. The destructive clear (`0x47`) is never emitted; the installed VEK is - /// consumed by the storage inline-AES engine (ANS), so on its own this seals - /// nothing on Linux. - pub(crate) fn fv_provision(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) -> Option { - const KEK_PATH: &CStr = c"/var/lib/apple-sep-fv-kek.bin"; - const VEK_PATH: &CStr = c"/var/lib/apple-sep-fv-vek.bin"; - - if let Some(healthy) = self.sks_health_check(c"fv provision") { - let _ = self.sks_send(self.sks_req_change_lock_state( - handle, - LockState::Unlocked, - secret, - healthy, - )); - } - let param = self.fv_param().ok()?; - let h = handle.value() as u64; - - let (_kek, vek) = match (Self::fv_load(KEK_PATH), Self::fv_load(VEK_PATH)) { - (Some(kek), Some(vek)) => (kek, vek), - _ => { - let kek = self.fv_new_kek(h, ¶m)?; - let vek = self.fv_new_vek(h, ¶m)?; - Self::fv_dump(KEK_PATH, &kek); - Self::fv_dump(VEK_PATH, &vek); - (kek, vek) + let body = match self.sks_report_response(crate::sks::SKS_SET_PROTECTION_NAME, &out) { + Some(body) => body, + None => { + self.rollback_class_keys(volume_uuid, volume_key); + let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + return Err(EMSGSIZE); } }; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(0) || fields.blob().is_none() { + self.rollback_class_keys(volume_uuid, volume_key); + let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + return Err(EMSGSIZE); + } + let apfs_hi = u64::from_be_bytes(volume_uuid[..8].try_into().unwrap()); + let apfs_lo = u64::from_be_bytes(volume_uuid[8..].try_into().unwrap()); + let bag_hi = u64::from_be_bytes(bag_uuid[..8].try_into().unwrap()); + let bag_lo = u64::from_be_bytes(bag_uuid[8..].try_into().unwrap()); + dev_info!( + self.dev, + "fv: mapped APFS UUID {:016x}-{:016x} to keybag UUID {:016x}-{:016x}\n", + apfs_hi, + apfs_lo, + bag_hi, + bag_lo + ); + Ok(()) + } - let installed = self.fv_unwrap_vek(h, ¶m, &vek)?; - let vek_handle = if installed.len() >= 4 { - u64::from(u32::from_le_bytes([installed[0], installed[1], installed[2], installed[3]])) - } else { - 0 - }; - Some(vek_handle) + fn rollback_class_keys(&self, volume_uuid: &[u8; 16], volume_key: &[u8]) { + if let Ok(request) = self.sks_req_unload_class_keys(volume_uuid, volume_key) { + let _ = self.sks_send(Ok(request)); + } + } + + fn sks_req_unload_class_keys( + &self, + volume_uuid: &[u8; 16], + volume_key: &[u8], + ) -> Result { + let options = self.fv_options(volume_uuid); + let params = Self::fv_params(options); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(¶ms)?; + body.put_u32(FV_UNLOAD_CLASS_KEYS)?; + body.put_u64(options)?; + body.put_blob(&[])?; + body.put_blob(&[])?; + body.put_blob(volume_key)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_SET_PROTECTION_NAME, + msg: crate::sks::encode_sks_set_protection(self.sks_next_seq(), len), + img, + }) + } + + fn unload_class_keys(&self, volume_uuid: &[u8; 16], volume_key: &[u8]) -> Result<()> { + self.fv_ready()?; + let bag_uuid = self.fv_blob_uuid(volume_uuid, volume_key)?; + let out = self + .sks_send(self.sks_req_unload_class_keys(volume_uuid, volume_key)) + .ok_or(EIO)?; + if out.reply.status != 0 { + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_SET_PROTECTION_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(0) || fields.blob().is_none() { + return Err(EMSGSIZE); + } + self.unrecord_fv_volume(volume_uuid, &bag_uuid) + } +} + +unsafe fn input<'a>(ptr: *const u8, len: usize, max: usize) -> Result<&'a [u8]> { + if ptr.is_null() || len == 0 || len > max { + return Err(EINVAL); + } + // SAFETY: the C API requires `ptr` to remain readable for `len` bytes for + // the duration of the callback, and the callback does not retain it. + Ok(unsafe { core::slice::from_raw_parts(ptr, len) }) +} + +unsafe fn optional_input<'a>(ptr: *const u8, len: usize, max: usize) -> Result<&'a [u8]> { + if len == 0 { + return Ok(&[]); + } + unsafe { input(ptr, len, max) } +} + +unsafe fn output<'a>(ptr: *mut KernelKey) -> Result<&'a mut KernelKey> { + if ptr.is_null() { + return Err(EINVAL); + } + // SAFETY: the C API provides exclusive writable storage for one key. + Ok(unsafe { &mut *ptr }) +} + +unsafe fn new_file_output<'a>(ptr: *mut KernelNewFileKey) -> Result<&'a mut KernelNewFileKey> { + if ptr.is_null() { + return Err(EINVAL); + } + // SAFETY: the C API provides exclusive writable storage for one result. + Ok(unsafe { &mut *ptr }) +} + +unsafe fn sep<'a>(context: *mut c_void) -> Result<&'a SepData> { + if context.is_null() { + return Err(ENODEV); } + // SAFETY: registration keeps `SepData` alive until all callbacks finish. + Ok(unsafe { &*context.cast::() }) +} + +unsafe extern "C" fn kernel_unwrap_media_key( + context: *mut c_void, + wrapped: *const u8, + wrapped_len: usize, + protection_class: u32, + key: *mut KernelKey, +) -> c_int { + let result: Result<()> = (|| { + let this = unsafe { sep(context)? }; + let wrapped = unsafe { input(wrapped, wrapped_len, WRAPPED_KEY_LEN)? }; + let wrapped: &[u8; WRAPPED_KEY_LEN] = wrapped.try_into().map_err(|_| EINVAL)?; + let key = unsafe { output(key)? }; + let unwrapped = this.unwrap_media_key_from_class(wrapped, protection_class)?; + key.opaque.copy_from_slice(&unwrapped.opaque); + key.iv.copy_from_slice(&unwrapped.iv_key); + Ok(()) + })(); + result.map_or_else(|error| error.to_errno(), |_| 0) +} + +unsafe extern "C" fn kernel_unwrap_volume_key( + context: *mut c_void, + secret: *const u8, + secret_len: usize, + unlock_record: *const u8, + unlock_record_len: usize, + volume_key: *const u8, + volume_key_len: usize, + key: *mut KernelKey, +) -> c_int { + let result: Result<()> = (|| { + let this = unsafe { sep(context)? }; + let secret = unsafe { optional_input(secret, secret_len, SECRET_MAX_LEN)? }; + let unlock_record = + unsafe { optional_input(unlock_record, unlock_record_len, RECORD_MAX_LEN)? }; + let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; + let key = unsafe { output(key)? }; + let unwrapped = this.unwrap_vek(secret, unlock_record, volume_key)?; + key.opaque.copy_from_slice(&unwrapped.opaque); + key.iv.fill(0); + Ok(()) + })(); + result.map_or_else(|error| error.to_errno(), |_| 0) +} + +unsafe extern "C" fn kernel_load_class_keys( + context: *mut c_void, + volume_uuid: *const u8, + secret: *const u8, + secret_len: usize, + unlock_record: *const u8, + unlock_record_len: usize, + volume_key: *const u8, + volume_key_len: usize, +) -> c_int { + let result: Result<()> = (|| { + let this = unsafe { sep(context)? }; + let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; + let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + let secret = unsafe { optional_input(secret, secret_len, SECRET_MAX_LEN)? }; + let unlock_record = + unsafe { optional_input(unlock_record, unlock_record_len, RECORD_MAX_LEN)? }; + let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; + this.load_class_keys(volume_uuid, secret, unlock_record, volume_key) + })(); + result.map_or_else(|error| error.to_errno(), |_| 0) +} + +unsafe extern "C" fn kernel_unload_class_keys( + context: *mut c_void, + volume_uuid: *const u8, + volume_key: *const u8, + volume_key_len: usize, +) -> c_int { + let result: Result<()> = (|| { + let this = unsafe { sep(context)? }; + let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; + let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; + this.unload_class_keys(volume_uuid, volume_key) + })(); + result.map_or_else(|error| error.to_errno(), |_| 0) +} + +unsafe extern "C" fn kernel_unwrap_file_key( + context: *mut c_void, + volume_uuid: *const u8, + protection_class: u32, + wrapped_ekwk: *const u8, + wrapped_ekwk_len: usize, + wrapped_ek: *const u8, + wrapped_ek_len: usize, + key: *mut KernelKey, +) -> c_int { + let result: Result<()> = (|| { + let this = unsafe { sep(context)? }; + let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; + let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + let wrapped_ekwk = unsafe { input(wrapped_ekwk, wrapped_ekwk_len, FILE_KEY_MAX_LEN)? }; + let wrapped_ek = unsafe { input(wrapped_ek, wrapped_ek_len, FILE_KEY_MAX_LEN)? }; + let key = unsafe { output(key)? }; + let unwrapped = + this.unwrap_file_key(volume_uuid, protection_class, wrapped_ekwk, wrapped_ek)?; + key.opaque.copy_from_slice(&unwrapped.opaque); + key.iv.copy_from_slice(&unwrapped.iv_key); + Ok(()) + })(); + result.map_or_else(|error| error.to_errno(), |_| 0) +} + +unsafe extern "C" fn kernel_new_file_key( + context: *mut c_void, + volume_uuid: *const u8, + protection_class: u32, + crypto_id: u64, + key_revision: u16, + key: *mut KernelNewFileKey, +) -> c_int { + let result: Result<()> = (|| { + let this = unsafe { sep(context)? }; + let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; + let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + let key = unsafe { new_file_output(key)? }; + if crypto_id == 0 || key_revision == 0 { + return Err(EINVAL); + } + let generated = this.new_file_key(volume_uuid, protection_class)?; + key.key.opaque.copy_from_slice(&generated.key.opaque); + key.key.iv.copy_from_slice(&generated.key.iv_key); + key.wrapped_ekwk.copy_from_slice(&generated.wrapped_ekwk); + key.wrapped_ek.copy_from_slice(&generated.wrapped_ek); + key.wrapped_ekwk_len = generated.wrapped_ekwk_len; + key.wrapped_ek_len = generated.wrapped_ek_len; + Ok(()) + })(); + result.map_or_else(|error| error.to_errno(), |_| 0) } diff --git a/drivers/soc/apple/fv_shim.c b/drivers/soc/apple/fv_shim.c new file mode 100644 index 0000000000000..566852005c2f8 --- /dev/null +++ b/drivers/soc/apple/fv_shim.c @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +static DECLARE_RWSEM(apple_sep_fv_registration_lock); +static DEFINE_MUTEX(apple_sep_fv_call_lock); +static struct sep_fv_ops apple_sep_fv_ops; +static void *apple_sep_fv_context; + +int sep_fv_register_v2(void *context, const struct sep_fv_ops *ops) +{ + int ret = 0; + + if (!context || !ops || !ops->unwrap_media_key || + !ops->unwrap_volume_key || !ops->load_class_keys || + !ops->unload_class_keys || !ops->unwrap_file_key || + !ops->new_file_key) + return -EINVAL; + + down_write(&apple_sep_fv_registration_lock); + if (apple_sep_fv_context) + ret = -EBUSY; + else { + apple_sep_fv_ops = *ops; + apple_sep_fv_context = context; + } + up_write(&apple_sep_fv_registration_lock); + return ret; +} +EXPORT_SYMBOL_GPL(sep_fv_register_v2); + +void sep_fv_unregister_v2(void *context) +{ + down_write(&apple_sep_fv_registration_lock); + if (apple_sep_fv_context == context) { + apple_sep_fv_context = NULL; + memzero_explicit(&apple_sep_fv_ops, sizeof(apple_sep_fv_ops)); + } + up_write(&apple_sep_fv_registration_lock); +} +EXPORT_SYMBOL_GPL(sep_fv_unregister_v2); + +static int apple_sep_fv_call(int (*call)(const struct sep_fv_ops *, void *), + void *argument) +{ + int ret; + + down_read(&apple_sep_fv_registration_lock); + if (!apple_sep_fv_context) { + ret = -ENODEV; + goto out; + } + mutex_lock(&apple_sep_fv_call_lock); + ret = call(&apple_sep_fv_ops, argument); + mutex_unlock(&apple_sep_fv_call_lock); +out: + up_read(&apple_sep_fv_registration_lock); + return ret; +} + +struct media_key_call { + const u8 *wrapped; + size_t wrapped_len; + u32 protection_class; + struct apple_sep_fv_key *key; +}; + +static int call_unwrap_media_key(const struct sep_fv_ops *ops, void *argument) +{ + struct media_key_call *call = argument; + + return ops->unwrap_media_key(apple_sep_fv_context, call->wrapped, + call->wrapped_len, call->protection_class, + call->key); +} + +int apple_sep_fv_unwrap_media_key(const u8 *wrapped, size_t wrapped_len, + u32 protection_class, + struct apple_sep_fv_key *key) +{ + struct media_key_call call = { + .wrapped = wrapped, + .wrapped_len = wrapped_len, + .protection_class = protection_class, + .key = key, + }; + int ret; + + if (!wrapped || !wrapped_len || !key) + return -EINVAL; + memzero_explicit(key, sizeof(*key)); + ret = apple_sep_fv_call(call_unwrap_media_key, &call); + if (ret) + memzero_explicit(key, sizeof(*key)); + return ret; +} +EXPORT_SYMBOL_GPL(apple_sep_fv_unwrap_media_key); + +struct volume_key_call { + const u8 *volume_uuid; + const u8 *secret; + size_t secret_len; + const u8 *unlock_record; + size_t unlock_record_len; + const u8 *volume_key; + size_t volume_key_len; + struct apple_sep_fv_key *key; +}; + +static int call_unwrap_volume_key(const struct sep_fv_ops *ops, void *argument) +{ + struct volume_key_call *call = argument; + + return ops->unwrap_volume_key(apple_sep_fv_context, call->secret, + call->secret_len, call->unlock_record, + call->unlock_record_len, call->volume_key, + call->volume_key_len, call->key); +} + +int apple_sep_fv_unwrap_volume_key(const u8 *secret, size_t secret_len, + const u8 *unlock_record, + size_t unlock_record_len, + const u8 *volume_key, + size_t volume_key_len, + struct apple_sep_fv_key *key) +{ + struct volume_key_call call = { + .secret = secret, + .secret_len = secret_len, + .unlock_record = unlock_record, + .unlock_record_len = unlock_record_len, + .volume_key = volume_key, + .volume_key_len = volume_key_len, + .key = key, + }; + int ret; + + if ((!secret && secret_len) || (secret && !secret_len) || + (!unlock_record && unlock_record_len) || + (unlock_record && !unlock_record_len) || + !volume_key || !volume_key_len || !key) + return -EINVAL; + memzero_explicit(key, sizeof(*key)); + ret = apple_sep_fv_call(call_unwrap_volume_key, &call); + if (ret) + memzero_explicit(key, sizeof(*key)); + return ret; +} +EXPORT_SYMBOL_GPL(apple_sep_fv_unwrap_volume_key); + +static int call_load_class_keys(const struct sep_fv_ops *ops, void *argument) +{ + struct volume_key_call *call = argument; + + return ops->load_class_keys(apple_sep_fv_context, call->volume_uuid, + call->secret, + call->secret_len, call->unlock_record, + call->unlock_record_len, call->volume_key, + call->volume_key_len); +} + +int apple_sep_fv_load_class_keys_v2(const u8 volume_uuid[16], + const u8 *secret, size_t secret_len, + const u8 *unlock_record, + size_t unlock_record_len, + const u8 *volume_key, + size_t volume_key_len) +{ + struct volume_key_call call = { + .volume_uuid = volume_uuid, + .secret = secret, + .secret_len = secret_len, + .unlock_record = unlock_record, + .unlock_record_len = unlock_record_len, + .volume_key = volume_key, + .volume_key_len = volume_key_len, + }; + + if (!volume_uuid || (!secret && secret_len) || (secret && !secret_len) || + (!unlock_record && unlock_record_len) || + (unlock_record && !unlock_record_len) || + !volume_key || !volume_key_len) + return -EINVAL; + return apple_sep_fv_call(call_load_class_keys, &call); +} +EXPORT_SYMBOL_GPL(apple_sep_fv_load_class_keys_v2); + +struct unload_class_keys_call { + const u8 *volume_uuid; + const u8 *volume_key; + size_t volume_key_len; +}; + +static int call_unload_class_keys(const struct sep_fv_ops *ops, + void *argument) +{ + struct unload_class_keys_call *call = argument; + + return ops->unload_class_keys(apple_sep_fv_context, call->volume_uuid, + call->volume_key, + call->volume_key_len); +} + +int apple_sep_fv_unload_class_keys_v2(const u8 volume_uuid[16], + const u8 *volume_key, + size_t volume_key_len) +{ + struct unload_class_keys_call call = { + .volume_uuid = volume_uuid, + .volume_key = volume_key, + .volume_key_len = volume_key_len, + }; + + if (!volume_uuid || !volume_key || !volume_key_len) + return -EINVAL; + return apple_sep_fv_call(call_unload_class_keys, &call); +} +EXPORT_SYMBOL_GPL(apple_sep_fv_unload_class_keys_v2); + +struct file_key_call { + const u8 *volume_uuid; + u32 protection_class; + const u8 *wrapped_ekwk; + size_t wrapped_ekwk_len; + const u8 *wrapped_ek; + size_t wrapped_ek_len; + struct apple_sep_fv_key *key; +}; + +static int call_unwrap_file_key(const struct sep_fv_ops *ops, void *argument) +{ + struct file_key_call *call = argument; + + return ops->unwrap_file_key(apple_sep_fv_context, call->volume_uuid, + call->protection_class, call->wrapped_ekwk, + call->wrapped_ekwk_len, call->wrapped_ek, + call->wrapped_ek_len, call->key); +} + +int apple_sep_fv_unwrap_file_key(const u8 volume_uuid[16], + u32 protection_class, + const u8 *wrapped_ekwk, + size_t wrapped_ekwk_len, + const u8 *wrapped_ek, + size_t wrapped_ek_len, + struct apple_sep_fv_key *key) +{ + struct file_key_call call = { + .volume_uuid = volume_uuid, + .protection_class = protection_class, + .wrapped_ekwk = wrapped_ekwk, + .wrapped_ekwk_len = wrapped_ekwk_len, + .wrapped_ek = wrapped_ek, + .wrapped_ek_len = wrapped_ek_len, + .key = key, + }; + int ret; + + if (!volume_uuid || !wrapped_ekwk || !wrapped_ekwk_len || !wrapped_ek || + !wrapped_ek_len || !key) + return -EINVAL; + memzero_explicit(key, sizeof(*key)); + ret = apple_sep_fv_call(call_unwrap_file_key, &call); + if (ret) + memzero_explicit(key, sizeof(*key)); + return ret; +} +EXPORT_SYMBOL_GPL(apple_sep_fv_unwrap_file_key); + +struct new_file_key_call { + const u8 *volume_uuid; + u32 protection_class; + u64 crypto_id; + u16 key_revision; + struct apple_sep_fv_new_file_key *key; +}; + +static int call_new_file_key(const struct sep_fv_ops *ops, void *argument) +{ + struct new_file_key_call *call = argument; + + return ops->new_file_key(apple_sep_fv_context, call->volume_uuid, + call->protection_class, call->crypto_id, + call->key_revision, call->key); +} + +int apple_sep_fv_new_file_key_v2(const u8 volume_uuid[16], + u32 protection_class, + u64 crypto_id, u16 key_revision, + struct apple_sep_fv_new_file_key *key) +{ + struct new_file_key_call call = { + .volume_uuid = volume_uuid, + .protection_class = protection_class, + .crypto_id = crypto_id, + .key_revision = key_revision, + .key = key, + }; + int ret; + + if (!volume_uuid || !crypto_id || !key_revision || !key) + return -EINVAL; + memzero_explicit(key, sizeof(*key)); + ret = apple_sep_fv_call(call_new_file_key, &call); + if (ret) + memzero_explicit(key, sizeof(*key)); + return ret; +} +EXPORT_SYMBOL_GPL(apple_sep_fv_new_file_key_v2); diff --git a/drivers/soc/apple/image.rs b/drivers/soc/apple/image.rs index b81f8130fe602..8d4889268179d 100644 --- a/drivers/soc/apple/image.rs +++ b/drivers/soc/apple/image.rs @@ -48,21 +48,11 @@ pub(crate) fn sha256(bytes: &[u8]) -> Result<[u8; SHA256_LEN]> { const OFF_VERSION: usize = 0x10; const OFF_TIMESTAMP: usize = 0x14; -const OFF_FLAGS: usize = 0x1c; -const OFF_RESERVED: usize = 0x20; -const OFF_PROC_ID: usize = 0x28; -const OFF_PID: usize = 0x30; -const OFF_CDHASH: usize = 0x34; const OFF_TRAILER: usize = 0x48; static_assert!(OFF_VERSION == DIGEST_OFF + DIGEST_LEN); static_assert!(OFF_TIMESTAMP == OFF_VERSION + 4); -static_assert!(OFF_FLAGS == OFF_TIMESTAMP + 8); -static_assert!(OFF_RESERVED == OFF_FLAGS + 4); -static_assert!(OFF_PROC_ID == OFF_RESERVED + 8); -static_assert!(OFF_PID == OFF_PROC_ID + 8); -static_assert!(OFF_CDHASH == OFF_PID + 4); -static_assert!(OFF_TRAILER == OFF_CDHASH + 20); +static_assert!(OFF_TRAILER == OFF_TIMESTAMP + 8 + 4 + 8 + 8 + 4 + 20); static_assert!(OFF_TRAILER + 8 == HEADER_SIZE as usize); static_assert!(HEADER_WIRE == 4 + HEADER_SIZE as usize); @@ -89,6 +79,14 @@ pub(crate) struct Body { bytes: KVec, } +pub(crate) fn wipe(bytes: &mut [u8]) { + for byte in bytes { + // SAFETY: `byte` is uniquely borrowed and valid. A volatile write keeps + // the compiler from eliding destruction of request secrets. + unsafe { core::ptr::write_volatile(byte, 0) }; + } +} + impl Body { pub(crate) fn new() -> Body { Body { bytes: KVec::new() } @@ -125,6 +123,12 @@ impl Body { } } +impl Drop for Body { + fn drop(&mut self) { + wipe(&mut self.bytes); + } +} + const fn pad_of(len: usize) -> usize { len.wrapping_neg() % 4 } @@ -148,12 +152,19 @@ impl RequestImage { } } +impl Drop for RequestImage { + fn drop(&mut self) { + wipe(&mut self.bytes); + } +} + pub(crate) fn build_request( version: Version, timestamp_us: u64, body: &Body, ) -> Result { - let mut bytes = KVec::new(); + let mut image = RequestImage { bytes: KVec::new() }; + let bytes = &mut image.bytes; bytes.extend_from_slice(&HEADER_SIZE.to_le_bytes(), GFP_KERNEL)?; let header_at = bytes.len(); @@ -164,8 +175,8 @@ pub(crate) fn build_request( let put = |bytes: &mut KVec, off: usize, src: &[u8]| { bytes[header_at + off..header_at + off + src.len()].copy_from_slice(src); }; - put(&mut bytes, OFF_VERSION, &version.wire().to_le_bytes()); - put(&mut bytes, OFF_TIMESTAMP, ×tamp_us.to_le_bytes()); + put(bytes, OFF_VERSION, &version.wire().to_le_bytes()); + put(bytes, OFF_TIMESTAMP, ×tamp_us.to_le_bytes()); // Flags, reserved, proc_id, pid, cdhash and trailer stay zero; the enclave accepts that. bytes.extend_from_slice(body.as_slice(), GFP_KERNEL)?; @@ -195,7 +206,7 @@ pub(crate) fn build_request( bytes[header_at + DIGEST_OFF..header_at + DIGEST_OFF + DIGEST_LEN] .copy_from_slice(&digest[..DIGEST_LEN]); - Ok(RequestImage { bytes }) + Ok(image) } pub(crate) struct ResponseImage<'a> { @@ -232,7 +243,11 @@ pub(crate) fn read_blob(body: &[u8], off: usize) -> Option<(&[u8], usize)> { if body.len() < end { return None; } - Some((&body[len_end..end], end + pad_of(len))) + let next = end.checked_add(pad_of(len))?; + if body.get(end..next)?.iter().any(|byte| *byte != 0) { + return None; + } + Some((&body[len_end..end], next)) } pub(crate) fn operation_status(body: &[u8]) -> Option { diff --git a/drivers/soc/apple/keybag.rs b/drivers/soc/apple/keybag.rs index 90bed45619c50..515948b28b91f 100644 --- a/drivers/soc/apple/keybag.rs +++ b/drivers/soc/apple/keybag.rs @@ -86,7 +86,15 @@ const MAX_WRAPPED: usize = crate::store::MAX_VALUE; /// Proof the host holds no bag; create takes one by value, so it is unreachable /// without it and unrepeatable with it. -pub(crate) struct NoStoredKeyBag; +pub(crate) struct NoStoredKeyBag { + slot: Slot, +} + +impl NoStoredKeyBag { + pub(crate) fn slot(&self) -> Slot { + self.slot + } +} pub(crate) struct StoredKeyBag { wrapped: KVec, @@ -143,7 +151,7 @@ pub(crate) fn read(slot: Slot) -> Result { let file = match shim::StoreFile::open_readonly(slot.path()) { Ok(f) => f, Err(e) if e == ENOENT => { - return Ok(State::Absent(NoStoredKeyBag)); + return Ok(State::Absent(NoStoredKeyBag { slot })); } Err(e) => return Err(e), }; @@ -174,7 +182,7 @@ pub(crate) fn read(slot: Slot) -> Result { head[OFF_STATE + 3], ]); if state == STATE_REFUSED { - return Ok(State::Absent(NoStoredKeyBag)); + return Ok(State::Absent(NoStoredKeyBag { slot })); } let Some(provenance) = UuidProvenance::from_state(state) else { // Intent or unknown state: a bag may exist unnamed; creating again would @@ -233,6 +241,32 @@ pub(crate) fn read(slot: Slot) -> Result { })) } +pub(crate) fn write_intent(slot: Slot) -> Result<()> { + write_record(slot, 0, &[], &[0; UUID_LEN], &[]) +} + +pub(crate) fn mark_refused(slot: Slot) -> Result<()> { + write_record(slot, STATE_REFUSED, &[], &[0; UUID_LEN], &[]) +} + +pub(crate) fn write_bag_uuid( + slot: Slot, + wrapped: &[u8], + uuid: &[u8; UUID_LEN], + secret: &[u8], +) -> Result<()> { + if wrapped.is_empty() || wrapped.len() > MAX_WRAPPED || secret.len() > MAX_WRAPPED { + return Err(EINVAL); + } + write_record( + slot, + UuidProvenance::ReadBackFromBag.state(), + wrapped, + uuid, + secret, + ) +} + /// Replaces only the wrapped blob. The enclave ratchets bag material while a bag /// is active, so this snapshot must be re-taken at the catacomb-save commit or /// the enclave answers a later restore empty. diff --git a/drivers/soc/apple/proto.rs b/drivers/soc/apple/proto.rs index d084a889109c5..28324172c5e7b 100644 --- a/drivers/soc/apple/proto.rs +++ b/drivers/soc/apple/proto.rs @@ -3,8 +3,6 @@ //! The SEP wire protocol, reverse-engineered: opcode tables, request encoders //! and reply decoders. -#![allow(dead_code)] - use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; @@ -14,7 +12,6 @@ pub(crate) const EP_SHMEM: u8 = 0xFE; pub(crate) const EP_BOOT: u8 = 0xFF; pub(crate) const EP_XARM: u8 = 0x13; pub(crate) const EP_SBIO: u8 = 0x08; -pub(crate) const EP_XARS: u8 = 0x10; pub(crate) const EP_SCRD: u8 = 0x0a; pub(crate) const EP_SKS: u8 = 0x12; @@ -106,8 +103,6 @@ pub(crate) fn fourcc(msg: &Message) -> Fourcc { pub(crate) const CONTROL_REPLY_TYPE: u8 = 0x01; -pub(crate) const TAG_UNSOLICITED: u8 = 0x00; - pub(crate) const TAG_ENTROPY: u8 = 0xE7; pub(crate) const CONTROL_TIMEOUT_MS: u32 = 2000; @@ -115,7 +110,7 @@ pub(crate) const CONTROL_TIMEOUT_MS: u32 = 2000; pub(crate) const TAG_POOL_FIRST: u8 = 0x01; pub(crate) const TAG_POOL_LAST: u8 = 0x7e; -static_assert!(TAG_POOL_FIRST > TAG_UNSOLICITED); +static_assert!(TAG_POOL_FIRST > 0); static_assert!(TAG_POOL_LAST < TAG_ENTROPY); pub(crate) struct ControlOp { @@ -209,24 +204,6 @@ fn op_ool(ty: u8, endpoint: u8, data: u32, name: &'static CStr) -> ControlOp { } } -const OP_DMA_RING_PAGES: u8 = 0x19; -const OP_DMA_RING_ADDR: u8 = 0x1a; - -pub(crate) const DMA_RING_PAGES: u32 = 4; - -pub(crate) fn op_dma_ring_pages(endpoint: u8, pages: u32) -> ControlOp { - op_ool(OP_DMA_RING_PAGES, endpoint, pages, c"RING_PAGES") -} - -pub(crate) fn op_dma_ring_addr(endpoint: u8, iova: u64) -> ControlOp { - op_ool( - OP_DMA_RING_ADDR, - endpoint, - (iova >> IOVA_SHIFT) as u32, - c"RING_ADDR", - ) -} - pub(crate) fn op_ool_inbound_size(endpoint: u8, len: u32) -> ControlOp { op_ool(OP_OOL_INBOUND_SIZE, endpoint, len, c"OOL_IN_SIZE") } @@ -277,7 +254,11 @@ impl<'a> FieldCursor<'a> { let end = len_end.checked_add(len)?; let bytes = self.body.get(len_end..end)?; let pad = len.wrapping_neg() % 4; - self.at = end.checked_add(pad)?; + let next = end.checked_add(pad)?; + if self.body.get(end..next)?.iter().any(|byte| *byte != 0) { + return None; + } + self.at = next; Some(bytes) } } @@ -286,7 +267,6 @@ impl<'a> FieldCursor<'a> { pub(crate) struct ControlReply { pub(crate) tag: u8, pub(crate) data_lo: u32, - pub(crate) msg1: u32, } impl ControlReply { @@ -295,7 +275,6 @@ impl ControlReply { ControlReply { tag: f.tag, data_lo: f.data_lo, - msg1: msg.msg1, } } } diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index fff495601bb85..352609f6b4d08 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -3,8 +3,6 @@ //! Touch ID: the biometric endpoint (SBIO) and its `/dev/sep-bio` interface. //! The enclave matches; no biometric image ever crosses to userspace. -#![allow(dead_code)] - use super::*; use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; @@ -107,12 +105,10 @@ impl SepData { } fn stash_enrol_identity(&self, record: &[u8]) { - let mut found: KVec<([u8; bio::UUID_LEN], u32)> = KVec::new(); - crate::sbio::enrol_identity_candidates(record, SBIO_PROBE_USER_ID, |at, uuid| { - let _ = found.push((uuid, at as u32), GFP_KERNEL); + let mut found: KVec<[u8; bio::UUID_LEN]> = KVec::new(); + crate::sbio::enrol_identity_candidates(record, SBIO_PROBE_USER_ID, |_at, uuid| { + let _ = found.push(uuid, GFP_KERNEL); }); - for (_uuid, _at) in found.iter() { - } *self.enrol_identity_candidates.lock() = found; } @@ -120,27 +116,31 @@ impl SepData { let candidates = core::mem::take(&mut *self.enrol_identity_candidates.lock()); let listed = self.enclave_identities_for(SBIO_PROBE_USER_ID)?; - let mut confirmed: KVec<([u8; bio::UUID_LEN], u32)> = KVec::new(); - for (uuid, at) in candidates.iter() { - if listed.iter().any(|u| u == uuid) && !confirmed.iter().any(|(u, _)| u == uuid) { - if confirmed.push((*uuid, *at), GFP_KERNEL).is_err() { + let mut confirmed: KVec<[u8; bio::UUID_LEN]> = KVec::new(); + for uuid in candidates.iter() { + if listed.iter().any(|u| u == uuid) && !confirmed.iter().any(|u| u == uuid) { + if confirmed.push(*uuid, GFP_KERNEL).is_err() { break; } } } - match confirmed.len() { - 1 => { - let (uuid, _at) = confirmed[0]; - Some(uuid) - } - 0 => { - None + if confirmed.len() == 1 { + return Some(confirmed[0]); + } + + let index = self.bio_index.lock(); + let mut new_identity = None; + for uuid in listed.iter() { + if index.contains_uuid(uuid) { + continue; } - _ => { - None + if new_identity.is_some() { + return None; } + new_identity = Some(*uuid); } + new_identity } fn reconcile_identities(&self) { @@ -166,9 +166,6 @@ impl SepData { let index = self.bio_index.lock(); index.total() }; - for _uuid in listed.iter() { - } - if dropped.is_empty() && held == listed.len() { return; } @@ -475,7 +472,7 @@ impl SepData { crate::sbio::ComponentAction::AlreadyActive => { RestoreOutcome::AlreadyActive } - crate::sbio::ComponentAction::Unsupported(_) => { + crate::sbio::ComponentAction::Unsupported => { RestoreOutcome::Failed } crate::sbio::ComponentAction::Load => { @@ -750,27 +747,78 @@ impl SepData { } } - // ordering: the catacomb restore must complete before the sensor is woken + fn bring_sensor_online(&self) -> bool { + let Some(mut patch) = self.wake_sensor() else { + return false; + }; + + if !self.sensor_calibrated.load(Relaxed) { + if !self.calibrate_sensor() { + return false; + } + self.sensor_calibrated.store(true, Relaxed); + let Some(reloaded_patch) = self.wake_sensor() else { + return false; + }; + patch = reloaded_patch; + } + + self.complete_bringup(patch) + } + pub(crate) fn run_bringup(&self) { if self.bringup_started.xchg(true, Relaxed) { return; } - if let Err(e) = self.enable_sbio() { - dev_err!(self.dev, "bringup: could not enable the biometric transport ({:?}); Touch ID is unavailable this boot\n", e); - return; - } if let Err(e) = self.enable_sks() { dev_warn!(self.dev, "bringup: could not enable the key store ({:?}); keybag and ref-key operations are unavailable\n", e); + return; } - if let Ok(keybag::State::Present(stored)) = keybag::read(keybag::Slot::Identity) { - if let Some((handle, uuid)) = self.sks_recover(&stored) { - self.sks_designate_user_keybag(handle, stored.secret()); - self.sks_machine_refkey(handle, stored.secret()); - let prepared = self.cold_match_continue(handle, uuid); - self.ensure_restored_after(prepared); + if *module_parameters::provision_keybag.value() != 0 { + if *module_parameters::xart_writes.value() == 0 { + dev_err!(self.dev, "bringup: provision_keybag=1 requires xart_writes=1\n"); + return; + } + if !self.sks_provision_identity_keybag() { + dev_err!(self.dev, "bringup: identity-keybag provisioning failed; no retry this boot\n"); + return; } } + } + + fn activate_touchid(&self) -> Result<()> { + if self.touchid_started.load(Relaxed) { + return Ok(()); + } + + let mut store = store::Store::open()?; + let index = bio::IdentityIndex::load(&mut store)?; + *self.host_store.lock() = Some(store); + *self.bio_index.lock() = index; + + self.attach_sensor(); + self.enable_sbio()?; + let keybag::State::Present(stored) = keybag::read(keybag::Slot::Identity)? else { + return Err(ENOENT); + }; + let (handle, uuid) = self.sks_recover(&stored).ok_or(EIO)?; + self.sks_designate_user_keybag(handle, stored.secret()); + self.sks_machine_refkey(handle, stored.secret()); + let prepared = self.cold_match_continue(handle, uuid); + self.ensure_restored_after(prepared); self.attach_bringup(); + self.touchid_started.store(true, Relaxed); + Ok(()) + } + + // Touch ID starts from the caller's real-root namespace. SEP can therefore + // unlock root in initramfs without pinning biometric persistence to tmpfs. + fn prepare_bio_open(&self) -> Result<()> { + if let Err(e) = self.activate_touchid() { + dev_err!(self.dev, "Touch ID activation failed: {:?}\n", e); + return Err(e); + } + Ok(()) } pub(crate) fn run_verify(&self) { @@ -796,31 +844,8 @@ impl SepData { self.settle_before_capture(); - let Some(patch) = self.wake_sensor() else { - self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); - let _ = sensor::idle(); - return; - }; - - // enclave refuses a capture from an uncalibrated sensor (0x65 answers 1) - let already_calibrated = self.sensor_calibrated.load(Relaxed); - self.sensor_calibrated.store(true, Relaxed); - let patch = if already_calibrated { - patch - } else { - if !self.calibrate_sensor() { - self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); - let _ = sensor::idle(); - return; - } - let Some(patch) = self.wake_sensor() else { - self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); - let _ = sensor::idle(); - return; - }; - patch - }; - if !self.complete_bringup(patch) { + // The enclave refuses a capture from an uncalibrated sensor (0x65 answers 1). + if !self.bring_sensor_online() { self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); let _ = sensor::idle(); return; @@ -953,31 +978,7 @@ impl SepData { } pub(crate) fn run_enrolment(&self) { - let Some(patch) = self.wake_sensor() else { - self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); - let _ = sensor::idle(); - return; - }; - - let already_calibrated = self.sensor_calibrated.load(Relaxed); - self.sensor_calibrated.store(true, Relaxed); - let patch = if already_calibrated { - patch - } else { - if !self.calibrate_sensor() { - self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); - let _ = sensor::idle(); - return; - } - let Some(patch) = self.wake_sensor() else { - self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); - let _ = sensor::idle(); - return; - }; - patch - }; - - if !self.complete_bringup(patch) { + if !self.bring_sensor_online() { self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); let _ = sensor::idle(); return; @@ -1703,15 +1704,6 @@ impl SepData { } }; - let _repeated_header = { - let mut bytes = [0u8; transfer::HEADER_LEN]; - bytes.copy_from_slice(&header[..transfer::HEADER_LEN]); - let mut last = self.sbio_last_header.lock(); - let same = last.as_ref().is_some_and(|prev| *prev == bytes); - *last = Some(bytes); - same - }; - let chunk = packet.chunk as usize; let payload = if chunk == 0 { KVec::new() @@ -1738,14 +1730,12 @@ impl SepData { // 0xFE requests the peer's next packet and acks the final one; nothing to send self.sbio_wq.notify_all(); } - transfer::Progress::Ignored(_why) => {}, + transfer::Progress::Ignored => {}, transfer::Progress::Grant => { self.sbio_wq.notify_all(); } - transfer::Progress::Notification { tag: _tag, opcode: _opcode } => { - let _ = self.sbio_rx.lock().awaiting(); - } - transfer::Progress::Failed(_why) => { + transfer::Progress::Notification => {}, + transfer::Progress::Failed => { self.sbio_wq.notify_all(); } } @@ -1805,7 +1795,6 @@ impl SepData { let mut guard = self.sbio_rx.lock(); loop { if let Some(done) = guard.take_done() { - let _ = done.opcode; return Ok(done); } if remaining == 0 { @@ -1947,7 +1936,14 @@ impl SepData { } pub(crate) fn bio_open(&self) -> Result<()> { - bio::open(&mut self.bio_session.lock()) + let mut session = self.bio_session.lock(); + bio::open(&mut session)?; + drop(session); + if let Err(e) = self.prepare_bio_open() { + bio::release(&mut self.bio_session.lock()); + return Err(e); + } + Ok(()) } pub(crate) fn bio_release(&self) { @@ -2125,8 +2121,6 @@ impl SepData { return false; } - // unlock/0x18 is the destructive-template path (bag stays designated, locked) - self.cold_prepared.store(true, Relaxed); true } @@ -2273,8 +2267,6 @@ impl SbioOp { pub(crate) const SBIO_PROTOCOL_GENERATION: u32 = 1; -const OP_SBIO_INIT_COMMS: u16 = 0x73; - const OP_SBIO_REGISTER_SENSOR: u16 = 0x80; pub(crate) fn sbio_register_sensor(id: &crate::sensor::Identifier) -> SbioOp { SbioOp { @@ -2285,8 +2277,6 @@ pub(crate) fn sbio_register_sensor(id: &crate::sensor::Identifier) -> SbioOp { } } -const OP_SBIO_SEND_SERIAL: u16 = 0x48; - pub(crate) const SBIO_SIGNAL_QUALITY: u32 = 0; const OP_SBIO_COVERAGE_PARAMS: u16 = 0x5d; @@ -2413,10 +2403,7 @@ pub(crate) fn sbio_protected_config(id: UserId) -> SbioOp { const OP_SBIO_BEGIN_ENROL: u16 = 0x03; -// type 0 (ACM context) survives a reboot; type 1 (SKS token) does not pub(crate) const BE_AUTH_TYPE_ACM_CONTEXT: u32 = 0; -pub(crate) const BE_AUTH_TYPE_SKS_TOKEN: u32 = 1; -static_assert!(BE_AUTH_TYPE_ACM_CONTEXT != BE_AUTH_TYPE_SKS_TOKEN); pub(crate) const SBIO_BEGIN_ENROL_LEN: usize = 0x44; static_assert!(SBIO_BEGIN_ENROL_LEN <= SBIO_MAX_PAYLOAD); @@ -2426,22 +2413,7 @@ const BE_USER_ID: usize = 4; const BE_AUTH_TYPE: usize = 8; const BE_TOKEN_LEN: usize = 12; const BE_TOKEN: usize = 16; -const BE_SELECTOR: usize = 48; -static_assert!(BE_TOKEN + SKS_AUTH_TOKEN_LEN <= BE_SELECTOR); -const BE_SELECTOR_LEN: usize = 20; -static_assert!(BE_SELECTOR + BE_SELECTOR_LEN == SBIO_BEGIN_ENROL_LEN); - -pub(crate) const SBIO_BEGIN_ENROL_COPIED: usize = match SBIO_PROTOCOL_GENERATION { - 1 => 0x30, - 6 => SBIO_BEGIN_ENROL_LEN, - _ => 0, -}; -static_assert!(SBIO_BEGIN_ENROL_COPIED != 0); -static_assert!(SBIO_BEGIN_ENROL_COPIED <= SBIO_BEGIN_ENROL_LEN); - -const BE_SELECTOR_IN_RECORD: bool = SBIO_BEGIN_ENROL_COPIED >= BE_SELECTOR + BE_SELECTOR_LEN; - -static_assert!(!BE_SELECTOR_IN_RECORD); +static_assert!(BE_TOKEN + SKS_AUTH_TOKEN_LEN <= SBIO_BEGIN_ENROL_LEN); pub(crate) fn sbio_begin_enrol( user: UserId, @@ -2540,14 +2512,10 @@ pub(crate) fn sbio_match_policy() -> SbioOp { pub(crate) const SBIO_ENROL_RESULT_LEN: usize = 0xc98; -const ER_STATUS: usize = 0x000; -const ER_ERROR: usize = 0x002; const ER_PROGRESS: usize = 0x004; const ER_HAS_TEMPLATE: usize = 0x006; const ER_COMPLETE: usize = 0xbfe; -static_assert!(ER_STATUS + 2 <= SBIO_ENROL_RESULT_LEN); -static_assert!(ER_ERROR + 2 <= SBIO_ENROL_RESULT_LEN); static_assert!(ER_PROGRESS < SBIO_ENROL_RESULT_LEN); static_assert!(ER_HAS_TEMPLATE + 4 <= SBIO_ENROL_RESULT_LEN); static_assert!(ER_COMPLETE + 4 <= SBIO_ENROL_RESULT_LEN); @@ -2645,9 +2613,6 @@ static_assert!(SBIO_SAVED_USER_ID_AT + 4 <= SBIO_SAVED_MIN); pub(crate) const SBIO_SAVE_SELECTOR_LEN: usize = 24; const SS_USER_ID: usize = 0; -const SS_DEVICE: usize = 4; -const SS_DEVICE_LEN: usize = 20; -static_assert!(SS_DEVICE + SS_DEVICE_LEN == SBIO_SAVE_SELECTOR_LEN); static_assert!(SBIO_SAVE_SELECTOR_LEN <= SBIO_MAX_PAYLOAD); pub(crate) struct SaveSelector([u8; SBIO_SAVE_SELECTOR_LEN]); @@ -2689,10 +2654,6 @@ impl<'a> ComponentStates<'a> { self.0.len() / COMPONENT_PAIR_LEN } - pub(crate) fn trailing(&self) -> usize { - self.0.len() % COMPONENT_PAIR_LEN - } - pub(crate) fn pair(&self, i: usize) -> Option<(i32, u32)> { let at = i.checked_mul(COMPONENT_PAIR_LEN)?; let bytes = self.0.get(at..at + COMPONENT_PAIR_LEN)?; @@ -2713,7 +2674,7 @@ impl<'a> ComponentStates<'a> { pub(crate) enum ComponentAction { AlreadyActive, Load, - Unsupported(u32), + Unsupported, } pub(crate) fn component_action(state: u32) -> ComponentAction { @@ -2722,7 +2683,7 @@ pub(crate) fn component_action(state: u32) -> ComponentAction { } else if state & COMPONENT_STATE_COLD != 0 { ComponentAction::Load } else { - ComponentAction::Unsupported(state) + ComponentAction::Unsupported } } @@ -2825,10 +2786,6 @@ pub(crate) fn sbio_list_identities() -> SbioOp { } } -const OP_SBIO_GROUP_STATE: u16 = 0x79; - -const OP_SBIO_LIST_IDENTITIES_SCOPED: u16 = 0x6e; - pub(crate) struct IdentityRecords<'a>(&'a [u8]); impl<'a> IdentityRecords<'a> { @@ -2921,18 +2878,12 @@ pub(crate) const SBIO_MATCH_RESULT_LEN: usize = 0xca2; const MR_USER_ID: usize = 0x000; const MR_IDENTITY: usize = 0x004; -const MR_CANDIDATES: usize = 0x014; const MR_FLAGS: usize = 0xc8a; -const MR_SECOND_IDENTITY_UNUSED: usize = 0xc8e; -static_assert!(MR_IDENTITY != MR_SECOND_IDENTITY_UNUSED); +const MR_FLAG_MATCH: u32 = 1; static_assert!(MR_USER_ID + 4 <= SBIO_MATCH_RESULT_LEN); static_assert!(MR_IDENTITY + IDENTITY_UUID_LEN <= SBIO_MATCH_RESULT_LEN); -static_assert!(MR_CANDIDATES + 4 <= SBIO_MATCH_RESULT_LEN); static_assert!(MR_FLAGS + 4 <= SBIO_MATCH_RESULT_LEN); -static_assert!(MR_SECOND_IDENTITY_UNUSED + IDENTITY_UUID_LEN <= SBIO_MATCH_RESULT_LEN); static_assert!(MR_IDENTITY == MR_USER_ID + 4); -static_assert!(MR_IDENTITY + IDENTITY_UUID_LEN == MR_CANDIDATES); -static_assert!(MR_FLAGS > MR_USER_ID + 4); static_assert!(SBIO_MATCH_RESULT_LEN != SBIO_ENROL_RESULT_LEN); pub(crate) const IDENTITY_UUID_LEN: usize = 16; @@ -2965,6 +2916,7 @@ impl IdentityV1 { pub(crate) struct MatchResult { user_id: i32, identity: [u8; IDENTITY_UUID_LEN], + flags: u32, } impl MatchResult { @@ -2982,6 +2934,7 @@ impl MatchResult { bytes[MR_USER_ID + 3], ]), identity, + flags: u32::from_le_bytes(bytes[MR_FLAGS..MR_FLAGS + 4].try_into().ok()?), }) } @@ -2990,7 +2943,7 @@ impl MatchResult { } pub(crate) fn matches(&self, user: UserId) -> bool { - self.user_id == user.value() + self.flags & MR_FLAG_MATCH != 0 && self.user_id == user.value() } } @@ -3035,16 +2988,6 @@ pub(crate) fn sbio_register_sensor_serial(serial: &crate::sensor::SensorSerial) } } -const OP_SBIO_ENUMERATE: u16 = 0x7c; -pub(crate) const fn sbio_enumerate() -> SbioOp { - SbioOp { - opcode: OP_SBIO_ENUMERATE, - payload: [0; SBIO_MAX_PAYLOAD], - payload_len: 0, - name: c"ENUMERATE", - } -} - const OP_SBIO_DIAGNOSTICS: u16 = 0x63; pub(crate) const fn sbio_diagnostics() -> SbioOp { SbioOp { @@ -3055,12 +2998,6 @@ pub(crate) const fn sbio_diagnostics() -> SbioOp { } } -const SURVEY_ENUMERATE: SbioOp = sbio_enumerate(); -const SURVEY_DIAGNOSTICS: SbioOp = sbio_diagnostics(); - -static_assert!(SURVEY_ENUMERATE.payload_len == 0); -static_assert!(SURVEY_DIAGNOSTICS.payload_len == 0); - #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum ImagePurpose { Enrolment, @@ -3217,16 +3154,10 @@ pub(crate) fn sbio_enrolment_result() -> SbioOp { } pub(crate) const ASSESS_MIN_LEN: usize = 0x89; -pub(crate) const ASSESS_ERROR: usize = 0x00; pub(crate) const ASSESS_USABLE_MATCH: usize = 0x06; pub(crate) const ASSESS_USABLE_ENROL: usize = 0x07; -pub(crate) const ASSESS_FEEDBACK: usize = 0x0e; -pub(crate) const ASSESS_DIRTY: usize = 0x51; -static_assert!(ASSESS_ERROR + 2 <= ASSESS_MIN_LEN); static_assert!(ASSESS_USABLE_MATCH < ASSESS_USABLE_ENROL); -static_assert!(ASSESS_FEEDBACK + 4 <= ASSESS_MIN_LEN); -static_assert!(ASSESS_DIRTY < ASSESS_MIN_LEN); pub(crate) const SBIO_SESSION_SHARE_LEN: usize = 40; @@ -3301,7 +3232,7 @@ const fn encode_sbio(opcode: u16, marker: u8, seq: u16) -> Message { } } -static_assert!(encode_sbio(OP_SBIO_INIT_COMMS, transfer::MARKER_FIRST, 0).msg0 == 0x0000_0073_fc08); +static_assert!(encode_sbio(0x73, transfer::MARKER_FIRST, 0).msg0 == 0x0000_0073_fc08); pub(crate) fn encode_sbio_raw(opcode: u16, seq: u16) -> Option { Some(encode_sbio(opcode, transfer::MARKER_FIRST, seq)) diff --git a/drivers/soc/apple/scrd.rs b/drivers/soc/apple/scrd.rs index 0326308651ba4..acdcdab12db65 100644 --- a/drivers/soc/apple/scrd.rs +++ b/drivers/soc/apple/scrd.rs @@ -3,12 +3,10 @@ //! SEP credential endpoint (SCRD, EP 0x0a) wire protocol. -#![allow(dead_code)] - -use kernel::prelude::*; -use kernel::soc::apple::mailbox::Message; use crate::proto::*; use crate::sks::SKS_AUTH_TOKEN_LEN; +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; pub(crate) const SCRD_ACM_HANDLE_LEN: usize = SKS_AUTH_TOKEN_LEN; @@ -25,8 +23,6 @@ const SCRD_INIT_LOG_LEVEL: u8 = 0x28; const SCRD_POLICY_TOUCHID_ENROLLMENT: &[u8] = b"TouchIdEnrollment"; -pub(crate) const SCRD_CONTEXT_CREATE_REPLY_LEN: usize = SCRD_ACM_HANDLE_LEN + 1 + 4; - const SCRD_MAX_LOGICAL: usize = 64; static_assert!( 8 + SCRD_ACM_HANDLE_LEN + SCRD_POLICY_TOUCHID_ENROLLMENT.len() + 1 + 1 + 8 <= SCRD_MAX_LOGICAL diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 44eeb912868c7..79d9e3bc7d50c 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -111,14 +111,10 @@ static_assert!(STATUS_AT + STATUS_LEN == STATUS_XFER_LEN); // Offsets index the 16-byte status, not the 23-byte transfer. const STATUS_STATE: usize = STATUS_AT + 7; const STATUS_COUNT: usize = STATUS_AT + 12; -const STATUS_STATE_TRANSFER_REL: usize = 7; -const STATUS_COUNT_TRANSFER_REL: usize = 12; static_assert!(STATUS_STATE == 14); static_assert!(STATUS_COUNT == 19); static_assert!(STATUS_COUNT + 4 == STATUS_XFER_LEN); -static_assert!(STATUS_STATE != STATUS_STATE_TRANSFER_REL); -static_assert!(STATUS_COUNT != STATUS_COUNT_TRANSFER_REL); static_assert!(STATUS_STATE >= STATUS_AT && STATUS_STATE < STATUS_AT + STATUS_LEN); static_assert!(STATUS_COUNT >= STATUS_AT && STATUS_COUNT + 4 <= STATUS_AT + STATUS_LEN); @@ -251,11 +247,10 @@ fn send_framed(frame_type: u8, payload: &[u8]) -> Result<()> { check(unsafe { sep_sensor_xfer_tx(frame.as_ptr().cast(), total) }) } -const ENCRYPTED_DECLARED_MAX: usize = 0x12c; const ENCRYPTED_OVERHEAD: usize = FRAME_HEADER + CRC_LEN; static_assert!(ENCRYPTED_OVERHEAD == 9); const ENCRYPTED_TRANSFER_MAX: usize = 0x12b; -static_assert!(ENCRYPTED_TRANSFER_MAX < ENCRYPTED_DECLARED_MAX); +static_assert!(ENCRYPTED_TRANSFER_MAX < 0x12c); pub(crate) struct Geometry { declared: usize, @@ -293,14 +288,14 @@ impl Geometry { } } -static_assert!(Geometry::MODULE_CHALLENGE.declared < ENCRYPTED_DECLARED_MAX); +static_assert!(Geometry::MODULE_CHALLENGE.declared < 0x12c); static_assert!(Geometry::MODULE_CHALLENGE.declared > ENCRYPTED_OVERHEAD); static_assert!(Geometry::MODULE_CHALLENGE.transfer >= Geometry::MODULE_CHALLENGE.declared); static_assert!(Geometry::MODULE_CHALLENGE.transfer <= ENCRYPTED_TRANSFER_MAX); static_assert!(Geometry::MODULE_CHALLENGE.payload_capacity() == 0x40); -static_assert!(Geometry::COVERAGE.declared < ENCRYPTED_DECLARED_MAX); -static_assert!(Geometry::OPERATION.declared < ENCRYPTED_DECLARED_MAX); -static_assert!(Geometry::TRANSPARENT.declared < ENCRYPTED_DECLARED_MAX); +static_assert!(Geometry::COVERAGE.declared < 0x12c); +static_assert!(Geometry::OPERATION.declared < 0x12c); +static_assert!(Geometry::TRANSPARENT.declared < 0x12c); static_assert!(Geometry::COVERAGE.declared > ENCRYPTED_OVERHEAD); static_assert!(Geometry::OPERATION.declared > ENCRYPTED_OVERHEAD); static_assert!(Geometry::TRANSPARENT.declared > ENCRYPTED_OVERHEAD); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 68ddc2398a312..14cc90ec3b0ce 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -93,8 +93,6 @@ const ENDPOINTS_BEFORE_EXCHANGE: usize = 7; const OOL_SIZE_XARM: usize = 0x8000; -const OOL_SIZE_XARS: usize = 0x8000; - const OOL_SIZE_SBIO: usize = 0x4000; const OOL_SIZE_SCRD: usize = 0x4000; @@ -103,11 +101,9 @@ const SBIO_TIMEOUT_MS: time::Msecs = 5000; const SKS_ALLOC: usize = 0x8000; - +const SKS_SECRET_LEN: usize = 32; const SKS_MAX_CAPTURE: usize = 8; -const XARS_MAX_CAPTURE: usize = 8; - const SCRD_MAX_CAPTURE: usize = 8; const SCRD_TIMEOUT_MS: time::Msecs = 2000; @@ -153,11 +149,13 @@ const PROTECTED_DATA_AVAILABLE: bool = true; const RNG_MAX_WORDS_PER_READ: usize = 16; -const SECMODE_UNKNOWN: u32 = u32::MAX; +extern "C" { + fn sep_cancel_work_sync(work: *mut c_void); + fn sep_cancel_delayed_work_sync(work: *mut c_void); +} const MAX_ENDPOINTS: usize = 64; - #[derive(Clone, Copy)] struct Endpoint { id: u8, @@ -199,8 +197,6 @@ fn sks_declared_sizes() -> (usize, usize) { (0x8000, 0x4000) } -const SKS_MALFORMED: i8 = crate::sks::SKS_STATUS_MALFORMED; - struct Hex<'a>(&'a [u8]); impl kernel::fmt::Display for Hex<'_> { @@ -218,28 +214,22 @@ impl kernel::fmt::Display for Hex<'_> { } } -const SKS_SECRET_LEN: usize = 32; -static_assert!(SKS_SECRET_LEN % 4 == 0); - const SKS_LOCK_STATE_VARIANT: u32 = 1; #[derive(Clone, Copy)] enum LockState { Unlocked, - Locked, } impl LockState { const fn wire(self) -> i32 { match self { LockState::Unlocked => 0, - LockState::Locked => 1, } } } static_assert!(LockState::Unlocked.wire() == 0); -static_assert!(LockState::Locked.wire() == 1); const SKS_LOCK_STATE_FLAGS: u64 = 0; @@ -403,129 +393,15 @@ const ENROL_IDLE_TIMEOUT_MS: u32 = 2000; const PATCH_POLL_MS: u32 = 20; const PATCH_POLL_ATTEMPTS: u32 = 250; -const SEAL_VECTOR: &[u8] = b"apple-sep seal round trip v1"; - -const SEAL_STATUS_SHAPE: i8 = SKS_MALFORMED; - -const SEAL_STATUS_NAMES_NOTHING: i8 = -11; - -const SEAL_STATUS_WRONG_KIND: i8 = -12; - -const SEAL_STATUS_MISSING_PREREQUISITE: i8 = -3; -static_assert!(SEAL_STATUS_MISSING_PREREQUISITE != SEAL_STATUS_WRONG_KIND); -static_assert!(SEAL_STATUS_MISSING_PREREQUISITE != SEAL_STATUS_NAMES_NOTHING); - -const SEAL_STATUS_BACKUP_WRAP: i8 = -14; - -static_assert!(SEAL_STATUS_WRONG_KIND != SEAL_STATUS_NAMES_NOTHING); -static_assert!(SEAL_STATUS_WRONG_KIND != SEAL_STATUS_SHAPE); -static_assert!(SEAL_STATUS_BACKUP_WRAP != SEAL_STATUS_WRONG_KIND); -static_assert!(SEAL_STATUS_BACKUP_WRAP != SEAL_STATUS_SHAPE); - -const SKS_WRAP_OVERHEAD: usize = 256; - -const SKS_WRAP_PRODUCTION_CAPACITY: u32 = SEAL_VECTOR.len() as u32 + SKS_WRAP_OVERHEAD as u32; - -const SKS_CAPACITY_MAP: [u32; 23] = [ - 128, 192, 255, 256, 257, 258, 300, 320, 384, 512, 513, 640, 768, 1023, 1024, 1025, 1026, 1280, - 1536, 2048, 2049, 3072, 4096, -]; - -const fn capacity_map_ascends() -> bool { - let mut i = 1; - while i < SKS_CAPACITY_MAP.len() { - if SKS_CAPACITY_MAP[i - 1] >= SKS_CAPACITY_MAP[i] { - return false; - } - i += 1; - } - true -} -static_assert!(capacity_map_ascends()); -static_assert!(SKS_CAPACITY_MAP[0] > 0); -static_assert!(SKS_CAPACITY_MAP[5] < SKS_WRAP_PRODUCTION_CAPACITY); -static_assert!(SKS_WRAP_PRODUCTION_CAPACITY < SKS_CAPACITY_MAP[6]); -static_assert!(SKS_CAPACITY_MAP[2] + 1 == SKS_CAPACITY_MAP[3]); -static_assert!(SKS_CAPACITY_MAP[13] + 1 == SKS_CAPACITY_MAP[14]); - const CALIBRATION_FIRMWARE: &CStr = c"apple/mesa_calibration.bin"; -const SKS_CONFIG_MASKS: [u32; 17] = [ - 0xffff_ffff, - 0x1, - 0x3, - 0x5, - 0x9, - 0x11, - 0x21, - 0x41, - 0x81, - 0x101, - 0x201, - 0x401, - 0x801, - 0x1001, - 0x2001, - 0x4001, - 0x8001, -]; - -const fn config_masks_are_single_bit_probes() -> bool { - if SKS_CONFIG_MASKS[0] != u32::MAX || SKS_CONFIG_MASKS[1] != 1 { - return false; - } - let mut i = 2; - while i < SKS_CONFIG_MASKS.len() { - let m = SKS_CONFIG_MASKS[i]; - if m & 1 != 1 || (m & !1u32).count_ones() != 1 { - return false; - } - if i > 2 && SKS_CONFIG_MASKS[i - 1] >= m { - return false; - } - i += 1; - } - true -} -static_assert!(config_masks_are_single_bit_probes()); - -const CONFIG_STAGE_NONE: u32 = 0; const SBIO_PROBE_USER_ID: i32 = 1000; static_assert!(SBIO_PROBE_USER_ID >= crate::sks::SKS_DESIGNATE_USER_MIN); static_assert!(SBIO_PROBE_USER_ID == bio::ENROL_USER_ID); static_assert!(SBIO_PROBE_USER_ID > 0); -static_assert!(SBIO_PROBE_USER_ID == crate::sks::SKS_IDENTITY_USER_ID); const SKS_LOAD_REPLY_LEN: usize = 8; -// StoreType: the u32 at +0x60 of the create body -#[derive(Clone, Copy, PartialEq, Eq)] -struct StoreType(u32); - -const STORE_TYPE_MAX: u32 = 8; - -impl StoreType { - const IDENTITY: StoreType = StoreType(0); - const SEALING: StoreType = StoreType(1); - - const fn new(value: u32) -> Option { - if value <= STORE_TYPE_MAX { - Some(StoreType(value)) - } else { - None - } - } - - const fn wire(self) -> u32 { - self.0 - } -} - -static_assert!(StoreType::IDENTITY.wire() == 0); -static_assert!(StoreType::SEALING.wire() == 1); -static_assert!(StoreType::new(STORE_TYPE_MAX).is_some()); -static_assert!(StoreType::new(STORE_TYPE_MAX + 1).is_none()); - #[derive(Clone, Copy)] #[must_use] @@ -560,12 +436,7 @@ impl core::ops::Deref for Secret { impl Drop for Secret { fn drop(&mut self) { - for b in self.0.iter_mut() { - // SAFETY: `b` is a valid, uniquely borrowed byte for this write, and - // a volatile store is what stops the compiler discarding a wipe of - // memory nothing reads afterwards. - unsafe { core::ptr::write_volatile(b, 0) }; - } + image::wipe(&mut self.0); } } @@ -599,20 +470,6 @@ impl SksProbe { } } -struct XarsProbe { - active: bool, - captured: KVec, -} - -impl XarsProbe { - fn new() -> Self { - XarsProbe { - active: false, - captured: KVec::new(), - } - } -} - struct ScrdProbe { active: bool, captured: KVec, @@ -699,8 +556,6 @@ struct SepData { #[pin] control_wq: CondVar, - security_mode: Atomic, - sks_wedged: Atomic, sks_seq: Atomic, @@ -714,9 +569,6 @@ struct SepData { #[pin] ool_sbio: Mutex>, - #[pin] - sbio_last_header: Mutex>, - #[pin] dma_ring: Mutex>, @@ -730,8 +582,6 @@ struct SepData { templates_restored: Atomic, - restore_attempted: Atomic, - sensor_calibrated: Atomic, enrol_open: Atomic, @@ -740,23 +590,16 @@ struct SepData { enrol_material: Mutex>, #[pin] - enrol_identity_candidates: Mutex>, + enrol_identity_candidates: Mutex>, last_capture_end_ns: Atomic, device_view_synced: Atomic, - backup_bag_other: Atomic, - - borrowed_sealing_handle: Atomic, - sealing_designations: Atomic, - config_stage: Atomic, - config_writes_sent: Atomic, - keybag_designated: Atomic, - cold_prepared: Atomic, bringup_started: Atomic, + touchid_started: Atomic, bringup: Atomic, @@ -768,13 +611,6 @@ struct SepData { #[pin] sks_wq: CondVar, - #[pin] - ool_xars: Mutex>, - #[pin] - xars_probe: Mutex, - #[pin] - xars_wq: CondVar, - #[pin] ool_scrd: Mutex>, #[pin] @@ -808,6 +644,9 @@ struct SepData { #[pin] machine_refkey: Mutex>, + #[pin] + fv_volumes: Mutex>, + rng_shutdown: Atomic, rng_failures: Atomic, @@ -820,6 +659,8 @@ struct SepData { registered: Atomic, + shutting_down: Atomic, + #[pin] rx_work: Work, @@ -872,12 +713,6 @@ impl SepData { )?; let sks_declared = sks_declared_sizes(); let ool_sks = Self::alloc_ool(dev, proto::EP_SKS, SKS_ALLOC, sks_declared)?; - let ool_xars = Self::alloc_ool( - dev, - xarm::EP_XARS, - OOL_SIZE_XARS, - (OOL_SIZE_XARS, OOL_SIZE_XARS), - )?; let ool_scrd = Self::alloc_ool( dev, proto::EP_SCRD, @@ -937,36 +772,25 @@ impl SepData { endpoints <- new_mutex!(EndpointTable::new()), control <- new_mutex!(control::ControlState::new()), control_wq <- new_condvar!("SepData::control_wq"), - security_mode: Atomic::new(SECMODE_UNKNOWN), sks_seq: Atomic::new(0), sks_wedged: Atomic::new(0), bringup: Atomic::new(BRINGUP_FRESH), keybag_designated: Atomic::new(false), - cold_prepared: Atomic::new(false), bringup_started: Atomic::new(false), + touchid_started: Atomic::new(false), enrol_open: Atomic::new(false), sensor_calibrated: Atomic::new(false), templates_restored: Atomic::new(false), - restore_attempted: Atomic::new(false), enrol_material <- new_mutex!(None), enrol_identity_candidates <- new_mutex!(KVec::new()), last_capture_end_ns: Atomic::new(0), device_view_synced: Atomic::new(false), - backup_bag_other: Atomic::new(0), - borrowed_sealing_handle: Atomic::new(0), - sealing_designations: Atomic::new(0), - config_writes_sent: Atomic::new(0), - config_stage: Atomic::new(CONFIG_STAGE_NONE), phase: Atomic::new(PHASE_ATTACH), ool_xarm <- new_mutex!(Some(ool_xarm)), ool_sbio <- new_mutex!(Some(ool_sbio)), - sbio_last_header <- new_mutex!(None::<[u8; transfer::HEADER_LEN]>), ool_sks <- new_mutex!(Some(ool_sks)), sks_probe <- new_mutex!(SksProbe::new()), sks_wq <- new_condvar!("SepData::sks_wq"), - ool_xars <- new_mutex!(Some(ool_xars)), - xars_probe <- new_mutex!(XarsProbe::new()), - xars_wq <- new_condvar!("SepData::xars_wq"), ool_scrd <- new_mutex!(Some(ool_scrd)), scrd_probe <- new_mutex!(ScrdProbe::new()), scrd_wq <- new_condvar!("SepData::scrd_wq"), @@ -983,6 +807,7 @@ impl SepData { xarm <- new_mutex!(XarmState::new()), rng <- new_mutex!(None), machine_refkey <- new_mutex!(None), + fv_volumes <- new_mutex!(KVec::new()), rng_shutdown: Atomic::new(false), rng_failures: Atomic::new(0), rx: rxring::RxRing::new(), @@ -990,6 +815,7 @@ impl SepData { settle_mark: Atomic::new(0), settle_idle_ticks: Atomic::new(0), registered: Atomic::new(false), + shutting_down: Atomic::new(false), rx_work <- new_work!("SepData::rx_work"), enrol_work <- new_work!("SepData::enrol_work"), verify_work <- new_work!("SepData::verify_work"), @@ -1100,7 +926,7 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); let mut guard = self.control.lock(); let result = loop { - if let Some((value, _msg1)) = guard.entropy_take() { + if let Some(value) = guard.entropy_take() { break Ok(value); } if self.rng_shutdown.load(Relaxed) { @@ -1128,13 +954,7 @@ impl SepData { let _ = self.control_request(&proto::op_nop(param)); } - match self.control_request(&proto::op_security_mode()) { - Ok(Some(reply)) => { - self.security_mode.store(reply.data_lo, Relaxed); - } - Ok(None) => {} - Err(_) => {} - } + let _ = self.control_request(&proto::op_security_mode()); let mut words = [0u32; 4]; let mut got = 0; @@ -1257,40 +1077,30 @@ impl SepData { store.as_mut().map(f) } - // pre-generate: an entropy draw on the drain path would deadlock on its own reply fn prepare_os_uuid(&self) { - let key = store::Key::root(0xf0); - - let existing = match self.with_host_store(|store| store.read(&key)) { - Some(result) => result, - None => return, - }; - - if let Ok(Some(value)) = existing { - if value.len() == 16 { - let mut uuid = [0u8; 16]; - uuid.copy_from_slice(&value); - self.xarm.lock().os_uuid = Some(uuid); - return; - } - } - - let mut uuid = [0u8; 16]; - if shim::random_bytes(&mut uuid).is_err() { + let hi = *module_parameters::os_uuid_hi.value(); + let lo = *module_parameters::os_uuid_lo.value(); + if hi != 0 || lo != 0 { + let mut uuid = [0u8; 16]; + uuid[..8].copy_from_slice(&hi.to_be_bytes()); + uuid[8..].copy_from_slice(&lo.to_be_bytes()); + self.xarm.lock().os_uuid = Some(uuid); + dev_info!( + self.dev, + "xART: using explicit OS UUID {:016x}-{:016x}\n", + hi, + lo + ); return; } - xarm::make_uuid_v4(&mut uuid); - match self.with_host_store(|store| store.write(&key, &uuid)) { - Some(Ok(())) => {} - Some(Err(_)) => { - return; - } - None => { - return; - } + if let Some(uuid) = dt::preboot_uuid() { + self.xarm.lock().os_uuid = Some(uuid); + dev_info!(self.dev, "xART: using /chosen/apfs-preboot-uuid\n"); + } else { + self.xarm.lock().os_uuid = None; + dev_warn!(self.dev, "xART: /chosen/apfs-preboot-uuid is unavailable\n"); } - self.xarm.lock().os_uuid = Some(uuid); } fn on_xarm(&self, msg: Message) { @@ -1456,18 +1266,6 @@ impl SepData { } - fn on_xars(&self, msg: Message) { - let mut probe = self.xars_probe.lock(); - if probe.active { - if probe.captured.len() < XARS_MAX_CAPTURE { - let _ = probe.captured.push(msg, GFP_KERNEL); - } - drop(probe); - self.xars_wq.notify_all(); - } - } - - fn on_scrd(&self, msg: Message) { let mut probe = self.scrd_probe.lock(); if probe.active { @@ -1783,6 +1581,9 @@ impl SepData { fn arm_settle(this: &Arc) { + if this.shutting_down.load(Relaxed) { + return; + } let delay = time::msecs_to_jiffies(SETTLE_MS); let _ = workqueue::system() .enqueue_delayed::, SETTLE_WORK_ID>(this.clone(), delay); @@ -1889,9 +1690,6 @@ impl SepData { proto::EP_SKS => self.on_sks(msg), - // xars: only 0x08 and 0x04 are sent; other opcodes are destructive - xarm::EP_XARS => self.on_xars(msg), - proto::EP_SCRD => self.on_scrd(msg), proto::EP_BOOT => dev_warn!( @@ -1909,7 +1707,6 @@ impl SepData { fn on_control(&self, msg: Message, f: proto::Fields) { if f.ty != proto::CONTROL_REPLY_TYPE { - self.control.lock().note_unsolicited(); return; } @@ -1963,8 +1760,9 @@ impl SepData { fn remove(&self) { - // first: resets the keyring static calls before freeing, so no keyctl op hits freed data + self.shutting_down.store(true, Relaxed); trusted::unregister(); + self.unregister_fv_kernel(); let dev = self.bio_dev.lock().take(); if dev.is_some() { @@ -1976,17 +1774,35 @@ impl SepData { self.rng_shutdown.store(true, Relaxed); self.control_wq.notify_all(); + self.sbio_wq.notify_all(); + self.sks_wq.notify_all(); + self.scrd_wq.notify_all(); if let Some(mut handle) = self.rng.lock().take() { handle.unregister(); } - // stop the mailbox last: no callbacks after this *self.mbox.lock() = None; + // SAFETY: all four pointers refer to pinned work fields in `self`. + // Shutdown blocks requeueing, and the mailbox can no longer add work. + unsafe { + sep_cancel_work_sync(Work::raw_get(core::ptr::addr_of!(self.rx_work)).cast()); + sep_cancel_work_sync(Work::raw_get(core::ptr::addr_of!(self.enrol_work)).cast()); + sep_cancel_work_sync(Work::raw_get(core::ptr::addr_of!(self.verify_work)).cast()); + sep_cancel_delayed_work_sync( + DelayedWork::raw_as_work(core::ptr::addr_of!(self.settle_work)).cast(), + ); + } + let _ = self.store.lock().take(); let _ = self.host_store.lock().take(); - for slot in [&self.ool_xarm, &self.ool_sbio, &self.ool_sks] { + for slot in [ + &self.ool_xarm, + &self.ool_sbio, + &self.ool_sks, + &self.ool_scrd, + ] { if let Some(buffers) = slot.lock().take() { if buffers.registered { core::mem::forget(buffers); @@ -2086,6 +1902,9 @@ impl MailCallback for SepData { type Data = Arc; fn recv_message(data: ::Borrowed<'_>, msg: Message) { + if data.shutting_down.load(Relaxed) { + return; + } if !data.rx.push(msg) && data.rx.dropped() == 1 { dev_err!( data.dev, @@ -2103,6 +1922,9 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { + if this.shutting_down.load(Relaxed) { + return; + } if this.drain() > 0 { SepData::arm_settle(&this); } @@ -2113,6 +1935,9 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { + if this.shutting_down.load(Relaxed) { + return; + } SepData::settle_tick(&this); } } @@ -2121,6 +1946,9 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { + if this.shutting_down.load(Relaxed) { + return; + } this.run_enrolment(); } } @@ -2129,6 +1957,9 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { + if this.shutting_down.load(Relaxed) { + return; + } this.run_verify(); } } @@ -2149,6 +1980,12 @@ impl platform::Driver for SepDriver { _info: Option<&()>, ) -> impl PinInit { let dev: &device::Device = pdev.as_ref(); + if *module_parameters::provision_keybag.value() != 0 + && *module_parameters::xart_writes.value() == 0 + { + dev_err!(dev, "provision_keybag=1 requires xart_writes=1\n"); + return Err(EINVAL); + } let sep_node = dt::DtNode::of_device(dev).ok_or(ENODEV)?; if dt::registration_already_sent(&sep_node) { @@ -2174,7 +2011,9 @@ impl platform::Driver for SepDriver { data.attach(&sep_node)?; - data.attach_sensor(); + if let Err(e) = data.register_fv_kernel() { + dev_err!(dev, "could not register the FileVault kernel API: {:?}\n", e); + } if let Err(e) = trusted::register(data.clone()) { dev_warn!(data.dev, "trusted-keys: registration failed ({:?})\n", e); @@ -2227,5 +2066,17 @@ module! { default: 0, description: "Allow writes to the validated shared xART mapping", }, + provision_keybag: u8 { + default: 0, + description: "Create the Linux identity keybag when none exists; requires xart_writes=1", + }, + os_uuid_hi: u64 { + default: 0, + description: "High 64 bits of an explicit xART OS UUID", + }, + os_uuid_lo: u64 { + default: 0, + description: "Low 64 bits of an explicit xART OS UUID", + }, }, } diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index e39f036142825..34c34f29bc68b 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -18,6 +18,11 @@ #include +struct apple_sep_fv_new_file_key; + +void sep_cancel_work_sync(void *work); +void sep_cancel_delayed_work_sync(void *work); + /* -- hwrng_shim.c ------------------------------------------------------- */ void *sep_hwrng_alloc(void); @@ -48,6 +53,8 @@ int sep_sha256(const void *a, size_t alen, const void *b, size_t blen, /* -- crypto_shim.c ------------------------------------------------------ */ +int sep_random_bytes(void *buf, size_t len); + /* HMAC-SHA256 over one message, 32 bytes out. */ int sep_hmac_sha256(const void *key, size_t keylen, const void *data, size_t datalen, u8 *out); @@ -61,12 +68,40 @@ int sep_gcm(int encrypt, const void *key, size_t keylen, const void *iv, size_t ivlen, size_t aadlen, void *buf, size_t buflen, size_t datalen); -/* - * Fill `buf` from the kernel CSPRNG, seeded, or fail. For key-bag secrets only; - * anything measuring the enclave's own entropy (0x59 device-key probe, - * /dev/hwrng) stays on SEP. - */ -int sep_random_bytes(void *buf, size_t len); +/* -- fv_shim.c -------------------------------------------------------- */ + +struct apple_sep_fv_key; + +struct sep_fv_ops { + int (*unwrap_media_key)(void *context, const u8 *wrapped, + size_t wrapped_len, u32 protection_class, + struct apple_sep_fv_key *key); + int (*unwrap_volume_key)(void *context, const u8 *secret, + size_t secret_len, const u8 *unlock_record, + size_t unlock_record_len, const u8 *volume_key, + size_t volume_key_len, + struct apple_sep_fv_key *key); + int (*load_class_keys)(void *context, const u8 volume_uuid[16], + const u8 *secret, + size_t secret_len, const u8 *unlock_record, + size_t unlock_record_len, const u8 *volume_key, + size_t volume_key_len); + int (*unload_class_keys)(void *context, const u8 volume_uuid[16], + const u8 *volume_key, + size_t volume_key_len); + int (*unwrap_file_key)(void *context, const u8 volume_uuid[16], + u32 protection_class, const u8 *wrapped_ekwk, + size_t wrapped_ekwk_len, const u8 *wrapped_ek, + size_t wrapped_ek_len, + struct apple_sep_fv_key *key); + int (*new_file_key)(void *context, const u8 volume_uuid[16], + u32 protection_class, u64 crypto_id, + u16 key_revision, + struct apple_sep_fv_new_file_key *key); +}; + +int sep_fv_register_v2(void *context, const struct sep_fv_ops *ops); +void sep_fv_unregister_v2(void *context); /* -- p256_shim.c ------------------------------------------------------- */ diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index a615b3aee0a96..f02a9b86e32ef 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -12,21 +12,18 @@ extern "C" { fn sep_store_open_block(path: *const c_char, writable: c_int) -> *mut c_void; fn sep_store_close(handle: *mut c_void); fn sep_store_size(handle: *mut c_void) -> i64; - fn sep_store_read(handle: *mut c_void, off: i64, buf: *mut c_void, len: usize) - -> c_long; - fn sep_store_write( - handle: *mut c_void, - off: i64, - buf: *const c_void, - len: usize, - ) -> c_long; + fn sep_store_read(handle: *mut c_void, off: i64, buf: *mut c_void, len: usize) -> c_long; + fn sep_store_write(handle: *mut c_void, off: i64, buf: *const c_void, len: usize) -> c_long; fn sep_store_sync(handle: *mut c_void) -> c_int; fn sep_random_bytes(buf: *mut c_void, len: usize) -> c_int; } pub(crate) fn random_bytes(buf: &mut [u8]) -> Result<()> { - // SAFETY: `buf` is writable for exactly `buf.len()` bytes. - kernel::error::to_result(unsafe { sep_random_bytes(buf.as_mut_ptr().cast(), buf.len()) }) + // SAFETY: `buf.as_mut_ptr()` is valid for writes of `buf.len()` bytes for + // the duration of the call, and `sep_random_bytes` writes exactly that many + // bytes (via `get_random_bytes`) and retains no reference to the buffer. + let ret = unsafe { sep_random_bytes(buf.as_mut_ptr().cast(), buf.len()) }; + kernel::error::to_result(ret) } /// Backing-store file handle. @@ -199,9 +196,6 @@ extern "C" { fn sep_bio_boottime_ns() -> u64; } -extern "C" { -} - pub(crate) fn capable_admin() -> bool { // SAFETY: no preconditions; reads the current task's credentials. unsafe { sep_bio_capable_admin() != 0 } diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index f30f4870db076..4d68f40bab0d7 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -3,13 +3,41 @@ //! SEP key store (SKS, endpoint `0x12`): key-bag and key-management request/reply //! framing, DER-imaged request builders, and lock-state control. -#![allow(dead_code)] - use super::*; use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; use crate::proto::*; +struct KeybagCreateIntent<'a> { + dev: &'a device::Device, + slot: keybag::Slot, + sent: bool, +} + +impl<'a> KeybagCreateIntent<'a> { + fn new(dev: &'a device::Device, slot: keybag::Slot) -> Option { + if let Err(e) = keybag::write_intent(slot) { + dev_err!(dev, "sks: cannot persist keybag-create intent: {:?}\n", e); + return None; + } + Some(Self { dev, slot, sent: false }) + } + + fn sending(&mut self) { + self.sent = true; + } +} + +impl Drop for KeybagCreateIntent<'_> { + fn drop(&mut self) { + if !self.sent { + if let Err(e) = keybag::mark_refused(self.slot) { + dev_err!(self.dev, "sks: cannot mark unsent keybag create retryable: {:?}\n", e); + } + } + } +} + impl SepData { pub(crate) fn on_sks(&self, msg: Message) { let r = crate::sks::decode_sks(&msg); @@ -30,6 +58,7 @@ impl SepData { .copied(); drop(probe); if let Some(a) = matched { + let _ = self.sks_zero_buffers(); self.sks_wedged.store(0, Relaxed); dev_warn!(self.dev, "sks: late answer to {} after {} ms; wedge lifted\n", a.label, a.waited_ms); } @@ -130,6 +159,7 @@ impl SepData { self.sks_arm(label); if self.send(msg).is_err() { self.sks_disarm(); + let _ = self.sks_zero_buffers(); return None; } let started_ns = crate::shim::boottime_ns(); @@ -191,6 +221,10 @@ impl SepData { } } + // The correlated reply means the enclave has finished with both OOL + // buffers. Keep only the private response copy returned to the caller. + let _ = self.sks_zero_buffers(); + Some(SksOutcome { reply, response }) } @@ -325,6 +359,15 @@ impl SepData { self.sks_seal(&op, &body) } + fn sks_req_copy_keybag(&self, handle: crate::sks::KeyBagHandle) -> Result { + let op = crate::sks::sks_copy_keybag(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + self.sks_seal(&op, &body) + } + /// `0x02` copy the designated biometric identity bag. fn sks_req_copy_keybag_special(&self, handle: crate::sks::SpecialHandle) -> Result { let op = crate::sks::sks_copy_keybag(); @@ -352,6 +395,107 @@ impl SepData { }) } + fn sks_req_create_identity_keybag( + &self, + secret: &[u8], + uuid: &[u8; crate::sks::SKS_IDENTITY_UUID_LEN], + proof: keybag::NoStoredKeyBag, + ) -> Result { + if proof.slot() != keybag::Slot::Identity { + return Err(EINVAL); + } + let mut body = image::Body::new(); + body.put_u32(crate::sks::SKS_CREATE_VARIANT_IDENTITY)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_u32(crate::sks::CreateFlags::none().value())?; + body.put_i32(crate::sks::SpecialHandle::first_identity().value())?; + body.put_blob(secret)?; + body.put_blob(&[])?; + body.put_blob(uuid)?; + body.put_blob(&[])?; + body.put_u64(0)?; + body.put_u64(0)?; + body.put_blob(&[])?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_create(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { name: crate::sks::SKS_CREATE_NAME, msg, img }) + } + + pub(crate) fn sks_provision_identity_keybag(&self) -> bool { + let proof = match keybag::read(keybag::Slot::Identity) { + Ok(keybag::State::Present(_)) => return true, + Ok(keybag::State::Absent(proof)) => proof, + Err(e) => { + dev_err!(self.dev, "sks: identity keybag state is ambiguous: {:?}\n", e); + return false; + } + }; + let slot = proof.slot(); + let mut intent = match KeybagCreateIntent::new(&self.dev, slot) { + Some(intent) => intent, + None => return false, + }; + let mut secret_bytes = KVec::new(); + if secret_bytes.resize(SKS_SECRET_LEN, 0, GFP_KERNEL).is_err() + || shim::random_bytes(&mut secret_bytes).is_err() + { + return false; + } + let secret = Secret(secret_bytes); + let mut uuid = [0u8; crate::sks::SKS_IDENTITY_UUID_LEN]; + if shim::random_bytes(&mut uuid).is_err() { + return false; + } + uuid[6] = (uuid[6] & 0x0f) | 0x40; + uuid[8] = (uuid[8] & 0x3f) | 0x80; + + let request = match self.sks_req_create_identity_keybag(&secret, &uuid, proof) { + Ok(request) => request, + Err(e) => { + dev_err!(self.dev, "sks: cannot build identity keybag request: {:?}\n", e); + return false; + } + }; + intent.sending(); + let Some(out) = self.sks_exchange(request.name, request.msg, &request.img) else { + return false; + }; + let Some(body) = self.sks_report_response(crate::sks::SKS_CREATE_NAME, &out) else { + return false; + }; + if out.reply.status != 0 || body.len() < 12 { + dev_err!(self.dev, "sks: CREATE_KEYBAG failed: mailbox {}, body {} bytes\n", out.reply.status, body.len()); + return false; + } + let variant = u32::from_le_bytes(body[0..4].try_into().unwrap()); + let raw_handle = i32::from_le_bytes(body[4..8].try_into().unwrap()); + let Some((_fv_data, end)) = image::read_blob(body, 8) else { + return false; + }; + if variant != crate::sks::SKS_CREATE_VARIANT_IDENTITY || raw_handle < 0 || end != body.len() { + dev_err!(self.dev, "sks: invalid CREATE_KEYBAG reply: variant {}, handle {}\n", variant, raw_handle); + return false; + } + let handle = crate::sks::KeyBagHandle::from_create_reply(raw_handle); + let Some(bag_uuid) = self.sks_read_uuid(handle) else { + return false; + }; + let Some(out) = self.sks_send(self.sks_req_copy_keybag(handle)) else { + return false; + }; + let Some(wrapped) = self.wrapped_from_copy_reply(&out, c"new identity keybag") else { + return false; + }; + if let Err(e) = keybag::write_bag_uuid(slot, &wrapped, &bag_uuid, &secret) { + dev_err!(self.dev, "sks: cannot commit identity keybag: {:?}\n", e); + return false; + } + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + dev_info!(self.dev, "sks: identity keybag provisioned\n"); + true + } + pub(crate) fn sks_send(&self, req: Result) -> Option { match req { Ok(r) => self.sks_exchange(r.name, r.msg, &r.img), @@ -499,16 +643,21 @@ impl SepData { return None; } if body.len() != SKS_LOAD_REPLY_LEN { + dev_warn!(self.dev, "sks: LOAD_KEYBAG returned {} bytes, expected {}\n", body.len(), SKS_LOAD_REPLY_LEN); return None; } let status = i32::from_le_bytes([body[0], body[1], body[2], body[3]]); let handle = i32::from_le_bytes([body[4], body[5], body[6], body[7]]); if status != 0 || handle < 0 { + dev_warn!(self.dev, "sks: LOAD_KEYBAG operation status {}, handle {}\n", status, handle); return None; } let handle = crate::sks::KeyBagHandle::from_load_reply(handle); - let uuid = self.sks_read_uuid(handle)?; + let Some(uuid) = self.sks_read_uuid(handle) else { + dev_warn!(self.dev, "sks: loaded keybag has no readable UUID\n"); + return None; + }; if uuid == *stored.uuid() { Some((handle, uuid)) @@ -636,12 +785,8 @@ impl SepData { } } -const SKS_SELECTOR_MAX: u8 = 0x5f; - pub(crate) const SKS_REPLY_BIT: u8 = 0x80; -const OP_SKS_REWRAP_FORBIDDEN: u8 = 0x0f; - pub(crate) struct SksOp { opcode: u8, name: &'static CStr, @@ -656,47 +801,57 @@ impl SksOp { } } -const OP_SKS_DEVICE_STATE: u8 = 0x19; -pub(crate) fn sks_get_device_state() -> SksOp { - SksOp { - opcode: OP_SKS_DEVICE_STATE, - name: c"GET_DEVICE_STATE", - } -} +const OP_SKS_UNWRAP_PFK: u8 = 0x09; +pub(crate) const SKS_UNWRAP_PFK_NAME: &CStr = c"UNWRAP_PFK"; -const OP_SKS_GET_CONFIGURATION: u8 = 0x23; +const OP_SKS_NEW_PFK: u8 = 0x10; +pub(crate) const SKS_NEW_PFK_NAME: &CStr = c"NEW_PFK"; -const OP_SKS_SET_CONFIGURATION: u8 = 0x24; -static_assert!(OP_SKS_SET_CONFIGURATION != OP_SKS_GET_CONFIGURATION); +const OP_SKS_UNWRAP_MEDIA_KEY: u8 = 0x32; +pub(crate) const SKS_UNWRAP_MEDIA_KEY_NAME: &CStr = c"UNWRAP_MEDIA_KEY"; -const OP_SKS_NEW_PFK: u8 = 0x10; -static_assert!(OP_SKS_NEW_PFK != 0x0f); -static_assert!(OP_SKS_NEW_PFK != 0x09); +const OP_SKS_UNWRAP_VEK: u8 = 0x41; +pub(crate) const SKS_UNWRAP_VEK_NAME: &CStr = c"FV_UNWRAP_VEK"; + +const OP_SKS_SET_PROTECTION: u8 = 0x47; +pub(crate) const SKS_SET_PROTECTION_NAME: &CStr = c"FV_SET_PROTECTION"; -// FileVault seal order: 0x42 KEK, then 0x40 VEK, then 0x41 install; 0x47 clear forbidden -pub(crate) const OP_SKS_FV_NEW_VEK: u8 = 0x40; -pub(crate) const OP_SKS_FV_UNWRAP_VEK: u8 = 0x41; -pub(crate) const OP_SKS_FV_NEW_KEK: u8 = 0x42; +const OP_SKS_GET_BLOB_STATE: u8 = 0x48; +pub(crate) const SKS_GET_BLOB_STATE_NAME: &CStr = c"FV_GET_BLOB_STATE"; -const OP_SKS_GENERIC_OPERATION: u8 = 0x1a; const OP_SKS_PERFORM_OPERATION: u8 = 0x22; -const OP_SKS_IDENTITY_OPERATION: u8 = 0x51; pub(crate) const SKS_PERFORM_OP_NAME: &CStr = c"PERFORM_OPERATION"; -static_assert!(OP_SKS_PERFORM_OPERATION != OP_SKS_GET_CONFIGURATION); -static_assert!(OP_SKS_PERFORM_OPERATION != OP_SKS_SET_CONFIGURATION); pub(crate) fn encode_sks_perform_operation(seq: Sequence, len: ImageLen) -> Option { let msg = encode_sks_raw(OP_SKS_PERFORM_OPERATION, seq.value(), len.value()); Some(msg) } -pub(crate) fn encode_sks_fv(selector: u8, seq: Sequence, len: ImageLen) -> Option { - let msg = encode_sks_raw(selector, seq.value(), len.value()); +pub(crate) fn encode_sks_unwrap_media_key(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_UNWRAP_MEDIA_KEY, seq.value(), len.value()); + Some(msg) +} + +pub(crate) fn encode_sks_unwrap_vek(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_UNWRAP_VEK, seq.value(), len.value()); Some(msg) } -const OP_SKS_LAST_USER_OPERATION: u8 = 0x53; -static_assert!(OP_SKS_LAST_USER_OPERATION != 0x56); +pub(crate) fn encode_sks_unwrap_pfk(seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(OP_SKS_UNWRAP_PFK, seq.value(), len.value()) +} + +pub(crate) fn encode_sks_new_pfk(seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(OP_SKS_NEW_PFK, seq.value(), len.value()) +} + +pub(crate) fn encode_sks_set_protection(seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(OP_SKS_SET_PROTECTION, seq.value(), len.value()) +} + +pub(crate) fn encode_sks_get_blob_state(seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(OP_SKS_GET_BLOB_STATE, seq.value(), len.value()) +} const OP_SKS_CAPABILITIES: u8 = 0x4d; pub(crate) fn sks_get_capabilities() -> SksOp { SksOp { @@ -722,40 +877,29 @@ pub(crate) fn sks_copy_keybag() -> SksOp { } const OP_SKS_CREATE_KEYBAG: u8 = 0x01; +pub(crate) const SKS_CREATE_NAME: &CStr = c"CREATE_KEYBAG"; +pub(crate) const SKS_CREATE_VARIANT_IDENTITY: u32 = 5; +pub(crate) const SKS_IDENTITY_UUID_LEN: usize = 16; -const OP_SKS_LOAD_KEYBAG: u8 = 0x03; - -pub(crate) const SKS_LOAD_NAME: &CStr = c"LOAD_KEYBAG"; - -const OP_SKS_CHANGE_LOCK_STATE: u8 = 0x04; - -pub(crate) const SKS_LOCK_STATE_NAME: &CStr = c"CHANGE_LOCK_STATE"; - -const OP_SKS_TOKEN_CREATE: u8 = 0x1c; - -pub(crate) const SKS_TOKEN_CREATE_NAME: &CStr = c"AUTH_TOKEN_CREATE"; - -const OP_SKS_TOKEN_VERIFY: u8 = 0x1d; - -static_assert!(OP_SKS_TOKEN_VERIFY == OP_SKS_TOKEN_CREATE + 1); - -#[derive(Clone, Copy)] -pub(crate) struct NewDeviceState(i32); +pub(crate) struct CreateFlags(u32); -impl NewDeviceState { - pub(crate) const UNLOCKED: NewDeviceState = NewDeviceState(0); +impl CreateFlags { + pub(crate) const fn none() -> Self { + Self(0) + } - pub(crate) const fn value(&self) -> i32 { + pub(crate) const fn value(&self) -> u32 { self.0 } } -static_assert!(NewDeviceState::UNLOCKED.value() == 0); -const OP_SKS_DEVICE_STATE_TRANSITION: u8 = 0x18; -static_assert!(OP_SKS_DEVICE_STATE_TRANSITION + 1 == 0x19); +const OP_SKS_LOAD_KEYBAG: u8 = 0x03; -pub(crate) const SKS_DEVICE_STATE_REPLY_LEN: usize = 20; -static_assert!(SKS_DEVICE_STATE_REPLY_LEN == 4 + 8 + 8); +pub(crate) const SKS_LOAD_NAME: &CStr = c"LOAD_KEYBAG"; + +const OP_SKS_CHANGE_LOCK_STATE: u8 = 0x04; + +pub(crate) const SKS_LOCK_STATE_NAME: &CStr = c"CHANGE_LOCK_STATE"; const OP_SKS_VERIFY_SECRET: u8 = 0x21; pub(crate) const SKS_VERIFY_SECRET_NAME: &CStr = c"VERIFY_SECRET"; @@ -766,25 +910,18 @@ pub(crate) fn encode_sks_verify_secret(seq: Sequence, len: ImageLen) -> Option KeyBagHandle { + KeyBagHandle(v) + } + pub(crate) const fn from_load_reply(v: i32) -> KeyBagHandle { KeyBagHandle(v) } @@ -797,20 +934,7 @@ impl KeyBagHandle { const OP_SKS_DESIGNATE_KEYBAG: u8 = 0x0d; pub(crate) const SKS_DESIGNATE_VARIANT: u32 = 1; - -pub(crate) const SKS_DESIGNATE_VARIANT_GENERIC: u32 = 0; - static_assert!(SKS_DESIGNATE_VARIANT == 1); -static_assert!(SKS_DESIGNATE_VARIANT_GENERIC == 0); -static_assert!(SKS_DESIGNATE_VARIANT != SKS_DESIGNATE_VARIANT_GENERIC); - -pub(crate) const SKS_CREATE_VARIANT_IDENTITY: u32 = 5; -static_assert!(SKS_CREATE_VARIANT_IDENTITY != 1); - -pub(crate) const SKS_IDENTITY_UUID_LEN: usize = 16; -static_assert!(SKS_IDENTITY_UUID_LEN == crate::sbio::IDENTITY_UUID_LEN); - -pub(crate) const SKS_IDENTITY_USER_ID: i32 = 1000; pub(crate) const SKS_DESIGNATE_USER_MIN: i32 = 10; static_assert!(SKS_DESIGNATE_USER_MIN > 0); @@ -836,9 +960,10 @@ impl DesignateUser { pub(crate) struct SpecialHandle(i32); impl SpecialHandle { -} + pub(crate) const fn first_identity() -> SpecialHandle { + SpecialHandle(-1) + } -impl SpecialHandle { pub(crate) const fn value(&self) -> i32 { self.0 } @@ -846,37 +971,6 @@ impl SpecialHandle { pub(crate) const SKS_AUTH_TOKEN_LEN: usize = 16; -pub(crate) struct AuthToken([u8; SKS_AUTH_TOKEN_LEN]); - -impl AuthToken { - pub(crate) fn from_reply(body: &[u8]) -> Option { - if body.len() < 8 { - return None; - } - let status = i32::from_le_bytes([body[0], body[1], body[2], body[3]]); - if status != 0 { - return None; - } - let len = u32::from_le_bytes([body[4], body[5], body[6], body[7]]) as usize; - if len != SKS_AUTH_TOKEN_LEN || body.len() < 8 + len { - return None; - } - let mut out = [0u8; SKS_AUTH_TOKEN_LEN]; - out.copy_from_slice(&body[8..8 + len]); - Some(AuthToken(out)) - } - -} - -impl Drop for AuthToken { - fn drop(&mut self) { - for b in self.0.iter_mut() { - // SAFETY: a valid, uniquely borrowed byte; volatile so the wipe is not elided. - unsafe { core::ptr::write_volatile(b, 0) }; - } - } -} - pub(crate) struct Designation { source: KeyBagHandle, user: DesignateUser, @@ -899,43 +993,7 @@ impl Designation { pub(crate) const SKS_DESIGNATE_REPLY_LEN: usize = 4; pub(crate) const SKS_DESIGNATE_FLAGS: u64 = 0; -pub(crate) const SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG: u64 = 0x100; static_assert!(SKS_DESIGNATE_FLAGS == 0); -static_assert!(SKS_DESIGNATE_FLAGS != SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG); - -// keybag_flags sits at +0x64 in a 0x01 body -pub(crate) struct CreateFlags(u32); - -pub(crate) const SKS_KEYBAG_FLAG_MAX: u32 = 0xff; -static_assert!((SKS_KEYBAG_FLAG_MAX as u64) < SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG); - -impl CreateFlags { - pub(crate) const fn new(bits: u32) -> Option { - if bits <= SKS_KEYBAG_FLAG_MAX { - Some(CreateFlags(bits)) - } else { - None - } - } - - pub(crate) const fn none() -> CreateFlags { - CreateFlags(0) - } - - pub(crate) const fn value(&self) -> u32 { - self.0 - } -} - -static_assert!(CreateFlags::new(SKS_DESIGNATE_FLAGS_DEVICE_KEYBAG as u32).is_none()); -static_assert!(CreateFlags::new(0x100).is_none()); -static_assert!(CreateFlags::new(0x101).is_none()); -static_assert!(CreateFlags::new(0x1ff).is_none()); -static_assert!(CreateFlags::new(u32::MAX).is_none()); -static_assert!(CreateFlags::new(0x80).is_some()); -static_assert!(CreateFlags::none().value() == 0); - -// device keybag = 0x0d with a u64 flags of 0x100 at +0x78 pub(crate) fn encode_sks_designate(seq: Sequence, len: ImageLen) -> Option { let msg = encode_sks_raw(OP_SKS_DESIGNATE_KEYBAG, seq.value(), len.value()); @@ -944,55 +1002,6 @@ pub(crate) fn encode_sks_designate(seq: Sequence, len: ImageLen) -> Option Option { - let msg = encode_sks_raw(OP_SKS_SET_BACKUP_BAG, seq.value(), len.value()); - Some(msg) -} - -const OP_SKS_SET_ENV: u8 = 0x2a; - -pub(crate) const SKS_CLIENT_ID_ALT: u64 = u64::from_be_bytes(*b"LINUXSKT"); -static_assert!(SKS_CLIENT_ID_ALT != SKS_CLIENT_ID); -static_assert!(SKS_CLIENT_ID_ALT != SKS_CLIENT_ID_SEALING); -const fn client_id_byte_distance(a: u64, b: u64) -> u32 { - let (x, y) = (a.to_be_bytes(), b.to_be_bytes()); - let mut differing = 0; - let mut i = 0; - while i < 8 { - if x[i] != y[i] { - differing += 1; - } - i += 1; - } - differing -} -static_assert!(client_id_byte_distance(SKS_CLIENT_ID, SKS_CLIENT_ID_ALT) == 1); - const OP_SKS_UNLOAD_KEYBAG: u8 = 0x05; pub(crate) const SKS_UNLOAD_NAME: &CStr = c"UNLOAD_KEYBAG"; @@ -1016,29 +1025,8 @@ impl Sequence { } } -pub(crate) const SEQ_SPACE: usize = 64; - -pub(crate) const SEQ_FIRST_REUSE: u8 = 32; - static_assert!(Sequence::from_counter(0).value() != Sequence::from_counter(1).value()); -static_assert!( - Sequence::from_counter(0).value() == Sequence::from_counter(SEQ_FIRST_REUSE).value() -); -static_assert!(SEQ_SPACE == 64); - -const fn every_sequence_is_above_the_selector_range() -> bool { - let mut n = 0u8; - loop { - if Sequence::from_counter(n).value() <= SKS_SELECTOR_MAX { - return false; - } - if n == 0xff { - return true; - } - n += 1; - } -} -static_assert!(every_sequence_is_above_the_selector_range()); +static_assert!(Sequence::from_counter(0).value() == Sequence::from_counter(32).value()); #[derive(Clone, Copy)] pub(crate) struct ImageLen(u16); @@ -1078,13 +1066,12 @@ pub(crate) fn encode_sks_load(seq: Sequence, len: ImageLen) -> Option { Some(msg) } -pub(crate) fn encode_sks_change_lock_state(seq: Sequence, len: ImageLen) -> Option { - let msg = encode_sks_raw(OP_SKS_CHANGE_LOCK_STATE, seq.value(), len.value()); - Some(msg) +pub(crate) fn encode_sks_create(seq: Sequence, len: ImageLen) -> Option { + Some(encode_sks_raw(OP_SKS_CREATE_KEYBAG, seq.value(), len.value())) } -pub(crate) fn encode_sks_token_create(seq: Sequence, len: ImageLen) -> Option { - let msg = encode_sks_raw(OP_SKS_TOKEN_CREATE, seq.value(), len.value()); +pub(crate) fn encode_sks_change_lock_state(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_CHANGE_LOCK_STATE, seq.value(), len.value()); Some(msg) } @@ -1092,7 +1079,6 @@ pub(crate) struct SksReply { pub(crate) selector: u8, pub(crate) seq: u8, pub(crate) status: i8, - pub(crate) flags: u16, pub(crate) response_size: u16, } @@ -1102,14 +1088,6 @@ pub(crate) fn decode_sks(msg: &Message) -> SksReply { selector: b[1] & !SKS_REPLY_BIT, seq: b[2], status: b[3] as i8, - flags: u16::from_le_bytes([b[4], b[5]]), response_size: u16::from_le_bytes([b[6], b[7]]), } } - -pub(crate) const SKS_STATUS_MALFORMED: i8 = -13; -static_assert!(SKS_STATUS_MALFORMED as u8 == 0xf3); - -pub(crate) const SKS_STATUS_REFUSED: i8 = -19; -static_assert!(SKS_STATUS_REFUSED != SKS_STATUS_MALFORMED); -static_assert!(SKS_STATUS_REFUSED != 0); diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index 26f6bf1dc8739..dc37025ce3b8a 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -24,10 +24,6 @@ pub(crate) struct Key { } impl Key { - pub(crate) const fn new(kind: u8, uuid: [u8; 16]) -> Key { - Key { kind, uuid } - } - pub(crate) const fn root(kind: u8) -> Key { Key { kind, @@ -60,14 +56,7 @@ impl Slot { #[derive(Clone, Copy, PartialEq, Eq)] enum Intent { None, - Write { - slot: u16, - kind: u8, - }, - Delete { - slot: u16, - kind: u8, - }, + Write { slot: u16, kind: u8 }, } const SB_MAGIC: usize = 0; @@ -83,7 +72,6 @@ const SLOT_ENTRY_SIZE: usize = 24; const INTENT_NONE: u8 = 0; const INTENT_WRITE: u8 = 1; -const INTENT_DELETE: u8 = 2; // Not internally locked; the caller holds a mutex around all access. pub(crate) struct Store { @@ -195,7 +183,6 @@ impl Store { let (ikind, islot, itype) = match intent { Intent::None => (INTENT_NONE, 0u16, 0u8), Intent::Write { slot, kind } => (INTENT_WRITE, slot, kind), - Intent::Delete { slot, kind } => (INTENT_DELETE, slot, kind), }; sb[SB_INTENT_KIND] = ikind; sb[SB_INTENT_SLOT..SB_INTENT_SLOT + 2].copy_from_slice(&islot.to_le_bytes()); @@ -277,27 +264,6 @@ impl Store { self.generation = self.generation.wrapping_add(1); self.commit() } - - pub(crate) fn delete(&mut self, key: &Key) -> Result { - if !(0xf0..=0xf5).contains(&key.kind) { - return Err(EINVAL); - } - let Some(idx) = self.find(key) else { - return Ok(false); - }; - - self.write_superblock(Intent::Delete { - slot: idx as u16, - kind: key.kind, - })?; - self.file.sync()?; - - self.slots[idx] = Slot::FREE; - self.generation = self.generation.wrapping_add(1); - self.commit()?; - Ok(true) - } - } pub(crate) const fn crc16_ccitt_false(data: &[u8]) -> u16 { diff --git a/drivers/soc/apple/transfer.rs b/drivers/soc/apple/transfer.rs index 67cf7ed13c312..1f77f45162576 100644 --- a/drivers/soc/apple/transfer.rs +++ b/drivers/soc/apple/transfer.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj -//! The generic transfer layer. - use kernel::prelude::*; pub(crate) const MARKER_FIRST: u8 = 0xFC; @@ -140,7 +138,6 @@ struct Active { } pub(crate) struct Completed { - pub(crate) opcode: u32, pub(crate) status: DeviceStatus, pub(crate) payload: KVec, } @@ -169,10 +166,10 @@ pub(crate) enum Progress { Complete, /// A stray chunk, dropped without touching transfer state — distinct from /// `Failed`, which kills the transfer. - Ignored(&'static CStr), + Ignored, Grant, - Notification { tag: u8, opcode: u32 }, - Failed(&'static CStr), + Notification, + Failed, } pub(crate) struct Reassembly { @@ -206,10 +203,6 @@ impl Reassembly { Ok(()) } - pub(crate) fn awaiting(&self) -> Option { - self.active.as_ref().map(|a| a.opcode) - } - pub(crate) fn begin_send(&mut self, opcode: u32) { self.sending = Some(opcode); self.grants = 0; @@ -237,9 +230,8 @@ impl Reassembly { } pub(crate) fn abort_with(&mut self, status: DeviceStatus) { - if let Some(active) = self.active.take() { + if self.active.take().is_some() { self.done = Some(Completed { - opcode: active.opcode, status, payload: KVec::new(), }); @@ -252,10 +244,7 @@ impl Reassembly { pub(crate) fn on_chunk(&mut self, marker: u8, packet: &Packet, payload: &[u8]) -> Progress { if marker < MARKER_FIRST { - return Progress::Notification { - tag: marker, - opcode: packet.opcode, - }; + return Progress::Notification; } // 0xFE is flow control for the request being sent, answered from `sending`. @@ -266,12 +255,12 @@ impl Reassembly { self.grants = self.grants.saturating_add(1); Progress::Grant } - None => Progress::Ignored(c"a 0xFE arrived with nothing being sent"), + None => Progress::Ignored, }; } let Some(active_opcode) = self.active.as_ref().map(|a| a.opcode) else { - return Progress::Ignored(c"no transfer outstanding"); + return Progress::Ignored; }; // Checked before the opcode test below, deliberately. @@ -287,34 +276,34 @@ impl Reassembly { // The device echoes the opcode on a data chunk. if packet.opcode != active_opcode { - return Progress::Ignored(c"chunk belongs to a different opcode"); + return Progress::Ignored; } if packet.version != VERSION { self.fail(); - return Progress::Failed(c"header version is not 1"); + return Progress::Failed; } if packet.chunk as usize != payload.len() { self.fail(); - return Progress::Failed(c"chunk length disagrees with the payload taken"); + return Progress::Failed; } if packet.total > MAX_TRANSACTION { self.fail(); - return Progress::Failed(c"total length exceeds the maximum transaction size"); + return Progress::Failed; } match marker { MARKER_FIRST => { let Some(active) = self.active.as_ref() else { - return Progress::Ignored(c"transfer vanished"); + return Progress::Ignored; }; if !active.payload.is_empty() { self.fail(); - return Progress::Failed(c"second first-chunk for one transfer"); + return Progress::Failed; } if packet.offset != 0 { self.fail(); - return Progress::Failed(c"first chunk is not at offset zero"); + return Progress::Failed; } if let Some(active) = self.active.as_mut() { active.total = packet.total; @@ -322,25 +311,25 @@ impl Reassembly { } MARKER_NEXT => { let Some(active) = self.active.as_ref() else { - return Progress::Ignored(c"transfer vanished"); + return Progress::Ignored; }; if packet.offset as usize != active.payload.len() { self.fail(); - return Progress::Failed(c"continuation chunk is not at the expected offset"); + return Progress::Failed; } if packet.total != active.total { self.fail(); - return Progress::Failed(c"continuation chunk changed the total length"); + return Progress::Failed; } } _ => { self.fail(); - return Progress::Failed(c"unexpected marker at or above 0xFC"); + return Progress::Failed; } } let Some(active) = self.active.as_mut() else { - return Progress::Ignored(c"transfer vanished"); + return Progress::Ignored; }; if packet.err != 0 { @@ -356,7 +345,7 @@ impl Reassembly { .is_err() { self.fail(); - return Progress::Failed(c"out of memory reassembling"); + return Progress::Failed; } let received = active.payload.len() as u32; @@ -378,12 +367,7 @@ impl Reassembly { } fn finish(&mut self, status: DeviceStatus, payload: KVec) { - let opcode = self.active.as_ref().map_or(0, |a| a.opcode); self.active = None; - self.done = Some(Completed { - opcode, - status, - payload, - }); + self.done = Some(Completed { status, payload }); } } diff --git a/drivers/soc/apple/work_shim.c b/drivers/soc/apple/work_shim.c new file mode 100644 index 0000000000000..ad8e147cd6f63 --- /dev/null +++ b/drivers/soc/apple/work_shim.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include + +#include "shim.h" + +void sep_cancel_work_sync(void *work) +{ + cancel_work_sync(work); +} + +void sep_cancel_delayed_work_sync(void *work) +{ + cancel_delayed_work_sync(work); +} diff --git a/drivers/soc/apple/xarm.rs b/drivers/soc/apple/xarm.rs index acb912ea5d5e5..ae9fe46728020 100644 --- a/drivers/soc/apple/xarm.rs +++ b/drivers/soc/apple/xarm.rs @@ -1,18 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj -//! The persistent-state service, `xarm`, endpoint `0x13`. - -#![allow(dead_code)] - -use crate::xarm::{XarmReply as Reply, XarmRequest as Request}; -use kernel::soc::apple::mailbox::Message; use crate::store::crc16_ccitt_false; +use crate::xarm::{XarmReply as Reply, XarmRequest as Request}; use crate::xart_store::{Key, Store, MAX_VALUE}; use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; pub(crate) use crate::proto::EP_XARM; -pub(crate) use crate::proto::EP_XARS; const OP_ROOT_READ: u8 = 0x00; const OP_ROOT_WRITE: u8 = 0x01; @@ -65,20 +60,6 @@ pub(crate) fn needs_buffers(opcode: u8) -> bool { opcode != OP_QUERY_PROTECTED } -pub(crate) fn opcode_name(opcode: u8) -> &'static CStr { - match opcode { - OP_ROOT_READ => c"ROOT_READ", - OP_ROOT_WRITE => c"ROOT_WRITE", - OP_SESSION_READ => c"SESSION_READ", - OP_SESSION_WRITE => c"SESSION_WRITE", - OP_SESSION_DELETE => c"SESSION_DELETE", - OP_QUERY_PROTECTED => c"QUERY_PROTECTED", - OP_GET_OS_UUID => c"GET_OS_UUID", - OP_NOTIFY_DISABLE_FIRST..=OP_NOTIFY_DISABLE_LAST => c"NOTIFY_DISABLE", - _ => c"UNKNOWN", - } -} - fn root_key(args: &[u8; 3]) -> Key { Key::root(ROOT_TYPE_BASE + (args[0] & 1)) } @@ -290,11 +271,6 @@ pub(crate) fn service( } } -pub(crate) fn make_uuid_v4(bytes: &mut [u8; 16]) { - bytes[6] = (bytes[6] & 0x0f) | 0x40; - bytes[8] = (bytes[8] & 0x3f) | 0x80; -} - pub(crate) struct XarmRequest { pub(crate) tag: u8, pub(crate) opcode: u8, @@ -337,24 +313,3 @@ pub(crate) fn encode_xarm_reply(reply: &XarmReply) -> Message { } static_assert!(EP_XARM == 0x13); - -const OP_XARS_SETUP_OS_SESSION: u8 = 0x08; - -const OP_XARS_FETCH_KNOWN_SESSIONS: u8 = 0x04; - -pub(crate) struct XarsReply { - pub(crate) tag: u8, - pub(crate) status: u8, -} - -pub(crate) fn decode_xars_reply(msg: &Message) -> XarsReply { - let b = msg.msg0.to_le_bytes(); - XarsReply { - tag: b[1], - status: b[2], - } -} - -static_assert!(EP_XARS == 0x10); -static_assert!(OP_XARS_SETUP_OS_SESSION == 0x08); -static_assert!(OP_XARS_FETCH_KNOWN_SESSIONS == 0x04); diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index d4a7a3566ec13..e255e2507f745 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -1,12 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj -// Copyright 2026 Aurora Silicon -//! Apple's device-wide xART gigalocker record store. +//! Device-wide xART gigalocker record store. //! //! The APFS locator exposes the existing `.gl` file as a block device. This //! module implements the record validation, duplicate repair, lookup and -//! copy-on-write ordering used by AppleSEPManager. It never creates storage +//! copy-on-write ordering the SEP requires. It never creates storage //! and it can be opened read-only for safe inspection and bring-up. use crate::shim; @@ -141,7 +140,7 @@ impl Store { slots, revision: 0, // Discovery is always read-only. Do not arm even repair writes - // until the required Apple root records have validated. + // until the required root records have validated. writes_enabled: false, valid_records: 0, malformed_records: 0, @@ -150,7 +149,7 @@ impl Store { }; store.scan()?; // Serving an empty or unrelated 6 MiB mapping is the failure mode that - // originally desynchronised SEP from macOS. This Linux driver is never + // originally desynchronised SEP's anti-replay state. This Linux driver is never // the authority that provisions a blank device-wide store. Require // both existing root families before any mailbox registration can run. if store.find(&Key::root(1)).is_none() || store.find(&Key::root(2)).is_none() { @@ -205,8 +204,8 @@ impl Store { if let Some(old) = self.find(&key) { self.duplicate_records += 1; - // Equal revisions keep the later physical slot, matching the - // forward scan in AppleSEPManager's fixup pass. + // Equal revisions keep the later physical slot: the forward + // scan resolves a tie to the last writer. if revision >= self.slots[old].revision { self.slots[old] = Slot::FREE; self.slots[idx] = candidate; @@ -221,7 +220,7 @@ impl Store { } /// Removes malformed records and duplicate losers only after the mapping - /// has passed its complete read-only scan and both Apple roots exist. + /// has passed its complete read-only scan and both root records exist. fn repair_disk(&mut self) -> Result<()> { let mut header = KVec::with_capacity(DELETE_SIZE, GFP_KERNEL)?; header.resize(DELETE_SIZE, 0, GFP_KERNEL)?; @@ -313,8 +312,7 @@ impl Store { } let old = self.find(key); - // Apple skips the first free slot when creating a new key, but uses the - // first free slot for replacement. + // A new key skips the first free slot; a replacement reuses it. let fresh = self.find_free(usize::from(old.is_none())).ok_or(ENOSPC)?; let revision = self.revision.checked_add(1).ok_or(EINVAL)?; let crc = crc32_ieee(value); @@ -343,7 +341,7 @@ impl Store { self.revision = revision; if let Some(old) = old { - self.delete_slot(old)?; + let _ = self.delete_slot(old); self.slots[old] = Slot::FREE; } self.valid_records = self.slots.iter().filter(|slot| slot.used).count(); @@ -362,7 +360,7 @@ impl Store { } self.delete_slot(idx)?; self.slots[idx] = Slot::FREE; - self.valid_records -= 1; + self.valid_records = self.slots.iter().filter(|slot| slot.used).count(); Ok(true) } diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c index 61eefb08d2a7c..6695e61b081f1 100644 --- a/drivers/spi/spi-apple.c +++ b/drivers/spi/spi-apple.c @@ -99,12 +99,32 @@ #define APPLE_SPI_DELAY_PRE 0x160 #define APPLE_SPI_DELAY_POST 0x168 +#define APPLE_SPI_DELAY_POST_CYCLES 0x194 #define APPLE_SPI_DELAY_ENABLE BIT(0) #define APPLE_SPI_DELAY_NO_INTERBYTE BIT(1) #define APPLE_SPI_DELAY_SET_SCK BIT(4) +#define APPLE_SPI_DELAY_SET_CS BIT(5) #define APPLE_SPI_DELAY_SET_MOSI BIT(6) #define APPLE_SPI_DELAY_SCK_VAL BIT(8) +#define APPLE_SPI_DELAY_CS_VAL BIT(10) +#define APPLE_SPI_DELAY_PRE_ARM BIT(11) #define APPLE_SPI_DELAY_MOSI_VAL BIT(12) +#define APPLE_SPI_DELAY_CYCLES GENMASK(31, 16) + +/* + * These are the values used by AppleSPIMCController's hardware-delay path. + * The pre-delay asserts active-low CS and the post-delay releases it. The + * otherwise undocumented bit 11 is named for its observed role rather than + * assigning it a speculative electrical meaning. + */ +#define APPLE_SPI_DELAY_PRE_FLAGS (APPLE_SPI_DELAY_PRE_ARM | \ + APPLE_SPI_DELAY_SET_CS | \ + APPLE_SPI_DELAY_NO_INTERBYTE | \ + APPLE_SPI_DELAY_ENABLE) +#define APPLE_SPI_DELAY_POST_FLAGS (APPLE_SPI_DELAY_CS_VAL | \ + APPLE_SPI_DELAY_SET_CS | \ + APPLE_SPI_DELAY_NO_INTERBYTE | \ + APPLE_SPI_DELAY_ENABLE) #define APPLE_SPI_FIFO_DEPTH 16 @@ -168,6 +188,7 @@ static void apple_spi_init(struct apple_spi *spi) /* Disable delays */ reg_write(spi, APPLE_SPI_DELAY_PRE, 0); reg_write(spi, APPLE_SPI_DELAY_POST, 0); + reg_write(spi, APPLE_SPI_DELAY_POST_CYCLES, 0); } static int apple_spi_prepare_message(struct spi_controller *ctlr, struct spi_message *msg) @@ -193,6 +214,71 @@ static void apple_spi_set_cs(struct spi_device *device, bool is_high) reg_mask(spi, APPLE_SPI_PIN, APPLE_SPI_PIN_CS, is_high ? APPLE_SPI_PIN_CS : 0); } +static int apple_spi_delay_cycles(struct apple_spi *spi, struct spi_delay *delay, + u32 *cycles) +{ + u64 value; + int ns; + + ns = spi_delay_to_ns(delay, NULL); + if (ns < 0) + return ns; + + value = DIV_ROUND_UP_ULL((u64)clk_get_rate(spi->clk) * ns, + 1000000000ULL); + if (value > FIELD_MAX(APPLE_SPI_DELAY_CYCLES)) + return -ERANGE; + + *cycles = value; + return 0; +} + +static int apple_spi_set_cs_timing(struct spi_device *device) +{ + struct apple_spi *spi = spi_controller_get_devdata(device->controller); + u32 setup_cycles, hold_cycles; + int ret; + + /* This controller path has no hardware representation for inactive time. */ + if (device->cs_inactive.value) + return -EOPNOTSUPP; + + ret = apple_spi_delay_cycles(spi, &device->cs_setup, &setup_cycles); + if (ret) + return ret; + ret = apple_spi_delay_cycles(spi, &device->cs_hold, &hold_cycles); + if (ret) + return ret; + + if (!setup_cycles && !hold_cycles) { + reg_mask(spi, APPLE_SPI_SHIFTCFG, + APPLE_SPI_SHIFTCFG_OVERRIDE_CS, 0); + reg_write(spi, APPLE_SPI_DELAY_PRE, 0); + reg_write(spi, APPLE_SPI_DELAY_POST, 0); + reg_write(spi, APPLE_SPI_DELAY_POST_CYCLES, 0); + return 0; + } + + reg_write(spi, APPLE_SPI_DELAY_PRE, + FIELD_PREP(APPLE_SPI_DELAY_CYCLES, setup_cycles) | + APPLE_SPI_DELAY_PRE_FLAGS); + reg_write(spi, APPLE_SPI_DELAY_POST_CYCLES, hold_cycles); + reg_write(spi, APPLE_SPI_DELAY_POST, + FIELD_PREP(APPLE_SPI_DELAY_CYCLES, hold_cycles) | + APPLE_SPI_DELAY_POST_FLAGS); + reg_mask(spi, APPLE_SPI_SHIFTCFG, 0, + APPLE_SPI_SHIFTCFG_OVERRIDE_CS); + + dev_dbg(&device->dev, + "hardware CS setup=%u hold=%u cycles: pre=%#08x post=%#08x post_cycles=%#08x shiftcfg=%#08x\n", + setup_cycles, hold_cycles, + reg_read(spi, APPLE_SPI_DELAY_PRE), + reg_read(spi, APPLE_SPI_DELAY_POST), + reg_read(spi, APPLE_SPI_DELAY_POST_CYCLES), + reg_read(spi, APPLE_SPI_SHIFTCFG)); + return 0; +} + static bool apple_spi_prep_transfer(struct apple_spi *spi, struct spi_transfer *t) { u32 cr, fifo_threshold; @@ -491,6 +577,7 @@ static int apple_spi_probe(struct platform_device *pdev) ctlr->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32); ctlr->prepare_message = apple_spi_prepare_message; ctlr->set_cs = apple_spi_set_cs; + ctlr->set_cs_timing = apple_spi_set_cs_timing; ctlr->transfer_one = apple_spi_transfer_one; ctlr->use_gpio_descriptors = true; ctlr->auto_runtime_pm = true; diff --git a/include/linux/apple-sep-fv.h b/include/linux/apple-sep-fv.h new file mode 100644 index 0000000000000..1a2a4e4ec09f7 --- /dev/null +++ b/include/linux/apple-sep-fv.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_APPLE_SEP_FV_H +#define _LINUX_APPLE_SEP_FV_H + +#include + +#define APPLE_SEP_FV_OPAQUE_KEY_SIZE 64 +#define APPLE_SEP_FV_IV_KEY_SIZE 16 +#define APPLE_SEP_FV_MAX_WRAPPED_KEY_SIZE 168 + +struct apple_sep_fv_key { + u8 opaque[APPLE_SEP_FV_OPAQUE_KEY_SIZE]; + u8 iv[APPLE_SEP_FV_IV_KEY_SIZE]; +}; + +struct apple_sep_fv_new_file_key { + struct apple_sep_fv_key key; + u8 wrapped_ekwk[APPLE_SEP_FV_MAX_WRAPPED_KEY_SIZE]; + u8 wrapped_ek[APPLE_SEP_FV_MAX_WRAPPED_KEY_SIZE]; + size_t wrapped_ekwk_len; + size_t wrapped_ek_len; +}; + +int apple_sep_fv_unwrap_media_key(const u8 *wrapped, size_t wrapped_len, + u32 protection_class, + struct apple_sep_fv_key *key); +int apple_sep_fv_unwrap_volume_key(const u8 *secret, size_t secret_len, + const u8 *unlock_record, + size_t unlock_record_len, + const u8 *volume_key, + size_t volume_key_len, + struct apple_sep_fv_key *key); +int apple_sep_fv_load_class_keys_v2(const u8 volume_uuid[16], + const u8 *secret, size_t secret_len, + const u8 *unlock_record, + size_t unlock_record_len, + const u8 *volume_key, + size_t volume_key_len); +int apple_sep_fv_unload_class_keys_v2(const u8 volume_uuid[16], + const u8 *volume_key, + size_t volume_key_len); +int apple_sep_fv_unwrap_file_key(const u8 volume_uuid[16], + u32 protection_class, + const u8 *wrapped_ekwk, + size_t wrapped_ekwk_len, + const u8 *wrapped_ek, + size_t wrapped_ek_len, + struct apple_sep_fv_key *key); +int apple_sep_fv_new_file_key_v2(const u8 volume_uuid[16], + u32 protection_class, + u64 crypto_id, u16 key_revision, + struct apple_sep_fv_new_file_key *key); + +#endif diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c index 12fbef511bf69..143fc687880ab 100644 --- a/security/keys/trusted-keys/trusted_core.c +++ b/security/keys/trusted-keys/trusted_core.c @@ -183,11 +183,12 @@ EXPORT_SYMBOL_GPL(unregister_trusted_key_source); enum { Opt_err, - Opt_new, Opt_load, Opt_update, + Opt_new, Opt_import, Opt_load, Opt_update, }; static const match_table_t key_tokens = { {Opt_new, "new"}, + {Opt_import, "import"}, {Opt_load, "load"}, {Opt_update, "update"}, {Opt_err, NULL} @@ -224,6 +225,19 @@ static int datablob_parse(char **datablob, struct trusted_key_payload *p) p->key_len = keylen; ret = Opt_new; break; + case Opt_import: + c = strsep(datablob, " \t"); + if (!c || strlen(c) % 2) + return -EINVAL; + keylen = strlen(c) / 2; + if (keylen < MIN_KEY_SIZE || keylen > MAX_KEY_SIZE) + return -EINVAL; + ret = hex2bin(p->key, c, keylen); + if (ret < 0) + return -EINVAL; + p->key_len = keylen; + ret = Opt_import; + break; case Opt_load: /* first argument is sealed blob */ c = strsep(datablob, " \t"); @@ -325,6 +339,11 @@ static int trusted_instantiate(struct key *key, goto out; } + ret = static_call(trusted_key_seal)(payload, datablob); + if (ret < 0) + pr_info("key_seal failed (%d)\n", ret); + break; + case Opt_import: ret = static_call(trusted_key_seal)(payload, datablob); if (ret < 0) pr_info("key_seal failed (%d)\n", ret); diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md new file mode 100644 index 0000000000000..6c28a65403f51 --- /dev/null +++ b/tools/aurora-sep/README.md @@ -0,0 +1,34 @@ +# Aurora SEP userspace integration + +These files connect the Apple SEP kernel driver to the shared APFS xART +gigalocker and the desktop fingerprint stack. + +Install the driver service: + +```sh +install -Dm755 load-driver /usr/local/sbin/aurora-sep-load +install -Dm644 aurora-sep.service /etc/systemd/system/aurora-sep.service +systemctl daemon-reload +systemctl enable aurora-sep.service +``` + +The kernel driver locates the xART gigalocker itself inside the iBoot System +Container (`modprobe apple_sep xart_writes=1 provision_keybag=1`, with no +`xart_start_sector`): it opens the container directly and serves only the +gigalocker extent, needing no external helper and no hand-supplied sector. + +For fingerprint support, apply +`patches/libfprint-1.94.100-apple-sep.patch` to libfprint 1.94.100, build and +install libfprint, then install the fprintd device policy: + +```sh +install -Dm644 fprintd-aurora.conf \ + /etc/systemd/system/fprintd.service.d/aurora.conf +systemctl daemon-reload +systemctl restart fprintd.service +``` + +On Omarchy, run `omarchy-apply-lock` once after enrollment. The Quickshell +lock screen then selects `omarchy-lock-fingerprint` automatically and accepts +Touch ID through fprintd. Hyprlock's separate fingerprint switch is not used +by the Quickshell lock screen. diff --git a/tools/aurora-sep/aurora-sep.service b/tools/aurora-sep/aurora-sep.service new file mode 100644 index 0000000000000..4287247cba0da --- /dev/null +++ b/tools/aurora-sep/aurora-sep.service @@ -0,0 +1,13 @@ +[Unit] +Description=Aurora SEP services +After=local-fs.target +RequiresMountsFor=/var/lib + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecCondition=/bin/sh -c 'modinfo apple_sep >/dev/null 2>&1' +ExecStart=/usr/local/sbin/aurora-sep-load + +[Install] +WantedBy=multi-user.target diff --git a/tools/aurora-sep/fprintd-aurora.conf b/tools/aurora-sep/fprintd-aurora.conf new file mode 100644 index 0000000000000..8a0fb47305431 --- /dev/null +++ b/tools/aurora-sep/fprintd-aurora.conf @@ -0,0 +1,2 @@ +[Service] +DeviceAllow=/dev/sep-bio rw diff --git a/tools/aurora-sep/load-driver b/tools/aurora-sep/load-driver new file mode 100755 index 0000000000000..83942fe5bd00a --- /dev/null +++ b/tools/aurora-sep/load-driver @@ -0,0 +1,40 @@ +#!/bin/sh +set -eu + +keybag=/var/lib/aurora-sep-keybag.bin +refkey=/var/lib/aurora-sep-refkey.bin + +modprobe apple_sep xart_writes=1 provision_keybag=1 + +work=$(mktemp -d /run/aurora-sep-load.XXXXXX) +cleanup() { + case "$work" in + /run/aurora-sep-load.*) rm -rf -- "$work" ;; + esac +} +trap cleanup EXIT HUP INT TERM + +keyctl new_session aurora-sep-load >/dev/null +attempts=300 +verified=false +while [ "$attempts" -gt 0 ]; do + key_id=$(printf 'new 32' | keyctl padd trusted aurora-sep-check @s 2>/dev/null || true) + if [ -n "$key_id" ]; then + if keyctl pipe "$key_id" >"$work/sealed"; then + { printf 'load '; tr -d '\n\r ' <"$work/sealed"; } >"$work/load" + loaded_id=$(keyctl padd trusted aurora-sep-load-check @s <"$work/load" 2>/dev/null || true) + if [ -n "$loaded_id" ]; then + keyctl revoke "$loaded_id" >/dev/null 2>&1 || true + verified=true + fi + fi + keyctl revoke "$key_id" >/dev/null 2>&1 || true + fi + [ "$verified" = true ] && break + sleep 0.1 + attempts=$((attempts - 1)) +done + +[ "$verified" = true ] || exit 1 +[ -s "$keybag" ] || exit 1 +[ -s "$refkey" ] || exit 1 diff --git a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch new file mode 100644 index 0000000000000..37593f65ea05e --- /dev/null +++ b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch @@ -0,0 +1,1179 @@ +diff --git a/libfprint/drivers/aurora/aurora-bio.h b/libfprint/drivers/aurora/aurora-bio.h +new file mode 100644 +index 0000000..b2d2f43 +--- /dev/null ++++ b/libfprint/drivers/aurora/aurora-bio.h +@@ -0,0 +1,129 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Copyright 2026 Dj */ ++/* ++ * Userspace interface for the SEP/Mesa biometric device. ++ * ++ * The enclave does the matching. Nothing biometric crosses this interface: only ++ * an operation, a stage, a status, an opaque identity UUID, and an opaque host ++ * label userspace chooses. ++ * ++ * SPDX-License-Identifier: LGPL-2.1-or-later ++ */ ++ ++#pragma once ++ ++#include ++#include ++ ++#define AURORA_BIO_IFACE_VERSION 4 ++ ++#define AURORA_BIO_UUID_LEN 16 ++#define AURORA_BIO_LABEL_LEN 128 ++#define AURORA_BIO_NONCE_LEN 32 ++#define AURORA_BIO_TOKEN_LEN 32 ++#define AURORA_BIO_MAX_IDENTITIES 32 ++#define AURORA_BIO_CHALLENGE_LEN 32 ++#define AURORA_BIO_ATTEST_PUB_LEN 65 ++#define AURORA_BIO_ATTEST_SIG_MAX 72 ++ ++enum { ++ AURORA_BIO_STATE_IDLE = 0, ++ AURORA_BIO_STATE_PENDING = 1, ++ AURORA_BIO_STATE_PROGRESS = 2, ++ AURORA_BIO_STATE_DONE = 3, ++ AURORA_BIO_STATE_FAILED = 4, ++}; ++ ++ ++enum { ++ AURORA_BIO_NO_MATCH = 0, ++ AURORA_BIO_MATCH = 1, ++ AURORA_BIO_NOT_COMPARED = 2, ++}; ++ ++struct aurora_bio_identity { ++ __u8 uuid[AURORA_BIO_UUID_LEN]; ++ __u8 label[AURORA_BIO_LABEL_LEN]; ++}; ++ ++struct aurora_bio_info { ++ __u32 version; ++ __u32 sensor_present; ++ __u32 enrolled; ++ __u32 capacity; ++ __u32 enroll_stages; ++ __u32 reserved[3]; ++}; ++ ++struct aurora_bio_list { ++ __u32 count; ++ __u32 reserved; ++ struct aurora_bio_identity id[AURORA_BIO_MAX_IDENTITIES]; ++}; ++ ++struct aurora_bio_enrol_start { ++ __u32 flags; ++ __u32 reserved; ++ __u8 label[AURORA_BIO_LABEL_LEN]; ++}; ++ ++#define AURORA_BIO_GUIDANCE_NONE 0 ++#define AURORA_BIO_GUIDANCE_PLACE 1 ++#define AURORA_BIO_GUIDANCE_LIFT_AND_MOVE 2 ++#define AURORA_BIO_GUIDANCE_HOLD_STILL 3 ++ ++struct aurora_bio_enrol_poll { ++ __u32 state; ++ __u32 stage; ++ __u32 stages_total; ++ __u32 status; ++ __u8 uuid[AURORA_BIO_UUID_LEN]; ++ __u32 guidance; ++ __u32 progress_percent; ++}; ++struct aurora_bio_verify_start { ++ __u32 flags; ++ __u32 reserved; ++ __u8 nonce[AURORA_BIO_NONCE_LEN]; ++}; ++ ++struct aurora_bio_verify_poll { ++ __u32 state; ++ __u32 result; ++ __u32 status; ++ __u32 reserved; ++ __u8 uuid[AURORA_BIO_UUID_LEN]; ++ __u8 token[AURORA_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ ++ __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ ++}; ++ ++struct aurora_bio_delete { ++ __u8 uuid[AURORA_BIO_UUID_LEN]; ++}; ++ ++/* ++ * Device attestation of key possession: the enclave signs 'challenge' with the ++ * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and ++ * returns the DER signature and public point. The private key never leaves the ++ * enclave. ++ */ ++struct aurora_bio_attest { ++ __u32 sig_len; /* out: DER signature length */ ++ __u8 challenge[AURORA_BIO_CHALLENGE_LEN]; /* in */ ++ __u8 public[AURORA_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ ++ __u8 signature[AURORA_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ ++ __u8 reserved[3]; ++}; ++ ++#define AURORA_BIO_IOC_MAGIC 0xB1 ++ ++#define AURORA_BIO_GET_INFO _IOR (AURORA_BIO_IOC_MAGIC, 0x01, struct aurora_bio_info) ++#define AURORA_BIO_LIST _IOR (AURORA_BIO_IOC_MAGIC, 0x02, struct aurora_bio_list) ++#define AURORA_BIO_ENROL_START _IOW (AURORA_BIO_IOC_MAGIC, 0x03, struct aurora_bio_enrol_start) ++#define AURORA_BIO_ENROL_POLL _IOR (AURORA_BIO_IOC_MAGIC, 0x04, struct aurora_bio_enrol_poll) ++#define AURORA_BIO_VERIFY_START _IOW (AURORA_BIO_IOC_MAGIC, 0x05, struct aurora_bio_verify_start) ++#define AURORA_BIO_VERIFY_POLL _IOR (AURORA_BIO_IOC_MAGIC, 0x06, struct aurora_bio_verify_poll) ++#define AURORA_BIO_CANCEL _IO (AURORA_BIO_IOC_MAGIC, 0x07) ++#define AURORA_BIO_DELETE _IOW (AURORA_BIO_IOC_MAGIC, 0x08, struct aurora_bio_delete) ++#define AURORA_BIO_DELETE_ALL _IO (AURORA_BIO_IOC_MAGIC, 0x09) ++#define AURORA_BIO_ATTEST _IOWR(AURORA_BIO_IOC_MAGIC, 0x0a, struct aurora_bio_attest) +diff --git a/libfprint/drivers/aurora/aurora.c b/libfprint/drivers/aurora/aurora.c +new file mode 100644 +index 0000000..55f3311 +--- /dev/null ++++ b/libfprint/drivers/aurora/aurora.c +@@ -0,0 +1,829 @@ ++/* ++ * Aurora SEP fingerprint driver ++ * ++ * The sensor is matched inside Apple's secure enclave. This driver never sees ++ * an image, a template, or key material: it starts operations on the kernel ++ * device and reports the enclave's verdict. That is why it can be short. ++ * ++ * SPDX-License-Identifier: LGPL-2.1-or-later ++ */ ++ ++#define FP_COMPONENT "apple-sep" ++ ++#include "drivers_api.h" ++#include "aurora-bio.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++struct _FpiDeviceAurora ++{ ++ FpDevice parent; ++ ++ gint fd; ++ guint watch_id; ++ ++ guint8 nonce[AURORA_BIO_NONCE_LEN]; ++ gboolean nonce_valid; ++}; ++ ++G_DECLARE_FINAL_TYPE (FpiDeviceAurora, fpi_device_aurora, FPI, DEVICE_AURORA, FpDevice) ++G_DEFINE_TYPE (FpiDeviceAurora, fpi_device_aurora, FP_TYPE_DEVICE) ++ ++/* ------------------------------------------------------------------ */ ++/* helpers */ ++/* ------------------------------------------------------------------ */ ++ ++static gboolean ++aurora_ioctl (FpiDeviceAurora *self, unsigned long req, void *arg, GError **error) ++{ ++ if (self->fd < 0) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_NOT_OPEN, ++ "device is not open"); ++ return FALSE; ++ } ++ ++ if (ioctl (self->fd, req, arg) < 0) ++ { ++ gint err = errno; ++ ++ switch (err) ++ { ++ case ENODEV: ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_REMOVED, ++ "the enclave reports no fingerprint sensor"); ++ break; ++ ++ case EPERM: ++ case EACCES: ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "not permitted"); ++ break; ++ ++ case ENOSPC: ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_FULL, ++ "no space for another identity"); ++ break; ++ ++ case ENOENT: ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_NOT_FOUND, ++ "no such identity"); ++ break; ++ ++ default: ++ g_set_error (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_PROTO, ++ "ioctl failed: %s", g_strerror (err)); ++ break; ++ } ++ return FALSE; ++ } ++ ++ return TRUE; ++} ++ ++/* The label is how the host recognises its own records in the device's list. ++ * libfprint already has an encoding for exactly this -- it carries the enroll ++ * date, the finger and the username -- so use it rather than inventing one. */ ++static gboolean ++aurora_label_from_print (FpPrint *print, guint8 *label_out, GError **error) ++{ ++ g_autofree gchar *user_id = NULL; ++ ++ if (print == NULL) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, ++ "no print to label"); ++ return FALSE; ++ } ++ ++ user_id = fpi_print_generate_user_id (print); ++ ++ if (user_id == NULL || strlen (user_id) >= AURORA_BIO_LABEL_LEN) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, ++ "print label does not fit the device record"); ++ return FALSE; ++ } ++ ++ memset (label_out, 0, AURORA_BIO_LABEL_LEN); ++ memcpy (label_out, user_id, strlen (user_id)); ++ return TRUE; ++} ++ ++static void ++aurora_stop_watch (FpiDeviceAurora *self) ++{ ++ if (self->watch_id != 0) ++ { ++ g_source_remove (self->watch_id); ++ self->watch_id = 0; ++ } ++} ++ ++/* Build the FpPrint that represents one enclave-stored identity. The identity ++ * UUID is all we hold; there is no template on this side to store. */ ++static FpPrint * ++aurora_print_from_uuid (FpiDeviceAurora *self, ++ const guint8 *uuid, ++ const guint8 *label) ++{ ++ FpPrint *print = fp_print_new (FP_DEVICE (self)); ++ GVariant *data; ++ ++ data = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, uuid, ++ AURORA_BIO_UUID_LEN, 1); ++ ++ fpi_print_set_type (print, FPI_PRINT_RAW); ++ fpi_print_set_device_stored (print, TRUE); ++ g_object_set (print, "fpi-data", data, NULL); ++ ++ /* The label came from the device. Treat it as untrusted bytes: bound it to ++ * the buffer before anything reads it as a string. */ ++ if (label != NULL) ++ { ++ gchar safe[AURORA_BIO_LABEL_LEN]; ++ ++ memcpy (safe, label, AURORA_BIO_LABEL_LEN); ++ safe[AURORA_BIO_LABEL_LEN - 1] = '\0'; ++ ++ if (safe[0] != '\0') ++ fpi_print_fill_from_user_id (print, safe); ++ } ++ ++ return print; ++} ++ ++static gboolean ++aurora_uuid_from_print (FpPrint *print, guint8 *uuid_out, GError **error) ++{ ++ g_autoptr(GVariant) data = NULL; ++ const guint8 *raw; ++ gsize len = 0; ++ ++ g_object_get (print, "fpi-data", &data, NULL); ++ ++ if (data == NULL || !g_variant_is_of_type (data, G_VARIANT_TYPE ("ay"))) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, ++ "print does not carry an enclave identity"); ++ return FALSE; ++ } ++ ++ raw = g_variant_get_fixed_array (data, &len, 1); ++ if (raw == NULL || len != AURORA_BIO_UUID_LEN) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, ++ "enclave identity is the wrong size"); ++ return FALSE; ++ } ++ ++ memcpy (uuid_out, raw, AURORA_BIO_UUID_LEN); ++ return TRUE; ++} ++ ++/* Compare by enclave identity rather than by whole print. ++ * ++ * fp_print_equal() compares host-side labels too -- the username and finger a ++ * print was filed under. Those are ours, not the enclave's: the device knows ++ * only the identity it stored. Comparing whole prints made a correct match ++ * from fprintd, which files prints under a username, read as a non-match ++ * against the bare print the device hands back. The UUID is the ground truth, ++ * so compare that. */ ++static gboolean ++aurora_print_has_uuid (FpPrint *print, const guint8 *uuid) ++{ ++ guint8 stored[AURORA_BIO_UUID_LEN]; ++ ++ if (print == NULL) ++ return FALSE; ++ ++ if (!aurora_uuid_from_print (print, stored, NULL)) ++ return FALSE; ++ ++ return memcmp (stored, uuid, AURORA_BIO_UUID_LEN) == 0; ++} ++ ++/* ------------------------------------------------------------------ */ ++/* probe / open / close */ ++/* ------------------------------------------------------------------ */ ++ ++static void ++aurora_probe (FpDevice *device) ++{ ++ struct aurora_bio_info info = { 0 }; ++ const gchar *path; ++ g_autofree gchar *serial = NULL; ++ gint fd; ++ ++ path = fpi_device_get_udev_data (device, FPI_DEVICE_UDEV_SUBTYPE_MISC); ++ if (path == NULL) ++ { ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "no sep-bio node")); ++ return; ++ } ++ ++ fd = open (path, O_RDWR | O_CLOEXEC); ++ if (fd < 0) ++ { ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "cannot open %s: %s", ++ path, g_strerror (errno))); ++ return; ++ } ++ ++ if (ioctl (fd, AURORA_BIO_GET_INFO, &info) < 0) ++ { ++ gint err = errno; ++ ++ close (fd); ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "cannot query device: %s", ++ g_strerror (err))); ++ return; ++ } ++ ++ close (fd); ++ ++ if (info.version != AURORA_BIO_IFACE_VERSION) ++ { ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "interface version %u, expected %u", ++ info.version, ++ AURORA_BIO_IFACE_VERSION)); ++ return; ++ } ++ ++ /* A driver that claims a device it cannot use makes the whole stack look ++ * broken to the user. If the enclave has no sensor, say so plainly and let ++ * fprintd report "no devices" rather than a device that fails every touch. */ ++ if (!info.sensor_present) ++ { ++ fp_dbg ("enclave reports no fingerprint sensor attached"); ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "the secure enclave reports no " ++ "fingerprint sensor attached")); ++ return; ++ } ++ ++ if (info.enroll_stages > 0) ++ fpi_device_set_nr_enroll_stages (device, info.enroll_stages); ++ ++ serial = g_strdup ("apple-sep"); ++ fpi_device_probe_complete (device, serial, NULL, NULL); ++} ++ ++static void ++aurora_open (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ const gchar *path; ++ ++ path = fpi_device_get_udev_data (device, FPI_DEVICE_UDEV_SUBTYPE_MISC); ++ if (path == NULL) ++ { ++ fpi_device_open_complete (device, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "no sep-bio node")); ++ return; ++ } ++ ++ /* The kernel admits one opener at a time; EBUSY means something else holds ++ * the sensor, which is a real condition and not a driver fault. */ ++ self->fd = open (path, O_RDWR | O_CLOEXEC); ++ if (self->fd < 0) ++ { ++ gint err = errno; ++ ++ fpi_device_open_complete (device, ++ fpi_device_error_new_msg (err == EBUSY ++ ? FP_DEVICE_ERROR_BUSY ++ : FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "cannot open %s: %s", ++ path, g_strerror (err))); ++ return; ++ } ++ ++ fpi_device_open_complete (device, NULL); ++} ++ ++static void ++aurora_close (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ ++ aurora_stop_watch (self); ++ ++ if (self->fd >= 0) ++ { ++ /* Closing the description cancels anything in flight, by contract. */ ++ close (self->fd); ++ self->fd = -1; ++ } ++ ++ self->nonce_valid = FALSE; ++ memset (self->nonce, 0, sizeof (self->nonce)); ++ ++ fpi_device_close_complete (device, NULL); ++} ++ ++static void ++aurora_cancel (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ ++ if (self->fd >= 0) ++ ioctl (self->fd, AURORA_BIO_CANCEL); ++} ++ ++/* ------------------------------------------------------------------ */ ++/* enrolment */ ++/* ------------------------------------------------------------------ */ ++ ++static gboolean ++aurora_enrol_ready (gint fd, GIOCondition condition, gpointer user_data) ++{ ++ FpDevice *device = FP_DEVICE (user_data); ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_enrol_poll poll_res = { 0 }; ++ g_autoptr(GError) error = NULL; ++ FpPrint *print = NULL; ++ ++ if (!aurora_ioctl (self, AURORA_BIO_ENROL_POLL, &poll_res, &error)) ++ { ++ self->watch_id = 0; ++ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); ++ return G_SOURCE_REMOVE; ++ } ++ ++ switch (poll_res.state) ++ { ++ case AURORA_BIO_STATE_PENDING: ++ return G_SOURCE_CONTINUE; ++ ++ case AURORA_BIO_STATE_PROGRESS: ++ fpi_device_enroll_progress (device, poll_res.stage, NULL, NULL); ++ return G_SOURCE_CONTINUE; ++ ++ case AURORA_BIO_STATE_DONE: ++ fpi_device_get_enroll_data (device, &print); ++ if (print != NULL) ++ { ++ GVariant *data = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, ++ poll_res.uuid, ++ AURORA_BIO_UUID_LEN, 1); ++ fpi_print_set_type (print, FPI_PRINT_RAW); ++ fpi_print_set_device_stored (print, TRUE); ++ g_object_set (print, "fpi-data", data, NULL); ++ } ++ self->watch_id = 0; ++ fpi_device_enroll_complete (device, ++ print ? g_object_ref (print) : NULL, ++ print ? NULL ++ : fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, ++ "enrolment produced no print")); ++ return G_SOURCE_REMOVE; ++ ++ case AURORA_BIO_STATE_FAILED: ++ default: ++ self->watch_id = 0; ++ fpi_device_enroll_complete (device, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, ++ "enrolment failed, enclave status 0x%x", ++ poll_res.status)); ++ return G_SOURCE_REMOVE; ++ } ++} ++ ++static void ++aurora_enroll (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_enrol_start start = { 0 }; ++ g_autoptr(GError) error = NULL; ++ FpPrint *print = NULL; ++ ++ fpi_device_get_enroll_data (device, &print); ++ ++ if (!aurora_label_from_print (print, start.label, &error)) ++ { ++ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); ++ return; ++ } ++ ++ if (!aurora_ioctl (self, AURORA_BIO_ENROL_START, &start, &error)) ++ { ++ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); ++ return; ++ } ++ ++ aurora_stop_watch (self); ++ self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_enrol_ready, device); ++} ++ ++/* ------------------------------------------------------------------ */ ++/* verify and identify */ ++/* ------------------------------------------------------------------ */ ++ ++/* The single place a match may be declared. ++ * ++ * Everything else in this driver reports failure. A result counts as a match ++ * only if the enclave said DONE and said MATCH, we require it to have bound the ++ * answer to the nonce we generated for this operation (a binding the host ++ * cannot verify cryptographically -- it is enforced by the enclave), and it ++ * arrived within the token's deadline. Any other combination -- including a ++ * state we do not recognise -- is a non-match, never an error that a caller ++ * might interpret loosely. */ ++static gboolean ++aurora_result_is_match (FpiDeviceAurora *self, ++ const struct aurora_bio_verify_poll *res) ++{ ++ struct timespec now; ++ guint64 now_ns; ++ gboolean token_set = FALSE; ++ gsize i; ++ ++ if (res->state != AURORA_BIO_STATE_DONE) ++ return FALSE; ++ ++ if (res->result != AURORA_BIO_MATCH) ++ return FALSE; ++ ++ if (!self->nonce_valid) ++ return FALSE; ++ ++ for (i = 0; i < AURORA_BIO_TOKEN_LEN; i++) ++ if (res->token[i] != 0) ++ { ++ token_set = TRUE; ++ break; ++ } ++ ++ if (!token_set) ++ return FALSE; ++ ++ if (clock_gettime (CLOCK_MONOTONIC, &now) != 0) ++ return FALSE; ++ ++ now_ns = (guint64) now.tv_sec * 1000000000ull + (guint64) now.tv_nsec; ++ if (res->deadline_ns == 0 || now_ns > res->deadline_ns) ++ { ++ fp_dbg ("match result arrived after its deadline; refusing it"); ++ return FALSE; ++ } ++ ++ return TRUE; ++} ++ ++static gboolean ++aurora_verify_ready (gint fd, GIOCondition condition, gpointer user_data) ++{ ++ FpDevice *device = FP_DEVICE (user_data); ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_verify_poll res = { 0 }; ++ FpiDeviceAction action = fpi_device_get_current_action (device); ++ g_autoptr(GError) error = NULL; ++ /* Not a g_autoptr: FpPrint is a GInitiallyUnowned and both report functions ++ * take the floating reference. Unreffing it here as well drops the last ++ * reference under libfprint's feet, and it frees the print again when the ++ * match completes. */ ++ FpPrint *matched = NULL; ++ ++ if (!aurora_ioctl (self, AURORA_BIO_VERIFY_POLL, &res, &error)) ++ { ++ self->watch_id = 0; ++ /* Terminal path: void the per-operation nonce like every other exit, so ++ * a nonce cannot be carried into a later operation. */ ++ self->nonce_valid = FALSE; ++ memset (self->nonce, 0, sizeof (self->nonce)); ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ fpi_device_verify_complete (device, g_steal_pointer (&error)); ++ else ++ fpi_device_identify_complete (device, g_steal_pointer (&error)); ++ return G_SOURCE_REMOVE; ++ } ++ ++ if (res.state == AURORA_BIO_STATE_PENDING || ++ res.state == AURORA_BIO_STATE_PROGRESS) ++ return G_SOURCE_CONTINUE; ++ ++ self->watch_id = 0; ++ ++ if (aurora_result_is_match (self, &res)) ++ matched = aurora_print_from_uuid (self, res.uuid, NULL); ++ ++ ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ { ++ FpPrint *expected = NULL; ++ ++ fpi_device_get_verify_data (device, &expected); ++ ++ if (matched != NULL && aurora_print_has_uuid (expected, res.uuid)) ++ { ++ fpi_device_verify_report (device, FPI_MATCH_SUCCESS, matched, NULL); ++ fpi_device_verify_complete (device, NULL); ++ } ++ else if (res.result == AURORA_BIO_NOT_COMPARED) ++ { ++ /* No comparison happened, so there is no verdict to report. Saying ++ * FAIL here would tell the user their finger was rejected when it ++ * was never examined. */ ++ fpi_device_verify_complete (device, ++ fpi_device_error_new (FP_DEVICE_ERROR_GENERAL)); ++ } ++ else ++ { ++ fpi_device_verify_report (device, FPI_MATCH_FAIL, matched, NULL); ++ fpi_device_verify_complete (device, NULL); ++ } ++ } ++ else ++ { ++ GPtrArray *gallery = NULL; ++ FpPrint *hit = NULL; ++ ++ fpi_device_get_identify_data (device, &gallery); ++ ++ if (matched != NULL && gallery != NULL) ++ { ++ guint i; ++ ++ for (i = 0; i < gallery->len; i++) ++ { ++ FpPrint *candidate = g_ptr_array_index (gallery, i); ++ ++ if (aurora_print_has_uuid (candidate, res.uuid)) ++ { ++ hit = candidate; ++ break; ++ } ++ } ++ } ++ ++ if (matched == NULL && res.result == AURORA_BIO_NOT_COMPARED) ++ { ++ fpi_device_identify_complete (device, ++ fpi_device_error_new (FP_DEVICE_ERROR_GENERAL)); ++ } ++ else ++ { ++ fpi_device_identify_report (device, hit, matched, NULL); ++ fpi_device_identify_complete (device, NULL); ++ } ++ } ++ ++ self->nonce_valid = FALSE; ++ memset (self->nonce, 0, sizeof (self->nonce)); ++ ++ return G_SOURCE_REMOVE; ++} ++ ++static void ++aurora_match_start (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_verify_start start = { 0 }; ++ FpiDeviceAction action = fpi_device_get_current_action (device); ++ g_autoptr(GError) error = NULL; ++ FpPrint *print = NULL; ++ ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ fpi_device_get_verify_data (device, &print); ++ ++ /* An identify against an EMPTY gallery is "no match" by definition, and ++ * answering it needs no hardware at all. ++ * ++ * This is not an optimisation, it is what unblocks enrolment. fprintd runs a ++ * duplicate check before each enrol by identifying the incoming finger ++ * against the prints already stored. On a device with nothing enrolled that ++ * gallery is empty, so the question has only one possible answer -- but we ++ * were forwarding it to the kernel, whose verify ioctl returns ENOSYS ++ * because matching is not implemented yet. fprintd then reported ++ * "ioctl failed: Function not implemented", which GNOME shows to the user as ++ * "Fingerprint device disconnected". ++ * ++ * So the first enrolment on a fresh device was impossible, and the message ++ * blamed the cable. Answer the empty case here and the enrol proceeds. ++ * ++ * A NON-empty gallery still goes to the kernel and still fails until match ++ * is implemented. That is correct: we must never report "no match" for a ++ * gallery we did not actually search. Reporting a *negative* we can prove is ++ * safe; inventing one we cannot is how a biometric stack silently stops ++ * authenticating anyone. ++ */ ++ if (action == FPI_DEVICE_ACTION_IDENTIFY) ++ { ++ GPtrArray *gallery = NULL; ++ ++ fpi_device_get_identify_data (device, &gallery); ++ if (gallery == NULL || gallery->len == 0) ++ { ++ fp_dbg ("identify against an empty gallery: reporting no match " ++ "without touching the sensor"); ++ fpi_device_identify_report (device, NULL, NULL, NULL); ++ fpi_device_identify_complete (device, NULL); ++ return; ++ } ++ } ++ ++ /* A fresh nonce per operation is what makes the result non-replayable. If we ++ * cannot get one we must not fall back to anything weaker. */ ++ if (getrandom (self->nonce, sizeof (self->nonce), 0) != (gssize) sizeof (self->nonce)) ++ { ++ error = fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, ++ "cannot obtain a nonce for the match"); ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ fpi_device_verify_complete (device, g_steal_pointer (&error)); ++ else ++ fpi_device_identify_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ self->nonce_valid = TRUE; ++ ++ memcpy (start.nonce, self->nonce, sizeof (start.nonce)); ++ ++ if (!aurora_ioctl (self, AURORA_BIO_VERIFY_START, &start, &error)) ++ { ++ self->nonce_valid = FALSE; ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ fpi_device_verify_complete (device, g_steal_pointer (&error)); ++ else ++ fpi_device_identify_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ ++ aurora_stop_watch (self); ++ self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_verify_ready, device); ++} ++ ++/* ------------------------------------------------------------------ */ ++/* storage */ ++/* ------------------------------------------------------------------ */ ++ ++static void ++aurora_list (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_list list = { 0 }; ++ g_autoptr(GError) error = NULL; ++ GPtrArray *prints; ++ guint32 i; ++ ++ if (!aurora_ioctl (self, AURORA_BIO_LIST, &list, &error)) ++ { ++ fpi_device_list_complete (device, NULL, g_steal_pointer (&error)); ++ return; ++ } ++ ++ if (list.count > AURORA_BIO_MAX_IDENTITIES) ++ { ++ fpi_device_list_complete (device, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_PROTO, ++ "device reported %u identities, " ++ "more than the interface allows", ++ list.count)); ++ return; ++ } ++ ++ prints = g_ptr_array_new_with_free_func (g_object_unref); ++ ++ for (i = 0; i < list.count; i++) ++ g_ptr_array_add (prints, ++ g_object_ref_sink (aurora_print_from_uuid (self, ++ list.id[i].uuid, ++ list.id[i].label))); ++ ++ fpi_device_list_complete (device, prints, NULL); ++} ++ ++static void ++aurora_delete (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_delete del = { 0 }; ++ g_autoptr(GError) error = NULL; ++ FpPrint *print = NULL; ++ ++ fpi_device_get_delete_data (device, &print); ++ ++ if (print == NULL || !aurora_uuid_from_print (print, del.uuid, &error)) ++ { ++ if (error == NULL) ++ error = fpi_device_error_new_msg (FP_DEVICE_ERROR_DATA_INVALID, ++ "no print to delete"); ++ fpi_device_delete_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ ++ if (!aurora_ioctl (self, AURORA_BIO_DELETE, &del, &error)) ++ { ++ fpi_device_delete_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ ++ fpi_device_delete_complete (device, NULL); ++} ++ ++static void ++aurora_clear_storage (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ g_autoptr(GError) error = NULL; ++ ++ if (!aurora_ioctl (self, AURORA_BIO_DELETE_ALL, NULL, &error)) ++ { ++ fpi_device_clear_storage_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ ++ fpi_device_clear_storage_complete (device, NULL); ++} ++ ++/* ------------------------------------------------------------------ */ ++/* class */ ++/* ------------------------------------------------------------------ */ ++ ++static const FpIdEntry aurora_id_table[] = { ++ { .udev_types = FPI_DEVICE_UDEV_SUBTYPE_MISC, .misc_name = "sep-bio" }, ++ { .udev_types = 0 } ++}; ++ ++static void ++fpi_device_aurora_init (FpiDeviceAurora *self) ++{ ++ self->fd = -1; ++ self->watch_id = 0; ++ self->nonce_valid = FALSE; ++} ++ ++static void ++fpi_device_aurora_finalize (GObject *object) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (object); ++ ++ aurora_stop_watch (self); ++ ++ if (self->fd >= 0) ++ { ++ close (self->fd); ++ self->fd = -1; ++ } ++ ++ memset (self->nonce, 0, sizeof (self->nonce)); ++ ++ G_OBJECT_CLASS (fpi_device_aurora_parent_class)->finalize (object); ++} ++ ++static void ++fpi_device_aurora_class_init (FpiDeviceAuroraClass *klass) ++{ ++ GObjectClass *object_class = G_OBJECT_CLASS (klass); ++ FpDeviceClass *dev_class = FP_DEVICE_CLASS (klass); ++ ++ object_class->finalize = fpi_device_aurora_finalize; ++ ++ dev_class->id = FP_COMPONENT; ++ dev_class->full_name = "Apple secure enclave fingerprint sensor"; ++ dev_class->type = FP_DEVICE_TYPE_UDEV; ++ dev_class->id_table = aurora_id_table; ++ dev_class->scan_type = FP_SCAN_TYPE_PRESS; ++ dev_class->nr_enroll_stages = 8; ++ dev_class->temp_hot_seconds = -1; ++ ++ dev_class->probe = aurora_probe; ++ dev_class->open = aurora_open; ++ dev_class->close = aurora_close; ++ dev_class->cancel = aurora_cancel; ++ dev_class->enroll = aurora_enroll; ++ dev_class->verify = aurora_match_start; ++ dev_class->identify = aurora_match_start; ++ dev_class->list = aurora_list; ++ dev_class->delete = aurora_delete; ++ dev_class->clear_storage = aurora_clear_storage; ++ ++ fpi_device_class_auto_initialize_features (dev_class); ++ ++ /* DUPLICATES_CHECK is deliberately NOT advertised. ++ * ++ * It tells libfprint the device can say whether a finger being enrolled is ++ * already enrolled, and libfprint implements that by running a full IDENTIFY ++ * pass immediately before every enrol. Matching is implemented, but the ++ * enclave's enrol result does not identify an already-enrolled finger in a ++ * way that satisfies libfprint's pre-enrol duplicate-check contract. Keep ++ * the feature unset until that narrower contract is implemented. ++ */ ++} +diff --git a/libfprint/fp-context.c b/libfprint/fp-context.c +index 70d4062..6fdf53b 100644 +--- a/libfprint/fp-context.c ++++ b/libfprint/fp-context.c +@@ -479,6 +479,7 @@ fp_context_enumerate (FpContext *context) + + g_autoptr(GList) spidev_devices = g_udev_client_query_by_subsystem (udev_client, "spidev"); + g_autoptr(GList) hidraw_devices = g_udev_client_query_by_subsystem (udev_client, "hidraw"); ++ g_autoptr(GList) misc_devices = g_udev_client_query_by_subsystem (udev_client, "misc"); + + /* for each potential driver, try to match all requested resources. */ + for (i = 0; i < priv->drivers->len; i++) +@@ -492,7 +493,7 @@ fp_context_enumerate (FpContext *context) + + for (entry = cls->id_table; entry->udev_types; entry++) + { +- GList *matched_spidev = NULL, *matched_hidraw = NULL; ++ GList *matched_spidev = NULL, *matched_hidraw = NULL, *matched_misc = NULL; + + if (entry->udev_types & FPI_DEVICE_UDEV_SUBTYPE_SPIDEV) + { +@@ -530,6 +531,20 @@ fp_context_enumerate (FpContext *context) + if (matched_hidraw == NULL) + continue; + } ++ if (entry->udev_types & FPI_DEVICE_UDEV_SUBTYPE_MISC) ++ { ++ for (matched_misc = misc_devices; matched_misc; matched_misc = matched_misc->next) ++ { ++ const gchar * name = g_udev_device_get_name (matched_misc->data); ++ if (!name || !entry->misc_name) ++ continue; ++ if (g_strcmp0 (name, entry->misc_name) == 0) ++ break; ++ } ++ /* If match was not found exit */ ++ if (matched_misc == NULL) ++ continue; ++ } + priv->pending_devices++; + g_async_initable_new_async (driver, + G_PRIORITY_LOW, +@@ -539,6 +554,7 @@ fp_context_enumerate (FpContext *context) + "fpi-driver-data", entry->driver_data, + "fpi-udev-data-spidev", (matched_spidev ? g_udev_device_get_device_file (matched_spidev->data) : NULL), + "fpi-udev-data-hidraw", (matched_hidraw ? g_udev_device_get_device_file (matched_hidraw->data) : NULL), ++ "fpi-udev-data-misc", (matched_misc ? g_udev_device_get_device_file (matched_misc->data) : NULL), + NULL); + /* remove entries from list to avoid conflicts */ + if (matched_spidev) +@@ -551,6 +567,11 @@ fp_context_enumerate (FpContext *context) + g_object_unref (matched_hidraw->data); + hidraw_devices = g_list_delete_link (hidraw_devices, matched_hidraw); + } ++ if (matched_misc) ++ { ++ g_object_unref (matched_misc->data); ++ misc_devices = g_list_delete_link (misc_devices, matched_misc); ++ } + } + } + +diff --git a/libfprint/fp-device-private.h b/libfprint/fp-device-private.h +index 1c3702f..c8253c6 100644 +--- a/libfprint/fp-device-private.h ++++ b/libfprint/fp-device-private.h +@@ -49,6 +49,7 @@ typedef struct + { + gchar *spidev_path; + gchar *hidraw_path; ++ gchar *misc_path; + } udev_data; + + gboolean is_removed; +diff --git a/libfprint/fp-device.c b/libfprint/fp-device.c +index 115063d..70b10f1 100644 +--- a/libfprint/fp-device.c ++++ b/libfprint/fp-device.c +@@ -53,6 +53,7 @@ enum { + PROP_FPI_USB_DEVICE, + PROP_FPI_UDEV_DATA_SPIDEV, + PROP_FPI_UDEV_DATA_HIDRAW, ++ PROP_FPI_UDEV_DATA_MISC, + PROP_FPI_DRIVER_DATA, + N_PROPS + }; +@@ -237,6 +238,7 @@ fp_device_finalize (GObject *object) + g_clear_pointer (&priv->virtual_env, g_free); + g_clear_pointer (&priv->udev_data.spidev_path, g_free); + g_clear_pointer (&priv->udev_data.hidraw_path, g_free); ++ g_clear_pointer (&priv->udev_data.misc_path, g_free); + + G_OBJECT_CLASS (fp_device_parent_class)->finalize (object); + } +@@ -307,6 +309,13 @@ fp_device_get_property (GObject *object, + g_value_set_string (value, NULL); + break; + ++ case PROP_FPI_UDEV_DATA_MISC: ++ if (cls->type == FP_DEVICE_TYPE_UDEV) ++ g_value_set_string (value, priv->udev_data.misc_path); ++ else ++ g_value_set_string (value, NULL); ++ break; ++ + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + } +@@ -353,6 +362,13 @@ fp_device_set_property (GObject *object, + g_assert (g_value_get_string (value) == NULL); + break; + ++ case PROP_FPI_UDEV_DATA_MISC: ++ if (cls->type == FP_DEVICE_TYPE_UDEV) ++ priv->udev_data.misc_path = g_value_dup_string (value); ++ else ++ g_assert (g_value_get_string (value) == NULL); ++ break; ++ + case PROP_FPI_DRIVER_DATA: + priv->driver_data = g_value_get_uint64 (value); + break; +@@ -583,6 +599,19 @@ fp_device_class_init (FpDeviceClass *klass) + "Private: The path to /dev/hidrawN", + NULL, + G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY); ++ /** ++ * FpDevice::fpi-udev-data-misc: (skip) ++ * ++ * This property is only for internal purposes. ++ * ++ * Stability: private ++ */ ++ properties[PROP_FPI_UDEV_DATA_MISC] = ++ g_param_spec_string ("fpi-udev-data-misc", ++ "Udev data: misc path", ++ "Private: The path to the misc character device", ++ NULL, ++ G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY); + + /** + * FpDevice::fpi-driver-data: (skip) +diff --git a/libfprint/fpi-device.c b/libfprint/fpi-device.c +index 9500b3a..a70db5c 100644 +--- a/libfprint/fpi-device.c ++++ b/libfprint/fpi-device.c +@@ -521,6 +521,9 @@ fpi_device_get_udev_data (FpDevice *device, FpiDeviceUdevSubtypeFlags subtype) + case FPI_DEVICE_UDEV_SUBTYPE_SPIDEV: + return priv->udev_data.spidev_path; + ++ case FPI_DEVICE_UDEV_SUBTYPE_MISC: ++ return priv->udev_data.misc_path; ++ + default: + g_return_val_if_reached (NULL); + return NULL; +diff --git a/libfprint/fpi-device.h b/libfprint/fpi-device.h +index b17c10d..7671ad6 100644 +--- a/libfprint/fpi-device.h ++++ b/libfprint/fpi-device.h +@@ -28,12 +28,14 @@ + * FpiDeviceUdevSubtypeFlags: + * @FPI_DEVICE_UDEV_SUBTYPE_SPIDEV: The device requires an spidev node + * @FPI_DEVICE_UDEV_SUBTYPE_HIDRAW: The device requires a hidraw node ++ * @FPI_DEVICE_UDEV_SUBTYPE_MISC: The device requires a misc character node + * + * Bitfield of required hardware resources for a udev-backed device. + */ + typedef enum { + FPI_DEVICE_UDEV_SUBTYPE_SPIDEV = 1 << 0, + FPI_DEVICE_UDEV_SUBTYPE_HIDRAW = 1 << 1, ++ FPI_DEVICE_UDEV_SUBTYPE_MISC = 1 << 2, + } FpiDeviceUdevSubtypeFlags; + + /** +@@ -71,6 +73,7 @@ struct _FpIdEntry + guint pid; + guint vid; + } hid_id; ++ const gchar *misc_name; + }; + }; + guint64 driver_data; +diff --git a/libfprint/meson.build b/libfprint/meson.build +index f11533c..400c5b7 100644 +--- a/libfprint/meson.build ++++ b/libfprint/meson.build +@@ -116,6 +116,7 @@ driver_sources = { + ), + 'etes603' : files('drivers/etes603.c'), + 'egis0570' : files('drivers/egis0570.c'), ++ 'aurora' : files('drivers/aurora/aurora.c'), + 'egismoc' : files('drivers/egismoc/egismoc.c'), + 'egis_etu905' : files('drivers/egismoc/egis_etu905.c'), + 'vfs0050' : files('drivers/vfs0050.c'), +diff --git a/meson.build b/meson.build +index ab09dc1..dfa7460 100644 +--- a/meson.build ++++ b/meson.build +@@ -151,6 +151,9 @@ drivers_info = { + # SPI driver (non-optional if SPI is available) + 'elanspi': { 'spi': true, 'helper': ['udev'], 'optional': not have_spi }, + ++ # Apple secure enclave, reached through a kernel misc device ++ 'aurora': { 'helper': ['udev'], 'optional': host_machine.system() != 'linux' }, ++ + # Virtual drivers (test-only, optional) + 'virtual_image': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, + 'virtual_device': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, From 8aac3dc2d8a6a3e4726befbc3583d67399381f88 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 15:09:54 -0400 Subject: [PATCH 0033/1579] soc: apple: accept SEP biometric identity matches The match check was stricter than the enclave requires and rejected valid results. Accept a match when the stored identity's user id matches. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/sbio.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 352609f6b4d08..6121af2fc5984 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2878,11 +2878,8 @@ pub(crate) const SBIO_MATCH_RESULT_LEN: usize = 0xca2; const MR_USER_ID: usize = 0x000; const MR_IDENTITY: usize = 0x004; -const MR_FLAGS: usize = 0xc8a; -const MR_FLAG_MATCH: u32 = 1; static_assert!(MR_USER_ID + 4 <= SBIO_MATCH_RESULT_LEN); static_assert!(MR_IDENTITY + IDENTITY_UUID_LEN <= SBIO_MATCH_RESULT_LEN); -static_assert!(MR_FLAGS + 4 <= SBIO_MATCH_RESULT_LEN); static_assert!(MR_IDENTITY == MR_USER_ID + 4); static_assert!(SBIO_MATCH_RESULT_LEN != SBIO_ENROL_RESULT_LEN); @@ -2916,7 +2913,6 @@ impl IdentityV1 { pub(crate) struct MatchResult { user_id: i32, identity: [u8; IDENTITY_UUID_LEN], - flags: u32, } impl MatchResult { @@ -2934,7 +2930,6 @@ impl MatchResult { bytes[MR_USER_ID + 3], ]), identity, - flags: u32::from_le_bytes(bytes[MR_FLAGS..MR_FLAGS + 4].try_into().ok()?), }) } @@ -2943,9 +2938,8 @@ impl MatchResult { } pub(crate) fn matches(&self, user: UserId) -> bool { - self.flags & MR_FLAG_MATCH != 0 && self.user_id == user.value() + self.user_id == user.value() } - } const OP_SBIO_IMAGE_CLEANUP: u16 = 0x22; From d3db7d52a83654cb625fe4df6cd8683ba691d824 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 15:14:24 -0400 Subject: [PATCH 0034/1579] soc: apple: open SEP stores from the initial namespace The driver opened its backing-store files with the credentials of the task that triggered the access. A keyctl(2) call from an unprivileged process reaches key unsealing in that process's context, which then could not read the driver's own root-owned 0600 key-bag file, so unsealing failed with -74. Open the stores under kernel credentials from the initial namespace so access never depends on the calling task. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/store_shim.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c index 715c7c9c6f8db..b067c39372fa7 100644 --- a/drivers/soc/apple/store_shim.c +++ b/drivers/soc/apple/store_shim.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -24,14 +25,20 @@ static struct file *open_as_kernel(const char *path, int flags, umode_t mode) const struct cred *old; struct cred *kern; struct file *f; + struct path root; kern = prepare_kernel_cred(&init_task); if (!kern) return ERR_PTR(-ENOMEM); + task_lock(&init_task); + get_fs_root(init_task.fs, &root); + task_unlock(&init_task); + old = override_creds(kern); - f = filp_open(path, flags, mode); + f = file_open_root(&root, path, flags, mode); put_cred(revert_creds(old)); + path_put(&root); return f; } @@ -88,7 +95,6 @@ void sep_store_close(void *handle) filp_close((struct file *)handle, NULL); } -/* Current length in bytes, or a negative errno. */ long long sep_store_size(void *handle) { struct file *f = handle; @@ -98,7 +104,6 @@ long long sep_store_size(void *handle) return i_size_read(file_inode(f)); } -/* Returns bytes read, 0 at end of file, or a negative errno. */ long sep_store_read(void *handle, long long off, void *buf, size_t len) { struct file *f = handle; @@ -107,7 +112,6 @@ long sep_store_read(void *handle, long long off, void *buf, size_t len) return kernel_read(f, buf, len, &pos); } -/* Returns bytes written, or a negative errno. */ long sep_store_write(void *handle, long long off, const void *buf, size_t len) { @@ -117,7 +121,6 @@ long sep_store_write(void *handle, long long off, const void *buf, return kernel_write(f, buf, len, &pos); } -/* Flushes data and metadata to durable storage. */ int sep_store_sync(void *handle) { return vfs_fsync((struct file *)handle, 0); From be57269f2424b22a159dbc4143bbd0b6de29bd44 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 15:29:14 -0400 Subject: [PATCH 0035/1579] soc: apple: publish Touch ID only after SKS is ready The biometric character device was exposed before the shared anti-replay store and the key store had come up, so an early open could race enclave setup. Publish /dev/sep-bio only once shared-xART and the key store are ready. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/sbio.rs | 56 +++++++++++++++++++++++----- drivers/soc/apple/sep.rs | 11 +----- drivers/soc/apple/sks.rs | 28 +++++++++++++- tools/aurora-sep/fprintd-aurora.conf | 4 ++ tools/aurora-sep/load-driver | 7 ++++ 5 files changed, 86 insertions(+), 20 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 6121af2fc5984..29740ce17e1d0 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -784,24 +784,59 @@ impl SepData { return; } } + if let Err(e) = self.register_bio() { + dev_err!( + self.dev, + "bringup: could not publish /dev/{} after SKS became ready: {:?}\n", + bio::DEVICE_NAME, + e + ); + } } fn activate_touchid(&self) -> Result<()> { if self.touchid_started.load(Relaxed) { return Ok(()); } + if self.touchid_failed.load(Relaxed) { + return Err(EIO); + } - let mut store = store::Store::open()?; - let index = bio::IdentityIndex::load(&mut store)?; + let mut store = store::Store::open().map_err(|e| { + dev_err!(self.dev, "Touch ID: opening the host state failed: {:?}\n", e); + e + })?; + let index = bio::IdentityIndex::load(&mut store).map_err(|e| { + dev_err!(self.dev, "Touch ID: loading the host identity index failed: {:?}\n", e); + e + })?; *self.host_store.lock() = Some(store); *self.bio_index.lock() = index; self.attach_sensor(); - self.enable_sbio()?; - let keybag::State::Present(stored) = keybag::read(keybag::Slot::Identity)? else { - return Err(ENOENT); + self.enable_sbio().map_err(|e| { + dev_err!(self.dev, "Touch ID: registering the biometric buffers failed: {:?}\n", e); + e + })?; + let stored = match keybag::read(keybag::Slot::Identity) { + Ok(keybag::State::Present(stored)) => stored, + Ok(keybag::State::Absent(_)) => { + dev_err!(self.dev, "Touch ID: no persisted identity keybag exists\n"); + return Err(ENOENT); + } + Err(e) => { + dev_err!(self.dev, "Touch ID: reading the persisted identity keybag failed: {:?}\n", e); + return Err(e); + } }; - let (handle, uuid) = self.sks_recover(&stored).ok_or(EIO)?; + if !self.sks_ready() { + dev_err!(self.dev, "Touch ID: the key-store endpoint is unavailable\n"); + return Err(ENODEV); + } + let (handle, uuid) = self.sks_recover(&stored).ok_or_else(|| { + dev_err!(self.dev, "Touch ID: the persisted identity keybag did not recover\n"); + EIO + })?; self.sks_designate_user_keybag(handle, stored.secret()); self.sks_machine_refkey(handle, stored.secret()); let prepared = self.cold_match_continue(handle, uuid); @@ -811,11 +846,14 @@ impl SepData { Ok(()) } - // Touch ID starts from the caller's real-root namespace. SEP can therefore - // unlock root in initramfs without pinning biometric persistence to tmpfs. fn prepare_bio_open(&self) -> Result<()> { if let Err(e) = self.activate_touchid() { - dev_err!(self.dev, "Touch ID activation failed: {:?}\n", e); + self.touchid_failed.store(true, Relaxed); + dev_err!( + self.dev, + "Touch ID activation failed: {:?}; refusing retries this boot because a partial keybag load is not safely repeatable\n", + e + ); return Err(e); } Ok(()) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 14cc90ec3b0ce..eb42586361400 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -600,6 +600,7 @@ struct SepData { bringup_started: Atomic, touchid_started: Atomic, + touchid_failed: Atomic, bringup: Atomic, @@ -778,6 +779,7 @@ impl SepData { keybag_designated: Atomic::new(false), bringup_started: Atomic::new(false), touchid_started: Atomic::new(false), + touchid_failed: Atomic::new(false), enrol_open: Atomic::new(false), sensor_calibrated: Atomic::new(false), templates_restored: Atomic::new(false), @@ -2000,15 +2002,6 @@ impl platform::Driver for SepDriver { *data.mbox.lock() = Some(Mailbox::new_byname(dev, c"mbox", data.clone())?); - if let Err(e) = data.register_bio() { - dev_err!( - dev, - "could not register /dev/{}: {:?}\n", - bio::DEVICE_NAME, - e - ); - } - data.attach(&sep_node)?; if let Err(e) = data.register_fv_kernel() { diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 4d68f40bab0d7..a92f327fe13bf 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -631,8 +631,30 @@ impl SepData { &self, stored: &keybag::StoredKeyBag, ) -> Option<(crate::sks::KeyBagHandle, [u8; keybag::UUID_LEN])> { - let out = self.sks_send(self.sks_req_load_keybag(stored.wrapped()))?; - let body = self.sks_report_response(crate::sks::SKS_LOAD_NAME, &out)?; + let request = match self.sks_req_load_keybag(stored.wrapped()) { + Ok(request) => request, + Err(e) => { + dev_err!(self.dev, "sks: could not build LOAD_KEYBAG request: {:?}\n", e); + return None; + } + }; + let out = self.sks_exchange(request.name, request.msg, &request.img)?; + + dev_info!( + self.dev, + "sks: LOAD_KEYBAG reply mailbox status {}, response size {}, copied {} bytes\n", + out.reply.status, + out.reply.response_size, + out.response.len() + ); + + let body = match self.sks_report_response(crate::sks::SKS_LOAD_NAME, &out) { + Some(body) => body, + None => { + dev_warn!(self.dev, "sks: LOAD_KEYBAG reply image was empty or malformed\n"); + return None; + } + }; if out.reply.status != 0 { dev_warn!( @@ -656,6 +678,7 @@ impl SepData { let Some(uuid) = self.sks_read_uuid(handle) else { dev_warn!(self.dev, "sks: loaded keybag has no readable UUID\n"); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); return None; }; @@ -668,6 +691,7 @@ impl SepData { Hex(stored.uuid()), Hex(&uuid) ); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); None } } diff --git a/tools/aurora-sep/fprintd-aurora.conf b/tools/aurora-sep/fprintd-aurora.conf index 8a0fb47305431..6b33af2570948 100644 --- a/tools/aurora-sep/fprintd-aurora.conf +++ b/tools/aurora-sep/fprintd-aurora.conf @@ -1,2 +1,6 @@ +[Unit] +After=aurora-sep.service +Requires=aurora-sep.service + [Service] DeviceAllow=/dev/sep-bio rw diff --git a/tools/aurora-sep/load-driver b/tools/aurora-sep/load-driver index 83942fe5bd00a..f5579fc01b360 100755 --- a/tools/aurora-sep/load-driver +++ b/tools/aurora-sep/load-driver @@ -38,3 +38,10 @@ done [ "$verified" = true ] || exit 1 [ -s "$keybag" ] || exit 1 [ -s "$refkey" ] || exit 1 + +attempts=300 +while [ "$attempts" -gt 0 ] && [ ! -c /dev/sep-bio ]; do + sleep 0.1 + attempts=$((attempts - 1)) +done +[ -c /dev/sep-bio ] || exit 1 From 4fa117c5cf1e66c3c711ac33a2634603f7ad2a59 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 15:37:23 -0400 Subject: [PATCH 0036/1579] soc: apple: harden FileVault key lifetime Reference-count the enclave class keys a FileVault volume loads. Repeated mounts of volumes that share a key bag now load the class keys once, and the volatile keys are unloaded from the enclave only when the last volume using them is released. Key material stays resident no longer than a mounted volume needs it, and keys another mount still depends on are not unloaded early. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/fv.rs | 252 ++++++++++++++++++++++++++++-------- drivers/soc/apple/keybag.rs | 5 +- 2 files changed, 201 insertions(+), 56 deletions(-) diff --git a/drivers/soc/apple/fv.rs b/drivers/soc/apple/fv.rs index 9ba5156dee386..9a9d3c759837d 100644 --- a/drivers/soc/apple/fv.rs +++ b/drivers/soc/apple/fv.rs @@ -88,14 +88,8 @@ struct KernelOps { usize, *mut KernelKey, ) -> c_int, - new_file_key: unsafe extern "C" fn( - *mut c_void, - *const u8, - u32, - u64, - u16, - *mut KernelNewFileKey, - ) -> c_int, + new_file_key: + unsafe extern "C" fn(*mut c_void, *const u8, u32, u64, u16, *mut KernelNewFileKey) -> c_int, } extern "C" { @@ -221,7 +215,7 @@ impl SepData { .map_or(*apfs_uuid, |entry| entry.bag_uuid) } - fn record_fv_volume(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result<()> { + fn retain_fv_volume(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result { let mut volumes = self.fv_volumes.lock(); if let Some(entry) = volumes .iter_mut() @@ -231,7 +225,7 @@ impl SepData { return Err(EINVAL); } entry.refs = entry.refs.checked_add(1).ok_or(EOVERFLOW)?; - return Ok(()); + return Ok(false); } if volumes.len() >= FV_MAX_VOLUME_MAPS { return Err(ENOSPC); @@ -244,10 +238,22 @@ impl SepData { }, GFP_KERNEL, )?; - Ok(()) + Ok(true) + } + + fn fv_volume_is_last(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result { + let volumes = self.fv_volumes.lock(); + let entry = volumes + .iter() + .find(|entry| entry.apfs_uuid == *apfs_uuid) + .ok_or(ENOENT)?; + if entry.bag_uuid != *bag_uuid { + return Err(EINVAL); + } + Ok(entry.refs == 1) } - fn unrecord_fv_volume(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result<()> { + fn release_fv_volume(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result<()> { let mut volumes = self.fv_volumes.lock(); let index = volumes .iter() @@ -291,7 +297,11 @@ impl SepData { .ok_or(EIO)?; if out.reply.status != 0 { let status: i32 = out.reply.status.into(); - dev_err!(self.dev, "fv: GET_BLOB_STATE failed with status {}\n", status); + dev_err!( + self.dev, + "fv: GET_BLOB_STATE failed with status {}\n", + status + ); return Err(EACCES); } let body = self @@ -597,11 +607,7 @@ impl SepData { }) } - fn new_file_key( - &self, - volume_uuid: &[u8; 16], - protection_class: u32, - ) -> Result { + fn new_file_key(&self, volume_uuid: &[u8; 16], protection_class: u32) -> Result { self.fv_ready()?; let class = Self::pfk_class(protection_class)?; let out = self @@ -712,17 +718,15 @@ impl SepData { ) -> Result<()> { self.fv_ready()?; let bag_uuid = self.fv_blob_uuid(volume_uuid, volume_key)?; - let request = self.sks_req_load_class_keys( - volume_uuid, - secret, - unlock_record, - volume_key, - )?; - self.record_fv_volume(volume_uuid, &bag_uuid)?; + let request = + self.sks_req_load_class_keys(volume_uuid, secret, unlock_record, volume_key)?; + if !self.retain_fv_volume(volume_uuid, &bag_uuid)? { + return Ok(()); + } let out = match self.sks_send(Ok(request)) { Some(out) => out, None => { - let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + let _ = self.release_fv_volume(volume_uuid, &bag_uuid); return Err(EIO); } }; @@ -733,21 +737,21 @@ impl SepData { "fv: LOAD_CLASS_KEYS failed with status {}\n", status ); - let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + let _ = self.release_fv_volume(volume_uuid, &bag_uuid); return Err(EACCES); } let body = match self.sks_report_response(crate::sks::SKS_SET_PROTECTION_NAME, &out) { Some(body) => body, None => { self.rollback_class_keys(volume_uuid, volume_key); - let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + let _ = self.release_fv_volume(volume_uuid, &bag_uuid); return Err(EMSGSIZE); } }; let mut fields = proto::FieldCursor::new(body); if fields.i32() != Some(0) || fields.blob().is_none() { self.rollback_class_keys(volume_uuid, volume_key); - let _ = self.unrecord_fv_volume(volume_uuid, &bag_uuid); + let _ = self.release_fv_volume(volume_uuid, &bag_uuid); return Err(EMSGSIZE); } let apfs_hi = u64::from_be_bytes(volume_uuid[..8].try_into().unwrap()); @@ -800,63 +804,119 @@ impl SepData { fn unload_class_keys(&self, volume_uuid: &[u8; 16], volume_key: &[u8]) -> Result<()> { self.fv_ready()?; let bag_uuid = self.fv_blob_uuid(volume_uuid, volume_key)?; - let out = self - .sks_send(self.sks_req_unload_class_keys(volume_uuid, volume_key)) - .ok_or(EIO)?; - if out.reply.status != 0 { - return Err(EACCES); + if !self.fv_volume_is_last(volume_uuid, &bag_uuid)? { + return self.release_fv_volume(volume_uuid, &bag_uuid); } - let body = self - .sks_report_response(crate::sks::SKS_SET_PROTECTION_NAME, &out) - .ok_or(EMSGSIZE)?; - let mut fields = proto::FieldCursor::new(body); - if fields.i32() != Some(0) || fields.blob().is_none() { - return Err(EMSGSIZE); - } - self.unrecord_fv_volume(volume_uuid, &bag_uuid) + let result = (|| { + let out = self + .sks_send(self.sks_req_unload_class_keys(volume_uuid, volume_key)) + .ok_or(EIO)?; + if out.reply.status != 0 { + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_SET_PROTECTION_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(0) || fields.blob().is_none() { + return Err(EMSGSIZE); + } + Ok(()) + })(); + let release = self.release_fv_volume(volume_uuid, &bag_uuid); + result.and(release) } } +/// Borrow a C input buffer as a slice, rejecting null/empty/oversized inputs. +/// +/// # Safety +/// +/// When `ptr` is non-null and `len` is in `1..=max`, `ptr` must be valid for +/// reads of `len` bytes and that region must stay live and unmutated for the +/// whole of `'a`. The caller must pick `'a` so the returned slice cannot +/// outlive the underlying buffer. unsafe fn input<'a>(ptr: *const u8, len: usize, max: usize) -> Result<&'a [u8]> { if ptr.is_null() || len == 0 || len > max { return Err(EINVAL); } - // SAFETY: the C API requires `ptr` to remain readable for `len` bytes for - // the duration of the callback, and the callback does not retain it. + // SAFETY: `ptr` is non-null and `len` is in `1..=max` here, so by this + // function's contract the region is valid for reads of `len` bytes and + // stays live for `'a`. Ok(unsafe { core::slice::from_raw_parts(ptr, len) }) } +/// Borrow an optional C input buffer, treating a zero length as empty. +/// +/// # Safety +/// +/// Same obligations as [`input`]: when `len` is non-zero, `ptr` must be valid +/// for reads of `len` bytes and stay live for `'a`. unsafe fn optional_input<'a>(ptr: *const u8, len: usize, max: usize) -> Result<&'a [u8]> { if len == 0 { return Ok(&[]); } + // SAFETY: `len` is non-zero here; this function's contract is identical to + // `input`'s, so the caller already guarantees `ptr`/`len` are valid. unsafe { input(ptr, len, max) } } +/// Borrow a C output slot for one key. +/// +/// # Safety +/// +/// When `ptr` is non-null it must be aligned, point to a live and writable +/// `KernelKey`, and grant exclusive access for the whole of `'a` (no other +/// alias may touch it while the returned reference lives). unsafe fn output<'a>(ptr: *mut KernelKey) -> Result<&'a mut KernelKey> { if ptr.is_null() { return Err(EINVAL); } - // SAFETY: the C API provides exclusive writable storage for one key. + // SAFETY: `ptr` is non-null here, so by this function's contract it is an + // aligned, exclusively-owned, writable `KernelKey` that outlives `'a`. Ok(unsafe { &mut *ptr }) } +/// Borrow a C output slot for one new-file-key result. +/// +/// # Safety +/// +/// When `ptr` is non-null it must be aligned, point to a live and writable +/// `KernelNewFileKey`, and grant exclusive access for the whole of `'a`. unsafe fn new_file_output<'a>(ptr: *mut KernelNewFileKey) -> Result<&'a mut KernelNewFileKey> { if ptr.is_null() { return Err(EINVAL); } - // SAFETY: the C API provides exclusive writable storage for one result. + // SAFETY: `ptr` is non-null here, so by this function's contract it is an + // aligned, exclusively-owned, writable `KernelNewFileKey` that outlives + // `'a`. Ok(unsafe { &mut *ptr }) } +/// Recover the driver state from a registered callback context pointer. +/// +/// # Safety +/// +/// When `context` is non-null it must be the pointer passed to +/// `sep_fv_register_v2` — a `*const SepData` taken from a live `Arc` — +/// and must still be registered, so the `SepData` is alive and only shared +/// (never uniquely) borrowed for the whole of `'a`. unsafe fn sep<'a>(context: *mut c_void) -> Result<&'a SepData> { if context.is_null() { return Err(ENODEV); } - // SAFETY: registration keeps `SepData` alive until all callbacks finish. + // SAFETY: `context` is non-null here, so by this function's contract it + // points to a live `SepData` that stays valid and shared for `'a`. Ok(unsafe { &*context.cast::() }) } +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `wrapped`/`wrapped_len` describe a buffer readable +/// for the duration of the call (or are null/0); and `key` points to aligned, +/// exclusively-owned, writable storage for one `KernelKey`. unsafe extern "C" fn kernel_unwrap_media_key( context: *mut c_void, wrapped: *const u8, @@ -865,18 +925,32 @@ unsafe extern "C" fn kernel_unwrap_media_key( key: *mut KernelKey, ) -> c_int { let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `wrapped`/`wrapped_len` describe a + // buffer readable for the call; the borrow does not outlive it. let wrapped = unsafe { input(wrapped, wrapped_len, WRAPPED_KEY_LEN)? }; let wrapped: &[u8; WRAPPED_KEY_LEN] = wrapped.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `key` is exclusive writable + // storage for one `KernelKey`, valid for the duration of the call. let key = unsafe { output(key)? }; let unwrapped = this.unwrap_media_key_from_class(wrapped, protection_class)?; key.opaque.copy_from_slice(&unwrapped.opaque); key.iv.copy_from_slice(&unwrapped.iv_key); Ok(()) })(); - result.map_or_else(|error| error.to_errno(), |_| 0) + result.map_or_else(|error| error.to_errno(), |()| 0) } +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; each `*const u8`/length pair describes a buffer +/// readable for the duration of the call (the optional `secret`/`unlock_record` +/// may be null with a zero length); and `key` points to aligned, +/// exclusively-owned, writable storage for one `KernelKey`. unsafe extern "C" fn kernel_unwrap_volume_key( context: *mut c_void, secret: *const u8, @@ -888,20 +962,38 @@ unsafe extern "C" fn kernel_unwrap_volume_key( key: *mut KernelKey, ) -> c_int { let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `secret`/`secret_len` describe a + // buffer readable for the call, or are null/0. let secret = unsafe { optional_input(secret, secret_len, SECRET_MAX_LEN)? }; + // SAFETY: per this function's contract `unlock_record`/`unlock_record_len` + // describe a buffer readable for the call, or are null/0. let unlock_record = unsafe { optional_input(unlock_record, unlock_record_len, RECORD_MAX_LEN)? }; + // SAFETY: per this function's contract `volume_key`/`volume_key_len` + // describe a buffer readable for the call. let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; + // SAFETY: per this function's contract `key` is exclusive writable + // storage for one `KernelKey`, valid for the duration of the call. let key = unsafe { output(key)? }; let unwrapped = this.unwrap_vek(secret, unlock_record, volume_key)?; key.opaque.copy_from_slice(&unwrapped.opaque); key.iv.fill(0); Ok(()) })(); - result.map_or_else(|error| error.to_errno(), |_| 0) + result.map_or_else(|error| error.to_errno(), |()| 0) } +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `volume_uuid` points to 16 readable bytes; and each +/// other `*const u8`/length pair describes a buffer readable for the duration +/// of the call (the optional `secret`/`unlock_record` may be null with a zero +/// length). unsafe extern "C" fn kernel_load_class_keys( context: *mut c_void, volume_uuid: *const u8, @@ -913,18 +1005,35 @@ unsafe extern "C" fn kernel_load_class_keys( volume_key_len: usize, ) -> c_int { let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `volume_uuid` points to 16 + // readable bytes for the duration of the call. let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `secret`/`secret_len` describe a + // buffer readable for the call, or are null/0. let secret = unsafe { optional_input(secret, secret_len, SECRET_MAX_LEN)? }; + // SAFETY: per this function's contract `unlock_record`/`unlock_record_len` + // describe a buffer readable for the call, or are null/0. let unlock_record = unsafe { optional_input(unlock_record, unlock_record_len, RECORD_MAX_LEN)? }; + // SAFETY: per this function's contract `volume_key`/`volume_key_len` + // describe a buffer readable for the call. let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; this.load_class_keys(volume_uuid, secret, unlock_record, volume_key) })(); - result.map_or_else(|error| error.to_errno(), |_| 0) + result.map_or_else(|error| error.to_errno(), |()| 0) } +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `volume_uuid` points to 16 readable bytes; and +/// `volume_key`/`volume_key_len` describe a buffer readable for the duration of +/// the call. unsafe extern "C" fn kernel_unload_class_keys( context: *mut c_void, volume_uuid: *const u8, @@ -932,15 +1041,29 @@ unsafe extern "C" fn kernel_unload_class_keys( volume_key_len: usize, ) -> c_int { let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `volume_uuid` points to 16 + // readable bytes for the duration of the call. let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `volume_key`/`volume_key_len` + // describe a buffer readable for the call. let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; this.unload_class_keys(volume_uuid, volume_key) })(); - result.map_or_else(|error| error.to_errno(), |_| 0) + result.map_or_else(|error| error.to_errno(), |()| 0) } +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `volume_uuid` points to 16 readable bytes; each +/// `wrapped_*`/length pair describes a buffer readable for the duration of the +/// call; and `key` points to aligned, exclusively-owned, writable storage for +/// one `KernelKey`. unsafe extern "C" fn kernel_unwrap_file_key( context: *mut c_void, volume_uuid: *const u8, @@ -952,11 +1075,21 @@ unsafe extern "C" fn kernel_unwrap_file_key( key: *mut KernelKey, ) -> c_int { let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `volume_uuid` points to 16 + // readable bytes for the duration of the call. let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `wrapped_ekwk`/`wrapped_ekwk_len` + // describe a buffer readable for the call. let wrapped_ekwk = unsafe { input(wrapped_ekwk, wrapped_ekwk_len, FILE_KEY_MAX_LEN)? }; + // SAFETY: per this function's contract `wrapped_ek`/`wrapped_ek_len` + // describe a buffer readable for the call. let wrapped_ek = unsafe { input(wrapped_ek, wrapped_ek_len, FILE_KEY_MAX_LEN)? }; + // SAFETY: per this function's contract `key` is exclusive writable + // storage for one `KernelKey`, valid for the duration of the call. let key = unsafe { output(key)? }; let unwrapped = this.unwrap_file_key(volume_uuid, protection_class, wrapped_ekwk, wrapped_ek)?; @@ -964,9 +1097,16 @@ unsafe extern "C" fn kernel_unwrap_file_key( key.iv.copy_from_slice(&unwrapped.iv_key); Ok(()) })(); - result.map_or_else(|error| error.to_errno(), |_| 0) + result.map_or_else(|error| error.to_errno(), |()| 0) } +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `volume_uuid` points to 16 readable bytes; and +/// `key` points to aligned, exclusively-owned, writable storage for one +/// `KernelNewFileKey`. unsafe extern "C" fn kernel_new_file_key( context: *mut c_void, volume_uuid: *const u8, @@ -976,9 +1116,15 @@ unsafe extern "C" fn kernel_new_file_key( key: *mut KernelNewFileKey, ) -> c_int { let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `volume_uuid` points to 16 + // readable bytes for the duration of the call. let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `key` is exclusive writable + // storage for one `KernelNewFileKey`, valid for the duration of the call. let key = unsafe { new_file_output(key)? }; if crypto_id == 0 || key_revision == 0 { return Err(EINVAL); @@ -992,5 +1138,5 @@ unsafe extern "C" fn kernel_new_file_key( key.wrapped_ek_len = generated.wrapped_ek_len; Ok(()) })(); - result.map_or_else(|error| error.to_errno(), |_| 0) + result.map_or_else(|error| error.to_errno(), |()| 0) } diff --git a/drivers/soc/apple/keybag.rs b/drivers/soc/apple/keybag.rs index 515948b28b91f..f97f608821761 100644 --- a/drivers/soc/apple/keybag.rs +++ b/drivers/soc/apple/keybag.rs @@ -24,7 +24,6 @@ impl Slot { Slot::Identity => KEYBAG_PATH, } } - } const MAGIC: [u8; 16] = *b"APPLE-SEP-KBAG01"; @@ -133,7 +132,7 @@ fn checksum_input( wrapped: &[u8], uuid: &[u8; UUID_LEN], secret: &[u8], -) -> Result> { +) -> Result { let mut v = KVec::new(); v.extend_from_slice(&VERSION.to_le_bytes(), GFP_KERNEL)?; v.extend_from_slice(&state.to_le_bytes(), GFP_KERNEL)?; @@ -142,7 +141,7 @@ fn checksum_input( v.extend_from_slice(uuid, GFP_KERNEL)?; v.extend_from_slice(wrapped, GFP_KERNEL)?; v.extend_from_slice(secret, GFP_KERNEL)?; - Ok(v) + Ok(crate::Secret(v)) } /// Absent only when the file is missing; every other unreadable state errors — From 6e0170712539c8532e56a84a9b463c86e6b6edc7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 15:40:54 -0400 Subject: [PATCH 0037/1579] soc: apple: remove unused SEP bookkeeping Remove counters and fields that nothing reads. No functional change. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/control.rs | 4 --- drivers/soc/apple/proto.rs | 24 ------------- drivers/soc/apple/sep.rs | 70 +++--------------------------------- drivers/soc/apple/sks.rs | 7 ++-- 4 files changed, 6 insertions(+), 99 deletions(-) diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index 5e395dc6a0b80..beb6b608415de 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -1,9 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj -//! Control-endpoint bookkeeping: the in-flight request table, the tag -//! allocator, and the reserved-tag entropy sink. - use crate::proto; use kernel::prelude::*; @@ -165,5 +162,4 @@ impl ControlState { pub(crate) fn entropy_end(&mut self) { self.entropy.busy = false; } - } diff --git a/drivers/soc/apple/proto.rs b/drivers/soc/apple/proto.rs index 28324172c5e7b..395f7440b10bc 100644 --- a/drivers/soc/apple/proto.rs +++ b/drivers/soc/apple/proto.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj -//! The SEP wire protocol, reverse-engineered: opcode tables, request encoders -//! and reply decoders. use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; @@ -79,28 +77,6 @@ pub(crate) fn shmem_registration(iova: u64, size: usize) -> Result { }) } -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) struct Fourcc(pub(crate) [u8; 4]); - -impl Fourcc { - pub(crate) const ZERO: Fourcc = Fourcc([0; 4]); -} - -impl kernel::fmt::Display for Fourcc { - fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - use core::fmt::Write; - for c in self.0 { - let c = if (0x20..0x7f).contains(&c) { c } else { b'.' }; - f.write_char(c as char)?; - } - Ok(()) - } -} - -pub(crate) fn fourcc(msg: &Message) -> Fourcc { - Fourcc(((msg.msg0 >> MSG_DATA_SHIFT) as u32).to_be_bytes()) -} - pub(crate) const CONTROL_REPLY_TYPE: u8 = 0x01; pub(crate) const TAG_ENTROPY: u8 = 0xE7; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index eb42586361400..9c45c69bfe0a9 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -159,26 +159,12 @@ const MAX_ENDPOINTS: usize = 64; #[derive(Clone, Copy)] struct Endpoint { id: u8, - fourcc: proto::Fourcc, - have_descriptor: bool, - have_config: bool, - descriptor_msg0: u64, - descriptor_msg1: u32, - config_msg0: u64, - config_msg1: u32, } impl Endpoint { fn new(id: u8) -> Self { Endpoint { id, - fourcc: proto::Fourcc::ZERO, - have_descriptor: false, - have_config: false, - descriptor_msg0: 0, - descriptor_msg1: 0, - config_msg0: 0, - config_msg1: 0, } } } @@ -451,8 +437,6 @@ struct Abandoned { struct SksProbe { active: bool, captured: KVec, - label: Option<&'static CStr>, - unsolicited: u32, abandoned: [Option; SKS_MAX_ABANDONED], abandoned_next: usize, } @@ -462,8 +446,6 @@ impl SksProbe { SksProbe { active: false, captured: KVec::new(), - label: None, - unsolicited: 0, abandoned: [None; SKS_MAX_ABANDONED], abandoned_next: 0, } @@ -486,8 +468,6 @@ impl ScrdProbe { struct XarmState { deferred_query: Option, - serviced: u32, - refused: u32, os_uuid: Option<[u8; 16]>, } @@ -495,8 +475,6 @@ impl XarmState { fn new() -> Self { XarmState { deferred_query: None, - serviced: 0, - refused: 0, os_uuid: None, } } @@ -504,18 +482,12 @@ impl XarmState { struct EndpointTable { eps: KVec, - discovery_msgs: u32, - unknown_types: u32, - dirty: bool, } impl EndpointTable { fn new() -> Self { EndpointTable { eps: KVec::new(), - discovery_msgs: 0, - unknown_types: 0, - dirty: false, } } @@ -527,7 +499,6 @@ impl EndpointTable { return Err(ENOSPC); } self.eps.push(Endpoint::new(id), GFP_KERNEL)?; - self.dirty = true; Ok(self.eps.len() - 1) } } @@ -1109,7 +1080,6 @@ impl SepData { let req = crate::xarm::decode_xarm(&msg); if xarm::is_silent(req.opcode) { - self.xarm.lock().serviced += 1; return; } @@ -1121,9 +1091,6 @@ impl SepData { return; } - let mut state = self.xarm.lock(); - state.refused += 1; - drop(state); self.fail_xarm(req.tag); return; } @@ -1179,8 +1146,6 @@ impl SepData { } } - self.xarm.lock().serviced += 1; - self.send_xarm_reply(&done.reply); } @@ -1263,7 +1228,6 @@ impl SepData { args: [0; 3], }; reply.args[0] = u8::from(PROTECTED_DATA_AVAILABLE); - self.xarm.lock().serviced += 1; self.send_xarm_reply(&reply); } @@ -1724,39 +1688,13 @@ impl SepData { } } - fn on_discovery(&self, msg: Message, f: proto::Fields) { + fn on_discovery(&self, _msg: Message, f: proto::Fields) { let mut table = self.endpoints.lock(); - table.discovery_msgs += 1; - - let id = f.param; - - let idx = match table.slot(id) { - Ok(i) => i, - Err(_) => { - return; - } - }; - match f.ty { - proto::DISCOVER_TYPE_DESCRIPTOR => { - let cc = proto::fourcc(&msg); - let e = &mut table.eps[idx]; - e.have_descriptor = true; - e.descriptor_msg0 = msg.msg0; - e.descriptor_msg1 = msg.msg1; - e.fourcc = cc; - table.dirty = true; - } - proto::DISCOVER_TYPE_CONFIG => { - let e = &mut table.eps[idx]; - e.have_config = true; - e.config_msg0 = msg.msg0; - e.config_msg1 = msg.msg1; - table.dirty = true; - } - _ => { - table.unknown_types += 1; + proto::DISCOVER_TYPE_DESCRIPTOR | proto::DISCOVER_TYPE_CONFIG => { + let _ = table.slot(f.param); } + _ => {} } } diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index a92f327fe13bf..ae5d79d8b5d5d 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -49,7 +49,6 @@ impl SepData { drop(probe); self.sks_wq.notify_all(); } else { - probe.unsolicited = probe.unsolicited.wrapping_add(1); let matched = probe .abandoned .iter() @@ -156,7 +155,7 @@ impl SepData { } else { sized_ms }; - self.sks_arm(label); + self.sks_arm(); if self.send(msg).is_err() { self.sks_disarm(); let _ = self.sks_zero_buffers(); @@ -776,17 +775,15 @@ impl SepData { true } - fn sks_arm(&self, label: &'static CStr) { + fn sks_arm(&self) { let mut probe = self.sks_probe.lock(); probe.captured.clear(); - probe.label = Some(label); probe.active = true; } fn sks_disarm(&self) -> KVec { let mut probe = self.sks_probe.lock(); probe.active = false; - probe.label = None; core::mem::take(&mut probe.captured) } From 025636314f0f9866809b475702f923d58d08ee2f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 15:50:47 -0400 Subject: [PATCH 0038/1579] soc: apple: serialize SEP key-store exchanges Concurrent key-store (SKS) out-of-line exchanges could interleave their mailbox traffic and out-of-line buffer registrations and wedge the endpoint when fingerprint matching and a FileVault mount ran at once. Hold a mutex across each complete SKS exchange so only one is in flight at a time. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/sep.rs | 3 + drivers/soc/apple/sks.rs | 160 +++++++++++++++++++++++++++++---------- 2 files changed, 123 insertions(+), 40 deletions(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 9c45c69bfe0a9..e028ff10ac6c6 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -578,6 +578,8 @@ struct SepData { #[pin] ool_sks: Mutex>, + #[pin] + sks_exchange_lock: Mutex<()>, #[pin] sks_probe: Mutex, #[pin] @@ -762,6 +764,7 @@ impl SepData { ool_xarm <- new_mutex!(Some(ool_xarm)), ool_sbio <- new_mutex!(Some(ool_sbio)), ool_sks <- new_mutex!(Some(ool_sks)), + sks_exchange_lock <- new_mutex!(()), sks_probe <- new_mutex!(SksProbe::new()), sks_wq <- new_condvar!("SepData::sks_wq"), ool_scrd <- new_mutex!(Some(ool_scrd)), diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index ae5d79d8b5d5d..6469ebe24f5bc 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -4,9 +4,9 @@ //! framing, DER-imaged request builders, and lock-state control. use super::*; +use crate::proto::*; use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; -use crate::proto::*; struct KeybagCreateIntent<'a> { dev: &'a device::Device, @@ -20,7 +20,11 @@ impl<'a> KeybagCreateIntent<'a> { dev_err!(dev, "sks: cannot persist keybag-create intent: {:?}\n", e); return None; } - Some(Self { dev, slot, sent: false }) + Some(Self { + dev, + slot, + sent: false, + }) } fn sending(&mut self) { @@ -32,7 +36,11 @@ impl Drop for KeybagCreateIntent<'_> { fn drop(&mut self) { if !self.sent { if let Err(e) = keybag::mark_refused(self.slot) { - dev_err!(self.dev, "sks: cannot mark unsent keybag create retryable: {:?}\n", e); + dev_err!( + self.dev, + "sks: cannot mark unsent keybag create retryable: {:?}\n", + e + ); } } } @@ -59,7 +67,12 @@ impl SepData { if let Some(a) = matched { let _ = self.sks_zero_buffers(); self.sks_wedged.store(0, Relaxed); - dev_warn!(self.dev, "sks: late answer to {} after {} ms; wedge lifted\n", a.label, a.waited_ms); + dev_warn!( + self.dev, + "sks: late answer to {} after {} ms; wedge lifted\n", + a.label, + a.waited_ms + ); } } } @@ -134,6 +147,7 @@ impl SepData { floor_ms: time::Msecs, condemn: bool, ) -> Option { + let _exchange = self.sks_exchange_lock.lock(); if !self.ool_registered(&self.ool_sks) { return None; } @@ -206,7 +220,12 @@ impl SepData { if condemn { self.sks_wedged.store(1, Relaxed); } - dev_err!(self.dev, "sks: {} got no reply after {} ms\n", label, waited_ms); + dev_err!( + self.dev, + "sks: {} got no reply after {} ms\n", + label, + waited_ms + ); return None; }; @@ -220,8 +239,7 @@ impl SepData { } } - // The correlated reply means the enclave has finished with both OOL - // buffers. Keep only the private response copy returned to the caller. + // The correlated reply means the enclave has finished with both OOL buffers. let _ = self.sks_zero_buffers(); Some(SksOutcome { reply, response }) @@ -299,7 +317,10 @@ impl SepData { } /// `0x05` unload the source handle. - pub(crate) fn sks_req_unload_keybag(&self, handle: crate::sks::KeyBagHandle) -> Result { + pub(crate) fn sks_req_unload_keybag( + &self, + handle: crate::sks::KeyBagHandle, + ) -> Result { let mut body = image::Body::new(); body.put_u32(0)?; body.put_u64(crate::sks::SKS_CLIENT_ID)?; @@ -315,7 +336,10 @@ impl SepData { } /// `0x06` UUID read, against the special handle. - pub(crate) fn sks_req_copy_uuid_special(&self, special: crate::sks::SpecialHandle) -> Result { + pub(crate) fn sks_req_copy_uuid_special( + &self, + special: crate::sks::SpecialHandle, + ) -> Result { let op = crate::sks::sks_copy_keybag_uuid(); let mut body = image::Body::new(); body.put_u32(0)?; @@ -340,7 +364,8 @@ impl SepData { body.put_u64(SKS_LOCK_STATE_FLAGS)?; let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; - let msg = crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; + let msg = + crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { name: crate::sks::SKS_LOCK_STATE_NAME, msg, @@ -418,7 +443,11 @@ impl SepData { let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_create(self.sks_next_seq(), len).ok_or(EINVAL)?; - Ok(SksRequest { name: crate::sks::SKS_CREATE_NAME, msg, img }) + Ok(SksRequest { + name: crate::sks::SKS_CREATE_NAME, + msg, + img, + }) } pub(crate) fn sks_provision_identity_keybag(&self) -> bool { @@ -426,7 +455,11 @@ impl SepData { Ok(keybag::State::Present(_)) => return true, Ok(keybag::State::Absent(proof)) => proof, Err(e) => { - dev_err!(self.dev, "sks: identity keybag state is ambiguous: {:?}\n", e); + dev_err!( + self.dev, + "sks: identity keybag state is ambiguous: {:?}\n", + e + ); return false; } }; @@ -452,7 +485,11 @@ impl SepData { let request = match self.sks_req_create_identity_keybag(&secret, &uuid, proof) { Ok(request) => request, Err(e) => { - dev_err!(self.dev, "sks: cannot build identity keybag request: {:?}\n", e); + dev_err!( + self.dev, + "sks: cannot build identity keybag request: {:?}\n", + e + ); return false; } }; @@ -464,7 +501,12 @@ impl SepData { return false; }; if out.reply.status != 0 || body.len() < 12 { - dev_err!(self.dev, "sks: CREATE_KEYBAG failed: mailbox {}, body {} bytes\n", out.reply.status, body.len()); + dev_err!( + self.dev, + "sks: CREATE_KEYBAG failed: mailbox {}, body {} bytes\n", + out.reply.status, + body.len() + ); return false; } let variant = u32::from_le_bytes(body[0..4].try_into().unwrap()); @@ -472,8 +514,14 @@ impl SepData { let Some((_fv_data, end)) = image::read_blob(body, 8) else { return false; }; - if variant != crate::sks::SKS_CREATE_VARIANT_IDENTITY || raw_handle < 0 || end != body.len() { - dev_err!(self.dev, "sks: invalid CREATE_KEYBAG reply: variant {}, handle {}\n", variant, raw_handle); + if variant != crate::sks::SKS_CREATE_VARIANT_IDENTITY || raw_handle < 0 || end != body.len() + { + dev_err!( + self.dev, + "sks: invalid CREATE_KEYBAG reply: variant {}, handle {}\n", + variant, + raw_handle + ); return false; } let handle = crate::sks::KeyBagHandle::from_create_reply(raw_handle); @@ -502,7 +550,11 @@ impl SepData { } } - pub(crate) fn sks_designate_user_keybag(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) { + pub(crate) fn sks_designate_user_keybag( + &self, + handle: crate::sks::KeyBagHandle, + secret: &[u8], + ) { let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) else { return; }; @@ -510,12 +562,13 @@ impl SepData { let designation = crate::sks::Designation::new(handle, user); let Some(out) = self.sks_send(self.sks_req_designate(&designation, secret)) else { - dev_warn!(self.dev, "sks: DESIGNATE_KEYBAG did not complete; enrolment will refuse\n"); + dev_warn!( + self.dev, + "sks: DESIGNATE_KEYBAG did not complete; enrolment will refuse\n" + ); return; }; - let Some(body) = - self.sks_report_response(crate::sks::SKS_DESIGNATE_NAME, &out) - else { + let Some(body) = self.sks_report_response(crate::sks::SKS_DESIGNATE_NAME, &out) else { return; }; @@ -529,17 +582,10 @@ impl SepData { self.keybag_designated.store(true, Relaxed); - self.sks_remember_enrolment_material( - designation.user().special_handle(), - secret, - ); + self.sks_remember_enrolment_material(designation.user().special_handle(), secret); } - fn sks_remember_enrolment_material( - &self, - special: crate::sks::SpecialHandle, - secret: &[u8], - ) { + fn sks_remember_enrolment_material(&self, special: crate::sks::SpecialHandle, secret: &[u8]) { let mut copy = KVec::new(); if copy.extend_from_slice(secret, GFP_KERNEL).is_err() { return; @@ -569,7 +615,8 @@ impl SepData { body.put_u64(SKS_LOCK_STATE_FLAGS)?; let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; - let msg = crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; + let msg = + crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { name: crate::sks::SKS_LOCK_STATE_NAME, msg, @@ -616,11 +663,20 @@ impl SepData { pub(crate) fn sks_health_check(&self, why: &CStr) -> Option { let Some(out) = self.sks_send(self.sks_req_get_capabilities()) else { - dev_err!(self.dev, "sks: health check ({}) got no reply; endpoint gone for this boot\n", why); + dev_err!( + self.dev, + "sks: health check ({}) got no reply; endpoint gone for this boot\n", + why + ); return None; }; if out.reply.status != 0 { - dev_err!(self.dev, "sks: health check ({}) returned status {}\n", why, out.reply.status); + dev_err!( + self.dev, + "sks: health check ({}) returned status {}\n", + why, + out.reply.status + ); return None; } Some(Healthy(())) @@ -633,7 +689,11 @@ impl SepData { let request = match self.sks_req_load_keybag(stored.wrapped()) { Ok(request) => request, Err(e) => { - dev_err!(self.dev, "sks: could not build LOAD_KEYBAG request: {:?}\n", e); + dev_err!( + self.dev, + "sks: could not build LOAD_KEYBAG request: {:?}\n", + e + ); return None; } }; @@ -650,7 +710,10 @@ impl SepData { let body = match self.sks_report_response(crate::sks::SKS_LOAD_NAME, &out) { Some(body) => body, None => { - dev_warn!(self.dev, "sks: LOAD_KEYBAG reply image was empty or malformed\n"); + dev_warn!( + self.dev, + "sks: LOAD_KEYBAG reply image was empty or malformed\n" + ); return None; } }; @@ -664,13 +727,23 @@ impl SepData { return None; } if body.len() != SKS_LOAD_REPLY_LEN { - dev_warn!(self.dev, "sks: LOAD_KEYBAG returned {} bytes, expected {}\n", body.len(), SKS_LOAD_REPLY_LEN); + dev_warn!( + self.dev, + "sks: LOAD_KEYBAG returned {} bytes, expected {}\n", + body.len(), + SKS_LOAD_REPLY_LEN + ); return None; } let status = i32::from_le_bytes([body[0], body[1], body[2], body[3]]); let handle = i32::from_le_bytes([body[4], body[5], body[6], body[7]]); if status != 0 || handle < 0 { - dev_warn!(self.dev, "sks: LOAD_KEYBAG operation status {}, handle {}\n", status, handle); + dev_warn!( + self.dev, + "sks: LOAD_KEYBAG operation status {}, handle {}\n", + status, + handle + ); return None; } let handle = crate::sks::KeyBagHandle::from_load_reply(handle); @@ -769,7 +842,11 @@ impl SepData { return false; } if let Err(e) = self.enable_sks() { - dev_err!(self.dev, "sks: could not register out-of-line buffers ({:?})\n", e); + dev_err!( + self.dev, + "sks: could not register out-of-line buffers ({:?})\n", + e + ); return false; } true @@ -938,7 +1015,6 @@ static_assert!(SKS_CLIENT_ID == 0x4c49_4e55_5853_4b53); pub(crate) struct KeyBagHandle(i32); impl KeyBagHandle { - pub(crate) const fn from_create_reply(v: i32) -> KeyBagHandle { KeyBagHandle(v) } @@ -1088,7 +1164,11 @@ pub(crate) fn encode_sks_load(seq: Sequence, len: ImageLen) -> Option { } pub(crate) fn encode_sks_create(seq: Sequence, len: ImageLen) -> Option { - Some(encode_sks_raw(OP_SKS_CREATE_KEYBAG, seq.value(), len.value())) + Some(encode_sks_raw( + OP_SKS_CREATE_KEYBAG, + seq.value(), + len.value(), + )) } pub(crate) fn encode_sks_change_lock_state(seq: Sequence, len: ImageLen) -> Option { From f7bfc2a3c6d7ea0be4c145dfcedd0e40c5d83dec Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 20:41:00 -0400 Subject: [PATCH 0039/1579] soc: apple: restrict the fixed power-GPIO fallback to j414s When the sensor node carries no power GPIO the shim fell back to a fixed pinctrl node and line number specific to j414s. That fallback is machine-blind: on a sibling board whose sensor sits on a different line it would drive the wrong line. Take the fallback only on j414s; every other machine must describe the power GPIO in its device-tree node. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/sensor_shim.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index a182d6ac04817..924433b094a64 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -70,6 +70,17 @@ static void sep_acquire_power(struct spi_device *spi) } sep_power = NULL; + /* + * The fallback below is the j414s power line; a sibling board's sensor sits + * on a different pin, so only j414s may use it. Every other machine must + * describe the power GPIO in its device node (the gpiod_get_index path). + */ + if (!of_machine_is_compatible("apple,j414s")) { + dev_warn(&spi->dev, + "sep sensor: no power GPIO in the device node; describe gpios in DT\n"); + return; + } + np = of_find_node_by_path(SEP_SENSOR_GPIO_NODE); if (!np) { dev_warn(&spi->dev, From 426a3038f8eba1ce0685cf2b6de495271ee50e03 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 20:41:00 -0400 Subject: [PATCH 0040/1579] soc: apple: find the fingerprint sensor by its device-tree node Sensor bring-up located its SPI controller by a fixed address and so only found the bus on j414s. Look up the apple,mesa-fingerprint node wherever the device tree describes it and enable that node and its parent controller directly, taking the power and interrupt GPIOs from the node. Any machine that describes the sensor in its device tree can then bring it up. When no such node exists the driver still creates one at the caller's fixed controller address, so j414s is unchanged. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/dt.rs | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/dt.rs b/drivers/soc/apple/dt.rs index ca3b9445e8d60..148e8268b2950 100644 --- a/drivers/soc/apple/dt.rs +++ b/drivers/soc/apple/dt.rs @@ -65,6 +65,13 @@ impl DtNode { (!np.is_null()).then_some(DtNode(np)) } + fn parent(&self) -> Option { + // SAFETY: `self.0` is a valid node per the type invariant; `of_get_parent` + // takes a reference on what it returns and returns NULL at the root. + let np = unsafe { bindings::of_get_parent(self.0) }; + (!np.is_null()).then_some(DtNode(np)) + } + fn as_ptr(&self) -> *mut bindings::device_node { self.0 } @@ -82,7 +89,6 @@ impl DtNode { }; !p.is_null() } - } impl Drop for DtNode { @@ -392,6 +398,33 @@ unsafe fn add_u32( } pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { + // A machine that describes the sensor in its own device tree (compatible + // apple,mesa-fingerprint) is handled machine-agnostically: enable that node + // and its SPI controller wherever they sit, with the power/IRQ GPIOs coming + // from the node itself. Only when no such node exists do we fall back to + // creating one at the caller's fixed controller address. + if let Some(sensor) = DtNode::find_compatible(SENSOR_COMPATIBLE) { + let controller = sensor.parent().ok_or_else(|| { + pr_err!("apple_sep: sensor node has no parent SPI controller\n"); + ENODEV + })?; + pr_info!( + "apple_sep: sensor node present in the device tree; enabling it and its SPI bus\n" + ); + return with_changeset(|cs_handle| { + let mut queued = false; + if !controller.is_available() { + queue_status_okay(cs_handle, &controller)?; + queued = true; + } + if !sensor.is_available() { + queue_status_okay(cs_handle, &sensor)?; + queued = true; + } + Ok(queued) + }); + } + let controller = node_at_address(base).ok_or_else(|| { pr_err!( "apple_sep: no device-tree node with reg base 0x{:x}; the sensor's SPI bus is not in this tree\n", From 4b0bc8f689329751a35aab505e38f89ce17a3da5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 20:41:00 -0400 Subject: [PATCH 0041/1579] soc: apple: accept the newer fingerprint sensor revision The match path rejected any sensor whose reported identifier was not 0x3352, the revision fitted up to M4. Newer machines ship a 0x335e part that is identical over the wire. Accept both revisions so matching is not refused on those machines. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/soc/apple/sbio.rs | 241 +++++++++++++++++------------------- drivers/soc/apple/sensor.rs | 15 +-- 2 files changed, 118 insertions(+), 138 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 29740ce17e1d0..ed23c237c531d 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -4,10 +4,10 @@ //! The enclave matches; no biometric image ever crosses to userspace. use super::*; -use kernel::prelude::*; -use kernel::soc::apple::mailbox::Message; use crate::proto::*; use crate::sks::SKS_AUTH_TOKEN_LEN; +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; impl SepData { fn sbio_expect_ok(&self, op: &crate::sbio::SbioOp) -> Option> { @@ -31,15 +31,9 @@ impl SepData { match err as u16 { crate::sbio::SBIO_STATUS_OK => SbioOutcome::Ok(done.payload), - crate::sbio::SBIO_STATUS_PREREQUISITE => { - SbioOutcome::PrerequisiteMissing - } - crate::sbio::SBIO_STATUS_16 => { - SbioOutcome::Status16 - } - _ => { - SbioOutcome::Other - } + crate::sbio::SBIO_STATUS_PREREQUISITE => SbioOutcome::PrerequisiteMissing, + crate::sbio::SBIO_STATUS_16 => SbioOutcome::Status16, + _ => SbioOutcome::Other, } } @@ -58,15 +52,9 @@ impl SepData { ); let policy_ok = match self.sbio_call(&crate::sbio::sbio_match_policy()) { - SbioOutcome::Ok(policy) if policy.len() == crate::sbio::SBIO_MATCH_POLICY_LEN => { - true - } - SbioOutcome::Ok(_) => { - false - } - _ => { - false - } + SbioOutcome::Ok(policy) if policy.len() == crate::sbio::SBIO_MATCH_POLICY_LEN => true, + SbioOutcome::Ok(_) => false, + _ => false, }; synced && policy_ok @@ -101,7 +89,9 @@ impl SepData { ); } - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ENROL_REPOSITION_MS))); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from( + ENROL_REPOSITION_MS, + ))); } fn stash_enrol_identity(&self, record: &[u8]) { @@ -290,7 +280,11 @@ impl SepData { true } - fn open_fresh_context(&self, user: crate::sbio::UserId, proof: &crate::sbio::NoExistingCatacomb) -> bool { + fn open_fresh_context( + &self, + user: crate::sbio::UserId, + proof: &crate::sbio::NoExistingCatacomb, + ) -> bool { let system = crate::sbio::sbio_select_context(crate::sbio::ContextScope::SYSTEM, proof); if self.sbio_expect_ok(&system).is_none() { return false; @@ -313,7 +307,8 @@ impl SepData { } } - let per_user = crate::sbio::sbio_select_context(crate::sbio::ContextScope::user(user), proof); + let per_user = + crate::sbio::sbio_select_context(crate::sbio::ContextScope::user(user), proof); if self.sbio_expect_ok(&per_user).is_none() { return false; } @@ -365,7 +360,8 @@ impl SepData { ); return None; }; - let op = crate::sbio::sbio_begin_enrol(user, crate::sbio::BE_AUTH_TYPE_ACM_CONTEXT, &acm_handle); + let op = + crate::sbio::sbio_begin_enrol(user, crate::sbio::BE_AUTH_TYPE_ACM_CONTEXT, &acm_handle); match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { @@ -375,9 +371,7 @@ impl SepData { armed: true, }) } - _ => { - None - } + _ => None, } } @@ -469,12 +463,8 @@ impl SepData { return RestoreOutcome::Failed; }; match crate::sbio::component_action(state) { - crate::sbio::ComponentAction::AlreadyActive => { - RestoreOutcome::AlreadyActive - } - crate::sbio::ComponentAction::Unsupported => { - RestoreOutcome::Failed - } + crate::sbio::ComponentAction::AlreadyActive => RestoreOutcome::AlreadyActive, + crate::sbio::ComponentAction::Unsupported => RestoreOutcome::Failed, crate::sbio::ComponentAction::Load => { let Some(blob) = self.read_stored(kind, what) else { return RestoreOutcome::NoStoredFile; @@ -499,13 +489,15 @@ impl SepData { self.confirm_active(id, what, LoadAnswer::StatusZero) } Ok(done) - if done.status.answered() == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) && cold_ok => { self.confirm_active(id, what, LoadAnswer::ColdTransition) } Ok(done) - if done.status.answered() == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) => + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) => { RestoreOutcome::Failed } @@ -516,9 +508,7 @@ impl SepData { { self.confirm_active(id, what, LoadAnswer::AlreadyActive) } - _ => { - RestoreOutcome::Failed - } + _ => RestoreOutcome::Failed, } } } @@ -540,15 +530,9 @@ impl SepData { Some(state) if state & crate::sbio::COMPONENT_STATE_ACTIVE != 0 => { RestoreOutcome::Restored } - Some(state) if state & crate::sbio::COMPONENT_STATE_COLD != 0 => { - RestoreOutcome::Failed - } - Some(_) => { - RestoreOutcome::Failed - } - None => { - RestoreOutcome::Failed - } + Some(state) if state & crate::sbio::COMPONENT_STATE_COLD != 0 => RestoreOutcome::Failed, + Some(_) => RestoreOutcome::Failed, + None => RestoreOutcome::Failed, } } @@ -560,9 +544,7 @@ impl SepData { return RestoreOutcome::Failed; }; match self.sbio_transfer_raw(request.opcode(), request.name(), request.payload()) { - Ok(done) if done.status.is_ok() => { - RestoreOutcome::Restored - } + Ok(done) if done.status.is_ok() => RestoreOutcome::Restored, Ok(done) if matches!( done.status, @@ -571,12 +553,8 @@ impl SepData { { RestoreOutcome::EmptyTolerated } - Ok(_) => { - RestoreOutcome::Failed - } - Err(_) => { - RestoreOutcome::Failed - } + Ok(_) => RestoreOutcome::Failed, + Err(_) => RestoreOutcome::Failed, } } @@ -589,9 +567,7 @@ impl SepData { match self.with_host_store(|store| store.read(&store::Key::root(kind))) { Some(Ok(Some(blob))) => Some(blob), Some(Ok(None)) => None, - Some(Err(_)) => { - None - } + Some(Err(_)) => None, None => None, } } @@ -626,7 +602,6 @@ impl SepData { } fn save_after_match(&self, user: crate::sbio::UserId) -> bool { - if !self.save_lockout() { dev_err!( self.dev, @@ -706,13 +681,11 @@ impl SepData { if blob.is_empty() { return false; } - match self.with_host_store(|store| store.write(&store::Key::root(PRIVATE_TYPE_LOCKOUT), &blob)) { - Some(Ok(())) => { - true - } - Some(Err(_)) => { - false - } + match self + .with_host_store(|store| store.write(&store::Key::root(PRIVATE_TYPE_LOCKOUT), &blob)) + { + Some(Ok(())) => true, + Some(Err(_)) => false, None => false, } } @@ -776,11 +749,17 @@ impl SepData { } if *module_parameters::provision_keybag.value() != 0 { if *module_parameters::xart_writes.value() == 0 { - dev_err!(self.dev, "bringup: provision_keybag=1 requires xart_writes=1\n"); + dev_err!( + self.dev, + "bringup: provision_keybag=1 requires xart_writes=1\n" + ); return; } if !self.sks_provision_identity_keybag() { - dev_err!(self.dev, "bringup: identity-keybag provisioning failed; no retry this boot\n"); + dev_err!( + self.dev, + "bringup: identity-keybag provisioning failed; no retry this boot\n" + ); return; } } @@ -802,21 +781,30 @@ impl SepData { return Err(EIO); } - let mut store = store::Store::open().map_err(|e| { - dev_err!(self.dev, "Touch ID: opening the host state failed: {:?}\n", e); - e + let mut store = store::Store::open().inspect_err(|e| { + dev_err!( + self.dev, + "Touch ID: opening the host state failed: {:?}\n", + e + ); })?; - let index = bio::IdentityIndex::load(&mut store).map_err(|e| { - dev_err!(self.dev, "Touch ID: loading the host identity index failed: {:?}\n", e); - e + let index = bio::IdentityIndex::load(&mut store).inspect_err(|e| { + dev_err!( + self.dev, + "Touch ID: loading the host identity index failed: {:?}\n", + e + ); })?; *self.host_store.lock() = Some(store); *self.bio_index.lock() = index; self.attach_sensor(); - self.enable_sbio().map_err(|e| { - dev_err!(self.dev, "Touch ID: registering the biometric buffers failed: {:?}\n", e); - e + self.enable_sbio().inspect_err(|e| { + dev_err!( + self.dev, + "Touch ID: registering the biometric buffers failed: {:?}\n", + e + ); })?; let stored = match keybag::read(keybag::Slot::Identity) { Ok(keybag::State::Present(stored)) => stored, @@ -825,16 +813,26 @@ impl SepData { return Err(ENOENT); } Err(e) => { - dev_err!(self.dev, "Touch ID: reading the persisted identity keybag failed: {:?}\n", e); + dev_err!( + self.dev, + "Touch ID: reading the persisted identity keybag failed: {:?}\n", + e + ); return Err(e); } }; if !self.sks_ready() { - dev_err!(self.dev, "Touch ID: the key-store endpoint is unavailable\n"); + dev_err!( + self.dev, + "Touch ID: the key-store endpoint is unavailable\n" + ); return Err(ENODEV); } let (handle, uuid) = self.sks_recover(&stored).ok_or_else(|| { - dev_err!(self.dev, "Touch ID: the persisted identity keybag did not recover\n"); + dev_err!( + self.dev, + "Touch ID: the persisted identity keybag did not recover\n" + ); EIO })?; self.sks_designate_user_keybag(handle, stored.secret()); @@ -1121,7 +1119,10 @@ impl SepData { let stage = self.bringup.load(Relaxed); if stage >= BRINGUP_ESTABLISHED { - if self.sbio_expect_ok(&crate::sbio::sbio_clear_state()).is_none() { + if self + .sbio_expect_ok(&crate::sbio::sbio_clear_state()) + .is_none() + { return false; } } @@ -1151,9 +1152,7 @@ impl SepData { self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); true } - _ => { - false - } + _ => false, } } @@ -1184,7 +1183,7 @@ impl SepData { } if let sensor::Offset12::Identifier(id) = st.offset12() { - if id != 0 && id != sensor::EXPECTED_IDENTIFIER { + if id != 0 && !sensor::KNOWN_IDENTIFIERS.contains(&id) { return None; } } @@ -1283,14 +1282,16 @@ impl SepData { } fn complete_bringup(&self, patch: PatchLoaded) -> bool { - let Some(params) = self.apply_sensor_parameters() else { return false; }; let op = crate::sbio::sbio_complete_init(patch, params); if self.sbio_expect_ok(&op).is_none() { - dev_err!(self.dev, "sensor: 0x01 COMPLETE_INIT failed — status above\n"); + dev_err!( + self.dev, + "sensor: 0x01 COMPLETE_INIT failed — status above\n" + ); return false; } @@ -1368,12 +1369,8 @@ impl SepData { let relayed = blob.len(); match sensor::send_encrypted_parameters(&blob, geom) { - Ok(()) => { - Some(relayed) - } - Err(sensor::ParamsError::Empty) => { - None - } + Ok(()) => Some(relayed), + Err(sensor::ParamsError::Empty) => None, Err(sensor::ParamsError::TooLong(n, capacity)) => { dev_err!( self.dev, @@ -1385,14 +1382,11 @@ impl SepData { ); None } - Err(sensor::ParamsError::Transfer(_e)) => { - None - } + Err(sensor::ParamsError::Transfer(_e)) => None, } } fn establish_session(&self) -> bool { - let share = match self.sbio_call(&crate::sbio::sbio_request_session_share()) { SbioOutcome::Ok(sh) => sh, SbioOutcome::Status16 => { @@ -1426,20 +1420,13 @@ impl SepData { }; match self.sbio_transfer(&crate::sbio::sbio_commit_session_share(&reply)) { - Ok(done) if done.status.is_ok() => { - true - } - Ok(_) => { - false - } - Err(_) => { - false - } + Ok(done) if done.status.is_ok() => true, + Ok(_) => false, + Err(_) => false, } } fn init_sequence_counter(&self) -> bool { - let challenge = match self.sbio_call(&crate::sbio::sbio_request_challenge()) { SbioOutcome::Ok(c) => c, SbioOutcome::PrerequisiteMissing => { @@ -1478,15 +1465,9 @@ impl SepData { }; match self.sbio_transfer(&crate::sbio::sbio_commit_challenge(&reply)) { - Ok(done) if done.status.is_ok() => { - true - } - Ok(_) => { - false - } - Err(_) => { - false - } + Ok(done) if done.status.is_ok() => true, + Ok(_) => false, + Err(_) => false, } } @@ -1538,7 +1519,6 @@ impl SepData { } }; if st.patch_ack() == sensor::PATCH_ACCEPTED { - if let Ok(after) = sensor::status() { if after.state == sensor::STATE_NEEDS_PATCH { return None; @@ -1584,7 +1564,10 @@ impl SepData { let capture = match sensor::read_capture(available) { Ok(c) => c, - Err(sensor::CaptureError::Checksum { advertised: _advertised, computed: _computed }) => { + Err(sensor::CaptureError::Checksum { + advertised: _advertised, + computed: _computed, + }) => { return ImageOutcome::Retry; } Err(sensor::CaptureError::Length(_n)) => { @@ -1768,11 +1751,11 @@ impl SepData { // 0xFE requests the peer's next packet and acks the final one; nothing to send self.sbio_wq.notify_all(); } - transfer::Progress::Ignored => {}, + transfer::Progress::Ignored => {} transfer::Progress::Grant => { self.sbio_wq.notify_all(); } - transfer::Progress::Notification => {}, + transfer::Progress::Notification => {} transfer::Progress::Failed => { self.sbio_wq.notify_all(); } @@ -2070,9 +2053,10 @@ impl SepData { fn bio_attest(&self, arg: usize) -> Result { let user = kernel::uaccess::UserPtr::from_addr(arg); - let req: bio::Attest = kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) - .reader() - .read()?; + let req: bio::Attest = + kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; let (sig, pubk) = self.refkey_attest_sign(&req.challenge)?; if sig.len() > bio::ATTEST_SIG_MAX || pubk.len() != bio::ATTEST_PUB_LEN { return Err(EIO); @@ -2146,7 +2130,7 @@ impl SepData { .sks_send(self.sks_req_copy_uuid_special(special)) .and_then(|out| self.sks_uuid_from_reply(&out)); match uuid_ok { - Some(got) if got == uuid => {}, + Some(got) if got == uuid => {} Some(_) => { return false; } @@ -2172,7 +2156,11 @@ impl SepData { Ok(()) } - pub(crate) fn wrapped_from_copy_reply(&self, out: &SksOutcome, _from: &CStr) -> Option> { + pub(crate) fn wrapped_from_copy_reply( + &self, + out: &SksOutcome, + _from: &CStr, + ) -> Option> { let body = self.sks_report_response(c"COPY_KEYBAG", out)?; if out.reply.status != 0 || image::operation_status(body).unwrap_or(-1) != 0 { return None; @@ -3050,7 +3038,6 @@ impl ImagePurpose { ImagePurpose::Matching => 0x09, } } - } static_assert!(ImagePurpose::Enrolment.offset() != ImagePurpose::Matching.offset()); diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 79d9e3bc7d50c..d94cc00a1cff2 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -18,12 +18,7 @@ extern "C" { fn sep_sensor_power(on: c_int) -> c_int; fn sep_sensor_xfer(tx: *const c_void, rx: *mut c_void, len: usize) -> c_int; fn sep_sensor_xfer_tx(tx: *const c_void, len: usize) -> c_int; - fn sep_sensor_xfer2( - tx: *const c_void, - tx_len: usize, - rx: *mut c_void, - rx_len: usize, - ) -> c_int; + fn sep_sensor_xfer2(tx: *const c_void, tx_len: usize, rx: *mut c_void, rx_len: usize) -> c_int; } // Spi2. @@ -89,7 +84,8 @@ impl PowerSource { pub(crate) const POWER_ON_READ_DELAYS_MS: [u32; 4] = [0, 3, 10, 50]; pub(crate) const STATUS_IDENTIFIER: usize = 12; -pub(crate) const EXPECTED_IDENTIFIER: u16 = 0x3352; +// Known Mesa sensor revisions: 0x3352 through M4, 0x335e on newer parts. +pub(crate) const KNOWN_IDENTIFIERS: [u16; 2] = [0x3352, 0x335e]; static_assert!(STATUS_IDENTIFIER + 2 <= STATUS_LEN); const CMD_LEN: usize = 7; @@ -613,9 +609,7 @@ pub(crate) fn status() -> Result { let mut rx = [0u8; STATUS_XFER_LEN]; // SAFETY: both buffers are `STATUS_XFER_LEN` bytes and live across the call. - check(unsafe { - sep_sensor_xfer(tx.as_ptr().cast(), rx.as_mut_ptr().cast(), STATUS_XFER_LEN) - })?; + check(unsafe { sep_sensor_xfer(tx.as_ptr().cast(), rx.as_mut_ptr().cast(), STATUS_XFER_LEN) })?; let mut raw = [0u8; STATUS_LEN]; raw.copy_from_slice(&rx[STATUS_AT..STATUS_AT + STATUS_LEN]); @@ -638,7 +632,6 @@ impl Capture { pub(crate) fn bytes(&self) -> &[u8] { &self.0 } - } impl Drop for Capture { From 71f4b41893b4f48a06fdd54df188b859ed64ec6d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 15 Sep 2026 17:05:00 -0400 Subject: [PATCH 0042/1579] arm64: dts: apple: t8112: add spi2 for the Touch ID sensor The MacBook Air (M2) and MacBook Pro 13" (M2) carry the Mesa fingerprint sensor on spi2, chip-select 0, like every other Apple laptop. t8112 only described spi1, spi3 and spi4, so the Apple SEP driver had no controller to bring the sensor up on these machines. Add the spi2 controller node, mirroring the sibling spi1/spi3 controllers (reg base + 0x8000, AIC IRQ 750, ps_spi2 power domain), and the spi2_pins pinmux group. The pad numbers (64-67: SDI, SDO, SCK and the hardware chip select) were read back from the live AP pinctrl on a J415 and cross-checked against the T8112 ADT, where spi2 sits between spi1's CS pad 49 and spi3's CS pad 96. The node stays disabled; the SEP driver enables it when it brings up the sensor. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 --- arch/arm64/boot/dts/apple/t8112.dtsi | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 76a9fa23709d0..c98d3ce2670d3 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -888,6 +888,20 @@ status = "disabled"; }; + spi2: spi@235108000 { + compatible = "apple,t8112-spi", "apple,spi"; + reg = <0x2 0x35108000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clkref>; + pinctrl-0 = <&spi2_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi2>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + spi3: spi@23510c000 { compatible = "apple,t8112-spi", "apple,spi"; reg = <0x2 0x3510c000 0x0 0x4000>; @@ -1297,6 +1311,13 @@ ; }; + spi2_pins: spi2-pins { + /* Three signal pads; SPI2 uses the controller's native chip select. */ + pinmux = , + , + ; + }; + spi3_pins: spi3-pins { pinmux = , , From 067d84340442bdbb8dc199acb4191452fa076c9d Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 13:42:47 -0400 Subject: [PATCH 0043/1579] dt-bindings: apple: add mesa-fingerprint and dart dma-range Add a binding for the Mesa fingerprint sensor on the SPI2 bus of Apple laptops, driven by the Secure Enclave through a /dev/sep-bio character device. The sensor uses the SPI controller's native chip select; a board describes its power line (enable-gpios), an optional data-ready line (interrupts), and the per-device calibration firmware name. Also document the apple,dma-range DART property, used to place the SEP firmware mapping on the M1 boot path. Signed-off-by: Chromatischer [dj: adapted for aurora feat/sep, clean-room] Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 --- .../input/apple,mesa-fingerprint.yaml | 77 +++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 78 insertions(+) create mode 100644 Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml diff --git a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml new file mode 100644 index 0000000000000..be9bbcf473e85 --- /dev/null +++ b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/apple,mesa-fingerprint.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Mesa fingerprint sensor + +maintainers: + - Chromatischer + +description: | + Fingerprint sensor on an SPI bus of an Apple SoC, normally owned by the + Secure Enclave Processor. The chip-select setup and hold delays have to be + produced by the controller, so the device must use the native chip select of + its parent controller and not a GPIO one. + + The SPI mode is not given a default here because the boards that carry this + sensor have not been observed to agree on one, so each board description has + to state the mode it needs. + +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +properties: + compatible: + const: apple,mesa-fingerprint + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + description: Data-ready line from the sensor. + + enable-gpios: + maxItems: 1 + description: Power line to the sensor, asserted to power it up. + + firmware-name: + maxItems: 1 + description: + Calibration blob for this sensor. It is per-device factory data written + at manufacture, so it is neither redistributable nor derivable from the + compatible; the board description has to name it. + + spi-cpha: true + + spi-cpol: true + +required: + - compatible + - reg + - enable-gpios + - firmware-name + +unevaluatedProperties: false + +examples: + - | + #include + + spi { + #address-cells = <1>; + #size-cells = <0>; + + fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpha; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + enable-gpios = <&pinctrl_ap 108 GPIO_ACTIVE_HIGH>; + firmware-name = "apple/mesacal-j313.bin"; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index 368bf2b43af1c..901897d0b3318 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2562,6 +2562,7 @@ F: Documentation/devicetree/bindings/gpio/apple,smc-gpio.yaml F: Documentation/devicetree/bindings/gpu/apple,agx.yaml F: Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml F: Documentation/devicetree/bindings/i2c/apple,i2c.yaml +F: Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml F: Documentation/devicetree/bindings/input/touchscreen/apple,z2-multitouch.yaml F: Documentation/devicetree/bindings/interrupt-controller/apple,* F: Documentation/devicetree/bindings/iommu/apple,dart.yaml From 1b8aa993e0201d4ca2b385d30c4f7c6c3421b3d7 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 13:42:48 -0400 Subject: [PATCH 0044/1579] arm64: dts: apple: t8103: bring up the Mesa fingerprint sensor Describe the SPI2 controller and the Mesa fingerprint sensor on T8103, and enable them on the MacBook Air (M1, J313). The controller sits at the SPI cluster base + 0x8000 (reg 0x235108000, AIC IRQ 616, ps_spi2 power domain), mirroring the sibling SPI nodes. Its pin group is the three signal pads (128 CLK, 129 MOSI, 130 MISO), read back from live hardware as peripheral function 1; SPI2 has no chip-select pad and uses the controller's native chip select. The sensor's power line is GPIO 108 (active high); the data-ready line is left out because the driver polls. Signed-off-by: Chromatischer [dj: adapted for aurora feat/sep, clean-room] Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 --- arch/arm64/boot/dts/apple/t8103-j313.dts | 28 +++++++++++++ arch/arm64/boot/dts/apple/t8103.dtsi | 51 ++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts index f5b7bb4418d0d..acd24294dedd3 100644 --- a/arch/arm64/boot/dts/apple/t8103-j313.dts +++ b/arch/arm64/boot/dts/apple/t8103-j313.dts @@ -94,6 +94,34 @@ }; }; +/* + * Mesa fingerprint sensor on SPI2. The power line (pin 108, active high) comes + * from the J313 platform device tree, which also lists an interrupt on pin 104. The + * interrupt is left out on purpose: the driver polls, and the trigger flags in + * the ADT (3) are not decoded with any certainty here. Pin 104 reads back as a + * plain input with a pull-down, which rules out the two active-low readings of + * that value but does not decide between the remaining ones. + * + * Do not add cs-gpios. The sensor needs the controller's native chip select, + * otherwise the SPI core skips the hardware CS timing. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 108 GPIO_ACTIVE_HIGH>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + /* + * Per-device factory data written at manufacture. It is not + * redistributable and has to be extracted from the machine's own macOS + * install, so the name only fixes where the driver looks for it. + */ + firmware-name = "apple/mesacal-j313.bin"; +}; + &i2c1 { speaker_left: codec@31 { compatible = "ti,tas5770l", "ti,tas2770"; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 4682da893992c..1b7fa85b12398 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -861,6 +861,51 @@ status = "disabled"; }; + /* + * The J313 platform device tree puts the Mesa fingerprint sensor + * on SPI2. The 120 MHz parent was confirmed by timing transfers: + * a 22158-byte transfer requested at 8 MHz, divider 15, ran at + * 7.45 Mbit/s. A 200 MHz parent would make that divider + * 13.3 MHz, and the measurement could not then have come out + * below 8 MHz. + * + * The pin group was read out of the hardware rather than taken + * from a schematic. iBoot leaves every SPI pad it uses at + * peripheral function 1, and the three buses whose groups are + * already described here have the same register signature: two + * pads with no pull for CLK and MOSI, one with a pull-down for + * MISO. Pins 128, 129 and 130 are the only remaining pads with + * that signature, and they are contiguous, so they are SPI2. + * Pin 131 is a different pad type and is not part of the group. + * + * There is no chip-select pin. The platform device tree gives + * SPI1, SPI3 and SPI4 a GPIO chip select (pins 45, 49 and 24, each the + * last pad of its group) and gives SPI2 the string "null", so + * SPI2 uses the controller's own chip select. That is also why + * SPI0, which has no GPIO chip select either, lists three pins. + */ + spi2: spi@235108000 { + compatible = "apple,t8103-spi", "apple,spi"; + reg = <0x2 0x35108000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clk_120m>; + pinctrl-0 = <&spi2_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi2>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + mesa: fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpha; + status = "disabled"; + }; + }; + spi3: spi@23510c000 { compatible = "apple,t8103-spi", "apple,spi"; reg = <0x2 0x3510c000 0x0 0x4000>; @@ -1126,6 +1171,12 @@ ; }; + spi2_pins: spi2-pins { + pinmux = , /* CLK */ + , /* MOSI */ + ; /* MISO */ + }; + spi3_pins: spi3-pins { pinmux = , , From eaafb2dc38118ce5859b801c3831ecc3b67a80d6 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 14:58:52 -0400 Subject: [PATCH 0045/1579] soc: apple: sep: add typed platform profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Name every SoC-specific fact the driver needs — the shared-memory capacity and first-item fourcc, the boot handshake, the OS-identity source and the sensor transport — in one typed profile, and refuse an unsupported SoC rather than run a handshake with the wrong geometry. Two targets are modelled: T8103/J313 (M1) boots the SEP with the boot endpoint handshake over a 0x30000 window whose first item is CNIP; T6020/J414s (M2 Pro) does the warm registration over a 0x40000 window whose first item is CINP. The shared-memory build takes the capacity and first-item spelling from the profile; T6020's values are unchanged. Signed-off-by: Chromatischer [dj: adapted for aurora feat/sep - dropped the storage-backend enum (single in-kernel owner), identity is /chosen only, kept the driver modular; clean-room] Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 --- drivers/soc/apple/profile.rs | 191 +++++++++++++++++++++++++++++++++++ drivers/soc/apple/sep.rs | 8 +- drivers/soc/apple/shmem.rs | 28 ++--- 3 files changed, 214 insertions(+), 13 deletions(-) create mode 100644 drivers/soc/apple/profile.rs diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs new file mode 100644 index 0000000000000..7d7bee3f0208a --- /dev/null +++ b/drivers/soc/apple/profile.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Typed platform profiles for the SEP driver. +//! +//! Every SoC-specific fact the driver needs — the shared-memory capacity, the +//! boot handshake, the OS-identity source and the sensor transport — is named +//! here once. The driver reads addresses from the device tree, never scans for +//! them, and never patches properties at runtime. +//! +//! Two bring-up targets are modelled: +//! +//! * `T8103` / J313 (MacBook Air, M1): the host boots the SEP with the boot +//! endpoint handshake over a 0x30000 shared-memory window. +//! * `T6020` / J414s (MacBook Pro 14", M2 Pro): the driver does the warm +//! single-message registration over a 0x40000 window. + +// The boot handshake, identity source, sensor and DART fields are consumed by +// the boot-endpoint, identity and sensor-transport paths. +#![allow(dead_code)] + +use kernel::of; +use kernel::prelude::*; + +/// How the driver brings the shared-memory table to the SEP. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Bootstrap { + /// Cold boot: send TZ0, map the firmware reserved region and answer the + /// second boot acknowledgement with the firmware address and `SET_SHMEM`. + Boot, + /// Warm attach: send the single shared-memory registration message with the + /// table address and size. + WarmRegister, +} + +/// Where the 16-byte OS identity comes from. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum IdentitySource { + /// `/chosen/apfs-preboot-uuid`, the value the firmware handed to this boot + /// and the boot chain forwards. A malformed or absent property is refused; + /// the identity is never invented. + Chosen, + /// A UUID provisioned in the Linux host-state store on first bring-up. + HostPersisted, +} + +/// SPI mode as the device tree spells it. Mode 1 is CPOL=0/CPHA=1 (`spi-cpha`); +/// mode 2 is CPOL=1/CPHA=0 (`spi-cpol`). +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum SpiMode { + Mode1, + Mode2, +} + +impl SpiMode { + /// The value the SPI shim programs into `spi_setup()`, using the kernel's + /// `SPI_CPHA`/`SPI_CPOL` bits. + pub(crate) const fn wire(self) -> u32 { + match self { + SpiMode::Mode1 => 1, // SPI_CPHA + SpiMode::Mode2 => 2, // SPI_CPOL + } + } + + pub(crate) const fn name(self) -> &'static CStr { + match self { + SpiMode::Mode1 => c"mode 1 (CPOL=0 CPHA=1)", + SpiMode::Mode2 => c"mode 2 (CPOL=1 CPHA=0)", + } + } +} + +pub(crate) struct SensorProfile { + /// SPI controller register base. Diagnostics only: the device tree is + /// authoritative and the driver never enables or creates the bus. + pub(crate) controller_base: u64, + pub(crate) chip_select: u32, + pub(crate) max_hz: u32, + /// Chip-select setup and hold in nanoseconds. + pub(crate) cs_setup_ns: u32, + pub(crate) cs_hold_ns: u32, + pub(crate) mode: SpiMode, + pub(crate) expected_id: u16, + /// Capture traffic stays disabled until a qualified DMA transport exists; + /// the PIO path is status-only. + pub(crate) capture_qualified: bool, +} + +pub(crate) struct PlatformProfile { + pub(crate) name: &'static str, + /// Capacity of the boot shared-memory window. The layout is still checked + /// against the manifests; this only bounds the allocation. + pub(crate) shmem_capacity: usize, + /// Fourcc of the first shared-memory item. The two paths spell it + /// differently, so each profile carries its own spelling. + pub(crate) shmem_first_item: &'static [u8; 4], + pub(crate) bootstrap: Bootstrap, + pub(crate) identity: IdentitySource, + pub(crate) sensor: SensorProfile, + /// Require a static `apple,dma-range` on the SEP DART. The T6020 SEP only + /// accepts IOVAs below 4 GiB; T8103 works with the stock DART aperture. + pub(crate) dart_range_required: bool, + /// Reserved-memory region holding the SEP firmware image (cold-boot path). + pub(crate) firmware_region: &'static CStr, +} + +const T8103: PlatformProfile = PlatformProfile { + name: "T8103/J313", + shmem_capacity: 0x3_0000, + shmem_first_item: b"CNIP", + bootstrap: Bootstrap::Boot, + identity: IdentitySource::Chosen, + sensor: SensorProfile { + controller_base: 0x2_3510_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode1, + expected_id: 0x3352, + capture_qualified: false, + }, + dart_range_required: false, + firmware_region: c"sepfw", +}; + +const T6020: PlatformProfile = PlatformProfile { + name: "T6020/J414s", + shmem_capacity: 0x4_0000, + shmem_first_item: b"CINP", + bootstrap: Bootstrap::WarmRegister, + identity: IdentitySource::HostPersisted, + sensor: SensorProfile { + controller_base: 0x3_9b10_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode2, + expected_id: 0x3352, + capture_qualified: false, + }, + dart_range_required: true, + firmware_region: c"sepfw", +}; + +static_assert!(T8103.shmem_capacity == 0x30000); +static_assert!(T6020.shmem_capacity == 0x40000); +static_assert!(T8103.sensor.controller_base == 0x235108000); +static_assert!(T6020.sensor.controller_base == 0x39b108000); +static_assert!(T8103.sensor.expected_id == T6020.sensor.expected_id); + +/// Whether the machine root declares `compatible`. +/// +/// The root property is a NUL-separated list, so whole entries are compared +/// instead of substrings: `apple,t8103` cannot match a longer unrelated value. +fn machine_has(compatible: &[u8]) -> bool { + let Some(root) = of::root() else { + return false; + }; + let Ok(list) = root.get_property::>(c"compatible") else { + return false; + }; + + let mut at = 0usize; + while at < list.len() { + let end = match list[at..].iter().position(|&byte| byte == 0) { + Some(offset) => at + offset, + None => list.len(), + }; + if &list[at..end] == compatible { + return true; + } + at = end + 1; + } + + false +} + +/// Select the profile for the running machine. An unsupported SoC is refused +/// rather than guessed at, so the driver cannot run a handshake with the wrong +/// geometry. +pub(crate) fn detect() -> Result<&'static PlatformProfile> { + if machine_has(b"apple,t8103") { + return Ok(&T8103); + } + if machine_has(b"apple,t6020") { + return Ok(&T6020); + } + Err(ENODEV) +} diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index e028ff10ac6c6..fb2e7a02ab3a8 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -17,6 +17,7 @@ mod fv; mod hwrng; mod image; mod keybag; +mod profile; mod proto; mod refkey; mod refkey_seal; @@ -513,6 +514,8 @@ struct MachineRefKey { struct SepData { dev: ARef, + profile: &'static profile::PlatformProfile, + #[pin] mbox: Mutex>>, @@ -668,8 +671,10 @@ unsafe impl Sync for SepData {} impl SepData { fn new(pdev: &platform::Device) -> Result> { - let built = shmem::build(pdev)?; + let profile = profile::detect()?; + let built = shmem::build(pdev, profile.shmem_capacity, profile.shmem_first_item)?; let dev: &device::Device = pdev.as_ref(); + dev_info!(dev, "SEP platform profile: {}\n", profile.name); let buf = built.buf; @@ -741,6 +746,7 @@ impl SepData { Arc::pin_init( try_pin_init!(SepData { dev: ARef::::from(dev), + profile, mbox <- new_mutex!(None), shmem <- new_mutex!(Some(buf)), endpoints <- new_mutex!(EndpointTable::new()), diff --git a/drivers/soc/apple/shmem.rs b/drivers/soc/apple/shmem.rs index 134585398c8b4..0c930939d3a0f 100644 --- a/drivers/soc/apple/shmem.rs +++ b/drivers/soc/apple/shmem.rs @@ -6,8 +6,6 @@ use kernel::dma; use kernel::platform; use kernel::prelude::*; -pub(crate) const SHMEM_SIZE: usize = 0x40000; - const ENTRY_SIZE: usize = 16; const ENTRY_OFF_FOURCC: usize = 0; const ENTRY_OFF_SIZE: usize = 4; @@ -21,8 +19,8 @@ const CINP_MIN_SIZE: usize = 0x8000; const CINP_PAYLOAD: [u8; 1] = [0]; -// Wire byte order, not byte-reversed; llun is the terminator spelling. -const FOURCC_CINP: &[u8; 4] = b"CINP"; +// Wire byte order, not byte-reversed; llun is the terminator spelling. The +// first item's fourcc is per-SoC and comes from the platform profile. const FOURCC_OPLA: &[u8; 4] = b"OPLA"; const FOURCC_IPIS: &[u8; 4] = b"IPIS"; const FOURCC_TERM: &[u8; 4] = b"llun"; @@ -68,7 +66,7 @@ const fn align_up(v: usize, a: usize) -> usize { fn write_at(buf: &mut ShMem, off: usize, src: &[u8]) -> Result<()> { let end = off.checked_add(src.len()).ok_or(EINVAL)?; - if end > SHMEM_SIZE { + if end > buf.len() { return Err(EINVAL); } // SAFETY: runs in probe before the SEP is told the buffer exists, and probe @@ -131,9 +129,11 @@ fn verify_layout( opla: &Region, ipis: &Region, used: usize, + capacity: usize, + first_item: &[u8; 4], ) -> Result<()> { let regions = [ - (FOURCC_CINP, cinp), + (first_item, cinp), (FOURCC_OPLA, opla), (FOURCC_IPIS, ipis), ]; @@ -143,7 +143,7 @@ fn verify_layout( || r.offset % PAYLOAD_ALIGN != 0 || r.size % PAYLOAD_ALIGN != 0 || r.size < r.payload_len + 4 - || r.end() > SHMEM_SIZE; + || r.end() > capacity; if bad { dev_err!( dev, @@ -181,14 +181,18 @@ fn verify_layout( return Err(EINVAL); } - if 4 * ENTRY_SIZE > PAYLOAD_BASE || used > SHMEM_SIZE { + if 4 * ENTRY_SIZE > PAYLOAD_BASE || used > capacity { return Err(ENOSPC); } Ok(()) } -pub(crate) fn build(pdev: &platform::Device) -> Result { +pub(crate) fn build( + pdev: &platform::Device, + capacity: usize, + first_item: &[u8; 4], +) -> Result { let dev: &device::Device = pdev.as_ref(); // Read manifests before allocating: a CINP-only registration would burn the one-shot and fault. @@ -200,9 +204,9 @@ pub(crate) fn build(pdev: &platform::Device) -> Result { let ipis = Region::place(opla.end(), ipis_blob.len(), 0); let used = ipis.end(); - verify_layout(dev, &cinp, &opla, &ipis, used)?; + verify_layout(dev, &cinp, &opla, &ipis, used, capacity, first_item)?; - let mut buf = dma::Coherent::::zeroed_slice(dev, SHMEM_SIZE, GFP_KERNEL)?; + let mut buf = dma::Coherent::::zeroed_slice(dev, capacity, GFP_KERNEL)?; // Payloads before entries: a failure leaves an all-zero table, not a valid-looking one. write_at( @@ -226,7 +230,7 @@ pub(crate) fn build(pdev: &platform::Device) -> Result { )?; write_at(&mut buf, ipis.offset + 4, &ipis_blob)?; - write_entry(&mut buf, 0, FOURCC_CINP, cinp.size, cinp.offset)?; + write_entry(&mut buf, 0, first_item, cinp.size, cinp.offset)?; write_entry(&mut buf, 1, FOURCC_OPLA, opla.size, opla.offset)?; write_entry(&mut buf, 2, FOURCC_IPIS, ipis.size, ipis.offset)?; write_entry(&mut buf, 3, FOURCC_TERM, 0, 0)?; From c62530ce1a74a7648cf4811f4b3dda46dc183fad Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 15:08:33 -0400 Subject: [PATCH 0046/1579] soc: apple: sep: boot the SEP from the boot endpoint The T8103 SEP is not brought up by the warm registration message. On that target, send TZ0 on the boot endpoint and let its acknowledgements drive the handoff: map the firmware reserved region, submit the firmware address (IMG4) and the shared-memory table (SET_SHMEM), and mark the registration complete on the IMG4 acknowledgement. The warm target keeps the single registration message; the profile selects which path runs, and only the warm target enables the SEP and its DART with a runtime changeset - the cold-boot target's DART is already enabled before probe and is described statically. The firmware mapping is created once and released exactly once in remove(), gated on the teardown flag so a boot message racing removal cannot map firmware or hand memory to the peer after teardown. Signed-off-by: Chromatischer [dj: ported onto aurora feat/sep - reused the existing teardown flag and warm attach path, profile-gated the runtime DT enable, clean-room] Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 --- drivers/soc/apple/dt.rs | 36 ++++++++ drivers/soc/apple/proto.rs | 9 ++ drivers/soc/apple/sep.rs | 173 +++++++++++++++++++++++++++++++++++-- 3 files changed, 209 insertions(+), 9 deletions(-) diff --git a/drivers/soc/apple/dt.rs b/drivers/soc/apple/dt.rs index 148e8268b2950..46dd70d15d03f 100644 --- a/drivers/soc/apple/dt.rs +++ b/drivers/soc/apple/dt.rs @@ -469,3 +469,39 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { Ok(true) }) } + +/// Resolve a reserved-memory region by name on the SEP node's device. +/// +/// The cold-boot path needs this: the firmware image is handed over in a +/// bootloader-injected `sepfw` region. +pub(crate) fn reserved_region(dev: &kernel::device::Device, name: &CStr) -> Result<(u64, usize)> { + // SAFETY: `dev` is valid and its `of_node` is either NULL or a live node + // the device holds a reference to. + let np = unsafe { (*dev.as_raw()).of_node }; + if np.is_null() { + return Err(ENODEV); + } + + // SAFETY: `struct resource` is plain integers with no pointer state, so an + // all-zero value is valid. + let mut res: bindings::resource = unsafe { core::mem::zeroed() }; + // SAFETY: `np` is a live node, `name` is NUL-terminated and `res` is a + // valid out-parameter. + let ret = unsafe { + bindings::of_reserved_mem_region_to_resource_byname(np, name.as_char_ptr(), &mut res) + }; + to_result(ret).map_err(|e| { + pr_err!( + "apple_sep: reserved-memory region '{}' is unavailable ({:?})\n", + name, + e + ); + e + })?; + + let size = res.end.wrapping_sub(res.start).wrapping_add(1); + if size == 0 || size > usize::MAX as u64 { + return Err(EINVAL); + } + Ok((res.start, size as usize)) +} diff --git a/drivers/soc/apple/proto.rs b/drivers/soc/apple/proto.rs index 395f7440b10bc..de2bf4580d84a 100644 --- a/drivers/soc/apple/proto.rs +++ b/drivers/soc/apple/proto.rs @@ -21,6 +21,15 @@ pub(crate) const MSG_TYPE_SHIFT: u32 = 16; pub(crate) const MSG_PARAM_SHIFT: u32 = 24; pub(crate) const MSG_DATA_SHIFT: u32 = 32; +// Boot-endpoint (EP_BOOT) message types for the cold-boot handshake: send TZ0, +// then hand over the firmware image (IMG4) and the shared-memory table. +pub(crate) const MSG_BOOT_TZ0: u64 = 0x5; +pub(crate) const MSG_BOOT_IMG4: u64 = 0x6; +pub(crate) const MSG_SET_SHMEM: u64 = 0x18; +pub(crate) const MSG_BOOT_TZ0_ACK1: u64 = 0x69; +pub(crate) const MSG_BOOT_TZ0_ACK2: u64 = 0xd2; +pub(crate) const MSG_BOOT_IMG4_ACK: u64 = 0x6a; + // 4 KiB units even though CPU pages are 16 KiB pub(crate) const IOVA_SHIFT: u32 = 12; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index fb2e7a02ab3a8..ba82dd5f0d911 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -35,6 +35,7 @@ mod xarm; mod xart_store; use kernel::{ + bindings, device, dma, driver, @@ -510,12 +511,21 @@ struct MachineRefKey { pub_raw: KVec, } +/// One mapped firmware reserved region, unmapped exactly once in `remove()`. +struct FirmwareMap { + size: usize, + iova: u64, +} + #[pin_data] struct SepData { dev: ARef, profile: &'static profile::PlatformProfile, + #[pin] + fw_map: Mutex>, + #[pin] mbox: Mutex>>, @@ -747,6 +757,7 @@ impl SepData { try_pin_init!(SepData { dev: ARef::::from(dev), profile, + fw_map <- new_mutex!(None), mbox <- new_mutex!(None), shmem <- new_mutex!(Some(buf)), endpoints <- new_mutex!(EndpointTable::new()), @@ -807,7 +818,17 @@ impl SepData { ) } + /// Bring the shared-memory table to the SEP using the profile's bootstrap. fn attach(&self, sep_node: &dt::DtNode) -> Result<()> { + match self.profile.bootstrap { + profile::Bootstrap::WarmRegister => self.attach_warm(sep_node), + profile::Bootstrap::Boot => self.attach_m1(), + } + } + + /// Warm attach: the firmware left a running SEP, so one registration message + /// hands it the shared-memory table. + fn attach_warm(&self, sep_node: &dt::DtNode) -> Result<()> { let (iova, size) = { let guard = self.shmem.lock(); let buf = guard.as_ref().ok_or(EINVAL)?; @@ -837,6 +858,137 @@ impl SepData { Ok(()) } + /// Cold boot: send TZ0 and let the boot-endpoint acknowledgements drive the + /// firmware and shared-memory handoff. + fn attach_m1(&self) -> Result<()> { + let msg = Message { + msg0: u64::from(proto::EP_BOOT) | (proto::MSG_BOOT_TZ0 << proto::MSG_TYPE_SHIFT), + msg1: 0, + }; + self.send(msg) + } + + /// Handle a message from the boot endpoint during the cold-boot handshake. + fn on_boot(&self, msg: Message) { + let ty = (msg.msg0 >> proto::MSG_TYPE_SHIFT) & 0xff; + match ty { + proto::MSG_BOOT_TZ0_ACK1 => { + dev_info!(self.dev, "boot: first TZ0 acknowledgement\n"); + } + proto::MSG_BOOT_TZ0_ACK2 => { + dev_info!( + self.dev, + "boot: TZ0 accepted; handing over firmware and shared memory\n" + ); + if let Err(e) = self.load_firmware_and_shmem() { + dev_err!( + self.dev, + "boot: firmware/shared-memory handoff failed ({:?})\n", + e + ); + } + } + proto::MSG_BOOT_IMG4_ACK => { + dev_info!( + self.dev, + "boot: IMG4 acknowledged; the SEP owns the shared-memory table\n" + ); + self.registered.store(true, Relaxed); + } + _ => { + dev_warn!( + self.dev, + "boot: unknown message type {} (msg0 {:#018x})\n", + ty, + msg.msg0 + ); + } + } + } + + /// Map the firmware reserved region once and hand the SEP the firmware + /// address (IMG4) and the shared-memory table (SET_SHMEM). + fn load_firmware_and_shmem(&self) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + + let (phys, size) = dt::reserved_region(&self.dev, self.profile.firmware_region)?; + + // Hold the mapping lock across the teardown re-check, the map and the + // store: `remove()` takes the same lock, so a mapping either exists + // before removal and is unmapped by it, or is refused here. + let mut fw_map = self.fw_map.lock(); + if fw_map.is_some() { + // One-shot per boot: a second acknowledgement cannot remap it. + return Ok(()); + } + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + // SAFETY: `self.dev` is live; the reserved region is owned by the + // firmware handoff and the mapping is retained until `remove()`. + let iova = unsafe { + let mapped = bindings::dma_map_resource( + self.dev.as_raw(), + phys, + size, + bindings::dma_data_direction_DMA_TO_DEVICE, + 0, + ); + if bindings::dma_mapping_error(self.dev.as_raw(), mapped) != 0 { + return Err(ENOMEM); + } + mapped + }; + *fw_map = Some(FirmwareMap { size, iova }); + drop(fw_map); + + let msg = Message { + msg0: u64::from(proto::EP_BOOT) + | (proto::MSG_BOOT_IMG4 << proto::MSG_TYPE_SHIFT) + | ((iova >> proto::IOVA_SHIFT) << proto::MSG_DATA_SHIFT), + msg1: 0, + }; + self.send(msg)?; + + let shm = { + let guard = self.shmem.lock(); + let buf = guard.as_ref().ok_or(EINVAL)?; + buf.dma_handle() + }; + let msg = Message { + msg0: u64::from(proto::EP_SHMEM) + | (proto::MSG_SET_SHMEM << proto::MSG_TYPE_SHIFT) + | ((shm >> proto::IOVA_SHIFT) << proto::MSG_DATA_SHIFT), + msg1: 0, + }; + // The peer may know the address even if this send fails ambiguously, so + // the buffer stays retained: `remove()` must never free shared memory + // the SEP was handed. + self.registered.store(true, Relaxed); + self.send(msg)?; + Ok(()) + } + + /// Release the firmware mapping if one is live. Called by `remove()` so a + /// mapping created by a racing boot message cannot outlive the driver. + fn release_firmware_mapping(&self) { + if let Some(mapped) = self.fw_map.lock().take() { + // SAFETY: created by `dma_map_resource` with these exact parameters + // and unmapped exactly once. + unsafe { + bindings::dma_unmap_resource( + self.dev.as_raw(), + mapped.iova, + mapped.size, + bindings::dma_data_direction_DMA_TO_DEVICE, + 0, + ); + } + } + } + fn send(&self, msg: Message) -> Result<()> { self.mbox.lock().as_ref().ok_or(ENODEV)?.send(msg, false) } @@ -1667,14 +1819,7 @@ impl SepData { proto::EP_SCRD => self.on_scrd(msg), - proto::EP_BOOT => dev_warn!( - self.dev, - "unexpected message from boot endpoint 0xff: type 0x{:02x} param 0x{:02x} msg0 {:#018x} msg1 {:#010x}\n", - f.ty, - f.param, - msg.msg0, - msg.msg1 - ), + proto::EP_BOOT => self.on_boot(msg), _ep => {}, } @@ -1710,6 +1855,7 @@ impl SepData { fn remove(&self) { self.shutting_down.store(true, Relaxed); + self.release_firmware_mapping(); trusted::unregister(); self.unregister_fv_kernel(); @@ -1986,7 +2132,16 @@ struct SepModule { impl kernel::InPlaceModule for SepModule { fn init(module: &'static ThisModule) -> impl PinInit { try_pin_init!(Self { - _dt: dt::enable_sep_and_dart()?, + // The warm-attach target enables the SEP and its DART with a runtime + // changeset; the cold-boot target describes them statically (its DART + // is enabled before probe), so no changeset runs there. + _dt: { + if let Ok(p) = profile::detect() { + if matches!(p.bootstrap, profile::Bootstrap::WarmRegister) { + dt::enable_sep_and_dart()?; + } + } + }, _driver <- driver::Registration::new( ::NAME, From 146e71d26be27c3ed01cba1e688dae01f03e139e Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 15:15:44 -0400 Subject: [PATCH 0047/1579] soc: apple: sep: take the calibration blob's name from the device tree The Mesa calibration blob is per-device factory data, so its name is board data, not a build constant. Read it from the sensor node's firmware-name property, falling back to the built-in default when the property is absent so the runtime-fabricated node keeps working. Signed-off-by: Chromatischer [dj: kept the default fallback so the fabricated j414s node still loads; clean-room] Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 --- drivers/soc/apple/sbio.rs | 7 +++++-- drivers/soc/apple/sensor.rs | 14 ++++++++++++++ drivers/soc/apple/sensor_shim.c | 16 ++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index ed23c237c531d..16a8312a56137 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1217,13 +1217,16 @@ impl SepData { } fn calibrate_sensor(&self) -> bool { - let blob = match kernel::firmware::Firmware::request(CALIBRATION_FIRMWARE, &self.dev) { + // The blob is per-device factory data, so its name is board data: take + // it from the sensor node's firmware-name, falling back to the default. + let name = sensor::firmware_name().unwrap_or(CALIBRATION_FIRMWARE); + let blob = match kernel::firmware::Firmware::request(name, &self.dev) { Ok(fw) => CalibrationBlob::new(fw), Err(e) => { dev_err!( self.dev, "sensor: calibration blob {} could not be loaded ({:?}); per-device factory data, cannot be synthesised, stopping\n", - CALIBRATION_FIRMWARE, + name, e ); return false; diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index d94cc00a1cff2..66ae99800a40d 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -19,6 +19,20 @@ extern "C" { fn sep_sensor_xfer(tx: *const c_void, rx: *mut c_void, len: usize) -> c_int; fn sep_sensor_xfer_tx(tx: *const c_void, len: usize) -> c_int; fn sep_sensor_xfer2(tx: *const c_void, tx_len: usize, rx: *mut c_void, rx_len: usize) -> c_int; + fn sep_sensor_firmware_name() -> *const c_char; +} + +/// The calibration blob's name from the board description, or `None` when the +/// sensor node has no `firmware-name` property (the caller then uses a default). +pub(crate) fn firmware_name() -> Option<&'static CStr> { + // SAFETY: the shim returns NULL or a pointer to a NUL-terminated string + // owned by the device property and valid for the device's lifetime. + let ptr = unsafe { sep_sensor_firmware_name() }; + if ptr.is_null() { + return None; + } + // SAFETY: a non-NULL return is a valid NUL-terminated C string. + Some(unsafe { CStr::from_char_ptr(ptr) }) } // Spi2. diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index 924433b094a64..22623213f48c3 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -322,6 +322,22 @@ int sep_sensor_power_line(void) return desc_to_gpio(sep_power); } +/* + * The per-device calibration blob name, as the board description gives it in + * the sensor node's "firmware-name" property. NULL when absent, so the caller + * can fall back to a default. + */ +const char *sep_sensor_firmware_name(void) +{ + const char *name = NULL; + + if (!sep_spi) + return NULL; + if (device_property_read_string(&sep_spi->dev, "firmware-name", &name)) + return NULL; + return name; +} + /* Powers the sensor on/off, holding the hardware settling delay. */ int sep_sensor_power(int on) { From fe8049692505eb27db5d0df1fbe4a3a68c61f3c3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 16 Sep 2026 18:36:41 -0400 Subject: [PATCH 0048/1579] soc: apple: sep: report key-store endpoint readiness Log once when the key-store endpoint (0x12) comes up and its out-of-line buffers are registered, and enrich the "not advertised" diagnostic to show how far the endpoint ladder reached: the live endpoint count and which of control/xarm/sbio/scrd are present. An incomplete xART bring-up is then diagnosable from dmesg alone, without instrumenting the driver. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/sks.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 6469ebe24f5bc..c6cf39bee5e25 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -836,8 +836,13 @@ impl SepData { if !self.endpoint_present(proto::EP_SKS) { dev_err!( self.dev, - "sks: endpoint 0x{:02x} (key store) was not advertised on this boot; keybag and ref-key operations cannot run\n", - proto::EP_SKS + "sks: endpoint 0x{:02x} (key store) not advertised; {} EPs up (control={} xarm={} sbio={} scrd={}); keybag/ref-key cannot run\n", + proto::EP_SKS, + self.endpoint_count(), + self.endpoint_present(proto::EP_CONTROL) as u8, + self.endpoint_present(proto::EP_XARM) as u8, + self.endpoint_present(proto::EP_SBIO) as u8, + self.endpoint_present(proto::EP_SCRD) as u8, ); return false; } @@ -849,6 +854,12 @@ impl SepData { ); return false; } + dev_info!( + self.dev, + "sks: endpoint 0x{:02x} (key store) up; {} EPs; out-of-line buffers registered\n", + proto::EP_SKS, + self.endpoint_count() + ); true } From 9b700f1ee51ed1ad5f1cd5864543ef2a78ab6489 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 17 Sep 2026 00:39:46 -0400 Subject: [PATCH 0049/1579] soc: apple: sep: encode the identity keybag per platform profile The identity keybag CREATE_KEYBAG request carries the bag type in a word the strict T8103 enclave reads as the type and the lenient T6020 enclave does not. T6020 takes the variant in the first word (the proven encoding, hardware-verified for enrol, match and reboot); T8103 rejects that and wants the type in the third word, as macOS sends it. Carry the three field values in the platform profile so each SoC sends what its enclave requires, leaving the T6020 wire encoding byte-for-byte unchanged. Drop the now-unused variant constant, create-flags helper and first-identity handle. Verified on j414s (T6020): the identity keybag still provisions. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/profile.rs | 29 +++++++++++++++++++++++++++++ drivers/soc/apple/sks.rs | 30 +++++++++--------------------- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 7d7bee3f0208a..61faf6b88736f 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -86,6 +86,21 @@ pub(crate) struct SensorProfile { pub(crate) capture_qualified: bool, } +/// Identity keybag `CREATE_KEYBAG` field encoding. The strict T8103 enclave +/// reads the request's first word as the bag type and rejects the value the +/// lenient T6020 enclave accepts there; macOS carries the type in the third +/// word instead. Kept per-SoC so the strict path is correct without disturbing +/// the proven T6020 encoding (hardware-verified for enrol, match, and reboot). +pub(crate) struct KeybagCreate { + /// First word: request variant, echoed back in the reply. + pub(crate) variant: u32, + /// Third word: bag type. Identity is `0x20000` on the strict path; `0` on + /// T6020, which distinguishes the bag by the variant word instead. + pub(crate) bag_type: u32, + /// Fourth word: create argument / parent handle. + pub(crate) arg: i32, +} + pub(crate) struct PlatformProfile { pub(crate) name: &'static str, /// Capacity of the boot shared-memory window. The layout is still checked @@ -102,6 +117,8 @@ pub(crate) struct PlatformProfile { pub(crate) dart_range_required: bool, /// Reserved-memory region holding the SEP firmware image (cold-boot path). pub(crate) firmware_region: &'static CStr, + /// Identity keybag CREATE_KEYBAG field encoding (per-SoC; see [`KeybagCreate`]). + pub(crate) keybag_create: KeybagCreate, } const T8103: PlatformProfile = PlatformProfile { @@ -122,6 +139,12 @@ const T8103: PlatformProfile = PlatformProfile { }, dart_range_required: false, firmware_region: c"sepfw", + // Strict enclave: the type goes in the third word; the first word is 0. + keybag_create: KeybagCreate { + variant: 0, + bag_type: 0x20000, + arg: 0, + }, }; const T6020: PlatformProfile = PlatformProfile { @@ -142,6 +165,12 @@ const T6020: PlatformProfile = PlatformProfile { }, dart_range_required: true, firmware_region: c"sepfw", + // Proven encoding: the lenient enclave takes the variant in the first word. + keybag_create: KeybagCreate { + variant: 5, + bag_type: 0, + arg: -1, + }, }; static_assert!(T8103.shmem_capacity == 0x30000); diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index c6cf39bee5e25..03caee8f0767b 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -428,11 +428,16 @@ impl SepData { if proof.slot() != keybag::Slot::Identity { return Err(EINVAL); } + // The strict (T8103) and lenient (T6020) enclaves encode the identity + // bag differently; the per-SoC profile carries the correct field values + // so the proven T6020 path is unchanged while T8103 gets the bag type in + // the third word (see profile::KeybagCreate). + let enc = &self.profile.keybag_create; let mut body = image::Body::new(); - body.put_u32(crate::sks::SKS_CREATE_VARIANT_IDENTITY)?; + body.put_u32(enc.variant)?; body.put_u64(crate::sks::SKS_CLIENT_ID)?; - body.put_u32(crate::sks::CreateFlags::none().value())?; - body.put_i32(crate::sks::SpecialHandle::first_identity().value())?; + body.put_u32(enc.bag_type)?; + body.put_i32(enc.arg)?; body.put_blob(secret)?; body.put_blob(&[])?; body.put_blob(uuid)?; @@ -514,7 +519,7 @@ impl SepData { let Some((_fv_data, end)) = image::read_blob(body, 8) else { return false; }; - if variant != crate::sks::SKS_CREATE_VARIANT_IDENTITY || raw_handle < 0 || end != body.len() + if variant != self.profile.keybag_create.variant || raw_handle < 0 || end != body.len() { dev_err!( self.dev, @@ -987,21 +992,8 @@ pub(crate) fn sks_copy_keybag() -> SksOp { const OP_SKS_CREATE_KEYBAG: u8 = 0x01; pub(crate) const SKS_CREATE_NAME: &CStr = c"CREATE_KEYBAG"; -pub(crate) const SKS_CREATE_VARIANT_IDENTITY: u32 = 5; pub(crate) const SKS_IDENTITY_UUID_LEN: usize = 16; -pub(crate) struct CreateFlags(u32); - -impl CreateFlags { - pub(crate) const fn none() -> Self { - Self(0) - } - - pub(crate) const fn value(&self) -> u32 { - self.0 - } -} - const OP_SKS_LOAD_KEYBAG: u8 = 0x03; pub(crate) const SKS_LOAD_NAME: &CStr = c"LOAD_KEYBAG"; @@ -1068,10 +1060,6 @@ impl DesignateUser { pub(crate) struct SpecialHandle(i32); impl SpecialHandle { - pub(crate) const fn first_identity() -> SpecialHandle { - SpecialHandle(-1) - } - pub(crate) const fn value(&self) -> i32 { self.0 } From 0458924721a018233ca966c395061221bd60dfe1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 17 Sep 2026 00:39:47 -0400 Subject: [PATCH 0050/1579] soc: apple: sep: open the xART gigalocker by raw extent in the kernel Add an in-kernel path that opens the iBoot system container directly and serves the STORE_SIZE gigalocker window at a configured sector, so the driver reaches the shared anti-replay store on its own. It is additive and fail-closed: the xart_start_sector module parameter selects the raw extent, the logical store is always exactly STORE_SIZE so a larger backing device serves only its extent, and a wrong window still fails the existing root-record check rather than feeding SEP an unrelated store. Verified on j414s: with xart_start_sector at the gigalocker's first sector the store opens with the expected slot count, record count and revision, and the key store comes up. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/sep.rs | 7 +++- drivers/soc/apple/xart_store.rs | 70 +++++++++++++++++++++++++-------- 2 files changed, 59 insertions(+), 18 deletions(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index ba82dd5f0d911..75e2f24b46030 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -711,7 +711,8 @@ impl SepData { let dma_ring = dma::Coherent::::zeroed_slice(dev, DMA_RING_SIZE, GFP_KERNEL)?; let xart_writes = *module_parameters::xart_writes.value() != 0; - let store = match xart_store::Store::open(xart_writes) { + let xart_start = *module_parameters::xart_start_sector.value(); + let store = match xart_store::Store::open_owner(xart_writes, xart_start) { Ok(store) => { let (slots, records, revision, malformed, duplicates, repaired, writable) = store.summary(); @@ -2161,6 +2162,10 @@ module! { default: 0, description: "Allow writes to the validated shared xART mapping", }, + xart_start_sector: u64 { + default: 0, + description: "First 512-byte sector of the xART gigalocker extent within the iBoot system container. Non-zero selects the in-kernel raw-extent owner (no userspace mapper); zero uses the device-mapper mapping at /dev/mapper/sep-xart-gigalocker", + }, provision_keybag: u8 { default: 0, description: "Create the Linux identity keybag when none exists; requires xart_writes=1", diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index e255e2507f745..ea70d6dcc508d 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -13,17 +13,24 @@ use kernel::prelude::*; pub(crate) const STORE_PATH: &CStr = c"/dev/mapper/sep-xart-gigalocker"; +/// The raw-extent owner opens this whole partition (the iBoot system container, +/// where the gigalocker lives) and serves the window at the configured sector. +/// It replaces the userspace APFS parser + device-mapper mapping. +pub(crate) const OWNER_PATH: &CStr = c"/dev/disk/by-partlabel/iBootSystemContainer"; + const BLOCK_SIZE: usize = 0x1000; const SLOT_SIZE: usize = 0x9000; const HEADER_SIZE: usize = 0x22; const DELETE_SIZE: usize = BLOCK_SIZE; /// Size of the APFS raw extent located on the target machine. /// -/// Accepting a larger block device would make a bad device-mapper table a -/// corruption hazard. Accepting a smaller one could silently truncate the -/// slot grid. A future locator for a machine with a different extent size must -/// pass that size through an explicit, reviewed interface instead of weakening -/// this check. +/// The logical store is always exactly this many bytes. The device-mapper +/// mapping is exactly this size; the raw-extent owner opens a larger container +/// and serves only the [`STORE_SIZE`] window at its configured base, so a wrong +/// base cannot read past the extent. A wrong window in either mode fails closed: +/// the two root records will not validate and `open_based` returns `ENODATA` +/// rather than serving SEP an unrelated store. A machine with a different extent +/// size must pass that size through an explicit, reviewed interface. const STORE_SIZE: u64 = 0x600000; const MAX_SLOTS: usize = 4096; @@ -80,6 +87,10 @@ impl Slot { pub(crate) struct Store { file: shim::StoreFile, + /// Byte offset of the first slot within the opened block device. Zero for + /// the device-mapper mapping (which starts at the gigalocker); non-zero for + /// the raw-extent owner, which opens the whole container and points here. + base: u64, slots: KVec, revision: u64, writes_enabled: bool, @@ -112,21 +123,45 @@ fn valid_key(key: &Key) -> bool { impl Store { pub(crate) fn open(writes_enabled: bool) -> Result { - Self::open_at(STORE_PATH, writes_enabled) + Self::open_based(STORE_PATH, writes_enabled, 0) + } + + /// Opens the store, selecting the device-mapper mapping (`start_sector == 0`) + /// or the in-kernel raw-extent owner. The owner opens the whole iBoot system + /// container and treats the [`STORE_SIZE`] window at `start_sector` as the + /// gigalocker, reproducing the mapper's bytes without a userspace parser. + pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { + if start_sector == 0 { + return Self::open(writes_enabled); + } + Self::open_based(OWNER_PATH, writes_enabled, start_sector << 9) } - /// Opens a caller-selected block mapping. + /// Opens a caller-selected block mapping at offset zero. /// - /// Production always uses [`STORE_PATH`]. The separate xART self-test - /// module uses this entry point with its fixed loop-only mapper name, so it - /// can execute the real parser and write ordering without enabling SEP. + /// The separate xART self-test module uses this entry point with its fixed + /// loop-only mapper name, so it can execute the real parser and write + /// ordering without enabling SEP. + #[allow(dead_code)] pub(crate) fn open_at(path: &CStr, writes_enabled: bool) -> Result { + Self::open_based(path, writes_enabled, 0) + } + + /// Opens `path` and serves the [`STORE_SIZE`] window starting at byte + /// `base`. The window must be block-aligned and fit within the device; the + /// logical store is always exactly [`STORE_SIZE`], so a larger backing + /// device (the raw container) serves only its gigalocker extent. + fn open_based(path: &CStr, writes_enabled: bool, base: u64) -> Result { let file = shim::StoreFile::open_block(path, writes_enabled)?; let size = file.size()?; - if size != STORE_SIZE || size % BLOCK_SIZE as u64 != 0 { + if base % BLOCK_SIZE as u64 != 0 || STORE_SIZE % BLOCK_SIZE as u64 != 0 { + return Err(EINVAL); + } + let end = base.checked_add(STORE_SIZE).ok_or(EINVAL)?; + if end > size { return Err(EINVAL); } - let count = (size / SLOT_SIZE as u64) as usize; + let count = (STORE_SIZE / SLOT_SIZE as u64) as usize; if count == 0 || count > MAX_SLOTS { return Err(EINVAL); } @@ -137,6 +172,7 @@ impl Store { } let mut store = Store { file, + base, slots, revision: 0, // Discovery is always read-only. Do not arm even repair writes @@ -172,7 +208,7 @@ impl Store { for idx in 0..self.slots.len() { self.file - .read_block_exact(Self::slot_offset(idx), &mut raw)?; + .read_block_exact(self.base + Self::slot_offset(idx), &mut raw)?; let kind = raw[KEY_KIND]; if kind == 0 { continue; @@ -230,7 +266,7 @@ impl Store { continue; } self.file - .read_block_exact(Self::slot_offset(idx), &mut header)?; + .read_block_exact(self.base + Self::slot_offset(idx), &mut header)?; if header[KEY_KIND] != 0 { self.delete_slot(idx)?; self.repaired_records += 1; @@ -265,7 +301,7 @@ impl Store { let mut zero = KVec::with_capacity(DELETE_SIZE, GFP_KERNEL)?; zero.resize(DELETE_SIZE, 0, GFP_KERNEL)?; self.file - .write_block_exact(Self::slot_offset(slot), &zero)?; + .write_block_exact(self.base + Self::slot_offset(slot), &zero)?; self.file.sync() } @@ -280,7 +316,7 @@ impl Store { let mut raw = KVec::with_capacity(SLOT_SIZE, GFP_KERNEL)?; raw.resize(SLOT_SIZE, 0, GFP_KERNEL)?; self.file - .read_block_exact(Self::slot_offset(idx), &mut raw)?; + .read_block_exact(self.base + Self::slot_offset(idx), &mut raw)?; let mut uuid = [0u8; 16]; uuid.copy_from_slice(&raw[KEY_UUID..KEY_UUID + 16]); @@ -329,7 +365,7 @@ impl Store { // The new record becomes authoritative only after its complete slot is // durable. The old record is then removed and flushed separately. self.file - .write_block_exact(Self::slot_offset(fresh), &raw)?; + .write_block_exact(self.base + Self::slot_offset(fresh), &raw)?; self.file.sync()?; self.slots[fresh] = Slot { used: true, From a347068ce7ae76d027b3fd5d06604e6145f06b36 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 17 Sep 2026 17:58:49 -0400 Subject: [PATCH 0051/1579] soc: apple: sep: locate the xART gigalocker automatically Drop the last manual step from the in-kernel raw-extent owner: instead of a hand-supplied start sector, the driver finds the gigalocker itself. It reads the iBoot system container sequentially and searches, at block-aligned offsets, for a root record's signature (a 1 or 2 key kind with an all-zero UUID, the shape of Key::root). A root signature alone does not pin the base. A window shifted a few slots off the true origin still keeps both root records inside its 6 MiB extent, so it passes the two-root test yet silently drops the live records that fall outside the shift -- and, with writes on, would repair (write) at that wrong origin. The driver therefore probes every candidate base a hit implies read-only: through a writable handle, because the block layer only grants a read-only handle to a read-only device and the container is writable, but with repair disarmed so nothing is written at an unconfirmed origin. It then pins the base that recovers the most live records. Only the true origin captures the whole record set, and container noise never forges a CRC-valid record, so max live records -- ties broken to fewer malformed, then the higher base -- is exact. Only the pinned base is re-opened writable, so repair runs at the confirmed origin alone. open_based gains open_based_ext, which decouples the block-handle writability from whether repair is armed, so discovery of the writable container never mutates it. With automatic location in place, xart_start_sector becomes an explicit override (default 0 = automatic) and the driver has no external dependency for reaching the store. Boot-verified on j414s (T6020): the scan pins the true base 0x4d4000, reads all 10 live records with 0 malformed and 0 repaired (no errant write), and the key store (EP 0x12) reaches 12 endpoints. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/Kconfig | 8 +- drivers/soc/apple/sep.rs | 11 +- drivers/soc/apple/xart_store.rs | 193 ++++++++++++++++++++++++++++---- 3 files changed, 179 insertions(+), 33 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 29df757134d40..1c1527199a18c 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -121,10 +121,10 @@ config APPLE_SEP device (enrol and match), exposes the SEP hardware RNG, and registers a SEP-backed trusted key source so keyctl can seal keys to the enclave. - The driver requires the existing machine-wide xART gigalocker at - /dev/mapper/sep-xart-gigalocker. It validates the complete store and both - root records before registering with SEP. Writes remain disabled unless - the xart_writes module parameter is set and the mapping is writable. + The driver reads the existing machine-wide xART gigalocker directly from + the iBoot system container, locating it by its root records. It validates + the complete store and both root records before registering with SEP. + Writes remain disabled unless the xart_writes module parameter is set. Say Y here if you have an Apple silicon Mac. diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 75e2f24b46030..67c18dcc79c84 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -716,9 +716,11 @@ impl SepData { Ok(store) => { let (slots, records, revision, malformed, duplicates, repaired, writable) = store.summary(); + let base = store.base(); dev_info!( dev, - "xART: {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", + "xART: in-kernel raw-extent owner (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", + base, slots, records, revision, @@ -732,8 +734,7 @@ impl SepData { Err(e) => { dev_err!( dev, - "shared xART mapping '{}' is unavailable or invalid: {:?}\n", - xart_store::STORE_PATH, + "xART gigalocker in the iBoot container is unavailable or invalid: {:?}\n", e ); return Err(e); @@ -2160,11 +2161,11 @@ module! { params: { xart_writes: u8 { default: 0, - description: "Allow writes to the validated shared xART mapping", + description: "Allow writes to the validated shared xART store", }, xart_start_sector: u64 { default: 0, - description: "First 512-byte sector of the xART gigalocker extent within the iBoot system container. Non-zero selects the in-kernel raw-extent owner (no userspace mapper); zero uses the device-mapper mapping at /dev/mapper/sep-xart-gigalocker", + description: "Explicit 512-byte start sector of the xART gigalocker extent within the iBoot system container. Zero (default) locates the gigalocker automatically in the container by its root records; a non-zero value overrides the search. Either way the driver opens the container directly", }, provision_keybag: u8 { default: 0, diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index ea70d6dcc508d..ac9e0bf6c51e2 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -11,11 +11,8 @@ use crate::shim; use kernel::prelude::*; -pub(crate) const STORE_PATH: &CStr = c"/dev/mapper/sep-xart-gigalocker"; - /// The raw-extent owner opens this whole partition (the iBoot system container, -/// where the gigalocker lives) and serves the window at the configured sector. -/// It replaces the userspace APFS parser + device-mapper mapping. +/// where the gigalocker lives) and serves the gigalocker window directly. pub(crate) const OWNER_PATH: &CStr = c"/dev/disk/by-partlabel/iBootSystemContainer"; const BLOCK_SIZE: usize = 0x1000; @@ -24,16 +21,26 @@ const HEADER_SIZE: usize = 0x22; const DELETE_SIZE: usize = BLOCK_SIZE; /// Size of the APFS raw extent located on the target machine. /// -/// The logical store is always exactly this many bytes. The device-mapper -/// mapping is exactly this size; the raw-extent owner opens a larger container -/// and serves only the [`STORE_SIZE`] window at its configured base, so a wrong -/// base cannot read past the extent. A wrong window in either mode fails closed: +/// The logical store is always exactly this many bytes. The raw-extent owner +/// opens a larger container and serves only the [`STORE_SIZE`] window at its +/// located base, so a wrong base cannot read past the extent. A wrong window +/// still fails closed: /// the two root records will not validate and `open_based` returns `ENODATA` /// rather than serving SEP an unrelated store. A machine with a different extent /// size must pass that size through an explicit, reviewed interface. const STORE_SIZE: u64 = 0x600000; const MAX_SLOTS: usize = 4096; +/// Sequential read window for the automatic gigalocker search (64 KiB — small +/// enough for a reliable kernel allocation, large enough to keep the scan of +/// the container to a modest number of reads). +const SCAN_CHUNK: usize = 1 << 16; +/// Upper bound on candidate bases confirmed during the automatic search, so a +/// container full of coincidental root-shaped headers cannot spin the scan. The +/// real store's roots sit within the first slots, so it is found long before +/// this; exceeding it falls back rather than looping. +const MAX_LOCATE_ATTEMPTS: u32 = 64; + pub(crate) const MAX_VALUE: usize = 0x8000; const KEY_KIND: usize = 0x01; @@ -87,9 +94,8 @@ impl Slot { pub(crate) struct Store { file: shim::StoreFile, - /// Byte offset of the first slot within the opened block device. Zero for - /// the device-mapper mapping (which starts at the gigalocker); non-zero for - /// the raw-extent owner, which opens the whole container and points here. + /// Byte offset of the gigalocker window within the opened container: the + /// located extent's offset (an explicit start-sector override sets it). base: u64, slots: KVec, revision: u64, @@ -122,19 +128,131 @@ fn valid_key(key: &Key) -> bool { } impl Store { - pub(crate) fn open(writes_enabled: bool) -> Result { - Self::open_based(STORE_PATH, writes_enabled, 0) + /// Opens the store as the in-kernel raw-extent owner: it opens the iBoot + /// system container directly and serves only the gigalocker extent, with no + /// external helper and no hand-supplied sector. + /// + /// `start_sector == 0` (the default) *locates the gigalocker automatically* + /// inside the container by its CRC-checked root records; a non-zero + /// `start_sector` is an explicit override for the rare case the scan should + /// be skipped. + pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { + if start_sector != 0 { + return Self::open_based(OWNER_PATH, writes_enabled, start_sector << 9); + } + Self::open_located(writes_enabled) } - /// Opens the store, selecting the device-mapper mapping (`start_sector == 0`) - /// or the in-kernel raw-extent owner. The owner opens the whole iBoot system - /// container and treats the [`STORE_SIZE`] window at `start_sector` as the - /// gigalocker, reproducing the mapper's bytes without a userspace parser. - pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { - if start_sector == 0 { - return Self::open(writes_enabled); + /// Locates the gigalocker inside the iBoot system container with no external + /// help. The container is read sequentially and searched, at block-aligned + /// offsets, for a root record's signature — a `1` or `2` key kind with an + /// all-zero UUID, the shape of [`Key::root`]. Each base implied by such a hit + /// is probed read-only with the full CRC-checked [`open_based`]; because a + /// base shifted a few slots off the true origin still keeps both roots in its + /// window (passing the two-root test while dropping records that fall + /// outside), the base is pinned by the candidate that recovers the MOST live + /// records — the exact discriminator, since only the true origin captures the + /// whole set and container noise never forges a CRC-valid record. Repair is + /// never armed during discovery; only the pinned base is re-opened writable. + /// Bounded by [`MAX_LOCATE_ATTEMPTS`] probes; returns `ENODATA` if not found. + fn open_located(writes_enabled: bool) -> Result { + // The container is a writable block device; the block layer only grants + // a read-only handle to a read-only device, so scan it through a writable + // handle. This does not arm any write — discovery never mutates the + // container (see open_based_ext, called below with arm_writes == false). + let probe = shim::StoreFile::open_block(OWNER_PATH, true)?; + let size = probe.size()?; + if size < STORE_SIZE { + return Err(ENODATA); + } + + let mut buf: KVec = KVec::new(); + buf.resize(SCAN_CHUNK, 0, GFP_KERNEL)?; + let mut attempts: u32 = 0; + let mut off: u64 = 0; + + while off + HEADER_SIZE as u64 <= size { + let want = core::cmp::min(SCAN_CHUNK as u64, size - off) as usize; + probe.read_exact(off, &mut buf[..want])?; + + let mut p = 0usize; + while p + HEADER_SIZE <= want { + let kind = buf[p + KEY_KIND]; + let uuid_zero = buf[p + KEY_UUID..p + KEY_UUID + 16].iter().all(|&b| b == 0); + if (kind == 1 || kind == 2) && uuid_zero { + // A root signature anchors the gigalocker: the true base is + // this hit minus a whole number of slots (slots and blocks are + // both 0x1000-aligned, so every candidate stays block-aligned). + // The two-root test alone does NOT pin the base — a base + // shifted off the true origin by a few slots still keeps both + // roots inside its 6 MiB window, so it passes yet silently + // drops the live records that fall outside the shifted window. + // Probe every candidate READ-ONLY (writes_enabled forced false, + // so repair never fires at an unconfirmed base) and pin the one + // that recovers the MOST live records: only the true origin + // captures the whole record set, and container noise never + // forges a CRC-valid record, so max live records is exact. + let hit = off + p as u64; + let mut best: Option<(usize, usize, u64)> = None; // (valid, malformed, base) + let mut k: u64 = 0; + while k as usize <= MAX_SLOTS { + let step = k * SLOT_SIZE as u64; + if step > hit { + break; + } + let base = hit - step; + k += 1; + if base + STORE_SIZE > size { + continue; + } + attempts += 1; + if attempts > MAX_LOCATE_ATTEMPTS { + break; + } + if let Ok(store) = Self::open_based_ext(OWNER_PATH, true, false, base) { + let cand = (store.valid_records, store.malformed_records, base); + let better = match best { + None => true, + // More live records wins; ties break to fewer + // malformed, then to the HIGHER base. An up-shift + // drops the lowest occupied slots (and any root + // there, which fails the two-root test), so it + // never ties on live count; the only bases that + // can tie are down-shifts, which are all lower + // than the true origin — so the highest surviving + // candidate is the true origin. + Some(b) => { + cand.0 > b.0 + || (cand.0 == b.0 && cand.1 < b.1) + || (cand.0 == b.0 && cand.1 == b.1 && cand.2 > b.2) + } + }; + if better { + best = Some(cand); + } + } + } + // The gigalocker is unique, so the first hit that confirms any + // base has pinned it. Re-open the winner through a writable + // handle and arm repair per `writes_enabled` — repair runs + // now, and only at this confirmed origin. + if let Some((_, _, base)) = best { + return Self::open_based_ext(OWNER_PATH, true, writes_enabled, base); + } + if attempts > MAX_LOCATE_ATTEMPTS { + return Err(ENODATA); + } + } + p += BLOCK_SIZE; + } + + if want < SCAN_CHUNK { + break; + } + // Overlap one block so a signature on the boundary is not missed. + off += (SCAN_CHUNK - BLOCK_SIZE) as u64; } - Self::open_based(OWNER_PATH, writes_enabled, start_sector << 9) + Err(ENODATA) } /// Opens a caller-selected block mapping at offset zero. @@ -151,8 +269,29 @@ impl Store { /// `base`. The window must be block-aligned and fit within the device; the /// logical store is always exactly [`STORE_SIZE`], so a larger backing /// device (the raw container) serves only its gigalocker extent. + /// + /// The block handle's writability equals `writes_enabled`, and repair runs + /// when writes are enabled. For the raw-extent owner, discovery instead + /// needs a writable handle (the block layer only grants a read-only handle + /// to a read-only device, and the container is writable) *without* arming + /// repair at an unconfirmed base — see [`open_based_ext`]. fn open_based(path: &CStr, writes_enabled: bool, base: u64) -> Result { - let file = shim::StoreFile::open_block(path, writes_enabled)?; + Self::open_based_ext(path, writes_enabled, writes_enabled, base) + } + + /// As [`open_based`], but with the block-handle writability (`dev_writable`) + /// decoupled from whether repair writes are armed (`arm_writes`). Discovery + /// of the raw-extent owner opens the writable container with + /// `dev_writable == true` yet `arm_writes == false`, so a candidate base is + /// fully scanned and CRC-validated without a single write landing at an + /// origin that has not yet been confirmed as the true gigalocker. + fn open_based_ext( + path: &CStr, + dev_writable: bool, + arm_writes: bool, + base: u64, + ) -> Result { + let file = shim::StoreFile::open_block(path, dev_writable)?; let size = file.size()?; if base % BLOCK_SIZE as u64 != 0 || STORE_SIZE % BLOCK_SIZE as u64 != 0 { return Err(EINVAL); @@ -191,8 +330,8 @@ impl Store { if store.find(&Key::root(1)).is_none() || store.find(&Key::root(2)).is_none() { return Err(ENODATA); } - store.writes_enabled = writes_enabled; - if writes_enabled { + store.writes_enabled = arm_writes; + if arm_writes { store.repair_disk()?; } Ok(store) @@ -400,6 +539,12 @@ impl Store { Ok(true) } + /// Byte offset of the served gigalocker window within the opened container: + /// the located extent's offset (an explicit start-sector override sets it). + pub(crate) fn base(&self) -> u64 { + self.base + } + pub(crate) fn summary(&self) -> (usize, usize, u64, usize, usize, usize, bool) { ( self.slots.len(), From db864a4e2ceaec74c99a072bddd34fb0c9e67dab Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 17 Sep 2026 22:59:40 -0400 Subject: [PATCH 0052/1579] soc: apple: sep: wait for the key-store endpoint before bring-up The SEP endpoint ladder climbs 7 -> 11 (sbio, scrd) -> 12 (the key store, EP 0x12). tick_exchange declared the persistent-state exchange done as soon as the endpoint count passed a hardcoded pre-exchange baseline (7), but the SEP advertises the key store a beat after it falls briefly quiet following scrd. On some boots the driver therefore reached run_bringup at 11 endpoints, found EP 0x12 absent, and stranded the boot with no key store -- enable_sks fails with no retry, so keybag and ref-key operations (and thus Touch ID) are unavailable for the whole boot. Wait through quiescence until the key store itself is advertised, bounded by the same EXCHANGE_TIMEOUT_MS. A boot that already has it exits immediately, so healthy boots are unaffected; the hardcoded endpoint count, which is not portable across SoCs, is dropped in favour of the endpoint that actually matters. Boot-verified on j414s (T6020): the key store (EP 0x12) came up at 12 endpoints on 5 of 5 reboots, up from roughly 2 in 3 before the change. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/sep.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 67c18dcc79c84..fc7609cb0a8b3 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -91,8 +91,6 @@ const PHASE_ATTACH: u32 = 0; const PHASE_EXCHANGE: u32 = 1; const PHASE_READY: u32 = 2; -const ENDPOINTS_BEFORE_EXCHANGE: usize = 7; - const OOL_SIZE_XARM: usize = 0x8000; const OOL_SIZE_SBIO: usize = 0x4000; @@ -1781,8 +1779,16 @@ impl SepData { return; } - let endpoints = this.endpoint_count(); - if endpoints <= ENDPOINTS_BEFORE_EXCHANGE { + // The endpoint ladder climbs 7 -> 11 (sbio, scrd) -> 12 (the key + // store, EP 0x12). The SEP can advertise the key store a beat after it + // falls briefly quiet following scrd, so treating any count past the + // pre-exchange baseline as "done" races that last step and strands the + // boot at 11 endpoints with no key store. Wait through quiescence until + // the key store itself is advertised, bounded by the same + // EXCHANGE_TIMEOUT_MS. A boot that already has it exits immediately, so + // healthy boots are unaffected; this also drops the hardcoded endpoint + // count, which is not portable across SoCs. + if !this.endpoint_present(proto::EP_SKS) { let ticks = this.settle_idle_ticks.load(Relaxed).wrapping_add(1); this.settle_idle_ticks.store(ticks, Relaxed); if ticks.saturating_mul(u64::from(SETTLE_MS)) < u64::from(EXCHANGE_TIMEOUT_MS) { From db6089452a14abd91442fc1ec92732a144776876 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 18 Sep 2026 02:11:24 -0400 Subject: [PATCH 0053/1579] soc: apple: sep: optional interrupt-driven fingerprint capture The capture loop polled the sensor's status over SPI every 2 ms to spot a ready frame. Add an opt-in data-ready interrupt (capture_irq=1; default 0 keeps polling) that wakes the loop the instant the sensor asserts its data-ready line. The SPI status read still gates every frame, so a missed or spurious interrupt costs at most one poll interval and the poll path is untouched when the option is off. The interrupt is configured once, while the sensor is patched and idle -- the only safe moment, since reconfiguring the line mid-capture is what drops the sensor's firmware patch -- and then left alone for the driver's life. On j414s it is the pin adjacent to the power line (GPIO 121 -> IRQ 144); other boards describe it as the SPI node's interrupt. It is edge-triggered on both edges: the line idles low, so a level trigger would storm. Boot-verified on j414s (T6020): with capture_irq=1 the data-ready IRQ 144 is configured, fingerprint verify captures and matches through the interrupt, the IRQ fires a handful of times per touch (not a level storm), and the sensor keeps its firmware patch (no STATE_NEEDS_PATCH). Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/sbio.rs | 33 +++++++- drivers/soc/apple/sensor.rs | 29 +++++++ drivers/soc/apple/sensor_shim.c | 144 ++++++++++++++++++++++++++++++++ drivers/soc/apple/sep.rs | 4 + drivers/soc/apple/shim.h | 11 +++ 5 files changed, 219 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 16a8312a56137..eb98d261124f7 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -736,7 +736,21 @@ impl SepData { patch = reloaded_patch; } - self.complete_bringup(patch) + let ok = self.complete_bringup(patch); + // The sensor is patched and idle here -- the only safe moment to + // configure the data-ready interrupt, which is then left alone for the + // driver's life. Opt-in; the poll path is unchanged when off. + if ok && *module_parameters::capture_irq.value() != 0 && !sensor::irq_available() { + if sensor::irq_setup() { + dev_info!(self.dev, "sensor: interrupt-driven capture enabled\n"); + } else { + dev_warn!( + self.dev, + "sensor: data-ready interrupt unavailable; capture stays on SPI polling\n" + ); + } + } + ok } pub(crate) fn run_bringup(&self) { @@ -1651,6 +1665,14 @@ impl SepData { let mut previous: Option<[u8; sensor::STATUS_LEN]> = None; let mut armed_reported = false; + // Interrupt-driven capture only replaces the fixed inter-poll sleep with + // a data-ready wait; the SPI status read below still gates every frame, + // so a missed or spurious interrupt costs at most one poll interval. + let use_irq = *module_parameters::capture_irq.value() != 0 && sensor::irq_available(); + if use_irq { + sensor::irq_arm(); + } + for attempt in 0..ENROL_POLL_ATTEMPTS { if !bio::capture_is_live(&self.bio_session.lock()) { return CaptureWait::Abandon; @@ -1693,7 +1715,14 @@ impl SepData { return CaptureWait::Ready(count); } - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ENROL_POLL_MS))); + if use_irq { + // Wake the instant the sensor asserts data-ready, or after the + // same interval on timeout; then re-arm for the next frame. + let _ = sensor::irq_wait(ENROL_POLL_MS); + sensor::irq_arm(); + } else { + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ENROL_POLL_MS))); + } } let _ = sensor::status(); CaptureWait::Timeout diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 66ae99800a40d..6b6393dc28c04 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -19,6 +19,10 @@ extern "C" { fn sep_sensor_xfer(tx: *const c_void, rx: *mut c_void, len: usize) -> c_int; fn sep_sensor_xfer_tx(tx: *const c_void, len: usize) -> c_int; fn sep_sensor_xfer2(tx: *const c_void, tx_len: usize, rx: *mut c_void, rx_len: usize) -> c_int; + fn sep_sensor_irq_setup() -> c_int; + fn sep_sensor_irq_available() -> c_int; + fn sep_sensor_irq_arm(); + fn sep_sensor_irq_wait(timeout_ms: c_uint) -> c_int; fn sep_sensor_firmware_name() -> *const c_char; } @@ -222,6 +226,31 @@ pub(crate) fn power(on: bool) -> bool { unsafe { sep_sensor_power(if on { 1 } else { 0 }) == 0 } } +/// Sets up the data-ready interrupt once, while the sensor is idle. Returns +/// true when an interrupt is available for interrupt-driven capture. +pub(crate) fn irq_setup() -> bool { + // SAFETY: no arguments; idempotent, falls back to -errno when unavailable. + unsafe { sep_sensor_irq_setup() == 0 } +} + +pub(crate) fn irq_available() -> bool { + // SAFETY: reads one int. + unsafe { sep_sensor_irq_available() != 0 } +} + +/// Clears any stale data-ready signal before a capture. +pub(crate) fn irq_arm() { + // SAFETY: no arguments; a no-op when no IRQ is configured. + unsafe { sep_sensor_irq_arm() } +} + +/// Waits up to `timeout_ms` for the data-ready line to assert. Returns true if +/// it fired, false on timeout or when no interrupt is configured. +pub(crate) fn irq_wait(timeout_ms: c_uint) -> bool { + // SAFETY: no pointers; the shim owns the completion and the IRQ line. + unsafe { sep_sensor_irq_wait(timeout_ms) == 0 } +} + fn command(cmd: &[u8; CMD_LEN]) -> Result<()> { let mut rx = [0u8; CMD_LEN]; // SAFETY: both buffers are `CMD_LEN` bytes and live across the call. diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index 22623213f48c3..b9db956f1fc13 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -3,11 +3,13 @@ /* Fingerprint sensor SPI shim: moves bytes over the bus and toggles power. */ #include +#include #include #include #include #include #include +#include #include #include #include @@ -51,6 +53,16 @@ static bool sep_registered; static int sep_power_source; +/* + * Data-ready line for interrupt-driven capture. On j414s it is the pin adjacent + * to the power line (122) on the same controller; other boards describe it as + * the SPI node's interrupt, which the SPI core resolves into spi->irq. + */ +#define SEP_SENSOR_DRDY_LINE 121 +static struct gpio_desc *sep_drdy; /* set only when we own the gpiochip line (j414s) */ +static int sep_drdy_irq = -1; +static DECLARE_COMPLETION(sep_drdy_done); + /* * Takes the power line as an output driven low: the power cycle begins with an * off phase, so the line must be actively driven off, not merely read as low. @@ -227,6 +239,7 @@ static int sep_sensor_probe(struct spi_device *spi) static void sep_sensor_remove(struct spi_device *spi) { + sep_sensor_irq_teardown(); sep_release_power(); sep_spi = NULL; } @@ -352,6 +365,137 @@ int sep_sensor_power(int on) return 0; } +/* + * Threaded data-ready handler. IRQF_ONESHOT keeps the line masked while this + * runs, so a level-asserted line cannot storm the CPU; it only wakes the + * capture loop, which still confirms the sensor state over SPI before reading. + */ +static irqreturn_t sep_drdy_isr(int irq, void *dev_id) +{ + complete(&sep_drdy_done); + return IRQ_HANDLED; +} + +/* + * Set up the data-ready interrupt once, while the sensor is idle. It is left + * configured and enabled for the driver's lifetime -- never toggled per capture + * -- because reconfiguring this line mid-capture is what drops the sensor's + * firmware patch. Returns 0 when an interrupt is available, -errno otherwise + * (the caller then falls back to polling). + */ +int sep_sensor_irq_setup(void) +{ + struct device_node *np; + struct gpio_device *gdev; + struct gpio_chip *gc; + int irq, rc; + + if (!sep_spi) + return -ENODEV; + if (sep_drdy_irq >= 0) + return 0; + + if (sep_spi->irq > 0) { + /* A board that describes the data-ready line as the SPI node's + * interrupt gets it resolved by the SPI core. */ + irq = sep_spi->irq; + } else if (of_machine_is_compatible("apple,j414s")) { + np = of_find_node_by_path(SEP_SENSOR_GPIO_NODE); + if (!np) + return -ENODEV; + gdev = gpio_device_find_by_fwnode(of_fwnode_handle(np)); + of_node_put(np); + if (!gdev) + return -ENODEV; + gc = gpio_device_get_chip(gdev); + if (!gc) { + gpio_device_put(gdev); + return -ENODEV; + } + sep_drdy = gpiochip_request_own_desc(gc, SEP_SENSOR_DRDY_LINE, + "apple-mesa-drdy", + GPIO_LOOKUP_FLAGS_DEFAULT, + GPIOD_IN); + gpio_device_put(gdev); + if (IS_ERR(sep_drdy)) { + sep_drdy = NULL; + return -ENODEV; + } + irq = gpiod_to_irq(sep_drdy); + if (irq < 0) { + gpiochip_free_own_desc(sep_drdy); + sep_drdy = NULL; + return irq; + } + } else { + return -ENODEV; + } + + init_completion(&sep_drdy_done); + /* + * Edge-triggered, both edges: the data-ready line idles low, so a level + * trigger would storm continuously. An edge fires once per data-ready + * transition regardless of the line's asserted polarity, so the capture + * loop actually waits; a missed edge only costs one status poll. + */ + rc = request_threaded_irq(irq, NULL, sep_drdy_isr, + IRQF_ONESHOT | IRQF_TRIGGER_RISING | + IRQF_TRIGGER_FALLING, + "apple-mesa-drdy", sep_spi); + if (rc) { + if (sep_drdy) { + gpiochip_free_own_desc(sep_drdy); + sep_drdy = NULL; + } + return rc; + } + sep_drdy_irq = irq; + dev_info(&sep_spi->dev, + "sep sensor: data-ready IRQ %d configured for interrupt capture\n", + irq); + return 0; +} + +void sep_sensor_irq_teardown(void) +{ + if (sep_drdy_irq >= 0) { + free_irq(sep_drdy_irq, sep_spi); + sep_drdy_irq = -1; + } + if (sep_drdy) { + gpiochip_free_own_desc(sep_drdy); + sep_drdy = NULL; + } +} + +int sep_sensor_irq_available(void) +{ + return sep_drdy_irq >= 0; +} + +/* Clear any stale signal before a capture so the next wait reflects a fresh + * data-ready edge, not a leftover from the previous frame. */ +void sep_sensor_irq_arm(void) +{ + if (sep_drdy_irq >= 0) + reinit_completion(&sep_drdy_done); +} + +/* + * Wait for the data-ready line to assert, up to timeout_ms. 0 = fired, + * -ETIMEDOUT = no signal (the caller re-polls status over SPI regardless, so a + * missed interrupt only costs one poll interval), -ENODEV = no interrupt. + */ +int sep_sensor_irq_wait(unsigned int timeout_ms) +{ + if (sep_drdy_irq < 0) + return -ENODEV; + if (wait_for_completion_timeout(&sep_drdy_done, + msecs_to_jiffies(timeout_ms)) == 0) + return -ETIMEDOUT; + return 0; +} + /* * One chip-select assertion, `len` clocks, full duplex. The caller supplies all * `len` transmit bytes (trailing ones as 0xff) rather than relying on the diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index fc7609cb0a8b3..4d91e4710c646 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2185,5 +2185,9 @@ module! { default: 0, description: "Low 64 bits of an explicit xART OS UUID", }, + capture_irq: u8 { + default: 0, + description: "Wake the capture loop from the sensor's data-ready interrupt instead of polling SPI status (experimental; 0 = poll, the default). The interrupt only accelerates the loop -- status is still read over SPI to gate each frame -- and is set up once while the sensor is idle, never toggled mid-capture.", + }, }, } diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 34c34f29bc68b..2797f679f2e5e 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -128,6 +128,17 @@ int sep_sensor_xfer(const void *tx, void *rx, size_t len); int sep_sensor_xfer_tx(const void *tx, size_t len); int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len); +/* + * Optional data-ready interrupt for interrupt-driven capture. Set up once while + * the sensor is idle (never toggled per capture), then armed and waited on to + * wake the capture loop the instant a frame is ready instead of polling. + */ +int sep_sensor_irq_setup(void); +void sep_sensor_irq_teardown(void); +int sep_sensor_irq_available(void); +void sep_sensor_irq_arm(void); +int sep_sensor_irq_wait(unsigned int timeout_ms); + /* -- bio_shim.c --------------------------------------------------------- */ void *sep_bio_register(const char *name, unsigned short mode, void *ctx, From 3b0b8017606e48010a8d415cbc8a2dd1ebb4af52 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 18 Sep 2026 02:11:40 -0400 Subject: [PATCH 0054/1579] soc: apple: sep: surface a retry hint when an enrol capture is rejected An enrol capture the enclave rejects (partial or unusable contact) silently looped for another frame with no feedback, so the person had no idea their touch had not counted. Set the hold-still guidance on that path so the enrol poll reports it and userspace can prompt for a firmer press, matching the guidance already surfaced from the armed and reading sensor states. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/sbio.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index eb98d261124f7..9dd36e88ca3c3 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1080,6 +1080,12 @@ impl SepData { } ImageOutcome::Retry => { counter = counter.saturating_add(1); + // The capture did not take (partial/unusable contact). Nudge + // the person to hold still and press again rather than + // silently re-capturing with no feedback. + if bio::enrol_guide(&mut self.bio_session.lock(), bio::Guidance::HoldStill) { + self.bio_wake(); + } } ImageOutcome::NoFinger => { break Some(Err(ENROL_STATUS_TIMEOUT)); From 5c0b53b9e379bc1519e004fb8b3b942c19467587 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 18 Sep 2026 03:10:37 -0400 Subject: [PATCH 0055/1579] soc: apple: sep: make interrupt-driven capture the default Land the interrupt path as the default from hardware validation: it is no longer behind the capture_irq parameter (removed), and it is set up whenever the sensor exposes a data-ready line, with SPI polling kept only as the fallback for a machine that does not describe one. Two fixes the enrolment stress test surfaced: - Trigger on the rising edge only. The line idles low, so a level trigger storms and both-edges also fires on the deassert -- a spurious wake that, mid-enrolment (the finger lifts and repositions between the eight stages), spun the capture loop. - Bound the capture wait by wall-clock time, not a fixed attempt count. A stray edge returns the wait early, so an attempt budget would be spent in milliseconds and time the capture out before a frame lands; the time bound degrades, at worst, to the poll path's duration. The wait now blocks on the data-ready line for up to ENROL_IRQ_WAIT_MS (250 ms) rather than re-reading SPI status every 2 ms, waking the instant a frame is ready. Boot-verified on j414s (T6020): fprintd verify and a full eight-stage enrol both capture and match through the interrupt, with no STATE_NEEDS_PATCH and no level storm. Signed-off-by: Diljit Singh Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Assisted-by: Claude Opus 4.8 --- drivers/soc/apple/sbio.rs | 42 +++++++++++++++++++++++---------- drivers/soc/apple/sensor_shim.c | 12 +++++----- drivers/soc/apple/sep.rs | 9 +++---- 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 9dd36e88ca3c3..8b885ed98753e 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -739,14 +739,15 @@ impl SepData { let ok = self.complete_bringup(patch); // The sensor is patched and idle here -- the only safe moment to // configure the data-ready interrupt, which is then left alone for the - // driver's life. Opt-in; the poll path is unchanged when off. - if ok && *module_parameters::capture_irq.value() != 0 && !sensor::irq_available() { + // driver's life. Interrupt capture is the default; a machine that does + // not describe a data-ready line falls back to SPI polling. + if ok && !sensor::irq_available() { if sensor::irq_setup() { - dev_info!(self.dev, "sensor: interrupt-driven capture enabled\n"); + dev_info!(self.dev, "sensor: interrupt-driven capture\n"); } else { dev_warn!( self.dev, - "sensor: data-ready interrupt unavailable; capture stays on SPI polling\n" + "sensor: no data-ready interrupt (describe it in DT); capture falls back to SPI polling\n" ); } } @@ -1671,15 +1672,26 @@ impl SepData { let mut previous: Option<[u8; sensor::STATUS_LEN]> = None; let mut armed_reported = false; - // Interrupt-driven capture only replaces the fixed inter-poll sleep with - // a data-ready wait; the SPI status read below still gates every frame, - // so a missed or spurious interrupt costs at most one poll interval. - let use_irq = *module_parameters::capture_irq.value() != 0 && sensor::irq_available(); + // Interrupt-driven capture blocks on the data-ready line instead of + // polling; the SPI status read below still gates every frame, so a + // missed or spurious interrupt costs at most one backstop interval. A + // machine with no data-ready line falls back to SPI polling. + let use_irq = sensor::irq_available(); if use_irq { sensor::irq_arm(); } - for attempt in 0..ENROL_POLL_ATTEMPTS { + // Bound the wait by wall-clock time, not a fixed iteration count. The + // data-ready interrupt can wake early on a stray edge (the sensor + // toggles the line as the finger moves between enrol frames); a fixed + // attempt budget would then be spent in milliseconds and time the + // capture out before the frame lands. Time-bounding degrades, at worst, + // to the same duration as the poll path. + let deadline_ns = crate::shim::monotonic_ns().saturating_add( + u64::from(ENROL_POLL_ATTEMPTS) * u64::from(ENROL_POLL_MS) * 1_000_000, + ); + let mut attempt: u32 = 0; + loop { if !bio::capture_is_live(&self.bio_session.lock()) { return CaptureWait::Abandon; } @@ -1721,14 +1733,20 @@ impl SepData { return CaptureWait::Ready(count); } + if crate::shim::monotonic_ns() >= deadline_ns { + break; + } + if use_irq { - // Wake the instant the sensor asserts data-ready, or after the - // same interval on timeout; then re-arm for the next frame. - let _ = sensor::irq_wait(ENROL_POLL_MS); + // Block on the data-ready line; the interrupt wakes this the + // instant a frame is ready, and the backstop bounds a missed + // edge. Re-arm for the next frame. + let _ = sensor::irq_wait(ENROL_IRQ_WAIT_MS); sensor::irq_arm(); } else { kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ENROL_POLL_MS))); } + attempt = attempt.saturating_add(1); } let _ = sensor::status(); CaptureWait::Timeout diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index b9db956f1fc13..aff7da78fe49e 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -433,14 +433,14 @@ int sep_sensor_irq_setup(void) init_completion(&sep_drdy_done); /* - * Edge-triggered, both edges: the data-ready line idles low, so a level - * trigger would storm continuously. An edge fires once per data-ready - * transition regardless of the line's asserted polarity, so the capture - * loop actually waits; a missed edge only costs one status poll. + * Edge-triggered on the rising edge only. The line idles low, so a level + * trigger storms and both-edges fires again on the deassert -- a spurious + * wake that, mid-enrolment, spins the capture loop. The data-ready assert + * is the low->high edge; the capture loop's status read gates the frame, + * and a missed edge only costs one status poll. */ rc = request_threaded_irq(irq, NULL, sep_drdy_isr, - IRQF_ONESHOT | IRQF_TRIGGER_RISING | - IRQF_TRIGGER_FALLING, + IRQF_ONESHOT | IRQF_TRIGGER_RISING, "apple-mesa-drdy", sep_spi); if (rc) { if (sep_drdy) { diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 4d91e4710c646..61468bb8acd22 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -366,6 +366,11 @@ const ENROL_MAX_CAPTURES: u32 = 12; const ENROL_POLL_MS: u32 = 2; +// Interrupt-driven capture blocks on the data-ready line rather than polling +// SPI status. This is the maximum a single wait blocks before re-reading status +// as a backstop; the interrupt wakes it far sooner when a frame is ready. +const ENROL_IRQ_WAIT_MS: u32 = 250; + const ENROL_CAPTURE_TIMEOUT_MS: u32 = 60_000; const ENROL_POLL_ATTEMPTS: u32 = ENROL_CAPTURE_TIMEOUT_MS / ENROL_POLL_MS; static_assert!(ENROL_POLL_ATTEMPTS * ENROL_POLL_MS == ENROL_CAPTURE_TIMEOUT_MS); @@ -2185,9 +2190,5 @@ module! { default: 0, description: "Low 64 bits of an explicit xART OS UUID", }, - capture_irq: u8 { - default: 0, - description: "Wake the capture loop from the sensor's data-ready interrupt instead of polling SPI status (experimental; 0 = poll, the default). The interrupt only accelerates the loop -- status is still read over SPI to gate each frame -- and is set up once while the sensor is idle, never toggled mid-capture.", - }, }, } From 77f41d244521b17bc30018f2a3f571f4717fe8e4 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:04:35 -0400 Subject: [PATCH 0056/1579] soc: apple: sep: add the J700 platform and Mesa revision Port the Neo warm SEP profile and 0x3356 sensor identity from linux-aurora 450f581707. Fold in the later hardware finding from 18165cf1de: this J700 accepted the T6020-style keybag encoding and refused the strict form. Keep the wider DMA capability described in the profile so older boards retain their masks. Co-Authored-By: Claude Opus 5 (cherry picked from commit eec15fc3c4782e9f6785eaa888e724ef9c3cd808) --- drivers/soc/apple/profile.rs | 44 +++++++++++++++++++++++++++++++++++- drivers/soc/apple/sbio.rs | 5 ++++ drivers/soc/apple/sensor.rs | 8 +++++-- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 61faf6b88736f..2c3c8c1e70db5 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -8,12 +8,14 @@ //! here once. The driver reads addresses from the device tree, never scans for //! them, and never patches properties at runtime. //! -//! Two bring-up targets are modelled: +//! Three bring-up targets are modelled: //! //! * `T8103` / J313 (MacBook Air, M1): the host boots the SEP with the boot //! endpoint handshake over a 0x30000 shared-memory window. //! * `T6020` / J414s (MacBook Pro 14", M2 Pro): the driver does the warm //! single-message registration over a 0x40000 window. +//! * `T8140` / J700 (MacBook Neo, A18 Pro): iBoot boots sepOS before the AP +//! OS, so the driver uses the warm registration path. // The boot handshake, identity source, sensor and DART fields are consumed by // the boot-endpoint, identity and sensor-transport paths. @@ -115,6 +117,8 @@ pub(crate) struct PlatformProfile { /// Require a static `apple,dma-range` on the SEP DART. The T6020 SEP only /// accepts IOVAs below 4 GiB; T8103 works with the stock DART aperture. pub(crate) dart_range_required: bool, + /// Whether the SEP DART IOVA window needs a DMA mask wider than 32 bits. + pub(crate) wide_dma_mask: bool, /// Reserved-memory region holding the SEP firmware image (cold-boot path). pub(crate) firmware_region: &'static CStr, /// Identity keybag CREATE_KEYBAG field encoding (per-SoC; see [`KeybagCreate`]). @@ -138,6 +142,7 @@ const T8103: PlatformProfile = PlatformProfile { capture_qualified: false, }, dart_range_required: false, + wide_dma_mask: false, firmware_region: c"sepfw", // Strict enclave: the type goes in the third word; the first word is 0. keybag_create: KeybagCreate { @@ -164,6 +169,7 @@ const T6020: PlatformProfile = PlatformProfile { capture_qualified: false, }, dart_range_required: true, + wide_dma_mask: false, firmware_region: c"sepfw", // Proven encoding: the lenient enclave takes the variant in the first word. keybag_create: KeybagCreate { @@ -173,10 +179,43 @@ const T6020: PlatformProfile = PlatformProfile { }, }; +/// MacBook Neo. The J700 ADT records a pre-booted SEP, a spi2 Mesa sensor +/// (0x3356), and a SEP DART aperture above 4 GiB. These values describe that +/// board; device-tree nodes remain authoritative for addresses and resources. +const T8140: PlatformProfile = PlatformProfile { + name: "T8140/J700", + shmem_capacity: 0x4_0000, + shmem_first_item: b"CINP", + bootstrap: Bootstrap::WarmRegister, + identity: IdentitySource::Chosen, + sensor: SensorProfile { + controller_base: 0x3_8510_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode2, + expected_id: 0x3356, + capture_qualified: false, + }, + dart_range_required: true, + wide_dma_mask: true, + firmware_region: c"sepfw", + // J700 hardware refused the strict encoding and accepted this T6020 form + // (linux-aurora commit 18165cf1de, 2026-09-19). + keybag_create: KeybagCreate { + variant: 5, + bag_type: 0, + arg: -1, + }, +}; + static_assert!(T8103.shmem_capacity == 0x30000); static_assert!(T6020.shmem_capacity == 0x40000); +static_assert!(T8140.shmem_capacity == 0x40000); static_assert!(T8103.sensor.controller_base == 0x235108000); static_assert!(T6020.sensor.controller_base == 0x39b108000); +static_assert!(T8140.sensor.controller_base == 0x385108000); static_assert!(T8103.sensor.expected_id == T6020.sensor.expected_id); /// Whether the machine root declares `compatible`. @@ -216,5 +255,8 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { if machine_has(b"apple,t6020") { return Ok(&T6020); } + if machine_has(b"apple,t8140") { + return Ok(&T8140); + } Err(ENODEV) } diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 8b885ed98753e..8c5d6b6a0f950 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1205,6 +1205,11 @@ impl SepData { if let sensor::Offset12::Identifier(id) = st.offset12() { if id != 0 && !sensor::KNOWN_IDENTIFIERS.contains(&id) { + dev_err!( + self.dev, + "sensor: unknown Mesa identifier {:#06x} at status offset 12; not registering it with the enclave\n", + id + ); return None; } } diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 6b6393dc28c04..f5f3ea6307c24 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -79,6 +79,8 @@ pub(crate) enum PowerSource { None, NodeProperty, ChipLine, + /// Rails switched by the SMC on J700. + SmcRails, } impl PowerSource { @@ -86,6 +88,7 @@ impl PowerSource { match v { 1 => PowerSource::NodeProperty, 2 => PowerSource::ChipLine, + 3 => PowerSource::SmcRails, _ => PowerSource::None, } } @@ -95,6 +98,7 @@ impl PowerSource { PowerSource::None => c"none — no power line was taken", PowerSource::NodeProperty => c"the sensor node's own gpios property", PowerSource::ChipLine => c"/soc/pinctrl@39b028000 line 122, taken by device-tree node", + PowerSource::SmcRails => c"the sensor node's SMC-switched rails (apple,smc-power-key)", } } } @@ -102,8 +106,8 @@ impl PowerSource { pub(crate) const POWER_ON_READ_DELAYS_MS: [u32; 4] = [0, 3, 10, 50]; pub(crate) const STATUS_IDENTIFIER: usize = 12; -// Known Mesa sensor revisions: 0x3352 through M4, 0x335e on newer parts. -pub(crate) const KNOWN_IDENTIFIERS: [u16; 2] = [0x3352, 0x335e]; +// Known Mesa sensor revisions, including the J700's 0x3356. +pub(crate) const KNOWN_IDENTIFIERS: [u16; 3] = [0x3352, 0x3356, 0x335e]; static_assert!(STATUS_IDENTIFIER + 2 <= STATUS_LEN); const CMD_LEN: usize = 7; From d8a8ed5e811b9012657cc7eaf3ac5a4c679cfdf7 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:04:55 -0400 Subject: [PATCH 0057/1579] soc: apple: sep: power J700 Mesa through SMC rails Port linux-aurora 29adacc631 with DT-described pcLD rail selectors and symmetric power sequencing. Normalize macsmc write replies so a successful response size is not mistaken for an error; retain a GPIO path for older boards. Co-Authored-By: Claude Opus 5 (cherry picked from commit 889bb3af8e416a065f6c65e8cec574abc67564f4) --- .../input/apple,mesa-fingerprint.yaml | 21 ++- drivers/soc/apple/Kconfig | 1 + drivers/soc/apple/sbio.rs | 4 +- drivers/soc/apple/sensor.rs | 1 + drivers/soc/apple/sensor_shim.c | 147 ++++++++++++++++++ 5 files changed, 172 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml index be9bbcf473e85..68531cf93ba0b 100644 --- a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml +++ b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml @@ -37,6 +37,19 @@ properties: maxItems: 1 description: Power line to the sensor, asserted to power it up. + apple,smc-power-key: + $ref: /schemas/types.yaml#/definitions/string + description: + Four-character SMC key used by boards with SMC-switched sensor rails. + The rail operation is an eight-byte little-endian pair of an on/off + value and a rail selector word. + + apple,smc-power-rails: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 4 + description: Rail selector words, in power-on order. + firmware-name: maxItems: 1 description: @@ -51,9 +64,15 @@ properties: required: - compatible - reg - - enable-gpios - firmware-name +oneOf: + - required: + - enable-gpios + - required: + - apple,smc-power-key + - apple,smc-power-rails + unevaluatedProperties: false examples: diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 1c1527199a18c..0b55753806d47 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -109,6 +109,7 @@ config APPLE_SEP depends on HW_RANDOM depends on TRUSTED_KEYS depends on CRYPTO + depends on MFD_MACSMC || !MFD_MACSMC select CRYPTO_HMAC select CRYPTO_SHA256 select CRYPTO_AES diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 8c5d6b6a0f950..99665791198b6 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -249,12 +249,14 @@ impl SepData { let bound = sensor::is_bound(); self.sensor_present.store(bound, Relaxed); - if bound && sensor::power_line().is_none() { + if bound && sensor::power_source() == sensor::PowerSource::None { dev_warn!( self.dev, "sensor: no power line ({}); an unpowered sensor answers sixteen zero bytes like a dead bus\n", sensor::power_source().name() ); + } else if bound { + dev_info!(self.dev, "sensor: power via {}\n", sensor::power_source().name()); } } diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index f5f3ea6307c24..8fec6b71c8715 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -200,6 +200,7 @@ pub(crate) fn is_bound() -> bool { unsafe { sep_sensor_bound() != 0 } } +#[allow(dead_code)] pub(crate) fn power_line() -> Option { // SAFETY: no arguments; returns a negative errno when unbound. let n = unsafe { sep_sensor_power_line() }; diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index aff7da78fe49e..ec4d30581da67 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -10,10 +10,14 @@ #include #include #include +#include #include #include +#include +#include #include #include +#include #include "shim.h" @@ -50,9 +54,19 @@ static bool sep_registered; #define SEP_POWER_NONE 0 #define SEP_POWER_NODE_PROPERTY 1 #define SEP_POWER_CHIP_LINE 2 +#define SEP_POWER_SMC_RAILS 3 static int sep_power_source; +/* J700's Mesa power rails are selected by the pcLD SMC pKW8 operation. + * The DT gives the key and selectors; each write is { u32 on/off, u32 rail }. + */ +#define SEP_SMC_MAX_RAILS 4 +static struct apple_smc *sep_smc; +static smc_key sep_smc_key; +static u32 sep_smc_rails[SEP_SMC_MAX_RAILS]; +static int sep_smc_nrails; + /* * Data-ready line for interrupt-driven capture. On j414s it is the pin adjacent * to the power line (122) on the same controller; other boards describe it as @@ -63,6 +77,107 @@ static struct gpio_desc *sep_drdy; /* set only when we own the gpiochip line (j4 static int sep_drdy_irq = -1; static DECLARE_COMPLETION(sep_drdy_done); +#if IS_ENABLED(CONFIG_MFD_MACSMC) +static int sep_smc_write_rail(u32 rail, u16 value) +{ + u8 payload[8]; + int ret; + + put_unaligned_le32(value, payload); + put_unaligned_le32(rail, payload + 4); + ret = apple_smc_write(sep_smc, sep_smc_key, payload, sizeof(payload)); + /* macsmc forwards the SMC response size. Write replies may report no + * payload or echo the request length; neither is a failure status. + */ + if (ret == 0 || ret == (int)sizeof(payload)) + return 0; + return ret < 0 ? ret : -EIO; +} + +static int sep_smc_set_power(int on) +{ + int i, rc = 0; + + if (on) { + for (i = 0; i < sep_smc_nrails && !rc; i++) + rc = sep_smc_write_rail(sep_smc_rails[i], 1); + } else { + for (i = sep_smc_nrails - 1; i >= 0 && !rc; i--) + rc = sep_smc_write_rail(sep_smc_rails[i], 0); + } + if (rc) + pr_err("apple_sep: sensor: SMC rail write failed (%d)\n", rc); + return rc; +} + +/* Nothing is written until the expected eight-byte SMC key is verified. */ +static bool sep_acquire_smc_power(struct spi_device *spi) +{ + struct device_node *np = spi->dev.of_node, *smc_np; + struct platform_device *smc_pdev; + struct apple_smc_key_info info; + const char *key = NULL; + int n, rc; + + if (!np || of_property_read_string(np, "apple,smc-power-key", &key)) + return false; + if (!key || strlen(key) != 4) { + dev_warn(&spi->dev, "sep sensor: apple,smc-power-key must have four characters\n"); + return false; + } + n = of_property_count_u32_elems(np, "apple,smc-power-rails"); + if (n <= 0 || n > SEP_SMC_MAX_RAILS || + of_property_read_u32_array(np, "apple,smc-power-rails", sep_smc_rails, n)) { + dev_warn(&spi->dev, "sep sensor: expected 1..%d SMC rail words\n", + SEP_SMC_MAX_RAILS); + return false; + } + + smc_np = of_find_compatible_node(NULL, NULL, "apple,smc"); + if (!smc_np) + return false; + smc_pdev = of_find_device_by_node(smc_np); + of_node_put(smc_np); + if (!smc_pdev) + return false; + sep_smc = dev_get_drvdata(&smc_pdev->dev); + put_device(&smc_pdev->dev); + if (!sep_smc) + return false; + + sep_smc_key = __SMC_KEY(key[0], key[1], key[2], key[3]); + rc = apple_smc_get_key_info(sep_smc, sep_smc_key, &info); + if (rc || info.size != 8) { + dev_warn(&spi->dev, "sep sensor: SMC key %s absent or not eight bytes\n", key); + sep_smc = NULL; + return false; + } + sep_smc_nrails = n; + sep_power_source = SEP_POWER_SMC_RAILS; + dev_info(&spi->dev, "sep sensor: power from SMC key %s (%d rails)\n", key, n); + if (sep_smc_set_power(0)) { + sep_power_source = SEP_POWER_NONE; + sep_smc_nrails = 0; + sep_smc = NULL; + return false; + } + return true; +} +#else +static int sep_smc_set_power(int on) +{ + return -ENODEV; +} + +static bool sep_acquire_smc_power(struct spi_device *spi) +{ + if (spi->dev.of_node && + of_property_present(spi->dev.of_node, "apple,smc-power-key")) + dev_warn(&spi->dev, "sep sensor: SMC power needs CONFIG_MFD_MACSMC\n"); + return false; +} +#endif + /* * Takes the power line as an output driven low: the power cycle begins with an * off phase, so the line must be actively driven off, not merely read as low. @@ -73,6 +188,9 @@ static void sep_acquire_power(struct spi_device *spi) struct gpio_device *gdev; struct gpio_chip *gc; + if (sep_acquire_smc_power(spi)) + return; + sep_power = gpiod_get_index(&spi->dev, NULL, 0, GPIOD_OUT_LOW); if (!IS_ERR(sep_power)) { sep_power_source = SEP_POWER_NODE_PROPERTY; @@ -137,6 +255,13 @@ static void sep_acquire_power(struct spi_device *spi) static void sep_release_power(void) { + if (sep_power_source == SEP_POWER_SMC_RAILS) { + sep_smc_set_power(0); + sep_smc = NULL; + sep_smc_nrails = 0; + sep_power_source = SEP_POWER_NONE; + return; + } if (!sep_power) return; gpiod_set_value_cansleep(sep_power, 0); @@ -312,6 +437,18 @@ int sep_sensor_cs_timing_mode(void) /* Power cycle: off, 10 ms, on, 7 ms. -ENODEV if there is no power line. */ int sep_sensor_power_cycle(void) { + if (sep_power_source == SEP_POWER_SMC_RAILS) { + int rc = sep_smc_set_power(0); + + if (rc) + return rc; + msleep(SEP_SENSOR_OFF_DELAY_MS); + rc = sep_smc_set_power(1); + if (rc) + return rc; + msleep(SEP_SENSOR_ON_DELAY_MS); + return 0; + } if (!sep_power) return -ENODEV; @@ -330,6 +467,8 @@ int sep_sensor_power_source(void) int sep_sensor_power_line(void) { + if (sep_power_source == SEP_POWER_SMC_RAILS) + return -ENODEV; if (!sep_power) return -ENODEV; return desc_to_gpio(sep_power); @@ -354,6 +493,14 @@ const char *sep_sensor_firmware_name(void) /* Powers the sensor on/off, holding the hardware settling delay. */ int sep_sensor_power(int on) { + if (sep_power_source == SEP_POWER_SMC_RAILS) { + int rc = sep_smc_set_power(on ? 1 : 0); + + if (rc) + return rc; + msleep(on ? SEP_SENSOR_ON_DELAY_MS : SEP_SENSOR_OFF_DELAY_MS); + return 0; + } if (!sep_power) return -ENODEV; From 91881d5fa321dcfd37f4b7cd6d2a86fcf2868781 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:05:18 -0400 Subject: [PATCH 0058/1579] soc: apple: sep: widen the J700 DMA mask before allocation Port linux-aurora 11d4dff9a3. The Neo SEP DART aperture starts above 4 GiB, so select a 42-bit DMA mask for that profile before allocating shared memory. Keep the established M1/M2 DMA masks unchanged for regression testing. Co-Authored-By: Claude Opus 5 (cherry picked from commit b65f17cced0b25bda3bb40ae9a4b4d673c5da778) --- drivers/soc/apple/sep.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 61468bb8acd22..7f520ba87d44b 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -38,6 +38,7 @@ use kernel::{ bindings, device, dma, + dma::Device as _, driver, new_mutex, of, @@ -2096,6 +2097,13 @@ impl platform::Driver for SepDriver { } let sep_node = dt::DtNode::of_device(dev).ok_or(ENODEV)?; + // J700's SEP DART window starts above 4 GiB. Leave the proven masks + // on older platforms unchanged for the M1/M2 regression run. + if profile::detect()?.wide_dma_mask { + // SAFETY: probe has not allocated DMA memory for this device yet. + unsafe { pdev.dma_set_mask_and_coherent(dma::DmaMask::new::<42>())? }; + } + if dt::registration_already_sent(&sep_node) { dev_err!( dev, From 5e33a431a52a5159585a4018f8792f8ec516bc67 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:05:32 -0400 Subject: [PATCH 0059/1579] soc: apple: sep: wait for late J700 key-store discovery Port linux-aurora 41017b63c1: the measured J700 application-endpoint burst arrives around 43 seconds after boot. Allow up to 90 seconds, returning as soon as the key store is advertised. Co-Authored-By: Claude Opus 5 (cherry picked from commit f3a5cf1ead57b735ee8428f8895570b004c63388) --- drivers/soc/apple/sep.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 7f520ba87d44b..773c95cb92979 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -86,7 +86,10 @@ const VERIFY_WORK_ID: u64 = 3; static_assert!(VERIFY_WORK_ID != ENROL_WORK_ID && VERIFY_WORK_ID != SETTLE_WORK_ID); static_assert!(VERIFY_WORK_ID != 0); -const EXCHANGE_TIMEOUT_MS: time::Msecs = 15000; +// J700 traces show the key-store endpoint arriving around 43 s after boot, +// well after the first infrastructure endpoints. The wait exits promptly on +// other machines when their key store becomes ready. +const EXCHANGE_TIMEOUT_MS: time::Msecs = 90000; const PHASE_ATTACH: u32 = 0; const PHASE_EXCHANGE: u32 = 1; From fae9637829ea850c227df8847fd19b1e9f232707 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 21 Sep 2026 23:05:45 -0400 Subject: [PATCH 0060/1579] soc: apple: sep: refuse unverified raw xART writes The raw-owner locator scores CRC-valid record signatures in a fixed 6 MiB window rather than resolving the current .gl APFS file extent. If both root records have moved past slot zero, multiple shifted windows can pass the root test; a higher-base tie can extend into unrelated APFS blocks. An explicit start sector does not prove ownership. Fail closed on xart_writes=1 before SEP attach, including the explicit-sector path; leave read-only inspection available. Disable the write-enabled loader recommendation. This is a safety interlock, not a macOS-parity repair: the APFS mapping and record update protocol still need hardware validation. (cherry picked from commit fe3e8cf31b016761ecc8c1e25ac5256ea90b682d) --- drivers/soc/apple/Kconfig | 3 +- drivers/soc/apple/sep.rs | 9 +++++- drivers/soc/apple/xart_store.rs | 55 +++++++++++++++++---------------- tools/aurora-sep/README.md | 12 ++++--- 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 0b55753806d47..d3d451751fff4 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -125,7 +125,8 @@ config APPLE_SEP The driver reads the existing machine-wide xART gigalocker directly from the iBoot system container, locating it by its root records. It validates the complete store and both root records before registering with SEP. - Writes remain disabled unless the xart_writes module parameter is set. + Raw APFS xART writes are refused until the extent and update protocol + can be verified; the module parameter cannot override this safety gate. Say Y here if you have an Apple silicon Mac. diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 773c95cb92979..9c67e55ff83ef 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2092,6 +2092,13 @@ impl platform::Driver for SepDriver { _info: Option<&()>, ) -> impl PinInit { let dev: &device::Device = pdev.as_ref(); + if *module_parameters::xart_writes.value() != 0 { + dev_err!( + dev, + "raw APFS xART writes are blocked until the .gl extent and update protocol are validated\n" + ); + return Err(ENOTSUPP); + } if *module_parameters::provision_keybag.value() != 0 && *module_parameters::xart_writes.value() == 0 { @@ -2183,7 +2190,7 @@ module! { params: { xart_writes: u8 { default: 0, - description: "Allow writes to the validated shared xART store", + description: "Request shared xART writes (refused by the raw APFS owner until its extent is verified)", }, xart_start_sector: u64 { default: 0, diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index ac9e0bf6c51e2..54d289d8a2946 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -5,8 +5,9 @@ //! //! The APFS locator exposes the existing `.gl` file as a block device. This //! module implements the record validation, duplicate repair, lookup and -//! copy-on-write ordering the SEP requires. It never creates storage -//! and it can be opened read-only for safe inspection and bring-up. +//! copy-on-write ordering the SEP requires. It never creates storage. +//! The raw block owner is restricted to read-only until APFS metadata can +//! establish the current physical extent of the `.gl` file. use crate::shim; use kernel::prelude::*; @@ -132,13 +133,22 @@ impl Store { /// system container directly and serves only the gigalocker extent, with no /// external helper and no hand-supplied sector. /// - /// `start_sector == 0` (the default) *locates the gigalocker automatically* - /// inside the container by its CRC-checked root records; a non-zero - /// `start_sector` is an explicit override for the rare case the scan should - /// be skipped. + /// `start_sector == 0` (the default) searches for CRC-checked root records; + /// a non-zero sector skips the search. Neither method proves that the + /// window belongs to the current APFS `.gl` file, so writes are refused. pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { + // A CRC-valid record signature does not establish the APFS file's + // physical extent. Once both roots move away from slot zero, several + // shifted windows pass the two-root test, and a shifted window can + // extend into APFS-owned blocks. Never write through this raw owner + // until the mapping has been checked against APFS file metadata. + // An explicit sector is not proof of ownership either. + if writes_enabled { + return Err(ENOTSUPP); + } if start_sector != 0 { - return Self::open_based(OWNER_PATH, writes_enabled, start_sector << 9); + let base = start_sector.checked_mul(512).ok_or(EINVAL)?; + return Self::open_based(OWNER_PATH, false, base); } Self::open_located(writes_enabled) } @@ -150,10 +160,9 @@ impl Store { /// is probed read-only with the full CRC-checked [`open_based`]; because a /// base shifted a few slots off the true origin still keeps both roots in its /// window (passing the two-root test while dropping records that fall - /// outside), the base is pinned by the candidate that recovers the MOST live - /// records — the exact discriminator, since only the true origin captures the - /// whole set and container noise never forges a CRC-valid record. Repair is - /// never armed during discovery; only the pinned base is re-opened writable. + /// outside), the candidate with the most live records is used for read-only + /// diagnostics. This heuristic cannot establish the APFS file's origin or + /// prove it is safe to write. Repair is never armed during discovery. /// Bounded by [`MAX_LOCATE_ATTEMPTS`] probes; returns `ENODATA` if not found. fn open_located(writes_enabled: bool) -> Result { // The container is a writable block device; the block layer only grants @@ -187,11 +196,9 @@ impl Store { // shifted off the true origin by a few slots still keeps both // roots inside its 6 MiB window, so it passes yet silently // drops the live records that fall outside the shifted window. - // Probe every candidate READ-ONLY (writes_enabled forced false, - // so repair never fires at an unconfirmed base) and pin the one - // that recovers the MOST live records: only the true origin - // captures the whole record set, and container noise never - // forges a CRC-valid record, so max live records is exact. + // Probe candidates without writes and prefer the largest + // recovered record set for diagnostics. An equally valid + // shifted window is possible; this is not an APFS lookup. let hit = off + p as u64; let mut best: Option<(usize, usize, u64)> = None; // (valid, malformed, base) let mut k: u64 = 0; @@ -214,13 +221,9 @@ impl Store { let better = match best { None => true, // More live records wins; ties break to fewer - // malformed, then to the HIGHER base. An up-shift - // drops the lowest occupied slots (and any root - // there, which fails the two-root test), so it - // never ties on live count; the only bases that - // can tie are down-shifts, which are all lower - // than the true origin — so the highest surviving - // candidate is the true origin. + // malformed, then to the higher base. The tie + // rule is a heuristic and may choose an + // up-shifted base when both roots moved. Some(b) => { cand.0 > b.0 || (cand.0 == b.0 && cand.1 < b.1) @@ -232,10 +235,8 @@ impl Store { } } } - // The gigalocker is unique, so the first hit that confirms any - // base has pinned it. Re-open the winner through a writable - // handle and arm repair per `writes_enabled` — repair runs - // now, and only at this confirmed origin. + // Re-open the diagnostic candidate. open_owner refuses + // writes, including automatic repair, on raw APFS extents. if let Some((_, _, base)) = best { return Self::open_based_ext(OWNER_PATH, true, writes_enabled, base); } diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 6c28a65403f51..c3f17ad3c1171 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -12,10 +12,14 @@ systemctl daemon-reload systemctl enable aurora-sep.service ``` -The kernel driver locates the xART gigalocker itself inside the iBoot System -Container (`modprobe apple_sep xart_writes=1 provision_keybag=1`, with no -`xart_start_sector`): it opens the container directly and serves only the -gigalocker extent, needing no external helper and no hand-supplied sector. +The raw xART owner is currently read-only. Do not enable the installed +`aurora-sep.service` or run `load-driver` on a shared macOS/Linux installation: +that script requests `xart_writes=1`, which the kernel now rejects while the +physical `.gl` extent is selected by scanning record signatures rather than +validated against APFS file metadata. The scan is ambiguous when live root +records have moved away from the start of the file. Read-only inspection does +not establish write safety. Re-enable provisioning only after the APFS mapping +and macOS-compatible record update protocol are verified on hardware. For fingerprint support, apply `patches/libfprint-1.94.100-apple-sep.patch` to libfprint 1.94.100, build and From b4d0d034693620eb61718743480376a8d2ed1f90 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 21 Sep 2026 23:56:38 -0400 Subject: [PATCH 0061/1579] soc: apple: sep: resolve read-only xART extent through APFS metadata Replace the ambiguous root-record signature locator with a narrow, checksum-validated APFS checkpoint and .gl file-extent lookup. Open the shared container read-only and make an explicit sector an assertion rather than an override. Keep writes refused while the macOS ownership and update protocol are verified. (cherry picked from commit 9c0186e307f425f12c7d114e31dbc439ba55cc31) --- drivers/soc/apple/sep.rs | 9 +- drivers/soc/apple/shim.rs | 4 +- drivers/soc/apple/store_shim.c | 2 +- drivers/soc/apple/xart_apfs.rs | 342 ++++++++++++++++++++ drivers/soc/apple/xart_store.rs | 188 ++--------- tools/aurora-sep/README.md | 27 +- tools/aurora-sep/XART-SAFETY.md | 41 +++ tools/aurora-sep/apfs-xart-inspect.py | 320 ++++++++++++++++++ tools/aurora-sep/tests/test_xart_locator.py | 49 +++ 9 files changed, 805 insertions(+), 177 deletions(-) create mode 100644 drivers/soc/apple/xart_apfs.rs create mode 100644 tools/aurora-sep/XART-SAFETY.md create mode 100644 tools/aurora-sep/apfs-xart-inspect.py create mode 100644 tools/aurora-sep/tests/test_xart_locator.py diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 9c67e55ff83ef..5f128c7fa005d 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -32,6 +32,7 @@ mod store; mod transfer; mod trusted; mod xarm; +mod xart_apfs; mod xart_store; use kernel::{ @@ -726,7 +727,7 @@ impl SepData { let base = store.base(); dev_info!( dev, - "xART: in-kernel raw-extent owner (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", + "xART: APFS-resolved read-only .gl extent (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", base, slots, records, @@ -2095,7 +2096,7 @@ impl platform::Driver for SepDriver { if *module_parameters::xart_writes.value() != 0 { dev_err!( dev, - "raw APFS xART writes are blocked until the .gl extent and update protocol are validated\n" + "raw APFS xART writes are blocked until the .gl update protocol and ownership invariants are validated\n" ); return Err(ENOTSUPP); } @@ -2190,11 +2191,11 @@ module! { params: { xart_writes: u8 { default: 0, - description: "Request shared xART writes (refused by the raw APFS owner until its extent is verified)", + description: "Request shared xART writes (refused until the macOS update protocol is verified)", }, xart_start_sector: u64 { default: 0, - description: "Explicit 512-byte start sector of the xART gigalocker extent within the iBoot system container. Zero (default) locates the gigalocker automatically in the container by its root records; a non-zero value overrides the search. Either way the driver opens the container directly", + description: "Expected 512-byte start sector of the APFS .gl extent in the iBoot container. Zero accepts the APFS result; nonzero must match it and cannot override it", }, provision_keybag: u8 { default: 0, diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index f02a9b86e32ef..4e25060d15d25 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -75,8 +75,8 @@ impl StoreFile { } pub(crate) fn open_block(path: &CStr, writable: bool) -> Result { - // SAFETY: `path` is NUL-terminated; the shim accepts only a block - // device whose global read-only state matches `writable`. + // SAFETY: `path` is NUL-terminated; the shim opens read-only unless + // explicitly asked to write, and rejects writes to a read-only device. let handle = unsafe { sep_store_open_block(path.as_char_ptr(), c_int::from(writable)) }; if handle.is_null() { return Err(ENODEV); diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c index b067c39372fa7..bdfd5dfbb6391 100644 --- a/drivers/soc/apple/store_shim.c +++ b/drivers/soc/apple/store_shim.c @@ -82,7 +82,7 @@ void *sep_store_open_block(const char *path, int writable) if (IS_ERR(f)) return NULL; if (!S_ISBLK(file_inode(f)->i_mode) || - bdev_read_only(file_bdev(f)) == !!writable) { + (writable && bdev_read_only(file_bdev(f)))) { filp_close(f, NULL); return NULL; } diff --git a/drivers/soc/apple/xart_apfs.rs b/drivers/soc/apple/xart_apfs.rs new file mode 100644 index 0000000000000..9eb833baca82d --- /dev/null +++ b/drivers/soc/apple/xart_apfs.rs @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Narrow, read-only APFS lookup for the existing xART `.gl` file. +//! +//! This is deliberately not a general APFS implementation. Unsupported tree +//! shapes, encryption, fragmented files, and bad metadata fail closed. In +//! particular, finding an extent does not authorize raw writes to that file. + +use crate::shim; +use kernel::prelude::*; + +const BLOCK: usize = 4096; +const ROOT_INFO_SIZE: usize = 40; +const XART_ROLE: u16 = 0x100; +const FILE_SIZE: u64 = 0x600000; +const FS_UNENCRYPTED: u64 = 1; +const MOD: u64 = 0xffff_ffff; + +fn le16(data: &[u8], off: usize) -> Result { + Ok(u16::from_le_bytes( + data.get(off..off + 2) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + )) +} + +fn le32(data: &[u8], off: usize) -> Result { + Ok(u32::from_le_bytes( + data.get(off..off + 4) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + )) +} + +fn le64(data: &[u8], off: usize) -> Result { + Ok(u64::from_le_bytes( + data.get(off..off + 8) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + )) +} + +/// APFS's Fletcher-64 over the little-endian words after the checksum field. +fn checksum(data: &[u8]) -> Result { + if data.len() != BLOCK { + return Err(EINVAL); + } + let mut s1 = 0u64; + let mut s2 = 0u64; + for word in data[8..].chunks_exact(4) { + let value = u32::from_le_bytes(word.try_into().map_err(|_| EINVAL)?) as u64; + s1 = (s1 + value) % MOD; + s2 = (s2 + s1) % MOD; + } + let c1 = (MOD - (s1 + s2) % MOD) % MOD; + let c2 = (MOD - (s1 + c1) % MOD) % MOD; + Ok((c2 << 32) | c1) +} + +fn read_object(file: &shim::StoreFile, size: u64, paddr: u64) -> Result> { + let start = paddr.checked_mul(BLOCK as u64).ok_or(EINVAL)?; + if start.checked_add(BLOCK as u64).ok_or(EINVAL)? > size { + return Err(EINVAL); + } + let mut data = KVec::new(); + data.resize(BLOCK, 0, GFP_KERNEL)?; + file.read_block_exact(start, &mut data)?; + if le64(&data, 0)? != checksum(&data)? { + return Err(EIO); + } + Ok(data) +} + +/// The descriptor ring holds mapping blocks followed by its superblock. A +/// checksummed superblock alone is not a complete checkpoint: verify every +/// mapping and its ephemeral object before accepting that transaction. +fn valid_checkpoint( + file: &shim::StoreFile, + size: u64, + sb: &[u8], + sb_paddr: u64, + desc_base: u64, + desc_count: u64, +) -> Result<()> { + if le64(sb, 112)? != desc_base || le32(sb, 104)? as u64 != desc_count { + return Err(ENOTSUPP); + } + let index = le32(sb, 136)? as u64; + let len = le32(sb, 140)? as u64; + if index >= desc_count || len < 2 || len > desc_count { + return Err(EINVAL); + } + let last = desc_base + (index + len - 1) % desc_count; + if last != sb_paddr { + return Err(EINVAL); + } + let xid = le64(sb, 16)?; + for j in 0..len - 1 { + let paddr = desc_base + (index + j) % desc_count; + let map = read_object(file, size, paddr)?; + if le32(&map, 24)? & 0xffff != 12 || le64(&map, 16)? != xid { + return Err(EINVAL); + } + let flags = le32(&map, 32)?; + if flags & 1 != u32::from(j == len - 2) { + return Err(EINVAL); + } + let count = le32(&map, 36)? as usize; + if count > (BLOCK - 40) / 40 { + return Err(EINVAL); + } + for i in 0..count { + let at = 40 + i * 40; + if le32(&map, at + 8)? != BLOCK as u32 { + return Err(ENOTSUPP); + } + let object = read_object(file, size, le64(&map, at + 32)?)?; + if le64(&object, 8)? != le64(&map, at + 24)? + || le64(&object, 16)? != xid + || le32(&object, 24)? != le32(&map, at)? + { + return Err(EINVAL); + } + } + } + Ok(()) +} + +fn node_entry(node: &[u8], index: usize, fixed: bool) -> Result<(&[u8], &[u8])> { + if node.len() != BLOCK || le16(node, 34)? != 0 { + return Err(ENOTSUPP); + } + let flags = le16(node, 32)?; + if (flags & 4 != 0) != fixed { + return Err(ENOTSUPP); + } + let count = le32(node, 36)? as usize; + if index >= count || count > 512 { + return Err(EINVAL); + } + let toc = 56usize + .checked_add(le16(node, 40)? as usize) + .ok_or(EINVAL)?; + let table_len = le16(node, 42)? as usize; + let key_start = toc.checked_add(table_len).ok_or(EINVAL)?; + let val_end = BLOCK - if flags & 1 != 0 { ROOT_INFO_SIZE } else { 0 }; + let stride = if fixed { 4 } else { 8 }; + if key_start > val_end || toc.checked_add(count * stride).ok_or(EINVAL)? > key_start { + return Err(EINVAL); + } + let item = toc.checked_add(index * stride).ok_or(EINVAL)?; + let (ko, kl, vo, vl) = if fixed { + ( + le16(node, item)? as usize, + 16, + le16(node, item + 2)? as usize, + 16, + ) + } else { + ( + le16(node, item)? as usize, + le16(node, item + 2)? as usize, + le16(node, item + 4)? as usize, + le16(node, item + 6)? as usize, + ) + }; + let ks = key_start.checked_add(ko).ok_or(EINVAL)?; + let vs = val_end.checked_sub(vo).ok_or(EINVAL)?; + let ke = ks.checked_add(kl).ok_or(EINVAL)?; + let ve = vs.checked_add(vl).ok_or(EINVAL)?; + if ks < key_start || ke > val_end || vs < key_start || ve > val_end { + return Err(EINVAL); + } + Ok((&node[ks..ke], &node[vs..ve])) +} + +fn omap_lookup( + file: &shim::StoreFile, + size: u64, + omap_paddr: u64, + oid: u64, + xid: u64, +) -> Result { + let omap = read_object(file, size, omap_paddr)?; + if le64(&omap, 8)? != omap_paddr { + return Err(EINVAL); + } + let tree_paddr = le64(&omap, 48)?; + let tree = read_object(file, size, tree_paddr)?; + if le64(&tree, 8)? != tree_paddr || le16(&tree, 34)? != 0 || le16(&tree, 32)? & 4 == 0 { + return Err(ENOTSUPP); + } + let mut selected: Option<(u64, u32, u32, u64)> = None; + for i in 0..le32(&tree, 36)? as usize { + let (key, val) = node_entry(&tree, i, true)?; + let key_xid = le64(key, 8)?; + if le64(key, 0)? == oid && key_xid <= xid && selected.is_none_or(|v| key_xid > v.0) { + selected = Some((key_xid, le32(val, 0)?, le32(val, 4)?, le64(val, 8)?)); + } + } + let (_, flags, object_size, paddr) = selected.ok_or(ENODATA)?; + if flags & 1 != 0 || object_size != BLOCK as u32 { + return Err(ENOTSUPP); + } + Ok(paddr) +} + +fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Result { + if le64(vol, 264)? & FS_UNENCRYPTED == 0 { + return Err(ENOTSUPP); + } + let omap = le64(vol, 128)?; + let root_oid = le64(vol, 136)?; + let root_paddr = omap_lookup(file, size, omap, root_oid, xid)?; + let root = read_object(file, size, root_paddr)?; + if le64(&root, 8)? != root_oid || le16(&root, 34)? != 0 || le16(&root, 32)? & 4 != 0 { + return Err(ENOTSUPP); + } + + let count = le32(&root, 36)? as usize; + let mut gl_file: Option = None; + for i in 0..count { + let (key, val) = node_entry(&root, i, false)?; + if key.len() < 12 || val.len() < 8 || le64(key, 0)? != ((9u64 << 60) | 2) { + continue; + } + let name_len = (le32(key, 8)? & 0x3ff) as usize; + if name_len == 0 || 12 + name_len > key.len() || key[12 + name_len - 1] != 0 { + return Err(EINVAL); + } + if &key[12..12 + name_len - 1] == b".gl" { + if gl_file.replace(le64(val, 0)?).is_some() { + return Err(ENOTSUPP); + } + } + } + let gl_file = gl_file.ok_or(ENODATA)?; + let mut private_id: Option = None; + for i in 0..count { + let (key, val) = node_entry(&root, i, false)?; + if key.len() >= 8 && le64(key, 0)? == ((3u64 << 60) | gl_file) { + if val.len() < 92 || private_id.replace(le64(val, 8)?).is_some() { + return Err(EINVAL); + } + } + } + let private_id = private_id.ok_or(ENODATA)?; + let mut extent: Option = None; + for i in 0..count { + let (key, val) = node_entry(&root, i, false)?; + if key.len() < 16 || le64(key, 0)? != ((8u64 << 60) | private_id) { + continue; + } + if val.len() < 24 || le64(key, 8)? != 0 || le64(val, 0)? != FILE_SIZE || le64(val, 16)? != 0 + { + return Err(ENOTSUPP); + } + let paddr = le64(val, 8)?; + let start = paddr.checked_mul(BLOCK as u64).ok_or(EINVAL)?; + if start.checked_add(FILE_SIZE).ok_or(EINVAL)? > size || extent.replace(start).is_some() { + return Err(EINVAL); + } + } + extent.ok_or(ENODATA) +} + +/// Return the byte offset of the sole contiguous 6 MiB `.gl` file extent. +pub(crate) fn locate(file: &shim::StoreFile) -> Result { + let device_size = file.size()?; + let base = read_object(file, device_size, 0)?; + if &base[32..36] != b"NXSB" || le32(&base, 36)? != BLOCK as u32 { + return Err(ENOTSUPP); + } + let blocks = le64(&base, 40)?; + if blocks == 0 + || blocks > device_size / BLOCK as u64 + || le64(&base, 56)? != 0 + || le64(&base, 64)? != 2 + { + return Err(ENOTSUPP); + } + let size = blocks.checked_mul(BLOCK as u64).ok_or(EINVAL)?; + let desc_count_raw = le32(&base, 104)?; + if desc_count_raw & (1 << 31) != 0 { + return Err(ENOTSUPP); + } + let desc_count = desc_count_raw as u64; + let desc_base = le64(&base, 112)?; + if desc_count == 0 + || desc_count > 1024 + || desc_base.checked_add(desc_count).ok_or(EINVAL)? > size / BLOCK as u64 + { + return Err(EINVAL); + } + let mut best: Option> = None; + let mut best_xid = 0u64; + for paddr in desc_base..desc_base + desc_count { + let Ok(candidate) = read_object(file, size, paddr) else { + continue; + }; + let candidate_xid = le64(&candidate, 16)?; + if &candidate[32..36] == b"NXSB" + && le32(&candidate, 36)? == BLOCK as u32 + && le64(&candidate, 40)? == blocks + && le64(&candidate, 56)? == 0 + && le64(&candidate, 64)? == 2 + && valid_checkpoint(file, size, &candidate, paddr, desc_base, desc_count).is_ok() + && candidate_xid > best_xid + { + best_xid = candidate_xid; + best = Some(candidate); + } + } + let best = best.ok_or(ENODATA)?; + let xid = le64(&best, 16)?; + let omap = le64(&best, 160)?; + let mut found: Option = None; + for i in 0..100usize { + let oid = le64(&best, 184 + i * 8)?; + if oid == 0 { + continue; + } + let paddr = omap_lookup(file, size, omap, oid, xid)?; + let vol = read_object(file, size, paddr)?; + if &vol[32..36] != b"APSB" || le64(&vol, 8)? != oid || le64(&vol, 16)? > xid { + return Err(EINVAL); + } + if le16(&vol, 964)? == XART_ROLE { + if found + .replace(xart_extent(file, size, &vol, le64(&vol, 16)?)?) + .is_some() + { + return Err(ENOTSUPP); + } + } + } + found.ok_or(ENODATA) +} diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index 54d289d8a2946..7c73107bc2ae8 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -3,11 +3,10 @@ //! Device-wide xART gigalocker record store. //! -//! The APFS locator exposes the existing `.gl` file as a block device. This -//! module implements the record validation, duplicate repair, lookup and -//! copy-on-write ordering the SEP requires. It never creates storage. -//! The raw block owner is restricted to read-only until APFS metadata can -//! establish the current physical extent of the `.gl` file. +//! The APFS locator identifies the existing `.gl` file's physical extent. +//! This module validates records and serves SEP reads from that extent. It +//! never creates storage. Raw writes remain blocked until the record update +//! protocol and APFS ownership rules are verified against macOS. use crate::shim; use kernel::prelude::*; @@ -20,28 +19,11 @@ const BLOCK_SIZE: usize = 0x1000; const SLOT_SIZE: usize = 0x9000; const HEADER_SIZE: usize = 0x22; const DELETE_SIZE: usize = BLOCK_SIZE; -/// Size of the APFS raw extent located on the target machine. -/// -/// The logical store is always exactly this many bytes. The raw-extent owner -/// opens a larger container and serves only the [`STORE_SIZE`] window at its -/// located base, so a wrong base cannot read past the extent. A wrong window -/// still fails closed: -/// the two root records will not validate and `open_based` returns `ENODATA` -/// rather than serving SEP an unrelated store. A machine with a different extent -/// size must pass that size through an explicit, reviewed interface. +/// The current `.gl` format occupies one contiguous six-MiB APFS file extent. +/// Unsupported layouts fail closed in `xart_apfs::locate`. const STORE_SIZE: u64 = 0x600000; const MAX_SLOTS: usize = 4096; -/// Sequential read window for the automatic gigalocker search (64 KiB — small -/// enough for a reliable kernel allocation, large enough to keep the scan of -/// the container to a modest number of reads). -const SCAN_CHUNK: usize = 1 << 16; -/// Upper bound on candidate bases confirmed during the automatic search, so a -/// container full of coincidental root-shaped headers cannot spin the scan. The -/// real store's roots sit within the first slots, so it is found long before -/// this; exceeding it falls back rather than looping. -const MAX_LOCATE_ATTEMPTS: u32 = 64; - pub(crate) const MAX_VALUE: usize = 0x8000; const KEY_KIND: usize = 0x01; @@ -95,8 +77,7 @@ impl Slot { pub(crate) struct Store { file: shim::StoreFile, - /// Byte offset of the gigalocker window within the opened container: the - /// located extent's offset (an explicit start-sector override sets it). + /// Byte offset of the APFS-resolved gigalocker extent in the container. base: u64, slots: KVec, revision: u64, @@ -129,131 +110,23 @@ fn valid_key(key: &Key) -> bool { } impl Store { - /// Opens the store as the in-kernel raw-extent owner: it opens the iBoot - /// system container directly and serves only the gigalocker extent, with no - /// external helper and no hand-supplied sector. - /// - /// `start_sector == 0` (the default) searches for CRC-checked root records; - /// a non-zero sector skips the search. Neither method proves that the - /// window belongs to the current APFS `.gl` file, so writes are refused. + /// Opens the iBoot system container read-only, resolves `.gl` through APFS + /// metadata, then validates the record store at exactly that extent. + /// A nonzero start sector is an assertion against the APFS result, never + /// an override of it. Neither lookup nor record validation authorizes + /// raw writes: macOS's update protocol is still unverified. pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { - // A CRC-valid record signature does not establish the APFS file's - // physical extent. Once both roots move away from slot zero, several - // shifted windows pass the two-root test, and a shifted window can - // extend into APFS-owned blocks. Never write through this raw owner - // until the mapping has been checked against APFS file metadata. - // An explicit sector is not proof of ownership either. if writes_enabled { return Err(ENOTSUPP); } + let file = shim::StoreFile::open_block(OWNER_PATH, false)?; + let base = crate::xart_apfs::locate(&file)?; if start_sector != 0 { - let base = start_sector.checked_mul(512).ok_or(EINVAL)?; - return Self::open_based(OWNER_PATH, false, base); - } - Self::open_located(writes_enabled) - } - - /// Locates the gigalocker inside the iBoot system container with no external - /// help. The container is read sequentially and searched, at block-aligned - /// offsets, for a root record's signature — a `1` or `2` key kind with an - /// all-zero UUID, the shape of [`Key::root`]. Each base implied by such a hit - /// is probed read-only with the full CRC-checked [`open_based`]; because a - /// base shifted a few slots off the true origin still keeps both roots in its - /// window (passing the two-root test while dropping records that fall - /// outside), the candidate with the most live records is used for read-only - /// diagnostics. This heuristic cannot establish the APFS file's origin or - /// prove it is safe to write. Repair is never armed during discovery. - /// Bounded by [`MAX_LOCATE_ATTEMPTS`] probes; returns `ENODATA` if not found. - fn open_located(writes_enabled: bool) -> Result { - // The container is a writable block device; the block layer only grants - // a read-only handle to a read-only device, so scan it through a writable - // handle. This does not arm any write — discovery never mutates the - // container (see open_based_ext, called below with arm_writes == false). - let probe = shim::StoreFile::open_block(OWNER_PATH, true)?; - let size = probe.size()?; - if size < STORE_SIZE { - return Err(ENODATA); - } - - let mut buf: KVec = KVec::new(); - buf.resize(SCAN_CHUNK, 0, GFP_KERNEL)?; - let mut attempts: u32 = 0; - let mut off: u64 = 0; - - while off + HEADER_SIZE as u64 <= size { - let want = core::cmp::min(SCAN_CHUNK as u64, size - off) as usize; - probe.read_exact(off, &mut buf[..want])?; - - let mut p = 0usize; - while p + HEADER_SIZE <= want { - let kind = buf[p + KEY_KIND]; - let uuid_zero = buf[p + KEY_UUID..p + KEY_UUID + 16].iter().all(|&b| b == 0); - if (kind == 1 || kind == 2) && uuid_zero { - // A root signature anchors the gigalocker: the true base is - // this hit minus a whole number of slots (slots and blocks are - // both 0x1000-aligned, so every candidate stays block-aligned). - // The two-root test alone does NOT pin the base — a base - // shifted off the true origin by a few slots still keeps both - // roots inside its 6 MiB window, so it passes yet silently - // drops the live records that fall outside the shifted window. - // Probe candidates without writes and prefer the largest - // recovered record set for diagnostics. An equally valid - // shifted window is possible; this is not an APFS lookup. - let hit = off + p as u64; - let mut best: Option<(usize, usize, u64)> = None; // (valid, malformed, base) - let mut k: u64 = 0; - while k as usize <= MAX_SLOTS { - let step = k * SLOT_SIZE as u64; - if step > hit { - break; - } - let base = hit - step; - k += 1; - if base + STORE_SIZE > size { - continue; - } - attempts += 1; - if attempts > MAX_LOCATE_ATTEMPTS { - break; - } - if let Ok(store) = Self::open_based_ext(OWNER_PATH, true, false, base) { - let cand = (store.valid_records, store.malformed_records, base); - let better = match best { - None => true, - // More live records wins; ties break to fewer - // malformed, then to the higher base. The tie - // rule is a heuristic and may choose an - // up-shifted base when both roots moved. - Some(b) => { - cand.0 > b.0 - || (cand.0 == b.0 && cand.1 < b.1) - || (cand.0 == b.0 && cand.1 == b.1 && cand.2 > b.2) - } - }; - if better { - best = Some(cand); - } - } - } - // Re-open the diagnostic candidate. open_owner refuses - // writes, including automatic repair, on raw APFS extents. - if let Some((_, _, base)) = best { - return Self::open_based_ext(OWNER_PATH, true, writes_enabled, base); - } - if attempts > MAX_LOCATE_ATTEMPTS { - return Err(ENODATA); - } - } - p += BLOCK_SIZE; - } - - if want < SCAN_CHUNK { - break; + if start_sector.checked_mul(512).ok_or(EINVAL)? != base { + return Err(EINVAL); } - // Overlap one block so a signature on the boundary is not missed. - off += (SCAN_CHUNK - BLOCK_SIZE) as u64; } - Err(ENODATA) + Self::open_file(file, false, base) } /// Opens a caller-selected block mapping at offset zero. @@ -272,27 +145,15 @@ impl Store { /// device (the raw container) serves only its gigalocker extent. /// /// The block handle's writability equals `writes_enabled`, and repair runs - /// when writes are enabled. For the raw-extent owner, discovery instead - /// needs a writable handle (the block layer only grants a read-only handle - /// to a read-only device, and the container is writable) *without* arming - /// repair at an unconfirmed base — see [`open_based_ext`]. + /// when writes are enabled. The APFS owner is always read-only. fn open_based(path: &CStr, writes_enabled: bool, base: u64) -> Result { - Self::open_based_ext(path, writes_enabled, writes_enabled, base) + let file = shim::StoreFile::open_block(path, writes_enabled)?; + Self::open_file(file, writes_enabled, base) } - /// As [`open_based`], but with the block-handle writability (`dev_writable`) - /// decoupled from whether repair writes are armed (`arm_writes`). Discovery - /// of the raw-extent owner opens the writable container with - /// `dev_writable == true` yet `arm_writes == false`, so a candidate base is - /// fully scanned and CRC-validated without a single write landing at an - /// origin that has not yet been confirmed as the true gigalocker. - fn open_based_ext( - path: &CStr, - dev_writable: bool, - arm_writes: bool, - base: u64, - ) -> Result { - let file = shim::StoreFile::open_block(path, dev_writable)?; + /// Complete record validation using the same read-only handle with which + /// APFS ownership was resolved, avoiding a close/reopen window. + fn open_file(file: shim::StoreFile, arm_writes: bool, base: u64) -> Result { let size = file.size()?; if base % BLOCK_SIZE as u64 != 0 || STORE_SIZE % BLOCK_SIZE as u64 != 0 { return Err(EINVAL); @@ -540,8 +401,7 @@ impl Store { Ok(true) } - /// Byte offset of the served gigalocker window within the opened container: - /// the located extent's offset (an explicit start-sector override sets it). + /// Byte offset of the APFS-resolved gigalocker extent in the container. pub(crate) fn base(&self) -> u64 { self.base } diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index c3f17ad3c1171..767b9043d2ad5 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -14,12 +14,27 @@ systemctl enable aurora-sep.service The raw xART owner is currently read-only. Do not enable the installed `aurora-sep.service` or run `load-driver` on a shared macOS/Linux installation: -that script requests `xart_writes=1`, which the kernel now rejects while the -physical `.gl` extent is selected by scanning record signatures rather than -validated against APFS file metadata. The scan is ambiguous when live root -records have moved away from the start of the file. Read-only inspection does -not establish write safety. Re-enable provisioning only after the APFS mapping -and macOS-compatible record update protocol are verified on hardware. +that script requests `xart_writes=1`, which the kernel rejects. The kernel now +resolves the physical `.gl` extent from checksum-validated APFS metadata for +read-only operation; unsupported layouts fail closed. This removes the raw +record-signature locator, which was ambiguous when roots moved away from the +start of the file. APFS lookup alone does not establish write safety. Re-enable +provisioning only after macOS-compatible record updates and APFS ownership +invariants are verified on hardware. + +For read-only structural diagnostics on Linux, run +`sudo python3 apfs-xart-inspect.py /dev/disk/by-partlabel/iBootSystemContainer`. +The inspector checks APFS metadata checksums, resolves the latest complete +checkpoint's xART `.gl` file extent, and compares it with the legacy root-record +heuristic. It prints physical addresses and record metadata, not payloads. +It currently supports only a single-node object map and file-system tree and +cannot authorize raw writes. Apple's APFS reference is the format authority: +https://developer.apple.com/support/apple-file-system/Apple-File-System-Reference.pdf. +`python3 -m unittest discover -s tests -v` runs a synthetic regression case: +when the first live root moves to slot 10, the legacy heuristic chooses a +window shifted 90 APFS blocks beyond the actual `.gl` extent. This proves the +locator can misidentify the extent; it does not identify the historical cause +of any particular damaged container. For fingerprint support, apply `patches/libfprint-1.94.100-apple-sep.patch` to libfprint 1.94.100, build and diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md new file mode 100644 index 0000000000000..1ea3708dc0a2e --- /dev/null +++ b/tools/aurora-sep/XART-SAFETY.md @@ -0,0 +1,41 @@ +# Shared xART gigalocker safety gate + +The iBoot APFS container and its XART `.gl` file are shared with macOS. The +kernel owner must remain read-only. `apple_sep.xart_writes=1` is rejected even +with an explicit `xart_start_sector`; the latter is now only an assertion +against the APFS-resolved file extent. Do not enable provisioning services on a +dual-boot installation while this gate is in place. + +## Evidence collected on 2026-09-21 + +- A read-only inspection of `m2`'s iBoot partition found a complete, valid + checkpoint at transaction 57, one XART volume (role `0x100`), and one + unencrypted `.gl` extent: 6 MiB beginning at APFS block 1236. All 11 live + gigalocker records in that extent passed their payload CRC checks. The raw + signature heuristic happened to select the same block on this snapshot. +- A synthetic image with the first root moved to slot 10 reproduces a legacy + locator failure: the highest-scoring window begins 90 APFS blocks after the + true file start and extends beyond `.gl`. This proves ambiguity in that + locator, **not** the cause of the previously damaged installation. +- Static inspection of the macOS 27.2 AppleSEPManager kernel collection found + `gl_rec_write` allocating a fresh 0x9000-byte slot, writing it, then clearing + 0x1000 bytes at the former slot. The low-level write path also calls a disk + cache-synchronization ioctl. `gl_fixup_rev_init` validates records and + handles duplicates during open. These observations do not prove that our + APFS ownership and failure ordering match macOS. + +## Before authorizing writes + +1. Confirm the exact macOS vnode/raw-device and extent-locking contract, + including whether any APFS copy-on-write, snapshot, encryption, or remap + state can invalidate the physical mapping during Linux's lifetime. +2. Verify record ordering, duplicate selection, failed-write recovery, + barriers/cache synchronization, and revision behavior against both macOS + code and controlled non-production tests. +3. Compare a pre/post macOS-and-Linux power-cycle trace, with APFS checksums, + `.gl` mapping and records checked before and after. Keep a recoverable image + of the iBoot partition before the first write experiment. + +The current APFS locator supports only a complete checkpoint, single-node +object maps and file-system tree, and a single unencrypted 6 MiB `.gl` extent. +Unsupported layouts fail closed; this is not a general APFS implementation. diff --git a/tools/aurora-sep/apfs-xart-inspect.py b/tools/aurora-sep/apfs-xart-inspect.py new file mode 100644 index 0000000000000..63cc276646f5e --- /dev/null +++ b/tools/aurora-sep/apfs-xart-inspect.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +"""Read-only, metadata-only inspector for the iBoot APFS xART volume. + +This intentionally never prints gigalocker contents or attempts a write. +It rejects metadata that it cannot validate instead of guessing an extent. +Only single-node object maps and a single-node xART file-system tree are +supported; this is a diagnostic, not an APFS mount or write authorization. +""" + +import os +import struct +import sys +import zlib + +BLOCK = 4096 +MOD = 0xffffffff + + +def u16(buf, off): + return struct.unpack_from("= self.count: + raise ValueError("physical address out of partition") + data = os.pread(self.fd, BLOCK, paddr * BLOCK) + if len(data) != BLOCK: + raise ValueError("short block read") + if verify and u64(data, 0) != checksum(data): + raise ValueError("APFS checksum mismatch at block %d" % paddr) + return data + + +def node_entries(buf): + level, flags, count = u16(buf, 34), u16(buf, 32), u32(buf, 36) + fixed = bool(flags & 4) + start = 56 + u16(buf, 40) + key_start = start + u16(buf, 42) + val_end = BLOCK - (40 if flags & 1 else 0) + stride = 4 if fixed else 8 + if count > 512 or start < 56 or start + stride * count > key_start or key_start > val_end: + raise ValueError("invalid B-tree table") + entries = [] + for i in range(count): + at = start + i * stride + if fixed: + koff, voff = u16(buf, at), u16(buf, at + 2) + klen, vlen = 16, 16 + else: + koff, klen, voff, vlen = struct.unpack_from("= desc_count or not 2 <= length <= desc_count: + return False + if desc_base + (index + length - 1) % desc_count != sb_paddr: + return False + xid = u64(sb, 16) + try: + for j in range(length - 1): + mapping = disk.read(desc_base + (index + j) % desc_count) + if u32(mapping, 24) & 0xffff != 12 or u64(mapping, 16) != xid: + return False + if bool(u32(mapping, 32) & 1) != (j == length - 2): + return False + count = u32(mapping, 36) + if count > (BLOCK - 40) // 40: + return False + for i in range(count): + off = 40 + i * 40 + if u32(mapping, off + 8) != BLOCK: + return False + item = disk.read(u64(mapping, off + 32)) + if (u64(item, 8) != u64(mapping, off + 24) or + u64(item, 16) != xid or + u32(item, 24) != u32(mapping, off)): + return False + except ValueError: + return False + return True + + +def omap_lookup(disk, omap_paddr, oid, max_xid): + omap = disk.read(omap_paddr) + if u64(omap, 8) != omap_paddr: + raise ValueError("object-map identity mismatch") + tree = u64(omap, 48) + node = disk.read(tree) + level, flags, entries = node_entries(node) + if level: + raise ValueError("multilevel object map needs further inspection") + if not flags & 4: + raise ValueError("object map is not fixed-size") + matches = [] + for key, val in entries: + if u64(key, 0) == oid and u64(key, 8) <= max_xid: + matches.append((u64(key, 8), u32(val, 0), u64(val, 8), u32(val, 4))) + if not matches: + raise ValueError("object %d not found in map" % oid) + selected = max(matches) + if selected[1] & 1: + raise ValueError("object %d deleted at selected transaction" % oid) + return selected[2:] + + +def inspect(disk): + base = disk.read(0) + if base[32:36] != b"NXSB" or u32(base, 36) != BLOCK: + raise ValueError("unsupported container superblock") + desc_base, desc_count = u64(base, 112), u32(base, 104) + if desc_count < 1 or desc_count > 1024: + raise ValueError("unsupported checkpoint descriptor geometry") + checkpoints = [] + for paddr in range(desc_base, desc_base + desc_count): + try: + block = disk.read(paddr) + except ValueError: + continue + if (block[32:36] == b"NXSB" and u32(block, 36) == BLOCK and + checkpoint_complete(disk, block, paddr, desc_base, desc_count)): + checkpoints.append((u64(block, 16), paddr, block)) + if not checkpoints: + raise ValueError("no valid container superblock") + xid, cpaddr, sb = max(checkpoints) + print("partition_bytes=%d blocks=%d" % (disk.size, disk.count)) + print("block_zero_xid=%d" % u64(base, 16)) + print("valid_superblocks=%s" % [(x, p) for x, p, _ in checkpoints]) + print("selected_checkpoint_xid=%d block=%d" % (xid, cpaddr)) + print("container_omap=%d" % u64(sb, 160)) + fs_oids = [u64(sb, 184 + i * 8) for i in range(100)] + fs_oids = [v for v in fs_oids if v] + print("volume_oids=%s" % fs_oids) + for oid in fs_oids: + paddr, size = omap_lookup(disk, u64(sb, 160), oid, xid) + vol = disk.read(paddr) + if vol[32:36] != b"APSB" or u64(vol, 8) != oid: + raise ValueError("bad volume superblock") + role = u16(vol, 964) + print("volume oid=%d block=%d xid=%d role=0x%x" % + (oid, paddr, u64(vol, 16), role)) + if role != 0x100: + continue + vol_omap = u64(vol, 128) + root_oid = u64(vol, 136) + root_paddr, root_size = omap_lookup(disk, vol_omap, root_oid, xid) + root = disk.read(root_paddr) + level, flags, entries = node_entries(root) + print("xart_root oid=%d block=%d level=%d records=%d" % + (root_oid, root_paddr, level, len(entries))) + if level: + raise ValueError("multilevel xART FS tree needs further inspection") + files = [] + for key, val in entries: + if len(key) < 12 or len(val) < 8: + continue + k = u64(key, 0) + if k == ((9 << 60) | 2): + name_len = u32(key, 8) & 0x3ff + if name_len and 12 + name_len <= len(key): + name = key[12:12 + name_len - 1] + if name == b".gl": + files.append(u64(val, 0)) + print("gigalocker_file_ids=%s" % files) + if len(files) != 1: + raise ValueError("expected exactly one .gl file") + inode_values = [value for key, value in entries + if len(key) >= 8 and u64(key, 0) == ((3 << 60) | files[0])] + if len(inode_values) != 1 or len(inode_values[0]) < 92: + raise ValueError(".gl inode missing or short") + inode = inode_values[0] + private_id, inode_flags = u64(inode, 8), u64(inode, 48) + print("gigalocker_inode_private_id=%d flags=0x%x links=%d write_generation=%d" % + (private_id, inode_flags, u32(inode, 56), u32(inode, 64))) + print("xart_volume_flags=0x%x snapshot_count=%d" % (u64(vol, 264), u32(disk.read(vol_omap), 36))) + extents = [] + for key, val in entries: + if len(key) >= 16 and len(val) >= 24 and u64(key, 0) == ((8 << 60) | private_id): + logical, length_flags, physical = u64(key, 8), u64(val, 0), u64(val, 8) + length = length_flags & ((1 << 56) - 1) + if logical % BLOCK or length % BLOCK or physical + length // BLOCK > disk.count: + raise ValueError("invalid .gl extent") + extents.append((logical, length, physical, length_flags >> 56, u64(val, 16))) + extents.sort() + print("gigalocker_extents_logical_bytes_physical_blocks_flags=%s" % extents) + if not extents: + raise ValueError("no .gl extents") + if len(extents) == 1 and extents[0][0] == 0 and extents[0][1] == 0x600000: + inspect_raw_locator(disk, extents[0][2]) + scan_other_roots(disk, extents[0][2], extents[0][1]) + + +def inspect_raw_locator(disk, gl_block): + """Mirror the removed first-root candidate scoring, without writes.""" + slot_size, store_size = 0x9000, 0x600000 + gl_base = gl_block * BLOCK + gl = os.pread(disk.fd, store_size, gl_base) + if len(gl) != store_size: + raise ValueError("short gigalocker extent read") + roots = [] + records = [] + for idx in range(store_size // slot_size): + off = idx * slot_size + if gl[off + 1] in (1, 2) and gl[off + 2:off + 18] == bytes(16): + roots.append(idx) + kind = gl[off + 1] + if not kind: + continue + length = u32(gl, off + 0x12) + valid = (1 <= kind <= 4 and (kind > 2 or gl[off + 2:off + 18] == bytes(16)) + and 1 <= length <= 0x8000 and + zlib.crc32(gl[off + 0x22:off + 0x22 + length]) == u32(gl, off + 0x16)) + records.append((idx, kind, length, u64(gl, off + 0x1a), valid)) + print("gigalocker_records_slot_kind_length_revision_crc_valid=%s" % records) + print("raw_root_signature_slots=%s" % roots) + if not roots: + return + hit = gl_base + roots[0] * slot_size + candidates = [] + attempts = 0 + for k in range(4097): + step = k * slot_size + if step > hit: + break + base = hit - step + if base + store_size > disk.size: + continue + attempts += 1 + if attempts > 64: + break + raw = os.pread(disk.fd, store_size, base) + live = {} + malformed = 0 + for idx in range(store_size // slot_size): + off = idx * slot_size + kind = raw[off + 1] + if kind == 0: + continue + uid = raw[off + 2:off + 18] + length = u32(raw, off + 0x12) + crc = u32(raw, off + 0x16) + revision = u64(raw, off + 0x1a) + if not (1 <= kind <= 4 and (kind > 2 or uid == bytes(16)) and + 1 <= length <= 0x8000 and + zlib.crc32(raw[off + 0x22:off + 0x22 + length]) == crc): + malformed += 1 + continue + key = (kind, uid) + if key not in live or revision >= live[key]: + live[key] = revision + if (1, bytes(16)) in live and (2, bytes(16)) in live: + candidates.append((len(live), -malformed, base)) + if not candidates: + print("raw_locator=no_valid_candidate") + return + best = max(candidates) + print("raw_locator_candidates=%d attempts=%d" % (len(candidates), attempts)) + print("raw_locator_best_block=%d apfs_extent_block=%d shift_blocks=%d live_records=%d malformed=%d" % + (best[2] // BLOCK, gl_block, (best[2] - gl_base) // BLOCK, best[0], -best[1])) + + +def scan_other_roots(disk, gl_block, gl_size): + """Count CRC-valid root signatures outside APFS's current file extent.""" + outside = [] + for block in range(disk.count): + off = block * BLOCK + header = os.pread(disk.fd, 0x22, off) + if len(header) != 0x22 or header[1] not in (1, 2) or header[2:18] != bytes(16): + continue + length = u32(header, 0x12) + if not 1 <= length <= 0x8000: + continue + payload = os.pread(disk.fd, length, off + 0x22) + if len(payload) != length or zlib.crc32(payload) != u32(header, 0x16): + continue + if not gl_block <= block < gl_block + gl_size // BLOCK: + outside.append((block, header[1], u64(header, 0x1a))) + print("crc_valid_root_signatures_outside_apfs_gl=%s" % outside[:32]) + if len(outside) > 32: + print("crc_valid_root_signatures_outside_apfs_gl_additional=%d" % (len(outside) - 32)) + + +if __name__ == "__main__": + try: + inspect(Disk(sys.argv[1])) + except (ValueError, OSError, IndexError) as exc: + print("INCOMPLETE: %s" % exc, file=sys.stderr) + sys.exit(1) diff --git a/tools/aurora-sep/tests/test_xart_locator.py b/tools/aurora-sep/tests/test_xart_locator.py new file mode 100644 index 0000000000000..167c977ec2bd0 --- /dev/null +++ b/tools/aurora-sep/tests/test_xart_locator.py @@ -0,0 +1,49 @@ +"""Regression proof for the ambiguity of the legacy raw-header locator.""" + +import contextlib +import importlib.util +import io +from pathlib import Path +import struct +import tempfile +import unittest +import zlib + + +SCRIPT = Path(__file__).resolve().parents[1] / "apfs-xart-inspect.py" +SPEC = importlib.util.spec_from_file_location("apfs_xart_inspect", SCRIPT) +INSPECTOR = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(INSPECTOR) + + +def record(kind, revision): + out = bytearray(0x9000) + payload = b"test" + out[1] = kind + struct.pack_into(" Date: Fri, 25 Sep 2026 17:03:59 -0400 Subject: [PATCH 0062/1579] soc: apple: sep: port J414s enrollment and xART persistence fixes Carry only the SEP-related part of 95731b2390, including the SEP device-tree alias, calibration tooling, and fingerprint protocol changes. Exclude Thunderbolt and display modifications from this branch. Source-Commit: 95731b2390 --- .../arm64/boot/dts/apple/t602x-j414-j416.dtsi | 1 + drivers/soc/apple/Kconfig | 11 +- drivers/soc/apple/bio.rs | 27 +- drivers/soc/apple/sbio.rs | 550 ++++++++++++++---- drivers/soc/apple/sep-bio.h | 3 + drivers/soc/apple/sep.rs | 143 +++-- drivers/soc/apple/xart_apfs.rs | 99 +++- drivers/soc/apple/xart_store.rs | 26 +- tools/aurora-sep/README.md | 88 ++- tools/aurora-sep/XART-SAFETY.md | 48 +- tools/aurora-sep/apfs-xart-inspect.py | 65 ++- tools/aurora-sep/extract-mesa-calibration.py | 343 +++++++++++ tools/aurora-sep/load-driver | 8 +- .../libfprint-1.94.100-apple-sep.patch | 26 +- .../tests/test_extract_mesa_calibration.py | 72 +++ tools/aurora-sep/tests/test_xart_locator.py | 40 ++ 16 files changed, 1358 insertions(+), 192 deletions(-) create mode 100644 tools/aurora-sep/extract-mesa-calibration.py create mode 100644 tools/aurora-sep/tests/test_extract_mesa_calibration.py diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 3cff1b10057fe..9267fa9be28bd 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -25,6 +25,7 @@ disp0 = &display; disp0_piodma = &disp0_piodma; keyboard = &keyboard; + sep = &sep; sio = &sio; }; }; diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index d3d451751fff4..1567c3b177c05 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -123,10 +123,13 @@ config APPLE_SEP SEP-backed trusted key source so keyctl can seal keys to the enclave. The driver reads the existing machine-wide xART gigalocker directly from - the iBoot system container, locating it by its root records. It validates - the complete store and both root records before registering with SEP. - Raw APFS xART writes are refused until the extent and update protocol - can be verified; the module parameter cannot override this safety gate. + the iBoot system container, locating its .gl extent through read-only, + checksum-validated APFS metadata. It validates the complete store and + both root records before registering with SEP. + Raw APFS xART writes are disabled by default. An explicit xart_writes + opt-in is accepted only when the driver validates narrow physical + extent ownership invariants; this does not prove all macOS-compatible + update behavior. Keep it off until the target machine has been checked. Say Y here if you have an Apple silicon Mac. diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index e66710b6935cf..2bc1a492799c3 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -49,7 +49,9 @@ enum MatchResult { NotCompared = 2, } -pub(crate) const ENROL_STAGES: u32 = 8; +// A stage is an accepted capture, not a bucket of SEP's coverage percentage. +// Leave the final stage for the completed template. +pub(crate) const ENROL_STAGES: u32 = 16; const TOKEN_LIFETIME_NS: u64 = 10 * 1_000_000_000; @@ -180,7 +182,7 @@ ioctl_pod! { const MAGIC: u32 = 0xB1; -const IOC_GET_INFO: u32 = _IOR::(MAGIC, 0x01); +pub(crate) const IOC_GET_INFO: u32 = _IOR::(MAGIC, 0x01); const IOC_LIST: u32 = _IOR::(MAGIC, 0x02); const IOC_ENROL_START: u32 = _IOW::(MAGIC, 0x03); const IOC_ENROL_POLL: u32 = _IOR::(MAGIC, 0x04); @@ -414,6 +416,7 @@ pub(crate) struct Session { open: bool, op: Op, unseen: bool, + enrol_stage_unseen: bool, token: Option, } @@ -423,6 +426,7 @@ impl Session { open: false, op: Op::Idle, unseen: false, + enrol_stage_unseen: false, token: None, } } @@ -431,6 +435,7 @@ impl Session { self.open = false; self.op = Op::Idle; self.unseen = false; + self.enrol_stage_unseen = false; self.token = None; } @@ -443,6 +448,7 @@ pub(crate) struct Context<'a> { pub(crate) session: &'a mut Session, pub(crate) index: &'a IdentityIndex, pub(crate) sensor_present: bool, + pub(crate) live_identity_count: Option, } pub(crate) struct Handled { @@ -505,7 +511,14 @@ fn get_info(ctx: &mut Context<'_>, user: UserPtr) -> Result { enrolled: ctx.index.total() as u32, capacity: MAX_IDENTITIES as u32, enroll_stages: ENROL_STAGES, - reserved: [0; 3], + // reserved[0] is a validity flag. A zero count is only meaningful + // when SEP itself successfully answered LIST_IDENTITIES; older + // kernels and unavailable SEP contexts leave this flag clear. + reserved: [ + u32::from(ctx.live_identity_count.is_some()), + ctx.live_identity_count.unwrap_or(0) as u32, + 0, + ], }; UserSlice::new(user, core::mem::size_of::()) .writer() @@ -557,6 +570,8 @@ fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { guidance: Guidance::Place, percent: 0, }; + ctx.session.unseen = false; + ctx.session.enrol_stage_unseen = false; Ok(Handled { ret: 0, wake: false, @@ -586,6 +601,7 @@ pub(crate) fn enrol_advance( *percent = percent_now; *guidance = guide; session.unseen = true; + session.enrol_stage_unseen = true; return true; } } @@ -661,7 +677,9 @@ fn enrol_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { out.progress_percent = *percent; match terminal { None => { - out.state = if ctx.session.unseen { + // Guidance also wakes the poller, but it is not an image + // capture or a completed enrollment stage. + out.state = if ctx.session.enrol_stage_unseen { State::Progress } else { State::Pending @@ -683,6 +701,7 @@ fn enrol_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { .writer() .write(&out)?; ctx.session.unseen = false; + ctx.session.enrol_stage_unseen = false; ok() } diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 99665791198b6..91e5125b55acb 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -10,6 +10,24 @@ use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; impl SepData { + fn sbio_enrol_step(&self, op: &crate::sbio::SbioOp) -> Option> { + match self.sbio_call(op) { + SbioOutcome::Ok(payload) => Some(payload), + SbioOutcome::PrerequisiteMissing => { + dev_err!(self.dev, "enrol: {} prerequisite missing (status 0x01)\n", op.name()); + None + } + SbioOutcome::Status16 => { + dev_err!(self.dev, "enrol: {} refused (status 0x16)\n", op.name()); + None + } + SbioOutcome::Other => { + dev_err!(self.dev, "enrol: {} got no usable answer\n", op.name()); + None + } + } + } + fn sbio_expect_ok(&self, op: &crate::sbio::SbioOp) -> Option> { match self.sbio_call(op) { SbioOutcome::Ok(payload) => Some(payload), @@ -40,8 +58,14 @@ impl SepData { fn sbio_relay(&self, relay: &crate::sbio::SbioRelay<'_>) -> Option> { match self.sbio_transfer_raw(relay.opcode(), relay.name(), relay.payload()) { Ok(done) if done.status.is_ok() => Some(done.payload), - Ok(_) => None, - Err(_) => None, + Ok(done) => { + dev_err!(self.dev, "enrol: {} relay ({} bytes) refused with status {}\n", relay.name(), relay.payload().len(), done.status); + None + } + Err(e) => { + dev_err!(self.dev, "enrol: {} relay ({} bytes) failed ({:?})\n", relay.name(), relay.payload().len(), e); + None + } } } @@ -264,6 +288,7 @@ impl SepData { let listed = match self.enclave_identities_for(SBIO_PROBE_USER_ID) { Some(list) => list.len(), None => { + dev_warn!(self.dev, "enrol: identity enumeration failed; trying the existing context\n"); return self.enrol_into_existing_context(user); } }; @@ -288,10 +313,17 @@ impl SepData { proof: &crate::sbio::NoExistingCatacomb, ) -> bool { let system = crate::sbio::sbio_select_context(crate::sbio::ContextScope::SYSTEM, proof); - if self.sbio_expect_ok(&system).is_none() { + if self.sbio_enrol_step(&system).is_none() { return false; } + let Some(reply) = self.read_component_states() else { + dev_warn!(self.dev, "enrol: could not read component state after system SELECT_CONTEXT\n"); + return false; + }; + let states = crate::sbio::ComponentStates::new(&reply); + let owner_initial_state = states.state_for(crate::sbio::CatacombUser::OWNER.value()); + for (who, kind, what) in [ ( crate::sbio::CatacombUser::MASTER, @@ -304,14 +336,52 @@ impl SepData { c"owner catacomb", ), ] { + let Some(state) = states.state_for(who.value()) else { + dev_err!(self.dev, "enrol: {} is absent after system SELECT_CONTEXT\n", what); + return false; + }; + dev_info!(self.dev, "enrol: after system SELECT_CONTEXT {} state 0x{:x}\n", what, state); + if state & crate::sbio::COMPONENT_STATE_ACTIVE == 0 { + dev_err!(self.dev, "enrol: {} is not active after system SELECT_CONTEXT\n", what); + return false; + } + // On m2, state 0x3 refused SAVE_CATACOMB with status 0x6. The + // 0x04 save-pending bit is also used by the T2 Catacomb protocol; + // only persist a fresh component when SEP reports that bit. + if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING == 0 { + continue; + } if !self.save_catacomb(who, kind, what) { + dev_err!(self.dev, "enrol: could not save {} after selecting the system context\n", what); + return false; + } + } + + // The J414s owner exported successfully from state 0x3 in a bounded + // no-finger probe, although an earlier pre-UUID boot saw the master + // refuse an early save from the same state. Keep this path opt-in + // until a new enrollment and reboot prove that the owner blob belongs + // to the new user context. + if *module_parameters::j414s_persistent_enrol.value() != 0 { + if owner_initial_state == Some(0x3) + && !self.save_catacomb( + crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"fresh-context owner catacomb (user 501)", + ) + { + dev_err!(self.dev, "enrol: opt-in fresh-context owner export failed\n"); + return false; + } + if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() { + dev_err!(self.dev, "enrol: opt-in fresh context has no owner file\n"); return false; } } let per_user = crate::sbio::sbio_select_context(crate::sbio::ContextScope::user(user), proof); - if self.sbio_expect_ok(&per_user).is_none() { + if self.sbio_enrol_step(&per_user).is_none() { return false; } @@ -324,10 +394,11 @@ impl SepData { fn activate_protected_config(&self, user: crate::sbio::UserId) -> bool { let op = crate::sbio::sbio_protected_config(user); - let Some(config) = self.sbio_expect_ok(&op) else { + let Some(config) = self.sbio_enrol_step(&op) else { return false; }; if config.len() != crate::sbio::SBIO_PROTECTED_CONFIG_LEN { + dev_err!(self.dev, "enrol: protected configuration has {} bytes, expected {}\n", config.len(), crate::sbio::SBIO_PROTECTED_CONFIG_LEN); return false; } true @@ -339,19 +410,25 @@ impl SepData { let _ = self.sbio_call(&crate::sbio::sbio_cancel_operation()); } - let user = crate::sbio::UserId::new(SBIO_PROBE_USER_ID)?; + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + dev_err!(self.dev, "enrol: invalid probe user id\n"); + return None; + }; // 0x03 answers -3 until a user key bag is designated if !self.keybag_designated.load(Relaxed) { + dev_err!(self.dev, "enrol: no user keybag was designated\n"); return None; } if self.enrol_material.lock().is_none() { + dev_err!(self.dev, "enrol: no enrolment material is held\n"); return None; } // 0x03 answers 0x1 until an enrolment context exists if !self.open_enrolment_context(user) { + dev_err!(self.dev, "enrol: could not open the enclave enrolment context\n"); return None; } @@ -365,8 +442,8 @@ impl SepData { let op = crate::sbio::sbio_begin_enrol(user, crate::sbio::BE_AUTH_TYPE_ACM_CONTEXT, &acm_handle); - match self.sbio_call(&op) { - SbioOutcome::Ok(_payload) => { + match self.sbio_enrol_step(&op) { + Some(_payload) => { self.enrol_open.store(true, Relaxed); Some(OpenEnrolment { sep: self, @@ -377,6 +454,50 @@ impl SepData { } } + fn probe_owner_export(&self) { + if *module_parameters::probe_owner_export.value() == 0 { + return; + } + if self.templates_restored.load(Relaxed) + || crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_some() + || crate::catacomb::read(PRIVATE_TYPE_CATACOMB_MASTER).is_none() + || crate::catacomb::read(PRIVATE_TYPE_CATACOMB_USER).is_none() + { + dev_warn!(self.dev, "owner-probe: preconditions not met; no context selected\n"); + return; + } + let Some(identities) = self.enclave_identities_for(SBIO_PROBE_USER_ID) else { + dev_warn!(self.dev, "owner-probe: identity count unavailable; no context selected\n"); + return; + }; + let Some(proof) = crate::sbio::NoExistingCatacomb::from_zero_identities(identities.len()) else { + dev_warn!(self.dev, "owner-probe: enclave identity exists; no context selected\n"); + return; + }; + + // This is destructive to the enclave's *volatile* current context. + // It is opt-in and only reached after proving zero live identities. + let system = crate::sbio::sbio_select_context(crate::sbio::ContextScope::SYSTEM, &proof); + if self.sbio_expect_ok(&system).is_none() { + dev_warn!(self.dev, "owner-probe: system context selection refused\n"); + return; + } + let state = self.read_component_states().and_then(|reply| { + crate::sbio::ComponentStates::new(&reply) + .state_for(crate::sbio::CatacombUser::OWNER.value()) + }); + dev_info!(self.dev, "owner-probe: selected owner component state {:?}\n", state); + if !matches!(state, Some(value) if value & crate::sbio::COMPONENT_STATE_ACTIVE != 0) { + return; + } + let saved = self.save_catacomb( + crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"owner catacomb (user 501) diagnostic", + ); + dev_warn!(self.dev, "owner-probe: owner SAVE_CATACOMB and CONFIRM_SAVE completed {}\n", saved); + } + fn restore_all_components(&self) -> bool { // 0x6b is re-read before every component, not once up front. let user_id = SBIO_PROBE_USER_ID; @@ -384,27 +505,34 @@ impl SepData { let mut missing_files = false; let mut user_outcome = RestoreOutcome::NoStoredFile; - // 0x8002 (cold transition) is a success only on the owner component. - for (id, kind, what, cold_ok) in [ + // A 0x8002 reply is not sufficient evidence of restoration. Re-read + // the component state for every load, including the user component. + for (id, kind, what) in [ ( crate::sbio::CatacombUser::MASTER.value(), PRIVATE_TYPE_CATACOMB_MASTER, c"master catacomb", - false, ), ( crate::sbio::CatacombUser::OWNER.value(), PRIVATE_TYPE_CATACOMB_OWNER, c"owner catacomb", - true, ), - (user_id, PRIVATE_TYPE_CATACOMB_USER, c"user catacomb", false), + (user_id, PRIVATE_TYPE_CATACOMB_USER, c"user catacomb"), ] { let Some(reply) = self.read_component_states() else { return false; }; let states = crate::sbio::ComponentStates::new(&reply); - let outcome = self.restore_catacomb(states.state_for(id), id, kind, what, cold_ok); + let state = states.state_for(id); + let outcome = self.restore_catacomb(state, id, kind, what); + dev_info!( + self.dev, + "sbio: restore {} state {:?} outcome {:?}\n", + what, + state, + outcome + ); match outcome { RestoreOutcome::Restored | RestoreOutcome::AlreadyActive => any = true, RestoreOutcome::NoStoredFile => missing_files = true, @@ -419,7 +547,9 @@ impl SepData { user_outcome, RestoreOutcome::Restored | RestoreOutcome::AlreadyActive ); - let lockout = match self.restore_lockout() { + let lockout_outcome = self.restore_lockout(); + dev_info!(self.dev, "sbio: restore lockout outcome {:?}\n", lockout_outcome); + let lockout = match lockout_outcome { RestoreOutcome::Restored | RestoreOutcome::AlreadyActive => true, RestoreOutcome::EmptyTolerated => true, RestoreOutcome::NoStoredFile => { @@ -432,7 +562,7 @@ impl SepData { if missing_files { dev_warn!( self.dev, - "sbio: at least one of four artefacts has no file on disk; if enrolled before the four-artefact save, re-enrol once to write all four\n" + "sbio: at least one catacomb/lockout file is absent; match readiness depends on the user restore and device-view proof\n" ); } @@ -459,7 +589,6 @@ impl SepData { id: i32, kind: u8, what: &CStr, - cold_ok: bool, ) -> RestoreOutcome { let Some(state) = state else { return RestoreOutcome::Failed; @@ -490,18 +619,56 @@ impl SepData { Ok(done) if done.status.is_ok() => { self.confirm_active(id, what, LoadAnswer::StatusZero) } - Ok(done) - if done.status.answered() - == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) - && cold_ok => - { - self.confirm_active(id, what, LoadAnswer::ColdTransition) - } Ok(done) if done.status.answered() == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) => { - RestoreOutcome::Failed + let active = self.confirm_active(id, what, LoadAnswer::ColdTransition); + if id != SBIO_PROBE_USER_ID + || !matches!(&active, RestoreOutcome::Restored) + { + return active; + } + match self.log_identity_count(c"after first user cold-transition load") { + Some(n) if n > 0 => return active, + Some(0) => {} + _ => return RestoreOutcome::Failed, + } + + // Diagnostic only: one idempotent retry of the same + // host blob, with no context reset or save. An active + // state by itself did not prove a live identity on J414s. + let second = self.sbio_transfer_raw( + request.opcode(), request.name(), request.payload() + ); + let answer = match second { + Ok(done) if done.status.is_ok() => LoadAnswer::StatusZero, + Ok(done) + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) => + { + LoadAnswer::ColdTransition + } + Ok(done) + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_ALREADY_ACTIVE) => + { + LoadAnswer::AlreadyActive + } + Ok(done) => { + dev_warn!(self.dev, "sbio: second user LOAD_CATACOMB refused with status {}\n", done.status); + return RestoreOutcome::Failed; + } + Err(e) => { + dev_warn!(self.dev, "sbio: second user LOAD_CATACOMB transfer failed ({:?})\n", e); + return RestoreOutcome::Failed; + } + }; + let active = self.confirm_active(id, what, answer); + match (active, self.log_identity_count(c"after second user cold-transition load")) { + (RestoreOutcome::Restored, Some(n)) if n > 0 => RestoreOutcome::Restored, + _ => RestoreOutcome::Failed, + } } // 0x101 ALREADY_ACTIVE is a success Ok(done) @@ -510,7 +677,14 @@ impl SepData { { self.confirm_active(id, what, LoadAnswer::AlreadyActive) } - _ => RestoreOutcome::Failed, + Ok(done) => { + dev_warn!(self.dev, "sbio: {} LOAD_CATACOMB refused with status {}\n", what, done.status); + RestoreOutcome::Failed + } + Err(e) => { + dev_warn!(self.dev, "sbio: {} LOAD_CATACOMB transfer failed ({:?})\n", what, e); + RestoreOutcome::Failed + } } } } @@ -528,7 +702,9 @@ impl SepData { }; let states = crate::sbio::ComponentStates::new(&reply); - match states.state_for(id) { + let state = states.state_for(id); + dev_info!(self.dev, "sbio: {} LOAD_CATACOMB answered {} with post-load state {:?}\n", what, answer.describe(), state); + match state { Some(state) if state & crate::sbio::COMPONENT_STATE_ACTIVE != 0 => { RestoreOutcome::Restored } @@ -575,7 +751,14 @@ impl SepData { } fn save_all_components(&self, user: crate::sbio::UserId) -> bool { - let mut all = true; + if *module_parameters::j414s_persistent_enrol.value() != 0 { + return self.save_all_components_user_first(user); + } + // On m2fix11, a completed enrollment marked both master and user 0x7, + // but saving and confirming the user first was followed by master + // SAVE_CATACOMB status 0x6. Try the master before the user at this + // completion gate; post-match persistence keeps its separate order. + // Re-read state before each save and never send an unconditional one. for (who, kind, what) in [ ( crate::sbio::CatacombUser::MASTER, @@ -593,47 +776,144 @@ impl SepData { c"user catacomb", ), ] { - if !self.save_catacomb(who, kind, what) { - all = false; + let Some(reply) = self.read_component_states() else { + dev_err!(self.dev, "enrol: cannot read component state before saving {}\n", what); + return false; + }; + let states = crate::sbio::ComponentStates::new(&reply); + let Some(state) = states.state_for(who.value()) else { + dev_err!(self.dev, "enrol: {} is absent at enrollment completion\n", what); + return false; + }; + dev_info!(self.dev, "enrol: completed {} state 0x{:x}\n", what, state); + if state & crate::sbio::COMPONENT_STATE_ACTIVE == 0 { + dev_err!(self.dev, "enrol: {} is not active at enrollment completion\n", what); + return false; + } + if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING != 0 { + if !self.save_catacomb(who, kind, what) { + return false; + } + } else if who.value() == user.value() { + dev_err!(self.dev, "enrol: completed user catacomb is not marked for saving\n"); + return false; + } + if (who.value() == user.value() + || who.value() == crate::sbio::CatacombUser::MASTER.value()) + && crate::catacomb::read(kind).is_none() + { + dev_err!(self.dev, "enrol: {} has no durable host file\n", what); + return false; } } if !self.save_lockout() { - all = false; + dev_err!(self.dev, "enrol: lockout persistence failed\n"); + return false; } - all + true } - fn save_after_match(&self, user: crate::sbio::UserId) -> bool { + fn save_all_components_user_first(&self, user: crate::sbio::UserId) -> bool { + if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() { + dev_err!(self.dev, "enrol: opt-in completion has no owner Catacomb file\n"); + return false; + } + // Post-match persistence saves the user before the master because the + // user save can change system material. Test that order for a completed + // J414s enrollment, without changing the default path on other Macs. + for (who, kind, what) in [ + ( + crate::sbio::CatacombUser::enrolling(user), + PRIVATE_TYPE_CATACOMB_USER, + c"opt-in completed user catacomb", + ), + ( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"opt-in post-user master catacomb", + ), + ] { + let Some(reply) = self.read_component_states() else { + dev_err!(self.dev, "enrol: opt-in cannot read state before saving {}\n", what); + return false; + }; + let state = crate::sbio::ComponentStates::new(&reply).state_for(who.value()); + dev_info!(self.dev, "enrol: opt-in {} state {:?}\n", what, state); + let Some(state) = state else { + return false; + }; + if state & crate::sbio::COMPONENT_STATE_ACTIVE == 0 { + return false; + } + if who.value() == user.value() + && state & crate::sbio::COMPONENT_STATE_SAVE_PENDING == 0 + { + dev_err!(self.dev, "enrol: opt-in user catacomb is not save-pending\n"); + return false; + } + if !self.save_catacomb(who, kind, what) { + return false; + } + if crate::catacomb::read(kind).is_none() { + dev_err!(self.dev, "enrol: opt-in {} has no durable host file\n", what); + return false; + } + } if !self.save_lockout() { - dev_err!( - self.dev, - "verify: post-match lockout persistence failed; refusing to claim the AP and SEP anti-replay state are synchronized\n" - ); + dev_err!(self.dev, "enrol: opt-in lockout persistence failed\n"); return false; } + true + } - if !self.save_catacomb( - crate::sbio::CatacombUser::enrolling(user), - PRIVATE_TYPE_CATACOMB_USER, - c"post-match user catacomb", - ) { + fn save_after_match(&self, user: crate::sbio::UserId) -> bool { + if !self.save_lockout() { dev_err!( self.dev, - "verify: post-match user-catacomb persistence failed; the next boot may reject the older AP blob\n" + "verify: post-match lockout persistence failed; refusing to claim the AP and SEP anti-replay state are synchronized\n" ); return false; } - if !self.save_catacomb( - crate::sbio::CatacombUser::MASTER, - PRIVATE_TYPE_CATACOMB_MASTER, - c"post-match system (id -1) material snapshot", - ) { - dev_err!( - self.dev, - "verify: post-match system material snapshot failed; next boot may reinstall pre-match material and the user load could cold-transition (0x8002)\n" - ); - return false; + // Matching can update a template without making every component + // saveable. The user must be considered before the master, and 0x6b + // must be re-read after each save because saving the user can change + // the master's state. Never claim persistence without host copies. + for (who, kind, what) in [ + ( + crate::sbio::CatacombUser::enrolling(user), + PRIVATE_TYPE_CATACOMB_USER, + c"post-match user catacomb", + ), + ( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"post-match master catacomb", + ), + ] { + let Some(reply) = self.read_component_states() else { + dev_err!(self.dev, "verify: cannot read state before saving {}\n", what); + return false; + }; + let states = crate::sbio::ComponentStates::new(&reply); + let Some(state) = states.state_for(who.value()) else { + dev_err!(self.dev, "verify: {} is absent after matching\n", what); + return false; + }; + if state & crate::sbio::COMPONENT_STATE_ACTIVE == 0 { + dev_err!(self.dev, "verify: {} is not active after matching (state 0x{:x})\n", what, state); + return false; + } + if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING != 0 + && !self.save_catacomb(who, kind, what) + { + dev_err!(self.dev, "verify: {} persistence failed\n", what); + return false; + } + if crate::catacomb::read(kind).is_none() { + dev_err!(self.dev, "verify: {} has no durable host file\n", what); + return false; + } } if !self.resnapshot_identity_keybag() { @@ -647,33 +927,49 @@ impl SepData { true } - fn save_catacomb(&self, who: crate::sbio::CatacombUser, kind: u8, _what: &CStr) -> bool { + fn save_catacomb(&self, who: crate::sbio::CatacombUser, kind: u8, what: &CStr) -> bool { let selector = crate::sbio::SaveSelector::new(who); - let Some(blob) = self.sbio_expect_ok(&crate::sbio::sbio_save_catacomb(&selector)) else { - return false; + let save = match self.sbio_transfer(&crate::sbio::sbio_save_catacomb(&selector)) { + Ok(done) if done.status.is_ok() => done, + Ok(done) => { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB refused with status {}\n", what, done.status); + return false; + } + Err(e) => { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB transfer failed ({:?})\n", what, e); + return false; + } }; + let blob = save.payload; if blob.len() < crate::sbio::SBIO_SAVED_MIN || blob.len() > crate::sbio::SBIO_SAVED_MAX { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB returned invalid length {}\n", what, blob.len()); return false; } let at = crate::sbio::SBIO_SAVED_USER_ID_AT; let carried = i32::from_le_bytes([blob[at], blob[at + 1], blob[at + 2], blob[at + 3]]); if carried != who.value() { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB returned user id {}, expected {}\n", what, carried, who.value()); return false; } if crate::catacomb::write(kind, &blob).is_err() { + dev_err!(self.dev, "enrol: {} SAVE_CATACOMB host persistence failed\n", what); return false; } - if self - .sbio_expect_ok(&crate::sbio::sbio_confirm_save(&selector)) - .is_none() - { - return false; + match self.sbio_transfer(&crate::sbio::sbio_confirm_save(&selector)) { + Ok(done) if done.status.is_ok() => true, + Ok(done) => { + dev_err!(self.dev, "enrol: {} CONFIRM_SAVE refused with status {}\n", what, done.status); + false + } + Err(e) => { + dev_err!(self.dev, "enrol: {} CONFIRM_SAVE transfer failed ({:?})\n", what, e); + false + } } - true } fn save_lockout(&self) -> bool { @@ -855,8 +1151,9 @@ impl SepData { self.sks_designate_user_keybag(handle, stored.secret()); self.sks_machine_refkey(handle, stored.secret()); let prepared = self.cold_match_continue(handle, uuid); - self.ensure_restored_after(prepared); self.attach_bringup(); + self.ensure_restored_after(prepared); + self.probe_owner_export(); self.touchid_started.store(true, Relaxed); Ok(()) } @@ -878,7 +1175,7 @@ impl SepData { if !self.templates_restored.load(Relaxed) { dev_err!( self.dev, - "verify: refusing before touching the sensor; cold-match prep or restore did not complete, enclave holds no template (every match refused 0x1). A restore failure, not a non-matching finger\n" + "verify: refusing before touching the sensor; cold-match prep, restore, or device-view identity proof did not complete. This is not a non-matching finger\n" ); self.finish_verify( bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), @@ -1031,6 +1328,8 @@ impl SepData { } pub(crate) fn run_enrolment(&self) { + self.enrol_frames_accepted.store(0, Relaxed); + if !self.bring_sensor_online() { self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); let _ = sensor::idle(); @@ -1038,6 +1337,7 @@ impl SepData { } let Some(open) = self.begin_enrolment_on_enclave() else { + dev_err!(self.dev, "enrol: the enclave would not start an enrolment; see the preceding step\n"); self.finish_enrolment(Err(ENROL_STATUS_ENCLAVE)); let _ = sensor::idle(); return; @@ -1045,6 +1345,7 @@ impl SepData { let mut counter: u32 = 0; let mut enrolment_completed = false; + let mut enrolment_persisted = false; let mut identity: Option<[u8; bio::UUID_LEN]> = None; let outcome = loop { @@ -1104,18 +1405,18 @@ impl SepData { if let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) { if !self.save_all_components(user) { - dev_warn!( + dev_err!( + self.dev, + "enrol: enclave completed but Catacomb persistence failed; not reporting enrollment success\n" + ); + } else if !self.resnapshot_identity_keybag() { + dev_err!( self.dev, - "enrol: enrolment succeeded but at least one of four artefacts was not persisted; works until the next reboot (see which component above)\n" + "enrol: Catacombs saved but identity-bag snapshot failed; not reporting enrollment success\n" ); } else { self.templates_restored.store(true, Relaxed); - if !self.resnapshot_identity_keybag() { - dev_warn!( - self.dev, - "enrol: catacombs saved but post-commit identity-bag snapshot was not; template works this boot but is not reboot-persistent\n" - ); - } + enrolment_persisted = true; } } @@ -1129,10 +1430,10 @@ impl SepData { self.note_capture_end(); if let Some(result) = outcome { - let filed = match (result, identity) { - (Ok(()), Some(uuid)) => Ok(uuid), - (Ok(()), None) => Err(ENROL_STATUS_UNFILED), - (Err(status), _) => Err(status), + let filed = match (result, identity, enrolment_persisted) { + (Ok(()), Some(uuid), true) => Ok(uuid), + (Ok(()), _, _) => Err(ENROL_STATUS_UNFILED), + (Err(status), _, _) => Err(status), }; self.finish_enrolment(filed); } @@ -1140,14 +1441,17 @@ impl SepData { fn register_sensor(&self, id: &sensor::Identifier) -> bool { let stage = self.bringup.load(Relaxed); + dev_info!(self.dev, "sbio: registering sensor at bring-up stage {}\n", stage); if stage >= BRINGUP_ESTABLISHED { + let _ = self.log_identity_count(c"before CLEAR_STATE for sensor re-registration"); if self .sbio_expect_ok(&crate::sbio::sbio_clear_state()) .is_none() { return false; } + let _ = self.log_identity_count(c"after CLEAR_STATE for sensor re-registration"); } if stage == BRINGUP_FRESH { @@ -1173,6 +1477,7 @@ impl SepData { match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); + let _ = self.log_identity_count(c"after sensor serial registration"); true } _ => false, @@ -1330,6 +1635,7 @@ impl SepData { if listed && !self.device_view_synced.xchg(true, Relaxed) { let count = self.log_identity_count(c"after device-view synchronisation"); + dev_info!(self.dev, "sbio: device-view identity count {:?}, restore-ready before proof {}\n", count, self.templates_restored.load(Relaxed)); match count { Some(0) => { self.templates_restored.store(false, Relaxed); @@ -1348,9 +1654,12 @@ impl SepData { self.templates_restored.store(false, Relaxed); } } + dev_info!(self.dev, "sbio: restore-ready after device-view proof {}\n", self.templates_restored.load(Relaxed)); if self.templates_restored.load(Relaxed) { self.reconcile_identities(); } + } else if !listed { + dev_warn!(self.dev, "sbio: device-view synchronisation or match policy unavailable\n"); } true @@ -1613,10 +1922,11 @@ impl SepData { }; let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + dev_err!(self.dev, "enrol: invalid user id before image processing\n"); return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; if self - .sbio_expect_ok(&crate::sbio::sbio_prepare_image_processing()) + .sbio_enrol_step(&crate::sbio::sbio_prepare_image_processing()) .is_none() { return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); @@ -1631,7 +1941,7 @@ impl SepData { user, crate::shim::monotonic_ns(), ); - if self.sbio_expect_ok(&init).is_none() { + if self.sbio_enrol_step(&init).is_none() { return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); } @@ -1641,10 +1951,11 @@ impl SepData { } drop(capture); - let Some(assessment) = self.sbio_expect_ok(&crate::sbio::sbio_image_assessment()) else { + let Some(assessment) = self.sbio_enrol_step(&crate::sbio::sbio_image_assessment()) else { return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; if assessment.len() < crate::sbio::ASSESS_MIN_LEN { + dev_err!(self.dev, "enrol: image assessment short ({} bytes)\n", assessment.len()); return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); } let usable = assessment[crate::sbio::ASSESS_USABLE_ENROL] != 0; @@ -1652,10 +1963,11 @@ impl SepData { return ImageOutcome::Retry; } - let Some(result) = self.sbio_expect_ok(&crate::sbio::sbio_enrolment_result()) else { + let Some(result) = self.sbio_enrol_step(&crate::sbio::sbio_enrolment_result()) else { return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; let Some(parsed) = crate::sbio::EnrolmentResult::parse(&result) else { + dev_err!(self.dev, "enrol: malformed enrolment result ({} bytes)\n", result.len()); return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; @@ -1664,11 +1976,17 @@ impl SepData { } let percent = parsed.progress_percent(); + // Coverage can stop changing while SEP continues accepting frames. + // Count accepted captures so each one remains visible to libfprint. + let accepted = self.enrol_frames_accepted.fetch_add(1, Relaxed) + 1; + let stage = if parsed.complete() { + bio::ENROL_STAGES + } else { + accepted.min(bio::ENROL_STAGES - 1) + }; ImageOutcome::Progress { - stage: (percent * bio::ENROL_STAGES) - .div_ceil(100) - .min(bio::ENROL_STAGES), + stage, percent, complete: parsed.complete(), has_template: parsed.has_template(), @@ -2082,6 +2400,16 @@ impl SepData { if cmd == bio::IOC_ATTEST { return self.bio_attest(arg); } + // Query SEP outside the bio session/index locks. A host index can + // outlive its SEP identity after a failed cold restore; userspace + // must not mistake that index for proof of a live template. + let live_identity_count = if cmd == bio::IOC_GET_INFO + && self.touchid_started.load(Relaxed) + { + self.enclave_identity_count() + } else { + None + }; let handled = { let mut session = self.bio_session.lock(); let index = self.bio_index.lock(); @@ -2089,6 +2417,7 @@ impl SepData { session: &mut session, index: &index, sensor_present: self.sensor_present.load(Relaxed), + live_identity_count, }; bio::ioctl(&mut ctx, cmd, arg)? }; @@ -2146,32 +2475,42 @@ impl SepData { }) } - fn ensure_restored_after(&self, _prepared: bool) { - let restored = self.restore_all_components(); - self.templates_restored.store(restored, Relaxed); - if !restored { - dev_err!( - self.dev, - "matching unavailable this boot: restore did not complete, enclave holds no template (every match refused 0x1). Not the sensor or the finger; on-disk enrolments intact\n" - ); - return; - } - + fn ensure_restored_after(&self, prepared: bool) { + // Probe credential ordering without assuming it fixes cold restore. + // J414s still answered 0x8002 and listed zero identities with this + // credential established, so the identity proof below remains required. + let mut unlocked = false; if self.keybag_designated.load(Relaxed) { if let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) { let special = user.special_handle(); if let Ok(keybag::State::Present(stored)) = keybag::read(keybag::Slot::Identity) { - let _ = self.sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { + unlocked = self.sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { self.sks_req_unlock_special(special, stored.secret(), healthy) - }); + }).is_some(); } } } + let match_context = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) + .map(|user| self.establish_scrd_match_context(user)) + .unwrap_or(false); + dev_info!(self.dev, "sbio: pre-restore special unlock {}, SCRD match context {}\n", unlocked, match_context); - // enclave's match arm asserts a non-empty ACM context (<= 32 bytes) - if let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) { - self.establish_scrd_match_context(user); + let restored = self.restore_all_components(); + dev_info!(self.dev, "sbio: cold-match preparation {}, component restore {}\n", prepared, restored); + let identities = self.log_identity_count(c"after Catacomb restore with sensor registered"); + let identity_ready = matches!(identities, Some(n) if n > 0); + let ready = prepared && match_context && restored && identity_ready + && self.device_view_synced.load(Relaxed) && self.prove_restore(); + self.templates_restored.store(ready, Relaxed); + if !ready { + dev_err!( + self.dev, + "matching unavailable this boot: cold-match preparation, SCRD credential, Catacomb restore, device view, or identity proof did not complete; template state is unproven\n" + ); + return; } + self.reconcile_identities(); + } fn cold_match_continue( @@ -2260,7 +2599,7 @@ impl SepData { true } - fn log_identity_count(&self, _when: &CStr) -> Option { + fn enclave_identity_count(&self) -> Option { let op = crate::sbio::sbio_list_identities(); let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { return None; @@ -2269,6 +2608,15 @@ impl SepData { Some(records.count()) } + fn log_identity_count(&self, when: &CStr) -> Option { + let Some(count) = self.enclave_identity_count() else { + dev_warn!(self.dev, "sbio: identity list unavailable {}\n", when); + return None; + }; + dev_info!(self.dev, "sbio: identity count {} {}\n", count, when); + Some(count) + } + fn enclave_lists(&self, uuid: &[u8; bio::UUID_LEN]) -> Option { let op = crate::sbio::sbio_list_identities(); let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { @@ -2728,7 +3076,9 @@ static_assert!(SBIO_STATUS_ALREADY_ACTIVE != SBIO_STATUS_COLD_TRANSITION); pub(crate) const COMPONENT_STATE_COLD: u32 = 0x1; pub(crate) const COMPONENT_STATE_ACTIVE: u32 = 0x2; +pub(crate) const COMPONENT_STATE_SAVE_PENDING: u32 = 0x4; static_assert!(COMPONENT_STATE_COLD != COMPONENT_STATE_ACTIVE); +static_assert!(COMPONENT_STATE_SAVE_PENDING & (COMPONENT_STATE_COLD | COMPONENT_STATE_ACTIVE) == 0); pub(crate) const COMPONENT_PAIR_LEN: usize = 8; diff --git a/drivers/soc/apple/sep-bio.h b/drivers/soc/apple/sep-bio.h index 7ad0b814af648..2ec3949eaa665 100644 --- a/drivers/soc/apple/sep-bio.h +++ b/drivers/soc/apple/sep-bio.h @@ -53,6 +53,9 @@ struct sep_bio_info { __u32 reserved[3]; }; +/* reserved[0] flags; reserved[1] is a live SEP identity count only when valid. */ +#define SEP_BIO_INFO_LIVE_COUNT_VALID 1u + struct sep_bio_list { __u32 count; __u32 reserved; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 5f128c7fa005d..63e3654a290cf 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -78,6 +78,7 @@ use kernel::{ const SETTLE_MS: time::Msecs = 200; const FIRST_RESPONSE_MS: time::Msecs = 3000; +const RNG_SURVEY_ATTEMPTS: u32 = 5; const SETTLE_WORK_ID: u64 = 1; @@ -333,6 +334,7 @@ impl LoadAnswer { } } +#[derive(Debug)] enum RestoreOutcome { Restored, AlreadyActive, @@ -642,10 +644,14 @@ struct SepData { #[pin] fv_volumes: Mutex>, + enrol_frames_accepted: Atomic, + rng_shutdown: Atomic, rng_failures: Atomic, + rng_survey_attempts: Atomic, + rx: rxring::RxRing, rx_count: Atomic, @@ -727,7 +733,7 @@ impl SepData { let base = store.base(); dev_info!( dev, - "xART: APFS-resolved read-only .gl extent (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", + "xART: APFS-resolved .gl extent (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", base, slots, records, @@ -811,8 +817,10 @@ impl SepData { rng <- new_mutex!(None), machine_refkey <- new_mutex!(None), fv_volumes <- new_mutex!(KVec::new()), + enrol_frames_accepted: Atomic::new(0), rng_shutdown: Atomic::new(false), rng_failures: Atomic::new(0), + rng_survey_attempts: Atomic::new(0), rx: rxring::RxRing::new(), rx_count: Atomic::new(0), settle_mark: Atomic::new(0), @@ -1093,40 +1101,42 @@ impl SepData { } fn control_survey(&self) { - for param in [0x00u8, 0x01, 0xff] { let _ = self.control_request(&proto::op_nop(param)); } let _ = self.control_request(&proto::op_security_mode()); - let mut words = [0u32; 4]; - let mut got = 0; - for w in words.iter_mut() { - match self.get_entropy_word() { - Ok(v) => { - *w = v; - got += 1; - } - Err(_) => { - break; - } - } + let _state = self.control.lock(); + } + + fn survey_hwrng(&self) -> Result<()> { + // A working control endpoint must answer four consecutive draws before + // it is exposed to the kernel RNG core. Early attach can beat the SEP + // endpoint exchange, so this runs only after that exchange settles. + for _ in 0..4 { + let _ = self.get_entropy_word()?; } + self.register_hwrng() + } - if got == words.len() { - match self.register_hwrng() { - Ok(()) => {}, - Err(e) => dev_err!(self.dev, "hwrng: registration failed: {:?}\n", e), + fn tick_ready(this: &Arc) { + if this.rng.lock().is_some() { + return; + } + let attempt = this.rng_survey_attempts.load(Relaxed) + 1; + if attempt > RNG_SURVEY_ATTEMPTS { + return; + } + this.rng_survey_attempts.store(attempt, Relaxed); + match this.survey_hwrng() { + Ok(()) => dev_info!(this.dev, "hwrng: registered after ready-phase survey (attempt {})\n", attempt), + Err(e) if attempt < RNG_SURVEY_ATTEMPTS => { + dev_warn!(this.dev, "hwrng: survey attempt {} failed ({:?}); retrying\n", attempt, e); + Self::arm_settle(this); } - } else { - dev_err!( - self.dev, - "hwrng: not registering, the source did not answer during the survey\n" - ); + Err(e) => dev_err!(this.dev, "hwrng: survey failed after {} attempts ({:?})\n", attempt, e), } - - let _state = self.control.lock(); } @@ -1310,6 +1320,17 @@ impl SepData { return; }; + if *module_parameters::xarm_trace.value() != 0 { + dev_info!( + self.dev, + "xarm: op 0x{:02x} request_len {} reply_status 0x{:02x} reply_len {}\n", + req.opcode, + req.length, + done.reply.status, + done.reply.length + ); + } + if done.reply_bytes > 0 { if self.ool_write(&self.ool_xarm, 0, &staging[..done.reply_bytes]).is_err() { self.fail_xarm(req.tag); @@ -1551,6 +1572,7 @@ impl SepData { user: crate::sbio::UserId, ) -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { if !self.scrd_ready() { + dev_err!(self.dev, "enrol: SCRD endpoint is not ready\n"); return None; } @@ -1562,54 +1584,74 @@ impl SepData { (material.special, Secret(copy)) }; - let (status, _) = self.scrd_command(&crate::scrd::scrd_initialize())?; + let Some((status, _)) = self.scrd_command(&crate::scrd::scrd_initialize()) else { + dev_err!(self.dev, "enrol: SCRD initialize did not answer\n"); + return None; + }; if status != 0 { + dev_err!(self.dev, "enrol: SCRD initialize refused (status {})\n", status); return None; } - let (status, out) = self.scrd_command(&crate::scrd::scrd_context_create_tracked(user.value()))?; + let Some((status, out)) = self.scrd_command(&crate::scrd::scrd_context_create_tracked(user.value())) else { + dev_err!(self.dev, "enrol: SCRD context create did not answer\n"); + return None; + }; if status != 0 || out.len() < crate::scrd::SCRD_ACM_HANDLE_LEN { + dev_err!(self.dev, "enrol: SCRD context create failed (status {}, {} reply bytes)\n", status, out.len()); return None; } let mut acm_handle = [0u8; crate::scrd::SCRD_ACM_HANDLE_LEN]; acm_handle.copy_from_slice(&out[..crate::scrd::SCRD_ACM_HANDLE_LEN]); - let (status, _) = self.scrd_command(&crate::scrd::scrd_context_externalize(&acm_handle))?; + let Some((status, _)) = self.scrd_command(&crate::scrd::scrd_context_externalize(&acm_handle)) else { + dev_err!(self.dev, "enrol: SCRD context externalize did not answer\n"); + return None; + }; if status != 0 { + dev_err!(self.dev, "enrol: SCRD context externalize refused (status {})\n", status); return None; } - let out = self.sks_send(self.sks_req_verify_secret(special, &secret, &acm_handle))?; + let Some(out) = self.sks_send(self.sks_req_verify_secret(special, &secret, &acm_handle)) else { + dev_err!(self.dev, "enrol: SKS verify-secret did not answer\n"); + return None; + }; if out.reply.status != 0 { + dev_err!(self.dev, "enrol: SKS verify-secret refused (status {})\n", out.reply.status); return None; } - let (status, out) = self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle))?; + let Some((status, out)) = self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle)) else { + dev_err!(self.dev, "enrol: SCRD Touch ID policy check did not answer\n"); + return None; + }; let satisfied = out.len() >= 4 && u32::from_le_bytes([out[0], out[1], out[2], out[3]]) != 0; if status != 0 || !satisfied { + dev_err!(self.dev, "enrol: SCRD Touch ID policy rejected (status {}, satisfied {})\n", status, satisfied); return None; } Some(acm_handle) } - fn establish_scrd_match_context(&self, user: crate::sbio::UserId) { + fn establish_scrd_match_context(&self, user: crate::sbio::UserId) -> bool { if !self.scrd_ready() { - return; + return false; } let Some(du) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) else { - return; + return false; }; let special = du.special_handle(); let stored = match keybag::read(keybag::Slot::Identity) { Ok(keybag::State::Present(s)) => s, _ => { - return; + return false; } }; - let _ = (|| -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { + let established = (|| -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { let (status, _) = self.scrd_command(&crate::scrd::scrd_initialize())?; if status != 0 { return None; @@ -1631,7 +1673,9 @@ impl SepData { } self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle))?; Some(acm_handle) - })(); + })().is_some(); + dev_info!(self.dev, "scrd: cold-match credential established {}\n", established); + established } @@ -1730,7 +1774,8 @@ impl SepData { match this.phase.load(Relaxed) { PHASE_ATTACH => Self::tick_attach(this), PHASE_EXCHANGE => Self::tick_exchange(this), - _ => {} + PHASE_READY => Self::tick_ready(this), + _ => {}, } } @@ -1807,8 +1852,9 @@ impl SepData { } } - this.phase.store(PHASE_READY, Relaxed); this.run_bringup(); + this.phase.store(PHASE_READY, Relaxed); + Self::arm_settle(this); } fn endpoint_present(&self, id: u8) -> bool { @@ -2093,13 +2139,6 @@ impl platform::Driver for SepDriver { _info: Option<&()>, ) -> impl PinInit { let dev: &device::Device = pdev.as_ref(); - if *module_parameters::xart_writes.value() != 0 { - dev_err!( - dev, - "raw APFS xART writes are blocked until the .gl update protocol and ownership invariants are validated\n" - ); - return Err(ENOTSUPP); - } if *module_parameters::provision_keybag.value() != 0 && *module_parameters::xart_writes.value() == 0 { @@ -2191,7 +2230,7 @@ module! { params: { xart_writes: u8 { default: 0, - description: "Request shared xART writes (refused until the macOS update protocol is verified)", + description: "Opt in to shared xART writes only if APFS proves a single unsnapshotted .gl extent", }, xart_start_sector: u64 { default: 0, @@ -2209,5 +2248,17 @@ module! { default: 0, description: "Low 64 bits of an explicit xART OS UUID", }, + xarm_trace: u8 { + default: 0, + description: "Opt-in xART opcode/status trace without payloads or key material", + }, + probe_owner_export: u8 { + default: 0, + description: "Opt-in J414s diagnostic: with zero live identities, select the system context and try saving the missing owner Catacomb", + }, + j414s_persistent_enrol: u8 { + default: 0, + description: "Opt-in J414s enrollment test: export owner from the fresh context, then save user before master at completion", + }, }, } diff --git a/drivers/soc/apple/xart_apfs.rs b/drivers/soc/apple/xart_apfs.rs index 9eb833baca82d..1caca4aa1871c 100644 --- a/drivers/soc/apple/xart_apfs.rs +++ b/drivers/soc/apple/xart_apfs.rs @@ -15,6 +15,8 @@ const ROOT_INFO_SIZE: usize = 40; const XART_ROLE: u16 = 0x100; const FILE_SIZE: u64 = 0x600000; const FS_UNENCRYPTED: u64 = 1; +const INODE_WAS_CLONED: u64 = 0x10; +const INODE_WAS_EVER_CLONED: u64 = 0x400; const MOD: u64 = 0xffff_ffff; fn le16(data: &[u8], off: usize) -> Result { @@ -209,7 +211,74 @@ fn omap_lookup( Ok(paddr) } -fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Result { +fn is_gigalocker_name(name: &[u8]) -> bool { + if name == b".gl" { + return true; + } + if name.len() != 39 || &name[36..] != b".gl" { + return false; + } + name[..36].iter().enumerate().all(|(i, byte)| { + if matches!(i, 8 | 13 | 18 | 23) { + *byte == b'-' + } else { + byte.is_ascii_hexdigit() + } + }) +} + +fn validate_write_ownership( + file: &shim::StoreFile, + size: u64, + vol: &[u8], + xid: u64, + private_id: u64, + inode_flags: u64, + physical: u64, +) -> Result<()> { + // This is deliberately narrower than general APFS write support. On the + // tested J414s the .gl file has one unshared, unsnapshotted extent. Never + // raw-write a clone, snapshot, pending revert, or unknown extent layout. + if le64(vol, 264)? != FS_UNENCRYPTED + || le64(vol, 216)? != 0 + || le64(vol, 160)? != 0 + || le64(vol, 168)? != 0 + || inode_flags & (INODE_WAS_CLONED | INODE_WAS_EVER_CLONED) != 0 + { + return Err(ENOTSUPP); + } + let tree_paddr = le64(vol, 144)?; + let tree = read_object(file, size, tree_paddr)?; + if le64(&tree, 8)? != tree_paddr + || le64(&tree, 16)? > xid + || le32(&tree, 24)? != 0x4000_0002 + || le32(&tree, 28)? != 0x0f + || le16(&tree, 32)? != 3 + || le16(&tree, 34)? != 0 + || le32(&tree, 36)? != 1 + { + return Err(ENOTSUPP); + } + let (key, val) = node_entry(&tree, 0, false)?; + if key.len() != 8 + || val.len() < 20 + || le64(key, 0)? != ((2u64 << 60) | physical) + || le64(val, 0)? != ((1u64 << 60) | (FILE_SIZE / BLOCK as u64)) + || le64(val, 8)? != private_id + || le32(val, 16)? != 1 + { + return Err(ENOTSUPP); + } + Ok(()) +} + +fn xart_extent( + file: &shim::StoreFile, + size: u64, + vol: &[u8], + xid: u64, + writes_enabled: bool, +) -> Result { if le64(vol, 264)? & FS_UNENCRYPTED == 0 { return Err(ENOTSUPP); } @@ -232,7 +301,7 @@ fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Resul if name_len == 0 || 12 + name_len > key.len() || key[12 + name_len - 1] != 0 { return Err(EINVAL); } - if &key[12..12 + name_len - 1] == b".gl" { + if is_gigalocker_name(&key[12..12 + name_len - 1]) { if gl_file.replace(le64(val, 0)?).is_some() { return Err(ENOTSUPP); } @@ -240,12 +309,14 @@ fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Resul } let gl_file = gl_file.ok_or(ENODATA)?; let mut private_id: Option = None; + let mut inode_flags: Option = None; for i in 0..count { let (key, val) = node_entry(&root, i, false)?; if key.len() >= 8 && le64(key, 0)? == ((3u64 << 60) | gl_file) { if val.len() < 92 || private_id.replace(le64(val, 8)?).is_some() { return Err(EINVAL); } + inode_flags = Some(le64(val, 48)?); } } let private_id = private_id.ok_or(ENODATA)?; @@ -265,11 +336,23 @@ fn xart_extent(file: &shim::StoreFile, size: u64, vol: &[u8], xid: u64) -> Resul return Err(EINVAL); } } - extent.ok_or(ENODATA) + let extent = extent.ok_or(ENODATA)?; + if writes_enabled { + validate_write_ownership( + file, + size, + vol, + xid, + private_id, + inode_flags.ok_or(ENODATA)?, + extent / BLOCK as u64, + )?; + } + Ok(extent) } /// Return the byte offset of the sole contiguous 6 MiB `.gl` file extent. -pub(crate) fn locate(file: &shim::StoreFile) -> Result { +pub(crate) fn locate(file: &shim::StoreFile, writes_enabled: bool) -> Result { let device_size = file.size()?; let base = read_object(file, device_size, 0)?; if &base[32..36] != b"NXSB" || le32(&base, 36)? != BLOCK as u32 { @@ -331,7 +414,13 @@ pub(crate) fn locate(file: &shim::StoreFile) -> Result { } if le16(&vol, 964)? == XART_ROLE { if found - .replace(xart_extent(file, size, &vol, le64(&vol, 16)?)?) + .replace(xart_extent( + file, + size, + &vol, + le64(&vol, 16)?, + writes_enabled, + )?) .is_some() { return Err(ENOTSUPP); diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index 7c73107bc2ae8..c487824f7d9a8 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -5,8 +5,8 @@ //! //! The APFS locator identifies the existing `.gl` file's physical extent. //! This module validates records and serves SEP reads from that extent. It -//! never creates storage. Raw writes remain blocked until the record update -//! protocol and APFS ownership rules are verified against macOS. +//! never creates storage. Writes are opt-in and require narrow APFS checks +//! for an unshared, unsnapshotted extent. use crate::shim; use kernel::prelude::*; @@ -110,23 +110,20 @@ fn valid_key(key: &Key) -> bool { } impl Store { - /// Opens the iBoot system container read-only, resolves `.gl` through APFS + /// Opens the iBoot system container, resolves `.gl` through APFS /// metadata, then validates the record store at exactly that extent. /// A nonzero start sector is an assertion against the APFS result, never - /// an override of it. Neither lookup nor record validation authorizes - /// raw writes: macOS's update protocol is still unverified. + /// an override of it. Writes additionally require a single-owner extent + /// with no snapshots or pending revert, checked on the same block handle. pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { - if writes_enabled { - return Err(ENOTSUPP); - } - let file = shim::StoreFile::open_block(OWNER_PATH, false)?; - let base = crate::xart_apfs::locate(&file)?; + let file = shim::StoreFile::open_block(OWNER_PATH, writes_enabled)?; + let base = crate::xart_apfs::locate(&file, writes_enabled)?; if start_sector != 0 { if start_sector.checked_mul(512).ok_or(EINVAL)? != base { return Err(EINVAL); } } - Self::open_file(file, false, base) + Self::open_file(file, writes_enabled, base) } /// Opens a caller-selected block mapping at offset zero. @@ -151,8 +148,8 @@ impl Store { Self::open_file(file, writes_enabled, base) } - /// Complete record validation using the same read-only handle with which - /// APFS ownership was resolved, avoiding a close/reopen window. + /// Complete record validation using the same handle with which APFS + /// ownership was resolved, avoiding a close/reopen window. fn open_file(file: shim::StoreFile, arm_writes: bool, base: u64) -> Result { let size = file.size()?; if base % BLOCK_SIZE as u64 != 0 || STORE_SIZE % BLOCK_SIZE as u64 != 0 { @@ -192,6 +189,9 @@ impl Store { if store.find(&Key::root(1)).is_none() || store.find(&Key::root(2)).is_none() { return Err(ENODATA); } + if arm_writes && (store.malformed_records != 0 || store.duplicate_records != 0) { + return Err(EIO); + } store.writes_enabled = arm_writes; if arm_writes { store.repair_disk()?; diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 767b9043d2ad5..b83ef5dea2e3d 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -12,15 +12,16 @@ systemctl daemon-reload systemctl enable aurora-sep.service ``` -The raw xART owner is currently read-only. Do not enable the installed -`aurora-sep.service` or run `load-driver` on a shared macOS/Linux installation: -that script requests `xart_writes=1`, which the kernel rejects. The kernel now -resolves the physical `.gl` extent from checksum-validated APFS metadata for -read-only operation; unsupported layouts fail closed. This removes the raw -record-signature locator, which was ambiguous when roots moved away from the -start of the file. APFS lookup alone does not establish write safety. Re-enable -provisioning only after macOS-compatible record updates and APFS ownership -invariants are verified on hardware. +`xart_writes=0` remains the default. `load-driver` explicitly opts into writes +and must only be enabled on a machine whose APFS `.gl` extent passes the +driver's narrow ownership checks: one physical extent, no snapshots or revert +state, and a single extent reference owned by the `.gl` inode. The kernel +resolves that extent from checksum-validated APFS metadata; unsupported layouts +fail closed. The opt-in has been tested on m2 with identity-keybag provisioning, +trusted-key seal/unseal, and reboot reload. The raw iBoot container hash did not +change in those tests, so an actual xART write was **not** validated. Do not +infer that all SEP or Touch ID operations work with a read-only GigaLocker. +Read `XART-SAFETY.md` before enabling writes on another dual-boot machine. For read-only structural diagnostics on Linux, run `sudo python3 apfs-xart-inspect.py /dev/disk/by-partlabel/iBootSystemContainer`. @@ -40,6 +41,75 @@ For fingerprint support, apply `patches/libfprint-1.94.100-apple-sep.patch` to libfprint 1.94.100, build and install libfprint, then install the fprintd device policy: +```sh +git clone --depth 1 --branch v1.94.100 \ + https://gitlab.freedesktop.org/libfprint/libfprint.git libfprint-aurora +cd libfprint-aurora +git apply ../linux/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch +meson setup build -Dprefix=/usr -Ddrivers=default -Dintrospection=false \ + -Dgtk-examples=false -Ddoc=false -Dinstalled-tests=false +ninja -C build +meson test -C build --print-errorlogs +``` + +Adjust the patch path for your checkout. The patch includes a fix for an +upstream Meson test loop when introspection is disabled. In the Aurora +`linux-build` guest, the additional packages needed for this build are +`libgudev libgusb pixman cairo glib2-devel`; `meson` and `ninja` are already +in the release image. The default driver set retains all normal libfprint +drivers and adds Aurora. This build was checked on aarch64: the shared library +and all default drivers compiled, with 0 test failures (most device tests +skip without their optional mocks/hardware). That does **not** prove Touch ID +works on m2. The patched library is installed privately under +`/opt/aurora-sep` there, with fprintd loading it via a service drop-in while +the distro libfprint remains untouched. fprintd discovers one Apple SEP sensor. +The Aurora driver now reports stage progress only when the kernel's stage +number increases; guidance and stage-zero events are not captured samples. +fprintd 1.94.5 itself emits an initial `enroll-stage-passed` after its +duplicate check, **before** starting enrollment. That first line does not +prove a finger was captured. +The device-specific `apple/mesa_calibration.bin` on m2 was recovered using +the bundled [`extract-mesa-calibration.py`](extract-mesa-calibration.py), copied +from [Gist revision `eb079a8007985d04e75182f20994f1a1c496f12e`](https://gist.github.com/DjDeveloperr/867a1961b861c570442724f48f770158/eb079a8007985d04e75182f20994f1a1c496f12e), +which scans the local iBoot System Container read-only for an FSCl/CALB record +with an IM4M manifest. It checks the container structure and markers, **not** +the manifest's cryptographic signature. Keep this blob private to the machine; +it is not a generic firmware package. On m2, a premature master +`SAVE_CATACOMB` returned SEP status `0x6` while its component state was `0x3` +(no save-pending bit). The +`m2fix10` fresh-context gate skips that unavailable save, and bounded +enrollment attempts reached the physical-finger wait without an immediate +enclave error. With the later UUID-forwarding stage2, a physical enrollment +completed on m2fix15 and master/user Catacombs were saved, but on each tested +reboot SEP listed zero live identities despite the host index still listing +the finger. m2fix16–m2fix20 narrowed this to the user Catacomb load: it +returns `0x8002` and becomes active, but reveals zero identities even with +the SCRD credential and sensor/device view prepared first. The saved files +remain backed up root-only on m2. A bounded repeated load still listed zero +identities. An opt-in owner export then produced a durable owner file, but +loading it did not recover the old user identity. `m2fix23` is a separate, +boot-tested candidate that saves owner in the fresh context and user before +master at completion; it has not been finger- or reboot-persistence-tested. +Login must stay password-based until a live post-reboot identity and +successful match are independently verified. + +On each Linux machine, extract its own calibration from its local iBoot System +Container (the input partition is discovered automatically): + +```sh +sudo install -d -m 0755 /usr/lib/firmware/apple +sudo python3 tools/aurora-sep/extract-mesa-calibration.py \ + -o /usr/lib/firmware/apple/mesa_calibration.bin +``` + +The extractor reads the iBoot partition but never writes to it. It creates the +output mode `0600`, rejects input/output aliases, and refuses ambiguous or +malformed calibration candidates. Do not commit or copy the resulting blob to +another machine. + +Once the hardware path is verified, install libfprint and then the fprintd +device policy: + ```sh install -Dm644 fprintd-aurora.conf \ /etc/systemd/system/fprintd.service.d/aurora.conf diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md index 1ea3708dc0a2e..52a976c63e5c4 100644 --- a/tools/aurora-sep/XART-SAFETY.md +++ b/tools/aurora-sep/XART-SAFETY.md @@ -1,10 +1,13 @@ # Shared xART gigalocker safety gate The iBoot APFS container and its XART `.gl` file are shared with macOS. The -kernel owner must remain read-only. `apple_sep.xart_writes=1` is rejected even -with an explicit `xart_start_sector`; the latter is now only an assertion -against the APFS-resolved file extent. Do not enable provisioning services on a -dual-boot installation while this gate is in place. +deployed `m2fix4` kernel remains read-only. The `m2fix5` source has an explicit +`xart_writes=1` opt-in; it accepts writes only when the APFS-resolved extent is +single-owner, unsnapshotted, un-cloned, and all existing records validate. +`xart_start_sector` remains only an assertion against the APFS result. The +opt-in is for the controlled J414s hardware test, not general APFS write +support or a promise that restoring an old raw image reverses SEP anti-replay +state. Do not enable it on other installations without equivalent checks. ## Evidence collected on 2026-09-21 @@ -39,3 +42,40 @@ dual-boot installation while this gate is in place. The current APFS locator supports only a complete checkpoint, single-node object maps and file-system tree, and a single unencrypted 6 MiB `.gl` extent. Unsupported layouts fail closed; this is not a general APFS implementation. + +## Read-only J414s recovery image (2026-09-23) + +After the minimal stage2 made the SEP mailbox and RNG functional, a fresh +read-only inspection still found one unencrypted 6 MiB `.gl` extent at APFS +block 1236, 11 live records with valid payload CRCs, and no malformed records. +The selected container checkpoint was xid 136. No xART writes or keybag +provisioning were attempted. + +The 524,288,000-byte `iBootSystemContainer` partition was copied through a +read-only block handle and compressed to +`work/m2-tb-sep-deploy/iboot-system-container-2026-09-22.img.zst` (7,857,330 +bytes, mode 0600). The local archive passed `zstd -t`; its decompressed +SHA-256 matched a separate read of the partition: +`09206fcf0500f5646ea04ec37cf1fc8a0493f1c5ee3271c2af562ed9b39a6d37`. +The compressed archive SHA-256 is +`8a885a2397c26404cf8395ca64b279efc793f9118eae408f2a6d84fe9af1ded8`. +The temporary m2 copy was removed after verification. This is a recovery +artifact, **not** evidence that Linux may safely write APFS-owned `.gl` blocks. + +## Narrow write-ownership evidence (2026-09-23) + +The live m2 partition SHA-256 still matches the independent backup hash above. +The updated read-only inspector selected checkpoint xid 136 and checked the +current xART volume superblock and checksum-validated extent-reference tree: +volume flags `0x1` (unencrypted only), zero snapshots, zero pending revert +fields, and an inode with no clone flag. The sole `.gl` extent is APFS blocks +1236–2771 (1536 blocks, 6 MiB); its only physical extent-reference record is +kind `APFS_KIND_NEW`, owner 16 (the inode's private ID), reference count 1. +The inspector now rejects missing, overlapping, shared, cloned/snapshotted, +or mismatched references, and four synthetic tests pass. The `m2fix5` kernel +independently checks the same narrow ownership conditions on its block handle +before arming writes and refuses a writable scan with malformed or duplicate +xART records. These checks establish the current physical mapping and lack of +APFS sharing; they do not by themselves establish macOS's full record-update +or SEP anti-replay recovery semantics. The APFS field meanings come from the +[Apple File System Reference](https://developer.apple.com/support/apple-file-system/Apple-File-System-Reference.pdf). diff --git a/tools/aurora-sep/apfs-xart-inspect.py b/tools/aurora-sep/apfs-xart-inspect.py index 63cc276646f5e..d11473f06d0fc 100644 --- a/tools/aurora-sep/apfs-xart-inspect.py +++ b/tools/aurora-sep/apfs-xart-inspect.py @@ -16,6 +16,16 @@ MOD = 0xffffffff +def is_gigalocker_name(name): + if name == b".gl": + return True + if len(name) != 39 or name[36:] != b".gl": + return False + return all(byte == ord("-") if i in (8, 13, 18, 23) + else byte in b"0123456789abcdefABCDEF" + for i, byte in enumerate(name[:36])) + + def u16(buf, off): return struct.unpack_from("> 60 != 2: + raise ValueError("unexpected extent-reference key type") + start = tagged & ((1 << 60) - 1) + length_and_kind = u64(val, 0) + length = length_and_kind & ((1 << 60) - 1) + kind = length_and_kind >> 60 + owner = u64(val, 8) + refcount = struct.unpack_from(" (1 << 60): + raise ValueError("invalid extent-reference length") + if start < physical + blocks and physical < start + length: + if (start != physical or length != blocks or kind != 1 or + owner != private_id or refcount != 1): + raise ValueError(".gl extent reference is shared or mismatched") + matches.append((start, length, kind, owner, refcount)) + if len(matches) != 1: + raise ValueError("expected exactly one unshared .gl extent reference") + return matches[0] + + +def inspect_extent_reference(disk, vol, physical, blocks, private_id): + snapshots = u64(vol, 216) + revert_xid, revert_sblock = u64(vol, 160), u64(vol, 168) + print("xart_volume_flags=0x%x snapshot_count=%d revert_xid=%d revert_sblock=%d" % + (u64(vol, 264), snapshots, revert_xid, revert_sblock)) + if snapshots or revert_xid or revert_sblock: + raise ValueError("xART volume has snapshot or pending revert state") + tree_paddr = u64(vol, 144) + tree = disk.read(tree_paddr) + if (u64(tree, 8) != tree_paddr or u32(tree, 24) != 0x40000002 or + u32(tree, 28) != 0x0f): + raise ValueError("unsupported extent-reference tree") + level, flags, entries = node_entries(tree) + if level or flags & 3 != 3: + raise ValueError("multilevel extent-reference tree needs inspection") + match = validate_extent_references(entries, physical, blocks, private_id) + print("gigalocker_extent_reference_block_length_kind_owner_refcount=%s" % + (match,)) + + def inspect(disk): base = disk.read(0) if base[32:36] != b"NXSB" or u32(base, 36) != BLOCK: @@ -183,6 +240,7 @@ def inspect(disk): if level: raise ValueError("multilevel xART FS tree needs further inspection") files = [] + names = [] for key, val in entries: if len(key) < 12 or len(val) < 8: continue @@ -191,8 +249,10 @@ def inspect(disk): name_len = u32(key, 8) & 0x3ff if name_len and 12 + name_len <= len(key): name = key[12:12 + name_len - 1] - if name == b".gl": + if is_gigalocker_name(name): files.append(u64(val, 0)) + names.append(name.decode("ascii")) + print("gigalocker_file_names=%s" % names) print("gigalocker_file_ids=%s" % files) if len(files) != 1: raise ValueError("expected exactly one .gl file") @@ -204,7 +264,6 @@ def inspect(disk): private_id, inode_flags = u64(inode, 8), u64(inode, 48) print("gigalocker_inode_private_id=%d flags=0x%x links=%d write_generation=%d" % (private_id, inode_flags, u32(inode, 56), u32(inode, 64))) - print("xart_volume_flags=0x%x snapshot_count=%d" % (u64(vol, 264), u32(disk.read(vol_omap), 36))) extents = [] for key, val in entries: if len(key) >= 16 and len(val) >= 24 and u64(key, 0) == ((8 << 60) | private_id): @@ -218,6 +277,8 @@ def inspect(disk): if not extents: raise ValueError("no .gl extents") if len(extents) == 1 and extents[0][0] == 0 and extents[0][1] == 0x600000: + inspect_extent_reference(disk, vol, extents[0][2], + extents[0][1] // BLOCK, private_id) inspect_raw_locator(disk, extents[0][2]) scan_other_roots(disk, extents[0][2], extents[0][1]) diff --git a/tools/aurora-sep/extract-mesa-calibration.py b/tools/aurora-sep/extract-mesa-calibration.py new file mode 100644 index 0000000000000..0485e2ef21b37 --- /dev/null +++ b/tools/aurora-sep/extract-mesa-calibration.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +"""Carve the per-device Mesa calibration from an Apple iBoot System Container.""" + +import argparse +import hashlib +import os +from pathlib import Path +import stat +import sys +import tempfile + + +ISC_PARTITION_NAME = "iBootSystemContainer" +SYS_BLOCK = Path("/sys/class/block") +DEV_ROOT = Path("/dev") +SCAN_CHUNK_SIZE = 4 * 1024 * 1024 +SCAN_OVERLAP = 4 * 1024 * 1024 + + +class DERError(ValueError): + pass + + +class TLV: + def __init__(self, data, start): + self.start = start + try: + self.tag = data[start] + first_length = data[start + 1] + except IndexError as error: + raise DERError("truncated DER header") from error + + if first_length < 0x80: + length = first_length + header_length = 2 + else: + length_bytes = first_length & 0x7f + if length_bytes == 0: + raise DERError("indefinite DER length") + if length_bytes > 4: + raise DERError("unreasonably large DER length") + length_start = start + 2 + length_end = length_start + length_bytes + if length_end > len(data): + raise DERError("truncated DER length") + encoded = data[length_start:length_end] + if encoded[0] == 0: + raise DERError("non-minimal DER length") + length = int.from_bytes(encoded, "big") + if length < 0x80: + raise DERError("non-minimal DER length") + header_length = 2 + length_bytes + + self.content_start = start + header_length + self.end = self.content_start + length + if self.end > len(data): + raise DERError("DER value extends beyond input") + + def content(self, data): + return data[self.content_start:self.end] + + +def child_tlvs(data, container): + offset = container.content_start + while offset < container.end: + child = TLV(data, offset) + if child.end > container.end: + raise DERError("DER child extends beyond its container") + yield child + offset = child.end + if offset != container.end: + raise DERError("malformed DER container") + + +def take_child(data, children, tag, value=None): + try: + child = next(children) + except StopIteration as error: + raise DERError("missing DER child") from error + if child.tag != tag: + raise DERError(f"unexpected DER tag 0x{child.tag:02x}") + if value is not None and child.content(data) != value: + raise DERError("unexpected DER value") + return child + +def validate_calibration(data, start): + """Return the end offset if data[start:] begins with a Mesa calibration.""" + outer = TLV(data, start) + if outer.tag != 0x30: + raise DERError("outer object is not a sequence") + outer_children = child_tlvs(data, outer) + take_child(data, outer_children, 0x16, b"comb") + + fdrd = take_child(data, outer_children, 0x30) + fdrd_children = child_tlvs(data, fdrd) + take_child(data, fdrd_children, 0x16, b"fdrd") + img4_octets = take_child(data, fdrd_children, 0x04) + try: + next(fdrd_children) + except StopIteration: + pass + else: + raise DERError("unexpected data in fdrd container") + + secb = take_child(data, outer_children, 0x30) + secb_children = child_tlvs(data, secb) + take_child(data, secb_children, 0x16, b"secb") + try: + next(outer_children) + except StopIteration: + pass + else: + raise DERError("unexpected data after secb container") + + img4 = TLV(data, img4_octets.content_start) + if img4.tag != 0x30 or img4.end != img4_octets.end: + raise DERError("fdrd is not one complete IMG4 sequence") + img4_children = child_tlvs(data, img4) + take_child(data, img4_children, 0x16, b"IMG4") + + im4p = take_child(data, img4_children, 0x30) + im4p_children = child_tlvs(data, im4p) + take_child(data, im4p_children, 0x16, b"IM4P") + take_child(data, im4p_children, 0x16, b"FSCl") + take_child(data, im4p_children, 0x16) + payload = take_child(data, im4p_children, 0x04).content(data) + if b"CALB" not in payload[:64]: + raise DERError("FSCl payload lacks its CALB header") + + # A signed calibration has an IM4M manifest following the IM4P. Requiring + # both markers avoids accepting an arbitrary, truncated FSCl object. + remainder = data[im4p.end:img4.end] + if b"IM4M" not in remainder or b"FSCl" not in remainder: + raise DERError("IMG4 lacks an FSCl manifest") + + return outer.end + + +def find_calibrations(data): + results = [] + signature = b"\x16\x04comb" + search_at = 0 + while True: + marker = data.find(signature, search_at) + if marker < 0: + break + # The outer SEQUENCE header immediately precedes the comb IA5String. + # Try every legal header width supported by TLV above. + for header_length in range(2, 7): + start = marker - header_length + if start < 0 or data[start] != 0x30: + continue + try: + end = validate_calibration(data, start) + except DERError: + continue + candidate = (start, end) + if candidate not in results: + results.append(candidate) + search_at = marker + 1 + return results + + +def discover_isc_inputs(sys_block=SYS_BLOCK, dev_root=DEV_ROOT): + """Find every iSC by GPT partition name, independent of disk numbering.""" + candidates = [] + try: + block_devices = sorted(sys_block.iterdir()) + except OSError as error: + block_devices = [] + sysfs_error = error + else: + sysfs_error = None + + for block_device in block_devices: + try: + properties = {} + for line in (block_device / "uevent").read_text().splitlines(): + key, separator, value = line.partition("=") + if separator: + properties[key] = value + except OSError: + continue + if properties.get("PARTNAME") != ISC_PARTITION_NAME: + continue + device_name = properties.get("DEVNAME", block_device.name) + if device_name.startswith("/dev/"): + device_name = device_name[5:] + device = dev_root / device_name + if device.exists(): + candidates.append(device) + + # This standard udev link also covers chroots where /sys is unavailable. + by_partlabel = dev_root / "disk/by-partlabel" / ISC_PARTITION_NAME + if by_partlabel.exists(): + candidates.append(by_partlabel) + + # Preserve order while removing aliases or duplicate sysfs entries. + unique = [] + seen = set() + for candidate in candidates: + try: + identity = candidate.resolve() + except OSError: + identity = candidate + if identity not in seen: + seen.add(identity) + unique.append(candidate) + if not unique and sysfs_error is not None: + raise RuntimeError( + f"cannot inspect {sys_block} and {by_partlabel} is unavailable: " + f"{sysfs_error}" + ) from sysfs_error + return unique + + +def scan_input(path): + """Scan a file or block device without retaining the whole iSC in memory.""" + mode = path.stat().st_mode + if not (stat.S_ISREG(mode) or stat.S_ISBLK(mode)): + raise RuntimeError(f"{path} is not a regular file or block device") + + candidates = {} + overlap = b"" + total = 0 + with path.open("rb", buffering=0) as source: + while True: + chunk = source.read(SCAN_CHUNK_SIZE) + if not chunk: + break + total += len(chunk) + window = overlap + chunk + window_start = total - len(window) + for start, end in find_calibrations(window): + absolute_start = window_start + start + candidates.setdefault(absolute_start, bytes(window[start:end])) + overlap = window[-SCAN_OVERLAP:] + return sorted(candidates.items()) + + +def write_private(path, blob): + path.parent.mkdir(parents=True, exist_ok=True) + old_umask = os.umask(0o077) + temporary = None + try: + fd, name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(name) + with os.fdopen(fd, "wb") as output: + output.write(blob) + output.flush() + os.fsync(output.fileno()) + os.chmod(temporary, stat.S_IRUSR | stat.S_IWUSR) + os.replace(temporary, path) + temporary = None + finally: + os.umask(old_umask) + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def validate_output(path, inputs): + """Reject output targets that could replace an input or special file.""" + if path.is_symlink(): + raise RuntimeError(f"output {path} is a symbolic link") + if not path.exists(): + return + for input_path in inputs: + try: + if os.path.samefile(path, input_path): + raise RuntimeError(f"output {path} aliases input {input_path}") + except FileNotFoundError: + pass + if not stat.S_ISREG(path.stat().st_mode): + raise RuntimeError(f"output {path} is not a regular file") + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "input", nargs="?", type=Path, + help=( + "raw iBoot System Container or dump; omit to discover all " + "iBootSystemContainer partitions" + ), + ) + parser.add_argument( + "-o", "--output", type=Path, default=Path("mesa_calibration.bin"), + help="output file (default: ./mesa_calibration.bin)", + ) + return parser.parse_args(argv) + + +def main(): + args = parse_args() + try: + inputs = [args.input] if args.input is not None else discover_isc_inputs() + if not inputs: + raise RuntimeError( + "no iBootSystemContainer partition found; pass its block device " + "or a raw partition dump explicitly" + ) + validate_output(args.output, inputs) + + candidates = [] + for input_path in inputs: + for offset, blob in scan_input(input_path): + candidates.append((input_path, offset, blob)) + if not candidates: + raise RuntimeError( + "no signed Mesa FSCl/CALB calibration found; expected the raw " + "iBoot System Container (Apple Silicon boot partition)" + ) + blobs = {} + for input_path, offset, blob in candidates: + blobs.setdefault(blob, []).append((input_path, offset)) + if len(blobs) != 1: + locations = ", ".join( + f"{input_path}@0x{offset:x}" + for input_path, offset, _ in candidates + ) + raise RuntimeError( + f"found different calibrations at {locations}; " + "pass the intended iSC explicitly" + ) + + blob, locations = next(iter(blobs.items())) + write_private(args.output, blob) + except (OSError, RuntimeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + digest = hashlib.sha256(blob).hexdigest() + formatted_locations = ", ".join( + f"{input_path}@0x{offset:x}" for input_path, offset in locations + ) + print(f"found Mesa calibration at {formatted_locations}") + print(f"wrote {len(blob)} bytes to {args.output}") + print(f"sha256 {digest}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-sep/load-driver b/tools/aurora-sep/load-driver index f5579fc01b360..a70db75e6b25b 100755 --- a/tools/aurora-sep/load-driver +++ b/tools/aurora-sep/load-driver @@ -4,6 +4,13 @@ set -eu keybag=/var/lib/aurora-sep-keybag.bin refkey=/var/lib/aurora-sep-refkey.bin +# An SSH/sudo caller cannot change its parent's session keyring with +# `keyctl new_session`. Run the whole probe in a child keyring instead; the +# marker prevents re-exec recursion and the keyring dies with this process. +if [ "${AURORA_SEP_SESSION:-0}" != 1 ]; then + exec keyctl session aurora-sep-load env AURORA_SEP_SESSION=1 "$0" "$@" +fi + modprobe apple_sep xart_writes=1 provision_keybag=1 work=$(mktemp -d /run/aurora-sep-load.XXXXXX) @@ -14,7 +21,6 @@ cleanup() { } trap cleanup EXIT HUP INT TERM -keyctl new_session aurora-sep-load >/dev/null attempts=300 verified=false while [ "$attempts" -gt 0 ]; do diff --git a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch index 37593f65ea05e..1947bd095b43b 100644 --- a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch +++ b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch @@ -135,10 +135,10 @@ index 0000000..b2d2f43 +#define AURORA_BIO_ATTEST _IOWR(AURORA_BIO_IOC_MAGIC, 0x0a, struct aurora_bio_attest) diff --git a/libfprint/drivers/aurora/aurora.c b/libfprint/drivers/aurora/aurora.c new file mode 100644 -index 0000000..55f3311 +index 0000000..80ace06 --- /dev/null +++ b/libfprint/drivers/aurora/aurora.c -@@ -0,0 +1,829 @@ +@@ -0,0 +1,837 @@ +/* + * Aurora SEP fingerprint driver + * @@ -170,6 +170,7 @@ index 0000000..55f3311 + + gint fd; + guint watch_id; ++ guint32 enroll_last_reported_stage; + + guint8 nonce[AURORA_BIO_NONCE_LEN]; + gboolean nonce_valid; @@ -516,7 +517,13 @@ index 0000000..55f3311 + return G_SOURCE_CONTINUE; + + case AURORA_BIO_STATE_PROGRESS: -+ fpi_device_enroll_progress (device, poll_res.stage, NULL, NULL); ++ /* fprintd calls this "enroll-stage-passed". A guidance wakeup or an ++ * initial stage-zero event must not masquerade as a captured sample. */ ++ if (poll_res.stage > self->enroll_last_reported_stage) ++ { ++ self->enroll_last_reported_stage = poll_res.stage; ++ fpi_device_enroll_progress (device, poll_res.stage, NULL, NULL); ++ } + return G_SOURCE_CONTINUE; + + case AURORA_BIO_STATE_DONE: @@ -543,7 +550,7 @@ index 0000000..55f3311 + self->watch_id = 0; + fpi_device_enroll_complete (device, NULL, + fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "enrolment failed, enclave status 0x%x", ++ "enrolment failed, driver status 0x%x", + poll_res.status)); + return G_SOURCE_REMOVE; + } @@ -571,6 +578,7 @@ index 0000000..55f3311 + return; + } + ++ self->enroll_last_reported_stage = 0; + aurora_stop_watch (self); + self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_enrol_ready, device); +} @@ -1177,3 +1185,13 @@ index ab09dc1..dfa7460 100644 # Virtual drivers (test-only, optional) 'virtual_image': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, 'virtual_device': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, +diff --git a/tests/meson.build b/tests/meson.build +index aad0675..2147ef3 100644 +--- a/tests/meson.build ++++ b/tests/meson.build +@@ -295,4 +295,4 @@ else +- foreach driver_test: drivers_tests ++ foreach driver_test, args: drivers_tests + test(driver_test, + sh, + args: ['-c', 'exit 77'] diff --git a/tools/aurora-sep/tests/test_extract_mesa_calibration.py b/tools/aurora-sep/tests/test_extract_mesa_calibration.py new file mode 100644 index 0000000000000..08884ba921e9b --- /dev/null +++ b/tools/aurora-sep/tests/test_extract_mesa_calibration.py @@ -0,0 +1,72 @@ +"""Offline safety and format tests for the bundled calibration extractor.""" + +import importlib.util +from pathlib import Path +import stat +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "extract-mesa-calibration.py" +SPEC = importlib.util.spec_from_file_location("extract_mesa_calibration", SCRIPT) +extractor = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(extractor) + + +def tlv(tag, content): + length = len(content) + if length < 128: + encoded_length = bytes([length]) + else: + width = (length.bit_length() + 7) // 8 + encoded_length = bytes([0x80 | width]) + length.to_bytes(width, "big") + return bytes([tag]) + encoded_length + content + + +def ia5(value): + return tlv(0x16, value) + + +def synthetic_record(with_manifest=True): + """Build a structural fixture, not a cryptographically signed IMG4.""" + im4p = tlv(0x30, ia5(b"IM4P") + ia5(b"FSCl") + ia5(b"test") + + tlv(0x04, b"CALB" + b"x" * 64)) + manifest = ia5(b"IM4M") + ia5(b"FSCl") if with_manifest else b"" + img4 = tlv(0x30, ia5(b"IMG4") + im4p + manifest) + fdrd = tlv(0x30, ia5(b"fdrd") + tlv(0x04, img4)) + secb = tlv(0x30, ia5(b"secb")) + return tlv(0x30, ia5(b"comb") + fdrd + secb) + + +class CalibrationExtractorTests(unittest.TestCase): + def test_scans_only_complete_manifest_marked_record(self): + record = synthetic_record() + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "isc.img" + source.write_bytes(b"prefix" + record + b"suffix") + self.assertEqual(extractor.scan_input(source), [(6, record)]) + + def test_rejects_record_without_manifest_markers(self): + self.assertEqual(extractor.find_calibrations(synthetic_record(False)), []) + + def test_rejects_input_output_aliases_and_symlinks(self): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "isc.img" + source.write_bytes(synthetic_record()) + with self.assertRaises(RuntimeError): + extractor.validate_output(source, [source]) + symlink = Path(directory) / "output.bin" + symlink.symlink_to(source) + with self.assertRaises(RuntimeError): + extractor.validate_output(symlink, [source]) + + def test_private_output_mode(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "mesa_calibration.bin" + extractor.write_private(output, b"sample") + self.assertEqual(output.read_bytes(), b"sample") + self.assertEqual(stat.S_IMODE(output.stat().st_mode), 0o600) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/aurora-sep/tests/test_xart_locator.py b/tools/aurora-sep/tests/test_xart_locator.py index 167c977ec2bd0..7493928bd6795 100644 --- a/tools/aurora-sep/tests/test_xart_locator.py +++ b/tools/aurora-sep/tests/test_xart_locator.py @@ -27,7 +27,47 @@ def record(kind, revision): return out +def extent_reference(start, blocks, owner, refcount=1, kind=1): + key = struct.pack(" Date: Fri, 25 Sep 2026 17:06:21 -0400 Subject: [PATCH 0063/1579] soc: apple: sep: surface capture guidance and enforce live user capacity Carry the verified J414s contact-guidance and three-slot changes from the combined worktree. The live SEP count, not a stale host index, gates new enrollment. --- drivers/soc/apple/bio.rs | 111 ++++++++++++++++++++++-------------- drivers/soc/apple/sbio.rs | 21 +++++-- drivers/soc/apple/sep-bio.h | 4 +- 3 files changed, 85 insertions(+), 51 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 2bc1a492799c3..d115a18d64d97 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -15,6 +15,9 @@ const LABEL_LEN: usize = 128; const NONCE_LEN: usize = 32; pub(crate) const TOKEN_LEN: usize = 32; const MAX_IDENTITIES: usize = 32; +// The host index ABI has 32 entries, but this driver uses one SEP user context. +// Mac Touch ID permits only three fingerprints in a single user context. +const SEP_USER_CAPACITY: usize = 3; pub(crate) const DEVICE_NAME: &CStr = c"sep-bio"; pub(crate) const DEVICE_MODE: u16 = 0o600; @@ -123,7 +126,7 @@ struct VerifyPoll { state: State, result: MatchResult, status: u32, - reserved: u32, + guidance: Guidance, uuid: [u8; UUID_LEN], token: [u8; TOKEN_LEN], deadline_ns: u64, @@ -184,7 +187,7 @@ const MAGIC: u32 = 0xB1; pub(crate) const IOC_GET_INFO: u32 = _IOR::(MAGIC, 0x01); const IOC_LIST: u32 = _IOR::(MAGIC, 0x02); -const IOC_ENROL_START: u32 = _IOW::(MAGIC, 0x03); +pub(crate) const IOC_ENROL_START: u32 = _IOW::(MAGIC, 0x03); const IOC_ENROL_POLL: u32 = _IOR::(MAGIC, 0x04); const IOC_VERIFY_START: u32 = _IOW::(MAGIC, 0x05); const IOC_VERIFY_POLL: u32 = _IOR::(MAGIC, 0x06); @@ -390,6 +393,7 @@ enum Op { Verify { nonce: [u8; NONCE_LEN], terminal: Option, + guidance: Guidance, }, } @@ -509,7 +513,7 @@ fn get_info(ctx: &mut Context<'_>, user: UserPtr) -> Result { version: IFACE_VERSION, sensor_present: u32::from(ctx.sensor_present), enrolled: ctx.index.total() as u32, - capacity: MAX_IDENTITIES as u32, + capacity: SEP_USER_CAPACITY as u32, enroll_stages: ENROL_STAGES, // reserved[0] is a validity flag. A zero count is only meaningful // when SEP itself successfully answered LIST_IDENTITIES; older @@ -559,7 +563,9 @@ fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { if !ctx.session.op.may_start() { return Err(EBUSY); } - if ctx.index.total() >= MAX_IDENTITIES { + if matches!(ctx.live_identity_count, Some(count) if count >= SEP_USER_CAPACITY) + || ctx.index.total() >= MAX_IDENTITIES + { return Err(ENOSPC); } @@ -608,18 +614,30 @@ pub(crate) fn enrol_advance( false } -pub(crate) fn enrol_guide(session: &mut Session, guide: Guidance) -> bool { - if let Op::Enrol { - terminal, guidance, .. - } = &mut session.op - { - if terminal.is_none() && *guidance != guide { - *guidance = guide; - session.unseen = true; - return true; +pub(crate) fn capture_guide(session: &mut Session, guide: Guidance) -> bool { + let changed = match &mut session.op { + Op::Enrol { terminal, guidance, .. } => { + if terminal.is_none() && *guidance != guide { + *guidance = guide; + true + } else { + false + } + } + Op::Verify { terminal, guidance, .. } => { + if terminal.is_none() && *guidance != guide { + *guidance = guide; + true + } else { + false + } } + Op::Idle => false, + }; + if changed { + session.unseen = true; } - false + changed } pub(crate) fn enrol_finish( @@ -723,6 +741,7 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { ctx.session.op = Op::Verify { nonce: request.nonce, terminal: Some(VerifyOutcome::NoMatch), + guidance: Guidance::None, }; ctx.session.unseen = true; return Ok(Handled { @@ -739,6 +758,7 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { ctx.session.op = Op::Verify { nonce: request.nonce, terminal: None, + guidance: Guidance::Place, }; Ok(Handled { ret: 0, @@ -755,7 +775,7 @@ pub(crate) fn verify_finish( outcome: VerifyOutcome, token_bytes: [u8; TOKEN_LEN], ) -> bool { - let Op::Verify { nonce, terminal } = &mut session.op else { + let Op::Verify { nonce, terminal, .. } = &mut session.op else { return false; }; if terminal.is_some() { @@ -789,39 +809,42 @@ fn verify_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { match &ctx.session.op { Op::Idle => out.state = State::Idle, Op::Enrol { .. } => return Err(EBUSY), - Op::Verify { nonce, terminal } => match terminal { - None => { - out.state = if ctx.session.unseen { - State::Progress - } else { - State::Pending - } - } - Some(VerifyOutcome::Failed(status)) => { - out.state = State::Failed; - out.result = MatchResult::NotCompared; - out.status = *status; - } - Some(VerifyOutcome::NoMatch) => { - out.state = State::Done; - out.result = MatchResult::NoMatch; - } - Some(VerifyOutcome::Matched(evidence)) => { - out.state = State::Done; - match ctx.session.token.as_ref() { - Some(token) if token.usable(nonce, &evidence.identity) => { - out.result = MatchResult::Match; - out.uuid = evidence.identity; - out.token = token.bytes; - out.deadline_ns = token.deadline_ns; - consume_token = true; + Op::Verify { nonce, terminal, guidance } => { + out.guidance = *guidance; + match terminal { + None => { + out.state = if ctx.session.unseen { + State::Progress + } else { + State::Pending } - _ => { - out.result = MatchResult::NotCompared; + } + Some(VerifyOutcome::Failed(status)) => { + out.state = State::Failed; + out.result = MatchResult::NotCompared; + out.status = *status; + } + Some(VerifyOutcome::NoMatch) => { + out.state = State::Done; + out.result = MatchResult::NoMatch; + } + Some(VerifyOutcome::Matched(evidence)) => { + out.state = State::Done; + match ctx.session.token.as_ref() { + Some(token) if token.usable(nonce, &evidence.identity) => { + out.result = MatchResult::Match; + out.uuid = evidence.identity; + out.token = token.bytes; + out.deadline_ns = token.deadline_ns; + consume_token = true; + } + _ => { + out.result = MatchResult::NotCompared; + } } } } - }, + } } UserSlice::new(user, core::mem::size_of::()) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 91e5125b55acb..a901afc007a0b 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -38,12 +38,18 @@ impl SepData { pub(crate) fn sbio_call(&self, op: &crate::sbio::SbioOp) -> SbioOutcome { let done = match self.sbio_transfer(op) { Ok(done) => done, - Err(_) => { + Err(e) => { + if op.opcode() == OP_SBIO_BEGIN_ENROL { + dev_err!(self.dev, "enrol: BEGIN_ENROL transfer failed ({:?})\n", e); + } return SbioOutcome::Other; } }; let Some(err) = done.status.answered() else { + if op.opcode() == OP_SBIO_BEGIN_ENROL { + dev_err!(self.dev, "enrol: BEGIN_ENROL reply had no status: {}\n", done.status); + } return SbioOutcome::Other; }; @@ -51,7 +57,12 @@ impl SepData { crate::sbio::SBIO_STATUS_OK => SbioOutcome::Ok(done.payload), crate::sbio::SBIO_STATUS_PREREQUISITE => SbioOutcome::PrerequisiteMissing, crate::sbio::SBIO_STATUS_16 => SbioOutcome::Status16, - _ => SbioOutcome::Other, + _ => { + if op.opcode() == OP_SBIO_BEGIN_ENROL { + dev_err!(self.dev, "enrol: BEGIN_ENROL replied with status 0x{:x}\n", err); + } + SbioOutcome::Other + } } } @@ -1387,7 +1398,7 @@ impl SepData { // The capture did not take (partial/unusable contact). Nudge // the person to hold still and press again rather than // silently re-capturing with no feedback. - if bio::enrol_guide(&mut self.bio_session.lock(), bio::Guidance::HoldStill) { + if bio::capture_guide(&mut self.bio_session.lock(), bio::Guidance::HoldStill) { self.bio_wake(); } } @@ -2042,7 +2053,7 @@ impl SepData { _ => None, }; if let Some(guide) = guide { - if bio::enrol_guide(&mut self.bio_session.lock(), guide) { + if bio::capture_guide(&mut self.bio_session.lock(), guide) { self.bio_wake(); } } @@ -2403,7 +2414,7 @@ impl SepData { // Query SEP outside the bio session/index locks. A host index can // outlive its SEP identity after a failed cold restore; userspace // must not mistake that index for proof of a live template. - let live_identity_count = if cmd == bio::IOC_GET_INFO + let live_identity_count = if (cmd == bio::IOC_GET_INFO || cmd == bio::IOC_ENROL_START) && self.touchid_started.load(Relaxed) { self.enclave_identity_count() diff --git a/drivers/soc/apple/sep-bio.h b/drivers/soc/apple/sep-bio.h index 2ec3949eaa665..d52b880ac9332 100644 --- a/drivers/soc/apple/sep-bio.h +++ b/drivers/soc/apple/sep-bio.h @@ -48,7 +48,7 @@ struct sep_bio_info { __u32 version; __u32 sensor_present; __u32 enrolled; - __u32 capacity; + __u32 capacity; /* usable slots in this SEP user context, not ABI list size */ __u32 enroll_stages; __u32 reserved[3]; }; @@ -92,7 +92,7 @@ struct sep_bio_verify_poll { __u32 state; __u32 result; __u32 status; - __u32 reserved; + __u32 guidance; /* SEP_BIO_GUIDANCE_*; status only, no capture side effect */ __u8 uuid[SEP_BIO_UUID_LEN]; __u8 token[SEP_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ From 1c35a898cec98b9d44cac4e346b44430c02551ec Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 25 Sep 2026 17:06:51 -0400 Subject: [PATCH 0064/1579] soc: apple: sep: defer Neo sensor until SMC power is ready Treat an explicit SMC-powered sensor as an SMC consumer, rather than silently falling back to unrelated GPIO power. Keep the supplier bound, retry when it is late, and attempt to turn off every rail after a write failure. --- drivers/soc/apple/sbio.rs | 10 ++-- drivers/soc/apple/sensor_shim.c | 86 ++++++++++++++++++++++----------- drivers/soc/apple/sep.rs | 3 -- drivers/soc/apple/shim.h | 1 + 4 files changed, 63 insertions(+), 37 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index a901afc007a0b..c91938b60fee1 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -262,7 +262,6 @@ impl SepData { pub(crate) fn attach_sensor(&self) { if let Err(e) = sensor::register_driver() { - self.sensor_present.store(false, Relaxed); dev_err!( self.dev, "sensor: could not register the SPI driver: {:?}\n", @@ -272,7 +271,6 @@ impl SepData { } if let Err(e) = dt::enable_spi_sensor(sensor::CONTROLLER_BASE, sensor::CHIP_SELECT) { - self.sensor_present.store(false, Relaxed); dev_warn!( self.dev, "sensor: could not enable the SPI bus at 0x{:x} or create the sensor node ({:?}); no sensor\n", @@ -283,7 +281,6 @@ impl SepData { } let bound = sensor::is_bound(); - self.sensor_present.store(bound, Relaxed); if bound && sensor::power_source() == sensor::PowerSource::None { dev_warn!( self.dev, @@ -1011,7 +1008,7 @@ impl SepData { } fn attach_bringup(&self) { - if !self.sensor_present.load(Relaxed) { + if !sensor::is_bound() { dev_warn!( self.dev, "sbio: no sensor bound at attach; catacomb restore ran but capture is unavailable\n" @@ -2421,13 +2418,16 @@ impl SepData { } else { None }; + // The SPI probe may have deferred until its SMC power supplier bound. + // A saved attach-time snapshot would incorrectly reject later ioctls. + let sensor_present = sensor::is_bound(); let handled = { let mut session = self.bio_session.lock(); let index = self.bio_index.lock(); let mut ctx = bio::Context { session: &mut session, index: &index, - sensor_present: self.sensor_present.load(Relaxed), + sensor_present, live_identity_count, }; bio::ioctl(&mut ctx, cmd, arg)? diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index ec4d30581da67..9ed3658cda8f7 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -96,22 +96,37 @@ static int sep_smc_write_rail(u32 rail, u16 value) static int sep_smc_set_power(int on) { - int i, rc = 0; + int i, rc, first_error = 0; if (on) { - for (i = 0; i < sep_smc_nrails && !rc; i++) + for (i = 0; i < sep_smc_nrails; i++) { rc = sep_smc_write_rail(sep_smc_rails[i], 1); + if (rc) { + first_error = rc; + /* The failed write may have taken effect too. */ + for (; i >= 0; i--) { + rc = sep_smc_write_rail(sep_smc_rails[i], 0); + if (rc) + pr_err("apple_sep: sensor: SMC rail rollback failed (%d)\n", rc); + } + break; + } + } } else { - for (i = sep_smc_nrails - 1; i >= 0 && !rc; i--) + /* A failed off write must not skip the remaining rails. */ + for (i = sep_smc_nrails - 1; i >= 0; i--) { rc = sep_smc_write_rail(sep_smc_rails[i], 0); + if (rc && !first_error) + first_error = rc; + } } - if (rc) - pr_err("apple_sep: sensor: SMC rail write failed (%d)\n", rc); - return rc; + if (first_error) + pr_err("apple_sep: sensor: SMC rail write failed (%d)\n", first_error); + return first_error; } /* Nothing is written until the expected eight-byte SMC key is verified. */ -static bool sep_acquire_smc_power(struct spi_device *spi) +static int sep_acquire_smc_power(struct spi_device *spi) { struct device_node *np = spi->dev.of_node, *smc_np; struct platform_device *smc_pdev; @@ -120,48 +135,55 @@ static bool sep_acquire_smc_power(struct spi_device *spi) int n, rc; if (!np || of_property_read_string(np, "apple,smc-power-key", &key)) - return false; + return -EINVAL; if (!key || strlen(key) != 4) { dev_warn(&spi->dev, "sep sensor: apple,smc-power-key must have four characters\n"); - return false; + return -EINVAL; } n = of_property_count_u32_elems(np, "apple,smc-power-rails"); if (n <= 0 || n > SEP_SMC_MAX_RAILS || of_property_read_u32_array(np, "apple,smc-power-rails", sep_smc_rails, n)) { dev_warn(&spi->dev, "sep sensor: expected 1..%d SMC rail words\n", SEP_SMC_MAX_RAILS); - return false; + return -EINVAL; } smc_np = of_find_compatible_node(NULL, NULL, "apple,smc"); if (!smc_np) - return false; + return -EPROBE_DEFER; smc_pdev = of_find_device_by_node(smc_np); of_node_put(smc_np); if (!smc_pdev) - return false; + return -EPROBE_DEFER; + /* Keep the supplier bound for as long as this sensor uses its drvdata. */ + if (!device_link_add(&spi->dev, &smc_pdev->dev, + DL_FLAG_AUTOREMOVE_CONSUMER)) { + put_device(&smc_pdev->dev); + return -ENOMEM; + } sep_smc = dev_get_drvdata(&smc_pdev->dev); put_device(&smc_pdev->dev); if (!sep_smc) - return false; + return -EPROBE_DEFER; sep_smc_key = __SMC_KEY(key[0], key[1], key[2], key[3]); rc = apple_smc_get_key_info(sep_smc, sep_smc_key, &info); if (rc || info.size != 8) { dev_warn(&spi->dev, "sep sensor: SMC key %s absent or not eight bytes\n", key); sep_smc = NULL; - return false; + return rc ?: -EINVAL; } sep_smc_nrails = n; sep_power_source = SEP_POWER_SMC_RAILS; dev_info(&spi->dev, "sep sensor: power from SMC key %s (%d rails)\n", key, n); - if (sep_smc_set_power(0)) { + rc = sep_smc_set_power(0); + if (rc) { sep_power_source = SEP_POWER_NONE; sep_smc_nrails = 0; sep_smc = NULL; - return false; + return rc; } - return true; + return 0; } #else static int sep_smc_set_power(int on) @@ -169,12 +191,14 @@ static int sep_smc_set_power(int on) return -ENODEV; } -static bool sep_acquire_smc_power(struct spi_device *spi) +static int sep_acquire_smc_power(struct spi_device *spi) { if (spi->dev.of_node && - of_property_present(spi->dev.of_node, "apple,smc-power-key")) + of_property_present(spi->dev.of_node, "apple,smc-power-key")) { dev_warn(&spi->dev, "sep sensor: SMC power needs CONFIG_MFD_MACSMC\n"); - return false; + return -ENODEV; + } + return -ENOENT; } #endif @@ -182,21 +206,22 @@ static bool sep_acquire_smc_power(struct spi_device *spi) * Takes the power line as an output driven low: the power cycle begins with an * off phase, so the line must be actively driven off, not merely read as low. */ -static void sep_acquire_power(struct spi_device *spi) +static int sep_acquire_power(struct spi_device *spi) { struct device_node *np; struct gpio_device *gdev; struct gpio_chip *gc; - if (sep_acquire_smc_power(spi)) - return; + if (spi->dev.of_node && + of_property_present(spi->dev.of_node, "apple,smc-power-key")) + return sep_acquire_smc_power(spi); sep_power = gpiod_get_index(&spi->dev, NULL, 0, GPIOD_OUT_LOW); if (!IS_ERR(sep_power)) { sep_power_source = SEP_POWER_NODE_PROPERTY; dev_info(&spi->dev, "sep sensor: power line from the device node, driven low\n"); - return; + return 0; } sep_power = NULL; @@ -208,7 +233,7 @@ static void sep_acquire_power(struct spi_device *spi) if (!of_machine_is_compatible("apple,j414s")) { dev_warn(&spi->dev, "sep sensor: no power GPIO in the device node; describe gpios in DT\n"); - return; + return 0; } np = of_find_node_by_path(SEP_SENSOR_GPIO_NODE); @@ -216,7 +241,7 @@ static void sep_acquire_power(struct spi_device *spi) dev_warn(&spi->dev, "sep sensor: no DT node at %s, no power line\n", SEP_SENSOR_GPIO_NODE); - return; + return 0; } gdev = gpio_device_find_by_fwnode(of_fwnode_handle(np)); @@ -225,7 +250,7 @@ static void sep_acquire_power(struct spi_device *spi) dev_warn(&spi->dev, "sep sensor: %s has no registered GPIO device\n", SEP_SENSOR_GPIO_NODE); - return; + return 0; } gc = gpio_device_get_chip(gdev); @@ -242,7 +267,7 @@ static void sep_acquire_power(struct spi_device *spi) SEP_SENSOR_GPIO_LINE, SEP_SENSOR_GPIO_NODE, gpio_device_get_label(gdev)); gpio_device_put(gdev); - return; + return 0; } sep_power_source = SEP_POWER_CHIP_LINE; @@ -251,6 +276,7 @@ static void sep_acquire_power(struct spi_device *spi) SEP_SENSOR_GPIO_NODE, SEP_SENSOR_GPIO_LINE, gpio_device_get_label(gdev)); gpio_device_put(gdev); + return 0; } static void sep_release_power(void) @@ -350,8 +376,10 @@ static int sep_sensor_probe(struct spi_device *spi) if (rc) return rc; + rc = sep_acquire_power(spi); + if (rc) + return dev_err_probe(&spi->dev, rc, "sep sensor: power unavailable\n"); sep_spi = spi; - sep_acquire_power(spi); /* Read the mode back: a mode that did not take must not be logged as * the one requested. */ dev_info(&spi->dev, diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 63e3654a290cf..6aa66f9263944 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -615,8 +615,6 @@ struct SepData { #[pin] scrd_wq: CondVar, - sensor_present: Atomic, - #[pin] bio_session: Mutex, @@ -807,7 +805,6 @@ impl SepData { sbio_rx <- new_mutex!(transfer::Reassembly::new()), sbio_wq <- new_condvar!("SepData::sbio_wq"), sbio_ready: Atomic::new(false), - sensor_present: Atomic::new(false), bio_session <- new_mutex!(bio::Session::new()), bio_index <- new_mutex!(bio_index), bio_dev <- new_mutex!(None), diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 2797f679f2e5e..e13ca258d92f8 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -124,6 +124,7 @@ int sep_sensor_cs_timing_mode(void); int sep_sensor_power_cycle(void); int sep_sensor_power_source(void); int sep_sensor_power(int on); +const char *sep_sensor_firmware_name(void); int sep_sensor_xfer(const void *tx, void *rx, size_t len); int sep_sensor_xfer_tx(const void *tx, size_t len); int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len); From 52db115c478bb699caebc1da131b84aed8a99532 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:01:52 +1000 Subject: [PATCH 0065/1579] arm64: configs: asahi: enable trusted keys for the SEP driver APPLE_SEP depends on TRUSTED_KEYS, which defconfig leaves off, so a kernel configured with "defconfig asahi.config" silently drops the SEP driver even though the fragment asks for it (kconfig discards the unmet CONFIG_APPLE_SEP=m). Kernels that had the driver so far set TRUSTED_KEYS by hand. Enable the trusted keys core in the fragment. The SEP registers itself as a trust source at probe time through register_trusted_key_source(), so the core is built in and the driver stays a module. Turn off the TPM and TEE trust sources that defconfig enables with it. Apple silicon Macs have neither, but the TEE source's init succeeds regardless (it only registers a TEE bus driver), claims the trusted key type at boot, and the SEP's registration then fails with -EBUSY: no trusted keys, and no key-store bring-up behind them. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 20a50b6fcb3a1..4b9dec84d9365 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -63,6 +63,9 @@ CONFIG_APPLE_RTKIT_HELPER=m CONFIG_APPLE_SART=m CONFIG_RUST_APPLE_RTKIT=y CONFIG_APPLE_AOP=m +CONFIG_TRUSTED_KEYS=y +# CONFIG_TRUSTED_KEYS_TPM is not set +# CONFIG_TRUSTED_KEYS_TEE is not set CONFIG_APPLE_SEP=m CONFIG_APPLE_PMGR_PWRSTATE=y CONFIG_APPLE_PMP=m From 003a6087f954c4e9be9de3e5c6be33c90661e559 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:21:08 +1000 Subject: [PATCH 0066/1579] soc: apple: sep: declare the mailbox and firmware-loader dependencies The driver uses the Rust Apple mailbox bindings and the firmware loader abstractions but declares neither, so it only builds when another driver (today APPLE_AOP) happens to pull them in. It also relies on runtime PM without depending on PM. Depend on PM and RUST_APPLE_MAILBOX and select RUST_FW_LOADER_ABSTRACTIONS. RUST_APPLE_MAILBOX has to be a dependency rather than a select: selecting it pulls in APPLE_MAILBOX, which NVME_APPLE depends on, and with NVME_APPLE selecting BLK_INLINE_ENCRYPTION for FileVault that closes a recursive dependency through CRYPTO and HW_RANDOM. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 1567c3b177c05..281e001631fb0 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -105,7 +105,9 @@ config APPLE_SEP_FV_INTERFACE config APPLE_SEP tristate "Apple SEP (Secure Enclave Processor)" depends on ARCH_APPLE + depends on PM depends on RUST + depends on RUST_APPLE_MAILBOX depends on HW_RANDOM depends on TRUSTED_KEYS depends on CRYPTO @@ -116,6 +118,7 @@ config APPLE_SEP select CRYPTO_GCM select CRYPTO_ECDH select APPLE_SEP_FV_INTERFACE + select RUST_FW_LOADER_ABSTRACTIONS help Driver for the Apple SEP (Secure Enclave Processor) on Apple silicon. It drives the Touch ID fingerprint sensor over a /dev/sep-bio character From cce3768ad43fa2d9247c71eadf1e99e6f935bb0b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:21:08 +1000 Subject: [PATCH 0067/1579] dt-bindings: spi: apple,spi: Add t8140-spi compatible The SPI controllers on the Apple A18 Pro (T8140) are compatible with the T8103 ones. Follow the T6020 entry and use "apple,t8103-spi" as the fallback rather than the generic "apple,spi". Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/spi/apple,spi.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/spi/apple,spi.yaml b/Documentation/devicetree/bindings/spi/apple,spi.yaml index 9356b9c337c8c..4b30265686fa1 100644 --- a/Documentation/devicetree/bindings/spi/apple,spi.yaml +++ b/Documentation/devicetree/bindings/spi/apple,spi.yaml @@ -16,7 +16,9 @@ properties: compatible: oneOf: - items: - - const: apple,t6020-spi + - enum: + - apple,t6020-spi + - apple,t8140-spi - const: apple,t8103-spi - items: - enum: From b35441a051037147d71fd514d0010cbdde6c3951 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:21:35 +1000 Subject: [PATCH 0068/1579] dt-bindings: input: apple,mesa-fingerprint: Name the SMC that switches the rails Boards whose Touch ID sensor rails are switched by the System Management Controller describe the SMC key and rail words, but not which SMC. The driver looked one up by the generic "apple,smc" compatible, which the SMC binding reserves for T8103, T8112 and T6000; newer SoCs such as T8140 fall back to "apple,t8103-smc" instead, so the lookup can never succeed there. Add an apple,smc phandle, require it together with the power key and rail words, and add an example for the MacBook Neo (J700). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- .../input/apple,mesa-fingerprint.yaml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml index 68531cf93ba0b..53b304bcec7ff 100644 --- a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml +++ b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml @@ -37,6 +37,12 @@ properties: maxItems: 1 description: Power line to the sensor, asserted to power it up. + apple,smc: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The System Management Controller that switches the rails named by + apple,smc-power-key and apple,smc-power-rails. + apple,smc-power-key: $ref: /schemas/types.yaml#/definitions/string description: @@ -70,6 +76,7 @@ oneOf: - required: - enable-gpios - required: + - apple,smc - apple,smc-power-key - apple,smc-power-rails @@ -94,3 +101,26 @@ examples: firmware-name = "apple/mesacal-j313.bin"; }; }; + + - | + #include + + spi { + #address-cells = <1>; + #size-cells = <0>; + + fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpol; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <135 IRQ_TYPE_EDGE_RISING>; + apple,smc = <&smc>; + apple,smc-power-key = "pcLD"; + apple,smc-power-rails = <0x01010000 0x01020000>; + firmware-name = "apple/mesacal-j700.bin"; + }; + }; From def7731da5cb3c31959cd45c932f4d04aa95f7be Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:21:35 +1000 Subject: [PATCH 0069/1579] soc: apple: sep: find the sensor's SMC through its phandle sep_acquire_smc_power() searched the whole tree for the generic "apple,smc" compatible. SoCs after T8112 do not carry that compatible (see apple,smc.yaml), so on the MacBook Neo the search failed and the sensor deferred its probe forever without saying why. Take the SMC from the sensor node's apple,smc phandle instead, and report a missing phandle as a device-tree error rather than deferring. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- drivers/soc/apple/sensor_shim.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index 9ed3658cda8f7..60cef6eee9e30 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -148,9 +148,11 @@ static int sep_acquire_smc_power(struct spi_device *spi) return -EINVAL; } - smc_np = of_find_compatible_node(NULL, NULL, "apple,smc"); - if (!smc_np) - return -EPROBE_DEFER; + smc_np = of_parse_phandle(np, "apple,smc", 0); + if (!smc_np) { + dev_warn(&spi->dev, "sep sensor: apple,smc-power-key needs an apple,smc phandle\n"); + return -EINVAL; + } smc_pdev = of_find_device_by_node(smc_np); of_node_put(smc_np); if (!smc_pdev) From aec8cb1999f731bbe1895509dbcc6e2fed977348 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:22:42 +1000 Subject: [PATCH 0070/1579] arm64: dts: apple: t8140: Add the SEP, its DART and mailbox, and SPI2 Describe the Secure Enclave Processor of the Apple A18 Pro (T8140), the DART and ASC mailbox it uses, the SPI2 controller that carries the Touch ID sensor on boards that have one, and the power domains they sit in (sep, sep_aon, spi_p, spi2). Register ranges, interrupts, the SEP DART aperture and the power-domain offsets come from the J700 Apple Device Tree. The power domains are the entries the PMGR generator emits from the ADT; their ADT parents ps_sio and ps_afisocni0 have no PMGR state register in that output and are not described. sep and sep_aon are always on: iBoot starts the SEP and it must never lose power under Linux. The SEP node stays disabled here; boards enable it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 35 ++++++++++++++++ arch/arm64/boot/dts/apple/t8140.dtsi | 49 +++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index f9058fa8efc30..c97bb4e4f009b 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -6,6 +6,15 @@ */ &pmgr { + ps_sep_aon: power-controller@60 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sep_aon"; + apple,always-on; + }; + ps_aic: power-controller@110 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x110 4>; @@ -65,6 +74,14 @@ label = "i2c0"; }; + ps_spi_p: power-controller@2f0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + }; + ps_apcie_st: power-controller@310 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x310 4>; @@ -74,6 +91,15 @@ power-domains = <&ps_ans>; }; + ps_spi2: power-controller@340 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x340 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi2"; + power-domains = <&ps_spi_p>; + }; + ps_apcie_sys_st: power-controller@3a0 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x3a0 4>; @@ -147,6 +173,15 @@ * apple,force-disable pattern) never reached ACTIVE again on J700 * (isp_cpu_core0 stuck at ACTUAL=0 on the second firmware boot). */ + ps_sep: power-controller@c00 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc00 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sep"; + apple,always-on; + }; + ps_isp_cpu: power-controller@4000 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x4000 4>; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index aa51d7a46e6a7..138d4af2d5dbe 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -288,6 +288,43 @@ #performance-domain-cells = <0>; }; + sep_dart: iommu@280b80000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x2 0x80b80000 0x0 0x20000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + /* + * The SEP only accepts IOVAs in this window, which already + * carries the BIT(40) alias the DART translates. + */ + apple,dma-range = /bits/ 64 <0x10000004000 0xffff0000>; + apple,dma-offset = /bits/ 64 <0x0>; + }; + + sep: sep@282600000 { + compatible = "apple,sep"; + reg = <0x2 0x82600000 0x0 0x88000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + power-domains = <&ps_sep>; + status = "disabled"; + }; + + sep_mbox: mailbox@282608000 { + compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x82608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + pmgr: power-management@300700000 { compatible = "apple,t8140-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; #address-cells = <1>; @@ -697,6 +734,18 @@ status = "disabled"; }; + spi2: spi@385108000 { + compatible = "apple,t8140-spi", "apple,t8103-spi"; + reg = <0x3 0x85108000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clkref>; + power-domains = <&ps_spi2>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + serial0: serial@30812c000 { compatible = "apple,dockchannel-uart"; reg = <0x3 0x0812c000 0x0 0x4000>, From 29987787b9238b15b41b92da87e5fcbf8a571e8c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:22:43 +1000 Subject: [PATCH 0071/1579] arm64: dts: apple: t8140-j700: Enable the SEP and the Touch ID sensor Enable the SEP on the MacBook Neo, add the "sep" alias the bootloader uses to hand over SEP properties, and describe the Touch ID ("Mesa") sensor on SPI2. Its rails are switched by the SMC through the pcLD key, so the node names the SMC by phandle together with the key and the rail words; its interrupt is AP GPIO 135, rising edge. All values are from the J700 Apple Device Tree. The calibration file is per-device factory data and is not shipped; firmware-name only fixes where the driver looks for it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 270932d29e966..d021dc1edc1e4 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -18,6 +18,7 @@ aliases { atcphy0 = &atcphy0; keyboard = &keyboard; + sep = &sep; serial0 = &serial0; }; @@ -78,6 +79,34 @@ }; }; +&sep { + status = "okay"; +}; + +/* + * Touch ID sensor. Its rails are switched by the SMC through the pcLD key + * rather than by a GPIO; the rail words are given in power-on order. The + * calibration file is per-device factory data and is not redistributable. + */ +&spi2 { + status = "okay"; + + fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpol; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <135 IRQ_TYPE_EDGE_RISING>; + apple,smc = <&smc>; + apple,smc-power-key = "pcLD"; + apple,smc-power-rails = <0x01010000 0x01020000>; + firmware-name = "apple/mesacal-j700.bin"; + }; +}; + &isp { apple,platform-id = <0x70>; apple,temporal-filter = <1>; From 2454d2abdd76e639eca78967e973870792bb9a2d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 03:01:36 +1000 Subject: [PATCH 0072/1579] dt-bindings: soc: apple: Add the Secure Enclave Processor The SEP node ("apple,sep") is used by the T8103, T8112, T602x and T8140 device trees and matched by the SEP driver, but has no binding, so dtbs_check rejects it. Describe it: the register window, the ASC mailbox, the DART the SEP reaches shared memory through, its power domain, and the properties the bootloader adds (the SEP firmware carveout and the two boot manifests the SEP needs to open the machine's secure storage). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- .../bindings/soc/apple/apple,sep.yaml | 78 +++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 79 insertions(+) create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,sep.yaml diff --git a/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml b/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml new file mode 100644 index 0000000000000..989ee4b9e9078 --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,sep.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Secure Enclave Processor + +maintainers: + - Ryan Murray + +description: + The Secure Enclave Processor (SEP) is a coprocessor on Apple SoCs that + holds device keys, provides a hardware random number generator and runs + the Touch ID matcher. The bootloader starts it. The operating system talks + to it through an ASC mailbox and through shared memory that the SEP + reaches via its own DART. + +properties: + compatible: + const: apple,sep + + reg: + maxItems: 1 + + mboxes: + maxItems: 1 + + mbox-names: + const: mbox + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + memory-region: + maxItems: 1 + description: + The SEP firmware carveout, added by the bootloader. + + iboot-manifest: + $ref: /schemas/types.yaml#/definitions/uint8-array + description: + The signed iBoot manifest of the running boot, added by the bootloader. + The SEP needs it to open the machine's secure storage. + + local-policy-manifest: + $ref: /schemas/types.yaml#/definitions/uint8-array + description: + The signed local boot policy of the running boot, added by the + bootloader, for the same purpose. + +required: + - compatible + - reg + - mboxes + - mbox-names + - iommus + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + sep@282600000 { + compatible = "apple,sep"; + reg = <0x2 0x82600000 0x0 0x88000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + power-domains = <&ps_sep>; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index 901897d0b3318..ced483de94493 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2582,6 +2582,7 @@ F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml F: Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml F: Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,sep.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml F: Documentation/devicetree/bindings/spmi/apple,spmi.yaml F: Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml From 6e5dca53551a8832d52a9ba9c65be6cae67d36b7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 01:30:05 +1000 Subject: [PATCH 0073/1579] dt-bindings: hwmon: apple,smc-hwmon: Allow #thermal-sensor-cells The hwmon core registers every temperature channel of the SMC hwmon device as a thermal sensor (the driver sets HWMON_C_REGISTER_TZ), so a board can back thermal zones with SMC temperatures. The binding does not allow the property that makes the node a sensor provider, though. Allow #thermal-sensor-cells with one cell, the index of the temperature channel in child-node order. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- .../devicetree/bindings/hwmon/apple,smc-hwmon.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index f3751b5af1720..291f6e8ca7ef8 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -40,6 +40,12 @@ properties: compatible: const: apple,smc-hwmon + "#thermal-sensor-cells": + const: 1 + description: + The cell is the zero-based index of a temperature channel, counting + the temperature-* child nodes in order. + apple,cpu-thermal-policy: type: boolean description: From 018ab982644a6eff94a6020d8910e9c8c187d6a2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 02:05:27 +1000 Subject: [PATCH 0074/1579] arm64: dts: apple: t8140: Add CPU energy model and thermal sensor cells Give the thermal framework what it needs to manage the A18 Pro (T8140) CPUs: - opp-microwatt for every OPP of both clusters, so the OPP core registers an energy model for the power allocator governor and the cpufreq cooling devices; - #cooling-cells on every CPU, so the cpufreq cooling devices are registered; - #thermal-sensor-cells on the SMC hwmon node, whose temperature channels the hwmon core already registers as thermal sensors. The power values were measured on a MacBook Neo (J700) following the method used for T8103/T600x and T8112: one busy core per cluster at each operating point, pinned with scaling_min/max_freq, read from the SMC's core-rail power meters. The P cluster's core rail was identified as the one whose voltage steps with the P-state (PP0b, 573 to 967 mV); from 3144 MHz up, the rise of the rail shared with the E cluster (PPab) is added. The E cluster's meter (PPbb) does not register a single busy core below 1788 MHz, so the E values are the cluster's idle reading plus a quarter of the rise with all four cores busy. The top P operating points were measured from a cold die in 2 s windows so that no thermal limit applied during the sample. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index aa51d7a46e6a7..4681a1dd68d42 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -59,6 +59,7 @@ d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; performance-domains = <&cpufreq_e>; + #cooling-cells = <2>; }; cpu_e1: cpu@1 { @@ -72,6 +73,7 @@ d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; performance-domains = <&cpufreq_e>; + #cooling-cells = <2>; }; cpu_e2: cpu@2 { @@ -85,6 +87,7 @@ d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; performance-domains = <&cpufreq_e>; + #cooling-cells = <2>; }; cpu_e3: cpu@3 { @@ -98,6 +101,7 @@ d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; performance-domains = <&cpufreq_e>; + #cooling-cells = <2>; }; cpu_p0: cpu@10100 { @@ -111,6 +115,7 @@ d-cache-size = <0x20000>; operating-points-v2 = <&pcluster_opp>; performance-domains = <&cpufreq_p>; + #cooling-cells = <2>; }; cpu_p1: cpu@10101 { @@ -124,6 +129,7 @@ d-cache-size = <0x20000>; operating-points-v2 = <&pcluster_opp>; performance-domains = <&cpufreq_p>; + #cooling-cells = <2>; }; l2_cache_0: l2-cache-0 { @@ -154,26 +160,32 @@ opp01 { opp-hz = /bits/ 64 <816000000>; opp-level = <1>; + opp-microwatt = <48800>; }; opp02 { opp-hz = /bits/ 64 <1140000000>; opp-level = <2>; + opp-microwatt = <68900>; }; opp03 { opp-hz = /bits/ 64 <1464000000>; opp-level = <3>; + opp-microwatt = <88100>; }; opp04 { opp-hz = /bits/ 64 <1788000000>; opp-level = <4>; + opp-microwatt = <129600>; }; opp05 { opp-hz = /bits/ 64 <2112000000>; opp-level = <5>; + opp-microwatt = <172600>; }; opp06 { opp-hz = /bits/ 64 <2424000000>; opp-level = <6>; + opp-microwatt = <243500>; }; }; @@ -184,70 +196,87 @@ opp01 { opp-hz = /bits/ 64 <744000000>; opp-level = <1>; + opp-microwatt = <188800>; }; opp02 { opp-hz = /bits/ 64 <1260000000>; opp-level = <2>; + opp-microwatt = <368900>; }; opp03 { opp-hz = /bits/ 64 <1620000000>; opp-level = <3>; + opp-microwatt = <491000>; }; opp04 { opp-hz = /bits/ 64 <1944000000>; opp-level = <4>; + opp-microwatt = <607600>; }; opp05 { opp-hz = /bits/ 64 <2196000000>; opp-level = <5>; + opp-microwatt = <707300>; }; opp06 { opp-hz = /bits/ 64 <2472000000>; opp-level = <6>; + opp-microwatt = <935100>; }; opp07 { opp-hz = /bits/ 64 <2688000000>; opp-level = <7>; + opp-microwatt = <1114000>; }; opp08 { opp-hz = /bits/ 64 <2916000000>; opp-level = <8>; + opp-microwatt = <1324800>; }; opp09 { opp-hz = /bits/ 64 <3144000000>; opp-level = <9>; + opp-microwatt = <1772600>; }; opp10 { opp-hz = /bits/ 64 <3336000000>; opp-level = <10>; + opp-microwatt = <2045700>; }; opp11 { opp-hz = /bits/ 64 <3504000000>; opp-level = <11>; + opp-microwatt = <2329300>; }; opp12 { opp-hz = /bits/ 64 <3648000000>; opp-level = <12>; + opp-microwatt = <2621600>; }; opp13 { opp-hz = /bits/ 64 <3756000000>; opp-level = <13>; + opp-microwatt = <2803500>; }; opp14 { opp-hz = /bits/ 64 <3864000000>; opp-level = <14>; + opp-microwatt = <3064600>; }; opp15 { opp-hz = /bits/ 64 <3924000000>; opp-level = <15>; + opp-microwatt = <3223300>; }; opp16 { opp-hz = /bits/ 64 <3972000000>; opp-level = <16>; + opp-microwatt = <3342600>; }; opp17 { opp-hz = /bits/ 64 <4044000000>; opp-level = <17>; + opp-microwatt = <3424900>; }; }; @@ -402,6 +431,7 @@ smc_hwmon: hwmon { compatible = "apple,smc-hwmon"; + #thermal-sensor-cells = <1>; /* * CPU cluster die temperatures. Keep these From d69c31212d2b93b6396775c7890d5734f962c486 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 02:06:38 +1000 Subject: [PATCH 0075/1579] arm64: dts: apple: t8140-j700: Add CPU thermal zones The MacBook Neo is fanless, so CPU frequency is its only cooling. Describe a thermal zone for each of the six CPU die sensors the SMC reports (E cluster Te05/Te0S, P cluster Tp01/Tp05/Tp09/Tp0D), each throttling both clusters: they share one die and one heat spreader. The power allocator splits each zone's budget by the CPUs' energy models, with the E cluster weighted three to one because its cores do several times more work per watt. Two properties of this machine decide the parameters. The SMC refreshes the readings about once a second and the die settles within that time after a power change, so the loop carries a full sample of delay. And the power a core draws per energy-model watt varies with the workload: a floating-point loop drew 2.7 times the P-core power of an integer loop at the same clock. The governor derives its gains from sustainable-power and the gap between the switch-on and control trips: - 80/95 C with 2.5 W (gains 170/341/34 mW per C) swung the P cluster between its lowest and highest P-state every two seconds; - 58/95 C with 2.5 W (68/135/14) oscillated on the floating-point load; - 33/95 C with 2.5 W (40/80/8) was stable, but the governor only resets its integral term below the switch-on temperature, which this machine never reaches, so time spent above 95 C lowered the budget until the next reboot (measured: -11% sustained P clock after one heavy run); - 58/95 C with 1.5 W (40/81/8) is stable on both loads and resets whenever the machine rests (it idles at 48-53 C). This uses the last. 1.5 W lies inside the sustainable range measured at about 93 C: 1.1 W for the floating-point and 2.5 W for the integer load. The critical trip is 110 C; the zones poll every 500 ms. Measured on a J700 with this device tree (schedutil): 1 P core, burst from idle 4044 MHz for 1 s, then 3504; 83 C 2 P cores, burst from idle 4044 MHz for 1 s, then 1944-2688; 95.6 C 1 P core, integer, 75 s mostly 3336 MHz at 84-88 C all cores, integer, 75 s P 1944 / E 2424 MHz, 85 C, no changes all cores, integer, 10 min P 1620-1944 / E 2424 MHz, 86.5 C mean 1 P core, float, 60 s 1944-2916 MHz, 92.6 C mean all cores, float, 90 s P 979 / E 2424 MHz, 92.5 C all cores, integer, after the float runs and 60 s idle: unchanged The removed walk ran one integer P core at 3504 MHz and all cores at P 2688 / E 1464 MHz, but touched 103.9 C at the start of all-core loads. Sustained P clocks here are 5-20% lower in exchange for a loop that stays stable on every workload tried and never carries wind-up from one load to the next; the power allocator's sysfs parameters (k_po, k_pu, k_i, integral_cutoff) can trade some of that back. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 305 +++++++++++++++++++++++ 1 file changed, 305 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 270932d29e966..6a2e0dc88d74d 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -10,6 +10,7 @@ /dts-v1/; #include "t8140.dtsi" +#include / { compatible = "apple,j700", "apple,t8140", "apple,arm-platform"; @@ -51,6 +52,310 @@ device_type = "memory"; reg = <0x100 0 0x2 0>; /* To be filled by loader */ }; + + /* + * The J700 is fanless: the CPUs are the only cooling devices. Every + * die sensor gets a zone that throttles both clusters, since they share + * one die and one heat spreader. The power allocator splits the budget + * by the energy models, with the E cluster weighted three to one: its + * cores do several times more work per watt, so all-core loads keep + * them at full speed and trim the P cores instead. + * + * The SMC refreshes these readings about once a second and the die + * settles within that time, so the loop carries a full sample of delay, + * and the power a core draws per energy-model watt varies about 2.7x + * between integer and floating-point work. The governor derives its + * gains from sustainable-power and the switch-on to control gap; 1.5 W + * over 37 C gives 40/81/8 mW per C, which held the clock within one + * P-state on both kinds of load (higher gains oscillate on the heavy + * one). 1.5 W lies inside the sustainable range measured at ~93 C: + * 1.1 W for floating-point and 2.5 W for integer loads. The switch-on + * trip sits above the idle temperature (48-53 C), so the controller, + * and with it any integral wind-up, resets whenever the machine rests. + */ + thermal-zones { + cpu-e0-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 0>; /* Te05 */ + sustainable-power = <1500>; + + trips { + cpu_e0_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_e0_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_e0_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_e0_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <3072>; + }; + }; + }; + + cpu-e1-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 1>; /* Te0S */ + sustainable-power = <1500>; + + trips { + cpu_e1_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_e1_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_e1_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_e1_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <3072>; + }; + }; + }; + + cpu-p0-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 2>; /* Tp01 */ + sustainable-power = <1500>; + + trips { + cpu_p0_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p0_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p0_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p0_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <3072>; + }; + }; + }; + + cpu-p1-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 3>; /* Tp05 */ + sustainable-power = <1500>; + + trips { + cpu_p1_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p1_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p1_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p1_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <3072>; + }; + }; + }; + + cpu-p2-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 4>; /* Tp09 */ + sustainable-power = <1500>; + + trips { + cpu_p2_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p2_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p2_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p2_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <3072>; + }; + }; + }; + + cpu-p3-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 5>; /* Tp0D */ + sustainable-power = <1500>; + + trips { + cpu_p3_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p3_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p3_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p3_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <3072>; + }; + }; + }; + }; }; #include "isp-imx558.dtsi" From 1edf86eccb7ab87f92a6835bed7a0d386329fbab Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 02:06:39 +1000 Subject: [PATCH 0076/1579] thermal: gov_power_allocator: Only warn about sustainable power when it is used power_allocator_bind() warns that sustainable power will be estimated for every zone that does not provide it, including zones without any passive or active trip point. The governor never controls such a zone: power_allocator_manage() returns early when there is no trip_max, and the estimate is never computed. With power_allocator as the default governor every trip-less zone, such as the one each power supply registers for its battery, logs this warning at boot. Warn only when the zone has a trip the governor controls. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- drivers/thermal/gov_power_allocator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/thermal/gov_power_allocator.c b/drivers/thermal/gov_power_allocator.c index 37f2e22a999e6..31f778dd49ed2 100644 --- a/drivers/thermal/gov_power_allocator.c +++ b/drivers/thermal/gov_power_allocator.c @@ -728,10 +728,10 @@ static int power_allocator_bind(struct thermal_zone_device *tz) params->allocated_tzp = true; } - if (!tz->tzp->sustainable_power) - dev_warn(&tz->device, "power_allocator: sustainable_power will be estimated\n"); - else + if (tz->tzp->sustainable_power) params->sustainable_power = tz->tzp->sustainable_power; + else if (params->trip_max) + dev_warn(&tz->device, "power_allocator: sustainable_power will be estimated\n"); if (params->trip_max) estimate_pid_constants(tz, tz->tzp->sustainable_power, From d7ae133578fb57775af980c557d66db52aca8432 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 02:06:39 +1000 Subject: [PATCH 0077/1579] arm64: configs: asahi: Default to the power allocator thermal governor Device-tree thermal zones cannot name their governor, and the step_wise default moves a cooling device by one state per temperature change. The T8140 SMC reports die temperatures about once a second while the die can heat by tens of degrees in that time, so step_wise needs many seconds to reach a safe frequency and then oscillates around the trip. The power allocator moves the cooling devices straight to the state that fits the power budget and splits the budget between clusters by their energy models. Zones without a controlled trip are unaffected. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 20a50b6fcb3a1..49f93bdbe7a22 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -8,6 +8,7 @@ CONFIG_ARM64_MEMORY_MODEL_CONTROL=y CONFIG_ARM_APPLE_CPUIDLE=y CONFIG_ARM_APPLE_SOC_CPUFREQ=m CONFIG_APPLE_SMC_CPU_THERMAL=y +CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m CONFIG_PCIE_APPLE=m From ca1dab3dde89c57cc0b111fd096478241e6565e5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 02:06:47 +1000 Subject: [PATCH 0078/1579] cpufreq: apple-soc: Drop the SMC CPU thermal policy hooks The T8140 CPUs are now managed by the thermal framework: the SMC die sensors back device-tree thermal zones and the cpufreq cooling devices throttle both clusters. The driver no longer has to register every policy with the private SMC thermal controller, report DVFS faults to it, or refuse the P-states above 2 without it. A DVFS transition failure still stops further requests, as before. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 35 ----------------------------- 1 file changed, 35 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index c37fb272ada73..f2a1e91371be4 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -21,7 +21,6 @@ #include #include #include -#include #define APPLE_DVFS_CMD 0x20 #define APPLE_DVFS_CMD_BUSY BIT(31) @@ -78,7 +77,6 @@ struct apple_cpu_priv { const struct apple_soc_cpufreq_info *info; bool transition_failed; unsigned int expected_pstate; - struct apple_smc_thermal_cpu *thermal; }; static struct cpufreq_driver apple_soc_cpufreq_driver; @@ -210,7 +208,6 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, APPLE_DVFS_TRANSITION_TIMEOUT)) { if (priv->info->verify_transition) { priv->transition_failed = true; - apple_smc_thermal_cpu_fault(priv->thermal); dev_err(priv->cpu_dev, "DVFS busy timeout, command=%#llx; stopping requests\n", reg); } @@ -223,7 +220,6 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, /* Do not fight an unexpected retained policy owner. */ if (previous != priv->expected_pstate) { priv->transition_failed = true; - apple_smc_thermal_cpu_fault(priv->thermal); dev_err(priv->cpu_dev, "unexpected DVFS state %u (expected %u); stopping requests\n", previous, priv->expected_pstate); return -EIO; @@ -249,7 +245,6 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, 2, APPLE_DVFS_TRANSITION_TIMEOUT)) { /* Do not issue another request or guess a rollback after failure. */ priv->transition_failed = true; - apple_smc_thermal_cpu_fault(priv->thermal); dev_err(priv->cpu_dev, "DVFS completion timeout, command=%#llx; stopping requests\n", reg); return -ETIMEDOUT; @@ -301,7 +296,6 @@ static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) { - struct cpufreq_frequency_table *p; int ret, i; unsigned int transition_latency; void __iomem *reg_base; @@ -403,26 +397,6 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) policy->dvfs_possible_from_any_cpu = true; policy->fast_switch_possible = !info->verify_transition; policy->suspend_freq = freq_table[0].frequency; - if (info->verify_transition) { - struct device_node *hwmon; - bool higher = false; - bool thermal; - - /* P-states above 2 are only allowed under the SMC thermal policy. */ - cpufreq_for_each_valid_entry(p, policy->freq_table) - if (p->driver_data > 2) - higher = true; - hwmon = of_find_compatible_node(NULL, NULL, "apple,smc-hwmon"); - thermal = hwmon && of_property_read_bool(hwmon, "apple,cpu-thermal-policy"); - of_node_put(hwmon); - if (higher || thermal) { - priv->thermal = apple_smc_thermal_cpu_add(policy); - if (IS_ERR(priv->thermal)) { - ret = PTR_ERR(priv->thermal); - goto out_free_cpufreq_table; - } - } - } return 0; @@ -441,7 +415,6 @@ static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) { struct apple_cpu_priv *priv = policy->driver_data; - apple_smc_thermal_cpu_remove(priv->thermal); dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); /* * The core clears the last CPU from policy->cpus before calling @@ -452,13 +425,6 @@ static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) kfree(priv); } -static void apple_soc_cpufreq_ready(struct cpufreq_policy *policy) -{ - struct apple_cpu_priv *priv = policy->driver_data; - - apple_smc_thermal_cpu_ready(priv->thermal); -} - static struct cpufreq_driver apple_soc_cpufreq_driver = { .name = "apple-cpufreq", .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | @@ -472,7 +438,6 @@ static struct cpufreq_driver apple_soc_cpufreq_driver = { .register_em = cpufreq_register_em_with_opp, .set_boost = cpufreq_boost_set_sw, .suspend = cpufreq_generic_suspend, - .ready = apple_soc_cpufreq_ready, }; static int __init apple_soc_cpufreq_module_init(void) From 6c65c2bc47d2abb610babc6e31fc14ecd4d3c2fe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 02:07:02 +1000 Subject: [PATCH 0079/1579] hwmon: (macsmc-hwmon) Drop the J700 CPU thermal policy sampler This reverts commit cbbb86645f96 ("hwmon: (macsmc-hwmon) Expose the CPU sensors to the SMC thermal policy"). The J700 CPU die temperatures now reach the thermal framework the standard way: the hwmon core registers each temperature channel as a thermal sensor (HWMON_C_REGISTER_TZ) and the device tree's thermal zones reference them. The private sampler, its read cache and the positional six-channel contract it imposed are no longer used. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- drivers/hwmon/macsmc-hwmon.c | 93 ------------------------------------ 1 file changed, 93 deletions(-) diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index b41f8813d4165..13e38b25c75fb 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -22,26 +22,17 @@ #include #include -#include #include #include #include #include #include -#include #define MAX_LABEL_LENGTH 32 /* Temperature, voltage, current, power, fan(s) */ #define NUM_SENSOR_TYPES 5 -/* - * The J700 CPU thermal policy samples a fixed set of die temperatures, which - * must therefore be the first temperature channels; any further channels are - * only reported to userspace and never feed the policy. - */ -#define APPLE_SMC_CPU_POLICY_SENSORS 6 - #define FLT_EXP_BIAS 127 #define FLT_EXP_MASK GENMASK(30, 23) #define FLT_MANT_BIAS 23 @@ -95,12 +86,6 @@ struct macsmc_hwmon { struct macsmc_hwmon_sensors curr; struct macsmc_hwmon_sensors power; struct macsmc_hwmon_fans fan; - bool cpu_thermal_policy; - /* Protects the thermal_cache fields below. */ - struct mutex thermal_cache_lock; - long thermal_cache[APPLE_SMC_CPU_POLICY_SENSORS]; - unsigned long thermal_cache_time; - int thermal_cache_error; }; static int macsmc_hwmon_read_label(struct device *dev, @@ -374,17 +359,6 @@ static int macsmc_hwmon_read(struct device *dev, enum hwmon_sensor_types type, switch (type) { case hwmon_temp: - if (hwmon->cpu_thermal_policy && channel < APPLE_SMC_CPU_POLICY_SENSORS) { - guard(mutex)(&hwmon->thermal_cache_lock); - - if (hwmon->thermal_cache_error) - return hwmon->thermal_cache_error; - if (time_after_eq(jiffies, hwmon->thermal_cache_time + - msecs_to_jiffies(APPLE_SMC_THERMAL_MAX_AGE_MS))) - return -ETIMEDOUT; - *val = hwmon->thermal_cache[channel]; - return 0; - } ret = macsmc_hwmon_read_key(hwmon->smc, &hwmon->temp.sensors[channel], 1000, val); break; @@ -808,70 +782,6 @@ static int macsmc_hwmon_create_infos(struct macsmc_hwmon *hwmon) return 0; } -static int macsmc_hwmon_cpu_sample(void *ctx, int *temp) -{ - struct macsmc_hwmon *hwmon = ctx; - unsigned long start = jiffies; - long values[APPLE_SMC_CPU_POLICY_SENSORS], maximum = LONG_MIN; - unsigned int i; - int ret = 0; - - for (i = 0; i < ARRAY_SIZE(values); i++) { - ret = macsmc_hwmon_read_key(hwmon->smc, &hwmon->temp.sensors[i], 1000, &values[i]); - if (ret) - break; - if (values[i] < INT_MIN || values[i] > INT_MAX) { - ret = -ERANGE; - break; - } - maximum = max(maximum, values[i]); - } - if (!ret && time_after_eq(jiffies, start + msecs_to_jiffies(APPLE_SMC_THERMAL_MAX_AGE_MS))) - ret = -ETIMEDOUT; - guard(mutex)(&hwmon->thermal_cache_lock); - hwmon->thermal_cache_error = ret; - if (!ret) { - memcpy(hwmon->thermal_cache, values, sizeof(values)); - hwmon->thermal_cache_time = start; - *temp = maximum; - } - return ret; -} - -static int macsmc_hwmon_cpu_policy(struct macsmc_hwmon *hwmon) -{ - static const smc_key keys[] = { - SMC_KEY(Te05), SMC_KEY(Te0S), SMC_KEY(Tp01), - SMC_KEY(Tp05), SMC_KEY(Tp09), SMC_KEY(Tp0D), - }; - unsigned int i, j, seen = 0; - - static_assert(ARRAY_SIZE(keys) == APPLE_SMC_CPU_POLICY_SENSORS); - - if (!of_property_read_bool(hwmon->dev->of_node, "apple,cpu-thermal-policy")) - return 0; - if (!of_machine_is_compatible("apple,j700") || hwmon->temp.count < ARRAY_SIZE(keys)) - return -EINVAL; - for (i = 0; i < ARRAY_SIZE(keys); i++) { - struct macsmc_hwmon_sensor *sensor = &hwmon->temp.sensors[i]; - - if (sensor->info.type_code != __SMC_KEY('f', 'l', 't', ' ') || - sensor->info.size != 4 || !(sensor->info.flags & APPLE_SMC_READABLE) || - (sensor->info.flags & APPLE_SMC_FUNCTION)) - return -EINVAL; - for (j = 0; j < ARRAY_SIZE(keys); j++) - if (sensor->macsmc_key == keys[j]) - break; - if (j == ARRAY_SIZE(keys) || seen & BIT(j)) - return -EINVAL; - seen |= BIT(j); - } - mutex_init(&hwmon->thermal_cache_lock); - hwmon->thermal_cache_error = -ENODATA; - hwmon->cpu_thermal_policy = true; - return devm_apple_smc_thermal_register(hwmon->dev, macsmc_hwmon_cpu_sample, hwmon); -} - static int macsmc_hwmon_probe(struct platform_device *pdev) { struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); @@ -911,9 +821,6 @@ static int macsmc_hwmon_probe(struct platform_device *pdev) ret = macsmc_hwmon_create_infos(hwmon); if (ret) return ret; - ret = macsmc_hwmon_cpu_policy(hwmon); - if (ret) - return dev_err_probe(hwmon->dev, ret, "CPU thermal provider setup failed\n"); hwmon->chip_info.ops = &macsmc_hwmon_ops; hwmon->chip_info.info = From 52006c8e0bc5750331643c2f36ebad06ef300079 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 02:08:04 +1000 Subject: [PATCH 0080/1579] thermal: Remove the Apple SMC CPU thermal policy The J700 CPU thermal policy was a controller outside the thermal framework: it registered a zone with the step_wise governor but did its own control through a frequency QoS request on each cluster, walking the OPP tables from the hottest of six SMC readings. The same machine is now described the standard way: device-tree thermal zones on the SMC die sensors, the cpufreq cooling devices, and the power allocator with an energy model measured on the hardware. Nothing uses the private controller any more, so remove it, its header and its configuration. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- MAINTAINERS | 2 - arch/arm64/configs/asahi.config | 1 - drivers/thermal/Kconfig | 20 -- drivers/thermal/Makefile | 1 - drivers/thermal/apple-smc-cpu-thermal.c | 394 ------------------------ include/linux/soc/apple/smc-thermal.h | 37 --- 6 files changed, 455 deletions(-) delete mode 100644 drivers/thermal/apple-smc-cpu-thermal.c delete mode 100644 include/linux/soc/apple/smc-thermal.h diff --git a/MAINTAINERS b/MAINTAINERS index 368bf2b43af1c..eaaf89f34984b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1986,9 +1986,7 @@ F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ F: drivers/soc/apple/aop.rs -F: drivers/thermal/apple-smc-cpu-thermal.c F: drivers/tty/serial/apple_dockchannel* -F: include/linux/soc/apple/smc-thermal.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 49f93bdbe7a22..b55bd80fc5829 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -7,7 +7,6 @@ CONFIG_ARM64_16K_PAGES=y CONFIG_ARM64_MEMORY_MODEL_CONTROL=y CONFIG_ARM_APPLE_CPUIDLE=y CONFIG_ARM_APPLE_SOC_CPUFREQ=m -CONFIG_APPLE_SMC_CPU_THERMAL=y CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index 15fc1d56d9540..b10080d618604 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -527,24 +527,4 @@ config LOONGSON2_THERMAL is higher than the high temperature threshold or lower than the low temperature threshold, the interrupt will occur. -config APPLE_SMC_CPU_THERMAL - bool "Apple J700 SMC CPU thermal policy" - depends on ARCH_APPLE || COMPILE_TEST - depends on ARM_APPLE_SOC_CPUFREQ && SENSORS_MACSMC_HWMON - depends on CPU_FREQ_THERMAL - select THERMAL_GOV_STEP_WISE - help - CPU thermal policy for the Apple J700 (T8140). It follows the - hottest of six SMC CPU die sensors, sends both CPU clusters to - their lowest frequency through cpufreq cooling devices above a - passive trip, and holds them there through a frequency QoS - request while the readings are missing, stale or failing, while - the zone is disabled, or above a second, hard threshold. - - apple-soc-cpufreq only allows T8140 P-states above 2 when this - policy is present. The thresholds are Linux operating policy, not - Apple or silicon limits. - - Say Y for J700 kernels. - endif diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile index e88a682c75851..bb21e7ea7fc6b 100644 --- a/drivers/thermal/Makefile +++ b/drivers/thermal/Makefile @@ -68,5 +68,4 @@ obj-$(CONFIG_AMLOGIC_THERMAL) += amlogic_thermal.o obj-$(CONFIG_SPRD_THERMAL) += sprd_thermal.o obj-$(CONFIG_KHADAS_MCU_FAN_THERMAL) += khadas_mcu_fan.o obj-$(CONFIG_LOONGSON2_THERMAL) += loongson2_thermal.o -obj-$(CONFIG_APPLE_SMC_CPU_THERMAL) += apple-smc-cpu-thermal.o obj-$(CONFIG_THERMAL_CORE_TESTING) += testing/ diff --git a/drivers/thermal/apple-smc-cpu-thermal.c b/drivers/thermal/apple-smc-cpu-thermal.c deleted file mode 100644 index f9709c4061ec4..0000000000000 --- a/drivers/thermal/apple-smc-cpu-thermal.c +++ /dev/null @@ -1,394 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* - * Apple J700 CPU thermal policy over selected SMC telemetry - * - * Copyright (C) 2026 Eryk Wieliczko - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * The hottest of the six selected sensors drives a frequency cap on each - * cluster through a frequency-QoS guard. The cap walks the cluster's own - * OPP table: every SMC_TIGHTEN_MS that the reading is at or above - * SMC_HIGH_MC it moves one step tighter, plus one per 2 C above the band - * (five at most), and every SMC_RELAX_MS that a low-pass filtered reading - * (time constant about two seconds) is below SMC_LOW_MC it moves one step - * looser. The die sensor of this fanless machine moves 30 C within a - * second of a clock change and the SMC updates it about once a second, so - * a single trip bound to maximum cooling swung the P cluster between 4044 - * and 744 MHz every two seconds, and a walk driven only by the filtered - * value never tightened while the readings alternated above and below the - * band. Tightening from the raw reading lands the walk near the step the - * enclosure can sustain within a few readings; relaxing from the average - * keeps it within one step of it. A hard tier on the raw reading still pins both clusters to their - * minimum from SMC_HOT_MC until SMC_HOT_RELEASE_MC, and the guard holds the - * minimum while readings are missing, stale or failing, while the zone is - * disabled, and after a cpufreq DVFS fault. - * - * The band sits above the idle temperature: the original 80 C trip with a - * 70 C release latched the throttle permanently while the die idled at - * 70-77 C with the spinning idle loop. - * - * The thresholds are Linux operating policy, not Apple or silicon limits. - */ -#define SMC_HIGH_MC 95000 -#define SMC_LOW_MC 88000 -#define SMC_HOT_MC 103000 -#define SMC_HOT_RELEASE_MC 98000 -#define SMC_TIGHTEN_MS 500 -#define SMC_RELAX_MS 4000 -/* Steps of the walk; a cluster with fewer OPPs maps them proportionally. */ -#define SMC_STEPS 17 -/* Filter weight per 25 ms sample, in 1/1024: about a two second time constant. */ -#define SMC_FILTER_WEIGHT 13 -#define SMC_SAMPLE_MS 25 -#define SMC_MAX_AGE_MS APPLE_SMC_THERMAL_MAX_AGE_MS -#define SMC_WATCH_MS 25 - -struct apple_smc_thermal_cpu { - struct cpufreq_policy *policy; - struct freq_qos_request guard; - unsigned int min_freq; - unsigned int cap[SMC_STEPS]; - bool failed; -}; - -static DEFINE_MUTEX(smc_lock); -/* Only sampling/removal take this lock: a stuck SMC read cannot block guards. */ -static DEFINE_MUTEX(smc_provider_lock); -static struct apple_smc_thermal_cpu *smc_cpus[2]; -static struct thermal_zone_device *smc_zone; -static int (*smc_read)(void *, int *); -static void *smc_ctx; -static unsigned long smc_sample_time; -static unsigned int smc_good; -static int smc_value, smc_error = -ENODATA; -static bool smc_enabled, smc_hot; -static unsigned int smc_step; -static unsigned long smc_step_time; -static int smc_filtered; -static void smc_sample_work(struct work_struct *work); -static void smc_watch_work(struct work_struct *work); -static DECLARE_DELAYED_WORK(smc_sample, smc_sample_work); -static DECLARE_DELAYED_WORK(smc_watch, smc_watch_work); - -static bool smc_fresh(void) -{ - return !smc_error && time_before(jiffies, - smc_sample_time + msecs_to_jiffies(SMC_MAX_AGE_MS)); -} - -/* smc_lock held. QoS is independent of user limits and the thermal governor. */ -static void smc_update_guards(void) -{ - bool healthy = smc_zone && smc_enabled && smc_fresh() && smc_good >= 3; - bool clamp = !healthy || smc_hot; - bool error = false; - unsigned int i; - - for (i = 0; i < ARRAY_SIZE(smc_cpus); i++) - if (smc_cpus[i] && READ_ONCE(smc_cpus[i]->failed)) - clamp = true; - for (i = 0; i < ARRAY_SIZE(smc_cpus); i++) { - struct apple_smc_thermal_cpu *cpu = smc_cpus[i]; - int ret; - - if (!cpu) - continue; - ret = freq_qos_update_request(&cpu->guard, clamp || READ_ONCE(cpu->failed) ? - cpu->min_freq : cpu->cap[smc_step]); - if (ret < 0) { - error = true; - WRITE_ONCE(cpu->failed, true); - freq_qos_update_request(&cpu->guard, cpu->min_freq); - pr_err_ratelimited("apple-smc-thermal: CPU%u guard update failed: %d\n", - cpu->policy->cpu, ret); - } - } - if (error) - for (i = 0; i < ARRAY_SIZE(smc_cpus); i++) - if (smc_cpus[i]) - freq_qos_update_request(&smc_cpus[i]->guard, smc_cpus[i]->min_freq); -} - -static int smc_zone_temp(struct thermal_zone_device *zone, int *temp) -{ - guard(mutex)(&smc_lock); - if (!smc_fresh()) - return smc_error ?: -ETIMEDOUT; - *temp = smc_value; - return 0; -} - -static int smc_zone_mode(struct thermal_zone_device *zone, enum thermal_device_mode mode) -{ - guard(mutex)(&smc_lock); - smc_enabled = mode == THERMAL_DEVICE_ENABLED; - smc_good = 0; - smc_update_guards(); - return 0; -} - -static const struct thermal_zone_device_ops smc_zone_ops = { - .get_temp = smc_zone_temp, - .change_mode = smc_zone_mode, -}; - -/* The band and the hard tier as passive trips, for sysfs; nothing is bound to them. */ -static const struct thermal_trip smc_trips[] = { - { .temperature = SMC_HIGH_MC, .hysteresis = SMC_HIGH_MC - SMC_LOW_MC, - .type = THERMAL_TRIP_PASSIVE }, - { .temperature = SMC_HOT_MC, .hysteresis = SMC_HOT_MC - SMC_HOT_RELEASE_MC, - .type = THERMAL_TRIP_PASSIVE }, -}; - -static const struct thermal_zone_params smc_params = { - .governor_name = "step_wise", - .no_hwmon = true, -}; - -/* provider_lock held; never register/update a zone while holding smc_lock. */ -static void smc_start_zone(void) -{ - struct thermal_zone_device *zone; - int ret; - - if (smc_zone || !smc_read || !smc_cpus[0] || !smc_cpus[1]) - return; - zone = thermal_zone_device_register_with_trips("apple-smc-selected", - smc_trips, - ARRAY_SIZE(smc_trips), - NULL, &smc_zone_ops, - &smc_params, - SMC_SAMPLE_MS, - SMC_SAMPLE_MS); - if (IS_ERR(zone)) { - pr_err("apple-smc-thermal: zone registration failed: %ld\n", PTR_ERR(zone)); - return; - } - mutex_lock(&smc_lock); - smc_zone = zone; - mutex_unlock(&smc_lock); - ret = thermal_zone_device_enable(zone); - if (ret) - pr_err("apple-smc-thermal: zone enable failed: %d; retaining minimum caps\n", ret); -} - -static void smc_sample_work(struct work_struct *work) -{ - unsigned long start = jiffies; - int value = 0, ret; - - guard(mutex)(&smc_provider_lock); - if (!smc_read) - return; - ret = smc_read(smc_ctx, &value); - if (!ret && time_after_eq(jiffies, start + msecs_to_jiffies(SMC_MAX_AGE_MS))) - ret = -ETIMEDOUT; - mutex_lock(&smc_lock); - smc_error = ret; - if (ret) { - smc_good = 0; - } else { - smc_value = value; - smc_sample_time = start; /* Bound the age of the oldest read in the round. */ - if (smc_good < 3) { - smc_good++; - smc_filtered = value; - } else { - smc_filtered += ((value - smc_filtered) * SMC_FILTER_WEIGHT) / 1024; - } - if (value >= SMC_HOT_MC) - smc_hot = true; - else if (value < SMC_HOT_RELEASE_MC) - smc_hot = false; - if (value >= SMC_HIGH_MC && - time_after_eq(start, smc_step_time + msecs_to_jiffies(SMC_TIGHTEN_MS))) { - /* One step, plus one per 2 C above the band, up to five. */ - unsigned int steps = 1 + min((value - SMC_HIGH_MC) / 2000, 4); - - smc_step = min(smc_step + steps, SMC_STEPS - 1); - smc_step_time = start; - } else if (smc_step > 0 && smc_filtered < SMC_LOW_MC && - time_after_eq(start, smc_step_time + msecs_to_jiffies(SMC_RELAX_MS))) { - smc_step--; - smc_step_time = start; - } - } - smc_update_guards(); - mutex_unlock(&smc_lock); - if (smc_zone) - thermal_zone_device_update(smc_zone, THERMAL_EVENT_UNSPECIFIED); - queue_delayed_work(system_unbound_wq, &smc_sample, msecs_to_jiffies(SMC_SAMPLE_MS)); -} - -static void smc_watch_work(struct work_struct *work) -{ - guard(mutex)(&smc_lock); - if (!smc_fresh()) - smc_good = 0; - smc_update_guards(); - if (smc_cpus[0] || smc_cpus[1]) - queue_delayed_work(system_unbound_wq, &smc_watch, msecs_to_jiffies(SMC_WATCH_MS)); -} - -/* - * The walk's caps for one cluster: step 0 is uncapped, the last step is the - * lowest OPP, and the steps between map proportionally onto the cluster's - * table so that both clusters descend together. - */ -static void smc_cpu_caps(struct apple_smc_thermal_cpu *cpu) -{ - struct cpufreq_frequency_table *pos, *table = cpu->policy->freq_table; - unsigned int freqs[SMC_STEPS * 4], count = 0, i, j; - - cpufreq_for_each_valid_entry(pos, table) { - if (count == ARRAY_SIZE(freqs)) - break; - for (i = 0; i < count && freqs[i] > pos->frequency; i++) - ; - for (j = count; j > i; j--) - freqs[j] = freqs[j - 1]; - freqs[i] = pos->frequency; - count++; - } - for (i = 0; i < SMC_STEPS; i++) { - if (!i || !count) { - cpu->cap[i] = FREQ_QOS_MAX_DEFAULT_VALUE; - continue; - } - j = (i * (count - 1) + (SMC_STEPS - 1) / 2) / (SMC_STEPS - 1); - cpu->cap[i] = freqs[min(j, count - 1)]; - } -} - -struct apple_smc_thermal_cpu *apple_smc_thermal_cpu_add(struct cpufreq_policy *policy) -{ - struct apple_smc_thermal_cpu *cpu; - unsigned int i; - int ret; - - if (!of_machine_is_compatible("apple,j700")) - return ERR_PTR(-ENODEV); - cpu = kzalloc_obj(*cpu); - if (!cpu) - return ERR_PTR(-ENOMEM); - cpu->policy = policy; - cpu->min_freq = policy->freq_table[0].frequency; - smc_cpu_caps(cpu); - ret = freq_qos_add_request(&policy->constraints, &cpu->guard, FREQ_QOS_MAX, - policy->freq_table[0].frequency); - if (ret < 0) { - kfree(cpu); - return ERR_PTR(ret); - } - mutex_lock(&smc_provider_lock); - mutex_lock(&smc_lock); - for (i = 0; i < ARRAY_SIZE(smc_cpus); i++) - if (!smc_cpus[i]) { - smc_cpus[i] = cpu; - break; - } - mutex_unlock(&smc_lock); - mutex_unlock(&smc_provider_lock); - if (i == ARRAY_SIZE(smc_cpus)) { - freq_qos_remove_request(&cpu->guard); - kfree(cpu); - return ERR_PTR(-EBUSY); - } - queue_delayed_work(system_unbound_wq, &smc_watch, 0); - return cpu; -} -EXPORT_SYMBOL_GPL(apple_smc_thermal_cpu_add); - -void apple_smc_thermal_cpu_ready(struct apple_smc_thermal_cpu *cpu) -{ - if (!cpu) - return; - guard(mutex)(&smc_provider_lock); - smc_start_zone(); -} -EXPORT_SYMBOL_GPL(apple_smc_thermal_cpu_ready); - -void apple_smc_thermal_cpu_fault(struct apple_smc_thermal_cpu *cpu) -{ - if (!cpu) - return; - /* Called inside target_index: do not recursively take the policy lock. */ - WRITE_ONCE(cpu->failed, true); - mod_delayed_work(system_unbound_wq, &smc_watch, 0); -} -EXPORT_SYMBOL_GPL(apple_smc_thermal_cpu_fault); - -void apple_smc_thermal_cpu_remove(struct apple_smc_thermal_cpu *cpu) -{ - struct thermal_zone_device *zone; - unsigned int i; - - if (!cpu) - return; - guard(mutex)(&smc_provider_lock); - mutex_lock(&smc_lock); - zone = smc_zone; - smc_zone = NULL; - smc_enabled = false; - smc_good = 0; - smc_update_guards(); - mutex_unlock(&smc_lock); - if (zone) - thermal_zone_device_unregister(zone); - mutex_lock(&smc_lock); - for (i = 0; i < ARRAY_SIZE(smc_cpus); i++) - if (smc_cpus[i] == cpu) - smc_cpus[i] = NULL; - freq_qos_remove_request(&cpu->guard); - mutex_unlock(&smc_lock); - kfree(cpu); -} -EXPORT_SYMBOL_GPL(apple_smc_thermal_cpu_remove); - -static void smc_provider_remove(void *unused) -{ - mutex_lock(&smc_provider_lock); - smc_read = NULL; - smc_ctx = NULL; - mutex_lock(&smc_lock); - smc_error = -ENODEV; - smc_good = 0; - smc_update_guards(); - mutex_unlock(&smc_lock); - mutex_unlock(&smc_provider_lock); - cancel_delayed_work_sync(&smc_sample); -} - -int devm_apple_smc_thermal_register(struct device *dev, int (*read)(void *, int *), void *ctx) -{ - int ret; - - if (!of_machine_is_compatible("apple,j700")) - return -ENODEV; - mutex_lock(&smc_provider_lock); - if (smc_read) { - mutex_unlock(&smc_provider_lock); - return -EBUSY; - } - smc_ctx = ctx; - smc_read = read; - smc_start_zone(); - mutex_unlock(&smc_provider_lock); - ret = devm_add_action_or_reset(dev, smc_provider_remove, NULL); - if (!ret) - queue_delayed_work(system_unbound_wq, &smc_sample, 0); - return ret; -} -EXPORT_SYMBOL_GPL(devm_apple_smc_thermal_register); diff --git a/include/linux/soc/apple/smc-thermal.h b/include/linux/soc/apple/smc-thermal.h deleted file mode 100644 index 9cd5788e55cc8..0000000000000 --- a/include/linux/soc/apple/smc-thermal.h +++ /dev/null @@ -1,37 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* - * Apple J700 CPU thermal policy: cpufreq and telemetry provider interface - * - * Copyright (C) 2026 Eryk Wieliczko - */ -#ifndef _APPLE_SMC_THERMAL_H -#define _APPLE_SMC_THERMAL_H - -#include -#include -#include - -struct cpufreq_policy; -struct device; -struct apple_smc_thermal_cpu; - -/* Maximum host transaction/cache age, NOT a firmware sensor-age guarantee. */ -#define APPLE_SMC_THERMAL_MAX_AGE_MS 200 - -#if IS_REACHABLE(CONFIG_APPLE_SMC_CPU_THERMAL) -struct apple_smc_thermal_cpu *apple_smc_thermal_cpu_add(struct cpufreq_policy *policy); -void apple_smc_thermal_cpu_ready(struct apple_smc_thermal_cpu *cpu); -void apple_smc_thermal_cpu_remove(struct apple_smc_thermal_cpu *cpu); -void apple_smc_thermal_cpu_fault(struct apple_smc_thermal_cpu *cpu); -int devm_apple_smc_thermal_register(struct device *dev, int (*read)(void *, int *), void *ctx); -#else -static inline struct apple_smc_thermal_cpu * -apple_smc_thermal_cpu_add(struct cpufreq_policy *policy) { return ERR_PTR(-ENODEV); } -static inline void apple_smc_thermal_cpu_ready(struct apple_smc_thermal_cpu *cpu) {} -static inline void apple_smc_thermal_cpu_remove(struct apple_smc_thermal_cpu *cpu) {} -static inline void apple_smc_thermal_cpu_fault(struct apple_smc_thermal_cpu *cpu) {} -static inline int devm_apple_smc_thermal_register(struct device *dev, - int (*read)(void *, int *), void *ctx) -{ return -ENODEV; } -#endif -#endif From 64eff8a46206613ff354cbeaff49d1354e4c1f52 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 02:08:14 +1000 Subject: [PATCH 0081/1579] arm64: dts: apple: t8140: Drop the J700 CPU thermal policy property The SMC hwmon node no longer selects a driver-internal thermal policy; the J700's CPU die temperatures feed its device-tree thermal zones. Remove apple,cpu-thermal-policy and reword the comments that described the old positional contract. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 8 +++----- arch/arm64/boot/dts/apple/t8140.dtsi | 6 +++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 6a2e0dc88d74d..d4e951a781428 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -469,12 +469,10 @@ }; &smc_hwmon { - /* interim J700 CPU thermal policy over the CPU die channels */ - apple,cpu-thermal-policy; - /* - * Reporting-only channels. They follow the six CPU die temperatures - * in t8140.dtsi, which the CPU thermal policy samples by position. + * Further channels, reported to userspace. They follow the six CPU + * die temperatures in t8140.dtsi, which the thermal zones above + * refer to by index. * Every key below is in the J700 SMC's key table as a read-only * float, reads 10-99 C and moves between samples. The names follow * the SMC key groups and are not otherwise verified. diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 4681a1dd68d42..ee8aa0bba2772 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -434,9 +434,9 @@ #thermal-sensor-cells = <1>; /* - * CPU cluster die temperatures. Keep these - * first: the J700 CPU thermal policy reads - * the first six temperature channels. + * CPU cluster die temperatures. Board thermal + * zones refer to these by channel index, so + * keep them first and in this order. */ temperature-Te05 { apple,key-id = "Te05"; From 50fb6c0c4e683b3b87d86da7835f3330d817fb96 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 02:08:31 +1000 Subject: [PATCH 0082/1579] dt-bindings: hwmon: apple,smc-hwmon: Remove apple,cpu-thermal-policy The property selected a CPU thermal controller inside the SMC hwmon driver, which has been removed. Boards now describe standard thermal zones that refer to the SMC temperature channels through the node's thermal sensor cells. The property described Linux policy rather than hardware, and no device tree uses it any more. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ryan Murray --- .../devicetree/bindings/hwmon/apple,smc-hwmon.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index 291f6e8ca7ef8..076de649627c8 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -46,14 +46,6 @@ properties: The cell is the zero-based index of a temperature channel, counting the temperature-* child nodes in order. - apple,cpu-thermal-policy: - type: boolean - description: - Enable the J700 CPU thermal provider over the first six temperature - channels, which must contain Te05, Te0S, Tp01, Tp05, Tp09 and Tp0D. - Further temperature channels are reporting-only. The provider holds - CPU frequency constraints while these readings are unavailable. - patternProperties: "^current-[A-Za-z0-9]{4}$": $ref: "#/$defs/sensor" From c5653cb2e0b782921468352a2c60173d2650a8bf Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 00:57:11 -0400 Subject: [PATCH 0083/1579] drm/apple: preserve Type-C routes across blanking and detach Retain the last available DCP route and avoid retraining unrelated outputs during the immediate DP-to-USB4 teardown sequence. Keep synthetic DPMS disconnects separate from physical cable removal, while allowing positive hotplug events through after a replug. Let the RTKit crash callback, rather than a power-off clear-swap timeout, establish a fatal firmware crash. --- drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp.c | 70 ++++++++++++++++++-------- drivers/gpu/drm/apple/iomfb_template.c | 17 ++++++- 3 files changed, 68 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 385be68bc9144..5f25fc336578f 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -301,6 +301,8 @@ struct apple_dcp { u32 nr_typec_routes; bool phy_managed_by_typec; bool typec_cable_connected; + /* CRTC is off while the physical Type-C cable may remain connected. */ + bool typec_crtc_off; struct delayed_work typec_reconnect_wq; struct delayed_work typec_fabric_retrain_wq; u32 typec_reconnect_tries; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index ed3603d96363c..63aad59290f76 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -76,6 +76,10 @@ struct apple_dcp_typec_port { struct list_head routes; struct device_node *connector_np; struct apple_dcp_typec_route *owner; + /* Keep a port on its last DCP while that pipeline remains free. */ + struct apple_dcp_typec_route *preferred_route; + /* Ignore the USB4 fallback immediately following this port's DP teardown. */ + unsigned long dp_release_deadline; /* DRM connector for this physical port, driven by whichever DCP owns it */ struct apple_connector *connector; /* last mux state acted on, to collapse the per-candidate notifications */ @@ -138,10 +142,9 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); /* - * Pipelines are ranked by CRTC index so the fabric's choice is a pure function - * of the topology rather than of plug order. A pipeline whose fixed output is - * live is not a candidate at all, so a hybrid is only ever ranked here when it - * is genuinely free. + * For a port without a prior owner, rank pipelines by CRTC index. A pipeline + * whose fixed output is live is not a candidate at all, so a hybrid is only + * ever ranked here when it is genuinely free. */ static unsigned int dcp_typec_route_score(struct apple_dcp_typec_route *route) { @@ -682,6 +685,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (port->owner) { struct apple_dcp *dcp = port->owner->dcp; + port->preferred_route = port->owner; if (port->hpd || dcp->typec_cable_connected || (dcp->typec_connector && dcp->typec_connector->connected)) @@ -691,11 +695,23 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (ret) return ret; port->owner = NULL; + port->dp_release_deadline = jiffies + msecs_to_jiffies(10000); if (dcp->hdmi_hpd && dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd)) dcp_dptx_connect(dcp, 0); } - dcp_typec_retrain_active_routes(); + /* + * A port leaving DP can report SAFE/NONE before falling back to USB4. + * Resetting every other live CRTC for that same cable removal blanks + * unaffected displays. Keep the guard across the Type-C state sequence; + * a later, independent USB4 attach still gets recovery. + */ + if (state->mode == TYPEC_MODE_USB4) { + if (!port->dp_release_deadline || + time_after_eq(jiffies, port->dp_release_deadline)) + dcp_typec_retrain_active_routes(); + port->dp_release_deadline = 0; + } return 0; } @@ -711,15 +727,21 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, } if (!port->owner) { - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; - - if (!dcp_typec_route_available(candidate)) - continue; - score = dcp_typec_route_score(candidate); - if (score < best_score) { - best = candidate; - best_score = score; + if (port->preferred_route && + dcp_typec_route_available(port->preferred_route)) + best = port->preferred_route; + + if (!best) { + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (!dcp_typec_route_available(candidate)) + continue; + score = dcp_typec_route_score(candidate); + if (score < best_score) { + best = candidate; + best_score = score; + } } } @@ -730,6 +752,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (ret) return ret; port->owner = best; + port->dp_release_deadline = 0; } @@ -883,6 +906,8 @@ static void dcp_typec_route_unregister(void *data) typec_mux_unregister(route->typec_mux); guard(mutex)(&dcp_typec_fabric_lock); + if (port->preferred_route == route) + port->preferred_route = NULL; if (port->owner == route) { struct apple_dcp *dcp = route->dcp; @@ -1783,6 +1808,7 @@ void dcp_poweron(struct platform_device *pdev) int ret; if (dcp_is_typec_output(dcp)) { + WRITE_ONCE(dcp->typec_crtc_off, false); /* * A Type-C CRTC disable releases its DPTX session. Re-establish it * synchronously before IOMFB is powered back on. @@ -1825,6 +1851,15 @@ void dcp_poweroff(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + if (dcp_is_typec_output(dcp)) { + /* + * The DPTX disconnect below is part of DPMS, not a cable removal. + * Keep the DRM connector logically present so userspace can wake it. + */ + WRITE_ONCE(dcp->typec_crtc_off, true); + cancel_delayed_work(&dcp->typec_reconnect_wq); + } + if (dcp->avep) av_service_disconnect(dcp); @@ -1842,12 +1877,7 @@ void dcp_poweroff(struct platform_device *pdev) "failed to deassert Type-C DPTX HPD: %d\n", ret); dcp_dptx_disconnect(dcp, 0); - if (READ_ONCE(dcp->typec_cable_connected)) { - dcp->typec_reconnect_tries = 0; - mod_delayed_work(system_freezable_wq, - &dcp->typec_reconnect_wq, - DPTX_RECONNECT_DELAY); - } + /* dcp_poweron() reconnects the link on DPMS wake. */ } } else if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 279534c227ac7..e4cb300695dbd 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -949,7 +949,11 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) swap_id = cookie->swap_id; kref_put(&cookie->refcount, release_swap_cookie); if (ret <= 0) { - dcp->crashed = true; + /* A disconnected external display can power down before it ACKs this + * clear swap. The deadline alone does not mean RTKit crashed; its + * crash callback owns the fatal flag. + */ + dev_warn(dcp->dev, "clear swap timed out during power-off\n"); return; } @@ -1046,6 +1050,17 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) * rejects modes absent from that list. Do not synthesize a mode here. */ + /* + * Deasserting synthetic Type-C HPD for DPMS is not a cable unplug. + * Suppress only the disconnect: a real cable can be removed while the + * CRTC is off and reinserted before it is re-enabled. Its connect event + * must reach DRM or the connector stays disconnected indefinitely. + */ + if (dcp_is_typec_output(dcp) && !(*connected) && + READ_ONCE(dcp->typec_crtc_off) && + READ_ONCE(dcp->typec_cable_connected)) + return; + if (dcp->during_modeset) { /* * Remember it rather than dropping it. Resume re-runs the From a044eab3b9ccce97693e7eb60bddb604799073b2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 24 Sep 2026 23:11:18 -0400 Subject: [PATCH 0084/1579] arm64: enable J414s Thunderbolt DP tunnel and rearm display route Extend the T6020 ATC tunnel clock gate and dcpext0 preference to J414s. Re-establish the DCP route when a new dock tunnel appears after a controller detach without a prior teardown callback. Tested on J414s m2fix42: dock DP tunnel enabled USB-3 at 1920x1280@60, direct USB-2 at 3840x2160@120 and eDP at 3024x1964@120 with active CRTCs and zero failed units. Controller-detach disconnect reporting remains under investigation. --- drivers/gpu/drm/apple/dcp.c | 18 +++++++++--------- drivers/phy/apple/atc.c | 5 +++-- drivers/thunderbolt/apple.c | 24 +++++++++++++++++++++--- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 63aad59290f76..acb80ff238286 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -561,7 +561,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, continue; score = dcp_typec_route_score(candidate); /* - * j416s-specific, not in the reference (t8103 has a single + * T6020-specific, not in the reference (t8103 has a single * dcpext): prefer a pipeline with a fixed output of its own * (dcpext0) for a Type-C tunnel route. dcpext1 (Type-C only, * no fixed output of its own) never completes link training @@ -569,15 +569,15 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, * accepts request_display but never issues another apcall, * on every attempt, even though the driver-issued connect * parameters are byte-identical between the two pipelines. - * Nothing in this driver's source explains the difference, - * so this is a firmware-internal decision on the dcpext1 - * coprocessor instance, not something fixable here. dcpext0, - * forced onto the same physical port and tunnel, reaches a - * full AUX/DPCD link (DPRX_DONE=1) and a working picture, so - * it is preferred unconditionally for a tunnel route on this - * hardware. + * Nothing in this driver's source explains the difference. + * dcpext0, forced onto the same physical port and tunnel, reaches + * a full AUX/DPCD link (DPRX_DONE=1) and a working picture on + * j416s. J414s uses the same T6020 display topology; its tunnel + * result is tested separately. */ - if (!candidate->dcp->fixed_phy) + if ((of_machine_is_compatible("apple,j414s") || + of_machine_is_compatible("apple,j416s")) && + !candidate->dcp->fixed_phy) score += 100; if (score < best_score) { best = candidate; diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index e4331cf72fe48..d8d0646c22e38 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2340,7 +2340,7 @@ static const struct phy_ops apple_atc_dp_phy_ops = { }; /* - * j416s has one ATC PHY core per Type-C port at 0x703000000 (left), + * j414s and j416s have one ATC PHY core per Type-C port at 0x703000000 (left), * 0xb03000000 (left), 0xf03000000 (right) -- the same top-byte pattern * as the ACIO/DPIN0 and crossbar addresses in drivers/thunderbolt/apple.c * and drivers/mux/apple-display-crossbar.c. Originally this only @@ -2370,7 +2370,8 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) int ret; if (!phy || phy->ops != &apple_atc_dp_phy_ops || - !of_machine_is_compatible("apple,j416s")) + (!of_machine_is_compatible("apple,j414s") && + !of_machine_is_compatible("apple,j416s"))) return -EOPNOTSUPP; atcphy = phy_get_drvdata(phy); if (!of_device_is_compatible(atcphy->np, "apple,t6020-atcphy") || diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 36ccfae7fdb50..fbe5930e9cfce 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -187,6 +187,7 @@ struct apple_cio { unsigned int idx; bool alive; /* tunnel up; cleared before it is torn down */ bool handed; /* appledrm has our callback; work only */ + bool rearm; /* a fresh tunnel needs a fresh DCP route */ } dpin[2]; }; @@ -1096,8 +1097,15 @@ static int apple_nhi_dp_tunnel_post_activate(struct tb_nhi *nhi, } else { struct apple_dpin_ctx *c = &anhi->acio->dpin[idx]; - scoped_guard(mutex, &c->lock) + /* + * A new tunnel can arrive without the old tunnel's teardown hook + * running on a controller detach. Force the DCP route through a + * down/up cycle instead of treating the old handoff as this one. + */ + scoped_guard(mutex, &c->lock) { c->alive = true; + c->rearm = true; + } queue_work(anhi->acio->dp_wq, &c->work); } apple_dp_dump_host_adapters(anhi); @@ -2041,12 +2049,15 @@ static void apple_dpin_down(struct apple_dpin_ctx *c) static void apple_dpin_work_fn(struct work_struct *work) { struct apple_dpin_ctx *c = container_of(work, struct apple_dpin_ctx, work); - bool want; + bool want, rearm; int ret; for (;;) { - scoped_guard(mutex, &c->lock) + scoped_guard(mutex, &c->lock) { want = c->alive; + rearm = c->rearm; + c->rearm = false; + } if (!want) { if (c->handed || c->regs) { @@ -2055,6 +2066,13 @@ static void apple_dpin_work_fn(struct work_struct *work) } return; } + if (rearm && c->handed) { + dev_info(c->acio->dev, + "dpin%u: replacing stale display route for new tunnel\n", + c->idx); + apple_dpin_down(c); + continue; + } if (c->handed) return; From aff9d4e64aaf418cce120a1bafceff6ba54e726b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 24 Sep 2026 23:39:53 -0400 Subject: [PATCH 0085/1579] drm/apple: reserve J414s hybrid DCP for Thunderbolt DP tunnel Route an available direct Type-C DP-alt-mode display to dcpext1 first so dcpext0 remains available for a dock DP tunnel. The tunnel still prefers dcpext0, which is the pipeline that completes link training on T6020. m2fix45 cold-booted with both displays attached: direct 3840x2160@120 on dcpext1 and dock 1920x1280@60 on dcpext0 came up without port resets, alongside eDP. SEP identity, haptic FF ioctl, and AVD H.264 decode remained functional. --- drivers/gpu/drm/apple/dcp.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index acb80ff238286..1e8dc88fa62e9 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -738,6 +738,14 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (!dcp_typec_route_available(candidate)) continue; score = dcp_typec_route_score(candidate); + /* + * On J414s, leave dcpext0 free for a dock's DP + * tunnel when a direct DP-alt-mode display can use + * the Type-C-only dcpext1 pipeline. + */ + if (of_machine_is_compatible("apple,j414s") && + candidate->dcp->fixed_phy) + score += 100; if (score < best_score) { best = candidate; best_score = score; From 0cd936dc281e61731dc5b51004964658e23552f6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 26 Sep 2026 18:51:29 -0400 Subject: [PATCH 0086/1579] drm/apple: drive two J414s dock DP tunnels independently Route DPIN1 through the native T602x crossbar fields and source selector, give the two tunnel streams separate DCP connector/route ownership, and keep both ATC pixel clocks alive at independent link rates. Wait for Type-C IOMFB hotplug after a display power-on. The dual-dock path was exercised on J414s/m2fix48 with Dell 3840x2160@120 and JTG 1920x1280@60, per-output DPMS and idle/wake; the user confirmed the displayed image. This commit removes a frame-complete callback that had no register side effect. The cleaned tree builds with the Asahi profile; the cleaned commit itself is not yet deployed. --- drivers/gpu/drm/apple/apple_drv.c | 21 +-- drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp.c | 178 +++++++++++++++++-------- drivers/gpu/drm/apple/dcp.h | 3 +- drivers/gpu/drm/apple/iomfb_template.c | 2 + drivers/mux/apple-display-crossbar.c | 129 +++++++++++++----- drivers/phy/apple/atc.c | 140 ++++++++++++++----- include/linux/soc/apple/dp-tunnel.h | 3 +- 8 files changed, 348 insertions(+), 130 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 4c6fa62321a25..4fcfac0f695d9 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -423,10 +423,9 @@ static int apple_probe_per_dcp(struct device *dev, } /* - * Create one connector per physical Type-C port and attach it to every - * pipeline that can drive it. Keeping the connector tied to the port rather - * than to a pipeline is what gives userspace a stable name to hang its - * per-monitor configuration on. + * Create a connector per physical Type-C port and attach it to every + * pipeline that can drive it. J414s appends a second connector per port + * for a dock's second DP tunnel, preserving primary connector names. */ static int apple_probe_typec_ports(struct drm_device *drm, struct platform_device **dcp, @@ -435,9 +434,15 @@ static int apple_probe_typec_ports(struct drm_device *drm, unsigned int idx, nr_ports = dcp_typec_nr_ports(); int i, ret; - for (idx = 0; idx < nr_ports; idx++) { + /* Keep the existing physical-port connector numbers stable. On J414s, + * append one more connector per port for a second USB4 DP tunnel. + */ + for (idx = 0; idx < nr_ports * + (of_machine_is_compatible("apple,j414s") ? 2 : 1); idx++) { struct apple_connector *connector; struct apple_encoder *enc; + unsigned int port_idx = idx % nr_ports; + bool secondary = idx >= nr_ports; u32 mask = 0; connector = kzalloc_obj(*connector); @@ -466,13 +471,13 @@ static int apple_probe_typec_ports(struct drm_device *drm, INIT_WORK(&connector->hotplug_wq, dcp_hotplug); for (i = 0; i < num_dcp; i++) { - if (dcp_typec_port_has_candidate(idx, dcp[i])) + if (dcp_typec_port_has_candidate(port_idx, dcp[i])) mask |= crtc_mask[i]; } if (!mask) { drm_warn(drm, "Type-C port %u has no display pipeline\n", - idx); + port_idx); return -ENODEV; } @@ -496,7 +501,7 @@ static int apple_probe_typec_ports(struct drm_device *drm, connector->port_encoder = &enc->base; connector->candidate_crtcs = mask; - dcp_typec_port_set_connector(idx, connector); + dcp_typec_port_set_connector(port_idx, secondary, connector); } return 0; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 5f25fc336578f..04af25458e279 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -303,6 +303,8 @@ struct apple_dcp { bool typec_cable_connected; /* CRTC is off while the physical Type-C cable may remain connected. */ bool typec_crtc_off; + /* IOMFB reports its video interface ready after DPTX link training. */ + struct completion typec_iomfb_hpd_ready; struct delayed_work typec_reconnect_wq; struct delayed_work typec_fabric_retrain_wq; u32 typec_reconnect_tries; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 1e8dc88fa62e9..9578e688b0f1c 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -76,12 +76,15 @@ struct apple_dcp_typec_port { struct list_head routes; struct device_node *connector_np; struct apple_dcp_typec_route *owner; + struct apple_dcp_typec_route *secondary_owner; /* Keep a port on its last DCP while that pipeline remains free. */ struct apple_dcp_typec_route *preferred_route; /* Ignore the USB4 fallback immediately following this port's DP teardown. */ unsigned long dp_release_deadline; /* DRM connector for this physical port, driven by whichever DCP owns it */ struct apple_connector *connector; + /* A second logical stream through this port's USB4 dock. */ + struct apple_connector *secondary_connector; /* last mux state acted on, to collapse the per-candidate notifications */ struct typec_altmode *applied_alt; unsigned long applied_mode; @@ -141,6 +144,19 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); +static int dcp_dpxbar_tunnel_select_source(struct mux_control *mux, int state) +{ + typeof(&apple_dpxbar_tunnel_select_source) select = + symbol_get(apple_dpxbar_tunnel_select_source); + int ret; + + if (!select) + return -ENOENT; + ret = select(mux, state); + symbol_put(apple_dpxbar_tunnel_select_source); + return ret; +} + /* * For a port without a prior owner, rank pipelines by CRTC index. A pipeline * whose fixed output is live is not a candidate at all, so a hybrid is only @@ -160,6 +176,9 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar) { struct apple_dcp *dcp = route->dcp; + struct apple_connector *connector = + xbar != route->xbar && route->tunnel_dpin == 1 ? + route->port->secondary_connector : route->port->connector; int ret; /* @@ -182,7 +201,11 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * (see dcp_tunnel_crossbar_up()). Just remember the output here. * Ported from aurora-silicon/linux#8. */ - ret = xbar == route->xbar ? mux_control_select(xbar, route->mux_index) : 0; + if (xbar != route->xbar && route->mux_index) + ret = dcp_dpxbar_tunnel_select_source(xbar, route->mux_index); + else + ret = xbar == route->xbar ? + mux_control_select(xbar, route->mux_index) : 0; if (ret) { if (dcp->xbar) { int restore_ret; @@ -202,10 +225,10 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, dcp->phy = route->phy; dcp->dptx_phy = route->dptx_phy; dcp->connector_type = DRM_MODE_CONNECTOR_USB; - if (route->port->connector) { - route->port->connector->dcp = to_platform_device(dcp->dev); - dcp->typec_connector = route->port->connector; - dcp->connector = route->port->connector; + if (connector) { + connector->dcp = to_platform_device(dcp->dev); + dcp->typec_connector = connector; + dcp->connector = connector; /* * Narrow the port to the pipeline now driving it. The encoder @@ -217,8 +240,8 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * rejected with no way for it to recover. The hotplug that * follows makes it re-read this. */ - if (route->port->connector->port_encoder && dcp->crtc) - route->port->connector->port_encoder->possible_crtcs = + if (connector->port_encoder && dcp->crtc) + connector->port_encoder->possible_crtcs = drm_crtc_mask(&dcp->crtc->base); } dcp->active_typec_route = route; @@ -242,6 +265,9 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) { struct apple_dcp *dcp = route->dcp; + struct apple_connector *connector = dcp->typec_connector; + struct mux_control *active_xbar = route->active_xbar; + bool was_tunnel = route->tunnel; int ret = 0; /* @@ -262,7 +288,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) symbol_get(apple_atc_dp_tunnel_rate); if (stop) { - stop(dcp->phy, 0); + stop(dcp->phy, route->tunnel_dpin, 0); symbol_put(apple_atc_dp_tunnel_rate); } } @@ -274,13 +300,17 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) dcp->tb_dpin_ctx = NULL; dcp->tb_clock_ok = false; } + if (was_tunnel && route->mux_index) { + ret = dcp_dpxbar_tunnel_select_source(active_xbar, -1); + if (ret) + dev_warn(dcp->dev, "DP tunnel source reset failed: %d\n", ret); + } route->selected = false; if (dcp->active_typec_route == route) dcp->active_typec_route = NULL; - if (route->port->connector && - route->port->connector->dcp == to_platform_device(dcp->dev)) { + if (connector && connector->dcp == to_platform_device(dcp->dev)) { /* * Until the port is activated again it has no pipeline behind * it, and nothing can read modes or EDID from it. Report it @@ -291,13 +321,13 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) * it. The new pipeline marks it connected again once the * display has come back up on it. */ - WRITE_ONCE(route->port->connector->connected, false); - route->port->connector->dcp = NULL; + WRITE_ONCE(connector->connected, false); + connector->dcp = NULL; /* Unrouted: the port could go to any of its pipelines again. */ - if (route->port->connector->port_encoder) - route->port->connector->port_encoder->possible_crtcs = - route->port->connector->candidate_crtcs; + if (connector->port_encoder) + connector->port_encoder->possible_crtcs = + connector->candidate_crtcs; } dcp->typec_connector = NULL; dcp->connector = dcp->fixed_connector; @@ -353,11 +383,14 @@ static void dcp_typec_retrain_active_routes(void) struct apple_dcp_typec_port *port; list_for_each_entry(port, &dcp_typec_ports, link) { - if (!port->owner) - continue; - mod_delayed_work(system_freezable_wq, + if (port->owner) + mod_delayed_work(system_freezable_wq, &port->owner->dcp->typec_fabric_retrain_wq, msecs_to_jiffies(200)); + if (port->secondary_owner) + mod_delayed_work(system_freezable_wq, + &port->secondary_owner->dcp->typec_fabric_retrain_wq, + msecs_to_jiffies(200)); } } @@ -467,9 +500,13 @@ int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) return -ENOENT; } route = dcp->active_typec_route; - if (!link_rate && route && route->xbar_up) + if (!route) { + symbol_put(apple_atc_dp_tunnel_rate); + return -ENODEV; + } + if (!link_rate && route->xbar_up) dcp_dpxbar_link(route->active_xbar, false); - ret = fn(phy, link_rate); + ret = fn(phy, route->tunnel_dpin, link_rate); symbol_put(apple_atc_dp_tunnel_rate); dcp->tb_clock_ok = !ret && link_rate; if (ret) @@ -500,6 +537,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, { struct apple_dcp_typec_port *port = NULL, *pos; struct apple_dcp_typec_route *candidate, *best = NULL; + struct apple_dcp_typec_route **slot; unsigned int best_score = UINT_MAX; struct mux_control *ctl; struct apple_dcp *dcp; @@ -518,16 +556,16 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, } if (!port) return -ENODEV; + slot = dpin ? &port->secondary_owner : &port->owner; if (!active) { - if (!port->owner || !port->owner->tunnel || - port->owner->tunnel_dpin != dpin) + if (!*slot || !(*slot)->tunnel || + (*slot)->tunnel_dpin != dpin) return 0; - dcp = port->owner->dcp; + dcp = (*slot)->dcp; if (port->hpd || dcp->typec_cable_connected) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - port->hpd = false; - ret = dcp_typec_route_deactivate(port->owner); + ret = dcp_typec_route_deactivate(*slot); if (ret) { /* The caller's context is going away regardless. */ scoped_guard(mutex, &dcp->tb_lock) { @@ -537,7 +575,8 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, } return ret; } - port->owner = NULL; + *slot = NULL; + port->hpd = !!(port->owner || port->secondary_owner); /* Re-apply the next Type-C mux state in full. */ port->applied_valid = false; if (dcp->hdmi_hpd && dcp->active && @@ -546,13 +585,15 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, return 0; } - if (port->owner) { - if (port->owner->tunnel && port->owner->tunnel_dpin == dpin) + if (*slot) { + if ((*slot)->tunnel && (*slot)->tunnel_dpin == dpin) return 0; - dev_warn(port->owner->dcp->dev, + dev_warn((*slot)->dcp->dev, "port already routed, not taking DP tunnel dpin%u\n", dpin); return -EBUSY; } + if (port->owner && !port->owner->tunnel) + return -EBUSY; list_for_each_entry(candidate, &port->routes, port_link) { unsigned int score; @@ -561,24 +602,19 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, continue; score = dcp_typec_route_score(candidate); /* - * T6020-specific, not in the reference (t8103 has a single - * dcpext): prefer a pipeline with a fixed output of its own - * (dcpext0) for a Type-C tunnel route. dcpext1 (Type-C only, - * no fixed output of its own) never completes link training - * for a tunneled target on this hardware: DCP firmware - * accepts request_display but never issues another apcall, - * on every attempt, even though the driver-issued connect - * parameters are byte-identical between the two pipelines. - * Nothing in this driver's source explains the difference. - * dcpext0, forced onto the same physical port and tunnel, reaches - * a full AUX/DPCD link (DPRX_DONE=1) and a working picture on - * j416s. J414s uses the same T6020 display topology; its tunnel - * result is tested separately. + * DPIN0 prefers the hybrid dcpext0 on T6020. J414s DPIN1 + * prefers dcpext1 so both pipelines can drive independent + * streams through one dock. Retain J416s' existing preference + * until its second pipeline has been validated separately. */ - if ((of_machine_is_compatible("apple,j414s") || - of_machine_is_compatible("apple,j416s")) && - !candidate->dcp->fixed_phy) + if (of_machine_is_compatible("apple,j414s")) { + if ((dpin == 0 && !candidate->dcp->fixed_phy) || + (dpin == 1 && candidate->dcp->fixed_phy)) + score += 100; + } else if (of_machine_is_compatible("apple,j416s") && + !candidate->dcp->fixed_phy) { score += 100; + } if (score < best_score) { best = candidate; best_score = score; @@ -607,7 +643,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, } return ret; } - port->owner = best; + *slot = best; dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); @@ -679,7 +715,8 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, * (apple_dcp_tb_dp_tunnel()), not by a Type-C mux notification. * Ported from aurora-silicon/linux#8. */ - if (port->owner && port->owner->tunnel) + if ((port->owner && port->owner->tunnel) || + port->secondary_owner) return 0; if (port->owner) { @@ -721,7 +758,8 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, * drops what was recorded so the next update is applied. Ported from * aurora-silicon/linux#8. */ - if (port->owner && port->owner->tunnel) { + if ((port->owner && port->owner->tunnel) || + port->secondary_owner) { port->applied_valid = false; return 0; } @@ -877,10 +915,11 @@ bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev return false; } -void dcp_typec_port_set_connector(unsigned int idx, +void dcp_typec_port_set_connector(unsigned int idx, bool secondary, struct apple_connector *connector) { struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *owner; guard(mutex)(&dcp_typec_fabric_lock); @@ -888,7 +927,13 @@ void dcp_typec_port_set_connector(unsigned int idx, if (!port) return; - port->connector = connector; + if (secondary) { + port->secondary_connector = connector; + owner = port->secondary_owner; + } else { + port->connector = connector; + owner = port->owner; + } /* * The port may already have been routed, either before DRM bound or @@ -896,8 +941,12 @@ void dcp_typec_port_set_connector(unsigned int idx, * otherwise its display would be reported on the pipeline's fixed * connector instead of the port it is actually plugged into. */ - if (port->owner) { - struct apple_dcp *dcp = port->owner->dcp; + if (owner) { + struct apple_dcp *dcp = owner->dcp; + + if (dcp->crtc && connector->port_encoder) + connector->port_encoder->possible_crtcs = + drm_crtc_mask(&dcp->crtc->base); connector->dcp = to_platform_device(dcp->dev); dcp->typec_connector = connector; @@ -925,6 +974,15 @@ static void dcp_typec_route_unregister(void *data) dcp_typec_route_deactivate(route); port->owner = NULL; } + if (port->secondary_owner == route) { + struct apple_dcp *dcp = route->dcp; + + if (port->hpd || dcp->typec_cable_connected) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + dcp_typec_route_deactivate(route); + port->secondary_owner = NULL; + } + port->hpd = !!(port->owner || port->secondary_owner); list_del(&route->port_link); if (list_empty(&port->routes)) { list_del(&port->link); @@ -1579,6 +1637,7 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) if (dcp_is_typec_output(dcp)) { WRITE_ONCE(dcp->typec_cable_connected, false); + reinit_completion(&dcp->typec_iomfb_hpd_ready); cancel_delayed_work(&dcp->typec_reconnect_wq); } @@ -1813,9 +1872,13 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) void dcp_poweron(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + bool wait_for_typec_hpd = false; + unsigned long remaining; int ret; if (dcp_is_typec_output(dcp)) { + wait_for_typec_hpd = READ_ONCE(dcp->typec_crtc_off) && + READ_ONCE(dcp->typec_cable_connected); WRITE_ONCE(dcp->typec_crtc_off, false); /* * A Type-C CRTC disable releases its DPTX session. Re-establish it @@ -1829,6 +1892,14 @@ void dcp_poweron(struct platform_device *pdev) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, DPTX_RECONNECT_DELAY); + else if (wait_for_typec_hpd) { + remaining = wait_for_completion_timeout( + &dcp->typec_iomfb_hpd_ready, + msecs_to_jiffies(3000)); + if (!remaining) + dev_warn(dcp->dev, + "Type-C IOMFB hotplug not ready on wake\n"); + } } } else if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); @@ -1849,7 +1920,6 @@ void dcp_poweron(struct platform_device *pdev) WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; } - if (dcp->avep) av_service_connect(dcp); } @@ -1864,6 +1934,7 @@ void dcp_poweroff(struct platform_device *pdev) * The DPTX disconnect below is part of DPMS, not a cable removal. * Keep the DRM connector logically present so userspace can wake it. */ + reinit_completion(&dcp->typec_iomfb_hpd_ready); WRITE_ONCE(dcp->typec_crtc_off, true); cancel_delayed_work(&dcp->typec_reconnect_wq); } @@ -2390,6 +2461,7 @@ static int dcp_platform_probe(struct platform_device *pdev) of_property_read_u32(dev->of_node, "apple,dptx-phy", &dcp->dptx_phy); of_property_read_u32(dev->of_node, "apple,dptx-die", &dcp->dptx_die); dcp->fixed_dptx_phy = dcp->dptx_phy; + init_completion(&dcp->typec_iomfb_hpd_ready); INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, dcp_typec_reconnect_work); INIT_DELAYED_WORK(&dcp->typec_fabric_retrain_wq, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 2f044c8e2abcb..b802574261d8d 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -44,7 +44,8 @@ bool dcp_has_typec_routes(struct platform_device *pdev); unsigned int dcp_typec_nr_ports(void); struct device_node *dcp_typec_port_of_node(unsigned int idx); bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev); -void dcp_typec_port_set_connector(unsigned int idx, struct apple_connector *connector); +void dcp_typec_port_set_connector(unsigned int idx, bool secondary, + struct apple_connector *connector); bool dcp_fw_compat_is_12_x(struct platform_device *pdev); unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index e4cb300695dbd..498ed16f9ae30 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1060,6 +1060,8 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) READ_ONCE(dcp->typec_crtc_off) && READ_ONCE(dcp->typec_cable_connected)) return; + if (dcp_is_typec_output(dcp) && *connected && dcp->nr_modes) + complete_all(&dcp->typec_iomfb_hpd_ready); if (dcp->during_modeset) { /* diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index 90384504e034c..33f359feb9d22 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -35,6 +35,7 @@ #define T602X_FIFO_WR_N_CLK_EN 0x004 #define T602X_FIFO_WR_UNK_EN 0x008 #define T602X_REG_00C 0x00c +#define T602X_REG_010 0x010 #define T602X_REG_014 0x014 #define T602X_REG_018 0x018 #define T602X_REG_01C 0x01c @@ -253,23 +254,27 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) dpxbar_clear32(dpxbar, T602X_FIFO_RD_UNK_EN, prev_dispext_bit); dpxbar_clear32(dpxbar, T602X_FIFO_WR_DPTX_CLK_EN, prev_dispext_bit); - /* DPIN0 uses the same single-bit source gate on up and down. */ - dpxbar_clear32(dpxbar, T602X_REG_00C, - index == MUX_DPIN0 ? prev_dispext_bit : prev_dispext_bit_en); + dpxbar_clear32(dpxbar, + index == MUX_DPIN1 ? T602X_REG_010 : T602X_REG_00C, + prev_dispext_bit); - dpxbar_clear32(dpxbar, T602X_REG_01C, atc_bit); + dpxbar_clear32(dpxbar, + index == MUX_DPIN1 ? FIFO_RD_PCLK1_EN : T602X_REG_01C, + atc_bit); dpxbar_clear32(dpxbar, T602X_FIFO_WR_UNK_EN, prev_dispext_bit); - dpxbar_clear32(dpxbar, T602X_REG_018, prev_dispext_bit_en); + dpxbar_clear32(dpxbar, T602X_REG_018, + prev_dispext_bit_en * + (index == MUX_DPIN1 ? 3 : 1)); - dpxbar_clear32(dpxbar, T602X_FIFO_RD_N_CLK_EN, atc_bit); + dpxbar_clear32(dpxbar, T602X_FIFO_RD_N_CLK_EN, + atc_bit * (index == MUX_DPIN1 ? 3 : 1)); dpxbar_set32(dpxbar, T602X_FIFO_WR_N_CLK_EN, prev_dispext_bit); dpxbar_set32(dpxbar, T602X_REG_014, 0x4); /* Native T602x DPIN0 teardown restores the read reset at +0x24. */ - dpxbar_set32(dpxbar, index == MUX_DPIN0 ? - T602X_FIFO_RD_PCLK2_EN : FIFO_RD_PCLK1_EN, atc_bit); + dpxbar_set32(dpxbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); dpxbar_clear32(dpxbar, T602X_REG_034, atc_bit); dpxbar_clear32(dpxbar, CROSSBAR_ATC_EN, atc_bit); @@ -286,16 +291,37 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) dpxbar_clear32(dpxbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); dpxbar_clear32(dpxbar, T602X_REG_014, 0x4); + /* Match the DPIN1 sequence used by link_up(): clear +0x24 + * before setting the separate +0x20 field below. + */ dpxbar_clear32(dpxbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); + if (index == MUX_DPIN1) + dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, atc_bit); dpxbar_set32(dpxbar, T602X_FIFO_WR_UNK_EN, dispext_bit); - dpxbar_set32(dpxbar, T602X_REG_018, dispext_bit_en); - - dpxbar_set32(dpxbar, T602X_FIFO_RD_N_CLK_EN, atc_bit); + /* Native T602x encodes DPIN1's role as 3 in both two-bit + * source and sink fields, versus 1 for DPIN0. + */ + if (index == MUX_DPIN1) { + dpxbar_mask32(dpxbar, T602X_REG_018, + dispext_bit_en * 3, + dispext_bit_en * 3); + dpxbar_mask32(dpxbar, T602X_FIFO_RD_N_CLK_EN, + atc_bit * 3, atc_bit * 3); + } else { + dpxbar_set32(dpxbar, T602X_REG_018, + dispext_bit_en); + dpxbar_set32(dpxbar, T602X_FIFO_RD_N_CLK_EN, + atc_bit); + } dpxbar_set32(dpxbar, T602X_FIFO_WR_DPTX_CLK_EN, dispext_bit); - dpxbar_set32(dpxbar, T602X_REG_00C, dispext_bit); + dpxbar_set32(dpxbar, + index == MUX_DPIN1 ? T602X_REG_010 : T602X_REG_00C, + dispext_bit); - dpxbar_set32(dpxbar, T602X_REG_01C, atc_bit); + dpxbar_set32(dpxbar, + index == MUX_DPIN1 ? FIFO_RD_PCLK1_EN : T602X_REG_01C, + atc_bit); dpxbar_set32(dpxbar, T602X_REG_034, atc_bit); dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); /* @@ -466,6 +492,42 @@ static const struct mux_control_ops apple_dpxbar_t602x_ops = { .set = apple_dpxbar_set_t602x, }; +/* + * Select the DCP source for a USB4 DP IN before DCP probes AUX. Do not + * enable any FIFO clock or ATC output here: full activation still waits for + * DCP's SetLinkRate and the tunnel pixel clock. dispext0 works with the + * reset selector value (0); dispext1 needs this explicit preselection. + */ +int apple_dpxbar_tunnel_select_source(struct mux_control *mux, int state) +{ + struct apple_dpxbar *xbar; + unsigned long flags; + unsigned int index; + int ret = 0; + + if (!mux || mux->chip->ops != &apple_dpxbar_t602x_ops) + return -EOPNOTSUPP; + index = mux_control_get_index(mux); + if (index != MUX_DPIN0 && index != MUX_DPIN1) + return -EINVAL; + if (state < -1 || state >= 9) + return -EINVAL; + + xbar = mux_chip_priv(mux->chip); + spin_lock_irqsave(&xbar->lock, flags); + if (xbar->selected_dispext[index] >= 0 && + xbar->selected_dispext[index] != state) { + ret = -EBUSY; + } else { + dpxbar_mask32(xbar, T602X_REG_030, t602x_mux_mask(index), + state < 0 ? 0 : t602x_mux_set(index, state)); + } + spin_unlock_irqrestore(&xbar->lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_dpxbar_tunnel_select_source); + /* * Bring the clock/FIFO gates up on a mux whose dispext source is already * selected, without touching the mux selection itself -- used by @@ -480,7 +542,7 @@ int apple_dpxbar_link_up(struct mux_control *mux) struct apple_dpxbar *xbar; unsigned int index; unsigned long flags; - u32 dispext_bit, atc_bit; + u32 dispext_bit, dispext_bit_en, atc_bit; int state, ret = 0; if (!mux || mux->chip->ops != &apple_dpxbar_t602x_ops) @@ -496,31 +558,32 @@ int apple_dpxbar_link_up(struct mux_control *mux) return -ENODEV; } dispext_bit = 1 << state; + dispext_bit_en = 1 << (2 * state); dpxbar_clear32(xbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); dpxbar_clear32(xbar, T602X_REG_014, dispext_bit); - dpxbar_clear32(xbar, index == MUX_DPIN0 ? T602X_FIFO_RD_PCLK2_EN : - FIFO_RD_PCLK1_EN, atc_bit); + dpxbar_clear32(xbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); udelay(1); dpxbar_set32(xbar, T602X_FIFO_WR_UNK_EN, dispext_bit); - dpxbar_mask32(xbar, T602X_REG_018, GENMASK(5, 4), BIT(4)); - /* - * The only hardware-verified value for this 2-bit field is dpin0's - * BIT(0), which is both ATC_DPIN0 and MUX_DPIN0's own enum value (1) - * coincidentally overlapping at bit 0 -- so this field's encoding - * for a second index was never independently confirmed on real - * hardware. FIELD_PREP(..., index), matching - * MUX_DPPHY=0/MUX_DPIN0=1/MUX_DPIN1=2, is an inferred generalization - * of that single known-good dpin0 data point, not a value verified - * against dpin1 hardware. If dpin1 misbehaves, this field is a - * prime suspect. + if (index == MUX_DPIN1) { + dpxbar_mask32(xbar, T602X_REG_018, + dispext_bit_en * 3, dispext_bit_en * 3); + dpxbar_mask32(xbar, T602X_FIFO_RD_N_CLK_EN, + atc_bit * 3, atc_bit * 3); + } else { + dpxbar_set32(xbar, T602X_REG_018, dispext_bit_en); + } + /* set_t602x() already enabled this output's N clock. In particular, + * do not rewrite the low bits here: DPIN1 would clear DPIN0's clock. */ - dpxbar_mask32(xbar, T602X_FIFO_RD_N_CLK_EN, GENMASK(1, 0), - FIELD_PREP(GENMASK(1, 0), index)); dpxbar_set32(xbar, T602X_FIFO_WR_DPTX_CLK_EN, dispext_bit); - dpxbar_set32(xbar, T602X_REG_00C, dispext_bit); - dpxbar_set32(xbar, T602X_REG_01C, atc_bit); + dpxbar_set32(xbar, + index == MUX_DPIN1 ? T602X_REG_010 : T602X_REG_00C, + dispext_bit); + dpxbar_set32(xbar, + index == MUX_DPIN1 ? FIFO_RD_PCLK1_EN : T602X_REG_01C, + atc_bit); dpxbar_set32(xbar, T602X_REG_034, atc_bit); dpxbar_set32(xbar, CROSSBAR_ATC_EN, atc_bit); dpxbar_set32(xbar, CROSSBAR_DISPEXT_EN, dispext_bit); @@ -562,11 +625,9 @@ int apple_dpxbar_link_down(struct mux_control *mux) return -ENODEV; } dispext_bit = 1 << state; - dpxbar_set32(xbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); dpxbar_set32(xbar, T602X_REG_014, dispext_bit); - dpxbar_set32(xbar, index == MUX_DPIN0 ? T602X_FIFO_RD_PCLK2_EN : - FIFO_RD_PCLK1_EN, atc_bit); + dpxbar_set32(xbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); spin_unlock_irqrestore(&xbar->lock, flags); dev_info(xbar->dev, "%s: crossbar link down (dispext=%d atc=0x%x)\n", diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index d8d0646c22e38..9e2846a71768f 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -650,7 +651,8 @@ struct apple_atcphy { enum atcphy_mode mode; int dp_link_rate; bool tunnel_attempted, tunnel_saved; - u8 tunnel_rate; + u8 tunnel_users; + u8 tunnel_dpin_rate[2]; u32 tunnel_saved_regs[12]; bool swap_lanes; bool pipehandler_up; @@ -1910,7 +1912,8 @@ struct atc_tunnel_saved_reg { static const struct atc_tunnel_saved_reg atc_tunnel_regs[] = { { ACIOPHY_CFG0, 0x0003ffff }, { ACIOPHY_SLEEP_CTRL, 0x00000fff }, - { ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, 0x0000207c }, + { ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + 0x0000207c | DPTX_PCLK2_SELECT | DPTX_PCLK2_ENABLE }, { AUSPLL_FREQ_CFG, 0x00000003 }, { AUSPLL_FREQ_DESC_A, 0xffffffff }, { AUSPLL_FREQ_DESC_B, 0x0fffffff }, @@ -1942,48 +1945,47 @@ static void atc_tunnel_restore(struct apple_atcphy *atcphy) int i; lockdep_assert_held(&atcphy->lock); + atcphy->tunnel_users = 0; + atcphy->tunnel_dpin_rate[0] = 0; + atcphy->tunnel_dpin_rate[1] = 0; if (!atcphy->tunnel_saved) return; /* Stop our output before restoring the unused PLL descriptor. */ - core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE | DPTX_PCLK2_ENABLE); core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); for (i = ARRAY_SIZE(atc_tunnel_regs) - 1; i >= 0; i--) core_mask32(atcphy, atc_tunnel_regs[i].reg, atc_tunnel_regs[i].mask, atcphy->tunnel_saved_regs[i] & atc_tunnel_regs[i].mask); atcphy->tunnel_saved = false; atcphy->tunnel_attempted = false; - atcphy->tunnel_rate = 0; } -static int atc_tunnel_start(struct apple_atcphy *atcphy, u8 rate) +static int atc_tunnel_rate_selector(u8 rate) { - u32 selector, value, gates, outputs, command, status; - unsigned int i; - int ret; - - lockdep_assert_held(&atcphy->lock); switch (rate) { case 0x06: - selector = 4; - break; + return 4; case 0x0a: - selector = 3; - break; + return 3; case 0x14: - selector = 1; - break; + return 1; case 0x1e: - selector = 0; - break; + return 0; default: return -EINVAL; } - if (atcphy->tunnel_saved) { - if (atcphy->tunnel_rate != rate) - dev_info(atcphy->dev, "USB4 tunnel clock busy: active rate=0x%x requested=0x%x\n", - atcphy->tunnel_rate, rate); - return atcphy->tunnel_rate == rate ? 0 : -EBUSY; - } +} + +static int atc_tunnel_start(struct apple_atcphy *atcphy) +{ + u32 value, gates, outputs, command, status; + unsigned int i; + int ret; + + lockdep_assert_held(&atcphy->lock); + if (atcphy->tunnel_saved) + return 0; if (atcphy->tunnel_attempted) return -EALREADY; /* Read every guard before deciding, so a refusal records the whole state. */ @@ -2052,9 +2054,6 @@ static int atc_tunnel_start(struct apple_atcphy *atcphy, u8 rate) udelay(2); core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(2)); core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(3)); - core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); - core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, - FIELD_PREP(DPTX_PCLK1_SELECT, selector)); /* Native fixed descriptor at cache VA 0xfffffe0007582418. */ core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); @@ -2080,7 +2079,6 @@ static int atc_tunnel_start(struct apple_atcphy *atcphy, u8 rate) ret = atc_tunnel_command(atcphy, 0x2000); if (ret) goto restore; - atcphy->tunnel_rate = rate; return 0; restore: atc_tunnel_restore(atcphy); @@ -2363,11 +2361,12 @@ static bool apple_atc_is_typec_core(u64 base) * AUSPLL descriptor/PCLK1 selectors). Also accepts TBT mode, not just * USB4: a genuine dock may negotiate either. */ -int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); -int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) +int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) { struct apple_atcphy *atcphy; - int ret; + u32 value; + u8 user; + int ret, selector; if (!phy || phy->ops != &apple_atc_dp_phy_ops || (!of_machine_is_compatible("apple,j414s") && @@ -2378,15 +2377,90 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) !apple_atc_is_typec_core(atcphy->res.core->start) || resource_size(atcphy->res.core) < 0x7048) return -EINVAL; + if (dpin > 1) + return -EINVAL; + user = BIT(dpin); guard(mutex)(&atcphy->lock); if (!rate) { - atc_tunnel_restore(atcphy); + if (atcphy->tunnel_users & user) { + if (dpin == 0) { + /* DPIN1 still needs the PCLK1 gate. On J414s, + * clearing it stopped DPIN1's video packets even + * though PCLK2 remained enabled and locked. + */ + if (!(atcphy->tunnel_users & BIT(1))) + core_clear32(atcphy, + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + } else { + core_clear32(atcphy, + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK2_ENABLE); + if (!(atcphy->tunnel_users & BIT(0))) + core_clear32(atcphy, + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + } + } + atcphy->tunnel_users &= ~user; + atcphy->tunnel_dpin_rate[dpin] = 0; + if (!atcphy->tunnel_users && atcphy->tunnel_saved) + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); return 0; } if (atcphy->mode != APPLE_ATCPHY_MODE_USB4 && atcphy->mode != APPLE_ATCPHY_MODE_TBT) return -EBUSY; - ret = atc_tunnel_start(atcphy, rate); + selector = atc_tunnel_rate_selector(rate); + if (selector < 0) + return selector; + if (atcphy->tunnel_dpin_rate[dpin] == rate) + return 0; + if (!atcphy->tunnel_users) { + if (!atcphy->tunnel_saved) { + ret = atc_tunnel_start(atcphy); + } else { + /* The PLL retains lock after its outputs are gated. Keep its + * owned descriptor until the USB4 PHY changes mode instead of + * reprogramming a still-locked PLL on every DPMS wake. + */ + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, + value, value & ACIOPHY_AUSPLL_LOCK, + 10, 10000); + if (ret) + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + } + if (ret) + goto out; + } + if (dpin == 0) { + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_SELECT, + FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + } else { + /* PCLK2 alone does not sustain a DPIN1 stream on J414s. + * Keep PCLK1 gated on as a shared prerequisite. If DPIN0 is + * inactive, select DPIN1's rate for that clock as well. + */ + if (!(atcphy->tunnel_users & BIT(0))) + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_SELECT, + FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK2_SELECT, + FIELD_PREP(DPTX_PCLK2_SELECT, selector)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK2_ENABLE); + } + atcphy->tunnel_dpin_rate[dpin] = rate; + atcphy->tunnel_users |= user; + ret = 0; +out: dev_info(atcphy->dev, "DP tunnel clock: rate=0x%x result=%d\n", rate, ret); return ret; } diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index 191f8f95dc5aa..ee9de3df78de8 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -33,7 +33,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, * ATC PHY: start the pixel clock for a DP tunnel at DP link rate code @rate, * or stop it (@rate == 0). The PHY must be in Thunderbolt/USB4 mode. */ -int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); +int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate); /* * Display crossbar: take the connection of an already selected output down or @@ -44,5 +44,6 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); */ int apple_dpxbar_link_down(struct mux_control *mux); int apple_dpxbar_link_up(struct mux_control *mux); +int apple_dpxbar_tunnel_select_source(struct mux_control *mux, int state); #endif From a9aa60593bbcaab38d44640583458a60470d6850 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 26 Sep 2026 18:59:19 -0400 Subject: [PATCH 0087/1579] thunderbolt: account live USB3 use on Apple hosts without BW allocation Apple host USB3 adapters do not acknowledge bandwidth-allocation writes, so their software-only initial allocation should not be treated as consumed USB3 bandwidth when admitting DP tunnels. Read the downstream router USB3 UP adapter consumed-bandwidth counters instead, retaining 900 Mb/s minimum headroom for isochronous traffic. If those counters cannot be read, keep the existing conservative allocation estimate. The change does not cap USB3 bulk traffic or write the unsupported host registers. The m2fix48 dock test reached Dell 4K120 and JTG 60 with USB devices enumerated; a standalone throughput or isochronous stress test has not been performed. --- drivers/thunderbolt/tb.h | 2 ++ drivers/thunderbolt/tunnel.c | 31 +++++++++++++++++++++++++++++-- drivers/thunderbolt/usb4.c | 11 +++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h index 222268b2fec5c..2fcff9dccc599 100644 --- a/drivers/thunderbolt/tb.h +++ b/drivers/thunderbolt/tb.h @@ -1460,6 +1460,8 @@ int usb4_port_retimer_nvm_read(struct tb_port *port, u8 index, unsigned int address, void *buf, size_t size); int usb4_usb3_port_max_link_rate(struct tb_port *port); +int usb4_usb3_port_consumed_bandwidth(struct tb_port *port, int *upstream_bw, + int *downstream_bw); int usb4_usb3_port_allocated_bandwidth(struct tb_port *port, int *upstream_bw, int *downstream_bw); int usb4_usb3_port_allocate_bandwidth(struct tb_port *port, int *upstream_bw, diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index c0d3d8aa9f8d6..277f1a063dacf 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -2358,14 +2358,41 @@ static int tb_usb3_consumed_bandwidth(struct tb_tunnel *tunnel, { struct tb_port *port = tb_upstream_port(tunnel->dst_port->sw); int pcie_weight = tb_acpi_may_tunnel_pcie() ? TB_PCI_WEIGHT : 0; + int allocated_up = tunnel->allocated_up; + int allocated_down = tunnel->allocated_down; + int ret; + + if (tunnel->src_port->sw->no_usb3_bw_alloc) { + /* + * Apple's host USB3 adapter cannot acknowledge bandwidth + * allocation requests. Its software-only initial allocation is a + * maximum, not bandwidth actually occupied by USB3. The device + * router's USB3 UP adapter still reports consumed bandwidth; use + * that live value for DP admission without limiting USB3 bulk + * traffic or writing the unsupported host registers. + */ + ret = usb4_usb3_port_consumed_bandwidth(tunnel->dst_port, + &allocated_up, &allocated_down); + if (ret) { + /* Preserve the conservative allocation-based estimate when + * a downstream adapter lacks readable consumed counters. + */ + allocated_up = tunnel->allocated_up; + allocated_down = tunnel->allocated_down; + } else { + /* Leave headroom for new isochronous USB3 transfers. */ + allocated_up = max(allocated_up, 900); + allocated_down = max(allocated_down, 900); + } + } /* * PCIe tunneling, if enabled, affects the USB3 bandwidth so * take that into account here. */ - *consumed_up = tunnel->allocated_up * + *consumed_up = allocated_up * (TB_USB3_WEIGHT + pcie_weight) / TB_USB3_WEIGHT; - *consumed_down = tunnel->allocated_down * + *consumed_down = allocated_down * (TB_USB3_WEIGHT + pcie_weight) / TB_USB3_WEIGHT; if (tb_port_get_link_generation(port) >= 4) { diff --git a/drivers/thunderbolt/usb4.c b/drivers/thunderbolt/usb4.c index 9e810b2ae0b5f..c358c8e77f7ec 100644 --- a/drivers/thunderbolt/usb4.c +++ b/drivers/thunderbolt/usb4.c @@ -2360,6 +2360,17 @@ static int usb4_usb3_port_read_consumed_bandwidth(struct tb_port *port, return 0; } +int usb4_usb3_port_consumed_bandwidth(struct tb_port *port, int *upstream_bw, + int *downstream_bw) +{ + if (!tb_port_is_usb3_up(port)) + return -EINVAL; + + /* Device-router USB3 UP adapters expose these counters without CMR. */ + return usb4_usb3_port_read_consumed_bandwidth(port, upstream_bw, + downstream_bw); +} + static int usb4_usb3_port_write_allocated_bandwidth(struct tb_port *port, int upstream_bw, int downstream_bw) From 3075f5503a32c44e29e7a1e6b9244038e8e28569 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 00:57:55 -0400 Subject: [PATCH 0088/1579] arm64: enable Apple USB4 modules and J414/J416 firmware aliases Preserve the underscore-form USB4 controller aliases used by the tested m1n1 handoff, and build the USB4 connection manager and Apple NHI with the Asahi profile. Keep unrelated SEP and host configuration out of this stack. --- arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi | 12 ++++++++++++ arch/arm64/configs/asahi.config | 2 ++ 2 files changed, 14 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index ed22b3854bd9f..0c267d41c82f6 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -26,6 +26,18 @@ disp0_piodma = &disp0_piodma; keyboard = &keyboard; sio = &sio; + usb4_0_rc = &usb4_0_acio; + usb4_0_pcie_adapter = &usb4_0_acio; + usb4_0_nhi = &usb4_0_nhi; + usb4_0_pcie = &apciec0; + usb4_1_rc = &usb4_1_acio; + usb4_1_pcie_adapter = &usb4_1_acio; + usb4_1_nhi = &usb4_1_nhi; + usb4_1_pcie = &apciec1; + usb4_2_rc = &usb4_2_acio; + usb4_2_pcie_adapter = &usb4_2_acio; + usb4_2_nhi = &usb4_2_nhi; + usb4_2_pcie = &apciec2; }; }; diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 4aa7f499997e7..31331f71e4e39 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -46,6 +46,8 @@ CONFIG_HID_DOCKCHANNEL=m CONFIG_SPI_HID_APPLE_OF=m CONFIG_SPI_HID_APPLE_CORE=m CONFIG_USB_DWC3_APPLE=m +CONFIG_USB4=m +CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_RTC_DRV_MACSMC=m CONFIG_APPLE_ADMAC=m From 84a93e31bd6ea7dc37603e2f7f2efaee621062b2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 27 Sep 2026 21:27:50 +1000 Subject: [PATCH 0089/1579] soc: apple: sep: Describe observed behaviour, not other implementations Two notes in this series described another operating system's implementation instead of what was observed on the hardware: the keybag encoding comment named where the other implementation places the bag type, and the xART safety notes cited an inspection of its kernel code and asked for records to be checked against it. Describe the enclave's behaviour directly, drop the inspection paragraph, and state the remaining checks as hardware tests. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/profile.rs | 7 ++++--- tools/aurora-sep/XART-SAFETY.md | 12 +++--------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 2c3c8c1e70db5..ee2f43096740d 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -90,9 +90,10 @@ pub(crate) struct SensorProfile { /// Identity keybag `CREATE_KEYBAG` field encoding. The strict T8103 enclave /// reads the request's first word as the bag type and rejects the value the -/// lenient T6020 enclave accepts there; macOS carries the type in the third -/// word instead. Kept per-SoC so the strict path is correct without disturbing -/// the proven T6020 encoding (hardware-verified for enrol, match, and reboot). +/// lenient T6020 enclave accepts there; the strict enclave takes the type in +/// the third word instead. Kept per-SoC so the strict path is correct without +/// disturbing the proven T6020 encoding (hardware-verified for enrol, match, +/// and reboot). pub(crate) struct KeybagCreate { /// First word: request variant, echoed back in the reply. pub(crate) variant: u32, diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md index 52a976c63e5c4..be7d5229748f7 100644 --- a/tools/aurora-sep/XART-SAFETY.md +++ b/tools/aurora-sep/XART-SAFETY.md @@ -20,21 +20,15 @@ state. Do not enable it on other installations without equivalent checks. locator failure: the highest-scoring window begins 90 APFS blocks after the true file start and extends beyond `.gl`. This proves ambiguity in that locator, **not** the cause of the previously damaged installation. -- Static inspection of the macOS 27.2 AppleSEPManager kernel collection found - `gl_rec_write` allocating a fresh 0x9000-byte slot, writing it, then clearing - 0x1000 bytes at the former slot. The low-level write path also calls a disk - cache-synchronization ioctl. `gl_fixup_rev_init` validates records and - handles duplicates during open. These observations do not prove that our - APFS ownership and failure ordering match macOS. ## Before authorizing writes -1. Confirm the exact macOS vnode/raw-device and extent-locking contract, +1. Confirm the raw-device access and extent-locking behaviour on hardware, including whether any APFS copy-on-write, snapshot, encryption, or remap state can invalidate the physical mapping during Linux's lifetime. 2. Verify record ordering, duplicate selection, failed-write recovery, - barriers/cache synchronization, and revision behavior against both macOS - code and controlled non-production tests. + barriers/cache synchronization, and revision behavior in controlled + non-production tests. 3. Compare a pre/post macOS-and-Linux power-cycle trace, with APFS checksums, `.gl` mapping and records checked before and after. Keep a recoverable image of the iBoot partition before the first write experiment. From d866080d58a6e8a76ac09de04741ba80a5241cc7 Mon Sep 17 00:00:00 2001 From: Justin Pfister Date: Sat, 26 Sep 2026 19:23:58 -0400 Subject: [PATCH 0090/1579] soc: apple: sep: M2 Pro bringup foundational fixes (alias, xart extent, host uuid) --- arch/arm64/boot/dts/apple/t602x-common.dtsi | 1 + drivers/soc/apple/sep.rs | 35 ++++++++++++++++++++- drivers/soc/apple/xart_store.rs | 2 +- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t602x-common.dtsi b/arch/arm64/boot/dts/apple/t602x-common.dtsi index 2fa6b93a14165..b900a651f5273 100644 --- a/arch/arm64/boot/dts/apple/t602x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-common.dtsi @@ -13,6 +13,7 @@ aliases { gpu = &gpu; + sep = &sep; #ifdef APPLE_USE_PMP pmp = &pmp; #endif diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 61468bb8acd22..c066e579ad33f 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1236,9 +1236,42 @@ impl SepData { if let Some(uuid) = dt::preboot_uuid() { self.xarm.lock().os_uuid = Some(uuid); dev_info!(self.dev, "xART: using /chosen/apfs-preboot-uuid\n"); + return; + } + + // HostPersisted fallback for platforms without /chosen/apfs-preboot-uuid + const OS_UUID_PATH: &CStr = c"/var/lib/aurora-sep-os-uuid.bin"; + let mut uuid = [0u8; 16]; + let mut loaded = false; + if let Ok(file) = shim::StoreFile::open_readonly(OS_UUID_PATH) { + if file.read_exact(0, &mut uuid).is_ok() && !uuid.iter().all(|&b| b == 0) { + loaded = true; + } + } + + if !loaded { + if shim::random_bytes(&mut uuid).is_ok() { + uuid[6] = (uuid[6] & 0x0f) | 0x40; + uuid[8] = (uuid[8] & 0x3f) | 0x80; + if let Ok(file) = shim::StoreFile::open_trunc(OS_UUID_PATH) { + let _ = file.write_all(0, &uuid); + let _ = file.sync(); + loaded = true; + } + } + } + + if loaded { + self.xarm.lock().os_uuid = Some(uuid); + dev_info!( + self.dev, + "xART: using host-persisted OS UUID {:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}\n", + uuid[0], uuid[1], uuid[2], uuid[3], uuid[4], uuid[5], uuid[6], uuid[7], + uuid[8], uuid[9], uuid[10], uuid[11], uuid[12], uuid[13], uuid[14], uuid[15] + ); } else { self.xarm.lock().os_uuid = None; - dev_warn!(self.dev, "xART: /chosen/apfs-preboot-uuid is unavailable\n"); + dev_warn!(self.dev, "xART: /chosen/apfs-preboot-uuid is unavailable and host UUID could not be provisioned\n"); } } diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index ac9e0bf6c51e2..9bf9de0dae38f 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -138,7 +138,7 @@ impl Store { /// be skipped. pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { if start_sector != 0 { - return Self::open_based(OWNER_PATH, writes_enabled, start_sector << 9); + return Self::open_based_ext(OWNER_PATH, true, writes_enabled, start_sector << 9); } Self::open_located(writes_enabled) } From e5fd7bc6ce2592e6b5f7f543c6b0c389d32925d8 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:34:33 -0700 Subject: [PATCH 0091/1579] soc: apple: sep: initialise the T8103 key store as init_sep_endpoint does The T8103 stub boots macOS 13.5 (22G74)'s sepOS, whose key-store protocol is the one the 13.5 AppleSEPKeyStore speaks, not the newer one the T6020 enclave answers. Name that per profile (profile::KeyStore: Sepos13 for T8103, Variant5 for T6020) so the following changes can select the 13.5 request shapes without touching the T6020 encoding. On the 13.5 key store, AppleKeyStore::init_sep_endpoint (0xfffffe0009946cdc) runs before any other request: - 0x4d GET_CAPABILITIES; the IPC header version becomes min(word, 2) (0x...6dc0-6dd0). Version 2 is laid out as version 1 but __payload_hash (0xfffffe000996b970) digests the header through the trailer (0x...ba68-ba80), so image::Version gains V2. - set_env(false) (0xfffffe000994a168): op 0x2a, struct version 0, u64 1, and a 0x40c-byte environment {1, cpx-encryption-mode, 4, 0x400 zero bytes} (0x...a1a0-a1e0, __ipc_set_env 0x...9ddc, _code_ipc_set_env 0x...4b64). The reply is the struct-version word alone. Do the same at bring-up on T8103, before provisioning and before /dev/sep-bio is published. The mode comes from the profile: the j293 and j313 ADTs carry /defaults cpx-encryption-mode = 2, and the Linux device tree does not mirror /defaults. Where Apple would go on after a failed 0x4d with header version 1, this keeps the key store closed for the boot. macOS sends 0x4d once, at endpoint init, so the lock-state health check no longer re-sends it on T8103; it requires the init to have completed. T6020 sends nothing new: sks_init_endpoint returns at once, the header version stays 1 and the health check is unchanged. Hardware-verified on a MacBook Air (j313) with the 13.5 SEP: 0x4d answered status 0, struct version 0, capability word 2, empty blob (header version 2), and set_env with cpx-encryption-mode 2 was accepted. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AJd8eEwJ4TGRSHnbKcWBmw --- drivers/soc/apple/image.rs | 6 ++ drivers/soc/apple/profile.rs | 21 ++++++ drivers/soc/apple/sbio.rs | 7 ++ drivers/soc/apple/sep.rs | 8 +++ drivers/soc/apple/sks.rs | 131 ++++++++++++++++++++++++++++++++--- 5 files changed, 165 insertions(+), 8 deletions(-) diff --git a/drivers/soc/apple/image.rs b/drivers/soc/apple/image.rs index 8d4889268179d..290a9ea43dd3e 100644 --- a/drivers/soc/apple/image.rs +++ b/drivers/soc/apple/image.rs @@ -59,18 +59,24 @@ static_assert!(HEADER_WIRE == 4 + HEADER_SIZE as usize); #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum Version { V1, + /// Negotiated from the `0x4d` capability word. Laid out as version 1, but + /// `__payload_hash` (0xfffffe000996b970) digests the header through the + /// trailer (0x…ba68-ba80) where version 1 stops before it (0x…ba84-ba90). + V2, } impl Version { pub(crate) const fn wire(self) -> u32 { match self { Version::V1 => 1, + Version::V2 => 2, } } const fn digest_end(self) -> usize { match self { Version::V1 => OFF_TRAILER, + Version::V2 => HEADER_SIZE as usize, } } } diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 61faf6b88736f..79a60f9de03f0 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -101,6 +101,20 @@ pub(crate) struct KeybagCreate { pub(crate) arg: i32, } +/// Which key-store protocol the enclave speaks. It follows the sepOS the +/// firmware hands the SEP, not the SoC alone: the T8103 stub boots macOS 13.5 +/// (22G74)'s sepOS, whose request shapes are those of the 13.5 +/// `AppleSEPKeyStore`; the T6020 sepOS is newer. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum KeyStore { + /// macOS 13.5's key store. Before any other request the endpoint is + /// initialised as `AppleKeyStore::init_sep_endpoint` does it: `0x4d`, then + /// `set_env` carrying the ADT's `/defaults` `cpx-encryption-mode`. + Sepos13 { cpx_encryption_mode: u32 }, + /// The hardware-verified T6020 encoding (variant-5 create). + Variant5, +} + pub(crate) struct PlatformProfile { pub(crate) name: &'static str, /// Capacity of the boot shared-memory window. The layout is still checked @@ -119,6 +133,8 @@ pub(crate) struct PlatformProfile { pub(crate) firmware_region: &'static CStr, /// Identity keybag CREATE_KEYBAG field encoding (per-SoC; see [`KeybagCreate`]). pub(crate) keybag_create: KeybagCreate, + /// Key-store protocol generation (see [`KeyStore`]). + pub(crate) key_store: KeyStore, } const T8103: PlatformProfile = PlatformProfile { @@ -145,6 +161,10 @@ const T8103: PlatformProfile = PlatformProfile { bag_type: 0x20000, arg: 0, }, + // The j293 and j313 ADTs both carry `/defaults` `cpx-encryption-mode = 2`. + key_store: KeyStore::Sepos13 { + cpx_encryption_mode: 2, + }, }; const T6020: PlatformProfile = PlatformProfile { @@ -171,6 +191,7 @@ const T6020: PlatformProfile = PlatformProfile { bag_type: 0, arg: -1, }, + key_store: KeyStore::Variant5, }; static_assert!(T8103.shmem_capacity == 0x30000); diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 8b885ed98753e..18b7e990ee2a0 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -762,6 +762,13 @@ impl SepData { dev_warn!(self.dev, "bringup: could not enable the key store ({:?}); keybag and ref-key operations are unavailable\n", e); return; } + if !self.sks_init_endpoint() { + dev_err!( + self.dev, + "bringup: key-store endpoint init failed; the key store stays closed\n" + ); + return; + } if *module_parameters::provision_keybag.value() != 0 { if *module_parameters::xart_writes.value() == 0 { dev_err!( diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index c066e579ad33f..5acc0dd8159fe 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -547,6 +547,12 @@ struct SepData { sks_seq: Atomic, + // IPC header version negotiated by the 13.5 endpoint init; 1 otherwise. + sks_header_version: Atomic, + + // The 13.5 endpoint init (0x4d, set_env) completed. + sks_initialized: Atomic, + // u32, not u8: the kernel's Rust atomics have no u8 AtomicType phase: Atomic, @@ -769,6 +775,8 @@ impl SepData { control <- new_mutex!(control::ControlState::new()), control_wq <- new_condvar!("SepData::control_wq"), sks_seq: Atomic::new(0), + sks_header_version: Atomic::new(1), + sks_initialized: Atomic::new(false), sks_wedged: Atomic::new(0), bringup: Atomic::new(BRINGUP_FRESH), keybag_designated: Atomic::new(false), diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 03caee8f0767b..092521c35f664 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -263,6 +263,15 @@ impl SepData { Some(parsed.body) } + /// The IPC header version: 1 unless the 13.5 endpoint init negotiated 2. + pub(crate) fn sks_ipc_version(&self) -> image::Version { + if self.sks_header_version.load(Relaxed) == 2 { + image::Version::V2 + } else { + image::Version::V1 + } + } + pub(crate) fn sks_timestamp_us(&self) -> u64 { crate::shim::boottime_ns() / 1000 } @@ -276,7 +285,7 @@ impl SepData { } fn sks_seal(&self, op: &crate::sks::SksOp, body: &image::Body) -> Result { - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), body)?; let len = self.sks_image_len(&img)?; Ok(SksRequest { name: op.name(), @@ -295,6 +304,89 @@ impl SepData { self.sks_seal(&op, &body) } + /// `0x2a` set_env, as `AppleKeyStore::set_env(false)` (0xfffffe000994a168) + /// builds it: a 0x40c-byte environment `{1, mode, 4, 0x400 zero bytes}` + /// (0x…a1a0-a1e0) sent by `__ipc_set_env` (0x…9ddc) as struct version 0, + /// u64 1, blob (`_code_ipc_set_env` 0x…4b64). + fn sks_req_set_env(&self, mode: u32) -> Result { + let op = crate::sks::sks_set_env(); + let mut env = KVec::new(); + env.resize(crate::sks::SKS_SET_ENV_LEN, 0u8, GFP_KERNEL)?; + env[0..4].copy_from_slice(&1u32.to_le_bytes()); + env[4..8].copy_from_slice(&mode.to_le_bytes()); + env[8..12].copy_from_slice(&crate::sks::SKS_SET_ENV_FALSE.to_le_bytes()); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(1)?; + body.put_blob(&env)?; + self.sks_seal(&op, &body) + } + + /// `AppleKeyStore::init_sep_endpoint` (0xfffffe0009946cdc) on the 13.5 key + /// store, before any other request: `0x4d`, whose capability word sets the + /// IPC header version to `min(word, 2)` (0x…6dc0-6dd0), then `set_env` + /// (0x…6e64). Returns false if the key store must stay closed this boot. + /// The T6020 path sends nothing here. + pub(crate) fn sks_init_endpoint(&self) -> bool { + let profile::KeyStore::Sepos13 { + cpx_encryption_mode, + } = self.profile.key_store + else { + return true; + }; + + let Some(out) = self.sks_send(self.sks_req_get_capabilities()) else { + dev_err!(self.dev, "sks: GET_CAPABILITIES got no reply\n"); + return false; + }; + // Reply: struct version 0, u64 capability word, blob. + let word = self + .sks_report_response(c"GET_CAPABILITIES", &out) + .filter(|body| out.reply.status == 0 && body.len() >= 12) + .map(|body| u64::from_le_bytes(body[4..12].try_into().unwrap())); + match word { + // Version 0 leaves out the header fields __payload_hash needs. + Some(0) | None => { + dev_err!( + self.dev, + "sks: GET_CAPABILITIES status {}, no usable capability word\n", + out.reply.status + ); + return false; + } + Some(word) => { + let version = if word < 2 { 1 } else { 2 }; + self.sks_header_version.store(version, Relaxed); + dev_info!( + self.dev, + "sks: capability word {}, IPC header version {}\n", + word, + version + ); + } + } + + let Some(out) = self.sks_send(self.sks_req_set_env(cpx_encryption_mode)) else { + dev_err!(self.dev, "sks: SET_ENV got no reply\n"); + return false; + }; + // The reply is the struct-version word 0 alone (0x…4c44-4c5c). + let accepted = self + .sks_report_response(c"SET_ENV", &out) + .is_some_and(|body| body.len() == 4 && image::operation_status(body) == Some(0)); + if out.reply.status != 0 || !accepted { + dev_err!( + self.dev, + "sks: SET_ENV (cpx-encryption-mode {}) refused: status {}\n", + cpx_encryption_mode, + out.reply.status + ); + return false; + } + self.sks_initialized.store(true, Relaxed); + true + } + /// `0x0d` designate. fn sks_req_designate(&self, d: &crate::sks::Designation, secret: &[u8]) -> Result { let mut body = image::Body::new(); @@ -306,7 +398,7 @@ impl SepData { // Flags is a u64, not u32; a u32 leaves the body short -> enclave answers -13. body.put_u64(crate::sks::SKS_DESIGNATE_FLAGS)?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_designate(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { @@ -325,7 +417,7 @@ impl SepData { body.put_u32(0)?; body.put_u64(crate::sks::SKS_CLIENT_ID)?; body.put_i32(handle.value())?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_unload_keybag(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { @@ -362,7 +454,7 @@ impl SepData { body.put_i32(LockState::Unlocked.wire())?; body.put_blob(secret)?; body.put_u64(SKS_LOCK_STATE_FLAGS)?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; @@ -409,7 +501,7 @@ impl SepData { body.put_u64(crate::sks::SKS_CLIENT_ID)?; body.put_blob(wrapped)?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_load(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { @@ -445,7 +537,7 @@ impl SepData { body.put_u64(0)?; body.put_u64(0)?; body.put_blob(&[])?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_create(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { @@ -618,7 +710,7 @@ impl SepData { body.put_blob(secret)?; // Trailing u64 goes after the blob (`0x18` puts it before). body.put_u64(SKS_LOCK_STATE_FLAGS)?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; @@ -643,7 +735,7 @@ impl SepData { body.put_blob(secret)?; body.put_blob(acm_context)?; body.put_u64(0)?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_verify_secret(self.sks_next_seq(), len).ok_or(EINVAL)?; Ok(SksRequest { @@ -667,6 +759,18 @@ impl SepData { } pub(crate) fn sks_health_check(&self, why: &CStr) -> Option { + // macOS 13.5 sends 0x4d once, at endpoint init, never again. + if self.profile.key_store != profile::KeyStore::Variant5 { + if !self.sks_initialized.load(Relaxed) { + dev_err!( + self.dev, + "sks: health check ({}): endpoint not initialised\n", + why + ); + return None; + } + return Some(Healthy(())); + } let Some(out) = self.sks_send(self.sks_req_get_capabilities()) else { dev_err!( self.dev, @@ -974,6 +1078,17 @@ pub(crate) fn sks_get_capabilities() -> SksOp { } } +const OP_SKS_SET_ENV: u8 = 0x2a; +pub(crate) fn sks_set_env() -> SksOp { + SksOp { + opcode: OP_SKS_SET_ENV, + name: c"SET_ENV", + } +} +pub(crate) const SKS_SET_ENV_LEN: usize = 12 + 0x400; +/// Third environment word for `set_env(false)`, the call init_sep_endpoint makes. +pub(crate) const SKS_SET_ENV_FALSE: u32 = 4; + const OP_SKS_COPY_UUID: u8 = 0x06; pub(crate) fn sks_copy_keybag_uuid() -> SksOp { SksOp { From 77e2618b55840851d3caeecdcfc26bed4d8d5cf2 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:35:38 -0700 Subject: [PATCH 0092/1579] soc: apple: sep: create the T8103 identity keybag as macOS 13.5 does The T8103 CREATE_KEYBAG matched no 13.5 call. Its first word is the codec's struct version, not a bag type, and _code_ipc_create_keybag (0xfffffe000995de6c) takes versions 0-2 only and decodes exact-length. The request sent version 0 with the variant-5 tail, type 0x20000 (no immediate in the 13.5 AppleSEPKeyStore) and parent 0 (which identity_create refuses), and the reply parser required at least 12 bytes where a 13.5 success is 8. Send what AppleKeyStore::identity_create (0xfffffe000994b2d8) sends through __ipc_create_keybag_v2 (0xfffffe0009955ce0), the shape of userspace AKSIdentityCreateFirst: version 2, the client, type 0x400000 (mov w3, #0x400000 at 0x...b4e8), parent -1, blob A = the bag secret, an empty blob B, and the 16-byte identity UUID. The reply is exactly the version word and a handle (0x...e11c-e12c). Follow identity_create after the reply as well: copy (0x02, identity_save 0xfffffe000994b1cc), then unload (0x05). No 0x06 UUID read-back is sent; the 13.5 enclave keeps the identity bag and names it by the UUID it was created with (identity_open loads it by that UUID, 0x...b11c), so the record now stores that host-generated UUID (keybag state 1, "as generated") next to the copied blob and the secret. The T6020 request, reply check and provisioning order are unchanged. Hardware-verified on a MacBook Air (j313) with the 13.5 SEP: CREATE (164- byte image) answered status 0 with handle 1, COPY returned 32 bytes, and UNLOAD answered status 0; the SEP committed the new identity to xART and accepted that store at the next boot. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AJd8eEwJ4TGRSHnbKcWBmw --- drivers/soc/apple/keybag.rs | 20 +++++++++++++ drivers/soc/apple/profile.rs | 25 ++++++++-------- drivers/soc/apple/sks.rs | 57 +++++++++++++++++++++++++++--------- 3 files changed, 76 insertions(+), 26 deletions(-) diff --git a/drivers/soc/apple/keybag.rs b/drivers/soc/apple/keybag.rs index f97f608821761..e295693421ad7 100644 --- a/drivers/soc/apple/keybag.rs +++ b/drivers/soc/apple/keybag.rs @@ -266,6 +266,26 @@ pub(crate) fn write_bag_uuid( ) } +/// Commits a bag named by the host-generated UUID it was created with: the +/// 13.5 enclave keeps an identity bag itself and loads it by that UUID. +pub(crate) fn write_generated_uuid( + slot: Slot, + wrapped: &[u8], + uuid: &[u8; UUID_LEN], + secret: &[u8], +) -> Result<()> { + if wrapped.is_empty() || wrapped.len() > MAX_WRAPPED || secret.len() > MAX_WRAPPED { + return Err(EINVAL); + } + write_record( + slot, + UuidProvenance::AsGenerated.state(), + wrapped, + uuid, + secret, + ) +} + /// Replaces only the wrapped blob. The enclave ratchets bag material while a bag /// is active, so this snapshot must be re-taken at the catacomb-save commit or /// the enclave answers a later restore empty. diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 79a60f9de03f0..e0eb6b3eb7872 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -86,16 +86,16 @@ pub(crate) struct SensorProfile { pub(crate) capture_qualified: bool, } -/// Identity keybag `CREATE_KEYBAG` field encoding. The strict T8103 enclave -/// reads the request's first word as the bag type and rejects the value the -/// lenient T6020 enclave accepts there; macOS carries the type in the third -/// word instead. Kept per-SoC so the strict path is correct without disturbing -/// the proven T6020 encoding (hardware-verified for enrol, match, and reboot). +/// Identity keybag `CREATE_KEYBAG` field encoding. The first word is the +/// codec's struct version: the 13.5 enclave (T8103) takes versions 0-2 only +/// and macOS creates an identity with version 2, while the T6020 enclave takes +/// variant 5. Kept per-SoC so the T8103 path is correct without disturbing the +/// proven T6020 encoding (hardware-verified for enrol, match, and reboot). pub(crate) struct KeybagCreate { - /// First word: request variant, echoed back in the reply. + /// First word: struct version (request variant), echoed back in the reply. pub(crate) variant: u32, - /// Third word: bag type. Identity is `0x20000` on the strict path; `0` on - /// T6020, which distinguishes the bag by the variant word instead. + /// Third word: bag type. Identity is `0x400000` on 13.5; `0` on T6020, + /// which distinguishes the bag by the variant word instead. pub(crate) bag_type: u32, /// Fourth word: create argument / parent handle. pub(crate) arg: i32, @@ -155,11 +155,12 @@ const T8103: PlatformProfile = PlatformProfile { }, dart_range_required: false, firmware_region: c"sepfw", - // Strict enclave: the type goes in the third word; the first word is 0. + // macOS 13.5's AppleKeyStore::identity_create (0xfffffe000994b2d8): + // version 2, type 0x400000, parent -1 (it accepts only -1 or <= -10). keybag_create: KeybagCreate { - variant: 0, - bag_type: 0x20000, - arg: 0, + variant: 2, + bag_type: 0x40_0000, + arg: -1, }, // The j293 and j313 ADTs both carry `/defaults` `cpx-encryption-mode = 2`. key_store: KeyStore::Sepos13 { diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 092521c35f664..92226c59d78d4 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -520,10 +520,12 @@ impl SepData { if proof.slot() != keybag::Slot::Identity { return Err(EINVAL); } - // The strict (T8103) and lenient (T6020) enclaves encode the identity - // bag differently; the per-SoC profile carries the correct field values - // so the proven T6020 path is unchanged while T8103 gets the bag type in - // the third word (see profile::KeybagCreate). + // The 13.5 (T8103) and T6020 enclaves encode the identity bag + // differently; the per-SoC profile carries the field values (see + // profile::KeybagCreate). On 13.5 this is identity_create's + // __ipc_create_keybag_v2 (0xfffffe0009955ce0): secret, an empty second + // blob and the 16-byte identity UUID, nothing after it; the codec + // (_code_ipc_create_keybag 0xfffffe000995de6c) decodes exact-length. let enc = &self.profile.keybag_create; let mut body = image::Body::new(); body.put_u32(enc.variant)?; @@ -533,10 +535,12 @@ impl SepData { body.put_blob(secret)?; body.put_blob(&[])?; body.put_blob(uuid)?; - body.put_blob(&[])?; - body.put_u64(0)?; - body.put_u64(0)?; - body.put_blob(&[])?; + if self.profile.key_store == profile::KeyStore::Variant5 { + body.put_blob(&[])?; + body.put_u64(0)?; + body.put_u64(0)?; + body.put_blob(&[])?; + } let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; let msg = crate::sks::encode_sks_create(self.sks_next_seq(), len).ok_or(EINVAL)?; @@ -597,7 +601,13 @@ impl SepData { let Some(body) = self.sks_report_response(crate::sks::SKS_CREATE_NAME, &out) else { return false; }; - if out.reply.status != 0 || body.len() < 12 { + // 13.5 replies with the struct version and the handle alone + // (0x…e11c-e12c); the T6020 reply adds a blob. + let reply_min = match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } => 8, + profile::KeyStore::Variant5 => 12, + }; + if out.reply.status != 0 || body.len() < reply_min { dev_err!( self.dev, "sks: CREATE_KEYBAG failed: mailbox {}, body {} bytes\n", @@ -608,8 +618,14 @@ impl SepData { } let variant = u32::from_le_bytes(body[0..4].try_into().unwrap()); let raw_handle = i32::from_le_bytes(body[4..8].try_into().unwrap()); - let Some((_fv_data, end)) = image::read_blob(body, 8) else { - return false; + let end = match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } => 8, + profile::KeyStore::Variant5 => { + let Some((_fv_data, end)) = image::read_blob(body, 8) else { + return false; + }; + end + } }; if variant != self.profile.keybag_create.variant || raw_handle < 0 || end != body.len() { @@ -622,8 +638,17 @@ impl SepData { return false; } let handle = crate::sks::KeyBagHandle::from_create_reply(raw_handle); - let Some(bag_uuid) = self.sks_read_uuid(handle) else { - return false; + // identity_create reads no UUID back: 13.5 names the identity by the + // UUID it was created with, and identity_save (0xfffffe000994b1cc) is + // just the copy below, then the unload. + let bag_uuid = match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } => None, + profile::KeyStore::Variant5 => { + let Some(bag_uuid) = self.sks_read_uuid(handle) else { + return false; + }; + Some(bag_uuid) + } }; let Some(out) = self.sks_send(self.sks_req_copy_keybag(handle)) else { return false; @@ -631,7 +656,11 @@ impl SepData { let Some(wrapped) = self.wrapped_from_copy_reply(&out, c"new identity keybag") else { return false; }; - if let Err(e) = keybag::write_bag_uuid(slot, &wrapped, &bag_uuid, &secret) { + let committed = match bag_uuid { + Some(bag_uuid) => keybag::write_bag_uuid(slot, &wrapped, &bag_uuid, &secret), + None => keybag::write_generated_uuid(slot, &wrapped, &uuid, &secret), + }; + if let Err(e) = committed { dev_err!(self.dev, "sks: cannot commit identity keybag: {:?}\n", e); return false; } From 79fbcfdc3f161fceb0963ba8acd091ecaad1824f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:36:48 -0700 Subject: [PATCH 0093/1579] soc: apple: sep: activate the T8103 identity as identity_load does On macOS 13.5 an identity session is opened by AppleKeyStore::identity_load (0xfffffe000994b888), which sends exactly three key-store requests: - 0x03 LOAD_KEYBAG through identity_open (0xfffffe000994b0e4), whose blob is the 16-byte identity UUID (mov w4, #0x10 at 0x...b11c): the enclave keeps the identity bag and the host never holds a wrapped one; - 0x0d make-system-keybag, binding the loaded handle to the session handle (0x...ba48-ba5c) with an empty blob. The codec (_code_ipc_make_system_keybag 0xfffffe000995fd8c) takes struct version 0 only (0x...fe78) and the reply is that version word alone; - 0x05 UNLOAD of the source handle (0x...bad4). On T8103 the driver loaded the wrapped blob, read the UUID back with 0x06, designated with version 1, the bag secret and a trailing u64, then made a machine ref-key (0x04 unlock of the source and 0x22 'oc'), listed SBIO identities and read the session UUID back before the unload. None of that has a 13.5 counterpart. On T8103, load by the recorded identity UUID, send no 0x06, designate with version 0 and an empty blob (checking the version-0 reply), and unload the source straight after. The ref-key flow, which no 13.5 code performs with an identity bag, is refused on that profile before anything is sent, both at activation and for trusted-key and attest callers. T6020 requests, their order and the ref-key flow are unchanged. Hardware-verified on a MacBook Air (j313) with the 13.5 SEP, on every activation after provisioning: LOAD_KEYBAG by UUID status 0 with a handle, DESIGNATE_KEYBAG (source, -1000, empty blob) status 0, UNLOAD_KEYBAG status 0. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AJd8eEwJ4TGRSHnbKcWBmw --- drivers/soc/apple/refkey.rs | 8 +++++++ drivers/soc/apple/sbio.rs | 5 +++++ drivers/soc/apple/sks.rs | 43 +++++++++++++++++++++++++++++++------ 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/drivers/soc/apple/refkey.rs b/drivers/soc/apple/refkey.rs index d2ffe586594ea..63671723fd7ed 100644 --- a/drivers/soc/apple/refkey.rs +++ b/drivers/soc/apple/refkey.rs @@ -90,6 +90,11 @@ impl SepData { pub(crate) fn sks_machine_refkey(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) { const MACHINE_REFKEY_PATH: &CStr = c"/var/lib/aurora-sep-refkey.bin"; + // No 13.5 code unlocks an identity source handle or makes a ref-key + // with it; identity_load is exactly 0x03, 0x0d, 0x05 there. + if self.profile.key_store != crate::profile::KeyStore::Variant5 { + return; + } if self.machine_refkey.lock().is_some() { return; } @@ -135,6 +140,9 @@ impl SepData { } fn ensure_machine_refkey(&self) -> Result<()> { + if self.profile.key_store != crate::profile::KeyStore::Variant5 { + return Err(ENODEV); + } if self.machine_refkey.lock().is_some() { return Ok(()); } diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 18b7e990ee2a0..ebdc4e0c61fe9 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2187,6 +2187,11 @@ impl SepData { return false; }; let special = user.special_handle(); + // identity_load ends with the source unload (0xfffffe000994bad4); + // 13.5 sends nothing between its 0x0d and that unload. + if self.profile.key_store != profile::KeyStore::Variant5 { + return self.sks_send(self.sks_req_unload_keybag(source)).is_some(); + } self.log_identity_count(c"before the cold-match preparation"); let uuid_ok = self diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 92226c59d78d4..d4a1bea71c6cd 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -387,16 +387,32 @@ impl SepData { true } - /// `0x0d` designate. + /// The designate struct version, which the reply echoes. 13.5's + /// `_code_ipc_make_system_keybag` (0xfffffe000995fd8c) takes version 0 + /// only (0x…fe78). + fn sks_designate_variant(&self) -> u32 { + match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } => crate::sks::SKS_DESIGNATE_VERSION_13, + profile::KeyStore::Variant5 => crate::sks::SKS_DESIGNATE_VARIANT, + } + } + + /// `0x0d` designate. On 13.5 this is `identity_load`'s make-system-keybag + /// (0xfffffe000994ba48-ba5c): source handle, session, an empty blob and + /// nothing after it. fn sks_req_designate(&self, d: &crate::sks::Designation, secret: &[u8]) -> Result { let mut body = image::Body::new(); - body.put_u32(crate::sks::SKS_DESIGNATE_VARIANT)?; + body.put_u32(self.sks_designate_variant())?; body.put_u64(crate::sks::SKS_CLIENT_ID)?; body.put_i32(d.source().value())?; body.put_i32(d.user().special_handle().value())?; - body.put_blob(secret)?; - // Flags is a u64, not u32; a u32 leaves the body short -> enclave answers -13. - body.put_u64(crate::sks::SKS_DESIGNATE_FLAGS)?; + if self.profile.key_store == profile::KeyStore::Variant5 { + body.put_blob(secret)?; + // Flags is a u64, not u32; a u32 leaves the body short -> enclave answers -13. + body.put_u64(crate::sks::SKS_DESIGNATE_FLAGS)?; + } else { + body.put_blob(&[])?; + } let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; let len = self.sks_image_len(&img)?; @@ -702,7 +718,7 @@ impl SepData { return; } let variant = u32::from_le_bytes([body[0], body[1], body[2], body[3]]); - if variant != crate::sks::SKS_DESIGNATE_VARIANT { + if variant != self.sks_designate_variant() { return; } @@ -824,7 +840,13 @@ impl SepData { &self, stored: &keybag::StoredKeyBag, ) -> Option<(crate::sks::KeyBagHandle, [u8; keybag::UUID_LEN])> { - let request = match self.sks_req_load_keybag(stored.wrapped()) { + // 13.5 keeps the identity bag in the enclave: identity_open + // (0xfffffe000994b0e4) loads it by its 16-byte UUID (0x…b11c-b124). + let key = match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } => &stored.uuid()[..], + profile::KeyStore::Variant5 => stored.wrapped(), + }; + let request = match self.sks_req_load_keybag(key) { Ok(request) => request, Err(e) => { dev_err!( @@ -886,6 +908,12 @@ impl SepData { } let handle = crate::sks::KeyBagHandle::from_load_reply(handle); + // identity_load sends 0x0d straight after the load (0x…ba5c) and + // reads no UUID back. + if self.profile.key_store != profile::KeyStore::Variant5 { + return Some((handle, *stored.uuid())); + } + let Some(uuid) = self.sks_read_uuid(handle) else { dev_warn!(self.dev, "sks: loaded keybag has no readable UUID\n"); let _ = self.sks_send(self.sks_req_unload_keybag(handle)); @@ -1179,6 +1207,7 @@ const OP_SKS_DESIGNATE_KEYBAG: u8 = 0x0d; pub(crate) const SKS_DESIGNATE_VARIANT: u32 = 1; static_assert!(SKS_DESIGNATE_VARIANT == 1); +pub(crate) const SKS_DESIGNATE_VERSION_13: u32 = 0; pub(crate) const SKS_DESIGNATE_USER_MIN: i32 = 10; static_assert!(SKS_DESIGNATE_USER_MIN > 0); From ca0264ffd9f531822e1f6eac2b3a3c5c6350b486 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:37:28 -0700 Subject: [PATCH 0094/1579] soc: apple: sep: unlock the T8103 identity session with 0x18 macOS 13.5 unlocks an identity session with AKSIdentityUnlockSession (userspace 0x1ed92da2c), user-client selector 0x7b. The kext passes the secret unchanged to unlock_the_device(client, h, secret, 5) (0xfffffe000993cee0, called at 0x...15a4), which calls device_state_transition with state 0 and, for mode 5, flags 0 (0xfffffe000993cf04-cf0c). That is SEP op 0x18, __ipc_device_state_transition (0xfffffe0009958204): struct version 0 only (_code_ipc_device_state_transition 0xfffffe0009961f20, 0x...200c), u64 client, u32 handle, u32 state, u64 flags, blob secret; the reply is the version word and two u64. No 13.5 identity path uses 0x04 change-lock-state, whose codec takes version 0 only as well, not the version 1 with a trailing u64 the driver sends. On T8103, unlock the designated session with that 0x18 request. The unlock also ran only when the template restore succeeded. On macOS the session is unlocked whenever it is opened, and a T8103 machine that has not enrolled yet has no template to restore, so its session was never unlocked. On T8103 the unlock now follows the restore whatever its result; the match context set-up that needs templates still runs only after a successful restore. T6020 keeps 0x04 and its restore-gated order. Hardware-verified on a MacBook Air (j313) with the 13.5 SEP: on every activation after LOAD, DESIGNATE and UNLOAD, DEVICE_STATE_TRANSITION {v0, client, -1000, state 0, flags 0, secret} answered status 0 with the 20-byte reply, and the enrol credential's VERIFY_SECRET (0x21) against -1000 then answered status 0. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AJd8eEwJ4TGRSHnbKcWBmw --- drivers/soc/apple/sbio.rs | 9 ++++++++- drivers/soc/apple/sks.rs | 40 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index ebdc4e0c61fe9..534846c454696 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2154,7 +2154,11 @@ impl SepData { self.dev, "matching unavailable this boot: restore did not complete, enclave holds no template (every match refused 0x1). Not the sensor or the finger; on-disk enrolments intact\n" ); - return; + // On 13.5 the identity session is unlocked on every activation, + // template or not, as identity_load plus AKSIdentityUnlockSession. + if self.profile.key_store == profile::KeyStore::Variant5 { + return; + } } if self.keybag_designated.load(Relaxed) { @@ -2167,6 +2171,9 @@ impl SepData { } } } + if !restored { + return; + } // enclave's match arm asserts a non-empty ACM context (<= 32 bytes) if let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) { diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index d4a1bea71c6cd..8adb821d8ef14 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -463,6 +463,9 @@ impl SepData { healthy: Healthy, ) -> Result { let Healthy(()) = healthy; + if self.profile.key_store != profile::KeyStore::Variant5 { + return self.sks_req_device_state_unlock(special, secret); + } let mut body = image::Body::new(); body.put_u32(SKS_LOCK_STATE_VARIANT)?; body.put_u64(crate::sks::SKS_CLIENT_ID)?; @@ -481,6 +484,36 @@ impl SepData { }) } + /// `0x18` device-state transition to unlocked, the 13.5 identity-session + /// unlock. `AKSIdentityUnlockSession` (user-client selector 0x7b) reaches + /// `unlock_the_device(client, h, secret, 5)` (0xfffffe000993cee0, called at + /// 0x…15a4), which sends `device_state_transition` with state 0 and flags 0 + /// (0x…cf04-cf0c) through `__ipc_device_state_transition` + /// (0xfffffe0009958204): struct version 0 only (0x…200c), u64 client, u32 + /// handle, u32 state, u64 flags, blob secret. The reply is the version word + /// and two u64. + fn sks_req_device_state_unlock( + &self, + special: crate::sks::SpecialHandle, + secret: &[u8], + ) -> Result { + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(special.value())?; + body.put_i32(LockState::Unlocked.wire())?; + body.put_u64(0)?; + body.put_blob(secret)?; + let img = image::build_request(self.sks_ipc_version(), self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_device_state_transition(self.sks_next_seq(), len); + Ok(SksRequest { + name: crate::sks::SKS_DEVICE_STATE_NAME, + msg, + img, + }) + } + /// `0x06` copy key-bag UUID. fn sks_req_copy_keybag_uuid(&self, handle: crate::sks::KeyBagHandle) -> Result { let op = crate::sks::sks_copy_keybag_uuid(); @@ -1174,6 +1207,13 @@ const OP_SKS_CHANGE_LOCK_STATE: u8 = 0x04; pub(crate) const SKS_LOCK_STATE_NAME: &CStr = c"CHANGE_LOCK_STATE"; +const OP_SKS_DEVICE_STATE_TRANSITION: u8 = 0x18; +pub(crate) const SKS_DEVICE_STATE_NAME: &CStr = c"DEVICE_STATE_TRANSITION"; + +pub(crate) fn encode_sks_device_state_transition(seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(OP_SKS_DEVICE_STATE_TRANSITION, seq.value(), len.value()) +} + const OP_SKS_VERIFY_SECRET: u8 = 0x21; pub(crate) const SKS_VERIFY_SECRET_NAME: &CStr = c"VERIFY_SECRET"; pub(crate) const SKS_VERIFY_SECRET_VARIANT: u32 = 1; From 7a43543e76db403597c8c2c2f73fd9890d604b4e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:02:28 -0700 Subject: [PATCH 0095/1579] soc: apple: sep: wait for the T8103 key store as the 13.5 kext does The 13.5 kext waits 6000 ms per key-store request (sep_deliver_msg_gated: getAdjustedTimeout(0x1770), 0xfffffe0009945a14). feat/sep waits 2000 ms for a small image. On a MacBook Air (j313) a 13.5 CREATE took 1.6 s and a COPY 1.66 s, both of which write xART, so a slightly slower one would be abandoned while its writes are in flight. Use a 6000 ms floor on the 13.5 key-store profile. T6020 is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AJd8eEwJ4TGRSHnbKcWBmw --- drivers/soc/apple/sep.rs | 2 ++ drivers/soc/apple/sks.rs | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 5acc0dd8159fe..db7ad728b20fa 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -109,6 +109,8 @@ const SCRD_MAX_CAPTURE: usize = 8; const SCRD_TIMEOUT_MS: time::Msecs = 2000; const SKS_TIMEOUT_MS: time::Msecs = 2000; +/// The 13.5 kext's per-try key-store wait. +const SKS_TIMEOUT_13_MS: time::Msecs = 6000; const SKS_TIMEOUT_PER_KIB_MS: time::Msecs = 6000; diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 8adb821d8ef14..ac864cf5f18ec 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -164,6 +164,15 @@ impl SepData { } let sized_ms = sks_timeout_for(img.len()); + // The 13.5 key store is waited on as sep_deliver_msg_gated does: + // getAdjustedTimeout(0x1770), 6000 ms per wait (0xfffffe0009945a14). + // On a j313 a CREATE took + // 1.6 s and a COPY 1.66 s, close enough to the 2 s base that a slow + // one would be abandoned mid-way through its xART writes. + let floor_ms = match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } if floor_ms < SKS_TIMEOUT_13_MS => SKS_TIMEOUT_13_MS, + _ => floor_ms, + }; let timeout_ms = if sized_ms < floor_ms { floor_ms } else { From dde59cef6b7fccdf86b5344fdafa9c7a817c3cbc Mon Sep 17 00:00:00 2001 From: Justin Pfister Date: Sat, 26 Sep 2026 19:27:07 -0400 Subject: [PATCH 0096/1579] soc: apple: sep: configure T6020 to use Sepos13 and mark refused on failed keybag create --- drivers/soc/apple/profile.rs | 25 ++++++++++++------------- drivers/soc/apple/sks.rs | 3 ++- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index e0eb6b3eb7872..ffab27e4b20b4 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -155,17 +155,13 @@ const T8103: PlatformProfile = PlatformProfile { }, dart_range_required: false, firmware_region: c"sepfw", - // macOS 13.5's AppleKeyStore::identity_create (0xfffffe000994b2d8): - // version 2, type 0x400000, parent -1 (it accepts only -1 or <= -10). + // Strict enclave: the type goes in the third word; the first word is 0. keybag_create: KeybagCreate { - variant: 2, - bag_type: 0x40_0000, - arg: -1, - }, - // The j293 and j313 ADTs both carry `/defaults` `cpx-encryption-mode = 2`. - key_store: KeyStore::Sepos13 { - cpx_encryption_mode: 2, + variant: 0, + bag_type: 0x20000, + arg: 0, }, + key_store: KeyStore::Variant5, }; const T6020: PlatformProfile = PlatformProfile { @@ -186,13 +182,16 @@ const T6020: PlatformProfile = PlatformProfile { }, dart_range_required: true, firmware_region: c"sepfw", - // Proven encoding: the lenient enclave takes the variant in the first word. + // Sepos13 enclave encoding: version 2, type 0x400000, parent -1. keybag_create: KeybagCreate { - variant: 5, - bag_type: 0, + variant: 2, + bag_type: 0x40_0000, arg: -1, }, - key_store: KeyStore::Variant5, + // The j414 ADT carries `/defaults` `cpx-encryption-mode = 2`. + key_store: KeyStore::Sepos13 { + cpx_encryption_mode: 2, + }, }; static_assert!(T8103.shmem_capacity == 0x30000); diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index ac864cf5f18ec..d22adce33e179 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -672,6 +672,7 @@ impl SepData { out.reply.status, body.len() ); + let _ = keybag::mark_refused(slot); return false; } let variant = u32::from_le_bytes(body[0..4].try_into().unwrap()); @@ -846,7 +847,7 @@ impl SepData { } pub(crate) fn sks_health_check(&self, why: &CStr) -> Option { - // macOS 13.5 sends 0x4d once, at endpoint init, never again. + // Sepos13 sends 0x4d once at endpoint init; the health check only asserts it completed. if self.profile.key_store != profile::KeyStore::Variant5 { if !self.sks_initialized.load(Relaxed) { dev_err!( From 7d07c719f4246b503fca89f4dd7ecbccdca882bb Mon Sep 17 00:00:00 2001 From: Justin Pfister Date: Sat, 26 Sep 2026 21:20:09 -0400 Subject: [PATCH 0097/1579] soc: apple: sep: resolve T6020 enrolment budget, catacomb persistence and CSPRNG token minting --- drivers/soc/apple/bio_shim.c | 6 ++ drivers/soc/apple/sbio.rs | 106 +++++++++++++++++++++++++++++------ drivers/soc/apple/sep.rs | 2 +- drivers/soc/apple/shim.h | 1 + drivers/soc/apple/shim.rs | 6 ++ drivers/soc/apple/sks.rs | 21 +++++-- 6 files changed, 119 insertions(+), 23 deletions(-) diff --git a/drivers/soc/apple/bio_shim.c b/drivers/soc/apple/bio_shim.c index 50f7e7ddf61ea..fa4c3bf98e1c1 100644 --- a/drivers/soc/apple/bio_shim.c +++ b/drivers/soc/apple/bio_shim.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -201,3 +202,8 @@ __u64 sep_bio_boottime_ns(void) { return ktime_get_boottime_ns(); } + +void sep_bio_get_random_bytes(void *buf, size_t nbytes) +{ + get_random_bytes(buf, nbytes); +} diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 534846c454696..4c289d2c5dbb0 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -573,7 +573,6 @@ impl SepData { } fn save_all_components(&self, user: crate::sbio::UserId) -> bool { - let mut all = true; for (who, kind, what) in [ ( crate::sbio::CatacombUser::MASTER, @@ -591,14 +590,35 @@ impl SepData { c"user catacomb", ), ] { - if !self.save_catacomb(who, kind, what) { - all = false; + let Some(reply) = self.read_component_states() else { + dev_err!(self.dev, "enrol: cannot read component state before saving {}\n", what); + return false; + }; + let states = crate::sbio::ComponentStates::new(&reply); + let Some(state) = states.state_for(who.value()) else { + dev_err!(self.dev, "enrol: {} is absent at enrollment completion\n", what); + return false; + }; + dev_info!(self.dev, "enrol: completed {} state 0x{:x}\n", what, state); + if state & crate::sbio::COMPONENT_STATE_ACTIVE == 0 { + dev_err!(self.dev, "enrol: {} is not active at enrollment completion\n", what); + return false; + } + if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING != 0 { + if !self.save_catacomb(who, kind, what) { + dev_err!(self.dev, "enrol: save_catacomb failed for {}\n", what); + return false; + } + } else if who.value() == user.value() { + dev_err!(self.dev, "enrol: completed user catacomb is not marked for saving\n"); + return false; } } if !self.save_lockout() { - all = false; + dev_err!(self.dev, "enrol: lockout persistence failed\n"); + return false; } - all + true } fn save_after_match(&self, user: crate::sbio::UserId) -> bool { @@ -916,6 +936,16 @@ impl SepData { sensor::power(false); self.note_capture_end(); + dev_info!( + self.dev, + "verify: outcome={}\n", + match &outcome { + bio::VerifyOutcome::Matched(_) => "MATCHED", + bio::VerifyOutcome::NoMatch => "NO_MATCH", + bio::VerifyOutcome::Failed(_) => "FAILED", + } + ); + let definitive = matches!( &outcome, bio::VerifyOutcome::Matched(_) | bio::VerifyOutcome::NoMatch @@ -1003,35 +1033,58 @@ impl SepData { drop(capture); let Some(assessment) = self.sbio_expect_ok(&crate::sbio::sbio_image_assessment()) else { + dev_err!(self.dev, "verify: sbio_image_assessment failed\n"); return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); }; if assessment.len() <= crate::sbio::ASSESS_USABLE_MATCH { + dev_err!(self.dev, "verify: assessment short ({} bytes)\n", assessment.len()); return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); } - if assessment[crate::sbio::ASSESS_USABLE_MATCH] == 0 { + let usable = assessment[crate::sbio::ASSESS_USABLE_MATCH]; + dev_info!(self.dev, "verify: assessment usable_match={}\n", usable); + if usable == 0 { + dev_warn!(self.dev, "verify: sensor image not usable for match (partial contact)\n"); return bio::VerifyOutcome::Failed(ENROL_STATUS_RETRY); } let Some(result) = self.sbio_expect_ok(&crate::sbio::sbio_match_result()) else { + dev_err!(self.dev, "verify: sbio_match_result failed\n"); return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); }; let Some(parsed) = crate::sbio::MatchResult::parse(&result) else { + dev_err!(self.dev, "verify: MatchResult parse failed (result {} bytes)\n", result.len()); return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); }; + let identity = *parsed.identity_uuid(); + let known = self.bio_index.lock().contains_uuid(&identity); + + dev_info!( + self.dev, + "verify: enclave match result uuid={} user_id={} (expected {}), matches={}, known_uuid={}\n", + Hex(&identity), + parsed.user_id, + user.value(), + parsed.matches(user), + known + ); + if !parsed.matches(user) { + dev_info!(self.dev, "verify: enclave reported no match for this touch\n"); return bio::VerifyOutcome::NoMatch; } - // match result: user id groups templates, UUID at offset 0x04 identifies one - let identity = *parsed.identity_uuid(); - let known = self.bio_index.lock().contains_uuid(&identity); - if !known { + dev_err!( + self.dev, + "verify: matched template UUID {:02x?} is not known in host bio_index\n", + identity + ); return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); } + dev_info!(self.dev, "verify: MATCH CONFIRMED for user {}!\n", user.value()); bio::VerifyOutcome::Matched(bio::MatchEvidence::from_enclave_reply(identity)) } @@ -1057,6 +1110,11 @@ impl SepData { break None; } if counter >= ENROL_MAX_CAPTURES { + dev_err!( + self.dev, + "enrol: reached maximum capture budget ({} attempts) without enclave declaring completion\n", + counter + ); break Some(Err(ENROL_STATUS_TOO_MANY)); } @@ -1665,6 +1723,16 @@ impl SepData { let percent = parsed.progress_percent(); + dev_info!( + self.dev, + "enrol: capture {} -> progress_raw 0x{:02x} ({}%), has_template {}, complete {}\n", + counter, + parsed.progress_raw, + percent, + parsed.has_template(), + parsed.complete(), + ); + ImageOutcome::Progress { stage: (percent * bio::ENROL_STAGES) .div_ceil(100) @@ -2067,13 +2135,9 @@ impl SepData { fn mint_token_bytes(&self) -> Option<[u8; bio::TOKEN_LEN]> { let mut bytes = [0u8; bio::TOKEN_LEN]; - for chunk in bytes.chunks_mut(4) { - match self.get_entropy_word() { - Ok(word) => chunk.copy_from_slice(&word.to_le_bytes()[..chunk.len()]), - Err(_) => { - return None; - } - } + crate::shim::get_random_bytes(&mut bytes); + if bytes.iter().all(|&b| b == 0) { + bytes[0] = 0x42; } Some(bytes) } @@ -2108,6 +2172,12 @@ impl SepData { return Err(e); } + if handled.delete_identity.is_some() || !handled.delete_identities.is_empty() { + if let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) { + let _ = self.save_all_components(user); + } + } + if handled.wake { self.bio_wake(); } @@ -2740,7 +2810,9 @@ static_assert!(SBIO_STATUS_ALREADY_ACTIVE != SBIO_STATUS_COLD_TRANSITION); pub(crate) const COMPONENT_STATE_COLD: u32 = 0x1; pub(crate) const COMPONENT_STATE_ACTIVE: u32 = 0x2; +pub(crate) const COMPONENT_STATE_SAVE_PENDING: u32 = 0x4; static_assert!(COMPONENT_STATE_COLD != COMPONENT_STATE_ACTIVE); +static_assert!(COMPONENT_STATE_SAVE_PENDING & (COMPONENT_STATE_COLD | COMPONENT_STATE_ACTIVE) == 0); pub(crate) const COMPONENT_PAIR_LEN: usize = 8; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index db7ad728b20fa..6711178c83e96 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -364,7 +364,7 @@ static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_TIMEOUT); static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_ENCLAVE); static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_SENSOR); -const ENROL_MAX_CAPTURES: u32 = 12; +const ENROL_MAX_CAPTURES: u32 = 36; const ENROL_POLL_MS: u32 = 2; diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 2797f679f2e5e..9d6cf78842a58 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -151,6 +151,7 @@ void sep_bio_wake(void *dev); int sep_bio_capable_admin(void); __u64 sep_bio_monotonic_ns(void); __u64 sep_bio_boottime_ns(void); +void sep_bio_get_random_bytes(void *buf, size_t nbytes); /* -- trusted_shim.c ---------------------------------------------------- */ diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index f02a9b86e32ef..1a311eefc2d0b 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -194,6 +194,7 @@ extern "C" { fn sep_bio_capable_admin() -> c_int; fn sep_bio_monotonic_ns() -> u64; fn sep_bio_boottime_ns() -> u64; + fn sep_bio_get_random_bytes(buf: *mut c_void, nbytes: usize); } pub(crate) fn capable_admin() -> bool { @@ -213,6 +214,11 @@ pub(crate) fn boottime_ns() -> u64 { unsafe { sep_bio_boottime_ns() } } +pub(crate) fn get_random_bytes(buf: &mut [u8]) { + // SAFETY: `buf` is a valid slice for the duration of the call. + unsafe { sep_bio_get_random_bytes(buf.as_mut_ptr().cast(), buf.len()) } +} + /// The registered character device. /// /// # Invariants diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index d22adce33e179..66ffb72e218a5 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -994,11 +994,22 @@ impl SepData { return false; }; - let Some(uuid) = self - .sks_send(self.sks_req_copy_uuid_special(special)) - .and_then(|out| self.sks_uuid_from_reply(&out)) - else { - return false; + let stored = match keybag::read(keybag::Slot::Identity) { + Ok(keybag::State::Present(stored)) => stored, + _ => return false, + }; + + let uuid = match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } => *stored.uuid(), + profile::KeyStore::Variant5 => { + let Some(uuid) = self + .sks_send(self.sks_req_copy_uuid_special(special)) + .and_then(|out| self.sks_uuid_from_reply(&out)) + else { + return false; + }; + uuid + } }; match keybag::replace_wrapped(keybag::Slot::Identity, &wrapped, &uuid) { From d2ebd832038b2f3eecea962b297414ce2f427ff1 Mon Sep 17 00:00:00 2001 From: Yicong Hui Date: Fri, 21 Aug 2026 20:18:49 +0200 Subject: [PATCH 0098/1579] HID: dockchannel-hid: forward correct report type on SET_REPORT dchid_set_report() always sends HID_OUTPUT_REPORT regardless of the report type requested by the caller. Inline it into its sole caller and forward rtype directly, so SET_REPORT works correctly for feature reports too, not just output reports, and to simplify function calls. Signed-off-by: Yicong Hui --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 32e54117bd569..789182a24e078 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -683,12 +683,6 @@ static int dchid_get_report_cmd(struct dchid_iface *iface, u8 reportnum, void *b return ret <= 0 ? ret : ret - 1; } -/* Note: buf includes report number! */ -static int dchid_set_report(struct dchid_iface *iface, void *buf, size_t len) -{ - return dchid_cmd(iface, HID_OUTPUT_REPORT, REQ_SET_REPORT, buf, len, NULL, 0); -} - static int dchid_raw_request(struct hid_device *hdev, unsigned char reportnum, __u8 *buf, size_t len, unsigned char rtype, int reqtype) @@ -703,7 +697,7 @@ static int dchid_raw_request(struct hid_device *hdev, buf[0] = reportnum; return dchid_cmd(iface, rtype, REQ_GET_REPORT, &reportnum, 1, buf + 1, len - 1); case HID_REQ_SET_REPORT: - return dchid_set_report(iface, buf, len); + return dchid_cmd(iface, rtype, REQ_SET_REPORT, buf, len, NULL, 0); default: return -EIO; } From d7a6de11e7ed94a1871f161e37733f967ce0aa7b Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:22:24 +0200 Subject: [PATCH 0099/1579] HID: spi-hid-apple: enumerate the fourth HID interface The J313 trackpad firmware reports five HID interfaces, but the driver clamps the count to three and only ever queries mngt, kbd and tp. The fourth one is the trackpad's haptic actuator: iface 3: name 'Actuator', country 0x0, in 16 out 64 ctrl 1024 Its report descriptor declares output report 0x53, which is the report a waveform is played with, so dropping the interface puts the taptic engine out of reach entirely. Raise the cap to four and give interface 3 a spihid_interface of its own so that it is queried, registered as a HID device and torn down with the others. The "expecting at least 3" sanity check tested against the same constant, so give it its own SPIHID_MIN_DEVICES; otherwise raising the cap would reject a machine that reports exactly three interfaces. Interface 4 is still not enumerated. What it is has not been established. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/spi-hid/spi-hid-apple-core.c | 34 ++++++++++++++++-------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c index 2ed909895391c..ba4b4b23e6fbc 100644 --- a/drivers/hid/spi-hid/spi-hid-apple-core.c +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -35,12 +35,14 @@ #define SPIHID_MAX_INPUT_REPORT_SIZE 0x800 -/* support only keyboard, trackpad and management dev for now */ -#define SPIHID_MAX_DEVICES 3 +/* management, keyboard, trackpad and the trackpad's actuator */ +#define SPIHID_MIN_DEVICES 3 +#define SPIHID_MAX_DEVICES 4 #define SPIHID_DEVICE_ID_MNGT 0x0 #define SPIHID_DEVICE_ID_KBD 0x1 #define SPIHID_DEVICE_ID_TP 0x2 +#define SPIHID_DEVICE_ID_AUX 0x3 #define SPIHID_DEVICE_ID_INFO 0xd0 #define SPIHID_READ_PACKET 0x20 @@ -86,6 +88,7 @@ struct spihid_apple { struct spihid_interface mngt; struct spihid_interface kbd; struct spihid_interface tp; + struct spihid_interface aux; wait_queue_head_t wait; struct mutex tx_lock; //< protects against concurrent SPI writes @@ -279,6 +282,8 @@ static struct spihid_apple *spihid_get_data(struct spihid_interface *idev) return container_of(idev, struct spihid_apple, kbd); case SPIHID_DEVICE_ID_TP: return container_of(idev, struct spihid_apple, tp); + case SPIHID_DEVICE_ID_AUX: + return container_of(idev, struct spihid_apple, aux); default: return NULL; } @@ -408,6 +413,8 @@ static struct spihid_interface *spihid_get_iface(struct spihid_apple *spihid, return &spihid->kbd; case SPIHID_DEVICE_ID_TP: return &spihid->tp; + case SPIHID_DEVICE_ID_AUX: + return &spihid->aux; default: return NULL; } @@ -494,18 +501,17 @@ static bool spihid_process_device_info(struct spihid_apple *spihid, u32 iface, num_devices = __le16_to_cpu(info->num_devices); - if (num_devices < SPIHID_MAX_DEVICES) { + if (num_devices < SPIHID_MIN_DEVICES) { dev_err(dev, - "Device info reports %u devices, expecting at least 3", - num_devices); + "Device info reports %u devices, expecting at least %u", + num_devices, SPIHID_MIN_DEVICES); return false; } spihid->num_devices = num_devices; if (spihid->num_devices > SPIHID_MAX_DEVICES) { - dev_info( - dev, - "limiting the number of devices to mngt, kbd and mouse"); + dev_info(dev, "limiting the number of devices to %u", + SPIHID_MAX_DEVICES); spihid->num_devices = SPIHID_MAX_DEVICES; } @@ -592,8 +598,12 @@ static bool spihid_process_iface_info(struct spihid_apple *spihid, u32 num, iface->name[name_len] = '\0'; } - dev_dbg(&spihid->spidev->dev, "Info for %s, country code: 0x%x", - iface->name, iface->country); + dev_dbg(&spihid->spidev->dev, + "iface %u: name '%s', country 0x%x, in %u out %u ctrl %u", + num, iface->name, iface->country, + iface->max_input_report_len, + iface->max_output_report_len, + iface->max_control_report_len); wake_up_interruptible(&spihid->wait); } @@ -1004,9 +1014,10 @@ int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops spihid->kbd.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); spihid->tp.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + spihid->aux.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); if (!spihid->report.buf || !spihid->kbd.hid_desc || - !spihid->tp.hid_desc) + !spihid->tp.hid_desc || !spihid->aux.hid_desc) return -ENOMEM; init_waitqueue_head(&spihid->wait); @@ -1081,6 +1092,7 @@ void spihid_apple_core_remove(struct spi_device *spi) /* destroy input devices */ + spihid_destroy_hid_device(&spihid->aux); spihid_destroy_hid_device(&spihid->tp); spihid_destroy_hid_device(&spihid->kbd); From 142da109c3939d5c0ccd8bb411d26dd2e327902e Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:22:24 +0200 Subject: [PATCH 0100/1579] HID: spi-hid-apple: address reports by report id, not interface id apple_ll_raw_request() refused any SET_REPORT whose report number differed from the interface id, and apple_ll_output_report() tagged its request with the interface id rather than the report id in buf[0]. The comment on the latter already suspected it: "second idev->id should maybe be buf[0]?". Both survived only because the report number and the interface id happened to coincide for every report the driver sent: the keyboard's LED report is report 1 on interface 1, and the SPI trackpad's multitouch enable is report 2 on interface 2. The actuator on interface 3 takes reports 0x21 and 0x53, so neither held. Its mode switch was rejected before it was sent, spi-hid-apple-of spi2.0: device:3 reportnum:33 mismatch and its waveforms went out labelled report 3, which the firmware ignored silently: every write was accepted and nothing moved. With both corrected the actuator latches host-controlled mode, feature report 0x21 reads back 01, and waveforms play on a J313. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/spi-hid/spi-hid-apple-core.c | 25 ++++++++++++------------ 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c index ba4b4b23e6fbc..0f8b13994f349 100644 --- a/drivers/hid/spi-hid/spi-hid-apple-core.c +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -362,15 +362,12 @@ static int apple_ll_raw_request(struct hid_device *hdev, return idev->reply_len; case HID_REQ_SET_REPORT: - if (buf[0] != reportnum) + if (!len || buf[0] != reportnum) return -EINVAL; - if (reportnum != idev->id) { - dev_warn(&spihid->spidev->dev, - "device:%u reportnum:" - "%hhu mismatch", - idev->id, reportnum); - return -EINVAL; - } + /* + * The report number is not the interface id: the keyboard's LED + * report is 1 on interface 1, the actuator's is 0x21 on 3. + */ return spihid_apple_request(spihid, idev->id, 0x52, reportnum, 0x00, 2, buf, len); default: return -EIO; @@ -385,11 +382,13 @@ static int apple_ll_output_report(struct hid_device *hdev, __u8 *buf, if (!spihid) return -1; - dev_dbg(&spihid->spidev->dev, - "apple_ll_output_report: device:%u len:%zu:", - idev->id, len); - // second idev->id should maybe be buf[0]? - return spihid_apple_request(spihid, idev->id, 0x51, idev->id, 0x00, 0, buf, len); + if (!len) + return -EINVAL; + + dev_dbg(&spihid->spidev->dev, "device:%u report:%u len:%zu", + idev->id, buf[0], len); + /* The request carries the report number, not the interface id. */ + return spihid_apple_request(spihid, idev->id, 0x51, buf[0], 0x00, 0, buf, len); } static struct hid_ll_driver apple_hid_ll = { From 0da20c72bf17ea538bdda28902f5b027781f5a26 Mon Sep 17 00:00:00 2001 From: Yicong Hui Date: Fri, 21 Aug 2026 20:29:06 +0200 Subject: [PATCH 0101/1579] HID: apple-mtp-haptic: add driver for the MTP haptic touchpad actuator Add a driver to control the actuator of the MTP trackpad, for M2 and later devices. It plays a waveform with output report 0x53 and switches the actuator between device-controlled and host-controlled mode with feature report 0x21. The specific hardware commands and their fields have been reverse engineered on my own machine (Macbook Pro 2023 M2 Max) using the m1n1 hypervisor. Signed-off-by: Yicong Hui Co-developed-by: Chromatischer Signed-off-by: Chromatischer --- arch/arm64/configs/asahi.config | 1 + drivers/hid/Kconfig | 10 +++ drivers/hid/Makefile | 1 + drivers/hid/hid-apple-mtp-haptic.c | 106 +++++++++++++++++++++++++++++ include/linux/soc/apple/actuator.h | 19 ++++++ 5 files changed, 137 insertions(+) create mode 100644 drivers/hid/hid-apple-mtp-haptic.c create mode 100644 include/linux/soc/apple/actuator.h diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 20a50b6fcb3a1..6d77a68fc17ed 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -42,6 +42,7 @@ CONFIG_SND_SOC_TAS2764=m CONFIG_SND_SOC_TAS2770=m CONFIG_HID_APPLE=m CONFIG_HID_MAGICMOUSE=m +CONFIG_HID_APPLE_MTP_HAPTIC=m CONFIG_SERIAL_SAMSUNG=y CONFIG_SERIAL_SAMSUNG_CONSOLE=y CONFIG_HID_DOCKCHANNEL=m diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index d44e73d0c3439..3371082e799fb 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -146,6 +146,16 @@ config HID_APPLE Say Y here if you want support for keyboards of Apple iBooks, PowerBooks, MacBooks, MacBook Pros and Apple Aluminum. +config HID_APPLE_MTP_HAPTIC + tristate "Apple MTP Haptic Actuator support" + depends on HID + help + Support for the haptic actuator of the internal trackpads of Apple + Silicon laptops. + + This driver can also be built as a module. If so, it will be called + hid-apple-mtp-haptic. + config HID_APPLEIR tristate "Apple infrared receiver" depends on (USB_HID) diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index 469aec73f9e2a..a37f8b911ecf9 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -29,6 +29,7 @@ obj-$(CONFIG_HID_ACCUTOUCH) += hid-accutouch.o obj-$(CONFIG_HID_ALPS) += hid-alps.o obj-$(CONFIG_HID_ACRUX) += hid-axff.o obj-$(CONFIG_HID_APPLE) += hid-apple.o +obj-$(CONFIG_HID_APPLE_MTP_HAPTIC) += hid-apple-mtp-haptic.o obj-$(CONFIG_HID_APPLEIR) += hid-appleir.o obj-$(CONFIG_HID_APPLETB_BL) += hid-appletb-bl.o obj-$(CONFIG_HID_APPLETB_KBD) += hid-appletb-kbd.o diff --git a/drivers/hid/hid-apple-mtp-haptic.c b/drivers/hid/hid-apple-mtp-haptic.c new file mode 100644 index 0000000000000..e4dca10249df6 --- /dev/null +++ b/drivers/hid/hid-apple-mtp-haptic.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include +#include + +#include "hid-ids.h" + +int apple_taptic_send(struct hid_device *haptic_hdev, u16 effect_type, u8 strength, u8 softness) +{ + u8 event_click[] = {0x53, 0x01, strength, softness, 0x03, 0x02, 0x22, 0x48, 0x49, + 0x01, 0x04, 0x60, 0x17, 0x2D, 0x03, 0x02, 0x18, 0x32, 0x1E}; + u8 event_deep_click[] = {0x53, 0x01, strength, softness, 0x01, 0x07, 0x18, 0x18, 0x4F, + 0x02, 0x02, 0x60, 0x0C, 0x25}; + u8 event_release[] = {0x53, 0x01, strength, softness}; + u8 *msg; + int msg_length; + int ret; + + switch (effect_type) { + case HID_HP_WAVEFORMPRESS & HID_USAGE: + msg = event_click; + msg_length = sizeof(event_click); + break; + case APPLE_HP_WAVEFORMDEEPCLICK & HID_USAGE: + msg = event_deep_click; + msg_length = sizeof(event_deep_click); + break; + case HID_HP_WAVEFORMRELEASE & HID_USAGE: + msg = event_release; + msg_length = sizeof(event_release); + break; + default: + return -EINVAL; + } + + ret = hid_hw_raw_request(haptic_hdev, msg[0], msg, msg_length, + HID_OUTPUT_REPORT, HID_REQ_SET_REPORT); + + if (ret < 0) + return ret; + return 0; +} +EXPORT_SYMBOL_GPL(apple_taptic_send); + +int apple_taptic_switch_modes(struct hid_device *haptic_hdev, bool enable_host_controlled) +{ + u8 msg[] = { 0x21, enable_host_controlled }; + int ret; + + ret = hid_hw_raw_request(haptic_hdev, msg[0], msg, sizeof(msg), + HID_FEATURE_REPORT, HID_REQ_SET_REPORT); + + if (ret < 0) + return ret; + return 0; +} +EXPORT_SYMBOL_GPL(apple_taptic_switch_modes); + +static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid_device_id *id) +{ + int ret; + + ret = hid_parse(haptic_hdev); + if (ret) { + hid_err(haptic_hdev, "hid parse failed\n"); + return ret; + } + + ret = hid_hw_start(haptic_hdev, 0); + if (ret) { + hid_err(haptic_hdev, "hw start failed\n"); + return ret; + } + + return 0; +} + +static bool actuator_match(struct hid_device *hdev, bool ignore_special_drivers) +{ + return (strcmp(hdev->name, "Apple MTP actuator") == 0); +} + +static void actuator_remove(struct hid_device *haptic_hdev) +{ + apple_taptic_switch_modes(haptic_hdev, false); + hid_hw_stop(haptic_hdev); +} + +static const struct hid_device_id apple_haptic_devices[] = { + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, + HID_ANY_ID), .driver_data = 0 }, + { } +}; +MODULE_DEVICE_TABLE(hid, apple_haptic_devices); + +static struct hid_driver apple_actuator_driver = { + .name = "hid-apple-mtp-haptic", + .id_table = apple_haptic_devices, + .probe = apple_actuator_probe, + .match = actuator_match, + .remove = actuator_remove +}; +module_hid_driver(apple_actuator_driver); + +MODULE_DESCRIPTION("Apple MTP Haptic Actuator driver"); +MODULE_LICENSE("GPL"); diff --git a/include/linux/soc/apple/actuator.h b/include/linux/soc/apple/actuator.h new file mode 100644 index 0000000000000..614f50d952424 --- /dev/null +++ b/include/linux/soc/apple/actuator.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef __APPLE_MTP_ACTUATOR_H__ +#define __APPLE_MTP_ACTUATOR_H__ +#include + +#define APPLE_HP_WAVEFORMDEEPCLICK 0x000e2001 + +#if IS_ENABLED(CONFIG_HID_APPLE_MTP_HAPTIC) +int apple_taptic_send(struct hid_device *hdev, u16 effect_type, u8 strength, u8 softness); +int apple_taptic_switch_modes(struct hid_device *hdev, bool currently_host_controlled); + +#else +static inline int apple_taptic_send(struct hid_device *hdev, u16 effect_type, + u8 strength, u8 softness) { return -ENODEV; } +static inline int apple_taptic_switch_modes(struct hid_device *hdev, + bool enable_host_controlled) { return -ENODEV; } +#endif +#endif /* __APPLE_MTP_ACTUATOR_H__ */ From dab1a263f006a0429f2d472193682fad4d6cbb0e Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:22:29 +0200 Subject: [PATCH 0102/1579] HID: apple-mtp-haptic: bind the actuator on SPI machines The actuator on the SPI machines takes the same output report 0x53 and the same mode-switch feature report 0x21 as the MTP one, so the driver needs no protocol change to drive it; it simply never matched. Its id table covers BUS_HOST only, and its .match insists on the name "Apple MTP actuator", while spi-hid names the interface "Apple SPI Actuator". Nothing else claims the device either: hid-apple's id table matches every Apple SPI device, which makes hid_generic_match() stand aside, and hid-apple's probe then rejects anything that is not HID_TYPE_SPI_KEYBOARD. The actuator is left bound to no driver at all. Match BUS_SPI Apple devices as well and accept either name, via a helper in the actuator header so hid-magicmouse can use the same test. Start with HID_CONNECT_HIDRAW so the waveform parameters can be exercised from userspace: the strength and softness that suit this engine are not known yet, and the MTP defaults feel harsh on a J313. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-apple-mtp-haptic.c | 6 ++++-- include/linux/soc/apple/actuator.h | 28 ++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/drivers/hid/hid-apple-mtp-haptic.c b/drivers/hid/hid-apple-mtp-haptic.c index e4dca10249df6..0678953acbb40 100644 --- a/drivers/hid/hid-apple-mtp-haptic.c +++ b/drivers/hid/hid-apple-mtp-haptic.c @@ -66,7 +66,7 @@ static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid return ret; } - ret = hid_hw_start(haptic_hdev, 0); + ret = hid_hw_start(haptic_hdev, HID_CONNECT_HIDRAW); if (ret) { hid_err(haptic_hdev, "hw start failed\n"); return ret; @@ -77,7 +77,7 @@ static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid static bool actuator_match(struct hid_device *hdev, bool ignore_special_drivers) { - return (strcmp(hdev->name, "Apple MTP actuator") == 0); + return apple_taptic_is_actuator(hdev); } static void actuator_remove(struct hid_device *haptic_hdev) @@ -89,6 +89,8 @@ static void actuator_remove(struct hid_device *haptic_hdev) static const struct hid_device_id apple_haptic_devices[] = { { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, HID_ANY_ID), .driver_data = 0 }, + { HID_DEVICE(BUS_SPI, HID_GROUP_ANY, SPI_VENDOR_ID_APPLE, + HID_ANY_ID), .driver_data = 0 }, { } }; MODULE_DEVICE_TABLE(hid, apple_haptic_devices); diff --git a/include/linux/soc/apple/actuator.h b/include/linux/soc/apple/actuator.h index 614f50d952424..0b565d5567eb0 100644 --- a/include/linux/soc/apple/actuator.h +++ b/include/linux/soc/apple/actuator.h @@ -2,18 +2,38 @@ #ifndef __APPLE_MTP_ACTUATOR_H__ #define __APPLE_MTP_ACTUATOR_H__ + #include +#include +#include #define APPLE_HP_WAVEFORMDEEPCLICK 0x000e2001 -#if IS_ENABLED(CONFIG_HID_APPLE_MTP_HAPTIC) +/* + * The actuator is a HID device of its own, named by the transport: "Apple + * MTP actuator" on the MTP machines, "Apple SPI Actuator" on the SPI ones. + */ +static inline bool apple_taptic_is_actuator(struct hid_device *hdev) +{ + return !strcmp(hdev->name, "Apple MTP actuator") || + !strcmp(hdev->name, "Apple SPI Actuator"); +} + +#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) int apple_taptic_send(struct hid_device *hdev, u16 effect_type, u8 strength, u8 softness); -int apple_taptic_switch_modes(struct hid_device *hdev, bool currently_host_controlled); +int apple_taptic_switch_modes(struct hid_device *hdev, bool host_controlled); #else static inline int apple_taptic_send(struct hid_device *hdev, u16 effect_type, - u8 strength, u8 softness) { return -ENODEV; } + u8 strength, u8 softness) +{ + return -ENODEV; +} + static inline int apple_taptic_switch_modes(struct hid_device *hdev, - bool enable_host_controlled) { return -ENODEV; } + bool host_controlled) +{ + return -ENODEV; +} #endif #endif /* __APPLE_MTP_ACTUATOR_H__ */ From cf248ccdaabc68aae214ebdaf5c8ce5235d4eb46 Mon Sep 17 00:00:00 2001 From: Yicong Hui Date: Fri, 21 Aug 2026 21:23:49 +0200 Subject: [PATCH 0103/1579] HID: magicmouse: add support for controlling MTP touchpad haptics Integrate the MTP actuator driver with the MTP trackpad driver to expose new functionality that allows userspace to control the haptic effects of the touchpad actuator as a force feedback device, following the convention for haptic touchpad functionality as specified in [1]. Set the INPUT_PROP_PRESSUREPAD property within the MTP trackpad device indicating that this device is a haptic touchpad, and set the FF_HAPTIC capability to indicate that force feedback haptic effects can be uploaded to the trackpad [2]. Userspace can thus choose to upload force feedback effects to the touchpad, which would trigger it to switch into "Host controlled" mode, where the touchpad will no longer click unless signalled to do so by userspace. This patch uses an unconventional means of doing so, by writing a separate loadable module for the haptic actuator. This is because the haptic actuator is considered a separate device on MTP trackpads, whereas haptics for HUTRR63 (and the other kernel drivers) considers the haptic actuator and the touchpad to be the very same device. Therefore, to conform to the standardized interface in [1] this patch creates a separate driver for the separate HID device, and is designed such that commands to the actuator are sent through the touchpad. This changeset is relevant because allowing userspace control of the haptics allows for user control of: 1. Click feedback intensity 2. Sensitivity of when clicks trigger 3. Configuring "Force Touch"-like deep click functionality The actuator is a sibling HID device that the transport can register after the trackpad, so it is resolved lazily on the first effect upload. The force feedback device is created before the input device is registered and destroyed explicitly before the driver state goes away, and the whole integration is compiled out when the actuator driver is not reachable, so this driver does not depend on it. [1] https://lore.kernel.org/all/20250818-support-forcepads-v3-0-e4f9ab0add84@google.com/T/#m12f9be6691015fb9b2e834248c368e5f90 [2] https://docs.kernel.org/input/event-codes.html#input-prop-pressurepad Signed-off-by: Yicong Hui Co-developed-by: Chromatischer Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 347 ++++++++++++++++++++++++++++++++++- 1 file changed, 346 insertions(+), 1 deletion(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index d6c43cc967e3b..35fbe163e0beb 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -19,7 +19,9 @@ #include #include #include +#include +#include "hid-haptic.h" #include "hid-ids.h" static bool emulate_3button = true; @@ -140,6 +142,14 @@ struct magicmouse_input_ops { int (*setup_input)(struct input_dev *input, struct hid_device *hdev); }; +struct effect_job { + struct work_struct work; + struct hid_device *taptic_hdev; + u16 effect_type; + u8 strength; + u8 softness; +}; + /** * struct magicmouse_sc - Tracks Magic Mouse-specific data. * @input: Input device through which we report events. @@ -152,6 +162,9 @@ struct magicmouse_input_ops { * @pos: multi touch position data of the last report. * @touches: Most recent data for a touch, indexed by tracking ID. * @tracking_ids: Mapping of current touch input data to @touches. + * @haptics: Haptic device state of an MTP trackpad, NULL otherwise. + * @taptic_hdev: Actuator HID device of an MTP trackpad, if resolved. + * @haptic_effects: Uploaded haptic effects of an MTP trackpad. * @hdev: Pointer to the underlying HID device. * @work: Workqueue to handle initialization retry for quirky devices. * @battery_timer: Timer for obtaining battery level information. @@ -179,6 +192,10 @@ struct magicmouse_sc { } touches[MAX_CONTACTS]; int tracking_ids[MAX_CONTACTS]; + struct hid_haptic_device *haptics; + struct hid_device *taptic_hdev; + struct effect_job *haptic_effects; + struct hid_device *hdev; struct delayed_work work; struct timer_list battery_timer; @@ -1127,6 +1144,7 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, struct magicmouse_sc *msc = hid_get_drvdata(hdev); __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); + __set_bit(INPUT_PROP_PRESSUREPAD, input->propbit); __clear_bit(BTN_0, input->keybit); __clear_bit(BTN_RIGHT, input->keybit); __clear_bit(BTN_MIDDLE, input->keybit); @@ -1134,7 +1152,7 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, __clear_bit(REL_X, input->relbit); __clear_bit(REL_Y, input->relbit); - mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK; + mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK | INPUT_MT_TOTAL_FORCE; /* * The C1FE report carries positions and the button only; advertise @@ -1254,6 +1272,57 @@ static int magicmouse_input_mapping(struct hid_device *hdev, return 0; } +static int magicmouse_init_haptics(struct magicmouse_sc *msc, + struct hid_device *hdev); + +#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +static int match_actuator(struct device *dev, const void *data) +{ + struct hid_device *hdev; + + if (dev->bus != &hid_bus_type) + return 0; + + hdev = to_hid_device(dev); + + return !strcmp(hdev->name, "Apple MTP actuator"); +} + +static struct hid_device *magicmouse_get_actuator(struct magicmouse_sc *msc) +{ + struct device *dev; + + if (msc->taptic_hdev) + return msc->taptic_hdev; + + if (!msc->hdev->dev.parent) + return NULL; + + dev = device_find_child(msc->hdev->dev.parent, NULL, match_actuator); + if (!dev) + return NULL; + + msc->taptic_hdev = to_hid_device(dev); + hid_dbg(msc->hdev, "bound to the MTP actuator\n"); + + return msc->taptic_hdev; +} + +static int magicmouse_switch_mode(struct input_dev *trackpad_idev, int mode) +{ + struct hid_device *trackpad_hdev = input_get_drvdata(trackpad_idev); + struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); + struct hid_device *actuator = magicmouse_get_actuator(msc); + + if (!actuator) + return -ENODEV; + + hid_dbg(trackpad_hdev, "switching haptics to mode %d\n", mode); + + return apple_taptic_switch_modes(actuator, mode); +} +#endif + static int magicmouse_input_configured(struct hid_device *hdev, struct hid_input *hi) @@ -1274,6 +1343,16 @@ static int magicmouse_input_configured(struct hid_device *hdev, return ret; } + /* + * This runs before the input device is registered, and the force + * feedback device has to exist by then for evdev clients to see + * FF_HAPTIC. + */ + ret = magicmouse_init_haptics(msc, hdev); + if (ret) + hid_warn(hdev, "haptics unavailable (%d), continuing without force feedback\n", + ret); + return 0; } @@ -1399,6 +1478,262 @@ static void magicmouse_battery_timer_tick(struct timer_list *t) } } +#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +static int apple_upload_effects(struct input_dev *trackpad_idev, + struct ff_effect *effect, struct ff_effect *old) +{ + struct hid_device *trackpad_hdev = input_get_drvdata(trackpad_idev); + struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); + struct hid_haptic_device *haptics = msc->haptics; + struct hid_device *actuator; + int ret; + + switch (effect->u.haptic.hid_usage) { + case (HID_HP_WAVEFORMPRESS & HID_USAGE): + case (HID_HP_WAVEFORMRELEASE & HID_USAGE): + case (APPLE_HP_WAVEFORMDEEPCLICK & HID_USAGE): + break; + default: + return -EINVAL; + } + + actuator = magicmouse_get_actuator(msc); + if (!actuator) + return -ENODEV; + + msc->haptic_effects[effect->id].taptic_hdev = actuator; + msc->haptic_effects[effect->id].effect_type = (effect->u.haptic.hid_usage) & HID_USAGE; + msc->haptic_effects[effect->id].strength = + (min(100, (effect->u.haptic.intensity)) * 255) / 100; + msc->haptic_effects[effect->id].softness = 0x90; + /* A future extension could add configurability to softness. */ + + /* The actuator may have been rebound or reset since the last upload. */ + ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_HOST); + if (ret) { + dev_err(&msc->hdev->dev, + "failed to switch to host-controlled mode: %d\n", ret); + msc->haptic_effects[effect->id].effect_type = 0; + return ret; + } + + haptics->mode = HID_HAPTIC_MODE_HOST; + + hid_dbg(trackpad_hdev, "uploaded haptic effect %d\n", effect->id); + return 0; +} + +static void haptic_playback_worker(struct work_struct *ws) +{ + struct effect_job *job = container_of(ws, struct effect_job, work); + + if (job->effect_type && + apple_taptic_send(job->taptic_hdev, job->effect_type, + job->strength, job->softness)) + hid_warn(job->taptic_hdev, "cannot play the haptic effect\n"); +} + +static int apple_taptic_playback(struct input_dev *trackpad_idev, int effect_id, int value) +{ + struct hid_device *trackpad_hdev = input_get_drvdata(trackpad_idev); + struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); + + if (value) + queue_work(msc->haptics->wq, &msc->haptic_effects[effect_id].work); + + return 0; +} + +static int apple_taptic_erase(struct input_dev *trackpad_idev, int effect_id) +{ + struct hid_device *trackpad_hdev = input_get_drvdata(trackpad_idev); + struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); + int i, ret = 0; + + msc->haptic_effects[effect_id].effect_type = 0; + + for (i = 0; i < FF_MAX_EFFECTS; i++) { + if (msc->haptic_effects[i].effect_type != 0) + return 0; + } + + /* Return to device-controlled mode if there are no effects left */ + if (msc->haptics->mode == HID_HAPTIC_MODE_HOST) { + flush_workqueue(msc->haptics->wq); + ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_DEVICE); + + if (ret) { + dev_err(&msc->hdev->dev, + "failed to switch back to device-controlled mode: %d\n", + ret); + return ret; + } + + msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + } + + return 0; +} + +static void apple_taptic_destroy(struct ff_device *ff) +{ + struct hid_haptic_device *haptic_dev = ff->private; + struct magicmouse_sc *msc = hid_get_drvdata(haptic_dev->hdev); + int ret; + + /* ff->private is devm-allocated; do not let ff-core kfree() it. */ + ff->private = NULL; + + if (msc->haptics && msc->haptics->mode == HID_HAPTIC_MODE_HOST) { + flush_workqueue(msc->haptics->wq); + ret = magicmouse_switch_mode(msc->haptics->input_dev, HID_HAPTIC_MODE_DEVICE); + if (ret) + hid_warn(msc->hdev, + "failed to switch back to device-controlled mode: %d\n", + ret); + else + msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + } + + destroy_workqueue(haptic_dev->wq); + haptic_dev->wq = NULL; + + kfree(msc->haptic_effects); + msc->haptic_effects = NULL; +} + +static int apple_taptic_init_mtp(struct magicmouse_sc *msc, + struct hid_haptic_device *haptic_dev) +{ + struct hid_device *trackpad_hdev = msc->hdev; + struct ff_device *ff; + int ret, i; + + haptic_dev->hdev = trackpad_hdev; + haptic_dev->input_dev = msc->input; + + msc->haptic_effects = kzalloc_objs(struct effect_job, FF_MAX_EFFECTS); + if (!msc->haptic_effects) { + dev_err(&trackpad_hdev->dev, "Cannot allocate haptic effects\n"); + return -ENOMEM; + } + + haptic_dev->wq = create_singlethread_workqueue("Apple trackpad haptics workqueue"); + if (!haptic_dev->wq) { + dev_err(&trackpad_hdev->dev, "Cannot allocate haptic workqueue\n"); + kfree(msc->haptic_effects); + msc->haptic_effects = NULL; + return -ENOMEM; + } + + for (i = 0; i < FF_MAX_EFFECTS; i++) + INIT_WORK(&msc->haptic_effects[i].work, haptic_playback_worker); + + /* FF_HAPTIC has to be in ffbit before the FF device is created. */ + input_set_capability(msc->input, EV_FF, FF_HAPTIC); + + ret = input_ff_create(msc->input, FF_MAX_EFFECTS); + if (ret) { + /* undo the capability set above, no FF device was created */ + __clear_bit(EV_FF, msc->input->evbit); + __clear_bit(FF_HAPTIC, msc->input->ffbit); + + kfree(msc->haptic_effects); + msc->haptic_effects = NULL; + + destroy_workqueue(haptic_dev->wq); + haptic_dev->wq = NULL; + + dev_err(&trackpad_hdev->dev, "Failed to create force-feedback device.\n"); + return ret; + } + + ff = msc->input->ff; + ff->private = haptic_dev; + ff->upload = apple_upload_effects; + ff->playback = apple_taptic_playback; + ff->erase = apple_taptic_erase; + ff->destroy = apple_taptic_destroy; + + /* + * Hold the input device until magicmouse_teardown_haptics(), which + * destroys the force feedback device while msc is still valid. + */ + input_get_device(msc->input); + + hid_info(trackpad_hdev, "initialised MTP haptics\n"); + return 0; +} + +static int magicmouse_init_haptics(struct magicmouse_sc *msc, struct hid_device *trackpad_hdev) +{ + struct hid_haptic_device *haptics; + int ret; + + if (msc->haptics) + return 0; + + /* + * Only the internal trackpads have an actuator. It is a separate HID + * device that may be registered after the trackpad, so it is resolved + * lazily when the first effect is uploaded. + */ + if (trackpad_hdev->bus != BUS_HOST || + trackpad_hdev->type != HID_TYPE_SPI_MOUSE) + return 0; + + haptics = devm_kzalloc(&trackpad_hdev->dev, sizeof(*haptics), GFP_KERNEL); + if (!haptics) + return -ENOMEM; + + haptics->hdev = trackpad_hdev; + + ret = apple_taptic_init_mtp(msc, haptics); + if (ret) { + devm_kfree(&trackpad_hdev->dev, haptics); + return ret; + } + + msc->haptics = haptics; + return 0; +} + +static void magicmouse_teardown_haptics(struct magicmouse_sc *msc) +{ + if (!msc->haptics) + return; + + /* + * Tear the force feedback device down while msc is still valid: an + * open evdev fd would otherwise run it from input_dev_release() after + * devres has freed the driver state. + */ + if (msc->haptics->input_dev) { + input_ff_destroy(msc->haptics->input_dev); + input_put_device(msc->haptics->input_dev); + msc->haptics->input_dev = NULL; + } + + msc->haptics = NULL; + msc->haptic_effects = NULL; + + if (msc->taptic_hdev) { + put_device(&msc->taptic_hdev->dev); + msc->taptic_hdev = NULL; + } +} +#else +static int magicmouse_init_haptics(struct magicmouse_sc *msc, + struct hid_device *hdev) +{ + return 0; +} + +static void magicmouse_teardown_haptics(struct magicmouse_sc *msc) +{ +} +#endif + static int magicmouse_probe(struct hid_device *hdev, const struct hid_device_id *id) { @@ -1452,6 +1787,11 @@ static int magicmouse_probe(struct hid_device *hdev, ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT); if (ret) { hid_err(hdev, "magicmouse hw start failed\n"); + /* + * hid_connect() cannot fail once hidinput_connect() has + * succeeded, so no input device is registered here. + */ + magicmouse_teardown_haptics(msc); return ret; } @@ -1554,6 +1894,8 @@ static int magicmouse_probe(struct hid_device *hdev, timer_delete_sync(&msc->battery_timer); hid_hw_stop(hdev); + magicmouse_teardown_haptics(msc); + return ret; } @@ -1569,6 +1911,9 @@ static void magicmouse_remove(struct hid_device *hdev) } hid_hw_stop(hdev); + + if (msc) + magicmouse_teardown_haptics(msc); } #ifdef CONFIG_PM From 2bea5b0af9e79d9df78254f94602495a1c8a6ad3 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:22:29 +0200 Subject: [PATCH 0104/1579] HID: magicmouse: allow haptics on SPI trackpads magicmouse_init_haptics() requires BUS_HOST, and match_actuator() looks for the MTP actuator by name, so the SPI trackpads were left without haptics even though their machines have an actuator that speaks the same protocol. Accept BUS_SPI as well and use the shared name test. The lookup itself needs no change: the actuator is a sibling HID device under the same SPI parent, which is what magicmouse_get_actuator() already searches. This does not change how a click feels by default. The actuator stays in device-controlled mode, where its firmware provides the click feedback, and only moves to host-controlled mode when userspace uploads an effect. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 35fbe163e0beb..556be8b1bb0e8 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -1285,7 +1285,7 @@ static int match_actuator(struct device *dev, const void *data) hdev = to_hid_device(dev); - return !strcmp(hdev->name, "Apple MTP actuator"); + return apple_taptic_is_actuator(hdev); } static struct hid_device *magicmouse_get_actuator(struct magicmouse_sc *msc) @@ -1674,11 +1674,12 @@ static int magicmouse_init_haptics(struct magicmouse_sc *msc, struct hid_device return 0; /* - * Only the internal trackpads have an actuator. It is a separate HID - * device that may be registered after the trackpad, so it is resolved - * lazily when the first effect is uploaded. + * Only the internal trackpads have an actuator: the MTP ones on + * BUS_HOST and the SPI ones, whose firmware exposes it as a separate + * "Actuator" interface. It may be registered after the trackpad, so + * it is resolved lazily on the first effect upload. */ - if (trackpad_hdev->bus != BUS_HOST || + if ((trackpad_hdev->bus != BUS_HOST && trackpad_hdev->bus != BUS_SPI) || trackpad_hdev->type != HID_TYPE_SPI_MOUSE) return 0; From acd5476d25c8fd441af7a92fa34c00f4843a1372 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 21:42:42 +0200 Subject: [PATCH 0105/1579] HID: magicmouse: make the haptic waveform shape configurable The waveform sent to the actuator carries a strength and a softness. Both were fixed: softness at 0x90, and strength at the full 255 scaled by the effect's intensity. Those suit the MTP actuator, but on a J313 the same numbers produce a hard knock rather than a click. Judged by feel there, a press wants roughly strength 0x28 at softness 0xf0, far weaker than the MTP default, while a deep click wants full strength 0xff at a softness of 0xd0. Add four parameters for the two waveforms, each defaulting per transport so the MTP path keeps the values its firmware was tuned with. Strength now names what 100% intensity maps to, so an effect asking for half intensity still gets half of it. They are writable at runtime and take effect on the next effect upload, which is what makes tuning by feel practical: the values that suit a given actuator cannot be derived, only tried. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 62 ++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 556be8b1bb0e8..7727cd87668d9 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -55,6 +55,27 @@ static bool report_undeciphered; module_param(report_undeciphered, bool, 0644); MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state field using a MSC_RAW event"); +/* + * Waveform shaping, writable at runtime and applied on the next effect + * upload. Zero means "use the per-transport default": the MTP machines keep + * the values their firmware was tuned with, the SPI actuator in a J313 needs + * a much softer, weaker waveform. The SPI numbers were judged by feel there. + */ +static u8 haptic_softness; +module_param(haptic_softness, byte, 0644); +MODULE_PARM_DESC(haptic_softness, "softness of the press and release waveforms, 1-255 (0: 0xf0 on SPI, 0x90 on MTP)"); + +static u8 haptic_deep_softness; +module_param(haptic_deep_softness, byte, 0644); +MODULE_PARM_DESC(haptic_deep_softness, "softness of the deep-click waveform, 1-255 (0: 0xd0 on SPI, 0x90 on MTP)"); + +static u8 haptic_strength; +module_param(haptic_strength, byte, 0644); +MODULE_PARM_DESC(haptic_strength, "strength at 100% intensity for press and release, 1-255 (0: 0x28 on SPI, 0xff on MTP)"); + +static u8 haptic_deep_strength; +module_param(haptic_deep_strength, byte, 0644); +MODULE_PARM_DESC(haptic_deep_strength, "strength at 100% intensity for a deep click, 1-255 (0: 0xff on SPI, 0xff on MTP)"); #define TRACKPAD2_2021_BT_VERSION 0x110 #define TRACKPAD_2024_BT_VERSION 0x314 @@ -1479,6 +1500,40 @@ static void magicmouse_battery_timer_tick(struct timer_list *t) } #if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +static bool magicmouse_effect_is_deep(const struct ff_effect *effect) +{ + return effect->u.haptic.hid_usage == + (APPLE_HP_WAVEFORMDEEPCLICK & HID_USAGE); +} + +static u8 magicmouse_haptic_softness(struct hid_device *hdev, + const struct ff_effect *effect) +{ + bool deep = magicmouse_effect_is_deep(effect); + u8 configured = deep ? haptic_deep_softness : haptic_softness; + + if (configured) + return configured; + if (hdev->bus != BUS_SPI) + return 0x90; + + return deep ? 0xd0 : 0xf0; +} + +static u8 magicmouse_haptic_strength(struct hid_device *hdev, + const struct ff_effect *effect) +{ + bool deep = magicmouse_effect_is_deep(effect); + u8 configured = deep ? haptic_deep_strength : haptic_strength; + + if (configured) + return configured; + if (hdev->bus != BUS_SPI) + return 0xff; + + return deep ? 0xff : 0x28; +} + static int apple_upload_effects(struct input_dev *trackpad_idev, struct ff_effect *effect, struct ff_effect *old) { @@ -1504,9 +1559,10 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, msc->haptic_effects[effect->id].taptic_hdev = actuator; msc->haptic_effects[effect->id].effect_type = (effect->u.haptic.hid_usage) & HID_USAGE; msc->haptic_effects[effect->id].strength = - (min(100, (effect->u.haptic.intensity)) * 255) / 100; - msc->haptic_effects[effect->id].softness = 0x90; - /* A future extension could add configurability to softness. */ + (min(100, (effect->u.haptic.intensity)) * + magicmouse_haptic_strength(trackpad_hdev, effect)) / 100; + msc->haptic_effects[effect->id].softness = + magicmouse_haptic_softness(trackpad_hdev, effect); /* The actuator may have been rebound or reset since the last upload. */ ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_HOST); From 73e4f72f05f124393a5c4fc871b715e583ccf082 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 22:13:13 +0200 Subject: [PATCH 0106/1579] HID: magicmouse: drive click feedback from the host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The actuator's firmware provides the feedback for a physical press, and how that press feels cannot be changed while it does. Playing the waveform from the host instead makes it adjustable, but it cannot be done by reacting to the button the trackpad reports: in host-controlled mode the firmware stops detecting clicks altogether. It keeps reporting fingers and never sets the button again, because deciding that a press happened belongs to whoever owns the actuator. Hooking the button bit therefore works exactly once — for the press already in flight when the mode changed — and then the pad has no click feedback and no button at all. So derive the click here. Sum the pressure of the fingers in each frame, call a press when it rises past haptic_press_threshold and a release when it falls below haptic_release_threshold, report BTN_MOUSE from that, and play the waveform from the haptics workqueue, which is where it can sleep. The thresholds default to 120 and 75, measured on a J313 against its own firmware: it asserts the button between 130 and 243 and releases between 26 and 73. The gap between the two is what stops a click chattering. This is off by default. Turning it off returns the actuator to the firmware on the next frame, and any failure while playing does the same, so a pad that cannot play a waveform falls back to working feedback rather than none. What the parameters do on an SPI actuator is narrower than the report layout suggests. The fifteen waveform bytes are ignored: zeroing them feels identical. Strength is close to on or off, with 0x00 silent and the values above it differing little. Softness is what shapes the pulse; swept from 0x10 to 0xff at a fixed strength it runs from a faint tick to a hard knock. A click that wants more weight than softness gives is made of more pulses, which is what haptic_press_pulses is for. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer [Satya Benson: rebased onto J700 C1FE support; C1FE reports carry no pressure, so those pads keep reporting the firmware's button] Assisted-by: Claude:claude-opus-5-5 --- drivers/hid/hid-magicmouse.c | 238 +++++++++++++++++++++++++++++++++-- 1 file changed, 227 insertions(+), 11 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 7727cd87668d9..b7049a888e023 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -11,6 +11,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -76,6 +77,42 @@ MODULE_PARM_DESC(haptic_strength, "strength at 100% intensity for press and rele static u8 haptic_deep_strength; module_param(haptic_deep_strength, byte, 0644); MODULE_PARM_DESC(haptic_deep_strength, "strength at 100% intensity for a deep click, 1-255 (0: 0xff on SPI, 0xff on MTP)"); + +/* + * Host-driven click feedback: hold the actuator in host-controlled mode and + * play the pulses from here, which is the only way to change how a press + * feels. With this off, the firmware provides the feedback itself. + */ +static bool haptic_press; +module_param(haptic_press, bool, 0644); +MODULE_PARM_DESC(haptic_press, "play click feedback from the host on physical press (default off: the actuator's firmware does it)"); + +static u8 haptic_press_pulses = 1; +module_param(haptic_press_pulses, byte, 0644); +MODULE_PARM_DESC(haptic_press_pulses, "pulses played on press (default 1)"); + +static u8 haptic_release_pulses = 1; +module_param(haptic_release_pulses, byte, 0644); +MODULE_PARM_DESC(haptic_release_pulses, "release waveforms played on release, minimum 1 (default 1)"); + +/* + * Click thresholds, in the units the trackpad reports per finger summed over + * the fingers in a frame. On a J313 the firmware asserts the button between + * 130 and 243 and releases between 26 and 73; these sit just inside that, with + * enough hysteresis that a click cannot chatter. + */ +static u16 haptic_press_threshold = 120; +module_param(haptic_press_threshold, ushort, 0644); +MODULE_PARM_DESC(haptic_press_threshold, "summed finger pressure at which a click starts (default 120)"); + +static u16 haptic_release_threshold = 75; +module_param(haptic_release_threshold, ushort, 0644); +MODULE_PARM_DESC(haptic_release_threshold, "summed finger pressure at which a click ends (default 75)"); + +static u8 haptic_pulse_gap_ms = 25; +module_param(haptic_pulse_gap_ms, byte, 0644); +MODULE_PARM_DESC(haptic_pulse_gap_ms, "milliseconds between pulses when more than one (default 25)"); + #define TRACKPAD2_2021_BT_VERSION 0x110 #define TRACKPAD_2024_BT_VERSION 0x314 @@ -216,6 +253,9 @@ struct magicmouse_sc { struct hid_haptic_device *haptics; struct hid_device *taptic_hdev; struct effect_job *haptic_effects; + struct work_struct haptic_press_work; + struct work_struct haptic_release_work; + bool haptic_button_down; struct hid_device *hdev; struct delayed_work work; @@ -839,6 +879,9 @@ static void report_c1fe_finger_data(struct input_dev *input, int slot, input_report_abs(input, ABS_MT_POSITION_Y, pos->y); } +static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, + bool firmware_button); + static void report_finger_data(struct input_dev *input, int slot, const struct input_mt_pos *pos, const struct tp_finger *f) @@ -868,6 +911,7 @@ static int magicmouse_raw_event_mtp(struct hid_device *hdev, struct input_dev *input = msc->input; struct tp_header *tp_hdr; struct tp_finger *f; + u32 pressure = 0; int i, n; u32 npoints; size_t hdr_sz = sizeof(struct tp_header); @@ -919,14 +963,20 @@ static int magicmouse_raw_event_mtp(struct hid_device *hdev, for (i = 0; i < n; i++) { int idx = map_contacs[i]; f = (struct tp_finger *)(data + hdr_sz + idx * touch_sz); - if (msc->mtp_c1fe) + if (msc->mtp_c1fe) { report_c1fe_finger_data(input, msc->tracking_ids[i], &msc->pos[i]); - else + } else { report_finger_data(input, msc->tracking_ids[i], &msc->pos[i], f); + pressure += le16_to_int(f->pressure); + } } input_mt_sync_frame(input); - input_report_key(input, BTN_MOUSE, tp_hdr->buttons & 1); + /* The C1FE report carries no pressure to derive a click from. */ + input_report_key(input, BTN_MOUSE, + msc->mtp_c1fe ? tp_hdr->buttons & 1 : + magicmouse_haptic_click(msc, pressure, + tp_hdr->buttons & 1)); input_sync(input); return 1; @@ -1342,6 +1392,168 @@ static int magicmouse_switch_mode(struct input_dev *trackpad_idev, int mode) return apple_taptic_switch_modes(actuator, mode); } + +static u8 magicmouse_haptic_softness(struct hid_device *hdev, bool deep); +static u8 magicmouse_haptic_strength(struct hid_device *hdev, bool deep); +static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc); + +/* + * Play the waveform @pulses times, @haptic_pulse_gap_ms apart. A failure here + * would leave host-controlled mode without any click feedback, so hand the + * actuator back to the firmware and turn the feature off instead. + */ +static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, + u8 pulses) +{ + struct hid_device *actuator = magicmouse_get_actuator(msc); + u8 strength, softness; + unsigned int i; + int ret; + + if (!pulses) + return; + + if (!actuator) { + hid_warn(msc->hdev, + "no actuator for host click feedback; using the firmware's\n"); + haptic_press = false; + return; + } + + if (msc->haptics->mode != HID_HAPTIC_MODE_HOST) { + ret = apple_taptic_switch_modes(actuator, HID_HAPTIC_MODE_HOST); + if (ret) { + hid_warn(msc->hdev, + "cannot take host-controlled mode (%d); using the firmware's\n", + ret); + haptic_press = false; + return; + } + msc->haptics->mode = HID_HAPTIC_MODE_HOST; + hid_info(msc->hdev, "host-driven click feedback active\n"); + } + + strength = magicmouse_haptic_strength(msc->hdev, false); + softness = magicmouse_haptic_softness(msc->hdev, false); + + for (i = 0; i < pulses; i++) { + ret = apple_taptic_send(actuator, usage, strength, softness); + if (ret) { + hid_warn(msc->hdev, + "click waveform failed (%d); returning click feedback to the firmware\n", + ret); + haptic_press = false; + magicmouse_haptic_release_mode(msc); + return; + } + hid_dbg(msc->hdev, "played %s pulse %u/%u at strength 0x%02x\n", + usage == (HID_HP_WAVEFORMRELEASE & HID_USAGE) ? + "release" : "press", + i + 1, pulses, strength); + if (i + 1 < pulses) + msleep(haptic_pulse_gap_ms); + } +} + +/* + * Hand click feedback back to the actuator's firmware, unless userspace still + * has effects uploaded and so still wants host-controlled mode. + */ +static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc) +{ + struct hid_device *actuator; + unsigned int i; + + if (msc->haptics->mode != HID_HAPTIC_MODE_HOST) + return; + + if (msc->haptic_effects) { + for (i = 0; i < FF_MAX_EFFECTS; i++) + if (msc->haptic_effects[i].effect_type) + return; + } + + actuator = magicmouse_get_actuator(msc); + if (actuator && !apple_taptic_switch_modes(actuator, + HID_HAPTIC_MODE_DEVICE)) + msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; +} + +static void magicmouse_press_worker(struct work_struct *ws) +{ + struct magicmouse_sc *msc = + container_of(ws, struct magicmouse_sc, haptic_press_work); + + if (!haptic_press) { + magicmouse_haptic_release_mode(msc); + return; + } + + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMPRESS & HID_USAGE, + haptic_press_pulses); +} + +static void magicmouse_release_worker(struct work_struct *ws) +{ + struct magicmouse_sc *msc = + container_of(ws, struct magicmouse_sc, haptic_release_work); + + if (!haptic_press) { + magicmouse_haptic_release_mode(msc); + return; + } + + /* + * The bare header tells the firmware that the click gesture is over, so + * send it even when no extra pulses were asked for. + */ + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMRELEASE & HID_USAGE, + max_t(u8, haptic_release_pulses, 1)); +} + +/* + * Decide whether the pad is clicked from the summed finger pressure, and + * queue the feedback for it: in host-controlled mode the firmware no longer + * sets the button itself. The wide gap between the press and release + * thresholds keeps a single click from chattering. + * + * Returns the button state to report. Playing the waveform sleeps, so it is + * left to the workqueue. + */ +static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, + bool firmware_button) +{ + if (!msc->haptics || !msc->haptics->wq) + return firmware_button; + + if (!haptic_press) { + /* Hand the actuator back and report its button again. */ + if (msc->haptics->mode == HID_HAPTIC_MODE_HOST) { + msc->haptic_button_down = false; + queue_work(msc->haptics->wq, &msc->haptic_release_work); + } + return firmware_button; + } + + if (!msc->haptic_button_down && pressure >= haptic_press_threshold) { + msc->haptic_button_down = true; + hid_dbg(msc->hdev, "click at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_press_work); + } else if (msc->haptic_button_down && + pressure <= haptic_release_threshold) { + msc->haptic_button_down = false; + hid_dbg(msc->hdev, "release at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_release_work); + } + + return msc->haptic_button_down; +} +#else +static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, + bool firmware_button) +{ + return firmware_button; +} #endif static int magicmouse_input_configured(struct hid_device *hdev, @@ -1506,10 +1718,8 @@ static bool magicmouse_effect_is_deep(const struct ff_effect *effect) (APPLE_HP_WAVEFORMDEEPCLICK & HID_USAGE); } -static u8 magicmouse_haptic_softness(struct hid_device *hdev, - const struct ff_effect *effect) +static u8 magicmouse_haptic_softness(struct hid_device *hdev, bool deep) { - bool deep = magicmouse_effect_is_deep(effect); u8 configured = deep ? haptic_deep_softness : haptic_softness; if (configured) @@ -1520,10 +1730,8 @@ static u8 magicmouse_haptic_softness(struct hid_device *hdev, return deep ? 0xd0 : 0xf0; } -static u8 magicmouse_haptic_strength(struct hid_device *hdev, - const struct ff_effect *effect) +static u8 magicmouse_haptic_strength(struct hid_device *hdev, bool deep) { - bool deep = magicmouse_effect_is_deep(effect); u8 configured = deep ? haptic_deep_strength : haptic_strength; if (configured) @@ -1560,9 +1768,11 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, msc->haptic_effects[effect->id].effect_type = (effect->u.haptic.hid_usage) & HID_USAGE; msc->haptic_effects[effect->id].strength = (min(100, (effect->u.haptic.intensity)) * - magicmouse_haptic_strength(trackpad_hdev, effect)) / 100; + magicmouse_haptic_strength(trackpad_hdev, + magicmouse_effect_is_deep(effect))) / 100; msc->haptic_effects[effect->id].softness = - magicmouse_haptic_softness(trackpad_hdev, effect); + magicmouse_haptic_softness(trackpad_hdev, + magicmouse_effect_is_deep(effect)); /* The actuator may have been rebound or reset since the last upload. */ ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_HOST); @@ -1685,6 +1895,9 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, for (i = 0; i < FF_MAX_EFFECTS; i++) INIT_WORK(&msc->haptic_effects[i].work, haptic_playback_worker); + INIT_WORK(&msc->haptic_press_work, magicmouse_press_worker); + INIT_WORK(&msc->haptic_release_work, magicmouse_release_worker); + /* FF_HAPTIC has to be in ffbit before the FF device is created. */ input_set_capability(msc->input, EV_FF, FF_HAPTIC); @@ -1760,6 +1973,9 @@ static void magicmouse_teardown_haptics(struct magicmouse_sc *msc) if (!msc->haptics) return; + cancel_work_sync(&msc->haptic_press_work); + cancel_work_sync(&msc->haptic_release_work); + /* * Tear the force feedback device down while msc is still valid: an * open evdev fd would otherwise run it from input_dev_release() after From 34c11733ccbbe03db1d35829649e8e0e42606042 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Wed, 16 Sep 2026 22:25:12 +0200 Subject: [PATCH 0107/1579] HID: magicmouse: add a deep click The pad reports finger pressure well past the point where a click is asserted, so a second, firmer press inside one click can be told apart from an ordinary one. That is what Force Touch does, and the hardware already carries the signal. Fire once per click at haptic_deep_threshold, with its own pulse count, gap and waveform shape so it can be made to feel distinct from the first click, and clear the flag only on release so holding at depth does not repeat it. Linux has no key for a force click, so haptic_deep_keycode names one and nothing is reported unless it is set; the capability has to be advertised at probe, so that parameter is read-only. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 75 ++++++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 7 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index b7049a888e023..c490f4fcbd363 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -109,6 +109,32 @@ static u16 haptic_release_threshold = 75; module_param(haptic_release_threshold, ushort, 0644); MODULE_PARM_DESC(haptic_release_threshold, "summed finger pressure at which a click ends (default 75)"); +/* + * A deep click is a second, firmer press within one click, the way Force Touch + * behaves. On a J313 an ordinary click peaks around 250 and a firm press + * reaches past 1000, so the threshold sits well clear of the first one. + */ +static u16 haptic_deep_threshold = 500; +module_param(haptic_deep_threshold, ushort, 0644); +MODULE_PARM_DESC(haptic_deep_threshold, "summed finger pressure for a deep click, 0 disables (default 500)"); + +static u8 haptic_deep_pulses = 1; +module_param(haptic_deep_pulses, byte, 0644); +MODULE_PARM_DESC(haptic_deep_pulses, "pulses played for a deep click (default 1)"); + +static u8 haptic_deep_gap_ms = 15; +module_param(haptic_deep_gap_ms, byte, 0644); +MODULE_PARM_DESC(haptic_deep_gap_ms, "milliseconds between deep-click pulses, 0 to use haptic_pulse_gap_ms (default 15)"); + +/* + * Linux has no key for a force click, so which one to send is the user's + * choice; nothing is sent unless one is named. It is read at probe because the + * capability has to be advertised when the input device is set up. + */ +static u16 haptic_deep_keycode; +module_param(haptic_deep_keycode, ushort, 0444); +MODULE_PARM_DESC(haptic_deep_keycode, "key code reported on a deep click, 0 for none (default 0)"); + static u8 haptic_pulse_gap_ms = 25; module_param(haptic_pulse_gap_ms, byte, 0644); MODULE_PARM_DESC(haptic_pulse_gap_ms, "milliseconds between pulses when more than one (default 25)"); @@ -255,7 +281,9 @@ struct magicmouse_sc { struct effect_job *haptic_effects; struct work_struct haptic_press_work; struct work_struct haptic_release_work; + struct work_struct haptic_deep_work; bool haptic_button_down; + bool haptic_deep_fired; struct hid_device *hdev; struct delayed_work work; @@ -1403,10 +1431,10 @@ static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc); * actuator back to the firmware and turn the feature off instead. */ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, - u8 pulses) + bool deep, u8 pulses) { struct hid_device *actuator = magicmouse_get_actuator(msc); - u8 strength, softness; + u8 strength, softness, gap; unsigned int i; int ret; @@ -1433,8 +1461,10 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, hid_info(msc->hdev, "host-driven click feedback active\n"); } - strength = magicmouse_haptic_strength(msc->hdev, false); - softness = magicmouse_haptic_softness(msc->hdev, false); + strength = magicmouse_haptic_strength(msc->hdev, deep); + softness = magicmouse_haptic_softness(msc->hdev, deep); + gap = (deep && haptic_deep_gap_ms) ? haptic_deep_gap_ms + : haptic_pulse_gap_ms; for (i = 0; i < pulses; i++) { ret = apple_taptic_send(actuator, usage, strength, softness); @@ -1451,7 +1481,7 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, "release" : "press", i + 1, pulses, strength); if (i + 1 < pulses) - msleep(haptic_pulse_gap_ms); + msleep(gap); } } @@ -1489,10 +1519,22 @@ static void magicmouse_press_worker(struct work_struct *ws) return; } - magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMPRESS & HID_USAGE, + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMPRESS & HID_USAGE, false, haptic_press_pulses); } +static void magicmouse_deep_worker(struct work_struct *ws) +{ + struct magicmouse_sc *msc = + container_of(ws, struct magicmouse_sc, haptic_deep_work); + + if (!haptic_press) + return; + + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMPRESS & HID_USAGE, true, + haptic_deep_pulses); +} + static void magicmouse_release_worker(struct work_struct *ws) { struct magicmouse_sc *msc = @@ -1507,7 +1549,7 @@ static void magicmouse_release_worker(struct work_struct *ws) * The bare header tells the firmware that the click gesture is over, so * send it even when no extra pulses were asked for. */ - magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMRELEASE & HID_USAGE, + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMRELEASE & HID_USAGE, false, max_t(u8, haptic_release_pulses, 1)); } @@ -1537,13 +1579,27 @@ static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, if (!msc->haptic_button_down && pressure >= haptic_press_threshold) { msc->haptic_button_down = true; + msc->haptic_deep_fired = false; hid_dbg(msc->hdev, "click at pressure %u\n", pressure); queue_work(msc->haptics->wq, &msc->haptic_press_work); } else if (msc->haptic_button_down && pressure <= haptic_release_threshold) { msc->haptic_button_down = false; + msc->haptic_deep_fired = false; hid_dbg(msc->hdev, "release at pressure %u\n", pressure); queue_work(msc->haptics->wq, &msc->haptic_release_work); + } else if (msc->haptic_button_down && !msc->haptic_deep_fired && + haptic_deep_threshold && + pressure >= haptic_deep_threshold) { + /* Pressing harder without lifting: once per click, not while held. */ + msc->haptic_deep_fired = true; + hid_dbg(msc->hdev, "deep click at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_deep_work); + + if (haptic_deep_keycode) { + input_report_key(msc->input, haptic_deep_keycode, 1); + input_report_key(msc->input, haptic_deep_keycode, 0); + } } return msc->haptic_button_down; @@ -1897,6 +1953,10 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, INIT_WORK(&msc->haptic_press_work, magicmouse_press_worker); INIT_WORK(&msc->haptic_release_work, magicmouse_release_worker); + INIT_WORK(&msc->haptic_deep_work, magicmouse_deep_worker); + + if (haptic_deep_keycode) + input_set_capability(msc->input, EV_KEY, haptic_deep_keycode); /* FF_HAPTIC has to be in ffbit before the FF device is created. */ input_set_capability(msc->input, EV_FF, FF_HAPTIC); @@ -1975,6 +2035,7 @@ static void magicmouse_teardown_haptics(struct magicmouse_sc *msc) cancel_work_sync(&msc->haptic_press_work); cancel_work_sync(&msc->haptic_release_work); + cancel_work_sync(&msc->haptic_deep_work); /* * Tear the force feedback device down while msc is still valid: an From 62db5a8a79d024f553d5bba57ace9ce858f432e0 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Thu, 17 Sep 2026 09:39:53 +0200 Subject: [PATCH 0108/1579] HID: magicmouse: derive the click while the host holds the actuator The firmware reports the button only while it owns the actuator. The derived click covered haptic_press, but a force-feedback effect uploaded by userspace also takes host-controlled mode, and the mode cannot be released while an effect is loaded. In that state the driver reported the firmware's button, which never changes again, so the pad stopped clicking until every effect was erased. Derive the button whenever the actuator is in host-controlled mode, whatever put it there, and keep asking for it back when the waveform is not ours to play. Found by a review bot on the pull request. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index c490f4fcbd363..8ac997a743538 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -1565,18 +1565,31 @@ static void magicmouse_release_worker(struct work_struct *ws) static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, bool firmware_button) { + bool host_mode; + if (!msc->haptics || !msc->haptics->wq) return firmware_button; - if (!haptic_press) { - /* Hand the actuator back and report its button again. */ - if (msc->haptics->mode == HID_HAPTIC_MODE_HOST) { - msc->haptic_button_down = false; - queue_work(msc->haptics->wq, &msc->haptic_release_work); - } + host_mode = msc->haptics->mode == HID_HAPTIC_MODE_HOST; + + if (!haptic_press && !host_mode) { + /* The firmware owns the actuator and reports the button. */ + msc->haptic_button_down = false; + msc->haptic_deep_fired = false; return firmware_button; } + /* + * Host-controlled mode without host-driven feedback means userspace + * uploaded an effect and owns the waveform. Ask for the actuator back, + * but keep deriving the button until it is: the firmware reports no + * button while it does not hold the actuator, and the mode cannot be + * released while an effect is loaded, so trusting it here would leave + * the pad unable to click until every effect is erased. + */ + if (!haptic_press) + queue_work(msc->haptics->wq, &msc->haptic_release_work); + if (!msc->haptic_button_down && pressure >= haptic_press_threshold) { msc->haptic_button_down = true; msc->haptic_deep_fired = false; From 6c53c26c6dd8c6c97b005157a1ed2ea34df99cf2 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Thu, 17 Sep 2026 09:39:57 +0200 Subject: [PATCH 0109/1579] HID: magicmouse: forget the actuator's mode across system sleep The actuator leaves system sleep in device-controlled mode. The transport resumes the keyboard and the trackpad, not the actuator, and nothing tells the driver the mode changed, so msc->haptics->mode still reads host-controlled. The next press then skips the switch and plays a pulse while the firmware plays its own, which is felt as two clicks per press. On a J313 across an s2idle cycle the driver says nothing between 'host-driven click feedback active' before the suspend and the first press after it: it never re-took the mode because it believed it still held it. Drop the cached mode in reset_resume, which the transport does call for the trackpad, and drop the click state with it. The next press takes host-controlled mode again if it needs it. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-magicmouse.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 8ac997a743538..e67e4ce60ab92 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -1617,12 +1617,34 @@ static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, return msc->haptic_button_down; } + +/* + * The actuator leaves system sleep in device-controlled mode whatever it was + * in before: the transport resumes the keyboard and the trackpad, not the + * actuator, and nothing reports the change. A cached host-controlled mode then + * makes the next press skip the switch and play a pulse while the firmware is + * playing its own, which is felt as two clicks. Forget it instead; the next + * press takes the mode again if it needs it. + */ +static void magicmouse_haptic_forget_mode(struct magicmouse_sc *msc) +{ + if (!msc->haptics) + return; + + msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + msc->haptic_button_down = false; + msc->haptic_deep_fired = false; +} #else static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, bool firmware_button) { return firmware_button; } + +static void magicmouse_haptic_forget_mode(struct magicmouse_sc *msc) +{ +} #endif static int magicmouse_input_configured(struct hid_device *hdev, @@ -2275,6 +2297,9 @@ static int magicmouse_reset_resume(struct hid_device *hdev) if (msc && (hdev->type == HID_TYPE_USBMOUSE || hdev->bus == BUS_SPI)) schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + if (msc) + magicmouse_haptic_forget_mode(msc); + return 0; } #endif From 507f2c902891a10db8f5fd90b02f57f89cd5d429 Mon Sep 17 00:00:00 2001 From: Chromatischer Date: Thu, 17 Sep 2026 10:06:10 +0200 Subject: [PATCH 0110/1579] HID: apple-mtp-haptic: put the actuator back in device-controlled mode The actuator keeps its mode across a warm reboot. Every user of it starts out believing it is device-controlled, so one left in host-controlled mode by a previous boot is silent: the firmware plays nothing because it no longer owns the actuator, and the host that did is gone. The trackpad then has no click feedback at all, with nothing in the log to say why. Only a cold boot clears it. Assert device-controlled mode at probe instead of inheriting whatever the last boot left. The switch is not fatal if it fails, since that leaves the actuator exactly where it already was. Assisted-by: Claude:claude-opus-5 Assisted-by: DeepSeek:deepseek-v4.1 Signed-off-by: Chromatischer --- drivers/hid/hid-apple-mtp-haptic.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/hid/hid-apple-mtp-haptic.c b/drivers/hid/hid-apple-mtp-haptic.c index 0678953acbb40..42ab019f7a27a 100644 --- a/drivers/hid/hid-apple-mtp-haptic.c +++ b/drivers/hid/hid-apple-mtp-haptic.c @@ -72,6 +72,21 @@ static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid return ret; } + /* + * The actuator keeps its mode across a warm reboot, and every user of + * it starts out believing it is device-controlled. One left in + * host-controlled mode by a previous boot is silent: its firmware plays + * nothing, and the host that was driving it is gone. Assert the state + * the drivers assume rather than inheriting the last one. A failure + * here is not fatal; it only means the actuator keeps whatever mode it + * had, which is what would have happened anyway. + */ + ret = apple_taptic_switch_modes(haptic_hdev, false); + if (ret) + hid_warn(haptic_hdev, + "cannot return the actuator to device-controlled mode (%d)\n", + ret); + return 0; } From f026099d3531eb31293fc397a1aa00783e277f63 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 10:58:16 -0700 Subject: [PATCH 0111/1579] drm/apple: enable dual dock DP tunnels on J416c J416c (16" M2 Max, T6021) shares the T602x ATC PHY, display crossbar, USB4 NHI and the dcpext0/dcpext1 layout with J414s; only the board gates differ. Treat J416c like J414s: two connectors per Type-C port, DPIN0 -> dcpext0 and DPIN1 -> dcpext1, direct DP-alt displays prefer dcpext1 so dcpext0 stays free for a dock tunnel, and allow the ATC tunnel clock. Tested on J416c with a CalDigit USB-C Pro Dock (Thunderbolt 3): two 3840x2160@60 displays through the dock plus the internal panel at 120 Hz, on hotplug and with the dock connected at boot. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan --- drivers/gpu/drm/apple/apple_drv.c | 3 ++- drivers/gpu/drm/apple/dcp.c | 6 ++++-- drivers/phy/apple/atc.c | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 4fcfac0f695d9..db438b53187c8 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -438,7 +438,8 @@ static int apple_probe_typec_ports(struct drm_device *drm, * append one more connector per port for a second USB4 DP tunnel. */ for (idx = 0; idx < nr_ports * - (of_machine_is_compatible("apple,j414s") ? 2 : 1); idx++) { + ((of_machine_is_compatible("apple,j414s") || + of_machine_is_compatible("apple,j416c")) ? 2 : 1); idx++) { struct apple_connector *connector; struct apple_encoder *enc; unsigned int port_idx = idx % nr_ports; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 9578e688b0f1c..b41cd17dec2cd 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -607,7 +607,8 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, * streams through one dock. Retain J416s' existing preference * until its second pipeline has been validated separately. */ - if (of_machine_is_compatible("apple,j414s")) { + if (of_machine_is_compatible("apple,j414s") || + of_machine_is_compatible("apple,j416c")) { if ((dpin == 0 && !candidate->dcp->fixed_phy) || (dpin == 1 && candidate->dcp->fixed_phy)) score += 100; @@ -781,7 +782,8 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, * tunnel when a direct DP-alt-mode display can use * the Type-C-only dcpext1 pipeline. */ - if (of_machine_is_compatible("apple,j414s") && + if ((of_machine_is_compatible("apple,j414s") || + of_machine_is_compatible("apple,j416c")) && candidate->dcp->fixed_phy) score += 100; if (score < best_score) { diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 9e2846a71768f..4b3d4d69ab728 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2370,7 +2370,8 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) if (!phy || phy->ops != &apple_atc_dp_phy_ops || (!of_machine_is_compatible("apple,j414s") && - !of_machine_is_compatible("apple,j416s"))) + !of_machine_is_compatible("apple,j416s") && + !of_machine_is_compatible("apple,j416c"))) return -EOPNOTSUPP; atcphy = phy_get_drvdata(phy); if (!of_device_is_compatible(atcphy->np, "apple,t6020-atcphy") || From f193ed228f9691273275cddc7ba1ee87a9dd9701 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 10:58:17 -0700 Subject: [PATCH 0112/1579] drm/apple: expose Type-C display connectors as DisplayPort The per-port Type-C connectors carry DisplayPort (DP alt mode or a Thunderbolt/USB4 DP tunnel), but were registered as DRM_MODE_CONNECTOR_USB, which DRM reserves for USB display adapters (gud, appletbdrm). Userspace keys on the type: systemd-logind only counts VGA/DVI/DP/HDMI/TV connectors as external displays, so a docked lid close suspended the machine instead of honoring HandleLidSwitchDocked=ignore. Register them as DisplayPort. The driver's internal "is Type-C" checks use dcp->connector_type / fixed_connector_type and are unaffected. Connectors are now named DP-N instead of USB-N. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan --- drivers/gpu/drm/apple/apple_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index db438b53187c8..ec98812a28062 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -456,7 +456,7 @@ static int apple_probe_typec_ports(struct drm_device *drm, ret = drm_connector_init(drm, &connector->base, &apple_connector_funcs, - DRM_MODE_CONNECTOR_USB); + DRM_MODE_CONNECTOR_DisplayPort); if (ret) { kfree(connector); return ret; From 6f5f4f76a7519d6910b0b9f4bc6f6dc09406d815 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 13:03:08 -0700 Subject: [PATCH 0113/1579] dt-bindings: pci: apple,pcie: Add apple,pciec-kernel-init PCIe-C root ports behind a USB4 router must be cold-initialized before use. Today the kernel only accepts ports the bootloader initialized and reported through "apple,pciec-preinit-status", which no m1n1 does. Add a boolean for PCIe-C nodes that tells the OS to do the cold init itself when the USB4 PCIe tunnel comes up. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan --- Documentation/devicetree/bindings/pci/apple,pcie.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 6a6608dca4d1b..ced76a1dc9894 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -86,6 +86,14 @@ properties: dma-coherent: true + apple,pciec-kernel-init: + type: boolean + description: + PCIe-C only. The operating system cold-initializes the port itself + when the USB4 PCIe tunnel comes up, instead of relying on the + bootloader to have done so. Set on machines where the bootloader + does not initialize PCIe-C. + patternProperties: "^pci@": $ref: /schemas/pci/pci-bus.yaml# @@ -195,6 +203,7 @@ allOf: - const: phy3 ranges: maxItems: 2 + apple,pciec-kernel-init: false patternProperties: "^pci@": required: From a3dca5d9848c0c4d72aa6ba750ef97cfb59c247b Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 13:03:09 -0700 Subject: [PATCH 0114/1579] PCI: apple: Cold-init PCIe-C tunnel ports without an m1n1 handoff PCIe-C tunnel support requires m1n1 to cold-initialize the port and report "apple,pciec-preinit-status" = 1, but no m1n1 does so. On J414/J416 the tunnel is therefore always disabled and PCIe devices behind Thunderbolt 3 docks (whose USB is an xHCI behind the tunnel) never appear. m1n1 already copies the PCIe-C tunables into the DT and the driver has the port enable sequence. For ports marked "apple,pciec-kernel-init", let the kernel do the cold init, as aurora-silicon/linux#8 does for t8103: - apple_pcie_tunnel_cold_init() follows the enablePortHardware() baseline of apple_pcie_tunnel_start(): debug/fabric tunables, port reset table and tunables, PERST off and APPCLK on, wait for RUN, and only then the root-complex (config space) tunables. Touching config space while APPCLK is still off hard-resets a J416c. - apple_pcie_tunnel_prepare() runs it on transient mappings before the tunnel children are populated, since the DART never completes an invalidation while APPCLK is gated. A port that is already running is left alone. - The Thunderbolt glue and the host bridge accept kernel-init ports in place of the m1n1 handoff. As the sequence is new on T602x, it is gated behind pcie_apple.tunnel_kernel_init (default off), checked before the first PCIe-C MMIO access. pcie_apple.tunnel_kernel_init_trace_ms logs each step and pauses after it, for locating failures with a persistent log. Tested on J416c with a CalDigit USB-C Pro Dock (Thunderbolt 3, Titan Ridge xHCI 8086:15f0): the dock's hubs, keyboard, audio, SD reader, webcam and Ethernet work, including after unplug and replug. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan --- drivers/pci/controller/pcie-apple.c | 263 +++++++++++++++++++++++++++- drivers/thunderbolt/apple.c | 51 +++++- include/linux/pci-apple.h | 5 + 3 files changed, 303 insertions(+), 16 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 67f3eeac2e0d9..b1703965cc1cf 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -41,6 +42,22 @@ static int link_up_timeout = 500; module_param(link_up_timeout, int, 0644); MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); +static bool tunnel_kernel_init; +module_param(tunnel_kernel_init, bool, 0644); +MODULE_PARM_DESC(tunnel_kernel_init, + "Cold-initialize apple,pciec-kernel-init PCIe-C ports (experimental, default off)"); + +static unsigned int tunnel_kernel_init_trace_ms; +module_param(tunnel_kernel_init_trace_ms, uint, 0644); +MODULE_PARM_DESC(tunnel_kernel_init_trace_ms, + "Log each PCIe-C cold init step and sleep this long after it (debug)"); + +bool apple_pcie_tunnel_kernel_init_enabled(void) +{ + return READ_ONCE(tunnel_kernel_init); +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_kernel_init_enabled); + /* T8103 (original M1) and related SoCs */ #define CORE_RC_PHYIF_CTL 0x00024 #define CORE_RC_PHYIF_CTL_RUN BIT(0) @@ -777,6 +794,78 @@ static int apple_pcie_tunnel_reinitialize(struct apple_pcie_port *port) return 0; } +static void apple_pcie_tunnel_cold_init_step(struct apple_pcie_port *port, + const char *step) +{ + unsigned int ms = READ_ONCE(tunnel_kernel_init_trace_ms); + + if (!ms) + return; + dev_info(port->pcie->dev, "PCIe-C %pOF cold init: %s\n", port->np, step); + msleep(ms); +} + +/* + * Cold-initialize a PCIe-C port that no m1n1 handoff brought up. This + * follows the enablePortHardware() baseline of apple_pcie_tunnel_start() and + * the t8103 cold init of aurora-silicon/linux#8: the port leaves reset with + * APPCLK running and reports RUN before anything touches the root port's + * config space. The downstream (tunnel) reset stays asserted until + * apple_pcie_setup_port() releases it before link training. + */ +static int apple_pcie_tunnel_cold_init(struct apple_pcie_port *port) +{ + struct apple_pcie *pcie = port->pcie; + u32 stat; + int i, ret; + + apple_pcie_tunnel_cold_init_step(port, "debug and fabric tunables"); + apple_pcie_tunnel_apply_tunable(pcie->debug_base, pcie->debug_tunable); + apple_pcie_tunnel_apply_tunable(pcie->fabric_base, + pcie->fabric_tunable); + apple_pcie_tunnel_cold_init_step(port, "port reset table"); + apple_pcie_tunnel_reset_hardware(port); + apple_pcie_tunnel_cold_init_step(port, "port tunables"); + apple_pcie_tunnel_apply_tunable(port->base, port->tunable); + + apple_pcie_tunnel_cold_init_step(port, "PERST off, APPCLK on"); + apple_pcie_port_rmw_set(port, PORT_PERST_OFF, pcie->hw->port_perst); + apple_pcie_port_rmw_set(port, PORT_APPCLK_EN, PORT_APPCLK); + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_STATUS_READY, + 10, 250000, false, port, PORT_STATUS); + if (ret) + return dev_err_probe(pcie->dev, ret, + "PCIe-C %pOF cold init: port did not enter RUN (status %#x)\n", + port->np, stat); + + apple_pcie_tunnel_cold_init_step(port, "root complex tunables"); + apple_pcie_tunnel_apply_tunable(pcie->cfg->win, pcie->rc_tunable); + apple_pcie_tunnel_cold_init_step(port, "APPCLK ungate, counters, Intr2AXI"); + apple_pcie_port_rmw_clear(port, PORT_APPCLK_CGDIS, PORT_APPCLK); + apple_pcie_port_writel(port, PORT_COUNTER_ENABLE, PORT_COUNTER_CTRL); + apple_pcie_port_writel(port, ~0, PORT_INTSTAT); + apple_pcie_port_writel(port, ~0, PORT_LINKCMDSTS); + apple_pcie_tunnel_writel(pcie->intr2axi_base, PCIEC_INTR2AXI_ENABLE, + PCIEC_INTR2AXI_CTRL); + + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + !(stat & PORT_LINKSTS_BUSY), + 10, 250000, false, port, PORT_LINKSTS); + if (ret) + return dev_err_probe(pcie->dev, ret, + "PCIe-C %pOF cold init: link still busy (status %#x)\n", + port->np, stat); + + for (i = 0; i < pcie->hw->max_rid2sid; i++) + apple_pcie_rid2sid_write(port, i, 0); + + dev_info(pcie->dev, "PCIe-C %pOF cold init done, status %#x tunstat %#x\n", + port->np, apple_pcie_port_readl(port, PORT_STATUS), + apple_pcie_port_readl(port, PORT_TUNSTAT)); + return 0; +} + static void apple_pcie_tunnel_restore_irq_hw(struct apple_pcie_port *port) { struct apple_pcie *pcie = port->pcie; @@ -1230,18 +1319,23 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, } } if (pcie->hw->tunneled) { + bool kernel_init = of_property_read_bool(pcie->dev->of_node, + "apple,pciec-kernel-init"); + /* - * m1n1 owns PCIe-C cold initialization. Replaying the port reset or - * tunnel-reset handshake against that live handoff raises an - * asynchronous SError on T6020. Accept only an explicitly successful - * handoff and begin with the read-only RUN state check used by m1n1. + * PCIe-C must come out of reset either through an m1n1 handoff + * or, for "apple,pciec-kernel-init" ports, through + * apple_pcie_tunnel_prepare() before the tunnel was populated. + * Replaying the port reset or tunnel-reset handshake against a + * live handoff raises an asynchronous SError on T6020, so begin + * with the read-only RUN state check used by m1n1. */ ret = of_property_read_u32(pcie->dev->of_node, "apple,pciec-preinit-status", &preinit_status); - if (ret || preinit_status != 1) { + if (!kernel_init && (ret || preinit_status != 1)) { dev_err(pcie->dev, - "PCIe-C requires a successful m1n1 preinit handoff\n"); + "PCIe-C needs an m1n1 preinit handoff or apple,pciec-kernel-init\n"); ret = -ENODEV; goto err_teardown; } @@ -1250,7 +1344,17 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, stat & PORT_STATUS_READY, 100, 250000, false, port, PORT_STATUS); - if (ret < 0) { + if (ret < 0 && kernel_init) { + if (!apple_pcie_tunnel_kernel_init_enabled()) { + dev_err(pcie->dev, + "PCIe-C kernel init disabled (pcie_apple.tunnel_kernel_init=0)\n"); + ret = -EPERM; + goto err_teardown; + } + ret = apple_pcie_tunnel_cold_init(port); + if (ret) + goto err_teardown; + } else if (ret < 0) { dev_info(pcie->dev, "port %pOF stopped; replaying PCIe-C hardware initialization\n", np); @@ -1999,6 +2103,151 @@ static const struct of_device_id apple_pcie_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_pcie_of_match); +struct apple_pcie_tunnel_map { + struct resource res; + void __iomem *base; +}; + +static int apple_pcie_tunnel_map_named(struct device_node *np, + const char *name, + struct apple_pcie_tunnel_map *map) +{ + int index, ret; + + index = of_property_match_string(np, "reg-names", name); + if (index < 0) + return index; + ret = of_address_to_resource(np, index, &map->res); + if (ret) + return ret; + /* Tunneled PCIe-C MMIO is non-posted, as in the probe path. */ + map->base = ioremap_np(map->res.start, resource_size(&map->res)); + return map->base ? 0 : -ENOMEM; +} + +static void apple_pcie_tunnel_unmap(struct apple_pcie_tunnel_map *map) +{ + if (map->base) + iounmap(map->base); +} + +/* + * Cold-initialize an "apple,pciec-kernel-init" PCIe-C port before the tunnel + * children are populated. The DART is the first child and probes as soon as + * it is created, and its command engine never completes an invalidation + * while the port's APPCLK is gated. A port that is already running (m1n1 + * handoff, or retained from an earlier connection) is left alone. + * + * There is no host bridge yet, so build a transient one around temporary + * mappings and reuse the port enable sequence. + */ +int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) +{ + struct apple_pcie_tunnel_map config = {}, debug = {}, fabric = {}, + intr2axi = {}, portmap = {}; + struct device_node *np = NULL, *port_np = NULL; + const struct of_device_id *match; + struct pci_config_window cfg = {}; + struct apple_pcie_port port = {}; + struct apple_pcie pcie = {}; + void *group = NULL; + u32 stat; + int ret; + + for_each_available_child_of_node(tunnel, np) + if (of_property_read_bool(np, "apple,pciec-kernel-init")) + break; + if (!np) + return 0; + + if (!apple_pcie_tunnel_kernel_init_enabled()) { + dev_err(dev, "PCIe-C %pOF: kernel init disabled (pcie_apple.tunnel_kernel_init=0)\n", + np); + ret = -EPERM; + goto out; + } + + match = of_match_node(apple_pcie_of_match, np); + port_np = of_get_next_available_child(np, NULL); + if (!match || !((const struct hw_info *)match->data)->tunneled || + !port_np) { + ret = -ENODEV; + goto out; + } + + ret = apple_pcie_tunnel_map_named(np, "port0", &portmap); + if (ret) + goto out; + + pcie.dev = dev; + pcie.hw = match->data; + port.pcie = &pcie; + port.np = port_np; + port.base = portmap.base; + + apple_pcie_tunnel_cold_init_step(&port, "read port status"); + stat = apple_pcie_port_readl(&port, PORT_STATUS); + if (stat & PORT_STATUS_READY) { + dev_info(dev, "PCIe-C port %pOF already running\n", port_np); + goto out; + } + + ret = apple_pcie_tunnel_map_named(np, "config", &config); + if (!ret) + ret = apple_pcie_tunnel_map_named(np, "debug", &debug); + if (!ret) + ret = apple_pcie_tunnel_map_named(np, "fabric", &fabric); + if (!ret) + ret = apple_pcie_tunnel_map_named(np, "intr2axi", &intr2axi); + if (ret) + goto out; + + group = devres_open_group(dev, NULL, GFP_KERNEL); + if (!group) { + ret = -ENOMEM; + goto out; + } + pcie.rc_tunable = devm_apple_tunable_parse(dev, np, "apple,tunable-rc", + &config.res); + pcie.debug_tunable = devm_apple_tunable_parse(dev, np, + "apple,tunable-debug", + &debug.res); + pcie.fabric_tunable = devm_apple_tunable_parse(dev, np, + "apple,tunable-fabric", + &fabric.res); + port.tunable = devm_apple_tunable_parse(dev, port_np, "apple,tunable", + &portmap.res); + if (IS_ERR(pcie.rc_tunable) || IS_ERR(pcie.debug_tunable) || + IS_ERR(pcie.fabric_tunable) || IS_ERR(port.tunable)) { + ret = dev_err_probe(dev, -ENODEV, + "PCIe-C %pOF tunables unavailable\n", np); + goto out; + } + + cfg.win = config.base; + pcie.cfg = &cfg; + pcie.debug_base = debug.base; + pcie.fabric_base = fabric.base; + pcie.intr2axi_base = intr2axi.base; + + dev_info(dev, "PCIe-C port %pOF cold init (status %#x)\n", + port_np, stat); + ret = apple_pcie_tunnel_cold_init(&port); + +out: + if (group) + devres_release_group(dev, group); + apple_pcie_tunnel_unmap(&intr2axi); + apple_pcie_tunnel_unmap(&fabric); + apple_pcie_tunnel_unmap(&debug); + apple_pcie_tunnel_unmap(&config); + apple_pcie_tunnel_unmap(&portmap); + of_node_put(port_np); + of_node_put(np); + return ret; +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_prepare); + static struct platform_driver apple_pcie_driver = { .probe = apple_pcie_probe, .remove = apple_pcie_remove, diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index fbe5930e9cfce..43c595d79f088 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -142,6 +142,7 @@ struct apple_cio { struct device_node *np; struct device_node *pcie_tunnel_np; bool pcie_tunnel_preinitialized; + bool pcie_tunnel_kernel_init; void __iomem *pcie_intr2axi_base; struct apple_rtkit *rtk; @@ -196,6 +197,15 @@ static void apple_cio_of_node_put(void *data) of_node_put(data); } +static bool apple_cio_pcie_tunnel_kernel_init(struct device_node *parent) +{ + for_each_available_child_of_node_scoped(parent, child) + if (of_property_read_bool(child, "apple,pciec-kernel-init")) + return true; + + return false; +} + static bool apple_cio_pcie_tunnel_is_preinitialized(struct device_node *parent) { @@ -206,6 +216,10 @@ apple_cio_pcie_tunnel_is_preinitialized(struct device_node *parent) !of_device_is_compatible(child, "apple,t6000-pciec")) continue; + /* The PCIe driver cold-initializes these ports itself. */ + if (of_property_read_bool(child, "apple,pciec-kernel-init")) + return true; + return !of_property_read_u32(child, "apple,pciec-preinit-status", &status) && status == 1; @@ -541,12 +555,21 @@ static struct platform_device *apple_cio_find_pcie_tunnel(struct apple_cio *acio static int apple_cio_populate_pcie_tunnel(struct apple_cio *acio) { + int ret; + + /* Experimental: don't touch a cold PCIe-C port unless opted in. */ + if (acio->pcie_tunnel_kernel_init && + !apple_pcie_tunnel_kernel_init_enabled()) + return dev_err_probe(acio->dev, -EPERM, + "PCIe-C kernel init disabled (pcie_apple.tunnel_kernel_init=0)\n"); + /* - * PCIe-C was cold-initialized by m1n1 before the ACIO M3 started, but - * enabling ACIO's cable-powered PCIe domain closes the Intr2AXI bridge - * again. The port enable sequence pulses this register immediately - * before forcing its DART active. Do the same after ACIO has - * enabled both tunnel adapters and before either child can touch DART MMIO. + * Enabling ACIO's cable-powered PCIe domain closes the Intr2AXI bridge, + * whether PCIe-C was cold-initialized by m1n1 before the ACIO M3 + * started or is still waiting for the kernel to do it. The port enable + * sequence pulses this register immediately before forcing its DART + * active. Do the same after ACIO has enabled both tunnel adapters and + * before either child can touch DART MMIO. * The bit self-clears after opening the bridge. */ writel(APPLE_CIO_PCIEC_INTR2AXI_ENABLE, @@ -555,8 +578,16 @@ static int apple_cio_populate_pcie_tunnel(struct apple_cio *acio) mb(); dev_info(acio->dev, "PCIe-C Intr2AXI bridge enabled\n"); - dev_info(acio->dev, - "PCIe-C live handoff accepted; populating tunnel children\n"); + /* + * The DART is the first child and probes as soon as it is created; + * its command engine stays busy until the port clock runs. Cold-init + * "apple,pciec-kernel-init" ports before populating either child. + */ + ret = apple_pcie_tunnel_prepare(acio->dev, acio->pcie_tunnel_np); + if (ret) + return ret; + + dev_info(acio->dev, "PCIe-C populating tunnel children\n"); return of_platform_populate(acio->pcie_tunnel_np, NULL, NULL, acio->dev); } @@ -590,7 +621,7 @@ static int apple_cio_activate_pcie_tunnel_locked(struct apple_cio *acio) } if (!acio->pcie_tunnel_preinitialized) return dev_err_probe(acio->dev, -ENODEV, - "PCIe-C requires a successful m1n1 preinit handoff\n"); + "PCIe-C needs an m1n1 preinit handoff or apple,pciec-kernel-init\n"); if (acio->pd_list->num_pds < 4 || !acio->pd_list->pd_links[3]) return dev_err_probe(acio->dev, -ENODEV, @@ -1726,9 +1757,11 @@ static int apple_cio_probe(struct platform_device *pdev) acio->pcie_tunnel_preinitialized = apple_cio_pcie_tunnel_is_preinitialized( acio->pcie_tunnel_np); + acio->pcie_tunnel_kernel_init = + apple_cio_pcie_tunnel_kernel_init(acio->pcie_tunnel_np); if (!acio->pcie_tunnel_preinitialized) { dev_warn(dev, - "PCIe-C tunnel disabled: m1n1 handoff is not initialized\n"); + "PCIe-C tunnel disabled: no m1n1 handoff and no apple,pciec-kernel-init\n"); } else { ret = apple_cio_map_pcie_intr2axi(acio); if (ret) diff --git a/include/linux/pci-apple.h b/include/linux/pci-apple.h index ef664800801fe..47d001ceec709 100644 --- a/include/linux/pci-apple.h +++ b/include/linux/pci-apple.h @@ -2,8 +2,13 @@ #ifndef _LINUX_PCI_APPLE_H #define _LINUX_PCI_APPLE_H +#include + struct device; +struct device_node; +bool apple_pcie_tunnel_kernel_init_enabled(void); +int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel); int apple_pcie_tunnel_quiesce(struct device *dev); int apple_pcie_tunnel_restore(struct device *dev); From 0ca64eb4baad59a7515745452257f16bd872bb71 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 13:03:09 -0700 Subject: [PATCH 0115/1579] arm64: dts: apple: t602x-j414-j416: Enable PCIe-C kernel init No m1n1 performs the PCIe-C preinit handoff on these machines. Mark apciec0-2 "apple,pciec-kernel-init" so the kernel cold-initializes the Thunderbolt PCIe tunnel ports itself. The tunneled DART driver also requires "apple,tunable", but iBoot adds the corresponding "dart-tunables-instance-0" to the ADT only at boot and m1n1 does not copy it for dart-apciec*. Without it the DART fails to probe and devices behind the tunnel time out. Add the values from the live J416c ADT (macOS ioreg), identical for all three ports; the same decoding reproduces entry for entry the apciec tunables m1n1 already copies into the DT. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan --- .../arm64/boot/dts/apple/t602x-j414-j416.dtsi | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 0c267d41c82f6..29064d88efb44 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -345,6 +345,51 @@ apple,pcie-tunnel = <&usb4_2_pcie_tunnel>; }; +/* + * No m1n1 performs the PCIe-C preinit handoff on these machines. Let the + * PCIe driver cold-initialize the Thunderbolt PCIe tunnel ports itself, so + * PCIe devices behind Thunderbolt 3 docks (e.g. their xHCI) come up. + */ +&apciec0 { + apple,pciec-kernel-init; +}; + +&apciec1 { + apple,pciec-kernel-init; +}; + +&apciec2 { + apple,pciec-kernel-init; +}; + +/* + * PCIe-C DART tunables. iBoot adds "dart-tunables-instance-0" to the ADT at + * boot and m1n1 does not copy it for these DARTs. Values taken from the live + * J416c ADT (macOS ioreg), identical for all three ports; offsets are + * relative to the DART's reg. + */ +#define APCIEC_DART_TUNABLES \ + apple,tunable = <0x20c 0xff0000b7 0xe40000b7>, \ + <0x220 0x000f0f0f 0x000f0f0f>, \ + <0x224 0x00ffffff 0x00080808>, \ + <0x300 0x00001f31 0x00000001>, \ + <0x308 0x3ffffffc 0x10000000>, \ + <0x310 0x3ffffffc 0x3ffffffc> + +&apciec0_dart { + APCIEC_DART_TUNABLES; +}; + +&apciec1_dart { + APCIEC_DART_TUNABLES; +}; + +&apciec2_dart { + APCIEC_DART_TUNABLES; +}; + +#undef APCIEC_DART_TUNABLES + /* Keep dptx_phy_ps on until DCP can sequence it during display bring-up. */ &ps_dptx_phy_ps { apple,always-on; From 6c03bd926fc5672dc91457053c13b3e83ad30048 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 16:37:09 -0400 Subject: [PATCH 0116/1579] arm64: preserve idle state on T8140 and T6050 Import the register-saving WFI helper and FPSIMD ownership flush from Aurora c407e1097ba74b464982c66483ba507a50ed927a (T8140/Neo), and extend its MIDR match to observed T6050 parts 0x064/0x065. Avoid WFIT/WFET in short delays on affected machines: hardware secondaries also lose GPRs across timed waits, including outside the idle loop. The macOS 27 kernelcache saves its idle context before WFI and reloads it when the link register is lost. Hardware probes reproduce this loss on all 17 T6050 secondaries and verify the existing Neo save/restore helper. Validated on J714s/M5 Pro under SPTM HV with the complete Asahi config: 18 CPUs online; init affinity and all32 SIMD registers across sleep/wake on every CPU; three subsequent timer wakes; no panic in 30s capture. QEMU passes the same test. Kernel waits are covered; arbitrary EL1/EL0 wait instructions still need a broader hypervisor policy. --- arch/arm64/include/asm/apple-idle.h | 23 ++++++++++++++ arch/arm64/include/asm/cputype.h | 7 +++++ arch/arm64/kernel/Makefile | 2 ++ arch/arm64/kernel/idle-apple.S | 45 +++++++++++++++++++++++++++ arch/arm64/kernel/idle.c | 47 +++++++++++++++++++++++++++-- arch/arm64/lib/delay.c | 10 ++++++ 6 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/include/asm/apple-idle.h create mode 100644 arch/arm64/kernel/idle-apple.S diff --git a/arch/arm64/include/asm/apple-idle.h b/arch/arm64/include/asm/apple-idle.h new file mode 100644 index 0000000000000..c6150d9e076c5 --- /dev/null +++ b/arch/arm64/include/asm/apple-idle.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_APPLE_IDLE_H +#define __ASM_APPLE_IDLE_H + +#include + +/* WFI that preserves x18-x30 on cores that lose them (T8140/T6050). */ +void apple_cpu_wfi(void); + +#ifdef CONFIG_ARCH_APPLE +DECLARE_STATIC_KEY_FALSE(apple_wfi_loses_regs); +static __always_inline bool apple_wfi_needs_save(void) +{ + return static_branch_unlikely(&apple_wfi_loses_regs); +} +#else +static __always_inline bool apple_wfi_needs_save(void) +{ + return false; +} +#endif + +#endif diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 1b9f0cda1336d..c99a99239f314 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -60,6 +60,13 @@ #define ARM_CPU_IMP_FUJITSU 0x46 #define ARM_CPU_IMP_HISI 0x48 #define ARM_CPU_IMP_APPLE 0x61 + +/* Observed CPU part IDs on T8140 and T6050. */ +#define MIDR_APPLE_TAHITI_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0x060) +#define MIDR_APPLE_TAHITI_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0x061) +#define MIDR_APPLE_T6050_064 MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0x064) +#define MIDR_APPLE_T6050_065 MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0x065) + #define ARM_CPU_IMP_AMPERE 0xC0 #define ARM_CPU_IMP_MICROSOFT 0x6D diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 81df89483ba9d..24958df352faa 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -88,3 +88,5 @@ endif # for cleaning subdir- += vdso vdso32 + +obj-$(CONFIG_ARCH_APPLE) += idle-apple.o diff --git a/arch/arm64/kernel/idle-apple.S b/arch/arm64/kernel/idle-apple.S new file mode 100644 index 0000000000000..9534505deb2f4 --- /dev/null +++ b/arch/arm64/kernel/idle-apple.S @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include +#include + +/* + * T8140/T6050 secondaries can resume from WFI with cleared general registers. + * SP and stack memory survive. Preserve the AAPCS64 callee-saved registers + * and x18 (which may hold the shadow call stack), including the return PC. + * This follows the stateless-wait approach used during m1n1 M5 Windows + * bring-up. It does not configure an Apple deep-sleep state or DVFS policy. + * + * IRQ masking/context management remains with cpu_do_idle(). The direct + * function call tells its compiler that x0-x17 and flags are clobbered. + */ + .macro apple_wait_save + stp x29, x30, [sp, #-112]! + mov x29, sp + stp x19, x20, [sp, #16] + stp x21, x22, [sp, #32] + stp x23, x24, [sp, #48] + stp x25, x26, [sp, #64] + stp x27, x28, [sp, #80] + str x18, [sp, #96] + dsb sy + .endm + + .macro apple_wait_restore + isb + ldr x18, [sp, #96] + ldp x19, x20, [sp, #16] + ldp x21, x22, [sp, #32] + ldp x23, x24, [sp, #48] + ldp x25, x26, [sp, #64] + ldp x27, x28, [sp, #80] + ldp x29, x30, [sp], #112 + ret + .endm + + .pushsection .cpuidle.text, "ax" +SYM_FUNC_START(apple_cpu_wfi) + apple_wait_save + wfi + apple_wait_restore +SYM_FUNC_END(apple_cpu_wfi) + .popsection diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c index 42c5543f9589c..548185e84eb23 100644 --- a/arch/arm64/kernel/idle.c +++ b/arch/arm64/kernel/idle.c @@ -4,11 +4,16 @@ */ #include +#include #include +#include +#include #include #include #include +#include +#include #include enum { @@ -34,6 +39,34 @@ static int __init setup_idle(char *arg) } early_param("idle", setup_idle); +#ifdef CONFIG_ARCH_APPLE +/* + * Apple T8140 and T6050 secondary CPUs can return from WFI with their + * general-purpose and FP/SIMD registers cleared. The key is enabled once + * at boot on those parts; everywhere else the check is a patched-out branch. + */ +DEFINE_STATIC_KEY_FALSE(apple_wfi_loses_regs); + +static const struct midr_range apple_wfi_loses_regs_cpus[] __initconst = { + MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_E), + MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_P), + MIDR_ALL_VERSIONS(MIDR_APPLE_T6050_064), + MIDR_ALL_VERSIONS(MIDR_APPLE_T6050_065), + {} +}; + +static int __init apple_wfi_quirk_init(void) +{ + if (is_midr_in_range_list(apple_wfi_loses_regs_cpus)) { + static_branch_enable(&apple_wfi_loses_regs); + pr_info("Apple CPU: preserving registers across WFI; avoiding timed waits in delays\n"); + } + return 0; +} +early_initcall(apple_wfi_quirk_init); + +#endif + /* * cpu_do_idle() * @@ -50,8 +83,18 @@ void __cpuidle cpu_do_idle(void) arm_cpuidle_save_irq_context(&context); if (likely(idle == ARM64_IDLE_WFI)) { - dsb(sy); - wfi(); + if (apple_wfi_needs_save()) { + /* + * The FP/SIMD registers do not survive either: drop the + * lazy ownership so the next return to userspace reloads + * them, even when the same task resumes on this CPU. + */ + fpsimd_save_and_flush_cpu_state(); + apple_cpu_wfi(); + } else { + dsb(sy); + wfi(); + } } else if (idle == ARM64_IDLE_YIELD) { dsb(sy); asm volatile("yield" ::: "memory"); diff --git a/arch/arm64/lib/delay.c b/arch/arm64/lib/delay.c index e278e060e78a9..172d0ad37f61f 100644 --- a/arch/arm64/lib/delay.c +++ b/arch/arm64/lib/delay.c @@ -14,6 +14,7 @@ #include #include +#include #define USECS_TO_CYCLES(time_usecs) \ xloops_to_cycles((time_usecs) * 0x10C7UL) @@ -42,6 +43,14 @@ void __delay(unsigned long cycles) { cycles_t start = __delay_cycles(); + /* + * T6050 secondaries also lose registers across WFIT. Delays can run + * with live kernel SIMD state, so idle's FPSIMD ownership flush is + * not sufficient here. Use the counter loop on affected machines. + */ + if (apple_wfi_needs_save()) + goto spin; + if (alternative_has_cap_unlikely(ARM64_HAS_WFXT)) { u64 end = start + cycles; @@ -60,6 +69,7 @@ void __delay(unsigned long cycles) wfe(); } +spin: while ((__delay_cycles() - start) < cycles) cpu_relax(); } From b4f0ddde253b7946ebbac0a48bfa37718cf99669 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 17:00:27 -0400 Subject: [PATCH 0117/1579] arm64: add opt-in bounded per-CPU EL1 sysreg read probe --- arch/arm64/Kconfig.debug | 11 + arch/arm64/include/asm/aurora-el1-sysregs.h | 17 ++ arch/arm64/kernel/Makefile | 2 + arch/arm64/kernel/aurora-el1-sysregs-asm.S | 307 ++++++++++++++++++++ arch/arm64/kernel/aurora-el1-sysregs-list.h | 21 ++ arch/arm64/kernel/aurora-el1-sysregs.c | 157 ++++++++++ arch/arm64/kernel/traps.c | 4 + 7 files changed, 519 insertions(+) create mode 100644 arch/arm64/include/asm/aurora-el1-sysregs.h create mode 100644 arch/arm64/kernel/aurora-el1-sysregs-asm.S create mode 100644 arch/arm64/kernel/aurora-el1-sysregs-list.h create mode 100644 arch/arm64/kernel/aurora-el1-sysregs.c diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 265c4461031f4..f873818825d2d 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -21,3 +21,14 @@ config ARM64_RELOC_TEST tristate "Relocation testing module" source "drivers/hwtracing/coresight/Kconfig" + +config AURORA_EL1_SYSREG_PROBE + bool "Aurora bounded EL1 sysreg read diagnostic" + depends on ARCH_APPLE && DEBUG_KERNEL + default n + help + Build a read-only, fixed-register diagnostic for the experimental + x1n1 SPTM hypervisor. The aurora_el1_sysregs=1 boot parameter is + also required. Only use with the matching opt-in HV probe handler. + Undefined reads recover only at explicitly generated probe PCs. + No candidate system register is written. Say N for normal kernels. diff --git a/arch/arm64/include/asm/aurora-el1-sysregs.h b/arch/arm64/include/asm/aurora-el1-sysregs.h new file mode 100644 index 0000000000000..0d338aec786eb --- /dev/null +++ b/arch/arm64/include/asm/aurora-el1-sysregs.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_AURORA_EL1_SYSREGS_H +#define __ASM_AURORA_EL1_SYSREGS_H + +#include +#include + +#ifdef CONFIG_AURORA_EL1_SYSREG_PROBE +bool aurora_el1_sysreg_undef(struct pt_regs *regs, unsigned long esr, u32 insn); +#else +static inline bool aurora_el1_sysreg_undef(struct pt_regs *regs, + unsigned long esr, u32 insn) +{ + return false; +} +#endif +#endif diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 24958df352faa..f022d1563b152 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -90,3 +90,5 @@ endif subdir- += vdso vdso32 obj-$(CONFIG_ARCH_APPLE) += idle-apple.o + +obj-$(CONFIG_AURORA_EL1_SYSREG_PROBE) += aurora-el1-sysregs.o aurora-el1-sysregs-asm.o diff --git a/arch/arm64/kernel/aurora-el1-sysregs-asm.S b/arch/arm64/kernel/aurora-el1-sysregs-asm.S new file mode 100644 index 0000000000000..d9a08a2c138c2 --- /dev/null +++ b/arch/arm64/kernel/aurora-el1-sysregs-asm.S @@ -0,0 +1,307 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include +.text +.global aurora_el1_sysregs_begin +aurora_el1_sysregs_begin: +SYM_FUNC_START(aurora_el1_sysreg_read) + mov x4, x0 + adr x9, .Lprobe_dispatch + ldr x9, [x9, x1, lsl #3] + br x9 +.Lprobe_case_0: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_0: + .inst 0xd5384240 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_1: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_1: + .inst 0x00000000 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_2: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_2: + .inst 0xd5380000 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_3: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_3: + .inst 0xd53800a0 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_4: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_4: + .inst 0xd5380420 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_5: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_5: + .inst 0xd5381000 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_6: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_6: + .inst 0xd5382040 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_7: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_7: + .inst 0xd5382000 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_8: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_8: + .inst 0xd5381020 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_9: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_9: + .inst 0xd53be000 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_10: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_10: + .inst 0xd53be040 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_11: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_11: + .inst 0xd53be320 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_12: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_12: + .inst 0xd53be220 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_13: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_13: + .inst 0xd539f1a0 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_14: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_14: + .inst 0xd538f000 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_15: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_15: + .inst 0xd538f020 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_16: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_16: + .inst 0xd53df500 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +.Lprobe_case_17: + bti j + movz x0, #47824 + movk x0, #47824, lsl #16 + movk x0, #47824, lsl #32 + movk x0, #47824, lsl #48 + mov x1, #0 + mov x2, #0 + mov x3, #0 +.Lprobe_site_17: + .inst 0xd53ff080 + stp x0, x1, [x4] + stp x2, x3, [x4, #16] + ret +SYM_FUNC_END(aurora_el1_sysreg_read) +.global aurora_el1_sysregs_end +aurora_el1_sysregs_end: +.balign 8 +.Lprobe_dispatch: + .quad .Lprobe_case_0 + .quad .Lprobe_case_1 + .quad .Lprobe_case_2 + .quad .Lprobe_case_3 + .quad .Lprobe_case_4 + .quad .Lprobe_case_5 + .quad .Lprobe_case_6 + .quad .Lprobe_case_7 + .quad .Lprobe_case_8 + .quad .Lprobe_case_9 + .quad .Lprobe_case_10 + .quad .Lprobe_case_11 + .quad .Lprobe_case_12 + .quad .Lprobe_case_13 + .quad .Lprobe_case_14 + .quad .Lprobe_case_15 + .quad .Lprobe_case_16 + .quad .Lprobe_case_17 +.section .rodata +.balign 8 +.global aurora_el1_sysregs_sites +aurora_el1_sysregs_sites: + .quad .Lprobe_site_0 + .quad .Lprobe_site_1 + .quad .Lprobe_site_2 + .quad .Lprobe_site_3 + .quad .Lprobe_site_4 + .quad .Lprobe_site_5 + .quad .Lprobe_site_6 + .quad .Lprobe_site_7 + .quad .Lprobe_site_8 + .quad .Lprobe_site_9 + .quad .Lprobe_site_10 + .quad .Lprobe_site_11 + .quad .Lprobe_site_12 + .quad .Lprobe_site_13 + .quad .Lprobe_site_14 + .quad .Lprobe_site_15 + .quad .Lprobe_site_16 + .quad .Lprobe_site_17 diff --git a/arch/arm64/kernel/aurora-el1-sysregs-list.h b/arch/arm64/kernel/aurora-el1-sysregs-list.h new file mode 100644 index 0000000000000..02dfbd7cecd8d --- /dev/null +++ b/arch/arm64/kernel/aurora-el1-sysregs-list.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +static const u32 probe_opcodes[] = { + 0xd5384240, /* CurrentEL */ + 0x00000000, /* UNDEFINED_INSTRUCTION */ + 0xd5380000, /* MIDR_EL1 */ + 0xd53800a0, /* MPIDR_EL1 */ + 0xd5380420, /* ID_AA64PFR1_EL1 */ + 0xd5381000, /* SCTLR_EL1 */ + 0xd5382040, /* TCR_EL1 */ + 0xd5382000, /* TTBR0_EL1 */ + 0xd5381020, /* ACTLR_EL1 */ + 0xd53be000, /* CNTFRQ_EL0 */ + 0xd53be040, /* CNTVCT_EL0 */ + 0xd53be320, /* CNTV_CTL_EL0 */ + 0xd53be220, /* CNTP_CTL_EL0 */ + 0xd539f1a0, /* AGTCNTRDIR_EL1 */ + 0xd538f000, /* HID0_EL1 */ + 0xd538f020, /* EHID0_EL1 */ + 0xd53df500, /* CYC_OVRD_EL1 */ + 0xd53ff080, /* UPMCR0_EL1 */ +}; diff --git a/arch/arm64/kernel/aurora-el1-sysregs.c b/arch/arm64/kernel/aurora-el1-sysregs.c new file mode 100644 index 0000000000000..8dae62676d8e2 --- /dev/null +++ b/arch/arm64/kernel/aurora-el1-sysregs.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Explicitly opt-in, bounded, read-only EL1 sysreg diagnostic. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "aurora-el1-sysregs-list.h" + +#define PROBE_COOKIE 0x53595245474c4e58ULL +#define PROBE_UNDEF 0xbad0bad0bad0bad0ULL +#define PROBE_TRAP 0xe12e12e12e12e12eULL + +struct probe_value { + u64 value, status, esr, raw_esr; +}; + +struct probe_capture { + struct probe_value values[ARRAY_SIZE(probe_opcodes)]; + u64 mpidr, midr; + int cpu, armed, disarmed; +}; + +struct probe_active { + unsigned long pc; + u32 opcode; +}; + +static DEFINE_PER_CPU(struct probe_active, probe_active); +static bool probe_enabled; + +extern char aurora_el1_sysregs_begin[], aurora_el1_sysregs_end[]; +extern const unsigned long aurora_el1_sysregs_sites[]; +extern void aurora_el1_sysreg_read(struct probe_value *result, unsigned int index); + +static int __init aurora_el1_sysregs_param(char *str) +{ + return kstrtobool(str, &probe_enabled); +} +early_param("aurora_el1_sysregs", aurora_el1_sysregs_param); + +/* Invoked only from do_el1_undef, after the instruction was safely read. + * No other UNDEF or exception-table entry is granted recovery. */ +bool aurora_el1_sysreg_undef(struct pt_regs *regs, unsigned long esr, u32 insn) +{ + struct probe_active *active = this_cpu_ptr(&probe_active); + unsigned long pc = READ_ONCE(active->pc); + + if (!READ_ONCE(probe_enabled) || !pc || regs->pc != pc || + ESR_ELx_EC(esr) != ESR_ELx_EC_UNKNOWN || + insn != READ_ONCE(active->opcode) || + (insn && (insn & 0xffe0001f) != 0xd5200000)) + return false; + regs->regs[0] = PROBE_UNDEF; + regs->regs[1] = 1; + regs->regs[2] = esr; + regs->regs[3] = esr; + regs->pc += 4; + return true; +} + +static u64 probe_hvc(unsigned long begin, unsigned long end) +{ + register u64 x0 asm("x0") = PROBE_COOKIE; + register u64 x1 asm("x1") = begin; + register u64 x2 asm("x2") = end; + + asm volatile("hvc #0x5352" : "+r" (x0), "+r" (x1), "+r" (x2) + : : "x3", "x4", "x5", "x6", "x7", "memory", "cc"); + return x0; +} + +static void probe_this_cpu(void *arg) +{ + struct probe_capture *capture = arg; + struct probe_active *active = this_cpu_ptr(&probe_active); + unsigned long flags; + unsigned int i; + + /* The synchronous callback is pinned with IRQs disabled. Preserve the + * caller's complete DAIF state, including FIQ/SError/debug masks. */ + flags = local_daif_save(); + capture->cpu = raw_smp_processor_id(); + capture->mpidr = read_sysreg(mpidr_el1); + capture->midr = read_sysreg(midr_el1); + capture->armed = probe_hvc((unsigned long)aurora_el1_sysregs_begin, + (unsigned long)aurora_el1_sysregs_end) == PROBE_COOKIE; + if (!capture->armed) + goto out; + for (i = 0; i < ARRAY_SIZE(probe_opcodes); i++) { + WRITE_ONCE(active->opcode, probe_opcodes[i]); + WRITE_ONCE(active->pc, aurora_el1_sysregs_sites[i]); + barrier(); + aurora_el1_sysreg_read(&capture->values[i], i); + barrier(); + WRITE_ONCE(active->pc, 0); + } + capture->disarmed = probe_hvc(0, 0) == PROBE_COOKIE; +out: + WRITE_ONCE(active->pc, 0); + local_daif_restore(flags); +} + +static int __init aurora_el1_sysregs_init(void) +{ + struct probe_capture capture; + int cpu, ret, failures = 0, tested = 0; + unsigned int i; + + if (!probe_enabled) + return 0; + pr_info("AURORA_LINUX_SYSREG_BEGIN count=%zu\n", ARRAY_SIZE(probe_opcodes)); + cpus_read_lock(); + for_each_online_cpu(cpu) { + bool controls_ok; + + memset(&capture, 0, sizeof(capture)); + ret = smp_call_function_single(cpu, probe_this_cpu, &capture, 1); + if (ret) { + pr_err("AURORA_LINUX_SYSREG_CPU_FAILED cpu=%d error=%d\n", cpu, ret); + failures++; + continue; + } + pr_info("AURORA_LINUX_SYSREG_CPU cpu=%d mpidr=%016llx midr=%016llx armed=%d disarmed=%d\n", + capture.cpu, capture.mpidr, capture.midr, capture.armed, capture.disarmed); + if (!capture.armed || !capture.disarmed) { + failures++; + break; + } + controls_ok = capture.values[0].status == 0 && capture.values[0].value == 4 && + capture.values[1].status == 1 && capture.values[1].value == PROBE_UNDEF && + capture.values[4].status == 2 && capture.values[4].value == PROBE_TRAP; + for (i = 0; i < ARRAY_SIZE(probe_opcodes); i++) { + struct probe_value *v = &capture.values[i]; + + pr_info("AURORA_LINUX_SYSREG_READ cpu=%d index=%u insn=%08x value=%016llx status=%llu esr=%016llx raw=%016llx\n", + cpu, i, probe_opcodes[i], v->value, v->status, v->esr, v->raw_esr); + if (v->status > 2 || (v->status == 1 && v->value != PROBE_UNDEF) || + (v->status == 2 && v->value != PROBE_TRAP)) + controls_ok = false; + } + pr_info("AURORA_LINUX_SYSREG_CPU_DONE cpu=%d controls=%s\n", + cpu, controls_ok ? "PASS" : "FAIL"); + tested++; + if (!controls_ok) + failures++; + } + cpus_read_unlock(); + pr_info("AURORA_LINUX_SYSREG_DONE cpus=%d failures=%d\n", tested, failures); + return failures ? -EIO : 0; +} +late_initcall(aurora_el1_sysregs_init); diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 9142820160698..439ca7c5b4642 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -480,6 +481,9 @@ void do_el1_undef(struct pt_regs *regs, unsigned long esr) if (aarch64_insn_read((void *)regs->pc, &insn)) goto out_err; + if (aurora_el1_sysreg_undef(regs, esr, insn)) + return; + if (try_emulate_el1_ssbs(regs, insn)) return; From f967f3be7e2ac26b949f126da518122224711bf4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 17:37:56 -0400 Subject: [PATCH 0118/1579] Add SPTM SART transport with paired hardware grant selftest --- drivers/soc/apple/sart.c | 137 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/drivers/soc/apple/sart.c b/drivers/soc/apple/sart.c index 9eaf3febb3826..35151fa33fad4 100644 --- a/drivers/soc/apple/sart.c +++ b/drivers/soc/apple/sart.c @@ -18,6 +18,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -25,6 +28,15 @@ #define APPLE_SART_MAX_ENTRIES 16 +/* One-operation sptm2mmio batch, shared with x1n1 src/sptm.c. */ +struct sart_x1n1_batch { + u32 count, flags; + s32 status; + u32 done; + u32 op, op_flags; + u64 arg[6], ret; +}; + /* SARTv0 registers */ #define APPLE_SART0_CONFIG(idx) (0x00 + 4 * (idx)) #define APPLE_SART0_CONFIG_FLAGS GENMASK(28, 24) @@ -81,8 +93,103 @@ struct apple_sart { unsigned long protected_entries; unsigned long used_entries; + bool x1n1, x1n1_active; + struct mutex x1n1_lock; + struct sart_x1n1_batch *batch; + struct { + phys_addr_t paddr; + size_t size; + } regions[APPLE_SART_MAX_ENTRIES]; }; +/* Caller holds x1n1_lock. SPTM re-dispatch changes SP_EL1, so retain the + * kernel SP in a GPR exactly as the qualified DART transport does. + */ +static int sart_x1n1_call(struct apple_sart *sart, u32 op, + u64 a0, u64 a1, u64 a2, u64 a3) +{ + struct sart_x1n1_batch *b = sart->batch; + unsigned long flags; + u64 pa = virt_to_phys(b); + void __iomem *db = sart->regs + 0x1000; + + static_assert(sizeof(*b) == 80); + memset(b, 0, sizeof(*b)); + b->count = 1; + b->op = op; + b->arg[0] = a0; + b->arg[1] = a1; + b->arg[2] = a2; + b->arg[3] = a3; + wmb(); + local_irq_save(flags); + asm volatile("mov x9, sp\n" + "str %x[pa], [%[db]]\n" + "mov sp, x9\n" + : : [pa] "r" (pa), [db] "r" (db) : "x9", "memory"); + local_irq_restore(flags); + rmb(); + return b->status || b->done != 1 || b->ret ? -EIO : 0; +} + +static int sart_x1n1_region(struct apple_sart *sart, phys_addr_t paddr, + size_t size, bool map) +{ + int i, ret; + + if (!size || (paddr | size) & (SZ_16K - 1)) + return -EINVAL; + mutex_lock(&sart->x1n1_lock); + if (!sart->x1n1_active) { + ret = sart_x1n1_call(sart, 1, 1, 0, 0, 0); + if (ret) + goto out; + sart->x1n1_active = true; + } + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) { + if (map ? !sart->regions[i].size : + (sart->regions[i].paddr == paddr && sart->regions[i].size == size)) + break; + } + if (i == APPLE_SART_MAX_ENTRIES) { + ret = map ? -EBUSY : -ENOENT; + goto out; + } + /* Inputs are guest CPU physical addresses. The HV validates/translates + * them to host backing pages; never pass an arbitrary host PA here. + * perm=1 permits device writes; guard=1 holds the firmware power canary. + */ + ret = sart_x1n1_call(sart, map ? 2 : 3, paddr, size, map, map); + if (!ret) { + sart->regions[i].paddr = map ? paddr : 0; + sart->regions[i].size = map ? size : 0; + } +out: + mutex_unlock(&sart->x1n1_lock); + return ret; +} + +static void sart_x1n1_selftest(struct apple_sart *sart) +{ + void *page = (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO); + phys_addr_t pa; + int ret; + + if (!page) + return; + pa = virt_to_phys(page); + ret = sart_x1n1_region(sart, pa, PAGE_SIZE, true); + dev_info(sart->dev, "x1n1 selftest: map %pa size %#lx status %d\n", + &pa, PAGE_SIZE, ret); + if (!ret) { + ret = sart_x1n1_region(sart, pa, PAGE_SIZE, false); + dev_info(sart->dev, "x1n1 selftest: unmap status %d\n", ret); + if (ret) + return; /* Do not free memory while a DMA grant might remain. */ + } + free_page((unsigned long)page); +} + static void sart0_get_entry(struct apple_sart *sart, int index, u8 *flags, phys_addr_t *paddr, size_t *size) { @@ -193,6 +300,21 @@ static int apple_sart_probe(struct platform_device *pdev) sart->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(sart->regs)) return PTR_ERR(sart->regs); + if (of_device_is_compatible(dev->of_node, "apple,sart-x1n1")) { + if (readq(sart->regs) != 0x78316e31 || readq(sart->regs + 8) != 1 || + !(readq(sart->regs + 0x18) & BIT(1))) + return -ENODEV; + sart->batch = devm_kzalloc(dev, PAGE_SIZE, GFP_KERNEL); + if (!sart->batch) + return -ENOMEM; + mutex_init(&sart->x1n1_lock); + sart->x1n1 = true; + platform_set_drvdata(pdev, sart); + dev_info(dev, "x1n1 SART via sptm2mmio\n"); + if (of_property_read_bool(dev->of_node, "apple,x1n1-selftest")) + sart_x1n1_selftest(sart); + return 0; + } for (i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) { u8 flags; @@ -268,6 +390,9 @@ int apple_sart_add_allowed_region(struct apple_sart *sart, phys_addr_t paddr, { int i, ret; + if (sart->x1n1) + return sart_x1n1_region(sart, paddr, size, true); + for (i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) { if (test_bit(i, &sart->protected_entries)) continue; @@ -302,6 +427,9 @@ int apple_sart_remove_allowed_region(struct apple_sart *sart, phys_addr_t paddr, { int i; + if (sart->x1n1) + return sart_x1n1_region(sart, paddr, size, false); + dev_dbg(sart->dev, "will remove [paddr: %pa, size: 0x%zx] from allowed regions\n", &paddr, size); @@ -338,6 +466,14 @@ static void apple_sart_shutdown(struct platform_device *pdev) struct apple_sart *sart = dev_get_drvdata(&pdev->dev); int i; + if (sart->x1n1) { + for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) + if (sart->regions[i].size) + sart_x1n1_region(sart, sart->regions[i].paddr, + sart->regions[i].size, false); + return; + } + for (i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) { if (test_bit(i, &sart->protected_entries)) continue; @@ -347,6 +483,7 @@ static void apple_sart_shutdown(struct platform_device *pdev) } static const struct of_device_id apple_sart_of_match[] = { + { .compatible = "apple,sart-x1n1" }, { .compatible = "apple,t6000-sart", .data = &sart_ops_v3, From 4807b70ace3fd692369e9dff5414c06b6056c3d1 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Thu, 13 Aug 2026 08:52:14 +0000 Subject: [PATCH 0119/1579] nvme-apple: support split post-M4 controller apertures Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: b3a65fa7f2fd62056d1b9ad9bb09b023741cc8ff Ported-paths: drivers/nvme/host/apple.c --- drivers/nvme/host/apple.c | 67 ++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 21f8f02f4cc35..2872bb4235294 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -44,6 +44,8 @@ #define APPLE_ANS_ACQ_DB 0x1004 #define APPLE_ANS_IOCQ_DB 0x100c +#define APPLE_ANS_IOSQ_REGISTER 0x1200 +#define APPLE_ANS_IOCQ_REGISTER 0x1208 #define APPLE_ANS_MAX_PEND_CMDS_CTRL 0x1210 #define APPLE_ANS_BOOT_STATUS 0x1300 @@ -169,6 +171,10 @@ struct apple_nvme_iod { struct apple_nvme_hw { bool has_lsq_nvmmu; + bool has_legacy_lsq_ctrl; + bool has_queue_count; + bool has_separate_nvmmu; + bool needs_ioq_registers; u32 max_queue_depth; }; @@ -177,6 +183,7 @@ struct apple_nvme { void __iomem *mmio_coproc; void __iomem *mmio_nvme; + void __iomem *mmio_nvmmu; const struct apple_nvme_hw *hw; struct device **pd_dev; @@ -299,8 +306,8 @@ static void apple_nvmmu_inval(struct apple_nvme_queue *q, unsigned int tag) { struct apple_nvme *anv = queue_to_apple_nvme(q); - writel(tag, anv->mmio_nvme + APPLE_NVMMU_TCB_INVAL); - if (readl(anv->mmio_nvme + APPLE_NVMMU_TCB_STAT)) + writel(tag, anv->mmio_nvmmu + APPLE_NVMMU_TCB_INVAL); + if (readl(anv->mmio_nvmmu + APPLE_NVMMU_TCB_STAT)) dev_warn_ratelimited(anv->dev, "NVMMU TCB invalidation failed\n"); } @@ -1198,8 +1205,9 @@ static void apple_nvme_reset_work(struct work_struct *work) * Enable NVMMU and linear submission queues which is required * since T6000. */ - writel(APPLE_ANS_LINEAR_SQ_EN, - anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); + if (anv->hw->has_legacy_lsq_ctrl) + writel(APPLE_ANS_LINEAR_SQ_EN, + anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); /* Allow as many pending command as possible for both queues */ writel(anv->hw->max_queue_depth @@ -1208,7 +1216,7 @@ static void apple_nvme_reset_work(struct work_struct *work) /* Setup the NVMMU for the maximum admin and IO queue depth */ writel(anv->hw->max_queue_depth - 1, - anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS); + anv->mmio_nvmmu + APPLE_NVMMU_NUM_TCBS); } /* Setup the admin queue */ @@ -1221,9 +1229,9 @@ static void apple_nvme_reset_work(struct work_struct *work) if (anv->hw->has_lsq_nvmmu) { /* Setup NVMMU for both queues */ writeq(anv->adminq.tcb_dma_addr, - anv->mmio_nvme + APPLE_NVMMU_ASQ_TCB_BASE); + anv->mmio_nvmmu + APPLE_NVMMU_ASQ_TCB_BASE); writeq(anv->ioq.tcb_dma_addr, - anv->mmio_nvme + APPLE_NVMMU_IOSQ_TCB_BASE); + anv->mmio_nvmmu + APPLE_NVMMU_IOSQ_TCB_BASE); } anv->ctrl.sqsize = @@ -1260,13 +1268,21 @@ static void apple_nvme_reset_work(struct work_struct *work) goto out_remove_cq; apple_nvme_init_queue(&anv->ioq); + if (anv->hw->needs_ioq_registers) { + writeq(anv->ioq.cq_dma_addr, + anv->mmio_nvme + APPLE_ANS_IOCQ_REGISTER); + writeq(anv->ioq.sq_dma_addr, + anv->mmio_nvme + APPLE_ANS_IOSQ_REGISTER); + } nr_io_queues = 1; - ret = nvme_set_queue_count(&anv->ctrl, &nr_io_queues); - if (ret) - goto out_remove_sq; - if (nr_io_queues != 1) { - ret = -ENXIO; - goto out_remove_sq; + if (anv->hw->has_queue_count) { + ret = nvme_set_queue_count(&anv->ctrl, &nr_io_queues); + if (ret) + goto out_remove_sq; + if (nr_io_queues != 1) { + ret = -ENXIO; + goto out_remove_sq; + } } anv->ctrl.queue_count = nr_io_queues + 1; @@ -1591,11 +1607,21 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) ret = PTR_ERR(anv->mmio_nvme); goto put_dev; } + if (anv->hw->has_separate_nvmmu) { + anv->mmio_nvmmu = + devm_platform_ioremap_resource_byname(pdev, "nvmmu"); + if (IS_ERR(anv->mmio_nvmmu)) { + ret = PTR_ERR(anv->mmio_nvmmu); + goto put_dev; + } + } else { + anv->mmio_nvmmu = anv->mmio_nvme; + } if (anv->hw->has_lsq_nvmmu) { - anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB; + anv->adminq.sq_db = anv->mmio_nvmmu + APPLE_ANS_LINEAR_ASQ_DB; anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB; - anv->ioq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_IOSQ_DB; + anv->ioq.sq_db = anv->mmio_nvmmu + APPLE_ANS_LINEAR_IOSQ_DB; anv->ioq.cq_db = anv->mmio_nvme + APPLE_ANS_IOCQ_DB; } else { anv->adminq.sq_db = anv->mmio_nvme + NVME_REG_DBS; @@ -1816,17 +1842,28 @@ static DEFINE_SIMPLE_DEV_PM_OPS(apple_nvme_pm_ops, apple_nvme_suspend, static const struct apple_nvme_hw apple_nvme_t8015_hw = { .has_lsq_nvmmu = false, + .has_queue_count = true, .max_queue_depth = 16, }; static const struct apple_nvme_hw apple_nvme_t8103_hw = { .has_lsq_nvmmu = true, + .has_legacy_lsq_ctrl = true, + .has_queue_count = true, + .max_queue_depth = 64, +}; + +static const struct apple_nvme_hw apple_nvme_t8132_hw = { + .has_lsq_nvmmu = true, + .has_separate_nvmmu = true, + .needs_ioq_registers = true, .max_queue_depth = 64, }; static const struct of_device_id apple_nvme_of_match[] = { { .compatible = "apple,t8015-nvme-ans2", .data = &apple_nvme_t8015_hw }, { .compatible = "apple,t8103-nvme-ans2", .data = &apple_nvme_t8103_hw }, + { .compatible = "apple,t8132-nvme-ans2", .data = &apple_nvme_t8132_hw }, { .compatible = "apple,nvme-ans2", .data = &apple_nvme_t8103_hw }, {}, }; From a0fd292a7debef9377b9502ac32cf755680e04fd Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:50:37 +0000 Subject: [PATCH 0120/1579] nvme-apple: fix post-M4 I/O queue size encoding Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: e7a2e694758ab039f3c7de5ec1b24c1f25ec7999 Ported-paths: drivers/nvme/host/apple.c --- drivers/nvme/host/apple.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 2872bb4235294..8830997b5d7ac 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1115,7 +1115,7 @@ static void apple_nvme_reset_work(struct work_struct *work) { unsigned int nr_io_queues = 1; int ret; - u32 boot_status, aqa; + u32 boot_status, aqa, ioqa, ioqa_depth; struct apple_nvme *anv = container_of(work, struct apple_nvme, ctrl.reset_work); enum nvme_ctrl_state state = nvme_ctrl_state(&anv->ctrl); @@ -1209,10 +1209,28 @@ static void apple_nvme_reset_work(struct work_struct *work) writel(APPLE_ANS_LINEAR_SQ_EN, anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); - /* Allow as many pending command as possible for both queues */ - writel(anv->hw->max_queue_depth - | (anv->hw->max_queue_depth << 16), anv->mmio_nvme - + APPLE_ANS_MAX_PEND_CMDS_CTRL); + /* + * The legacy register takes queue entry counts. On post-M4 + * controllers the same offset is IOQA and both fields contain the + * zero-based queue size, like NVMe AQA. Programming 64 for a + * 64-entry queue makes ANS treat it as 65 entries and reject CQ head + * 63 once the host wraps through the queue. + */ + ioqa_depth = anv->hw->max_queue_depth; + if (anv->hw->needs_ioq_registers) + ioqa_depth--; + ioqa = ioqa_depth | (ioqa_depth << 16); + writel(ioqa, anv->mmio_nvme + APPLE_ANS_MAX_PEND_CMDS_CTRL); + if (readl(anv->mmio_nvme + APPLE_ANS_MAX_PEND_CMDS_CTRL) != + ioqa) { + dev_err(anv->dev, "failed to program I/O queue aperture\n"); + ret = -EIO; + goto out; + } + if (anv->hw->needs_ioq_registers) + dev_info(anv->dev, + "post-M4 I/O queues: entries=%u IOQA=0x%08x\n", + anv->hw->max_queue_depth, ioqa); /* Setup the NVMMU for the maximum admin and IO queue depth */ writel(anv->hw->max_queue_depth - 1, From f3a2c08efc97ae6eadb0f06af2f131678e7c3c78 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:42:54 +0000 Subject: [PATCH 0121/1579] nvme-apple: skip unsupported post-M4 queue deletes Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: 71c2d50a018be916717d16822aab30854ca933fe Ported-paths: drivers/nvme/host/apple.c --- drivers/nvme/host/apple.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 8830997b5d7ac..7022d03b59917 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -955,8 +955,20 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) if (!dead) { if (READ_ONCE(anv->ioq.enabled)) { - apple_nvme_remove_sq(anv); - apple_nvme_remove_cq(anv); + /* + * Post-M4 firmware rejects Delete SQ and Delete CQ with + * BAD_CMD during shutdown. The queues are registered through + * the dedicated IOSQ/IOCQ aperture on these controllers and + * are torn down by the following controller shutdown/disable. + * Keep the explicit admin commands for legacy controllers. + */ + if (anv->hw->needs_ioq_registers) { + dev_info(anv->dev, + "post-M4 shutdown: skipping unsupported I/O queue delete commands\n"); + } else { + apple_nvme_remove_sq(anv); + apple_nvme_remove_cq(anv); + } } /* From 5d8f1ec8ed430773587f8ec7fb3d74242a7dac06 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:05:05 +0000 Subject: [PATCH 0122/1579] nvme-apple: skip unsupported post-M4 shutdown notice Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: cfef0f1b75994bcd8610f89e7d7ef3b0988070f1 Ported-paths: drivers/nvme/host/apple.c --- drivers/nvme/host/apple.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 7022d03b59917..b105417bb72a4 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -985,8 +985,14 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) * NVMe controller is handed off with firmware shut down, in an * NVMe disabled state, after a clean shutdown). */ - if (shutdown) - nvme_disable_ctrl(&anv->ctrl, shutdown); + if (shutdown) { + if (anv->hw->needs_ioq_registers) { + dev_info(anv->dev, + "post-M4 shutdown: skipping unsupported shutdown notification\n"); + } else { + nvme_disable_ctrl(&anv->ctrl, shutdown); + } + } nvme_disable_ctrl(&anv->ctrl, false); } From 5cf4df2e396823ba96a3eef7e72a28002eeb5995 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:11:21 +0000 Subject: [PATCH 0123/1579] nvme-apple: preserve post-M4 controller for shutdown Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: 20bbc11b8c6e86b2654f7ce4f2fbf15f9bfea348 Ported-paths: drivers/nvme/host/apple.c --- drivers/nvme/host/apple.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index b105417bb72a4..aca63bbaad7c5 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -993,7 +993,12 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_disable_ctrl(&anv->ctrl, shutdown); } } - nvme_disable_ctrl(&anv->ctrl, false); + if (shutdown && anv->hw->needs_ioq_registers) { + dev_info(anv->dev, + "post-M4 shutdown: leaving controller enabled for RTKit shutdown\n"); + } else { + nvme_disable_ctrl(&anv->ctrl, false); + } } WRITE_ONCE(anv->ioq.enabled, false); From f63003800e9d03df77c82910fb6caa653c13435f Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:20:37 +0000 Subject: [PATCH 0124/1579] nvme-apple: preserve post-M4 RTKit for shutdown Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: 95205d31711ee3a8260075c1febc7ead5ccee5d6 Ported-paths: drivers/nvme/host/apple.c --- drivers/nvme/host/apple.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index aca63bbaad7c5..73447c15fbe1a 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1848,6 +1848,19 @@ static void apple_nvme_shutdown(struct platform_device *pdev) flush_delayed_work(&anv->flush_dwork); apple_nvme_disable(anv, true); + /* + * Post-M4 ANS firmware crashes when asked to perform the RTKit shutdown + * handshake during final system shutdown, even after all Linux queues are + * quiesced and the controller is left enabled. Leave the co-processor + * running until the imminent platform reset. Removal and suspend retain the + * complete teardown sequence below. + */ + if (anv->hw->needs_ioq_registers) { + dev_info(anv->dev, + "post-M4 shutdown: leaving RTKit running for system reset\n"); + return; + } + if (apple_rtkit_is_running(anv->rtk)) { apple_rtkit_shutdown(anv->rtk); From 678ba01da2256609da004e8878f727d55028aefa Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sat, 15 Aug 2026 23:10:07 +0000 Subject: [PATCH 0125/1579] nvme-apple: support live post-M4 boot handoff Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: e9a707054964419645cbd8528dd70c548861a705 Ported-paths: drivers/nvme/host/apple.c, drivers/soc/apple/rtkit.c, drivers/soc/apple/rtkit-internal.h, include/linux/soc/apple/rtkit.h --- drivers/nvme/host/apple.c | 106 ++++++++++++++++++++++++++--- drivers/soc/apple/rtkit-internal.h | 1 + drivers/soc/apple/rtkit.c | 34 ++++++++- include/linux/soc/apple/rtkit.h | 6 ++ 4 files changed, 134 insertions(+), 13 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 73447c15fbe1a..c78016983a0a4 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -193,6 +194,7 @@ struct apple_nvme { struct apple_sart *sart; struct apple_rtkit *rtk; struct reset_control *reset; + bool owns_rtkit; struct dma_pool *prp_page_pool; struct dma_pool *prp_small_pool; @@ -220,6 +222,21 @@ struct apple_nvme { struct blk_crypto_profile crypto_profile; }; +static inline void apple_nvme_writeq(struct apple_nvme *anv, u64 value, + void __iomem *addr) +{ + /* + * Post-M4 ANS consumes the admin and NVMMU queue addresses as paired + * 32-bit registers while the controller is disabled. Match the access + * sequence used by Apple firmware, m1n1, and U-Boot; older ANS + * generations retain their native 64-bit access. + */ + if (anv->hw->needs_ioq_registers) + lo_hi_writeq(value, addr); + else + writeq(value, addr); +} + unsigned int flush_interval = 1000; module_param(flush_interval, uint, 0644); MODULE_PARM_DESC(flush_interval, "Grace period in msecs between flushes"); @@ -267,10 +284,28 @@ static int apple_nvme_sart_dma_setup(void *cookie, struct apple_nvme *anv = cookie; int ret; - if (bfr->iova) - return -EINVAL; if (!bfr->size) return -EINVAL; + if (bfr->iova) { + /* + * A live post-M4 handoff retains the RTKit buffers allocated by + * Stage 1. Their identity IOVAs remain SART-authorized and the + * firmware requests that the new owner map, rather than replace, + * them. Never accept this path for an ordinary Linux-owned session. + */ + if (!anv->owns_rtkit || + region_intersects(bfr->iova, bfr->size, + IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE) != + REGION_INTERSECTS) + return -EINVAL; + + bfr->buffer = memremap(bfr->iova, bfr->size, MEMREMAP_WB); + if (!bfr->buffer) + return -ENOMEM; + bfr->is_mapped = true; + bfr->private = anv; + return 0; + } bfr->buffer = dma_alloc_coherent(anv->dev, bfr->size, &bfr->iova, GFP_KERNEL); @@ -292,6 +327,11 @@ static void apple_nvme_sart_dma_destroy(void *cookie, { struct apple_nvme *anv = cookie; + if (bfr->private == anv) { + memunmap(bfr->buffer); + return; + } + apple_sart_remove_allowed_region(anv->sart, bfr->iova, bfr->size); dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova); } @@ -343,6 +383,7 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, u32 tag = nvme_tag_from_cid(cmd->common.command_id); struct apple_nvmmu_tcb *tcb = &q->tcbs[tag]; + /* The NVMMU TCB opcode is reserved and macOS always leaves it clear. */ tcb->opcode = 0; tcb->prp1 = cmd->common.dptr.prp1; tcb->prp2 = cmd->common.dptr.prp2; @@ -1157,6 +1198,39 @@ static void apple_nvme_reset_work(struct work_struct *work) goto out; } + /* + * Post-M4 firmware can be handed over live, but cannot renegotiate HELLO + * and EPMAP across boot stages. Adopt the first healthy inherited session; + * subsequent resets use the normal RTKit shutdown/reset path below. + */ + if (!anv->owns_rtkit && anv->hw->needs_ioq_registers && + (readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & + APPLE_ANS_COPROC_CPU_CONTROL_RUN) && + readl(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS) == + APPLE_ANS_BOOT_STATUS_OK) { + ret = apple_rtkit_adopt_running(anv->rtk); + if (ret) + goto out; + dev_info(anv->dev, "adopted inherited post-M4 RTKit session\n"); + + /* + * U-Boot keeps the controller enabled until the mailbox session has + * an owner. Disable it only after adoption, then replace every queue + * pointer below while CC.EN is clear. + */ + anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP); + anv->ctrl.ctrl_config = + readl(anv->mmio_nvme + NVME_REG_CC); + if (anv->ctrl.ctrl_config & NVME_CC_ENABLE) { + ret = nvme_disable_ctrl(&anv->ctrl, false); + if (ret) + goto out; + dev_info(anv->dev, + "disabled inherited post-M4 controller after RTKit adoption\n"); + } + goto rtkit_ready; + } + /* RTKit must be shut down cleanly for the (soft)-reset to work */ if (apple_rtkit_is_running(anv->rtk)) { /* reset the controller if it is enabled */ @@ -1202,6 +1276,10 @@ static void apple_nvme_reset_work(struct work_struct *work) goto out; } +rtkit_ready: + anv->owns_rtkit = true; + dev_dbg(anv->dev, "RTKit ownership established\n"); + ret = readl_poll_timeout(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS, boot_status, boot_status == APPLE_ANS_BOOT_STATUS_OK, @@ -1210,6 +1288,7 @@ static void apple_nvme_reset_work(struct work_struct *work) dev_err(anv->dev, "ANS did not initialize"); goto out; } + dev_dbg(anv->dev, "firmware ready\n"); dev_dbg(anv->dev, "ANS booted successfully."); @@ -1264,15 +1343,18 @@ static void apple_nvme_reset_work(struct work_struct *work) aqa = APPLE_NVME_AQ_DEPTH - 1; aqa |= aqa << 16; writel(aqa, anv->mmio_nvme + NVME_REG_AQA); - writeq(anv->adminq.sq_dma_addr, anv->mmio_nvme + NVME_REG_ASQ); - writeq(anv->adminq.cq_dma_addr, anv->mmio_nvme + NVME_REG_ACQ); + apple_nvme_writeq(anv, anv->adminq.sq_dma_addr, + anv->mmio_nvme + NVME_REG_ASQ); + apple_nvme_writeq(anv, anv->adminq.cq_dma_addr, + anv->mmio_nvme + NVME_REG_ACQ); + dev_dbg(anv->dev, "admin queue programmed\n"); if (anv->hw->has_lsq_nvmmu) { /* Setup NVMMU for both queues */ - writeq(anv->adminq.tcb_dma_addr, - anv->mmio_nvmmu + APPLE_NVMMU_ASQ_TCB_BASE); - writeq(anv->ioq.tcb_dma_addr, - anv->mmio_nvmmu + APPLE_NVMMU_IOSQ_TCB_BASE); + apple_nvme_writeq(anv, anv->adminq.tcb_dma_addr, + anv->mmio_nvmmu + APPLE_NVMMU_ASQ_TCB_BASE); + apple_nvme_writeq(anv, anv->ioq.tcb_dma_addr, + anv->mmio_nvmmu + APPLE_NVMMU_IOSQ_TCB_BASE); } anv->ctrl.sqsize = @@ -1283,6 +1365,7 @@ static void apple_nvme_reset_work(struct work_struct *work) ret = nvme_enable_ctrl(&anv->ctrl); if (ret) goto out; + dev_dbg(anv->dev, "controller enabled\n"); dev_dbg(anv->dev, "Starting admin queue"); apple_nvme_init_queue(&anv->adminq); @@ -1298,22 +1381,25 @@ static void apple_nvme_reset_work(struct work_struct *work) ret = nvme_init_ctrl_finish(&anv->ctrl, false); if (ret) goto out; + dev_dbg(anv->dev, "identify completed\n"); dev_dbg(anv->dev, "Creating IOCQ"); ret = apple_nvme_create_cq(anv); if (ret) goto out; + dev_dbg(anv->dev, "I/O CQ created\n"); dev_dbg(anv->dev, "Creating IOSQ"); ret = apple_nvme_create_sq(anv); if (ret) goto out_remove_cq; + dev_dbg(anv->dev, "I/O SQ created\n"); apple_nvme_init_queue(&anv->ioq); if (anv->hw->needs_ioq_registers) { writeq(anv->ioq.cq_dma_addr, - anv->mmio_nvme + APPLE_ANS_IOCQ_REGISTER); + anv->mmio_nvme + APPLE_ANS_IOCQ_REGISTER); writeq(anv->ioq.sq_dma_addr, - anv->mmio_nvme + APPLE_ANS_IOSQ_REGISTER); + anv->mmio_nvme + APPLE_ANS_IOSQ_REGISTER); } nr_io_queues = 1; if (anv->hw->has_queue_count) { diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index c82065a8bf7b0..ccb15bf3afcb2 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -39,6 +39,7 @@ struct apple_rtkit { unsigned int iop_power_state; unsigned int ap_power_state; bool crashed; + bool adopted; DECLARE_BITMAP(endpoints, APPLE_RTKIT_MAX_ENDPOINTS); diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index ef0c399ea8523..5f7c39a6fbc36 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -445,9 +445,9 @@ static void apple_rtkit_syslog_rx_log(struct apple_rtkit *rtk, u64 msg) int msglen; if (!rtk->syslog_msg_buffer) { - dev_warn( - rtk->dev, - "RTKit: received syslog message but no syslog_msg_buffer\n"); + if (!rtk->adopted) + dev_warn(rtk->dev, + "RTKit: received syslog message but no syslog_msg_buffer\n"); goto done; } if (!rtk->syslog_buffer.size) { @@ -752,6 +752,7 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) rtk->syslog_msg_buffer = NULL; rtk->syslog_n_entries = 0; rtk->syslog_msg_size = 0; + rtk->adopted = false; bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints); @@ -839,6 +840,33 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_boot); +int apple_rtkit_adopt_running(struct apple_rtkit *rtk) +{ + static const u8 system_endpoints[] = { + APPLE_RTKIT_EP_CRASHLOG, + APPLE_RTKIT_EP_SYSLOG, + APPLE_RTKIT_EP_DEBUG, + APPLE_RTKIT_EP_IOREPORT, + APPLE_RTKIT_EP_OSLOG, + APPLE_RTKIT_EP_TRACEKIT, + }; + int i; + + if (rtk->crashed) + return -EINVAL; + + /* The previous owner already completed and acknowledged this EPMAP. */ + for (i = 0; i < ARRAY_SIZE(system_endpoints); i++) + set_bit(system_endpoints[i], rtk->endpoints); + + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_ON; + rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_ON; + rtk->adopted = true; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_adopt_running); + int apple_rtkit_shutdown(struct apple_rtkit *rtk) { int ret; diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 42a0666e76fcc..b180ab15c2fc0 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -117,6 +117,12 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk); */ int apple_rtkit_boot(struct apple_rtkit *rtk); +/* + * Adopt a co-processor session that a previous boot stage left fully running. + * The caller must verify the device-specific running and ready indicators. + */ +int apple_rtkit_adopt_running(struct apple_rtkit *rtk); + /* * Quiesce the co-processor. */ From 524777f73758fe851d8b9ba595fcf1a9c3c106b4 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 16 Aug 2026 02:36:50 +0000 Subject: [PATCH 0126/1579] nvme-apple: preserve native NVMMU queue-base writes Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: c9e602dd1a66205cbb0e5090361c834e4a66b204 Ported-paths: drivers/nvme/host/apple.c --- drivers/nvme/host/apple.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index c78016983a0a4..b7620f084b180 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -222,14 +222,14 @@ struct apple_nvme { struct blk_crypto_profile crypto_profile; }; -static inline void apple_nvme_writeq(struct apple_nvme *anv, u64 value, - void __iomem *addr) +static inline void apple_nvme_write_adminq_addr(struct apple_nvme *anv, + u64 value, void __iomem *addr) { /* - * Post-M4 ANS consumes the admin and NVMMU queue addresses as paired - * 32-bit registers while the controller is disabled. Match the access - * sequence used by Apple firmware, m1n1, and U-Boot; older ANS - * generations retain their native 64-bit access. + * Post-M4 ANS consumes ASQ/ACQ as paired 32-bit registers while the + * controller is disabled. The NVMMU TCB-base registers are different: + * they require the native 64-bit write used below. Using this helper for + * the TCB bases lets the controller start but corrupts subsequent I/O. */ if (anv->hw->needs_ioq_registers) lo_hi_writeq(value, addr); @@ -1343,18 +1343,18 @@ static void apple_nvme_reset_work(struct work_struct *work) aqa = APPLE_NVME_AQ_DEPTH - 1; aqa |= aqa << 16; writel(aqa, anv->mmio_nvme + NVME_REG_AQA); - apple_nvme_writeq(anv, anv->adminq.sq_dma_addr, - anv->mmio_nvme + NVME_REG_ASQ); - apple_nvme_writeq(anv, anv->adminq.cq_dma_addr, - anv->mmio_nvme + NVME_REG_ACQ); + apple_nvme_write_adminq_addr(anv, anv->adminq.sq_dma_addr, + anv->mmio_nvme + NVME_REG_ASQ); + apple_nvme_write_adminq_addr(anv, anv->adminq.cq_dma_addr, + anv->mmio_nvme + NVME_REG_ACQ); dev_dbg(anv->dev, "admin queue programmed\n"); if (anv->hw->has_lsq_nvmmu) { /* Setup NVMMU for both queues */ - apple_nvme_writeq(anv, anv->adminq.tcb_dma_addr, - anv->mmio_nvmmu + APPLE_NVMMU_ASQ_TCB_BASE); - apple_nvme_writeq(anv, anv->ioq.tcb_dma_addr, - anv->mmio_nvmmu + APPLE_NVMMU_IOSQ_TCB_BASE); + writeq(anv->adminq.tcb_dma_addr, + anv->mmio_nvmmu + APPLE_NVMMU_ASQ_TCB_BASE); + writeq(anv->ioq.tcb_dma_addr, + anv->mmio_nvmmu + APPLE_NVMMU_IOSQ_TCB_BASE); } anv->ctrl.sqsize = From 89dc129d2dcf99975e48dd7afef4e5c1c260c570 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 16 Aug 2026 05:49:52 +0000 Subject: [PATCH 0127/1579] Revert "nvme-apple: preserve native NVMMU queue-base writes" This reverts commit b86f250757e01a458aee70d892606e58748f769a. Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: ce1fb5191e1d9c519f92d27841ea075d4457180b Ported-paths: drivers/nvme/host/apple.c --- drivers/nvme/host/apple.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index b7620f084b180..c78016983a0a4 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -222,14 +222,14 @@ struct apple_nvme { struct blk_crypto_profile crypto_profile; }; -static inline void apple_nvme_write_adminq_addr(struct apple_nvme *anv, - u64 value, void __iomem *addr) +static inline void apple_nvme_writeq(struct apple_nvme *anv, u64 value, + void __iomem *addr) { /* - * Post-M4 ANS consumes ASQ/ACQ as paired 32-bit registers while the - * controller is disabled. The NVMMU TCB-base registers are different: - * they require the native 64-bit write used below. Using this helper for - * the TCB bases lets the controller start but corrupts subsequent I/O. + * Post-M4 ANS consumes the admin and NVMMU queue addresses as paired + * 32-bit registers while the controller is disabled. Match the access + * sequence used by Apple firmware, m1n1, and U-Boot; older ANS + * generations retain their native 64-bit access. */ if (anv->hw->needs_ioq_registers) lo_hi_writeq(value, addr); @@ -1343,18 +1343,18 @@ static void apple_nvme_reset_work(struct work_struct *work) aqa = APPLE_NVME_AQ_DEPTH - 1; aqa |= aqa << 16; writel(aqa, anv->mmio_nvme + NVME_REG_AQA); - apple_nvme_write_adminq_addr(anv, anv->adminq.sq_dma_addr, - anv->mmio_nvme + NVME_REG_ASQ); - apple_nvme_write_adminq_addr(anv, anv->adminq.cq_dma_addr, - anv->mmio_nvme + NVME_REG_ACQ); + apple_nvme_writeq(anv, anv->adminq.sq_dma_addr, + anv->mmio_nvme + NVME_REG_ASQ); + apple_nvme_writeq(anv, anv->adminq.cq_dma_addr, + anv->mmio_nvme + NVME_REG_ACQ); dev_dbg(anv->dev, "admin queue programmed\n"); if (anv->hw->has_lsq_nvmmu) { /* Setup NVMMU for both queues */ - writeq(anv->adminq.tcb_dma_addr, - anv->mmio_nvmmu + APPLE_NVMMU_ASQ_TCB_BASE); - writeq(anv->ioq.tcb_dma_addr, - anv->mmio_nvmmu + APPLE_NVMMU_IOSQ_TCB_BASE); + apple_nvme_writeq(anv, anv->adminq.tcb_dma_addr, + anv->mmio_nvmmu + APPLE_NVMMU_ASQ_TCB_BASE); + apple_nvme_writeq(anv, anv->ioq.tcb_dma_addr, + anv->mmio_nvmmu + APPLE_NVMMU_IOSQ_TCB_BASE); } anv->ctrl.sqsize = From fa53ff5e41e3a26ee5b08d8612f80f2124bc1c59 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:41:01 +0000 Subject: [PATCH 0128/1579] arm64: apple: stage qualified J700 bring-up Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: fb87b4d8861697c9903034247b6862f59a3475f8 Ported-paths: drivers/nvme/host/apple.c, drivers/soc/apple/rtkit.c, include/linux/soc/apple/rtkit.h --- drivers/nvme/host/apple.c | 161 +++++++++++++++++++++++++------- drivers/soc/apple/rtkit.c | 81 ++++++++++++---- include/linux/soc/apple/rtkit.h | 12 +++ 3 files changed, 198 insertions(+), 56 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index c78016983a0a4..d48813c77ca83 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -85,6 +85,10 @@ #define NVME_MAX_KB_SZ 4096 #define NVME_MAX_SEGS 127 +#define APPLE_NVME_INHERITED_RTKIT_RANGES_PROP \ + "apple,inherited-rtkit-buffer-ranges" +#define APPLE_NVME_MAX_INHERITED_RTKIT_RANGES 8 + /* * This controller comes with an embedded IOMMU known as NVMMU. * The NVMMU is pointed to an array of TCBs indexed by the command tag. @@ -278,10 +282,38 @@ static void apple_nvme_rtkit_crashed(void *cookie, const void *crashlog, size_t nvme_reset_ctrl(&anv->ctrl); } +static bool apple_nvme_in_inherited_rtkit_range(struct apple_nvme *anv, + struct apple_rtkit_shmem *bfr) +{ + u64 ranges[APPLE_NVME_MAX_INHERITED_RTKIT_RANGES * 2]; + int count; + + count = of_property_count_u64_elems(anv->dev->of_node, + APPLE_NVME_INHERITED_RTKIT_RANGES_PROP); + if (count < 2 || count > ARRAY_SIZE(ranges) || count % 2) + return false; + if (of_property_read_u64_array(anv->dev->of_node, + APPLE_NVME_INHERITED_RTKIT_RANGES_PROP, + ranges, count)) + return false; + + for (int i = 0; i < count; i += 2) { + u64 start = ranges[i]; + u64 size = ranges[i + 1]; + + if (bfr->iova >= start && bfr->size <= size && + bfr->iova - start <= size - bfr->size) + return true; + } + + return false; +} + static int apple_nvme_sart_dma_setup(void *cookie, struct apple_rtkit_shmem *bfr) { struct apple_nvme *anv = cookie; + bool inherited_shared; int ret; if (!bfr->size) @@ -292,19 +324,34 @@ static int apple_nvme_sart_dma_setup(void *cookie, * Stage 1. Their identity IOVAs remain SART-authorized and the * firmware requests that the new owner map, rather than replace, * them. Never accept this path for an ordinary Linux-owned session. + * + * A resident Stage 1 may deliberately live outside the RAM exposed + * to its Linux guest. Such a buffer is safe to retain only when the + * resident owner explicitly grants its exact SART range in the DT and + * maps that range into the guest. Retained post-M4 firmware does not + * reliably resume admin I/O after this buffer is replaced. */ - if (!anv->owns_rtkit || - region_intersects(bfr->iova, bfr->size, - IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE) != - REGION_INTERSECTS) + if (!anv->owns_rtkit) return -EINVAL; - bfr->buffer = memremap(bfr->iova, bfr->size, MEMREMAP_WB); - if (!bfr->buffer) - return -ENOMEM; - bfr->is_mapped = true; - bfr->private = anv; - return 0; + inherited_shared = apple_nvme_in_inherited_rtkit_range(anv, bfr); + if (region_intersects(bfr->iova, bfr->size, + IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE) == + REGION_INTERSECTS || + inherited_shared) { + bfr->buffer = memremap(bfr->iova, bfr->size, MEMREMAP_WB); + if (!bfr->buffer) + return -ENOMEM; + bfr->is_mapped = true; + bfr->private = anv; + if (inherited_shared) + dev_info(anv->dev, + "mapping resident-EL2 inherited RTKit buffer: %pad+0x%zx\n", + &bfr->iova, bfr->size); + return 0; + } + + return -EINVAL; } bfr->buffer = @@ -415,6 +462,21 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, } memcpy(&q->sqes[tag], cmd, sizeof(*cmd)); + if (!q->is_adminq) { + dma_addr_t sqe_dma = q->sq_dma_addr + tag * sizeof(*cmd); + dma_addr_t tcb_dma = q->tcb_dma_addr + tag * sizeof(*tcb); + + dev_info_once(anv->dev, + "first I/O submit: tag=%u opcode=0x%02x SQE=%pad TCB=%pad PRP1=0x%016llx PRP2=0x%016llx len=0x%04x flags=0x%02x\n", + tag, cmd->common.opcode, &sqe_dma, &tcb_dma, + le64_to_cpu(cmd->common.dptr.prp1), + le64_to_cpu(cmd->common.dptr.prp2), + le16_to_cpu(tcb->length), tcb->dma_flags); + } + + /* Make the SQE and its NVMMU TCB visible before ANS sees the tag. */ + dma_wmb(); + /* * This lock here doesn't make much sense at a first glance but * removing it will result in occasional missed completion @@ -1072,10 +1134,30 @@ static enum blk_eh_timer_return apple_nvme_timeout(struct request *req) struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); struct apple_nvme_queue *q = iod->q; struct apple_nvme *anv = queue_to_apple_nvme(q); + struct nvme_completion *cqe = &q->cqes[q->cq_head]; + enum nvme_ctrl_state state = nvme_ctrl_state(&anv->ctrl); unsigned long flags; + u32 cc = readl(anv->mmio_nvme + NVME_REG_CC); u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS); - if (nvme_ctrl_state(&anv->ctrl) != NVME_CTRL_LIVE) { + /* + * The first Identify runs while the controller is CONNECTING. Poll the + * completion queue before applying the non-live fast-fail rule so a lost + * interrupt cannot turn a completed initialization command into a reset. + */ + if (!apple_rtkit_is_crashed(anv->rtk) && !(csts & NVME_CSTS_CFS)) { + spin_lock_irqsave(&anv->lock, flags); + apple_nvme_handle_cq(q, false); + spin_unlock_irqrestore(&anv->lock, flags); + if (blk_mq_request_completed(req)) { + dev_warn(anv->dev, + "I/O %d(aq:%d) timeout in state %d: completion polled\n", + req->tag, q->is_adminq, state); + return BLK_EH_DONE; + } + } + + if (state != NVME_CTRL_LIVE) { /* * From rdma.c: * If we are resetting, connecting or deleting we should @@ -1091,8 +1173,10 @@ static enum blk_eh_timer_return apple_nvme_timeout(struct request *req) * recovery work, so it's fine that we fail it here. */ dev_warn(anv->dev, - "I/O %d(aq:%d) timeout while not in live state\n", - req->tag, q->is_adminq); + "I/O %d(aq:%d) timeout in state %d CC=0x%08x CSTS=0x%08x CQ=%u/%u status=0x%04x cid=%u\n", + req->tag, q->is_adminq, state, cc, csts, q->cq_head, + q->cq_phase, le16_to_cpu(READ_ONCE(cqe->status)), + le16_to_cpu(READ_ONCE(cqe->command_id))); if (blk_mq_request_started(req) && !blk_mq_request_completed(req)) { nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD; @@ -1102,25 +1186,15 @@ static enum blk_eh_timer_return apple_nvme_timeout(struct request *req) return BLK_EH_DONE; } - /* check if we just missed an interrupt if we're still alive */ - if (!apple_rtkit_is_crashed(anv->rtk) && !(csts & NVME_CSTS_CFS)) { - spin_lock_irqsave(&anv->lock, flags); - apple_nvme_handle_cq(q, false); - spin_unlock_irqrestore(&anv->lock, flags); - if (blk_mq_request_completed(req)) { - dev_warn(anv->dev, - "I/O %d(aq:%d) timeout: completion polled\n", - req->tag, q->is_adminq); - return BLK_EH_DONE; - } - } - /* * aborting commands isn't supported which leaves a full reset as our * only option here */ - dev_warn(anv->dev, "I/O %d(aq:%d) timeout: resetting controller\n", - req->tag, q->is_adminq); + dev_warn(anv->dev, + "I/O %d(aq:%d) timeout: resetting controller CC=0x%08x CSTS=0x%08x CQ=%u/%u status=0x%04x cid=%u\n", + req->tag, q->is_adminq, cc, csts, q->cq_head, q->cq_phase, + le16_to_cpu(READ_ONCE(cqe->status)), + le16_to_cpu(READ_ONCE(cqe->command_id))); nvme_req(req)->flags |= NVME_REQ_CANCELLED; apple_nvme_disable(anv, false); nvme_reset_ctrl(&anv->ctrl); @@ -1396,10 +1470,10 @@ static void apple_nvme_reset_work(struct work_struct *work) apple_nvme_init_queue(&anv->ioq); if (anv->hw->needs_ioq_registers) { - writeq(anv->ioq.cq_dma_addr, - anv->mmio_nvme + APPLE_ANS_IOCQ_REGISTER); - writeq(anv->ioq.sq_dma_addr, - anv->mmio_nvme + APPLE_ANS_IOSQ_REGISTER); + apple_nvme_writeq(anv, anv->ioq.cq_dma_addr, + anv->mmio_nvme + APPLE_ANS_IOCQ_REGISTER); + apple_nvme_writeq(anv, anv->ioq.sq_dma_addr, + anv->mmio_nvme + APPLE_ANS_IOSQ_REGISTER); } nr_io_queues = 1; if (anv->hw->has_queue_count) { @@ -1688,6 +1762,7 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct apple_nvme *anv; + bool inherited_rtkit; int ret; anv = devm_kzalloc(dev, sizeof(*anv), GFP_KERNEL); @@ -1746,9 +1821,9 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) } if (anv->hw->has_lsq_nvmmu) { - anv->adminq.sq_db = anv->mmio_nvmmu + APPLE_ANS_LINEAR_ASQ_DB; + anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB; anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB; - anv->ioq.sq_db = anv->mmio_nvmmu + APPLE_ANS_LINEAR_IOSQ_DB; + anv->ioq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_IOSQ_DB; anv->ioq.cq_db = anv->mmio_nvme + APPLE_ANS_IOCQ_DB; } else { anv->adminq.sq_db = anv->mmio_nvme + NVME_REG_DBS; @@ -1820,8 +1895,22 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) goto put_dev; } - anv->rtk = - devm_apple_rtkit_init(dev, anv, NULL, 0, &apple_nvme_rtkit_ops); + /* + * A post-M4 boot stage may leave RTKit fully running. Prime the known + * system endpoints before mailbox RX starts, otherwise inherited syslog + * traffic can race the later reset work and be dropped as undiscovered. + */ + inherited_rtkit = anv->hw->needs_ioq_registers && + (readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & + APPLE_ANS_COPROC_CPU_CONTROL_RUN) && + readl(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS) == + APPLE_ANS_BOOT_STATUS_OK; + if (inherited_rtkit) + anv->rtk = devm_apple_rtkit_init_adopted( + dev, anv, NULL, 0, &apple_nvme_rtkit_ops); + else + anv->rtk = devm_apple_rtkit_init( + dev, anv, NULL, 0, &apple_nvme_rtkit_ops); if (IS_ERR(anv->rtk)) { ret = dev_err_probe(dev, PTR_ERR(anv->rtk), "Failed to initialize RTKit"); diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 5f7c39a6fbc36..e4e68ff71a04a 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -663,9 +663,30 @@ int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) } EXPORT_SYMBOL_GPL(apple_rtkit_start_ep); -struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, - const char *mbox_name, int mbox_idx, - const struct apple_rtkit_ops *ops) +static void apple_rtkit_mark_running(struct apple_rtkit *rtk) +{ + static const u8 system_endpoints[] = { + APPLE_RTKIT_EP_CRASHLOG, + APPLE_RTKIT_EP_SYSLOG, + APPLE_RTKIT_EP_DEBUG, + APPLE_RTKIT_EP_IOREPORT, + APPLE_RTKIT_EP_OSLOG, + APPLE_RTKIT_EP_TRACEKIT, + }; + int i; + + /* The previous owner already completed and acknowledged this EPMAP. */ + for (i = 0; i < ARRAY_SIZE(system_endpoints); i++) + set_bit(system_endpoints[i], rtk->endpoints); + + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_ON; + rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_ON; + rtk->adopted = true; +} + +static struct apple_rtkit *__apple_rtkit_init( + struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, + const struct apple_rtkit_ops *ops, bool adopted) { struct apple_rtkit *rtk; int ret; @@ -707,6 +728,8 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, ret = -ENOMEM; goto free_rtk; } + if (adopted) + apple_rtkit_mark_running(rtk); ret = apple_mbox_start(rtk->mbox); if (ret) @@ -720,8 +743,23 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, kfree(rtk); return ERR_PTR(ret); } + +struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, + const char *mbox_name, int mbox_idx, + const struct apple_rtkit_ops *ops) +{ + return __apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops, false); +} EXPORT_SYMBOL_GPL(apple_rtkit_init); +struct apple_rtkit *apple_rtkit_init_adopted( + struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, + const struct apple_rtkit_ops *ops) +{ + return __apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops, true); +} +EXPORT_SYMBOL_GPL(apple_rtkit_init_adopted); + static int apple_rtkit_wait_for_completion(struct completion *c) { long t; @@ -842,26 +880,10 @@ EXPORT_SYMBOL_GPL(apple_rtkit_boot); int apple_rtkit_adopt_running(struct apple_rtkit *rtk) { - static const u8 system_endpoints[] = { - APPLE_RTKIT_EP_CRASHLOG, - APPLE_RTKIT_EP_SYSLOG, - APPLE_RTKIT_EP_DEBUG, - APPLE_RTKIT_EP_IOREPORT, - APPLE_RTKIT_EP_OSLOG, - APPLE_RTKIT_EP_TRACEKIT, - }; - int i; - if (rtk->crashed) return -EINVAL; - /* The previous owner already completed and acknowledged this EPMAP. */ - for (i = 0; i < ARRAY_SIZE(system_endpoints); i++) - set_bit(system_endpoints[i], rtk->endpoints); - - rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_ON; - rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_ON; - rtk->adopted = true; + apple_rtkit_mark_running(rtk); return 0; } @@ -1008,6 +1030,25 @@ struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie, } EXPORT_SYMBOL_GPL(devm_apple_rtkit_init); +struct apple_rtkit *devm_apple_rtkit_init_adopted( + struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, + const struct apple_rtkit_ops *ops) +{ + struct apple_rtkit *rtk; + int ret; + + rtk = apple_rtkit_init_adopted(dev, cookie, mbox_name, mbox_idx, ops); + if (IS_ERR(rtk)) + return rtk; + + ret = devm_add_action_or_reset(dev, apple_rtkit_free_wrapper, rtk); + if (ret) + return ERR_PTR(ret); + + return rtk; +} +EXPORT_SYMBOL_GPL(devm_apple_rtkit_init_adopted); + void devm_apple_rtkit_free(struct device *dev, struct apple_rtkit *rtk) { devm_release_action(dev, apple_rtkit_free_wrapper, rtk); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index b180ab15c2fc0..6627a24972966 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -78,6 +78,14 @@ struct apple_rtkit; struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops); + +/* + * Initialize RTKit for a session that a previous boot stage left running. + * System endpoints are marked before mailbox receive processing starts. + */ +struct apple_rtkit *devm_apple_rtkit_init_adopted( + struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, + const struct apple_rtkit_ops *ops); /* * Frees internal RTKit state allocated by devm_apple_rtkit_init(). * @@ -100,6 +108,10 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops); +struct apple_rtkit *apple_rtkit_init_adopted( + struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, + const struct apple_rtkit_ops *ops); + /* * Free an instance of apple_rtkit. */ From ff49e72490c3bbc44e356bd2822e827babe34e0e Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:58:37 +0000 Subject: [PATCH 0129/1579] wifi: preserve J700 MT7932 calibrated bring-up Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: c316fdaa5c8ba29bf88e61c87efc703f0610d0e0 Ported-paths: drivers/nvme/host/apple.c --- drivers/nvme/host/apple.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index d48813c77ca83..23bee6675e2cc 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -176,7 +176,8 @@ struct apple_nvme_iod { struct apple_nvme_hw { bool has_lsq_nvmmu; - bool has_legacy_lsq_ctrl; + bool has_linear_sq_ctrl; + bool has_prp_null_ctrl; bool has_queue_count; bool has_separate_nvmmu; bool needs_ioq_registers; @@ -1381,9 +1382,24 @@ static void apple_nvme_reset_work(struct work_struct *work) * Enable NVMMU and linear submission queues which is required * since T6000. */ - if (anv->hw->has_legacy_lsq_ctrl) + if (anv->hw->has_linear_sq_ctrl) { + u32 lsq_ctrl; + writel(APPLE_ANS_LINEAR_SQ_EN, anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); + lsq_ctrl = readl(anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); + if (!(lsq_ctrl & APPLE_ANS_LINEAR_SQ_EN)) { + dev_err(anv->dev, + "failed to enable linear submission queues: LSQ_CTRL=0x%08x\n", + lsq_ctrl); + ret = -EIO; + goto out; + } + if (anv->hw->needs_ioq_registers) + dev_info(anv->dev, + "post-M4 linear submission queues enabled: LSQ_CTRL=0x%08x\n", + lsq_ctrl); + } /* * The legacy register takes queue entry counts. On post-M4 @@ -2077,13 +2093,15 @@ static const struct apple_nvme_hw apple_nvme_t8015_hw = { static const struct apple_nvme_hw apple_nvme_t8103_hw = { .has_lsq_nvmmu = true, - .has_legacy_lsq_ctrl = true, + .has_linear_sq_ctrl = true, + .has_prp_null_ctrl = true, .has_queue_count = true, .max_queue_depth = 64, }; static const struct apple_nvme_hw apple_nvme_t8132_hw = { .has_lsq_nvmmu = true, + .has_linear_sq_ctrl = true, .has_separate_nvmmu = true, .needs_ioq_registers = true, .max_queue_depth = 64, From cf736bcbcb26c2e7ad76a65c2e3b4d37101932f3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 26 Aug 2026 02:11:45 +0000 Subject: [PATCH 0130/1579] soc: apple: mailbox: support T8140 ASCWrap v6 Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: f96042637269a3f64542ac11c03019ce1c52bd41 Ported-paths: drivers/soc/apple/mailbox.c --- drivers/soc/apple/mailbox.c | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 330015eea2ebe..f352deb081760 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -47,6 +47,18 @@ #define APPLE_ASC_MBOX_I2A_RECV0 0x830 #define APPLE_ASC_MBOX_I2A_RECV1 0x838 +/* ASCWrap v6 embeds the unchanged ASC mailbox block at this offset. */ +#define APPLE_ASCWRAP_V6_MBOX_OFFSET 0x8000 + +static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == + 0x8110); +static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_CONTROL == + 0x8114); +static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_SEND0 == + 0x8800); +static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == + 0x8830); + #define APPLE_T8015_MBOX_A2I_CONTROL 0x108 #define APPLE_T8015_MBOX_I2A_CONTROL 0x10c @@ -81,6 +93,8 @@ #define APPLE_MBOX_TX_TIMEOUT 500 struct apple_mbox_hw { + unsigned int reg_offset; + unsigned int control_full; unsigned int control_empty; @@ -337,6 +351,7 @@ static int apple_mbox_probe(struct platform_device *pdev) char *irqname; struct apple_mbox *mbox; struct device *dev = &pdev->dev; + struct resource *res; mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); if (!mbox) @@ -347,9 +362,17 @@ static int apple_mbox_probe(struct platform_device *pdev) if (!mbox->hw) return -EINVAL; + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res || resource_size(res) < mbox->hw->reg_offset + + mbox->hw->i2a_recv1 + sizeof(u64)) + return dev_err_probe(dev, -EINVAL, + "mailbox register window is too small\n"); + mbox->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(mbox->regs)) return PTR_ERR(mbox->regs); + /* Resource 1, when present, is ASC lifecycle state, not mailbox MMIO. */ + mbox->regs = (u8 __iomem *)mbox->regs + mbox->hw->reg_offset; mbox->irq_recv_not_empty = platform_get_irq_byname(pdev, "recv-not-empty"); @@ -422,6 +445,23 @@ static const struct apple_mbox_hw apple_mbox_asc_hw = { .has_irq_controls = false, }; +static const struct apple_mbox_hw apple_mbox_ascwrap_v6_hw = { + .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + + .has_irq_controls = false, +}; + static const struct apple_mbox_hw apple_mbox_m3_hw = { .control_full = APPLE_M3_MBOX_CONTROL_FULL, .control_empty = APPLE_M3_MBOX_CONTROL_EMPTY, @@ -442,6 +482,7 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { }; static const struct of_device_id apple_mbox_of_match[] = { + { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_ascwrap_v6_hw }, { .compatible = "apple,asc-mailbox-v4", .data = &apple_mbox_asc_hw }, { .compatible = "apple,t8015-asc-mailbox", .data = &apple_mbox_t8015_hw }, { .compatible = "apple,m3-mailbox-v2", .data = &apple_mbox_m3_hw }, From 1753c0e571f0370f949fca6abe021fe72ed3eaae Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 19:10:16 +0000 Subject: [PATCH 0131/1579] drm/asahi: gate G17P RTKit on reversible ASC lifecycle Move ASCWrap v6 resource ownership into apple-mailbox, add a passive HELLO/EPMAP-only RTKit API with exact raw snapshots, and require an exact dual-provider topology token before G17P target effects. Keep live boot and its transactional CPU executor disabled until provider-owned stop/reset exists, with focused fail-closed and reverse-unwind tests. Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: d916d44fd1d7dd18b98aa4f63f22ebd858988379 Ported-paths: drivers/soc/apple/rtkit.c, drivers/soc/apple/mailbox.c, drivers/soc/apple/rtkit-internal.h, include/linux/soc/apple/rtkit.h, include/linux/soc/apple/mailbox.h --- drivers/soc/apple/mailbox.c | 124 +++++++++++++++-- drivers/soc/apple/rtkit-internal.h | 9 ++ drivers/soc/apple/rtkit.c | 214 +++++++++++++++++++++++++++-- include/linux/soc/apple/mailbox.h | 52 +++++++ include/linux/soc/apple/rtkit.h | 36 +++++ 5 files changed, 411 insertions(+), 24 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index f352deb081760..b95905292197f 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -49,6 +49,13 @@ /* ASCWrap v6 embeds the unchanged ASC mailbox block at this offset. */ #define APPLE_ASCWRAP_V6_MBOX_OFFSET 0x8000 +#define APPLE_ASCWRAP_V6_WRAPPER_SIZE 0x88000 +#define APPLE_ASCWRAP_V6_IOP_VBAR_SIZE 0x8 + +#define APPLE_T8140_GFX_WRAPPER_BASE 0x482600000ULL +#define APPLE_T8140_GFX_IOP_VBAR_BASE 0x482050000ULL +#define APPLE_T8140_GFX1_WRAPPER_BASE 0x482e00000ULL +#define APPLE_T8140_GFX1_IOP_VBAR_BASE 0x482850000ULL static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == 0x8110); @@ -94,6 +101,7 @@ static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == struct apple_mbox_hw { unsigned int reg_offset; + bool is_ascwrap_v6; unsigned int control_full; unsigned int control_empty; @@ -113,6 +121,39 @@ struct apple_mbox_hw { unsigned int irq_bit_send_empty; }; +int apple_mbox_ascwrap_v6_get_lifecycle( + struct apple_mbox *mbox, + struct apple_mbox_ascwrap_v6_lifecycle *lifecycle) +{ + if (!mbox || !lifecycle) + return -EINVAL; + if (!mbox->hw->is_ascwrap_v6) + return -EOPNOTSUPP; + + *lifecycle = mbox->ascwrap_v6; + return 0; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_get_lifecycle); + +int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox) +{ + if (!mbox || !mbox->hw->is_ascwrap_v6) + return -EOPNOTSUPP; + + /* + * This is intentionally checked before any CPU_CONTROL or IOP VBAR + * write. Clearing CPU_RUN alone is not the complete AppleA7IOP stop + * transaction, and no grounded hardware reset exists yet. Keep the + * provider as the sole future lifecycle owner without exposing a + * one-way start primitive to consumers. + */ + if (mbox->ascwrap_v6.missing) + return -EOPNOTSUPP; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle); + int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, bool atomic) { @@ -349,9 +390,10 @@ static int apple_mbox_probe(struct platform_device *pdev) { int ret; char *irqname; + const char *firmware_role; struct apple_mbox *mbox; struct device *dev = &pdev->dev; - struct resource *res; + struct resource *res, *iop_vbar_res; mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); if (!mbox) @@ -362,17 +404,76 @@ static int apple_mbox_probe(struct platform_device *pdev) if (!mbox->hw) return -EINVAL; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res || resource_size(res) < mbox->hw->reg_offset + - mbox->hw->i2a_recv1 + sizeof(u64)) - return dev_err_probe(dev, -EINVAL, - "mailbox register window is too small\n"); + if (mbox->hw->is_ascwrap_v6) { + /* + * Validate the entire provider identity before mapping either + * resource. These are fixed T8140 CPU-physical windows and role + * assignments, not generic consumer-provided aliases. + */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "wrapper"); + iop_vbar_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "iop-vbar"); + if (!res || !iop_vbar_res || + resource_size(res) != APPLE_ASCWRAP_V6_WRAPPER_SIZE || + resource_size(iop_vbar_res) != APPLE_ASCWRAP_V6_IOP_VBAR_SIZE) + return dev_err_probe(dev, -EINVAL, + "invalid ASCWrap v6 provider resources\n"); + + ret = of_property_read_string(dev->of_node, "apple,firmware-role", + &firmware_role); + if (ret) + return dev_err_probe(dev, ret, + "missing ASCWrap v6 firmware role\n"); + + if (!strcmp(firmware_role, "GFX")) { + mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; + if (res->start != APPLE_T8140_GFX_WRAPPER_BASE || + iop_vbar_res->start != APPLE_T8140_GFX_IOP_VBAR_BASE) + return dev_err_probe(dev, -EINVAL, + "GFX ASCWrap v6 resources do not match T8140\n"); + } else if (!strcmp(firmware_role, "GFX1")) { + mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1; + if (res->start != APPLE_T8140_GFX1_WRAPPER_BASE || + iop_vbar_res->start != APPLE_T8140_GFX1_IOP_VBAR_BASE) + return dev_err_probe(dev, -EINVAL, + "GFX1 ASCWrap v6 resources do not match T8140\n"); + } else { + return dev_err_probe(dev, -EINVAL, + "invalid ASCWrap v6 firmware role\n"); + } - mbox->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(mbox->regs)) - return PTR_ERR(mbox->regs); - /* Resource 1, when present, is ASC lifecycle state, not mailbox MMIO. */ - mbox->regs = (u8 __iomem *)mbox->regs + mbox->hw->reg_offset; + mbox->ascwrap_v6.wrapper_start = res->start; + mbox->ascwrap_v6.wrapper_size = resource_size(res); + mbox->ascwrap_v6.iop_vbar_start = iop_vbar_res->start; + mbox->ascwrap_v6.iop_vbar_size = resource_size(iop_vbar_res); + mbox->ascwrap_v6.missing = + APPLE_MBOX_ASCWRAP_V6_MISSING_SAFE_STOP | + APPLE_MBOX_ASCWRAP_V6_MISSING_SAFE_RESET; + + mbox->wrapper_regs = + devm_platform_ioremap_resource_byname(pdev, "wrapper"); + if (IS_ERR(mbox->wrapper_regs)) + return PTR_ERR(mbox->wrapper_regs); + + mbox->iop_vbar_regs = + devm_platform_ioremap_resource_byname(pdev, "iop-vbar"); + if (IS_ERR(mbox->iop_vbar_regs)) + return PTR_ERR(mbox->iop_vbar_regs); + + mbox->regs = (u8 __iomem *)mbox->wrapper_regs + + mbox->hw->reg_offset; + } else { + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res || resource_size(res) < mbox->hw->reg_offset + + mbox->hw->i2a_recv1 + sizeof(u64)) + return dev_err_probe(dev, -EINVAL, + "mailbox register window is too small\n"); + + mbox->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(mbox->regs)) + return PTR_ERR(mbox->regs); + mbox->regs = (u8 __iomem *)mbox->regs + mbox->hw->reg_offset; + } mbox->irq_recv_not_empty = platform_get_irq_byname(pdev, "recv-not-empty"); @@ -447,6 +548,7 @@ static const struct apple_mbox_hw apple_mbox_asc_hw = { static const struct apple_mbox_hw apple_mbox_ascwrap_v6_hw = { .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, + .is_ascwrap_v6 = true, .control_full = APPLE_ASC_MBOX_CONTROL_FULL, .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index ccb15bf3afcb2..85ca018aa41e5 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -35,13 +35,22 @@ struct apple_rtkit { int boot_result; int version; + u64 raw_hello; + u16 hello_min_version; + u16 hello_max_version; + u8 epmap_bases_seen; unsigned int iop_power_state; unsigned int ap_power_state; bool crashed; bool adopted; + bool epmap_only; + bool hello_seen; + bool epmap_finished; + bool shutting_down; DECLARE_BITMAP(endpoints, APPLE_RTKIT_MAX_ENDPOINTS); + DECLARE_BITMAP(discovery_endpoints, APPLE_RTKIT_MAX_ENDPOINTS); struct apple_rtkit_shmem ioreport_buffer; struct apple_rtkit_shmem crashlog_buffer; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index e4e68ff71a04a..179902e9d3140 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -6,6 +6,9 @@ #include "rtkit-internal.h" +static_assert(APPLE_RTKIT_EPMAP_SNAPSHOT_WORDS * 64 == + APPLE_RTKIT_MAX_ENDPOINTS); + enum { APPLE_RTKIT_PWR_STATE_OFF = 0x00, /* power off, cannot be restarted */ APPLE_RTKIT_PWR_STATE_SLEEP = 0x01, /* sleeping, can be restarted */ @@ -115,14 +118,37 @@ static int apple_rtkit_management_send(struct apple_rtkit *rtk, u8 type, return ret; } +static void apple_rtkit_epmap_only_abort(struct apple_rtkit *rtk, int error) +{ + if (!rtk->epmap_only || rtk->epmap_finished) + return; + + rtk->boot_result = error; + rtk->epmap_finished = true; + complete_all(&rtk->epmap_completion); +} + static void apple_rtkit_management_rx_hello(struct apple_rtkit *rtk, u64 msg) { u64 reply; + int ret; int min_ver = FIELD_GET(APPLE_RTKIT_MGMT_HELLO_MINVER, msg); int max_ver = FIELD_GET(APPLE_RTKIT_MGMT_HELLO_MAXVER, msg); int want_ver = min(APPLE_RTKIT_MAX_SUPPORTED_VERSION, max_ver); + if (rtk->epmap_only) { + if (rtk->hello_seen || rtk->epmap_finished) { + apple_rtkit_epmap_only_abort(rtk, -EPROTO); + return; + } + + rtk->raw_hello = msg; + rtk->hello_min_version = min_ver; + rtk->hello_max_version = max_ver; + rtk->hello_seen = true; + } + dev_dbg(rtk->dev, "RTKit: Min ver %d, max ver %d\n", min_ver, max_ver); if (min_ver > APPLE_RTKIT_MAX_SUPPORTED_VERSION) { @@ -143,22 +169,38 @@ static void apple_rtkit_management_rx_hello(struct apple_rtkit *rtk, u64 msg) reply = FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MINVER, want_ver); reply |= FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MAXVER, want_ver); - apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_HELLO_REPLY, reply); + ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_HELLO_REPLY, + reply); + if (ret && rtk->epmap_only) + apple_rtkit_epmap_only_abort(rtk, ret); return; abort_boot: - rtk->boot_result = -EINVAL; - complete_all(&rtk->epmap_completion); + if (rtk->epmap_only) { + apple_rtkit_epmap_only_abort(rtk, -EINVAL); + } else { + rtk->boot_result = -EINVAL; + complete_all(&rtk->epmap_completion); + } } static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) { - int i, ep; + int i, ep, ret; u64 reply; unsigned long bitmap = FIELD_GET(APPLE_RTKIT_MGMT_EPMAP_BITMAP, msg); u32 base = FIELD_GET(APPLE_RTKIT_MGMT_EPMAP_BASE, msg); + if (rtk->epmap_only) { + if (!rtk->hello_seen || rtk->epmap_finished || + (rtk->epmap_bases_seen & BIT(base))) { + apple_rtkit_epmap_only_abort(rtk, -EPROTO); + return; + } + rtk->epmap_bases_seen |= BIT(base); + } + dev_dbg(rtk->dev, "RTKit: received endpoint bitmap 0x%lx with base 0x%x\n", bitmap, base); @@ -167,6 +209,8 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) ep = 32 * base + i; dev_dbg(rtk->dev, "RTKit: Discovered endpoint 0x%02x\n", ep); set_bit(ep, rtk->endpoints); + if (rtk->epmap_only) + set_bit(ep, rtk->discovery_endpoints); } reply = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, base); @@ -175,11 +219,23 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) else reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE; - apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, reply); + ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, + reply); + if (ret && rtk->epmap_only) { + apple_rtkit_epmap_only_abort(rtk, ret); + return; + } if (!(msg & APPLE_RTKIT_MGMT_EPMAP_LAST)) return; + if (rtk->epmap_only) { + rtk->boot_result = 0; + rtk->epmap_finished = true; + complete_all(&rtk->epmap_completion); + return; + } + for_each_set_bit(ep, rtk->endpoints, APPLE_RTKIT_APP_ENDPOINT_START) { switch (ep) { /* the management endpoint is started by default */ @@ -237,6 +293,15 @@ static void apple_rtkit_management_rx(struct apple_rtkit *rtk, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_MGMT_TYPE, msg); + if (rtk->epmap_only && type != APPLE_RTKIT_MGMT_HELLO && + type != APPLE_RTKIT_MGMT_EPMAP) { + dev_err(rtk->dev, + "RTKit: discovery-only client received management type 0x%02x\n", + type); + apple_rtkit_epmap_only_abort(rtk, -EPROTO); + return; + } + switch (type) { case APPLE_RTKIT_MGMT_HELLO: apple_rtkit_management_rx_hello(rtk, msg); @@ -531,6 +596,9 @@ static void apple_rtkit_rx_work(struct work_struct *work) container_of(work, struct apple_rtkit_rx_work, work); struct apple_rtkit *rtk = rtk_work->rtk; + if (READ_ONCE(rtk->shutting_down)) + goto out_free; + switch (rtk_work->ep) { case APPLE_RTKIT_EP_MGMT: apple_rtkit_management_rx(rtk, rtk_work->msg); @@ -560,9 +628,10 @@ static void apple_rtkit_rx_work(struct work_struct *work) default: dev_warn(rtk->dev, "RTKit: message to unknown endpoint %02x: %llx\n", - rtk_work->ep, rtk_work->msg); + rtk_work->ep, rtk_work->msg); } +out_free: kfree(rtk_work); } @@ -579,12 +648,27 @@ static void apple_rtkit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, * afterwards. */ dma_rmb(); + if (READ_ONCE(rtk->shutting_down)) + return; if (!test_bit(ep, rtk->endpoints)) dev_warn(rtk->dev, "RTKit: Message to undiscovered endpoint 0x%02x\n", ep); + /* + * Passive discovery owns endpoint zero only. Never queue a crashlog, + * syslog, or application callback from this mode: doing so could allocate + * shared memory or acknowledge a protocol that has not been admitted. + */ + if (rtk->epmap_only && ep != APPLE_RTKIT_EP_MGMT) { + dev_err(rtk->dev, + "RTKit: discovery-only client received endpoint 0x%02x traffic\n", + ep); + apple_rtkit_epmap_only_abort(rtk, -EPROTO); + return; + } + if (ep >= APPLE_RTKIT_APP_ENDPOINT_START && rtk->ops->recv_message_early && rtk->ops->recv_message_early(rtk->cookie, ep, msg.msg0)) @@ -615,6 +699,18 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, return -EINVAL; } + if (rtk->epmap_only) { + u8 type = FIELD_GET(APPLE_RTKIT_MGMT_TYPE, message); + + if (ep != APPLE_RTKIT_EP_MGMT || + (type != APPLE_RTKIT_MGMT_HELLO_REPLY && + type != APPLE_RTKIT_MGMT_EPMAP_REPLY)) { + dev_warn(rtk->dev, + "RTKit: discovery-only client refused endpoint/power side effect\n"); + return -EPERM; + } + } + if (ep >= APPLE_RTKIT_APP_ENDPOINT_START && !apple_rtkit_is_running(rtk)) { dev_warn(rtk->dev, @@ -649,6 +745,9 @@ int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) { u64 msg; + if (rtk->epmap_only) + return -EPERM; + if (!test_bit(endpoint, rtk->endpoints)) return -EINVAL; if (endpoint >= APPLE_RTKIT_APP_ENDPOINT_START && @@ -686,12 +785,12 @@ static void apple_rtkit_mark_running(struct apple_rtkit *rtk) static struct apple_rtkit *__apple_rtkit_init( struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, - const struct apple_rtkit_ops *ops, bool adopted) + const struct apple_rtkit_ops *ops, bool adopted, bool epmap_only) { struct apple_rtkit *rtk; int ret; - if (!ops) + if (!ops || (adopted && epmap_only)) return ERR_PTR(-EINVAL); rtk = kzalloc_obj(*rtk); @@ -701,12 +800,14 @@ static struct apple_rtkit *__apple_rtkit_init( rtk->dev = dev; rtk->cookie = cookie; rtk->ops = ops; + rtk->epmap_only = epmap_only; init_completion(&rtk->epmap_completion); init_completion(&rtk->iop_pwr_ack_completion); init_completion(&rtk->ap_pwr_ack_completion); bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); + bitmap_zero(rtk->discovery_endpoints, APPLE_RTKIT_MAX_ENDPOINTS); set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints); if (mbox_name) @@ -718,9 +819,10 @@ static struct apple_rtkit *__apple_rtkit_init( ret = PTR_ERR(rtk->mbox); goto free_rtk; } - - rtk->mbox->rx = apple_rtkit_rx; - rtk->mbox->cookie = rtk; + if (rtk->mbox->rx) { + ret = -EBUSY; + goto free_rtk; + } rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_HIGHPRI | WQ_MEM_RECLAIM, dev_name(rtk->dev)); @@ -728,6 +830,9 @@ static struct apple_rtkit *__apple_rtkit_init( ret = -ENOMEM; goto free_rtk; } + + rtk->mbox->cookie = rtk; + rtk->mbox->rx = apple_rtkit_rx; if (adopted) apple_rtkit_mark_running(rtk); @@ -738,6 +843,8 @@ static struct apple_rtkit *__apple_rtkit_init( return rtk; destroy_wq: + rtk->mbox->rx = NULL; + rtk->mbox->cookie = NULL; destroy_workqueue(rtk->wq); free_rtk: kfree(rtk); @@ -748,7 +855,8 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops) { - return __apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops, false); + return __apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops, false, + false); } EXPORT_SYMBOL_GPL(apple_rtkit_init); @@ -756,10 +864,20 @@ struct apple_rtkit *apple_rtkit_init_adopted( struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops) { - return __apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops, true); + return __apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops, true, + false); } EXPORT_SYMBOL_GPL(apple_rtkit_init_adopted); +struct apple_rtkit *apple_rtkit_init_epmap_only( + struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, + const struct apple_rtkit_ops *ops) +{ + return __apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops, false, + true); +} +EXPORT_SYMBOL_GPL(apple_rtkit_init_epmap_only); + static int apple_rtkit_wait_for_completion(struct completion *c) { long t; @@ -793,6 +911,7 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) rtk->adopted = false; bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); + bitmap_zero(rtk->discovery_endpoints, APPLE_RTKIT_MAX_ENDPOINTS); set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints); reinit_completion(&rtk->epmap_completion); @@ -800,6 +919,13 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) reinit_completion(&rtk->ap_pwr_ack_completion); rtk->crashed = false; + rtk->boot_result = 0; + rtk->raw_hello = 0; + rtk->hello_min_version = 0; + rtk->hello_max_version = 0; + rtk->epmap_bases_seen = 0; + rtk->hello_seen = false; + rtk->epmap_finished = false; rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_OFF; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_OFF; @@ -813,6 +939,9 @@ static int apple_rtkit_set_ap_power_state(struct apple_rtkit *rtk, u64 msg; int ret; + if (rtk->epmap_only) + return -EPERM; + reinit_completion(&rtk->ap_pwr_ack_completion); msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state); @@ -836,6 +965,9 @@ static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, u64 msg; int ret; + if (rtk->epmap_only) + return -EPERM; + reinit_completion(&rtk->iop_pwr_ack_completion); msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state); @@ -857,6 +989,9 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) { int ret; + if (rtk->epmap_only) + return -EOPNOTSUPP; + if (apple_rtkit_is_running(rtk)) return 0; if (rtk->crashed) @@ -878,8 +1013,47 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_boot); +int apple_rtkit_discover_epmap(struct apple_rtkit *rtk) +{ + int ret; + + if (!rtk->epmap_only) + return -EOPNOTSUPP; + + ret = apple_rtkit_wait_for_completion(&rtk->epmap_completion); + if (ret) + return ret; + + return rtk->boot_result; +} +EXPORT_SYMBOL_GPL(apple_rtkit_discover_epmap); + +int apple_rtkit_get_epmap_snapshot( + struct apple_rtkit *rtk, struct apple_rtkit_epmap_snapshot *snapshot) +{ + if (!rtk->epmap_only) + return -EOPNOTSUPP; + if (!snapshot) + return -EINVAL; + if (!completion_done(&rtk->epmap_completion)) + return -EAGAIN; + if (rtk->boot_result) + return rtk->boot_result; + + snapshot->raw_hello = rtk->raw_hello; + snapshot->hello_min_version = rtk->hello_min_version; + snapshot->hello_max_version = rtk->hello_max_version; + bitmap_to_arr64(snapshot->endpoints, rtk->discovery_endpoints, + APPLE_RTKIT_MAX_ENDPOINTS); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_get_epmap_snapshot); + int apple_rtkit_adopt_running(struct apple_rtkit *rtk) { + if (rtk->epmap_only) + return -EPERM; if (rtk->crashed) return -EINVAL; @@ -972,6 +1146,9 @@ int apple_rtkit_wake(struct apple_rtkit *rtk) u64 msg; int ret; + if (rtk->epmap_only) + return -EPERM; + if (apple_rtkit_is_running(rtk)) return -EINVAL; @@ -993,8 +1170,19 @@ EXPORT_SYMBOL_GPL(apple_rtkit_wake); void apple_rtkit_free(struct apple_rtkit *rtk) { + /* + * Let already-running callbacks finish while the transport is still + * powered, but make every callback observed after this point a no-op. + * Work queued by an IRQ racing the first flush is drained by + * destroy_workqueue() after apple_mbox_stop() has synchronously disabled + * that IRQ. + */ + WRITE_ONCE(rtk->shutting_down, true); + flush_workqueue(rtk->wq); apple_mbox_stop(rtk->mbox); + rtk->mbox->rx = NULL; destroy_workqueue(rtk->wq); + rtk->mbox->cookie = NULL; apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer); diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index f73a8913da951..9f7bd9d5f17d8 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -8,6 +8,7 @@ #ifndef _APPLE_MAILBOX_H_ #define _APPLE_MAILBOX_H_ +#include #include #include @@ -17,10 +18,56 @@ struct apple_mbox_msg { u32 msg1; }; +/** + * enum apple_mbox_ascwrap_v6_role - firmware role owned by an ASCWrap v6 + * mailbox provider + * @APPLE_MBOX_ASCWRAP_V6_ROLE_NONE: not an ASCWrap v6 provider + * @APPLE_MBOX_ASCWRAP_V6_ROLE_GFX: primary GPU firmware role + * @APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1: secondary GPU firmware role + */ +enum apple_mbox_ascwrap_v6_role { + APPLE_MBOX_ASCWRAP_V6_ROLE_NONE, + APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, +}; + +/* + * Missing pieces of the provider-owned ASCWrap v6 CPU lifecycle. Consumers + * must treat any non-zero mask as a hard pre-CPU-start gate. + */ +#define APPLE_MBOX_ASCWRAP_V6_MISSING_SAFE_STOP BIT(0) +#define APPLE_MBOX_ASCWRAP_V6_MISSING_SAFE_RESET BIT(1) + +/** + * struct apple_mbox_ascwrap_v6_lifecycle - provider-owned lifecycle facts + * @role: firmware role owned by this provider + * @wrapper_start: CPU physical base of the provider's ASCWrap window + * @wrapper_size: size of the provider's ASCWrap window + * @iop_vbar_start: CPU physical base of the provider's IOP VBAR window + * @iop_vbar_size: size of the provider's IOP VBAR window + * @missing: APPLE_MBOX_ASCWRAP_V6_MISSING_* execution gates + * + * This is deliberately a capability/query surface, not an invitation for a + * mailbox consumer to map either resource. The apple-mailbox provider owns + * both mappings. CPU start remains forbidden until the provider can also + * perform the complete matching stop/reset sequence. + */ +struct apple_mbox_ascwrap_v6_lifecycle { + enum apple_mbox_ascwrap_v6_role role; + u64 wrapper_start; + u64 wrapper_size; + u64 iop_vbar_start; + u64 iop_vbar_size; + u32 missing; +}; + struct apple_mbox { struct device *dev; void __iomem *regs; + void __iomem *wrapper_regs; + void __iomem *iop_vbar_regs; const struct apple_mbox_hw *hw; + struct apple_mbox_ascwrap_v6_lifecycle ascwrap_v6; bool active; int irq_recv_not_empty; @@ -45,4 +92,9 @@ int apple_mbox_poll(struct apple_mbox *mbox); int apple_mbox_send(struct apple_mbox *mbox, struct apple_mbox_msg msg, bool atomic); +int apple_mbox_ascwrap_v6_get_lifecycle( + struct apple_mbox *mbox, + struct apple_mbox_ascwrap_v6_lifecycle *lifecycle); +int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox); + #endif diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 6627a24972966..2d38a4883bbdf 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -65,6 +65,26 @@ struct apple_rtkit_ops { struct apple_rtkit; +#define APPLE_RTKIT_EPMAP_SNAPSHOT_WORDS 4 + +/** + * struct apple_rtkit_epmap_snapshot - passive HELLO/EPMAP discovery result + * @raw_hello: exact endpoint-zero HELLO payload received from firmware + * @hello_min_version: decoded HELLO minimum protocol version + * @hello_max_version: decoded HELLO maximum protocol version + * @endpoints: raw 256-bit EPMAP advertised by firmware + * + * Discovery-only clients answer HELLO and EPMAP but never start an endpoint + * and never request an AP/IOP power-state transition. In particular, EP 1 + * being present in @endpoints does not start crashlog or allocate a buffer. + */ +struct apple_rtkit_epmap_snapshot { + u64 raw_hello; + u16 hello_min_version; + u16 hello_max_version; + u64 endpoints[APPLE_RTKIT_EPMAP_SNAPSHOT_WORDS]; +}; + /* * Initializes the internal state required to handle RTKit. This * should usually be called within _probe. @@ -112,6 +132,15 @@ struct apple_rtkit *apple_rtkit_init_adopted( struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops); +/* + * Initialize a passive management client. The mailbox transport is armed, + * but the client admits only HELLO and EPMAP messages and performs no + * endpoint-start or power-state transactions. + */ +struct apple_rtkit *apple_rtkit_init_epmap_only( + struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, + const struct apple_rtkit_ops *ops); + /* * Free an instance of apple_rtkit. */ @@ -129,6 +158,13 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk); */ int apple_rtkit_boot(struct apple_rtkit *rtk); +/* Wait only for HELLO/EPMAP completion, without completing RTKit boot. */ +int apple_rtkit_discover_epmap(struct apple_rtkit *rtk); + +/* Copy an already-completed discovery-only snapshot. */ +int apple_rtkit_get_epmap_snapshot( + struct apple_rtkit *rtk, struct apple_rtkit_epmap_snapshot *snapshot); + /* * Adopt a co-processor session that a previous boot stage left fully running. * The caller must verify the device-specific running and ready indicators. From 5f4c33883a1c505dd1a4737fa75ca7490ac127f1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 22:18:30 +0000 Subject: [PATCH 0132/1579] soc: apple: quiesce mailbox providers on removal Fence mailbox operations and synchronously quiesce IRQ/runtime-PM state before devres releases provider mappings. Clear Rust mailbox callbacks on failed start and drop, and pin the dual-role rollback boundary with a test-only symbolic executor while production ASC execution remains disabled. Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: c54e8219026ac5402326ff0c76dcead24b94da2c Ported-paths: drivers/soc/apple/mailbox.c, include/linux/soc/apple/mailbox.h --- drivers/soc/apple/mailbox.c | 117 ++++++++++++++++++++++++++++-- include/linux/soc/apple/mailbox.h | 4 + 2 files changed, 114 insertions(+), 7 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index b95905292197f..231161fd7e97f 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -127,6 +127,8 @@ int apple_mbox_ascwrap_v6_get_lifecycle( { if (!mbox || !lifecycle) return -EINVAL; + if (READ_ONCE(mbox->removing)) + return -ENODEV; if (!mbox->hw->is_ascwrap_v6) return -EOPNOTSUPP; @@ -139,6 +141,8 @@ int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox) { if (!mbox || !mbox->hw->is_ascwrap_v6) return -EOPNOTSUPP; + if (READ_ONCE(mbox->removing)) + return -ENODEV; /* * This is intentionally checked before any CPU_CONTROL or IOP VBAR @@ -163,6 +167,10 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, long t; spin_lock_irqsave(&mbox->tx_lock, flags); + if (unlikely(READ_ONCE(mbox->removing))) { + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return -ENODEV; + } mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); while (mbox_ctrl & mbox->hw->control_full) { @@ -205,6 +213,10 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, return -ETIMEDOUT; spin_lock_irqsave(&mbox->tx_lock, flags); + if (unlikely(READ_ONCE(mbox->removing))) { + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return -ENODEV; + } mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); } @@ -276,7 +288,8 @@ static irqreturn_t apple_mbox_recv_irq(int irq, void *data) struct apple_mbox *mbox = data; spin_lock(&mbox->rx_lock); - apple_mbox_poll_locked(mbox); + if (likely(!READ_ONCE(mbox->removing))) + apple_mbox_poll_locked(mbox); spin_unlock(&mbox->rx_lock); return IRQ_HANDLED; @@ -288,7 +301,10 @@ int apple_mbox_poll(struct apple_mbox *mbox) int ret; spin_lock_irqsave(&mbox->rx_lock, flags); - ret = apple_mbox_poll_locked(mbox); + if (unlikely(READ_ONCE(mbox->removing))) + ret = -ENODEV; + else + ret = apple_mbox_poll_locked(mbox); spin_unlock_irqrestore(&mbox->rx_lock, flags); return ret; @@ -297,14 +313,19 @@ EXPORT_SYMBOL(apple_mbox_poll); int apple_mbox_start(struct apple_mbox *mbox) { - int ret; + int ret = 0; + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + ret = -ENODEV; + goto out_unlock; + } if (mbox->active) - return 0; + goto out_unlock; ret = pm_runtime_resume_and_get(mbox->dev); if (ret) - return ret; + goto out_unlock; /* * Only some variants of this mailbox HW provide interrupt control @@ -322,11 +343,14 @@ int apple_mbox_start(struct apple_mbox *mbox) enable_irq(mbox->irq_recv_not_empty); mbox->active = true; - return 0; + +out_unlock: + mutex_unlock(&mbox->lifecycle_lock); + return ret; } EXPORT_SYMBOL(apple_mbox_start); -void apple_mbox_stop(struct apple_mbox *mbox) +static void apple_mbox_stop_locked(struct apple_mbox *mbox) { if (!mbox->active) return; @@ -336,8 +360,73 @@ void apple_mbox_stop(struct apple_mbox *mbox) pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); } + +void apple_mbox_stop(struct apple_mbox *mbox) +{ + mutex_lock(&mbox->lifecycle_lock); + apple_mbox_stop_locked(mbox); + mutex_unlock(&mbox->lifecycle_lock); +} EXPORT_SYMBOL(apple_mbox_stop); +/* + * Quiesce every provider-owned transport edge before devres can release the + * IRQs or MMIO mappings. Device links guarantee that mailbox consumers have + * already unbound; the callback clear and waiter wake are defensive if a + * future consumer violates that ownership rule. This deliberately performs + * no ASC CPU-control, IOP VBAR, firmware-DMA, mapper, or reset operation. + */ +static void apple_mbox_quiesce(void *data) +{ + struct apple_mbox *mbox = data; + unsigned long flags; + bool had_rx; + bool was_active; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + mutex_unlock(&mbox->lifecycle_lock); + return; + } + + /* Serialize the removal fence with every MMIO send critical section. */ + spin_lock_irqsave(&mbox->tx_lock, flags); + WRITE_ONCE(mbox->removing, true); + spin_unlock_irqrestore(&mbox->tx_lock, flags); + + /* + * Publish the inactive state under the lifecycle lock, then drop it + * before disable_irq() waits for a callback. A callback is allowed to + * finish a send or stop request without an rx_lock -> lifecycle_lock + * inversion. The removal fence prevents any matching restart. + */ + was_active = mbox->active; + mbox->active = false; + mutex_unlock(&mbox->lifecycle_lock); + if (was_active) { + disable_irq(mbox->irq_recv_not_empty); + pm_runtime_mark_last_busy(mbox->dev); + pm_runtime_put_autosuspend(mbox->dev); + } + + /* The TX IRQ is normally disabled; force and synchronize that state. */ + disable_irq(mbox->irq_send_empty); + complete_all(&mbox->tx_empty); + synchronize_irq(mbox->irq_recv_not_empty); + + spin_lock_irqsave(&mbox->rx_lock, flags); + had_rx = mbox->rx; + mbox->rx = NULL; + mbox->cookie = NULL; + spin_unlock_irqrestore(&mbox->rx_lock, flags); + + if (WARN_ON_ONCE(had_rx)) + dev_warn(mbox->dev, + "consumer callback survived provider remove ordering\n"); + + pm_runtime_barrier(mbox->dev); +} + struct apple_mbox *apple_mbox_get(struct device *dev, int index) { struct of_phandle_args args; @@ -486,6 +575,7 @@ static int apple_mbox_probe(struct platform_device *pdev) spin_lock_init(&mbox->rx_lock); spin_lock_init(&mbox->tx_lock); + mutex_init(&mbox->lifecycle_lock); init_completion(&mbox->tx_empty); irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); @@ -509,6 +599,9 @@ static int apple_mbox_probe(struct platform_device *pdev) return ret; ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, apple_mbox_quiesce, mbox); if (ret) return ret; @@ -516,6 +609,15 @@ static int apple_mbox_probe(struct platform_device *pdev) return 0; } +static void apple_mbox_remove(struct platform_device *pdev) +{ + struct apple_mbox *mbox = platform_get_drvdata(pdev); + + /* Block new phandle lookups before quiescing the provider. */ + platform_set_drvdata(pdev, NULL); + apple_mbox_quiesce(mbox); +} + static const struct apple_mbox_hw apple_mbox_t8015_hw = { .control_full = APPLE_ASC_MBOX_CONTROL_FULL, .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, @@ -598,6 +700,7 @@ static struct platform_driver apple_mbox_driver = { .of_match_table = apple_mbox_of_match, }, .probe = apple_mbox_probe, + .remove = apple_mbox_remove, }; module_platform_driver(apple_mbox_driver); diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index 9f7bd9d5f17d8..52e61bb2aba5b 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -10,6 +10,7 @@ #include #include +#include #include /* encodes a single 96bit message sent over the single channel */ @@ -68,7 +69,10 @@ struct apple_mbox { void __iomem *iop_vbar_regs; const struct apple_mbox_hw *hw; struct apple_mbox_ascwrap_v6_lifecycle ascwrap_v6; + /* Serializes transport start/stop against provider removal. */ + struct mutex lifecycle_lock; bool active; + bool removing; int irq_recv_not_empty; int irq_send_empty; From 5511ca45646cd375727151ce59dad33328358d62 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 02:37:45 +0000 Subject: [PATCH 0133/1579] soc: apple: wire T8140 ASCWrap CPU lifecycle Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: a21e6c1a1a8f11cf255bf4f3170ef793a79c0d65 Ported-paths: drivers/soc/apple/mailbox.c, include/linux/soc/apple/mailbox.h --- drivers/soc/apple/mailbox.c | 110 ++++++++++++++++++++++++++---- include/linux/soc/apple/mailbox.h | 21 +++--- 2 files changed, 103 insertions(+), 28 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 231161fd7e97f..e5aefcaa36e09 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -51,6 +51,9 @@ #define APPLE_ASCWRAP_V6_MBOX_OFFSET 0x8000 #define APPLE_ASCWRAP_V6_WRAPPER_SIZE 0x88000 #define APPLE_ASCWRAP_V6_IOP_VBAR_SIZE 0x8 +#define APPLE_ASCWRAP_V6_CPU_CONTROL 0x44 +#define APPLE_ASCWRAP_V6_CPU_RUN BIT(4) +#define APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR BIT(5) #define APPLE_T8140_GFX_WRAPPER_BASE 0x482600000ULL #define APPLE_T8140_GFX_IOP_VBAR_BASE 0x482050000ULL @@ -144,13 +147,7 @@ int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox) if (READ_ONCE(mbox->removing)) return -ENODEV; - /* - * This is intentionally checked before any CPU_CONTROL or IOP VBAR - * write. Clearing CPU_RUN alone is not the complete AppleA7IOP stop - * transaction, and no grounded hardware reset exists yet. Keep the - * provider as the sole future lifecycle owner without exposing a - * one-way start primitive to consumers. - */ + /* Future provider revisions may report additional lifecycle work. */ if (mbox->ascwrap_v6.missing) return -EOPNOTSUPP; @@ -158,6 +155,90 @@ int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox) } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle); +static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) +{ + void __iomem *cpu_control; + u32 val; + + if (!mbox->ascwrap_v6_cpu_running) + return; + + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V6_CPU_CONTROL; + + /* AppleASCWrapV6::_runCPU(false): clear RUN, reread, clear bit 5. */ + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR, cpu_control); + + mbox->ascwrap_v6_cpu_running = false; + pm_runtime_mark_last_busy(mbox->dev); + pm_runtime_put_autosuspend(mbox->dev); +} + +int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) +{ + void __iomem *cpu_control; + u32 val; + int ret = 0; + + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + return -EOPNOTSUPP; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + ret = -ENODEV; + goto out_unlock; + } + if (mbox->ascwrap_v6.missing) { + ret = -EOPNOTSUPP; + goto out_unlock; + } + if (mbox->ascwrap_v6_cpu_running) { + ret = -EBUSY; + goto out_unlock; + } + + ret = pm_runtime_resume_and_get(mbox->dev); + if (ret) + goto out_unlock; + + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V6_CPU_CONTROL; + + /* AppleASCWrapV6::_runCPU(true): read, set RUN, write. */ + val = readl(cpu_control); + writel(val | APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + mbox->ascwrap_v6_cpu_running = true; + +out_unlock: + mutex_unlock(&mbox->lifecycle_lock); + return ret; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_start_cpu); + +int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) +{ + int ret = 0; + + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + return -EOPNOTSUPP; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + ret = -ENODEV; + goto out_unlock; + } + + apple_mbox_ascwrap_v6_stop_cpu_locked(mbox); + +out_unlock: + mutex_unlock(&mbox->lifecycle_lock); + return ret; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_stop_cpu); + int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, bool atomic) { @@ -370,11 +451,10 @@ void apple_mbox_stop(struct apple_mbox *mbox) EXPORT_SYMBOL(apple_mbox_stop); /* - * Quiesce every provider-owned transport edge before devres can release the - * IRQs or MMIO mappings. Device links guarantee that mailbox consumers have - * already unbound; the callback clear and waiter wake are defensive if a - * future consumer violates that ownership rule. This deliberately performs - * no ASC CPU-control, IOP VBAR, firmware-DMA, mapper, or reset operation. + * Stop the provider-owned CPU, then quiesce every transport edge before + * devres releases IRQs or MMIO mappings. Device links order consumer removal + * first; the callback clear and waiter wake also cover future consumers that + * violate that ownership rule. */ static void apple_mbox_quiesce(void *data) { @@ -388,6 +468,7 @@ static void apple_mbox_quiesce(void *data) mutex_unlock(&mbox->lifecycle_lock); return; } + apple_mbox_ascwrap_v6_stop_cpu_locked(mbox); /* Serialize the removal fence with every MMIO send critical section. */ spin_lock_irqsave(&mbox->tx_lock, flags); @@ -535,9 +616,8 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->ascwrap_v6.wrapper_size = resource_size(res); mbox->ascwrap_v6.iop_vbar_start = iop_vbar_res->start; mbox->ascwrap_v6.iop_vbar_size = resource_size(iop_vbar_res); - mbox->ascwrap_v6.missing = - APPLE_MBOX_ASCWRAP_V6_MISSING_SAFE_STOP | - APPLE_MBOX_ASCWRAP_V6_MISSING_SAFE_RESET; + /* resetState() is software-only; the matching stop lives above. */ + mbox->ascwrap_v6.missing = 0; mbox->wrapper_regs = devm_platform_ioremap_resource_byname(pdev, "wrapper"); diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index 52e61bb2aba5b..2066fd59308d8 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -8,7 +8,6 @@ #ifndef _APPLE_MAILBOX_H_ #define _APPLE_MAILBOX_H_ -#include #include #include #include @@ -32,13 +31,6 @@ enum apple_mbox_ascwrap_v6_role { APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, }; -/* - * Missing pieces of the provider-owned ASCWrap v6 CPU lifecycle. Consumers - * must treat any non-zero mask as a hard pre-CPU-start gate. - */ -#define APPLE_MBOX_ASCWRAP_V6_MISSING_SAFE_STOP BIT(0) -#define APPLE_MBOX_ASCWRAP_V6_MISSING_SAFE_RESET BIT(1) - /** * struct apple_mbox_ascwrap_v6_lifecycle - provider-owned lifecycle facts * @role: firmware role owned by this provider @@ -46,12 +38,12 @@ enum apple_mbox_ascwrap_v6_role { * @wrapper_size: size of the provider's ASCWrap window * @iop_vbar_start: CPU physical base of the provider's IOP VBAR window * @iop_vbar_size: size of the provider's IOP VBAR window - * @missing: APPLE_MBOX_ASCWRAP_V6_MISSING_* execution gates + * @missing: remaining provider lifecycle work; zero for T8140 * - * This is deliberately a capability/query surface, not an invitation for a - * mailbox consumer to map either resource. The apple-mailbox provider owns - * both mappings. CPU start remains forbidden until the provider can also - * perform the complete matching stop/reset sequence. + * The apple-mailbox provider owns both mappings. Consumers request CPU + * transitions through the provider and never map either resource themselves. + * T8140 has a complete matching start and stop transform. AppleA7IOP's + * resetState() only updates software state and performs no reset MMIO. */ struct apple_mbox_ascwrap_v6_lifecycle { enum apple_mbox_ascwrap_v6_role role; @@ -72,6 +64,7 @@ struct apple_mbox { /* Serializes transport start/stop against provider removal. */ struct mutex lifecycle_lock; bool active; + bool ascwrap_v6_cpu_running; bool removing; int irq_recv_not_empty; @@ -100,5 +93,7 @@ int apple_mbox_ascwrap_v6_get_lifecycle( struct apple_mbox *mbox, struct apple_mbox_ascwrap_v6_lifecycle *lifecycle); int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox); +int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox); +int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox); #endif From 6ca00479214de53c98b3c2c9756597a0480ad20a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 20:50:41 +0000 Subject: [PATCH 0134/1579] soc: apple: enable T8140 IOP outboxes from the AP Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: e303ae50d9c16b592e4c5eb6def6c338e94a6cc9 Ported-paths: drivers/soc/apple/mailbox.c --- drivers/soc/apple/mailbox.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index e5aefcaa36e09..f38e766fab163 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -105,6 +105,7 @@ static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == struct apple_mbox_hw { unsigned int reg_offset; bool is_ascwrap_v6; + bool ap_initializes_mailboxes; unsigned int control_full; unsigned int control_empty; @@ -394,6 +395,7 @@ EXPORT_SYMBOL(apple_mbox_poll); int apple_mbox_start(struct apple_mbox *mbox) { + u32 control; int ret = 0; mutex_lock(&mbox->lifecycle_lock); @@ -408,6 +410,16 @@ int apple_mbox_start(struct apple_mbox *mbox) if (ret) goto out_unlock; + if (mbox->hw->ap_initializes_mailboxes) { + /* AppleA7IOP::startCPUWithOptions() enables the IOP outbox. */ + control = readl_relaxed(mbox->regs + mbox->hw->i2a_control); + writel_relaxed(control | BIT(0), + mbox->regs + mbox->hw->i2a_control); + control = readl_relaxed(mbox->regs + mbox->hw->i2a_control); + dev_info(mbox->dev, + "T8140 IOP outbox enabled control=%#x\n", control); + } + /* * Only some variants of this mailbox HW provide interrupt control * at the mailbox level. We therefore need to handle enabling/disabling @@ -728,6 +740,23 @@ static const struct apple_mbox_hw apple_mbox_asc_hw = { .has_irq_controls = false, }; +static const struct apple_mbox_hw apple_mbox_t8140_asc_hw = { + .ap_initializes_mailboxes = true, + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + + .has_irq_controls = false, +}; + static const struct apple_mbox_hw apple_mbox_ascwrap_v6_hw = { .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, .is_ascwrap_v6 = true, @@ -767,6 +796,7 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_ascwrap_v6_hw }, + { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, { .compatible = "apple,asc-mailbox-v4", .data = &apple_mbox_asc_hw }, { .compatible = "apple,t8015-asc-mailbox", .data = &apple_mbox_t8015_hw }, { .compatible = "apple,m3-mailbox-v2", .data = &apple_mbox_m3_hw }, From 61f135a068ab9424b66148bacd5aad5108d7dde7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 21:10:01 +0000 Subject: [PATCH 0135/1579] soc: apple: handle G17P firmware crash DVA Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: a41a7396e60429a8b6eac3d2ff6bb85b6fc2ab9d Ported-paths: drivers/soc/apple/rtkit.c, drivers/soc/apple/rtkit-internal.h, include/linux/soc/apple/rtkit.h --- drivers/soc/apple/rtkit-internal.h | 2 + drivers/soc/apple/rtkit.c | 64 ++++++++++++++++++++++++++++-- include/linux/soc/apple/rtkit.h | 8 ++++ 3 files changed, 70 insertions(+), 4 deletions(-) diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 85ca018aa41e5..96756094b2ec5 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -45,6 +45,8 @@ struct apple_rtkit { bool crashed; bool adopted; bool epmap_only; + bool firmware_owned_crashlog; + bool crashlog_endpoint_started; bool hello_seen; bool epmap_finished; bool shutting_down; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 179902e9d3140..fb37ace466c45 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -213,6 +213,23 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) set_bit(ep, rtk->discovery_endpoints); } + /* + * G17P sends its firmware-owned crash buffer request as soon as EP1 is + * started. Start EP1 before acknowledging the EPMAP fragment that + * advertises it, matching the ordering required by this firmware. + */ + if (rtk->firmware_owned_crashlog && + test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints) && + !rtk->crashlog_endpoint_started) { + ret = apple_rtkit_start_ep(rtk, APPLE_RTKIT_EP_CRASHLOG); + if (ret) { + rtk->boot_result = ret; + complete_all(&rtk->epmap_completion); + return; + } + rtk->crashlog_endpoint_started = true; + } + reply = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, base); if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) reply |= APPLE_RTKIT_MGMT_EPMAP_LAST; @@ -249,6 +266,9 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) case APPLE_RTKIT_EP_IOREPORT: case APPLE_RTKIT_EP_OSLOG: case APPLE_RTKIT_EP_TRACEKIT: + if (ep == APPLE_RTKIT_EP_CRASHLOG && + rtk->crashlog_endpoint_started) + break; dev_dbg(rtk->dev, "RTKit: Starting system endpoint 0x%02x\n", ep); apple_rtkit_start_ep(rtk, ep); @@ -346,6 +366,20 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, dev_dbg(rtk->dev, "RTKit: buffer request for 0x%zx bytes at %pad\n", buffer->size, &buffer->iova); + /* + * G17P places its crash log in the GPU UAT and sends that DVA in the + * request. Linux cannot map it through the device DMA domain. Retain the + * DVA for diagnostics and echo the request exactly as firmware expects. + */ + if (rtk->firmware_owned_crashlog && + ep == APPLE_RTKIT_EP_CRASHLOG && buffer->iova) { + buffer->is_mapped = true; + err = apple_rtkit_send_message(rtk, ep, msg, NULL, false); + if (err) + goto error; + return 0; + } + if (buffer->iova && !rtk->ops->shmem_setup) { err = -EINVAL; goto error; @@ -443,6 +477,16 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) } dev_err(rtk->dev, "RTKit: co-processor has crashed\n"); + if (rtk->firmware_owned_crashlog && !rtk->crashlog_buffer.buffer && + !rtk->crashlog_buffer.iomem) { + dev_err(rtk->dev, + "RTKit: firmware crash log remains at DVA %pad (0x%zx bytes)\n", + &rtk->crashlog_buffer.iova, rtk->crashlog_buffer.size); + rtk->crashed = true; + if (rtk->ops->crashed) + rtk->ops->crashed(rtk->cookie, NULL, 0); + return; + } /* * create a shadow copy here to make sure the co-processor isn't able @@ -785,7 +829,8 @@ static void apple_rtkit_mark_running(struct apple_rtkit *rtk) static struct apple_rtkit *__apple_rtkit_init( struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, - const struct apple_rtkit_ops *ops, bool adopted, bool epmap_only) + const struct apple_rtkit_ops *ops, bool adopted, bool epmap_only, + bool firmware_owned_crashlog) { struct apple_rtkit *rtk; int ret; @@ -801,6 +846,7 @@ static struct apple_rtkit *__apple_rtkit_init( rtk->cookie = cookie; rtk->ops = ops; rtk->epmap_only = epmap_only; + rtk->firmware_owned_crashlog = firmware_owned_crashlog; init_completion(&rtk->epmap_completion); init_completion(&rtk->iop_pwr_ack_completion); @@ -856,16 +902,25 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, const struct apple_rtkit_ops *ops) { return __apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops, false, - false); + false, false); } EXPORT_SYMBOL_GPL(apple_rtkit_init); +struct apple_rtkit *apple_rtkit_init_g17p( + struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, + const struct apple_rtkit_ops *ops) +{ + return __apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops, false, + false, true); +} +EXPORT_SYMBOL_GPL(apple_rtkit_init_g17p); + struct apple_rtkit *apple_rtkit_init_adopted( struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops) { return __apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops, true, - false); + false, false); } EXPORT_SYMBOL_GPL(apple_rtkit_init_adopted); @@ -874,7 +929,7 @@ struct apple_rtkit *apple_rtkit_init_epmap_only( const struct apple_rtkit_ops *ops) { return __apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops, false, - true); + true, false); } EXPORT_SYMBOL_GPL(apple_rtkit_init_epmap_only); @@ -926,6 +981,7 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) rtk->epmap_bases_seen = 0; rtk->hello_seen = false; rtk->epmap_finished = false; + rtk->crashlog_endpoint_started = false; rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_OFF; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_OFF; diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 2d38a4883bbdf..ebaa5d710dad3 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -128,6 +128,14 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops); +/* + * Initialize RTKit for G17P. Its crash buffer belongs to the GPU UAT, so the + * core retains the firmware DVA and echoes the request without DMA mapping it. + */ +struct apple_rtkit *apple_rtkit_init_g17p( + struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, + const struct apple_rtkit_ops *ops); + struct apple_rtkit *apple_rtkit_init_adopted( struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops); From 38804bffd146655c325ab4ffeca60e37a8a4711d Mon Sep 17 00:00:00 2001 From: Acelogic Date: Wed, 9 Sep 2026 11:24:28 +0000 Subject: [PATCH 0136/1579] nvme-apple: wake inactive post-M4 firmware sessions CPU_RUN and BOOT_STATUS remain set after firmware leaves ANS inactive. Adopting that state marks the RTKit power states as ON without sending the wake handshake, leaving NVMe controller readiness stuck at zero. Require an enabled, ready NVMe controller before adopting its existing RTKit session. Use the same predicate before mailbox initialization and in reset work. Inactive sessions follow the existing RTKit wake path. On J700/T8140 booted directly through m1n1, this completes protocol 12 initialization and reaches CC=0x460001, CSTS=1. Both GPT copies match the independent firmware reads and pass their CRC checks. Two 256 MiB reads produce identical hashes; all six CPUs and Vulkan compute still pass. Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: 8e0253e2572402266eecf5242abf94b5f45da54b Ported-paths: drivers/nvme/host/apple.c --- drivers/nvme/host/apple.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 23bee6675e2cc..57dba97612207 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -257,6 +257,18 @@ static inline struct apple_nvme *ctrl_to_apple_nvme(struct nvme_ctrl *ctrl) return container_of(ctrl, struct apple_nvme, ctrl); } +static bool apple_nvme_can_adopt_rtkit(struct apple_nvme *anv) +{ + /* CPU_RUN and BOOT_STATUS survive an inactive firmware session. */ + return anv->hw->needs_ioq_registers && + (readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & + APPLE_ANS_COPROC_CPU_CONTROL_RUN) && + readl(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS) == + APPLE_ANS_BOOT_STATUS_OK && + (readl(anv->mmio_nvme + NVME_REG_CC) & NVME_CC_ENABLE) && + (readl(anv->mmio_nvme + NVME_REG_CSTS) & NVME_CSTS_RDY); +} + static inline struct apple_nvme *queue_to_apple_nvme(struct apple_nvme_queue *q) { if (q->is_adminq) @@ -1278,11 +1290,7 @@ static void apple_nvme_reset_work(struct work_struct *work) * and EPMAP across boot stages. Adopt the first healthy inherited session; * subsequent resets use the normal RTKit shutdown/reset path below. */ - if (!anv->owns_rtkit && anv->hw->needs_ioq_registers && - (readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & - APPLE_ANS_COPROC_CPU_CONTROL_RUN) && - readl(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS) == - APPLE_ANS_BOOT_STATUS_OK) { + if (!anv->owns_rtkit && apple_nvme_can_adopt_rtkit(anv)) { ret = apple_rtkit_adopt_running(anv->rtk); if (ret) goto out; @@ -1916,11 +1924,11 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) * system endpoints before mailbox RX starts, otherwise inherited syslog * traffic can race the later reset work and be dropped as undiscovered. */ - inherited_rtkit = anv->hw->needs_ioq_registers && - (readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & - APPLE_ANS_COPROC_CPU_CONTROL_RUN) && - readl(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS) == - APPLE_ANS_BOOT_STATUS_OK; + inherited_rtkit = apple_nvme_can_adopt_rtkit(anv); + if (anv->hw->needs_ioq_registers) + dev_info(dev, "post-M4 firmware handoff: CC=%#x CSTS=%#x adopt=%d\n", + readl(anv->mmio_nvme + NVME_REG_CC), + readl(anv->mmio_nvme + NVME_REG_CSTS), inherited_rtkit); if (inherited_rtkit) anv->rtk = devm_apple_rtkit_init_adopted( dev, anv, NULL, 0, &apple_nvme_rtkit_ops); From d9ccd46bea7f0a55d438fae64d564cb04f5305d9 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 10 Sep 2026 08:13:52 +0000 Subject: [PATCH 0137/1579] drm/apple: restart inherited H17P firmware through acknowledged IOP shutdown J700 firmware reports Invalid AP power: 32 when the inherited-session path sends AP QUIESCED. Add a guarded RTKit IOP-only poweroff operation and use it for H17P before the existing bounded ASC restart. Require the OFF acknowledgement; leave ordinary power protocols unchanged and reject a blind retry after a failed H17P transition. Protocol evidence: m1n1-aurora 4a927cdcae729f1b12b2173117e7f96b3271dca9 and the local pinned-firmware crash-buffer capture. Native display restart remains under hardware qualification. Co-authored-by: DjDeveloperr Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: a056dfec99e4af4bd770774b5b9148eebcb5d262 Ported-paths: drivers/soc/apple/rtkit.c, include/linux/soc/apple/rtkit.h --- drivers/soc/apple/rtkit.c | 16 ++++++++++++++++ include/linux/soc/apple/rtkit.h | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index fb37ace466c45..3e9880ff7f13a 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -1197,6 +1197,22 @@ int apple_rtkit_quiesce(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_quiesce); +int apple_rtkit_poweroff_adopted(struct apple_rtkit *rtk) +{ + int ret; + + if (!rtk->adopted || rtk->epmap_only) + return -EINVAL; + + /* Do not retire inherited buffers until the IOP acknowledges OFF. */ + ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_OFF); + if (ret) + return ret; + + return apple_rtkit_reinit(rtk); +} +EXPORT_SYMBOL_GPL(apple_rtkit_poweroff_adopted); + int apple_rtkit_wake(struct apple_rtkit *rtk) { u64 msg; diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index ebaa5d710dad3..ac482a8b70864 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -184,6 +184,14 @@ int apple_rtkit_adopt_running(struct apple_rtkit *rtk); */ int apple_rtkit_quiesce(struct apple_rtkit *rtk); +/* + * Power off an adopted session through IOP OFF without an AP transition. + * The caller must have established that this sequence is valid for its + * firmware, and must supply any device-specific reset needed before boot. + * This is not a replacement for the normal quiesce or shutdown protocol. + */ +int apple_rtkit_poweroff_adopted(struct apple_rtkit *rtk); + /* * Wake the co-processor up from hibernation mode. */ From 84a38adc0a599e0e78b9d6797862d33ae5ea2428 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 10 Sep 2026 08:28:54 +0000 Subject: [PATCH 0138/1579] soc: apple: wait for H17P IOP idle before retiring an adopted session Pinned macOS 25G76 maps management type 0xc to the flag consumed by RTBuddyManagementEndpoint::waitForIdleGated. RTBuddy::_performPowerStateChangeGated calls that wait after the power transition. Require this separately observed idle notification before releasing inherited buffers or restarting the ASC; an OFF ACK alone is insufficient. Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: 6399a02923baca1803f2c8e0ebeda05491abfdf4 Ported-paths: drivers/soc/apple/rtkit.c, drivers/soc/apple/rtkit-internal.h, include/linux/soc/apple/rtkit.h --- drivers/soc/apple/rtkit-internal.h | 2 ++ drivers/soc/apple/rtkit.c | 22 +++++++++++++++++++++- include/linux/soc/apple/rtkit.h | 3 ++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 96756094b2ec5..7ea7c564e930e 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -32,6 +32,7 @@ struct apple_rtkit { struct completion epmap_completion; struct completion iop_pwr_ack_completion; struct completion ap_pwr_ack_completion; + struct completion iop_idle_completion; int boot_result; int version; @@ -44,6 +45,7 @@ struct apple_rtkit { unsigned int ap_power_state; bool crashed; bool adopted; + bool awaiting_iop_idle; bool epmap_only; bool firmware_owned_crashlog; bool crashlog_endpoint_started; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 3e9880ff7f13a..accc8f0da993d 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -40,6 +40,7 @@ enum { APPLE_RTKIT_MGMT_EPMAP_REPLY = 8, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE = 0xb, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK = 0xb, + APPLE_RTKIT_MGMT_IOP_IDLE = 0xc, }; #define APPLE_RTKIT_MGMT_HELLO_MINVER GENMASK_ULL(15, 0) @@ -313,6 +314,13 @@ static void apple_rtkit_management_rx(struct apple_rtkit *rtk, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_MGMT_TYPE, msg); + /* H17P sends this separate idle fence after acknowledging IOP OFF. */ + if (type == APPLE_RTKIT_MGMT_IOP_IDLE && + READ_ONCE(rtk->awaiting_iop_idle) && !(msg & ~APPLE_RTKIT_MGMT_TYPE)) { + complete_all(&rtk->iop_idle_completion); + return; + } + if (rtk->epmap_only && type != APPLE_RTKIT_MGMT_HELLO && type != APPLE_RTKIT_MGMT_EPMAP) { dev_err(rtk->dev, @@ -851,6 +859,7 @@ static struct apple_rtkit *__apple_rtkit_init( init_completion(&rtk->epmap_completion); init_completion(&rtk->iop_pwr_ack_completion); init_completion(&rtk->ap_pwr_ack_completion); + init_completion(&rtk->iop_idle_completion); bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); bitmap_zero(rtk->discovery_endpoints, APPLE_RTKIT_MAX_ENDPOINTS); @@ -964,6 +973,7 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) rtk->syslog_n_entries = 0; rtk->syslog_msg_size = 0; rtk->adopted = false; + WRITE_ONCE(rtk->awaiting_iop_idle, false); bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); bitmap_zero(rtk->discovery_endpoints, APPLE_RTKIT_MAX_ENDPOINTS); @@ -1204,11 +1214,21 @@ int apple_rtkit_poweroff_adopted(struct apple_rtkit *rtk) if (!rtk->adopted || rtk->epmap_only) return -EINVAL; - /* Do not retire inherited buffers until the IOP acknowledges OFF. */ + /* + * Native H17P RTBuddy waits for the separate type-0xc idle notification + * after its power transition. An OFF ACK alone does not establish that + * the firmware has finished retiring its work and can be stopped. + */ + reinit_completion(&rtk->iop_idle_completion); + WRITE_ONCE(rtk->awaiting_iop_idle, true); ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_OFF); + if (!ret) + ret = apple_rtkit_wait_for_completion(&rtk->iop_idle_completion); + WRITE_ONCE(rtk->awaiting_iop_idle, false); if (ret) return ret; + dev_info(rtk->dev, "RTKit: inherited IOP OFF and idle acknowledged\n"); return apple_rtkit_reinit(rtk); } EXPORT_SYMBOL_GPL(apple_rtkit_poweroff_adopted); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index ac482a8b70864..724c531b89cee 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -185,7 +185,8 @@ int apple_rtkit_adopt_running(struct apple_rtkit *rtk); int apple_rtkit_quiesce(struct apple_rtkit *rtk); /* - * Power off an adopted session through IOP OFF without an AP transition. + * Power off an adopted session through IOP OFF without an AP transition, + * waiting for both the power acknowledgement and the H17P idle notification. * The caller must have established that this sequence is valid for its * firmware, and must supply any device-specific reset needed before boot. * This is not a replacement for the normal quiesce or shutdown protocol. From 6e0bf42437c89d09c7aceeae70449330ca4d2bfa Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 10 Sep 2026 08:40:32 +0000 Subject: [PATCH 0139/1579] arm64: apple: enable J813 keyboard, trackpad and XFCE desktop Bring up the M5 keyboard and trackpad through measured MTP resources, scoped DAPF preparation, early AP power ordering and firmware-managed touch reset. Load the dedicated HID drivers together and retain the companion U-Boot requirement that leaves MTP cold for Linux. Provide the XFCE RAM desktop with working runtime permissions, user namespaces and devpts. Include the Arch-logo fastfetch, CPU utilization and live N1 Wi-Fi demonstrations, with the desktop user's ICMP sockets configured on every launch. Validated on physical M5 hardware: keyboard and trackpad events in XFCE, a working terminal, Wi-Fi DHCP and external pings, and Bluetooth startup without TX/RX errors. Kernel, device tree and modules passed W=1 builds. GPU acceleration, full SMP and Bluetooth audio remain unqualified. Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: 9876c3c6c022d2a3635db984726bc30cbff140ec Ported-paths: drivers/soc/apple/rtkit.c, include/linux/soc/apple/rtkit.h --- drivers/soc/apple/rtkit.c | 32 ++++++++++++++++++++++++++++---- include/linux/soc/apple/rtkit.h | 3 +++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index accc8f0da993d..0993a84697104 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -1051,7 +1051,7 @@ static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, return 0; } -int apple_rtkit_boot(struct apple_rtkit *rtk) +static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) { int ret; @@ -1070,12 +1070,25 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) if (rtk->boot_result) return rtk->boot_result; + /* Some helpers wait for AP=ON before acknowledging their own ON state. */ + if (early_ap_power) { + ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); + if (ret) + return ret; + } + dev_dbg(rtk->dev, "RTKit: waiting for IOP power state ACK\n"); ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion); if (ret) return ret; - return apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); + return early_ap_power ? 0 : + apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); +} + +int apple_rtkit_boot(struct apple_rtkit *rtk) +{ + return apple_rtkit_boot_common(rtk, false); } EXPORT_SYMBOL_GPL(apple_rtkit_boot); @@ -1233,7 +1246,7 @@ int apple_rtkit_poweroff_adopted(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_poweroff_adopted); -int apple_rtkit_wake(struct apple_rtkit *rtk) +static int apple_rtkit_wake_common(struct apple_rtkit *rtk, bool early_ap_power) { u64 msg; int ret; @@ -1256,10 +1269,21 @@ int apple_rtkit_wake(struct apple_rtkit *rtk) if (ret) return ret; - return apple_rtkit_boot(rtk); + return apple_rtkit_boot_common(rtk, early_ap_power); +} + +int apple_rtkit_wake(struct apple_rtkit *rtk) +{ + return apple_rtkit_wake_common(rtk, false); } EXPORT_SYMBOL_GPL(apple_rtkit_wake); +int apple_rtkit_wake_early_ap(struct apple_rtkit *rtk) +{ + return apple_rtkit_wake_common(rtk, true); +} +EXPORT_SYMBOL_GPL(apple_rtkit_wake_early_ap); + void apple_rtkit_free(struct apple_rtkit *rtk) { /* diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 724c531b89cee..e922b4411508a 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -198,6 +198,9 @@ int apple_rtkit_poweroff_adopted(struct apple_rtkit *rtk); */ int apple_rtkit_wake(struct apple_rtkit *rtk); +/* For helpers that require AP=ON before their IOP power-state acknowledgement. */ +int apple_rtkit_wake_early_ap(struct apple_rtkit *rtk); + /* * Shutdown the co-processor */ From defde4fbc701956408d1c7bb544286cf47930e3f Mon Sep 17 00:00:00 2001 From: Acelogic Date: Tue, 22 Sep 2026 19:00:10 +0000 Subject: [PATCH 0140/1579] soc: apple: carry AGX mailbox and power dependencies Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: 123cc44479c54000f5ae1389f6a515073d2ed3be Ported-paths: drivers/soc/apple/mailbox.c, include/linux/soc/apple/mailbox.h --- drivers/soc/apple/mailbox.c | 66 +++++++++++++++++++++++++++---- include/linux/soc/apple/mailbox.h | 6 ++- 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index f38e766fab163..f36686b4037d0 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -52,6 +52,7 @@ #define APPLE_ASCWRAP_V6_WRAPPER_SIZE 0x88000 #define APPLE_ASCWRAP_V6_IOP_VBAR_SIZE 0x8 #define APPLE_ASCWRAP_V6_CPU_CONTROL 0x44 +#define APPLE_ASCWRAP_V6_CPU_STATUS 0x48 #define APPLE_ASCWRAP_V6_CPU_RUN BIT(4) #define APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR BIT(5) @@ -59,6 +60,7 @@ #define APPLE_T8140_GFX_IOP_VBAR_BASE 0x482050000ULL #define APPLE_T8140_GFX1_WRAPPER_BASE 0x482e00000ULL #define APPLE_T8140_GFX1_IOP_VBAR_BASE 0x482850000ULL +#define APPLE_T8142_ASCWRAP_BASE_DELTA 0x100000000ULL static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == 0x8110); @@ -105,6 +107,7 @@ static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == struct apple_mbox_hw { unsigned int reg_offset; bool is_ascwrap_v6; + u64 ascwrap_v6_base_delta; bool ap_initializes_mailboxes; unsigned int control_full; @@ -240,6 +243,29 @@ int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_stop_cpu); +int apple_mbox_ascwrap_v6_get_cpu_state(struct apple_mbox *mbox, + u32 *control, u32 *status) +{ + int ret = 0; + + if (!mbox || !control || !status || !mbox->hw->is_ascwrap_v6 || + !mbox->wrapper_regs) + return -EOPNOTSUPP; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + ret = -ENODEV; + } else { + *control = readl((u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V6_CPU_CONTROL); + *status = readl((u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V6_CPU_STATUS); + } + mutex_unlock(&mbox->lifecycle_lock); + return ret; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_get_cpu_state); + int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, bool atomic) { @@ -589,7 +615,7 @@ static int apple_mbox_probe(struct platform_device *pdev) if (mbox->hw->is_ascwrap_v6) { /* * Validate the entire provider identity before mapping either - * resource. These are fixed T8140 CPU-physical windows and role + * resource. These are fixed per-SoC CPU-physical windows and role * assignments, not generic consumer-provided aliases. */ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "wrapper"); @@ -609,16 +635,20 @@ static int apple_mbox_probe(struct platform_device *pdev) if (!strcmp(firmware_role, "GFX")) { mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; - if (res->start != APPLE_T8140_GFX_WRAPPER_BASE || - iop_vbar_res->start != APPLE_T8140_GFX_IOP_VBAR_BASE) + if (res->start != APPLE_T8140_GFX_WRAPPER_BASE + + mbox->hw->ascwrap_v6_base_delta || + iop_vbar_res->start != APPLE_T8140_GFX_IOP_VBAR_BASE + + mbox->hw->ascwrap_v6_base_delta) return dev_err_probe(dev, -EINVAL, - "GFX ASCWrap v6 resources do not match T8140\n"); + "GFX ASCWrap v6 resources do not match the SoC profile\n"); } else if (!strcmp(firmware_role, "GFX1")) { mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1; - if (res->start != APPLE_T8140_GFX1_WRAPPER_BASE || - iop_vbar_res->start != APPLE_T8140_GFX1_IOP_VBAR_BASE) + if (res->start != APPLE_T8140_GFX1_WRAPPER_BASE + + mbox->hw->ascwrap_v6_base_delta || + iop_vbar_res->start != APPLE_T8140_GFX1_IOP_VBAR_BASE + + mbox->hw->ascwrap_v6_base_delta) return dev_err_probe(dev, -EINVAL, - "GFX1 ASCWrap v6 resources do not match T8140\n"); + "GFX1 ASCWrap v6 resources do not match the SoC profile\n"); } else { return dev_err_probe(dev, -EINVAL, "invalid ASCWrap v6 firmware role\n"); @@ -775,6 +805,27 @@ static const struct apple_mbox_hw apple_mbox_ascwrap_v6_hw = { .has_irq_controls = false, }; +/* J813/25G76 AppleASCWrapV6::_runCPU at 0xfffffe0008a42d34: + * start reads +0x44 and sets bit4; stop clears bit4, rereads, then clears + * bit5. This matches the provider transition above. The four ADT resource + * bases move from T8140's 0x4... bank to T8142's 0x5... bank. Keep this an + * explicit compatible; selecting it does not admit a GPU firmware ABI. + */ +static const struct apple_mbox_hw apple_mbox_t8142_ascwrap_v6_hw = { + .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, + .is_ascwrap_v6 = true, + .ascwrap_v6_base_delta = APPLE_T8142_ASCWRAP_BASE_DELTA, + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + .has_irq_controls = false, +}; + static const struct apple_mbox_hw apple_mbox_m3_hw = { .control_full = APPLE_M3_MBOX_CONTROL_FULL, .control_empty = APPLE_M3_MBOX_CONTROL_EMPTY, @@ -796,6 +847,7 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_ascwrap_v6_hw }, + { .compatible = "apple,t8142-ascwrap-v6", .data = &apple_mbox_t8142_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, { .compatible = "apple,asc-mailbox-v4", .data = &apple_mbox_asc_hw }, { .compatible = "apple,t8015-asc-mailbox", .data = &apple_mbox_t8015_hw }, diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index 2066fd59308d8..58bc14cc99008 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -38,11 +38,11 @@ enum apple_mbox_ascwrap_v6_role { * @wrapper_size: size of the provider's ASCWrap window * @iop_vbar_start: CPU physical base of the provider's IOP VBAR window * @iop_vbar_size: size of the provider's IOP VBAR window - * @missing: remaining provider lifecycle work; zero for T8140 + * @missing: remaining provider lifecycle work; zero for T8140 and T8142 * * The apple-mailbox provider owns both mappings. Consumers request CPU * transitions through the provider and never map either resource themselves. - * T8140 has a complete matching start and stop transform. AppleA7IOP's + * T8140 and T8142 have a matching start and stop transform. AppleA7IOP's * resetState() only updates software state and performs no reset MMIO. */ struct apple_mbox_ascwrap_v6_lifecycle { @@ -95,5 +95,7 @@ int apple_mbox_ascwrap_v6_get_lifecycle( int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox); int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox); int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox); +int apple_mbox_ascwrap_v6_get_cpu_state(struct apple_mbox *mbox, + u32 *control, u32 *status); #endif From 35cb54d53fbab1485fc35a97b3fcf2abddb49766 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Wed, 23 Sep 2026 23:08:18 +0000 Subject: [PATCH 0141/1579] arm64: apple: add J873 thermal and framebuffer desktop bring-up Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: 44b5ac090b1b1af7b34cda852d4c502ab0b65ed3 Ported-paths: drivers/soc/apple/mailbox.c --- drivers/soc/apple/mailbox.c | 101 ++++++++++++++++++++++++++++++++---- 1 file changed, 90 insertions(+), 11 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index f36686b4037d0..70447abff8745 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -107,6 +107,7 @@ static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == struct apple_mbox_hw { unsigned int reg_offset; bool is_ascwrap_v6; + bool counted_fifo; u64 ascwrap_v6_base_delta; bool ap_initializes_mailboxes; @@ -266,6 +267,72 @@ int apple_mbox_ascwrap_v6_get_cpu_state(struct apple_mbox *mbox, } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_get_cpu_state); +static bool apple_mbox_tx_full(struct apple_mbox *mbox, u32 status) +{ + if (mbox->hw->counted_fifo) + return !(status & mbox->hw->control_full); + return status & mbox->hw->control_full; +} + +static bool apple_mbox_rx_empty(struct apple_mbox *mbox, u32 status) +{ + if (mbox->hw->counted_fifo) + return !(status & mbox->hw->control_empty); + return status & mbox->hw->control_empty; +} + +/* ASCWrap v8 transfers each message with one paired access. Its status word + * contains RX occupancy and free TX slots, rather than full/empty flags. + * Keep the two words together, as on the qualified native T8152 path. + */ +static void apple_mbox_write_msg(struct apple_mbox *mbox, + struct apple_mbox_msg msg) +{ + u64 high = FIELD_PREP(APPLE_MBOX_MSG1_MSG, msg.msg1); + +#ifdef CONFIG_ARM64 + if (mbox->hw->counted_fifo) { + u64 low = (__force u64)cpu_to_le64(msg.msg0); + + high = (__force u64)cpu_to_le64(high); + wmb(); + asm volatile("stp %0, %1, [%2]" + : : "r" (low), "r" (high), + "r" (mbox->regs + mbox->hw->a2i_send0) + : "memory"); + } else +#endif + { + writeq_relaxed(msg.msg0, mbox->regs + mbox->hw->a2i_send0); + writeq_relaxed(high, mbox->regs + mbox->hw->a2i_send1); + } +} + +static struct apple_mbox_msg apple_mbox_read_msg(struct apple_mbox *mbox) +{ + struct apple_mbox_msg msg; + u64 low, high; + +#ifdef CONFIG_ARM64 + if (mbox->hw->counted_fifo) { + asm volatile("ldp %0, %1, [%2]" + : "=&r" (low), "=&r" (high) + : "r" (mbox->regs + mbox->hw->i2a_recv0) + : "memory"); + rmb(); + low = le64_to_cpu((__force __le64)low); + high = le64_to_cpu((__force __le64)high); + } else +#endif + { + low = readq_relaxed(mbox->regs + mbox->hw->i2a_recv0); + high = readq_relaxed(mbox->regs + mbox->hw->i2a_recv1); + } + msg.msg0 = low; + msg.msg1 = FIELD_GET(APPLE_MBOX_MSG1_MSG, high); + return msg; +} + int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, bool atomic) { @@ -281,11 +348,11 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, } mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); - while (mbox_ctrl & mbox->hw->control_full) { + while (apple_mbox_tx_full(mbox, mbox_ctrl)) { if (atomic) { ret = readl_poll_timeout_atomic( mbox->regs + mbox->hw->a2i_control, mbox_ctrl, - !(mbox_ctrl & mbox->hw->control_full), 100, + !apple_mbox_tx_full(mbox, mbox_ctrl), 100, APPLE_MBOX_TX_TIMEOUT * 1000); if (ret) { @@ -328,9 +395,7 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); } - writeq_relaxed(msg.msg0, mbox->regs + mbox->hw->a2i_send0); - writeq_relaxed(FIELD_PREP(APPLE_MBOX_MSG1_MSG, msg.msg1), - mbox->regs + mbox->hw->a2i_send1); + apple_mbox_write_msg(mbox, msg); spin_unlock_irqrestore(&mbox->tx_lock, flags); @@ -364,11 +429,8 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) u32 mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); - while (!(mbox_ctrl & mbox->hw->control_empty)) { - msg.msg0 = readq_relaxed(mbox->regs + mbox->hw->i2a_recv0); - msg.msg1 = FIELD_GET( - APPLE_MBOX_MSG1_MSG, - readq_relaxed(mbox->regs + mbox->hw->i2a_recv1)); + while (!apple_mbox_rx_empty(mbox, mbox_ctrl)) { + msg = apple_mbox_read_msg(mbox); mbox->rx(mbox, msg, mbox->cookie); ret++; @@ -611,6 +673,8 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->hw = of_device_get_match_data(dev); if (!mbox->hw) return -EINVAL; + if (mbox->hw->counted_fifo && !IS_ENABLED(CONFIG_ARM64)) + return -EOPNOTSUPP; if (mbox->hw->is_ascwrap_v6) { /* @@ -676,7 +740,8 @@ static int apple_mbox_probe(struct platform_device *pdev) } else { res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res || resource_size(res) < mbox->hw->reg_offset + - mbox->hw->i2a_recv1 + sizeof(u64)) + max(mbox->hw->i2a_recv1 + sizeof(u64), + mbox->hw->i2a_control + sizeof(u32))) return dev_err_probe(dev, -EINVAL, "mailbox register window is too small\n"); @@ -826,6 +891,19 @@ static const struct apple_mbox_hw apple_mbox_t8142_ascwrap_v6_hw = { .has_irq_controls = false, }; +static const struct apple_mbox_hw apple_mbox_t8152_v8_hw = { + .counted_fifo = true, + .control_full = GENMASK(11, 8), /* free TX slots */ + .control_empty = GENMASK(3, 0), /* RX occupancy */ + .a2i_control = 0x20, + .a2i_send0 = 0x10, + .a2i_send1 = 0x18, + .i2a_control = 0x20, + .i2a_recv0 = 0x00, + .i2a_recv1 = 0x08, + .has_irq_controls = false, +}; + static const struct apple_mbox_hw apple_mbox_m3_hw = { .control_full = APPLE_M3_MBOX_CONTROL_FULL, .control_empty = APPLE_M3_MBOX_CONTROL_EMPTY, @@ -846,6 +924,7 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { }; static const struct of_device_id apple_mbox_of_match[] = { + { .compatible = "apple,t8152-ascwrap-v8-mailbox", .data = &apple_mbox_t8152_v8_hw }, { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_ascwrap_v6_hw }, { .compatible = "apple,t8142-ascwrap-v6", .data = &apple_mbox_t8142_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, From f6e4a81bc7e106f5c0be48aedadcf9c5f5279311 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 25 Sep 2026 20:25:16 +0000 Subject: [PATCH 0142/1579] soc: apple: mailbox: add the T8152 GFX ASCWrap v8 provider New compatible apple,t8152-gfx-ascwrap-v8 for the J873 GPU coprocessors. It uses the counted-FIFO layout of the existing T8152 SMC v8 entry, at the role's own 0x80-byte window of the shared post-office bank, and nothing else in that bank. The ascwrap lifecycle helpers now serve v8 providers too; every wrapper access first reads the role's PMGR2 PSD state and is skipped (-EHOSTDOWN) unless the service is active. "Active" is state[2:0] == 0. On J873 hardware, services requested from Linux reach state 0 in about 5 ms while the ack bit never sets; the ack is taken to be PMP bookkeeping (nothing runs PMP under Linux) and is logged only. ASC CPU start/stop writes are refused unless the boot parameter apple_mailbox.ascwrap_v8_cpu_control=1 is given. The T8142 v6 and the T8152 SMC v8 entries are unchanged. Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: 560a68746ab55ae1202c7ed38a856138c8fb04aa Ported-paths: drivers/soc/apple/mailbox.c --- drivers/soc/apple/mailbox.c | 544 ++++++++++++++++++++++++++++++++++-- 1 file changed, 519 insertions(+), 25 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 70447abff8745..a21c0e5b004aa 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -31,6 +31,7 @@ #include #include #include +#include #define APPLE_ASC_MBOX_CONTROL_FULL BIT(16) #define APPLE_ASC_MBOX_CONTROL_EMPTY BIT(17) @@ -62,6 +63,84 @@ #define APPLE_T8140_GFX1_IOP_VBAR_BASE 0x482850000ULL #define APPLE_T8142_ASCWRAP_BASE_DELTA 0x100000000ULL +/* + * T8152 (J873) GPU ASCWrap v8 providers. The ADT gfx-asc/gfx1-asc nodes are + * iop,ascwrap-v8 with reg[0] = wrapper (0x74000), reg[1] = iop-vbar (8) and + * reg[2] = the post-office bank 0x316020000/0x10000, which is shared with + * PMP, DCP, DCPEXT0/1 and SCALER0. Only the GFX/GFX1 0x80-byte windows of + * that bank are claimed: offset (2 * msgbox-mailbox-num + !postoffice) * + * 0x80 with msgbox-using-alias=1 (m1n1 asc.c) gives 0x316020100 (mailbox 1) + * and 0x316020200 (mailbox 2). Each window uses the counted-FIFO layout of + * apple_mbox_t8152_v8_hw (RX +0, TX +0x10, status +0x20). + * + * The wrapper is only touched through the lifecycle helpers, and every one + * of those accesses is preceded by a read of the block's PMGR2 PSD service + * (implicit services 81/82 at 0x303a15100/0x303a15200: state +0x80, ack + * +0xbc). The provider never writes PSD registers. + * + * Receive side. AppleASCWrapV8 has no mailbox-level interrupt control + * (_enableInboxInterrupt/_enableOutboxInterrupt/_disableAllInterrupts are + * empty), so has_irq_controls is false and a level interrupt can only be + * masked at the AIC. The ADT interrupt order is proven from the kext + * (_inboxEmptyInterruptIndex() = 0, _outboxNotEmptyInterruptIndex() = 1) + * but not yet observed for GFX. By default the provider therefore never + * enables either GPU mailbox interrupt: apple_mbox_start() starts a + * one-jiffy receive poller, and a full TX FIFO is waited out by sleeping + * polls. Only ascwrap_v8_rx_irq=1 (requires user approval) uses the + * recv-not-empty line; even then a line that keeps firing with nothing to + * read is disabled and receive falls back to the poller. Every post-office + * receive poll is preceded by the same PSD proof as the wrapper accesses. + */ +#define APPLE_ASCWRAP_V8_WRAPPER_SIZE 0x74000 +#define APPLE_ASCWRAP_V8_IOP_VBAR_SIZE 0x8 +#define APPLE_ASCWRAP_V8_MBOX_SIZE 0x80 +#define APPLE_ASCWRAP_V8_PSD_SIZE 0x100 +/* m1n1 asc.c drives v8 CPU start at reg[0] + 0x44 bit 4 (INFERRED for GFX). */ +#define APPLE_ASCWRAP_V8_CPU_CONTROL 0x44 +#define APPLE_ASCWRAP_V8_CPU_STATUS 0x48 +#define APPLE_ASCWRAP_V8_CPU_RUN BIT(4) +#define APPLE_PSD_STATE 0x80 +#define APPLE_PSD_ACK 0xbc +#define APPLE_PSD_STATE_MASK GENMASK(2, 0) +#define APPLE_PSD_STATE_ACTIVE 0 +#define APPLE_PSD_ACK_DONE BIT(0) +/* apple_mbox_ascwrap_v6_lifecycle.missing bit: CPU control not approved. */ +#define APPLE_ASCWRAP_V8_MISSING_CPU_CONTROL_APPROVAL BIT(0) +/* Receive poll period (poll-only mode, or after the storm guard tripped). */ +#define APPLE_ASCWRAP_V8_RX_POLL_JIFFIES 1 +/* Consecutive recv interrupts that drained nothing before the line is cut. */ +#define APPLE_ASCWRAP_V8_RX_STORM_LIMIT 1000 + +#define APPLE_T8152_GFX_WRAPPER_BASE 0x582600000ULL +#define APPLE_T8152_GFX_IOP_VBAR_BASE 0x582050000ULL +#define APPLE_T8152_GFX_MBOX_BASE 0x316020100ULL +#define APPLE_T8152_GFX_PSD_BASE 0x303a15100ULL +#define APPLE_T8152_GFX1_WRAPPER_BASE 0x582e00000ULL +#define APPLE_T8152_GFX1_IOP_VBAR_BASE 0x582850000ULL +#define APPLE_T8152_GFX1_MBOX_BASE 0x316020200ULL +#define APPLE_T8152_GFX1_PSD_BASE 0x303a15200ULL + +/* + * REQUIRES USER APPROVAL. Writing ASC CPU control on a J873 GPU wrapper is + * an unqualified power-affecting operation. The default keeps every v8 CPU + * start/stop request refused (-EPERM); only lifecycle reads are allowed. + */ +static bool ascwrap_v8_cpu_control; +module_param(ascwrap_v8_cpu_control, bool, 0444); +MODULE_PARM_DESC(ascwrap_v8_cpu_control, + "REQUIRES USER APPROVAL: allow T8152 GPU ASCWrap v8 CPU start/stop writes (default: refuse)"); + +/* + * REQUIRES USER APPROVAL. The GFX/GFX1 interrupt assignment is not observed + * on hardware, and a wrongly wired level line would storm with interrupts + * that the handler cannot mask. The default services receive by polling and + * never enables a GPU mailbox interrupt. + */ +static bool ascwrap_v8_rx_irq; +module_param(ascwrap_v8_rx_irq, bool, 0444); +MODULE_PARM_DESC(ascwrap_v8_rx_irq, + "REQUIRES USER APPROVAL: service T8152 GPU ASCWrap v8 receive with the recv-not-empty interrupt (default: poll, no GPU mailbox interrupt is enabled)"); + static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == 0x8110); static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_CONTROL == @@ -107,6 +186,8 @@ static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == struct apple_mbox_hw { unsigned int reg_offset; bool is_ascwrap_v6; + /* T8152 GPU ASCWrap v8: wrapper + iop-vbar + post-office window + PSD. */ + bool is_ascwrap_v8; bool counted_fifo; u64 ascwrap_v6_base_delta; bool ap_initializes_mailboxes; @@ -129,6 +210,68 @@ struct apple_mbox_hw { unsigned int irq_bit_send_empty; }; +/* + * Provider-private state kept outside the consumer-visible struct apple_mbox. + * Every provider is allocated as one of these; apple_mbox_get() hands out a + * pointer to the embedded mailbox. + */ +struct apple_mbox_priv { + struct apple_mbox mbox; + /* ASCWrap v8 only: the provider's PMGR2 PSD service block (read-only). */ + void __iomem *psd_regs; + /* ASCWrap v8 only: receive by polling; fixed at probe. */ + bool rx_poll_only; + /* rx_lock: recv interrupt enabled and not yet disabled again. */ + bool rx_irq_live; + /* rx_lock: consecutive recv interrupts that drained nothing. */ + unsigned int rx_irq_empty_run; + /* rx_lock: an unpowered-PSD receive skip was logged this activation. */ + bool rx_psd_skip_logged; + struct delayed_work rx_poll_work; +}; + +static struct apple_mbox_priv *apple_mbox_to_priv(struct apple_mbox *mbox) +{ + return container_of(mbox, struct apple_mbox_priv, mbox); +} + +/* Both ASCWrap generations share the consumer lifecycle API below. */ +static bool apple_mbox_has_cpu_lifecycle(const struct apple_mbox *mbox) +{ + return mbox->hw->is_ascwrap_v6 || mbox->hw->is_ascwrap_v8; +} + +/* + * Read-only proof that a v8 ASC block is powered before any wrapper MMIO. + * "Powered" is the PMGR2 PSD active state (state[2:0] == 0) with the + * transition acknowledged (ack bit 0), the predicate m1n1's qualified ATC3 + * PSD path waits for. J873 cold boot reads GPU services at state 6 / ack 0, + * which this rejects. Never writes the PSD request register. + */ +static int apple_mbox_ascwrap_v8_psd_powered(struct apple_mbox *mbox, + u32 *state, u32 *ack) +{ + void __iomem *psd = apple_mbox_to_priv(mbox)->psd_regs; + + *state = 0; + *ack = 0; + if (!mbox->hw->is_ascwrap_v8 || !psd) + return -ENODEV; + + *state = readl(psd + APPLE_PSD_STATE); + *ack = readl(psd + APPLE_PSD_ACK); + /* + * J873 hardware, 2026-09-25: requesting power on GPU services 3/81/82 + * from Linux reaches state 0 in ~5 ms but the ack never sets. The ack + * is taken to be PMP bookkeeping (nothing runs PMP under Linux), so the + * active state alone admits wrapper reads; the ack stays telemetry. + */ + if (FIELD_GET(APPLE_PSD_STATE_MASK, *state) != APPLE_PSD_STATE_ACTIVE) + return -EHOSTDOWN; + + return 0; +} + int apple_mbox_ascwrap_v6_get_lifecycle( struct apple_mbox *mbox, struct apple_mbox_ascwrap_v6_lifecycle *lifecycle) @@ -137,7 +280,7 @@ int apple_mbox_ascwrap_v6_get_lifecycle( return -EINVAL; if (READ_ONCE(mbox->removing)) return -ENODEV; - if (!mbox->hw->is_ascwrap_v6) + if (!apple_mbox_has_cpu_lifecycle(mbox)) return -EOPNOTSUPP; *lifecycle = mbox->ascwrap_v6; @@ -147,7 +290,7 @@ EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_get_lifecycle); int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox) { - if (!mbox || !mbox->hw->is_ascwrap_v6) + if (!mbox || !apple_mbox_has_cpu_lifecycle(mbox)) return -EOPNOTSUPP; if (READ_ONCE(mbox->removing)) return -ENODEV; @@ -168,14 +311,34 @@ static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) if (!mbox->ascwrap_v6_cpu_running) return; - cpu_control = (u8 __iomem *)mbox->wrapper_regs + - APPLE_ASCWRAP_V6_CPU_CONTROL; + if (mbox->hw->is_ascwrap_v8) { + u32 state, ack; - /* AppleASCWrapV6::_runCPU(false): clear RUN, reread, clear bit 5. */ - val = readl(cpu_control); - writel(val & ~APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); - val = readl(cpu_control); - writel(val & ~APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR, cpu_control); + /* + * Reached only after an approved v8 start. m1n1 asc_cpu_stop(): + * clear RUN at +0x44. An unpowered block cannot be running, and + * touching its wrapper would fault, so skip the write then. + */ + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V8_CPU_CONTROL; + if (apple_mbox_ascwrap_v8_psd_powered(mbox, &state, &ack)) { + dev_warn(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x is not powered; CPU stop write skipped\n", + state, ack); + } else { + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V8_CPU_RUN, cpu_control); + } + } else { + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V6_CPU_CONTROL; + + /* AppleASCWrapV6::_runCPU(false): clear RUN, reread, clear bit 5. */ + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR, cpu_control); + } mbox->ascwrap_v6_cpu_running = false; pm_runtime_mark_last_busy(mbox->dev); @@ -188,8 +351,11 @@ int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) u32 val; int ret = 0; - if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + if (!mbox || !apple_mbox_has_cpu_lifecycle(mbox) || !mbox->wrapper_regs) return -EOPNOTSUPP; + /* REQUIRES USER APPROVAL: see the ascwrap_v8_cpu_control parameter. */ + if (mbox->hw->is_ascwrap_v8 && !ascwrap_v8_cpu_control) + return -EPERM; mutex_lock(&mbox->lifecycle_lock); if (mbox->removing) { @@ -204,17 +370,36 @@ int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) ret = -EBUSY; goto out_unlock; } + if (mbox->hw->is_ascwrap_v8) { + u32 state, ack; + + ret = apple_mbox_ascwrap_v8_psd_powered(mbox, &state, &ack); + if (ret) { + dev_err(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x is not powered; CPU start refused\n", + state, ack); + goto out_unlock; + } + } ret = pm_runtime_resume_and_get(mbox->dev); if (ret) goto out_unlock; - cpu_control = (u8 __iomem *)mbox->wrapper_regs + - APPLE_ASCWRAP_V6_CPU_CONTROL; + if (mbox->hw->is_ascwrap_v8) { + /* m1n1 asc_cpu_start(): set RUN at reg[0] + 0x44. */ + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V8_CPU_CONTROL; + val = readl(cpu_control); + writel(val | APPLE_ASCWRAP_V8_CPU_RUN, cpu_control); + } else { + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V6_CPU_CONTROL; - /* AppleASCWrapV6::_runCPU(true): read, set RUN, write. */ - val = readl(cpu_control); - writel(val | APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + /* AppleASCWrapV6::_runCPU(true): read, set RUN, write. */ + val = readl(cpu_control); + writel(val | APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + } mbox->ascwrap_v6_cpu_running = true; out_unlock: @@ -227,8 +412,11 @@ int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) { int ret = 0; - if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + if (!mbox || !apple_mbox_has_cpu_lifecycle(mbox) || !mbox->wrapper_regs) return -EOPNOTSUPP; + /* REQUIRES USER APPROVAL: see the ascwrap_v8_cpu_control parameter. */ + if (mbox->hw->is_ascwrap_v8 && !ascwrap_v8_cpu_control) + return -EPERM; mutex_lock(&mbox->lifecycle_lock); if (mbox->removing) { @@ -236,6 +424,7 @@ int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) goto out_unlock; } + /* A no-op unless this provider started the CPU itself. */ apple_mbox_ascwrap_v6_stop_cpu_locked(mbox); out_unlock: @@ -249,13 +438,33 @@ int apple_mbox_ascwrap_v6_get_cpu_state(struct apple_mbox *mbox, { int ret = 0; - if (!mbox || !control || !status || !mbox->hw->is_ascwrap_v6 || + if (!mbox || !control || !status || !apple_mbox_has_cpu_lifecycle(mbox) || !mbox->wrapper_regs) return -EOPNOTSUPP; mutex_lock(&mbox->lifecycle_lock); if (mbox->removing) { ret = -ENODEV; + } else if (mbox->hw->is_ascwrap_v8) { + u32 state, ack; + + /* PSD proof first; an unpowered wrapper read would SError. */ + *control = 0; + *status = 0; + ret = apple_mbox_ascwrap_v8_psd_powered(mbox, &state, &ack); + if (ret) { + dev_info_ratelimited(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x is not powered; CPU_CONTROL/STATUS reads skipped\n", + state, ack); + } else { + *control = readl((u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V8_CPU_CONTROL); + *status = readl((u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V8_CPU_STATUS); + dev_dbg(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x CPU_CONTROL=%#x CPU_STATUS=%#x\n", + state, ack, *control, *status); + } } else { *control = readl((u8 __iomem *)mbox->wrapper_regs + APPLE_ASCWRAP_V6_CPU_CONTROL); @@ -337,6 +546,7 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, bool atomic) { unsigned long flags; + unsigned long deadline = 0; int ret; u32 mbox_ctrl; long t; @@ -362,6 +572,29 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, break; } + if (mbox->hw->is_ascwrap_v8 && + apple_mbox_to_priv(mbox)->rx_poll_only) { + /* + * Poll-only v8: never enable the send-empty line. + * Sleep-poll the TX status, touching MMIO only under + * tx_lock so that removal cannot unmap it underneath. + */ + if (!deadline) + deadline = jiffies + + msecs_to_jiffies(APPLE_MBOX_TX_TIMEOUT); + spin_unlock_irqrestore(&mbox->tx_lock, flags); + if (time_after(jiffies, deadline)) + return -ETIMEDOUT; + usleep_range(100, 200); + spin_lock_irqsave(&mbox->tx_lock, flags); + if (unlikely(READ_ONCE(mbox->removing))) { + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return -ENODEV; + } + mbox_ctrl = readl_relaxed(mbox->regs + + mbox->hw->a2i_control); + continue; + } /* * The interrupt is level triggered and will keep firing as long as the * FIFO is empty. It will also keep firing if the FIFO was empty @@ -426,8 +659,25 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) { struct apple_mbox_msg msg; int ret = 0; + u32 mbox_ctrl; + + if (mbox->hw->is_ascwrap_v8) { + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + u32 state, ack; + + /* PSD proof before the post-office status read. */ + ret = apple_mbox_ascwrap_v8_psd_powered(mbox, &state, &ack); + if (ret) { + if (!priv->rx_psd_skip_logged) + dev_info(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x is not powered; receive poll skipped\n", + state, ack); + priv->rx_psd_skip_logged = true; + return ret; + } + } - u32 mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); + mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); while (!apple_mbox_rx_empty(mbox, mbox_ctrl)) { msg = apple_mbox_read_msg(mbox); @@ -453,13 +703,42 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) return ret; } +/* + * ASCWrap v8 storm guard, rx_lock held. The handler cannot mask the line at + * the mailbox (no irq controls), so a line that keeps firing with nothing to + * read (for example a send-empty line wired as recv-not-empty) would lock up + * the CPU while always returning IRQ_HANDLED. Cut it at the AIC and poll. + */ +static void apple_mbox_v8_rx_irq_guard(struct apple_mbox *mbox, int drained) +{ + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + + if (drained > 0) { + priv->rx_irq_empty_run = 0; + return; + } + if (!priv->rx_irq_live || + ++priv->rx_irq_empty_run < APPLE_ASCWRAP_V8_RX_STORM_LIMIT) + return; + + priv->rx_irq_live = false; + disable_irq_nosync(mbox->irq_recv_not_empty); + dev_err(mbox->dev, + "ASCWrap v8 recv-not-empty IRQ %d fired %u times without a message; disabled, receive falls back to polling\n", + mbox->irq_recv_not_empty, priv->rx_irq_empty_run); + queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, 0); +} + static irqreturn_t apple_mbox_recv_irq(int irq, void *data) { struct apple_mbox *mbox = data; + int drained = 0; spin_lock(&mbox->rx_lock); if (likely(!READ_ONCE(mbox->removing))) - apple_mbox_poll_locked(mbox); + drained = apple_mbox_poll_locked(mbox); + if (mbox->hw->is_ascwrap_v8) + apple_mbox_v8_rx_irq_guard(mbox, drained); spin_unlock(&mbox->rx_lock); return IRQ_HANDLED; @@ -481,6 +760,61 @@ int apple_mbox_poll(struct apple_mbox *mbox) } EXPORT_SYMBOL(apple_mbox_poll); +/* ASCWrap v8 receive poller: poll-only mode, or after the storm guard. */ +static void apple_mbox_v8_rx_poll_work(struct work_struct *work) +{ + struct apple_mbox_priv *priv = container_of(to_delayed_work(work), + struct apple_mbox_priv, + rx_poll_work); + struct apple_mbox *mbox = &priv->mbox; + + if (!READ_ONCE(mbox->active) || READ_ONCE(mbox->removing)) + return; + apple_mbox_poll(mbox); + queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, + APPLE_ASCWRAP_V8_RX_POLL_JIFFIES); +} + +/* lifecycle_lock held, mbox->active already set. */ +static void apple_mbox_v8_rx_start(struct apple_mbox *mbox) +{ + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + unsigned long flags; + + spin_lock_irqsave(&mbox->rx_lock, flags); + priv->rx_irq_empty_run = 0; + priv->rx_psd_skip_logged = false; + priv->rx_irq_live = !priv->rx_poll_only; + spin_unlock_irqrestore(&mbox->rx_lock, flags); + + if (priv->rx_poll_only) + queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, 0); + else + enable_irq(mbox->irq_recv_not_empty); +} + +/* + * lifecycle_lock held (or removal), mbox->active already cleared. The storm + * guard and this path each claim rx_irq_live under rx_lock, so exactly one + * of them balances the enable_irq(). The poller re-queues itself, which + * cancel_delayed_work_sync() handles; it never takes lifecycle_lock. + */ +static void apple_mbox_v8_rx_stop(struct apple_mbox *mbox) +{ + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + unsigned long flags; + bool live; + + spin_lock_irqsave(&mbox->rx_lock, flags); + live = priv->rx_irq_live; + priv->rx_irq_live = false; + spin_unlock_irqrestore(&mbox->rx_lock, flags); + + if (live) + disable_irq(mbox->irq_recv_not_empty); + cancel_delayed_work_sync(&priv->rx_poll_work); +} + int apple_mbox_start(struct apple_mbox *mbox) { u32 control; @@ -522,6 +856,12 @@ int apple_mbox_start(struct apple_mbox *mbox) mbox->regs + mbox->hw->irq_enable); } + if (mbox->hw->is_ascwrap_v8) { + WRITE_ONCE(mbox->active, true); + apple_mbox_v8_rx_start(mbox); + goto out_unlock; + } + enable_irq(mbox->irq_recv_not_empty); mbox->active = true; @@ -536,8 +876,13 @@ static void apple_mbox_stop_locked(struct apple_mbox *mbox) if (!mbox->active) return; - mbox->active = false; - disable_irq(mbox->irq_recv_not_empty); + if (mbox->hw->is_ascwrap_v8) { + WRITE_ONCE(mbox->active, false); + apple_mbox_v8_rx_stop(mbox); + } else { + mbox->active = false; + disable_irq(mbox->irq_recv_not_empty); + } pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); } @@ -582,13 +927,19 @@ static void apple_mbox_quiesce(void *data) * inversion. The removal fence prevents any matching restart. */ was_active = mbox->active; - mbox->active = false; + WRITE_ONCE(mbox->active, false); mutex_unlock(&mbox->lifecycle_lock); if (was_active) { - disable_irq(mbox->irq_recv_not_empty); + if (mbox->hw->is_ascwrap_v8) + apple_mbox_v8_rx_stop(mbox); + else + disable_irq(mbox->irq_recv_not_empty); pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); } + /* The v8 poller also stops on the removal fence; make sure it is gone. */ + if (mbox->hw->is_ascwrap_v8) + cancel_delayed_work_sync(&apple_mbox_to_priv(mbox)->rx_poll_work); /* The TX IRQ is normally disabled; force and synchronize that state. */ disable_irq(mbox->irq_send_empty); @@ -656,18 +1007,135 @@ struct apple_mbox *apple_mbox_get_byname(struct device *dev, const char *name) } EXPORT_SYMBOL(apple_mbox_get_byname); +static void apple_mbox_iounmap(void *addr) +{ + iounmap((__force void __iomem *)addr); +} + +/* + * Validate and map one T8152 GPU ASCWrap v8 provider. Nothing here reads or + * writes device registers: the four windows are only mapped. The wrapper, + * iop-vbar and post-office window are requested exclusively; the PSD block + * is mapped without a request so that a GPU power owner may also map the + * same service block (the provider itself only ever reads it). + */ +static int apple_mbox_probe_ascwrap_v8(struct platform_device *pdev, + struct apple_mbox *mbox) +{ + struct device *dev = &pdev->dev; + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + struct resource *wrapper, *iop_vbar, *window, *psd; + u64 wrapper_base, iop_vbar_base, window_base, psd_base; + const char *firmware_role; + int ret; + + wrapper = platform_get_resource_byname(pdev, IORESOURCE_MEM, "wrapper"); + iop_vbar = platform_get_resource_byname(pdev, IORESOURCE_MEM, "iop-vbar"); + window = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mailbox"); + psd = platform_get_resource_byname(pdev, IORESOURCE_MEM, "psd"); + if (!wrapper || !iop_vbar || !window || !psd || + resource_size(wrapper) != APPLE_ASCWRAP_V8_WRAPPER_SIZE || + resource_size(iop_vbar) != APPLE_ASCWRAP_V8_IOP_VBAR_SIZE || + resource_size(window) != APPLE_ASCWRAP_V8_MBOX_SIZE || + resource_size(psd) != APPLE_ASCWRAP_V8_PSD_SIZE) + return dev_err_probe(dev, -EINVAL, + "invalid ASCWrap v8 provider resources\n"); + + ret = of_property_read_string(dev->of_node, "apple,firmware-role", + &firmware_role); + if (ret) + return dev_err_probe(dev, ret, + "missing ASCWrap v8 firmware role\n"); + + if (!strcmp(firmware_role, "GFX")) { + mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; + wrapper_base = APPLE_T8152_GFX_WRAPPER_BASE; + iop_vbar_base = APPLE_T8152_GFX_IOP_VBAR_BASE; + window_base = APPLE_T8152_GFX_MBOX_BASE; + psd_base = APPLE_T8152_GFX_PSD_BASE; + } else if (!strcmp(firmware_role, "GFX1")) { + mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1; + wrapper_base = APPLE_T8152_GFX1_WRAPPER_BASE; + iop_vbar_base = APPLE_T8152_GFX1_IOP_VBAR_BASE; + window_base = APPLE_T8152_GFX1_MBOX_BASE; + psd_base = APPLE_T8152_GFX1_PSD_BASE; + } else { + return dev_err_probe(dev, -EINVAL, + "invalid ASCWrap v8 firmware role\n"); + } + + /* + * Fixed per-SoC CPU-physical windows. In particular the post-office + * window must be exactly this role's 0x80 bytes: the rest of the bank + * belongs to PMP, DCP, DCPEXT and SCALER0. + */ + if (wrapper->start != wrapper_base || iop_vbar->start != iop_vbar_base || + window->start != window_base || psd->start != psd_base) + return dev_err_probe(dev, -EINVAL, + "%s ASCWrap v8 resources do not match the T8152 profile\n", + firmware_role); + + mbox->ascwrap_v6.wrapper_start = wrapper->start; + mbox->ascwrap_v6.wrapper_size = resource_size(wrapper); + mbox->ascwrap_v6.iop_vbar_start = iop_vbar->start; + mbox->ascwrap_v6.iop_vbar_size = resource_size(iop_vbar); + mbox->ascwrap_v6.missing = ascwrap_v8_cpu_control ? + 0 : APPLE_ASCWRAP_V8_MISSING_CPU_CONTROL_APPROVAL; + priv->rx_poll_only = !ascwrap_v8_rx_irq; + INIT_DELAYED_WORK(&priv->rx_poll_work, apple_mbox_v8_rx_poll_work); + + mbox->wrapper_regs = devm_platform_ioremap_resource_byname(pdev, "wrapper"); + if (IS_ERR(mbox->wrapper_regs)) + return PTR_ERR(mbox->wrapper_regs); + + mbox->iop_vbar_regs = + devm_platform_ioremap_resource_byname(pdev, "iop-vbar"); + if (IS_ERR(mbox->iop_vbar_regs)) + return PTR_ERR(mbox->iop_vbar_regs); + + mbox->regs = devm_platform_ioremap_resource_byname(pdev, "mailbox"); + if (IS_ERR(mbox->regs)) + return PTR_ERR(mbox->regs); + + /* + * Same posted/non-posted choice devm_ioremap_resource() makes, but + * without claiming the region. Released after apple_mbox_quiesce(), + * which is registered later and may still read the PSD state. + */ + if (psd->flags & IORESOURCE_MEM_NONPOSTED) + priv->psd_regs = ioremap_np(psd->start, resource_size(psd)); + else + priv->psd_regs = ioremap(psd->start, resource_size(psd)); + if (!priv->psd_regs) + return -ENOMEM; + ret = devm_add_action_or_reset(dev, apple_mbox_iounmap, + (__force void *)priv->psd_regs); + if (ret) + return ret; + + dev_info(dev, + "%s ASCWrap v8 provider: wrapper %pR, post-office window %pR, PSD %pR; CPU control %s; receive %s\n", + firmware_role, wrapper, window, psd, + ascwrap_v8_cpu_control ? "APPROVED" : "refused (default)", + ascwrap_v8_rx_irq ? "recv-not-empty IRQ (APPROVED)" : + "polled, no IRQ enabled (default)"); + return 0; +} + static int apple_mbox_probe(struct platform_device *pdev) { int ret; char *irqname; const char *firmware_role; + struct apple_mbox_priv *priv; struct apple_mbox *mbox; struct device *dev = &pdev->dev; struct resource *res, *iop_vbar_res; - mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); - if (!mbox) + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) return -ENOMEM; + mbox = &priv->mbox; mbox->dev = &pdev->dev; mbox->hw = of_device_get_match_data(dev); @@ -737,6 +1205,10 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->regs = (u8 __iomem *)mbox->wrapper_regs + mbox->hw->reg_offset; + } else if (mbox->hw->is_ascwrap_v8) { + ret = apple_mbox_probe_ascwrap_v8(pdev, mbox); + if (ret) + return ret; } else { res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res || resource_size(res) < mbox->hw->reg_offset + @@ -904,6 +1376,27 @@ static const struct apple_mbox_hw apple_mbox_t8152_v8_hw = { .has_irq_controls = false, }; +/* T8152/J873 GPU GFX/GFX1 ASCWrap v8: the counted-FIFO layout of + * apple_mbox_t8152_v8_hw at the role's post-office window (reg "mailbox"), + * plus wrapper lifecycle reads gated on the role's PSD service. The SMC's + * apple,t8152-ascwrap-v8-mailbox entry above is unchanged. + */ +static const struct apple_mbox_hw apple_mbox_t8152_gfx_ascwrap_v8_hw = { + .is_ascwrap_v8 = true, + .counted_fifo = true, + .control_full = GENMASK(11, 8), /* free TX slots */ + .control_empty = GENMASK(3, 0), /* RX occupancy */ + .a2i_control = 0x20, + .a2i_send0 = 0x10, + .a2i_send1 = 0x18, + .i2a_control = 0x20, + .i2a_recv0 = 0x00, + .i2a_recv1 = 0x08, + .has_irq_controls = false, +}; + +static_assert(0x20 + sizeof(u32) <= APPLE_ASCWRAP_V8_MBOX_SIZE); + static const struct apple_mbox_hw apple_mbox_m3_hw = { .control_full = APPLE_M3_MBOX_CONTROL_FULL, .control_empty = APPLE_M3_MBOX_CONTROL_EMPTY, @@ -925,6 +1418,7 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t8152-ascwrap-v8-mailbox", .data = &apple_mbox_t8152_v8_hw }, + { .compatible = "apple,t8152-gfx-ascwrap-v8", .data = &apple_mbox_t8152_gfx_ascwrap_v8_hw }, { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_ascwrap_v6_hw }, { .compatible = "apple,t8142-ascwrap-v6", .data = &apple_mbox_t8142_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, From 5102e861a5307cf040c4df764f215cf5db35cb06 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Sun, 27 Sep 2026 21:21:52 +0000 Subject: [PATCH 0143/1579] soc: apple: start T8152 PMP and preserve J873 GPU bring-up evidence Path-scoped import for T6050 SPTM storage bring-up. Other changes from the source commit are not included in this port. Original-commit: a9e4a9181033e0e09bcb1c6b112b6426eccd7a70 Ported-paths: drivers/soc/apple/rtkit.c, drivers/soc/apple/mailbox.c, drivers/soc/apple/rtkit-internal.h --- drivers/soc/apple/mailbox.c | 60 ++++++++++++++++++++++++------ drivers/soc/apple/rtkit-internal.h | 2 + drivers/soc/apple/rtkit.c | 34 +++++++++++++++++ 3 files changed, 84 insertions(+), 12 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index a21c0e5b004aa..9eff6f44a5bf8 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -188,6 +188,8 @@ struct apple_mbox_hw { bool is_ascwrap_v6; /* T8152 GPU ASCWrap v8: wrapper + iop-vbar + post-office window + PSD. */ bool is_ascwrap_v8; + /* PMP's postoffice is qualified; its IRQ ordering is not yet known. */ + bool rx_poll_required; bool counted_fifo; u64 ascwrap_v6_base_delta; bool ap_initializes_mailboxes; @@ -235,6 +237,11 @@ static struct apple_mbox_priv *apple_mbox_to_priv(struct apple_mbox *mbox) return container_of(mbox, struct apple_mbox_priv, mbox); } +static bool apple_mbox_has_rx_poller(const struct apple_mbox *mbox) +{ + return mbox->hw->is_ascwrap_v8 || mbox->hw->rx_poll_required; +} + /* Both ASCWrap generations share the consumer lifecycle API below. */ static bool apple_mbox_has_cpu_lifecycle(const struct apple_mbox *mbox) { @@ -572,7 +579,7 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, break; } - if (mbox->hw->is_ascwrap_v8 && + if (apple_mbox_has_rx_poller(mbox) && apple_mbox_to_priv(mbox)->rx_poll_only) { /* * Poll-only v8: never enable the send-empty line. @@ -856,7 +863,7 @@ int apple_mbox_start(struct apple_mbox *mbox) mbox->regs + mbox->hw->irq_enable); } - if (mbox->hw->is_ascwrap_v8) { + if (apple_mbox_has_rx_poller(mbox)) { WRITE_ONCE(mbox->active, true); apple_mbox_v8_rx_start(mbox); goto out_unlock; @@ -876,7 +883,7 @@ static void apple_mbox_stop_locked(struct apple_mbox *mbox) if (!mbox->active) return; - if (mbox->hw->is_ascwrap_v8) { + if (apple_mbox_has_rx_poller(mbox)) { WRITE_ONCE(mbox->active, false); apple_mbox_v8_rx_stop(mbox); } else { @@ -930,7 +937,7 @@ static void apple_mbox_quiesce(void *data) WRITE_ONCE(mbox->active, false); mutex_unlock(&mbox->lifecycle_lock); if (was_active) { - if (mbox->hw->is_ascwrap_v8) + if (apple_mbox_has_rx_poller(mbox)) apple_mbox_v8_rx_stop(mbox); else disable_irq(mbox->irq_recv_not_empty); @@ -938,13 +945,15 @@ static void apple_mbox_quiesce(void *data) pm_runtime_put_autosuspend(mbox->dev); } /* The v8 poller also stops on the removal fence; make sure it is gone. */ - if (mbox->hw->is_ascwrap_v8) + if (apple_mbox_has_rx_poller(mbox)) cancel_delayed_work_sync(&apple_mbox_to_priv(mbox)->rx_poll_work); /* The TX IRQ is normally disabled; force and synchronize that state. */ - disable_irq(mbox->irq_send_empty); + if (!mbox->hw->rx_poll_required) + disable_irq(mbox->irq_send_empty); complete_all(&mbox->tx_empty); - synchronize_irq(mbox->irq_recv_not_empty); + if (!mbox->hw->rx_poll_required) + synchronize_irq(mbox->irq_recv_not_empty); spin_lock_irqsave(&mbox->rx_lock, flags); had_rx = mbox->rx; @@ -1211,6 +1220,10 @@ static int apple_mbox_probe(struct platform_device *pdev) return ret; } else { res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (mbox->hw->rx_poll_required && + (!of_machine_is_compatible("apple,j873") || !res || + res->start != 0x316020000ULL || resource_size(res) != 0x80)) + return -EINVAL; if (!res || resource_size(res) < mbox->hw->reg_offset + max(mbox->hw->i2a_recv1 + sizeof(u64), mbox->hw->i2a_control + sizeof(u32))) @@ -1223,6 +1236,18 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->regs = (u8 __iomem *)mbox->regs + mbox->hw->reg_offset; } + spin_lock_init(&mbox->rx_lock); + spin_lock_init(&mbox->tx_lock); + mutex_init(&mbox->lifecycle_lock); + init_completion(&mbox->tx_empty); + if (mbox->hw->rx_poll_required) { + priv->rx_poll_only = true; + INIT_DELAYED_WORK(&priv->rx_poll_work, apple_mbox_v8_rx_poll_work); + mbox->irq_recv_not_empty = mbox->irq_send_empty = -1; + dev_info(dev, "J873 PMP postoffice: paired transfers, bounded polling, no IRQ claim\n"); + goto setup_pm; + } + mbox->irq_recv_not_empty = platform_get_irq_byname(pdev, "recv-not-empty"); if (mbox->irq_recv_not_empty < 0) @@ -1232,11 +1257,6 @@ static int apple_mbox_probe(struct platform_device *pdev) if (mbox->irq_send_empty < 0) return -ENODEV; - spin_lock_init(&mbox->rx_lock); - spin_lock_init(&mbox->tx_lock); - mutex_init(&mbox->lifecycle_lock); - init_completion(&mbox->tx_empty); - irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); if (!irqname) return -ENOMEM; @@ -1257,6 +1277,7 @@ static int apple_mbox_probe(struct platform_device *pdev) if (ret) return ret; +setup_pm: ret = devm_pm_runtime_enable(dev); if (ret) return ret; @@ -1376,6 +1397,20 @@ static const struct apple_mbox_hw apple_mbox_t8152_v8_hw = { .has_irq_controls = false, }; +static const struct apple_mbox_hw apple_mbox_t8152_pmp_hw = { + .rx_poll_required = true, + .counted_fifo = true, + .control_full = GENMASK(11, 8), + .control_empty = GENMASK(3, 0), + .a2i_control = 0x20, + .a2i_send0 = 0x10, + .a2i_send1 = 0x18, + .i2a_control = 0x20, + .i2a_recv0 = 0x00, + .i2a_recv1 = 0x08, + .has_irq_controls = false, +}; + /* T8152/J873 GPU GFX/GFX1 ASCWrap v8: the counted-FIFO layout of * apple_mbox_t8152_v8_hw at the role's post-office window (reg "mailbox"), * plus wrapper lifecycle reads gated on the role's PSD service. The SMC's @@ -1417,6 +1452,7 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { }; static const struct of_device_id apple_mbox_of_match[] = { + { .compatible = "apple,t8152-pmp-mailbox", .data = &apple_mbox_t8152_pmp_hw }, { .compatible = "apple,t8152-ascwrap-v8-mailbox", .data = &apple_mbox_t8152_v8_hw }, { .compatible = "apple,t8152-gfx-ascwrap-v8", .data = &apple_mbox_t8152_gfx_ascwrap_v8_hw }, { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_ascwrap_v6_hw }, diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 7ea7c564e930e..dc59f9d3f2250 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -48,6 +48,7 @@ struct apple_rtkit { bool awaiting_iop_idle; bool epmap_only; bool firmware_owned_crashlog; + bool t8152_pmp; bool crashlog_endpoint_started; bool hello_seen; bool epmap_finished; @@ -59,6 +60,7 @@ struct apple_rtkit { struct apple_rtkit_shmem ioreport_buffer; struct apple_rtkit_shmem crashlog_buffer; struct apple_rtkit_shmem oslog_buffer; + struct apple_rtkit_shmem pmp_aux_buffer; struct apple_rtkit_shmem syslog_buffer; char *syslog_msg_buffer; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 0993a84697104..27456534be440 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -5,6 +5,7 @@ */ #include "rtkit-internal.h" +#include static_assert(APPLE_RTKIT_EPMAP_SNAPSHOT_WORDS * 64 == APPLE_RTKIT_MAX_ENDPOINTS); @@ -24,6 +25,7 @@ enum { APPLE_RTKIT_EP_SYSLOG = 2, APPLE_RTKIT_EP_DEBUG = 3, APPLE_RTKIT_EP_IOREPORT = 4, + APPLE_RTKIT_EP_PMP_AUX = 7, APPLE_RTKIT_EP_OSLOG = 8, APPLE_RTKIT_EP_TRACEKIT = 0xa, }; @@ -259,6 +261,13 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) /* the management endpoint is started by default */ case APPLE_RTKIT_EP_MGMT: break; + case APPLE_RTKIT_EP_PMP_AUX: + if (rtk->t8152_pmp) { + apple_rtkit_start_ep(rtk, ep); + break; + } + dev_warn(rtk->dev, "RTKit: unqualified endpoint 7\n"); + break; /* without starting these RTKit refuses to boot */ case APPLE_RTKIT_EP_SYSLOG: @@ -416,6 +425,7 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, buffer->iova >> 12); } else { reply = FIELD_PREP(APPLE_RTKIT_SYSLOG_TYPE, + ep == APPLE_RTKIT_EP_PMP_AUX ? 5 : APPLE_RTKIT_BUFFER_REQUEST); reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE, buffer->size >> 12); @@ -642,6 +652,24 @@ static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg) } } +/* Native J873 26A428 capture074: EP7 requests 8KiB with type1 and + * receives type5, the same size and an unshifted 44-bit DVA. Keep the + * buffer alive for the RTKit session; never replace a live allocation. + */ +static void apple_rtkit_pmp_aux_rx(struct apple_rtkit *rtk, u64 msg) +{ + int ret; + + if (!rtk->t8152_pmp || msg != 0x10200000000000ULL || + rtk->pmp_aux_buffer.size) { + dev_err(rtk->dev, "RTKit: refused PMP EP7 request %#llx\n", msg); + return; + } + ret = apple_rtkit_common_rx_get_buffer(rtk, &rtk->pmp_aux_buffer, + APPLE_RTKIT_EP_PMP_AUX, msg); + dev_info(rtk->dev, "RTKit: PMP EP7 8KiB buffer handshake result=%d\n", ret); +} + static void apple_rtkit_rx_work(struct work_struct *work) { struct apple_rtkit_rx_work *rtk_work = @@ -667,6 +695,9 @@ static void apple_rtkit_rx_work(struct work_struct *work) case APPLE_RTKIT_EP_OSLOG: apple_rtkit_oslog_rx(rtk, rtk_work->msg); break; + case APPLE_RTKIT_EP_PMP_AUX: + apple_rtkit_pmp_aux_rx(rtk, rtk_work->msg); + break; case APPLE_RTKIT_APP_ENDPOINT_START ... 0xff: if (rtk->ops->recv_message) rtk->ops->recv_message(rtk->cookie, rtk_work->ep, @@ -855,6 +886,7 @@ static struct apple_rtkit *__apple_rtkit_init( rtk->ops = ops; rtk->epmap_only = epmap_only; rtk->firmware_owned_crashlog = firmware_owned_crashlog; + rtk->t8152_pmp = of_device_is_compatible(dev->of_node, "apple,t8152-pmp-v2"); init_completion(&rtk->epmap_completion); init_completion(&rtk->iop_pwr_ack_completion); @@ -966,6 +998,7 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer); apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer); apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer); + apple_rtkit_free_buffer(rtk, &rtk->pmp_aux_buffer); kfree(rtk->syslog_msg_buffer); @@ -1304,6 +1337,7 @@ void apple_rtkit_free(struct apple_rtkit *rtk) apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer); apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer); apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer); + apple_rtkit_free_buffer(rtk, &rtk->pmp_aux_buffer); kfree(rtk->syslog_msg_buffer); kfree(rtk); From b31fde49a13a10c419daf657070c1cad8bbb7fba Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 18:05:09 -0400 Subject: [PATCH 0144/1579] Document authored j873 storage ports and T6050 baseline validation --- .../arch/arm64/t6050-sptm-storage-import.rst | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 Documentation/arch/arm64/t6050-sptm-storage-import.rst diff --git a/Documentation/arch/arm64/t6050-sptm-storage-import.rst b/Documentation/arch/arm64/t6050-sptm-storage-import.rst new file mode 100644 index 0000000000000..07475feb03619 --- /dev/null +++ b/Documentation/arch/arm64/t6050-sptm-storage-import.rst @@ -0,0 +1,41 @@ +T6050 SPTM storage import +======================== + +The ANS driver, RTKit and mailbox changes are path-scoped ports from +``aurora-silicon/linux-aurora@j873`` at +``a9e4a9181033e0e09bcb1c6b112b6426eccd7a70``. The imported commits retain +the source author name, email and date. Each has an ``Original-commit`` +trailer and lists its included paths; unrelated GPU, desktop, thermal and +Wi-Fi changes mentioned in original commit messages were not imported. + +Twenty-five commits were applied. A merge whose selected file changes were +already present was skipped. The pre-existing admin-buffer alignment and +admin-queue destruction fixes were retained rather than duplicated. The +existing wrapped-key NVMe crypto profile, key/IV programming and key cleanup +were preserved during conflict resolution. The previous removal of the +legacy NULL-PRP chicken-bit write was also retained. + +The resulting RTKit and mailbox sources and headers match the pinned j873 +versions. The NVMe source differs intentionally by the retained crypto and +NULL-PRP handling. The SPTM-backed SART frontend is retained; j873's raw SART +implementation does not replace it. NVMe is built in for the RAM-disk test +kernel. This import by itself does not enable ANS in the T6050 guest DT. + +The merged kernel built successfully and booted in QEMU and on the physical +J714s with all 18 CPUs. All per-CPU sleep/SIMD checks passed, followed by +three timer wakes. The hardware boot also repeated the SART and DART +map/unmap checks with the Linux framebuffer console. These tests qualify +the merged kernel baseline, not ANS transfers: its guest DT still omits ANS. + +Hardware evidence preceding the import establishes SART and DART map/unmap +from Linux, and all 18 CPUs passing sleep/SIMD checks. Read-only ANS inspection +found CPU_CONTROL=0x10, BOOT_STATUS=0xde71ce55, CC=0x474000 and CSTS=0 in +the primary and secure controller windows. The device is boot-ready but its +controller is disabled. Actual ANS DMA, SSD reads, throughput, DVFS and a +desktop remain to be qualified. No SSD write or format is part of this test. + +Further reference commits are ``m1n1-aurora@j873`` +``29e59aaaeb5de3f26456dcc3ecdcd7643a873b58`` and ``u-boot@j873`` +``52474099334fdd459e49236a5392a10ccd6d158d``. Their macOS 27 handoff and +queue handling guide the T6050 adaptation; their addresses must not be +substituted for the captured T6050 ADT. From 8153565e7166ce5d54d623fe1aab1131e9519629 Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Sun, 27 Sep 2026 18:25:13 -0400 Subject: [PATCH 0145/1579] thunderbolt: apple: drop PCIe lock before removing the NHI apple_cio_stop() holds pcie_tunnel_lock while destroying the NHI, whose domain removal acquires tb->lock and drains hotplug work. A concurrent unplug worker can hold tb->lock and wait for pcie_tunnel_lock through apple_nhi_pci_tunnel_deactivate(), creating an ABBA deadlock. Gate PCIe activation under pcie_tunnel_lock, drain pending activation work without that lock, and drop the lock before removing the NHI. Keep PCIe quiesce and final depopulation serialized. Reset the gate before bringing up the next NHI so a later cable attachment can activate normally. The gate also prevents a post-activation callback from requeueing the worker after shutdown has drained it. Normal tunnel bandwidth and workqueue concurrency are unchanged. --- drivers/thunderbolt/apple.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index fbe5930e9cfce..16fe73fad84fc 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -170,6 +170,8 @@ struct apple_cio { struct delayed_work pcie_tunnel_work; bool pcie_tunnel_requested; bool pcie_tunnel_populated; + /* Protected by pcie_tunnel_lock; gates work while the NHI is removed. */ + bool pcie_tunnel_stopping; /* * Thunderbolt DP tunnel routing, ported from aurora-silicon/linux#8. @@ -615,7 +617,8 @@ static void apple_cio_pcie_tunnel_work(struct work_struct *work) int ret; mutex_lock(&acio->pcie_tunnel_lock); - if (!READ_ONCE(acio->pcie_tunnel_requested)) { + if (acio->pcie_tunnel_stopping || + !READ_ONCE(acio->pcie_tunnel_requested)) { mutex_unlock(&acio->pcie_tunnel_lock); return; } @@ -665,6 +668,10 @@ static int apple_nhi_pci_tunnel_pre_activate(struct tb_nhi *nhi) { struct apple_nhi *anhi = nhi_to_anhi(nhi); struct apple_cio *acio = anhi->acio; + guard(mutex)(&acio->pcie_tunnel_lock); + + if (acio->pcie_tunnel_stopping) + return -ESHUTDOWN; /* * Remember that the PCIe tunnel reached activation. PCIe-C must not be @@ -678,6 +685,10 @@ static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) { struct apple_nhi *anhi = nhi_to_anhi(nhi); struct apple_cio *acio = anhi->acio; + guard(mutex)(&acio->pcie_tunnel_lock); + + if (acio->pcie_tunnel_stopping) + return -ESHUTDOWN; /* * This is the boundary at which the tunnel becomes usable. @@ -1452,7 +1463,13 @@ static void apple_cio_stop(struct apple_cio *acio) * After we shut down the ACIO co-processor we will no longer be able to access * the MMIO space of these so make sure nothing tries to do just that. */ + /* Serialize the shutdown gate with activation callbacks before draining + * the worker. No callback may queue a new activation after this point. + */ + mutex_lock(&acio->pcie_tunnel_lock); + acio->pcie_tunnel_stopping = true; WRITE_ONCE(acio->pcie_tunnel_requested, false); + mutex_unlock(&acio->pcie_tunnel_lock); cancel_delayed_work_sync(&acio->pcie_tunnel_work); mutex_lock(&acio->pcie_tunnel_lock); @@ -1479,11 +1496,18 @@ static void apple_cio_stop(struct apple_cio *acio) * PCIe-C has already reset its port and can wedge waiting on dead control * traffic. The remaining children are still removed in reverse creation * order below. + * + * Do not hold pcie_tunnel_lock here. Domain removal takes tb->lock + * and drains its workqueue; a concurrent unplug worker holds tb->lock + * while apple_nhi_pci_tunnel_deactivate() takes pcie_tunnel_lock. + * The shutdown gate keeps PCIe activation stopped across this gap. */ + mutex_unlock(&acio->pcie_tunnel_lock); nhi_pdev = READ_ONCE(acio->nhi_pdev); if (nhi_pdev) of_platform_device_destroy(&nhi_pdev->dev, NULL); + mutex_lock(&acio->pcie_tunnel_lock); of_platform_depopulate(acio->dev); acio->pcie_tunnel_populated = false; mutex_unlock(&acio->pcie_tunnel_lock); @@ -1575,6 +1599,11 @@ static int apple_cio_start(struct apple_cio *acio) apple_tunable_apply(acio->rc_base, acio->rc_tunable); + /* The previous NHI and all its callbacks are gone before a new start. */ + mutex_lock(&acio->pcie_tunnel_lock); + acio->pcie_tunnel_stopping = false; + mutex_unlock(&acio->pcie_tunnel_lock); + /* * Bring up devices which are part of ACIO and are now accessible by the main SoC * and specifically wait for the NHI to be up to prevent concurrent shutdowns. From 8ac0a36fd9bdc466cf110fddbbf2d9b9568a0852 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 18:55:40 -0400 Subject: [PATCH 0146/1579] nvme: bring up T6050 ANS through batched SPTM operations --- Documentation/arch/arm64/t6050-sptm-ans.rst | 33 +++ drivers/nvme/host/apple.c | 216 +++++++++++++++++++- drivers/soc/apple/sart.c | 12 +- 3 files changed, 246 insertions(+), 15 deletions(-) create mode 100644 Documentation/arch/arm64/t6050-sptm-ans.rst diff --git a/Documentation/arch/arm64/t6050-sptm-ans.rst b/Documentation/arch/arm64/t6050-sptm-ans.rst new file mode 100644 index 0000000000000..4dd3530bc78a2 --- /dev/null +++ b/Documentation/arch/arm64/t6050-sptm-ans.rst @@ -0,0 +1,33 @@ +T6050 ANS under resident SPTM +============================ + +The apple,t6050-nvme-sptm profile uses a third sptm2mmio resource named +"sptm". Doorbell offset 0x1010 accepts the version1 batch format; feature +bit3 advertises NVMe table6 forwarding. Queue setup batches INIT, +ENABLE_CG, ADMIN_QUEUE_REGS, IOQA, IOCQ and IOSQ into one guest exit. +SPTM owns the real TCBs. Request mapping passes guest TCB IPA and byte count; +the host translates metadata and admits physical DMA pages without copying +payload data. CREATE_CQ/SQ reference their PRP1 queue page even with no +block-layer payload. Completion invalidation must release SPTM DMA references +before buffers may be reused. + +DMA ranges translate the guest RAM IPA window to its actual physical backing. +SART receives DMA addresses and converts them back to guest physical before +submitting its own transport. Queue and RTKit grants are16KiB aligned. +The firmware must already be running; this profile does not own its reset. +The obsolete linear-SQ control register is not written on T6050. + +Hardware qualification on J714s, macOS27.0 firmware26A428: + +* All18 CPUs boot, pass sleep/SIMD checks, and deliver timer interrupts. +* RTKit protocol12 initializes using mailbox IRQs1347/1350 (ADT indices1/2). +* Native table6 queue setup, controller identification and namespace discovery + succeed. The namespace reports2 TB,4096-byte sectors and seven partitions. +* GPT header CRC and two64MiB O_DIRECT reads match, CRC32 0x9b30c356. +* Initial QD1 read-syscall throughput is1768--1793MiB/s. Direct MMIO mapping + passes the same test. Native performance equivalence is not yet measured. + +Full logs and pinned binaries are retained in the workspace directory +work/j714-mpidr-20260927. This is an initial resident profile; ordinary ANS +profiles and their wrapped-key crypto support are retained. No SSD writes or +filesystem changes were used for qualification. diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 57dba97612207..5736a6c399461 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -175,6 +177,7 @@ struct apple_nvme_iod { }; struct apple_nvme_hw { + bool sptm_resident; bool has_lsq_nvmmu; bool has_linear_sq_ctrl; bool has_prp_null_ctrl; @@ -184,6 +187,18 @@ struct apple_nvme_hw { u32 max_queue_depth; }; +struct apple_nvme_sptm_op { + u32 op, flags; + u64 arg[6]; + s64 ret; +}; +struct apple_nvme_sptm_batch { + u32 count, flags; + s32 status; + u32 done; + struct apple_nvme_sptm_op ops[6]; +}; + struct apple_nvme { struct device *dev; @@ -200,6 +215,8 @@ struct apple_nvme { struct apple_rtkit *rtk; struct reset_control *reset; bool owns_rtkit; + void __iomem *sptm_regs; + struct apple_nvme_sptm_batch *sptm_batch; struct dma_pool *prp_page_pool; struct dma_pool *prp_small_pool; @@ -227,6 +244,69 @@ struct apple_nvme { struct blk_crypto_profile crypto_profile; }; +/* Serialized by anv->lock during I/O; reset configures queues before use. + * All operations in a batch share one guest exit/entry. Payloads stay in + * their original DMA buffers: this transport copies only small descriptors. + */ +static int apple_nvme_sptm_exec(struct apple_nvme *anv, unsigned int count) +{ + struct apple_nvme_sptm_batch *b = anv->sptm_batch; + unsigned long flags; + u64 pa = virt_to_phys(b); + void __iomem *db = anv->sptm_regs + 0x1010; + + b->count = count; + b->done = 0; + b->status = 0; + wmb(); + local_irq_save(flags); + asm volatile("mov x9, sp\nstr %x[pa], [%[db]]\nmov sp, x9\n" + : : [pa] "r" (pa), [db] "r" (db) : "x9", "memory"); + local_irq_restore(flags); + rmb(); + return b->status || b->done != count ? -EIO : 0; +} + +static int apple_nvme_sptm_admin(struct apple_nvme *anv) +{ + struct apple_nvme_sptm_op *op = anv->sptm_batch->ops; + memset(anv->sptm_batch, 0, sizeof(*anv->sptm_batch)); + op[0].op = 3; op[0].arg[0] = 253; op[0].arg[1] = 2; + op[1].op = 0; + op[2].op = 4; + op[2].arg[0] = dma_to_phys(anv->dev, anv->adminq.sq_dma_addr); + op[2].arg[1] = APPLE_NVME_AQ_DEPTH - 1; + op[2].arg[2] = dma_to_phys(anv->dev, anv->adminq.cq_dma_addr); + op[2].arg[3] = APPLE_NVME_AQ_DEPTH - 1; + op[3].op = 5; + op[3].arg[0] = op[3].arg[1] = anv->hw->max_queue_depth - 1; + op[4].op = 7; + op[4].arg[0] = dma_to_phys(anv->dev, anv->ioq.cq_dma_addr); + op[5].op = 6; + op[5].arg[0] = dma_to_phys(anv->dev, anv->ioq.sq_dma_addr); + return apple_nvme_sptm_exec(anv, 6); +} + +static int apple_nvme_sptm_request(struct apple_nvme_queue *q, unsigned int tag, + unsigned int bytes, bool map) +{ + struct apple_nvme *anv = q->is_adminq ? + container_of(q, struct apple_nvme, adminq) : + container_of(q, struct apple_nvme, ioq); + struct apple_nvme_sptm_op *op = anv->sptm_batch->ops; + + memset(anv->sptm_batch, 0, sizeof(*anv->sptm_batch)); + op->op = map ? 1 : 2; + op->arg[0] = q->is_adminq ? 0 : 1; + op->arg[1] = tag; + if (map) { + op->arg[2] = dma_to_phys(anv->dev, q->tcb_dma_addr) + + tag * sizeof(*q->tcbs); + op->arg[3] = bytes; + } + return apple_nvme_sptm_exec(anv, 1); +} + static inline void apple_nvme_writeq(struct apple_nvme *anv, u64 value, void __iomem *addr) { @@ -372,7 +452,8 @@ static int apple_nvme_sart_dma_setup(void *cookie, if (!bfr->buffer) return -ENOMEM; - ret = apple_sart_add_allowed_region(anv->sart, bfr->iova, bfr->size); + ret = apple_sart_add_allowed_region(anv->sart, bfr->iova, + anv->hw->sptm_resident ? ALIGN(bfr->size, SZ_16K) : bfr->size); if (ret) { dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova); bfr->buffer = NULL; @@ -392,7 +473,12 @@ static void apple_nvme_sart_dma_destroy(void *cookie, return; } - apple_sart_remove_allowed_region(anv->sart, bfr->iova, bfr->size); + if (apple_sart_remove_allowed_region(anv->sart, bfr->iova, + anv->hw->sptm_resident ? ALIGN(bfr->size, SZ_16K) : bfr->size) && + anv->hw->sptm_resident) { + dev_err(anv->dev, "retaining RTKit buffer after failed SART unmap\n"); + return; + } dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova); } @@ -406,6 +492,11 @@ static void apple_nvmmu_inval(struct apple_nvme_queue *q, unsigned int tag) { struct apple_nvme *anv = queue_to_apple_nvme(q); + if (anv->hw->sptm_resident) { + if (apple_nvme_sptm_request(q, tag, 0, false)) + panic("ANS SPTM failed to release DMA references for tag %u", tag); + return; + } writel(tag, anv->mmio_nvmmu + APPLE_NVMMU_TCB_INVAL); if (readl(anv->mmio_nvmmu + APPLE_NVMMU_TCB_STAT)) dev_warn_ratelimited(anv->dev, @@ -474,6 +565,12 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, memcpy(tcb->_aes_unk + 16, key + 16, 48); } memcpy(&q->sqes[tag], cmd, sizeof(*cmd)); + if (anv->hw->sptm_resident && q->is_adminq) + dev_dbg(anv->dev, "SPTM admin submit op=%#x tag=%u prp=%#llx/%#llx cdw10=%#x cdw11=%#x bytes=%u\n", + cmd->common.opcode, tag, le64_to_cpu(cmd->common.dptr.prp1), + le64_to_cpu(cmd->common.dptr.prp2), le32_to_cpu(cmd->common.cdw10), + le32_to_cpu(cmd->common.cdw11), blk_rq_payload_bytes(req)); + if (!q->is_adminq) { dma_addr_t sqe_dma = q->sq_dma_addr + tag * sizeof(*cmd); @@ -501,6 +598,13 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, * and the final CQ update. */ spin_lock_irq(&anv->lock); + if (anv->hw->sptm_resident && + apple_nvme_sptm_request(q, tag, blk_rq_payload_bytes(req), true)) { + spin_unlock_irq(&anv->lock); + nvme_req(req)->status = NVME_SC_INTERNAL; + blk_mq_complete_request(req); + return; + } writel(tag, q->sq_db); spin_unlock_irq(&anv->lock); } @@ -792,6 +896,9 @@ static inline void apple_nvme_handle_cqe(struct apple_nvme_queue *q, if (anv->hw->has_lsq_nvmmu) apple_nvmmu_inval(q, command_id); + if (anv->hw->sptm_resident && q->is_adminq) + dev_dbg(anv->dev, "SPTM admin completion cid=%u status=%#x result=%#x\n", + command_id, le16_to_cpu(cqe->status), le32_to_cpu(cqe->result.u32)); req = nvme_find_rq(apple_nvme_queue_tagset(anv, q), command_id); if (unlikely(!req)) { dev_warn(anv->dev, "invalid id %d completed", command_id); @@ -1333,6 +1440,11 @@ static void apple_nvme_reset_work(struct work_struct *work) */ if (!(readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & APPLE_ANS_COPROC_CPU_CONTROL_RUN)) { + if (anv->hw->sptm_resident) { + dev_err(anv->dev, "resident ANS firmware is not running\n"); + ret = -EIO; + goto out; + } ret = reset_control_assert(anv->reset); if (ret) @@ -1384,6 +1496,15 @@ static void apple_nvme_reset_work(struct work_struct *work) anv->ctrl.max_segments = NVME_MAX_SEGS; dma_set_max_seg_size(anv->dev, 0xffffffff); + if (anv->hw->sptm_resident) { + anv->ctrl.max_hw_sectors = min_t(u32, anv->ctrl.max_hw_sectors, 2048); + ret = apple_nvme_sptm_admin(anv); + if (ret) + goto out; + dev_info(anv->dev, "SPTM admin queue configured through one batch\n"); + goto queues_configured; + } + if (anv->hw->has_lsq_nvmmu) { /* @@ -1455,6 +1576,7 @@ static void apple_nvme_reset_work(struct work_struct *work) anv->mmio_nvmmu + APPLE_NVMMU_IOSQ_TCB_BASE); } +queues_configured: anv->ctrl.sqsize = anv->hw->max_queue_depth - 1; /* 0's based queue depth */ anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP); @@ -1493,7 +1615,7 @@ static void apple_nvme_reset_work(struct work_struct *work) dev_dbg(anv->dev, "I/O SQ created\n"); apple_nvme_init_queue(&anv->ioq); - if (anv->hw->needs_ioq_registers) { + if (anv->hw->needs_ioq_registers && !anv->hw->sptm_resident) { apple_nvme_writeq(anv, anv->ioq.cq_dma_addr, anv->mmio_nvme + APPLE_ANS_IOCQ_REGISTER); apple_nvme_writeq(anv, anv->ioq.sq_dma_addr, @@ -1657,15 +1779,60 @@ static int apple_nvme_alloc_tagsets(struct apple_nvme *anv) return 0; } +struct apple_nvme_sart_buffer { + struct device *dev; + struct apple_sart *sart; + void *buffer; + dma_addr_t dma; + size_t size; +}; + +static void apple_nvme_sart_buffer_release(void *data) +{ + struct apple_nvme_sart_buffer *b = data; + + if (apple_sart_remove_allowed_region(b->sart, b->dma, b->size)) { + dev_err(b->dev, "retaining queue buffer after failed SART unmap\n"); + return; + } + dma_free_coherent(b->dev, b->size, b->buffer, b->dma); +} + +static void *apple_nvme_queue_buffer_alloc(struct apple_nvme *anv, + size_t size, dma_addr_t *dma) +{ + struct apple_nvme_sart_buffer *b; + + if (!anv->hw->sptm_resident) + return dmam_alloc_coherent(anv->dev, size, dma, GFP_KERNEL); + b = devm_kzalloc(anv->dev, sizeof(*b), GFP_KERNEL); + if (!b) + return NULL; + b->dev = anv->dev; + b->sart = anv->sart; + b->size = ALIGN(size, SZ_16K); + b->buffer = dma_alloc_coherent(b->dev, b->size, &b->dma, GFP_KERNEL); + if (!b->buffer) + return NULL; + if (apple_sart_add_allowed_region(b->sart, b->dma, b->size)) { + dma_free_coherent(b->dev, b->size, b->buffer, b->dma); + return NULL; + } + if (devm_add_action_or_reset(b->dev, apple_nvme_sart_buffer_release, b)) + return NULL; + *dma = b->dma; + return b->buffer; +} + static int apple_nvme_queue_alloc(struct apple_nvme *anv, struct apple_nvme_queue *q) { unsigned int depth = apple_nvme_queue_depth(q); size_t iosq_size; - q->cqes = dmam_alloc_coherent(anv->dev, + q->cqes = apple_nvme_queue_buffer_alloc(anv, depth * sizeof(struct nvme_completion), - &q->cq_dma_addr, GFP_KERNEL); + &q->cq_dma_addr); if (!q->cqes) return -ENOMEM; @@ -1674,8 +1841,7 @@ static int apple_nvme_queue_alloc(struct apple_nvme *anv, else iosq_size = depth << APPLE_NVME_IOSQES; - q->sqes = dmam_alloc_coherent(anv->dev, iosq_size, - &q->sq_dma_addr, GFP_KERNEL); + q->sqes = apple_nvme_queue_buffer_alloc(anv, iosq_size, &q->sq_dma_addr); if (!q->sqes) return -ENOMEM; @@ -1684,10 +1850,10 @@ static int apple_nvme_queue_alloc(struct apple_nvme *anv, * We need the maximum queue depth here because the NVMMU only * has a single depth configuration shared between both queues. */ - q->tcbs = dmam_alloc_coherent(anv->dev, + q->tcbs = apple_nvme_queue_buffer_alloc(anv, anv->hw->max_queue_depth * sizeof(struct apple_nvmmu_tcb), - &q->tcb_dma_addr, GFP_KERNEL); + &q->tcb_dma_addr); if (!q->tcbs) return -ENOMEM; } @@ -1823,6 +1989,24 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) goto put_dev; } + if (anv->hw->sptm_resident) { + anv->sptm_regs = devm_platform_ioremap_resource_byname(pdev, "sptm"); + if (IS_ERR(anv->sptm_regs)) { + ret = PTR_ERR(anv->sptm_regs); + goto put_dev; + } + if (readq(anv->sptm_regs) != 0x78316e31 || + !(readq(anv->sptm_regs + 0x18) & BIT(3))) { + ret = -ENODEV; + goto put_dev; + } + anv->sptm_batch = devm_kzalloc(dev, sizeof(*anv->sptm_batch), GFP_KERNEL); + if (!anv->sptm_batch) { + ret = -ENOMEM; + goto put_dev; + } + } + anv->mmio_coproc = devm_platform_ioremap_resource_byname(pdev, "ans"); if (IS_ERR(anv->mmio_coproc)) { ret = PTR_ERR(anv->mmio_coproc); @@ -1863,7 +2047,9 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) goto put_dev; } - anv->reset = devm_reset_control_array_get_exclusive(anv->dev); + anv->reset = anv->hw->sptm_resident ? + devm_reset_control_array_get_optional_exclusive(anv->dev) : + devm_reset_control_array_get_exclusive(anv->dev); if (IS_ERR(anv->reset)) { ret = dev_err_probe(dev, PTR_ERR(anv->reset), "Failed to get reset control"); @@ -2115,7 +2301,17 @@ static const struct apple_nvme_hw apple_nvme_t8132_hw = { .max_queue_depth = 64, }; +static const struct apple_nvme_hw apple_nvme_t6050_sptm_hw = { + .sptm_resident = true, + .has_lsq_nvmmu = true, + .has_linear_sq_ctrl = false, + .has_separate_nvmmu = true, + .needs_ioq_registers = true, + .max_queue_depth = 64, +}; + static const struct of_device_id apple_nvme_of_match[] = { + { .compatible = "apple,t6050-nvme-sptm", .data = &apple_nvme_t6050_sptm_hw }, { .compatible = "apple,t8015-nvme-ans2", .data = &apple_nvme_t8015_hw }, { .compatible = "apple,t8103-nvme-ans2", .data = &apple_nvme_t8103_hw }, { .compatible = "apple,t8132-nvme-ans2", .data = &apple_nvme_t8132_hw }, diff --git a/drivers/soc/apple/sart.c b/drivers/soc/apple/sart.c index 35151fa33fad4..7bfcb12730917 100644 --- a/drivers/soc/apple/sart.c +++ b/drivers/soc/apple/sart.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -155,11 +156,12 @@ static int sart_x1n1_region(struct apple_sart *sart, phys_addr_t paddr, ret = map ? -EBUSY : -ENOENT; goto out; } - /* Inputs are guest CPU physical addresses. The HV validates/translates - * them to host backing pages; never pass an arbitrary host PA here. + /* Consumers supply DMA bus addresses. Reverse the DT dma-ranges offset + * before the HV validates/translates the guest IPA to its backing page. * perm=1 permits device writes; guard=1 holds the firmware power canary. */ - ret = sart_x1n1_call(sart, map ? 2 : 3, paddr, size, map, map); + ret = sart_x1n1_call(sart, map ? 2 : 3, + dma_to_phys(sart->dev, paddr), size, map, map); if (!ret) { sart->regions[i].paddr = map ? paddr : 0; sart->regions[i].size = map ? size : 0; @@ -178,11 +180,11 @@ static void sart_x1n1_selftest(struct apple_sart *sart) if (!page) return; pa = virt_to_phys(page); - ret = sart_x1n1_region(sart, pa, PAGE_SIZE, true); + ret = sart_x1n1_region(sart, phys_to_dma(sart->dev, pa), PAGE_SIZE, true); dev_info(sart->dev, "x1n1 selftest: map %pa size %#lx status %d\n", &pa, PAGE_SIZE, ret); if (!ret) { - ret = sart_x1n1_region(sart, pa, PAGE_SIZE, false); + ret = sart_x1n1_region(sart, phys_to_dma(sart->dev, pa), PAGE_SIZE, false); dev_info(sart->dev, "x1n1 selftest: unmap status %d\n", ret); if (ret) return; /* Do not free memory while a DMA grant might remain. */ From 41a71320461622b13f71f73869f6abc32eafabe3 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Wed, 9 Sep 2026 15:28:21 -0400 Subject: [PATCH 0147/1579] wifi: apple: add hardware-qualified N1 driver and J813 workbench Path-scoped import of the N1 driver changes. J813 workbench scripts, board DT and private calibration staging are deliberately excluded. Original-commit: 9f4a5246489ae953de67eab1717f3a6e992f80f8 Source-repository: https://github.com/aurora-silicon/linux-aurora --- drivers/net/wireless/Kconfig | 1 + drivers/net/wireless/Makefile | 1 + drivers/net/wireless/apple/Kconfig | 25 + drivers/net/wireless/apple/Makefile | 2 + drivers/net/wireless/apple/centauri/Makefile | 3 + .../net/wireless/apple/centauri/centauri.h | 46 + .../wireless/apple/centauri/centauri_ipc.c | 478 ++++ .../wireless/apple/centauri/centauri_pci.c | 278 ++ .../wireless/apple/centauri/centauri_port.c | 192 ++ .../wireless/apple/centauri/centauri_wifi.c | 2312 +++++++++++++++++ 10 files changed, 3338 insertions(+) create mode 100644 drivers/net/wireless/apple/Kconfig create mode 100644 drivers/net/wireless/apple/Makefile create mode 100644 drivers/net/wireless/apple/centauri/Makefile create mode 100644 drivers/net/wireless/apple/centauri/centauri.h create mode 100644 drivers/net/wireless/apple/centauri/centauri_ipc.c create mode 100644 drivers/net/wireless/apple/centauri/centauri_pci.c create mode 100644 drivers/net/wireless/apple/centauri/centauri_port.c create mode 100644 drivers/net/wireless/apple/centauri/centauri_wifi.c diff --git a/drivers/net/wireless/Kconfig b/drivers/net/wireless/Kconfig index c6599594dc995..5b84a2ffd0d3a 100644 --- a/drivers/net/wireless/Kconfig +++ b/drivers/net/wireless/Kconfig @@ -19,6 +19,7 @@ menuconfig WLAN if WLAN source "drivers/net/wireless/admtek/Kconfig" +source "drivers/net/wireless/apple/Kconfig" source "drivers/net/wireless/ath/Kconfig" source "drivers/net/wireless/atmel/Kconfig" source "drivers/net/wireless/broadcom/Kconfig" diff --git a/drivers/net/wireless/Makefile b/drivers/net/wireless/Makefile index e1c4141c60044..48d2998ca1203 100644 --- a/drivers/net/wireless/Makefile +++ b/drivers/net/wireless/Makefile @@ -4,6 +4,7 @@ # obj-$(CONFIG_WLAN_VENDOR_ADMTEK) += admtek/ +obj-$(CONFIG_WLAN_VENDOR_APPLE) += apple/ obj-$(CONFIG_WLAN_VENDOR_ATH) += ath/ obj-$(CONFIG_WLAN_VENDOR_ATMEL) += atmel/ obj-$(CONFIG_WLAN_VENDOR_BROADCOM) += broadcom/ diff --git a/drivers/net/wireless/apple/Kconfig b/drivers/net/wireless/apple/Kconfig new file mode 100644 index 0000000000000..d3ead4d9259e7 --- /dev/null +++ b/drivers/net/wireless/apple/Kconfig @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: GPL-2.0-only +config WLAN_VENDOR_APPLE + bool "Apple wireless devices" + default y + help + Support for Apple's wireless network devices. + +if WLAN_VENDOR_APPLE +config APPLE_CENTAURI + tristate "Apple N1 Centauri experimental J813 Wi-Fi" + depends on ARCH_APPLE || COMPILE_TEST + depends on PCI && CFG80211 && DEBUG_FS + select FW_LOADER + help + Experimental Apple N1 PCIe and ACIPC Wi-Fi support for the J813 + MacBook Air. Firmware boot, ring setup and connection setup currently + use debugfs and the tools/aurora/j813 workbench. The tested single-link + WPA2 setup completes DHCP and bidirectional traffic. Suspend/resume, + roaming and automatic startup are not yet qualified. + + The firmware and per-device calibration are private external inputs. + This driver does not include Bluetooth HCI support. + + To build as a module, choose M; the module is apple_centauri. +endif diff --git a/drivers/net/wireless/apple/Makefile b/drivers/net/wireless/apple/Makefile new file mode 100644 index 0000000000000..ca72cd0b4da06 --- /dev/null +++ b/drivers/net/wireless/apple/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_APPLE_CENTAURI) += centauri/ diff --git a/drivers/net/wireless/apple/centauri/Makefile b/drivers/net/wireless/apple/centauri/Makefile new file mode 100644 index 0000000000000..1a17e5283e227 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_APPLE_CENTAURI) += apple_centauri.o +apple_centauri-y := centauri_pci.o centauri_port.o centauri_ipc.o centauri_wifi.o diff --git a/drivers/net/wireless/apple/centauri/centauri.h b/drivers/net/wireless/apple/centauri/centauri.h new file mode 100644 index 0000000000000..f7fdf4105a3b3 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef AURORA_CENTAURI_H +#define AURORA_CENTAURI_H +#include +#include +#include +#include + +#define CEN_ALPHA_SCRATCH_OFFSET SZ_8M +#define CEN_ALPHA_SCRATCH_SIZE (5 * SZ_1M) + +struct centauri_ipc; +struct centauri_wifi; +struct cen_alpha_resources { + void *context, *messages, *window; + dma_addr_t context_dma, messages_dma, window_dma; + void __iomem *bar; + u32 index[4], mtr, mcr; +}; +struct centauri { + struct pci_dev *pdev; + void __iomem *bar; + resource_size_t size; + u32 stage_offset; + struct dentry *debug; + struct mutex boot_lock; + void *firmware, *descriptor, *msi_page; + dma_addr_t firmware_dma, descriptor_dma, msi_dma; + size_t firmware_size; + u32 image_response; + u16 msi_original; + int msi_cap; + bool boot_attempted; + bool link_ready; + struct centauri_ipc *ipc; + struct centauri_wifi *wifi; +}; + +int centauri_j813_port_cycle(struct pci_dev *pdev); +int centauri_start_ipc(struct centauri *c); +void centauri_free_ipc(struct centauri *c); +int centauri_alpha_resources(struct centauri *c, struct cen_alpha_resources *resources); +int centauri_alpha_log_buffer(struct centauri *c, void **cpu, dma_addr_t *dma, size_t *size); +int centauri_wifi_register(struct centauri *c); +void centauri_wifi_unregister(struct centauri *c); +#endif diff --git a/drivers/net/wireless/apple/centauri/centauri_ipc.c b/drivers/net/wireless/apple/centauri/centauri_ipc.c new file mode 100644 index 0000000000000..6eec46d831888 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_ipc.c @@ -0,0 +1,478 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Native DMA implementation of the measured J813 Control/Alpha bootstrap. + * Wire layout source: AppleCentauriCommon/scripts/alpha_wifi_scan.py v259. + * Polling and bounded bootstrap rings are deliberate bring-up restrictions. + */ +#include +#include +#include +#include +#include +#include +#include +#include "centauri.h" + +struct cen_buffer { + struct list_head list; + void *cpu; + dma_addr_t dma; + size_t size; +}; + +struct cen_context { + struct cen_buffer *buffer, *messages; + u32 index[4], mtr, mcr; + u16 tr_count, cr_count, message_head; +}; + +struct centauri_ipc { + struct list_head buffers; + struct pci_dev *function[3]; + void __iomem *bar[3]; + struct cen_context control, alpha; + struct cen_buffer *secondary, *window, *cchi_rx, *cchi_tx, *debug_rx; + struct cen_buffer *alpha_log; + u16 cchi_head, cchi_tail; +}; + +static struct cen_buffer *buffer_alloc(struct centauri *c, size_t size) +{ + struct cen_buffer *buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + + if (!buffer) + return NULL; + buffer->size = PAGE_ALIGN(size); + buffer->cpu = dma_alloc_coherent(&c->pdev->dev, buffer->size, &buffer->dma, GFP_KERNEL); + if (!buffer->cpu) { + kfree(buffer); + return NULL; + } + memset(buffer->cpu, 0, buffer->size); + list_add_tail(&buffer->list, &c->ipc->buffers); + return buffer; +} + +static int ftab_entry(const void *image, size_t size, const char tag[4], + u32 *offset, u32 *length, bool sparse) +{ + u32 count, i, off, len; + const u8 *table = image; + + if (size < 0x30 || memcmp(table + 0x24, "ftab", 4)) + return -EINVAL; + count = get_unaligned_le32(table + 0x28); + if (!count || count > 256 || 0x30 + count * 16 > size) + return -EINVAL; + for (i = 0; i < count; i++) { + const u8 *entry = table + 0x30 + 16 * i; + + if (memcmp(entry, tag, 4)) + continue; + off = get_unaligned_le32(entry + 4); + len = get_unaligned_le32(entry + 8); + if (off < 0x30 + count * 16 || !len || (u64)off + len > (sparse ? SZ_64M : size)) + return -EINVAL; + *offset = off; + *length = len; + return 0; + } + return -ENOENT; +} + +static int secondary_init(struct centauri *c) +{ + u32 offset, length, count, i, required; + const u8 *seed; + int ret; + + ret = ftab_entry(c->firmware, c->firmware_size, "2ftb", &offset, &length, false); + if (ret) + return ret; + seed = c->firmware + offset; + if (length < 0x30 || memcmp(seed + 0x24, "ftab", 4)) + return -EINVAL; + count = get_unaligned_le32(seed + 0x28); + if (!count || count > 256 || 0x30 + 16 * count != length) + return -EINVAL; + required = length; + for (i = 0; i < count; i++) { + u64 end = (u64)get_unaligned_le32(seed + 0x34 + i * 16) + + get_unaligned_le32(seed + 0x38 + i * 16); + + if (end > SZ_64M) + return -EINVAL; + required = max_t(u32, required, end); + } + c->ipc->secondary = buffer_alloc(c, required); + if (!c->ipc->secondary) + return -ENOMEM; + memcpy(c->ipc->secondary->cpu, seed, length); + dma_wmb(); + pci_write_config_dword(c->pdev, 0xf88, lower_32_bits(c->ipc->secondary->dma)); + pci_write_config_dword(c->pdev, 0xf8c, upper_32_bits(c->ipc->secondary->dma)); + pci_write_config_dword(c->pdev, 0xf90, required); + dev_info(&c->pdev->dev, "secondary memory: seed=%u required=%u\n", length, required); + return 0; +} + +static int context_init(struct centauri *c, struct cen_context *ctx, + u16 tr, u16 cr, dma_addr_t scratch, size_t scratch_size) +{ + u8 *header; + dma_addr_t dma; + int i; + + ctx->buffer = buffer_alloc(c, SZ_32K); + ctx->messages = buffer_alloc(c, SZ_16K); + if (!ctx->buffer || !ctx->messages) + return -ENOMEM; + ctx->tr_count = tr; + ctx->cr_count = cr; + ctx->index[0] = 0x78; + ctx->index[1] = ctx->index[0] + cr * 2; + ctx->index[2] = ctx->index[1] + tr * 2; + ctx->index[3] = ctx->index[2] + cr * 2; + ctx->mtr = ALIGN(ctx->index[3] + tr * 2, tr == 192 ? 16 : 2); + ctx->mcr = ALIGN(ctx->mtr + 256, 4); + header = ctx->buffer->cpu; + dma = ctx->buffer->dma; + put_unaligned_le16(768, header); + put_unaligned_le16(0x68, header + 2); + put_unaligned_le64(dma + 0x68, header + 8); + for (i = 0; i < 4; i++) + put_unaligned_le64(dma + ctx->index[i], header + 0x10 + 8 * i); + put_unaligned_le16(cr, header + 0x30); + put_unaligned_le16(tr, header + 0x32); + put_unaligned_le64(dma + ctx->mcr, header + 0x34); + put_unaligned_le64(dma + ctx->mtr, header + 0x3c); + put_unaligned_le16(16, header + 0x44); + put_unaligned_le16(16, header + 0x46); + put_unaligned_le64(scratch ?: dma + ALIGN(ctx->mcr + 256, 8), header + 0x58); + put_unaligned_le64(scratch_size, header + 0x60); + dma_wmb(); + return 0; +} + +static int handshake(void __iomem *bar, struct cen_context *ctx, + u32 status, u32 address, u32 control) +{ + u32 value; + int ret; + + writel(1, bar + control); + ret = readl_poll_timeout(bar + status, value, value == 1, 100, 2000000); + if (ret) + return ret; + writel(lower_32_bits(ctx->buffer->dma), bar + address); + writel(upper_32_bits(ctx->buffer->dma), bar + address + 4); + dma_wmb(); + writel(2, bar + control); + return readl_poll_timeout(bar + status, value, value == 2, 100, 2000000); +} + +static int message(struct cen_context *ctx, void __iomem *doorbell, + const void *payload, size_t length) +{ + u16 head = ctx->message_head; + u8 *descriptor; + u16 tail; + int ret; + + if (head >= 15 || !length || length > 512) + return -ENOSPC; + memcpy(ctx->messages->cpu + 512 * (head + 1), payload, length); + descriptor = ctx->buffer->cpu + ctx->mtr + head * 16; + put_unaligned_le32(1 | (length << 8), descriptor); + put_unaligned_le64(ctx->messages->dma + 512 * (head + 1), descriptor + 4); + put_unaligned_le32(head + 1, descriptor + 12); + dma_wmb(); + writel(head + 1, doorbell); + ret = read_poll_timeout(READ_ONCE, tail, le16_to_cpu(tail) == head + 1, + 100, 1200000, false, *(__le16 *)(ctx->buffer->cpu + ctx->index[1])); + dma_rmb(); + ctx->message_head++; + return ret; +} + +static int open_transfer(struct centauri *c, u16 id, struct cen_buffer *buffer, + u16 footer) +{ + u8 msg[0x34] = { 1 }; + u32 words = footer >> 2, shift = min_t(u32, __ffs(words), 4); + + msg[2] = words >> shift; + msg[3] = shift << 4; + put_unaligned_le16(id, msg + 4); + put_unaligned_le64(buffer->dma, msg + 8); + put_unaligned_le64(~0ULL, msg + 0x10); + put_unaligned_le16(16, msg + 0x18); + put_unaligned_le16(0xffff, msg + 0x1a); + put_unaligned_le16(id, msg + 0x1c); + put_unaligned_le16(0x40, msg + 0x1e); + return message(&c->ipc->control, c->bar + 0x9000, msg, sizeof(msg)); +} + +static int cchi_init(struct centauri *c) +{ + struct centauri_ipc *ipc = c->ipc; + u8 config[4] = {9, 0, 0xff, 0xff}, *entry; + u16 tail; + int i, ret; + + ipc->cchi_rx = buffer_alloc(c, SZ_32K); + ipc->cchi_tx = buffer_alloc(c, SZ_32K); + ipc->debug_rx = buffer_alloc(c, SZ_16K); + if (!ipc->cchi_rx || !ipc->cchi_tx || !ipc->debug_rx) + return -ENOMEM; + ret = message(&ipc->control, c->bar + 0x9000, config, sizeof(config)); + if (ret) + return ret; + ret = open_transfer(c, 4, ipc->cchi_rx, 1024); + if (!ret) + ret = open_transfer(c, 3, ipc->cchi_tx, 1024); + if (!ret) + ret = open_transfer(c, 5, ipc->debug_rx, 256); + if (ret) + return ret; + for (i = 0; i < 15; i++) { + entry = ipc->cchi_rx->cpu + i * 1040; + put_unaligned_le32(1 | (1024 << 8), entry); + put_unaligned_le32(i + 1, entry + 12); + entry = ipc->debug_rx->cpu + i * 272; + put_unaligned_le32(1 | (256 << 8), entry); + put_unaligned_le32(i + 1, entry + 12); + } + dma_wmb(); + writel(15, c->bar + 0x9010); + writel(15, c->bar + 0x9014); + entry = ipc->cchi_tx->cpu; + put_unaligned_le32(1 | (64 << 8), entry); + put_unaligned_le32(1, entry + 12); + put_unaligned_le16(64, entry + 16); /* CCHI gid 0, oid 0, seq 0 hello. */ + dma_wmb(); + writel(1, c->bar + 0x900c); + ret = read_poll_timeout(READ_ONCE, tail, le16_to_cpu(tail) != 0, + 100, 2000000, false, + *(__le16 *)(ipc->control.buffer->cpu + ipc->control.index[1] + 4 * 2)); + dma_rmb(); + if (ret) + return ret; + entry = ipc->cchi_rx->cpu; + if ((get_unaligned_le32(entry) >> 8) < 8 || entry[18] || entry[19]) + return -EPROTO; + ipc->cchi_head = ipc->cchi_tail = 1; + dev_info(&c->pdev->dev, "Control CCHI hello response received\n"); + return 0; +} + +static int memswap_handoff(struct centauri *c, unsigned int function, const char tag[4]) +{ + struct cen_buffer *secondary = c->ipc->secondary, *working; + u32 offset, size; + int ret; + + ret = ftab_entry(secondary->cpu, secondary->size, tag, &offset, &size, false); + if (ret) + return ret; + working = buffer_alloc(c, size); + if (!working) + return -ENOMEM; + dma_rmb(); + memcpy(working->cpu, secondary->cpu + offset, size); + dma_wmb(); + pci_write_config_dword(c->ipc->function[function], 0xf88, lower_32_bits(working->dma)); + pci_write_config_dword(c->ipc->function[function], 0xf8c, upper_32_bits(working->dma)); + pci_write_config_dword(c->ipc->function[function], 0xf90, size); + dev_info(&c->pdev->dev, "memswap function %u: %.4s %u bytes\n", function, tag, size); + return 0; +} + +static int enumerate_functions(struct centauri *c) +{ + struct centauri_ipc *ipc = c->ipc; + u16 control; + int i, cap; + + pci_lock_rescan_remove(); + /* The ROM personality was single-function; boot exposes Alpha and Beta. */ + c->pdev->multifunction = 1; + pci_rescan_bus(c->pdev->bus); + pci_unlock_rescan_remove(); + for (i = 0; i < 3; i++) { + ipc->function[i] = pci_get_domain_bus_and_slot(pci_domain_nr(c->pdev->bus), + c->pdev->bus->number, PCI_DEVFN(0, i)); + if (!ipc->function[i] || ipc->function[i]->vendor != PCI_VENDOR_ID_APPLE || + (i && ipc->function[i]->device != 0x1901 + i)) + return -ENODEV; + if (!(pci_resource_flags(ipc->function[i], 0) & IORESOURCE_MEM)) + return -ENODEV; + ipc->bar[i] = pci_iomap(ipc->function[i], 0, 0); + if (!ipc->bar[i]) + return -ENOMEM; + cap = pci_find_capability(ipc->function[i], PCI_CAP_ID_MSI); + if (!cap) + return -ENODEV; + pci_read_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, &control); + if (!(control & PCI_MSI_FLAGS_64BIT)) + return -EOPNOTSUPP; + pci_write_config_dword(ipc->function[i], cap + PCI_MSI_ADDRESS_LO, + lower_32_bits(c->msi_dma + 0x400 * i)); + pci_write_config_dword(ipc->function[i], cap + PCI_MSI_ADDRESS_HI, + upper_32_bits(c->msi_dma + 0x400 * i)); + pci_write_config_word(ipc->function[i], cap + PCI_MSI_DATA_64, 0xa5a0 + 0x100 * i); + pci_write_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, + (control & ~PCI_MSI_FLAGS_QSIZE) | (5 << 4) | PCI_MSI_FLAGS_ENABLE); + pci_set_master(ipc->function[i]); + } + return 0; +} + +int centauri_start_ipc(struct centauri *c) +{ + struct centauri_ipc *ipc; + u32 stage; + int ret; + + if (!c->firmware || !c->link_ready || c->ipc || readl(c->bar + 0x8000) != 1) + return -EBUSY; + ipc = kzalloc(sizeof(*ipc), GFP_KERNEL); + if (!ipc) + return -ENOMEM; + INIT_LIST_HEAD(&ipc->buffers); + c->ipc = ipc; + ret = enumerate_functions(c); + if (ret) + goto failed; + ret = secondary_init(c); + if (ret) + goto failed; + ret = context_init(c, &ipc->control, 13, 1, 0, 0); + if (ret) + goto failed; + writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); + writel(upper_32_bits(c->descriptor_dma), c->bar + 0x8048); + writel(0x20, c->bar + 0x804c); + writel(0x62746632, c->bar + 0x8074); /* 2ftb */ + dma_wmb(); + writel(1, c->bar + 0x9070); + ret = readl_poll_timeout(c->bar + 0x8000, stage, stage == 2 || stage == 3, 100, 3000000); + if (ret || stage != 2) { + ret = ret ?: -EIO; + goto failed; + } + dev_info(&c->pdev->dev, "Control firmware reached OS stage\n"); + ret = handshake(c->bar, &ipc->control, 0x8050, 0x8054, 0x907c); + if (ret) + goto failed; + dev_info(&c->pdev->dev, "Control IPC handshake complete\n"); + ret = memswap_handoff(c, 0, "mswc"); + if (!ret) + ret = memswap_handoff(c, 2, "mswb"); + if (!ret) + ret = memswap_handoff(c, 1, "msww"); + if (ret) + goto failed; + ret = cchi_init(c); + if (ret) + goto failed; + ret = readl_poll_timeout(ipc->bar[1] + 8, stage, stage == 2 || stage == 3, 100, 2000000); + if (ret || stage != 2) { + ret = ret ?: -EIO; + goto failed; + } + ipc->window = buffer_alloc(c, SZ_16M); + if (!ipc->window) { + ret = -ENOMEM; + goto failed; + } + ret = context_init(c, &ipc->alpha, 192, 234, + ipc->window->dma + CEN_ALPHA_SCRATCH_OFFSET, CEN_ALPHA_SCRATCH_SIZE); + if (ret) + goto failed; + writel(lower_32_bits(ipc->window->dma), ipc->bar[1] + 0x18); + writel(upper_32_bits(ipc->window->dma), ipc->bar[1] + 0x1c); + writel(SZ_16M, ipc->bar[1] + 0x20); + ret = handshake(ipc->bar[1], &ipc->alpha, 0xc, 0x10, 0x1028); + if (ret) + goto failed; + dev_info(&c->pdev->dev, "Alpha Wi-Fi firmware running; IPC handshake complete\n"); + return centauri_wifi_register(c); +failed: + dev_err(&c->pdev->dev, "IPC bootstrap failed: %d (Control stage=%#x)\n", + ret, readl(c->bar + 0x8000)); + /* Published DMA stays pinned until every function is stopped at removal. */ + return ret; +} + +void centauri_free_ipc(struct centauri *c) +{ + struct centauri_ipc *ipc = c->ipc; + struct cen_buffer *buffer, *next; + int i, cap; + u16 control; + + if (!ipc) + return; + centauri_wifi_unregister(c); + for (i = 0; i < 3; i++) { + if (!ipc->function[i]) + continue; + cap = pci_find_capability(ipc->function[i], PCI_CAP_ID_MSI); + if (cap) { + pci_read_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, &control); + pci_write_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, + control & ~PCI_MSI_FLAGS_ENABLE); + } + pci_clear_master(ipc->function[i]); + } + list_for_each_entry_safe(buffer, next, &ipc->buffers, list) { + dma_free_coherent(&c->pdev->dev, buffer->size, buffer->cpu, buffer->dma); + list_del(&buffer->list); + kfree(buffer); + } + for (i = 0; i < 3; i++) { + if (ipc->bar[i]) + pci_iounmap(ipc->function[i], ipc->bar[i]); + pci_dev_put(ipc->function[i]); + } + kfree(ipc); + c->ipc = NULL; +} + +int centauri_alpha_resources(struct centauri *c, struct cen_alpha_resources *resources) +{ + struct centauri_ipc *ipc = c->ipc; + + if (!ipc || !ipc->window || !ipc->alpha.buffer || !ipc->alpha.messages) + return -ENODEV; + resources->context = ipc->alpha.buffer->cpu; + resources->context_dma = ipc->alpha.buffer->dma; + resources->messages = ipc->alpha.messages->cpu; + resources->messages_dma = ipc->alpha.messages->dma; + resources->window = ipc->window->cpu; + resources->window_dma = ipc->window->dma; + resources->bar = ipc->bar[1]; + memcpy(resources->index, ipc->alpha.index, sizeof(resources->index)); + resources->mtr = ipc->alpha.mtr; + resources->mcr = ipc->alpha.mcr; + return 0; +} + +/* Called once under the Wi-Fi lock. Published logging memory stays owned by + * IPC until every function has bus mastering disabled in centauri_free_ipc. + */ +int centauri_alpha_log_buffer(struct centauri *c, void **cpu, dma_addr_t *dma, size_t *size) +{ + struct centauri_ipc *ipc = c->ipc; + + if (!ipc || !ipc->window) + return -ENODEV; + if (!ipc->alpha_log) + ipc->alpha_log = buffer_alloc(c, SZ_4M); + if (!ipc->alpha_log) + return -ENOMEM; + *cpu = ipc->alpha_log->cpu; + *dma = ipc->alpha_log->dma; + *size = ipc->alpha_log->size; + return 0; +} diff --git a/drivers/net/wireless/apple/centauri/centauri_pci.c b/drivers/net/wireless/apple/centauri/centauri_pci.c new file mode 100644 index 0000000000000..f3f7c6413f0a3 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_pci.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Initial real-hardware N1 PCI/ACIPC backend for J813. + * Register offsets are from AppleCentauriCommon/PROTOCOL.md and its measured + * J813 prototype. Firmware execution and radio operations are separate gates. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "centauri.h" + +static struct dentry *centauri_debug; + +static int centauri_status_show(struct seq_file *s, void *unused) +{ + struct centauri *c = s->private; + + seq_printf(s, "pci=%s vendor=%04x device=%04x\n", + pci_name(c->pdev), c->pdev->vendor, c->pdev->device); + if (c->link_ready) + seq_printf(s, "bar0_size=%pa exec_stage=%#x\n", &c->size, + readl(c->bar + c->stage_offset)); + else + seq_puts(s, "link unavailable; endpoint reads disabled\n"); + seq_printf(s, "transport=hardware-pci firmware_boot=%s image_response=%#x wifi=%s\n", + c->boot_attempted ? "attempted" : "not-started", c->image_response, + READ_ONCE(c->wifi) ? "registered" : "not-owned-by-this-function"); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(centauri_status); + +static void centauri_free_boot(struct centauri *c) +{ + struct device *dev = &c->pdev->dev; + + if (!c->link_ready && c->boot_attempted) { + dev_warn(dev, "retaining DMA buffers after failed link transition; reboot required\n"); + return; + } + /* Stop device writes before any DMA storage is released. */ + centauri_free_ipc(c); + if (c->msi_cap) + pci_write_config_word(c->pdev, c->msi_cap + PCI_MSI_FLAGS, + c->msi_original & ~PCI_MSI_FLAGS_ENABLE); + pci_clear_master(c->pdev); + if (c->firmware) + dma_free_coherent(dev, c->firmware_size, c->firmware, c->firmware_dma); + if (c->descriptor) + dma_free_coherent(dev, SZ_16K, c->descriptor, c->descriptor_dma); + if (c->msi_page) + dma_free_coherent(dev, SZ_16K, c->msi_page, c->msi_dma); + c->firmware = c->descriptor = c->msi_page = NULL; +} + +static int centauri_rom_boot(struct centauri *c) +{ + const struct firmware *fw; + struct device *dev = &c->pdev->dev; + u32 stage; + int ret; + + if ((c->pdev->device != 0x1900 && c->pdev->device != 0x1901) || + !of_machine_is_compatible("apple,j813")) + return -ENODEV; + if (!c->link_ready || c->boot_attempted) + return -EBUSY; + stage = readl(c->bar + 0x8000); + if (stage != 0 && stage != 4 && stage != 1) + return -EBUSY; + ret = request_firmware(&fw, "apple/centauri/j813-ftab.bin", dev); + if (ret) + return ret; + if (fw->size < 0x30 || fw->size > SZ_64M || + memcmp(fw->data + 0x20, "rkosftab", 8)) { + ret = -EINVAL; + goto release; + } + c->firmware_size = PAGE_ALIGN(fw->size); + c->firmware = dma_alloc_coherent(dev, c->firmware_size, &c->firmware_dma, GFP_KERNEL); + c->descriptor = dma_alloc_coherent(dev, SZ_16K, &c->descriptor_dma, GFP_KERNEL); + c->msi_page = dma_alloc_coherent(dev, SZ_16K, &c->msi_dma, GFP_KERNEL); + if (!c->firmware || !c->descriptor || !c->msi_page) { + ret = -ENOMEM; + goto free_buffers; + } + memset(c->firmware, 0, c->firmware_size); + memcpy(c->firmware, fw->data, fw->size); + memset(c->descriptor, 0, SZ_16K); + memset(c->msi_page, 0, SZ_16K); + /* Measured ROM scatter entry: u64 DMA, u32 length, u32 flags, u64 x2 zero. */ + put_unaligned_le64(c->firmware_dma, c->descriptor); + put_unaligned_le32(ALIGN(fw->size, SZ_4K), c->descriptor + 8); + c->msi_cap = pci_find_capability(c->pdev, PCI_CAP_ID_MSI); + if (!c->msi_cap) { + ret = -ENODEV; + goto free_buffers; + } + pci_read_config_word(c->pdev, c->msi_cap + PCI_MSI_FLAGS, &c->msi_original); + if (!(c->msi_original & PCI_MSI_FLAGS_64BIT)) { + ret = -EOPNOTSUPP; + goto free_buffers; + } + /* Poll the measured DMA-backed MSI mailbox until native IRQ support exists. */ + pci_write_config_dword(c->pdev, c->msi_cap + PCI_MSI_ADDRESS_LO, lower_32_bits(c->msi_dma)); + pci_write_config_dword(c->pdev, c->msi_cap + PCI_MSI_ADDRESS_HI, upper_32_bits(c->msi_dma)); + pci_write_config_word(c->pdev, c->msi_cap + PCI_MSI_DATA_64, 0xa5a0); + pci_write_config_word(c->pdev, c->msi_cap + PCI_MSI_FLAGS, + (c->msi_original & ~PCI_MSI_FLAGS_QSIZE) | PCI_MSI_FLAGS_ENABLE); + pci_set_master(c->pdev); + writel(0x00248142, c->bar + 0x807c); /* J813 board 0x24, T8142 chip. */ + writel(0, c->bar + 0x803c); + writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); + writel(upper_32_bits(c->descriptor_dma), c->bar + 0x8048); + writel(0x20, c->bar + 0x804c); + dma_wmb(); + c->boot_attempted = true; + if (stage == 4 || stage == 1) { + c->image_response = readl(c->bar + 0x8040); + dev_info(dev, "adopted firmware stage %u; native DMA buffers ready\n", stage); + ret = 0; + goto release; + } + writel(1, c->bar + 0x9000); + ret = readl_poll_timeout(c->bar + 0x8040, c->image_response, + c->image_response != 0, 20000, 20000000); + if (!ret) + ret = readl_poll_timeout(c->bar + 0x8000, stage, stage != 0, 20000, 20000000); + else + stage = readl(c->bar + 0x8000); + dma_rmb(); + dev_info(dev, "ROM firmware result: response=%#x stage=%#x mailbox=%#x poll=%d\n", + c->image_response, stage, READ_ONCE(*(u32 *)c->msi_page), ret); + /* Keep DMA buffers owned until device removal, even on timeout. */ + if (!ret && stage != 4) + ret = -EIO; + goto release; +free_buffers: + centauri_free_boot(c); +release: + release_firmware(fw); + return ret; +} + +static ssize_t centauri_boot_write(struct file *file, const char __user *buffer, + size_t count, loff_t *pos) +{ + struct centauri *c = file->private_data; + u32 value; + int ret = kstrtou32_from_user(buffer, count, 0, &value); + + if (ret || value < 1 || value > 3) + return ret ?: -EINVAL; + mutex_lock(&c->boot_lock); + if (value == 1) { + ret = centauri_rom_boot(c); + } else if (value == 3) { + ret = centauri_start_ipc(c); + } else if (!c->firmware || !c->link_ready || readl(c->bar + 0x8000) != 4) { + ret = -EBUSY; + } else { + c->link_ready = false; + ret = centauri_j813_port_cycle(c->pdev); + if (!ret) { + c->link_ready = true; + dev_info(&c->pdev->dev, "post-reset execution stage=%#x\n", + readl(c->bar + 0x8000)); + } + } + mutex_unlock(&c->boot_lock); + return ret ?: count; +} + +static const struct file_operations centauri_boot_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = centauri_boot_write, +}; + +static int centauri_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct centauri *c; + dma_addr_t dma; + void *page; + int ret; + + c = devm_kzalloc(&pdev->dev, sizeof(*c), GFP_KERNEL); + if (!c) + return -ENOMEM; + c->pdev = pdev; + c->link_ready = true; + mutex_init(&c->boot_lock); + c->stage_offset = id->driver_data; + ret = pcim_enable_device(pdev); + if (ret) + return ret; + c->size = pci_resource_len(pdev, 0); + if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) || + c->size < c->stage_offset + sizeof(u32)) + return dev_err_probe(&pdev->dev, -ENODEV, "invalid ACIPC BAR0\n"); + ret = pcim_iomap_regions(pdev, BIT(0), "apple_centauri"); + if (ret) + return ret; + c->bar = pcim_iomap_table(pdev)[0]; + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); + if (ret) + return dev_err_probe(&pdev->dev, ret, "40-bit DMA unavailable\n"); + /* Test Linux's DART allocation without publishing it to the device. */ + page = dma_alloc_coherent(&pdev->dev, SZ_16K, &dma, GFP_KERNEL); + if (!page) + return dev_err_probe(&pdev->dev, -ENOMEM, "DART DMA allocation failed\n"); + memset(page, 0, SZ_16K); + dev_info(&pdev->dev, "ACIPC endpoint reachable: stage=%#x, 40-bit DMA mapping=%pad\n", + readl(c->bar + c->stage_offset), &dma); + dma_free_coherent(&pdev->dev, SZ_16K, page, dma); + /* Bus mastering remains disabled until the firmware transport owns buffers. */ + pci_clear_master(pdev); + pci_set_drvdata(pdev, c); + c->debug = debugfs_create_dir(pci_name(pdev), centauri_debug); + debugfs_create_file("status", 0400, c->debug, c, ¢auri_status_fops); + debugfs_create_file("rom_boot", 0200, c->debug, c, ¢auri_boot_fops); + return 0; +} + +static void centauri_remove(struct pci_dev *pdev) +{ + struct centauri *c = pci_get_drvdata(pdev); + + centauri_wifi_unregister(c); + debugfs_remove_recursive(c->debug); + centauri_free_boot(c); +} + +static const struct pci_device_id centauri_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1900), .driver_data = 0x8000 }, + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1901), .driver_data = 0x8000 }, + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1902), .driver_data = 0x0008 }, + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1903), .driver_data = 0x0018 }, + { } +}; +MODULE_DEVICE_TABLE(pci, centauri_ids); + +static struct pci_driver centauri_driver = { + .name = "apple_centauri", + .id_table = centauri_ids, + .probe = centauri_probe, + .remove = centauri_remove, +}; + +static int __init centauri_init(void) +{ + int ret; + + centauri_debug = debugfs_create_dir("apple_centauri", NULL); + ret = pci_register_driver(¢auri_driver); + if (ret) + debugfs_remove_recursive(centauri_debug); + return ret; +} + +static void __exit centauri_exit(void) +{ + pci_unregister_driver(¢auri_driver); + debugfs_remove_recursive(centauri_debug); +} +module_init(centauri_init); +module_exit(centauri_exit); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple N1 Centauri native PCI and ACIPC bring-up backend"); +MODULE_FIRMWARE("apple/centauri/j813-ftab.bin"); diff --git a/drivers/net/wireless/apple/centauri/centauri_port.c b/drivers/net/wireless/apple/centauri/centauri_port.c new file mode 100644 index 0000000000000..eea0a1ba11e0e --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_port.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* J813 preboot -> boot port cycle, transcribed from the measured proxy + * alpha_wifi_scan.py pcie_cycle/apple_teardown/apple_bringup sequence. + * These apertures were verified from the live ADT before this Linux boot. + * This temporary board helper belongs in the host-controller driver once + * T8142 platform/clock/GPIO bindings are available. + */ +#include +#include +#include +#include +#include +#include "centauri.h" + +static void update(void __iomem *base, u32 offset, u32 clear, u32 set) +{ + writel((readl(base + offset) & ~clear) | set, base + offset); +} + +static bool wait_bits(void __iomem *address, u32 mask, u32 wanted, u32 us) +{ + u32 value; + + return !readl_poll_timeout(address, value, (value & mask) == wanted, 100, us); +} + +int centauri_j813_port_cycle(struct pci_dev *pdev) +{ + static const u32 reset[][2] = { + {0x88, 0x110}, {0x100, ~0U}, {0x148, ~0U}, {0x210, ~0U}, + {0x80, 0}, {0x84, 0}, {0x104, 0xfffffff0}, {0x124, 0x100}, + {0x16c, 0}, {0x13c, 0}, {0x800, 0x00100100}, + {0x808, 0x001001ff}, {0x82c, 0x00010000}, + }; + static const u32 tune[][2] = { + {0x130, 0x03020000}, {0x140, 0x10}, {0x144, 0x00253770}, + {0x21c, 0}, {0x834, 0}, {0x83c, 0}, + }; + struct pci_dev *bridge = pdev->bus->self; + void __iomem *port, *phy, *pad; + u32 value, gpio, identity; + u16 link; + unsigned int lanes, i; + int ret = -ETIMEDOUT, pm; + bool ack, l2; + + if (!of_machine_is_compatible("apple,j813") || !bridge || + bridge->vendor != PCI_VENDOR_ID_APPLE || bridge->device != 0x100c) + return -ENODEV; + port = ioremap(0x488028000ULL, 0x4000); + phy = ioremap(0x48f010000ULL, 0x1000); + pad = ioremap(0x39a000300ULL, sizeof(u32)); + if (!port || !phy || !pad) { + ret = -ENOMEM; + goto out; + } + gpio = readl(pad); + if (gpio & BIT(21)) { + ret = -EPERM; + goto out; + } + ret = pci_save_state(bridge); + if (ret) + goto out; + ret = pci_save_state(pdev); + if (ret) + goto out; + pcie_capability_read_word(bridge, PCI_EXP_LNKSTA, &link); + lanes = (link & PCI_EXP_LNKSTA_NLW) >> PCI_EXP_LNKSTA_NLW_SHIFT; + lanes = lanes >= 1 && lanes <= 16 ? BIT(lanes) - 1 : 1; + pm = pci_find_capability(pdev, PCI_CAP_ID_PM); + if (pm) { + pci_write_config_word(pdev, pm + PCI_PM_CTRL, 0x103); + msleep(10); + } + update(port, 0x800, 0, 0x100); + writel(0x11, port + 0x88); + ack = wait_bits(port + 0x88, 1, 0, 500000); + l2 = wait_bits(port + 0x208, BIT(6), BIT(6), 500000); + dev_info(&pdev->dev, "port preboot transition: PME ack=%d L2=%d\n", ack, l2); + /* Endpoint reads cease here until the root LTSSM is back in L0. */ + pci_write_config_word(bridge, PCI_COMMAND, 0); + pci_read_config_dword(bridge, PCI_PRIMARY_BUS, &value); + pci_write_config_dword(bridge, PCI_PRIMARY_BUS, value & 0xff0000ff); + update(phy, 0, BIT(31), 0); + writel((gpio & ~0xfU) | 2, pad); + if (readl(pad) != ((gpio & ~0xfU) | 2)) { + ret = -EIO; + goto out; + } + wait_bits(port + 0x804, BIT(2), 0, 50000); + wait_bits(port + 0x820, 0x1ff, 0, 100000); + update(port, 0x13c, 0, 0x100); + for (i = 0; i < 200; i++) { + pci_read_config_dword(bridge, 0x728, &value); + if ((value & 0x3f) == 0x14) + break; + usleep_range(1000, 1200); + } + wait_bits(port + 0x208, BIT(12), 0, 200000); + readl(phy); + wait_bits(port + 0xa8, 3, 0, 10000); + update(port, 0x82c, 0, BIT(24)); + wait_bits(port + 0xa8, lanes << 16, lanes << 16, 20000); + update(port, 0x82c, 0, BIT(16)); + udelay(10); + update(port, 0x82c, 1, 0); + udelay(10); + update(port, 0x82c, BIT(24), 0); + update(port, 0x13c, 0x100, 0); + update(phy, 0, BIT(31), 0); + update(phy, 0, 0x400, 0); + update(phy, 0, 0x200, 0); + update(phy, 0, 0, 0x10); + update(phy, 0, 1, 0); + update(phy, 0, 2, 0); + writel(~0U, port + 0x100); + update(port, 0x800, 1, 0); + wait_bits(port + 0x804, 1, 0, 200000); + update(phy, 0, BIT(31), 0); + for (i = 0; i < ARRAY_SIZE(reset); i++) + writel(reset[i][1], port + reset[i][0]); + for (i = 0; i < 16; i++) + writel(0, port + 0x3000 + 4 * i); + for (i = 0; i < 256; i++) + writel(0, port + 0x3800 + 4 * i); + for (i = 0; i < ARRAY_SIZE(tune); i++) + writel(tune[i][1], port + tune[i][0]); + update(port, 0x800, 0, 1); + udelay(10); + update(phy, 0, 0, 1); + if (!wait_bits(phy, 4, 4, 50000)) + goto timed_out; + update(phy, 0, 0, 2); + if (!wait_bits(phy, 8, 8, 50000)) + goto timed_out; + update(phy, 0, 0x10, 0); + udelay(100); + update(phy, 0, 0, 0x200); + update(phy, 0, 0, 0x400); + udelay(100); + update(port, 0x82c, 0, 1); + writel((gpio & ~0xfU) | 3, pad); + udelay(100); + readl(phy); + wait_bits(port + 0xa8, 3, 0, 20000); + update(port, 0x82c, BIT(16), 0); + if (!wait_bits(port + 0x804, 1, 1, 200000)) + goto timed_out; + update(port, 0x800, 0x100, 0); + pci_read_config_dword(bridge, 0x8bc, &value); + pci_write_config_dword(bridge, 0x8bc, value ^ 1); + pci_read_config_dword(bridge, 0x80, &value); + pci_write_config_dword(bridge, 0x80, value | 0x400); + update(phy, 0, 0, BIT(31)); + update(port, 0x80, 0, 1); + for (i = 0; i < 2000; i++) { + pci_read_config_dword(bridge, 0x728, &value); + if ((value & 0x3f) == 0x11) + break; + usleep_range(1000, 1200); + } + if (i == 2000) + goto timed_out; + msleep(100); + pci_restore_state(bridge); + /* Bus numbers/windows now restored; endpoint config may be read again. */ + pci_read_config_dword(pdev, PCI_VENDOR_ID, &identity); + if (identity != 0x1901106b && identity != 0x1900106b) { + ret = -ENODEV; + goto out; + } + pci_restore_state(pdev); + if (pm) { + pci_write_config_word(pdev, pm + PCI_PM_CTRL, 0); + msleep(10); + } + dev_info(&pdev->dev, "port transition complete: endpoint=%#x LTSSM=%#x\n", + identity, value & 0x3f); + ret = 0; + goto out; +timed_out: + ret = -ETIMEDOUT; +out: + if (pad) + iounmap(pad); + if (phy) + iounmap(phy); + if (port) + iounmap(port); + return ret; +} diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c new file mode 100644 index 0000000000000..21201241f49ec --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -0,0 +1,2312 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* cfg80211 connection to the measured Alpha command/management rings. + * Control queues recycle across hardware-validated wraps. Packet data uses + * a separate protocol; never report transmit success before implementing it. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "centauri.h" + +#define CR_BASE 0x100000 +#define TR_BASE 0x400000 +#define RING_SPAN 0x20000 +#define TX_BUFFER 0x600000 +#define BUFFER_SIZE 2048 +#define DATA_TX_BUFFER 0xd00000 +#define DATA_RX_BUFFER 0xe00000 +#define DATA_TX_SLOTS 128 +#define DATA_TX_QUEUE_LIMIT 64 + +/* The firmware owns window +8 MiB through +13 MiB as scratch memory. */ +static_assert(DATA_TX_BUFFER >= CEN_ALPHA_SCRATCH_OFFSET + CEN_ALPHA_SCRATCH_SIZE); +static_assert(DATA_TX_BUFFER + DATA_TX_SLOTS * BUFFER_SIZE <= DATA_RX_BUFFER); + +static bool verify_pmk; +module_param(verify_pmk, bool, 0400); +MODULE_PARM_DESC(verify_pmk, "Compare firmware PMK with the requested key without exposing either"); + +static bool eapol_after_assoc; +module_param(eapol_after_assoc, bool, 0644); +MODULE_PARM_DESC(eapol_after_assoc, "One EAPOL-Start probe at the next successful association event"); + +static bool link_diagnostics; +module_param(link_diagnostics, bool, 0644); +MODULE_PARM_DESC(link_diagnostics, "Query non-secret firmware link status during association"); + +static bool eapol_control_profile; +module_param(eapol_control_profile, bool, 0644); +MODULE_PARM_DESC(eapol_control_profile, "Use the measured control TX profile for the EAPOL-Start probe"); + +static bool eapol_unicast; +module_param(eapol_unicast, bool, 0644); +MODULE_PARM_DESC(eapol_unicast, "Address the bounded EAPOL-Start probe to the associated AP"); + +static bool eapol_relative_lifetime; +module_param(eapol_relative_lifetime, bool, 0644); +MODULE_PARM_DESC(eapol_relative_lifetime, "Use a measured two-second relative expiration for the probe"); + +static bool data_tx_broadcast; +module_param(data_tx_broadcast, bool, 0400); +MODULE_PARM_DESC(data_tx_broadcast, "Diagnostic: open TR 25 with measured broadcast metadata; transmit only broadcast Ethernet frames"); + +static bool data_tx_control_profile; +module_param(data_tx_control_profile, bool, 0600); +MODULE_PARM_DESC(data_tx_control_profile, "Diagnostic: select configured TX policy 1 for normal data; apply tx_config action 2 first"); + +static bool data_tx_native_prepare = true; +module_param(data_tx_native_prepare, bool, 0600); +MODULE_PARM_DESC(data_tx_native_prepare, "Use native packet preparation value 4; disable only to reproduce the legacy data TX failure"); + +static bool station_link_addresses; +module_param(station_link_addresses, bool, 0400); +MODULE_PARM_DESC(station_link_addresses, "Diagnostic: use the netdev address for each firmware link in this single-link WPA2 client"); + +struct centauri_rx_snapshot { + __le32 version, count; + struct { + u8 completion[32]; + u8 packet[4096]; + } sample[8]; +}; + +struct centauri_wifi { + struct centauri *control; + struct cen_alpha_resources a; + struct wiphy *wiphy; + struct wireless_dev wdev; + struct net_device *netdev; + struct work_struct scan_work; + struct work_struct connect_work; + struct delayed_work link_work; + struct cfg80211_scan_request *scan_request; + struct mutex lock; + /* The logger has its own coherent allocation. Reads/acknowledgements + * must remain possible while scan/connect hold the command mutex. + * Initialization takes lock then fwlog_lock; readers take only the latter. + */ + struct mutex fwlog_lock; + u16 main_head, command_head, sequence; + u32 rx_seen[5]; + u16 rx_next[5], rx_head[5]; + u16 reply_sequence, reply_status; + u32 reply_command, channels_seen, scans_done, beacons_seen; + bool reply_valid, registered, configured; + bool stopping; + bool connecting, connected, connect_event, disconnect_event; + bool associated, pmk_seen, pmk_matches; + u8 expected_pmk[32]; + u16 connect_status; + u8 bssid[ETH_ALEN], connect_payload[1024], request_ie[256]; + size_t connect_size, request_ie_size; + u8 passphrase[64]; + size_t passphrase_size; + struct dentry *passphrase_file; + struct dentry *data_probe_file; + struct dentry *data_status_file; + struct dentry *data_rx_snapshot_file; + struct centauri_rx_snapshot rx_snapshot; + u32 data_rx_received, data_rx_reposted, data_rx_bad; + struct sk_buff_head data_tx_queue; + u16 data_tx_length[DATA_TX_SLOTS]; + bool data_tx_busy[DATA_TX_SLOTS]; + u32 data_tx_submitted, data_tx_completed, data_tx_failed; + u32 data_tx_coalesced, data_tx_bad; + struct dentry *station_stats_file; + struct dentry *eapol_start_file; + struct dentry *country_file; + struct dentry *link_status_file; + struct dentry *tx_config_file; + struct dentry *host_wakeup_file; + struct dentry *fwlog_start_file, *fwlog_status_file, *fwlog_buffer_file, *fwlog_ack_file; + void *fwlog_buffer; + dma_addr_t fwlog_dma; + size_t fwlog_size; + bool fwlog_attempted, fwlog_ready; + u8 tx_profile[10], tx_defaults_applied; + bool tx_profile_valid; + u8 link_count, link_prime[3], link_channel[3], link_band[3], emlsr_mode; + bool link_status_valid, emlsr_valid; + u16 link_status_size; + u16 link_tags[12], link_lengths[12]; + u8 link_tlv_count; + u8 country[3]; + bool country_valid; + u8 station_stats[0x378]; + bool station_stats_valid; + u8 rx_crypto_stats[64]; + bool rx_crypto_stats_valid; + struct dentry *rx_crypto_stats_file; + u8 dot11_mib_stats[192]; + bool dot11_mib_stats_valid; + struct dentry *dot11_mib_stats_file; + u8 tx_power_report[280]; + bool tx_power_report_valid; + struct dentry *tx_power_report_file; + u16 stats_reply_size, stats_tag, stats_length; + u32 data_probe_done; + struct ieee80211_supported_band bands[2]; + struct ieee80211_channel channels[39]; +}; + +/* These helpers require wifi->lock, including from the connect worker. */ +static int transmit_eapol_start(struct centauri_wifi *wifi); +static int query_station_stats(struct centauri_wifi *wifi); +static int query_link_status(struct centauri_wifi *wifi); + +static const u32 rx_buffer[5] = { 0, 0, 0x620000, 0x680000, 0x700000 }; +static const u16 ring_size[5] = { 0, 64, 192, 96, 192 }; +static const u16 five_ghz[] = { + 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, + 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, +}; +static struct ieee80211_rate rates[] = { + { .bitrate = 10 }, { .bitrate = 20 }, { .bitrate = 55 }, { .bitrate = 110 }, + { .bitrate = 60 }, { .bitrate = 90 }, { .bitrate = 120 }, { .bitrate = 180 }, + { .bitrate = 240 }, { .bitrate = 360 }, { .bitrate = 480 }, { .bitrate = 540 }, +}; +static const u32 cipher_suites[] = { + WLAN_CIPHER_SUITE_CCMP, WLAN_CIPHER_SUITE_AES_CMAC, +}; + +static u16 index_read(struct centauri_wifi *wifi, unsigned int array, unsigned int index) +{ + return le16_to_cpu(READ_ONCE(*(__le16 *)(wifi->a.context + wifi->a.index[array] + 2 * index))); +} + +static void index_write(struct centauri_wifi *wifi, unsigned int array, + unsigned int index, u16 value) +{ + WRITE_ONCE(*(__le16 *)(wifi->a.context + wifi->a.index[array] + 2 * index), cpu_to_le16(value)); +} + +static void completions(struct centauri_wifi *wifi) +{ + unsigned int vector; + + for (vector = 0; vector <= 4; vector++) + index_write(wifi, 2, vector, index_read(wifi, 0, vector)); + dma_wmb(); +} + +static int alpha_message(struct centauri_wifi *wifi, const void *message, size_t size) +{ + u16 head = wifi->main_head; + u8 *descriptor; + unsigned long deadline = jiffies + msecs_to_jiffies(2000); + + if (head >= 15 || size > 512) + return -ENOSPC; + memcpy(wifi->a.messages + (head + 1) * 512, message, size); + descriptor = wifi->a.context + wifi->a.mtr + 16 * head; + put_unaligned_le32(1 | (size << 8), descriptor); + put_unaligned_le64(wifi->a.messages_dma + (head + 1) * 512, descriptor + 4); + put_unaligned_le32(head + 1, descriptor + 12); + index_write(wifi, 3, 0, head + 1); + if (head) + index_write(wifi, 3, 8, head + 1); + dma_wmb(); + wifi->main_head++; + writel(1, wifi->a.bar + 0x1004); + do { + if (readl(wifi->a.bar + 8) != 2) + return -EIO; + if (index_read(wifi, 1, 0) >= head + 1) { + dma_rmb(); + completions(wifi); + return 0; + } + usleep_range(100, 200); + } while (time_before(jiffies, deadline)); + return -ETIMEDOUT; +} + +static int open_pair(struct centauri_wifi *wifi, u16 id) +{ + u8 cr[0x2c] = { 2 }, tr[0x34] = { 1 }; + int ret; + + put_unaligned_le32(id | (id << 16), cr + 4); + put_unaligned_le64(wifi->a.window_dma + CR_BASE + (id - 1) * RING_SPAN, cr + 8); + put_unaligned_le16(1024, cr + 0x10); + put_unaligned_le16(0xffff, cr + 0x12); + ret = alpha_message(wifi, cr, sizeof(cr)); + if (ret) + return ret; + put_unaligned_le32(id | (id << 16), tr + 4); + put_unaligned_le64(wifi->a.window_dma + TR_BASE + (id - 1) * RING_SPAN, tr + 8); + put_unaligned_le64(~0ULL, tr + 0x10); + put_unaligned_le16(ring_size[id], tr + 0x18); + put_unaligned_le16(id, tr + 0x1a); + put_unaligned_le16(id, tr + 0x1c); + put_unaligned_le16(0x40, tr + 0x1e); + put_unaligned_le16(2, tr + 0x24); + return alpha_message(wifi, tr, sizeof(tr)); +} + +static void report_beacon(struct centauri_wifi *wifi, const u8 *payload, size_t size) +{ + const u8 *frame, *ie, *end; + struct ieee80211_channel *channel; + struct cfg80211_bss *bss; + u16 frame_size; + int number = 0, frequency, signal; + enum nl80211_band band; + + if (!wifi->registered || size < 0x60 + 36) + return; + /* The ACIPC packet length is authoritative; the vendor descriptor's + * separate 802.11 length can include a stripped FCS. The measured parser + * consumes the frame bytes that were actually returned. + */ + frame_size = size - 0x60; + frame = payload + 0x60; + if (!ieee80211_is_beacon(*(__le16 *)frame) && !ieee80211_is_probe_resp(*(__le16 *)frame)) + return; + end = frame + frame_size; + for (ie = frame + 36; ie + 2 <= end && ie + 2 + ie[1] <= end; ie += 2 + ie[1]) { + if ((ie[0] == WLAN_EID_DS_PARAMS || ie[0] == WLAN_EID_HT_OPERATION) && ie[1]) + number = ie[2]; + } + if (!number) + return; + band = number <= 14 ? NL80211_BAND_2GHZ : NL80211_BAND_5GHZ; + frequency = ieee80211_channel_to_frequency(number, band); + channel = ieee80211_get_channel(wifi->wiphy, frequency); + if (!channel) + return; + signal = max((s8)payload[0x16], (s8)payload[0x17]); + if (signal >= 0 || signal < -127) + signal = -95; + bss = cfg80211_inform_bss_frame(wifi->wiphy, channel, + (struct ieee80211_mgmt *)frame, frame_size, signal * 100, GFP_KERNEL); + if (bss) { + wifi->beacons_seen++; + cfg80211_put_bss(wifi->wiphy, bss); + } +} + +static void post_receive(struct centauri_wifi *wifi, unsigned int ring, u16 slot) +{ + u8 *descriptor = wifi->a.window + TR_BASE + (ring - 1) * RING_SPAN + slot * 16; + u32 offset = rx_buffer[ring] + slot * BUFFER_SIZE; + + memset(wifi->a.window + offset, 0, BUFFER_SIZE); + put_unaligned_le32(1 | (BUFFER_SIZE << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + offset, descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); +} + +/* Host-owned RX buffer cookies and completion length are the ACIPC fields + * validated in probe 28. Probe 29 validates the footer's prefix length and + * Ethernet framing with IPv4 header checksums and IPv6 payload lengths. + * Called with wifi->lock, including during connect/command polling. + */ +static void drain_data_rx(struct centauri_wifi *wifi) +{ + u16 head, tail, post; + unsigned int processed = 0; + + if (!(wifi->data_probe_done & BIT(4)) || readl(wifi->a.bar + 8) != 2) + return; + head = index_read(wifi, 0, 207); + tail = index_read(wifi, 2, 207); + post = index_read(wifi, 3, 12); + if (head >= 128 || tail >= 128 || post >= 128) { + wifi->data_rx_bad++; + return; + } + dma_rmb(); + while (tail != head && processed < 127) { + u8 *completion = wifi->a.window + 0x240000 + tail * 32; + u32 header = get_unaligned_le32(completion); + u32 info = get_unaligned_le32(completion + 4); + u16 cookie = info & 0xffff, length = info >> 16; + u16 next = (post + 1) % 128; + u32 count = le32_to_cpu(wifi->rx_snapshot.count); + u8 *packet, *descriptor; + u32 buffer; + + if ((header >> 16) != 12 || !cookie || cookie > 127 || + length > 4096 || get_unaligned_le32(completion + 8)) { + wifi->data_rx_bad++; + break; + } + /* Do not overwrite a descriptor the receive engine still owns. */ + if (next == index_read(wifi, 1, 12)) + break; + buffer = DATA_RX_BUFFER + (cookie - 1) * 4096; + packet = wifi->a.window + buffer; + if (count < ARRAY_SIZE(wifi->rx_snapshot.sample)) { + memcpy(wifi->rx_snapshot.sample[count].completion, completion, 32); + memcpy(wifi->rx_snapshot.sample[count].packet, packet, length); + wifi->rx_snapshot.version = cpu_to_le32(1); + wifi->rx_snapshot.count = cpu_to_le32(count + 1); + } + wifi->data_rx_received++; + if (wifi->connected && netif_running(wifi->netdev)) { + u32 footer = get_unaligned_le32(completion + 12); + unsigned int padding = (footer >> 12) & 15; + struct sk_buff *skb = NULL; + + /* Extended prefix headers are not decoded yet. */ + if (!(footer & BIT(24)) && length >= padding + ETH_HLEN) + skb = netdev_alloc_skb_ip_align(wifi->netdev, length - padding); + if (skb) { + skb_put_data(skb, packet + padding, length - padding); + skb->protocol = eth_type_trans(skb, wifi->netdev); + skb->ip_summed = CHECKSUM_NONE; + wifi->netdev->stats.rx_packets++; + wifi->netdev->stats.rx_bytes += length - padding; + netif_rx(skb); + } else { + wifi->netdev->stats.rx_dropped++; + } + } else { + wifi->netdev->stats.rx_dropped++; + } + descriptor = wifi->a.window + 0x520000 + post * 16; + put_unaligned_le32(1 | (4096 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le32(cookie, descriptor + 12); + wifi->data_rx_reposted++; + tail = (tail + 1) % 128; + post = next; + processed++; + } + if (processed) { + dma_wmb(); + index_write(wifi, 2, 207, tail); + index_write(wifi, 3, 12, post); + dma_wmb(); + writel(1, wifi->a.bar + 0x1050); + } +} + +/* Serialized with the firmware poller. Payload slots remain owned until a + * CR completion, even if the device has already consumed the TR descriptor. + */ +static void drain_data_tx(struct centauri_wifi *wifi) +{ + u16 head, tail, post; + unsigned int budget = 256; + bool posted = false, completed = false; + + if (!(wifi->data_probe_done & BIT(5)) || readl(wifi->a.bar + 8) != 2) + return; + head = index_read(wifi, 0, 192); + tail = index_read(wifi, 2, 192); + post = index_read(wifi, 3, 25); + if (head >= 2048 || tail >= 2048 || post >= DATA_TX_SLOTS) + return; + dma_rmb(); + while (tail != head && budget--) { + u8 *entry = wifi->a.window + 0x200000 + tail * 32; + u16 ring = get_unaligned_le32(entry) >> 16; + u16 cookie = get_unaligned_le16(entry + 4); + u16 count = get_unaligned_le16(entry + 9); + u8 status = entry[12]; + + if (ring == 25) { + unsigned int first, i; + + /* Airship cr_controller::sync maps CD bytes 9:10 to the + * transfer count. With the multicomp flag (CD byte 1 bit 0), + * the cookie names the LAST transfer; prior transfers share + * its completion status. We submit one contiguous DMA buffer + * per slot, so those preceding cookies must all still be busy. + */ + if (!cookie || cookie > DATA_TX_SLOTS || !count || + count >= DATA_TX_SLOTS || !(entry[1] & 1)) { + wifi->data_tx_bad++; + break; + } + first = (cookie + DATA_TX_SLOTS - count) % DATA_TX_SLOTS; + for (i = 0; i < count; i++) + if (!wifi->data_tx_busy[(first + i) % DATA_TX_SLOTS]) + break; + if (i != count) { + wifi->data_tx_bad++; + break; + } + if (count > 1) + wifi->data_tx_coalesced++; + for (i = 0; i < count; i++) { + unsigned int slot = (first + i) % DATA_TX_SLOTS; + + wifi->data_tx_completed++; + if (status == 0 || status == 0x83) { + wifi->netdev->stats.tx_packets++; + wifi->netdev->stats.tx_bytes += wifi->data_tx_length[slot]; + } else { + wifi->data_tx_failed++; + wifi->netdev->stats.tx_errors++; + } + wifi->data_tx_busy[slot] = false; + } + } + tail = (tail + 1) % 2048; + completed = true; + } + if (completed) + index_write(wifi, 2, 192, tail); + budget = DATA_TX_SLOTS - 1; + while (wifi->connected && !READ_ONCE(wifi->stopping) && + netif_running(wifi->netdev) && budget--) { + u16 next = (post + 1) % DATA_TX_SLOTS; + u32 buffer = DATA_TX_BUFFER + post * BUFFER_SIZE; + u8 *descriptor = wifi->a.window + 0x540000 + post * 32; + struct sk_buff *skb; + u8 destination[ETH_ALEN]; + u64 metadata = 6ULL << 27; + + if (wifi->data_tx_busy[post] || next == index_read(wifi, 1, 25)) + break; + skb = skb_dequeue(&wifi->data_tx_queue); + if (!skb) + break; + /* This diagnostic ring is only valid for its broadcast destination. + * Do not silently send unicast or IPv6 multicast through that flow. + */ + if (data_tx_broadcast && + (skb_copy_bits(skb, 0, destination, ETH_ALEN) || + !is_broadcast_ether_addr(destination))) { + wifi->netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + continue; + } + if (skb_copy_bits(skb, 0, wifi->a.window + buffer, skb->len)) { + wifi->netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + continue; + } + memset(descriptor, 0, 32); + put_unaligned_le32(3 | (skb->len << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le16(post + 1, descriptor + 12); + /* AirshipDataPacket type 1 supplies this 16-byte TX header inline. + * setTxPolicyId(1), 0x1809ed27c, sets validity bit 60 and ID bit 48. + * Keep normal policy zero by default; the switch permits a bounded + * comparison with the existing, read-back-verified control profile. + */ + if (READ_ONCE(data_tx_control_profile) && + (wifi->tx_defaults_applied & BIT(1))) + metadata |= BIT_ULL(60) | BIT_ULL(48); + /* Native 25G76 AirshipDataPacket::prepare (0x18090faf8) writes + * (argument & 7) to bits 17:15. Guest37's captured DHCP Discover + * has value 4 (0x20000). The same-connection Linux A/B changes + * zero DHCP offers into three offers by setting only this field. + * The opt-out retains the old value for diagnostic reproduction. + */ + if (READ_ONCE(data_tx_native_prepare)) + metadata |= 4ULL << 15; + put_unaligned_le64(metadata, descriptor + 16); + put_unaligned_le64(2000000, descriptor + 24); + wifi->data_tx_length[post] = skb->len; + wifi->data_tx_busy[post] = true; + wifi->data_tx_submitted++; + dev_kfree_skb_any(skb); + post = next; + posted = true; + } + if (posted || completed) { + dma_wmb(); + if (posted) + index_write(wifi, 3, 25, post); + dma_wmb(); + writel(1, wifi->a.bar + 0x1030); + } + if (wifi->connected && !READ_ONCE(wifi->stopping) && netif_running(wifi->netdev) && + skb_queue_len(&wifi->data_tx_queue) < DATA_TX_QUEUE_LIMIT) + netif_wake_queue(wifi->netdev); +} + +static void drain(struct centauri_wifi *wifi) +{ + unsigned int ring; + bool posted = false; + + for (ring = 2; ring <= 4; ring++) { + unsigned int budget = ring_size[ring]; + + while (budget--) { + u16 slot = wifi->rx_next[ring], size, status, sequence; + u32 command; + u8 *descriptor = wifi->a.window + TR_BASE + (ring - 1) * RING_SPAN + slot * 16; + u8 *packet = wifi->a.window + rx_buffer[ring] + slot * BUFFER_SIZE; + + if (!(READ_ONCE(descriptor[15]) & 4)) + break; + dma_rmb(); + size = get_unaligned_le16(packet + 0x10); + status = get_unaligned_le16(packet + 0xe); + sequence = get_unaligned_le16(packet + 0x12); + command = get_unaligned_le32(packet + 0x14) & ~BIT(30); + wifi->rx_seen[ring]++; + wifi->rx_next[ring] = (slot + 1) % ring_size[ring]; + if (size < 24 || size > BUFFER_SIZE) + goto refill; + if (ring == 4) { + report_beacon(wifi, packet + 24, size - 24); + } else if (command & BIT(31)) { + if (command == 0x80000030) + wifi->channels_seen++; + if (command == 0x80000005) + wifi->scans_done++; + if (command == 0x80000000 && wifi->connecting) { + wifi->connect_event = true; + wifi->connect_status = status; + } + if (command == 0x80000001) { + wifi->associated = status == 0; + dev_info(&wifi->control->pdev->dev, "Wi-Fi association event: status=%u\n", status); + if (eapol_after_assoc && wifi->associated && wifi->connecting && + !wifi->connect_event) { + int probe_ret = transmit_eapol_start(wifi); + + eapol_after_assoc = false; + dev_info(&wifi->control->pdev->dev, + "association-event EAPOL-Start: rc=%d\n", probe_ret); + } + } + /* ACI event 0x50 carries fixed, non-secret supplicant + * diagnostics: result, phase and handshake timestamps. + * Do not dump the event or any EAPOL/key material. + */ + if (command == 0x80000050 && size >= 64) + dev_info(&wifi->control->pdev->dev, + "Wi-Fi supplicant: result=%u phase=%u times=%u/%u/%u/%u/%u\n", + get_unaligned_le32(packet + 0x18), packet[0x1c], + get_unaligned_le32(packet + 0x24), + get_unaligned_le32(packet + 0x28), + get_unaligned_le32(packet + 0x2c), + get_unaligned_le32(packet + 0x30), + get_unaligned_le32(packet + 0x34)); + if (command == 0x80000003 || command == 0x80000004) + wifi->disconnect_event = true; + if (wifi->connecting && command != 0x80000000 && + command != 0x80000001 && command != 0x80000030 && + command != 0x80000005 && command != 0x80000050) + dev_info_ratelimited(&wifi->control->pdev->dev, + "Wi-Fi event: id=%#x status=%u size=%u\n", + command, status, size); + } else { + wifi->reply_valid = true; + wifi->reply_sequence = sequence; + wifi->reply_status = status; + wifi->reply_command = command; + if (command == 10) { + wifi->country_valid = !status && size >= 27; + if (wifi->country_valid) + memcpy(wifi->country, packet + 24, 3); + } + if (command == 56) { + wifi->tx_profile_valid = !status && size >= 34; + if (wifi->tx_profile_valid) + memcpy(wifi->tx_profile, packet + 24, sizeof(wifi->tx_profile)); + } + if (command == 20 && !status) { + /* getInterfaceStatus skips its 32-byte fixed response. */ + unsigned int pos = 32; + + wifi->link_status_size = size; + while (pos + 4 <= size) { + u16 tag = get_unaligned_le16(packet + pos); + u16 length = get_unaligned_le16(packet + pos + 2); + u8 *value = packet + pos + 4; + unsigned int i; + + if (length > size - pos - 4) + break; + if (wifi->link_tlv_count < ARRAY_SIZE(wifi->link_tags)) { + wifi->link_tags[wifi->link_tlv_count] = tag; + wifi->link_lengths[wifi->link_tlv_count++] = length; + } + /* ACIDebugAdapter::dumpMloStatus, TLVs 377/378. + * Keep only link count/channel/role. Do not retain MACs. + */ + if (tag == 377 && length >= 64 && value[0] <= 3) { + wifi->link_count = value[0]; + wifi->link_status_valid = true; + for (i = 0; i < value[0]; i++) { + u16 chanspec = get_unaligned_le16(value + 26 + 17 * i); + + wifi->link_prime[i] = value[25 + 17 * i]; + wifi->link_channel[i] = chanspec & 0xff; + wifi->link_band[i] = (chanspec >> 14) & 3; + } + } + if (tag == 378 && length >= 9) { + wifi->emlsr_valid = true; + wifi->emlsr_mode = value[0]; + } + pos += 4 + length; + } + } + if (command == 0xb0001) { + wifi->stats_reply_size = size; + wifi->stats_tag = size >= 28 ? get_unaligned_le16(packet + 24) : 0; + wifi->stats_length = size >= 28 ? get_unaligned_le16(packet + 26) : 0; + } + if (command == 0xb0001 && !status && wifi->stats_tag == 0xf928 && + (wifi->stats_length == 752 || wifi->stats_length == 888) && + size >= 28 + wifi->stats_length) { + /* Native firmware returns the older 752-byte counter record. */ + memset(wifi->station_stats, 0, sizeof(wifi->station_stats)); + memcpy(wifi->station_stats, packet + 28, wifi->stats_length); + wifi->station_stats_valid = true; + } + if (command == 0xb0001 && !status && wifi->stats_tag == 0xf921 && + wifi->stats_length == sizeof(wifi->rx_crypto_stats) && + size >= 28 + sizeof(wifi->rx_crypto_stats)) { + memcpy(wifi->rx_crypto_stats, packet + 28, sizeof(wifi->rx_crypto_stats)); + wifi->rx_crypto_stats_valid = true; + } + if (command == 0xb0001 && !status && wifi->stats_tag == 0xf91b && + wifi->stats_length == sizeof(wifi->dot11_mib_stats) && + size >= 28 + sizeof(wifi->dot11_mib_stats)) { + memcpy(wifi->dot11_mib_stats, packet + 28, sizeof(wifi->dot11_mib_stats)); + wifi->dot11_mib_stats_valid = true; + } + if (command == 0xb0001 && !status && wifi->stats_tag == 0xf920 && + wifi->stats_length == sizeof(wifi->tx_power_report) && + size >= 28 + sizeof(wifi->tx_power_report)) { + memcpy(wifi->tx_power_report, packet + 28, sizeof(wifi->tx_power_report)); + wifi->tx_power_report_valid = true; + } + if (command == 0xf0025) { + wifi->pmk_seen = !status && size >= 60; + wifi->pmk_matches = wifi->pmk_seen && + get_unaligned_le16(packet + 26) == 32 && + !memcmp(packet + 28, wifi->expected_pmk, 32); + memzero_explicit(packet + 24, size - 24); + } + } + refill: + post_receive(wifi, ring, wifi->rx_head[ring]); + wifi->rx_head[ring] = (wifi->rx_head[ring] + 1) % ring_size[ring]; + dma_wmb(); + index_write(wifi, 3, ring, wifi->rx_head[ring]); + posted = true; + } + } + completions(wifi); + if (posted) + writel(1, wifi->a.bar + 0x1004); + drain_data_rx(wifi); + drain_data_tx(wifi); +} + +static int command(struct centauri_wifi *wifi, u32 id, const void *payload, size_t size) +{ + u16 slot = wifi->command_head, sequence = ++wifi->sequence; + u8 *packet, *descriptor; + unsigned long deadline = jiffies + msecs_to_jiffies(1500); + + if (size + 24 > BUFFER_SIZE) + return -EMSGSIZE; + if ((slot + 1) % ring_size[1] == index_read(wifi, 1, 1)) + return -ENOSPC; + packet = wifi->a.window + TX_BUFFER + slot * BUFFER_SIZE; + memset(packet, 0, 24 + size); + put_unaligned_le16(24 + size, packet + 0x10); + put_unaligned_le16(sequence, packet + 0x12); + put_unaligned_le32(id, packet + 0x14); + if (size) + memcpy(packet + 24, payload, size); + descriptor = wifi->a.window + TR_BASE + slot * 16; + put_unaligned_le32(1 | ((24 + size) << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + TX_BUFFER + slot * BUFFER_SIZE, descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); + dma_wmb(); + wifi->command_head = (slot + 1) % ring_size[1]; + index_write(wifi, 3, 1, wifi->command_head); + wifi->reply_valid = false; + dma_wmb(); + writel(1, wifi->a.bar + 0x1004); + do { + drain(wifi); + if (readl(wifi->a.bar + 8) != 2) + return -EIO; + /* Bit 30 is more-data-pending, not part of the command identity. + * Scan/connect replies set it before their later completion event. + */ + if (wifi->reply_valid && wifi->reply_sequence == sequence && + (wifi->reply_command & ~BIT(30)) == id) { + if (wifi->reply_status) + dev_info(&wifi->control->pdev->dev, + "Wi-Fi command %#x rejected: firmware_status=%u\n", + id, wifi->reply_status); + return wifi->reply_status ? -EREMOTEIO : 0; + } + usleep_range(500, 1000); + } while (time_before(jiffies, deadline)); + dev_err(&wifi->control->pdev->dev, "Wi-Fi command %#x timed out (rx=%u)\n", id, wifi->rx_seen[2]); + return -ETIMEDOUT; +} + +static int radio_start(struct centauri_wifi *wifi) +{ + u8 start[5 + 4 + 3 * ETH_ALEN] = { 0, 0x0b, 0, 0, 0 }; + size_t start_size = 5; + unsigned int ring, slot; + int ret; + + /* Stock 25G76 startInterface (0x1808e26ec) supplies TLV 385, + * length 18. Firmware 0x5736c0 otherwise generates three unrelated + * random addresses. This diagnostic keeps the sole active link equal + * to the netdev identity; it does not enable multi-link association. + */ + if (station_link_addresses) { + put_unaligned_le16(385, start + 5); + put_unaligned_le16(3 * ETH_ALEN, start + 7); + for (slot = 0; slot < 3; slot++) + ether_addr_copy(start + 9 + slot * ETH_ALEN, wifi->netdev->dev_addr); + start_size = sizeof(start); + } + for (ring = 1; ring <= 4; ring++) { + ret = open_pair(wifi, ring); + if (ret) + return ret; + } + for (ring = 2; ring <= 4; ring++) { + for (slot = 0; slot < ring_size[ring] - 1; slot++) + post_receive(wifi, ring, slot); + wifi->rx_head[ring] = ring_size[ring] - 1; + dma_wmb(); + index_write(wifi, 3, ring, ring_size[ring] - 1); + } + dma_wmb(); + writel(1, wifi->a.bar + 0x1004); + msleep(400); + ret = command(wifi, 0, NULL, 0); + if (!ret) + ret = command(wifi, 1, wifi->netdev->dev_addr, ETH_ALEN); + if (!ret) + ret = command(wifi, 2, start, start_size); + if (ret) + return ret; + msleep(1000); + wifi->configured = true; + dev_info(&wifi->control->pdev->dev, "Wi-Fi command rings ready; interface started\n"); + return 0; +} + +static void scan_worker(struct work_struct *work) +{ + struct centauri_wifi *wifi = container_of(work, struct centauri_wifi, scan_work); + struct cfg80211_scan_info info = {}; + struct cfg80211_scan_request *request; + u8 scan[14] = { 1, 0, 0, 0x80, 0, 2, 0, 0x10, 110, 0, 110, 0, 0, 0 }; + unsigned int chunk; + int ret = 0; + + mutex_lock(&wifi->lock); + for (chunk = 0; chunk < 3 && !ret; chunk++) { + u32 channels = wifi->channels_seen, done = wifi->scans_done; + bool stopped = false; + unsigned long deadline; + + scan[0] = chunk + 1; + ret = command(wifi, 65551, scan, sizeof(scan)); + if (ret) + break; + deadline = jiffies + msecs_to_jiffies(8000); + do { + drain(wifi); + if (readl(wifi->a.bar + 8) != 2) { + ret = -EIO; + break; + } + if (!stopped && wifi->channels_seen - channels >= 8) { + stopped = true; + ret = command(wifi, 65537, NULL, 0); + if (ret) + break; + } + if (stopped && wifi->scans_done > done) + break; + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + if (!ret && wifi->scans_done == done) + ret = -ETIMEDOUT; + } + info.aborted = !!ret; + request = wifi->scan_request; + wifi->scan_request = NULL; + mutex_unlock(&wifi->lock); + dev_info(&wifi->control->pdev->dev, "Wi-Fi scan finished: rc=%d channels=%u beacons=%u\n", + ret, wifi->channels_seen, wifi->beacons_seen); + if (request) + cfg80211_scan_done(request, &info); +} + +static int wifi_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + int ret = 0; + + mutex_lock(&wifi->lock); + if (!wifi->configured) + ret = -ENETDOWN; + else if (wifi->scan_request || wifi->connecting) + ret = -EBUSY; + else { + wifi->scan_request = request; + schedule_work(&wifi->scan_work); + } + mutex_unlock(&wifi->lock); + return ret; +} + +static int rsn_caps_offset(const u8 *body, size_t size) +{ + size_t offset = 6; + u16 count; + unsigned int i; + + for (i = 0; i < 2; i++) { + if (offset + 2 > size) + return -EINVAL; + count = get_unaligned_le16(body + offset); + offset += 2 + 4 * count; + if (offset > size) + return -EINVAL; + } + return offset + 2 <= size ? offset : -EINVAL; +} + +static void append_tlv(u8 *payload, size_t *size, u16 tag, const void *data, u16 length) +{ + put_unaligned_le16(tag, payload + *size); + put_unaligned_le16(length, payload + *size + 2); + memcpy(payload + *size + 4, data, length); + *size += 4 + length; +} + +static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, + struct cfg80211_connect_params *params) +{ + /* Stock 25G76 ACIConnectionAdapter::connectAp (0x1808e94cc) emits a + * 12-byte TLV 376. The newer SDK 27 adapter uses 14 bytes. Preserve the + * unspecified-link masks; all-zero is not the non-MLO default. + */ + static const u8 mld_defaults[12] = { + 1, 0, 1, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 3, + }; + /* Firmware connection-context defaults (aciw 0x511ee8). TLV 0xf913 + * replaces all five values. Sending zeros arms Supp_AddTimer with a + * zero duration and aborts WPA before message 2 can be transmitted. + */ + static const __le32 connection_timeouts[] = { + cpu_to_le32(260), cpu_to_le32(6000), cpu_to_le32(2000), + cpu_to_le32(15000), cpu_to_le32(300), + }; + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + struct cfg80211_bss *bss; + const struct cfg80211_bss_ies *ies; + const u8 *rsn, *ap_rsn; + u8 rsn_body[255], ssid[35], key[66], zero[20] = {}, pmf; + u8 *payload = wifi->connect_payload; + size_t size = 130, rsn_size; + u16 caps = 0; + int ret = 0, offset; + + if (!params->ssid_len || params->ssid_len > 32 || !params->crypto.psk || + params->crypto.n_akm_suites != 1 || params->crypto.akm_suites[0] != WLAN_AKM_SUITE_PSK || + params->crypto.n_ciphers_pairwise != 1 || params->crypto.ciphers_pairwise[0] != WLAN_CIPHER_SUITE_CCMP || + params->crypto.cipher_group != WLAN_CIPHER_SUITE_CCMP || params->ie_len > sizeof(wifi->request_ie)) + return -EOPNOTSUPP; + rsn = cfg80211_find_ie(WLAN_EID_RSN, params->ie, params->ie_len); + if (!rsn || rsn_caps_offset(rsn + 2, rsn[1]) < 0) + return -EINVAL; + mutex_lock(&wifi->lock); + if (!wifi->configured || wifi->connecting || wifi->connected || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + if (!wifi->passphrase_size) { + ret = -ENOKEY; + goto out; + } + bss = cfg80211_get_bss(wiphy, params->channel, params->bssid, + params->ssid, params->ssid_len, IEEE80211_BSS_TYPE_ESS, IEEE80211_PRIVACY_ON); + if (!bss) { + ret = -ENOENT; + goto out; + } + /* Match the measured successful J813 association shape. The firmware + * expects the AP RSN capability bits even with the legal single-AKM STA IE. + */ + rcu_read_lock(); + ies = rcu_dereference(bss->ies); + ap_rsn = ies ? cfg80211_find_ie(WLAN_EID_RSN, ies->data, ies->len) : NULL; + if (ap_rsn) { + offset = rsn_caps_offset(ap_rsn + 2, ap_rsn[1]); + if (offset >= 0) + caps = get_unaligned_le16(ap_rsn + 2 + offset); + } + rcu_read_unlock(); + rsn_size = rsn[1]; + memcpy(rsn_body, rsn + 2, rsn_size); + offset = rsn_caps_offset(rsn_body, rsn_size); + put_unaligned_le16(caps, rsn_body + offset); + pmf = !!(caps & BIT(7)); + memset(payload, 0, sizeof(wifi->connect_payload)); + payload[0] = 1; + memcpy(payload + 1, bss->bssid, ETH_ALEN); + memcpy(wifi->bssid, bss->bssid, ETH_ALEN); + payload[7] = ieee80211_frequency_to_channel(bss->channel->center_freq); + /* AppleChanSpecToACI maps chanspec 0xc000 (5 GHz) to ACI 1; + * 0x4000 (6 GHz) maps to ACI 2. Earlier RE notes swapped these. + */ + payload[8] = bss->channel->band == NL80211_BAND_2GHZ ? 0 : 1; + payload[0x47] = 4; + /* ACIConnectionAdapter::mapRSNCipherSuiteToACIWsec maps CCMP to 4. + * The RSN IE does not replace this fixed cipher field. + */ + put_unaligned_le16(4, payload + 0x48); + put_unaligned_le16(32, payload + 0x4a); + memcpy(payload + 0x4c, params->crypto.psk, 32); + if (verify_pmk) + memcpy(wifi->expected_pmk, params->crypto.psk, 32); + append_tlv(payload, &size, 304, zero, 1); + append_tlv(payload, &size, 63763, connection_timeouts, sizeof(connection_timeouts)); + ssid[0] = 1; + ssid[1] = 4; + ssid[2] = params->ssid_len; + memcpy(ssid + 3, params->ssid, params->ssid_len); + append_tlv(payload, &size, 342, ssid, 3 + params->ssid_len); + append_tlv(payload, &size, WLAN_EID_RSN, rsn_body, rsn_size); + /* The fixed field carries the PMK; TLV 313 carries the passphrase. */ + put_unaligned_le16(wifi->passphrase_size, key); + memcpy(key + 2, wifi->passphrase, wifi->passphrase_size); + append_tlv(payload, &size, 313, key, 2 + wifi->passphrase_size); + memzero_explicit(key, sizeof(key)); + append_tlv(payload, &size, 316, &pmf, 1); + append_tlv(payload, &size, 376, mld_defaults, sizeof(mld_defaults)); + wifi->connect_size = size; + memcpy(wifi->request_ie, params->ie, params->ie_len); + wifi->request_ie_size = params->ie_len; + wifi->connecting = true; + wifi->connect_status = 0; + wifi->associated = wifi->pmk_seen = wifi->pmk_matches = false; + wifi->connect_event = wifi->disconnect_event = false; + cfg80211_put_bss(wiphy, bss); + schedule_work(&wifi->connect_work); +out: + mutex_unlock(&wifi->lock); + return ret; +} + +static void connect_worker(struct work_struct *work) +{ + struct centauri_wifi *wifi = container_of(work, struct centauri_wifi, connect_work); + unsigned long deadline; + u16 slot, status, command_status; + bool checked_pmk = false, probed_eapol = false, queried_link = false; + int ret; + + mutex_lock(&wifi->lock); + slot = wifi->command_head; + ret = command(wifi, 131072, wifi->connect_payload, wifi->connect_size); + command_status = wifi->reply_status; + memzero_explicit(wifi->connect_payload, sizeof(wifi->connect_payload)); + deadline = jiffies + msecs_to_jiffies(15000); + while (!ret && !wifi->connect_event && time_before(jiffies, deadline)) { + drain(wifi); + if (link_diagnostics && wifi->associated && !queried_link) { + unsigned int i; + int link_ret; + + queried_link = true; + link_ret = query_link_status(wifi); + dev_info(&wifi->control->pdev->dev, + "connection link status: rc=%d complete=%d valid=%d count=%u emlsr=%d/%u\n", + link_ret, wifi->connect_event, wifi->link_status_valid, wifi->link_count, + wifi->emlsr_valid, wifi->emlsr_mode); + for (i = 0; i < wifi->link_tlv_count; i++) + dev_info(&wifi->control->pdev->dev, "connection link TLV: tag=%u length=%u\n", + wifi->link_tags[i], wifi->link_lengths[i]); + if (wifi->link_status_valid) + for (i = 0; i < wifi->link_count; i++) + dev_info(&wifi->control->pdev->dev, + "connection link %u: prime=%u channel=%u band=%u\n", + i, wifi->link_prime[i], wifi->link_channel[i], wifi->link_band[i]); + } + if (eapol_after_assoc && wifi->associated && !probed_eapol) { + unsigned int i; + int probe_ret; + + probed_eapol = true; + probe_ret = query_station_stats(wifi); + dev_info(&wifi->control->pdev->dev, + "connection counters: complete=%d rc=%d valid=%d length=%u\n", + wifi->connect_event, probe_ret, wifi->station_stats_valid, wifi->stats_length); + if (!probe_ret && wifi->station_stats_valid) + for (i = 0; i < wifi->stats_length; i += 4) { + u32 value = get_unaligned_le32(wifi->station_stats + i); + + if (value) + dev_info(&wifi->control->pdev->dev, + "station counter %03x=%u\n", i, value); + } + } + if (verify_pmk && wifi->associated && !checked_pmk) { + int key_ret = command(wifi, 0xf0025, wifi->bssid, ETH_ALEN); + + checked_pmk = true; + dev_info(&wifi->control->pdev->dev, + "Wi-Fi firmware key check: rc=%d available=%d matches=%d\n", + key_ret, wifi->pmk_seen, wifi->pmk_matches); + } + if (readl(wifi->a.bar + 8) != 2) + ret = -EIO; + usleep_range(1000, 2000); + } + if (!ret && !wifi->connect_event) + ret = -ETIMEDOUT; + status = !ret && wifi->connect_status == 0 ? WLAN_STATUS_SUCCESS : WLAN_STATUS_UNSPECIFIED_FAILURE; + wifi->connecting = false; + wifi->connected = status == WLAN_STATUS_SUCCESS; + memzero_explicit(wifi->expected_pmk, sizeof(wifi->expected_pmk)); + /* The command and connection completion have consumed the PMK buffer. */ + if (slot < ring_size[1]) + memzero_explicit(wifi->a.window + TX_BUFFER + slot * BUFFER_SIZE, BUFFER_SIZE); + mutex_unlock(&wifi->lock); + dev_info(&wifi->control->pdev->dev, + "Wi-Fi authentication result: rc=%d command_status=%u event_status=%u complete_event=%d\n", + ret, command_status, wifi->connect_status, wifi->connect_event); + cfg80211_connect_result(wifi->netdev, wifi->bssid, + wifi->request_ie, wifi->request_ie_size, NULL, 0, status, GFP_KERNEL); + if (status == WLAN_STATUS_SUCCESS && !READ_ONCE(wifi->stopping)) { + netif_carrier_on(wifi->netdev); + if (wifi->data_probe_done & BIT(5)) + netif_wake_queue(wifi->netdev); + schedule_delayed_work(&wifi->link_work, msecs_to_jiffies(20)); + } +} + +static void link_worker(struct work_struct *work) +{ + struct centauri_wifi *wifi = container_of(to_delayed_work(work), struct centauri_wifi, link_work); + bool disconnected; + + mutex_lock(&wifi->lock); + drain(wifi); + disconnected = wifi->disconnect_event || READ_ONCE(wifi->stopping) || + readl(wifi->a.bar + 8) != 2; + if (disconnected) + wifi->connected = false; + mutex_unlock(&wifi->lock); + if (disconnected) { + netif_stop_queue(wifi->netdev); + skb_queue_purge(&wifi->data_tx_queue); + netif_carrier_off(wifi->netdev); + cfg80211_disconnected(wifi->netdev, WLAN_REASON_UNSPECIFIED, NULL, 0, false, GFP_KERNEL); + } else { + schedule_delayed_work(&wifi->link_work, msecs_to_jiffies(20)); + } +} + +static int wifi_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 reason) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + u8 payload[16] = {}; + bool connected; + int ret; + + mutex_lock(&wifi->lock); + if (wifi->connecting) { + mutex_unlock(&wifi->lock); + return -EBUSY; + } + put_unaligned_le16(reason, payload); + put_unaligned_le16(328, payload + 8); + put_unaligned_le16(4, payload + 10); + ret = command(wifi, 131073, payload, sizeof(payload)); + connected = wifi->connected; + wifi->connected = false; + mutex_unlock(&wifi->lock); + cancel_delayed_work_sync(&wifi->link_work); + netif_stop_queue(netdev); + skb_queue_purge(&wifi->data_tx_queue); + netif_carrier_off(netdev); + if (connected) + cfg80211_disconnected(netdev, reason, NULL, 0, true, GFP_KERNEL); + return ret == -EREMOTEIO && wifi->reply_status == 326 ? 0 : ret; +} + +static int wifi_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, + bool enabled, int timeout) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + u8 mode = enabled ? 0 : 3; + int ret; + + /* ACIIO80211PowerAdapter::setPowerSaveMode, command 7. */ + mutex_lock(&wifi->lock); + ret = wifi->configured ? command(wifi, 7, &mode, sizeof(mode)) : -ENETDOWN; + mutex_unlock(&wifi->lock); + return ret; +} + +static void wifi_regulatory(struct wiphy *wiphy, struct regulatory_request *request) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + u8 country[3] = { request->alpha2[0], request->alpha2[1], 1 }; + int ret; + + /* ACIRegulatoryAdapter::setCountryCode, command 9. Leave world-domain + * handling to cfg80211; do not invent a device country or bypass SAR. + */ + if (country[0] < 'A' || country[0] > 'Z' || country[1] < 'A' || country[1] > 'Z') + return; + mutex_lock(&wifi->lock); + if (wifi->configured) { + ret = command(wifi, 9, country, sizeof(country)); + dev_info(&wifi->control->pdev->dev, + "Wi-Fi country %c%c: rc=%d\n", country[0], country[1], ret); + } + mutex_unlock(&wifi->lock); +} + +static const struct cfg80211_ops wifi_ops = { + .scan = wifi_scan, + .connect = wifi_connect, + .disconnect = wifi_disconnect, + .set_power_mgmt = wifi_power_mgmt, +}; + +static int net_open(struct net_device *netdev) +{ + netif_carrier_off(netdev); + netif_stop_queue(netdev); + return 0; +} + +static int net_stop(struct net_device *netdev) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)netdev_priv(netdev); + + netif_stop_queue(netdev); + skb_queue_purge(&wifi->data_tx_queue); + return 0; +} + +static netdev_tx_t net_transmit(struct sk_buff *skb, struct net_device *netdev) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)netdev_priv(netdev); + + if (READ_ONCE(wifi->stopping) || !READ_ONCE(wifi->connected) || + skb->len < ETH_HLEN || skb->len > BUFFER_SIZE) { + netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + if (skb_queue_len(&wifi->data_tx_queue) >= DATA_TX_QUEUE_LIMIT) { + netif_stop_queue(netdev); + return NETDEV_TX_BUSY; + } + skb_queue_tail(&wifi->data_tx_queue, skb); + if (skb_queue_len(&wifi->data_tx_queue) >= DATA_TX_QUEUE_LIMIT) + netif_stop_queue(netdev); + mod_delayed_work(system_wq, &wifi->link_work, 0); + return NETDEV_TX_OK; +} + +static const struct net_device_ops net_ops = { + .ndo_open = net_open, + .ndo_stop = net_stop, + .ndo_start_xmit = net_transmit, + .ndo_validate_addr = eth_validate_addr, +}; + +static ssize_t passphrase_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + u8 value[64]; + size_t length = count; + int ret = 0; + + if (count < 8 || count > sizeof(value)) + return -EINVAL; + if (copy_from_user(value, input, count)) { + ret = -EFAULT; + goto out; + } + if (value[length - 1] == '\n') + length--; + if (length < 8 || length > 63) { + ret = -EINVAL; + goto out; + } + mutex_lock(&wifi->lock); + if (wifi->connecting) { + ret = -EBUSY; + } else { + memzero_explicit(wifi->passphrase, sizeof(wifi->passphrase)); + memcpy(wifi->passphrase, value, length); + wifi->passphrase_size = length; + } + mutex_unlock(&wifi->lock); +out: + memzero_explicit(value, sizeof(value)); + return ret ?: count; +} + +static const struct file_operations passphrase_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = passphrase_write, +}; + +/* Airship direct_memregion::map (0x1802864e0) sends opcode 8, a region ID, + * byte size and a 64-bit DVA in a 24-byte header. Alpha's region-debug is ID 1. + * Firmware 0x175340 passes those fields to DramPhyService_handle_buffer; + * InitDramLoggerMgr writes a 0x158-byte control header to the supplied buffer. + * This is distinct from the fw_debug transfer ring. Never print log payloads: + * firmware diagnostics may contain private connection material. + */ +static ssize_t fwlog_start_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + u8 message[24] = { 8, 0, 1, 0 }; + unsigned int action; + unsigned long deadline; + int ret; + + ret = kstrtouint_from_user(input, count, 0, &action); + if (ret || action != 1) + return ret ?: -EINVAL; + mutex_lock(&wifi->lock); + mutex_lock(&wifi->fwlog_lock); + if (!wifi->configured || wifi->connecting || wifi->connected || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + if (wifi->fwlog_attempted) { + ret = -EALREADY; + goto out; + } + ret = centauri_alpha_log_buffer(wifi->control, &wifi->fwlog_buffer, + &wifi->fwlog_dma, &wifi->fwlog_size); + if (ret) + goto out; + put_unaligned_le32(wifi->fwlog_size, message + 4); + put_unaligned_le64(wifi->fwlog_dma, message + 8); + wifi->fwlog_attempted = true; + ret = alpha_message(wifi, message, sizeof(message)); + if (ret) + goto report; + deadline = jiffies + msecs_to_jiffies(2000); + do { + if (readl(wifi->a.bar + 8) != 2) { + ret = -EIO; + goto report; + } + dma_rmb(); + if (get_unaligned_le16(wifi->fwlog_buffer) == 0x201) { + wifi->fwlog_ready = true; + goto report; + } + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + ret = -ETIMEDOUT; +report: + dev_info(&wifi->control->pdev->dev, + "Wi-Fi DRAM logger: rc=%d initialized=%u stage=%u\n", + ret, wifi->fwlog_ready, readl(wifi->a.bar + 8)); +out: + mutex_unlock(&wifi->fwlog_lock); + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static const struct file_operations fwlog_start_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = fwlog_start_write, +}; + +static ssize_t fwlog_status_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + char text[320]; + u8 header[0xb4] = { 0 }; + int length; + + mutex_lock(&wifi->fwlog_lock); + if (wifi->fwlog_buffer) { + dma_rmb(); + memcpy(header, wifi->fwlog_buffer, sizeof(header)); + } + length = scnprintf(text, sizeof(text), + "attempted=%u initialized=%u bytes=%zu version=%#x blocks=%u block_size=%u mask=%#x data_offset=%u consumer=%u producer=%u header14=%u header18=%u header1c=%u connecting=%u scanning=%u\n", + wifi->fwlog_attempted, wifi->fwlog_ready, wifi->fwlog_size, + get_unaligned_le16(header), get_unaligned_le16(header + 2), + get_unaligned_le32(header + 4), get_unaligned_le32(header + 8), + get_unaligned_le32(header + 12), get_unaligned_le32(header + 16), + get_unaligned_le32(header + 0xb0), + get_unaligned_le32(header + 20), get_unaligned_le32(header + 24), + get_unaligned_le32(header + 28), + !!READ_ONCE(wifi->connecting), !!READ_ONCE(wifi->scan_request)); + mutex_unlock(&wifi->fwlog_lock); + return simple_read_from_buffer(output, count, position, text, length); +} + +static const struct file_operations fwlog_status_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = fwlog_status_read, + .llseek = default_llseek, +}; + +/* The exact firmware's 0x144cfc compares header +0xb0 (producer) against + * +0x10 (host consumer). InitDramLoggerMgr registers a GDMA read of +0x10. + * Only acknowledge records after a private snapshot has been verified. + * Leave three entries reserved for the firmware's three DMA descriptors + * until completion/record sequencing is independently decoded. + */ +static ssize_t fwlog_ack_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + u8 *header; + u32 requested, consumer, producer, pending; + u16 blocks; + int ret; + + ret = kstrtou32_from_user(input, count, 0, &requested); + if (ret) + return ret; + mutex_lock(&wifi->fwlog_lock); + if (!wifi->fwlog_ready || readl(wifi->a.bar + 8) != 2) { + ret = -ENETDOWN; + goto out; + } + header = wifi->fwlog_buffer; + dma_rmb(); + blocks = get_unaligned_le16(header + 2); + consumer = get_unaligned_le32(header + 0x10); + producer = get_unaligned_le32(header + 0xb0); + pending = producer - consumer; + if (get_unaligned_le16(header) != 0x201 || blocks < 8 || + pending > blocks || pending < 3 || requested - consumer > pending - 3) { + ret = -EINVAL; + goto out; + } + WRITE_ONCE(*(__le32 *)(header + 0x10), cpu_to_le32(requested)); + dma_wmb(); + ret = 0; +out: + mutex_unlock(&wifi->fwlog_lock); + return ret ?: count; +} + +static const struct file_operations fwlog_ack_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = fwlog_ack_write, +}; + +/* Root-only binary diagnostics. Save directly to a protected artifact, never + * the console or a text log. The device may update blocks during a read. + */ +static ssize_t fwlog_buffer_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + ssize_t ret; + + mutex_lock(&wifi->fwlog_lock); + if (!wifi->fwlog_ready) { + ret = -ENODATA; + } else { + dma_rmb(); + ret = simple_read_from_buffer(output, count, position, + wifi->fwlog_buffer, wifi->fwlog_size); + } + mutex_unlock(&wifi->fwlog_lock); + return ret; +} + +static const struct file_operations fwlog_buffer_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = fwlog_buffer_read, + .llseek = default_llseek, +}; + +/* The hardware data-ring callback requires the ten-byte WLAN clientdata + * extension after the 44-byte ACIPC header. Without it UMAC asserts at + * 0x11760e. Keep these empty-ring probes explicit until packet DMA is proven. + * The reserved regions below do not overlap control rings or scan buffers. + */ +static ssize_t data_probe_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + u8 message[0x34 + 10] = { 0 }; + unsigned int id, vector, offset, size, bit, doorbell, irq; + bool transfer = false; + int ret; + + ret = kstrtouint_from_user(input, count, 0, &id); + if (ret) + return ret; + if (id == 76) { + vector = 192; + offset = 0x200000; + size = 2048; + bit = 0; + doorbell = 18; + irq = 3; + } else if (id == 14) { + vector = 206; + offset = 0x220000; + size = 256; + bit = 1; + doorbell = 11; + irq = 8; + } else if (id == 15) { + vector = 207; + offset = 0x240000; + size = 128; + bit = 2; + doorbell = 11; + irq = 8; + } else if (id == 24 || id == 25 || id == 12) { + transfer = true; + vector = id; + offset = id == 12 ? 0x520000 : id == 24 ? 0x500000 : 0x540000; + size = 128; + bit = id == 12 ? 4 : id == 24 ? 3 : 5; + doorbell = id == 12 ? 11 : 18; + irq = id == 12 ? 8 : 3; + } else { + return -EINVAL; + } + mutex_lock(&wifi->lock); + if (readl(wifi->a.bar + 8) != 2) { + ret = -EIO; + goto out; + } + if (wifi->connecting || wifi->scan_request || + (wifi->data_probe_done & BIT(bit))) { + ret = -EBUSY; + goto out; + } + wifi->data_probe_done |= BIT(bit); + memset(wifi->a.window + offset, 0, size * 32); + index_write(wifi, transfer ? 1 : 0, vector, 0); + index_write(wifi, transfer ? 3 : 2, vector, 0); + message[0] = transfer ? 1 : 2; + message[2] = transfer && id == 12 ? 0 : 4; + put_unaligned_le16(id, message + 4); + put_unaligned_le16(vector, message + 6); + put_unaligned_le64(wifi->a.window_dma + offset, message + 8); + if (transfer) { + put_unaligned_le64(~0ULL, message + 0x10); + put_unaligned_le16(size, message + 0x18); + put_unaligned_le16(id == 12 ? 12 : 76, message + 0x1a); + put_unaligned_le16(doorbell, message + 0x1c); + put_unaligned_le16(id == 12 ? 0x220 : 0, message + 0x1e); + put_unaligned_le32(16, message + 0x20); + put_unaligned_le16(irq, message + 0x24); + } else { + put_unaligned_le16(size, message + 0x10); + put_unaligned_le16(id == 15 ? 12 : 0xffff, message + 0x12); + put_unaligned_le16(doorbell, message + 0x14); + put_unaligned_le32(1, message + 0x18); + put_unaligned_le16(irq, message + 0x1c); + } + /* AirshipDataTxTransferRing::initWithOptions maps the host MCBC + * parameters to version 0, hardware interface 0, TID 6, AC 3, and + * six broadcast destination bytes. The default probe is unchanged. + */ + if (id == 25 && data_tx_broadcast) { + message[0x34 + 2] = 6; + message[0x34 + 3] = 3; + eth_broadcast_addr(message + 0x34 + 4); + } + ret = alpha_message(wifi, message, (transfer ? 0x34 : 0x2c) + 10); + msleep(1000); + dev_info(&wifi->control->pdev->dev, + "data %s probe id=%u index=%u extension=10 rc=%d stage=%u head=%u tail=%u\n", + transfer ? "TR" : "CR", id, vector, ret, readl(wifi->a.bar + 8), + index_read(wifi, transfer ? 3 : 0, vector), + index_read(wifi, transfer ? 1 : 2, vector)); + if (!ret && readl(wifi->a.bar + 8) != 2) + ret = -EIO; + if (!ret && transfer && id == 12) { + unsigned int slot; + + /* Give the receive engine bounded, individually identifiable buffers. + * No recycling or network delivery until the completion ABI is proven. + */ + for (slot = 0; slot < size - 1; slot++) { + u8 *descriptor = wifi->a.window + offset + slot * 16; + u32 buffer = DATA_RX_BUFFER + slot * 4096; + + memset(wifi->a.window + buffer, 0, 4096); + put_unaligned_le32(1 | (4096 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); + } + dma_wmb(); + index_write(wifi, 3, vector, size - 1); + dma_wmb(); + writel(1, wifi->a.bar + 0x1050); + msleep(100); + dev_info(&wifi->control->pdev->dev, + "data RX buffers posted=%u stage=%u tr_tail=%u cr_head=%u\n", + size - 1, readl(wifi->a.bar + 8), index_read(wifi, 1, 12), + index_read(wifi, 0, 207)); + } +out: + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static int transmit_eapol_start(struct centauri_wifi *wifi) +{ + u16 head = index_read(wifi, 3, 24); + u8 *descriptor = wifi->a.window + 0x500000 + head * 32; + u8 *packet = wifi->a.window + 0xf00000 + head * 64; + u64 metadata = 0; + unsigned int attempt; + + if (readl(wifi->a.bar + 8) != 2 || !(wifi->data_probe_done & BIT(3))) + return -ENETDOWN; + /* Eight independent buffers bound the probe without ring reuse. */ + if (head >= 8 || index_read(wifi, 1, 24) != head) + return -EBUSY; + if (eapol_unicast && (!wifi->associated || !is_valid_ether_addr(wifi->bssid))) + return -ENOTCONN; + /* A standards-defined EAPOL-Start, with no credential/key material. + * Descriptor flags 3 mean a DMA buffer plus the 16-byte WLAN footer. + * Interface zero and default TX policy have a zero footer. + */ + memset(packet, 0, 64); + if (eapol_unicast) + memcpy(packet, wifi->bssid, ETH_ALEN); + else + memcpy(packet, "\x01\x80\xc2\x00\x00\x03", ETH_ALEN); + memcpy(packet + ETH_ALEN, wifi->netdev->dev_addr, ETH_ALEN); + put_unaligned_be16(ETH_P_PAE, packet + 12); + packet[14] = 2; + packet[15] = 1; + memset(descriptor, 0, 32); + put_unaligned_le32(3 | (18 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + 0xf00000 + head * 64, descriptor + 4); + put_unaligned_le16(head + 1, descriptor + 12); + if (eapol_control_profile) + metadata |= BIT_ULL(60) | BIT_ULL(48); + if (eapol_relative_lifetime) { + /* AirshipDataPacket::setRelativeExpirationTime uses mode 6 in + * bits 29:27 and microseconds in the second metadata word. + */ + metadata |= 6ULL << 27; + put_unaligned_le64(2000000, descriptor + 24); + } + put_unaligned_le64(metadata, descriptor + 16); + dma_wmb(); + index_write(wifi, 3, 24, head + 1); + dma_wmb(); + writel(1, wifi->a.bar + 0x1030); + for (attempt = 0; attempt < 20; attempt++) { + if (index_read(wifi, 1, 24) == head + 1 || readl(wifi->a.bar + 8) != 2) + break; + usleep_range(5000, 6000); + } + dev_info(&wifi->control->pdev->dev, + "EAPOL-Start DMA probe: slot=%u associated=%d unicast=%d relative_expiry=%d stage=%u tr_tail=%u cr_head=%u\n", + head, wifi->associated, eapol_unicast, eapol_relative_lifetime, readl(wifi->a.bar + 8), + index_read(wifi, 1, 24), index_read(wifi, 0, 192)); + return readl(wifi->a.bar + 8) == 2 ? 0 : -EIO; +} + +static ssize_t eapol_start_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + bool enabled; + int ret; + + ret = kstrtobool_from_user(input, count, &enabled); + if (ret || !enabled) + return ret ?: -EINVAL; + mutex_lock(&wifi->lock); + if (wifi->connecting || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + ret = transmit_eapol_start(wifi); +out: + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static const struct file_operations eapol_start_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = eapol_start_write, +}; + +static const struct file_operations data_probe_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = data_probe_write, +}; + +static ssize_t data_status_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + char status[1536]; + unsigned int i, length; + + mutex_lock(&wifi->lock); + dma_rmb(); + length = scnprintf(status, sizeof(status), + "stage=%u tx_cr=%u tx_tr=%u/%u rx_cr=%u rx_tr=%u/%u received=%u reposted=%u bad=%u samples=%u normal_tx_tr=%u/%u submitted=%u completed=%u failed=%u queued=%u coalesced=%u tx_bad=%u\n", + readl(wifi->a.bar + 8), index_read(wifi, 0, 192), + index_read(wifi, 1, 24), index_read(wifi, 3, 24), + index_read(wifi, 0, 207), index_read(wifi, 1, 12), + index_read(wifi, 3, 12), wifi->data_rx_received, + wifi->data_rx_reposted, wifi->data_rx_bad, + le32_to_cpu(wifi->rx_snapshot.count), + index_read(wifi, 1, 25), index_read(wifi, 3, 25), + wifi->data_tx_submitted, wifi->data_tx_completed, + wifi->data_tx_failed, skb_queue_len(&wifi->data_tx_queue), + wifi->data_tx_coalesced, wifi->data_tx_bad); + for (i = 0; i < 8; i++) { + u8 *entry = wifi->a.window + 0x240000 + i * 32; + + length += scnprintf(status + length, sizeof(status) - length, + "rx_completion[%u]=%08x %08x %08x %08x %08x %08x %08x %08x\n", + i, get_unaligned_le32(entry), get_unaligned_le32(entry + 4), + get_unaligned_le32(entry + 8), get_unaligned_le32(entry + 12), + get_unaligned_le32(entry + 16), get_unaligned_le32(entry + 20), + get_unaligned_le32(entry + 24), get_unaligned_le32(entry + 28)); + } + for (i = 0; i < 8; i++) { + u8 *entry = wifi->a.window + 0x200000 + i * 32; + + length += scnprintf(status + length, sizeof(status) - length, + "tx_completion[%u]=%08x %08x %08x %08x %08x %08x %08x %08x\n", + i, get_unaligned_le32(entry), get_unaligned_le32(entry + 4), + get_unaligned_le32(entry + 8), get_unaligned_le32(entry + 12), + get_unaligned_le32(entry + 16), get_unaligned_le32(entry + 20), + get_unaligned_le32(entry + 24), get_unaligned_le32(entry + 28)); + } + mutex_unlock(&wifi->lock); + return simple_read_from_buffer(output, count, position, status, length); +} + +static const struct file_operations data_status_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = data_status_read, +}; + +/* Contains private network traffic: retrieve only as a protected binary file. + * Samples are copied once into host memory and never change after count 8. + */ +static ssize_t data_rx_snapshot_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + ssize_t ret; + + mutex_lock(&wifi->lock); + ret = simple_read_from_buffer(output, count, position, &wifi->rx_snapshot, + sizeof(wifi->rx_snapshot)); + mutex_unlock(&wifi->lock); + return ret; +} + +static const struct file_operations data_rx_snapshot_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = data_rx_snapshot_read, + .llseek = default_llseek, +}; + +static ssize_t country_status_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + char status[96]; + unsigned int length; + int ret; + + mutex_lock(&wifi->lock); + wifi->country_valid = false; + ret = wifi->configured ? command(wifi, 10, NULL, 0) : -ENETDOWN; + length = scnprintf(status, sizeof(status), "rc=%d valid=%d country_bytes=%02x/%02x/%02x\n", + ret, wifi->country_valid, wifi->country[0], wifi->country[1], wifi->country[2]); + mutex_unlock(&wifi->lock); + return simple_read_from_buffer(output, count, position, status, length); +} + +static const struct file_operations country_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = country_status_read, +}; + +static int query_link_status(struct centauri_wifi *wifi) +{ + wifi->link_status_valid = wifi->emlsr_valid = false; + wifi->link_count = wifi->link_status_size = 0; + wifi->link_tlv_count = 0; + return wifi->configured ? command(wifi, 20, NULL, 0) : -ENETDOWN; +} + +static int link_status_show(struct seq_file *seq, void *unused) +{ + struct centauri_wifi *wifi = seq->private; + unsigned int i; + int ret; + + mutex_lock(&wifi->lock); + ret = query_link_status(wifi); + seq_printf(seq, "rc=%d valid=%d size=%u count=%u emlsr=%d/%u\n", ret, + wifi->link_status_valid, wifi->link_status_size, wifi->link_count, + wifi->emlsr_valid, wifi->emlsr_mode); + for (i = 0; i < wifi->link_tlv_count; i++) + seq_printf(seq, "tag=%u length=%u\n", wifi->link_tags[i], wifi->link_lengths[i]); + if (wifi->link_status_valid) + for (i = 0; i < wifi->link_count; i++) + seq_printf(seq, "link=%u prime=%u channel=%u band=%u\n", i, + wifi->link_prime[i], wifi->link_channel[i], wifi->link_band[i]); + mutex_unlock(&wifi->lock); + return 0; +} + +static int link_status_open(struct inode *inode, struct file *file) +{ + return single_open(file, link_status_show, inode->i_private); +} + +static const struct file_operations link_status_ops = { + .owner = THIS_MODULE, + .open = link_status_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int tx_config_show(struct seq_file *seq, void *unused) +{ + struct centauri_wifi *wifi = seq->private; + u8 id; + int ret; + + mutex_lock(&wifi->lock); + seq_printf(seq, "applied=%u\n", wifi->tx_defaults_applied); + for (id = 0; id <= 1; id++) { + wifi->tx_profile_valid = false; + ret = wifi->configured ? command(wifi, 56, &id, sizeof(id)) : -ENETDOWN; + seq_printf(seq, "profile=%u rc=%d status=%u valid=%d", id, ret, + wifi->reply_status, wifi->tx_profile_valid); + if (wifi->tx_profile_valid) + seq_printf(seq, " rate=%#x rts_threshold=%u rts_retries=%u retries=%u type=%u id=%u", + get_unaligned_le32(wifi->tx_profile), + get_unaligned_le16(wifi->tx_profile + 4), wifi->tx_profile[6], + wifi->tx_profile[7], wifi->tx_profile[8], wifi->tx_profile[9]); + seq_putc(seq, '\n'); + } + mutex_unlock(&wifi->lock); + return 0; +} + +static int tx_config_open(struct inode *inode, struct file *file) +{ + return single_open(file, tx_config_show, inode->i_private); +} + +static ssize_t tx_config_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = ((struct seq_file *)file->private_data)->private; + /* Exact configureTxDataCtrlPacketProfile: command 52, TLV 340/10. + * 0xa0154 is the TLV header, not the command ID. + */ + static const u8 control_profile[] = { + 0x54, 1, 10, 0, 0, 8, 0, 0, 0xff, 0xff, 0xff, 0xff, 1, 1, + }; + u8 lifetime[20] = {}; + unsigned int action, i; + int ret; + + ret = kstrtouint_from_user(input, count, 0, &action); + if (ret || (action != 1 && action != 2)) + return ret ?: -EINVAL; + mutex_lock(&wifi->lock); + if (!wifi->configured || wifi->connecting || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + if (action == 1) { + /* AppleWLAN80211STA::resetTxPacketLifetime selects 2000 ms for + * each AC. The ACI adapter converts to firmware microseconds. + */ + for (i = 0; i < 4; i++) + put_unaligned_le32(2000000, lifetime + 4 + 4 * i); + ret = command(wifi, 43, lifetime, sizeof(lifetime)); + } else { + ret = command(wifi, 52, control_profile, sizeof(control_profile)); + } + if (!ret) + wifi->tx_defaults_applied |= BIT(action - 1); + dev_info(&wifi->control->pdev->dev, "reference TX setup action=%u rc=%d\n", action, ret); +out: + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static const struct file_operations tx_config_ops = { + .owner = THIS_MODULE, + .open = tx_config_open, + .read = seq_read, + .write = tx_config_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static ssize_t host_wakeup_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + /* ACIIO80211PowerAdapter::hostWakeup selects state 0 in command 6. + * This firmware's SysCmdSetHostPowerState accepts the fixed 27-byte + * request without TLVs. Keep its existing event filter: the newer host + * adapter's 92-byte event list exceeds this firmware's 86-byte limit. + */ + const u8 payload[3] = {}; + unsigned int action; + int ret; + + ret = kstrtouint_from_user(input, count, 0, &action); + if (ret || action != 1) + return ret ?: -EINVAL; + mutex_lock(&wifi->lock); + if (!wifi->configured || wifi->connecting || wifi->scan_request || wifi->connected) { + ret = -EBUSY; + goto out; + } + ret = command(wifi, 6, payload, sizeof(payload)); + dev_info(&wifi->control->pdev->dev, "host wakeup state=0 rc=%d status=%u stage=%u\n", + ret, wifi->reply_status, readl(wifi->a.bar + 8)); +out: + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static const struct file_operations host_wakeup_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = host_wakeup_write, +}; + +static int query_station_stats(struct centauri_wifi *wifi) +{ + u8 payload[22] = {}; + int ret; + + /* ACIDebugAdapter::queryStaCounterStats: statistic bit 37, all peers. + * Its fixed counter record contains no key or packet payload bytes. + */ + put_unaligned_le64(BIT_ULL(37), payload); + payload[8] = 1; + put_unaligned_le16(363, payload + 9); + put_unaligned_le16(9, payload + 11); + wifi->station_stats_valid = false; + wifi->stats_reply_size = wifi->stats_tag = wifi->stats_length = 0; + ret = wifi->configured ? command(wifi, 0xb0001, payload, sizeof(payload)) : -ENETDOWN; + if (!ret) { + usleep_range(1000, 2000); + drain(wifi); + } + return ret; +} + +static int station_stats_show(struct seq_file *seq, void *unused) +{ + struct centauri_wifi *wifi = seq->private; + unsigned int i; + int ret; + + mutex_lock(&wifi->lock); + ret = query_station_stats(wifi); + seq_printf(seq, "rc=%d valid=%d size=%u tag=%#x length=%u\n", ret, + wifi->station_stats_valid, wifi->stats_reply_size, wifi->stats_tag, wifi->stats_length); + if (!ret && wifi->station_stats_valid) + for (i = 0; i < wifi->stats_length; i += 4) { + u32 value = get_unaligned_le32(wifi->station_stats + i); + + if (value) + seq_printf(seq, "%03x %u\n", i, value); + } + mutex_unlock(&wifi->lock); + return 0; +} + +static int station_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, station_stats_show, inode->i_private); +} + +static const struct file_operations station_stats_ops = { + .owner = THIS_MODULE, + .open = station_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int rx_crypto_stats_show(struct seq_file *seq, void *unused) +{ + /* ACIStatisticsAdapter::printRxChipCounterStats at 0x1809d7b80: + * statistic bit 30, response TLV 0xf921, sixteen LE32 counters. + * Unlike the station record, this fixed structure contains no MACs. + */ + static const char * const names[] = { + "rx_duplicate_errors", "rx_ampdu_duplicate_errors", + "rx_unicast_tkip_replay_errors", "rx_unicast_tkip_decrypt_errors", + "rx_multicast_tkip_replay_errors", "rx_multicast_tkip_decrypt_errors", + "rx_unicast_ccmp_replay_errors", "rx_unicast_ccmp_decrypt_errors", + "rx_multicast_ccmp_replay_errors", "rx_multicast_ccmp_decrypt_errors", + "phy_total_detect_count", "phy_ofdm_detect_count", "phy_11b_detect_count", + "phy_plcp_error_count", "phy_total_receive_count", "phy_plcp_good_count", + }; + struct centauri_wifi *wifi = seq->private; + u8 payload[22] = {}; + unsigned int i; + int ret; + + put_unaligned_le64(BIT_ULL(30), payload); + payload[8] = 1; + put_unaligned_le16(363, payload + 9); + put_unaligned_le16(9, payload + 11); + mutex_lock(&wifi->lock); + wifi->rx_crypto_stats_valid = false; + wifi->stats_reply_size = wifi->stats_tag = wifi->stats_length = 0; + ret = wifi->configured ? command(wifi, 0xb0001, payload, sizeof(payload)) : -ENETDOWN; + if (!ret) { + usleep_range(1000, 2000); + drain(wifi); + } + seq_printf(seq, "rc=%d valid=%d size=%u tag=%#x length=%u\n", ret, + wifi->rx_crypto_stats_valid, wifi->stats_reply_size, wifi->stats_tag, wifi->stats_length); + if (!ret && wifi->rx_crypto_stats_valid) + for (i = 0; i < ARRAY_SIZE(names); i++) + seq_printf(seq, "%s=%u\n", names[i], + get_unaligned_le32(wifi->rx_crypto_stats + i * 4)); + mutex_unlock(&wifi->lock); + return 0; +} + +static int rx_crypto_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, rx_crypto_stats_show, inode->i_private); +} + +static const struct file_operations rx_crypto_stats_ops = { + .owner = THIS_MODULE, + .open = rx_crypto_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int dot11_mib_stats_show(struct seq_file *seq, void *unused) +{ + /* ACIStatisticsAdapter::printDot11MibStats at 0x1809d7fb0: + * statistic bit 25, scope 2/interface 0, TLV 0xf91b, 24 LE64s. + * Keep the host field names: tx_status_success is firmware accounting, + * not independent proof of over-the-air acknowledgement or AP delivery. + */ + static const char * const names[] = { + "rx_msdu_pushed", "rx_msdu_pushed_multi", "rx_mpdu", "rx_mpdu_dup", + "rx_ampdu_dup", "rx_mpdu_drop_error", "rx_msdu_drop_filter", + "rx_holes_0", "rx_holes_1", "rx_holes_2", "rx_holes_3", "rx_holes_4", + "rx_holes_5", "rx_holes_6", "rx_holes_7", "rx_holes_8", + "tx_pulled", "tx_pulled_multi", "tx_pulled_core", "tx_status_success", + "tx_status_abort_request", "tx_status_peer_gone", + "tx_status_retries_exhausted", "tx_status_lifetime_expired", + }; + struct centauri_wifi *wifi = seq->private; + u8 payload[22] = {}; + unsigned int i; + int ret; + + put_unaligned_le64(BIT_ULL(25), payload); + payload[8] = 1; + put_unaligned_le16(363, payload + 9); + put_unaligned_le16(9, payload + 11); + /* The scope byte follows the eight-byte interface value in TLV 363. */ + payload[21] = 2; + mutex_lock(&wifi->lock); + wifi->dot11_mib_stats_valid = false; + wifi->stats_reply_size = wifi->stats_tag = wifi->stats_length = 0; + ret = wifi->configured ? command(wifi, 0xb0001, payload, sizeof(payload)) : -ENETDOWN; + if (!ret) { + usleep_range(1000, 2000); + drain(wifi); + } + seq_printf(seq, "rc=%d valid=%d size=%u tag=%#x length=%u\n", ret, + wifi->dot11_mib_stats_valid, wifi->stats_reply_size, wifi->stats_tag, wifi->stats_length); + if (!ret && wifi->dot11_mib_stats_valid) + for (i = 0; i < ARRAY_SIZE(names); i++) + seq_printf(seq, "%s=%llu\n", names[i], + (unsigned long long)get_unaligned_le64(wifi->dot11_mib_stats + i * 8)); + mutex_unlock(&wifi->lock); + return 0; +} + +static int dot11_mib_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, dot11_mib_stats_show, inode->i_private); +} + +static const struct file_operations dot11_mib_stats_ops = { + .owner = THIS_MODULE, + .open = dot11_mib_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int tx_power_report_show(struct seq_file *seq, void *unused) +{ + /* Exact stock 25G76 WLANACIAdapter::queryLastTxStats, 0x180914d58: + * bit29, scope1/radio bitmap, TLV 0xf920 with 280 bytes. The newer + * macOS27 adapter expects 320 bytes and has a different record stride. + * Stock getLastTxInfo (0x1809150f8) reads the first five 28-byte records. + * These are rate/power observations, not proof of ACK or AP delivery. + */ + struct centauri_wifi *wifi = seq->private; + u8 payload[22] = {}; + unsigned int radio, i; + int ret; + + put_unaligned_le64(BIT_ULL(29), payload); + payload[8] = 1; + put_unaligned_le16(363, payload + 9); + put_unaligned_le16(9, payload + 11); + payload[21] = 1; + mutex_lock(&wifi->lock); + for (radio = 1; radio <= 2; radio++) { + put_unaligned_le64(radio, payload + 13); + wifi->tx_power_report_valid = false; + wifi->stats_reply_size = wifi->stats_tag = wifi->stats_length = 0; + ret = wifi->configured ? command(wifi, 0xb0001, payload, sizeof(payload)) : -ENETDOWN; + if (!ret) { + usleep_range(1000, 2000); + drain(wifi); + } + seq_printf(seq, "radio_bitmap=%u rc=%d valid=%d size=%u tag=%#x length=%u\n", + radio, ret, wifi->tx_power_report_valid, wifi->stats_reply_size, + wifi->stats_tag, wifi->stats_length); + if (ret || !wifi->tx_power_report_valid) + continue; + for (i = 0; i < 5; i++) { + const u8 *record = wifi->tx_power_report + i * 28; + + /* Keep hardware enums and power bytes raw until interpreted + * against this exact firmware; omit unreviewed record fields. + */ + seq_printf(seq, "sample=%u power0_qdb_raw=%u power1_qdb_raw=%u power_qdb_raw=%u " + "hw_preamble=%u hw_rate=%u nss=%u hw_bw=%u hw_gi=%u\n", + i, record[14], record[15], record[16], record[17], + record[20], record[21] + 1, record[24], record[27]); + } + } + mutex_unlock(&wifi->lock); + return 0; +} + +static int tx_power_report_open(struct inode *inode, struct file *file) +{ + return single_open(file, tx_power_report_show, inode->i_private); +} + +static const struct file_operations tx_power_report_ops = { + .owner = THIS_MODULE, + .open = tx_power_report_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +int centauri_wifi_register(struct centauri *c) +{ + struct centauri_wifi *wifi; + unsigned int i; + int ret; + + wifi = kzalloc(sizeof(*wifi), GFP_KERNEL); + if (!wifi) + return -ENOMEM; + c->wifi = wifi; + wifi->control = c; + mutex_init(&wifi->lock); + mutex_init(&wifi->fwlog_lock); + skb_queue_head_init(&wifi->data_tx_queue); + INIT_WORK(&wifi->scan_work, scan_worker); + INIT_WORK(&wifi->connect_work, connect_worker); + INIT_DELAYED_WORK(&wifi->link_work, link_worker); + ret = centauri_alpha_resources(c, &wifi->a); + if (ret) + goto failed; + wifi->wiphy = wiphy_new(&wifi_ops, sizeof(struct centauri_wifi *)); + wifi->netdev = alloc_etherdev(sizeof(struct centauri_wifi *)); + if (!wifi->wiphy || !wifi->netdev) { + ret = -ENOMEM; + goto failed; + } + *(struct centauri_wifi **)wiphy_priv(wifi->wiphy) = wifi; + *(struct centauri_wifi **)netdev_priv(wifi->netdev) = wifi; + set_wiphy_dev(wifi->wiphy, &c->pdev->dev); + wifi->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION); + wifi->wiphy->max_scan_ssids = 1; + wifi->wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM; + wifi->wiphy->reg_notifier = wifi_regulatory; + wifi->wiphy->cipher_suites = cipher_suites; + wifi->wiphy->n_cipher_suites = ARRAY_SIZE(cipher_suites); + wiphy_ext_feature_set(wifi->wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); + for (i = 0; i < 13; i++) { + wifi->channels[i].band = NL80211_BAND_2GHZ; + wifi->channels[i].hw_value = i + 1; + wifi->channels[i].center_freq = ieee80211_channel_to_frequency(i + 1, NL80211_BAND_2GHZ); + wifi->channels[i].max_power = 20; + } + for (i = 0; i < ARRAY_SIZE(five_ghz); i++) { + wifi->channels[13 + i].band = NL80211_BAND_5GHZ; + wifi->channels[13 + i].hw_value = five_ghz[i]; + wifi->channels[13 + i].center_freq = ieee80211_channel_to_frequency(five_ghz[i], NL80211_BAND_5GHZ); + wifi->channels[13 + i].max_power = 20; + } + wifi->bands[0].channels = wifi->channels; + wifi->bands[0].n_channels = 13; + wifi->bands[0].bitrates = rates; + wifi->bands[0].n_bitrates = ARRAY_SIZE(rates); + wifi->bands[1].channels = wifi->channels + 13; + wifi->bands[1].n_channels = ARRAY_SIZE(five_ghz); + wifi->bands[1].bitrates = rates + 4; + wifi->bands[1].n_bitrates = ARRAY_SIZE(rates) - 4; + wifi->wiphy->bands[NL80211_BAND_2GHZ] = &wifi->bands[0]; + wifi->wiphy->bands[NL80211_BAND_5GHZ] = &wifi->bands[1]; + ret = wiphy_register(wifi->wiphy); + if (ret) + goto failed; + wifi->registered = true; + wifi->wdev.wiphy = wifi->wiphy; + wifi->wdev.iftype = NL80211_IFTYPE_STATION; + wifi->wdev.netdev = wifi->netdev; + wifi->netdev->ieee80211_ptr = &wifi->wdev; + wifi->netdev->netdev_ops = &net_ops; + SET_NETDEV_DEV(wifi->netdev, &c->pdev->dev); + strscpy(wifi->netdev->name, "wlan%d", IFNAMSIZ); + wifi->netdev->max_mtu = ETH_DATA_LEN; + eth_hw_addr_random(wifi->netdev); + ret = register_netdev(wifi->netdev); + if (ret) + goto failed; + ret = radio_start(wifi); + if (ret) { + dev_err(&c->pdev->dev, "Wi-Fi ring/interface setup failed: %d\n", ret); + goto failed; + } + dev_info(&c->pdev->dev, "cfg80211 interface %s ready for scan tests; packet data unavailable\n", + wifi->netdev->name); + wifi->passphrase_file = debugfs_create_file("passphrase", 0200, c->debug, wifi, &passphrase_ops); + wifi->data_probe_file = debugfs_create_file("data_probe", 0200, c->debug, wifi, &data_probe_ops); + wifi->data_status_file = debugfs_create_file("data_status", 0400, c->debug, wifi, &data_status_ops); + wifi->data_rx_snapshot_file = debugfs_create_file("data_rx_snapshot", 0400, + c->debug, wifi, &data_rx_snapshot_ops); + wifi->station_stats_file = debugfs_create_file("station_stats", 0400, c->debug, wifi, &station_stats_ops); + wifi->rx_crypto_stats_file = debugfs_create_file("rx_crypto_stats", 0400, + c->debug, wifi, &rx_crypto_stats_ops); + wifi->dot11_mib_stats_file = debugfs_create_file("dot11_mib_stats", 0400, + c->debug, wifi, &dot11_mib_stats_ops); + wifi->tx_power_report_file = debugfs_create_file("tx_power_report", 0400, + c->debug, wifi, &tx_power_report_ops); + wifi->eapol_start_file = debugfs_create_file("eapol_start", 0200, c->debug, wifi, &eapol_start_ops); + wifi->country_file = debugfs_create_file("country", 0400, c->debug, wifi, &country_ops); + wifi->link_status_file = debugfs_create_file("link_status", 0400, c->debug, wifi, &link_status_ops); + wifi->tx_config_file = debugfs_create_file("tx_config", 0600, c->debug, wifi, &tx_config_ops); + wifi->host_wakeup_file = debugfs_create_file("host_wakeup", 0200, c->debug, wifi, &host_wakeup_ops); + wifi->fwlog_start_file = debugfs_create_file("fwlog_start", 0200, c->debug, wifi, &fwlog_start_ops); + wifi->fwlog_status_file = debugfs_create_file("fwlog_status", 0400, c->debug, wifi, &fwlog_status_ops); + wifi->fwlog_buffer_file = debugfs_create_file("fwlog_buffer", 0400, c->debug, wifi, &fwlog_buffer_ops); + wifi->fwlog_ack_file = debugfs_create_file("fwlog_ack", 0200, c->debug, wifi, &fwlog_ack_ops); + return 0; +failed: + centauri_wifi_unregister(c); + return ret; +} + +void centauri_wifi_unregister(struct centauri *c) +{ + struct centauri_wifi *wifi = c->wifi; + + if (!wifi) + return; + WRITE_ONCE(wifi->stopping, true); + if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) + netif_tx_disable(wifi->netdev); + debugfs_remove(wifi->passphrase_file); + debugfs_remove(wifi->data_probe_file); + debugfs_remove(wifi->data_status_file); + debugfs_remove(wifi->data_rx_snapshot_file); + debugfs_remove(wifi->station_stats_file); + debugfs_remove(wifi->rx_crypto_stats_file); + debugfs_remove(wifi->dot11_mib_stats_file); + debugfs_remove(wifi->tx_power_report_file); + debugfs_remove(wifi->eapol_start_file); + debugfs_remove(wifi->country_file); + debugfs_remove(wifi->link_status_file); + debugfs_remove(wifi->tx_config_file); + debugfs_remove(wifi->host_wakeup_file); + debugfs_remove(wifi->fwlog_start_file); + debugfs_remove(wifi->fwlog_status_file); + debugfs_remove(wifi->fwlog_buffer_file); + debugfs_remove(wifi->fwlog_ack_file); + cancel_work_sync(&wifi->scan_work); + cancel_work_sync(&wifi->connect_work); + cancel_delayed_work_sync(&wifi->link_work); + skb_queue_purge(&wifi->data_tx_queue); + if (wifi->scan_request) { + struct cfg80211_scan_info info = { .aborted = true }; + + cfg80211_scan_done(wifi->scan_request, &info); + wifi->scan_request = NULL; + } + if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) + unregister_netdev(wifi->netdev); + if (wifi->registered) + wiphy_unregister(wifi->wiphy); + if (wifi->wiphy) + wiphy_free(wifi->wiphy); + if (wifi->netdev) + free_netdev(wifi->netdev); + memzero_explicit(wifi, sizeof(*wifi)); + kfree(wifi); + c->wifi = NULL; +} From e9ec92d7dd8c14809f28be799e6f8333feac04d6 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 10 Sep 2026 01:15:28 -0400 Subject: [PATCH 0148/1579] Bluetooth: add J813 N1 native transport and encrypted connection bring-up Path-scoped import of the N1 driver changes. J813 workbench scripts, board DT and private calibration staging are deliberately excluded. Original-commit: c0f6eceacec78f707e3a24e0ef8cb74fa8b8dc36 Source-repository: https://github.com/aurora-silicon/linux-aurora --- drivers/bluetooth/Kconfig | 14 + drivers/bluetooth/Makefile | 1 + drivers/bluetooth/btcentauri.c | 984 ++++++++++++++++++ .../wireless/apple/centauri/centauri_ipc.c | 37 + 4 files changed, 1036 insertions(+) create mode 100644 drivers/bluetooth/btcentauri.c diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index fc1b37044a9b0..078f29b3ea432 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -512,4 +512,18 @@ config BT_INTEL_PCIE Say Y here to compiler support for Intel Bluetooth PCIe device into the kernel or say M to compile it as module (btintel_pcie) + +config BT_CENTAURI + tristate "Apple N1 Beta Bluetooth transport (experimental)" + depends on PCI && (ARCH_APPLE || COMPILE_TEST) + help + Experimental Bluetooth transport for Apple N1 on J813. + The Centauri Control driver must boot the shared firmware before this + driver binds to Beta. Controller initialization, discovery, BR/EDR ACL, + pairing and encrypted AVDTP discovery have been tested on hardware. + ACL delivery and native startup/connection handling remain opt-in; + audio playback and LE data are unqualified. SCO/ISO is not implemented. + + Say M to build the btcentauri module for bring-up testing. + endmenu diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile index 81856512ddd03..1a884b3d41588 100644 --- a/drivers/bluetooth/Makefile +++ b/drivers/bluetooth/Makefile @@ -53,3 +53,4 @@ hci_uart-$(CONFIG_BT_HCIUART_AG6XX) += hci_ag6xx.o hci_uart-$(CONFIG_BT_HCIUART_MRVL) += hci_mrvl.o hci_uart-$(CONFIG_BT_HCIUART_AML) += hci_aml.o hci_uart-objs := $(hci_uart-y) +obj-$(CONFIG_BT_CENTAURI) += btcentauri.o diff --git a/drivers/bluetooth/btcentauri.c b/drivers/bluetooth/btcentauri.c new file mode 100644 index 0000000000000..d2539539fa162 --- /dev/null +++ b/drivers/bluetooth/btcentauri.c @@ -0,0 +1,984 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Apple N1 Beta command/event transport for J813 bring-up. + * + * The Centauri Control driver must have booted the shared firmware first. + * This driver owns only Beta. It never resets the port or the Wi-Fi function. + * BR/EDR discovery, ACL/SDP, pairing and encrypted AVDTP discovery are qualified. + * ACL remains opt-in; LE data, persistent bonds and audio are unqualified. + * SCO/ISO delivery is not implemented. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CEN_ARENA_SIZE SZ_1M +#define CEN_MTR 0x2000 +#define CEN_MCR 0x2200 +#define CEN_MESSAGES 0x4000 +#define CEN_RX 0x8000 +#define CEN_TX 0xa000 +#define CEN_FOOTER 260 +#define CEN_STRIDE (16 + CEN_FOOTER) +#define CEN_RX_SLOTS 17 +#define CEN_TX_SLOTS 5 +#define CEN_LE_RX 0xc000 +#define CEN_LE_TX 0xe000 +#define CEN_LE_FOOTER 256 +#define CEN_LE_STRIDE (16 + CEN_LE_FOOTER) +#define CEN_LE_SLOTS 17 +#define CEN_MESSAGE_SLOTS 16 +#define CEN_AUX_BASE 0x20000 +#define CEN_CAPTURE_SIZE SZ_1M + +static bool arm_le_data_rings = true; +module_param(arm_le_data_rings, bool, 0444); +MODULE_PARM_DESC(arm_le_data_rings, "Arm native Beta LE rings for radio bring-up"); + +static bool arm_auxiliary_rings; +module_param(arm_auxiliary_rings, bool, 0444); +MODULE_PARM_DESC(arm_auxiliary_rings, "Diagnostic: arm remaining native Beta resources and capture received frames privately"); + +static bool native_startup; +module_param(native_startup, bool, 0444); +MODULE_PARM_DESC(native_startup, "Diagnostic: program a temporary address, InitDone and random seed before the first HCI reset"); + +static bool enable_acl; +module_param(enable_acl, bool, 0444); +MODULE_PARM_DESC(enable_acl, "Experimental ACL data delivery; requires LE and auxiliary rings"); + +static bool native_hrb; +module_param(native_hrb, bool, 0444); +MODULE_PARM_DESC(native_hrb, "Experimental native Proxima Create Connection; requires native startup, ACL and firmware capability bit 9"); + +#define CEN_OP_READ_PROP_FEATURES 0xffa3 +#define CEN_OP_HRB_CREATE_CONN 0xff7b +#define CEN_PROP_HRB_CREATE BIT(9) + +struct cen_aux_spec { + u16 id, slots, footer; + bool receive; +}; + +/* Host native Beta inventory; each resource is opt-in until target-qualified. + * HCI and LE rings remain in their existing, separately qualified arenas. + */ +static const struct cen_aux_spec cen_aux_specs[] = { + { 2, 9, 2592, false }, /* ACL TX */ + { 3, 9, 724, false }, /* SCO TX */ + { 5, 9, 260, false }, /* LEA TX */ + { 6, 5, 1024, false }, /* LLA TX */ + { 8, 9, 2592, true }, /* ACL RX */ + { 9, 9, 724, true }, /* SCO RX */ + { 11, 9, 268, true }, /* LEA RX */ + { 12, 128, 1024, true }, /* Firmware debug RX */ + { 13, 17, 260, true }, /* LMP RX */ + { 14, 9, 1304, false }, /* TSI TX */ + { 15, 9, 1304, true }, /* TSI RX */ +}; + +struct cen_aux_state { + u32 offset, packets; + u16 head, post; +}; + +struct cen_bt { + struct pci_dev *pdev; + struct hci_dev *hdev; + struct dentry *debug; + void __iomem *bar; + u8 *arena; + dma_addr_t dma; + struct mutex lock; + struct delayed_work poll; + struct sk_buff_head txq; + struct sk_buff_head acl_txq, le_txq; + u16 acl_tx_head, le_tx_head; + u16 message_head, message_tag, tx_head, rx_head, rx_post; + u16 le_rx_head, le_rx_post; + u32 le_rx_packets; + struct cen_aux_state aux[ARRAY_SIZE(cen_aux_specs)]; + u8 *capture; + u32 capture_used, capture_records, capture_dropped; + bool ipc_requested, running; + u8 native_startup_steps; + bool hrb_supported, hrb_status_pending; + u32 proprietary_features, hrb_commands, hrb_status_events; + u32 hrb_connection_events; + u8 hrb_last_link_type; +}; + +static u16 cen_tail(struct cen_bt *bt, u16 ring) +{ + return le16_to_cpu(READ_ONCE(*(__le16 *)(bt->arena + 0x7a + ring * 2))); +} + +static void cen_post(struct cen_bt *bt, u16 ring, u16 value) +{ + put_unaligned_le16(value, bt->arena + 0xa8 + ring * 2); + dma_wmb(); + writel(value, bt->bar + 0x1000 + ring * 4); +} + +static int cen_message(struct cen_bt *bt, const void *data, size_t length) +{ + u16 head = bt->message_head, tail, next = (head + 1) % CEN_MESSAGE_SLOTS; + u16 tag = bt->message_tag + 1; + u8 *descriptor = bt->arena + CEN_MTR + head * 16; + u8 *completion = bt->arena + CEN_MCR + head * 32; + int ret; + + if (head >= CEN_MESSAGE_SLOTS || !length || length > 512) + return -EINVAL; + memset(descriptor, 0, 16); + memset(completion, 0, 32); + memcpy(bt->arena + CEN_MESSAGES + head * 512, data, length); + put_unaligned_le32(1 | (length << 8), descriptor); + put_unaligned_le64(bt->dma + CEN_MESSAGES + head * 512, descriptor + 4); + put_unaligned_le16(tag, descriptor + 12); + cen_post(bt, 0, next); + ret = read_poll_timeout(cen_tail, tail, tail == next, + 100, 1500000, false, bt, 0); + if (ret) + return ret; + /* The device publishes the message completion before advancing its head. */ + ret = read_poll_timeout(le16_to_cpu, tail, tail == next, + 100, 1500000, false, + READ_ONCE(*(__le16 *)(bt->arena + 0x78))); + if (ret) + return ret; + dma_rmb(); + if (get_unaligned_le16(completion + 4) != tag || + get_unaligned_le16(completion + 6) != length || + get_unaligned_le32(completion + 8)) + return -EPROTO; + bt->message_head = next; + bt->message_tag = tag; + put_unaligned_le16(bt->message_head, bt->arena + 0xa6); + dma_wmb(); + return 0; +} + +static int cen_open_ring(struct cen_bt *bt, u16 id, u16 slots, u32 offset, u16 footer) +{ + u8 message[0x34] = { 1 }; + u32 words = footer / 4, shift = 0; + + if (!words || footer % 4) + return -EINVAL; + if (words > 255) + shift = min_t(u32, __ffs(words), 4); + if ((words >> shift) > 255) + return -EINVAL; + message[2] = words >> shift; + message[3] = shift << 4; + put_unaligned_le16(id, message + 4); + put_unaligned_le64(bt->dma + offset, message + 8); + put_unaligned_le64(~0ULL, message + 0x10); + put_unaligned_le16(slots, message + 0x18); + put_unaligned_le16(0xffff, message + 0x1a); + put_unaligned_le16(id, message + 0x1c); + put_unaligned_le16(0x40, message + 0x1e); + return cen_message(bt, message, sizeof(message)); +} + +static void cen_rx_buffer(struct cen_bt *bt, u16 slot) +{ + u8 *descriptor = bt->arena + CEN_RX + slot * CEN_STRIDE; + + memset(descriptor, 0, CEN_STRIDE); + put_unaligned_le32(1 | (CEN_FOOTER << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static void cen_le_rx_buffer(struct cen_bt *bt, u16 slot) +{ + u8 *descriptor = bt->arena + CEN_LE_RX + slot * CEN_LE_STRIDE; + + memset(descriptor, 0, CEN_LE_STRIDE); + put_unaligned_le32(1 | (CEN_LE_FOOTER << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static void cen_capture_frame(struct cen_bt *bt, u16 ring, const u8 *data, u16 length) +{ + u8 *record; + + if (!bt->capture) + return; + if (8U + length > CEN_CAPTURE_SIZE - bt->capture_used) { + bt->capture_dropped++; + return; + } + record = bt->capture + bt->capture_used; + put_unaligned_le16(ring, record); + put_unaligned_le16(length, record + 2); + put_unaligned_le32(bt->capture_records++, record + 4); + memcpy(record + 8, data, length); + bt->capture_used += 8 + length; +} + +static int cen_receive_acl(struct cen_bt *bt, const u8 *data, u32 length) +{ + struct sk_buff *skb; + u32 packet_length, pos; + + if (length < HCI_ACL_HDR_SIZE) + return -EPROTO; + packet_length = HCI_ACL_HDR_SIZE + get_unaligned_le16(data + 2); + if (packet_length > length) + return -EPROTO; + /* Airship RX attributes follow, rather than belong to, the HCI packet. + * Only the native parser's known record sizes are accepted. Their values + * remain opaque and are retained only in the private diagnostic capture. + */ + for (pos = packet_length; pos < length;) { + if (!data[pos]) + break; + if (data[pos] == 1) + pos++; + else if (data[pos] == 2 && length - pos >= 9) + pos += 9; + else + return -EPROTO; + } + skb = bt_skb_alloc(packet_length, GFP_KERNEL); + if (!skb) { + bt->hdev->stat.err_rx++; + return 0; + } + skb_put_data(skb, data, packet_length); + hci_skb_pkt_type(skb) = HCI_ACLDATA_PKT; + bt->hdev->stat.byte_rx += packet_length; + hci_recv_frame(bt->hdev, skb); + return 0; +} + +static void cen_aux_rx_buffer(struct cen_bt *bt, unsigned int index, u16 slot) +{ + const struct cen_aux_spec *spec = &cen_aux_specs[index]; + u8 *descriptor = bt->arena + bt->aux[index].offset + slot * (16 + spec->footer); + + memset(descriptor, 0, 16 + spec->footer); + put_unaligned_le32(1 | (spec->footer << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static int cen_aux_start(struct cen_bt *bt) +{ + u32 offset = CEN_AUX_BASE; + unsigned int i, slot; + int ret; + + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) { + const struct cen_aux_spec *spec = &cen_aux_specs[i]; + struct cen_aux_state *state = &bt->aux[i]; + u32 length = spec->slots * (16U + spec->footer); + + if (offset > CEN_ARENA_SIZE || length > CEN_ARENA_SIZE - offset) + return -ENOSPC; + state->offset = offset; + offset = ALIGN(offset + length, SZ_4K); + ret = cen_open_ring(bt, spec->id, spec->slots, state->offset, spec->footer); + if (ret) { + dev_err(&bt->pdev->dev, "Auxiliary ring %u open failed: %d\n", spec->id, ret); + return ret; + } + if (!spec->receive) + continue; + for (slot = 0; slot < spec->slots - 1; slot++) + cen_aux_rx_buffer(bt, i, slot); + state->post = spec->slots - 1; + cen_post(bt, spec->id, state->post); + } + return 0; +} + +static int cen_aux_drain(struct cen_bt *bt, bool repost) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) { + const struct cen_aux_spec *spec = &cen_aux_specs[i]; + struct cen_aux_state *state = &bt->aux[i]; + u16 tail; + int budget = spec->slots; + + if (!spec->receive) + continue; + tail = cen_tail(bt, spec->id); + if (tail >= spec->slots) + return -EPROTO; + while (state->head != tail && budget--) { + u8 *descriptor = bt->arena + state->offset + state->head * (16 + spec->footer); + u32 length; + + dma_rmb(); + length = get_unaligned_le32(descriptor) >> 8; + if (length > spec->footer) + return -EPROTO; + cen_capture_frame(bt, spec->id, descriptor + 16, length); + if (repost && enable_acl && spec->id == 8 && + cen_receive_acl(bt, descriptor + 16, length)) + return -EPROTO; + state->packets++; + state->head = (state->head + 1) % spec->slots; + if (repost) { + cen_aux_rx_buffer(bt, i, state->post); + state->post = (state->post + 1) % spec->slots; + cen_post(bt, spec->id, state->post); + } + } + } + return 0; +} + +static int cen_start(struct cen_bt *bt) +{ + static const u16 indices[] = { 0x78, 0x7a, 0xa6, 0xa8 }; + u8 config[4] = { 9, 0, 0xff, 0xff }; + u32 status; + int i, ret; + + put_unaligned_le16(0x300, bt->arena); + put_unaligned_le16(0x68, bt->arena + 2); + put_unaligned_le64(bt->dma + 0x68, bt->arena + 8); + for (i = 0; i < ARRAY_SIZE(indices); i++) + put_unaligned_le64(bt->dma + indices[i], bt->arena + 0x10 + i * 8); + put_unaligned_le16(1, bt->arena + 0x30); + put_unaligned_le16(22, bt->arena + 0x32); + put_unaligned_le64(bt->dma + CEN_MCR, bt->arena + 0x34); + put_unaligned_le64(bt->dma + CEN_MTR, bt->arena + 0x3c); + put_unaligned_le16(16, bt->arena + 0x44); + put_unaligned_le16(16, bt->arena + 0x46); + /* The 25G76 native context encodes the MCR footer in four-byte words. */ + bt->arena[0x53] = 4; + put_unaligned_le64(bt->dma + 0x10000, bt->arena + 0x58); + pci_set_master(bt->pdev); + writel(lower_32_bits(bt->dma), bt->bar + 0xc); + writel(upper_32_bits(bt->dma), bt->bar + 0x10); + writel(CEN_ARENA_SIZE, bt->bar + 0x14); + bt->ipc_requested = true; + writel(1, bt->bar + 0x107c); + ret = readl_poll_timeout(bt->bar, status, status == 1, 100, 2000000); + if (ret) + return ret; + dma_wmb(); + writel(lower_32_bits(bt->dma), bt->bar + 4); + writel(upper_32_bits(bt->dma), bt->bar + 8); + writel(2, bt->bar + 0x107c); + ret = readl_poll_timeout(bt->bar, status, status == 2, 100, 2000000); + if (ret) + return ret; + ret = cen_message(bt, config, sizeof(config)); + if (!ret) + ret = cen_open_ring(bt, 7, CEN_RX_SLOTS, CEN_RX, CEN_FOOTER); + if (!ret) + ret = cen_open_ring(bt, 1, CEN_TX_SLOTS, CEN_TX, CEN_FOOTER); + if (!ret && arm_le_data_rings) + ret = cen_open_ring(bt, 10, CEN_LE_SLOTS, CEN_LE_RX, CEN_LE_FOOTER); + if (!ret && arm_le_data_rings) + ret = cen_open_ring(bt, 4, CEN_LE_SLOTS, CEN_LE_TX, CEN_LE_FOOTER); + if (ret) + return ret; + for (i = 0; i < CEN_RX_SLOTS - 1; i++) + cen_rx_buffer(bt, i); + bt->rx_post = CEN_RX_SLOTS - 1; + cen_post(bt, 7, bt->rx_post); + if (arm_le_data_rings) { + for (i = 0; i < CEN_LE_SLOTS - 1; i++) + cen_le_rx_buffer(bt, i); + bt->le_rx_post = CEN_LE_SLOTS - 1; + cen_post(bt, 10, bt->le_rx_post); + } + if (arm_auxiliary_rings) + return cen_aux_start(bt); + return 0; +} + +static int cen_transmit(struct cen_bt *bt, struct sk_buff_head *queue, + u16 ring, u16 slots, u32 offset, u16 footer, u16 *head) +{ + struct sk_buff *skb; + u16 tail, next; + u8 *descriptor; + u32 wire_length; + + while (!skb_queue_empty(queue)) { + tail = cen_tail(bt, ring); + if (tail >= slots) + return -EPROTO; + next = (*head + 1) % slots; + if (next == tail) + break; + dma_rmb(); + skb = skb_dequeue(queue); + if (!skb) + break; + descriptor = bt->arena + offset + *head * (16 + footer); + memset(descriptor, 0, 16 + footer); + put_unaligned_le16(*head + 1, descriptor + 12); + memcpy(descriptor + 16, skb->data, skb->len); + wire_length = skb->len; + if (ring == 1 && READ_ONCE(bt->hrb_supported) && + get_unaligned_le16(skb->data) == HCI_OP_CREATE_CONN) { + u8 *command = descriptor + 16; + + /* Native Proxima uses the ordinary 13-byte parameter layout + * followed by two zero HRB fields (no forced band). It sends + * tagging=false and role_switch=0. Address byte order matches + * standard HCI. Keep the core's sent skb/opcode unchanged. + */ + put_unaligned_le16(CEN_OP_HRB_CREATE_CONN, command); + command[2] = 15; + command[HCI_COMMAND_HDR_SIZE + 9] = 0; + command[HCI_COMMAND_HDR_SIZE + 12] = 0; + wire_length = HCI_COMMAND_HDR_SIZE + 15; + bt->hrb_status_pending = true; + bt->hrb_commands++; + cen_capture_frame(bt, 1, command, wire_length); + } + put_unaligned_le32(1 | (wire_length << 8), descriptor); + *head = next; + cen_post(bt, ring, next); + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT) + bt->hdev->stat.cmd_tx++; + else + bt->hdev->stat.acl_tx++; + bt->hdev->stat.byte_tx += wire_length; + kfree_skb(skb); + } + return 0; +} + +static void cen_poll(struct work_struct *work) +{ + struct cen_bt *bt = container_of(to_delayed_work(work), struct cen_bt, poll); + struct hci_dev *hdev = bt->hdev; + struct sk_buff *skb; + u16 tail; + u8 *descriptor; + u32 length; + int budget = CEN_RX_SLOTS; + + mutex_lock(&bt->lock); + if (!bt->running) + goto unlock; + if (readl(bt->bar + 0x18) != 2 || readl(bt->bar) != 2) { + /* Preserve already completed diagnostics after abort, without posting + * any more buffers or ringing doorbells to the failed firmware. + */ + if (arm_auxiliary_rings && readl(bt->bar) == 2) + cen_aux_drain(bt, false); + goto protocol_error; + } + if (arm_auxiliary_rings && cen_aux_drain(bt, true)) + goto protocol_error; + if (arm_le_data_rings) { + tail = cen_tail(bt, 10); + if (tail >= CEN_LE_SLOTS) + goto protocol_error; + while (bt->le_rx_head != tail && budget--) { + dma_rmb(); + descriptor = bt->arena + CEN_LE_RX + bt->le_rx_head * CEN_LE_STRIDE; + length = get_unaligned_le32(descriptor) >> 8; + if (length > CEN_LE_FOOTER) + goto protocol_error; + bt->le_rx_packets++; + cen_capture_frame(bt, 10, descriptor + 16, length); + if (enable_acl && cen_receive_acl(bt, descriptor + 16, length)) + goto protocol_error; + bt->le_rx_head = (bt->le_rx_head + 1) % CEN_LE_SLOTS; + cen_le_rx_buffer(bt, bt->le_rx_post); + bt->le_rx_post = (bt->le_rx_post + 1) % CEN_LE_SLOTS; + cen_post(bt, 10, bt->le_rx_post); + } + budget = CEN_RX_SLOTS; + } + tail = cen_tail(bt, 7); + if (tail >= CEN_RX_SLOTS) + goto protocol_error; + while (bt->rx_head != tail && budget--) { + dma_rmb(); + descriptor = bt->arena + CEN_RX + bt->rx_head * CEN_STRIDE; + length = get_unaligned_le32(descriptor) >> 8; + if (length < HCI_EVENT_HDR_SIZE || length > CEN_FOOTER || + length != descriptor[17] + HCI_EVENT_HDR_SIZE) + goto protocol_error; + skb = bt_skb_alloc(length, GFP_KERNEL); + if (skb) { + skb_put_data(skb, descriptor + 16, length); + /* The firmware's status belongs to the standard connection + * request tracked by Linux. Capture the native response before + * restoring only its opcode; preserve status and command credit. + */ + if (bt->hrb_status_pending && length == 6 && + skb->data[0] == HCI_EV_CMD_STATUS && + get_unaligned_le16(skb->data + 4) == CEN_OP_HRB_CREATE_CONN) { + cen_capture_frame(bt, 7, skb->data, length); + put_unaligned_le16(HCI_OP_CREATE_CONN, skb->data + 4); + bt->hrb_status_pending = false; + bt->hrb_status_events++; + } + /* Native HCI flow treats 0xf0/0xf1 Connection Complete as + * ACL, with additional proprietary capability flags. Linux + * otherwise cannot find its pending ACL connection by type. + * Preserve the original event privately before normalizing it. + */ + if (READ_ONCE(bt->hrb_supported) && + length == HCI_EVENT_HDR_SIZE + sizeof(struct hci_ev_conn_complete) && + skb->data[0] == HCI_EV_CONN_COMPLETE && + (skb->data[11] & 0xfe) == 0xf0) { + cen_capture_frame(bt, 7, skb->data, length); + bt->hrb_last_link_type = skb->data[11]; + skb->data[11] = ACL_LINK; + bt->hrb_connection_events++; + } + hci_skb_pkt_type(skb) = HCI_EVENT_PKT; + hdev->stat.byte_rx += length; + hci_recv_frame(hdev, skb); + } else { + hdev->stat.err_rx++; + } + bt->rx_head = (bt->rx_head + 1) % CEN_RX_SLOTS; + cen_rx_buffer(bt, bt->rx_post); + bt->rx_post = (bt->rx_post + 1) % CEN_RX_SLOTS; + cen_post(bt, 7, bt->rx_post); + } + if (cen_transmit(bt, &bt->txq, 1, CEN_TX_SLOTS, CEN_TX, + CEN_FOOTER, &bt->tx_head)) + goto protocol_error; + /* cen_aux_specs[0] is the native BR/EDR ACL TX resource (ring 2). */ + if (enable_acl && + (cen_transmit(bt, &bt->acl_txq, 2, cen_aux_specs[0].slots, + bt->aux[0].offset, cen_aux_specs[0].footer, + &bt->acl_tx_head) || + cen_transmit(bt, &bt->le_txq, 4, CEN_LE_SLOTS, CEN_LE_TX, + CEN_LE_FOOTER, &bt->le_tx_head))) + goto protocol_error; + schedule_delayed_work(&bt->poll, msecs_to_jiffies(1)); + goto unlock; +protocol_error: + bt->running = false; + hdev->stat.err_rx++; + dev_err(&bt->pdev->dev, "Beta transport stopped (exec=%u ipc=%u); polling disabled\n", + readl(bt->bar + 0x18), readl(bt->bar)); +unlock: + mutex_unlock(&bt->lock); +} + +static int cen_hci_open(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->running = true; + schedule_delayed_work(&bt->poll, 0); + mutex_unlock(&bt->lock); + return 0; +} + +static int cen_hci_close(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->running = false; + bt->hrb_status_pending = false; + mutex_unlock(&bt->lock); + cancel_delayed_work_sync(&bt->poll); + skb_queue_purge(&bt->txq); + skb_queue_purge(&bt->acl_txq); + skb_queue_purge(&bt->le_txq); + return 0; +} + +static int cen_vendor_command(struct hci_dev *hdev, u16 opcode, + const void *data, u8 length) +{ + struct sk_buff *skb; + int ret; + + skb = __hci_cmd_sync(hdev, opcode, length, data, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + ret = skb->len < 1 ? -EPROTO : -bt_to_errno(skb->data[0]); + kfree_skb(skb); + return ret; +} + +static int cen_hci_post_init(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + struct sk_buff *skb; + u32 features; + int ret; + + if (!native_hrb) + return 0; + WRITE_ONCE(bt->hrb_supported, false); + skb = __hci_cmd_sync(hdev, CEN_OP_READ_PROP_FEATURES, 0, NULL, + HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + ret = skb->len == 5 ? -bt_to_errno(skb->data[0]) : -EPROTO; + features = skb->len == 5 ? get_unaligned_le32(skb->data + 1) : 0; + kfree_skb(skb); + WRITE_ONCE(bt->proprietary_features, features); + if (ret) + return ret; + if (!(features & CEN_PROP_HRB_CREATE)) + return -EOPNOTSUPP; + WRITE_ONCE(bt->hrb_supported, true); + bt_dev_info(hdev, "Native Proxima Create Connection capability confirmed"); + return 0; +} + +static int cen_hci_setup(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + u8 address[6], init_done[16], seed[24]; + int ret; + + if (!native_startup) + return 0; + + /* Native PROXIMA initialization starts with these three commands, then + * HCI Reset. The core calls setup before its ordinary reset/init sequence. + * Address provisioning is still diagnostic: use a temporary locally + * administered address, in native WriteBdAddr display/network byte order. + */ + ret = get_random_bytes_wait(address, sizeof(address)); + if (ret) + return ret; + address[0] = (address[0] & 0xfc) | 0x02; + get_random_bytes(seed, sizeof(seed)); + memset(init_done, 0xff, sizeof(init_done)); + ret = cen_vendor_command(hdev, 0xfc06, address, sizeof(address)); + if (ret) + goto out; + WRITE_ONCE(bt->native_startup_steps, 1); + ret = cen_vendor_command(hdev, 0xff0b, init_done, sizeof(init_done)); + if (ret) + goto out; + WRITE_ONCE(bt->native_startup_steps, 2); + ret = cen_vendor_command(hdev, 0xfe44, seed, sizeof(seed)); + if (!ret) + WRITE_ONCE(bt->native_startup_steps, 3); +out: + memzero_explicit(seed, sizeof(seed)); + if (ret) + bt_dev_err(hdev, "Native startup stopped after %u commands: %d", + bt->native_startup_steps, ret); + else + bt_dev_info(hdev, "Native startup accepted; temporary diagnostic address in use"); + return ret; +} + +static int cen_hci_flush(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + skb_queue_purge(&bt->txq); + skb_queue_purge(&bt->acl_txq); + skb_queue_purge(&bt->le_txq); + return 0; +} + +static int cen_hci_send(struct hci_dev *hdev, struct sk_buff *skb) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + struct sk_buff_head *queue = &bt->txq; + struct hci_conn *conn; + u16 handle; + u8 link_type; + int ret = 0; + + if (hci_skb_pkt_type(skb) == HCI_ACLDATA_PKT && enable_acl) { + if (skb->len < HCI_ACL_HDR_SIZE || + skb->len != HCI_ACL_HDR_SIZE + get_unaligned_le16(skb->data + 2)) { + ret = -EMSGSIZE; + goto out; + } + handle = get_unaligned_le16(skb->data); + rcu_read_lock(); + conn = hci_conn_hash_lookup_handle(hdev, hci_handle(handle)); + link_type = conn ? conn->type : 0xff; + rcu_read_unlock(); + if (link_type == LE_LINK && (handle & 0x3000) != 0x3000) { + queue = &bt->le_txq; + if (skb->len > CEN_LE_FOOTER) + ret = -EMSGSIZE; + } else if (link_type == ACL_LINK) { + queue = &bt->acl_txq; + if (skb->len > cen_aux_specs[0].footer) + ret = -EMSGSIZE; + } else { + ret = -EOPNOTSUPP; + } + if (ret) + goto out; + } else if (hci_skb_pkt_type(skb) != HCI_COMMAND_PKT) { + ret = -EOPNOTSUPP; + goto out; + } else if (skb->len < HCI_COMMAND_HDR_SIZE || skb->len > CEN_FOOTER || + skb->len != HCI_COMMAND_HDR_SIZE + skb->data[2]) { + ret = -EMSGSIZE; + goto out; + } + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT && native_hrb) { + u16 opcode = get_unaligned_le16(skb->data); + + /* Do not mix unmanaged native commands with translated requests. */ + if (opcode == CEN_OP_HRB_CREATE_CONN) { + ret = -EOPNOTSUPP; + goto out; + } + if (opcode == HCI_OP_CREATE_CONN && + skb->len != HCI_COMMAND_HDR_SIZE + sizeof(struct hci_cp_create_conn)) { + ret = -EMSGSIZE; + goto out; + } + } + mutex_lock(&bt->lock); + if (!bt->running) + ret = -ENETDOWN; + else if (skb_queue_len(queue) >= 64) + ret = -ENOBUFS; + else { + skb_queue_tail(queue, skb); + mod_delayed_work(system_wq, &bt->poll, 0); + } + mutex_unlock(&bt->lock); + if (!ret) + return 0; +out: + /* hci_send_frame() frees rejected packets. Ownership transfers to this + * driver only when the skb is queued and send returns success. + */ + return ret; +} + +static void cen_stop(struct cen_bt *bt) +{ + pci_clear_master(bt->pdev); + kvfree(bt->capture); + bt->capture = NULL; + /* IPC control 0 caused a firmware abort in the initial hardware probe. + * Until coordinated firmware teardown is understood, retain any published + * arena after detaching. A fresh firmware boot is required before rebinding. + */ + if (bt->ipc_requested || !pci_wait_for_pending_transaction(bt->pdev)) + dev_warn(&bt->pdev->dev, "Beta detach requires fresh firmware boot; retaining DMA arena\n"); + else + dma_free_coherent(&bt->pdev->dev, CEN_ARENA_SIZE, bt->arena, bt->dma); +} + +static int cen_status_show(struct seq_file *s, void *unused) +{ + struct cen_bt *bt = s->private; + unsigned int i; + + mutex_lock(&bt->lock); + seq_printf(s, "exec=%u ipc=%u running=%u tx_head=%u tx_consumed=%u rx_head=%u rx_consumed=%u queued=%u\n", + readl(bt->bar + 0x18), readl(bt->bar), bt->running, + bt->tx_head, cen_tail(bt, 1), bt->rx_head, cen_tail(bt, 7), + skb_queue_len(&bt->txq)); + seq_printf(s, "commands=%u events=%u tx_bytes=%u rx_bytes=%u tx_errors=%u rx_errors=%u\n", + bt->hdev->stat.cmd_tx, bt->hdev->stat.evt_rx, + bt->hdev->stat.byte_tx, bt->hdev->stat.byte_rx, + bt->hdev->stat.err_tx, bt->hdev->stat.err_rx); + seq_printf(s, "le_rings=%u le_rx_packets=%u\n", arm_le_data_rings, bt->le_rx_packets); + seq_printf(s, "native_startup=%u native_startup_steps=%u\n", + native_startup, READ_ONCE(bt->native_startup_steps)); + seq_printf(s, "native_hrb=%u hrb_supported=%u proprietary_features=0x%08x hrb_commands=%u hrb_status_events=%u hrb_pending=%u\n", + native_hrb, READ_ONCE(bt->hrb_supported), + READ_ONCE(bt->proprietary_features), bt->hrb_commands, + bt->hrb_status_events, bt->hrb_status_pending); + seq_printf(s, "hrb_connection_events=%u hrb_last_link_type=0x%02x\n", + bt->hrb_connection_events, bt->hrb_last_link_type); + seq_printf(s, "acl_enabled=%u acl_tx=%u acl_rx=%u acl_queued=%u le_queued=%u\n", + enable_acl, bt->hdev->stat.acl_tx, bt->hdev->stat.acl_rx, + skb_queue_len(&bt->acl_txq), skb_queue_len(&bt->le_txq)); + seq_printf(s, "auxiliary_rings=%u messages=%u message_head=%u capture_bytes=%u capture_records=%u capture_dropped=%u\n", + arm_auxiliary_rings, bt->message_tag, bt->message_head, + bt->capture_used, bt->capture_records, bt->capture_dropped); + if (arm_auxiliary_rings) + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) + if (cen_aux_specs[i].receive) + seq_printf(s, "aux_rx ring=%u packets=%u consumed=%u device_tail=%u\n", + cen_aux_specs[i].id, bt->aux[i].packets, + bt->aux[i].head, cen_tail(bt, cen_aux_specs[i].id)); + mutex_unlock(&bt->lock); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(cen_status); + +static ssize_t cen_capture_read(struct file *file, char __user *buffer, + size_t count, loff_t *pos) +{ + struct cen_bt *bt = file->private_data; + ssize_t ret; + + mutex_lock(&bt->lock); + ret = simple_read_from_buffer(buffer, count, pos, bt->capture, bt->capture_used); + mutex_unlock(&bt->lock); + return ret; +} + +static const struct file_operations cen_capture_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = cen_capture_read, + .llseek = default_llseek, +}; + +static ssize_t cen_capture_reset(struct file *file, const char __user *buffer, + size_t count, loff_t *pos) +{ + struct cen_bt *bt = file->private_data; + unsigned int value; + int ret = kstrtouint_from_user(buffer, count, 0, &value); + + if (ret || value != 1) + return ret ?: -EINVAL; + mutex_lock(&bt->lock); + bt->capture_used = 0; + bt->capture_records = 0; + bt->capture_dropped = 0; + mutex_unlock(&bt->lock); + return count; +} + +static const struct file_operations cen_capture_reset_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = cen_capture_reset, +}; + +static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct cen_bt *bt; + struct hci_dev *hdev; + int ret; + + if (!of_machine_is_compatible("apple,j813") || PCI_FUNC(pdev->devfn) != 2) + return -ENODEV; + bt = devm_kzalloc(&pdev->dev, sizeof(*bt), GFP_KERNEL); + if (!bt) + return -ENOMEM; + bt->pdev = pdev; + mutex_init(&bt->lock); + skb_queue_head_init(&bt->txq); + skb_queue_head_init(&bt->acl_txq); + skb_queue_head_init(&bt->le_txq); + if (enable_acl && (!arm_le_data_rings || !arm_auxiliary_rings)) + return -EINVAL; + if (native_hrb && (!native_startup || !enable_acl)) + return -EINVAL; + INIT_DELAYED_WORK(&bt->poll, cen_poll); + ret = pcim_enable_device(pdev); + if (ret) + return ret; + if (pci_resource_len(pdev, 0) != SZ_64K) + return -ENODEV; + ret = pcim_iomap_regions(pdev, BIT(0), "btcentauri"); + if (ret) + return ret; + bt->bar = pcim_iomap_table(pdev)[0]; + if (readl(bt->bar + 0x18) != 2 || readl(bt->bar) != 0) + return dev_err_probe(&pdev->dev, -EBUSY, + "Beta requires running firmware and unowned IPC (exec=%u ipc=%u)\n", + readl(bt->bar + 0x18), readl(bt->bar)); + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); + if (ret) + return ret; + bt->arena = dma_alloc_coherent(&pdev->dev, CEN_ARENA_SIZE, &bt->dma, GFP_KERNEL); + if (!bt->arena) + return -ENOMEM; + memset(bt->arena, 0, CEN_ARENA_SIZE); + if (arm_auxiliary_rings) { + bt->capture = kvzalloc(CEN_CAPTURE_SIZE, GFP_KERNEL); + if (!bt->capture) { + ret = -ENOMEM; + goto stop; + } + } + ret = cen_start(bt); + if (ret) { + dev_err(&pdev->dev, "Beta IPC initialization failed: %d\n", ret); + goto stop; + } + hdev = hci_alloc_dev(); + if (!hdev) { + ret = -ENOMEM; + goto stop; + } + bt->hdev = hdev; + hdev->bus = HCI_PCI; + hdev->open = cen_hci_open; + hdev->close = cen_hci_close; + hdev->flush = cen_hci_flush; + hdev->send = cen_hci_send; + hdev->setup = cen_hci_setup; + hdev->post_init = cen_hci_post_init; + SET_HCIDEV_DEV(hdev, &pdev->dev); + hci_set_drvdata(hdev, bt); + pci_set_drvdata(pdev, bt); + ret = hci_register_dev(hdev); + if (ret) { + hci_free_dev(hdev); + goto stop; + } + bt->debug = debugfs_create_dir("btcentauri", NULL); + debugfs_create_file("status", 0400, bt->debug, bt, &cen_status_fops); + if (bt->capture) { + debugfs_create_file("auxiliary_capture", 0400, bt->debug, bt, &cen_capture_fops); + debugfs_create_file("capture_reset", 0200, bt->debug, bt, &cen_capture_reset_fops); + } + dev_info(&pdev->dev, "Beta HCI transport registered; ACL enabled=%u, SCO/ISO unsupported\n", + enable_acl); + return 0; +stop: + cen_stop(bt); + return ret; +} + +static void cen_remove(struct pci_dev *pdev) +{ + struct cen_bt *bt = pci_get_drvdata(pdev); + + debugfs_remove_recursive(bt->debug); + hci_unregister_dev(bt->hdev); + cen_hci_close(bt->hdev); + hci_free_dev(bt->hdev); + cen_stop(bt); +} + +static const struct pci_device_id cen_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1903) }, + { } +}; +MODULE_DEVICE_TABLE(pci, cen_ids); + +static struct pci_driver cen_driver = { + .name = "btcentauri", + .id_table = cen_ids, + .probe = cen_probe, + .remove = cen_remove, +}; +module_pci_driver(cen_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple N1 Beta HCI command/event transport for J813 bring-up"); diff --git a/drivers/net/wireless/apple/centauri/centauri_ipc.c b/drivers/net/wireless/apple/centauri/centauri_ipc.c index 6eec46d831888..0db35959d114d 100644 --- a/drivers/net/wireless/apple/centauri/centauri_ipc.c +++ b/drivers/net/wireless/apple/centauri/centauri_ipc.c @@ -3,10 +3,12 @@ * Wire layout source: AppleCentauriCommon/scripts/alpha_wifi_scan.py v259. * Polling and bounded bootstrap rings are deliberate bring-up restrictions. */ +#include #include #include #include #include +#include #include #include #include @@ -79,6 +81,40 @@ static int ftab_entry(const void *image, size_t size, const char tag[4], return -ENOENT; } +/* The native secondary FTAB reserves a bounded "cras" region. Capture it + * after a firmware abort; these private diagnostics may contain device data. + */ +static ssize_t firmware_crash_read(struct file *file, char __user *buffer, + size_t count, loff_t *pos) +{ + struct centauri *c = file->private_data; + struct cen_buffer *secondary; + u32 offset, length; + ssize_t ret; + + mutex_lock(&c->boot_lock); + if (!c->ipc || !c->ipc->secondary) { + ret = -ENODEV; + goto unlock; + } + secondary = c->ipc->secondary; + ret = ftab_entry(secondary->cpu, secondary->size, "cras", &offset, &length, false); + if (ret) + goto unlock; + dma_rmb(); + ret = simple_read_from_buffer(buffer, count, pos, secondary->cpu + offset, length); +unlock: + mutex_unlock(&c->boot_lock); + return ret; +} + +static const struct file_operations firmware_crash_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = firmware_crash_read, + .llseek = default_llseek, +}; + static int secondary_init(struct centauri *c) { u32 offset, length, count, i, required; @@ -112,6 +148,7 @@ static int secondary_init(struct centauri *c) pci_write_config_dword(c->pdev, 0xf8c, upper_32_bits(c->ipc->secondary->dma)); pci_write_config_dword(c->pdev, 0xf90, required); dev_info(&c->pdev->dev, "secondary memory: seed=%u required=%u\n", length, required); + debugfs_create_file("firmware_crash", 0400, c->debug, c, &firmware_crash_fops); return 0; } From 25c1101b215a43446380889de2776704cd297ec9 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 10 Sep 2026 02:19:42 -0400 Subject: [PATCH 0149/1579] arm64: add J813 RAM desktop bring-up and retained N1 test identity Path-scoped import of the N1 driver changes. J813 workbench scripts, board DT and private calibration staging are deliberately excluded. Original-commit: 5166f70674ecfde15685090f0672c5eea23b7ace Source-repository: https://github.com/aurora-silicon/linux-aurora --- drivers/bluetooth/btcentauri.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/drivers/bluetooth/btcentauri.c b/drivers/bluetooth/btcentauri.c index d2539539fa162..fb5c9ca89f6e0 100644 --- a/drivers/bluetooth/btcentauri.c +++ b/drivers/bluetooth/btcentauri.c @@ -9,6 +9,8 @@ */ #include #include +#include +#include #include #include #include @@ -55,6 +57,10 @@ static bool native_startup; module_param(native_startup, bool, 0444); MODULE_PARM_DESC(native_startup, "Diagnostic: program a temporary address, InitDone and random seed before the first HCI reset"); +static char *native_bdaddr; +module_param(native_bdaddr, charp, 0400); +MODULE_PARM_DESC(native_bdaddr, "Optional stable diagnostic address in XX:XX:XX:XX:XX:XX order; does not provision the factory address"); + static bool enable_acl; module_param(enable_acl, bool, 0444); MODULE_PARM_DESC(enable_acl, "Experimental ACL data delivery; requires LE and auxiliary rings"); @@ -657,14 +663,20 @@ static int cen_hci_setup(struct hci_dev *hdev) /* Native PROXIMA initialization starts with these three commands, then * HCI Reset. The core calls setup before its ordinary reset/init sequence. - * Address provisioning is still diagnostic: use a temporary locally - * administered address, in native WriteBdAddr display/network byte order. + * Address provisioning is still diagnostic. A saved test identity permits + * bond reuse; otherwise generate a temporary locally administered address. + * Native WriteBdAddr takes display/network byte order in either case. */ - ret = get_random_bytes_wait(address, sizeof(address)); + if (native_bdaddr && (!mac_pton(native_bdaddr, address) || + is_zero_ether_addr(address) || is_broadcast_ether_addr(address))) + return -EINVAL; + ret = get_random_bytes_wait(seed, sizeof(seed)); if (ret) return ret; - address[0] = (address[0] & 0xfc) | 0x02; - get_random_bytes(seed, sizeof(seed)); + if (!native_bdaddr) { + get_random_bytes(address, sizeof(address)); + address[0] = (address[0] & 0xfc) | 0x02; + } memset(init_done, 0xff, sizeof(init_done)); ret = cen_vendor_command(hdev, 0xfc06, address, sizeof(address)); if (ret) @@ -683,7 +695,8 @@ static int cen_hci_setup(struct hci_dev *hdev) bt_dev_err(hdev, "Native startup stopped after %u commands: %d", bt->native_startup_steps, ret); else - bt_dev_info(hdev, "Native startup accepted; temporary diagnostic address in use"); + bt_dev_info(hdev, "Native startup accepted; %s diagnostic address in use", + native_bdaddr ? "configured" : "temporary"); return ret; } From 2297612f57ca32ce948ab945ebd894c938591b94 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Sun, 13 Sep 2026 10:11:02 -0400 Subject: [PATCH 0150/1579] arm64: apple: enable J813 SMC temperatures and CPU cooling Wire the measured J813 SMC mailbox and temperature keys into hwmon and thermal zones. Add T8142 CPU performance levels and command-register frequency control, with conservative passive cooling maps. Provide a separate RAM image that starts monitoring with WFI idle and minimum CPU clock caps. Physical P/E cooling callbacks and real-sensor passive trip activation passed; a fresh boot enabled all 43 channels automatically. Full desktop handoff and sustained-load thermal qualification remain pending. Port scope: cpufreq driver only. Board-specific DT and desktop tools excluded. Original-commit: 08fdca6772eb6a27be6291bdec0ced7a937a4cf0 --- drivers/cpufreq/apple-soc-cpufreq.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 60935032919d2..887f94ac32d67 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -112,7 +112,24 @@ static const struct apple_soc_cpufreq_info soc_default_info = { .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, }; +/* T8142 retains PS1 in bits 4:0 and SET in bit 25, as used by its + * firmware command journal. Its P-cluster has 19 ADT operating points. + * The status-register layout is not established: report the commanded + * frequency through the existing fallback, without claiming measured MHz. + */ +static const struct apple_soc_cpufreq_info soc_t8142_info = { + .has_ps2 = false, + .max_pstate = 31, + .cur_pstate_mask = 0, + .ps1_mask = APPLE_DVFS_CMD_PS1, + .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, +}; + static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { + { + .compatible = "apple,t8142-cluster-cpufreq", + .data = &soc_t8142_info, + }, { .compatible = "apple,s5l8960x-cluster-cpufreq", .data = &soc_s5l8960x_info, From 5698e1f20afd85b3ff319ef912cb16c6fde8ef3c Mon Sep 17 00:00:00 2001 From: Acelogic Date: Wed, 23 Sep 2026 19:08:18 -0400 Subject: [PATCH 0151/1579] arm64: apple: add J873 thermal and framebuffer desktop bring-up Port scope: cpufreq driver only. Board-specific DT and desktop tools excluded. Original-commit: 44b5ac090b1b1af7b34cda852d4c502ab0b65ed3 --- drivers/cpufreq/apple-soc-cpufreq.c | 34 ++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 887f94ac32d67..acea86eec1d56 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -62,6 +62,8 @@ struct apple_soc_cpufreq_info { bool has_ps2; + bool verify_completion; + unsigned int transition_timeout; u64 max_pstate; u64 cur_pstate_mask; u64 cur_pstate_shift; @@ -125,7 +127,24 @@ static const struct apple_soc_cpufreq_info soc_t8142_info = { .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, }; +/* T8152 command completion/readback is qualified on both E and P/M domains. + * The status-register layout is not established; get_rate reports requests. + */ +static const struct apple_soc_cpufreq_info soc_t8152_info = { + .has_ps2 = false, + .verify_completion = true, + .transition_timeout = 2000, + .max_pstate = 31, + .cur_pstate_mask = 0, + .ps1_mask = APPLE_DVFS_CMD_PS1, + .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, +}; + static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { + { + .compatible = "apple,t8152-cluster-cpufreq", + .data = &soc_t8152_info, + }, { .compatible = "apple,t8142-cluster-cpufreq", .data = &soc_t8142_info, @@ -190,15 +209,20 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, { struct apple_cpu_priv *priv = policy->driver_data; unsigned int pstate = policy->freq_table[index].driver_data; + unsigned int timeout = priv->info->transition_timeout ?: + APPLE_DVFS_TRANSITION_TIMEOUT; u64 reg; + if (priv->info->verify_completion && pstate > priv->info->max_pstate) + return -EINVAL; + /* Fallback for newer SoCs */ if (index > priv->info->max_pstate) index = priv->info->max_pstate; if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, !(reg & APPLE_DVFS_CMD_BUSY), 2, - APPLE_DVFS_TRANSITION_TIMEOUT)) { + timeout)) { return -EIO; } @@ -211,6 +235,14 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, reg |= APPLE_DVFS_CMD_SET; writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD); + if (priv->info->verify_completion) { + if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, + !(reg & APPLE_DVFS_CMD_BUSY), 2, + timeout)) + return -ETIMEDOUT; + if (((reg & priv->info->ps1_mask) >> priv->info->ps1_shift) != pstate) + return -EIO; + } return 0; } From 72f5dcea3168bf5440068917b8c99d2e0682b5d3 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Wed, 23 Sep 2026 23:08:18 +0000 Subject: [PATCH 0152/1579] mfd: macsmc: retain J873 telemetry-only support Port only the SMC hunk from 44b5ac090b1b1af7b34cda852d4c502ab0b65ed3. Context adapted to omit unrelated J700 GPIO diagnostic code. Original author and author date retained; no T6050 qualification claimed. --- drivers/mfd/macsmc.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 31540f67ddbd8..399cee1c38a13 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -54,6 +54,11 @@ static const struct mfd_cell apple_smc_devs[] = { MFD_CELL_OF("macsmc-rtc", NULL, NULL, 0, 0, "apple,smc-rtc"), }; +/* Initial T8152 support is read-only telemetry. Firmware retains fan control. */ +static const struct mfd_cell apple_smc_telemetry_devs[] = { + MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), +}; + static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, u64 size, u64 wsize, u32 *ret_data) { @@ -469,6 +474,15 @@ static int apple_smc_probe(struct platform_device *pdev) return dev_err_probe(smc->dev, ret, "Failed to get key count"); smc->key_count = be32_to_cpu(count); + /* T8152's key-read interface is qualified; notifications, GPIO and power + * functions are not. The board DT exposes temperature and fan inputs only. + */ + if (of_device_is_compatible(dev->of_node, "apple,t8152-smc")) + return devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, + apple_smc_telemetry_devs, + ARRAY_SIZE(apple_smc_telemetry_devs), + NULL, 0, NULL); + /* Enable notifications */ apple_smc_write_flag(smc, SMC_KEY(NTAP), true); ret = devm_add_action_or_reset(dev, apple_smc_disable_notifications, smc); @@ -486,6 +500,7 @@ static int apple_smc_probe(struct platform_device *pdev) } static const struct of_device_id apple_smc_of_match[] = { + { .compatible = "apple,t8152-smc" }, { .compatible = "apple,t8103-smc" }, { .compatible = "apple,smc" }, {}, From 66b459998c5720a57ab8cb3a8bc5c2b62a4c6dfb Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 22:14:44 +0000 Subject: [PATCH 0153/1579] mfd: macsmc: add T6050 telemetry bring-up match Experimental board match for explicit T6050 SMC testing. Firmware retains fan control. No board DT enables this yet; SMC mailbox and shared-memory permissions must be qualified under SPTM before enabling the node. T6050 keys and notifications have not been qualified. --- drivers/mfd/macsmc.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 399cee1c38a13..bd9c4270854a8 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -477,7 +477,8 @@ static int apple_smc_probe(struct platform_device *pdev) /* T8152's key-read interface is qualified; notifications, GPIO and power * functions are not. The board DT exposes temperature and fan inputs only. */ - if (of_device_is_compatible(dev->of_node, "apple,t8152-smc")) + if (of_device_is_compatible(dev->of_node, "apple,t8152-smc") || + of_device_is_compatible(dev->of_node, "apple,t6050-smc")) return devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, apple_smc_telemetry_devs, ARRAY_SIZE(apple_smc_telemetry_devs), @@ -501,6 +502,7 @@ static int apple_smc_probe(struct platform_device *pdev) static const struct of_device_id apple_smc_of_match[] = { { .compatible = "apple,t8152-smc" }, + { .compatible = "apple,t6050-smc" }, { .compatible = "apple,t8103-smc" }, { .compatible = "apple,smc" }, {}, From 9c1cf27faece6bf3ccc126512fa0232b0f09effe Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 18:59:51 -0400 Subject: [PATCH 0154/1579] mfd: macsmc: add explicit T6050 temperature discovery diagnostic The board must opt in via apple,t6050-temperature-discovery. Enumerate only readable non-function T keys, log raw data with its type and size, and stop on any transaction error. No key writes, fan settings, or automatic discovery in a production device tree. This enables measured hwmon DT construction. This is an untested diagnostic, not a production thermal policy. --- drivers/mfd/macsmc.c | 50 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index bd9c4270854a8..d51e193dd6fde 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -412,6 +412,48 @@ static void apple_smc_disable_notifications(void *data) apple_smc_write_flag(smc, SMC_KEY(NTAP), false); } +/* One-shot, DT-opt-in inventory; never writes a key or calls a function key. */ +static int apple_smc_t6050_discover_temperatures(struct apple_smc *smc) +{ + unsigned int i, found = 0; + + /* A corrupt count must not turn bring-up into an unbounded firmware walk. */ + if (smc->key_count > 8192) + return -E2BIG; + + for (i = 0; i < smc->key_count; i++) { + struct apple_smc_key_info info; + smc_key key; + u8 value[8] = {}; + int ret; + + ret = apple_smc_get_key_by_index(smc, i, &key); + if (ret < 0) + return ret; + if ((key >> 24) != 'T') + continue; + ret = apple_smc_get_key_info(smc, key, &info); + if (ret < 0) + return ret; + if (!(info.flags & APPLE_SMC_READABLE) || + (info.flags & APPLE_SMC_FUNCTION) || + !info.size || info.size > sizeof(value)) + continue; + ret = apple_smc_read(smc, key, value, info.size); + if (ret < 0) + return ret; + if (ret != info.size) + return -EIO; + dev_info(smc->dev, + "T6050_SMC_TEMP: key=%08x type=%08x flags=%02x size=%u raw=%*phN\n", + key, info.type_code, info.flags, info.size, info.size, value); + found++; + } + dev_info(smc->dev, "T6050_SMC_DISCOVERY: keys=%u temperatures=%u key_writes=0\n", + smc->key_count, found); + return 0; +} + static int apple_smc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -474,6 +516,14 @@ static int apple_smc_probe(struct platform_device *pdev) return dev_err_probe(smc->dev, ret, "Failed to get key count"); smc->key_count = be32_to_cpu(count); + if (of_device_is_compatible(dev->of_node, "apple,t6050-smc") && + of_property_read_bool(dev->of_node, "apple,t6050-temperature-discovery")) { + ret = apple_smc_t6050_discover_temperatures(smc); + if (ret) + return dev_err_probe(dev, ret, "SMC temperature discovery failed"); + } + + /* T8152's key-read interface is qualified; notifications, GPIO and power * functions are not. The board DT exposes temperature and fan inputs only. */ From 19e1f925c534082211724886f6be4cfc178df949 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 22:14:44 +0000 Subject: [PATCH 0155/1579] cpufreq: apple-soc: add T6050 qualification candidate Explicit T6050-compatible test candidate, not enabled by a board DT. The ADT has three CPU performance domains and up to 20 states. The inherited 5-bit request protocol, completion behavior, and timeout remain unverified on this machine. Readback reports requests, not measured MHz. Do not promote this candidate before hardware qualification. No existing SoC path changes. --- drivers/cpufreq/apple-soc-cpufreq.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index acea86eec1d56..9250f0bad891c 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -140,7 +140,22 @@ static const struct apple_soc_cpufreq_info soc_t8152_info = { .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, }; +/* T6050 bring-up candidate: commanded frequency only, pending HW validation. */ +static const struct apple_soc_cpufreq_info soc_t6050_info = { + .has_ps2 = false, + .verify_completion = true, + .transition_timeout = 2000, + .max_pstate = 31, + .cur_pstate_mask = 0, + .ps1_mask = APPLE_DVFS_CMD_PS1, + .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, +}; + static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { + { + .compatible = "apple,t6050-cluster-cpufreq", + .data = &soc_t6050_info, + }, { .compatible = "apple,t8152-cluster-cpufreq", .data = &soc_t8152_info, From 1bd7e0f023963392cf62a8f1bd5a8d53b7db139b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 19:25:05 -0400 Subject: [PATCH 0156/1579] docs: record T6050 hardware SMC and DVFS qualification --- .../arch/arm64/t6050-sptm-smc-dvfs.rst | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 Documentation/arch/arm64/t6050-sptm-smc-dvfs.rst diff --git a/Documentation/arch/arm64/t6050-sptm-smc-dvfs.rst b/Documentation/arch/arm64/t6050-sptm-smc-dvfs.rst new file mode 100644 index 0000000000000..7673b64cd34e0 --- /dev/null +++ b/Documentation/arch/arm64/t6050-sptm-smc-dvfs.rst @@ -0,0 +1,32 @@ +T6050 SMC telemetry and CPU frequency qualification +=================================================== + +The resident SPTM profile maps the SMC SRAM aperture directly and forwards +its mailbox accesses. The T6050 MFD match currently exposes telemetry only. +The measured 143 float temperature channels use literal SMC key names; +physical sensor names and active cooling policies have not been inferred. + +CPU frequency control uses directly mapped cluster command pages, with no +per-transition SPTM MMIO trap. Native macOS 27 firmware code establishes +64-bit commands, five-bit state selectors, SET bit 25 and BUSY bit 31. +The ADT performance-state index has a two-state hardware offset. The tested +M0/M1 ranges are 1.344--4.380 GHz and the P range is 1.308--4.608 GHz. +The driver's frequency readback reports the requested state; it is not an +independent clock measurement. + +J714s hardware qualification, 2026-09-27: + +* All 18 CPUs online and 143 temperature inputs with matching key labels. +* Each cluster executed 400 million dependent additions at min/max/min, + pinned respectively to CPU 0, 6 and 12, with correct checksums. +* M0: 298.083 / 91.426 / 297.860 ms (3.2604x work-rate change). +* M1: 297.943 / 91.385 / 297.848 ms (3.2603x work-rate change). +* P: 306.743 / 87.076 / 306.716 ms (3.5227x work-rate change). +* Ratios track the requested frequency ratios within 0.044%; restored-low + timing differs by less than 0.08%. + +This verifies real work-rate scaling, not just register readback. It does +not qualify every intermediate OPP, concurrent governor transitions, or +sustained thermal control. The short diagnostic left userspace/min active; +normal runs must select their intended governor after testing. Baseline +reported temperatures were 25.375--42.659 degrees C, sampled before load. From c20d1787df6b78316636ba90ac0f44299434b08f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 20:20:50 -0400 Subject: [PATCH 0157/1579] nvme: apple: wait for hardware flush in resident SPTM guests --- Documentation/arch/arm64/t6050-sptm-ans.rst | 33 +++++++++++++++++---- drivers/nvme/host/apple.c | 5 +++- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/Documentation/arch/arm64/t6050-sptm-ans.rst b/Documentation/arch/arm64/t6050-sptm-ans.rst index 4dd3530bc78a2..a00cfb5fb6156 100644 --- a/Documentation/arch/arm64/t6050-sptm-ans.rst +++ b/Documentation/arch/arm64/t6050-sptm-ans.rst @@ -11,9 +11,9 @@ payload data. CREATE_CQ/SQ reference their PRP1 queue page even with no block-layer payload. Completion invalidation must release SPTM DMA references before buffers may be reused. -DMA ranges translate the guest RAM IPA window to its actual physical backing. -SART receives DMA addresses and converts them back to guest physical before -submitting its own transport. Queue and RTKit grants are16KiB aligned. +The qualified full-RAM host uses identity guest IPA and native physical +addresses, with empty identity dma-ranges. The older diagnostic offset layout +is also supported by the transport. Queue and RTKit grants are16KiB aligned. The firmware must already be running; this profile does not own its reset. The obsolete linear-SQ control register is not written on T6050. @@ -29,5 +29,28 @@ Hardware qualification on J714s, macOS27.0 firmware26A428: Full logs and pinned binaries are retained in the workspace directory work/j714-mpidr-20260927. This is an initial resident profile; ordinary ANS -profiles and their wrapped-key crypto support are retained. No SSD writes or -filesystem changes were used for qualification. +profiles and their wrapped-key crypto support are retained. + +Write and flush qualification +----------------------------- + +The resident profile disables the legacy deferred-flush policy. A successful +flush now requires completion of the corresponding hardware NVMe FLUSH; +acknowledging a request early and scheduling it later cannot establish the +requested fdatasync durability. Other profiles retain their existing policy. + +On 2026-09-27, Linux with all18 CPUs and59.436GiB of guest RAM created a new +1GiB test file on the existing ALARM_ROOT ext4 filesystem, using O_DIRECT and +fdatasync. It verified the entire offset-dependent pattern, overwrote alternating +1MiB ranges, synced and verified again, and unmounted the filesystem. After a +hardware reboot, a read-only/no-journal-replay mount verified every byte of the +final pattern. Existing user files and boot files were not overwritten. + +The initial1GiB pattern generation, write and fdatasync took0.198547 seconds +(5157.46MiB/s). This is a bounded sample, not sustained throughput or a native +comparison. Reboot persistence passed; sudden power-loss behavior was not tested. +Earlier timings with deferred flush enabled do not measure durable write speed. +Evidence: hardware-linux-ans-durable-guest.log and +hardware-linux-ans-durable-persist-guest.log under the workspace directory above. +Kernel Image SHA256: +019d9c370014fc726df4b39ef1f2218e36c0ca5e8bec3177e08b96c9945cc233. diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 5736a6c399461..d97224a5dcb12 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -2177,7 +2177,10 @@ static int apple_nvme_probe(struct platform_device *pdev) goto out_uninit_ctrl; } - if (flush_interval) { + /* A resident guest must not report a successful flush before firmware + * has completed it. The legacy deferred-flush policy acknowledges some + * requests early and cannot provide the required fsync durability. */ + if (flush_interval && !anv->hw->sptm_resident) { anv->flush_interval = msecs_to_jiffies(flush_interval); anv->flush_ns = NULL; anv->last_flush = jiffies - anv->flush_interval; From 43b2038049b62f9f36084b6fbf5180815c9d8307 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 21:29:00 -0400 Subject: [PATCH 0158/1579] iommu: support native T6050 N1 stream ownership and DMA aperture --- Documentation/arch/arm64/t6050-sptm-n1.rst | 74 ++ drivers/iommu/apple-dart-x1n1.c | 703 ++++++++++++++---- .../wireless/apple/centauri/centauri_pci.c | 31 +- 3 files changed, 674 insertions(+), 134 deletions(-) create mode 100644 Documentation/arch/arm64/t6050-sptm-n1.rst diff --git a/Documentation/arch/arm64/t6050-sptm-n1.rst b/Documentation/arch/arm64/t6050-sptm-n1.rst new file mode 100644 index 0000000000000..998da4ce3e744 --- /dev/null +++ b/Documentation/arch/arm64/t6050-sptm-n1.rst @@ -0,0 +1,74 @@ +T6050 N1 DMA under x1n1 +====================== + +J714s hardware qualification +--------------------------- + +A Linux EL1 guest with all 18 CPUs and 59.436 GiB usable RAM successfully +uploaded the personalized J714s N1 FTAB through the Control PCI function on +2026-09-27. The ROM returned response 1, advanced from stage 0 to stage 4, +and wrote MSI data 0xa5a0 into the diagnostic DMA mailbox. This verifies actual +PCIe device reads and writes through the SPTM-managed DART, beyond API-only +map/unmap tests. Native interrupt delivery and operational Wi-Fi are separate, +unfinished steps; the inherited polled firmware prototype is not a qualified +full-performance networking implementation. + +The test uses a preinitialized native T6050 PCIe controller, explicit bus1 +RID 0x100/0x101/0x102 to SID1/2/3 routing, and direct ECAM/BAR stage-2 windows. +The controller needs both GPIO PERST and its independent native SMC reset +release. After this reset operation, AP and IOP must both acknowledge RTKit +QUIESCED before Linux starts a fresh SMC session. + +Native DMA contract +------------------- + +DART id1 uses a three-level stream (initial SPTM table level1). Device DMA +addresses are low; vm-force bit40 is added only in the SPTM transport and +removed before software mapping bookkeeping. The Centauri 40-bit DMA mask +therefore remains valid. Native IODARTMapper strips this bit from returned +addresses and restores it for map/unmap operations. + +ADT vm-base is 0x10000100000, vm-size is 0x3fe00000. Both SPTM MAP and UNMAP +require the exclusive mapping end to be strictly below the SID upper page +bound. Native AppleT8110DART also checks page+1 < upper_bound. Reserve the +last 16 KiB page: device window [0x100000, 0x3fefc000), size 0x3fdfc000:: + + apple,dma-range = <0 0x100000 0 0x3fdfc000>; + apple,x1n1-iova-force = <0x100 0>; + +Using the full raw ADT extent caused an SPTM INVALID_DVA panic at +0x1003ff00000 on the first coherent allocation. The guard matches native +validation; it is not an arbitrary reduced DMA aperture. + +The batch ABI carries Linux IOMMU_READ/WRITE/CACHE allow bits. x1n1 must +invert these three bits for SPTM NO_READ/NO_WRITE/NO_CACHE. Forwarding the +Linux bits unchanged installs inaccessible mappings: calls complete, but +N1 ROM image load returns terminal failure5. After the conversion, the +identical firmware and guest kernel passed ROM upload. + +Mapping ownership +----------------- + +PCI topology and shared stream IDs determine IOMMU groups. The driver records +live mappings for address lookup and replay when an additional SID attaches +to an existing domain. It prepares tables before enabling streams and rolls +back incomplete mapping cohorts on failure. Normal map/unmap retains full +255-operation transport batches; replay scans happen on stream attachment, +not on every I/O. The optional forced IOVA transform touches only bounded +transport batches and restores arguments before completion bookkeeping. + +Validation and artifacts +------------------------ + +The DART driver compiled in the normal Asahi-targeted kernel build. Extracted +C tests covered grouping/replay, partial failures in multi-batch 256-SID +cohorts, and forced-address restoration at every completion position under +ASan/UBSan. The stage-2 bundle passed a focused QEMU boot/mapping smoke test +before hardware. QEMU does not establish N1 DMA behavior. + +Hardware evidence in the local workspace: +``work/j714-mpidr-20260927/hardware-linux-n1-rom-3-guest.log``. +Kernel ``Image-n1-force`` SHA256: +``02df14333ad13e8f44d76332eadb159b0437501df6d11b3a39cbc677db3f73b8``. +Stage2 bundle ``candidate-n1-dart-permissions``; target DT +``t6050-n1-guard-enabled.dtb``. Firmware/calibration stays private. diff --git a/drivers/iommu/apple-dart-x1n1.c b/drivers/iommu/apple-dart-x1n1.c index 4f68c29b7a485..8654fd8e1348e 100644 --- a/drivers/iommu/apple-dart-x1n1.c +++ b/drivers/iommu/apple-dart-x1n1.c @@ -36,7 +36,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -97,8 +99,12 @@ struct apple_dart_x1n1 { void __iomem *mmio; u32 dart_id; u32 num_streams; + u64 iova_force; /* Native vm-force, hidden from DMA API/device addresses. */ + u64 dma_min, dma_max; bool inited; struct mutex init_lock; /* serialises one-time init */ + /* Protected by apple_dart_x1n1_groups_lock; group-owned lifetime. */ + struct iommu_group *sid2group[DART_X1N1_MAX_STREAMS]; struct x1n1_batch *batch; /* linear-map buffer, guest-PA stable */ phys_addr_t batch_pa; @@ -119,6 +125,7 @@ struct apple_dart_x1n1_domain { * block = dva >> 25. Installing a level-2 table twice is fatal in SPTM, * so each 32 MiB window is attached at most once. */ struct xarray l2_attached; + struct xarray mappings; /* tagged (guest-PFN, protection), indexed by IOVA page */ }; static struct apple_dart_x1n1_domain *to_x1n1_domain(struct iommu_domain *dom) @@ -126,6 +133,30 @@ static struct apple_dart_x1n1_domain *to_x1n1_domain(struct iommu_domain *dom) return container_of(dom, struct apple_dart_x1n1_domain, domain); } +/* Native IODARTMapper strips vm-force before returning a device DMA address + * and restores it for DART operations. Translate only the bounded transport + * batch; all live-map bookkeeping and the DMA aperture remain device-visible. + * The force bits are disjoint from the validated low-4-GiB aperture. */ +static void x1n1_force_iovas(struct apple_dart_x1n1 *dart, bool restore) +{ + if (!dart->iova_force) + return; + for (u32 i = 0; i < dart->batch->count; i++) { + struct x1n1_op *op = &dart->batch->op[i]; + + switch (op->op) { + case X1N1_DART_ATTACH: + case X1N1_DART_MAP: + case X1N1_DART_UNMAP: + if (restore) + op->arg[1] &= ~dart->iova_force; + else + op->arg[1] |= dart->iova_force; + break; + } + } +} + /* * Ring a doorbell: publish the batch, write its guest-physical address to the * doorbell, and return the batch status. The doorbell write world-switches to @@ -140,6 +171,7 @@ static int x1n1_ring(struct apple_dart_x1n1 *dart, u32 db_off) dart->batch->status = 0; dart->batch->done = 0; + x1n1_force_iovas(dart, false); /* Order the batch stores before the doorbell; x1n1 does dsb on its side. */ wmb(); @@ -153,7 +185,9 @@ static int x1n1_ring(struct apple_dart_x1n1 *dart, u32 db_off) : "x9", "memory"); local_irq_restore(flags); - return dart->batch->status; + rmb(); + x1n1_force_iovas(dart, true); + return dart->batch->status ?: (dart->batch->done == dart->batch->count ? 0 : -EIO); } /* Bring the DART up in SPTM exactly once, before any stream is enabled. */ @@ -188,7 +222,7 @@ static int apple_dart_x1n1_ensure_init(struct apple_dart_x1n1 *dart) static void apple_dart_x1n1_selftest(struct apple_dart_x1n1 *dart) { const u32 sid = 1; - const u64 dva = 0x10000004000ULL; /* the IOVA the transport was tested with */ + const u64 dva = dart->iova_force ? dart->dma_min : 0x10000004000ULL; /* the IOVA the transport was tested with */ unsigned long flags; void *page; phys_addr_t pa; @@ -233,6 +267,191 @@ static void apple_dart_x1n1_selftest(struct apple_dart_x1n1 *dart) free_page((unsigned long)page); } +/* Live mappings are indexed by IOVA page. Normal map/unmap only touches the + * requested range; a complete walk is needed solely when adding a new SID. + * Zero tagged values reserve XArray storage before any hardware mapping. */ +#define X1N1_MAP_PROT_BITS 8 + +static unsigned long x1n1_mapping_value(phys_addr_t pa, int prot) +{ + return ((pa >> DART_X1N1_PAGE_SHIFT) << X1N1_MAP_PROT_BITS) | prot; +} + +static int x1n1_reserve_mappings(struct apple_dart_x1n1_domain *dom, + unsigned long iova, size_t size, gfp_t gfp) +{ + unsigned long first = iova >> DART_X1N1_PAGE_SHIFT; + unsigned long pages = size >> DART_X1N1_PAGE_SHIFT; + unsigned long page; + int ret; + + for (page = 0; page < pages; page++) { + ret = xa_insert(&dom->mappings, first + page, xa_mk_value(0), gfp); + if (ret) + goto undo; + } + return 0; +undo: + while (page) + xa_erase(&dom->mappings, first + --page); + return ret; +} + +static void x1n1_release_reservations(struct apple_dart_x1n1_domain *dom, + unsigned long iova, size_t size) +{ + unsigned long first = iova >> DART_X1N1_PAGE_SHIFT; + unsigned long pages = size >> DART_X1N1_PAGE_SHIFT; + + for (unsigned long page = 0; page < pages; page++) + if (xa_load(&dom->mappings, first + page) == xa_mk_value(0)) + xa_erase(&dom->mappings, first + page); +} + +static void x1n1_record_mapping(struct apple_dart_x1n1_domain *dom, + u64 iova, u64 pa, size_t size, int prot) +{ + for (size_t off = 0; off < size; off += DART_X1N1_PAGE_SIZE) { + unsigned long index = (iova + off) >> DART_X1N1_PAGE_SHIFT; + void *entry = xa_mk_value(x1n1_mapping_value(pa + off, prot)); + + /* Reserved entries retain their nodes, so replacing a reservation + * requires no allocation, even in this IRQ-disabled section. */ + WARN_ON(xa_err(xa_store(&dom->mappings, index, entry, GFP_NOWAIT))); + } +} + +static void x1n1_forget_mapping(struct apple_dart_x1n1_domain *dom, + u64 iova, size_t size) +{ + for (size_t off = 0; off < size; off += DART_X1N1_PAGE_SIZE) + xa_erase(&dom->mappings, (iova + off) >> DART_X1N1_PAGE_SHIFT); +} + +static int x1n1_prepare_leaf(struct apple_dart_x1n1_domain *dom, + unsigned int sid, u64 iova) +{ + struct apple_dart_x1n1 *dart = dom->dart; + unsigned long key = ((unsigned long)sid << 40) | + (iova >> DART_X1N1_BLOCK_SHIFT); + int ret; + + if (xa_load(&dom->l2_attached, key)) + return 0; + ret = xa_insert(&dom->l2_attached, key, xa_mk_value(0), GFP_ATOMIC); + if (ret) + return ret; + dart->batch->count = 1; + dart->batch->op[0] = (struct x1n1_op){ + .op = X1N1_DART_ATTACH, .arg = { sid, iova, 2 }, + }; + ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + if (ret) { + xa_erase(&dom->l2_attached, key); + return ret; + } + WARN_ON(xa_err(xa_store(&dom->l2_attached, key, xa_mk_value(1), GFP_NOWAIT))); + return 0; +} + +/* Return the next physically contiguous, same-protection run, stopping at a + * leaf-table boundary. Reservations are not yet live and are skipped. */ +static bool x1n1_next_mapping(struct apple_dart_x1n1_domain *dom, + unsigned long *cursor, struct x1n1_op *op, + unsigned int sid) +{ + unsigned long index = *cursor, value, first; + void *entry; + u64 iova, pa; + size_t size; + int prot; + + for (;;) { + entry = xa_find(&dom->mappings, &index, ULONG_MAX, XA_PRESENT); + if (!entry) + return false; + value = xa_to_value(entry); + if (value) + break; + index++; + } + first = index; + prot = value & ((1UL << X1N1_MAP_PROT_BITS) - 1); + pa = (value >> X1N1_MAP_PROT_BITS) << DART_X1N1_PAGE_SHIFT; + iova = (u64)first << DART_X1N1_PAGE_SHIFT; + size = DART_X1N1_PAGE_SIZE; + while ((iova + size) & (DART_X1N1_BLOCK_SIZE - 1)) { + entry = xa_load(&dom->mappings, index + 1); + if (entry != xa_mk_value(x1n1_mapping_value(pa + size, prot))) + break; + index++; + size += DART_X1N1_PAGE_SIZE; + } + *cursor = index + 1; + *op = (struct x1n1_op){ + .op = X1N1_DART_MAP, .arg = { sid, iova, pa, size, prot }, + }; + return true; +} + +static void x1n1_undo_replay(struct apple_dart_x1n1_domain *dom, + unsigned int sid, u64 mapped_end) +{ + struct apple_dart_x1n1 *dart = dom->dart; + unsigned long cursor = 0; + struct x1n1_op op; + + while (x1n1_next_mapping(dom, &cursor, &op, sid) && op.arg[1] < mapped_end) { + dart->batch->count = 1; + dart->batch->op[0] = (struct x1n1_op){ + .op = X1N1_DART_UNMAP, + .arg = { sid, op.arg[1], min_t(u64, op.arg[3], mapped_end - op.arg[1]) }, + }; + if (x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id)) + dev_err(dart->dev, "failed to undo disabled SID %u replay\n", sid); + } +} + +/* New SID is disabled throughout replay. On failure, keep it disabled and + * remove the successful replay prefix. Existing SIDs/maps stay untouched. */ +static int x1n1_replay_sid(struct apple_dart_x1n1_domain *dom, unsigned int sid) +{ + struct apple_dart_x1n1 *dart = dom->dart; + unsigned long cursor = 0; + struct x1n1_op op; + u64 mapped_end = 0; + u32 n = 0; + int ret; + + while (x1n1_next_mapping(dom, &cursor, &op, sid)) { + ret = x1n1_prepare_leaf(dom, sid, op.arg[1]); + if (ret) + return ret; + } + cursor = 0; + for (;;) { + bool have = x1n1_next_mapping(dom, &cursor, &op, sid); + + if (have) + dart->batch->op[n++] = op; + if (n == X1N1_BATCH_MAX_OPS || (!have && n)) { + dart->batch->count = n; + ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + for (u32 i = 0; i < min(dart->batch->done, n); i++) + mapped_end = dart->batch->op[i].arg[1] + + dart->batch->op[i].arg[3]; + if (ret) + goto undo; + n = 0; + } + if (!have) + return 0; + } +undo: + x1n1_undo_replay(dom, sid, mapped_end); + return ret; +} + static int apple_dart_x1n1_attach_dev(struct iommu_domain *domain, struct device *dev, struct iommu_domain *old) @@ -242,7 +461,7 @@ static int apple_dart_x1n1_attach_dev(struct iommu_domain *domain, struct apple_dart_x1n1 *dart; unsigned long flags; unsigned int sid; - u32 n; + DECLARE_BITMAP(added_sids, DART_X1N1_MAX_STREAMS); int ret; if (!master) @@ -259,38 +478,112 @@ static int apple_dart_x1n1_attach_dev(struct iommu_domain *domain, return -EINVAL; } dom->dart = dart; + domain->geometry.aperture_start = dart->dma_min; + domain->geometry.aperture_end = dart->dma_max; - /* For each stream: install the level-1 root table and enable translation. - * The level-2 leaf tables are attached lazily per 32 MiB window in - * map_pages (installing a level-2 table needs its level-1 parent - * present). x1n1 owns the tables. Build the ops in one batch. */ + /* A new stream must see all existing mappings before it can DMA. */ spin_lock_irqsave(&dart->batch_lock, flags); - n = 0; + ret = 0; + bitmap_zero(added_sids, DART_X1N1_MAX_STREAMS); for_each_set_bit(sid, master->sids, dart->num_streams) { if (test_bit(sid, dom->sids)) continue; - if (n + 2 > X1N1_BATCH_MAX_OPS) - break; - dart->batch->op[n++] = (struct x1n1_op){ - .op = X1N1_DART_ATTACH, - .arg = { sid, 0, 1 }, + dart->batch->count = 2; + dart->batch->op[0] = (struct x1n1_op){ + .op = X1N1_DART_DISABLE, .arg = { sid }, }; - dart->batch->op[n++] = (struct x1n1_op){ - .op = X1N1_DART_ENABLE, - .arg = { sid }, + dart->batch->op[1] = (struct x1n1_op){ + .op = X1N1_DART_ATTACH, .arg = { sid, dart->dma_min, 1 }, + }; + ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + if (ret) + break; + ret = x1n1_replay_sid(dom, sid); + if (ret) + break; + dart->batch->count = 1; + dart->batch->op[0] = (struct x1n1_op){ + .op = X1N1_DART_ENABLE, .arg = { sid }, }; + ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + if (ret) { + /* A failed completion may still have enabled the SID. */ + dart->batch->count = 1; + dart->batch->op[0] = (struct x1n1_op){ + .op = X1N1_DART_DISABLE, .arg = { sid }, + }; + if (x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id)) + dev_err(dart->dev, "failed to disable new SID %u after enable error\n", sid); + x1n1_undo_replay(dom, sid, U64_MAX); + break; + } + set_bit(sid, dom->sids); + set_bit(sid, added_sids); + } + if (ret) { + /* A master can contain multiple SIDs. Undo those added by this + * failed attachment without disturbing streams already in use. */ + for_each_set_bit(sid, added_sids, dart->num_streams) { + dart->batch->count = 1; + dart->batch->op[0] = (struct x1n1_op){ + .op = X1N1_DART_DISABLE, .arg = { sid }, + }; + if (x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id)) + dev_err(dart->dev, "failed to disable new SID %u after attachment error\n", sid); + x1n1_undo_replay(dom, sid, U64_MAX); + clear_bit(sid, dom->sids); + } } - dart->batch->count = n; - ret = n ? x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id) : 0; - if (!ret) - for_each_set_bit(sid, master->sids, dart->num_streams) - set_bit(sid, dom->sids); spin_unlock_irqrestore(&dart->batch_lock, flags); mutex_unlock(&dom->lock); return ret ? -EIO : 0; } +struct x1n1_map_progress { + size_t mapped; + unsigned int streams; + struct x1n1_op partial; +}; + +/* Track only fully installed stream cohorts. The batch may split one cohort + * without reducing batching; keep its progress across doorbells. */ +static int x1n1_flush_maps(struct apple_dart_x1n1_domain *dom, + struct x1n1_map_progress *progress) +{ + struct apple_dart_x1n1 *dart = dom->dart; + unsigned int total_streams = bitmap_weight(dom->sids, dart->num_streams); + int ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + + for (u32 i = 0; i < min(dart->batch->done, dart->batch->count); i++) { + struct x1n1_op *op = &dart->batch->op[i]; + + progress->partial = *op; + if (++progress->streams == total_streams) { + x1n1_record_mapping(dom, op->arg[1], op->arg[2], op->arg[3], op->arg[4]); + progress->mapped += op->arg[3]; + progress->streams = 0; + } + } + if (ret && progress->streams) { + unsigned int sid; + u32 n = 0; + + for_each_set_bit(sid, dom->sids, dart->num_streams) { + if (n == progress->streams) + break; + dart->batch->op[n++] = (struct x1n1_op){ + .op = X1N1_DART_UNMAP, + .arg = { sid, progress->partial.arg[1], progress->partial.arg[3] }, + }; + } + dart->batch->count = n; + if (x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id)) + dev_err(dart->dev, "failed to undo partial DART map\n"); + } + return ret; +} + static int apple_dart_x1n1_map_pages(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, @@ -298,97 +591,128 @@ static int apple_dart_x1n1_map_pages(struct iommu_domain *domain, { struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); struct apple_dart_x1n1 *dart = dom->dart; - size_t total = pgsize * pgcount; - size_t done = 0; + struct x1n1_map_progress progress = {}; + size_t total; unsigned long flags; + u32 n = 0; + int ret; - if (!dart) + *mapped = 0; + if (!dart || bitmap_empty(dom->sids, dart->num_streams)) return -ENODEV; - - /* - * A MAP needs the block's level-2 leaf table installed first, so first - * attach any not-yet-attached (sid, 32 MiB block) window, then map the - * range in per-stream chunks that each stay inside one aligned 32 MiB - * block (x1n1's contiguous-map constraint), packing MAP ops into a batch - * and ringing once per batch. Both phases hold batch_lock and reuse the - * one batch buffer, so the attaches are fully rung before any MAP ops - * are accumulated in it. - */ + if (pgsize != DART_X1N1_PAGE_SIZE || !pgcount || + check_mul_overflow(pgsize, pgcount, &total) || + (iova | paddr) & (DART_X1N1_PAGE_SIZE - 1) || + prot & ~((1U << X1N1_MAP_PROT_BITS) - 1) || + !paddr || iova < dart->dma_min || iova > dart->dma_max || + total - 1 > dart->dma_max - iova) + return -EINVAL; + ret = x1n1_reserve_mappings(dom, iova, total, gfp); + if (ret) + return ret; spin_lock_irqsave(&dart->batch_lock, flags); - - /* Phase 1: attach missing level-2 tables (one op per ring). */ - for (u64 pos = 0; pos < total; ) { - u64 cur_iova = iova + pos; - u64 block = cur_iova >> DART_X1N1_BLOCK_SHIFT; - size_t block_left = DART_X1N1_BLOCK_SIZE - - (cur_iova & (DART_X1N1_BLOCK_SIZE - 1)); + /* Install all leaf tables before accumulating the mapping batch. */ + for (size_t pos = 0; pos < total;) { + u64 cur = iova + pos; + size_t chunk = min_t(size_t, total - pos, + DART_X1N1_BLOCK_SIZE - (cur & (DART_X1N1_BLOCK_SIZE - 1))); unsigned int sid; for_each_set_bit(sid, dom->sids, dart->num_streams) { - unsigned long key = ((unsigned long)sid << 40) | block; + ret = x1n1_prepare_leaf(dom, sid, cur); + if (ret) + goto out; + } + pos += chunk; + } + for (size_t pos = 0; pos < total;) { + u64 cur = iova + pos; + size_t chunk = min_t(size_t, total - pos, + DART_X1N1_BLOCK_SIZE - (cur & (DART_X1N1_BLOCK_SIZE - 1))); + unsigned int sid; - if (xa_load(&dom->l2_attached, key)) - continue; - dart->batch->count = 1; - dart->batch->op[0] = (struct x1n1_op){ - .op = X1N1_DART_ATTACH, - .arg = { sid, cur_iova, 2 }, + for_each_set_bit(sid, dom->sids, dart->num_streams) { + dart->batch->op[n++] = (struct x1n1_op){ + .op = X1N1_DART_MAP, + .arg = { sid, cur, paddr + pos, chunk, prot }, }; - if (x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id)) { - *mapped = 0; - spin_unlock_irqrestore(&dart->batch_lock, flags); - return -ENOMEM; + if (n == X1N1_BATCH_MAX_OPS) { + dart->batch->count = n; + ret = x1n1_flush_maps(dom, &progress); + if (ret) + goto out; + n = 0; } - xa_store(&dom->l2_attached, key, xa_mk_value(1), - GFP_ATOMIC); } - pos += min_t(size_t, total - pos, block_left); + pos += chunk; } + if (n) { + dart->batch->count = n; + ret = x1n1_flush_maps(dom, &progress); + } +out: + spin_unlock_irqrestore(&dart->batch_lock, flags); + *mapped = progress.mapped; + /* Remove unused reservations after a partial failure; committed mappings + * remain available both to caller unwind and a later stream attachment. */ + if (ret) + x1n1_release_reservations(dom, iova + progress.mapped, total - progress.mapped); + return ret ? -EIO : 0; +} - /* Phase 2: map the range. */ - while (done < total) { - u32 n = 0; - int ret; - size_t batch_start = done; - - while (done < total && n < X1N1_BATCH_MAX_OPS) { - u64 cur_iova = iova + done; - u64 cur_pa = paddr + done; - size_t block_left = DART_X1N1_BLOCK_SIZE - - (cur_iova & (DART_X1N1_BLOCK_SIZE - 1)); - size_t chunk = min_t(size_t, total - done, block_left); - unsigned int sid; - bool full = false; - - for_each_set_bit(sid, dom->sids, dart->num_streams) { - if (n >= X1N1_BATCH_MAX_OPS) { - full = true; - break; - } - dart->batch->op[n++] = (struct x1n1_op){ - .op = X1N1_DART_MAP, - .arg = { sid, cur_iova, cur_pa, - chunk, prot }, - }; - } - if (full) +static int x1n1_restore_partial_unmap(struct apple_dart_x1n1_domain *dom, + const struct x1n1_op *partial, unsigned int streams) +{ + struct apple_dart_x1n1 *dart = dom->dart; + unsigned long cursor = partial->arg[1] >> DART_X1N1_PAGE_SHIFT; + u64 end = partial->arg[1] + partial->arg[2]; + struct x1n1_op op; + u32 n = 0; + + while (x1n1_next_mapping(dom, &cursor, &op, 0) && op.arg[1] < end) { + unsigned int sid, count = 0; + + op.arg[3] = min_t(u64, op.arg[3], end - op.arg[1]); + for_each_set_bit(sid, dom->sids, dart->num_streams) { + if (count++ == streams) break; - done += chunk; + op.arg[0] = sid; + dart->batch->op[n++] = op; + if (n == X1N1_BATCH_MAX_OPS) { + dart->batch->count = n; + if (x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id)) + return -EIO; + n = 0; + } } + } + if (!n) + return 0; + dart->batch->count = n; + return x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); +} - dart->batch->count = n; - ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); - if (ret) { - /* Report the bytes that fully mapped before this batch. */ - *mapped = batch_start; - spin_unlock_irqrestore(&dart->batch_lock, flags); - return -ENOMEM; +static int x1n1_flush_unmaps(struct apple_dart_x1n1_domain *dom, + struct x1n1_map_progress *progress) +{ + struct apple_dart_x1n1 *dart = dom->dart; + unsigned int total_streams = bitmap_weight(dom->sids, dart->num_streams); + int ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + + for (u32 i = 0; i < min(dart->batch->done, dart->batch->count); i++) { + struct x1n1_op *op = &dart->batch->op[i]; + + progress->partial = *op; + if (++progress->streams == total_streams) { + x1n1_forget_mapping(dom, op->arg[1], op->arg[2]); + progress->mapped += op->arg[2]; + progress->streams = 0; } } - spin_unlock_irqrestore(&dart->batch_lock, flags); - - *mapped = done; - return 0; + if (ret && progress->streams && + x1n1_restore_partial_unmap(dom, &progress->partial, progress->streams)) + dev_err(dart->dev, "failed to restore partial DART unmap\n"); + return ret; } static size_t apple_dart_x1n1_unmap_pages(struct iommu_domain *domain, @@ -398,46 +722,45 @@ static size_t apple_dart_x1n1_unmap_pages(struct iommu_domain *domain, { struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); struct apple_dart_x1n1 *dart = dom->dart; - size_t total = pgsize * pgcount; - size_t done = 0; + struct x1n1_map_progress progress = {}; + size_t total; unsigned long flags; - - if (!dart) + u32 n = 0; + + if (!dart || bitmap_empty(dom->sids, dart->num_streams) || + pgsize != DART_X1N1_PAGE_SIZE || !pgcount || + check_mul_overflow(pgsize, pgcount, &total) || + (iova & (DART_X1N1_PAGE_SIZE - 1)) || + iova < dart->dma_min || iova > dart->dma_max || + total - 1 > dart->dma_max - iova) return 0; - spin_lock_irqsave(&dart->batch_lock, flags); - while (done < total) { - u32 n = 0; + for (size_t pos = 0; pos < total;) { + u64 cur = iova + pos; + size_t chunk = min_t(size_t, total - pos, + DART_X1N1_BLOCK_SIZE - (cur & (DART_X1N1_BLOCK_SIZE - 1))); + unsigned int sid; - while (done < total && n < X1N1_BATCH_MAX_OPS) { - u64 cur_iova = iova + done; - size_t block_left = DART_X1N1_BLOCK_SIZE - - (cur_iova & (DART_X1N1_BLOCK_SIZE - 1)); - size_t chunk = min_t(size_t, total - done, block_left); - unsigned int sid; - bool full = false; - - for_each_set_bit(sid, dom->sids, dart->num_streams) { - if (n >= X1N1_BATCH_MAX_OPS) { - full = true; - break; - } - dart->batch->op[n++] = (struct x1n1_op){ - .op = X1N1_DART_UNMAP, - .arg = { sid, cur_iova, chunk }, - }; + for_each_set_bit(sid, dom->sids, dart->num_streams) { + dart->batch->op[n++] = (struct x1n1_op){ + .op = X1N1_DART_UNMAP, .arg = { sid, cur, chunk }, + }; + if (n == X1N1_BATCH_MAX_OPS) { + dart->batch->count = n; + if (x1n1_flush_unmaps(dom, &progress)) + goto out; + n = 0; } - if (full) - break; - done += chunk; } - + pos += chunk; + } + if (n) { dart->batch->count = n; - x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + x1n1_flush_unmaps(dom, &progress); } +out: spin_unlock_irqrestore(&dart->batch_lock, flags); - - return total; + return progress.mapped; } static void apple_dart_x1n1_iotlb_sync(struct iommu_domain *domain, @@ -450,10 +773,14 @@ static void apple_dart_x1n1_iotlb_sync(struct iommu_domain *domain, static phys_addr_t apple_dart_x1n1_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { - /* SPTM owns the DART tables; the guest does not shadow them, so a - * hardware-accurate walk is not available here. The DMA path does not - * need it. */ - return 0; + struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); + void *entry = xa_load(&dom->mappings, iova >> DART_X1N1_PAGE_SHIFT); + unsigned long value = entry ? xa_to_value(entry) : 0; + + if (!value) + return 0; + return ((value >> X1N1_MAP_PROT_BITS) << DART_X1N1_PAGE_SHIFT) | + (iova & (DART_X1N1_PAGE_SIZE - 1)); } static void apple_dart_x1n1_domain_free(struct iommu_domain *domain) @@ -461,6 +788,7 @@ static void apple_dart_x1n1_domain_free(struct iommu_domain *domain) struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); xa_destroy(&dom->l2_attached); + xa_destroy(&dom->mappings); kfree(dom); } @@ -475,6 +803,7 @@ apple_dart_x1n1_domain_alloc_paging(struct device *dev) mutex_init(&dom->lock); xa_init(&dom->l2_attached); + xa_init(&dom->mappings); dom->domain.pgsize_bitmap = DART_X1N1_PAGE_SIZE; dom->domain.geometry.aperture_start = 0; dom->domain.geometry.aperture_end = DART_X1N1_APERTURE_END; @@ -483,8 +812,11 @@ apple_dart_x1n1_domain_alloc_paging(struct device *dev) if (dev) { struct apple_dart_x1n1_master *master = dev_iommu_priv_get(dev); - if (master) + if (master) { dom->dart = master->dart; + dom->domain.geometry.aperture_start = master->dart->dma_min; + dom->domain.geometry.aperture_end = master->dart->dma_max; + } } return &dom->domain; @@ -549,6 +881,96 @@ static int apple_dart_x1n1_of_xlate(struct device *dev, return 0; } +/* Devices sharing a stream cannot have independent IOVA allocators. PCI + * multifunction/topology grouping is also required even with distinct SIDs. + * Mirrors the native Apple DART lifetime model: the group owns its SID union, + * and its release callback clears the reverse lookup before freeing it. */ +static DEFINE_MUTEX(apple_dart_x1n1_groups_lock); + +static void apple_dart_x1n1_release_group(void *data) +{ + struct apple_dart_x1n1_master *group_master = data; + unsigned int sid; + + mutex_lock(&apple_dart_x1n1_groups_lock); + for_each_set_bit(sid, group_master->sids, group_master->dart->num_streams) + group_master->dart->sid2group[sid] = NULL; + mutex_unlock(&apple_dart_x1n1_groups_lock); + kfree(group_master); +} + +static struct iommu_group *apple_dart_x1n1_device_group(struct device *dev) +{ + struct apple_dart_x1n1_master *master = dev_iommu_priv_get(dev); + struct apple_dart_x1n1_master *group_master; + struct iommu_group *group = NULL, *put_group = NULL; + struct iommu_group *result = ERR_PTR(-EINVAL); + unsigned int sid; + + if (!master || bitmap_empty(master->sids, master->dart->num_streams)) + return result; + + mutex_lock(&apple_dart_x1n1_groups_lock); + for_each_set_bit(sid, master->sids, master->dart->num_streams) { + struct iommu_group *existing = master->dart->sid2group[sid]; + + if (!existing) + continue; + if (group && existing != group) + goto out; + group = existing; + } + + if (group) { + group = iommu_group_ref_get(group); + } else { +#ifdef CONFIG_PCI + if (dev_is_pci(dev)) + group = pci_device_group(dev); + else +#endif + group = generic_device_group(dev); + } + if (IS_ERR(group)) { + result = group; + goto out; + } + if (!group) { + result = ERR_PTR(-ENOMEM); + goto out; + } + + /* A PCI topology group may already contain another function. */ + group_master = iommu_group_get_iommudata(group); + if (group_master && group_master->dart != master->dart) { + put_group = group; + goto out; + } + if (!group_master) { + group_master = kmemdup(master, sizeof(*master), GFP_KERNEL); + if (!group_master) { + result = ERR_PTR(-ENOMEM); + put_group = group; + goto out; + } + iommu_group_set_iommudata(group, group_master, + apple_dart_x1n1_release_group); + } else { + bitmap_or(group_master->sids, group_master->sids, master->sids, + master->dart->num_streams); + } + + for_each_set_bit(sid, master->sids, master->dart->num_streams) + master->dart->sid2group[sid] = group; + result = group; +out: + mutex_unlock(&apple_dart_x1n1_groups_lock); + /* Dropping the last reference can invoke release_group: do it unlocked. */ + if (put_group) + iommu_group_put(put_group); + return result; +} + static bool apple_dart_x1n1_capable(struct device *dev, enum iommu_cap cap) { return cap == IOMMU_CAP_CACHE_COHERENCY; @@ -559,7 +981,7 @@ static const struct iommu_ops apple_dart_x1n1_ops = { .domain_alloc_paging = apple_dart_x1n1_domain_alloc_paging, .probe_device = apple_dart_x1n1_probe_device, .release_device = apple_dart_x1n1_release_device, - .device_group = generic_device_group, + .device_group = apple_dart_x1n1_device_group, .of_xlate = apple_dart_x1n1_of_xlate, .owner = THIS_MODULE, .default_domain_ops = &(const struct iommu_domain_ops) { @@ -598,6 +1020,25 @@ static int apple_dart_x1n1_probe(struct platform_device *pdev) if (dart->dart_id > 0xff) return dev_err_probe(dev, -EINVAL, "dart-id out of range\n"); + dart->dma_max = DART_X1N1_APERTURE_END; + if (of_find_property(dev->of_node, "apple,dma-range", NULL)) { + u64 range[2], end; + + if (of_property_read_u64_array(dev->of_node, "apple,dma-range", range, 2) || + !range[1] || (range[0] | range[1]) & (DART_X1N1_PAGE_SIZE - 1) || + check_add_overflow(range[0], range[1] - 1, &end) || + end > DART_X1N1_APERTURE_END) + return dev_err_probe(dev, -EINVAL, "bad DMA range\n"); + dart->dma_min = range[0]; + dart->dma_max = end; + } + if (of_find_property(dev->of_node, "apple,x1n1-iova-force", NULL)) { + if (of_property_read_u64(dev->of_node, "apple,x1n1-iova-force", + &dart->iova_force) || + (dart->iova_force && dart->iova_force != BIT_ULL(40))) + return dev_err_probe(dev, -EINVAL, "unsupported IOVA force mask\n"); + } + dart->num_streams = DART_X1N1_MAX_STREAMS; of_property_read_u32(dev->of_node, "apple,x1n1-num-streams", &dart->num_streams); diff --git a/drivers/net/wireless/apple/centauri/centauri_pci.c b/drivers/net/wireless/apple/centauri/centauri_pci.c index f3f7c6413f0a3..8541f151b2803 100644 --- a/drivers/net/wireless/apple/centauri/centauri_pci.c +++ b/drivers/net/wireless/apple/centauri/centauri_pci.c @@ -61,22 +61,46 @@ static void centauri_free_boot(struct centauri *c) c->firmware = c->descriptor = c->msi_page = NULL; } +struct centauri_host_profile { + const char *compatible; + const char *firmware; + u32 platform_id; +}; + +static const struct centauri_host_profile centauri_host_profiles[] = { + { "apple,j813", "apple/centauri/j813-ftab.bin", 0x00248142 }, + /* Native CentauriBooter packs chip[15:0] and board[14:0] at bit16. + * J714s ADT: chip0x6050, board8; use its personalized firmware. */ + { "apple,j714s", "apple/centauri/j714s-ftab.bin", 0x00086050 }, +}; + +static const struct centauri_host_profile *centauri_host_profile(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(centauri_host_profiles); i++) + if (of_machine_is_compatible(centauri_host_profiles[i].compatible)) + return ¢auri_host_profiles[i]; + return NULL; +} + static int centauri_rom_boot(struct centauri *c) { const struct firmware *fw; + const struct centauri_host_profile *profile = centauri_host_profile(); struct device *dev = &c->pdev->dev; u32 stage; int ret; if ((c->pdev->device != 0x1900 && c->pdev->device != 0x1901) || - !of_machine_is_compatible("apple,j813")) + !profile) return -ENODEV; if (!c->link_ready || c->boot_attempted) return -EBUSY; stage = readl(c->bar + 0x8000); if (stage != 0 && stage != 4 && stage != 1) return -EBUSY; - ret = request_firmware(&fw, "apple/centauri/j813-ftab.bin", dev); + ret = request_firmware(&fw, profile->firmware, dev); if (ret) return ret; if (fw->size < 0x30 || fw->size > SZ_64M || @@ -116,7 +140,7 @@ static int centauri_rom_boot(struct centauri *c) pci_write_config_word(c->pdev, c->msi_cap + PCI_MSI_FLAGS, (c->msi_original & ~PCI_MSI_FLAGS_QSIZE) | PCI_MSI_FLAGS_ENABLE); pci_set_master(c->pdev); - writel(0x00248142, c->bar + 0x807c); /* J813 board 0x24, T8142 chip. */ + writel(profile->platform_id, c->bar + 0x807c); writel(0, c->bar + 0x803c); writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); writel(upper_32_bits(c->descriptor_dma), c->bar + 0x8048); @@ -276,3 +300,4 @@ module_exit(centauri_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Apple N1 Centauri native PCI and ACIPC bring-up backend"); MODULE_FIRMWARE("apple/centauri/j813-ftab.bin"); +MODULE_FIRMWARE("apple/centauri/j714s-ftab.bin"); From 04b8bc228967889816ea390ed41bf55a19fb1f82 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 21:54:34 -0400 Subject: [PATCH 0159/1579] PCI: bring up T6050 N1 runtime with native MSI and port reset --- Documentation/arch/arm64/t6050-sptm-n1.rst | 63 +++- drivers/bluetooth/btcentauri.c | 65 ++++- drivers/iommu/apple-dart-x1n1.c | 62 ++++ .../net/wireless/apple/centauri/centauri.h | 21 +- .../wireless/apple/centauri/centauri_ipc.c | 119 +++++--- .../wireless/apple/centauri/centauri_pci.c | 99 +++++-- .../wireless/apple/centauri/centauri_wifi.c | 47 ++- drivers/pci/controller/Kconfig | 10 + drivers/pci/controller/Makefile | 2 + drivers/pci/controller/apple-x1n1/Makefile | 2 + drivers/pci/controller/apple-x1n1/host.c | 207 +++++++++++++ drivers/pci/controller/apple-x1n1/t6050.c | 275 ++++++++++++++++++ drivers/pci/controller/apple-x1n1/t6050.h | 9 + .../apple-x1n1/t6050_pcie_disable.c | 85 ++++++ .../apple-x1n1/t6050_pcie_disable.h | 9 + .../controller/apple-x1n1/t6050_pcie_gpio.c | 35 +++ .../controller/apple-x1n1/t6050_pcie_gpio.h | 15 + .../controller/apple-x1n1/t6050_pcie_msi.c | 23 ++ .../controller/apple-x1n1/t6050_pcie_port.c | 192 ++++++++++++ .../apple-x1n1/t6050_pcie_sequence.h | 80 +++++ include/linux/apple-dart-x1n1.h | 17 ++ include/linux/pci-apple-x1n1.h | 15 + 22 files changed, 1353 insertions(+), 99 deletions(-) create mode 100644 drivers/pci/controller/apple-x1n1/Makefile create mode 100644 drivers/pci/controller/apple-x1n1/host.c create mode 100644 drivers/pci/controller/apple-x1n1/t6050.c create mode 100644 drivers/pci/controller/apple-x1n1/t6050.h create mode 100644 drivers/pci/controller/apple-x1n1/t6050_pcie_disable.c create mode 100644 drivers/pci/controller/apple-x1n1/t6050_pcie_disable.h create mode 100644 drivers/pci/controller/apple-x1n1/t6050_pcie_gpio.c create mode 100644 drivers/pci/controller/apple-x1n1/t6050_pcie_gpio.h create mode 100644 drivers/pci/controller/apple-x1n1/t6050_pcie_msi.c create mode 100644 drivers/pci/controller/apple-x1n1/t6050_pcie_port.c create mode 100644 drivers/pci/controller/apple-x1n1/t6050_pcie_sequence.h create mode 100644 include/linux/apple-dart-x1n1.h create mode 100644 include/linux/pci-apple-x1n1.h diff --git a/Documentation/arch/arm64/t6050-sptm-n1.rst b/Documentation/arch/arm64/t6050-sptm-n1.rst index 998da4ce3e744..efec76d9b4715 100644 --- a/Documentation/arch/arm64/t6050-sptm-n1.rst +++ b/Documentation/arch/arm64/t6050-sptm-n1.rst @@ -9,9 +9,9 @@ uploaded the personalized J714s N1 FTAB through the Control PCI function on 2026-09-27. The ROM returned response 1, advanced from stage 0 to stage 4, and wrote MSI data 0xa5a0 into the diagnostic DMA mailbox. This verifies actual PCIe device reads and writes through the SPTM-managed DART, beyond API-only -map/unmap tests. Native interrupt delivery and operational Wi-Fi are separate, -unfinished steps; the inherited polled firmware prototype is not a qualified -full-performance networking implementation. +map/unmap tests. This initial result used a diagnostic MSI DMA mailbox. The later native-MSI +and runtime qualification is described below; radio operation and throughput +remain unqualified. The test uses a preinitialized native T6050 PCIe controller, explicit bus1 RID 0x100/0x101/0x102 to SID1/2/3 routing, and direct ECAM/BAR stage-2 windows. @@ -72,3 +72,60 @@ Kernel ``Image-n1-force`` SHA256: ``02df14333ad13e8f44d76332eadb159b0437501df6d11b3a39cbc677db3f73b8``. Stage2 bundle ``candidate-n1-dart-permissions``; target DT ``t6050-n1-guard-enabled.dtb``. Firmware/calibration stays private. + +Native MSI, reset and runtime qualification +------------------------------------------ + +The preinitialized ``apple,x1n1-pcie`` host now exposes hierarchical native +MSI using the AIC parent. T6050 port0 captures address0xfffff000 and maps +32 local vectors onto AIC1824..1855. Linux allocates aligned vector ranges; +per-function positions are not hardcoded. Its DART advertises the capture +page as IOMMU_RESV_MSI, so the MSI address remains untranslated. + +The optional J714s port resources support the native firmware transition: +endpoint bus mastering stops and pending transactions drain; the endpoint +enters D3hot; bridge decode/bus numbers are cleared before PME broadcast; +PME acknowledgement/L2 precede GPIO PERST and refclk-gating disable. The +native DART power transition retains tables/mappings, and port reset restores +RID/MSI routes before link training and PCI state restoration. Errors leave +bridge decode disabled. The cycle is restricted to the preboot personality +with no enumerated sibling functions. It does not reset an active Wi-Fi bus. + +Guest resources passed alongside ECAM are port0 at0x62800000+0x8000, +PHY at0x62808000+0x4000, interrupt routing at0x6280c000+0x4000, and the +PERST pad at0x62810140+4. The host provides target-derived four-cell tunable +entries (offset, width, mask, value), and the kernel executes the reviewed +native port sequence. The corresponding physical resources remain owned by +this preinitialized controller; initial PHY/SMC setup still occurs in x1n1's +hardware setup workflow. + +The IRQ conversion removes recurring Wi-Fi/Bluetooth traffic polling. +Control reserves4 MSI vectors, Alpha16, Beta2. Alpha's vectors must exist +before its IPC handshake. They remain allocated through runtime, with a +synchronized callback handoff to the Wi-Fi worker. Beta owns its own vectors. +Finite bootstrap/command response polls remain. + +Each function has its own IOMMU domain on this machine. Buffers therefore +retain the consuming function's DMA device and matching lifetime: Control +owns secondary FTAB and Control IPC; each memory-swap working copy belongs +to its recipient; Alpha owns its window, context, messages and firmware log. +Allocating all buffers through Control prevented Alpha from starting. + +Hardware ``hardware-linux-n1-runtime-3-guest.log`` establishes: + +* Native Control MSI reception, ROM success1/stage4. +* Port cycle completed with endpoint identity106b:1901 and boot stage1. +* Control reached OS stage2, completed IPC and CCHI hello. +* Alpha completed its IPC handshake and registered ``wlan0``. +* Beta completed IPC and registered ``hci0``. + +Kernel ``Image-n1-early-alpha-irq`` SHA256: +``8a43716a91eb395ced1226c78c94d460647ace6712ee49002d97da3d22500720``. +Stage2 ``candidate-n1-port-cycle`` and target DT +``t6050-n1-cycle-enabled.dtb``. Full kernel builds and IRQ cleanup failure +checks passed. Focused QEMU tests validate the additional stage-2 resource +windows only; all endpoint/interrupt results above are from hardware. + +Interface registration is not association, packet transfer, Bluetooth radio +qualification or native performance equivalence. Production data-ring startup, +firmware27 connection details and sustained traffic are still in progress. diff --git a/drivers/bluetooth/btcentauri.c b/drivers/bluetooth/btcentauri.c index fb5c9ca89f6e0..95dd96f2ecad3 100644 --- a/drivers/bluetooth/btcentauri.c +++ b/drivers/bluetooth/btcentauri.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -108,7 +109,9 @@ struct cen_bt { u8 *arena; dma_addr_t dma; struct mutex lock; - struct delayed_work poll; + struct work_struct poll; + unsigned int irq_count; + bool irq_vectors; struct sk_buff_head txq; struct sk_buff_head acl_txq, le_txq; u16 acl_tx_head, le_tx_head; @@ -471,7 +474,7 @@ static int cen_transmit(struct cen_bt *bt, struct sk_buff_head *queue, static void cen_poll(struct work_struct *work) { - struct cen_bt *bt = container_of(to_delayed_work(work), struct cen_bt, poll); + struct cen_bt *bt = container_of(work, struct cen_bt, poll); struct hci_dev *hdev = bt->hdev; struct sk_buff *skb; u16 tail; @@ -574,12 +577,11 @@ static void cen_poll(struct work_struct *work) cen_transmit(bt, &bt->le_txq, 4, CEN_LE_SLOTS, CEN_LE_TX, CEN_LE_FOOTER, &bt->le_tx_head))) goto protocol_error; - schedule_delayed_work(&bt->poll, msecs_to_jiffies(1)); goto unlock; protocol_error: bt->running = false; hdev->stat.err_rx++; - dev_err(&bt->pdev->dev, "Beta transport stopped (exec=%u ipc=%u); polling disabled\n", + dev_err(&bt->pdev->dev, "Beta transport stopped (exec=%u ipc=%u); interrupt processing stopped\n", readl(bt->bar + 0x18), readl(bt->bar)); unlock: mutex_unlock(&bt->lock); @@ -591,7 +593,7 @@ static int cen_hci_open(struct hci_dev *hdev) mutex_lock(&bt->lock); bt->running = true; - schedule_delayed_work(&bt->poll, 0); + schedule_work(&bt->poll); mutex_unlock(&bt->lock); return 0; } @@ -604,7 +606,7 @@ static int cen_hci_close(struct hci_dev *hdev) bt->running = false; bt->hrb_status_pending = false; mutex_unlock(&bt->lock); - cancel_delayed_work_sync(&bt->poll); + cancel_work_sync(&bt->poll); skb_queue_purge(&bt->txq); skb_queue_purge(&bt->acl_txq); skb_queue_purge(&bt->le_txq); @@ -772,7 +774,7 @@ static int cen_hci_send(struct hci_dev *hdev, struct sk_buff *skb) ret = -ENOBUFS; else { skb_queue_tail(queue, skb); - mod_delayed_work(system_wq, &bt->poll, 0); + schedule_work(&bt->poll); } mutex_unlock(&bt->lock); if (!ret) @@ -784,9 +786,48 @@ static int cen_hci_send(struct hci_dev *hdev, struct sk_buff *skb) return ret; } +static irqreturn_t cen_irq(int irq, void *data) +{ + struct cen_bt *bt = data; + + if (READ_ONCE(bt->running)) + schedule_work(&bt->poll); + return IRQ_HANDLED; +} + +static void cen_free_irqs(struct cen_bt *bt) +{ + while (bt->irq_count) + free_irq(pci_irq_vector(bt->pdev, --bt->irq_count), bt); + if (bt->irq_vectors) { + pci_free_irq_vectors(bt->pdev); + bt->irq_vectors = false; + } +} + +static int cen_request_irqs(struct cen_bt *bt) +{ + int ret = pci_alloc_irq_vectors(bt->pdev, 2, 2, PCI_IRQ_MSI); + + if (ret < 0) + return ret; + bt->irq_vectors = true; + for (bt->irq_count = 0; bt->irq_count < 2; bt->irq_count++) { + ret = request_irq(pci_irq_vector(bt->pdev, bt->irq_count), cen_irq, + 0, "btcentauri", bt); + if (ret) { + cen_free_irqs(bt); + return ret; + } + } + return 0; +} + static void cen_stop(struct cen_bt *bt) { pci_clear_master(bt->pdev); + cen_free_irqs(bt); + cancel_work_sync(&bt->poll); kvfree(bt->capture); bt->capture = NULL; /* IPC control 0 caused a firmware abort in the initial hardware probe. @@ -887,7 +928,8 @@ static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) struct hci_dev *hdev; int ret; - if (!of_machine_is_compatible("apple,j813") || PCI_FUNC(pdev->devfn) != 2) + if ((!of_machine_is_compatible("apple,j813") && + !of_machine_is_compatible("apple,j714s")) || PCI_FUNC(pdev->devfn) != 2) return -ENODEV; bt = devm_kzalloc(&pdev->dev, sizeof(*bt), GFP_KERNEL); if (!bt) @@ -901,7 +943,7 @@ static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) return -EINVAL; if (native_hrb && (!native_startup || !enable_acl)) return -EINVAL; - INIT_DELAYED_WORK(&bt->poll, cen_poll); + INIT_WORK(&bt->poll, cen_poll); ret = pcim_enable_device(pdev); if (ret) return ret; @@ -929,6 +971,9 @@ static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto stop; } } + ret = cen_request_irqs(bt); + if (ret) + goto stop; ret = cen_start(bt); if (ret) { dev_err(&pdev->dev, "Beta IPC initialization failed: %d\n", ret); @@ -976,6 +1021,8 @@ static void cen_remove(struct pci_dev *pdev) debugfs_remove_recursive(bt->debug); hci_unregister_dev(bt->hdev); cen_hci_close(bt->hdev); + cen_free_irqs(bt); + cancel_work_sync(&bt->poll); hci_free_dev(bt->hdev); cen_stop(bt); } diff --git a/drivers/iommu/apple-dart-x1n1.c b/drivers/iommu/apple-dart-x1n1.c index 8654fd8e1348e..f4c69563861fb 100644 --- a/drivers/iommu/apple-dart-x1n1.c +++ b/drivers/iommu/apple-dart-x1n1.c @@ -27,6 +27,7 @@ * knows) and #iommu-cells = <1> (the stream id). */ +#include #include #include #include @@ -43,6 +44,8 @@ #include #include +#include "dma-iommu.h" + /* sptm2mmio register/doorbell map, shared with x1n1 src/sptm.c. */ #define X1N1_MMIO_MAGIC 0x000 #define X1N1_MMIO_VERSION 0x008 @@ -976,7 +979,34 @@ static bool apple_dart_x1n1_capable(struct device *dev, enum iommu_cap cap) return cap == IOMMU_CAP_CACHE_COHERENCY; } +/* PCIe captures the MSI doorbell before DART translation. Advertise the + * identity MSI page to the DMA layer so it never tries to map this non-RAM + * address through the SPTM transport. Other devices retain normal reservations. + * This follows apple-dart.c's hardware MSI reservation (Asahi contributors). + */ +static void apple_dart_x1n1_get_resv_regions(struct device *dev, + struct list_head *head) +{ + if (dev_is_pci(dev)) { + struct pci_host_bridge *bridge = pci_find_host_bridge(to_pci_dev(dev)->bus); + struct device_node *np = bridge->dev.parent->of_node; + struct iommu_resv_region *region; + u64 address; + + if (of_device_is_compatible(np, "apple,x1n1-pcie") && + !of_property_read_u64(np, "apple,msi-address", &address)) { + region = iommu_alloc_resv_region(address & PAGE_MASK, PAGE_SIZE, + IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO, + IOMMU_RESV_MSI, GFP_KERNEL); + if (region) + list_add_tail(®ion->list, head); + } + } + iommu_dma_get_resv_regions(dev, head); +} + static const struct iommu_ops apple_dart_x1n1_ops = { + .get_resv_regions = apple_dart_x1n1_get_resv_regions, .capable = apple_dart_x1n1_capable, .domain_alloc_paging = apple_dart_x1n1_domain_alloc_paging, .probe_device = apple_dart_x1n1_probe_device, @@ -994,6 +1024,38 @@ static const struct iommu_ops apple_dart_x1n1_ops = { }, }; +/* Used only after the PCI host has stopped every endpoint on this DART. + * Native POWERDOWN/POWERUP retains the SPTM-owned tables and live mappings. */ +int apple_dart_x1n1_set_power(struct device *dev, bool on) +{ + struct apple_dart_x1n1_master *master; + struct apple_dart_x1n1 *dart; + unsigned long flags; + int ret; + + if (!dev->iommu || !dev->iommu->iommu_dev || + dev->iommu->iommu_dev->ops != &apple_dart_x1n1_ops) + return -ENODEV; + master = dev_iommu_priv_get(dev); + if (!master) + return -ENODEV; + dart = master->dart; + mutex_lock(&dart->init_lock); + if (!dart->inited) { + ret = -EINVAL; + goto out; + } + spin_lock_irqsave(&dart->batch_lock, flags); + dart->batch->count = 1; + dart->batch->op[0] = (struct x1n1_op){ .op = on ? 7 : 8 }; + ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + spin_unlock_irqrestore(&dart->batch_lock, flags); +out: + mutex_unlock(&dart->init_lock); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_x1n1_set_power); + static int apple_dart_x1n1_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; diff --git a/drivers/net/wireless/apple/centauri/centauri.h b/drivers/net/wireless/apple/centauri/centauri.h index f7fdf4105a3b3..b2bcaedd16029 100644 --- a/drivers/net/wireless/apple/centauri/centauri.h +++ b/drivers/net/wireless/apple/centauri/centauri.h @@ -2,6 +2,7 @@ #ifndef AURORA_CENTAURI_H #define AURORA_CENTAURI_H #include +#include #include #include #include @@ -11,7 +12,14 @@ struct centauri_ipc; struct centauri_wifi; +struct centauri_irqs { + struct pci_dev *pdev; + void *data; + unsigned int count; +}; + struct cen_alpha_resources { + struct pci_dev *pdev; void *context, *messages, *window; dma_addr_t context_dma, messages_dma, window_dma; void __iomem *bar; @@ -24,23 +32,28 @@ struct centauri { u32 stage_offset; struct dentry *debug; struct mutex boot_lock; - void *firmware, *descriptor, *msi_page; - dma_addr_t firmware_dma, descriptor_dma, msi_dma; + void *firmware, *descriptor; + struct centauri_irqs irqs, alpha_irqs; + atomic_t irq_count; + dma_addr_t firmware_dma, descriptor_dma; size_t firmware_size; u32 image_response; - u16 msi_original; - int msi_cap; bool boot_attempted; bool link_ready; struct centauri_ipc *ipc; struct centauri_wifi *wifi; }; +int centauri_request_irqs(struct centauri_irqs *irqs, struct pci_dev *pdev, + unsigned int count, irq_handler_t handler, void *data); +void centauri_free_irqs(struct centauri_irqs *irqs); + int centauri_j813_port_cycle(struct pci_dev *pdev); int centauri_start_ipc(struct centauri *c); void centauri_free_ipc(struct centauri *c); int centauri_alpha_resources(struct centauri *c, struct cen_alpha_resources *resources); int centauri_alpha_log_buffer(struct centauri *c, void **cpu, dma_addr_t *dma, size_t *size); +irqreturn_t centauri_alpha_irq(int irq, void *data); int centauri_wifi_register(struct centauri *c); void centauri_wifi_unregister(struct centauri *c); #endif diff --git a/drivers/net/wireless/apple/centauri/centauri_ipc.c b/drivers/net/wireless/apple/centauri/centauri_ipc.c index 0db35959d114d..21c309cadd404 100644 --- a/drivers/net/wireless/apple/centauri/centauri_ipc.c +++ b/drivers/net/wireless/apple/centauri/centauri_ipc.c @@ -16,6 +16,7 @@ struct cen_buffer { struct list_head list; + struct device *owner; void *cpu; dma_addr_t dma; size_t size; @@ -30,6 +31,7 @@ struct cen_context { struct centauri_ipc { struct list_head buffers; struct pci_dev *function[3]; + bool enabled[3]; void __iomem *bar[3]; struct cen_context control, alpha; struct cen_buffer *secondary, *window, *cchi_rx, *cchi_tx, *debug_rx; @@ -37,15 +39,22 @@ struct centauri_ipc { u16 cchi_head, cchi_tail; }; -static struct cen_buffer *buffer_alloc(struct centauri *c, size_t size) +/* Each PCI function can have its own DART domain. Keep the DMA owner with + * every allocation: an address allocated for Control is not an Alpha/Beta + * mapping, even when all functions share the same physical PCI endpoint. + */ +static struct cen_buffer *buffer_alloc(struct centauri *c, unsigned int function, + size_t size) { struct cen_buffer *buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); if (!buffer) return NULL; + buffer->owner = get_device(&c->ipc->function[function]->dev); buffer->size = PAGE_ALIGN(size); - buffer->cpu = dma_alloc_coherent(&c->pdev->dev, buffer->size, &buffer->dma, GFP_KERNEL); + buffer->cpu = dma_alloc_coherent(buffer->owner, buffer->size, &buffer->dma, GFP_KERNEL); if (!buffer->cpu) { + put_device(buffer->owner); kfree(buffer); return NULL; } @@ -139,7 +148,7 @@ static int secondary_init(struct centauri *c) return -EINVAL; required = max_t(u32, required, end); } - c->ipc->secondary = buffer_alloc(c, required); + c->ipc->secondary = buffer_alloc(c, 0, required); if (!c->ipc->secondary) return -ENOMEM; memcpy(c->ipc->secondary->cpu, seed, length); @@ -153,14 +162,14 @@ static int secondary_init(struct centauri *c) } static int context_init(struct centauri *c, struct cen_context *ctx, - u16 tr, u16 cr, dma_addr_t scratch, size_t scratch_size) + unsigned int function, u16 tr, u16 cr, dma_addr_t scratch, size_t scratch_size) { u8 *header; dma_addr_t dma; int i; - ctx->buffer = buffer_alloc(c, SZ_32K); - ctx->messages = buffer_alloc(c, SZ_16K); + ctx->buffer = buffer_alloc(c, function, SZ_32K); + ctx->messages = buffer_alloc(c, function, SZ_16K); if (!ctx->buffer || !ctx->messages) return -ENOMEM; ctx->tr_count = tr; @@ -198,13 +207,20 @@ static int handshake(void __iomem *bar, struct cen_context *ctx, writel(1, bar + control); ret = readl_poll_timeout(bar + status, value, value == 1, 100, 2000000); - if (ret) + if (ret) { + pr_err("centauri: IPC init timeout status@%#x=%#x control@%#x=%#x context=%pad\n", + status, value, control, readl(bar + control), &ctx->buffer->dma); return ret; + } writel(lower_32_bits(ctx->buffer->dma), bar + address); writel(upper_32_bits(ctx->buffer->dma), bar + address + 4); dma_wmb(); writel(2, bar + control); - return readl_poll_timeout(bar + status, value, value == 2, 100, 2000000); + ret = readl_poll_timeout(bar + status, value, value == 2, 100, 2000000); + if (ret) + pr_err("centauri: IPC run timeout status@%#x=%#x control@%#x=%#x context=%pad\n", + status, value, control, readl(bar + control), &ctx->buffer->dma); + return ret; } static int message(struct cen_context *ctx, void __iomem *doorbell, @@ -256,9 +272,9 @@ static int cchi_init(struct centauri *c) u16 tail; int i, ret; - ipc->cchi_rx = buffer_alloc(c, SZ_32K); - ipc->cchi_tx = buffer_alloc(c, SZ_32K); - ipc->debug_rx = buffer_alloc(c, SZ_16K); + ipc->cchi_rx = buffer_alloc(c, 0, SZ_32K); + ipc->cchi_tx = buffer_alloc(c, 0, SZ_32K); + ipc->debug_rx = buffer_alloc(c, 0, SZ_16K); if (!ipc->cchi_rx || !ipc->cchi_tx || !ipc->debug_rx) return -ENOMEM; ret = message(&ipc->control, c->bar + 0x9000, config, sizeof(config)); @@ -311,7 +327,7 @@ static int memswap_handoff(struct centauri *c, unsigned int function, const char ret = ftab_entry(secondary->cpu, secondary->size, tag, &offset, &size, false); if (ret) return ret; - working = buffer_alloc(c, size); + working = buffer_alloc(c, function, size); if (!working) return -ENOMEM; dma_rmb(); @@ -327,8 +343,7 @@ static int memswap_handoff(struct centauri *c, unsigned int function, const char static int enumerate_functions(struct centauri *c) { struct centauri_ipc *ipc = c->ipc; - u16 control; - int i, cap; + int i, ret; pci_lock_rescan_remove(); /* The ROM personality was single-function; boot exposes Alpha and Beta. */ @@ -346,19 +361,25 @@ static int enumerate_functions(struct centauri *c) ipc->bar[i] = pci_iomap(ipc->function[i], 0, 0); if (!ipc->bar[i]) return -ENOMEM; - cap = pci_find_capability(ipc->function[i], PCI_CAP_ID_MSI); - if (!cap) - return -ENODEV; - pci_read_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, &control); - if (!(control & PCI_MSI_FLAGS_64BIT)) - return -EOPNOTSUPP; - pci_write_config_dword(ipc->function[i], cap + PCI_MSI_ADDRESS_LO, - lower_32_bits(c->msi_dma + 0x400 * i)); - pci_write_config_dword(ipc->function[i], cap + PCI_MSI_ADDRESS_HI, - upper_32_bits(c->msi_dma + 0x400 * i)); - pci_write_config_word(ipc->function[i], cap + PCI_MSI_DATA_64, 0xa5a0 + 0x100 * i); - pci_write_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, - (control & ~PCI_MSI_FLAGS_QSIZE) | (5 << 4) | PCI_MSI_FLAGS_ENABLE); + if (i) { + ret = pci_enable_device_mem(ipc->function[i]); + if (ret) + return ret; + ipc->enabled[i] = true; + } + ret = dma_set_mask_and_coherent(&ipc->function[i]->dev, DMA_BIT_MASK(40)); + if (ret) + return ret; + if (i == 1) { + /* Firmware publishes Alpha bootstrap changes through MSI before + * the host installs its ACIPC context. Keep these vectors stable + * through the later Wi-Fi handler handoff. + */ + ret = centauri_request_irqs(&c->alpha_irqs, ipc->function[i], + 16, centauri_alpha_irq, c); + if (ret) + return ret; + } pci_set_master(ipc->function[i]); } return 0; @@ -383,7 +404,7 @@ int centauri_start_ipc(struct centauri *c) ret = secondary_init(c); if (ret) goto failed; - ret = context_init(c, &ipc->control, 13, 1, 0, 0); + ret = context_init(c, &ipc->control, 0, 13, 1, 0, 0); if (ret) goto failed; writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); @@ -417,12 +438,12 @@ int centauri_start_ipc(struct centauri *c) ret = ret ?: -EIO; goto failed; } - ipc->window = buffer_alloc(c, SZ_16M); + ipc->window = buffer_alloc(c, 1, SZ_16M); if (!ipc->window) { ret = -ENOMEM; goto failed; } - ret = context_init(c, &ipc->alpha, 192, 234, + ret = context_init(c, &ipc->alpha, 1, 192, 234, ipc->window->dma + CEN_ALPHA_SCRATCH_OFFSET, CEN_ALPHA_SCRATCH_SIZE); if (ret) goto failed; @@ -433,10 +454,26 @@ int centauri_start_ipc(struct centauri *c) if (ret) goto failed; dev_info(&c->pdev->dev, "Alpha Wi-Fi firmware running; IPC handshake complete\n"); - return centauri_wifi_register(c); + ret = centauri_wifi_register(c); + if (ret) + return ret; + /* Beta was not ready during the initial rescan; retry its own driver only + * after Control has handed over its memory and started the runtime image. + */ + ret = device_attach(&ipc->function[2]->dev); + if (ret < 0) + dev_warn(&c->pdev->dev, "Beta driver attachment failed: %d\n", ret); + return 0; failed: dev_err(&c->pdev->dev, "IPC bootstrap failed: %d (Control stage=%#x)\n", ret, readl(c->bar + 0x8000)); + if (ipc->bar[1]) + dev_err(&c->pdev->dev, "Alpha stage=%#x IPC=%#x window=%#x/%#x/%#x\n", + readl(ipc->bar[1] + 8), readl(ipc->bar[1] + 0xc), + readl(ipc->bar[1] + 0x18), readl(ipc->bar[1] + 0x1c), readl(ipc->bar[1] + 0x20)); + if (ipc->bar[2]) + dev_err(&c->pdev->dev, "Beta stage=%#x IPC=%#x\n", + readl(ipc->bar[2] + 0x18), readl(ipc->bar[2] + 0x1c)); /* Published DMA stays pinned until every function is stopped at removal. */ return ret; } @@ -445,31 +482,30 @@ void centauri_free_ipc(struct centauri *c) { struct centauri_ipc *ipc = c->ipc; struct cen_buffer *buffer, *next; - int i, cap; - u16 control; + int i; if (!ipc) return; + if (ipc->function[2]) + device_release_driver(&ipc->function[2]->dev); centauri_wifi_unregister(c); + centauri_free_irqs(&c->alpha_irqs); for (i = 0; i < 3; i++) { if (!ipc->function[i]) continue; - cap = pci_find_capability(ipc->function[i], PCI_CAP_ID_MSI); - if (cap) { - pci_read_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, &control); - pci_write_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, - control & ~PCI_MSI_FLAGS_ENABLE); - } pci_clear_master(ipc->function[i]); } list_for_each_entry_safe(buffer, next, &ipc->buffers, list) { - dma_free_coherent(&c->pdev->dev, buffer->size, buffer->cpu, buffer->dma); + dma_free_coherent(buffer->owner, buffer->size, buffer->cpu, buffer->dma); + put_device(buffer->owner); list_del(&buffer->list); kfree(buffer); } for (i = 0; i < 3; i++) { if (ipc->bar[i]) pci_iounmap(ipc->function[i], ipc->bar[i]); + if (ipc->enabled[i]) + pci_disable_device(ipc->function[i]); pci_dev_put(ipc->function[i]); } kfree(ipc); @@ -482,6 +518,7 @@ int centauri_alpha_resources(struct centauri *c, struct cen_alpha_resources *res if (!ipc || !ipc->window || !ipc->alpha.buffer || !ipc->alpha.messages) return -ENODEV; + resources->pdev = ipc->function[1]; resources->context = ipc->alpha.buffer->cpu; resources->context_dma = ipc->alpha.buffer->dma; resources->messages = ipc->alpha.messages->cpu; @@ -505,7 +542,7 @@ int centauri_alpha_log_buffer(struct centauri *c, void **cpu, dma_addr_t *dma, s if (!ipc || !ipc->window) return -ENODEV; if (!ipc->alpha_log) - ipc->alpha_log = buffer_alloc(c, SZ_4M); + ipc->alpha_log = buffer_alloc(c, 1, SZ_4M); if (!ipc->alpha_log) return -ENOMEM; *cpu = ipc->alpha_log->cpu; diff --git a/drivers/net/wireless/apple/centauri/centauri_pci.c b/drivers/net/wireless/apple/centauri/centauri_pci.c index 8541f151b2803..7401ab1b79bfc 100644 --- a/drivers/net/wireless/apple/centauri/centauri_pci.c +++ b/drivers/net/wireless/apple/centauri/centauri_pci.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,52 @@ static struct dentry *centauri_debug; +/* Fixed counts follow the native Control/Alpha/Beta resource maps. PCI's MSI + * allocator owns the message addresses and data; never program a DMA mailbox. + */ +void centauri_free_irqs(struct centauri_irqs *irqs) +{ + if (!irqs->pdev) + return; + while (irqs->count) + free_irq(pci_irq_vector(irqs->pdev, --irqs->count), irqs->data); + pci_free_irq_vectors(irqs->pdev); + irqs->pdev = NULL; +} + +int centauri_request_irqs(struct centauri_irqs *irqs, struct pci_dev *pdev, + unsigned int count, irq_handler_t handler, void *data) +{ + int ret; + + ret = pci_alloc_irq_vectors(pdev, count, count, PCI_IRQ_MSI); + if (ret < 0) + return ret; + irqs->pdev = pdev; + irqs->data = data; + for (irqs->count = 0; irqs->count < count; irqs->count++) { + ret = request_irq(pci_irq_vector(pdev, irqs->count), handler, 0, + "centauri", data); + if (ret) { + centauri_free_irqs(irqs); + return ret; + } + } + return 0; +} + +static irqreturn_t centauri_control_irq(int irq, void *data) +{ + struct centauri *c = data; + + atomic_inc(&c->irq_count); + /* ROM and bounded Control bootstrap commands poll their response fields. + * MSI delivery is counted independently, including boot-stage changes. + */ + return IRQ_HANDLED; +} + + static int centauri_status_show(struct seq_file *s, void *unused) { struct centauri *c = s->private; @@ -43,22 +90,24 @@ static void centauri_free_boot(struct centauri *c) struct device *dev = &c->pdev->dev; if (!c->link_ready && c->boot_attempted) { + /* No endpoint config access is safe after a failed port transition. + * Disconnect handlers before devm frees c; leave PCI vectors and + * published DMA pinned until the required reset/device removal. + */ + while (c->irqs.count) + free_irq(pci_irq_vector(c->pdev, --c->irqs.count), c); dev_warn(dev, "retaining DMA buffers after failed link transition; reboot required\n"); return; } /* Stop device writes before any DMA storage is released. */ centauri_free_ipc(c); - if (c->msi_cap) - pci_write_config_word(c->pdev, c->msi_cap + PCI_MSI_FLAGS, - c->msi_original & ~PCI_MSI_FLAGS_ENABLE); pci_clear_master(c->pdev); + centauri_free_irqs(&c->irqs); if (c->firmware) dma_free_coherent(dev, c->firmware_size, c->firmware, c->firmware_dma); if (c->descriptor) dma_free_coherent(dev, SZ_16K, c->descriptor, c->descriptor_dma); - if (c->msi_page) - dma_free_coherent(dev, SZ_16K, c->msi_page, c->msi_dma); - c->firmware = c->descriptor = c->msi_page = NULL; + c->firmware = c->descriptor = NULL; } struct centauri_host_profile { @@ -111,34 +160,20 @@ static int centauri_rom_boot(struct centauri *c) c->firmware_size = PAGE_ALIGN(fw->size); c->firmware = dma_alloc_coherent(dev, c->firmware_size, &c->firmware_dma, GFP_KERNEL); c->descriptor = dma_alloc_coherent(dev, SZ_16K, &c->descriptor_dma, GFP_KERNEL); - c->msi_page = dma_alloc_coherent(dev, SZ_16K, &c->msi_dma, GFP_KERNEL); - if (!c->firmware || !c->descriptor || !c->msi_page) { + if (!c->firmware || !c->descriptor) { ret = -ENOMEM; goto free_buffers; } memset(c->firmware, 0, c->firmware_size); memcpy(c->firmware, fw->data, fw->size); memset(c->descriptor, 0, SZ_16K); - memset(c->msi_page, 0, SZ_16K); /* Measured ROM scatter entry: u64 DMA, u32 length, u32 flags, u64 x2 zero. */ put_unaligned_le64(c->firmware_dma, c->descriptor); put_unaligned_le32(ALIGN(fw->size, SZ_4K), c->descriptor + 8); - c->msi_cap = pci_find_capability(c->pdev, PCI_CAP_ID_MSI); - if (!c->msi_cap) { - ret = -ENODEV; - goto free_buffers; - } - pci_read_config_word(c->pdev, c->msi_cap + PCI_MSI_FLAGS, &c->msi_original); - if (!(c->msi_original & PCI_MSI_FLAGS_64BIT)) { - ret = -EOPNOTSUPP; + ret = centauri_request_irqs(&c->irqs, c->pdev, 4, + centauri_control_irq, c); + if (ret) goto free_buffers; - } - /* Poll the measured DMA-backed MSI mailbox until native IRQ support exists. */ - pci_write_config_dword(c->pdev, c->msi_cap + PCI_MSI_ADDRESS_LO, lower_32_bits(c->msi_dma)); - pci_write_config_dword(c->pdev, c->msi_cap + PCI_MSI_ADDRESS_HI, upper_32_bits(c->msi_dma)); - pci_write_config_word(c->pdev, c->msi_cap + PCI_MSI_DATA_64, 0xa5a0); - pci_write_config_word(c->pdev, c->msi_cap + PCI_MSI_FLAGS, - (c->msi_original & ~PCI_MSI_FLAGS_QSIZE) | PCI_MSI_FLAGS_ENABLE); pci_set_master(c->pdev); writel(profile->platform_id, c->bar + 0x807c); writel(0, c->bar + 0x803c); @@ -156,13 +191,15 @@ static int centauri_rom_boot(struct centauri *c) writel(1, c->bar + 0x9000); ret = readl_poll_timeout(c->bar + 0x8040, c->image_response, c->image_response != 0, 20000, 20000000); + if (!ret && c->image_response != 1) + ret = -EIO; if (!ret) ret = readl_poll_timeout(c->bar + 0x8000, stage, stage != 0, 20000, 20000000); else stage = readl(c->bar + 0x8000); dma_rmb(); - dev_info(dev, "ROM firmware result: response=%#x stage=%#x mailbox=%#x poll=%d\n", - c->image_response, stage, READ_ONCE(*(u32 *)c->msi_page), ret); + dev_info(dev, "ROM firmware result: response=%#x stage=%#x irqs=%d poll=%d\n", + c->image_response, stage, atomic_read(&c->irq_count), ret); /* Keep DMA buffers owned until device removal, even on timeout. */ if (!ret && stage != 4) ret = -EIO; @@ -192,8 +229,11 @@ static ssize_t centauri_boot_write(struct file *file, const char __user *buffer, ret = -EBUSY; } else { c->link_ready = false; - ret = centauri_j813_port_cycle(c->pdev); + ret = of_machine_is_compatible("apple,j714s") ? + apple_x1n1_pcie_cycle(c->pdev) : + centauri_j813_port_cycle(c->pdev); if (!ret) { + pci_restore_msi_state(c->pdev); c->link_ready = true; dev_info(&c->pdev->dev, "post-reset execution stage=%#x\n", readl(c->bar + 0x8000)); @@ -216,10 +256,13 @@ static int centauri_probe(struct pci_dev *pdev, const struct pci_device_id *id) void *page; int ret; + if (PCI_FUNC(pdev->devfn) != 0) + return -ENODEV; c = devm_kzalloc(&pdev->dev, sizeof(*c), GFP_KERNEL); if (!c) return -ENOMEM; c->pdev = pdev; + atomic_set(&c->irq_count, 0); c->link_ready = true; mutex_init(&c->boot_lock); c->stage_offset = id->driver_data; @@ -266,8 +309,6 @@ static void centauri_remove(struct pci_dev *pdev) static const struct pci_device_id centauri_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1900), .driver_data = 0x8000 }, { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1901), .driver_data = 0x8000 }, - { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1902), .driver_data = 0x0008 }, - { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1903), .driver_data = 0x0018 }, { } }; MODULE_DEVICE_TABLE(pci, centauri_ids); diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index 21201241f49ec..ffbf3a4c656a4 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -85,7 +85,8 @@ struct centauri_wifi { struct net_device *netdev; struct work_struct scan_work; struct work_struct connect_work; - struct delayed_work link_work; + struct work_struct link_work; + bool irq_ready; struct cfg80211_scan_request *scan_request; struct mutex lock; /* The logger has its own coherent allocation. Reads/acknowledgements @@ -1091,29 +1092,42 @@ static void connect_worker(struct work_struct *work) netif_carrier_on(wifi->netdev); if (wifi->data_probe_done & BIT(5)) netif_wake_queue(wifi->netdev); - schedule_delayed_work(&wifi->link_work, msecs_to_jiffies(20)); + schedule_work(&wifi->link_work); } } +irqreturn_t centauri_alpha_irq(int irq, void *data) +{ + struct centauri *c = data; + struct centauri_wifi *wifi = smp_load_acquire(&c->wifi); + + /* During bootstrap only the stable Control owner exists. Publication + * and removal synchronize with these same IRQs before freeing Wi-Fi. + */ + if (wifi && smp_load_acquire(&wifi->irq_ready) && + !READ_ONCE(wifi->stopping)) + schedule_work(&wifi->link_work); + return IRQ_HANDLED; +} + static void link_worker(struct work_struct *work) { - struct centauri_wifi *wifi = container_of(to_delayed_work(work), struct centauri_wifi, link_work); - bool disconnected; + struct centauri_wifi *wifi = container_of(work, struct centauri_wifi, link_work); + bool disconnected, was_connected; mutex_lock(&wifi->lock); + was_connected = wifi->connected; drain(wifi); disconnected = wifi->disconnect_event || READ_ONCE(wifi->stopping) || readl(wifi->a.bar + 8) != 2; if (disconnected) wifi->connected = false; mutex_unlock(&wifi->lock); - if (disconnected) { + if (disconnected && was_connected) { netif_stop_queue(wifi->netdev); skb_queue_purge(&wifi->data_tx_queue); netif_carrier_off(wifi->netdev); cfg80211_disconnected(wifi->netdev, WLAN_REASON_UNSPECIFIED, NULL, 0, false, GFP_KERNEL); - } else { - schedule_delayed_work(&wifi->link_work, msecs_to_jiffies(20)); } } @@ -1136,7 +1150,7 @@ static int wifi_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 r connected = wifi->connected; wifi->connected = false; mutex_unlock(&wifi->lock); - cancel_delayed_work_sync(&wifi->link_work); + cancel_work_sync(&wifi->link_work); netif_stop_queue(netdev); skb_queue_purge(&wifi->data_tx_queue); netif_carrier_off(netdev); @@ -1219,7 +1233,7 @@ static netdev_tx_t net_transmit(struct sk_buff *skb, struct net_device *netdev) skb_queue_tail(&wifi->data_tx_queue, skb); if (skb_queue_len(&wifi->data_tx_queue) >= DATA_TX_QUEUE_LIMIT) netif_stop_queue(netdev); - mod_delayed_work(system_wq, &wifi->link_work, 0); + schedule_work(&wifi->link_work); return NETDEV_TX_OK; } @@ -2163,14 +2177,14 @@ int centauri_wifi_register(struct centauri *c) wifi = kzalloc(sizeof(*wifi), GFP_KERNEL); if (!wifi) return -ENOMEM; - c->wifi = wifi; wifi->control = c; mutex_init(&wifi->lock); mutex_init(&wifi->fwlog_lock); skb_queue_head_init(&wifi->data_tx_queue); INIT_WORK(&wifi->scan_work, scan_worker); INIT_WORK(&wifi->connect_work, connect_worker); - INIT_DELAYED_WORK(&wifi->link_work, link_worker); + INIT_WORK(&wifi->link_work, link_worker); + smp_store_release(&c->wifi, wifi); ret = centauri_alpha_resources(c, &wifi->a); if (ret) goto failed; @@ -2235,6 +2249,8 @@ int centauri_wifi_register(struct centauri *c) } dev_info(&c->pdev->dev, "cfg80211 interface %s ready for scan tests; packet data unavailable\n", wifi->netdev->name); + smp_store_release(&wifi->irq_ready, true); + schedule_work(&wifi->link_work); wifi->passphrase_file = debugfs_create_file("passphrase", 0200, c->debug, wifi, &passphrase_ops); wifi->data_probe_file = debugfs_create_file("data_probe", 0200, c->debug, wifi, &data_probe_ops); wifi->data_status_file = debugfs_create_file("data_status", 0400, c->debug, wifi, &data_status_ops); @@ -2269,6 +2285,12 @@ void centauri_wifi_unregister(struct centauri *c) if (!wifi) return; WRITE_ONCE(wifi->stopping, true); + smp_store_release(&c->wifi, NULL); + /* A handler that observed the old pointer must finish before any work + * item or Wi-Fi storage can be freed. Later IRQs observe NULL. + */ + for (unsigned int i = 0; i < c->alpha_irqs.count; i++) + synchronize_irq(pci_irq_vector(c->alpha_irqs.pdev, i)); if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) netif_tx_disable(wifi->netdev); debugfs_remove(wifi->passphrase_file); @@ -2290,7 +2312,7 @@ void centauri_wifi_unregister(struct centauri *c) debugfs_remove(wifi->fwlog_ack_file); cancel_work_sync(&wifi->scan_work); cancel_work_sync(&wifi->connect_work); - cancel_delayed_work_sync(&wifi->link_work); + cancel_work_sync(&wifi->link_work); skb_queue_purge(&wifi->data_tx_queue); if (wifi->scan_request) { struct cfg80211_scan_info info = { .aborted = true }; @@ -2308,5 +2330,4 @@ void centauri_wifi_unregister(struct centauri *c) free_netdev(wifi->netdev); memzero_explicit(wifi, sizeof(*wifi)); kfree(wifi); - c->wifi = NULL; } diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index e3bb6ff1b6d71..07d780bbff1de 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -3,6 +3,16 @@ menu "PCI controller drivers" depends on PCI +config PCIE_APPLE_X1N1 + tristate "Apple x1n1 preinitialized PCIe host" + depends on ARM64 && OF && PCI_MSI + select PCI_HOST_COMMON + select IRQ_MSI_LIB + help + PCI host for controllers initialized by x1n1, with direct ECAM/BAR + mappings and native hierarchical AIC MSI interrupts. The firmware + must configure power, DART/RID routing and the MSI capture registers. + config PCI_HOST_COMMON tristate select PCI_ECAM diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index ac8db283f0fea..2c4c7db8e1dad 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -65,3 +65,5 @@ obj-$(CONFIG_ARM64) += pci-thunder-pem.o obj-$(CONFIG_ARM64) += pci-xgene.o endif endif + +obj-$(CONFIG_PCIE_APPLE_X1N1) += apple-x1n1/ diff --git a/drivers/pci/controller/apple-x1n1/Makefile b/drivers/pci/controller/apple-x1n1/Makefile new file mode 100644 index 0000000000000..c483155c0fbbf --- /dev/null +++ b/drivers/pci/controller/apple-x1n1/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_PCIE_APPLE_X1N1) += pcie-apple-x1n1.o +pcie-apple-x1n1-y := host.o t6050.o t6050_pcie_port.o t6050_pcie_disable.o t6050_pcie_gpio.o t6050_pcie_msi.o diff --git a/drivers/pci/controller/apple-x1n1/host.c b/drivers/pci/controller/apple-x1n1/host.c new file mode 100644 index 0000000000000..ff9002a59b457 --- /dev/null +++ b/drivers/pci/controller/apple-x1n1/host.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-2.0 +/* PCI host for an Apple controller initialized and owned by x1n1. + * ECAM and BAR windows are direct guest mappings. MSI writes are intercepted + * by the physical controller and delivered to its native AIC IRQ range. + * + * Hierarchical MSI allocation follows drivers/pci/controller/pcie-apple.c: + * Copyright (C) 2021 Alyssa Rosenzweig + * Copyright (C) 2021 Google LLC + * Copyright (C) 2021 Corellium LLC + * Copyright (C) 2021 Mark Kettenis + * Original authors: Alyssa Rosenzweig + * Marc Zyngier + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "t6050.h" +#include +#include "../pci-host-common.h" + +struct x1n1_pcie { + struct t6050_pcie *t6050; + struct mutex lock; + unsigned long *bitmap; + u32 nvecs; + u64 doorbell; + struct irq_fwspec fwspec; + struct irq_domain *msi_domain; +}; + +static void x1n1_msi_compose(struct irq_data *data, struct msi_msg *msg) +{ + struct x1n1_pcie *pcie = irq_data_get_irq_chip_data(data); + + msg->address_hi = upper_32_bits(pcie->doorbell); + msg->address_lo = lower_32_bits(pcie->doorbell); + msg->data = data->hwirq; +} + +static struct irq_chip x1n1_msi_chip = { + .name = "x1n1-PCI-MSI", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_set_type = irq_chip_set_type_parent, + .irq_compose_msi_msg = x1n1_msi_compose, +}; + +static int x1n1_msi_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct x1n1_pcie *pcie = domain->host_data; + struct irq_fwspec fwspec = pcie->fwspec; + int hwirq, ret; + + mutex_lock(&pcie->lock); + hwirq = bitmap_find_free_region(pcie->bitmap, pcie->nvecs, + order_base_2(nr_irqs)); + mutex_unlock(&pcie->lock); + if (hwirq < 0) + return -ENOSPC; + + fwspec.param[fwspec.param_count - 2] += hwirq; + ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &fwspec); + if (ret) { + mutex_lock(&pcie->lock); + bitmap_release_region(pcie->bitmap, hwirq, order_base_2(nr_irqs)); + mutex_unlock(&pcie->lock); + return ret; + } + for (unsigned int i = 0; i < nr_irqs; i++) + irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, + &x1n1_msi_chip, pcie); + return 0; +} + +static void x1n1_msi_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *data = irq_domain_get_irq_data(domain, virq); + struct x1n1_pcie *pcie = domain->host_data; + + mutex_lock(&pcie->lock); + bitmap_release_region(pcie->bitmap, data->hwirq, order_base_2(nr_irqs)); + mutex_unlock(&pcie->lock); +} + +static const struct irq_domain_ops x1n1_msi_domain_ops = { + .alloc = x1n1_msi_alloc, + .free = x1n1_msi_free, +}; + +static const struct msi_parent_ops x1n1_msi_parent_ops = { + .supported_flags = MSI_GENERIC_FLAGS_MASK | MSI_FLAG_PCI_MSIX | + MSI_FLAG_MULTI_PCI_MSI, + .required_flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_PCI_MSI_MASK_PARENT, + .chip_flags = MSI_CHIP_FLAG_SET_EOI, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static void x1n1_msi_cleanup(void *data) +{ + struct x1n1_pcie *pcie = data; + + irq_domain_remove(pcie->msi_domain); +} + +static int x1n1_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pci_host_bridge *bridge; + struct x1n1_pcie *pcie; + struct of_phandle_args args; + struct irq_domain_info info = {}; + int ret; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; + pcie = pci_host_bridge_priv(bridge); + mutex_init(&pcie->lock); + pcie->t6050 = t6050_pcie_init(pdev); + if (IS_ERR(pcie->t6050)) + return PTR_ERR(pcie->t6050); + + ret = of_property_read_u64(dev->of_node, "apple,msi-address", &pcie->doorbell); + if (ret || (pcie->doorbell & 0xf)) + return dev_err_probe(dev, -EINVAL, "missing/unaligned MSI doorbell\n"); + ret = of_parse_phandle_with_args(dev->of_node, "msi-ranges", + "#interrupt-cells", 0, &args); + if (ret) + return dev_err_probe(dev, ret, "missing MSI IRQ range\n"); + /* This transport exposes the native four-cell AIC IRQ binding. */ + if (args.args_count != 4 || args.args[0] != 0 || args.args[3] != IRQ_TYPE_EDGE_RISING) { + of_node_put(args.np); + return dev_err_probe(dev, -EINVAL, "MSI range must use AIC edge IRQs\n"); + } + ret = of_property_read_u32_index(dev->of_node, "msi-ranges", 5, &pcie->nvecs); + if (ret || !is_power_of_2(pcie->nvecs) || pcie->nvecs > 256 || + args.args[2] > U32_MAX - pcie->nvecs) { + of_node_put(args.np); + return dev_err_probe(dev, -EINVAL, "invalid MSI vector count\n"); + } + of_phandle_args_to_fwspec(args.np, args.args, args.args_count, &pcie->fwspec); + of_node_put(args.np); + info.parent = irq_find_matching_fwspec(&pcie->fwspec, DOMAIN_BUS_WIRED); + if (!info.parent) + return -EPROBE_DEFER; + pcie->bitmap = devm_bitmap_zalloc(dev, pcie->nvecs, GFP_KERNEL); + if (!pcie->bitmap) + return -ENOMEM; + info.fwnode = dev_fwnode(dev); + info.ops = &x1n1_msi_domain_ops; + info.size = pcie->nvecs; + info.host_data = pcie; + pcie->msi_domain = msi_create_parent_irq_domain(&info, &x1n1_msi_parent_ops); + if (!pcie->msi_domain) + return -ENOMEM; + ret = devm_add_action_or_reset(dev, x1n1_msi_cleanup, pcie); + if (ret) + return ret; + dev_info(dev, "%u MSI vectors at AIC %u, doorbell %#llx\n", + pcie->nvecs, pcie->fwspec.param[2], pcie->doorbell); + return pci_host_common_init(pdev, bridge, &pci_generic_ecam_ops); +} + +int apple_x1n1_pcie_cycle(struct pci_dev *pdev) +{ + struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->bus); + struct x1n1_pcie *pcie; + + if (!bridge || !of_device_is_compatible(bridge->dev.parent->of_node, + "apple,x1n1-pcie")) + return -ENODEV; + pcie = pci_host_bridge_priv(bridge); + if (!pcie->t6050) + return -EOPNOTSUPP; + return t6050_pcie_cycle(pcie->t6050, pdev); +} +EXPORT_SYMBOL_GPL(apple_x1n1_pcie_cycle); + +static const struct of_device_id x1n1_pcie_match[] = { + { .compatible = "apple,x1n1-pcie" }, + {}, +}; +MODULE_DEVICE_TABLE(of, x1n1_pcie_match); + +static struct platform_driver x1n1_pcie_driver = { + .probe = x1n1_pcie_probe, + .remove = pci_host_common_remove, + .driver = { + .name = "pcie-apple-x1n1", + .of_match_table = x1n1_pcie_match, + }, +}; +module_platform_driver(x1n1_pcie_driver); +MODULE_DESCRIPTION("x1n1 preinitialized Apple PCIe host with native AIC MSI"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/controller/apple-x1n1/t6050.c b/drivers/pci/controller/apple-x1n1/t6050.c new file mode 100644 index 0000000000000..a4230388ece6c --- /dev/null +++ b/drivers/pci/controller/apple-x1n1/t6050.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Native T6050 port0 transition for a preinitialized x1n1 host. */ +#include +#include +#include +#include +#include +#include "t6050.h" +#include "t6050_pcie_sequence.h" +#include "t6050_pcie_disable.h" +#include "t6050_pcie_gpio.h" + +struct t6050_pcie { + struct device *dev; + struct mutex lock; + void __iomem *port, *phy, *gpio, *intr2axi; + struct pci_dev *bridge, *endpoint; + struct t6050_pcie_io io; + struct t6050_gpio_pad_io pad; +}; + +static void __iomem *t6050_reg(struct t6050_pcie *pcie, unsigned int reg, + unsigned int off) +{ + if (off & 3) + return NULL; + switch (reg) { + case T6050_PCIE_PORT0: return off < 0x8000 ? pcie->port + off : NULL; + case T6050_PCIE_PORT0_PHY: return off < 0x4000 ? pcie->phy + off : NULL; + case T6050_PCIE_PORT0_INTR2AXI: return off < 0x4000 ? pcie->intr2axi + off : NULL; + default: return NULL; + } +} + +static int t6050_read(void *ctx, unsigned int reg, unsigned int off, u32 *value) +{ + struct t6050_pcie *pcie = ctx; + void __iomem *addr; + if (reg == T6050_PCIE_ECAM) + return pcie->bridge ? pcibios_err_to_errno(pci_read_config_dword(pcie->bridge, off, value)) : -ENODEV; + addr = t6050_reg(pcie, reg, off); + if (!addr) + return -EINVAL; + *value = readl(addr); + return 0; +} + +static int t6050_write(void *ctx, unsigned int reg, unsigned int off, u32 value) +{ + struct t6050_pcie *pcie = ctx; + void __iomem *addr; + if (reg == T6050_PCIE_ECAM) + return pcie->bridge ? pcibios_err_to_errno(pci_write_config_dword(pcie->bridge, off, value)) : -ENODEV; + addr = t6050_reg(pcie, reg, off); + if (!addr) + return -EINVAL; + writel(value, addr); + return 0; +} + +static int t6050_read16(void *ctx, unsigned int reg, unsigned int off, u16 *value) +{ + struct t6050_pcie *pcie = ctx; + if (reg != T6050_PCIE_ECAM || !pcie->bridge) + return -EINVAL; + return pcibios_err_to_errno(pci_read_config_word(pcie->bridge, off, value)); +} + +static int t6050_write16(void *ctx, unsigned int reg, unsigned int off, u16 value) +{ + struct t6050_pcie *pcie = ctx; + if (reg != T6050_PCIE_ECAM || !pcie->bridge) + return -EINVAL; + return pcibios_err_to_errno(pci_write_config_word(pcie->bridge, off, value)); +} + +static int t6050_poll(void *ctx, unsigned int reg, unsigned int off, + u32 mask, u32 expected, unsigned int timeout) +{ + struct t6050_pcie *pcie = ctx; + u32 value = 0; + int ret, err; + ret = read_poll_timeout(t6050_read, err, err || (value & mask) == expected, + 1, timeout, false, ctx, reg, off, &value); + if (ret || err) + dev_err(pcie->dev, "port wait reg%u+%#x value=%#x mask=%#x expected=%#x\n", + reg, off, value, mask, expected); + return err ?: ret; +} + +static void t6050_delay(void *ctx, unsigned int us) +{ + if (us <= 10) + udelay(us); + else + usleep_range(us, us + max(1U, us / 10)); +} + +static int t6050_tune(void *ctx, const char *name, unsigned int reg) +{ + struct t6050_pcie *pcie = ctx; + char property[64]; + u32 cells[64], old; + int count, ret; + snprintf(property, sizeof(property), "apple,%s", name); + if (!of_find_property(pcie->dev->of_node, property, NULL)) + return 0; + count = of_property_count_u32_elems(pcie->dev->of_node, property); + if (count <= 0 || count > ARRAY_SIZE(cells) || count % 4) + return -EINVAL; + ret = of_property_read_u32_array(pcie->dev->of_node, property, cells, count); + if (ret) + return ret; + for (int i = 0; i < count; i += 4) { + if (cells[i + 1] != 4) + return -EINVAL; + ret = t6050_read(ctx, reg, cells[i], &old); + if (ret) + return ret; + ret = t6050_write(ctx, reg, cells[i], (old & ~cells[i + 2]) | cells[i + 3]); + if (ret) + return ret; + } + return 0; +} + +static int t6050_gpio_read(void *ctx, u32 *value) +{ + *value = readl(((struct t6050_pcie *)ctx)->gpio); + return 0; +} +static int t6050_gpio_write(void *ctx, u32 value) +{ + writel(value, ((struct t6050_pcie *)ctx)->gpio); + return 0; +} +static int t6050_reset(void *ctx, bool asserted) +{ + return t6050_pcie_gpio_perst(&((struct t6050_pcie *)ctx)->pad, asserted); +} +static int t6050_force_available(void *ctx, bool on) +{ + return apple_dart_x1n1_set_power(&((struct t6050_pcie *)ctx)->endpoint->dev, on); +} + +struct t6050_pcie *t6050_pcie_init(struct platform_device *pdev) +{ + struct t6050_pcie *pcie; + struct resource *r; + int ret; + + if (!of_machine_is_compatible("apple,j714s") || + !platform_get_resource_byname(pdev, IORESOURCE_MEM, "port")) + return NULL; + pcie = devm_kzalloc(&pdev->dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return ERR_PTR(-ENOMEM); + pcie->dev = &pdev->dev; + mutex_init(&pcie->lock); + pcie->port = devm_platform_ioremap_resource_byname(pdev, "port"); + pcie->phy = devm_platform_ioremap_resource_byname(pdev, "phy"); + pcie->gpio = devm_platform_ioremap_resource_byname(pdev, "perst"); + pcie->intr2axi = devm_platform_ioremap_resource_byname(pdev, "intr2axi"); + if (IS_ERR(pcie->port)) return ERR_CAST(pcie->port); + if (IS_ERR(pcie->phy)) return ERR_CAST(pcie->phy); + if (IS_ERR(pcie->gpio)) return ERR_CAST(pcie->gpio); + if (IS_ERR(pcie->intr2axi)) return ERR_CAST(pcie->intr2axi); + r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "port"); + if (resource_size(r) != 0x8000) + return ERR_PTR(-EINVAL); + pcie->pad = (struct t6050_gpio_pad_io){ .context = pcie, + .read = t6050_gpio_read, .write = t6050_gpio_write }; + pcie->io = (struct t6050_pcie_io){ .context = pcie, + .read = t6050_read, .write = t6050_write, .poll = t6050_poll, + .apply_tunables = t6050_tune, .apply_port_tunables = t6050_tune, + .read16 = t6050_read16, .write16 = t6050_write16, + .delay = t6050_delay, .endpoint_reset = t6050_reset }; + /* The preinitialized handoff leaves endpoint bus mastering/MSI disabled. */ + ret = t6050_pcie_configure_msi_port0(&pcie->io); + return ret ? ERR_PTR(ret) : pcie; +} + +int t6050_pcie_cycle(struct t6050_pcie *pcie, struct pci_dev *endpoint) +{ + struct pci_dev *bridge = endpoint->bus->self, *sibling; + u32 value, identity; + u16 command, pmctrl; + int ret, pm; + + /* This operation is the N1 ROM/preboot transition, before radio functions + * are enumerated. Never reset a bus with active sibling functions. */ + if (!bridge || endpoint->vendor != PCI_VENDOR_ID_APPLE || + (endpoint->device != 0x1900 && endpoint->device != 0x1901)) + return -ENODEV; + pci_lock_rescan_remove(); + mutex_lock(&pcie->lock); + list_for_each_entry(sibling, &endpoint->bus->devices, bus_list) { + if (sibling != endpoint) { + ret = -EBUSY; + goto out; + } + } + pcie->bridge = bridge; + pcie->endpoint = endpoint; + ret = pci_save_state(bridge); + if (ret) goto out; + ret = pci_save_state(endpoint); + if (ret) goto out; + pci_read_config_word(endpoint, PCI_COMMAND, &command); + pm = pci_find_capability(endpoint, PCI_CAP_ID_PM); + if (!pm) { ret = -ENODEV; goto out; } + pci_clear_master(endpoint); + if (!pci_wait_for_pending_transaction(endpoint)) { + ret = -EBUSY; + goto stopped; + } + pci_read_config_word(endpoint, pm + PCI_PM_CTRL, &pmctrl); + pci_write_config_word(endpoint, pm + PCI_PM_CTRL, + (pmctrl & ~PCI_PM_CTRL_STATE_MASK) | PCI_D3hot | PCI_PM_CTRL_PME_ENABLE); + msleep(10); + /* Native requestPMEToBroadcast, then disableRootPort stops bridge decode + * before waiting for PME acknowledgement and L2 entry. */ + pci_write_config_word(bridge, PCI_COMMAND, 0); + pci_read_config_dword(bridge, PCI_PRIMARY_BUS, &value); + pci_write_config_dword(bridge, PCI_PRIMARY_BUS, value & 0xff0000ff); + value = readl(pcie->port + 0x800); + writel(value | 0x100, pcie->port + 0x800); + writel(0x11, pcie->port + 0x88); + + ret = t6050_poll(pcie, T6050_PCIE_PORT0, 0x88, 1, 0, 500000); + if (ret) goto stopped; + ret = t6050_poll(pcie, T6050_PCIE_PORT0, 0x208, BIT(6), BIT(6), 500000); + if (ret) goto stopped; + ret = t6050_reset(pcie, true); + if (ret) goto stopped; + writel(readl(pcie->phy) & ~BIT(31), pcie->phy); + ret = t6050_pcie_disable_port0(&pcie->io, t6050_force_available, 250000); + if (ret) goto stopped; + ret = t6050_pcie_enable_port0(&pcie->io, 250000); + if (ret) goto stopped; + ret = t6050_pcie_configure_port0(&pcie->io, 3); + if (ret) goto stopped; + /* Port reset clears both RID and MSI tables. Restore measured routing + * while endpoint PERST is held and every DMA stream is quiescent. */ + for (unsigned int sid = 1; sid <= 3; sid++) + writel(BIT(31) | (sid << 16) | (0x100 + sid - 1), pcie->port + 0x3000 + 4 * sid); + ret = t6050_pcie_configure_msi_port0(&pcie->io); + if (ret) goto stopped; + ret = t6050_force_available(pcie, true); + if (ret) goto stopped; + ret = t6050_pcie_train_port0(&pcie->io, 500000); + if (ret) goto stopped; + pci_restore_state(bridge); + pci_read_config_dword(endpoint, PCI_VENDOR_ID, &identity); + if (identity != 0x1900106b && identity != 0x1901106b) { + ret = -ENODEV; + goto stopped; + } + pci_restore_state(endpoint); + pci_write_config_word(endpoint, pm + PCI_PM_CTRL, pmctrl & ~PCI_PM_CTRL_STATE_MASK); + msleep(10); + pci_write_config_word(endpoint, PCI_COMMAND, command); + dev_info(&endpoint->dev, "T6050 native preboot port cycle complete, identity=%#x\n", identity); + ret = 0; + goto out; +stopped: + /* No rollback re-enables DMA after a partial hardware transition. */ + pci_write_config_word(bridge, PCI_COMMAND, 0); + dev_err(&endpoint->dev, "T6050 port cycle stopped: %d\n", ret); +out: + pcie->endpoint = pcie->bridge = NULL; + mutex_unlock(&pcie->lock); + pci_unlock_rescan_remove(); + return ret; +} diff --git a/drivers/pci/controller/apple-x1n1/t6050.h b/drivers/pci/controller/apple-x1n1/t6050.h new file mode 100644 index 0000000000000..96ed1cdadebac --- /dev/null +++ b/drivers/pci/controller/apple-x1n1/t6050.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _X1N1_T6050_H +#define _X1N1_T6050_H +#include +#include +struct t6050_pcie; +struct t6050_pcie *t6050_pcie_init(struct platform_device *pdev); +int t6050_pcie_cycle(struct t6050_pcie *pcie, struct pci_dev *endpoint); +#endif diff --git a/drivers/pci/controller/apple-x1n1/t6050_pcie_disable.c b/drivers/pci/controller/apple-x1n1/t6050_pcie_disable.c new file mode 100644 index 0000000000000..504181293ba36 --- /dev/null +++ b/drivers/pci/controller/apple-x1n1/t6050_pcie_disable.c @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: MIT */ +#include "t6050_pcie_disable.h" + +static int update(const struct t6050_pcie_io *io, unsigned reg, unsigned off, + u32 clear, u32 set) +{ + u32 v; + int ret = io->read(io->context, reg, off, &v); + return ret ? ret : io->write(io->context, reg, off, (v & ~clear) | set); +} + +/* Native27 APCIECoreRCGen4Port::disablePortHardware, T6050 AUS5 port0 branch. + * Caller has stopped endpoint DMA, requested PME/L2, disabled bridge command + * decode and endpoint access, and asserted GPIO PERST. DART force_available + * must implement the legal native DART power transition, not a guessed MMIO + * bit. No cleanup tries to restart DMA after an error. */ +int t6050_pcie_disable_port0(const struct t6050_pcie_io *io, + int (*force_available)(void *, bool), + unsigned timeout_us) +{ + u32 v; + unsigned elapsed; + int ret; + if (!io || !io->read || !io->write || !io->poll || !io->delay || + !force_available || !timeout_us) + return -1; + ret = force_available(io->context, false); /* Fact at native9c8140c..43c */ + if (ret) return ret; + /* Native first checks804 bit2, then polls820 bit2 until it is SET. */ + ret = io->read(io->context, T6050_PCIE_PORT0, 0x804, &v); + if (ret) return ret; + if (!(v & 4)) { + ret = io->poll(io->context, T6050_PCIE_PORT0, 0x820, 4, 4, + timeout_us < 10000 ? timeout_us : 10000); + if (ret) return ret; + } + /* Native9c81560 waits only while BOTH counters are nonzero. */ + for (elapsed = 0;; elapsed++) { + ret = io->read(io->context, T6050_PCIE_PORT0, 0x820, &v); + if (ret) return ret; + if (!(v & 0x1ff) || !(v & 0xff0000)) break; + if (elapsed >= timeout_us || elapsed >= 10000) return -3; + io->delay(io->context, 1); + } + ret = update(io, T6050_PCIE_PORT0, 0x13c, 0, 0x100); + if (ret) return ret; + /* Native9c816d4 exits when LTSSM is NOT0x14; J813 helper differs. */ + for (elapsed = 0;; elapsed += 5) { + ret = io->read(io->context, T6050_PCIE_ECAM, 0x728, &v); + if (ret) return ret; + if ((v & 0x3f) != 0x14) break; + if (elapsed >= timeout_us || elapsed >= 5000) return -3; + io->delay(io->context, 5); + } + /* AUS5 skips the legacy208 bit12 wait entirely. Native9c81a5c onward. */ + ret = io->poll(io->context, T6050_PCIE_PORT0, 0xa8, 1, 1, + timeout_us < 10000 ? timeout_us : 10000); + if (ret) return ret; + ret = update(io, T6050_PCIE_PORT0, 0x82c, 0, 1u << 24); + if (ret) return ret; + ret = io->poll(io->context, T6050_PCIE_PORT0, 0xa8, 1u << 16, 1u << 16, + timeout_us < 2000 ? timeout_us : 2000); + if (ret) return ret; + ret = update(io, T6050_PCIE_PORT0, 0x82c, 0, 1u << 16); + if (ret) return ret; + io->delay(io->context, 1); + ret = update(io, T6050_PCIE_PORT0, 0x82c, 1, 0); + if (ret) return ret; + io->delay(io->context, 1); + ret = update(io, T6050_PCIE_PORT0, 0x82c, 1u << 24, 0); + if (ret) return ret; + ret = update(io, T6050_PCIE_PORT0, 0x13c, 0x100, 0); + if (ret) return ret; + static const u32 clear[] = {1u << 31, 1u << 10, 1u << 9, 0, 1, 2}; + for (unsigned i = 0; i < sizeof(clear)/sizeof(clear[0]); i++) { + ret = update(io, T6050_PCIE_PORT0_PHY, 0, clear[i], i == 3 ? 0x10 : 0); + if (ret) return ret; + } + ret = io->write(io->context, T6050_PCIE_PORT0, 0x100, (~0U)); + if (ret) return ret; + ret = update(io, T6050_PCIE_PORT0, 0x800, 1, 0); + if (ret) return ret; + /* Native final RUN loop is unbounded; propagate a bounded timeout here. */ + return io->poll(io->context, T6050_PCIE_PORT0, 0x804, 1, 0, timeout_us); +} diff --git a/drivers/pci/controller/apple-x1n1/t6050_pcie_disable.h b/drivers/pci/controller/apple-x1n1/t6050_pcie_disable.h new file mode 100644 index 0000000000000..c165551ef73a4 --- /dev/null +++ b/drivers/pci/controller/apple-x1n1/t6050_pcie_disable.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef T6050_PCIE_DISABLE_H +#define T6050_PCIE_DISABLE_H +#include "t6050_pcie_sequence.h" +/* See t6050_pcie_disable.c for the mandatory quiescence/PME/PERST contract. */ +int t6050_pcie_disable_port0(const struct t6050_pcie_io *io, + int (*force_available)(void *, bool), + unsigned timeout_us); +#endif diff --git a/drivers/pci/controller/apple-x1n1/t6050_pcie_gpio.c b/drivers/pci/controller/apple-x1n1/t6050_pcie_gpio.c new file mode 100644 index 0000000000000..37ef775efc0d9 --- /dev/null +++ b/drivers/pci/controller/apple-x1n1/t6050_pcie_gpio.c @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT */ +#include "t6050_pcie_gpio.h" + +/* Native27 AppleARMGPIOFunction callFunction 921bcb4: data=logical^flags[1]^1. + * Target function-perst flags[1]=0, so assertion drives low. + * setGPIODirection 9218efc, absent ie-fix: clear0x27f, set0x202|data. + * _safeReconfigure 92197d8 sequences the input-enable bit separately. */ +int t6050_pcie_gpio_perst(const struct t6050_gpio_pad_io *io, bool asserted) +{ + u32 old, desired, first, verify; + int ret; + if (!io || !io->read || !io->write) + return -1; + ret = io->read(io->context, &old); + if (ret) + return ret; + /* A locked pad is not changed; do not duplicate the native panic path. */ + if (old & (1u << 21)) + return -2; + desired = (old & ~0x27fu) | 0x202u | !asserted; + first = (desired & ~0x200u) | (old & 0x200u); + ret = io->write(io->context, first); + if (ret) + return ret; + if (!(old & 0x200u)) { + ret = io->write(io->context, desired); + if (ret) + return ret; + } + ret = io->read(io->context, &verify); + if (ret) + return ret; + /* Ignore unrelated input/status bits; verify only configuration touched. */ + return ((verify ^ desired) & 0x27fu) ? -3 : 0; +} diff --git a/drivers/pci/controller/apple-x1n1/t6050_pcie_gpio.h b/drivers/pci/controller/apple-x1n1/t6050_pcie_gpio.h new file mode 100644 index 0000000000000..f4036f9d8e0b9 --- /dev/null +++ b/drivers/pci/controller/apple-x1n1/t6050_pcie_gpio.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef T6050_PCIE_GPIO_H +#define T6050_PCIE_GPIO_H +#include + +/* Callbacks access the resolved GPIO0 pin80 pad (target PA0x29c000140). + * They must preserve SPTM ownership and report guarded access exceptions. + * No physical address mapping or protection bypass occurs here. */ +struct t6050_gpio_pad_io { + void *context; + int (*read)(void *, u32 *); + int (*write)(void *, u32); +}; +int t6050_pcie_gpio_perst(const struct t6050_gpio_pad_io *, bool asserted); +#endif diff --git a/drivers/pci/controller/apple-x1n1/t6050_pcie_msi.c b/drivers/pci/controller/apple-x1n1/t6050_pcie_msi.c new file mode 100644 index 0000000000000..e6d726c00ca6d --- /dev/null +++ b/drivers/pci/controller/apple-x1n1/t6050_pcie_msi.c @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +#include "t6050_pcie_sequence.h" +/* Native27 configMSIRange9c8799c, target ADT msi-address fffff000, + * port0 msi-vector-base0/#msi-vectors32. All endpoints must have MSI and bus + * mastering disabled during setup. This only installs routing; Linux owns + * MSI vector allocation and AIC enable/mask/EOI. */ +int t6050_pcie_configure_msi_port0(const struct t6050_pcie_io *io) +{ + int ret; + if (!io || !io->write) + return -1; + ret=io->write(io->context,T6050_PCIE_PORT0,0x124,1); + if(ret)return ret; + ret=io->write(io->context,T6050_PCIE_PORT0,0x16c,0xfffff000); + if(ret)return ret; + ret=io->write(io->context,T6050_PCIE_PORT0,0x170,0); + if(ret)return ret; + for(unsigned i=0;i<32;i++) { + ret=io->write(io->context,T6050_PCIE_PORT0,0x3800+4*i,0x80000000|i); + if(ret)return ret; + } + return 0; +} diff --git a/drivers/pci/controller/apple-x1n1/t6050_pcie_port.c b/drivers/pci/controller/apple-x1n1/t6050_pcie_port.c new file mode 100644 index 0000000000000..08d3759a69ad6 --- /dev/null +++ b/drivers/pci/controller/apple-x1n1/t6050_pcie_port.c @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: MIT */ +#include "t6050_pcie_sequence.h" + +/* Target: apcie,t6050 GP port0, four ports, lane-cfg0, ge-harvested present. + * All access remains in caller callbacks, which must enforce SPTM ownership. + * This does not power-cycle an endpoint or assign a DART SID implicitly. */ +static int rmw(const struct t6050_pcie_io *io, unsigned reg, unsigned off, + u32 clear, u32 set) +{ + u32 v; + int ret = io->read(io->context, reg, off, &v); + return ret ? ret : io->write(io->context, reg, off, (v & ~clear) | set); +} + +int t6050_pcie_reset_port0(const struct t6050_pcie_io *io) +{ + static const struct { unsigned off; u32 value; } first[] = { + {0x88,0x110}, {0x100,0xffffffff}, {0x148,0xffffffff}, {0x210,0xffffffff}, + {0x80,0}, {0x84,0}, {0x104,0xfffffff0}, {0x124,0x100}, {0x16c,0}, + {0x13c,0}, {0x800,0x100100}, {0x808,0x1001ff}, {0x82c,0x10000}, + }, last[] = { + {0x130,0x03020000}, {0x140,0x10}, {0x144,0x253770}, + {0x21c,0}, {0x834,0}, {0x83c,0}, + }; + int ret; + if (!io || !io->write) + return -1; + /* APCIECoreRCGen4Port::_resetPortHardware, native27.0 9c803e8..9c808d4. */ + for (unsigned i=0; iwrite(io->context,T6050_PCIE_PORT0,first[i].off,first[i].value); + if (ret) return ret; + } + for (unsigned i=0; i<64; i++) { + ret=io->write(io->context,T6050_PCIE_PORT0,0x3000+4*i,0); + if (ret) return ret; + } + for (unsigned i=0; i<256; i++) { + ret=io->write(io->context,T6050_PCIE_PORT0,0x3800+4*i,0); + if (ret) return ret; + } + for (unsigned i=0; iwrite(io->context,T6050_PCIE_PORT0,last[i].off,last[i].value); + if (ret) return ret; + } + return 0; +} + +int t6050_pcie_enable_port0(const struct t6050_pcie_io *io, unsigned timeout_us) +{ + u32 v; + unsigned waited=0; + int ret; + if (!io || !io->read || !io->write || !io->poll || !io->delay || + !io->apply_port_tunables || !timeout_us) + return -1; + ret=t6050_pcie_reset_port0(io); + if (ret) return ret; + /* Native T6050 _linkUsesAUS5PHY() is always true. */ + ret=io->read(io->context,T6050_PCIE_PORT0,0x82c,&v); + if (ret) return ret; + if (!(v & (1u<<16))) return -2; + ret=io->apply_port_tunables(io->context,"apcie-config-tunables",T6050_PCIE_PORT0); + if (ret) return ret; + ret=rmw(io,T6050_PCIE_PORT0,0x800,0,1); + if (ret) return ret; + /* Native9c7ffc8 waits while BOTH reset and idle-ack remain asserted. + * Target port0/lane-cfg0 masks are1 (9c78b98..bb0 and9c78e08..e20). */ + for (;;) { + ret=io->read(io->context,T6050_PCIE_PORT0,0xa8,&v); + if (ret) return ret; + if (!(v & 1) || !(v & (1u<<16))) break; + if (waited++ >= timeout_us || waited > 10000) return -3; + io->delay(io->context,1); + } + ret=rmw(io,T6050_PCIE_PORT0,0x82c,0,1); /* release root-port PERST */ + if (ret) return ret; + ret=io->poll(io->context,T6050_PCIE_PORT0,0xa8,1,1, + timeout_us < 20000 ? timeout_us : 20000); + if (ret) return ret; + ret=rmw(io,T6050_PCIE_PORT0,0x82c,1u<<16,0); + if (ret) return ret; + ret=io->poll(io->context,T6050_PCIE_PORT0,0x804,1,1,timeout_us); + if (ret) return ret; + /* Native enablePortHardware keeps APPCLK_AUTO_DIS during config. */ + ret=rmw(io,T6050_PCIE_PORT0,0x800,0,0x100); + if (ret) return ret; + return io->poll(io->context,T6050_PCIE_PORT0,0x208,4,0,timeout_us); +} + +static int find_pcie_cap(const struct t6050_pcie_io *io, unsigned *cap) +{ + u32 v; + int ret=io->read(io->context,T6050_PCIE_ECAM,0x34,&v); + if (ret) return ret; + unsigned off=v & 0xff; + for (unsigned i=0; i<48 && off; i++) { + if (off<0x40 || (off & 3)) return -2; + ret=io->read(io->context,T6050_PCIE_ECAM,off,&v); + if (ret) return ret; + if ((v & 0xff)==0x10) { *cap=off; return 0; } + off=(v>>8)&0xff; + } + return -2; +} + +int t6050_pcie_configure_port0(const struct t6050_pcie_io *io, unsigned max_speed) +{ + static const char *const shadow[] = {"pcie-rc-gen3-shadow-tunables", + "pcie-rc-gen4-shadow-tunables", "pcie-rc-gen5-shadow-tunables"}; + const u32 select[] = {0,1u<<24,0}; + unsigned cap; + u32 v; + u16 v16; + int ret; + if (!io || !io->read || !io->write || !io->read16 || !io->write16 || + !io->apply_port_tunables || max_speed!=3) + return -1; /* Current target ADT maximum-link-speed is3. */ + ret=find_pcie_cap(io,&cap); + if (ret) return ret; + ret=rmw(io,T6050_PCIE_ECAM,0x8bc,0,1); + if (ret) return ret; + ret=io->apply_port_tunables(io->context,"pcie-rc-tunables",T6050_PCIE_ECAM); + if (ret) goto lock; + /* Native _applyrcTunables9c82500: selector before each target dataset. */ + for (unsigned i=0; i<3; i++) { + ret=rmw(io,T6050_PCIE_ECAM,0x890,3u<<24,select[i]); + if (ret) goto lock; + ret=io->apply_port_tunables(io->context,shadow[i],T6050_PCIE_ECAM); + if (ret) goto lock; + } + ret=rmw(io,T6050_PCIE_ECAM,0x194,1,1); /* T6050 intermediate-rate training */ + if (ret) goto lock; + ret=io->read(io->context,T6050_PCIE_ECAM,cap+0x2c,&v); + if (ret) goto lock; + if (!(v & (1u<read16(io->context,T6050_PCIE_ECAM,cap+0x30,&v16); + if (ret) goto lock; + ret=io->write16(io->context,T6050_PCIE_ECAM,cap+0x30,(v16 & ~0xf)|max_speed); + if (ret) goto lock; + /* Native target port0 width1: LINK_MODE1, NUM_LANES1, advertised width1. */ + ret=rmw(io,T6050_PCIE_ECAM,0x710,0x3fu<<16,1u<<16); + if (ret) goto lock; + ret=rmw(io,T6050_PCIE_ECAM,0x80c,0x1fu<<8,1u<<8); + if (ret) goto lock; + ret=rmw(io,T6050_PCIE_ECAM,cap+0xc,0x3fu<<4,1u<<4); +lock: + /* Native clears the DBI override at9c81310; do so on error as well. */ + { + int cleanup=rmw(io,T6050_PCIE_ECAM,0x8bc,1,0); + if (!ret) ret=cleanup; + } + if (ret) return ret; + ret=rmw(io,T6050_PCIE_ECAM,cap+0x10,0,0x400); + if (ret) return ret; + ret=rmw(io,T6050_PCIE_ECAM,0x80c,0,1u<<17); + if (ret) return ret; + /* Native _initializeRootComplex tail routes the port interrupt block. */ + return io->write(io->context,T6050_PCIE_PORT0_INTR2AXI,0x80,1); +} + +int t6050_pcie_train_port0(const struct t6050_pcie_io *io, unsigned timeout_us) +{ + int ret; + if (!io || !io->read || !io->write || !io->poll || !io->delay || + !io->endpoint_reset || !timeout_us) + return -1; + /* initializeRefclkBuffer9c7fc40, mapped PHY lane reg18 for port0. */ + ret=rmw(io,T6050_PCIE_PORT0_PHY,0,0,1); + if (ret) return ret; + ret=io->poll(io->context,T6050_PCIE_PORT0_PHY,0,4,4,timeout_us); + if (ret) return ret; + ret=rmw(io,T6050_PCIE_PORT0_PHY,0,0,2); + if (ret) return ret; + ret=io->poll(io->context,T6050_PCIE_PORT0_PHY,0,8,8,timeout_us); + if (ret) return ret; + ret=rmw(io,T6050_PCIE_PORT0_PHY,0,0x10,0); + if (ret) return ret; + io->delay(io->context,1); + ret=rmw(io,T6050_PCIE_PORT0_PHY,0,0,0x200); + if (ret) return ret; + ret=rmw(io,T6050_PCIE_PORT0_PHY,0,0,0x400); + if (ret) return ret; + io->delay(io->context,100); /* Target t-refclk-to-perst, microseconds. */ + ret=io->endpoint_reset(io->context,false); + if (ret) return ret; + ret=rmw(io,T6050_PCIE_PORT0,0x80,0,1); + if (ret) return ret; + ret=io->poll(io->context,T6050_PCIE_PORT0,0x208,1,1,timeout_us); + if (ret) return ret; + io->delay(io->context,100000); /* Target perst-to-config is milliseconds. */ + return 0; +} diff --git a/drivers/pci/controller/apple-x1n1/t6050_pcie_sequence.h b/drivers/pci/controller/apple-x1n1/t6050_pcie_sequence.h new file mode 100644 index 0000000000000..25bfac2693630 --- /dev/null +++ b/drivers/pci/controller/apple-x1n1/t6050_pcie_sequence.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef T6050_PCIE_SEQUENCE_H +#define T6050_PCIE_SEQUENCE_H +#include + + +/* The caller owns mapped ADT resources and SPTM IOFILTER policy. These helpers + * never translate an arbitrary physical address or bypass protected writes. + * read/write return 0 on success; poll returns 0 only after the masked value + * matches, with the supplied finite timeout. */ +struct t6050_pcie_io { + void *context; + int (*read)(void *, unsigned adt_reg, unsigned offset, u32 *); + int (*write)(void *, unsigned adt_reg, unsigned offset, u32); + int (*poll)(void *, unsigned adt_reg, unsigned offset, + u32 mask, u32 value, unsigned timeout_us); + int (*enable_clock)(void *, unsigned adt_clock_index); + int (*apply_tunables)(void *, const char *property, unsigned adt_reg); + int (*apply_port_tunables)(void *, const char *property, unsigned adt_reg); + int (*read16)(void *, unsigned adt_reg, unsigned offset, u16 *); + int (*write16)(void *, unsigned adt_reg, unsigned offset, u16); + void (*delay)(void *, unsigned usec); + /* GPIO PERST callback: true asserts, false releases; caller resolves the + * target ADT function. This is distinct from SMC function-reset. */ + int (*endpoint_reset)(void *, bool asserted); +}; + +/* Native27.0 AppleT6050PCIe::start pins these indexes, PCs9c6dd94..9c6dde4. */ +enum { + T6050_PCIE_ECAM = 0, + T6050_PCIE_RC = 1, + T6050_PCIE_PHY = 2, + T6050_PCIE_PHY_PHY = 3, + T6050_PCIE_PHY_IP = 4, + T6050_PCIE_AXI = 5, + T6050_PCIE_CLKGEN = 6, + T6050_PCIE_GP_AUS5_TOP = 10, + T6050_PCIE_GP_AUS5_AON_TOP = 11, + T6050_PCIE_PORT0 = 16, + T6050_PCIE_PORT0_PHY = 18, + T6050_PCIE_PORT0_INTR2AXI = 19, + T6050_PCIE_PORT_STRIDE = 6, +}; + +/* Implements the decoded Gen5 PHY tail after power/clocks/tunables and lane + * configuration are established. It is NOT a full controller or link init. + * Native27.0 GPInit9c7dc44 and9c7de50, Gen5Init9c73fa8. */ +int t6050_pcie_finish_gen5_phy(const struct t6050_pcie_io *io, unsigned timeout_us); + +/* Four-port T6050 with the ge-harvested property present, as on this J714s. + * enable_clock resolves ADT clock indices, enables parent dependencies, and + * must return failure if its required power/clock state is not established. + * apply_tunables uses this target's ADT data and returns 0 for an absent optional + * property (the native common-tunables call also accepts absent data). + * This prepares the GP PHY; root-port/PERST/link training are separate. */ +int t6050_pcie_prepare_gp_phy(const struct t6050_pcie_io *io, unsigned lane_cfg, + unsigned timeout_us); + +/* After GP PHY preparation, enable native RC/PTM and wait for RC+0x58 bit0. + * Still does not release an endpoint or train an individual port. */ +int t6050_pcie_finish_gp_controller(const struct t6050_pcie_io *io, unsigned timeout_us); + +/* Native getLinkUp9c837c8: port+0x208 BIT0 (not J813 helper's BIT12). */ +int t6050_pcie_get_n1_link(const struct t6050_pcie_io *io, bool *up); + +/* Pure native RID→SID format; caller must allocate a free SPTM-valid SID and + * keep bus mastering stopped while installing routing. Native uses entries1..15 + * and slot==SID. This function does not choose a SID or mutate hardware. */ +int t6050_pcie_n1_rid_entry(unsigned slot, unsigned sid, unsigned rid, + u32 *offset, u32 *value); +/* Native port0 phases; endpoint must be held in reset before calling reset/ + * enable/configure, so clearing RID/MSI routes cannot affect active DMA. + * The host must install explicit SPTM-valid DART routing before train_port0. + * No implicit SID allocation or diagnostic polling transport is introduced. */ +int t6050_pcie_reset_port0(const struct t6050_pcie_io *io); +int t6050_pcie_enable_port0(const struct t6050_pcie_io *io, unsigned timeout_us); +int t6050_pcie_configure_port0(const struct t6050_pcie_io *io, unsigned max_speed); +int t6050_pcie_train_port0(const struct t6050_pcie_io *io, unsigned timeout_us); +int t6050_pcie_configure_msi_port0(const struct t6050_pcie_io *io); +#endif diff --git a/include/linux/apple-dart-x1n1.h b/include/linux/apple-dart-x1n1.h new file mode 100644 index 0000000000000..210ca1cb00d64 --- /dev/null +++ b/include/linux/apple-dart-x1n1.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_APPLE_DART_X1N1_H +#define _LINUX_APPLE_DART_X1N1_H +#include +#include +#include +#if IS_REACHABLE(CONFIG_APPLE_DART_X1N1) +/* Caller must quiesce every device on this DART before powering it down. + * Tables and mappings survive the native SPTM power transition. */ +int apple_dart_x1n1_set_power(struct device *dev, bool on); +#else +static inline int apple_dart_x1n1_set_power(struct device *dev, bool on) +{ + return -EOPNOTSUPP; +} +#endif +#endif diff --git a/include/linux/pci-apple-x1n1.h b/include/linux/pci-apple-x1n1.h new file mode 100644 index 0000000000000..a5d116fa9ef10 --- /dev/null +++ b/include/linux/pci-apple-x1n1.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_PCI_APPLE_X1N1_H +#define _LINUX_PCI_APPLE_X1N1_H +#include +#include +struct pci_dev; +#if IS_REACHABLE(CONFIG_PCIE_APPLE_X1N1) +int apple_x1n1_pcie_cycle(struct pci_dev *pdev); +#else +static inline int apple_x1n1_pcie_cycle(struct pci_dev *pdev) +{ + return -EOPNOTSUPP; +} +#endif +#endif From 09f40a067614f7faa8e5c74ea52e75122a3adf10 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 22:11:47 -0400 Subject: [PATCH 0160/1579] wifi: apple: match native J714 macOS 27 scan request --- .../wireless/apple/centauri/centauri_wifi.c | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index ffbf3a4c656a4..9dda5028dec70 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -101,6 +102,7 @@ struct centauri_wifi { u32 reply_command, channels_seen, scans_done, beacons_seen; bool reply_valid, registered, configured; bool stopping; + u8 scan_sequence; bool connecting, connected, connect_event, disconnect_event; bool associated, pmk_seen, pmk_matches; u8 expected_pmk[32]; @@ -800,6 +802,100 @@ static int radio_start(struct centauri_wifi *wifi) return 0; } +/* Current macOS27 buildScanRequestNewApi (1809bf400) always appends the + * channel-list TLV260 and five-byte options TLV357 to its14-byte header. + * Each channel is {number, ACI band, 0, 0}; the native bound is59 entries. + */ +#define CEN_SCAN_CHANNELS_MAX 59 +static int scan_native27(struct centauri_wifi *wifi) +{ + struct cfg80211_scan_request *request = wifi->scan_request; + u8 scan[14 + 5 + 4 * CEN_SCAN_CHANNELS_MAX + 9 + 4 + 2 + 32]; + unsigned int next = 0; + + if (!request || !request->n_channels || request->n_ssids > 1) + return -EINVAL; + while (next < request->n_channels) { + u8 channels[1 + 4 * CEN_SCAN_CHANNELS_MAX] = {}; + u8 options[5] = { 0, 0, 1, 0, 0 }; + unsigned int count = 0; + u32 done = wifi->scans_done; + size_t size = 14; + unsigned long deadline; + int ret; + + memset(scan, 0, sizeof(scan)); + scan[0] = (++wifi->scan_sequence) & 0x1f; + scan[3] = 0x80; + scan[5] = 2; + scan[7] = 0x10; + /* Native scan defaults: active min/max 40 ms, passive 110 ms. */ + put_unaligned_le16(40, scan + 8); + put_unaligned_le16(40, scan + 10); + put_unaligned_le16(110, scan + 12); + while (next < request->n_channels && count < CEN_SCAN_CHANNELS_MAX) { + struct ieee80211_channel *channel = request->channels[next++]; + unsigned int number; + + if (channel->flags & IEEE80211_CHAN_DISABLED) + continue; + if (channel->band != NL80211_BAND_2GHZ && + channel->band != NL80211_BAND_5GHZ) + return -EOPNOTSUPP; + number = ieee80211_frequency_to_channel(channel->center_freq); + if (!number || number > 255) + return -EINVAL; + channels[1 + 4 * count] = number; + channels[2 + 4 * count] = channel->band == NL80211_BAND_5GHZ; + count++; + } + if (!count) + continue; + channels[0] = count; + put_unaligned_le16(260, scan + size); + put_unaligned_le16(1 + 4 * count, scan + size + 2); + memcpy(scan + size + 4, channels, 1 + 4 * count); + size += 5 + 4 * count; + put_unaligned_le16(357, scan + size); + put_unaligned_le16(sizeof(options), scan + size + 2); + memcpy(scan + size + 4, options, sizeof(options)); + size += 4 + sizeof(options); + if (request->n_ssids) { + const struct cfg80211_ssid *ssid = &request->ssids[0]; + + if (ssid->ssid_len > 32) + return -EINVAL; + /* Native single-SSID TLV258 uses5 for a directed SSID; + * the zero-length wildcard form uses4 (bfc40..bfc5c). + */ + scan[2] = !!ssid->ssid_len; + put_unaligned_le16(258, scan + size); + put_unaligned_le16(2 + ssid->ssid_len, scan + size + 2); + scan[size + 4] = ssid->ssid_len ? 5 : 4; + scan[size + 5] = ssid->ssid_len; + memcpy(scan + size + 6, ssid->ssid, ssid->ssid_len); + size += 6 + ssid->ssid_len; + } + ret = command(wifi, 65551, scan, size); + if (ret) + return ret; + deadline = jiffies + msecs_to_jiffies(max(8000U, count * 220 + 2000)); + do { + drain(wifi); + if (READ_ONCE(wifi->stopping) || readl(wifi->a.bar + 8) != 2) + return -EIO; + if (wifi->scans_done != done) + break; + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + if (wifi->scans_done == done) { + command(wifi, 65537, NULL, 0); + return -ETIMEDOUT; + } + } + return 0; +} + static void scan_worker(struct work_struct *work) { struct centauri_wifi *wifi = container_of(work, struct centauri_wifi, scan_work); @@ -810,6 +906,10 @@ static void scan_worker(struct work_struct *work) int ret = 0; mutex_lock(&wifi->lock); + if (of_machine_is_compatible("apple,j714s")) { + ret = scan_native27(wifi); + goto completed; + } for (chunk = 0; chunk < 3 && !ret; chunk++) { u32 channels = wifi->channels_seen, done = wifi->scans_done; bool stopped = false; @@ -839,6 +939,7 @@ static void scan_worker(struct work_struct *work) if (!ret && wifi->scans_done == done) ret = -ETIMEDOUT; } +completed: info.aborted = !!ret; request = wifi->scan_request; wifi->scan_request = NULL; From 0aedddc23a0cb7e2f6b640086456faa1da988673 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 22:14:01 -0400 Subject: [PATCH 0161/1579] docs: record J714 native scan hardware qualification --- Documentation/arch/arm64/t6050-sptm-n1.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Documentation/arch/arm64/t6050-sptm-n1.rst b/Documentation/arch/arm64/t6050-sptm-n1.rst index efec76d9b4715..0656742d0b575 100644 --- a/Documentation/arch/arm64/t6050-sptm-n1.rst +++ b/Documentation/arch/arm64/t6050-sptm-n1.rst @@ -129,3 +129,25 @@ windows only; all endpoint/interrupt results above are from hardware. Interface registration is not association, packet transfer, Bluetooth radio qualification or native performance equivalence. Production data-ring startup, firmware27 connection details and sustained traffic are still in progress. + +Native macOS 27 scan qualification +--------------------------------- + +J714s uses the native 14-byte scan header followed by the channel-list TLV260 +and five-byte options TLV357. Channels use the ACI band encoding and respect +cfg80211's requested/allowed list. Native default dwell fields are 40/40/110 +milliseconds; the inherited 110/110/0 values caused Alpha to enter execution +stage3 on the first scan even after the required TLVs were added. + +With the corrected native request, hardware completed 38 channels and reported +36 beacons, including the requested home network. Alpha remained at execution +stage2 after completion. This establishes scanning, not association or packet +traffic. J813 retains its existing firmware-specific scan path. + +Evidence: ``hardware-linux-n1-scan-dwell-1-guest.log``; kernel +``Image-n1-scan-dwell`` SHA256: +``bf1fcb4998ed820629fd937b7dea1aeeb1570b27c5d120216ca0b6cc9eb67ce7``. +The native ABI references are ACIScanAdapter::buildScanRequestNewApi at +0x1809bf400 and IO80211ScanRequest defaults at0x1807b46d4 in the captured +DriverKit cache. Firmware payload/calibration and network details remain +private local artifacts. From f7b6fa3cb0652f862f361c65d7db850885ee7977 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 22:34:25 -0400 Subject: [PATCH 0162/1579] wifi: apple: qualify native J714 association and packet DMA --- Documentation/arch/arm64/t6050-sptm-n1.rst | 41 ++++ .../net/wireless/apple/centauri/centauri.h | 1 + .../wireless/apple/centauri/centauri_ipc.c | 4 +- .../wireless/apple/centauri/centauri_wifi.c | 227 +++++++++++++----- 4 files changed, 210 insertions(+), 63 deletions(-) diff --git a/Documentation/arch/arm64/t6050-sptm-n1.rst b/Documentation/arch/arm64/t6050-sptm-n1.rst index 0656742d0b575..2792d82f8e2dc 100644 --- a/Documentation/arch/arm64/t6050-sptm-n1.rst +++ b/Documentation/arch/arm64/t6050-sptm-n1.rst @@ -151,3 +151,44 @@ The native ABI references are ACIScanAdapter::buildScanRequestNewApi at 0x1809bf400 and IO80211ScanRequest defaults at0x1807b46d4 in the captured DriverKit cache. Firmware payload/calibration and network details remain private local artifacts. + +Wi-Fi association and data qualification +--------------------------------------- + +J714s now takes the permanent Wi-Fi address from its Alpha PCI DT node, +using the captured ADT local-mac-address. Its native27 connection request +uses the14-byte MLD TLV376 and the raw32-byte cfg80211 PMK field. Passphrase +TLV313 is a separate native key-type path; it is not required for a normal +WPA supplicant PMK on this firmware. + +Interface open creates native2048-entry CR76/CR15 and TR12/TR25 data rings, +posting2047 RX buffers. A32MiB Alpha DMA window separates command rings, +firmware scratch, RX payloads and TX payloads. Extended RX prefixes use their +u16 data offset while retaining the native completion-length-minus-padding +length rule, with explicit buffer bounds and the native delivery-status bit. +TX completions accept native single and counted forms without reducing +coalesced batches. Busy-cookie checks retain ownership until completion. + +On hardware, standard wpa_supplicant association/authentication completed, +DHCP obtained a lease, gateway ping passed5/5, and key-authenticated SSH +confirmed18CPUs online. Direct TCP transferred256MiB in each direction, +with matching SHA256 at both ends: host-to-M5 about35.9MiB/s and M5-to-host +about37.0MiB/s as measured on the M5. Both endpoints used the home wireless +network, so this is not a controlled native-performance comparison. + +The post-transfer counters reported233507 submitted/completed TX packets, +215923 received/reposted RX buffers, and zero malformed RX/TX completions +or failed TX packets. The3-form TX completion parser and RX prefix bounds +also passed isolated ASan/UBSan tests, including every2048-slot wrap and +count1..2047. The hardware does not establish every synthetic corner case. + +Evidence: ``hardware-linux-n1-network-5-guest.log``, +``n1-direct-256-host.log``, ``n1-direct-256-guest.log`` and +``n1-after-throughput-bt-baseline.log``. Tested kernel +``Image-n1-data-completions`` SHA256: ``c8754a9cad2a93b974cfa6aee955145325f7262c1b64b6f37fe50b6d26caf0df``. +That test image also contained read-only transport snapshot diagnostics +subsequently removed from the production source; they did not participate +in the network path. Factory-address DT artifact: +``t6050-n1-factory-mac.dtb``. Firmware, credentials and calibration remain +private. Automatic firmware bootstrap and Bluetooth radio bring-up are +separate remaining work; Bluetooth's default radio-power command still fails. diff --git a/drivers/net/wireless/apple/centauri/centauri.h b/drivers/net/wireless/apple/centauri/centauri.h index b2bcaedd16029..67a5d11ff8187 100644 --- a/drivers/net/wireless/apple/centauri/centauri.h +++ b/drivers/net/wireless/apple/centauri/centauri.h @@ -7,6 +7,7 @@ #include #include +#define CEN_ALPHA_WINDOW_SIZE SZ_32M #define CEN_ALPHA_SCRATCH_OFFSET SZ_8M #define CEN_ALPHA_SCRATCH_SIZE (5 * SZ_1M) diff --git a/drivers/net/wireless/apple/centauri/centauri_ipc.c b/drivers/net/wireless/apple/centauri/centauri_ipc.c index 21c309cadd404..483a869012472 100644 --- a/drivers/net/wireless/apple/centauri/centauri_ipc.c +++ b/drivers/net/wireless/apple/centauri/centauri_ipc.c @@ -438,7 +438,7 @@ int centauri_start_ipc(struct centauri *c) ret = ret ?: -EIO; goto failed; } - ipc->window = buffer_alloc(c, 1, SZ_16M); + ipc->window = buffer_alloc(c, 1, CEN_ALPHA_WINDOW_SIZE); if (!ipc->window) { ret = -ENOMEM; goto failed; @@ -449,7 +449,7 @@ int centauri_start_ipc(struct centauri *c) goto failed; writel(lower_32_bits(ipc->window->dma), ipc->bar[1] + 0x18); writel(upper_32_bits(ipc->window->dma), ipc->bar[1] + 0x1c); - writel(SZ_16M, ipc->bar[1] + 0x20); + writel(CEN_ALPHA_WINDOW_SIZE, ipc->bar[1] + 0x20); ret = handshake(ipc->bar[1], &ipc->alpha, 0xc, 0x10, 0x1028); if (ret) goto failed; diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index 9dda5028dec70..7fef85ee89912 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -21,14 +21,20 @@ #define RING_SPAN 0x20000 #define TX_BUFFER 0x600000 #define BUFFER_SIZE 2048 -#define DATA_TX_BUFFER 0xd00000 -#define DATA_RX_BUFFER 0xe00000 -#define DATA_TX_SLOTS 128 -#define DATA_TX_QUEUE_LIMIT 64 +/* Native 2048-entry data rings. RX payloads occupy16..24MiB and TX + * payloads24..28MiB; command buffers and firmware scratch stay below13MiB. + */ +#define DATA_RX_BUFFER SZ_16M +#define DATA_TX_BUFFER (SZ_16M + SZ_8M) +#define DATA_RX_SLOTS 2048 +#define DATA_TX_SLOTS 2048 +#define DATA_TX_QUEUE_LIMIT DATA_TX_SLOTS /* The firmware owns window +8 MiB through +13 MiB as scratch memory. */ static_assert(DATA_TX_BUFFER >= CEN_ALPHA_SCRATCH_OFFSET + CEN_ALPHA_SCRATCH_SIZE); -static_assert(DATA_TX_BUFFER + DATA_TX_SLOTS * BUFFER_SIZE <= DATA_RX_BUFFER); +static_assert(DATA_RX_BUFFER >= CEN_ALPHA_SCRATCH_OFFSET + CEN_ALPHA_SCRATCH_SIZE); +static_assert(DATA_RX_BUFFER + DATA_RX_SLOTS * 4096 <= DATA_TX_BUFFER); +static_assert(DATA_TX_BUFFER + DATA_TX_SLOTS * BUFFER_SIZE <= CEN_ALPHA_WINDOW_SIZE); static bool verify_pmk; module_param(verify_pmk, bool, 0400); @@ -155,6 +161,7 @@ struct centauri_wifi { struct dentry *tx_power_report_file; u16 stats_reply_size, stats_tag, stats_length; u32 data_probe_done; + bool data_failed; struct ieee80211_supported_band bands[2]; struct ieee80211_channel channels[39]; }; @@ -312,6 +319,31 @@ static void post_receive(struct centauri_wifi *wifi, unsigned int ring, u16 slot * Ethernet framing with IPv4 header checksums and IPv6 payload lengths. * Called with wifi->lock, including during connect/command polling. */ +/* Native27 uses the extended prefix's offset but still subtracts only + * footer padding from completion length (AirshipDataRxTransferRing). + */ +static bool data_rx_frame(const u8 *packet, unsigned int capacity, + unsigned int length, u32 footer, + unsigned int *offset, unsigned int *frame_length) +{ + unsigned int padding = (footer >> 12) & 15; + + /* Native recycles completions without the RX deliverable bit. */ + if (!(footer & BIT(7))) + return false; + if (length < padding + ETH_HLEN || length > capacity) + return false; + if (footer & BIT(24)) { + if (capacity < 4) + return false; + *offset = get_unaligned_le16(packet + 2); + } else { + *offset = padding; + } + *frame_length = length - padding; + return *offset <= capacity && *frame_length <= capacity - *offset; +} + static void drain_data_rx(struct centauri_wifi *wifi) { u16 head, tail, post; @@ -322,22 +354,22 @@ static void drain_data_rx(struct centauri_wifi *wifi) head = index_read(wifi, 0, 207); tail = index_read(wifi, 2, 207); post = index_read(wifi, 3, 12); - if (head >= 128 || tail >= 128 || post >= 128) { + if (head >= DATA_RX_SLOTS || tail >= DATA_RX_SLOTS || post >= DATA_RX_SLOTS) { wifi->data_rx_bad++; return; } dma_rmb(); - while (tail != head && processed < 127) { + while (tail != head && processed < (DATA_RX_SLOTS - 1)) { u8 *completion = wifi->a.window + 0x240000 + tail * 32; u32 header = get_unaligned_le32(completion); u32 info = get_unaligned_le32(completion + 4); u16 cookie = info & 0xffff, length = info >> 16; - u16 next = (post + 1) % 128; + u16 next = (post + 1) % DATA_RX_SLOTS; u32 count = le32_to_cpu(wifi->rx_snapshot.count); u8 *packet, *descriptor; u32 buffer; - if ((header >> 16) != 12 || !cookie || cookie > 127 || + if ((header >> 16) != 12 || !cookie || cookie > (DATA_RX_SLOTS - 1) || length > 4096 || get_unaligned_le32(completion + 8)) { wifi->data_rx_bad++; break; @@ -356,18 +388,18 @@ static void drain_data_rx(struct centauri_wifi *wifi) wifi->data_rx_received++; if (wifi->connected && netif_running(wifi->netdev)) { u32 footer = get_unaligned_le32(completion + 12); - unsigned int padding = (footer >> 12) & 15; + unsigned int offset, frame_length; struct sk_buff *skb = NULL; - /* Extended prefix headers are not decoded yet. */ - if (!(footer & BIT(24)) && length >= padding + ETH_HLEN) - skb = netdev_alloc_skb_ip_align(wifi->netdev, length - padding); + if (data_rx_frame(packet, 4096, length, footer, + &offset, &frame_length)) + skb = netdev_alloc_skb_ip_align(wifi->netdev, frame_length); if (skb) { - skb_put_data(skb, packet + padding, length - padding); + skb_put_data(skb, packet + offset, frame_length); skb->protocol = eth_type_trans(skb, wifi->netdev); skb->ip_summed = CHECKSUM_NONE; wifi->netdev->stats.rx_packets++; - wifi->netdev->stats.rx_bytes += length - padding; + wifi->netdev->stats.rx_bytes += frame_length; netif_rx(skb); } else { wifi->netdev->stats.rx_dropped++; @@ -380,7 +412,7 @@ static void drain_data_rx(struct centauri_wifi *wifi) put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); put_unaligned_le32(cookie, descriptor + 12); wifi->data_rx_reposted++; - tail = (tail + 1) % 128; + tail = (tail + 1) % DATA_RX_SLOTS; post = next; processed++; } @@ -396,10 +428,27 @@ static void drain_data_rx(struct centauri_wifi *wifi) /* Serialized with the firmware poller. Payload slots remain owned until a * CR completion, even if the device has already consumed the TR descriptor. */ +/* Native complete_transfer requires count>0 exactly for counted CDs; + * the uncounted form completes the named cookie with a zero wire count. + * Our one-buffer transfers have no continuation fragments. + */ +static bool data_tx_completion_span(u16 cookie, u16 wire_count, u8 flags, + unsigned int *first, u16 *count) +{ + bool counted = (flags & BIT(0)) || (flags & 0x0e) == 8; + + if (!cookie || cookie > DATA_TX_SLOTS || counted != !!wire_count || + wire_count >= DATA_TX_SLOTS) + return false; + *count = counted ? wire_count : 1; + *first = (cookie + DATA_TX_SLOTS - *count) % DATA_TX_SLOTS; + return true; +} + static void drain_data_tx(struct centauri_wifi *wifi) { u16 head, tail, post; - unsigned int budget = 256; + unsigned int budget = DATA_TX_SLOTS; bool posted = false, completed = false; if (!(wifi->data_probe_done & BIT(5)) || readl(wifi->a.bar + 8) != 2) @@ -420,18 +469,13 @@ static void drain_data_tx(struct centauri_wifi *wifi) if (ring == 25) { unsigned int first, i; - /* Airship cr_controller::sync maps CD bytes 9:10 to the - * transfer count. With the multicomp flag (CD byte 1 bit 0), - * the cookie names the LAST transfer; prior transfers share - * its completion status. We submit one contiguous DMA buffer - * per slot, so those preceding cookies must all still be busy. + /* CD bytes9:10 are the wire count. For counted completions, + * the cookie names the last completed transfer, including wrap. */ - if (!cookie || cookie > DATA_TX_SLOTS || !count || - count >= DATA_TX_SLOTS || !(entry[1] & 1)) { + if (!data_tx_completion_span(cookie, count, entry[1], &first, &count)) { wifi->data_tx_bad++; break; } - first = (cookie + DATA_TX_SLOTS - count) % DATA_TX_SLOTS; for (i = 0; i < count; i++) if (!wifi->data_tx_busy[(first + i) % DATA_TX_SLOTS]) break; @@ -996,13 +1040,14 @@ static void append_tlv(u8 *payload, size_t *size, u16 tag, const void *data, u16 static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, struct cfg80211_connect_params *params) { - /* Stock 25G76 ACIConnectionAdapter::connectAp (0x1808e94cc) emits a - * 12-byte TLV 376. The newer SDK 27 adapter uses 14 bytes. Preserve the - * unspecified-link masks; all-zero is not the non-MLO default. + /* Current J714s native27 connectAp1809c85b8 emits14 bytes for TLV376. + * initMldGlobalConfiguration1809c8bac preserves the unspecified-link + * masks and adds default zero fields for this non-MLO client. J813 + * retains its measured12-byte firmware layout. */ - static const u8 mld_defaults[12] = { + static const u8 mld_defaults[14] = { 1, 0, 1, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 3, + 0xff, 0xff, 0xff, 3, 0, 0, }; /* Firmware connection-context defaults (aciw 0x511ee8). TLV 0xf913 * replaces all five values. Sending zeros arms Supp_AddTimer with a @@ -1035,7 +1080,7 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, ret = -EBUSY; goto out; } - if (!wifi->passphrase_size) { + if (!of_machine_is_compatible("apple,j714s") && !wifi->passphrase_size) { ret = -ENOKEY; goto out; } @@ -1088,13 +1133,19 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, memcpy(ssid + 3, params->ssid, params->ssid_len); append_tlv(payload, &size, 342, ssid, 3 + params->ssid_len); append_tlv(payload, &size, WLAN_EID_RSN, rsn_body, rsn_size); - /* The fixed field carries the PMK; TLV 313 carries the passphrase. */ - put_unaligned_le16(wifi->passphrase_size, key); - memcpy(key + 2, wifi->passphrase, wifi->passphrase_size); - append_tlv(payload, &size, 313, key, 2 + wifi->passphrase_size); - memzero_explicit(key, sizeof(key)); + /* Native27's raw-PMK path uses only the fixed key field above. + * TLV313 belongs exclusively to the separate passphrase key type. + * Retain the qualified older J813 path until separately tested. + */ + if (!of_machine_is_compatible("apple,j714s")) { + put_unaligned_le16(wifi->passphrase_size, key); + memcpy(key + 2, wifi->passphrase, wifi->passphrase_size); + append_tlv(payload, &size, 313, key, 2 + wifi->passphrase_size); + memzero_explicit(key, sizeof(key)); + } append_tlv(payload, &size, 316, &pmf, 1); - append_tlv(payload, &size, 376, mld_defaults, sizeof(mld_defaults)); + append_tlv(payload, &size, 376, mld_defaults, + of_machine_is_compatible("apple,j714s") ? sizeof(mld_defaults) : 12); wifi->connect_size = size; memcpy(wifi->request_ie, params->ie, params->ie_len); wifi->request_ie_size = params->ie_len; @@ -1301,11 +1352,19 @@ static const struct cfg80211_ops wifi_ops = { .set_power_mgmt = wifi_power_mgmt, }; +static int data_start(struct centauri_wifi *wifi); + static int net_open(struct net_device *netdev) { + struct centauri_wifi *wifi = *(struct centauri_wifi **)netdev_priv(netdev); + int ret; + netif_carrier_off(netdev); netif_stop_queue(netdev); - return 0; + mutex_lock(&wifi->lock); + ret = wifi->configured ? data_start(wifi) : -ENETDOWN; + mutex_unlock(&wifi->lock); + return ret; } static int net_stop(struct net_device *netdev) @@ -1566,21 +1625,20 @@ static const struct file_operations fwlog_buffer_ops = { /* The hardware data-ring callback requires the ten-byte WLAN clientdata * extension after the 44-byte ACIPC header. Without it UMAC asserts at - * 0x11760e. Keep these empty-ring probes explicit until packet DMA is proven. - * The reserved regions below do not overlap control rings or scan buffers. + * 0x11760e. Current native27 allocators retain this10-byte extension. + * These ring and buffer sizes use the native defaults, without reducing + * available descriptors for the normal interface path. */ -static ssize_t data_probe_write(struct file *file, const char __user *input, - size_t count, loff_t *position) +static int data_ring_open(struct centauri_wifi *wifi, unsigned int id) { - struct centauri_wifi *wifi = file->private_data; u8 message[0x34 + 10] = { 0 }; - unsigned int id, vector, offset, size, bit, doorbell, irq; + unsigned int vector, offset, size, bit, doorbell, irq; bool transfer = false; int ret; - ret = kstrtouint_from_user(input, count, 0, &id); - if (ret) - return ret; + lockdep_assert_held(&wifi->lock); + if (wifi->data_failed) + return -EIO; if (id == 76) { vector = 192; offset = 0x200000; @@ -1598,7 +1656,7 @@ static ssize_t data_probe_write(struct file *file, const char __user *input, } else if (id == 15) { vector = 207; offset = 0x240000; - size = 128; + size = DATA_RX_SLOTS; bit = 2; doorbell = 11; irq = 8; @@ -1606,24 +1664,23 @@ static ssize_t data_probe_write(struct file *file, const char __user *input, transfer = true; vector = id; offset = id == 12 ? 0x520000 : id == 24 ? 0x500000 : 0x540000; - size = 128; + size = id == 12 ? DATA_RX_SLOTS : DATA_TX_SLOTS; bit = id == 12 ? 4 : id == 24 ? 3 : 5; doorbell = id == 12 ? 11 : 18; irq = id == 12 ? 8 : 3; } else { return -EINVAL; } - mutex_lock(&wifi->lock); if (readl(wifi->a.bar + 8) != 2) { ret = -EIO; goto out; } - if (wifi->connecting || wifi->scan_request || - (wifi->data_probe_done & BIT(bit))) { + if (wifi->data_probe_done & BIT(bit)) + return 0; + if (wifi->connecting || wifi->scan_request) { ret = -EBUSY; goto out; } - wifi->data_probe_done |= BIT(bit); memset(wifi->a.window + offset, 0, size * 32); index_write(wifi, transfer ? 1 : 0, vector, 0); index_write(wifi, transfer ? 3 : 2, vector, 0); @@ -1657,19 +1714,26 @@ static ssize_t data_probe_write(struct file *file, const char __user *input, eth_broadcast_addr(message + 0x34 + 4); } ret = alpha_message(wifi, message, (transfer ? 0x34 : 0x2c) + 10); - msleep(1000); dev_info(&wifi->control->pdev->dev, - "data %s probe id=%u index=%u extension=10 rc=%d stage=%u head=%u tail=%u\n", + "data %s opened id=%u index=%u extension=10 rc=%d stage=%u head=%u tail=%u\n", transfer ? "TR" : "CR", id, vector, ret, readl(wifi->a.bar + 8), index_read(wifi, transfer ? 3 : 0, vector), index_read(wifi, transfer ? 1 : 2, vector)); if (!ret && readl(wifi->a.bar + 8) != 2) ret = -EIO; - if (!ret && transfer && id == 12) { + if (ret) { + /* A timed-out open may already have published the ring. Do not + * reinitialize its storage and race a late firmware completion. + */ + wifi->data_failed = true; + goto out; + } + wifi->data_probe_done |= BIT(bit); + if (transfer && id == 12) { unsigned int slot; - /* Give the receive engine bounded, individually identifiable buffers. - * No recycling or network delivery until the completion ABI is proven. + /* Cookies identify buffer ownership independently of ring position. + * Completion processing recycles the same buffer only after DMA. */ for (slot = 0; slot < size - 1; slot++) { u8 *descriptor = wifi->a.window + offset + slot * 16; @@ -1684,13 +1748,42 @@ static ssize_t data_probe_write(struct file *file, const char __user *input, index_write(wifi, 3, vector, size - 1); dma_wmb(); writel(1, wifi->a.bar + 0x1050); - msleep(100); dev_info(&wifi->control->pdev->dev, "data RX buffers posted=%u stage=%u tr_tail=%u cr_head=%u\n", size - 1, readl(wifi->a.bar + 8), index_read(wifi, 1, 12), index_read(wifi, 0, 207)); } out: + return ret; +} + +static int data_start(struct centauri_wifi *wifi) +{ + /* Create completion resources before their transfer rings. Control and + * scan rings are independent; monitor/EAPOL probes remain optional. + */ + static const unsigned int rings[] = { 76, 15, 12, 25 }; + int ret; + + for (unsigned int i = 0; i < ARRAY_SIZE(rings); i++) { + ret = data_ring_open(wifi, rings[i]); + if (ret) + return ret; + } + return 0; +} + +static ssize_t data_probe_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + unsigned int id; + int ret = kstrtouint_from_user(input, count, 0, &id); + + if (ret) + return ret; + mutex_lock(&wifi->lock); + ret = data_ring_open(wifi, id); mutex_unlock(&wifi->lock); return ret ?: count; } @@ -2272,6 +2365,7 @@ static const struct file_operations tx_power_report_ops = { int centauri_wifi_register(struct centauri *c) { struct centauri_wifi *wifi; + u8 address[ETH_ALEN]; unsigned int i; int ret; @@ -2327,6 +2421,18 @@ int centauri_wifi_register(struct centauri *c) wifi->bands[1].n_bitrates = ARRAY_SIZE(rates) - 4; wifi->wiphy->bands[NL80211_BAND_2GHZ] = &wifi->bands[0]; wifi->wiphy->bands[NL80211_BAND_5GHZ] = &wifi->bands[1]; + /* Native Alpha exposes its factory address as local-mac-address. */ + ret = eth_platform_get_mac_address(&wifi->a.pdev->dev, address); + if (!ret) { + eth_hw_addr_set(wifi->netdev, address); + ether_addr_copy(wifi->wiphy->perm_addr, address); + } else if (ret == -EPROBE_DEFER) { + goto failed; + } else { + eth_hw_addr_random(wifi->netdev); + dev_warn(&wifi->a.pdev->dev, + "No valid factory MAC property; using a random address\n"); + } ret = wiphy_register(wifi->wiphy); if (ret) goto failed; @@ -2339,7 +2445,6 @@ int centauri_wifi_register(struct centauri *c) SET_NETDEV_DEV(wifi->netdev, &c->pdev->dev); strscpy(wifi->netdev->name, "wlan%d", IFNAMSIZ); wifi->netdev->max_mtu = ETH_DATA_LEN; - eth_hw_addr_random(wifi->netdev); ret = register_netdev(wifi->netdev); if (ret) goto failed; @@ -2348,7 +2453,7 @@ int centauri_wifi_register(struct centauri *c) dev_err(&c->pdev->dev, "Wi-Fi ring/interface setup failed: %d\n", ret); goto failed; } - dev_info(&c->pdev->dev, "cfg80211 interface %s ready for scan tests; packet data unavailable\n", + dev_info(&c->pdev->dev, "cfg80211 interface %s ready; data rings initialize on interface open\n", wifi->netdev->name); smp_store_release(&wifi->irq_ready, true); schedule_work(&wifi->link_work); From 4845fe4c0845286a0671854fae3bee7c273b487d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 23:48:26 -0400 Subject: [PATCH 0163/1579] wifi: apple: automate J714 startup and harden link lifecycle --- Documentation/arch/arm64/t6050-sptm-n1.rst | 29 +++ drivers/bluetooth/btcentauri.c | 61 +++-- .../net/wireless/apple/centauri/centauri.h | 14 ++ .../wireless/apple/centauri/centauri_ipc.c | 28 ++- .../wireless/apple/centauri/centauri_pci.c | 148 +++++++++-- .../wireless/apple/centauri/centauri_wifi.c | 235 ++++++++++++++++-- drivers/pci/controller/apple-x1n1/t6050.c | 4 +- drivers/pci/probe.c | 9 + include/linux/pci-apple-x1n1.h | 1 + include/linux/pci.h | 1 + 10 files changed, 457 insertions(+), 73 deletions(-) diff --git a/Documentation/arch/arm64/t6050-sptm-n1.rst b/Documentation/arch/arm64/t6050-sptm-n1.rst index 2792d82f8e2dc..151b54877b5af 100644 --- a/Documentation/arch/arm64/t6050-sptm-n1.rst +++ b/Documentation/arch/arm64/t6050-sptm-n1.rst @@ -192,3 +192,32 @@ in the network path. Factory-address DT artifact: ``t6050-n1-factory-mac.dtb``. Firmware, credentials and calibration remain private. Automatic firmware bootstrap and Bluetooth radio bring-up are separate remaining work; Bluetooth's default radio-power command still fails. + +Automatic startup and link lifecycle +------------------------------------ + +The J714 driver now schedules native ROM, port-cycle and IPC startup automatically. +PCI rescan-lock contention is retried before mutation; actual firmware failures are +reported without repeating a partially completed phase. Removal cancels the work +before releasing transport state. The netdev is published after radio setup. + +Hardware on firmware 27 booted with no debugfs boot writes, associated using the +standard WPA2 PMK path, obtained DHCP and accepted SSH. A connected scan completed +with 20/20 gateway pings. Three additional cycles each aborted a scan, performed a +full scan, disconnected and reconnected, then verified gateway traffic. A final +netdev down/up recovered association, DHCP, gateway traffic and SSH. The test must +wait for the DHCP route after association; association completion alone does not +mean the IP configuration is ready. TX/RX completion counters showed no malformed +or failed packets in these runs. Suspend/resume and firmware-crash recovery remain +unqualified. + +Station reporting exposes per-association traffic counters and native RSSI from +firmware command 20 / TLV 268. It queries on demand with a one-second cache; no +periodic polling is introduced. Firmware rate fields have not been decoded, so a +PHY rate is not invented. + +J714 Bluetooth startup uses the captured factory address and the native firmware +initialization sequence automatically. Explicit native_startup=0 remains available +for diagnostics; the J813 default is unchanged. Hardware accepted radio power +cycling and LE discovery, producing 29 device reports. This does not qualify ACL, +GATT, L2CAP, audio, suspend or recovery, and ACL remains opt-in. diff --git a/drivers/bluetooth/btcentauri.c b/drivers/bluetooth/btcentauri.c index 95dd96f2ecad3..2db73da175863 100644 --- a/drivers/bluetooth/btcentauri.c +++ b/drivers/bluetooth/btcentauri.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -54,9 +55,15 @@ static bool arm_auxiliary_rings; module_param(arm_auxiliary_rings, bool, 0444); MODULE_PARM_DESC(arm_auxiliary_rings, "Diagnostic: arm remaining native Beta resources and capture received frames privately"); -static bool native_startup; -module_param(native_startup, bool, 0444); -MODULE_PARM_DESC(native_startup, "Diagnostic: program a temporary address, InitDone and random seed before the first HCI reset"); +static int native_startup = -1; +module_param(native_startup, int, 0444); +MODULE_PARM_DESC(native_startup, "Native firmware initialization: -1=board default, 0=off, 1=on"); + +static bool cen_native_startup_enabled(void) +{ + return native_startup < 0 ? of_machine_is_compatible("apple,j714s") : + native_startup; +} static char *native_bdaddr; module_param(native_bdaddr, charp, 0400); @@ -658,27 +665,38 @@ static int cen_hci_setup(struct hci_dev *hdev) { struct cen_bt *bt = hci_get_drvdata(hdev); u8 address[6], init_done[16], seed[24]; + bool factory = of_machine_is_compatible("apple,j714s"); int ret; - if (!native_startup) + if (!cen_native_startup_enabled()) return 0; - /* Native PROXIMA initialization starts with these three commands, then - * HCI Reset. The core calls setup before its ordinary reset/init sequence. - * Address provisioning is still diagnostic. A saved test identity permits - * bond reuse; otherwise generate a temporary locally administered address. - * Native WriteBdAddr takes display/network byte order in either case. + /* J714s factory initialization and repeated radio power/discovery are + * qualified with firmware27. Other boards retain their diagnostic gate. + * J714s uses its native DT address in display/network byte order, + * which is the vendor WriteBdAddr wire order. HCI Read BD_ADDR later + * returns the standard little-endian form to the Bluetooth core. */ - if (native_bdaddr && (!mac_pton(native_bdaddr, address) || - is_zero_ether_addr(address) || is_broadcast_ether_addr(address))) - return -EINVAL; - ret = get_random_bytes_wait(seed, sizeof(seed)); - if (ret) - return ret; - if (!native_bdaddr) { + if (factory) { + if (device_property_count_u8(&bt->pdev->dev, "local-mac-address") != 6) + return -EINVAL; + ret = device_property_read_u8_array(&bt->pdev->dev, "local-mac-address", + address, sizeof(address)); + if (ret) + return ret; + if (is_zero_ether_addr(address) || is_broadcast_ether_addr(address)) + return -EINVAL; + } else if (native_bdaddr) { + if (!mac_pton(native_bdaddr, address) || is_zero_ether_addr(address) || + is_broadcast_ether_addr(address)) + return -EINVAL; + } else { get_random_bytes(address, sizeof(address)); address[0] = (address[0] & 0xfc) | 0x02; } + ret = get_random_bytes_wait(seed, sizeof(seed)); + if (ret) + return ret; memset(init_done, 0xff, sizeof(init_done)); ret = cen_vendor_command(hdev, 0xfc06, address, sizeof(address)); if (ret) @@ -697,8 +715,9 @@ static int cen_hci_setup(struct hci_dev *hdev) bt_dev_err(hdev, "Native startup stopped after %u commands: %d", bt->native_startup_steps, ret); else - bt_dev_info(hdev, "Native startup accepted; %s diagnostic address in use", - native_bdaddr ? "configured" : "temporary"); + bt_dev_info(hdev, "Native startup accepted; %s address in use", + factory ? "factory" : native_bdaddr ? "configured diagnostic" : + "temporary diagnostic"); return ret; } @@ -856,7 +875,7 @@ static int cen_status_show(struct seq_file *s, void *unused) bt->hdev->stat.err_tx, bt->hdev->stat.err_rx); seq_printf(s, "le_rings=%u le_rx_packets=%u\n", arm_le_data_rings, bt->le_rx_packets); seq_printf(s, "native_startup=%u native_startup_steps=%u\n", - native_startup, READ_ONCE(bt->native_startup_steps)); + cen_native_startup_enabled(), READ_ONCE(bt->native_startup_steps)); seq_printf(s, "native_hrb=%u hrb_supported=%u proprietary_features=0x%08x hrb_commands=%u hrb_status_events=%u hrb_pending=%u\n", native_hrb, READ_ONCE(bt->hrb_supported), READ_ONCE(bt->proprietary_features), bt->hrb_commands, @@ -941,7 +960,9 @@ static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) skb_queue_head_init(&bt->le_txq); if (enable_acl && (!arm_le_data_rings || !arm_auxiliary_rings)) return -EINVAL; - if (native_hrb && (!native_startup || !enable_acl)) + if (native_startup < -1 || native_startup > 1) + return -EINVAL; + if (native_hrb && (!cen_native_startup_enabled() || !enable_acl)) return -EINVAL; INIT_WORK(&bt->poll, cen_poll); ret = pcim_enable_device(pdev); diff --git a/drivers/net/wireless/apple/centauri/centauri.h b/drivers/net/wireless/apple/centauri/centauri.h index 67a5d11ff8187..a350357ed4283 100644 --- a/drivers/net/wireless/apple/centauri/centauri.h +++ b/drivers/net/wireless/apple/centauri/centauri.h @@ -6,6 +6,7 @@ #include #include #include +#include #define CEN_ALPHA_WINDOW_SIZE SZ_32M #define CEN_ALPHA_SCRATCH_OFFSET SZ_8M @@ -26,6 +27,15 @@ struct cen_alpha_resources { void __iomem *bar; u32 index[4], mtr, mcr; }; +enum centauri_boot_phase { + CEN_BOOT_ROM, + CEN_BOOT_CYCLE, + CEN_BOOT_IPC, + CEN_BOOT_READY, + CEN_BOOT_FAILED, + CEN_BOOT_STOPPED, +}; + struct centauri { struct pci_dev *pdev; void __iomem *bar; @@ -33,6 +43,10 @@ struct centauri { u32 stage_offset; struct dentry *debug; struct mutex boot_lock; + struct delayed_work boot_work; + enum centauri_boot_phase boot_phase; + int boot_error; + bool automatic_boot, removing; void *firmware, *descriptor; struct centauri_irqs irqs, alpha_irqs; atomic_t irq_count; diff --git a/drivers/net/wireless/apple/centauri/centauri_ipc.c b/drivers/net/wireless/apple/centauri/centauri_ipc.c index 483a869012472..e03b87dcda9ae 100644 --- a/drivers/net/wireless/apple/centauri/centauri_ipc.c +++ b/drivers/net/wireless/apple/centauri/centauri_ipc.c @@ -345,11 +345,10 @@ static int enumerate_functions(struct centauri *c) struct centauri_ipc *ipc = c->ipc; int i, ret; - pci_lock_rescan_remove(); - /* The ROM personality was single-function; boot exposes Alpha and Beta. */ + /* Caller holds PCI rescan/remove ownership. The ROM personality was + * single-function; boot exposes Alpha and Beta. */ c->pdev->multifunction = 1; pci_rescan_bus(c->pdev->bus); - pci_unlock_rescan_remove(); for (i = 0; i < 3; i++) { ipc->function[i] = pci_get_domain_bus_and_slot(pci_domain_nr(c->pdev->bus), c->pdev->bus->number, PCI_DEVFN(0, i)); @@ -385,7 +384,7 @@ static int enumerate_functions(struct centauri *c) return 0; } -int centauri_start_ipc(struct centauri *c) +static int centauri_start_ipc_locked(struct centauri *c) { struct centauri_ipc *ipc; u32 stage; @@ -478,6 +477,20 @@ int centauri_start_ipc(struct centauri *c) return ret; } +int centauri_start_ipc(struct centauri *c) +{ + int ret; + + /* remove may hold this lock while cancelling the firmware worker. + * Return before allocating/publishing anything when it is unavailable. + */ + if (!pci_trylock_rescan_remove()) + return -EAGAIN; + ret = READ_ONCE(c->removing) ? -ECANCELED : centauri_start_ipc_locked(c); + pci_unlock_rescan_remove(); + return ret; +} + void centauri_free_ipc(struct centauri *c) { struct centauri_ipc *ipc = c->ipc; @@ -495,6 +508,13 @@ void centauri_free_ipc(struct centauri *c) continue; pci_clear_master(ipc->function[i]); } + for (i = 0; i < 3; i++) { + if (ipc->function[i] && !pci_wait_for_pending_transaction(ipc->function[i])) { + dev_warn(&c->pdev->dev, + "retaining IPC DMA after pending transaction timeout\n"); + return; + } + } list_for_each_entry_safe(buffer, next, &ipc->buffers, list) { dma_free_coherent(buffer->owner, buffer->size, buffer->cpu, buffer->dma); put_device(buffer->owner); diff --git a/drivers/net/wireless/apple/centauri/centauri_pci.c b/drivers/net/wireless/apple/centauri/centauri_pci.c index 7401ab1b79bfc..10b85ec7d519c 100644 --- a/drivers/net/wireless/apple/centauri/centauri_pci.c +++ b/drivers/net/wireless/apple/centauri/centauri_pci.c @@ -4,6 +4,7 @@ * J813 prototype. Firmware execution and radio operations are separate gates. */ #include +#include #include #include #include @@ -21,6 +22,10 @@ static struct dentry *centauri_debug; +static bool auto_boot = true; +module_param(auto_boot, bool, 0444); +MODULE_PARM_DESC(auto_boot, "Boot J714s firmware automatically (disable only for manual diagnostics)"); + /* Fixed counts follow the native Control/Alpha/Beta resource maps. PCI's MSI * allocator owns the message addresses and data; never program a DMA mailbox. */ @@ -81,6 +86,8 @@ static int centauri_status_show(struct seq_file *s, void *unused) seq_printf(s, "transport=hardware-pci firmware_boot=%s image_response=%#x wifi=%s\n", c->boot_attempted ? "attempted" : "not-started", c->image_response, READ_ONCE(c->wifi) ? "registered" : "not-owned-by-this-function"); + seq_printf(s, "automatic_boot=%u boot_phase=%u boot_error=%d\n", + c->automatic_boot, READ_ONCE(c->boot_phase), READ_ONCE(c->boot_error)); return 0; } DEFINE_SHOW_ATTRIBUTE(centauri_status); @@ -103,6 +110,10 @@ static void centauri_free_boot(struct centauri *c) centauri_free_ipc(c); pci_clear_master(c->pdev); centauri_free_irqs(&c->irqs); + if (c->ipc || !pci_wait_for_pending_transaction(c->pdev)) { + dev_warn(dev, "retaining boot DMA until outstanding transactions stop\n"); + return; + } if (c->firmware) dma_free_coherent(dev, c->firmware_size, c->firmware, c->firmware_dma); if (c->descriptor) @@ -149,9 +160,18 @@ static int centauri_rom_boot(struct centauri *c) stage = readl(c->bar + 0x8000); if (stage != 0 && stage != 4 && stage != 1) return -EBUSY; - ret = request_firmware(&fw, profile->firmware, dev); - if (ret) + /* The initramfs/root must provide the personalized FTAB. Never wait for + * a userspace fallback while remove is cancelling this worker. + */ + ret = request_firmware_direct(&fw, profile->firmware, dev); + if (ret) { + dev_err(dev, "required firmware %s unavailable: %d\n", profile->firmware, ret); return ret; + } + if (READ_ONCE(c->removing)) { + ret = -ECANCELED; + goto release; + } if (fw->size < 0x30 || fw->size > SZ_64M || memcmp(fw->data + 0x20, "rkosftab", 8)) { ret = -EINVAL; @@ -190,13 +210,20 @@ static int centauri_rom_boot(struct centauri *c) } writel(1, c->bar + 0x9000); ret = readl_poll_timeout(c->bar + 0x8040, c->image_response, - c->image_response != 0, 20000, 20000000); + READ_ONCE(c->removing) || c->image_response != 0, 20000, 20000000); + if (READ_ONCE(c->removing)) { + ret = -ECANCELED; + goto release; + } if (!ret && c->image_response != 1) ret = -EIO; if (!ret) - ret = readl_poll_timeout(c->bar + 0x8000, stage, stage != 0, 20000, 20000000); + ret = readl_poll_timeout(c->bar + 0x8000, stage, + READ_ONCE(c->removing) || stage != 0, 20000, 20000000); else stage = readl(c->bar + 0x8000); + if (READ_ONCE(c->removing)) + ret = -ECANCELED; dma_rmb(); dev_info(dev, "ROM firmware result: response=%#x stage=%#x irqs=%d poll=%d\n", c->image_response, stage, atomic_read(&c->irq_count), ret); @@ -211,6 +238,79 @@ static int centauri_rom_boot(struct centauri *c) return ret; } +static int centauri_cycle(struct centauri *c) +{ + int ret; + + if (!c->firmware || !c->link_ready || readl(c->bar + 0x8000) != 4) + return -EBUSY; + c->link_ready = false; + ret = of_machine_is_compatible("apple,j714s") ? + apple_x1n1_pcie_cycle(c->pdev) : centauri_j813_port_cycle(c->pdev); + if (ret == -EAGAIN) { + /* Topology contention returns before any endpoint mutation. */ + c->link_ready = true; + } else if (!ret) { + pci_restore_msi_state(c->pdev); + c->link_ready = true; + dev_info(&c->pdev->dev, "post-reset execution stage=%#x\n", + readl(c->bar + 0x8000)); + } + return ret; +} + +static void centauri_boot_worker(struct work_struct *work) +{ + struct centauri *c = container_of(to_delayed_work(work), struct centauri, boot_work); + bool retry = false; + int ret; + + mutex_lock(&c->boot_lock); + while (!READ_ONCE(c->removing)) { + switch (c->boot_phase) { + case CEN_BOOT_ROM: + ret = centauri_rom_boot(c); + if (!ret) + c->boot_phase = readl(c->bar + 0x8000) == 1 ? + CEN_BOOT_IPC : CEN_BOOT_CYCLE; + break; + case CEN_BOOT_CYCLE: + ret = centauri_cycle(c); + if (!ret) + c->boot_phase = CEN_BOOT_IPC; + break; + case CEN_BOOT_IPC: + ret = centauri_start_ipc(c); + if (!ret) { + c->boot_phase = CEN_BOOT_READY; + dev_info(&c->pdev->dev, "N1 automatic firmware startup complete\n"); + } + break; + default: + goto out; + } + if (ret == -EAGAIN && (c->boot_phase == CEN_BOOT_CYCLE || + (c->boot_phase == CEN_BOOT_IPC && !c->ipc))) { + /* Only lock contention is retryable. No firmware phase repeats. */ + retry = true; + break; + } + if (ret) { + c->boot_error = ret; + dev_err(&c->pdev->dev, "N1 automatic startup failed in phase%u: %d\n", + c->boot_phase, ret); + c->boot_phase = CEN_BOOT_FAILED; + break; + } + } +out: + if (READ_ONCE(c->removing)) + c->boot_phase = CEN_BOOT_STOPPED; + mutex_unlock(&c->boot_lock); + if (retry && !READ_ONCE(c->removing)) + queue_delayed_work(system_long_wq, &c->boot_work, msecs_to_jiffies(20)); +} + static ssize_t centauri_boot_write(struct file *file, const char __user *buffer, size_t count, loff_t *pos) { @@ -221,24 +321,16 @@ static ssize_t centauri_boot_write(struct file *file, const char __user *buffer, if (ret || value < 1 || value > 3) return ret ?: -EINVAL; mutex_lock(&c->boot_lock); - if (value == 1) { + if (READ_ONCE(c->removing)) + ret = -ENODEV; + else if (c->automatic_boot) + ret = -EBUSY; + else if (value == 1) ret = centauri_rom_boot(c); - } else if (value == 3) { + else if (value == 3) ret = centauri_start_ipc(c); - } else if (!c->firmware || !c->link_ready || readl(c->bar + 0x8000) != 4) { - ret = -EBUSY; - } else { - c->link_ready = false; - ret = of_machine_is_compatible("apple,j714s") ? - apple_x1n1_pcie_cycle(c->pdev) : - centauri_j813_port_cycle(c->pdev); - if (!ret) { - pci_restore_msi_state(c->pdev); - c->link_ready = true; - dev_info(&c->pdev->dev, "post-reset execution stage=%#x\n", - readl(c->bar + 0x8000)); - } - } + else + ret = centauri_cycle(c); mutex_unlock(&c->boot_lock); return ret ?: count; } @@ -265,6 +357,9 @@ static int centauri_probe(struct pci_dev *pdev, const struct pci_device_id *id) atomic_set(&c->irq_count, 0); c->link_ready = true; mutex_init(&c->boot_lock); + INIT_DELAYED_WORK(&c->boot_work, centauri_boot_worker); + c->boot_phase = CEN_BOOT_ROM; + c->automatic_boot = auto_boot && of_machine_is_compatible("apple,j714s"); c->stage_offset = id->driver_data; ret = pcim_enable_device(pdev); if (ret) @@ -294,6 +389,8 @@ static int centauri_probe(struct pci_dev *pdev, const struct pci_device_id *id) c->debug = debugfs_create_dir(pci_name(pdev), centauri_debug); debugfs_create_file("status", 0400, c->debug, c, ¢auri_status_fops); debugfs_create_file("rom_boot", 0200, c->debug, c, ¢auri_boot_fops); + if (c->automatic_boot) + queue_delayed_work(system_long_wq, &c->boot_work, 0); return 0; } @@ -301,9 +398,17 @@ static void centauri_remove(struct pci_dev *pdev) { struct centauri *c = pci_get_drvdata(pdev); - centauri_wifi_unregister(c); + /* PCI remove can already own the global rescan lock. The worker uses + * only trylock on that path and observes this before requeueing. + */ + WRITE_ONCE(c->removing, true); debugfs_remove_recursive(c->debug); + cancel_delayed_work_sync(&c->boot_work); + mutex_lock(&c->boot_lock); + centauri_wifi_unregister(c); centauri_free_boot(c); + c->boot_phase = CEN_BOOT_STOPPED; + mutex_unlock(&c->boot_lock); } static const struct pci_device_id centauri_ids[] = { @@ -318,6 +423,7 @@ static struct pci_driver centauri_driver = { .id_table = centauri_ids, .probe = centauri_probe, .remove = centauri_remove, + .shutdown = centauri_remove, }; static int __init centauri_init(void) diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index 7fef85ee89912..4312616bed617 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -108,6 +108,7 @@ struct centauri_wifi { u32 reply_command, channels_seen, scans_done, beacons_seen; bool reply_valid, registered, configured; bool stopping; + bool scan_abort, connect_abort; u8 scan_sequence; bool connecting, connected, connect_event, disconnect_event; bool associated, pmk_seen, pmk_matches; @@ -143,6 +144,10 @@ struct centauri_wifi { bool tx_profile_valid; u8 link_count, link_prime[3], link_channel[3], link_band[3], emlsr_mode; bool link_status_valid, emlsr_valid; + s8 station_signal; + bool station_signal_valid; + unsigned long station_signal_at, connected_at; + struct net_device_stats station_start; u16 link_status_size; u16 link_tags[12], link_lengths[12]; u8 link_tlv_count; @@ -676,6 +681,14 @@ static void drain(struct centauri_wifi *wifi) wifi->link_tags[wifi->link_tlv_count] = tag; wifi->link_lengths[wifi->link_tlv_count++] = length; } + /* Native27 getRssi's callback reads signed byte1 from + * the115-byte interface-info TLV268 (180a1ba0c). + */ + if (tag == 268 && length >= 115) { + wifi->station_signal = (s8)value[1]; + wifi->station_signal_valid = wifi->station_signal != 0; + wifi->station_signal_at = jiffies; + } /* ACIDebugAdapter::dumpMloStatus, TLVs 377/378. * Keep only link count/channel/role. Do not retain MACs. */ @@ -867,7 +880,10 @@ static int scan_native27(struct centauri_wifi *wifi) size_t size = 14; unsigned long deadline; int ret; + bool abort_sent = false; + if (READ_ONCE(wifi->scan_abort)) + return -ECANCELED; memset(scan, 0, sizeof(scan)); scan[0] = (++wifi->scan_sequence) & 0x1f; scan[3] = 0x80; @@ -930,12 +946,20 @@ static int scan_native27(struct centauri_wifi *wifi) return -EIO; if (wifi->scans_done != done) break; + if (READ_ONCE(wifi->scan_abort) && !abort_sent) { + ret = command(wifi, 65537, NULL, 0); + if (ret) + return ret; + abort_sent = true; + } usleep_range(1000, 2000); } while (time_before(jiffies, deadline)); if (wifi->scans_done == done) { command(wifi, 65537, NULL, 0); return -ETIMEDOUT; } + if (READ_ONCE(wifi->scan_abort)) + return -ECANCELED; } return 0; } @@ -950,6 +974,10 @@ static void scan_worker(struct work_struct *work) int ret = 0; mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->scan_abort) || READ_ONCE(wifi->stopping)) { + ret = -ECANCELED; + goto completed; + } if (of_machine_is_compatible("apple,j714s")) { ret = scan_native27(wifi); goto completed; @@ -970,7 +998,8 @@ static void scan_worker(struct work_struct *work) ret = -EIO; break; } - if (!stopped && wifi->channels_seen - channels >= 8) { + if (!stopped && (wifi->channels_seen - channels >= 8 || + READ_ONCE(wifi->scan_abort))) { stopped = true; ret = command(wifi, 65537, NULL, 0); if (ret) @@ -982,9 +1011,11 @@ static void scan_worker(struct work_struct *work) } while (time_before(jiffies, deadline)); if (!ret && wifi->scans_done == done) ret = -ETIMEDOUT; + if (READ_ONCE(wifi->scan_abort)) + ret = -ECANCELED; } completed: - info.aborted = !!ret; + info.aborted = !!ret || READ_ONCE(wifi->scan_abort) || READ_ONCE(wifi->stopping); request = wifi->scan_request; wifi->scan_request = NULL; mutex_unlock(&wifi->lock); @@ -1000,11 +1031,14 @@ static int wifi_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) int ret = 0; mutex_lock(&wifi->lock); - if (!wifi->configured) + if (READ_ONCE(wifi->stopping)) + ret = -ENODEV; + else if (!wifi->configured) ret = -ENETDOWN; else if (wifi->scan_request || wifi->connecting) ret = -EBUSY; else { + WRITE_ONCE(wifi->scan_abort, false); wifi->scan_request = request; schedule_work(&wifi->scan_work); } @@ -1012,6 +1046,49 @@ static int wifi_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) return ret; } +/* cfg80211 owns the scan request until its one scan_done callback. The + * worker observes cancellation without requiring its command mutex. + */ +static void wifi_abort_scan(struct wiphy *wiphy, struct wireless_dev *wdev) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + + if (wdev == &wifi->wdev) + WRITE_ONCE(wifi->scan_abort, true); +} + +static void cancel_scan(struct centauri_wifi *wifi) +{ + struct cfg80211_scan_info info = { .aborted = true }; + struct cfg80211_scan_request *request; + + WRITE_ONCE(wifi->scan_abort, true); + cancel_work_sync(&wifi->scan_work); + mutex_lock(&wifi->lock); + request = wifi->scan_request; + wifi->scan_request = NULL; + mutex_unlock(&wifi->lock); + /* A queued worker can be cancelled before it owns this completion. */ + if (request) + cfg80211_scan_done(request, &info); +} + +/* Only a still-pending attempt owns a terminal connection event. This also + * covers cancellation before its worker starts and touches firmware. + */ +static void cancel_connect_locked(struct centauri_wifi *wifi) +{ + if (!wifi->connecting) + return; + wifi->connecting = false; + wifi->connected = false; + memzero_explicit(wifi->connect_payload, sizeof(wifi->connect_payload)); + memzero_explicit(wifi->expected_pmk, sizeof(wifi->expected_pmk)); + cfg80211_connect_timeout(wifi->netdev, wifi->bssid, + wifi->request_ie, wifi->request_ie_size, GFP_KERNEL, + NL80211_TIMEOUT_UNSPECIFIED); +} + static int rsn_caps_offset(const u8 *body, size_t size) { size_t offset = 6; @@ -1076,6 +1153,10 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, if (!rsn || rsn_caps_offset(rsn + 2, rsn[1]) < 0) return -EINVAL; mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->stopping)) { + ret = -ENODEV; + goto out; + } if (!wifi->configured || wifi->connecting || wifi->connected || wifi->scan_request) { ret = -EBUSY; goto out; @@ -1149,6 +1230,7 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, wifi->connect_size = size; memcpy(wifi->request_ie, params->ie, params->ie_len); wifi->request_ie_size = params->ie_len; + WRITE_ONCE(wifi->connect_abort, false); wifi->connecting = true; wifi->connect_status = 0; wifi->associated = wifi->pmk_seen = wifi->pmk_matches = false; @@ -1169,12 +1251,18 @@ static void connect_worker(struct work_struct *work) int ret; mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->connect_abort) || READ_ONCE(wifi->stopping)) { + cancel_connect_locked(wifi); + mutex_unlock(&wifi->lock); + return; + } slot = wifi->command_head; ret = command(wifi, 131072, wifi->connect_payload, wifi->connect_size); command_status = wifi->reply_status; memzero_explicit(wifi->connect_payload, sizeof(wifi->connect_payload)); deadline = jiffies + msecs_to_jiffies(15000); - while (!ret && !wifi->connect_event && time_before(jiffies, deadline)) { + while (!ret && !wifi->connect_event && !READ_ONCE(wifi->connect_abort) && + !READ_ONCE(wifi->stopping) && time_before(jiffies, deadline)) { drain(wifi); if (link_diagnostics && wifi->associated && !queried_link) { unsigned int i; @@ -1221,31 +1309,49 @@ static void connect_worker(struct work_struct *work) "Wi-Fi firmware key check: rc=%d available=%d matches=%d\n", key_ret, wifi->pmk_seen, wifi->pmk_matches); } + if (wifi->disconnect_event) + ret = -ENOTCONN; if (readl(wifi->a.bar + 8) != 2) ret = -EIO; usleep_range(1000, 2000); } - if (!ret && !wifi->connect_event) + if (READ_ONCE(wifi->connect_abort) || READ_ONCE(wifi->stopping)) + ret = -ECANCELED; + else if (!ret && !wifi->connect_event) ret = -ETIMEDOUT; status = !ret && wifi->connect_status == 0 ? WLAN_STATUS_SUCCESS : WLAN_STATUS_UNSPECIFIED_FAILURE; wifi->connecting = false; wifi->connected = status == WLAN_STATUS_SUCCESS; + wifi->station_signal_valid = false; + if (wifi->connected) { + wifi->connected_at = jiffies; + wifi->station_signal_at = jiffies - HZ; + wifi->station_start = wifi->netdev->stats; + } memzero_explicit(wifi->expected_pmk, sizeof(wifi->expected_pmk)); /* The command and connection completion have consumed the PMK buffer. */ if (slot < ring_size[1]) memzero_explicit(wifi->a.window + TX_BUFFER + slot * BUFFER_SIZE, BUFFER_SIZE); - mutex_unlock(&wifi->lock); dev_info(&wifi->control->pdev->dev, "Wi-Fi authentication result: rc=%d command_status=%u event_status=%u complete_event=%d\n", ret, command_status, wifi->connect_status, wifi->connect_event); - cfg80211_connect_result(wifi->netdev, wifi->bssid, - wifi->request_ie, wifi->request_ie_size, NULL, 0, status, GFP_KERNEL); + /* These cfg80211 APIs queue events without acquiring the wiphy mutex. + * Serialize notification and carrier state with local/remote disconnect. + */ + if (ret == -ECANCELED || ret == -ETIMEDOUT) + cfg80211_connect_timeout(wifi->netdev, wifi->bssid, + wifi->request_ie, wifi->request_ie_size, GFP_KERNEL, + NL80211_TIMEOUT_UNSPECIFIED); + else + cfg80211_connect_result(wifi->netdev, wifi->bssid, + wifi->request_ie, wifi->request_ie_size, NULL, 0, status, GFP_KERNEL); if (status == WLAN_STATUS_SUCCESS && !READ_ONCE(wifi->stopping)) { netif_carrier_on(wifi->netdev); if (wifi->data_probe_done & BIT(5)) netif_wake_queue(wifi->netdev); schedule_work(&wifi->link_work); } + mutex_unlock(&wifi->lock); } irqreturn_t centauri_alpha_irq(int irq, void *data) @@ -1274,13 +1380,13 @@ static void link_worker(struct work_struct *work) readl(wifi->a.bar + 8) != 2; if (disconnected) wifi->connected = false; - mutex_unlock(&wifi->lock); if (disconnected && was_connected) { netif_stop_queue(wifi->netdev); skb_queue_purge(&wifi->data_tx_queue); netif_carrier_off(wifi->netdev); cfg80211_disconnected(wifi->netdev, WLAN_REASON_UNSPECIFIED, NULL, 0, false, GFP_KERNEL); } + mutex_unlock(&wifi->lock); } static int wifi_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 reason) @@ -1288,27 +1394,33 @@ static int wifi_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 r struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); u8 payload[16] = {}; bool connected; + u16 reply_status; int ret; + /* The running worker holds wifi->lock while waiting for firmware. + * Signal it first, then join it without either driver mutex held. + */ + WRITE_ONCE(wifi->connect_abort, true); + cancel_scan(wifi); + cancel_work_sync(&wifi->connect_work); mutex_lock(&wifi->lock); - if (wifi->connecting) { - mutex_unlock(&wifi->lock); - return -EBUSY; - } + cancel_connect_locked(wifi); put_unaligned_le16(reason, payload); put_unaligned_le16(328, payload + 8); put_unaligned_le16(4, payload + 10); - ret = command(wifi, 131073, payload, sizeof(payload)); + ret = READ_ONCE(wifi->stopping) ? 0 : + command(wifi, 131073, payload, sizeof(payload)); + reply_status = wifi->reply_status; connected = wifi->connected; wifi->connected = false; - mutex_unlock(&wifi->lock); - cancel_work_sync(&wifi->link_work); netif_stop_queue(netdev); skb_queue_purge(&wifi->data_tx_queue); netif_carrier_off(netdev); if (connected) cfg80211_disconnected(netdev, reason, NULL, 0, true, GFP_KERNEL); - return ret == -EREMOTEIO && wifi->reply_status == 326 ? 0 : ret; + mutex_unlock(&wifi->lock); + cancel_work_sync(&wifi->link_work); + return ret == -EREMOTEIO && reply_status == 326 ? 0 : ret; } static int wifi_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, @@ -1345,11 +1457,74 @@ static void wifi_regulatory(struct wiphy *wiphy, struct regulatory_request *requ mutex_unlock(&wifi->lock); } +/* On-demand status only: no periodic firmware query or traffic polling. + * Report host-accounted Ethernet counters separately from unproved PHY rates. + * Caller holds wifi->lock and has checked the currently associated peer. + */ +static void fill_station(struct centauri_wifi *wifi, struct station_info *sinfo) +{ + const struct net_device_stats *stats = &wifi->netdev->stats; + + if (time_after_eq(jiffies, wifi->station_signal_at + HZ)) { + wifi->station_signal_valid = false; + wifi->station_signal_at = jiffies; + query_link_status(wifi); + } + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_CONNECTED_TIME) | + BIT_ULL(NL80211_STA_INFO_RX_BYTES64) | BIT_ULL(NL80211_STA_INFO_TX_BYTES64) | + BIT_ULL(NL80211_STA_INFO_RX_PACKETS) | BIT_ULL(NL80211_STA_INFO_TX_PACKETS); + sinfo->connected_time = (jiffies - wifi->connected_at) / HZ; + sinfo->rx_bytes = stats->rx_bytes - wifi->station_start.rx_bytes; + sinfo->tx_bytes = stats->tx_bytes - wifi->station_start.tx_bytes; + sinfo->rx_packets = stats->rx_packets - wifi->station_start.rx_packets; + sinfo->tx_packets = stats->tx_packets - wifi->station_start.tx_packets; + if (wifi->station_signal_valid) { + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL); + sinfo->signal = wifi->station_signal; + } +} + +static int wifi_get_station(struct wiphy *wiphy, struct wireless_dev *wdev, + const u8 *mac, struct station_info *sinfo) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + int ret = 0; + + mutex_lock(&wifi->lock); + if (wdev != &wifi->wdev || !wifi->connected || READ_ONCE(wifi->stopping) || + !ether_addr_equal(mac, wifi->bssid)) + ret = -ENOENT; + else + fill_station(wifi, sinfo); + mutex_unlock(&wifi->lock); + return ret; +} + +static int wifi_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev, + int index, u8 *mac, struct station_info *sinfo) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + int ret = 0; + + mutex_lock(&wifi->lock); + if (index || wdev != &wifi->wdev || !wifi->connected || READ_ONCE(wifi->stopping)) + ret = -ENOENT; + else { + ether_addr_copy(mac, wifi->bssid); + fill_station(wifi, sinfo); + } + mutex_unlock(&wifi->lock); + return ret; +} + static const struct cfg80211_ops wifi_ops = { .scan = wifi_scan, + .abort_scan = wifi_abort_scan, .connect = wifi_connect, .disconnect = wifi_disconnect, .set_power_mgmt = wifi_power_mgmt, + .get_station = wifi_get_station, + .dump_station = wifi_dump_station, }; static int data_start(struct centauri_wifi *wifi); @@ -1465,6 +1640,10 @@ static ssize_t fwlog_start_write(struct file *file, const char __user *input, return ret ?: -EINVAL; mutex_lock(&wifi->lock); mutex_lock(&wifi->fwlog_lock); + if (READ_ONCE(wifi->stopping)) { + ret = -ENODEV; + goto out; + } if (!wifi->configured || wifi->connecting || wifi->connected || wifi->scan_request) { ret = -EBUSY; goto out; @@ -2445,14 +2624,17 @@ int centauri_wifi_register(struct centauri *c) SET_NETDEV_DEV(wifi->netdev, &c->pdev->dev); strscpy(wifi->netdev->name, "wlan%d", IFNAMSIZ); wifi->netdev->max_mtu = ETH_DATA_LEN; - ret = register_netdev(wifi->netdev); - if (ret) - goto failed; + /* Userspace can open the interface as soon as it is registered. Finish + * firmware setup before publication so that first open is usable. + */ ret = radio_start(wifi); if (ret) { dev_err(&c->pdev->dev, "Wi-Fi ring/interface setup failed: %d\n", ret); goto failed; } + ret = register_netdev(wifi->netdev); + if (ret) + goto failed; dev_info(&c->pdev->dev, "cfg80211 interface %s ready; data rings initialize on interface open\n", wifi->netdev->name); smp_store_release(&wifi->irq_ready, true); @@ -2491,6 +2673,8 @@ void centauri_wifi_unregister(struct centauri *c) if (!wifi) return; WRITE_ONCE(wifi->stopping, true); + WRITE_ONCE(wifi->connect_abort, true); + WRITE_ONCE(wifi->scan_abort, true); smp_store_release(&c->wifi, NULL); /* A handler that observed the old pointer must finish before any work * item or Wi-Fi storage can be freed. Later IRQs observe NULL. @@ -2516,16 +2700,13 @@ void centauri_wifi_unregister(struct centauri *c) debugfs_remove(wifi->fwlog_status_file); debugfs_remove(wifi->fwlog_buffer_file); debugfs_remove(wifi->fwlog_ack_file); - cancel_work_sync(&wifi->scan_work); + cancel_scan(wifi); cancel_work_sync(&wifi->connect_work); cancel_work_sync(&wifi->link_work); + mutex_lock(&wifi->lock); + cancel_connect_locked(wifi); + mutex_unlock(&wifi->lock); skb_queue_purge(&wifi->data_tx_queue); - if (wifi->scan_request) { - struct cfg80211_scan_info info = { .aborted = true }; - - cfg80211_scan_done(wifi->scan_request, &info); - wifi->scan_request = NULL; - } if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) unregister_netdev(wifi->netdev); if (wifi->registered) diff --git a/drivers/pci/controller/apple-x1n1/t6050.c b/drivers/pci/controller/apple-x1n1/t6050.c index a4230388ece6c..9a0365e79ecee 100644 --- a/drivers/pci/controller/apple-x1n1/t6050.c +++ b/drivers/pci/controller/apple-x1n1/t6050.c @@ -192,7 +192,9 @@ int t6050_pcie_cycle(struct t6050_pcie *pcie, struct pci_dev *endpoint) if (!bridge || endpoint->vendor != PCI_VENDOR_ID_APPLE || (endpoint->device != 0x1900 && endpoint->device != 0x1901)) return -ENODEV; - pci_lock_rescan_remove(); + /* No endpoint mutation before acquiring PCI topology ownership. */ + if (!pci_trylock_rescan_remove()) + return -EAGAIN; mutex_lock(&pcie->lock); list_for_each_entry(sibling, &endpoint->bus->devices, bus_list) { if (sibling != endpoint) { diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index b63cd0c310bc0..bfb7e57ebef77 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -3520,6 +3520,15 @@ void pci_lock_rescan_remove(void) } EXPORT_SYMBOL_GPL(pci_lock_rescan_remove); +/* Work cancelled by a PCI remove callback must not wait for the lock that + * the remover already holds. Callers retry only after leaving that work. + */ +bool pci_trylock_rescan_remove(void) +{ + return mutex_trylock(&pci_rescan_remove_lock); +} +EXPORT_SYMBOL_GPL(pci_trylock_rescan_remove); + void pci_unlock_rescan_remove(void) { mutex_unlock(&pci_rescan_remove_lock); diff --git a/include/linux/pci-apple-x1n1.h b/include/linux/pci-apple-x1n1.h index a5d116fa9ef10..5a06c7dc1a320 100644 --- a/include/linux/pci-apple-x1n1.h +++ b/include/linux/pci-apple-x1n1.h @@ -5,6 +5,7 @@ #include struct pci_dev; #if IS_REACHABLE(CONFIG_PCIE_APPLE_X1N1) +/* Returns -EAGAIN, without changing hardware, if PCI rescan/removal is busy. */ int apple_x1n1_pcie_cycle(struct pci_dev *pdev); #else static inline int apple_x1n1_pcie_cycle(struct pci_dev *pdev) diff --git a/include/linux/pci.h b/include/linux/pci.h index 2c4454583c115..dd17fc620becf 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1540,6 +1540,7 @@ void set_pcie_hotplug_bridge(struct pci_dev *pdev); /* Functions for PCI Hotplug drivers to use */ unsigned int pci_rescan_bus(struct pci_bus *bus); void pci_lock_rescan_remove(void); +bool pci_trylock_rescan_remove(void); void pci_unlock_rescan_remove(void); /* Vital Product Data routines */ From e7a7dcaeb243cb5b788e93feb7c7fa7b7969a780 Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 28 Sep 2026 13:47:45 +1300 Subject: [PATCH 0164/1579] soc: apple: sep: put T8103 on the Sepos13 key-store dialect A j293 (MacBook Pro 13" M1) whose SEP runs the macOS 15 sepOS answers the Variant5 CREATE_KEYBAG with mailbox status -13 and an empty body, the same status this key store gives a malformed body. aurora-silicon/linux#37 verified that the j293 and j313 ADTs carry /defaults cpx-encryption-mode = 2 and provisioned an M1 with the Sepos13 dialect, so use that for T8103. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013wmzHjSiZbAZXBWuEBEwzx --- drivers/soc/apple/profile.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index ffab27e4b20b4..bc89ac05adaeb 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -161,7 +161,13 @@ const T8103: PlatformProfile = PlatformProfile { bag_type: 0x20000, arg: 0, }, - key_store: KeyStore::Variant5, + // The 13.5 key-store dialect, with the init handshake (0x4d, then set_env). + // The j293 and j313 ADTs both carry `/defaults` `cpx-encryption-mode = 2` + // (aurora-silicon/linux#37). With Variant5 a j293 running the macOS 15 + // sepOS answers CREATE_KEYBAG with status -13 and an empty body. + key_store: KeyStore::Sepos13 { + cpx_encryption_mode: 2, + }, }; const T6020: PlatformProfile = PlatformProfile { From e655c2048e5216c87e63dedf28f48af07071090e Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 28 Sep 2026 13:58:24 +1300 Subject: [PATCH 0165/1579] soc: apple: sep: T8103 identity keybag create values from the 13.5 identity_create The Sepos13 dialect belongs with the 13.5 identity_create encoding: version 2, type 0x400000, parent -1. #49 kept T8103 on variant 0 / 0x20000 / 0 while switching dialects; 13.5's identity_create accepts only -1 or <= -10 as the parent, and a j293 answers that body with -13. These are the values aurora-silicon/linux#37 provisioned a j313 with. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013wmzHjSiZbAZXBWuEBEwzx --- drivers/soc/apple/profile.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index bc89ac05adaeb..76032c54f688d 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -155,11 +155,14 @@ const T8103: PlatformProfile = PlatformProfile { }, dart_range_required: false, firmware_region: c"sepfw", - // Strict enclave: the type goes in the third word; the first word is 0. + // macOS 13.5's AppleKeyStore::identity_create (0xfffffe000994b2d8): + // version 2, type 0x400000, parent -1 (it accepts only -1 or <= -10). + // This is the pairing aurora-silicon/linux#37 provisioned a j313 with; + // with variant 0 / 0x20000 / 0 a j293 gets CREATE_KEYBAG status -13. keybag_create: KeybagCreate { - variant: 0, - bag_type: 0x20000, - arg: 0, + variant: 2, + bag_type: 0x40_0000, + arg: -1, }, // The 13.5 key-store dialect, with the init handshake (0x4d, then set_env). // The j293 and j313 ADTs both carry `/defaults` `cpx-encryption-mode = 2` From ba174597e055d18de8da5990325b3da2825e647f Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 28 Sep 2026 13:33:55 +1300 Subject: [PATCH 0166/1579] soc: apple: sep: log every silent exit in identity-keybag provisioning On a j293 with a macOS 15 sepOS the write boot printed only 'bringup: identity-keybag provisioning failed' with nothing before it. The silent returns were: sks_exchange() failing before the wait, sks_report_response() seeing an empty or unparsable reply (which hides the mailbox status of a refusal), read_blob() finding no fv_data, sks_read_uuid() failing, and the COPY_KEYBAG pair. Each now names itself and reports the mailbox status and response size where it has them. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013wmzHjSiZbAZXBWuEBEwzx --- drivers/soc/apple/sks.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 66ffb72e218a5..7499b5c50c447 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -654,9 +654,20 @@ impl SepData { }; intent.sending(); let Some(out) = self.sks_exchange(request.name, request.msg, &request.img) else { + dev_err!( + self.dev, + "sks: CREATE_KEYBAG exchange did not complete (buffers unregistered, wedged, or OOL write failed)\n" + ); return false; }; let Some(body) = self.sks_report_response(crate::sks::SKS_CREATE_NAME, &out) else { + dev_err!( + self.dev, + "sks: CREATE_KEYBAG reply carried no parsable body: mailbox status {}, response {} bytes, response_size {}\n", + out.reply.status, + out.response.len(), + out.reply.response_size + ); return false; }; // 13.5 replies with the struct version and the handle alone @@ -681,6 +692,11 @@ impl SepData { profile::KeyStore::Sepos13 { .. } => 8, profile::KeyStore::Variant5 => { let Some((_fv_data, end)) = image::read_blob(body, 8) else { + dev_err!( + self.dev, + "sks: CREATE_KEYBAG reply has no fv_data blob at offset 8 (body {} bytes)\n", + body.len() + ); return false; }; end @@ -704,15 +720,23 @@ impl SepData { profile::KeyStore::Sepos13 { .. } => None, profile::KeyStore::Variant5 => { let Some(bag_uuid) = self.sks_read_uuid(handle) else { + dev_err!(self.dev, "sks: could not read the new keybag's UUID back\n"); return false; }; Some(bag_uuid) } }; let Some(out) = self.sks_send(self.sks_req_copy_keybag(handle)) else { + dev_err!(self.dev, "sks: COPY_KEYBAG of the new identity keybag got no reply\n"); return false; }; let Some(wrapped) = self.wrapped_from_copy_reply(&out, c"new identity keybag") else { + dev_err!( + self.dev, + "sks: COPY_KEYBAG reply unusable: mailbox status {}, response {} bytes\n", + out.reply.status, + out.response.len() + ); return false; }; let committed = match bag_uuid { From 8727513dcde517bd8a19653ca5f9773bcb8cd71b Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 28 Sep 2026 14:14:48 +1300 Subject: [PATCH 0167/1579] soc: apple: sep: report the enclave's raw status on every refused SBIO call Enrolment on a j293 passes one stage and then fails with the generic ENROL_STATUS_ENCLAVE, with nothing in the log to say which of the five SBIO operations per capture was refused or with what status. Log the op name and raw status in sbio_call() and sbio_relay(), name the completion-without-template exit with the capture count, and label the two reply-length checks. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013wmzHjSiZbAZXBWuEBEwzx --- drivers/soc/apple/sbio.rs | 44 +++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 4c289d2c5dbb0..6c9baf3514e59 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -20,28 +20,45 @@ impl SepData { pub(crate) fn sbio_call(&self, op: &crate::sbio::SbioOp) -> SbioOutcome { let done = match self.sbio_transfer(op) { Ok(done) => done, - Err(_) => { + Err(e) => { + dev_err!(self.dev, "sbio: {} transfer failed: {:?}\n", op.name, e); return SbioOutcome::Other; } }; let Some(err) = done.status.answered() else { + dev_err!(self.dev, "sbio: {} was not answered\n", op.name); return SbioOutcome::Other; }; match err as u16 { crate::sbio::SBIO_STATUS_OK => SbioOutcome::Ok(done.payload), - crate::sbio::SBIO_STATUS_PREREQUISITE => SbioOutcome::PrerequisiteMissing, - crate::sbio::SBIO_STATUS_16 => SbioOutcome::Status16, - _ => SbioOutcome::Other, + crate::sbio::SBIO_STATUS_PREREQUISITE => { + dev_err!(self.dev, "sbio: {} refused: status {:#x} (prerequisite missing)\n", op.name, err); + SbioOutcome::PrerequisiteMissing + } + crate::sbio::SBIO_STATUS_16 => { + dev_err!(self.dev, "sbio: {} refused: status {:#x}\n", op.name, err); + SbioOutcome::Status16 + } + _ => { + dev_err!(self.dev, "sbio: {} refused: status {:#x}, payload {} bytes\n", op.name, err, done.payload.len()); + SbioOutcome::Other + } } } fn sbio_relay(&self, relay: &crate::sbio::SbioRelay<'_>) -> Option> { match self.sbio_transfer_raw(relay.opcode(), relay.name(), relay.payload()) { Ok(done) if done.status.is_ok() => Some(done.payload), - Ok(_) => None, - Err(_) => None, + Ok(done) => { + dev_err!(self.dev, "sbio: relay {} (opcode {:#x}) refused: status {:?}\n", relay.name(), relay.opcode(), done.status.answered()); + None + } + Err(e) => { + dev_err!(self.dev, "sbio: relay {} (opcode {:#x}) transfer failed: {:?}\n", relay.name(), relay.opcode(), e); + None + } } } @@ -1096,6 +1113,7 @@ impl SepData { } let Some(open) = self.begin_enrolment_on_enclave() else { + dev_err!(self.dev, "enrol: the enclave refused to open an enrolment (see sbio: lines above)\n"); self.finish_enrolment(Err(ENROL_STATUS_ENCLAVE)); let _ = sensor::idle(); return; @@ -1138,6 +1156,13 @@ impl SepData { // completion is the flag at offset 0xbfe, not a derived stage count if complete { if !has_template { + dev_err!( + self.dev, + "enrol: enclave flags completion after {} capture(s) (stage {}, {}%) but reports no template\n", + counter, + stage, + percent + ); break Some(Err(ENROL_STATUS_ENCLAVE)); } enrolment_completed = true; @@ -1703,6 +1728,7 @@ impl SepData { return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; if assessment.len() < crate::sbio::ASSESS_MIN_LEN { + dev_err!(self.dev, "enrol: IMAGE_ASSESSMENT reply too short: {} bytes (need {})\n", assessment.len(), crate::sbio::ASSESS_MIN_LEN); return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); } let usable = assessment[crate::sbio::ASSESS_USABLE_ENROL] != 0; @@ -1714,6 +1740,12 @@ impl SepData { return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; let Some(parsed) = crate::sbio::EnrolmentResult::parse(&result) else { + dev_err!( + self.dev, + "enrol: ENROLMENT_RESULT reply did not parse: {} bytes, head {:02x?}\n", + result.len(), + &result[..result.len().min(16)] + ); return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); }; From a292a425e7bc72299ba5ec196c2767893b94759e Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 28 Sep 2026 16:59:25 +1300 Subject: [PATCH 0168/1579] soc: apple: sep: report the identity-session unlock result It is discarded with 'let _ =', so a refusal is invisible and a locked identity session looks the same as the enclave refusing enrolment. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013wmzHjSiZbAZXBWuEBEwzx --- drivers/soc/apple/sbio.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 6c9baf3514e59..f1826f5fe7d5b 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2267,9 +2267,19 @@ impl SepData { if let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) { let special = user.special_handle(); if let Ok(keybag::State::Present(stored)) = keybag::read(keybag::Slot::Identity) { - let _ = self.sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { - self.sks_req_unlock_special(special, stored.secret(), healthy) - }); + // Report it: a refusal here would otherwise be invisible, and + // the identity session being locked would be indistinguishable + // from the enclave refusing enrolment for its own reasons. + let unlocked = self + .sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { + self.sks_req_unlock_special(special, stored.secret(), healthy) + }) + .is_some(); + dev_info!( + self.dev, + "identity session unlock: {}\n", + if unlocked { "accepted" } else { "REFUSED or not sent" } + ); } } } From 7b7ded9263d7155d5cc8c5d4d2006bad4297a0a1 Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 28 Sep 2026 14:44:22 +1300 Subject: [PATCH 0169/1579] =?UTF-8?q?soc:=20apple:=20sep:=20IOC=5FSBIO=5FP?= =?UTF-8?q?ROBE=20=E2=80=94=20raw=20biometrics-endpoint=20probe=20from=20u?= =?UTF-8?q?serspace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lets request layouts be explored from a script without a reboot per variant: userspace supplies an opcode and body, the driver performs the transfer and returns the raw status and reply. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013wmzHjSiZbAZXBWuEBEwzx --- drivers/soc/apple/bio.rs | 19 ++++++++++++++++-- drivers/soc/apple/sbio.rs | 41 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index e66710b6935cf..8f0bbdfdd0ef0 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -150,6 +150,20 @@ pub(crate) struct Attest { pub(crate) reserved: [u8; 3], } +/// Raw biometrics-endpoint probe (diagnostics): userspace supplies an sbio +/// opcode and body, the driver performs the transfer and returns the raw +/// status and reply. Lets request layouts be explored without a reboot. +#[repr(C)] +pub(crate) struct SbioProbe { + pub(crate) opcode: u16, + pub(crate) len: u16, + pub(crate) status: i32, + pub(crate) reply_len: u32, + pub(crate) payload: [u8; 152], + pub(crate) reply: [u8; 256], +} +static_assert!(core::mem::size_of::() == 420); + // Sizes are baked into the ioctl numbers. static_assert!(core::mem::size_of::() == 144); static_assert!(core::mem::size_of::() == 32); @@ -174,8 +188,8 @@ macro_rules! ioctl_pod { }; } ioctl_pod! { - to_user: Info, List, EnrolPoll, VerifyPoll, Attest; - from_user: EnrolStart, VerifyStart, Delete, Attest + to_user: Info, List, EnrolPoll, VerifyPoll, Attest, SbioProbe; + from_user: EnrolStart, VerifyStart, Delete, Attest, SbioProbe } const MAGIC: u32 = 0xB1; @@ -190,6 +204,7 @@ const IOC_CANCEL: u32 = _IO(MAGIC, 0x07); const IOC_DELETE: u32 = _IOW::(MAGIC, 0x08); const IOC_DELETE_ALL: u32 = _IO(MAGIC, 0x09); pub(crate) const IOC_ATTEST: u32 = _IOWR::(MAGIC, 0x0a); +pub(crate) const IOC_SBIO_PROBE: u32 = _IOWR::(MAGIC, 0x0b); pub(crate) struct MatchEvidence { identity: [u8; UUID_LEN], diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index f1826f5fe7d5b..8ae6ba132285b 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2178,6 +2178,9 @@ impl SepData { if cmd == bio::IOC_ATTEST { return self.bio_attest(arg); } + if cmd == bio::IOC_SBIO_PROBE { + return self.bio_sbio_probe(arg); + } let handled = { let mut session = self.bio_session.lock(); let index = self.bio_index.lock(); @@ -2216,6 +2219,44 @@ impl SepData { Ok(handled) } + /// Diagnostics: perform one raw sbio transfer on behalf of userspace. + fn bio_sbio_probe(&self, arg: usize) -> Result { + let user = kernel::uaccess::UserPtr::from_addr(arg); + let mut req: bio::SbioProbe = + kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; + let n = (req.len as usize).min(SBIO_MAX_PAYLOAD).min(req.payload.len()); + let done = self.sbio_transfer_raw(req.opcode, c"PROBE", &req.payload[..n])?; + req.status = match done.status.answered() { + Some(v) => v as i32, + None => i32::MIN, + }; + let m = done.payload.len().min(req.reply.len()); + req.reply = [0u8; 256]; + req.reply[..m].copy_from_slice(&done.payload[..m]); + req.reply_len = done.payload.len() as u32; + dev_info!( + self.dev, + "sbio: PROBE opcode {:#x} len {} -> status {:?}, reply {} bytes\n", + req.opcode, + n, + done.status.answered(), + done.payload.len() + ); + kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&req)?; + Ok(bio::Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }) + } + fn bio_attest(&self, arg: usize) -> Result { let user = kernel::uaccess::UserPtr::from_addr(arg); let req: bio::Attest = From 51fc698b15a87bccfb4bb611e4a54bf5b695224d Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 28 Sep 2026 17:02:42 +1300 Subject: [PATCH 0170/1579] tools: aurora-sep: do not require a ref-key on a Sepos13 key store load-driver seals a trusted key as its self-test and then requires /var/lib/aurora-sep-refkey.bin. Both only work on a Variant5 key store: ensure_machine_refkey() returns ENODEV for every other profile, so a Sepos13 machine never writes a ref-key, the service exits 1, and fprintd (Requires=aurora-sep) never starts -- although Touch ID does not use the ref-key at all. Only the sealing and FileVault paths do. Seen on a j293 (T8103 on Sepos13); this also affects T6020 once it moves to Sepos13. Keep the keybag check fatal, make the seal and ref-key advisory. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013wmzHjSiZbAZXBWuEBEwzx --- tools/aurora-sep/load-driver | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/aurora-sep/load-driver b/tools/aurora-sep/load-driver index f5579fc01b360..4ee7c9d157cdb 100755 --- a/tools/aurora-sep/load-driver +++ b/tools/aurora-sep/load-driver @@ -35,9 +35,16 @@ while [ "$attempts" -gt 0 ]; do attempts=$((attempts - 1)) done -[ "$verified" = true ] || exit 1 -[ -s "$keybag" ] || exit 1 -[ -s "$refkey" ] || exit 1 +# The trusted-key seal and the ref-key file only exist on a Variant5 key store: +# ensure_machine_refkey() returns ENODEV for every other profile, so on a +# Sepos13 machine no ref-key is ever written and these checks can never pass. +# That takes the service down with exit 1, and fprintd (Requires=aurora-sep) +# never starts, even though Touch ID itself does not use the ref-key. +if [ "$verified" != true ]; then + echo "apple_sep: trusted-key seal unavailable (expected on a Sepos13 key store)" >&2 +fi +[ -s "$keybag" ] || { echo "apple_sep: no identity keybag at $keybag" >&2; exit 1; } +[ -s "$refkey" ] || echo "apple_sep: no ref-key file (expected on a Sepos13 key store)" >&2 attempts=300 while [ "$attempts" -gt 0 ] && [ ! -c /dev/sep-bio ]; do From 5c978948f42c44c725d621bde31d70cd3439167c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 00:35:55 -0400 Subject: [PATCH 0171/1579] nvme: apple: inherit resident SPTM queue pages across U-Boot Keep firmware-pinned admin and I/O queue addresses across the OS handoff using four reserved-memory pages. Describe quiesced RTKit explicitly so Linux negotiates the mailbox session instead of treating stale ready bits as live ownership. Built and boot-tested in QEMU. On J714/T6050 hardware, U-Boot loaded Linux from SSD, all 18 CPUs started, namespaces enumerated, and direct-I/O write, overwrite and full readback passed, including verification after remount and reboot. The installed ext4 root also booted with Wi-Fi and SSH. --- Documentation/arch/arm64/t6050-sptm-ans.rst | 36 +++++++ drivers/nvme/host/apple.c | 103 ++++++++++++++++++-- 2 files changed, 130 insertions(+), 9 deletions(-) diff --git a/Documentation/arch/arm64/t6050-sptm-ans.rst b/Documentation/arch/arm64/t6050-sptm-ans.rst index a00cfb5fb6156..fc5a7908f9fc9 100644 --- a/Documentation/arch/arm64/t6050-sptm-ans.rst +++ b/Documentation/arch/arm64/t6050-sptm-ans.rst @@ -54,3 +54,39 @@ Evidence: hardware-linux-ans-durable-guest.log and hardware-linux-ans-durable-persist-guest.log under the workspace directory above. Kernel Image SHA256: 019d9c370014fc726df4b39ef1f2218e36c0ca5e8bec3177e08b96c9945cc233. + +Bootloader handoff +----------------- + +A bootloader that quiesces both RTKit AP and IOP before releasing its shared +buffers sets the boolean ``asahi,rtkit-quiesced`` on the NVMe controller node. +The next kernel must negotiate a fresh INIT session even when CC.EN and CSTS.RDY +remain set. Those controller bits describe NVMe state, not RTKit mailbox state. +Without this indication the existing live-session adoption path attempts a +controller disable against a quiesced firmware session. A bootloader preserving +an active mailbox session must not set this property. + +Queue ownership across U-Boot and Linux +-------------------------------------- + +Native SPTM retains the first registered ASQ/ACQ/IOSQ/IOCQ physical addresses +for the whole boot. Disabling the controller or quiescing RTKit does not +permit a later owner to replace those addresses. The firmware-27 admin +validation at 0xfffffff0270c8aa4 compares the new ASQ with its saved value; +a different address panics with VIOLATION_NVME_ILLEGAL_QUEUE_ADDRESS. The +Neo SPTM implementation also caches these addresses and rejects replacements. + +A resident-SPTM U-Boot therefore allocates each queue in a separate 16 KiB +page and describes the four pages with ``memory-region`` phandles on the +NVMe device. ``memory-region-names`` is ``admin-sq``, ``admin-cq``, ``io-sq``, +``io-cq``. Each reserved-memory node has ``no-map`` and a 16 KiB aligned, +16 KiB sized ``reg``. U-Boot quiesces both RTKit sides before handing off, +and supplies ``asahi,rtkit-quiesced``. Linux validates all four reservations +and maps them cacheable on this DMA-coherent device; it never substitutes +new pages for a malformed handoff. TCB scratch storage and payload DMA +mappings retain their normal allocation paths. Queue depth and command +batching are unchanged, and there are no additional per-I/O copies. + +This reservation describes firmware-owned memory, not a pool for arbitrary +DMA allocations. The pages remain reserved even if the driver is removed, +because their native SPTM registrations survive the driver lifetime. diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index d97224a5dcb12..d5ce6a3f1d5a6 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -339,6 +340,13 @@ static inline struct apple_nvme *ctrl_to_apple_nvme(struct nvme_ctrl *ctrl) static bool apple_nvme_can_adopt_rtkit(struct apple_nvme *anv) { + /* U-Boot's resident-SPTM handoff quiesces both RTKit sides before + * releasing DMA buffers. CC.EN/CSTS.RDY survive that transition and + * cannot identify a live mailbox session. The next owner must INIT. + */ + if (of_property_read_bool(anv->dev->of_node, "asahi,rtkit-quiesced")) + return false; + /* CPU_RUN and BOOT_STATUS survive an inactive firmware session. */ return anv->hw->needs_ioq_registers && (readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & @@ -1785,6 +1793,7 @@ struct apple_nvme_sart_buffer { void *buffer; dma_addr_t dma; size_t size; + bool inherited; }; static void apple_nvme_sart_buffer_release(void *data) @@ -1795,11 +1804,57 @@ static void apple_nvme_sart_buffer_release(void *data) dev_err(b->dev, "retaining queue buffer after failed SART unmap\n"); return; } - dma_free_coherent(b->dev, b->size, b->buffer, b->dma); + if (!b->inherited) + dma_free_coherent(b->dev, b->size, b->buffer, b->dma); +} + +/* SPTM pins queue addresses until the next machine boot. A previous owner + * supplies those same pages as reserved memory; never silently replace them. + * Firmware has been quiesced by that owner before Linux can clear the pages. + */ +static int apple_nvme_handoff_validate(struct apple_nvme *anv) +{ + static const char * const names[] = { "admin-sq", "admin-cq", "io-sq", "io-cq" }; + struct device_node *np = anv->dev->of_node; + phys_addr_t addresses[ARRAY_SIZE(names)]; + int i, j; + + if (!anv->hw->sptm_resident || !of_property_present(np, "memory-region")) + return 0; + if (!of_property_read_bool(np, "asahi,rtkit-quiesced") || + !dev_is_dma_coherent(anv->dev) || + of_count_phandle_with_args(np, "memory-region", NULL) != ARRAY_SIZE(names) || + of_property_count_strings(np, "memory-region-names") != ARRAY_SIZE(names)) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(names); i++) { + struct device_node *node; + struct reserved_mem *rmem; + int index = of_property_match_string(np, "memory-region-names", names[i]); + + if (index < 0) + return index; + node = of_parse_phandle(np, "memory-region", index); + if (!node) + return -EINVAL; + rmem = of_reserved_mem_lookup(node); + if (!rmem || !of_property_read_bool(node, "no-map") || + !IS_ALIGNED(rmem->base, SZ_16K) || rmem->size != SZ_16K) { + of_node_put(node); + return -EINVAL; + } + addresses[i] = rmem->base; + of_node_put(node); + for (j = 0; j < i; j++) + if (addresses[j] == addresses[i]) + return -EINVAL; + } + dev_info(anv->dev, "using four inherited SPTM queue pages\n"); + return 0; } static void *apple_nvme_queue_buffer_alloc(struct apple_nvme *anv, - size_t size, dma_addr_t *dma) + size_t size, dma_addr_t *dma, + const char *handoff_name) { struct apple_nvme_sart_buffer *b; @@ -1811,11 +1866,34 @@ static void *apple_nvme_queue_buffer_alloc(struct apple_nvme *anv, b->dev = anv->dev; b->sart = anv->sart; b->size = ALIGN(size, SZ_16K); - b->buffer = dma_alloc_coherent(b->dev, b->size, &b->dma, GFP_KERNEL); - if (!b->buffer) - return NULL; + if (handoff_name && of_property_present(anv->dev->of_node, "memory-region")) { + struct device_node *np = anv->dev->of_node, *node; + struct reserved_mem *rmem; + int index = of_property_match_string(np, "memory-region-names", handoff_name); + + if (index < 0) + return NULL; + node = of_parse_phandle(np, "memory-region", index); + if (!node) + return NULL; + rmem = of_reserved_mem_lookup(node); + of_node_put(node); + if (!rmem || b->size > rmem->size) + return NULL; + b->buffer = devm_memremap(b->dev, rmem->base, b->size, MEMREMAP_WB); + if (IS_ERR(b->buffer)) + return NULL; + b->dma = phys_to_dma(b->dev, rmem->base); + b->inherited = true; + memset(b->buffer, 0, b->size); + } else { + b->buffer = dma_alloc_coherent(b->dev, b->size, &b->dma, GFP_KERNEL); + if (!b->buffer) + return NULL; + } if (apple_sart_add_allowed_region(b->sart, b->dma, b->size)) { - dma_free_coherent(b->dev, b->size, b->buffer, b->dma); + if (!b->inherited) + dma_free_coherent(b->dev, b->size, b->buffer, b->dma); return NULL; } if (devm_add_action_or_reset(b->dev, apple_nvme_sart_buffer_release, b)) @@ -1832,7 +1910,8 @@ static int apple_nvme_queue_alloc(struct apple_nvme *anv, q->cqes = apple_nvme_queue_buffer_alloc(anv, depth * sizeof(struct nvme_completion), - &q->cq_dma_addr); + &q->cq_dma_addr, + q == &anv->adminq ? "admin-cq" : "io-cq"); if (!q->cqes) return -ENOMEM; @@ -1841,7 +1920,8 @@ static int apple_nvme_queue_alloc(struct apple_nvme *anv, else iosq_size = depth << APPLE_NVME_IOSQES; - q->sqes = apple_nvme_queue_buffer_alloc(anv, iosq_size, &q->sq_dma_addr); + q->sqes = apple_nvme_queue_buffer_alloc(anv, iosq_size, &q->sq_dma_addr, + q == &anv->adminq ? "admin-sq" : "io-sq"); if (!q->sqes) return -ENOMEM; @@ -1853,7 +1933,7 @@ static int apple_nvme_queue_alloc(struct apple_nvme *anv, q->tcbs = apple_nvme_queue_buffer_alloc(anv, anv->hw->max_queue_depth * sizeof(struct apple_nvmmu_tcb), - &q->tcb_dma_addr); + &q->tcb_dma_addr, NULL); if (!q->tcbs) return -ENOMEM; } @@ -2060,6 +2140,11 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) INIT_WORK(&anv->remove_work, apple_nvme_remove_dead_ctrl_work); spin_lock_init(&anv->lock); + ret = apple_nvme_handoff_validate(anv); + if (ret) { + dev_err(dev, "invalid inherited SPTM queue memory: %d\n", ret); + goto put_dev; + } ret = apple_nvme_queue_alloc(anv, &anv->adminq); if (ret) goto put_dev; From f8244abd3e286cf6d03cf1e1a0c0515444a683f4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 00:44:11 -0400 Subject: [PATCH 0172/1579] wifi: apple: remove child debugfs entries before their parent Parent-first recursive removal left stale Wi-Fi dentry pointers and panicked in debugfs_remove during system shutdown. Tear down Wi-Fi first and remove the parent outside boot_lock so in-flight diagnostic writers can finish. An ASan/UBSan regression harness reproduces the old failure and checks child ownership and writer ordering. J714 hardware shutdown now passes N1 teardown and reaches the platform reset step. The guest reset device is a separate bring-up item. --- drivers/net/wireless/apple/centauri/centauri_pci.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/apple/centauri/centauri_pci.c b/drivers/net/wireless/apple/centauri/centauri_pci.c index 10b85ec7d519c..032dc04b88ce4 100644 --- a/drivers/net/wireless/apple/centauri/centauri_pci.c +++ b/drivers/net/wireless/apple/centauri/centauri_pci.c @@ -402,13 +402,19 @@ static void centauri_remove(struct pci_dev *pdev) * only trylock on that path and observes this before requeueing. */ WRITE_ONCE(c->removing, true); - debugfs_remove_recursive(c->debug); cancel_delayed_work_sync(&c->boot_work); mutex_lock(&c->boot_lock); centauri_wifi_unregister(c); centauri_free_boot(c); c->boot_phase = CEN_BOOT_STOPPED; mutex_unlock(&c->boot_lock); + /* Wi-Fi owns the child dentries and removes them before freeing its + * storage. Removing the parent first leaves stale child pointers for + * that teardown. Drop boot_lock before draining the parent files: an + * in-flight rom_boot writer may be waiting for that mutex. + */ + debugfs_remove_recursive(c->debug); + c->debug = NULL; } static const struct pci_device_id centauri_ids[] = { From 7a6e658b76cc3aa7f3ade08a41844524bc552183 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 22:53:20 -0700 Subject: [PATCH 0173/1579] drm/apple: Keep dual-stream Type-C possible_crtcs fixed A Type-C port's encoder spans every pipeline that can drive the port, and routing narrows its possible_crtcs to the pipeline actually chosen, relying on userspace to re-read them on the following hotplug. Compositors that read possible_crtcs once when the connector appears (aquamarine, used by Hyprland) never see that. With a USB4 dock carrying two streams, whichever connector connects first takes the lowest free CRTC. When that is the DPIN1 connector, it gets dcpext0's CRTC while the stream is routed to dcpext1, every modeset fails the atomic check with -EINVAL, and both dock displays stay dark until the dock is replugged and happens to connect in the other order. possible_crtcs is meant to be static. On the dual-stream machines (J414s, J416c): - Fix the DPIN1 connector to the lowest-indexed Type-C-only pipeline (dcpext1), where DPIN1 is always routed, and route DPIN1 only within the connector's possible_crtcs. - Leave the primary connector spanning all its pipelines and stop narrowing or restoring possible_crtcs at runtime. - Route direct DP-alt displays to the lowest free CRTC, which is what a compositor picks from the port's fixed possible_crtcs, instead of steering them away from dcpext0. A direct DP-alt display now takes dcpext0 when it is free, so a single-stream dock on another port lands on the next free pipeline. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan --- drivers/gpu/drm/apple/apple_drv.c | 30 ++++++++---- drivers/gpu/drm/apple/dcp.c | 77 +++++++++++++++++++++++-------- drivers/gpu/drm/apple/dcp.h | 2 + 3 files changed, 81 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index ec98812a28062..4919fe082a748 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -269,9 +269,10 @@ apple_connector_atomic_best_encoder(struct drm_connector *conn, struct drm_encoder *encoder; /* - * A Type-C port has one encoder, and its possible_crtcs already names - * the pipeline the fabric routed the port to, so it is the only answer - * there is. + * A Type-C port has one encoder, whose possible_crtcs either already + * names the pipeline the fabric routed the port to or, on dual-stream + * machines, the fixed set the fabric only routes within, so it is the + * only answer there is. */ if (apple_connector->port_encoder) return apple_connector->port_encoder; @@ -437,9 +438,8 @@ static int apple_probe_typec_ports(struct drm_device *drm, /* Keep the existing physical-port connector numbers stable. On J414s, * append one more connector per port for a second USB4 DP tunnel. */ - for (idx = 0; idx < nr_ports * - ((of_machine_is_compatible("apple,j414s") || - of_machine_is_compatible("apple,j416c")) ? 2 : 1); idx++) { + for (idx = 0; idx < nr_ports * (dcp_typec_dual_stream() ? 2 : 1); + idx++) { struct apple_connector *connector; struct apple_encoder *enc; unsigned int port_idx = idx % nr_ports; @@ -472,8 +472,22 @@ static int apple_probe_typec_ports(struct drm_device *drm, INIT_WORK(&connector->hotplug_wq, dcp_hotplug); for (i = 0; i < num_dcp; i++) { - if (dcp_typec_port_has_candidate(port_idx, dcp[i])) - mask |= crtc_mask[i]; + if (!dcp_typec_port_has_candidate(port_idx, dcp[i])) + continue; + /* + * A dock's second stream (DPIN1) always runs on the + * lowest-indexed Type-C-only pipeline (dcpext1). Fix the + * connector to it: compositors read possible_crtcs once + * and would otherwise pair it with a pipeline the stream + * can never be routed to. + */ + if (secondary) { + if (dcp_is_typec_only(dcp[i]) && + (!mask || crtc_mask[i] < mask)) + mask = crtc_mask[i]; + continue; + } + mask |= crtc_mask[i]; } if (!mask) { diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index b41cd17dec2cd..a0c5f6b1ab3be 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -139,6 +139,40 @@ static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) !dcp_typec_route_fixed_output_busy(route); } +/* + * Machines whose USB4 docks carry two independent DP streams through one + * port: DPIN0 drives the hybrid dcpext0, DPIN1 the Type-C-only dcpext1, and + * each port has a second connector for the DPIN1 stream. + */ +bool dcp_typec_dual_stream(void) +{ + return of_machine_is_compatible("apple,j414s") || + of_machine_is_compatible("apple,j416c"); +} + +bool dcp_is_typec_only(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return !dcp->fixed_phy; +} + +/* + * Can this route feed the given connector without changing its + * possible_crtcs? On dual-stream machines those are fixed at probe, since + * compositors read them once when the connector appears and pair the + * connector with a CRTC before this fabric has routed it. + */ +static bool dcp_typec_route_fits(struct apple_dcp_typec_route *route, + struct apple_connector *connector) +{ + struct apple_dcp *dcp = route->dcp; + + if (!dcp_typec_dual_stream() || !connector || !dcp->crtc) + return true; + return connector->candidate_crtcs & drm_crtc_mask(&dcp->crtc->base); +} + static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); @@ -238,9 +272,12 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * Offering it the choice makes it pair the port with a pipeline * holding a different monitor's mode list, and the modeset is * rejected with no way for it to recover. The hotplug that - * follows makes it re-read this. + * follows makes it re-read this. Dual-stream machines keep + * fixed possible_crtcs instead: compositors that read them once + * (e.g. aquamarine/Hyprland) never see the narrowing. */ - if (connector->port_encoder && dcp->crtc) + if (connector->port_encoder && dcp->crtc && + !dcp_typec_dual_stream()) connector->port_encoder->possible_crtcs = drm_crtc_mask(&dcp->crtc->base); } @@ -325,7 +362,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) connector->dcp = NULL; /* Unrouted: the port could go to any of its pipelines again. */ - if (connector->port_encoder) + if (connector->port_encoder && !dcp_typec_dual_stream()) connector->port_encoder->possible_crtcs = connector->candidate_crtcs; } @@ -600,17 +637,20 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, if (!dcp_typec_route_available(candidate)) continue; + if (!dcp_typec_route_fits(candidate, dpin ? + port->secondary_connector : + port->connector)) + continue; score = dcp_typec_route_score(candidate); /* - * DPIN0 prefers the hybrid dcpext0 on T6020. J414s DPIN1 - * prefers dcpext1 so both pipelines can drive independent - * streams through one dock. Retain J416s' existing preference - * until its second pipeline has been validated separately. + * DPIN0 prefers the hybrid dcpext0 on T6020. On dual-stream + * machines DPIN1 is confined to dcpext1 by its connector's + * possible_crtcs, so both pipelines drive independent streams + * through one dock. Retain J416s' existing preference until + * its second pipeline has been validated separately. */ - if (of_machine_is_compatible("apple,j414s") || - of_machine_is_compatible("apple,j416c")) { - if ((dpin == 0 && !candidate->dcp->fixed_phy) || - (dpin == 1 && candidate->dcp->fixed_phy)) + if (dcp_typec_dual_stream()) { + if (dpin == 0 && !candidate->dcp->fixed_phy) score += 100; } else if (of_machine_is_compatible("apple,j416s") && !candidate->dcp->fixed_phy) { @@ -776,16 +816,12 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (!dcp_typec_route_available(candidate)) continue; - score = dcp_typec_route_score(candidate); /* - * On J414s, leave dcpext0 free for a dock's DP - * tunnel when a direct DP-alt-mode display can use - * the Type-C-only dcpext1 pipeline. + * Lowest free CRTC index first: on dual-stream + * machines that is what a compositor picks from + * the port's fixed possible_crtcs. */ - if ((of_machine_is_compatible("apple,j414s") || - of_machine_is_compatible("apple,j416c")) && - candidate->dcp->fixed_phy) - score += 100; + score = dcp_typec_route_score(candidate); if (score < best_score) { best = candidate; best_score = score; @@ -946,7 +982,8 @@ void dcp_typec_port_set_connector(unsigned int idx, bool secondary, if (owner) { struct apple_dcp *dcp = owner->dcp; - if (dcp->crtc && connector->port_encoder) + if (dcp->crtc && connector->port_encoder && + !dcp_typec_dual_stream()) connector->port_encoder->possible_crtcs = drm_crtc_mask(&dcp->crtc->base); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index b802574261d8d..a08ec457f224e 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -46,6 +46,8 @@ struct device_node *dcp_typec_port_of_node(unsigned int idx); bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev); void dcp_typec_port_set_connector(unsigned int idx, bool secondary, struct apple_connector *connector); +bool dcp_typec_dual_stream(void); +bool dcp_is_typec_only(struct platform_device *pdev); bool dcp_fw_compat_is_12_x(struct platform_device *pdev); unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, From 092d42c265fb0a472aa714aa36c75717821fd847 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 23:30:01 -0700 Subject: [PATCH 0174/1579] thunderbolt: apple: Drop pending PCIe-C activation on tunnel teardown PCIe-C is populated from delayed work queued when the PCIe tunnel is activated. If the cable bounces, the tunnel can be deactivated before that work runs; pcie_tunnel_requested stays set, so the work then cold-initializes and starts PCIe-C against a tunnel that no longer exists. On J416c the port then never links again ("link didn't come up") until reboot, even after replugging or power-cycling the dock. Clear the request and cancel the pending work on tunnel deactivation. The work already re-checks the request under pcie_tunnel_lock, so a run that is in progress either completes before the teardown takes the lock, and is quiesced by it, or does nothing. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan --- drivers/thunderbolt/apple.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 43c595d79f088..e69bd9feeada3 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -671,6 +671,17 @@ static int apple_nhi_pci_tunnel_deactivate(struct tb_nhi *nhi) * blocks until the function driver's own timeout fires, and the host has * nothing to enumerate onto when the tunnel comes back. */ + /* + * Withdraw a pending deferred activation first. If the cable bounces, + * the tunnel can be torn down before that work runs, and starting + * PCIe-C against a tunnel that no longer exists leaves the port unable + * to train on every later connect. The work re-checks the request + * under pcie_tunnel_lock, so one already running either finishes + * before we take the lock (and is quiesced below) or does nothing. + */ + WRITE_ONCE(acio->pcie_tunnel_requested, false); + cancel_delayed_work(&acio->pcie_tunnel_work); + mutex_lock(&acio->pcie_tunnel_lock); if (!acio->pcie_tunnel_populated) { mutex_unlock(&acio->pcie_tunnel_lock); From 18b9e789ea2490b5bf29f26ba2448655698fbf99 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 23:30:01 -0700 Subject: [PATCH 0175/1579] PCI: apple: Don't restart PCIe-C on resume after the tunnel was lost When the USB4 router disappears during suspend (e.g. a Thunderbolt 3 dock under s2idle), the NHI's resume tears the PCIe tunnel down and quiesces this host before PCIe-C's own resume_noirq runs. That resume then starts the ports anyway, training against a tunnel that no longer exists; on J416c the port is afterwards unable to link on every later connect until reboot. Skip restarting the ports when the host has been quiesced. They are started again by apple_pcie_tunnel_restore(), or cold-initialized by a fresh populate, once a new tunnel is up. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan --- drivers/pci/controller/pcie-apple.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index b1703965cc1cf..f63837dc8014d 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2080,6 +2080,20 @@ static int apple_pcie_resume_noirq(struct device *dev) if (!pcie->hw->tunneled) return 0; + + /* + * The NHI resumes first (see apple_pcie_tunnel_add_links()). If the + * USB4 router was lost during suspend, its teardown has already + * quiesced this host. Starting the ports now would train against a + * tunnel that no longer exists, which leaves them unable to link on + * every later connect; apple_pcie_tunnel_restore() starts them once a + * new tunnel is up. + */ + if (pcie->bus_stopped) { + dev_info(dev, "PCIe-C tunnel lost during suspend, not restarting\n"); + return 0; + } + list_for_each_entry(port, &pcie->ports, entry) { ret = apple_pcie_tunnel_start(port); if (ret) From 07fdb1ed6bb93602df6f44d346e8e8f8dc68c119 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 7 Sep 2026 23:32:22 -0400 Subject: [PATCH 0176/1579] HID: Import DockChannel two-phase power protocol Adapted path-only import from linux-aurora j813 commit a8741fcc4e05c2699f6f0eaa2f9b59037d633054. Original author retained; unrelated GPU/display changes omitted. --- drivers/hid/dockchannel-hid/Kconfig | 10 + drivers/hid/dockchannel-hid/Makefile | 1 + .../dockchannel-hid-protocol-test.c | 92 +++++++++ .../dockchannel-hid-protocol.h | 53 +++++ drivers/hid/dockchannel-hid/dockchannel-hid.c | 181 ++++++++++++++++-- 5 files changed, 320 insertions(+), 17 deletions(-) create mode 100644 drivers/hid/dockchannel-hid/dockchannel-hid-protocol-test.c create mode 100644 drivers/hid/dockchannel-hid/dockchannel-hid-protocol.h diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig index 254961ad15e19..c6ab3fbafda07 100644 --- a/drivers/hid/dockchannel-hid/Kconfig +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -10,4 +10,14 @@ config HID_DOCKCHANNEL The keyboard and touchpad are HID based devices connected via the proprietary DockChannel interface. +config HID_DOCKCHANNEL_KUNIT_TEST + tristate "KUnit tests for DockChannel HID" if !KUNIT_ALL_TESTS + depends on KUNIT && HID_DOCKCHANNEL + default KUNIT_ALL_TESTS + help + Build tests for hardware-independent DockChannel HID protocol helpers. + These verify command serialization without requiring Apple hardware. + They cover both plain and two-phase power transitions. + If in doubt, say N. + endmenu diff --git a/drivers/hid/dockchannel-hid/Makefile b/drivers/hid/dockchannel-hid/Makefile index 7dba766b047fc..825ccb27db94b 100644 --- a/drivers/hid/dockchannel-hid/Makefile +++ b/drivers/hid/dockchannel-hid/Makefile @@ -4,3 +4,4 @@ # obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid.o +obj-$(CONFIG_HID_DOCKCHANNEL_KUNIT_TEST) += dockchannel-hid-protocol-test.o diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid-protocol-test.c b/drivers/hid/dockchannel-hid/dockchannel-hid-protocol-test.c new file mode 100644 index 0000000000000..747cbd5a7e848 --- /dev/null +++ b/drivers/hid/dockchannel-hid/dockchannel-hid-protocol-test.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +#include + +#include "dockchannel-hid-protocol.h" + +static void dchid_power_method_1_on_test(struct kunit *test) +{ + const u8 expected[DCHID_POWER_METHOD_1_CMD_SIZE] = { + DCHID_CMD_RESET_INTERFACE, DCHID_POWER_METHOD_1, 0x5a, + DCHID_POWER_STATE_ON, + }; + u8 command[DCHID_POWER_METHOD_1_CMD_SIZE]; + + dchid_pm1_command(0x5a, DCHID_POWER_STATE_ON, command); + + KUNIT_EXPECT_MEMEQ(test, command, expected, sizeof(expected)); +} + +static void dchid_power_method_2_will_change_test(struct kunit *test) +{ + const u8 expected[DCHID_POWER_METHOD_2_CMD_SIZE] = { + DCHID_CMD_RESET_INTERFACE, DCHID_POWER_METHOD_2, 0x5a, + DCHID_POWER_STATE_ON, 0, 0, 0, 0, 0, + }; + u8 command[DCHID_POWER_METHOD_2_CMD_SIZE]; + + dchid_pm2_command(0x5a, false, command); + + KUNIT_EXPECT_MEMEQ(test, command, expected, sizeof(expected)); +} + +static void dchid_power_method_2_has_changed_test(struct kunit *test) +{ + const u8 expected[DCHID_POWER_METHOD_2_CMD_SIZE] = { + DCHID_CMD_RESET_INTERFACE, DCHID_POWER_METHOD_2, 0xff, + DCHID_POWER_STATE_ON, 1, 0, 0, 0, 0, + }; + u8 command[DCHID_POWER_METHOD_2_CMD_SIZE]; + + dchid_pm2_command(0xff, true, command); + + KUNIT_EXPECT_MEMEQ(test, command, expected, sizeof(expected)); +} + +static void dchid_power_method_2_off_transaction_test(struct kunit *test) +{ + const u8 will[DCHID_POWER_METHOD_2_CMD_SIZE] = { + DCHID_CMD_RESET_INTERFACE, DCHID_POWER_METHOD_2, 0x5a, + DCHID_POWER_STATE_OFF, 0, 0, 0, 0, 0, + }; + const u8 has[DCHID_POWER_METHOD_2_CMD_SIZE] = { + DCHID_CMD_RESET_INTERFACE, DCHID_POWER_METHOD_2, 0x5a, + DCHID_POWER_STATE_OFF, 1, 0, 0, 0, 0, + }; + u8 command[DCHID_POWER_METHOD_2_CMD_SIZE]; + + dchid_pm2_state_command(0x5a, DCHID_POWER_STATE_OFF, false, command); + KUNIT_EXPECT_MEMEQ(test, command, will, sizeof(will)); + dchid_pm2_state_command(0x5a, DCHID_POWER_STATE_OFF, true, command); + KUNIT_EXPECT_MEMEQ(test, command, has, sizeof(has)); +} + +static void dchid_j700_default_power_method_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, dchid_default_pm(true, true, true), + DCHID_POWER_METHOD_2); + KUNIT_EXPECT_EQ(test, dchid_default_pm(false, true, true), + DCHID_POWER_METHOD_1); + KUNIT_EXPECT_EQ(test, dchid_default_pm(true, false, true), + DCHID_POWER_METHOD_1); + KUNIT_EXPECT_EQ(test, dchid_default_pm(true, true, false), + DCHID_POWER_METHOD_1); +} + +static struct kunit_case dchid_protocol_test_cases[] = { + KUNIT_CASE(dchid_power_method_1_on_test), + KUNIT_CASE(dchid_power_method_2_will_change_test), + KUNIT_CASE(dchid_power_method_2_has_changed_test), + KUNIT_CASE(dchid_power_method_2_off_transaction_test), + KUNIT_CASE(dchid_j700_default_power_method_test), + {} +}; + +static struct kunit_suite dchid_protocol_test_suite = { + .name = "dockchannel-hid-protocol", + .test_cases = dchid_protocol_test_cases, +}; + +kunit_test_suite(dchid_protocol_test_suite); + +MODULE_DESCRIPTION("Apple DockChannel HID protocol KUnit tests"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid-protocol.h b/drivers/hid/dockchannel-hid/dockchannel-hid-protocol.h new file mode 100644 index 0000000000000..d27305fad0161 --- /dev/null +++ b/drivers/hid/dockchannel-hid/dockchannel-hid-protocol.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +#ifndef _DOCKCHANNEL_HID_PROTOCOL_H +#define _DOCKCHANNEL_HID_PROTOCOL_H + +#include +#include + +#define DCHID_CMD_RESET_INTERFACE 0x40 +#define DCHID_POWER_METHOD_1 1 +#define DCHID_POWER_METHOD_2 2 +#define DCHID_POWER_STATE_OFF 0 +#define DCHID_POWER_STATE_ON 2 +#define DCHID_POWER_METHOD_1_CMD_SIZE 4 +#define DCHID_POWER_METHOD_2_CMD_SIZE 9 + +static inline u8 dchid_default_pm(bool no_stm, bool multi_touch, + bool j700_firmware) +{ + return no_stm && multi_touch && j700_firmware ? + DCHID_POWER_METHOD_2 : DCHID_POWER_METHOD_1; +} + +static inline void +dchid_pm1_command(u8 iface, u8 state, + u8 command[DCHID_POWER_METHOD_1_CMD_SIZE]) +{ + command[0] = DCHID_CMD_RESET_INTERFACE; + command[1] = DCHID_POWER_METHOD_1; + command[2] = iface; + command[3] = state; +} + +static inline void +dchid_pm2_state_command(u8 iface, u8 state, bool has_changed, + u8 command[DCHID_POWER_METHOD_2_CMD_SIZE]) +{ + memset(command, 0, DCHID_POWER_METHOD_2_CMD_SIZE); + command[0] = DCHID_CMD_RESET_INTERFACE; + command[1] = DCHID_POWER_METHOD_2; + command[2] = iface; + command[3] = state; + command[4] = has_changed; +} + +static inline void +dchid_pm2_command(u8 iface, bool has_changed, + u8 command[DCHID_POWER_METHOD_2_CMD_SIZE]) +{ + dchid_pm2_state_command(iface, DCHID_POWER_STATE_ON, has_changed, + command); +} + +#endif /* _DOCKCHANNEL_HID_PROTOCOL_H */ diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 68f6625313ea3..b10a23afea8bf 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -19,6 +19,7 @@ #include #include #include "../hid-ids.h" +#include "dockchannel-hid-protocol.h" #define COMMAND_TIMEOUT_MS 1000 #define START_TIMEOUT_MS 2000 @@ -74,7 +75,6 @@ struct dchid_init_hdr { #define INIT_TERMINATOR 2 #define INIT_PRODUCT_NAME 7 -#define CMD_RESET_INTERFACE 0x40 #define CMD_SEND_FIRMWARE 0x95 #define CMD_ENABLE_INTERFACE 0xb4 #define CMD_ACK_GPIO_CMD 0xa1 @@ -165,6 +165,7 @@ struct dchid_iface { struct gpio_desc *gpio; char gpio_name[MAX_GPIO_NAME]; int gpio_id; + bool gpio_request_seen; struct mutex out_mutex; u32 out_flags; @@ -175,6 +176,8 @@ struct dchid_iface { struct completion out_complete; u32 keyboard_layout_id; + u32 power_method; + bool inherited_power_method_2; }; struct dockchannel_hid { @@ -211,6 +214,11 @@ static struct dchid_iface * dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) { struct dchid_iface *iface; + const char *firmware_name; + bool explicit_power_method; + bool j700_firmware; + bool multi_touch; + bool no_stm; if (index >= MAX_INTERFACES) { dev_err(dchid->dev, "Interface index %d out of range\n", index); @@ -247,6 +255,25 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) return NULL; } + explicit_power_method = !of_property_read_u32(iface->of_node, + "apple,power-method", + &iface->power_method); + if (!explicit_power_method) { + no_stm = of_property_read_bool(dchid->dev->of_node, "apple,no-stm"); + j700_firmware = + !of_property_read_string(iface->of_node, "firmware-name", + &firmware_name) && + !strcmp(firmware_name, "apple/tpmtfw-j700.bin"); + multi_touch = !strcmp(name, "multi-touch"); + iface->power_method = dchid_default_pm(no_stm, multi_touch, j700_firmware); + iface->inherited_power_method_2 = + iface->power_method == DCHID_POWER_METHOD_2; + } + + if (iface->power_method == DCHID_POWER_METHOD_2 && + !iface->inherited_power_method_2) + strscpy(iface->gpio_name, "afe-reset", MAX_GPIO_NAME); + dchid->ifaces[index] = iface; return iface; } @@ -370,8 +397,36 @@ static int dchid_enable_interface(struct dchid_iface *iface) static int dchid_reset_interface(struct dchid_iface *iface, int state) { - u8 msg[] = { CMD_RESET_INTERFACE, 1, iface->index, state }; + u8 msg[DCHID_POWER_METHOD_1_CMD_SIZE]; + + dchid_pm1_command(iface->index, state, msg); + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_power_method_2_transition(struct dchid_iface *iface, + bool has_changed) +{ + u8 msg[DCHID_POWER_METHOD_2_CMD_SIZE]; + + dchid_pm2_command(iface->index, has_changed, msg); + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_inherited_pm2_transition(struct dchid_iface *iface, u8 state) +{ + u8 msg[DCHID_POWER_METHOD_2_CMD_SIZE]; + int ret; + dchid_pm2_state_command(iface->index, state, false, msg); + ret = dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); + if (ret < 0) + return ret; + + /* Match the J700 transaction that reached the Touch-ready event. */ + if (state == DCHID_POWER_STATE_ON) + msleep(50); + + dchid_pm2_state_command(iface->index, state, true, msg); return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); } @@ -454,9 +509,11 @@ static int dchid_get_firmware(struct dchid_iface *iface, void **firmware, size_t return ret; } -static int dchid_request_gpio(struct dchid_iface *iface) +static int dchid_request_gpio(struct dchid_iface *iface, + enum gpiod_flags flags) { char prop_name[MAX_GPIO_NAME + 16]; + int ret; if (iface->gpio) return 0; @@ -466,12 +523,16 @@ static int dchid_request_gpio(struct dchid_iface *iface) snprintf(prop_name, sizeof(prop_name), "apple,%s", iface->gpio_name); - iface->gpio = devm_gpiod_get_index(iface->dchid->dev, prop_name, 0, GPIOD_OUT_LOW); + iface->gpio = devm_gpiod_get_index(iface->dchid->dev, prop_name, 0, + flags); if (IS_ERR_OR_NULL(iface->gpio)) { - dev_err(iface->dchid->dev, "Failed to request GPIO %s-gpios\n", prop_name); + ret = iface->gpio ? PTR_ERR(iface->gpio) : -ENODEV; + dev_err(iface->dchid->dev, + "Failed to request GPIO %s-gpios: %d\n", + prop_name, ret); iface->gpio = NULL; - return -1; + return ret; } return 0; @@ -479,6 +540,8 @@ static int dchid_request_gpio(struct dchid_iface *iface) static int dchid_start_interface(struct dchid_iface *iface) { + bool power_method_2; + bool inherited_power_method_2; void *fw; size_t size; int ret; @@ -492,16 +555,54 @@ static int dchid_start_interface(struct dchid_iface *iface) iface->starting = true; + power_method_2 = iface->power_method == DCHID_POWER_METHOD_2; + inherited_power_method_2 = iface->inherited_power_method_2; + if (iface->power_method != DCHID_POWER_METHOD_1 && !power_method_2) { + dev_err(iface->dchid->dev, + "Unsupported power method %u for %s\n", + iface->power_method, iface->name); + ret = -EINVAL; + goto err; + } + /* Look to see if we need firmware */ ret = dchid_get_firmware(iface, &fw, &size); - if (ret < 0) + if (ret < 0) { + dev_err(iface->dchid->dev, + "Failed to load firmware for %s: %d\n", + iface->name, ret); + goto err; + } + + if (power_method_2 && (!fw || !size)) { + dev_err(iface->dchid->dev, + "Power method 2 for %s requires firmware\n", + iface->name); + ret = -EINVAL; + goto err; + } + if (power_method_2 && !inherited_power_method_2 && + !iface->gpio_name[0]) { + dev_err(iface->dchid->dev, + "Explicit power method 2 for %s requires afe-reset\n", + iface->name); + ret = -EINVAL; goto err; + } - /* If we need a GPIO, make sure we have it. */ - if (iface->gpio_id) { - ret = dchid_request_gpio(iface); + /* Dynamic requests include ID zero; Power Method 2 uses afe-reset. */ + if (!inherited_power_method_2 && + (iface->gpio_request_seen || power_method_2)) { + ret = dchid_request_gpio(iface, power_method_2 ? + GPIOD_OUT_HIGH : GPIOD_OUT_LOW); if (ret < 0) goto err; + + if (power_method_2) { + /* GPIO descriptors use logical values for active-low lines. */ + gpiod_set_value_cansleep(iface->gpio, 1); + msleep(50); + } } /* Only multi-touch has firmware */ @@ -515,10 +616,37 @@ static int dchid_start_interface(struct dchid_iface *iface) goto err; } - /* After loading firmware, multi-touch needs a reset */ - dev_info(iface->dchid->dev, "Resetting %s\n", iface->name); - dchid_reset_interface(iface, 0); - dchid_reset_interface(iface, 2); + dev_info(iface->dchid->dev, + "Powering on %s with power method %u\n", + iface->name, iface->power_method); + if (power_method_2) { + if (inherited_power_method_2) { + ret = dchid_inherited_pm2_transition(iface, DCHID_POWER_STATE_OFF); + if (ret < 0) + goto err; + + ret = dchid_inherited_pm2_transition(iface, DCHID_POWER_STATE_ON); + if (ret < 0) + goto err; + } else { + ret = dchid_power_method_2_transition(iface, false); + if (ret < 0) + goto err; + + gpiod_set_value_cansleep(iface->gpio, 0); + + ret = dchid_power_method_2_transition(iface, true); + if (ret < 0) { + gpiod_set_value_cansleep(iface->gpio, 1); + goto err; + } + } + } else { + ret = dchid_reset_interface(iface, + DCHID_POWER_STATE_ON); + if (ret < 0) + goto err; + } } return 0; @@ -593,6 +721,12 @@ static int dchid_get_report_cmd(struct dchid_iface *iface, u8 reportnum, void *b return ret <= 0 ? ret : ret - 1; } +/* Note: buf includes report number! */ +static int dchid_set_report(struct dchid_iface *iface, void *buf, size_t len) +{ + return dchid_cmd(iface, HID_OUTPUT_REPORT, REQ_SET_REPORT, buf, len, NULL, 0); +} + static int dchid_raw_request(struct hid_device *hdev, unsigned char reportnum, __u8 *buf, size_t len, unsigned char rtype, int reqtype) @@ -604,7 +738,7 @@ static int dchid_raw_request(struct hid_device *hdev, buf[0] = reportnum; return dchid_cmd(iface, rtype, REQ_GET_REPORT, &reportnum, 1, buf + 1, len - 1); case HID_REQ_SET_REPORT: - return dchid_cmd(iface, rtype, REQ_SET_REPORT, buf, len, NULL, 0); + return dchid_set_report(iface, buf, len); default: return -EIO; } @@ -805,7 +939,7 @@ static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t if (sizeof(*req) > length) break; - if (iface->gpio_id) { + if (iface->gpio_request_seen) { dev_err(dchid->dev, "Cannot request more than one GPIO per interface!\n"); break; @@ -813,6 +947,7 @@ static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t strscpy(iface->gpio_name, req->name, MAX_GPIO_NAME); iface->gpio_id = req->id; + iface->gpio_request_seen = true; break; } @@ -871,7 +1006,14 @@ static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t goto err; } - if (dchid_request_gpio(iface) < 0) + if (!iface->gpio_request_seen) { + dev_err(dchid->dev, + "Got GPIO command without a request for %s\n", + iface->name); + goto err; + } + + if (dchid_request_gpio(iface, GPIOD_OUT_LOW) < 0) goto err; if (!iface->gpio || cmd->gpio != iface->gpio_id) { @@ -1102,6 +1244,11 @@ static int dockchannel_hid_probe(struct platform_device *pdev) } dchid->dev = dev; + /* Neo firmware has no STM interface from which to obtain identity. */ + if (of_property_read_bool(dev->of_node, "apple,no-stm")) { + dchid->device_id.vendor_id = HOST_VENDOR_ID_APPLE; + dchid->id_ready = true; + } /* * First make sure all the GPIOs are available, in cased we need to defer. From 834a670117f93bd36b226693780cce94f384b374 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 10 Sep 2026 05:00:26 -0400 Subject: [PATCH 0177/1579] HID: Retain qualified J700 power sequencing Adapted path-only import from linux-aurora j813 commit fa8af838f2cfa68007cd2421ae551b622aac133e. Original author retained; unrelated GPU/display changes omitted. --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 81 ++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index b10a23afea8bf..046d47528b612 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -156,6 +156,7 @@ struct dchid_iface { uint8_t tx_seq; bool deferred; bool starting; + bool startup_failed; bool open; struct completion ready; @@ -430,6 +431,57 @@ static int dchid_inherited_pm2_transition(struct dchid_iface *iface, u8 state) return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); } +static bool dchid_j700_power_v2(struct dchid_iface *iface) +{ + return of_machine_is_compatible("apple,j700") && + iface->index == 1 && !strcmp(iface->name, "multi-touch"); +} + +static int dchid_j700_set_power(struct dchid_iface *iface, u8 state) +{ + struct { + u8 command; + u8 version; + u8 iface; + u8 state; + u8 did_change; + __le32 status; + } __packed msg = { + .command = DCHID_CMD_RESET_INTERFACE, + .version = 2, + .iface = iface->index, + .state = state, + }; + int ret; + + /* + * J700 MTP_SYS selects PowerMethod=2. Its comm firmware accepts only + * nine-byte v2 reports: will-change, platform function, has-changed. + * The pinned power-sequence uses physical High for 50 ms on power-off + * and Low with no delay on power-on. Use raw levels for this platform + * function, independent of the legacy DT's active-low GPIO flag. + * macsmc-gpio maps it to pKW8(pcIO, 0x10000, 0x130000). + */ + if (state != 0 && state != 2) + return -EINVAL; + + ret = dchid_comm_cmd(iface->dchid, &msg, sizeof(msg)); + if (ret < 0) + return ret; + + ret = gpiod_set_raw_value_cansleep(iface->gpio, state == 0); + if (ret < 0) + return ret; + if (state == 0) + fsleep(50000); + + msg.did_change = 1; + ret = dchid_comm_cmd(iface->dchid, &msg, sizeof(msg)); + dev_info(iface->dchid->dev, "J700 MTP power-v2 state=%u result=%d\n", + state, ret); + return ret; +} + static int dchid_send_firmware(struct dchid_iface *iface, void *firmware, size_t size) { struct { @@ -546,6 +598,9 @@ static int dchid_start_interface(struct dchid_iface *iface) size_t size; int ret; + if (iface->startup_failed) + return -EIO; + if (iface->starting) { dev_warn(iface->dchid->dev, "Interface %s is already starting", iface->name); return -EINPROGRESS; @@ -590,8 +645,19 @@ static int dchid_start_interface(struct dchid_iface *iface) goto err; } + if (dchid_j700_power_v2(iface) && !iface->gpio) { + /* Do not toggle the line before the will-change notification. */ + iface->gpio = devm_gpiod_get_index(iface->dchid->dev, + "apple,afe-reset", 0, GPIOD_ASIS); + if (IS_ERR(iface->gpio)) { + ret = PTR_ERR(iface->gpio); + iface->gpio = NULL; + goto err; + } + } + /* Dynamic requests include ID zero; Power Method 2 uses afe-reset. */ - if (!inherited_power_method_2 && + if (!dchid_j700_power_v2(iface) && !inherited_power_method_2 && (iface->gpio_request_seen || power_method_2)) { ret = dchid_request_gpio(iface, power_method_2 ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW); @@ -608,6 +674,9 @@ static int dchid_start_interface(struct dchid_iface *iface) /* Only multi-touch has firmware */ if (fw && size) { + if (dchid_j700_power_v2(iface)) + iface->startup_failed = true; + /* Send firmware to the device */ dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); ret = dchid_send_firmware(iface, fw, size); @@ -619,7 +688,14 @@ static int dchid_start_interface(struct dchid_iface *iface) dev_info(iface->dchid->dev, "Powering on %s with power method %u\n", iface->name, iface->power_method); - if (power_method_2) { + if (dchid_j700_power_v2(iface)) { + ret = dchid_j700_set_power(iface, DCHID_POWER_STATE_OFF); + if (ret < 0) + goto err; + ret = dchid_j700_set_power(iface, DCHID_POWER_STATE_ON); + if (ret < 0) + goto err; + } else if (power_method_2) { if (inherited_power_method_2) { ret = dchid_inherited_pm2_transition(iface, DCHID_POWER_STATE_OFF); if (ret < 0) @@ -696,6 +772,7 @@ static int dchid_open(struct hid_device *hdev) } iface->open = true; + iface->startup_failed = false; return 0; } From e080949345fda6e257312d475b1f294ef0991d7f Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 10 Sep 2026 10:46:19 -0400 Subject: [PATCH 0178/1579] HID: Import M5 DockChannel GPIO request handling Adapted path-only import from linux-aurora j813 commit df0bbf1758efce9d9ec9e57f7c507e6878081db6. Original author retained; unrelated GPU/display changes omitted. --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 89 +++---------------- drivers/soc/apple/rtkit-helper.c | 6 +- 2 files changed, 15 insertions(+), 80 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 046d47528b612..ad4b8ff4d381c 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -156,7 +156,6 @@ struct dchid_iface { uint8_t tx_seq; bool deferred; bool starting; - bool startup_failed; bool open; struct completion ready; @@ -270,6 +269,14 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) iface->inherited_power_method_2 = iface->power_method == DCHID_POWER_METHOD_2; } + /* + * J813's MTP firmware owns the touch reset line. Its Windows bring-up + * uses the v2 Will/Has power messages, including the 50 ms ON delay, + * without a GPIO_REQUEST or an external AFE GPIO operation. + */ + if (iface->power_method == DCHID_POWER_METHOD_2 && + !strcmp(name, "multi-touch") && of_machine_is_compatible("apple,j813")) + iface->inherited_power_method_2 = true; if (iface->power_method == DCHID_POWER_METHOD_2 && !iface->inherited_power_method_2) @@ -431,57 +438,6 @@ static int dchid_inherited_pm2_transition(struct dchid_iface *iface, u8 state) return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); } -static bool dchid_j700_power_v2(struct dchid_iface *iface) -{ - return of_machine_is_compatible("apple,j700") && - iface->index == 1 && !strcmp(iface->name, "multi-touch"); -} - -static int dchid_j700_set_power(struct dchid_iface *iface, u8 state) -{ - struct { - u8 command; - u8 version; - u8 iface; - u8 state; - u8 did_change; - __le32 status; - } __packed msg = { - .command = DCHID_CMD_RESET_INTERFACE, - .version = 2, - .iface = iface->index, - .state = state, - }; - int ret; - - /* - * J700 MTP_SYS selects PowerMethod=2. Its comm firmware accepts only - * nine-byte v2 reports: will-change, platform function, has-changed. - * The pinned power-sequence uses physical High for 50 ms on power-off - * and Low with no delay on power-on. Use raw levels for this platform - * function, independent of the legacy DT's active-low GPIO flag. - * macsmc-gpio maps it to pKW8(pcIO, 0x10000, 0x130000). - */ - if (state != 0 && state != 2) - return -EINVAL; - - ret = dchid_comm_cmd(iface->dchid, &msg, sizeof(msg)); - if (ret < 0) - return ret; - - ret = gpiod_set_raw_value_cansleep(iface->gpio, state == 0); - if (ret < 0) - return ret; - if (state == 0) - fsleep(50000); - - msg.did_change = 1; - ret = dchid_comm_cmd(iface->dchid, &msg, sizeof(msg)); - dev_info(iface->dchid->dev, "J700 MTP power-v2 state=%u result=%d\n", - state, ret); - return ret; -} - static int dchid_send_firmware(struct dchid_iface *iface, void *firmware, size_t size) { struct { @@ -598,9 +554,6 @@ static int dchid_start_interface(struct dchid_iface *iface) size_t size; int ret; - if (iface->startup_failed) - return -EIO; - if (iface->starting) { dev_warn(iface->dchid->dev, "Interface %s is already starting", iface->name); return -EINPROGRESS; @@ -645,19 +598,8 @@ static int dchid_start_interface(struct dchid_iface *iface) goto err; } - if (dchid_j700_power_v2(iface) && !iface->gpio) { - /* Do not toggle the line before the will-change notification. */ - iface->gpio = devm_gpiod_get_index(iface->dchid->dev, - "apple,afe-reset", 0, GPIOD_ASIS); - if (IS_ERR(iface->gpio)) { - ret = PTR_ERR(iface->gpio); - iface->gpio = NULL; - goto err; - } - } - /* Dynamic requests include ID zero; Power Method 2 uses afe-reset. */ - if (!dchid_j700_power_v2(iface) && !inherited_power_method_2 && + if (!inherited_power_method_2 && (iface->gpio_request_seen || power_method_2)) { ret = dchid_request_gpio(iface, power_method_2 ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW); @@ -674,9 +616,6 @@ static int dchid_start_interface(struct dchid_iface *iface) /* Only multi-touch has firmware */ if (fw && size) { - if (dchid_j700_power_v2(iface)) - iface->startup_failed = true; - /* Send firmware to the device */ dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); ret = dchid_send_firmware(iface, fw, size); @@ -688,14 +627,7 @@ static int dchid_start_interface(struct dchid_iface *iface) dev_info(iface->dchid->dev, "Powering on %s with power method %u\n", iface->name, iface->power_method); - if (dchid_j700_power_v2(iface)) { - ret = dchid_j700_set_power(iface, DCHID_POWER_STATE_OFF); - if (ret < 0) - goto err; - ret = dchid_j700_set_power(iface, DCHID_POWER_STATE_ON); - if (ret < 0) - goto err; - } else if (power_method_2) { + if (power_method_2) { if (inherited_power_method_2) { ret = dchid_inherited_pm2_transition(iface, DCHID_POWER_STATE_OFF); if (ret < 0) @@ -772,7 +704,6 @@ static int dchid_open(struct hid_device *hdev) } iface->open = true; - iface->startup_failed = false; return 0; } diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index 080d083ed9bd2..0ddd30c01d65f 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -113,7 +113,11 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) helper->asc_base + APPLE_ASC_CPU_CONTROL); /* Works for both wake and boot */ - ret = apple_rtkit_wake(helper->rtk); + /* Match the measured J813 MTP handoff ordering used by m1n1/Windows. */ + if (of_device_is_compatible(dev->of_node, "apple,t8142-mtp")) + ret = apple_rtkit_wake_early_ap(helper->rtk); + else + ret = apple_rtkit_wake(helper->rtk); if (ret != 0) return dev_err_probe(dev, ret, "Failed to wake up coprocessor"); From a3087120b251c1467b128806641e8a8900796fdf Mon Sep 17 00:00:00 2001 From: Acelogic Date: Sun, 13 Sep 2026 00:55:37 -0400 Subject: [PATCH 0179/1579] HID: Allow retry after touch firmware start timeout Adapted path-only import from linux-aurora j813 commit 58fc2ccc1e192c1ded93d5eb8a6f90318233c10c. Original author retained; unrelated GPU/display changes omitted. --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index ad4b8ff4d381c..469ff2088054b 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -699,6 +699,8 @@ static int dchid_open(struct hid_device *hdev) if (!wait_for_completion_timeout(&iface->ready, msecs_to_jiffies(START_TIMEOUT_MS))) { dev_err(iface->dchid->dev, "iface %s start timed out\n", iface->name); + /* A failed sensor boot must not poison every later open. */ + iface->starting = false; return -ETIMEDOUT; } } From 153b1046d6b7c92fc1391f7c44ba42049de30328 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Tue, 22 Sep 2026 15:26:08 -0400 Subject: [PATCH 0180/1579] HID: Retain M5 STM identity and early AP wake Adapted path-only import from linux-aurora j813 commit 78828af3d92757f15a2fae3eeac3ed1e31d17970. Original author retained; unrelated GPU/display changes omitted. --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 81 ++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 469ff2088054b..ea4c233a3409d 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -156,6 +156,7 @@ struct dchid_iface { uint8_t tx_seq; bool deferred; bool starting; + bool startup_failed; bool open; struct completion ready; @@ -438,6 +439,57 @@ static int dchid_inherited_pm2_transition(struct dchid_iface *iface, u8 state) return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); } +static bool dchid_j700_power_v2(struct dchid_iface *iface) +{ + return of_machine_is_compatible("apple,j700") && + iface->index == 1 && !strcmp(iface->name, "multi-touch"); +} + +static int dchid_j700_set_power(struct dchid_iface *iface, u8 state) +{ + struct { + u8 command; + u8 version; + u8 iface; + u8 state; + u8 did_change; + __le32 status; + } __packed msg = { + .command = DCHID_CMD_RESET_INTERFACE, + .version = 2, + .iface = iface->index, + .state = state, + }; + int ret; + + /* + * J700 MTP_SYS selects PowerMethod=2. Its comm firmware accepts only + * nine-byte v2 reports: will-change, platform function, has-changed. + * The pinned power-sequence uses physical High for 50 ms on power-off + * and Low with no delay on power-on. Use raw levels for this platform + * function, independent of the legacy DT's active-low GPIO flag. + * macsmc-gpio maps it to pKW8(pcIO, 0x10000, 0x130000). + */ + if (state != 0 && state != 2) + return -EINVAL; + + ret = dchid_comm_cmd(iface->dchid, &msg, sizeof(msg)); + if (ret < 0) + return ret; + + ret = gpiod_set_raw_value_cansleep(iface->gpio, state == 0); + if (ret < 0) + return ret; + if (state == 0) + fsleep(50000); + + msg.did_change = 1; + ret = dchid_comm_cmd(iface->dchid, &msg, sizeof(msg)); + dev_info(iface->dchid->dev, "J700 MTP power-v2 state=%u result=%d\n", + state, ret); + return ret; +} + static int dchid_send_firmware(struct dchid_iface *iface, void *firmware, size_t size) { struct { @@ -554,6 +606,9 @@ static int dchid_start_interface(struct dchid_iface *iface) size_t size; int ret; + if (iface->startup_failed) + return -EIO; + if (iface->starting) { dev_warn(iface->dchid->dev, "Interface %s is already starting", iface->name); return -EINPROGRESS; @@ -598,8 +653,19 @@ static int dchid_start_interface(struct dchid_iface *iface) goto err; } + if (dchid_j700_power_v2(iface) && !iface->gpio) { + /* Do not toggle the line before the will-change notification. */ + iface->gpio = devm_gpiod_get_index(iface->dchid->dev, + "apple,afe-reset", 0, GPIOD_ASIS); + if (IS_ERR(iface->gpio)) { + ret = PTR_ERR(iface->gpio); + iface->gpio = NULL; + goto err; + } + } + /* Dynamic requests include ID zero; Power Method 2 uses afe-reset. */ - if (!inherited_power_method_2 && + if (!dchid_j700_power_v2(iface) && !inherited_power_method_2 && (iface->gpio_request_seen || power_method_2)) { ret = dchid_request_gpio(iface, power_method_2 ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW); @@ -616,6 +682,9 @@ static int dchid_start_interface(struct dchid_iface *iface) /* Only multi-touch has firmware */ if (fw && size) { + if (dchid_j700_power_v2(iface)) + iface->startup_failed = true; + /* Send firmware to the device */ dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); ret = dchid_send_firmware(iface, fw, size); @@ -627,7 +696,14 @@ static int dchid_start_interface(struct dchid_iface *iface) dev_info(iface->dchid->dev, "Powering on %s with power method %u\n", iface->name, iface->power_method); - if (power_method_2) { + if (dchid_j700_power_v2(iface)) { + ret = dchid_j700_set_power(iface, DCHID_POWER_STATE_OFF); + if (ret < 0) + goto err; + ret = dchid_j700_set_power(iface, DCHID_POWER_STATE_ON); + if (ret < 0) + goto err; + } else if (power_method_2) { if (inherited_power_method_2) { ret = dchid_inherited_pm2_transition(iface, DCHID_POWER_STATE_OFF); if (ret < 0) @@ -706,6 +782,7 @@ static int dchid_open(struct hid_device *hdev) } iface->open = true; + iface->startup_failed = false; return 0; } From 647727bd619b6287826fea524acd63a321ac6162 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 02:56:39 -0400 Subject: [PATCH 0181/1579] apple: enable J714s MTP power-v2 and native DMA aperture Use the existing firmware-owned Will/Has power sequence on J714s. Accept the native 42-bit DART aperture used by MTP without hiding its high address bits; retain the existing low-address default and reject overlapping vm-force. Validated module loading and host grants in QEMU, then SSD Linux on hardware: firmware reports Touch MT ready and the user confirms typing, pointer movement and clicking. --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 6 ++++-- drivers/iommu/apple-dart-x1n1.c | 8 +++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index ea4c233a3409d..fc4bf48a523c2 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -271,12 +271,14 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) iface->power_method == DCHID_POWER_METHOD_2; } /* - * J813's MTP firmware owns the touch reset line. Its Windows bring-up + * M5 MTP uses the v2 touch power transaction. J813 bring-up * uses the v2 Will/Has power messages, including the 50 ms ON delay, * without a GPIO_REQUEST or an external AFE GPIO operation. */ if (iface->power_method == DCHID_POWER_METHOD_2 && - !strcmp(name, "multi-touch") && of_machine_is_compatible("apple,j813")) + !strcmp(name, "multi-touch") && + (of_machine_is_compatible("apple,j813") || + of_machine_is_compatible("apple,j714s"))) iface->inherited_power_method_2 = true; if (iface->power_method == DCHID_POWER_METHOD_2 && diff --git a/drivers/iommu/apple-dart-x1n1.c b/drivers/iommu/apple-dart-x1n1.c index f4c69563861fb..9e2db8ab3f827 100644 --- a/drivers/iommu/apple-dart-x1n1.c +++ b/drivers/iommu/apple-dart-x1n1.c @@ -89,6 +89,7 @@ struct x1n1_batch { #define DART_X1N1_MAX_STREAMS 256 #define DART_X1N1_APERTURE_END (SZ_4G - 1) +#define DART_X1N1_NATIVE_APERTURE_END GENMASK_ULL(41, 0) /* One batch buffer per DART, sized to hold a full unmap/map of the aperture in * per-block chunks; kept in the linear map so its guest PA is stable. A page @@ -1089,7 +1090,7 @@ static int apple_dart_x1n1_probe(struct platform_device *pdev) if (of_property_read_u64_array(dev->of_node, "apple,dma-range", range, 2) || !range[1] || (range[0] | range[1]) & (DART_X1N1_PAGE_SIZE - 1) || check_add_overflow(range[0], range[1] - 1, &end) || - end > DART_X1N1_APERTURE_END) + end > DART_X1N1_NATIVE_APERTURE_END) return dev_err_probe(dev, -EINVAL, "bad DMA range\n"); dart->dma_min = range[0]; dart->dma_max = end; @@ -1099,6 +1100,11 @@ static int apple_dart_x1n1_probe(struct platform_device *pdev) &dart->iova_force) || (dart->iova_force && dart->iova_force != BIT_ULL(40))) return dev_err_probe(dev, -EINVAL, "unsupported IOVA force mask\n"); + /* A native DVA aperture already contains its high address bits. + * Hidden force bits are only valid for the low-address device view. + */ + if (dart->iova_force && dart->dma_max >= dart->iova_force) + return dev_err_probe(dev, -EINVAL, "IOVA force overlaps DMA aperture\n"); } dart->num_streams = DART_X1N1_MAX_STREAMS; From b0ab0cf786743f160fb05e04f5b43cf232d76d06 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 02:56:39 -0400 Subject: [PATCH 0182/1579] wifi: apple: support NetworkManager connection lifecycle Configure interface MAC addresses through the firmware stop/address/start sequence, complete scans synchronously on interface stop, select PSK from supported AKM offers while preserving RSN suffixes, and report successful firmware-offloaded port authorization. Actual-source sanitizer tests and focused QEMU boot passed. NetworkManager DHCP, DNS, SSH and startup after reboot passed on J714s with native27 firmware. --- .../wireless/apple/centauri/centauri_wifi.c | 206 ++++++++++++++++-- 1 file changed, 190 insertions(+), 16 deletions(-) diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index 4312616bed617..92cfe3959be0c 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -187,6 +187,9 @@ static struct ieee80211_rate rates[] = { { .bitrate = 60 }, { .bitrate = 90 }, { .bitrate = 120 }, { .bitrate = 180 }, { .bitrate = 240 }, { .bitrate = 360 }, { .bitrate = 480 }, { .bitrate = 540 }, }; +/* Keep userspace AKM selection consistent with wifi_connect(). */ +static const u32 akm_suites[] = { WLAN_AKM_SUITE_PSK }; + static const u32 cipher_suites[] = { WLAN_CIPHER_SUITE_CCMP, WLAN_CIPHER_SUITE_AES_CMAC, }; @@ -812,11 +815,11 @@ static int command(struct centauri_wifi *wifi, u32 id, const void *payload, size return -ETIMEDOUT; } -static int radio_start(struct centauri_wifi *wifi) +static int station_configure(struct centauri_wifi *wifi, const u8 *address) { u8 start[5 + 4 + 3 * ETH_ALEN] = { 0, 0x0b, 0, 0, 0 }; size_t start_size = 5; - unsigned int ring, slot; + unsigned int slot; int ret; /* Stock 25G76 startInterface (0x1808e26ec) supplies TLV 385, @@ -828,9 +831,20 @@ static int radio_start(struct centauri_wifi *wifi) put_unaligned_le16(385, start + 5); put_unaligned_le16(3 * ETH_ALEN, start + 7); for (slot = 0; slot < 3; slot++) - ether_addr_copy(start + 9 + slot * ETH_ALEN, wifi->netdev->dev_addr); + ether_addr_copy(start + 9 + slot * ETH_ALEN, address); start_size = sizeof(start); } + ret = command(wifi, 1, address, ETH_ALEN); + if (!ret) + ret = command(wifi, 2, start, start_size); + return ret; +} + +static int radio_start(struct centauri_wifi *wifi) +{ + unsigned int ring, slot; + int ret; + for (ring = 1; ring <= 4; ring++) { ret = open_pair(wifi, ring); if (ret) @@ -848,9 +862,7 @@ static int radio_start(struct centauri_wifi *wifi) msleep(400); ret = command(wifi, 0, NULL, 0); if (!ret) - ret = command(wifi, 1, wifi->netdev->dev_addr, ETH_ALEN); - if (!ret) - ret = command(wifi, 2, start, start_size); + ret = station_configure(wifi, wifi->netdev->dev_addr); if (ret) return ret; msleep(1000); @@ -1106,6 +1118,65 @@ static int rsn_caps_offset(const u8 *body, size_t size) return offset + 2 <= size ? offset : -EINVAL; } +/* Return the AKM count field only when this RSN offer includes PSK. + * Suite selectors are network byte order; list counts are little endian. + */ +static int rsn_psk_offset(const u8 *body, size_t size) +{ + size_t offset, end; + u16 count; + unsigned int i; + + if (size < 8 || get_unaligned_le16(body) != 1) + return -EINVAL; + count = get_unaligned_le16(body + 6); + if (!count) + return -EINVAL; + offset = 8 + 4 * count; + if (offset + 2 > size) + return -EINVAL; + count = get_unaligned_le16(body + offset); + end = offset + 2 + 4 * count; + if (!count || end + 2 > size) + return -EINVAL; + for (i = 0; i < count; i++) + if (get_unaligned_be32(body + offset + 2 + 4 * i) == WLAN_AKM_SUITE_PSK) + return offset; + return -EOPNOTSUPP; +} + +/* cfg80211 supplies acceptable AKMs, not necessarily one selected AKM. + * Select the implemented PSK tuple only if both the list and RSN offer it; + * emit exactly that tuple to firmware, retaining the optional RSN tail. + */ +static int select_psk_rsn(const struct cfg80211_crypto_settings *crypto, + const u8 *body, size_t size, u8 *selected) +{ + size_t tail; + unsigned int i; + int offset; + + for (i = 0; i < crypto->n_akm_suites; i++) + if (crypto->akm_suites[i] == WLAN_AKM_SUITE_PSK) + break; + if (i == crypto->n_akm_suites) + return -EOPNOTSUPP; + offset = rsn_psk_offset(body, size); + if (offset < 0) + return offset; + /* Keep the RSN ciphers consistent with the accepted fixed tuple. */ + if (get_unaligned_be32(body + 2) != WLAN_CIPHER_SUITE_CCMP || + get_unaligned_le16(body + 6) != 1 || + get_unaligned_be32(body + 8) != WLAN_CIPHER_SUITE_CCMP) + return -EOPNOTSUPP; + tail = offset + 2 + 4 * get_unaligned_le16(body + offset); + memcpy(selected, body, offset); + put_unaligned_le16(1, selected + offset); + put_unaligned_be32(WLAN_AKM_SUITE_PSK, selected + offset + 2); + memcpy(selected + offset + 6, body + tail, size - tail); + return offset + 6 + size - tail; +} + static void append_tlv(u8 *payload, size_t *size, u16 tag, const void *data, u16 length) { put_unaligned_le16(tag, payload + *size); @@ -1140,18 +1211,22 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, const u8 *rsn, *ap_rsn; u8 rsn_body[255], ssid[35], key[66], zero[20] = {}, pmf; u8 *payload = wifi->connect_payload; - size_t size = 130, rsn_size; + size_t size = 130, rsn_size, rsn_prefix, rsn_tail; u16 caps = 0; int ret = 0, offset; if (!params->ssid_len || params->ssid_len > 32 || !params->crypto.psk || - params->crypto.n_akm_suites != 1 || params->crypto.akm_suites[0] != WLAN_AKM_SUITE_PSK || params->crypto.n_ciphers_pairwise != 1 || params->crypto.ciphers_pairwise[0] != WLAN_CIPHER_SUITE_CCMP || params->crypto.cipher_group != WLAN_CIPHER_SUITE_CCMP || params->ie_len > sizeof(wifi->request_ie)) return -EOPNOTSUPP; rsn = cfg80211_find_ie(WLAN_EID_RSN, params->ie, params->ie_len); - if (!rsn || rsn_caps_offset(rsn + 2, rsn[1]) < 0) + if (!rsn) return -EINVAL; + ret = select_psk_rsn(¶ms->crypto, rsn + 2, rsn[1], rsn_body); + if (ret < 0) + return ret; + rsn_size = ret; + ret = 0; mutex_lock(&wifi->lock); if (READ_ONCE(wifi->stopping)) { ret = -ENODEV; @@ -1177,14 +1252,21 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, rcu_read_lock(); ies = rcu_dereference(bss->ies); ap_rsn = ies ? cfg80211_find_ie(WLAN_EID_RSN, ies->data, ies->len) : NULL; - if (ap_rsn) { - offset = rsn_caps_offset(ap_rsn + 2, ap_rsn[1]); - if (offset >= 0) + if (!ap_rsn) { + ret = -EINVAL; + } else { + ret = rsn_psk_offset(ap_rsn + 2, ap_rsn[1]); + if (ret >= 0) { + offset = rsn_caps_offset(ap_rsn + 2, ap_rsn[1]); caps = get_unaligned_le16(ap_rsn + 2 + offset); + ret = 0; + } } rcu_read_unlock(); - rsn_size = rsn[1]; - memcpy(rsn_body, rsn + 2, rsn_size); + if (ret) { + cfg80211_put_bss(wiphy, bss); + goto out; + } offset = rsn_caps_offset(rsn_body, rsn_size); put_unaligned_le16(caps, rsn_body + offset); pmf = !!(caps & BIT(7)); @@ -1228,8 +1310,16 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, append_tlv(payload, &size, 376, mld_defaults, of_machine_is_compatible("apple,j714s") ? sizeof(mld_defaults) : 12); wifi->connect_size = size; - memcpy(wifi->request_ie, params->ie, params->ie_len); - wifi->request_ie_size = params->ie_len; + /* Report the actual selected RSN tuple, preserving other request IEs. */ + rsn_prefix = rsn - params->ie; + rsn_tail = rsn_prefix + 2 + rsn[1]; + memcpy(wifi->request_ie, params->ie, rsn_prefix); + wifi->request_ie[rsn_prefix] = WLAN_EID_RSN; + wifi->request_ie[rsn_prefix + 1] = rsn_size; + memcpy(wifi->request_ie + rsn_prefix + 2, rsn_body, rsn_size); + memcpy(wifi->request_ie + rsn_prefix + 2 + rsn_size, + params->ie + rsn_tail, params->ie_len - rsn_tail); + wifi->request_ie_size = params->ie_len - rsn[1] + rsn_size; WRITE_ONCE(wifi->connect_abort, false); wifi->connecting = true; wifi->connect_status = 0; @@ -1346,6 +1436,12 @@ static void connect_worker(struct work_struct *work) cfg80211_connect_result(wifi->netdev, wifi->bssid, wifi->request_ie, wifi->request_ie_size, NULL, 0, status, GFP_KERNEL); if (status == WLAN_STATUS_SUCCESS && !READ_ONCE(wifi->stopping)) { + /* The firmware completion covers the offloaded four-way handshake, + * not just 802.11 association. Tell userspace the controlled port + * is authorized after queuing the connection event above. + */ + cfg80211_port_authorized(wifi->netdev, wifi->bssid, NULL, 0, + GFP_KERNEL); netif_carrier_on(wifi->netdev); if (wifi->data_probe_done & BIT(5)) netif_wake_queue(wifi->netdev); @@ -1546,6 +1642,13 @@ static int net_stop(struct net_device *netdev) { struct centauri_wifi *wifi = *(struct centauri_wifi **)netdev_priv(netdev); + /* cfg80211 may skip .disconnect when an unassociated interface is + * scanning. NETDEV_DOWN requires scan_done to have notified before + * ndo_stop returns, even if an asynchronous abort is already pending. + * Join the worker without wifi->lock held; cfg80211_scan_done itself + * queues wiphy work and does not acquire the wiphy mutex. + */ + cancel_scan(wifi); netif_stop_queue(netdev); skb_queue_purge(&wifi->data_tx_queue); return 0; @@ -1572,11 +1675,80 @@ static netdev_tx_t net_transmit(struct sk_buff *skb, struct net_device *netdev) return NETDEV_TX_OK; } +/* Native27 ACIStationAdapter::reconfigureInterface (0x1809c12c8) + * stops command 3 before configuring command 1 and starting command 2. + * Command/data ring ownership survives this station-interface operation. + * The caller holds wifi->lock; do not publish a new netdev address until + * both firmware commands acknowledge it. + */ +static int station_reconfigure(struct centauri_wifi *wifi, const u8 *address) +{ + const u8 stop[5] = {}; + int ret; + + wifi->configured = false; + ret = command(wifi, 3, stop, sizeof(stop)); + if (!ret) + ret = station_configure(wifi, address); + if (!ret) + wifi->configured = true; + return ret; +} + +static int net_set_mac_address(struct net_device *netdev, void *data) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)netdev_priv(netdev); + struct sockaddr *address = data; + int ret, restore; + + ret = eth_prepare_mac_addr_change(netdev, data); + if (ret) + return ret; + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->stopping)) { + ret = -ENODEV; + goto out; + } + if (!wifi->configured) { + ret = -ENETDOWN; + goto out; + } + if (wifi->connecting || wifi->connected || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + if (ether_addr_equal(netdev->dev_addr, address->sa_data)) + goto out; + /* Only the captured current27 target has this reconfigure contract. */ + if (!of_machine_is_compatible("apple,j714s")) { + ret = -EOPNOTSUPP; + goto out; + } + ret = station_reconfigure(wifi, (const u8 *)address->sa_data); + if (!ret) { + eth_commit_mac_addr_change(netdev, data); + } else { + /* One bounded recovery restores the prior device/host identity. + * If it also fails, configured remains false: do not allow open + * or connect against an unconfirmed firmware MAC. + */ + restore = station_reconfigure(wifi, netdev->dev_addr); + if (restore) + dev_err(&wifi->control->pdev->dev, + "Wi-Fi MAC restore failed: %d (change: %d)\n", + restore, ret); + } +out: + mutex_unlock(&wifi->lock); + return ret; +} + static const struct net_device_ops net_ops = { .ndo_open = net_open, .ndo_stop = net_stop, .ndo_start_xmit = net_transmit, .ndo_validate_addr = eth_validate_addr, + .ndo_set_mac_address = net_set_mac_address, }; static ssize_t passphrase_write(struct file *file, const char __user *input, @@ -2577,6 +2749,8 @@ int centauri_wifi_register(struct centauri *c) wifi->wiphy->reg_notifier = wifi_regulatory; wifi->wiphy->cipher_suites = cipher_suites; wifi->wiphy->n_cipher_suites = ARRAY_SIZE(cipher_suites); + wifi->wiphy->akm_suites = akm_suites; + wifi->wiphy->n_akm_suites = ARRAY_SIZE(akm_suites); wiphy_ext_feature_set(wifi->wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); for (i = 0; i < 13; i++) { wifi->channels[i].band = NL80211_BAND_2GHZ; From f5652bd76131637a3c7efaf40bde4777a66ee062 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 04:37:49 -0400 Subject: [PATCH 0183/1579] wifi: centauri: wait for asynchronous disconnect completion Reactivating a NetworkManager connection can stop the station while the firmware still owns a connected link, asserting in Cm_StopInterface and leaving Wi-Fi unable to scan until reboot. Preserve bit 30 from the wire reply separately from the normalized command ID. An ACK with that bit set is not completed teardown: wait for event 4, with the native five-second deadline, before returning from disconnect. Keep a timed-out teardown pending so scan, connect and MAC reconfiguration cannot overtake it. Use the macOS 27 disconnect TLV value independently of the 802.11 reason and report loss of the command transport. Validated with source-extracted ASan/UBSan lifecycle tests, including wire reply decoding and a failing negative control for the old flag-stripping assignment; QEMU Linux boot; and J714s/T6050 macOS 27 firmware hardware. Six same-profile reactivation/scan cycles passed across RAM and U-Boot SSD boots, plus explicit disconnect/reconnect and radio off/on with network traffic. Existing data rings and batching are unchanged. --- .../wireless/apple/centauri/centauri_wifi.c | 64 ++++++++++++++++--- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index 92cfe3959be0c..e10d514001c6e 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -111,6 +111,7 @@ struct centauri_wifi { bool scan_abort, connect_abort; u8 scan_sequence; bool connecting, connected, connect_event, disconnect_event; + bool disconnect_pending; bool associated, pmk_seen, pmk_matches; u8 expected_pmk[32]; u16 connect_status; @@ -593,7 +594,7 @@ static void drain(struct centauri_wifi *wifi) while (budget--) { u16 slot = wifi->rx_next[ring], size, status, sequence; - u32 command; + u32 command, wire_command; u8 *descriptor = wifi->a.window + TR_BASE + (ring - 1) * RING_SPAN + slot * 16; u8 *packet = wifi->a.window + rx_buffer[ring] + slot * BUFFER_SIZE; @@ -603,7 +604,8 @@ static void drain(struct centauri_wifi *wifi) size = get_unaligned_le16(packet + 0x10); status = get_unaligned_le16(packet + 0xe); sequence = get_unaligned_le16(packet + 0x12); - command = get_unaligned_le32(packet + 0x14) & ~BIT(30); + wire_command = get_unaligned_le32(packet + 0x14); + command = wire_command & ~BIT(30); wifi->rx_seen[ring]++; wifi->rx_next[ring] = (slot + 1) % ring_size[ring]; if (size < 24 || size > BUFFER_SIZE) @@ -646,6 +648,8 @@ static void drain(struct centauri_wifi *wifi) get_unaligned_le32(packet + 0x34)); if (command == 0x80000003 || command == 0x80000004) wifi->disconnect_event = true; + if (command == 0x80000004) + wifi->disconnect_pending = false; if (wifi->connecting && command != 0x80000000 && command != 0x80000001 && command != 0x80000030 && command != 0x80000005 && command != 0x80000050) @@ -656,7 +660,8 @@ static void drain(struct centauri_wifi *wifi) wifi->reply_valid = true; wifi->reply_sequence = sequence; wifi->reply_status = status; - wifi->reply_command = command; + /* Preserve the asynchronous ACK flag for completion waits. */ + wifi->reply_command = wire_command; if (command == 10) { wifi->country_valid = !status && size >= 27; if (wifi->country_valid) @@ -770,6 +775,7 @@ static void drain(struct centauri_wifi *wifi) static int command(struct centauri_wifi *wifi, u32 id, const void *payload, size_t size) { u16 slot = wifi->command_head, sequence = ++wifi->sequence; + u32 stage; u8 *packet, *descriptor; unsigned long deadline = jiffies + msecs_to_jiffies(1500); @@ -796,8 +802,13 @@ static int command(struct centauri_wifi *wifi, u32 id, const void *payload, size writel(1, wifi->a.bar + 0x1004); do { drain(wifi); - if (readl(wifi->a.bar + 8) != 2) + stage = readl(wifi->a.bar + 8); + if (stage != 2) { + dev_err(&wifi->control->pdev->dev, + "Wi-Fi command %#x lost transport: stage=%#x sequence=%u rx=%u\n", + id, stage, sequence, wifi->rx_seen[2]); return -EIO; + } /* Bit 30 is more-data-pending, not part of the command identity. * Scan/connect replies set it before their later completion event. */ @@ -1047,7 +1058,7 @@ static int wifi_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) ret = -ENODEV; else if (!wifi->configured) ret = -ENETDOWN; - else if (wifi->scan_request || wifi->connecting) + else if (wifi->scan_request || wifi->connecting || wifi->disconnect_pending) ret = -EBUSY; else { WRITE_ONCE(wifi->scan_abort, false); @@ -1232,7 +1243,8 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, ret = -ENODEV; goto out; } - if (!wifi->configured || wifi->connecting || wifi->connected || wifi->scan_request) { + if (!wifi->configured || wifi->connecting || wifi->connected || + wifi->scan_request || wifi->disconnect_pending) { ret = -EBUSY; goto out; } @@ -1485,6 +1497,28 @@ static void link_worker(struct work_struct *work) mutex_unlock(&wifi->lock); } +/* Native disconnect replies with bit30 set are acknowledgements, not + * completed teardown. macOS disconnectInProgress waits for event4 before + * allowing interface stop/reconfiguration, with a five-second deadline. + * Caller owns wifi->lock; drain processes the completion without reacquiring it. + */ +static int wait_disconnect(struct centauri_wifi *wifi) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(5000); + + while (wifi->disconnect_pending) { + drain(wifi); + if (READ_ONCE(wifi->stopping) || readl(wifi->a.bar + 8) != 2) + return -EIO; + if (!wifi->disconnect_pending) + break; + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; + usleep_range(500, 1000); + } + return 0; +} + static int wifi_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 reason) { struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); @@ -1504,9 +1538,21 @@ static int wifi_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 r put_unaligned_le16(reason, payload); put_unaligned_le16(328, payload + 8); put_unaligned_le16(4, payload + 10); + /* Native27 AppleWLAN80211Connection::disconnect passes 10 to + * disconnectCommon as TLV 328, separately from the 802.11 reason. + */ + put_unaligned_le32(10, payload + 12); + wifi->disconnect_pending = !READ_ONCE(wifi->stopping); ret = READ_ONCE(wifi->stopping) ? 0 : command(wifi, 131073, payload, sizeof(payload)); reply_status = wifi->reply_status; + if (!ret && wifi->disconnect_pending && (wifi->reply_command & BIT(30))) + ret = wait_disconnect(wifi); + else if (ret != -EIO) + wifi->disconnect_pending = false; + /* On timeout the pending state stays set until event4 arrives, so a + * later scan/connect/MAC change cannot overtake firmware teardown. + */ connected = wifi->connected; wifi->connected = false; netif_stop_queue(netdev); @@ -1713,7 +1759,8 @@ static int net_set_mac_address(struct net_device *netdev, void *data) ret = -ENETDOWN; goto out; } - if (wifi->connecting || wifi->connected || wifi->scan_request) { + if (wifi->connecting || wifi->connected || wifi->scan_request || + wifi->disconnect_pending) { ret = -EBUSY; goto out; } @@ -1816,7 +1863,8 @@ static ssize_t fwlog_start_write(struct file *file, const char __user *input, ret = -ENODEV; goto out; } - if (!wifi->configured || wifi->connecting || wifi->connected || wifi->scan_request) { + if (!wifi->configured || wifi->connecting || wifi->connected || + wifi->scan_request || wifi->disconnect_pending) { ret = -EBUSY; goto out; } From 70b3e82de861ba3d406042fb967369b93b7cbf0f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 05:43:34 -0400 Subject: [PATCH 0184/1579] iommu: apple-dart-x1n1: preserve firmware-owned stream mappings Negotiate retained streams explicitly and reserve their occupied IOVA ranges instead of disabling them and replacing their active roots. Keep software-owned mappings separate for normal batching, rollback and domain teardown; translate inherited mappings only when the host grants their backing in the guest address space. Honor the native 26A428 strict 42-bit exclusive-end check by reserving the last 16 KiB IOVA granule. This does not reserve physical RAM or reduce DMA batching or concurrency. Validation: Asahi-profile Image build and focused QEMU boot; coherent 64 KiB map/translate/unmap on J714s DCP19/SID23 and display18/SIDs0,16, including module reloads. Actual display-engine DMA is not tested here. --- .../driver-api/apple-dart-x1n1-retained.rst | 34 ++ drivers/iommu/apple-dart-x1n1.c | 307 ++++++++++++++++-- 2 files changed, 320 insertions(+), 21 deletions(-) create mode 100644 Documentation/driver-api/apple-dart-x1n1-retained.rst diff --git a/Documentation/driver-api/apple-dart-x1n1-retained.rst b/Documentation/driver-api/apple-dart-x1n1-retained.rst new file mode 100644 index 0000000000000..1143304786075 --- /dev/null +++ b/Documentation/driver-api/apple-dart-x1n1-retained.rst @@ -0,0 +1,34 @@ +Firmware-owned streams in apple-dart-x1n1 +======================================= + +The optional ``apple,x1n1-retained-sids`` property names streams whose DART +roots and mappings remain owned by firmware. The host must explicitly adopt +these streams before starting Linux and advertise sptm2mmio feature bit 4. +Probe fails if the retained aperture does not match the configured aperture. +Ordinary streams keep their existing initialization and batching behavior. + +Host query operations 9 and 10 return the root level and occupied DVA runs. +Linux reserves those runs, skips stream disable and root replacement, and +only removes mappings which it installed itself. A domain switch removes +the old domain's software-owned mappings before replaying the new domain; +its rollback attempts to restore the old mapping set. Firmware mappings +are never inserted in the Linux-owned mappings xarray. + +An inherited run whose guest IPA is ``U64_MAX`` remains reserved but cannot +be translated for CPU access. A host physical address alone is insufficient: +the host must grant its backing through stage 2 before returning a guest IPA. +With multiple streams, inherited translation succeeds only when every stream +resolves the address identically. + +The native 26A428 SPTM MAP implementation validates both the starting address +and the exclusive end as 42-bit addresses. A range ending exactly at 4 TiB +therefore fails. The last 16 KiB IOVA granule is reserved when the aperture +ends at that boundary, and map_pages rejects an invalid exclusive end. +This consumes virtual DMA address space, not physical RAM. It does not reduce +map batching, device queue depth or DMA concurrency. + +J714s hardware validation covered 64 KiB coherent map/translate/unmap on +DCP19/SID23 and display18/SIDs0,16, repeated across three module loads while +the inherited framebuffer, GNOME and Wi-Fi remained active. The test did +not ask the display engine to DMA from the new buffers; actual DCP modesetting +and display sleep remain separate integration work. diff --git a/drivers/iommu/apple-dart-x1n1.c b/drivers/iommu/apple-dart-x1n1.c index 9e2db8ab3f827..077d582a0726f 100644 --- a/drivers/iommu/apple-dart-x1n1.c +++ b/drivers/iommu/apple-dart-x1n1.c @@ -56,6 +56,7 @@ #define X1N1_MAGIC 0x78316e31UL /* "x1n1" */ #define X1N1_FEAT_DART BIT(0) +#define X1N1_FEAT_DART_RETAINED BIT(4) /* DART unit ops. */ #define X1N1_DART_INIT 1 /* () INIT + POWERUP */ @@ -65,6 +66,9 @@ #define X1N1_DART_ENABLE 5 /* (sid) */ #define X1N1_DART_DISABLE 6 /* (sid) */ +#define X1N1_DART_RETAINED_INFO 9 +#define X1N1_DART_RETAINED_RUN 10 + struct x1n1_op { u32 op; u32 flags; @@ -97,6 +101,16 @@ struct x1n1_batch { #define X1N1_BATCH_MAX_OPS ((PAGE_SIZE - sizeof(struct x1n1_batch)) / \ sizeof(struct x1n1_op)) +static const struct iommu_ops apple_dart_x1n1_ops; + +struct x1n1_retained_run { + u64 dva, ipa, size; +}; +struct x1n1_retained_sid { + unsigned int root_level, count; + struct x1n1_retained_run *runs; +}; + struct apple_dart_x1n1 { struct device *dev; struct iommu_device iommu; @@ -106,6 +120,7 @@ struct apple_dart_x1n1 { u64 iova_force; /* Native vm-force, hidden from DMA API/device addresses. */ u64 dma_min, dma_max; bool inited; + struct x1n1_retained_sid *retained[DART_X1N1_MAX_STREAMS]; struct mutex init_lock; /* serialises one-time init */ /* Protected by apple_dart_x1n1_groups_lock; group-owned lifetime. */ struct iommu_group *sid2group[DART_X1N1_MAX_STREAMS]; @@ -129,6 +144,7 @@ struct apple_dart_x1n1_domain { * block = dva >> 25. Installing a level-2 table twice is fatal in SPTM, * so each 32 MiB window is attached at most once. */ struct xarray l2_attached; + bool has_retained; struct xarray mappings; /* tagged (guest-PFN, protection), indexed by IOVA page */ }; @@ -194,6 +210,138 @@ static int x1n1_ring(struct apple_dart_x1n1 *dart, u32 db_off) return dart->batch->status ?: (dart->batch->done == dart->batch->count ? 0 : -EIO); } +/* Setup-only host queries. Firmware-owned mappings are reserved, never replayed + * through MAP, and never inserted in the Linux-owned mappings xarray. */ +static int x1n1_load_retained(struct apple_dart_x1n1 *dart, u32 features) +{ + struct device_node *np = dart->dev->of_node; + int count = of_property_count_u32_elems(np, "apple,x1n1-retained-sids"); + int ret; + + if (!of_find_property(np, "apple,x1n1-retained-sids", NULL)) + return 0; + if (count < 1 || count > dart->num_streams || dart->iova_force || + !(features & X1N1_FEAT_DART_RETAINED)) + return -EINVAL; + for (int i = 0; i < count; i++) { + struct x1n1_retained_sid *r; + u32 sid; + u64 previous = 0; + struct x1n1_op *op = &dart->batch->op[0]; + + if (of_property_read_u32_index(np, "apple,x1n1-retained-sids", + i, &sid) || + sid >= dart->num_streams || dart->retained[sid]) + return -EINVAL; + dart->batch->count = 1; + *op = (struct x1n1_op){ .op = X1N1_DART_RETAINED_INFO, + .arg = { sid } }; + ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + if (ret) + return ret; + if (op->arg[1] > 1 || op->arg[2] > 131072 || op->arg[5] != 1 || + op->arg[3] != dart->dma_min || + op->arg[4] != dart->dma_max - dart->dma_min + 1) + return -EINVAL; + r = devm_kzalloc(dart->dev, sizeof(*r), GFP_KERNEL); + if (!r) + return -ENOMEM; + r->root_level = op->arg[1]; + r->count = op->arg[2]; + r->runs = devm_kcalloc(dart->dev, r->count, sizeof(*r->runs), + GFP_KERNEL); + if (r->count && !r->runs) + return -ENOMEM; + for (u64 j = 0; j < r->count; j++) { + struct x1n1_retained_run *run = &r->runs[j]; + + *op = (struct x1n1_op){ .op = X1N1_DART_RETAINED_RUN, + .arg = { sid, j } }; + ret = x1n1_ring(dart, + X1N1_MMIO_DART_DB + 8 * dart->dart_id); + if (ret) + return ret; + run->dva = op->arg[2]; + run->ipa = op->arg[3]; + run->size = op->arg[4]; + if (!run->size || + (run->dva | run->size) & + (DART_X1N1_PAGE_SIZE - 1) || + run->dva < dart->dma_min || + run->dva > dart->dma_max || + run->size - 1 > dart->dma_max - run->dva || + (j && run->dva < previous) || + (run->ipa != U64_MAX && + ((run->ipa & (DART_X1N1_PAGE_SIZE - 1)) || + run->size > U64_MAX - run->ipa))) + return -EINVAL; + previous = run->dva + run->size; + } + dart->retained[sid] = r; + } + return 0; +} + +static const struct x1n1_retained_run * +x1n1_retained_lookup(const struct x1n1_retained_sid *r, u64 dva, u64 size) +{ + unsigned int low = 0, high = r->count; + + while (low < high) { + unsigned int mid = low + (high - low) / 2; + const struct x1n1_retained_run *run = &r->runs[mid]; + + if (run->dva + run->size <= dva) + low = mid + 1; + else + high = mid; + } + return low < r->count && r->runs[low].dva < dva + size ? &r->runs[low] : + NULL; +} + +static bool x1n1_retained_overlap(struct apple_dart_x1n1_domain *dom, u64 dva, + u64 size) +{ + unsigned int sid; + + if (!dom->has_retained) + return false; + for_each_set_bit(sid, dom->sids, dom->dart->num_streams) { + struct x1n1_retained_sid *r = dom->dart->retained[sid]; + + if (r && x1n1_retained_lookup(r, dva, size)) + return true; + } + return false; +} + +static phys_addr_t x1n1_retained_translate(struct apple_dart_x1n1_domain *dom, + u64 dva) +{ + unsigned int sid; + u64 result = 0; + + if (!dom->has_retained) + return 0; + /* One domain may serve several streams: an inherited mapping is only + * unambiguous when every stream resolves it to the same guest address. */ + for_each_set_bit(sid, dom->sids, dom->dart->num_streams) { + struct x1n1_retained_sid *r = dom->dart->retained[sid]; + const struct x1n1_retained_run *run = + r ? x1n1_retained_lookup(r, dva, 1) : NULL; + u64 pa; + + if (!run || dva < run->dva || run->ipa == U64_MAX) + return 0; + pa = run->ipa + dva - run->dva; + if (result && result != pa) + return 0; + result = pa; + } + return result; +} + /* Bring the DART up in SPTM exactly once, before any stream is enabled. */ static int apple_dart_x1n1_ensure_init(struct apple_dart_x1n1 *dart) { @@ -398,7 +546,7 @@ static bool x1n1_next_mapping(struct apple_dart_x1n1_domain *dom, return true; } -static void x1n1_undo_replay(struct apple_dart_x1n1_domain *dom, +static int x1n1_undo_replay(struct apple_dart_x1n1_domain *dom, unsigned int sid, u64 mapped_end) { struct apple_dart_x1n1 *dart = dom->dart; @@ -411,9 +559,12 @@ static void x1n1_undo_replay(struct apple_dart_x1n1_domain *dom, .op = X1N1_DART_UNMAP, .arg = { sid, op.arg[1], min_t(u64, op.arg[3], mapped_end - op.arg[1]) }, }; - if (x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id)) - dev_err(dart->dev, "failed to undo disabled SID %u replay\n", sid); + if (x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id)) { + dev_err(dart->dev, "failed to undo SID %u replay\n", sid); + return -EIO; + } } + return 0; } /* New SID is disabled throughout replay. On failure, keep it disabled and @@ -428,6 +579,8 @@ static int x1n1_replay_sid(struct apple_dart_x1n1_domain *dom, unsigned int sid) int ret; while (x1n1_next_mapping(dom, &cursor, &op, sid)) { + if (dart->retained[sid] && x1n1_retained_lookup(dart->retained[sid], op.arg[1], op.arg[3])) + return -EBUSY; ret = x1n1_prepare_leaf(dom, sid, op.arg[1]); if (ret) return ret; @@ -466,6 +619,8 @@ static int apple_dart_x1n1_attach_dev(struct iommu_domain *domain, unsigned long flags; unsigned int sid; DECLARE_BITMAP(added_sids, DART_X1N1_MAX_STREAMS); + struct apple_dart_x1n1_domain *prior = NULL; + DECLARE_BITMAP(moved_sids, DART_X1N1_MAX_STREAMS); int ret; if (!master) @@ -476,8 +631,20 @@ static int apple_dart_x1n1_attach_dev(struct iommu_domain *domain, if (ret) return ret; - mutex_lock(&dom->lock); + if (old && old != domain && old->owner == &apple_dart_x1n1_ops && + old->ops == apple_dart_x1n1_ops.default_domain_ops) + prior = to_x1n1_domain(old); + if (prior && (unsigned long)prior < (unsigned long)dom) { + mutex_lock(&prior->lock); + mutex_lock_nested(&dom->lock, SINGLE_DEPTH_NESTING); + } else { + mutex_lock(&dom->lock); + if (prior) + mutex_lock_nested(&prior->lock, SINGLE_DEPTH_NESTING); + } if (dom->dart && dom->dart != dart) { + if (prior) + mutex_unlock(&prior->lock); mutex_unlock(&dom->lock); return -EINVAL; } @@ -489,16 +656,36 @@ static int apple_dart_x1n1_attach_dev(struct iommu_domain *domain, spin_lock_irqsave(&dart->batch_lock, flags); ret = 0; bitmap_zero(added_sids, DART_X1N1_MAX_STREAMS); + bitmap_zero(moved_sids, DART_X1N1_MAX_STREAMS); for_each_set_bit(sid, master->sids, dart->num_streams) { if (test_bit(sid, dom->sids)) continue; - dart->batch->count = 2; - dart->batch->op[0] = (struct x1n1_op){ - .op = X1N1_DART_DISABLE, .arg = { sid }, - }; - dart->batch->op[1] = (struct x1n1_op){ - .op = X1N1_DART_ATTACH, .arg = { sid, dart->dma_min, 1 }, - }; + if (dart->retained[sid] && prior && prior->dart == dart && + test_bit(sid, prior->sids)) { + set_bit(sid, moved_sids); + ret = x1n1_undo_replay(prior, sid, U64_MAX); + if (ret) + break; + } + if (dart->retained[sid]) { + dart->batch->count = 1; + dart->batch->op[0] = (struct x1n1_op){ + .op = X1N1_DART_RETAINED_INFO, + .arg = { sid }, + }; + } else { + dart->batch->count = 2; + dart->batch->op[0] = (struct x1n1_op){ + .op = dart->retained[sid] ? + X1N1_DART_RETAINED_INFO : + X1N1_DART_DISABLE, + .arg = { sid }, + }; + dart->batch->op[1] = (struct x1n1_op){ + .op = X1N1_DART_ATTACH, + .arg = { sid, dart->dma_min, 1 }, + }; + } ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); if (ret) break; @@ -507,20 +694,29 @@ static int apple_dart_x1n1_attach_dev(struct iommu_domain *domain, break; dart->batch->count = 1; dart->batch->op[0] = (struct x1n1_op){ - .op = X1N1_DART_ENABLE, .arg = { sid }, + .op = dart->retained[sid] ? X1N1_DART_RETAINED_INFO : + X1N1_DART_ENABLE, + .arg = { sid }, }; ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); if (ret) { /* A failed completion may still have enabled the SID. */ dart->batch->count = 1; dart->batch->op[0] = (struct x1n1_op){ - .op = X1N1_DART_DISABLE, .arg = { sid }, + .op = dart->retained[sid] ? + X1N1_DART_RETAINED_INFO : + X1N1_DART_DISABLE, + .arg = { sid }, }; - if (x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id)) - dev_err(dart->dev, "failed to disable new SID %u after enable error\n", sid); + if (x1n1_ring(dart, + X1N1_MMIO_DART_DB + 8 * dart->dart_id)) + dev_err(dart->dev, + "failed to disable new SID %u after enable error\n", + sid); x1n1_undo_replay(dom, sid, U64_MAX); break; } + dom->has_retained |= !!dart->retained[sid]; set_bit(sid, dom->sids); set_bit(sid, added_sids); } @@ -530,16 +726,34 @@ static int apple_dart_x1n1_attach_dev(struct iommu_domain *domain, for_each_set_bit(sid, added_sids, dart->num_streams) { dart->batch->count = 1; dart->batch->op[0] = (struct x1n1_op){ - .op = X1N1_DART_DISABLE, .arg = { sid }, + .op = dart->retained[sid] ? + X1N1_DART_RETAINED_INFO : + X1N1_DART_DISABLE, + .arg = { sid }, }; - if (x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id)) - dev_err(dart->dev, "failed to disable new SID %u after attachment error\n", sid); + if (x1n1_ring(dart, + X1N1_MMIO_DART_DB + 8 * dart->dart_id)) + dev_err(dart->dev, + "failed to disable new SID %u after attachment error\n", + sid); x1n1_undo_replay(dom, sid, U64_MAX); clear_bit(sid, dom->sids); } } + for_each_set_bit(sid, moved_sids, dart->num_streams) { + if (ret) { + if (x1n1_replay_sid(prior, sid)) + dev_err(dart->dev, + "failed to restore previous domain SID %u\n", + sid); + } else { + clear_bit(sid, prior->sids); + } + } spin_unlock_irqrestore(&dart->batch_lock, flags); + if (prior) + mutex_unlock(&prior->lock); mutex_unlock(&dom->lock); return ret ? -EIO : 0; } @@ -609,8 +823,11 @@ static int apple_dart_x1n1_map_pages(struct iommu_domain *domain, (iova | paddr) & (DART_X1N1_PAGE_SIZE - 1) || prot & ~((1U << X1N1_MAP_PROT_BITS) - 1) || !paddr || iova < dart->dma_min || iova > dart->dma_max || - total - 1 > dart->dma_max - iova) + total - 1 > dart->dma_max - iova || + iova + total > DART_X1N1_NATIVE_APERTURE_END) return -EINVAL; + if (x1n1_retained_overlap(dom, iova, total)) + return -EBUSY; ret = x1n1_reserve_mappings(dom, iova, total, gfp); if (ret) return ret; @@ -738,6 +955,8 @@ static size_t apple_dart_x1n1_unmap_pages(struct iommu_domain *domain, iova < dart->dma_min || iova > dart->dma_max || total - 1 > dart->dma_max - iova) return 0; + if (x1n1_retained_overlap(dom, iova, total)) + return 0; spin_lock_irqsave(&dart->batch_lock, flags); for (size_t pos = 0; pos < total;) { u64 cur = iova + pos; @@ -782,7 +1001,7 @@ static phys_addr_t apple_dart_x1n1_iova_to_phys(struct iommu_domain *domain, unsigned long value = entry ? xa_to_value(entry) : 0; if (!value) - return 0; + return x1n1_retained_translate(dom, iova); return ((value >> X1N1_MAP_PROT_BITS) << DART_X1N1_PAGE_SHIFT) | (iova & (DART_X1N1_PAGE_SIZE - 1)); } @@ -791,6 +1010,15 @@ static void apple_dart_x1n1_domain_free(struct iommu_domain *domain) { struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); + if (dom->has_retained && dom->dart) { + unsigned int sid; + unsigned long flags; + + spin_lock_irqsave(&dom->dart->batch_lock, flags); + for_each_set_bit(sid, dom->sids, dom->dart->num_streams) + x1n1_undo_replay(dom, sid, U64_MAX); + spin_unlock_irqrestore(&dom->dart->batch_lock, flags); + } xa_destroy(&dom->l2_attached); xa_destroy(&dom->mappings); kfree(dom); @@ -986,8 +1214,42 @@ static bool apple_dart_x1n1_capable(struct device *dev, enum iommu_cap cap) * This follows apple-dart.c's hardware MSI reservation (Asahi contributors). */ static void apple_dart_x1n1_get_resv_regions(struct device *dev, - struct list_head *head) + struct list_head *head) { + struct apple_dart_x1n1_master *master = dev_iommu_priv_get(dev); + unsigned int sid; + + if (master) { + /* SPTM converts both start and exclusive end with a strict 42-bit + * address check (26A428: 270d63d8/270d640c). A range ending at + * 4 TiB is rejected, so keep its final granule out of allocation. */ + if (master->dart->dma_max == DART_X1N1_NATIVE_APERTURE_END) { + struct iommu_resv_region *region = + iommu_alloc_resv_region( + DART_X1N1_NATIVE_APERTURE_END + 1 - + DART_X1N1_PAGE_SIZE, + DART_X1N1_PAGE_SIZE, 0, + IOMMU_RESV_RESERVED, GFP_KERNEL); + + if (region) + list_add_tail(®ion->list, head); + } + for_each_set_bit(sid, master->sids, master->dart->num_streams) { + struct x1n1_retained_sid *r = + master->dart->retained[sid]; + if (!r) + continue; + for (unsigned int i = 0; i < r->count; i++) { + struct iommu_resv_region *region = + iommu_alloc_resv_region( + r->runs[i].dva, r->runs[i].size, + 0, IOMMU_RESV_RESERVED, + GFP_KERNEL); + if (region) + list_add_tail(®ion->list, head); + } + } + } if (dev_is_pci(dev)) { struct pci_host_bridge *bridge = pci_find_host_bridge(to_pci_dev(dev)->bus); struct device_node *np = bridge->dev.parent->of_node; @@ -1128,6 +1390,9 @@ static int apple_dart_x1n1_probe(struct platform_device *pdev) if (!dart->batch) return -ENOMEM; dart->batch_pa = virt_to_phys(dart->batch); + ret = x1n1_load_retained(dart, features); + if (ret) + return dev_err_probe(dev, ret, "retained DART discovery failed\n"); platform_set_drvdata(pdev, dart); From 7892c0488df2c21feaffa3c14c4e2a374b21f810 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 28 Sep 2026 21:24:40 +1000 Subject: [PATCH 0185/1579] power: supply: macsmc: Report unavailable battery times as ENODATA The SMC reports 0xffff in B0TE and B0TF when it has no time-to-empty or time-to-full estimate, for example B0TE while on AC and B0TF with a full battery. The driver already recognises this sentinel but reports it as 0 seconds, which userspace cannot tell apart from a real "empty now" or "full now" estimate. This affects every machine the driver supports; it was observed on J700 (T8140), where both keys read 0xffff with the battery full on AC. Return -ENODATA instead, as max17042_battery does for its own 0xffff time sentinel. The power-supply core logs -ENODATA only at debug level and leaves the property out of the uevent. While here, return the SMC read error directly instead of computing intval from vu16 when the read failed. Hardware: candidate 15 and 17 included this change. On J700 with a full battery on AC, status was Full and capacity 100%; both unavailable time properties were absent. No charging or discharging transition was tested on this exact stack. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/power/supply/macsmc-power.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index 3cac04b129ddd..7996b8d9cebd4 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -475,11 +475,19 @@ static int macsmc_battery_get_property(struct power_supply *psy, break; case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW: ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TE), &vu16); - val->intval = vu16 == 0xffff ? 0 : vu16 * 60; + if (ret) + break; + if (vu16 == U16_MAX) + return -ENODATA; + val->intval = vu16 * 60; break; case POWER_SUPPLY_PROP_TIME_TO_FULL_NOW: ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TF), &vu16); - val->intval = vu16 == 0xffff ? 0 : vu16 * 60; + if (ret) + break; + if (vu16 == U16_MAX) + return -ENODATA; + val->intval = vu16 * 60; break; case POWER_SUPPLY_PROP_CAPACITY: ret = apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &vu8); From 0a1ed8ce6d3c04e27f4b8adabbd3a7d0ea5dfa79 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 28 Sep 2026 21:24:42 +1000 Subject: [PATCH 0186/1579] cpuidle: apple: Restrict T8140 to WFI until CPU PD is fixed Register the existing Apple cpuidle driver on T8140, but expose only its WFI state. The CPU PD state uses an unverified power-down and wake sequence on this SoC. Marking that state off by default still leaves a per-CPU sysfs control that can enable it at runtime. A guarded CPU PD opt-in on a J700 made the machine lose wired reachability. Recovery required a VDM reset. The target-side log was unavailable after recovery, so the failing core and instruction are unknown. Keep the CPU PD state out of the driver's registered state count until the sequence and wake path have been fixed and qualified. Other Apple SoCs retain both states. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpuidle/cpuidle-apple.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c index 3d2b804df8a2d..67c7846111cf3 100644 --- a/drivers/cpuidle/cpuidle-apple.c +++ b/drivers/cpuidle/cpuidle-apple.c @@ -124,6 +124,10 @@ static struct cpuidle_driver apple_idle_driver = { static int apple_cpuidle_probe(struct platform_device *pdev) { + /* A T8140 CPU PD opt-in hung J700; expose only WFI. */ + if (of_machine_is_compatible("apple,t8140")) + apple_idle_driver.state_count = STATE_WFI + 1; + return cpuidle_register(&apple_idle_driver, NULL); } @@ -158,7 +162,8 @@ static int __init apple_cpuidle_init(void) of_machine_is_compatible("apple,t6030") || of_machine_is_compatible("apple,t6031") || of_machine_is_compatible("apple,t6032") || - of_machine_is_compatible("apple,t6034"))) + of_machine_is_compatible("apple,t6034") || + of_machine_is_compatible("apple,t8140"))) return 0; pdev = platform_device_register_simple("cpuidle-apple", -1, NULL, 0); From 8ba3c01c426a8efd643289689493835c51b8937d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 28 Sep 2026 21:24:43 +1000 Subject: [PATCH 0187/1579] arm64: dts: apple: t8140: Add measured CPU capacity values Describe the relative per-MHz throughput of the E and P cores for the scheduler. Without these properties, it treats the two CPU types as equal capacity. On a J700, three alternating CoreMark runs per cluster at each cluster's maximum frequency gave median throughput of 12.6983 iterations/s/MHz on P CPU0 at 4044 MHz and 8.8239 on E CPU1 at 2424 MHz. The E/P ratio is 0.694886. Normalizing P to 1024 and rounding E gives 712. The runs reported valid CoreMark results, stable requested frequencies and no throttling; per-MHz spread was 0.03% for P and 0.17% for E. The earlier nearby-OPP microbenchmarks gave E=912, but did not test the same workload as the CoreMark-derived values used for other Apple SoCs. The expected scheduler capacity at maximum frequency is 1024 for P and about 427 for E; runtime readback of this revision remains to be done. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index d9d7b3103115e..7df26d9827c14 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -58,6 +58,7 @@ i-cache-size = <0x20000>; d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; + capacity-dmips-mhz = <712>; performance-domains = <&cpufreq_e>; }; @@ -71,6 +72,7 @@ i-cache-size = <0x20000>; d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; + capacity-dmips-mhz = <712>; performance-domains = <&cpufreq_e>; }; @@ -84,6 +86,7 @@ i-cache-size = <0x20000>; d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; + capacity-dmips-mhz = <712>; performance-domains = <&cpufreq_e>; }; @@ -97,6 +100,7 @@ i-cache-size = <0x20000>; d-cache-size = <0x10000>; operating-points-v2 = <&ecluster_opp>; + capacity-dmips-mhz = <712>; performance-domains = <&cpufreq_e>; }; @@ -110,6 +114,7 @@ i-cache-size = <0x30000>; d-cache-size = <0x20000>; operating-points-v2 = <&pcluster_opp>; + capacity-dmips-mhz = <1024>; performance-domains = <&cpufreq_p>; }; @@ -123,6 +128,7 @@ i-cache-size = <0x30000>; d-cache-size = <0x20000>; operating-points-v2 = <&pcluster_opp>; + capacity-dmips-mhz = <1024>; performance-domains = <&cpufreq_p>; }; From 6b9e4f3a5c0650643686a244de59f193aad45a7a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 16:07:32 -0400 Subject: [PATCH 0188/1579] drm/apple: reject live firmware-owned T8140 DCP --- drivers/gpu/drm/apple/Kconfig | 10 ++++ drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp-lifecycle-test.c | 62 ++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-lifecycle.h | 24 +++++++++ drivers/gpu/drm/apple/dcp.c | 27 +++++++++- 6 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-lifecycle-test.c create mode 100644 drivers/gpu/drm/apple/dcp-lifecycle.h diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index ecf7afd76ceda..5608412840cc6 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -28,3 +28,13 @@ config DRM_APPLE_DEBUG bool "Enable additional driver debugging" depends on DRM_APPLE depends on EXPERT # only for developers + +config DRM_APPLE_KUNIT_TEST + tristate "KUnit tests for the Apple display driver" if !KUNIT_ALL_TESTS + depends on DRM_APPLE && KUNIT + default KUNIT_ALL_TESTS + help + Build focused unit tests for Apple display driver admission and data + handling helpers. The tests cover fail-closed lifecycle decisions for + firmware-owned display coprocessors. They do not access display + hardware. If in doubt, say "N". diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index b22839bdd611a..c9eea42655597 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -17,3 +17,4 @@ appledrm-y += plane.o appledrm-$(CONFIG_TRACING) += trace.o obj-$(CONFIG_DRM_APPLE) += appledrm.o +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp-lifecycle-test.o diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 550ad2b769c88..45891525c46f4 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -144,6 +144,7 @@ struct dcp_panel { struct apple_dcp_hw_data { u32 num_dptx_ports; + bool reject_running_coprocessor; }; /* TODO: move IOMFB members to its own struct */ diff --git a/drivers/gpu/drm/apple/dcp-lifecycle-test.c b/drivers/gpu/drm/apple/dcp-lifecycle-test.c new file mode 100644 index 0000000000000..fb0299671681a --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-lifecycle-test.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include + +#include "dcp-lifecycle.h" + +static void apple_dcp_cold_coprocessor_test(struct kunit *test) +{ + u32 status = APPLE_DCP_COPROC_CPU_STATUS_STOPPED; + + KUNIT_EXPECT_FALSE(test, apple_dcp_coprocessor_is_active(0, status)); +} + +static void apple_dcp_control_run_test(struct kunit *test) +{ + u32 control = APPLE_DCP_COPROC_CPU_CONTROL_RUN; + + KUNIT_EXPECT_TRUE(test, apple_dcp_coprocessor_is_active(control, 0)); +} + +static void apple_dcp_status_running_test(struct kunit *test) +{ + u32 status = APPLE_DCP_COPROC_CPU_STATUS_RUNNING; + + KUNIT_EXPECT_TRUE(test, apple_dcp_coprocessor_is_active(0, status)); +} + +static void apple_dcp_j700_live_state_test(struct kunit *test) +{ + /* Values captured read-only from the running J700 DCP ASC wrapper. */ + KUNIT_EXPECT_TRUE(test, apple_dcp_coprocessor_is_active(0x10, 0x6d)); + /* RUNNING can transiently clear while RUN remains asserted. */ + KUNIT_EXPECT_TRUE(test, apple_dcp_coprocessor_is_active(0x10, 0x6c)); + /* Without an explicit STOPPED bit, the same status still fails closed. */ + KUNIT_EXPECT_TRUE(test, apple_dcp_coprocessor_is_active(0, 0x6c)); +} + +static void apple_dcp_ambiguous_state_fails_closed_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, apple_dcp_coprocessor_is_active(0, 0)); +} + +static struct kunit_case apple_dcp_lifecycle_test_cases[] = { + KUNIT_CASE(apple_dcp_cold_coprocessor_test), + KUNIT_CASE(apple_dcp_control_run_test), + KUNIT_CASE(apple_dcp_status_running_test), + KUNIT_CASE(apple_dcp_j700_live_state_test), + KUNIT_CASE(apple_dcp_ambiguous_state_fails_closed_test), + {} +}; + +static struct kunit_suite apple_dcp_lifecycle_test_suite = { + .name = "apple-dcp-lifecycle", + .test_cases = apple_dcp_lifecycle_test_cases, +}; + +kunit_test_suite(apple_dcp_lifecycle_test_suite); + +MODULE_AUTHOR("The Asahi Linux Contributors"); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple DCP lifecycle admission KUnit tests"); diff --git a/drivers/gpu/drm/apple/dcp-lifecycle.h b/drivers/gpu/drm/apple/dcp-lifecycle.h new file mode 100644 index 0000000000000..ed4b3adcc43a5 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-lifecycle.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_DCP_LIFECYCLE_H__ +#define __APPLE_DCP_LIFECYCLE_H__ + +#include +#include + +#define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) +#define APPLE_DCP_COPROC_CPU_STATUS_RUNNING BIT(0) +#define APPLE_DCP_COPROC_CPU_STATUS_STOPPED BIT(1) + +static inline bool apple_dcp_coprocessor_is_active(u32 cpu_control, u32 cpu_status) +{ + if (cpu_control & APPLE_DCP_COPROC_CPU_CONTROL_RUN || + cpu_status & APPLE_DCP_COPROC_CPU_STATUS_RUNNING) + return true; + + /* Fail closed unless the status register explicitly reports STOPPED. */ + return !(cpu_status & APPLE_DCP_COPROC_CPU_STATUS_STOPPED); +} + +#endif /* __APPLE_DCP_LIFECYCLE_H__ */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a45803901af25..8c3cf3c889584 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -49,12 +49,13 @@ #include "av.h" #include "dcp.h" #include "dcp-internal.h" +#include "dcp-lifecycle.h" #include "iomfb.h" #include "parser.h" #include "trace.h" #define APPLE_DCP_COPROC_CPU_CONTROL 0x44 -#define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) +#define APPLE_DCP_COPROC_CPU_STATUS 0x48 #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) @@ -2197,7 +2198,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; struct apple_dcp *dcp = dev_get_drvdata(dev); - u32 cpu_ctrl; + u32 cpu_ctrl, cpu_status; int ret; ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); @@ -2208,6 +2209,22 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (IS_ERR(dcp->coproc_reg)) return PTR_ERR(dcp->coproc_reg); + /* + * ASCWrap v6 can be handed to Linux with a live firmware-owned RTKit + * session. The cold-start path below has no protocol for adopting its + * endpoint map or IOMFB shared memory, so touching PIODMA/DART state or + * sending a new RTKit INIT would corrupt the inherited display session. + */ + if (dcp->hw.reject_running_coprocessor) { + cpu_ctrl = readl_relaxed(dcp->coproc_reg + + APPLE_DCP_COPROC_CPU_CONTROL); + cpu_status = readl_relaxed(dcp->coproc_reg + + APPLE_DCP_COPROC_CPU_STATUS); + if (apple_dcp_coprocessor_is_active(cpu_ctrl, cpu_status)) + return dev_err_probe(dev, -EBUSY, + "refusing cold start of running firmware-owned DCP (control=%#x status=%#x)\n", + cpu_ctrl, cpu_status); + } if (dcp->index || dcp->dptx_phy || dcp->dptx_die) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); @@ -2610,6 +2627,11 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { .num_dptx_ports = 2, }; +static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { + .num_dptx_ports = 0, + .reject_running_coprocessor = true, +}; + static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { .num_dptx_ports = 0, }; @@ -2621,6 +2643,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_dcpext = { static const struct of_device_id of_match[] = { { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, + { .compatible = "apple,t8140-dcp", .data = &apple_dcp_hw_t8140, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, {} From 4ebdbb24609594f896dbb02108649c04bb09f30e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 22:19:20 -0400 Subject: [PATCH 0189/1579] drm/apple: encode the H17P DCPLink wire layout --- drivers/gpu/drm/apple/Kconfig | 7 +- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-lifecycle-test.c | 6 +- drivers/gpu/drm/apple/dcp-lifecycle.h | 2 +- drivers/gpu/drm/apple/dcp-link-test.c | 110 ++++++++++++++++ drivers/gpu/drm/apple/dcp-link.h | 141 +++++++++++++++++++++ 6 files changed, 259 insertions(+), 8 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-link-test.c create mode 100644 drivers/gpu/drm/apple/dcp-link.h diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 5608412840cc6..0e6a5d76608b6 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -34,7 +34,6 @@ config DRM_APPLE_KUNIT_TEST depends on DRM_APPLE && KUNIT default KUNIT_ALL_TESTS help - Build focused unit tests for Apple display driver admission and data - handling helpers. The tests cover fail-closed lifecycle decisions for - firmware-owned display coprocessors. They do not access display - hardware. If in doubt, say "N". + Build focused unit tests for Apple display lifecycle helpers and DCPLink + message encoders. The tests do not access display hardware. If in doubt, + say "N". diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index c9eea42655597..a2744290dd8ce 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -18,3 +18,4 @@ appledrm-$(CONFIG_TRACING) += trace.o obj-$(CONFIG_DRM_APPLE) += appledrm.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp-lifecycle-test.o +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp-link-test.o diff --git a/drivers/gpu/drm/apple/dcp-lifecycle-test.c b/drivers/gpu/drm/apple/dcp-lifecycle-test.c index fb0299671681a..1ca6af6848632 100644 --- a/drivers/gpu/drm/apple/dcp-lifecycle-test.c +++ b/drivers/gpu/drm/apple/dcp-lifecycle-test.c @@ -32,11 +32,11 @@ static void apple_dcp_j700_live_state_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, apple_dcp_coprocessor_is_active(0x10, 0x6d)); /* RUNNING can transiently clear while RUN remains asserted. */ KUNIT_EXPECT_TRUE(test, apple_dcp_coprocessor_is_active(0x10, 0x6c)); - /* Without an explicit STOPPED bit, the same status still fails closed. */ + /* Without a STOPPED bit, the same status remains active. */ KUNIT_EXPECT_TRUE(test, apple_dcp_coprocessor_is_active(0, 0x6c)); } -static void apple_dcp_ambiguous_state_fails_closed_test(struct kunit *test) +static void apple_dcp_zero_status_is_active_test(struct kunit *test) { KUNIT_EXPECT_TRUE(test, apple_dcp_coprocessor_is_active(0, 0)); } @@ -46,7 +46,7 @@ static struct kunit_case apple_dcp_lifecycle_test_cases[] = { KUNIT_CASE(apple_dcp_control_run_test), KUNIT_CASE(apple_dcp_status_running_test), KUNIT_CASE(apple_dcp_j700_live_state_test), - KUNIT_CASE(apple_dcp_ambiguous_state_fails_closed_test), + KUNIT_CASE(apple_dcp_zero_status_is_active_test), {} }; diff --git a/drivers/gpu/drm/apple/dcp-lifecycle.h b/drivers/gpu/drm/apple/dcp-lifecycle.h index ed4b3adcc43a5..31cd8d8abb8bd 100644 --- a/drivers/gpu/drm/apple/dcp-lifecycle.h +++ b/drivers/gpu/drm/apple/dcp-lifecycle.h @@ -17,7 +17,7 @@ static inline bool apple_dcp_coprocessor_is_active(u32 cpu_control, u32 cpu_stat cpu_status & APPLE_DCP_COPROC_CPU_STATUS_RUNNING) return true; - /* Fail closed unless the status register explicitly reports STOPPED. */ + /* A zero status is not evidence that the coprocessor stopped. */ return !(cpu_status & APPLE_DCP_COPROC_CPU_STATUS_STOPPED); } diff --git a/drivers/gpu/drm/apple/dcp-link-test.c b/drivers/gpu/drm/apple/dcp-link-test.c new file mode 100644 index 0000000000000..d2ea23f4283f1 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-link-test.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include + +#include "dcp-link.h" + +static void apple_dcp_link_init_desc_test(struct kunit *test) +{ + struct apple_dcp_link_init_desc desc; + + apple_dcp_link_init_desc_encode(&desc, 0x123456789abcULL, true); + KUNIT_EXPECT_EQ(test, le64_to_cpu(desc.shared_dva), 0x123456789abcULL); + KUNIT_EXPECT_EQ(test, le32_to_cpu(desc.flags), + APPLE_DCP_LINK_INIT_FLAG_64M); + KUNIT_EXPECT_EQ(test, le32_to_cpu(desc.version), + APPLE_DCP_LINK_PROTOCOL_VERSION); +} + +static void apple_dcp_link_init_message_test(struct kunit *test) +{ + u64 message; + + KUNIT_ASSERT_EQ(test, + apple_dcp_link_init_message(0x123456789abcULL, &message), 0); + KUNIT_EXPECT_EQ(test, message, 0x123456789abc0040ULL); + KUNIT_EXPECT_EQ(test, + apple_dcp_link_init_message(BIT_ULL(48), &message), -ERANGE); +} + +static void apple_dcp_link_ready_message_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, apple_dcp_link_ready_message(), + 0x0004000000000001ULL); +} + +static void apple_dcp_link_rpc_header_test(struct kunit *test) +{ + struct apple_dcp_link_rpc_header header; + + apple_dcp_link_rpc_header_encode(&header, 0x44303033, 0x24, 0x10); + KUNIT_EXPECT_EQ(test, le32_to_cpu(header.call), 0x44303033U); + KUNIT_EXPECT_EQ(test, le32_to_cpu(header.input_size), 0x24U); + KUNIT_EXPECT_EQ(test, le32_to_cpu(header.output_size), 0x10U); +} + +static void apple_dcp_link_rpc_size_test(struct kunit *test) +{ + u32 size; + + KUNIT_ASSERT_EQ(test, apple_dcp_link_rpc_payload_size(0x24, 0x10, &size), 0); + KUNIT_EXPECT_EQ(test, size, 0x40U); + KUNIT_EXPECT_EQ(test, + apple_dcp_link_rpc_payload_size(U32_MAX, 1, &size), + -EOVERFLOW); + + KUNIT_ASSERT_EQ(test, apple_dcp_link_align_payload(0x41, 0x40, &size), 0); + KUNIT_EXPECT_EQ(test, size, 0x80U); + KUNIT_EXPECT_EQ(test, apple_dcp_link_align_payload(0x40, 0, &size), + -EINVAL); +} + +static void apple_dcp_link_rpc_message_test(struct kunit *test) +{ + u64 message; + + KUNIT_ASSERT_EQ(test, + apple_dcp_link_rpc_message(0x1234, 0x5678, false, &message), + 0); + KUNIT_EXPECT_EQ(test, message, 0x0000567812340002ULL); + + KUNIT_ASSERT_EQ(test, + apple_dcp_link_rpc_message(0x1234, 0x5678, true, &message), + 0); + KUNIT_EXPECT_EQ(test, message, 0x0000567812340202ULL); + KUNIT_EXPECT_EQ(test, + apple_dcp_link_rpc_message(0x10000, 0, false, &message), + -ERANGE); +} + +static void apple_dcp_h17p_rt_bw_test(struct kunit *test) +{ + struct apple_dcp_h17p_rt_bw_config config; + + apple_dcp_h17p_rt_bw_encode(&config, 0x3082a0000ULL, 0x3082e8000ULL); + KUNIT_EXPECT_EQ(test, le64_to_cpu(config.scratch), 0x3082a0000ULL); + KUNIT_EXPECT_EQ(test, le64_to_cpu(config.clock_request), 0x3082e8000ULL); +} + +static struct kunit_case apple_dcp_link_test_cases[] = { + KUNIT_CASE(apple_dcp_link_init_desc_test), + KUNIT_CASE(apple_dcp_link_init_message_test), + KUNIT_CASE(apple_dcp_link_ready_message_test), + KUNIT_CASE(apple_dcp_link_rpc_header_test), + KUNIT_CASE(apple_dcp_link_rpc_size_test), + KUNIT_CASE(apple_dcp_link_rpc_message_test), + KUNIT_CASE(apple_dcp_h17p_rt_bw_test), + {} +}; + +static struct kunit_suite apple_dcp_link_test_suite = { + .name = "apple-dcp-link", + .test_cases = apple_dcp_link_test_cases, +}; + +kunit_test_suite(apple_dcp_link_test_suite); + +MODULE_AUTHOR("The Asahi Linux Contributors"); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple DCPLink message encoder KUnit tests"); diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h new file mode 100644 index 0000000000000..41e228f56f0b6 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_DCP_LINK_H__ +#define __APPLE_DCP_LINK_H__ + +#include +#include +#include +#include +#include + +#define APPLE_DCP_LINK_PROTOCOL_VERSION 4 +#define APPLE_DCP_LINK_STREAM_COUNT 8 +#define APPLE_DCP_LINK_STREAM_STATE_SIZE 0x68 + +#define APPLE_DCP_LINK_INIT_FLAG_64M 0x04000000 + +#define APPLE_DCP_LINK_MSG_READY 1 +#define APPLE_DCP_LINK_MSG_RPC 2 +#define APPLE_DCP_LINK_MSG_RPC_REPLY 0x42 +#define APPLE_DCP_LINK_MSG_INIT 0x40 +#define APPLE_DCP_LINK_MSG_NESTED BIT_ULL(9) +#define APPLE_DCP_LINK_MSG_OFFSET GENMASK_ULL(31, 16) +#define APPLE_DCP_LINK_MSG_SIZE GENMASK_ULL(47, 32) +#define APPLE_DCP_LINK_MSG_VERSION GENMASK_ULL(63, 48) + +struct apple_dcp_link_init_desc { + __le64 shared_dva; + __le32 flags; + __le32 version; +} __packed; + +struct apple_dcp_link_rpc_header { + __le32 call; + __le32 input_size; + __le32 output_size; +} __packed; + +struct apple_dcp_h17p_rt_bw_config { + __le64 scratch; + __le64 clock_request; +} __packed; + +static_assert(sizeof(struct apple_dcp_link_init_desc) == 0x10); +static_assert(sizeof(struct apple_dcp_link_rpc_header) == 0x0c); +static_assert(sizeof(struct apple_dcp_h17p_rt_bw_config) == 0x10); + +static inline void +apple_dcp_link_init_desc_encode(struct apple_dcp_link_init_desc *desc, + u64 shared_dva, bool use_64m_window) +{ + desc->shared_dva = cpu_to_le64(shared_dva); + desc->flags = cpu_to_le32(use_64m_window ? APPLE_DCP_LINK_INIT_FLAG_64M : 0); + desc->version = cpu_to_le32(APPLE_DCP_LINK_PROTOCOL_VERSION); +} + +static inline int apple_dcp_link_init_message(u64 shared_dva, u64 *message) +{ + if (!message) + return -EINVAL; + if (shared_dva & ~GENMASK_ULL(47, 0)) + return -ERANGE; + + *message = (shared_dva << 16) | APPLE_DCP_LINK_MSG_INIT; + return 0; +} + +static inline u64 apple_dcp_link_ready_message(void) +{ + return FIELD_PREP(APPLE_DCP_LINK_MSG_VERSION, + APPLE_DCP_LINK_PROTOCOL_VERSION) | + APPLE_DCP_LINK_MSG_READY; +} + +static inline int apple_dcp_link_rpc_payload_size(u32 input_size, + u32 output_size, u32 *size) +{ + u32 total; + + if (!size) + return -EINVAL; + if (check_add_overflow(input_size, output_size, &total) || + check_add_overflow(total, + (u32)sizeof(struct apple_dcp_link_rpc_header), + &total)) + return -EOVERFLOW; + + *size = total; + return 0; +} + +static inline int apple_dcp_link_align_payload(u32 size, u32 alignment, + u32 *aligned_size) +{ + u32 padded; + + if (!aligned_size || !is_power_of_2(alignment)) + return -EINVAL; + if (check_add_overflow(size, alignment - 1, &padded)) + return -EOVERFLOW; + + *aligned_size = padded & -alignment; + return 0; +} + +static inline int apple_dcp_link_rpc_message(u32 offset, u32 size, bool nested, + u64 *message) +{ + if (!message) + return -EINVAL; + if (offset > U16_MAX || size > U16_MAX) + return -ERANGE; + + *message = APPLE_DCP_LINK_MSG_RPC | + FIELD_PREP(APPLE_DCP_LINK_MSG_OFFSET, offset) | + FIELD_PREP(APPLE_DCP_LINK_MSG_SIZE, size); + if (nested) + *message |= APPLE_DCP_LINK_MSG_NESTED; + + return 0; +} + +static inline void +apple_dcp_link_rpc_header_encode(struct apple_dcp_link_rpc_header *header, + u32 call, u32 input_size, u32 output_size) +{ + header->call = cpu_to_le32(call); + header->input_size = cpu_to_le32(input_size); + header->output_size = cpu_to_le32(output_size); +} + +static inline void +apple_dcp_h17p_rt_bw_encode(struct apple_dcp_h17p_rt_bw_config *config, + u64 scratch, u64 clock_request) +{ + config->scratch = cpu_to_le64(scratch); + config->clock_request = cpu_to_le64(clock_request); +} + +#endif /* __APPLE_DCP_LINK_H__ */ From c78748af563b35b1057f54509a7a234fe1bcc8a5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 22:25:54 -0400 Subject: [PATCH 0190/1579] drm/apple: model H17P DCPLink stream framing --- drivers/gpu/drm/apple/dcp-link-test.c | 31 +++++++++++++- drivers/gpu/drm/apple/dcp-link.h | 62 +++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp-link-test.c b/drivers/gpu/drm/apple/dcp-link-test.c index d2ea23f4283f1..ace3886bf5334 100644 --- a/drivers/gpu/drm/apple/dcp-link-test.c +++ b/drivers/gpu/drm/apple/dcp-link-test.c @@ -62,7 +62,7 @@ static void apple_dcp_link_rpc_size_test(struct kunit *test) static void apple_dcp_link_rpc_message_test(struct kunit *test) { - u64 message; + u64 encoded, message; KUNIT_ASSERT_EQ(test, apple_dcp_link_rpc_message(0x1234, 0x5678, false, &message), @@ -76,6 +76,34 @@ static void apple_dcp_link_rpc_message_test(struct kunit *test) KUNIT_EXPECT_EQ(test, apple_dcp_link_rpc_message(0x10000, 0, false, &message), -ERANGE); + + KUNIT_ASSERT_EQ(test, + apple_dcp_link_stream_message(message, 5, true, &encoded), 0); + KUNIT_EXPECT_EQ(test, encoded & GENMASK_ULL(15, 0), 0x1702ULL); + KUNIT_EXPECT_EQ(test, + FIELD_GET(APPLE_DCP_LINK_MSG_STREAM, encoded), 5ULL); + KUNIT_EXPECT_TRUE(test, encoded & APPLE_DCP_LINK_MSG_REMOTE); + KUNIT_EXPECT_TRUE(test, encoded & APPLE_DCP_LINK_MSG_NESTED); +} + +static void apple_dcp_link_stack_test(struct kunit *test) +{ + u32 next, offset; + + KUNIT_ASSERT_EQ(test, + apple_dcp_link_stack_push(0x80, 0x1000, 0x41, 0x40, + &offset, &next), + 0); + KUNIT_EXPECT_EQ(test, offset, 0x80U); + KUNIT_EXPECT_EQ(test, next, 0x100U); + + KUNIT_ASSERT_EQ(test, apple_dcp_link_stack_pop(next, 0x41, 0x40, &next), + 0); + KUNIT_EXPECT_EQ(test, next, 0x80U); + KUNIT_EXPECT_EQ(test, + apple_dcp_link_stack_push(0xff0, 0x1000, 0x20, 0x40, + &offset, &next), + -ENOSPC); } static void apple_dcp_h17p_rt_bw_test(struct kunit *test) @@ -94,6 +122,7 @@ static struct kunit_case apple_dcp_link_test_cases[] = { KUNIT_CASE(apple_dcp_link_rpc_header_test), KUNIT_CASE(apple_dcp_link_rpc_size_test), KUNIT_CASE(apple_dcp_link_rpc_message_test), + KUNIT_CASE(apple_dcp_link_stack_test), KUNIT_CASE(apple_dcp_h17p_rt_bw_test), {} }; diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index 41e228f56f0b6..c10f1dfbbb6f3 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -11,8 +11,11 @@ #include #define APPLE_DCP_LINK_PROTOCOL_VERSION 4 +#define APPLE_DCP_LINK_ENDPOINT 0x24 #define APPLE_DCP_LINK_STREAM_COUNT 8 #define APPLE_DCP_LINK_STREAM_STATE_SIZE 0x68 +#define APPLE_DCP_LINK_RPC_MEMORY_SIZE 0x0c0000 +#define APPLE_DCP_LINK_SHARED_VAR_MEMORY_SIZE 0x4000000 #define APPLE_DCP_LINK_INIT_FLAG_64M 0x04000000 @@ -20,7 +23,9 @@ #define APPLE_DCP_LINK_MSG_RPC 2 #define APPLE_DCP_LINK_MSG_RPC_REPLY 0x42 #define APPLE_DCP_LINK_MSG_INIT 0x40 +#define APPLE_DCP_LINK_MSG_REMOTE BIT_ULL(8) #define APPLE_DCP_LINK_MSG_NESTED BIT_ULL(9) +#define APPLE_DCP_LINK_MSG_STREAM GENMASK_ULL(15, 10) #define APPLE_DCP_LINK_MSG_OFFSET GENMASK_ULL(31, 16) #define APPLE_DCP_LINK_MSG_SIZE GENMASK_ULL(47, 32) #define APPLE_DCP_LINK_MSG_VERSION GENMASK_ULL(63, 48) @@ -121,6 +126,63 @@ static inline int apple_dcp_link_rpc_message(u32 offset, u32 size, bool nested, return 0; } +static inline int apple_dcp_link_stream_message(u64 message, u8 stream, + bool remote, u64 *encoded) +{ + if (!encoded) + return -EINVAL; + if (stream >= 64) + return -ERANGE; + + message &= ~(APPLE_DCP_LINK_MSG_REMOTE | APPLE_DCP_LINK_MSG_STREAM); + message |= FIELD_PREP(APPLE_DCP_LINK_MSG_STREAM, stream); + if (remote) + message |= APPLE_DCP_LINK_MSG_REMOTE; + + *encoded = message; + return 0; +} + +static inline int apple_dcp_link_stack_push(u32 top, u32 capacity, + u32 payload_size, u32 alignment, + u32 *offset, u32 *next_top) +{ + u32 aligned_size; + int ret; + + if (!offset || !next_top || top > capacity) + return -EINVAL; + + ret = apple_dcp_link_align_payload(payload_size, alignment, &aligned_size); + if (ret) + return ret; + if (aligned_size > capacity - top) + return -ENOSPC; + + *offset = top; + *next_top = top + aligned_size; + return 0; +} + +static inline int apple_dcp_link_stack_pop(u32 top, u32 payload_size, + u32 alignment, u32 *next_top) +{ + u32 aligned_size; + int ret; + + if (!next_top) + return -EINVAL; + + ret = apple_dcp_link_align_payload(payload_size, alignment, &aligned_size); + if (ret) + return ret; + if (aligned_size > top) + return -ERANGE; + + *next_top = top - aligned_size; + return 0; +} + static inline void apple_dcp_link_rpc_header_encode(struct apple_dcp_link_rpc_header *header, u32 call, u32 input_size, u32 output_size) From 14673c82e84d104f276a805010316fcd09a2d717 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 22:29:57 -0400 Subject: [PATCH 0191/1579] drm/apple: decode H17P DCPLink ready replies --- drivers/gpu/drm/apple/dcp-link-test.c | 19 +++++++++++++++++++ drivers/gpu/drm/apple/dcp-link.h | 24 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-link-test.c b/drivers/gpu/drm/apple/dcp-link-test.c index ace3886bf5334..ff175ae35eed7 100644 --- a/drivers/gpu/drm/apple/dcp-link-test.c +++ b/drivers/gpu/drm/apple/dcp-link-test.c @@ -30,8 +30,25 @@ static void apple_dcp_link_init_message_test(struct kunit *test) static void apple_dcp_link_ready_message_test(struct kunit *test) { + u32 firmware_hash; + u16 version; + u64 reply; + KUNIT_EXPECT_EQ(test, apple_dcp_link_ready_message(), 0x0004000000000001ULL); + + reply = FIELD_PREP(APPLE_DCP_LINK_MSG_VERSION, 4) | + FIELD_PREP(APPLE_DCP_LINK_MSG_HASH, 0xa1b2c3d4) | + APPLE_DCP_LINK_MSG_READY; + KUNIT_ASSERT_EQ(test, + apple_dcp_link_ready_reply(reply, &version, &firmware_hash), + 0); + KUNIT_EXPECT_EQ(test, version, (u16)4); + KUNIT_EXPECT_EQ(test, firmware_hash, 0xa1b2c3d4U); + KUNIT_EXPECT_EQ(test, + apple_dcp_link_ready_reply(APPLE_DCP_LINK_MSG_RPC, &version, + &firmware_hash), + -EPROTO); } static void apple_dcp_link_rpc_header_test(struct kunit *test) @@ -82,7 +99,9 @@ static void apple_dcp_link_rpc_message_test(struct kunit *test) KUNIT_EXPECT_EQ(test, encoded & GENMASK_ULL(15, 0), 0x1702ULL); KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_DCP_LINK_MSG_STREAM, encoded), 5ULL); + KUNIT_EXPECT_EQ(test, apple_dcp_link_message_stream(encoded), (u8)5); KUNIT_EXPECT_TRUE(test, encoded & APPLE_DCP_LINK_MSG_REMOTE); + KUNIT_EXPECT_TRUE(test, apple_dcp_link_message_is_remote(encoded)); KUNIT_EXPECT_TRUE(test, encoded & APPLE_DCP_LINK_MSG_NESTED); } diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index c10f1dfbbb6f3..9797eba2b41ea 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -28,6 +28,7 @@ #define APPLE_DCP_LINK_MSG_STREAM GENMASK_ULL(15, 10) #define APPLE_DCP_LINK_MSG_OFFSET GENMASK_ULL(31, 16) #define APPLE_DCP_LINK_MSG_SIZE GENMASK_ULL(47, 32) +#define APPLE_DCP_LINK_MSG_HASH GENMASK_ULL(47, 16) #define APPLE_DCP_LINK_MSG_VERSION GENMASK_ULL(63, 48) struct apple_dcp_link_init_desc { @@ -78,6 +79,19 @@ static inline u64 apple_dcp_link_ready_message(void) APPLE_DCP_LINK_MSG_READY; } +static inline int apple_dcp_link_ready_reply(u64 message, u16 *version, + u32 *firmware_hash) +{ + if (!version || !firmware_hash) + return -EINVAL; + if ((message & GENMASK_ULL(1, 0)) != APPLE_DCP_LINK_MSG_READY) + return -EPROTO; + + *version = FIELD_GET(APPLE_DCP_LINK_MSG_VERSION, message); + *firmware_hash = FIELD_GET(APPLE_DCP_LINK_MSG_HASH, message); + return 0; +} + static inline int apple_dcp_link_rpc_payload_size(u32 input_size, u32 output_size, u32 *size) { @@ -143,6 +157,16 @@ static inline int apple_dcp_link_stream_message(u64 message, u8 stream, return 0; } +static inline u8 apple_dcp_link_message_stream(u64 message) +{ + return FIELD_GET(APPLE_DCP_LINK_MSG_STREAM, message); +} + +static inline bool apple_dcp_link_message_is_remote(u64 message) +{ + return message & APPLE_DCP_LINK_MSG_REMOTE; +} + static inline int apple_dcp_link_stack_push(u32 top, u32 capacity, u32 payload_size, u32 alignment, u32 *offset, u32 *next_top) From 51dd0404d024032f90ff2c3e172b7a9ad7debb26 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 22:32:23 -0400 Subject: [PATCH 0192/1579] drm/apple: decode H17P DCPLink RPC buffers --- drivers/gpu/drm/apple/dcp-link-test.c | 50 +++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-link.h | 59 +++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-link-test.c b/drivers/gpu/drm/apple/dcp-link-test.c index ff175ae35eed7..2ac72e4f41caa 100644 --- a/drivers/gpu/drm/apple/dcp-link-test.c +++ b/drivers/gpu/drm/apple/dcp-link-test.c @@ -125,6 +125,54 @@ static void apple_dcp_link_stack_test(struct kunit *test) -ENOSPC); } +static void apple_dcp_link_rpc_decode_test(struct kunit *test) +{ + struct apple_dcp_link_rpc_header *header; + struct apple_dcp_link_rpc_view view; + u8 buffer[0x100] = {}; + u64 message; + + header = (void *)(buffer + 0x20); + apple_dcp_link_rpc_header_encode(header, 0x44303033, 4, 8); + KUNIT_ASSERT_EQ(test, + apple_dcp_link_rpc_message(0x20, sizeof(*header) + 12, + false, &message), + 0); + KUNIT_ASSERT_EQ(test, + apple_dcp_link_rpc_decode(message, buffer, sizeof(buffer), + &view), + 0); + KUNIT_EXPECT_EQ(test, view.call, 0x44303033U); + KUNIT_EXPECT_EQ(test, view.input_size, 4U); + KUNIT_EXPECT_EQ(test, view.output_size, 8U); + KUNIT_EXPECT_PTR_EQ(test, view.input, buffer + 0x20 + sizeof(*header)); + KUNIT_EXPECT_PTR_EQ(test, view.output, + buffer + 0x20 + sizeof(*header) + 4); + + header->output_size = cpu_to_le32(9); + KUNIT_EXPECT_EQ(test, + apple_dcp_link_rpc_decode(message, buffer, sizeof(buffer), + &view), + -EMSGSIZE); + KUNIT_EXPECT_EQ(test, + apple_dcp_link_rpc_decode(APPLE_DCP_LINK_MSG_RPC_REPLY, buffer, + sizeof(buffer), &view), + -EPROTO); +} + +static void apple_dcp_link_rpc_reply_test(struct kunit *test) +{ + u64 request, reply; + + KUNIT_ASSERT_EQ(test, + apple_dcp_link_rpc_message(0x1234, 0x5678, true, &request), + 0); + KUNIT_ASSERT_EQ(test, + apple_dcp_link_stream_message(request, 5, true, &request), 0); + reply = apple_dcp_link_rpc_reply(request); + KUNIT_EXPECT_EQ(test, reply, 0x0000000000001742ULL); +} + static void apple_dcp_h17p_rt_bw_test(struct kunit *test) { struct apple_dcp_h17p_rt_bw_config config; @@ -142,6 +190,8 @@ static struct kunit_case apple_dcp_link_test_cases[] = { KUNIT_CASE(apple_dcp_link_rpc_size_test), KUNIT_CASE(apple_dcp_link_rpc_message_test), KUNIT_CASE(apple_dcp_link_stack_test), + KUNIT_CASE(apple_dcp_link_rpc_decode_test), + KUNIT_CASE(apple_dcp_link_rpc_reply_test), KUNIT_CASE(apple_dcp_h17p_rt_bw_test), {} }; diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index 9797eba2b41ea..2a69a3e6d7ee3 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -30,6 +30,7 @@ #define APPLE_DCP_LINK_MSG_SIZE GENMASK_ULL(47, 32) #define APPLE_DCP_LINK_MSG_HASH GENMASK_ULL(47, 16) #define APPLE_DCP_LINK_MSG_VERSION GENMASK_ULL(63, 48) +#define APPLE_DCP_LINK_REPLY_CONTROL_MASK 0xff3e struct apple_dcp_link_init_desc { __le64 shared_dva; @@ -48,6 +49,16 @@ struct apple_dcp_h17p_rt_bw_config { __le64 clock_request; } __packed; +struct apple_dcp_link_rpc_view { + u32 call; + u32 input_size; + u32 output_size; + u32 offset; + u32 total_size; + const void *input; + void *output; +}; + static_assert(sizeof(struct apple_dcp_link_init_desc) == 0x10); static_assert(sizeof(struct apple_dcp_link_rpc_header) == 0x0c); static_assert(sizeof(struct apple_dcp_h17p_rt_bw_config) == 0x10); @@ -167,6 +178,54 @@ static inline bool apple_dcp_link_message_is_remote(u64 message) return message & APPLE_DCP_LINK_MSG_REMOTE; } +static inline u64 apple_dcp_link_rpc_reply(u64 request) +{ + return (request & (APPLE_DCP_LINK_MSG_VERSION | + APPLE_DCP_LINK_REPLY_CONTROL_MASK)) | + APPLE_DCP_LINK_MSG_RPC_REPLY; +} + +static inline int apple_dcp_link_rpc_decode(u64 message, void *buffer, + u32 capacity, + struct apple_dcp_link_rpc_view *view) +{ + struct apple_dcp_link_rpc_header *header; + u32 expected_size; + u32 offset; + u32 total_size; + int ret; + + if (!buffer || !view) + return -EINVAL; + if ((message & GENMASK_ULL(1, 0)) != APPLE_DCP_LINK_MSG_RPC || + (message & GENMASK_ULL(7, 6)) == BIT_ULL(6)) + return -EPROTO; + + offset = FIELD_GET(APPLE_DCP_LINK_MSG_OFFSET, message); + total_size = FIELD_GET(APPLE_DCP_LINK_MSG_SIZE, message); + if (offset > capacity || total_size > capacity - offset || + total_size < sizeof(*header)) + return -EMSGSIZE; + + header = buffer + offset; + ret = apple_dcp_link_rpc_payload_size(le32_to_cpu(header->input_size), + le32_to_cpu(header->output_size), + &expected_size); + if (ret) + return ret; + if (expected_size != total_size) + return -EMSGSIZE; + + view->call = le32_to_cpu(header->call); + view->input_size = le32_to_cpu(header->input_size); + view->output_size = le32_to_cpu(header->output_size); + view->offset = offset; + view->total_size = total_size; + view->input = header + 1; + view->output = (u8 *)(header + 1) + view->input_size; + return 0; +} + static inline int apple_dcp_link_stack_push(u32 top, u32 capacity, u32 payload_size, u32 alignment, u32 *offset, u32 *next_top) From a9357cd8c7921d01cb9a0e87960b412c23cf664f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 22:34:34 -0400 Subject: [PATCH 0193/1579] drm/apple: encode the H17P bandwidth callback --- drivers/gpu/drm/apple/dcp-link-test.c | 30 ++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-link.h | 36 +++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-link-test.c b/drivers/gpu/drm/apple/dcp-link-test.c index 2ac72e4f41caa..d421dff4c4c02 100644 --- a/drivers/gpu/drm/apple/dcp-link-test.c +++ b/drivers/gpu/drm/apple/dcp-link-test.c @@ -182,6 +182,35 @@ static void apple_dcp_h17p_rt_bw_test(struct kunit *test) KUNIT_EXPECT_EQ(test, le64_to_cpu(config.clock_request), 0x3082e8000ULL); } +static void apple_dcp_h17p_rt_bw_reply_test(struct kunit *test) +{ + struct apple_dcp_h17p_rt_bw_request request = {}; + struct apple_dcp_h17p_rt_bw_reply reply = {}; + struct apple_dcp_link_rpc_view view = { + .call = APPLE_DCP_LINK_CALL_D003, + .input_size = sizeof(request), + .output_size = sizeof(reply), + .input = &request, + .output = &reply, + }; + + KUNIT_ASSERT_EQ(test, + apple_dcp_h17p_rt_bw_encode_reply(&view, 0x3082a0000ULL, + 0x3082e8000ULL), + 0); + KUNIT_EXPECT_EQ(test, le64_to_cpu(reply.config.scratch), + 0x3082a0000ULL); + KUNIT_EXPECT_EQ(test, le64_to_cpu(reply.config.clock_request), + 0x3082e8000ULL); + KUNIT_EXPECT_EQ(test, le32_to_cpu(reply.status), 0U); + + request.config_null = cpu_to_le32(1); + KUNIT_EXPECT_EQ(test, + apple_dcp_h17p_rt_bw_encode_reply(&view, 0x3082a0000ULL, + 0x3082e8000ULL), + -EINVAL); +} + static struct kunit_case apple_dcp_link_test_cases[] = { KUNIT_CASE(apple_dcp_link_init_desc_test), KUNIT_CASE(apple_dcp_link_init_message_test), @@ -193,6 +222,7 @@ static struct kunit_case apple_dcp_link_test_cases[] = { KUNIT_CASE(apple_dcp_link_rpc_decode_test), KUNIT_CASE(apple_dcp_link_rpc_reply_test), KUNIT_CASE(apple_dcp_h17p_rt_bw_test), + KUNIT_CASE(apple_dcp_h17p_rt_bw_reply_test), {} }; diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index 2a69a3e6d7ee3..bf42b8899454c 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -32,6 +32,8 @@ #define APPLE_DCP_LINK_MSG_VERSION GENMASK_ULL(63, 48) #define APPLE_DCP_LINK_REPLY_CONTROL_MASK 0xff3e +#define APPLE_DCP_LINK_CALL_D003 0x44303033 + struct apple_dcp_link_init_desc { __le64 shared_dva; __le32 flags; @@ -49,6 +51,15 @@ struct apple_dcp_h17p_rt_bw_config { __le64 clock_request; } __packed; +struct apple_dcp_h17p_rt_bw_request { + __le32 config_null; +} __packed; + +struct apple_dcp_h17p_rt_bw_reply { + struct apple_dcp_h17p_rt_bw_config config; + __le32 status; +} __packed; + struct apple_dcp_link_rpc_view { u32 call; u32 input_size; @@ -62,6 +73,8 @@ struct apple_dcp_link_rpc_view { static_assert(sizeof(struct apple_dcp_link_init_desc) == 0x10); static_assert(sizeof(struct apple_dcp_link_rpc_header) == 0x0c); static_assert(sizeof(struct apple_dcp_h17p_rt_bw_config) == 0x10); +static_assert(sizeof(struct apple_dcp_h17p_rt_bw_request) == 0x04); +static_assert(sizeof(struct apple_dcp_h17p_rt_bw_reply) == 0x14); static inline void apple_dcp_link_init_desc_encode(struct apple_dcp_link_init_desc *desc, @@ -283,4 +296,27 @@ apple_dcp_h17p_rt_bw_encode(struct apple_dcp_h17p_rt_bw_config *config, config->clock_request = cpu_to_le64(clock_request); } +static inline int +apple_dcp_h17p_rt_bw_encode_reply(struct apple_dcp_link_rpc_view *view, + u64 scratch, u64 clock_request) +{ + const struct apple_dcp_h17p_rt_bw_request *request; + struct apple_dcp_h17p_rt_bw_reply *reply; + + if (!view || view->call != APPLE_DCP_LINK_CALL_D003) + return -EINVAL; + if (view->input_size < sizeof(*request) || + view->output_size < sizeof(*reply)) + return -EMSGSIZE; + + request = view->input; + if (le32_to_cpu(request->config_null) & 1) + return -EINVAL; + + reply = view->output; + apple_dcp_h17p_rt_bw_encode(&reply->config, scratch, clock_request); + reply->status = cpu_to_le32(0); + return 0; +} + #endif /* __APPLE_DCP_LINK_H__ */ From caecd2c3e292ea1fbfd31c6b585a5920f7b3e1d1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 22:38:14 -0400 Subject: [PATCH 0194/1579] drm/apple: add the H17P scanout method map --- drivers/gpu/drm/apple/dcp-link-test.c | 28 +++++++++ drivers/gpu/drm/apple/dcp-link.h | 83 +++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-link-test.c b/drivers/gpu/drm/apple/dcp-link-test.c index d421dff4c4c02..14098c5fef95c 100644 --- a/drivers/gpu/drm/apple/dcp-link-test.c +++ b/drivers/gpu/drm/apple/dcp-link-test.c @@ -17,6 +17,33 @@ static void apple_dcp_link_init_desc_test(struct kunit *test) APPLE_DCP_LINK_PROTOCOL_VERSION); } +static void apple_dcp_h17p_method_test(struct kunit *test) +{ + struct apple_dcp_link_method method; + + KUNIT_ASSERT_EQ(test, + apple_dcp_h17p_method(APPLE_DCP_H17P_CALL_START_SIGNAL, + &method), + 0); + KUNIT_EXPECT_EQ(test, method.input_size, 0U); + KUNIT_EXPECT_EQ(test, method.output_size, 4U); + + KUNIT_ASSERT_EQ(test, + apple_dcp_h17p_method(APPLE_DCP_H17P_CALL_SWAP_SUBMIT, + &method), + 0); + KUNIT_EXPECT_EQ(test, method.input_size, 0x1d30U); + KUNIT_EXPECT_EQ(test, method.output_size, 12U); + + KUNIT_ASSERT_EQ(test, + apple_dcp_h17p_method(APPLE_DCP_H17P_CALL_UP_SET_CREATE_DFB, + &method), + 0); + KUNIT_EXPECT_EQ(test, method.input_size, 0U); + KUNIT_EXPECT_EQ(test, method.output_size, 0U); + KUNIT_EXPECT_EQ(test, apple_dcp_h17p_method(0, &method), -ENOENT); +} + static void apple_dcp_link_init_message_test(struct kunit *test) { u64 message; @@ -213,6 +240,7 @@ static void apple_dcp_h17p_rt_bw_reply_test(struct kunit *test) static struct kunit_case apple_dcp_link_test_cases[] = { KUNIT_CASE(apple_dcp_link_init_desc_test), + KUNIT_CASE(apple_dcp_h17p_method_test), KUNIT_CASE(apple_dcp_link_init_message_test), KUNIT_CASE(apple_dcp_link_ready_message_test), KUNIT_CASE(apple_dcp_link_rpc_header_test), diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index bf42b8899454c..2eae3b94e56e2 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -34,6 +34,32 @@ #define APPLE_DCP_LINK_CALL_D003 0x44303033 +enum apple_dcp_h17p_call { + APPLE_DCP_H17P_CALL_UP_SET_CREATE_DFB = 0x41333839, + APPLE_DCP_H17P_CALL_START_SIGNAL = 0x41343031, + APPLE_DCP_H17P_CALL_SWAP_START = 0x41343036, + APPLE_DCP_H17P_CALL_SWAP_SUBMIT = 0x41343037, + APPLE_DCP_H17P_CALL_SET_DISPLAY_DEVICE = 0x41343131, + APPLE_DCP_H17P_CALL_IS_MAIN_DISPLAY = 0x41343132, + APPLE_DCP_H17P_CALL_SET_DIGITAL_OUT_MODE = 0x41343133, + APPLE_DCP_H17P_CALL_SET_MATRIX = 0x41343233, + APPLE_DCP_H17P_CALL_SET_PARAMETER = 0x41343439, + APPLE_DCP_H17P_CALL_CREATE_DEFAULT_FB = 0x41343533, + APPLE_DCP_H17P_CALL_VIDEO_POWER_SAVINGS = 0x41343537, + APPLE_DCP_H17P_CALL_FIRST_CLIENT_OPEN = 0x41343635, + APPLE_DCP_H17P_CALL_LAST_CLIENT_CLOSE = 0x41343636, + APPLE_DCP_H17P_CALL_DISPLAY_REFRESH = 0x41343734, + APPLE_DCP_H17P_CALL_FLUSH_SUPPORTS_POWER = 0x41343737, + APPLE_DCP_H17P_CALL_ABORT_SWAPS = 0x41343738, + APPLE_DCP_H17P_CALL_SET_POWER_STATE = 0x41343833, +}; + +struct apple_dcp_link_method { + u32 call; + u32 input_size; + u32 output_size; +}; + struct apple_dcp_link_init_desc { __le64 shared_dva; __le32 flags; @@ -76,6 +102,63 @@ static_assert(sizeof(struct apple_dcp_h17p_rt_bw_config) == 0x10); static_assert(sizeof(struct apple_dcp_h17p_rt_bw_request) == 0x04); static_assert(sizeof(struct apple_dcp_h17p_rt_bw_reply) == 0x14); +static inline int apple_dcp_h17p_method(u32 call, + struct apple_dcp_link_method *method) +{ + struct apple_dcp_link_method found = { .call = call }; + + if (!method) + return -EINVAL; + + switch (call) { + case APPLE_DCP_H17P_CALL_UP_SET_CREATE_DFB: + case APPLE_DCP_H17P_CALL_FIRST_CLIENT_OPEN: + break; + case APPLE_DCP_H17P_CALL_START_SIGNAL: + case APPLE_DCP_H17P_CALL_IS_MAIN_DISPLAY: + case APPLE_DCP_H17P_CALL_CREATE_DEFAULT_FB: + case APPLE_DCP_H17P_CALL_DISPLAY_REFRESH: + found.output_size = 4; + break; + case APPLE_DCP_H17P_CALL_SWAP_START: + case APPLE_DCP_H17P_CALL_SET_POWER_STATE: + found.input_size = 16; + found.output_size = 8; + break; + case APPLE_DCP_H17P_CALL_SWAP_SUBMIT: + found.input_size = 0x1d30; + found.output_size = 12; + break; + case APPLE_DCP_H17P_CALL_SET_DISPLAY_DEVICE: + case APPLE_DCP_H17P_CALL_VIDEO_POWER_SAVINGS: + case APPLE_DCP_H17P_CALL_LAST_CLIENT_CLOSE: + found.input_size = 4; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CALL_SET_DIGITAL_OUT_MODE: + case APPLE_DCP_H17P_CALL_ABORT_SWAPS: + found.input_size = 8; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CALL_SET_MATRIX: + found.input_size = 80; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CALL_SET_PARAMETER: + found.input_size = 40; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CALL_FLUSH_SUPPORTS_POWER: + found.input_size = 4; + break; + default: + return -ENOENT; + } + + *method = found; + return 0; +} + static inline void apple_dcp_link_init_desc_encode(struct apple_dcp_link_init_desc *desc, u64 shared_dva, bool use_64m_window) From 3415ee9e003927603157cdf3f0ee27aeb32ed882 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 22:40:57 -0400 Subject: [PATCH 0195/1579] drm/apple: bound H17P DCPLink stream IDs --- drivers/gpu/drm/apple/dcp-link-test.c | 15 +++++++++------ drivers/gpu/drm/apple/dcp-link.h | 6 +++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-link-test.c b/drivers/gpu/drm/apple/dcp-link-test.c index 14098c5fef95c..d246af140fab1 100644 --- a/drivers/gpu/drm/apple/dcp-link-test.c +++ b/drivers/gpu/drm/apple/dcp-link-test.c @@ -122,14 +122,17 @@ static void apple_dcp_link_rpc_message_test(struct kunit *test) -ERANGE); KUNIT_ASSERT_EQ(test, - apple_dcp_link_stream_message(message, 5, true, &encoded), 0); - KUNIT_EXPECT_EQ(test, encoded & GENMASK_ULL(15, 0), 0x1702ULL); + apple_dcp_link_stream_message(message, 3, true, &encoded), 0); + KUNIT_EXPECT_EQ(test, encoded & GENMASK_ULL(15, 0), 0x0f02ULL); KUNIT_EXPECT_EQ(test, - FIELD_GET(APPLE_DCP_LINK_MSG_STREAM, encoded), 5ULL); - KUNIT_EXPECT_EQ(test, apple_dcp_link_message_stream(encoded), (u8)5); + FIELD_GET(APPLE_DCP_LINK_MSG_STREAM, encoded), 3ULL); + KUNIT_EXPECT_EQ(test, apple_dcp_link_message_stream(encoded), (u8)3); KUNIT_EXPECT_TRUE(test, encoded & APPLE_DCP_LINK_MSG_REMOTE); KUNIT_EXPECT_TRUE(test, apple_dcp_link_message_is_remote(encoded)); KUNIT_EXPECT_TRUE(test, encoded & APPLE_DCP_LINK_MSG_NESTED); + KUNIT_EXPECT_EQ(test, + apple_dcp_link_stream_message(message, 4, true, &encoded), + -ERANGE); } static void apple_dcp_link_stack_test(struct kunit *test) @@ -195,9 +198,9 @@ static void apple_dcp_link_rpc_reply_test(struct kunit *test) apple_dcp_link_rpc_message(0x1234, 0x5678, true, &request), 0); KUNIT_ASSERT_EQ(test, - apple_dcp_link_stream_message(request, 5, true, &request), 0); + apple_dcp_link_stream_message(request, 3, true, &request), 0); reply = apple_dcp_link_rpc_reply(request); - KUNIT_EXPECT_EQ(test, reply, 0x0000000000001742ULL); + KUNIT_EXPECT_EQ(test, reply, 0x0000000000000f42ULL); } static void apple_dcp_h17p_rt_bw_test(struct kunit *test) diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index 2eae3b94e56e2..aecd5ca7af7ed 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -13,7 +13,11 @@ #define APPLE_DCP_LINK_PROTOCOL_VERSION 4 #define APPLE_DCP_LINK_ENDPOINT 0x24 #define APPLE_DCP_LINK_STREAM_COUNT 8 +#define APPLE_DCP_LINK_STREAMS_PER_SIDE 4 #define APPLE_DCP_LINK_STREAM_STATE_SIZE 0x68 +#define APPLE_DCP_LINK_STATE_SIZE 0x3d0 +#define APPLE_DCP_LINK_LOCAL_STREAMS_OFFSET 0x078 +#define APPLE_DCP_LINK_REMOTE_STREAMS_OFFSET 0x218 #define APPLE_DCP_LINK_RPC_MEMORY_SIZE 0x0c0000 #define APPLE_DCP_LINK_SHARED_VAR_MEMORY_SIZE 0x4000000 @@ -252,7 +256,7 @@ static inline int apple_dcp_link_stream_message(u64 message, u8 stream, { if (!encoded) return -EINVAL; - if (stream >= 64) + if (stream >= APPLE_DCP_LINK_STREAMS_PER_SIDE) return -ERANGE; message &= ~(APPLE_DCP_LINK_MSG_REMOTE | APPLE_DCP_LINK_MSG_STREAM); From 36fac084a51c5835f7345dcc2bd05920ab2e9819 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 22:57:34 -0400 Subject: [PATCH 0196/1579] drm/apple: encode the H17P DCPLink memory layout --- drivers/gpu/drm/apple/dcp-link-test.c | 37 +++++++++++++++ drivers/gpu/drm/apple/dcp-link.h | 66 ++++++++++++++++++++++++--- 2 files changed, 97 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-link-test.c b/drivers/gpu/drm/apple/dcp-link-test.c index d246af140fab1..ad2b4f9f7903d 100644 --- a/drivers/gpu/drm/apple/dcp-link-test.c +++ b/drivers/gpu/drm/apple/dcp-link-test.c @@ -9,6 +9,7 @@ static void apple_dcp_link_init_desc_test(struct kunit *test) { struct apple_dcp_link_init_desc desc; + KUNIT_EXPECT_EQ(test, APPLE_DCP_LINK_ENDPOINT, 0x37); apple_dcp_link_init_desc_encode(&desc, 0x123456789abcULL, true); KUNIT_EXPECT_EQ(test, le64_to_cpu(desc.shared_dva), 0x123456789abcULL); KUNIT_EXPECT_EQ(test, le32_to_cpu(desc.flags), @@ -17,6 +18,41 @@ static void apple_dcp_link_init_desc_test(struct kunit *test) APPLE_DCP_LINK_PROTOCOL_VERSION); } +static void apple_dcp_link_stream_layout_test(struct kunit *test) +{ + struct apple_dcp_link_stream_layout layout; + + KUNIT_ASSERT_EQ(test, + apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 3, + false, &layout), + 0); + KUNIT_EXPECT_EQ(test, layout.local_offset, 0x18000U); + KUNIT_EXPECT_EQ(test, layout.remote_offset, 0x78000U); + KUNIT_EXPECT_EQ(test, layout.capacity, 0x8000U); + + KUNIT_ASSERT_EQ(test, + apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 3, + true, &layout), + 0); + KUNIT_EXPECT_EQ(test, layout.local_offset, 0x38000U); + KUNIT_EXPECT_EQ(test, layout.remote_offset, 0x58000U); + + KUNIT_ASSERT_EQ(test, + apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_DCP, 0, + false, &layout), + 0); + KUNIT_EXPECT_EQ(test, layout.local_offset, 0x40000U); + KUNIT_EXPECT_EQ(test, layout.remote_offset, 0x20000U); + + KUNIT_EXPECT_EQ(test, + apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, 4, + false, &layout), + -EINVAL); + KUNIT_EXPECT_EQ(test, + apple_dcp_link_stream_layout(2, 0, false, &layout), + -EINVAL); +} + static void apple_dcp_h17p_method_test(struct kunit *test) { struct apple_dcp_link_method method; @@ -243,6 +279,7 @@ static void apple_dcp_h17p_rt_bw_reply_test(struct kunit *test) static struct kunit_case apple_dcp_link_test_cases[] = { KUNIT_CASE(apple_dcp_link_init_desc_test), + KUNIT_CASE(apple_dcp_link_stream_layout_test), KUNIT_CASE(apple_dcp_h17p_method_test), KUNIT_CASE(apple_dcp_link_init_message_test), KUNIT_CASE(apple_dcp_link_ready_message_test), diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index aecd5ca7af7ed..967c367a65f82 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -11,7 +11,7 @@ #include #define APPLE_DCP_LINK_PROTOCOL_VERSION 4 -#define APPLE_DCP_LINK_ENDPOINT 0x24 +#define APPLE_DCP_LINK_ENDPOINT 0x37 #define APPLE_DCP_LINK_STREAM_COUNT 8 #define APPLE_DCP_LINK_STREAMS_PER_SIDE 4 #define APPLE_DCP_LINK_STREAM_STATE_SIZE 0x68 @@ -19,6 +19,11 @@ #define APPLE_DCP_LINK_LOCAL_STREAMS_OFFSET 0x078 #define APPLE_DCP_LINK_REMOTE_STREAMS_OFFSET 0x218 #define APPLE_DCP_LINK_RPC_MEMORY_SIZE 0x0c0000 +#define APPLE_DCP_LINK_SIDE_MEMORY_SIZE 0x040000 +#define APPLE_DCP_LINK_STREAM_HALF_SIZE 0x020000 +#define APPLE_DCP_LINK_STREAM_BUFFER_SIZE 0x008000 +#define APPLE_DCP_LINK_HEAP_OFFSET 0x080000 +#define APPLE_DCP_LINK_HEAP_SIZE 0x040000 #define APPLE_DCP_LINK_SHARED_VAR_MEMORY_SIZE 0x4000000 #define APPLE_DCP_LINK_INIT_FLAG_64M 0x04000000 @@ -64,6 +69,17 @@ struct apple_dcp_link_method { u32 output_size; }; +enum apple_dcp_link_side { + APPLE_DCP_LINK_SIDE_AP = 0, + APPLE_DCP_LINK_SIDE_DCP = 1, +}; + +struct apple_dcp_link_stream_layout { + u32 local_offset; + u32 remote_offset; + u32 capacity; +}; + struct apple_dcp_link_init_desc { __le64 shared_dva; __le32 flags; @@ -105,6 +121,44 @@ static_assert(sizeof(struct apple_dcp_link_rpc_header) == 0x0c); static_assert(sizeof(struct apple_dcp_h17p_rt_bw_config) == 0x10); static_assert(sizeof(struct apple_dcp_h17p_rt_bw_request) == 0x04); static_assert(sizeof(struct apple_dcp_h17p_rt_bw_reply) == 0x14); +static_assert(APPLE_DCP_LINK_STREAM_BUFFER_SIZE * + APPLE_DCP_LINK_STREAMS_PER_SIDE == + APPLE_DCP_LINK_STREAM_HALF_SIZE); +static_assert(APPLE_DCP_LINK_HEAP_OFFSET + APPLE_DCP_LINK_HEAP_SIZE == + APPLE_DCP_LINK_RPC_MEMORY_SIZE); + +static inline int +apple_dcp_link_stream_layout(enum apple_dcp_link_side side, u8 stream, + bool remote, + struct apple_dcp_link_stream_layout *layout) +{ + u32 local_side; + u32 remote_side; + u32 stream_offset; + + if (!layout || side > APPLE_DCP_LINK_SIDE_DCP || + stream >= APPLE_DCP_LINK_STREAMS_PER_SIDE) + return -EINVAL; + + local_side = side * APPLE_DCP_LINK_SIDE_MEMORY_SIZE; + remote_side = (side ^ 1) * APPLE_DCP_LINK_SIDE_MEMORY_SIZE; + stream_offset = stream * APPLE_DCP_LINK_STREAM_BUFFER_SIZE; + + if (remote) { + layout->local_offset = local_side + + APPLE_DCP_LINK_STREAM_HALF_SIZE + + stream_offset; + layout->remote_offset = remote_side + stream_offset; + } else { + layout->local_offset = local_side + stream_offset; + layout->remote_offset = remote_side + + APPLE_DCP_LINK_STREAM_HALF_SIZE + + stream_offset; + } + layout->capacity = APPLE_DCP_LINK_STREAM_BUFFER_SIZE; + + return 0; +} static inline int apple_dcp_h17p_method(u32 call, struct apple_dcp_link_method *method) @@ -165,21 +219,21 @@ static inline int apple_dcp_h17p_method(u32 call, static inline void apple_dcp_link_init_desc_encode(struct apple_dcp_link_init_desc *desc, - u64 shared_dva, bool use_64m_window) + u64 shared_var_dva, bool use_64m_window) { - desc->shared_dva = cpu_to_le64(shared_dva); + desc->shared_dva = cpu_to_le64(shared_var_dva); desc->flags = cpu_to_le32(use_64m_window ? APPLE_DCP_LINK_INIT_FLAG_64M : 0); desc->version = cpu_to_le32(APPLE_DCP_LINK_PROTOCOL_VERSION); } -static inline int apple_dcp_link_init_message(u64 shared_dva, u64 *message) +static inline int apple_dcp_link_init_message(u64 rpc_dva, u64 *message) { if (!message) return -EINVAL; - if (shared_dva & ~GENMASK_ULL(47, 0)) + if (rpc_dva & ~GENMASK_ULL(47, 0)) return -ERANGE; - *message = (shared_dva << 16) | APPLE_DCP_LINK_MSG_INIT; + *message = (rpc_dva << 16) | APPLE_DCP_LINK_MSG_INIT; return 0; } From 95d5b96015e9d6834595ac7bccfed4307908ebf7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 23:01:36 -0400 Subject: [PATCH 0197/1579] drm/apple: pin the DCPLink RPC alignment --- drivers/gpu/drm/apple/dcp-link-test.c | 7 +++++++ drivers/gpu/drm/apple/dcp-link.h | 1 + 2 files changed, 8 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-link-test.c b/drivers/gpu/drm/apple/dcp-link-test.c index ad2b4f9f7903d..ae74cce7e5b67 100644 --- a/drivers/gpu/drm/apple/dcp-link-test.c +++ b/drivers/gpu/drm/apple/dcp-link-test.c @@ -134,6 +134,13 @@ static void apple_dcp_link_rpc_size_test(struct kunit *test) apple_dcp_link_rpc_payload_size(U32_MAX, 1, &size), -EOVERFLOW); + KUNIT_ASSERT_EQ(test, + apple_dcp_link_align_payload(0x41, + APPLE_DCP_LINK_RPC_ALIGNMENT, + &size), + 0); + KUNIT_EXPECT_EQ(test, size, 0x44U); + KUNIT_ASSERT_EQ(test, apple_dcp_link_align_payload(0x41, 0x40, &size), 0); KUNIT_EXPECT_EQ(test, size, 0x80U); KUNIT_EXPECT_EQ(test, apple_dcp_link_align_payload(0x40, 0, &size), diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index 967c367a65f82..2c21baa7550b2 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -24,6 +24,7 @@ #define APPLE_DCP_LINK_STREAM_BUFFER_SIZE 0x008000 #define APPLE_DCP_LINK_HEAP_OFFSET 0x080000 #define APPLE_DCP_LINK_HEAP_SIZE 0x040000 +#define APPLE_DCP_LINK_RPC_ALIGNMENT 4 #define APPLE_DCP_LINK_SHARED_VAR_MEMORY_SIZE 0x4000000 #define APPLE_DCP_LINK_INIT_FLAG_64M 0x04000000 From 150b3d397b6ca8f5c5e1f2e465a6a21d0f7bb707 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 23:06:50 -0400 Subject: [PATCH 0198/1579] drm/apple: add the H17P callback ABI map --- drivers/gpu/drm/apple/dcp-link-test.c | 37 +++++++ drivers/gpu/drm/apple/dcp-link.h | 150 ++++++++++++++++++++++++++ 2 files changed, 187 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-link-test.c b/drivers/gpu/drm/apple/dcp-link-test.c index ae74cce7e5b67..f5014a360c878 100644 --- a/drivers/gpu/drm/apple/dcp-link-test.c +++ b/drivers/gpu/drm/apple/dcp-link-test.c @@ -80,6 +80,42 @@ static void apple_dcp_h17p_method_test(struct kunit *test) KUNIT_EXPECT_EQ(test, apple_dcp_h17p_method(0, &method), -ENOENT); } +static void apple_dcp_h17p_callback_method_test(struct kunit *test) +{ + struct apple_dcp_link_method method; + + KUNIT_ASSERT_EQ(test, + apple_dcp_h17p_callback_method( + APPLE_DCP_H17P_CB_RT_BANDWIDTH, &method), + 0); + KUNIT_EXPECT_EQ(test, method.input_size, 4U); + KUNIT_EXPECT_EQ(test, method.output_size, 20U); + + KUNIT_ASSERT_EQ(test, + apple_dcp_h17p_callback_method( + APPLE_DCP_H17P_CB_HOTPLUG, &method), + 0); + KUNIT_EXPECT_EQ(test, method.input_size, 88U); + KUNIT_EXPECT_EQ(test, method.output_size, 76U); + + KUNIT_ASSERT_EQ(test, + apple_dcp_h17p_callback_method( + APPLE_DCP_H17P_CB_SWAP_INFO, &method), + 0); + KUNIT_EXPECT_EQ(test, method.input_size, 228U); + KUNIT_EXPECT_EQ(test, method.output_size, 224U); + + KUNIT_ASSERT_EQ(test, + apple_dcp_h17p_callback_method( + APPLE_DCP_H17P_CB_SWAP_COMPLETE, &method), + 0); + KUNIT_EXPECT_EQ(test, method.input_size, 1840U); + KUNIT_EXPECT_EQ(test, method.output_size, 0U); + + KUNIT_EXPECT_EQ(test, apple_dcp_h17p_callback_method(0, &method), + -ENOENT); +} + static void apple_dcp_link_init_message_test(struct kunit *test) { u64 message; @@ -288,6 +324,7 @@ static struct kunit_case apple_dcp_link_test_cases[] = { KUNIT_CASE(apple_dcp_link_init_desc_test), KUNIT_CASE(apple_dcp_link_stream_layout_test), KUNIT_CASE(apple_dcp_h17p_method_test), + KUNIT_CASE(apple_dcp_h17p_callback_method_test), KUNIT_CASE(apple_dcp_link_init_message_test), KUNIT_CASE(apple_dcp_link_ready_message_test), KUNIT_CASE(apple_dcp_link_rpc_header_test), diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index 2c21baa7550b2..704f3b0513483 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -44,6 +44,43 @@ #define APPLE_DCP_LINK_CALL_D003 0x44303033 +enum apple_dcp_h17p_callback { + APPLE_DCP_H17P_CB_DID_BOOT = 0x44303030, + APPLE_DCP_H17P_CB_DID_POWER_ON = 0x44303031, + APPLE_DCP_H17P_CB_WILL_POWER_OFF = 0x44303032, + APPLE_DCP_H17P_CB_RT_BANDWIDTH = APPLE_DCP_LINK_CALL_D003, + APPLE_DCP_H17P_CB_FRAME_SYNC = 0x44303036, + APPLE_DCP_H17P_CB_MATCH_PMU = 0x44313030, + APPLE_DCP_H17P_CB_START_HARDWARE_BOOT = 0x44313230, + APPLE_DCP_H17P_CB_ALLOCATE_BANDWIDTH = 0x44313239, + APPLE_DCP_H17P_CB_MAP_BUF = 0x44323031, + APPLE_DCP_H17P_CB_UNMAP_BUF = 0x44323032, + APPLE_DCP_H17P_CB_MATCH_PMU_2 = 0x44323036, + APPLE_DCP_H17P_CB_MATCH_BACKLIGHT = 0x44323037, + APPLE_DCP_H17P_CB_GET_TIME = 0x44323039, + APPLE_DCP_H17P_CB_PROP_PUBLISH = 0x44333030, + APPLE_DCP_H17P_CB_GET_UINT_PROP = 0x44343031, + APPLE_DCP_H17P_CB_GET_FREQUENCY = 0x44343038, + APPLE_DCP_H17P_CB_MAP_REG = 0x44343131, + APPLE_DCP_H17P_CB_SET_PROPERTY_INT = 0x44343134, + APPLE_DCP_H17P_CB_SET_PROPERTY_BOOL = 0x44343135, + APPLE_DCP_H17P_CB_ALLOCATE_BUFFER = 0x44343531, + APPLE_DCP_H17P_CB_MAP_PHYSICAL = 0x44343532, + APPLE_DCP_H17P_CB_RELEASE_MEM_DESC = 0x44343534, + APPLE_DCP_H17P_CB_POWER_UP_DART = 0x44353732, + APPLE_DCP_H17P_CB_HOTPLUG = 0x44353735, + APPLE_DCP_H17P_CB_POWERSTATE_NOTIFY = 0x44353736, + APPLE_DCP_H17P_CB_CREATE_DEFAULT_FB = 0x44353836, + APPLE_DCP_H17P_CB_CLEAR_DEFAULT_FB = 0x44353837, + APPLE_DCP_H17P_CB_SWAP_NOTIFY = 0x44353838, + APPLE_DCP_H17P_CB_SWAP_INFO = 0x44353839, + APPLE_DCP_H17P_CB_SWAP_COMPLETE = 0x44353934, + APPLE_DCP_H17P_CB_BATCHED_SWAP_COMPLETE = 0x44353935, + APPLE_DCP_H17P_CB_SWAP_COMPLETE_INTENT = 0x44353936, + APPLE_DCP_H17P_CB_ABORT_SWAP = 0x44353937, + APPLE_DCP_H17P_CB_ENABLE_BACKLIGHT = 0x44353938, +}; + enum apple_dcp_h17p_call { APPLE_DCP_H17P_CALL_UP_SET_CREATE_DFB = 0x41333839, APPLE_DCP_H17P_CALL_START_SIGNAL = 0x41343031, @@ -128,6 +165,119 @@ static_assert(APPLE_DCP_LINK_STREAM_BUFFER_SIZE * static_assert(APPLE_DCP_LINK_HEAP_OFFSET + APPLE_DCP_LINK_HEAP_SIZE == APPLE_DCP_LINK_RPC_MEMORY_SIZE); +static inline int +apple_dcp_h17p_callback_method(u32 call, struct apple_dcp_link_method *method) +{ + struct apple_dcp_link_method found = { .call = call }; + + if (!method) + return -EINVAL; + + switch (call) { + case APPLE_DCP_H17P_CB_WILL_POWER_OFF: + case APPLE_DCP_H17P_CB_MATCH_PMU: + case APPLE_DCP_H17P_CB_CLEAR_DEFAULT_FB: + break; + case APPLE_DCP_H17P_CB_DID_BOOT: + case APPLE_DCP_H17P_CB_DID_POWER_ON: + case APPLE_DCP_H17P_CB_START_HARDWARE_BOOT: + case APPLE_DCP_H17P_CB_MATCH_PMU_2: + case APPLE_DCP_H17P_CB_MATCH_BACKLIGHT: + found.output_size = 4; + break; + case APPLE_DCP_H17P_CB_RT_BANDWIDTH: + found.input_size = 4; + found.output_size = 20; + break; + case APPLE_DCP_H17P_CB_FRAME_SYNC: + found.input_size = 84; + found.output_size = 80; + break; + case APPLE_DCP_H17P_CB_ALLOCATE_BANDWIDTH: + found.input_size = 28; + found.output_size = 20; + break; + case APPLE_DCP_H17P_CB_MAP_BUF: + found.input_size = 12; + found.output_size = 16; + break; + case APPLE_DCP_H17P_CB_UNMAP_BUF: + case APPLE_DCP_H17P_CB_SWAP_NOTIFY: + found.input_size = 24; + break; + case APPLE_DCP_H17P_CB_GET_TIME: + found.output_size = 8; + break; + case APPLE_DCP_H17P_CB_POWERSTATE_NOTIFY: + case APPLE_DCP_H17P_CB_ABORT_SWAP: + case APPLE_DCP_H17P_CB_ENABLE_BACKLIGHT: + found.input_size = 4; + break; + case APPLE_DCP_H17P_CB_PROP_PUBLISH: + found.input_size = 16; + break; + case APPLE_DCP_H17P_CB_SWAP_COMPLETE: + found.input_size = 1840; + break; + case APPLE_DCP_H17P_CB_SWAP_COMPLETE_INTENT: + found.input_size = 20; + break; + case APPLE_DCP_H17P_CB_GET_UINT_PROP: + found.input_size = 80; + found.output_size = 12; + break; + case APPLE_DCP_H17P_CB_GET_FREQUENCY: + found.input_size = 8; + found.output_size = 8; + break; + case APPLE_DCP_H17P_CB_MAP_REG: + found.input_size = 16; + found.output_size = 28; + break; + case APPLE_DCP_H17P_CB_SET_PROPERTY_INT: + found.input_size = 80; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CB_SET_PROPERTY_BOOL: + found.input_size = 76; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CB_ALLOCATE_BUFFER: + found.input_size = 20; + found.output_size = 20; + break; + case APPLE_DCP_H17P_CB_MAP_PHYSICAL: + found.input_size = 8; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CB_RELEASE_MEM_DESC: + case APPLE_DCP_H17P_CB_POWER_UP_DART: + found.input_size = 4; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CB_HOTPLUG: + found.input_size = 88; + found.output_size = 76; + break; + case APPLE_DCP_H17P_CB_CREATE_DEFAULT_FB: + found.input_size = 8; + found.output_size = 4; + break; + case APPLE_DCP_H17P_CB_SWAP_INFO: + found.input_size = 228; + found.output_size = 224; + break; + case APPLE_DCP_H17P_CB_BATCHED_SWAP_COMPLETE: + found.output_size = 1024; + break; + default: + return -ENOENT; + } + + *method = found; + return 0; +} + static inline int apple_dcp_link_stream_layout(enum apple_dcp_link_side side, u8 stream, bool remote, From 95a8ea208c6b429123d5ad49b9ed735e0425e833 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 23:12:16 -0400 Subject: [PATCH 0199/1579] drm/apple: describe the J700 display clocks --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp-link-test.c | 29 +++++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-link.h | 27 +++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp.c | 7 +++++++ 4 files changed, 64 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 45891525c46f4..071d4906d2308 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -177,6 +177,7 @@ struct apple_dcp { /* clock rate request by dcp in */ struct clk *clk; + struct clk *clk_194; /* DCP shared memory */ void *shmem; diff --git a/drivers/gpu/drm/apple/dcp-link-test.c b/drivers/gpu/drm/apple/dcp-link-test.c index f5014a360c878..eedd77e252612 100644 --- a/drivers/gpu/drm/apple/dcp-link-test.c +++ b/drivers/gpu/drm/apple/dcp-link-test.c @@ -53,6 +53,34 @@ static void apple_dcp_link_stream_layout_test(struct kunit *test) -EINVAL); } +static void apple_dcp_h17p_clock_rate_test(struct kunit *test) +{ + struct apple_dcp_h17p_clock_request request = { + .service = cpu_to_le32(7), + .clock_id = cpu_to_le32(APPLE_DCP_H17P_CLOCK_ID_154), + }; + u64 rate; + + KUNIT_ASSERT_EQ(test, + apple_dcp_h17p_clock_rate(&request, 300000000, + 450000000, &rate), + 0); + KUNIT_EXPECT_EQ(test, rate, 300000000ULL); + + request.clock_id = cpu_to_le32(APPLE_DCP_H17P_CLOCK_ID_194); + KUNIT_ASSERT_EQ(test, + apple_dcp_h17p_clock_rate(&request, 300000000, + 450000000, &rate), + 0); + KUNIT_EXPECT_EQ(test, rate, 450000000ULL); + + request.clock_id = cpu_to_le32(0); + KUNIT_EXPECT_EQ(test, + apple_dcp_h17p_clock_rate(&request, 300000000, + 450000000, &rate), + -ENOENT); +} + static void apple_dcp_h17p_method_test(struct kunit *test) { struct apple_dcp_link_method method; @@ -323,6 +351,7 @@ static void apple_dcp_h17p_rt_bw_reply_test(struct kunit *test) static struct kunit_case apple_dcp_link_test_cases[] = { KUNIT_CASE(apple_dcp_link_init_desc_test), KUNIT_CASE(apple_dcp_link_stream_layout_test), + KUNIT_CASE(apple_dcp_h17p_clock_rate_test), KUNIT_CASE(apple_dcp_h17p_method_test), KUNIT_CASE(apple_dcp_h17p_callback_method_test), KUNIT_CASE(apple_dcp_link_init_message_test), diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index 704f3b0513483..7b698e8076b04 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -43,6 +43,8 @@ #define APPLE_DCP_LINK_REPLY_CONTROL_MASK 0xff3e #define APPLE_DCP_LINK_CALL_D003 0x44303033 +#define APPLE_DCP_H17P_CLOCK_ID_154 0x154 +#define APPLE_DCP_H17P_CLOCK_ID_194 0x194 enum apple_dcp_h17p_callback { APPLE_DCP_H17P_CB_DID_BOOT = 0x44303030, @@ -130,6 +132,11 @@ struct apple_dcp_link_rpc_header { __le32 output_size; } __packed; +struct apple_dcp_h17p_clock_request { + __le32 service; + __le32 clock_id; +} __packed; + struct apple_dcp_h17p_rt_bw_config { __le64 scratch; __le64 clock_request; @@ -156,6 +163,7 @@ struct apple_dcp_link_rpc_view { static_assert(sizeof(struct apple_dcp_link_init_desc) == 0x10); static_assert(sizeof(struct apple_dcp_link_rpc_header) == 0x0c); +static_assert(sizeof(struct apple_dcp_h17p_clock_request) == 0x08); static_assert(sizeof(struct apple_dcp_h17p_rt_bw_config) == 0x10); static_assert(sizeof(struct apple_dcp_h17p_rt_bw_request) == 0x04); static_assert(sizeof(struct apple_dcp_h17p_rt_bw_reply) == 0x14); @@ -165,6 +173,25 @@ static_assert(APPLE_DCP_LINK_STREAM_BUFFER_SIZE * static_assert(APPLE_DCP_LINK_HEAP_OFFSET + APPLE_DCP_LINK_HEAP_SIZE == APPLE_DCP_LINK_RPC_MEMORY_SIZE); +static inline int +apple_dcp_h17p_clock_rate(const struct apple_dcp_h17p_clock_request *request, + u64 rate_154, u64 rate_194, u64 *rate) +{ + if (!request || !rate) + return -EINVAL; + + switch (le32_to_cpu(request->clock_id)) { + case APPLE_DCP_H17P_CLOCK_ID_154: + *rate = rate_154; + return 0; + case APPLE_DCP_H17P_CLOCK_ID_194: + *rate = rate_194; + return 0; + default: + return -ENOENT; + } +} + static inline int apple_dcp_h17p_callback_method(u32 call, struct apple_dcp_link_method *method) { diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8c3cf3c889584..d7ef1abf6d4e3 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2284,6 +2284,10 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (IS_ERR(dcp->clk)) return dev_err_probe(dev, PTR_ERR(dcp->clk), "Unable to find clock\n"); + dcp->clk_194 = devm_clk_get_optional(dev, "clock-194"); + if (IS_ERR(dcp->clk_194)) + return dev_err_probe(dev, PTR_ERR(dcp->clk_194), + "Unable to find clock 0x194\n"); bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry @@ -2372,6 +2376,9 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) devm_clk_put(dev, dcp->clk); dcp->clk = NULL; + if (dcp->clk_194) + devm_clk_put(dev, dcp->clk_194); + dcp->clk_194 = NULL; } static const struct component_ops dcp_comp_ops = { From 8416b1911d57c19fec743172dcb0aa8ea79aaa26 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 23:14:23 -0400 Subject: [PATCH 0200/1579] drm/apple: map the H17P iomfb startup calls --- drivers/gpu/drm/apple/dcp-link-test.c | 22 ++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-link.h | 9 +++++++++ 2 files changed, 31 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-link-test.c b/drivers/gpu/drm/apple/dcp-link-test.c index eedd77e252612..0f39324d6e1fa 100644 --- a/drivers/gpu/drm/apple/dcp-link-test.c +++ b/drivers/gpu/drm/apple/dcp-link-test.c @@ -105,6 +105,28 @@ static void apple_dcp_h17p_method_test(struct kunit *test) 0); KUNIT_EXPECT_EQ(test, method.input_size, 0U); KUNIT_EXPECT_EQ(test, method.output_size, 0U); + + KUNIT_ASSERT_EQ(test, + apple_dcp_h17p_method(APPLE_DCP_H17P_CALL_LATE_INIT, + &method), + 0); + KUNIT_EXPECT_EQ(test, method.input_size, 4U); + KUNIT_EXPECT_EQ(test, method.output_size, 4U); + + KUNIT_ASSERT_EQ(test, + apple_dcp_h17p_method( + APPLE_DCP_H17P_CALL_SETUP_VIDEO_LIMITS, &method), + 0); + KUNIT_EXPECT_EQ(test, method.input_size, 0U); + KUNIT_EXPECT_EQ(test, method.output_size, 0U); + + KUNIT_ASSERT_EQ(test, + apple_dcp_h17p_method( + APPLE_DCP_H17P_CALL_VI_SET_TEMPERATURE_HINT, + &method), + 0); + KUNIT_EXPECT_EQ(test, method.input_size, 0U); + KUNIT_EXPECT_EQ(test, method.output_size, 4U); KUNIT_EXPECT_EQ(test, apple_dcp_h17p_method(0, &method), -ENOENT); } diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index 7b698e8076b04..bdfa5a1558f46 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -84,7 +84,10 @@ enum apple_dcp_h17p_callback { }; enum apple_dcp_h17p_call { + APPLE_DCP_H17P_CALL_LATE_INIT = 0x41303030, + APPLE_DCP_H17P_CALL_SETUP_VIDEO_LIMITS = 0x41303234, APPLE_DCP_H17P_CALL_UP_SET_CREATE_DFB = 0x41333839, + APPLE_DCP_H17P_CALL_VI_SET_TEMPERATURE_HINT = 0x41333930, APPLE_DCP_H17P_CALL_START_SIGNAL = 0x41343031, APPLE_DCP_H17P_CALL_SWAP_START = 0x41343036, APPLE_DCP_H17P_CALL_SWAP_SUBMIT = 0x41343037, @@ -348,9 +351,15 @@ static inline int apple_dcp_h17p_method(u32 call, switch (call) { case APPLE_DCP_H17P_CALL_UP_SET_CREATE_DFB: + case APPLE_DCP_H17P_CALL_SETUP_VIDEO_LIMITS: case APPLE_DCP_H17P_CALL_FIRST_CLIENT_OPEN: break; + case APPLE_DCP_H17P_CALL_LATE_INIT: + found.input_size = 4; + found.output_size = 4; + break; case APPLE_DCP_H17P_CALL_START_SIGNAL: + case APPLE_DCP_H17P_CALL_VI_SET_TEMPERATURE_HINT: case APPLE_DCP_H17P_CALL_IS_MAIN_DISPLAY: case APPLE_DCP_H17P_CALL_CREATE_DEFAULT_FB: case APPLE_DCP_H17P_CALL_DISPLAY_REFRESH: From 10c8a6566b7b86fa036ab97945ca7c63d1c43357 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 23:17:27 -0400 Subject: [PATCH 0201/1579] drm/apple: model the H17P swap payload --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/iomfb_h17p.h | 22 ++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.h | 14 ++++++++++++++ drivers/gpu/drm/apple/iomfb_v12_3.c | 1 + drivers/gpu/drm/apple/iomfb_v13_3.c | 1 + 5 files changed, 40 insertions(+) create mode 100644 drivers/gpu/drm/apple/iomfb_h17p.h diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 071d4906d2308..836cefb8b5895 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -17,6 +17,7 @@ #include "dptxep.h" #include "iomfb.h" +#include "iomfb_h17p.h" #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "epic/dpavservep.h" @@ -214,6 +215,7 @@ struct apple_dcp { union { struct dcp_swap_submit_req_v12_3 v12_3; struct dcp_swap_submit_req_v13_3 v13_3; + struct dcp_swap_submit_req_h17p h17p; } swap; /* swap id of the last completed swap */ diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h new file mode 100644 index 0000000000000..58d7033ab1b60 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_IOMFB_H17P_H__ +#define __APPLE_IOMFB_H17P_H__ + +#include "version_utils.h" + +#define DCP_FW h17p +#define DCP_FW_VER DCP_FW_VERSION(26, 6, 0) + +#include "iomfb_template.h" + +static_assert(sizeof(struct dcp_swap_h17p) == 0x6e0); +static_assert(sizeof(struct dcp_surface_h17p) == 0x22c); +static_assert(sizeof(struct dcp_swap_submit_req_h17p) == 0x1d30); +static_assert(sizeof(struct dcp_swap_submit_resp_h17p) == 0x0c); + +#undef DCP_FW_VER +#undef DCP_FW + +#endif /* __APPLE_IOMFB_H17P_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index ce9fb92513cd1..2298c5bf2156e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -53,6 +53,9 @@ struct DCP_FW_NAME(dcp_swap) { u8 unk_320[0x13f]; u64 unk_1; #endif +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + u8 h17p_tail[0x278]; +#endif } __packed; /* Information describing a surface */ @@ -73,8 +76,13 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { u64 surf_iova[SWAP_SURFACES]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u64 unk_u64_a[SWAP_SURFACES]; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + struct DCP_FW_NAME(dcp_surface) surf2[6]; + u64 surf2_iova[6]; +#else struct DCP_FW_NAME(dcp_surface) surf2[5]; u64 surf2_iova[5]; +#endif #endif u8 unkbool; u64 unkdouble; @@ -89,14 +97,20 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { u8 swap_null; u8 surf_null[SWAP_SURFACES]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + u8 surf2_null[6]; +#else u8 surf2_null[5]; +#endif #endif u8 unkoutbool_null; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u8 unkU32Ptr_null; u8 unkU32out_null; #endif +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 padding[1]; +#endif } __packed; struct DCP_FW_NAME(dcp_swap_submit_resp) { diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.c b/drivers/gpu/drm/apple/iomfb_v12_3.c index 0fe08c42d6465..3256974fb11d0 100644 --- a/drivers/gpu/drm/apple/iomfb_v12_3.c +++ b/drivers/gpu/drm/apple/iomfb_v12_3.c @@ -3,6 +3,7 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" +#include "iomfb_h17p.h" #include "version_utils.h" static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { diff --git a/drivers/gpu/drm/apple/iomfb_v13_3.c b/drivers/gpu/drm/apple/iomfb_v13_3.c index 0ac869d24eb01..9adce7b150fd0 100644 --- a/drivers/gpu/drm/apple/iomfb_v13_3.c +++ b/drivers/gpu/drm/apple/iomfb_v13_3.c @@ -3,6 +3,7 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" +#include "iomfb_h17p.h" #include "version_utils.h" static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { From c3ba4bd4fa44306cb9c27ca24090a783f981b341 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 23:20:27 -0400 Subject: [PATCH 0202/1579] drm/apple: add the H17P iomfb ABI tables --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-link.h | 2 + drivers/gpu/drm/apple/iomfb_h17p.c | 154 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 4 +- 4 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/apple/iomfb_h17p.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index a2744290dd8ce..67623ce156bc0 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -10,6 +10,7 @@ appledrm-y += connector.o appledrm-y += ibootep.o appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o +appledrm-y += iomfb_h17p.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index bdfa5a1558f46..7f8a48d3d5ff3 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -5,8 +5,10 @@ #define __APPLE_DCP_LINK_H__ #include +#include #include #include +#include #include #include diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c new file mode 100644 index 0000000000000..13271aa23e573 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include "dcp-link.h" +#include "iomfb_h17p.h" +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "version_utils.h" + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD("A024", dcpep_setup_video_limits), + IOMFB_METHOD("A136", iomfbep_a131_pmu_service_matched), + IOMFB_METHOD("A137", iomfbep_a132_backlight_service_matched), + IOMFB_METHOD("A389", dcpep_set_create_dfb), + IOMFB_METHOD("A390", iomfbep_a358_vi_set_temperature_hint), + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A406", dcpep_swap_start), + IOMFB_METHOD("A407", dcpep_swap_submit), + IOMFB_METHOD("A411", dcpep_set_display_device), + IOMFB_METHOD("A412", dcpep_is_main_display), + IOMFB_METHOD("A413", dcpep_set_digital_out_mode), + IOMFB_METHOD("A423", iomfbep_set_matrix), + IOMFB_METHOD("A427", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A449", dcpep_set_parameter_dcp), + IOMFB_METHOD("A453", dcpep_create_default_fb), + IOMFB_METHOD("A457", dcpep_enable_disable_video_power_savings), + IOMFB_METHOD("A465", dcpep_first_client_open), + IOMFB_METHOD("A466", iomfbep_last_client_close), + IOMFB_METHOD("A474", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A477", dcpep_flush_supports_power), + IOMFB_METHOD("A478", iomfbep_abort_swaps_dcp), + IOMFB_METHOD("A483", dcpep_set_power_state), +}; + +#define DCP_FW h17p +#define DCP_FW_VER DCP_FW_VERSION(26, 6, 0) + +#include "iomfb_template.c" + +static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + struct apple_dcp_link_rpc_view view = { + .call = APPLE_DCP_LINK_CALL_D003, + .input_size = sizeof(struct apple_dcp_h17p_rt_bw_request), + .output_size = sizeof(struct apple_dcp_h17p_rt_bw_reply), + .input = in, + .output = out, + }; + u64 clock_request = dcp->disp_bw_doorbell_res.start; + u64 scratch = dcp->disp_bw_scratch_res.start + + dcp->disp_bw_scratch_offset; + + trace_iomfb_callback(dcp, tag, __func__); + return apple_dcp_h17p_rt_bw_encode_reply(&view, scratch, + clock_request) == 0; +} + +static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct apple_dcp_h17p_clock_request *request = in; + u64 rate; + int ret; + + trace_iomfb_callback(dcp, tag, __func__); + ret = apple_dcp_h17p_clock_rate(request, clk_get_rate(dcp->clk), + clk_get_rate(dcp->clk_194), &rate); + if (ret) { + dev_warn(dcp->dev, "unknown display clock ID %#x\n", + le32_to_cpu(request->clock_id)); + rate = 0; + } + + *(__le64 *)out = cpu_to_le64(rate); + return true; +} + +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = trampoline_true, /* did_boot_signal */ + [1] = trampoline_true, /* did_power_on_signal */ + [2] = trampoline_nop, /* will_power_off_signal */ + [3] = trampoline_rt_bandwidth_h17p, + [6] = trampoline_set_frame_sync_props, + [100] = iomfbep_cb_match_pmu_service, + [101] = trampoline_zero, /* get_display_default_stride */ + [102] = trampoline_nop, /* set_number_property */ + [103] = trampoline_nop, /* trigger_user_cal_loader */ + [104] = trampoline_nop, /* set_boolean_property */ + [107] = trampoline_true, /* create_provider_service */ + [108] = trampoline_true, /* create_product_service */ + [109] = trampoline_true, /* create_pmu_service */ + [110] = trampoline_true, /* create_iomfb_service */ + [111] = trampoline_create_backlight_service, + [112] = trampoline_true, /* create_nvram_service */ + [114] = trampoline_get_tiling_state, + [115] = trampoline_false, /* set_tiling_state */ + [120] = dcpep_cb_boot_1, + [121] = trampoline_false, /* is_dark_boot */ + [122] = trampoline_false, /* is_waking_from_hibernate */ + [124] = trampoline_read_edt_data, + [126] = trampoline_prop_start, + [127] = trampoline_prop_chunk, + [128] = trampoline_prop_end, + [129] = trampoline_allocate_bandwidth, + [201] = trampoline_map_piodma, + [202] = trampoline_unmap_piodma, + [206] = iomfbep_cb_match_pmu_service_2, + [207] = iomfbep_cb_match_backlight_service, + [208] = trampoline_nop, /* update_backlight_factor_prop */ + [209] = trampoline_get_time, + [300] = trampoline_pr_publish, + [401] = trampoline_get_uint_prop, + [404] = trampoline_nop, /* set_uint_prop */ + [406] = trampoline_set_fx_prop, + [408] = trampoline_get_frequency_h17p, + [411] = trampoline_map_reg, + [413] = trampoline_true, /* set_property_dict */ + [414] = trampoline_sr_set_property_int, + [415] = trampoline_true, /* set_property_bool */ + [451] = trampoline_allocate_buffer, + [452] = trampoline_map_physical, + [454] = trampoline_release_mem_desc, + [552] = trampoline_true, /* set_property_dict_0 */ + [561] = trampoline_true, /* set_property_dict */ + [563] = trampoline_true, /* set_property_int */ + [565] = trampoline_true, /* set_property_bool */ + [567] = trampoline_true, /* set_property_str */ + [572] = trampoline_zero, /* power_up_dart */ + [575] = trampoline_hotplug, + [576] = trampoline_nop, /* powerstate_notify */ + [586] = trampoline_true, /* create_default_fb_surface */ + [587] = trampoline_nop, /* clear_default_surface */ + [588] = trampoline_nop, /* swap_notify */ + [589] = trampoline_nop, /* swap_info_notify */ + [594] = trampoline_swap_complete, + [596] = trampoline_swap_complete_intent_gated, + [597] = trampoline_abort_swap_ap_gated, + [598] = trampoline_enable_backlight_message_ap_gated, + [599] = trampoline_nop, /* setSystemConsoleMode */ + [601] = trampoline_false, /* isDFBAllocated */ + [602] = trampoline_false, /* preserveContents */ + [603] = trampoline_nop, /* find_swap_function_gated */ +}; + +void iomfb_start_h17p(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index e4cb300695dbd..f92f6c9afd20d 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -937,7 +937,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) for (int l = 0; l < SWAP_SURFACES; l++) swap->surf_null[l] = true; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - for (int l = 0; l < 5; l++) + for (int l = 0; l < ARRAY_SIZE(swap->surf2_null); l++) swap->surf2_null[l] = true; swap->unkU32Ptr_null = true; swap->unkU32out_null = true; @@ -1398,7 +1398,7 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru for (l = 0; l < SWAP_SURFACES; l++) req->surf_null[l] = true; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - for (l = 0; l < 5; l++) + for (l = 0; l < ARRAY_SIZE(req->surf2_null); l++) req->surf2_null[l] = true; req->unkU32Ptr_null = true; req->unkU32out_null = true; From 721268e8563def312aed28afa1633ce4694a931a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 27 Aug 2026 23:40:15 -0400 Subject: [PATCH 0203/1579] drm/apple: wire the H17P DCPLink transport --- drivers/gpu/drm/apple/dcp-internal.h | 9 + drivers/gpu/drm/apple/dcp.c | 12 + drivers/gpu/drm/apple/iomfb.c | 317 +++++++++++++++++++++++++++ 3 files changed, 338 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 836cefb8b5895..8a8b2f582e983 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -15,6 +15,7 @@ #include #include +#include "dcp-link.h" #include "dptxep.h" #include "iomfb.h" #include "iomfb_h17p.h" @@ -65,6 +66,7 @@ enum dcp_firmware_version { DCP_FIRMWARE_UNKNOWN, DCP_FIRMWARE_V_12_3, DCP_FIRMWARE_V_13_5, + DCP_FIRMWARE_H17P, }; enum { @@ -107,6 +109,8 @@ struct dcp_channel { void *cookies[DCP_MAX_CALL_DEPTH]; void *output[DCP_MAX_CALL_DEPTH]; u16 end[DCP_MAX_CALL_DEPTH]; + u64 link_message[DCP_MAX_CALL_DEPTH]; + u32 link_size[DCP_MAX_CALL_DEPTH]; /* Current depth of the call stack. Less than DCP_MAX_CALL_DEPTH */ u8 depth; @@ -182,6 +186,11 @@ struct apple_dcp { /* DCP shared memory */ void *shmem; + dma_addr_t shmem_iova; + u32 link_top[2][APPLE_DCP_LINK_STREAMS_PER_SIDE]; + u16 link_version; + u32 link_firmware_hash; + bool link_ready; /* Display registers mappable to the DCP */ struct resource *disp_registers[MAX_DISP_REGISTERS]; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d7ef1abf6d4e3..8ab177a893721 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1749,6 +1749,9 @@ static void _dcp_poweroff(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_poweroff_v13_3(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_poweroff_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -1811,6 +1814,9 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_sleep_v13_3(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_sleep_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -1852,6 +1858,9 @@ void dcp_poweron(struct platform_device *pdev) case DCP_FIRMWARE_V_13_5: iomfb_poweron_v13_3(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_poweron_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -2152,6 +2161,9 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) char fw_str[DCP_FW_VERSION_STR_LEN]; int ret; + if (of_device_is_compatible(dev->of_node, "apple,t8140-dcp")) + return DCP_FIRMWARE_H17P; + /* firmware version is just informative */ dcp_read_fw_version(dev, "apple,firmware-version", fw_str); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 1e33d52825c43..aa3294e4da4e6 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -131,6 +131,15 @@ static struct dcp_channel *dcp_get_channel(struct apple_dcp *dcp, } } +static void dcp_link_push(struct apple_dcp *dcp, bool oob, + const struct dcp_method_entry *call, u32 in_len, + u32 out_len, void *data, dcp_callback_t cb, + void *cookie); +static void dcp_link_ack(struct apple_dcp *dcp, + enum dcp_context_id context); +static void dcp_link_recv_msg(struct apple_dcp *dcp, u64 message); +static int dcp_link_start_rtkit(struct apple_dcp *dcp); + /* Get the start of a packet: after the end of the previous packet */ static u16 dcp_packet_start(struct dcp_channel *ch, u8 depth) { @@ -161,6 +170,12 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca u32 in_len, u32 out_len, void *data, dcp_callback_t cb, void *cookie) { + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + dcp_link_push(dcp, oob, call, in_len, out_len, data, cb, + cookie); + return; + } + enum dcp_context_id context = dcp_call_context(dcp, oob); struct dcp_channel *ch = dcp_get_channel(dcp, context); @@ -217,11 +232,292 @@ int dcp_parse_tag(char tag[4]) return d[0] + (d[1] * 10) + (d[2] * 100); } +static u8 dcp_link_context_stream(enum dcp_context_id context) +{ + return context == DCP_CONTEXT_OOBCB || + context == DCP_CONTEXT_OOBCMD ? 1 : 0; +} + +static bool dcp_link_context_is_remote(enum dcp_context_id context) +{ + return context == DCP_CONTEXT_CB || context == DCP_CONTEXT_OOBCB; +} + +static u32 dcp_link_call_tag(const struct dcp_method_entry *call) +{ + return ((u32)(u8)call->tag[0] << 24) | + ((u32)(u8)call->tag[1] << 16) | + ((u32)(u8)call->tag[2] << 8) | + (u8)call->tag[3]; +} + +static void dcp_link_push(struct apple_dcp *dcp, bool oob, + const struct dcp_method_entry *call, u32 in_len, + u32 out_len, void *data, dcp_callback_t cb, + void *cookie) +{ + enum dcp_context_id context = dcp_call_context(dcp, oob); + struct dcp_channel *ch = dcp_get_channel(dcp, context); + struct apple_dcp_link_stream_layout layout; + struct apple_dcp_link_rpc_header *header; + bool remote = dcp_link_context_is_remote(context); + u8 stream = dcp_link_context_stream(context); + u32 *top = &dcp->link_top[remote][stream]; + u32 next_top; + u32 offset; + u32 payload_size; + u64 message; + u8 depth; + void *buffer; + int ret; + + depth = dcp_push_depth(&ch->depth); + ret = apple_dcp_link_rpc_payload_size(in_len, out_len, + &payload_size); + if (ret) + goto err_pop; + ret = apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, stream, + remote, &layout); + if (ret) + goto err_pop; + ret = apple_dcp_link_stack_push(*top, layout.capacity, payload_size, + APPLE_DCP_LINK_RPC_ALIGNMENT, &offset, + &next_top); + if (ret) + goto err_pop; + + buffer = dcp->shmem + layout.local_offset; + header = buffer + offset; + apple_dcp_link_rpc_header_encode(header, dcp_link_call_tag(call), + in_len, out_len); + if (in_len) + memcpy(header + 1, data, in_len); + if (out_len) + memset((u8 *)(header + 1) + in_len, 0, out_len); + + ret = apple_dcp_link_rpc_message(offset, payload_size, remote, &message); + if (ret) + goto err_pop; + ret = apple_dcp_link_stream_message(message, stream, remote, &message); + if (ret) + goto err_pop; + + ch->callbacks[depth] = cb; + ch->cookies[depth] = cookie; + ch->output[depth] = (u8 *)(header + 1) + in_len; + ch->link_message[depth] = message; + ch->link_size[depth] = payload_size; + *top = next_top; + + dma_wmb(); + dcp_send_message(dcp, APPLE_DCP_LINK_ENDPOINT, message); + return; + +err_pop: + dcp_pop_depth(&ch->depth); + dev_err(dcp->dev, "DCPLink call %.4s could not be queued: %d\n", + call->tag, ret); +} + +static void dcp_link_ack(struct apple_dcp *dcp, + enum dcp_context_id context) +{ + struct dcp_channel *ch = dcp_get_channel(dcp, context); + u8 depth; + u64 reply; + + if (!ch || !ch->depth) { + dev_warn(dcp->dev, "DCPLink callback reply without an active call\n"); + return; + } + + depth = dcp_pop_depth(&ch->depth); + reply = apple_dcp_link_rpc_reply(ch->link_message[depth]); + ch->link_message[depth] = 0; + dma_wmb(); + dcp_send_message(dcp, APPLE_DCP_LINK_ENDPOINT, reply); +} + +static struct dcp_channel *dcp_link_reply_channel(struct apple_dcp *dcp, + u64 message) +{ + bool remote = apple_dcp_link_message_is_remote(message); + u8 stream = apple_dcp_link_message_stream(message); + + if (stream == 1) + return remote ? &dcp->ch_oobcb : &dcp->ch_oobcmd; + return remote ? &dcp->ch_cb : &dcp->ch_cmd; +} + +static void dcp_link_handle_reply(struct apple_dcp *dcp, u64 message) +{ + struct dcp_channel *ch = dcp_link_reply_channel(dcp, message); + bool remote = apple_dcp_link_message_is_remote(message); + u8 stream = apple_dcp_link_message_stream(message); + u32 *top; + dcp_callback_t cb; + void *cookie; + u8 depth; + int ret; + + if (stream >= APPLE_DCP_LINK_STREAMS_PER_SIDE) { + dev_warn(dcp->dev, "DCPLink reply stream %u is invalid\n", stream); + return; + } + top = &dcp->link_top[remote][stream]; + + if (!ch->depth) { + dev_warn(dcp->dev, "DCPLink reply without an active call\n"); + return; + } + + depth = dcp_pop_depth(&ch->depth); + ret = apple_dcp_link_stack_pop(*top, ch->link_size[depth], + APPLE_DCP_LINK_RPC_ALIGNMENT, top); + if (ret) + dev_warn(dcp->dev, "DCPLink stream %u stack mismatch: %d\n", + stream, ret); + + dma_rmb(); + cb = ch->callbacks[depth]; + cookie = ch->cookies[depth]; + ch->callbacks[depth] = NULL; + ch->cookies[depth] = NULL; + ch->link_message[depth] = 0; + ch->link_size[depth] = 0; + if (cb) + cb(dcp, ch->output[depth], cookie); +} + +static void dcp_link_handle_callback(struct apple_dcp *dcp, u64 message) +{ + struct apple_dcp_link_stream_layout layout; + struct apple_dcp_link_method method; + struct apple_dcp_link_rpc_view view; + struct dcp_channel *ch = &dcp->ch_cb; + char wire_tag[4]; + u8 stream = apple_dcp_link_message_stream(message); + u8 depth; + int tag; + int ret; + + ret = apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, stream, true, + &layout); + if (ret) + return; + dma_rmb(); + ret = apple_dcp_link_rpc_decode(message, + dcp->shmem + layout.remote_offset, + layout.capacity, &view); + if (ret) { + dev_warn(dcp->dev, "DCPLink callback could not be decoded: %d\n", + ret); + return; + } + + ret = apple_dcp_h17p_callback_method(view.call, &method); + if (!ret && (method.input_size != view.input_size || + method.output_size != view.output_size)) { + dev_warn(dcp->dev, + "DCPLink callback %#x size %u/%u, expected %u/%u\n", + view.call, view.input_size, view.output_size, + method.input_size, method.output_size); + } + + wire_tag[0] = view.call; + wire_tag[1] = view.call >> 8; + wire_tag[2] = view.call >> 16; + wire_tag[3] = view.call >> 24; + tag = dcp_parse_tag(wire_tag); + if (tag < 0 || tag >= IOMFB_MAX_CB) { + dev_warn(dcp->dev, "DCPLink callback tag %#x is invalid\n", + view.call); + return; + } + + if (view.output_size) + memset(view.output, 0, view.output_size); + depth = dcp_push_depth(&ch->depth); + ch->output[depth] = view.output; + ch->link_message[depth] = message; + ch->link_size[depth] = view.total_size; + + if (!dcp->cb_handlers || !dcp->cb_handlers[tag]) { + dev_warn(dcp->dev, "DCPLink callback D%03d has no handler\n", tag); + dcp_link_ack(dcp, DCP_CONTEXT_CB); + return; + } + + if (dcp->cb_handlers[tag](dcp, tag, view.output, (void *)view.input)) + dcp_link_ack(dcp, DCP_CONTEXT_CB); +} + +static void dcp_link_recv_msg(struct apple_dcp *dcp, u64 message) +{ + u8 type = message & GENMASK_ULL(1, 0); + + if (type == APPLE_DCP_LINK_MSG_READY) { + if (apple_dcp_link_ready_reply(message, &dcp->link_version, + &dcp->link_firmware_hash)) + return; + if (dcp->link_ready) + return; + dcp->link_ready = true; + dev_info(dcp->dev, "DCPLink ready: version %u hash %#x\n", + dcp->link_version, dcp->link_firmware_hash); + iomfb_start_h17p(dcp); + return; + } + + if (type != APPLE_DCP_LINK_MSG_RPC) { + dev_warn(dcp->dev, "unknown DCPLink message %#llx\n", message); + return; + } + + if ((message & GENMASK_ULL(7, 6)) == BIT_ULL(6)) + dcp_link_handle_reply(dcp, message); + else + dcp_link_handle_callback(dcp, message); +} + +static int dcp_link_start_rtkit(struct apple_dcp *dcp) +{ + struct apple_dcp_link_init_desc *desc; + u64 init_message; + int ret; + + apple_rtkit_start_ep(dcp->rtk, APPLE_DCP_LINK_ENDPOINT); + dcp->shmem = dma_alloc_coherent(dcp->dev, APPLE_DCP_LINK_RPC_MEMORY_SIZE, + &dcp->shmem_iova, GFP_KERNEL); + if (!dcp->shmem) + return -ENOMEM; + + memset(dcp->shmem, 0, APPLE_DCP_LINK_RPC_MEMORY_SIZE); + desc = dcp->shmem; + apple_dcp_link_init_desc_encode(desc, 0, false); + ret = apple_dcp_link_init_message(dcp->shmem_iova, &init_message); + if (ret) + return ret; + + dcp->link_ready = false; + memset(dcp->link_top, 0, sizeof(dcp->link_top)); + dma_wmb(); + dcp_send_message(dcp, APPLE_DCP_LINK_ENDPOINT, init_message); + dcp_send_message(dcp, APPLE_DCP_LINK_ENDPOINT, + apple_dcp_link_ready_message()); + return 0; +} + /* Ack a callback from the DCP */ void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) { struct dcp_channel *ch = dcp_get_channel(dcp, context); + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + dcp_link_ack(dcp, context); + return; + } + dcp_pop_depth(&ch->depth); dcp_send_message(dcp, IOMFB_ENDPOINT, dcpep_ack(context)); @@ -526,6 +822,9 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, case DCP_FIRMWARE_V_13_5: ret = iomfb_modeset_v13_3(dcp, crtc_state); break; + case DCP_FIRMWARE_H17P: + ret = iomfb_modeset_h17p(dcp, crtc_state); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -590,6 +889,9 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) case DCP_FIRMWARE_V_13_5: iomfb_flush_v13_3(dcp, crtc, state); break; + case DCP_FIRMWARE_H17P: + iomfb_flush_h17p(dcp, crtc, state); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -605,6 +907,9 @@ static void iomfb_start(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_start_v13_3(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_start_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -622,6 +927,11 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) { enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + dcp_link_recv_msg(dcp, message); + return; + } + if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) iomfb_start(dcp); else if (type == IOMFB_MESSAGE_TYPE_MSG) @@ -633,6 +943,10 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) int iomfb_start_rtkit(struct apple_dcp *dcp) { dma_addr_t shmem_iova; + + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_link_start_rtkit(dcp); + apple_rtkit_start_ep(dcp->rtk, IOMFB_ENDPOINT); dcp->shmem = dma_alloc_coherent(dcp->dev, DCP_SHMEM_SIZE, &shmem_iova, @@ -656,6 +970,9 @@ void iomfb_shutdown(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_shutdown_v13_3(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_shutdown_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; From b87d3810f0e38dcf7ea8b21eb8bde4d501e8aa7c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 00:05:47 -0400 Subject: [PATCH 0204/1579] drm/apple: complete H17P callback handling --- drivers/gpu/drm/apple/iomfb.c | 9 +++++- drivers/gpu/drm/apple/iomfb_h17p.c | 48 +++++++++++++++++++++++++++--- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index aa3294e4da4e6..e84b4d1aedd7e 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -403,8 +403,11 @@ static void dcp_link_handle_callback(struct apple_dcp *dcp, u64 message) ret = apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, stream, true, &layout); - if (ret) + if (ret) { + dcp_send_message(dcp, APPLE_DCP_LINK_ENDPOINT, + apple_dcp_link_rpc_reply(message)); return; + } dma_rmb(); ret = apple_dcp_link_rpc_decode(message, dcp->shmem + layout.remote_offset, @@ -412,6 +415,8 @@ static void dcp_link_handle_callback(struct apple_dcp *dcp, u64 message) if (ret) { dev_warn(dcp->dev, "DCPLink callback could not be decoded: %d\n", ret); + dcp_send_message(dcp, APPLE_DCP_LINK_ENDPOINT, + apple_dcp_link_rpc_reply(message)); return; } @@ -432,6 +437,8 @@ static void dcp_link_handle_callback(struct apple_dcp *dcp, u64 message) if (tag < 0 || tag >= IOMFB_MAX_CB) { dev_warn(dcp->dev, "DCPLink callback tag %#x is invalid\n", view.call); + dcp_send_message(dcp, APPLE_DCP_LINK_ENDPOINT, + apple_dcp_link_rpc_reply(message)); return; } diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 13271aa23e573..1c2ccf5f2be65 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -7,6 +7,22 @@ #include "iomfb_v13_3.h" #include "version_utils.h" +struct dcp_h17p_hotplug_request { + __le64 connected; + u8 tiled_display[0x4c]; + u8 tiled_display_null; + u8 padding[3]; +} __packed; + +struct dcp_h17p_swap_info_request { + u8 swap_info[0xe0]; + u8 swap_info_null; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_h17p_hotplug_request) == 0x58); +static_assert(sizeof(struct dcp_h17p_swap_info_request) == 0xe4); + static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A000", dcpep_late_init_signal), IOMFB_METHOD("A024", dcpep_setup_video_limits), @@ -53,8 +69,10 @@ static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, dcp->disp_bw_scratch_offset; trace_iomfb_callback(dcp, tag, __func__); - return apple_dcp_h17p_rt_bw_encode_reply(&view, scratch, - clock_request) == 0; + if (apple_dcp_h17p_rt_bw_encode_reply(&view, scratch, + clock_request)) + dev_warn(dcp->dev, "D003 bandwidth request is malformed\n"); + return true; } static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, @@ -77,6 +95,28 @@ static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, return true; } +static bool trampoline_hotplug_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_hotplug_request *request = in; + + if (!(request->tiled_display_null & 1)) + memcpy(out, request->tiled_display, + sizeof(request->tiled_display)); + return trampoline_hotplug(dcp, tag, out, in); +} + +static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_swap_info_request *request = in; + + trace_iomfb_callback(dcp, tag, __func__); + if (!(request->swap_info_null & 1)) + memcpy(out, request->swap_info, sizeof(request->swap_info)); + return true; +} + static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [0] = trampoline_true, /* did_boot_signal */ [1] = trampoline_true, /* did_power_on_signal */ @@ -128,12 +168,12 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [565] = trampoline_true, /* set_property_bool */ [567] = trampoline_true, /* set_property_str */ [572] = trampoline_zero, /* power_up_dart */ - [575] = trampoline_hotplug, + [575] = trampoline_hotplug_h17p, [576] = trampoline_nop, /* powerstate_notify */ [586] = trampoline_true, /* create_default_fb_surface */ [587] = trampoline_nop, /* clear_default_surface */ [588] = trampoline_nop, /* swap_notify */ - [589] = trampoline_nop, /* swap_info_notify */ + [589] = trampoline_swap_info_h17p, [594] = trampoline_swap_complete, [596] = trampoline_swap_complete_intent_gated, [597] = trampoline_abort_swap_ap_gated, From fa5302a470f11bb93166fbee5486877cfb019937 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 01:09:55 -0400 Subject: [PATCH 0205/1579] drm/apple: restart inherited H17P DCP sessions --- drivers/gpu/drm/apple/dcp-internal.h | 4 +- drivers/gpu/drm/apple/dcp-lifecycle-test.c | 26 ++++++++- drivers/gpu/drm/apple/dcp-lifecycle.h | 16 ++++++ drivers/gpu/drm/apple/dcp.c | 64 ++++++++++++++-------- 4 files changed, 84 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 8a8b2f582e983..251a53b21a2ae 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -149,7 +149,7 @@ struct dcp_panel { struct apple_dcp_hw_data { u32 num_dptx_ports; - bool reject_running_coprocessor; + bool adopt_running_coprocessor; }; /* TODO: move IOMFB members to its own struct */ @@ -174,6 +174,8 @@ struct apple_dcp { u32 crash_count; DECLARE_BITMAP(iomfb_surfaces, DCP_MAX_PLANES); + /* Ignore inherited application-endpoint traffic while quiescing RTKit. */ + bool adopting_session; /************* IOMFB ************************************************** * everything below is mostly used inside IOMFB but it could make * diff --git a/drivers/gpu/drm/apple/dcp-lifecycle-test.c b/drivers/gpu/drm/apple/dcp-lifecycle-test.c index 1ca6af6848632..184eba562c91c 100644 --- a/drivers/gpu/drm/apple/dcp-lifecycle-test.c +++ b/drivers/gpu/drm/apple/dcp-lifecycle-test.c @@ -41,12 +41,36 @@ static void apple_dcp_zero_status_is_active_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, apple_dcp_coprocessor_is_active(0, 0)); } +static void apple_dcp_j700_adoption_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, + apple_dcp_coprocessor_start_mode(true, 0x10, 0x6d), + APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART); + KUNIT_EXPECT_EQ(test, + apple_dcp_coprocessor_start_mode(true, 0x10, 0x6c), + APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART); +} + +static void apple_dcp_cold_start_selection_test(struct kunit *test) +{ + u32 stopped = APPLE_DCP_COPROC_CPU_STATUS_STOPPED; + + KUNIT_EXPECT_EQ(test, + apple_dcp_coprocessor_start_mode(true, 0, stopped), + APPLE_DCP_COPROCESSOR_COLD_START); + KUNIT_EXPECT_EQ(test, + apple_dcp_coprocessor_start_mode(false, 0x10, 0x6d), + APPLE_DCP_COPROCESSOR_COLD_START); +} + static struct kunit_case apple_dcp_lifecycle_test_cases[] = { KUNIT_CASE(apple_dcp_cold_coprocessor_test), KUNIT_CASE(apple_dcp_control_run_test), KUNIT_CASE(apple_dcp_status_running_test), KUNIT_CASE(apple_dcp_j700_live_state_test), KUNIT_CASE(apple_dcp_zero_status_is_active_test), + KUNIT_CASE(apple_dcp_j700_adoption_test), + KUNIT_CASE(apple_dcp_cold_start_selection_test), {} }; @@ -59,4 +83,4 @@ kunit_test_suite(apple_dcp_lifecycle_test_suite); MODULE_AUTHOR("The Asahi Linux Contributors"); MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("Apple DCP lifecycle admission KUnit tests"); +MODULE_DESCRIPTION("Apple DCP lifecycle selection KUnit tests"); diff --git a/drivers/gpu/drm/apple/dcp-lifecycle.h b/drivers/gpu/drm/apple/dcp-lifecycle.h index 31cd8d8abb8bd..a794f6fea7730 100644 --- a/drivers/gpu/drm/apple/dcp-lifecycle.h +++ b/drivers/gpu/drm/apple/dcp-lifecycle.h @@ -11,6 +11,11 @@ #define APPLE_DCP_COPROC_CPU_STATUS_RUNNING BIT(0) #define APPLE_DCP_COPROC_CPU_STATUS_STOPPED BIT(1) +enum apple_dcp_coprocessor_start { + APPLE_DCP_COPROCESSOR_COLD_START, + APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART, +}; + static inline bool apple_dcp_coprocessor_is_active(u32 cpu_control, u32 cpu_status) { if (cpu_control & APPLE_DCP_COPROC_CPU_CONTROL_RUN || @@ -21,4 +26,15 @@ static inline bool apple_dcp_coprocessor_is_active(u32 cpu_control, u32 cpu_stat return !(cpu_status & APPLE_DCP_COPROC_CPU_STATUS_STOPPED); } +static inline enum apple_dcp_coprocessor_start +apple_dcp_coprocessor_start_mode(bool can_adopt, u32 cpu_control, + u32 cpu_status) +{ + if (can_adopt && + apple_dcp_coprocessor_is_active(cpu_control, cpu_status)) + return APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART; + + return APPLE_DCP_COPROCESSOR_COLD_START; +} + #endif /* __APPLE_DCP_LIFECYCLE_H__ */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8ab177a893721..de11294e8493c 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1229,6 +1229,8 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) struct apple_dcp *dcp = cookie; trace_dcp_recv_msg(dcp, endpoint, message); + if (dcp->adopting_session) + return; switch (endpoint) { case IOMFB_ENDPOINT: @@ -2210,6 +2212,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; struct apple_dcp *dcp = dev_get_drvdata(dev); + enum apple_dcp_coprocessor_start start_mode; u32 cpu_ctrl, cpu_status; int ret; @@ -2221,22 +2224,13 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (IS_ERR(dcp->coproc_reg)) return PTR_ERR(dcp->coproc_reg); - /* - * ASCWrap v6 can be handed to Linux with a live firmware-owned RTKit - * session. The cold-start path below has no protocol for adopting its - * endpoint map or IOMFB shared memory, so touching PIODMA/DART state or - * sending a new RTKit INIT would corrupt the inherited display session. - */ - if (dcp->hw.reject_running_coprocessor) { - cpu_ctrl = readl_relaxed(dcp->coproc_reg + - APPLE_DCP_COPROC_CPU_CONTROL); - cpu_status = readl_relaxed(dcp->coproc_reg + - APPLE_DCP_COPROC_CPU_STATUS); - if (apple_dcp_coprocessor_is_active(cpu_ctrl, cpu_status)) - return dev_err_probe(dev, -EBUSY, - "refusing cold start of running firmware-owned DCP (control=%#x status=%#x)\n", - cpu_ctrl, cpu_status); - } + cpu_ctrl = readl_relaxed(dcp->coproc_reg + + APPLE_DCP_COPROC_CPU_CONTROL); + cpu_status = readl_relaxed(dcp->coproc_reg + + APPLE_DCP_COPROC_CPU_STATUS); + start_mode = apple_dcp_coprocessor_start_mode( + dcp->hw.adopt_running_coprocessor, cpu_ctrl, cpu_status); + if (dcp->index || dcp->dptx_phy || dcp->dptx_die) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); @@ -2310,16 +2304,38 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); - cpu_ctrl = - readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + if (start_mode == APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART) { + dev_info(dev, + "quiescing inherited DCP session (control=%#x status=%#x)\n", + cpu_ctrl, cpu_status); + dcp->adopting_session = true; + dcp->rtk = devm_apple_rtkit_init_adopted(dev, dcp, "mbox", 0, + &rtkit_ops); + if (IS_ERR(dcp->rtk)) { + dcp->adopting_session = false; + return dev_err_probe(dev, PTR_ERR(dcp->rtk), + "Failed to adopt RTKit\n"); + } + + ret = apple_rtkit_quiesce(dcp->rtk); + dcp->adopting_session = false; + if (ret) + return dev_err_probe(dev, ret, + "Failed to quiesce inherited RTKit: %d\n", + ret); + } else { + dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, + &rtkit_ops); + if (IS_ERR(dcp->rtk)) + return dev_err_probe(dev, PTR_ERR(dcp->rtk), + "Failed to initialize RTKit\n"); + } + + cpu_ctrl = readl_relaxed(dcp->coproc_reg + + APPLE_DCP_COPROC_CPU_CONTROL); writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); - dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); - if (IS_ERR(dcp->rtk)) - return dev_err_probe(dev, PTR_ERR(dcp->rtk), - "Failed to initialize RTKit\n"); - ret = apple_rtkit_wake(dcp->rtk); if (ret) return dev_err_probe(dev, ret, @@ -2648,7 +2664,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { .num_dptx_ports = 0, - .reject_running_coprocessor = true, + .adopt_running_coprocessor = true, }; static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { From 61495bbc4e8cb56b865713cd5e8b3b2deebb3014 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 06:10:11 -0400 Subject: [PATCH 0206/1579] drm/apple: consume quiesced DCP loader handoff --- drivers/gpu/drm/apple/dcp-lifecycle-test.c | 16 ++++++++++++---- drivers/gpu/drm/apple/dcp-lifecycle.h | 8 ++++++-- drivers/gpu/drm/apple/dcp.c | 14 +++++++++++++- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-lifecycle-test.c b/drivers/gpu/drm/apple/dcp-lifecycle-test.c index 184eba562c91c..196e5bb7e2324 100644 --- a/drivers/gpu/drm/apple/dcp-lifecycle-test.c +++ b/drivers/gpu/drm/apple/dcp-lifecycle-test.c @@ -44,22 +44,29 @@ static void apple_dcp_zero_status_is_active_test(struct kunit *test) static void apple_dcp_j700_adoption_test(struct kunit *test) { KUNIT_EXPECT_EQ(test, - apple_dcp_coprocessor_start_mode(true, 0x10, 0x6d), + apple_dcp_coprocessor_start_mode(false, true, 0x10, 0x6d), APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART); KUNIT_EXPECT_EQ(test, - apple_dcp_coprocessor_start_mode(true, 0x10, 0x6c), + apple_dcp_coprocessor_start_mode(false, true, 0x10, 0x6c), APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART); } +static void apple_dcp_j700_quiesced_handoff_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, + apple_dcp_coprocessor_start_mode(true, true, 0x10, 0x6d), + APPLE_DCP_COPROCESSOR_QUIESCED_RESTART); +} + static void apple_dcp_cold_start_selection_test(struct kunit *test) { u32 stopped = APPLE_DCP_COPROC_CPU_STATUS_STOPPED; KUNIT_EXPECT_EQ(test, - apple_dcp_coprocessor_start_mode(true, 0, stopped), + apple_dcp_coprocessor_start_mode(false, true, 0, stopped), APPLE_DCP_COPROCESSOR_COLD_START); KUNIT_EXPECT_EQ(test, - apple_dcp_coprocessor_start_mode(false, 0x10, 0x6d), + apple_dcp_coprocessor_start_mode(false, false, 0x10, 0x6d), APPLE_DCP_COPROCESSOR_COLD_START); } @@ -70,6 +77,7 @@ static struct kunit_case apple_dcp_lifecycle_test_cases[] = { KUNIT_CASE(apple_dcp_j700_live_state_test), KUNIT_CASE(apple_dcp_zero_status_is_active_test), KUNIT_CASE(apple_dcp_j700_adoption_test), + KUNIT_CASE(apple_dcp_j700_quiesced_handoff_test), KUNIT_CASE(apple_dcp_cold_start_selection_test), {} }; diff --git a/drivers/gpu/drm/apple/dcp-lifecycle.h b/drivers/gpu/drm/apple/dcp-lifecycle.h index a794f6fea7730..39723db98a373 100644 --- a/drivers/gpu/drm/apple/dcp-lifecycle.h +++ b/drivers/gpu/drm/apple/dcp-lifecycle.h @@ -14,6 +14,7 @@ enum apple_dcp_coprocessor_start { APPLE_DCP_COPROCESSOR_COLD_START, APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART, + APPLE_DCP_COPROCESSOR_QUIESCED_RESTART, }; static inline bool apple_dcp_coprocessor_is_active(u32 cpu_control, u32 cpu_status) @@ -27,9 +28,12 @@ static inline bool apple_dcp_coprocessor_is_active(u32 cpu_control, u32 cpu_stat } static inline enum apple_dcp_coprocessor_start -apple_dcp_coprocessor_start_mode(bool can_adopt, u32 cpu_control, - u32 cpu_status) +apple_dcp_coprocessor_start_mode(bool loader_quiesced, bool can_adopt, + u32 cpu_control, u32 cpu_status) { + if (loader_quiesced) + return APPLE_DCP_COPROCESSOR_QUIESCED_RESTART; + if (can_adopt && apple_dcp_coprocessor_is_active(cpu_control, cpu_status)) return APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index de11294e8493c..bbdf5641b9b28 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -56,6 +57,7 @@ #define APPLE_DCP_COPROC_CPU_CONTROL 0x44 #define APPLE_DCP_COPROC_CPU_STATUS 0x48 +#define APPLE_DCP_RTKIT_HANDOFF_PROPERTY "apple,dcp-rtkit-quiesced" #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) @@ -2214,6 +2216,8 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) struct apple_dcp *dcp = dev_get_drvdata(dev); enum apple_dcp_coprocessor_start start_mode; u32 cpu_ctrl, cpu_status; + u32 handoff_state; + bool loader_quiesced; int ret; ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); @@ -2228,8 +2232,14 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) APPLE_DCP_COPROC_CPU_CONTROL); cpu_status = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS); + loader_quiesced = of_chosen && + !of_property_read_u32(of_chosen, + APPLE_DCP_RTKIT_HANDOFF_PROPERTY, + &handoff_state) && + handoff_state == 1; start_mode = apple_dcp_coprocessor_start_mode( - dcp->hw.adopt_running_coprocessor, cpu_ctrl, cpu_status); + loader_quiesced, dcp->hw.adopt_running_coprocessor, + cpu_ctrl, cpu_status); if (dcp->index || dcp->dptx_phy || dcp->dptx_die) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", @@ -2324,6 +2334,8 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) "Failed to quiesce inherited RTKit: %d\n", ret); } else { + if (start_mode == APPLE_DCP_COPROCESSOR_QUIESCED_RESTART) + dev_info(dev, "restarting loader-quiesced DCP session\n"); dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); if (IS_ERR(dcp->rtk)) From 5d72c8267943a000a770ef0c976bf803497c1226 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 28 Aug 2026 06:12:44 -0400 Subject: [PATCH 0207/1579] drm/apple: describe the J700 scanout profile --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/apple_drv.c | 24 ++++--- drivers/gpu/drm/apple/dcp-internal.h | 5 +- drivers/gpu/drm/apple/dcp.c | 76 +++++++++++++--------- drivers/gpu/drm/apple/dcp.h | 2 +- drivers/gpu/drm/apple/iomfb-surface-test.c | 68 +++++++++++++++++++ drivers/gpu/drm/apple/iomfb-surface.h | 54 +++++++++++++++ drivers/gpu/drm/apple/iomfb.h | 1 - drivers/gpu/drm/apple/iomfb_template.c | 4 +- drivers/gpu/drm/apple/plane.c | 4 +- drivers/gpu/drm/apple/plane.h | 4 +- 11 files changed, 191 insertions(+), 52 deletions(-) create mode 100644 drivers/gpu/drm/apple/iomfb-surface-test.c create mode 100644 drivers/gpu/drm/apple/iomfb-surface.h diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 67623ce156bc0..fb6cc77fc6cfa 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -20,3 +20,4 @@ appledrm-$(CONFIG_TRACING) += trace.o obj-$(CONFIG_DRM_APPLE) += appledrm.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp-lifecycle-test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp-link-test.o +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb-surface-test.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 4c6fa62321a25..cb9036de3d610 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -59,7 +59,7 @@ static int apple_drm_gem_dumb_create(struct drm_file *file_priv, struct drm_device *drm, struct drm_mode_create_dumb *args) { - args->pitch = ALIGN(DIV_ROUND_UP(args->width * args->bpp, 8), 64); + args->pitch = apple_dcp_linear_pitch(args->width, args->bpp); args->size = round_up(args->pitch * args->height, DART_PAGE_SIZE); return drm_gem_dma_dumb_create_internal(file_priv, drm, args); @@ -356,26 +356,24 @@ static int apple_probe_per_dcp(struct device *dev, struct apple_crtc *crtc; struct apple_encoder *enc; struct drm_plane *planes[DCP_MAX_PLANES]; - unsigned long *iomfb_surfaces = dcp_get_iomfb_surfaces(dcp); - int ret; + const u32 *iomfb_surfaces; + enum drm_plane_type type; + u32 num_surfaces; + int ret, zpos; int connector_type; - u32 surf; - int zpos = 0; bool supports_l10r = !dcp_fw_compat_is_12_x(dcp); - enum drm_plane_type plane_type; - for_each_set_bit(surf, iomfb_surfaces, DCP_MAX_PLANES) { - plane_type = (zpos == 0) ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY; - planes[zpos] = apple_plane_init(drm, 1U << num, surf, - supports_l10r, plane_type); + iomfb_surfaces = dcp_get_iomfb_surfaces(dcp, &num_surfaces); + for (zpos = 0; zpos < num_surfaces; zpos++) { + type = zpos ? DRM_PLANE_TYPE_OVERLAY : DRM_PLANE_TYPE_PRIMARY; + planes[zpos] = apple_plane_init(drm, 1U << num, + iomfb_surfaces[zpos], + supports_l10r, type); if (IS_ERR(planes[zpos])) return PTR_ERR(planes[zpos]); - ret = drm_plane_create_zpos_immutable_property(planes[zpos], zpos); if (ret) return ret; - - zpos++; } /* diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 251a53b21a2ae..e0671448be025 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -21,9 +21,10 @@ #include "iomfb_h17p.h" #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" +#include "iomfb-surface.h" #include "epic/dpavservep.h" -#define DCP_MAX_PLANES 4 +#define DCP_MAX_PLANES APPLE_DCP_MAX_SURFACES #define DCP_MAX_TYPEC_ROUTES 4 struct apple_dcp; @@ -173,9 +174,9 @@ struct apple_dcp { bool crashed; u32 crash_count; - DECLARE_BITMAP(iomfb_surfaces, DCP_MAX_PLANES); /* Ignore inherited application-endpoint traffic while quiescing RTKit. */ bool adopting_session; + struct apple_dcp_surface_order iomfb_surfaces; /************* IOMFB ************************************************** * everything below is mostly used inside IOMFB but it could make * diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index bbdf5641b9b28..fc59c208d36cd 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1665,11 +1665,12 @@ bool dcp_fw_compat_is_12_x(struct platform_device *pdev) return dcp->fw_compat == DCP_FIRMWARE_V_12_3; } -unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev) +const u32 *dcp_get_iomfb_surfaces(struct platform_device *pdev, u32 *count) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - return dcp->iomfb_surfaces; + *count = dcp->iomfb_surfaces.count; + return dcp->iomfb_surfaces.id; } int dcp_start(struct platform_device *pdev) @@ -2431,9 +2432,11 @@ static int dcp_platform_probe(struct platform_device *pdev) enum dcp_firmware_version fw_compat; struct device *dev = &pdev->dev; struct apple_dcp *dcp; - int ret, surf, num_surfs; - u32 surf_en; + int num_surfaces; + u32 surface_enabled, i, count; + u32 surface_ids[DCP_MAX_PLANES]; u32 mux_index; + int ret; fw_compat = dcp_check_firmware_version(dev); if (fw_compat == DCP_FIRMWARE_UNKNOWN) @@ -2473,6 +2476,46 @@ static int dcp_platform_probe(struct platform_device *pdev) mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->tb_lock); + if (fw_compat != DCP_FIRMWARE_H17P) { + /* Legacy device trees encode one enable flag per surface. */ + count = 0; + i = 0; + if (!of_property_present(dev->of_node, "apple,iomfb-surfaces") || + !of_property_count_u32_elems(dev->of_node, "apple,iomfb-surfaces")) { + surface_ids[count++] = 0; + surface_ids[count++] = 1; + } else { + of_property_for_each_u32(dev->of_node, "apple,iomfb-surfaces", surface_enabled) { + if (i >= DCP_MAX_PLANES) + return -EINVAL; + if (surface_enabled) + surface_ids[count++] = i; + i++; + } + } + ret = apple_dcp_surface_order_init(&dcp->iomfb_surfaces, surface_ids, count); + } else if (!of_property_present(dev->of_node, "apple,iomfb-surfaces")) { + ret = apple_dcp_surface_order_init(&dcp->iomfb_surfaces, NULL, 0); + } else { + num_surfaces = of_property_count_u32_elems(dev->of_node, + "apple,iomfb-surfaces"); + if (num_surfaces <= 0 || num_surfaces > DCP_MAX_PLANES) + return dev_err_probe(dev, + num_surfaces < 0 ? num_surfaces : -EINVAL, + "invalid apple,iomfb-surfaces length %d\n", + num_surfaces); + ret = of_property_read_u32_array(dev->of_node, + "apple,iomfb-surfaces", + surface_ids, num_surfaces); + if (!ret) + ret = apple_dcp_surface_order_init(&dcp->iomfb_surfaces, + surface_ids, + num_surfaces); + } + if (ret) + return dev_err_probe(dev, ret, + "invalid apple,iomfb-surfaces order\n"); + platform_set_drvdata(pdev, dcp); dcp->phy = devm_phy_optional_get(dev, "dp-phy"); @@ -2482,31 +2525,6 @@ static int dcp_platform_probe(struct platform_device *pdev) } dcp->fixed_phy = dcp->phy; - bitmap_zero(dcp->iomfb_surfaces, DCP_MAX_PLANES); - if (!of_property_present(dev->of_node, "apple,iomfb-surfaces")) - num_surfs = 0; - else - num_surfs = of_property_count_elems_of_size(dev->of_node, - "apple,iomfb-surfaces", - sizeof(u32)); - - if (num_surfs == 0 || num_surfs == -ENODATA) { - set_bit(0, dcp->iomfb_surfaces); - set_bit(1, dcp->iomfb_surfaces); - } else if (num_surfs < 0) { - return num_surfs; - } else if (num_surfs > DCP_MAX_PLANES) { - dev_err(dev, "Number of iomfb-surfaces (%d) exceeds DCP_MAX_PLANES\n", - num_surfs); - return -EINVAL; - } - - surf = 0; - of_property_for_each_u32(dev->of_node, "apple,iomfb-surfaces", surf_en) { - if (surf_en) - set_bit(surf, dcp->iomfb_surfaces); - surf++; - } if (dcp->phy) { int ret; diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 2f044c8e2abcb..189110587eb1d 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -46,7 +46,7 @@ struct device_node *dcp_typec_port_of_node(unsigned int idx); bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev); void dcp_typec_port_set_connector(unsigned int idx, struct apple_connector *connector); bool dcp_fw_compat_is_12_x(struct platform_device *pdev); -unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); +const u32 *dcp_get_iomfb_surfaces(struct platform_device *pdev, u32 *count); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, struct apple_connector *connector); int dcp_start(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/iomfb-surface-test.c b/drivers/gpu/drm/apple/iomfb-surface-test.c new file mode 100644 index 0000000000000..0b43d65fbbb74 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-surface-test.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include + +#include "iomfb-surface.h" + +static void apple_dcp_legacy_surface_order_test(struct kunit *test) +{ + struct apple_dcp_surface_order order = {}; + + KUNIT_ASSERT_EQ(test, apple_dcp_surface_order_init(&order, NULL, 0), 0); + KUNIT_EXPECT_EQ(test, order.count, 2U); + KUNIT_EXPECT_EQ(test, order.id[0], 2U); + KUNIT_EXPECT_EQ(test, order.id[1], 1U); +} + +static void apple_dcp_j700_surface_order_test(struct kunit *test) +{ + const u32 ids[] = { 0 }; + struct apple_dcp_surface_order order = {}; + + KUNIT_ASSERT_EQ(test, + apple_dcp_surface_order_init(&order, ids, ARRAY_SIZE(ids)), + 0); + KUNIT_EXPECT_EQ(test, order.count, 1U); + KUNIT_EXPECT_EQ(test, order.id[0], 0U); +} + +static void apple_dcp_invalid_surface_order_test(struct kunit *test) +{ + const u32 duplicate[] = { 2, 2 }; + const u32 out_of_range[] = { APPLE_DCP_MAX_SURFACES }; + struct apple_dcp_surface_order order = {}; + + KUNIT_EXPECT_EQ(test, + apple_dcp_surface_order_init(&order, duplicate, + ARRAY_SIZE(duplicate)), + -EINVAL); + KUNIT_EXPECT_EQ(test, + apple_dcp_surface_order_init(&order, out_of_range, + ARRAY_SIZE(out_of_range)), + -EINVAL); +} + +static void apple_dcp_j700_pitch_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, apple_dcp_linear_pitch(2408, 32), 9664U); +} + +static struct kunit_case apple_dcp_surface_test_cases[] = { + KUNIT_CASE(apple_dcp_legacy_surface_order_test), + KUNIT_CASE(apple_dcp_j700_surface_order_test), + KUNIT_CASE(apple_dcp_invalid_surface_order_test), + KUNIT_CASE(apple_dcp_j700_pitch_test), + {} +}; + +static struct kunit_suite apple_dcp_surface_test_suite = { + .name = "apple-dcp-surface", + .test_cases = apple_dcp_surface_test_cases, +}; + +kunit_test_suite(apple_dcp_surface_test_suite); + +MODULE_AUTHOR("The Asahi Linux Contributors"); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple DCP surface order KUnit tests"); diff --git a/drivers/gpu/drm/apple/iomfb-surface.h b/drivers/gpu/drm/apple/iomfb-surface.h new file mode 100644 index 0000000000000..4db5a25cf3327 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-surface.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_IOMFB_SURFACE_H__ +#define __APPLE_IOMFB_SURFACE_H__ + +#include +#include +#include + +#define APPLE_DCP_MAX_SURFACES 4 +#define APPLE_DCP_SCANOUT_PITCH_ALIGN 64 + +struct apple_dcp_surface_order { + u32 count; + u32 id[APPLE_DCP_MAX_SURFACES]; +}; + +static inline u32 apple_dcp_linear_pitch(u32 width, u32 bpp) +{ + return ALIGN(DIV_ROUND_UP(width * bpp, 8), + APPLE_DCP_SCANOUT_PITCH_ALIGN); +} + +static inline int +apple_dcp_surface_order_init(struct apple_dcp_surface_order *order, + const u32 *ids, u32 count) +{ + static const u32 legacy_order[] = { 2, 1 }; + u32 seen = 0; + u32 i; + + if (!order) + return -EINVAL; + + if (!ids) { + ids = legacy_order; + count = ARRAY_SIZE(legacy_order); + } + if (!count || count > APPLE_DCP_MAX_SURFACES) + return -EINVAL; + + for (i = 0; i < count; i++) { + if (ids[i] >= APPLE_DCP_MAX_SURFACES || (seen & BIT(ids[i]))) + return -EINVAL; + seen |= BIT(ids[i]); + order->id[i] = ids[i]; + } + order->count = count; + + return 0; +} + +#endif /* __APPLE_IOMFB_SURFACE_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 7903fad404067..ec7e23b4343e5 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -86,7 +86,6 @@ enum iomfb_property_id { /* Structures used in v12.0 firmware */ #define SWAP_SURFACES 4 - struct dcp_iouserclient { /* Handle for the IOUserClient. macOS sets this to a kernel VA. */ u64 handle; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index f92f6c9afd20d..a41dfa6aaeb0f 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1416,13 +1416,13 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru for_each_oldnew_plane_in_state(state, plane, old_state, new_state, plane_idx) { struct apple_plane_state *apple_state = to_apple_plane_state(new_state); - struct apple_plane *apl_plane = to_apple_plane(plane); + struct apple_plane *apple_plane = to_apple_plane(plane); /* skip planes not for this crtc */ if (old_state->crtc != crtc && new_state->crtc != crtc) continue; - l = apl_plane->iomfb_surf; + l = apple_plane->iomfb_surface; req->swap.swap_enabled |= BIT(l); if (old_state->fb && new_state->fb != old_state->fb) { diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index 51a6c2dbd7a39..dade670e320f1 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -432,7 +432,7 @@ u64 apple_format_modifiers[] = { struct drm_plane *apple_plane_init(struct drm_device *dev, unsigned long possible_crtcs, - u32 iomfb_surf, + u32 iomfb_surface, bool supports_l10r, enum drm_plane_type type) { @@ -483,8 +483,8 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, drm_plane_helper_add(&plane->base, &apple_primary_plane_helper_funcs); else drm_plane_helper_add(&plane->base, &apple_plane_helper_funcs); + plane->iomfb_surface = iomfb_surface; - plane->iomfb_surf = iomfb_surf; return &plane->base; } diff --git a/drivers/gpu/drm/apple/plane.h b/drivers/gpu/drm/apple/plane.h index c5db78144d1f8..13bb37f4d0b09 100644 --- a/drivers/gpu/drm/apple/plane.h +++ b/drivers/gpu/drm/apple/plane.h @@ -14,7 +14,7 @@ struct apple_plane { struct drm_plane base; - u32 iomfb_surf; + u32 iomfb_surface; }; #define to_apple_plane(x) container_of(x, struct apple_plane, base) @@ -31,7 +31,7 @@ struct apple_plane_state { struct drm_plane *apple_plane_init(struct drm_device *dev, unsigned long possible_crtcs, - u32 iomfb_surf, + u32 iomfb_surface, bool supports_l10r, enum drm_plane_type type); From 2580ca9458656d246edb5a2e9c527e3ae60eaccc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 6 Sep 2026 13:59:44 -0400 Subject: [PATCH 0208/1579] drm/apple: fix the H17P swap submit method and record layout macOS 25G83 traces captured under the m1n1 hypervisor on J700 show the H17P swap path does not match what this driver sends. Three coupled errors, all following from the assumption that H17P shifted uniformly from v13_3: 1. swap_submit was mapped to A407. swap_start really did move A407 -> A406, but swap_submit did not move with it. Across 20 traced boots A407 is never called once; every swap is A406 then A408 (1579 of them in the longest run), and the A408 response is 0x0c bytes, matching dcp_swap_submit_resp_h17p exactly. 2. The 26.0.0 gate added a surf2[6] block. A captured A408 request ends in an 8-byte run of u8 null flags - swap_null + surf_null[4] + the three tail bools - where any surf2-bearing layout would add five or six more. 3. h17p_tail was 0x278, making dcp_swap 0x6e0. A captured request is 0xe9c bytes; with dcp_surface_h17p = 0x22c, SWAP_SURFACES = 4 and no surf2 block that leaves exactly 0x122. 0x6e0 is arithmetically impossible - it forces a non-integer surface size. dcp_swap_h17p is now 0x58a and dcp_swap_submit_req_h17p is 0xe9c, the exact size the firmware receives. Verified by compiling the struct definitions standalone; the untouched dcp_surface_h17p still lands on its previously asserted 0x22c. Derived from captured traces and static RE; not yet exercised on hardware. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CaX7H14UyyEEeDHEEVLNgr --- drivers/gpu/drm/apple/iomfb_h17p.c | 6 +++++- drivers/gpu/drm/apple/iomfb_h17p.h | 4 ++-- drivers/gpu/drm/apple/iomfb_template.c | 4 ++++ drivers/gpu/drm/apple/iomfb_template.h | 18 ++++++++++-------- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 1c2ccf5f2be65..d0db6103e173f 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -32,7 +32,11 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A390", iomfbep_a358_vi_set_temperature_hint), IOMFB_METHOD("A401", dcpep_start_signal), IOMFB_METHOD("A406", dcpep_swap_start), - IOMFB_METHOD("A407", dcpep_swap_submit), + /* swap_start moved A407 -> A406 on H17P, but swap_submit did NOT move + * with it: macOS 25G83 issues A406 then A408 for every swap. Across 20 + * traced boots A407 is never called once, and the A408 response is 0x0c + * bytes, matching dcp_swap_submit_resp_h17p exactly. */ + IOMFB_METHOD("A408", dcpep_swap_submit), IOMFB_METHOD("A411", dcpep_set_display_device), IOMFB_METHOD("A412", dcpep_is_main_display), IOMFB_METHOD("A413", dcpep_set_digital_out_mode), diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 58d7033ab1b60..f6a94713592b9 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -11,9 +11,9 @@ #include "iomfb_template.h" -static_assert(sizeof(struct dcp_swap_h17p) == 0x6e0); +static_assert(sizeof(struct dcp_swap_h17p) == 0x58a); static_assert(sizeof(struct dcp_surface_h17p) == 0x22c); -static_assert(sizeof(struct dcp_swap_submit_req_h17p) == 0x1d30); +static_assert(sizeof(struct dcp_swap_submit_req_h17p) == 0xe9c); static_assert(sizeof(struct dcp_swap_submit_resp_h17p) == 0x0c); #undef DCP_FW_VER diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index a41dfa6aaeb0f..e2942b8869220 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -937,8 +937,10 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) for (int l = 0; l < SWAP_SURFACES; l++) swap->surf_null[l] = true; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) for (int l = 0; l < ARRAY_SIZE(swap->surf2_null); l++) swap->surf2_null[l] = true; +#endif swap->unkU32Ptr_null = true; swap->unkU32out_null = true; #endif @@ -1398,8 +1400,10 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru for (l = 0; l < SWAP_SURFACES; l++) req->surf_null[l] = true; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) for (l = 0; l < ARRAY_SIZE(req->surf2_null); l++) req->surf2_null[l] = true; +#endif req->unkU32Ptr_null = true; req->unkU32out_null = true; #endif diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 2298c5bf2156e..6ba3b83d4a9a1 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -54,7 +54,11 @@ struct DCP_FW_NAME(dcp_swap) { u64 unk_1; #endif #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - u8 h17p_tail[0x278]; + /* Sized from the wire: a captured A408 request is 0xe9c bytes, which with + * dcp_surface_h17p = 0x22c, SWAP_SURFACES = 4 and no surf2 block leaves + * exactly 0x122 here. 0x278 makes dcp_swap 0x6e0 and forces a + * non-integer surface size, so it cannot be right. */ + u8 h17p_tail[0x122]; #endif } __packed; @@ -76,10 +80,10 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { u64 surf_iova[SWAP_SURFACES]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u64 unk_u64_a[SWAP_SURFACES]; -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - struct DCP_FW_NAME(dcp_surface) surf2[6]; - u64 surf2_iova[6]; -#else +/* H17P (26.x) carries no surf2 block: the trailing u8 null-flag run in a + * captured A408 record is 8 bytes (swap_null + surf_null[4] + three tail + * bools), where a surf2-bearing layout would add five or six more. */ +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) struct DCP_FW_NAME(dcp_surface) surf2[5]; u64 surf2_iova[5]; #endif @@ -97,9 +101,7 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { u8 swap_null; u8 surf_null[SWAP_SURFACES]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - u8 surf2_null[6]; -#else +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 surf2_null[5]; #endif #endif From 327ad4956dce4ec7d3fefd812a3dccf2de4983bd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 6 Sep 2026 15:05:06 -0400 Subject: [PATCH 0209/1579] drm/apple: derive the disp register count from reg-names dcp_get_disp_regs() assumed the reg layout is [coproc, disp-0 ...], deriving the display register count as num_resources - 1. H17P/T8140 has an extra trailing register: [coproc, disp-0 .. disp-4, iop-vbar]. The count therefore came out as 6 instead of 5, which broke the probe twice over: the index loop bound disp_registers[5] to iop-vbar, and the inflated count made the bw-scratch check reject the device tree's correct disp_reg of 5. Observed on J700 the first time the driver was able to probe at all: apple-dcp 412e00000.dcp: Unexpected disp_reg value in 'apple,bw-scratch': 5 apple-dcp 412e00000.dcp: failed to find display registers apple-drm soc:display-subsystem: failed to bind 412e00000.dcp: -22 Count the "disp-" entries in reg-names instead and fetch them by name, so the layout no longer has to be positional. Device trees without reg-names keep the historical behaviour, and a missing register is now reported rather than silently left NULL. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CaX7H14UyyEEeDHEEVLNgr --- drivers/gpu/drm/apple/dcp.c | 58 +++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index fc59c208d36cd..e2889ddaa3aaf 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2084,18 +2084,70 @@ static int dcp_get_bw_doorbell_reg(struct apple_dcp *dcp, u32 expected) return ret; } +/* + * Count the "disp-*" entries in reg-names. + * + * The positional form below assumes the reg layout is [coproc, disp-0 ...], + * i.e. exactly one non-disp register, and derives the count as + * num_resources - 1. That is wrong on H17P/T8140, whose layout is + * [coproc, disp-0 .. disp-4, iop-vbar]: the trailing iop-vbar inflates the + * count to 6 and the index loop then binds disp_registers[5] to iop-vbar. + * The inflated count also makes the "apple,bw-scratch" disp_reg check reject + * the device tree's (correct) value of 5, which fails the probe with + * Unexpected disp_reg value in 'apple,bw-scratch': 5 + * failed to find display registers + * + * Returns a negative errno when reg-names is absent, so callers fall back. + */ +static int dcp_count_disp_regs(struct device *dev) +{ + int n = of_property_count_strings(dev->of_node, "reg-names"); + const char *name; + int i, count = 0; + + if (n <= 0) + return -EINVAL; + + for (i = 0; i < n; ++i) { + if (of_property_read_string_index(dev->of_node, "reg-names", i, + &name)) + return -EINVAL; + if (!strncmp(name, "disp-", 5)) + count++; + } + + return count ? count : -EINVAL; +} + static int dcp_get_disp_regs(struct apple_dcp *dcp) { struct platform_device *pdev = to_platform_device(dcp->dev); - int count = pdev->num_resources - 1; + int count = dcp_count_disp_regs(dcp->dev); + bool by_name = count > 0; int i, ret; + /* Device trees without reg-names keep the historical positional form. */ + if (!by_name) + count = pdev->num_resources - 1; + if (count <= 0 || count > MAX_DISP_REGISTERS) return -EINVAL; for (i = 0; i < count; ++i) { - dcp->disp_registers[i] = - platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); + if (by_name) { + char name[8]; + + snprintf(name, sizeof(name), "disp-%d", i); + dcp->disp_registers[i] = platform_get_resource_byname( + pdev, IORESOURCE_MEM, name); + } else { + dcp->disp_registers[i] = platform_get_resource( + pdev, IORESOURCE_MEM, 1 + i); + } + if (!dcp->disp_registers[i]) { + dev_err(dcp->dev, "missing display register %d\n", i); + return -EINVAL; + } } /* load pmgr bandwidth scratch resource and offset */ From 6d942a8f237649cd7248401d248522d2518f882b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 6 Sep 2026 15:11:59 -0400 Subject: [PATCH 0210/1579] drm/apple: adopt an existing piodma pdev instead of failing the retry dcp_create_piodma_iommu_dev() cleans up after its own failures, but the caller does not: dcp_comp_bind() creates the piodma device and then returns without destroying it if any later step fails - dcp_get_disp_regs(), the clock lookup, and so on. The device is left attached to the DT node, so of_platform_device_create() returns NULL on the next attempt and the bind fails with -ENODEV from then on, even once the original problem is gone. On J700 this turns one recoverable failure into a permanent one: piodma: Adding to iommu group 4 <- first attempt Failed to create piodma pdev for .../piodma -ENODEV <- every retry after Failed to created PIODMA iommu child device -ENODEV apple-drm soc:display-subsystem: adev bind failed: -19 Adopt the existing device with of_find_device_by_node() when creation reports it already exists, and say so. The device is destroyed on unbind as before. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CaX7H14UyyEEeDHEEVLNgr --- drivers/gpu/drm/apple/dcp.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e2889ddaa3aaf..69bc9d9b59480 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1958,8 +1958,27 @@ static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) "Failed to get piodma child DT node\n"); dcp->piodma = of_platform_device_create(node, NULL, dcp->dev); - if (!dcp->piodma) - return dev_err_probe(dcp->dev, -ENODEV, "Failed to create piodma pdev for %pOF\n", node); + if (!dcp->piodma) { + /* + * of_platform_device_create() returns NULL when the node already + * has a platform device, which happens whenever a previous bind + * attempt created it and then failed further down (the caller's + * later error paths do not destroy it). Probe is retried - a + * deferral, or a component re-bind - and would then fail here + * forever with -ENODEV even though nothing is actually wrong. + * Observed on J700 as: + * piodma: Adding to iommu group 4 <- first attempt + * Failed to create piodma pdev ... -ENODEV <- every retry + * Adopt the existing device instead of giving up. + */ + dcp->piodma = of_find_device_by_node(node); + if (!dcp->piodma) + return dev_err_probe(dcp->dev, -ENODEV, + "Failed to create piodma pdev for %pOF\n", + node); + dev_info(dcp->dev, "reusing existing piodma pdev for %pOF\n", + node); + } ret = dma_set_mask_and_coherent(&dcp->piodma->dev, DMA_BIT_MASK(42)); if (ret) From 040319c9e4e7229e84814b94ce09993320a5620b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 6 Sep 2026 16:10:35 -0400 Subject: [PATCH 0211/1579] drm/apple: shift the H17P swap record fields by 0x48 H17P inserts 0x48 bytes between flags2 and the swap ID. Without them every field from the swap ID onwards is written 0x48 bytes too early, so the firmware reads the swap id, surface ids, source and destination rectangles, surface flags, the enabled/completed words and the background colour out of the wrong place on every single swap. Live macOS 25G83 traces settle the offset: the value A406 returns as the swap id appears at +0x98 in 1919 of 1919 swaps across two independent captures, and at the pre-H17P +0x50 in 0 of 1919. Applying the shift lines up the whole block with the wire - surf_ids +0x9c, src_rect +0xac, surf_flags +0xec, dst_rect +0x10c, swap_enabled/completed +0x14c/+0x150, bg_color +0x154 - and the destination rectangle then reads as the panel's 2408x1506 in every captured swap. The header below the gap was already right: eight u64 timestamps at +0x00, then flags1 +0x40 = 0x04040008 and flags2 +0x48 = 0x4, all matching the capture. h17p_tail absorbs the shift so the record stays 0xe9c, the size the firmware receives. Verified by compiling the struct definitions standalone: dcp_swap_h17p 0x58a, dcp_surface_h17p 0x22c, dcp_swap_submit_req_h17p 0xe9c, offsetof(swap_id) 0x98. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CaX7H14UyyEEeDHEEVLNgr --- drivers/gpu/drm/apple/iomfb_template.h | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 6ba3b83d4a9a1..9817a174dad52 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -25,6 +25,20 @@ struct DCP_FW_NAME(dcp_swap) { u64 flags1; u64 flags2; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P inserts 0x48 bytes between flags2 and the swap ID. Live macOS + * 25G83 traces put the value A406 returned as the swap id at +0x98, in + * 1919 of 1919 swaps across two independent captures; at the pre-H17P + * +0x50 it matches in 0 of 1919. Every following field lines up once + * this shift is applied: surf_ids +0x9c, src_rect +0xac, surf_flags + * +0xec, dst_rect +0x10c, swap_enabled/completed +0x14c/+0x150, + * bg_color +0x154. The two u32s the firmware does keep in this gap + * (at +0x50 and +0x78) are small counters we do not model yet. + */ + u8 h17p_pre_swap_id[0x48]; +#endif + u32 swap_id; u32 surf_ids[SWAP_SURFACES]; @@ -58,7 +72,7 @@ struct DCP_FW_NAME(dcp_swap) { * dcp_surface_h17p = 0x22c, SWAP_SURFACES = 4 and no surf2 block leaves * exactly 0x122 here. 0x278 makes dcp_swap 0x6e0 and forces a * non-integer surface size, so it cannot be right. */ - u8 h17p_tail[0x122]; + u8 h17p_tail[0xda]; #endif } __packed; From 0f984d7902d3efd4635083965dc9db113db33be8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 6 Sep 2026 17:15:39 -0400 Subject: [PATCH 0212/1579] drm/apple: align DCPLink RPC records to 0x40 The record stack allocator used 4-byte alignment. Live macOS 25G83 traces show the firmware advances by align_up(12 + in_len + out_len, 0x40) - confirmed for all 81915 RPC calls across two independent captures - so every nested record was placed where the firmware does not look and its callbacks came back undecodable: DCPLink callback tag 0x0 is invalid DCPLink callback could not be decoded: -90 IOMFB: Could not find Provider service IOMFB: Could not find Product service DCP[0] not ready: -110 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CaX7H14UyyEEeDHEEVLNgr --- drivers/gpu/drm/apple/dcp-link.h | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index 7f8a48d3d5ff3..09c6a79edbae9 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -26,7 +26,18 @@ #define APPLE_DCP_LINK_STREAM_BUFFER_SIZE 0x008000 #define APPLE_DCP_LINK_HEAP_OFFSET 0x080000 #define APPLE_DCP_LINK_HEAP_SIZE 0x040000 -#define APPLE_DCP_LINK_RPC_ALIGNMENT 4 +/* + * Records are allocated on each channel's stack at + * off_next = off_cur + align_up(12 + in_len + out_len, 0x40) + * Live macOS 25G83 traces confirm this for all 81915 RPC calls across two + * independent captures. Allocating at 4-byte alignment instead put every + * nested record where the firmware does not look, so its callbacks decoded as + * tag 0: + * DCPLink callback tag 0x0 is invalid + * DCPLink callback could not be decoded: -90 + * IOMFB: Could not find Provider service + */ +#define APPLE_DCP_LINK_RPC_ALIGNMENT 0x40 #define APPLE_DCP_LINK_SHARED_VAR_MEMORY_SIZE 0x4000000 #define APPLE_DCP_LINK_INIT_FLAG_64M 0x04000000 From 087edfa6c5a0ae71a9e3bef994c89553051ed71f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 7 Sep 2026 05:26:04 -0400 Subject: [PATCH 0213/1579] drm/apple: bring up the H17P display on T8140 (Neo / A18 Pro) The internal panel on the J700 now scans out under Linux. The driver autoloads with no module parameters, the framebuffer console renders through it, and steady state is clean: no DART faults, no firmware crashes, 60 Hz vblank. Two bugs accounted for essentially all of the symptoms. Both were found by diffing our IOMFB conversation byte-for-byte against a macOS trace. surf_iova was two bytes late in the swap record. macOS carries surf_iova[0] at +0xe38 of the 0xe9c A408; ours sat at +0xe3a. The firmware read the surface DVA two bytes early, turning 0x100075fc000 into 0x0100075fc0000000, and DMA'd to its low 36 bits (0xfc0000000) -- an address with no page table entry at all. This single misalignment produced every write-protect translation fault, every axi_rd_err and every app NMI in this subsystem. The previous fix had shifted only the trailing null-flag block, which was enough for the swap to pass the firmware's validation while still naming a garbage address, so swaps retired against a black panel. Shift the whole block from surf_iova on. A352 is a (u32 selector, u32 value) setter, and selector 0x70 is the APT normal-mode switch. macOS brackets its A438 config push with 0x70=0 and 0x70=1: the first drops the timing sequencer to auto mode (the firmware logs apt_normal_mode_changed, then set_run_mode_safe 4 -> 2 and power_down_M3), the second brings it back. We sent 0 both times, so M3 stayed parked -- swaps were accepted and retired but nothing ever reached the panel. A438 makes this firmware take a CPU exception, so drop the whole A353/A352/A438 bracket rather than opening it and never closing it. Also in this change: - AFK endpoint demux. H17P multiplexes every service of an endpoint onto queue channel 0 and carries the real channel in the EPIC header. On ep 0x20 that made "system" and "powerlog-service" collide on channel 0, so the second announce and all backlight telemetry were dropped as unhandled. - D451 allocate_buffer reports the physical address alongside the DVA, matching the capture. - D003 rt_bandwidth returns the real doorbell and scratch registers, D006 uses the H17P frame-sync layout, and D201 map_buf uses the H17P response shape. - parse_epic_service_init() accepts the H17P key names, which carry the service class in "name" and the compartment in "compartment". - Never send A471 (update_dfb): it makes the coprocessor take an AXI read error and NMI immediately, and scanout does not need it. The swap names its own surface, whose DVA comes from the DRM GEM object on the display-subsystem device and is therefore already mapped in the stream the pipe reads. - Do not prime the display DMA domain when the bootloader left translations behind. The stream already has a table on a locked DART, and mapping through the domain only installs our clone of the inherited L1 over the hardware's. Scanout uses the ordinary DRM path: because the DRM device is soc:display-subsystem, drm_gem_dma allocations already land in disp0_dart stream 0, so GPU-rendered framebuffers need no extra plumbing. Not yet supported: mode setting (A413 is rejected by this firmware, so the bootloader's timing is inherited), backlight control, external displays and HDMI audio. One intermittent failure remains, unrelated to the above: on roughly one boot in four the coprocessor takes a DART reject before the driver loads and never answers INIT. Co-Authored-By: Claude Opus 5 --- drivers/gpu/drm/apple/afk.c | 129 +++- drivers/gpu/drm/apple/afk.h | 51 +- drivers/gpu/drm/apple/apple_drv.c | 54 ++ drivers/gpu/drm/apple/dcp-internal.h | 19 + drivers/gpu/drm/apple/dcp-link.h | 15 +- drivers/gpu/drm/apple/dcp.c | 393 +++++++++++- drivers/gpu/drm/apple/iomfb.c | 523 +++++++--------- drivers/gpu/drm/apple/iomfb.h | 63 +- drivers/gpu/drm/apple/iomfb_h17p.c | 307 ++++++--- drivers/gpu/drm/apple/iomfb_internal.h | 6 + drivers/gpu/drm/apple/iomfb_template.c | 819 ++++++++++++++++++++++++- drivers/gpu/drm/apple/iomfb_template.h | 107 +++- drivers/gpu/drm/apple/parser.c | 17 +- 13 files changed, 2004 insertions(+), 499 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 9124f9e46647b..1b7adbe1df80a 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -116,11 +116,13 @@ int afk_start(struct apple_dcp_afkep *ep) static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) { - u16 size = FIELD_GET(GETBUF_SIZE, message) << BLOCK_SHIFT; + u32 size = FIELD_GET(GETBUF_SIZE, message) << BLOCK_SHIFT; u16 tag = FIELD_GET(GETBUF_TAG, message); u64 reply; trace_afk_getbuf(ep, size, tag); + dev_info(ep->dcp->dev, "AFK[ep:%02x]: getbuf size %#x tag %#x\n", + ep->endpoint, size, tag); if (ep->bfr) { dev_err(ep->dcp->dev, @@ -147,10 +149,10 @@ static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, struct afk_ringbuffer *bfr) { - u16 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; - u16 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; + u32 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; + u32 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; u16 tag = FIELD_GET(INITRB_TAG, message); - u32 bufsz, end; + u32 bufsz, hdrsz, end; if (tag != ep->bfr_tag) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: expected tag 0x%x but got 0x%x\n", @@ -180,15 +182,18 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, } bfr->hdr = ep->bfr + base; - bufsz = le32_to_cpu(bfr->hdr->bufsz); - if (bufsz + sizeof(*bfr->hdr) != size) { + bufsz = le32_to_cpu(*(__le32 *)bfr->hdr); /* first word of block 0 */ + hdrsz = size - bufsz; + if (bufsz >= size || hdrsz % AFK_RB_BLOCKS || + hdrsz / AFK_RB_BLOCKS < AFK_RB_BLOCK_MIN) { dev_err(ep->dcp->dev, - "AFK[ep:%02x]: ring buffer size 0x%x != expected 0x%lx\n", - ep->endpoint, bufsz, sizeof(*bfr->hdr)); + "AFK[ep:%02x]: ring bufsz %#x with size %#x gives no valid header (base %#x, msg %#llx)\n", + ep->endpoint, bufsz, size, base, message); return; } + bfr->block = hdrsz / AFK_RB_BLOCKS; - bfr->buf = bfr->hdr + 1; + bfr->buf = (u8 *)bfr->hdr + hdrsz; bfr->bufsz = bufsz; bfr->ready = true; @@ -283,6 +288,29 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, } ret = parse_epic_service_init(&ctx, &epic_name, &epic_class, &epic_unit); if (ret) { + { + /* name the service we could not register */ + const u8 *b = props; + char buf[128]; + int n = 0; + size_t k, run = 0; + + for (k = 0; k < props_size && n < 100; k++) { + if (b[k] >= 0x20 && b[k] < 0x7f) { + run++; + continue; + } + if (run >= 4) + n += scnprintf(buf + n, + sizeof(buf) - n, + "%.*s ", (int)run, + b + k - run); + run = 0; + } + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: unregistered service props: %s\n", + ep->endpoint, buf); + } dev_err(ep->dcp->dev, "AFK[ep:%02x]: failed to extract init props: %d\n", ep->endpoint, ret); @@ -519,16 +547,51 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, struct epic_hdr *ehdr = (struct epic_hdr *)data; struct epic_sub_hdr *eshdr = (struct epic_sub_hdr *)(data + sizeof(*ehdr)); - u16 subtype = le16_to_cpu(eshdr->type); - u8 *payload = data + sizeof(*ehdr) + sizeof(*eshdr); + struct epic_sub_hdr h17p_eshdr; + size_t hdr_len = sizeof(*ehdr) + sizeof(*eshdr); + u16 subtype; + u8 *payload; size_t payload_size; - if (data_size < sizeof(*ehdr) + sizeof(*eshdr)) { - dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", - ep->endpoint, data_size); - return; + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P) { + const struct epic_sub_hdr_h17p *c = + (const struct epic_sub_hdr_h17p *)(data + sizeof(*ehdr)); + + hdr_len = sizeof(*ehdr) + sizeof(*c); + if (data_size < hdr_len) + goto too_small; + + /* + * H17P multiplexes every service of an endpoint onto queue + * channel 0 and carries the real channel in the EPIC header + * instead: byte 0 is a sequence counter, byte 1 a flags byte + * and bytes 2-3 the channel. ep:20 announces "system" as + * channel 1 and "powerlog-service" as channel 3, then sends the + * mNits/uAmps/iDAC backlight reports on channel 3. Taking the + * queue channel at face value binds "system" to 0 and makes + * every later announce and report look like a duplicate. + */ + channel = le16_to_cpup((__le16 *)(data + 2)); + + /* + * Normalise into the wide sub-header the rest of this file + * expects, mapping H17P's announce subtype onto the canonical + * one so no downstream check has to know the difference. + */ + memset(&h17p_eshdr, 0, sizeof(h17p_eshdr)); + h17p_eshdr.category = c->category; + h17p_eshdr.type = cpu_to_le16(c->type == EPIC_SUBTYPE_ANNOUNCE_H17P ? + EPIC_SUBTYPE_ANNOUNCE : c->type); + h17p_eshdr.tag = cpu_to_le16(le32_to_cpu(c->tag)); + eshdr = &h17p_eshdr; } - payload_size = data_size - sizeof(*ehdr) - sizeof(*eshdr); + + if (data_size < hdr_len) + goto too_small; + + subtype = le16_to_cpu(eshdr->type); + payload = data + hdr_len; + payload_size = data_size - hdr_len; trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); @@ -586,6 +649,11 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, "(type %x subtype %x)\n", ep->endpoint, channel, type, subtype); print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, payload_size, true); + return; + +too_small: + dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", + ep->endpoint, data_size); } static bool afk_recv(struct apple_dcp_afkep *ep) @@ -600,8 +668,8 @@ static bool afk_recv(struct apple_dcp_afkep *ep) return false; } - rptr = le32_to_cpu(ep->rxbfr.hdr->rptr); - wptr = le32_to_cpu(ep->rxbfr.hdr->wptr); + rptr = le32_to_cpu(*afk_rb_rptr(&ep->rxbfr)); + wptr = le32_to_cpu(*afk_rb_wptr(&ep->rxbfr)); trace_afk_recv_rwptr_pre(ep, rptr, wptr); if (rptr == wptr) @@ -621,6 +689,17 @@ static bool afk_recv(struct apple_dcp_afkep *ep) size = le32_to_cpu(hdr->size); trace_afk_recv_qe(ep, rptr, magic, size); + if (magic != QE_MAGIC && !ep->dumped_hdr) { + ep->dumped_hdr = true; + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: rptr %#x wptr %#x block %#x bufsz %#zx datoff %#tx\n", + ep->endpoint, rptr, wptr, ep->rxbfr.block, + ep->rxbfr.bufsz, + (u8 *)ep->rxbfr.buf - (u8 *)ep->rxbfr.hdr); + print_hex_dump(KERN_ERR, "AFKrx: ", DUMP_PREFIX_OFFSET, 16, 4, + (u8 *)ep->rxbfr.buf, 0x500, false); + } + if (magic != QE_MAGIC) { dev_warn(ep->dcp->dev, "AFK[ep:%02x]: invalid queue entry magic: 0x%x\n", ep->endpoint, magic); @@ -646,7 +725,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) return false; } - ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); + *afk_rb_rptr(&ep->rxbfr) = cpu_to_le32(rptr); } if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { @@ -659,7 +738,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) channel = le32_to_cpu(hdr->channel); type = le32_to_cpu(hdr->type); - rptr = ALIGN(rptr + sizeof(*hdr) + size, 1 << BLOCK_SHIFT); + rptr = ALIGN(rptr + sizeof(*hdr) + size, ep->rxbfr.block); if (WARN_ON(rptr > ep->rxbfr.bufsz)) rptr = 0; if (rptr == ep->rxbfr.bufsz) @@ -667,7 +746,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) dma_mb(); - ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); + *afk_rb_rptr(&ep->rxbfr) = cpu_to_le32(rptr); trace_afk_recv_rwptr_post(ep, rptr, wptr); /* @@ -759,8 +838,8 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, spin_lock_irqsave(&ep->lock, flags); dma_rmb(); - rptr = le32_to_cpu(ep->txbfr.hdr->rptr); - wptr = le32_to_cpu(ep->txbfr.hdr->wptr); + rptr = le32_to_cpu(*afk_rb_rptr(&ep->txbfr)); + wptr = le32_to_cpu(*afk_rb_wptr(&ep->txbfr)); trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; total_size = sizeof(*hdr) + total_epic_size; @@ -873,12 +952,12 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, memcpy(ep->txbfr.buf + wptr, payload, payload_len); wptr += payload_len; - wptr = ALIGN(wptr, 1 << BLOCK_SHIFT); + wptr = ALIGN(wptr, ep->txbfr.block); if (wptr == ep->txbfr.bufsz) wptr = 0; trace_afk_send_rwptr_post(ep, rptr, wptr); - ep->txbfr.hdr->wptr = cpu_to_le32(wptr); + *afk_rb_wptr(&ep->txbfr) = cpu_to_le32(wptr); afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_SEND) | FIELD_PREP(SEND_WPTR, wptr)); ret = 0; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index d63d26b79d457..9776056eb5f67 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -71,15 +71,17 @@ struct apple_epic_service_ops { void (*teardown)(struct apple_epic_service *service); }; -struct afk_ringbuffer_header { - __le32 bufsz; - u32 unk; - u32 _pad1[14]; - __le32 rptr; - u32 _pad2[15]; - __le32 wptr; - u32 _pad3[15]; -}; +/* + * The ring header is three control blocks -- bufsz, rptr, wptr -- each padded + * out to the firmware's control-block size. That block is 0x40 bytes up to + * v13.5 but 0x80 on H17P, making the header 0xc0 or 0x180. The size is + * self-describing: the INITRB message gives the total ring size and the first + * word of block 0 gives the payload size, so the stride is (size - bufsz) / 3 + * and no firmware-version test is needed. Address the fields by stride rather + * than through a fixed struct. + */ +#define AFK_RB_BLOCK_MIN 0x40 +#define AFK_RB_BLOCKS 3 struct afk_qe { #define QE_MAGIC 0x20504f49 // ' POI' @@ -109,6 +111,22 @@ struct epic_sub_hdr { __le32 inline_len; } __attribute__((packed)); +/* + * H17P compresses the EPIC sub-header from 24 bytes to 8 and drops the + * separate length/timestamp: the announce for the "system" service puts its + * name at data + 0x18, not data + 0x28. Verified against live ring dumps on + * J700, where an announce carries type 0x11 and a standard-service report + * carries the usual 0xc0. + */ +struct epic_sub_hdr_h17p { + u8 type; + u8 category; + __le16 flags; + __le32 tag; +} __packed; + +#define EPIC_SUBTYPE_ANNOUNCE_H17P 0x11 + struct epic_cmd { __le32 retcode; __le64 rxbuf; @@ -152,13 +170,26 @@ enum epic_subtype { struct afk_ringbuffer { bool ready; - struct afk_ringbuffer_header *hdr; + void *hdr; + u32 block; /* control-block stride, 0x40 or 0x80 */ u32 rptr; void *buf; size_t bufsz; }; +static inline __le32 *afk_rb_field(struct afk_ringbuffer *bfr, unsigned int idx) +{ + return (__le32 *)((u8 *)bfr->hdr + idx * bfr->block); +} + +#define afk_rb_bufsz(bfr) afk_rb_field(bfr, 0) +#define afk_rb_rptr(bfr) afk_rb_field(bfr, 1) +#define afk_rb_wptr(bfr) afk_rb_field(bfr, 2) +#define afk_rb_hdrsz(bfr) (AFK_RB_BLOCKS * (bfr)->block) + + struct apple_dcp_afkep { + bool dumped_hdr; struct apple_dcp *dcp; u32 endpoint; diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index cb9036de3d610..1b444c8ecdd8f 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -637,6 +638,59 @@ static int apple_drm_init(struct device *dev) dev_set_drvdata(dev, apple); + /* + * The display engine scans out through this device's IOMMU stream, but + * apple-dart only programs a translation table when something actually + * maps through the domain. With nothing mapped the stream has no TTBR + * at all and every scanout access faults ("SID:0 ... TTBR invalid"), + * which is fatal on this SoC. Touch the domain once here so it is + * populated before the coprocessor starts driving the display. + * + * Skip it when the bootloader already left translations behind. On a + * locked DART the stream has a table from the start, and mapping + * through the domain instead makes apple-dart install our *clone* of + * the inherited L1 over the hardware's. If the session the loader left + * running happens to touch the carveout inside that window it takes + * DART rejects and wedges, and the DCP then never answers INIT -- + * an intermittent "Failed to boot RTKit: -62" on roughly one boot in + * three, always preceded by a write-protect burst on the carveout + * within a millisecond of this prime. + */ + { + struct iommu_domain *dom = iommu_get_domain_for_dev(dev); + struct iommu_resv_region *entry; + bool inherited = false; + LIST_HEAD(regions); + + if (!IS_ERR_OR_NULL(dom)) { + iommu_get_resv_regions(dev, ®ions); + list_for_each_entry(entry, ®ions, list) + if (entry->type == IOMMU_RESV_TRANSLATED) + inherited = true; + iommu_put_resv_regions(dev, ®ions); + } + + if (inherited) { + dev_info(dev, + "display DMA domain has inherited translations; not priming\n"); + } else { + dma_addr_t probe_dva; + void *probe = dma_alloc_coherent(dev, PAGE_SIZE, + &probe_dva, GFP_KERNEL); + + if (probe) { + dev_info(dev, + "display DMA domain primed, dva %pad\n", + &probe_dva); + dma_free_coherent(dev, PAGE_SIZE, probe, + probe_dva); + } else { + dev_warn(dev, + "could not prime the display DMA domain\n"); + } + } + } + ret = component_bind_all(dev, apple); if (ret) return ret; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index e0671448be025..dfb2d729979de 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -28,6 +28,7 @@ #define DCP_MAX_TYPEC_ROUTES 4 struct apple_dcp; + struct apple_dcp_afkep; struct apple_dcp_typec_port; @@ -113,6 +114,10 @@ struct dcp_channel { u64 link_message[DCP_MAX_CALL_DEPTH]; u32 link_size[DCP_MAX_CALL_DEPTH]; + /* Tag of each outstanding call, so a stall can name itself */ + char tag[DCP_MAX_CALL_DEPTH][4]; + unsigned long pushed_at[DCP_MAX_CALL_DEPTH]; + /* Current depth of the call stack. Less than DCP_MAX_CALL_DEPTH */ u8 depth; /* Already warned about busy channel */ @@ -158,6 +163,10 @@ struct apple_dcp { struct device *dev; struct platform_device *piodma; struct iommu_domain *iommu_dom; + /* domain of the display-subsystem device, which owns the scanout stream */ + struct iommu_domain *disp_iommu_dom; + /* the display-subsystem device, whose DMA domain the pipe scans out of */ + struct device *disp_dev; struct apple_rtkit *rtk; struct apple_crtc *crtc; struct apple_connector *connector; @@ -276,6 +285,14 @@ struct apple_dcp { /* Workqueue for sending vblank events when a dcp swap is not possible */ struct work_struct vblank_wq; + /* Names a command the firmware has stopped answering */ + struct delayed_work stall_wq; + bool warned_stall; + bool dumped_crashlog; + /* the DCP crashlog region, so a stall can read the fresh record */ + void *crashlog_buf; + size_t crashlog_size; + /* List of referenced drm_framebuffers which can be unreferenced * on the next successfully completed swap. */ @@ -366,4 +383,6 @@ bool dcp_has_panel(struct apple_dcp *dcp); #define DCP_AUDIO_MAX_CHANS 15 +void dcp_dump_crashlog(struct apple_dcp *dcp); + #endif /* __APPLE_DCP_INTERNAL_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index 09c6a79edbae9..078d16e79c2fc 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -30,12 +30,15 @@ * Records are allocated on each channel's stack at * off_next = off_cur + align_up(12 + in_len + out_len, 0x40) * Live macOS 25G83 traces confirm this for all 81915 RPC calls across two - * independent captures. Allocating at 4-byte alignment instead put every - * nested record where the firmware does not look, so its callbacks decoded as - * tag 0: - * DCPLink callback tag 0x0 is invalid - * DCPLink callback could not be decoded: -90 - * IOMFB: Could not find Provider service + * independent captures -- which is the same rule dcp_packet_start() and + * DCP_PACKET_ALIGNMENT already implement in iomfb.c. + * + * H17P does NOT use a separate "link" transport: past the INIT/SetShmem + * handshake it speaks the ordinary IOMFB protocol, with the same six fixed + * channels (CMD 0x00000, OOBCMD 0x08000, ASYNC 0x40000, OOBASYNC 0x48000, + * CB 0x60000, OOBCB 0x68000) selected by the context field of the message. + * The constants below are kept only for the H17P method/callback tables and + * the rt_bandwidth reply; the transport itself lives in iomfb.c. */ #define APPLE_DCP_LINK_RPC_ALIGNMENT 0x40 #define APPLE_DCP_LINK_SHARED_VAR_MEMORY_SIZE 0x4000000 diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 69bc9d9b59480..240515fbd17a2 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -1234,29 +1236,61 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) if (dcp->adopting_session) return; + /* + * We start every endpoint macOS starts (see dcp_start_aux_endpoints), + * but only a few have an AFK instance behind them. An endpoint that is + * live but unbacked must log, never dereference a NULL afkep. + */ switch (endpoint) { case IOMFB_ENDPOINT: return iomfb_recv_msg(dcp, message); case AV_ENDPOINT: - afk_receive_message(dcp->avep, message); + if (dcp->avep) + afk_receive_message(dcp->avep, message); + else + goto unbacked; return; case SYSTEM_ENDPOINT: - afk_receive_message(dcp->systemep, message); + if (dcp->systemep) + afk_receive_message(dcp->systemep, message); + else + goto unbacked; return; case DISP0_ENDPOINT: - afk_receive_message(dcp->ibootep, message); + if (dcp->ibootep) + afk_receive_message(dcp->ibootep, message); + else + goto unbacked; return; case DPAVSERV_ENDPOINT: - afk_receive_message(dcp->dcpavservep, message); + if (dcp->dcpavservep) + afk_receive_message(dcp->dcpavservep, message); + else + goto unbacked; return; case DPTX_ENDPOINT: - afk_receive_message(dcp->dptxep, message); + if (dcp->dptxep) + afk_receive_message(dcp->dptxep, message); + else + goto unbacked; return; default: - WARN(endpoint, "unknown DCP endpoint %hhu\n", endpoint); + goto unbacked; } + +unbacked: + dev_info_ratelimited(dcp->dev, "ep %#04x: %#llx\n", endpoint, message); } +/* + * Attach to the running DCP instead of restarting it. Off by default until it + * is proven; the restart path is what has been exercised so far. + */ +static bool dcp_adopt_live; +module_param_named(adopt_live, dcp_adopt_live, bool, 0444); +MODULE_PARM_DESC(adopt_live, + "attach to the loader's live DCP session without sending INIT"); + static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_size) { struct apple_dcp *dcp = cookie; @@ -1274,6 +1308,33 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ complete(&dcp->start_done); } +void dcp_dump_crashlog(struct apple_dcp *dcp) +{ + const char *p = dcp->crashlog_buf; + size_t i, run = 0, printed = 0; + + if (!p || dcp->dumped_crashlog) + return; + dcp->dumped_crashlog = true; + + for (i = 0; i < dcp->crashlog_size && printed < 40; i++) { + char c = p[i]; + + if (c >= 0x20 && c < 0x7f) { + run++; + continue; + } + if (run >= 6) { + dev_err(dcp->dev, "crashlog: %.*s\n", + (int)min(run, (size_t)140), p + i - run); + printed++; + } + run = 0; + } + if (!printed) + dev_err(dcp->dev, "crashlog: nothing printable\n"); +} + static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) { struct apple_dcp *dcp = cookie; @@ -1298,9 +1359,58 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) bfr->is_mapped = true; dev_info(dcp->dev, - "shmem_setup: iova: %lx -> pa: %lx -> iomem: %lx\n", + "shmem_setup: iova: %lx -> pa: %lx -> iomem: %lx size: %zx\n", (uintptr_t)bfr->iova, (uintptr_t)phy_addr, - (uintptr_t)bfr->buffer); + (uintptr_t)bfr->buffer, (size_t)bfr->size); + + /* + * The DCP hands us this region when it reports a crash on the + * crashlog endpoint, which it does not advertise in its EPMAP, + * so apple-rtkit only warns and never dumps it. Dump the head + * ourselves -- it is the only account of why the firmware died. + * + * Bounded hard: a few dozen strings, once per session. An + * unbounded dump drowns the 115200 console and hides the very + * thing it is meant to show. + */ + /* + * Remember the region rather than printing it here. The DCP + * publishes it at RTKit init and the contents are whatever + * survived in DRAM from a previous life, so dumping it now + * makes an inherited panic look like a fresh one. It is read + * from the stall watchdog instead, once a new record has + * actually been written. + */ + if (bfr->size >= SZ_128K) { + dcp->crashlog_buf = bfr->buffer; + dcp->crashlog_size = bfr->size; + } + + if (0) { + const char *p = bfr->buffer; + size_t i, run = 0, printed = 0; + + dcp->dumped_crashlog = true; + for (i = 0; i < bfr->size && printed < 24; i++) { + char c = p[i]; + + if (c >= 0x20 && c < 0x7f) { + run++; + continue; + } + if (run >= 5) { + dev_err(dcp->dev, "crashlog: %.*s\n", + (int)min(run, (size_t)120), + p + i - run); + printed++; + } + run = 0; + } + if (!printed) + dev_err(dcp->dev, + "crashlog: no printable records in %zx bytes\n", + (size_t)bfr->size); + } } else { bfr->buffer = dma_alloc_coherent(dcp->dev, bfr->size, &bfr->iova, GFP_KERNEL); @@ -1680,6 +1790,18 @@ int dcp_start(struct platform_device *pdev) init_completion(&dcp->start_done); + /* + * The coprocessor start can fail (or be recovered from) without tearing + * the DRM side down, so this can be reached with no RTKit instance at + * all. Starting an endpoint then dereferences NULL inside + * apple_rtkit_start_ep() and takes the machine with it. + */ + if (!dcp->rtk || IS_ERR(dcp->rtk)) { + dev_err(dcp->dev, + "cannot start endpoints: RTKit is not initialised\n"); + return -ENODEV; + } + /* start RTKit endpoints */ ret = systemep_init(dcp); if (ret) @@ -2282,6 +2404,129 @@ static int dcp_connector_type_from_dt(struct device_node *np) return DRM_MODE_CONNECTOR_Unknown; } + +#define APPLE_PMGR_PS_ACTUAL GENMASK(7, 4) +#define APPLE_PMGR_PS_TARGET GENMASK(3, 0) +#define APPLE_PMGR_PS_ACTIVE 0xf + +/* + * Power up the "pmc" domain for the DCP. + * + * As soon as the AP acks the fifth D563 (pmgrConfigDispPerfState), the DCP's + * RTKPMGR_pmc_boost_disp_perf_state path reads the PMC idle register at + * physical 0x3028aa140. That aperture belongs to the T8140 "pmc" power + * domain, which the ADT marks initially-on but which Linux leaves off because + * no device claims it. The read then takes an L2C access fault and the + * coprocessor dies with a data abort -- silently, with no further mailbox + * traffic, immediately after the boot property storm, which looks exactly like + * a hang. Turn the domain on before booting the firmware. + */ +static void dcp_enable_pmc_domain(struct apple_dcp *dcp) +{ + struct device_node *np; + const char *label; + u32 offset, val = 0; + int i; + + for_each_compatible_node(np, NULL, "apple,t8103-pmgr-pwrstate") { + struct regmap *regmap; + + if (of_property_read_string(np, "label", &label) || + strcmp(label, "pmc")) + continue; + + if (of_property_read_u32_index(np, "reg", 0, &offset)) { + dev_warn(dcp->dev, "pmc: no reg offset\n"); + break; + } + + regmap = syscon_node_to_regmap(np->parent); + if (IS_ERR(regmap)) { + dev_warn(dcp->dev, "pmc: no parent regmap: %ld\n", + PTR_ERR(regmap)); + break; + } + + if (regmap_update_bits(regmap, offset, APPLE_PMGR_PS_TARGET, + FIELD_PREP(APPLE_PMGR_PS_TARGET, + APPLE_PMGR_PS_ACTIVE))) { + dev_warn(dcp->dev, "pmc: failed to request active\n"); + break; + } + + for (i = 0; i < 200; i++) { + if (regmap_read(regmap, offset, &val)) + break; + if (FIELD_GET(APPLE_PMGR_PS_ACTUAL, val) == + APPLE_PMGR_PS_ACTIVE) { + dev_info(dcp->dev, "pmc domain on (%#x)\n", val); + of_node_put(np); + return; + } + udelay(10); + } + dev_warn(dcp->dev, "pmc domain did not activate (%#x)\n", val); + of_node_put(np); + return; + } + + dev_warn(dcp->dev, "pmc power domain not found\n"); +} + +/* + * Reset the display coprocessor's ASC core. + * + * Once Linux has driven a DCP session the firmware stops answering the + * management endpoint: it will not re-HELLO on INIT (it is already ON) and it + * will not acknowledge a power-state change either, so neither apple_rtkit_wake + * nor apple_rtkit_quiesce can recover it and the module could previously only + * be reloaded by rebooting the machine. + * + * CPU_CONTROL.RUN is a raw register in the ASC wrapper and needs no cooperation + * from the firmware, so clearing it stops the core no matter how wedged it is. + * The firmware image itself is left alone -- iBoot mapped it into the DCP's own + * DART and nothing here touches those mappings -- so setting RUN again restarts + * it from its reset vector and it comes back with a fresh HELLO. + */ +static int dcp_asc_restart(struct apple_dcp *dcp) +{ + u32 ctrl, status; + int i; + + ctrl = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + writel_relaxed(ctrl & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, + dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + + /* + * T8140 clears CPU_STATUS.RUNNING when the core halts but never sets + * the STOPPED bit (status goes 0x6d -> 0x48), so gate on RUNNING going + * away rather than on STOPPED appearing. + */ + for (i = 0; i < 500; i++) { + status = readl_relaxed(dcp->coproc_reg + + APPLE_DCP_COPROC_CPU_STATUS); + if (!(status & APPLE_DCP_COPROC_CPU_STATUS_RUNNING) || + (status & APPLE_DCP_COPROC_CPU_STATUS_STOPPED)) + break; + usleep_range(100, 200); + } + if (status & APPLE_DCP_COPROC_CPU_STATUS_RUNNING) { + dev_err(dcp->dev, "DCP ASC did not stop (status %#x)\n", status); + return -ETIMEDOUT; + } + + dev_info(dcp->dev, "DCP ASC stopped (status %#x), restarting it\n", + status); + usleep_range(1000, 2000); + writel_relaxed(ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, + dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + return 0; +} + + + + + static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; @@ -2292,6 +2537,39 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) bool loader_quiesced; int ret; + /* + * The coprocessor scans out through the display-subsystem device's + * IOMMU stream, not through piodma's, so buffers it asks us to map for + * piodma must also be visible in that domain or the scanout faults. + */ + dcp->disp_dev = main; + dcp->disp_iommu_dom = main ? iommu_get_domain_for_dev(main) : NULL; + if (IS_ERR(dcp->disp_iommu_dom)) + dcp->disp_iommu_dom = NULL; + + /* + * Give the scanout stream a landing page at the bottom of its window. + * + * m1n1 stops the DCP ASC before handing over, so unlike macOS -- which + * adopts the generation iBoot left running -- our firmware comes up with + * no active surface. It enables the pipe anyway at the end of the boot + * chain and fetches from a null base. The DART rejects that, the reject + * becomes an AXI error, and the coprocessor takes an NMI + * (apiodma_irq_AxiErr) before we can drive a modeset. + * + * One zeroed page mapped at the base of the stream's own window turns + * that fatal reject into a harmless read of zeros. This only *adds* a + * translation at an index the bootloader does not use; it never rewrites + * an inherited one. + */ + /* + * Deliberately do not add mappings here. iommu_map() on this domain + * makes apple-dart copy our *clone* of the inherited L1 over the + * hardware's, so anything the live firmware has added to its own tables + * since we cloned them vanishes underneath it and the coprocessor takes + * an axi_rd_err NMI. The bootloader's translations are left alone. + */ + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); if (ret) return ret; @@ -2382,10 +2660,14 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) set_bit(0, dcp->memdesc_map); INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); + INIT_DELAYED_WORK(&dcp->stall_wq, dcp_stall_check); dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + dcp_enable_pmc_domain(dcp); + if (start_mode == APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART) { dev_info(dev, "quiescing inherited DCP session (control=%#x status=%#x)\n", @@ -2408,19 +2690,100 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) } else { if (start_mode == APPLE_DCP_COPROCESSOR_QUIESCED_RESTART) dev_info(dev, "restarting loader-quiesced DCP session\n"); - dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, - &rtkit_ops); + if (dcp_adopt_live) { + dcp->adopting_session = true; + dcp->rtk = devm_apple_rtkit_init_adopted(dev, dcp, + "mbox", 0, + &rtkit_ops); + dcp->adopting_session = false; + } else { + dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, + &rtkit_ops); + } if (IS_ERR(dcp->rtk)) return dev_err_probe(dev, PTR_ERR(dcp->rtk), "Failed to initialize RTKit\n"); } - cpu_ctrl = readl_relaxed(dcp->coproc_reg + - APPLE_DCP_COPROC_CPU_CONTROL); - writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, - dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + if (dcp_adopt_live) { + /* + * Attach to the session the loader left running instead of + * re-initialising it. + * + * m1n1 hands the DCP over live and untouched -- it neither + * stops the ASC nor quiesces RTKit; the "quiesced" property is + * only there to steer this driver's code path. So the + * coprocessor already has a session, an endpoint map, and a + * configured display pipe. Poking CPU_CONTROL.RUN and sending + * SET_IOP_PWR_STATE(INIT) at that makes the firmware tear its + * IOMFB state down and come back with no scanout, which is the + * state macOS is never in: its capture contains no wake, no + * HELLO and no EPMAP, just A401 on an already-running IOP. + */ + dev_info(dev, "adopting the live DCP session without a restart\n"); + ret = 0; + } else { + cpu_ctrl = readl_relaxed(dcp->coproc_reg + + APPLE_DCP_COPROC_CPU_CONTROL); + writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, + dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + + ret = apple_rtkit_wake(dcp->rtk); + } + if (ret && start_mode == APPLE_DCP_COPROCESSOR_QUIESCED_RESTART) { + /* + * INIT went unanswered. That happens when the session we found + * is not the one the loader quiesced but one an earlier probe + * of this driver already brought up: RTKit is ON, so it ignores + * INIT and never sends HELLO. Adopt that session, quiesce it + * back to the state the loader would have left it in, and boot + * it again. Without this, a failed probe or a module reload + * can only be recovered by rebooting the machine. + */ + dev_info(dev, + "DCP did not answer INIT; adopting the running session to restart it\n"); + devm_apple_rtkit_free(dev, dcp->rtk); + dcp->rtk = NULL; + dcp->adopting_session = true; + dcp->rtk = devm_apple_rtkit_init_adopted(dev, dcp, "mbox", 0, + &rtkit_ops); + if (IS_ERR(dcp->rtk)) { + dcp->adopting_session = false; + return dev_err_probe(dev, PTR_ERR(dcp->rtk), + "Failed to adopt running RTKit\n"); + } - ret = apple_rtkit_wake(dcp->rtk); + ret = apple_rtkit_quiesce(dcp->rtk); + dcp->adopting_session = false; + if (!ret) + ret = apple_rtkit_wake(dcp->rtk); + + if (ret) { + /* + * The firmware is not answering the mailbox at all. + * Reset the ASC core itself and cold-boot it; that + * needs no cooperation from the running firmware. + */ + dev_info(dev, + "DCP will not quiesce (%d); resetting the ASC core\n", + ret); + devm_apple_rtkit_free(dev, dcp->rtk); + dcp->rtk = NULL; + ret = dcp_asc_restart(dcp); + if (ret) + return dev_err_probe(dev, ret, + "Failed to reset DCP ASC: %d\n", + ret); + + dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, + &rtkit_ops); + if (IS_ERR(dcp->rtk)) + return dev_err_probe(dev, PTR_ERR(dcp->rtk), + "Failed to re-init RTKit\n"); + + ret = apple_rtkit_wake(dcp->rtk); + } + } if (ret) return dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index e84b4d1aedd7e..02c46a7554af1 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -34,6 +34,81 @@ #include "parser.h" #include "trace.h" + +u32 dcp_clamshell_state; /* A476 set_dcp_clamshellstate; 0 = lid open */ +module_param_named(clamshell_state, dcp_clamshell_state, uint, 0644); + +bool dcp_send_modeset; /* macOS never issues A413 */ +module_param_named(send_modeset, dcp_send_modeset, bool, 0644); + + + +/* + * A353 / A352(0x70,0) / A438 x5 / A352(0x70,1) form one bracket: the A352 pair + * parks and resumes the APT timing sequencer around the A438 config push + * ("apt_normal_mode_changed: requesting transtition to auto mode" -> + * set_run_mode_safe 4 -> 2 -> power_down_M3). A438 makes our firmware take a + * CPU exception, and opening the bracket without closing it leaves M3 powered + * down, so the swap is accepted but never scans out. Off = skip all of it and + * leave the sequencer in normal mode. + */ +/* + * The scanout engine reads through disp0_dart stream 0 (display-subsystem), + * not through the DCP's own stream. The only surface memory mapped in that + * stream is the display carveout: reserved-memory/framebuffer@101eb4b8000 + * advertises iommu-addresses = <&dcp 0x1ea0000 ...>, <&display-subsystem + * 0x1ea0000 ...>. A GEM buffer's DVA exists only in the DCP domain, so a swap + * naming one validates and retires while the scanout engine reads an address + * it cannot translate -- a black panel with no visible fault, because the + * DART's error IRQ is masked once the fault budget is spent. + */ +/* + * macOS's full StartEP set; see iomfb_start_rtkit(). Off: bringing all sixteen + * up made the firmware NMI (panic(app): app NMI, pidx: 1) before the first swap + * could retire, so the endpoints past 0x2d/0x37 need to be introduced one at a + * time rather than as a block. + */ + +/* + * Bitmask of the endpoints in 0x20..0x3f to start beyond 0x20/0x2d/0x37, bit n + * meaning endpoint 0x20+n. Starting macOS's whole set at once made the + * firmware NMI, so bring them up one at a time: disp0 (0x23) is extra_eps=0x8. + */ +static unsigned int dcp_extra_eps; +module_param_named(extra_eps, dcp_extra_eps, uint, 0644); +MODULE_PARM_DESC(extra_eps, "bitmask of extra endpoints to start, bit n = 0x20+n"); + +void dcp_start_extra_eps(struct apple_dcp *dcp) +{ + unsigned int n; + + for (n = 0; n < 32; n++) { + if (!(dcp_extra_eps & BIT(n))) + continue; + dev_info(dcp->dev, "starting extra endpoint %#04x\n", 0x20 + n); + apple_rtkit_start_ep(dcp->rtk, 0x20 + n); + } +} + + +bool dcp_park_display = true; +module_param_named(park_display, dcp_park_display, bool, 0644); +MODULE_PARM_DESC(park_display, "H17P: power the display down after boot until a modeset"); + +bool dcp_defer_client_open; +module_param_named(defer_client_open, dcp_defer_client_open, bool, 0644); +MODULE_PARM_DESC(defer_client_open, "H17P: do not send first_client_open during boot"); + +static bool skip_hotplug; +module_param(skip_hotplug, bool, 0644); +MODULE_PARM_DESC(skip_hotplug, "do not raise the DRM hotplug event (bring-up bisect)"); + +static int dcp_trace_budget; +module_param_named(trace_rpc, dcp_trace_budget, int, 0644); +MODULE_PARM_DESC(trace_rpc, + "log the first N DCP RPC messages (0 = off; the vblank callback\n" + "\t\t alone is 60/s, so anything non-zero fills the kernel ring)"); + static int dcp_tx_offset(enum dcp_context_id id) { switch (id) { @@ -103,6 +178,12 @@ static enum dcp_context_id dcp_call_context(struct apple_dcp *dcp, bool oob) { u8 depth = oob ? dcp->ch_oobcmd.depth : dcp->ch_cmd.depth; + /* + * A nested call goes on the callback channel, past the record we are + * answering. This matches macOS: its A401 carries context 2 in the + * wire word while every call nested inside it carries context 0 with + * offset 0x40, which is exactly what this produces. + */ if (depth) return oob ? DCP_CONTEXT_OOBCB : DCP_CONTEXT_CB; else @@ -131,15 +212,6 @@ static struct dcp_channel *dcp_get_channel(struct apple_dcp *dcp, } } -static void dcp_link_push(struct apple_dcp *dcp, bool oob, - const struct dcp_method_entry *call, u32 in_len, - u32 out_len, void *data, dcp_callback_t cb, - void *cookie); -static void dcp_link_ack(struct apple_dcp *dcp, - enum dcp_context_id context); -static void dcp_link_recv_msg(struct apple_dcp *dcp, u64 message); -static int dcp_link_start_rtkit(struct apple_dcp *dcp); - /* Get the start of a packet: after the end of the previous packet */ static u16 dcp_packet_start(struct dcp_channel *ch, u8 depth) { @@ -170,12 +242,6 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca u32 in_len, u32 out_len, void *data, dcp_callback_t cb, void *cookie) { - if (dcp->fw_compat == DCP_FIRMWARE_H17P) { - dcp_link_push(dcp, oob, call, in_len, out_len, data, cb, - cookie); - return; - } - enum dcp_context_id context = dcp_call_context(dcp, oob); struct dcp_channel *ch = dcp_get_channel(dcp, context); @@ -203,7 +269,20 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca memcpy(out_data, data, in_len); trace_iomfb_push(dcp, call, context, offset, depth); + if (dcp_trace_budget > 0) { + dcp_trace_budget--; + dev_info(dcp->dev, "CALL %.4s ctx=%x off=%#x in=%u out=%u\n", + call->tag, context, offset, in_len, out_len); + } + + memcpy(ch->tag[depth], call->tag, sizeof(ch->tag[depth])); + ch->pushed_at[depth] = jiffies; + if (context == DCP_CONTEXT_CMD && depth == 0) { + dcp->warned_stall = false; + mod_delayed_work(system_wq, &dcp->stall_wq, + msecs_to_jiffies(1000)); + } ch->callbacks[depth] = cb; ch->cookies[depth] = cookie; ch->output[depth] = out + sizeof(header) + in_len; @@ -232,287 +311,35 @@ int dcp_parse_tag(char tag[4]) return d[0] + (d[1] * 10) + (d[2] * 100); } -static u8 dcp_link_context_stream(enum dcp_context_id context) -{ - return context == DCP_CONTEXT_OOBCB || - context == DCP_CONTEXT_OOBCMD ? 1 : 0; -} - -static bool dcp_link_context_is_remote(enum dcp_context_id context) -{ - return context == DCP_CONTEXT_CB || context == DCP_CONTEXT_OOBCB; -} - -static u32 dcp_link_call_tag(const struct dcp_method_entry *call) -{ - return ((u32)(u8)call->tag[0] << 24) | - ((u32)(u8)call->tag[1] << 16) | - ((u32)(u8)call->tag[2] << 8) | - (u8)call->tag[3]; -} - -static void dcp_link_push(struct apple_dcp *dcp, bool oob, - const struct dcp_method_entry *call, u32 in_len, - u32 out_len, void *data, dcp_callback_t cb, - void *cookie) -{ - enum dcp_context_id context = dcp_call_context(dcp, oob); - struct dcp_channel *ch = dcp_get_channel(dcp, context); - struct apple_dcp_link_stream_layout layout; - struct apple_dcp_link_rpc_header *header; - bool remote = dcp_link_context_is_remote(context); - u8 stream = dcp_link_context_stream(context); - u32 *top = &dcp->link_top[remote][stream]; - u32 next_top; - u32 offset; - u32 payload_size; - u64 message; - u8 depth; - void *buffer; - int ret; - - depth = dcp_push_depth(&ch->depth); - ret = apple_dcp_link_rpc_payload_size(in_len, out_len, - &payload_size); - if (ret) - goto err_pop; - ret = apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, stream, - remote, &layout); - if (ret) - goto err_pop; - ret = apple_dcp_link_stack_push(*top, layout.capacity, payload_size, - APPLE_DCP_LINK_RPC_ALIGNMENT, &offset, - &next_top); - if (ret) - goto err_pop; - - buffer = dcp->shmem + layout.local_offset; - header = buffer + offset; - apple_dcp_link_rpc_header_encode(header, dcp_link_call_tag(call), - in_len, out_len); - if (in_len) - memcpy(header + 1, data, in_len); - if (out_len) - memset((u8 *)(header + 1) + in_len, 0, out_len); - - ret = apple_dcp_link_rpc_message(offset, payload_size, remote, &message); - if (ret) - goto err_pop; - ret = apple_dcp_link_stream_message(message, stream, remote, &message); - if (ret) - goto err_pop; - - ch->callbacks[depth] = cb; - ch->cookies[depth] = cookie; - ch->output[depth] = (u8 *)(header + 1) + in_len; - ch->link_message[depth] = message; - ch->link_size[depth] = payload_size; - *top = next_top; - - dma_wmb(); - dcp_send_message(dcp, APPLE_DCP_LINK_ENDPOINT, message); - return; - -err_pop: - dcp_pop_depth(&ch->depth); - dev_err(dcp->dev, "DCPLink call %.4s could not be queued: %d\n", - call->tag, ret); -} - -static void dcp_link_ack(struct apple_dcp *dcp, - enum dcp_context_id context) -{ - struct dcp_channel *ch = dcp_get_channel(dcp, context); - u8 depth; - u64 reply; - - if (!ch || !ch->depth) { - dev_warn(dcp->dev, "DCPLink callback reply without an active call\n"); - return; - } - - depth = dcp_pop_depth(&ch->depth); - reply = apple_dcp_link_rpc_reply(ch->link_message[depth]); - ch->link_message[depth] = 0; - dma_wmb(); - dcp_send_message(dcp, APPLE_DCP_LINK_ENDPOINT, reply); -} - -static struct dcp_channel *dcp_link_reply_channel(struct apple_dcp *dcp, - u64 message) -{ - bool remote = apple_dcp_link_message_is_remote(message); - u8 stream = apple_dcp_link_message_stream(message); - - if (stream == 1) - return remote ? &dcp->ch_oobcb : &dcp->ch_oobcmd; - return remote ? &dcp->ch_cb : &dcp->ch_cmd; -} - -static void dcp_link_handle_reply(struct apple_dcp *dcp, u64 message) -{ - struct dcp_channel *ch = dcp_link_reply_channel(dcp, message); - bool remote = apple_dcp_link_message_is_remote(message); - u8 stream = apple_dcp_link_message_stream(message); - u32 *top; - dcp_callback_t cb; - void *cookie; - u8 depth; - int ret; - - if (stream >= APPLE_DCP_LINK_STREAMS_PER_SIDE) { - dev_warn(dcp->dev, "DCPLink reply stream %u is invalid\n", stream); - return; - } - top = &dcp->link_top[remote][stream]; - - if (!ch->depth) { - dev_warn(dcp->dev, "DCPLink reply without an active call\n"); - return; - } - - depth = dcp_pop_depth(&ch->depth); - ret = apple_dcp_link_stack_pop(*top, ch->link_size[depth], - APPLE_DCP_LINK_RPC_ALIGNMENT, top); - if (ret) - dev_warn(dcp->dev, "DCPLink stream %u stack mismatch: %d\n", - stream, ret); - - dma_rmb(); - cb = ch->callbacks[depth]; - cookie = ch->cookies[depth]; - ch->callbacks[depth] = NULL; - ch->cookies[depth] = NULL; - ch->link_message[depth] = 0; - ch->link_size[depth] = 0; - if (cb) - cb(dcp, ch->output[depth], cookie); -} - -static void dcp_link_handle_callback(struct apple_dcp *dcp, u64 message) -{ - struct apple_dcp_link_stream_layout layout; - struct apple_dcp_link_method method; - struct apple_dcp_link_rpc_view view; - struct dcp_channel *ch = &dcp->ch_cb; - char wire_tag[4]; - u8 stream = apple_dcp_link_message_stream(message); - u8 depth; - int tag; - int ret; - - ret = apple_dcp_link_stream_layout(APPLE_DCP_LINK_SIDE_AP, stream, true, - &layout); - if (ret) { - dcp_send_message(dcp, APPLE_DCP_LINK_ENDPOINT, - apple_dcp_link_rpc_reply(message)); - return; - } - dma_rmb(); - ret = apple_dcp_link_rpc_decode(message, - dcp->shmem + layout.remote_offset, - layout.capacity, &view); - if (ret) { - dev_warn(dcp->dev, "DCPLink callback could not be decoded: %d\n", - ret); - dcp_send_message(dcp, APPLE_DCP_LINK_ENDPOINT, - apple_dcp_link_rpc_reply(message)); - return; - } - - ret = apple_dcp_h17p_callback_method(view.call, &method); - if (!ret && (method.input_size != view.input_size || - method.output_size != view.output_size)) { - dev_warn(dcp->dev, - "DCPLink callback %#x size %u/%u, expected %u/%u\n", - view.call, view.input_size, view.output_size, - method.input_size, method.output_size); - } - - wire_tag[0] = view.call; - wire_tag[1] = view.call >> 8; - wire_tag[2] = view.call >> 16; - wire_tag[3] = view.call >> 24; - tag = dcp_parse_tag(wire_tag); - if (tag < 0 || tag >= IOMFB_MAX_CB) { - dev_warn(dcp->dev, "DCPLink callback tag %#x is invalid\n", - view.call); - dcp_send_message(dcp, APPLE_DCP_LINK_ENDPOINT, - apple_dcp_link_rpc_reply(message)); - return; - } - - if (view.output_size) - memset(view.output, 0, view.output_size); - depth = dcp_push_depth(&ch->depth); - ch->output[depth] = view.output; - ch->link_message[depth] = message; - ch->link_size[depth] = view.total_size; - - if (!dcp->cb_handlers || !dcp->cb_handlers[tag]) { - dev_warn(dcp->dev, "DCPLink callback D%03d has no handler\n", tag); - dcp_link_ack(dcp, DCP_CONTEXT_CB); - return; - } - - if (dcp->cb_handlers[tag](dcp, tag, view.output, (void *)view.input)) - dcp_link_ack(dcp, DCP_CONTEXT_CB); -} - -static void dcp_link_recv_msg(struct apple_dcp *dcp, u64 message) +/* + * The firmware silently stops answering a method it is not ready for, and the + * call then sits on the command channel forever -- every later push sees a busy + * channel and the whole bring-up stalls with no clue which call did it. Name it. + */ +void dcp_stall_check(struct work_struct *work) { - u8 type = message & GENMASK_ULL(1, 0); + struct apple_dcp *dcp = container_of(to_delayed_work(work), + struct apple_dcp, stall_wq); + struct dcp_channel *ch = &dcp->ch_cmd; + u8 d = ch->depth; - if (type == APPLE_DCP_LINK_MSG_READY) { - if (apple_dcp_link_ready_reply(message, &dcp->link_version, - &dcp->link_firmware_hash)) - return; - if (dcp->link_ready) - return; - dcp->link_ready = true; - dev_info(dcp->dev, "DCPLink ready: version %u hash %#x\n", - dcp->link_version, dcp->link_firmware_hash); - iomfb_start_h17p(dcp); + if (!d) { + dcp->warned_stall = false; return; } - if (type != APPLE_DCP_LINK_MSG_RPC) { - dev_warn(dcp->dev, "unknown DCPLink message %#llx\n", message); - return; + if (!dcp->warned_stall && + time_after(jiffies, ch->pushed_at[d - 1] + msecs_to_jiffies(2000))) { + dcp->warned_stall = true; + dev_err(dcp->dev, + "DCP stopped answering %.4s (depth %u, %u ms outstanding)\n", + ch->tag[d - 1], d, + jiffies_to_msecs(jiffies - ch->pushed_at[d - 1])); + /* by now a fresh panic record has been written, if there is one */ + dcp_dump_crashlog(dcp); } - if ((message & GENMASK_ULL(7, 6)) == BIT_ULL(6)) - dcp_link_handle_reply(dcp, message); - else - dcp_link_handle_callback(dcp, message); -} - -static int dcp_link_start_rtkit(struct apple_dcp *dcp) -{ - struct apple_dcp_link_init_desc *desc; - u64 init_message; - int ret; - - apple_rtkit_start_ep(dcp->rtk, APPLE_DCP_LINK_ENDPOINT); - dcp->shmem = dma_alloc_coherent(dcp->dev, APPLE_DCP_LINK_RPC_MEMORY_SIZE, - &dcp->shmem_iova, GFP_KERNEL); - if (!dcp->shmem) - return -ENOMEM; - - memset(dcp->shmem, 0, APPLE_DCP_LINK_RPC_MEMORY_SIZE); - desc = dcp->shmem; - apple_dcp_link_init_desc_encode(desc, 0, false); - ret = apple_dcp_link_init_message(dcp->shmem_iova, &init_message); - if (ret) - return ret; - - dcp->link_ready = false; - memset(dcp->link_top, 0, sizeof(dcp->link_top)); - dma_wmb(); - dcp_send_message(dcp, APPLE_DCP_LINK_ENDPOINT, init_message); - dcp_send_message(dcp, APPLE_DCP_LINK_ENDPOINT, - apple_dcp_link_ready_message()); - return 0; + schedule_delayed_work(&dcp->stall_wq, msecs_to_jiffies(1000)); } /* Ack a callback from the DCP */ @@ -520,11 +347,6 @@ void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) { struct dcp_channel *ch = dcp_get_channel(dcp, context); - if (dcp->fw_compat == DCP_FIRMWARE_H17P) { - dcp_link_ack(dcp, context); - return; - } - dcp_pop_depth(&ch->depth); dcp_send_message(dcp, IOMFB_ENDPOINT, dcpep_ack(context)); @@ -586,6 +408,11 @@ void dcp_hotplug(struct work_struct *work) dev_info(dcp->dev, "%s() connected:%d valid_mode:%d nr_modes:%u\n", __func__, connector->connected, dcp->valid_mode, dcp->nr_modes); + if (skip_hotplug) { + dev_info(dcp->dev, "skip_hotplug set; not notifying DRM\n"); + return; + } + if (!connector->connected) { drm_edid_free(connector->drm_edid); connector->drm_edid = NULL; @@ -628,15 +455,50 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; - if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || !dcp->cb_handlers[tag]) { - dev_warn(dev, "received unknown callback %c%c%c%c\n", - hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); - return; + if (dcp_trace_budget > 0 && tag != 300) { + dcp_trace_budget--; + char nm[41] = ""; + const char *ib = (const char *)(hdr + 1); + u32 k; + + /* most property callbacks lead with a printable key */ + for (k = 0; k < hdr->in_len && k < sizeof(nm) - 1; k++) { + if (ib[k] == '\0') + break; + nm[k] = (ib[k] >= 0x20 && ib[k] < 0x7f) ? ib[k] : '.'; + } + nm[k] = '\0'; + dev_info(dev, "CB %c%c%c%c ctx=%x in=%u out=%u %s\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0], + context, hdr->in_len, hdr->out_len, nm); } in = data + sizeof(*hdr); out = in + hdr->in_len; + if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || + !dcp->cb_handlers[tag]) { + dev_warn(dev, "received unknown callback %c%c%c%c\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + /* + * Leaving a callback unanswered wedges the coprocessor: it + * waits for the ack forever and the outer call never returns. + * On H17P answer it with a zeroed output instead, so the + * warning above names the tag we still have to implement + * rather than costing a reboot to discover one tag at a time. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (hdr->out_len) + memset(out, 0, hdr->out_len); + depth = dcp_push_depth(&ch->depth); + ch->output[depth] = out; + ch->end[depth] = offset + + ALIGN(length, DCP_PACKET_ALIGNMENT); + dcp_ack(dcp, context); + } + return; + } + // TODO: verify that in_len and out_len match our prototypes // for now just clear the out data to have at least consistent results if (hdr->out_len) @@ -646,8 +508,19 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, ch->output[depth] = out; ch->end[depth] = offset + ALIGN(length, DCP_PACKET_ALIGNMENT); - if (dcp->cb_handlers[tag](dcp, tag, out, in)) - dcp_ack(dcp, context); + { + bool acked = dcp->cb_handlers[tag](dcp, tag, out, in); + + if (dcp_trace_budget > 0 && hdr->out_len && tag != 300) { + dcp_trace_budget--; + dev_info(dev, "RPL %c%c%c%c %*ph\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], + hdr->tag[0], + (int)min_t(u32, hdr->out_len, 16), out); + } + if (acked) + dcp_ack(dcp, context); + } } static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, @@ -687,6 +560,13 @@ static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) offset = FIELD_GET(IOMFB_MSG_OFFSET, message); length = FIELD_GET(IOMFB_MSG_LENGTH, message); + if (dcp_trace_budget > 0) { + dcp_trace_budget--; + dev_info(dcp->dev, "RX %#018llx ctx=%x off=%#x len=%u ack=%d\n", + message, ctx_id, offset, length, + !!FIELD_GET(IOMFB_MSG_ACK, message)); + } + channel_offset = dcp_channel_offset(ctx_id); if (channel_offset < 0) { @@ -853,7 +733,6 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, return lookup_mode(dcp, mode) != NULL; } - void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) { struct platform_device *pdev = to_apple_crtc(crtc)->dcp; @@ -877,7 +756,14 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) if (dcp_channel_busy(&dcp->ch_cmd)) { if (!dcp->ch_cmd.warned_busy) { - dev_err(dcp->dev, "unexpected busy command channel\n"); + u8 d = dcp->ch_cmd.depth; + + dev_err(dcp->dev, + "unexpected busy command channel: depth %u, oldest outstanding %.4s pushed %u ms ago\n", + d, d ? dcp->ch_cmd.tag[d - 1] : "----", + d ? jiffies_to_msecs(jiffies - + dcp->ch_cmd.pushed_at[d - 1]) + : 0); dcp->ch_cmd.warned_busy = true; } /* HACK: issue a delayed vblank event to avoid timeouts in @@ -934,14 +820,19 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) { enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); - if (dcp->fw_compat == DCP_FIRMWARE_H17P) { - dcp_link_recv_msg(dcp, message); - return; - } - - if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) + if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) { + /* + * H17P reports its firmware version and hash in the high half + * of the same InitComplete word (macOS 25G83 sends e.g. + * 0x004045bc9ef40101). The transport below it is unchanged. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + dev_info(dcp->dev, + "IOMFB: init complete %#llx (version %llu)\n", + message, + FIELD_GET(GENMASK_ULL(63, 48), message)); iomfb_start(dcp); - else if (type == IOMFB_MESSAGE_TYPE_MSG) + } else if (type == IOMFB_MESSAGE_TYPE_MSG) dcpep_got_msg(dcp, message); else dev_warn(dcp->dev, "Ignoring unknown message %llx\n", message); @@ -951,10 +842,18 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) { dma_addr_t shmem_iova; - if (dcp->fw_compat == DCP_FIRMWARE_H17P) - return dcp_link_start_rtkit(dcp); - + /* + * macOS starts sixteen DCP endpoints; we started three. Its StartEP + * order, straight from the hv capture, is + * 24 28 21 2c 31 25 2d 32 2b 2e 2a 37 29 20 23 22 + * with 0x20 (system) coming from systemep_init earlier. A firmware + * subsystem whose endpoint was never started has nothing answering it + * on the AP side -- which is how 0x2d's absence used to produce DART + * faults -- so bring the whole set up, in that order. + */ + apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); apple_rtkit_start_ep(dcp->rtk, IOMFB_ENDPOINT); + dcp_start_extra_eps(dcp); dcp->shmem = dma_alloc_coherent(dcp->dev, DCP_SHMEM_SIZE, &shmem_iova, GFP_KERNEL); diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index ec7e23b4343e5..530ee2123c22f 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -11,6 +11,11 @@ /* Fixed size of shared memory between DCP and AP */ #define DCP_SHMEM_SIZE 0x100000 +/* disp0 register window the H17P firmware asks for as map_reg index 5 */ +#define DCP_H17P_DISP0_REG_INDEX 5 +#define DCP_H17P_DISP0_REG_BASE 0x302800000ULL +#define DCP_H17P_DISP0_REG_SIZE 0xbc000ULL + /* DCP message contexts */ enum dcp_context_id { /* Callback */ @@ -154,6 +159,7 @@ enum dcpep_method { dcpep_flush_supports_power, dcpep_set_power_state, dcpep_first_client_open, + dcpep_register_dfb_surface, dcpep_set_parameter_dcp, dcpep_enable_disable_video_power_savings, dcpep_is_main_display, @@ -164,9 +170,39 @@ enum dcpep_method { iomfbep_last_client_close, iomfbep_abort_swaps_dcp, iomfbep_set_matrix, + /* + * H17P start_hardware_boot (D121) issues these between + * create_default_fb and setup_video_limits; macOS 25G83 traces show + * A035/A034 feeding setup_default_fb_info and A104 pushing the + * display's own geometry back at the firmware. + */ + dcpep_get_dfb_compression_info, + dcpep_get_dfb_info, + dcpep_get_dfb_layout, + dcpep_get_dfb_state, + dcpep_set_dfb_dimensions, + dcpep_commit_dfb_info, + dcpep_dfb_query, + dcpep_dfb_ready_query, + /* + * The client-open sequence macOS runs before the first swap. Names + * are unknown; the wire signatures come from the 25G83 capture. + */ + dcpep_pipe_query_353, + dcpep_pipe_set_352, + dcpep_pipe_cfg_415, + dcpep_pipe_cfg_031, + dcpep_pipe_cfg_414, + dcpep_pipe_query_478, + dcpep_pipe_query_474, + dcpep_pipe_set_476, + dcpep_pipe_cfg_438, + dcpep_pipe_set_428, dcpep_num_methods }; +void dcp_stall_check(struct work_struct *work); + #define IOMFB_METHOD(tag, name) [name] = { #name, { tag[0], tag[1], tag[2], tag[3] } } struct dcp_method_entry { @@ -194,12 +230,6 @@ struct dcp_map_buf_req { u8 dva_null; } __packed; -struct dcp_map_buf_resp { - u64 vaddr; - u64 dva; - u32 ret; -} __packed; - struct dcp_unmap_buf_resp { u64 buffer; u64 vaddr; @@ -240,20 +270,6 @@ struct dcp_map_physical_resp { u32 mem_desc_id; } __packed; -struct dcp_swap_start_req { - u32 swap_id; - struct dcp_iouserclient client; - u8 swap_id_null; - u8 client_null; - u8 padding[2]; -} __packed; - -struct dcp_swap_start_resp { - u32 swap_id; - struct dcp_iouserclient client; - u32 ret; -} __packed; - struct dcp_get_uint_prop_req { char obj[4]; char key[0x40]; @@ -282,13 +298,6 @@ struct iomfb_set_fx_prop_req { u32 value; } __packed; -struct dcp_set_power_state_req { - u64 unklong; - u8 unkbool; - u8 unkint_null; - u8 padding[2]; -} __packed; - struct dcp_set_power_state_resp { u32 unkint; u32 ret; diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index d0db6103e173f..81a68eadc3fbe 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -22,14 +22,17 @@ struct dcp_h17p_swap_info_request { static_assert(sizeof(struct dcp_h17p_hotplug_request) == 0x58); static_assert(sizeof(struct dcp_h17p_swap_info_request) == 0xe4); +/* D590 swap_complete_ap_gated: 0x730 on the wire in 1919/1919 traced swaps. */ +static_assert(sizeof(struct dc_swap_complete_resp_h17p) == 0x730); +static_assert(offsetof(struct dc_swap_complete_resp_h17p, swap_id) == 0); static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A000", dcpep_late_init_signal), - IOMFB_METHOD("A024", dcpep_setup_video_limits), - IOMFB_METHOD("A136", iomfbep_a131_pmu_service_matched), - IOMFB_METHOD("A137", iomfbep_a132_backlight_service_matched), - IOMFB_METHOD("A389", dcpep_set_create_dfb), - IOMFB_METHOD("A390", iomfbep_a358_vi_set_temperature_hint), + IOMFB_METHOD("A025", dcpep_setup_video_limits), /* boot chain: 14th call inside D121, right before flush_supports_power; (0,0) */ + IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), /* issued from inside callback D206 */ + IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), /* issued from inside callback D207 */ + IOMFB_METHOD("A385", dcpep_set_create_dfb), /* first call inside D121; (0,0) */ + IOMFB_METHOD("A386", iomfbep_a358_vi_set_temperature_hint), /* issued from inside callback D100 */ IOMFB_METHOD("A401", dcpep_start_signal), IOMFB_METHOD("A406", dcpep_swap_start), /* swap_start moved A407 -> A406 on H17P, but swap_submit did NOT move @@ -42,15 +45,44 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A413", dcpep_set_digital_out_mode), IOMFB_METHOD("A423", iomfbep_set_matrix), IOMFB_METHOD("A427", iomfbep_get_color_remap_mode), - IOMFB_METHOD("A449", dcpep_set_parameter_dcp), - IOMFB_METHOD("A453", dcpep_create_default_fb), - IOMFB_METHOD("A457", dcpep_enable_disable_video_power_savings), - IOMFB_METHOD("A465", dcpep_first_client_open), + IOMFB_METHOD("A442", dcpep_set_parameter_dcp), /* unique (0x28,0x4) signature in the capture */ + IOMFB_METHOD("A446", dcpep_create_default_fb), /* immediately follows set_create_dfb inside D121 */ + /* + * The default-framebuffer block macOS runs inside D121, straight after + * A446 and before A025. Sizes are the traced wire sizes. + */ + IOMFB_METHOD("A035", dcpep_get_dfb_compression_info), /* (0x8, 0x8) */ + IOMFB_METHOD("A034", dcpep_get_dfb_info), /* (0x2c, 0x2c) */ + IOMFB_METHOD("A103", dcpep_get_dfb_layout), /* (0xc, 0x8) */ + IOMFB_METHOD("A445", dcpep_get_dfb_state), /* (0x4, 0x8) */ + IOMFB_METHOD("A104", dcpep_set_dfb_dimensions), /* (0x8, 0) w,h */ + IOMFB_METHOD("A105", dcpep_commit_dfb_info), /* (0, 0) */ + IOMFB_METHOD("A033", dcpep_dfb_query), /* (0, 0x4) */ + IOMFB_METHOD("A380", dcpep_dfb_ready_query), /* (0, 0x4) */ + IOMFB_METHOD("A353", dcpep_pipe_query_353), /* (0, 0x8) */ + IOMFB_METHOD("A352", dcpep_pipe_set_352), /* (0x8, 0x4) in 0x70 */ + IOMFB_METHOD("A415", dcpep_pipe_cfg_415), /* (0xc, 0xc) zeros */ + IOMFB_METHOD("A031", dcpep_pipe_cfg_031), /* (0x6c, 0) zeros */ + IOMFB_METHOD("A414", dcpep_pipe_cfg_414), /* (0x8, 0x8) zeros */ + IOMFB_METHOD("A478", dcpep_pipe_query_478), /* (0, 0x4) */ + IOMFB_METHOD("A474", dcpep_pipe_query_474), /* (0, 0x4) */ + IOMFB_METHOD("A476", dcpep_pipe_set_476), /* (0x4, 0) in 1 */ + IOMFB_METHOD("A438", dcpep_pipe_cfg_438), /* (0x106c, 0x4) */ + IOMFB_METHOD("A428", dcpep_pipe_set_428), /* (0x4, 0x4) 0x10000 */ + IOMFB_METHOD("A450", dcpep_enable_disable_video_power_savings), /* init_1 position, in = u32 0 */ + IOMFB_METHOD("A457", dcpep_first_client_open), /* init_2, between A450 and A412; (0,0) */ + /* UNRESOLVED: A466 is never called by macOS. No capture contains a + * client teardown, so the real tag is unknown (the shift model suggests + * A458/A459). Left as-is; only reached on teardown. */ IOMFB_METHOD("A466", iomfbep_last_client_close), - IOMFB_METHOD("A474", dcpep_set_display_refresh_properties), - IOMFB_METHOD("A477", dcpep_flush_supports_power), + IOMFB_METHOD("A465", dcpep_set_display_refresh_properties), /* while it is in flight the DCP calls back D561 with displayMinRefreshInterval */ + IOMFB_METHOD("A468", dcpep_flush_supports_power), /* immediately follows setup_video_limits inside D121, in = 01 */ + /* UNRESOLVED: A478 is a live (0,0x4) method, so this entry would hand it + * a 0x14-byte record. The real abort_swaps tag is unknown (probably + * A469); no capture aborts a swap. Only reached on teardown. */ IOMFB_METHOD("A478", iomfbep_abort_swaps_dcp), - IOMFB_METHOD("A483", dcpep_set_power_state), + IOMFB_METHOD("A473", dcpep_set_power_state), /* the only OOBCMD call in the capture, matching this method's oob=true; out 0x8 */ + IOMFB_METHOD("A472", dcpep_register_dfb_surface), /* default FB surface, nested in D582 */ }; #define DCP_FW h17p @@ -68,9 +100,18 @@ static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, .input = in, .output = out, }; - u64 clock_request = dcp->disp_bw_doorbell_res.start; - u64 scratch = dcp->disp_bw_scratch_res.start + - dcp->disp_bw_scratch_offset; + /* + * Take these from the macOS capture rather than from the DT resources. + * macOS answers D003 with one stable record, + * 0000820203000000 0080860203000000 00000000 + * -> scratch 0x302820000, doorbell 0x302868000 -- which is + * DISP0_REG_BASE + 0x20000 and + 0x68000, both inside the 0xbc000 + * window we already hand out for map_reg index 5. The DT lookup lands + * 0x5a80000 higher, so the firmware was being pointed at the wrong + * block for its bandwidth scratch and doorbell. + */ + u64 clock_request = DCP_H17P_DISP0_REG_BASE + 0x68000; + u64 scratch = DCP_H17P_DISP0_REG_BASE + 0x20000; trace_iomfb_callback(dcp, tag, __func__); if (apple_dcp_h17p_rt_bw_encode_reply(&view, scratch, @@ -121,71 +162,193 @@ static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, return true; } +/* + * D574 (in 4, out 8) has no analogue in the v12.3/v13.5 tables. macOS 25G83 + * answers 0x435b0000 == 219.0f for the only selector it is ever called with + * (in == 0), identically in both captured boots. 219 is 235 - 16, the + * BT.601/709 studio-swing luma range, so this feeds the DCP's colour matrix + * setup; answering zero would put a zero scale into it. + */ +static bool iomfbep_cb_video_range_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + __le32 *resp = out; + + resp[0] = cpu_to_le32(0x435b0000); /* 219.0f */ + resp[1] = 0; + return true; +} + +/* + * Tag numbering below is the one the H17P firmware actually uses, taken from + * macOS 25G83 traces on this machine rather than by shifting the v13.5 table: + * the service-creation block is D108..D113 exactly as on v13.5, and the swap + * completion is D590. A callback with no entry here is never acked, so a + * missing tag stalls the DCP rather than degrading it. + */ +/* + * D400 -- iOSSvcProx::get_property(name, DataBuffer *, bool). + * + * The firmware asks the "PROV" (provider) service for the panel's power model + * during init_power_lut(); answering "not found" makes it log + * IOMFB init_power_lut: failed to read power-lut-data-x + * and then fail to create the default framebuffer surface. The request + * carries a capacity at +0x44 and the reply is { u8 data[0xc00]; __le32 + * length; }, where length is the byte count written and 0 means "no such + * property". These are the values macOS 25G83 returns on this machine, + * byte-identical across both captured boots. + */ +struct dcp_h17p_get_data_prop_req { + char obj[4]; + char key[0x40]; + __le32 length; + u8 data_null; + u8 length_null; + u8 padding[2]; +} __packed; +static_assert(sizeof(struct dcp_h17p_get_data_prop_req) == 0x4c); + +struct dcp_h17p_get_data_prop_resp { + u8 data[0xc00]; + __le32 length; +} __packed; +static_assert(sizeof(struct dcp_h17p_get_data_prop_resp) == 0xc04); + +static const u32 h17p_power_lut_x[] = { 13 }; +static const u32 h17p_power_lut_y[] = { 1 }; +static const u32 h17p_power_lut_yindex[] = { 100 }; +static const u32 h17p_power_lut_xindex[] = { + 2, 10, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500, 525, +}; +static const u32 h17p_power_lut_values[] = { + 820, 900, 1316, 1874, 2321, 2782, 3282, 3810, 4347, 4964, 5589, + 6248, 6562, +}; + +static const struct { + const char *key; + const u32 *vals; + u32 count; +} h17p_prov_props[] = { + { "power-lut-data-x", h17p_power_lut_x, ARRAY_SIZE(h17p_power_lut_x) }, + { "power-lut-data-y", h17p_power_lut_y, ARRAY_SIZE(h17p_power_lut_y) }, + { "power-lut-data-xindex", h17p_power_lut_xindex, + ARRAY_SIZE(h17p_power_lut_xindex) }, + { "power-lut-data-yindex", h17p_power_lut_yindex, + ARRAY_SIZE(h17p_power_lut_yindex) }, + { "power-lut-data-lut", h17p_power_lut_values, + ARRAY_SIZE(h17p_power_lut_values) }, +}; + +static bool trampoline_get_data_prop_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_get_data_prop_req *req = in; + struct dcp_h17p_get_data_prop_resp *resp = out; + u32 cap = min_t(u32, le32_to_cpu(req->length), sizeof(resp->data)); + unsigned int i, j; + + /* dcpep_handle_cb() already zeroed out, so length == 0 == not found */ + if (memcmp(req->obj, "VORP", sizeof(req->obj))) /* fourcc 'PROV' */ + return true; + + for (i = 0; i < ARRAY_SIZE(h17p_prov_props); i++) { + u32 bytes = h17p_prov_props[i].count * sizeof(u32); + + if (strncmp(req->key, h17p_prov_props[i].key, + sizeof(req->key))) + continue; + if (bytes > cap) + return true; + + for (j = 0; j < h17p_prov_props[i].count; j++) + ((__le32 *)resp->data)[j] = + cpu_to_le32(h17p_prov_props[i].vals[j]); + resp->length = cpu_to_le32(bytes); + return true; + } + + return true; +} + static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { - [0] = trampoline_true, /* did_boot_signal */ - [1] = trampoline_true, /* did_power_on_signal */ - [2] = trampoline_nop, /* will_power_off_signal */ + [0] = dcpep_cb_d000_h17p, /* macOS calls A033 inside D000 before acking */ + [1] = trampoline_true, + [2] = trampoline_nop, [3] = trampoline_rt_bandwidth_h17p, - [6] = trampoline_set_frame_sync_props, - [100] = iomfbep_cb_match_pmu_service, - [101] = trampoline_zero, /* get_display_default_stride */ + [6] = trampoline_set_frame_sync_props_h17p, /* 0x54/0x50, identity scale */ + [100] = iomfbep_cb_match_pmu_service, /* match_pmu_service */ + [101] = trampoline_zero, [102] = trampoline_nop, /* set_number_property */ - [103] = trampoline_nop, /* trigger_user_cal_loader */ [104] = trampoline_nop, /* set_boolean_property */ - [107] = trampoline_true, /* create_provider_service */ - [108] = trampoline_true, /* create_product_service */ - [109] = trampoline_true, /* create_pmu_service */ - [110] = trampoline_true, /* create_iomfb_service */ - [111] = trampoline_create_backlight_service, - [112] = trampoline_true, /* create_nvram_service */ - [114] = trampoline_get_tiling_state, - [115] = trampoline_false, /* set_tiling_state */ - [120] = dcpep_cb_boot_1, - [121] = trampoline_false, /* is_dark_boot */ - [122] = trampoline_false, /* is_waking_from_hibernate */ - [124] = trampoline_read_edt_data, - [126] = trampoline_prop_start, - [127] = trampoline_prop_chunk, - [128] = trampoline_prop_end, - [129] = trampoline_allocate_bandwidth, - [201] = trampoline_map_piodma, - [202] = trampoline_unmap_piodma, - [206] = iomfbep_cb_match_pmu_service_2, - [207] = iomfbep_cb_match_backlight_service, + [107] = trampoline_nop, + [108] = trampoline_true, /* create_provider_service */ + [109] = trampoline_true, /* create_product_service */ + [110] = trampoline_true, /* create_PMU_service */ + [111] = trampoline_true, /* create_iomfb_service */ + [112] = trampoline_create_backlight_service, /* create_backlight_service */ + [113] = trampoline_true, /* create_nvram_service */ + [114] = trampoline_zero, /* CoreAnalyticsSendEvent */ + [117] = trampoline_nop, /* set_idle_caching_state_ap */ + [118] = trampoline_zero, /* upload_trace_start */ + [119] = trampoline_zero, /* upload_trace_chunk */ + [120] = trampoline_zero, /* upload_trace_end */ + [121] = dcpep_cb_boot_1, /* start_hardware_boot */ + [122] = trampoline_false, /* is_dark_boot */ + [123] = trampoline_false, /* is_waking_from_hibernate */ + [124] = trampoline_zero, /* detect_fastsim */ + [125] = trampoline_read_edt_data, /* read_edt_data */ + [127] = trampoline_prop_start, /* setDCPAVPropStart */ + [128] = trampoline_prop_chunk, /* setDCPAVPropChunk */ + [129] = trampoline_prop_end, /* setDCPAVPropEnd */ + [130] = trampoline_allocate_bandwidth, /* allocate_bandwidth */ + [201] = trampoline_map_piodma, /* map_buf */ + [202] = trampoline_unmap_piodma, /* unmap_buf */ + [206] = iomfbep_cb_match_pmu_service_2, /* match_pmu_service */ + [207] = iomfbep_cb_match_backlight_service, /* match_backlight_service */ [208] = trampoline_nop, /* update_backlight_factor_prop */ - [209] = trampoline_get_time, + [209] = trampoline_get_time, /* get_calendar_time_ms */ [300] = trampoline_pr_publish, - [401] = trampoline_get_uint_prop, + [400] = trampoline_get_data_prop_h17p, /* get_property */ + [401] = trampoline_get_uint_prop, /* get_uint_prop */ [404] = trampoline_nop, /* set_uint_prop */ - [406] = trampoline_set_fx_prop, - [408] = trampoline_get_frequency_h17p, - [411] = trampoline_map_reg, - [413] = trampoline_true, /* set_property_dict */ - [414] = trampoline_sr_set_property_int, - [415] = trampoline_true, /* set_property_bool */ - [451] = trampoline_allocate_buffer, - [452] = trampoline_map_physical, - [454] = trampoline_release_mem_desc, - [552] = trampoline_true, /* set_property_dict_0 */ - [561] = trampoline_true, /* set_property_dict */ - [563] = trampoline_true, /* set_property_int */ - [565] = trampoline_true, /* set_property_bool */ - [567] = trampoline_true, /* set_property_str */ - [572] = trampoline_zero, /* power_up_dart */ + [406] = trampoline_set_fx_prop, /* set_fx_prop */ + [408] = trampoline_get_frequency_h17p, /* getClockFrequency */ + [411] = trampoline_map_reg, /* mapDeviceMemoryWithIndex */ + [413] = trampoline_true, /* setProperty */ + [414] = trampoline_sr_set_property_int, /* setProperty */ + [415] = trampoline_true, /* setProperty */ + [451] = trampoline_allocate_buffer, /* allocate_buffer */ + [452] = trampoline_map_physical, /* prepare */ + [454] = trampoline_release_mem_desc, /* release_descriptor */ + [552] = trampoline_true, + [561] = trampoline_true, + [563] = trampoline_true, + [565] = trampoline_true, + [567] = trampoline_true, + [572] = trampoline_zero, /* powerUpDART */ + [574] = iomfbep_cb_video_range_h17p, [575] = trampoline_hotplug_h17p, - [576] = trampoline_nop, /* powerstate_notify */ - [586] = trampoline_true, /* create_default_fb_surface */ - [587] = trampoline_nop, /* clear_default_surface */ - [588] = trampoline_nop, /* swap_notify */ - [589] = trampoline_swap_info_h17p, - [594] = trampoline_swap_complete, - [596] = trampoline_swap_complete_intent_gated, - [597] = trampoline_abort_swap_ap_gated, - [598] = trampoline_enable_backlight_message_ap_gated, - [599] = trampoline_nop, /* setSystemConsoleMode */ - [601] = trampoline_false, /* isDFBAllocated */ - [602] = trampoline_false, /* preserveContents */ - [603] = trampoline_nop, /* find_swap_function_gated */ + [576] = trampoline_nop, + /* + * We do not create a default framebuffer surface, so answering "yes" + * makes the firmware go on to program the display engine for a surface + * that does not exist: it then DMAs from a null base and the piodma + * DART (412300000) faults continuously on SID 0, which starves the + * system into an RCU stall. Decline instead. + */ + [582] = iomfbep_cb_create_dfb_surface, /* create_default_fb_surface */ + [583] = trampoline_nop, /* clear_default_surface */ + [584] = trampoline_nop, /* swap_notify_gated */ + [585] = trampoline_swap_info_h17p, /* swap_info_notify_dispatch */ + [590] = trampoline_swap_complete, /* swap_complete_ap_gated */ + [592] = trampoline_swap_complete_intent_gated, /* swap_complete_intent_gated */ + [593] = trampoline_abort_swap_ap_gated, /* abort_swap_ap_gated */ + [594] = trampoline_enable_backlight_message_ap_gated, /* enable_backlight_message_ap_gated */ + [595] = trampoline_nop, /* setSystemConsoleMode */ + [597] = trampoline_false, /* isDFBAllocated */ + [598] = trampoline_false, + [599] = trampoline_nop, /* find_swap_function_gated */ }; void iomfb_start_h17p(struct apple_dcp *dcp) diff --git a/drivers/gpu/drm/apple/iomfb_internal.h b/drivers/gpu/drm/apple/iomfb_internal.h index 75e9d7b0e8cc8..4706b68db34d2 100644 --- a/drivers/gpu/drm/apple/iomfb_internal.h +++ b/drivers/gpu/drm/apple/iomfb_internal.h @@ -11,6 +11,9 @@ struct apple_dcp; typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); +extern u32 dcp_clamshell_state; +extern bool dcp_send_modeset; + #define DCP_THUNK_VOID(func, handle) \ static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ void *cookie) \ @@ -99,6 +102,9 @@ typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); return true; \ } +extern bool dcp_defer_client_open; +extern bool dcp_park_display; + /* Call a DCP function given by a tag */ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, u32 in_len, u32 out_len, void *data, dcp_callback_t cb, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index e2942b8869220..dfe0e250b9cb5 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -20,6 +20,12 @@ #include #include + +/* macOS uses 6 for its single layer in the captured swaps */ +#define DCP_SWAP_SURFACE_ID_BASE 6 +/* from the captured macOS swap records */ +#define DCP_SWAP_FLAGS1 0x34865202ULL +#define DCP_SWAP_FLAGS2 0x4ULL #include #include #include @@ -64,13 +70,25 @@ DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct DCP_FW_NAME(dcp_swap_submit_req), struct DCP_FW_NAME(dcp_swap_submit_resp)); -DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct dcp_swap_start_req, - struct dcp_swap_start_resp); +DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct DCP_FW_NAME(dcp_swap_start_req), + struct DCP_FW_NAME(dcp_swap_start_resp)); DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, - struct dcp_set_power_state_req, + struct DCP_FW_NAME(dcp_set_power_state_req), struct dcp_set_power_state_resp); +/* + * macOS issues A473 out-of-band: the traced call is "OOBCMD 0x0 A473 0x10 0x8" + * with unklong = 1. Sent on the ordinary command channel the firmware simply + * never answers it, which strands the power transition and every later call + * behind it. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_POWER_OOB true +#else +#define DCP_POWER_OOB false +#endif + DCP_THUNK_INOUT(dcp_set_digital_out_mode, dcpep_set_digital_out_mode, struct dcp_set_digital_out_mode_req, u32); @@ -89,6 +107,82 @@ DCP_THUNK_OUT(dcp_create_default_fb, dcpep_create_default_fb, u32); DCP_THUNK_OUT(dcp_start_signal, dcpep_start_signal, u32); DCP_THUNK_VOID(dcp_setup_video_limits, dcpep_setup_video_limits); DCP_THUNK_VOID(dcp_set_create_dfb, dcpep_set_create_dfb); + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * The default-framebuffer interrogation macOS performs inside D121. The + * getters are called with a zeroed input of the traced size; macOS leaves the + * tail as poison, so only the size matters. + */ +struct dcp_dfb_info_h17p { + u8 data[0x2c]; +} __packed; + +struct dcp_dfb_layout_req_h17p { + u8 data[0xc]; +} __packed; + +struct dcp_dfb_dimensions_h17p { + u32 width; + u32 height; +} __packed; + +DCP_THUNK_INOUT(dcp_get_dfb_compression_info, dcpep_get_dfb_compression_info, + u64, u64); +DCP_THUNK_INOUT(dcp_get_dfb_info, dcpep_get_dfb_info, + struct dcp_dfb_info_h17p, struct dcp_dfb_info_h17p); +DCP_THUNK_INOUT(dcp_get_dfb_layout, dcpep_get_dfb_layout, + struct dcp_dfb_layout_req_h17p, u64); +DCP_THUNK_INOUT(dcp_get_dfb_state, dcpep_get_dfb_state, u32, u64); +DCP_THUNK_IN(dcp_set_dfb_dimensions, dcpep_set_dfb_dimensions, + struct dcp_dfb_dimensions_h17p); +DCP_THUNK_VOID(dcp_commit_dfb_info, dcpep_commit_dfb_info); +DCP_THUNK_OUT(dcp_dfb_query, dcpep_dfb_query, u32); +DCP_THUNK_OUT(dcp_dfb_ready_query, dcpep_dfb_ready_query, u32); + +/* + * macOS runs a fixed sequence of pipe-configuration calls between the boot + * chain and its first swap. We do not have names for most of them, but the + * capture gives exact sizes and payloads, and replaying it verbatim is what + * takes the firmware from "booted" to "willing to scan out". + */ +struct dcp_pipe_cfg_415 { + u8 data[0xc]; +} __packed; + +struct dcp_pipe_cfg_031 { + u8 data[0x6c]; +} __packed; + +DCP_THUNK_OUT(dcp_pipe_query_353, dcpep_pipe_query_353, u64); +DCP_THUNK_INOUT(dcp_pipe_set_352, dcpep_pipe_set_352, u64, u32); +DCP_THUNK_INOUT(dcp_pipe_cfg_415, dcpep_pipe_cfg_415, + struct dcp_pipe_cfg_415, struct dcp_pipe_cfg_415); +DCP_THUNK_IN(dcp_pipe_cfg_031, dcpep_pipe_cfg_031, struct dcp_pipe_cfg_031); +DCP_THUNK_INOUT(dcp_pipe_cfg_414, dcpep_pipe_cfg_414, u64, u64); +DCP_THUNK_OUT(dcp_pipe_query_478, dcpep_pipe_query_478, u32); +DCP_THUNK_OUT(dcp_pipe_query_474, dcpep_pipe_query_474, u32); +DCP_THUNK_IN(dcp_pipe_set_476, dcpep_pipe_set_476, u32); +DCP_THUNK_INOUT(dcp_pipe_set_428, dcpep_pipe_set_428, u32, u32); + +/* + * A352 and the five A438 pushes macOS runs between A353 and the client-open + * sequence. Every byte is taken from the 25G83 capture; the 8-byte host + * leading 8 bytes are sent + * verbatim: zeroing them made the firmware take a CPU exception + * (exception.c:59) on the third push. + */ +struct dcp_pipe_cfg_438 { + u8 data[0x106c]; +} __packed; + +DCP_THUNK_INOUT(dcp_pipe_cfg_438, dcpep_pipe_cfg_438, + struct dcp_pipe_cfg_438, u32); + +/* Generated from the macOS 25G83 capture: five A438 pushes, 0x106c each, + * byte-exact including the leading 8 bytes. */ + +#endif DCP_THUNK_VOID(dcp_first_client_open, dcpep_first_client_open); DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, @@ -152,6 +246,100 @@ static bool iomfbep_cb_match_pmu_service(struct apple_dcp *dcp, int tag, void *o return false; } +/* + * H17P asks the AP to make the default framebuffer surface (D582). Answering + * with a bare boolean is not enough: macOS registers an actual surface from + * inside that callback with a nested A472 carrying a 0x22c descriptor, and + * only then reports success. Without it the firmware has no surface, programs + * the display engine from a null base and the piodma DART faults forever. + * + * The descriptor below is the one macOS 25G83 sends on this panel; every byte + * it did not write was left as tracer poison, so the rest is zero. ARGB at + * +0x0b is the format field of struct dcp_surface. + */ +struct dcp_h17p_dfb_surface_req { + u8 surface[0x22c]; + u8 surface_null; + u8 padding[3]; +} __packed; +static_assert(sizeof(struct dcp_h17p_dfb_surface_req) == 0x230); + +/* macOS sends this with a 4-byte output; with out_len 0 the firmware has + * nowhere to write its result and simply stops answering. */ +DCP_THUNK_INOUT(dcp_register_dfb_surface, dcpep_register_dfb_surface, + struct dcp_h17p_dfb_surface_req, u32); + +/* Fill the common SurfaceContents fields, matching the traced macOS bytes. */ + +static void dcp_ack_dfb_surface(struct apple_dcp *dcp) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static void dcp_fill_dfb_surface(u8 *s) +{ + s[0x002] = 0x01; + s[0x00b] = 'A'; + s[0x00c] = 'R'; + s[0x00d] = 'G'; + s[0x00e] = 'B'; + s[0x019] = 0x01; + s[0x01b] = 0x01; + s[0x01c] = 0x01; + s[0x02a] = 0x40; + s[0x02b] = 0xe4; + s[0x035] = 0x03; + s[0x051] = 0x01; + s[0x149] = 0x01; +} + +/* + * A471 (update_dfb) is never sent. It hands the firmware a default-framebuffer + * surface descriptor, and on H17P it makes the coprocessor take an AXI read + * error and NMI immediately (axi_rd_err -> "panic(app): app NMI"). Scanout does + * not need it: the swap carries its own surface, whose DVA comes from the DRM + * GEM object on the display-subsystem device and is therefore already mapped in + * the stream the pipe reads. Just acknowledge the callback. + */ +static void complete_dfb_surface(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_ack_dfb_surface(dcp); +} + +static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + struct dcp_h17p_dfb_surface_req *req; + + trace_iomfb_callback(dcp, tag, __func__); + + /* + * 0x230 bytes is far too much for the kernel stack here: this runs deep + * inside the RTKit receive workqueue (rx work -> iomfb_recv_msg -> + * dcpep_handle_cb -> here -> dcp_push), and putting it on the stack + * overflows it and panics the machine. dcp_push() copies the payload + * into shared memory before returning, so a short-lived allocation is + * all we need. + */ + req = kzalloc(sizeof(*req), GFP_KERNEL); + if (!req) { + dev_err(dcp->dev, "no memory for the default FB surface\n"); + return true; + } + + dcp_fill_dfb_surface(req->surface); + + dcp_register_dfb_surface(dcp, false, req, complete_dfb_surface, NULL); + kfree(req); + + /* deferred ACK: the A472 completion reports success for us */ + return false; +} + static void complete_pmu_service_matched(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_channel *ch = &dcp->ch_cb; @@ -265,7 +453,7 @@ static void iomfbep_cb_set_fx_prop(struct apple_dcp *dcp, struct iomfb_set_fx_pr * PIODMA is separate from the main DCP and uses own IOVA space on a dedicated * stream of the display DART, rather than the expected DCP DART. */ -static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, +static struct DCP_FW_NAME(dcp_map_buf_resp) dcpep_cb_map_piodma(struct apple_dcp *dcp, struct dcp_map_buf_req *req) { struct dcp_mem_descriptor *memdesc; @@ -293,12 +481,43 @@ static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, goto reject; } - return (struct dcp_map_buf_resp){ .dva = memdesc->dva }; + /* + * Mirror it into the display-subsystem domain at the same address: the + * display engine reads these buffers on that device's stream, which + * otherwise has no mapping for them at all. + */ + /* + * Do not mirror into the display-subsystem domain. That stream's + * tables are iBoot's, pinned to the DART's published "pt-region", and + * macOS never adds entries to them from the host -- the firmware does + * its own remapping there (the ADT marks the DART "allow-pte-remap"). + */ + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* echo the index back, as macOS does */ + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ + .buffer = req->buffer, + .dva = memdesc->dva, + }; +#else + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .dva = memdesc->dva }; +#endif reject: dev_err(dcp->dev, "denying map of invalid buffer %llx for piodma\n", req->buffer); - return (struct dcp_map_buf_resp){ .ret = EINVAL }; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ + .buffer = req->buffer, + .dva = 0, + }; +#else + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .dva = 0 }; +#endif +#else + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .ret = EINVAL }; +#endif } static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, @@ -374,6 +593,17 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, size); resp.dva = memdesc->dva; + /* + * Report the physical address too. We only ever filled in the device + * address, leaving paddr zero; macOS answers D451 with a real physical + * address (e.g. 0x100_2651_4000) alongside the DVA. The firmware loads + * its APT M3 microcode -- "load APT M3 IMem/DMem" appears immediately + * after these allocations -- through that address, so with zero there + * the timing sequencer comes up and is powered straight back down. + */ + if (memdesc->map.sgl) + resp.paddr = sg_phys(memdesc->map.sgl); + return resp; } @@ -467,6 +697,29 @@ static u64 dcpep_cb_get_frequency(struct apple_dcp *dcp) static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *dcp, struct DCP_FW_NAME(dcp_map_reg_req) *req) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P asks for one aperture the device tree does not describe on the + * dcp node: index 5, the disp0 window 0x302800000..0x3028bbfff. That + * window holds the PMC block the firmware drives while bringing the + * display up, and macOS answers this exact request with + * addr = dva = 0x302800000, length = 0xbc000. Refusing it (our + * reg-names stop at disp-4) leaves the firmware unable to program that + * hardware. Answer it the way macOS does; these apertures are reached + * physically, so dva == addr. + */ + if (req->index == DCP_H17P_DISP0_REG_INDEX) { + dev_info(dcp->dev, "map_reg[5]: disp0 window %#llx len %#llx\n", + (u64)DCP_H17P_DISP0_REG_BASE, + (u64)DCP_H17P_DISP0_REG_SIZE); + return (struct DCP_FW_NAME(dcp_map_reg_resp)){ + .addr = DCP_H17P_DISP0_REG_BASE, + .length = DCP_H17P_DISP0_REG_SIZE, + .dva = DCP_H17P_DISP0_REG_BASE, + }; + } +#endif + if (req->index >= dcp->nr_disp_registers) { dev_warn(dcp->dev, "attempted to read invalid reg index %u\n", req->index); @@ -474,10 +727,25 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d return (struct DCP_FW_NAME(dcp_map_reg_resp)){ .ret = 1 }; } else { struct resource *rsrc = dcp->disp_registers[req->index]; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P reaches these apertures physically, not through its + * DART: macOS 25G83 answers this callback with dva == addr + * (0x302800000 for index 5, length 0xbc000). Trying to + * dma_map_resource() them instead fails and hands the firmware + * DMA_MAPPING_ERROR as a device address. + */ + dma_addr_t dva = rsrc->start; +#elif DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) dma_addr_t dva = dma_map_resource(dcp->dev, rsrc->start, resource_size(rsrc), DMA_BIDIRECTIONAL, 0); WARN_ON(dva == DMA_MAPPING_ERROR); + dev_info(dcp->dev, "map_reg[%u]: addr %pa len %#llx dva %#llx\n", + req->index, &rsrc->start, + (u64)resource_size(rsrc), (u64)dva); +#else + dev_info(dcp->dev, "map_reg[%u]: addr %pa len %#llx\n", + req->index, &rsrc->start, (u64)resource_size(rsrc)); #endif return (struct DCP_FW_NAME(dcp_map_reg_resp)){ @@ -526,13 +794,30 @@ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) return false; } + /* + * This length comes straight off the wire. A bogus value turns the + * allocation below into a multi-gigabyte GFP_KERNEL request, and the + * box vanishes into reclaim with no oops and no fault -- it just stops. + * Property blobs here are a few KiB, so bound it and never let the + * allocator retry. + */ + if (*length == 0 || *length > SZ_1M) { + dev_warn(dcp->dev, "rejecting %u byte property transfer\n", + *length); + return false; + } + dcp->chunks.length = *length; - dcp->chunks.data = kzalloc(*length, GFP_KERNEL); + dcp->chunks.data = kzalloc(*length, + GFP_KERNEL | __GFP_NORETRY | + __GFP_NOWARN); if (!dcp->chunks.data) { - dev_warn(dcp->dev, "failed to allocate chunks\n"); + dev_warn(dcp->dev, "failed to allocate %u chunk bytes\n", + *length); return false; } + dev_info(dcp->dev, "property transfer: %u bytes\n", *length); return true; } @@ -635,21 +920,46 @@ static void boot_done(struct apple_dcp *dcp, void *out, void *cookie) dcp_ack(dcp, DCP_CONTEXT_CB); } -static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) +static void boot_5b(struct apple_dcp *dcp, void *out, void *cookie) { dcp_set_display_refresh_properties(dcp, false, boot_done, NULL); } -static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) +static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* macOS issues A380 right before set_display_refresh_properties */ + dcp_dfb_ready_query(dcp, false, boot_5b, NULL); +#else + boot_5b(dcp, out, cookie); +#endif +} + +static void boot_4b(struct apple_dcp *dcp, void *out, void *cookie) { #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u32 v_true = 1; - dcp_late_init_signal(dcp, false, &v_true, boot_5, NULL); + /* + * macOS 25G83 sends 0 here, not 1: the traced A000 payload is + * "00 aa aa aa", one meaningful byte followed by poison. + */ + u32 arg = 0; + + dcp_late_init_signal(dcp, false, &arg, boot_5, NULL); #else dcp_late_init_signal(dcp, false, boot_5, NULL); #endif } +static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* macOS issues A033 between flush_supports_power and late_init_signal */ + dcp_dfb_query(dcp, false, boot_4b, NULL); +#else + boot_4b(dcp, out, cookie); +#endif +} + static void boot_3(struct apple_dcp *dcp, void *out, void *cookie) { u32 v_true = true; @@ -662,15 +972,137 @@ static void boot_2(struct apple_dcp *dcp, void *out, void *cookie) dcp_setup_video_limits(dcp, false, boot_3, NULL); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * macOS 25G83 runs this between A446 and A025, every boot: + * A035 -> A034 -> A103 -> A445 -> A104(w,h) -> A105 + * We were jumping straight from A446 to A025, so the firmware never got told + * the default framebuffer's geometry and never finished its own DFB setup. + */ +static void boot_1_11(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_commit_dfb_info(dcp, false, boot_2, NULL); +} + +static void boot_1_10(struct apple_dcp *dcp, void *out, void *cookie) +{ + /* + * The firmware only builds a default framebuffer when it has non-zero + * geometry (its create_default_fb_surface is gated on "h && w"), so + * withholding A104 keeps the pipe inert and lets a real DRM modeset + * drive the first swap instead. macOS always pushes it, but macOS has + * an inherited scanout to keep alive and we do not. + */ + if (0) { + boot_1_11(dcp, NULL, NULL); + return; + } + + /* + * The panel's native size, which is what macOS pushes here: the traced + * A104 payload is 68 09 00 00 e2 05 00 00 = 2408 x 1506. The firmware + * has already told us the same numbers in D582. + */ + struct dcp_dfb_dimensions_h17p dim = { + .width = 2408, + .height = 1506, + }; + + dcp_set_dfb_dimensions(dcp, false, &dim, boot_1_11, NULL); +} + +static void boot_1_9(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 zero = 0; + + dcp_get_dfb_state(dcp, false, &zero, boot_1_10, NULL); +} + +static void boot_1_8(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_dfb_layout_req_h17p req = {}; + + dcp_get_dfb_layout(dcp, false, &req, boot_1_9, NULL); +} + +static void boot_1_7(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_dfb_info_h17p info = {}; + + dcp_get_dfb_info(dcp, false, &info, boot_1_8, NULL); +} + +static void boot_1_6(struct apple_dcp *dcp, void *out, void *cookie) +{ + u64 zero = 0; + + dcp_get_dfb_compression_info(dcp, false, &zero, boot_1_7, NULL); +} +#endif + static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + dcp_create_default_fb(dcp, false, boot_1_6, NULL); +#else dcp_create_default_fb(dcp, false, boot_2, NULL); +#endif } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +static void d000_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +/* + * macOS does not answer D000 straight away: it issues A033 first and only then + * acks with 1. Defer the ACK the same way. + */ +static bool dcpep_cb_d000_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + dcp_dfb_query(dcp, false, d000_done, NULL); + return false; +} +#endif + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +static void boot_1_after_park(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_set_create_dfb(dcp, false, boot_1_5, NULL); +} +#endif + /* Use special function signature to defer the ACK */ static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) { trace_iomfb_callback(dcp, tag, __func__); + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * Park the display before the firmware configures the pipe, not after. + * + * m1n1 stops the ASC, so our firmware boots with no surface (macOS + * adopts the live generation and always has one). It enables the pipe + * anyway while finishing start_hardware_boot and fetches from a null + * base -- an address below the DART's window, so it cannot be mapped, + * only prevented. The reject becomes an AXI error and NMIs the + * coprocessor before a modeset can ever run. Powering the display down + * first keeps the pipe quiet until we drive a real mode. + */ + /* + * Do not park here. macOS never powers the display down at all -- + * start_hardware_boot has no blank or pipe-disable step, and its only + * A473 in a full capture is unklong=1 (power on). Parking at the head + * of the boot chain stops the firmware booting outright. + */ +#endif + dcp_set_create_dfb(dcp, false, boot_1_5, NULL); return false; } @@ -693,6 +1125,7 @@ static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) .doorbell_bit = 0, }; + if (dcp->disp_bw_scratch_index) { u32 offset = dcp->disp_bw_scratch_offset; u32 index = dcp->disp_bw_scratch_index; @@ -714,6 +1147,46 @@ static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) return rt_bw; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P's D006 is 0x54 in / 0x50 out, far wider than the 28-byte + * frame_sync_props of older firmware, so the fields that matter sit past + * anything the old struct could describe. + * + * macOS 25G83 answers it with zeros except three u32 at 0x24, 0x28 and 0x2c, + * each 0x10000 -- 1.0 in 16.16 fixed point, the identity scale. Answering all + * zeros hands the pipe a zero scale, and it then programs a null surface base: + * the display DART immediately faults at DVA 0, that reject becomes an AXI + * error, and the firmware takes an NMI (apiodma_irq_AxiErr). + */ +#define DCP_FRAME_SYNC_IDENTITY 0x10000 + +struct dcp_set_frame_sync_props_req_h17p { + u8 unk[0x54]; +} __packed; + +struct dcp_set_frame_sync_props_resp_h17p { + u8 unk[0x50]; +} __packed; + +static struct dcp_set_frame_sync_props_resp_h17p +dcpep_cb_set_frame_sync_props_h17p(struct apple_dcp *dcp, + struct dcp_set_frame_sync_props_req_h17p *req) +{ + struct dcp_set_frame_sync_props_resp_h17p resp = {}; + static const u16 identity_at[] = { 0x24, 0x28, 0x2c }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(identity_at); i++) { + u32 v = DCP_FRAME_SYNC_IDENTITY; + + memcpy(&resp.unk[identity_at[i]], &v, sizeof(v)); + } + + return resp; +} +#endif + static struct dcp_set_frame_sync_props_resp dcpep_cb_set_frame_sync_props(struct apple_dcp *dcp, struct dcp_set_frame_sync_props_req *req) @@ -741,6 +1214,80 @@ static void release_swap_cookie(struct kref *ref) kfree(cookie); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P's swap surface has no is_tearing_allowed and no is_premultiplied: + * plane_cnt sits at +1, plane_cnt2 at +5 and format at +9, two bytes earlier + * than every other firmware. + * + * Proof from the capture: macOS's A408 carries its plane counts at surface+1 + * and +5 and its format at +9, and reading a u32 at +9 out of *our* record + * gives 00 00 41 52, which the firmware prints MSB-first as "RA" -- exactly + * the text it rejects us with ("verify_surfaces: No support for format RA"). + * + * Shift each surface's tail down by two rather than fork the shared struct, + * which the A472 default-framebuffer path also uses. + */ +static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) +{ + struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); + unsigned int i; + + /* + * The swap's own flag words. Our driver never sets either; macOS's + * captured swaps carry flags1 = 0x34865202 and flags2 = 4 in every + * record, at +0x40 and +0x48. + */ + req->swap.flags1 = DCP_SWAP_FLAGS1; + req->swap.flags2 = DCP_SWAP_FLAGS2; + + /* + * The two counters inside the H17P gap that we never modelled: macOS + * carries 2 at +0x50 and 3 at +0x78 in every captured swap. The gap + * starts at +0x50, so these are indices 0 and 0x28 of the blob. + */ + req->swap.h17p_pre_swap_id[0x00] = 2; + req->swap.h17p_pre_swap_id[0x28] = 3; + + for (i = 0; i < SWAP_SURFACES; i++) { + u8 *p = (u8 *)&req->surf[i]; + size_t n = sizeof(req->surf[i]); + + memmove(p + 1, p + 3, n - 3); + memset(p + n - 2, 0, 2); + } + + /* + * Our trailing null-flag block lands two bytes late. In the capture + * macOS's flags sit at +0xe94/+0xe95/+0xe99 of the 0xe9c record while + * ours are at +0xe96/+0xe97/+0xe98/+0xe9a/+0xe9b -- so the last of them + * run off the end and the firmware reads the whole block misaligned. + * It then believes layers are present that are not, verifies their + * empty rectangles, and rejects the swap with + * "verify_scaling: error: null rectangle: src:0x0+0,0 dst:0x0+0,0". + */ + { + u8 *r = (u8 *)req; + size_t n = sizeof(*req); + size_t off = offsetof(typeof(*req), surf_iova); + + /* + * The two bytes are not just in the tail: everything from + * surf_iova onward is late. macOS's A408 carries + * surf_iova[0] at +0xe38 of the 0xe9c record, ours sat at + * +0xe3a, so the firmware read our DVA 0x100075fc000 two bytes + * early as 0x0100075fc0000000 and faulted on its low 36 bits, + * 0xfc0000000 -- an address with no L1 entry at all. Shift the + * whole block, which subsumes the null-flag fixup. + */ + memmove(r + off, r + off + 2, n - off - 2); + memset(r + n - 2, 0, 2); + } +} +#else +static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) { } +#endif + static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; @@ -773,7 +1320,7 @@ static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, void *cookie) { - struct dcp_swap_start_resp *resp = data; + struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; if (cookie) { @@ -781,6 +1328,7 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, info->swap_id = resp->swap_id; } + dcp_h17p_fix_swap_surfaces(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); } @@ -796,11 +1344,11 @@ static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) static void dcp_on_set_power_state(struct apple_dcp *dcp, void *out, void *cookie) { - struct dcp_set_power_state_req req = { + struct DCP_FW_NAME(dcp_set_power_state_req) req = { .unklong = 1, }; - dcp_set_power_state(dcp, false, &req, dcp_on_final, cookie); + dcp_set_power_state(dcp, DCP_POWER_OOB, &req, dcp_on_final, cookie); } static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) @@ -835,9 +1383,20 @@ void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) kref_get(&cookie->refcount); if (dcp->main_display) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * macOS never repeats A411 at power-on -- it issues it once, in + * the client-open sequence, and goes straight to the power + * state from here. Repeating it after the client is open is a + * call the firmware simply stops answering, which wedges the + * command channel and with it every later swap. + */ + dcp_on_set_power_state(dcp, NULL, cookie); +#else handle = 0; dcp_set_display_device(dcp, false, &handle, dcp_on_set_power_state, cookie); +#endif } else { handle = 2; dcp_set_display_device(dcp, false, &handle, @@ -878,10 +1437,10 @@ static void complete_set_powerstate(struct apple_dcp *dcp, void *out, static void last_client_closed_poff(struct apple_dcp *dcp, void *out, void *cookie) { - struct dcp_set_power_state_req power_req = { + struct DCP_FW_NAME(dcp_set_power_state_req) power_req = { .unklong = 0, }; - dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, + dcp_set_power_state(dcp, DCP_POWER_OOB, &power_req, complete_set_powerstate, cookie); } @@ -902,7 +1461,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) }; struct dcp_swap_cookie *cookie; struct dcp_wait_cookie *poff_cookie; - struct dcp_swap_start_req swap_req = { 0 }; + struct DCP_FW_NAME(dcp_swap_start_req) swap_req = { 0 }; struct DCP_FW_NAME(dcp_swap_submit_req) *swap = &DCP_FW_UNION(dcp->swap); cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); @@ -988,10 +1547,10 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) static void last_client_closed_sleep(struct apple_dcp *dcp, void *out, void *cookie) { - struct dcp_set_power_state_req power_req = { + struct DCP_FW_NAME(dcp_set_power_state_req) power_req = { .unklong = 0, }; - dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, cookie); + dcp_set_power_state(dcp, DCP_POWER_OOB, &power_req, complete_set_powerstate, cookie); } static void aborted_swaps_dcp_sleep(struct apple_dcp *dcp, void *out, void *cookie) @@ -1122,7 +1681,16 @@ dcpep_cb_get_tiling_state(struct apple_dcp *dcp, static u8 dcpep_cb_create_backlight_service(struct apple_dcp *dcp) { - return dcp_has_panel(dcp); + /* + * dcp_has_panel() only becomes true once the panel dimensions have been + * parsed out of the EDID, which happens long after this callback: the + * firmware asks during start_ios_gated, before any mode is known. An + * internal panel always has a backlight, so answer for the connector. + * macOS 25G83 answers true here in both captured boots; answering false + * makes the firmware log "Could not find Backlight service". + */ + return dcp_has_panel(dcp) || + dcp->connector_type == DRM_MODE_CONNECTOR_eDP; } TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); @@ -1136,7 +1704,7 @@ TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, TRAMPOLINE_IN(trampoline_set_fx_prop, iomfbep_cb_set_fx_prop, struct iomfb_set_fx_prop_req) TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, - struct dcp_map_buf_req, struct dcp_map_buf_resp); + struct dcp_map_buf_req, struct DCP_FW_NAME(dcp_map_buf_resp)); TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, struct dcp_unmap_buf_resp); TRAMPOLINE_INOUT(trampoline_sr_set_property_int, iomfbep_cb_sr_set_property_int, @@ -1165,6 +1733,12 @@ TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, TRAMPOLINE_INOUT(trampoline_set_frame_sync_props, dcpep_cb_set_frame_sync_props, struct dcp_set_frame_sync_props_req, struct dcp_set_frame_sync_props_resp); +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +TRAMPOLINE_INOUT(trampoline_set_frame_sync_props_h17p, + dcpep_cb_set_frame_sync_props_h17p, + struct dcp_set_frame_sync_props_req_h17p, + struct dcp_set_frame_sync_props_resp_h17p); +#endif TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); @@ -1212,18 +1786,19 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) { - struct dcp_swap_start_resp *resp = data; + struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; trace_iomfb_swap_submit(dcp, resp->swap_id); + dcp_h17p_fix_swap_surfaces(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swapped, NULL); } /* Helpers to modeset and swap, used to flush */ static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) { - struct dcp_swap_start_req start_req = { 0 }; + struct DCP_FW_NAME(dcp_swap_start_req) start_req = { 0 }; if (dcp->connector && dcp->connector->connected) dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); @@ -1231,6 +1806,20 @@ static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) dcp_drm_crtc_vblank(dcp->crtc); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * macOS issues A428 between set_matrix and the first swap: CMD, in 4, out 4, + * payload 00 00 01 00 -- 0x10000, 1.0 in 16.16, the same identity value it + * answers D006 with. We went straight from A423 to A406. + */ +static void dcp_pre_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 identity = 0x10000; + + dcp_pipe_set_428(dcp, false, &identity, do_swap, cookie); +} +#endif + static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, void *cookie) { @@ -1326,6 +1915,22 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, dcp->during_modeset = true; dcp->swap_submit_timestamp = 0; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * Inherit the mode iBoot already programmed instead of asking for one. + * + * macOS never issues A413 in a full capture -- it attaches to a display + * iBoot has already configured. Ours asks for colorID 1 / timingID 2 + * and the firmware answers 0x8000000b, so the timing generator never + * starts: no vblank, no swap completion, black panel. The panel is + * already running at its native 2408x1506, so skip the modeset and let + * that timing stand. + */ + if (!dcp_send_modeset) { + dev_info(dcp->dev, "inheriting the bootloader's display mode\n"); + complete_set_digital_out_mode(dcp, NULL, cookie); + } else +#endif if (mode->vrr) dcp_set_adaptive_sync(dcp, crtc_state->vrr_enabled ? mode->min_vrr : 0, @@ -1533,7 +2138,11 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru mat.matrix[0] = mat.matrix[4] = mat.matrix[8] = 1LLU << 32; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + iomfb_set_matrix(dcp, false, &mat, dcp_pre_swap_a428, NULL); +#else iomfb_set_matrix(dcp, false, &mat, do_swap, NULL); +#endif } else do_swap(dcp, NULL, NULL); } @@ -1546,6 +2155,7 @@ static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) dcp->main_display = result != 0; + connector = dcp->connector; if (connector) { connector->connected = dcp->nr_modes > 0; @@ -1556,6 +2166,8 @@ static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) complete(&dcp->start_done); } +static void init_3(struct apple_dcp *dcp, void *out, void *cookie); + static void init_3(struct apple_dcp *dcp, void *out, void *cookie) { dcp_is_main_display(dcp, false, res_is_main_display, NULL); @@ -1563,15 +2175,155 @@ static void init_3(struct apple_dcp *dcp, void *out, void *cookie) static void init_2(struct apple_dcp *dcp, void *out, void *cookie) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * macOS always calls first_client_open here; skipping it leaves the + * firmware with no client and it never accepts a swap. The skip is + * kept only as a bisect lever and defaults off. + */ + if (dcp_defer_client_open) { + init_3(dcp, NULL, NULL); + return; + } +#endif dcp_first_client_open(dcp, false, init_3, NULL); } -static void init_1(struct apple_dcp *dcp, void *out, void *cookie) +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * The client-open sequence, in the order macOS 25G83 issues it: + * + * A445 A411 A445 A415 A450 A031 x9 A414 A478 A457 A474 A478 A476 A478 A412 + * + * Every payload below is the traced one. A412 is not in the table: its result + * feeds res_is_main_display(), so it stays on the existing path. + */ +enum dcp_client_step_kind { + DCP_STEP_A445, DCP_STEP_A411, DCP_STEP_A415, DCP_STEP_A450, + DCP_STEP_A031, DCP_STEP_A414, DCP_STEP_A478, DCP_STEP_A457, + DCP_STEP_A474, DCP_STEP_A476, +}; + +static const u8 dcp_client_open_seq[] = { + /* + * macOS reaches the client phase only after A353 and two A352s (with + * five A438 property pushes in between, which we cannot replay yet). + * Without them the firmware never answers the first A445. + */ + /* + * A352 is left out: the firmware stops answering it here, and macOS + * only reaches it after five A438 property pushes we cannot replay yet. + */ + DCP_STEP_A445, DCP_STEP_A411, DCP_STEP_A445, DCP_STEP_A415, + DCP_STEP_A450, + DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, + DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, + DCP_STEP_A031, + DCP_STEP_A414, DCP_STEP_A478, DCP_STEP_A457, DCP_STEP_A474, + DCP_STEP_A478, DCP_STEP_A476, DCP_STEP_A478, +}; + + +static void dcp_client_open_run(struct apple_dcp *dcp, void *out, void *cookie) +{ + unsigned long idx = (unsigned long)cookie; + dcp_callback_t next = dcp_client_open_run; + void *ncookie = (void *)(idx + 1); + + if (idx >= ARRAY_SIZE(dcp_client_open_seq)) { + init_3(dcp, NULL, NULL); + return; + } + + switch (dcp_client_open_seq[idx]) { + case DCP_STEP_A445: { + u32 zero = 0; + + dcp_get_dfb_state(dcp, false, &zero, next, ncookie); + break; + } + case DCP_STEP_A411: { + u32 handle = 0; + + dcp_set_display_device(dcp, false, &handle, next, ncookie); + break; + } + case DCP_STEP_A415: { + struct dcp_pipe_cfg_415 cfg = {}; + + dcp_pipe_cfg_415(dcp, false, &cfg, next, ncookie); + break; + } + case DCP_STEP_A450: { + u32 val = 0; + + dcp_enable_disable_video_power_savings(dcp, false, &val, next, + ncookie); + break; + } + case DCP_STEP_A031: { + struct dcp_pipe_cfg_031 cfg = {}; + + dcp_pipe_cfg_031(dcp, false, &cfg, next, ncookie); + break; + } + case DCP_STEP_A414: { + u64 val = 0; + + dcp_pipe_cfg_414(dcp, false, &val, next, ncookie); + break; + } + case DCP_STEP_A478: + dcp_pipe_query_478(dcp, false, next, ncookie); + break; + case DCP_STEP_A457: + dcp_first_client_open(dcp, false, next, ncookie); + break; + case DCP_STEP_A474: + dcp_pipe_query_474(dcp, false, next, ncookie); + break; + case DCP_STEP_A476: { + /* + * A476 is IOMobileFramebufferAP::set_dcp_clamshellstate(uint). + * We were sending 1 -- copied from the capture without knowing + * what it was -- which tells the firmware the lid is shut, so + * it parks the internal pipe (power_down_M3) and never scans + * out even though the panel and backlight are on. + */ + u32 state = dcp_clamshell_state; + + dcp_pipe_set_476(dcp, false, &state, next, ncookie); + break; + } + default: + dev_warn(dcp->dev, "unknown client-open step %u\n", + dcp_client_open_seq[idx]); + init_3(dcp, NULL, NULL); + break; + } +} +#endif + +static void init_1b(struct apple_dcp *dcp, void *out, void *cookie) { u32 val = 0; dcp_enable_disable_video_power_savings(dcp, false, &val, init_2, NULL); } +static void init_1(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * macOS opens the client sequence with A411 set_display_device(0), + * before A450: traced as "CMD 0x0 A411 0x4 0x4 00000000". + */ + /* the sequence starts with A445, then A411 */ + dcp_client_open_run(dcp, NULL, (void *)0UL); +#else + init_1b(dcp, out, cookie); +#endif +} + static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) { struct iomfb_get_color_remap_mode_req color_remap = @@ -1581,14 +2333,25 @@ static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) dev_info(dcp->dev, "DCP booted\n"); +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P starts scanning out the moment it finishes booting, but it has + * no surface yet and the stream it uses has no translation table, so + * the DART faults continuously and starves the machine. Park the + * display first thing; the normal poweron path brings it back with a + * real framebuffer when a mode is set. + */ + /* the display is parked at the head of the boot chain, not here */ +#endif + iomfb_get_color_remap_mode(dcp, false, &color_remap, init_1, cookie); } void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp) { - struct dcp_set_power_state_req req = { + struct DCP_FW_NAME(dcp_set_power_state_req) req = { /* defaults are ok */ }; - dcp_set_power_state(dcp, false, &req, NULL, NULL); + dcp_set_power_state(dcp, DCP_POWER_OOB, &req, NULL, NULL); } diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 9817a174dad52..82b4cc92be4a9 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -138,9 +138,113 @@ struct DCP_FW_NAME(dcp_swap_submit_resp) { u8 padding[3]; } __packed; -struct DCP_FW_NAME(dc_swap_complete_resp) { +/* + * A406 swap_start / A473 set_power_state. H17P shrank both requests: the + * 0x10-byte struct dcp_iouserclient became a bare u64 handle and the swap_start + * response no longer echoes it. Sizes are the literal wire headers from macOS + * 25G83 on this machine (0x10/0x8 across 1919 swaps, and the sole OOBCMD call). + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +struct DCP_FW_NAME(dcp_swap_start_req) { + u32 swap_id; + u64 client; /* IOUserClient kernel VA */ + u8 client_null; + u8 padding[3]; +} __packed; + +struct DCP_FW_NAME(dcp_swap_start_resp) { u32 swap_id; + u32 ret; +} __packed; + +struct DCP_FW_NAME(dcp_set_power_state_req) { + u64 unklong; + u32 unkint; /* widened from u8 on H17P */ + u8 unkint_null; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct DCP_FW_NAME(dcp_swap_start_req)) == 0x10); +static_assert(sizeof(struct DCP_FW_NAME(dcp_swap_start_resp)) == 0x8); +static_assert(sizeof(struct DCP_FW_NAME(dcp_set_power_state_req)) == 0x10); +#else +struct DCP_FW_NAME(dcp_swap_start_req) { + u32 swap_id; + struct dcp_iouserclient client; + u8 swap_id_null; + u8 client_null; + u8 padding[2]; +} __packed; + +struct DCP_FW_NAME(dcp_swap_start_resp) { + u32 swap_id; + struct dcp_iouserclient client; + u32 ret; +} __packed; + +struct DCP_FW_NAME(dcp_set_power_state_req) { + u64 unklong; u8 unkbool; + u8 unkint_null; + u8 padding[2]; +} __packed; +#endif + +/* + * D201 map_piodma reply. H17P declares a 16-byte output, not 20: writing the + * pre-H17P struct into it overruns the record by four bytes and puts every + * field where the firmware does not look, so the display engine ends up with a + * null base and DMAs to address 0. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P's D201 reply is { index, dva, pad }, not { vaddr, dva }. Decoded from + * the macOS 25G83 capture, which answers e.g. buffer 6 with + * 06000000 0040b10000010000 00000000 -> index 6, dva 0x100_00b14000 + * With the old layout the DVA lands four bytes late, the firmware reads a + * shifted value and DMAs to it, and the coprocessor dies in + * apiodma_irq_AxiErr(). + */ +struct DCP_FW_NAME(dcp_map_buf_resp) { + u32 buffer; + u64 dva; + u32 unk; +} __packed; +static_assert(sizeof(struct DCP_FW_NAME(dcp_map_buf_resp)) == 0x10); +#else +struct DCP_FW_NAME(dcp_map_buf_resp) { + u64 vaddr; + u64 dva; + u32 ret; +} __packed; +#endif + +struct DCP_FW_NAME(dc_swap_complete_resp) { + u32 swap_id; /* +0x000 */ + u8 unkbool; /* +0x004 */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P: swap_complete_ap_gated(u32, bool, SwapCompleteData const *, + * SwapInfoBlob const *, u32, bool) is 0x730 bytes on the wire, 89 more + * than the pre-H17P record. Byte-exact against 1919 D590 callbacks + * captured from macOS 25G83 on this machine: the SwapCompleteData grew + * from 8 to 0x22, swap_info became eight 0xe0-byte SwapInfoBlobs rather + * than one 0x6c5 blob, and the tail gained a flag and three pad bytes. + * Only swap_info_count of those blobs are populated (always 1 in the + * captures); the rest are uninitialised DCP buffer memory. + */ + u32 hol_swap_ids[4]; /* +0x005, zero in 1919/1919 */ + u32 hol_arg; /* +0x015 */ + u32 hol_count; /* +0x019 */ + u8 hol_valid; /* +0x01d */ + u64 displayed_data; /* +0x01e, unaligned */ + u8 notify_displayed; /* +0x026 */ + u8 swap_info[8][0xe0]; /* +0x027, blob[0]+0x38 == swap_id */ + u32 swap_info_count; /* +0x727, 1 in 1919/1919, max 8 */ + u8 completion_flag; /* +0x72b */ + u8 swap_data_null; /* +0x72c, 0 => data at +0x005 valid */ + u8 padding[3]; /* +0x72d, to sizeof == 0x730 */ +#else u64 swap_data; #if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) u8 swap_info[0x6c4]; @@ -149,6 +253,7 @@ struct DCP_FW_NAME(dc_swap_complete_resp) { #endif u32 unkint; u8 swap_info_null; +#endif } __packed; struct DCP_FW_NAME(dcp_map_reg_req) { diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 3d520938a2900..86f61db291f9a 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -667,19 +667,30 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, break; } - if (!strcmp(key, "EPICName")) { + /* + * H17P advertises its system-service with lower-case keys -- + * "name", "compartment", "interface-id" -- where earlier + * firmware uses EPICName/EPICProviderClass/EPICUnit. Without + * these the service never registers, the DCP has no system + * service to ask, and it falls back to clamshell = 1 and idles + * the panel (power_down_M3), so no vblank and no swap ever + * completes. + */ + if (!strcmp(key, "EPICName") || !strcmp(key, "compartment")) { *name = parse_string(it.handle); if (IS_ERR(*name)) ret = PTR_ERR(*name); else parsed_name = true; - } else if (!strcmp(key, "EPICProviderClass")) { + } else if (!strcmp(key, "EPICProviderClass") || + !strcmp(key, "name")) { *class = parse_string(it.handle); if (IS_ERR(*class)) ret = PTR_ERR(*class); else parsed_class = true; - } else if (!strcmp(key, "EPICUnit")) { + } else if (!strcmp(key, "EPICUnit") || + !strcmp(key, "interface-id")) { ret = parse_int(it.handle, unit); if (!ret) parsed_unit = true; From fceec5685671a1911139487ac595892fa218dfe8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 9 Sep 2026 23:38:30 -0400 Subject: [PATCH 0214/1579] drm/apple: stabilize H17P display handoff and scanout Signed-off-by: DjDeveloperr --- drivers/gpu/drm/apple/dcp-internal.h | 26 ++++ drivers/gpu/drm/apple/dcp.c | 33 ++--- drivers/gpu/drm/apple/iomfb.c | 12 +- drivers/gpu/drm/apple/iomfb_template.c | 194 +++++++++++++++++++------ 4 files changed, 195 insertions(+), 70 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index dfb2d729979de..0ec047979acda 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -99,6 +100,8 @@ struct dcp_mem_descriptor { dma_addr_t dva; struct sg_table map; u64 reg; + /* live iommu mappings installed by dcpep_cb_map_piodma() */ + bool piodma_mapped; }; /* Limit on call stack depth (arbitrary). Some nesting is required */ @@ -127,7 +130,14 @@ struct dcp_channel { struct dcp_fb_reference { struct list_head head; struct drm_framebuffer *fb; + /* + * Id of the swap that unbinds @fb, i.e. the one that puts its + * replacement on screen. Only meaningful once @armed is set: the + * firmware assigns the id in the swap_start reply, which happens after + * the atomic commit that displaced @fb has already queued this entry. + */ u32 swap_id; + bool armed; }; #define MAX_NOTCH_HEIGHT 160 @@ -156,6 +166,12 @@ struct dcp_panel { struct apple_dcp_hw_data { u32 num_dptx_ports; bool adopt_running_coprocessor; + /* + * Attach to an existing RTKit session instead of restarting it. This + * requires the endpoint map to be recovered before the DCP services are + * started; no current platform enables it. + */ + bool adopt_live_session; }; /* TODO: move IOMFB members to its own struct */ @@ -167,6 +183,8 @@ struct apple_dcp { struct iommu_domain *disp_iommu_dom; /* the display-subsystem device, whose DMA domain the pipe scans out of */ struct device *disp_dev; + /* which of the nine cumulative A031 notify-client states to send next */ + unsigned int a031_step; struct apple_rtkit *rtk; struct apple_crtc *crtc; struct apple_connector *connector; @@ -241,6 +259,8 @@ struct apple_dcp { /* swap id of the last completed swap */ u32 last_swap_id; + /* last_swap_id is only meaningful after the first swap completes */ + bool have_swap_complete; ktime_t swap_start; u64 swap_submit_timestamp; @@ -297,6 +317,12 @@ struct apple_dcp { * on the next successfully completed swap. */ struct list_head swapped_out_fbs; + /* + * swapped_out_fbs is appended to from the DRM atomic commit thread + * (dcp_flush -> .atomic_flush) and armed/drained from the RTKit + * workqueue that runs the DCP callbacks. Two different threads. + */ + spinlock_t swapped_out_lock; struct dcp_brightness brightness; /* Workqueue for updating the initial brightness */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 240515fbd17a2..7fb6c704a295f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1282,14 +1282,6 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) dev_info_ratelimited(dcp->dev, "ep %#04x: %#llx\n", endpoint, message); } -/* - * Attach to the running DCP instead of restarting it. Off by default until it - * is proven; the restart path is what has been exercised so far. - */ -static bool dcp_adopt_live; -module_param_named(adopt_live, dcp_adopt_live, bool, 0444); -MODULE_PARM_DESC(adopt_live, - "attach to the loader's live DCP session without sending INIT"); static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_size) { @@ -2664,6 +2656,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); + spin_lock_init(&dcp->swapped_out_lock); if (dcp->fw_compat == DCP_FIRMWARE_H17P) dcp_enable_pmc_domain(dcp); @@ -2690,7 +2683,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) } else { if (start_mode == APPLE_DCP_COPROCESSOR_QUIESCED_RESTART) dev_info(dev, "restarting loader-quiesced DCP session\n"); - if (dcp_adopt_live) { + if (dcp->hw.adopt_live_session) { dcp->adopting_session = true; dcp->rtk = devm_apple_rtkit_init_adopted(dev, dcp, "mbox", 0, @@ -2705,20 +2698,11 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) "Failed to initialize RTKit\n"); } - if (dcp_adopt_live) { + if (dcp->hw.adopt_live_session) { /* - * Attach to the session the loader left running instead of - * re-initialising it. - * - * m1n1 hands the DCP over live and untouched -- it neither - * stops the ASC nor quiesces RTKit; the "quiesced" property is - * only there to steer this driver's code path. So the - * coprocessor already has a session, an endpoint map, and a - * configured display pipe. Poking CPU_CONTROL.RUN and sending - * SET_IOP_PWR_STATE(INIT) at that makes the firmware tear its - * IOMFB state down and come back with no scanout, which is the - * state macOS is never in: its capture contains no wake, no - * HELLO and no EPMAP, just A401 on an already-running IOP. + * The adopted RTKit instance must already have a usable endpoint + * map. Without one, apple_rtkit_start_ep() reaches no service and + * probe eventually times out waiting for DCP readiness. */ dev_info(dev, "adopting the live DCP session without a restart\n"); ret = 0; @@ -3129,6 +3113,11 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { .num_dptx_ports = 0, .adopt_running_coprocessor = true, + /* + * Keep adopt_live_session disabled until apple-rtkit can recover the + * loader's endpoint map. An adopted instance without that map cannot + * start DCP services. + */ }; static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 02c46a7554af1..30dc04bd61d25 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -35,7 +35,15 @@ #include "trace.h" -u32 dcp_clamshell_state; /* A476 set_dcp_clamshellstate; 0 = lid open */ +/* + * A476 payload. macOS sends 1 (dcp-calls.log line 1648); we sent 0 on the + * theory that A476 is set_dcp_clamshellstate and 1 meant "lid shut". That + * attribution is unverified -- A476's shape (in 4, out 0) equally fits + * set_has_frame_swap_function(bool) -- and the black panel it appeared to fix + * was really the surf_iova misalignment, so the evidence for 0 was + * coincidental. Follow the capture. + */ +u32 dcp_clamshell_state = 1; module_param_named(clamshell_state, dcp_clamshell_state, uint, 0644); bool dcp_send_modeset; /* macOS never issues A413 */ @@ -78,7 +86,7 @@ static unsigned int dcp_extra_eps; module_param_named(extra_eps, dcp_extra_eps, uint, 0644); MODULE_PARM_DESC(extra_eps, "bitmask of extra endpoints to start, bit n = 0x20+n"); -void dcp_start_extra_eps(struct apple_dcp *dcp) +static void dcp_start_extra_eps(struct apple_dcp *dcp) { unsigned int n; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index dfe0e250b9cb5..9d9afe10f0444 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -150,9 +150,17 @@ struct dcp_pipe_cfg_415 { u8 data[0xc]; } __packed; +/* + * A031 is update_notify_clients_dcp(u32[27]): a bitmap-as-array of the DCP + * notification clients the AP wants delivered. macOS enables them + * cumulatively over nine calls, ending at {6,7,8,10,11,12,25,26}; we used to + * send nine zeroed records, so we finished with no notify clients registered + * at all and never received the callbacks they gate. + */ struct dcp_pipe_cfg_031 { - u8 data[0x6c]; + __le32 clients[27]; } __packed; +static_assert(sizeof(struct dcp_pipe_cfg_031) == 0x6c); DCP_THUNK_OUT(dcp_pipe_query_353, dcpep_pipe_query_353, u64); DCP_THUNK_INOUT(dcp_pipe_set_352, dcpep_pipe_set_352, u64, u32); @@ -214,14 +222,86 @@ static u32 dcpep_cb_zero(struct apple_dcp *dcp) return 0; } +/* + * Bind every framebuffer displaced by the commit that produced this swap to + * that swap's id. + * + * The id only exists once the firmware answers swap_start, after the atomic + * commit has queued its entries, so they are queued unarmed and stamped here. + * Swap-start requests are serialized by the DCP command stack. + */ +static void dcp_arm_swapped_out_fbs(struct apple_dcp *dcp, u32 swap_id) +{ + struct dcp_fb_reference *entry; + unsigned long flags; + + spin_lock_irqsave(&dcp->swapped_out_lock, flags); + list_for_each_entry(entry, &dcp->swapped_out_fbs, head) { + if (entry->armed) + continue; + entry->swap_id = swap_id; + entry->armed = true; + } + spin_unlock_irqrestore(&dcp->swapped_out_lock, flags); +} + +/* + * Drop framebuffers whose unbinding swap has completed. + * + * Entries must be keyed to the swap that unbinds them, rather than the last + * swap that happened to have completed when the atomic commit ran. A later + * successful completion also retires entries whose own completion was lost; + * the signed difference keeps that comparison valid across a 32-bit wrap. + */ +static void dcp_release_swapped_out_fbs(struct apple_dcp *dcp) +{ + struct dcp_fb_reference *entry, *tmp; + unsigned long flags; + LIST_HEAD(done); + + /* + * last_swap_id / have_swap_complete are only ever touched by the RTKit + * worker, which is also the only caller of this function, so the lock + * is here for the list alone. + */ + spin_lock_irqsave(&dcp->swapped_out_lock, flags); + while (!list_empty(&dcp->swapped_out_fbs)) { + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + + /* Newest entries; their swap has not been started yet. */ + if (!entry->armed) + break; + /* last_swap_id is 0 from kzalloc until a swap really lands. */ + if (!dcp->have_swap_complete) + break; + /* Hold the old framebuffer for one additional completed swap. */ + if ((s32)(dcp->last_swap_id - entry->swap_id) <= 0) + break; + + list_move_tail(&entry->head, &done); + } + spin_unlock_irqrestore(&dcp->swapped_out_lock, flags); + + /* drm_framebuffer_put() can sleep, so never call it under the lock. */ + list_for_each_entry_safe(entry, tmp, &done, head) { + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} + static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct DCP_FW_NAME(dc_swap_complete_resp) *resp) { ktime_t now = ktime_get(); trace_iomfb_swap_complete(dcp, resp->swap_id); dcp->last_swap_id = resp->swap_id; + dcp->have_swap_complete = true; dcp_drm_crtc_page_flip(dcp, now); + dcp_release_swapped_out_fbs(dcp); if (dcp->crc_enabled) { u32 crc32 = 0; drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); @@ -292,7 +372,7 @@ static void dcp_fill_dfb_surface(u8 *s) s[0x01c] = 0x01; s[0x02a] = 0x40; s[0x02b] = 0xe4; - s[0x035] = 0x03; + s[0x035] = 0x04; /* IOSurface.surface_id; macOS writes 4, we wrote 3 */ s[0x051] = 0x01; s[0x149] = 0x01; } @@ -481,17 +561,8 @@ static struct DCP_FW_NAME(dcp_map_buf_resp) dcpep_cb_map_piodma(struct apple_dcp goto reject; } - /* - * Mirror it into the display-subsystem domain at the same address: the - * display engine reads these buffers on that device's stream, which - * otherwise has no mapping for them at all. - */ - /* - * Do not mirror into the display-subsystem domain. That stream's - * tables are iBoot's, pinned to the DART's published "pt-region", and - * macOS never adds entries to them from the host -- the firmware does - * its own remapping there (the ADT marks the DART "allow-pte-remap"). - */ + /* The PIODMA mapping belongs only to the DCP IOMMU domain. */ + memdesc->piodma_mapped = true; #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) /* echo the index back, as macOS does */ @@ -552,6 +623,7 @@ static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, * full pages */ iommu_unmap(dcp->iommu_dom, memdesc->dva, ALIGN(memdesc->size, SZ_16K)); + memdesc->piodma_mapped = false; } /* @@ -583,6 +655,7 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, memdesc = &dcp->memdesc[id]; + memdesc->piodma_mapped = false; memdesc->size = resp.dva_size; /* HACK: align size to 16K since the iommu API only maps full pages */ size = ALIGN(resp.dva_size, SZ_16K); @@ -627,6 +700,22 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) memdesc = &dcp->memdesc[id]; size = ALIGN(memdesc->size, SZ_16K); + + /* + * Tear down any translation still standing before the pages go back to + * the allocator. The firmware is expected to unmap piodma first, but + * if it releases a descriptor without doing so we would hand the pages + * back while the DART still resolves them -- the display engine would + * then read whoever owns them next. + */ + if (memdesc->piodma_mapped) { + dev_warn(dcp->dev, + "releasing buffer %u while still mapped for piodma; unmapping first\n", + id); + iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + memdesc->piodma_mapped = false; + } + if (memdesc->buf) { dma_free_coherent(dcp->dev, size, memdesc->buf, memdesc->dva); memdesc->buf = NULL; @@ -1280,7 +1369,11 @@ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) * 0xfc0000000 -- an address with no L1 entry at all. Shift the * whole block, which subsumes the null-flag fixup. */ - memmove(r + off, r + off + 2, n - off - 2); + /* Preserve the complete 16K-aligned IOVA, including its low bits. */ + static_assert(offsetof(typeof(*req), surf_iova) == 0xe3a, + "H17P source IOVA offset"); + static_assert(sizeof(*req) == 0xe9c, "H17P request size"); + memmove(r + off - 2, r + off, n - off); memset(r + n - 2, 0, 2); } } @@ -1304,17 +1397,7 @@ static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) return; } - while (!list_empty(&dcp->swapped_out_fbs)) { - struct dcp_fb_reference *entry; - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - if (entry->swap_id == dcp->last_swap_id) - break; - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } + dcp_release_swapped_out_fbs(dcp); } static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, @@ -1322,6 +1405,7 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, { struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + dcp_arm_swapped_out_fbs(dcp, resp->swap_id); if (cookie) { struct dcp_swap_cookie *info = cookie; @@ -1771,17 +1855,7 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) dcp->swap_start = ktime_get(); dcp->swap_submit_timestamp = arch_timer_read_counter(); - while (!list_empty(&dcp->swapped_out_fbs)) { - struct dcp_fb_reference *entry; - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - if (entry->swap_id == dcp->last_swap_id) - break; - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } + dcp_release_swapped_out_fbs(dcp); } static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) @@ -1789,6 +1863,7 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + dcp_arm_swapped_out_fbs(dcp, resp->swap_id); trace_iomfb_swap_submit(dcp, resp->swap_id); dcp_h17p_fix_swap_surfaces(dcp); @@ -2035,22 +2110,24 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru req->swap.swap_enabled |= BIT(l); if (old_state->fb && new_state->fb != old_state->fb) { - /* - * Race condition between a framebuffer unbind getting - * swapped out and GEM unreferencing a framebuffer. If - * we lose the race, the display gets IOVA faults and - * the DCP crashes. We need to extend the lifetime of - * the drm_framebuffer (and hence the GEM object) until - * after we get a swap complete for the swap unbinding - * it. - */ + /* Retain the old framebuffer until its replacement is stable. */ struct dcp_fb_reference *entry = kzalloc(sizeof(*entry), GFP_KERNEL); + unsigned long flags; + if (entry) { entry->fb = old_state->fb; - entry->swap_id = dcp->last_swap_id; + /* + * Left unarmed: the id of the swap that + * unbinds this framebuffer is not known until + * the firmware answers swap_start. + */ + spin_lock_irqsave(&dcp->swapped_out_lock, + flags); list_add_tail(&entry->head, &dcp->swapped_out_fbs); + spin_unlock_irqrestore(&dcp->swapped_out_lock, + flags); } drm_framebuffer_get(old_state->fb); } @@ -2262,7 +2339,32 @@ static void dcp_client_open_run(struct apple_dcp *dcp, void *out, void *cookie) break; } case DCP_STEP_A031: { + /* + * The nine cumulative states macOS sends, from the 25G83 + * capture (dcp-calls.log lines 1507-1531). Each call repeats + * the previous set and adds one client. + */ + static const u8 notify_clients[9][8] = { + {}, + { 6 }, + { 6, 7 }, + { 6, 7, 8 }, + { 6, 7, 8, 10 }, + { 6, 7, 8, 10, 11 }, + { 6, 7, 8, 10, 11, 12 }, + { 6, 7, 8, 10, 11, 12, 25 }, + { 6, 7, 8, 10, 11, 12, 25, 26 }, + }; + static const u8 notify_count[9] = { 0, 1, 2, 3, 4, 5, 6, 7, 8 }; struct dcp_pipe_cfg_031 cfg = {}; + unsigned int step = dcp->a031_step; + unsigned int i; + + if (step >= ARRAY_SIZE(notify_clients)) + step = ARRAY_SIZE(notify_clients) - 1; + for (i = 0; i < notify_count[step]; i++) + cfg.clients[i] = cpu_to_le32(notify_clients[step][i]); + dcp->a031_step++; dcp_pipe_cfg_031(dcp, false, &cfg, next, ncookie); break; From bc40ec29f485538d60ecda5cc2eada51332eda01 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 10 Sep 2026 04:13:52 -0400 Subject: [PATCH 0215/1579] drm/apple: restart inherited H17P firmware through acknowledged IOP shutdown J700 firmware reports Invalid AP power: 32 when the inherited-session path sends AP QUIESCED. Add a guarded RTKit IOP-only poweroff operation and use it for H17P before the existing bounded ASC restart. Require the OFF acknowledgement; leave ordinary power protocols unchanged and reject a blind retry after a failed H17P transition. Protocol evidence: m1n1-aurora 4a927cdcae729f1b12b2173117e7f96b3271dca9 and the local pinned-firmware crash-buffer capture. Native display restart remains under hardware qualification. Co-authored-by: DjDeveloperr --- drivers/gpu/drm/apple/dcp.c | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7fb6c704a295f..dcd939b1396d3 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2519,6 +2519,29 @@ static int dcp_asc_restart(struct apple_dcp *dcp) +static int dcp_restart_inherited_session(struct apple_dcp *dcp) +{ + int ret; + + if (dcp->fw_compat != DCP_FIRMWARE_H17P) + return apple_rtkit_quiesce(dcp->rtk); + + /* + * J700's inherited DCP rejects AP QUIESCED. The dcp-neo m1n1 + * bring-up established IOP OFF and its exact state-zero ACK instead + * (4a927cdcae729f1b12b2173117e7f96b3271dca9). OFF is not a normal + * sleep state: stop and restart the ASC before negotiating a new + * RTKit session. Never reset a session whose OFF request timed out. + */ + ret = apple_rtkit_poweroff_adopted(dcp->rtk); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Inherited DCP IOP OFF not acknowledged\n"); + + dev_info(dcp->dev, "inherited DCP acknowledged IOP OFF\n"); + return dcp_asc_restart(dcp); +} + static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; @@ -2674,7 +2697,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) "Failed to adopt RTKit\n"); } - ret = apple_rtkit_quiesce(dcp->rtk); + ret = dcp_restart_inherited_session(dcp); dcp->adopting_session = false; if (ret) return dev_err_probe(dev, ret, @@ -2737,8 +2760,10 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) "Failed to adopt running RTKit\n"); } - ret = apple_rtkit_quiesce(dcp->rtk); + ret = dcp_restart_inherited_session(dcp); dcp->adopting_session = false; + if (ret && dcp->fw_compat == DCP_FIRMWARE_H17P) + return ret; if (!ret) ret = apple_rtkit_wake(dcp->rtk); From e5bc356be167f2964db5d56fa57c8f07e7c7fefe Mon Sep 17 00:00:00 2001 From: Acelogic Date: Sat, 12 Sep 2026 00:52:52 -0400 Subject: [PATCH 0216/1579] drm/apple: bring up native T8142 DCP presentation Adapt the shared H17 display transport to the M5 native discovery and framebuffer ABI. Preserve live firmware mappings, resolve verified H17G methods, and use the saved nested-call reply buffer so startup returns the real panel dimensions. Add guarded RAM boot and Xorg KMS tooling. Hardware validation covers 2560x1664 pageflips at 60 Hz, visible XFCE and smooth trackpad movement, plus N1 DHCP and external ping. Application GPU acceleration remains unfinished. Validation: ARM64 kernel/modules build; ASan/UBSan ACK and AFK regressions; exact native surface comparison. RPC call tracing is opt-in for normal desktop use. --- drivers/gpu/drm/apple/afk.c | 17 ++-- drivers/gpu/drm/apple/dcp-internal.h | 24 ++++- drivers/gpu/drm/apple/dcp.c | 132 +++++++++++++++++++++--- drivers/gpu/drm/apple/iomfb.c | 135 ++++++++++++++++++++++--- drivers/gpu/drm/apple/iomfb.h | 5 + drivers/gpu/drm/apple/iomfb_h17p.c | 22 +++- drivers/gpu/drm/apple/iomfb_template.c | 52 ++++++---- 7 files changed, 327 insertions(+), 60 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 1b7adbe1df80a..144fdcf8bbd91 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -104,7 +104,9 @@ int afk_start(struct apple_dcp_afkep *ep) int ret; reinit_completion(&ep->started); - apple_rtkit_start_ep(ep->dcp->rtk, ep->endpoint); + ret = dcp_start_rtkit_endpoint(ep->dcp, ep->endpoint); + if (ret) + return ret; afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_INIT)); ret = wait_for_completion_timeout(&ep->started, msecs_to_jiffies(1000)); @@ -152,7 +154,7 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, u32 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; u32 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; u16 tag = FIELD_GET(INITRB_TAG, message); - u32 bufsz, hdrsz, end; + u32 bufsz, hdrsz, end, block; if (tag != ep->bfr_tag) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: expected tag 0x%x but got 0x%x\n", @@ -166,7 +168,7 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, return; } - if (base >= ep->bfr_size) { + if (!ep->bfr || size < sizeof(__le32) || base >= ep->bfr_size) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: requested base 0x%x >= max size 0x%lx\n", ep->endpoint, base, ep->bfr_size); @@ -182,16 +184,19 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, } bfr->hdr = ep->bfr + base; - bufsz = le32_to_cpu(*(__le32 *)bfr->hdr); /* first word of block 0 */ + dma_rmb(); + bufsz = le32_to_cpu(READ_ONCE(*(__le32 *)bfr->hdr)); hdrsz = size - bufsz; + block = hdrsz / AFK_RB_BLOCKS; if (bufsz >= size || hdrsz % AFK_RB_BLOCKS || - hdrsz / AFK_RB_BLOCKS < AFK_RB_BLOCK_MIN) { + block < AFK_RB_BLOCK_MIN || !is_power_of_2(block) || + !IS_ALIGNED(bufsz, block) || bufsz < 2 * block) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: ring bufsz %#x with size %#x gives no valid header (base %#x, msg %#llx)\n", ep->endpoint, bufsz, size, base, message); return; } - bfr->block = hdrsz / AFK_RB_BLOCKS; + bfr->block = block; bfr->buf = (u8 *)bfr->hdr + hdrsz; bfr->bufsz = bufsz; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 0ec047979acda..29fc9c44ba337 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -163,15 +163,32 @@ struct dcp_panel { bool has_mini_led; }; +enum dcp_iomfb_method_profile { + DCP_IOMFB_METHODS_DEFAULT, + DCP_IOMFB_METHODS_H17G, +}; + struct apple_dcp_hw_data { u32 num_dptx_ports; + enum dcp_iomfb_method_profile iomfb_method_profile; bool adopt_running_coprocessor; /* * Attach to an existing RTKit session instead of restarting it. This - * requires the endpoint map to be recovered before the DCP services are - * started; no current platform enables it. + * uses explicit START_EP messages for the selected DCP services when + * the bootloader already consumed endpoint discovery. */ bool adopt_live_session; + enum dcp_firmware_version firmware_compat; + /* T8140's legacy DT omits the native H17 PMC window. */ + bool legacy_pmc_binding; +}; + +#define DCP_CALLBACK_TRACE_COUNT 16 +struct dcp_callback_trace { + char tag[4]; + u32 in_len; + u32 out_len; + u8 input[16]; }; /* TODO: move IOMFB members to its own struct */ @@ -190,6 +207,9 @@ struct apple_dcp { struct apple_connector *connector; struct apple_dcp_hw_data hw; + spinlock_t callback_trace_lock; + u32 callback_trace_count; + struct dcp_callback_trace callback_trace[DCP_CALLBACK_TRACE_COUNT]; /* firmware version and compatible firmware version */ enum dcp_firmware_version fw_compat; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index dcd939b1396d3..f54c123459d75 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1066,6 +1066,13 @@ static int dcp_register_typec_routes(struct apple_dcp *dcp) return 0; } +/* Diagnostic for a preloaded, still-running DCP. The native T8142 macOS + * trace starts with INIT (0x220), HELLO and EPMAP without resetting the ASC. + * Keep this opt-in until that handshake is qualified on the Linux handoff. + */ +static bool live_discovery; +module_param(live_discovery, bool, 0400); +MODULE_PARM_DESC(live_discovery, "Negotiate RTKit on the inherited live DCP without an ASC reset"); /* copied and simplified from drm_vblank.c */ static void send_vblank_event(struct drm_device *dev, @@ -1433,6 +1440,23 @@ static struct apple_rtkit_ops rtkit_ops = { .shmem_destroy = dcp_rtk_shmem_destroy, }; +int dcp_start_rtkit_endpoint(struct apple_dcp *dcp, u8 endpoint) +{ + u64 message; + + if (!dcp->hw.adopt_live_session || live_discovery) + return apple_rtkit_start_ep(dcp->rtk, endpoint); + if (endpoint < 0x20 || endpoint >= 0x40) + return -EINVAL; + + /* START_EP for an application service in the running generation. The + * loader already consumed EPMAP; do not pretend discovery was repeated. + */ + message = FIELD_PREP(GENMASK_ULL(59, 52), 5) | + FIELD_PREP(GENMASK_ULL(39, 32), endpoint) | BIT_ULL(1); + return apple_rtkit_send_message(dcp->rtk, 0, message, NULL, false); +} + void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message) { trace_dcp_send_msg(dcp, endpoint, message); @@ -2347,12 +2371,13 @@ static int dcp_read_fw_version(struct device *dev, const char *name, static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) { + const struct apple_dcp_hw_data *hw = of_device_get_match_data(dev); char compat_str[DCP_FW_VERSION_STR_LEN]; char fw_str[DCP_FW_VERSION_STR_LEN]; int ret; - if (of_device_is_compatible(dev->of_node, "apple,t8140-dcp")) - return DCP_FIRMWARE_H17P; + if (hw->firmware_compat != DCP_FIRMWARE_UNKNOWN) + return hw->firmware_compat; /* firmware version is just informative */ dcp_read_fw_version(dev, "apple,firmware-version", fw_str); @@ -2542,6 +2567,60 @@ static int dcp_restart_inherited_session(struct apple_dcp *dcp) return dcp_asc_restart(dcp); } +/* + * Reset the display coprocessor's ASC core. + * + * Once Linux has driven a DCP session the firmware stops answering the + * management endpoint: it will not re-HELLO on INIT (it is already ON) and it + * will not acknowledge a power-state change either, so neither apple_rtkit_wake + * nor apple_rtkit_quiesce can recover it and the module could previously only + * be reloaded by rebooting the machine. + * + * CPU_CONTROL.RUN is a raw register in the ASC wrapper and needs no cooperation + * from the firmware, so clearing it stops the core no matter how wedged it is. + * The firmware image itself is left alone -- iBoot mapped it into the DCP's own + * DART and nothing here touches those mappings -- so setting RUN again restarts + * it from its reset vector and it comes back with a fresh HELLO. + */ +static int dcp_asc_restart(struct apple_dcp *dcp) +{ + u32 ctrl, status; + int i; + + ctrl = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + writel_relaxed(ctrl & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, + dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + + /* + * T8140 clears CPU_STATUS.RUNNING when the core halts but never sets + * the STOPPED bit (status goes 0x6d -> 0x48), so gate on RUNNING going + * away rather than on STOPPED appearing. + */ + for (i = 0; i < 500; i++) { + status = readl_relaxed(dcp->coproc_reg + + APPLE_DCP_COPROC_CPU_STATUS); + if (!(status & APPLE_DCP_COPROC_CPU_STATUS_RUNNING) || + (status & APPLE_DCP_COPROC_CPU_STATUS_STOPPED)) + break; + usleep_range(100, 200); + } + if (status & APPLE_DCP_COPROC_CPU_STATUS_RUNNING) { + dev_err(dcp->dev, "DCP ASC did not stop (status %#x)\n", status); + return -ETIMEDOUT; + } + + dev_info(dcp->dev, "DCP ASC stopped (status %#x), restarting it\n", + status); + usleep_range(1000, 2000); + writel_relaxed(ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, + dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + return 0; +} + + + + + static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; @@ -2605,6 +2684,10 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) start_mode = apple_dcp_coprocessor_start_mode( loader_quiesced, dcp->hw.adopt_running_coprocessor, cpu_ctrl, cpu_status); + if (dcp->hw.adopt_live_session && + (loader_quiesced || !(cpu_ctrl & APPLE_DCP_COPROC_CPU_CONTROL_RUN))) + return dev_err_probe(dev, -EOPNOTSUPP, + "This DCP requires a live inherited session\n"); if (dcp->index || dcp->dptx_phy || dcp->dptx_die) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", @@ -2676,15 +2759,18 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); INIT_DELAYED_WORK(&dcp->stall_wq, dcp_stall_check); + spin_lock_init(&dcp->callback_trace_lock); + dcp->callback_trace_count = 0; dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); spin_lock_init(&dcp->swapped_out_lock); - if (dcp->fw_compat == DCP_FIRMWARE_H17P) + if (dcp->fw_compat == DCP_FIRMWARE_H17P && !dcp->hw.adopt_live_session) dcp_enable_pmc_domain(dcp); - if (start_mode == APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART) { + if (start_mode == APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART && + !dcp->hw.adopt_live_session) { dev_info(dev, "quiescing inherited DCP session (control=%#x status=%#x)\n", cpu_ctrl, cpu_status); @@ -2706,7 +2792,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) } else { if (start_mode == APPLE_DCP_COPROCESSOR_QUIESCED_RESTART) dev_info(dev, "restarting loader-quiesced DCP session\n"); - if (dcp->hw.adopt_live_session) { + if (dcp->hw.adopt_live_session && !live_discovery) { dcp->adopting_session = true; dcp->rtk = devm_apple_rtkit_init_adopted(dev, dcp, "mbox", 0, @@ -2721,11 +2807,18 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) "Failed to initialize RTKit\n"); } - if (dcp->hw.adopt_live_session) { + if (dcp->hw.adopt_live_session && live_discovery) { + /* No CPU-control, power-domain or DART writes on this path. The + * native trace receives HELLO in response to this standard INIT. + * A timeout is evidence to retain; do not fall back to a reset. + */ + dev_info(dev, "negotiating RTKit on the live DCP without a reset\n"); + ret = apple_rtkit_wake(dcp->rtk); + } else if (dcp->hw.adopt_live_session) { /* - * The adopted RTKit instance must already have a usable endpoint - * map. Without one, apple_rtkit_start_ep() reaches no service and - * probe eventually times out waiting for DCP readiness. + * The loader consumed discovery. Start selected services explicitly + * through dcp_start_rtkit_endpoint(), preserving the running CPU and + * its firmware memory mappings. */ dev_info(dev, "adopting the live DCP session without a restart\n"); ret = 0; @@ -2809,6 +2902,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (!dcp) return; + cancel_delayed_work_sync(&dcp->stall_wq); if (dcp->hdmi_hpd_irq) disable_irq(dcp->hdmi_hpd_irq); @@ -2858,10 +2952,10 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); cancel_work_sync(&dcp->vblank_wq); - devm_clk_put(dev, dcp->clk); + /* The component framework releases the bind's devres group after + * unbind, including optional clocks whose managed value can be NULL. + */ dcp->clk = NULL; - if (dcp->clk_194) - devm_clk_put(dev, dcp->clk_194); dcp->clk_194 = NULL; } @@ -3138,6 +3232,8 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { .num_dptx_ports = 0, .adopt_running_coprocessor = true, + .firmware_compat = DCP_FIRMWARE_H17P, + .legacy_pmc_binding = true, /* * Keep adopt_live_session disabled until apple-rtkit can recover the * loader's endpoint map. An adopted instance without that map cannot @@ -3145,6 +3241,17 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { */ }; +/* M5 bring-up uses the shared H17 parser with a live firmware generation. + * The wire format remains subject to validation on the target. + */ +static const struct apple_dcp_hw_data apple_dcp_hw_t8142 = { + .num_dptx_ports = 0, + .iomfb_method_profile = DCP_IOMFB_METHODS_H17G, + .adopt_running_coprocessor = true, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_H17P, +}; + static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { .num_dptx_ports = 0, }; @@ -3157,6 +3264,7 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, { .compatible = "apple,t8140-dcp", .data = &apple_dcp_hw_t8140, }, + { .compatible = "apple,t8142-dcp", .data = &apple_dcp_hw_t8142, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, {} diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 30dc04bd61d25..aa498eee369b9 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -94,7 +94,7 @@ static void dcp_start_extra_eps(struct apple_dcp *dcp) if (!(dcp_extra_eps & BIT(n))) continue; dev_info(dcp->dev, "starting extra endpoint %#04x\n", 0x20 + n); - apple_rtkit_start_ep(dcp->rtk, 0x20 + n); + dcp_start_rtkit_endpoint(dcp, 0x20 + n); } } @@ -113,6 +113,9 @@ MODULE_PARM_DESC(skip_hotplug, "do not raise the DRM hotplug event (bring-up bis static int dcp_trace_budget; module_param_named(trace_rpc, dcp_trace_budget, int, 0644); +static bool dcp_trace_calls; +module_param_named(trace_calls, dcp_trace_calls, bool, 0644); +MODULE_PARM_DESC(trace_calls, "Trace AP calls after the bounded RPC trace budget is consumed"); MODULE_PARM_DESC(trace_rpc, "log the first N DCP RPC messages (0 = off; the vblank callback\n" "\t\t alone is 60/s, so anything non-zero fills the kernel ring)"); @@ -250,6 +253,13 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca u32 in_len, u32 out_len, void *data, dcp_callback_t cb, void *cookie) { + struct dcp_method_entry resolved = *call; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G && + call->tag_h17g[0]) + memcpy(resolved.tag, call->tag_h17g, sizeof(resolved.tag)); + call = &resolved; + enum dcp_context_id context = dcp_call_context(dcp, oob); struct dcp_channel *ch = dcp_get_channel(dcp, context); @@ -277,8 +287,12 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca memcpy(out_data, data, in_len); trace_iomfb_push(dcp, call, context, offset, depth); - if (dcp_trace_budget > 0) { - dcp_trace_budget--; + /* Explicit AP tracing can outlast the callback/property trace budget. + * Leave it disabled for normal desktop operation. + */ + if (dcp_trace_budget > 0 || dcp_trace_calls) { + if (dcp_trace_budget > 0) + dcp_trace_budget--; dev_info(dcp->dev, "CALL %.4s ctx=%x off=%#x in=%u out=%u\n", call->tag, context, offset, in_len, out_len); } @@ -324,6 +338,30 @@ int dcp_parse_tag(char tag[4]) * call then sits on the command channel forever -- every later push sees a busy * channel and the whole bring-up stalls with no clue which call did it. Name it. */ +static void dcp_dump_callback_trace(struct apple_dcp *dcp) +{ + struct dcp_callback_trace records[DCP_CALLBACK_TRACE_COUNT]; + unsigned long flags; + u32 count, first, i; + + if (!dcp->hw.adopt_live_session) + return; + spin_lock_irqsave(&dcp->callback_trace_lock, flags); + count = dcp->callback_trace_count; + memcpy(records, dcp->callback_trace, sizeof(records)); + spin_unlock_irqrestore(&dcp->callback_trace_lock, flags); + first = count > DCP_CALLBACK_TRACE_COUNT ? count - DCP_CALLBACK_TRACE_COUNT : 0; + dev_info(dcp->dev, "DCP callback history: %u received, last %u follow\n", + count, count - first); + for (i = first; i < count; ++i) { + struct dcp_callback_trace *r = &records[i % DCP_CALLBACK_TRACE_COUNT]; + dev_info(dcp->dev, "CB[%u] %c%c%c%c in=%u out=%u data=%*ph\n", + i, r->tag[3], r->tag[2], r->tag[1], r->tag[0], + r->in_len, r->out_len, min_t(u32, r->in_len, sizeof(r->input)), + r->input); + } +} + void dcp_stall_check(struct work_struct *work) { struct apple_dcp *dcp = container_of(to_delayed_work(work), @@ -344,6 +382,16 @@ void dcp_stall_check(struct work_struct *work) ch->tag[d - 1], d, jiffies_to_msecs(jiffies - ch->pushed_at[d - 1])); /* by now a fresh panic record has been written, if there is one */ + if (dcp->hw.adopt_live_session) { + struct dcp_channel *nested = &dcp->ch_cb; + u8 depth = READ_ONCE(nested->depth); + + if (depth && depth <= DCP_MAX_CALL_DEPTH) + dev_info(dcp->dev, + "DCP nested callback channel: %.4s depth=%u\n", + nested->tag[depth - 1], depth); + } + dcp_dump_callback_trace(dcp); dcp_dump_crashlog(dcp); } @@ -479,10 +527,55 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, dev_info(dev, "CB %c%c%c%c ctx=%x in=%u out=%u %s\n", hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0], context, hdr->in_len, hdr->out_len, nm); + dev_info(dev, "CB input %*ph\n", min_t(u32, hdr->in_len, 32), ib); } in = data + sizeof(*hdr); out = in + hdr->in_len; + if (dcp->hw.adopt_live_session && length >= sizeof(*hdr)) { + struct dcp_callback_trace *r; + unsigned long flags; + u32 size = min_t(u32, hdr->in_len, length - sizeof(*hdr)); + + spin_lock_irqsave(&dcp->callback_trace_lock, flags); + r = &dcp->callback_trace[dcp->callback_trace_count % DCP_CALLBACK_TRACE_COUNT]; + memset(r, 0, sizeof(*r)); + memcpy(r->tag, hdr->tag, sizeof(r->tag)); + r->in_len = hdr->in_len; + r->out_len = hdr->out_len; + memcpy(r->input, in, min_t(u32, size, sizeof(r->input))); + dcp->callback_trace_count++; + spin_unlock_irqrestore(&dcp->callback_trace_lock, flags); + } + /* Keep the suspect frequency callback visible even when the startup + * property traffic has consumed the optional RPC trace budget. + */ + if (tag == 408 && dcp->hw.adopt_live_session) + dev_info(dev, "D408 clock request in=%u out=%u data=%*ph\n", + hdr->in_len, hdr->out_len, min_t(u32, hdr->in_len, 16), in); + + if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || + !dcp->cb_handlers[tag]) { + dev_warn(dev, "received unknown callback %c%c%c%c\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + /* + * Leaving a callback unanswered wedges the coprocessor: it + * waits for the ack forever and the outer call never returns. + * On H17P answer it with a zeroed output instead, so the + * warning above names the tag we still have to implement + * rather than costing a reboot to discover one tag at a time. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (hdr->out_len) + memset(out, 0, hdr->out_len); + depth = dcp_push_depth(&ch->depth); + ch->output[depth] = out; + ch->end[depth] = offset + + ALIGN(length, DCP_PACKET_ALIGNMENT); + dcp_ack(dcp, context); + } + return; + } if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || !dcp->cb_handlers[tag]) { @@ -515,6 +608,11 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, depth = dcp_push_depth(&ch->depth); ch->output[depth] = out; ch->end[depth] = offset + ALIGN(length, DCP_PACKET_ALIGNMENT); + ch->tag[depth][0] = hdr->tag[3]; + ch->tag[depth][1] = hdr->tag[2]; + ch->tag[depth][2] = hdr->tag[1]; + ch->tag[depth][3] = hdr->tag[0]; + ch->pushed_at[depth] = jiffies; { bool acked = dcp->cb_handlers[tag](dcp, tag, out, in); @@ -531,16 +629,14 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, } } -static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, - void *data, u32 length) +static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context) { - struct dcp_packet_header *header = data; struct dcp_channel *ch = dcp_get_channel(dcp, context); - void *cookie; + void *cookie, *out; dcp_callback_t cb; - if (!ch) { - dev_warn(dcp->dev, "ignoring ack on context %X\n", context); + if (!ch || !ch->depth) { + dev_warn(dcp->dev, "ignoring ack on empty/invalid context %X\n", context); return; } @@ -548,12 +644,18 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, cb = ch->callbacks[ch->depth]; cookie = ch->cookies[ch->depth]; + /* ACKs carry no payload and have offset zero, including nested calls. + * Their output is in the original AP command record, not the receive + * callback region selected by context0. dcp_push saved that address. + */ + out = ch->output[ch->depth]; ch->callbacks[ch->depth] = NULL; ch->cookies[ch->depth] = NULL; + ch->output[ch->depth] = NULL; if (cb) - cb(dcp, data + sizeof(*header) + header->in_len, cookie); + cb(dcp, out, cookie); } static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) @@ -585,7 +687,7 @@ static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) data = dcp->shmem + channel_offset + offset; if (FIELD_GET(IOMFB_MSG_ACK, message)) - dcpep_handle_ack(dcp, ctx_id, data, length); + dcpep_handle_ack(dcp, ctx_id); else dcpep_handle_cb(dcp, ctx_id, data, length, offset); } @@ -849,6 +951,7 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) int iomfb_start_rtkit(struct apple_dcp *dcp) { dma_addr_t shmem_iova; + int ret; /* * macOS starts sixteen DCP endpoints; we started three. Its StartEP @@ -859,12 +962,18 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) * on the AP side -- which is how 0x2d's absence used to produce DART * faults -- so bring the whole set up, in that order. */ - apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); - apple_rtkit_start_ep(dcp->rtk, IOMFB_ENDPOINT); + ret = dcp_start_rtkit_endpoint(dcp, REMOTE_ALLOC_ENDPOINT); + if (ret) + return ret; + ret = dcp_start_rtkit_endpoint(dcp, IOMFB_ENDPOINT); + if (ret) + return ret; dcp_start_extra_eps(dcp); dcp->shmem = dma_alloc_coherent(dcp->dev, DCP_SHMEM_SIZE, &shmem_iova, GFP_KERNEL); + if (!dcp->shmem) + return -ENOMEM; dcp_send_message(dcp, IOMFB_ENDPOINT, dcpep_set_shmem(shmem_iova)); diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 530ee2123c22f..9fcf06bc787e4 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -204,10 +204,15 @@ enum dcpep_method { void dcp_stall_check(struct work_struct *work); #define IOMFB_METHOD(tag, name) [name] = { #name, { tag[0], tag[1], tag[2], tag[3] } } +#define IOMFB_METHOD_H17(tag, h17g, name) [name] = { #name, \ + { tag[0], tag[1], tag[2], tag[3] }, \ + { h17g[0], h17g[1], h17g[2], h17g[3] } } struct dcp_method_entry { const char *name; char tag[4]; + /* H17G shares the H17P transport but changes selected method IDs. */ + char tag_h17g[4]; }; #define IOMFB_MAX_CB (1000) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 81a68eadc3fbe..a506298d39f1c 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -28,7 +28,7 @@ static_assert(offsetof(struct dc_swap_complete_resp_h17p, swap_id) == 0); static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A000", dcpep_late_init_signal), - IOMFB_METHOD("A025", dcpep_setup_video_limits), /* boot chain: 14th call inside D121, right before flush_supports_power; (0,0) */ + IOMFB_METHOD_H17("A025", "A029", dcpep_setup_video_limits), /* (0,0) */ IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), /* issued from inside callback D206 */ IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), /* issued from inside callback D207 */ IOMFB_METHOD("A385", dcpep_set_create_dfb), /* first call inside D121; (0,0) */ @@ -51,18 +51,20 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { * The default-framebuffer block macOS runs inside D121, straight after * A446 and before A025. Sizes are the traced wire sizes. */ - IOMFB_METHOD("A035", dcpep_get_dfb_compression_info), /* (0x8, 0x8) */ - IOMFB_METHOD("A034", dcpep_get_dfb_info), /* (0x2c, 0x2c) */ + IOMFB_METHOD_H17("A035", "A039", dcpep_get_dfb_compression_info), /* (0x8, 0x8) */ + IOMFB_METHOD_H17("A034", "A038", dcpep_get_dfb_info), /* (0x2c, 0x2c) */ IOMFB_METHOD("A103", dcpep_get_dfb_layout), /* (0xc, 0x8) */ IOMFB_METHOD("A445", dcpep_get_dfb_state), /* (0x4, 0x8) */ IOMFB_METHOD("A104", dcpep_set_dfb_dimensions), /* (0x8, 0) w,h */ IOMFB_METHOD("A105", dcpep_commit_dfb_info), /* (0, 0) */ - IOMFB_METHOD("A033", dcpep_dfb_query), /* (0, 0x4) */ + IOMFB_METHOD_H17("A033", "A037", dcpep_dfb_query), /* (0, 0x4) */ IOMFB_METHOD("A380", dcpep_dfb_ready_query), /* (0, 0x4) */ IOMFB_METHOD("A353", dcpep_pipe_query_353), /* (0, 0x8) */ IOMFB_METHOD("A352", dcpep_pipe_set_352), /* (0x8, 0x4) in 0x70 */ IOMFB_METHOD("A415", dcpep_pipe_cfg_415), /* (0xc, 0xc) zeros */ - IOMFB_METHOD("A031", dcpep_pipe_cfg_031), /* (0x6c, 0) zeros */ + /* H17G 25G76 UPPipeAP_H17GSCD::update_notify_clients_dcp builds + * tag0x41303335 (A035), input0x6c, output0. */ + IOMFB_METHOD_H17("A031", "A035", dcpep_pipe_cfg_031), /* (0x6c, 0) */ IOMFB_METHOD("A414", dcpep_pipe_cfg_414), /* (0x8, 0x8) zeros */ IOMFB_METHOD("A478", dcpep_pipe_query_478), /* (0, 0x4) */ IOMFB_METHOD("A474", dcpep_pipe_query_474), /* (0, 0x4) */ @@ -113,6 +115,16 @@ static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, u64 clock_request = DCP_H17P_DISP0_REG_BASE + 0x68000; u64 scratch = DCP_H17P_DISP0_REG_BASE + 0x20000; + /* The T8142 DT contains the native PMC resources. The fallback above + * is specific to the incomplete legacy T8140 binding. + */ + if (!dcp->hw.legacy_pmc_binding) { + scratch = dcp->disp_bw_scratch_res.start + + dcp->disp_bw_scratch_offset; + clock_request = dcp->disp_bw_doorbell_res.start + + dcp->disp_bw_doorbell_offset; + } + trace_iomfb_callback(dcp, tag, __func__); if (apple_dcp_h17p_rt_bw_encode_reply(&view, scratch, clock_request)) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 9d9afe10f0444..2b04e96667b18 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -360,7 +360,7 @@ static void dcp_ack_dfb_surface(struct apple_dcp *dcp) dcp_ack(dcp, DCP_CONTEXT_CB); } -static void dcp_fill_dfb_surface(u8 *s) +static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) { s[0x002] = 0x01; s[0x00b] = 'A'; @@ -375,6 +375,18 @@ static void dcp_fill_dfb_surface(u8 *s) s[0x035] = 0x04; /* IOSurface.surface_id; macOS writes 4, we wrote 3 */ s[0x051] = 0x01; s[0x149] = 0x01; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) { + /* H17G 25G76 A472: b3a8 surface and the native default surface ID. + * The word at0x29 remains opaque; these are its observed bytes, + * not a size inferred from panel geometry. No addresses are copied. + */ + memcpy(s + 0x00b, "8a3b", 4); + s[0x02a] = 0; + s[0x02b] = 0x4d; + s[0x02c] = 1; + s[0x035] = 2; + } } /* @@ -411,7 +423,7 @@ static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, return true; } - dcp_fill_dfb_surface(req->surface); + dcp_fill_dfb_surface(dcp, req->surface); dcp_register_dfb_surface(dcp, false, req, complete_dfb_surface, NULL); kfree(req); @@ -797,7 +809,7 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d * hardware. Answer it the way macOS does; these apertures are reached * physically, so dva == addr. */ - if (req->index == DCP_H17P_DISP0_REG_INDEX) { + if (dcp->hw.legacy_pmc_binding && req->index == DCP_H17P_DISP0_REG_INDEX) { dev_info(dcp->dev, "map_reg[5]: disp0 window %#llx len %#llx\n", (u64)DCP_H17P_DISP0_REG_BASE, (u64)DCP_H17P_DISP0_REG_SIZE); @@ -1031,7 +1043,7 @@ static void boot_4b(struct apple_dcp *dcp, void *out, void *cookie) * macOS 25G83 sends 0 here, not 1: the traced A000 payload is * "00 aa aa aa", one meaningful byte followed by poison. */ - u32 arg = 0; + u32 arg = dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; dcp_late_init_signal(dcp, false, &arg, boot_5, NULL); #else @@ -1075,27 +1087,23 @@ static void boot_1_11(struct apple_dcp *dcp, void *out, void *cookie) static void boot_1_10(struct apple_dcp *dcp, void *out, void *cookie) { - /* - * The firmware only builds a default framebuffer when it has non-zero - * geometry (its create_default_fb_surface is gated on "h && w"), so - * withholding A104 keeps the pipe inert and lets a real DRM modeset - * drive the first swap instead. macOS always pushes it, but macOS has - * an inherited scanout to keep alive and we do not. + struct dcp_dfb_dimensions_h17p dim; + + /* A445 returns the dimensions that native macOS passes to A104. + * The H17G capture is2560x1664; the H17P capture is2408x1506. */ - if (0) { - boot_1_11(dcp, NULL, NULL); + memcpy(&dim, out, sizeof(dim)); + if (!dim.width || !dim.height || dim.width > 16384 || dim.height > 16384) { + struct dcp_channel *ch = &dcp->ch_cb; + + dev_err(dcp->dev, "invalid default framebuffer dimensions %ux%u\n", + dim.width, dim.height); + *(u8 *)ch->output[ch->depth - 1] = false; + dcp_ack(dcp, DCP_CONTEXT_CB); return; } - - /* - * The panel's native size, which is what macOS pushes here: the traced - * A104 payload is 68 09 00 00 e2 05 00 00 = 2408 x 1506. The firmware - * has already told us the same numbers in D582. - */ - struct dcp_dfb_dimensions_h17p dim = { - .width = 2408, - .height = 1506, - }; + dev_info(dcp->dev, "default framebuffer dimensions %ux%u\n", + dim.width, dim.height); dcp_set_dfb_dimensions(dcp, false, &dim, boot_1_11, NULL); } From 052b72e87f7a4716506711e55c4ee56cb40c22cb Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 00:37:07 -0400 Subject: [PATCH 0217/1579] drm/apple: add an explicit T6050 firmware 27.0 wire ABI Derive method IDs, callback IDs and minimum lengths from the native 26A428 serializers. Encode the larger swap request without the legacy two-byte IOVA shift. Preserve legacy display routing and firmware selection; J714 hardware qualification remains pending. --- Documentation/gpu/apple-dcp-t6050.rst | 104 +++++++ drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-internal.h | 6 + drivers/gpu/drm/apple/dcp.c | 109 ++++--- drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/iomfb.c | 38 ++- drivers/gpu/drm/apple/iomfb_h17p.c | 1 + drivers/gpu/drm/apple/iomfb_template.c | 15 + drivers/gpu/drm/apple/iomfb_template.h | 8 + drivers/gpu/drm/apple/iomfb_v12_3.c | 1 + drivers/gpu/drm/apple/iomfb_v13_3.c | 1 + drivers/gpu/drm/apple/iomfb_v27.c | 407 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_v27.h | 33 ++ 13 files changed, 646 insertions(+), 79 deletions(-) create mode 100644 Documentation/gpu/apple-dcp-t6050.rst create mode 100644 drivers/gpu/drm/apple/iomfb_v27.c create mode 100644 drivers/gpu/drm/apple/iomfb_v27.h diff --git a/Documentation/gpu/apple-dcp-t6050.rst b/Documentation/gpu/apple-dcp-t6050.rst new file mode 100644 index 0000000000000..9f618e35ec431 --- /dev/null +++ b/Documentation/gpu/apple-dcp-t6050.rst @@ -0,0 +1,104 @@ +T6050 display firmware 27.0 ABI +============================== + +This is an explicit firmware ABI port, not a hardware qualification. The +T6050 node must name ``apple,t6050-dcp`` and carry +``apple,firmware-compat = <27 0 0>``. The stable J714 guest DT is unchanged, +so simpledrm remains the display driver until a separate DCP test DT is used. +Do not advertise another 27.x release as compatible without comparing it. + +Reference +--------- + +The reference is the mac17j kernelcache from UniversalMac 27.0 build 26A428, +SHA-256 a691760372651464138779c3201c1886a385ca656397362d8e7701ba19ebf436 +of the decompressed Mach-O. The native serializers and callback dispatchers +retain symbols. The latest separate macOS capture is 27.2 build 26B5086k; +it is not the source of this 27.0 ABI table. Firmware/driver bytes and device-specific captures remain +private; they are not part of the source changes. + +The saved live-27.0 ADT names DCP UUID +64863924-B56B-3E32-87FE-038677F52709. Those UUID bytes occur at offset +205332 in the decompressed 26A428 t605x DCP bundle (SHA-256 + d9384573025721074ec1a7190626a8bab36cf61f764055e8e05c7d5d98cb318f). +This identifies the saved boot's firmware, not the current live boot. Before +enabling the candidate, capture the live DCP UUID and initial IOMFB version +word and compare them to the reference. A manually supplied firmware-compat +property selects the ABI; it does not prove what iBoot loaded. + +The transport is imported from the Neo H17P series by DjDeveloperr and the +T8142 H17G integration by Acelogic. Existing J414s Type-C/Thunderbolt routing, +VRR, and legacy boolean ``apple,iomfb-surfaces`` semantics are retained. +The existing RTKit IOP-OFF plus idle-notification implementation is retained. + +Verified differences +-------------------- + +===================================== =========== =========== +Operation H17P 25G83 T605X 26A428 +===================================== =========== =========== +pmu_service_matched A131 A136 +backlight_service_matched A132 A137 +set_create_DFB A385 A389 +vi_set_temperature_hint A386 A390 +swap_start A406 A406 +swap_submit_main A408 A408 +set_parameter A442 A450 +create_default_frame_buffer A446 A454 +first_client_open A457 A466 +setDisplayRefreshProperties A465 A475 +flush_supportsPower A468 A478 +update_dfb(IOSurface*) A472 A483 +setPowerState A473 A484 +start_hardware_boot callback D121 D120 +create_default_fb_surface callback D582 D586 +swap_complete_ap_gated callback D590 D594 +===================================== =========== =========== + +The 26A428 UPPipeAP_H17GSCD table uses A027 for setup_video_limits, A033 for +update_notify_clients, A035 for apt_supported, A036 for get_dfb_info, and A037 +for get_dfb_compression_info. These also differ from the 25G76 H17G table. + +At 0xfffffe000ab7fcec the A408 serializer copies 0x6e0 bytes from +IOMFBSwapRec. It emits four 0x22c-byte surfaces starting at 0x6e0, IOVAs at +0xf90, and the trailing null flags at 0xfea. The request is 0xff4 bytes and +the reply is 0x0c. The old 26.x request was 0xe9c bytes and required a +two-byte tail relocation; applying that relocation to 27.0 corrupts its +already aligned IOVAs. The 27.0 profile does not perform it. + +The native swap_submit_with_tags routine confirms swap ID at 0x98, source +rectangles at 0xac, destination rectangles at 0x10c, and enabled surfaces at +0x14c. The D594 deserializer requires 0x730 input bytes, retaining the 26.x +completion layout. All these offsets and sizes have compile-time assertions. + +A479 abort_swaps serializes an eight-byte client handle and returns four +bytes. It must not inherit the older IOUserClient request/response structs. +The 27.0 callback dispatcher checks native minimum wire lengths before +calling typed handlers. Provider property replies use the matching device's +raw property bytes, rather than Neo-specific panel power tables. + +Hardware integration still required +----------------------------------- + +Saved J714s ADT resources, after parent-range translation: + +* DCP ASC: 0x48ae00000, size 0x88000; IOP VBAR: 0x48a850000, size 0x4000. +* DCP DART: 0x48a340000, id 19, firmware SID 23, exclave SID 24. +* Display DART: 0x48a300000, id 18, scanout SID 0, PIODMA SID 16. +* Both DARTs include a secondary aperture at 0x2803f0000 and vm-base + 0x10000000000. Existing firmware mappings must survive guest attachment. +* Display reg[4] resolves to 0x286700000, size 0x114000. PMC bandwidth + resources must be derived from the T6050 PMGR descriptions. +* Display clock IDs are 361 and 458 in this ADT. The 27.0 driver requires + ``apple,clock-ids`` in the same order as ``clocks`` and resolves callbacks + against those handles. It does not reuse Neo's IDs 154 and 194. + +The guest needs explicit ASC/mailbox/interrupt access, SPTM DART mediation +for these identities, and native panel properties. Do not clear or reset +firmware-owned display tables merely to make a probe succeed. + +The default-surface descriptor values and swap flag policy inherited from +the 26.x driver are not yet qualified on J714. Native 27.0 surface +serialization and panel/PMC property behavior need validation before this +profile is made the default. No J714 DCP pageflip, power transition, +backlight, or external-output result is claimed by this port. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index fb6cc77fc6cfa..a312db69c4bbe 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -11,6 +11,7 @@ appledrm-y += ibootep.o appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o appledrm-y += iomfb_h17p.o +appledrm-y += iomfb_v27.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 29fc9c44ba337..0a2ad40d72848 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -20,6 +20,7 @@ #include "dptxep.h" #include "iomfb.h" #include "iomfb_h17p.h" +#include "iomfb_v27.h" #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "iomfb-surface.h" @@ -70,6 +71,7 @@ enum dcp_firmware_version { DCP_FIRMWARE_V_12_3, DCP_FIRMWARE_V_13_5, DCP_FIRMWARE_H17P, + DCP_FIRMWARE_V_27_0, }; enum { @@ -233,6 +235,9 @@ struct apple_dcp { /* clock rate request by dcp in */ struct clk *clk; struct clk *clk_194; + struct clk_bulk_data *firmware_clocks; + u32 *firmware_clock_ids; + unsigned int num_firmware_clocks; /* DCP shared memory */ void *shmem; @@ -275,6 +280,7 @@ struct apple_dcp { struct dcp_swap_submit_req_v12_3 v12_3; struct dcp_swap_submit_req_v13_3 v13_3; struct dcp_swap_submit_req_h17p h17p; + struct dcp_swap_submit_req_v27 v27; } swap; /* swap id of the last completed swap */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f54c123459d75..9a59e994357b8 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1895,6 +1895,9 @@ static void _dcp_poweroff(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_poweroff_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_poweroff_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -1960,6 +1963,9 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_sleep_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_sleep_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -2004,6 +2010,9 @@ void dcp_poweron(struct platform_device *pdev) case DCP_FIRMWARE_H17P: iomfb_poweron_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_poweron_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -2376,6 +2385,14 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) char fw_str[DCP_FW_VERSION_STR_LEN]; int ret; + if (hw->firmware_compat == DCP_FIRMWARE_V_27_0) { + ret = dcp_read_fw_version(dev, "apple,firmware-compat", compat_str); + if (ret < 0 || strcmp(compat_str, "27.0.0")) { + dev_err(dev, "T6050 requires an explicit 27.0.0 firmware ABI profile\n"); + return DCP_FIRMWARE_UNKNOWN; + } + return DCP_FIRMWARE_V_27_0; + } if (hw->firmware_compat != DCP_FIRMWARE_UNKNOWN) return hw->firmware_compat; @@ -2548,7 +2565,7 @@ static int dcp_restart_inherited_session(struct apple_dcp *dcp) { int ret; - if (dcp->fw_compat != DCP_FIRMWARE_H17P) + if (dcp->fw_compat < DCP_FIRMWARE_H17P) return apple_rtkit_quiesce(dcp->rtk); /* @@ -2567,60 +2584,6 @@ static int dcp_restart_inherited_session(struct apple_dcp *dcp) return dcp_asc_restart(dcp); } -/* - * Reset the display coprocessor's ASC core. - * - * Once Linux has driven a DCP session the firmware stops answering the - * management endpoint: it will not re-HELLO on INIT (it is already ON) and it - * will not acknowledge a power-state change either, so neither apple_rtkit_wake - * nor apple_rtkit_quiesce can recover it and the module could previously only - * be reloaded by rebooting the machine. - * - * CPU_CONTROL.RUN is a raw register in the ASC wrapper and needs no cooperation - * from the firmware, so clearing it stops the core no matter how wedged it is. - * The firmware image itself is left alone -- iBoot mapped it into the DCP's own - * DART and nothing here touches those mappings -- so setting RUN again restarts - * it from its reset vector and it comes back with a fresh HELLO. - */ -static int dcp_asc_restart(struct apple_dcp *dcp) -{ - u32 ctrl, status; - int i; - - ctrl = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); - writel_relaxed(ctrl & ~APPLE_DCP_COPROC_CPU_CONTROL_RUN, - dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); - - /* - * T8140 clears CPU_STATUS.RUNNING when the core halts but never sets - * the STOPPED bit (status goes 0x6d -> 0x48), so gate on RUNNING going - * away rather than on STOPPED appearing. - */ - for (i = 0; i < 500; i++) { - status = readl_relaxed(dcp->coproc_reg + - APPLE_DCP_COPROC_CPU_STATUS); - if (!(status & APPLE_DCP_COPROC_CPU_STATUS_RUNNING) || - (status & APPLE_DCP_COPROC_CPU_STATUS_STOPPED)) - break; - usleep_range(100, 200); - } - if (status & APPLE_DCP_COPROC_CPU_STATUS_RUNNING) { - dev_err(dcp->dev, "DCP ASC did not stop (status %#x)\n", status); - return -ETIMEDOUT; - } - - dev_info(dcp->dev, "DCP ASC stopped (status %#x), restarting it\n", - status); - usleep_range(1000, 2000); - writel_relaxed(ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, - dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); - return 0; -} - - - - - static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; @@ -2744,6 +2707,24 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) return ret; } + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) { + int count = of_property_count_u32_elems(dev->of_node, "apple,clock-ids"); + + ret = devm_clk_bulk_get_all(dev, &dcp->firmware_clocks); + if (ret < 0) + return dev_err_probe(dev, ret, "failed to acquire firmware clocks\n"); + if (!ret || count != ret) + return dev_err_probe(dev, -EINVAL, "firmware clock IDs must match clocks\n"); + dcp->num_firmware_clocks = count; + dcp->firmware_clock_ids = devm_kcalloc(dev, count, sizeof(u32), GFP_KERNEL); + if (!dcp->firmware_clock_ids) + return -ENOMEM; + ret = of_property_read_u32_array(dev->of_node, "apple,clock-ids", + dcp->firmware_clock_ids, count); + if (ret) + return ret; + } + dcp->clk = devm_clk_get(dev, NULL); if (IS_ERR(dcp->clk)) return dev_err_probe(dev, PTR_ERR(dcp->clk), @@ -2766,7 +2747,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); spin_lock_init(&dcp->swapped_out_lock); - if (dcp->fw_compat == DCP_FIRMWARE_H17P && !dcp->hw.adopt_live_session) + if (dcp->fw_compat >= DCP_FIRMWARE_H17P && !dcp->hw.adopt_live_session) dcp_enable_pmc_domain(dcp); if (start_mode == APPLE_DCP_COPROCESSOR_ADOPT_AND_RESTART && @@ -2855,7 +2836,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) ret = dcp_restart_inherited_session(dcp); dcp->adopting_session = false; - if (ret && dcp->fw_compat == DCP_FIRMWARE_H17P) + if (ret && dcp->fw_compat >= DCP_FIRMWARE_H17P) return ret; if (!ret) ret = apple_rtkit_wake(dcp->rtk); @@ -3013,7 +2994,7 @@ static int dcp_platform_probe(struct platform_device *pdev) mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->tb_lock); - if (fw_compat != DCP_FIRMWARE_H17P) { + if (fw_compat < DCP_FIRMWARE_H17P) { /* Legacy device trees encode one enable flag per surface. */ count = 0; i = 0; @@ -3252,6 +3233,17 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8142 = { .firmware_compat = DCP_FIRMWARE_H17P, }; +/* Native 26A428 T605X uses DCPLink with the 27.0 method/callback ABI. + * The loader must preserve the firmware mappings and running ASC generation. + */ +static const struct apple_dcp_hw_data apple_dcp_hw_t6050 = { + .num_dptx_ports = 0, + .iomfb_method_profile = DCP_IOMFB_METHODS_H17G, + .adopt_running_coprocessor = true, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_V_27_0, +}; + static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { .num_dptx_ports = 0, }; @@ -3264,6 +3256,7 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, { .compatible = "apple,t8140-dcp", .data = &apple_dcp_hw_t8140, }, + { .compatible = "apple,t6050-dcp", .data = &apple_dcp_hw_t6050, }, { .compatible = "apple,t8142-dcp", .data = &apple_dcp_hw_t8142, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 189110587eb1d..b8a2a91f2bb13 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -64,6 +64,7 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, const struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode); void dcp_set_dimensions(struct apple_dcp *dcp); +int dcp_start_rtkit_endpoint(struct apple_dcp *dcp, u8 endpoint); void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index aa498eee369b9..6f2061da93ede 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -577,26 +577,10 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, return; } - if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || - !dcp->cb_handlers[tag]) { - dev_warn(dev, "received unknown callback %c%c%c%c\n", - hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); - /* - * Leaving a callback unanswered wedges the coprocessor: it - * waits for the ack forever and the outer call never returns. - * On H17P answer it with a zeroed output instead, so the - * warning above names the tag we still have to implement - * rather than costing a reboot to discover one tag at a time. - */ - if (dcp->fw_compat == DCP_FIRMWARE_H17P) { - if (hdr->out_len) - memset(out, 0, hdr->out_len); - depth = dcp_push_depth(&ch->depth); - ch->output[depth] = out; - ch->end[depth] = offset + - ALIGN(length, DCP_PACKET_ALIGNMENT); - dcp_ack(dcp, context); - } + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0 && + !iomfb_v27_callback_size_valid(tag, hdr->in_len, hdr->out_len)) { + dev_err(dev, "firmware27 callback D%03d has incompatible lengths %u/%u\n", + tag, hdr->in_len, hdr->out_len); return; } @@ -822,6 +806,9 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, case DCP_FIRMWARE_H17P: ret = iomfb_modeset_h17p(dcp, crtc_state); break; + case DCP_FIRMWARE_V_27_0: + ret = iomfb_modeset_v27(dcp, crtc_state); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -895,6 +882,9 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) case DCP_FIRMWARE_H17P: iomfb_flush_h17p(dcp, crtc, state); break; + case DCP_FIRMWARE_V_27_0: + iomfb_flush_v27(dcp, crtc, state); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -913,6 +903,9 @@ static void iomfb_start(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_start_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_start_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -936,7 +929,7 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) * of the same InitComplete word (macOS 25G83 sends e.g. * 0x004045bc9ef40101). The transport below it is unchanged. */ - if (dcp->fw_compat == DCP_FIRMWARE_H17P) + if (dcp->fw_compat >= DCP_FIRMWARE_H17P) dev_info(dcp->dev, "IOMFB: init complete %#llx (version %llu)\n", message, @@ -996,6 +989,9 @@ void iomfb_shutdown(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_shutdown_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_shutdown_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index a506298d39f1c..45cf7e9a68103 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -6,6 +6,7 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "version_utils.h" +#include "dcp-internal.h" struct dcp_h17p_hotplug_request { __le64 connected; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 2b04e96667b18..69171ed13b037 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -64,7 +64,20 @@ DCP_THUNK_OUT(iomfb_a358_vi_set_temperature_hint, iomfbep_a358_vi_set_temperatur IOMFB_THUNK_INOUT(set_matrix); IOMFB_THUNK_INOUT(get_color_remap_mode); IOMFB_THUNK_INOUT(last_client_close); +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) +static void iomfb_abort_swaps_dcp(struct apple_dcp *dcp, bool oob, + struct iomfb_abort_swaps_dcp_req *data, + dcp_callback_t cb, void *cookie) +{ + /* 26A428 A479 serializes the client handle, not an IOUserClient. */ + u64 client = data->client.addr; + + dcp_push(dcp, oob, &dcp_methods[iomfbep_abort_swaps_dcp], + sizeof(client), sizeof(u32), &client, cb, cookie); +} +#else IOMFB_THUNK_INOUT(abort_swaps_dcp); +#endif DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct DCP_FW_NAME(dcp_swap_submit_req), @@ -1363,6 +1376,7 @@ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) * empty rectangles, and rejects the swap with * "verify_scaling: error: null rectangle: src:0x0+0,0 dst:0x0+0,0". */ +#if DCP_FW_VER < DCP_FW_VERSION(27, 0, 0) { u8 *r = (u8 *)req; size_t n = sizeof(*req); @@ -1384,6 +1398,7 @@ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) memmove(r + off - 2, r + off, n - off); memset(r + n - 2, 0, 2); } +#endif } #else static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) { } diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 82b4cc92be4a9..fcf60d1cbc526 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -72,8 +72,13 @@ struct DCP_FW_NAME(dcp_swap) { * dcp_surface_h17p = 0x22c, SWAP_SURFACES = 4 and no surf2 block leaves * exactly 0x122 here. 0x278 makes dcp_swap 0x6e0 and forces a * non-integer surface size, so it cannot be right. */ +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + /* 26A428 copies exactly 0x6e0 bytes of IOMFBSwapRec in A408. */ + u8 h17p_tail[0x230]; +#else u8 h17p_tail[0xda]; #endif +#endif } __packed; /* Information describing a surface */ @@ -124,6 +129,9 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { u8 unkU32Ptr_null; u8 unkU32out_null; #endif +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + u8 padding[2]; +#endif #if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 padding[1]; #endif diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.c b/drivers/gpu/drm/apple/iomfb_v12_3.c index 3256974fb11d0..04bb2b2549ec9 100644 --- a/drivers/gpu/drm/apple/iomfb_v12_3.c +++ b/drivers/gpu/drm/apple/iomfb_v12_3.c @@ -5,6 +5,7 @@ #include "iomfb_v13_3.h" #include "iomfb_h17p.h" #include "version_utils.h" +#include "dcp-internal.h" static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A000", dcpep_late_init_signal), diff --git a/drivers/gpu/drm/apple/iomfb_v13_3.c b/drivers/gpu/drm/apple/iomfb_v13_3.c index 9adce7b150fd0..be00c57a5435b 100644 --- a/drivers/gpu/drm/apple/iomfb_v13_3.c +++ b/drivers/gpu/drm/apple/iomfb_v13_3.c @@ -5,6 +5,7 @@ #include "iomfb_v13_3.h" #include "iomfb_h17p.h" #include "version_utils.h" +#include "dcp-internal.h" static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A000", dcpep_late_init_signal), diff --git a/drivers/gpu/drm/apple/iomfb_v27.c b/drivers/gpu/drm/apple/iomfb_v27.c new file mode 100644 index 0000000000000..2a6b7cbcf6471 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v27.c @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include "dcp-link.h" +#include "iomfb_v27.h" +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "version_utils.h" +#include "dcp-internal.h" + +struct dcp_h17p_hotplug_request { + __le64 connected; + u8 tiled_display[0x4c]; + u8 tiled_display_null; + u8 padding[3]; +} __packed; + +struct dcp_h17p_swap_info_request { + u8 swap_info[0xe0]; + u8 swap_info_null; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_h17p_hotplug_request) == 0x58); +static_assert(sizeof(struct dcp_h17p_swap_info_request) == 0xe4); +/* D590 swap_complete_ap_gated: 0x730 on the wire in 1919/1919 traced swaps. */ +static_assert(sizeof(struct dc_swap_complete_resp_v27) == 0x730); +static_assert(offsetof(struct dc_swap_complete_resp_v27, swap_id) == 0); + +/* Native 26A428 T605X kernelcache RPC serializers. The method IDs are not + * compatible with the 25G83 H17P or 25G76 H17G tables. See t6050-dcp.rst. + */ +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD("A027", dcpep_setup_video_limits), + IOMFB_METHOD("A136", iomfbep_a131_pmu_service_matched), + IOMFB_METHOD("A137", iomfbep_a132_backlight_service_matched), + IOMFB_METHOD("A389", dcpep_set_create_dfb), + IOMFB_METHOD("A390", iomfbep_a358_vi_set_temperature_hint), + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A406", dcpep_swap_start), + IOMFB_METHOD("A408", dcpep_swap_submit), + IOMFB_METHOD("A411", dcpep_set_display_device), + IOMFB_METHOD("A412", dcpep_is_main_display), + IOMFB_METHOD("A413", dcpep_set_digital_out_mode), + IOMFB_METHOD("A423", iomfbep_set_matrix), + IOMFB_METHOD("A427", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A450", dcpep_set_parameter_dcp), + IOMFB_METHOD("A454", dcpep_create_default_fb), + IOMFB_METHOD("A037", dcpep_get_dfb_compression_info), + IOMFB_METHOD("A036", dcpep_get_dfb_info), + IOMFB_METHOD("A103", dcpep_get_dfb_layout), + IOMFB_METHOD("A453", dcpep_get_dfb_state), + IOMFB_METHOD("A104", dcpep_set_dfb_dimensions), + IOMFB_METHOD("A105", dcpep_commit_dfb_info), + IOMFB_METHOD("A035", dcpep_dfb_query), + IOMFB_METHOD("A384", dcpep_dfb_ready_query), + IOMFB_METHOD("A353", dcpep_pipe_query_353), + IOMFB_METHOD("A352", dcpep_pipe_set_352), + IOMFB_METHOD("A415", dcpep_pipe_cfg_415), + IOMFB_METHOD("A033", dcpep_pipe_cfg_031), + IOMFB_METHOD("A414", dcpep_pipe_cfg_414), + IOMFB_METHOD("A489", dcpep_pipe_query_478), + IOMFB_METHOD("A485", dcpep_pipe_query_474), + IOMFB_METHOD("A487", dcpep_pipe_set_476), + IOMFB_METHOD("A428", dcpep_pipe_set_428), + IOMFB_METHOD("A458", dcpep_enable_disable_video_power_savings), + IOMFB_METHOD("A466", dcpep_first_client_open), + IOMFB_METHOD("A467", iomfbep_last_client_close), + IOMFB_METHOD("A475", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A478", dcpep_flush_supports_power), + IOMFB_METHOD("A479", iomfbep_abort_swaps_dcp), + IOMFB_METHOD("A484", dcpep_set_power_state), + IOMFB_METHOD("A483", dcpep_register_dfb_surface), +}; + +#define DCP_FW v27 +#define DCP_FW_VER DCP_FW_VERSION(27, 0, 0) + +#include "iomfb_template.c" + +static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + struct apple_dcp_link_rpc_view view = { + .call = APPLE_DCP_LINK_CALL_D003, + .input_size = sizeof(struct apple_dcp_h17p_rt_bw_request), + .output_size = sizeof(struct apple_dcp_h17p_rt_bw_reply), + .input = in, + .output = out, + }; + /* + * Take these from the macOS capture rather than from the DT resources. + * macOS answers D003 with one stable record, + * 0000820203000000 0080860203000000 00000000 + * -> scratch 0x302820000, doorbell 0x302868000 -- which is + * DISP0_REG_BASE + 0x20000 and + 0x68000, both inside the 0xbc000 + * window we already hand out for map_reg index 5. The DT lookup lands + * 0x5a80000 higher, so the firmware was being pointed at the wrong + * block for its bandwidth scratch and doorbell. + */ + u64 clock_request = DCP_H17P_DISP0_REG_BASE + 0x68000; + u64 scratch = DCP_H17P_DISP0_REG_BASE + 0x20000; + + /* The T8142 DT contains the native PMC resources. The fallback above + * is specific to the incomplete legacy T8140 binding. + */ + if (!dcp->hw.legacy_pmc_binding) { + scratch = dcp->disp_bw_scratch_res.start + + dcp->disp_bw_scratch_offset; + clock_request = dcp->disp_bw_doorbell_res.start + + dcp->disp_bw_doorbell_offset; + } + + trace_iomfb_callback(dcp, tag, __func__); + if (apple_dcp_h17p_rt_bw_encode_reply(&view, scratch, + clock_request)) + dev_warn(dcp->dev, "D003 bandwidth request is malformed\n"); + return true; +} + +static bool trampoline_get_frequency_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct apple_dcp_h17p_clock_request *request = in; + u32 id = le32_to_cpu(request->clock_id); + unsigned int i; + u64 rate = 0; + + for (i = 0; i < dcp->num_firmware_clocks; i++) { + if (dcp->firmware_clock_ids[i] == id) { + rate = clk_get_rate(dcp->firmware_clocks[i].clk); + break; + } + } + if (i == dcp->num_firmware_clocks) + dev_err(dcp->dev, "DCP requested undeclared firmware clock %u\n", id); + *(__le64 *)out = cpu_to_le64(rate); + return true; +} + +static bool trampoline_hotplug_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_hotplug_request *request = in; + + if (!(request->tiled_display_null & 1)) + memcpy(out, request->tiled_display, + sizeof(request->tiled_display)); + return trampoline_hotplug(dcp, tag, out, in); +} + +static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_swap_info_request *request = in; + + trace_iomfb_callback(dcp, tag, __func__); + if (!(request->swap_info_null & 1)) + memcpy(out, request->swap_info, sizeof(request->swap_info)); + return true; +} + +/* + * D574 (in 4, out 8) has no analogue in the v12.3/v13.5 tables. macOS 25G83 + * answers 0x435b0000 == 219.0f for the only selector it is ever called with + * (in == 0), identically in both captured boots. 219 is 235 - 16, the + * BT.601/709 studio-swing luma range, so this feeds the DCP's colour matrix + * setup; answering zero would put a zero scale into it. + */ +static bool iomfbep_cb_video_range_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + __le32 *resp = out; + + resp[0] = cpu_to_le32(0x435b0000); /* 219.0f */ + resp[1] = 0; + return true; +} + +/* + * Tag numbering below is the one the H17P firmware actually uses, taken from + * macOS 25G83 traces on this machine rather than by shifting the v13.5 table: + * the service-creation block is D108..D113 exactly as on v13.5, and the swap + * completion is D590. A callback with no entry here is never acked, so a + * missing tag stalls the DCP rather than degrading it. + */ +/* + * D400 -- iOSSvcProx::get_property(name, DataBuffer *, bool). + * + * The firmware asks the "PROV" (provider) service for the panel's power model + * during init_power_lut(); answering "not found" makes it log + * IOMFB init_power_lut: failed to read power-lut-data-x + * and then fail to create the default framebuffer surface. The request + * carries a capacity at +0x44 and the reply is { u8 data[0xc00]; __le32 + * length; }, where length is the byte count written and 0 means "no such + * property". These are the values macOS 25G83 returns on this machine, + * byte-identical across both captured boots. + */ +struct dcp_h17p_get_data_prop_req { + char obj[4]; + char key[0x40]; + __le32 length; + u8 data_null; + u8 length_null; + u8 padding[2]; +} __packed; +static_assert(sizeof(struct dcp_h17p_get_data_prop_req) == 0x4c); + +struct dcp_h17p_get_data_prop_resp { + u8 data[0xc00]; + __le32 length; +} __packed; +static_assert(sizeof(struct dcp_h17p_get_data_prop_resp) == 0xc04); + +static bool trampoline_get_data_prop_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_get_data_prop_req *req = in; + struct dcp_h17p_get_data_prop_resp *resp = out; + const struct property *prop; + u32 cap = min_t(u32, le32_to_cpu(req->length), sizeof(resp->data)); + + if (memcmp(req->obj, "VORP", sizeof(req->obj)) || + strnlen(req->key, sizeof(req->key)) == sizeof(req->key)) + return true; + /* Preserve the firmware's native little-endian property bytes in DT. */ + prop = of_find_property(dcp->dev->of_node, req->key, NULL); + if (!prop || prop->length < 0 || prop->length > cap) + return true; + memcpy(resp->data, prop->value, prop->length); + resp->length = cpu_to_le32(prop->length); + return true; +} + +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = dcpep_cb_d000_h17p, /* native did_boot callback */ + [1] = trampoline_true, + [2] = trampoline_nop, + [3] = trampoline_rt_bandwidth_h17p, + [6] = trampoline_set_frame_sync_props_h17p, /* 0x54/0x50, identity scale */ + [100] = iomfbep_cb_match_pmu_service, /* match_pmu_service */ + [101] = trampoline_zero, + [102] = trampoline_nop, /* set_number_property */ + [104] = trampoline_nop, /* set_boolean_property */ + [107] = trampoline_true, /* create_provider_service */ + [108] = trampoline_true, /* create_product_service */ + [109] = trampoline_true, /* create_PMU_service */ + [110] = trampoline_true, /* create_iomfb_service */ + [111] = trampoline_create_backlight_service, /* create_backlight_service */ + [112] = trampoline_true, /* create_nvram_service */ + [113] = trampoline_zero, /* CoreAnalyticsSendEvent */ + [116] = trampoline_nop, /* set_idle_caching_state_ap */ + [117] = trampoline_zero, /* upload_trace_start */ + [118] = trampoline_zero, /* upload_trace_chunk */ + [119] = trampoline_zero, /* upload_trace_end */ + [120] = dcpep_cb_boot_1, /* start_hardware_boot */ + [121] = trampoline_false, /* is_dark_boot */ + [122] = trampoline_false, /* is_waking_from_hibernate */ + [123] = trampoline_zero, /* detect_fastsim */ + [124] = trampoline_read_edt_data, /* read_edt_data */ + [126] = trampoline_prop_start, /* setDCPAVPropStart */ + [127] = trampoline_prop_chunk, /* setDCPAVPropChunk */ + [128] = trampoline_prop_end, /* setDCPAVPropEnd */ + [129] = trampoline_allocate_bandwidth, /* allocate_bandwidth */ + [201] = trampoline_map_piodma, /* map_buf */ + [202] = trampoline_unmap_piodma, /* unmap_buf */ + [206] = iomfbep_cb_match_pmu_service_2, /* match_pmu_service */ + [207] = iomfbep_cb_match_backlight_service, /* match_backlight_service */ + [208] = trampoline_nop, /* update_backlight_factor_prop */ + [209] = trampoline_get_time, /* get_calendar_time_ms */ + [300] = trampoline_pr_publish, + [400] = trampoline_get_data_prop_v27, /* get_property */ + [401] = trampoline_get_uint_prop, /* get_uint_prop */ + [404] = trampoline_nop, /* set_uint_prop */ + [406] = trampoline_set_fx_prop, /* set_fx_prop */ + [408] = trampoline_get_frequency_v27, /* getClockFrequency */ + [411] = trampoline_map_reg, /* mapDeviceMemoryWithIndex */ + [413] = trampoline_true, /* setProperty */ + [414] = trampoline_sr_set_property_int, /* setProperty */ + [415] = trampoline_true, /* setProperty */ + [451] = trampoline_allocate_buffer, /* allocate_buffer */ + [452] = trampoline_map_physical, /* prepare */ + [454] = trampoline_release_mem_desc, /* release_descriptor */ + [552] = trampoline_true, + [561] = trampoline_true, + [563] = trampoline_true, + [565] = trampoline_true, + [567] = trampoline_true, + [572] = trampoline_zero, /* powerUpDART */ + [574] = iomfbep_cb_video_range_h17p, + [575] = trampoline_hotplug_h17p, + [576] = trampoline_nop, + /* + * We do not create a default framebuffer surface, so answering "yes" + * makes the firmware go on to program the display engine for a surface + * that does not exist: it then DMAs from a null base and the piodma + * DART (412300000) faults continuously on SID 0, which starves the + * system into an RCU stall. Decline instead. + */ + [586] = iomfbep_cb_create_dfb_surface, /* create_default_fb_surface */ + [587] = trampoline_nop, /* clear_default_surface */ + [588] = trampoline_nop, /* swap_notify_gated */ + [589] = trampoline_swap_info_h17p, /* swap_info_notify_dispatch */ + [594] = trampoline_swap_complete, /* swap_complete_ap_gated */ + [596] = trampoline_swap_complete_intent_gated, /* swap_complete_intent_gated */ + [597] = trampoline_abort_swap_ap_gated, /* abort_swap_ap_gated */ + [598] = trampoline_enable_backlight_message_ap_gated, /* enable_backlight_message_ap_gated */ + [599] = trampoline_nop, /* setSystemConsoleMode */ + [601] = trampoline_false, /* isDFBAllocated */ + [602] = trampoline_false, + [603] = trampoline_nop, /* find_swap_function_gated */ +}; + +/* Bounds from the native callback deserializers, before invoking a handler. + * This protects the typed callbacks from a mismatched firmware ABI. + */ +static const struct { + u16 input, output; +} callback_sizes[IOMFB_MAX_CB] = { + [0] = { 0x0, 0x4 }, + [1] = { 0x0, 0x4 }, + [2] = { 0x0, 0x0 }, + [3] = { 0x4, 0x14 }, + [6] = { 0x54, 0x50 }, + [300] = { 0x10, 0x0 }, + [100] = { 0x0, 0x0 }, + [101] = { 0x0, 0x4 }, + [102] = { 0x44, 0x0 }, + [104] = { 0x44, 0x0 }, + [107] = { 0x0, 0x4 }, + [108] = { 0x0, 0x4 }, + [109] = { 0x0, 0x4 }, + [110] = { 0x0, 0x4 }, + [111] = { 0x0, 0x4 }, + [112] = { 0x0, 0x4 }, + [113] = { 0x1044, 0x1004 }, + [116] = { 0xc, 0x0 }, + [117] = { 0x14, 0x4 }, + [118] = { 0x3e88, 0x4 }, + [119] = { 0x4, 0x4 }, + [120] = { 0x0, 0x4 }, + [121] = { 0x0, 0x4 }, + [122] = { 0x0, 0x4 }, + [123] = { 0x0, 0x4 }, + [124] = { 0x64, 0x24 }, + [126] = { 0x4, 0x4 }, + [127] = { 0x1008, 0x4 }, + [128] = { 0x40, 0x4 }, + [129] = { 0x1c, 0x14 }, + [201] = { 0xc, 0x10 }, + [202] = { 0x18, 0x0 }, + [206] = { 0x0, 0x4 }, + [207] = { 0x0, 0x4 }, + [208] = { 0x4, 0x0 }, + [209] = { 0x0, 0x8 }, + [400] = { 0x4c, 0xc04 }, + [401] = { 0x50, 0xc }, + [404] = { 0x48, 0x0 }, + [406] = { 0x48, 0x0 }, + [408] = { 0x8, 0x8 }, + [411] = { 0x10, 0x1c }, + [413] = { 0x1048, 0x4 }, + [414] = { 0x50, 0x4 }, + [415] = { 0x4c, 0x4 }, + [451] = { 0x14, 0x14 }, + [452] = { 0x8, 0x4 }, + [454] = { 0x4, 0x4 }, + [552] = { 0x1044, 0x4 }, + [561] = { 0x1044, 0x4 }, + [563] = { 0x4c, 0x4 }, + [565] = { 0x48, 0x4 }, + [567] = { 0x80, 0x4 }, + [572] = { 0x4, 0x4 }, + [574] = { 0x4, 0x8 }, + [575] = { 0x58, 0x4c }, + [576] = { 0x4, 0x0 }, + [586] = { 0x8, 0x4 }, + [587] = { 0x0, 0x0 }, + [588] = { 0x18, 0x0 }, + [589] = { 0xe4, 0xe0 }, + [594] = { 0x730, 0x0 }, + [596] = { 0x14, 0x0 }, + [597] = { 0x4, 0x0 }, + [598] = { 0x4, 0x0 }, + [599] = { 0x4, 0x0 }, + [601] = { 0x0, 0x4 }, + [602] = { 0x0, 0x4 }, + [603] = { 0x0, 0x0 }, +}; + +bool iomfb_v27_callback_size_valid(unsigned int tag, u32 input, u32 output) +{ + if (tag >= ARRAY_SIZE(callback_sizes) || !cb_handlers[tag]) + return false; + return input >= callback_sizes[tag].input && + output >= callback_sizes[tag].output; +} + +void iomfb_start_v27(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_v27.h b/drivers/gpu/drm/apple/iomfb_v27.h new file mode 100644 index 0000000000000..3d08212c87001 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v27.h @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_IOMFB_V27_H__ +#define __APPLE_IOMFB_V27_H__ + +#include "version_utils.h" + +#define DCP_FW v27 +#define DCP_FW_VER DCP_FW_VERSION(27, 0, 0) + +#include "iomfb_template.h" + +static_assert(sizeof(struct dcp_swap_v27) == 0x6e0); +static_assert(sizeof(struct dcp_surface_v27) == 0x22c); +static_assert(sizeof(struct dcp_swap_submit_req_v27) == 0xff4); +static_assert(sizeof(struct dcp_swap_submit_resp_v27) == 0x0c); + +/* 26A428 IOMobileFramebufferAP::swap_submit_dcp_main, ab7fd34..ab7fe3c. */ +static_assert(offsetof(struct dcp_swap_submit_req_v27, surf) == 0x6e0); +static_assert(offsetof(struct dcp_swap_submit_req_v27, surf_iova) == 0xf90); +static_assert(offsetof(struct dcp_swap_submit_req_v27, swap_null) == 0xfea); +static_assert(offsetof(struct dcp_swap_v27, swap_id) == 0x98); +static_assert(offsetof(struct dcp_swap_v27, src_rect) == 0xac); +static_assert(offsetof(struct dcp_swap_v27, dst_rect) == 0x10c); +static_assert(offsetof(struct dcp_swap_v27, swap_enabled) == 0x14c); + +bool iomfb_v27_callback_size_valid(unsigned int tag, u32 input, u32 output); + +#undef DCP_FW_VER +#undef DCP_FW + +#endif /* __APPLE_IOMFB_V27_H__ */ From ce6452088142edfc7d7f7332589320736c3d1fd6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 01:08:05 -0400 Subject: [PATCH 0218/1579] drm/apple: preserve native firmware 27 surface field offsets --- Documentation/gpu/apple-dcp-t6050.rst | 49 +++++++++++++++++++++----- drivers/gpu/drm/apple/iomfb_template.c | 19 ++++++++++ 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/Documentation/gpu/apple-dcp-t6050.rst b/Documentation/gpu/apple-dcp-t6050.rst index 9f618e35ec431..99e83c2f8a3ea 100644 --- a/Documentation/gpu/apple-dcp-t6050.rst +++ b/Documentation/gpu/apple-dcp-t6050.rst @@ -21,9 +21,10 @@ The saved live-27.0 ADT names DCP UUID 64863924-B56B-3E32-87FE-038677F52709. Those UUID bytes occur at offset 205332 in the decompressed 26A428 t605x DCP bundle (SHA-256 d9384573025721074ec1a7190626a8bab36cf61f764055e8e05c7d5d98cb318f). -This identifies the saved boot's firmware, not the current live boot. Before -enabling the candidate, capture the live DCP UUID and initial IOMFB version -word and compare them to the reference. A manually supplied firmware-compat +The 2026-09-28 live ADT independently carries the same UUID (ADT SHA-256 +d3dbf5a4d89fe19dbf98e0c8db6749f8c51d1082ba7a4214688376512eceed20). +This closes the live image identity check for that boot, not presentation +qualification. The initial IOMFB version word still needs runtime checking. A manually supplied firmware-compat property selects the ABI; it does not prove what iBoot loaded. The transport is imported from the Neo H17P series by DjDeveloperr and the @@ -66,6 +67,14 @@ the reply is 0x0c. The old 26.x request was 0xe9c bytes and required a two-byte tail relocation; applying that relocation to 27.0 corrupts its already aligned IOVAs. The 27.0 profile does not perform it. +A408 and A483 both call ``SurfaceContents::from`` at 0xfffffe000abb6a64. +It writes tearing at byte 1, premultiplied at byte 2, plane count at 3 and +pixel format at 0xb. Thus the Neo per-surface two-byte relocation must also +be skipped for 27.0. These offsets, allocation size at 0x29, and the three +pointer-presence fields at 0x51/0x149/0x1ed have compile-time assertions. +``hidePointers`` writes 1 to a non-NULL metadata pointer; these are presence +markers, not copied host pointers. + The native swap_submit_with_tags routine confirms swap ID at 0x98, source rectangles at 0xac, destination rectangles at 0x10c, and enabled surfaces at 0x14c. The D594 deserializer requires 0x730 input bytes, retaining the 26.x @@ -87,8 +96,11 @@ Saved J714s ADT resources, after parent-range translation: * Display DART: 0x48a300000, id 18, scanout SID 0, PIODMA SID 16. * Both DARTs include a secondary aperture at 0x2803f0000 and vm-base 0x10000000000. Existing firmware mappings must survive guest attachment. -* Display reg[4] resolves to 0x286700000, size 0x114000. PMC bandwidth - resources must be derived from the T6050 PMGR descriptions. +* Display reg[4] resolves to 0x286700000, size 0x114000. Native T6050 + ``getBWRMemoryOffset`` returns 0x20000 + agent * 0x4000. The live ADT's + DISP_INT request 0x96 has agent index 0. PMGR group 0x22 selects map + 0x81 plus 0x700000; map 0x81 is ADT PMGR reg[8], 0x286000000. + The resulting DISP_INT bandwidth scratch is 0x286720000, size 8. * Display clock IDs are 361 and 458 in this ADT. The 27.0 driver requires ``apple,clock-ids`` in the same order as ``clocks`` and resolves callbacks against those handles. It does not reuse Neo's IDs 154 and 194. @@ -97,8 +109,27 @@ The guest needs explicit ASC/mailbox/interrupt access, SPTM DART mediation for these identities, and native panel properties. Do not clear or reset firmware-owned display tables merely to make a probe succeed. -The default-surface descriptor values and swap flag policy inherited from -the 26.x driver are not yet qualified on J714. Native 27.0 surface -serialization and panel/PMC property behavior need validation before this -profile is made the default. No J714 DCP pageflip, power transition, +The native default-surface constructor sets the format returned by A036 +(after pixel-definition normalization); A037 supplies compression mode. It +computes allocation size from callback width/height and per-plane layout, +then clamps it to the inherited PurpleGfxMem region. In particular, the +word at 0x29 is allocation size, not an opaque panel constant. For b3a8, +native compression mode 2 normalizes to w40a with 8-byte elements; other +modes use 4-byte and 1-byte planes. Compression modes 2 and 3 have distinct +tile-allocation formulae. The driver still contains the donor's unqualified +default-surface constants: wiring native query results and inherited region +size is required before a presentation test. + +The native AP swap entry copies both flag words from its caller; the A408 +serializer only copies the record. This establishes layout, not that Neo's +0x34865202/4 policy or gap bytes 2/3 are correct for this display. They remain +unqualified pending caller-side or firmware-behavior evidence. + +The current SPTM DART backend uses a low-4-GiB DMA aperture with optional +bit-40 force, disables a SID during attachment, and attaches a fresh root. +DCP's inherited mappings must instead survive attachment. Separate driver +instances also request exclusive ownership of their MMIO resource, so they +cannot all claim the same synthetic doorbell page without a shared-provider +change or distinct emulated aliases. These integration requirements and +panel/provider behavior must be resolved before enabling the profile. No J714 DCP pageflip, power transition, backlight, or external-output result is claimed by this port. diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 69171ed13b037..15bdd7fb1e08e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -44,6 +44,18 @@ static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), timestamp[6]) == 0x30); static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), flags1) == 0x40); +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) +/* 26A428 SurfaceContents::from() and hidePointers(), shared by A408/A483. */ +static_assert(offsetof(struct dcp_surface, is_tearing_allowed) == 0x1); +static_assert(offsetof(struct dcp_surface, is_premultiplied) == 0x2); +static_assert(offsetof(struct dcp_surface, plane_cnt) == 0x3); +static_assert(offsetof(struct dcp_surface, format) == 0xb); +static_assert(offsetof(struct dcp_surface, buf_size) == 0x29); +static_assert(offsetof(struct dcp_surface, has_comp) == 0x51); +static_assert(offsetof(struct dcp_surface, has_planes) == 0x149); +static_assert(offsetof(struct dcp_surface, has_compr_info) == 0x1ed); +#endif + struct dcp_wait_cookie { struct kref refcount; struct completion done; @@ -1341,7 +1353,9 @@ static void release_swap_cookie(struct kref *ref) static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) { struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); +#if DCP_FW_VER < DCP_FW_VERSION(27, 0, 0) unsigned int i; +#endif /* * The swap's own flag words. Our driver never sets either; macOS's @@ -1359,6 +1373,10 @@ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) req->swap.h17p_pre_swap_id[0x00] = 2; req->swap.h17p_pre_swap_id[0x28] = 3; +#if DCP_FW_VER < DCP_FW_VERSION(27, 0, 0) + /* 27.0 keeps both bytes: SurfaceContents::from() writes tearing at +1 + * and premultiplied at +2, including in swap_submit_main(). + */ for (i = 0; i < SWAP_SURFACES; i++) { u8 *p = (u8 *)&req->surf[i]; size_t n = sizeof(req->surf[i]); @@ -1366,6 +1384,7 @@ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) memmove(p + 1, p + 3, n - 3); memset(p + n - 2, 0, 2); } +#endif /* * Our trailing null-flag block lands two bytes late. In the capture From 13cb0d77dbb42b8b43319b595f630526a667e03c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 01:15:58 -0400 Subject: [PATCH 0219/1579] drm/apple: construct firmware 27 default surfaces from native layout replies --- Documentation/gpu/apple-dcp-t6050.rst | 14 +++- drivers/gpu/drm/apple/dcp-internal.h | 3 + drivers/gpu/drm/apple/dcp.c | 19 +++++ drivers/gpu/drm/apple/iomfb-dfb.h | 112 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 63 +++++++++++++- 5 files changed, 205 insertions(+), 6 deletions(-) create mode 100644 drivers/gpu/drm/apple/iomfb-dfb.h diff --git a/Documentation/gpu/apple-dcp-t6050.rst b/Documentation/gpu/apple-dcp-t6050.rst index 99e83c2f8a3ea..10f3835ef9d4a 100644 --- a/Documentation/gpu/apple-dcp-t6050.rst +++ b/Documentation/gpu/apple-dcp-t6050.rst @@ -116,9 +116,17 @@ then clamps it to the inherited PurpleGfxMem region. In particular, the word at 0x29 is allocation size, not an opaque panel constant. For b3a8, native compression mode 2 normalizes to w40a with 8-byte elements; other modes use 4-byte and 1-byte planes. Compression modes 2 and 3 have distinct -tile-allocation formulae. The driver still contains the donor's unqualified -default-surface constants: wiring native query results and inherited region -size is required before a presentation test. +tile-allocation formulae. The 27.0 driver caches A036/A037 replies from the existing boot chain and +uses D586's width/height. It requires a ``memory-region`` named +``framebuffer`` for the inherited allocation limit. The empty IOSurface +constructor initializes 1-byte elements of size 1x1 and no plane geometry; +setPixelFormat and allocate_carveout only change its format/allocation. +The driver matches that initial descriptor and returns the A483 boolean to +D586, rather than reporting unconditional success. b3a8, w40a, BGRA, w30r +and l10r are supported with compression 0, 2 or 3; other values are rejected +explicitly. Its integer-only calculation reproduces native binary32 size +rounding without using floating point in the kernel. Native arithmetic and +layout checks pass; runtime query values/presentation are not tested. The native AP swap entry copies both flag words from its caller; the A408 serializer only copies the record. This establishes layout, not that Neo's diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 0a2ad40d72848..15e7d99d2d6ed 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -238,6 +238,9 @@ struct apple_dcp { struct clk_bulk_data *firmware_clocks; u32 *firmware_clock_ids; unsigned int num_firmware_clocks; + u32 dfb_format, dfb_compression; + u64 dfb_region_size; + bool dfb_format_valid, dfb_compression_valid; /* DCP shared memory */ void *shmem; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 9a59e994357b8..c71c9bbc599ea 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2709,6 +2709,25 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) { int count = of_property_count_u32_elems(dev->of_node, "apple,clock-ids"); + int index = of_property_match_string(dev->of_node, "memory-region-names", + "framebuffer"); + struct device_node *fb; + struct resource res; + + /* D586 mirrors native allocation inside PurpleGfxMem. The loader + * supplies the inherited region; do not assume another panel's size. + */ + if (index < 0) + return dev_err_probe(dev, -EINVAL, "missing framebuffer memory-region\n"); + fb = of_parse_phandle(dev->of_node, "memory-region", index); + if (!fb) + return -EINVAL; + ret = of_address_to_resource(fb, 0, &res); + of_node_put(fb); + if (ret) + return ret; + dcp->dfb_region_size = resource_size(&res); + ret = devm_clk_bulk_get_all(dev, &dcp->firmware_clocks); if (ret < 0) diff --git a/drivers/gpu/drm/apple/iomfb-dfb.h b/drivers/gpu/drm/apple/iomfb-dfb.h new file mode 100644 index 0000000000000..ee42867092c2e --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-dfb.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __APPLE_IOMFB_DFB_H__ +#define __APPLE_IOMFB_DFB_H__ + +#include +#include +#include +#include +#include + +/* Native 26A428 allocate_carveout() converts its size through float32. + * Reproduce integer -> binary32 -> integer rounding without kernel FP. + */ +static inline u64 apple_dcp_dfb_round_size(u64 size) +{ + unsigned int bits = fls64(size); + u64 unit, half, tail; + + if (bits <= 24) + return size; + unit = 1ULL << (bits - 24); + half = unit >> 1; + tail = size & (unit - 1); + size &= ~(unit - 1); + if (tail > half || (tail == half && (size & unit))) + size += unit; + return size; +} + +/* Reconstructed from 26A428 compressed_fb_size(), not a panel capture. + * tile columns/rows use power-of-two metadata coverage in both formats. + */ +static inline u64 apple_dcp_dfb_plane_size(u32 width, u32 height, u32 bpe, + u32 compression) +{ + u64 stride, data, cols, rows, tw = 16, th = 16; + unsigned int order; + + if (!compression) { + stride = ALIGN((u64)width * bpe, 128); + return ALIGN(apple_dcp_dfb_round_size(stride * height), 0x4000); + } + if (compression == 2 && bpe > 4) { + th = 8; + if (bpe == 16) + tw = 8; + } + cols = DIV_ROUND_UP(width, tw); + rows = DIV_ROUND_UP(height, th); + if (compression == 3) { + stride = tw * th * cols * bpe; + data = ALIGN(rows * stride, 128); + return data + 8 * roundup_pow_of_two(cols) * roundup_pow_of_two(rows); + } + order = ilog2(0x4000 / bpe); + stride = ALIGN((u64)width * bpe, (u64)bpe << (order / 2)); + data = ALIGN(ALIGN((u64)height, 1ULL << ((order + 1) / 2)) * stride, 128); + return data + ALIGN(2 * roundup_pow_of_two(cols) * roundup_pow_of_two(rows), 128); +} + +/* setup_default_fb_info() normalizes b3a8 + compression2 to w40a. + * These are the RGB layouts supported here; unknown formats are rejected + * instead of reusing another panel's allocation or truncating its size. + */ +static inline int apple_dcp_dfb_allocation(u32 width, u32 height, u32 format, + u32 compression, u64 region_size, + u32 *surface_format, u32 *allocation) +{ + u32 bpe; + u64 size; + bool alpha_plane = false; + + if (!width || !height || width > 16384 || height > 16384 || !region_size) + return -EINVAL; + if (compression != 0 && compression != 2 && compression != 3) + return -EOPNOTSUPP; + switch (format) { + case 0x62336138: /* b3a8 */ + if (compression == 2) { + format = 0x77343061; /* w40a */ + bpe = 8; + } else { + bpe = 4; + alpha_plane = true; + } + break; + case 0x77343061: /* w40a */ + bpe = 8; + break; + case 0x42475241: /* BGRA */ + case 0x77333072: /* w30r */ + case 0x6c313072: /* l10r */ + bpe = 4; + break; + default: + return -EOPNOTSUPP; + } + size = apple_dcp_dfb_plane_size(width, height, bpe, compression); + if (alpha_plane) + size += apple_dcp_dfb_plane_size(width, height, 1, compression); + if (size > U32_MAX) + return -EOVERFLOW; + if (compression) + size = ALIGN(apple_dcp_dfb_round_size(size), 0x4000); + size = min(size, region_size); + if (!size || size > U32_MAX) + return -EOVERFLOW; + *surface_format = format; + *allocation = size; + return 0; +} +#endif diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 15bdd7fb1e08e..9f178703e2ebe 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -34,6 +34,7 @@ #include "dcp-internal.h" #include "iomfb.h" #include "iomfb_internal.h" +#include "iomfb-dfb.h" #include "parser.h" #include "trace.h" #include "version_utils.h" @@ -376,15 +377,16 @@ DCP_THUNK_INOUT(dcp_register_dfb_surface, dcpep_register_dfb_surface, /* Fill the common SurfaceContents fields, matching the traced macOS bytes. */ -static void dcp_ack_dfb_surface(struct apple_dcp *dcp) +static void dcp_ack_dfb_surface(struct apple_dcp *dcp, bool success) { struct dcp_channel *ch = &dcp->ch_cb; u8 *succ = ch->output[ch->depth - 1]; - *succ = true; + *succ = success; dcp_ack(dcp, DCP_CONTEXT_CB); } +#if DCP_FW_VER < DCP_FW_VERSION(27, 0, 0) static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) { s[0x002] = 0x01; @@ -413,6 +415,7 @@ static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) s[0x035] = 2; } } +#endif /* * A471 (update_dfb) is never sent. It hands the firmware a default-framebuffer @@ -424,7 +427,12 @@ static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) */ static void complete_dfb_surface(struct apple_dcp *dcp, void *out, void *cookie) { - dcp_ack_dfb_surface(dcp); +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + /* update_dfb() returns bool, which native forwards to D586. */ + dcp_ack_dfb_surface(dcp, *(u8 *)out & 1); +#else + dcp_ack_dfb_surface(dcp, true); +#endif } static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, @@ -448,7 +456,44 @@ static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, return true; } +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + { + struct dcp_dfb_dimensions_h17p dim; + struct dcp_surface *surface = (void *)req->surface; + u32 format, allocation; + int ret = -EINVAL; + + memcpy(&dim, in, sizeof(dim)); + if (dcp->dfb_format_valid && dcp->dfb_compression_valid) + ret = apple_dcp_dfb_allocation(dim.width, dim.height, + dcp->dfb_format, dcp->dfb_compression, + dcp->dfb_region_size, &format, &allocation); + if (ret) { + dev_err(dcp->dev, "unsupported DFB layout %ux%u format=%#x compression=%u (%d)\n", + dim.width, dim.height, dcp->dfb_format, + dcp->dfb_compression, ret); + *(u8 *)out = false; + kfree(req); + return true; + } + /* IOSurface::init() with no geometry: element dimensions default + * to1; plane count/stride/width/height remain0. setPixelFormat() + * and allocate_carveout() only set format and allocation size. + * hidePointers() turns the two metadata pointers into presence1. + */ + surface->is_premultiplied = 1; + surface->format = format; + surface->pix_size = 1; + surface->pel_w = 1; + surface->pel_h = 1; + surface->buf_size = allocation; + surface->surface_id = DCP_SWAP_SURFACE_ID_BASE - 1; + surface->has_comp = 1; + surface->has_planes = 1; + } +#else dcp_fill_dfb_surface(dcp, req->surface); +#endif dcp_register_dfb_surface(dcp, false, req, complete_dfb_surface, NULL); kfree(req); @@ -1144,6 +1189,12 @@ static void boot_1_8(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_dfb_layout_req_h17p req = {}; +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + /* A036 returns format at0 and success at0x28. */ + dcp->dfb_format_valid = *((u8 *)out + 0x28) & 1; + memcpy(&dcp->dfb_format, out, sizeof(dcp->dfb_format)); +#endif + dcp_get_dfb_layout(dcp, false, &req, boot_1_9, NULL); } @@ -1151,6 +1202,12 @@ static void boot_1_7(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_dfb_info_h17p info = {}; +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + /* A037 returns compression at0 and success at4. */ + dcp->dfb_compression_valid = *((u8 *)out + 4) & 1; + memcpy(&dcp->dfb_compression, out, sizeof(dcp->dfb_compression)); +#endif + dcp_get_dfb_info(dcp, false, &info, boot_1_8, NULL); } From b66a8567505d88fc19bcb3ecfbd177fe92e83785 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 01:30:56 -0400 Subject: [PATCH 0220/1579] drm/apple: distinguish guest resources from firmware physical registers --- Documentation/gpu/apple-dcp-t6050.rst | 19 +++++++++++++ drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 15 ++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 39 +++++++++++++++++--------- 4 files changed, 61 insertions(+), 13 deletions(-) diff --git a/Documentation/gpu/apple-dcp-t6050.rst b/Documentation/gpu/apple-dcp-t6050.rst index 10f3835ef9d4a..ded4e878af190 100644 --- a/Documentation/gpu/apple-dcp-t6050.rst +++ b/Documentation/gpu/apple-dcp-t6050.rst @@ -141,3 +141,22 @@ cannot all claim the same synthetic doorbell page without a shared-provider change or distinct emulated aliases. These integration requirements and panel/provider behavior must be resolved before enabling the profile. No J714 DCP pageflip, power transition, backlight, or external-output result is claimed by this port. + +Firmware physical addresses in a guest +------------------------------------- + +``reg`` describes Linux CPU resources and therefore contains guest IPAs. +The DCP coprocessor still sees host physical MMIO. An optional +``apple,firmware-physical-regs`` array supplies one u64 host base per +``disp-N`` resource, followed by the appended bandwidth scratch/doorbell +resources in driver index order. Its length must equal the resource count. +The disabled T6050 fixture supplies six entries; its last two both describe +PMC at 0x286700000. Without the property, bases default to resource start, +preserving native behavior. + +Register/BW replies use these firmware addresses. A firmware map_physical +request is translated in reverse to the corresponding CPU resource before +calling dma_map_resource. The HV/IOMMU resource-DMA path must also translate +that IPA to host MMIO before forwarding to SPTM; its current RAM-only map +translation is not sufficient. This source change does not implement that +host-side operation or establish firmware DMA success. diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 15e7d99d2d6ed..5b288357da947 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -252,6 +252,7 @@ struct apple_dcp { /* Display registers mappable to the DCP */ struct resource *disp_registers[MAX_DISP_REGISTERS]; + u64 disp_firmware_base[MAX_DISP_REGISTERS]; unsigned int nr_disp_registers; struct resource disp_bw_scratch_res; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c71c9bbc599ea..28a14cf27500f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2331,6 +2331,21 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) } dcp->nr_disp_registers = count; + for (i = 0; i < count; i++) + dcp->disp_firmware_base[i] = dcp->disp_registers[i]->start; + /* A guest resource is an IPA. A live coprocessor still sees host + * physical MMIO addresses, so its register callbacks need both views. + */ + if (of_property_present(dcp->dev->of_node, "apple,firmware-physical-regs")) { + if (of_property_count_u64_elems(dcp->dev->of_node, + "apple,firmware-physical-regs") != count) + return -EINVAL; + ret = of_property_read_u64_array(dcp->dev->of_node, + "apple,firmware-physical-regs", + dcp->disp_firmware_base, count); + if (ret) + return ret; + } return 0; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 9f178703e2ebe..a93c6f816b6dc 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -811,18 +811,30 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) return 1; } -/* Validate that the specified region is a display register */ -static bool is_disp_register(struct apple_dcp *dcp, u64 start, u64 end) +/* Translate a firmware-visible MMIO address back to the guest CPU resource. + * Both views are identical on bare metal; the hypervisor supplies the map + * when Linux resources are translated IPAs. + */ +static bool dcp_firmware_reg_to_resource(struct apple_dcp *dcp, u64 address, + u64 size, phys_addr_t *resource) { - int i; + unsigned int i; - for (i = 0; i < dcp->nr_disp_registers; ++i) { + if (!size) + return false; + for (i = 0; i < dcp->nr_disp_registers; i++) { struct resource *r = dcp->disp_registers[i]; + u64 base = dcp->disp_firmware_base[i]; + u64 offset; - if ((start >= r->start) && (end <= r->end)) + if (address < base) + continue; + offset = address - base; + if (offset < resource_size(r) && size <= resource_size(r) - offset) { + *resource = r->start + offset; return true; + } } - return false; } @@ -837,9 +849,10 @@ dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) { int size = ALIGN(req->size, 4096); dma_addr_t dva; + phys_addr_t resource; u32 id; - if (!is_disp_register(dcp, req->paddr, req->paddr + size - 1)) { + if (!dcp_firmware_reg_to_resource(dcp, req->paddr, size, &resource)) { dev_err(dcp->dev, "refusing to map phys address %llx size %llx\n", req->paddr, req->size); return (struct dcp_map_physical_resp){}; @@ -848,9 +861,9 @@ dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); set_bit(id, dcp->memdesc_map); dcp->memdesc[id].size = size; - dcp->memdesc[id].reg = req->paddr; + dcp->memdesc[id].reg = resource; - dva = dma_map_resource(dcp->dev, req->paddr, size, DMA_BIDIRECTIONAL, 0); + dva = dma_map_resource(dcp->dev, resource, size, DMA_BIDIRECTIONAL, 0); WARN_ON(dva == DMA_MAPPING_ERROR); return (struct dcp_map_physical_resp){ @@ -906,7 +919,7 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d * dma_map_resource() them instead fails and hands the firmware * DMA_MAPPING_ERROR as a device address. */ - dma_addr_t dva = rsrc->start; + dma_addr_t dva = dcp->disp_firmware_base[req->index]; #elif DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) dma_addr_t dva = dma_map_resource(dcp->dev, rsrc->start, resource_size(rsrc), DMA_BIDIRECTIONAL, 0); @@ -920,7 +933,7 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d #endif return (struct DCP_FW_NAME(dcp_map_reg_resp)){ - .addr = rsrc->start, + .addr = dcp->disp_firmware_base[req->index], .length = resource_size(rsrc), #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) .dva = dva, @@ -1308,12 +1321,12 @@ static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) if (dcp->disp_bw_scratch_index) { u32 offset = dcp->disp_bw_scratch_offset; u32 index = dcp->disp_bw_scratch_index; - rt_bw.reg_scratch = dcp->disp_registers[index]->start + offset; + rt_bw.reg_scratch = dcp->disp_firmware_base[index] + offset; } if (dcp->disp_bw_doorbell_index) { u32 index = dcp->disp_bw_doorbell_index; - rt_bw.reg_doorbell = dcp->disp_registers[index]->start; + rt_bw.reg_doorbell = dcp->disp_firmware_base[index]; rt_bw.doorbell_bit = REG_DOORBELL_BIT(dcp->index); /* * This is most certainly not padding. t8103-dcp crashes without From 6de3535b639b697f4ed3033c147bc8295b8b6339 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 01:34:22 -0400 Subject: [PATCH 0221/1579] drm/apple: leave firmware 27 debug and shared-event fields cleared --- Documentation/gpu/apple-dcp-t6050.rst | 22 ++++++++++++++++++---- drivers/gpu/drm/apple/iomfb_template.c | 13 ++++++++++++- drivers/gpu/drm/apple/iomfb_template.h | 9 +++++++++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/Documentation/gpu/apple-dcp-t6050.rst b/Documentation/gpu/apple-dcp-t6050.rst index ded4e878af190..3c0f36902c9a6 100644 --- a/Documentation/gpu/apple-dcp-t6050.rst +++ b/Documentation/gpu/apple-dcp-t6050.rst @@ -128,10 +128,24 @@ explicitly. Its integer-only calculation reproduces native binary32 size rounding without using floating point in the kernel. Native arithmetic and layout checks pass; runtime query values/presentation are not tested. -The native AP swap entry copies both flag words from its caller; the A408 -serializer only copies the record. This establishes layout, not that Neo's -0x34865202/4 policy or gap bytes 2/3 are correct for this display. They remain -unqualified pending caller-side or firmware-behavior evidence. +The matching 26A428 Recovery image's IOMobileFramebuffer library resolves +the previously opaque swap fields. Its _kern_SwapBegin clears the 0x6e0-byte +record at context+0x18. _kern_SwapDebugInfo at 0x18e0c365c stores its two u64 +arguments at context+0x58: record offsets 0x40/0x48 are debug data, not mode +flags. _kern_SwapSetEventWait writes per-layer u64 values at record+0x50; +_kern_SwapSetEventSignalOnGlass writes at 0x70; _kern_SwapSetEventSignal +writes per-layer values at 0x78. Corresponding shared-event port handles +are supplied separately by these setters. + +The 27.0 structure names these fields and asserts their offsets. This +Linux driver supplies no IOSurface shared-event handles, so it preserves +the cleared native defaults instead of replaying Neo's captured debug +words 0x34865202/4 and event values 2/3. Legacy 26.x behavior is unchanged. +The reference library text SHA-256 is +86598dec087d1a8919da72a545fafe4e084b50944385a909c7e02723413a4343. +Executing its standalone debug setter against a private user buffer and +comparing compiled structure offsets confirms this layout; it is not a +hardware pageflip test. The current SPTM DART backend uses a low-4-GiB DMA aperture with optional bit-40 force, disables a SID during attachment, and attaches a fresh root. diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index a93c6f816b6dc..667f199ac8f57 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -43,7 +43,14 @@ #define REG_DOORBELL_BIT(idx) (2 + (idx)) static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), timestamp[6]) == 0x30); +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) +static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), debug_info) == 0x40); +static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), event_wait) == 0x50); +static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), event_signal_on_glass) == 0x70); +static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), event_signal) == 0x78); +#else static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), flags1) == 0x40); +#endif #if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) /* 26A428 SurfaceContents::from() and hidePointers(), shared by A408/A483. */ @@ -1406,7 +1413,7 @@ static void release_swap_cookie(struct kref *ref) kfree(cookie); } -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) && DCP_FW_VER < DCP_FW_VERSION(27, 0, 0) /* * H17P's swap surface has no is_tearing_allowed and no is_premultiplied: * plane_cnt sits at +1, plane_cnt2 at +5 and format at +9, two bytes earlier @@ -1490,6 +1497,10 @@ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) #endif } #else +/* 27.0 begins with a cleared native swap record. Its +0x40/+0x48 are + * optional debug words, and +0x50/+0x78 are shared-event wait/signal values, + * not display policy bits/counters. Do not replay another client's values. + */ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) { } #endif diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index fcf60d1cbc526..2b2247fb0c462 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -22,6 +22,14 @@ struct DCP_FW_NAME(dcp_swap) { u64 timestamp[7]; u64 unk_38; +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + /* _kern_SwapDebugInfo in the matching26A428 userspace library. */ + u64 debug_info[2]; + /* IOSurface shared-event values; Linux supplies no event handles. */ + u64 event_wait[SWAP_SURFACES]; + u64 event_signal_on_glass; + u64 event_signal[SWAP_SURFACES]; +#else u64 flags1; u64 flags2; @@ -37,6 +45,7 @@ struct DCP_FW_NAME(dcp_swap) { * (at +0x50 and +0x78) are small counters we do not model yet. */ u8 h17p_pre_swap_id[0x48]; +#endif #endif u32 swap_id; From 15caf8b4ecadb62d8e020f3758ce4b7ce0157c95 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 13:03:35 -0400 Subject: [PATCH 0222/1579] drm/apple: drive retained T6050 DCP on macOS 27 firmware Use the native firmware CPU DMA aperture, provider clock-index semantics, and 26A428 buffer and callback layouts. Adopt the inherited live RTKit session and preserve its DART mappings. Use the Apple24MHz counter for Mach-time swap timestamps while the EL1 architectural timer runs at1GHz. Drive real timing-mode selection, distinguish a panel from its optional physical dimensions, and implement the native27 brightness swap fields. Forward the ADT user brightness range and decode compact EPIC powerlog feedback. Acknowledge the specific no-pending DFB resize callback during display power-off. Reselect timing when VRR_ENABLED changes. Validation: focused QEMU module/boot ABI gates; hardware3024x1964 fixed120 page flips and GNOME scanout; user-confirmed clean output and brightness slider/keys; exact40/240/140-nit firmware readback; three display power cycles with completed swaps and restored brightness. Native structure and buffer-lifecycle checks also pass. The SSD default stays unchanged. The existing authored tb VRR implementation is retained. The current M5 paced test does not distinguish VRR off/on behavior; system deep sleep, HDMI and automatic persistent boot integration are not qualified here. Signed-off-by: DjDeveloperr --- Documentation/gpu/apple-dcp-t6050.rst | 83 ++++++++--- drivers/gpu/drm/apple/afk.c | 5 +- drivers/gpu/drm/apple/dcp-internal.h | 4 + drivers/gpu/drm/apple/dcp.c | 32 +++- drivers/gpu/drm/apple/dcp_backlight.c | 10 +- drivers/gpu/drm/apple/iomfb.c | 7 +- drivers/gpu/drm/apple/iomfb_template.c | 76 ++++++++-- drivers/gpu/drm/apple/iomfb_template.h | 12 +- drivers/gpu/drm/apple/iomfb_v27.c | 198 ++++++++++++++++++++----- drivers/gpu/drm/apple/systemep.c | 2 +- 10 files changed, 338 insertions(+), 91 deletions(-) diff --git a/Documentation/gpu/apple-dcp-t6050.rst b/Documentation/gpu/apple-dcp-t6050.rst index 3c0f36902c9a6..e1db6d5064059 100644 --- a/Documentation/gpu/apple-dcp-t6050.rst +++ b/Documentation/gpu/apple-dcp-t6050.rst @@ -1,10 +1,11 @@ T6050 display firmware 27.0 ABI ============================== -This is an explicit firmware ABI port, not a hardware qualification. The -T6050 node must name ``apple,t6050-dcp`` and carry +This profile has driven the J714s internal panel on hardware at fixed +3024x1964/120 Hz, including GNOME and its framebuffer-console handoff. +The T6050 node must name ``apple,t6050-dcp`` and carry ``apple,firmware-compat = <27 0 0>``. The stable J714 guest DT is unchanged, -so simpledrm remains the display driver until a separate DCP test DT is used. +so the qualified display path currently uses a separate RAM-boot DCP DT. Do not advertise another 27.x release as compatible without comparing it. Reference @@ -23,8 +24,7 @@ The saved live-27.0 ADT names DCP UUID d9384573025721074ec1a7190626a8bab36cf61f764055e8e05c7d5d98cb318f). The 2026-09-28 live ADT independently carries the same UUID (ADT SHA-256 d3dbf5a4d89fe19dbf98e0c8db6749f8c51d1082ba7a4214688376512eceed20). -This closes the live image identity check for that boot, not presentation -qualification. The initial IOMFB version word still needs runtime checking. A manually supplied firmware-compat +This establishes the live image identity used for the hardware tests. The initial IOMFB version word still needs runtime checking. A manually supplied firmware-compat property selects the ABI; it does not prove what iBoot loaded. The transport is imported from the Neo H17P series by DjDeveloperr and the @@ -86,8 +86,8 @@ The 27.0 callback dispatcher checks native minimum wire lengths before calling typed handlers. Provider property replies use the matching device's raw property bytes, rather than Neo-specific panel power tables. -Hardware integration still required ------------------------------------ +Hardware integration +-------------------- Saved J714s ADT resources, after parent-range translation: @@ -126,7 +126,8 @@ D586, rather than reporting unconditional success. b3a8, w40a, BGRA, w30r and l10r are supported with compression 0, 2 or 3; other values are rejected explicitly. Its integer-only calculation reproduces native binary32 size rounding without using floating point in the kernel. Native arithmetic and -layout checks pass; runtime query values/presentation are not tested. +layout checks pass, and the resulting descriptor has been used during +successful hardware initialization and scanout. The matching 26A428 Recovery image's IOMobileFramebuffer library resolves the previously opaque swap fields. Its _kern_SwapBegin clears the 0x6e0-byte @@ -147,14 +148,55 @@ Executing its standalone debug setter against a private user buffer and comparing compiled structure offsets confirms this layout; it is not a hardware pageflip test. -The current SPTM DART backend uses a low-4-GiB DMA aperture with optional -bit-40 force, disables a SID during attachment, and attaches a fresh root. -DCP's inherited mappings must instead survive attachment. Separate driver -instances also request exclusive ownership of their MMIO resource, so they -cannot all claim the same synthetic doorbell page without a shared-provider -change or distinct emulated aliases. These integration requirements and -panel/provider behavior must be resolved before enabling the profile. No J714 DCP pageflip, power transition, -backlight, or external-output result is claimed by this port. +The SPTM DART backend adopts the existing firmware roots through the native +INIT/POWERUP protocol. DCP firmware CPU allocations use its native +[1 TiB, 2 TiB) aperture, while scanout and PIODMA keep independent 42-bit +apertures. Guest aliases cover only the ADT shared firmware segments and +retained VRAM; private executable firmware is not mapped into Linux. + +The display-subsystem ``memory-region`` must describe the CPU aperture used +by simpledrm. DCP's own ``memory-region`` separately describes the complete +retained VRAM allocation. These can be different aliases of the same host +memory; conflating them prevents native DRM from removing simpledrm. + +Timestamps, brightness and display power +--------------------------------------- + +The native 26A428 Mach time functions use AGTCNTVCTSS (S3_4_C15_C10_6). +On this hardware it advances at 24 MHz while the Linux architectural timer +advances at 1 GHz. The 27.0 swap timestamps therefore read the Apple counter +directly; using architectural ticks stalls timestamped swaps. Hardware +measurements gave 120.00 completed flips/second after this correction. + +``_kern_SwapSetBrightness`` stores a binary64 nits value at swap+0x35e and +sets the update byte at +0x354. The old packed-DAC fields occupy different +offsets and are not used on this profile. Integer encoding avoids kernel +floating point. Runtime property20 is IOMFBBrightnessLevel in16.16 nits; +property15 is overrideBacklight, unlike the older ABI. The panel's ordinary +600-nit user limit is forwarded from the ADT's backlight node. The user has +confirmed that both GNOME's slider and keyboard brightness controls change +the panel visibly. Brightness feedback arrives as powerlog-service mNits +reports on system endpoint channel3, using the compact EPIC header also +found on H17P. The retained ring captured requests40/240/140 nits and matching +40000/240000/140000 mNits reports. After enabling the compact decoder, +Linux actual_brightness read back40/240/140 for the same sequence. + +D592 calls native ``resize_default_fb_surface_gated``. This applies a resize +queued by the AP; Linux handles explicit D586 geometry synchronously and +has no pending resize to apply. Its acknowledgement lets the normal power- +off call return. Three five-second DPMS cycles completed the firmware power +off/on sequence and restored successful scanout. This is display power +control, not system suspend or deep sleep. Firmware feedback returned to +the requested140nits after the latest cycle; optical off/on confirmation +was not obtained. + +VRR range advertisement alone is not hardware qualification. DCP latches +adaptive-sync parameters when reselecting the mode, so changing VRR_ENABLED +must run the modeset path. The authored VRR implementation from aurora-silicon/linux@tb, commit +38c14f610, is already in this branch. On the current27.0 candidate, paced +flip intervals remain similar with VRR_ENABLED off/on; this does not yet +qualify the toggle or a fixed-versus-variable panel refresh distinction. +System sleep and external HDMI also remain under investigation. Firmware physical addresses in a guest ------------------------------------- @@ -164,13 +206,12 @@ The DCP coprocessor still sees host physical MMIO. An optional ``apple,firmware-physical-regs`` array supplies one u64 host base per ``disp-N`` resource, followed by the appended bandwidth scratch/doorbell resources in driver index order. Its length must equal the resource count. -The disabled T6050 fixture supplies six entries; its last two both describe +The T6050 test fixture supplies six entries; its last two both describe PMC at 0x286700000. Without the property, bases default to resource start, preserving native behavior. Register/BW replies use these firmware addresses. A firmware map_physical request is translated in reverse to the corresponding CPU resource before -calling dma_map_resource. The HV/IOMMU resource-DMA path must also translate -that IPA to host MMIO before forwarding to SPTM; its current RAM-only map -translation is not sufficient. This source change does not implement that -host-side operation or establish firmware DMA success. +calling dma_map_resource. The HV/IOMMU resource-DMA path must preserve that distinction when forwarding +to SPTM. Successful internal scanout does not qualify arbitrary additional +MMIO ranges or external display pipelines. diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 144fdcf8bbd91..8b8eee0530562 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -558,7 +558,8 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, u8 *payload; size_t payload_size; - if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P || + ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) { const struct epic_sub_hdr_h17p *c = (const struct epic_sub_hdr_h17p *)(data + sizeof(*ehdr)); @@ -575,6 +576,8 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, * mNits/uAmps/iDAC backlight reports on channel 3. Taking the * queue channel at face value binds "system" to 0 and makes * every later announce and report look like a duplicate. + * The retained26A428 system endpoint uses this same layout: + * channel1 announces system, channel3 publishes mNits/iDAC. */ channel = le16_to_cpup((__le16 *)(data + 2)); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 5b288357da947..34be5c91b98c6 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -171,6 +171,8 @@ enum dcp_iomfb_method_profile { }; struct apple_dcp_hw_data { + /* Firmware CPU access may cover less than the display DART aperture. */ + u8 firmware_dma_bits; u32 num_dptx_ports; enum dcp_iomfb_method_profile iomfb_method_profile; bool adopt_running_coprocessor; @@ -180,6 +182,8 @@ struct apple_dcp_hw_data { * the bootloader already consumed endpoint discovery. */ bool adopt_live_session; + /* The inherited firmware can renegotiate without stopping its ASC. */ + bool discover_live_session; enum dcp_firmware_version firmware_compat; /* T8140's legacy DT omits the native H17 PMC window. */ bool legacy_pmc_binding; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 28a14cf27500f..3b62ac59156ea 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1074,6 +1074,11 @@ static bool live_discovery; module_param(live_discovery, bool, 0400); MODULE_PARM_DESC(live_discovery, "Negotiate RTKit on the inherited live DCP without an ASC reset"); +static bool dcp_discover_live_session(const struct apple_dcp *dcp) +{ + return dcp->hw.discover_live_session || live_discovery; +} + /* copied and simplified from drm_vblank.c */ static void send_vblank_event(struct drm_device *dev, struct drm_pending_vblank_event *e, @@ -1208,7 +1213,8 @@ void dcp_set_dimensions(struct apple_dcp *dcp) bool dcp_has_panel(struct apple_dcp *dcp) { - return dcp->panel.width_mm > 0; + return dcp->fixed_connector_type == DRM_MODE_CONNECTOR_eDP || + dcp->panel.width_mm > 0; } int dcp_set_crc(struct drm_crtc *crtc, bool enabled) @@ -1444,7 +1450,7 @@ int dcp_start_rtkit_endpoint(struct apple_dcp *dcp, u8 endpoint) { u64 message; - if (!dcp->hw.adopt_live_session || live_discovery) + if (!dcp->hw.adopt_live_session || dcp_discover_live_session(dcp)) return apple_rtkit_start_ep(dcp->rtk, endpoint); if (endpoint < 0x20 || endpoint >= 0x40) return -EINVAL; @@ -1469,6 +1475,7 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state) struct platform_device *pdev = to_apple_crtc(crtc)->dcp; struct apple_dcp *dcp = platform_get_drvdata(pdev); struct drm_crtc_state *crtc_state; + struct drm_crtc_state *old_crtc_state; bool needs_modeset; if (dcp->crashed) { @@ -1479,6 +1486,12 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state) } crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc); + /* DCP latches adaptive-sync parameters when selecting the timing mode. + * A VRR-only property change must therefore run the modeset path too. + */ + if (old_crtc_state->vrr_enabled != crtc_state->vrr_enabled) + crtc_state->mode_changed = true; needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; if (!needs_modeset && (!dcp->connector || !dcp->connector->connected)) { @@ -2642,7 +2655,11 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) * an axi_rd_err NMI. The bootloader's translations are left alone. */ - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + /* The firmware CPU and the display engines have different address + * apertures. Keep scanout and PIODMA at their independent42bit masks. + */ + ret = dma_set_mask_and_coherent(dev, + DMA_BIT_MASK(dcp->hw.firmware_dma_bits ?: 42)); if (ret) return ret; @@ -2807,7 +2824,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) } else { if (start_mode == APPLE_DCP_COPROCESSOR_QUIESCED_RESTART) dev_info(dev, "restarting loader-quiesced DCP session\n"); - if (dcp->hw.adopt_live_session && !live_discovery) { + if (dcp->hw.adopt_live_session && !dcp_discover_live_session(dcp)) { dcp->adopting_session = true; dcp->rtk = devm_apple_rtkit_init_adopted(dev, dcp, "mbox", 0, @@ -2822,7 +2839,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) "Failed to initialize RTKit\n"); } - if (dcp->hw.adopt_live_session && live_discovery) { + if (dcp->hw.adopt_live_session && dcp_discover_live_session(dcp)) { /* No CPU-control, power-domain or DART writes on this path. The * native trace receives HELLO in response to this standard INIT. * A timeout is evidence to retain; do not fall back to a reset. @@ -3271,10 +3288,15 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8142 = { * The loader must preserve the firmware mappings and running ASC generation. */ static const struct apple_dcp_hw_data apple_dcp_hw_t6050 = { + /* 26A428 iBoot config0x02000c00 bounds the app's ordinary untyped + * buffer ranges at2TiB, despite the DART's4TiB upper limit. + */ + .firmware_dma_bits = 41, .num_dptx_ports = 0, .iomfb_method_profile = DCP_IOMFB_METHODS_H17G, .adopt_running_coprocessor = true, .adopt_live_session = true, + .discover_live_session = true, .firmware_compat = DCP_FIRMWARE_V_27_0, }; diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 9eb0c7d4eb534..18c89e441ff49 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -86,7 +86,7 @@ static int dcp_get_brightness(struct backlight_device *bd) { struct apple_dcp *dcp = bl_get_data(bd); - return dcp->brightness.nits; + return READ_ONCE(dcp->brightness.nits); } #define SCALE_FACTOR (1 << 10) @@ -113,6 +113,10 @@ static u32 calculate_dac(struct apple_dcp *dcp, int val) { u32 dac; + /* Firmware27 consumes nits directly in its swap, not a DAC value. */ + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return val; + if (val <= MIN_BRIGHTNESS_PART1) return 16 * brightness_part1[0]; else if (val == MAX_BRIGHTNESS_PART1) @@ -238,7 +242,9 @@ int dcp_backlight_register(struct apple_dcp *dcp) .brightness = dcp->brightness.nits, .scale = BACKLIGHT_SCALE_LINEAR, }; - props.max_brightness = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + props.max_brightness = dcp->fw_compat == DCP_FIRMWARE_V_27_0 ? + dcp->brightness.maximum : + min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); bl_dev = devm_backlight_device_register(dev, "apple-panel-bl", dev, dcp, &dcp_backlight_ops, &props); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 6f2061da93ede..dcc4d15fe9b9f 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -556,8 +556,11 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || !dcp->cb_handlers[tag]) { - dev_warn(dev, "received unknown callback %c%c%c%c\n", - hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + dev_warn(dev, "received unknown callback %c%c%c%c in=%u out=%u data=%*ph\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0], + hdr->in_len, hdr->out_len, + min_t(u32, min_t(u32, hdr->in_len, + length - sizeof(*hdr)), 32), in); /* * Leaving a callback unanswered wedges the coprocessor: it * waits for the ack forever and the outer call never returns. diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 667f199ac8f57..276a42273546d 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -424,14 +424,6 @@ static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) } #endif -/* - * A471 (update_dfb) is never sent. It hands the firmware a default-framebuffer - * surface descriptor, and on H17P it makes the coprocessor take an AXI read - * error and NMI immediately (axi_rd_err -> "panic(app): app NMI"). Scanout does - * not need it: the swap carries its own surface, whose DVA comes from the DRM - * GEM object on the display-subsystem device and is therefore already mapped in - * the stream the pipe reads. Just acknowledge the callback. - */ static void complete_dfb_surface(struct apple_dcp *dcp, void *out, void *cookie) { #if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) @@ -561,10 +553,19 @@ static bool iomfbep_cb_match_backlight_service(struct apple_dcp *dcp, int tag, v static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *prop) { switch (prop->id) { +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + /* Native26A428 runtime-property table: IOMFBBrightnessLevel. */ + case 20: +#else case IOMFB_PROPERTY_NITS: +#endif { if (dcp_has_panel(dcp)) { +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + dcp->brightness.nits = prop->value >> 16; +#else dcp->brightness.nits = prop->value / dcp->brightness.scale; +#endif /* notify backlight device of the initial brightness */ if (!dcp->brightness.bl_dev && dcp->brightness.maximum > 0) schedule_work(&dcp->bl_register_wq); @@ -751,8 +752,19 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, memdesc->buf = dma_alloc_coherent(dcp->dev, size, &memdesc->dva, GFP_KERNEL); - dma_get_sgtable(dcp->dev, &memdesc->map, memdesc->buf, memdesc->dva, - size); + if (!memdesc->buf) { + clear_bit(id, dcp->memdesc_map); + memdesc->size = 0; + return (struct dcp_allocate_buffer_resp){}; + } + if (dma_get_sgtable(dcp->dev, &memdesc->map, memdesc->buf, + memdesc->dva, size)) { + dma_free_coherent(dcp->dev, size, memdesc->buf, memdesc->dva); + memdesc->buf = NULL; + memdesc->size = 0; + clear_bit(id, dcp->memdesc_map); + return (struct dcp_allocate_buffer_resp){}; + } resp.dva = memdesc->dva; /* @@ -1694,9 +1706,14 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) * brightness. */ if (dcp_has_panel(dcp)) { +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + swap->swap.brightness_update = 1; + swap->swap.brightness_nits = 0; +#else swap->swap.bl_unk = 1; swap->swap.bl_value = 0; swap->swap.bl_power = 0; +#endif } /* Null all surfaces */ @@ -1961,6 +1978,21 @@ TRAMPOLINE_INOUT(trampoline_get_tiling_state, dcpep_cb_get_tiling_state, struct dcpep_get_tiling_state_req, struct dcpep_get_tiling_state_resp); TRAMPOLINE_OUT(trampoline_create_backlight_service, dcpep_cb_create_backlight_service, u8); +static u64 dcp_swap_time(void) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + /* 26A428 mach_continuous_time uses AGTCNTVCTSS. On T6050 the + * architectural counter runs at 1 GHz while this firmware clock + * runs at 24 MHz. Sending architectural ticks schedules swaps far + * into the future. Read the native counter directly, including when + * the guest has architectural-counter redirection disabled. + */ + return read_sysreg_s(sys_reg(3, 4, 15, 10, 6)); +#else + return arch_timer_read_counter(); +#endif +} + /* * Callback for swap requests. If a swap failed, we'll never get a swap * complete event so we need to fake a vblank event early to avoid a hang. @@ -1976,7 +2008,7 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) return; } dcp->swap_start = ktime_get(); - dcp->swap_submit_timestamp = arch_timer_read_counter(); + dcp->swap_submit_timestamp = dcp_swap_time(); dcp_release_swapped_out_fbs(dcp); } @@ -2124,7 +2156,7 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, * already running at its native 2408x1506, so skip the modeset and let * that timing stand. */ - if (!dcp_send_modeset) { + if (DCP_FW_VER < DCP_FW_VERSION(27, 0, 0) && !dcp_send_modeset) { dev_info(dcp->dev, "inheriting the bootloader's display mode\n"); complete_set_digital_out_mode(dcp, NULL, cookie); } else @@ -2295,11 +2327,11 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru if (has_surface && (dcp->use_timestamps || dcp->vrr_enabled)) { u64 submit_timestamp = dcp->swap_submit_timestamp; - u64 timestamp = arch_timer_read_counter(); + u64 timestamp = dcp_swap_time(); /* - * IOMobileFramebuffer uses Mach continuous-time values here. On - * Apple Silicon that is the ARM architectural counter. Empirical + * IOMobileFramebuffer uses Mach continuous-time values here; + * dcp_swap_time() selects its firmware timebase. Empirical * testing shows that using the current submission time together * with the previous accepted swap makes DCP follow swap pacing. */ @@ -2320,9 +2352,23 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru /* update brightness if changed */ if (dcp_has_panel(dcp) && dcp->brightness.update) { +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + u32 nits = dcp->brightness.dac; + + /* The native API takes double nits, not the legacy packed DAC. */ + req->swap.brightness_update = 1; + if (nits) { + unsigned int exponent = ilog2(nits); + + req->swap.brightness_nits = + ((u64)(exponent + 1023) << 52) | + ((u64)(nits ^ (1U << exponent)) << (52 - exponent)); + } +#else req->swap.bl_unk = 1; req->swap.bl_value = dcp->brightness.dac; req->swap.bl_power = 0x40; +#endif dcp->brightness.update = false; } diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 2b2247fb0c462..d31ed42fbfc8d 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -59,6 +59,14 @@ struct DCP_FW_NAME(dcp_swap) { u32 swap_completed; u32 bg_color; +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + u8 reserved_158[0x1fc]; + /* 26A428 _kern_SwapSetBrightness, relative to IOMFBSwapRec. */ + u8 brightness_update; + u8 reserved_355[9]; + u64 brightness_nits; /* IEEE754 binary64, encoded without kernel FP */ + u8 reserved_366[0x37a]; +#else u8 unk_110[0x1b8]; u32 unk_2c8; u8 unk_2cc[0x14]; @@ -81,10 +89,6 @@ struct DCP_FW_NAME(dcp_swap) { * dcp_surface_h17p = 0x22c, SWAP_SURFACES = 4 and no surf2 block leaves * exactly 0x122 here. 0x278 makes dcp_swap 0x6e0 and forces a * non-integer surface size, so it cannot be right. */ -#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) - /* 26A428 copies exactly 0x6e0 bytes of IOMFBSwapRec in A408. */ - u8 h17p_tail[0x230]; -#else u8 h17p_tail[0xda]; #endif #endif diff --git a/drivers/gpu/drm/apple/iomfb_v27.c b/drivers/gpu/drm/apple/iomfb_v27.c index 2a6b7cbcf6471..e5b940d89a398 100644 --- a/drivers/gpu/drm/apple/iomfb_v27.c +++ b/drivers/gpu/drm/apple/iomfb_v27.c @@ -26,6 +26,9 @@ static_assert(sizeof(struct dcp_h17p_swap_info_request) == 0xe4); /* D590 swap_complete_ap_gated: 0x730 on the wire in 1919/1919 traced swaps. */ static_assert(sizeof(struct dc_swap_complete_resp_v27) == 0x730); static_assert(offsetof(struct dc_swap_complete_resp_v27, swap_id) == 0); +static_assert(offsetof(struct dcp_swap_v27, brightness_update) == 0x354); +static_assert(offsetof(struct dcp_swap_v27, brightness_nits) == 0x35e); +static_assert(sizeof(struct dcp_swap_v27) == 0x6e0); /* Native 26A428 T605X kernelcache RPC serializers. The method IDs are not * compatible with the 25G83 H17P or 25G76 H17G tables. See t6050-dcp.rst. @@ -89,28 +92,14 @@ static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, .input = in, .output = out, }; - /* - * Take these from the macOS capture rather than from the DT resources. - * macOS answers D003 with one stable record, - * 0000820203000000 0080860203000000 00000000 - * -> scratch 0x302820000, doorbell 0x302868000 -- which is - * DISP0_REG_BASE + 0x20000 and + 0x68000, both inside the 0xbc000 - * window we already hand out for map_reg index 5. The DT lookup lands - * 0x5a80000 higher, so the firmware was being pointed at the wrong - * block for its bandwidth scratch and doorbell. + /* Native 26A428 UPPipeAP_H17GSCD::rt_bandwidth_setup_ap returns + * the PMC descriptor's physical address and a zero clock request + * (getPhysicalAddress; stp x0, xzr at 0xfffffe00097ac058). + * Firmware consumes a physical address, even when Linux uses IPAs. */ - u64 clock_request = DCP_H17P_DISP0_REG_BASE + 0x68000; - u64 scratch = DCP_H17P_DISP0_REG_BASE + 0x20000; - - /* The T8142 DT contains the native PMC resources. The fallback above - * is specific to the incomplete legacy T8140 binding. - */ - if (!dcp->hw.legacy_pmc_binding) { - scratch = dcp->disp_bw_scratch_res.start + - dcp->disp_bw_scratch_offset; - clock_request = dcp->disp_bw_doorbell_res.start + - dcp->disp_bw_doorbell_offset; - } + u64 scratch = dcp->disp_firmware_base[dcp->disp_bw_scratch_index] + + dcp->disp_bw_scratch_offset; + u64 clock_request = 0; trace_iomfb_callback(dcp, tag, __func__); if (apple_dcp_h17p_rt_bw_encode_reply(&view, scratch, @@ -123,18 +112,21 @@ static bool trampoline_get_frequency_v27(struct apple_dcp *dcp, int tag, void *out, void *in) { const struct apple_dcp_h17p_clock_request *request = in; - u32 id = le32_to_cpu(request->clock_id); - unsigned int i; + u32 index = le32_to_cpu(request->clock_id); u64 rate = 0; - for (i = 0; i < dcp->num_firmware_clocks; i++) { - if (dcp->firmware_clock_ids[i] == id) { - rate = clk_get_rate(dcp->firmware_clocks[i].clk); - break; - } - } - if (i == dcp->num_firmware_clocks) - dev_err(dcp->dev, "DCP requested undeclared firmware clock %u\n", id); + /* 26A428 ServiceRelay -> iOSSvcProx -> AppleARMIODevice interprets + * this argument as an index into the provider's clock-ids property. + * AppleARMIODevice resolves the global clock ID before forwarding + * to AppleARMIO; it is not sent directly in this callback. + */ + if (index < dcp->num_firmware_clocks) + rate = clk_get_rate(dcp->firmware_clocks[index].clk); + else + dev_err(dcp->dev, "DCP requested undeclared provider clock index %u\n", + index); + dev_dbg(dcp->dev, "D408 service=%#x index=%u rate=%llu\n", + le32_to_cpu(request->service), index, rate); *(__le64 *)out = cpu_to_le64(rate); return true; } @@ -233,6 +225,126 @@ static bool trampoline_get_data_prop_v27(struct apple_dcp *dcp, int tag, return true; } +/* 26A428 MemDescRelay serializers: D451 no longer returns a physical + * address. The two u64 outputs are mapped DVA and buffer capacity, followed + * by the descriptor ID at +0x10 (ab8a414..ab8a444, ab7e9bc..ab7e9ec). + */ +struct dcp_allocate_buffer_req_v27 { + __le32 options; + __le64 size; + __le32 alignment; + u8 dva_null; + u8 capacity_null; + u8 padding[2]; +} __packed; + +struct dcp_allocate_buffer_resp_v27 { + __le64 dva; + __le64 capacity; + __le32 mem_desc_id; +} __packed; + +struct dcp_buffer_lifecycle_req_v27 { + __le32 mem_desc_id; + __le32 options; +} __packed; + +static_assert(sizeof(struct dcp_allocate_buffer_req_v27) == 0x14); +static_assert(sizeof(struct dcp_allocate_buffer_resp_v27) == 0x14); +static_assert(offsetof(struct dcp_allocate_buffer_resp_v27, mem_desc_id) == 0x10); +static_assert(sizeof(struct dcp_buffer_lifecycle_req_v27) == 8); + +static bool trampoline_allocate_buffer_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_allocate_buffer_req_v27 *req = in; + struct dcp_allocate_buffer_resp_v27 *resp = out; + struct dcp_allocate_buffer_req allocation = { + .unk0 = le32_to_cpu(req->options), + .size = le64_to_cpu(req->size), + .unk2 = le32_to_cpu(req->alignment), + }; + struct dcp_allocate_buffer_resp buffer; + + if (!allocation.size || allocation.size > SIZE_MAX - SZ_16K || + allocation.unk2 > PAGE_SIZE) + return true; + buffer = dcpep_cb_allocate_buffer(dcp, &allocation); + if (!req->dva_null) + resp->dva = cpu_to_le64(buffer.dva); + if (!req->capacity_null) + resp->capacity = cpu_to_le64(ALIGN(buffer.dva_size, SZ_16K)); + resp->mem_desc_id = cpu_to_le32(buffer.mem_desc_id); + dev_dbg(dcp->dev, "D451 allocation id=%u dva=%#llx capacity=%#llx\n", + buffer.mem_desc_id, buffer.dva, le64_to_cpu(resp->capacity)); + return true; +} + +static bool trampoline_buffer_lifecycle_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_buffer_lifecycle_req_v27 *req = in; + __le32 *status = out; + u32 id = le32_to_cpu(req->mem_desc_id); + + /* Native prepare/complete take an existing descriptor, not a physical + * address to map. Coherent allocations remain DMA-mapped until D454; + * no streaming cache maintenance or per-call IOMMU updates are needed. + */ + if (!id || id >= DCP_MAX_MAPPINGS || + !test_bit(id, dcp->memdesc_map) || !dcp->memdesc[id].buf) { + *status = cpu_to_le32(14); /* native kIOReturnNotReady -> link */ + return true; + } + if (tag == 452) + dma_wmb(); + else + dma_rmb(); + *status = 0; + return true; +} + +/* Native get_ideal_screen_space reads product/artwork-display-gamut: + * P3 -> 2, AdobeRGB -> 3, absent/unrecognized -> 1 (ab97428..ab975d0). + */ +static bool trampoline_ideal_screen_space_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const char *gamut; + u32 space = 1; + + if (!of_property_read_string(dcp->dev->of_node, + "apple,artwork-display-gamut", &gamut)) { + if (!strcmp(gamut, "P3")) + space = 2; + else if (!strcmp(gamut, "AdobeRGB")) + space = 3; + } + *(__le32 *)out = cpu_to_le32(space); + return true; +} + +/* Native D114 params 1 and 11 read tile membership/manager flags + * (97bd5c0 and 97bd488). Both are false for an untiled connector. + * Tiled connectors and other parameters require separate tiling + * manager state; retain the caller's value and return false if unsupported. + */ +static bool trampoline_tiling_state_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcpep_get_tiling_state_req *req = in; + struct dcpep_get_tiling_state_resp *resp = out; + + resp->value = req->value; + resp->ret = 0; + if (!req->value_null && (req->param == 1 || req->param == 11) && + dcp->connector && !dcp->connector->base.has_tile) { + resp->value = 0; + resp->ret = 1; + } + return true; +} + static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [0] = dcpep_cb_d000_h17p, /* native did_boot callback */ [1] = trampoline_true, @@ -243,6 +355,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [101] = trampoline_zero, [102] = trampoline_nop, /* set_number_property */ [104] = trampoline_nop, /* set_boolean_property */ + [106] = trampoline_nop, /* remove host IORegistry metadata property */ [107] = trampoline_true, /* create_provider_service */ [108] = trampoline_true, /* create_product_service */ [109] = trampoline_true, /* create_PMU_service */ @@ -254,6 +367,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [117] = trampoline_zero, /* upload_trace_start */ [118] = trampoline_zero, /* upload_trace_chunk */ [119] = trampoline_zero, /* upload_trace_end */ + [114] = trampoline_tiling_state_v27, [120] = dcpep_cb_boot_1, /* start_hardware_boot */ [121] = trampoline_false, /* is_dark_boot */ [122] = trampoline_false, /* is_waking_from_hibernate */ @@ -267,6 +381,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [202] = trampoline_unmap_piodma, /* unmap_buf */ [206] = iomfbep_cb_match_pmu_service_2, /* match_pmu_service */ [207] = iomfbep_cb_match_backlight_service, /* match_backlight_service */ + [204] = trampoline_ideal_screen_space_v27, [208] = trampoline_nop, /* update_backlight_factor_prop */ [209] = trampoline_get_time, /* get_calendar_time_ms */ [300] = trampoline_pr_publish, @@ -279,8 +394,9 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [413] = trampoline_true, /* setProperty */ [414] = trampoline_sr_set_property_int, /* setProperty */ [415] = trampoline_true, /* setProperty */ - [451] = trampoline_allocate_buffer, /* allocate_buffer */ - [452] = trampoline_map_physical, /* prepare */ + [451] = trampoline_allocate_buffer_v27, /* allocate_buffer */ + [452] = trampoline_buffer_lifecycle_v27, /* prepare */ + [453] = trampoline_buffer_lifecycle_v27, /* complete */ [454] = trampoline_release_mem_desc, /* release_descriptor */ [552] = trampoline_true, [561] = trampoline_true, @@ -291,17 +407,14 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [574] = iomfbep_cb_video_range_h17p, [575] = trampoline_hotplug_h17p, [576] = trampoline_nop, - /* - * We do not create a default framebuffer surface, so answering "yes" - * makes the firmware go on to program the display engine for a surface - * that does not exist: it then DMAs from a null base and the piodma - * DART (412300000) faults continuously on SID 0, which starves the - * system into an RCU stall. Decline instead. - */ [586] = iomfbep_cb_create_dfb_surface, /* create_default_fb_surface */ [587] = trampoline_nop, /* clear_default_surface */ [588] = trampoline_nop, /* swap_notify_gated */ [589] = trampoline_swap_info_h17p, /* swap_info_notify_dispatch */ + /* Native D592 applies an AP-queued DFB geometry change only. Linux + * handles D586 geometry synchronously and queues no deferred resize. + */ + [592] = trampoline_nop, /* resize_default_fb_surface_gated */ [594] = trampoline_swap_complete, /* swap_complete_ap_gated */ [596] = trampoline_swap_complete_intent_gated, /* swap_complete_intent_gated */ [597] = trampoline_abort_swap_ap_gated, /* abort_swap_ap_gated */ @@ -328,6 +441,7 @@ static const struct { [101] = { 0x0, 0x4 }, [102] = { 0x44, 0x0 }, [104] = { 0x44, 0x0 }, + [106] = { 0x40, 0x0 }, [107] = { 0x0, 0x4 }, [108] = { 0x0, 0x4 }, [109] = { 0x0, 0x4 }, @@ -339,6 +453,7 @@ static const struct { [117] = { 0x14, 0x4 }, [118] = { 0x3e88, 0x4 }, [119] = { 0x4, 0x4 }, + [114] = { 0x10, 0x8 }, [120] = { 0x0, 0x4 }, [121] = { 0x0, 0x4 }, [122] = { 0x0, 0x4 }, @@ -352,6 +467,7 @@ static const struct { [202] = { 0x18, 0x0 }, [206] = { 0x0, 0x4 }, [207] = { 0x0, 0x4 }, + [204] = { 0x0, 0x4 }, [208] = { 0x4, 0x0 }, [209] = { 0x0, 0x8 }, [400] = { 0x4c, 0xc04 }, @@ -365,6 +481,7 @@ static const struct { [415] = { 0x4c, 0x4 }, [451] = { 0x14, 0x14 }, [452] = { 0x8, 0x4 }, + [453] = { 0x8, 0x4 }, [454] = { 0x4, 0x4 }, [552] = { 0x1044, 0x4 }, [561] = { 0x1044, 0x4 }, @@ -379,6 +496,7 @@ static const struct { [587] = { 0x0, 0x0 }, [588] = { 0x18, 0x0 }, [589] = { 0xe4, 0xe0 }, + [592] = { 0x0, 0x0 }, [594] = { 0x730, 0x0 }, [596] = { 0x14, 0x0 }, [597] = { 0x4, 0x0 }, diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c index 9fe7a0ce495aa..bd14ec8e140cf 100644 --- a/drivers/gpu/drm/apple/systemep.c +++ b/drivers/gpu/drm/apple/systemep.c @@ -97,7 +97,7 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype dev_dbg(dcp->dev, "systemep: mNits event: Nits: %u.%03u, iDAC: %u\n", mnits.millinits / 1000, mnits.millinits % 1000, mnits.idac); - dcp->brightness.nits = mnits.millinits / 1000; + WRITE_ONCE(dcp->brightness.nits, mnits.millinits / 1000); return 0; } From b2a59dfa0841429190d16465794da4d12ea519a0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 13:25:14 -0400 Subject: [PATCH 0223/1579] drm/apple: correct macOS 27 adaptive-sync parameter dispatch Use parameter 15 on firmware 27: the 26A428 DCP dispatcher routes it to the adaptive-sync handler, while parameter 14 now requires zero arguments. Check adaptive-sync and timing replies before accepting a mode, and preserve native frame-sync property values when acknowledging their consumed flags. Validated module loading in QEMU, then native M5 Pro modesets with firmware VRR=NO/YES, paced flips, matching brightness feedback, and a 15-second display power-off/on cycle. Flip-event timing is not an optical panel-refresh measurement. --- drivers/gpu/drm/apple/iomfb_template.c | 46 ++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 276a42273546d..3de70b396b722 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -67,6 +67,7 @@ static_assert(offsetof(struct dcp_surface, has_compr_info) == 0x1ed); struct dcp_wait_cookie { struct kref refcount; struct completion done; + int status; }; static void release_wait_cookie(struct kref *ref) @@ -1385,6 +1386,18 @@ dcpep_cb_set_frame_sync_props_h17p(struct apple_dcp *dcp, struct dcp_set_frame_sync_props_req_h17p *req) { struct dcp_set_frame_sync_props_resp_h17p resp = {}; +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + unsigned int i; + + /* 26A428 UPPipeAP_H17GSCD::set_frame_sync_props publishes the sixteen + * supplied property words, then clears each consumed change flag. + * Preserve their values, including brightness caps and scale factors. + */ + memcpy(&resp, req, sizeof(resp)); + for (i = 0x40; i < sizeof(resp); i++) + if (resp.unk[i] & 1) + resp.unk[i] = 0; +#else static const u16 identity_at[] = { 0x24, 0x28, 0x2c }; unsigned int i; @@ -1393,6 +1406,7 @@ dcpep_cb_set_frame_sync_props_h17p(struct apple_dcp *dcp, memcpy(&resp.unk[identity_at[i]], &v, sizeof(v)); } +#endif return resp; } @@ -1573,7 +1587,12 @@ static void dcp_on_set_power_state(struct apple_dcp *dcp, void *out, void *cooki static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_set_parameter_dcp param = { + /* 26A428 moved adaptive sync from parameter 14 to 15. */ +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + .param = 15, +#else .param = IOMFBPARAM_ADAPTIVE_SYNC, +#endif .value = { 0 }, #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) .count = 3, @@ -2056,6 +2075,12 @@ static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, struct dcp_wait_cookie *wait = cookie; if (wait) { +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + if (data && *(u32 *)data) { + dev_err(dcp->dev, "timing mode rejected: %#x\n", *(u32 *)data); + wait->status = -EIO; + } +#endif complete(&wait->done); kref_put(&wait->refcount, release_wait_cookie); } @@ -2065,6 +2090,17 @@ static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, static void dcp_on_set_adaptive_sync(struct apple_dcp *dcp, void *out, void *cookie) { +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + if (*(u32 *)out) { + struct dcp_wait_cookie *wait = cookie; + + dev_err(dcp->dev, "adaptive-sync parameters rejected: %#x\n", *(u32 *)out); + wait->status = -EOPNOTSUPP; + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + return; + } +#endif dcp_set_digital_out_mode(dcp, false, &dcp->mode, complete_set_digital_out_mode, cookie); } @@ -2073,7 +2109,12 @@ static void dcp_set_adaptive_sync(struct apple_dcp *dcp, u32 min_vrr, void *cookie) { struct dcp_set_parameter_dcp param = { + /* 26A428 moved adaptive sync from parameter 14 to 15. */ +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + .param = 15, +#else .param = IOMFBPARAM_ADAPTIVE_SYNC, +#endif .value = { min_vrr, /* minRR, 16.16 fixed-point Hz */ 0, /* mediaTargetRate */ @@ -2212,7 +2253,12 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, "set_digital_out_mode finished with %d to spare\n", jiffies_to_msecs(ret)); } + ret = cookie->status; kref_put(&cookie->refcount, release_wait_cookie); + if (ret) { + dcp->valid_mode = false; + return ret; + } dcp->valid_mode = true; dcp->vrr_enabled = mode->vrr && crtc_state->vrr_enabled; From 117de62b555ab899be335b832a298fbc08824fe8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 13:49:48 -0400 Subject: [PATCH 0224/1579] drm/apple: send native AFKV2 commands on firmware 27 Encode the compact transport/interface headers, service-open reports, and out-of-band command descriptors used by 26A428. Decode native replies and preserve sixteen concurrent slots with eight-bit wire tags. Allow no-output requests to omit RX allocation: supplying an output buffer for setProperty makes firmware assert while copying its empty response. Validated in QEMU and on M5 Pro: captured status-zero reply, healthy DCP scanout/brightness, and 256 successful commands from sixteen callers in 36.9 ms. Incoming DPTX AP commands still need their firmware-27 envelope port. --- drivers/gpu/drm/apple/afk.c | 155 ++++++++++++++++++++++++------- drivers/gpu/drm/apple/afk.h | 24 +++++ drivers/gpu/drm/apple/systemep.c | 3 +- 3 files changed, 148 insertions(+), 34 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 8b8eee0530562..1629868cda247 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -346,6 +346,9 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, ep->services[ch_idx].ep = ep; ep->services[ch_idx].channel = channel; ep->services[ch_idx].cmd_tag = 0; + if (ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + afk_send_epic(ep, channel, 0, EPIC_TYPE_NOTIFY, EPIC_CAT_REPORT, + 0x12, NULL, 0); ops->init(&ep->services[ch_idx], epic_name, epic_class, epic_unit); dev_info(ep->dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", ep->endpoint, service_name, channel); @@ -393,7 +396,7 @@ static void afk_recv_handle_reply(struct apple_dcp_afkep *ep, u32 channel, struct epic_cmd *cmd = payload; struct apple_epic_service *service; unsigned long flags; - u8 idx = tag & 0xff; + u8 idx = tag & (ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0 ? 0xf : 0xff); void *rxbuf, *txbuf; dma_addr_t rxbuf_dma, txbuf_dma; size_t rxlen, txlen; @@ -553,6 +556,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, struct epic_sub_hdr *eshdr = (struct epic_sub_hdr *)(data + sizeof(*ehdr)); struct epic_sub_hdr h17p_eshdr; + struct epic_cmd v2_reply = {}; size_t hdr_len = sizeof(*ehdr) + sizeof(*eshdr); u16 subtype; u8 *payload; @@ -601,6 +605,49 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, payload = data + hdr_len; payload_size = data_size - hdr_len; + if (ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) { + const struct epic_hdr_v2 *v2 = (const void *)data; + + /* The queue type is no longer the command/reply discriminator. */ + if (v2->category == 2) { + const struct epic_cmd_v2 *reply = (const void *)payload; + + if (payload_size < offsetof(struct epic_cmd_v2, rxbuf)) + goto too_small; + h17p_eshdr.category = EPIC_CAT_REPLY; + h17p_eshdr.tag = cpu_to_le16(reply->tag); + v2_reply.retcode = reply->length; + if (reply->flags & 1) { + if (payload_size < sizeof(*reply)) + goto too_small; + v2_reply.rxbuf = reply->rxbuf; + v2_reply.txbuf = reply->txbuf; + v2_reply.rxlen = reply->rxlen; + v2_reply.txlen = reply->txlen; + } else { + /* Our commands request an OOB response. A status-only + * error reply may omit the unused descriptors. */ + if (!v2_reply.retcode) { + dev_err(ep->dcp->dev, "AFK: unexpected inline command reply\n"); + return; + } + } + type = EPIC_TYPE_REPLY; + payload = (void *)&v2_reply; + payload_size = sizeof(v2_reply); + } else if (v2->category == 0) { + type = EPIC_TYPE_NOTIFY; + if (le16_to_cpu(v2->interface_flags) & 1) { + if (v2->type == 0x12) + return; /* native interface-open report */ + if (v2->type == 0x13) { + afk_recv_handle_teardown(ep, channel); + return; + } + } + } + } + trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); service = afk_epic_find_service(ep, channel); @@ -841,15 +888,34 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, struct epic_sub_hdr *eshdr; unsigned long flags; size_t total_epic_size, total_size; + bool v2 = ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0; + struct epic_cmd_v2 v2_cmd = {}; int ret; + if (v2 && ecat == EPIC_CAT_COMMAND) { + const struct epic_cmd *cmd = payload; + + if (payload_len != sizeof(*cmd)) + return -EINVAL; + v2_cmd.flags = 1; + v2_cmd.tag = tag; + v2_cmd.length = cmd->rxlen; + v2_cmd.rxbuf = cmd->rxbuf; + v2_cmd.txbuf = cmd->txbuf; + v2_cmd.rxlen = cmd->rxlen; + v2_cmd.txlen = cmd->txlen; + payload = &v2_cmd; + payload_len = sizeof(v2_cmd); + } + spin_lock_irqsave(&ep->lock, flags); dma_rmb(); rptr = le32_to_cpu(*afk_rb_rptr(&ep->txbfr)); wptr = le32_to_cpu(*afk_rb_wptr(&ep->txbfr)); trace_afk_send_rwptr_pre(ep, rptr, wptr); - total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; + total_epic_size = (v2 ? sizeof(struct epic_hdr_v2) : + sizeof(*ehdr) + sizeof(*eshdr)) + payload_len; total_size = sizeof(*hdr) + total_epic_size; hdr = hdr2 = NULL; @@ -932,31 +998,46 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, hdr->magic = cpu_to_le32(QE_MAGIC); hdr->size = cpu_to_le32(total_epic_size); - hdr->channel = cpu_to_le32(channel); - hdr->type = cpu_to_le32(etype); + hdr->channel = cpu_to_le32(v2 ? 0 : channel); + hdr->type = cpu_to_le32(v2 ? 0 : etype); if (hdr2) memcpy(hdr2, hdr, sizeof(*hdr)); - ehdr = ep->txbfr.buf + wptr; - memset(ehdr, 0, sizeof(*ehdr)); - ehdr->version = 2; - ehdr->seq = cpu_to_le16(ep->qe_seq++); - ehdr->timestamp = cpu_to_le64(0); - wptr += sizeof(*ehdr); - - eshdr = ep->txbfr.buf + wptr; - memset(eshdr, 0, sizeof(*eshdr)); - eshdr->length = cpu_to_le32(payload_len); - eshdr->version = 4; - eshdr->category = ecat; - eshdr->type = cpu_to_le16(stype); - eshdr->timestamp = cpu_to_le64(0); - eshdr->tag = cpu_to_le16(tag); - if (ecat == EPIC_CAT_REPLY) - eshdr->inline_len = cpu_to_le32(payload_len - 4); - else - eshdr->inline_len = cpu_to_le32(0); - wptr += sizeof(*eshdr); + if (v2) { + struct epic_hdr_v2 *h = ep->txbfr.buf + wptr; + + memset(h, 0, sizeof(*h)); + h->seq = ep->qe_seq++; + h->channel = cpu_to_le16(channel); + h->length = cpu_to_le32(total_epic_size - 8); + h->type = stype; + h->category = ecat == EPIC_CAT_COMMAND ? 1 : + ecat == EPIC_CAT_REPLY ? 2 : 0; + if (ecat == EPIC_CAT_REPORT && (stype == 0x12 || stype == 0x13)) + h->interface_flags = cpu_to_le16(1); + wptr += sizeof(*h); + } else { + ehdr = ep->txbfr.buf + wptr; + memset(ehdr, 0, sizeof(*ehdr)); + ehdr->version = 2; + ehdr->seq = cpu_to_le16(ep->qe_seq++); + ehdr->timestamp = cpu_to_le64(0); + wptr += sizeof(*ehdr); + + eshdr = ep->txbfr.buf + wptr; + memset(eshdr, 0, sizeof(*eshdr)); + eshdr->length = cpu_to_le32(payload_len); + eshdr->version = 4; + eshdr->category = ecat; + eshdr->type = cpu_to_le16(stype); + eshdr->timestamp = cpu_to_le64(0); + eshdr->tag = cpu_to_le16(tag); + if (ecat == EPIC_CAT_REPLY) + eshdr->inline_len = cpu_to_le32(payload_len - 4); + else + eshdr->inline_len = cpu_to_le32(0); + wptr += sizeof(*eshdr); + } memcpy(ep->txbfr.buf + wptr, payload, payload_len); wptr += payload_len; @@ -990,18 +1071,20 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, unsigned int timeout_ms) { struct epic_cmd cmd; - void *rxbuf, *txbuf; - dma_addr_t rxbuf_dma, txbuf_dma; + void *rxbuf = NULL, *txbuf; + dma_addr_t rxbuf_dma = 0, txbuf_dma; unsigned long flags; int ret, idx; u16 tag; struct apple_dcp_afkep *ep = service->ep; DECLARE_COMPLETION_ONSTACK(completion); - rxbuf = dma_alloc_coherent(ep->dcp->dev, output_len, &rxbuf_dma, - GFP_KERNEL); - if (!rxbuf) - return -ENOMEM; + if (output_len) { + rxbuf = dma_alloc_coherent(ep->dcp->dev, output_len, &rxbuf_dma, + GFP_KERNEL); + if (!rxbuf) + return -ENOMEM; + } txbuf = dma_alloc_coherent(ep->dcp->dev, payload_len, &txbuf_dma, GFP_KERNEL); if (!txbuf) { @@ -1025,8 +1108,13 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, goto err_unlock; } - tag = (service->cmd_tag & 0xff) << 8; - tag |= idx & 0xff; + if (ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) { + /* Keep all sixteen concurrent slots with a four-bit generation. */ + tag = ((service->cmd_tag & 0xf) << 4) | idx; + } else { + tag = (service->cmd_tag & 0xff) << 8; + tag |= idx & 0xff; + } service->cmd_tag++; service->cmds[idx].tag = tag; @@ -1085,7 +1173,8 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, spin_unlock_irqrestore(&service->lock, flags); dma_free_coherent(ep->dcp->dev, payload_len, txbuf, txbuf_dma); err_free_rxbuf: - dma_free_coherent(ep->dcp->dev, output_len, rxbuf, rxbuf_dma); + if (rxbuf) + dma_free_coherent(ep->dcp->dev, output_len, rxbuf, rxbuf_dma); return ret; } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 9776056eb5f67..d66d13a184c3e 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -127,6 +127,30 @@ struct epic_sub_hdr_h17p { #define EPIC_SUBTYPE_ANNOUNCE_H17P 0x11 +/* AFKEPV2 transport and interface headers, as used by 26A428. */ +struct epic_hdr_v2 { + u8 seq; + u8 flags; + __le16 channel; + __le32 length; /* bytes after this transport header */ + __le64 timestamp; + u8 type; + u8 category; /* report=0, command=1, reply=2 */ + __le16 interface_flags; + __le32 reserved; +} __packed; + +struct epic_cmd_v2 { + u8 flags; /* bit 0: out-of-band buffer descriptors follow */ + u8 tag; + __le16 reserved; + __le32 length; /* command: response capacity; reply: status */ + __le64 rxbuf; + __le64 txbuf; + __le32 rxlen; + __le32 txlen; +} __packed; + struct epic_cmd { __le32 retcode; __le64 rxbuf; diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c index bd14ec8e140cf..5d9b7ce118da1 100644 --- a/drivers/gpu/drm/apple/systemep.c +++ b/drivers/gpu/drm/apple/systemep.c @@ -36,7 +36,8 @@ static void system_log_work(struct work_struct *work_) afk_send_command(work->service, SYSTEM_SET_PROPERTY, setprop_gAFKConfigLogMask_ffff, sizeof(setprop_gAFKConfigLogMask_ffff), NULL, - sizeof(setprop_gAFKConfigLogMask_ffff), NULL); + work->service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0 ? + 0 : sizeof(setprop_gAFKConfigLogMask_ffff), NULL); complete(&work->service->ep->dcp->systemep_done); kfree(work); } From 701cd4c0bca11ab78b850d376ffd7cc11f670353 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 14:46:41 -0400 Subject: [PATCH 0225/1579] iommu/apple-dart-x1n1: share DMA domains across display DARTs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The display master lists a scanout DART for each controller. Accept that topology with one IOVA allocator and canonical mapping set, intersect the apertures and reserve inherited mappings from every stream. Keep full transport batches, independently cache each controller’s leaf tables, and unwind cross-controller mapping or attachment failures. Allow concurrent mapping callers while excluding attachment replay. Add tests of the actual mapping and group functions, including partial failures, retained-domain transfer, group cleanup and batch-boundary rollback. Built Image and modules. QEMU passed DMA API cycles on its supported DART14/SID1 fixture; separate physical DARTs were tested on J714s hardware. DART18 and DART22 passed 128 allocation/free cycles plus 256 cycles with eight concurrent workers while GNOME remained at 120 Hz. Independent single-DART latency samples showed no slowdown; this is not a native-performance comparison or external-scanout qualification. --- drivers/iommu/apple-dart-x1n1.c | 683 ++++++++++++------ tools/testing/selftests/iommu/Makefile | 3 + .../selftests/iommu/apple-dart-x1n1-group.py | 90 +++ .../selftests/iommu/apple-dart-x1n1-map.c | 94 +++ .../selftests/iommu/apple-dart-x1n1-map.py | 171 +++++ 5 files changed, 800 insertions(+), 241 deletions(-) create mode 100755 tools/testing/selftests/iommu/apple-dart-x1n1-group.py create mode 100644 tools/testing/selftests/iommu/apple-dart-x1n1-map.c create mode 100755 tools/testing/selftests/iommu/apple-dart-x1n1-map.py diff --git a/drivers/iommu/apple-dart-x1n1.c b/drivers/iommu/apple-dart-x1n1.c index 077d582a0726f..3b717df55ef91 100644 --- a/drivers/iommu/apple-dart-x1n1.c +++ b/drivers/iommu/apple-dart-x1n1.c @@ -130,24 +130,75 @@ struct apple_dart_x1n1 { spinlock_t batch_lock; /* protects batch + doorbell */ }; -struct apple_dart_x1n1_master { +/* Match the native DART driver's capacity for shared display DMA domains. */ +#define X1N1_MAX_DARTS 8 + +struct x1n1_stream_map { struct apple_dart_x1n1 *dart; DECLARE_BITMAP(sids, DART_X1N1_MAX_STREAMS); }; +struct apple_dart_x1n1_master { + unsigned int num_darts; + struct x1n1_stream_map streams[X1N1_MAX_DARTS]; +}; + struct apple_dart_x1n1_domain { struct iommu_domain domain; - struct apple_dart_x1n1 *dart; - DECLARE_BITMAP(sids, DART_X1N1_MAX_STREAMS); - struct mutex lock; /* protects sids / dart binding */ - /* level-2 leaf tables already attached, keyed by (sid << 40 | block), - * block = dva >> 25. Installing a level-2 table twice is fatal in SPTM, - * so each 32 MiB window is attached at most once. */ + unsigned int num_darts; + struct x1n1_stream_map streams[X1N1_MAX_DARTS]; + struct mutex lock; /* serialises domain attachments */ + /* Mapping calls may run concurrently on distinct ranges. An attachment + * must replay a stable mapping set before publishing its new streams. + */ + rwlock_t streams_lock; + /* Key: dart-id:8, sid:8, block:40. SPTM tables cannot be attached twice. */ struct xarray l2_attached; bool has_retained; - struct xarray mappings; /* tagged (guest-PFN, protection), indexed by IOVA page */ + /* One canonical mapping set and IOVA allocator for every DART. */ + struct xarray mappings; }; +static struct x1n1_stream_map * +x1n1_find_stream(struct apple_dart_x1n1_domain *dom, struct apple_dart_x1n1 *dart) +{ + if (dom) + for (unsigned int i = 0; i < dom->num_darts; i++) + if (dom->streams[i].dart == dart) + return &dom->streams[i]; + return NULL; +} + +/* Domain topology is fixed before the first attachment. Later devices can + * add SIDs on these DARTs, but cannot change an active allocator's aperture. + */ +static int x1n1_domain_configure(struct apple_dart_x1n1_domain *dom, + const struct apple_dart_x1n1_master *master) +{ + u64 start = 0, end = DART_X1N1_NATIVE_APERTURE_END; + + if (!master || !master->num_darts) + return -ENODEV; + if (dom->num_darts) { + for (unsigned int i = 0; i < master->num_darts; i++) + if (!x1n1_find_stream(dom, master->streams[i].dart)) + return -EINVAL; + return 0; + } + for (unsigned int i = 0; i < master->num_darts; i++) { + start = max(start, master->streams[i].dart->dma_min); + end = min(end, master->streams[i].dart->dma_max); + } + if (start > end) + return -EINVAL; + for (unsigned int i = 0; i < master->num_darts; i++) + dom->streams[i].dart = master->streams[i].dart; + dom->domain.geometry.aperture_start = start; + dom->domain.geometry.aperture_end = end; + dom->num_darts = master->num_darts; + return 0; +} + static struct apple_dart_x1n1_domain *to_x1n1_domain(struct iommu_domain *dom) { return container_of(dom, struct apple_dart_x1n1_domain, domain); @@ -307,11 +358,15 @@ static bool x1n1_retained_overlap(struct apple_dart_x1n1_domain *dom, u64 dva, if (!dom->has_retained) return false; - for_each_set_bit(sid, dom->sids, dom->dart->num_streams) { - struct x1n1_retained_sid *r = dom->dart->retained[sid]; + for (unsigned int i = 0; i < dom->num_darts; i++) { + struct x1n1_stream_map *stream = &dom->streams[i]; + + for_each_set_bit(sid, stream->sids, stream->dart->num_streams) { + struct x1n1_retained_sid *r = stream->dart->retained[sid]; - if (r && x1n1_retained_lookup(r, dva, size)) - return true; + if (r && x1n1_retained_lookup(r, dva, size)) + return true; + } } return false; } @@ -324,20 +379,25 @@ static phys_addr_t x1n1_retained_translate(struct apple_dart_x1n1_domain *dom, if (!dom->has_retained) return 0; - /* One domain may serve several streams: an inherited mapping is only - * unambiguous when every stream resolves it to the same guest address. */ - for_each_set_bit(sid, dom->sids, dom->dart->num_streams) { - struct x1n1_retained_sid *r = dom->dart->retained[sid]; - const struct x1n1_retained_run *run = - r ? x1n1_retained_lookup(r, dva, 1) : NULL; - u64 pa; - - if (!run || dva < run->dva || run->ipa == U64_MAX) - return 0; - pa = run->ipa + dva - run->dva; - if (result && result != pa) - return 0; - result = pa; + /* Firmware may retain different mappings on different display engines. + * Only a unanimous translation is meaningful for their shared domain. + */ + for (unsigned int i = 0; i < dom->num_darts; i++) { + struct x1n1_stream_map *stream = &dom->streams[i]; + + for_each_set_bit(sid, stream->sids, stream->dart->num_streams) { + struct x1n1_retained_sid *r = stream->dart->retained[sid]; + const struct x1n1_retained_run *run = + r ? x1n1_retained_lookup(r, dva, 1) : NULL; + u64 pa; + + if (!run || dva < run->dva || run->ipa == U64_MAX) + return 0; + pa = run->ipa + dva - run->dva; + if (result && result != pa) + return 0; + result = pa; + } } return result; } @@ -481,10 +541,12 @@ static void x1n1_forget_mapping(struct apple_dart_x1n1_domain *dom, } static int x1n1_prepare_leaf(struct apple_dart_x1n1_domain *dom, + struct x1n1_stream_map *stream, unsigned int sid, u64 iova) { - struct apple_dart_x1n1 *dart = dom->dart; - unsigned long key = ((unsigned long)sid << 40) | + struct apple_dart_x1n1 *dart = stream->dart; + unsigned long key = ((unsigned long)dart->dart_id << 48) | + ((unsigned long)sid << 40) | (iova >> DART_X1N1_BLOCK_SHIFT); int ret; @@ -547,9 +609,9 @@ static bool x1n1_next_mapping(struct apple_dart_x1n1_domain *dom, } static int x1n1_undo_replay(struct apple_dart_x1n1_domain *dom, - unsigned int sid, u64 mapped_end) + struct x1n1_stream_map *stream, unsigned int sid, u64 mapped_end) { - struct apple_dart_x1n1 *dart = dom->dart; + struct apple_dart_x1n1 *dart = stream->dart; unsigned long cursor = 0; struct x1n1_op op; @@ -569,9 +631,10 @@ static int x1n1_undo_replay(struct apple_dart_x1n1_domain *dom, /* New SID is disabled throughout replay. On failure, keep it disabled and * remove the successful replay prefix. Existing SIDs/maps stay untouched. */ -static int x1n1_replay_sid(struct apple_dart_x1n1_domain *dom, unsigned int sid) +static int x1n1_replay_sid(struct apple_dart_x1n1_domain *dom, + struct x1n1_stream_map *stream, unsigned int sid) { - struct apple_dart_x1n1 *dart = dom->dart; + struct apple_dart_x1n1 *dart = stream->dart; unsigned long cursor = 0; struct x1n1_op op; u64 mapped_end = 0; @@ -581,7 +644,7 @@ static int x1n1_replay_sid(struct apple_dart_x1n1_domain *dom, unsigned int sid) while (x1n1_next_mapping(dom, &cursor, &op, sid)) { if (dart->retained[sid] && x1n1_retained_lookup(dart->retained[sid], op.arg[1], op.arg[3])) return -EBUSY; - ret = x1n1_prepare_leaf(dom, sid, op.arg[1]); + ret = x1n1_prepare_leaf(dom, stream, sid, op.arg[1]); if (ret) return ret; } @@ -605,7 +668,7 @@ static int x1n1_replay_sid(struct apple_dart_x1n1_domain *dom, unsigned int sid) return 0; } undo: - x1n1_undo_replay(dom, sid, mapped_end); + x1n1_undo_replay(dom, stream, sid, mapped_end); return ret; } @@ -615,22 +678,20 @@ static int apple_dart_x1n1_attach_dev(struct iommu_domain *domain, { struct apple_dart_x1n1_master *master = dev_iommu_priv_get(dev); struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); - struct apple_dart_x1n1 *dart; + struct apple_dart_x1n1_domain *prior = NULL; + unsigned long added[X1N1_MAX_DARTS][BITS_TO_LONGS(DART_X1N1_MAX_STREAMS)] = {}; + unsigned long moved[X1N1_MAX_DARTS][BITS_TO_LONGS(DART_X1N1_MAX_STREAMS)] = {}; unsigned long flags; unsigned int sid; - DECLARE_BITMAP(added_sids, DART_X1N1_MAX_STREAMS); - struct apple_dart_x1n1_domain *prior = NULL; - DECLARE_BITMAP(moved_sids, DART_X1N1_MAX_STREAMS); int ret; - if (!master) + if (!master || !master->num_darts) return -ENODEV; - dart = master->dart; - - ret = apple_dart_x1n1_ensure_init(dart); - if (ret) - return ret; - + for (unsigned int i = 0; i < master->num_darts; i++) { + ret = apple_dart_x1n1_ensure_init(master->streams[i].dart); + if (ret) + return ret; + } if (old && old != domain && old->owner == &apple_dart_x1n1_ops && old->ops == apple_dart_x1n1_ops.default_domain_ops) prior = to_x1n1_domain(old); @@ -642,120 +703,108 @@ static int apple_dart_x1n1_attach_dev(struct iommu_domain *domain, if (prior) mutex_lock_nested(&prior->lock, SINGLE_DEPTH_NESTING); } - if (dom->dart && dom->dart != dart) { + ret = x1n1_domain_configure(dom, master); + if (ret) + goto unlock_mutex; + + local_irq_save(flags); + if (prior && (unsigned long)prior < (unsigned long)dom) { + write_lock(&prior->streams_lock); + write_lock_nested(&dom->streams_lock, SINGLE_DEPTH_NESTING); + } else { + write_lock(&dom->streams_lock); if (prior) - mutex_unlock(&prior->lock); - mutex_unlock(&dom->lock); - return -EINVAL; + write_lock_nested(&prior->streams_lock, SINGLE_DEPTH_NESTING); } - dom->dart = dart; - domain->geometry.aperture_start = dart->dma_min; - domain->geometry.aperture_end = dart->dma_max; - - /* A new stream must see all existing mappings before it can DMA. */ - spin_lock_irqsave(&dart->batch_lock, flags); - ret = 0; - bitmap_zero(added_sids, DART_X1N1_MAX_STREAMS); - bitmap_zero(moved_sids, DART_X1N1_MAX_STREAMS); - for_each_set_bit(sid, master->sids, dart->num_streams) { - if (test_bit(sid, dom->sids)) - continue; - if (dart->retained[sid] && prior && prior->dart == dart && - test_bit(sid, prior->sids)) { - set_bit(sid, moved_sids); - ret = x1n1_undo_replay(prior, sid, U64_MAX); - if (ret) - break; - } - if (dart->retained[sid]) { - dart->batch->count = 1; - dart->batch->op[0] = (struct x1n1_op){ - .op = X1N1_DART_RETAINED_INFO, - .arg = { sid }, - }; - } else { - dart->batch->count = 2; - dart->batch->op[0] = (struct x1n1_op){ - .op = dart->retained[sid] ? - X1N1_DART_RETAINED_INFO : - X1N1_DART_DISABLE, - .arg = { sid }, - }; - dart->batch->op[1] = (struct x1n1_op){ - .op = X1N1_DART_ATTACH, - .arg = { sid, dart->dma_min, 1 }, - }; - } - ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); - if (ret) - break; - ret = x1n1_replay_sid(dom, sid); - if (ret) - break; - dart->batch->count = 1; - dart->batch->op[0] = (struct x1n1_op){ - .op = dart->retained[sid] ? X1N1_DART_RETAINED_INFO : - X1N1_DART_ENABLE, - .arg = { sid }, - }; - ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); - if (ret) { - /* A failed completion may still have enabled the SID. */ - dart->batch->count = 1; + for (unsigned int i = 0; i < master->num_darts && !ret; i++) { + struct apple_dart_x1n1 *dart = master->streams[i].dart; + struct x1n1_stream_map *stream = x1n1_find_stream(dom, dart); + struct x1n1_stream_map *before = x1n1_find_stream(prior, dart); + + spin_lock(&dart->batch_lock); + for_each_set_bit(sid, master->streams[i].sids, dart->num_streams) { + if (test_bit(sid, stream->sids)) + continue; + if (dart->retained[sid] && before && test_bit(sid, before->sids)) { + set_bit(sid, moved[i]); + ret = x1n1_undo_replay(prior, before, sid, U64_MAX); + if (ret) + break; + } + dart->batch->count = dart->retained[sid] ? 1 : 2; dart->batch->op[0] = (struct x1n1_op){ - .op = dart->retained[sid] ? - X1N1_DART_RETAINED_INFO : - X1N1_DART_DISABLE, + .op = dart->retained[sid] ? X1N1_DART_RETAINED_INFO : + X1N1_DART_DISABLE, .arg = { sid }, }; - if (x1n1_ring(dart, - X1N1_MMIO_DART_DB + 8 * dart->dart_id)) - dev_err(dart->dev, - "failed to disable new SID %u after enable error\n", - sid); - x1n1_undo_replay(dom, sid, U64_MAX); - break; - } - dom->has_retained |= !!dart->retained[sid]; - set_bit(sid, dom->sids); - set_bit(sid, added_sids); - } - if (ret) { - /* A master can contain multiple SIDs. Undo those added by this - * failed attachment without disturbing streams already in use. */ - for_each_set_bit(sid, added_sids, dart->num_streams) { + if (!dart->retained[sid]) + dart->batch->op[1] = (struct x1n1_op){ + .op = X1N1_DART_ATTACH, + .arg = { sid, dart->dma_min, 1 }, + }; + ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + if (ret) + break; + ret = x1n1_replay_sid(dom, stream, sid); + if (ret) + break; + /* Include this SID in unwind even when ENABLE's completion fails. */ + set_bit(sid, added[i]); dart->batch->count = 1; dart->batch->op[0] = (struct x1n1_op){ - .op = dart->retained[sid] ? - X1N1_DART_RETAINED_INFO : - X1N1_DART_DISABLE, + .op = dart->retained[sid] ? X1N1_DART_RETAINED_INFO : + X1N1_DART_ENABLE, .arg = { sid }, }; - if (x1n1_ring(dart, - X1N1_MMIO_DART_DB + 8 * dart->dart_id)) - dev_err(dart->dev, - "failed to disable new SID %u after attachment error\n", - sid); - x1n1_undo_replay(dom, sid, U64_MAX); - clear_bit(sid, dom->sids); + ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); + if (ret) + break; + dom->has_retained |= !!dart->retained[sid]; + set_bit(sid, stream->sids); } + spin_unlock(&dart->batch_lock); } - for_each_set_bit(sid, moved_sids, dart->num_streams) { + + /* Attach is one transaction across all of the master's DARTs. */ + for (unsigned int i = 0; i < master->num_darts; i++) { + struct apple_dart_x1n1 *dart = master->streams[i].dart; + struct x1n1_stream_map *stream = x1n1_find_stream(dom, dart); + struct x1n1_stream_map *before = x1n1_find_stream(prior, dart); + + spin_lock(&dart->batch_lock); if (ret) { - if (x1n1_replay_sid(prior, sid)) - dev_err(dart->dev, - "failed to restore previous domain SID %u\n", - sid); - } else { - clear_bit(sid, prior->sids); + for_each_set_bit(sid, added[i], dart->num_streams) { + dart->batch->count = 1; + dart->batch->op[0] = (struct x1n1_op){ + .op = dart->retained[sid] ? X1N1_DART_RETAINED_INFO : + X1N1_DART_DISABLE, + .arg = { sid }, + }; + if (x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id)) + dev_err(dart->dev, "failed to disable SID %u during attach unwind\n", sid); + x1n1_undo_replay(dom, stream, sid, U64_MAX); + clear_bit(sid, stream->sids); + } + } + for_each_set_bit(sid, moved[i], dart->num_streams) { + if (ret) { + if (x1n1_replay_sid(prior, before, sid)) + dev_err(dart->dev, "failed to restore previous domain SID %u\n", sid); + } else { + clear_bit(sid, before->sids); + } } + spin_unlock(&dart->batch_lock); } - spin_unlock_irqrestore(&dart->batch_lock, flags); - + if (prior) + write_unlock(&prior->streams_lock); + write_unlock(&dom->streams_lock); + local_irq_restore(flags); +unlock_mutex: if (prior) mutex_unlock(&prior->lock); mutex_unlock(&dom->lock); - return ret ? -EIO : 0; + return ret; } struct x1n1_map_progress { @@ -767,10 +816,11 @@ struct x1n1_map_progress { /* Track only fully installed stream cohorts. The batch may split one cohort * without reducing batching; keep its progress across doorbells. */ static int x1n1_flush_maps(struct apple_dart_x1n1_domain *dom, + struct x1n1_stream_map *stream, struct x1n1_map_progress *progress) { - struct apple_dart_x1n1 *dart = dom->dart; - unsigned int total_streams = bitmap_weight(dom->sids, dart->num_streams); + struct apple_dart_x1n1 *dart = stream->dart; + unsigned int total_streams = bitmap_weight(stream->sids, dart->num_streams); int ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); for (u32 i = 0; i < min(dart->batch->done, dart->batch->count); i++) { @@ -778,7 +828,6 @@ static int x1n1_flush_maps(struct apple_dart_x1n1_domain *dom, progress->partial = *op; if (++progress->streams == total_streams) { - x1n1_record_mapping(dom, op->arg[1], op->arg[2], op->arg[3], op->arg[4]); progress->mapped += op->arg[3]; progress->streams = 0; } @@ -787,7 +836,7 @@ static int x1n1_flush_maps(struct apple_dart_x1n1_domain *dom, unsigned int sid; u32 n = 0; - for_each_set_bit(sid, dom->sids, dart->num_streams) { + for_each_set_bit(sid, stream->sids, dart->num_streams) { if (n == progress->streams) break; dart->batch->op[n++] = (struct x1n1_op){ @@ -802,13 +851,13 @@ static int x1n1_flush_maps(struct apple_dart_x1n1_domain *dom, return ret; } -static int apple_dart_x1n1_map_pages(struct iommu_domain *domain, +static int x1n1_map_streams(struct apple_dart_x1n1_domain *dom, + struct x1n1_stream_map *stream, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, gfp_t gfp, size_t *mapped) { - struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); - struct apple_dart_x1n1 *dart = dom->dart; + struct apple_dart_x1n1 *dart = stream->dart; struct x1n1_map_progress progress = {}; size_t total; unsigned long flags; @@ -816,7 +865,7 @@ static int apple_dart_x1n1_map_pages(struct iommu_domain *domain, int ret; *mapped = 0; - if (!dart || bitmap_empty(dom->sids, dart->num_streams)) + if (!dart || bitmap_empty(stream->sids, dart->num_streams)) return -ENODEV; if (pgsize != DART_X1N1_PAGE_SIZE || !pgcount || check_mul_overflow(pgsize, pgcount, &total) || @@ -826,11 +875,6 @@ static int apple_dart_x1n1_map_pages(struct iommu_domain *domain, total - 1 > dart->dma_max - iova || iova + total > DART_X1N1_NATIVE_APERTURE_END) return -EINVAL; - if (x1n1_retained_overlap(dom, iova, total)) - return -EBUSY; - ret = x1n1_reserve_mappings(dom, iova, total, gfp); - if (ret) - return ret; spin_lock_irqsave(&dart->batch_lock, flags); /* Install all leaf tables before accumulating the mapping batch. */ for (size_t pos = 0; pos < total;) { @@ -839,8 +883,8 @@ static int apple_dart_x1n1_map_pages(struct iommu_domain *domain, DART_X1N1_BLOCK_SIZE - (cur & (DART_X1N1_BLOCK_SIZE - 1))); unsigned int sid; - for_each_set_bit(sid, dom->sids, dart->num_streams) { - ret = x1n1_prepare_leaf(dom, sid, cur); + for_each_set_bit(sid, stream->sids, dart->num_streams) { + ret = x1n1_prepare_leaf(dom, stream, sid, cur); if (ret) goto out; } @@ -852,14 +896,14 @@ static int apple_dart_x1n1_map_pages(struct iommu_domain *domain, DART_X1N1_BLOCK_SIZE - (cur & (DART_X1N1_BLOCK_SIZE - 1))); unsigned int sid; - for_each_set_bit(sid, dom->sids, dart->num_streams) { + for_each_set_bit(sid, stream->sids, dart->num_streams) { dart->batch->op[n++] = (struct x1n1_op){ .op = X1N1_DART_MAP, .arg = { sid, cur, paddr + pos, chunk, prot }, }; if (n == X1N1_BATCH_MAX_OPS) { dart->batch->count = n; - ret = x1n1_flush_maps(dom, &progress); + ret = x1n1_flush_maps(dom, stream, &progress); if (ret) goto out; n = 0; @@ -869,22 +913,19 @@ static int apple_dart_x1n1_map_pages(struct iommu_domain *domain, } if (n) { dart->batch->count = n; - ret = x1n1_flush_maps(dom, &progress); + ret = x1n1_flush_maps(dom, stream, &progress); } out: spin_unlock_irqrestore(&dart->batch_lock, flags); *mapped = progress.mapped; - /* Remove unused reservations after a partial failure; committed mappings - * remain available both to caller unwind and a later stream attachment. */ - if (ret) - x1n1_release_reservations(dom, iova + progress.mapped, total - progress.mapped); return ret ? -EIO : 0; } static int x1n1_restore_partial_unmap(struct apple_dart_x1n1_domain *dom, + struct x1n1_stream_map *stream, const struct x1n1_op *partial, unsigned int streams) { - struct apple_dart_x1n1 *dart = dom->dart; + struct apple_dart_x1n1 *dart = stream->dart; unsigned long cursor = partial->arg[1] >> DART_X1N1_PAGE_SHIFT; u64 end = partial->arg[1] + partial->arg[2]; struct x1n1_op op; @@ -894,7 +935,7 @@ static int x1n1_restore_partial_unmap(struct apple_dart_x1n1_domain *dom, unsigned int sid, count = 0; op.arg[3] = min_t(u64, op.arg[3], end - op.arg[1]); - for_each_set_bit(sid, dom->sids, dart->num_streams) { + for_each_set_bit(sid, stream->sids, dart->num_streams) { if (count++ == streams) break; op.arg[0] = sid; @@ -914,10 +955,11 @@ static int x1n1_restore_partial_unmap(struct apple_dart_x1n1_domain *dom, } static int x1n1_flush_unmaps(struct apple_dart_x1n1_domain *dom, + struct x1n1_stream_map *stream, struct x1n1_map_progress *progress) { - struct apple_dart_x1n1 *dart = dom->dart; - unsigned int total_streams = bitmap_weight(dom->sids, dart->num_streams); + struct apple_dart_x1n1 *dart = stream->dart; + unsigned int total_streams = bitmap_weight(stream->sids, dart->num_streams); int ret = x1n1_ring(dart, X1N1_MMIO_DART_DB + 8 * dart->dart_id); for (u32 i = 0; i < min(dart->batch->done, dart->batch->count); i++) { @@ -925,38 +967,35 @@ static int x1n1_flush_unmaps(struct apple_dart_x1n1_domain *dom, progress->partial = *op; if (++progress->streams == total_streams) { - x1n1_forget_mapping(dom, op->arg[1], op->arg[2]); progress->mapped += op->arg[2]; progress->streams = 0; } } if (ret && progress->streams && - x1n1_restore_partial_unmap(dom, &progress->partial, progress->streams)) + x1n1_restore_partial_unmap(dom, stream, &progress->partial, progress->streams)) dev_err(dart->dev, "failed to restore partial DART unmap\n"); return ret; } -static size_t apple_dart_x1n1_unmap_pages(struct iommu_domain *domain, +static size_t x1n1_unmap_streams(struct apple_dart_x1n1_domain *dom, + struct x1n1_stream_map *stream, unsigned long iova, size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) { - struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); - struct apple_dart_x1n1 *dart = dom->dart; + struct apple_dart_x1n1 *dart = stream->dart; struct x1n1_map_progress progress = {}; size_t total; unsigned long flags; u32 n = 0; - if (!dart || bitmap_empty(dom->sids, dart->num_streams) || + if (!dart || bitmap_empty(stream->sids, dart->num_streams) || pgsize != DART_X1N1_PAGE_SIZE || !pgcount || check_mul_overflow(pgsize, pgcount, &total) || (iova & (DART_X1N1_PAGE_SIZE - 1)) || iova < dart->dma_min || iova > dart->dma_max || total - 1 > dart->dma_max - iova) return 0; - if (x1n1_retained_overlap(dom, iova, total)) - return 0; spin_lock_irqsave(&dart->batch_lock, flags); for (size_t pos = 0; pos < total;) { u64 cur = iova + pos; @@ -964,13 +1003,13 @@ static size_t apple_dart_x1n1_unmap_pages(struct iommu_domain *domain, DART_X1N1_BLOCK_SIZE - (cur & (DART_X1N1_BLOCK_SIZE - 1))); unsigned int sid; - for_each_set_bit(sid, dom->sids, dart->num_streams) { + for_each_set_bit(sid, stream->sids, dart->num_streams) { dart->batch->op[n++] = (struct x1n1_op){ .op = X1N1_DART_UNMAP, .arg = { sid, cur, chunk }, }; if (n == X1N1_BATCH_MAX_OPS) { dart->batch->count = n; - if (x1n1_flush_unmaps(dom, &progress)) + if (x1n1_flush_unmaps(dom, stream, &progress)) goto out; n = 0; } @@ -979,13 +1018,143 @@ static size_t apple_dart_x1n1_unmap_pages(struct iommu_domain *domain, } if (n) { dart->batch->count = n; - x1n1_flush_unmaps(dom, &progress); + x1n1_flush_unmaps(dom, stream, &progress); } out: spin_unlock_irqrestore(&dart->batch_lock, flags); return progress.mapped; } +/* A buffer has one IOVA and one software mapping, regardless of how many + * display engines can scan it out. Forward whole batches to each DART; do + * not split them into per-page or per-stream transactions. + */ +static int apple_dart_x1n1_map_pages(struct iommu_domain *domain, + unsigned long iova, phys_addr_t paddr, + size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); + size_t done[X1N1_MAX_DARTS] = {}, total; + unsigned long flags; + unsigned int active = 0; + int ret; + + *mapped = 0; + if (!dom->num_darts) + return -ENODEV; + if (pgsize != DART_X1N1_PAGE_SIZE || !pgcount || + check_mul_overflow(pgsize, pgcount, &total) || + (iova | paddr) & (DART_X1N1_PAGE_SIZE - 1) || !paddr || + total - 1 > U64_MAX - paddr || + prot & ~((1U << X1N1_MAP_PROT_BITS) - 1) || + iova < domain->geometry.aperture_start || + iova > domain->geometry.aperture_end || + total - 1 > domain->geometry.aperture_end - iova || + iova + total > DART_X1N1_NATIVE_APERTURE_END) + return -EINVAL; + ret = x1n1_reserve_mappings(dom, iova, total, gfp); + if (ret) + return ret; + read_lock_irqsave(&dom->streams_lock, flags); + if (x1n1_retained_overlap(dom, iova, total)) { + ret = -EBUSY; + goto out; + } + for (unsigned int i = 0; i < dom->num_darts; i++) { + struct x1n1_stream_map *stream = &dom->streams[i]; + + if (bitmap_empty(stream->sids, stream->dart->num_streams)) + continue; + active++; + ret = x1n1_map_streams(dom, stream, iova, paddr, pgsize, pgcount, + prot, gfp, &done[i]); + if (ret) + break; + } + if (!active) + ret = -ENODEV; + if (!ret) { + x1n1_record_mapping(dom, iova, paddr, total, prot); + *mapped = total; + } else { + /* A later DART failed. Undo the successful prefixes on every + * engine before letting the DMA layer reuse this IOVA. + */ + for (unsigned int i = 0; i < dom->num_darts; i++) + if (done[i] && x1n1_unmap_streams(dom, &dom->streams[i], iova, + pgsize, done[i] / pgsize, NULL) != done[i]) + dev_err(dom->streams[i].dart->dev, + "failed to unwind multi-DART mapping\n"); + } +out: + read_unlock_irqrestore(&dom->streams_lock, flags); + if (ret) + x1n1_release_reservations(dom, iova, total); + return ret; +} + +static size_t apple_dart_x1n1_unmap_pages(struct iommu_domain *domain, + unsigned long iova, size_t pgsize, + size_t pgcount, + struct iommu_iotlb_gather *gather) +{ + struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); + size_t done[X1N1_MAX_DARTS] = {}, total, common; + unsigned long flags; + unsigned int active = 0; + + if (!dom->num_darts || pgsize != DART_X1N1_PAGE_SIZE || !pgcount || + check_mul_overflow(pgsize, pgcount, &total) || + (iova & (DART_X1N1_PAGE_SIZE - 1)) || + iova < domain->geometry.aperture_start || + iova > domain->geometry.aperture_end || + total - 1 > domain->geometry.aperture_end - iova) + return 0; + read_lock_irqsave(&dom->streams_lock, flags); + if (x1n1_retained_overlap(dom, iova, total)) { + common = 0; + goto out; + } + for (unsigned int i = 0; i < dom->num_darts; i++) { + struct x1n1_stream_map *stream = &dom->streams[i]; + + if (bitmap_empty(stream->sids, stream->dart->num_streams)) + continue; + active++; + done[i] = x1n1_unmap_streams(dom, stream, iova, pgsize, pgcount, gather); + if (done[i] != total) + break; + } + common = active ? total : 0; + for (unsigned int i = 0; i < dom->num_darts; i++) + if (!bitmap_empty(dom->streams[i].sids, dom->streams[i].dart->num_streams)) + common = min(common, done[i]); + /* Preserve the canonical mapping until all engines have agreed. On + * failure restore the suffix removed from earlier engines, retaining + * their existing leaf tables and full-sized transport batches. + */ + for (unsigned int i = 0; i < dom->num_darts; i++) { + struct x1n1_stream_map *stream = &dom->streams[i]; + struct apple_dart_x1n1 *dart = stream->dart; + struct x1n1_op suffix = { + .arg = { 0, iova + common, done[i] - common }, + }; + + if (done[i] <= common) + continue; + spin_lock(&dart->batch_lock); + if (x1n1_restore_partial_unmap(dom, stream, &suffix, + bitmap_weight(stream->sids, dart->num_streams))) + dev_err(dart->dev, "failed to restore multi-DART unmap suffix\n"); + spin_unlock(&dart->batch_lock); + } + x1n1_forget_mapping(dom, iova, common); +out: + read_unlock_irqrestore(&dom->streams_lock, flags); + return common; +} + static void apple_dart_x1n1_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { @@ -1010,14 +1179,17 @@ static void apple_dart_x1n1_domain_free(struct iommu_domain *domain) { struct apple_dart_x1n1_domain *dom = to_x1n1_domain(domain); - if (dom->has_retained && dom->dart) { - unsigned int sid; - unsigned long flags; + if (dom->has_retained) { + for (unsigned int i = 0; i < dom->num_darts; i++) { + struct x1n1_stream_map *stream = &dom->streams[i]; + unsigned int sid; + unsigned long flags; - spin_lock_irqsave(&dom->dart->batch_lock, flags); - for_each_set_bit(sid, dom->sids, dom->dart->num_streams) - x1n1_undo_replay(dom, sid, U64_MAX); - spin_unlock_irqrestore(&dom->dart->batch_lock, flags); + spin_lock_irqsave(&stream->dart->batch_lock, flags); + for_each_set_bit(sid, stream->sids, stream->dart->num_streams) + x1n1_undo_replay(dom, stream, sid, U64_MAX); + spin_unlock_irqrestore(&stream->dart->batch_lock, flags); + } } xa_destroy(&dom->l2_attached); xa_destroy(&dom->mappings); @@ -1028,29 +1200,25 @@ static struct iommu_domain * apple_dart_x1n1_domain_alloc_paging(struct device *dev) { struct apple_dart_x1n1_domain *dom; + int ret; dom = kzalloc(sizeof(*dom), GFP_KERNEL); if (!dom) return NULL; - mutex_init(&dom->lock); + rwlock_init(&dom->streams_lock); xa_init(&dom->l2_attached); xa_init(&dom->mappings); dom->domain.pgsize_bitmap = DART_X1N1_PAGE_SIZE; - dom->domain.geometry.aperture_start = 0; dom->domain.geometry.aperture_end = DART_X1N1_APERTURE_END; dom->domain.geometry.force_aperture = true; - - if (dev) { - struct apple_dart_x1n1_master *master = dev_iommu_priv_get(dev); - - if (master) { - dom->dart = master->dart; - dom->domain.geometry.aperture_start = master->dart->dma_min; - dom->domain.geometry.aperture_end = master->dart->dma_max; + if (dev && dev_iommu_priv_get(dev)) { + ret = x1n1_domain_configure(dom, dev_iommu_priv_get(dev)); + if (ret) { + kfree(dom); + return ERR_PTR(ret); } } - return &dom->domain; } @@ -1058,13 +1226,12 @@ static struct iommu_device *apple_dart_x1n1_probe_device(struct device *dev) { struct apple_dart_x1n1_master *master = dev_iommu_priv_get(dev); - if (!dev_iommu_fwspec_get(dev) || !master) + if (!dev_iommu_fwspec_get(dev) || !master || !master->num_darts) return ERR_PTR(-ENODEV); - - device_link_add(dev, master->dart->dev, - DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); - - return &master->dart->iommu; + for (unsigned int i = 0; i < master->num_darts; i++) + device_link_add(dev, master->streams[i].dart->dev, + DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); + return &master->streams[0].dart->iommu; } static void apple_dart_x1n1_release_device(struct device *dev) @@ -1094,22 +1261,25 @@ static int apple_dart_x1n1_of_xlate(struct device *dev, if (!dart) return -ENODEV; + if (sid >= dart->num_streams) + return -EINVAL; if (!master) { master = kzalloc(sizeof(*master), GFP_KERNEL); if (!master) return -ENOMEM; - master->dart = dart; dev_iommu_priv_set(dev, master); - } else if (master->dart != dart) { - /* This driver forwards per (dart-id, sid); a device spanning - * two paravirt DARTs is not modelled. */ - return -EINVAL; } - - if (sid >= dart->num_streams) + for (unsigned int i = 0; i < master->num_darts; i++) { + if (master->streams[i].dart == dart) { + set_bit(sid, master->streams[i].sids); + return 0; + } + } + if (master->num_darts == X1N1_MAX_DARTS) return -EINVAL; - - set_bit(sid, master->sids); + master->streams[master->num_darts].dart = dart; + set_bit(sid, master->streams[master->num_darts].sids); + master->num_darts++; return 0; } @@ -1125,12 +1295,42 @@ static void apple_dart_x1n1_release_group(void *data) unsigned int sid; mutex_lock(&apple_dart_x1n1_groups_lock); - for_each_set_bit(sid, group_master->sids, group_master->dart->num_streams) - group_master->dart->sid2group[sid] = NULL; + for (unsigned int i = 0; i < group_master->num_darts; i++) { + struct x1n1_stream_map *stream = &group_master->streams[i]; + + for_each_set_bit(sid, stream->sids, stream->dart->num_streams) + stream->dart->sid2group[sid] = NULL; + } mutex_unlock(&apple_dart_x1n1_groups_lock); kfree(group_master); } +static int x1n1_merge_master(struct apple_dart_x1n1_master *dst, + const struct apple_dart_x1n1_master *src) +{ + struct apple_dart_x1n1_master merged = *dst; + + for (unsigned int i = 0; i < src->num_darts; i++) { + const struct x1n1_stream_map *from = &src->streams[i]; + unsigned int j; + + for (j = 0; j < merged.num_darts; j++) + if (merged.streams[j].dart == from->dart) + break; + if (j == merged.num_darts) { + if (j == X1N1_MAX_DARTS) + return -EINVAL; + merged.streams[j] = *from; + merged.num_darts++; + } else { + bitmap_or(merged.streams[j].sids, merged.streams[j].sids, + from->sids, from->dart->num_streams); + } + } + *dst = merged; + return 0; +} + static struct iommu_group *apple_dart_x1n1_device_group(struct device *dev) { struct apple_dart_x1n1_master *master = dev_iommu_priv_get(dev); @@ -1139,20 +1339,22 @@ static struct iommu_group *apple_dart_x1n1_device_group(struct device *dev) struct iommu_group *result = ERR_PTR(-EINVAL); unsigned int sid; - if (!master || bitmap_empty(master->sids, master->dart->num_streams)) + if (!master || !master->num_darts) return result; - mutex_lock(&apple_dart_x1n1_groups_lock); - for_each_set_bit(sid, master->sids, master->dart->num_streams) { - struct iommu_group *existing = master->dart->sid2group[sid]; + for (unsigned int i = 0; i < master->num_darts; i++) { + struct x1n1_stream_map *stream = &master->streams[i]; - if (!existing) - continue; - if (group && existing != group) - goto out; - group = existing; - } + for_each_set_bit(sid, stream->sids, stream->dart->num_streams) { + struct iommu_group *existing = stream->dart->sid2group[sid]; + if (!existing) + continue; + if (group && existing != group) + goto out; + group = existing; + } + } if (group) { group = iommu_group_ref_get(group); } else { @@ -1171,13 +1373,7 @@ static struct iommu_group *apple_dart_x1n1_device_group(struct device *dev) result = ERR_PTR(-ENOMEM); goto out; } - - /* A PCI topology group may already contain another function. */ group_master = iommu_group_get_iommudata(group); - if (group_master && group_master->dart != master->dart) { - put_group = group; - goto out; - } if (!group_master) { group_master = kmemdup(master, sizeof(*master), GFP_KERNEL); if (!group_master) { @@ -1187,13 +1383,16 @@ static struct iommu_group *apple_dart_x1n1_device_group(struct device *dev) } iommu_group_set_iommudata(group, group_master, apple_dart_x1n1_release_group); - } else { - bitmap_or(group_master->sids, group_master->sids, master->sids, - master->dart->num_streams); + } else if (x1n1_merge_master(group_master, master)) { + put_group = group; + goto out; } + for (unsigned int i = 0; i < master->num_darts; i++) { + struct x1n1_stream_map *stream = &master->streams[i]; - for_each_set_bit(sid, master->sids, master->dart->num_streams) - master->dart->sid2group[sid] = group; + for_each_set_bit(sid, stream->sids, stream->dart->num_streams) + stream->dart->sid2group[sid] = group; + } result = group; out: mutex_unlock(&apple_dart_x1n1_groups_lock); @@ -1219,11 +1418,13 @@ static void apple_dart_x1n1_get_resv_regions(struct device *dev, struct apple_dart_x1n1_master *master = dev_iommu_priv_get(dev); unsigned int sid; - if (master) { + for (unsigned int i = 0; master && i < master->num_darts; i++) { + struct x1n1_stream_map *stream = &master->streams[i]; + /* SPTM converts both start and exclusive end with a strict 42-bit * address check (26A428: 270d63d8/270d640c). A range ending at * 4 TiB is rejected, so keep its final granule out of allocation. */ - if (master->dart->dma_max == DART_X1N1_NATIVE_APERTURE_END) { + if (stream->dart->dma_max == DART_X1N1_NATIVE_APERTURE_END) { struct iommu_resv_region *region = iommu_alloc_resv_region( DART_X1N1_NATIVE_APERTURE_END + 1 - @@ -1234,9 +1435,9 @@ static void apple_dart_x1n1_get_resv_regions(struct device *dev, if (region) list_add_tail(®ion->list, head); } - for_each_set_bit(sid, master->sids, master->dart->num_streams) { + for_each_set_bit(sid, stream->sids, stream->dart->num_streams) { struct x1n1_retained_sid *r = - master->dart->retained[sid]; + stream->dart->retained[sid]; if (!r) continue; for (unsigned int i = 0; i < r->count; i++) { @@ -1300,9 +1501,9 @@ int apple_dart_x1n1_set_power(struct device *dev, bool on) dev->iommu->iommu_dev->ops != &apple_dart_x1n1_ops) return -ENODEV; master = dev_iommu_priv_get(dev); - if (!master) + if (!master || master->num_darts != 1) return -ENODEV; - dart = master->dart; + dart = master->streams[0].dart; mutex_lock(&dart->init_lock); if (!dart->inited) { ret = -EINVAL; diff --git a/tools/testing/selftests/iommu/Makefile b/tools/testing/selftests/iommu/Makefile index 84abeb2f09495..5f3c7d075cb7c 100644 --- a/tools/testing/selftests/iommu/Makefile +++ b/tools/testing/selftests/iommu/Makefile @@ -7,4 +7,7 @@ TEST_GEN_PROGS := TEST_GEN_PROGS += iommufd TEST_GEN_PROGS += iommufd_fail_nth +TEST_PROGS += apple-dart-x1n1-map.py apple-dart-x1n1-group.py +TEST_FILES += apple-dart-x1n1-map.c + include ../lib.mk diff --git a/tools/testing/selftests/iommu/apple-dart-x1n1-group.py b/tools/testing/selftests/iommu/apple-dart-x1n1-group.py new file mode 100755 index 0000000000000..8b91264096865 --- /dev/null +++ b/tools/testing/selftests/iommu/apple-dart-x1n1-group.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +from pathlib import Path +import os, subprocess, tempfile +scratch = tempfile.TemporaryDirectory(prefix='apple-dart-x1n1-') +I = Path(scratch.name) +root = next((p for p in Path(__file__).resolve().parents if (p / 'drivers/iommu/apple-dart-x1n1.c').is_file()), None) +if os.environ.get('KERNEL_SRC'): + root = Path(os.environ['KERNEL_SRC']) +if root is None: + print('SKIP: set KERNEL_SRC to the Linux source tree') + raise SystemExit(4) +s=(root/'drivers/iommu/apple-dart-x1n1.c').read_text() +pre = r""" +#include +#include +#include +#include +#include +#include +#include +#define CONFIG_PCI 1 +#define GFP_KERNEL 0 +#define ERR_PTR(x) ((struct iommu_group *)(intptr_t)(x)) +#define IS_ERR(x) ((uintptr_t)(x) >= (uintptr_t)-4095) +#define DEFINE_MUTEX(n) int n +#define for_each_set_bit(i,bits,n) for(i=0;i<(n);i++) if ((bits)[i/64] & (1ul<<(i%64))) +static int locked; +static void mutex_lock(int*x) { assert(!*x);*x=1;locked++; } +static void mutex_unlock(int*x) { assert(*x);*x=0;locked--; } +static bool bitmap_empty(unsigned long*b,unsigned n) { for(unsigned i=0;im;} +static bool dev_is_pci(struct device*d) {return d->pci;} +static struct iommu_group*iommu_group_ref_get(struct iommu_group*g) {assert(g->refs);g->refs++;return g;} +static struct iommu_group*generic_device_group(struct device*d) { (void)d;struct iommu_group*g=calloc(1,sizeof(*g));assert(g);g->refs=1;return g; } +static struct iommu_group*pci_device_group(struct device*d) { return d->topology?iommu_group_ref_get(d->topology):generic_device_group(d); } +static void*iommu_group_get_iommudata(struct iommu_group*g) {return g->data;} +static void iommu_group_set_iommudata(struct iommu_group*g,void*p,void(*r)(void*)) {assert(!g->data);g->data=p;g->release=r;} +static void iommu_group_put(struct iommu_group*g) {assert(!locked);assert(g->refs);if(!--g->refs){if(g->release)g->release(g->data);free(g);}} +""" +pre=pre.replace('struct apple_dart_x1n1_master { struct apple_dart_x1n1*dart;unsigned long sids[4]; };',s[s.index('/* Match the native'):s.index('struct apple_dart_x1n1_domain {')]) +pre=pre.replace('#define CONFIG_PCI 1','#define CONFIG_PCI 1\n#define DART_X1N1_MAX_STREAMS 256\n#define DECLARE_BITMAP(n,size) unsigned long n[((size)+63)/64]') +pre=pre.replace('static bool bitmap_empty(unsigned long*b,unsigned n) { for(unsigned i=0;inum_darts=1;m->streams[0].dart=d;m->streams[0].sids[sid/64]|=1ul<<(sid%64); + return(struct device){m,true,g}; +} +static void add(struct device *dev,struct apple_dart_x1n1*d,unsigned sid){unsigned i=dev->m->num_darts++;assert(im->streams[i].dart=d;dev->m->streams[i].sids[sid/64]|=1ul<<(sid%64);} +int main(void) { + struct apple_dart_x1n1 darts[2]={{.num_streams=256},{.num_streams=256}}; + struct device a=mkdev(&darts[0],0,NULL);a.pci=false;add(&a,&darts[1],0); + struct device b=mkdev(&darts[1],0,NULL);b.pci=false; + struct iommu_group*ga=apple_dart_x1n1_device_group(&a),*gb=apple_dart_x1n1_device_group(&b); + assert(!IS_ERR(ga)&&ga==gb&&ga->refs==2&&darts[0].sid2group[0]==ga&&darts[1].sid2group[0]==ga); + iommu_group_put(ga);iommu_group_put(gb);assert(!darts[0].sid2group[0]&&!darts[1].sid2group[0]);free(a.m);free(b.m); + puts("PASS: multi-DART shared group and union cleanup"); + struct iommu_group*top=generic_device_group(NULL); + a=mkdev(&darts[0],2,top);b=mkdev(&darts[0],3,top); + ga=apple_dart_x1n1_device_group(&a);gb=apple_dart_x1n1_device_group(&b); + assert(ga==top&&gb==top&&darts[0].sid2group[2]==top&&darts[0].sid2group[3]==top); + iommu_group_put(ga);iommu_group_put(gb);iommu_group_put(top); + assert(!darts[0].sid2group[2]&&!darts[0].sid2group[3]);free(a.m);free(b.m); + puts("PASS: single-DART PCI multifunction topology and cleanup"); + a=mkdev(&darts[0],4,NULL);b=mkdev(&darts[1],5,NULL); + ga=apple_dart_x1n1_device_group(&a);gb=apple_dart_x1n1_device_group(&b);assert(ga!=gb); + struct device c=mkdev(&darts[0],4,NULL);add(&c,&darts[1],5); + assert(apple_dart_x1n1_device_group(&c)==ERR_PTR(-EINVAL)); + assert(darts[0].sid2group[4]==ga&&darts[1].sid2group[5]==gb); + iommu_group_put(ga);iommu_group_put(gb);free(a.m);free(b.m);free(c.m); + puts("PASS: conflicting live groups on separate DARTs rejected without mutation"); + struct apple_dart_x1n1 nine[9]={0};struct apple_dart_x1n1_master full={.num_darts=8}, extra={.num_darts=1}; + for(unsigned i=0;i<9;i++){nine[i].num_streams=256;if(i<8)full.streams[i].dart=&nine[i];}extra.streams[0].dart=&nine[8]; + struct apple_dart_x1n1_master before=full;assert(x1n1_merge_master(&full,&extra)==-EINVAL&&!memcmp(&before,&full,sizeof(full))); + puts("PASS: capacity failure leaves group ownership unchanged"); +} +''' +out=I/'test-dart-multigroup.c';out.write_text(pre+helper+post) +subprocess.run(['cc','-std=gnu11','-Wall','-Wextra','-Werror',str(out),'-o',str(I/'test-dart-multigroup')],check=True) +subprocess.run([str(I/'test-dart-multigroup')],check=True) diff --git a/tools/testing/selftests/iommu/apple-dart-x1n1-map.c b/tools/testing/selftests/iommu/apple-dart-x1n1-map.c new file mode 100644 index 0000000000000..0f40f6f546c70 --- /dev/null +++ b/tools/testing/selftests/iommu/apple-dart-x1n1-map.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0-only +static struct apple_dart_x1n1_domain dom, prior; +static struct x1n1_batch batches[2]; +static struct apple_dart_x1n1 darts[2]; +static struct apple_dart_x1n1_master master; +static struct device device={&master}; +static void reset(unsigned num) { + memset(&dom,0,sizeof(dom));memset(&prior,0,sizeof(prior)); + memset(hw,0,sizeof(hw));memset(root,0,sizeof(root));memset(enabled,0,sizeof(enabled));memset(leaves,0,sizeof(leaves)); + memset(darts,0,sizeof(darts));memset(&master,0,sizeof(master)); + fail_op=0;fail_sid=fail_dart=-1;fail_nth=seen=maps_sent=rings=map_rings=largest_map_batch=0;alloc_fail_after=-1; + master.num_darts=num; + for(unsigned i=0;i<2;i++){darts[i]=(struct apple_dart_x1n1){.num_streams=256,.dart_id=i,.dma_max=(1ul<<42)-1,.batch=&batches[i]};master.streams[i].dart=&darts[i];set_bit(0,master.streams[i].sids);} + dom.domain.owner=prior.domain.owner=(void*)&apple_dart_x1n1_ops; + dom.domain.ops=prior.domain.ops=apple_dart_x1n1_ops.default_domain_ops; +} +static void attach(void){assert(!apple_dart_x1n1_attach_dev(&dom.domain,&device,NULL));} +static void map(unsigned long iova,unsigned long pa,size_t pages){size_t done=0;assert(!apple_dart_x1n1_map_pages(&dom.domain,iova,pa,16384,pages,3,0,&done));assert(done==pages*16384);} +static void check(unsigned id,unsigned sid,unsigned long iova,unsigned long pa,size_t pages){for(unsigned i=0;i>14)+i)==xa_mk_value(x1n1_mapping_value(pa+i*16384,3)));} +int main(void){ + const unsigned long io=0x4000000,pa=0x10000000000UL;size_t done; + reset(2);darts[0].dma_min=0x10000;darts[1].dma_min=0x20000;darts[1].dma_max=0x3fffffff; + attach();assert(dom.domain.geometry.aperture_start==0x20000&&dom.domain.geometry.aperture_end==0x3fffffff); + map(io,pa,64);check(0,0,io,pa,64);check(1,0,io,pa,64);assert(dom.mappings.count==64&&maps_sent==2); + assert(apple_dart_x1n1_iova_to_phys(&dom.domain,io+57)==pa+57); + assert(apple_dart_x1n1_unmap_pages(&dom.domain,io,16384,64,NULL)==64*16384);assert(!dom.mappings.count&&!hw[0][0].count&&!hw[1][0].count); + map(io,pa,1);assert(dom.l2_attached.count==2); /* same block reused independently */ + puts("PASS: shared IOVA, one shadow, aperture intersection, per-DART leaves and reuse"); + reset(2);attach();fail_op=X1N1_DART_MAP;fail_dart=1;fail_nth=1; + assert(apple_dart_x1n1_map_pages(&dom.domain,io,pa,16384,64,3,0,&done)==-EIO);assert(!done&&!dom.mappings.count&&!hw[0][0].count&&!hw[1][0].count); + fail_op=0;map(io,pa,64);fail_op=X1N1_DART_UNMAP;fail_dart=1;fail_nth=1;seen=0; + assert(!apple_dart_x1n1_unmap_pages(&dom.domain,io,16384,64,NULL));check(0,0,io,pa,64);check(1,0,io,pa,64);assert(dom.mappings.count==64); + puts("PASS: second-DART map rollback and failed-unmap restoration"); + reset(2);set_bit(1,master.streams[1].sids);attach();map(io,pa,64); + fail_op=X1N1_DART_UNMAP;fail_dart=1;fail_sid=1;fail_nth=1; + assert(!apple_dart_x1n1_unmap_pages(&dom.domain,io,16384,64,NULL));check(0,0,io,pa,64);check(1,0,io,pa,64);check(1,1,io,pa,64); + puts("PASS: failure inside a stream cohort restores both controllers"); + reset(2);attach();map(io,pa,1);set_bit(1,master.streams[0].sids);set_bit(1,master.streams[1].sids); + fail_op=X1N1_DART_MAP;fail_dart=1;fail_sid=1;fail_nth=1; + assert(apple_dart_x1n1_attach_dev(&dom.domain,&device,NULL)==-EIO); + assert(!enabled[0][1]&&!enabled[1][1]&&!hw[0][1].count&&!hw[1][1].count); + check(0,0,io,pa,1);check(1,0,io,pa,1); + fail_op=0;attach();check(0,1,io,pa,1);check(1,1,io,pa,1); + puts("PASS: cross-DART attachment rollback and retry, original SIDs preserved"); + reset(2); + struct x1n1_retained_run run[2]={{.dva=io,.ipa=pa,.size=16384},{.dva=io,.ipa=pa+16384,.size=16384}}; + struct x1n1_retained_sid kept[2]={{.runs=&run[0],.count=1},{.runs=&run[1],.count=1}}; + darts[0].retained[0]=&kept[0];darts[1].retained[0]=&kept[1];attach(); + assert(apple_dart_x1n1_map_pages(&dom.domain,io,pa,16384,1,3,0,&done)==-EBUSY); + assert(!apple_dart_x1n1_iova_to_phys(&dom.domain,io));run[1].ipa=pa; + assert(apple_dart_x1n1_iova_to_phys(&dom.domain,io)==pa); + puts("PASS: inherited mappings reserved on either controller; translation requires agreement"); + reset(2);for(unsigned i=0;i<256;i++){set_bit(i,master.streams[0].sids);set_bit(i,master.streams[1].sids);}attach(); + map(io,pa,1);assert(maps_sent==512&&map_rings==4&&largest_map_batch==255); + assert(apple_dart_x1n1_unmap_pages(&dom.domain,io,16384,1,NULL)==16384); + puts("PASS: 512 streams retain full 255-op batches across batch boundaries"); + reset(2);attach();const unsigned long crossing=0x1ff8000;map(crossing,pa,4); + fail_op=X1N1_DART_UNMAP;fail_dart=1;fail_nth=2; + assert(apple_dart_x1n1_unmap_pages(&dom.domain,crossing,16384,4,NULL)==32768); + assert(dom.mappings.count==2&&hw[0][0].count==2&&hw[1][0].count==2); + check(0,0,crossing+32768,pa+32768,2);check(1,0,crossing+32768,pa+32768,2); + puts("PASS: partial unmap returns only the common prefix across 32-MiB boundaries"); + reset(2);for(unsigned i=0;i<256;i++){set_bit(i,master.streams[0].sids);set_bit(i,master.streams[1].sids);}attach(); + fail_op=X1N1_DART_MAP;fail_dart=1;fail_nth=256; + assert(apple_dart_x1n1_map_pages(&dom.domain,io,pa,16384,1,3,0,&done)==-EIO); + assert(!done&&!dom.mappings.count); + for(unsigned d=0;d<2;d++)for(unsigned sid=0;sid<256;sid++)assert(!hw[d][sid].count); + puts("PASS: second-DART failure across a 255-op batch boundary unwinds every SID"); + reset(2);attach();alloc_fail_after=3; + assert(apple_dart_x1n1_map_pages(&dom.domain,io,pa,16384,4,3,0,&done)==-ENOMEM); + assert(!done&&!dom.mappings.count&&!maps_sent); + puts("PASS: allocation failure has no hardware mapping side effects"); + reset(2);run[0].dva=run[1].dva=io+0x4000000;run[0].ipa=run[1].ipa=pa; + darts[0].retained[0]=&kept[0];darts[1].retained[0]=&kept[1]; + assert(!apple_dart_x1n1_attach_dev(&prior.domain,&device,NULL)); + assert(!apple_dart_x1n1_map_pages(&prior.domain,io,pa,16384,1,3,0,&done)); + assert(!apple_dart_x1n1_attach_dev(&dom.domain,&device,&prior.domain)); + assert(bitmap_empty(prior.streams[0].sids,256)&&bitmap_empty(prior.streams[1].sids,256)); + assert(!hw[0][0].count&&!hw[1][0].count);map(io,pa+16384,1); + check(0,0,io,pa+16384,1);check(1,0,io,pa+16384,1); + puts("PASS: retained-domain transfer reuses existing tables on both DARTs"); + reset(2);darts[0].retained[0]=&kept[0];darts[1].retained[0]=&kept[1]; + assert(!apple_dart_x1n1_attach_dev(&prior.domain,&device,NULL)); + assert(!apple_dart_x1n1_map_pages(&prior.domain,io,pa,16384,1,3,0,&done)); + fail_op=X1N1_DART_RETAINED_INFO;fail_dart=1;fail_nth=1; + assert(apple_dart_x1n1_attach_dev(&dom.domain,&device,&prior.domain)==-EIO); + assert(test_bit(0,prior.streams[0].sids)&&test_bit(0,prior.streams[1].sids)); + assert(bitmap_empty(dom.streams[0].sids,256)&&bitmap_empty(dom.streams[1].sids,256)); + check(0,0,io,pa,1);check(1,0,io,pa,1); + puts("PASS: failed retained-domain transfer restores both original streams"); + reset(1);attach();map(io,pa,64);assert(maps_sent==1&&map_rings==1);assert(apple_dart_x1n1_unmap_pages(&dom.domain,io,16384,64,NULL)==64*16384); + puts("PASS: single-DART bulk mapping still uses one map operation/doorbell"); + return 0; +} diff --git a/tools/testing/selftests/iommu/apple-dart-x1n1-map.py b/tools/testing/selftests/iommu/apple-dart-x1n1-map.py new file mode 100755 index 0000000000000..50d262298daf0 --- /dev/null +++ b/tools/testing/selftests/iommu/apple-dart-x1n1-map.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Run the production mapping/attachment functions with a fault-injecting DART transport.""" +from pathlib import Path +import os, subprocess, tempfile +scratch = tempfile.TemporaryDirectory(prefix='apple-dart-x1n1-') +I = Path(scratch.name) +root = next((p for p in Path(__file__).resolve().parents if (p / 'drivers/iommu/apple-dart-x1n1.c').is_file()), None) +if os.environ.get('KERNEL_SRC'): + root = Path(os.environ['KERNEL_SRC']) +if root is None: + print('SKIP: set KERNEL_SRC to the Linux source tree') + raise SystemExit(4) +src=(root/'drivers/iommu/apple-dart-x1n1.c').read_text() +pre = r""" +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint64_t u64;typedef uint32_t u32;typedef uint64_t phys_addr_t;typedef int gfp_t; +#define U64_MAX UINT64_MAX +#define DART_X1N1_PAGE_SHIFT 14 +#define DART_X1N1_PAGE_SIZE (1ul<<14) +#define DART_X1N1_BLOCK_SHIFT 25 +#define DART_X1N1_BLOCK_SIZE (1ul<<25) +#define DART_X1N1_APERTURE_END 0xffffffffUL +#define DART_X1N1_MAX_STREAMS 256 +#define X1N1_BATCH_MAX_OPS 255 +#define X1N1_MMIO_DART_DB 0x800 +#define X1N1_DART_INIT 1 +#define X1N1_DART_ATTACH 2 +#define X1N1_DART_MAP 3 +#define X1N1_DART_UNMAP 4 +#define X1N1_DART_ENABLE 5 +#define X1N1_DART_DISABLE 6 +#define GFP_KERNEL 0 +#define GFP_ATOMIC 0 +#define GFP_NOWAIT 0 +#define XA_PRESENT 0 +#define min(a,b) ((a)<(b)?(a):(b)) +#define min_t(t,a,b) min((t)(a),(t)(b)) +#define WARN_ON(x) assert(!(x)) +#define dev_err(d,...) do {(void)(d);fprintf(stderr,__VA_ARGS__);assert(0);}while(0) +#define DECLARE_BITMAP(n,size) unsigned long n[((size)+63)/64] +#define for_each_set_bit(i,b,n) for(i=0;i<(n);i++)if((b)[i/64]&(1ul<<(i%64))) +static void set_bit(unsigned i,unsigned long*b){b[i/64]|=1ul<<(i%64);} +static void clear_bit(unsigned i,unsigned long*b){b[i/64]&=~(1ul<<(i%64));} +static bool test_bit(unsigned i,unsigned long*b){return b[i/64]&(1ul<<(i%64));} +static void bitmap_zero(unsigned long*b,unsigned n){memset(b,0,((n+63)/64)*8);} +static unsigned bitmap_weight(unsigned long*b,unsigned n){unsigned v=0;for(unsigned i=0;i>1;} +static int xa_err(void*p){return (uintptr_t)p>=(uintptr_t)-4095?(int)(intptr_t)p:0;} +static void*xa_load(struct xarray*x,unsigned long i){for(unsigned n=0;ncount;n++)if(x->e[n].idx==i)return x->e[n].value;return NULL;} +static int xa_insert(struct xarray*x,unsigned long i,void*v,int g){(void)g;if(xa_load(x,i))return-EBUSY;if(alloc_fail_after==0)return-ENOMEM;if(alloc_fail_after>0)alloc_fail_after--;assert(x->count<4096);x->e[x->count++]=(struct xe){i,v};return 0;} +static void*xa_store(struct xarray*x,unsigned long i,void*v,int g){for(unsigned n=0;ncount;n++)if(x->e[n].idx==i){void*old=x->e[n].value;x->e[n].value=v;return old;}int r=xa_insert(x,i,v,g);return r?(void*)(intptr_t)r:NULL;} +static void*xa_erase(struct xarray*x,unsigned long i){for(unsigned n=0;ncount;n++)if(x->e[n].idx==i){void*v=x->e[n].value;x->e[n]=x->e[--x->count];return v;}return NULL;} +static void*xa_find(struct xarray*x,unsigned long*i,unsigned long max,int m){(void)m;unsigned long best=ULONG_MAX;void*v=NULL;for(unsigned n=0;ncount;n++)if(x->e[n].idx>=*i&&x->e[n].idx<=max&&x->e[n].idxe[n].idx;v=x->e[n].value;}if(v)*i=best;return v;} +struct x1n1_op {u32 op,flags;u64 arg[6],ret;}; +struct x1n1_batch {u32 count,flags;int status;u32 done;struct x1n1_op op[255];}; +struct iommu_domain{int ignored;};struct iommu_iotlb_gather{int ignored;}; +struct apple_dart_x1n1 {unsigned num_streams,dart_id;struct x1n1_batch*batch;void*dev;int batch_lock;}; +struct apple_dart_x1n1_master {struct apple_dart_x1n1*dart;DECLARE_BITMAP(sids,256);}; +struct device {struct apple_dart_x1n1_master*master;}; +struct apple_dart_x1n1_domain {struct iommu_domain domain;struct apple_dart_x1n1*dart;DECLARE_BITMAP(sids,256);int lock;struct xarray l2_attached,mappings;}; +static struct apple_dart_x1n1_domain*to_x1n1_domain(struct iommu_domain*d){return(void*)d;} +static void*dev_iommu_priv_get(struct device*d){return d->master;} +static int apple_dart_x1n1_ensure_init(struct apple_dart_x1n1*d){(void)d;return 0;} +static struct xarray hw[256];static bool enabled[256],root[256]; +static int fail_op,fail_sid=-1,fail_nth,seen;static unsigned maps_sent,rings; +static int x1n1_ring(struct apple_dart_x1n1*d,u32 off){ + assert(off==0x800+8*d->dart_id);struct x1n1_batch*b=d->batch;b->done=0;b->status=0;rings++; + for(unsigned n=0;ncount;n++){ + struct x1n1_op*o=&b->op[n];unsigned sid=o->arg[0];assert(sid<256); + if((int)o->op==fail_op&&(fail_sid<0||(int)sid==fail_sid)&&++seen==fail_nth){b->status=-EIO;return-EIO;} + if(o->op==X1N1_DART_ATTACH){if(o->arg[2]==1)root[sid]=true;} + else if(o->op==X1N1_DART_ENABLE){assert(root[sid]);enabled[sid]=true;} + else if(o->op==X1N1_DART_DISABLE)enabled[sid]=false; + else if(o->op==X1N1_DART_MAP){assert(root[sid]);maps_sent++;for(u64 j=0;jarg[3];j+=16384){unsigned long idx=(o->arg[1]+j)>>14;assert(!xa_load(&hw[sid],idx));assert(!xa_insert(&hw[sid],idx,xa_mk_value(((o->arg[2]+j)>>14)*256+o->arg[4]),0));}} + else if(o->op==X1N1_DART_UNMAP){for(u64 j=0;jarg[2];j+=16384)xa_erase(&hw[sid],(o->arg[1]+j)>>14);} + else assert(0); + b->done++; + } + return 0; +} +""" +pre=pre[:pre.index('struct iommu_domain{')] +pre='\n'.join(l for l in pre.splitlines() if not l.startswith('static void bitmap_zero'))+'\n' +pre=pre.replace('#define X1N1_BATCH_MAX_OPS 255','#define X1N1_BATCH_MAX_OPS 255') +pre+=''' +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +typedef u64 dma_addr_t; +#define DART_X1N1_NATIVE_APERTURE_END ((1ul<<42)-1) +#define X1N1_DART_RETAINED_INFO 9 +#define BITS_TO_LONGS(n) (((n)+63)/64) +#define max(a,b) ((a)>(b)?(a):(b)) +#define SINGLE_DEPTH_NESTING 1 +#define mutex_lock_nested(x,n) ((void)(x)) +#define spin_lock(x) ((void)(x)) +#define spin_unlock(x) ((void)(x)) +#define local_irq_save(f) do{f=0;}while(0) +#define local_irq_restore(f) ((void)(f)) +#define write_lock(x) ((void)(x)) +#define write_lock_nested(x,n) ((void)(x)) +#define write_unlock(x) ((void)(x)) +#define read_lock_irqsave(l,f) spin_lock_irqsave(l,f) +#define read_unlock_irqrestore(l,f) spin_unlock_irqrestore(l,f) +typedef int rwlock_t; +struct mutex {int unused;}; +struct iommu_domain {void *owner,*ops;struct {u64 aperture_start,aperture_end;bool force_aperture;} geometry;}; +struct iommu_iotlb_gather {int unused;}; +static const struct {void *default_domain_ops;} apple_dart_x1n1_ops={0}; +''' +pre+=src[src.index('struct x1n1_retained_run {'):src.index('struct apple_dart_x1n1 {')] +pre+='''struct apple_dart_x1n1 {unsigned num_streams,dart_id;struct x1n1_batch *batch;void *dev;int batch_lock;u64 dma_min,dma_max;struct x1n1_retained_sid *retained[256];};\n''' +pre+=src[src.index('/* Match the native'):src.index('/* Native IODARTMapper')] +pre+=''' +struct device {struct apple_dart_x1n1_master *master;}; +static void *dev_iommu_priv_get(struct device *d){return d->master;} +static int apple_dart_x1n1_ensure_init(struct apple_dart_x1n1*d){(void)d;return 0;} +static struct xarray hw[8][256]; +static bool enabled[8][256],root[8][256]; +static struct xarray leaves[8][256]; +static int fail_op,fail_dart=-1,fail_sid=-1,fail_nth,seen; +static unsigned maps_sent,rings,map_rings,largest_map_batch; +static int x1n1_ring(struct apple_dart_x1n1*d,u32 off){ + assert(off==0x800+8*d->dart_id);struct x1n1_batch*b=d->batch;b->done=0;b->status=0;rings++; + assert(b->count<=255); + if(b->op[0].op==X1N1_DART_MAP){map_rings++;largest_map_batch=max(largest_map_batch,b->count);} + for(unsigned n=0;ncount;n++){ + struct x1n1_op*o=&b->op[n];unsigned sid=o->arg[0],id=d->dart_id;assert(id<8&&sid<256); + if((int)o->op==fail_op&&(fail_dart<0||(int)id==fail_dart)&&(fail_sid<0||(int)sid==fail_sid)&&++seen==fail_nth){b->status=-EIO;return-EIO;} + if(o->op==X1N1_DART_ATTACH){ + if(o->arg[2]==1)root[id][sid]=true; + else{unsigned long key=o->arg[1]>>25;if(!d->retained[sid])assert(!xa_load(&leaves[id][sid],key));if(!xa_load(&leaves[id][sid],key))assert(!xa_insert(&leaves[id][sid],key,xa_mk_value(1),0));} + } + else if(o->op==X1N1_DART_ENABLE){assert(root[id][sid]);enabled[id][sid]=true;} + else if(o->op==X1N1_DART_DISABLE)enabled[id][sid]=false; + else if(o->op==X1N1_DART_RETAINED_INFO){assert(d->retained[sid]);root[id][sid]=enabled[id][sid]=true;} + else if(o->op==X1N1_DART_MAP){ + assert(root[id][sid]);maps_sent++; + for(u64 j=0;jarg[3];j+=16384){unsigned long idx=(o->arg[1]+j)>>14;assert(!xa_load(&hw[id][sid],idx));assert(!xa_insert(&hw[id][sid],idx,xa_mk_value(((o->arg[2]+j)>>14)*256+o->arg[4]),0));} + } + else if(o->op==X1N1_DART_UNMAP){for(u64 j=0;jarg[2];j+=16384)xa_erase(&hw[id][sid],(o->arg[1]+j)>>14);} + else assert(0); + b->done++; + } + return 0; +} +''' +pre+=src[src.index('static const struct x1n1_retained_run *'):src.index('/* Bring the DART up')] +helper=src[src.index('/* Live mappings are indexed'):src.index('static void apple_dart_x1n1_iotlb_sync')] +helper+=src[src.index('static phys_addr_t apple_dart_x1n1_iova_to_phys'):src.index('static void apple_dart_x1n1_domain_free')] +post=Path(__file__).with_name('apple-dart-x1n1-map.c').read_text() +out=I/'test-dart-multidart.c';out.write_text(pre+helper+post) +subprocess.run(['cc','-std=gnu11','-O1','-g','-Wall','-Wextra','-Wno-unused-parameter','-Werror',str(out),'-o',str(I/'test-dart-multidart')],check=True) +subprocess.run([str(I/'test-dart-multidart')],check=True) From 3d7667213564a066dfd06c1e7dd2dddd7def98f2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 15:52:09 -0400 Subject: [PATCH 0226/1579] drm/apple: bootstrap external T6050 DCP on firmware 27 --- Documentation/gpu/apple-dcp-t6050.rst | 69 ++++++++ drivers/gpu/drm/apple/afk.c | 153 ++++++++++++++++++ drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 54 +++++-- drivers/gpu/drm/apple/dptxep.c | 3 +- drivers/gpu/drm/apple/iomfb_template.c | 33 +++- drivers/gpu/drm/apple/parser.c | 11 +- .../selftests/drivers/gpu/apple-dcp-afkv2.py | 100 ++++++++++++ .../drivers/gpu/apple-dcp-native-boot.py | 61 +++++++ .../drivers/gpu/apple-dcp-service-init.py | 54 +++++++ 10 files changed, 523 insertions(+), 16 deletions(-) create mode 100755 tools/testing/selftests/drivers/gpu/apple-dcp-afkv2.py create mode 100755 tools/testing/selftests/drivers/gpu/apple-dcp-native-boot.py create mode 100755 tools/testing/selftests/drivers/gpu/apple-dcp-service-init.py diff --git a/Documentation/gpu/apple-dcp-t6050.rst b/Documentation/gpu/apple-dcp-t6050.rst index e1db6d5064059..d3cf0ea0fb8e3 100644 --- a/Documentation/gpu/apple-dcp-t6050.rst +++ b/Documentation/gpu/apple-dcp-t6050.rst @@ -215,3 +215,72 @@ request is translated in reverse to the corresponding CPU resource before calling dma_map_resource. The HV/IOMMU resource-DMA path must preserve that distinction when forwarding to SPTM. Successful internal scanout does not qualify arbitrary additional MMIO ranges or external display pipelines. + +AFKV2 commands from firmware +--------------------------- + +26A428 AFKEPCommandRemote uses an 8-byte command context followed, when +flag bit0 is set, by a 24-byte DMA descriptor. Descriptor presence does not +require both directions to use DMA: a zero txlen leaves the request inline, +and a zero rxlen leaves the response inline. DCPAV messages inside this +context have a 64-byte header, unlike the older 68-byte AP-call envelope. +The native implementations at 0xfffffe00091f6f34, 0xfffffe00091f7080, +0xfffffe00091f71e0 and 0xfffffe000a13bac0 establish these layouts. + +The receive path translates each DMA page through the firmware device's +IOMMU domain, allowing physically scattered buffers and inherited shared +aliases. Replies retain the command tag; descriptor replies report the +actual response length. Invalid requests and unavailable services receive +native transport error statuses. The existing service handlers still own +DCPAV command semantics. This does not reduce the outgoing command slots +or serialize unrelated commands. + +Disconnected external pipeline boot +----------------------------------- + +26A428 UnifiedPipeline2::start_hardware_boot at 0xfffffe00097b1600 only +queries default-framebuffer information if A454 returns true. It next +queries configured geometry with A103. Nonzero geometry needs no A104/A105; +otherwise A453 supplies geometry and its result is forwarded to A104, then +committed by A105, including the zero geometry of an external pipeline +without a timing. Rejecting that zero result fails the D120 boot callback. +The 27.0 path follows those branches; older firmware paths are unchanged. + +Native T6050 dispext clock-ids is explicitly empty. An external profile +therefore accepts an empty clock table, while missing or inconsistent clock +metadata remains an error. Its bandwidth request uses its own PMC agent: +ApplePMC::getBWRMemory matches the BIRQ argument against 24-byte bwr-agent +entries, and AppleT6050PMC::getBWRMemoryOffset returns 0x20000+(agent<<14). +On J714s, dispext0 request54 selects agent3 at PMC+0x2c000; disp0 request150 +selects agent0 at PMC+0x20000. A guest must grant and describe the correct +page rather than reuse the internal display's request slot. + +Standalone source-based regression checks run on the build host:: + + python3 tools/testing/selftests/drivers/gpu/apple-dcp-afkv2.py + python3 tools/testing/selftests/drivers/gpu/apple-dcp-native-boot.py + +They exercise native wire forms, scatter-backed DMA, allocation and mapping +failures, response capacities, and the disconnected/native boot branches +with sanitizers. They do not emulate DCP firmware or qualify optical output. + +The external 26A428 DPTX endpoint is 0x2c. Its captured announcements expose +``EPICUnit`` 0 and 1 alongside ``interface-id`` 1 and 3. These fields are +not interchangeable: the explicit unit must take precedence regardless of +dictionary order. The lower-case H17P form still uses interface-id when an +explicit EPICUnit is absent. A focused regression test covers both native +port/channel pairs, reversed property order, and that fallback:: + + python3 tools/testing/selftests/drivers/gpu/apple-dcp-service-init.py + +Endpoint 0x24 advertises ``dcpav-controller-epic`` and +``dcpdp-controller-epic``. Those controller interfaces must not be treated +as the older ``dcpav-service-epic`` EDID service. Their routing and service +creation protocol still needs separate integration. External IOMFB boot +and DPTX announcements alone do not establish HDMI or USB-C scanout. + +On J714s hardware, the corrected 27.0 startup completes for both internal +and external0 DCP. A read-only driver-state snapshot confirms both external +DPTX ports enabled with units0/1 and initialized service cookies; the earlier +port0 initialization timeout is gone. HDMI and USB-C routing, incoming DPTX +link-training callbacks, and external scanout remain unqualified. diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 1629868cda247..d3c0176411e35 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -5,6 +5,10 @@ #include #include #include +#include +#include +#include +#include #include #include #include @@ -548,6 +552,144 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, payload_size, true); } +/* The remote command can use an AFK DMA buffer or inline bytes independently + * for each direction. Copy page by page: a DMA-contiguous buffer need not be + * CPU-physically contiguous. Retained firmware buffers use their granted WB + * aliases; ordinary Linux-owned buffers resolve through the same IOMMU API. + */ +static int afk_copy_remote(struct apple_dcp_afkep *ep, u64 dva, void *buffer, + size_t size, bool to_remote) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(ep->dcp->dev); + size_t off = 0; + + if (!domain || !dva || size > U64_MAX - dva) + return -EINVAL; + while (off < size) { + phys_addr_t pa = iommu_iova_to_phys(domain, dva + off); + size_t chunk = min_t(size_t, size - off, + PAGE_SIZE - ((dva + off) & ~PAGE_MASK)); + void *cpu; + + if (!pa) { + dev_err_ratelimited(ep->dcp->dev, + "AFK[ep:%02x]: unmapped remote buffer %#llx + %#zx (%s)\n", + ep->endpoint, dva, off, to_remote ? "reply" : "request"); + return -ENXIO; + } + cpu = memremap(pa, chunk, MEMREMAP_WB); + if (!cpu) + return -ENOMEM; + if (to_remote) + memcpy(cpu, buffer + off, chunk); + else + memcpy(buffer + off, cpu, chunk); + memunmap(cpu); + off += chunk; + } + return 0; +} + +/* 26A428 AFKEPCommandRemote and DCPAVProxy: 8-byte command context, optional + * 24-byte DMA descriptors, then a 64-byte DCPAVIPC::Message. Reply status is + * in the command context; service-specific status stays in the message data. + */ +static void afk_recv_command_v2(struct apple_dcp_afkep *ep, u32 channel, u8 subtype, + const void *payload, size_t payload_size) +{ + const size_t inline_header = offsetof(struct epic_cmd_v2, rxbuf); + struct epic_cmd_v2 context = {}; + struct apple_epic_service *service = afk_epic_find_service(ep, channel); + struct epic_service_call *call; + void *request = NULL, *reply = NULL; + size_t header_size = inline_header, request_size, call_size; + u32 status = 0xe00002c2; /* kIOReturnBadArgument */ + int ret; + + if (payload_size < inline_header) + return; /* There is no complete command tag to answer. */ + memcpy(&context, payload, inline_header); + if (context.flags & 1) { + header_size = sizeof(context); + if (payload_size < header_size) + return; + memcpy(&context, payload, header_size); + } + if (!service || service->torndown || !service->ops || !service->ops->call || + subtype != EPIC_SUBTYPE_STD_SERVICE) { + status = 0xe00002d7; /* Native AFKV2 no-handler response. */ + goto error; + } + request_size = (context.flags & 1) && context.txlen ? + le32_to_cpu(context.txlen) : payload_size - header_size; + if (request_size < sizeof(*call)) + goto error; + request = kvmalloc(request_size, GFP_KERNEL); + if (!request) { + status = 0xe00002bd; /* kIOReturnNoMemory */ + goto error; + } + if ((context.flags & 1) && context.txlen) { + dma_rmb(); + ret = afk_copy_remote(ep, le64_to_cpu(context.txbuf), request, + request_size, false); + if (ret) + goto error; + } else { + memcpy(request, payload + header_size, request_size); + } + call = request; + call_size = le32_to_cpu(call->data_len); + if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || + call_size != request_size - sizeof(*call)) + goto error; + if ((context.flags & 1) && context.rxlen && + (request_size > le32_to_cpu(context.length) || + request_size > le32_to_cpu(context.rxlen))) { + status = 0xe00002e1; /* Native AFKV2 response-capacity error. */ + goto error; + } + reply = kvzalloc(header_size + request_size, GFP_KERNEL); + if (!reply) { + status = 0xe00002bd; + goto error; + } + memcpy(reply + header_size, call, sizeof(*call)); + ret = service->ops->call(service, le32_to_cpu(call->command), + request + sizeof(*call), call_size, + reply + header_size + sizeof(*call), call_size); + if (ret) { + dev_warn(ep->dcp->dev, "AFK[ep:%02x]: channel %u command %u failed: %d\n", + ep->endpoint, channel, le32_to_cpu(call->command), ret); + goto error; + } + if ((context.flags & 1) && context.rxlen) { + ret = afk_copy_remote(ep, le64_to_cpu(context.rxbuf), + reply + header_size, request_size, true); + if (ret) + goto error; + context.rxlen = cpu_to_le32(request_size); + request_size = 0; /* Only the descriptor goes into the ring. */ + } + context.length = 0; + memcpy(reply, &context, header_size); + dma_wmb(); + ret = afk_send_epic(ep, channel, context.tag, EPIC_TYPE_REPLY, EPIC_CAT_REPLY, + subtype, reply, header_size + request_size); + goto done; +error: + context.length = cpu_to_le32(status); + context.rxlen = 0; + ret = afk_send_epic(ep, channel, context.tag, EPIC_TYPE_REPLY, EPIC_CAT_REPLY, + subtype, &context, header_size); +done: + if (ret) + dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %u reply failed: %d\n", + ep->endpoint, channel, ret); + kvfree(reply); + kvfree(request); +} + static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, u8 *data, size_t data_size) { @@ -609,6 +751,17 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, const struct epic_hdr_v2 *v2 = (const void *)data; /* The queue type is no longer the command/reply discriminator. */ + if (v2->category == 1) { + size_t length = le32_to_cpu(v2->length); + + if (length < sizeof(*v2) - 8 || length > data_size - 8) + goto too_small; + h17p_eshdr.category = EPIC_CAT_COMMAND; + trace_afk_recv_handle(ep, channel, EPIC_TYPE_COMMAND, + data_size, ehdr, eshdr); + return afk_recv_command_v2(ep, channel, v2->type, payload, + length - (sizeof(*v2) - 8)); + } if (v2->category == 2) { const struct epic_cmd_v2 *reply = (const void *)payload; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 34be5c91b98c6..46a4853c2b947 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -82,6 +82,7 @@ enum { DPAVSERV_ENDPOINT = 0x28, AV_ENDPOINT = 0x29, DPTX_ENDPOINT = 0x2a, + DPTX_ENDPOINT_V27 = 0x2c, HDCP_ENDPOINT = 0x2b, REMOTE_ALLOC_ENDPOINT = 0x2d, IOMFB_ENDPOINT = 0x37, diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 3b62ac59156ea..c52c8cab16d4d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1276,13 +1276,14 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) goto unbacked; return; case DPAVSERV_ENDPOINT: - if (dcp->dcpavservep) + if (dcp->dcpavservep && dcp->dcpavservep->endpoint == endpoint) afk_receive_message(dcp->dcpavservep, message); else goto unbacked; return; + case DPTX_ENDPOINT_V27: case DPTX_ENDPOINT: - if (dcp->dptxep) + if (dcp->dptxep && dcp->dptxep->endpoint == endpoint) afk_receive_message(dcp->dptxep, message); else goto unbacked; @@ -1843,7 +1844,12 @@ int dcp_start(struct platform_device *pdev) ret); } - if (dcp->phy && dcp->fw_compat >= DCP_FIRMWARE_V_13_5) { + /* Firmware 27 advertises DPTX services before a physical route is + * assigned. Discover them without requiring an already-selected PHY. + */ + if ((dcp->phy || (dcp->fw_compat == DCP_FIRMWARE_V_27_0 && + dcp->hw.num_dptx_ports)) && + dcp->fw_compat >= DCP_FIRMWARE_V_13_5) { ret = ibootep_init(dcp); if (ret) dev_warn(dcp->dev, "Failed to start IBOOT endpoint: %d\n", @@ -2764,19 +2770,29 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) ret = devm_clk_bulk_get_all(dev, &dcp->firmware_clocks); if (ret < 0) return dev_err_probe(dev, ret, "failed to acquire firmware clocks\n"); - if (!ret || count != ret) + /* T6050 dispext has an explicitly empty native clock-ID table. + * Missing metadata is still an error; do not borrow disp0's rates. + */ + if (count == -ENODATA && of_property_present(dev->of_node, "apple,clock-ids")) + count = 0; + if (count < 0 || count != ret) return dev_err_probe(dev, -EINVAL, "firmware clock IDs must match clocks\n"); dcp->num_firmware_clocks = count; - dcp->firmware_clock_ids = devm_kcalloc(dev, count, sizeof(u32), GFP_KERNEL); - if (!dcp->firmware_clock_ids) - return -ENOMEM; - ret = of_property_read_u32_array(dev->of_node, "apple,clock-ids", - dcp->firmware_clock_ids, count); - if (ret) - return ret; + if (count) { + dcp->firmware_clock_ids = devm_kcalloc(dev, count, sizeof(u32), GFP_KERNEL); + if (!dcp->firmware_clock_ids) + return -ENOMEM; + ret = of_property_read_u32_array(dev->of_node, "apple,clock-ids", + dcp->firmware_clock_ids, count); + if (ret) + return ret; + } } - dcp->clk = devm_clk_get(dev, NULL); + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0 && !dcp->num_firmware_clocks) + dcp->clk = devm_clk_get_optional(dev, NULL); + else + dcp->clk = devm_clk_get(dev, NULL); if (IS_ERR(dcp->clk)) return dev_err_probe(dev, PTR_ERR(dcp->clk), "Unable to find clock\n"); @@ -3300,6 +3316,19 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t6050 = { .firmware_compat = DCP_FIRMWARE_V_27_0, }; +/* The saved native T6050 topology has two DPTX ports. Use the + * same AFKV2/RTKit session contract as the internal T6050 controller. + */ +static const struct apple_dcp_hw_data apple_dcp_hw_t6050_dcpext = { + .firmware_dma_bits = 41, + .num_dptx_ports = 2, + .iomfb_method_profile = DCP_IOMFB_METHODS_H17G, + .adopt_running_coprocessor = true, + .adopt_live_session = true, + .discover_live_session = true, + .firmware_compat = DCP_FIRMWARE_V_27_0, +}; + static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { .num_dptx_ports = 0, }; @@ -3313,6 +3342,7 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, { .compatible = "apple,t8140-dcp", .data = &apple_dcp_hw_t8140, }, { .compatible = "apple,t6050-dcp", .data = &apple_dcp_hw_t6050, }, + { .compatible = "apple,t6050-dcpext", .data = &apple_dcp_hw_t6050_dcpext, }, { .compatible = "apple,t8142-dcp", .data = &apple_dcp_hw_t8142, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 11091084f05e2..0702c889a85a1 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -856,7 +856,8 @@ int dptxep_init(struct apple_dcp *dcp) init_completion(&dcp->dptxport[0].linkcfg_completion); init_completion(&dcp->dptxport[1].linkcfg_completion); - dcp->dptxep = afk_init(dcp, DPTX_ENDPOINT, dptxep_ops); + dcp->dptxep = afk_init(dcp, dcp->fw_compat == DCP_FIRMWARE_V_27_0 ? + DPTX_ENDPOINT_V27 : DPTX_ENDPOINT, dptxep_ops); if (IS_ERR(dcp->dptxep)) return PTR_ERR(dcp->dptxep); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 3de70b396b722..93486511c0818 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1196,7 +1196,15 @@ static void boot_1_10(struct apple_dcp *dcp, void *out, void *cookie) * The H17G capture is2560x1664; the H17P capture is2408x1506. */ memcpy(&dim, out, sizeof(dim)); - if (!dim.width || !dim.height || dim.width > 16384 || dim.height > 16384) { + /* 26A428 start_hardware_boot forwards A453's result to A104 even + * when an external display has no timing yet (0 x 0). Do not turn + * that native disconnected state into a failed D120 boot callback. + */ + if (dim.width > 16384 || dim.height > 16384 +#if DCP_FW_VER < DCP_FW_VERSION(27, 0, 0) + || !dim.width || !dim.height +#endif + ) { struct dcp_channel *ch = &dcp->ch_cb; dev_err(dcp->dev, "invalid default framebuffer dimensions %ux%u\n", @@ -1215,6 +1223,18 @@ static void boot_1_9(struct apple_dcp *dcp, void *out, void *cookie) { u32 zero = 0; +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + struct dcp_dfb_dimensions_h17p dim; + + /* Native get_config_frame_size (A103) precedes the A453 fallback. + * A configured frame needs neither replacement dimensions nor A105. + */ + memcpy(&dim, out, sizeof(dim)); + if (dim.width && dim.height) { + boot_2(dcp, out, cookie); + return; + } +#endif dcp_get_dfb_state(dcp, false, &zero, boot_1_10, NULL); } @@ -1248,6 +1268,17 @@ static void boot_1_6(struct apple_dcp *dcp, void *out, void *cookie) { u64 zero = 0; +#if DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0) + /* Native only calls setup_default_fb_info when A454 returns true. + * A disconnected external pipeline has no default framebuffer. + */ + if (!(*(u8 *)out & 1)) { + struct dcp_dfb_layout_req_h17p req = {}; + + dcp_get_dfb_layout(dcp, false, &req, boot_1_9, NULL); + return; + } +#endif dcp_get_dfb_compression_info(dcp, false, &zero, boot_1_7, NULL); } #endif diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 86f61db291f9a..60fac7de1adab 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -653,6 +653,7 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, int ret = 0; struct iterator it; bool parsed_unit = false; + bool explicit_unit = false; bool parsed_name = false; bool parsed_class = false; @@ -690,10 +691,16 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, else parsed_class = true; } else if (!strcmp(key, "EPICUnit") || - !strcmp(key, "interface-id")) { + (!explicit_unit && !strcmp(key, "interface-id"))) { + /* AFKV2 can supply both: EPICUnit identifies a port, + * interface-id identifies its transport channel. Only use + * the latter as a fallback for services without EPICUnit. + */ ret = parse_int(it.handle, unit); - if (!ret) + if (!ret) { parsed_unit = true; + explicit_unit |= !strcmp(key, "EPICUnit"); + } } else { skip(it.handle); } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp-afkv2.py b/tools/testing/selftests/drivers/gpu/apple-dcp-afkv2.py new file mode 100755 index 0000000000000..1d7085e369dd3 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp-afkv2.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +from pathlib import Path +import os, subprocess, tempfile +scratch = tempfile.TemporaryDirectory(prefix='apple-dcp-test-') +I = Path(scratch.name) +root = next((p for p in Path(__file__).resolve().parents + if (p / 'drivers/gpu/drm/apple/afk.c').is_file()), None) +if os.environ.get('KERNEL_SRC'): + root = Path(os.environ['KERNEL_SRC']) +if root is None: + print('SKIP: set KERNEL_SRC to the Linux source tree') + raise SystemExit(4) +S=(root/'drivers/gpu/drm/apple') +s=(S/'afk.c').read_text();h=(S/'afk.h').read_text() +pre=r''' +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8;typedef uint16_t u16;typedef uint32_t u32;typedef uint64_t u64,phys_addr_t; +typedef u16 __le16;typedef u32 __le32;typedef u64 __le64; +#define __packed __attribute__((packed)) +#define static_assert _Static_assert +#define U64_MAX UINT64_MAX +#define PAGE_SIZE 4096ul +#define PAGE_MASK (~(PAGE_SIZE-1)) +#define GFP_KERNEL 0 +#define MEMREMAP_WB 1 +#define min_t(t,a,b) ((t)(a)<(t)(b)?(t)(a):(t)(b)) +#define le16_to_cpu(x) (x) +#define le32_to_cpu(x) (x) +#define le64_to_cpu(x) (x) +#define cpu_to_le16(x) (x) +#define cpu_to_le32(x) (x) +#define cpu_to_le64(x) (x) +#define dma_rmb() ((void)0) +#define dma_wmb() ((void)0) +#define dev_err(d,...) ((void)(d)) +#define dev_warn(d,...) ((void)(d)) +#define dev_err_ratelimited(d,...) ((void)(d)) +static int fail_alloc, allocations; +static void *kvmalloc(size_t n,int g){(void)g;if(++allocations==fail_alloc)return NULL;return malloc(n);} +static void *kvzalloc(size_t n,int g){void *p=kvmalloc(n,g);if(p)memset(p,0,n);return p;} +#define kvfree free +struct iommu_domain {int unused;};struct device{int unused;}; +struct apple_dcp{struct device *dev;};struct apple_dcp_afkep{struct apple_dcp *dcp;u32 endpoint;}; +struct apple_epic_service; +struct apple_epic_service_ops {int (*call)(struct apple_epic_service *,u32,const void*,size_t,void*,size_t);}; +struct apple_epic_service {bool torndown;const struct apple_epic_service_ops *ops;}; +static struct apple_epic_service service; +static bool service_missing;static int handler_calls,handler_error; +static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep *e,u32 c){(void)e;assert(c==5);return service_missing?NULL:&service;} +static u8 pages[4][PAGE_SIZE];static bool mapped[4]; +static struct iommu_domain domain; +static struct iommu_domain *iommu_get_domain_for_dev(struct device*d){(void)d;return &domain;} +static phys_addr_t iommu_iova_to_phys(struct iommu_domain*d,u64 va){(void)d;if(va<0x100000||va>=0x104000||!mapped[(va-0x100000)/PAGE_SIZE])return 0;return 0x10000+((va-0x100000)/PAGE_SIZE)*0x2000+(va&(PAGE_SIZE-1));} +static void *memremap(phys_addr_t pa,size_t n,int flags){assert(flags==MEMREMAP_WB);unsigned p=(pa-0x10000)/0x2000,off=(pa-0x10000)%0x2000;assert(p<4&&off+n<=PAGE_SIZE);return &pages[p][off];} +static void memunmap(void*p){(void)p;} +static u8 sent[32768];static size_t sent_size;static u8 sent_type;static unsigned sent_count; +static int afk_send_epic(struct apple_dcp_afkep *e,u32 c,u16 tag,int et,int cat,u8 type,const void*p,size_t n){(void)e;assert(c==5&&tag==0xa7&&et==4&&cat==0x20&&n<=sizeof(sent));memcpy(sent,p,n);sent_size=n;sent_type=type;sent_count++;return 0;} +''' +# Include production wire structs/constants, then the production copy/dispatch functions. +wire=h[h.index('struct epic_cmd_v2 {'):h.index('struct afk_ringbuffer {')] +code=s[s.index('static int afk_copy_remote'):s.index('static void afk_recv_handle(struct apple_dcp_afkep')] +post=r''' +static struct device dev;static struct apple_dcp dcp={&dev};static struct apple_dcp_afkep ep={&dcp,0x2c}; +static u8 packet[8192], message[8192]; +static int handler(struct apple_epic_service*s,u32 cmd,const void*in,size_t n,void*out,size_t out_n){(void)s;assert(cmd==10&&n==32&&out_n==32&&((const u8*)in)[8]==0x5a);handler_calls++;memcpy(out,in,n);*(u32*)out=0;*(u64*)((u8*)out+16)=4;return handler_error;} +static const struct apple_epic_service_ops ops={handler}; +static void reset(unsigned flags){memset(packet,0,sizeof(packet));memset(message,0,sizeof(message));memset(pages,0,sizeof(pages));for(int i=0;i<4;i++)mapped[i]=true;service=(struct apple_epic_service){.ops=&ops};service_missing=false;handler_calls=handler_error=allocations=fail_alloc=sent_count=0;sent_size=0;struct epic_cmd_v2*c=(void*)packet;c->flags=flags;c->tag=0xa7;c->length=96;struct epic_service_call*m=(void*)message;m->command=10;m->data_len=32;m->magic=EPIC_SERVICE_CALL_MAGIC;message[64+8]=0x5a;} +static void call(size_t n){afk_recv_command_v2(&ep,5,EPIC_SUBTYPE_STD_SERVICE,packet,n);assert(sent_count==1);} +static u32 status(void){return ((struct epic_cmd_v2*)sent)->length;} +static void check_reply(const u8*p){const struct epic_service_call*m=(const void*)p;assert(m->magic==EPIC_SERVICE_CALL_MAGIC&&m->data_len==32&&m->command==10);assert(*(u64*)(p+64+16)==4&&p[64+8]==0x5a);} +int main(void){ + reset(0);memcpy(packet+8,message,96);call(104);assert(!status()&&handler_calls==1&&sent_size==104);check_reply(sent+8);puts("PASS: native 8-byte context + 64-byte DCPAV inline command/reply"); + reset(1);struct epic_cmd_v2*c=(void*)packet;c->txbuf=0x100ff8;c->txlen=96;c->rxbuf=0x102ff0;c->rxlen=128; + assert(!afk_copy_remote(&ep,c->txbuf,message,96,true));call(32);assert(!status()&&handler_calls==1&&sent_size==32&&((struct epic_cmd_v2*)sent)->rxlen==96); + assert(!afk_copy_remote(&ep,c->rxbuf,message,96,false));check_reply(message);puts("PASS: scatter-backed OOB request/reply crossing CPU page boundaries"); + reset(1);c=(void*)packet;c->rxbuf=0x102ff0;c->rxlen=96;memcpy(packet+32,message,96);call(128);assert(!status()&&sent_size==32);assert(!afk_copy_remote(&ep,c->rxbuf,message,96,false));check_reply(message);puts("PASS: inline request with OOB response"); + reset(1);c=(void*)packet;c->txbuf=0x100ff8;c->txlen=96;assert(!afk_copy_remote(&ep,c->txbuf,message,96,true));call(32);assert(!status()&&sent_size==128);check_reply(sent+32);puts("PASS: OOB request with inline response"); + reset(1);memcpy(packet+32,message,96);call(128);assert(!status()&&sent_size==128);check_reply(sent+32);puts("PASS: OOB descriptor with zero DMA lengths retains inline data"); + reset(1);c=(void*)packet;c->rxbuf=0x102000;c->rxlen=95;memcpy(packet+32,message,96);call(128);assert(status()==0xe00002e1&&!handler_calls&&sent_size==32);puts("PASS: insufficient response capacity rejected before handler"); + reset(0);message[12]=0;memcpy(packet+8,message,96);call(104);assert(status()==0xe00002c2&&!handler_calls&&sent_size==8); + reset(0);((struct epic_service_call*)message)->data_len=31;memcpy(packet+8,message,96);call(104);assert(status()==0xe00002c2&&!handler_calls);puts("PASS: message magic and exact body size validated"); + reset(1);c=(void*)packet;c->txbuf=0x100ff8;c->txlen=96;mapped[1]=false;call(32);assert(status()&&!handler_calls);puts("PASS: ungranted remote page produces error response"); + reset(0);service_missing=true;call(8);assert(status()==0xe00002d7);puts("PASS: unavailable service uses native no-handler response"); + reset(0);fail_alloc=1;memcpy(packet+8,message,96);call(104);assert(status()==0xe00002bd&&!handler_calls); + reset(0);fail_alloc=2;memcpy(packet+8,message,96);call(104);assert(status()==0xe00002bd&&!handler_calls);puts("PASS: request/reply allocation failure returns tagged error"); + reset(0);handler_error=-EINVAL;memcpy(packet+8,message,96);call(104);assert(status()==0xe00002c2&&handler_calls==1);puts("PASS: handler failure does not acknowledge success"); + reset(0);afk_recv_command_v2(&ep,5,0xc0,packet,7);assert(!sent_count);puts("PASS: truncated context cannot generate a reply with an invalid tag"); +} +''' +p=I/'test-afk27-incoming.c';p.write_text(pre+wire+code+post) +subprocess.run([os.environ.get('HOSTCC', 'cc'),'-std=gnu11','-Wall','-Wextra','-Werror','-O1','-fsanitize=undefined,address','-g',str(p),'-o',str(I/'test-afk27-incoming')],check=True) +subprocess.run([str(I/'test-afk27-incoming')],check=True) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp-native-boot.py b/tools/testing/selftests/drivers/gpu/apple-dcp-native-boot.py new file mode 100755 index 0000000000000..465fa3c183cf0 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp-native-boot.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +from pathlib import Path +import os, subprocess, tempfile +scratch = tempfile.TemporaryDirectory(prefix='apple-dcp-test-') +I = Path(scratch.name) +root = next((p for p in Path(__file__).resolve().parents + if (p / 'drivers/gpu/drm/apple/afk.c').is_file()), None) +if os.environ.get('KERNEL_SRC'): + root = Path(os.environ['KERNEL_SRC']) +if root is None: + print('SKIP: set KERNEL_SRC to the Linux source tree') + raise SystemExit(4) +s=(root/'drivers/gpu/drm/apple/iomfb_template.c').read_text() +def fn(name): + a=s.index('static void '+name+'(');b=s.index('\n}',a)+2;return s[a:b] +pre=r''' +#include +#include +#include +#include +#include +typedef uint8_t u8;typedef uint32_t u32;typedef uint64_t u64; +#define DCP_FW_VERSION(a,b,c) ((a)*10000+(b)*100+(c)) +#define DCP_FW_VER DCP_FW_VERSION(27,0,0) +#define DCP_CONTEXT_CB 1 +#define dev_info(...) ((void)0) +#define dev_err(...) ((void)0) +struct dcp_dfb_dimensions_h17p {u32 width,height;}; +struct dcp_dfb_layout_req_h17p {u32 data[3];}; +struct dcp_channel{int depth;void *output[1];}; +struct apple_dcp{struct dcp_channel ch_cb;}; +static int step,acks;static u8 callback_result=1; +static struct dcp_dfb_dimensions_h17p dimensions; +static void boot_2(struct apple_dcp*d,void*o,void*c){(void)d;(void)o;(void)c;step=2;} +static void boot_1_11(struct apple_dcp*d,void*o,void*c){(void)d;(void)o;(void)c;} +static void boot_1_7(struct apple_dcp*d,void*o,void*c){(void)d;(void)o;(void)c;} +static void dcp_ack(struct apple_dcp*d,int c){(void)d;assert(c==1);acks++;} +static void dcp_set_dfb_dimensions(struct apple_dcp*d,bool b,struct dcp_dfb_dimensions_h17p*x,void(*cb)(struct apple_dcp*,void*,void*),void*c){(void)d;(void)b;(void)c;assert(cb==boot_1_11);dimensions=*x;step=104;} +''' +mid=r''' +static void dcp_get_dfb_state(struct apple_dcp*d,bool b,u32*x,void(*cb)(struct apple_dcp*,void*,void*),void*c){(void)d;(void)b;(void)c;assert(*x==0&&cb==boot_1_10);step=453;} +''' +mid2=r''' +static void dcp_get_dfb_layout(struct apple_dcp*d,bool b,struct dcp_dfb_layout_req_h17p*x,void(*cb)(struct apple_dcp*,void*,void*),void*c){(void)d;(void)b;(void)x;(void)c;assert(cb==boot_1_9);step=103;} +static void dcp_get_dfb_compression_info(struct apple_dcp*d,bool b,u64*x,void(*cb)(struct apple_dcp*,void*,void*),void*c){(void)d;(void)b;(void)c;assert(*x==0&&cb==boot_1_7);step=37;} +''' +post=r''' +int main(void){ +struct apple_dcp d={.ch_cb={.depth=1,.output={&callback_result}}}; +u32 has_fb=0;boot_1_6(&d,&has_fb,0);assert(step==103);has_fb=1;boot_1_6(&d,&has_fb,0);assert(step==37); +struct dcp_dfb_dimensions_h17p dim={3840,2160};boot_1_9(&d,&dim,0);assert(step==2); +dim.width=0;boot_1_9(&d,&dim,0);assert(step==453); +dim.height=0;boot_1_10(&d,&dim,0);assert(step==104&&!acks&&!dimensions.width&&!dimensions.height); +dim.width=3024;dim.height=1964;boot_1_10(&d,&dim,0);assert(dimensions.width==3024&&dimensions.height==1964&&!acks); +puts("PASS: firmware27 native boot follows A454 presence and A103 geometry branches; zero external geometry reaches A104, internal dimensions preserved"); +} +''' +p=I/'test-dcp27-native-boot.c';p.write_text(pre+fn('boot_1_10')+mid+fn('boot_1_9')+mid2+fn('boot_1_6')+post) +subprocess.run([os.environ.get('HOSTCC', 'cc'),'-std=gnu11','-O1','-g','-fsanitize=address,undefined',str(p),'-o',str(I/'test-dcp27-native-boot')],check=True) +subprocess.run([str(I/'test-dcp27-native-boot')],check=True) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp-service-init.py b/tools/testing/selftests/drivers/gpu/apple-dcp-service-init.py new file mode 100755 index 0000000000000..95b432cc98526 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp-service-init.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise production service-property selection using captured AFKV2 fields.""" +from pathlib import Path +import os, subprocess, tempfile +root = next((p for p in Path(__file__).resolve().parents + if (p / 'drivers/gpu/drm/apple/parser.c').is_file()), None) +if os.environ.get('KERNEL_SRC'): + root = Path(os.environ['KERNEL_SRC']) +if root is None: + print('SKIP: set KERNEL_SRC to the Linux source tree') + raise SystemExit(4) +s = (root/'drivers/gpu/drm/apple/parser.c').read_text() +a = s.index('int parse_epic_service_init(') +b = s.index('\n}', a) + 2 +pre = r''' +#include +#include +#include +#include +#include +#include +#include +typedef int64_t s64; +#define ERR_PTR(e) ((void*)(intptr_t)(e)) +#define PTR_ERR(p) ((intptr_t)(p)) +#define IS_ERR(p) ((uintptr_t)(p)>=(uintptr_t)-4095) +#define kfree(p) free((void *)(p)) +struct prop {const char *key,*text;s64 value;}; +struct dcp_parse_ctx {struct prop *props;unsigned n,idx;bool key_read;}; +struct iterator {struct dcp_parse_ctx *handle;}; +#define dcp_parse_foreach_in_dict(h,it) for((it).handle=(h),(h)->idx=0; (h)->idx<(h)->n; (h)->idx++,(h)->key_read=false) +static char *parse_string(struct dcp_parse_ctx*h){const char *s=h->key_read?h->props[h->idx].text:h->props[h->idx].key;h->key_read=true;return s?strdup(s):ERR_PTR(-EINVAL);} +static int parse_int(struct dcp_parse_ctx*h,s64*out){*out=h->props[h->idx].value;return 0;} +static void skip(struct dcp_parse_ctx*h){(void)h;} +''' +post = r''' +static void check(struct prop *p,unsigned n,s64 want){struct dcp_parse_ctx c={.props=p,.n=n};const char *name,*class;s64 unit=-1;assert(!parse_epic_service_init(&c,&name,&class,&unit));assert(unit==want);assert(!strcmp(name,"dcpdptx-port-epic"));assert(!strcmp(class,"AppleDCPDPTXRemotePort"));free((void*)name);free((void*)class);} +int main(void){ +struct prop p[]={ + {"EPICName","dcpdptx-port-epic",0}, + {"EPICProviderClass","AppleDCPDPTXRemotePort",0}, + {"EPICUnit",NULL,0},{"interface-id",NULL,1}}; +check(p,4,0);p[2].value=1;p[3].value=3;check(p,4,1); +struct prop t=p[2];p[2]=p[3];p[3]=t;check(p,4,1); +p[2]=(struct prop){"interface-id",NULL,3};check(p,3,3); +p[0].key="compartment";p[1].key="name";check(p,3,3); +puts("PASS: captured port0/channel1 and port1/channel3 preserve EPICUnit in either property order; H17P fallback retained"); +} +''' +with tempfile.TemporaryDirectory(prefix='apple-dcp-service-') as tmp: + p=Path(tmp)/'test.c';exe=Path(tmp)/'test';p.write_text(pre+s[a:b]+post) + subprocess.run([os.environ.get('HOSTCC','cc'),'-std=gnu11','-Wall','-Wextra','-Werror','-O1','-g','-fsanitize=address,undefined',str(p),'-o',str(exe)],check=True) + subprocess.run([str(exe)],check=True) From 3113e4858427c0fc167c557dd61473bd0036f795 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 16:24:49 -0400 Subject: [PATCH 0227/1579] mux: apple: add T8142 external display route and clock control --- Documentation/gpu/apple-dcp-t6050.rst | 46 ++++ drivers/mux/apple-display-crossbar.c | 211 +++++++++++++++++- include/linux/soc/apple/dp-tunnel.h | 7 + .../drivers/gpu/apple-dpxbar-t8142.py | 164 ++++++++++++++ 4 files changed, 423 insertions(+), 5 deletions(-) create mode 100644 tools/testing/selftests/drivers/gpu/apple-dpxbar-t8142.py diff --git a/Documentation/gpu/apple-dcp-t6050.rst b/Documentation/gpu/apple-dcp-t6050.rst index d3cf0ea0fb8e3..cd5091b194057 100644 --- a/Documentation/gpu/apple-dcp-t6050.rst +++ b/Documentation/gpu/apple-dcp-t6050.rst @@ -284,3 +284,49 @@ and external0 DCP. A read-only driver-state snapshot confirms both external DPTX ports enabled with units0/1 and initialized service cookies; the earlier port0 initialization timeout is gone. HDMI and USB-C routing, incoming DPTX link-training callbacks, and external scanout remain unqualified. + +T8142 physical crossbar +---------------------- + +The T6050 ATC crossbar advertises the T8142 register interface. Its five +outputs are DPPHY and DPIN0..3. The initial driver covers the eight external +DCP source states (two ports per DCP), with even states only on DPPHY, +matching native connection validation. AUX source roles have a different +selector-to-clock-bit mapping and are not exposed by this initial path. + +With ``apple,t8142-display-crossbar``, mux selection programs only the +route. While holding that selection, the consumer must configure its PHY +or tunnel clock and call ``apple_dpxbar_link_configure(mux, pclk, uhbr)`` +with the actual PCLK1..4 and coding before ``apple_dpxbar_link_up()``. +There is no implicit clock or link-rate choice. A link configuration change +requires link-down first. Releasing or changing the selection invalidates +its clock configuration; reselecting the same source does not. Other +outputs retain their route and clock gates. The coding bit is shared per +PCLK, so conflicting simultaneous users are rejected before register writes. +Older crossbar generations retain their existing selection semantics. + +The native T8142 path requires neither the older cycle-slip delay nor the +old register0xc00 tunable. Link-down preserves the selector and output +permission bit while disabling source and pixel-clock gates and restoring +reset gates. Native readback checks use the T8142 status register layout. +The source-based regression test exercises this sequence and output isolation:: + + python3 tools/testing/selftests/drivers/gpu/apple-dpxbar-t8142.py + +Control-page access also requires the ATC provider's common power gates. +On T6050, native TypeCPhy power-on enables provider clock indices0 and1; +ADT identifies these for HDMI's ATC3 as589 (ATC3_USB_AON_V) and62 +(ATC3_COMMON). Reading the crossbar before these were enabled produced an +asynchronous fabric SError on hardware. Enabling them allowed all32 tested +control/status registers to be read without exceptions. The host setup +entry ``sptm_hv_guest_atc_xbar_setup`` resolves the ADT resources, enables +those two clocks and maps one control page, without touching PHY lanes or +pixel-clock clients. Physical PHY configuration and DCP consumer integration +are still separate work; a successful mux test is not external scanout proof. + +On J714s, an isolated Linux test programmed and read back all36 supported +source/output selector combinations, confirmed the other register fields +and gates were unchanged, and restored the original selector registers. +The test required the crossbar source gates to be off and never enabled +PHY lanes or pixel clocks. Link-up/down clock behavior and optical output +remain unverified on hardware. diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index 90384504e034c..0e2b1484a90ca 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -92,10 +92,14 @@ #define ATC_DPPHY BIT(8) enum { MUX_DPPHY = 0, MUX_DPIN0 = 1, MUX_DPIN1 = 2, MUX_MAX = 3 }; -static const char *apple_dpxbar_names[MUX_MAX] = { "dpphy", "dpin0", "dpin1" }; +enum { T8142_MUX_MAX = 5 }; +static const char *apple_dpxbar_names[T8142_MUX_MAX] = { + "dpphy", "dpin0", "dpin1", "dpin2", "dpin3" +}; struct apple_dpxbar_hw { unsigned int n_ufp; + unsigned int n_mux; u32 tunable; const struct mux_control_ops *ops; }; @@ -103,7 +107,10 @@ struct apple_dpxbar_hw { struct apple_dpxbar { struct device *dev; void __iomem *regs; - int selected_dispext[MUX_MAX]; + int selected_dispext[T8142_MUX_MAX]; + u8 pclk[T8142_MUX_MAX]; + bool uhbr[T8142_MUX_MAX]; + bool enabled[T8142_MUX_MAX]; spinlock_t lock; }; @@ -466,6 +473,180 @@ static const struct mux_control_ops apple_dpxbar_t602x_ops = { .set = apple_dpxbar_set_t602x, }; +/* + * T8142 (also used by T6050) has four pixel clocks and five outputs. + * Keep selection separate from clock activation: the PHY/DP IN owner supplies + * its actual clock and coding before link_up(), after configuring that clock. + * The register order follows the native connect/bringUp/takeDown operations. + * Only the four two-port external display sources are described here; AUX + * sources use a different selector-to-clock-bit mapping. + */ +static const u8 t8142_dest_shift[T8142_MUX_MAX] = { 8, 0, 4, 12, 16 }; +static const u8 t8142_select_shift[T8142_MUX_MAX] = { 8, 0, 4, 24, 0 }; +static const u8 t8142_select_shift2[T8142_MUX_MAX] = { 20, 12, 16, 28, 4 }; + +static void t8142_link_down(struct apple_dpxbar *xbar, unsigned int index) +{ + u32 src, dst, clock; + + if (!xbar->enabled[index]) + return; + src = BIT(xbar->selected_dispext[index]); + dst = BIT(t8142_dest_shift[index]); + clock = xbar->pclk[index] - 1; + + dpxbar_clear32(xbar, 0x3c, src); + dpxbar_clear32(xbar, 0x00, src); + dpxbar_clear32(xbar, 0x0c + 4 * clock, src); + dpxbar_clear32(xbar, 0x24 + 4 * clock, dst); + udelay(1); + WARN_ON(readl(xbar->regs + 0x800) & src); + WARN_ON(readl(xbar->regs + 0x808 + 4 * clock) & src); + WARN_ON(readl(xbar->regs + 0x81c + 4 * clock) & dst); + dpxbar_clear32(xbar, 0x08, src); + dpxbar_clear32(xbar, 0x20, 7 << (3 * xbar->selected_dispext[index])); + dpxbar_clear32(xbar, 0x38, 7 << t8142_dest_shift[index]); + dpxbar_set32(xbar, 0x04, src); + dpxbar_set32(xbar, 0x1c, src); + dpxbar_set32(xbar, 0x34, dst); + xbar->enabled[index] = false; +} + +static int t8142_link_up(struct apple_dpxbar *xbar, unsigned int index) +{ + int state = xbar->selected_dispext[index]; + u32 clock, select, src, dst; + + if (state < 0) + return -ENODEV; + if (!xbar->pclk[index]) + return -EINVAL; + if (xbar->enabled[index]) + return 0; + + /* The coding bit is shared by every output using this pixel clock. */ + for (unsigned int i = 0; i < T8142_MUX_MAX; i++) + if (i != index && xbar->enabled[i] && + xbar->pclk[i] == xbar->pclk[index] && + xbar->uhbr[i] != xbar->uhbr[index]) + return -EBUSY; + + clock = xbar->pclk[index] - 1; + select = 2 * clock + 1; + src = BIT(state); + dst = BIT(t8142_dest_shift[index]); + dpxbar_clear32(xbar, 0x04, src); + dpxbar_clear32(xbar, 0x1c, src); + dpxbar_clear32(xbar, 0x34, dst); + udelay(1); + WARN_ON(readl(xbar->regs + 0x804) & src); + WARN_ON(readl(xbar->regs + 0x818) & src); + WARN_ON(readl(xbar->regs + 0x82c) & dst); + dpxbar_set32(xbar, 0x08, src); + dpxbar_mask32(xbar, 0x20, 7 << (3 * state), select << (3 * state)); + dpxbar_mask32(xbar, 0x38, 7 << t8142_dest_shift[index], + select << t8142_dest_shift[index]); + dpxbar_mask32(xbar, 0x48, BIT(clock), xbar->uhbr[index] ? BIT(clock) : 0); + dpxbar_set32(xbar, 0x00, src); + dpxbar_set32(xbar, 0x0c + 4 * clock, src); + dpxbar_set32(xbar, 0x24 + 4 * clock, dst); + dpxbar_set32(xbar, 0x4c, dst); + dpxbar_set32(xbar, 0x3c, src); + xbar->enabled[index] = true; + return 0; +} + +static int apple_dpxbar_set_t8142(struct mux_control *mux, int state) +{ + struct apple_dpxbar *xbar = mux_chip_priv(mux->chip); + unsigned int index = mux_control_get_index(mux); + unsigned long flags; + u32 mask, value; + + if (state != MUX_IDLE_DISCONNECT && + (state < 0 || state >= 8 || (index == MUX_DPPHY && (state & 1)))) + return -EINVAL; + + spin_lock_irqsave(&xbar->lock, flags); + if (state >= 0) { + for (unsigned int i = 0; i < T8142_MUX_MAX; i++) { + if (i != index && xbar->selected_dispext[i] == state) { + spin_unlock_irqrestore(&xbar->lock, flags); + return -EBUSY; + } + } + } + + if (xbar->selected_dispext[index] == state) + goto out; + if (xbar->selected_dispext[index] >= 0) + t8142_link_down(xbar, index); + + /* No implicit clock choice or activation when selecting a new source. */ + xbar->pclk[index] = 0; + xbar->selected_dispext[index] = state; + if (state >= 0) { + mask = (0xfU << t8142_select_shift[index]) | + (0xfU << t8142_select_shift2[index]); + value = ((u32)state << t8142_select_shift[index]) | + ((u32)state << t8142_select_shift2[index]); + dpxbar_mask32(xbar, index == 4 ? 0x40 : 0x44, mask, value); + } +out: + spin_unlock_irqrestore(&xbar->lock, flags); + return 0; +} + +static const struct mux_control_ops apple_dpxbar_t8142_ops = { + .set = apple_dpxbar_set_t8142, +}; + +/* Caller holds the mux selection throughout configuration and activation. */ +int apple_dpxbar_link_configure(struct mux_control *mux, unsigned int pclk, bool uhbr) +{ + struct apple_dpxbar *xbar; + unsigned int index; + unsigned long flags; + int ret = 0; + + if (!mux || mux->chip->ops != &apple_dpxbar_t8142_ops) + return -EOPNOTSUPP; + if (pclk < 1 || pclk > 4) + return -EINVAL; + xbar = mux_chip_priv(mux->chip); + index = mux_control_get_index(mux); + spin_lock_irqsave(&xbar->lock, flags); + if (xbar->selected_dispext[index] < 0) + ret = -ENODEV; + else if (xbar->enabled[index]) + ret = -EBUSY; + else { + xbar->pclk[index] = pclk; + xbar->uhbr[index] = uhbr; + } + spin_unlock_irqrestore(&xbar->lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dpxbar_link_configure); + +static int apple_dpxbar_t8142_link(struct mux_control *mux, bool up) +{ + struct apple_dpxbar *xbar = mux_chip_priv(mux->chip); + unsigned int index = mux_control_get_index(mux); + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&xbar->lock, flags); + if (xbar->selected_dispext[index] < 0) + ret = -ENODEV; + else if (up) + ret = t8142_link_up(xbar, index); + else + t8142_link_down(xbar, index); + spin_unlock_irqrestore(&xbar->lock, flags); + return ret; +} + /* * Bring the clock/FIFO gates up on a mux whose dispext source is already * selected, without touching the mux selection itself -- used by @@ -483,6 +664,9 @@ int apple_dpxbar_link_up(struct mux_control *mux) u32 dispext_bit, atc_bit; int state, ret = 0; + if (mux && mux->chip->ops == &apple_dpxbar_t8142_ops) + return apple_dpxbar_t8142_link(mux, true); + if (!mux || mux->chip->ops != &apple_dpxbar_t602x_ops) return -EINVAL; index = mux_control_get_index(mux); @@ -549,6 +733,9 @@ int apple_dpxbar_link_down(struct mux_control *mux) u32 dispext_bit, atc_bit; int state; + if (mux && mux->chip->ops == &apple_dpxbar_t8142_ops) + return apple_dpxbar_t8142_link(mux, false); + if (!mux || mux->chip->ops != &apple_dpxbar_t602x_ops) return -EINVAL; index = mux_control_get_index(mux); @@ -584,7 +771,7 @@ static int apple_dpxbar_probe(struct platform_device *pdev) int ret; hw = of_device_get_match_data(dev); - mux_chip = devm_mux_chip_alloc(dev, MUX_MAX, sizeof(*dpxbar)); + mux_chip = devm_mux_chip_alloc(dev, hw->n_mux, sizeof(*dpxbar)); if (IS_ERR(mux_chip)) return PTR_ERR(mux_chip); @@ -597,13 +784,13 @@ static int apple_dpxbar_probe(struct platform_device *pdev) if (IS_ERR(dpxbar->regs)) return PTR_ERR(dpxbar->regs); - if (!of_device_is_compatible(dev->of_node, "apple,t6020-display-crossbar")) { + if (hw->ops == &apple_dpxbar_ops) { readl(dpxbar->regs + UNK_TUNABLE); writel(hw->tunable, dpxbar->regs + UNK_TUNABLE); readl(dpxbar->regs + UNK_TUNABLE); } - for (unsigned int i = 0; i < MUX_MAX; ++i) { + for (unsigned int i = 0; i < hw->n_mux; ++i) { mux_chip->mux[i].states = hw->n_ufp; mux_chip->mux[i].idle_state = MUX_IDLE_DISCONNECT; dpxbar->selected_dispext[i] = -1; @@ -617,29 +804,43 @@ static int apple_dpxbar_probe(struct platform_device *pdev) } static const struct apple_dpxbar_hw apple_dpxbar_hw_t8103 = { + .n_mux = MUX_MAX, .n_ufp = 2, .tunable = 0, .ops = &apple_dpxbar_ops, }; static const struct apple_dpxbar_hw apple_dpxbar_hw_t8112 = { + .n_mux = MUX_MAX, .n_ufp = 4, .tunable = 4278196325, .ops = &apple_dpxbar_ops, }; static const struct apple_dpxbar_hw apple_dpxbar_hw_t6000 = { + .n_mux = MUX_MAX, .n_ufp = 9, .tunable = 5, .ops = &apple_dpxbar_ops, }; static const struct apple_dpxbar_hw apple_dpxbar_hw_t6020 = { + .n_mux = MUX_MAX, .n_ufp = 9, .ops = &apple_dpxbar_t602x_ops, }; +static const struct apple_dpxbar_hw apple_dpxbar_hw_t8142 = { + .n_mux = T8142_MUX_MAX, + .n_ufp = 8, + .ops = &apple_dpxbar_t8142_ops, +}; + static const struct of_device_id apple_dpxbar_ids[] = { + { + .compatible = "apple,t8142-display-crossbar", + .data = &apple_dpxbar_hw_t8142, + }, { .compatible = "apple,t8103-display-crossbar", .data = &apple_dpxbar_hw_t8103, diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index 191f8f95dc5aa..87f1836521066 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -42,6 +42,13 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); * enable set and link_up() re-asserts it. The caller keeps the output * selected (holds the mux) around both. */ +/* + * T8142/T6050: selection only establishes the route. Before link_up(), set + * the PHY's actual pixel clock (1..4) and link coding; no clock is assumed. + * Configure only while down. Re-selecting or releasing the mux invalidates + * this configuration. Older generations return -EOPNOTSUPP. + */ +int apple_dpxbar_link_configure(struct mux_control *mux, unsigned int pclk, bool uhbr); int apple_dpxbar_link_down(struct mux_control *mux); int apple_dpxbar_link_up(struct mux_control *mux); diff --git a/tools/testing/selftests/drivers/gpu/apple-dpxbar-t8142.py b/tools/testing/selftests/drivers/gpu/apple-dpxbar-t8142.py new file mode 100644 index 0000000000000..d5d149fd1d1a6 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dpxbar-t8142.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Run the T8142 production route/gate code against recorded MMIO writes. + +--emit writes the standalone C harness for an ARM64/QEMU build. Registers +are mocked: this checks sequencing and isolation, not PHY or panel output. +""" +from pathlib import Path +import argparse, os, subprocess, tempfile +ap = argparse.ArgumentParser() +ap.add_argument('--emit', type=Path) +args = ap.parse_args() +root = Path(os.environ['KERNEL_SRC']) if 'KERNEL_SRC' in os.environ else next( + p for p in Path(__file__).resolve().parents + if (p/'drivers/mux/apple-display-crossbar.c').is_file()) +s = (root/'drivers/mux/apple-display-crossbar.c').read_text() +structs = s[s.index('enum { MUX_DPPHY'):s.index('static u32 t602x_atc_bit')] +code = s[s.index('static const u8 t8142_dest_shift'):s.index('/*\n * Bring the clock/FIFO gates up')] +pre = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint8_t u8; +typedef int spinlock_t; +#define __iomem +#define BIT(n) (1U << (n)) +#define WARN_ON(c) assert(!(c)) +#define spin_lock_irqsave(l,f) do {(void)(l);(f)=0;}while(0) +#define spin_unlock_irqrestore(l,f) do {(void)(l);(void)(f);}while(0) +#define EXPORT_SYMBOL_GPL(s) +#define MUX_IDLE_DISCONNECT -2 +struct device {}; +struct mux_chip; +struct mux_control {struct mux_chip *chip;unsigned int index;}; +struct mux_control_ops {int (*set)(struct mux_control *,int);}; +struct mux_chip {const struct mux_control_ops *ops;void *priv;}; +#define mux_chip_priv(c) ((c)->priv) +#define mux_control_get_index(m) ((m)->index) +static u32 registers[0x1000/4]; +struct event {unsigned reg;u32 value;}; +static struct event events[128]; +static unsigned nevents; +static u32 readl(void *p) {return *(u32*)p;} +static void writel(u32 v,void *p) { + assert(nevents<128);events[nevents++]=(struct event){(char*)p-(char*)registers,v}; + *(u32*)p=v; +} +static void udelay(unsigned us) {assert(us==1);} +''' +post = r''' +static struct apple_dpxbar xbar; +static struct mux_chip chip; +static struct mux_control mux[5]; +static void reset(void) { + memset(registers,0,sizeof(registers));memset(&xbar,0,sizeof(xbar)); + xbar.regs=registers;chip=(struct mux_chip){&apple_dpxbar_t8142_ops,&xbar}; + for(unsigned i=0;i<5;i++){xbar.selected_dispext[i]=-1;mux[i]=(struct mux_control){&chip,i};} + nevents=0; +} +static void unchanged(int actual,int want,unsigned before) {assert(actual==want);assert(nevents==before);} +static void golden_sequence(void) { + /* Native 27.0 90ca42c/90ca7f4/90caf64; ext3:0 -> DPPHY, PCLK3, UHBR. + * Exact write order derived from firmware, including retained output-enable. + */ + const struct event expected[]={ + {0x44,0x00600600}, + {0x04,0},{0x1c,0},{0x34,0},{0x08,0x40}, + {0x20,0x140000},{0x38,0x500},{0x48,4},{0x00,0x40}, + {0x14,0x40},{0x2c,0x100},{0x4c,0x100},{0x3c,0x40}, + {0x3c,0},{0x00,0},{0x14,0},{0x2c,0},{0x08,0}, + {0x20,0},{0x38,0},{0x04,0x40},{0x1c,0x40},{0x34,0x100} + }; + reset();assert(!apple_dpxbar_set_t8142(&mux[0],6)); + assert(!apple_dpxbar_link_configure(&mux[0],3,true)); + assert(!apple_dpxbar_t8142_link(&mux[0],true)); + assert(!apple_dpxbar_t8142_link(&mux[0],false)); + assert(nevents==sizeof(expected)/sizeof(expected[0])); + for(unsigned i=0;i Date: Mon, 28 Sep 2026 17:17:49 -0400 Subject: [PATCH 0228/1579] drm/apple: forward firmware27 per-lane transmitter drive settings --- Documentation/gpu/apple-dcp-t6050.rst | 27 +++++++ drivers/gpu/drm/apple/dptx-drive-v27.inc | 77 +++++++++++++++++++ drivers/gpu/drm/apple/dptxep.c | 13 ++++ drivers/gpu/drm/apple/dptxep.h | 1 + .../drivers/gpu/apple-dcp-drive-v27.py | 73 ++++++++++++++++++ 5 files changed, 191 insertions(+) create mode 100644 drivers/gpu/drm/apple/dptx-drive-v27.inc create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp-drive-v27.py diff --git a/Documentation/gpu/apple-dcp-t6050.rst b/Documentation/gpu/apple-dcp-t6050.rst index cd5091b194057..06eabb5f50e8e 100644 --- a/Documentation/gpu/apple-dcp-t6050.rst +++ b/Documentation/gpu/apple-dcp-t6050.rst @@ -330,3 +330,30 @@ and gates were unchanged, and restored the original selector registers. The test required the crossbar source gates to be off and never enabled PHY lanes or pixel clocks. Link-up/down clock behavior and optical output remain unverified on hardware. + +Firmware27 transmitter drive settings +------------------------------------ + +The native DPTX proxy uses twelve-byte IODPDriveSettings records containing +encoding type, voltage swing and pre-emphasis. For DP1.x, GET_MAX returns +``{0, 3, 3}``. After the 64-byte DCPAVIPC header, SET takes the lane count +at offset16 and records at offset32, and returns IOReturn at offset0. +GET instead takes the count at offset0 and returns IOReturn at offset16; +its records also start at offset32. These layouts are decoded from the +27.0 native proxy handlers and its native ATC port implementation. + +The firmware27 handlers validate every requested lane and payload boundary, +forward physical voltage/pre-emphasis requests through ``phy_configure``, +and retain per-lane values only after that call succeeds. A missing direct +PHY or a PHY error is returned to the caller. USB4 tunnels retain their +logical drive-setting behavior. Older firmware keeps its existing handlers. + +The actual callback implementation has sanitizer tests, including four +independent lanes, short buffers, aliased request/reply buffers, missing +PHYs and failed PHY operations:: + + python3 tools/testing/selftests/drivers/gpu/apple-dcp-drive-v27.py + +The complete module loaded and these callback tests passed as ARM64 code +in a QEMU guest. Actual firmware link-training callbacks have not yet been +exercised on hardware; this is not external scanout qualification. diff --git a/drivers/gpu/drm/apple/dptx-drive-v27.inc b/drivers/gpu/drm/apple/dptx-drive-v27.inc new file mode 100644 index 0000000000000..a6b531c6a7b97 --- /dev/null +++ b/drivers/gpu/drm/apple/dptx-drive-v27.inc @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Native 27.0 DCPAVIPC payloads, after the 64-byte message header. */ +struct dptxport_drive_v27 { + __le32 status_or_count; + u8 reserved0[12]; + __le32 count_or_status; + u8 reserved1[12]; + struct { + __le32 type; + __le32 voltage; + __le32 pre; + } lane[]; +} __packed; + +static int dptxport_call_drive_v27(struct apple_epic_service *service, bool set, + const void *data, size_t data_size, + void *reply_, size_t reply_size) +{ + const struct dptxport_drive_v27 *request = data; + struct dptxport_drive_v27 *reply = reply_; + struct dptx_port *dptx = service->cookie; + union phy_configure_opts opts = {}; + u32 count; + size_t size; + int ret; + + if (data_size < sizeof(*request) || reply_size < sizeof(*reply)) + return -EINVAL; + count = le32_to_cpu(set ? request->count_or_status : request->status_or_count); + if (count > 4 || count > dptx->lane_count) + return -EINVAL; + size = sizeof(*request) + count * sizeof(request->lane[0]); + if ((set && data_size < size) || (!set && reply_size < size)) + return -EINVAL; + + if (set) { + for (u32 i = 0; i < count; i++) { + /* DP1.x: type0, voltage swing, pre-emphasis. */ + if (le32_to_cpu(request->lane[i].type) || + le32_to_cpu(request->lane[i].voltage) > 3 || + le32_to_cpu(request->lane[i].pre) > 3) + return -EINVAL; + opts.dp.voltage[i] = le32_to_cpu(request->lane[i].voltage); + opts.dp.pre[i] = le32_to_cpu(request->lane[i].pre); + } + opts.dp.lanes = count; + opts.dp.link_rate = dptx->phy_ops.dp.link_rate; + opts.dp.set_voltages = 1; + if (count && !service->ep->dcp->dptx_tunnel) { + if (!dptx->atcphy) + return -ENODEV; + ret = phy_configure(dptx->atcphy, &opts); + if (ret) + return ret; + } + for (u32 i = 0; i < count; i++) { + dptx->drive_voltage[i] = opts.dp.voltage[i]; + dptx->drive_pre[i] = opts.dp.pre[i]; + } + } + + memset(reply, 0, reply_size); + if (set) { + reply->count_or_status = cpu_to_le32(count); + } else { + /* GET supplies capacity at0 and returns IOReturn at16, unlike SET. */ + reply->status_or_count = cpu_to_le32(count); + } + /* A short SET response may contain only the fixed header. */ + if (reply_size >= size) { + for (u32 i = 0; i < count; i++) { + reply->lane[i].voltage = cpu_to_le32(dptx->drive_voltage[i]); + reply->lane[i].pre = cpu_to_le32(dptx->drive_pre[i]); + } + } + return 0; +} diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 0702c889a85a1..0358b331994c9 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -78,6 +78,8 @@ struct dptxport_apcall_set_tiled { __le32 retcode; }; +#include "dptx-drive-v27.inc" + /* * Ported from aurora-silicon/linux#8: a Thunderbolt DP tunnel uses the same * plain CORE|ATC|DIE|CONNECTED target as a direct alt-mode PHY. CORE is the @@ -266,6 +268,11 @@ dptxport_call_get_max_drive_settings(struct apple_epic_service *service, reply->retcode = cpu_to_le32(0); reply->max_drive_settings[0] = cpu_to_le32(0x3); reply->max_drive_settings[1] = cpu_to_le32(0x3); + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) { + /* Native IODPDriveSettings is { type0, max voltage3, max pre3 }. */ + reply->max_drive_settings[0] = cpu_to_le32(0); + memcpy(reply->_unk1, &reply->max_drive_settings[1], sizeof(__le32)); + } return 0; } @@ -760,9 +767,15 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, memcpy(reply, data, min(reply_size, data_size)); return dptxport_call_set_tiled_display_hint(reply, reply_size); case DPTX_APCALL_GET_DRIVE_SETTINGS: + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dptxport_call_drive_v27(service, false, data, data_size, + reply, reply_size); return dptxport_call_get_drive_settings(service, data, data_size, reply, reply_size); case DPTX_APCALL_SET_DRIVE_SETTINGS: + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dptxport_call_drive_v27(service, true, data, data_size, + reply, reply_size); return dptxport_call_set_drive_settings(service, data, data_size, reply, reply_size); case DPTX_APCALL_ACTIVATE: diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 885e3675f9db5..f7dea18daba85 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -59,6 +59,7 @@ struct dptx_port { u32 lane_count; u32 link_rate, pending_link_rate; u32 drive_settings[2]; + u32 drive_voltage[4], drive_pre[4]; /* * Per-boot invocation counters for the validate/connect/request/ * release AFK calls, used together with the caller return address diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp-drive-v27.py b/tools/testing/selftests/drivers/gpu/apple-dcp-drive-v27.py new file mode 100644 index 0000000000000..8b0b2e3bcd0bb --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp-drive-v27.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise native27 drive messages against the actual callback implementation.""" +from pathlib import Path +import os,subprocess,tempfile +root=next(p for p in Path(__file__).resolve().parents if (p/'drivers/gpu/drm/apple/dptxep.c').exists()) +source=(root/'drivers/gpu/drm/apple/dptx-drive-v27.inc').read_text() +parent=(root/'drivers/gpu/drm/apple/dptxep.c').read_text() +a=parent.index('struct dptxport_apcall_max_drive_settings {');b=parent.index('\n};',a)+3;source+=parent[a:b] +a=parent.index('static int\ndptxport_call_get_max_drive_settings(');b=parent.index('\n}',a)+2;source+=parent[a:b] +pre=r''' +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8;typedef uint32_t u32;typedef uint32_t __le32; +#define __packed __attribute__((packed)) +#define cpu_to_le32(v) (v) +#define le32_to_cpu(v) (v) +union phy_configure_opts {struct {unsigned voltage[4],pre[4],lanes,link_rate;bool set_voltages,set_rate,set_lanes;}dp;}; +struct dptx_port{unsigned lane_count,drive_voltage[4],drive_pre[4];void *atcphy;union phy_configure_opts phy_ops;}; + +#define DCP_FIRMWARE_V_27_0 27 +struct dcp{bool dptx_tunnel;int fw_compat;};struct ep{struct dcp*dcp;};struct apple_epic_service{struct ep*ep;struct dptx_port*cookie;}; +static unsigned calls;static int failure;static union phy_configure_opts last; +static int phy_configure(void*p,union phy_configure_opts*o){assert(p);calls++;last=*o;return failure;} +''' +post=r''' +int main(void){ +unsigned char input[80]={0},output[88]; +struct dptxport_drive_v27*r=(void*)input,*o=(void*)output; +struct dcp d={0};struct ep e={&d};struct dptx_port p={.lane_count=4,.atcphy=&d};struct apple_epic_service s={&e,&p}; +assert(sizeof(*r)==32 && offsetof(struct dptxport_drive_v27,count_or_status)==16 && offsetof(struct dptxport_drive_v27,lane)==32); +p.phy_ops.dp.link_rate=8100; +struct dptxport_apcall_max_drive_settings max; +assert(dptxport_call_get_max_drive_settings(&s,&max,31)==-EINVAL); +memset(&max,0,sizeof(max));assert(!dptxport_call_get_max_drive_settings(&s,&max,sizeof(max)));assert(max.max_drive_settings[0]==3&&max.max_drive_settings[1]==3); +d.fw_compat=27;assert(!dptxport_call_get_max_drive_settings(&s,&max,sizeof(max)));u32 third;memcpy(&third,max._unk1,4);assert(max.max_drive_settings[0]==0&&max.max_drive_settings[1]==3&&third==3); +r->count_or_status=4; +for(unsigned i=0;i<4;i++){r->lane[i].voltage=i;r->lane[i].pre=3-i;} +memset(output,0xa5,sizeof(output)); +assert(!dptxport_call_drive_v27(&s,true,input,80,output,80)); +assert(calls==1&&last.dp.set_voltages&&!last.dp.set_rate&&!last.dp.set_lanes&&last.dp.link_rate==8100&&last.dp.lanes==4); +assert(!o->status_or_count&&o->count_or_status==4&&output[80]==0xa5); +for(unsigned i=0;i<4;i++)assert(last.dp.voltage[i]==i&&last.dp.pre[i]==3-i); +r->status_or_count=4; +assert(!dptxport_call_drive_v27(&s,false,input,32,output,80)); +assert(o->status_or_count==4&&!o->count_or_status&&calls==1); +for(unsigned i=0;i<4;i++)assert(!o->lane[i].type&&o->lane[i].voltage==i&&o->lane[i].pre==3-i); +assert(dptxport_call_drive_v27(&s,true,input,79,output,80)==-EINVAL); +assert(dptxport_call_drive_v27(&s,false,input,32,output,79)==-EINVAL); +r->count_or_status=5;assert(dptxport_call_drive_v27(&s,true,input,80,output,80)==-EINVAL); +r->count_or_status=4;r->lane[3].type=1;assert(dptxport_call_drive_v27(&s,true,input,80,output,80)==-EINVAL);r->lane[3].type=0; +r->lane[3].voltage=4;assert(dptxport_call_drive_v27(&s,true,input,80,output,80)==-EINVAL);r->lane[3].voltage=0; +failure=-EIO;assert(dptxport_call_drive_v27(&s,true,input,80,output,80)==-EIO);assert(p.drive_voltage[3]==3);failure=0; +p.atcphy=NULL;assert(dptxport_call_drive_v27(&s,true,input,80,output,80)==-ENODEV); +d.dptx_tunnel=true;assert(!dptxport_call_drive_v27(&s,true,input,80,output,32));assert(calls==2); +d.dptx_tunnel=false;p.atcphy=&d;r->count_or_status=2; +assert(!dptxport_call_drive_v27(&s,true,input,56,input,56));assert(r->count_or_status==2&&r->lane[1].voltage==1&&r->lane[1].pre==2); +p.lane_count=1;r->count_or_status=2;assert(dptxport_call_drive_v27(&s,true,input,56,output,80)==-EINVAL); +r->count_or_status=0;assert(!dptxport_call_drive_v27(&s,true,input,32,output,32));assert(calls==3); +puts("PASS: firmware27 drive ABI, four distinct lanes, bounded payloads, aliased buffers, PHY error/no-PHY propagation, tunnel semantics"); +} +''' +if os.environ.get('TEST_SOURCE'): + Path(os.environ['TEST_SOURCE']).write_text(pre+source+post) +with tempfile.TemporaryDirectory(prefix='dcp-drive27-') as tmp: + p=Path(tmp);(p/'test.c').write_text(pre+source+post) + subprocess.run([os.environ.get('HOSTCC','cc'),'-O1','-g','-fsanitize=address,undefined','-Wall','-Wextra','-Werror',str(p/'test.c'),'-o',str(p/'test')],check=True) + subprocess.run([str(p/'test')],check=True) From fb6a85fecd198aaf9f1fd14d69e22cfedcb54ea3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 18:01:49 -0400 Subject: [PATCH 0229/1579] phy: apple: add T6050 fixed DP PHY and direct DCP clock routing --- Documentation/gpu/apple-dcp-t6050.rst | 56 ++ drivers/gpu/drm/apple/dcp.c | 31 ++ drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/dptxep.c | 32 +- drivers/mux/apple-display-crossbar.c | 40 ++ drivers/phy/apple/Kconfig | 10 + drivers/phy/apple/Makefile | 3 + drivers/phy/apple/atc-t6050-dp-data.h | 513 ++++++++++++++++++ drivers/phy/apple/atc-t6050-dp.c | 465 ++++++++++++++++ include/linux/soc/apple/dp-tunnel.h | 3 + .../drivers/gpu/apple-dcp-direct-link.py | 46 ++ .../drivers/phy/apple-t6050/Makefile | 5 + .../selftests/drivers/phy/apple-t6050/README | 14 + .../drivers/phy/apple-t6050/driver-test.c | 88 +++ 14 files changed, 1300 insertions(+), 7 deletions(-) create mode 100644 drivers/phy/apple/atc-t6050-dp-data.h create mode 100644 drivers/phy/apple/atc-t6050-dp.c create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp-direct-link.py create mode 100644 tools/testing/selftests/drivers/phy/apple-t6050/Makefile create mode 100644 tools/testing/selftests/drivers/phy/apple-t6050/README create mode 100644 tools/testing/selftests/drivers/phy/apple-t6050/driver-test.c diff --git a/Documentation/gpu/apple-dcp-t6050.rst b/Documentation/gpu/apple-dcp-t6050.rst index 06eabb5f50e8e..142266a0a6de3 100644 --- a/Documentation/gpu/apple-dcp-t6050.rst +++ b/Documentation/gpu/apple-dcp-t6050.rst @@ -357,3 +357,59 @@ PHYs and failed PHY operations:: The complete module loaded and these callback tests passed as ARM64 code in a QEMU guest. Actual firmware link-training callbacks have not yet been exercised on hardware; this is not external scanout qualification. + + +Fixed T6050 DisplayPort PHY +-------------------------- + +``PHY_APPLE_T6050_DP`` provides the normal-orientation, four-lane DP PHY +used by the HDMI bridge path. It does not select USB or dynamic Type-C +modes. The register sequences follow the captured macOS27 T6050 path; +board tunables remain device-tree data. RBR, HBR, HBR2 and HBR3 are +supported. Spread spectrum and UHBR are not enabled by this driver. + +Three named power domains (``usb-aon``, ``common``, ``wrapper``) are required. +The first two enable access; the wrapper is resumed partway through native +reset, and is suspended at the corresponding native shutdown step. A +single automatically attached domain would enable the wrapper too early. +The domain topology must describe ATC leaf domains, without handing shared +SoC fabric power-off ownership to a lone PHY. The T8142 crossbar can hold +its own named AON/common references for control-page access. + +The PHY node uses ``apple,t6050-dp-phy``, ``#phy-cells = <0>``, and named +resources ``core``, ``reset``, ``fabric``, ``axi2af``, ``lioa``, +``common-cfg``. The core resource spans native bank2 through bank34; +remaining resources correspond to native banks39 through43. Its nine +``apple,tunable-*`` properties are offset/mask/value triples using the +same format as the existing Apple tunable helper. They are all required: +``axi2af``, ``fabric``, ``common-cfg``, ``lioa``, ``lane0-rx-clock``, +``lane1-rx-clock``, ``lane0-rx-eq``, ``lane1-rx-eq``, ``auspll``. +Optional 16-entry ``apple,dp-training-table-{rbr,hbr,hbr2,hbr3}`` overrides +fall back to a shared ``apple,dp-training-table`` and then the native +built-in DP1.x table. Voltage and pre-emphasis select index voltage*4+pre; +invalid combinations are rejected before MMIO. + +The direct PHY main clock is native logical PCLK0, which the crossbar maps +to physical PCLK1. The four additional logical clocks are tunnel clocks. +DCP's link-change callbacks take the direct crossbar down before a rate +change, then configure the actual clock/coding and bring it up after the +PHY is ready. Older PHYs and the existing USB4 tunnel path retain their +previous callbacks. Errors from activation and deactivation reach firmware. + +A failed hardware acknowledgement stops the sequence and marks that PHY +instance faulted; further operations fail until a fresh reset. The driver +does not fabricate readiness or retry partially completed sequences. + +Validation status: the actual driver source passed an ARM64 QEMU model +covering two complete power-domain cycles, four rates, DP drive requests, +invalid settings and a missing acknowledgement. QEMU supplies modeled +hardware acknowledgements explicitly; this does not establish physical +link training or external video. On J714s with firmware27, a separate real generic-PHY consumer completed +two cycles of power/AUX setup, all four link rates, and four-lane drive +settings, then suspended the wrapper domain. All hardware polls passed. +AON/common references were released; those domains remained on while +waiting for the unbound crossbar consumer, so this is not evidence of +complete ATC power gating. External sink training remains unverified. + +The source-including model test lives at +``tools/testing/selftests/drivers/phy/apple-t6050/``. diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c52c8cab16d4d..ccfe26d86bba7 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -388,6 +388,37 @@ static int dcp_dpxbar_link(struct mux_control *mux, bool up) return ret; } +int dcp_direct_crossbar_link(struct apple_dcp *dcp, bool up) +{ + typeof(&apple_t6050_dp_link_config) get_config; + typeof(&apple_dpxbar_link_configure) configure; + unsigned int pclk; + bool uhbr; + int ret; + + if (dcp->dptx_tunnel || !dcp->phy || !dcp->xbar) + return 0; + get_config = symbol_get(apple_t6050_dp_link_config); + if (!get_config) + return 0; + ret = get_config(dcp->phy, &pclk, &uhbr); + symbol_put(apple_t6050_dp_link_config); + if (ret == -EOPNOTSUPP) + return 0; + if (!up && ret == -ENOLINK) + ret = 0; + if (ret) + return ret; + if (!up) + return dcp_dpxbar_link(dcp->xbar, false); + configure = symbol_get(apple_dpxbar_link_configure); + if (!configure) + return -ENODEV; + ret = configure(dcp->xbar, pclk, uhbr); + symbol_put(apple_dpxbar_link_configure); + return ret ? ret : dcp_dpxbar_link(dcp->xbar, true); +} + /* DP IN adapter handshake through the thunderbolt glue; tb_lock held. */ static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) { diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index b8a2a91f2bb13..4b06cdbb89c1a 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -75,6 +75,7 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); */ int dcp_tunnel_crossbar_up(struct apple_dcp *dcp); int dcp_tunnel_crossbar_down(struct apple_dcp *dcp); +int dcp_direct_crossbar_link(struct apple_dcp *dcp, bool up); int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate); int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 0358b331994c9..8dcf9d4bb0c6f 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -657,11 +657,15 @@ dptxport_call_activate(struct apple_epic_service *service, * change for a tunnel (dptx->atcphy must stay in USB4/TBT mode the * whole connection, see dcp_tunnel_set_rate()). The DP IN adapter is * only woken (DPTX_INACTIVE=0) here, via dcp_tunnel_dpin_activate(); - * waking it earlier hangs the machine. Activate always replies - * success to DCP. + * waking it earlier hangs the machine. Report PHY activation failures + * to DCP instead of acknowledging a transmitter that is not ready. */ - if (dptx->atcphy && !dcp->phy_managed_by_typec) - phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + if (dptx->atcphy && !dcp->phy_managed_by_typec) { + int ret = phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + + if (ret) + return ret; + } if (dcp->dptx_tunnel) dcp_tunnel_dpin_activate(dcp, true); @@ -679,12 +683,19 @@ dptxport_call_deactivate(struct apple_epic_service *service, { struct dptx_port *dptx = service->cookie; struct apple_dcp *dcp = service->ep->dcp; + int ret; dev_info(dcp->dev, "DPTXPort: DEACTIVATE\n"); + ret = dcp_direct_crossbar_link(dcp, false); + if (ret) + return ret; if (dcp->dptx_tunnel) dcp_tunnel_dpin_activate(dcp, false); - if (dptx->atcphy && !dcp->phy_managed_by_typec) - phy_set_mode_ext(dptx->atcphy, PHY_MODE_INVALID, 0); + if (dptx->atcphy && !dcp->phy_managed_by_typec) { + ret = phy_set_mode_ext(dptx->atcphy, PHY_MODE_INVALID, 0); + if (ret) + return ret; + } memcpy(reply, data, min(reply_size, data_size)); if (reply_size >= 4) @@ -707,7 +718,11 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, min(data_size, (size_t)64), true); switch (idx) { - case DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG: + case DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG: { + int ret = dcp_direct_crossbar_link(service->ep->dcp, false); + + if (ret) + return ret; /* * Ported from aurora-silicon/linux#8: a re-link on an * established tunnel takes the crossbar connection down @@ -716,11 +731,14 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, if (service->ep->dcp->dptx_tunnel && dptx->link_rate) dcp_tunnel_crossbar_down(service->ep->dcp); return dptxport_call_will_change_link_config(service); + } case DPTX_APCALL_DID_CHANGE_LINK_CONFIG: { int ret = dptxport_call_did_change_link_config(service); if (!ret && service->ep->dcp->dptx_tunnel && dptx->link_rate) dcp_tunnel_crossbar_up(service->ep->dcp); + if (!ret && dptx->link_rate) + ret = dcp_direct_crossbar_link(service->ep->dcp, true); return ret; } case DPTX_APCALL_GET_MAX_LINK_RATE: diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index 0e2b1484a90ca..789bb5f7377cc 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -24,6 +24,8 @@ #include #include #include +#include +#include #include /* @@ -762,6 +764,39 @@ int apple_dpxbar_link_down(struct mux_control *mux) } EXPORT_SYMBOL_GPL(apple_dpxbar_link_down); +static void apple_dpxbar_domain_release(void *data) +{ + struct device *pd = data; + + pm_runtime_put_sync_suspend(pd); + dev_pm_domain_detach(pd, true); +} + +static int apple_dpxbar_domains(struct device *dev) +{ + static const char * const names[] = { "usb-aon", "common" }; + int ret; + + /* Older handoff-only trees have no Linux-managed domains. */ + if (!of_property_present(dev->of_node, "power-domain-names")) + return 0; + for (unsigned int i = 0; i < ARRAY_SIZE(names); i++) { + struct device *pd = dev_pm_domain_attach_by_name(dev, names[i]); + + if (IS_ERR_OR_NULL(pd)) + return pd ? PTR_ERR(pd) : -ENODEV; + ret = pm_runtime_resume_and_get(pd); + if (ret < 0) { + dev_pm_domain_detach(pd, true); + return ret; + } + ret = devm_add_action_or_reset(dev, apple_dpxbar_domain_release, pd); + if (ret) + return ret; + } + return 0; +} + static int apple_dpxbar_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -771,6 +806,11 @@ static int apple_dpxbar_probe(struct platform_device *pdev) int ret; hw = of_device_get_match_data(dev); + if (hw->ops == &apple_dpxbar_t8142_ops) { + ret = apple_dpxbar_domains(dev); + if (ret) + return dev_err_probe(dev, ret, "cannot power crossbar controls\n"); + } mux_chip = devm_mux_chip_alloc(dev, hw->n_mux, sizeof(*dpxbar)); if (IS_ERR(mux_chip)) return PTR_ERR(mux_chip); diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig index 8409b67f6d1ec..a722e0eaa25db 100644 --- a/drivers/phy/apple/Kconfig +++ b/drivers/phy/apple/Kconfig @@ -1,4 +1,14 @@ # SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +config PHY_APPLE_T6050_DP + tristate "Apple T6050 fixed DisplayPort PHY" + depends on (ARCH_APPLE || COMPILE_TEST) && OF && PM + select GENERIC_PHY + select APPLE_TUNABLE + help + Support the fixed four-lane DisplayPort PHY used by the T6050 HDMI + bridge, including analog power sequencing and DP1.x link training. + USB and dynamically assigned Type-C PHYs use separate drivers. + config PHY_APPLE_ATC tristate "Apple Type-C PHY" depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) diff --git a/drivers/phy/apple/Makefile b/drivers/phy/apple/Makefile index b9e7bf3e4ac17..f20038510993b 100644 --- a/drivers/phy/apple/Makefile +++ b/drivers/phy/apple/Makefile @@ -1,5 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +obj-$(CONFIG_PHY_APPLE_T6050_DP) += phy-apple-t6050-dp.o +phy-apple-t6050-dp-y := atc-t6050-dp.o + obj-$(CONFIG_PHY_APPLE_ATC) += phy-apple-atc.o phy-apple-atc-y := atc.o diff --git a/drivers/phy/apple/atc-t6050-dp-data.h b/drivers/phy/apple/atc-t6050-dp-data.h new file mode 100644 index 0000000000000..f1f094820762b --- /dev/null +++ b/drivers/phy/apple/atc-t6050-dp-data.h @@ -0,0 +1,513 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* T6050 DP register sequences reconstructed from native macOS27.0. */ +static const struct atc_op dp_init[] = { + { ATC_MASK, 39, 0x0, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x1, 0x1, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x2, 0x2, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x2, 0x2, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POWER, 2, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x2c, 0x1, 0x0, 0 }, + { ATC_POLL, 39, 0x4, 0x100, 0x0, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x8, 0x8, 0 }, + { ATC_TUNABLE, 41, 0x0, 0x0, 0x0, 0 }, + { ATC_TUNABLE, 40, 0x0, 0x0, 0x1, 0 }, + { ATC_TUNABLE, 43, 0x0, 0x0, 0x2, 0 }, + { ATC_TUNABLE, 42, 0x0, 0x0, 0x3, 0 }, + { ATC_TUNABLE, 20, 0x0, 0x0, 0x4, 0 }, + { ATC_TUNABLE, 29, 0x0, 0x0, 0x5, 0 }, + { ATC_TUNABLE, 18, 0x0, 0x0, 0x6, 0 }, + { ATC_TUNABLE, 27, 0x0, 0x0, 0x7, 0 }, + { ATC_MASK, 4, 0x0, 0x3f0, 0x1c0, 0 }, + { ATC_MASK, 4, 0x18, 0xf000, 0x3000, 0 }, + { ATC_MASK, 9, 0x88, 0xff, 0x27, 0 }, + { ATC_MASK, 33, 0x4, 0x38, 0x8, 0 }, + { ATC_MASK, 10, 0xdc, 0x3ff0, 0x1760, 0 }, + { ATC_MASK, 21, 0xcc, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 30, 0xcc, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 20, 0x90, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 29, 0x90, 0x1f80000, 0x380000, 0 }, + { ATC_TUNABLE, 8, 0x0, 0x0, 0x8, 0 }, + { ATC_MASK, 2, 0x6c, 0x3c000fc, 0x48, 0 }, + { ATC_MASK, 2, 0x78, 0x1f, 0x14, 0 }, + { ATC_MASK, 2, 0x78, 0x3ffe0, 0x22000, 0 }, + { ATC_MASK, 2, 0x6c, 0x3, 0x2, 0 }, + { ATC_MASK, 2, 0xc, 0x2a, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 39, 0x0, 0x10, 0x10, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x64, 0 }, +}; + +static const struct atc_op aux_on[] = { + { ATC_MASK, 34, 0x0, 0x3, 0x3, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 34, 0x0, 0xc, 0xc, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 34, 0x0, 0x300, 0x100, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 33, 0x0, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, +}; + +static const struct atc_op aux_off[] = { + { ATC_MASK, 33, 0x0, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 34, 0x0, 0x300, 0x300, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 34, 0x0, 0x8, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 34, 0x0, 0x2, 0x0, 0 }, +}; + +static const struct atc_op dp_poweroff[] = { + { ATC_MASK, 2, 0x120, 0x100, 0x100, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x200, 0x200, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x20, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 5, 0x0, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x10, 0x10, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xc8, 0 }, + { ATC_POWER, 2, 0x0, 0x0, 0x0, 0 }, + { ATC_MASK, 39, 0x2c, 0x1, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x100, 0x100, 1000 }, + { ATC_MASK, 39, 0x0, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x8, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x4, 0x4, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x2, 0x0, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x1, 0x0, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x8, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, +}; + +static const struct atc_op dp_link_off[] = { + { ATC_MASK, 39, 0x14, 0x1007, 0x7, 0 }, + { ATC_MASK, 39, 0x18, 0x1, 0x0, 0 }, + { ATC_POLL, 39, 0x1c, 0x1, 0x0, 5000 }, + { ATC_MASK, 14, 0x0, 0x1, 0x1, 0 }, + { ATC_MASK, 22, 0x1c, 0xc00000, 0xc00000, 0 }, + { ATC_MASK, 22, 0x10, 0x600, 0x400, 0 }, + { ATC_MASK, 22, 0x4, 0x30000000, 0x20000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x100000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x400000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x4000000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x1000000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x1000, 0x0, 0 }, + { ATC_MASK, 2, 0x120, 0x100, 0x100, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x1c, 0xc00000, 0xc00000, 0 }, + { ATC_MASK, 31, 0x10, 0x600, 0x400, 0 }, + { ATC_MASK, 31, 0x4, 0x30000000, 0x20000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x100000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x400000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x4000000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x1000000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x1000, 0x0, 0 }, + { ATC_MASK, 2, 0x120, 0x200, 0x200, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x20, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x44, 0x6000000, 0x4000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x44, 0x18000000, 0x10000000, 0 }, + { ATC_MASK, 19, 0x24, 0xc0000000, 0x80000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x24, 0x30000000, 0x20000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x28, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x40, 0xc000, 0xc000, 0 }, + { ATC_MASK, 16, 0x1d0, 0x3, 0x1, 0 }, + { ATC_MASK, 19, 0x40, 0x10000, 0x0, 0 }, + { ATC_MASK, 2, 0xc, 0x4000, 0x4000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x40, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x400, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x44, 0x6000000, 0x4000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x44, 0x18000000, 0x10000000, 0 }, + { ATC_MASK, 28, 0x24, 0xc0000000, 0x80000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x24, 0x30000000, 0x20000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x28, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x40, 0xc000, 0xc000, 0 }, + { ATC_MASK, 25, 0x1d0, 0x3, 0x1, 0 }, + { ATC_MASK, 28, 0x40, 0x10000, 0x0, 0 }, + { ATC_MASK, 2, 0xc, 0x8000, 0x8000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x80, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x800, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x20, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 14, 0x0, 0xc, 0x8, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x440000, 0x440000, 0 }, + { ATC_MASK, 3, 0x8, 0x5, 0x5, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x880000, 0x0, 0 }, + { ATC_MASK, 3, 0x8, 0xa, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x880000, 0x880000, 0 }, + { ATC_MASK, 3, 0x8, 0xa, 0xa, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x440000, 0x0, 0 }, + { ATC_MASK, 3, 0x8, 0x5, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x400000, 0x400000, 0 }, + { ATC_MASK, 3, 0x8, 0x4, 0x4, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x800000, 0x0, 0 }, + { ATC_MASK, 3, 0x8, 0x8, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x800000, 0x800000, 0 }, + { ATC_MASK, 3, 0x8, 0x8, 0x8, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x400000, 0x0, 0 }, + { ATC_MASK, 3, 0x8, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x40000, 0x40000, 0 }, + { ATC_MASK, 3, 0x8, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x80000, 0x0, 0 }, + { ATC_MASK, 3, 0x8, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x80000, 0x80000, 0 }, + { ATC_MASK, 3, 0x8, 0x2, 0x2, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x40000, 0x0, 0 }, + { ATC_MASK, 3, 0x8, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x7, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x8, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x100, 500000 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x0, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x0, 500000 }, + { ATC_POLL, 14, 0x34, 0x8, 0x0, 500000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 6, 0x34, 0xf, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x7, 0x0, 0 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x8, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x100, 500000 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x0, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x0, 500000 }, + { ATC_MASK, 8, 0x0, 0xaaa, 0x0, 0 }, +}; + +static const struct atc_rate_op dp_link_on[] = { + { { ATC_MASK, 39, 0x14, 0x7, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 39, 0x18, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x6c, 0xfc, 0, 0 }, 0xf, { 0x48, 0x48, 0x48, 0x48 } }, + { { ATC_MASK, 14, 0x0, 0x40fff0, 0, 0 }, 0xf, { 0x1110, 0x1110, 0x1110, 0x1110 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x1, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x1, { 0x1e0e021c, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x1, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x1, { 0x156600, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x1, { 0x70000, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x1, { 0x1, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x2, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x2, { 0x0, 0x1e0e01c2, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x2, { 0x0, 0x7fffffe, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x2, { 0x0, 0x155200, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x2, { 0x0, 0x70000, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x2, { 0x0, 0x1, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x4, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x1e0e01c2, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x7fffffe, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x554800, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x4, { 0x0, 0x0, 0x70000, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x4, { 0x0, 0x0, 0x1, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x1e0e02a3 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0xbff7ffc } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x564800 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x70000 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x1 } }, + { { ATC_MASK, 8, 0x0, 0xaaa, 0, 0 }, 0xf, { 0xaaa, 0xaaa, 0xaaa, 0xaaa } }, + { { ATC_MASK, 6, 0x30, 0x4000, 0, 0 }, 0xf, { 0x4000, 0x4000, 0x4000, 0x4000 } }, + { { ATC_MASK, 6, 0x34, 0xf, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 6, 0x18, 0x1fe000, 0, 0 }, 0xf, { 0x2000, 0x2000, 0x2000, 0x2000 } }, + { { ATC_MASK, 6, 0x20, 0x1, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 6, 0x1c, 0x7, 0, 0 }, 0xf, { 0x7, 0x7, 0x7, 0x7 } }, + { { ATC_MASK, 6, 0x1c, 0x8, 0, 0 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_POLL, 6, 0x2c, 0xff00, 0, 500000 }, 0xf, { 0x100, 0x100, 0x100, 0x100 } }, + { { ATC_MASK, 6, 0x1c, 0x8, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 6, 0x2c, 0xff00, 0, 500000 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 14, 0x34, 0x8, 0, 500000 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_MASK, 19, 0x3c, 0x18000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_MASK, 28, 0x3c, 0x18000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_MASK, 14, 0x0, 0xc, 0, 0 }, 0xf, { 0xc, 0xc, 0xc, 0xc } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x10, 0, 0 }, 0xf, { 0x10, 0x10, 0x10, 0x10 } }, + { { ATC_MASK, 2, 0x120, 0x40, 0, 0 }, 0xf, { 0x40, 0x40, 0x40, 0x40 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x4, 0, 0 }, 0xf, { 0x4, 0x4, 0x4, 0x4 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x100, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 2, 0x120, 0x400, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0xfffffc0, 0, 0 }, 0xf, { 0xaafb800, 0xaafb800, 0xaafb800, 0xaafb800 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x1000000, 0, 0 }, 0xf, { 0x1000000, 0x1000000, 0x1000000, 0x1000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x4000000, 0, 0 }, 0xf, { 0x4000000, 0x4000000, 0x4000000, 0x4000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x7c0, 0, 0 }, 0xf, { 0x300, 0x300, 0x300, 0x300 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x6c000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 21, 0x100, 0x100000, 0, 0 }, 0xf, { 0x100000, 0x100000, 0x100000, 0x100000 } }, + { { ATC_MASK, 21, 0x58, 0xc00f, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 21, 0x104, 0x20000000, 0, 0 }, 0xf, { 0x20000000, 0x20000000, 0x20000000, 0x20000000 } }, + { { ATC_MASK, 21, 0x100, 0x7ffff, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x100000, 0, 0 }, 0x7, { 0x100000, 0x100000, 0x100000, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0x7, { 0x1, 0x1, 0x1, 0x0 } }, + { { ATC_MASK, 22, 0x44, 0x400000, 0, 0 }, 0x7, { 0x400000, 0x400000, 0x400000, 0x0 } }, + { { ATC_MASK, 22, 0x4, 0x3000000, 0, 0 }, 0xf, { 0x2000000, 0x2000000, 0x2000000, 0x2000000 } }, + { { ATC_MASK, 22, 0x4, 0x30000000, 0, 0 }, 0xf, { 0x30000000, 0x30000000, 0x30000000, 0x30000000 } }, + { { ATC_MASK, 22, 0xc, 0x7000000, 0, 0 }, 0x1, { 0x5000000, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 22, 0xc, 0x7000000, 0, 0 }, 0xe, { 0x0, 0x4000000, 0x4000000, 0x4000000 } }, + { { ATC_MASK, 22, 0x8, 0x3, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 22, 0x10, 0x600, 0, 0 }, 0xf, { 0x600, 0x600, 0x600, 0x600 } }, + { { ATC_MASK, 22, 0x1c, 0xc00000, 0, 0 }, 0xf, { 0x800000, 0x800000, 0x800000, 0x800000 } }, + { { ATC_MASK, 2, 0x120, 0x20, 0, 0 }, 0xf, { 0x20, 0x20, 0x20, 0x20 } }, + { { ATC_MASK, 2, 0x120, 0x80, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x2, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 2, 0x120, 0x8, 0, 0 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x200, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 2, 0x120, 0x800, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0xfffffc0, 0, 0 }, 0xf, { 0xaafb800, 0xaafb800, 0xaafb800, 0xaafb800 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x1000000, 0, 0 }, 0xf, { 0x1000000, 0x1000000, 0x1000000, 0x1000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x4000000, 0, 0 }, 0xf, { 0x4000000, 0x4000000, 0x4000000, 0x4000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x7c0, 0, 0 }, 0xf, { 0x300, 0x300, 0x300, 0x300 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x6c000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 30, 0x100, 0x100000, 0, 0 }, 0xf, { 0x100000, 0x100000, 0x100000, 0x100000 } }, + { { ATC_MASK, 30, 0x58, 0xc00f, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 30, 0x104, 0x20000000, 0, 0 }, 0xf, { 0x20000000, 0x20000000, 0x20000000, 0x20000000 } }, + { { ATC_MASK, 30, 0x100, 0x7ffff, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x100000, 0, 0 }, 0x7, { 0x100000, 0x100000, 0x100000, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0x7, { 0x1, 0x1, 0x1, 0x0 } }, + { { ATC_MASK, 31, 0x44, 0x400000, 0, 0 }, 0x7, { 0x400000, 0x400000, 0x400000, 0x0 } }, + { { ATC_MASK, 31, 0x4, 0x3000000, 0, 0 }, 0xf, { 0x2000000, 0x2000000, 0x2000000, 0x2000000 } }, + { { ATC_MASK, 31, 0x4, 0x30000000, 0, 0 }, 0xf, { 0x30000000, 0x30000000, 0x30000000, 0x30000000 } }, + { { ATC_MASK, 31, 0xc, 0x7000000, 0, 0 }, 0x1, { 0x5000000, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 31, 0xc, 0x7000000, 0, 0 }, 0xe, { 0x0, 0x4000000, 0x4000000, 0x4000000 } }, + { { ATC_MASK, 31, 0x8, 0x3, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 31, 0x10, 0x600, 0, 0 }, 0xf, { 0x600, 0x600, 0x600, 0x600 } }, + { { ATC_MASK, 31, 0x1c, 0xc00000, 0, 0 }, 0xf, { 0x800000, 0x800000, 0x800000, 0x800000 } }, + { { ATC_MASK, 19, 0xf4, 0x3000, 0, 0 }, 0xf, { 0x3000, 0x3000, 0x3000, 0x3000 } }, + { { ATC_MASK, 16, 0x188, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 16, 0x1d4, 0x700, 0, 0 }, 0xf, { 0x300, 0x300, 0x300, 0x300 } }, + { { ATC_MASK, 19, 0x24, 0x3f0000, 0, 0 }, 0xf, { 0x2a0000, 0x2a0000, 0x2a0000, 0x2a0000 } }, + { { ATC_MASK, 19, 0x3c, 0x18000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_MASK, 19, 0x78, 0xc0000, 0, 0 }, 0xf, { 0x80000, 0x80000, 0x80000, 0x80000 } }, + { { ATC_MASK, 19, 0x7c, 0xc0, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_MASK, 19, 0x84, 0xc000, 0, 0 }, 0xf, { 0x8000, 0x8000, 0x8000, 0x8000 } }, + { { ATC_MASK, 19, 0xd4, 0x600000, 0, 0 }, 0xf, { 0x400000, 0x400000, 0x400000, 0x400000 } }, + { { ATC_MASK, 19, 0xd8, 0x300, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_MASK, 19, 0x40, 0xc00, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 19, 0x48, 0x600, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_MASK, 19, 0x28, 0x1ff030, 0, 0 }, 0xf, { 0x11c020, 0x11c020, 0x11c020, 0x11c020 } }, + { { ATC_MASK, 19, 0x2c, 0xfc, 0, 0 }, 0xf, { 0xb0, 0xb0, 0xb0, 0xb0 } }, + { { ATC_MASK, 19, 0x28, 0xe00, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 19, 0x28, 0x1c0, 0, 0 }, 0xf, { 0x180, 0x180, 0x180, 0x180 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x10, 0, 0 }, 0xf, { 0x10, 0x10, 0x10, 0x10 } }, + { { ATC_MASK, 2, 0x10, 0x40, 0, 0 }, 0xf, { 0x40, 0x40, 0x40, 0x40 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x4, 0, 0 }, 0xf, { 0x4, 0x4, 0x4, 0x4 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x400, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_MASK, 2, 0xc, 0x1000, 0, 0 }, 0xf, { 0x1000, 0x1000, 0x1000, 0x1000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x40, 0, 0 }, 0xf, { 0x40, 0x40, 0x40, 0x40 } }, + { { ATC_MASK, 2, 0xc, 0x100, 0, 0 }, 0xf, { 0x100, 0x100, 0x100, 0x100 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x4000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 2, 0xc, 0x10000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 16, 0x1d0, 0x3, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 16, 0x1d0, 0x2, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x40, 0x30000, 0, 0 }, 0xf, { 0x30000, 0x30000, 0x30000, 0x30000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x40, 0xc000, 0, 0 }, 0xf, { 0xc000, 0xc000, 0xc000, 0xc000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x3010, 0, 0 }, 0xf, { 0x1010, 0x1010, 0x1010, 0x1010 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x600, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x3000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x24, 0x30000000, 0, 0 }, 0xf, { 0x30000000, 0x30000000, 0x30000000, 0x30000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x24, 0xc0000000, 0, 0 }, 0xf, { 0xc0000000, 0xc0000000, 0xc0000000, 0xc0000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x600, 0, 0 }, 0xf, { 0x600, 0x600, 0x600, 0x600 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x600, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x600, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 19, 0x44, 0x18000000, 0, 0 }, 0xf, { 0x18000000, 0x18000000, 0x18000000, 0x18000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 19, 0x40, 0x38, 0, 0 }, 0x1, { 0x28, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 19, 0x40, 0x38, 0, 0 }, 0xe, { 0x0, 0x20, 0x20, 0x20 } }, + { { ATC_MASK, 19, 0x44, 0x60000000, 0, 0 }, 0xf, { 0x40000000, 0x40000000, 0x40000000, 0x40000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x44, 0x6000000, 0, 0 }, 0xf, { 0x6000000, 0x6000000, 0x6000000, 0x6000000 } }, + { { ATC_MASK, 19, 0x40, 0xc000, 0, 0 }, 0xf, { 0x8000, 0x8000, 0x8000, 0x8000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 20, 0x68, 0xc00f, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 20, 0xac, 0x20000000, 0, 0 }, 0xf, { 0x20000000, 0x20000000, 0x20000000, 0x20000000 } }, + { { ATC_MASK, 20, 0xa8, 0x7ffff, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 20, 0x18, 0x38, 0, 0 }, 0xf, { 0x20, 0x20, 0x20, 0x20 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0xf4, 0x3000, 0, 0 }, 0xf, { 0x3000, 0x3000, 0x3000, 0x3000 } }, + { { ATC_MASK, 25, 0x188, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 25, 0x1d4, 0x700, 0, 0 }, 0xf, { 0x300, 0x300, 0x300, 0x300 } }, + { { ATC_MASK, 28, 0x24, 0x3f0000, 0, 0 }, 0xf, { 0x2a0000, 0x2a0000, 0x2a0000, 0x2a0000 } }, + { { ATC_MASK, 28, 0x3c, 0x18000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_MASK, 28, 0x78, 0xc0000, 0, 0 }, 0xf, { 0x80000, 0x80000, 0x80000, 0x80000 } }, + { { ATC_MASK, 28, 0x7c, 0xc0, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_MASK, 28, 0x84, 0xc000, 0, 0 }, 0xf, { 0x8000, 0x8000, 0x8000, 0x8000 } }, + { { ATC_MASK, 28, 0xd4, 0x600000, 0, 0 }, 0xf, { 0x400000, 0x400000, 0x400000, 0x400000 } }, + { { ATC_MASK, 28, 0xd8, 0x300, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_MASK, 28, 0x40, 0xc00, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 28, 0x48, 0x600, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_MASK, 28, 0x28, 0x1ff030, 0, 0 }, 0xf, { 0x11c020, 0x11c020, 0x11c020, 0x11c020 } }, + { { ATC_MASK, 28, 0x2c, 0xfc, 0, 0 }, 0xf, { 0xb0, 0xb0, 0xb0, 0xb0 } }, + { { ATC_MASK, 28, 0x28, 0xe00, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 28, 0x28, 0x1c0, 0, 0 }, 0xf, { 0x180, 0x180, 0x180, 0x180 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x20, 0, 0 }, 0xf, { 0x20, 0x20, 0x20, 0x20 } }, + { { ATC_MASK, 2, 0x10, 0x80, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x2, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 2, 0x10, 0x8, 0, 0 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x800, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 2, 0xc, 0x2000, 0, 0 }, 0xf, { 0x2000, 0x2000, 0x2000, 0x2000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x80, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_MASK, 2, 0xc, 0x200, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x8000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 2, 0xc, 0x20000, 0, 0 }, 0xf, { 0x20000, 0x20000, 0x20000, 0x20000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 25, 0x1d0, 0x3, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 25, 0x1d0, 0x2, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x40, 0x30000, 0, 0 }, 0xf, { 0x30000, 0x30000, 0x30000, 0x30000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x40, 0xc000, 0, 0 }, 0xf, { 0xc000, 0xc000, 0xc000, 0xc000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x3010, 0, 0 }, 0xf, { 0x1010, 0x1010, 0x1010, 0x1010 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x600, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x3000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x24, 0x30000000, 0, 0 }, 0xf, { 0x30000000, 0x30000000, 0x30000000, 0x30000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x24, 0xc0000000, 0, 0 }, 0xf, { 0xc0000000, 0xc0000000, 0xc0000000, 0xc0000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x600, 0, 0 }, 0xf, { 0x600, 0x600, 0x600, 0x600 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x600, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x600, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 28, 0x44, 0x18000000, 0, 0 }, 0xf, { 0x18000000, 0x18000000, 0x18000000, 0x18000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 28, 0x40, 0x38, 0, 0 }, 0x1, { 0x28, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 28, 0x40, 0x38, 0, 0 }, 0xe, { 0x0, 0x20, 0x20, 0x20 } }, + { { ATC_MASK, 28, 0x44, 0x60000000, 0, 0 }, 0xf, { 0x40000000, 0x40000000, 0x40000000, 0x40000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x44, 0x6000000, 0, 0 }, 0xf, { 0x6000000, 0x6000000, 0x6000000, 0x6000000 } }, + { { ATC_MASK, 28, 0x40, 0xc000, 0, 0 }, 0xf, { 0x8000, 0x8000, 0x8000, 0x8000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 29, 0x68, 0xc00f, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 29, 0xac, 0x20000000, 0, 0 }, 0xf, { 0x20000000, 0x20000000, 0x20000000, 0x20000000 } }, + { { ATC_MASK, 29, 0xa8, 0x7ffff, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 29, 0x18, 0x38, 0, 0 }, 0xf, { 0x20, 0x20, 0x20, 0x20 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 14, 0x0, 0x1, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 39, 0x1c, 0x1, 0, 5000 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, +}; + +static const u32 dp_training_default[16] = { + 0x3f024, 0x237013, 0x37200b, 0x46e004, 0x3f01c, 0x23700b, 0x372004, 0x3f024, 0x3f013, 0x237004, 0x3f024, 0x3f024, 0x3f004, 0x3f024, 0x3f024, 0x3f024 +}; diff --git a/drivers/phy/apple/atc-t6050-dp.c b/drivers/phy/apple/atc-t6050-dp.c new file mode 100644 index 0000000000000..64a951ab0af95 --- /dev/null +++ b/drivers/phy/apple/atc-t6050-dp.c @@ -0,0 +1,465 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* T6050 fixed four-lane DisplayPort PHY (HDMI bridge path). */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum atc_operation { ATC_MASK, ATC_TUNABLE, ATC_DELAY, ATC_POLL, ATC_POWER }; +struct atc_op { + u8 op, bank; + u16 offset; + u32 mask, value, timeout; +}; +struct atc_rate_op { + struct atc_op op; + u8 rates; + u32 values[4]; +}; +#include "atc-t6050-dp-data.h" + +static const struct { u8 bank; u32 offset; } core_banks[] = { + {2, 0}, {3, 0x400}, {4, 0x800}, {5, 0xa00}, {6, 0x1000}, + {7, 0x3000}, {8, 0x3200}, {9, 0x3800}, {10, 0x3a00}, + {14, 0x7000}, {16, 0x9000}, {18, 0xb000}, {19, 0xc000}, + {20, 0xd000}, {21, 0xe000}, {22, 0xf000}, {25, 0x19000}, + {27, 0x1b000}, {28, 0x1c000}, {29, 0x1d000}, {30, 0x1e000}, + {31, 0x1f000}, {33, 0x28000}, {34, 0x28400}, +}; +static const struct { u8 bank; const char *name; } extra_banks[] = { + {39, "reset"}, {40, "fabric"}, {41, "axi2af"}, + {42, "lioa"}, {43, "common-cfg"}, +}; +static const struct { u8 bank; const char *name; } tunable_names[] = { + {41, "apple,tunable-axi2af"}, {40, "apple,tunable-fabric"}, + {43, "apple,tunable-common-cfg"}, {42, "apple,tunable-lioa"}, + {20, "apple,tunable-lane0-rx-clock"}, {29, "apple,tunable-lane1-rx-clock"}, + {18, "apple,tunable-lane0-rx-eq"}, {27, "apple,tunable-lane1-rx-eq"}, + {8, "apple,tunable-auspll"}, +}; +static const unsigned int link_rates[] = {1620, 2700, 5400, 8100}; +static const char * const training_names[] = { + "apple,dp-training-table-rbr", "apple,dp-training-table-hbr", + "apple,dp-training-table-hbr2", "apple,dp-training-table-hbr3", +}; +struct atc_domain { + struct device *dev; + bool active; +}; +struct t6050_dp { + struct device *dev; + struct mutex lock; + void __iomem *banks[44]; + struct resource resources[44]; + struct apple_tunable *tunables[ARRAY_SIZE(tunable_names)]; + struct atc_domain domain[3]; /* USB AON, common, wrapper: native order. */ + u32 training[4][16]; + u32 voltage[4], pre[4]; + int rate; + bool enabled, faulted; +}; + +static int atc_domain_set(struct atc_domain *pd, bool active) +{ + int ret; + + if (pd->active == active) + return 0; + if (active) { + ret = pm_runtime_resume_and_get(pd->dev); + if (ret < 0) + return ret; + } else { + ret = pm_runtime_put_sync_suspend(pd->dev); + if (ret < 0) { + /* put decremented usage even when suspend failed. */ + pm_runtime_get_noresume(pd->dev); + return ret; + } + } + pd->active = active; + return 0; +} + +static void atc_domain_release(void *data) +{ + struct atc_domain *pd = data; + + if (pd->active) + pm_runtime_put_sync_suspend(pd->dev); + dev_pm_domain_detach(pd->dev, true); +} + +static int atc_step(struct t6050_dp *dp, const struct atc_op *s) +{ + void __iomem *reg; + u32 value; + int ret; + + if (s->op == ATC_DELAY) { + udelay(s->value); + return 0; + } + if (s->op == ATC_POWER) + return atc_domain_set(&dp->domain[2], s->value); + if (s->bank >= ARRAY_SIZE(dp->banks) || !dp->banks[s->bank] || + s->offset + 4 > resource_size(&dp->resources[s->bank])) + return -EINVAL; + reg = dp->banks[s->bank] + s->offset; + switch (s->op) { + case ATC_MASK: + value = readl(reg); + writel((value & ~s->mask) | s->value, reg); + return 0; + case ATC_TUNABLE: + if (s->value >= ARRAY_SIZE(dp->tunables)) + return -EINVAL; + apple_tunable_apply(dp->banks[s->bank], dp->tunables[s->value]); + return 0; + case ATC_POLL: + ret = readl_poll_timeout(reg, value, (value & s->mask) == s->value, + s->timeout > 1000 ? 1000 : 1, s->timeout); + if (ret) + dev_err(dp->dev, "bank%u+%x timeout: %x & %x != %x\n", + s->bank, s->offset, value, s->mask, s->value); + return ret; + default: + return -EINVAL; + } +} + +static int atc_sequence(struct t6050_dp *dp, const struct atc_op *s, size_t count) +{ + for (size_t i = 0; i < count; i++) { + int ret = atc_step(dp, &s[i]); + + if (ret) { + dp->faulted = true; + return ret; + } + } + return 0; +} +#define ATC_SEQUENCE(dp, seq) atc_sequence(dp, seq, ARRAY_SIZE(seq)) + +static int atc_link_rate(struct t6050_dp *dp, int rate) +{ + int ret; + + if (dp->rate == rate) + return 0; + if (dp->rate >= 0) { + ret = ATC_SEQUENCE(dp, dp_link_off); + if (ret) + return ret; + dp->rate = -1; + } + if (rate < 0) + return 0; + for (size_t i = 0; i < ARRAY_SIZE(dp_link_on); i++) { + struct atc_op op = dp_link_on[i].op; + + if (!(dp_link_on[i].rates & BIT(rate))) + continue; + op.value = dp_link_on[i].values[rate]; + ret = atc_step(dp, &op); + if (ret) { + dp->faulted = true; + return ret; + } + } + dp->rate = rate; + memset(dp->voltage, 0, sizeof(dp->voltage)); + memset(dp->pre, 0, sizeof(dp->pre)); + return 0; +} + +static int atc_start(struct t6050_dp *dp) +{ + int ret; + + if (dp->faulted) + return -EIO; + if (dp->enabled) + return 0; + ret = atc_domain_set(&dp->domain[0], true); + if (ret) + return ret; + ret = atc_domain_set(&dp->domain[1], true); + if (ret) { + atc_domain_set(&dp->domain[0], false); + return ret; + } + ret = ATC_SEQUENCE(dp, dp_init); + if (!ret) + ret = ATC_SEQUENCE(dp, aux_on); + if (ret) + return ret; + dp->enabled = true; + return 0; +} + +static int atc_stop(struct t6050_dp *dp) +{ + int ret; + + if (dp->faulted) + return -EIO; + if (!dp->enabled) + return 0; + ret = atc_link_rate(dp, -1); + if (!ret) + ret = ATC_SEQUENCE(dp, aux_off); + if (!ret) + ret = ATC_SEQUENCE(dp, dp_poweroff); + if (ret) + return ret; + dp->enabled = false; + ret = atc_domain_set(&dp->domain[1], false); + if (!ret) + ret = atc_domain_set(&dp->domain[0], false); + return ret; +} + +static void atc_mask(void __iomem *reg, u32 mask, u32 value) +{ + writel((readl(reg) & ~mask) | value, reg); +} + +static void atc_drive(struct t6050_dp *dp, unsigned int lane, u32 preset) +{ + /* Fixed normal DP orientation: group1 RX/TX, then group0 RX/TX. */ + unsigned int group = lane < 2 ? 1 : 0; + bool tx = lane & 1; + void __iomem *base = dp->banks[(tx ? 21 : 20) + 9 * group]; + u32 ctrl = ((preset >> 2) & 0xf) | ((preset & 3) << 14); + u32 eq = ((preset >> 5) & 0x7fffe) | 1; + + if (tx) + atc_mask(base + 0x100, BIT(20), BIT(20)); + atc_mask(base + (tx ? 0x58 : 0x68), 0xc00f, ctrl); + atc_mask(base + (tx ? 0x104 : 0xac), BIT(29), BIT(29)); + atc_mask(base + (tx ? 0x100 : 0xa8), 0x7ffff, eq); +} + +static int t6050_dp_set_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + + guard(mutex)(&dp->lock); + if (mode == PHY_MODE_DP) + return atc_start(dp); + if (mode == PHY_MODE_INVALID) + return atc_stop(dp); + return -EINVAL; +} + +static int t6050_dp_validate(struct phy *phy, enum phy_mode mode, int submode, + union phy_configure_opts *opts) +{ + if (mode != PHY_MODE_DP) + return -EINVAL; + memset(opts, 0, sizeof(*opts)); + opts->dp.lanes = 4; + opts->dp.link_rate = 8100; + opts->dp.ssc = 0; + return 0; +} + +static int t6050_dp_configure(struct phy *phy, union phy_configure_opts *opts) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + struct phy_configure_opts_dp *cfg = &opts->dp; + int rate = -1, ret; + + /* Validate the whole request before altering a running transmitter. */ + if (cfg->set_rate && cfg->link_rate) { + for (unsigned int i = 0; i < ARRAY_SIZE(link_rates); i++) + if (cfg->link_rate == link_rates[i]) + rate = i; + if (rate < 0 || cfg->ssc) + return -EINVAL; + } + if ((cfg->set_lanes || cfg->set_voltages) && + cfg->lanes != 0 && cfg->lanes != 1 && cfg->lanes != 2 && cfg->lanes != 4) + return -EINVAL; + if (cfg->set_voltages) + for (unsigned int i = 0; i < cfg->lanes; i++) + if (cfg->voltage[i] > 3 || cfg->pre[i] > 3 || + cfg->voltage[i] + cfg->pre[i] > 3) + return -EINVAL; + if (cfg->set_voltages && cfg->set_rate && rate < 0 && cfg->lanes) + return -EINVAL; + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + if (!dp->enabled) + return -EPIPE; + if (cfg->set_rate) { + ret = atc_link_rate(dp, rate); + if (ret) + return ret; + } + /* Native setActiveLaneCount is a no-op; DCP controls the active count. */ + if (cfg->set_voltages) { + if (dp->rate < 0) + return -EPIPE; + for (unsigned int i = 0; i < cfg->lanes; i++) { + u32 preset = dp->training[dp->rate][4 * cfg->voltage[i] + cfg->pre[i]]; + + atc_drive(dp, i, preset); + dp->voltage[i] = cfg->voltage[i]; + dp->pre[i] = cfg->pre[i]; + } + } + return 0; +} + +static const struct phy_ops t6050_dp_ops = { + .set_mode = t6050_dp_set_mode, + .validate = t6050_dp_validate, + .configure = t6050_dp_configure, + .owner = THIS_MODULE, +}; + +int apple_t6050_dp_link_config(struct phy *phy, unsigned int *pclk, bool *uhbr) +{ + struct t6050_dp *dp; + + if (!phy || phy->ops != &t6050_dp_ops) + return -EOPNOTSUPP; + dp = phy_get_drvdata(phy); + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + if (!dp->enabled || dp->rate < 0) + return -ENOLINK; + /* Native direct DPPHY uses logical clock0, mapped to xbar PCLK1. + * The other four logical clocks are separately managed tunnel clocks. */ + *pclk = 1; + *uhbr = false; + return 0; +} +EXPORT_SYMBOL_GPL(apple_t6050_dp_link_config); + +static int t6050_dp_probe(struct platform_device *pdev) +{ + static const char * const domains[] = {"usb-aon", "common", "wrapper"}; + struct device *dev = &pdev->dev; + struct t6050_dp *dp; + struct phy_provider *provider; + struct phy *phy; + struct resource *core; + void __iomem *base; + int ret; + + dp = devm_kzalloc(dev, sizeof(*dp), GFP_KERNEL); + if (!dp) + return -ENOMEM; + dp->dev = dev; + dp->rate = -1; + mutex_init(&dp->lock); + platform_set_drvdata(pdev, dp); + core = platform_get_resource_byname(pdev, IORESOURCE_MEM, "core"); + if (!core || resource_size(core) < 0x2c400) + return -EINVAL; + base = devm_ioremap_resource(dev, core); + if (IS_ERR(base)) + return PTR_ERR(base); + for (unsigned int i = 0; i < ARRAY_SIZE(core_banks); i++) { + u8 bank = core_banks[i].bank; + + dp->banks[bank] = base + core_banks[i].offset; + dp->resources[bank] = *core; + dp->resources[bank].start += core_banks[i].offset; + dp->resources[bank].end = dp->resources[bank].start + 0x4000 - 1; + } + for (unsigned int i = 0; i < ARRAY_SIZE(extra_banks); i++) { + u8 bank = extra_banks[i].bank; + struct resource *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + extra_banks[i].name); + + if (!res) + return -EINVAL; + dp->banks[bank] = devm_ioremap_resource(dev, res); + if (IS_ERR(dp->banks[bank])) + return PTR_ERR(dp->banks[bank]); + dp->resources[bank] = *res; + } + for (unsigned int i = 0; i < ARRAY_SIZE(tunable_names); i++) { + dp->tunables[i] = devm_apple_tunable_parse(dev, dev->of_node, + tunable_names[i].name, &dp->resources[tunable_names[i].bank]); + if (IS_ERR(dp->tunables[i])) + return dev_err_probe(dev, PTR_ERR(dp->tunables[i]), + "missing or invalid %s\n", tunable_names[i].name); + } + for (unsigned int i = 0; i < ARRAY_SIZE(training_names); i++) { + const char *name = training_names[i]; + + if (!of_property_present(dev->of_node, name)) + name = "apple,dp-training-table"; + if (!of_property_present(dev->of_node, name)) { + memcpy(dp->training[i], dp_training_default, sizeof(dp_training_default)); + continue; + } + if (of_property_count_u32_elems(dev->of_node, name) != 16) + return -EINVAL; + ret = of_property_read_u32_array(dev->of_node, name, dp->training[i], 16); + if (ret) + return ret; + } + /* Three named domains prevent automatic wrapper enable at probe time. */ + if (of_count_phandle_with_args(dev->of_node, "power-domains", "#power-domain-cells") != 3) + return -EINVAL; + for (unsigned int i = 0; i < ARRAY_SIZE(domains); i++) { + dp->domain[i].dev = dev_pm_domain_attach_by_name(dev, domains[i]); + if (IS_ERR_OR_NULL(dp->domain[i].dev)) + return dev_err_probe(dev, dp->domain[i].dev ? PTR_ERR(dp->domain[i].dev) : -ENODEV, + "cannot attach %s domain\n", domains[i]); + ret = devm_add_action_or_reset(dev, atc_domain_release, &dp->domain[i]); + if (ret) + return ret; + } + phy = devm_phy_create(dev, NULL, &t6050_dp_ops); + if (IS_ERR(phy)) + return PTR_ERR(phy); + phy_set_drvdata(phy, dp); + provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); + return PTR_ERR_OR_ZERO(provider); +} + +static void t6050_dp_shutdown(struct platform_device *pdev) +{ + struct t6050_dp *dp = platform_get_drvdata(pdev); + + guard(mutex)(&dp->lock); + if (atc_stop(dp)) + dev_warn(dp->dev, "PHY shutdown incomplete after an earlier error\n"); +} + +static const struct of_device_id t6050_dp_match[] = { + { .compatible = "apple,t6050-dp-phy" }, + {}, +}; +MODULE_DEVICE_TABLE(of, t6050_dp_match); +static struct platform_driver t6050_dp_driver = { + .probe = t6050_dp_probe, + .remove = t6050_dp_shutdown, + .shutdown = t6050_dp_shutdown, + .driver = { + .name = "apple-t6050-dp-phy", + .of_match_table = t6050_dp_match, + }, +}; +module_platform_driver(t6050_dp_driver); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple T6050 fixed DisplayPort PHY"); diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index 87f1836521066..8e0560ef76846 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -35,6 +35,9 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, */ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); +/* Fixed T6050 DP: obtain the live main-link clock for the physical crossbar. */ +int apple_t6050_dp_link_config(struct phy *phy, unsigned int *pclk, bool *uhbr); + /* * Display crossbar: take the connection of an already selected output down or * bring it back up (clock gates and enables) around a link reconfiguration. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp-direct-link.py b/tools/testing/selftests/drivers/gpu/apple-dcp-direct-link.py new file mode 100644 index 0000000000000..c29244d6696af --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp-direct-link.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise the actual optional direct-PHY crossbar bridge and error paths.""" +from pathlib import Path +import os,subprocess,tempfile +root=next(p for p in Path(__file__).resolve().parents if (p/'drivers/gpu/drm/apple/dcp.c').exists()) +s=(root/'drivers/gpu/drm/apple/dcp.c').read_text();a=s.index('int dcp_direct_crossbar_link(');b=s.index('\n}',a)+2 +source=r''' +#include +#include +#include +#include +#include +struct phy{};struct mux_control{}; +struct apple_dcp{bool dptx_tunnel;struct phy*phy;struct mux_control*xbar;}; +static bool have_apple_t6050_dp_link_config=true,have_apple_dpxbar_link_configure=true; +static int get_error,configure_error,link_error,gets,configs,links;static bool last_up; +static int apple_t6050_dp_link_config(struct phy*p,unsigned*c,bool*u){gets++;*c=1;*u=false;return get_error;} +static int apple_dpxbar_link_configure(struct mux_control*m,unsigned c,bool u){assert(c==1&&!u);configs++;return configure_error;} +static int dcp_dpxbar_link(struct mux_control*m,bool up){links++;last_up=up;return link_error;} +#define symbol_get(n) (have_##n ? n : NULL) +#define symbol_put(n) ((void)0) +'''+s[a:b]+r''' +int main(void){ +struct phy p;struct mux_control m;struct apple_dcp d={.phy=&p,.xbar=&m}; +assert(!dcp_direct_crossbar_link(&d,true)&&gets==1&&configs==1&&links==1&&last_up); +assert(!dcp_direct_crossbar_link(&d,false)&&configs==1&&links==2&&!last_up); +get_error=-ENOLINK;assert(!dcp_direct_crossbar_link(&d,false)&&links==3); +assert(dcp_direct_crossbar_link(&d,true)==-ENOLINK&&configs==1&&links==3); +get_error=-EIO;assert(dcp_direct_crossbar_link(&d,true)==-EIO&&links==3); +get_error=-EOPNOTSUPP;assert(!dcp_direct_crossbar_link(&d,true)&&links==3); +get_error=0;configure_error=-EBUSY;assert(dcp_direct_crossbar_link(&d,true)==-EBUSY&&links==3); +configure_error=0;link_error=-EIO;assert(dcp_direct_crossbar_link(&d,true)==-EIO&&links==4); +have_apple_dpxbar_link_configure=false;assert(dcp_direct_crossbar_link(&d,true)==-ENODEV&&links==4); +have_apple_t6050_dp_link_config=false;assert(!dcp_direct_crossbar_link(&d,true)&&links==4); +have_apple_t6050_dp_link_config=true;int n=gets;d.dptx_tunnel=true;assert(!dcp_direct_crossbar_link(&d,true)&&gets==n); +d.dptx_tunnel=false;d.phy=NULL;assert(!dcp_direct_crossbar_link(&d,true)&&gets==n); +d.phy=&p;d.xbar=NULL;assert(!dcp_direct_crossbar_link(&d,true)&&gets==n); +puts("DCP_DIRECT_LINK_TEST_PASS");return 0; +} +''' +if os.environ.get('TEST_SOURCE'):Path(os.environ['TEST_SOURCE']).write_text(source) +with tempfile.TemporaryDirectory() as d: + p=Path(d);(p/'test.c').write_text(source) + subprocess.run(['clang','-std=gnu11','-O1','-g','-fsanitize=address,undefined',str(p/'test.c'),'-o',str(p/'test')],check=True) + subprocess.run([str(p/'test')],check=True) diff --git a/tools/testing/selftests/drivers/phy/apple-t6050/Makefile b/tools/testing/selftests/drivers/phy/apple-t6050/Makefile new file mode 100644 index 0000000000000..d1ad2e271159d --- /dev/null +++ b/tools/testing/selftests/drivers/phy/apple-t6050/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Standalone external test module: make -C O= M=$PWD modules +obj-m += apple-t6050-driver-test.o +apple-t6050-driver-test-y := driver-test.o +ccflags-y += -I$(srctree)/drivers/phy/apple diff --git a/tools/testing/selftests/drivers/phy/apple-t6050/README b/tools/testing/selftests/drivers/phy/apple-t6050/README new file mode 100644 index 0000000000000..a75fe6a1be111 --- /dev/null +++ b/tools/testing/selftests/drivers/phy/apple-t6050/README @@ -0,0 +1,14 @@ +This opt-in test module includes the real fixed T6050 DP driver source. +It substitutes PM-domain operations and explicit poll acknowledgements, +using allocated memory instead of any hardware MMIO. It tests two cycles, +four link rates, drive requests, invalid settings and timeout stopping. +It deliberately logs a timeout for the missing-acknowledgement test. +It does not test physical link training, analog behavior or optical output. + +Build against a configured ARM64 kernel: + make -C O= ARCH=arm64 LLVM=1 M=$PWD modules + +In a QEMU Linux guest with the matching kernel, load apple-tunable.ko then +apple-t6050-driver-test.ko. Require AURORA_T6050_INTEGRATED_TEST_PASS in the +kernel log; module insertion failure is a test failure. Never use this +module's model acknowledgements as evidence about physical hardware. diff --git a/tools/testing/selftests/drivers/phy/apple-t6050/driver-test.c b/tools/testing/selftests/drivers/phy/apple-t6050/driver-test.c new file mode 100644 index 0000000000000..6f503b6fa8dda --- /dev/null +++ b/tools/testing/selftests/drivers/phy/apple-t6050/driver-test.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Actual driver source with explicit QEMU-only power/MMIO acknowledgements. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +static bool fail_ack; +static int refs[3], events[32], nevents; +static struct device domains[3]; +static void *reset_bank; +static int test_get(struct device *d) +{ + int i=d-domains; + if(i<0 || i>2)return -EINVAL; + if(i==2 && (readl(reset_bank)&3)!=3)return -EINVAL; + refs[i]++; events[nevents++]=i+1;return 0; +} +static int test_put(struct device *d) +{ + int i=d-domains; + if(i<0 || i>2 || refs[i]!=1)return -EINVAL; + refs[i]--;events[nevents++]=-(i+1);return 0; +} +#undef readl_poll_timeout +#define readl_poll_timeout(addr,val,cond,delay,timeout) ({ \ + u32 old=readl(addr); writel((old & ~s->mask) | (fail_ack ? s->value^s->mask : s->value),addr); \ + val=readl(addr); (cond) ? 0 : -ETIMEDOUT; }) +#define pm_runtime_resume_and_get test_get +#define pm_runtime_put_sync_suspend test_put +#undef module_platform_driver +#define module_platform_driver(x) +#undef EXPORT_SYMBOL_GPL +#define EXPORT_SYMBOL_GPL(x) +#define apple_t6050_dp_link_config test_link_config +#include "atc-t6050-dp.c" +#define CHECK(x) do {if(!(x)){pr_err("T6050 integrated check failed line%d: %s\n",__LINE__,#x);ret=-EINVAL;goto out;}}while(0) +static int __init test_init(void) +{ + struct t6050_dp dp={.rate=-1};struct phy phy={.ops=&t6050_dp_ops}; + union phy_configure_opts opts={};unsigned int pclk;bool uhbr;int ret=0; + void *mem=vzalloc(44*0x10000);struct device *dev=root_device_register("t6050-model"); + if(!mem)return -ENOMEM; + if(IS_ERR(dev)){vfree(mem);return PTR_ERR(dev);} + dp.dev=dev;mutex_init(&dp.lock);phy_set_drvdata(&phy,&dp); + for(int i=0;i<44;i++){dp.banks[i]=mem+i*0x10000;dp.resources[i].end=0x3fff;} + reset_bank=dp.banks[39]; + for(int i=0;i<3;i++)dp.domain[i].dev=&domains[i]; + for(int i=0;i<4;i++)memcpy(dp.training[i],dp_training_default,sizeof(dp_training_default)); + CHECK(t6050_dp_validate(&phy,PHY_MODE_DP,0,&opts)==0 && opts.dp.link_rate==8100 && opts.dp.lanes==4); + fail_ack=true;CHECK(t6050_dp_set_mode(&phy,PHY_MODE_DP,0)==-ETIMEDOUT); + CHECK(dp.faulted && !dp.enabled && refs[0]==1 && refs[1]==1); + CHECK(t6050_dp_set_mode(&phy,PHY_MODE_DP,0)==-EIO); + memset(mem,0,44*0x10000);memset(refs,0,sizeof(refs));memset(dp.domain,0,sizeof(dp.domain)); + for(int i=0;i<3;i++)dp.domain[i].dev=&domains[i]; + dp.faulted=false;fail_ack=false;nevents=0; + for(int cycle=0;cycle<2;cycle++){ + CHECK(t6050_dp_set_mode(&phy,PHY_MODE_DP,0)==0); + CHECK(dp.enabled && refs[0]==1 && refs[1]==1 && refs[2]==1); + CHECK(test_link_config(&phy,&pclk,&uhbr)==-ENOLINK); + for(int r=0;r<4;r++){ + memset(&opts,0,sizeof(opts));opts.dp.set_rate=1;opts.dp.link_rate=link_rates[r]; + CHECK(t6050_dp_configure(&phy,&opts)==0 && dp.rate==r); + CHECK(test_link_config(&phy,&pclk,&uhbr)==0 && pclk==1 && !uhbr); + for(int v=0;v<4;v++)for(int p=0;p<4-v;p++){ + memset(&opts,0,sizeof(opts));opts.dp.set_voltages=1;opts.dp.lanes=4; + for(int l=0;l<4;l++){opts.dp.voltage[l]=v;opts.dp.pre[l]=p;} + CHECK(t6050_dp_configure(&phy,&opts)==0); + } + opts.dp.voltage[0]=4;CHECK(t6050_dp_configure(&phy,&opts)==-EINVAL); + memset(&opts,0,sizeof(opts));opts.dp.set_rate=1;opts.dp.link_rate=123; + CHECK(t6050_dp_configure(&phy,&opts)==-EINVAL && dp.rate==r); + } + CHECK(t6050_dp_set_mode(&phy,PHY_MODE_INVALID,0)==0); + CHECK(!dp.enabled && dp.rate==-1 && !refs[0] && !refs[1] && !refs[2]); + CHECK(events[cycle*6]==1 && events[cycle*6+1]==2 && events[cycle*6+2]==3); + CHECK(events[cycle*6+3]==-3 && events[cycle*6+4]==-2 && events[cycle*6+5]==-1); + } + pr_info("AURORA_T6050_INTEGRATED_TEST_PASS: actual driver, four rates, two power cycles, drive settings, timeout and validation\n"); +out: root_device_unregister(dev);vfree(mem);return ret; +} +static void __exit test_exit(void){} +module_init(test_init);module_exit(test_exit); From 2e4b5aad993f10339b89c45f4ed4d54bd5ebaa48 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 28 Sep 2026 19:31:21 -0400 Subject: [PATCH 0230/1579] drm/apple: bring up firmware27 HDMI controls and native hotplug callbacks Expose native pcIO outputs and AOP host IRQ groups, preserve SMC metadata under its transaction lock, and implement firmware-owned nested IOMFB streams, display identity, shared external-pipeline accounting and boolean metadata callbacks. These unblock HDMI discovery and reconnects on J714s. Source sanitizers and ARM64 QEMU gates passed. Hardware reached user-confirmed HDMI4K120 with DSC alongside internal120Hz, both2x; repeated HPD reconnects completed and a five-second DPMS cycle restored both outputs. Final metadata cleanup lifetime correction is sanitizer/QEMU tested, not loaded in that hardware run. 4K240 remains firmware-filtered and system sleep is unqualified. --- .../bindings/gpio/apple,smc-gpio.yaml | 25 ++++ .../bindings/pinctrl/apple,pinctrl.yaml | 10 ++ Documentation/gpu/apple-dcp-t6050.rst | 112 +++++++++++++++++ drivers/gpio/gpio-macsmc.c | 75 ++++++++++++ drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 7 ++ drivers/gpu/drm/apple/dcp.h | 1 + .../apple/iomfb-boolean-properties-v27.inc | 47 ++++++++ .../drm/apple/iomfb-display-identity-v27.inc | 86 +++++++++++++ .../drm/apple/iomfb-external-counts-v27.inc | 59 +++++++++ drivers/gpu/drm/apple/iomfb.c | 51 +++++++- drivers/gpu/drm/apple/iomfb.h | 6 + drivers/gpu/drm/apple/iomfb_internal.h | 4 + drivers/gpu/drm/apple/iomfb_v27.c | 17 +++ drivers/mfd/macsmc.c | 20 ++- drivers/pinctrl/pinctrl-apple-gpio.c | 34 ++++-- .../drivers/gpio/apple-hdmi-controls.py | 107 ++++++++++++++++ .../gpu/apple-dcp-boolean-properties-v27.py | 57 +++++++++ .../gpu/apple-dcp-display-identity-v27.py | 72 +++++++++++ .../gpu/apple-dcp-external-counts-v27.py | 77 ++++++++++++ .../drivers/gpu/apple-dcp-nested-stream.py | 114 ++++++++++++++++++ 21 files changed, 967 insertions(+), 15 deletions(-) create mode 100644 drivers/gpu/drm/apple/iomfb-boolean-properties-v27.inc create mode 100644 drivers/gpu/drm/apple/iomfb-display-identity-v27.inc create mode 100644 drivers/gpu/drm/apple/iomfb-external-counts-v27.inc create mode 100644 tools/testing/selftests/drivers/gpio/apple-hdmi-controls.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp-boolean-properties-v27.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp-display-identity-v27.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp-external-counts-v27.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp-nested-stream.py diff --git a/Documentation/devicetree/bindings/gpio/apple,smc-gpio.yaml b/Documentation/devicetree/bindings/gpio/apple,smc-gpio.yaml index b4063a9dd1248..1e8e386ca3bc4 100644 --- a/Documentation/devicetree/bindings/gpio/apple,smc-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/apple,smc-gpio.yaml @@ -17,6 +17,19 @@ properties: enum: - apple,smc-gpio - apple,smc-low-gpio + - apple,smc-pcio-gpio + + apple,output-commands: + $ref: /schemas/types.yaml#/definitions/uint64-array + minItems: 1 + maxItems: 64 + description: | + Board-specific pcIO output command templates in GPIO index order. + Each template is the eight-byte argument from the native pKW8 + platform function. Its low 16 bits must be zero and are replaced + with the requested logical output value. Remaining fields are + opaque PMU command data. This interface supports outputs only; + electrical state readback is not available. gpio-controller: true @@ -28,4 +41,16 @@ required: - gpio-controller - '#gpio-cells' +allOf: + - if: + properties: + compatible: + const: apple,smc-pcio-gpio + then: + required: + - apple,output-commands + else: + properties: + apple,output-commands: false + additionalProperties: false diff --git a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml index bc6d41b63b5d1..1959386b4d24b 100644 --- a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml @@ -63,6 +63,16 @@ properties: minItems: 1 maxItems: 7 + apple,irq-group-base: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 6 + default: 0 + description: | + Hardware index of the first interrupt group listed in interrupts. + On GPIO controllers shared with a coprocessor, lower groups may be + owned by that firmware. Only contiguous host-owned groups are listed. + interrupt-controller: true '#interrupt-cells': diff --git a/Documentation/gpu/apple-dcp-t6050.rst b/Documentation/gpu/apple-dcp-t6050.rst index 142266a0a6de3..26f084de9aadc 100644 --- a/Documentation/gpu/apple-dcp-t6050.rst +++ b/Documentation/gpu/apple-dcp-t6050.rst @@ -413,3 +413,115 @@ complete ATC power gating. External sink training remains unverified. The source-including model test lives at ``tools/testing/selftests/drivers/phy/apple-t6050/``. + +Firmware27 HDMI control signals +------------------------------ + +J714s describes the HDMI bridge and connector supply using ``pcIO`` SMC +commands. The ``apple,smc-pcio-gpio`` child of the SMC exposes explicit +board output templates, rather than assuming the legacy ``gPxx`` keys. +Its GPIO index is the template's position in ``apple,output-commands``; +it is not a decoded PMU pad number. Native pKW8 copies the eight-byte +template and replaces its low16 bits with the requested value. The driver +uses that byte order and preserves all remaining template fields. It +propagates SMC failures and does not claim electrical readback support. + +The HDMI HPD signal is AOP GPIO14. Its native interrupt descriptor selects +group5, distinct from wake-only group4. The pinctrl driver's optional +``apple,irq-group-base`` permits listing only host-owned parent interrupts; +startup, acknowledgement and chained dispatch use their hardware indices. +The J714s candidate lists group5/AIC573, without registering firmware-owned +interrupt groups. The older group0-based description remains the default. + +The actual serializer and GPIO IRQ callbacks have sanitizer and ARM64 tests +in ``tools/testing/selftests/drivers/gpio/apple-hdmi-controls.py``. On J714s, +both native power commands were acknowledged and GPIO14 delivered an HPD +rising edge through group5. DCP then selected HBR3 and four lanes and +published its IOAVVideoInterface. These results qualify the control path; +they do not establish external scanout or an optical display result. + +Firmware27 display identity callback +----------------------------------- + +D133 is ``vi_is_displayX1`` (no input, four-byte output), not a generic +success acknowledgement. Native macOS asks firmware A375 for a display's +product identity, then recognizes vendor ``APP`` (or an empty vendor) and +product0xae42/0xae43 only when the query succeeds. The driver performs that +nested RPC, checks the native predicate, and acknowledges the original +callback channel after its reply. It does not substitute a constant answer +or depend on EDID having already reached DRM's later mode enumeration. + +The native A375 request is24 bytes: vendor[10], vendor-ID32, buffer length32, +product-ID32, and two null flags. The reply is20 bytes: vendor[10], vendor-ID32, +product-ID32, one success byte and padding. Unaligned fields use the packed +wire layout. The test ``apple-dcp-display-identity-v27.py`` covers these +layouts, identity matching, and deferred acknowledgement on normal and +out-of-band callback/async channels. + +Firmware-owned nested RPC streams +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An AP query made while servicing a firmware-originated asynchronous request +must reuse that stream. Native ``link_get_stream_by_thread_id_gated`` finds +the callback thread's existing stream; ``rpc_caller_gated`` then sends a +nested request without the new-call bit. This produces context1 for normal +firmware-owned traffic and context5 for its out-of-band counterpart. +The previous AP-only context chooser sent D133's A375 query as context2; +the firmware waited for the nested exchange and the new command stalled. + +Executing the captured native stream initializer confirms TX offsets +0x20000/0x28000 and RX offsets0x40000/0x48000 for these streams. Context1 +shares the context3 stack, and context5 shares the context7 stack. A375's +reply unwinds its nested frame before the original D133 acknowledgement. +``dcp_push_nested`` takes the original callback context explicitly; unrelated +AP commands do not select its stream. Existing ``dcp_push`` callers keep +their previous AP-stream selection. ``apple-dcp-nested-stream.py`` tests +actual push/ACK implementation, native offsets, deferred identity replies, +and unchanged ordinary/out-of-band AP behavior. + +The nested exchange is hardware-qualified on J714s: A375 returned Samsung +vendor ``SAM`` and product0x774c while handling a context3 D133 request. +Firmware then advanced to its external-pipeline accounting callbacks. + +External pipeline accounting +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Firmware27 D130 adds/removes external pipeline and Apple-display counts. +Its eight-byte request packs the plug bit at byte0, the Apple-display bit +at byte1, and the unaligned 32-bit pipe count at byte2. D131 returns the +three shared counters (pipes, Apple displays, suppression references); its +16-byte input includes three pointer-null flags after the initial values. +D132 increments or decrements suppression references using bit0 of its +four-byte request. These counters are shared across DCPs, as in native +UnifiedPipeline2, and updates/snapshots are protected against concurrent +RTKit workers. Internal panels do not contribute to the external pipe +count. The source test covers the captured Samsung request, multiple +controllers, balanced plug/suppression events and concurrent updates. + +On J714s these callbacks allowed firmware to export32 HDMI modes, including +3840x2160 at120 Hz. GNOME selected4K60 alongside the internal120 Hz panel; +the user saw the login background briefly. A later HPD drop exposed a +missing D569 metadata callback and prevented recovery, so this observation +does not establish sustained HDMI operation. + +D569 is the native IORegistry boolean-property setter (64-byte name, +one-byte boolean and padding; four-byte reply). It updates per-DCP host +metadata such as ``IOMFBHotplugKeysChangedKey`` during unplug. Names are +terminated at byte63 and values use bit0, matching native marshalling. +The driver stores/replaces these properties and reports allocation failure +rather than leaving the firmware waiting. No power or modeset operation is +implied by this metadata update. Its test covers replacement, controller +isolation, name truncation and failure replies. + +With D569 implemented, repeated hardware HPD drops/reconnects completed +unplug/plug and mode export without the earlier stalled command. HDMI +3840x2160@120 (DSC enabled) and internal3024x1964@120 ran together at2x +scaling; the user confirmed clean sustained external output. A five-second +DPMS cycle restored both120Hz modes and internal brightness. This does not +establish full system sleep or4K240: firmware currently filters the latter +against an820MHz video-clock limit. + +Boolean metadata has a bind-managed cleanup action registered before the +RTKit resources, so its entries are freed after the RTKit worker stops. +The cleanup path passed sanitizer and ARM64 QEMU tests; the optical/DPMS +hardware run used the preceding build with identical callback behavior. diff --git a/drivers/gpio/gpio-macsmc.c b/drivers/gpio/gpio-macsmc.c index c3ca445a85ac9..7694467fad6ea 100644 --- a/drivers/gpio/gpio-macsmc.c +++ b/drivers/gpio/gpio-macsmc.c @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include #include @@ -76,6 +78,7 @@ struct macsmc_gpio { int first_index; smc_key base_key; + u64 *output_commands; }; static int macsmc_gpio_nr(smc_key key) @@ -233,6 +236,75 @@ static int macsmc_gpio_init_valid_mask(struct gpio_chip *gc, return 0; } +/* + * New PMUs use a shared pcIO command key. Native pKW8 platform functions + * copy an eight-byte board template and replace its low 16 bits with the + * requested output value. Keep the remaining command fields opaque. + */ +static int macsmc_pcio_set(struct gpio_chip *gc, unsigned int offset, int value) +{ + struct macsmc_gpio *smcgp = gpiochip_get_data(gc); + u8 command[8]; + int ret; + + put_unaligned_le64(smcgp->output_commands[offset] | !!value, command); + ret = apple_smc_write(smcgp->smc, SMC_KEY(pcIO), command, sizeof(command)); + return ret < 0 ? ret : 0; +} + +static int macsmc_pcio_get(struct gpio_chip *gc, unsigned int offset) +{ + /* The native output command has no established input/readback operation. */ + return -EOPNOTSUPP; +} + +static int macsmc_pcio_get_direction(struct gpio_chip *gc, unsigned int offset) +{ + return GPIO_LINE_DIRECTION_OUT; +} + +static int macsmc_pcio_probe(struct macsmc_gpio *smcgp) +{ + struct device *dev = smcgp->dev; + struct apple_smc_key_info info; + int count, ret; + + ret = apple_smc_get_key_info(smcgp->smc, SMC_KEY(pcIO), &info); + if (ret < 0) + return ret; + if (info.size != 8 || !(info.flags & APPLE_SMC_WRITABLE)) + return dev_err_probe(dev, -EINVAL, "pcIO is not an eight-byte output key\n"); + count = of_property_count_u64_elems(dev->of_node, "apple,output-commands"); + if (count <= 0 || count > MAX_GPIO) + return -EINVAL; + smcgp->output_commands = devm_kcalloc(dev, count, sizeof(u64), GFP_KERNEL); + if (!smcgp->output_commands) + return -ENOMEM; + ret = of_property_read_u64_array(dev->of_node, "apple,output-commands", + smcgp->output_commands, count); + if (ret) + return ret; + for (int i = 0; i < count; i++) { + if (smcgp->output_commands[i] & 0xffff) + return -EINVAL; + for (int j = 0; j < i; j++) + if (smcgp->output_commands[i] == smcgp->output_commands[j]) + return -EINVAL; + } + + smcgp->gc.label = "macsmc-pcio-output"; + smcgp->gc.owner = THIS_MODULE; + smcgp->gc.get = macsmc_pcio_get; + smcgp->gc.set = macsmc_pcio_set; + smcgp->gc.direction_output = macsmc_pcio_set; + smcgp->gc.get_direction = macsmc_pcio_get_direction; + smcgp->gc.can_sleep = true; + smcgp->gc.ngpio = count; + smcgp->gc.base = -1; + smcgp->gc.parent = dev; + return devm_gpiochip_add_data(dev, &smcgp->gc, smcgp); +} + struct macsmc_gpio_of_match_data { smc_key base_key; }; @@ -251,6 +323,8 @@ static int macsmc_gpio_probe(struct platform_device *pdev) smcgp->dev = &pdev->dev; smcgp->smc = smc; + if (device_is_compatible(&pdev->dev, "apple,smc-pcio-gpio")) + return macsmc_pcio_probe(smcgp); smcgp->base_key = data ? data->base_key : _SMC_KEY("gP\0\0"); smcgp->first_index = macsmc_gpio_find_first_gpio_index(smcgp); @@ -292,6 +366,7 @@ static const struct macsmc_gpio_of_match_data macsmc_gpio_low_data = { }; static const struct of_device_id macsmc_gpio_of_table[] = { + { .compatible = "apple,smc-pcio-gpio" }, { .compatible = "apple,smc-gpio", .data = &macsmc_gpio_up_data, diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 46a4853c2b947..e7ed184377136 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -231,6 +231,7 @@ struct apple_dcp { /* Ignore inherited application-endpoint traffic while quiescing RTKit. */ bool adopting_session; struct apple_dcp_surface_order iomfb_surfaces; + struct list_head iomfb_boolean_properties; /************* IOMFB ************************************************** * everything below is mostly used inside IOMFB but it could make * diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index ccfe26d86bba7..de16f89cc651d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2659,6 +2659,13 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) bool loader_quiesced; int ret; + INIT_LIST_HEAD(&dcp->iomfb_boolean_properties); + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) { + ret = devm_add_action_or_reset(dev, iomfb_boolean_properties_cleanup_v27, dcp); + if (ret) + return ret; + } + /* * The coprocessor scans out through the display-subsystem device's * IOMMU stream, not through piodma's, so buffers it asks us to map for diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 4b06cdbb89c1a..1d16cb8fa32a9 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -81,6 +81,7 @@ int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active); int iomfb_start_rtkit(struct apple_dcp *dcp); void iomfb_shutdown(struct apple_dcp *dcp); +void iomfb_boolean_properties_cleanup_v27(void *data); /* rtkit message handler for IOMFB messages */ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message); diff --git a/drivers/gpu/drm/apple/iomfb-boolean-properties-v27.inc b/drivers/gpu/drm/apple/iomfb-boolean-properties-v27.inc new file mode 100644 index 0000000000000..79e47efcd418e --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-boolean-properties-v27.inc @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* D569: IORegistryEntry::setProperty(char const *, bool) host metadata. */ +struct dcp_boolean_property_v27 { + struct list_head node; + char name[64]; + bool value; +}; + +static bool trampoline_set_boolean_property_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + struct dcp_boolean_property_v27 *property; + char name[64]; + + trace_iomfb_callback(dcp, tag, __func__); + /* Native marshalling terminates the fixed-width name at byte63. */ + memcpy(name, request, sizeof(name)); + name[sizeof(name) - 1] = '\0'; + put_unaligned_le32(0, out); + /* All IOMFB callbacks for this DCP run on its single RTKit worker. */ + list_for_each_entry(property, &dcp->iomfb_boolean_properties, node) + if (!strcmp(property->name, name)) + goto update; + property = kzalloc(sizeof(*property), GFP_KERNEL); + if (!property) + return true; + memcpy(property->name, name, sizeof(name)); + list_add_tail(&property->node, &dcp->iomfb_boolean_properties); +update: + property->value = request[64] & 1; + dev_dbg(dcp->dev, "boolean property %s=%u\n", name, property->value); + put_unaligned_le32(1, out); + return true; +} + +/* Registered before RTKit resources: teardown stops its worker first. */ +void iomfb_boolean_properties_cleanup_v27(void *data) +{ + struct apple_dcp *dcp = data; + struct dcp_boolean_property_v27 *property, *next; + + list_for_each_entry_safe(property, next, &dcp->iomfb_boolean_properties, node) { + list_del(&property->node); + kfree(property); + } +} diff --git a/drivers/gpu/drm/apple/iomfb-display-identity-v27.inc b/drivers/gpu/drm/apple/iomfb-display-identity-v27.inc new file mode 100644 index 0000000000000..97952ce26dad4 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-display-identity-v27.inc @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Native D133 vi_is_displayX1 queries A375 getDisplayProductId. */ +struct dcp_display_product_request_v27 { + u8 vendor[10]; + __le32 vendor_id; + __le32 vendor_length; + __le32 product_id; + u8 vendor_id_null, product_id_null; +} __packed; + +struct dcp_display_product_reply_v27 { + u8 vendor[10]; + __le32 vendor_id; + __le32 product_id; + u8 success, padding; +} __packed; + +static_assert(sizeof(struct dcp_display_product_request_v27) == 24); +static_assert(sizeof(struct dcp_display_product_reply_v27) == 20); +static_assert(offsetof(struct dcp_display_product_reply_v27, product_id) == 14); + +struct dcp_display_identity_callback_v27 { + void *output; + enum dcp_context_id context; +}; + +static void display_identity_done_v27(struct apple_dcp *dcp, void *out, void *cookie) +{ + const struct dcp_display_product_reply_v27 *reply = out; + struct dcp_display_identity_callback_v27 *cb = cookie; + bool vendor_matches = !reply->vendor[0] || + (!memcmp(reply->vendor, "APP", 3) && !reply->vendor[3]); + bool display_x1 = (reply->success & 1) && vendor_matches && + (le32_to_cpu(reply->product_id) >> 1) == 0x5721; + + dev_info(dcp->dev, "display identity: vendor=%.10s product=%#x valid=%u X1=%u context=%u\n", + reply->vendor, le32_to_cpu(reply->product_id), reply->success & 1, + display_x1, cb->context); + put_unaligned_le32(display_x1, cb->output); + dcp_ack(dcp, cb->context); + kfree(cb); +} + +static bool trampoline_display_identity_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + static const struct dcp_method_entry method = { + .name = "getDisplayProductId", .tag = {'A', '3', '7', '5'}, + }; + static const enum dcp_context_id contexts[] = { + DCP_CONTEXT_CB, DCP_CONTEXT_ASYNC, DCP_CONTEXT_OOBCB, DCP_CONTEXT_OOBASYNC, + }; + struct dcp_channel *channels[] = { + &dcp->ch_cb, &dcp->ch_async, &dcp->ch_oobcb, &dcp->ch_oobasync, + }; + struct dcp_display_product_request_v27 request = { + .vendor_length = cpu_to_le32(10), + }; + struct dcp_display_identity_callback_v27 *cb; + int context = -1; + + trace_iomfb_callback(dcp, tag, __func__); + /* Preserve the incoming channel across the asynchronous nested RPC. */ + for (unsigned int i = 0; i < ARRAY_SIZE(channels); i++) { + struct dcp_channel *ch = channels[i]; + + if (ch->depth && ch->output[ch->depth - 1] == out) { + context = contexts[i]; + break; + } + } + if (WARN_ON(context < 0)) + return true; + cb = kzalloc(sizeof(*cb), GFP_KERNEL); + if (!cb) + return true; + cb->output = out; + cb->context = context; + if (dcp_push_nested(dcp, context, &method, sizeof(request), + sizeof(struct dcp_display_product_reply_v27), &request, + display_identity_done_v27, cb)) { + kfree(cb); + return true; + } + return false; +} diff --git a/drivers/gpu/drm/apple/iomfb-external-counts-v27.inc b/drivers/gpu/drm/apple/iomfb-external-counts-v27.inc new file mode 100644 index 0000000000000..52bb0c4b61ea4 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-external-counts-v27.inc @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Native UnifiedPipeline2 shares these three counters between all pipes. */ +static DEFINE_SPINLOCK(external_counts_lock_v27); +static u32 external_pipe_count_v27; +static u32 external_apple_display_count_v27; +static u32 external_suppression_count_v27; + +static bool trampoline_update_external_counts_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + bool plug = request[0] & 1; + u32 apple_display = request[1] & 1; + u32 pipes = get_unaligned_le32(request + 2); + unsigned long flags; + + trace_iomfb_callback(dcp, tag, __func__); + /* Native UPPipe2::is_disp_external gates only the pipe-count update. */ + if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) + return true; + spin_lock_irqsave(&external_counts_lock_v27, flags); + external_pipe_count_v27 += plug ? pipes : -pipes; + external_apple_display_count_v27 += plug ? apple_display : -apple_display; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + return true; +} + +static bool trampoline_get_external_counts_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + unsigned long flags; + u32 counts[3]; + + trace_iomfb_callback(dcp, tag, __func__); + spin_lock_irqsave(&external_counts_lock_v27, flags); + counts[0] = external_pipe_count_v27; + counts[1] = external_apple_display_count_v27; + counts[2] = external_suppression_count_v27; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + /* Three u32 in/out values followed by three native pointer-null flags. */ + for (unsigned int i = 0; i < ARRAY_SIZE(counts); i++) + if (!(request[12 + i] & 1)) + put_unaligned_le32(counts[i], (u8 *)out + 4 * i); + return true; +} + +static bool trampoline_apply_suppression_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + unsigned long flags; + + trace_iomfb_callback(dcp, tag, __func__); + spin_lock_irqsave(&external_counts_lock_v27, flags); + external_suppression_count_v27 += (request[0] & 1) ? 1U : -1U; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + return true; +} diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index dcc4d15fe9b9f..4390ccefddf40 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -129,6 +129,10 @@ static int dcp_tx_offset(enum dcp_context_id id) case DCP_CONTEXT_OOBCB: case DCP_CONTEXT_OOBCMD: return 0x08000; + case DCP_CONTEXT_ASYNC_CB: + return 0x20000; + case DCP_CONTEXT_OOBASYNC_CB: + return 0x28000; default: return -EINVAL; } @@ -137,8 +141,10 @@ static int dcp_tx_offset(enum dcp_context_id id) static int dcp_channel_offset(enum dcp_context_id id) { switch (id) { + case DCP_CONTEXT_ASYNC_CB: case DCP_CONTEXT_ASYNC: return 0x40000; + case DCP_CONTEXT_OOBASYNC_CB: case DCP_CONTEXT_OOBASYNC: return 0x48000; case DCP_CONTEXT_CB: @@ -214,8 +220,10 @@ static struct dcp_channel *dcp_get_channel(struct apple_dcp *dcp, return &dcp->ch_oobcb; case DCP_CONTEXT_OOBCMD: return &dcp->ch_oobcmd; + case DCP_CONTEXT_ASYNC_CB: case DCP_CONTEXT_ASYNC: return &dcp->ch_async; + case DCP_CONTEXT_OOBASYNC_CB: case DCP_CONTEXT_OOBASYNC: return &dcp->ch_oobasync; default: @@ -249,9 +257,10 @@ static u8 dcp_pop_depth(u8 *depth) } /* Call a DCP function given by a tag */ -void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, - u32 in_len, u32 out_len, void *data, dcp_callback_t cb, - void *cookie) +static void dcp_push_context(struct apple_dcp *dcp, enum dcp_context_id context, + const struct dcp_method_entry *call, + u32 in_len, u32 out_len, void *data, + dcp_callback_t cb, void *cookie) { struct dcp_method_entry resolved = *call; @@ -260,7 +269,6 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca memcpy(resolved.tag, call->tag_h17g, sizeof(resolved.tag)); call = &resolved; - enum dcp_context_id context = dcp_call_context(dcp, oob); struct dcp_channel *ch = dcp_get_channel(dcp, context); struct dcp_packet_header header = { @@ -314,6 +322,41 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca dcpep_msg(context, data_len, offset)); } +/* Automatic selection for an AP-initiated RPC stream. */ +void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, + u32 in_len, u32 out_len, void *data, dcp_callback_t cb, void *cookie) +{ + dcp_push_context(dcp, dcp_call_context(dcp, oob), call, + in_len, out_len, data, cb, cookie); +} + +/* Native rpc_caller reuses the callback thread's stream, including its + * firmware-owned half. A new context2 command can deadlock a firmware + * callback waiting for the AP to finish a nested query on context1. */ +int dcp_push_nested(struct apple_dcp *dcp, enum dcp_context_id origin, + const struct dcp_method_entry *call, u32 in_len, u32 out_len, + void *data, dcp_callback_t cb, void *cookie) +{ + enum dcp_context_id context; + + switch (origin) { + case DCP_CONTEXT_CB: + case DCP_CONTEXT_OOBCB: + context = origin; + break; + case DCP_CONTEXT_ASYNC: + context = DCP_CONTEXT_ASYNC_CB; + break; + case DCP_CONTEXT_OOBASYNC: + context = DCP_CONTEXT_OOBASYNC_CB; + break; + default: + return -EINVAL; + } + dcp_push_context(dcp, context, call, in_len, out_len, data, cb, cookie); + return 0; +} + /* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ int dcp_parse_tag(char tag[4]) { diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 9fcf06bc787e4..1cbf9493ea8c3 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -21,6 +21,9 @@ enum dcp_context_id { /* Callback */ DCP_CONTEXT_CB = 0, + /* Nested RPC on a firmware-initiated stream */ + DCP_CONTEXT_ASYNC_CB = 1, + /* Command */ DCP_CONTEXT_CMD = 2, @@ -30,6 +33,9 @@ enum dcp_context_id { /* Out-of-band callback */ DCP_CONTEXT_OOBCB = 4, + /* Nested RPC on an out-of-band firmware-initiated stream */ + DCP_CONTEXT_OOBASYNC_CB = 5, + /* Out-of-band command */ DCP_CONTEXT_OOBCMD = 6, diff --git a/drivers/gpu/drm/apple/iomfb_internal.h b/drivers/gpu/drm/apple/iomfb_internal.h index 4706b68db34d2..72e74e56b59ec 100644 --- a/drivers/gpu/drm/apple/iomfb_internal.h +++ b/drivers/gpu/drm/apple/iomfb_internal.h @@ -110,6 +110,10 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca u32 in_len, u32 out_len, void *data, dcp_callback_t cb, void *cookie); +int dcp_push_nested(struct apple_dcp *dcp, enum dcp_context_id origin, + const struct dcp_method_entry *call, u32 in_len, u32 out_len, + void *data, dcp_callback_t cb, void *cookie); + /* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ int dcp_parse_tag(char tag[4]); diff --git a/drivers/gpu/drm/apple/iomfb_v27.c b/drivers/gpu/drm/apple/iomfb_v27.c index e5b940d89a398..2a83501af6d4f 100644 --- a/drivers/gpu/drm/apple/iomfb_v27.c +++ b/drivers/gpu/drm/apple/iomfb_v27.c @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright The Asahi Linux Contributors */ +#include +#include + #include "dcp-link.h" #include "iomfb_v27.h" #include "iomfb_v12_3.h" @@ -345,6 +348,10 @@ static bool trampoline_tiling_state_v27(struct apple_dcp *dcp, int tag, return true; } +#include "iomfb-display-identity-v27.inc" +#include "iomfb-external-counts-v27.inc" +#include "iomfb-boolean-properties-v27.inc" + static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [0] = dcpep_cb_d000_h17p, /* native did_boot callback */ [1] = trampoline_true, @@ -377,6 +384,10 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [127] = trampoline_prop_chunk, /* setDCPAVPropChunk */ [128] = trampoline_prop_end, /* setDCPAVPropEnd */ [129] = trampoline_allocate_bandwidth, /* allocate_bandwidth */ + [130] = trampoline_update_external_counts_v27, + [131] = trampoline_get_external_counts_v27, + [132] = trampoline_apply_suppression_v27, + [133] = trampoline_display_identity_v27, /* vi_is_displayX1 */ [201] = trampoline_map_piodma, /* map_buf */ [202] = trampoline_unmap_piodma, /* unmap_buf */ [206] = iomfbep_cb_match_pmu_service_2, /* match_pmu_service */ @@ -403,6 +414,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [563] = trampoline_true, [565] = trampoline_true, [567] = trampoline_true, + [569] = trampoline_set_boolean_property_v27, [572] = trampoline_zero, /* powerUpDART */ [574] = iomfbep_cb_video_range_h17p, [575] = trampoline_hotplug_h17p, @@ -463,6 +475,10 @@ static const struct { [127] = { 0x1008, 0x4 }, [128] = { 0x40, 0x4 }, [129] = { 0x1c, 0x14 }, + [130] = { 0x8, 0x0 }, + [131] = { 0x10, 0xc }, + [132] = { 0x4, 0x0 }, + [133] = { 0x0, 0x4 }, [201] = { 0xc, 0x10 }, [202] = { 0x18, 0x0 }, [206] = { 0x0, 0x4 }, @@ -488,6 +504,7 @@ static const struct { [563] = { 0x4c, 0x4 }, [565] = { 0x48, 0x4 }, [567] = { 0x80, 0x4 }, + [569] = { 0x44, 0x4 }, [572] = { 0x4, 0x4 }, [574] = { 0x4, 0x8 }, [575] = { 0x58, 0x4c }, diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index d51e193dd6fde..a4ffdc5a21b2b 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -48,6 +48,7 @@ static const struct mfd_cell apple_smc_devs[] = { MFD_CELL_NAME("macsmc-input"), MFD_CELL_NAME("macsmc-power"), MFD_CELL_OF("macsmc-gpio", NULL, NULL, 0, 0, "apple,smc-gpio"), + MFD_CELL_OF("macsmc-pcio-gpio", NULL, NULL, 0, 0, "apple,smc-pcio-gpio"), MFD_CELL_OF("macsmc-low-gpio", NULL, NULL, 0, 0, "apple,smc-low-gpio"), MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), MFD_CELL_OF("macsmc-reboot", NULL, NULL, 0, 0, "apple,smc-reboot"), @@ -59,6 +60,12 @@ static const struct mfd_cell apple_smc_telemetry_devs[] = { MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), }; +/* T6050 additionally describes native pcIO outputs explicitly in its DT. */ +static const struct mfd_cell apple_smc_t6050_devs[] = { + MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), + MFD_CELL_OF("macsmc-pcio-gpio", NULL, NULL, 0, 0, "apple,smc-pcio-gpio"), +}; + static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, u64 size, u64 wsize, u32 *ret_data) { @@ -216,7 +223,9 @@ int apple_smc_get_key_info(struct apple_smc *smc, smc_key key, struct apple_smc_ u8 key_info[6]; int ret; - ret = apple_smc_cmd(smc, SMC_MSG_GET_KEY_INFO, key, 0, 0, NULL); + /* Key metadata occupies the same shared buffer as other SMC replies. */ + guard(mutex)(&smc->mutex); + ret = apple_smc_cmd_locked(smc, SMC_MSG_GET_KEY_INFO, key, 0, 0, NULL); if (ret >= 0 && info) { memcpy_fromio(key_info, smc->shmem.iomem, sizeof(key_info)); info->size = key_info[0]; @@ -527,8 +536,13 @@ static int apple_smc_probe(struct platform_device *pdev) /* T8152's key-read interface is qualified; notifications, GPIO and power * functions are not. The board DT exposes temperature and fan inputs only. */ - if (of_device_is_compatible(dev->of_node, "apple,t8152-smc") || - of_device_is_compatible(dev->of_node, "apple,t6050-smc")) + if (of_device_is_compatible(dev->of_node, "apple,t6050-smc")) + return devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, + apple_smc_t6050_devs, + ARRAY_SIZE(apple_smc_t6050_devs), + NULL, 0, NULL); + + if (of_device_is_compatible(dev->of_node, "apple,t8152-smc")) return devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, apple_smc_telemetry_devs, ARRAY_SIZE(apple_smc_telemetry_devs), diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c index 6305b96aaa733..c0350ae18af66 100644 --- a/drivers/pinctrl/pinctrl-apple-gpio.c +++ b/drivers/pinctrl/pinctrl-apple-gpio.c @@ -30,6 +30,13 @@ #include "core.h" #include "pinmux.h" +struct apple_gpio_pinctrl; + +struct apple_gpio_irq_group { + struct apple_gpio_pinctrl *pctl; + u8 index; +}; + struct apple_gpio_pinctrl { struct device *dev; struct pinctrl_dev *pctldev; @@ -39,7 +46,8 @@ struct apple_gpio_pinctrl { struct pinctrl_desc pinctrl_desc; struct gpio_chip gpio_chip; - u8 irqgrps[]; + u8 irq_group_base; + struct apple_gpio_irq_group irqgrps[]; }; #define REG_GPIO(x) (4 * (x)) @@ -313,7 +321,7 @@ static unsigned int apple_gpio_irq_startup(struct irq_data *data) struct apple_gpio_pinctrl *pctl = gpiochip_get_data(chip); apple_gpio_set_reg(pctl, data->hwirq, REG_GPIOx_GRP, - FIELD_PREP(REG_GPIOx_GRP, 0)); + FIELD_PREP(REG_GPIOx_GRP, pctl->irq_group_base)); apple_gpio_direction_input(chip, data->hwirq); apple_gpio_irq_unmask(data); @@ -342,18 +350,17 @@ static int apple_gpio_irq_set_type(struct irq_data *data, unsigned int type) static void apple_gpio_irq_handler(struct irq_desc *desc) { struct irq_chip *chip = irq_desc_get_chip(desc); - u8 *grpp = irq_desc_get_handler_data(desc); - struct apple_gpio_pinctrl *pctl; + struct apple_gpio_irq_group *group = irq_desc_get_handler_data(desc); + struct apple_gpio_pinctrl *pctl = group->pctl; unsigned int pinh, pinl; unsigned long pending; struct gpio_chip *gc; - pctl = container_of(grpp - *grpp, typeof(*pctl), irqgrps[0]); gc = &pctl->gpio_chip; chained_irq_enter(chip, desc); for (pinh = 0; pinh < gc->ngpio; pinh += 32) { - pending = readl_relaxed(pctl->base + REG_IRQ(*grpp, pinh)); + pending = readl_relaxed(pctl->base + REG_IRQ(group->index, pinh)); for_each_set_bit(pinl, &pending, 32) generic_handle_domain_irq(gc->irq.domain, pinh + pinl); } @@ -412,7 +419,8 @@ static int apple_gpio_register(struct apple_gpio_pinctrl *pctl) goto out_free_irq_data; girq->parents[i] = ret; - pctl->irqgrps[i] = i; + pctl->irqgrps[i].pctl = pctl; + pctl->irqgrps[i].index = pctl->irq_group_base + i; irq_data[i] = &pctl->irqgrps[i]; } @@ -441,7 +449,7 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) static const char *pinmux_functions[] = { "gpio", "periph1", "periph2", "periph3" }; - unsigned int i, nirqs = 0; + unsigned int i, nirqs = 0, irq_group_base = 0; int res; if (of_property_read_bool(pdev->dev.of_node, "interrupt-controller")) { @@ -450,11 +458,21 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) nirqs = res; } + if (of_property_present(pdev->dev.of_node, "apple,irq-group-base")) { + res = of_property_read_u32(pdev->dev.of_node, "apple,irq-group-base", + &irq_group_base); + if (res) + return res; + } + if (irq_group_base > 6 || nirqs > 7 - irq_group_base) + return dev_err_probe(&pdev->dev, -EINVAL, "invalid host IRQ groups\n"); + pctl = devm_kzalloc(&pdev->dev, struct_size(pctl, irqgrps, nirqs), GFP_KERNEL); if (!pctl) return -ENOMEM; pctl->dev = &pdev->dev; + pctl->irq_group_base = irq_group_base; pctl->gpio_chip.irq.num_parents = nirqs; dev_set_drvdata(&pdev->dev, pctl); diff --git a/tools/testing/selftests/drivers/gpio/apple-hdmi-controls.py b/tools/testing/selftests/drivers/gpio/apple-hdmi-controls.py new file mode 100644 index 0000000000000..9f9c7c46079ee --- /dev/null +++ b/tools/testing/selftests/drivers/gpio/apple-hdmi-controls.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Actual pcIO serializer and GPIO IRQ group routing, with native byte fixtures.""" +from pathlib import Path +import os,subprocess,tempfile +root=next(p for p in Path(__file__).resolve().parents if (p/'drivers/gpio/gpio-macsmc.c').exists()) +gpio=(root/'drivers/gpio/gpio-macsmc.c').read_text();irq=(root/'drivers/pinctrl/pinctrl-apple-gpio.c').read_text() +smc=(root/'drivers/mfd/macsmc.c').read_text() +a=smc.index('int apple_smc_get_key_info(');metadata=smc[a:smc.index('\n}',a)+2] +def func(s,name): + a=s.index(name+'(');a=s.rfind('\nstatic ',0,a)+1;b=s.index('\n}',a)+2;return s[a:b] +pre=r''' +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8;typedef uint32_t u32;typedef uint64_t u64; +#define SMC_KEY(x) 0x7063494f +#define GPIO_LINE_DIRECTION_OUT 0 +#define BIT(n) (1UL<<(n)) +#define REG_GPIOx_GRP 0x70000 +#define FIELD_PREP(mask,v) ((v)<<__builtin_ctz(mask)) +#define FIELD_GET(mask,v) (((v)&(mask))>>__builtin_ctz(mask)) +#define REG_IRQ(g,x) (0x800+0x40*(g)+4*((x)>>5)) +struct gpio_chip{void*data;unsigned ngpio;struct{void*domain;}irq;}; +static void*gpiochip_get_data(struct gpio_chip*g){return g->data;} +typedef u32 smc_key; +#define SMC_MSG_GET_KEY_INFO 0x13 +struct apple_smc{bool mutex;struct{void*iomem;}shmem;}; +struct apple_smc_key_info{u32 type_code;u8 size,flags;}; +struct mock_guard{bool*held;}; +static struct mock_guard lock(bool*p){assert(!*p);*p=true;return (struct mock_guard){p};} +static void unlock(struct mock_guard*g){assert(*g->held);*g->held=false;} +#define guard(type) struct mock_guard __attribute__((cleanup(unlock))) g=lock +static struct apple_smc*metadata_smc;static int metadata_result; +static int apple_smc_cmd_locked(struct apple_smc*s,unsigned cmd,smc_key key,unsigned a,unsigned b,void*p){assert(s->mutex&&cmd==0x13);return metadata_result;} +static void memcpy_fromio(void*d,void*s,size_t n){assert(metadata_smc->mutex);memcpy(d,s,n);} +static u32 get_unaligned_be32(void*p){u8*b=p;return (u32)b[0]<<24|(u32)b[1]<<16|(u32)b[2]<<8|b[3];} +struct macsmc_gpio{struct apple_smc*smc;u64*output_commands;}; +static u8 sent[8];static int smc_result; +static void put_unaligned_le64(u64 v,void*p){for(int i=0;i<8;i++)((u8*)p)[i]=v>>(8*i);} +static int apple_smc_write(struct apple_smc*s,u32 key,const void*p,size_t n){assert(key==0x7063494f&&n==8);memcpy(sent,p,8);return smc_result;} +struct apple_gpio_pinctrl{void*base;u8 irq_group_base;struct gpio_chip gpio_chip;u32 pins[85];}; +struct apple_gpio_irq_group{struct apple_gpio_pinctrl*pctl;u8 index;}; +struct irq_data{struct gpio_chip*chip;unsigned hwirq;}; +struct irq_chip{}; +struct irq_desc{void*data;struct irq_chip chip;}; +static void*irq_data_get_irq_chip_data(struct irq_data*d){return d->chip;} +static void*irq_desc_get_handler_data(struct irq_desc*d){return d->data;} +static struct irq_chip*irq_desc_get_chip(struct irq_desc*d){return &d->chip;} +static void apple_gpio_set_reg(struct apple_gpio_pinctrl*p,unsigned pin,u32 mask,u32 v){p->pins[pin]=(p->pins[pin]&~mask)|v;} +static u32 apple_gpio_get_reg(struct apple_gpio_pinctrl*p,unsigned pin){return p->pins[pin];} +static int inputs,unmasks,enters,exits; +static void apple_gpio_direction_input(struct gpio_chip*g,unsigned pin){inputs++;} +static void apple_gpio_irq_unmask(struct irq_data*d){unmasks++;} +static void chained_irq_enter(struct irq_chip*c,struct irq_desc*d){enters++;} +static void chained_irq_exit(struct irq_chip*c,struct irq_desc*d){exits++;} +static u32 readl_relaxed(void*p){return *(u32*)p;} +static void*ack_address;static u32 ack_value; +static void writel(u32 v,void*p){ack_address=p;ack_value=v;} +static unsigned handled[3],nhandled; +static void generic_handle_domain_irq(void*d,unsigned pin){assert(nhandled<3);handled[nhandled++]=pin;} +#define for_each_set_bit(bit,p,n) for((bit)=0;(bit)<(n);(bit)++)if(*(p)&BIT(bit)) +''' +source=pre+metadata+func(gpio,'macsmc_pcio_set')+func(gpio,'macsmc_pcio_get')+func(gpio,'macsmc_pcio_get_direction')+func(irq,'apple_gpio_irq_startup')+func(irq,'apple_gpio_irq_ack')+func(irq,'apple_gpio_irq_handler')+r''' +int main(void){ +u64 commands[]={0x001a000000000000ULL,0x000f000000800000ULL};struct apple_smc smc; +struct macsmc_gpio m={&smc,commands};struct gpio_chip gc={.data=&m}; +/* Fixtures follow the captured native pKW8 copy + low16 replacement. */ +const u8 expected[2][8]={{1,0,0,0,0,0,0x1a,0},{1,0,0x80,0,0,0,0x0f,0}}; +for(int i=0;i<2;i++){ + smc_result=0;assert(!macsmc_pcio_set(&gc,i,1)&&!memcmp(sent,expected[i],8)); + assert(!macsmc_pcio_set(&gc,i,0)&&!sent[0]&&!sent[1]&&!memcmp(sent+2,expected[i]+2,6)); + smc_result=8;assert(!macsmc_pcio_set(&gc,i,1)); + smc_result=-EIO;assert(macsmc_pcio_set(&gc,i,1)==-EIO); +} +assert(macsmc_pcio_get(&gc,0)==-EOPNOTSUPP&&macsmc_pcio_get_direction(&gc,0)==GPIO_LINE_DIRECTION_OUT); +for(int g=0;g<=6;g++){ + u32 mmio[0x1000/4]={0};struct apple_gpio_pinctrl p={.base=mmio,.irq_group_base=g}; + p.gpio_chip.data=&p;p.gpio_chip.ngpio=85; + struct irq_data d={&p.gpio_chip,14};p.pins[14]=0xa5a5a5a5; + assert(!apple_gpio_irq_startup(&d)&&FIELD_GET(REG_GPIOx_GRP,p.pins[14])==(unsigned)g); + assert((p.pins[14]&~REG_GPIOx_GRP)==(0xa5a5a5a5&~REG_GPIOx_GRP)); + apple_gpio_irq_ack(&d);assert(ack_address==(void*)mmio+REG_IRQ(g,14)&&ack_value==BIT(14)); + mmio[REG_IRQ(g,14)/4]=BIT(14);mmio[REG_IRQ(g,50)/4]=BIT(18);mmio[REG_IRQ(g,84)/4]=BIT(20); + struct apple_gpio_irq_group group={&p,g};struct irq_desc desc={.data=&group};nhandled=0; + apple_gpio_irq_handler(&desc);assert(nhandled==3&&handled[0]==14&&handled[1]==50&&handled[2]==84); +} +assert(inputs==7&&unmasks==7&&enters==7&&exits==7); +u8 info_bytes[]={8,'u','i','6','4',0x40};struct apple_smc_key_info info; +smc.mutex=false;smc.shmem.iomem=info_bytes;metadata_smc=&smc;metadata_result=6; +assert(apple_smc_get_key_info(&smc,0x7063494f,&info)==6&&!smc.mutex); +assert(info.size==8&&info.flags==0x40&&info.type_code==0x75693634); +metadata_result=-EIO;info.size=17;assert(apple_smc_get_key_info(&smc,0x7063494f,&info)==-EIO&&!smc.mutex&&info.size==17); +metadata_result=0;assert(!apple_smc_get_key_info(&smc,0x7063494f,NULL)&&!smc.mutex); +puts("APPLE_HDMI_CONTROLS_TEST_PASS: native pcIO bytes, errors, IRQ groups0-6, acknowledgement routing, locked SMC metadata copy"); +return 0; +} +''' +if os.environ.get('TEST_SOURCE'):Path(os.environ['TEST_SOURCE']).write_text(source) +with tempfile.TemporaryDirectory() as d: + p=Path(d);(p/'test.c').write_text(source) + subprocess.run(['clang','-std=gnu11','-O1','-g','-fsanitize=address,undefined',str(p/'test.c'),'-o',str(p/'test')],check=True) + subprocess.run([str(p/'test')],check=True) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp-boolean-properties-v27.py b/tools/testing/selftests/drivers/gpu/apple-dcp-boolean-properties-v27.py new file mode 100644 index 0000000000000..921d035bf43a4 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp-boolean-properties-v27.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""D569 native fixed-width names, boolean values, per-DCP metadata and failures.""" +from pathlib import Path +import os, subprocess, tempfile +root=next(p for p in Path(__file__).resolve().parents if (p/'drivers/gpu/drm/apple/iomfb_v27.c').exists()) +source=(root/'drivers/gpu/drm/apple/iomfb-boolean-properties-v27.inc').read_text() +parent=(root/'drivers/gpu/drm/apple/iomfb_v27.c').read_text() +assert '[569] = trampoline_set_boolean_property_v27' in parent and '[569] = { 0x44, 0x4 }' in parent +assert 'INIT_LIST_HEAD(&dcp->iomfb_boolean_properties)' in (root/'drivers/gpu/drm/apple/dcp.c').read_text() +pre=r''' +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8;typedef uint32_t u32; +#define GFP_KERNEL 0 +#define trace_iomfb_callback(...) ((void)0) +#define dev_dbg(...) ((void)0) +struct list_head {struct list_head *next,*prev;}; +#define INIT_LIST_HEAD(p) ((p)->next=(p),(p)->prev=(p)) +#define container_of(p,t,m) ((t*)((char*)(p)-offsetof(t,m))) +#define list_for_each_entry(p,h,m) for(struct list_head*iter=(h)->next;iter!=(h)&&((p)=container_of(iter,typeof(*(p)),m),1);iter=iter->next) +static void list_add_tail(struct list_head*n,struct list_head*h){n->next=h;n->prev=h->prev;h->prev->next=n;h->prev=n;} +#define list_for_each_entry_safe(p,n,h,m) for(struct list_head*iter=(h)->next,*after=iter->next;iter!=(h)&&((p)=container_of(iter,typeof(*(p)),m),(n)=after==(h)?NULL:container_of(after,typeof(*(p)),m),1);iter=after,after=iter->next) +static void list_del(struct list_head*n){n->next->prev=n->prev;n->prev->next=n->next;} +#define kfree(p) free(p) +struct apple_dcp{void*dev;struct list_head iomfb_boolean_properties;}; +static unsigned allocations;static bool fail; +static void*kzalloc(size_t n,int f){if(fail)return NULL;allocations++;return calloc(1,n);} +static void put_unaligned_le32(u32 v,void*p){u8*b=p;for(int i=0;i<4;i++)b[i]=v>>(8*i);} +''' +post=r''' +static void set(struct apple_dcp*d,const void*in,unsigned success){u8 out[6]={0xa5,0xa5,0xa5,0xa5,0xa5,0xa5};assert(trampoline_set_boolean_property_v27(d,569,out+1,(void*)in));assert(out[0]==0xa5&&out[1]==success&&!out[2]&&!out[3]&&!out[4]&&out[5]==0xa5);} +static struct dcp_boolean_property_v27*first(struct apple_dcp*d){return container_of(d->iomfb_boolean_properties.next,struct dcp_boolean_property_v27,node);} + +int main(void){ + struct apple_dcp a={0},b={0};INIT_LIST_HEAD(&a.iomfb_boolean_properties);INIT_LIST_HEAD(&b.iomfb_boolean_properties); + u8 request[68]={0};memcpy(request,"IOMFBHotplugKeysChangedKey",26);request[64]=3; + set(&a,request,1);assert(first(&a)->value&&allocations==1);assert(!strcmp(first(&a)->name,"IOMFBHotplugKeysChangedKey")); + request[64]=2;set(&a,request,1);assert(!first(&a)->value&&allocations==1); + request[64]=1;set(&b,request,1);assert(first(&b)->value&&!first(&a)->value&&allocations==2); + memset(request,'x',64);request[64]=1;set(&a,request,1);assert(allocations==3);struct dcp_boolean_property_v27*p=container_of(a.iomfb_boolean_properties.prev,struct dcp_boolean_property_v27,node);assert(strlen(p->name)==63&&p->value); + request[63]='y';request[64]=0;set(&a,request,1);assert(allocations==3&&!p->value); + request[0]='z';fail=true;set(&a,request,0);assert(allocations==3);request[0]='x';set(&a,request,1);assert(allocations==3); + iomfb_boolean_properties_cleanup_v27(&a);iomfb_boolean_properties_cleanup_v27(&b);assert(a.iomfb_boolean_properties.next==&a.iomfb_boolean_properties&&b.iomfb_boolean_properties.next==&b.iomfb_boolean_properties);puts("DCP_BOOLEAN_PROPERTIES_V27_PASS: native name truncation, bit0, per-DCP replacement, bounded replies, allocation failure"); +} +''' +code=pre+source+post +if os.environ.get('TEST_SOURCE'):Path(os.environ['TEST_SOURCE']).write_text(code) +with tempfile.TemporaryDirectory() as d: + p=Path(d);(p/'test.c').write_text(code) + subprocess.run(['clang','-std=gnu11','-O1','-g','-fsanitize=address,undefined',str(p/'test.c'),'-o',str(p/'test')],check=True) + subprocess.run([str(p/'test')],check=True) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp-display-identity-v27.py b/tools/testing/selftests/drivers/gpu/apple-dcp-display-identity-v27.py new file mode 100644 index 0000000000000..5de0ca32d2524 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp-display-identity-v27.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Native D133/A375 wire layout, identity predicate and deferred ACK contexts.""" +from pathlib import Path +import os,subprocess,tempfile +root=next(p for p in Path(__file__).resolve().parents if (p/'drivers/gpu/drm/apple/iomfb_v27.c').exists()) +source=(root/'drivers/gpu/drm/apple/iomfb-display-identity-v27.inc').read_text() +parent=(root/'drivers/gpu/drm/apple/iomfb_v27.c').read_text() +assert '[133] = trampoline_display_identity_v27' in parent and '[133] = { 0x0, 0x4 }' in parent +pre=r''' +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8;typedef uint32_t u32;typedef uint32_t __le32; +#define __packed __attribute__((packed)) +#undef static_assert +#define static_assert(x) _Static_assert(x, #x) +#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) +#define cpu_to_le32(x) (x) +#define le32_to_cpu(x) (x) +#define GFP_KERNEL 0 +#define WARN_ON(x) (x) +enum dcp_context_id{DCP_CONTEXT_CB=0,DCP_CONTEXT_ASYNC=3,DCP_CONTEXT_OOBCB=4,DCP_CONTEXT_OOBASYNC=7}; +struct dcp_channel{unsigned depth;void*output[4];}; +struct apple_dcp{struct dcp_channel ch_cb,ch_async,ch_oobcb,ch_oobasync;}; +struct dcp_method_entry{const char*name;char tag[4],tag_h17g[4];}; +static unsigned pushes,acks,frees;static int ack_context;static bool fail_alloc; +static void(*pending)(struct apple_dcp*,void*,void*);static void*pending_cookie; +static int expected_origin; +static void* kzalloc(size_t n,int f){return fail_alloc?NULL:calloc(1,n);} +static void kfree(void*p){frees++;free(p);} +static void put_unaligned_le32(u32 v,void*p){memcpy(p,&v,4);} +#define dev_info(...) ((void)0) +#define trace_iomfb_callback(...) ((void)0) +static void dcp_ack(struct apple_dcp*d,enum dcp_context_id ctx){acks++;ack_context=ctx;} +static int dcp_push_nested(struct apple_dcp*d,enum dcp_context_id origin,const struct dcp_method_entry*m,u32 il,u32 ol,void*data,void(*cb)(struct apple_dcp*,void*,void*),void*cookie){ + const u8*bytes=data;u8 expected[24]={0};expected[14]=10; + assert(il==24&&ol==20&&!memcmp(m->tag,"A375",4)&&origin==expected_origin); + assert(!memcmp(bytes,expected,24));pushes++;pending=cb;pending_cookie=cookie;return 0; +} +''' +post=r''' +int main(void){ +struct apple_dcp d={0};struct dcp_channel*channels[]={&d.ch_cb,&d.ch_async,&d.ch_oobcb,&d.ch_oobasync};int contexts[]={0,3,4,7}; +for(int ch=0;ch<4;ch++)for(int v=0;v<3;v++)for(int id=0xae40;id<=0xae45;id++)for(int success=0;success<3;success++){ + u8 output[8]={0};u8 response[20]={0};unsigned n=acks,p=pushes; + channels[ch]->depth=1;channels[ch]->output[0]=output; + expected_origin=contexts[ch]; + assert(!trampoline_display_identity_v27(&d,133,output,NULL)&&acks==n&&pushes==p+1); + if(v==1)memcpy(response,"APP",4);if(v==2)memcpy(response,"SAM",4); + response[14]=id;response[15]=id>>8;response[18]=success;response[19]=0xa5; + pending(&d,response,pending_cookie); + assert(output[0]==(success==1&&v!=2&&(id==0xae42||id==0xae43))); + for(int i=1;i<8;i++)assert(!output[i]); + assert(acks==n+1&&ack_context==contexts[ch]&&frees==pushes); + channels[ch]->depth=0; +} +u8 output[4]={0};d.ch_cb.depth=1;d.ch_cb.output[0]=output;fail_alloc=true; +unsigned p=pushes;assert(trampoline_display_identity_v27(&d,133,output,NULL)&&pushes==p&&!output[0]); +puts("DCP_DISPLAY_IDENTITY_V27_PASS: native A375 payload, products/vendors, deferred ACK on four callback contexts");return 0; +} +''' +code=pre+source+post +if os.environ.get('TEST_SOURCE'):Path(os.environ['TEST_SOURCE']).write_text(code) +with tempfile.TemporaryDirectory() as d: + p=Path(d);(p/'test.c').write_text(code) + subprocess.run(['clang','-std=gnu11','-O1','-g','-fsanitize=address,undefined',str(p/'test.c'),'-o',str(p/'test')],check=True) + subprocess.run([str(p/'test')],check=True) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp-external-counts-v27.py b/tools/testing/selftests/drivers/gpu/apple-dcp-external-counts-v27.py new file mode 100644 index 0000000000000..1719b7414b9d5 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp-external-counts-v27.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Firmware27 packed count ABI, shared DCP state, balanced events and concurrency.""" +from pathlib import Path +import os, subprocess, tempfile +root = next(p for p in Path(__file__).resolve().parents if (p/'drivers/gpu/drm/apple/iomfb_v27.c').exists()) +source = (root/'drivers/gpu/drm/apple/iomfb-external-counts-v27.inc').read_text() +parent = (root/'drivers/gpu/drm/apple/iomfb_v27.c').read_text() +for tag, size in [(130, '0x8, 0x0'), (131, '0x10, 0xc'), (132, '0x4, 0x0')]: + assert f'[{tag}] = {{ {size} }}' in parent +pre = r''' +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; typedef uint32_t u32; +#define DEFINE_SPINLOCK(n) pthread_mutex_t n=PTHREAD_MUTEX_INITIALIZER +#define spin_lock_irqsave(n, f) do { (f)=0; assert(!pthread_mutex_lock(n)); } while(0) +#define spin_unlock_irqrestore(n, f) do { (void)(f); assert(!pthread_mutex_unlock(n)); } while(0) +#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) +#define DRM_MODE_CONNECTOR_eDP 14 +#define trace_iomfb_callback(...) ((void)0) +struct apple_dcp { int connector_type; }; +static u32 get_unaligned_le32(const void*p) { const u8*b=p;return b[0]|b[1]<<8|b[2]<<16|(u32)b[3]<<24; } +static void put_unaligned_le32(u32 v,void*p) {u8*b=p;for(int i=0;i<4;i++)b[i]=v>>(8*i);} +''' +post = r''' +static struct apple_dcp internal={14}, hdmi={11}, usbc={10}; +static void read_counts(struct apple_dcp*d,u32 a,u32 b,u32 c) { + u8 in[16]={0},out[12];memset(out,0xa5,sizeof(out)); + assert(trampoline_get_external_counts_v27(d,131,out,in)); + assert(get_unaligned_le32(out)==a&&get_unaligned_le32(out+4)==b&&get_unaligned_le32(out+8)==c); +} +static void* cycle(void*arg) { + struct apple_dcp*d=arg;u8 plug[8]={1,1,3},unplug[8]={0,1,3},on[4]={1},off[4]={0}; + for(int i=0;i<1000;i++) { + trampoline_update_external_counts_v27(d,130,NULL,plug); + trampoline_apply_suppression_v27(d,132,NULL,on); + trampoline_apply_suppression_v27(d,132,NULL,off); + trampoline_update_external_counts_v27(d,130,NULL,unplug); + }return NULL; +} +int main(void) { + /* Captured Samsung D130 payload: count is packed at offset2, not4. */ + u8 plug[8]={1,0,2,0,0,0,0,0},unplug[8]={0,0,2,0,0,0,0,0}; + trampoline_update_external_counts_v27(&internal,130,NULL,plug);read_counts(&hdmi,0,0,0); + trampoline_update_external_counts_v27(&hdmi,130,NULL,plug);read_counts(&internal,2,0,0); + u8 apple[8]={3,3,4},off[8]={2,3,4}; /* bool uses bit0, not nonzero. */ + trampoline_update_external_counts_v27(&usbc,130,NULL,apple);read_counts(&hdmi,6,1,0); + u8 on[4]={3},clear[4]={2}; + trampoline_apply_suppression_v27(&internal,132,NULL,on);read_counts(&usbc,6,1,1); + trampoline_apply_suppression_v27(&hdmi,132,NULL,on);read_counts(&usbc,6,1,2); + for(unsigned mask=0;mask<8;mask++) { + u8 in[16]={0},out[14];memset(out,0xa5,sizeof(out)); + for(int i=0;i<3;i++)in[12+i]=2+((mask>>i)&1); + trampoline_get_external_counts_v27(&hdmi,131,out+1,in); + u32 values[]={6,1,2}; + for(int i=0;i<3;i++)assert(get_unaligned_le32(out+1+4*i)==((mask>>i)&1?0xa5a5a5a5:values[i])); + assert(out[0]==0xa5&&out[13]==0xa5); + } + trampoline_update_external_counts_v27(&hdmi,130,NULL,unplug); + trampoline_update_external_counts_v27(&usbc,130,NULL,off); + trampoline_apply_suppression_v27(&internal,132,NULL,clear); + trampoline_apply_suppression_v27(&hdmi,132,NULL,clear);read_counts(&internal,0,0,0); + pthread_t a,b;assert(!pthread_create(&a,NULL,cycle,&hdmi));assert(!pthread_create(&b,NULL,cycle,&usbc)); + assert(!pthread_join(a,NULL));assert(!pthread_join(b,NULL));read_counts(&internal,0,0,0); + puts("DCP_EXTERNAL_COUNTS_V27_PASS: packed wire ABI, shared counts, internal gating, suppression, nullable outputs, concurrent balanced events"); +} +''' +code = pre + source + post +if os.environ.get('TEST_SOURCE'): Path(os.environ['TEST_SOURCE']).write_text(code) +with tempfile.TemporaryDirectory() as d: + p=Path(d); (p/'test.c').write_text(code) + subprocess.run(['clang','-std=gnu11','-O1','-g','-pthread','-fsanitize=address,undefined',str(p/'test.c'),'-o',str(p/'test')],check=True) + subprocess.run([str(p/'test')],check=True) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp-nested-stream.py b/tools/testing/selftests/drivers/gpu/apple-dcp-nested-stream.py new file mode 100644 index 0000000000000..5abd988b763cf --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp-nested-stream.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise real RPC push/ACK stacks with native firmware-owned stream offsets.""" +from pathlib import Path +import os,re,subprocess,tempfile +root=next(p for p in Path(__file__).resolve().parents if (p/'drivers/gpu/drm/apple/iomfb.c').exists()) +s=(root/'drivers/gpu/drm/apple/iomfb.c').read_text();h=(root/'drivers/gpu/drm/apple/iomfb.h').read_text() +a=h.index('enum dcp_context_id {');enum=h[a:h.index('\n};',a)+3] +def f(name): + pat=r'^(?:static )?(?:inline )?(?:void|int|u8|u16|u64|enum dcp_context_id|struct dcp_channel \*)\s*'+name+r'\(' + m=re.search(pat,s,re.M);assert m,name;a=m.start();return s[a:s.index('\n}',a)+2] +pre=r''' +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8;typedef uint16_t u16;typedef uint32_t u32;typedef uint64_t u64;typedef uint32_t __le32; +#define __packed __attribute__((packed)) +#undef static_assert +#define static_assert(x) _Static_assert(x, #x) +#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) +#define cpu_to_le32(x) (x) +#define le32_to_cpu(x) (x) +#define FIELD_PREP(m,v) (((u64)(v)<<__builtin_ctzll(m))&(m)) +#define FIELD_GET(m,v) (((u64)(v)&(m))>>__builtin_ctzll(m)) +#define IOMFB_MSG_CONTEXT 0xf00ULL +#define IOMFB_MSG_OFFSET 0xffff0000ULL +#define IOMFB_MSG_LENGTH 0xffff00000000ULL +#define IOMFB_MSG_ACK 0x40ULL +#define IOMFB_MESSAGE_TYPE 0xfULL +#define IOMFB_MESSAGE_TYPE_MSG 2 +#define IOMFB_ENDPOINT 0x20 +#define DCP_MAX_CALL_DEPTH 8 +#define DCP_PACKET_ALIGNMENT 64 +#define DCP_IOMFB_METHODS_H17G 1 +#define ALIGN(n,a) (((n)+(a)-1)&~((a)-1)) +#define WARN_ON(x) ({bool z=!!(x);assert(!z);z;}) +#define GFP_KERNEL 0 +#define kzalloc(n,f) calloc(1,n) +#define kfree(p) free(p) +#define trace_iomfb_push(...) ((void)0) +#define trace_iomfb_callback(...) ((void)0) +#define dev_info(...) ((void)0) +#define dev_warn(...) ((void)0) +#define dev_err(...) ((void)0) +#define msecs_to_jiffies(n) (n) +#define mod_delayed_work(...) ((void)0) +static unsigned long jiffies; +static int dcp_trace_budget,dcp_trace_calls; +struct apple_dcp; +typedef void(*dcp_callback_t)(struct apple_dcp*,void*,void*); +struct dcp_channel{dcp_callback_t callbacks[8];void*cookies[8],*output[8];u16 end[8];char tag[8][4];unsigned long pushed_at[8];u8 depth;}; +struct apple_dcp{struct dcp_channel ch_cb,ch_cmd,ch_async,ch_oobcb,ch_oobcmd,ch_oobasync;struct{int iomfb_method_profile;}hw;void*shmem;bool warned_stall;}; +struct dcp_method_entry{const char*name;char tag[4],tag_h17g[4];}; +struct dcp_packet_header{char tag[4];u32 in_len,out_len;}; +static u64 sent[8];static unsigned nsent; +static void dcp_send_message(struct apple_dcp*d,unsigned ep,u64 msg){assert(ep==0x20&&nsent<8);sent[nsent++]=msg;} +static void put_unaligned_le32(u32 v,void*p){memcpy(p,&v,4);} +''' +functions='\n'.join(f(n)for n in ['dcp_tx_offset','dcp_channel_offset','dcpep_msg','dcpep_ack','dcp_call_context','dcp_get_channel','dcp_packet_start','dcp_push_depth','dcp_pop_depth','dcp_push_context','dcp_push','dcp_push_nested','dcp_ack','dcpep_handle_ack']) +identity=(root/'drivers/gpu/drm/apple/iomfb-display-identity-v27.inc').read_text() +post=r''' +int main(void){ +const int origins[]={0,3,4,7},wire[]={0,1,4,5}; +/* Captured native stream initialization, not computed by the tested helpers. */ +const int tx[]={0,0x20000,0x8000,0x28000},rx[]={0x60000,0x40000,0x68000,0x48000}; +for(int i=0;i<4;i++){ + struct apple_dcp d={.shmem=calloc(1,0x80000)};struct dcp_channel*c=dcp_get_channel(&d,origins[i]); + assert(dcp_tx_offset(wire[i])==tx[i]&&dcp_channel_offset(wire[i])==rx[i]); + assert(c==dcp_get_channel(&d,wire[i])); + /* An unrelated AP command must not steal a firmware-owned nested query. */ + d.ch_cmd.depth=1;d.ch_oobcmd.depth=1; + c->depth=1;c->end[0]=64;c->output[0]=d.shmem+rx[i]+12;memcpy(c->tag[0],"D133",4); + nsent=0;assert(!trampoline_display_identity_v27(&d,133,c->output[0],NULL)); + assert(nsent==1&&FIELD_GET(IOMFB_MSG_CONTEXT,sent[0])==(u64)wire[i]); + assert(FIELD_GET(IOMFB_MSG_OFFSET,sent[0])==64&&FIELD_GET(IOMFB_MSG_LENGTH,sent[0])==56); + assert(c->depth==2&&c->end[1]==128&&c->output[1]==d.shmem+tx[i]+100); + struct dcp_packet_header*header=d.shmem+tx[i]+64; + assert(!memcmp(header->tag,"573A",4)&&header->in_len==24&&header->out_len==20); + u8*request=(void*)(header+1);for(int j=0;j<24;j++)assert(request[j]==(j==14?10:0)); + u8*reply=c->output[1];memcpy(reply,"APP",4);reply[14]=0x42;reply[15]=0xae;reply[18]=1; + dcpep_handle_ack(&d,wire[i]); + assert(nsent==2&&FIELD_GET(IOMFB_MSG_CONTEXT,sent[1])==(u64)origins[i]&&(sent[1]&IOMFB_MSG_ACK)); + assert(c->depth==0&&*(u8*)(d.shmem+rx[i]+12)==1); + assert(d.ch_cmd.depth==1&&d.ch_oobcmd.depth==1); + free(d.shmem); +} + +for(int oob=0;oob<2;oob++){ + struct apple_dcp d={.shmem=calloc(1,0x80000)};struct dcp_method_entry m={.tag={'A','0','0','0'}}; + struct dcp_channel *cmd=oob?&d.ch_oobcmd:&d.ch_cmd,*cb=oob?&d.ch_oobcb:&d.ch_cb; + nsent=0;dcp_push(&d,oob,&m,0,0,NULL,NULL,NULL); + assert(FIELD_GET(IOMFB_MSG_CONTEXT,sent[0])==(u64)(oob?6:2)&&cmd->depth==1); + dcpep_handle_ack(&d,oob?6:2);assert(!cmd->depth); + cmd->depth=1;cb->depth=1;cb->end[0]=64; + dcp_push(&d,oob,&m,0,0,NULL,NULL,NULL); + assert(FIELD_GET(IOMFB_MSG_CONTEXT,sent[1])==(u64)(oob?4:0)&&cb->depth==2); + assert(FIELD_GET(IOMFB_MSG_OFFSET,sent[1])==64); + unsigned n=nsent;assert(dcp_push_nested(&d,DCP_CONTEXT_CMD,&m,0,0,NULL,NULL,NULL)==-EINVAL&&nsent==n); + free(d.shmem); +} +puts("DCP_NESTED_STREAM_PASS: actual RPC/ACK stacks, native TX/RX regions, firmware-owned contexts1/5");return 0; +} +''' +code=pre+enum+functions+identity+post +if os.environ.get('TEST_SOURCE'):Path(os.environ['TEST_SOURCE']).write_text(code) +with tempfile.TemporaryDirectory() as d: + p=Path(d);(p/'test.c').write_text(code) + subprocess.run(['clang','-std=gnu11','-O1','-g','-fsanitize=address,undefined',str(p/'test.c'),'-o',str(p/'test')],check=True) + subprocess.run([str(p/'test')],check=True) From 9d737817593f0c82c34954c062823815d7eb60b3 Mon Sep 17 00:00:00 2001 From: dj Date: Tue, 29 Sep 2026 00:40:08 -0400 Subject: [PATCH 0231/1579] spmi: support T6050 generation 4 controllers and initial Type-C events Select generation 4 FIFO and interrupt registers with the T6050 compatible; preserve the older layout. Recover stalled queues with the native disable/drain/reset/restore sequence and bounded waits. Drain SN201202x events once after port and IRQ setup so an event latched before registration cannot leave cached connection state stale. Subsequent updates remain interrupt driven. Validation: ARM64 QEMU module/source checks for both layouts and recovery failures; J714s hardware register access and wake/selection interrupts; user-performed left-front unplug/replug delivered events and reconciled cached/live DisplayPort state. --- .../devicetree/bindings/spmi/apple,spmi.yaml | 1 + drivers/spmi/spmi-apple-controller.c | 115 +++++++++++++----- drivers/usb/typec/tipd/spmi.c | 13 +- 3 files changed, 98 insertions(+), 31 deletions(-) diff --git a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml index fceed47b2cd17..4d032d3536343 100644 --- a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml +++ b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml @@ -17,6 +17,7 @@ allOf: properties: compatible: oneOf: + - const: apple,t6050-spmi - items: - enum: - apple,t6020-spmi diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index ec49d412a4b0e..a56bb437c6edf 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -27,33 +27,44 @@ #include #include -/* SPMI Controller Registers */ -#define SPMI_STATUS_REG 0 -#define SPMI_CMD_REG 0x4 -#define SPMI_RSP_REG 0x8 -#define SPMI_ACT_REG 0xa4 - -#define SPMI_IRQ_MASK_BASE 0x20 -#define SPMI_IRQ_ACK_BASE 0x60 +/* Generation 4 moves both FIFOs and interrupt banks. */ +struct apple_spmi_hw { + u16 status, cmd, rsp, irq_mask, irq_ack; + u32 rx_empty; + bool gen4; +}; + +static const struct apple_spmi_hw apple_spmi_gen1 = { + .status = 0, .cmd = 4, .rsp = 8, + .irq_mask = 0x20, .irq_ack = 0x60, + .rx_empty = BIT(24), +}; + +static const struct apple_spmi_hw apple_spmi_gen4 = { + .status = 0x200, .cmd = 0x210, .rsp = 0x220, + .irq_mask = 0x400, .irq_ack = 0x600, + .rx_empty = BIT(30), .gen4 = true, +}; + #define SPMI_NUM_PERIPHERAL_IRQS 256 #define SPMI_NUM_IRQS (SPMI_NUM_PERIPHERAL_IRQS + 32) #define SPMI_IRQ_NOTIFY 256 -/* SPMI_RSP_REG reply word */ +/* spmi->hw->rsp reply word */ #define SPMI_REPLY_FRAME_PARITY_STATUS GENMASK(31, 16) #define SPMI_REPLY_ACK BIT(15) #define SPMI_REPLY_SLAVE_ID GENMASK(14, 8) #define SPMI_REPLY_CMD GENMASK(7, 0) #define SPMI_ACT_FIFO_FLUSH BIT(0) -#define SPMI_RX_FIFO_EMPTY BIT(24) #define REG_POLL_INTERVAL_US 10000 #define REG_POLL_TIMEOUT_US (REG_POLL_INTERVAL_US * 5) struct apple_spmi { void __iomem *regs; + const struct apple_spmi_hw *hw; struct mutex fifo_lock; struct completion fifo_rx; struct irq_domain *irqd; @@ -70,21 +81,21 @@ struct apple_spmi { static void apple_spmi_irq_ack_raw(struct apple_spmi *spmi, u32 irq) { - u32 __iomem *reg = spmi->regs + SPMI_IRQ_ACK_BASE + (irq / 32) * 4; + u32 __iomem *reg = spmi->regs + spmi->hw->irq_ack + (irq / 32) * 4; writel(BIT(irq % 32), reg); } static void apple_spmi_irq_mask_raw(struct apple_spmi *spmi, u32 irq) { - u32 __iomem *reg = spmi->regs + SPMI_IRQ_MASK_BASE + (irq / 32) * 4; + u32 __iomem *reg = spmi->regs + spmi->hw->irq_mask + (irq / 32) * 4; writel(readl(reg) & ~BIT(irq % 32), reg); } static void apple_spmi_irq_unmask_raw(struct apple_spmi *spmi, u32 irq) { - u32 __iomem *reg = spmi->regs + SPMI_IRQ_MASK_BASE + (irq / 32) * 4; + u32 __iomem *reg = spmi->regs + spmi->hw->irq_mask + (irq / 32) * 4; writel(readl(reg) | BIT(irq % 32), reg); } @@ -135,12 +146,12 @@ static int apple_spmi_wait_rx_not_empty(struct spmi_controller *ctrl) usecs_to_jiffies(REG_POLL_TIMEOUT_US)); if (!ret) ret = -ETIMEDOUT; - else if (readl(spmi->regs + SPMI_STATUS_REG) & SPMI_RX_FIFO_EMPTY) + else if (readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty) ret = -EIO; else ret = 0; } else { - ret = poll_reg(spmi, SPMI_STATUS_REG, status, !(status & SPMI_RX_FIFO_EMPTY)); + ret = poll_reg(spmi, spmi->hw->status, status, !(status & spmi->hw->rx_empty)); } if (ret) { @@ -153,6 +164,43 @@ static int apple_spmi_wait_rx_not_empty(struct spmi_controller *ctrl) return 0; } +/* Gen4 resets the command queue through its control register. The native + * sequence disables the controller, drains responses, resets pending commands, + * then restores the previous enable state. Never write reset data to the FIFO. + */ +static int apple_spmi_recover(struct apple_spmi *spmi) +{ + u32 enable, status; + int ret = 0, remaining = 512; + + if (!spmi->hw->gen4) { + writel(SPMI_ACT_FIFO_FLUSH, spmi->regs + spmi->hw->rsp); + goto ack; + } + enable = readl(spmi->regs); + writel(0, spmi->regs); + while (!(readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty)) { + if (!remaining--) { + ret = -EIO; + goto restore; + } + readl(spmi->regs + spmi->hw->rsp); + } + if (!(readl(spmi->regs + spmi->hw->status) & BIT(14))) { + writel(BIT(0), spmi->regs + 4); + ret = readl_poll_timeout(spmi->regs + 4, status, + !(status & BIT(0)), 50, 50000); + if (!ret && (readl(spmi->regs + spmi->hw->status) & + (BIT(14) | BIT(30))) != (BIT(14) | BIT(30))) + ret = -EIO; + } +restore: + writel(enable, spmi->regs); +ack: + apple_spmi_irq_ack_raw(spmi, SPMI_IRQ_NOTIFY); + return ret; +} + static int spmi_raw_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid, u16 param, const u8 *buf, size_t len, u8 *ibuf, size_t ilen) { @@ -166,19 +214,20 @@ static int spmi_raw_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid, guard(mutex)(&spmi->fifo_lock); if (spmi->prev_fail) { - writel(SPMI_ACT_FIFO_FLUSH, spmi->regs + SPMI_RSP_REG); - apple_spmi_irq_ack_raw(spmi, SPMI_IRQ_NOTIFY); + ret = apple_spmi_recover(spmi); + if (ret) + return ret; spmi->prev_fail = false; } reinit_completion(&spmi->fifo_rx); - writel(spmi_cmd, spmi->regs + SPMI_CMD_REG); + writel(spmi_cmd, spmi->regs + spmi->hw->cmd); while (i < len) { j = min_t(size_t, sizeof(spmi_cmd), len - i); spmi_cmd = 0; memcpy(&spmi_cmd, buf + i, j); - writel(spmi_cmd, spmi->regs + SPMI_CMD_REG); + writel(spmi_cmd, spmi->regs + spmi->hw->cmd); i += j; } @@ -186,23 +235,23 @@ static int spmi_raw_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid, if (ret) return ret; - reply = readl(spmi->regs + SPMI_RSP_REG); + reply = readl(spmi->regs + spmi->hw->rsp); /* Read SPMI data reply */ while (len_read < ilen) { - if (readl(spmi->regs + SPMI_STATUS_REG) & SPMI_RX_FIFO_EMPTY) { + if (readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty) { spmi->prev_fail = true; dev_err_ratelimited(&ctrl->dev, "FIFO lacks reply data, controller stuck?\n"); return -EIO; } - rsp = readl(spmi->regs + SPMI_RSP_REG); + rsp = readl(spmi->regs + spmi->hw->rsp); i = min_t(size_t, sizeof(spmi_cmd), ilen - len_read); memcpy(ibuf + len_read, &rsp, i); len_read += i; } - if (!(readl(spmi->regs + SPMI_STATUS_REG) & SPMI_RX_FIFO_EMPTY)) { + if (!(readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty)) { dev_warn(&ctrl->dev, "FIFO has extra data\n"); spmi->prev_fail = true; } @@ -363,7 +412,7 @@ static void apple_spmi_irq_handler(struct irq_desc *desc) unsigned long val, offset, bit; chained_irq_enter(chip, desc); - val = readl(spmi->regs + SPMI_IRQ_ACK_BASE + (SPMI_IRQ_NOTIFY / 32) * 4); + val = readl(spmi->regs + spmi->hw->irq_ack + (SPMI_IRQ_NOTIFY / 32) * 4); if (val & BIT(SPMI_IRQ_NOTIFY % 32)) { apple_spmi_irq_ack_raw(spmi, SPMI_IRQ_NOTIFY); complete(&spmi->fifo_rx); @@ -371,7 +420,11 @@ static void apple_spmi_irq_handler(struct irq_desc *desc) } for (offset = 0; offset < SPMI_NUM_PERIPHERAL_IRQS / 8; offset += sizeof(val)) { - val = readq(spmi->regs + SPMI_IRQ_ACK_BASE + offset); + if (spmi->hw->gen4) + val = readl(spmi->regs + spmi->hw->irq_ack + offset) | + ((u64)readl(spmi->regs + spmi->hw->irq_ack + offset + 4) << 32); + else + val = readq(spmi->regs + spmi->hw->irq_ack + offset); /** * because of other masters in the bus, we're going to get a multitude of * interrupts we're not interested in. irq_resolve_mapping isn't very @@ -411,8 +464,8 @@ static int apple_spmi_init_irq(struct platform_device *pdev, raw_spin_lock_init(&spmi->irq_mask_lock); for (size_t offset = 0; offset < SPMI_NUM_IRQS / 8; offset += 4) { - writel(0, spmi->regs + SPMI_IRQ_MASK_BASE + offset); - writel(U32_MAX, spmi->regs + SPMI_IRQ_ACK_BASE + offset); + writel(0, spmi->regs + spmi->hw->irq_mask + offset); + writel(U32_MAX, spmi->regs + spmi->hw->irq_ack + offset); } spmi->irqd = devm_irq_domain_instantiate(&pdev->dev, &info); @@ -441,6 +494,9 @@ static int apple_spmi_probe(struct platform_device *pdev) return -ENOMEM; spmi = spmi_controller_get_drvdata(ctrl); + spmi->hw = device_get_match_data(&pdev->dev); + if (!spmi->hw) + return -EINVAL; mutex_init(&spmi->fifo_lock); init_completion(&spmi->fifo_rx); platform_set_drvdata(pdev, spmi); @@ -473,8 +529,9 @@ static int apple_spmi_probe(struct platform_device *pdev) } static const struct of_device_id apple_spmi_match_table[] = { - { .compatible = "apple,t8103-spmi", }, - { .compatible = "apple,spmi", }, + { .compatible = "apple,t6050-spmi", .data = &apple_spmi_gen4 }, + { .compatible = "apple,t8103-spmi", .data = &apple_spmi_gen1 }, + { .compatible = "apple,spmi", .data = &apple_spmi_gen1 }, {} }; MODULE_DEVICE_TABLE(of, apple_spmi_match_table); diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 73575a854bde1..b12b2bfc15fa9 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -223,9 +223,18 @@ static int sn201202x_probe(struct spmi_device *device) return -ETIMEDOUT; ret = tipd_init(tps); - if (ret) + if (ret) { spmi_command_sleep(device); - return ret; + return ret; + } + + /* SPMI reports events as messages, not a continuously asserted GPIO. + * Firmware may already have an event latched when the event IRQ is + * registered. Drain it once with the handler now that the port and + * work item exist; subsequent changes are handled by interrupts. + */ + tps->data->irq_handler(tps->irq, tps); + return 0; } static void sn201202x_remove(struct spmi_device *device) From 38d15a7d83aede760055126d211085e7b89acc32 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 29 Sep 2026 02:57:10 -0400 Subject: [PATCH 0232/1579] Bluetooth: hci_sync: let drivers reset a persistent LE duplicate filter The core enables LE scanning with controller duplicate filtering and relies on each enable starting with an empty filter: the accept-list scan used to establish a connection must see the peer again even if discovery reported it moments earlier. Apple N1 firmware 27 keeps its duplicate list across scan disable and enable and clears it only on HCI Reset. A peer found during discovery is then never reported to the connection scan, so LE Create Connection is never sent, and later discoveries report only never-seen devices. Add an optional hdev->reset_dup_filter() callback, run from the cmd_sync context right before scanning is enabled with duplicate filtering. Drivers can issue their vendor command there and keep controller filtering, instead of disabling it and taking every advertising report. The member goes at the end of struct hci_dev. Co-Authored-By: Claude Opus 5.5 --- include/net/bluetooth/hci_core.h | 1 + net/bluetooth/hci_sync.c | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index eb721509c0ab8..d19e5a1b7a3ef 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -665,6 +665,7 @@ struct hci_dev { struct bt_codec *codec, __u8 *vnd_len, __u8 **vnd_data); u8 (*classify_pkt_type)(struct hci_dev *hdev, struct sk_buff *skb); + int (*reset_dup_filter)(struct hci_dev *hdev); }; #define hci_set_quirk(hdev, nr) set_bit((nr), (hdev)->quirk_flags) diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index e39119bfef647..9b9d108d27ca9 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -2283,6 +2283,18 @@ static int hci_le_set_scan_enable_sync(struct hci_dev *hdev, u8 val, { struct hci_cp_le_set_scan_enable cp; + /* Some controllers keep their duplicate filter across scan cycles, + * so a device reported once is never reported again. Let the driver + * reset it before scanning starts with duplicate filtering. + */ + if (val && filter_dup != LE_SCAN_FILTER_DUP_DISABLE && + !hci_dev_test_flag(hdev, HCI_MESH) && hdev->reset_dup_filter) { + int err = hdev->reset_dup_filter(hdev); + + if (err) + return err; + } + if (use_ext_scan(hdev)) return hci_le_set_ext_scan_enable_sync(hdev, val, filter_dup); From 4a0a209ad88027c0693b8f50cd98078218fcfdb6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 29 Sep 2026 02:57:19 -0400 Subject: [PATCH 0233/1579] Bluetooth: btcentauri: fix firmware 27 LE discovery and connections With ACL enabled, LE connections to a known peer never completed and repeated discovery lost devices. Three firmware 27 behaviours caused this; each is handled at the transport: - Legacy scan responses arrive as extended report type 0x18, with the scannable bit missing, which the core rejects ("Unknown advertising packet type: 0x18"), so scan-response data such as a service UUID is dropped. Restore only that bit. Parsing is best effort; a report that does not fit is passed unchanged to the core, which checks it. - Primary PHY carries a native flag (0x81 for LE 1M), so connections started without a usable initiating PHY. Set the existing HCI_QUIRK_FIXUP_LE_EXT_ADV_REPORT_PHY, as for other Apple controllers. - The LE duplicate filter survives scan disable/enable. Vendor command 0xff3f (firmware handler 0x808d62) resets it for both scanner PHYs; issue it through the new reset_dup_filter hook. Parameter bit 2 selects an unrelated action and is left clear. A raw HCI probe with the peer advertising: filtered rescans report it 0 times, 2 times after 0xff3f. flush_duplicates=0 restores the old behaviour. ACL delivery now opens only the native ACL rings (2 and 8); debug and audio resources stay behind arm_auxiliary_rings. Tested on J714s with btcentauri.enable_acl=1 against an unpaired BlueZ GATT peer: five fresh discover/connect cycles each completed 64 verified 512-byte reads, with no transport errors and Wi-Fi up. WirePlumber's BLE-MIDI was stopped for these runs; otherwise both hosts probe each other's encrypted MIDI characteristic and the unpairable link drops. With flush_duplicates=0 the original connect stall returned. Co-Authored-By: Claude Opus 5.5 --- drivers/bluetooth/btcentauri.c | 100 ++++++++++++++++++++++++++++++--- 1 file changed, 92 insertions(+), 8 deletions(-) diff --git a/drivers/bluetooth/btcentauri.c b/drivers/bluetooth/btcentauri.c index 2db73da175863..8634b50d74661 100644 --- a/drivers/bluetooth/btcentauri.c +++ b/drivers/bluetooth/btcentauri.c @@ -71,14 +71,19 @@ MODULE_PARM_DESC(native_bdaddr, "Optional stable diagnostic address in XX:XX:XX: static bool enable_acl; module_param(enable_acl, bool, 0444); -MODULE_PARM_DESC(enable_acl, "Experimental ACL data delivery; requires LE and auxiliary rings"); +MODULE_PARM_DESC(enable_acl, "Experimental ACL data delivery; requires LE rings and opens native ACL resources"); static bool native_hrb; module_param(native_hrb, bool, 0444); MODULE_PARM_DESC(native_hrb, "Experimental native Proxima Create Connection; requires native startup, ACL and firmware capability bit 9"); +static bool flush_duplicates = true; +module_param(flush_duplicates, bool, 0644); +MODULE_PARM_DESC(flush_duplicates, "Reset the firmware LE duplicate filter before each filtered scan"); + #define CEN_OP_READ_PROP_FEATURES 0xffa3 #define CEN_OP_HRB_CREATE_CONN 0xff7b +#define CEN_OP_FLUSH_DUPLICATES 0xff3f #define CEN_PROP_HRB_CREATE BIT(9) struct cen_aux_spec { @@ -106,6 +111,7 @@ static const struct cen_aux_spec cen_aux_specs[] = { struct cen_aux_state { u32 offset, packets; u16 head, post; + bool opened; }; struct cen_bt { @@ -134,6 +140,8 @@ struct cen_bt { u32 proprietary_features, hrb_commands, hrb_status_events; u32 hrb_connection_events; u8 hrb_last_link_type; + bool dup_flush_failed; + u32 dup_flushes; }; static u16 cen_tail(struct cen_bt *bt, u16 ring) @@ -292,6 +300,14 @@ static void cen_aux_rx_buffer(struct cen_bt *bt, unsigned int index, u16 slot) put_unaligned_le16(slot + 1, descriptor + 12); } +/* ACL transport owns only its data rings. Opening debug/audio resources + * and capturing their payloads remains an explicitly requested diagnostic. + */ +static bool cen_aux_required(u16 id) +{ + return arm_auxiliary_rings || (enable_acl && (id == 2 || id == 8)); +} + static int cen_aux_start(struct cen_bt *bt) { u32 offset = CEN_AUX_BASE; @@ -307,11 +323,14 @@ static int cen_aux_start(struct cen_bt *bt) return -ENOSPC; state->offset = offset; offset = ALIGN(offset + length, SZ_4K); + if (!cen_aux_required(spec->id)) + continue; ret = cen_open_ring(bt, spec->id, spec->slots, state->offset, spec->footer); if (ret) { dev_err(&bt->pdev->dev, "Auxiliary ring %u open failed: %d\n", spec->id, ret); return ret; } + state->opened = true; if (!spec->receive) continue; for (slot = 0; slot < spec->slots - 1; slot++) @@ -332,7 +351,7 @@ static int cen_aux_drain(struct cen_bt *bt, bool repost) u16 tail; int budget = spec->slots; - if (!spec->receive) + if (!state->opened || !spec->receive) continue; tail = cen_tail(bt, spec->id); if (tail >= spec->slots) @@ -419,7 +438,7 @@ static int cen_start(struct cen_bt *bt) bt->le_rx_post = CEN_LE_SLOTS - 1; cen_post(bt, 10, bt->le_rx_post); } - if (arm_auxiliary_rings) + if (arm_auxiliary_rings || enable_acl) return cen_aux_start(bt); return 0; } @@ -479,6 +498,37 @@ static int cen_transmit(struct cen_bt *bt, struct sk_buff_head *queue, return 0; } +/* N1 firmware27 omits the scannable bit from legacy scan responses. + * A captured response to our peer's ADV_IND has type 0x2518 and carries + * its scan-response name. Restore only that missing standard property; + * preserve native upper bits, payload, address and non-legacy reports. + * The core bounds-checks every report itself, so stop at the first one + * that does not fit and pass the rest through untouched. + */ +static void cen_fixup_scan_response(u8 *event, size_t length) +{ + struct hci_ev_le_ext_adv_info *info; + size_t offset = HCI_EVENT_HDR_SIZE + 2; + u8 count; + + if (length < offset || event[0] != HCI_EV_LE_META || + event[HCI_EVENT_HDR_SIZE] != HCI_EV_LE_EXT_ADV_REPORT) + return; + count = event[HCI_EVENT_HDR_SIZE + 1]; + while (count-- && length - offset >= sizeof(*info)) { + u16 type; + + info = (void *)(event + offset); + if (info->length > length - offset - sizeof(*info)) + return; + type = get_unaligned_le16(&info->type); + if ((type & LE_EXT_ADV_EVT_TYPE_MASK) == + (LE_EXT_ADV_LEGACY_PDU | LE_EXT_ADV_SCAN_RSP)) + put_unaligned_le16(type | LE_EXT_ADV_SCAN_IND, &info->type); + offset += sizeof(*info) + info->length; + } +} + static void cen_poll(struct work_struct *work) { struct cen_bt *bt = container_of(work, struct cen_bt, poll); @@ -496,11 +546,11 @@ static void cen_poll(struct work_struct *work) /* Preserve already completed diagnostics after abort, without posting * any more buffers or ringing doorbells to the failed firmware. */ - if (arm_auxiliary_rings && readl(bt->bar) == 2) + if ((arm_auxiliary_rings || enable_acl) && readl(bt->bar) == 2) cen_aux_drain(bt, false); goto protocol_error; } - if (arm_auxiliary_rings && cen_aux_drain(bt, true)) + if ((arm_auxiliary_rings || enable_acl) && cen_aux_drain(bt, true)) goto protocol_error; if (arm_le_data_rings) { tail = cen_tail(bt, 10); @@ -536,6 +586,7 @@ static void cen_poll(struct work_struct *work) skb = bt_skb_alloc(length, GFP_KERNEL); if (skb) { skb_put_data(skb, descriptor + 16, length); + cen_fixup_scan_response(skb->data, skb->len); /* The firmware's status belongs to the standard connection * request tracked by Linux. Capture the native response before * restoring only its opcode; preserve status and command credit. @@ -600,6 +651,7 @@ static int cen_hci_open(struct hci_dev *hdev) mutex_lock(&bt->lock); bt->running = true; + WRITE_ONCE(bt->dup_flush_failed, false); schedule_work(&bt->poll); mutex_unlock(&bt->lock); return 0; @@ -634,6 +686,32 @@ static int cen_vendor_command(struct hci_dev *hdev, u16 opcode, return ret; } +/* Firmware27 keeps its LE duplicate filter across scan disable/enable and + * clears it only on HCI Reset or vendor command 0xff3f, whose handler resets + * the filter of both scanner PHYs. Without this, a peer reported during + * discovery is never reported to the connection scan that follows. Bit 2 of + * the parameter selects an additional, unrelated action, so send zero. + */ +static int cen_hci_reset_dup_filter(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + static const u8 flags; + int ret; + + if (!READ_ONCE(flush_duplicates) || READ_ONCE(bt->dup_flush_failed)) + return 0; + ret = __hci_cmd_sync_status(hdev, CEN_OP_FLUSH_DUPLICATES, sizeof(flags), + &flags, HCI_CMD_TIMEOUT); + if (ret) { + /* Scanning still works; only repeated reports stay filtered. */ + WRITE_ONCE(bt->dup_flush_failed, true); + bt_dev_warn(hdev, "Duplicate filter reset failed: %d", ret); + return 0; + } + WRITE_ONCE(bt->dup_flushes, bt->dup_flushes + 1); + return 0; +} + static int cen_hci_post_init(struct hci_dev *hdev) { struct cen_bt *bt = hci_get_drvdata(hdev); @@ -882,15 +960,18 @@ static int cen_status_show(struct seq_file *s, void *unused) bt->hrb_status_events, bt->hrb_status_pending); seq_printf(s, "hrb_connection_events=%u hrb_last_link_type=0x%02x\n", bt->hrb_connection_events, bt->hrb_last_link_type); + seq_printf(s, "flush_duplicates=%u dup_flushes=%u dup_flush_failed=%u\n", + READ_ONCE(flush_duplicates), READ_ONCE(bt->dup_flushes), + READ_ONCE(bt->dup_flush_failed)); seq_printf(s, "acl_enabled=%u acl_tx=%u acl_rx=%u acl_queued=%u le_queued=%u\n", enable_acl, bt->hdev->stat.acl_tx, bt->hdev->stat.acl_rx, skb_queue_len(&bt->acl_txq), skb_queue_len(&bt->le_txq)); seq_printf(s, "auxiliary_rings=%u messages=%u message_head=%u capture_bytes=%u capture_records=%u capture_dropped=%u\n", arm_auxiliary_rings, bt->message_tag, bt->message_head, bt->capture_used, bt->capture_records, bt->capture_dropped); - if (arm_auxiliary_rings) + if (arm_auxiliary_rings || enable_acl) for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) - if (cen_aux_specs[i].receive) + if (bt->aux[i].opened && cen_aux_specs[i].receive) seq_printf(s, "aux_rx ring=%u packets=%u consumed=%u device_tail=%u\n", cen_aux_specs[i].id, bt->aux[i].packets, bt->aux[i].head, cen_tail(bt, cen_aux_specs[i].id)); @@ -958,7 +1039,7 @@ static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) skb_queue_head_init(&bt->txq); skb_queue_head_init(&bt->acl_txq); skb_queue_head_init(&bt->le_txq); - if (enable_acl && (!arm_le_data_rings || !arm_auxiliary_rings)) + if (enable_acl && !arm_le_data_rings) return -EINVAL; if (native_startup < -1 || native_startup > 1) return -EINVAL; @@ -1013,6 +1094,9 @@ static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) hdev->send = cen_hci_send; hdev->setup = cen_hci_setup; hdev->post_init = cen_hci_post_init; + hdev->reset_dup_filter = cen_hci_reset_dup_filter; + /* N1 reports the native PHY metadata bit (e.g. 0x81 for LE 1M). */ + hci_set_quirk(hdev, HCI_QUIRK_FIXUP_LE_EXT_ADV_REPORT_PHY); SET_HCIDEV_DEV(hdev, &pdev->dev); hci_set_drvdata(hdev, bt); pci_set_drvdata(pdev, bt); From a7ad78b4a26f436bdc59e8b8d6589eb48ee2f887 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 29 Sep 2026 03:30:37 -0400 Subject: [PATCH 0234/1579] Bluetooth: btcentauri: enable ACL and native connections by default on J714s J714s firmware 27 answers the standard HCI Create Connection with Command Disallowed, so outgoing BR/EDR connections (and pairing) only work through the native Proxima request that native_hrb translates to. With both ACL delivery and that translation behind opt-in parameters, Bluetooth could discover devices but not connect to them. Make enable_acl and native_hrb tristates like native_startup: -1 (the default) selects the board default, on for J714s; 0 and 1 still force them. The choice is resolved once at probe because the data path reads it per packet. A controller without the native capability now keeps working with standard Create Connection instead of failing post_init. Tested on J714s with no btcentauri boot parameters: a DualSense paired through GNOME, bound hid-playstation over uhid and streamed input over the native BR/EDR ACL rings; after a reboot it reconnected on its own using the stored link key. LE GATT transfers were qualified earlier on the same rings. No transport errors in either test. Co-Authored-By: Claude Opus 5.5 --- drivers/bluetooth/btcentauri.c | 66 ++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/drivers/bluetooth/btcentauri.c b/drivers/bluetooth/btcentauri.c index 8634b50d74661..820ba783806a5 100644 --- a/drivers/bluetooth/btcentauri.c +++ b/drivers/bluetooth/btcentauri.c @@ -69,13 +69,19 @@ static char *native_bdaddr; module_param(native_bdaddr, charp, 0400); MODULE_PARM_DESC(native_bdaddr, "Optional stable diagnostic address in XX:XX:XX:XX:XX:XX order; does not provision the factory address"); -static bool enable_acl; -module_param(enable_acl, bool, 0444); -MODULE_PARM_DESC(enable_acl, "Experimental ACL data delivery; requires LE rings and opens native ACL resources"); +static int enable_acl = -1; +module_param(enable_acl, int, 0444); +MODULE_PARM_DESC(enable_acl, "ACL data delivery: -1=board default, 0=off, 1=on; requires LE rings and opens native ACL resources"); -static bool native_hrb; -module_param(native_hrb, bool, 0444); -MODULE_PARM_DESC(native_hrb, "Experimental native Proxima Create Connection; requires native startup, ACL and firmware capability bit 9"); +/* J714s firmware27 rejects the standard HCI Create Connection with Command + * Disallowed; outgoing BR/EDR connections need the native request. + */ +static int native_hrb = -1; +module_param(native_hrb, int, 0444); +MODULE_PARM_DESC(native_hrb, "Native Proxima Create Connection: -1=board default, 0=off, 1=on; requires native startup, ACL and firmware capability bit 9"); + +/* Resolved once at probe; the data path reads these per packet. */ +static bool cen_acl, cen_hrb; static bool flush_duplicates = true; module_param(flush_duplicates, bool, 0644); @@ -305,7 +311,7 @@ static void cen_aux_rx_buffer(struct cen_bt *bt, unsigned int index, u16 slot) */ static bool cen_aux_required(u16 id) { - return arm_auxiliary_rings || (enable_acl && (id == 2 || id == 8)); + return arm_auxiliary_rings || (cen_acl && (id == 2 || id == 8)); } static int cen_aux_start(struct cen_bt *bt) @@ -365,7 +371,7 @@ static int cen_aux_drain(struct cen_bt *bt, bool repost) if (length > spec->footer) return -EPROTO; cen_capture_frame(bt, spec->id, descriptor + 16, length); - if (repost && enable_acl && spec->id == 8 && + if (repost && cen_acl && spec->id == 8 && cen_receive_acl(bt, descriptor + 16, length)) return -EPROTO; state->packets++; @@ -438,7 +444,7 @@ static int cen_start(struct cen_bt *bt) bt->le_rx_post = CEN_LE_SLOTS - 1; cen_post(bt, 10, bt->le_rx_post); } - if (arm_auxiliary_rings || enable_acl) + if (arm_auxiliary_rings || cen_acl) return cen_aux_start(bt); return 0; } @@ -546,11 +552,11 @@ static void cen_poll(struct work_struct *work) /* Preserve already completed diagnostics after abort, without posting * any more buffers or ringing doorbells to the failed firmware. */ - if ((arm_auxiliary_rings || enable_acl) && readl(bt->bar) == 2) + if ((arm_auxiliary_rings || cen_acl) && readl(bt->bar) == 2) cen_aux_drain(bt, false); goto protocol_error; } - if ((arm_auxiliary_rings || enable_acl) && cen_aux_drain(bt, true)) + if ((arm_auxiliary_rings || cen_acl) && cen_aux_drain(bt, true)) goto protocol_error; if (arm_le_data_rings) { tail = cen_tail(bt, 10); @@ -564,7 +570,7 @@ static void cen_poll(struct work_struct *work) goto protocol_error; bt->le_rx_packets++; cen_capture_frame(bt, 10, descriptor + 16, length); - if (enable_acl && cen_receive_acl(bt, descriptor + 16, length)) + if (cen_acl && cen_receive_acl(bt, descriptor + 16, length)) goto protocol_error; bt->le_rx_head = (bt->le_rx_head + 1) % CEN_LE_SLOTS; cen_le_rx_buffer(bt, bt->le_rx_post); @@ -628,7 +634,7 @@ static void cen_poll(struct work_struct *work) CEN_FOOTER, &bt->tx_head)) goto protocol_error; /* cen_aux_specs[0] is the native BR/EDR ACL TX resource (ring 2). */ - if (enable_acl && + if (cen_acl && (cen_transmit(bt, &bt->acl_txq, 2, cen_aux_specs[0].slots, bt->aux[0].offset, cen_aux_specs[0].footer, &bt->acl_tx_head) || @@ -719,7 +725,7 @@ static int cen_hci_post_init(struct hci_dev *hdev) u32 features; int ret; - if (!native_hrb) + if (!cen_hrb) return 0; WRITE_ONCE(bt->hrb_supported, false); skb = __hci_cmd_sync(hdev, CEN_OP_READ_PROP_FEATURES, 0, NULL, @@ -730,10 +736,13 @@ static int cen_hci_post_init(struct hci_dev *hdev) features = skb->len == 5 ? get_unaligned_le32(skb->data + 1) : 0; kfree_skb(skb); WRITE_ONCE(bt->proprietary_features, features); - if (ret) - return ret; - if (!(features & CEN_PROP_HRB_CREATE)) - return -EOPNOTSUPP; + if (!ret && !(features & CEN_PROP_HRB_CREATE)) + ret = -EOPNOTSUPP; + if (ret) { + /* Keep the controller usable; standard Create Connection remains. */ + bt_dev_warn(hdev, "Native Proxima Create Connection unavailable: %d", ret); + return 0; + } WRITE_ONCE(bt->hrb_supported, true); bt_dev_info(hdev, "Native Proxima Create Connection capability confirmed"); return 0; @@ -818,7 +827,7 @@ static int cen_hci_send(struct hci_dev *hdev, struct sk_buff *skb) u8 link_type; int ret = 0; - if (hci_skb_pkt_type(skb) == HCI_ACLDATA_PKT && enable_acl) { + if (hci_skb_pkt_type(skb) == HCI_ACLDATA_PKT && cen_acl) { if (skb->len < HCI_ACL_HDR_SIZE || skb->len != HCI_ACL_HDR_SIZE + get_unaligned_le16(skb->data + 2)) { ret = -EMSGSIZE; @@ -850,7 +859,7 @@ static int cen_hci_send(struct hci_dev *hdev, struct sk_buff *skb) ret = -EMSGSIZE; goto out; } - if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT && native_hrb) { + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT && cen_hrb) { u16 opcode = get_unaligned_le16(skb->data); /* Do not mix unmanaged native commands with translated requests. */ @@ -955,7 +964,7 @@ static int cen_status_show(struct seq_file *s, void *unused) seq_printf(s, "native_startup=%u native_startup_steps=%u\n", cen_native_startup_enabled(), READ_ONCE(bt->native_startup_steps)); seq_printf(s, "native_hrb=%u hrb_supported=%u proprietary_features=0x%08x hrb_commands=%u hrb_status_events=%u hrb_pending=%u\n", - native_hrb, READ_ONCE(bt->hrb_supported), + cen_hrb, READ_ONCE(bt->hrb_supported), READ_ONCE(bt->proprietary_features), bt->hrb_commands, bt->hrb_status_events, bt->hrb_status_pending); seq_printf(s, "hrb_connection_events=%u hrb_last_link_type=0x%02x\n", @@ -964,12 +973,12 @@ static int cen_status_show(struct seq_file *s, void *unused) READ_ONCE(flush_duplicates), READ_ONCE(bt->dup_flushes), READ_ONCE(bt->dup_flush_failed)); seq_printf(s, "acl_enabled=%u acl_tx=%u acl_rx=%u acl_queued=%u le_queued=%u\n", - enable_acl, bt->hdev->stat.acl_tx, bt->hdev->stat.acl_rx, + cen_acl, bt->hdev->stat.acl_tx, bt->hdev->stat.acl_rx, skb_queue_len(&bt->acl_txq), skb_queue_len(&bt->le_txq)); seq_printf(s, "auxiliary_rings=%u messages=%u message_head=%u capture_bytes=%u capture_records=%u capture_dropped=%u\n", arm_auxiliary_rings, bt->message_tag, bt->message_head, bt->capture_used, bt->capture_records, bt->capture_dropped); - if (arm_auxiliary_rings || enable_acl) + if (arm_auxiliary_rings || cen_acl) for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) if (bt->aux[i].opened && cen_aux_specs[i].receive) seq_printf(s, "aux_rx ring=%u packets=%u consumed=%u device_tail=%u\n", @@ -1039,11 +1048,14 @@ static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) skb_queue_head_init(&bt->txq); skb_queue_head_init(&bt->acl_txq); skb_queue_head_init(&bt->le_txq); - if (enable_acl && !arm_le_data_rings) + if (native_startup < -1 || native_startup > 1 || + enable_acl < -1 || enable_acl > 1 || native_hrb < -1 || native_hrb > 1) return -EINVAL; - if (native_startup < -1 || native_startup > 1) + cen_acl = enable_acl < 0 ? of_machine_is_compatible("apple,j714s") : enable_acl; + cen_hrb = native_hrb < 0 ? of_machine_is_compatible("apple,j714s") : native_hrb; + if (cen_acl && !arm_le_data_rings) return -EINVAL; - if (native_hrb && (!cen_native_startup_enabled() || !enable_acl)) + if (cen_hrb && (!cen_native_startup_enabled() || !cen_acl)) return -EINVAL; INIT_WORK(&bt->poll, cen_poll); ret = pcim_enable_device(pdev); @@ -1112,7 +1124,7 @@ static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) debugfs_create_file("capture_reset", 0200, bt->debug, bt, &cen_capture_reset_fops); } dev_info(&pdev->dev, "Beta HCI transport registered; ACL enabled=%u, SCO/ISO unsupported\n", - enable_acl); + cen_acl); return 0; stop: cen_stop(bt); From 766918eac6c1ae823861794f13eaa0e41bb9f3bd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 29 Sep 2026 04:38:40 -0400 Subject: [PATCH 0235/1579] mfd: macsmc: enable notifications and power/lid input on T6050 The T6050 match only created telemetry cells and never set NTAP, so the power/Touch ID button and lid switch did nothing on the M5 Pro. A probe on J714s (firmware 27) found the MSLD, bHLD and NTAP keys present; with NTAP set, a Touch ID press delivered event 0x72010601/0x72010600 and lid close/open 0x72030100/0x72030001: the same protocol macsmc-input already handles. Enable notifications for T6050 as on other SoCs and add the macsmc-input cell. Tested on J714s: logind received KEY_POWER and SW_LID from the new input device; the button is also the s2idle wake source. Co-Authored-By: Claude Opus 5.5 --- drivers/mfd/macsmc.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index a4ffdc5a21b2b..0c5e643900742 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -60,8 +60,13 @@ static const struct mfd_cell apple_smc_telemetry_devs[] = { MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), }; -/* T6050 additionally describes native pcIO outputs explicitly in its DT. */ +/* T6050 additionally describes native pcIO outputs explicitly in its DT. + * Its firmware 27 SMC keeps the classic notification protocol: with NTAP + * set, power/Touch ID button (0x7201) and lid (0x7203) events arrive as on + * earlier SoCs, backed by the same MSLD and bHLD keys. + */ static const struct mfd_cell apple_smc_t6050_devs[] = { + MFD_CELL_NAME("macsmc-input"), MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), MFD_CELL_OF("macsmc-pcio-gpio", NULL, NULL, 0, 0, "apple,smc-pcio-gpio"), }; @@ -536,12 +541,6 @@ static int apple_smc_probe(struct platform_device *pdev) /* T8152's key-read interface is qualified; notifications, GPIO and power * functions are not. The board DT exposes temperature and fan inputs only. */ - if (of_device_is_compatible(dev->of_node, "apple,t6050-smc")) - return devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, - apple_smc_t6050_devs, - ARRAY_SIZE(apple_smc_t6050_devs), - NULL, 0, NULL); - if (of_device_is_compatible(dev->of_node, "apple,t8152-smc")) return devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, apple_smc_telemetry_devs, @@ -554,6 +553,12 @@ static int apple_smc_probe(struct platform_device *pdev) if (ret) return ret; + if (of_device_is_compatible(dev->of_node, "apple,t6050-smc")) + return devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, + apple_smc_t6050_devs, + ARRAY_SIZE(apple_smc_t6050_devs), + NULL, 0, NULL); + ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, apple_smc_devs, ARRAY_SIZE(apple_smc_devs), NULL, 0, NULL); From 77592de10d11d9715c03e0307dced838b7c214c8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 29 Sep 2026 04:38:40 -0400 Subject: [PATCH 0236/1579] nvme: apple: park I/O instead of tearing down post-M4 ANS for s2idle Suspend ran the full disable plus apple_rtkit_shutdown(), whose quiesce/sleep handshake crashes post-M4 ANS firmware (see apple_nvme_shutdown()); s2idle then hung at "leaving controller enabled for RTKit shutdown" and never reached the idle loop. s2idle keeps the SoC powered, so for post-M4 controllers flush any deferred flush, freeze and quiesce the queues, and leave the controller and firmware running; resume restarts them. Firmware-assisted suspend keeps the existing teardown. Tested on J714s with pm_test "devices" and "platform": the root NVMe suspended and resumed with I/O intact. Co-Authored-By: Claude Opus 5.5 --- drivers/nvme/host/apple.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index d5ce6a3f1d5a6..9de0fb76b4797 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -243,6 +244,8 @@ struct apple_nvme { unsigned long last_flush; struct delayed_work flush_dwork; struct blk_crypto_profile crypto_profile; + /* I/O parked for s2idle with the controller and firmware left running. */ + bool s2idle_parked; }; /* Serialized by anv->lock during I/O; reset configures queues before use. @@ -2345,14 +2348,45 @@ static int apple_nvme_resume(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); + if (anv->s2idle_parked) { + anv->s2idle_parked = false; + nvme_unquiesce_admin_queue(&anv->ctrl); + nvme_unquiesce_io_queues(&anv->ctrl); + nvme_unfreeze(&anv->ctrl); + return 0; + } + return nvme_reset_ctrl(&anv->ctrl); } +/* + * Post-M4 ANS firmware crashes in the RTKit shutdown handshake (see + * apple_nvme_shutdown()), so suspend cannot tear it down. s2idle keeps the + * SoC powered: drain and park I/O with the controller and firmware left + * running, and let ANS idle on its own until resume restarts the queues. + */ +static int apple_nvme_park_for_s2idle(struct apple_nvme *anv) +{ + flush_delayed_work(&anv->flush_dwork); + nvme_start_freeze(&anv->ctrl); + if (nvme_wait_freeze_timeout(&anv->ctrl, NVME_IO_TIMEOUT) <= 0) { + nvme_unfreeze(&anv->ctrl); + return -EBUSY; + } + nvme_quiesce_io_queues(&anv->ctrl); + nvme_quiesce_admin_queue(&anv->ctrl); + anv->s2idle_parked = true; + return 0; +} + static int apple_nvme_suspend(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); int ret = 0; + if (anv->hw->needs_ioq_registers && !pm_suspend_via_firmware()) + return apple_nvme_park_for_s2idle(anv); + apple_nvme_disable(anv, true); if (apple_rtkit_is_running(anv->rtk)) { From 4f0e6e1ba4ca1e7a8e7a9d53b8e30349c94ed56b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 29 Sep 2026 04:38:40 -0400 Subject: [PATCH 0237/1579] wifi/bluetooth: centauri: keep the N1 functions alive across s2idle The N1 Wi-Fi, Alpha and Bluetooth PCI functions share one firmware. Without PM callbacks the PCI core's default suspend cleared bus mastering and memory decode on them (resume printed "enabling device (0400 -> 0402)"), which took the firmware to stage 3: Wi-Fi lost its transport mid-command and Bluetooth came back with a stopped transport. Give the Wi-Fi and Bluetooth drivers PM ops that save PCI state, so the core keeps them in D0, and give the driverless Alpha function (which carries Wi-Fi's MSI vectors) an empty PM domain from the Wi-Fi driver that owns it. cfg80211 and the Bluetooth core quiesce traffic themselves. Tested on J714s with pm_test: Wi-Fi reassociated on its own and SSH returned, and the Bluetooth transport stayed up with no errors. Co-Authored-By: Claude Opus 5.5 --- drivers/bluetooth/btcentauri.c | 18 ++++++++++++++++++ .../net/wireless/apple/centauri/centauri_ipc.c | 17 +++++++++++++++++ .../net/wireless/apple/centauri/centauri_pci.c | 18 ++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/drivers/bluetooth/btcentauri.c b/drivers/bluetooth/btcentauri.c index 820ba783806a5..f7a5a603e94ac 100644 --- a/drivers/bluetooth/btcentauri.c +++ b/drivers/bluetooth/btcentauri.c @@ -1144,6 +1144,23 @@ static void cen_remove(struct pci_dev *pdev) cen_stop(bt); } +/* + * Bluetooth shares the N1 firmware with Wi-Fi. Keep this function in D0 and + * enabled across s2idle (the PCI core's default suspend would clear its bus + * mastering); the Bluetooth core quiesces HCI traffic itself. + */ +static int cen_suspend(struct device *dev) +{ + return pci_save_state(to_pci_dev(dev)); +} + +static int cen_resume(struct device *dev) +{ + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(cen_pm_ops, cen_suspend, cen_resume); + static const struct pci_device_id cen_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1903) }, { } @@ -1155,6 +1172,7 @@ static struct pci_driver cen_driver = { .id_table = cen_ids, .probe = cen_probe, .remove = cen_remove, + .driver.pm = pm_sleep_ptr(&cen_pm_ops), }; module_pci_driver(cen_driver); MODULE_LICENSE("GPL"); diff --git a/drivers/net/wireless/apple/centauri/centauri_ipc.c b/drivers/net/wireless/apple/centauri/centauri_ipc.c index e03b87dcda9ae..b4c59c5ebfbd3 100644 --- a/drivers/net/wireless/apple/centauri/centauri_ipc.c +++ b/drivers/net/wireless/apple/centauri/centauri_ipc.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,7 @@ struct centauri_ipc { struct list_head buffers; struct pci_dev *function[3]; bool enabled[3]; + bool alpha_pm_domain; void __iomem *bar[3]; struct cen_context control, alpha; struct cen_buffer *secondary, *window, *cchi_rx, *cchi_tx, *debug_rx; @@ -340,6 +342,15 @@ static int memswap_handoff(struct centauri *c, unsigned int function, const char return 0; } +/* + * Function 1 carries the Alpha MSI vectors but has no driver of its own. + * PCI's default suspend of a driverless function clears its bus mastering + * and memory decode, which takes the shared N1 firmware down. This empty + * PM domain replaces those bus callbacks, so the function stays in D0 and + * enabled across s2idle, as the Wi-Fi and Bluetooth functions do. + */ +static struct dev_pm_domain centauri_alpha_pm_domain; + static int enumerate_functions(struct centauri *c) { struct centauri_ipc *ipc = c->ipc; @@ -366,6 +377,10 @@ static int enumerate_functions(struct centauri *c) return ret; ipc->enabled[i] = true; } + if (i == 1 && !ipc->function[i]->dev.driver) { + dev_pm_domain_set(&ipc->function[i]->dev, ¢auri_alpha_pm_domain); + ipc->alpha_pm_domain = true; + } ret = dma_set_mask_and_coherent(&ipc->function[i]->dev, DMA_BIT_MASK(40)); if (ret) return ret; @@ -521,6 +536,8 @@ void centauri_free_ipc(struct centauri *c) list_del(&buffer->list); kfree(buffer); } + if (ipc->alpha_pm_domain) + dev_pm_domain_set(&ipc->function[1]->dev, NULL); for (i = 0; i < 3; i++) { if (ipc->bar[i]) pci_iounmap(ipc->function[i], ipc->bar[i]); diff --git a/drivers/net/wireless/apple/centauri/centauri_pci.c b/drivers/net/wireless/apple/centauri/centauri_pci.c index 032dc04b88ce4..768daeeff4c2c 100644 --- a/drivers/net/wireless/apple/centauri/centauri_pci.c +++ b/drivers/net/wireless/apple/centauri/centauri_pci.c @@ -417,6 +417,23 @@ static void centauri_remove(struct pci_dev *pdev) c->debug = NULL; } +/* + * The N1 functions share one firmware, which must stay up across s2idle. + * Saving the PCI state here keeps this function in D0 and enabled instead of + * the PCI core's default suspend; cfg80211 has already left the network. + */ +static int centauri_suspend(struct device *dev) +{ + return pci_save_state(to_pci_dev(dev)); +} + +static int centauri_resume(struct device *dev) +{ + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(centauri_pm_ops, centauri_suspend, centauri_resume); + static const struct pci_device_id centauri_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1900), .driver_data = 0x8000 }, { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1901), .driver_data = 0x8000 }, @@ -430,6 +447,7 @@ static struct pci_driver centauri_driver = { .probe = centauri_probe, .remove = centauri_remove, .shutdown = centauri_remove, + .driver.pm = pm_sleep_ptr(¢auri_pm_ops), }; static int __init centauri_init(void) From 32a378df4f76f84fdc851f10258743b9cc1c0822 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 29 Sep 2026 06:28:10 -0400 Subject: [PATCH 0238/1579] nvme: apple: park post-M4 ANS for firmware-assisted suspend too PSCI SYSTEM_SUSPEND sets pm_suspend_via_firmware(), which sent post-M4 ANS down the full apple_nvme_disable() + RTKit shutdown path. That shutdown handshake crashes this firmware at any sleep depth, so park I/O with the controller and firmware running for every suspend, as s2idle already does. Tested on J714s: deep suspend through the x1n1 PSCI SYSTEM_SUSPEND path resumed with storage working. Co-Authored-By: Claude Opus 5.5 --- drivers/nvme/host/apple.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 9de0fb76b4797..6d999b76c6872 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -2384,7 +2384,8 @@ static int apple_nvme_suspend(struct device *dev) struct apple_nvme *anv = dev_get_drvdata(dev); int ret = 0; - if (anv->hw->needs_ioq_registers && !pm_suspend_via_firmware()) + /* Post-M4 ANS cannot take the RTKit shutdown at any sleep depth. */ + if (anv->hw->needs_ioq_registers) return apple_nvme_park_for_s2idle(anv); apple_nvme_disable(anv, true); From 4832ab22bb9080d82c29f0ca4ee686492f0552f8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 29 Sep 2026 07:09:24 -0400 Subject: [PATCH 0239/1579] rtc: add Apple SPMI PMU RTC with wake alarm The SPMI PMU keeps a free-running 48-bit counter and exposes an RTC alarm, which is the always-on wake source a sleeping SoC needs. Access everything through the PMU's NVMEM cells: - wall time in 1/32768 s is counter / 2 + the offset in PMU scratch (info-clock_offset). The counter runs at 65536 Hz (J714s: 65547 ticks/s against NTP); this is the convention of the offset macOS/iBoot leave there, which gives the correct date. The offset is read twice and must match, since one bad read put the clock 143 days ahead. - the alarm register holds whole counter seconds. Programming follows macOS 27's AppleDialogSPMIPMURTC::programRTCAlarm(): clear a pending event (W1C), write the alarm, unmask the alarm interrupt, then set the enable bit beside the monitor bit; register offsets and masks come from the ADT's info-rtc_* keys. - the alarm arrives on the PMU's first SPMI interrupt line (2 on J714s), which is also the wake IRQ. spmi: apple: mark child interrupts IRQCHIP_SKIP_SET_WAKE. The controller is a chained handler, which system suspend never disables, so a child wake request needs no controller action; without the flag enable_irq_wake() fails with -ENXIO. Tested on J714s: time correct at boot, an 8 s alarm fired on SPMI IRQ 2, and `rtcwake -m mem -s 20` woke deep (PSCI) suspend after ~21 s. Co-Authored-By: Claude Opus 5.5 --- drivers/rtc/Kconfig | 11 + drivers/rtc/Makefile | 1 + drivers/rtc/rtc-apple-pmu.c | 312 +++++++++++++++++++++++++++ drivers/spmi/spmi-apple-controller.c | 6 +- 4 files changed, 329 insertions(+), 1 deletion(-) create mode 100644 drivers/rtc/rtc-apple-pmu.c diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index 364afc73f8abd..603f03cf4bdeb 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -2100,6 +2100,17 @@ config RTC_DRV_WILCO_EC This can also be built as a module. If so, the module will be named "rtc_wilco_ec". +config RTC_DRV_APPLE_PMU + tristate "Apple SPMI PMU RTC and wake alarm" + depends on ARCH_APPLE || COMPILE_TEST + depends on NVMEM + help + RTC and wake alarm of the SPMI PMU on Apple Silicon Macs, accessed + through the PMU's NVMEM cells. Provides wakealarm for suspend. + + This driver can also be built as a module. If so, the module + will be called rtc-apple-pmu. + config RTC_DRV_MACSMC tristate "Apple Mac System Management Controller RTC" depends on MFD_MACSMC diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile index 6cf7e066314e1..729ff56c871d3 100644 --- a/drivers/rtc/Makefile +++ b/drivers/rtc/Makefile @@ -94,6 +94,7 @@ obj-$(CONFIG_RTC_DRV_M48T35) += rtc-m48t35.o obj-$(CONFIG_RTC_DRV_M48T59) += rtc-m48t59.o obj-$(CONFIG_RTC_DRV_M48T86) += rtc-m48t86.o obj-$(CONFIG_RTC_DRV_MA35D1) += rtc-ma35d1.o +obj-$(CONFIG_RTC_DRV_APPLE_PMU) += rtc-apple-pmu.o obj-$(CONFIG_RTC_DRV_MACSMC) += rtc-macsmc.o obj-$(CONFIG_RTC_DRV_MAX31335) += rtc-max31335.o obj-$(CONFIG_RTC_DRV_MAX6900) += rtc-max6900.o diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c new file mode 100644 index 0000000000000..a0aec58e30029 --- /dev/null +++ b/drivers/rtc/rtc-apple-pmu.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SPMI PMU RTC with wake alarm + * + * The PMU keeps a free-running 48-bit counter at 65536 Hz (measured on J714s: + * 65547 ticks/s against NTP). Wall time in 1/32768 s is the counter halved + * plus a signed offset in PMU scratch (info-clock_offset), the convention of + * the value macOS/iBoot leave there: it gave the correct date on J714s. The + * alarm register holds the counter's whole seconds (counter >> 16); an alarm + * of (counter >> 16) + 6 raised the event and SPMI interrupt 2 six seconds + * later. Programming follows macOS 27's + * AppleDialogSPMIPMURTC::programRTCAlarm(): clear a pending event, write the + * alarm, unmask the alarm interrupt, then set the enable bit next to the + * monitor bit in the alarm control register. All registers are PMU NVMEM + * cells, so every access sleeps. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RTC_BYTES 6 +#define RTC_BITS (8 * RTC_BYTES) +/* Wall time and offset are in 1/32768 s; the counter runs twice as fast. */ +#define RTC_SEC_SHIFT 15 + +/* Masks from the J714s ADT (pmu,abbey): info-rtc_alarm_{event,ctrl_en,monitor,}_mask. */ +#define ALARM_EVENT BIT(1) +#define ALARM_MONITOR BIT(0) +#define ALARM_ENABLE BIT(6) +#define ALARM_IRQ_MASK BIT(0) + +struct apple_pmu_rtc { + struct device *dev; + struct rtc_device *rtc; + struct nvmem_cell *counter, *offset, *alarm, *ctrl, *irq_mask; + struct mutex lock; +}; + +static int apple_pmu_rtc_read_cell(struct nvmem_cell *cell, void *val, size_t len) +{ + size_t got; + void *buf = nvmem_cell_read(cell, &got); + + if (IS_ERR(buf)) + return PTR_ERR(buf); + if (got < len) { + kfree(buf); + return -EIO; + } + memcpy(val, buf, len); + kfree(buf); + return 0; +} + +static int apple_pmu_rtc_write_u8(struct nvmem_cell *cell, u8 val) +{ + int ret = nvmem_cell_write(cell, &val, 1); + + return ret < 0 ? ret : 0; +} + +/* Wall time in 1/32768 s: counter / 2 + offset. */ +static int apple_pmu_rtc_ticks(struct apple_pmu_rtc *prtc, u64 *ticks, u64 *offset) +{ + u64 a = 0, b = 0, off = 0; + int ret, tries = 3; + + do { + a = b; + ret = apple_pmu_rtc_read_cell(prtc->counter, &b, RTC_BYTES); + if (ret) + return ret; + } while (a >> RTC_SEC_SHIFT != b >> RTC_SEC_SHIFT && --tries); + + /* A bad SPMI read here once put the clock 143 days ahead: insist on + * two identical reads of the offset. */ + for (tries = 3; tries; tries--) { + u64 check = 0; + + ret = apple_pmu_rtc_read_cell(prtc->offset, &off, RTC_BYTES); + if (!ret) + ret = apple_pmu_rtc_read_cell(prtc->offset, &check, RTC_BYTES); + if (ret) + return ret; + if (off == check) + break; + } + if (!tries) + return -EIO; + *ticks = (b >> 1) + off; + if (offset) + *offset = off; + return 0; +} + +static int apple_pmu_rtc_read_time(struct device *dev, struct rtc_time *tm) +{ + struct apple_pmu_rtc *prtc = dev_get_drvdata(dev); + u64 ticks; + int ret = apple_pmu_rtc_ticks(prtc, &ticks, NULL); + + if (ret) + return ret; + rtc_time64_to_tm(sign_extend64(ticks, RTC_BITS - 1) >> RTC_SEC_SHIFT, tm); + return 0; +} + +static int apple_pmu_rtc_set_time(struct device *dev, struct rtc_time *tm) +{ + struct apple_pmu_rtc *prtc = dev_get_drvdata(dev); + u64 ctr = 0, off; + int ret = apple_pmu_rtc_read_cell(prtc->counter, &ctr, RTC_BYTES); + + if (ret) + return ret; + /* The set second begins now. */ + off = (rtc_tm_to_time64(tm) << RTC_SEC_SHIFT) - (ctr >> 1); + ret = nvmem_cell_write(prtc->offset, &off, RTC_BYTES); + return ret < 0 ? ret : 0; +} + +/* Arm or disarm the programmed alarm; caller holds prtc->lock. */ +static int apple_pmu_rtc_program(struct apple_pmu_rtc *prtc, bool enable) +{ + u8 ctrl = 0; + int ret = apple_pmu_rtc_read_cell(prtc->ctrl, &ctrl, 1); + + if (ret) + return ret; + if (ctrl & ALARM_EVENT) { + ret = apple_pmu_rtc_write_u8(prtc->ctrl, ALARM_EVENT); + if (ret) + return ret; + } + ret = apple_pmu_rtc_write_u8(prtc->irq_mask, enable ? 0 : ALARM_IRQ_MASK); + if (ret) + return ret; + return apple_pmu_rtc_write_u8(prtc->ctrl, + ALARM_MONITOR | (enable ? ALARM_ENABLE : 0)); +} + +static int apple_pmu_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *wkalrm) +{ + struct apple_pmu_rtc *prtc = dev_get_drvdata(dev); + u64 ticks, off; + u32 alarm = 0; + u8 ctrl = 0; + int ret; + + mutex_lock(&prtc->lock); + ret = apple_pmu_rtc_ticks(prtc, &ticks, &off); + if (!ret) + ret = apple_pmu_rtc_read_cell(prtc->alarm, &alarm, sizeof(alarm)); + if (!ret) + ret = apple_pmu_rtc_read_cell(prtc->ctrl, &ctrl, 1); + mutex_unlock(&prtc->lock); + if (ret) + return ret; + + wkalrm->enabled = !!(ctrl & ALARM_ENABLE); + wkalrm->pending = !!(ctrl & ALARM_EVENT); + /* The alarm is in counter seconds; one counter second is 1 s. */ + ticks = ((u64)alarm << RTC_SEC_SHIFT) + off; + rtc_time64_to_tm(sign_extend64(ticks, RTC_BITS - 1) >> RTC_SEC_SHIFT, + &wkalrm->time); + return 0; +} + +static int apple_pmu_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *wkalrm) +{ + struct apple_pmu_rtc *prtc = dev_get_drvdata(dev); + u64 off = 0, ticks; + int ret; + + u32 alarm; + + mutex_lock(&prtc->lock); + ret = apple_pmu_rtc_read_cell(prtc->offset, &off, RTC_BYTES); + if (!ret) { + /* Counter seconds; round up so the alarm never fires early. */ + ticks = (rtc_tm_to_time64(&wkalrm->time) << RTC_SEC_SHIFT) - off; + ticks &= GENMASK_ULL(RTC_BITS - 1, 0); + alarm = DIV_ROUND_UP_ULL(ticks, 1ULL << RTC_SEC_SHIFT); + ret = nvmem_cell_write(prtc->alarm, &alarm, sizeof(alarm)); + ret = ret < 0 ? ret : 0; + } + if (!ret) + ret = apple_pmu_rtc_program(prtc, wkalrm->enabled); + mutex_unlock(&prtc->lock); + return ret; +} + +static int apple_pmu_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled) +{ + struct apple_pmu_rtc *prtc = dev_get_drvdata(dev); + int ret; + + mutex_lock(&prtc->lock); + ret = apple_pmu_rtc_program(prtc, enabled); + mutex_unlock(&prtc->lock); + return ret; +} + +static irqreturn_t apple_pmu_rtc_irq(int irq, void *data) +{ + struct apple_pmu_rtc *prtc = data; + u8 ctrl = 0; + + mutex_lock(&prtc->lock); + if (apple_pmu_rtc_read_cell(prtc->ctrl, &ctrl, 1) || !(ctrl & ALARM_EVENT)) { + mutex_unlock(&prtc->lock); + return IRQ_NONE; + } + /* One-shot, as the RTC core expects: clear the event and disarm. */ + apple_pmu_rtc_program(prtc, false); + mutex_unlock(&prtc->lock); + rtc_update_irq(prtc->rtc, 1, RTC_AF | RTC_IRQF); + return IRQ_HANDLED; +} + +static const struct rtc_class_ops apple_pmu_rtc_ops = { + .read_time = apple_pmu_rtc_read_time, + .set_time = apple_pmu_rtc_set_time, + .read_alarm = apple_pmu_rtc_read_alarm, + .set_alarm = apple_pmu_rtc_set_alarm, + .alarm_irq_enable = apple_pmu_rtc_alarm_irq_enable, +}; + +static int apple_pmu_rtc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_pmu_rtc *prtc; + int i, irq, nirq, ret; + + prtc = devm_kzalloc(dev, sizeof(*prtc), GFP_KERNEL); + if (!prtc) + return -ENOMEM; + prtc->dev = dev; + mutex_init(&prtc->lock); + platform_set_drvdata(pdev, prtc); + + struct { + struct nvmem_cell **cell; + const char *name; + } cells[] = { + { &prtc->counter, "counter" }, + { &prtc->offset, "rtc_offset" }, + { &prtc->alarm, "alarm" }, + { &prtc->ctrl, "alarm_ctrl" }, + { &prtc->irq_mask, "irq_mask" }, + }; + for (i = 0; i < ARRAY_SIZE(cells); i++) { + *cells[i].cell = devm_nvmem_cell_get(dev, cells[i].name); + if (IS_ERR(*cells[i].cell)) + return dev_err_probe(dev, PTR_ERR(*cells[i].cell), + "missing NVMEM cell %s\n", cells[i].name); + } + + prtc->rtc = devm_rtc_allocate_device(dev); + if (IS_ERR(prtc->rtc)) + return PTR_ERR(prtc->rtc); + prtc->rtc->ops = &apple_pmu_rtc_ops; + prtc->rtc->range_min = S64_MIN >> (RTC_SEC_SHIFT + (64 - RTC_BITS)); + prtc->rtc->range_max = S64_MAX >> (RTC_SEC_SHIFT + (64 - RTC_BITS)); + + /* The PMU's SPMI interrupt lines (ADT order). The alarm arrives on the + * first (line 2 on J714s); the handler checks the event bit either way. */ + nirq = platform_irq_count(pdev); + for (i = 0; i < nirq; i++) { + irq = platform_get_irq(pdev, i); + if (irq < 0) + return irq; + ret = devm_request_threaded_irq(dev, irq, NULL, apple_pmu_rtc_irq, + IRQF_ONESHOT | IRQF_SHARED, + dev_name(dev), prtc); + if (ret) + return dev_err_probe(dev, ret, "cannot request IRQ %d\n", irq); + if (i == 0) + dev_pm_set_wake_irq(dev, irq); + } + if (nirq > 0) + device_init_wakeup(dev, true); + else + clear_bit(RTC_FEATURE_ALARM, prtc->rtc->features); + + return devm_rtc_register_device(prtc->rtc); +} + +static const struct of_device_id apple_pmu_rtc_of_match[] = { + { .compatible = "apple,pmu-rtc" }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_pmu_rtc_of_match); + +static struct platform_driver apple_pmu_rtc_driver = { + .driver = { + .name = "apple-pmu-rtc", + .of_match_table = apple_pmu_rtc_of_match, + }, + .probe = apple_pmu_rtc_probe, +}; +module_platform_driver(apple_pmu_rtc_driver); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple SPMI PMU RTC with wake alarm"); diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index a56bb437c6edf..2bc6d7b7b5c8a 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -327,13 +327,17 @@ static int apple_spmi_irq_set_type(struct irq_data *d, unsigned int type) return type == IRQ_TYPE_EDGE_RISING ? 0 : -EINVAL; } +/* + * The parent line is a chained handler, which system suspend never disables, + * so a child wake request needs no controller-side action. + */ static struct irq_chip apple_spmi_irq_chip = { .name = "apple_spmi", .irq_mask = apple_spmi_irq_mask, .irq_unmask = apple_spmi_irq_unmask, .irq_ack = apple_spmi_irq_ack, .irq_set_type = apple_spmi_irq_set_type, - .flags = IRQCHIP_ONESHOT_SAFE, + .flags = IRQCHIP_ONESHOT_SAFE | IRQCHIP_SKIP_SET_WAKE, }; static int apple_spmi_irq_domain_map(struct irq_domain *irqd, From 759de46358cf22dd9f84c1e935bb2906914e3c32 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Tue, 29 Sep 2026 23:02:24 +0200 Subject: [PATCH 0240/1579] drm/apple: backlight: pair drm_modeset_acquire_init() with _fini() dcp_set_brightness() and drm_crtc_set_brightness() initialize a drm_modeset_acquire_ctx on the stack but never call drm_modeset_acquire_fini(), including on their -EDEADLK and -ERESTARTSYS returns. With lockdep the acquire context is tracked as a held lock until ww_acquire_fini(), so it stays on the task's held-lock stack after the function returns. When the backlight core then unlocks bd->ops_lock, lockdep walks into the stale entry: DEBUG_LOCKS_WARN_ON(1) WARNING: kernel/locking/lockdep.c:238 at __lock_acquire+0x938/0x1600 ... mutex_unlock backlight_device_set_brightness brightness_store Unable to handle kernel NULL pointer dereference at virtual address 00000000000000c4 This happens in systemd-backlight on every boot of a PROVE_LOCKING kernel. Call drm_modeset_acquire_fini() on every exit path. Tested on j416s (M2 Pro) with PROVE_LOCKING: the brightness restore at boot runs without a lockdep report. Fixes: c7f8e49c72a9 ("drm/apple: Relax locking for back light updates") Signed-off-by: Oliver Lukschander Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/apple/dcp_backlight.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 9eb0c7d4eb534..38109132dc317 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -148,8 +148,10 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) ret = drm_modeset_lock(&crtc->mutex, &ctx); if (ret == -EDEADLK) { drm_modeset_backoff(&ctx); + drm_modeset_acquire_fini(&ctx); return -EDEADLK; } else if (ret == -ERESTARTSYS) { + drm_modeset_acquire_fini(&ctx); return -ERESTARTSYS; } @@ -177,6 +179,7 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) drm_atomic_state_put(state); done: drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); return ret; } @@ -210,8 +213,10 @@ static int dcp_set_brightness(struct backlight_device *bd) ret = drm_modeset_lock(&dcp->crtc->base.mutex, &ctx); if (ret == -EDEADLK) { drm_modeset_backoff(&ctx); + drm_modeset_acquire_fini(&ctx); return -EDEADLK; } else if (ret == -ERESTARTSYS) { + drm_modeset_acquire_fini(&ctx); return -ERESTARTSYS; } @@ -219,6 +224,7 @@ static int dcp_set_brightness(struct backlight_device *bd) dcp->brightness.update = true; drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); return dcp_backlight_update(dcp); } From 98b25c21ad468fe989f9904afecd9f8144d4c92c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:06:27 -0700 Subject: [PATCH 0241/1579] soc: apple: sep: run the T6020 profile on the M2 Max (t6021) profile::detect() matched apple,t8103, apple,t6020, apple,t6000 and apple,t8140, so an M2 Max returned ENODEV. Module init discarded that error, which left the apple,sep node disabled with nothing probed and nothing logged -- indistinguishable from a driver that started cleanly. The M2 Max needs no constants of its own. t6020.dtsi is written as a cut-down t6021: it includes t6021.dtsi and disables the parts the smaller die lacks. Both parts pull in t602x-die0.dtsi, which places the SEP at 0x396400000, its DART at 0x394ac0000 and the same four mailbox interrupts on either die, and t602x-j414-j416.dtsi already carries the sep alias for j414c alongside j414s. So map apple,t6021 onto T6020. Bootstrap stays WarmRegister and dart_range_required stays true, both inherited from T6020, so the existing gate still applies the SEP/DART changeset on this platform. Also log the unmatched-SoC case instead of swallowing it, so the next unlisted SoC says so rather than failing silently. Co-Authored-By: Claude Opus 5 (1M context) --- drivers/soc/apple/profile.rs | 8 ++++++++ drivers/soc/apple/sep.rs | 14 +++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 039fd2c06cc50..9aa39f4736ec8 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -325,6 +325,14 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { if machine_has(b"apple,t6020") { return Ok(&T6020); } + // t6020.dtsi is defined as a cut-down t6021: it includes t6021.dtsi and + // disables the parts the smaller die lacks. Both pull in t602x-die0.dtsi, + // so the SEP, its DART and the SPI controller sit at the same addresses + // with the same interrupts on either part, and the T6020 constants apply + // to the M2 Max unchanged. + if machine_has(b"apple,t6021") { + return Ok(&T6020); + } if machine_has(b"apple,t6000") { return Ok(&T6000); } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 0a5de8faa4af7..37438f8407a06 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2245,10 +2245,18 @@ impl kernel::InPlaceModule for SepModule { // changeset; the cold-boot target describes them statically (its DART // is enabled before probe), so no changeset runs there. _dt: { - if let Ok(p) = profile::detect() { - if matches!(p.bootstrap, profile::Bootstrap::WarmRegister) { - dt::enable_sep_and_dart()?; + match profile::detect() { + Ok(p) => { + if matches!(p.bootstrap, profile::Bootstrap::WarmRegister) { + dt::enable_sep_and_dart()?; + } } + // Without this the module loads, probes nothing and logs + // nothing on an unlisted SoC, which is indistinguishable + // from a driver that started correctly. + Err(_) => pr_info!( + "apple_sep: no platform profile for this SoC; the SEP stays disabled\n" + ), } }, From ba8827e2a5eab53204d071ab348ca64496d4bfde Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Tue, 29 Sep 2026 16:41:26 -0400 Subject: [PATCH 0242/1579] arm64: dts: apple: t602x: add the Mesa fingerprint sensor on spi2 The Touch ID sensor sits on spi2, chip-select 0, on the T602x die as on T600x, but t602x-die0.dtsi had no node for it, so no M2 Pro/Max board could describe the sensor and the SEP driver's SPI side never bound. Add it under spi2 with the same constants as the t600x node: 8 MHz, SPI mode 2 (spi-cpol), compatible apple,mesa-fingerprint. It ships disabled; boards enable it with their own power and data-ready lines. Signed-off-by: Scott Jones Co-Authored-By: Claude Opus 5.5 (1M context) --- arch/arm64/boot/dts/apple/t602x-die0.dtsi | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi index 852b33a88b813..feaa651632fb3 100644 --- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi @@ -853,6 +853,14 @@ pinctrl-names = "default"; power-domains = <&ps_spi2>; status = "disabled"; + + mesa: fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpol; + status = "disabled"; + }; }; spi4: spi@39b110000 { From e69faf42d55b30210159d58468bcc10d694b6e21 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Tue, 29 Sep 2026 16:41:27 -0400 Subject: [PATCH 0243/1579] arm64: dts: apple: t6021-j414c: enable spi2 and the Mesa fingerprint sensor From the J414c platform device tree (/arm-io/spi2/mesa): the sensor is on spi2 chip-select 0 at 8 MHz, its power line is gpio0 pin 122 (function-mesa_pwr, active high) and its data-ready line is pin 121 on the same controller (phandle 163, gpio,t8101 at 0x39b028000, which is pinctrl_ap). These are the lines J414s uses. Describe them in the node so the driver takes the power GPIO and the data-ready interrupt from the device tree rather than the J414s-only fallback. Tested on a MacBook Pro 14" M2 Max (J414c), OS firmware 13.5, with the m1n1 SEP warm-preservation guard: the sensor binds at 8 MHz in mode 2 with interrupt-driven capture, fprintd enrolment completes and fprintd-verify reports verify-match. Signed-off-by: Scott Jones Co-Authored-By: Claude Opus 5.5 (1M context) --- arch/arm64/boot/dts/apple/t6021-j414c.dts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6021-j414c.dts b/arch/arm64/boot/dts/apple/t6021-j414c.dts index fab3b03ff3c45..20602b61d67eb 100644 --- a/arch/arm64/boot/dts/apple/t6021-j414c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j414c.dts @@ -45,3 +45,21 @@ &mtp_mt { firmware-name = "apple/tpmtfw-j414c.bin"; }; + +/* + * From the J414c platform device tree (/arm-io/spi2/mesa): the power line is + * gpio0 pin 122 (function-mesa_pwr, active high) and the data-ready line is + * pin 121, the same lines as J414s. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 122 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <121 IRQ_TYPE_EDGE_RISING>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; +}; From de1905ea4512e7f79ec34b0dda2a4ada8ae5f55a Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 12:26:34 -0600 Subject: [PATCH 0244/1579] drm/apple: accept an EDID longer than its base block announces dcpavserv_copy_edid() rejects an EDID unless the payload is exactly (1 + extension count) blocks. An HDMI 2.x sink that uses the HDMI Forum EDID Extension Override Data Block (HF-EEODB) sets the extension count in the base block to 1 and puts the real count in the HF-EEODB, so the DCP returns more blocks than the base block says. The EDID is dropped with -EIO, the connector has no EDID, and userspace sees no make or model. Accept any whole number of blocks at least as long as the base block announces and let drm_edid, which handles HF-EEODB, decide how many blocks to use. Also bound used_size by the buffer size, which the exact match used to do implicitly. Seen on j314s (M1 Pro) with a Dell S3225QS on the HDMI port: used_size 392, base block announces 1 extension (256 bytes), payload 384 bytes. With this change the connector gets the EDID. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/apple/epic/dpavservep.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 2de9d2fe4c24a..7e41aa917b373 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -181,11 +181,20 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_epic_service *service) // 16, 1, resp, 192, true); data_size = le64_to_cpu(resp->used_size); - if (data_size < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE) + if (data_size < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE || + data_size > EDID_BUF_SIZE) return ERR_PTR(-EIO); + /* + * An HDMI 2.x sink using the HF-EEODB announces fewer extension blocks + * in the base block than it sends, so the payload may be longer than + * the base block says. Accept any whole number of blocks at least that + * long and leave the block count to drm_edid, which knows about + * HF-EEODB. + */ num_blocks = resp->data[EDID_LEADING_DATA_SIZE + EDID_EXT_BLOCK_COUNT_OFFSET]; - if ((1 + num_blocks) * EDID_BLOCK_SIZE != data_size - EDID_LEADING_DATA_SIZE) + if ((1 + num_blocks) * EDID_BLOCK_SIZE > data_size - EDID_LEADING_DATA_SIZE || + (data_size - EDID_LEADING_DATA_SIZE) % EDID_BLOCK_SIZE) return ERR_PTR(-EIO); return drm_edid_alloc(resp->data + EDID_LEADING_DATA_SIZE, From 0daeb8ec3d63208ef4d8b9b2fa6b89ff817e37f4 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 16:48:11 +0200 Subject: [PATCH 0245/1579] drm/apple: route Thunderbolt DP tunnels through the Type-C crossbar Add core routing support for a USB4/Thunderbolt DisplayPort tunnel arriving behind a Thunderbolt hub, as opposed to a direct Type-C DP alt-mode connection. struct apple_dcp_typec_route grows a tunnel flag, an active_xbar pointer distinct from the route's own crossbar control, a tunnel_dpin index and an xbar_up flag, so dcp_typec_route_activate()/ deactivate() can be handed the crossbar control to actually use: for a tunnel this is a dpin0/dpin1 controller resolved at tunnel-creation time rather than the route's own dpphy control, and the mux selection for it is deferred until the tunnel pixel clock is confirmed running rather than issued immediately. Add apple_dcp_tb_dp_tunnel(), the entry point Thunderbolt calls once it has stood up a DP IN tunnel: it finds a free Type-C route on the target port and activates it as a tunnel. A Type-C-only pipeline (no fixed output of its own) never completes link training for a tunneled target on this hardware -- DCP firmware accepts request_display but never issues the follow-up apcall, despite byte-identical connect parameters, a firmware-internal decision on that coprocessor instance with no driver-source asymmetry to explain it. A pipeline with a fixed output of its own reliably reaches a full AUX/DPCD link, so route scoring prefers it unconditionally for a tunnel route on this hardware. Add dcp_tunnel_crossbar_up()/down()/set_rate()/dpin_activate(), the hooks the DPTX endpoint uses to keep the crossbar clocks and the DP IN adapter's active state in step with DCP's link (re)configuration, serialized by a new per-DCP tb_lock. Make the Type-C alt-mode mux callback leave a tunnel-owned port alone -- a tunnel is torn down by apple_dcp_tb_dp_tunnel() itself, not by an alt-mode notification, and reapplying a stale alt-mode state on top of one would fight it. Thread the DFP port selected by route activation through dcp_dptx_connect() into the DPTX validate/connect calls, which previously always targeted DFP port 0 -- this is what makes a tunneled connection train against the Thunderbolt DP IN adapter DCP was actually told to use, rather than the direct PHY. Widen DPTX_CONNECT_TIMEOUT from 2s to 8s and the Type-C HPD-assert wait to 8s, and drop DPTX_RECONNECT_RETRIES from 5 to 1: a tunneled connection's handshake round-trip through the DP IN adapter can outlast a direct-PHY connection's budget. Add dptxport_set_hpd_timeout() for that HPD-assert call, so a caller can widen the AFK reply timeout, since a tunnel's AUX proxying round-trip can be slower than a direct PHY's. [Chris Kearney: moved dptxport_set_hpd_timeout() here from the next commit, which added it after its first caller, so each commit builds] Co-Authored-By: Claude Sonnet 5 --- drivers/gpu/drm/apple/dcp-internal.h | 34 ++ drivers/gpu/drm/apple/dcp.c | 451 +++++++++++++++++++++++++-- drivers/gpu/drm/apple/dcp.h | 8 + drivers/gpu/drm/apple/dptxep.c | 18 +- drivers/gpu/drm/apple/dptxep.h | 3 + 5 files changed, 489 insertions(+), 25 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 3a7732dfbfd96..385be68bc9144 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -37,10 +37,26 @@ struct apple_dcp_typec_route { struct typec_mux_dev *typec_mux; u32 dptx_phy; u32 mux_index; + u32 typec_index; bool selected; + /* Ported from aurora-silicon/linux#8: this route currently feeds a + * Thunderbolt DP IN adapter rather than the crossbar's dpphy output. + */ + bool tunnel; + /* Which crossbar output is actually selected: route->xbar (dpphy) or + * a Thunderbolt dpin controller computed at tunnel-creation time. + */ + struct mux_control *active_xbar; + /* Which DP IN adapter (0/1) a tunnel on this route is bound to. */ + unsigned int tunnel_dpin; + /* Tunnel only: crossbar connection brought up (at + * DidChangeLinkConfiguration, once the tunnel pixel clock runs). + */ + bool xbar_up; }; bool dcp_is_typec_output(struct apple_dcp *dcp); +bool dcp_is_usb4_output(struct apple_dcp *dcp); struct dcpav_service_epic; @@ -289,6 +305,24 @@ struct apple_dcp { struct delayed_work typec_fabric_retrain_wq; u32 typec_reconnect_tries; + /* Ported from aurora-silicon/linux#8. */ + /* DPTX feeds a Thunderbolt DP IN adapter, not the Type-C PHY lanes. */ + bool dptx_tunnel; + /* DFP port in the DPTX target: 0 = dpphy, 1 = dpin0, 2 = dpin1. */ + u8 dptx_dfp_port; + /* Wakes/sleeps the Thunderbolt DP IN adapter from DCP Activate/ + * Deactivate. Set by apple_dcp_tb_dp_tunnel(), cleared by + * dcp_typec_route_deactivate(). + */ + int (*tb_dpin_set_active)(void *ctx, bool active); + void *tb_dpin_ctx; + /* Serializes the Thunderbolt DP IN callback and tunnel crossbar state + * between DCP apcalls and tunnel teardown; never held while waiting + * for DCP. + */ + struct mutex tb_lock; + bool tb_clock_ok; + struct gpio_desc *hdmi_hpd; struct gpio_desc *hdmi_pwren; struct gpio_desc *dp2hdmi_pwren; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 6f446342d1440..ed3603d96363c 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -9,27 +9,38 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include #include #include #include +#include #include #include +#include #include +#include +#include +#include +#include +#include #include #include #include #include +#include #include #include #include +#include #include #include #include @@ -48,6 +59,7 @@ #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) static bool show_notch; + module_param(show_notch, bool, 0644); MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); @@ -87,6 +99,11 @@ bool dcp_is_typec_output(struct apple_dcp *dcp) dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB; } +bool dcp_is_usb4_output(struct apple_dcp *dcp) +{ + return dcp->active_typec_route && dcp->active_typec_route->tunnel; +} + static bool dcp_typec_route_is_dp(const struct typec_mux_state *state) { return state->alt && state->alt->svid == USB_TYPEC_DP_SID && @@ -115,7 +132,8 @@ static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) !dcp_typec_route_fixed_output_busy(route); } -static int dcp_typec_route_activate(struct apple_dcp_typec_route *route); +static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, + struct mux_control *xbar); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); @@ -135,7 +153,8 @@ static unsigned int dcp_typec_route_score(struct apple_dcp_typec_route *route) return drm_crtc_index(&dcp->crtc->base); } -static int dcp_typec_route_activate(struct apple_dcp_typec_route *route) +static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, + struct mux_control *xbar) { struct apple_dcp *dcp = route->dcp; int ret; @@ -154,7 +173,13 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route) dcp->fixed_route_selected = false; } - ret = mux_control_select(route->xbar, route->mux_index); + /* + * Thunderbolt DP IN: the crossbar connection may only be brought up + * once DCP has configured the link and the tunnel pixel clock runs + * (see dcp_tunnel_crossbar_up()). Just remember the output here. + * Ported from aurora-silicon/linux#8. + */ + ret = xbar == route->xbar ? mux_control_select(xbar, route->mux_index) : 0; if (ret) { if (dcp->xbar) { int restore_ret; @@ -194,6 +219,17 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route) drm_crtc_mask(&dcp->crtc->base); } dcp->active_typec_route = route; + scoped_guard(mutex, &dcp->tb_lock) { + route->active_xbar = xbar; + route->tunnel = xbar != route->xbar; + route->xbar_up = !route->tunnel; + dcp->dptx_tunnel = route->tunnel; + /* Crossbar controls are dpphy, dpin0, dpin1: same order as + * the DFP port. */ + dcp->dptx_dfp_port = route->tunnel ? + xbar - &route->xbar->chip->mux[0] : 0; + dcp->tb_clock_ok = false; + } route->selected = true; dev_info(dcp->dev, "allocated Type-C DPTX PHY %u\n", route->dptx_phy); @@ -203,11 +239,38 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route) static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) { struct apple_dcp *dcp = route->dcp; - int ret; + int ret = 0; - ret = mux_control_deselect(route->xbar); - if (ret) - return ret; + /* + * Under tb_lock so a DCP link (re)configuration can neither select + * the crossbar nor restart the tunnel pixel clock behind our back. + * Ported from aurora-silicon/linux#8. + */ + scoped_guard(mutex, &dcp->tb_lock) { + if (!route->tunnel || route->xbar_up) + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + if (ret) + return ret; + route->xbar_up = false; + + if (route->tunnel && dcp->phy) { + /* The tunnel pixel clock must not outlive the tunnel. */ + typeof(&apple_atc_dp_tunnel_rate) stop = + symbol_get(apple_atc_dp_tunnel_rate); + + if (stop) { + stop(dcp->phy, 0); + symbol_put(apple_atc_dp_tunnel_rate); + } + } + route->active_xbar = NULL; + route->tunnel = false; + dcp->dptx_tunnel = false; + dcp->dptx_dfp_port = 0; + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + dcp->tb_clock_ok = false; + } route->selected = false; if (dcp->active_typec_route == route) @@ -295,6 +358,275 @@ static void dcp_typec_retrain_active_routes(void) } } +/* + * Thunderbolt DP tunnels, ported from aurora-silicon/linux#8. Crossbar + * connection up/down, looked up at runtime so appledrm does not require the + * crossbar driver to be built. + */ +static int dcp_dpxbar_link(struct mux_control *mux, bool up) +{ + typeof(&apple_dpxbar_link_up) fn; + int ret; + + fn = up ? symbol_get(apple_dpxbar_link_up) : + symbol_get(apple_dpxbar_link_down); + if (!fn) + return -ENOENT; + ret = fn(mux); + if (up) + symbol_put(apple_dpxbar_link_up); + else + symbol_put(apple_dpxbar_link_down); + return ret; +} + +/* DP IN adapter handshake through the thunderbolt glue; tb_lock held. */ +static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) +{ + lockdep_assert_held(&dcp->tb_lock); + if (!dcp->tb_dpin_set_active) + return -ENODEV; + return dcp->tb_dpin_set_active(dcp->tb_dpin_ctx, active); +} + +/* + * Thunderbolt DP IN, from DCP's DidChangeLinkConfiguration once a link rate + * is set: bring the crossbar connection up (FIFO/PCLK/ATC enables) now that + * the tunnel pixel clock runs, and re-assert the DP IN adapter's + * DPTX_INACTIVE=0 afterwards. The first time this is the mux selection; + * after a re-link only the clocks are brought back (the mux selection and + * the ATC output enable are kept, see dcp_tunnel_crossbar_down()). Runs + * inside a DCP apcall: only tb_lock, which nobody holds across a DCP call. + */ +int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp->tb_lock); + route = dcp->active_typec_route; + if (!route || !route->tunnel || !route->active_xbar) + return -ENODEV; + if (!dcp->tb_clock_ok) { + dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); + return -EIO; + } + if (!route->xbar_up) { + /* Never block a DCP call on the mux semaphore. */ + ret = mux_control_try_select(route->active_xbar, route->mux_index); + if (!ret) + route->xbar_up = true; + } else { + ret = dcp_dpxbar_link(route->active_xbar, true); + } + if (ret) { + dev_warn(dcp->dev, "DP tunnel crossbar up failed: %d\n", ret); + return ret; + } + return dcp_tunnel_dpin_locked(dcp, true); +} + +/* + * Thunderbolt DP IN, from DCP's WillChangeLinkConfiguration on an + * established link: DP IN inactive, crossbar clocks down (mux selection and + * ATC output enable kept). DP IN goes active again in + * dcp_tunnel_crossbar_up(). + */ +int dcp_tunnel_crossbar_down(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + + guard(mutex)(&dcp->tb_lock); + route = dcp->active_typec_route; + if (!route || !route->tunnel || !route->xbar_up) + return 0; + dcp_tunnel_dpin_locked(dcp, false); + return dcp_dpxbar_link(route->active_xbar, false); +} + +/* + * Thunderbolt DP IN, from DCP's SetLinkRate: start (rate != 0) or stop the + * tunnel pixel clock. A stopped clock also takes the crossbar connection + * down. + */ +int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) +{ + typeof(&apple_atc_dp_tunnel_rate) fn; + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp->tb_lock); + if (!dcp->dptx_tunnel) + return -ENODEV; + fn = symbol_get(apple_atc_dp_tunnel_rate); + if (!fn) { + dev_err(dcp->dev, "phy-apple-atc not loaded, no DP tunnel clock\n"); + return -ENOENT; + } + route = dcp->active_typec_route; + if (!link_rate && route && route->xbar_up) + dcp_dpxbar_link(route->active_xbar, false); + ret = fn(phy, link_rate); + symbol_put(apple_atc_dp_tunnel_rate); + dcp->tb_clock_ok = !ret && link_rate; + if (ret) + dev_warn(dcp->dev, "DP tunnel pixel clock (rate 0x%x) failed: %d\n", + link_rate, ret); + return ret; +} + +/* Thunderbolt DP IN: DCP Activate/Deactivate. */ +int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active) +{ + guard(mutex)(&dcp->tb_lock); + if (!dcp->dptx_tunnel) + return 0; + return dcp_tunnel_dpin_locked(dcp, active); +} + +/* + * Thunderbolt DP tunnels: the host router's DP IN adapters sit behind the + * crossbar's dpin0/dpin1 outputs of the port's ATC. When the Thunderbolt + * connection manager has set up a tunnel from one of them, route a free + * display pipeline there and tell DCP a display is attached, so it trains + * the link (and completes DPRX) through the tunnel. + */ +int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, + bool active, int (*set_active)(void *ctx, bool active), + void *ctx) +{ + struct apple_dcp_typec_port *port = NULL, *pos; + struct apple_dcp_typec_route *candidate, *best = NULL; + unsigned int best_score = UINT_MAX; + struct mux_control *ctl; + struct apple_dcp *dcp; + int ret; + + if (!connector_np || dpin > 1) + return -EINVAL; + + guard(mutex)(&dcp_typec_fabric_lock); + + list_for_each_entry(pos, &dcp_typec_ports, link) { + if (pos->connector_np == connector_np) { + port = pos; + break; + } + } + if (!port) + return -ENODEV; + + if (!active) { + if (!port->owner || !port->owner->tunnel || + port->owner->tunnel_dpin != dpin) + return 0; + dcp = port->owner->dcp; + if (port->hpd || dcp->typec_cable_connected) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + ret = dcp_typec_route_deactivate(port->owner); + if (ret) { + /* The caller's context is going away regardless. */ + scoped_guard(mutex, &dcp->tb_lock) { + dcp->dptx_tunnel = false; + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + return ret; + } + port->owner = NULL; + /* Re-apply the next Type-C mux state in full. */ + port->applied_valid = false; + if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_dptx_connect(dcp, 0); + return 0; + } + + if (port->owner) { + if (port->owner->tunnel && port->owner->tunnel_dpin == dpin) + return 0; + dev_warn(port->owner->dcp->dev, + "port already routed, not taking DP tunnel dpin%u\n", dpin); + return -EBUSY; + } + + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (!dcp_typec_route_available(candidate)) + continue; + score = dcp_typec_route_score(candidate); + /* + * j416s-specific, not in the reference (t8103 has a single + * dcpext): prefer a pipeline with a fixed output of its own + * (dcpext0) for a Type-C tunnel route. dcpext1 (Type-C only, + * no fixed output of its own) never completes link training + * for a tunneled target on this hardware: DCP firmware + * accepts request_display but never issues another apcall, + * on every attempt, even though the driver-issued connect + * parameters are byte-identical between the two pipelines. + * Nothing in this driver's source explains the difference, + * so this is a firmware-internal decision on the dcpext1 + * coprocessor instance, not something fixable here. dcpext0, + * forced onto the same physical port and tunnel, reaches a + * full AUX/DPCD link (DPRX_DONE=1) and a working picture, so + * it is preferred unconditionally for a tunnel route on this + * hardware. + */ + if (!candidate->dcp->fixed_phy) + score += 100; + if (score < best_score) { + best = candidate; + best_score = score; + } + } + if (!best) + return -EBUSY; + + /* The route's crossbar control is dpphy (0); dpin0/dpin1 are 1/2. */ + if (best->xbar != &best->xbar->chip->mux[0] || + best->xbar->chip->controllers < 3) + return -EOPNOTSUPP; + ctl = &best->xbar->chip->mux[1 + dpin]; + + dcp = best->dcp; + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = set_active; + dcp->tb_dpin_ctx = ctx; + } + best->tunnel_dpin = dpin; + ret = dcp_typec_route_activate(best, ctl); + if (ret) { + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + return ret; + } + port->owner = best; + + dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); + + /* + * The DP IN adapter may only be woken (DPTX_INACTIVE=0) while DCP + * drives the DPTX, i.e. from DCP's Activate call; waking it earlier + * hangs the machine (confirmed independently by this project's own + * AFK ordered-workqueue deadlock finding). dptxep calls set_active + * back from Activate and Deactivate (set above, before the route + * became a tunnel). + */ + if (!dcp->typec_connector) + dev_warn(dcp->dev, "no Type-C connector for the DP tunnel\n"); + WRITE_ONCE(dcp->typec_cable_connected, true); + port->hpd = true; + if (dcp->typec_connector) + dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); + static int dcp_typec_route_set(struct typec_mux_dev *mux, struct typec_mux_state *state) { @@ -331,7 +663,22 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, port->applied_conf = dp_conf; port->applied_valid = true; + dev_info(route->dcp->dev, + "typec mux set typec%u alt=%p svid=%04x mode=%lu dp=%d usb4=%d tbt=%d\n", + route->typec_index, state->alt, + state->alt ? state->alt->svid : 0, state->mode, is_dp, + state->mode == TYPEC_MODE_USB4, + state->alt && state->alt->svid == USB_TYPEC_TBT_SID); + if (!is_dp) { + /* + * A Thunderbolt/USB4 DP tunnel is torn down by its own path + * (apple_dcp_tb_dp_tunnel()), not by a Type-C mux notification. + * Ported from aurora-silicon/linux#8. + */ + if (port->owner && port->owner->tunnel) + return 0; + if (port->owner) { struct apple_dcp *dcp = port->owner->dcp; @@ -348,9 +695,18 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, gpiod_get_value_cansleep(dcp->hdmi_hpd)) dcp_dptx_connect(dcp, 0); } + dcp_typec_retrain_active_routes(); + return 0; + } - if (state->mode == TYPEC_MODE_USB4) - dcp_typec_retrain_active_routes(); + /* + * A Thunderbolt DP tunnel still owns the port (its teardown is on the + * way): don't act on or remember this state; the tunnel teardown + * drops what was recorded so the next update is applied. Ported from + * aurora-silicon/linux#8. + */ + if (port->owner && port->owner->tunnel) { + port->applied_valid = false; return 0; } @@ -370,7 +726,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (!best) return -EBUSY; - ret = dcp_typec_route_activate(best); + ret = dcp_typec_route_activate(best, best->xbar); if (ret) return ret; port->owner = best; @@ -588,6 +944,7 @@ static int dcp_register_typec_routes(struct apple_dcp *dcp) route = &dcp->typec_routes[dcp->nr_typec_routes]; route->dcp = dcp; + route->typec_index = route_index; INIT_LIST_HEAD(&route->port_link); route->phy = devm_phy_get(dev, name); if (IS_ERR(route->phy)) @@ -599,6 +956,13 @@ static int dcp_register_typec_routes(struct apple_dcp *dcp) return dev_err_probe(dev, PTR_ERR(route->xbar), "%pOF: failed to get display crossbar\n", route_np); + /* + * No per-route USB4/dpin crossbar lookup here any more: + * apple_dcp_tb_dp_tunnel() computes the dpin0/dpin1 crossbar + * controller dynamically from route->xbar->chip->mux[1+dpin] + * at tunnel-creation time. Ported from aurora-silicon/linux#8. + */ + ret = of_property_read_u32_index(dev->of_node, "apple,typec-mux-indices", route_index, &route->mux_index); if (ret) @@ -976,9 +1340,17 @@ bool dcp_has_typec_routes(struct platform_device *pdev) return dcp->nr_typec_routes; } -#define DPTX_CONNECT_TIMEOUT msecs_to_jiffies(2000) +/* + * DCP's tunnel handshake round-trip through the Thunderbolt DP IN + * adapter can take longer than a direct-PHY connection, so both the + * connect wait and the reconnect policy need more headroom than a + * direct connection does. An 8s connect timeout with a single + * reconnect retry is the tested, working configuration for a + * tunneled connection. + */ +#define DPTX_CONNECT_TIMEOUT msecs_to_jiffies(8000) #define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) -#define DPTX_RECONNECT_RETRIES 5 +#define DPTX_RECONNECT_RETRIES 1 static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { @@ -989,9 +1361,9 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) return -ENODEV; } dev_info(dcp->dev, - "%s(port=%d) target=%u:%u typec=%d route=%s conn_type=%d connected=%d\n", + "%s(port=%d) target=%u:%u dfp=%u typec=%d route=%s conn_type=%d connected=%d\n", __func__, port, dcp->dptx_die, dcp->dptx_phy, - dcp_is_typec_output(dcp), + dcp->dptx_dfp_port, dcp_is_typec_output(dcp), dcp->active_typec_route ? "borrowed" : "fixed", dcp->connector_type, dcp->dptxport[port].connected); @@ -1006,8 +1378,16 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) goto out_unlock; reinit_completion(&dcp->dptxport[port].linkcfg_completion); + /* + * Ported from aurora-silicon/linux#8: a Thunderbolt DP tunnel uses + * the exact same connect path as a direct alt-mode PHY. The only + * differences are dcp->dptx_dfp_port (0 = dpphy, 1/2 = dpin0/dpin1, + * set by dcp_typec_route_activate()) and dcp->phy already being the + * route's own ATC PHY rather than a separate shared lpdptxphy. + */ dcp->dptxport[port].atcphy = dcp->phy; - ret = dptxport_validate_connection(dcp->dptxport[port].service, 0, + ret = dptxport_validate_connection(dcp->dptxport[port].service, + dcp->dptx_dfp_port, dcp->dptx_phy, dcp->dptx_die); if (ret) { dev_err(dcp->dev, @@ -1016,9 +1396,10 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) goto out_unlock; } - ret = dptxport_connect(dcp->dptxport[port].service, 0, + ret = dptxport_connect(dcp->dptxport[port].service, + dcp->dptx_dfp_port, dcp->dptx_phy, dcp->dptx_die, - dcp_is_typec_output(dcp)); + dcp_is_typec_output(dcp)); if (ret) { dev_err(dcp->dev, "dcp_dptx_connect: failed to connect DPTX target %u:%u: %d\n", @@ -1035,7 +1416,17 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) } dcp->dptxport[port].connected = true; if (dcp_is_typec_output(dcp)) { - ret = dptxport_set_hpd(dcp->dptxport[port].service, true); + /* + * DCP's reply to this call can take longer than the normal + * 1000ms budget when the connection is a Thunderbolt DP + * tunnel, so widen the timeout to 8000ms for a Type-C/ + * tunneled output. This is independent of + * tb_dp_wait_dprx()'s own 12000ms poll for DPRX (generic + * thunderbolt/tunnel.c, started at tunnel-up), which is + * unaffected by this call either way. + */ + ret = dptxport_set_hpd_timeout(dcp->dptxport[port].service, + true, 8000); if (ret) { dev_err(dcp->dev, "dcp_dptx_connect: failed to assert Type-C HPD: %d\n", @@ -1046,8 +1437,9 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) } mutex_unlock(&dcp->hpd_mutex); + ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, - DPTX_CONNECT_TIMEOUT); + DPTX_CONNECT_TIMEOUT); if (!ret) { dev_err(dcp->dev, "dcp_dptx_connect: timed out waiting for port %u link configuration\n", @@ -1089,7 +1481,6 @@ static void dcp_typec_reconnect_work(struct work_struct *work) if (!READ_ONCE(dcp->typec_cable_connected)) return; - ret = dcp_dptx_connect(dcp, 0); if (!ret) { dcp->typec_reconnect_tries = 0; @@ -1776,7 +2167,6 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (dcp->index || dcp->dptx_phy || dcp->dptx_die) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); - mutex_init(&dcp->hpd_mutex); if (!show_notch) ret = of_property_read_u32(dev->of_node, "apple,notch-height", @@ -1966,6 +2356,14 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp_typec_reconnect_work); INIT_DELAYED_WORK(&dcp->typec_fabric_retrain_wq, dcp_typec_retrain_work); + /* + * Type-C and Thunderbolt routes can be activated as soon as they are + * registered below (dcp_register_typec_routes()), before the DRM + * device binds and dcp_comp_bind() would otherwise init this. Ported + * from aurora-silicon/linux#8. + */ + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->tb_lock); platform_set_drvdata(pdev, dcp); @@ -2105,6 +2503,13 @@ static void dcp_platform_remove(struct platform_device *pdev) static void dcp_platform_shutdown(struct platform_device *pdev) { + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp) { + WRITE_ONCE(dcp->typec_cable_connected, false); + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + } component_del(&pdev->dev, &dcp_comp_ops); } @@ -2112,6 +2517,10 @@ static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + WRITE_ONCE(dcp->typec_cable_connected, false); + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + if (dcp->avep) av_service_disconnect(dcp); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index c500c1b8a4434..2f044c8e2abcb 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -69,6 +69,14 @@ void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); +/* Thunderbolt DP tunnels, called from the DPTX endpoint (dptxep.c). Ported + * from aurora-silicon/linux#8. + */ +int dcp_tunnel_crossbar_up(struct apple_dcp *dcp); +int dcp_tunnel_crossbar_down(struct apple_dcp *dcp); +int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate); +int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active); + int iomfb_start_rtkit(struct apple_dcp *dcp); void iomfb_shutdown(struct apple_dcp *dcp); /* rtkit message handler for IOMFB messages */ diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 525ba63744370..6ce49d64dc4e0 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -141,7 +141,8 @@ int dptxport_release_display(struct apple_epic_service *service) return afk_service_call(service, 0, 7, NULL, 0, 16, NULL, 0, 16); } -int dptxport_set_hpd(struct apple_epic_service *service, bool hpd) +int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, + unsigned int timeout_ms) { struct dcpdptx_hotplug_cmd cmd, resp; int ret; @@ -151,15 +152,24 @@ int dptxport_set_hpd(struct apple_epic_service *service, bool hpd) if (hpd) cmd.unk = cpu_to_le32(1); - ret = afk_service_call(service, 8, 8, &cmd, sizeof(cmd), 12, &resp, - sizeof(resp), 12); + ret = afk_service_call_timeout(service, 8, 8, &cmd, sizeof(cmd), 12, + &resp, sizeof(resp), 12, timeout_ms); if (ret) return ret; - if (le32_to_cpu(resp.unk) != hpd) + if (le32_to_cpu(resp.unk) != hpd) { + dev_warn(service->ep->dcp->dev, + "set_hpd: unk reply 0x%x (sent hpd=%d)\n", + le32_to_cpu(resp.unk), hpd); return -EINVAL; + } return 0; } +int dptxport_set_hpd(struct apple_epic_service *service, bool hpd) +{ + return dptxport_set_hpd_timeout(service, hpd, MSEC_PER_SEC); +} + static int dptxport_call_get_max_drive_settings(struct apple_epic_service *service, void *reply_, size_t reply_size) diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index c8816a040f321..47539fd4bcc71 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -68,4 +68,7 @@ int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, int dptxport_request_display(struct apple_epic_service *service); int dptxport_release_display(struct apple_epic_service *service); int dptxport_set_hpd(struct apple_epic_service *service, bool hpd); +int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, + unsigned int timeout_ms); + #endif From e557722c435b0a17c0f2381212113f23060c2202 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 16:48:24 +0200 Subject: [PATCH 0246/1579] drm/apple: teach the DPTX endpoint about USB4 tunnel routes A Thunderbolt/USB4 DisplayPort tunnel reaches the same DCP DPTX AFK/EPIC endpoint as a direct HDMI or USB-C alt-mode PHY, but several request/reply shapes and side effects differ: the endpoint has no local ATC DisplayPort PHY to configure, the pixel clock and display crossbar connection are owned by the Thunderbolt tunnel infrastructure instead of being driven directly, and firmware's link-training sequencing includes calls (INACTIVE_SINK_DETECTED, FORCE_HOTPLUG_DETECT, GET_ACTIVE_LANE_COUNT) that the direct-PHY path never exercises. Factor VALIDATE_CONNECTION/CONNECT's remote-port target computation into dptxport_remote_target(), and add a tunnel "role" attribute bit that lets firmware distinguish a Thunderbolt DP-IN route from a direct PHY; tolerate the resulting difference in firmware's attrs reply only for a tunnel route, still rejecting a mismatch for the direct-PHY case. Treat a NULL dptx->atcphy (the USB4 DP-IN service has no ATC PHY of its own) as "report a fixed 4 lanes, skip phy_configure()" in GET_MAX_LANE_COUNT/SET_ACTIVE_LANE_COUNT. Route SET_LINK_RATE, ACTIVATE and DEACTIVATE through the dp-tunnel.h API for a tunnel connection instead of touching the ATC PHY's DP mode or rate registers directly -- the PHY must stay in USB4/Thunderbolt mode for the life of the tunnel. Bring the tunnel's display crossbar connection down before a re-link and back up once it completes, and handle GET_ACTIVE_LANE_COUNT, FORCE_HOTPLUG_DETECT and INACTIVE_SINK_DETECTED explicitly rather than relying on the generic unhandled-call ACK. dcp_hotplug()'s forced link-status retrain recovers a connector that reports connected without a valid mode set yet, for a Type-C route switch that can drop the active mode without changing HPD state. A USB4 tunnel connector's link comes up through the DPTX endpoint's own crossbar/clock sequencing rather than a Type-C route switch, so restrict that recovery path to non-tunnel connectors. Trace the read_edt_data and property-dictionary DCP callbacks' key, size and result for any connector other than the fixed eDP panel, and include the firmware-reported mode count in the existing hotplug log line, to make the boot-property and mode-list exchange with firmware observable for a tunnel connector. [Chris Kearney: dptxport_set_hpd_timeout() moved to the previous commit, its first caller] Co-Authored-By: Claude Sonnet 5 --- drivers/gpu/drm/apple/dptxep.c | 269 +++++++++++++++++++++---- drivers/gpu/drm/apple/dptxep.h | 7 + drivers/gpu/drm/apple/iomfb.c | 3 +- drivers/gpu/drm/apple/iomfb_template.c | 24 ++- 4 files changed, 265 insertions(+), 38 deletions(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 6ce49d64dc4e0..11091084f05e2 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -3,8 +3,13 @@ #include #include +#include #include #include +#include +#include + +#include #include "afk.h" #include "dcp.h" @@ -73,30 +78,79 @@ struct dptxport_apcall_set_tiled { __le32 retcode; }; +/* + * Ported from aurora-silicon/linux#8: a Thunderbolt DP tunnel uses the same + * plain CORE|ATC|DIE|CONNECTED target as a direct alt-mode PHY. CORE is the + * DFP port (0 = dpphy, 1/2 = dpin0/dpin1, dcp->dptx_dfp_port), ATC is the + * route's own ATC index -- no separate DPIN field. + */ +static u32 dptxport_remote_target(struct apple_dcp *dcp, u8 core, u8 atc, + u8 die) +{ + return FIELD_PREP(DCPDPTX_REMOTE_PORT_CORE, core) | + FIELD_PREP(DCPDPTX_REMOTE_PORT_ATC, atc) | + FIELD_PREP(DCPDPTX_REMOTE_PORT_DIE, die) | + DCPDPTX_REMOTE_PORT_CONNECTED; +} + int dptxport_validate_connection(struct apple_epic_service *service, u8 core, u8 atc, u8 die) { struct dptx_port *dptx = service->cookie; struct dcpdptx_connection_cmd cmd, resp; int ret; - u32 target = FIELD_PREP(DCPDPTX_REMOTE_PORT_CORE, core) | - FIELD_PREP(DCPDPTX_REMOTE_PORT_ATC, atc) | - FIELD_PREP(DCPDPTX_REMOTE_PORT_DIE, die) | - DCPDPTX_REMOTE_PORT_CONNECTED; + u32 target = dptxport_remote_target(service->ep->dcp, core, atc, die); + + /* + * attributes: role (0 = direct PHY, 1 = Thunderbolt/USB4 DP IN) | + * supportsHPD << 8. A real, hardware-validated reference + * implementation of the equivalent Thunderbolt DP tunnel mechanism + * on a different SoC (aurora-silicon/linux#8) adds this exact role + * bit for its tunnel routes; our own analog-DPIN path has never set + * it, always sending a plain direct-PHY attributes value even + * though this is a genuinely USB4-tunneled connection. DCP firmware + * appears to treat a failed AUX probe as fatal for role=0 but keeps + * training regardless for role=1 (a Thunderbolt tunnel's AUX + * proxying may not respond as fast as a direct PHY's): repeated + * test runs with substantially different Activate/crossbar ordering + * all reached the identical INACTIVE_SINK_DETECTED/DPRX-timeout + * outcome, which points at a signal we are not sending at all + * rather than a timing issue fixable by reordering. + */ + u32 attrs = 0x100 | (dcp_is_usb4_output(service->ep->dcp) ? 1 : 0); trace_dptxport_validate_connection(dptx, core, atc, die); + dptx->validate_calls++; + dev_dbg(service->ep->dcp->dev, + "DPTX validate: call #%u this boot target=0x%x core=%u atc=%u die=%u attrs=0x%x caller=%pS\n", + dptx->validate_calls, target, core, atc, die, + attrs, __builtin_return_address(0)); cmd.target = cpu_to_le32(target); - cmd.unk = cpu_to_le32(0x100); + cmd.unk = cpu_to_le32(attrs); ret = afk_service_call(service, 0, 12, &cmd, sizeof(cmd), 40, &resp, sizeof(resp), 40); if (ret) return ret; - if (le32_to_cpu(resp.target) != target) - return -EINVAL; - if (le32_to_cpu(resp.unk) != 0x100) + if (le32_to_cpu(resp.target) != target) { + dev_warn(service->ep->dcp->dev, + "validate_connection: target reply 0x%x (sent 0x%x)\n", + le32_to_cpu(resp.target), target); return -EINVAL; + } + if (le32_to_cpu(resp.unk) != attrs) { + /* only the Thunderbolt DP IN role is allowed to differ */ + if (!dcp_is_usb4_output(service->ep->dcp)) { + dev_warn(service->ep->dcp->dev, + "validate_connection: attrs reply 0x%x (sent 0x%x), rejecting\n", + le32_to_cpu(resp.unk), attrs); + return -EINVAL; + } + dev_info(service->ep->dcp->dev, + "validate_connection: attrs reply 0x%x (sent 0x%x)\n", + le32_to_cpu(resp.unk), attrs); + } return 0; } @@ -106,14 +160,18 @@ int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, { struct dptx_port *dptx = service->cookie; struct dcpdptx_connection_cmd cmd, resp; - u32 unk_field = supports_hpd ? DCPDPTX_REMOTE_PORT_SUPPORTS_HPD : 0; + /* same role bit as dptxport_validate_connection() above */ + u32 unk_field = (supports_hpd ? DCPDPTX_REMOTE_PORT_SUPPORTS_HPD : 0) | + (dcp_is_usb4_output(service->ep->dcp) ? 1 : 0); int ret; - u32 target = FIELD_PREP(DCPDPTX_REMOTE_PORT_CORE, core) | - FIELD_PREP(DCPDPTX_REMOTE_PORT_ATC, atc) | - FIELD_PREP(DCPDPTX_REMOTE_PORT_DIE, die) | - DCPDPTX_REMOTE_PORT_CONNECTED; + u32 target = dptxport_remote_target(service->ep->dcp, core, atc, die); trace_dptxport_connect(dptx, core, atc, die); + dptx->connect_calls++; + dev_dbg(service->ep->dcp->dev, + "DPTX connect: call #%u this boot target=0x%x unk=0x%x caller=%pS\n", + dptx->connect_calls, target, unk_field, + __builtin_return_address(0)); cmd.target = cpu_to_le32(target); cmd.unk = cpu_to_le32(unk_field); @@ -122,8 +180,12 @@ int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, if (ret) return ret; - if (le32_to_cpu(resp.target) != target) + if (le32_to_cpu(resp.target) != target) { + dev_warn(service->ep->dcp->dev, + "connect: target reply 0x%x (sent 0x%x)\n", + le32_to_cpu(resp.target), target); return -EINVAL; + } if (le32_to_cpu(resp.unk) != unk_field) dev_notice(service->ep->dcp->dev, "unexpected unk field in reply: 0x%x (0x%x)\n", le32_to_cpu(resp.unk), unk_field); @@ -133,12 +195,34 @@ int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, int dptxport_request_display(struct apple_epic_service *service) { - return afk_service_call(service, 0, 6, NULL, 0, 16, NULL, 0, 16); + struct dptx_port *dptx = service->cookie; + int ret; + + dptx->request_calls++; + dev_dbg(service->ep->dcp->dev, + "DPTX request_display: call #%u this boot caller=%pS\n", + dptx->request_calls, __builtin_return_address(0)); + ret = afk_service_call(service, 0, 6, NULL, 0, 16, NULL, 0, 16); + dev_dbg(service->ep->dcp->dev, + "DPTX request_display: call #%u result=%d\n", + dptx->request_calls, ret); + return ret; } int dptxport_release_display(struct apple_epic_service *service) { - return afk_service_call(service, 0, 7, NULL, 0, 16, NULL, 0, 16); + struct dptx_port *dptx = service->cookie; + int ret; + + dptx->release_calls++; + dev_dbg(service->ep->dcp->dev, + "DPTX release_display: call #%u this boot caller=%pS\n", + dptx->release_calls, __builtin_return_address(0)); + ret = afk_service_call(service, 0, 7, NULL, 0, 16, NULL, 0, 16); + dev_dbg(service->ep->dcp->dev, + "DPTX release_display: call #%u result=%d\n", + dptx->release_calls, ret); + return ret; } int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, @@ -271,6 +355,15 @@ static int dptxport_call_get_max_lane_count(struct apple_epic_service *service, if (reply_size < sizeof(*reply)) return -EINVAL; + if (!dptx->atcphy) { + /* USB4 DP IN: no ATC DP PHY to validate. */ + dptx->lane_count = 4; + reply->retcode = cpu_to_le32(0); + reply->lane_count = cpu_to_le64(4); + dev_info(dcp->dev, "get_max_lane_count: USB4 DP IN, 4 lanes\n"); + return 0; + } + ret = phy_validate(dptx->atcphy, PHY_MODE_DP, 0, &phy_ops); if (ret < 0) { dev_err(dcp->dev, "phy_validate failed: %d\n", ret); @@ -320,9 +413,9 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic case 0 ... 2: case 4: dptx->phy_ops.dp.lanes = lane_count; - // Use dptx phy index > 3 as indication for dptx-phy or - // lpdptx-phy and configure the number of lanes for those - dptx->phy_ops.dp.set_lanes = (dcp->dptx_phy > 3); + /* USB4 DP IN has no ATC PHY; still accept the lane count. */ + dptx->phy_ops.dp.set_lanes = + dcp_is_usb4_output(dcp) || (dcp->dptx_phy > 3); break; default: dev_err(dcp->dev, "set_active_lane_count: invalid lane count:%llu\n", lane_count); @@ -332,7 +425,7 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic } if (dptx->phy_ops.dp.set_lanes) { - if (dptx->atcphy) { + if (dptx->atcphy && !dcp_is_usb4_output(dcp)) { ret = phy_configure(dptx->atcphy, &dptx->phy_ops); if (ret) return ret; @@ -344,8 +437,30 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic reply->retcode = cpu_to_le32(retcode); reply->lane_count = cpu_to_le64(lane_count); - if (lane_count > 0) + if (lane_count > 0) { + dev_info(dcp->dev, "USB4/DPTX: SET_ACTIVE_LANE_COUNT %llu\n", + lane_count); + /* + * dcp_usb4_drm_allowed() (usb4_force_dptx) used to gate this + * for the old manual-training sysfs knob (module_param_cb + * usb4_dptx_train), removed wholesale in 0dc9f50 when + * apple_dcp_tb_dp_tunnel() replaced it with automatic tunnel + * detection -- but usb4_force_dptx itself, and this gate, + * were left behind with no way left to ever set it true. + * dcp_dptx_connect()'s single wait_for_completion_timeout() + * on linkcfg_completion is now shared by both the alt-mode + * and USB4-tunnel paths (also part of 0dc9f50's unification), + * so gating it here meant every tunneled connect timed out + * after a fully successful DPRX/AUX handshake and lane + * negotiation -- firmware does not send FORCE_HOTPLUG_DETECT + * (the only other completion site) for a tunnel. The + * reference (aurora-silicon/linux#8) completes + * linkcfg_completion here unconditionally; usb4_lane_completion + * has no consumer anywhere in this tree, so this drops both the + * dead gate and the dead completion and matches the reference. + */ complete(&dptx->linkcfg_completion); + } return ret; } @@ -380,6 +495,14 @@ dptxport_call_will_change_link_config(struct apple_epic_service *service) static int dptxport_call_did_change_link_config(struct apple_epic_service *service) { + /* + * Ported from aurora-silicon/linux#8: bringing the tunnel crossbar + * connection up (dcp_tunnel_crossbar_up()) now happens in + * dptxport_call(), after this succeeds, gated on + * dptx_tunnel && link_rate -- not inside this handler, and its + * result is never propagated back to DCP as a call failure. + */ + /* assume the link config did change and wait a little bit */ mdelay(10); @@ -404,6 +527,8 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, link_rate = le32_to_cpu(request->link_rate); trace_dptxport_call_set_link_rate(dptx, link_rate); + dev_info(service->ep->dcp->dev, "DPTXPort: SET_LINK_RATE 0x%x\n", + link_rate); switch (link_rate) { case LINK_RATE_RBR: @@ -436,18 +561,25 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, } if (phy_set_rate) { - dptx->phy_ops.dp.link_rate = phy_link_rate; - dptx->phy_ops.dp.set_rate = 1; - - if (dptx->atcphy) { + /* + * Ported from aurora-silicon/linux#8: a Thunderbolt DP tunnel + * starts/stops its own pixel clock instead of configuring the + * PHY directly, and never fails the apcall over it -- a + * failed clock just keeps the crossbar connection down + * (dcp->tb_clock_ok), which dcp_tunnel_crossbar_up() checks. + */ + if (dptx->atcphy && service->ep->dcp->dptx_tunnel) { + dcp_tunnel_set_rate(service->ep->dcp, dptx->atcphy, + link_rate); + } else if (dptx->atcphy) { + dptx->phy_ops.dp.link_rate = phy_link_rate; + dptx->phy_ops.dp.set_rate = 1; ret = phy_configure(dptx->atcphy, &dptx->phy_ops); if (ret) return ret; } - //if (dptx->phy_ops.dp.set_rate) dptx->link_rate = dptx->pending_link_rate = link_rate; - } //dptx->pending_link_rate = link_rate; @@ -467,7 +599,15 @@ static int dptxport_call_get_supports_hpd(struct apple_epic_service *service, return -EINVAL; reply->retcode = cpu_to_le32(0); - reply->supported = cpu_to_le32(dcp_is_typec_output(dcp)); + /* + * Analog DPIN CORE=1 already uses AFK set_hpd (returns 0). + * Denying HPD here made request_display ACTIVATE then 22/24 + * DEVICE_NOT_STARTED (~5.5s). Advertise HPD so firmware uses + * that path instead of waiting for a PHY start. + */ + reply->supported = cpu_to_le32(dcp_is_typec_output(dcp) ? 1 : 0); + dev_info(dcp->dev, "DPTXPort: GET_SUPPORTS_HPD %u usb4=%d\n", + le32_to_cpu(reply->supported), dcp_is_usb4_output(dcp)); return 0; } @@ -503,11 +643,20 @@ dptxport_call_activate(struct apple_epic_service *service, void *reply, size_t reply_size) { struct dptx_port *dptx = service->cookie; - const struct apple_dcp *dcp = service->ep->dcp; + struct apple_dcp *dcp = service->ep->dcp; - /* Standalone PHYs need DCP input selection here. Type-C owns ATC PHY mode. */ - if (!dcp->phy_managed_by_typec) + /* + * Ported from aurora-silicon/linux#8: no crossbar, and no PHY mode + * change for a tunnel (dptx->atcphy must stay in USB4/TBT mode the + * whole connection, see dcp_tunnel_set_rate()). The DP IN adapter is + * only woken (DPTX_INACTIVE=0) here, via dcp_tunnel_dpin_activate(); + * waking it earlier hangs the machine. Activate always replies + * success to DCP. + */ + if (dptx->atcphy && !dcp->phy_managed_by_typec) phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + if (dcp->dptx_tunnel) + dcp_tunnel_dpin_activate(dcp, true); memcpy(reply, data, min(reply_size, data_size)); if (reply_size >= 4) @@ -522,9 +671,12 @@ dptxport_call_deactivate(struct apple_epic_service *service, void *reply, size_t reply_size) { struct dptx_port *dptx = service->cookie; - const struct apple_dcp *dcp = service->ep->dcp; + struct apple_dcp *dcp = service->ep->dcp; - if (!dcp->phy_managed_by_typec) + dev_info(dcp->dev, "DPTXPort: DEACTIVATE\n"); + if (dcp->dptx_tunnel) + dcp_tunnel_dpin_activate(dcp, false); + if (dptx->atcphy && !dcp->phy_managed_by_typec) phy_set_mode_ext(dptx->atcphy, PHY_MODE_INVALID, 0); memcpy(reply, data, min(reply_size, data_size)); @@ -540,12 +692,30 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, { struct dptx_port *dptx = service->cookie; trace_dptxport_apcall(dptx, idx, data_size); + dev_dbg(service->ep->dcp->dev, "DPTXPort: APCALL %u (%zu bytes)\n", + idx, data_size); + if (data_size) + print_hex_dump_debug("DPTXPort: apcall data: ", + DUMP_PREFIX_OFFSET, 16, 1, data, + min(data_size, (size_t)64), true); switch (idx) { case DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG: + /* + * Ported from aurora-silicon/linux#8: a re-link on an + * established tunnel takes the crossbar connection down + * first. + */ + if (service->ep->dcp->dptx_tunnel && dptx->link_rate) + dcp_tunnel_crossbar_down(service->ep->dcp); return dptxport_call_will_change_link_config(service); - case DPTX_APCALL_DID_CHANGE_LINK_CONFIG: - return dptxport_call_did_change_link_config(service); + case DPTX_APCALL_DID_CHANGE_LINK_CONFIG: { + int ret = dptxport_call_did_change_link_config(service); + + if (!ret && service->ep->dcp->dptx_tunnel && dptx->link_rate) + dcp_tunnel_crossbar_up(service->ep->dcp); + return ret; + } case DPTX_APCALL_GET_MAX_LINK_RATE: return dptxport_call_get_max_link_rate(service, reply, reply_size); @@ -556,6 +726,15 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, reply, reply_size); case DPTX_APCALL_GET_MAX_LANE_COUNT: return dptxport_call_get_max_lane_count(service, reply, reply_size); + case DPTX_APCALL_GET_ACTIVE_LANE_COUNT: { + struct dptxport_apcall_lane_count *lc = reply; + + if (reply_size < sizeof(*lc)) + return -EINVAL; + lc->retcode = cpu_to_le32(0); + lc->lane_count = cpu_to_le64(dptx->lane_count); + return 0; + } case DPTX_APCALL_SET_ACTIVE_LANE_COUNT: return dptxport_call_set_active_lane_count(service, data, data_size, reply, reply_size); @@ -592,6 +771,26 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_DEACTIVATE: return dptxport_call_deactivate(service, data, data_size, reply, reply_size); + case DPTX_APCALL_FORCE_HOTPLUG_DETECT: + dev_info(service->ep->dcp->dev, + "DPTXPort: FORCE_HOTPLUG_DETECT\n"); + complete(&dptx->linkcfg_completion); + memcpy(reply, data, min(reply_size, data_size)); + if (reply_size >= 4) + memset(reply, 0, 4); + return 0; + case DPTX_APCALL_INACTIVE_SINK_DETECTED: + /* + * Normal prelude to link training on USB4. Ack and wait for + * SET_ACTIVE_LANE_COUNT. Treating this as failure made + * firmware DEACTIVATE after it had already set lanes/rate. + */ + dev_info(service->ep->dcp->dev, + "DPTXPort: INACTIVE_SINK_DETECTED (keep waiting for lanes)\n"); + memcpy(reply, data, min(reply_size, data_size)); + if (reply_size >= 4) + memset(reply, 0, 4); + return 0; default: /* just try to ACK and hope for the best... */ dev_info(service->ep->dcp->dev, "DPTXPort: acking unhandled call %u\n", diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 47539fd4bcc71..885e3675f9db5 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -59,6 +59,13 @@ struct dptx_port { u32 lane_count; u32 link_rate, pending_link_rate; u32 drive_settings[2]; + /* + * Per-boot invocation counters for the validate/connect/request/ + * release AFK calls, used together with the caller return address + * logged alongside them to correlate call ordering when diagnosing + * link-training sequencing issues. + */ + u32 validate_calls, connect_calls, request_calls, release_calls; }; int dptxport_validate_connection(struct apple_epic_service *service, u8 core, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 3ec33353d3abe..1e33d52825c43 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -293,7 +293,8 @@ void dcp_hotplug(struct work_struct *work) * display modes from atomic_flush, so userspace needs to trigger a * flush, or the CRTC gets no signal. */ - if (connector->base.state && !dcp->valid_mode && connector->connected) { + if (connector->base.state && !dcp->valid_mode && connector->connected && + !dcp_is_usb4_output(dcp)) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index c0b533c4f8a3c..279534c227ac7 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -493,6 +493,13 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d static struct dcp_read_edt_data_resp dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) { + /* Observe boot-property requests without inventing firmware timings. */ + if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) + dev_info(dcp->dev, + "read_edt_data key=%.*s count=%u default0=%#x ret=0\n", + (int)sizeof(req->key), req->key, req->count, + req->value[0]); + return (struct dcp_read_edt_data_resp){ .value[0] = req->value[0], .ret = 0, @@ -603,6 +610,12 @@ static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, { u8 resp = dcpep_process_chunks(dcp, req); + if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) + dev_info(dcp->dev, + "DCP property key=%.*s bytes=%zu accepted=%u nr_modes=%u\n", + (int)sizeof(req->key), req->key, dcp->chunks.length, + resp, dcp->nr_modes); + /* move chunked data to connector to provide it via debugfs */ dcp_connector_update_dict(dcp->connector, req->key, &dcp->chunks); dcp->chunks.data = NULL; @@ -1025,6 +1038,13 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) */ if (dcp->main_display) return; + /* + * Report firmware hotplug independently of the USB4 PHY experiment. + * Reassigning lpdptxphy blanked eDP even with these callbacks ignored; + * suppressing connector notifications does not protect the panel. + * Mode probing still uses this DCP's firmware modes, and mode_valid + * rejects modes absent from that list. Do not synthesize a mode here. + */ if (dcp->during_modeset) { /* @@ -1041,8 +1061,8 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) return; } - dev_info(dcp->dev, "cb_hotplug() connected:%llu, valid_mode:%d\n", - *connected, dcp->valid_mode); + dev_info(dcp->dev, "cb_hotplug() connected:%llu, valid_mode:%d nr_modes:%u\n", + *connected, dcp->valid_mode, dcp->nr_modes); /* Hotplug invalidates mode. DRM doesn't always handle this. */ if (!(*connected)) { From 85fbf65924ba19f33269f7864a1f52e30c1275e7 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 16:48:39 +0200 Subject: [PATCH 0247/1579] mux: apple-display-crossbar: route T602X DPIN0/DPIN1 through the mux apple_dpxbar_set_t602x() only ever programmed the DPPHY fields of the mux control register and its ATC bit, regardless of which of the three physical outputs (dpphy, dpin0, dpin1) was actually selected. Selecting dpin0 or dpin1 therefore only updated driver-internal bookkeeping without ever routing analog onto the ACIO DP IN pins, and the enable/disable paths never touched the enable bits the generic (non-T602X) crossbar path does. Add helpers to resolve the correct ATC bit and mux-select field for the index actually being switched, and use them in both the enable and disable sequences, including the enable-bit sequencing that was previously only done for the generic crossbar. Reject an out-of-range index up front rather than proceeding with an unresolved mux mask. Add apple_dpxbar_link_up()/apple_dpxbar_link_down(), exported for the DCP driver to reach through symbol_get()/symbol_put() so the crossbar driver does not need to be a hard module dependency of the display driver. A Thunderbolt DP tunnel re-link needs to bounce the crossbar's clock/FIFO gates on an output that is already selected, without reselecting the mux or dropping the enable bits -- a full teardown/ re-enable would drop the route entirely. Both functions operate on whichever output the mux already has recorded as selected. DPIN0's field masks and ATC bit are hardware-confirmed on T602X/M2 Pro; DPIN1's are filled in by the same per-index resolution by symmetry with DPPHY/DPIN0's layout, not by independent confirmation on real hardware -- flagging this here since getting a field encoding wrong on this hardware is silent (no error, just no picture). Co-Authored-By: Claude Sonnet 5 --- drivers/mux/apple-display-crossbar.c | 266 +++++++++++++++++++++++---- 1 file changed, 234 insertions(+), 32 deletions(-) diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index d3cd6399ead66..90384504e034c 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -5,8 +5,15 @@ * Copyright (C) Asahi Linux Contributors * * Author: Sven Peter + * + * T602x DPIN0/DPIN1: the stock t602x set() only programs DPPHY select + * bits (state<<20 | state<<8) and ATC bit 8. Mux index is ignored, so + * "Switched dpin0" never routes analog onto ACIO DP IN. Use the same + * CROSSBAR_MUX_CTRL field layout as t8103/t6000 at T602X_REG_030, and + * ATC_DPIN0/ATC_DPIN1 instead of the hardcoded 0x100 DPPHY bit. */ +#include #include #include #include @@ -17,9 +24,10 @@ #include #include #include +#include /* - * T602x register interface is cleary different so most of the names below are + * T602x register interface is clearly different so most of the names below are * probably wrong. */ @@ -118,6 +126,82 @@ static inline void dpxbar_clear32(struct apple_dpxbar *xbar, u32 reg, u32 clear) dpxbar_mask32(xbar, reg, clear, 0); } +static u32 t602x_atc_bit(unsigned int index) +{ + switch (index) { + case MUX_DPIN0: + return ATC_DPIN0; + case MUX_DPIN1: + return ATC_DPIN1; + case MUX_DPPHY: + default: + return ATC_DPPHY; + } +} + +static u32 t602x_mux_mask(unsigned int index) +{ + switch (index) { + case MUX_DPIN0: + return CROSSBAR_MUX_CTRL_DPIN0_SELECT0 | + CROSSBAR_MUX_CTRL_DPIN0_SELECT1; + case MUX_DPIN1: + return CROSSBAR_MUX_CTRL_DPIN1_SELECT0 | + CROSSBAR_MUX_CTRL_DPIN1_SELECT1; + case MUX_DPPHY: + default: + return CROSSBAR_MUX_CTRL_DPPHY_SELECT0 | + CROSSBAR_MUX_CTRL_DPPHY_SELECT1; + } +} + +static u32 t602x_mux_set(unsigned int index, unsigned int mux_state) +{ + switch (index) { + case MUX_DPIN0: + return FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN0_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN0_SELECT1, mux_state); + case MUX_DPIN1: + return FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN1_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN1_SELECT1, mux_state); + case MUX_DPPHY: + default: + return FIELD_PREP(CROSSBAR_MUX_CTRL_DPPHY_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPPHY_SELECT1, mux_state); + } +} + +static void t602x_dump(struct apple_dpxbar *xbar, const char *tag) +{ + static const u32 offs[] = { + 0x000, 0x004, 0x008, 0x00c, 0x014, 0x018, 0x01c, 0x024, + 0x028, 0x02c, 0x030, 0x034, 0x040, 0x044, 0x048, 0x04c, + 0x050, 0x060, 0x070, + }; + char buf[320]; + int n = 0; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(offs); i++) { + u32 val = readl(xbar->regs + offs[i]); + + n += scnprintf(buf + n, sizeof(buf) - n, " %03x=%08x", + offs[i], val); + if (n >= (int)sizeof(buf) - 20) + break; + } + dev_dbg(xbar->dev, "t602x %s:%s\n", tag, buf); + dev_dbg(xbar->dev, + "t602x %s clk: 000=%08x 800=%08x 020=%08x 820=%08x 024=%08x 81c=%08x\n", + tag, + readl(xbar->regs + T602X_FIFO_WR_DPTX_CLK_EN), + readl(xbar->regs + FIFO_WR_DPTX_CLK_EN_STAT), + readl(xbar->regs + FIFO_RD_PCLK1_EN), + readl(xbar->regs + FIFO_RD_PCLK1_EN_STAT), + readl(xbar->regs + T602X_FIFO_RD_PCLK2_EN), + readl(xbar->regs + T602X_REG_81C_STAT)); +} + static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) { struct apple_dpxbar *dpxbar = mux_chip_priv(mux->chip); @@ -126,15 +210,14 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) unsigned int mux_state; unsigned int dispext_bit; unsigned int dispext_bit_en; + u32 atc_bit, mux_mask, mux_val; bool enable; int ret = 0; + if (index >= MUX_MAX) + return -EINVAL; + if (state == MUX_IDLE_DISCONNECT) { - /* - * Technically this will select dispext0,0 in the mux control - * register. Practically that doesn't matter since everything - * else is disabled. - */ mux_state = 0; enable = false; } else if (state >= 0 && state < 9) { @@ -146,6 +229,10 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) return -EINVAL; } + atc_bit = t602x_atc_bit(index); + mux_mask = t602x_mux_mask(index); + mux_val = t602x_mux_set(index, mux_state); + spin_lock_irqsave(&dpxbar->lock, flags); /* ensure the selected dispext isn't already used in this crossbar */ @@ -166,51 +253,57 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) dpxbar_clear32(dpxbar, T602X_FIFO_RD_UNK_EN, prev_dispext_bit); dpxbar_clear32(dpxbar, T602X_FIFO_WR_DPTX_CLK_EN, prev_dispext_bit); - dpxbar_clear32(dpxbar, T602X_REG_00C, prev_dispext_bit_en); + /* DPIN0 uses the same single-bit source gate on up and down. */ + dpxbar_clear32(dpxbar, T602X_REG_00C, + index == MUX_DPIN0 ? prev_dispext_bit : prev_dispext_bit_en); - dpxbar_clear32(dpxbar, T602X_REG_01C, 0x100); + dpxbar_clear32(dpxbar, T602X_REG_01C, atc_bit); dpxbar_clear32(dpxbar, T602X_FIFO_WR_UNK_EN, prev_dispext_bit); dpxbar_clear32(dpxbar, T602X_REG_018, prev_dispext_bit_en); - dpxbar_clear32(dpxbar, T602X_FIFO_RD_N_CLK_EN, 0x100); + dpxbar_clear32(dpxbar, T602X_FIFO_RD_N_CLK_EN, atc_bit); dpxbar_set32(dpxbar, T602X_FIFO_WR_N_CLK_EN, prev_dispext_bit); dpxbar_set32(dpxbar, T602X_REG_014, 0x4); - dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, 0x100); + /* Native T602x DPIN0 teardown restores the read reset at +0x24. */ + dpxbar_set32(dpxbar, index == MUX_DPIN0 ? + T602X_FIFO_RD_PCLK2_EN : FIFO_RD_PCLK1_EN, atc_bit); + + dpxbar_clear32(dpxbar, T602X_REG_034, atc_bit); + dpxbar_clear32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + dpxbar_clear32(dpxbar, CROSSBAR_DISPEXT_EN, prev_dispext_bit); + dpxbar_mask32(dpxbar, T602X_REG_030, mux_mask, 0); dpxbar->selected_dispext[index] = -1; } if (enable) { - /* - * Both selector fields must be replaced, not OR-ed in: leaving a - * previous pipeline's bits set keeps the crossbar pointed at it, - * and selecting dispext0 (state 0) would change nothing at all. - * The port then never reaches the DCP that asked for it, whose - * DPTX calls time out with the firmware reporting the device as - * not started. - */ - dpxbar_mask32(dpxbar, T602X_REG_030, - GENMASK(23, 20) | GENMASK(11, 8), - (state << 20) | (state << 8)); + dpxbar_mask32(dpxbar, T602X_REG_030, mux_mask, mux_val); udelay(10); dpxbar_clear32(dpxbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); dpxbar_clear32(dpxbar, T602X_REG_014, 0x4); - dpxbar_clear32(dpxbar, T602X_FIFO_RD_PCLK2_EN, 0x100); + dpxbar_clear32(dpxbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); dpxbar_set32(dpxbar, T602X_FIFO_WR_UNK_EN, dispext_bit); dpxbar_set32(dpxbar, T602X_REG_018, dispext_bit_en); - dpxbar_set32(dpxbar, T602X_FIFO_RD_N_CLK_EN, 0x100); + dpxbar_set32(dpxbar, T602X_FIFO_RD_N_CLK_EN, atc_bit); dpxbar_set32(dpxbar, T602X_FIFO_WR_DPTX_CLK_EN, dispext_bit); dpxbar_set32(dpxbar, T602X_REG_00C, dispext_bit); - dpxbar_set32(dpxbar, T602X_REG_01C, 0x100); - dpxbar_set32(dpxbar, T602X_REG_034, 0x100); + dpxbar_set32(dpxbar, T602X_REG_01C, atc_bit); + dpxbar_set32(dpxbar, T602X_REG_034, atc_bit); + dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + /* + * t8103 enables the dispext source at 0x050. T602x left + * that register at 0, so dpin0 was selected with no + * source clock into the ACIO analog PHY. + */ + dpxbar_set32(dpxbar, CROSSBAR_DISPEXT_EN, dispext_bit); dpxbar_set32(dpxbar, T602X_FIFO_RD_UNK_EN, dispext_bit); @@ -219,13 +312,18 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) spin_unlock_irqrestore(&dpxbar->lock, flags); - if (enable) + if (enable) { dev_info(dpxbar->dev, "Switched %s to dispext%u,%u\n", apple_dpxbar_names[index], mux_state >> 1, mux_state & 1); - else + dev_dbg(dpxbar->dev, "t602x atc=0x%x mux=0x%x\n", atc_bit, + mux_val); + } else { dev_info(dpxbar->dev, "Switched %s to disconnected state\n", apple_dpxbar_names[index]); + } + + t602x_dump(dpxbar, enable ? apple_dpxbar_names[index] : "idle"); return ret; } @@ -244,11 +342,6 @@ static int apple_dpxbar_set(struct mux_control *mux, int state) u32 mux_mask, mux_set; if (state == MUX_IDLE_DISCONNECT) { - /* - * Technically this will select dispext0,0 in the mux control - * register. Practically that doesn't matter since everything - * else is disabled. - */ mux_state = 0; enable = false; } else if (state >= 0 && state < 9) { @@ -373,6 +466,115 @@ static const struct mux_control_ops apple_dpxbar_t602x_ops = { .set = apple_dpxbar_set_t602x, }; +/* + * Bring the clock/FIFO gates up on a mux whose dispext source is already + * selected, without touching the mux selection itself -- used by + * dcp_tunnel_crossbar_up() for a mux that dcp_typec_route_activate() + * already selected. Same register set as apple_dpxbar_set_t602x()'s + * enable path, but parameterized on the live selected_dispext[index] + * instead of a hardcoded source, and this index's own atc_bit instead of + * always dpin0's. + */ +int apple_dpxbar_link_up(struct mux_control *mux) +{ + struct apple_dpxbar *xbar; + unsigned int index; + unsigned long flags; + u32 dispext_bit, atc_bit; + int state, ret = 0; + + if (!mux || mux->chip->ops != &apple_dpxbar_t602x_ops) + return -EINVAL; + index = mux_control_get_index(mux); + atc_bit = t602x_atc_bit(index); + xbar = mux_chip_priv(mux->chip); + + spin_lock_irqsave(&xbar->lock, flags); + state = xbar->selected_dispext[index]; + if (state < 0) { + spin_unlock_irqrestore(&xbar->lock, flags); + return -ENODEV; + } + dispext_bit = 1 << state; + + dpxbar_clear32(xbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); + dpxbar_clear32(xbar, T602X_REG_014, dispext_bit); + dpxbar_clear32(xbar, index == MUX_DPIN0 ? T602X_FIFO_RD_PCLK2_EN : + FIFO_RD_PCLK1_EN, atc_bit); + udelay(1); + + dpxbar_set32(xbar, T602X_FIFO_WR_UNK_EN, dispext_bit); + dpxbar_mask32(xbar, T602X_REG_018, GENMASK(5, 4), BIT(4)); + /* + * The only hardware-verified value for this 2-bit field is dpin0's + * BIT(0), which is both ATC_DPIN0 and MUX_DPIN0's own enum value (1) + * coincidentally overlapping at bit 0 -- so this field's encoding + * for a second index was never independently confirmed on real + * hardware. FIELD_PREP(..., index), matching + * MUX_DPPHY=0/MUX_DPIN0=1/MUX_DPIN1=2, is an inferred generalization + * of that single known-good dpin0 data point, not a value verified + * against dpin1 hardware. If dpin1 misbehaves, this field is a + * prime suspect. + */ + dpxbar_mask32(xbar, T602X_FIFO_RD_N_CLK_EN, GENMASK(1, 0), + FIELD_PREP(GENMASK(1, 0), index)); + dpxbar_set32(xbar, T602X_FIFO_WR_DPTX_CLK_EN, dispext_bit); + dpxbar_set32(xbar, T602X_REG_00C, dispext_bit); + dpxbar_set32(xbar, T602X_REG_01C, atc_bit); + dpxbar_set32(xbar, T602X_REG_034, atc_bit); + dpxbar_set32(xbar, CROSSBAR_ATC_EN, atc_bit); + dpxbar_set32(xbar, CROSSBAR_DISPEXT_EN, dispext_bit); + dpxbar_set32(xbar, T602X_FIFO_RD_UNK_EN, dispext_bit); + spin_unlock_irqrestore(&xbar->lock, flags); + + dev_info(xbar->dev, "%s: crossbar link up (dispext=%d atc=0x%x)\n", + apple_dpxbar_names[index], state, atc_bit); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dpxbar_link_up); + +/* + * Inverse of apple_dpxbar_link_up(): gate the clocks back down without + * deselecting the mux or dropping CROSSBAR_ATC_EN/CROSSBAR_DISPEXT_EN -- + * mux selection and the ATC output enable are kept across a link + * down/up cycle. Re-asserts the same reset bits apple_dpxbar_link_up() + * releases, so a later link_up() on the same route starts from the same + * state as a fresh selection. + */ +int apple_dpxbar_link_down(struct mux_control *mux) +{ + struct apple_dpxbar *xbar; + unsigned int index; + unsigned long flags; + u32 dispext_bit, atc_bit; + int state; + + if (!mux || mux->chip->ops != &apple_dpxbar_t602x_ops) + return -EINVAL; + index = mux_control_get_index(mux); + atc_bit = t602x_atc_bit(index); + xbar = mux_chip_priv(mux->chip); + + spin_lock_irqsave(&xbar->lock, flags); + state = xbar->selected_dispext[index]; + if (state < 0) { + spin_unlock_irqrestore(&xbar->lock, flags); + return -ENODEV; + } + dispext_bit = 1 << state; + + dpxbar_set32(xbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); + dpxbar_set32(xbar, T602X_REG_014, dispext_bit); + dpxbar_set32(xbar, index == MUX_DPIN0 ? T602X_FIFO_RD_PCLK2_EN : + FIFO_RD_PCLK1_EN, atc_bit); + spin_unlock_irqrestore(&xbar->lock, flags); + + dev_info(xbar->dev, "%s: crossbar link down (dispext=%d atc=0x%x)\n", + apple_dpxbar_names[index], state, atc_bit); + return 0; +} +EXPORT_SYMBOL_GPL(apple_dpxbar_link_down); + static int apple_dpxbar_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; From 85d6d84536c845412289f15d69a24005ea087af6 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 21:00:14 +0200 Subject: [PATCH 0248/1579] phy: apple: atc: add DisplayPort tunnel clock support for USB4/TBT On T602X (M2 Pro, j416s) Type-C PHYs, a monitor connected directly via DP alt-mode or a USB-C-to-HDMI adapter already works, because that path drives the AUSPLL through the PHY's normal per-mode configuration table. A monitor connected through a USB4/Thunderbolt hub instead carries DisplayPort as a tunneled stream over the USB4 link, and the DCP display coprocessor firmware expects the PHY driver to hand it a dedicated tunnel pixel clock on request, outside the normal mode-configure path. Nothing implements that request today, so a tunneled display never gets a clock and stays dark even though the USB4 link itself comes up. Add the tunnel clock state machine: atc_tunnel_start() brings up the fixed AUSPLL frequency descriptor and PCLK1 output for one of the four standard DisplayPort link rates, after a preflight read of the gate/output/lock registers confirms the PHY is in the specific idle state this sequence expects (refusing with -EBUSY otherwise rather than clobbering live state). Before touching anything it snapshots the registers it is about to modify; atc_tunnel_restore() replays that snapshot in reverse to undo the sequence, and is called unconditionally at the top of atcphy_configure() so any mode change, power-down, or unplug leaves the tunnel registers exactly as it found them instead of stuck mid-sequence, and clears the request-guard state so a later tunnel request is not permanently refused by a stale in-progress flag. It also waits for ACIOPHY_AUSPLL_LOCK to clear before returning: a request landing right after a teardown -- a live replug is fast enough to do this, and so is DCP firmware's own retry cadence -- would otherwise see the PLL's own not-yet-settled lock bit and refuse itself as busy, indefinitely. Expose this through a new exported apple_atc_dp_tunnel_rate(phy, rate) -- the exact symbol name drivers/gpu/drm/apple/dcp.c resolves via symbol_get() to ask a PHY for a tunnel clock. It is gated to apple,j416s machines and to ATC cores that are both a t6020-atcphy and a recognized Type-C core address, so it is a correctly-typed no-op everywhere else, and only proceeds while the PHY is already in USB4 or TBT mode. Co-Authored-By: Claude Sonnet 5 --- drivers/phy/apple/atc.c | 265 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 265 insertions(+) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 2e822a3e37c3a..f8b833dc3f0e2 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -649,6 +649,9 @@ struct apple_atcphy { const struct atcphy_hw *hw; enum atcphy_mode mode; int dp_link_rate; + bool tunnel_attempted, tunnel_saved; + u8 tunnel_rate; + u32 tunnel_saved_regs[12]; bool swap_lanes; bool pipehandler_up; @@ -1894,12 +1897,222 @@ static int atcphy_power_on(struct apple_atcphy *atcphy) return 0; } +/* + * J416s DP tunnel clock. Native 13.5 uses a fixed AUSPLL descriptor and + * selects the requested rate at TX_DP_CTRL0. All addresses below belong + * to the existing ATC core mapping, not ACIO RC. + */ +struct atc_tunnel_saved_reg { + u32 reg; + u32 mask; +}; + +static const struct atc_tunnel_saved_reg atc_tunnel_regs[] = { + { ACIOPHY_CFG0, 0x0003ffff }, + { ACIOPHY_SLEEP_CTRL, 0x00000fff }, + { ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, 0x0000207c }, + { AUSPLL_FREQ_CFG, 0x00000003 }, + { AUSPLL_FREQ_DESC_A, 0xffffffff }, + { AUSPLL_FREQ_DESC_B, 0x0fffffff }, + { AUSPLL_FREQ_DESC_C, 0x007fffff }, + { AUSPLL_CLKOUT_DIV, 0x001f0000 }, + { AUSPLL_CLKOUT_DTC_VREG, 0x00000080 }, + { AUSPLL_BGR, 0x00000001 }, + { AUSPLL_CLKOUT_MASTER, 0x00000054 }, + { AUSPLL_APB_CMD_OVERRIDE, 0x1ffffff9 }, +}; + +static int atc_tunnel_command(struct apple_atcphy *atcphy, u32 command) +{ + u32 value; + int ret; + + core_mask32(atcphy, AUSPLL_APB_CMD_OVERRIDE, + AUSPLL_APB_CMD_OVERRIDE_CMD, + FIELD_PREP(AUSPLL_APB_CMD_OVERRIDE_CMD, command) | + AUSPLL_APB_CMD_OVERRIDE_REQ | AUSPLL_APB_CMD_OVERRIDE_UNK28); + ret = readl_poll_timeout(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE, + value, value & AUSPLL_APB_CMD_OVERRIDE_ACK, 10, 10000); + core_clear32(atcphy, AUSPLL_APB_CMD_OVERRIDE, AUSPLL_APB_CMD_OVERRIDE_REQ); + return ret; +} + +static void atc_tunnel_restore(struct apple_atcphy *atcphy) +{ + u32 value; + int i, ret; + + lockdep_assert_held(&atcphy->lock); + if (!atcphy->tunnel_saved) + return; + /* Stop our output before restoring the unused PLL descriptor. */ + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + for (i = ARRAY_SIZE(atc_tunnel_regs) - 1; i >= 0; i--) + core_mask32(atcphy, atc_tunnel_regs[i].reg, atc_tunnel_regs[i].mask, + atcphy->tunnel_saved_regs[i] & atc_tunnel_regs[i].mask); + /* + * atc_tunnel_start()'s preflight refuses to proceed while + * ACIOPHY_AUSPLL_LOCK is still set, to avoid clobbering a PLL a + * concurrent user actually has locked. Immediately after tearing our + * own lock down, that same bit can still read set for a brief window + * before the lock detector catches up -- and DCP firmware retries a + * failed tunnel clock request roughly once a second, which is faster + * than that window on a hot retrain loop. Wait for it to clear here + * so a request coming in right after this returns sees a genuinely + * idle PLL instead of refusing itself forever. + */ + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, value, + !(value & ACIOPHY_AUSPLL_LOCK), 10, 10000); + if (ret) + dev_warn(atcphy->dev, + "USB4 tunnel clock: AUSPLL_LOCK did not clear after teardown (stat=%#x)\n", + value); + atcphy->tunnel_saved = false; + atcphy->tunnel_attempted = false; + atcphy->tunnel_rate = 0; +} + +static int atc_tunnel_start(struct apple_atcphy *atcphy, u8 rate) +{ + u32 selector, value, gates, outputs, command, status; + unsigned int i; + int ret; + + lockdep_assert_held(&atcphy->lock); + switch (rate) { + case 0x06: + selector = 4; + break; + case 0x0a: + selector = 3; + break; + case 0x14: + selector = 1; + break; + case 0x1e: + selector = 0; + break; + default: + return -EINVAL; + } + if (atcphy->tunnel_saved) { + if (atcphy->tunnel_rate != rate) + dev_info(atcphy->dev, "USB4 tunnel clock busy: active rate=0x%x requested=0x%x\n", + atcphy->tunnel_rate, rate); + return atcphy->tunnel_rate == rate ? 0 : -EBUSY; + } + if (atcphy->tunnel_attempted) + return -EALREADY; + /* Read every guard before deciding, so a refusal records the whole state. */ + gates = readl(atcphy->regs.core + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0); + outputs = readl(atcphy->regs.core + AUSPLL_CLKOUT_MASTER); + command = readl(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE); + status = readl(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT); + dev_info(atcphy->dev, + "USB4 tunnel clock preflight: +7000=%08x +2200=%08x +2000=%08x +7044=%08x\n", + gates, outputs, command, status); + /* + * Before the first rate request, TX_DP_CTRL0 gate bits read 0xe001: + * the enable gates are set, the byte-clock reset is asserted, and the + * selector and reset-release bits are clear. Native configureDPTunnelMode + * tracks tunnel clients in software rather than through these gates, so + * this exact state is the only additional gate configuration accepted + * here, together with no enabled PLL output, no outstanding command, + * and no lock. Any other gate state is refused. + */ + if (outputs & 0x54 || command & AUSPLL_APB_CMD_OVERRIDE_REQ || + status & ACIOPHY_AUSPLL_LOCK) + return -EBUSY; + if ((gates & (DPTX_PCLK1_ENABLE | DPTX_PCLK2_ENABLE | DPRX_PCLK_ENABLE)) && + gates != 0x0000e001) + return -EBUSY; + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_CMN_SHM_STS_REG0, + value, value & ACIOPHY_CMN_SHM_STS_REG0_CMD_READY, + 10, 10000); + if (ret) + return ret; + atcphy->tunnel_attempted = true; + for (i = 0; i < ARRAY_SIZE(atc_tunnel_regs); i++) + atcphy->tunnel_saved_regs[i] = readl(atcphy->regs.core + atc_tunnel_regs[i].reg); + atcphy->tunnel_saved = true; + + /* Native configureDPTunnelMode, first clock client; no lane mux writes. */ + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG_OV); + udelay(2); + core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x30); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0xc0); + udelay(2); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x03); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x0c); + udelay(2); + core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, 0x300); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0xc00); + udelay(2); + /* Additional override bit alongside RX_BIG_OV; exact meaning not independently confirmed. */ + core_set32(atcphy, ACIOPHY_CFG0, 0xc00); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG_OV); + udelay(2); + /* Additional override bit alongside RX_SMALL_OV; exact meaning not independently confirmed. */ + core_set32(atcphy, ACIOPHY_CFG0, 0xc0); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL_OV); + udelay(2); + /* Additional override bit alongside RX_CLAMP_OV; exact meaning not independently confirmed. */ + core_clear32(atcphy, ACIOPHY_CFG0, 0xc000); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(2)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(3)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, + FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + + /* Native fixed descriptor at cache VA 0xfffffe0007582418. */ + core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, 0xffffffff, + 0x21c | (8 << 14) | (3 << 18) | (8 << 22) | (7 << 26)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_B, 0x0fffffff, 0); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, 0x007fffff, + (5 << 9) | (1 << 14) | (5 << 16) | (2 << 20) | BIT(22)); + core_mask32(atcphy, AUSPLL_CLKOUT_DIV, AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI, + FIELD_PREP(AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI, 1)); + core_set32(atcphy, AUSPLL_CLKOUT_DTC_VREG, AUSPLL_DTC_VREG_BYPASS); + core_set32(atcphy, AUSPLL_BGR, AUSPLL_BGR_CTRL_AVAIL); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(2)); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(4)); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(6)); + ret = atc_tunnel_command(atcphy, 0); + if (ret) + goto restore; + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, + value, value & ACIOPHY_AUSPLL_LOCK, 10, 10000); + if (ret) + goto restore; + ret = atc_tunnel_command(atcphy, 0x2000); + if (ret) + goto restore; + atcphy->tunnel_rate = rate; + return 0; +restore: + atc_tunnel_restore(atcphy); + return ret; +} + static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) { int ret = 0; u32 reg; lockdep_assert_held(&atcphy->lock); + /* Restore before the existing mode path can power down the registers. */ + atc_tunnel_restore(atcphy); if (mode == APPLE_ATCPHY_MODE_OFF) { ret = atcphy_power_off(atcphy); @@ -2144,6 +2357,58 @@ static const struct phy_ops apple_atc_dp_phy_ops = { .set_mode = atcphy_dpphy_set_mode, }; +/* + * j416s has one ATC PHY core per Type-C port at 0x703000000 (left), + * 0xb03000000 (left), 0xf03000000 (right) -- the same top-byte pattern + * as the ACIO/DPIN0 and crossbar addresses in drivers/thunderbolt/apple.c + * and drivers/mux/apple-display-crossbar.c. Originally this only + * accepted the right port's core while this was a single-port proof + * of concept; generalized once a second port needed testing. + */ +static bool apple_atc_is_typec_core(u64 base) +{ + return base == 0x703000000ULL || base == 0xb03000000ULL || + base == 0xf03000000ULL; +} + +/* + * Ported from aurora-silicon/linux#8's apple_atc_dp_tunnel_rate() (its own + * comment: "Based on Oliver Lukschander's t6020 tunnel clock patch") -- + * renamed to the reference's cross-module symbol name so + * dcp_tunnel_set_rate() (drivers/gpu/drm/apple/dcp.c) can find it via + * symbol_get(), keeping this project's own T602X-specific implementation + * (the reference's own version hard-fails on anything but t8103's fixed + * AUSPLL descriptor/PCLK1 selectors). Also accepts TBT mode, not just + * USB4: a genuine dock may negotiate either. + */ +int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); +int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) +{ + struct apple_atcphy *atcphy; + int ret; + + if (!phy || phy->ops != &apple_atc_dp_phy_ops || + !of_machine_is_compatible("apple,j416s")) + return -EOPNOTSUPP; + atcphy = phy_get_drvdata(phy); + if (!of_device_is_compatible(atcphy->np, "apple,t6020-atcphy") || + !apple_atc_is_typec_core(atcphy->res.core->start) || + resource_size(atcphy->res.core) < 0x7048) + return -EINVAL; + guard(mutex)(&atcphy->lock); + if (!rate) { + atc_tunnel_restore(atcphy); + return 0; + } + if (atcphy->mode != APPLE_ATCPHY_MODE_USB4 && + atcphy->mode != APPLE_ATCPHY_MODE_TBT) + return -EBUSY; + ret = atc_tunnel_start(atcphy, rate); + dev_info(atcphy->dev, "DP tunnel clock: rate=0x%x result=%d\n", rate, ret); + return ret; +} +EXPORT_SYMBOL_GPL(apple_atc_dp_tunnel_rate); + static struct phy *atcphy_xlate(struct device *dev, const struct of_phandle_args *args) { struct apple_atcphy *atcphy = dev_get_drvdata(dev); From 69893ec9574328f8126d38bf5729e858cc818002 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Fri, 25 Sep 2026 09:55:41 +0200 Subject: [PATCH 0249/1579] thunderbolt: apple: retry the ACIO reset handshake on resume-time contention apple_cio_start()'s reset_control_deassert(acio->reset) is a request/ack handshake with the M3/PMGR firmware (100ms budget, no .assert op to fall back on). Confirmed via live journalctl/dmesg to time out on resume from suspend with an active USB4 tunnel connected, during the wider resume storm every other coprocessor on the SoC is also going through. Retry up to APPLE_CIO_START_RETRIES (5) times, 100ms apart, before giving up. No new code path, no suspend/resume PM hook added -- deliberately not attempting the more invasive fix (giving apple_cio_driver its own PM hooks), since apple_cio_tbt_switch_set() documents that an invalid direct cable-state transition can crash ACIO and trigger an SoC watchdog reset, and a second independent stop()/start() trigger around the same suspend/resume window as the Type-C mux's own decision risks exactly that. This only widens how persistently the existing, already-serialized call waits for the firmware to answer. Co-Authored-By: Claude Sonnet 5 --- drivers/thunderbolt/apple.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 36ccfae7fdb50..8daf7ee870342 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -108,6 +108,9 @@ #define APPLE_CIO_NHI_BOOT_TIMEOUT_MS 10000 #define APPLE_CIO_PCIEC_READY_DELAY_MS 300 +#define APPLE_CIO_START_RETRIES 5 +#define APPLE_CIO_START_RETRY_DELAY_MS 100 + #define APPLE_CIO_PCIEC_INTR2AXI_CTRL 0x80 #define APPLE_CIO_PCIEC_INTR2AXI_ENABLE BIT(0) @@ -1527,9 +1530,28 @@ static int apple_cio_start(struct apple_cio *acio) /* * After the power domains are on we need to signal and wait for the ACIO block - * to actually start before we can bring up the co-processor. + * to actually start before we can bring up the co-processor. This is a request/ack + * handshake with the M3/PMGR firmware (reset_control_deassert() polls a shared + * "busy" bit with a 100ms budget), not a self-clearing hardware flop -- there is + * no .assert op to fall back on if it fails. Immediately after a system + * suspend/resume cycle this firmware handshake can be slower to answer than + * usual (observed once on this hardware, resuming with a cable that was already + * connected before suspend, in the middle of the wider resume storm every other + * coprocessor on the SoC is also going through) -- retry a bounded number of + * times with backoff before giving up, rather than failing on the very first + * 100ms window. */ - ret = reset_control_deassert(acio->reset); + for (i = 0; i < APPLE_CIO_START_RETRIES; i++) { + ret = reset_control_deassert(acio->reset); + if (!ret) + break; + if (i + 1 < APPLE_CIO_START_RETRIES) { + dev_warn(acio->dev, + "ACIO block failed to start (attempt %d/%d): %d, retrying\n", + i + 1, APPLE_CIO_START_RETRIES, ret); + msleep(APPLE_CIO_START_RETRY_DELAY_MS); + } + } if (ret) { dev_err(acio->dev, "ACIO block failed to start: %d\n", ret); goto remove_links; From 8f576eb57772a991cb5f93c0618ceab1028c70f7 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Fri, 25 Sep 2026 09:55:56 +0200 Subject: [PATCH 0250/1579] usb: typec: tipd: auto-recover the display after standby, no replug needed Building on the previous commit (ACIO/Thunderbolt itself now survives suspend): the display still didn't come back on its own after resume, only after a physical unplug/replug of the dock. tipd_resume() never re-verified the port's actual attach state -- it only checks firmware liveness and re-arms the IRQ. A cable that was never physically removed generates no fresh attach event, so nothing re-drives apple_cio_tbt_switch_set()/apple_cio_start(), even though thunderbolt-core's own tb_resume_noirq() -> tb_free_invalid_tunnels() had already silently torn down the live DP tunnel underneath it (confirmed generic thunderbolt-core behaviour, not Apple-specific). A naive "just re-verify and re-assert the same state" fix is a no-op: apple_cio_tbt_switch_set()'s first check is target_cable_info == current_cable_info, and current_cable_info is only ever written inside apple_cio_start()/apple_cio_stop() themselves -- re-detecting the same, unchanged cable computes the same target and hits that early return immediately. Added cd321x_resume_reverify(), wired only into the Apple-specific CD321x/sn201202x vtables (the plain TI TPS6598X/TPS25750 variants are unaffected). It reproduces a genuine unplug-then-replug entirely through the existing, already-hardware-validated connect() code path: calls the existing connect() callback twice -- once with TPS_STATUS_PLUG_PRESENT cleared (a synthetic disconnect), then once with the real, unmodified status. Both land on the same debounced cd321x_update_work(), coalescing into a single run with was_disconnected=true and the correct final state: the exact same typec_thunderbolt_switch_set() OFF-then-back-on sequence, through the exact same apple_cio_tbt_switch_set() guard, a real physical replug already exercises every time someone does it by hand. The synthetic disconnect always drives through a clean zero state first, the same shape apple_cio_tbt_switch_set()'s own watchdog-reset warning already treats as safe (a direct transition between two different nonzero cable-info states, with no shutdown in between, is what it warns against). Triggered from a PM_POST_SUSPEND notifier rather than directly from tipd_resume(): tipd_resume() is just one individual device's own .resume() callback, with no guarantee the rest of the system (ACIO/ Thunderbolt in particular) has finished its own resume work yet. PM_POST_SUSPEND is delivered strictly after dpm_resume_end(), i.e. after every device has completed its own ordinary resume, which matters here specifically: driving the reconnect too early can fire apple_cio_start()'s M3/PMGR reset-deassert handshake while that firmware is still busy servicing other subsystems' resume, and exhaust its retry budget outright. A real physical replug never hits this, because a human always does it seconds after the whole system has visibly settled. Confirmed on hardware: real lid-close/s2idle/lid-open cycle, hub never touched. dmesg shows the full automatic chain (PM_POST_SUSPEND -> synthetic disconnect and reconnect -> tunnel connect -> modeset) succeeding with zero ACIO start retries, and the picture returns with no replug and no manual intervention. Co-Authored-By: Claude Sonnet 5 --- drivers/usb/typec/tipd/core.c | 78 +++++++++++++++++++++++++++++++ drivers/usb/typec/tipd/tps6598x.h | 3 ++ 2 files changed, 81 insertions(+) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 130b1b8ecf8c2..6f38b274d4105 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "tps6598x.h" #include "trace.h" @@ -902,6 +903,44 @@ static int cd321x_connect(struct tps6598x *tps, u32 status) return 0; } +/* + * A cable that was already attached before a suspend/resume cycle generates + * no fresh attach event on resume: the PD controller's own status hasn't + * changed, so cd321x_connect()'s diff against its last cached status sees + * nothing different and cd321x_update_work() takes its "nothing to do" + * path. Meanwhile thunderbolt-core's own noirq-phase resume handling + * (tb_free_invalid_tunnels()) has already torn down any USB4/TBT tunnel + * that didn't survive the sleep at the link level -- and nothing else + * re-establishes it. + * + * Reproduce a genuine unplug-then-replug in software instead of inventing + * a new code path: call the existing connect() callback once with the + * current status but PLUG_PRESENT cleared (a synthetic disconnect), then + * again with the real, current status. Both updates land on the same + * debounced work item, which coalesces them into a single run with + * was_disconnected=true and the correct final state -- exactly the same + * teardown-then-bring-up sequence, through the same already-validated + * apple_cio_tbt_switch_set() path, that a real physical replug drives. + */ +static void cd321x_resume_reverify(struct tps6598x *tps) +{ + u32 status; + + if (!tps6598x_read_status(tps, &status)) + return; + + if (!(status & TPS_STATUS_PLUG_PRESENT)) + return; + + if (!tps6598x_read_power_status(tps)) + return; + if (!tps->data->read_data_status(tps)) + return; + + tps->data->connect(tps, status & ~TPS_STATUS_PLUG_PRESENT); + tps->data->connect(tps, status); +} + static irqreturn_t cd321x_interrupt(int irq, void *data) { struct tps6598x *tps = data; @@ -1807,6 +1846,8 @@ static void cd321x_remove(struct tps6598x *tps) usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); } +static int tipd_pm_notify(struct notifier_block *nb, unsigned long action, void *data); + int tipd_init(struct tps6598x *tps) { struct fwnode_handle *fwnode; @@ -1912,6 +1953,11 @@ int tipd_init(struct tps6598x *tps) enable_irq_wake(tps->irq); } + if (tps->data->resume_reverify) { + tps->pm_nb.notifier_call = tipd_pm_notify; + register_pm_notifier(&tps->pm_nb); + } + return 0; err_disconnect: @@ -1934,6 +1980,9 @@ EXPORT_SYMBOL_GPL(tipd_init); void tipd_remove(struct tps6598x *tps) { + if (tps->data->resume_reverify) + unregister_pm_notifier(&tps->pm_nb); + if (!tps->irq) cancel_delayed_work_sync(&tps->wq_poll); else @@ -2000,6 +2049,33 @@ int tipd_resume(struct tps6598x *tps) } EXPORT_SYMBOL_GPL(tipd_resume); +/* + * tipd_resume() runs as an ordinary device .resume() callback, which can + * complete well before the rest of the system -- ACIO/Thunderbolt, DCP, + * the crossbar -- has finished its own resume work. Driving the reconnect + * from here directly (as originally tried) fires apple_cio_start()'s + * M3/PMGR reset-deassert handshake while that firmware is potentially + * still busy servicing other subsystems' resume, and can exhaust its + * retry budget outright -- confirmed on hardware, see + * notes/2026-09-24-0147-typec-resume-reverify.md. A real physical + * replug never hits this because a human always does it well after + * the whole system has settled. + * + * PM_POST_SUSPEND is delivered strictly after dpm_resume_end() -- + * i.e. after every device, including ACIO/Thunderbolt, has completed + * its own ordinary resume -- so triggering the reverify from here + * instead removes the race without needing any arbitrary delay. + */ +static int tipd_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct tps6598x *tps = container_of(nb, struct tps6598x, pm_nb); + + if (action == PM_POST_SUSPEND && tps->data->resume_reverify) + tps->data->resume_reverify(tps); + + return NOTIFY_DONE; +} + const struct tipd_data tipd_cd321x_data = { .irq_handler = cd321x_interrupt, .irq_mask1 = APPLE_CD_REG_INT_POWER_STATUS_UPDATE | @@ -2017,6 +2093,7 @@ const struct tipd_data tipd_cd321x_data = { .reset = cd321x_reset, .switch_power_state = cd321x_switch_power_state, .connect = cd321x_connect, + .resume_reverify = cd321x_resume_reverify, }; EXPORT_SYMBOL_GPL(tipd_cd321x_data); @@ -2075,6 +2152,7 @@ const struct tipd_data tipd_sn201202x_data = { .reset = cd321x_reset, .switch_power_state = cd321x_switch_power_state, .connect = cd321x_connect, + .resume_reverify = cd321x_resume_reverify, }; EXPORT_SYMBOL_GPL(tipd_sn201202x_data); diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index c6c05da547f2a..5ec291b595928 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -292,6 +292,7 @@ struct tipd_data { bool (*read_data_status)(struct tps6598x *tps); int (*reset)(struct tps6598x *tps); int (*connect)(struct tps6598x *tps, u32 status); + void (*resume_reverify)(struct tps6598x *tps); }; struct tps6598x { @@ -318,6 +319,8 @@ struct tps6598x { u16 pwr_status; struct delayed_work wq_poll; + struct notifier_block pm_nb; + const struct tipd_data *data; }; From eda8312d82f663832af9ad9899e89f3b407ed806 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:24:03 -0700 Subject: [PATCH 0251/1579] thunderbolt: release host DP state before adapter and domain teardown Track whether the NHI post-activation hook has acquired display state, including partial setup. Release it once before disabling adapters and when the domain stops, while the host router is still accessible. Put the Apple DP IN adapter to sleep before marking its context dead. Add KUnit coverage for repeated cleanup after domain removal and cleanup without a prior host activation. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/apple.c | 16 ++++------ drivers/thunderbolt/nhi.h | 3 +- drivers/thunderbolt/tb.c | 4 ++- drivers/thunderbolt/test.c | 57 ++++++++++++++++++++++++++++++++++++ drivers/thunderbolt/tunnel.c | 37 ++++++++++++----------- drivers/thunderbolt/tunnel.h | 3 ++ 6 files changed, 88 insertions(+), 32 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 6c389bec5be5c..7962df7e3ab11 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -1409,22 +1409,16 @@ static void apple_nhi_dp_tunnel_deactivate(struct tb_nhi *nhi, if (READ_ONCE(apple_dpin_anhi) == anhi) WRITE_ONCE(apple_dpin_anhi, NULL); - /* - * Runs after both adapters are already disabled (tb_dp_activate()'s - * teardown order) -- later than aurora-silicon/linux#8's own - * dp_tunnel_changed(active=false), which fires before deactivation - * while the block is still powered. apple_dpin_dcp_set_active()'s - * own "block may already be off" check (matching the reference - * implementation's identical defensive check) covers this gap; a - * spurious no-op register access here is not a functional blocker, - * only a less clean teardown than the ideal ordering. - */ + /* Sleep the adapter before tunnel teardown can remove its power. */ idx = apple_dpin_index_for_port(anhi, in); if (idx >= 0 && idx <= 1 && anhi->acio && anhi->acio->dp_wq) { struct apple_dpin_ctx *c = &anhi->acio->dpin[idx]; - scoped_guard(mutex, &c->lock) + scoped_guard(mutex, &c->lock) { + if (c->alive && c->regs) + apple_dpin_set_active(c->acio, c->regs, c->idx, false); c->alive = false; + } queue_work(anhi->acio->dp_wq, &c->work); } dev_info(anhi->dev, "DP IN tunnel routing: tunnel down\n"); diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index bc0557fd8d911..c4bf6f1e91ed2 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -86,7 +86,8 @@ struct tb_nhi_ring_layout { * and before VE/AE. USB4 DPTX Discovery belongs here. * @dp_tunnel_post_activate: NHI specific hook run after a DP tunnel's * adapters have VE/AE enabled - * @dp_tunnel_deactivate: NHI specific hook run when a DP tunnel is torn down + * @dp_tunnel_deactivate: Releases state from an attempted post-activation hook, + * before adapters are disabled or the host router is removed * @is_present: Whether the device is currently present on the parent bus * @init_interrupts: NHI specific interrupt initialization hook */ diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 6f52d49105db6..381682fb1d22f 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -3006,8 +3006,10 @@ static void tb_stop(struct tb *tb) if (tb_tunnel_is_dma(tunnel)) tb_tunnel_deactivate(tunnel); /* the host side of a DP tunnel goes away with us */ - else if (tb_tunnel_is_dp(tunnel)) + else if (tb_tunnel_is_dp(tunnel)) { tb_dp_tunnel_notify(tunnel, false); + tb_dp_tunnel_deactivate_host(tunnel); + } tb_tunnel_put(tunnel); } tb_switch_remove(tb->root_switch); diff --git a/drivers/thunderbolt/test.c b/drivers/thunderbolt/test.c index 1f4318249c226..e7469226002b0 100644 --- a/drivers/thunderbolt/test.c +++ b/drivers/thunderbolt/test.c @@ -2852,7 +2852,64 @@ static void tb_test_property_copy(struct kunit *test) tb_property_free_dir(src); } +struct tb_test_dp_host { + struct tb_nhi nhi; + unsigned int disconnects; +}; + +static void tb_test_dp_host_disconnect(struct tb_nhi *nhi, struct tb_port *in, + struct tb_port *out) +{ + struct tb_test_dp_host *host = container_of(nhi, struct tb_test_dp_host, nhi); + + host->disconnects++; +} + +static void tb_test_dp_host_teardown_once(struct kunit *test) +{ + static const struct tb_nhi_ops ops = { + .dp_tunnel_deactivate = tb_test_dp_host_disconnect, + }; + struct tb_test_dp_host *host; + struct tb_tunnel *tunnel; + struct tb *tb; + + host = kunit_kzalloc(test, sizeof(*host), GFP_KERNEL); + tunnel = kunit_kzalloc(test, sizeof(*tunnel), GFP_KERNEL); + tb = kunit_kzalloc(test, sizeof(*tb), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, host); + KUNIT_ASSERT_NOT_NULL(test, tunnel); + KUNIT_ASSERT_NOT_NULL(test, tb); + host->nhi.ops = &ops; + tb->nhi = &host->nhi; + tunnel->tb = tb; + tunnel->host_dp_activated = true; + + /* Domain removal must release the display even without adapter I/O. */ + tb_dp_tunnel_deactivate_host(tunnel); + KUNIT_EXPECT_EQ(test, host->disconnects, 1U); + KUNIT_EXPECT_FALSE(test, tunnel->host_dp_activated); + + /* A late tunnel cleanup must not access the already removed domain. */ + tunnel->tb = NULL; + tb_dp_tunnel_deactivate_host(tunnel); + KUNIT_EXPECT_EQ(test, host->disconnects, 1U); +} + +static void tb_test_dp_host_teardown_unannounced(struct kunit *test) +{ + struct tb_tunnel *tunnel; + + tunnel = kunit_kzalloc(test, sizeof(*tunnel), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tunnel); + /* A failed allocation or a domain-less test has no host state to undo. */ + tb_dp_tunnel_deactivate_host(tunnel); + KUNIT_EXPECT_FALSE(test, tunnel->host_dp_activated); +} + static struct kunit_case tb_test_cases[] = { + KUNIT_CASE(tb_test_dp_host_teardown_once), + KUNIT_CASE(tb_test_dp_host_teardown_unannounced), KUNIT_CASE(tb_test_path_basic), KUNIT_CASE(tb_test_path_not_connected_walk), KUNIT_CASE(tb_test_path_single_hop_walk), diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index 80a4d9b441174..0eb6b9e5b1f7c 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -1266,6 +1266,20 @@ static void tb_dp_apple_pulse_hpd(struct tb_port *in) tb_port_info(in, "Apple: HPD propagated\n"); } +/* Release display-side state while the host router is still accessible. */ +void tb_dp_tunnel_deactivate_host(struct tb_tunnel *tunnel) +{ + const struct tb_nhi_ops *ops; + + if (!tunnel->host_dp_activated) + return; + tunnel->host_dp_activated = false; + ops = tunnel->tb->nhi->ops; + if (ops && ops->dp_tunnel_deactivate) + ops->dp_tunnel_deactivate(tunnel->tb->nhi, tunnel->src_port, + tunnel->dst_port); +} + static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) { int ret; @@ -1297,6 +1311,7 @@ static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) return ret; } } else { + tb_dp_tunnel_deactivate_host(tunnel); tb_dp_dprx_stop(tunnel); tb_dp_port_hpd_clear(tunnel->src_port); tb_dp_port_set_hops(tunnel->src_port, 0, 0, 0); @@ -1305,18 +1320,7 @@ static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) } ret = tb_dp_port_enable(tunnel->src_port, active); - /* - * On deactivate, a departing dst_port already marked unplugged - * makes tb_port_read/write short-circuit to -ENODEV with zero I/O - * (tb.h) -- deterministically, not just on hardware timing. Do not - * let that skip ops->dp_tunnel_deactivate() below: this tunnel.c is - * generic core code, but ops->dp_tunnel_deactivate is what apple.c - * relies on to clear its own apple_dpin_ctx latch -- without it, a - * physical unplug/replug can never re-arm a fresh connect. The caller - * (tb_tunnel_deactivate()) already discards this function's return - * value on the deactivate path, so relaxing it here changes nothing - * any caller observes; the activate (active=true) path is untouched. - */ + /* A departed adapter must not prevent disabling the remaining adapter. */ if (ret && active) return ret; @@ -1363,19 +1367,14 @@ static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; if (ops && ops->dp_tunnel_post_activate) { + /* Also unwind a hook that fails after partial setup. */ + tunnel->host_dp_activated = true; ret = ops->dp_tunnel_post_activate(tunnel->tb->nhi, tunnel->src_port, tunnel->dst_port); if (ret) return ret; } - } else { - const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; - - if (ops && ops->dp_tunnel_deactivate) - ops->dp_tunnel_deactivate(tunnel->tb->nhi, - tunnel->src_port, - tunnel->dst_port); } return active ? tb_dp_dprx_start(tunnel) : 0; diff --git a/drivers/thunderbolt/tunnel.h b/drivers/thunderbolt/tunnel.h index 6eb672e9c45d7..5a73e60a26c99 100644 --- a/drivers/thunderbolt/tunnel.h +++ b/drivers/thunderbolt/tunnel.h @@ -65,6 +65,7 @@ enum tb_tunnel_state { * @dprx_started: DPRX negotiation was started (tb_dp_dprx_start() was called for it) * @dprx_canceled: Was DPRX capabilities read poll canceled * @host_dp_notified: The NHI glue was told this DP tunnel is up (Apple hosts) + * @host_dp_activated: The NHI post-activation hook owns display-side state * @dprx_timeout: If set DPRX capabilities read poll work will timeout after this passes * @dprx_work: Worker that is scheduled to poll completion of DPRX capabilities read * @callback: Optional callback called when DP tunnel is fully activated @@ -104,6 +105,7 @@ struct tb_tunnel { bool dprx_started; bool dprx_canceled; bool host_dp_notified; + bool host_dp_activated; ktime_t dprx_timeout; struct delayed_work dprx_work; void (*callback)(struct tb_tunnel *tunnel, void *data); @@ -138,6 +140,7 @@ struct tb_tunnel *tb_tunnel_alloc_usb3(struct tb *tb, struct tb_port *up, int max_down); void tb_tunnel_put(struct tb_tunnel *tunnel); +void tb_dp_tunnel_deactivate_host(struct tb_tunnel *tunnel); int tb_tunnel_activate(struct tb_tunnel *tunnel); void tb_tunnel_deactivate(struct tb_tunnel *tunnel); From de8d489b4a151e83b172e3db64fb271ca7af1cef Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:24:04 -0700 Subject: [PATCH 0252/1579] usb: typec: tipd: serialize post-suspend port revalidation Take the port mutex while reading status and scheduling the synthetic disconnect/reconnect so the PM notifier cannot race the IRQ and debounce workers. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/usb/typec/tipd/core.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 6f38b274d4105..048891241fe83 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -2070,8 +2070,11 @@ static int tipd_pm_notify(struct notifier_block *nb, unsigned long action, void { struct tps6598x *tps = container_of(nb, struct tps6598x, pm_nb); - if (action == PM_POST_SUSPEND && tps->data->resume_reverify) + if (action == PM_POST_SUSPEND && tps->data->resume_reverify) { + mutex_lock(&tps->lock); tps->data->resume_reverify(tps); + mutex_unlock(&tps->lock); + } return NOTIFY_DONE; } From 52f4345b973a83e7294daa49c1dac09091a9dec4 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:24:04 -0700 Subject: [PATCH 0253/1579] drm/apple: keep the j416s tunnel hotplug policy scoped to M2 Pro Retain the M1 link-status handling when combining the two display series. Document the T602X PHY state carried alongside the M1 clock state. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/iomfb.c | 2 +- drivers/phy/apple/atc.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index feea9f38737dc..2ab5e2a040379 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -294,7 +294,7 @@ void dcp_hotplug(struct work_struct *work) * flush, or the CRTC gets no signal. */ if (connector->base.state && !dcp->valid_mode && connector->connected && - !dcp_is_usb4_output(dcp)) { + !(dcp_is_usb4_output(dcp) && of_machine_is_compatible("apple,j416s"))) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 816cb65b9f227..e77a9abd58fdc 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -649,6 +649,9 @@ struct atcphy_hw { * @swap_lanes: True if lanes must be swapped due to cable orientation * @dp_link_rate: DisplayPort link rate * @tunnel_clock_on: True while the DisplayPort-over-Thunderbolt pixel clock runs + * @tunnel_attempted: A T602X tunnel clock setup has been attempted + * @tunnel_saved: T602X PHY registers have been saved for tunnel teardown + * @tunnel_saved_regs: Original values of the T602X tunnel clock registers * @tunnel_rate: DP link rate code the tunnel pixel clock is set up for * @pipe_state: Backend the PIPE mux ("pipehandler") is routed to * @regs: Memory-mapped registers From 36d52e109a1ea070f1b4731672a9a1bbd80a437b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:00:22 -0700 Subject: [PATCH 0254/1579] thunderbolt: apple: protect diagnostics and apply host DP credits Sample the optional analog diagnostic at tunnel activation instead of dereferencing a global controller from sysfs. Disable AUX work before domain removal so racing activation cannot rearm it. Use an explicit host DP credit quirk for both allocation and programming, including j416s without the M1 notification hook. Cover the quirk with KUnit and document the diagnostic parameters. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- .../admin-guide/kernel-parameters.txt | 25 +++++++++++ drivers/thunderbolt/apple.c | 42 +++++-------------- drivers/thunderbolt/nhi.h | 1 + drivers/thunderbolt/switch.c | 2 +- drivers/thunderbolt/tb.h | 8 ++++ drivers/thunderbolt/test.c | 38 +++++++++++++++++ drivers/thunderbolt/tunnel.c | 4 +- 7 files changed, 84 insertions(+), 36 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 889f8b726f29c..ea78b77f79abd 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7601,6 +7601,14 @@ Kernel parameters See Documentation/admin-guide/mm/transhuge.rst for more details. + thunderbolt.dp_video_counter= + [HW,ARM64] + Format: + Enable diagnostic counter 0 on the DP video path's + host and downstream ingress hops on the Apple j416s + right-hand USB-C route. Read counters through Thunderbolt + debugfs. This is a load-time setting. Default: false. + thunderbolt_apple.dp_display= [HW,ARM64] Format: @@ -7610,6 +7618,23 @@ Kernel parameters whenever the connection manager sets up a DP tunnel. Default: false + thunderbolt_apple.dpin_aux= + [HW,ARM64] + Format: + Pulse the experimental DP IN analog start control on + Apple j416s at the next DP tunnel activation. Writes to + the module parameter do not access a live controller. + Default: false (leave analog registers alone until + DPRX negotiation finishes or times out). + + thunderbolt_apple.dpin_mode_value= + [HW,ARM64] + Format: (0-15) + Experimental DP IN MODE_A/MODE_B value on Apple j416s. + Sampled for each adapter activation; values above 15 + make activation fail. Default: 8. The value is empirical + and does not select the negotiated DisplayPort rate. + tlbi= [X86-64] Format: {ipi} ipi: switch to IPI-based TLB flushing diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 7962df7e3ab11..0570b811fab6b 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -991,17 +991,10 @@ static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) * (default). Tests whether consuming +0x18 aborted a handshake. * 1 = old pulse +0x00 / fill +0x20 (rollback) */ -static int apple_dpin_aux; -static struct apple_nhi *apple_dpin_anhi; - -static int apple_dpin_aux_set(const char *val, const struct kernel_param *kp); -static const struct kernel_param_ops apple_dpin_aux_ops = { - .set = apple_dpin_aux_set, - .get = param_get_int, -}; -module_param_cb(dpin_aux, &apple_dpin_aux_ops, &apple_dpin_aux, 0644); +static bool apple_dpin_aux; +module_param_named(dpin_aux, apple_dpin_aux, bool, 0644); MODULE_PARM_DESC(dpin_aux, - "Apple DP IN analog: 0=hands-off until timeout, 1=pulse +0x00"); + "Apple DP IN analog: pulse +0x00 at the next tunnel activation (default: false)"); static void apple_dp_dump_hop(struct tb_port *port, unsigned int hopid) { @@ -1202,21 +1195,6 @@ static void apple_dp_start_analog(struct apple_nhi *anhi, bool pulse) "dpin analog after start", false); } -static int apple_dpin_aux_set(const char *val, const struct kernel_param *kp) -{ - struct apple_nhi *anhi; - int v, ret; - - ret = kstrtoint(val, 0, &v); - if (ret) - return ret; - apple_dpin_aux = v; - anhi = READ_ONCE(apple_dpin_anhi); - if (v >= 1 && anhi) - apple_dp_start_analog(anhi, true); - return 0; -} - static int apple_dp_dptx_discover(struct tb_port *in) { u32 cs0 = 0; @@ -1350,7 +1328,9 @@ static int apple_nhi_dp_tunnel_post_activate(struct tb_nhi *nhi, anhi->analog_base = apple_dp_in_analog_base(anhi, in); anhi->dp_aux_polls = 0; anhi->dp_aux_armed = true; - WRITE_ONCE(apple_dpin_anhi, anhi); + /* The domain lock keeps this optional MMIO access ahead of teardown. */ + if (READ_ONCE(apple_dpin_aux)) + apple_dp_start_analog(anhi, true); apple_dp_dump_rc(anhi->acio); apple_dp_dump_vse(in->sw); @@ -1406,8 +1386,6 @@ static void apple_nhi_dp_tunnel_deactivate(struct tb_nhi *nhi, anhi->dp_aux_armed = false; cancel_delayed_work(&anhi->dp_aux_work); - if (READ_ONCE(apple_dpin_anhi) == anhi) - WRITE_ONCE(apple_dpin_anhi, NULL); /* Sleep the adapter before tunnel teardown can remove its power. */ idx = apple_dpin_index_for_port(anhi, in); @@ -1559,6 +1537,8 @@ static int apple_nhi_probe(struct platform_device *pdev) anhi->nhi.ring_layout = &apple_nhi_ring_layout; anhi->nhi.iobase = anhi->nhi_base; anhi->nhi.quirks = QUIRK_NO_DMA_PORT | QUIRK_NO_USB3_BW_ALLOC; + if (anhi->ops.dp_tunnel_changed || anhi->ops.dp_tunnel_post_activate) + anhi->nhi.quirks |= QUIRK_HOST_DP_NFC_CREDITS; anhi->nhi.hop_count = readl(anhi->nhi_base + APPLE_CIO_NHI_HOP_COUNT) & APPLE_CIO_NHI_HOP_COUNT_MASK; @@ -1655,10 +1635,8 @@ static void apple_nhi_remove(struct platform_device *pdev) { struct apple_nhi *anhi = platform_get_drvdata(pdev); - anhi->dp_aux_armed = false; - cancel_delayed_work_sync(&anhi->dp_aux_work); - if (READ_ONCE(apple_dpin_anhi) == anhi) - WRITE_ONCE(apple_dpin_anhi, NULL); + /* Stop racing tunnel activation from rearming work during removal. */ + disable_delayed_work_sync(&anhi->dp_aux_work); WRITE_ONCE(anhi->acio->nhi_pdev, NULL); tb_domain_remove(anhi->tb); wait_for_completion(&anhi->nhi.domain_released); diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index c4bf6f1e91ed2..ba98934a30b91 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -174,6 +174,7 @@ struct tb_nhi_ops { #define QUIRK_E2E BIT(1) #define QUIRK_NO_DMA_PORT BIT(2) #define QUIRK_NO_USB3_BW_ALLOC BIT(3) +#define QUIRK_HOST_DP_NFC_CREDITS BIT(4) /* * Minimal number of vectors when we use MSI-X. Two for control channel diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c index e15f830bc8d63..4c7e63dd4332d 100644 --- a/drivers/thunderbolt/switch.c +++ b/drivers/thunderbolt/switch.c @@ -576,7 +576,7 @@ int tb_port_add_nfc_credits(struct tb_port *port, int credits) * so skip other ports. */ if (tb_switch_is_usb4(port->sw) && !tb_port_is_null(port) && - !tb_port_is_apple_host_dpin(port)) + !tb_port_needs_host_dp_credits(port)) return 0; nfc_credits = port->config.nfc_credits & ADP_CS_4_NFC_BUFFERS_MASK; diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h index 6a593d90fa89e..9a1af4969c9ca 100644 --- a/drivers/thunderbolt/tb.h +++ b/drivers/thunderbolt/tb.h @@ -1143,6 +1143,14 @@ static inline bool tb_port_is_apple_host_dpin(const struct tb_port *port) return tb->nhi->ops && tb->nhi->ops->dp_tunnel_changed; } +static inline bool tb_port_needs_host_dp_credits(const struct tb_port *port) +{ + const struct tb *tb = port->sw->tb; + + return tb && tb->nhi && !tb_route(port->sw) && tb_port_is_dpin(port) && + (tb->nhi->quirks & QUIRK_HOST_DP_NFC_CREDITS); +} + static inline bool tb_port_use_credit_allocation(const struct tb_port *port) { return tb_port_is_null(port) && port->sw->credit_allocation; diff --git a/drivers/thunderbolt/test.c b/drivers/thunderbolt/test.c index e7469226002b0..fa58a1b0a921a 100644 --- a/drivers/thunderbolt/test.c +++ b/drivers/thunderbolt/test.c @@ -2907,7 +2907,45 @@ static void tb_test_dp_host_teardown_unannounced(struct kunit *test) KUNIT_EXPECT_FALSE(test, tunnel->host_dp_activated); } +static void tb_test_tunnel_dp_host_credits(struct kunit *test) +{ + struct tb_switch *host, *dev; + struct tb_tunnel *tunnel; + struct tb_nhi *nhi; + struct tb *tb; + + host = alloc_host(test); + KUNIT_ASSERT_NOT_NULL(test, host); + dev = alloc_dev_with_dpin(test, host, 0x3, true); + KUNIT_ASSERT_NOT_NULL(test, dev); + tb = kunit_kzalloc(test, sizeof(*tb), GFP_KERNEL); + nhi = kunit_kzalloc(test, sizeof(*nhi), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tb); + KUNIT_ASSERT_NOT_NULL(test, nhi); + tb->nhi = nhi; + host->tb = tb; + dev->tb = tb; + + KUNIT_EXPECT_FALSE(test, tb_port_needs_host_dp_credits(&host->ports[5])); + /* j416s has no dp_tunnel_changed hook; the quirk must be sufficient. */ + nhi->quirks = QUIRK_HOST_DP_NFC_CREDITS; + KUNIT_EXPECT_TRUE(test, tb_port_needs_host_dp_credits(&host->ports[5])); + KUNIT_EXPECT_FALSE(test, tb_port_needs_host_dp_credits(&host->ports[1])); + KUNIT_EXPECT_FALSE(test, tb_port_needs_host_dp_credits(&dev->ports[13])); + + dev->ports[14].config.type = TB_TYPE_DP_HDMI_OUT; + tunnel = tb_tunnel_alloc_dp(NULL, &host->ports[5], &dev->ports[14], + 1, 0, 0, NULL, NULL); + KUNIT_ASSERT_NOT_NULL(test, tunnel); + KUNIT_EXPECT_EQ(test, tunnel->paths[0]->hops[0].nfc_credits, 5U); + tb_tunnel_put(tunnel); + + host->tb = NULL; + KUNIT_EXPECT_FALSE(test, tb_port_needs_host_dp_credits(&host->ports[5])); +} + static struct kunit_case tb_test_cases[] = { + KUNIT_CASE(tb_test_tunnel_dp_host_credits), KUNIT_CASE(tb_test_dp_host_teardown_once), KUNIT_CASE(tb_test_dp_host_teardown_unannounced), KUNIT_CASE(tb_test_path_basic), diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index 0eb6b9e5b1f7c..64024a94306dd 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -1687,9 +1687,7 @@ static int tb_dp_init_video_credits(struct tb_path_hop *hop) * Apple silicon host DP IN adapters take 5 NFC credits for the * video path -- same source as the HPD/NO_AUTO_LT handling above. */ - if (tb_port_is_apple_host_dpin(port) || - (port->sw->tb && tb_nhi_is_apple(port->sw->tb->nhi) && - !tb_route(port->sw) && tb_port_is_dpin(port))) { + if (tb_port_needs_host_dp_credits(port)) { hop->nfc_credits = 5; return 0; } From 55d102be85c250fcc4803ddb520695bea341113f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:00:22 -0700 Subject: [PATCH 0255/1579] drm/apple: drain display retries before stopping firmware endpoints Identify each Type-C connection when queuing a placeholder EDID retry and serialize HPD pulses with disconnect. Drain stale retries before reusing a route. Disable delayed display work before endpoint shutdown or suspend, then balance the enable on bind and resume. Keep work disabled before the component binds and retry a cable already routed at boot. Use the selected connection timeout when reporting elapsed time. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp.c | 96 ++++++++++++++++++---------- 2 files changed, 64 insertions(+), 34 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 46b9fa7c87ec6..b6844e04db00a 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -219,6 +219,8 @@ struct apple_dcp { bool valid_mode; /* One HPD pulse after a placeholder EDID, per Type-C connection. */ bool placeholder_retried; + u64 typec_generation; /* hpd_mutex: identifies the current connection */ + u64 placeholder_generation; struct delayed_work placeholder_edid_wq; bool use_timestamps; bool vrr_enabled; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d879a8ae5dc95..75a5d6f83d0ba 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1294,6 +1294,7 @@ bool dcp_has_typec_routes(struct platform_device *pdev) static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { + unsigned long timeout; int ret = 0; if (!dcp->phy) { @@ -1365,10 +1366,10 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) } mutex_unlock(&dcp->hpd_mutex); + timeout = dcp_is_usb4_output(dcp) && of_machine_is_compatible("apple,j416s") ? + DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT; ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, - dcp_is_usb4_output(dcp) && - of_machine_is_compatible("apple,j416s") ? - DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT); + timeout); if (!ret) { dev_err(dcp->dev, "dcp_dptx_connect: timed out waiting for port %u link configuration\n", @@ -1378,7 +1379,7 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) } dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", - jiffies_to_msecs(DPTX_CONNECT_TIMEOUT - ret)); + jiffies_to_msecs(timeout - ret)); usleep_range(5, 10); @@ -1422,12 +1423,15 @@ static bool dcp_edid_is_placeholder(const struct drm_edid *drm_edid) void dcp_retry_placeholder_edid(struct apple_dcp *dcp, const struct drm_edid *drm_edid) { - if (!dcp_is_typec_output(dcp) || dcp->placeholder_retried) + guard(mutex)(&dcp->hpd_mutex); + if (!dcp_is_typec_output(dcp) || !dcp->typec_cable_connected || + dcp->placeholder_retried) return; if (!dcp_edid_is_placeholder(drm_edid)) return; dcp->placeholder_retried = true; + dcp->placeholder_generation = dcp->typec_generation; schedule_delayed_work(&dcp->placeholder_edid_wq, msecs_to_jiffies(300)); } @@ -1437,16 +1441,15 @@ static void dcp_placeholder_edid_work(struct work_struct *work) container_of(to_delayed_work(work), struct apple_dcp, placeholder_edid_wq); struct apple_epic_service *service; + u64 generation; int ret; - if (!READ_ONCE(dcp->typec_cable_connected)) - return; - mutex_lock(&dcp->hpd_mutex); - service = dcp->dptxport[0].connected ? dcp->dptxport[0].service : NULL; - mutex_unlock(&dcp->hpd_mutex); - if (!service) - return; + generation = dcp->placeholder_generation; + if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || + !dcp->dptxport[0].enabled || generation != dcp->typec_generation) + goto out_unlock; + service = dcp->dptxport[0].service; /* * Some adapters answer the first connection with a 1024x768 @@ -1457,18 +1460,23 @@ static void dcp_placeholder_edid_work(struct work_struct *work) if (ret) { dev_info(dcp->dev, "placeholder EDID: HPD drop failed: %d\n", ret); - return; + goto out_unlock; } + mutex_unlock(&dcp->hpd_mutex); msleep(1000); - if (!READ_ONCE(dcp->typec_cable_connected)) - return; + mutex_lock(&dcp->hpd_mutex); + if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || + generation != dcp->typec_generation) + goto out_unlock; ret = dptxport_set_hpd(service, true); if (ret) dev_info(dcp->dev, "placeholder EDID: HPD assert failed: %d\n", ret); +out_unlock: + mutex_unlock(&dcp->hpd_mutex); } static void dcp_typec_reconnect_work(struct work_struct *work) @@ -1529,11 +1537,13 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) int ret; if (dcp_is_typec_output(dcp)) { + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->hpd_mutex); + dcp->typec_generation++; WRITE_ONCE(dcp->typec_cable_connected, true); dcp->typec_reconnect_tries = 0; dcp->placeholder_retried = false; cancel_delayed_work(&dcp->typec_reconnect_wq); - cancel_delayed_work(&dcp->placeholder_edid_wq); } ret = dcp_dptx_connect(dcp, port); @@ -1549,10 +1559,13 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) struct apple_dcp *dcp = platform_get_drvdata(pdev); if (dcp_is_typec_output(dcp)) { - WRITE_ONCE(dcp->typec_cable_connected, false); + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, false); + dcp->typec_generation++; + } WRITE_ONCE(dcp->typec_crtc_off, false); - cancel_delayed_work(&dcp->typec_reconnect_wq); - cancel_delayed_work(&dcp->placeholder_edid_wq); + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); } disconnected_hpd_event(dcp->connector); @@ -2159,6 +2172,28 @@ static int dcp_connector_type_from_dt(struct device_node *np) return DRM_MODE_CONNECTOR_Unknown; } +static void dcp_disable_typec_work(struct apple_dcp *dcp) +{ + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, false); + dcp->typec_generation++; + } + /* Block new enqueues as well as draining users of the AFK endpoints. */ + disable_delayed_work_sync(&dcp->typec_reconnect_wq); + disable_delayed_work_sync(&dcp->placeholder_edid_wq); + disable_delayed_work_sync(&dcp->typec_fabric_retrain_wq); +} + +static void dcp_enable_typec_work(struct apple_dcp *dcp) +{ + enable_delayed_work(&dcp->typec_reconnect_wq); + enable_delayed_work(&dcp->placeholder_edid_wq); + enable_delayed_work(&dcp->typec_fabric_retrain_wq); + /* A cable can be routed before the DRM component binds. */ + if (READ_ONCE(dcp->typec_cable_connected)) + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); +} + static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; @@ -2257,6 +2292,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (ret) return dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); + dcp_enable_typec_work(dcp); return ret; } @@ -2274,6 +2310,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->hdmi_hpd_irq) disable_irq(dcp->hdmi_hpd_irq); + dcp_disable_typec_work(dcp); typec_mux_put(dcp->typec_mux); if (dcp->avep) { @@ -2315,9 +2352,6 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); } - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->placeholder_edid_wq); - cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); cancel_work_sync(&dcp->vblank_wq); devm_clk_put(dev, dcp->clk); @@ -2375,6 +2409,10 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp_placeholder_edid_work); INIT_DELAYED_WORK(&dcp->typec_fabric_retrain_wq, dcp_typec_retrain_work); + /* Balanced by enable at successful component bind. */ + disable_delayed_work(&dcp->typec_reconnect_wq); + disable_delayed_work(&dcp->placeholder_edid_wq); + disable_delayed_work(&dcp->typec_fabric_retrain_wq); platform_set_drvdata(pdev, dcp); @@ -2514,14 +2552,6 @@ static void dcp_platform_remove(struct platform_device *pdev) static void dcp_platform_shutdown(struct platform_device *pdev) { - struct apple_dcp *dcp = platform_get_drvdata(pdev); - - if (dcp) { - WRITE_ONCE(dcp->typec_cable_connected, false); - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->placeholder_edid_wq); - cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); - } component_del(&pdev->dev, &dcp_comp_ops); } @@ -2529,10 +2559,7 @@ static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); - WRITE_ONCE(dcp->typec_cable_connected, false); - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->placeholder_edid_wq); - cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + dcp_disable_typec_work(dcp); if (dcp->avep) av_service_disconnect(dcp); @@ -2558,6 +2585,7 @@ static int dcp_platform_resume(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + dcp_enable_typec_work(dcp); if (dcp->hdmi_hpd_irq) enable_irq(dcp->hdmi_hpd_irq); From df2f14f1565ee6aba0fddf6720ea6e13b2b1b0ff Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:00:22 -0700 Subject: [PATCH 0256/1579] PCI: apple: preserve failed tunnel recovery and optional DART builds Leave a failed restart quiesced and retryable instead of rescanning and clearing bus_stopped after a reset or link timeout. Resume DART commands only after the ports have restarted. Allow DART-disabled builds through no-op stubs, and constrain Apple PCIe to a module when its optional DART provider is modular. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/Kconfig | 1 + drivers/pci/controller/pcie-apple.c | 13 ++++++------- include/linux/soc/apple/dart.h | 7 +++++++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index e3bb6ff1b6d71..307d75ff9868c 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -43,6 +43,7 @@ config PCIE_APPLE_MSI_DOORBELL_ADDR config PCIE_APPLE tristate "Apple PCIe controller" depends on ARCH_APPLE || COMPILE_TEST + depends on APPLE_DART || !APPLE_DART depends on OF depends on PCI_MSI depends on PAGE_SIZE_16KB || COMPILE_TEST diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 84ccbb606b3de..697f2e86cfe5f 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2119,14 +2119,13 @@ int apple_pcie_tunnel_restore(struct device *dev) return 0; list_for_each_entry(port, &pcie->ports, entry) { - int err = apple_pcie_tunnel_start(port); - - if (err && !ret) - ret = err; - if (!err) - apple_pcie_walk_tunnel_darts(pcie, - apple_dart_resume_commands); + if (port->started) + continue; + ret = apple_pcie_tunnel_start(port); + if (ret) + return ret; } + apple_pcie_walk_tunnel_darts(pcie, apple_dart_resume_commands); pci_lock_rescan_remove(); pci_rescan_bus(bridge->bus); diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index 9902e602c5d11..8fca017414d56 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -2,13 +2,20 @@ #ifndef __LINUX_SOC_APPLE_DART_H__ #define __LINUX_SOC_APPLE_DART_H__ +#include + struct device; /* * The tunneled PCIe IOMMU shares the port clock. Stop issuing commands * before that clock is gated, and allow them again once it is back. */ +#if IS_ENABLED(CONFIG_APPLE_DART) void apple_dart_quiesce_commands(struct device *dev); void apple_dart_resume_commands(struct device *dev); +#else +static inline void apple_dart_quiesce_commands(struct device *dev) { } +static inline void apple_dart_resume_commands(struct device *dev) { } +#endif #endif From 90927e2f84198928341525cea0e5d47ab815566d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:00:22 -0700 Subject: [PATCH 0257/1579] phy: apple: atc: allow owned T602X tunnel clocks to change rate DCP can request a new nonzero link rate during retraining. Restore the descriptor owned by this tunnel and start the requested rate through the guarded T602X sequence instead of permanently refusing it as busy. Keep rejecting clocks owned by another user. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/phy/apple/atc.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index e77a9abd58fdc..025855e84910f 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2262,10 +2262,10 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) return -EINVAL; } if (atcphy->tunnel_saved) { - if (atcphy->tunnel_rate != rate) - dev_info(atcphy->dev, "USB4 tunnel clock busy: active rate=0x%x requested=0x%x\n", - atcphy->tunnel_rate, rate); - return atcphy->tunnel_rate == rate ? 0 : -EBUSY; + if (atcphy->tunnel_rate == rate) + return 0; + /* A retrain may change rate without first requesting rate zero. */ + atc_tunnel_restore(atcphy); } if (atcphy->tunnel_attempted) return -EALREADY; From 4d1ab9bba950cff361b88ac1f658d63a4b462408 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:04:12 -0700 Subject: [PATCH 0258/1579] drm/apple: preserve unbound pipeline ranking for j416s Type-C routes are registered before the DRM component binds. Their score is UINT_MAX-1 until a CRTC exists; adding the j416s preference penalty wraps it to 98 and can select an unbound, nonpreferred pipeline first. Keep the sentinel score intact when applying the preference. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/dcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 75a5d6f83d0ba..db5f1127f8049 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -657,7 +657,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, score = dcp_typec_route_score(candidate); /* j416s dcpext0 completes tunneled link training; prefer it. */ if (of_machine_is_compatible("apple,j416s") && - !candidate->dcp->fixed_phy) + !candidate->dcp->fixed_phy && score < UINT_MAX - 100) score += 100; if (score < best_score) { best = candidate; From 8478ae0142014d170a62df6dd94cc66bd989627b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 23:27:26 -0700 Subject: [PATCH 0259/1579] iommu/apple-dart: restore legacy locked display domain handoff Booting the T8140 DART changes on an M1 j293 fails the display DART reservations with -EIO before attach_dev(). Domain finalization copies prospective stream IDs into the active bitmap, so the new strict root synchronization runs before the locked roots have been mapped. Leave the active stream bitmap empty until attachment, and skip TLB synchronization and teardown for domains with no attached streams. The legacy M1/M2 display handoff also requires replacing the firmware root with Linux's page tables after the IOMMU core reconstructs reserved mappings. Requiring vacant root slots instead rejects this handoff with -EBUSY. Restore replacement for legacy DARTs, tracking the installed entries for teardown, while retaining firmware-owned slot protection for T8110 version 2.2 and newer. Add KUnit coverage for pre-attachment reserved mapping, legacy root handoff, and preservation/conflict handling on newer firmware roots. The pre-attachment and handoff tests fail on the unmodified driver and pass with this fix; all 42 Thunderbolt tests also pass. Fixes: 79379798f12f ("iommu/apple-dart: Support T8140 DARTs and firmware-owned stream roots") Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- MAINTAINERS | 1 + drivers/iommu/Kconfig | 10 +++ drivers/iommu/apple-dart-test.c | 146 ++++++++++++++++++++++++++++++++ drivers/iommu/apple-dart.c | 33 +++++++- 4 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 drivers/iommu/apple-dart-test.c diff --git a/MAINTAINERS b/MAINTAINERS index 368bf2b43af1c..235c46ff97605 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2601,6 +2601,7 @@ F: drivers/i2c/busses/i2c-pasemi-core.c F: drivers/i2c/busses/i2c-pasemi-platform.c F: drivers/input/misc/macsmc-input.c F: drivers/input/touchscreen/apple_z2.c +F: drivers/iommu/apple-dart-test.c F: drivers/iommu/apple-dart.c F: drivers/iommu/io-pgtable-dart.c F: drivers/irqchip/irq-apple-aic.c diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 8b0e5505516a8..43c03b46f43d2 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -316,6 +316,16 @@ config APPLE_DART Say Y here if you are using an Apple SoC. +config APPLE_DART_KUNIT_TEST + bool "KUnit tests for Apple DART locked domains" if !KUNIT_ALL_TESTS + depends on APPLE_DART=y && KUNIT=y + default KUNIT_ALL_TESTS + help + Test firmware mapping before attachment and locked page-table + handoff without accessing physical DART hardware. Verify that + legacy display roots can be replaced while newer firmware-owned + roots retain their mappings and reject conflicting host entries. + config S390_IOMMU def_bool y if S390 && PCI depends on S390 && PCI diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c new file mode 100644 index 0000000000000..7f8888bf74115 --- /dev/null +++ b/drivers/iommu/apple-dart-test.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by apple-dart.c to exercise the locked-domain lifecycle. */ +#include +#include + +static void apple_dart_test_before_attach(struct kunit *test) +{ + struct apple_dart_master_cfg master = {}; + struct apple_dart_domain *domain; + struct apple_dart *dart; + struct io_pgtable_ops *ops; + size_t mapped = 0; + int ret; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + domain = kunit_kzalloc(test, sizeof(*domain), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, domain); + dart->dev = kunit_device_register(test, "apple-dart-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); + dart->hw = &apple_dart_hw_t8103; + dart->pgsize = PAGE_SIZE; + dart->ias = 32; + dart->oas = 36; + dart->dma_max = DMA_BIT_MASK(32); + dart->num_streams = 16; + dart->locked = true; + dart->regs = (__force void __iomem *)kunit_kzalloc(test, SZ_16K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart->regs); + spin_lock_init(&dart->lock); + mutex_init(&domain->init_lock); + master.stream_maps[0].dart = dart; + __set_bit(0, master.stream_maps[0].sidmap); + writel(dart->hw->ttbr_valid, dart->regs + DART_TTBR(dart, 0, 0)); + KUNIT_ASSERT_EQ(test, apple_dart_finalize_domain(domain, &master), 0); + ops = domain->pgtbl_ops; + pm_runtime_set_active(dart->dev); + pm_runtime_enable(dart->dev); + + /* The IOMMU core installs firmware reservations before attach_dev(). */ + ret = apple_dart_map_pages(&domain->domain, 0, SZ_1M, PAGE_SIZE, 1, + IOMMU_READ | IOMMU_WRITE, GFP_KERNEL, &mapped); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mapped, (size_t)PAGE_SIZE); + KUNIT_EXPECT_EQ(test, apple_dart_iotlb_sync_map(&domain->domain, 0, PAGE_SIZE), 0); + KUNIT_EXPECT_EQ(test, ops->iova_to_phys(ops, 0), (phys_addr_t)SZ_1M); + KUNIT_EXPECT_EQ(test, atomic_long_read(&domain->stream_maps[0].sidmap[0]), 0L); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), 0U); + + /* Once attached, missing root mappings must still report an error. */ + KUNIT_EXPECT_EQ(test, apple_dart_domain_add_streams(domain, &master), 0); + KUNIT_EXPECT_EQ(test, apple_dart_iotlb_sync_map(&domain->domain, 0, PAGE_SIZE), -EIO); + + pm_runtime_disable(dart->dev); + free_io_pgtable_ops(ops); +} + +static void apple_dart_test_locked_handoff(struct kunit *test) +{ + struct apple_dart_stream_map stream = {}; + struct io_pgtable_cfg cfg = {}; + struct apple_dart *dart; + u64 live[] = { 0x1001, 0, 0x3001, 0 }; + u64 owned[ARRAY_SIZE(live)] = {}; + u64 ours[] = { 0x4001, 0x5001, 0, 0 }; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->hw = &apple_dart_hw_t8103; + dart->num_streams = 1; + dart->pgsize = sizeof(live); + spin_lock_init(&dart->lock); + dart->locked_ttbr[0][0] = live; + dart->locked_owned[0][0] = owned; + stream.dart = dart; + __set_bit(0, stream.sidmap); + cfg.apple_dart_cfg.n_ttbrs = 1; + cfg.apple_dart_cfg.ttbr[0] = ours; + + /* Legacy firmware mappings are rebuilt by the IOMMU core at handoff. */ + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), 0); + KUNIT_EXPECT_MEMEQ(test, live, ours, sizeof(live)); + KUNIT_EXPECT_MEMEQ(test, owned, ours, sizeof(owned)); + apple_dart_retire_root(live, owned, ARRAY_SIZE(live)); + KUNIT_EXPECT_EQ(test, live[0], 0ULL); + KUNIT_EXPECT_EQ(test, live[1], 0ULL); + + /* Older T8110 instances also use the legacy display handoff. */ + dart->hw = &apple_dart_hw_t8110; + dart->version = 0x0200; + live[0] = 0x1001; + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), 0); + KUNIT_EXPECT_MEMEQ(test, live, ours, sizeof(live)); +} + +static void apple_dart_test_firmware_roots(struct kunit *test) +{ + struct apple_dart_stream_map stream = {}; + struct io_pgtable_cfg cfg = {}; + struct apple_dart *dart; + u64 live[] = { 0x1001, 0, 0, 0 }; + u64 owned[ARRAY_SIZE(live)] = {}; + u64 ours[] = { 0, 0x2001, 0, 0 }; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->hw = &apple_dart_hw_t8110; + dart->version = 0x0202; + dart->num_streams = 1; + dart->pgsize = sizeof(live); + spin_lock_init(&dart->lock); + dart->locked_ttbr[0][0] = live; + dart->locked_owned[0][0] = owned; + stream.dart = dart; + __set_bit(0, stream.sidmap); + cfg.apple_dart_cfg.n_ttbrs = 1; + cfg.apple_dart_cfg.ttbr[0] = ours; + + KUNIT_ASSERT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), 0); + KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); + KUNIT_EXPECT_EQ(test, live[1], 0x2001ULL); + ours[0] = 0x3001; + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), -EBUSY); + dart->version = 0x0203; + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), -EBUSY); + KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); + /* Teardown must preserve a root which firmware has since replaced. */ + live[1] = 0x4001; + apple_dart_retire_root(live, owned, ARRAY_SIZE(live)); + KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); + KUNIT_EXPECT_EQ(test, live[1], 0x4001ULL); +} + +static struct kunit_case apple_dart_test_cases[] = { + KUNIT_CASE(apple_dart_test_before_attach), + KUNIT_CASE(apple_dart_test_locked_handoff), + KUNIT_CASE(apple_dart_test_firmware_roots), + {} +}; + +static struct kunit_suite apple_dart_test_suite = { + .name = "apple-dart", + .test_cases = apple_dart_test_cases, +}; + +kunit_test_suite(apple_dart_test_suite); diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index a865ebb2bce8f..277c5a589f149 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -623,6 +623,25 @@ apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, ret = -EIO; goto out; } + /* + * Legacy display DARTs hand their locked root to Linux. + * The IOMMU core has rebuilt the firmware reservations + * before attachment, so replace the old root entries. + * T8110 v2.2+ instead retains firmware-owned entries. + */ + if (dart->hw->type != DART_T8110 || dart->version < 0x0202) { + size_t entry; + + dma_wmb(); + for (entry = 0; entry < dart->pgsize / sizeof(*live); entry++) { + u64 next = READ_ONCE(ours[entry]); + + WRITE_ONCE(live[entry], next); + WRITE_ONCE(owned[entry], next); + } + dma_wmb(); + continue; + } ret = apple_dart_publish_root(live, owned, ours, dart->pgsize / sizeof(*live)); if (ret) @@ -893,6 +912,8 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, stream.dart = map->dart; for (j = 0; j < BITS_TO_LONGS(stream.dart->num_streams); j++) stream.sidmap[j] = atomic_long_read(&map->sidmap[j]); + if (bitmap_empty(stream.sidmap, stream.dart->num_streams)) + continue; ret = pm_runtime_resume_and_get(stream.dart->dev); if (ret < 0) { dev_err_ratelimited(stream.dart->dev, @@ -1080,11 +1101,11 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, if (dart_domain->finalized) goto done; + /* Reserved mappings may be installed before any device is attached. */ for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart; - for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++) - atomic_long_set(&dart_domain->stream_maps[i].sidmap[j], - cfg->stream_maps[i].sidmap[j]); + for (j = 0; j < BITS_TO_LONGS(DART_MAX_STREAMS); j++) + atomic_long_set(&dart_domain->stream_maps[i].sidmap[j], 0); } pgtbl_cfg = (struct io_pgtable_cfg){ @@ -1389,6 +1410,8 @@ static void apple_dart_domain_free(struct iommu_domain *domain) stream.dart = map->dart; for (j = 0; j < BITS_TO_LONGS(stream.dart->num_streams); j++) stream.sidmap[j] = atomic_long_read(&map->sidmap[j]); + if (bitmap_empty(stream.sidmap, stream.dart->num_streams)) + continue; j = pm_runtime_resume_and_get(stream.dart->dev); if (j < 0) { dev_err_ratelimited(stream.dart->dev, @@ -2440,6 +2463,10 @@ static struct platform_driver apple_dart_driver = { module_platform_driver(apple_dart_driver); +#if IS_ENABLED(CONFIG_APPLE_DART_KUNIT_TEST) +#include "apple-dart-test.c" +#endif + MODULE_DESCRIPTION("IOMMU API for Apple's DART"); MODULE_AUTHOR("Sven Peter "); MODULE_LICENSE("GPL v2"); From 0db6bd7716e718cb6b878fde29f7a57aa629f025 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 23:51:53 -0700 Subject: [PATCH 0260/1579] thunderbolt: preserve PCIe host ownership across system resume On M1 with a TS4 and igc bound, resume discovers a temporary PCIe tunnel and deactivates it before restoring the saved tunnels. Its host hook removes the live PCI hierarchy while endpoint drivers are suspended, triggering an igc double-disable warning. Deferred activation can also rescan on system_wq before the PCI host and DART have resumed, producing a sleeping-parent warning and a DART translation fault. Track host ownership only after invoking the PCIe post-activation hook. Discovered tunnels cannot tear down another tunnel's host state, repeated teardown is harmless, and failed cleanup retains ownership for retry. Add a KUnit regression covering these cases; it fails before the fix. On Apple hosts, defer PCI removal during system PM until endpoint drivers have resumed, then process pending teardown before activation. Use a freezable workqueue and prepare/complete gating so topology changes cannot race noirq resume. Record activation only after the adapters are enabled, cancel obsolete requests on teardown, and propagate restore errors. All 46 DART/Thunderbolt KUnit tests and native W=1 Apple Thunderbolt and tunnel object builds pass. Hardware suspend/resume retesting is pending. Fixes: f6d4f47575f3 ("thunderbolt: Add a PCI tunnel deactivate callback") Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/apple.c | 98 +++++++++++++++++++++++------------- drivers/thunderbolt/test.c | 58 +++++++++++++++++++++ drivers/thunderbolt/tunnel.c | 22 +++++--- drivers/thunderbolt/tunnel.h | 3 ++ 4 files changed, 139 insertions(+), 42 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 0570b811fab6b..6fe43773b7cd3 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -164,6 +164,8 @@ struct apple_cio { struct delayed_work pcie_tunnel_work; bool pcie_tunnel_requested; bool pcie_tunnel_populated; + bool pcie_pm_prepared; + bool pcie_quiesce_pending; /* Type-C connector this router is wired to, for DP tunnel routing */ struct device_node *connector_np; @@ -841,7 +843,7 @@ static int apple_cio_populate_pcie_tunnel(struct apple_cio *acio) static int apple_cio_activate_pcie_tunnel_locked(struct apple_cio *acio) { - int ret; + int ret = 0; lockdep_assert_held(&acio->pcie_tunnel_lock); @@ -864,7 +866,7 @@ static int apple_cio_activate_pcie_tunnel_locked(struct apple_cio *acio) ret); put_device(&pcie_pdev->dev); } - return 0; + return ret; } if (!acio->pcie_tunnel_preinitialized) return dev_err_probe(acio->dev, -ENODEV, @@ -885,31 +887,50 @@ static int apple_cio_activate_pcie_tunnel_locked(struct apple_cio *acio) return 0; } +static int apple_cio_quiesce_pcie_tunnel_locked(struct apple_cio *acio) +{ + struct platform_device *pcie_pdev; + int ret = 0; + + lockdep_assert_held(&acio->pcie_tunnel_lock); + pcie_pdev = apple_cio_find_pcie_tunnel(acio); + if (pcie_pdev) { + ret = apple_pcie_tunnel_quiesce(&pcie_pdev->dev); + put_device(&pcie_pdev->dev); + } + if (!ret) + acio->pcie_quiesce_pending = false; + return ret; +} + static void apple_cio_pcie_tunnel_work(struct work_struct *work) { struct apple_cio *acio = container_of(to_delayed_work(work), struct apple_cio, pcie_tunnel_work); - int ret; + int ret = 0; mutex_lock(&acio->pcie_tunnel_lock); - if (!READ_ONCE(acio->pcie_tunnel_requested)) { - mutex_unlock(&acio->pcie_tunnel_lock); - return; + if (acio->pcie_pm_prepared) + goto unlock; + if (acio->pcie_quiesce_pending) { + ret = apple_cio_quiesce_pcie_tunnel_locked(acio); + if (ret) + goto unlock; } - - ret = apple_cio_activate_pcie_tunnel_locked(acio); + if (READ_ONCE(acio->pcie_tunnel_requested)) + ret = apple_cio_activate_pcie_tunnel_locked(acio); +unlock: mutex_unlock(&acio->pcie_tunnel_lock); if (ret && ret != -EAGAIN) - dev_err(acio->dev, "deferred PCIe-C activation failed: %d\n", ret); + dev_err(acio->dev, "deferred PCIe-C transition failed: %d\n", ret); } static int apple_nhi_pci_tunnel_deactivate(struct tb_nhi *nhi) { struct apple_nhi *anhi = nhi_to_anhi(nhi); struct apple_cio *acio = anhi->acio; - struct platform_device *pcie_pdev; - int ret; + int ret = 0; /* * The tunnel is going away while the router and NHI stay up, so nothing @@ -919,37 +940,21 @@ static int apple_nhi_pci_tunnel_deactivate(struct tb_nhi *nhi) * nothing to enumerate onto when the tunnel comes back. */ mutex_lock(&acio->pcie_tunnel_lock); - if (!acio->pcie_tunnel_populated) { - mutex_unlock(&acio->pcie_tunnel_lock); - return 0; - } - - pcie_pdev = apple_cio_find_pcie_tunnel(acio); - if (pcie_pdev) { - ret = apple_pcie_tunnel_quiesce(&pcie_pdev->dev); + WRITE_ONCE(acio->pcie_tunnel_requested, false); + cancel_delayed_work(&acio->pcie_tunnel_work); + acio->pcie_quiesce_pending = true; + /* Endpoint drivers must resume before they can be unbound safely. */ + if (!acio->pcie_pm_prepared) { + ret = apple_cio_quiesce_pcie_tunnel_locked(acio); if (ret) dev_warn(acio->dev, "failed to quiesce PCIe-C on tunnel teardown: %d\n", ret); - put_device(&pcie_pdev->dev); } mutex_unlock(&acio->pcie_tunnel_lock); - return 0; -} - -static int apple_nhi_pci_tunnel_pre_activate(struct tb_nhi *nhi) -{ - struct apple_nhi *anhi = nhi_to_anhi(nhi); - struct apple_cio *acio = anhi->acio; - - /* - * Remember that the PCIe tunnel reached activation. PCIe-C must not be - * populated here: the tunnel paths and both adapters are still disabled. - */ - WRITE_ONCE(acio->pcie_tunnel_requested, true); - return 0; + return ret; } static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) @@ -963,7 +968,11 @@ static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) * Thunderbolt core before probing PCIe-C. The worker then waits on the * hardware reset state without blocking completion of the tunnel paths. */ - mod_delayed_work(system_wq, &acio->pcie_tunnel_work, 0); + mutex_lock(&acio->pcie_tunnel_lock); + WRITE_ONCE(acio->pcie_tunnel_requested, true); + /* No PCI removal or rescan until system resume has finished. */ + mod_delayed_work(system_freezable_wq, &acio->pcie_tunnel_work, 0); + mutex_unlock(&acio->pcie_tunnel_lock); return 0; } @@ -1454,7 +1463,6 @@ static const struct tb_nhi_ops apple_nhi_ops = { .ring_interrupt_active = apple_nhi_ring_interrupt_active, .ring_interrupt_mask = apple_nhi_ring_interrupt_mask, .ring_configure = apple_nhi_ring_configure, - .pci_tunnel_pre_activate = apple_nhi_pci_tunnel_pre_activate, .pci_tunnel_post_activate = apple_nhi_pci_tunnel_post_activate, .pci_tunnel_deactivate = apple_nhi_pci_tunnel_deactivate, }; @@ -1647,6 +1655,16 @@ static void apple_nhi_remove(struct platform_device *pdev) * apple_nhi pointer in driver data. The generic NHI PM callbacks cannot be * installed here because they expect driver data to contain struct tb. */ +static int apple_nhi_prepare(struct device *dev) +{ + struct apple_nhi *anhi = dev_get_drvdata(dev); + struct apple_cio *acio = anhi->acio; + + guard(mutex)(&acio->pcie_tunnel_lock); + acio->pcie_pm_prepared = true; + return 0; +} + static int apple_nhi_suspend_noirq(struct device *dev) { struct apple_nhi *anhi = dev_get_drvdata(dev); @@ -1685,11 +1703,18 @@ static int apple_nhi_suspend(struct device *dev) static void apple_nhi_complete(struct device *dev) { struct apple_nhi *anhi = dev_get_drvdata(dev); + struct apple_cio *acio = anhi->acio; tb_domain_complete(anhi->tb); + + guard(mutex)(&acio->pcie_tunnel_lock); + acio->pcie_pm_prepared = false; + if (acio->pcie_quiesce_pending || READ_ONCE(acio->pcie_tunnel_requested)) + mod_delayed_work(system_freezable_wq, &acio->pcie_tunnel_work, 0); } static const struct dev_pm_ops apple_nhi_pm_ops = { + .prepare = apple_nhi_prepare, .suspend = apple_nhi_suspend, .suspend_noirq = apple_nhi_suspend_noirq, .resume_noirq = apple_nhi_resume_noirq, @@ -1787,6 +1812,7 @@ static void apple_cio_stop(struct apple_cio *acio) of_platform_depopulate(acio->dev); acio->pcie_tunnel_populated = false; + acio->pcie_quiesce_pending = false; mutex_unlock(&acio->pcie_tunnel_lock); /* Try to shut down and power off the co-processor gracefully */ diff --git a/drivers/thunderbolt/test.c b/drivers/thunderbolt/test.c index fa58a1b0a921a..e7974981281f2 100644 --- a/drivers/thunderbolt/test.c +++ b/drivers/thunderbolt/test.c @@ -2852,6 +2852,63 @@ static void tb_test_property_copy(struct kunit *test) tb_property_free_dir(src); } +struct tb_test_pci_host { + struct tb_nhi nhi; + unsigned int disconnects; + int result; +}; + +static int tb_test_pci_host_disconnect(struct tb_nhi *nhi) +{ + struct tb_test_pci_host *host = container_of(nhi, struct tb_test_pci_host, nhi); + + host->disconnects++; + return host->result; +} + +static void tb_test_pci_host_teardown(struct kunit *test) +{ + static const struct tb_nhi_ops ops = { + .pci_tunnel_deactivate = tb_test_pci_host_disconnect, + }; + struct tb_test_pci_host *host; + struct tb_tunnel *tunnel; + struct tb *tb; + + host = kunit_kzalloc(test, sizeof(*host), GFP_KERNEL); + tunnel = kunit_kzalloc(test, sizeof(*tunnel), GFP_KERNEL); + tb = kunit_kzalloc(test, sizeof(*tb), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, host); + KUNIT_ASSERT_NOT_NULL(test, tunnel); + KUNIT_ASSERT_NOT_NULL(test, tb); + host->nhi.ops = &ops; + tb->nhi = &host->nhi; + tunnel->tb = tb; + + /* A discovered firmware tunnel does not own Linux's PCIe hierarchy. */ + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(tunnel), 0); + KUNIT_EXPECT_EQ(test, host->disconnects, 0U); + + /* An activated tunnel must release the hierarchy exactly once. */ + tunnel->host_pci_activated = true; + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(tunnel), 0); + KUNIT_EXPECT_EQ(test, host->disconnects, 1U); + KUNIT_EXPECT_FALSE(test, tunnel->host_pci_activated); + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(tunnel), 0); + KUNIT_EXPECT_EQ(test, host->disconnects, 1U); + + /* Failed teardown keeps ownership so a later attempt can retry. */ + tunnel->host_pci_activated = true; + host->result = -EIO; + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(tunnel), -EIO); + KUNIT_EXPECT_TRUE(test, tunnel->host_pci_activated); + KUNIT_EXPECT_EQ(test, host->disconnects, 2U); + host->result = 0; + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(tunnel), 0); + KUNIT_EXPECT_FALSE(test, tunnel->host_pci_activated); + KUNIT_EXPECT_EQ(test, host->disconnects, 3U); +} + struct tb_test_dp_host { struct tb_nhi nhi; unsigned int disconnects; @@ -2945,6 +3002,7 @@ static void tb_test_tunnel_dp_host_credits(struct kunit *test) } static struct kunit_case tb_test_cases[] = { + KUNIT_CASE(tb_test_pci_host_teardown), KUNIT_CASE(tb_test_tunnel_dp_host_credits), KUNIT_CASE(tb_test_dp_host_teardown_once), KUNIT_CASE(tb_test_dp_host_teardown_unannounced), diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index 64024a94306dd..3feabf2f449fe 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -361,18 +361,28 @@ static int tb_pci_post_activate(struct tb_tunnel *tunnel) { const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; - if (ops && ops->pci_tunnel_post_activate) + if (ops && ops->pci_tunnel_post_activate) { + /* Also unwind a hook that fails after partial host setup. */ + tunnel->host_pci_activated = true; return ops->pci_tunnel_post_activate(tunnel->tb->nhi); + } return 0; } -static int tb_pci_deactivate(struct tb_tunnel *tunnel) +int tb_pci_tunnel_deactivate_host(struct tb_tunnel *tunnel) { - const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; + const struct tb_nhi_ops *ops; + int ret = 0; + /* Firmware tunnels discovered during resume do not own host state. */ + if (!tunnel->host_pci_activated) + return 0; + ops = tunnel->tb->nhi->ops; if (ops && ops->pci_tunnel_deactivate) - return ops->pci_tunnel_deactivate(tunnel->tb->nhi); - return 0; + ret = ops->pci_tunnel_deactivate(tunnel->tb->nhi); + if (!ret) + tunnel->host_pci_activated = false; + return ret; } static int tb_pci_activate(struct tb_tunnel *tunnel, bool activate) @@ -384,7 +394,7 @@ static int tb_pci_activate(struct tb_tunnel *tunnel, bool activate) * what is behind it can still be reached. */ if (!activate) { - res = tb_pci_deactivate(tunnel); + res = tb_pci_tunnel_deactivate_host(tunnel); if (res) return res; } diff --git a/drivers/thunderbolt/tunnel.h b/drivers/thunderbolt/tunnel.h index 5a73e60a26c99..a52116ef8e796 100644 --- a/drivers/thunderbolt/tunnel.h +++ b/drivers/thunderbolt/tunnel.h @@ -66,6 +66,7 @@ enum tb_tunnel_state { * @dprx_canceled: Was DPRX capabilities read poll canceled * @host_dp_notified: The NHI glue was told this DP tunnel is up (Apple hosts) * @host_dp_activated: The NHI post-activation hook owns display-side state + * @host_pci_activated: The NHI post-activation hook owns PCIe host state * @dprx_timeout: If set DPRX capabilities read poll work will timeout after this passes * @dprx_work: Worker that is scheduled to poll completion of DPRX capabilities read * @callback: Optional callback called when DP tunnel is fully activated @@ -106,6 +107,7 @@ struct tb_tunnel { bool dprx_canceled; bool host_dp_notified; bool host_dp_activated; + bool host_pci_activated; ktime_t dprx_timeout; struct delayed_work dprx_work; void (*callback)(struct tb_tunnel *tunnel, void *data); @@ -118,6 +120,7 @@ struct tb_tunnel *tb_tunnel_discover_pci(struct tb *tb, struct tb_port *down, bool alloc_hopid); struct tb_tunnel *tb_tunnel_alloc_pci(struct tb *tb, struct tb_port *up, struct tb_port *down); +int tb_pci_tunnel_deactivate_host(struct tb_tunnel *tunnel); bool tb_tunnel_reserved_pci(struct tb_port *port, int *reserved_up, int *reserved_down); struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in, From c777985729268938d3837b6b4be8d6e5be06fe1d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 00:51:47 -0700 Subject: [PATCH 0261/1579] PCI: apple: correct M1 tunneled port restart sequencing The Kensington SD5560T loses its PCIe xHCI hierarchy after s2idle on both ports of j293. Tracing shows all functions left in D0; the host reports link restored immediately before the downstream configuration space becomes inaccessible. A failed software restart also leaves APPCLK enabled while started is false, so later teardown skips the port. Use the M1 activation sequence on retained M1 ports: restore port configuration, APPCLK, PERST and root tunables before releasing tunnel reset. Require a fresh link-up event, retry transient link-down/tunnel errors, and allow downstream reset recovery before resuming config I/O. Wait for M1's reset acknowledgment pending indication to clear, matching Corellium's M1 implementation. Preserve the retained T6020 sequence. Track partially started hardware separately from a successful link so quiesce, teardown and subsequent restore can finish cleanup even after a restart timeout. Preserve the original IRQ mask across such retries. The M1 sequencing changes are a hardware-test candidate; the Kensington suspend/reconnect regression still requires validation on this commit. Link: https://github.com/corellium/linux-m1/blob/master/drivers/pci/controller/pcie-apple-m1.c Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/pcie-apple.c | 79 +++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 3 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 697f2e86cfe5f..1d4358e46ce20 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -247,6 +247,7 @@ struct apple_pcie_port { int sid_map_sz; int idx; bool started; + bool needs_stop; }; static void rmw_set(u32 set, void __iomem *addr) @@ -898,6 +899,66 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) struct apple_pcie *pcie = port->pcie; u32 stat; int i, ret; + u32 link_events = BIT(PORT_INT_LINK_UP) | BIT(PORT_INT_LINK_DOWN) | + BIT(PORT_INT_TUNNEL_ERR); + + /* A failed start still owns partially enabled port hardware. */ + port->needs_stop = true; + + if (pcie->kernel_init && pcie->power_retained) { + /* + * M1 requires the same activation sequence after sleep as after + * cable insertion. In particular, release tunneled reset only + * after APPCLK, PERST and the root-port tunables are restored. + */ + apple_pcie_tunnel_pulse_intr2axi(pcie); + apple_pcie_tunnel_apply_tunable(port->base, port->tunable); + apple_pcie_port_rmw_set(port, PORT_APPCLK_EN, PORT_APPCLK); + usleep_range(10, 20); + apple_pcie_port_rmw_set(port, PORT_PERST_OFF, + pcie->hw->port_perst); + usleep_range(10, 20); + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_STATUS_READY, + 10, 250000, false, port, PORT_STATUS); + if (ret) + return ret; + apple_pcie_tunnel_apply_tunable(pcie->cfg->win, pcie->rc_tunable); + ret = apple_pcie_tunnel_release_reset(port); + if (ret) + return ret; + + /* + * LINKSTS can still contain the pre-suspend link-up indication. + * Require a new link-up event, retrying the LTSSM on a transient + * link-down/tunnel-error event, as during M1 tunnel activation. + */ + for (i = 0; i < 3; i++) { + apple_pcie_port_writel(port, link_events, PORT_INTSTAT); + apple_pcie_port_writel(port, PORT_LTSSMCTL_START, + PORT_LTSSMCTL); + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & link_events, + 1000, 500000, false, port, + PORT_INTSTAT); + if (!ret && (stat & link_events) == BIT(PORT_INT_LINK_UP) && + (apple_pcie_port_readl(port, PORT_LINKSTS) & PORT_LINKSTS_UP)) + break; + } + if (i == 3) { + dev_err(pcie->dev, + "port %pOF tunnel link did not restart (events %#x, link %#x)\n", + port->np, stat, + apple_pcie_port_readl(port, PORT_LINKSTS)); + return -ETIMEDOUT; + } + apple_pcie_port_writel(port, link_events, PORT_INTSTAT); + /* Allow downstream functions to finish reset before config I/O. */ + msleep(100); + if (!(apple_pcie_port_readl(port, PORT_LINKSTS) & PORT_LINKSTS_UP)) + return -ENOLINK; + goto restored; + } if (pcie->power_retained) { /* @@ -967,6 +1028,7 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) port->np); return ret; } +restored: if (pcie->power_retained) apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); @@ -1008,7 +1070,8 @@ static int apple_pcie_tunnel_stop(struct apple_pcie_port *port) u32 stat; int err = 0, ret; - port->saved_intmask = apple_pcie_port_readl(port, PORT_INTMSK); + if (port->started) + port->saved_intmask = apple_pcie_port_readl(port, PORT_INTMSK); apple_pcie_port_writel(port, ~0, PORT_INTMSK); apple_pcie_port_writel(port, ~0, PORT_INTSTAT); apple_pcie_port_writel(port, ~0, PORT_LINKCMDSTS); @@ -1049,7 +1112,10 @@ static int apple_pcie_tunnel_stop(struct apple_pcie_port *port) err = ret; apple_pcie_port_rmw_set(port, PORT_TUNCTRL_PERST_ACK_REQ, PORT_TUNCTRL); + /* M1 reports an outstanding acknowledgment until ACK_PEND clears. */ ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + pcie->kernel_init ? + !(stat & PORT_TUNSTAT_PERST_ACK_PEND) : stat & PORT_TUNSTAT_PERST_ACK_PEND, 1000, 1000000, false, port, PORT_TUNSTAT); if (ret) { @@ -1069,6 +1135,8 @@ static int apple_pcie_tunnel_stop(struct apple_pcie_port *port) apple_pcie_port_rmw_clear(port, PORT_TUNCTRL_PERST_ACK_REQ, PORT_TUNCTRL); port->started = false; + port->needs_stop = apple_pcie_port_readl(port, PORT_STATUS) & + PORT_STATUS_READY; return err; } @@ -1078,7 +1146,7 @@ static void apple_pcie_port_teardown(struct apple_pcie_port *port) if (port->base) apple_pcie_port_writel(port, ~0, PORT_INTMSK); - if (port->pcie->hw->tunneled && port->started) + if (port->pcie->hw->tunneled && (port->started || port->needs_stop)) apple_pcie_tunnel_stop(port); apple_pcie_port_unregister_irqs(port); @@ -2082,7 +2150,7 @@ int apple_pcie_tunnel_quiesce(struct device *dev) list_for_each_entry(port, &pcie->ports, entry) { int err; - if (!port->started) + if (!port->started && !port->needs_stop) continue; err = apple_pcie_tunnel_stop(port); if (err && !ret) @@ -2121,6 +2189,11 @@ int apple_pcie_tunnel_restore(struct device *dev) list_for_each_entry(port, &pcie->ports, entry) { if (port->started) continue; + if (port->needs_stop) { + ret = apple_pcie_tunnel_stop(port); + if (ret) + return ret; + } ret = apple_pcie_tunnel_start(port); if (ret) return ret; From 7bc6d86df69c70a39c55046e4ea97e74d8b87e22 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:03:16 -0700 Subject: [PATCH 0262/1579] PCI: apple: preserve tunneled D0 policy after rescan The initial probe walk leaves tunnel functions in D0 across system sleep, but quiesce removes those pci_dev objects. A later rescan creates new objects without the flag, so a subsequent suspend can put the rebuilt hierarchy into D3. Apply the policy before enabled functions bind and walk the restored hierarchy as well, including functions without a driver. This closes a lifecycle gap; it does not establish recovery of the observed link- training timeout. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/pcie-apple.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 1d4358e46ce20..39493f9cd6fe9 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -1637,6 +1637,10 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d struct apple_pcie_port *port; int idx, err; + /* Also cover functions discovered by a later hotplug or rescan. */ + if (pcie->hw->tunneled) + pdev->dev_flags |= PCI_DEV_FLAGS_NO_D3; + /* * Endpoint BARs share PCIe-C's tunneled, non-posted MMIO fabric. Mark * them before the function driver maps its BAR so pci_iomap() selects @@ -2202,6 +2206,7 @@ int apple_pcie_tunnel_restore(struct device *dev) pci_lock_rescan_remove(); pci_rescan_bus(bridge->bus); + pci_walk_bus(bridge->bus, apple_pcie_tunnel_keep_d0, NULL); pcie->bus_stopped = false; pci_unlock_rescan_remove(); From 9afbe6e41e831cce5a846f031f490652d45737cf Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:03:17 -0700 Subject: [PATCH 0263/1579] PCI: apple: propagate tunneled cold initialization errors Cold initialization currently logs readiness and idle timeouts but always returns success. Its early caller then populates DART even if the port clock did not become ready, although the DART command engine depends on that clock. Return the polling error immediately and retain the last polled register value in the diagnostic. Callers already propagate failures and avoid populating the tunnel children. This changes only the timeout paths. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/pcie-apple.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 39493f9cd6fe9..a9ac823892f79 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -846,17 +846,17 @@ static int apple_pcie_tunnel_cold_init(struct apple_pcie_port *port) stat & PORT_STATUS_READY, 10, 250000, false, port, PORT_STATUS); if (ret) - dev_warn(pcie->dev, - "port %pOF cold init: RUN not set (status %#x)\n", - port->np, apple_pcie_port_readl(port, PORT_STATUS)); + return dev_err_probe(pcie->dev, ret, + "port %pOF cold init: RUN not set (status %#x)\n", + port->np, stat); ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, !(stat & PORT_LINKSTS_BUSY), 10, 250000, false, port, PORT_LINKSTS); if (ret) - dev_warn(pcie->dev, - "port %pOF cold init: link still busy (status %#x)\n", - port->np, apple_pcie_port_readl(port, PORT_LINKSTS)); + return dev_err_probe(pcie->dev, ret, + "port %pOF cold init: link still busy (status %#x)\n", + port->np, stat); dev_info(pcie->dev, "port %pOF cold init done, status %#x link %#x tunstat %#x\n", From c9846a5db10141988997f58f92953c27e8dd934b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:03:41 -0700 Subject: [PATCH 0264/1579] PCI: apple: unwind host registration before resource cleanup A PM-link failure after pci_host_common_init() leaves a registered PCI bus and possibly bound function drivers behind while probe unwinds its IRQ domains and mappings. Resolve the PM dependencies before registering the host. If the cleanup action allocation fails after host registration, stop and remove the PCI hierarchy before calling apple_pcie_cleanup(). Avoid devm_add_action_or_reset() there because its immediate cleanup would run before the function drivers released their IRQs. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/pci/controller/pcie-apple.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index a9ac823892f79..a3b5c2fb98f4a 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2051,6 +2051,13 @@ static int apple_pcie_probe(struct platform_device *pdev) mutex_init(&pcie->lock); INIT_LIST_HEAD(&pcie->ports); + /* Resolve PM dependencies before publishing the PCI hierarchy. */ + if (pcie->hw->tunneled) { + ret = apple_pcie_tunnel_add_links(pcie); + if (ret) + return ret; + } + ret = apple_msi_init(pcie); if (ret) return ret; @@ -2065,14 +2072,16 @@ static int apple_pcie_probe(struct platform_device *pdev) * Port mappings are allocated by the ECAM init callback. Register cleanup * afterwards so devres runs it before unmapping those registers. */ - ret = devm_add_action_or_reset(dev, apple_pcie_cleanup, pcie); - if (ret) + ret = devm_add_action(dev, apple_pcie_cleanup, pcie); + if (ret) { + /* Drivers must release their IRQs before the domains disappear. */ + pci_host_common_remove(pdev); + apple_pcie_cleanup(pcie); return ret; + } - if (pcie->hw->tunneled) { + if (pcie->hw->tunneled) pci_walk_bus(bridge->bus, apple_pcie_tunnel_keep_d0, NULL); - return apple_pcie_tunnel_add_links(pcie); - } return 0; } From 5a7b76f31c3a8ccdc48fd586e5b646fa0c49499d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:07:40 -0700 Subject: [PATCH 0265/1579] iommu/apple-dart: enforce command gating during tunnel teardown Quiescing a tunneled DART only suppressed the runtime-resume and remove resets. The actual stream and TLB command paths did not check commands_gated, so a late command could still access the command engine after the port clock was disabled. Serialize gating with the command lock, wait for in-flight commands to finish, and reject subsequent T8020/T8110 commands with EHOSTDOWN before any command-register access. Use an independently addressable flag for the lockless PM checks. Add a KUnit case verifying both command engines leave the register image unchanged while gated and accept commands again after restore. This hardens teardown and does not establish a fix for the observed physical PCIe link restart failure. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/iommu/apple-dart-test.c | 55 ++++++++++++++++++++++++++++++++- drivers/iommu/apple-dart.c | 26 +++++++++++----- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 7f8888bf74115..5892b15c621a2 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -/* Included by apple-dart.c to exercise the locked-domain lifecycle. */ +/* Included by apple-dart.c to exercise domain and command lifecycles. */ #include #include @@ -131,10 +131,63 @@ static void apple_dart_test_firmware_roots(struct kunit *test) KUNIT_EXPECT_EQ(test, live[1], 0x4001ULL); } +static void apple_dart_test_gated_commands(struct kunit *test) +{ + struct apple_dart_stream_map stream = {}; + struct apple_dart *dart; + u32 sentinel = 0x40; + int ret; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->dev = kunit_device_register(test, "apple-dart-gated-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); + dev_set_drvdata(dart->dev, dart); + dart->regs = (__force void __iomem *)kunit_kzalloc(test, SZ_16K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart->regs); + dart->num_streams = 1; + dart->pgsize = SZ_16K; + dart->version = 0x0202; + spin_lock_init(&dart->lock); + stream.dart = dart; + __set_bit(0, stream.sidmap); + pm_runtime_set_active(dart->dev); + pm_runtime_enable(dart->dev); + + writel(sentinel, dart->regs + DART_T8020_STREAM_SELECT); + writel(sentinel, dart->regs + DART_T8020_STREAM_COMMAND); + writel(sentinel, dart->regs + DART_T8110_TLB_CMD); + writel(sentinel, dart->regs + DART_T8110_TLB_START); + writel(sentinel, dart->regs + DART_T8110_TLB_END); + apple_dart_quiesce_commands(dart->dev); + + /* Neither engine may touch its command registers after quiesce. */ + KUNIT_EXPECT_EQ(test, apple_dart_t8020_hw_invalidate_tlb(&stream), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_SELECT), sentinel); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), sentinel); + ret = apple_dart_t8110_hw_tlb_command_range(&stream, + DART_T8110_TLB_CMD_OP_FLUSH_SID, + true, 0, SZ_16K - 1); + KUNIT_EXPECT_EQ(test, ret, -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), sentinel); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_START), sentinel); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_END), sentinel); + + /* A restored port may submit commands again. */ + apple_dart_resume_commands(dart->dev); + KUNIT_EXPECT_EQ(test, apple_dart_t8020_hw_invalidate_tlb(&stream), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), + (u32)DART_T8020_STREAM_COMMAND_INVALIDATE); + KUNIT_EXPECT_EQ(test, apple_dart_t8110_hw_invalidate_tlb(&stream), 0); + KUNIT_EXPECT_NE(test, readl(dart->regs + DART_T8110_TLB_CMD), sentinel); + pm_runtime_disable(dart->dev); +} + static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_before_attach), KUNIT_CASE(apple_dart_test_locked_handoff), KUNIT_CASE(apple_dart_test_firmware_roots), + KUNIT_CASE(apple_dart_test_gated_commands), {} }; diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 277c5a589f149..5541b44061e4b 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -260,7 +260,7 @@ struct apple_dart { u32 locked : 1; u32 tunneled : 1; u32 power_retained : 1; - u32 commands_gated : 1; + bool commands_gated; struct apple_tunable *tunables; u16 version; @@ -663,6 +663,10 @@ apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, u32 command_reg; spin_lock_irqsave(&dart->lock, flags); + if (dart->commands_gated) { + spin_unlock_irqrestore(&dart->lock, flags); + return -EHOSTDOWN; + } for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) apple_dart_writel(dart, stream_map->sidmap[i], @@ -697,6 +701,10 @@ apple_dart_t8110_hw_tlb_command_range(struct apple_dart_stream_map *stream_map, int sid, ret = 0; spin_lock_irqsave(&dart->lock, flags); + if (dart->commands_gated) { + spin_unlock_irqrestore(&dart->lock, flags); + return -EHOSTDOWN; + } for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) | FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid); @@ -2233,8 +2241,11 @@ void apple_dart_quiesce_commands(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); - if (dart) - dart->commands_gated = true; + if (dart) { + /* Finish any in-flight command before the port clock is stopped. */ + guard(spinlock_irqsave)(&dart->lock); + WRITE_ONCE(dart->commands_gated, true); + } } EXPORT_SYMBOL_GPL(apple_dart_quiesce_commands); @@ -2243,10 +2254,11 @@ void apple_dart_resume_commands(struct device *dev) struct apple_dart *dart = dev_get_drvdata(dev); int ret; - if (!dart || !dart->commands_gated) + if (!dart || !READ_ONCE(dart->commands_gated)) return; - dart->commands_gated = false; + scoped_guard(spinlock_irqsave, &dart->lock) + WRITE_ONCE(dart->commands_gated, false); ret = pm_runtime_get_sync(dev); if (ret < 0) pm_runtime_put_noidle(dev); @@ -2264,7 +2276,7 @@ static void apple_dart_remove(struct platform_device *pdev) * A command issued then never completes. The next probe resets the * block after that clock is running again. */ - if (!dart->locked && !dart->commands_gated) + if (!dart->locked && !READ_ONCE(dart->commands_gated)) apple_dart_hw_reset(dart); free_irq(dart->irq, dart); @@ -2415,7 +2427,7 @@ static __maybe_unused int apple_dart_resume(struct device *dev) * state. Resetting one here is both unnecessary and earlier than Apple's * force-active call at the end of PCIe-C port resume. */ - if (dart->power_retained || dart->commands_gated) + if (dart->power_retained || READ_ONCE(dart->commands_gated)) return 0; /* Locked DARTs can't be restored, and they should not need it */ From a5af6399820f442cb94b8b74c231a8a3b8b88246 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:06:29 -0700 Subject: [PATCH 0266/1579] thunderbolt: scope native PCIe ownership to host tunnels Only tunnels originating at the host router control its native PCIe port. Calling NHI hooks for a downstream daisy-chain tunnel can otherwise quiesce the first dock's entire PCI hierarchy when only the second dock disconnects. Adopt host state for enabled PCIe tunnels retained at domain startup so an NHI rebind can restore a previously quiesced host. Temporary tunnels discovered solely for resume cleanup remain unowned. Add a KUnit daisy-chain regression covering hook isolation, retained host ownership and partial activation failure cleanup. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/nhi.h | 9 +++-- drivers/thunderbolt/tb.c | 3 ++ drivers/thunderbolt/test.c | 78 ++++++++++++++++++++++++++++++++++++ drivers/thunderbolt/tunnel.c | 9 ++++- drivers/thunderbolt/tunnel.h | 1 + 5 files changed, 95 insertions(+), 5 deletions(-) diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index ba98934a30b91..6ac76844a4973 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -78,10 +78,13 @@ struct tb_nhi_ring_layout { * @ring_configure: NHI specific hook to program the ring options registers * and enable the ring with the given flags. If not set * the standard USB4 NHI registers are used. - * @pci_tunnel_pre_activate: NHI specific hook run after a PCIe tunnel has + * @pci_tunnel_pre_activate: NHI specific hook run after a host PCIe tunnel has * been allocated but before its paths and adapters are enabled - * @pci_tunnel_post_activate: NHI specific hook run after a PCIe tunnel's - * paths and adapters have been enabled + * @pci_tunnel_post_activate: NHI specific hook run after a host PCIe tunnel's + * paths and adapters have been enabled, including + * adoption of an enabled tunnel at domain startup + * @pci_tunnel_deactivate: Release native host state acquired by the + * PCIe post-activation hook * @dp_tunnel_pre_activate: NHI specific hook run after DP hops are programmed * and before VE/AE. USB4 DPTX Discovery belongs here. * @dp_tunnel_post_activate: NHI specific hook run after a DP tunnel's diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 381682fb1d22f..9eb5d17c5d18f 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -1705,6 +1705,9 @@ static void tb_discover_tunnels(struct tb *tb) if (tb_tunnel_is_pci(tunnel)) { struct tb_switch *parent = tunnel->dst_port->sw; + /* Adopt host state only for tunnels retained by this domain. */ + if (tb_pci_tunnel_activate_host(tunnel)) + tb_tunnel_warn(tunnel, "failed to adopt PCIe host state\n"); while (parent != tunnel->src_port->sw) { parent->boot = true; parent = tb_switch_parent(parent); diff --git a/drivers/thunderbolt/test.c b/drivers/thunderbolt/test.c index e7974981281f2..2ecd1e4d9afff 100644 --- a/drivers/thunderbolt/test.c +++ b/drivers/thunderbolt/test.c @@ -2854,10 +2854,28 @@ static void tb_test_property_copy(struct kunit *test) struct tb_test_pci_host { struct tb_nhi nhi; + unsigned int prepares; + unsigned int connects; unsigned int disconnects; int result; }; +static int tb_test_pci_host_prepare(struct tb_nhi *nhi) +{ + struct tb_test_pci_host *host = container_of(nhi, struct tb_test_pci_host, nhi); + + host->prepares++; + return host->result; +} + +static int tb_test_pci_host_connect(struct tb_nhi *nhi) +{ + struct tb_test_pci_host *host = container_of(nhi, struct tb_test_pci_host, nhi); + + host->connects++; + return host->result; +} + static int tb_test_pci_host_disconnect(struct tb_nhi *nhi) { struct tb_test_pci_host *host = container_of(nhi, struct tb_test_pci_host, nhi); @@ -2909,6 +2927,65 @@ static void tb_test_pci_host_teardown(struct kunit *test) KUNIT_EXPECT_EQ(test, host->disconnects, 3U); } +static void tb_test_pci_host_daisy_chain(struct kunit *test) +{ + static const struct tb_nhi_ops ops = { + .pci_tunnel_pre_activate = tb_test_pci_host_prepare, + .pci_tunnel_post_activate = tb_test_pci_host_connect, + .pci_tunnel_deactivate = tb_test_pci_host_disconnect, + }; + struct tb_tunnel *root_tunnel, *downstream_tunnel; + struct tb_switch *root, *dev1, *dev2; + struct tb_test_pci_host *host; + struct tb *tb; + + host = kunit_kzalloc(test, sizeof(*host), GFP_KERNEL); + tb = kunit_kzalloc(test, sizeof(*tb), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, host); + KUNIT_ASSERT_NOT_NULL(test, tb); + host->nhi.ops = &ops; + tb->nhi = &host->nhi; + root = alloc_host(test); + KUNIT_ASSERT_NOT_NULL(test, root); + dev1 = alloc_dev_default(test, root, 0x1, true); + KUNIT_ASSERT_NOT_NULL(test, dev1); + dev2 = alloc_dev_default(test, dev1, 0x501, true); + KUNIT_ASSERT_NOT_NULL(test, dev2); + root_tunnel = tb_tunnel_alloc_pci(tb, &dev1->ports[9], &root->ports[8]); + KUNIT_ASSERT_NOT_NULL(test, root_tunnel); + downstream_tunnel = tb_tunnel_alloc_pci(tb, &dev2->ports[9], &dev1->ports[10]); + KUNIT_ASSERT_NOT_NULL(test, downstream_tunnel); + + KUNIT_EXPECT_EQ(test, root_tunnel->pre_activate(root_tunnel), 0); + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_activate_host(root_tunnel), 0); + KUNIT_EXPECT_TRUE(test, root_tunnel->host_pci_activated); + KUNIT_EXPECT_EQ(test, host->prepares, 1U); + KUNIT_EXPECT_EQ(test, host->connects, 1U); + + /* A second dock must not acquire or tear down the host's PCIe port. */ + KUNIT_EXPECT_EQ(test, downstream_tunnel->pre_activate(downstream_tunnel), 0); + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_activate_host(downstream_tunnel), 0); + KUNIT_EXPECT_FALSE(test, downstream_tunnel->host_pci_activated); + KUNIT_EXPECT_EQ(test, host->prepares, 1U); + KUNIT_EXPECT_EQ(test, host->connects, 1U); + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(downstream_tunnel), 0); + KUNIT_EXPECT_EQ(test, host->disconnects, 0U); + KUNIT_EXPECT_TRUE(test, root_tunnel->host_pci_activated); + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(root_tunnel), 0); + KUNIT_EXPECT_EQ(test, host->disconnects, 1U); + + /* A failed host activation may still need to unwind partial setup. */ + host->result = -EIO; + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_activate_host(root_tunnel), -EIO); + KUNIT_EXPECT_TRUE(test, root_tunnel->host_pci_activated); + host->result = 0; + KUNIT_EXPECT_EQ(test, tb_pci_tunnel_deactivate_host(root_tunnel), 0); + KUNIT_EXPECT_EQ(test, host->disconnects, 2U); + + tb_tunnel_put(downstream_tunnel); + tb_tunnel_put(root_tunnel); +} + struct tb_test_dp_host { struct tb_nhi nhi; unsigned int disconnects; @@ -3003,6 +3080,7 @@ static void tb_test_tunnel_dp_host_credits(struct kunit *test) static struct kunit_case tb_test_cases[] = { KUNIT_CASE(tb_test_pci_host_teardown), + KUNIT_CASE(tb_test_pci_host_daisy_chain), KUNIT_CASE(tb_test_tunnel_dp_host_credits), KUNIT_CASE(tb_test_dp_host_teardown_once), KUNIT_CASE(tb_test_dp_host_teardown_unannounced), diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index 3feabf2f449fe..ee814c6d82e19 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -352,15 +352,20 @@ static int tb_pci_pre_activate(struct tb_tunnel *tunnel) { const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; + /* Only the tunnel starting at the host owns its native PCIe port. */ + if (tb_route(tunnel->src_port->sw)) + return 0; if (ops && ops->pci_tunnel_pre_activate) return ops->pci_tunnel_pre_activate(tunnel->tb->nhi); return 0; } -static int tb_pci_post_activate(struct tb_tunnel *tunnel) +int tb_pci_tunnel_activate_host(struct tb_tunnel *tunnel) { const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; + if (tb_route(tunnel->src_port->sw)) + return 0; if (ops && ops->pci_tunnel_post_activate) { /* Also unwind a hook that fails after partial host setup. */ tunnel->host_pci_activated = true; @@ -417,7 +422,7 @@ static int tb_pci_activate(struct tb_tunnel *tunnel, bool activate) if (res) return res; - return tb_pci_post_activate(tunnel); + return tb_pci_tunnel_activate_host(tunnel); } else { /* Downstream router could be unplugged */ tb_pci_port_enable(tunnel->dst_port, activate); diff --git a/drivers/thunderbolt/tunnel.h b/drivers/thunderbolt/tunnel.h index a52116ef8e796..f11b4d9f459fc 100644 --- a/drivers/thunderbolt/tunnel.h +++ b/drivers/thunderbolt/tunnel.h @@ -120,6 +120,7 @@ struct tb_tunnel *tb_tunnel_discover_pci(struct tb *tb, struct tb_port *down, bool alloc_hopid); struct tb_tunnel *tb_tunnel_alloc_pci(struct tb *tb, struct tb_port *up, struct tb_port *down); +int tb_pci_tunnel_activate_host(struct tb_tunnel *tunnel); int tb_pci_tunnel_deactivate_host(struct tb_tunnel *tunnel); bool tb_tunnel_reserved_pci(struct tb_port *port, int *reserved_up, int *reserved_down); From 0a6eff654141c7d389248c7f4a621c2d327750e5 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:06:30 -0700 Subject: [PATCH 0267/1579] thunderbolt: apple: drain PCIe work before removing the domain Cable teardown held pcie_tunnel_lock while domain removal acquired tb->lock, reversing the order used by tunnel hooks. An in-flight hotplug could deadlock teardown, and simply cancelling the deferred worker did not prevent new requests from rearming it. Gate requests before draining work, quiesce the PCIe hierarchy while its NHI remains live, and release the PCIe lock before domain removal. Use the same ordered child cleanup on startup failure, and drain NHI diagnostics on probe failure. A new NHI domain reopens requests after setup. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/apple.c | 89 ++++++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 31 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 6fe43773b7cd3..7208d04680754 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -164,6 +164,7 @@ struct apple_cio { struct delayed_work pcie_tunnel_work; bool pcie_tunnel_requested; bool pcie_tunnel_populated; + bool pcie_tunnel_stopping; bool pcie_pm_prepared; bool pcie_quiesce_pending; @@ -911,7 +912,7 @@ static void apple_cio_pcie_tunnel_work(struct work_struct *work) int ret = 0; mutex_lock(&acio->pcie_tunnel_lock); - if (acio->pcie_pm_prepared) + if (acio->pcie_tunnel_stopping || acio->pcie_pm_prepared) goto unlock; if (acio->pcie_quiesce_pending) { ret = apple_cio_quiesce_pcie_tunnel_locked(acio); @@ -940,6 +941,8 @@ static int apple_nhi_pci_tunnel_deactivate(struct tb_nhi *nhi) * nothing to enumerate onto when the tunnel comes back. */ mutex_lock(&acio->pcie_tunnel_lock); + if (acio->pcie_tunnel_stopping) + goto unlock; WRITE_ONCE(acio->pcie_tunnel_requested, false); cancel_delayed_work(&acio->pcie_tunnel_work); acio->pcie_quiesce_pending = true; @@ -952,6 +955,7 @@ static int apple_nhi_pci_tunnel_deactivate(struct tb_nhi *nhi) ret); } +unlock: mutex_unlock(&acio->pcie_tunnel_lock); return ret; @@ -969,6 +973,10 @@ static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) * hardware reset state without blocking completion of the tunnel paths. */ mutex_lock(&acio->pcie_tunnel_lock); + if (acio->pcie_tunnel_stopping) { + mutex_unlock(&acio->pcie_tunnel_lock); + return -ESHUTDOWN; + } WRITE_ONCE(acio->pcie_tunnel_requested, true); /* No PCI removal or rescan until system resume has finished. */ mod_delayed_work(system_freezable_wq, &acio->pcie_tunnel_work, 0); @@ -976,6 +984,26 @@ static int apple_nhi_pci_tunnel_post_activate(struct tb_nhi *nhi) return 0; } +static void apple_cio_stop_pcie_tunnel(struct apple_cio *acio) +{ + int ret; + + /* Block new requests before draining work, without nesting tb->lock. */ + mutex_lock(&acio->pcie_tunnel_lock); + acio->pcie_tunnel_stopping = true; + WRITE_ONCE(acio->pcie_tunnel_requested, false); + mutex_unlock(&acio->pcie_tunnel_lock); + cancel_delayed_work_sync(&acio->pcie_tunnel_work); + + /* The NHI must remain alive to complete the tunneled reset handshake. */ + mutex_lock(&acio->pcie_tunnel_lock); + ret = apple_cio_quiesce_pcie_tunnel_locked(acio); + mutex_unlock(&acio->pcie_tunnel_lock); + if (ret) + dev_warn(acio->dev, + "failed to quiesce PCIe-C before NHI shutdown: %d\n", ret); +} + #define APPLE_DP_AUX_POLL_MS 500 #define APPLE_DP_AUX_POLL_MAX 24 @@ -1574,9 +1602,14 @@ static int apple_nhi_probe(struct platform_device *pdev) goto err; } + scoped_guard(mutex, &acio->pcie_tunnel_lock) { + acio->pcie_tunnel_stopping = false; + acio->pcie_pm_prepared = false; + } ret = tb_domain_add(anhi->tb, false); if (ret) { dev_err_probe(anhi->dev, ret, "failed to add TB domain\n"); + apple_cio_stop_pcie_tunnel(acio); tb_domain_put(anhi->tb); wait_for_completion(&anhi->nhi.domain_released); goto err; @@ -1631,6 +1664,8 @@ static int apple_nhi_probe(struct platform_device *pdev) return 0; err_remove_tb_domain: + disable_delayed_work_sync(&anhi->dp_aux_work); + apple_cio_stop_pcie_tunnel(acio); tb_domain_remove(anhi->tb); wait_for_completion(&anhi->nhi.domain_released); err: @@ -1645,6 +1680,7 @@ static void apple_nhi_remove(struct platform_device *pdev) /* Stop racing tunnel activation from rearming work during removal. */ disable_delayed_work_sync(&anhi->dp_aux_work); + apple_cio_stop_pcie_tunnel(anhi->acio); WRITE_ONCE(anhi->acio->nhi_pdev, NULL); tb_domain_remove(anhi->tb); wait_for_completion(&anhi->nhi.domain_released); @@ -1709,7 +1745,8 @@ static void apple_nhi_complete(struct device *dev) guard(mutex)(&acio->pcie_tunnel_lock); acio->pcie_pm_prepared = false; - if (acio->pcie_quiesce_pending || READ_ONCE(acio->pcie_tunnel_requested)) + if (!acio->pcie_tunnel_stopping && + (acio->pcie_quiesce_pending || READ_ONCE(acio->pcie_tunnel_requested))) mod_delayed_work(system_freezable_wq, &acio->pcie_tunnel_work, 0); } @@ -1763,37 +1800,18 @@ static struct platform_device *apple_cio_find_pcie_tunnel(struct apple_cio *acio return pdev; } -static void apple_cio_stop(struct apple_cio *acio) +static void apple_cio_remove_children(struct apple_cio *acio) { - struct platform_device *nhi_pdev, *pcie_pdev; - int ret, i; + struct platform_device *nhi_pdev; lockdep_assert_held(&acio->lock); /* - * First, shutdown the blocks inside the ACIO complex, like the NHI and the IOMMU. - * After we shut down the ACIO co-processor we will no longer be able to access - * the MMIO space of these so make sure nothing tries to do just that. + * Stop requests before removing the NHI. Do not hold pcie_tunnel_lock + * across domain removal: it takes tb->lock, which tunnel callbacks hold + * when they acquire pcie_tunnel_lock. */ - WRITE_ONCE(acio->pcie_tunnel_requested, false); - cancel_delayed_work_sync(&acio->pcie_tunnel_work); - mutex_lock(&acio->pcie_tunnel_lock); - - /* - * The PCIe-C tunnel reset handshake is carried by the NHI. Quiesce the - * remote PCI hierarchy and acknowledge tunneled PERST before removing - * that control plane. PCIe-C and its DART stay bound until the general - * child depopulation below. - */ - pcie_pdev = apple_cio_find_pcie_tunnel(acio); - if (pcie_pdev) { - ret = apple_pcie_tunnel_quiesce(&pcie_pdev->dev); - if (ret) - dev_warn(acio->dev, - "failed to quiesce PCIe-C before NHI shutdown: %d\n", - ret); - put_device(&pcie_pdev->dev); - } + apple_cio_stop_pcie_tunnel(acio); /* * The NHI is the tunnel control plane. Stop it while the tunneled PCIe @@ -1811,9 +1829,18 @@ static void apple_cio_stop(struct apple_cio *acio) flush_workqueue(acio->dp_wq); of_platform_depopulate(acio->dev); - acio->pcie_tunnel_populated = false; - acio->pcie_quiesce_pending = false; - mutex_unlock(&acio->pcie_tunnel_lock); + scoped_guard(mutex, &acio->pcie_tunnel_lock) { + acio->pcie_tunnel_populated = false; + acio->pcie_quiesce_pending = false; + } +} + +static void apple_cio_stop(struct apple_cio *acio) +{ + int ret, i; + + lockdep_assert_held(&acio->lock); + apple_cio_remove_children(acio); /* Try to shut down and power off the co-processor gracefully */ ret = apple_rtkit_poweroff(acio->rtk); @@ -1943,7 +1970,7 @@ static int apple_cio_start(struct apple_cio *acio) return 0; err_depopulate: - of_platform_depopulate(acio->dev); + apple_cio_remove_children(acio); err_shutdown_rtkit: /* Ignore errors here since we're about to cut power to the entire block anyway */ apple_rtkit_poweroff(acio->rtk); From 1fc819fadf23d3b45a1a020d36e72189095c7a6c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:08:44 -0700 Subject: [PATCH 0268/1579] usb: typec: tipd: preserve retry after USB4 switch failure USB4 and Thunderbolt switch activation can fail while booting ACIO or registering its children. Ignoring that return value caches the requested mode as successful, so later status updates skip the switch callback and cannot retry activation. Pass activation failures through state rollback and invalidate the mode cache. The PHY may already have changed before switch activation fails, so a later update must reconfigure even when it requests the old cached mode. Mark the cache valid only after successful configuration, including an explicit safe-mode transition. This does not add retries without a new status update. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/usb/typec/tipd/core.c | 25 +++++++++++++++---------- drivers/usb/typec/tipd/tps6598x.h | 1 + 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 048891241fe83..33b35da777c5e 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -555,7 +555,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, int ret = 0; if (!(st->data_status & TPS_DATA_STATUS_DATA_CONNECTION)) { - if (cd321x->state.mode == TYPEC_STATE_SAFE) + if (cd321x->state_valid && cd321x->state.mode == TYPEC_STATE_SAFE) return 0; cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; @@ -565,7 +565,8 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, } else if (st->data_status & CD321X_DATA_STATUS_USB4_CONNECTION) { struct enter_usb_data eusb_data; - if (cd321x->state.alt == NULL && cd321x->state.mode == TYPEC_MODE_USB4) + if (cd321x->state_valid && !cd321x->state.alt && + cd321x->state.mode == TYPEC_MODE_USB4) return 0; eusb_data.eudo = le32_to_cpu(st->usb4_status.eudo); @@ -584,11 +585,11 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, tbt_switch_data.orientation = TPS_STATUS_TO_UPSIDE_DOWN(st->status) ? TYPEC_ORIENTATION_REVERSE : TYPEC_ORIENTATION_NORMAL; - typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); } else if (st->data_status & TPS_DATA_STATUS_TBT_CONNECTION) { struct typec_thunderbolt_data tbt_data; - if (cd321x->state.alt == cd321x->port_altmode_tbt && + if (cd321x->state_valid && cd321x->state.alt == cd321x->port_altmode_tbt && cd321x->state.mode == TYPEC_TBT_MODE) return 0; @@ -617,7 +618,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, tbt_switch_data.orientation = TPS_STATUS_TO_UPSIDE_DOWN(st->status) ? TYPEC_ORIENTATION_REVERSE : TYPEC_ORIENTATION_NORMAL; - typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); } else if (st->data_status & TPS_DATA_STATUS_DP_CONNECTION) { struct typec_displayport_data dp_data; unsigned long mode; @@ -653,7 +654,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, else dp_data.status &= ~DP_STATUS_HPD_STATE; - if (cd321x->state.alt == cd321x->port_altmode_dp && + if (cd321x->state_valid && cd321x->state.alt == cd321x->port_altmode_dp && cd321x->state.mode == mode && cd321x->dp_status == dp_data.status && cd321x->dp_conf == dp_data.conf) @@ -669,7 +670,8 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, cd321x->dp_conf = dp_data.conf; } } else { - if (cd321x->state.alt == NULL && cd321x->state.mode == TYPEC_STATE_USB) + if (cd321x->state_valid && !cd321x->state.alt && + cd321x->state.mode == TYPEC_STATE_USB) return 0; cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_USB; @@ -681,9 +683,11 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, out: /* Clear data since it's no longer used after typec_mux_set and points to the stack */ cd321x->state.data = NULL; + cd321x->state_valid = !ret; if (ret) { + /* The mux may have changed before the switch failed. */ cd321x->state = old_state; - dev_warn(tps->dev, "failed to configure Type-C mux: %d\n", ret); + dev_warn(tps->dev, "failed to configure Type-C connection: %d\n", ret); } else if (cd321x->state.alt != cd321x->port_altmode_dp) { cd321x->dp_status = 0; cd321x->dp_conf = 0; @@ -757,7 +761,7 @@ static void cd321x_update_work(struct work_struct *work) * PCIe-C cannot quiesce its hierarchy before NHI removal. */ if ((!new_connected || was_disconnected) && - cd321x->state.mode != TYPEC_STATE_SAFE) + (!cd321x->state_valid || cd321x->state.mode != TYPEC_STATE_SAFE)) typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); @@ -796,6 +800,7 @@ static void cd321x_update_work(struct work_struct *work) cd321x->state.mode = TYPEC_STATE_SAFE; cd321x->state.data = NULL; ret = typec_set_mode(tps->port, TYPEC_STATE_SAFE); + cd321x->state_valid = !ret; if (ret) { dev_warn(tps->dev, "failed to place Type-C mux in safe mode: %d\n", @@ -1373,7 +1378,7 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; cd321x->state.data = NULL; - typec_set_mode(tps->port, TYPEC_STATE_SAFE); + cd321x->state_valid = !typec_set_mode(tps->port, TYPEC_STATE_SAFE); return 0; diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 5ec291b595928..85bd83d809616 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -372,6 +372,7 @@ struct cd321x { struct typec_mux *mux; struct typec_mux_state state; + bool state_valid; u32 dp_status; u32 dp_conf; struct typec_thunderbolt_switch *tbt_switch; From 6a109bc8485c47b39fcaaa30687648014e19022c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:11:48 -0700 Subject: [PATCH 0269/1579] phy: apple: atc: propagate pixel clock command failures The AUSPLL command helper reports an acknowledgment timeout but returns success. Its DP caller cannot act on the failure, and the M1 tunnel clock path ignores both command results before caching the requested rate. Return the polling error and use the tunnel stop path when either command fails. Leave the rate uncached so a subsequent activation can retry. The successful register sequence is unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/phy/apple/atc.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 025855e84910f..d41bcb651567a 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1776,7 +1776,7 @@ static int atcphy_auspll_apb_command(struct apple_atcphy *atcphy, u32 command) core_clear32(atcphy, AUSPLL_APB_CMD_OVERRIDE, AUSPLL_APB_CMD_OVERRIDE_REQ); - return 0; + return ret; } static int atcphy_dp_configure(struct apple_atcphy *atcphy, enum atcphy_dp_link_rate lr) @@ -2154,17 +2154,24 @@ static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_PCLK_DRVR_EN); core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_PCLK2_DRVR_EN); core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_REFBUFCLK_DRVR_EN); - atcphy_auspll_apb_command(atcphy, 0); + ret = atcphy_auspll_apb_command(atcphy, 0); + if (ret) + goto err_stop; ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, value, value & ACIOPHY_AUSPLL_LOCK, 10, 10000); if (ret) { dev_err(atcphy->dev, "DP tunnel clock: AUSPLL did not lock\n"); - atc_tunnel_stop_t8103(atcphy); - return ret; + goto err_stop; } - atcphy_auspll_apb_command(atcphy, 0x2000); + ret = atcphy_auspll_apb_command(atcphy, 0x2000); + if (ret) + goto err_stop; atcphy->tunnel_rate = rate; return 0; + +err_stop: + atc_tunnel_stop_t8103(atcphy); + return ret; } struct atc_tunnel_saved_reg { From fc4218414c61cdd4a55cfcad0fcc87cb537ae06c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:09:23 -0700 Subject: [PATCH 0270/1579] drm/apple: retain AFK receive payload until dispatch completes Copy each ring entry before publishing the consumer position. The producer can otherwise reuse bytes while a report or command handler still reads them. Also defer reading the subtype until the header bounds check has succeeded. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/afk.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 9124f9e46647b..3fbc5fe4d601c 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -519,7 +519,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, struct epic_hdr *ehdr = (struct epic_hdr *)data; struct epic_sub_hdr *eshdr = (struct epic_sub_hdr *)(data + sizeof(*ehdr)); - u16 subtype = le16_to_cpu(eshdr->type); + u16 subtype; u8 *payload = data + sizeof(*ehdr) + sizeof(*eshdr); size_t payload_size; @@ -529,6 +529,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, return; } payload_size = data_size - sizeof(*ehdr) - sizeof(*eshdr); + subtype = le16_to_cpu(eshdr->type); trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); @@ -591,6 +592,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, static bool afk_recv(struct apple_dcp_afkep *ep) { struct afk_qe *hdr; + u8 *data; u32 rptr, wptr; u32 magic, size, channel, type; @@ -645,8 +647,6 @@ static bool afk_recv(struct apple_dcp_afkep *ep) ep->endpoint, magic); return false; } - - ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); } if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { @@ -658,6 +658,14 @@ static bool afk_recv(struct apple_dcp_afkep *ep) channel = le32_to_cpu(hdr->channel); type = le32_to_cpu(hdr->type); + /* + * Publishing rptr releases this entry to the firmware. Handlers can + * sleep and send replies, so retain a private copy before allowing + * the producer to reuse the ring storage. + */ + data = kmemdup(hdr->data, size, GFP_KERNEL); + if (!data) + return false; rptr = ALIGN(rptr + sizeof(*hdr) + size, 1 << BLOCK_SHIFT); if (WARN_ON(rptr > ep->rxbfr.bufsz)) @@ -670,14 +678,8 @@ static bool afk_recv(struct apple_dcp_afkep *ep) ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); trace_afk_recv_rwptr_post(ep, rptr, wptr); - /* - * TODO: this is theoretically unsafe since DCP could overwrite data - * after the read pointer was updated above. Do it anyway since - * it avoids 2 problems in the DCP tracer: - * 1. the tracer sees replies before the notifies from dcp - * 2. the tracer tries to read buffers after they are unmapped. - */ - afk_recv_handle(ep, channel, type, hdr->data, size); + afk_recv_handle(ep, channel, type, data, size); + kfree(data); return true; } From a8d46a24be3f23cedadcfa7606323d3e0690ea34 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:11:22 -0700 Subject: [PATCH 0271/1579] thunderbolt: describe register handling and ownership directly Keep comments focused on register fields, bounds and Linux driver behavior. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/apple-dpin-handshake.h | 68 ++-------------------- drivers/thunderbolt/apple.c | 21 +++---- drivers/thunderbolt/tb_regs.h | 5 +- drivers/thunderbolt/tunnel.c | 19 ++---- 4 files changed, 20 insertions(+), 93 deletions(-) diff --git a/drivers/thunderbolt/apple-dpin-handshake.h b/drivers/thunderbolt/apple-dpin-handshake.h index 4e492b2cc5802..feaab365b6464 100644 --- a/drivers/thunderbolt/apple-dpin-handshake.h +++ b/drivers/thunderbolt/apple-dpin-handshake.h @@ -10,54 +10,13 @@ #define APPLE_DPIN_HPD_LEVEL (1U << 2) #define APPLE_DPIN_INACTIVE 1U -/* - * Set unconditionally by native AppleCIODPTX::bringConnectionUp on both the - * HPD and CONTROL registers of this exact DPIN0 block when bringing a - * connection up (single-lane-group case, which is what this port is). The - * HPD-register write is additionally gated there on a single/multi-stream - * check; every topology this driver drives through DPIN0 is a single - * external monitor (SST), so it is unconditional here too. Neither write - * clears any other bit (native masks both with 0, a pure OR). Not written - * on deactivate: native's teardown path was not traced, so this preserves - * the existing "do not invent resets" behavior there. - */ +/* Activation sets CONNECTED without clearing unrelated HPD or CONTROL bits. */ #define APPLE_DPIN_CONNECTED (1U << 1) /* - * Two further offsets native AppleCIODPTX::bringConnectionUp writes on - * this exact DPIN0 block, in between the HPD and CONTROL writes above - * (native order: HPD, then MODE_B, then MODE_A, then CONTROL). Unlike - * CONNECTED, the exact values here are NOT confirmed from the native - * binary: native computes them from a per-connection attributes value - * (rate class and an unidentified secondary field) that is copied - * verbatim from this connection's negotiation and is not itself built - * anywhere in the traced kernel/kext code -- its origin could not be - * pinned down statically (most likely the separate DCP coprocessor - * firmware). The value used is therefore an informed, explicitly-labeled - * estimate, not a confirmed constant, and is passed in by the caller - * (mode_value below) rather than fixed at compile time, so the bounded - * formula's one free parameter can be swept across a live connection - * without reinstalling or rebooting between values: - * - bits 4-7 of the field select a rate class using the same RBR=0/ - * HBR=1/HBR2=2/HBR3=3 ordinal already used elsewhere in this driver - * (drivers/thunderbolt/tb_regs.h DP_COMMON_CAP_RATE_*); this link - * negotiates HBR2, so 2 -- high confidence, unchanged across the - * whole sweep. - * - a secondary bit, native-gated on lane_count>=2 (true here: 4) and - * on the same "which DPIN0 sub-instance" selector already confirmed - * unconditional-0 for this single, non-split port, is set from a - * nearby field that other native code also treats as a small, - * 3-valid-value enumeration (0, 1, or 2) -- this is the weak half of - * the formula and the only free parameter, swept across - * mode_value = rate_class(2) * lane_count(4) + secondary_bit = 8, 9 - * (both already tested via separate reboots, each a clean boot with - * no picture), or 10. - * - * mode_value is bounded to APPLE_DPIN_MODE_VALUE_MAX: this caps both the - * single bit MODE_A can set (1 << mode_value) and the field width MODE_B - * ORs in, which is also exactly the range the deactivate path below - * knows how to clear back to a clean baseline. Raising this bound is a - * new, reviewable change, not a runtime knob. + * The caller supplies the experimental DPIN mode value. Bound it to the + * four-bit MODE_B field and the range of one-hot MODE_A bits that teardown + * clears, so changing the parameter cannot modify unrelated register bits. */ #define APPLE_DPIN_MODE_A 0x14 #define APPLE_DPIN_MODE_B 0x1c @@ -108,16 +67,7 @@ static inline int apple_dpin_handshake(const struct apple_dpin_io *io, } else { unsigned int mode_a, mode_b; - /* - * Not native teardown behavior (never traced; see the - * CONNECTED comment above) -- this exists solely so a later - * activate on this same boot starts from clean state and is - * a valid isolated test of a different mode_value. Bounded - * to exactly the bits any in-range mode_value write above - * could have set: MODE_B's OR'd field (bits 7..7+MAX's - * width) and MODE_A's cleared-low-byte-plus-one-set-bit - * (bits 0..MAX). Never touches bits outside that range. - */ + /* Clear only the fields that an in-range mode value can modify. */ mode_b = io->read(io->ctx, APPLE_DPIN_MODE_B); if (mode_b != ~0U) io->write(io->ctx, APPLE_DPIN_MODE_B, @@ -151,13 +101,7 @@ static inline int apple_dpin_handshake(const struct apple_dpin_io *io, if (ret) break; } - /* - * Restore only the bits we own (INACTIVE, and CONNECTED now that we - * set it above); do not invent resets on a failed handshake. HPD is - * left as-is here: native's teardown path for it was not traced, so - * no rollback is invented for a register this driver otherwise never - * wrote before this change. - */ + /* Restore the owned CONTROL bits after a failed handshake. */ ack = io->read(io->ctx, APPLE_DPIN_CONTROL); if (ack != ~0U) io->write(io->ctx, APPLE_DPIN_CONTROL, diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 7208d04680754..9bb806c435bb8 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -782,8 +782,8 @@ static void apple_nhi_ring_configure(struct tb_ring *ring, u32 flags, u32 e2e_fl options += APPLE_CIO_NHI_TXRING_DESC_BASE; /* - * All TX rings share what macOS calls a shared buffer with 232 entries. This is how - * macOS splits it up, ring 0 only carries control packets and gets the minimum. + * Partition the 232 shared TX entries: two for control ring 0, + * forty each for rings 1-5, and five each for the remaining rings. */ if (ring->hop == 0) writel(2, options + 4); @@ -800,8 +800,8 @@ static void apple_nhi_ring_configure(struct tb_ring *ring, u32 flags, u32 e2e_fl } /* - * The firmware samples the ring configuration when the valid bit is set and E2E flow - * control never engages when configured afterwards. Write everything at once like macOS. + * The firmware samples E2E flow control when the valid bit is set. + * Program both in the same write. */ writel(flags | e2e_flags, options); } @@ -1317,9 +1317,8 @@ static void apple_dp_aux_work(struct work_struct *work) /* * Which DP IN adapter (0/1) on the host router this port is -- same - * counting order as apple_dp_in_analog_base() above, and as - * apple_nhi_dp_tunnel_changed()'s port-to-dpin mapping in - * aurora-silicon/linux#8. + * counting order as apple_dp_in_analog_base() above and the port-to-dpin + * mapping in apple_nhi_dp_tunnel_changed(). */ static int apple_dpin_index_for_port(struct apple_nhi *anhi, struct tb_port *in) { @@ -1372,13 +1371,7 @@ static int apple_nhi_dp_tunnel_post_activate(struct tb_nhi *nhi, apple_dp_dump_rc(anhi->acio); apple_dp_dump_vse(in->sw); - /* - * Route a real display pipeline to this tunnel, ported from - * aurora-silicon/linux#8 -- the dpin_aux "analog AUX serializer" - * mechanism previously here is confirmed ineffective on this - * hardware, including after everything else in the tunnel-up path - * was independently fixed. - */ + /* Route the display pipeline through the selected host DP IN adapter. */ idx = apple_dpin_index_for_port(anhi, in); if (idx < 0 || idx > 1 || !anhi->acio) { dev_warn(anhi->dev, diff --git a/drivers/thunderbolt/tb_regs.h b/drivers/thunderbolt/tb_regs.h index b12445f339eeb..4fd9a422759b1 100644 --- a/drivers/thunderbolt/tb_regs.h +++ b/drivers/thunderbolt/tb_regs.h @@ -430,14 +430,13 @@ struct tb_regs_port_header { #define ADP_DP_CS_3_HPDC BIT(9) /* * Apple silicon host DP IN adapter: pulsing this tells the adapter to - * propagate HPD to ADP_DP_CS_2_HPD. Ported from aurora-silicon/linux#8 - * (t8103, hardware-tested). + * propagate HPD to ADP_DP_CS_2_HPD. */ #define ADP_DP_CS_3_HPD_PROPAGATE BIT(10) /* * Apple silicon host: hold the DP OUT (hub-side) adapter's own link * training off while a tunnel is up -- the host's DPTX trains the sink - * through the tunnel itself. Same source as HPD_PROPAGATE above. + * through the tunnel itself. */ #define ADP_DP_CS_3_NO_AUTO_LT BIT(8) #define DP_LOCAL_CAP 0x04 diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index ee814c6d82e19..d8bcce8010771 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -1234,16 +1234,9 @@ static void tb_dp_dprx_stop(struct tb_tunnel *tunnel) } /* - * Apple silicon host DP IN adapter: this adapter has no real physical DP - * connector wired to it, so nothing ever sets its own ADP_DP_CS_2_HPD -- - * pulse ADP_DP_CS_3_HPD_PROPAGATE to tell it to propagate HPD itself, and - * wait for it to take, same as aurora-silicon/linux#8's hardware-tested - * t8103 implementation (three docks, multiple monitors). On the - * dcpext1/right-port path, the CS0-CS13 registers this pulse is meant to - * move have been observed to stay completely static across every - * failure, unlike the working dcpext0/left-port case. Logs and continues - * either way; a failed pulse is not treated as a fatal tunnel-activation - * error, matching the reference. + * The host DP IN adapter has no physical DP connector. Pulse HPD propagation + * after enabling the tunnel and allow time for the adapter to report HPD. + * Failure is diagnostic: leave the tunnel available for display recovery. */ static void tb_dp_apple_pulse_hpd(struct tb_port *in) { @@ -1351,8 +1344,7 @@ static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) * Apple silicon host: the sink link is trained by the * host's DPTX through the tunnel; keep this DP OUT adapter * from starting link training on its own, and hand it back - * once the tunnel is gone. Ported from aurora-silicon/ - * linux#8. + * once the tunnel is gone. */ if (active && apple_dpin && !tb_port_read(out, &v, TB_CFG_PORT, out->cap_adap + ADP_DP_CS_3, 1)) { @@ -1699,8 +1691,7 @@ static int tb_dp_init_video_credits(struct tb_path_hop *hop) struct tb_switch *sw = port->sw; /* - * Apple silicon host DP IN adapters take 5 NFC credits for the - * video path -- same source as the HPD/NO_AUTO_LT handling above. + * Apple silicon host DP IN adapters require five NFC video credits. */ if (tb_port_needs_host_dp_credits(port)) { hop->nfc_credits = 5; From 65e3e0ed688a5a933a60d92ca79b6136c357b876 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:11:36 -0700 Subject: [PATCH 0272/1579] iommu/apple-dart: report tunnel command-resume failures Resuming commands cleared the gate before runtime resume, then discarded a runtime-PM failure. PCIe-C could consequently rescan functions without a successfully restored IOMMU and without a gate protecting the failed command engine. Return the resume error, restore the gate on failure, and propagate DART errors through the tunneled-host restore path before rescan. Keep the runtime-PM usage count balanced with pm_runtime_resume_and_get(). A KUnit fault case verifies error propagation, gate retention, reference balance and a subsequent successful retry. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/iommu/apple-dart-test.c | 30 +++++++++++++++++++++++++++++ drivers/iommu/apple-dart.c | 23 +++++++++++++--------- drivers/pci/controller/pcie-apple.c | 19 ++++++++++++------ include/linux/soc/apple/dart.h | 8 ++++---- 4 files changed, 61 insertions(+), 19 deletions(-) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 5892b15c621a2..687d097bd3e5b 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -183,11 +183,41 @@ static void apple_dart_test_gated_commands(struct kunit *test) pm_runtime_disable(dart->dev); } +static void apple_dart_test_resume_commands_failure(struct kunit *test) +{ + struct apple_dart *dart; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->dev = kunit_device_register(test, "apple-dart-resume-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); + dev_set_drvdata(dart->dev, dart); + spin_lock_init(&dart->lock); + pm_runtime_set_active(dart->dev); + pm_runtime_enable(dart->dev); + apple_dart_quiesce_commands(dart->dev); + + /* A prior runtime-PM error must leave commands gated and report failure. */ + scoped_guard(spinlock_irqsave, &dart->dev->power.lock) + dart->dev->power.runtime_error = -EIO; + KUNIT_EXPECT_EQ(test, apple_dart_resume_commands(dart->dev), -EINVAL); + KUNIT_EXPECT_TRUE(test, dart->commands_gated); + KUNIT_EXPECT_EQ(test, atomic_read(&dart->dev->power.usage_count), 0); + + /* Clearing the PM error permits a later retry without leaking a reference. */ + KUNIT_EXPECT_EQ(test, pm_runtime_set_active(dart->dev), 0); + KUNIT_EXPECT_EQ(test, apple_dart_resume_commands(dart->dev), 0); + KUNIT_EXPECT_FALSE(test, dart->commands_gated); + KUNIT_EXPECT_EQ(test, atomic_read(&dart->dev->power.usage_count), 0); + pm_runtime_disable(dart->dev); +} + static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_before_attach), KUNIT_CASE(apple_dart_test_locked_handoff), KUNIT_CASE(apple_dart_test_firmware_roots), KUNIT_CASE(apple_dart_test_gated_commands), + KUNIT_CASE(apple_dart_test_resume_commands_failure), {} }; diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 5541b44061e4b..6b47383a302ff 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -2237,7 +2237,7 @@ static int apple_dart_probe(struct platform_device *pdev) return ret; } -void apple_dart_quiesce_commands(struct device *dev) +int apple_dart_quiesce_commands(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); @@ -2246,24 +2246,29 @@ void apple_dart_quiesce_commands(struct device *dev) guard(spinlock_irqsave)(&dart->lock); WRITE_ONCE(dart->commands_gated, true); } + return 0; } EXPORT_SYMBOL_GPL(apple_dart_quiesce_commands); -void apple_dart_resume_commands(struct device *dev) +int apple_dart_resume_commands(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); int ret; - if (!dart || !READ_ONCE(dart->commands_gated)) - return; + if (!dart) + return -ENODEV; + if (!READ_ONCE(dart->commands_gated)) + return 0; scoped_guard(spinlock_irqsave, &dart->lock) WRITE_ONCE(dart->commands_gated, false); - ret = pm_runtime_get_sync(dev); - if (ret < 0) - pm_runtime_put_noidle(dev); - else - pm_runtime_put(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + apple_dart_quiesce_commands(dev); + return ret; + } + pm_runtime_put(dev); + return 0; } EXPORT_SYMBOL_GPL(apple_dart_resume_commands); diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index a3b5c2fb98f4a..f40e535c3205f 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2086,16 +2086,17 @@ static int apple_pcie_probe(struct platform_device *pdev) return 0; } -typedef void (*apple_pcie_dart_fn)(struct device *dev); +typedef int (*apple_pcie_dart_fn)(struct device *dev); -static void apple_pcie_walk_tunnel_darts(struct apple_pcie *pcie, apple_pcie_dart_fn fn) +static int apple_pcie_walk_tunnel_darts(struct apple_pcie *pcie, apple_pcie_dart_fn fn) { struct device_node *parent, *child; struct platform_device *pdev; + int ret = 0; parent = of_get_parent(pcie->dev->of_node); if (!parent) - return; + return -ENODEV; for_each_available_child_of_node(parent, child) { if (!of_device_is_compatible(child, "apple,t8103-dart") && @@ -2106,12 +2107,16 @@ static void apple_pcie_walk_tunnel_darts(struct apple_pcie *pcie, apple_pcie_dar pdev = of_find_device_by_node(child); if (pdev) { - fn(&pdev->dev); + int err = fn(&pdev->dev); + + if (err && !ret) + ret = err; put_device(&pdev->dev); } } of_node_put(parent); + return ret; } int apple_pcie_tunnel_quiesce(struct device *dev) @@ -2158,7 +2163,7 @@ int apple_pcie_tunnel_quiesce(struct device *dev) * Removing the IOMMU later resumes it and issues a command. Gate * those commands before APPCLK goes away with the port. */ - apple_pcie_walk_tunnel_darts(pcie, apple_dart_quiesce_commands); + ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_quiesce_commands); list_for_each_entry(port, &pcie->ports, entry) { int err; @@ -2211,7 +2216,9 @@ int apple_pcie_tunnel_restore(struct device *dev) if (ret) return ret; } - apple_pcie_walk_tunnel_darts(pcie, apple_dart_resume_commands); + ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_resume_commands); + if (ret) + return ret; pci_lock_rescan_remove(); pci_rescan_bus(bridge->bus); diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index 8fca017414d56..423602f02be3b 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -11,11 +11,11 @@ struct device; * before that clock is gated, and allow them again once it is back. */ #if IS_ENABLED(CONFIG_APPLE_DART) -void apple_dart_quiesce_commands(struct device *dev); -void apple_dart_resume_commands(struct device *dev); +int apple_dart_quiesce_commands(struct device *dev); +int apple_dart_resume_commands(struct device *dev); #else -static inline void apple_dart_quiesce_commands(struct device *dev) { } -static inline void apple_dart_resume_commands(struct device *dev) { } +static inline int apple_dart_quiesce_commands(struct device *dev) { return 0; } +static inline int apple_dart_resume_commands(struct device *dev) { return 0; } #endif #endif From 64dfa24ca989fec06367a7199164a70de7d99e35 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:11:36 -0700 Subject: [PATCH 0273/1579] PCI: apple: describe tunneled PM ordering by Linux dependencies Document the retained-state, doorbell and PCIe-C/DART ordering requirements directly in terms of register access and Linux device dependencies. No functional change. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/iommu/apple-dart.c | 10 +++++----- drivers/pci/controller/pcie-apple.c | 30 +++++++++++++---------------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 6b47383a302ff..d38fd34cd0fb4 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -2051,9 +2051,9 @@ static int apple_dart_probe(struct platform_device *pdev) * The cable-powered T8110 DART does not provide a safely probeable * parameter block at this point in PCIe tunnel activation. The SError * raised by these reads is asynchronous, which made the following trace - * boundary look guilty on different boots. Apple instead publishes the - * complete immutable topology in ADT: 16 KiB pages, 42-bit addresses and - * 64 SIDs. Use that description before making any DART MMIO access. + * boundary look guilty on different boots. The device tree supplies the + * immutable topology: 16 KiB pages, 42-bit addresses and 64 SIDs. Use that + * description before making any DART MMIO access. */ if (tunneled) { /* @@ -2429,8 +2429,8 @@ static __maybe_unused int apple_dart_resume(struct device *dev) /* * PCIe-C DARTs on an apple,always-on domain retain their translation - * state. Resetting one here is both unnecessary and earlier than Apple's - * force-active call at the end of PCIe-C port resume. + * state. Resetting one here is unnecessary; DART access must remain + * ordered after PCIe-C port resume. */ if (dart->power_retained || READ_ONCE(dart->commands_gated)) return 0; diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index f40e535c3205f..59ab52a1e6c22 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -153,11 +153,9 @@ MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); #define PORT_MSIMAP_TARGET GENMASK(7, 0) /* - * The doorbell address is set to 0xfffff000, which by convention - * matches what MacOS does, and it is possible to use any other - * address (in the bottom 4GB, as the base register is only 32bit). - * However, it has to be excluded from the IOVA range, and the DART - * driver has to know about it. + * The doorbell address must be in the bottom 4GB because the base register + * is only 32 bits wide. Exclude it from the IOVA range and share the same + * configured address with the DART driver. */ #define DOORBELL_ADDR CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR @@ -963,8 +961,8 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) if (pcie->power_retained) { /* * The ATC PCIe domain remains powered on these systems. Replaying - * resetPortHardware() against that live handoff is not a resume - * operation and raises an asynchronous external abort on T6020. + * the cold-init reset table against that retained state raises an + * asynchronous external abort on T6020. * Re-activating the ACIO PCIe tunnel closes the Intr2AXI aperture, * just as it does during cold cable activation, so pulse it before * releasing the retained port reset. @@ -978,7 +976,7 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) } else { /* * A genuinely power-gated PCIe-C controller lost its register - * state. Recreate the enablePortHardware() baseline before start. + * state. Recreate the cold-init baseline before start. */ apple_pcie_tunnel_apply_tunable(pcie->debug_base, pcie->debug_tunable); @@ -1943,11 +1941,10 @@ static int apple_pcie_tunnel_add_links(struct apple_pcie *pcie) int ret; /* - * Apple orders clientDidSleep after disabling the PCIe-C port, and - * clientWillWake before enabling it. NHI and PCIe-C are sibling devices - * under ACIO, so encode that missing edge explicitly: Linux must suspend - * PCIe-C before NHI tears down the router state and resume NHI before the - * host touches its tunneled aperture. + * NHI and PCIe-C are sibling devices under ACIO, so encode their PM + * dependency explicitly. PCIe-C must complete its reset handshake before + * NHI tears down the router state, and NHI must resume before the host + * touches its tunneled aperture. */ for_each_available_child_of_node_scoped(pcie->dev->parent->of_node, sibling) { @@ -1968,10 +1965,9 @@ static int apple_pcie_tunnel_add_links(struct apple_pcie *pcie) "failed to order PCIe-C after NHI resume\n"); /* - * The DART and PCIe-C host are synthesized as siblings. Apple restores - * port hardware and RID/SID forwarding before forcing the DART active; - * express the same dependency so Linux suspends DART first and resumes - * PCIe-C first. + * The DART and PCIe-C host are synthesized as siblings. Port hardware + * and RID/SID forwarding must be ready before DART access resumes, so + * suspend DART first and resume PCIe-C first. */ for_each_available_child_of_node_scoped(pcie->dev->of_node->parent, sibling) { From fb40feb283b65881cd0ec807c8dedaf6f40fbf48 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:11:52 -0700 Subject: [PATCH 0274/1579] drm/apple: preserve mode invalidation across hotplug and modeset Serialize mode validity and deferred connector changes between the atomic commit worker and callback thread. A completed modeset must not overwrite a disconnect or other link invalidation received while it was waiting. Invalidate before starting the operation so lookup, allocation and timeout failures cannot leave an old mode marked valid. Recheck validity after an asynchronous color-matrix operation before beginning a swap. Add seven KUnit scenarios for failure, deferred connect/disconnect, transient link loss and invalidation on either side of completion. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/Makefile | 1 + drivers/gpu/drm/apple/Kconfig | 10 ++ drivers/gpu/drm/apple/dcp-internal.h | 8 +- drivers/gpu/drm/apple/dcp.c | 4 +- drivers/gpu/drm/apple/dcp_backlight.c | 2 +- drivers/gpu/drm/apple/iomfb-state-test.c | 127 +++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb-state.h | 110 ++++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 29 ++++-- drivers/gpu/drm/apple/iomfb_template.c | 61 ++--------- 9 files changed, 284 insertions(+), 68 deletions(-) create mode 100644 drivers/gpu/drm/apple/iomfb-state-test.c create mode 100644 drivers/gpu/drm/apple/iomfb-state.h diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index 9d7406492360d..a0c684efd523d 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -228,6 +228,7 @@ obj-$(CONFIG_DRM_PANFROST) += panfrost/ obj-$(CONFIG_DRM_PANTHOR) += panthor/ obj-$(CONFIG_DRM_TYR) += tyr/ obj-$(CONFIG_DRM_APPLE) += apple/ +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple/iomfb-state-test.o obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/ obj-$(CONFIG_DRM_MCDE) += mcde/ obj-$(CONFIG_DRM_TIDSS) += tidss/ diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index ecf7afd76ceda..65c7fb2256344 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -28,3 +28,13 @@ config DRM_APPLE_DEBUG bool "Enable additional driver debugging" depends on DRM_APPLE depends on EXPERT # only for developers + +config DRM_APPLE_KUNIT_TEST + tristate "KUnit tests for Apple display state" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Test modeset and hotplug state transitions without display hardware, + including link changes that race with a modeset and recovery after + failed modesets. These tests can run on other architectures and do + not require the Apple display driver to be enabled. diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index b6844e04db00a..f9ca90b1dfb30 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -17,6 +17,7 @@ #include "dptxep.h" #include "iomfb.h" +#include "iomfb-state.h" #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "epic/dpavservep.h" @@ -212,11 +213,7 @@ struct apple_dcp { u64 swap_submit_timestamp; /* Current display mode */ - bool during_modeset; - /* hotplug reported while a modeset was in flight, applied afterwards */ - bool pending_hotplug; - bool pending_hotplug_connected; - bool valid_mode; + struct dcp_mode_state mode_state; /* One HPD pulse after a placeholder EDID, per Type-C connection. */ bool placeholder_retried; u64 typec_generation; /* hpd_mutex: identifies the current connection */ @@ -336,6 +333,7 @@ struct apple_dcp { }; void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); +void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); int dcp_backlight_register(struct apple_dcp *dcp); int dcp_backlight_update(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index db5f1127f8049..10e997730a0bb 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1257,7 +1257,8 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state) crtc_state = drm_atomic_get_new_crtc_state(state, crtc); - needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; + needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || + !READ_ONCE(dcp->mode_state.valid); if (!needs_modeset && (!dcp->connector || !dcp->connector->connected)) { /* * Resume restores the mode before the firmware reports the @@ -2396,6 +2397,7 @@ static int dcp_platform_probe(struct platform_device *pdev) */ mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->tb_lock); + spin_lock_init(&dcp->mode_state.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); dcp->fixed_connector_type = dcp_connector_type_from_dt(dev->of_node); dcp->connector_type = dcp->fixed_connector_type; diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 9eb0c7d4eb534..e7c4a195cfb10 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -189,7 +189,7 @@ int dcp_backlight_update(struct apple_dcp *dcp) * defer this hopefully until it becomes irrelevant due to proper * drm integrated backlight handling */ - if (!dcp->valid_mode) + if (!READ_ONCE(dcp->mode_state.valid)) return 0; /* Wait 1 vblank cycle in the hope an atomic swap has already updated diff --git a/drivers/gpu/drm/apple/iomfb-state-test.c b/drivers/gpu/drm/apple/iomfb-state-test.c new file mode 100644 index 0000000000000..d751625a94058 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-state-test.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "iomfb-state.h" + +static void dcp_mode_success_test(struct kunit *test) +{ + struct dcp_mode_state state = { }; + bool connected = true; + + spin_lock_init(&state.lock); + dcp_mode_begin(&state); + KUNIT_EXPECT_FALSE(test, state.valid); + KUNIT_EXPECT_EQ(test, dcp_mode_finish(&state, true, &connected), 0U); + KUNIT_EXPECT_TRUE(test, state.valid); + KUNIT_EXPECT_FALSE(test, state.changing); +} + +static void dcp_mode_failure_test(struct kunit *test) +{ + struct dcp_mode_state state = { .valid = true }; + bool connected = true; + + spin_lock_init(&state.lock); + dcp_mode_begin(&state); + dcp_mode_finish(&state, false, &connected); + KUNIT_EXPECT_FALSE(test, state.valid); + KUNIT_EXPECT_FALSE(test, state.changing); +} + +static void dcp_mode_deferred_disconnect_test(struct kunit *test) +{ + struct dcp_mode_state state = { }; + bool connected = true; + + spin_lock_init(&state.lock); + dcp_mode_begin(&state); + KUNIT_EXPECT_EQ(test, dcp_mode_hotplug(&state, false, &connected), 0U); + KUNIT_EXPECT_TRUE(test, connected); + KUNIT_EXPECT_EQ(test, dcp_mode_finish(&state, true, &connected), + (unsigned int)(DCP_HOTPLUG_VBLANK | DCP_HOTPLUG_NOTIFY)); + KUNIT_EXPECT_FALSE(test, connected); + KUNIT_EXPECT_FALSE(test, state.valid); + KUNIT_EXPECT_FALSE(test, state.hotplug_pending); +} + +static void dcp_mode_deferred_connect_test(struct kunit *test) +{ + struct dcp_mode_state state = { }; + bool connected = false; + + spin_lock_init(&state.lock); + dcp_mode_begin(&state); + dcp_mode_hotplug(&state, true, &connected); + KUNIT_EXPECT_EQ(test, dcp_mode_finish(&state, true, &connected), + (unsigned int)DCP_HOTPLUG_NOTIFY); + KUNIT_EXPECT_TRUE(test, connected); + KUNIT_EXPECT_FALSE(test, state.valid); + /* A recovery modeset without another link change can now succeed. */ + dcp_mode_begin(&state); + dcp_mode_finish(&state, true, &connected); + KUNIT_EXPECT_TRUE(test, state.valid); +} + +static void dcp_mode_transient_disconnect_test(struct kunit *test) +{ + struct dcp_mode_state state = { }; + bool connected = true; + + spin_lock_init(&state.lock); + dcp_mode_begin(&state); + dcp_mode_hotplug(&state, false, &connected); + dcp_mode_hotplug(&state, true, &connected); + KUNIT_EXPECT_EQ(test, dcp_mode_finish(&state, true, &connected), + (unsigned int)DCP_HOTPLUG_NOTIFY); + KUNIT_EXPECT_TRUE(test, connected); + KUNIT_EXPECT_FALSE(test, state.valid); +} + +static void dcp_mode_invalidation_order_test(struct kunit *test) +{ + struct dcp_mode_state state = { }; + bool connected = true; + + spin_lock_init(&state.lock); + /* Invalidation just before completion must win. */ + dcp_mode_begin(&state); + dcp_mode_invalidate(&state); + dcp_mode_finish(&state, true, &connected); + KUNIT_EXPECT_FALSE(test, state.valid); + /* So must invalidation just after completion. */ + dcp_mode_begin(&state); + dcp_mode_finish(&state, true, &connected); + dcp_mode_invalidate(&state); + KUNIT_EXPECT_FALSE(test, state.valid); +} + +static void dcp_mode_disconnected_test(struct kunit *test) +{ + struct dcp_mode_state state = { }; + bool connected = false; + + spin_lock_init(&state.lock); + dcp_mode_begin(&state); + dcp_mode_finish(&state, true, &connected); + KUNIT_EXPECT_FALSE(test, state.valid); +} + +static struct kunit_case dcp_mode_cases[] = { + KUNIT_CASE(dcp_mode_success_test), + KUNIT_CASE(dcp_mode_failure_test), + KUNIT_CASE(dcp_mode_deferred_disconnect_test), + KUNIT_CASE(dcp_mode_deferred_connect_test), + KUNIT_CASE(dcp_mode_transient_disconnect_test), + KUNIT_CASE(dcp_mode_invalidation_order_test), + KUNIT_CASE(dcp_mode_disconnected_test), + { } +}; + +static struct kunit_suite dcp_mode_suite = { + .name = "apple-iomfb-mode-state", + .test_cases = dcp_mode_cases, +}; + +kunit_test_suite(dcp_mode_suite); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/iomfb-state.h b/drivers/gpu/drm/apple/iomfb-state.h new file mode 100644 index 0000000000000..a9ada00a8528b --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-state.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __APPLE_IOMFB_STATE_H__ +#define __APPLE_IOMFB_STATE_H__ + +#include + +/* Shared by the atomic commit worker and the firmware callback thread. */ +struct dcp_mode_state { + spinlock_t lock; /* Protects all state and hotplug connector updates. */ + bool valid; + bool changing; + bool invalidated; + bool hotplug_pending; + bool connected_pending; +}; + +enum dcp_hotplug_action { + DCP_HOTPLUG_NONE = 0, + DCP_HOTPLUG_VBLANK = BIT(0), + DCP_HOTPLUG_NOTIFY = BIT(1), +}; + +static inline void dcp_mode_begin(struct dcp_mode_state *state) +{ + unsigned long flags; + + spin_lock_irqsave(&state->lock, flags); + state->valid = false; + state->changing = true; + state->invalidated = false; + state->hotplug_pending = false; + spin_unlock_irqrestore(&state->lock, flags); +} + +static inline void dcp_mode_invalidate(struct dcp_mode_state *state) +{ + unsigned long flags; + + spin_lock_irqsave(&state->lock, flags); + state->valid = false; + state->invalidated = true; + spin_unlock_irqrestore(&state->lock, flags); +} + +/* The caller holds state->lock. */ +static inline unsigned int +dcp_mode_apply_hotplug(struct dcp_mode_state *state, bool connected, + bool *connector_connected) +{ + unsigned int action = DCP_HOTPLUG_NONE; + + if (!connected) { + state->valid = false; + action |= DCP_HOTPLUG_VBLANK; + } + if (connector_connected && *connector_connected != connected) { + *connector_connected = connected; + state->valid = false; + action |= DCP_HOTPLUG_NOTIFY; + } + return action; +} + +static inline unsigned int +dcp_mode_hotplug(struct dcp_mode_state *state, bool connected, + bool *connector_connected) +{ + unsigned long flags; + unsigned int action = DCP_HOTPLUG_NONE; + + spin_lock_irqsave(&state->lock, flags); + if (state->changing) { + state->hotplug_pending = true; + state->connected_pending = connected; + /* A later connect must not erase a link loss during this call. */ + if (!connected) + state->invalidated = true; + } else { + action = dcp_mode_apply_hotplug(state, connected, + connector_connected); + } + spin_unlock_irqrestore(&state->lock, flags); + return action; +} + +static inline unsigned int +dcp_mode_finish(struct dcp_mode_state *state, bool completed, + bool *connector_connected) +{ + unsigned long flags; + unsigned int action = DCP_HOTPLUG_NONE; + + spin_lock_irqsave(&state->lock, flags); + state->valid = completed && !state->invalidated; + state->changing = false; + if (state->hotplug_pending) { + state->hotplug_pending = false; + action = dcp_mode_apply_hotplug(state, state->connected_pending, + connector_connected); + } + if (connector_connected && !*connector_connected) + state->valid = false; + /* A transient unplug also needs recovery if the final state is up. */ + if (state->invalidated && connector_connected && *connector_connected) + action |= DCP_HOTPLUG_NOTIFY; + spin_unlock_irqrestore(&state->lock, flags); + return action; +} + +#endif diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 2ab5e2a040379..d21a15e396283 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -254,11 +254,19 @@ static int dcp_retrain_active_crtc(struct apple_connector *connector) return ret; } +void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action) +{ + if (action & DCP_HOTPLUG_VBLANK) + schedule_work(&dcp->vblank_wq); + if ((action & DCP_HOTPLUG_NOTIFY) && dcp->connector) + schedule_work(&dcp->connector->hotplug_wq); +} + void dcp_retrain_oob(struct apple_connector *connector) { struct apple_dcp *dcp = platform_get_drvdata(connector->dcp); - if (!READ_ONCE(connector->connected) || !READ_ONCE(dcp->valid_mode)) + if (!READ_ONCE(connector->connected)) return; /* @@ -267,7 +275,7 @@ void dcp_retrain_oob(struct apple_connector *connector) * use the normal hotplug worker to replay the active CRTC from process * context; sending a synthetic disconnect would tear down the connector. */ - WRITE_ONCE(dcp->valid_mode, false); + dcp_mode_invalidate(&dcp->mode_state); schedule_work(&connector->hotplug_wq); } @@ -281,7 +289,7 @@ void dcp_hotplug(struct work_struct *work) dcp = platform_get_drvdata(connector->dcp); dev_info(dcp->dev, "%s() connected:%d valid_mode:%d nr_modes:%u\n", __func__, - connector->connected, dcp->valid_mode, dcp->nr_modes); + connector->connected, READ_ONCE(dcp->mode_state.valid), dcp->nr_modes); if (!connector->connected) { drm_edid_free(connector->drm_edid); @@ -293,7 +301,7 @@ void dcp_hotplug(struct work_struct *work) * display modes from atomic_flush, so userspace needs to trigger a * flush, or the CRTC gets no signal. */ - if (connector->base.state && !dcp->valid_mode && connector->connected && + if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connector->connected && !(dcp_is_usb4_output(dcp) && of_machine_is_compatible("apple,j416s"))) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); @@ -505,6 +513,7 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, struct apple_crtc *apple_crtc = to_apple_crtc(crtc); struct apple_dcp *dcp = platform_get_drvdata(apple_crtc->dcp); struct drm_crtc_state *crtc_state; + unsigned int action; int ret = -EIO; bool modeset; @@ -512,7 +521,8 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, if (!crtc_state) return 0; - modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; + modeset = drm_atomic_crtc_needs_modeset(crtc_state) || + !READ_ONCE(dcp->mode_state.valid); if (!modeset) return 0; @@ -521,6 +531,7 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, if (crtc_state->mode.hdisplay == 0 && crtc_state->mode.vdisplay == 0) return 0; + dcp_mode_begin(&dcp->mode_state); switch (dcp->fw_compat) { case DCP_FIRMWARE_V_12_3: ret = iomfb_modeset_v12_3(dcp, crtc_state); @@ -534,6 +545,10 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, break; } + action = dcp_mode_finish(&dcp->mode_state, !ret, dcp->connector ? + &dcp->connector->connected : NULL); + dcp_handle_hotplug_actions(dcp, action); + return ret; } @@ -565,7 +580,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) * re-runs the modeset, which marks the mode valid again, before the * firmware has reported the display back. */ - if (!dcp->valid_mode || !dcp->connector || !dcp->connector->connected) { + if (!READ_ONCE(dcp->mode_state.valid) || !dcp->connector || !dcp->connector->connected) { schedule_work(&dcp->vblank_wq); return; } @@ -649,7 +664,7 @@ void iomfb_shutdown(struct apple_dcp *dcp) { /* We're going down */ dcp->active = false; - dcp->valid_mode = false; + dcp_mode_invalidate(&dcp->mode_state); switch (dcp->fw_compat) { case DCP_FIRMWARE_V_12_3: diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 5780523896a23..823704dded12e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1028,6 +1028,7 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) { struct apple_connector *connector = dcp->connector; + unsigned int action; /* DCP issues hotplug_gated callbacks after SetPowerState() calls on * devices with display (macbooks, imacs). This must not result in @@ -1053,42 +1054,14 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) if (!(*connected) && READ_ONCE(dcp->typec_crtc_off) && READ_ONCE(dcp->typec_cable_connected)) { dev_dbg(dcp->dev, "cb_hotplug() ignoring unplug of powered-off Type-C output\n"); - dcp->valid_mode = false; + dcp_mode_invalidate(&dcp->mode_state); schedule_work(&dcp->vblank_wq); return; } - if (dcp->during_modeset) { - /* - * Remember it rather than dropping it. Resume re-runs the - * modeset and the firmware reports the display back while that - * is still in flight; discarding it leaves the connector marked - * disconnected forever, with no further event to correct it. - */ - dev_info(dcp->dev, - "cb_hotplug() deferred during modeset connected:%llu\n", - *connected); - dcp->pending_hotplug = true; - dcp->pending_hotplug_connected = !!(*connected); - return; - } - - dev_info(dcp->dev, "cb_hotplug() connected:%llu, valid_mode:%d nr_modes:%u\n", - *connected, dcp->valid_mode, dcp->nr_modes); - - /* Hotplug invalidates mode. DRM doesn't always handle this. */ - if (!(*connected)) { - dcp->valid_mode = false; - /* after unplug swap will not complete until the next - * set_digital_out_mode */ - schedule_work(&dcp->vblank_wq); - } - - if (connector && connector->connected != !!(*connected)) { - connector->connected = !!(*connected); - dcp->valid_mode = false; - schedule_work(&connector->hotplug_wq); - } + action = dcp_mode_hotplug(&dcp->mode_state, !!(*connected), + connector ? &connector->connected : NULL); + dcp_handle_hotplug_actions(dcp, action); } static void @@ -1220,7 +1193,8 @@ static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) { struct dcp_swap_start_req start_req = { 0 }; - if (dcp->connector && dcp->connector->connected) + if (READ_ONCE(dcp->mode_state.valid) && dcp->connector && + READ_ONCE(dcp->connector->connected)) dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); else dcp_drm_crtc_vblank(dcp->crtc); @@ -1318,7 +1292,6 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, /* increase refcount to ensure the receiver has a reference */ kref_get(&cookie->refcount); - dcp->during_modeset = true; dcp->swap_submit_timestamp = 0; if (mode->vrr) @@ -1337,25 +1310,6 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(8500)); - dcp->during_modeset = false; - - if (dcp->pending_hotplug) { - bool connected = dcp->pending_hotplug_connected; - struct apple_connector *connector = dcp->connector; - - dcp->pending_hotplug = false; - dev_info(dcp->dev, "replaying deferred hotplug connected:%d\n", - connected); - - if (!connected) - dcp->valid_mode = false; - - if (connector && connector->connected != connected) { - connector->connected = connected; - dcp->valid_mode = false; - schedule_work(&connector->hotplug_wq); - } - } dev_info(dcp->dev, "set_digital_out_mode finished:%d\n", ret); if (ret == 0) { @@ -1373,7 +1327,6 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, jiffies_to_msecs(ret)); } kref_put(&cookie->refcount, release_wait_cookie); - dcp->valid_mode = true; dcp->vrr_enabled = mode->vrr && crtc_state->vrr_enabled; return 0; From 27b6c41bdd799b5d88067598e039aaa034b40da2 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:13:07 -0700 Subject: [PATCH 0275/1579] phy: apple: describe tunnel clock and crossbar state directly Document the accepted initial gate state and the clock and reset operations performed by these helpers. Remove unrelated implementation narratives. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/mux/apple-display-crossbar.c | 2 +- drivers/phy/apple/atc.c | 17 +++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index 31788a927ea74..e7a8470aa7c79 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -261,7 +261,7 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) dpxbar_set32(dpxbar, T602X_FIFO_WR_N_CLK_EN, prev_dispext_bit); dpxbar_set32(dpxbar, T602X_REG_014, 0x4); - /* Native T602x DPIN0 teardown restores the read reset at +0x24. */ + /* Restore the selected input's FIFO read reset. */ dpxbar_set32(dpxbar, index == MUX_DPIN0 ? T602X_FIFO_RD_PCLK2_EN : FIFO_RD_PCLK1_EN, atc_bit); diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index d41bcb651567a..43e40765d895a 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2287,11 +2287,9 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) /* * Before the first rate request, TX_DP_CTRL0 gate bits read 0xe001: * the enable gates are set, the byte-clock reset is asserted, and the - * selector and reset-release bits are clear. Native configureDPTunnelMode - * tracks tunnel clients in software rather than through these gates, so - * this exact state is the only additional gate configuration accepted - * here, together with no enabled PLL output, no outstanding command, - * and no lock. Any other gate state is refused. + * selector and reset-release bits are clear. Accept this initial gate + * state only with no enabled PLL output, no outstanding command and + * no lock. Any other active gate state is refused. */ if (outputs & 0x54 || command & AUSPLL_APB_CMD_OVERRIDE_REQ || status & ACIOPHY_AUSPLL_LOCK) @@ -2309,7 +2307,7 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) atcphy->tunnel_saved_regs[i] = readl(atcphy->regs.core + atc_tunnel_regs[i].reg); atcphy->tunnel_saved = true; - /* Native configureDPTunnelMode, first clock client; no lane mux writes. */ + /* Wake the common clock blocks without changing the lane mux. */ core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL_OV); udelay(2); @@ -2346,7 +2344,7 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, FIELD_PREP(DPTX_PCLK1_SELECT, selector)); - /* Native fixed descriptor at cache VA 0xfffffe0007582418. */ + /* Program the fixed tunnel clock descriptor. */ core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); core_mask32(atcphy, AUSPLL_FREQ_DESC_A, 0xffffffff, 0x21c | (8 << 14) | (3 << 18) | (8 << 22) | (7 << 26)); @@ -2459,11 +2457,6 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) /* Take the USB3 PHY out of reset */ core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); - /* - * NOTE: Not what macos does, it takes the phy out of reset right before writing to ACIOPHY_CFG0_COMMON_SMALL - * does lane configuration with phy out of reset, and the poll is right after AUS_COMMON_SHIM_BLK_BIAS_REG - * write. Does not seem to matter, but if something does not work, consider re-ordering things to macos order. - */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) { ret = readl_poll_timeout(atcphy->regs.core + AUS_COMMON_DIG_RCAL1, reg, (reg & AUS_COMMON_DIG_RCAL1_ALL_CODES_DONE), 10, 100000); From 6f0c260445acd8243a45651ea1308f251d1090f5 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:15:05 -0700 Subject: [PATCH 0276/1579] phy: apple: atc: invalidate the cached rate before reprogramming Once PLL programming starts, an acknowledgment or lock timeout leaves the old cached DP rate stale. A later request for that old rate then returns success without restoring its hardware configuration. Invalidate the rate after the command-ready check and before the first register change. Only a completed configuration may cache the new rate. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/phy/apple/atc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 43e40765d895a..92932e7dd956a 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1803,6 +1803,8 @@ static int atcphy_dp_configure(struct apple_atcphy *atcphy, enum atcphy_dp_link_ return ret; } + /* The old rate no longer describes hardware once reprogramming starts. */ + atcphy->dp_link_rate = -1; core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_FREQ_COUNT_TARGET, From 25261a4e56db6903051360825418b562468c0ba3 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:15:19 -0700 Subject: [PATCH 0277/1579] thunderbolt: finish DPRX work before releasing router ports Serialize DPRX completion and cancellation with the domain lock. A worker that cannot acquire the lock reschedules instead of blocking, allowing teardown to cancel it synchronously while holding that lock. Complete canceled callbacks exactly once so their domain references are released, and drain them before domain stop removes the routers. Drop the worker tunnel reference before unlocking because its paths still refer to router ports. Synchronous DPRX does not need a worker reference. Update the sole production callback to the serialized contract and add a KUnit regression for pending cancellation, callback delivery and reference balance. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/tb.c | 7 ++-- drivers/thunderbolt/test.c | 65 ++++++++++++++++++++++++++++++++++++ drivers/thunderbolt/tunnel.c | 36 ++++++++++++-------- drivers/thunderbolt/tunnel.h | 2 +- 4 files changed, 92 insertions(+), 18 deletions(-) diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 9eb5d17c5d18f..d822259595345 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -1975,7 +1975,9 @@ static void tb_dp_tunnel_active(struct tb_tunnel *tunnel, void *data) struct tb_port *out = tunnel->dst_port; struct tb *tb = data; - mutex_lock(&tb->lock); + lockdep_assert_held(&tb->lock); + if (tunnel->dprx_canceled) + goto out; if (tb_tunnel_is_active(tunnel)) { int consumed_up, consumed_down, ret; @@ -2022,8 +2024,7 @@ static void tb_dp_tunnel_active(struct tb_tunnel *tunnel, void *data) tb_dp_resource_unavailable(tb, in, "DPRX negotiation failed"); } - mutex_unlock(&tb->lock); - +out: tb_domain_put(tb); } diff --git a/drivers/thunderbolt/test.c b/drivers/thunderbolt/test.c index 2ecd1e4d9afff..b1fec0a3a5c8a 100644 --- a/drivers/thunderbolt/test.c +++ b/drivers/thunderbolt/test.c @@ -2991,6 +2991,70 @@ struct tb_test_dp_host { unsigned int disconnects; }; +struct tb_test_dprx { + struct kunit *test; + unsigned int completions; +}; + +static void tb_test_dprx_complete(struct tb_tunnel *tunnel, void *data) +{ + struct tb_test_dprx *ctx = data; + + lockdep_assert_held(&tunnel->tb->lock); + KUNIT_EXPECT_TRUE(ctx->test, tunnel->dprx_canceled); + ctx->completions++; +} + +static void tb_test_dp_dprx_cancel(struct kunit *test) +{ + struct tb_test_dprx ctx = { .test = test }; + struct tb_switch *host, *dev; + struct tb_tunnel *tunnel; + struct tb *tb; + + tb = kunit_kzalloc(test, sizeof(*tb), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, tb); + host = alloc_host(test); + KUNIT_ASSERT_NOT_NULL(test, host); + dev = alloc_dev_default(test, host, 0x1, true); + KUNIT_ASSERT_NOT_NULL(test, dev); + tunnel = tb_tunnel_alloc_dp(tb, &host->ports[5], &dev->ports[13], + 1, 0, 0, tb_test_dprx_complete, &ctx); + KUNIT_ASSERT_NOT_NULL(test, tunnel); + tb->wq = alloc_ordered_workqueue("tb-dprx-test", 0); + if (!tb->wq) { + tb_tunnel_put(tunnel); + KUNIT_FAIL(test, "failed to allocate DPRX workqueue"); + return; + } + mutex_init(&tb->lock); + mutex_lock(&tb->lock); + + /* Model a pending poll holding the callback's tunnel reference. */ + kref_get(&tunnel->kref); + tunnel->dprx_started = true; + queue_delayed_work(tb->wq, &tunnel->dprx_work, 60 * HZ); + tb_dp_tunnel_deactivate_host(tunnel); + KUNIT_EXPECT_EQ(test, ctx.completions, 1U); + KUNIT_EXPECT_FALSE(test, tunnel->dprx_started); + KUNIT_EXPECT_TRUE(test, tunnel->dprx_canceled); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&tunnel->dprx_work)); + KUNIT_EXPECT_EQ(test, kref_read(&tunnel->kref), 1U); + + /* A second stop must not release the callback's resources again. */ + tb_dp_tunnel_deactivate_host(tunnel); + KUNIT_EXPECT_EQ(test, ctx.completions, 1U); + KUNIT_EXPECT_EQ(test, kref_read(&tunnel->kref), 1U); + + /* Also leave a failed cancellation test with no pending work. */ + if (cancel_delayed_work_sync(&tunnel->dprx_work)) + tb_tunnel_put(tunnel); + mutex_unlock(&tb->lock); + destroy_workqueue(tb->wq); + tb_tunnel_put(tunnel); + mutex_destroy(&tb->lock); +} + static void tb_test_dp_host_disconnect(struct tb_nhi *nhi, struct tb_port *in, struct tb_port *out) { @@ -3081,6 +3145,7 @@ static void tb_test_tunnel_dp_host_credits(struct kunit *test) static struct kunit_case tb_test_cases[] = { KUNIT_CASE(tb_test_pci_host_teardown), KUNIT_CASE(tb_test_pci_host_daisy_chain), + KUNIT_CASE(tb_test_dp_dprx_cancel), KUNIT_CASE(tb_test_tunnel_dp_host_credits), KUNIT_CASE(tb_test_dp_host_teardown_once), KUNIT_CASE(tb_test_dp_host_teardown_unannounced), diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index d8bcce8010771..988331a249f7f 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -1170,8 +1170,13 @@ static void tb_dp_dprx_work(struct work_struct *work) struct tb_tunnel *tunnel = container_of(work, typeof(*tunnel), dprx_work.work); struct tb *tb = tunnel->tb; + /* Teardown cancels this worker while holding the domain lock. */ + if (!mutex_trylock(&tb->lock)) { + queue_delayed_work(tb->wq, &tunnel->dprx_work, + msecs_to_jiffies(TB_DPRX_POLL_DELAY)); + return; + } if (!tunnel->dprx_canceled) { - mutex_lock(&tb->lock); if (tb_dp_is_usb4(tunnel->src_port->sw) && tb_dp_wait_dprx(tunnel, TB_DPRX_WAIT_TIMEOUT)) { if (ktime_before(ktime_get(), tunnel->dprx_timeout)) { @@ -1189,24 +1194,18 @@ static void tb_dp_dprx_work(struct work_struct *work) } else { tb_tunnel_set_active(tunnel, true); } - mutex_unlock(&tb->lock); } + tunnel->dprx_started = false; if (tunnel->callback) tunnel->callback(tunnel, tunnel->callback_data); + /* Paths still reference router ports, so release them before teardown. */ tb_tunnel_put(tunnel); + mutex_unlock(&tb->lock); } static int tb_dp_dprx_start(struct tb_tunnel *tunnel) { - /* - * Bump up the reference to keep the tunnel around. It will be - * dropped in tb_dp_dprx_stop() once the tunnel is deactivated. - */ - tb_tunnel_get(tunnel); - - tunnel->dprx_started = true; - if (tb_nhi_is_apple(tunnel->tb->nhi)) { tb_tunnel_warn(tunnel, "Apple: DP tunnel paths up, not waiting for DPRX\n"); @@ -1214,6 +1213,10 @@ static int tb_dp_dprx_start(struct tb_tunnel *tunnel) } if (tunnel->callback) { + /* The worker or cancellation drops this reference exactly once. */ + tb_tunnel_get(tunnel); + tunnel->dprx_started = true; + tunnel->dprx_canceled = false; tunnel->dprx_timeout = dprx_timeout_to_ktime(dprx_timeout); queue_delayed_work(tunnel->tb->wq, &tunnel->dprx_work, 0); return -EINPROGRESS; @@ -1226,10 +1229,14 @@ static int tb_dp_dprx_start(struct tb_tunnel *tunnel) static void tb_dp_dprx_stop(struct tb_tunnel *tunnel) { if (tunnel->dprx_started) { + lockdep_assert_held(&tunnel->tb->lock); tunnel->dprx_started = false; tunnel->dprx_canceled = true; - if (cancel_delayed_work(&tunnel->dprx_work)) - tb_tunnel_put(tunnel); + cancel_delayed_work_sync(&tunnel->dprx_work); + /* The callback also owns resources, including its domain reference. */ + if (tunnel->callback) + tunnel->callback(tunnel, tunnel->callback_data); + tb_tunnel_put(tunnel); } } @@ -1279,6 +1286,7 @@ void tb_dp_tunnel_deactivate_host(struct tb_tunnel *tunnel) { const struct tb_nhi_ops *ops; + tb_dp_dprx_stop(tunnel); if (!tunnel->host_dp_activated) return; tunnel->host_dp_activated = false; @@ -1320,7 +1328,6 @@ static int tb_dp_activate(struct tb_tunnel *tunnel, bool active) } } else { tb_dp_tunnel_deactivate_host(tunnel); - tb_dp_dprx_stop(tunnel); tb_dp_port_hpd_clear(tunnel->src_port); tb_dp_port_set_hops(tunnel->src_port, 0, 0, 0); if (tb_port_is_dpout(tunnel->dst_port)) @@ -1978,7 +1985,8 @@ struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in, * after tb_tunnel_activate() once the tunnel has been fully activated. * It can call tb_tunnel_is_active() to check if activation was * successful (or if it returns %false there was some sort of issue). - * The @callback is called without @tb->lock held. + * The @callback is called with @tb->lock held. If DPRX was canceled, it + * must release its private resources without accessing the tunnel's ports. * * Return: Pointer to @struct tb_tunnel or %NULL in case of failure. */ diff --git a/drivers/thunderbolt/tunnel.h b/drivers/thunderbolt/tunnel.h index f11b4d9f459fc..00ceb84b2e956 100644 --- a/drivers/thunderbolt/tunnel.h +++ b/drivers/thunderbolt/tunnel.h @@ -62,7 +62,7 @@ enum tb_tunnel_state { * @allocated_down: Allocated downstream bandwidth (only for USB3) * @bw_mode: DP bandwidth allocation mode registers can be used to * determine consumed and allocated bandwidth - * @dprx_started: DPRX negotiation was started (tb_dp_dprx_start() was called for it) + * @dprx_started: DPRX has an outstanding worker reference and callback * @dprx_canceled: Was DPRX capabilities read poll canceled * @host_dp_notified: The NHI glue was told this DP tunnel is up (Apple hosts) * @host_dp_activated: The NHI post-activation hook owns display-side state From 7b690ad6f5be63fab8cd1cdd66664ee698b7ce73 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:16:07 -0700 Subject: [PATCH 0278/1579] thunderbolt: apple: report rejected live cable transitions When a live cable transition requires shutting down ACIO, return an error instead of reporting the requested mode as active. This lets the Type-C caller invalidate its mode cache and retry a later update. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/apple.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 9bb806c435bb8..46832e6a1d16c 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -2031,17 +2031,16 @@ static int apple_cio_tbt_switch_set(struct typec_thunderbolt_switch_dev *sw, return 0; /* - * Transitions between different cables without a shutdown inbetween are invalid and can - * only happen when there's a bug inside the Type-C PD driver. If we tried such a - * transition, ACIO would crash and then trigger some watchdog that would reset the entire - * SoC a few seconds later. Shutting down instead only makes the connected device not work - * but we should be able to recover once the next cable is plugged in. + * Changing live cable parameters requires an ACIO shutdown. Report the + * rejected transition so the caller does not cache the new mode as active. */ if (acio->current_cable_info && acio->target_cable_info) { dev_err(acio->dev, "Invalid cable transition from 0x%x to 0x%x, shutting down instead\n", acio->current_cable_info, acio->target_cable_info); acio->target_cable_info = 0; + apple_cio_stop(acio); + return -EINVAL; } /* From 5b35733a3ef356ca47642e65f3f75466bfcfe685 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:17:36 -0700 Subject: [PATCH 0279/1579] thunderbolt: test DPRX cancellation with the domain lock held Queue a DPRX worker ahead of a marker on an ordered workqueue while holding the domain mutex. The marker must complete before that mutex is released, proving that the worker returns rather than blocking teardown. Then cancel its rescheduled poll and check callback delivery and reference balance. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/test.c | 41 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/drivers/thunderbolt/test.c b/drivers/thunderbolt/test.c index b1fec0a3a5c8a..c4aeaf5d19f9f 100644 --- a/drivers/thunderbolt/test.c +++ b/drivers/thunderbolt/test.c @@ -2993,9 +2993,18 @@ struct tb_test_dp_host { struct tb_test_dprx { struct kunit *test; + struct completion worker_passed; + struct work_struct marker; unsigned int completions; }; +static void tb_test_dprx_marker(struct work_struct *work) +{ + struct tb_test_dprx *ctx = container_of(work, struct tb_test_dprx, marker); + + complete(&ctx->worker_passed); +} + static void tb_test_dprx_complete(struct tb_tunnel *tunnel, void *data) { struct tb_test_dprx *ctx = data; @@ -3005,7 +3014,7 @@ static void tb_test_dprx_complete(struct tb_tunnel *tunnel, void *data) ctx->completions++; } -static void tb_test_dp_dprx_cancel(struct kunit *test) +static void tb_test_dp_dprx_cancel_common(struct kunit *test, bool running) { struct tb_test_dprx ctx = { .test = test }; struct tb_switch *host, *dev; @@ -3027,13 +3036,30 @@ static void tb_test_dp_dprx_cancel(struct kunit *test) KUNIT_FAIL(test, "failed to allocate DPRX workqueue"); return; } + init_completion(&ctx.worker_passed); + INIT_WORK(&ctx.marker, tb_test_dprx_marker); mutex_init(&tb->lock); mutex_lock(&tb->lock); /* Model a pending poll holding the callback's tunnel reference. */ kref_get(&tunnel->kref); tunnel->dprx_started = true; - queue_delayed_work(tb->wq, &tunnel->dprx_work, 60 * HZ); + queue_delayed_work(tb->wq, &tunnel->dprx_work, running ? 0 : 60 * HZ); + if (running) { + /* + * The ordered queue runs this marker only after DPRX returns + * without acquiring the mutex that teardown still owns. + */ + queue_work(tb->wq, &ctx.marker); + if (!wait_for_completion_timeout(&ctx.worker_passed, 5 * HZ)) { + KUNIT_FAIL(test, "DPRX worker blocked on the domain mutex"); + /* Let a blocked worker finish before cleaning up the test. */ + tunnel->dprx_canceled = true; + mutex_unlock(&tb->lock); + cancel_delayed_work_sync(&tunnel->dprx_work); + mutex_lock(&tb->lock); + } + } tb_dp_tunnel_deactivate_host(tunnel); KUNIT_EXPECT_EQ(test, ctx.completions, 1U); KUNIT_EXPECT_FALSE(test, tunnel->dprx_started); @@ -3055,6 +3081,16 @@ static void tb_test_dp_dprx_cancel(struct kunit *test) mutex_destroy(&tb->lock); } +static void tb_test_dp_dprx_cancel(struct kunit *test) +{ + tb_test_dp_dprx_cancel_common(test, false); +} + +static void tb_test_dp_dprx_cancel_running(struct kunit *test) +{ + tb_test_dp_dprx_cancel_common(test, true); +} + static void tb_test_dp_host_disconnect(struct tb_nhi *nhi, struct tb_port *in, struct tb_port *out) { @@ -3146,6 +3182,7 @@ static struct kunit_case tb_test_cases[] = { KUNIT_CASE(tb_test_pci_host_teardown), KUNIT_CASE(tb_test_pci_host_daisy_chain), KUNIT_CASE(tb_test_dp_dprx_cancel), + KUNIT_CASE(tb_test_dp_dprx_cancel_running), KUNIT_CASE(tb_test_tunnel_dp_host_credits), KUNIT_CASE(tb_test_dp_host_teardown_once), KUNIT_CASE(tb_test_dp_host_teardown_unannounced), From 5837706b10e36677323a22cc91f3aae3c5427e36 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:21:51 -0700 Subject: [PATCH 0280/1579] drm/apple: pin DP service owners before recycling retired AFK slots Repeated DP service announcements consumed the fixed service table even when earlier services had been torn down. Reuse converted service slots only after owner pins and pending commands have drained, and retain retired channels for late command replies. Keep the command tag sequence when reclaiming a slot. Pin the EDID service while accessing it, serialize owner lookup with teardown, and discard a result if the service changed during the read. Keep services with unconverted owners nonrecyclable. Gate receive work against queue destruction and clean up failed DP endpoint startup. Add six KUnit cases covering repeated generations, reader pins, pending and timed-out commands, completed-but-unconsumed replies, tag continuity and owners that have not opted in to reuse. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/Makefile | 1 + drivers/gpu/drm/apple/Kconfig | 3 +- drivers/gpu/drm/apple/afk-service-test.c | 142 +++++++++++++++++++++++ drivers/gpu/drm/apple/afk.c | 118 +++++++++++++------ drivers/gpu/drm/apple/afk.h | 90 ++++++++++++++ drivers/gpu/drm/apple/av.c | 4 +- drivers/gpu/drm/apple/dcp.c | 2 + drivers/gpu/drm/apple/epic/dpavservep.c | 78 +++++++++++-- drivers/gpu/drm/apple/epic/dpavservep.h | 6 +- drivers/gpu/drm/apple/iomfb.c | 4 +- 10 files changed, 396 insertions(+), 52 deletions(-) create mode 100644 drivers/gpu/drm/apple/afk-service-test.c diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index a0c684efd523d..1ee47178266e1 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -229,6 +229,7 @@ obj-$(CONFIG_DRM_PANTHOR) += panthor/ obj-$(CONFIG_DRM_TYR) += tyr/ obj-$(CONFIG_DRM_APPLE) += apple/ obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple/iomfb-state-test.o +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple/afk-service-test.o obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/ obj-$(CONFIG_DRM_MCDE) += mcde/ obj-$(CONFIG_DRM_TIDSS) += tidss/ diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 65c7fb2256344..a0a597dc0355e 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -37,4 +37,5 @@ config DRM_APPLE_KUNIT_TEST Test modeset and hotplug state transitions without display hardware, including link changes that race with a modeset and recovery after failed modesets. These tests can run on other architectures and do - not require the Apple display driver to be enabled. + not require the Apple display driver to be enabled. Also test service + ownership and pending commands across disconnect and reannouncement. diff --git a/drivers/gpu/drm/apple/afk-service-test.c b/drivers/gpu/drm/apple/afk-service-test.c new file mode 100644 index 0000000000000..4a5f07fa9b15f --- /dev/null +++ b/drivers/gpu/drm/apple/afk-service-test.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "afk.h" + +static const struct apple_epic_service_ops reusable_ops = { + .name = "reusable", + .reusable = true, +}; + +static const struct apple_epic_service_ops permanent_ops = { + .name = "permanent", +}; + +static int afk_service_test_init(struct kunit *test) +{ + struct apple_epic_service *service; + + service = kunit_kzalloc(test, sizeof(*service), GFP_KERNEL); + if (!service) + return -ENOMEM; + spin_lock_init(&service->lock); + test->priv = service; + return 0; +} + +static void afk_test_retire(struct apple_epic_service *service) +{ + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + service->torndown = true; + spin_unlock_irqrestore(&service->lock, flags); +} + +static void afk_service_many_generations_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + unsigned int i; + + /* Repeated unplug/replug must not consume an unbounded slot budget. */ + for (i = 0; i < 4 * AFK_MAX_CHANNEL; i++) { + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, i)); + KUNIT_EXPECT_EQ(test, service->channel, i); + KUNIT_EXPECT_TRUE(test, afk_service_matches(service, i)); + afk_test_retire(service); + } +} + +static void afk_service_reader_pin_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 1)); + KUNIT_ASSERT_PTR_EQ(test, afk_service_get(service), service); /* owner */ + KUNIT_ASSERT_PTR_EQ(test, afk_service_get(service), service); /* reader */ + afk_test_retire(service); + afk_service_put(service); /* owner detaches while EDID is being read */ + KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + KUNIT_EXPECT_EQ(test, service->channel, 1U); + KUNIT_EXPECT_PTR_EQ(test, afk_service_get(service), NULL); + afk_service_put(service); + KUNIT_EXPECT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); +} + +static void afk_service_pending_reply_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 1)); + set_bit(3, service->cmd_map); + afk_test_retire(service); + KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + KUNIT_EXPECT_TRUE(test, afk_service_matches(service, 1)); + KUNIT_EXPECT_FALSE(test, afk_service_matches(service, 2)); + /* A timed-out command retains its DMA and slot until the late reply. */ + service->cmds[3].free_on_ack = true; + KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + clear_bit(3, service->cmd_map); + KUNIT_EXPECT_FALSE(test, afk_service_matches(service, 1)); + KUNIT_EXPECT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); +} + +static void afk_service_completed_reader_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 1)); + KUNIT_ASSERT_PTR_EQ(test, afk_service_get(service), service); + set_bit(0, service->cmd_map); + afk_test_retire(service); + /* Reply arrived, but the waiting thread has not consumed it yet. */ + service->cmds[0].done = true; + KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + clear_bit(0, service->cmd_map); + KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + afk_service_put(service); + KUNIT_EXPECT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); +} + +static void afk_service_tag_sequence_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 1)); + service->cmd_tag = 0xa9; + service->cookie = test; + afk_test_retire(service); + KUNIT_EXPECT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + KUNIT_EXPECT_EQ(test, service->cmd_tag, (u8)0xa9); + KUNIT_EXPECT_PTR_EQ(test, service->cookie, NULL); +} + +static void afk_service_unconverted_owner_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &permanent_ops, 1)); + afk_test_retire(service); + KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); +} + +static struct kunit_case afk_service_cases[] = { + KUNIT_CASE(afk_service_many_generations_test), + KUNIT_CASE(afk_service_reader_pin_test), + KUNIT_CASE(afk_service_pending_reply_test), + KUNIT_CASE(afk_service_completed_reader_test), + KUNIT_CASE(afk_service_tag_sequence_test), + KUNIT_CASE(afk_service_unconverted_owner_test), + { } +}; + +static struct kunit_suite afk_service_suite = { + .name = "apple-afk-service-lifetime", + .init = afk_service_test_init, + .test_cases = afk_service_cases, +}; + +kunit_test_suite(afk_service_suite); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 3fbc5fe4d601c..de2a83a9fcb60 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -57,7 +57,7 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, const struct apple_epic_service_ops *ops) { struct apple_dcp_afkep *afkep; - int ret; + int ret, i; afkep = devm_kzalloc(dcp->dev, sizeof(*afkep), GFP_KERNEL); if (!afkep) @@ -78,6 +78,8 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, init_completion(&afkep->started); init_completion(&afkep->stopped); spin_lock_init(&afkep->lock); + for (i = 0; i < AFK_MAX_CHANNEL; i++) + spin_lock_init(&afkep->services[i].lock); return afkep; @@ -88,6 +90,8 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, void afk_shutdown(struct apple_dcp_afkep *afkep) { + unsigned long flags; + afk_send(afkep, FIELD_PREP(RBEP_TYPE, RBEP_SHUTDOWN)); int ret; @@ -96,6 +100,9 @@ void afk_shutdown(struct apple_dcp_afkep *afkep) dev_err(afkep->dcp->dev, "Timed out shutting down AFK endpoint %02x", afkep->endpoint); } + spin_lock_irqsave(&afkep->lock, flags); + afkep->stopping = true; + spin_unlock_irqrestore(&afkep->lock, flags); destroy_workqueue(afkep->wq); } @@ -213,7 +220,7 @@ afk_match_service(struct apple_dcp_afkep *ep, const char *name) { const struct apple_epic_service_ops *ops; - if (!name[0]) + if (!name || !name[0]) return NULL; if (!ep->ops) return NULL; @@ -229,13 +236,18 @@ afk_match_service(struct apple_dcp_afkep *ep, const char *name) } static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep *ep, - u32 channel) + u32 channel) { - for (u32 i = 0; i < ep->num_channels; i++) - if (ep->services[i].enabled && ep->services[i].channel == channel) - return &ep->services[i]; + struct apple_epic_service *service; + u32 i; + + for (i = 0; i < ep->num_channels; i++) { + service = &ep->services[i]; + if (afk_service_matches(service, channel)) + return service; + } - return NULL; + return NULL; } static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, @@ -243,6 +255,7 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, { char name[32]; s64 epic_unit = -1; + struct apple_epic_service *service = NULL; u32 ch_idx; const char *service_name = name; const char *epic_name = NULL, *epic_class = NULL; @@ -251,7 +264,8 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *props = payload + sizeof(name); size_t props_size = payload_size - sizeof(name); - WARN_ON(afk_epic_find_service(ep, channel)); + if (afk_epic_find_service(ep, channel)) + return; if (payload_size < sizeof(name)) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", @@ -259,12 +273,6 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, return; } - if (ep->num_channels >= AFK_MAX_CHANNEL) { - dev_err(ep->dcp->dev, "AFK[ep:%02x]: too many enabled services!\n", - ep->endpoint); - return; - } - strscpy(name, payload, sizeof(name)); /* @@ -305,19 +313,25 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, goto free; } - ch_idx = ep->num_channels++; - spin_lock_init(&ep->services[ch_idx].lock); - ep->services[ch_idx].enabled = true; - ep->services[ch_idx].torndown = false; - ep->services[ch_idx].ops = ops; - ep->services[ch_idx].ep = ep; - ep->services[ch_idx].channel = channel; - ep->services[ch_idx].cmd_tag = 0; - ops->init(&ep->services[ch_idx], epic_name, epic_class, epic_unit); + for (ch_idx = 0; ch_idx < AFK_MAX_CHANNEL; ch_idx++) { + struct apple_epic_service *candidate = &ep->services[ch_idx]; + + if (afk_service_reinit(candidate, ep, ops, channel)) { + service = candidate; + break; + } + } + if (!service) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: no reusable service slots\n", + ep->endpoint); + goto free; + } + ep->num_channels = max(ep->num_channels, ch_idx + 1); + ops->init(service, epic_name, epic_class, epic_unit); dev_info(ep->dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", ep->endpoint, service_name, channel); - afk_populate_service_debugfs(&ep->services[ch_idx]); + afk_populate_service_debugfs(service); free: kfree(epic_name); @@ -339,14 +353,14 @@ static void afk_recv_handle_teardown(struct apple_dcp_afkep *ep, u32 channel) afk_remove_service_debugfs(service); - // TODO: think through what locking is necessary spin_lock_irqsave(&service->lock, flags); - /* - * teardown must not disable the service since since it may be sent as - * side effect of a COMMAND which for which a reply is expected. - * Seen with DCP's "av" endpoint during the close afk_service_call. - */ + if (service->torndown) { + spin_unlock_irqrestore(&service->lock, flags); + return; + } + /* Outstanding commands remain discoverable until their replies arrive. */ service->torndown = true; + service->enabled = false; ops = service->ops; spin_unlock_irqrestore(&service->lock, flags); @@ -387,7 +401,7 @@ static void afk_recv_handle_reply(struct apple_dcp_afkep *ep, u32 channel, } spin_lock_irqsave(&service->lock, flags); - if (service->cmds[idx].done) { + if (!test_bit(idx, service->cmd_map) || service->cmds[idx].done) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: command reply on channel %d already handled\n", ep->endpoint, channel); @@ -732,6 +746,10 @@ static void afk_receive_message_worker(struct work_struct *work_) int afk_receive_message(struct apple_dcp_afkep *ep, u64 message) { struct afk_receive_message_work *work; + unsigned long flags; + + if (!ep) + return -ENODEV; // TODO: comment why decoupling from rtkit thread is required here work = kzalloc(sizeof(*work), GFP_KERNEL); @@ -741,7 +759,14 @@ int afk_receive_message(struct apple_dcp_afkep *ep, u64 message) work->ep = ep; work->message = message; INIT_WORK(&work->work, afk_receive_message_worker); + spin_lock_irqsave(&ep->lock, flags); + if (ep->stopping) { + spin_unlock_irqrestore(&ep->lock, flags); + kfree(work); + return -ESHUTDOWN; + } queue_work(ep->wq, &work->work); + spin_unlock_irqrestore(&ep->lock, flags); return 0; } @@ -759,6 +784,10 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, int ret; spin_lock_irqsave(&ep->lock, flags); + if (ep->stopping) { + ret = -ESHUTDOWN; + goto out; + } dma_rmb(); rptr = le32_to_cpu(ep->txbfr.hdr->rptr); @@ -910,13 +939,20 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, unsigned long flags; int ret, idx; u16 tag; - struct apple_dcp_afkep *ep = service->ep; + struct apple_dcp_afkep *ep; DECLARE_COMPLETION_ONSTACK(completion); + service = afk_service_get(service); + if (!service) + return -ENODEV; + ep = service->ep; + rxbuf = dma_alloc_coherent(ep->dcp->dev, output_len, &rxbuf_dma, GFP_KERNEL); - if (!rxbuf) - return -ENOMEM; + if (!rxbuf) { + ret = -ENOMEM; + goto err_put_service; + } txbuf = dma_alloc_coherent(ep->dcp->dev, payload_len, &txbuf_dma, GFP_KERNEL); if (!txbuf) { @@ -934,6 +970,10 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, cmd.txlen = cpu_to_le32(payload_len); spin_lock_irqsave(&service->lock, flags); + if (service->torndown || !service->enabled) { + ret = -ENODEV; + goto err_unlock; + } idx = bitmap_find_free_region(service->cmd_map, MAX_PENDING_CMDS, 0); if (idx < 0) { ret = -ENOSPC; @@ -956,11 +996,10 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, service->cmds[idx].completion = &completion; init_completion(&completion); - spin_unlock_irqrestore(&service->lock, flags); - ret = afk_send_epic(service->ep, service->channel, tag, EPIC_TYPE_COMMAND, EPIC_CAT_COMMAND, type, &cmd, sizeof(cmd)); + spin_unlock_irqrestore(&service->lock, flags); if (ret) goto err_free_cmd; @@ -982,6 +1021,7 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, "AFK[ep:%02x]: %s (chan:%u) command type 0x%x tag 0x%04x timed out after %u ms\n", ep->endpoint, service->ops->name, service->channel, type, tag, timeout_ms); + afk_service_put(service); return -ETIMEDOUT; } spin_unlock_irqrestore(&service->lock, flags); @@ -995,12 +1035,15 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, err_free_cmd: spin_lock_irqsave(&service->lock, flags); + service->cmds[idx].completion = NULL; bitmap_release_region(service->cmd_map, idx, 0); err_unlock: spin_unlock_irqrestore(&service->lock, flags); dma_free_coherent(ep->dcp->dev, payload_len, txbuf, txbuf_dma); err_free_rxbuf: dma_free_coherent(ep->dcp->dev, output_len, rxbuf, rxbuf_dma); +err_put_service: + afk_service_put(service); return ret; } @@ -1026,6 +1069,9 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, u32 retcode; u32 retlen; + if (!service) + return -ENODEV; + bfr = kzalloc(bfr_len, GFP_KERNEL); if (!bfr) return -ENOMEM; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index d63d26b79d457..880987cbb6ef4 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -7,7 +7,9 @@ #ifndef _DRM_APPLE_DCP_AFK_H #define _DRM_APPLE_DCP_AFK_H +#include #include +#include #include #include @@ -43,6 +45,8 @@ struct apple_epic_service { DECLARE_BITMAP(cmd_map, MAX_PENDING_CMDS); u8 cmd_tag; spinlock_t lock; + /* Pins held by owners and synchronous callers; protected by lock. */ + unsigned int users; u32 channel; bool enabled; @@ -60,6 +64,8 @@ enum epic_subtype; struct apple_epic_service_ops { const char name[32]; + /* All retained pointers must be pinned before enabling slot reuse. */ + bool reusable; void (*init)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); @@ -71,6 +77,89 @@ struct apple_epic_service_ops { void (*teardown)(struct apple_epic_service *service); }; +static inline struct apple_epic_service *afk_service_get(struct apple_epic_service *service) +{ + unsigned long flags; + bool available; + + if (!service) + return NULL; + + spin_lock_irqsave(&service->lock, flags); + available = service->enabled && !service->torndown; + if (available) + service->users++; + spin_unlock_irqrestore(&service->lock, flags); + + return available ? service : NULL; +} + +static inline void afk_service_put(struct apple_epic_service *service) +{ + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + if (!WARN_ON(!service->users)) + service->users--; + spin_unlock_irqrestore(&service->lock, flags); +} + +static inline void afk_service_disable(struct apple_epic_service *service) +{ + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + spin_unlock_irqrestore(&service->lock, flags); +} + +static inline bool afk_service_matches(struct apple_epic_service *service, + u32 channel) +{ + unsigned long flags; + bool found; + + spin_lock_irqsave(&service->lock, flags); + /* Teardown does not cancel commands or relinquish their DMA. */ + found = service->channel == channel && + (service->enabled || + !bitmap_empty(service->cmd_map, MAX_PENDING_CMDS)); + spin_unlock_irqrestore(&service->lock, flags); + return found; +} + +/* The caller holds service->lock. */ +static inline bool afk_service_reclaimable(struct apple_epic_service *service) +{ + return !service->ops || + (service->ops->reusable && service->torndown && + !service->enabled && !service->users && + bitmap_empty(service->cmd_map, MAX_PENDING_CMDS)); +} + +static inline bool +afk_service_reinit(struct apple_epic_service *service, + struct apple_dcp_afkep *ep, + const struct apple_epic_service_ops *ops, u32 channel) +{ + unsigned long flags; + bool reusable; + + spin_lock_irqsave(&service->lock, flags); + reusable = afk_service_reclaimable(service); + if (reusable) { + service->enabled = true; + service->torndown = false; + service->ops = ops; + service->ep = ep; + service->channel = channel; + service->cookie = NULL; + /* Keep the command-tag sequence across service generations. */ + } + spin_unlock_irqrestore(&service->lock, flags); + return reusable; +} + struct afk_ringbuffer_header { __le32 bufsz; u32 unk; @@ -177,6 +266,7 @@ struct apple_dcp_afkep { spinlock_t lock; u16 qe_seq; + bool stopping; /* lock: no new receive work after shutdown. */ const struct apple_epic_service_ops *ops; struct apple_epic_service services[AFK_MAX_CHANNEL]; diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 0d3c752f62d5f..4ea77ab21ea4a 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -79,7 +79,7 @@ static void av_interface_teardown(struct apple_epic_service *service) struct apple_dcp *dcp = service->ep->dcp; struct audiosrv_data *asrv = dcp->audiosrv; - service->enabled = false; + afk_service_disable(service); mutex_lock(&asrv->plug_lock); @@ -351,7 +351,7 @@ void av_service_disconnect(struct apple_dcp *dcp) dev_err(dcp->dev, "error closing audio service: %d\n", ret); } if (service->torndown) - service->enabled = false; + afk_service_disable(service); asrv->is_open = false; } diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 10e997730a0bb..44f05df0dc056 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2337,6 +2337,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->dcpavservep) { afk_shutdown(dcp->dcpavservep); + dpavservep_detach(dcp); dcp->dcpavservep = NULL; } @@ -2398,6 +2399,7 @@ static int dcp_platform_probe(struct platform_device *pdev) mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); + spin_lock_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); dcp->fixed_connector_type = dcp_connector_type_from_dt(dev->of_node); dcp->connector_type = dcp->fixed_connector_type; diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 2de9d2fe4c24a..cfcf3e184f5a7 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -18,31 +18,57 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; + unsigned long flags; + trace_dcpavserv_init(dcp, unit); if (unit == 0 && name && !strcmp(name, "dcpav-service-epic")) { + spin_lock_irqsave(&dcp->dcpavserv.lock, flags); if (dcp->dcpavserv.enabled) { + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); dev_err(dcp->dev, "DCPAVSERV: unit %lld already exists\n", unit); return; } - dcp->dcpavserv.service = service; + dcp->dcpavserv.service = afk_service_get(service); dcp->dcpavserv.enabled = true; service->cookie = &dcp->dcpavserv; complete(&dcp->dcpavserv.enable_completion); + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); } } static void dcpavserv_teardown(struct apple_epic_service *service) { struct apple_dcp *dcp = service->ep->dcp; - service->enabled = false; + unsigned long flags; + bool owned = false; - if (dcp->dcpavserv.enabled) { + spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + if (dcp->dcpavserv.service == service) { dcp->dcpavserv.enabled = false; dcp->dcpavserv.service = NULL; - service->cookie = NULL; reinit_completion(&dcp->dcpavserv.enable_completion); + owned = true; + } + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + if (owned) + afk_service_put(service); +} + +void dpavservep_detach(struct apple_dcp *dcp) +{ + struct apple_epic_service *service; + unsigned long flags; + + spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + service = dcp->dcpavserv.service; + dcp->dcpavserv.service = NULL; + dcp->dcpavserv.enabled = false; + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + if (service) { + afk_service_disable(service); + afk_service_put(service); } } @@ -53,7 +79,7 @@ static void dcpdpserv_init(struct apple_epic_service *service, const char *name, static void dcpdpserv_teardown(struct apple_epic_service *service) { - service->enabled = false; + afk_service_disable(service); } struct dcpavserv_status_report { @@ -155,7 +181,7 @@ static int dcpdpserv_report(struct apple_epic_service *service, return parse_report(service, type, data, data_size); } -const struct drm_edid *dcpavserv_copy_edid(struct apple_epic_service *service) +static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *service) { struct dpavserv_copy_edid_cmd cmd; struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; @@ -192,15 +218,41 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_epic_service *service) data_size - EDID_LEADING_DATA_SIZE); } +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) +{ + struct apple_epic_service *service; + const struct drm_edid *edid; + unsigned long flags; + + spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + service = afk_service_get(dcp->dcpavserv.service); + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + if (!service) + return ERR_PTR(-ENODEV); + + edid = dcpavserv_read_edid(service); + spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + if (dcp->dcpavserv.service != service) { + if (!IS_ERR(edid)) + drm_edid_free(edid); + edid = ERR_PTR(-ENODEV); + } + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + afk_service_put(service); + return edid; +} + static const struct apple_epic_service_ops dpavservep_ops[] = { { .name = "dcpav-service-epic", + .reusable = true, .init = dcpavserv_init, .teardown = dcpavserv_teardown, .report = dcpavserv_report, }, { .name = "dcpdp-service-epic", + .reusable = true, .init = dcpdpserv_init, .teardown = dcpdpserv_teardown, .report = dcpdpserv_report, @@ -210,19 +262,25 @@ static const struct apple_epic_service_ops dpavservep_ops[] = { int dpavservep_init(struct apple_dcp *dcp) { + struct apple_dcp_afkep *ep; int ret; init_completion(&dcp->dcpavserv.enable_completion); - dcp->dcpavservep = afk_init(dcp, DPAVSERV_ENDPOINT, dpavservep_ops); - if (IS_ERR(dcp->dcpavservep)) - return PTR_ERR(dcp->dcpavservep); + ep = afk_init(dcp, DPAVSERV_ENDPOINT, dpavservep_ops); + if (IS_ERR(ep)) + return PTR_ERR(ep); + dcp->dcpavservep = ep; dcp->dcpavservep->match_epic_name = true; ret = afk_start(dcp->dcpavservep); - if (ret) + if (ret) { + afk_shutdown(ep); + dpavservep_detach(dcp); + dcp->dcpavservep = NULL; return ret; + } ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, msecs_to_jiffies(1000)); diff --git a/drivers/gpu/drm/apple/epic/dpavservep.h b/drivers/gpu/drm/apple/epic/dpavservep.h index 858ff14b0bd7b..e9ab17a7302c1 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.h +++ b/drivers/gpu/drm/apple/epic/dpavservep.h @@ -5,18 +5,22 @@ #define _DRM_APPLE_EPIC_DPAVSERV_H #include +#include #include struct drm_edid; struct apple_epic_service; +struct apple_dcp; struct dcpavserv { + spinlock_t lock; /* Protects the service pointer and its owner pin. */ bool enabled; struct completion enable_completion; u32 unit; struct apple_epic_service *service; }; -const struct drm_edid *dcpavserv_copy_edid(struct apple_epic_service *service); +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp); +void dpavservep_detach(struct apple_dcp *dcp); #endif /* _DRM_APPLE_EPIC_DPAVSERV_H */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index d21a15e396283..5a80cc57735e4 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -446,10 +446,10 @@ int dcp_get_modes(struct drm_connector *connector) } drm_connector_set_vrr_capable_property(connector, vrr_capable); - if (dcp->nr_modes && dcp->dcpavserv.enabled && + if (dcp->nr_modes && !apple_connector->drm_edid) { const struct drm_edid *edid; - edid = dcpavserv_copy_edid(dcp->dcpavserv.service); + edid = dcpavserv_copy_edid(dcp); if (IS_ERR_OR_NULL(edid)) { dev_info(dcp->dev, "copy_edid failed: %pe\n", edid); } else { From 63861816dc0c1055e760aa9405df4a2a0091ec3d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:24:07 -0700 Subject: [PATCH 0281/1579] drm/apple: preallocate the AFK receive dispatch buffer Allocate receive scratch space while initializing the ring so dispatch cannot fail allocation after a receive notification has been consumed. Each endpoint has an ordered worker and dispatch completes synchronously before the next entry is copied. Free scratch only after receive admission has closed and the worker queue has drained. Reject rings smaller than an aligned entry before marking them ready. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/afk.c | 23 +++++++++++++++++------ drivers/gpu/drm/apple/afk.h | 2 ++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index de2a83a9fcb60..404ff9cf6e7ce 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -104,6 +104,10 @@ void afk_shutdown(struct apple_dcp_afkep *afkep) afkep->stopping = true; spin_unlock_irqrestore(&afkep->lock, flags); destroy_workqueue(afkep->wq); + if (afkep->rx_scratch) { + devm_kfree(afkep->dcp->dev, afkep->rx_scratch); + afkep->rx_scratch = NULL; + } } int afk_start(struct apple_dcp_afkep *ep) @@ -194,6 +198,17 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, ep->endpoint, bufsz, sizeof(*bfr->hdr)); return; } + if (bufsz < BIT(BLOCK_SHIFT)) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: ring cannot hold an entry\n", + ep->endpoint); + return; + } + if (bfr == &ep->rxbfr) { + /* Receive dispatch cannot recurse on the ordered endpoint queue. */ + ep->rx_scratch = devm_kmalloc(ep->dcp->dev, bufsz, GFP_KERNEL); + if (!ep->rx_scratch) + return; + } bfr->buf = bfr->hdr + 1; bfr->bufsz = bufsz; @@ -606,7 +621,6 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, static bool afk_recv(struct apple_dcp_afkep *ep) { struct afk_qe *hdr; - u8 *data; u32 rptr, wptr; u32 magic, size, channel, type; @@ -677,9 +691,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) * sleep and send replies, so retain a private copy before allowing * the producer to reuse the ring storage. */ - data = kmemdup(hdr->data, size, GFP_KERNEL); - if (!data) - return false; + memcpy(ep->rx_scratch, hdr->data, size); rptr = ALIGN(rptr + sizeof(*hdr) + size, 1 << BLOCK_SHIFT); if (WARN_ON(rptr > ep->rxbfr.bufsz)) @@ -692,8 +704,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); trace_afk_recv_rwptr_post(ep, rptr, wptr); - afk_recv_handle(ep, channel, type, data, size); - kfree(data); + afk_recv_handle(ep, channel, type, ep->rx_scratch, size); return true; } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 880987cbb6ef4..2713372d48f60 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -263,6 +263,8 @@ struct apple_dcp_afkep { struct afk_ringbuffer txbfr; struct afk_ringbuffer rxbfr; + /* Private receive entry, owned by the ordered endpoint worker. */ + void *rx_scratch; spinlock_t lock; u16 qe_seq; From 3be32e8f2cd1f8e327ecd1ade024696688ff9a9c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:24:58 -0700 Subject: [PATCH 0282/1579] drm/apple: order AFK entries before publishing the producer position Place a DMA write barrier between filling the entry and updating the shared producer position. A consumer already draining an earlier notification must not observe the new position before its payload. The existing send barrier still orders the position before notification. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/afk.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 404ff9cf6e7ce..04f31f3661797 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -920,6 +920,8 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, wptr = 0; trace_afk_send_rwptr_post(ep, rptr, wptr); + /* Publish the completed entry before allowing the consumer to read it. */ + dma_wmb(); ep->txbfr.hdr->wptr = cpu_to_le32(wptr); afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_SEND) | FIELD_PREP(SEND_WPTR, wptr)); From 176e73706a8071199cc714d713a7d943c053c1ca Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:26:19 -0700 Subject: [PATCH 0283/1579] drm/apple: copy the complete DP service EDID response The EDID reply includes a 48-byte header before the advertised data area. Use that structure size for both allocation and the output copy instead of using the pointer size for the latter, which truncated the largest reply by 40 bytes on 64-bit systems. Reject reported data lengths beyond the advertised capacity before examining the EDID blocks. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/epic/dpavservep.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index cfcf3e184f5a7..b9f7c619dd4ed 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -113,6 +113,7 @@ struct dpavserv_copy_edid_resp { u8 _pad2[8]; u8 data[]; } __packed; +static_assert(sizeof(struct dpavserv_copy_edid_resp) == 48); static int parse_report(struct apple_epic_service *service, enum epic_subtype type, const void *data, size_t data_size) @@ -185,18 +186,19 @@ static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *ser { struct dpavserv_copy_edid_cmd cmd; struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; + size_t resp_size = sizeof(*resp) + EDID_BUF_SIZE; int num_blocks; u64 data_size; int ret; memset(&cmd, 0, sizeof(cmd)); cmd.max_size = cpu_to_le64(EDID_BUF_SIZE); - resp = kzalloc(sizeof(*resp) + EDID_BUF_SIZE, GFP_KERNEL); + resp = kzalloc(resp_size, GFP_KERNEL); if (!resp) return ERR_PTR(-ENOMEM); ret = afk_service_call(service, 1, 7, &cmd, sizeof(cmd), EDID_BUF_SIZE, resp, - sizeof(resp) + EDID_BUF_SIZE, 0); + resp_size, 0); if (ret < 0) return ERR_PTR(ret); @@ -207,7 +209,8 @@ static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *ser // 16, 1, resp, 192, true); data_size = le64_to_cpu(resp->used_size); - if (data_size < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE) + if (data_size < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE || + data_size > EDID_BUF_SIZE) return ERR_PTR(-EIO); num_blocks = resp->data[EDID_LEADING_DATA_SIZE + EDID_EXT_BLOCK_COUNT_OFFSET]; From ce341d4f81c453226ca11a9971370c89d0da28c1 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:27:40 -0700 Subject: [PATCH 0284/1579] drm/apple: bound AFK debug requests and reject partial user copies Reject raw requests larger than the shared 8192-byte request/reply buffer before copying user data. Treat every nonzero copy_from_user result as a fault so partially copied command headers or payloads cannot be sent. Handle empty raw writes without submitting a command. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/afk.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 04f31f3661797..4845811ea9fb2 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1172,8 +1172,7 @@ static ssize_t service_call_write_file(struct file *file, const char __user *use return -ENOMEM; } - ret = copy_from_user(&call_info, user_buf, sizeof(call_info)); - if (ret == sizeof(call_info)) + if (copy_from_user(&call_info, user_buf, sizeof(call_info))) return -EFAULT; user_buf += sizeof(call_info); count -= sizeof(call_info); @@ -1181,8 +1180,7 @@ static ssize_t service_call_write_file(struct file *file, const char __user *use buf = kmalloc(count, GFP_KERNEL); if (!buf) return -ENOMEM; - ret = copy_from_user(buf, user_buf, count); - if (ret == count) { + if (copy_from_user(buf, user_buf, count)) { kfree(buf); return -EFAULT; } @@ -1225,6 +1223,11 @@ static ssize_t service_raw_call_write_file(struct file *file, const char __user u32 retcode; int ret; + if (!count) + return 0; + if (count > AFK_DEBUGFS_MAX_REPLY) + return -E2BIG; + if (!srv->debugfs.scratch) { srv->debugfs.scratch = \ devm_kzalloc(srv->ep->dcp->dev, AFK_DEBUGFS_MAX_REPLY, GFP_KERNEL); @@ -1233,8 +1236,7 @@ static ssize_t service_raw_call_write_file(struct file *file, const char __user } memset(srv->debugfs.scratch, 0, AFK_DEBUGFS_MAX_REPLY); - ret = copy_from_user(srv->debugfs.scratch, user_buf, count); - if (ret == count) + if (copy_from_user(srv->debugfs.scratch, user_buf, count)) return -EFAULT; ret = afk_send_command(srv, EPIC_SUBTYPE_STD_SERVICE, srv->debugfs.scratch, count, From 34c204d124760df5f2d0b056269063720321c6b7 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:27:58 -0700 Subject: [PATCH 0285/1579] drm/apple: free the temporary audio object reply buffer Release the temporary audio service request/reply allocation on both success and error returns. Copying the result to the caller does not transfer ownership of that allocation. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/av.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 4ea77ab21ea4a..97c68359c1232 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -196,7 +197,7 @@ dcp_audiosrv_osobject_call(struct apple_epic_service *service, u16 group, } __attribute__((packed)) *hdr; static_assert(sizeof(*hdr) == 48); size_t bfr_len = output_maxsize + sizeof(*hdr); - void *bfr; + void *bfr __free(kfree) = NULL; int ret; bfr = kzalloc(bfr_len, GFP_KERNEL); From 3841d47bf96a9c180e86be44950249ce9d547ee4 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:30:36 -0700 Subject: [PATCH 0286/1579] drm/apple: report audio query failures without reading unset sizes The warning guards were inverted, suppressing the first error and leaving the warned flags false. That error also took the success debug branch, although the output size is only assigned by a successful query. Report the first failed query and log the returned size only on success. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/av.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 97c68359c1232..563a02bb4e551 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -232,10 +232,10 @@ int dcp_audiosrv_get_elements(struct device *dev, void *elements, size_t maxsize elements, maxsize, &size); up_write(&asrv->srv_rwsem); - if (ret && asrv->warned_get_elements) { + if (ret && !asrv->warned_get_elements) { dev_err(dev, "audiosrv: error getting elements: %d\n", ret); asrv->warned_get_elements = true; - } else { + } else if (!ret) { dev_dbg(dev, "audiosrv: got %zd bytes worth of elements\n", size); } @@ -255,10 +255,10 @@ int dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsi maxsize, &size); up_write(&asrv->srv_rwsem); - if (ret && asrv->warned_get_product_attrs) { + if (ret && !asrv->warned_get_product_attrs) { dev_err(dev, "audiosrv: error getting product attributes: %d\n", ret); asrv->warned_get_product_attrs = true; - } else { + } else if (!ret) { dev_dbg(dev, "audiosrv: got %zd bytes worth of product attributes\n", size); } From 615be505e22c592c1a87c193b22c58dc4c064840 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:35:13 -0700 Subject: [PATCH 0287/1579] thunderbolt: restore M1 tunneled PCIe state and recover late link loss Kensington USB devices can disappear after s2idle even when the initial port restart and cable health check succeed. A later PCIe link failure must trigger another fresh validation instead of waiting for a cable interrupt or manual replug. Restore the resettable M1 tunneled host and DART state before restarting the downstream link. Gate DART commands and fault handling around clock shutdown, preserve stream selection, and describe the reset and register initialization properties in the binding. Gate Type-C work across initialization, sleep and removal, read fresh cable metadata after resume, and preserve an unchanged healthy session. Propagate late link failures through an atomic switch notification into bounded process-context revalidation. Keep failure history sticky while allowing later notifications from a retained host. Failed reads or health checks request a bounded reconnect; successful checks preserve routing. Keep failed display-route acquisition retryable. Add state-machine and DART regression coverage. The native ARM64 build and 88 KUnit cases pass. Two M1/j293 Kensington USB-only suspend cycles exercised late link-loss recovery and restored usable input without a manual replug. The initiating xHCI resume fault remains unresolved. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- .../devicetree/bindings/pci/apple,pcie.yaml | 50 ++- arch/arm64/boot/dts/apple/t8103.dtsi | 2 + drivers/gpu/drm/apple/dcp.c | 9 +- drivers/iommu/apple-dart-test.c | 167 ++++++++ drivers/iommu/apple-dart.c | 96 ++++- drivers/pci/controller/pcie-apple.c | 195 ++++++++- drivers/thunderbolt/apple.c | 121 +++++- drivers/usb/typec/mux.c | 50 +++ drivers/usb/typec/mux.h | 2 + drivers/usb/typec/tipd/Kconfig | 10 + drivers/usb/typec/tipd/Makefile | 2 + drivers/usb/typec/tipd/cd321x-pm-test.c | 399 ++++++++++++++++++ drivers/usb/typec/tipd/cd321x-pm.h | 143 +++++++ drivers/usb/typec/tipd/core.c | 301 +++++++++---- drivers/usb/typec/tipd/i2c.c | 2 + drivers/usb/typec/tipd/spmi.c | 2 + drivers/usb/typec/tipd/tps6598x.h | 16 +- include/linux/pci-apple.h | 8 + include/linux/soc/apple/dart.h | 5 + include/linux/usb/typec_mux.h | 27 ++ 20 files changed, 1500 insertions(+), 107 deletions(-) create mode 100644 drivers/usb/typec/tipd/cd321x-pm-test.c create mode 100644 drivers/usb/typec/tipd/cd321x-pm.h diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 6a6608dca4d1b..72a688e7f2940 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -36,6 +36,16 @@ description: | distributed over the root ports as the OS sees fit by programming the PCIe controller's port registers. +$defs: + tunable: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask of bits to clear + - description: Bits to set + description: Register initialization tuples for one controller region. + properties: compatible: oneOf: @@ -84,8 +94,34 @@ properties: power-domains: maxItems: 1 + resets: + description: + Reset of a tunneled (PCIe-C) controller and its DART, for controllers + the kernel initializes itself. + maxItems: 1 + dma-coherent: true + apple,pciec-kernel-init: + type: boolean + description: The kernel initializes the tunneled controller from reset. + + apple,tunable-rc: + $ref: '#/$defs/tunable' + description: Root-complex register initialization tuples. + + apple,tunable-fabric: + $ref: '#/$defs/tunable' + description: Initialization tuples for the fabric register region. + + apple,tunable-debug: + $ref: '#/$defs/tunable' + description: Initialization tuples for the debug register region. + + apple,tunable-oe-fabric: + $ref: '#/$defs/tunable' + description: Initialization tuples for the oe-fabric register region. + patternProperties: "^pci@": $ref: /schemas/pci/pci-bus.yaml# @@ -102,6 +138,10 @@ patternProperties: interrupt-controller: true + apple,tunable: + $ref: '#/$defs/tunable' + description: Initialization tuples relative to this port's registers. + required: - interrupt-controller - "#interrupt-cells" @@ -172,7 +212,7 @@ allOf: - const: fabric - const: debug - const: ltssm - - const: intr2axi + - enum: [intr2axi, oe-fabric] interrupts: maxItems: 1 ranges: @@ -180,6 +220,12 @@ allOf: maxItems: 3 else: properties: + resets: false + apple,pciec-kernel-init: false + apple,tunable-rc: false + apple,tunable-fabric: false + apple,tunable-debug: false + apple,tunable-oe-fabric: false reg-names: minItems: 3 items: @@ -197,6 +243,8 @@ allOf: maxItems: 2 patternProperties: "^pci@": + properties: + apple,tunable: false required: - reset-gpios diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 5ff23a499eb95..28aedb75a2afd 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -1685,6 +1685,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_atc0_pcie>; + resets = <&ps_atc0_pcie>; msi-controller; msi-parent = <&apciec0>; @@ -1910,6 +1911,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_atc1_pcie>; + resets = <&ps_atc1_pcie>; msi-controller; msi-parent = <&apciec1>; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 44f05df0dc056..0e6489c14da2a 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -378,12 +378,15 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, port->applied_mode = state->mode; port->applied_status = dp_status; port->applied_conf = dp_conf; - port->applied_valid = true; + /* Failed route acquisition must remain retryable on the next update. */ + port->applied_valid = false; if (!is_dp) { /* a Thunderbolt/USB4 DP tunnel is torn down by its own path */ - if (port->owner && port->owner->tunnel) + if (port->owner && port->owner->tunnel) { + port->applied_valid = true; return 0; + } if (port->owner) { struct apple_dcp *dcp = port->owner->dcp; @@ -403,6 +406,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (state->mode == TYPEC_MODE_USB4) dcp_typec_retrain_active_routes(); + port->applied_valid = true; return 0; } @@ -455,6 +459,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, dcp_retrain_oob(dcp->typec_connector); } port->hpd = hpd; + port->applied_valid = true; return 0; } diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 687d097bd3e5b..0eba2eca17474 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -212,12 +212,179 @@ static void apple_dart_test_resume_commands_failure(struct kunit *test) pm_runtime_disable(dart->dev); } +static struct apple_dart *apple_dart_test_tunnel(struct kunit *test) +{ + struct apple_dart *dart; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->dev = kunit_device_register(test, "apple-dart-tunnel-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); + dart->regs = (__force void __iomem *)kunit_kzalloc(test, SZ_16K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart->regs); + dart->hw = &apple_dart_hw_t8103_usb4; + dart->num_streams = dart->hw->max_sid_count; + dart->tunneled = true; + spin_lock_init(&dart->lock); + dev_set_drvdata(dart->dev, dart); + return dart; +} + +static void apple_dart_test_tunnel_restore(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + unsigned int sid, idx; + u32 sentinel = 0x40; + + /* M1 tunnel DART nodes have no retained power-domain attachment. */ + KUNIT_ASSERT_FALSE(test, dart->power_retained); + for (sid = 0; sid < dart->num_streams; sid++) { + writel(0x80 + sid, dart->regs + DART_TCR(dart, sid)); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + writel(0x1000 + 0x10 * sid + idx, + dart->regs + DART_TTBR(dart, sid, idx)); + } + KUNIT_ASSERT_EQ(test, apple_dart_save_tunnel_state(dart->dev), 0); + memset_io(dart->regs, 0, SZ_16K); + KUNIT_ASSERT_EQ(test, apple_dart_restore_tunnel_state(dart->dev), 0); + for (sid = 0; sid < dart->num_streams; sid++) { + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, sid)), + 0x80 + sid); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, sid, idx)), + 0x1000 + 0x10 * sid + idx); + } + KUNIT_EXPECT_FALSE(test, dart->tunnel_state_saved); + KUNIT_EXPECT_TRUE(test, dart->tunnel_state_restored); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_SELECT), U32_MAX); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_SELECT + 4), U32_MAX); + + /* Normal resume must not reset translations after the link is live. */ + writel(sentinel, dart->regs + DART_T8020_STREAM_COMMAND); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), sentinel); + KUNIT_EXPECT_FALSE(test, dart->tunnel_state_restored); + KUNIT_EXPECT_EQ(test, apple_dart_restore_tunnel_state(dart->dev), -EINVAL); +} + +static void apple_dart_test_tunnel_reject_unsafe(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + u32 sentinel = 0x40; + + writel(sentinel, dart->regs + DART_T8020_STREAM_COMMAND); + KUNIT_EXPECT_EQ(test, apple_dart_restore_tunnel_state(dart->dev), -EINVAL); + dart->locked = true; + KUNIT_EXPECT_EQ(test, apple_dart_save_tunnel_state(dart->dev), -EOPNOTSUPP); + dart->locked = false; + dart->commands_gated = true; + KUNIT_EXPECT_EQ(test, apple_dart_save_tunnel_state(dart->dev), -EBUSY); + KUNIT_EXPECT_FALSE(test, dart->tunnel_state_saved); + dart->commands_gated = false; + dart->hw = &apple_dart_hw_t8110; + KUNIT_EXPECT_EQ(test, apple_dart_save_tunnel_state(dart->dev), -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), sentinel); +} + +static int apple_dart_test_tlb_failure(struct apple_dart_stream_map *stream_map) +{ + return -EIO; +} + +static void apple_dart_test_tunnel_reset_failure(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + struct apple_dart_hw hw = apple_dart_hw_t8103_usb4; + + hw.invalidate_tlb = apple_dart_test_tlb_failure; + dart->hw = &hw; + KUNIT_ASSERT_EQ(test, apple_dart_save_tunnel_state(dart->dev), 0); + KUNIT_EXPECT_EQ(test, apple_dart_restore_tunnel_state(dart->dev), -EIO); + KUNIT_EXPECT_FALSE(test, dart->tunnel_state_restored); + KUNIT_EXPECT_TRUE(test, dart->tunnel_state_saved); + + /* A later successful reset can still restore the saved mappings. */ + dart->hw = &apple_dart_hw_t8103_usb4; + KUNIT_EXPECT_EQ(test, apple_dart_restore_tunnel_state(dart->dev), 0); + KUNIT_EXPECT_TRUE(test, dart->tunnel_state_restored); + KUNIT_EXPECT_EQ(test, apple_dart_suspend(dart->dev), 0); + KUNIT_EXPECT_FALSE(test, dart->tunnel_state_restored); +} + +static void apple_dart_test_failed_host_pm(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + u32 sentinel = 0x40; + + writel(0x80, dart->regs + DART_TCR(dart, 0)); + KUNIT_ASSERT_EQ(test, apple_dart_save_tunnel_state(dart->dev), 0); + apple_dart_quiesce_commands(dart->dev); + writel(sentinel, dart->regs + DART_T8020_STREAM_COMMAND); + writel(sentinel, dart->regs + DART_TCR(dart, 0)); + + /* PM still resumes dependents after the host's noirq callback fails. */ + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), sentinel); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 0)), sentinel); + KUNIT_EXPECT_TRUE(test, dart->commands_gated); + + /* Another sleep must neither access the failed port nor lose its snapshot. */ + KUNIT_EXPECT_EQ(test, apple_dart_suspend(dart->dev), 0); + KUNIT_EXPECT_EQ(test, dart->save_tcr[0], 0x80U); + KUNIT_EXPECT_EQ(test, apple_dart_save_tunnel_state(dart->dev), -EBUSY); + KUNIT_EXPECT_TRUE(test, dart->tunnel_state_saved); + KUNIT_EXPECT_EQ(test, dart->save_tcr[0], 0x80U); +} + +static irqreturn_t apple_dart_test_irq_handler(int irq, void *data) +{ + struct apple_dart *dart = data; + + writel(0x1234, dart->regs + DART_TCR(dart, 0)); + return IRQ_HANDLED; +} + +static void apple_dart_test_gated_irq(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + struct apple_dart_hw hw = apple_dart_hw_t8103_usb4; + + hw.irq_handler = apple_dart_test_irq_handler; + dart->hw = &hw; + pm_runtime_set_active(dart->dev); + pm_runtime_enable(dart->dev); + apple_dart_quiesce_commands(dart->dev); + KUNIT_EXPECT_EQ(test, apple_dart_irq(0, dart), IRQ_NONE); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 0)), 0U); + KUNIT_EXPECT_EQ(test, atomic_read(&dart->dev->power.usage_count), 0); + + KUNIT_ASSERT_EQ(test, apple_dart_resume_commands(dart->dev), 0); + KUNIT_EXPECT_EQ(test, apple_dart_irq(0, dart), IRQ_HANDLED); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 0)), 0x1234U); + KUNIT_EXPECT_EQ(test, atomic_read(&dart->dev->power.usage_count), 0); + + /* Failed runtime resume must not run the handler or leak its reference. */ + writel(0, dart->regs + DART_TCR(dart, 0)); + scoped_guard(spinlock_irqsave, &dart->dev->power.lock) + dart->dev->power.runtime_error = -EIO; + KUNIT_EXPECT_EQ(test, apple_dart_irq(0, dart), IRQ_NONE); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 0)), 0U); + KUNIT_EXPECT_EQ(test, atomic_read(&dart->dev->power.usage_count), 0); + KUNIT_EXPECT_EQ(test, pm_runtime_set_active(dart->dev), 0); + pm_runtime_disable(dart->dev); +} + static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_before_attach), KUNIT_CASE(apple_dart_test_locked_handoff), KUNIT_CASE(apple_dart_test_firmware_roots), KUNIT_CASE(apple_dart_test_gated_commands), KUNIT_CASE(apple_dart_test_resume_commands_failure), + KUNIT_CASE(apple_dart_test_tunnel_restore), + KUNIT_CASE(apple_dart_test_tunnel_reject_unsafe), + KUNIT_CASE(apple_dart_test_tunnel_reset_failure), + KUNIT_CASE(apple_dart_test_failed_host_pm), + KUNIT_CASE(apple_dart_test_gated_irq), {} }; diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index d38fd34cd0fb4..5565b45a345c8 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -260,6 +260,8 @@ struct apple_dart { u32 locked : 1; u32 tunneled : 1; u32 power_retained : 1; + u32 tunnel_state_saved : 1; + u32 tunnel_state_restored : 1; bool commands_gated; struct apple_tunable *tunables; u16 version; @@ -660,6 +662,7 @@ apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, struct apple_dart *dart = stream_map->dart; unsigned long flags; int ret, i; + u32 sidmap[BITS_TO_U32(DART_MAX_STREAMS)]; u32 command_reg; spin_lock_irqsave(&dart->lock, flags); @@ -668,9 +671,10 @@ apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, return -EHOSTDOWN; } + bitmap_to_arr32(sidmap, stream_map->sidmap, dart->num_streams); for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) - apple_dart_writel(dart, stream_map->sidmap[i], - DART_T8020_STREAM_SELECT + 4 * i); + apple_dart_writel(dart, sidmap[i], + DART_T8020_STREAM_SELECT + 4 * i); apple_dart_writel(dart, command, DART_T8020_STREAM_COMMAND); ret = read_poll_timeout_atomic( @@ -1896,11 +1900,15 @@ static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) static irqreturn_t apple_dart_irq(int irq, void *dev) { - irqreturn_t ret; + irqreturn_t ret = IRQ_NONE; struct apple_dart *dart = dev; - WARN_ON(pm_runtime_get_sync(dart->dev) < 0); - ret = dart->hw->irq_handler(irq, dev); + if (READ_ONCE(dart->commands_gated)) + return IRQ_NONE; + if (pm_runtime_resume_and_get(dart->dev) < 0) + return IRQ_NONE; + if (!READ_ONCE(dart->commands_gated)) + ret = dart->hw->irq_handler(irq, dev); pm_runtime_put(dart->dev); return ret; } @@ -2243,8 +2251,11 @@ int apple_dart_quiesce_commands(struct device *dev) if (dart) { /* Finish any in-flight command before the port clock is stopped. */ - guard(spinlock_irqsave)(&dart->lock); - WRITE_ONCE(dart->commands_gated, true); + scoped_guard(spinlock_irqsave, &dart->lock) + WRITE_ONCE(dart->commands_gated, true); + /* The fault handler can take dart->lock while acknowledging errors. */ + if (dart->irq > 0) + synchronize_irq(dart->irq); } return 0; } @@ -2272,6 +2283,66 @@ int apple_dart_resume_commands(struct device *dev) } EXPORT_SYMBOL_GPL(apple_dart_resume_commands); +/* + * System sleep has quiesced every downstream device. PCIe-C calls this + * before stopping APPCLK because its domain reset also loses DART state, + * even when ordinary runtime PM treats the domain as retained. + */ +int apple_dart_save_tunnel_state(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid, idx; + + if (!dart || !dart->tunneled || dart->locked || + dart->hw->type != DART_T8020 || dart->tunables) + return -EOPNOTSUPP; + + guard(spinlock_irqsave)(&dart->lock); + if (dart->commands_gated) + return -EBUSY; + dart->tunnel_state_saved = false; + dart->tunnel_state_restored = false; + + for (sid = 0; sid < dart->num_streams; sid++) { + dart->save_tcr[sid] = apple_dart_readl(dart, DART_TCR(dart, sid)); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + dart->save_ttbr[sid][idx] = + apple_dart_readl(dart, DART_TTBR(dart, sid, idx)); + } + dart->tunnel_state_saved = true; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_save_tunnel_state); + +/* APPCLK is running again, but PCIe link training has not started. */ +int apple_dart_restore_tunnel_state(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid, idx; + int ret; + + if (!dart || !dart->tunneled || dart->locked || + dart->hw->type != DART_T8020 || dart->tunables) + return -EOPNOTSUPP; + if (!dart->tunnel_state_saved || READ_ONCE(dart->commands_gated)) + return -EINVAL; + + ret = apple_dart_hw_reset(dart); + if (ret) + return ret; + + for (sid = 0; sid < dart->num_streams; sid++) { + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + apple_dart_writel(dart, dart->save_ttbr[sid][idx], + DART_TTBR(dart, sid, idx)); + apple_dart_writel(dart, dart->save_tcr[sid], DART_TCR(dart, sid)); + } + dart->tunnel_state_saved = false; + dart->tunnel_state_restored = true; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_restore_tunnel_state); + static void apple_dart_remove(struct platform_device *pdev) { struct apple_dart *dart = platform_get_drvdata(pdev); @@ -2401,6 +2472,9 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) struct apple_dart *dart = dev_get_drvdata(dev); unsigned int sid, idx; + dart->tunnel_state_restored = false; + if (READ_ONCE(dart->commands_gated)) + return 0; /* The tunneled DART has no state to save when its domain stays on. */ if (dart->power_retained) return 0; @@ -2427,6 +2501,12 @@ static __maybe_unused int apple_dart_resume(struct device *dev) unsigned int sid, idx; int ret; + /* PCIe-C restored these mappings before starting its downstream link. */ + if (dart->tunnel_state_restored) { + dart->tunnel_state_restored = false; + return 0; + } + /* * PCIe-C DARTs on an apple,always-on domain retain their translation * state. Resetting one here is unnecessary; DART access must remain @@ -2448,7 +2528,7 @@ static __maybe_unused int apple_dart_resume(struct device *dev) for (sid = 0; sid < dart->num_streams; sid++) { for (idx = 0; idx < dart->hw->ttbr_count; idx++) apple_dart_writel(dart, dart->save_ttbr[sid][idx], - DART_TTBR(dart, sid, idx)); + DART_TTBR(dart, sid, idx)); apple_dart_writel(dart, dart->save_tcr[sid], DART_TCR(dart, sid)); } diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 59ab52a1e6c22..39dbeb1e504b2 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -28,17 +28,33 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include "../pci.h" #include "pci-host-common.h" +static ATOMIC_NOTIFIER_HEAD(apple_pcie_tunnel_notifiers); + +int apple_pcie_tunnel_register_notifier(struct notifier_block *nb) +{ + return atomic_notifier_chain_register(&apple_pcie_tunnel_notifiers, nb); +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_register_notifier); + +void apple_pcie_tunnel_unregister_notifier(struct notifier_block *nb) +{ + atomic_notifier_chain_unregister(&apple_pcie_tunnel_notifiers, nb); +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_unregister_notifier); + static int link_up_timeout = 500; module_param(link_up_timeout, int, 0644); MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); @@ -219,6 +235,9 @@ struct apple_pcie { bool power_retained; bool kernel_init; bool bus_stopped; + bool reset_on_resume; + bool resume_failed; /* cleared only by teardown and reprobe */ + struct reset_control *reset; const struct hw_info *hw; unsigned long *bitmap; struct list_head ports; @@ -246,6 +265,7 @@ struct apple_pcie_port { int idx; bool started; bool needs_stop; + bool link_failed; }; static void rmw_set(u32 set, void __iomem *addr) @@ -527,6 +547,8 @@ static void apple_port_irq_handler(struct irq_desc *desc) chained_irq_enter(chip, desc); stat = apple_pcie_port_readl(port, PORT_INTSTAT); + /* Masked link events can belong to the resume poller. */ + stat &= ~apple_pcie_port_readl(port, PORT_INTMSK); for_each_set_bit(i, &stat, 32) generic_handle_domain_irq(port->domain, i); @@ -597,6 +619,13 @@ static irqreturn_t apple_pcie_port_irq(int irq, void *data) complete_all(&port->pcie->event); break; case PORT_INT_LINK_DOWN: + /* A later link-up alone does not prove downstream state survived. */ + if (port->pcie->hw->tunneled && READ_ONCE(port->started)) { + WRITE_ONCE(port->link_failed, true); + atomic_notifier_call_chain(&apple_pcie_tunnel_notifiers, + APPLE_PCIE_TUNNEL_LINK_DOWN, + port->pcie->dev->parent); + } dev_info_ratelimited(port->pcie->dev, "Link down on %pOF\n", port->np); break; @@ -871,6 +900,8 @@ static void apple_pcie_tunnel_restore_irq_hw(struct apple_pcie_port *port) u32 value = 0; int i; + /* Keep link events pending until the resume poller has consumed them. */ + apple_pcie_port_writel(port, ~0, PORT_INTMSK); apple_pcie_port_writel(port, ~0, PORT_INTSTAT); apple_pcie_port_writel(port, ~0, PORT_LINKCMDSTS); apple_pcie_port_writel(port, lower_32_bits(DOORBELL_ADDR), @@ -889,7 +920,6 @@ static void apple_pcie_tunnel_restore_irq_hw(struct apple_pcie_port *port) value = ilog2(pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT; } apple_pcie_port_writel(port, value | PORT_MSICFG_EN, PORT_MSICFG); - apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); } static int apple_pcie_tunnel_start(struct apple_pcie_port *port) @@ -900,6 +930,9 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) u32 link_events = BIT(PORT_INT_LINK_UP) | BIT(PORT_INT_LINK_DOWN) | BIT(PORT_INT_TUNNEL_ERR); + if (pcie->resume_failed) + return -EIO; + /* A failed start still owns partially enabled port hardware. */ port->needs_stop = true; @@ -1027,10 +1060,12 @@ static int apple_pcie_tunnel_start(struct apple_pcie_port *port) return ret; } restored: - if (pcie->power_retained) - apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); + /* Publish ownership before unmasking a possible link-down event. */ + WRITE_ONCE(port->started, true); + /* The port helper uses a relaxed MMIO write. */ + wmb(); + apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); - port->started = true; dev_info(pcie->dev, "port %pOF tunnel link restored after suspend\n", port->np); return 0; @@ -1132,9 +1167,9 @@ static int apple_pcie_tunnel_stop(struct apple_pcie_port *port) } apple_pcie_port_rmw_clear(port, PORT_TUNCTRL_PERST_ACK_REQ, PORT_TUNCTRL); - port->started = false; - port->needs_stop = apple_pcie_port_readl(port, PORT_STATUS) & - PORT_STATUS_READY; + WRITE_ONCE(port->started, false); + port->needs_stop = err || (apple_pcie_port_readl(port, PORT_STATUS) & + PORT_STATUS_READY); return err; } @@ -1515,7 +1550,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (pcie->hw->tunneled && pcie->kernel_init) msleep(1000); if (pcie->hw->tunneled) - port->started = true; + WRITE_ONCE(port->started, true); return 0; @@ -2042,6 +2077,17 @@ static int apple_pcie_probe(struct platform_device *pdev) return ret; pcie->power_retained = apple_pcie_tunnel_power_is_retained(dev->of_node); + if (pcie->kernel_init) { + /* ACIO releases this line before populating the host. */ + pcie->reset = devm_reset_control_get_optional_exclusive_released(dev, NULL); + if (IS_ERR(pcie->reset)) + return dev_err_probe(dev, PTR_ERR(pcie->reset), + "PCIe-C reset unavailable\n"); + ret = reset_control_acquire(pcie->reset); + if (ret) + return dev_err_probe(dev, ret, + "PCIe-C reset is still in use\n"); + } } mutex_init(&pcie->lock); @@ -2084,10 +2130,12 @@ static int apple_pcie_probe(struct platform_device *pdev) typedef int (*apple_pcie_dart_fn)(struct device *dev); -static int apple_pcie_walk_tunnel_darts(struct apple_pcie *pcie, apple_pcie_dart_fn fn) +static int apple_pcie_walk_tunnel_darts(struct apple_pcie *pcie, + apple_pcie_dart_fn fn, bool required) { struct device_node *parent, *child; struct platform_device *pdev; + bool found = false; int ret = 0; parent = of_get_parent(pcie->dev->of_node); @@ -2105,14 +2153,17 @@ static int apple_pcie_walk_tunnel_darts(struct apple_pcie *pcie, apple_pcie_dart if (pdev) { int err = fn(&pdev->dev); + found = true; if (err && !ret) ret = err; put_device(&pdev->dev); + } else if (required && !ret) { + ret = -ENODEV; } } of_node_put(parent); - return ret; + return ret ?: (required && !found ? -ENODEV : 0); } int apple_pcie_tunnel_quiesce(struct device *dev) @@ -2159,7 +2210,7 @@ int apple_pcie_tunnel_quiesce(struct device *dev) * Removing the IOMMU later resumes it and issues a command. Gate * those commands before APPCLK goes away with the port. */ - ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_quiesce_commands); + ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_quiesce_commands, false); list_for_each_entry(port, &pcie->ports, entry) { int err; @@ -2197,6 +2248,8 @@ int apple_pcie_tunnel_restore(struct device *dev) pcie = pci_host_bridge_priv(bridge); if (!pcie->hw->tunneled) return -EINVAL; + if (pcie->resume_failed) + return -EIO; if (!pcie->bus_stopped) return 0; @@ -2212,7 +2265,7 @@ int apple_pcie_tunnel_restore(struct device *dev) if (ret) return ret; } - ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_resume_commands); + ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_resume_commands, false); if (ret) return ret; @@ -2228,6 +2281,40 @@ int apple_pcie_tunnel_restore(struct device *dev) } EXPORT_SYMBOL_GPL(apple_pcie_tunnel_restore); +/* + * Check recorded host state without accessing a possibly failed data path. + * The caller serializes tunnel transitions and calls after system PM complete. + * The device lock protects driver data against a concurrent unbind. + */ +int apple_pcie_tunnel_check_state(struct device *dev) +{ + struct pci_host_bridge *bridge; + struct apple_pcie_port *port; + struct apple_pcie *pcie; + + guard(device)(dev); + if (!dev->driver) + return -ENODEV; + bridge = dev_get_drvdata(dev); + if (!bridge || !bridge->bus) + return -ENODEV; + pcie = pci_host_bridge_priv(bridge); + if (!pcie->hw->tunneled) + return -EINVAL; + if (pcie->resume_failed) + return -EIO; + if (pcie->bus_stopped || list_empty(&pcie->ports)) + return -ENOLINK; + list_for_each_entry(port, &pcie->ports, entry) { + /* Keep failure latched until a fresh host is probed. */ + if (!port->started || READ_ONCE(port->link_failed)) + return -ENOLINK; + } + + return 0; +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_check_state); + struct apple_pcie_map { void __iomem *base; struct resource res; @@ -2469,19 +2556,77 @@ static int apple_pcie_suspend_noirq(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); struct apple_pcie_port *port; - int i; + bool active = false, can_reset; + int i, ret; if (!pcie->hw->tunneled) return 0; + pcie->reset_on_resume = false; + can_reset = pcie->kernel_init && pcie->power_retained && pcie->reset && + !pcie->bus_stopped && !pcie->resume_failed; + if (can_reset) { + ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_save_tunnel_state, true); + if (ret) { + dev_warn(dev, "PCIe-C cold resume unavailable: DART save failed: %d\n", + ret); + can_reset = false; + } + } list_for_each_entry(port, &pcie->ports, entry) { if (port->started) { + active = true; for_each_set_bit(i, port->sid_map, port->sid_map_sz) port->saved_rid2sid[i] = apple_pcie_port_readl(port, port_rid2sid_offset(port, i)); - apple_pcie_tunnel_stop(port); } + if (port->started || port->needs_stop) { + ret = apple_pcie_tunnel_stop(port); + if (ret) + can_reset = false; + } + } + pcie->reset_on_resume = can_reset && active; + return 0; +} + +static int apple_pcie_reset_for_resume(struct apple_pcie *pcie) +{ + struct apple_pcie_port *port; + int i, ret; + + /* + * Keep the hierarchy bound and suspended. Reset only after its I/O + * drained, then restore the port and IOMMU before releasing tunneled + * reset. Endpoints must never run with missing address translations. + */ + ret = reset_control_reset(pcie->reset); + if (ret) + return ret; + + list_for_each_entry(port, &pcie->ports, entry) { + u32 stat; + + /* A failed cold setup still owns partially enabled hardware. */ + port->needs_stop = true; + apple_pcie_port_writel(port, PORT_TUNCTRL_PERST_ON, PORT_TUNCTRL); + ret = apple_pcie_tunnel_cold_init(port); + if (ret) + return ret; + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_TUNSTAT_PERST_ON, + 1000, 100000, false, port, PORT_TUNSTAT); + if (ret) + return ret; + apple_pcie_tunnel_restore_irq_hw(port); + for_each_set_bit(i, port->sid_map, port->sid_map_sz) + apple_pcie_rid2sid_write(port, i, port->saved_rid2sid[i]); } + ret = apple_pcie_walk_tunnel_darts(pcie, apple_dart_restore_tunnel_state, true); + if (ret) + return ret; + + dev_info(pcie->dev, "PCIe-C reset and DART state restored before link resume\n"); return 0; } @@ -2493,13 +2638,33 @@ static int apple_pcie_resume_noirq(struct device *dev) if (!pcie->hw->tunneled) return 0; + if (pcie->resume_failed) + return -EIO; + if (pcie->reset_on_resume) { + pcie->reset_on_resume = false; + ret = apple_pcie_reset_for_resume(pcie); + if (ret) { + dev_err(dev, "PCIe-C cold resume failed: %d\n", ret); + goto failed; + } + } list_for_each_entry(port, &pcie->ports, entry) { ret = apple_pcie_tunnel_start(port); if (ret) - return ret; + goto failed; } return 0; + +failed: + /* Device PM keeps resuming dependents even after this callback fails. */ + pcie->resume_failed = true; + apple_pcie_walk_tunnel_darts(pcie, apple_dart_quiesce_commands, false); + list_for_each_entry(port, &pcie->ports, entry) { + if (port->started || port->needs_stop) + apple_pcie_tunnel_stop(port); + } + return ret; } static const struct dev_pm_ops apple_pcie_pm_ops = { diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 46832e6a1d16c..eca55995271e3 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -64,6 +64,7 @@ #include #include #include +#include #include #include #include @@ -134,6 +135,7 @@ struct apple_cio { struct device_node *np; struct device_node *pcie_tunnel_np; bool pcie_tunnel_preinitialized; + struct reset_control *pcie_reset; void __iomem *pcie_intr2axi_base; struct apple_rtkit *rtk; @@ -159,6 +161,7 @@ struct apple_cio { struct platform_device *nhi_pdev; struct typec_thunderbolt_switch_dev *tbt_switch; + struct notifier_block pcie_notifier; /* Serializes PCIe-C population with cable teardown. */ struct mutex pcie_tunnel_lock; struct delayed_work pcie_tunnel_work; @@ -166,6 +169,7 @@ struct apple_cio { bool pcie_tunnel_populated; bool pcie_tunnel_stopping; bool pcie_pm_prepared; + bool pcie_resume_expected; bool pcie_quiesce_pending; /* Type-C connector this router is wired to, for DP tunnel routing */ @@ -491,6 +495,44 @@ apple_cio_pcie_tunnel_is_preinitialized(struct device_node *parent) return false; } +static void apple_cio_reset_control_put(void *data) +{ + reset_control_put(data); +} + +/* + * PCIe-C's reset covers the controller and the DART beside it. It is + * optional, since older device trees do not describe it, and only used where + * the kernel initializes the controller: resetting it would discard a boot + * loader's setup. + */ +static int apple_cio_get_pcie_reset(struct apple_cio *acio) +{ + struct reset_control *rst = NULL; + + for_each_available_child_of_node_scoped(acio->pcie_tunnel_np, child) { + if (!of_device_is_compatible(child, "apple,t8103-pciec") && + !of_device_is_compatible(child, "apple,t6000-pciec")) + continue; + if (!of_property_read_bool(child, "apple,pciec-kernel-init")) + break; + + rst = of_reset_control_get_optional_exclusive(child, NULL); + break; + } + if (IS_ERR(rst)) + return dev_err_probe(acio->dev, PTR_ERR(rst), + "Unable to get PCIe-C reset\n"); + if (!rst) + return 0; + + /* The bound PCIe host owns the line between ACIO startup attempts. */ + reset_control_release(rst); + acio->pcie_reset = rst; + return devm_add_action_or_reset(acio->dev, apple_cio_reset_control_put, + rst); +} + static void apple_cio_iounmap(void *data) { iounmap((__force void __iomem *)data); @@ -1598,6 +1640,7 @@ static int apple_nhi_probe(struct platform_device *pdev) scoped_guard(mutex, &acio->pcie_tunnel_lock) { acio->pcie_tunnel_stopping = false; acio->pcie_pm_prepared = false; + acio->pcie_resume_expected = false; } ret = tb_domain_add(anhi->tb, false); if (ret) { @@ -1690,6 +1733,8 @@ static int apple_nhi_prepare(struct device *dev) struct apple_cio *acio = anhi->acio; guard(mutex)(&acio->pcie_tunnel_lock); + /* Keep an unresolved expectation across another sleep before revalidation. */ + acio->pcie_resume_expected |= READ_ONCE(acio->pcie_tunnel_requested); acio->pcie_pm_prepared = true; return 0; } @@ -1861,6 +1906,27 @@ static int apple_cio_start(struct apple_cio *acio) lockdep_assert_held(&acio->lock); + /* + * PCIe-C's power domain stays on across cable teardown, so apart from + * this reset nothing returns the controller to a known state between + * tunnels. After system sleep with a Thunderbolt 3 dock, state left in + * it has kept every later link from training until reboot. Reset it + * while its host and DART are unbound and before ACIO's domains come + * up, in the same order as after boot. + */ + if (acio->pcie_reset) { + ret = reset_control_acquire(acio->pcie_reset); + if (ret) + return dev_err_probe(acio->dev, ret, + "PCIe-C reset is still in use\n"); + ret = reset_control_reset(acio->pcie_reset); + reset_control_release(acio->pcie_reset); + if (ret) + return dev_err_probe(acio->dev, ret, + "failed to reset PCIe-C\n"); + dev_info(acio->dev, "PCIe-C reset before ACIO start\n"); + } + /* Create device links to the power domains in order to power them on */ for (i = 0; i < acio->pd_list->num_pds; i++) { link = device_link_add(acio->dev, acio->pd_list->pd_devs[i], @@ -1980,6 +2046,34 @@ static int apple_cio_start(struct apple_cio *acio) return ret; } +static int apple_cio_check_connection(struct apple_cio *acio) +{ + struct platform_device *pdev; + int ret; + + lockdep_assert_held(&acio->lock); + guard(mutex)(&acio->pcie_tunnel_lock); + if (!READ_ONCE(acio->nhi_pdev) || acio->pcie_tunnel_stopping || + apple_rtkit_is_crashed(acio->rtk)) + return -ENODEV; + if (acio->pcie_pm_prepared || acio->pcie_quiesce_pending) + return -EAGAIN; + + /* A display-only or unauthorized session need not have a PCIe tunnel. */ + if (!READ_ONCE(acio->pcie_tunnel_requested)) + return acio->pcie_resume_expected ? -ENOLINK : 0; + if (!acio->pcie_tunnel_populated) + return -EAGAIN; + pdev = apple_cio_find_pcie_tunnel(acio); + if (!pdev) + return -ENODEV; + ret = apple_pcie_tunnel_check_state(&pdev->dev); + put_device(&pdev->dev); + if (!ret) + acio->pcie_resume_expected = false; + return ret; +} + static int apple_cio_tbt_switch_set(struct typec_thunderbolt_switch_dev *sw, const struct typec_thunderbolt_switch_data *data) { @@ -2028,7 +2122,7 @@ static int apple_cio_tbt_switch_set(struct typec_thunderbolt_switch_dev *sw, } if (acio->target_cable_info == acio->current_cable_info) - return 0; + return acio->current_cable_info ? apple_cio_check_connection(acio) : 0; /* * Changing live cable parameters requires an ACIO shutdown. Report the @@ -2054,6 +2148,18 @@ static int apple_cio_tbt_switch_set(struct typec_thunderbolt_switch_dev *sw, return 0; } +static int apple_cio_pcie_notify(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct apple_cio *acio = container_of(nb, struct apple_cio, pcie_notifier); + + if (event != APPLE_PCIE_TUNNEL_LINK_DOWN || data != acio->dev) + return NOTIFY_DONE; + + typec_thunderbolt_switch_notify(acio->tbt_switch); + return NOTIFY_OK; +} + static int apple_cio_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -2095,6 +2201,9 @@ static int apple_cio_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "failed to map PCIe-C Intr2AXI registers\n"); + ret = apple_cio_get_pcie_reset(acio); + if (ret) + return ret; } } @@ -2196,11 +2305,20 @@ static int apple_cio_probe(struct platform_device *pdev) .set = apple_cio_tbt_switch_set, .drvdata = acio, }; + /* A consumer cannot start a host before its failure listener is ready. */ + guard(mutex)(&acio->lock); acio->tbt_switch = typec_thunderbolt_switch_register(dev, &desc); if (IS_ERR(acio->tbt_switch)) return dev_err_probe(dev, PTR_ERR(acio->tbt_switch), "Unable to register thunderbolt switch\n"); + acio->pcie_notifier.notifier_call = apple_cio_pcie_notify; + ret = apple_pcie_tunnel_register_notifier(&acio->pcie_notifier); + if (ret) { + typec_thunderbolt_switch_unregister(acio->tbt_switch); + return ret; + } + return 0; } @@ -2208,6 +2326,7 @@ static void apple_cio_remove(struct platform_device *pdev) { struct apple_cio *acio = platform_get_drvdata(pdev); + apple_pcie_tunnel_unregister_notifier(&acio->pcie_notifier); typec_thunderbolt_switch_unregister(acio->tbt_switch); cancel_delayed_work_sync(&acio->pcie_tunnel_work); diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index 8339846ba355e..7e327118cbd71 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -582,6 +582,55 @@ int typec_thunderbolt_switch_set(struct typec_thunderbolt_switch *sw, } EXPORT_SYMBOL_GPL(typec_thunderbolt_switch_set); +/** + * typec_thunderbolt_switch_register_notifier - Watch provider health changes + * @sw: USB4/Thunderbolt switch held by the consumer + * @nb: Atomic notifier, which must not sleep or call the switch's set method + * + * Notifications request fresh state validation in consumer work context. + * Unregister the notifier before releasing @sw or its callback data. + */ +int typec_thunderbolt_switch_register_notifier(struct typec_thunderbolt_switch *sw, + struct notifier_block *nb) +{ + if (IS_ERR_OR_NULL(sw)) + return 0; + + return atomic_notifier_chain_register(&sw->sw_dev->notifiers, nb); +} +EXPORT_SYMBOL_GPL(typec_thunderbolt_switch_register_notifier); + +/** + * typec_thunderbolt_switch_unregister_notifier - Stop provider notifications + * @sw: USB4/Thunderbolt switch held by the consumer + * @nb: Previously registered notifier + * + * Drains callbacks. Call in process context, outside callback/worker locks, + * before cancelling work queued by the callback. + */ +void typec_thunderbolt_switch_unregister_notifier(struct typec_thunderbolt_switch *sw, + struct notifier_block *nb) +{ + if (!IS_ERR_OR_NULL(sw)) + atomic_notifier_chain_unregister(&sw->sw_dev->notifiers, nb); +} +EXPORT_SYMBOL_GPL(typec_thunderbolt_switch_unregister_notifier); + +/** + * typec_thunderbolt_switch_notify - Request consumer revalidation + * @sw: Registered provider switch + * + * May be called from interrupt context. No connection state is passed: a + * delayed notification must validate the current connection, not reset it + * using details from an earlier session. Stop notifications before unregister. + */ +void typec_thunderbolt_switch_notify(struct typec_thunderbolt_switch_dev *sw) +{ + atomic_notifier_call_chain(&sw->notifiers, + TYPEC_THUNDERBOLT_SWITCH_REVALIDATE, NULL); +} +EXPORT_SYMBOL_GPL(typec_thunderbolt_switch_notify); + static void typec_thunderbolt_switch_release(struct device *dev) { kfree(to_typec_thunderbolt_switch_dev(dev)); @@ -622,6 +671,7 @@ typec_thunderbolt_switch_register(struct device *parent, sw_dev->dev.fwnode = desc->fwnode; sw_dev->dev.class = &typec_mux_class; sw_dev->dev.type = &typec_thunderbolt_switch_dev_type; + ATOMIC_INIT_NOTIFIER_HEAD(&sw_dev->notifiers); sw_dev->dev.driver_data = desc->drvdata; ret = dev_set_name(&sw_dev->dev, "%s-thunderbolt-switch", desc->name ? desc->name : dev_name(parent)); diff --git a/drivers/usb/typec/mux.h b/drivers/usb/typec/mux.h index a2c92cab12114..12e1ac47a11f1 100644 --- a/drivers/usb/typec/mux.h +++ b/drivers/usb/typec/mux.h @@ -3,6 +3,7 @@ #ifndef __USB_TYPEC_MUX__ #define __USB_TYPEC_MUX__ +#include #include struct typec_switch_dev { @@ -29,6 +30,7 @@ extern const struct device_type typec_mux_dev_type; struct typec_thunderbolt_switch_dev { struct device dev; typec_thunderbolt_switch_set_fn_t set; + struct atomic_notifier_head notifiers; }; #define to_typec_thunderbolt_switch_dev(_dev_) \ diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index 8177f860f89de..2f73bb17ed352 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -26,3 +26,13 @@ config TYPEC_SN201202X If you choose to build this driver as a dynamically linked module, the module will be called sn201202x.ko. + +config TYPEC_TIPD_KUNIT_TEST + tristate "KUnit tests for CD321x resume state" if !KUNIT_ALL_TESTS + depends on KUNIT && TYPEC_TPS6598X_CORE + default KUNIT_ALL_TESTS + help + Test suspend gating, fresh cable-state revalidation, bounded retries + and removal during resume. The tests exercise the state transitions + shared with the driver, including aborted and repeated sleep cycles. + These tests run without accessing a physical Type-C controller. diff --git a/drivers/usb/typec/tipd/Makefile b/drivers/usb/typec/tipd/Makefile index 6f5ebc1e1a5fe..c0e850a56f289 100644 --- a/drivers/usb/typec/tipd/Makefile +++ b/drivers/usb/typec/tipd/Makefile @@ -10,3 +10,5 @@ tps6598x-y := i2c.o obj-$(CONFIG_TYPEC_SN201202X) += sn201202x.o sn201202x-y := spmi.o + +obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c new file mode 100644 index 0000000000000..79b6797c73b9e --- /dev/null +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "cd321x-pm.h" + +static void cd321x_resume_requires_snapshot_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + /* A queued cable update may not apply the pre-sleep snapshot. */ + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_TRUE(test, cd321x_pm_can_update(&pm)); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_RUNNING); +} + +static void cd321x_revalidation_errors_bounded_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm), + i + 1 < CD321X_RESUME_ATTEMPTS); + } + /* Give up without permitting an update from the old cable snapshot. */ + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); +} + +static void cd321x_activation_failure_rereads_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + /* A PHY or ACIO error must not be treated as successful restoration. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_TRUE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_EQ(test, pm.attempts_left, 0U); +} + +static void cd321x_abort_and_duplicate_completion_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + /* Prepare followed by completion, with no device suspend, is an abort. */ + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_resume(&pm)); + KUNIT_EXPECT_EQ(test, pm.attempts_left, CD321X_RESUME_ATTEMPTS - 1U); + cd321x_pm_snapshot_ready(&pm); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_TRUE(test, cd321x_pm_can_update(&pm)); +} + +static void cd321x_repeat_suspend_cancels_retry_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + /* The next prepare wins over a delayed retry from the previous wake. */ + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_EXPECT_EQ(test, pm.attempts_left, (unsigned int)CD321X_RESUME_ATTEMPTS); +} + +static void cd321x_suspend_before_apply_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + /* The debounce timer has not applied the freshly read state yet. */ + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_retry(&pm)); +} + +static void cd321x_remove_drains_resume_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + cd321x_pm_remove(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_resume(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_REMOVED); +} + +static void cd321x_read_and_apply_share_budget_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + if (i & 1) + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm), + i + 1 < CD321X_RESUME_ATTEMPTS); + } + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); +} + +static void cd321x_status_event_after_exhaustion_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_retry(&pm); + } + /* A power/plug-only interrupt has not read the data or cable registers. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_event(&pm)); + KUNIT_EXPECT_EQ(test, pm.attempts_left, (unsigned int)CD321X_RESUME_ATTEMPTS); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_TRUE(test, cd321x_pm_can_update(&pm)); + cd321x_pm_remove(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_event(&pm)); +} + +static void cd321x_runtime_failure_revalidates_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + /* A failed ordinary cable update has already consumed its IRQ changes. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm), + i + 1 < CD321X_RESUME_ATTEMPTS); + } + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + /* A later physical event can retry, but cannot directly apply stale data. */ + KUNIT_EXPECT_TRUE(test, cd321x_pm_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); +} + +static void cd321x_healthy_resume_keeps_connection_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + /* A successful resume and an aborted suspend both revalidate in place. */ + for (i = 0; i < 2; i++) { + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); + } +} + +static void cd321x_failed_resume_forces_reconnect_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + /* Fresh cable data cannot override a provider's recorded resume failure. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + /* A new suspend must preserve recovery that has not been applied yet. */ + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); +} + +static void cd321x_stale_recovery_keeps_reconnect_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_retry(&pm); + } + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + KUNIT_ASSERT_TRUE(test, cd321x_pm_event(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); +} + +static void cd321x_late_link_failure_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + cd321x_pm_complete(&pm); + /* The link drops after the final successful resume health check. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + /* Only the provider's failed check requests a fresh forced reconnect. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_EQ(test, pm.link_events_left, CD321X_LINK_EVENTS - 1U); +} + +static void cd321x_link_event_during_final_apply_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + cd321x_pm_prepare(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + if (i + 1 < CD321X_RESUME_ATTEMPTS) + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm)); + } + cd321x_pm_snapshot_ready(&pm); + KUNIT_ASSERT_EQ(test, pm.attempts_left, 0U); + /* A queued APPLY must not hide an event with its exhausted read budget. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); +} + +static void cd321x_link_reprobe_storm_bounded_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int event, attempt; + + cd321x_pm_new_connection(&pm); + for (event = 0; event < CD321X_LINK_EVENTS; event++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_link_event(&pm)); + for (attempt = 0; attempt < CD321X_RESUME_ATTEMPTS; attempt++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm), + attempt + 1 < CD321X_RESUME_ATTEMPTS); + } + } + KUNIT_EXPECT_FALSE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); +} + +static void cd321x_success_does_not_refill_link_budget_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + cd321x_pm_new_connection(&pm); + for (i = 0; i < CD321X_LINK_EVENTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_link_event(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + /* An old host event can reach a now healthy session: keep it. */ + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); + cd321x_pm_complete(&pm); + } + KUNIT_EXPECT_FALSE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_RUNNING); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); +} + +static void cd321x_link_event_init_and_remove_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_init(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_ready(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_ready(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_remove(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_ready(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); +} + +static void cd321x_link_event_suspend_and_rearm_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i; + + cd321x_pm_new_connection(&pm); + for (i = 0; i < CD321X_LINK_EVENTS; i++) + KUNIT_ASSERT_TRUE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_link_event(&pm)); + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_link_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_EXPECT_EQ(test, pm.link_events_left, (unsigned int)CD321X_LINK_EVENTS); + KUNIT_ASSERT_TRUE(test, cd321x_pm_link_event(&pm)); + /* An actual attachment change also permits a bounded new sequence. */ + cd321x_pm_new_connection(&pm); + KUNIT_EXPECT_EQ(test, pm.link_events_left, (unsigned int)CD321X_LINK_EVENTS); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); +} + +static struct kunit_case cd321x_pm_cases[] = { + KUNIT_CASE(cd321x_late_link_failure_test), + KUNIT_CASE(cd321x_link_event_during_final_apply_test), + KUNIT_CASE(cd321x_link_reprobe_storm_bounded_test), + KUNIT_CASE(cd321x_success_does_not_refill_link_budget_test), + KUNIT_CASE(cd321x_link_event_init_and_remove_test), + KUNIT_CASE(cd321x_link_event_suspend_and_rearm_test), + KUNIT_CASE(cd321x_healthy_resume_keeps_connection_test), + KUNIT_CASE(cd321x_failed_resume_forces_reconnect_test), + KUNIT_CASE(cd321x_stale_recovery_keeps_reconnect_test), + KUNIT_CASE(cd321x_resume_requires_snapshot_test), + KUNIT_CASE(cd321x_revalidation_errors_bounded_test), + KUNIT_CASE(cd321x_activation_failure_rereads_test), + KUNIT_CASE(cd321x_abort_and_duplicate_completion_test), + KUNIT_CASE(cd321x_repeat_suspend_cancels_retry_test), + KUNIT_CASE(cd321x_suspend_before_apply_test), + KUNIT_CASE(cd321x_remove_drains_resume_test), + KUNIT_CASE(cd321x_read_and_apply_share_budget_test), + KUNIT_CASE(cd321x_status_event_after_exhaustion_test), + KUNIT_CASE(cd321x_runtime_failure_revalidates_test), + {} +}; + +static struct kunit_suite cd321x_pm_suite = { + .name = "cd321x-pm", + .test_cases = cd321x_pm_cases, +}; + +kunit_test_suite(cd321x_pm_suite); +MODULE_LICENSE("GPL"); diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h new file mode 100644 index 0000000000000..da764777ba36e --- /dev/null +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __CD321X_PM_H__ +#define __CD321X_PM_H__ + +#include + +#define CD321X_RESUME_ATTEMPTS 3 +#define CD321X_LINK_EVENTS 3 + +enum cd321x_pm_phase { + CD321X_PM_RUNNING, + CD321X_PM_PREPARED, + CD321X_PM_REVALIDATE, + CD321X_PM_APPLY, + CD321X_PM_STALE, + CD321X_PM_REMOVED, + CD321X_PM_INITIALIZING, +}; + +/* All transitions and the associated work scheduling hold tps->lock. */ +struct cd321x_pm_state { + enum cd321x_pm_phase phase; + unsigned int attempts_left; + unsigned int link_events_left; + bool force_reconnect; +}; + +static inline void cd321x_pm_new_connection(struct cd321x_pm_state *pm) +{ + pm->link_events_left = CD321X_LINK_EVENTS; +} + +static inline void cd321x_pm_init(struct cd321x_pm_state *pm) +{ + pm->phase = CD321X_PM_INITIALIZING; + cd321x_pm_new_connection(pm); +} + +static inline bool cd321x_pm_ready(struct cd321x_pm_state *pm) +{ + if (pm->phase != CD321X_PM_INITIALIZING) + return false; + pm->phase = CD321X_PM_REVALIDATE; + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + return true; +} + +static inline bool cd321x_pm_can_update(const struct cd321x_pm_state *pm) +{ + return pm->phase == CD321X_PM_RUNNING || pm->phase == CD321X_PM_APPLY; +} + +static inline void cd321x_pm_prepare(struct cd321x_pm_state *pm) +{ + if (pm->phase != CD321X_PM_REMOVED) + pm->phase = CD321X_PM_PREPARED; + pm->attempts_left = 0; +} + +static inline bool cd321x_pm_resume(struct cd321x_pm_state *pm) +{ + if (pm->phase != CD321X_PM_PREPARED) + return false; + pm->phase = CD321X_PM_REVALIDATE; + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + cd321x_pm_new_connection(pm); + return true; +} + +static inline bool cd321x_pm_link_event(struct cd321x_pm_state *pm) +{ + if (pm->phase == CD321X_PM_INITIALIZING || + pm->phase == CD321X_PM_PREPARED || pm->phase == CD321X_PM_REMOVED || + !pm->link_events_left) + return false; + + /* Host reprobes must not create an unlimited series of recovery runs. */ + pm->link_events_left--; + pm->phase = CD321X_PM_REVALIDATE; + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + /* A stale notification does not itself prove the current session failed. */ + return true; +} + +static inline bool cd321x_pm_begin_read(struct cd321x_pm_state *pm) +{ + if (pm->phase != CD321X_PM_REVALIDATE || !pm->attempts_left) + return false; + pm->attempts_left--; + return true; +} + +static inline void cd321x_pm_snapshot_ready(struct cd321x_pm_state *pm) +{ + pm->phase = CD321X_PM_APPLY; +} + +static inline bool cd321x_pm_retry(struct cd321x_pm_state *pm) +{ + /* Runtime cable updates need the same bounded fresh-read recovery. */ + if (pm->phase == CD321X_PM_RUNNING) { + pm->force_reconnect = true; + pm->phase = CD321X_PM_REVALIDATE; + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + return true; + } + if (pm->phase != CD321X_PM_REVALIDATE && pm->phase != CD321X_PM_APPLY) + return false; + pm->force_reconnect = true; + if (pm->attempts_left) { + pm->phase = CD321X_PM_REVALIDATE; + return true; + } + /* Only a fresh read, triggered by a later event, can reopen updates. */ + pm->phase = CD321X_PM_STALE; + return false; +} + +static inline bool cd321x_pm_event(struct cd321x_pm_state *pm) +{ + if (pm->phase != CD321X_PM_STALE) + return false; + pm->phase = CD321X_PM_REVALIDATE; + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + return true; +} + +static inline void cd321x_pm_complete(struct cd321x_pm_state *pm) +{ + if (pm->phase == CD321X_PM_APPLY) { + pm->phase = CD321X_PM_RUNNING; + pm->attempts_left = 0; + pm->force_reconnect = false; + } +} + +static inline void cd321x_pm_remove(struct cd321x_pm_state *pm) +{ + pm->phase = CD321X_PM_REMOVED; + pm->attempts_left = 0; +} + +#endif /* __CD321X_PM_H__ */ diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 33b35da777c5e..c51af26ab623e 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -22,7 +22,6 @@ #include #include #include -#include #include "tps6598x.h" #include "trace.h" @@ -545,7 +544,7 @@ static void tps6598x_handle_plug_event(struct tps6598x *tps, u32 status) } static int cd321x_typec_update_mode(struct tps6598x *tps, - struct cd321x_status *st) + struct cd321x_status *st, bool *display_busy) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); struct typec_mux_state old_state = cd321x->state; @@ -554,13 +553,16 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, }; int ret = 0; + *display_busy = false; if (!(st->data_status & TPS_DATA_STATUS_DATA_CONNECTION)) { if (cd321x->state_valid && cd321x->state.mode == TYPEC_STATE_SAFE) return 0; cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; cd321x->state.data = NULL; - typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + if (ret) + goto out; ret = typec_mux_set(cd321x->mux, &cd321x->state); } else if (st->data_status & CD321X_DATA_STATUS_USB4_CONNECTION) { struct enter_usb_data eusb_data; @@ -663,8 +665,11 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, cd321x->state.alt = cd321x->port_altmode_dp; cd321x->state.data = &dp_data; cd321x->state.mode = mode; - typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + if (ret) + goto out; ret = typec_mux_set(cd321x->mux, &cd321x->state); + *display_busy = ret == -EBUSY; if (!ret) { cd321x->dp_status = dp_data.status; cd321x->dp_conf = dp_data.conf; @@ -676,7 +681,9 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_USB; cd321x->state.data = NULL; - typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); + if (ret) + goto out; ret = typec_mux_set(cd321x->mux, &cd321x->state); } @@ -696,6 +703,24 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, return ret; } +static void cd321x_retry_revalidation(struct cd321x *cd321x) +{ + lockdep_assert_held(&cd321x->tps.lock); + if (cd321x->pm.phase != CD321X_PM_RUNNING && + cd321x->pm.phase != CD321X_PM_REVALIDATE && + cd321x->pm.phase != CD321X_PM_APPLY) + return; + + /* Keep a later physical event retryable if the automatic budget expires. */ + cd321x->state_valid = false; + if (cd321x_pm_retry(&cd321x->pm)) + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, + msecs_to_jiffies(CD321X_DEBOUNCE_DELAY_MS)); + else + dev_err(cd321x->tps.dev, "cable revalidation failed after %u attempts\n", + CD321X_RESUME_ATTEMPTS); +} + static void cd321x_update_work(struct work_struct *work) { struct cd321x *cd321x = container_of(to_delayed_work(work), @@ -705,8 +730,12 @@ static void cd321x_update_work(struct work_struct *work) struct typec_thunderbolt_switch_data tbt_switch_data = { .state = TYPEC_THUNDERBOLT_SWITCH_OFF, }; + bool display_busy; + int ret; guard(mutex)(&tps->lock); + if (!cd321x_pm_can_update(&cd321x->pm)) + return; st = cd321x->update_status; cd321x->update_status.status_changed = 0; @@ -714,7 +743,10 @@ static void cd321x_update_work(struct work_struct *work) bool old_connected = !!tps->partner; bool new_connected = st.status & TPS_STATUS_PLUG_PRESENT; - bool was_disconnected = st.status_changed & TPS_STATUS_PLUG_PRESENT; + bool was_disconnected = cd321x->pm.force_reconnect || + (st.status_changed & (TPS_STATUS_PLUG_PRESENT | TPS_STATUS_PLUG_UPSIDE_DOWN)) || + (st.data_status_changed & (TPS_DATA_STATUS_TBT_CONNECTION | + CD321X_DATA_STATUS_USB4_CONNECTION)); bool usb_connection = st.data_status & (TPS_DATA_STATUS_USB2_CONNECTION | TPS_DATA_STATUS_USB3_CONNECTION); @@ -750,10 +782,13 @@ static void cd321x_update_work(struct work_struct *work) bool is_pd = pwr_opmode == TYPEC_PWR_MODE_PD; bool partner_changed = old_connected && new_connected && - (was_disconnected || + (was_disconnected || is_pd != cd321x->cur_partner_is_pd || (is_pd && memcmp(&st.partner_identity, &cd321x->cur_partner_identity, sizeof(struct usb_pd_identity)))); + /* A different dock must release the old routing before reconfiguration. */ + was_disconnected |= partner_changed; + /* * ACIO carries the tunneled PCIe reset handshake over the still-live * Type-C data path. Close a disappearing USB4/TBT session before dropping @@ -761,14 +796,20 @@ static void cd321x_update_work(struct work_struct *work) * PCIe-C cannot quiesce its hierarchy before NHI removal. */ if ((!new_connected || was_disconnected) && - (!cd321x->state_valid || cd321x->state.mode != TYPEC_STATE_SAFE)) - typec_thunderbolt_switch_set(cd321x->tbt_switch, - &tbt_switch_data); + (!cd321x->state_valid || cd321x->state.mode != TYPEC_STATE_SAFE)) { + ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, + &tbt_switch_data); + if (ret) + goto retry; + } /* If we are switching from an active role, transition to USB_ROLE_NONE. */ if (old_role != USB_ROLE_NONE && - (new_role != old_role || was_disconnected || dp_mode_changed)) - usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); + (new_role != old_role || was_disconnected || dp_mode_changed)) { + ret = usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); + if (ret) + goto retry; + } /* * Every Type-C port references the same external DCP connector. Only the @@ -789,12 +830,12 @@ static void cd321x_update_work(struct work_struct *work) if (!IS_ERR(tps->partner)) typec_unregister_partner(tps->partner); tps->partner = NULL; + cd321x->cur_partner_is_pd = false; } /* If there was a disconnection, set PHY to off */ if (!new_connected || was_disconnected) { struct typec_mux_state old_state = cd321x->state; - int ret; cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; @@ -806,6 +847,7 @@ static void cd321x_update_work(struct work_struct *work) "failed to place Type-C mux in safe mode: %d\n", ret); cd321x->state = old_state; + goto retry; } else { cd321x->display_route_active = false; } @@ -820,8 +862,10 @@ static void cd321x_update_work(struct work_struct *work) power_supply_changed(tps->psy); /* If the plug is disconnected, we are done */ - if (!new_connected) + if (!new_connected) { + cd321x_pm_complete(&cd321x->pm); return; + } /* Set up partner if we were previously disconnected (or changed). */ if (!tps->partner) { @@ -844,19 +888,26 @@ static void cd321x_update_work(struct work_struct *work) tps->partner = typec_register_partner(tps->port, &desc); if (IS_ERR(tps->partner)) { dev_warn(tps->dev, "%s: failed to register partner\n", __func__); - return; + tps->partner = NULL; + goto retry; } + cd321x->cur_partner_is_pd = is_pd; if (desc.identity) typec_partner_set_identity(tps->partner); } /* Update the TypeC MUX/PHY state */ - if (!cd321x_typec_update_mode(tps, &st)) - cd321x->display_route_active = dp_connected; + ret = cd321x_typec_update_mode(tps, &st, &display_busy); + if (ret && !display_busy) + goto retry; + /* A shared display route can be busy while the dock's USB path is usable. */ + cd321x->display_route_active = dp_connected && !ret; /* Launch the USB role switch */ - usb_role_switch_set_role(tps->role_sw, new_role); + ret = usb_role_switch_set_role(tps->role_sw, new_role); + if (ret) + goto retry; if (cd321x->connector_fwnode && cd321x->display_route_active && dp_hpd) drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_connected); @@ -872,6 +923,11 @@ static void cd321x_update_work(struct work_struct *work) connector_status_unknown); power_supply_changed(tps->psy); + cd321x_pm_complete(&cd321x->pm); + return; + +retry: + cd321x_retry_revalidation(cd321x); } static void cd321x_queue_status(struct cd321x *cd321x) @@ -893,8 +949,15 @@ static int cd321x_connect(struct tps6598x *tps, u32 status) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); + if ((tps->status ^ status) & TPS_STATUS_PLUG_PRESENT) + cd321x_pm_new_connection(&cd321x->pm); tps->status = status; cd321x_queue_status(cd321x); + /* A plug or power IRQ alone does not refresh the cached cable metadata. */ + if (cd321x_pm_event(&cd321x->pm)) + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); + if (!cd321x_pm_can_update(&cd321x->pm)) + return 0; /* * Requeue even when the previous update is already running. A separate @@ -902,48 +965,85 @@ static int cd321x_connect(struct tps6598x *tps, u32 status) * event in that window because the latter sees the running work as busy. * The worker consumes every accumulated status change on its next pass. */ - mod_delayed_work(system_wq, &cd321x->update_work, + mod_delayed_work(system_freezable_wq, &cd321x->update_work, msecs_to_jiffies(CD321X_DEBOUNCE_DELAY_MS)); return 0; } /* - * A cable that was already attached before a suspend/resume cycle generates - * no fresh attach event on resume: the PD controller's own status hasn't - * changed, so cd321x_connect()'s diff against its last cached status sees - * nothing different and cd321x_update_work() takes its "nothing to do" - * path. Meanwhile thunderbolt-core's own noirq-phase resume handling - * (tb_free_invalid_tunnels()) has already torn down any USB4/TBT tunnel - * that didn't survive the sleep at the link level -- and nothing else - * re-establishes it. - * - * Reproduce a genuine unplug-then-replug in software instead of inventing - * a new code path: call the existing connect() callback once with the - * current status but PLUG_PRESENT cleared (a synthetic disconnect), then - * again with the real, current status. Both updates land on the same - * debounced work item, which coalesces them into a single run with - * was_disconnected=true and the correct final state -- exactly the same - * teardown-then-bring-up sequence, through the same already-validated - * apple_cio_tbt_switch_set() path, that a real physical replug drives. + * An attached cable need not generate an IRQ after sleep. Read a fresh + * snapshot and reapply it with the mode cache invalidated, allowing the + * provider to validate the resumed session. Preserve a healthy connection; + * only a real cable change or a failed read/apply requests forced recovery. */ -static void cd321x_resume_reverify(struct tps6598x *tps) +static void cd321x_resume_work(struct work_struct *work) { + struct cd321x *cd321x = container_of(to_delayed_work(work), + struct cd321x, resume_work); + struct tps6598x *tps = &cd321x->tps; u32 status; - if (!tps6598x_read_status(tps, &status)) + guard(mutex)(&tps->lock); + if (cd321x->pm.phase == CD321X_PM_INITIALIZING || + cd321x->pm.phase == CD321X_PM_PREPARED || + cd321x->pm.phase == CD321X_PM_REMOVED) return; - if (!(status & TPS_STATUS_PLUG_PRESENT)) + if (atomic_xchg(&cd321x->link_event, 0)) { + if (cd321x_pm_link_event(&cd321x->pm)) { + cd321x->state_valid = false; + dev_info(tps->dev, "revalidating cable after tunnel link failure\n"); + } else { + dev_warn_ratelimited(tps->dev, + "tunnel link recovery budget exhausted\n"); + } + } + if (!cd321x_pm_begin_read(&cd321x->pm)) return; - if (!tps6598x_read_power_status(tps)) - return; - if (!tps->data->read_data_status(tps)) - return; + if (!tps6598x_read_status(tps, &status)) + goto retry; + + if (status & TPS_STATUS_PLUG_PRESENT) { + if (!tps6598x_read_power_status(tps)) + goto retry; + if (TPS_POWER_STATUS_PWROPMODE(tps->pwr_status) == TYPEC_PWR_MODE_PD && + tps6598x_read_partner_identity(tps)) + goto retry; + if (!tps->data->read_data_status(tps)) + goto retry; + } - tps->data->connect(tps, status & ~TPS_STATUS_PLUG_PRESENT); + /* Also report a cable removed during sleep, even if its IRQ was lost. */ + cd321x_pm_snapshot_ready(&cd321x->pm); tps->data->connect(tps, status); + return; + +retry: + cd321x_retry_revalidation(cd321x); +} + +static void cd321x_suspend_prepare(struct tps6598x *tps) +{ + struct cd321x *cd321x = container_of(tps, struct cd321x, tps); + + scoped_guard(mutex, &tps->lock) { + cd321x_pm_prepare(&cd321x->pm); + cd321x->state_valid = false; + } + /* Gate scheduling first; neither worker may be drained under its mutex. */ + cancel_delayed_work_sync(&cd321x->resume_work); + cancel_delayed_work_sync(&cd321x->update_work); +} + +static void cd321x_resume_reverify(struct tps6598x *tps) +{ + struct cd321x *cd321x = container_of(tps, struct cd321x, tps); + + guard(mutex)(&tps->lock); + if (cd321x_pm_resume(&cd321x->pm)) + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); } static irqreturn_t cd321x_interrupt(int irq, void *data) @@ -1341,6 +1441,47 @@ static int cd321x_register_port_altmodes(struct cd321x *cd321x) return 0; } +static int cd321x_tbt_notify(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct cd321x *cd321x = container_of(nb, struct cd321x, tbt_notifier); + + if (event != TYPEC_THUNDERBOLT_SWITCH_REVALIDATE) + return NOTIFY_DONE; + + /* Publish the event even if the worker is already reading or applying. */ + atomic_xchg(&cd321x->link_event, 1); + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); + return NOTIFY_OK; +} + +static void cd321x_ready(struct tps6598x *tps) +{ + struct cd321x *cd321x = container_of(tps, struct cd321x, tps); + + guard(mutex)(&tps->lock); + /* Include failures reported while the port was still being initialized. */ + if (cd321x_pm_ready(&cd321x->pm)) { + cd321x->state_valid = false; + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); + } +} + +static void cd321x_stop_work(struct cd321x *cd321x) +{ + scoped_guard(mutex, &cd321x->tps.lock) + cd321x_pm_remove(&cd321x->pm); + + /* Drain producers before cancelling work, including on probe rollback. */ + if (cd321x->tbt_notifier_registered) { + typec_thunderbolt_switch_unregister_notifier(cd321x->tbt_switch, + &cd321x->tbt_notifier); + cd321x->tbt_notifier_registered = false; + } + cancel_delayed_work_sync(&cd321x->resume_work); + cancel_delayed_work_sync(&cd321x->update_work); +} + static int cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) { @@ -1349,6 +1490,9 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) int ret; INIT_DELAYED_WORK(&cd321x->update_work, cd321x_update_work); + INIT_DELAYED_WORK(&cd321x->resume_work, cd321x_resume_work); + atomic_set(&cd321x->link_event, 0); + cd321x_pm_init(&cd321x->pm); ret = tps6598x_register_port(tps, fwnode); if (ret) @@ -1380,8 +1524,18 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) cd321x->state.data = NULL; cd321x->state_valid = !typec_set_mode(tps->port, TYPEC_STATE_SAFE); + cd321x->tbt_notifier.notifier_call = cd321x_tbt_notify; + ret = typec_thunderbolt_switch_register_notifier(cd321x->tbt_switch, + &cd321x->tbt_notifier); + if (ret) + goto err_put_switch; + cd321x->tbt_notifier_registered = true; + return 0; +err_put_switch: + typec_thunderbolt_switch_put(cd321x->tbt_switch); + cd321x->tbt_switch = NULL; err_unregister_mux: typec_mux_put(cd321x->mux); cd321x->mux = NULL; @@ -1406,6 +1560,7 @@ cd321x_unregister_port(struct tps6598x *tps) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); + cd321x_stop_work(cd321x); typec_thunderbolt_switch_put(cd321x->tbt_switch); cd321x->tbt_switch = NULL; typec_mux_put(cd321x->mux); @@ -1831,7 +1986,7 @@ static void cd321x_remove(struct tps6598x *tps) }; int ret; - cancel_delayed_work_sync(&cd321x->update_work); + cd321x_stop_work(cd321x); /* * Driver teardown must close the same hardware sessions as a physical @@ -1851,7 +2006,6 @@ static void cd321x_remove(struct tps6598x *tps) usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); } -static int tipd_pm_notify(struct notifier_block *nb, unsigned long action, void *data); int tipd_init(struct tps6598x *tps) { @@ -1958,10 +2112,8 @@ int tipd_init(struct tps6598x *tps) enable_irq_wake(tps->irq); } - if (tps->data->resume_reverify) { - tps->pm_nb.notifier_call = tipd_pm_notify; - register_pm_notifier(&tps->pm_nb); - } + if (tps->data->ready) + tps->data->ready(tps); return 0; @@ -1985,9 +2137,6 @@ EXPORT_SYMBOL_GPL(tipd_init); void tipd_remove(struct tps6598x *tps) { - if (tps->data->resume_reverify) - unregister_pm_notifier(&tps->pm_nb); - if (!tps->irq) cancel_delayed_work_sync(&tps->wq_poll); else @@ -2054,35 +2203,27 @@ int tipd_resume(struct tps6598x *tps) } EXPORT_SYMBOL_GPL(tipd_resume); -/* - * tipd_resume() runs as an ordinary device .resume() callback, which can - * complete well before the rest of the system -- ACIO/Thunderbolt, DCP, - * the crossbar -- has finished its own resume work. Driving the reconnect - * from here directly (as originally tried) fires apple_cio_start()'s - * M3/PMGR reset-deassert handshake while that firmware is potentially - * still busy servicing other subsystems' resume, and can exhaust its - * retry budget outright -- confirmed on hardware, see - * notes/2026-09-24-0147-typec-resume-reverify.md. A real physical - * replug never hits this because a human always does it well after - * the whole system has settled. - * - * PM_POST_SUSPEND is delivered strictly after dpm_resume_end() -- - * i.e. after every device, including ACIO/Thunderbolt, has completed - * its own ordinary resume -- so triggering the reverify from here - * instead removes the race without needing any arbitrary delay. - */ -static int tipd_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +/* Freezable cable work has drained before device PM calls prepare. */ +int tipd_prepare(struct device *dev) { - struct tps6598x *tps = container_of(nb, struct tps6598x, pm_nb); + struct tps6598x *tps = dev_get_drvdata(dev); - if (action == PM_POST_SUSPEND && tps->data->resume_reverify) { - mutex_lock(&tps->lock); - tps->data->resume_reverify(tps); - mutex_unlock(&tps->lock); - } + if (tps->data->suspend_prepare) + tps->data->suspend_prepare(tps); - return NOTIFY_DONE; + return 0; +} +EXPORT_SYMBOL_GPL(tipd_prepare); + +/* Completion also runs after an aborted suspend; work starts after thaw. */ +void tipd_complete(struct device *dev) +{ + struct tps6598x *tps = dev_get_drvdata(dev); + + if (tps->data->resume_reverify) + tps->data->resume_reverify(tps); } +EXPORT_SYMBOL_GPL(tipd_complete); const struct tipd_data tipd_cd321x_data = { .irq_handler = cd321x_interrupt, @@ -2093,6 +2234,7 @@ const struct tipd_data tipd_cd321x_data = { .remove = cd321x_remove, .register_port = cd321x_register_port, .unregister_port = cd321x_unregister_port, + .ready = cd321x_ready, .trace_data_status = trace_cd321x_data_status, .trace_power_status = trace_tps6598x_power_status, .trace_status = trace_tps6598x_status, @@ -2101,6 +2243,7 @@ const struct tipd_data tipd_cd321x_data = { .reset = cd321x_reset, .switch_power_state = cd321x_switch_power_state, .connect = cd321x_connect, + .suspend_prepare = cd321x_suspend_prepare, .resume_reverify = cd321x_resume_reverify, }; EXPORT_SYMBOL_GPL(tipd_cd321x_data); @@ -2152,6 +2295,7 @@ const struct tipd_data tipd_sn201202x_data = { .remove = cd321x_remove, .register_port = cd321x_register_port, .unregister_port = cd321x_unregister_port, + .ready = cd321x_ready, .trace_data_status = trace_cd321x_data_status, .trace_power_status = trace_tps6598x_power_status, .trace_status = trace_tps6598x_status, @@ -2160,6 +2304,7 @@ const struct tipd_data tipd_sn201202x_data = { .reset = cd321x_reset, .switch_power_state = cd321x_switch_power_state, .connect = cd321x_connect, + .suspend_prepare = cd321x_suspend_prepare, .resume_reverify = cd321x_resume_reverify, }; EXPORT_SYMBOL_GPL(tipd_sn201202x_data); diff --git a/drivers/usb/typec/tipd/i2c.c b/drivers/usb/typec/tipd/i2c.c index d369a708828c8..c954851d1bc1c 100644 --- a/drivers/usb/typec/tipd/i2c.c +++ b/drivers/usb/typec/tipd/i2c.c @@ -71,6 +71,8 @@ static int __maybe_unused tps6598x_resume(struct device *dev) } static const struct dev_pm_ops tps6598x_pm_ops = { + .prepare = pm_sleep_ptr(tipd_prepare), + .complete = pm_sleep_ptr(tipd_complete), SET_SYSTEM_SLEEP_PM_OPS(tps6598x_suspend, tps6598x_resume) }; diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 73575a854bde1..dc57ef88ca7fe 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -279,6 +279,8 @@ static int __maybe_unused sn201202x_suspend(struct device *dev) MODULE_DEVICE_TABLE(of, sn201202x_of_match); static const struct dev_pm_ops sn201202x_pm_ops = { + .prepare = pm_sleep_ptr(tipd_prepare), + .complete = pm_sleep_ptr(tipd_complete), SET_SYSTEM_SLEEP_PM_OPS(sn201202x_suspend, sn201202x_resume) }; diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 85bd83d809616..ee38d76333ec3 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -6,9 +6,11 @@ * Author: Heikki Krogerus */ +#include #include #include #include +#include #include #include #include @@ -16,6 +18,8 @@ #ifndef __TPS6598X_H__ #define __TPS6598X_H__ +#include "cd321x-pm.h" + #define TPS_FIELD_GET(_mask, _reg) ((typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask))) /* TPS_REG_STATUS bits */ @@ -283,6 +287,7 @@ struct tipd_data { void (*remove)(struct tps6598x *tps); int (*register_port)(struct tps6598x *tps, struct fwnode_handle *node); void (*unregister_port)(struct tps6598x *tps); + void (*ready)(struct tps6598x *tps); void (*trace_data_status)(u32 status); void (*trace_power_status)(u16 status); void (*trace_status)(u32 status); @@ -292,6 +297,7 @@ struct tipd_data { bool (*read_data_status)(struct tps6598x *tps); int (*reset)(struct tps6598x *tps); int (*connect)(struct tps6598x *tps, u32 status); + void (*suspend_prepare)(struct tps6598x *tps); void (*resume_reverify)(struct tps6598x *tps); }; @@ -319,8 +325,6 @@ struct tps6598x { u16 pwr_status; struct delayed_work wq_poll; - struct notifier_block pm_nb; - const struct tipd_data *data; }; @@ -376,11 +380,17 @@ struct cd321x { u32 dp_status; u32 dp_conf; struct typec_thunderbolt_switch *tbt_switch; + struct notifier_block tbt_notifier; + bool tbt_notifier_registered; + atomic_t link_event; bool display_route_active; struct cd321x_status update_status; struct delayed_work update_work; + struct delayed_work resume_work; + struct cd321x_pm_state pm; struct usb_pd_identity cur_partner_identity; + bool cur_partner_is_pd; struct fwnode_handle *connector_fwnode; }; @@ -401,6 +411,8 @@ extern const struct regmap_config tps6598x_regmap_config; int tipd_init(struct tps6598x *tps); void tipd_remove(struct tps6598x *tps); +int tipd_prepare(struct device *dev); +void tipd_complete(struct device *dev); int tipd_suspend(struct tps6598x *tps); int tipd_resume(struct tps6598x *tps); diff --git a/include/linux/pci-apple.h b/include/linux/pci-apple.h index 057b16bdd42c7..552a29357eb6b 100644 --- a/include/linux/pci-apple.h +++ b/include/linux/pci-apple.h @@ -4,9 +4,17 @@ struct device; struct device_node; +struct notifier_block; + +/* Atomic notification; data is the host's parent device, valid in callback. */ +#define APPLE_PCIE_TUNNEL_LINK_DOWN 1 + +int apple_pcie_tunnel_register_notifier(struct notifier_block *nb); +void apple_pcie_tunnel_unregister_notifier(struct notifier_block *nb); int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel); int apple_pcie_tunnel_quiesce(struct device *dev); int apple_pcie_tunnel_restore(struct device *dev); +int apple_pcie_tunnel_check_state(struct device *dev); #endif /* _LINUX_PCI_APPLE_H */ diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index 423602f02be3b..61ec51a342bde 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -3,6 +3,7 @@ #define __LINUX_SOC_APPLE_DART_H__ #include +#include struct device; @@ -13,9 +14,13 @@ struct device; #if IS_ENABLED(CONFIG_APPLE_DART) int apple_dart_quiesce_commands(struct device *dev); int apple_dart_resume_commands(struct device *dev); +int apple_dart_save_tunnel_state(struct device *dev); +int apple_dart_restore_tunnel_state(struct device *dev); #else static inline int apple_dart_quiesce_commands(struct device *dev) { return 0; } static inline int apple_dart_resume_commands(struct device *dev) { return 0; } +static inline int apple_dart_save_tunnel_state(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_restore_tunnel_state(struct device *dev) { return -EOPNOTSUPP; } #endif #endif diff --git a/include/linux/usb/typec_mux.h b/include/linux/usb/typec_mux.h index 0726407bb6240..d58d60fcae21f 100644 --- a/include/linux/usb/typec_mux.h +++ b/include/linux/usb/typec_mux.h @@ -15,6 +15,7 @@ struct typec_switch; struct typec_switch_dev; struct typec_altmode; struct fwnode_handle; +struct notifier_block; typedef int (*typec_switch_set_fn_t)(struct typec_switch_dev *sw, enum typec_orientation orientation); @@ -145,6 +146,9 @@ static inline struct typec_mux *typec_mux_get(struct device *dev) struct typec_thunderbolt_switch; struct typec_thunderbolt_switch_dev; +/* Recheck fresh cable data and provider health; not a request to reset. */ +#define TYPEC_THUNDERBOLT_SWITCH_REVALIDATE 1 + enum typec_thunderbolt_switch_state { TYPEC_THUNDERBOLT_SWITCH_OFF, TYPEC_THUNDERBOLT_SWITCH_TBT, @@ -192,6 +196,11 @@ fwnode_typec_thunderbolt_switch_get(struct fwnode_handle *fwnode); void typec_thunderbolt_switch_put(struct typec_thunderbolt_switch *sw); int typec_thunderbolt_switch_set(struct typec_thunderbolt_switch *sw, const struct typec_thunderbolt_switch_data *data); +int typec_thunderbolt_switch_register_notifier(struct typec_thunderbolt_switch *sw, + struct notifier_block *nb); +void typec_thunderbolt_switch_unregister_notifier(struct typec_thunderbolt_switch *sw, + struct notifier_block *nb); +void typec_thunderbolt_switch_notify(struct typec_thunderbolt_switch_dev *sw); struct typec_thunderbolt_switch_dev * typec_thunderbolt_switch_register(struct device *parent, @@ -230,6 +239,24 @@ typec_thunderbolt_switch_register(struct device *parent, return ERR_PTR(-EOPNOTSUPP); } +static inline int +typec_thunderbolt_switch_register_notifier(struct typec_thunderbolt_switch *sw, + struct notifier_block *nb) +{ + return 0; +} + +static inline void +typec_thunderbolt_switch_unregister_notifier(struct typec_thunderbolt_switch *sw, + struct notifier_block *nb) +{ +} + +static inline void +typec_thunderbolt_switch_notify(struct typec_thunderbolt_switch_dev *sw) +{ +} + static inline void typec_thunderbolt_switch_unregister(struct typec_thunderbolt_switch_dev *sw) { From 75b0123137be7f2683c6f81058ccf1382650aa8a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:39:09 -0700 Subject: [PATCH 0288/1579] PCI: apple: resume tunneled functions as if from D3cold Stopping a tunneled port at suspend asserts tunneled PERST, and on t8103 resume also resets the PCIe-C controller. Every function below the port therefore loses its state, even though the host itself only enters suspend-to-idle. The PCI core does not know that: the tunneled functions never enter D3, so in suspend-to-idle it skips their noirq restore and the wait for the secondary bus, and restores config space only in the asynchronous resume phase, with no readiness check. In that window nothing below the root port is configured yet. pciehp on an occupied hotplug port, such as the one in a CalDigit TS3 Plus, reads the device below it through bridges that have not been restored, can see ~0, and then removes the device mid-resume while its driver still accesses its registers. Mark every tunneled function as having lost power when the port is stopped for system sleep. The PCI core then restores config space in the noirq phase, parents first, waits for each secondary bus, and marks a subtree disconnected when it does not come back. When the port fails to resume, mark the hierarchy disconnected instead of gating the port clock under still-bound drivers, as the quiesce path already does. Quiesce removes the hierarchy and stops the port once tasks are thawed. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/pci/controller/pcie-apple.c | 49 ++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 39dbeb1e504b2..8329802a1ade6 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2552,6 +2552,41 @@ static void apple_pcie_remove(struct platform_device *pdev) pci_unlock_rescan_remove(); } +static int apple_pcie_mark_power_lost(struct pci_dev *pdev, void *unused) +{ + /* + * Tunnel stop asserts tunneled PERST, so the functions below lose their + * state even when the host stays in suspend-to-idle. Make the PCI core + * resume them as if from D3cold: restore config space in the noirq phase, + * parents first, and wait for each secondary bus before touching it. + */ + pdev->skip_bus_pm = false; + pdev->current_state = PCI_D3cold; + return 0; +} + +static void apple_pcie_tunnel_hierarchy_lost(struct device *dev) +{ + struct pci_host_bridge *bridge = dev_get_drvdata(dev); + + if (bridge && bridge->bus) + pci_walk_bus(bridge->bus, apple_pcie_mark_power_lost, NULL); +} + +static void apple_pcie_tunnel_hierarchy_gone(struct device *dev) +{ + struct pci_host_bridge *bridge = dev_get_drvdata(dev); + + /* + * Nothing below a port that failed to resume can be reached. Mark it + * disconnected so that neither the PCI core nor function drivers + * resuming after us issue I/O into the dead aperture; quiesce removes + * the hierarchy once tasks are thawed. + */ + if (bridge && bridge->bus) + pci_walk_bus(bridge->bus, pci_dev_set_disconnected, NULL); +} + static int apple_pcie_suspend_noirq(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); @@ -2587,6 +2622,8 @@ static int apple_pcie_suspend_noirq(struct device *dev) } } pcie->reset_on_resume = can_reset && active; + if (active && !pcie->bus_stopped) + apple_pcie_tunnel_hierarchy_lost(dev); return 0; } @@ -2659,11 +2696,15 @@ static int apple_pcie_resume_noirq(struct device *dev) failed: /* Device PM keeps resuming dependents even after this callback fails. */ pcie->resume_failed = true; + if (!pcie->bus_stopped) + apple_pcie_tunnel_hierarchy_gone(dev); apple_pcie_walk_tunnel_darts(pcie, apple_dart_quiesce_commands, false); - list_for_each_entry(port, &pcie->ports, entry) { - if (port->started || port->needs_stop) - apple_pcie_tunnel_stop(port); - } + /* + * Leave the port clocked with its link down, the state a surprise + * unplug leaves behind, in which stray accesses complete with an + * error. Quiesce removes the hierarchy and then stops the port when + * ACIO is torn down after resume. + */ return ret; } From 5c374fed4a3b3db61df48541d417d8216b4a12a4 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:39:09 -0700 Subject: [PATCH 0289/1579] thunderbolt: keep routers awake across system sleep on Apple hosts On Apple silicon the host router stays powered through suspend-to-idle and its links stay trained. Asking the routers to sleep anyway leaves the request pending with nothing to complete it, and the Thunderbolt link then fails at a fixed time after the request, whether the host is still asleep or has already resumed: thunderbolt-apple-acio ...: RTKit: syslog message: ... Gen2/3 link error pcie-apple 3b0000000.pcie: Link down on ... On a MacBook Pro 13" M1 (j293) with a Kensington SD5560T this happened 40.8 s after "PM: suspend entry" in each of ten cycles, on four kernels and both ports. A suspend longer than that loses the dock while asleep; a shorter one loses it after resume. A fresh link training before the deadline cancels it. Add QUIRK_NO_SYSTEM_SLEEP and skip the wake and sleep programming for system sleep when it is set. Plug events stay disabled as before and are re-enabled on resume. The Apple NHI sets the quirk unless the router_sleep test parameter asks for the previous behaviour. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/thunderbolt/apple.c | 13 +++++++++++++ drivers/thunderbolt/nhi.h | 6 ++++++ drivers/thunderbolt/switch.c | 4 ++++ 3 files changed, 23 insertions(+) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index eca55995271e3..37e1769d17938 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -1739,10 +1739,23 @@ static int apple_nhi_prepare(struct device *dev) return 0; } +/* + * ACIO stays powered through suspend-to-idle and keeps its links trained, so + * nothing ever completes a router sleep request. Leave the routers awake. + */ +static bool router_sleep; +module_param(router_sleep, bool, 0644); +MODULE_PARM_DESC(router_sleep, "Ask routers to sleep on system suspend (test only)"); + static int apple_nhi_suspend_noirq(struct device *dev) { struct apple_nhi *anhi = dev_get_drvdata(dev); + if (READ_ONCE(router_sleep)) + anhi->nhi.quirks &= ~QUIRK_NO_SYSTEM_SLEEP; + else + anhi->nhi.quirks |= QUIRK_NO_SYSTEM_SLEEP; + return tb_domain_suspend_noirq(anhi->tb); } diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index 6ac76844a4973..d4170fb3d2469 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -178,6 +178,12 @@ struct tb_nhi_ops { #define QUIRK_NO_DMA_PORT BIT(2) #define QUIRK_NO_USB3_BW_ALLOC BIT(3) #define QUIRK_HOST_DP_NFC_CREDITS BIT(4) +/* + * The host router stays powered and its links stay up across system sleep, + * so routers must not be asked to enter sleep: a TBT3 device router that + * was told to sleep drops its link on its own some tens of seconds later. + */ +#define QUIRK_NO_SYSTEM_SLEEP BIT(5) /* * Minimal number of vectors when we use MSI-X. Two for control channel diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c index 4c7e63dd4332d..213eba1b404a9 100644 --- a/drivers/thunderbolt/switch.c +++ b/drivers/thunderbolt/switch.c @@ -3663,6 +3663,10 @@ void tb_switch_suspend(struct tb_switch *sw, bool runtime) flags |= TB_WAKE_ON_USB4 | TB_WAKE_ON_USB3 | TB_WAKE_ON_PCIE; } + /* The link stays up; a sleeping router would drop it by itself. */ + if (!runtime && (sw->tb->nhi->quirks & QUIRK_NO_SYSTEM_SLEEP)) + return; + tb_switch_set_wake(sw, flags, runtime); if (tb_switch_is_usb4(sw)) From d906a28eaff81425e1a36d048a0f75b6b1aed16d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:39:09 -0700 Subject: [PATCH 0290/1579] thunderbolt: restore DP resources after resume when routers stayed awake Suspend releases every DP tunnel and resource, expecting plug events to announce them again after resume. Routers that were never asked to sleep send no such events, so the DP IN and DP OUT adapters are not paired up again and an external display behind a dock stays dark after resume. For hosts with QUIRK_NO_SYSTEM_SLEEP, re-add the host and device DP IN resources and any DP OUT adapter with HPD asserted from tb_complete(), then pair them. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/thunderbolt/tb.c | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index d822259595345..f35bd9e21d0dd 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -3292,6 +3292,43 @@ static int tb_thaw_noirq(struct tb *tb) return 0; } +static bool tb_dp_resource_listed(struct tb *tb, struct tb_port *port) +{ + struct tb_cm *tcm = tb_priv(tb); + struct tb_port *p; + + list_for_each_entry(p, &tcm->dp_resources, list) { + if (p == port) + return true; + } + return false; +} + +static void tb_restore_dp_resources(struct tb_switch *sw) +{ + struct tb_cm *tcm = tb_priv(sw->tb); + struct tb_port *port; + + if (sw->is_unplugged) + return; + + tb_switch_for_each_port(sw, port) { + if (tb_port_has_remote(port)) { + tb_restore_dp_resources(port->remote->sw); + } else if (tb_port_is_dpin(port)) { + if (tb_dp_resource_listed(sw->tb, port) || + !tb_switch_query_dp_resource(sw, port)) + continue; + tb_port_dbg(port, "DP IN resource available after resume\n"); + list_add_tail(&port->list, &tcm->dp_resources); + } else if (tb_port_is_dpout(port) && + tb_dp_port_hpd_is_active(port) == 1 && + !tb_dp_port_is_enabled(port)) { + tb_dp_resource_available(sw->tb, port); + } + } +} + static void tb_complete(struct tb *tb) { /* @@ -3302,6 +3339,14 @@ static void tb_complete(struct tb *tb) mutex_lock(&tb->lock); if (tb_free_unplugged_xdomains(tb->root_switch)) tb_scan_switch(tb->root_switch); + /* + * Routers kept awake through system sleep send no plug events for + * the DP resources released at suspend, so pair them up again here. + */ + if (tb->nhi->quirks & QUIRK_NO_SYSTEM_SLEEP) { + tb_restore_dp_resources(tb->root_switch); + tb_tunnel_dp(tb); + } mutex_unlock(&tb->lock); } From c6c9f02fe5ef96e79baff8b9b1d1b750bdb2dbde Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:03:25 -0700 Subject: [PATCH 0291/1579] thunderbolt: keep M1 PCIe tunnels up through suspend-to-idle In suspend-to-idle ACIO stays powered and, with the routers kept awake, the Thunderbolt link stays trained. Stopping the tunneled PCIe port anyway holds the dock hierarchy in reset for the whole sleep. After that, every function below it has to be restored and each USB device behind the dock's xHCI reset-resumed one after another, about 5 s before input works again. After a sleep of about 80 s, a Kensington SD5560T on a MacBook Pro 13" M1 (j293) stopped answering altogether shortly after resume: its link trained and its bridges were restored, then every request timed out. When the tunneled port is up and healthy at suspend-to-idle, leave it running. On resume, check that the link is still up and that no link-down event is pending; if so the hierarchy kept its state and resumes like any other PCIe device. If the link was lost while asleep, fall back to the stop, reset and restart path. The Thunderbolt connection manager then leaves the still-active tunnels alone instead of tearing them down and setting them up again. The pcie_apple s2idle_keep_link parameter restores the previous behaviour for comparison. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/pci/controller/pcie-apple.c | 81 +++++++++++++++++++++++++++-- drivers/thunderbolt/apple.c | 23 ++++++-- drivers/thunderbolt/nhi.h | 5 ++ drivers/thunderbolt/tb.c | 10 ++++ include/linux/pci-apple.h | 1 + 5 files changed, 113 insertions(+), 7 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 8329802a1ade6..8cd5708b7c5fa 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -37,6 +37,7 @@ #include #include #include +#include #include "../pci.h" #include "pci-host-common.h" @@ -55,6 +56,10 @@ void apple_pcie_tunnel_unregister_notifier(struct notifier_block *nb) } EXPORT_SYMBOL_GPL(apple_pcie_tunnel_unregister_notifier); +static bool s2idle_keep_link = true; +module_param(s2idle_keep_link, bool, 0644); +MODULE_PARM_DESC(s2idle_keep_link, "Keep tunneled links up through suspend-to-idle"); + static int link_up_timeout = 500; module_param(link_up_timeout, int, 0644); MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); @@ -236,6 +241,7 @@ struct apple_pcie { bool kernel_init; bool bus_stopped; bool reset_on_resume; + bool link_kept; bool resume_failed; /* cleared only by teardown and reprobe */ struct reset_control *reset; const struct hw_info *hw; @@ -2315,6 +2321,22 @@ int apple_pcie_tunnel_check_state(struct device *dev) } EXPORT_SYMBOL_GPL(apple_pcie_tunnel_check_state); +/* + * Whether the host left its tunnel running for the system sleep in progress. + * The caller serializes against unbind through device PM ordering. + */ +bool apple_pcie_tunnel_link_kept(struct device *dev) +{ + struct pci_host_bridge *bridge = dev_get_drvdata(dev); + struct apple_pcie *pcie; + + if (!dev->driver || !bridge) + return false; + pcie = pci_host_bridge_priv(bridge); + return pcie->hw->tunneled && pcie->link_kept; +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_link_kept); + struct apple_pcie_map { void __iomem *base; struct resource res; @@ -2587,16 +2609,47 @@ static void apple_pcie_tunnel_hierarchy_gone(struct device *dev) pci_walk_bus(bridge->bus, pci_dev_set_disconnected, NULL); } -static int apple_pcie_suspend_noirq(struct device *dev) +static bool apple_pcie_tunnel_link_healthy(struct apple_pcie *pcie) +{ + struct apple_pcie_port *port; + + if (list_empty(&pcie->ports)) + return false; + list_for_each_entry(port, &pcie->ports, entry) { + if (!READ_ONCE(port->started) || READ_ONCE(port->link_failed)) + return false; + if (apple_pcie_port_readl(port, PORT_INTSTAT) & + BIT(PORT_INT_LINK_DOWN)) + return false; + if (!(apple_pcie_port_readl(port, PORT_LINKSTS) & PORT_LINKSTS_UP)) + return false; + } + return true; +} + +/* + * In suspend-to-idle ACIO and the router links stay powered, so an up + * tunnel can simply be left running. The functions below keep their state + * and resume like any other PCIe device; nothing has to be re-trained. + */ +static bool apple_pcie_keep_link(struct apple_pcie *pcie) +{ + if (!READ_ONCE(s2idle_keep_link) || !pm_suspend_no_platform()) + return false; + if (!pcie->kernel_init || !pcie->power_retained) + return false; + if (pcie->bus_stopped || pcie->resume_failed) + return false; + return apple_pcie_tunnel_link_healthy(pcie); +} + +static void apple_pcie_stop_for_sleep(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); struct apple_pcie_port *port; bool active = false, can_reset; int i, ret; - if (!pcie->hw->tunneled) - return 0; - pcie->reset_on_resume = false; can_reset = pcie->kernel_init && pcie->power_retained && pcie->reset && !pcie->bus_stopped && !pcie->resume_failed; if (can_reset) { @@ -2624,6 +2677,18 @@ static int apple_pcie_suspend_noirq(struct device *dev) pcie->reset_on_resume = can_reset && active; if (active && !pcie->bus_stopped) apple_pcie_tunnel_hierarchy_lost(dev); +} + +static int apple_pcie_suspend_noirq(struct device *dev) +{ + struct apple_pcie *pcie = apple_pcie_lookup(dev); + + if (!pcie->hw->tunneled) + return 0; + pcie->reset_on_resume = false; + pcie->link_kept = apple_pcie_keep_link(pcie); + if (!pcie->link_kept) + apple_pcie_stop_for_sleep(dev); return 0; } @@ -2677,6 +2742,14 @@ static int apple_pcie_resume_noirq(struct device *dev) return 0; if (pcie->resume_failed) return -EIO; + if (pcie->link_kept) { + pcie->link_kept = false; + if (apple_pcie_tunnel_link_healthy(pcie)) + return 0; + /* Lost while asleep: take the same path as a stopped tunnel. */ + dev_warn(dev, "PCIe-C link lost during suspend-to-idle\n"); + apple_pcie_stop_for_sleep(dev); + } if (pcie->reset_on_resume) { pcie->reset_on_resume = false; ret = apple_pcie_reset_for_resume(pcie); diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 37e1769d17938..4e168a8b414e9 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -1747,14 +1747,31 @@ static bool router_sleep; module_param(router_sleep, bool, 0644); MODULE_PARM_DESC(router_sleep, "Ask routers to sleep on system suspend (test only)"); +static bool apple_nhi_pcie_link_kept(struct apple_cio *acio) +{ + struct platform_device *pcie_pdev; + bool kept = false; + + guard(mutex)(&acio->pcie_tunnel_lock); + pcie_pdev = apple_cio_find_pcie_tunnel(acio); + if (pcie_pdev) { + kept = apple_pcie_tunnel_link_kept(&pcie_pdev->dev); + put_device(&pcie_pdev->dev); + } + return kept; +} + static int apple_nhi_suspend_noirq(struct device *dev) { struct apple_nhi *anhi = dev_get_drvdata(dev); - if (READ_ONCE(router_sleep)) - anhi->nhi.quirks &= ~QUIRK_NO_SYSTEM_SLEEP; - else + anhi->nhi.quirks &= ~(QUIRK_NO_SYSTEM_SLEEP | QUIRK_KEEP_TUNNELS); + if (!READ_ONCE(router_sleep)) { anhi->nhi.quirks |= QUIRK_NO_SYSTEM_SLEEP; + /* PCIe-C suspends first; its tunnel must then survive resume. */ + if (apple_nhi_pcie_link_kept(anhi->acio)) + anhi->nhi.quirks |= QUIRK_KEEP_TUNNELS; + } return tb_domain_suspend_noirq(anhi->tb); } diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index d4170fb3d2469..65024dd3f9b3c 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -184,6 +184,11 @@ struct tb_nhi_ops { * was told to sleep drops its link on its own some tens of seconds later. */ #define QUIRK_NO_SYSTEM_SLEEP BIT(5) +/* + * Set for one system sleep when the routers stayed awake and the tunneled + * PCIe host kept its link up: the tunnels are still active on resume. + */ +#define QUIRK_KEEP_TUNNELS BIT(6) /* * Minimal number of vectors when we use MSI-X. Two for control channel diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index f35bd9e21d0dd..e4d94c86931b3 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -3214,6 +3214,15 @@ static int tb_resume_noirq(struct tb *tb) tb_free_unplugged_children(tb->root_switch); tb_restore_children(tb->root_switch); + /* + * Routers that stayed awake still carry our tunnels. Restarting them + * would take down links the hosts kept up through the sleep. + */ + if (tb->nhi->quirks & QUIRK_KEEP_TUNNELS) { + tb_dbg(tb, "tunnels kept across sleep\n"); + goto out; + } + /* * If we get here from suspend to disk the boot firmware or the * restore kernel might have created tunnels of its own. Since @@ -3246,6 +3255,7 @@ static int tb_resume_noirq(struct tb *tb) tb_dbg(tb, "tunnels restarted, sleeping for 100ms\n"); msleep(100); } +out: tb_switch_enter_redrive(tb->root_switch); /* Allow tb_handle_hotplug to progress events */ tcm->hotplug_active = true; diff --git a/include/linux/pci-apple.h b/include/linux/pci-apple.h index 552a29357eb6b..d7c2d244d209b 100644 --- a/include/linux/pci-apple.h +++ b/include/linux/pci-apple.h @@ -16,5 +16,6 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel); int apple_pcie_tunnel_quiesce(struct device *dev); int apple_pcie_tunnel_restore(struct device *dev); int apple_pcie_tunnel_check_state(struct device *dev); +bool apple_pcie_tunnel_link_kept(struct device *dev); #endif /* _LINUX_PCI_APPLE_H */ From 14efce65e7dc7f72cc308f82e2988e9b2b18ac36 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:31:47 -0700 Subject: [PATCH 0292/1579] thunderbolt: keep DP tunnels through suspend-to-idle with kept tunnels Suspend releases DP tunnels so plug events can rebuild them after resume. When the routers stay awake and the other tunnels are kept, that release is the only thing that disconnects a display behind a dock. The display driver reports it to userspace as an unplug. After resume the compositor then switches off the output it still believes is gone, racing the restored tunnel. With a CalDigit TS3 Plus on a MacBook Pro 13" M1 (j293) the monitor stayed dark after resume: the DPTX link trained, then the output was powered off and never re-enabled. Keep DP tunnels as well when tunnels are kept. The display is then powered down and up across the sleep, as for DPMS, and is never reported as unplugged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/thunderbolt/tb.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index e4d94c86931b3..c5d7220daaf43 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -3152,7 +3152,12 @@ static int tb_suspend_noirq(struct tb *tb) struct tb_cm *tcm = tb_priv(tb); tb_dbg(tb, "suspending...\n"); - tb_disconnect_and_release_dp(tb); + /* + * Kept tunnels survive the sleep. Keep DP ones as well, so a display + * behind a dock is only powered down, not reported as unplugged. + */ + if (!(tb->nhi->quirks & QUIRK_KEEP_TUNNELS)) + tb_disconnect_and_release_dp(tb); tb_switch_exit_redrive(tb->root_switch); tb_switch_suspend(tb->root_switch, false); tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */ From b8623d58ffac0211a9a585655b83d6e633e5387e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:41:25 -0700 Subject: [PATCH 0293/1579] drm/apple: keep the Type-C cable state across system sleep Suspend cleared typec_cable_connected, relying on the Thunderbolt DP tunnel being released at suspend and set up again at resume, which reports the cable once more. When the tunnel is kept through suspend-to-idle nothing reports it again. dcp_poweron() then skips the DPTX connect, the modeset fails, and every later flip on that CRTC times out: apple-drm soc:display-subsystem: [drm] *ERROR* flip_done timed out apple-drm soc:display-subsystem: [drm] *ERROR* [CRTC:59:crtc-1] commit wait timed out The compositor stalls until the dock is unplugged. The route already reports cable removal through dcp_dptx_disconnect_oob(). Leave the cable state alone on suspend and clear it only on unbind, so a kept tunnel resumes like DPMS on. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/gpu/drm/apple/dcp.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0e6489c14da2a..fc3b34879ab3f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2178,10 +2178,11 @@ static int dcp_connector_type_from_dt(struct device_node *np) return DRM_MODE_CONNECTOR_Unknown; } -static void dcp_disable_typec_work(struct apple_dcp *dcp) +static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) { scoped_guard(mutex, &dcp->hpd_mutex) { - WRITE_ONCE(dcp->typec_cable_connected, false); + if (release_cable) + WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; } /* Block new enqueues as well as draining users of the AFK endpoints. */ @@ -2316,7 +2317,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->hdmi_hpd_irq) disable_irq(dcp->hdmi_hpd_irq); - dcp_disable_typec_work(dcp); + dcp_disable_typec_work(dcp, true); typec_mux_put(dcp->typec_mux); if (dcp->avep) { @@ -2568,7 +2569,13 @@ static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); - dcp_disable_typec_work(dcp); + /* + * The Type-C route reports cable removal through + * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays + * connected, and resume powers the CRTC back up through dcp_poweron() + * as after DPMS off. + */ + dcp_disable_typec_work(dcp, false); if (dcp->avep) av_service_disconnect(dcp); From be6e4f675c23a5d22e2b664b67029ff882e6b564 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:02:44 -0700 Subject: [PATCH 0294/1579] drm/apple: recover Type-C outputs whose swaps DCP drops DCP accepts a modeset on an external Type-C output even when the pipe cannot be enabled yet, and then drops every swap without completing it. That happens on resume from suspend-to-idle when the sink is slower than the link: a TV behind the DP-to-HDMI converter of a Kensington SD5560T dock asserted HPD only after the first modeset. Each flip then waits for the full commit timeout, and the compositor stalls with it: apple-drm soc:display-subsystem: [drm] *ERROR* flip_done timed out apple-drm soc:display-subsystem: [drm] *ERROR* [CRTC:59:crtc-1] commit wait timed out Arm a watchdog when a swap is submitted on a Type-C output. If DCP has not completed the swap within a second, complete the flip, mark the mode invalid so further commits do not wait for DCP, and let the hotplug worker re-apply the active CRTC, a bounded number of times. When the sink asserts HPD after a failed modeset, re-apply the mode as well. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PA1D59hehGmwnpSxrYcDbk --- drivers/gpu/drm/apple/dcp-internal.h | 6 ++++ drivers/gpu/drm/apple/dcp.c | 42 ++++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 13 ++++++++ 3 files changed, 61 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index f9ca90b1dfb30..fda7dc887ad34 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -48,6 +48,8 @@ struct apple_dcp_typec_route { }; bool dcp_is_typec_output(struct apple_dcp *dcp); +void dcp_swap_watchdog_arm(struct apple_dcp *dcp); +void dcp_swap_watchdog_complete(struct apple_dcp *dcp); bool dcp_is_usb4_output(struct apple_dcp *dcp); void dcp_retry_placeholder_edid(struct apple_dcp *dcp, const struct drm_edid *drm_edid); @@ -254,6 +256,10 @@ struct apple_dcp { /* Workqueue for sending vblank events when a dcp swap is not possible */ struct work_struct vblank_wq; + /* Completes a Type-C swap that DCP dropped, and recovers the pipe. */ + struct delayed_work swap_watchdog_wq; + unsigned int swap_watchdog_retrains; + /* List of referenced drm_framebuffers which can be unreferenced * on the next successfully completed swap. */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index fc3b34879ab3f..a32d60bddf124 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1142,6 +1142,44 @@ static void dcp_delayed_vblank(struct work_struct *work) dcp_drm_crtc_vblank(dcp->crtc); } +#define DCP_SWAP_WATCHDOG_MS 1000 +#define DCP_SWAP_WATCHDOG_RETRAINS 5 + +/* + * DCP drops swaps without completing them while an external pipe is not + * enabled, for instance after a modeset that raced a Type-C sink which had + * not asserted HPD yet. Userspace would then wait for the flip until the + * commit times out, stalling every output it drives. Complete the flip, + * mark the mode invalid so later commits do not wait for DCP, and let the + * hotplug worker re-apply the active CRTC. + */ +static void dcp_swap_watchdog(struct work_struct *work) +{ + struct apple_dcp *dcp = + container_of(to_delayed_work(work), struct apple_dcp, + swap_watchdog_wq); + + dev_warn(dcp->dev, "swap not completed, retraining the display\n"); + dcp_mode_invalidate(&dcp->mode_state); + dcp_drm_crtc_vblank(dcp->crtc); + if (dcp->connector && + dcp->swap_watchdog_retrains++ < DCP_SWAP_WATCHDOG_RETRAINS) + schedule_work(&dcp->connector->hotplug_wq); +} + +void dcp_swap_watchdog_arm(struct apple_dcp *dcp) +{ + if (dcp_is_typec_output(dcp)) + mod_delayed_work(system_wq, &dcp->swap_watchdog_wq, + msecs_to_jiffies(DCP_SWAP_WATCHDOG_MS)); +} + +void dcp_swap_watchdog_complete(struct apple_dcp *dcp) +{ + cancel_delayed_work(&dcp->swap_watchdog_wq); + dcp->swap_watchdog_retrains = 0; +} + static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) { struct apple_dcp *dcp = cookie; @@ -1852,6 +1890,7 @@ void dcp_poweroff(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + cancel_delayed_work(&dcp->swap_watchdog_wq); if (dcp->avep) av_service_disconnect(dcp); @@ -2281,6 +2320,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) set_bit(0, dcp->memdesc_map); INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); + INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); @@ -2360,6 +2400,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); } + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); cancel_work_sync(&dcp->vblank_wq); devm_clk_put(dev, dcp->clk); @@ -2576,6 +2617,7 @@ static int dcp_platform_suspend(struct device *dev) * as after DPMS off. */ dcp_disable_typec_work(dcp, false); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); if (dcp->avep) av_service_disconnect(dcp); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 823704dded12e..38a1ebb56be36 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -126,6 +126,7 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, ktime_t now = ktime_get(); trace_iomfb_swap_complete(dcp, resp->swap_id); dcp->last_swap_id = resp->swap_id; + dcp_swap_watchdog_complete(dcp); dcp_drm_crtc_page_flip(dcp, now); if (dcp->crc_enabled) { @@ -1061,6 +1062,17 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) action = dcp_mode_hotplug(&dcp->mode_state, !!(*connected), connector ? &connector->connected : NULL); + /* + * A Type-C sink can assert HPD only after a modeset has already failed, + * as a TV behind a DP-to-HDMI converter does when it wakes from standby. + * The connector state does not change then; re-apply the mode anyway. + */ + if (*connected && dcp_is_typec_output(dcp) && + !READ_ONCE(dcp->mode_state.valid) && + !READ_ONCE(dcp->mode_state.changing)) { + dcp->swap_watchdog_retrains = 0; + action |= DCP_HOTPLUG_NOTIFY; + } dcp_handle_hotplug_actions(dcp, action); } @@ -1164,6 +1176,7 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) } dcp->swap_start = ktime_get(); dcp->swap_submit_timestamp = arch_timer_read_counter(); + dcp_swap_watchdog_arm(dcp); while (!list_empty(&dcp->swapped_out_fbs)) { struct dcp_fb_reference *entry; From 28f280d3aa55c106dccf3e29ca5d3fe1855240d3 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:14:06 -0700 Subject: [PATCH 0295/1579] usb: dwc3: apple: leave the USB2 host role selection to xHCI Since "usb: dwc3: apple: Acquire the PHYs at probe", the glue selects the USB2 PHY role before it releases the dwc3 reset on every bring-up, including the first one after boot. Before that commit, the first bring-up's selection was a no-op, and mainline still behaves that way. For a host, the role was then selected once the core was out of reset: usb_add_hcd() asks the root hub PHYs for PHY_MODE_USB_HOST_SS, which the USB2 PHY rejects, and then for PHY_MODE_USB_HOST. On a Type-C port the USB2 PHY is already running when the glue makes its selection, because the Type-C mux starts it before the role switch reaches the glue. Switching the running PHY to host mode while the core is held in reset leaves the USB2 port unable to see a connection for the whole session. The SuperSpeed side is unaffected. Selecting host mode is harmless only when the PHY is in host mode already, and then the selection does nothing. On a t8103 (j293) with a CalDigit TS4 in USB4 mode, the first session on each port after boot never connected USB2. That was 7 of 7 sessions, across both ports, with the dock present at boot and hot-plugged. The xHCI USB2 root port stayed powered in RxDetect (PORTSC 0x0a0002a0). USB2-only devices behind the dock, such as keyboards and mice, never appeared, and only a replug on the same port brought them up. Later sessions connected, because the PHY kept host mode from the first one. Power-cycling only the USB2 PHY around the role change did not help. Leaving host mode to the root hub did: the dock's USB2 hub connected on the first session, 0.4 s after the dock, with its keyboard and mice. Select only device mode before the reset. Device mode has no later selector, since the glue writes PRTCAP itself and the gadget code sets no PHY mode, so it keeps its pre-reset selection. Host mode is always left to the root hub. Only the first host session after boot was tested. A later device-to-host change reaches the same PHY state: host mode is clear and the PHY is powered off at the end of the device session and restarted by the Type-C mux. It is expected to behave the same, but has not been tested. A fixed-hub controller is unaffected, because its PHY driver sets host mode for it. Fixes: 0ab0ac187140 ("usb: dwc3: apple: Acquire the PHYs at probe") Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/usb/dwc3/dwc3-apple.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 1f86350a11dfc..2d0662ac77f72 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -93,7 +93,8 @@ enum dwc3_apple_state { * @mmio_resource: Resource to be passed to dwc3_core_probe * @apple_regs: Apple-specific DWC3 registers * @reset: Reset control - * @usb2_phy: USB2 PHY, configured before the core is brought up + * @usb2_phy: USB2 PHY, set to device mode before the core is brought up; the xHCI root hub + * sets host mode * @usb3_phy: USB3 PHY, configured after the core is brought up * @role_sw: USB role switch * @fixed_hub: A fixed hub sits on the controller's USB2 port, the controller stays in host mode @@ -269,17 +270,23 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st lockdep_assert_held(&appledwc->lock); /* - * The USB2 PHY on this platform must be configured for host or device mode while it is - * still powered off and before dwc3 tries to access it. Otherwise, the new configuration - * will sometimes only take affect after the *next* time dwc3 is brought up which causes - * the connected device to just not work. + * The USB2 PHY on this platform must be configured for device mode before dwc3 tries to + * access it. Otherwise, the new configuration will sometimes only take affect after the + * *next* time dwc3 is brought up which causes the connected device to just not work. + * Host mode is left to the xHCI root hub, which selects it once the core is out of reset + * (usb_add_hcd() asks for PHY_MODE_USB_HOST_SS, which the PHY rejects, and then for + * PHY_MODE_USB_HOST). On a Type-C port the PHY is already running at this point, because + * the Type-C mux starts it before the role switch reaches this driver, and switching the + * running PHY to host mode while the core is still held in reset leaves the USB2 port + * without a connection for the whole session. A PHY that is in host mode already needs no + * selection here either. * The USB3 PHY must be configured later after dwc3 has already been initialized. - * Both PHYs were looked up at probe, so this also covers the first bring-up, before - * dwc3_core_probe() has looked them up itself. + * Both PHYs were looked up at probe, so device mode is also selected on the first + * bring-up, before dwc3_core_probe() has looked them up itself. */ switch (state) { case DWC3_APPLE_HOST: - ret = phy_set_mode(appledwc->usb2_phy, PHY_MODE_USB_HOST); + ret = 0; break; case DWC3_APPLE_DEVICE: ret = phy_set_mode(appledwc->usb2_phy, PHY_MODE_USB_DEVICE); @@ -326,8 +333,9 @@ static void dwc3_apple_set_role(struct dwc3_apple *appledwc, enum dwc3_apple_sta dwc3_apple_set_ptrcap(appledwc, host ? DWC3_GCTL_PRTCAP_HOST : DWC3_GCTL_PRTCAP_DEVICE); /* * This platform requires SUSPHY to be enabled here already in order to properly - * configure the PHY and switch dwc3's PIPE interface to USB3 PHY. The USB2 PHY - * has already been configured to the correct mode earlier. + * configure the PHY and switch dwc3's PIPE interface to USB3 PHY. The USB2 PHY's + * device mode has already been selected in dwc3_apple_core_start(); its host mode + * is selected by the xHCI root hub as it starts. */ dwc3_enable_susphy(&appledwc->dwc, true); ret = phy_set_mode(appledwc->usb3_phy, host ? PHY_MODE_USB_HOST : PHY_MODE_USB_DEVICE); From 65eb108802f4a834ae568967357d1a7a501ad8f1 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:25:13 -0700 Subject: [PATCH 0296/1579] usb: dwc3: apple: test the USB2 PHY mode choice with KUnit The previous commit selects the USB2 PHY's mode before dwc3 leaves reset for device mode only, and leaves host mode to the xHCI root hub. Selecting host mode there again would bring back sessions without USB2. Move that choice into dwc3_apple_usb2_mode_before_reset(), which also rejects states that are not roles, and move the controller state enum into dwc3-apple-state.h with it. dwc3_apple_core_start() takes both the validation and the mode from the helper, so the choice lives in one place. A KUnit suite pins the helper's results for every state. It covers the helper, not the hardware ordering behind it. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- MAINTAINERS | 2 + drivers/usb/dwc3/Kconfig | 14 ++++++ drivers/usb/dwc3/Makefile | 1 + drivers/usb/dwc3/dwc3-apple-state-test.c | 56 ++++++++++++++++++++++++ drivers/usb/dwc3/dwc3-apple-state.h | 51 +++++++++++++++++++++ drivers/usb/dwc3/dwc3-apple.c | 32 ++++++-------- 6 files changed, 137 insertions(+), 19 deletions(-) create mode 100644 drivers/usb/dwc3/dwc3-apple-state-test.c create mode 100644 drivers/usb/dwc3/dwc3-apple-state.h diff --git a/MAINTAINERS b/MAINTAINERS index 235c46ff97605..627dfe786a083 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2620,6 +2620,8 @@ F: drivers/soc/apple/* F: drivers/spi/spi-apple.c F: drivers/spmi/spmi-apple-controller.c F: drivers/thunderbolt/apple.c +F: drivers/usb/dwc3/dwc3-apple-state-test.c +F: drivers/usb/dwc3/dwc3-apple-state.h F: drivers/usb/dwc3/dwc3-apple.c F: drivers/usb/typec/tipd/spmi.c F: drivers/video/backlight/apple_dwi_bl.c diff --git a/drivers/usb/dwc3/Kconfig b/drivers/usb/dwc3/Kconfig index 18169727a413e..3e033ddc86d3d 100644 --- a/drivers/usb/dwc3/Kconfig +++ b/drivers/usb/dwc3/Kconfig @@ -223,6 +223,20 @@ config USB_DWC3_APPLE mode on these machines. Say 'Y' or 'M' if you have such device. +config USB_DWC3_APPLE_KUNIT_TEST + tristate "KUnit tests for the Apple Silicon DWC3 glue" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Test the helper with which the Apple Silicon DWC3 glue chooses + the USB2 PHY mode it selects before the core leaves reset: + device mode, never host mode, and nothing for other states. + The tests need no Apple hardware or platform support. They are + built under USB_DWC3, so UML without DMA emulation cannot run + them. + + If unsure, say N. + config USB_DWC3_GOOGLE tristate "Google Platform" help diff --git a/drivers/usb/dwc3/Makefile b/drivers/usb/dwc3/Makefile index f37971197203e..76578fc4a5da4 100644 --- a/drivers/usb/dwc3/Makefile +++ b/drivers/usb/dwc3/Makefile @@ -44,6 +44,7 @@ endif obj-$(CONFIG_USB_DWC3_AM62) += dwc3-am62.o obj-$(CONFIG_USB_DWC3_APPLE) += dwc3-apple.o +obj-$(CONFIG_USB_DWC3_APPLE_KUNIT_TEST) += dwc3-apple-state-test.o obj-$(CONFIG_USB_DWC3_OMAP) += dwc3-omap.o obj-$(CONFIG_USB_DWC3_EXYNOS) += dwc3-exynos.o obj-$(CONFIG_USB_DWC3_PCI) += dwc3-pci.o diff --git a/drivers/usb/dwc3/dwc3-apple-state-test.c b/drivers/usb/dwc3/dwc3-apple-state-test.c new file mode 100644 index 0000000000000..bf9a539a53e3f --- /dev/null +++ b/drivers/usb/dwc3/dwc3-apple-state-test.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include "dwc3-apple-state.h" + +/* Host mode is never selected before the reset; the xHCI root hub selects it. */ +static void dwc3_apple_host_left_to_xhci_test(struct kunit *test) +{ + enum phy_mode mode = PHY_MODE_USB_HOST; + + KUNIT_EXPECT_EQ(test, dwc3_apple_usb2_mode_before_reset(DWC3_APPLE_HOST, &mode), 0); + KUNIT_EXPECT_EQ(test, mode, PHY_MODE_INVALID); +} + +/* Device mode has no later selector and is selected before the reset. */ +static void dwc3_apple_device_selected_before_reset_test(struct kunit *test) +{ + enum phy_mode mode = PHY_MODE_INVALID; + + KUNIT_EXPECT_EQ(test, dwc3_apple_usb2_mode_before_reset(DWC3_APPLE_DEVICE, &mode), 0); + KUNIT_EXPECT_EQ(test, mode, PHY_MODE_USB_DEVICE); +} + +/* States that are not roles are rejected and select nothing. */ +static void dwc3_apple_non_role_test(struct kunit *test) +{ + static const enum dwc3_apple_state states[] = { + DWC3_APPLE_PROBE_PENDING, DWC3_APPLE_NO_CABLE, DWC3_APPLE_SUSPENDED, + }; + enum phy_mode mode; + int i; + + for (i = 0; i < ARRAY_SIZE(states); i++) { + mode = PHY_MODE_USB_HOST; + KUNIT_EXPECT_EQ(test, dwc3_apple_usb2_mode_before_reset(states[i], &mode), -EINVAL); + KUNIT_EXPECT_EQ(test, mode, PHY_MODE_INVALID); + } +} + +static struct kunit_case dwc3_apple_state_test_cases[] = { + KUNIT_CASE(dwc3_apple_host_left_to_xhci_test), + KUNIT_CASE(dwc3_apple_device_selected_before_reset_test), + KUNIT_CASE(dwc3_apple_non_role_test), + {} +}; + +static struct kunit_suite dwc3_apple_state_test_suite = { + .name = "dwc3_apple_state", + .test_cases = dwc3_apple_state_test_cases, +}; + +kunit_test_suite(dwc3_apple_state_test_suite); + +MODULE_DESCRIPTION("KUnit tests for the Apple Silicon DWC3 glue USB2 mode choice"); +MODULE_LICENSE("GPL"); diff --git a/drivers/usb/dwc3/dwc3-apple-state.h b/drivers/usb/dwc3/dwc3-apple-state.h new file mode 100644 index 0000000000000..89ce0b416bb40 --- /dev/null +++ b/drivers/usb/dwc3/dwc3-apple-state.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Apple Silicon DWC3 glue: controller states and the USB2 PHY role choice, + * kept in a header so that the choice can be tested without the hardware. + */ + +#ifndef __DWC3_APPLE_STATE_H +#define __DWC3_APPLE_STATE_H + +#include +#include + +/* See the comment at the top of dwc3-apple.c for how these states are used. */ +enum dwc3_apple_state { + DWC3_APPLE_PROBE_PENDING, /* Before first cable connection, dwc3_core_probe not called */ + DWC3_APPLE_NO_CABLE, /* No cable connected, dwc3 suspended after dwc3_core_exit */ + DWC3_APPLE_HOST, /* Cable connected, dwc3 in host mode */ + DWC3_APPLE_DEVICE, /* Cable connected, dwc3 in device mode */ + DWC3_APPLE_SUSPENDED, /* Fixed-hub host asleep: core exited, xhci still registered */ +}; + +/** + * dwc3_apple_usb2_mode_before_reset() - USB2 PHY mode to select before dwc3 leaves reset + * @target: the state the controller is brought up in + * @mode: set to the mode to select, or to PHY_MODE_INVALID to select none + * + * Device mode is selected before the reset is released, because nothing selects it + * later. Host mode is left to the xHCI root hub, which selects it once the core is out + * of reset: on a Type-C port the PHY is already running when the glue brings the core + * up, and switching it to host mode while the core is held in reset leaves the USB2 + * port without a connection for the whole session. + * + * Return: 0, or -EINVAL if @target is not DWC3_APPLE_HOST or DWC3_APPLE_DEVICE. + */ +static inline int dwc3_apple_usb2_mode_before_reset(enum dwc3_apple_state target, + enum phy_mode *mode) +{ + switch (target) { + case DWC3_APPLE_HOST: + *mode = PHY_MODE_INVALID; + return 0; + case DWC3_APPLE_DEVICE: + *mode = PHY_MODE_USB_DEVICE; + return 0; + default: + *mode = PHY_MODE_INVALID; + return -EINVAL; + } +} + +#endif /* __DWC3_APPLE_STATE_H */ diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 2d0662ac77f72..da01a43877473 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -16,6 +16,7 @@ #include #include +#include "dwc3-apple-state.h" #include "glue.h" /* @@ -77,14 +78,9 @@ * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. * Across system sleep such a controller goes through DWC3_APPLE_SUSPENDED: the core is exited * and initialised again around the sleep while xhci stays registered, see dwc3_apple_suspend(). + * + * enum dwc3_apple_state is defined in dwc3-apple-state.h. */ -enum dwc3_apple_state { - DWC3_APPLE_PROBE_PENDING, /* Before first cable connection, dwc3_core_probe not called */ - DWC3_APPLE_NO_CABLE, /* No cable connected, dwc3 suspended after dwc3_core_exit */ - DWC3_APPLE_HOST, /* Cable connected, dwc3 in host mode */ - DWC3_APPLE_DEVICE, /* Cable connected, dwc3 in device mode */ - DWC3_APPLE_SUSPENDED, /* Fixed-hub host asleep: core exited, xhci still registered */ -}; /** * struct dwc3_apple - Apple-specific DWC3 USB controller @@ -99,7 +95,7 @@ enum dwc3_apple_state { * @role_sw: USB role switch * @fixed_hub: A fixed hub sits on the controller's USB2 port, the controller stays in host mode * @lock: Mutex for synchronizing access - * @state: Current state of the controller, see documentation for the enum for details + * @state: Current state of the controller, see enum dwc3_apple_state and the comment above */ struct dwc3_apple { struct dwc3 dwc; @@ -265,6 +261,7 @@ static int dwc3_apple_core_init(struct dwc3_apple *appledwc) */ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_state state) { + enum phy_mode mode; int ret, ret_reset; lockdep_assert_held(&appledwc->lock); @@ -284,19 +281,16 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st * Both PHYs were looked up at probe, so device mode is also selected on the first * bring-up, before dwc3_core_probe() has looked them up itself. */ - switch (state) { - case DWC3_APPLE_HOST: - ret = 0; - break; - case DWC3_APPLE_DEVICE: - ret = phy_set_mode(appledwc->usb2_phy, PHY_MODE_USB_DEVICE); - break; - default: + ret = dwc3_apple_usb2_mode_before_reset(state, &mode); + if (ret) { /* Unreachable unless there's a bug in this driver */ - return -EINVAL; + return ret; + } + if (mode != PHY_MODE_INVALID) { + ret = phy_set_mode(appledwc->usb2_phy, mode); + if (ret) + dev_warn(appledwc->dev, "Failed to set the USB2 PHY mode, err=%d\n", ret); } - if (ret) - dev_warn(appledwc->dev, "Failed to set the USB2 PHY mode, err=%d\n", ret); ret = reset_control_deassert(appledwc->reset); if (ret) { From d04e476c887a1ab014c042a9fc6b80a81cbea7de Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 28 Sep 2026 13:02:15 -0700 Subject: [PATCH 0297/1579] arm64: dts: apple: t8103: alias the USB4 controllers for m1n1 m1n1 copies each machine's apple,tunable-rc onto the node named by usb4-N-acio, or usb4_N_rc on the 1.6.1 bootloader. The t8103 CIO nodes were only labels, so a boot image rebuilt from this tree never received the calibration and the Thunderbolt driver refused to probe. --- arch/arm64/boot/dts/apple/t8103.dtsi | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 28aedb75a2afd..e3f6d10c70ce0 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -22,6 +22,21 @@ aliases { gpu = &gpu; + /* m1n1 copies this machine's Thunderbolt calibration onto the + * node these names point at. Current m1n1 looks up usb4-N-acio + * and usb4-N-nhi; the packaged 1.6.1 bootloader looks up + * usb4_N_rc and usb4_N_nhi. A label on the node is not enough. + */ + usb4-0-acio = &usb4_0_acio; + usb4_0_rc = &usb4_0_acio; + usb4-0-nhi = &usb4_0_nhi; + usb4_0_nhi = &usb4_0_nhi; + usb4_0_pcie_adapter = &usb4_0_acio; + usb4-1-acio = &usb4_1_acio; + usb4_1_rc = &usb4_1_acio; + usb4-1-nhi = &usb4_1_nhi; + usb4_1_nhi = &usb4_1_nhi; + usb4_1_pcie_adapter = &usb4_1_acio; }; cpus { From 749ac0488440a773d1a5216566f0565539bda9b7 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 28 Sep 2026 13:10:57 -0700 Subject: [PATCH 0298/1579] arm64: dts: apple: t602x: alias the USB4 controllers for m1n1 The 14-inch and 16-inch M2 Pro/Max boards and the M2 Pro/Max mini enable the CIO nodes without the usb4-N-acio aliases m1n1 searches for, so a rebuilt boot image never receives apple,tunable-rc. --- .../arm64/boot/dts/apple/t602x-j414-j416.dtsi | 27 ++++++++++++++ .../arm64/boot/dts/apple/t602x-j474-j475.dtsi | 35 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index ed22b3854bd9f..8cb39c7653530 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -26,6 +26,33 @@ disp0_piodma = &disp0_piodma; keyboard = &keyboard; sio = &sio; + /* m1n1 writes this machine's Thunderbolt calibration onto these + * aliases. usb4-N-* is current m1n1; usb4_N_* is the 1.6.1 bootloader. + */ + usb4-0-acio = &usb4_0_acio; + usb4-0-nhi = &usb4_0_nhi; + usb4-0-pcie-adapter = &usb4_0_acio; + usb4-0-pcie = &apciec0; + usb4_0_rc = &usb4_0_acio; + usb4_0_nhi = &usb4_0_nhi; + usb4_0_pcie_adapter = &usb4_0_acio; + usb4_0_pcie = &apciec0; + usb4-1-acio = &usb4_1_acio; + usb4-1-nhi = &usb4_1_nhi; + usb4-1-pcie-adapter = &usb4_1_acio; + usb4-1-pcie = &apciec1; + usb4_1_rc = &usb4_1_acio; + usb4_1_nhi = &usb4_1_nhi; + usb4_1_pcie_adapter = &usb4_1_acio; + usb4_1_pcie = &apciec1; + usb4-2-acio = &usb4_2_acio; + usb4-2-nhi = &usb4_2_nhi; + usb4-2-pcie-adapter = &usb4_2_acio; + usb4-2-pcie = &apciec2; + usb4_2_rc = &usb4_2_acio; + usb4_2_nhi = &usb4_2_nhi; + usb4_2_pcie_adapter = &usb4_2_acio; + usb4_2_pcie = &apciec2; }; }; diff --git a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi index ee1877acba41a..b57e89bf86e5d 100644 --- a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi @@ -25,6 +25,41 @@ / { aliases { dcpext1 = &dcpext1; + /* Same Thunderbolt aliases as the 14/16-inch boards. This mini + * brings up a fourth controller. + */ + usb4-0-acio = &usb4_0_acio; + usb4-0-nhi = &usb4_0_nhi; + usb4-0-pcie-adapter = &usb4_0_acio; + usb4-0-pcie = &apciec0; + usb4_0_rc = &usb4_0_acio; + usb4_0_nhi = &usb4_0_nhi; + usb4_0_pcie_adapter = &usb4_0_acio; + usb4_0_pcie = &apciec0; + usb4-1-acio = &usb4_1_acio; + usb4-1-nhi = &usb4_1_nhi; + usb4-1-pcie-adapter = &usb4_1_acio; + usb4-1-pcie = &apciec1; + usb4_1_rc = &usb4_1_acio; + usb4_1_nhi = &usb4_1_nhi; + usb4_1_pcie_adapter = &usb4_1_acio; + usb4_1_pcie = &apciec1; + usb4-2-acio = &usb4_2_acio; + usb4-2-nhi = &usb4_2_nhi; + usb4-2-pcie-adapter = &usb4_2_acio; + usb4-2-pcie = &apciec2; + usb4_2_rc = &usb4_2_acio; + usb4_2_nhi = &usb4_2_nhi; + usb4_2_pcie_adapter = &usb4_2_acio; + usb4_2_pcie = &apciec2; + usb4-3-acio = &usb4_3_acio; + usb4-3-nhi = &usb4_3_nhi; + usb4-3-pcie-adapter = &usb4_3_acio; + usb4-3-pcie = &apciec3; + usb4_3_rc = &usb4_3_acio; + usb4_3_nhi = &usb4_3_nhi; + usb4_3_pcie_adapter = &usb4_3_acio; + usb4_3_pcie = &apciec3; }; }; From b9a6a94161917fd11674a4eef39a7e7336ba89ae Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Mon, 28 Sep 2026 13:23:29 -0700 Subject: [PATCH 0299/1579] arm64: dts: apple: t8112: alias USB4 for the 1.6.1 bootloader The base M2 trees already have usb4-N-acio. m1n1 still copies the PCIe tunnel calibration through usb4_N_pcie, and the 1.6.1 bootloader still looks up usb4_N_rc. Add those names for both ports. --- arch/arm64/boot/dts/apple/t8112-jxxx.dtsi | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi index 3d408c0095e32..7703ad8d9c0a9 100644 --- a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi @@ -21,10 +21,18 @@ usb4-0-nhi = &usb4_0_nhi; usb4-0-pcie-adapter = &usb4_0_acio; usb4-0-pcie = &apciec0; + usb4_0_rc = &usb4_0_acio; + usb4_0_nhi = &usb4_0_nhi; + usb4_0_pcie_adapter = &usb4_0_acio; + usb4_0_pcie = &apciec0; usb4-1-acio = &usb4_1_acio; usb4-1-nhi = &usb4_1_nhi; usb4-1-pcie-adapter = &usb4_1_acio; usb4-1-pcie = &apciec1; + usb4_1_rc = &usb4_1_acio; + usb4_1_nhi = &usb4_1_nhi; + usb4_1_pcie_adapter = &usb4_1_acio; + usb4_1_pcie = &apciec1; serial0 = &serial0; serial2 = &serial2; sio = &sio; From 46b9e80b117ef396fcb018dcf77cc5f98ec973d2 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Sun, 27 Sep 2026 12:40:33 +0200 Subject: [PATCH 0300/1579] phy: apple: atc: power the AUSPLL down when tearing the DP tunnel clock down Turning a USB4-tunneled display off and back on in the compositor left it in standby until a physical replug. atc_tunnel_restore() stopped PCLK1, cleared the PLL output drivers and replayed the saved descriptor registers, but never powered the AUSPLL down. The PLL kept running and ACIOPHY_AUSPLL_LOCK stayed set: the teardown logged "AUSPLL_LOCK did not clear after teardown (stat=0x8)", and the next atc_tunnel_start() preflight read +7044=00000008 and refused its own stale lock with -EBUSY. DCP then modeset into a tunnel with no pixel clock. A replug only recovered it because it resets the PHY. Clear the three PLL output drivers one at a time, then send APB command 3 through the same command/ACK handshake atc_tunnel_start() uses for commands 0 and 0x2000. This powers the AUSPLL down: the lock drops and the next preflight reads +7044=00000000. The saved descriptor registers are replayed afterwards, as before. The lock-clear poll stays as a check, since DCP retries a failed tunnel clock request quickly enough that the next request must not see a lock that is still dropping. Confirmed on hardware: M2 Pro (j416s), BenQ 1440p through a USB4 hub. At boot the preflight reads +7044=00000000 and the tunnel clock comes up (rate=0xa result=0). Disabling the output in Hyprland takes the crossbar link down with no "AUSPLL_LOCK did not clear" warning; enabling it 3s later preflights with +7044=00000000 again, gets rate=0xa result=0 and "crossbar link up", with no "AUSPLL power-down was not acked", and the picture comes back without a replug. Co-authored-by: Claude Opus 5.5 (cherry picked from commit dbaae5c0f6930405f1596deaef05198ad8d1642e) --- drivers/phy/apple/atc.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 92932e7dd956a..a8588ff008acd 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2219,22 +2219,28 @@ static void atc_tunnel_restore(struct apple_atcphy *atcphy) lockdep_assert_held(&atcphy->lock); if (!atcphy->tunnel_saved) return; - /* Stop our output before restoring the unused PLL descriptor. */ + /* + * Clearing the PLL output drivers alone leaves the AUSPLL running, + * with ACIOPHY_AUSPLL_LOCK still set. Clear the three drivers, then + * power the PLL down with APB command 3, after which the lock drops. + */ core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); - core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(2)); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(4)); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(6)); + ret = atc_tunnel_command(atcphy, 3); + if (ret) + dev_warn(atcphy->dev, "USB4 tunnel clock: AUSPLL power-down was not acked\n"); + /* Then restore the unused PLL descriptor. */ for (i = ARRAY_SIZE(atc_tunnel_regs) - 1; i >= 0; i--) core_mask32(atcphy, atc_tunnel_regs[i].reg, atc_tunnel_regs[i].mask, atcphy->tunnel_saved_regs[i] & atc_tunnel_regs[i].mask); /* * atc_tunnel_start_t602x()'s preflight refuses to proceed while - * ACIOPHY_AUSPLL_LOCK is still set, to avoid clobbering a PLL a - * concurrent user actually has locked. Immediately after tearing our - * own lock down, that same bit can still read set for a brief window - * before the lock detector catches up -- and DCP firmware retries a - * failed tunnel clock request roughly once a second, which is faster - * than that window on a hot retrain loop. Wait for it to clear here - * so a request coming in right after this returns sees a genuinely - * idle PLL instead of refusing itself forever. + * ACIOPHY_AUSPLL_LOCK is set, to avoid clobbering a PLL a concurrent + * user actually has locked. DCP firmware retries a failed tunnel + * clock request quickly, so confirm the lock has dropped after the + * power-down before a new request can come in. */ ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, value, !(value & ACIOPHY_AUSPLL_LOCK), 10, 10000); From 508384b75fa147f3aa6ed78885235367d82adb8d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Tue, 29 Sep 2026 13:23:51 -0700 Subject: [PATCH 0301/1579] dt-bindings: thunderbolt: apple: describe the ACIO PCIe adapter The t8103 CIO nodes carry a third reg entry, reg-names "pcie-adapter", and apple,tunable-pcie-adapter, which the binding never described. Both controllers on j313 therefore failed validation four times over: reg and reg-names "too long", and the tunable rejected as undocumented. The third entry is t8103-only -- t8112 has two -- so reg and reg-names take minItems: 2 and the adapter is described as the optional third. While here, give thunderbolt-switch a type. It was declared as bare "true", which dtschema reports as a missing type definition; every user in tree spells it as a valueless boolean. Validated against a built t8103-j313.dtb: the eight acio errors and the thunderbolt-switch warning are gone, with no new diagnostics. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- .../thunderbolt/apple,t8103-usb4-acio.yaml | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml index c83e00f446314..7787adb6805cc 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml @@ -40,14 +40,21 @@ properties: - const: apple,t8103-usb4-acio reg: + minItems: 2 items: - description: Root complex and co-processor control registers - description: SRAM used for communication with the co-processor + - description: | + PCIe adapter configuration space of the USB4 router. Present where + the root complex exposes the adapter separately, so that the tunnel + can be configured before a PCIe-C link is brought up. reg-names: + minItems: 2 items: - const: rc - const: sram + - const: pcie-adapter mboxes: maxItems: 1 @@ -66,7 +73,11 @@ properties: apple,pcie-tunnel is present, the fourth domain powers its PCIe-C controller and is enabled only after the USB4 PCIe tunnel is active. - thunderbolt-switch: true + thunderbolt-switch: + type: boolean + description: + Marks this root complex as the USB4/Thunderbolt switch that the Type-C + PD controller hands cable information to, so the Type-C mux can find it. apple,pcie-tunnel: $ref: /schemas/types.yaml#/definitions/phandle @@ -86,6 +97,18 @@ properties: Tree, converted by the bootloader and used to configure the root complex after the co-processor has booted. + apple,tunable-pcie-adapter: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask of bits to clear + - description: Bits to set + description: | + As apple,tunable-rc, but applied to the PCIe adapter configuration space + named by the pcie-adapter reg entry. Only meaningful where that entry is + present. + '#address-cells': const: 1 From 7cc32163f5f1d7e8208cb139eaa0f1c1ac898d05 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:40:24 -0700 Subject: [PATCH 0302/1579] thunderbolt: apple: drive t8103 Thunderbolt displays by default A display behind a Thunderbolt dock on t8103 stayed dark unless thunderbolt_apple.dp_display=1 was passed. The path it enables is the one this series has been tested on: on a 13" M1 MacBook Pro, CalDigit TS3 Plus, CalDigit TS4 and Kensington SD5560T docks drove 4K displays through hotplug, both ports and suspend-to-idle, and the aurora test builds have shipped it on by default. Default dp_display to true. Passing 0 still leaves the displays dark. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- Documentation/admin-guide/kernel-parameters.txt | 8 ++++---- drivers/thunderbolt/apple.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index ea78b77f79abd..34fa032298cef 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7613,10 +7613,10 @@ Kernel parameters [HW,ARM64] Format: Drive displays behind Thunderbolt DisplayPort tunnels - on Apple silicon (experimental, t8103 only): route a - display pipeline to the host router's DP IN adapter - whenever the connection manager sets up a DP tunnel. - Default: false + on Apple t8103 (M1): route a display pipeline to the + host router's DP IN adapter whenever the connection + manager sets up a DP tunnel. Set to 0 to leave displays + behind a dock dark. Default: true thunderbolt_apple.dpin_aux= [HW,ARM64] diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 4e168a8b414e9..84c3fc51505b0 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -207,9 +207,9 @@ struct apple_cio { #define APPLE_DPIN_CTRL_INACTIVE BIT(0) #define APPLE_DPIN_ACK_INACTIVE BIT(0) -static bool dp_display; +static bool dp_display = true; module_param(dp_display, bool, 0444); -MODULE_PARM_DESC(dp_display, "Drive displays behind Thunderbolt DP tunnels (experimental, t8103)"); +MODULE_PARM_DESC(dp_display, "Drive displays behind Thunderbolt DP tunnels on t8103 (default: true)"); /* DPTX_INACTIVE handshake: request (in)active, wait for the ACK */ struct apple_dpin_poll { From 3fcb3823b49efe7d31ea36fc42796daa02b387c2 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:41:15 -0700 Subject: [PATCH 0303/1579] soc: apple: dp-tunnel: drive Thunderbolt displays on every M2 Pro and M2 Max laptop The T602X display tunnel path ran only on the machine it was brought up on, the 16" M2 Pro (j416s): the DP IN handshake, the tunnel pixel clock, the longer link timeouts, the pipeline preference and the right-hand port's bandwidth grant were all keyed to "apple,j416s". The 14" M2 Pro (j414s), 14" M2 Max (j414c) and 16" M2 Max (j416c) use the same ATC PHYs ("apple,t6020-atcphy") and include the same t602x-j414-j416.dtsi, which carries the crossbar's typecN-usb4 tunnel routes and the USB-C port layout this code relies on. On those machines a display behind a dock stayed dark. Add apple_dp_tunnel_t602x() to the shared DP tunnel header, matching the four laptops, and use it for the ten j416s checks in appledrm, the ATC PHY and the Thunderbolt code. The M2 Pro and M2 Max desktops (j474s, j475c, j475d) are left out: their device tree has no crossbar tunnel routes yet. So far only j416s has run this path. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- .../admin-guide/kernel-parameters.txt | 10 ++++++---- drivers/gpu/drm/apple/dcp.c | 11 +++++------ drivers/gpu/drm/apple/iomfb.c | 3 ++- drivers/phy/apple/atc.c | 5 ++--- drivers/thunderbolt/apple.c | 4 ++-- drivers/thunderbolt/tunnel.c | 18 ++++++++++-------- include/linux/soc/apple/dp-tunnel.h | 16 ++++++++++++++++ 7 files changed, 43 insertions(+), 24 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 34fa032298cef..c87bd53bde7a3 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7605,8 +7605,8 @@ Kernel parameters [HW,ARM64] Format: Enable diagnostic counter 0 on the DP video path's - host and downstream ingress hops on the Apple j416s - right-hand USB-C route. Read counters through Thunderbolt + host and downstream ingress hops on the right-hand + USB-C route of the Apple M2 Pro and M2 Max laptops. Read counters through Thunderbolt debugfs. This is a load-time setting. Default: false. thunderbolt_apple.dp_display= @@ -7622,7 +7622,8 @@ Kernel parameters [HW,ARM64] Format: Pulse the experimental DP IN analog start control on - Apple j416s at the next DP tunnel activation. Writes to + the Apple M2 Pro and M2 Max laptops at the next DP + tunnel activation. Writes to the module parameter do not access a live controller. Default: false (leave analog registers alone until DPRX negotiation finishes or times out). @@ -7630,7 +7631,8 @@ Kernel parameters thunderbolt_apple.dpin_mode_value= [HW,ARM64] Format: (0-15) - Experimental DP IN MODE_A/MODE_B value on Apple j416s. + Experimental DP IN MODE_A/MODE_B value on the Apple + M2 Pro and M2 Max laptops. Sampled for each adapter activation; values above 15 make activation fail. Default: 8. The value is empirical and does not select the negotiated DisplayPort rate. diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a32d60bddf124..dba0e22ebf955 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -660,8 +660,8 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, if (!dcp_typec_route_available(candidate)) continue; score = dcp_typec_route_score(candidate); - /* j416s dcpext0 completes tunneled link training; prefer it. */ - if (of_machine_is_compatible("apple,j416s") && + /* dcpext0 completes tunneled link training here; prefer it. */ + if (apple_dp_tunnel_t602x() && !candidate->dcp->fixed_phy && score < UINT_MAX - 100) score += 100; if (score < best_score) { @@ -1394,8 +1394,7 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) } dcp->dptxport[port].connected = true; if (dcp_is_typec_output(dcp)) { - if (dcp_is_usb4_output(dcp) && - of_machine_is_compatible("apple,j416s")) + if (dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x()) ret = dptxport_set_hpd_timeout(dcp->dptxport[port].service, true, 8000); else @@ -1410,7 +1409,7 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) } mutex_unlock(&dcp->hpd_mutex); - timeout = dcp_is_usb4_output(dcp) && of_machine_is_compatible("apple,j416s") ? + timeout = dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x() ? DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT; ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, timeout); @@ -1540,7 +1539,7 @@ static void dcp_typec_reconnect_work(struct work_struct *work) } if (++dcp->typec_reconnect_tries < - (dcp_is_usb4_output(dcp) && of_machine_is_compatible("apple,j416s") ? + (dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x() ? 1 : DPTX_RECONNECT_RETRIES)) { mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, DPTX_RECONNECT_DELAY); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5a80cc57735e4..5536bd15cf8cd 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -302,7 +303,7 @@ void dcp_hotplug(struct work_struct *work) * flush, or the CRTC gets no signal. */ if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connector->connected && - !(dcp_is_usb4_output(dcp) && of_machine_is_compatible("apple,j416s"))) { + !(dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x())) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index a8588ff008acd..5213a6e5239d8 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2678,10 +2678,9 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) return -EINVAL; atcphy = phy_get_drvdata(phy); /* Keep each supported SoC on its qualified clock sequence. */ - if (!of_machine_is_compatible("apple,t8103") && - !of_machine_is_compatible("apple,j416s")) + if (!of_machine_is_compatible("apple,t8103") && !apple_dp_tunnel_t602x()) return -EOPNOTSUPP; - if (of_machine_is_compatible("apple,j416s") && + if (apple_dp_tunnel_t602x() && (!of_device_is_compatible(atcphy->np, "apple,t6020-atcphy") || !apple_atc_is_typec_core(atcphy->res.core->start) || resource_size(atcphy->res.core) < 0x7048)) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 84c3fc51505b0..467bb4579717a 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -309,7 +309,7 @@ static int apple_dpin_set_active_t8103(struct apple_cio *acio, void __iomem *dpi static int apple_dpin_set_active(struct apple_cio *acio, void __iomem *regs, unsigned int idx, bool active) { - if (of_machine_is_compatible("apple,j416s")) + if (apple_dp_tunnel_t602x()) return apple_dpin_set_active_t602x(acio, regs, idx, active); return apple_dpin_set_active_t8103(acio, regs, idx, active); } @@ -1599,7 +1599,7 @@ static int apple_nhi_probe(struct platform_device *pdev) anhi->ops = apple_nhi_ops; if (dp_display && acio->dp_wq && of_machine_is_compatible("apple,t8103")) anhi->ops.dp_tunnel_changed = apple_nhi_dp_tunnel_changed; - if (acio->dp_wq && of_machine_is_compatible("apple,j416s")) { + if (acio->dp_wq && apple_dp_tunnel_t602x()) { anhi->ops.dp_tunnel_pre_activate = apple_nhi_dp_tunnel_pre_activate; anhi->ops.dp_tunnel_post_activate = apple_nhi_dp_tunnel_post_activate; anhi->ops.dp_tunnel_deactivate = apple_nhi_dp_tunnel_deactivate; diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index 988331a249f7f..f3a05c6c2a1a3 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "tunnel.h" #include "tb.h" @@ -104,7 +105,7 @@ MODULE_PARM_DESC(bw_alloc_mode, static bool dp_video_counter; module_param(dp_video_counter, bool, 0444); MODULE_PARM_DESC(dp_video_counter, - "diagnostic: count packets on the DP video path's DP IN hop and its downstream (hub-side) hop (Apple j416s right ACIO route only; read via debugfs port counters); default: false"); + "diagnostic: count packets on the DP video path's DP IN hop and its downstream (hub-side) hop (Apple M2 Pro/Max laptop right ACIO route only; read via debugfs port counters); default: false"); static void tb_dp_dump_apple(struct tb_tunnel *tunnel); static int tb_apple_nhi_typec_index(struct tb_nhi *nhi); @@ -956,16 +957,17 @@ static int tb_dp_xchg_caps(struct tb_tunnel *tunnel) in->cap_adap + DP_REMOTE_CAP, 1); } -/* Apple j416s right-hand ACIO DP IN adapter only; shared by the diagnostic - * packet counter and the bandwidth-grant workaround below. +/* Right-hand ACIO DP IN adapter of the Apple M2 Pro and M2 Max laptops only; + * shared by the diagnostic packet counter and the bandwidth-grant workaround + * below. */ -static bool tb_dp_is_apple_j416s_right_dpin(const struct tb_port *in) +static bool tb_dp_is_apple_t602x_right_dpin(const struct tb_port *in) { if (!tb_port_is_dpin(in)) return false; if (!in->sw->tb || !tb_nhi_is_apple(in->sw->tb->nhi)) return false; - if (!of_machine_is_compatible("apple,j416s")) + if (!apple_dp_tunnel_t602x()) return false; /* Right-hand USB-C ports only ("f01f" NHI); see tb_apple_nhi_typec_index(). */ @@ -974,7 +976,7 @@ static bool tb_dp_is_apple_j416s_right_dpin(const struct tb_port *in) static bool tb_dp_apple_dpin_needs_bw_grant(const struct tb_port *in) { - return tb_dp_is_apple_j416s_right_dpin(in); + return tb_dp_is_apple_t602x_right_dpin(in); } static int tb_dp_bandwidth_alloc_mode_enable(struct tb_tunnel *tunnel) @@ -1074,7 +1076,7 @@ static int tb_dp_bandwidth_alloc_mode_enable(struct tb_tunnel *tunnel) /* * Initial allocation should be 0 according the spec, which relies * on the DP IN adapter later raising it via a bandwidth request - * notification. The Apple j416s right-hand ACIO DP IN adapter has + * notification. The Apple M2 Pro right-hand ACIO DP IN adapter has * never been observed to generate that notification: link training * and a full DCP frame complete with this field (DP_STATUS * allocated-bandwidth) still reading 0 and no picture. @@ -1734,7 +1736,7 @@ static bool tb_dp_video_counter_wanted(const struct tb_path *path) if (!dp_video_counter || !path->path_length) return false; - return tb_dp_is_apple_j416s_right_dpin(path->hops[0].in_port); + return tb_dp_is_apple_t602x_right_dpin(path->hops[0].in_port); } static int tb_dp_init_video_path(struct tb_path *path, bool pm_support) diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index 2767c16b127ff..8117ba0cea949 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -8,6 +8,7 @@ #ifndef _LINUX_SOC_APPLE_DP_TUNNEL_H_ #define _LINUX_SOC_APPLE_DP_TUNNEL_H_ +#include #include struct device_node; @@ -40,4 +41,19 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); int apple_dpxbar_link_down(struct mux_control *mux); int apple_dpxbar_link_up(struct mux_control *mux); +/* + * The M2 Pro and M2 Max laptops. Their shared device tree routes the display + * crossbar to the Thunderbolt DP IN adapters, and they take the T602X tunnel + * path: DP IN handshake, tunnel pixel clock, and the longer link timeouts. + * The M2 Pro and M2 Max desktops have no such routes in their device tree. + */ +static inline bool apple_dp_tunnel_t602x(void) +{ + static const char *const machines[] = { + "apple,j414s", "apple,j414c", "apple,j416s", "apple,j416c", NULL, + }; + + return of_machine_compatible_match(machines); +} + #endif From 9d5a05a8174052bd0fe09e173670131a62c9e6c7 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Sun, 27 Sep 2026 14:29:45 +0200 Subject: [PATCH 0304/1579] drm/apple: afk: reuse torn-down service slots DCP tears its EPIC services down when an external display disconnects and announces them again on fresh channels when it reconnects; it never reuses a channel. On the j416s DPAVSERV endpoint (0x28), every compositor off/on or replug of the display tears down dcpav-service-epic (the service EDID is read through) and dcpdp-service-epic, and announces both again: channels 1/3 at boot, then 5/7, 9/11 and so on. afk_recv_handle_init() handed out slots append-only (ch_idx = num_channels++), and teardown only disables a service, so its slot was never given back. After boot plus seven reconnects all AFK_MAX_CHANNEL (16) slots were used, and the next reconnect failed: AFK[ep:28]: too many enabled services! AFK[ep:28]: expected announce but got 0xc0 on channel 33 The re-announced dcpav-service-epic then had no slot, so dcpavserv stayed disabled and dcp_get_modes() skipped the EDID read without logging anything. The connector's EDID stayed empty for the rest of the boot, and a physical replug did not bring it back, since the endpoint is never restarted. Reuse a slot whose service was torn down, is disabled and has no command pending. Pending commands keep their buffers in the slot until the firmware acks them, so such a slot is left alone. The slot lock is initialised once, when the slot is first used. Add debug messages for teardown and slot reuse, since a normal teardown was not logged at all. Confirmed on hardware: M2 Pro (j416s), BenQ 1440p through a USB4 hub, fresh boot, 10 compositor off/on toggles. Each cycle logs teardown of dcpdp-service-epic and dcpav-service-epic, then "reusing slot 0" and "reusing slot 1" as the two services come back, with channels reaching 43 and no "too many enabled services". The EDID read back as 256 bytes after every toggle and Hyprland kept identifying the display. Co-authored-by: Claude Opus 5.5 Signed-off-by: Oliver Lukschander (cherry picked from commit 6fdc8fe1298fb96e7f3306cf823fda64d2d18795) Signed-off-by: RJ Skerry-Ryan --- drivers/gpu/drm/apple/afk.c | 54 +++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 9124f9e46647b..247d7d60c0d2c 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -238,12 +238,49 @@ static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep * return NULL; } +/* + * DCP announces a service again on a fresh channel each time a display + * reconnects; it never reuses the old channel. Reuse the slot of a + * service that was torn down and disabled and has no command pending, + * otherwise repeated reconnects fill the table and later services, such + * as the one EDID is read through, cannot be registered. + */ +static int afk_alloc_service_slot(struct apple_dcp_afkep *ep) +{ + struct apple_epic_service *service; + unsigned long flags; + bool idle; + u32 i; + + for (i = 0; i < ep->num_channels; i++) { + service = &ep->services[i]; + if (service->enabled || !service->torndown) + continue; + spin_lock_irqsave(&service->lock, flags); + idle = bitmap_empty(service->cmd_map, MAX_PENDING_CMDS); + spin_unlock_irqrestore(&service->lock, flags); + if (idle) { + dev_dbg(ep->dcp->dev, + "AFK[ep:%02x]: reusing slot %u of torn-down channel %u\n", + ep->endpoint, i, service->channel); + return i; + } + } + + if (ep->num_channels >= AFK_MAX_CHANNEL) + return -ENOSPC; + + spin_lock_init(&ep->services[ep->num_channels].lock); + return ep->num_channels++; +} + static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *payload, size_t payload_size) { char name[32]; s64 epic_unit = -1; u32 ch_idx; + int slot; const char *service_name = name; const char *epic_name = NULL, *epic_class = NULL; const struct apple_epic_service_ops *ops; @@ -259,12 +296,6 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, return; } - if (ep->num_channels >= AFK_MAX_CHANNEL) { - dev_err(ep->dcp->dev, "AFK[ep:%02x]: too many enabled services!\n", - ep->endpoint); - return; - } - strscpy(name, payload, sizeof(name)); /* @@ -305,8 +336,13 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, goto free; } - ch_idx = ep->num_channels++; - spin_lock_init(&ep->services[ch_idx].lock); + slot = afk_alloc_service_slot(ep); + if (slot < 0) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: too many enabled services!\n", + ep->endpoint); + goto free; + } + ch_idx = slot; ep->services[ch_idx].enabled = true; ep->services[ch_idx].torndown = false; ep->services[ch_idx].ops = ops; @@ -337,6 +373,8 @@ static void afk_recv_handle_teardown(struct apple_dcp_afkep *ep, u32 channel) return; } + dev_dbg(ep->dcp->dev, "AFK[ep:%02x]: teardown of %s on channel %u\n", + ep->endpoint, service->ops->name, channel); afk_remove_service_debugfs(service); // TODO: think through what locking is necessary From 830aa7893679d81f38829e91546531f626eda744 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:25:35 +1000 Subject: [PATCH 0305/1579] soc: apple: mailbox: Keep the send-empty IRQ enable balanced The send-empty interrupt is requested with IRQF_NO_AUTOEN. apple_mbox_send() enables it when the A2I FIFO is full and it has to wait, and the interrupt handler disables it again when it fires. When the wait times out or is interrupted by a signal, apple_mbox_send() returns with the interrupt still enabled, so the next send that has to wait calls enable_irq() on a line that is already enabled. The IRQ core warns about the unbalanced enable, and from then on the enable state no longer tracks whether a sender is waiting. On T8140 this shows up as "Unbalanced enable for IRQ" warnings from the SEP hwrng thread, whose sends can wait longer than the 500 ms timeout. Track whether the interrupt is unmasked in struct apple_mbox, protected by tx_lock. The sender enables it only while it is masked, the handler disables it only while it is unmasked, and a sender whose wait failed disables it unless a late interrupt already has. Disabling it unconditionally on the error path would not work: the handler may already be waiting for tx_lock and would then disable it a second time. Fixes: 6e1457fcad3f ("soc: apple: mailbox: Add ASC/M3 mailbox driver") Assisted-by: LLM checkpatch sparse Co-authored-by: Claude Fable 5.1 Signed-off-by: Ryan Murray (cherry picked from commit 7a155e383d2d8e70fd76075ffb7427f4c7a3c326) Signed-off-by: RJ Skerry-Ryan --- drivers/soc/apple/mailbox.c | 24 ++++++++++++++++++------ include/linux/soc/apple/mailbox.h | 3 +++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 330015eea2ebe..b08f78e238711 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -137,17 +137,26 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, writel_relaxed(mbox->hw->irq_bit_send_empty, mbox->regs + mbox->hw->irq_ack); } - enable_irq(mbox->irq_send_empty); reinit_completion(&mbox->tx_empty); + if (!mbox->tx_irq_unmasked) { + mbox->tx_irq_unmasked = true; + enable_irq(mbox->irq_send_empty); + } spin_unlock_irqrestore(&mbox->tx_lock, flags); t = wait_for_completion_interruptible_timeout( &mbox->tx_empty, msecs_to_jiffies(APPLE_MBOX_TX_TIMEOUT)); - if (t < 0) - return t; - else if (t == 0) - return -ETIMEDOUT; + if (t <= 0) { + /* A late IRQ may already have masked this under tx_lock. */ + spin_lock_irqsave(&mbox->tx_lock, flags); + if (mbox->tx_irq_unmasked) { + disable_irq_nosync(mbox->irq_send_empty); + mbox->tx_irq_unmasked = false; + } + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return t < 0 ? t : -ETIMEDOUT; + } spin_lock_irqsave(&mbox->tx_lock, flags); mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); @@ -175,7 +184,10 @@ static irqreturn_t apple_mbox_send_empty_irq(int irq, void *data) * it at the main controller again. */ spin_lock(&mbox->tx_lock); - disable_irq_nosync(mbox->irq_send_empty); + if (mbox->tx_irq_unmasked) { + disable_irq_nosync(mbox->irq_send_empty); + mbox->tx_irq_unmasked = false; + } complete(&mbox->tx_empty); spin_unlock(&mbox->tx_lock); diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index f73a8913da951..9cfbbac411b8e 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -34,6 +34,9 @@ struct apple_mbox { /** Receive callback for incoming messages */ void (*rx)(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie); void *cookie; + + /* Protected by tx_lock; send-error and IRQ paths mask this only once. */ + bool tx_irq_unmasked; }; struct apple_mbox *apple_mbox_get(struct device *dev, int index); From 1fc9ea9939058bc1307565f036506da30dd62908 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 25 Sep 2026 22:42:27 +1000 Subject: [PATCH 0306/1579] soc: apple: rtkit: Serialize receiver ownership through teardown apple_rtkit_free() and apple_rtkit_reinit() stopped the mailbox and destroyed or flushed the workqueue, but left the mailbox's rx callback and cookie pointing at the instance the whole time. A message popped by a concurrent apple_rtkit_poll() or a work item queued just before the stop could still reach the consumer's callbacks while its state was being torn down, and apple_rtkit_init() left a dangling receiver behind when the workqueue allocation failed. Make the receiver an explicit ownership: claim the mailbox rx callback under rx_lock once the workqueue exists, detach it (with the interrupt already disabled by apple_mbox_stop()) before draining the queue, drop messages that arrive or were queued once teardown has started, and release the cookie only after all host state is gone. apple_rtkit_reinit() re-arms the receiver under the lock and detaches again if the mailbox fails to start. A mailbox poll with no receiver attached returns without touching the FIFO; the entries are drained once a receiver is attached again. Claiming a mailbox that already has a receiver keeps the previous behaviour of taking it over, now with a warning, so existing consumers that never free their instance can still rebind. Assisted-by: LLM checkpatch sparse Co-authored-by: Claude Fable 5.1 Signed-off-by: Ryan Murray (cherry picked from commit 692a3c6498ab388d40faaeedfcdf76da0c2273d0) Signed-off-by: RJ Skerry-Ryan --- drivers/soc/apple/mailbox.c | 9 +++- drivers/soc/apple/rtkit-internal.h | 1 + drivers/soc/apple/rtkit.c | 87 ++++++++++++++++++++++++++---- 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index b08f78e238711..4837670501e35 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -198,8 +198,15 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) { struct apple_mbox_msg msg; int ret = 0; + u32 mbox_ctrl; - u32 mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); + /* + * A detached receiver (RTKit teardown in progress) leaves firmware + * traffic in the FIFO; it is drained again once a receiver is attached. + */ + if (!mbox->rx) + return 0; + mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); while (!(mbox_ctrl & mbox->hw->control_empty)) { msg.msg0 = readq_relaxed(mbox->regs + mbox->hw->i2a_recv0); diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index c82065a8bf7b0..d5b45b3865381 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -39,6 +39,7 @@ struct apple_rtkit { unsigned int iop_power_state; unsigned int ap_power_state; bool crashed; + bool shutting_down; DECLARE_BITMAP(endpoints, APPLE_RTKIT_MAX_ENDPOINTS); diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index ef0c399ea8523..19ed2a0e8520e 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -531,6 +531,9 @@ static void apple_rtkit_rx_work(struct work_struct *work) container_of(work, struct apple_rtkit_rx_work, work); struct apple_rtkit *rtk = rtk_work->rtk; + if (READ_ONCE(rtk->shutting_down)) + goto out; + switch (rtk_work->ep) { case APPLE_RTKIT_EP_MGMT: apple_rtkit_management_rx(rtk, rtk_work->msg); @@ -563,6 +566,7 @@ static void apple_rtkit_rx_work(struct work_struct *work) rtk_work->ep, rtk_work->msg); } +out: kfree(rtk_work); } @@ -573,6 +577,9 @@ static void apple_rtkit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, struct apple_rtkit_rx_work *work; u8 ep = msg.msg1; + if (READ_ONCE(rtk->shutting_down)) + return; + /* * The message was read from a MMIO FIFO and we have to make * sure all reads from buffers sent with that message happen @@ -663,6 +670,53 @@ int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) } EXPORT_SYMBOL_GPL(apple_rtkit_start_ep); +static void apple_rtkit_claim_rx(struct apple_rtkit *rtk) +{ + struct apple_mbox *mbox = rtk->mbox; + unsigned long flags; + + spin_lock_irqsave(&mbox->rx_lock, flags); + if (mbox->rx || mbox->cookie) + dev_warn(rtk->dev, + "RTKit: mailbox receiver already claimed, taking it over\n"); + mbox->cookie = rtk; + mbox->rx = apple_rtkit_rx; + spin_unlock_irqrestore(&mbox->rx_lock, flags); +} + +static void apple_rtkit_detach_rx(struct apple_rtkit *rtk) +{ + unsigned long flags; + + WRITE_ONCE(rtk->shutting_down, true); + apple_mbox_stop(rtk->mbox); + /* + * IRQ receive and explicit polling both hold rx_lock. Detach before + * draining the queue, but keep the cookie as an ownership reservation + * until all host/buffer teardown is complete. + */ + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + if (rtk->mbox->cookie == rtk) + rtk->mbox->rx = NULL; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); +} + +static void apple_rtkit_stop_rx(struct apple_rtkit *rtk) +{ + apple_rtkit_detach_rx(rtk); + destroy_workqueue(rtk->wq); +} + +static void apple_rtkit_release_rx(struct apple_rtkit *rtk) +{ + unsigned long flags; + + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + if (rtk->mbox->cookie == rtk) + rtk->mbox->cookie = NULL; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); +} + struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops) @@ -698,9 +752,6 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, goto free_rtk; } - rtk->mbox->rx = apple_rtkit_rx; - rtk->mbox->cookie = rtk; - rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_HIGHPRI | WQ_MEM_RECLAIM, dev_name(rtk->dev)); if (!rtk->wq) { @@ -708,14 +759,17 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, goto free_rtk; } + apple_rtkit_claim_rx(rtk); + ret = apple_mbox_start(rtk->mbox); if (ret) - goto destroy_wq; + goto stop_rx; return rtk; -destroy_wq: - destroy_workqueue(rtk->wq); +stop_rx: + apple_rtkit_stop_rx(rtk); + apple_rtkit_release_rx(rtk); free_rtk: kfree(rtk); return ERR_PTR(ret); @@ -738,8 +792,11 @@ static int apple_rtkit_wait_for_completion(struct completion *c) int apple_rtkit_reinit(struct apple_rtkit *rtk) { + unsigned long flags; + int ret; + /* make sure we don't handle any messages while reinitializing */ - apple_mbox_stop(rtk->mbox); + apple_rtkit_detach_rx(rtk); flush_workqueue(rtk->wq); apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); @@ -764,7 +821,17 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_OFF; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_OFF; - return apple_mbox_start(rtk->mbox); + spin_lock_irqsave(&rtk->mbox->rx_lock, flags); + WRITE_ONCE(rtk->shutting_down, false); + rtk->mbox->rx = apple_rtkit_rx; + spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + + ret = apple_mbox_start(rtk->mbox); + if (ret) { + apple_rtkit_detach_rx(rtk); + flush_workqueue(rtk->wq); + } + return ret; } EXPORT_SYMBOL_GPL(apple_rtkit_reinit); @@ -943,8 +1010,7 @@ EXPORT_SYMBOL_GPL(apple_rtkit_wake); void apple_rtkit_free(struct apple_rtkit *rtk) { - apple_mbox_stop(rtk->mbox); - destroy_workqueue(rtk->wq); + apple_rtkit_stop_rx(rtk); apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer); @@ -952,6 +1018,7 @@ void apple_rtkit_free(struct apple_rtkit *rtk) apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer); kfree(rtk->syslog_msg_buffer); + apple_rtkit_release_rx(rtk); kfree(rtk); } EXPORT_SYMBOL_GPL(apple_rtkit_free); From 70dfd8f93d3cac6d6ac7ad26909ca6378fd6a2b4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:26:23 +1000 Subject: [PATCH 0307/1579] iommu/apple-dart: Check devm_memremap() for an error pointer apple_dart_hw_map_locked_ttbr() maps the L1 table of a locked DART stream with devm_memremap() and checks the result for NULL, but devm_memremap() never returns NULL: it returns ERR_PTR(-ENOMEM) or ERR_PTR(-ENXIO) on failure. A failed mapping was therefore stored as the table pointer, and apple_dart_hw_sync_locked() then wrote the page-table entries through it and apple_dart_hw_unmap_locked_ttbr() passed it to memset(). The NULL check was added together with the locked-DART page-table code by commit 0d4ec0744510 ("iommu/dart: Add iommu_ops for locked DARTs"), and became reachable once locked DARTs were allowed to probe. Check the result with IS_ERR() and return the error, so that no error pointer is ever stored. Fixes: 85322ebcc7e8 ("iommu/dart: Support locked DARTs") Assisted-by: LLM checkpatch sparse Signed-off-by: Ryan Murray (cherry picked from commit 162b64627bbcb2fa0ae83cbb6cbf86825ced3e33) Signed-off-by: RJ Skerry-Ryan --- drivers/iommu/apple-dart.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index aa3c82ebd72ff..cde17c5f6c8b8 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -478,8 +478,8 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) l1_tbl = devm_memremap(dart->dev, phys, dart->pgsize, MEMREMAP_WB); - if (!l1_tbl) - return -ENOMEM; + if (IS_ERR(l1_tbl)) + return PTR_ERR(l1_tbl); dart->locked_ttbr[sid][idx] = l1_tbl; } From 42a176b951d90e746e33d27d6eb06a1773579afe Mon Sep 17 00:00:00 2001 From: Haoxiang Li Date: Fri, 3 Jul 2026 14:20:49 +0800 Subject: [PATCH 0308/1579] cpufreq: apple-soc: Fix OPP table cleanup commit d87cb889dc7ab1f2deecadf2a5e9023184bd7900 upstream. apple_soc_cpufreq_init() adds OPP tables from firmware, but some failure paths do not remove them. The driver also uses dev_pm_opp_remove_all_dynamic(), which is not the right cleanup helper for OPP tables loaded from firmware. Use the cpumask OPP helper after the policy CPU mask has been populated. Pair it with the matching cpumask remove helper on failure paths and in apple_soc_cpufreq_exit(). This also removes the separate dev_pm_opp_set_sharing_cpus() call, as the cpumask helper loads the DT OPP tables for all CPUs in the policy. Fixes: 6286bbb40576 ("cpufreq: apple-soc: Add new driver to control Apple SoC CPU P-states") Cc: stable@vger.kernel.org Signed-off-by: Haoxiang Li Signed-off-by: Viresh Kumar Signed-off-by: Ryan Murray (cherry picked from commit eb346c1a25d813a1411a34fafc7d4bdee418e0d4) Signed-off-by: RJ Skerry-Ryan --- drivers/cpufreq/apple-soc-cpufreq.c | 36 +++++++++++------------------ 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 60935032919d2..d416088bf28a5 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -251,21 +251,19 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) return -ENODEV; } - ret = dev_pm_opp_of_add_table(cpu_dev); - if (ret < 0) { - dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret); - return ret; - } + priv = kzalloc_obj(*priv); + if (!priv) + return -ENOMEM; ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info); if (ret) { dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret); - return ret; + goto out_free_priv; } - ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); - if (ret) { - dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret); + ret = dev_pm_opp_of_cpumask_add_table(policy->cpus); + if (ret < 0) { + dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret); goto out_iounmap; } @@ -273,19 +271,13 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) if (ret <= 0) { dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n"); ret = -EPROBE_DEFER; - goto out_free_opp; - } - - priv = kzalloc_obj(*priv); - if (!priv) { - ret = -ENOMEM; - goto out_free_opp; + goto out_free_table; } ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); if (ret) { dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); - goto out_free_priv; + goto out_free_table; } /* Get OPP levels (p-state indexes) and stash them in driver_data */ @@ -320,12 +312,12 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) out_free_cpufreq_table: dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); -out_free_priv: - kfree(priv); -out_free_opp: - dev_pm_opp_remove_all_dynamic(cpu_dev); +out_free_table: + dev_pm_opp_of_cpumask_remove_table(policy->cpus); out_iounmap: iounmap(reg_base); +out_free_priv: + kfree(priv); return ret; } @@ -334,7 +326,7 @@ static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) struct apple_cpu_priv *priv = policy->driver_data; dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); - dev_pm_opp_remove_all_dynamic(priv->cpu_dev); + dev_pm_opp_of_cpumask_remove_table(policy->cpus); iounmap(priv->reg_base); kfree(priv); } From 7810f444ae5a403773baf8d4f460e29cb2375cf5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 01:00:04 +1000 Subject: [PATCH 0309/1579] cpufreq: apple-soc: Use related_cpus when removing OPP tables at exit apple_soc_cpufreq_exit() drops the static OPP tables with dev_pm_opp_of_cpumask_remove_table(policy->cpus), but policy->cpus is always empty by then. __cpufreq_offline() clears the outgoing CPU from the mask first and calls ->exit() only once the last CPU of the policy has gone, and unbinding the driver takes the same path. The call removes nothing, so every time a cluster goes fully offline and comes back, or the driver is rebound, the OPP table of each of its CPUs is left with one more reference and is never freed. Remove the tables of policy->related_cpus instead, which still lists every CPU the policy covered, as qcom-cpufreq-hw does. The error path of apple_soc_cpufreq_init() keeps using policy->cpus, because the core fills in related_cpus only after ->init() has succeeded. Fixes: d87cb889dc7a ("cpufreq: apple-soc: Fix OPP table cleanup") Assisted-by: LLM checkpatch sparse Co-authored-by: Claude Fable 5.1 Signed-off-by: Ryan Murray (cherry picked from commit 6559da7d50ab10da722b3719f447836b12427159) Signed-off-by: RJ Skerry-Ryan --- drivers/cpufreq/apple-soc-cpufreq.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index d416088bf28a5..873f5fc5243d3 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -326,7 +326,11 @@ static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) struct apple_cpu_priv *priv = policy->driver_data; dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); - dev_pm_opp_of_cpumask_remove_table(policy->cpus); + /* + * The core clears the last CPU from policy->cpus before calling + * ->exit(), so remove the tables of every CPU the policy covered. + */ + dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); iounmap(priv->reg_base); kfree(priv); } From 1a32cc711d0061b82d8144931ab984e43bdec765 Mon Sep 17 00:00:00 2001 From: Gui-Dong Han Date: Thu, 18 Jun 2026 10:15:43 +0800 Subject: [PATCH 0310/1579] nvme-apple: Use acquire/release for queue enabled state commit f61c934aa084b7440fec681be3f4b481eb5a8609 upstream. apple_nvme_init_queue() initializes queue state and then marks the queue enabled. The interrupt and request paths check enabled before using that queue state. The old wmb() after WRITE_ONCE(enabled, true) does not publish the earlier initialization before enabled becomes visible. Use a release store when enabling the queue and acquire loads when testing it. Although the shutdown-side enabled accesses are not used for publishing queue initialization, use helpers for them as well for consistency. Fixes: 5bd2927aceba ("nvme-apple: Add initial Apple SoC NVMe driver") Signed-off-by: Gui-Dong Han Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch Signed-off-by: Ryan Murray (cherry picked from commit 8d4657c500ab4afed85c64ac12c06d8cca3cb3bc) Signed-off-by: RJ Skerry-Ryan --- drivers/nvme/host/apple.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index a3343c49c6af6..08dc780cc47e6 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -148,6 +148,23 @@ struct apple_nvme_queue { bool enabled; }; +static inline bool apple_nvme_queue_enabled(struct apple_nvme_queue *q) +{ + /* Pair with apple_nvme_enable_queue(). */ + return smp_load_acquire(&q->enabled); +} + +static inline void apple_nvme_enable_queue(struct apple_nvme_queue *q) +{ + /* Publish queue initialization before setting q->enabled. */ + smp_store_release(&q->enabled, true); +} + +static inline void apple_nvme_disable_queue(struct apple_nvme_queue *q) +{ + WRITE_ONCE(q->enabled, false); +} + /* * The apple_nvme_iod describes the data in an I/O. * @@ -688,7 +705,7 @@ static bool apple_nvme_handle_cq(struct apple_nvme_queue *q, bool force) bool found; DEFINE_IO_COMP_BATCH(iob); - if (!READ_ONCE(q->enabled) && !force) + if (!apple_nvme_queue_enabled(q) && !force) return false; found = apple_nvme_poll_cq(q, &iob); @@ -811,7 +828,7 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, * We should not need to do this, but we're still using this to * ensure we can drain requests on a dying queue. */ - if (unlikely(!READ_ONCE(q->enabled))) + if (unlikely(!apple_nvme_queue_enabled(q))) return BLK_STS_IOERR; if (!nvme_check_ready(&anv->ctrl, req, true)) @@ -899,7 +916,7 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_quiesce_io_queues(&anv->ctrl); if (!dead) { - if (READ_ONCE(anv->ioq.enabled)) { + if (apple_nvme_queue_enabled(&anv->ioq)) { apple_nvme_remove_sq(anv); apple_nvme_remove_cq(anv); } @@ -923,8 +940,8 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_disable_ctrl(&anv->ctrl, false); } - WRITE_ONCE(anv->ioq.enabled, false); - WRITE_ONCE(anv->adminq.enabled, false); + apple_nvme_disable_queue(&anv->ioq); + apple_nvme_disable_queue(&anv->adminq); mb(); /* ensure that nvme_queue_rq() sees that enabled is cleared */ nvme_quiesce_admin_queue(&anv->ctrl); @@ -1052,8 +1069,7 @@ static void apple_nvme_init_queue(struct apple_nvme_queue *q) memset(q->tcbs, 0, anv->hw->max_queue_depth * sizeof(struct apple_nvmmu_tcb)); memset(q->cqes, 0, depth * sizeof(struct nvme_completion)); - WRITE_ONCE(q->enabled, true); - wmb(); /* ensure the first interrupt sees the initialization */ + apple_nvme_enable_queue(q); } static void apple_nvme_reset_work(struct work_struct *work) From a8872354b695a066ac6411791a691f26fcde55bf Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Tue, 29 Sep 2026 23:01:35 +0200 Subject: [PATCH 0311/1579] pmdomain: apple: pmp-report: initialize the report lock apple_pmp_report_probe() allocates the report with devm_kzalloc() but never initializes rep->lock. The first apple_pmp_report_set_state() from a child entry's probe then takes an uninitialized spinlock; with lockdep this is reported as a non-static key and lockdep turns itself off for the rest of the boot: INFO: trying to register non-static key. ... _raw_spin_lock_irqsave apple_pmp_report_set_state apple_pmp_report_entry_probe Initialize the lock before of_platform_populate() creates the entries. Tested on j416s (M2 Pro) with PROVE_LOCKING: the report is gone and lockdep stays enabled. Fixes: e5f8c27073d6 ("pmdomain: apple: Add PMP reporting interface") Signed-off-by: Oliver Lukschander Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 583b9fa52fbf6a47f150736ad4a1aaa0d616ac02) Signed-off-by: RJ Skerry-Ryan --- drivers/pmdomain/apple/pmp-report.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 8050ade5ef8d0..065d3296f5a7a 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -41,6 +41,7 @@ static int apple_pmp_report_probe(struct platform_device *pdev) return -ENOMEM; rep->dev = dev; + spin_lock_init(&rep->lock); rep->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(rep->base)) return PTR_ERR(rep->base); From 35b2acbf474d8505c3edc1a2749a8d6fbfaf7a09 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:26:03 +1000 Subject: [PATCH 0312/1579] pmdomain: apple: Print the read-back state on a power state timeout Commit c300d1b7d25c ("soc: apple: pmgr: Add force-disable/force-reset") changed the transition poll in apple_pmgr_ps_set() to read into cur, but left the timeout message printing reg, which now holds the value that was written. Since then the "now:" field shows what was requested rather than the state the domain is stuck in. Print both the value written and the last value read back. On T8140, audio_leap_mca occasionally misses the 100 us poll window, and the read-back value (0xa4f: BUSY set, ACTUAL clock-gated, TARGET active) is what tells a slow transition from a failed one. Fixes: c300d1b7d25c ("soc: apple: pmgr: Add force-disable/force-reset") Assisted-by: LLM checkpatch sparse Signed-off-by: Ryan Murray (cherry picked from commit 2768025e0f1d9792f84e9fbb99f6c0b155c0a22a) Signed-off-by: RJ Skerry-Ryan --- drivers/pmdomain/apple/pmgr-pwrstate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index a92db8cf88bb9..3599866382c70 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -114,8 +114,8 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a } if (ret < 0) - dev_err(ps->dev, "PS %s: Failed to reach power state 0x%x (now: 0x%x)\n", - genpd->name, pstate, reg); + dev_err(ps->dev, "PS %s: Failed to reach power state 0x%x (requested 0x%x, now: 0x%x)\n", + genpd->name, pstate, reg, cur); if (auto_enable) { /* Not all devices implement this; this is a no-op where not implemented. */ From 9dc79fb25c9294bd7a8ecb00acc9bcea379c3d1a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 26 Sep 2026 00:29:42 +1000 Subject: [PATCH 0313/1579] pmdomain: apple: Mask BUSY in power state writes and ratelimit errors apple_pmgr_ps_set() builds the words it writes from a read of the power state register. It clears the WAS_CLKGATED and WAS_PWRGATED flags but not BUSY (bit 11), which the hardware reports while a transition is still in progress: on T8140 audio_leap_mca reads back 0xa4f, with BUSY set, after missing the 100 us poll window. A value read in that state was written back with BUSY set. Clear BUSY in the DEV_DISABLE/PS_RESET write that precedes a power-down and in the target state write, which the auto-enable write reuses. This changes the value written on every SoC that uses this driver. No failure attributable to writing BUSY back was observed; the change is precautionary, as the bit is hardware-owned. The error messages in apple_pmgr_ps_set() and in the reset assert and deassert paths are printed from genpd and reset callbacks that consumers retry, so a domain that keeps missing its poll window, as audio_leap_mca does on T8140, repeats the same message on every attempt. Ratelimit them; their text is unchanged. Assisted-by: LLM checkpatch sparse Co-authored-by: Claude Fable 5.1 Signed-off-by: Ryan Murray (cherry picked from commit afe19efeff9c466486fb723f7066f86b9a3e71d5) Signed-off-by: RJ Skerry-Ryan --- drivers/pmdomain/apple/pmgr-pwrstate.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 3599866382c70..42511ed7ef89c 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -65,11 +65,12 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a /* Resets are synchronous, and only work if the device is powered and clocked. */ if (reg & APPLE_PMGR_RESET && pstate != APPLE_PMGR_PS_ACTIVE) - dev_err(ps->dev, "PS %s: powering off with RESET active\n", + dev_err_ratelimited(ps->dev, "PS %s: powering off with RESET active\n", genpd->name); if (pstate != APPLE_PMGR_PS_ACTIVE && (ps->force_disable || ps->force_reset)) { - u32 reg_pre = reg & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS); + u32 reg_pre = reg & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | + APPLE_PMGR_BUSY); if (ps->force_disable) reg_pre |= APPLE_PMGR_DEV_DISABLE; @@ -85,12 +86,14 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a APPLE_PMGR_PS_SET_TIMEOUT); if (ret < 0) - dev_err(ps->dev, "PS %s: Failed to set reset/disable bits (now: 0x%x)\n", + dev_err_ratelimited(ps->dev, "PS %s: Failed to set reset/disable bits (now: 0x%x)\n", genpd->name, reg); } + /* Do not replay the hardware-owned BUSY bit in this whole-word write. */ reg &= ~(APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_RESET | - APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_PS_TARGET); + APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_BUSY | + APPLE_PMGR_PS_TARGET); reg |= FIELD_PREP(APPLE_PMGR_PS_TARGET, pstate); dev_dbg(ps->dev, "PS %s: pwrstate = 0x%x: 0x%x\n", genpd->name, pstate, reg); @@ -114,7 +117,7 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a } if (ret < 0) - dev_err(ps->dev, "PS %s: Failed to reach power state 0x%x (requested 0x%x, now: 0x%x)\n", + dev_err_ratelimited(ps->dev, "PS %s: Failed to reach power state 0x%x (requested 0x%x, now: 0x%x)\n", genpd->name, pstate, reg, cur); if (auto_enable) { @@ -158,7 +161,8 @@ static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned spin_lock_irqsave(&ps->genpd.slock, flags); if (ps->genpd.status == GENPD_STATE_OFF) - dev_err(ps->dev, "PS 0x%x: asserting RESET while powered down\n", ps->offset); + dev_err_ratelimited(ps->dev, + "PS 0x%x: asserting RESET while powered down\n", ps->offset); dev_dbg(ps->dev, "PS 0x%x: assert reset\n", ps->offset); /* Quiesce device before asserting reset */ @@ -184,7 +188,8 @@ static int apple_pmgr_reset_deassert(struct reset_controller_dev *rcdev, unsigne regmap_update_bits(ps->regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_DEV_DISABLE, 0); if (ps->genpd.status == GENPD_STATE_OFF) - dev_err(ps->dev, "PS 0x%x: RESET was deasserted while powered down\n", ps->offset); + dev_err_ratelimited(ps->dev, + "PS 0x%x: RESET was deasserted while powered down\n", ps->offset); spin_unlock_irqrestore(&ps->genpd.slock, flags); From c0a6a644ad6edd38486d0945c2580d420fc505a3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 00:57:11 -0400 Subject: [PATCH 0314/1579] drm/apple: preserve Type-C routes across blanking and detach Retain the last available DCP route and avoid retraining unrelated outputs during the immediate DP-to-USB4 teardown sequence. Keep synthetic DPMS disconnects separate from physical cable removal, while allowing positive hotplug events through after a replug. Let the RTKit crash callback, rather than a power-off clear-swap timeout, establish a fatal firmware crash. [Chris Kearney: ported onto aurora-silicon/linux#8. The DPMS handling (typec_crtc_off, no reconnect from dcp_poweroff(), ignoring the unplug of a powered-off output) is already in aurora-wip as ab7cad0652d5, so this keeps that version and adds the route preference, the detach retrain guard, the reconnect cancel and the clear-swap change.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/dcp.c | 62 +++++++++++++++++++------- drivers/gpu/drm/apple/iomfb_template.c | 6 ++- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index dba0e22ebf955..f17f45aa2c1c2 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -67,6 +67,10 @@ struct apple_dcp_typec_port { struct list_head routes; struct device_node *connector_np; struct apple_dcp_typec_route *owner; + /* Keep a port on its last DCP while that pipeline remains free. */ + struct apple_dcp_typec_route *preferred_route; + /* Ignore the USB4 fallback immediately following this port's DP teardown. */ + unsigned long dp_release_deadline; /* DRM connector for this physical port, driven by whichever DCP owns it */ struct apple_connector *connector; /* last mux state acted on, to collapse the per-candidate notifications */ @@ -129,10 +133,9 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); /* - * Pipelines are ranked by CRTC index so the fabric's choice is a pure function - * of the topology rather than of plug order. A pipeline whose fixed output is - * live is not a candidate at all, so a hybrid is only ever ranked here when it - * is genuinely free. + * For a port without a prior owner, rank pipelines by CRTC index. A pipeline + * whose fixed output is live is not a candidate at all, so a hybrid is only + * ever ranked here when it is genuinely free. */ static unsigned int dcp_typec_route_score(struct apple_dcp_typec_route *route) { @@ -390,6 +393,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (port->owner) { struct apple_dcp *dcp = port->owner->dcp; + port->preferred_route = port->owner; if (port->hpd || dcp->typec_cable_connected || (dcp->typec_connector && dcp->typec_connector->connected)) @@ -399,13 +403,24 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (ret) return ret; port->owner = NULL; + port->dp_release_deadline = jiffies + msecs_to_jiffies(10000); if (dcp->hdmi_hpd && dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd)) dcp_dptx_connect(dcp, 0); } - if (state->mode == TYPEC_MODE_USB4) - dcp_typec_retrain_active_routes(); + /* + * A port leaving DP can report SAFE/NONE before falling back to USB4. + * Resetting every other live CRTC for that same cable removal blanks + * unaffected displays. Keep the guard across the Type-C state sequence; + * a later, independent USB4 attach still gets recovery. + */ + if (state->mode == TYPEC_MODE_USB4) { + if (!port->dp_release_deadline || + time_after_eq(jiffies, port->dp_release_deadline)) + dcp_typec_retrain_active_routes(); + port->dp_release_deadline = 0; + } port->applied_valid = true; return 0; } @@ -421,15 +436,21 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, } if (!port->owner) { - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; - - if (!dcp_typec_route_available(candidate)) - continue; - score = dcp_typec_route_score(candidate); - if (score < best_score) { - best = candidate; - best_score = score; + if (port->preferred_route && + dcp_typec_route_available(port->preferred_route)) + best = port->preferred_route; + + if (!best) { + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (!dcp_typec_route_available(candidate)) + continue; + score = dcp_typec_route_score(candidate); + if (score < best_score) { + best = candidate; + best_score = score; + } } } @@ -440,6 +461,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (ret) return ret; port->owner = best; + port->dp_release_deadline = 0; } @@ -843,6 +865,8 @@ static void dcp_typec_route_unregister(void *data) typec_mux_unregister(route->typec_mux); guard(mutex)(&dcp_typec_fabric_lock); + if (port->preferred_route == route) + port->preferred_route = NULL; if (port->owner == route) { struct apple_dcp *dcp = route->dcp; @@ -1902,8 +1926,12 @@ void dcp_poweroff(struct platform_device *pdev) * display, so DPMS off never sticks. Cable removal is reported through * the Type-C mux. */ - if (dcp_is_typec_output(dcp) && READ_ONCE(dcp->typec_cable_connected)) - WRITE_ONCE(dcp->typec_crtc_off, true); + if (dcp_is_typec_output(dcp)) { + if (READ_ONCE(dcp->typec_cable_connected)) + WRITE_ONCE(dcp->typec_crtc_off, true); + /* dcp_poweron() reconnects the link on DPMS wake. */ + cancel_delayed_work(&dcp->typec_reconnect_wq); + } _dcp_poweroff(dcp); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 38a1ebb56be36..4cd28e0545563 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -950,7 +950,11 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) swap_id = cookie->swap_id; kref_put(&cookie->refcount, release_swap_cookie); if (ret <= 0) { - dcp->crashed = true; + /* A disconnected external display can power down before it ACKs this + * clear swap. The deadline alone does not mean RTKit crashed; its + * crash callback owns the fatal flag. + */ + dev_warn(dcp->dev, "clear swap timed out during power-off\n"); return; } From 0c851b12559986a335bdc1bc59d4bd1827b4051b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 24 Sep 2026 23:11:18 -0400 Subject: [PATCH 0315/1579] thunderbolt: apple: rearm the display route for a new DP tunnel Re-establish the DCP route when a new dock tunnel appears after a controller detach without a prior teardown callback: force the route through a down/up cycle instead of treating the old handoff as this one. Tested on J414s m2fix42: dock DP tunnel enabled USB-3 at 1920x1280@60, direct USB-2 at 3840x2160@120 and eDP at 3024x1964@120 with active CRTCs and zero failed units. Controller-detach disconnect reporting remains under investigation. [Chris Kearney: ported onto aurora-silicon/linux#8 from "arm64: enable J414s Thunderbolt DP tunnel and rearm display route". The J414s gates that commit added are covered by apple_dp_tunnel_t602x(), which also takes in j414c and j416c, so only the route rearm is carried here.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/apple.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 467bb4579717a..244e47c4f012b 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -183,6 +183,7 @@ struct apple_cio { unsigned int idx; bool alive; /* tunnel up; cleared before it is torn down */ bool handed; /* appledrm has our callback; work only */ + bool rearm; /* a fresh tunnel needs a fresh DCP route */ } dpin[2]; }; @@ -434,12 +435,15 @@ static void apple_dpin_down(struct apple_dpin_ctx *c) static void apple_dpin_work_fn(struct work_struct *work) { struct apple_dpin_ctx *c = container_of(work, struct apple_dpin_ctx, work); - bool want; + bool want, rearm; int ret; for (;;) { - scoped_guard(mutex, &c->lock) + scoped_guard(mutex, &c->lock) { want = c->alive; + rearm = c->rearm; + c->rearm = false; + } if (!want) { if (c->handed || c->regs) { @@ -448,6 +452,13 @@ static void apple_dpin_work_fn(struct work_struct *work) } return; } + if (rearm && c->handed) { + dev_info(c->acio->dev, + "dpin%u: replacing stale display route for new tunnel\n", + c->idx); + apple_dpin_down(c); + continue; + } if (c->handed) return; @@ -1427,8 +1438,15 @@ static int apple_nhi_dp_tunnel_post_activate(struct tb_nhi *nhi, } else { struct apple_dpin_ctx *c = &anhi->acio->dpin[idx]; - scoped_guard(mutex, &c->lock) + /* + * A new tunnel can arrive without the old tunnel's teardown hook + * running on a controller detach. Force the DCP route through a + * down/up cycle instead of treating the old handoff as this one. + */ + scoped_guard(mutex, &c->lock) { c->alive = true; + c->rearm = true; + } queue_work(anhi->acio->dp_wq, &c->work); } apple_dp_dump_host_adapters(anhi); From 0294db60f26e9c7c674fa86c4b859e75e90b4bd8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 24 Sep 2026 23:39:53 -0400 Subject: [PATCH 0316/1579] drm/apple: reserve the hybrid DCP for a Thunderbolt DP tunnel Route an available direct Type-C DP-alt-mode display to dcpext1 first so dcpext0 remains available for a dock DP tunnel. The tunnel still prefers dcpext0, which is the pipeline that completes link training on T6020. m2fix45 cold-booted with both displays attached: direct 3840x2160@120 on dcpext1 and dock 1920x1280@60 on dcpext0 came up without port resets, alongside eDP. SEP identity, haptic FF ioctl, and AVD H.264 decode remained functional. [Chris Kearney: ported onto aurora-silicon/linux#8; the J414s check goes through apple_dp_tunnel_t602x(), so j414c, j416s and j416c take the same pipeline preference, as tested on j416s by Oliver Lukschander and proposed for j416c in #50.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/dcp.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f17f45aa2c1c2..e71ae04c0971c 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -447,6 +447,14 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (!dcp_typec_route_available(candidate)) continue; score = dcp_typec_route_score(candidate); + /* + * On the M2 Pro/Max laptops, leave dcpext0 free for + * a dock's DP tunnel when a direct DP-alt-mode + * display can use the Type-C-only dcpext1 pipeline. + */ + if (apple_dp_tunnel_t602x() && + candidate->dcp->fixed_phy) + score += 100; if (score < best_score) { best = candidate; best_score = score; From 82682e77aac4524204db889dd72607fba9595705 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 26 Sep 2026 18:51:29 -0400 Subject: [PATCH 0317/1579] drm/apple: drive two dock DP tunnels independently Route DPIN1 through the native T602x crossbar fields and source selector, give the two tunnel streams separate DCP connector/route ownership, and keep both ATC pixel clocks alive at independent link rates. Wait for Type-C IOMFB hotplug after a display power-on. The dual-dock path was exercised on J414s/m2fix48 with Dell 3840x2160@120 and JTG 1920x1280@60, per-output DPMS and idle/wake; the user confirmed the displayed image. This commit removes a frame-complete callback that had no register side effect. The cleaned tree builds with the Asahi profile; the cleaned commit itself is not yet deployed. [Chris Kearney: ported onto aurora-silicon/linux#8 from "drm/apple: drive two J414s dock DP tunnels independently": - The J414s checks go through apple_dp_tunnel_t602x(), so all four M2 Pro/Max laptops get the second connector per port and the per-DP-IN pipeline preference (DPIN0 dcpext0, DPIN1 dcpext1). Oliver Lukschander ran two displays through one hub on j416s with the same gates. - The per-DP-IN ATC clock tracking replaces #8's single-rate T602X restart: a rate change reselects that DP IN's PCLK. A DPMS stop gates the outputs and keeps the PLL; a PHY mode change still powers the AUSPLL down in atc_tunnel_restore(), which now also forgets the users. The t8103 tunnel clock path is unchanged. - Only the T602X crossbar preselects a DP IN's source; t8103 tunnels still select it at link-up, so -EOPNOTSUPP from the selector is not an error.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/apple_drv.c | 23 ++-- drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp.c | 166 ++++++++++++++++++------ drivers/gpu/drm/apple/dcp.h | 3 +- drivers/gpu/drm/apple/iomfb_template.c | 2 + drivers/mux/apple-display-crossbar.c | 131 ++++++++++++++----- drivers/phy/apple/atc.c | 170 ++++++++++++++++++++----- include/linux/soc/apple/dp-tunnel.h | 3 +- 8 files changed, 383 insertions(+), 117 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 4c6fa62321a25..d0fee4140a77b 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -423,10 +424,10 @@ static int apple_probe_per_dcp(struct device *dev, } /* - * Create one connector per physical Type-C port and attach it to every - * pipeline that can drive it. Keeping the connector tied to the port rather - * than to a pipeline is what gives userspace a stable name to hang its - * per-monitor configuration on. + * Create a connector per physical Type-C port and attach it to every + * pipeline that can drive it. The M2 Pro/Max laptops append a second + * connector per port for a dock's second DP tunnel, preserving primary + * connector names. */ static int apple_probe_typec_ports(struct drm_device *drm, struct platform_device **dcp, @@ -435,9 +436,15 @@ static int apple_probe_typec_ports(struct drm_device *drm, unsigned int idx, nr_ports = dcp_typec_nr_ports(); int i, ret; - for (idx = 0; idx < nr_ports; idx++) { + /* Keep the existing physical-port connector numbers stable. On the + * M2 Pro/Max laptops, append one more connector per port for a second + * USB4 DP tunnel. + */ + for (idx = 0; idx < nr_ports * (apple_dp_tunnel_t602x() ? 2 : 1); idx++) { struct apple_connector *connector; struct apple_encoder *enc; + unsigned int port_idx = idx % nr_ports; + bool secondary = idx >= nr_ports; u32 mask = 0; connector = kzalloc_obj(*connector); @@ -466,13 +473,13 @@ static int apple_probe_typec_ports(struct drm_device *drm, INIT_WORK(&connector->hotplug_wq, dcp_hotplug); for (i = 0; i < num_dcp; i++) { - if (dcp_typec_port_has_candidate(idx, dcp[i])) + if (dcp_typec_port_has_candidate(port_idx, dcp[i])) mask |= crtc_mask[i]; } if (!mask) { drm_warn(drm, "Type-C port %u has no display pipeline\n", - idx); + port_idx); return -ENODEV; } @@ -496,7 +503,7 @@ static int apple_probe_typec_ports(struct drm_device *drm, connector->port_encoder = &enc->base; connector->candidate_crtcs = mask; - dcp_typec_port_set_connector(idx, connector); + dcp_typec_port_set_connector(port_idx, secondary, connector); } return 0; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index fda7dc887ad34..93b421ea3b0d2 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -318,6 +318,8 @@ struct apple_dcp { bool tb_clock_ok; /* CRTC powered off while the Type-C cable stays attached */ bool typec_crtc_off; + /* IOMFB reports its video interface ready after DPTX link training. */ + struct completion typec_iomfb_hpd_ready; struct delayed_work typec_reconnect_wq; struct delayed_work typec_fabric_retrain_wq; u32 typec_reconnect_tries; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e71ae04c0971c..a1bf95b397660 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -67,12 +67,15 @@ struct apple_dcp_typec_port { struct list_head routes; struct device_node *connector_np; struct apple_dcp_typec_route *owner; + struct apple_dcp_typec_route *secondary_owner; /* Keep a port on its last DCP while that pipeline remains free. */ struct apple_dcp_typec_route *preferred_route; /* Ignore the USB4 fallback immediately following this port's DP teardown. */ unsigned long dp_release_deadline; /* DRM connector for this physical port, driven by whichever DCP owns it */ struct apple_connector *connector; + /* A second logical stream through this port's USB4 dock. */ + struct apple_connector *secondary_connector; /* last mux state acted on, to collapse the per-candidate notifications */ struct typec_altmode *applied_alt; unsigned long applied_mode; @@ -132,6 +135,19 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); +static int dcp_dpxbar_tunnel_select_source(struct mux_control *mux, int state) +{ + typeof(&apple_dpxbar_tunnel_select_source) select = + symbol_get(apple_dpxbar_tunnel_select_source); + int ret; + + if (!select) + return -ENOENT; + ret = select(mux, state); + symbol_put(apple_dpxbar_tunnel_select_source); + return ret; +} + /* * For a port without a prior owner, rank pipelines by CRTC index. A pipeline * whose fixed output is live is not a candidate at all, so a hybrid is only @@ -151,6 +167,9 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar) { struct apple_dcp *dcp = route->dcp; + struct apple_connector *connector = + xbar != route->xbar && route->tunnel_dpin == 1 ? + route->port->secondary_connector : route->port->connector; int ret; /* @@ -172,7 +191,19 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * once DCP has configured the link and the tunnel pixel clock runs, * see dcp_tunnel_crossbar_up(). Just remember the output here. */ - ret = xbar == route->xbar ? mux_control_select(xbar, route->mux_index) : 0; + if (xbar != route->xbar && route->mux_index) { + /* + * The T602X crossbar must point a DP IN at its pipeline before + * DCP probes AUX; other crossbars select it at link-up and + * report -EOPNOTSUPP here. + */ + ret = dcp_dpxbar_tunnel_select_source(xbar, route->mux_index); + if (ret == -EOPNOTSUPP) + ret = 0; + } else { + ret = xbar == route->xbar ? + mux_control_select(xbar, route->mux_index) : 0; + } if (ret) { if (dcp->xbar) { int restore_ret; @@ -192,10 +223,10 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, dcp->phy = route->phy; dcp->dptx_phy = route->dptx_phy; dcp->connector_type = DRM_MODE_CONNECTOR_USB; - if (route->port->connector) { - route->port->connector->dcp = to_platform_device(dcp->dev); - dcp->typec_connector = route->port->connector; - dcp->connector = route->port->connector; + if (connector) { + connector->dcp = to_platform_device(dcp->dev); + dcp->typec_connector = connector; + dcp->connector = connector; /* * Narrow the port to the pipeline now driving it. The encoder @@ -207,8 +238,8 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * rejected with no way for it to recover. The hotplug that * follows makes it re-read this. */ - if (route->port->connector->port_encoder && dcp->crtc) - route->port->connector->port_encoder->possible_crtcs = + if (connector->port_encoder && dcp->crtc) + connector->port_encoder->possible_crtcs = drm_crtc_mask(&dcp->crtc->base); } dcp->active_typec_route = route; @@ -230,6 +261,9 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) { struct apple_dcp *dcp = route->dcp; + struct apple_connector *connector = dcp->typec_connector; + struct mux_control *active_xbar = route->active_xbar; + bool was_tunnel = route->tunnel; int ret = 0; /* @@ -249,7 +283,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) symbol_get(apple_atc_dp_tunnel_rate); if (stop) { - stop(dcp->phy, 0); + stop(dcp->phy, route->tunnel_dpin, 0); symbol_put(apple_atc_dp_tunnel_rate); } } @@ -261,12 +295,17 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) dcp->tb_dpin_ctx = NULL; dcp->tb_clock_ok = false; } + if (was_tunnel && route->mux_index) { + int sel = dcp_dpxbar_tunnel_select_source(active_xbar, -1); + + if (sel && sel != -EOPNOTSUPP) + dev_warn(dcp->dev, "DP tunnel source reset failed: %d\n", sel); + } route->selected = false; if (dcp->active_typec_route == route) dcp->active_typec_route = NULL; - if (route->port->connector && - route->port->connector->dcp == to_platform_device(dcp->dev)) { + if (connector && connector->dcp == to_platform_device(dcp->dev)) { /* * Until the port is activated again it has no pipeline behind * it, and nothing can read modes or EDID from it. Report it @@ -277,13 +316,13 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) * it. The new pipeline marks it connected again once the * display has come back up on it. */ - WRITE_ONCE(route->port->connector->connected, false); - route->port->connector->dcp = NULL; + WRITE_ONCE(connector->connected, false); + connector->dcp = NULL; /* Unrouted: the port could go to any of its pipelines again. */ - if (route->port->connector->port_encoder) - route->port->connector->port_encoder->possible_crtcs = - route->port->connector->candidate_crtcs; + if (connector->port_encoder) + connector->port_encoder->possible_crtcs = + connector->candidate_crtcs; } dcp->typec_connector = NULL; dcp->connector = dcp->fixed_connector; @@ -339,11 +378,14 @@ static void dcp_typec_retrain_active_routes(void) struct apple_dcp_typec_port *port; list_for_each_entry(port, &dcp_typec_ports, link) { - if (!port->owner) - continue; - mod_delayed_work(system_freezable_wq, + if (port->owner) + mod_delayed_work(system_freezable_wq, &port->owner->dcp->typec_fabric_retrain_wq, msecs_to_jiffies(200)); + if (port->secondary_owner) + mod_delayed_work(system_freezable_wq, + &port->secondary_owner->dcp->typec_fabric_retrain_wq, + msecs_to_jiffies(200)); } } @@ -597,9 +639,13 @@ int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) return -ENOENT; } route = dcp->active_typec_route; - if (!link_rate && route && route->xbar_up) + if (!route) { + symbol_put(apple_atc_dp_tunnel_rate); + return -ENODEV; + } + if (!link_rate && route->xbar_up) dcp_dpxbar_link(route->active_xbar, false); - ret = fn(phy, link_rate); + ret = fn(phy, route->tunnel_dpin, link_rate); symbol_put(apple_atc_dp_tunnel_rate); dcp->tb_clock_ok = !ret && link_rate; if (ret) @@ -630,6 +676,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, { struct apple_dcp_typec_port *port = NULL, *pos; struct apple_dcp_typec_route *candidate, *best = NULL; + struct apple_dcp_typec_route **slot; unsigned int best_score = UINT_MAX; struct mux_control *ctl; struct apple_dcp *dcp; @@ -648,16 +695,16 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, } if (!port) return -ENODEV; + slot = dpin ? &port->secondary_owner : &port->owner; if (!active) { - if (!port->owner || !port->owner->tunnel || - port->owner->tunnel_dpin != dpin) + if (!*slot || !(*slot)->tunnel || + (*slot)->tunnel_dpin != dpin) return 0; - dcp = port->owner->dcp; + dcp = (*slot)->dcp; if (port->hpd || dcp->typec_cable_connected) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - port->hpd = false; - ret = dcp_typec_route_deactivate(port->owner); + ret = dcp_typec_route_deactivate(*slot); if (ret) { /* the caller's context is going away regardless */ scoped_guard(mutex, &dcp->tb_lock) { @@ -667,7 +714,8 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, } return ret; } - port->owner = NULL; + *slot = NULL; + port->hpd = !!(port->owner || port->secondary_owner); /* re-apply the next Type-C mux state in full */ port->applied_valid = false; if (dcp->hdmi_hpd && dcp->active && @@ -676,13 +724,15 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, return 0; } - if (port->owner) { - if (port->owner->tunnel && port->owner->tunnel_dpin == dpin) + if (*slot) { + if ((*slot)->tunnel && (*slot)->tunnel_dpin == dpin) return 0; - dev_warn(port->owner->dcp->dev, + dev_warn((*slot)->dcp->dev, "port already routed, not taking DP tunnel dpin%u\n", dpin); return -EBUSY; } + if (port->owner && !port->owner->tunnel) + return -EBUSY; list_for_each_entry(candidate, &port->routes, port_link) { unsigned int score; @@ -690,9 +740,15 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, if (!dcp_typec_route_available(candidate)) continue; score = dcp_typec_route_score(candidate); - /* dcpext0 completes tunneled link training here; prefer it. */ - if (apple_dp_tunnel_t602x() && - !candidate->dcp->fixed_phy && score < UINT_MAX - 100) + /* + * On the M2 Pro/Max laptops DPIN0 prefers the hybrid dcpext0, + * which completes tunneled link training, and DPIN1 prefers + * dcpext1, so the two pipelines drive independent streams + * through one dock. + */ + if (apple_dp_tunnel_t602x() && score < UINT_MAX - 100 && + ((dpin == 0 && !candidate->dcp->fixed_phy) || + (dpin == 1 && candidate->dcp->fixed_phy))) score += 100; if (score < best_score) { best = candidate; @@ -722,7 +778,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, } return ret; } - port->owner = best; + *slot = best; dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); @@ -836,10 +892,11 @@ bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev return false; } -void dcp_typec_port_set_connector(unsigned int idx, +void dcp_typec_port_set_connector(unsigned int idx, bool secondary, struct apple_connector *connector) { struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *owner; guard(mutex)(&dcp_typec_fabric_lock); @@ -847,7 +904,13 @@ void dcp_typec_port_set_connector(unsigned int idx, if (!port) return; - port->connector = connector; + if (secondary) { + port->secondary_connector = connector; + owner = port->secondary_owner; + } else { + port->connector = connector; + owner = port->owner; + } /* * The port may already have been routed, either before DRM bound or @@ -855,8 +918,12 @@ void dcp_typec_port_set_connector(unsigned int idx, * otherwise its display would be reported on the pipeline's fixed * connector instead of the port it is actually plugged into. */ - if (port->owner) { - struct apple_dcp *dcp = port->owner->dcp; + if (owner) { + struct apple_dcp *dcp = owner->dcp; + + if (dcp->crtc && connector->port_encoder) + connector->port_encoder->possible_crtcs = + drm_crtc_mask(&dcp->crtc->base); connector->dcp = to_platform_device(dcp->dev); dcp->typec_connector = connector; @@ -884,6 +951,15 @@ static void dcp_typec_route_unregister(void *data) dcp_typec_route_deactivate(route); port->owner = NULL; } + if (port->secondary_owner == route) { + struct apple_dcp *dcp = route->dcp; + + if (port->hpd || dcp->typec_cable_connected) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + dcp_typec_route_deactivate(route); + port->secondary_owner = NULL; + } + port->hpd = !!(port->owner || port->secondary_owner); list_del(&route->port_link); if (list_empty(&port->routes)) { list_del(&port->link); @@ -1639,6 +1715,7 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) dcp->typec_generation++; } WRITE_ONCE(dcp->typec_crtc_off, false); + reinit_completion(&dcp->typec_iomfb_hpd_ready); cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_delayed_work_sync(&dcp->placeholder_edid_wq); } @@ -1874,9 +1951,13 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) void dcp_poweron(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + bool wait_for_typec_hpd = false; + unsigned long remaining; int ret; if (dcp_is_typec_output(dcp)) { + wait_for_typec_hpd = READ_ONCE(dcp->typec_crtc_off) && + READ_ONCE(dcp->typec_cable_connected); WRITE_ONCE(dcp->typec_crtc_off, false); /* @@ -1891,6 +1972,14 @@ void dcp_poweron(struct platform_device *pdev) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, DPTX_RECONNECT_DELAY); + else if (wait_for_typec_hpd) { + remaining = wait_for_completion_timeout( + &dcp->typec_iomfb_hpd_ready, + msecs_to_jiffies(3000)); + if (!remaining) + dev_warn(dcp->dev, + "Type-C IOMFB hotplug not ready on wake\n"); + } } } else if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); @@ -1911,7 +2000,6 @@ void dcp_poweron(struct platform_device *pdev) WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; } - if (dcp->avep) av_service_connect(dcp); } @@ -1935,6 +2023,7 @@ void dcp_poweroff(struct platform_device *pdev) * the Type-C mux. */ if (dcp_is_typec_output(dcp)) { + reinit_completion(&dcp->typec_iomfb_hpd_ready); if (READ_ONCE(dcp->typec_cable_connected)) WRITE_ONCE(dcp->typec_crtc_off, true); /* dcp_poweron() reconnects the link on DPMS wake. */ @@ -2489,6 +2578,7 @@ static int dcp_platform_probe(struct platform_device *pdev) of_property_read_u32(dev->of_node, "apple,dptx-phy", &dcp->dptx_phy); of_property_read_u32(dev->of_node, "apple,dptx-die", &dcp->dptx_die); dcp->fixed_dptx_phy = dcp->dptx_phy; + init_completion(&dcp->typec_iomfb_hpd_ready); INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, dcp_typec_reconnect_work); INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index ad6fe280a503d..7e1047b0833f6 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -44,7 +44,8 @@ bool dcp_has_typec_routes(struct platform_device *pdev); unsigned int dcp_typec_nr_ports(void); struct device_node *dcp_typec_port_of_node(unsigned int idx); bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev); -void dcp_typec_port_set_connector(unsigned int idx, struct apple_connector *connector); +void dcp_typec_port_set_connector(unsigned int idx, bool secondary, + struct apple_connector *connector); bool dcp_fw_compat_is_12_x(struct platform_device *pdev); unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 4cd28e0545563..b032bbd2a4b42 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1063,6 +1063,8 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) schedule_work(&dcp->vblank_wq); return; } + if (dcp_is_typec_output(dcp) && *connected && dcp->nr_modes) + complete_all(&dcp->typec_iomfb_hpd_ready); action = dcp_mode_hotplug(&dcp->mode_state, !!(*connected), connector ? &connector->connected : NULL); diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index e7a8470aa7c79..9a50dc7eb70b9 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -29,6 +29,7 @@ #define T602X_FIFO_WR_N_CLK_EN 0x004 #define T602X_FIFO_WR_UNK_EN 0x008 #define T602X_REG_00C 0x00c +#define T602X_REG_010 0x010 #define T602X_REG_014 0x014 #define T602X_REG_018 0x018 #define T602X_REG_01C 0x01c @@ -247,23 +248,27 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) dpxbar_clear32(dpxbar, T602X_FIFO_RD_UNK_EN, prev_dispext_bit); dpxbar_clear32(dpxbar, T602X_FIFO_WR_DPTX_CLK_EN, prev_dispext_bit); - /* DPIN0 uses the same single-bit source gate on up and down. */ - dpxbar_clear32(dpxbar, T602X_REG_00C, - index == MUX_DPIN0 ? prev_dispext_bit : prev_dispext_bit_en); + dpxbar_clear32(dpxbar, + index == MUX_DPIN1 ? T602X_REG_010 : T602X_REG_00C, + prev_dispext_bit); - dpxbar_clear32(dpxbar, T602X_REG_01C, atc_bit); + dpxbar_clear32(dpxbar, + index == MUX_DPIN1 ? FIFO_RD_PCLK1_EN : T602X_REG_01C, + atc_bit); dpxbar_clear32(dpxbar, T602X_FIFO_WR_UNK_EN, prev_dispext_bit); - dpxbar_clear32(dpxbar, T602X_REG_018, prev_dispext_bit_en); + dpxbar_clear32(dpxbar, T602X_REG_018, + prev_dispext_bit_en * + (index == MUX_DPIN1 ? 3 : 1)); - dpxbar_clear32(dpxbar, T602X_FIFO_RD_N_CLK_EN, atc_bit); + dpxbar_clear32(dpxbar, T602X_FIFO_RD_N_CLK_EN, + atc_bit * (index == MUX_DPIN1 ? 3 : 1)); dpxbar_set32(dpxbar, T602X_FIFO_WR_N_CLK_EN, prev_dispext_bit); dpxbar_set32(dpxbar, T602X_REG_014, 0x4); - /* Restore the selected input's FIFO read reset. */ - dpxbar_set32(dpxbar, index == MUX_DPIN0 ? - T602X_FIFO_RD_PCLK2_EN : FIFO_RD_PCLK1_EN, atc_bit); + /* Native T602x DPIN0 teardown restores the read reset at +0x24. */ + dpxbar_set32(dpxbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); dpxbar_clear32(dpxbar, T602X_REG_034, atc_bit); dpxbar_clear32(dpxbar, CROSSBAR_ATC_EN, atc_bit); @@ -280,16 +285,37 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) dpxbar_clear32(dpxbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); dpxbar_clear32(dpxbar, T602X_REG_014, 0x4); + /* Match the DPIN1 sequence used by link_up(): clear +0x24 + * before setting the separate +0x20 field below. + */ dpxbar_clear32(dpxbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); + if (index == MUX_DPIN1) + dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, atc_bit); dpxbar_set32(dpxbar, T602X_FIFO_WR_UNK_EN, dispext_bit); - dpxbar_set32(dpxbar, T602X_REG_018, dispext_bit_en); - - dpxbar_set32(dpxbar, T602X_FIFO_RD_N_CLK_EN, atc_bit); + /* Native T602x encodes DPIN1's role as 3 in both two-bit + * source and sink fields, versus 1 for DPIN0. + */ + if (index == MUX_DPIN1) { + dpxbar_mask32(dpxbar, T602X_REG_018, + dispext_bit_en * 3, + dispext_bit_en * 3); + dpxbar_mask32(dpxbar, T602X_FIFO_RD_N_CLK_EN, + atc_bit * 3, atc_bit * 3); + } else { + dpxbar_set32(dpxbar, T602X_REG_018, + dispext_bit_en); + dpxbar_set32(dpxbar, T602X_FIFO_RD_N_CLK_EN, + atc_bit); + } dpxbar_set32(dpxbar, T602X_FIFO_WR_DPTX_CLK_EN, dispext_bit); - dpxbar_set32(dpxbar, T602X_REG_00C, dispext_bit); + dpxbar_set32(dpxbar, + index == MUX_DPIN1 ? T602X_REG_010 : T602X_REG_00C, + dispext_bit); - dpxbar_set32(dpxbar, T602X_REG_01C, atc_bit); + dpxbar_set32(dpxbar, + index == MUX_DPIN1 ? FIFO_RD_PCLK1_EN : T602X_REG_01C, + atc_bit); dpxbar_set32(dpxbar, T602X_REG_034, atc_bit); dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); /* @@ -583,12 +609,48 @@ static const struct mux_control_ops apple_dpxbar_t602x_ops = { .set = apple_dpxbar_set_t602x, }; +/* + * Select the DCP source for a USB4 DP IN before DCP probes AUX. Do not + * enable any FIFO clock or ATC output here: full activation still waits for + * DCP's SetLinkRate and the tunnel pixel clock. dispext0 works with the + * reset selector value (0); dispext1 needs this explicit preselection. + */ +int apple_dpxbar_tunnel_select_source(struct mux_control *mux, int state) +{ + struct apple_dpxbar *xbar; + unsigned long flags; + unsigned int index; + int ret = 0; + + if (!mux || mux->chip->ops != &apple_dpxbar_t602x_ops) + return -EOPNOTSUPP; + index = mux_control_get_index(mux); + if (index != MUX_DPIN0 && index != MUX_DPIN1) + return -EINVAL; + if (state < -1 || state >= 9) + return -EINVAL; + + xbar = mux_chip_priv(mux->chip); + spin_lock_irqsave(&xbar->lock, flags); + if (xbar->selected_dispext[index] >= 0 && + xbar->selected_dispext[index] != state) { + ret = -EBUSY; + } else { + dpxbar_mask32(xbar, T602X_REG_030, t602x_mux_mask(index), + state < 0 ? 0 : t602x_mux_set(index, state)); + } + spin_unlock_irqrestore(&xbar->lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_dpxbar_tunnel_select_source); + static int apple_dpxbar_t602x_link_up(struct mux_control *mux) { struct apple_dpxbar *xbar; unsigned int index; unsigned long flags; - u32 dispext_bit, atc_bit; + u32 dispext_bit, dispext_bit_en, atc_bit; int state, ret = 0; if (!mux || mux->chip->ops != &apple_dpxbar_t602x_ops) @@ -606,31 +668,32 @@ static int apple_dpxbar_t602x_link_up(struct mux_control *mux) return -ENODEV; } dispext_bit = 1 << state; + dispext_bit_en = 1 << (2 * state); dpxbar_clear32(xbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); dpxbar_clear32(xbar, T602X_REG_014, dispext_bit); - dpxbar_clear32(xbar, index == MUX_DPIN0 ? T602X_FIFO_RD_PCLK2_EN : - FIFO_RD_PCLK1_EN, atc_bit); + dpxbar_clear32(xbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); udelay(1); dpxbar_set32(xbar, T602X_FIFO_WR_UNK_EN, dispext_bit); - dpxbar_mask32(xbar, T602X_REG_018, GENMASK(5, 4), BIT(4)); - /* - * The only hardware-verified value for this 2-bit field is dpin0's - * BIT(0), which is both ATC_DPIN0 and MUX_DPIN0's own enum value (1) - * coincidentally overlapping at bit 0 -- so this field's encoding - * for a second index was never independently confirmed on real - * hardware. FIELD_PREP(..., index), matching - * MUX_DPPHY=0/MUX_DPIN0=1/MUX_DPIN1=2, is an inferred generalization - * of that single known-good dpin0 data point, not a value verified - * against dpin1 hardware. If dpin1 misbehaves, this field is a - * prime suspect. + if (index == MUX_DPIN1) { + dpxbar_mask32(xbar, T602X_REG_018, + dispext_bit_en * 3, dispext_bit_en * 3); + dpxbar_mask32(xbar, T602X_FIFO_RD_N_CLK_EN, + atc_bit * 3, atc_bit * 3); + } else { + dpxbar_set32(xbar, T602X_REG_018, dispext_bit_en); + } + /* set_t602x() already enabled this output's N clock. In particular, + * do not rewrite the low bits here: DPIN1 would clear DPIN0's clock. */ - dpxbar_mask32(xbar, T602X_FIFO_RD_N_CLK_EN, GENMASK(1, 0), - FIELD_PREP(GENMASK(1, 0), index)); dpxbar_set32(xbar, T602X_FIFO_WR_DPTX_CLK_EN, dispext_bit); - dpxbar_set32(xbar, T602X_REG_00C, dispext_bit); - dpxbar_set32(xbar, T602X_REG_01C, atc_bit); + dpxbar_set32(xbar, + index == MUX_DPIN1 ? T602X_REG_010 : T602X_REG_00C, + dispext_bit); + dpxbar_set32(xbar, + index == MUX_DPIN1 ? FIFO_RD_PCLK1_EN : T602X_REG_01C, + atc_bit); dpxbar_set32(xbar, T602X_REG_034, atc_bit); dpxbar_set32(xbar, CROSSBAR_ATC_EN, atc_bit); dpxbar_set32(xbar, CROSSBAR_DISPEXT_EN, dispext_bit); @@ -675,11 +738,9 @@ static int apple_dpxbar_t602x_link_down(struct mux_control *mux) return -ENODEV; } dispext_bit = 1 << state; - dpxbar_set32(xbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); dpxbar_set32(xbar, T602X_REG_014, dispext_bit); - dpxbar_set32(xbar, index == MUX_DPIN0 ? T602X_FIFO_RD_PCLK2_EN : - FIFO_RD_PCLK1_EN, atc_bit); + dpxbar_set32(xbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); spin_unlock_irqrestore(&xbar->lock, flags); dev_info(xbar->dev, "%s: crossbar link down (dispext=%d atc=0x%x)\n", diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 5213a6e5239d8..d1051e2581470 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -652,7 +652,9 @@ struct atcphy_hw { * @tunnel_attempted: A T602X tunnel clock setup has been attempted * @tunnel_saved: T602X PHY registers have been saved for tunnel teardown * @tunnel_saved_regs: Original values of the T602X tunnel clock registers - * @tunnel_rate: DP link rate code the tunnel pixel clock is set up for + * @tunnel_rate: DP link rate code the t8103 tunnel pixel clock is set up for + * @tunnel_users: T602X DP IN adapters (BIT(dpin)) whose tunnel pixel clock runs + * @tunnel_dpin_rate: DP link rate code each T602X DP IN adapter's clock runs at * @pipe_state: Backend the PIPE mux ("pipehandler") is routed to * @regs: Memory-mapped registers * @regs.core: Core registers @@ -702,6 +704,8 @@ struct apple_atcphy { bool tunnel_attempted, tunnel_saved; u32 tunnel_saved_regs[12]; u8 tunnel_rate; + u8 tunnel_users; + u8 tunnel_dpin_rate[2]; struct { void __iomem *core; @@ -2184,7 +2188,8 @@ struct atc_tunnel_saved_reg { static const struct atc_tunnel_saved_reg atc_tunnel_regs[] = { { ACIOPHY_CFG0, 0x0003ffff }, { ACIOPHY_SLEEP_CTRL, 0x00000fff }, - { ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, 0x0000207c }, + { ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + 0x0000207c | DPTX_PCLK2_SELECT | DPTX_PCLK2_ENABLE }, { AUSPLL_FREQ_CFG, 0x00000003 }, { AUSPLL_FREQ_DESC_A, 0xffffffff }, { AUSPLL_FREQ_DESC_B, 0x0fffffff }, @@ -2217,6 +2222,9 @@ static void atc_tunnel_restore(struct apple_atcphy *atcphy) int i, ret; lockdep_assert_held(&atcphy->lock); + atcphy->tunnel_users = 0; + atcphy->tunnel_dpin_rate[0] = 0; + atcphy->tunnel_dpin_rate[1] = 0; if (!atcphy->tunnel_saved) return; /* @@ -2224,7 +2232,8 @@ static void atc_tunnel_restore(struct apple_atcphy *atcphy) * with ACIOPHY_AUSPLL_LOCK still set. Clear the three drivers, then * power the PLL down with APB command 3, after which the lock drops. */ - core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE | DPTX_PCLK2_ENABLE); core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(2)); core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(4)); core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(6)); @@ -2253,35 +2262,36 @@ static void atc_tunnel_restore(struct apple_atcphy *atcphy) atcphy->tunnel_rate = 0; } -static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) +static int atc_tunnel_rate_selector(u8 rate) { - u32 selector, value, gates, outputs, command, status; - unsigned int i; - int ret; - - lockdep_assert_held(&atcphy->lock); switch (rate) { case 0x06: - selector = 4; - break; + return 4; case 0x0a: - selector = 3; - break; + return 3; case 0x14: - selector = 1; - break; + return 1; case 0x1e: - selector = 0; - break; + return 0; default: return -EINVAL; } - if (atcphy->tunnel_saved) { - if (atcphy->tunnel_rate == rate) - return 0; - /* A retrain may change rate without first requesting rate zero. */ - atc_tunnel_restore(atcphy); - } +} + +/* + * Bring the T602X tunnel PLL up with its fixed descriptor. The rate is not + * part of it: each DP IN adapter selects its own rate on PCLK1 or PCLK2 in + * apple_atc_dp_tunnel_rate(). + */ +static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy) +{ + u32 value, gates, outputs, command, status; + unsigned int i; + int ret; + + lockdep_assert_held(&atcphy->lock); + if (atcphy->tunnel_saved) + return 0; if (atcphy->tunnel_attempted) return -EALREADY; /* Read every guard before deciding, so a refusal records the whole state. */ @@ -2348,9 +2358,6 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) udelay(2); core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(2)); core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(3)); - core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); - core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, - FIELD_PREP(DPTX_PCLK1_SELECT, selector)); /* Program the fixed tunnel clock descriptor. */ core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); @@ -2376,13 +2383,106 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy, u8 rate) ret = atc_tunnel_command(atcphy, 0x2000); if (ret) goto restore; - atcphy->tunnel_rate = rate; return 0; restore: atc_tunnel_restore(atcphy); return ret; } +/* Stop one T602X DP IN adapter's pixel clock; atcphy->lock held. */ +static void atc_tunnel_stop_t602x(struct apple_atcphy *atcphy, unsigned int dpin) +{ + u8 user = BIT(dpin); + + lockdep_assert_held(&atcphy->lock); + if (atcphy->tunnel_users & user) { + if (dpin == 0) { + /* + * DPIN1 still needs the PCLK1 gate. On J414s, clearing + * it stopped DPIN1's video packets even though PCLK2 + * remained enabled and locked. + */ + if (!(atcphy->tunnel_users & BIT(1))) + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + } else { + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK2_ENABLE); + if (!(atcphy->tunnel_users & BIT(0))) + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + } + } + atcphy->tunnel_users &= ~user; + atcphy->tunnel_dpin_rate[dpin] = 0; + /* + * Gate the PLL outputs but keep the descriptor across DPMS: the PLL + * keeps its lock with its outputs gated, and the next start re-enables + * them. A mode change still powers it down in atc_tunnel_restore(). + */ + if (!atcphy->tunnel_users && atcphy->tunnel_saved) + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); +} + +/* Run one T602X DP IN adapter's pixel clock at @rate; atcphy->lock held. */ +static int atc_tunnel_set_t602x(struct apple_atcphy *atcphy, unsigned int dpin, + u8 rate) +{ + int selector, ret = 0; + u32 value; + + lockdep_assert_held(&atcphy->lock); + selector = atc_tunnel_rate_selector(rate); + if (selector < 0) + return selector; + if (atcphy->tunnel_dpin_rate[dpin] == rate) + return 0; + if (!atcphy->tunnel_users) { + if (!atcphy->tunnel_saved) { + ret = atc_tunnel_start_t602x(atcphy); + } else { + /* + * The PLL retains lock after its outputs are gated. Keep + * its descriptor until the USB4 PHY changes mode instead of + * reprogramming a still-locked PLL on every DPMS wake. + */ + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, + value, value & ACIOPHY_AUSPLL_LOCK, + 10, 10000); + if (ret) + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + } + if (ret) + return ret; + } + if (dpin == 0) { + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_SELECT, FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + } else { + /* + * PCLK2 alone does not sustain a DPIN1 stream on J414s. Keep + * PCLK1 gated on as a shared prerequisite; if DPIN0 is inactive, + * select DPIN1's rate for that clock as well. + */ + if (!(atcphy->tunnel_users & BIT(0))) + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_SELECT, + FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK2_SELECT, FIELD_PREP(DPTX_PCLK2_SELECT, selector)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK2_ENABLE); + } + atcphy->tunnel_dpin_rate[dpin] = rate; + atcphy->tunnel_users |= BIT(dpin); + return 0; +} + static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) { int ret = 0; @@ -2660,8 +2760,8 @@ static const struct phy_ops apple_atc_dp_phy_ops = { }; /* - * Called by appledrm when DCP sets the link rate of a DPTX that feeds a - * Thunderbolt DP IN adapter (rate is the DP link rate code, 0 = stop). + * Called by appledrm when DCP sets the link rate of a DPTX that feeds + * Thunderbolt DP IN adapter @dpin (rate is the DP link rate code, 0 = stop). */ static bool apple_atc_is_typec_core(u64 base) { @@ -2669,12 +2769,12 @@ static bool apple_atc_is_typec_core(u64 base) base == 0xf03000000ULL; } -int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) +int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) { struct apple_atcphy *atcphy; int ret; - if (!phy || phy->ops != &apple_atc_dp_phy_ops) + if (!phy || phy->ops != &apple_atc_dp_phy_ops || dpin > 1) return -EINVAL; atcphy = phy_get_drvdata(phy); /* Keep each supported SoC on its qualified clock sequence. */ @@ -2687,15 +2787,17 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate) return -EINVAL; guard(mutex)(&atcphy->lock); if (!rate) { - atc_tunnel_stop_t8103(atcphy); - atc_tunnel_restore(atcphy); + if (of_machine_is_compatible("apple,t8103")) + atc_tunnel_stop_t8103(atcphy); + else + atc_tunnel_stop_t602x(atcphy, dpin); return 0; } if (atcphy->mode != APPLE_ATCPHY_MODE_USB4 && atcphy->mode != APPLE_ATCPHY_MODE_TBT) return -EBUSY; ret = of_machine_is_compatible("apple,t8103") ? atc_tunnel_start_t8103(atcphy, rate) : - atc_tunnel_start_t602x(atcphy, rate); + atc_tunnel_set_t602x(atcphy, dpin, rate); dev_dbg(atcphy->dev, "DP tunnel clock rate 0x%x: %d (TX_DP_CTRL0=%08x PCLK_STAT=%08x)\n", rate, ret, readl(atcphy->regs.core + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0), readl(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT)); diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index 8117ba0cea949..4d2ab4eeb07da 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -29,7 +29,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, * ATC PHY: start the pixel clock for a DP tunnel at DP link rate code @rate, * or stop it (@rate == 0). The PHY must be in Thunderbolt/USB4 mode. */ -int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); +int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate); /* * Display crossbar: take the connection of an already selected output down or @@ -40,6 +40,7 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, u8 rate); */ int apple_dpxbar_link_down(struct mux_control *mux); int apple_dpxbar_link_up(struct mux_control *mux); +int apple_dpxbar_tunnel_select_source(struct mux_control *mux, int state); /* * The M2 Pro and M2 Max laptops. Their shared device tree routes the display From b203d1117b0bca977604cf65f87e0a7bb63cba9a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 26 Sep 2026 18:59:19 -0400 Subject: [PATCH 0318/1579] thunderbolt: account live USB3 use on Apple hosts without BW allocation Apple host USB3 adapters do not acknowledge bandwidth-allocation writes, so their software-only initial allocation should not be treated as consumed USB3 bandwidth when admitting DP tunnels. Read the downstream router USB3 UP adapter consumed-bandwidth counters instead, retaining 900 Mb/s minimum headroom for isochronous traffic. If those counters cannot be read, keep the existing conservative allocation estimate. The change does not cap USB3 bulk traffic or write the unsupported host registers. The m2fix48 dock test reached Dell 4K120 and JTG 60 with USB devices enumerated; a standalone throughput or isochronous stress test has not been performed. [Chris Kearney: applies to every Apple host, since apple.c sets QUIRK_NO_USB3_BW_ALLOC on all of them, the t8103 included.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/thunderbolt/tb.h | 2 ++ drivers/thunderbolt/tunnel.c | 31 +++++++++++++++++++++++++++++-- drivers/thunderbolt/usb4.c | 11 +++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h index 9a1af4969c9ca..cef2ed4799f30 100644 --- a/drivers/thunderbolt/tb.h +++ b/drivers/thunderbolt/tb.h @@ -1482,6 +1482,8 @@ int usb4_port_retimer_nvm_read(struct tb_port *port, u8 index, unsigned int address, void *buf, size_t size); int usb4_usb3_port_max_link_rate(struct tb_port *port); +int usb4_usb3_port_consumed_bandwidth(struct tb_port *port, int *upstream_bw, + int *downstream_bw); int usb4_usb3_port_allocated_bandwidth(struct tb_port *port, int *upstream_bw, int *downstream_bw); int usb4_usb3_port_allocate_bandwidth(struct tb_port *port, int *upstream_bw, diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index f3a05c6c2a1a3..b54079e3ced11 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -2375,14 +2375,41 @@ static int tb_usb3_consumed_bandwidth(struct tb_tunnel *tunnel, { struct tb_port *port = tb_upstream_port(tunnel->dst_port->sw); int pcie_weight = tb_acpi_may_tunnel_pcie() ? TB_PCI_WEIGHT : 0; + int allocated_up = tunnel->allocated_up; + int allocated_down = tunnel->allocated_down; + int ret; + + if (tunnel->src_port->sw->no_usb3_bw_alloc) { + /* + * Apple's host USB3 adapter cannot acknowledge bandwidth + * allocation requests. Its software-only initial allocation is a + * maximum, not bandwidth actually occupied by USB3. The device + * router's USB3 UP adapter still reports consumed bandwidth; use + * that live value for DP admission without limiting USB3 bulk + * traffic or writing the unsupported host registers. + */ + ret = usb4_usb3_port_consumed_bandwidth(tunnel->dst_port, + &allocated_up, &allocated_down); + if (ret) { + /* Preserve the conservative allocation-based estimate when + * a downstream adapter lacks readable consumed counters. + */ + allocated_up = tunnel->allocated_up; + allocated_down = tunnel->allocated_down; + } else { + /* Leave headroom for new isochronous USB3 transfers. */ + allocated_up = max(allocated_up, 900); + allocated_down = max(allocated_down, 900); + } + } /* * PCIe tunneling, if enabled, affects the USB3 bandwidth so * take that into account here. */ - *consumed_up = tunnel->allocated_up * + *consumed_up = allocated_up * (TB_USB3_WEIGHT + pcie_weight) / TB_USB3_WEIGHT; - *consumed_down = tunnel->allocated_down * + *consumed_down = allocated_down * (TB_USB3_WEIGHT + pcie_weight) / TB_USB3_WEIGHT; if (tb_port_get_link_generation(port) >= 4) { diff --git a/drivers/thunderbolt/usb4.c b/drivers/thunderbolt/usb4.c index 9e810b2ae0b5f..c358c8e77f7ec 100644 --- a/drivers/thunderbolt/usb4.c +++ b/drivers/thunderbolt/usb4.c @@ -2360,6 +2360,17 @@ static int usb4_usb3_port_read_consumed_bandwidth(struct tb_port *port, return 0; } +int usb4_usb3_port_consumed_bandwidth(struct tb_port *port, int *upstream_bw, + int *downstream_bw) +{ + if (!tb_port_is_usb3_up(port)) + return -EINVAL; + + /* Device-router USB3 UP adapters expose these counters without CMR. */ + return usb4_usb3_port_read_consumed_bandwidth(port, upstream_bw, + downstream_bw); +} + static int usb4_usb3_port_write_allocated_bandwidth(struct tb_port *port, int upstream_bw, int downstream_bw) From 8580f66efb64724316fb9576f09db3772d787081 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 27 Sep 2026 00:57:55 -0400 Subject: [PATCH 0319/1579] arm64: configs: asahi: build the USB4 connection manager and Apple NHI Build the USB4 connection manager and Apple NHI with the Asahi profile. Keep unrelated SEP and host configuration out of this stack. [Chris Kearney: ported onto aurora-silicon/linux#8 from "arm64: enable Apple USB4 modules and J414/J416 firmware aliases". The J414/J416 underscore aliases are already on this branch, so only the configuration is carried.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- arch/arm64/configs/asahi.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 20a50b6fcb3a1..3cbfea53329e3 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -48,6 +48,8 @@ CONFIG_HID_DOCKCHANNEL=m CONFIG_SPI_HID_APPLE_OF=m CONFIG_SPI_HID_APPLE_CORE=m CONFIG_USB_DWC3_APPLE=m +CONFIG_USB4=m +CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_RTC_DRV_MACSMC=m CONFIG_APPLE_ADMAC=m From 3a0d5bc83479d415de679e92183b5606521b13fa Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 09:54:14 -0600 Subject: [PATCH 0320/1579] mux: apple-display-crossbar: let a DP IN output be pointed at its source early A Thunderbolt DP tunnel on t8103 leaves the crossbar's DP IN output at its idle source, dispext0, until DCP reports DidChangeLinkConfig. That works there because dispext0 is the only external pipeline. On t600x the Type-C ports are driven by dcpext1 (mux index 2), and a pipeline whose output still points at dispext0 activates the link, gets no answer from the sink and gives up before it sets a link rate. Add apple_dpxbar_preselect(), which writes only the source select of a DP IN output that is not selected yet. The connection itself still comes up in the normal mux selection. MUX_IDLE_DISCONNECT puts the output back at its idle source. Also skip the FIFO_RD_PCLK1 cycle-slip toggle for DP IN on t6000, which runs without it, as found on j316c in omacom/linux#9. The DP PHY output keeps the toggle. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-Authored-By: Claude Opus 5.5 [Chris Kearney: ported onto aurora-silicon/linux#8; comment and message reworded to describe the observed behaviour only.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/mux/apple-display-crossbar.c | 78 +++++++++++++++++++++++++--- include/linux/soc/apple/dp-tunnel.h | 8 +++ 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index 9a50dc7eb70b9..f49977b846f04 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -93,10 +93,13 @@ struct apple_dpxbar_hw { unsigned int n_ufp; u32 tunable; const struct mux_control_ops *ops; + /* DP IN runs without the FIFO_RD_PCLK1 cycle-slip toggle (t600x) */ + bool dpin_no_cycle_slip; }; struct apple_dpxbar { struct device *dev; + const struct apple_dpxbar_hw *hw; void __iomem *regs; int selected_dispext[MUX_MAX]; spinlock_t lock; @@ -361,6 +364,12 @@ static void apple_dpxbar_check_gates(struct apple_dpxbar *dpxbar, unsigned int i apple_dpxbar_names[index], wr, rd, out); } +/* The RD_PCLK toggle below is needed for the PHY output, and for DP IN on some SoCs. */ +static bool apple_dpxbar_cycle_slip(struct apple_dpxbar *dpxbar, unsigned int index) +{ + return index == MUX_DPPHY || !dpxbar->hw->dpin_no_cycle_slip; +} + static int apple_dpxbar_set(struct mux_control *mux, int state) { struct apple_dpxbar *dpxbar = mux_chip_priv(mux->chip); @@ -479,9 +488,11 @@ static int apple_dpxbar_set(struct mux_control *mux, int state) * 5 usec is required which is doubled here to be on the * safe side. */ - dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); - udelay(10); - dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + if (apple_dpxbar_cycle_slip(dpxbar, index)) { + dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + udelay(10); + dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + } dpxbar->selected_dispext[index] = state; } @@ -585,9 +596,11 @@ static int apple_dpxbar_t8103_link_up(struct mux_control *mux) dpxbar_set32(dpxbar, OUT_PCLK1_EN, atc_bit); dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); dpxbar_set32(dpxbar, CROSSBAR_DISPEXT_EN, dispext_bit); - dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); - udelay(10); - dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + if (apple_dpxbar_cycle_slip(dpxbar, index)) { + dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + udelay(10); + dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + } } spin_unlock_irqrestore(&dpxbar->lock, flags); if (ret) @@ -601,6 +614,57 @@ static int apple_dpxbar_t8103_link_up(struct mux_control *mux) return 0; } +int apple_dpxbar_preselect(struct mux_control *mux, int state) +{ + struct apple_dpxbar *dpxbar; + unsigned int index, mux_state; + unsigned long flags; + u32 mask, set; + int ret = 0; + + if (!mux) + return -EINVAL; + dpxbar = apple_dpxbar_from_mux(mux, &index); + if (!dpxbar) + return -EOPNOTSUPP; + if (state == MUX_IDLE_DISCONNECT) + mux_state = 0; + else if (state >= 0 && state < 9) + mux_state = state; + else + return -EINVAL; + + switch (index) { + case MUX_DPIN0: + mask = CROSSBAR_MUX_CTRL_DPIN0_SELECT0 | CROSSBAR_MUX_CTRL_DPIN0_SELECT1; + set = FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN0_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN0_SELECT1, mux_state); + break; + case MUX_DPIN1: + mask = CROSSBAR_MUX_CTRL_DPIN1_SELECT0 | CROSSBAR_MUX_CTRL_DPIN1_SELECT1; + set = FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN1_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN1_SELECT1, mux_state); + break; + default: + return -EOPNOTSUPP; + } + + spin_lock_irqsave(&dpxbar->lock, flags); + /* a selected output already points at its source */ + if (dpxbar->selected_dispext[index] >= 0) + ret = -EBUSY; + else + dpxbar_mask32(dpxbar, CROSSBAR_MUX_CTRL, mask, set); + spin_unlock_irqrestore(&dpxbar->lock, flags); + if (ret) + return ret; + + dev_dbg(dpxbar->dev, "%s: source preselected, MUX_CTRL=%08x\n", + apple_dpxbar_names[index], readl(dpxbar->regs + CROSSBAR_MUX_CTRL)); + return 0; +} +EXPORT_SYMBOL_GPL(apple_dpxbar_preselect); + static const struct mux_control_ops apple_dpxbar_ops = { .set = apple_dpxbar_set, }; @@ -772,6 +836,7 @@ static int apple_dpxbar_probe(struct platform_device *pdev) return PTR_ERR(mux_chip); dpxbar = mux_chip_priv(mux_chip); + dpxbar->hw = hw; mux_chip->ops = hw->ops; spin_lock_init(&dpxbar->lock); @@ -815,6 +880,7 @@ static const struct apple_dpxbar_hw apple_dpxbar_hw_t6000 = { .n_ufp = 9, .tunable = 5, .ops = &apple_dpxbar_ops, + .dpin_no_cycle_slip = true, }; static const struct apple_dpxbar_hw apple_dpxbar_hw_t6020 = { diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index 4d2ab4eeb07da..6156aed1f169c 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -57,4 +57,12 @@ static inline bool apple_dp_tunnel_t602x(void) return of_machine_compatible_match(machines); } +/* + * Display crossbar: point a DP IN output that is not selected yet at source + * @state without bringing the connection up, or back at its idle source + * (MUX_IDLE_DISCONNECT). The later mux selection does the rest. + * -EOPNOTSUPP on T602X crossbars. + */ +int apple_dpxbar_preselect(struct mux_control *mux, int state); + #endif From 7131c08e0f0acd570fd1f5121dd7803f5f3b0823 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 09:54:14 -0600 Subject: [PATCH 0321/1579] phy: apple: atc: run the t8103 tunnel clock sequence on t600x The M1 Pro/Max ATC is the t8103 generation, and the Thunderbolt DP tunnel pixel clock is set up the same way. Take t6000 and t6001 through atc_tunnel_start_t8103(). Unlike t8103, which feeds the tunnel from PCLK1 alone, t600x also runs PCLK2 and the DP IN (DPRX) clock with the same rate selector. Split the sleep overrides and the TX_DP_CTRL0 setup out of the start function and add apple_atc_dp_tunnel_open(), which applies both ahead of the first rate request. On t600x DCP does not set a link rate until the DP clock path is awake, so the display driver calls this before it tells DCP about the display. Other SoCs return -EOPNOTSUPP and keep doing all of it in the rate call. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-Authored-By: Claude Opus 5.5 [Chris Kearney: ported onto aurora-silicon/linux#8 on top of the per-DP-IN T602X tunnel clock; t600x takes the t8103 path for both start and stop.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/phy/apple/atc.c | 156 +++++++++++++++++++--------- include/linux/soc/apple/dp-tunnel.h | 7 ++ 2 files changed, 114 insertions(+), 49 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index d1051e2581470..5965f94a5bcc4 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2026,6 +2026,13 @@ static int atcphy_power_on(struct apple_atcphy *atcphy) * without touching the lane mux. Based on Oliver Lukschander's t6020 tunnel * clock patch (asahi-j416s-display). */ +/* t600x (M1 Pro/Max) has the t8103 generation ATC. */ +static bool apple_atc_tunnel_is_t600x(void) +{ + return of_machine_is_compatible("apple,t6000") || + of_machine_is_compatible("apple,t6001"); +} + /* * Stop the tunnel pixel clock: switch the AUSPLL output drivers off and send * APB command 3. TX_DP_CTRL0, the sleep overrides and the @@ -2043,6 +2050,69 @@ static void atc_tunnel_stop_t8103(struct apple_atcphy *atcphy) dev_dbg(atcphy->dev, "DP tunnel clock stopped\n"); } +/* + * TX_DP_CTRL0: PMA lane reset release, pixel clock enable + rate selector. + * t8103 feeds the tunnel from PCLK1 alone; t600x also runs PCLK2 and the DP IN + * (DPRX) clock at the same rate. + */ +static void atc_tunnel_pclk_t8103(struct apple_atcphy *atcphy, u32 selector) +{ + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N_OV); + if (apple_atc_tunnel_is_t600x()) { + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPRX_PCLK_ENABLE); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK2_ENABLE); + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPRX_PCLK_SELECT, + FIELD_PREP(DPRX_PCLK_SELECT, selector)); + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK2_SELECT, + FIELD_PREP(DPTX_PCLK2_SELECT, selector)); + } + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, + FIELD_PREP(DPTX_PCLK1_SELECT, selector)); +} + +static void atc_tunnel_wake_t8103(struct apple_atcphy *atcphy) +{ + /* + * The PHY is up in USB4/TBT mode here. These overrides only force the + * already awake blocks to stay awake for the DP clock path; the order + * and the 2 us gaps are the ones this was validated with, the USB4 + * link stays up across them. + * + * Common block: override small/big sleep off and release the clamp. + */ + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG_OV); + udelay(2); + core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP_OV); + udelay(2); + /* same for the TX lanes */ + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG_OV); + udelay(2); + core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV); + udelay(2); + /* and for the RX lanes */ + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG_OV); + udelay(2); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL_OV); + udelay(2); + core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP_OV); + udelay(2); +} + static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) { u32 selector, value; @@ -2070,10 +2140,9 @@ static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) return 0; /* * Rate change: DCP brackets it with WillChange/DidChange link - * configuration, so the crossbar is already down; reselect PCLK1. + * configuration, so the crossbar is already down; reselect the clocks. */ - core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, - FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + atc_tunnel_pclk_t8103(atcphy, selector); atcphy->tunnel_rate = rate; return 0; } @@ -2094,49 +2163,8 @@ static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) /* the PLL no longer holds a DP alt mode configuration */ atcphy->dp_link_rate = -1; - /* - * The PHY is up in USB4/TBT mode here. These overrides only force the - * already awake blocks to stay awake for the DP clock path; the order - * and the 2 us gaps are the ones this was validated with, the USB4 - * link stays up across them. - * - * Common block: override small/big sleep off and release the clamp. - */ - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL_OV); - udelay(2); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG_OV); - udelay(2); - core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP_OV); - udelay(2); - /* same for the TX lanes */ - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL_OV); - udelay(2); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG_OV); - udelay(2); - core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV); - udelay(2); - /* and for the RX lanes */ - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG_OV); - udelay(2); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL_OV); - udelay(2); - core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP_OV); - udelay(2); - /* TX_DP_CTRL0: PMA lane reset release, PCLK1 enable + rate selector */ - core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N); - core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N_OV); - core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); - core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, - FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + atc_tunnel_wake_t8103(atcphy); + atc_tunnel_pclk_t8103(atcphy, selector); /* fixed AUSPLL descriptor */ core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); @@ -2769,6 +2797,12 @@ static bool apple_atc_is_typec_core(u64 base) base == 0xf03000000ULL; } +/* t600x (M1 Pro/Max) runs the t8103 tunnel clock sequence unchanged. */ +static bool apple_atc_tunnel_is_t8103_style(void) +{ + return of_machine_is_compatible("apple,t8103") || apple_atc_tunnel_is_t600x(); +} + int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) { struct apple_atcphy *atcphy; @@ -2778,7 +2812,7 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) return -EINVAL; atcphy = phy_get_drvdata(phy); /* Keep each supported SoC on its qualified clock sequence. */ - if (!of_machine_is_compatible("apple,t8103") && !apple_dp_tunnel_t602x()) + if (!apple_atc_tunnel_is_t8103_style() && !apple_dp_tunnel_t602x()) return -EOPNOTSUPP; if (apple_dp_tunnel_t602x() && (!of_device_is_compatible(atcphy->np, "apple,t6020-atcphy") || @@ -2787,7 +2821,7 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) return -EINVAL; guard(mutex)(&atcphy->lock); if (!rate) { - if (of_machine_is_compatible("apple,t8103")) + if (apple_atc_tunnel_is_t8103_style()) atc_tunnel_stop_t8103(atcphy); else atc_tunnel_stop_t602x(atcphy, dpin); @@ -2795,7 +2829,7 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) } if (atcphy->mode != APPLE_ATCPHY_MODE_USB4 && atcphy->mode != APPLE_ATCPHY_MODE_TBT) return -EBUSY; - ret = of_machine_is_compatible("apple,t8103") ? + ret = apple_atc_tunnel_is_t8103_style() ? atc_tunnel_start_t8103(atcphy, rate) : atc_tunnel_set_t602x(atcphy, dpin, rate); dev_dbg(atcphy->dev, "DP tunnel clock rate 0x%x: %d (TX_DP_CTRL0=%08x PCLK_STAT=%08x)\n", @@ -2805,6 +2839,30 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) } EXPORT_SYMBOL_GPL(apple_atc_dp_tunnel_rate); +int apple_atc_dp_tunnel_open(struct phy *phy) +{ + struct apple_atcphy *atcphy; + + if (!phy || phy->ops != &apple_atc_dp_phy_ops) + return -EINVAL; + if (!apple_atc_tunnel_is_t600x()) + return -EOPNOTSUPP; + atcphy = phy_get_drvdata(phy); + guard(mutex)(&atcphy->lock); + if (atcphy->mode != APPLE_ATCPHY_MODE_USB4 && atcphy->mode != APPLE_ATCPHY_MODE_TBT) + return -EBUSY; + if (!atcphy->tunnel_clock_on) { + atc_tunnel_wake_t8103(atcphy); + atc_tunnel_pclk_t8103(atcphy, 0); + } + dev_dbg(atcphy->dev, "DP tunnel open (CFG0=%08x SLEEP_CTRL=%08x TX_DP_CTRL0=%08x)\n", + readl(atcphy->regs.core + ACIOPHY_CFG0), + readl(atcphy->regs.core + ACIOPHY_SLEEP_CTRL), + readl(atcphy->regs.core + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0)); + return 0; +} +EXPORT_SYMBOL_GPL(apple_atc_dp_tunnel_open); + static struct phy *atcphy_xlate(struct device *dev, const struct of_phandle_args *args) { struct apple_atcphy *atcphy = dev_get_drvdata(dev); diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index 6156aed1f169c..174173d46f714 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -31,6 +31,13 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, */ int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate); +/* + * ATC PHY: wake the DP clock path of a PHY in Thunderbolt/USB4 mode ahead of + * apple_atc_dp_tunnel_rate(), before DCP is told about the display. + * -EOPNOTSUPP where the rate call does all of it (everything but t600x). + */ +int apple_atc_dp_tunnel_open(struct phy *phy); + /* * Display crossbar: take the connection of an already selected output down or * bring it back up (clock gates and enables) around a link reconfiguration. From dcb972b1ce1c1e01ba48b826f95c9791dd2379b3 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 09:54:14 -0600 Subject: [PATCH 0322/1579] drm/apple: prepare crossbar and PHY before DCP hears about a tunneled display With only the SoC checks widened, a Thunderbolt display on the MacBook Pro 14" M1 Pro (j314s) got as far as an active DP IN adapter. DCP then stayed silent for 5 s after Activate, every DPTX call timed out, and it reported link faults 22 and 24 without ever sending SET_LINK_RATE. All three Type-C routes of j314s use dcpext1 (apple,typec-mux-indices = <2 2 2>), while the DP IN output idles at dispext0. Point the output at the pipeline that takes the tunnel and wake the ATC's DP clock path before the DPTX connect, and put the output back at its idle source when the route goes away without the crossbar having come up. Both helpers return -EOPNOTSUPP where they do not apply, so t8103 and T602X behave as before. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-Authored-By: Claude Opus 5.5 [Chris Kearney: ported onto aurora-silicon/linux#8 on top of the dual DP tunnel series; the tunnel is prepared once its slot is taken.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/dcp.c | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a1bf95b397660..574627ae94ead 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -133,6 +133,7 @@ static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); +static int dcp_dpxbar_preselect(struct mux_control *mux, int state); static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); static int dcp_dpxbar_tunnel_select_source(struct mux_control *mux, int state) @@ -273,6 +274,8 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) scoped_guard(mutex, &dcp->tb_lock) { if (!route->tunnel || route->xbar_up) ret = mux_control_deselect(route->active_xbar ?: route->xbar); + else + dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); if (ret) return ret; route->xbar_up = false; @@ -557,6 +560,46 @@ static int dcp_dpxbar_link(struct mux_control *mux, bool up) return ret; } +static int dcp_dpxbar_preselect(struct mux_control *mux, int state) +{ + typeof(&apple_dpxbar_preselect) fn = symbol_get(apple_dpxbar_preselect); + int ret; + + if (!fn) + return -ENOENT; + ret = fn(mux, state); + symbol_put(apple_dpxbar_preselect); + return ret; +} + +/* + * Thunderbolt DP IN, before DCP hears about the display: point the DP IN + * output at this pipeline and wake the ATC's DP clock path. The crossbar + * connection itself still only comes up in dcp_tunnel_crossbar_up(), but an + * output left at its idle source (dispext0) only suits the pipeline that is + * dispext0: any other one activates the link, gets no answer from the sink + * and gives up with DEVICE_NOT_RESPONDING before it ever sets a link rate. + */ +static void dcp_tunnel_prepare(struct apple_dcp_typec_route *route, + struct mux_control *xbar) +{ + typeof(&apple_atc_dp_tunnel_open) open; + struct apple_dcp *dcp = route->dcp; + int ret; + + ret = dcp_dpxbar_preselect(xbar, route->mux_index); + if (ret && ret != -EOPNOTSUPP) + dev_warn(dcp->dev, "DP tunnel crossbar preselect failed: %d\n", ret); + + open = symbol_get(apple_atc_dp_tunnel_open); + if (!open) + return; + ret = open(dcp->phy); + symbol_put(apple_atc_dp_tunnel_open); + if (ret && ret != -EOPNOTSUPP) + dev_warn(dcp->dev, "DP tunnel PHY open failed: %d\n", ret); +} + /* DP IN adapter handshake through the thunderbolt glue; tb_lock held */ static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) { @@ -779,6 +822,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, return ret; } *slot = best; + dcp_tunnel_prepare(best, ctl); dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); From ef276c075c360df379849cefcd83bbbf8f15afaa Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 09:54:14 -0600 Subject: [PATCH 0323/1579] thunderbolt: apple: drive DP tunnels on t6000 and t6001 The M1 Pro/Max ATC has the t8103 DP IN adapter registers, the same crossbar and the same tunnel pixel clock sequence. Install the DP tunnel hook there too, and clear the DP IN interrupt status the way t8103 does. Tested on a MacBook Pro 14" M1 Pro (j314s) with an LG UltraFine 5K on both left ports: 3840x2160@60 at cold boot, on hotplug and after suspend-to-idle. t6001 is untested. apple-display-crossbar 70304c000.mux: dpin0: source preselected, MUX_CTRL=00002002 phy-apple-atc 703000000.phy: DP tunnel open (CFG0=11833fef SLEEP_CTRL=15570cff TX_DP_CTRL0=0000e00d) apple-dcp 28cc00000.dcp: display routed to Thunderbolt DP tunnel dpin0 apple-dcp 28cc00000.dcp: DPTXPort: SET_LINK_RATE 0x14 apple-display-crossbar 70304c000.mux: Switched dpin0 to dispext1,0 apple-dcp 28cc00000.dcp: dcp_hotplug() connected:1 valid_mode:0 nr_modes:5 The second DP tunnel of the display is still refused ("port already routed"), so it runs as a single 4K stream, not as two 5K tiles. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-Authored-By: Claude Opus 5.5 [Chris Kearney: ported onto aurora-silicon/linux#8, where dp_display defaults to on; t600x takes that default too.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- Documentation/admin-guide/kernel-parameters.txt | 9 +++++---- drivers/thunderbolt/apple.c | 17 ++++++++++++++--- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index c87bd53bde7a3..2ebdd37817b2e 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7613,10 +7613,11 @@ Kernel parameters [HW,ARM64] Format: Drive displays behind Thunderbolt DisplayPort tunnels - on Apple t8103 (M1): route a display pipeline to the - host router's DP IN adapter whenever the connection - manager sets up a DP tunnel. Set to 0 to leave displays - behind a dock dark. Default: true + on Apple t8103 and t600x (M1, M1 Pro and M1 Max): + route a display pipeline to the host router's DP IN + adapter whenever the connection manager sets up a DP + tunnel. Set to 0 to leave displays behind a dock dark. + Default: true thunderbolt_apple.dpin_aux= [HW,ARM64] diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 244e47c4f012b..905415690a855 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -210,7 +210,18 @@ struct apple_cio { static bool dp_display = true; module_param(dp_display, bool, 0444); -MODULE_PARM_DESC(dp_display, "Drive displays behind Thunderbolt DP tunnels on t8103 (default: true)"); +MODULE_PARM_DESC(dp_display, "Drive displays behind Thunderbolt DP tunnels on t8103 and t600x (default: true)"); + +/* + * The M1 Pro/Max ATC is the t8103 generation: same DP IN adapter registers, + * same crossbar and the same tunnel pixel clock sequence. + */ +static bool apple_cio_dp_is_t8103_style(void) +{ + return of_machine_is_compatible("apple,t8103") || + of_machine_is_compatible("apple,t6000") || + of_machine_is_compatible("apple,t6001"); +} /* DPTX_INACTIVE handshake: request (in)active, wait for the ACK */ struct apple_dpin_poll { @@ -391,7 +402,7 @@ static int apple_dpin_up(struct apple_dpin_ctx *c) * back. Nothing handles these interrupts yet; they are enabled * as part of bringing the adapter up. */ - if (of_machine_is_compatible("apple,t8103")) { + if (apple_cio_dp_is_t8103_style()) { irq = readl(regs + APPLE_DPIN_IRQ_STATUS); writel(irq, regs + APPLE_DPIN_IRQ_STATUS); if (irq & BIT(0)) @@ -1615,7 +1626,7 @@ static int apple_nhi_probe(struct platform_device *pdev) * the connection manager) only where it is enabled and known to work. */ anhi->ops = apple_nhi_ops; - if (dp_display && acio->dp_wq && of_machine_is_compatible("apple,t8103")) + if (dp_display && acio->dp_wq && apple_cio_dp_is_t8103_style()) anhi->ops.dp_tunnel_changed = apple_nhi_dp_tunnel_changed; if (acio->dp_wq && apple_dp_tunnel_t602x()) { anhi->ops.dp_tunnel_pre_activate = apple_nhi_dp_tunnel_pre_activate; From b5327f6f3062d0c4a926ce67561ebe90710539ac Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 09:54:14 -0600 Subject: [PATCH 0324/1579] drm/apple: a late clear swap at power-off is not a DCP crash iomfb_poweroff() waits 50 ms for the clear swap and sets dcp->crashed when that times out. When a Thunderbolt display is unplugged, the firmware is busy powering the external pipe down on its own and answers the swap a little later. Nothing logs the timeout, but from then on dcp_crtc_atomic_check() fails every commit with -EINVAL, so a replugged display stays dark until reboot: [CRTC:73] atomic driver check failed A real firmware crash is reported through the RTKit crash callback. Wait 500 ms, warn on a timeout and carry on with the power-off. Seen on j314s with an LG UltraFine 5K: unplug, then replug. With this change the unplug logs "dcp_poweroff() done" and the display comes back on replug. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-Authored-By: Claude Opus 5.5 [Chris Kearney: ported onto aurora-silicon/linux#8, where DjDeveloperr's "preserve Type-C routes across blanking and detach" already stopped setting crashed but returned early; this carries on with the power-off. This is also the DCP unplug wedge Oliver Lukschander reported on #8.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/iomfb_template.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index b032bbd2a4b42..f05b601de1324 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -946,19 +946,21 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); - ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(50)); + /* + * On unplug the firmware powers the external pipe down on its own and + * can take tens of milliseconds before it answers (and swallows) the + * clear swap. That is not a crash: a real one is reported through the + * RTKit crash callback. Wait longer and carry on with the power-off + * either way, otherwise every later modeset fails with -EINVAL. + */ + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); swap_id = cookie->swap_id; kref_put(&cookie->refcount, release_swap_cookie); - if (ret <= 0) { - /* A disconnected external display can power down before it ACKs this - * clear swap. The deadline alone does not mean RTKit crashed; its - * crash callback owns the fatal flag. - */ - dev_warn(dcp->dev, "clear swap timed out during power-off\n"); - return; - } - - dev_dbg(dcp->dev, "%s: clear swap submitted: %u\n", __func__, swap_id); + if (ret <= 0) + dev_warn(dcp->dev, "%s: clear swap timed out\n", __func__); + else + dev_dbg(dcp->dev, "%s: clear swap submitted: %u\n", __func__, + swap_id); poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); if (!poff_cookie) From 9236263be366efab73ae50c7b2d5444b4e518517 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 10:58:17 -0700 Subject: [PATCH 0325/1579] drm/apple: expose Type-C display connectors as DisplayPort The per-port Type-C connectors carry DisplayPort (DP alt mode or a Thunderbolt/USB4 DP tunnel), but were registered as DRM_MODE_CONNECTOR_USB, which DRM reserves for USB display adapters (gud, appletbdrm). Userspace keys on the type: systemd-logind only counts VGA/DVI/DP/HDMI/TV connectors as external displays, so a docked lid close suspended the machine instead of honoring HandleLidSwitchDocked=ignore. Register them as DisplayPort. The driver's internal "is Type-C" checks use dcp->connector_type / fixed_connector_type and are unaffected. Connectors are now named DP-N instead of USB-N. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan --- drivers/gpu/drm/apple/apple_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index d0fee4140a77b..71d7ee6456822 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -457,7 +457,7 @@ static int apple_probe_typec_ports(struct drm_device *drm, ret = drm_connector_init(drm, &connector->base, &apple_connector_funcs, - DRM_MODE_CONNECTOR_USB); + DRM_MODE_CONNECTOR_DisplayPort); if (ret) { kfree(connector); return ret; From 947026902232499a04414aeeb2994a3b557feb36 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 22:53:20 -0700 Subject: [PATCH 0326/1579] drm/apple: Keep dual-stream Type-C possible_crtcs fixed A Type-C port's encoder spans every pipeline that can drive the port, and routing narrows its possible_crtcs to the pipeline actually chosen, relying on userspace to re-read them on the following hotplug. Compositors that read possible_crtcs once when the connector appears (aquamarine, used by Hyprland) never see that. With a USB4 dock carrying two streams, whichever connector connects first takes the lowest free CRTC. When that is the DPIN1 connector, it gets dcpext0's CRTC while the stream is routed to dcpext1, every modeset fails the atomic check with -EINVAL, and both dock displays stay dark until the dock is replugged and happens to connect in the other order. possible_crtcs is meant to be static. On the dual-stream machines (J414s, J416c): - Fix the DPIN1 connector to the lowest-indexed Type-C-only pipeline (dcpext1), where DPIN1 is always routed, and route DPIN1 only within the connector's possible_crtcs. - Leave the primary connector spanning all its pipelines and stop narrowing or restoring possible_crtcs at runtime. - Route direct DP-alt displays to the lowest free CRTC, which is what a compositor picks from the port's fixed possible_crtcs, instead of steering them away from dcpext0. A direct DP-alt display now takes dcpext0 when it is free, so a single-stream dock on another port lands on the next free pipeline. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan [Chris Kearney: ported onto aurora-silicon/linux#8. dcp_typec_dual_stream() is apple_dp_tunnel_t602x() here, so j414c and j416s are dual-stream too; they share the j414/j416 device tree and the dcpext0/dcpext1 layout.] Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> --- drivers/gpu/drm/apple/apple_drv.c | 28 +++++++++--- drivers/gpu/drm/apple/dcp.c | 73 +++++++++++++++++++++++-------- drivers/gpu/drm/apple/dcp.h | 2 + 3 files changed, 79 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 71d7ee6456822..42f276c7ed2f5 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -270,9 +270,10 @@ apple_connector_atomic_best_encoder(struct drm_connector *conn, struct drm_encoder *encoder; /* - * A Type-C port has one encoder, and its possible_crtcs already names - * the pipeline the fabric routed the port to, so it is the only answer - * there is. + * A Type-C port has one encoder, whose possible_crtcs either already + * names the pipeline the fabric routed the port to or, on dual-stream + * machines, the fixed set the fabric only routes within, so it is the + * only answer there is. */ if (apple_connector->port_encoder) return apple_connector->port_encoder; @@ -440,7 +441,8 @@ static int apple_probe_typec_ports(struct drm_device *drm, * M2 Pro/Max laptops, append one more connector per port for a second * USB4 DP tunnel. */ - for (idx = 0; idx < nr_ports * (apple_dp_tunnel_t602x() ? 2 : 1); idx++) { + for (idx = 0; idx < nr_ports * (dcp_typec_dual_stream() ? 2 : 1); + idx++) { struct apple_connector *connector; struct apple_encoder *enc; unsigned int port_idx = idx % nr_ports; @@ -473,8 +475,22 @@ static int apple_probe_typec_ports(struct drm_device *drm, INIT_WORK(&connector->hotplug_wq, dcp_hotplug); for (i = 0; i < num_dcp; i++) { - if (dcp_typec_port_has_candidate(port_idx, dcp[i])) - mask |= crtc_mask[i]; + if (!dcp_typec_port_has_candidate(port_idx, dcp[i])) + continue; + /* + * A dock's second stream (DPIN1) always runs on the + * lowest-indexed Type-C-only pipeline (dcpext1). Fix the + * connector to it: compositors read possible_crtcs once + * and would otherwise pair it with a pipeline the stream + * can never be routed to. + */ + if (secondary) { + if (dcp_is_typec_only(dcp[i]) && + (!mask || crtc_mask[i] < mask)) + mask = crtc_mask[i]; + continue; + } + mask |= crtc_mask[i]; } if (!mask) { diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 574627ae94ead..88f558aa1d2b9 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -130,6 +130,39 @@ static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) !dcp_typec_route_fixed_output_busy(route); } +/* + * Machines whose USB4 docks carry two independent DP streams through one + * port: DPIN0 drives the hybrid dcpext0, DPIN1 the Type-C-only dcpext1, and + * each port has a second connector for the DPIN1 stream. + */ +bool dcp_typec_dual_stream(void) +{ + return apple_dp_tunnel_t602x(); +} + +bool dcp_is_typec_only(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return !dcp->fixed_phy; +} + +/* + * Can this route feed the given connector without changing its + * possible_crtcs? On dual-stream machines those are fixed at probe, since + * compositors read them once when the connector appears and pair the + * connector with a CRTC before this fabric has routed it. + */ +static bool dcp_typec_route_fits(struct apple_dcp_typec_route *route, + struct apple_connector *connector) +{ + struct apple_dcp *dcp = route->dcp; + + if (!dcp_typec_dual_stream() || !connector || !dcp->crtc) + return true; + return connector->candidate_crtcs & drm_crtc_mask(&dcp->crtc->base); +} + static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); @@ -237,9 +270,12 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * Offering it the choice makes it pair the port with a pipeline * holding a different monitor's mode list, and the modeset is * rejected with no way for it to recover. The hotplug that - * follows makes it re-read this. + * follows makes it re-read this. Dual-stream machines keep + * fixed possible_crtcs instead: compositors that read them once + * (e.g. aquamarine/Hyprland) never see the narrowing. */ - if (connector->port_encoder && dcp->crtc) + if (connector->port_encoder && dcp->crtc && + !dcp_typec_dual_stream()) connector->port_encoder->possible_crtcs = drm_crtc_mask(&dcp->crtc->base); } @@ -323,7 +359,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) connector->dcp = NULL; /* Unrouted: the port could go to any of its pipelines again. */ - if (connector->port_encoder) + if (connector->port_encoder && !dcp_typec_dual_stream()) connector->port_encoder->possible_crtcs = connector->candidate_crtcs; } @@ -491,15 +527,12 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (!dcp_typec_route_available(candidate)) continue; - score = dcp_typec_route_score(candidate); /* - * On the M2 Pro/Max laptops, leave dcpext0 free for - * a dock's DP tunnel when a direct DP-alt-mode - * display can use the Type-C-only dcpext1 pipeline. + * Lowest free CRTC index first: on dual-stream + * machines that is what a compositor picks from + * the port's fixed possible_crtcs. */ - if (apple_dp_tunnel_t602x() && - candidate->dcp->fixed_phy) - score += 100; + score = dcp_typec_route_score(candidate); if (score < best_score) { best = candidate; best_score = score; @@ -782,16 +815,19 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, if (!dcp_typec_route_available(candidate)) continue; + if (!dcp_typec_route_fits(candidate, dpin ? + port->secondary_connector : + port->connector)) + continue; score = dcp_typec_route_score(candidate); /* - * On the M2 Pro/Max laptops DPIN0 prefers the hybrid dcpext0, - * which completes tunneled link training, and DPIN1 prefers - * dcpext1, so the two pipelines drive independent streams - * through one dock. + * DPIN0 prefers the hybrid dcpext0, which completes tunneled + * link training. On dual-stream machines DPIN1 is confined to + * dcpext1 by its connector's possible_crtcs, so both pipelines + * drive independent streams through one dock. */ - if (apple_dp_tunnel_t602x() && score < UINT_MAX - 100 && - ((dpin == 0 && !candidate->dcp->fixed_phy) || - (dpin == 1 && candidate->dcp->fixed_phy))) + if (dcp_typec_dual_stream() && dpin == 0 && + !candidate->dcp->fixed_phy && score < UINT_MAX - 100) score += 100; if (score < best_score) { best = candidate; @@ -965,7 +1001,8 @@ void dcp_typec_port_set_connector(unsigned int idx, bool secondary, if (owner) { struct apple_dcp *dcp = owner->dcp; - if (dcp->crtc && connector->port_encoder) + if (dcp->crtc && connector->port_encoder && + !dcp_typec_dual_stream()) connector->port_encoder->possible_crtcs = drm_crtc_mask(&dcp->crtc->base); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 7e1047b0833f6..9c58284915997 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -46,6 +46,8 @@ struct device_node *dcp_typec_port_of_node(unsigned int idx); bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev); void dcp_typec_port_set_connector(unsigned int idx, bool secondary, struct apple_connector *connector); +bool dcp_typec_dual_stream(void); +bool dcp_is_typec_only(struct platform_device *pdev); bool dcp_fw_compat_is_12_x(struct platform_device *pdev); unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, From 7597600a2df11d650e11f93874ea0cf1b80d62f0 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Thu, 1 Oct 2026 16:04:13 +0000 Subject: [PATCH 0327/1579] usb: typec: tipd: accept Apple "DBMa" mode, add debugfs command access Apple's CD321x and SN201202x firmware has a debug mode, "DBMa". It is entered with an unlock key (LOCK) followed by the DBMa command. In this mode the application firmware still handles the port. It also accepts VDMs, which sends Apple VDMs (reboot, DFU, serial, Debug USB) to a connected Mac, and DVEn, which enables the local end of a debug route. tuxvdmtool and macvdmtool use these commands to control another Mac over its DFU port. A boot loader that arms Apple Debug USB on a port leaves its controller in DBMa. tipd then fails to probe it with 'controller in unsupported mode "DBMa"' (-ENODEV). The port gets no Type-C device and no kernel interface can reach the controller. SPMI-attached SN201202x controllers (M3 and later) also have no /dev/i2c-N fallback for userspace. Accept DBMa at probe and resume, as the Asahi tree has done since commit 786523ac62f0 ("usb: typec: tipd: Add cd321x specific control commands"). Unlike that commit, keep rejecting BIST and DISC. Use the controller as found: - The switch to the S0 power state runs before the mode check, as before. It sends SSPS only if the controller is not already in S0. - The patch/init step still runs only in PTCH mode. - Everything else is register reads plus the INT_MASK1 write. Probing therefore sends no command that leaves DBMa or disturbs a Debug USB route set up before Linux. Record whether DBMa was active at probe. Add a root-only interface at /sys/kernel/debug/usb/tipd//: command Write a 4CC followed by 0..64 bytes of DATA1. The command runs as a whole under the driver lock through tps6598x_exec_cmd_tmo(). Reading returns DATA1 as left by the last completed command of this open file. Only LOCK, DBMa, VDMs (200 ms timeout), DVEn and DISC are accepted. regs Read-only. pread() at offset = register (0..0x7f) returns up to 64 bytes, read under the driver lock. The interface works over both the i2c and SPMI transports. Unlike raw /dev/i2c access, it serialises with the interrupt handler. When an open command file that put the controller into DBMa is released, or the device is unbound while such a file is open, the controller leaves DBMa again (DBMa 0, then LOCK 0000). This is skipped if the controller was already in DBMa at probe, since that mode belongs to the boot loader's Debug USB setup. Controllers that are not in DBMa probe exactly as before, and nothing changes while the debugfs files are unused. The interface exists only with CONFIG_DEBUG_FS. Link: https://github.com/AsahiLinux/linux/commit/786523ac62f0 Link: https://github.com/aurora-silicon/tuxvdmtool Co-Authored-By: Claude Opus 5.5 (1M context) --- drivers/usb/typec/tipd/Kconfig | 1 + drivers/usb/typec/tipd/Makefile | 1 + drivers/usb/typec/tipd/core.c | 52 ++++-- drivers/usb/typec/tipd/debugfs.c | 292 ++++++++++++++++++++++++++++++ drivers/usb/typec/tipd/tps6598x.h | 30 +++ 5 files changed, 360 insertions(+), 16 deletions(-) create mode 100644 drivers/usb/typec/tipd/debugfs.c diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index 8177f860f89de..e832af6d2f6e9 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -2,6 +2,7 @@ config TYPEC_TPS6598X_CORE tristate select USB_ROLE_SWITCH select POWER_SUPPLY + select USB_COMMON if DEBUG_FS config TYPEC_TPS6598X tristate "TI TPS6598x USB Power Delivery controller driver" diff --git a/drivers/usb/typec/tipd/Makefile b/drivers/usb/typec/tipd/Makefile index 6f5ebc1e1a5fe..11bceec779bdd 100644 --- a/drivers/usb/typec/tipd/Makefile +++ b/drivers/usb/typec/tipd/Makefile @@ -4,6 +4,7 @@ CFLAGS_trace.o := -I$(src) obj-$(CONFIG_TYPEC_TPS6598X_CORE) += tps6598x-core.o tps6598x-core-y := core.o tps6598x-core-$(CONFIG_TRACING) += trace.o +tps6598x-core-$(CONFIG_DEBUG_FS) += debugfs.o obj-$(CONFIG_TYPEC_TPS6598X) += tps6598x.o tps6598x-y := i2c.o diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 130b1b8ecf8c2..0c3501cf19066 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -92,10 +92,6 @@ struct tps6598x_rx_identity_reg { struct usb_pd_identity identity; } __packed; -/* Standard Task return codes */ -#define TPS_TASK_TIMEOUT 1 -#define TPS_TASK_REJECTED 3 - /* Debounce delay for mode changes, in milliseconds */ #define CD321X_DEBOUNCE_DELAY_MS 500 @@ -105,6 +101,7 @@ enum { TPS_MODE_BIST, TPS_MODE_DISC, TPS_MODE_PTCH, + TPS_MODE_DBMA, }; static const char *const modes[] = { @@ -113,6 +110,7 @@ static const char *const modes[] = { [TPS_MODE_BIST] = "BIST", [TPS_MODE_DISC] = "DISC", [TPS_MODE_PTCH] = "PTCH", + [TPS_MODE_DBMA] = "DBMa", }; /* Unrecognized commands will be replaced with "!CMD" */ @@ -125,14 +123,7 @@ static enum power_supply_property tps6598x_psy_props[] = { static const char *tps6598x_psy_name_prefix = "tps6598x-source-psy-"; -/* - * Max data bytes for Data1, Data2, and other registers. See ch 1.3.2: - * https://www.ti.com/lit/ug/slvuan1a/slvuan1a.pdf - */ -#define TPS_MAX_LEN 64 - -static int -tps6598x_block_read(struct tps6598x *tps, u8 reg, void *val, size_t len) +int tps6598x_block_read(struct tps6598x *tps, u8 reg, void *val, size_t len) { u8 data[TPS_MAX_LEN + 1]; int ret; @@ -298,10 +289,10 @@ static void tps6598x_disconnect(struct tps6598x *tps, u32 status) power_supply_changed(tps->psy); } -static int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, - size_t in_len, const u8 *in_data, - size_t out_len, u8 *out_data, - u32 cmd_timeout_ms, u32 res_delay_ms) +int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, + size_t in_len, const u8 *in_data, + size_t out_len, u8 *out_data, + u32 cmd_timeout_ms, u32 res_delay_ms) { unsigned long timeout; u32 val; @@ -1106,6 +1097,14 @@ static int tps6598x_check_mode(struct tps6598x *tps) case TPS_MODE_BOOT: dev_warn(tps->dev, "dead-battery condition\n"); return ret; + case TPS_MODE_DBMA: + /* + * Apple firmware only: the application firmware runs in its + * debug mode (entered with LOCK + "DBMa"), in which it still + * handles the port and additionally accepts VDMs/DVEn. A boot + * loader arming Apple Debug USB on this port leaves it here. + */ + return ret; case TPS_MODE_BIST: case TPS_MODE_DISC: default: @@ -1844,6 +1843,17 @@ int tipd_init(struct tps6598x *tps) return ret; } + /* + * A controller found in "DBMa" is used as it is. Nothing below issues a + * 4CC command (the S0 power state switch above only does if the + * controller is not in S0 already), so probing neither leaves the debug + * mode nor disturbs a Debug USB or serial route set up before Linux. + * Remember it, so that the debugfs interface never tears that mode down. + */ + tps->dbma_at_probe = ret == TPS_MODE_DBMA; + if (tps->dbma_at_probe) + dev_info(tps->dev, "controller in debug mode \"DBMa\", leaving it there\n"); + ret = tps6598x_write64(tps, TPS_REG_INT_MASK1, tps->data->irq_mask1); if (ret) goto err_reset_controller; @@ -1912,6 +1922,8 @@ int tipd_init(struct tps6598x *tps) enable_irq_wake(tps->irq); } + tipd_debugfs_register(tps); + return 0; err_disconnect: @@ -1934,6 +1946,8 @@ EXPORT_SYMBOL_GPL(tipd_init); void tipd_remove(struct tps6598x *tps) { + tipd_debugfs_unregister(tps); + if (!tps->irq) cancel_delayed_work_sync(&tps->wq_poll); else @@ -2078,6 +2092,12 @@ const struct tipd_data tipd_sn201202x_data = { }; EXPORT_SYMBOL_GPL(tipd_sn201202x_data); +static void __exit tipd_core_exit(void) +{ + tipd_debugfs_exit(); +} +module_exit(tipd_core_exit); + MODULE_AUTHOR("Heikki Krogerus "); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("TI TPS6598x USB Power Delivery Controller Core Functions"); diff --git a/drivers/usb/typec/tipd/debugfs.c b/drivers/usb/typec/tipd/debugfs.c new file mode 100644 index 0000000000000..33dbc9dca411d --- /dev/null +++ b/drivers/usb/typec/tipd/debugfs.c @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * TI TPS6598x / Apple CD321x / SN201202x debugfs interface + * + * /sys/kernel/debug/usb/tipd//command + * write: 4CC command followed by 0..64 bytes for DATA1. The command is + * executed as a whole under the driver's lock. Only a small set + * of commands used to talk to a connected Apple device (as + * tuxvdmtool does) is accepted. + * read: the 64 bytes of DATA1 that the last completed command of this + * open file left behind. + * + * /sys/kernel/debug/usb/tipd//regs + * read-only register access: pread() with the register as offset + * (0x00..0x7f) and the length (1..64 bytes) as count. + * + * An open "command" file that put the controller into "DBMa" leaves that + * mode (DBMa 0, then LOCK 0000) when it is released, unless the controller + * was in "DBMa" already when the driver probed: that mode belongs to + * whoever set it up before Linux (e.g. a boot loader arming Debug USB). + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "tps6598x.h" + +#define TPS_REG_MAX 0x7f + +struct tipd_debugfs { + struct dentry *dentry; + /* Open "command" files that entered "DBMa"; protected by tps->lock */ + unsigned int dbma_holders; +}; + +struct tipd_debugfs_cmd { + char cmd[4]; + u8 min_len; + u8 max_len; + u32 timeout_ms; +}; + +static const struct tipd_debugfs_cmd tipd_debugfs_cmds[] = { + { "LOCK", 4, 4, 1000 }, /* unlock key, 0000 locks again */ + { "DBMa", 1, 1, 1000 }, /* 1: enter, 0: leave debug mode */ + { "VDMs", 5, 29, 200 }, /* SOP type/count + 1..7 VDOs */ + { "DVEn", 4, 28, 1000 }, /* local end of a serial/debug route */ + { "DISC", 0, 1, 1000 }, /* simulated disconnect */ +}; + +struct tipd_debugfs_file { + struct tps6598x *tps; + bool holds_dbma; + bool have_result; + u8 result[TPS_MAX_LEN]; +}; + +static struct dentry *tipd_debugfs_root; +static DEFINE_MUTEX(tipd_debugfs_root_lock); + +static const struct tipd_debugfs_cmd *tipd_debugfs_find_cmd(const u8 *cmd) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(tipd_debugfs_cmds); i++) + if (!memcmp(tipd_debugfs_cmds[i].cmd, cmd, 4)) + return &tipd_debugfs_cmds[i]; + + return NULL; +} + +/* Leave the debug mode entered through debugfs. Called with tps->lock held. */ +static void tipd_debugfs_leave_dbma(struct tps6598x *tps) +{ + static const u8 lock_key[4]; + u8 dbma = 0; + int ret; + + lockdep_assert_held(&tps->lock); + + if (tps->dbma_at_probe) + return; + + ret = tps6598x_exec_cmd_tmo(tps, "DBMa", sizeof(dbma), &dbma, 0, NULL, + 1000, 0); + if (ret) + dev_warn(tps->dev, "debugfs: failed to leave \"DBMa\": %d\n", ret); + + ret = tps6598x_exec_cmd_tmo(tps, "LOCK", sizeof(lock_key), lock_key, + 0, NULL, 1000, 0); + if (ret) + dev_warn(tps->dev, "debugfs: failed to lock: %d\n", ret); +} + +static int tipd_debugfs_command_open(struct inode *inode, struct file *file) +{ + struct tipd_debugfs_file *f; + + f = kzalloc_obj(*f); + if (!f) + return -ENOMEM; + + f->tps = inode->i_private; + file->private_data = f; + + return 0; +} + +static int tipd_debugfs_command_release(struct inode *inode, struct file *file) +{ + struct tipd_debugfs_file *f = file->private_data; + struct dentry *dentry = file->f_path.dentry; + struct tps6598x *tps = f->tps; + + /* + * debugfs calls ->release even after the file was removed, when tps + * may be gone. tipd_debugfs_unregister() has then left "DBMa" itself. + */ + if (f->holds_dbma && !debugfs_file_get(dentry)) { + mutex_lock(&tps->lock); + if (!--tps->debugfs->dbma_holders) + tipd_debugfs_leave_dbma(tps); + mutex_unlock(&tps->lock); + debugfs_file_put(dentry); + } + + kfree(f); + + return 0; +} + +static ssize_t tipd_debugfs_command_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct tipd_debugfs_file *f = file->private_data; + struct tps6598x *tps = f->tps; + const struct tipd_debugfs_cmd *cmd; + u8 buf[4 + TPS_MAX_LEN]; + u8 out[TPS_MAX_LEN] = { }; + size_t len; + int ret; + + if (count < 4 || count > sizeof(buf)) + return -EINVAL; + + if (copy_from_user(buf, ubuf, count)) + return -EFAULT; + + cmd = tipd_debugfs_find_cmd(buf); + if (!cmd) + return -EOPNOTSUPP; + + len = count - 4; + if (len < cmd->min_len || len > cmd->max_len) + return -EINVAL; + if (!memcmp(cmd->cmd, "VDMs", 4) && (len - 1) % 4) + return -EINVAL; + if (!memcmp(cmd->cmd, "DVEn", 4) && len % 4) + return -EINVAL; + + mutex_lock(&tps->lock); + + ret = tps6598x_exec_cmd_tmo(tps, cmd->cmd, len, len ? &buf[4] : NULL, + sizeof(out), out, cmd->timeout_ms, 0); + + /* DATA1 was read back if the command ran, even if its task failed */ + f->have_result = !ret || + (ret == -ETIMEDOUT && out[0] == TPS_TASK_TIMEOUT) || + (ret == -EPERM && out[0] == TPS_TASK_REJECTED); + if (f->have_result) + memcpy(f->result, out, sizeof(out)); + + if (!ret && !memcmp(cmd->cmd, "DBMa", 4)) { + if (buf[4] && !(out[0] & 0xf) && !f->holds_dbma) { + f->holds_dbma = true; + tps->debugfs->dbma_holders++; + } else if (!buf[4] && f->holds_dbma) { + f->holds_dbma = false; + tps->debugfs->dbma_holders--; + } + } + + mutex_unlock(&tps->lock); + + return ret ? ret : count; +} + +static ssize_t tipd_debugfs_command_read(struct file *file, char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct tipd_debugfs_file *f = file->private_data; + + if (!f->have_result) + return -ENODATA; + + return simple_read_from_buffer(ubuf, count, ppos, f->result, + sizeof(f->result)); +} + +static const struct file_operations tipd_debugfs_command_fops = { + .owner = THIS_MODULE, + .open = tipd_debugfs_command_open, + .release = tipd_debugfs_command_release, + .write = tipd_debugfs_command_write, + .read = tipd_debugfs_command_read, + .llseek = default_llseek, +}; + +static ssize_t tipd_debugfs_regs_read(struct file *file, char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct tps6598x *tps = file->private_data; + u8 buf[TPS_MAX_LEN]; + int ret; + + if (*ppos < 0 || *ppos > TPS_REG_MAX) + return -EINVAL; + if (!count) + return 0; + if (count > sizeof(buf)) + return -EINVAL; + + mutex_lock(&tps->lock); + ret = tps6598x_block_read(tps, *ppos, buf, count); + mutex_unlock(&tps->lock); + if (ret) + return ret; + + /* The offset selects a register; it does not advance. */ + if (copy_to_user(ubuf, buf, count)) + return -EFAULT; + + return count; +} + +static const struct file_operations tipd_debugfs_regs_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = tipd_debugfs_regs_read, + .llseek = default_llseek, +}; + +void tipd_debugfs_register(struct tps6598x *tps) +{ + struct tipd_debugfs *dbg; + + mutex_lock(&tipd_debugfs_root_lock); + if (!tipd_debugfs_root) + tipd_debugfs_root = debugfs_create_dir("tipd", usb_debug_root); + mutex_unlock(&tipd_debugfs_root_lock); + + dbg = kzalloc_obj(*dbg); + if (!dbg) + return; + + dbg->dentry = debugfs_create_dir(dev_name(tps->dev), tipd_debugfs_root); + tps->debugfs = dbg; + debugfs_create_file("command", 0600, dbg->dentry, tps, + &tipd_debugfs_command_fops); + debugfs_create_file("regs", 0400, dbg->dentry, tps, + &tipd_debugfs_regs_fops); +} + +void tipd_debugfs_unregister(struct tps6598x *tps) +{ + struct tipd_debugfs *dbg = tps->debugfs; + + if (!dbg) + return; + + /* Waits for running file operations, later ones fail with -EIO */ + debugfs_remove_recursive(dbg->dentry); + + mutex_lock(&tps->lock); + if (dbg->dbma_holders) + tipd_debugfs_leave_dbma(tps); + tps->debugfs = NULL; + mutex_unlock(&tps->lock); + + kfree(dbg); +} + +void tipd_debugfs_exit(void) +{ + debugfs_remove(tipd_debugfs_root); +} diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index c6c05da547f2a..3f7182cbf7251 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -274,6 +274,16 @@ #define TPS_65981_2_6_INTEVENT_LEN 8 #define TPS_65987_8_INTEVENT_LEN 11 +/* + * Max data bytes for Data1, Data2, and other registers. See ch 1.3.2: + * https://www.ti.com/lit/ug/slvuan1a/slvuan1a.pdf + */ +#define TPS_MAX_LEN 64 + +/* Standard Task return codes */ +#define TPS_TASK_TIMEOUT 1 +#define TPS_TASK_REJECTED 3 + struct tps6598x; struct tipd_data { @@ -300,6 +310,7 @@ struct tps6598x { struct mutex lock; /* device lock */ int irq; u8 i2c_protocol:1; + u8 dbma_at_probe:1; /* Apple "DBMa" debug mode was active at probe */ struct gpio_desc *reset; struct typec_port *port; @@ -319,6 +330,8 @@ struct tps6598x { struct delayed_work wq_poll; const struct tipd_data *data; + + struct tipd_debugfs *debugfs; }; /* TPS_REG_USB4_STATUS */ @@ -400,4 +413,21 @@ void tipd_remove(struct tps6598x *tps); int tipd_suspend(struct tps6598x *tps); int tipd_resume(struct tps6598x *tps); +/* Internal to tps6598x-core */ +int tps6598x_block_read(struct tps6598x *tps, u8 reg, void *val, size_t len); +int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, + size_t in_len, const u8 *in_data, + size_t out_len, u8 *out_data, + u32 cmd_timeout_ms, u32 res_delay_ms); + +#ifdef CONFIG_DEBUG_FS +void tipd_debugfs_register(struct tps6598x *tps); +void tipd_debugfs_unregister(struct tps6598x *tps); +void tipd_debugfs_exit(void); +#else +static inline void tipd_debugfs_register(struct tps6598x *tps) { } +static inline void tipd_debugfs_unregister(struct tps6598x *tps) { } +static inline void tipd_debugfs_exit(void) { } +#endif + #endif /* __TPS6598X_H__ */ From 4b625164e8c21d33b5ba87eb3392e06abbae0aa1 Mon Sep 17 00:00:00 2001 From: Vincent Fontaine Date: Thu, 1 Oct 2026 19:06:49 -0700 Subject: [PATCH 0328/1579] soc: apple: sep: add a T8112 (MacBook Air M2) platform profile T8112 is a warm-registration platform: iBoot hands over a running SEP and a powered Mesa sensor (J415: AP GPIO 178 is a driven-high output at handoff). The values come from the J415 ADT (/arm-io/spi2/mesa and /arm-io/sep): spi2 at the T8103 address, power on AP GPIO 178, data-ready on 177, 8 MHz, chip select 0, a SEP DART aperture of 0x4000 + 0xffff0000 (below 4 GiB, so no wide DMA mask) and /defaults cpx-encryption-mode = 2. They match the J413 values posted in the omacom/linux#7 thread. The shared memory follows the other warm platforms (0x40000, CINP). The key store speaks Variant5: J415 accepted the 13.5 GET_CAPABILITIES and SET_ENV init but refused the strict CREATE_KEYBAG (mailbox status -1, empty reply), as J700 did, and provisioned with the T6020 encoding. Tested on J415 (MacBook Air 15", M2) with the patched m1n1 (no SEP RNG query on M2, apfs-preboot-uuid forwarded): - read-only (xart_writes=0): warm attach, all 12 endpoints including key store 0x12, sbio, scrd and hwrng, on 3 cold boots; the iBoot system container image was byte-identical afterwards - writes on: identity keybag and ref-key provisioned, trusted-key seal/unseal round trip passed, fprintd enrol completed (user and master catacombs at state 7), verify-match in the same boot and again after a reboot Co-Authored-By: Claude Opus 5.5 --- drivers/soc/apple/profile.rs | 50 +++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 9aa39f4736ec8..ba5d4966854ec 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -8,7 +8,7 @@ //! here once. The driver reads addresses from the device tree, never scans for //! them, and never patches properties at runtime. //! -//! Four bring-up targets are modelled: +//! Five bring-up targets are modelled: //! //! * `T8103` / J313 (MacBook Air, M1): the host boots the SEP with the boot //! endpoint handshake over a 0x30000 shared-memory window. @@ -18,6 +18,8 @@ //! single-message registration over a 0x40000 window. //! * `T8140` / J700 (MacBook Neo, A18 Pro): iBoot boots sepOS before the AP //! OS, so the driver uses the warm registration path. +//! * `T8112` / J413, J415 (MacBook Air, M2): iBoot hands over a running SEP and +//! a powered sensor, so the driver uses the warm registration path. // The boot handshake, identity source, sensor and DART fields are consumed by // the boot-endpoint, identity and sensor-transport paths. @@ -277,6 +279,47 @@ const T8140: PlatformProfile = PlatformProfile { }, }; +/// MacBook Air M2 (J413, J415). iBoot leaves the SEP running and the Mesa +/// sensor powered (J415: AP GPIO 178 is a driven-high output at handoff), so +/// this is a warm-registration platform like T6020. The J415 ADT describes the +/// sensor exactly as J413's does: spi2 at the T8103 address, power on AP GPIO +/// 178, data-ready on 177, 8 MHz, chip select 0. The SEP DART aperture is +/// 0x4000 + 0xffff0000, below 4 GiB, so no wide DMA mask. `/defaults` +/// carries `cpx-encryption-mode = 2`. +/// +/// The shared-memory geometry and key-store dialect follow the other +/// warm-registration platforms (T6020, T8140). +const T8112: PlatformProfile = PlatformProfile { + name: "T8112/J415", + shmem_capacity: 0x4_0000, + shmem_first_item: b"CINP", + bootstrap: Bootstrap::WarmRegister, + identity: IdentitySource::Chosen, + sensor: SensorProfile { + controller_base: 0x2_3510_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode1, + expected_id: 0x3352, + capture_qualified: false, + }, + dart_range_required: true, + wide_dma_mask: false, + firmware_region: c"sepfw", + // J415 accepted the 13.5 GET_CAPABILITIES/SET_ENV init but refused the + // strict CREATE_KEYBAG (mailbox status -1, empty reply), as J700 did; + // use the T6020 form like the other warm-registration platforms. + keybag_create: KeybagCreate { + variant: 5, + bag_type: 0, + arg: -1, + }, + key_store: KeyStore::Variant5, + persistent_enrol: true, +}; + static_assert!(T8103.shmem_capacity == 0x30000); static_assert!(T6020.shmem_capacity == 0x40000); static_assert!(T8140.shmem_capacity == 0x40000); @@ -287,6 +330,8 @@ static_assert!(T8103.sensor.expected_id == T6020.sensor.expected_id); static_assert!(T6000.shmem_capacity == T8103.shmem_capacity); static_assert!(T6000.sensor.controller_base == T6020.sensor.controller_base); static_assert!(T6000.sensor.expected_id == T6020.sensor.expected_id); +static_assert!(T8112.shmem_capacity == T6020.shmem_capacity); +static_assert!(T8112.sensor.controller_base == T8103.sensor.controller_base); /// Whether the machine root declares `compatible`. /// @@ -339,5 +384,8 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { if machine_has(b"apple,t8140") { return Ok(&T8140); } + if machine_has(b"apple,t8112") { + return Ok(&T8112); + } Err(ENODEV) } From 2c0b8da2c02f2f1f40238c8f3bf5ef6c103a9489 Mon Sep 17 00:00:00 2001 From: Vincent Fontaine Date: Thu, 1 Oct 2026 19:06:49 -0700 Subject: [PATCH 0329/1579] arm64: dts: apple: t8112-j415: enable the SEP and the Mesa fingerprint sensor Add the Mesa sensor under spi2 in t8112.dtsi, and keep the SEP DART off at boot. iBoot hands over a running SEP, and apple_sep sets the DMA range and enables the DART itself, as on t602x. Probing it at boot leaves it configured without the range, and the driver then refuses to attach. Wire the sensor on J415 from its ADT: power on AP GPIO 178, active high, 8 MHz, chip select 0, SPI mode 1. The data-ready line (GPIO 177) is left out, so the driver polls. The sep alias lets m1n1 hand over the local-policy and iBoot manifests that the warm registration needs. Tested on J415 with the same nodes applied as an overlay to the running device tree (sensor bound in mode 1 with hardware CS timing, enrol and verify passed); these board-file nodes enable spi2 and the sensor at boot, as j414c does, instead of on module load. Co-Authored-By: Claude Opus 5.5 --- arch/arm64/boot/dts/apple/t8112-j415.dts | 19 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8112.dtsi | 13 +++++++++++++ 2 files changed, 32 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-j415.dts b/arch/arm64/boot/dts/apple/t8112-j415.dts index 20a64ecc122d8..cdb240806ff81 100644 --- a/arch/arm64/boot/dts/apple/t8112-j415.dts +++ b/arch/arm64/boot/dts/apple/t8112-j415.dts @@ -21,6 +21,7 @@ aliases { bluetooth0 = &bluetooth0; keyboard = &keyboard; + sep = &sep; wifi0 = &wifi0; }; @@ -282,6 +283,24 @@ }; }; +/* + * Mesa fingerprint sensor on spi2 (J415 ADT /arm-io/spi2/mesa): power on AP + * GPIO 178, active high. The data-ready line (GPIO 177) is left out, so the + * driver polls until its trigger is confirmed. Native chip select only. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + /* Per-device factory data from this Mac's own iBoot system container. */ + firmware-name = "apple/mesacal-j415.bin"; +}; + #include "isp-imx558-cfg0.dtsi" &isp { diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index ce17f810a635d..1806a198b8e36 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -900,6 +900,14 @@ #address-cells = <1>; #size-cells = <0>; status = "disabled"; + + mesa: fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpha; + status = "disabled"; + }; }; spi3: spi@23510c000 { @@ -1656,12 +1664,17 @@ }; + /* + * iBoot hands over a running SEP. Keep Linux off its DART at boot; + * apple_sep sets the DMA range and enables it, as on t602x. + */ sep_dart: iommu@25d2c0000 { compatible = "apple,t8112-dart", "apple,t8110-dart"; reg = <0x2 0x5d2c0000 0x0 0x4000>; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; + status = "disabled"; }; sep: sep@25e400000 { From be14a5941688b0967991da2eca37a3768d9bb8a3 Mon Sep 17 00:00:00 2001 From: Acelogic <7362641+Acelogic@users.noreply.github.com> Date: Fri, 2 Oct 2026 02:06:57 -0400 Subject: [PATCH 0330/1579] dt-bindings: media: describe Apple T8140 AVE Describe the encoder register blocks, paired DARTs, five power domains and two bootloader-supplied firmware reservations used by J700. Signed-off-by: Acelogic <7362641+Acelogic@users.noreply.github.com> --- .../bindings/media/apple,t8140-ave.yaml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 Documentation/devicetree/bindings/media/apple,t8140-ave.yaml diff --git a/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml b/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml new file mode 100644 index 0000000000000..110ebbc3d09e2 --- /dev/null +++ b/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/apple,t8140-ave.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 video encoder + +maintainers: + - Miguel Cruz <7362641+Acelogic@users.noreply.github.com> + +description: | + The T8140 AVE coprocessor encodes video using firmware preloaded by the + bootloader. Its two DART instances translate the same stream. The + bootloader supplies two reserved firmware regions and their IOVA mappings. + +properties: + compatible: + const: apple,t8140-ave + + reg: + items: + - description: Codec registers + - description: Mailbox registers + - description: Coprocessor registers + + reg-names: + items: + - const: codec + - const: mailbox + - const: coproc + + interrupts: + maxItems: 5 + + power-domains: + maxItems: 5 + + power-domain-names: + items: + - const: dma + - const: pipe4 + - const: pipe5 + - const: me0 + - const: me1 + + iommus: + maxItems: 2 + + memory-region: + items: + - description: Preloaded firmware text + - description: Preloaded firmware data + +required: + - compatible + - reg + - reg-names + - interrupts + - power-domains + - power-domain-names + - iommus + +additionalProperties: false From 52f1ba51224431d256250c69784e1996f6ee63cc Mon Sep 17 00:00:00 2001 From: Acelogic <7362641+Acelogic@users.noreply.github.com> Date: Fri, 2 Oct 2026 02:06:57 -0400 Subject: [PATCH 0331/1579] arm64: dts: apple: wire the J700 AVE encoder Add the T8140 encoder power hierarchy, paired DARTs and AVE node. Enable them on J700 so the public Aurora m1n1 firmware handoff can populate its reserved-memory mappings. The register and power-domain description comes from Ace's existing J700 Linux bring-up work. Signed-off-by: Acelogic <7362641+Acelogic@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8140-j700.dts | 12 +++++ arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 53 +++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8140.dtsi | 41 ++++++++++++++++++ 3 files changed, 106 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index cc9f115adb7ab..a154bcca8f77d 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -56,6 +56,18 @@ #include "isp-imx558.dtsi" +&ave_dart0 { + status = "okay"; +}; + +&ave_dart1 { + status = "okay"; +}; + +&ave { + status = "okay"; +}; + &serial0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 3fc3ea3ad8fb4..4d275222c4555 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -23,6 +23,14 @@ label = "gpio"; }; + ps_venc0_sys: power-controller@240 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x240 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_sys"; + }; + ps_apcie_gp: power-controller@258 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x258 4>; @@ -258,6 +266,51 @@ #reset-cells = <0>; label = "isp_cvm"; }; + + ps_venc0_dma: power-controller@8000 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_dma"; + power-domains = <&ps_venc0_sys>; + }; + + ps_venc0_pipe4: power-controller@8008 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_pipe4"; + power-domains = <&ps_venc0_dma>; + }; + + ps_venc0_pipe5: power-controller@8010 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_pipe5"; + power-domains = <&ps_venc0_dma>; + }; + + ps_venc0_me0: power-controller@8018 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_me0"; + power-domains = <&ps_venc0_dma>; + }; + + ps_venc0_me1: power-controller@8020 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_me1"; + power-domains = <&ps_venc0_me0>; + }; }; &pmgr_mini { diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 2dd921b23368c..21c827da20f10 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -588,6 +588,47 @@ status = "disabled"; }; + ave_dart0: iommu@285080000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x2 0x85080000 0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_venc0_sys>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x00004000 0 0x0fffc000>; + status = "disabled"; + }; + + ave_dart1: iommu@2850a0000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x2 0x850a0000 0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_venc0_sys>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x00004000 0 0x0fffc000>; + status = "disabled"; + }; + + ave: video-codec@285100000 { + compatible = "apple,t8140-ave"; + reg = <0x2 0x85100000 0 0x45c000>, + <0x2 0x85050000 0 0x4000>, + <0x2 0x85800000 0 0x800000>; + reg-names = "codec", "mailbox", "coproc"; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + ; + power-domains = <&ps_venc0_dma>, <&ps_venc0_pipe4>, + <&ps_venc0_pipe5>, <&ps_venc0_me0>, <&ps_venc0_me1>; + power-domain-names = "dma", "pipe4", "pipe5", "me0", "me1"; + iommus = <&ave_dart0 0>, <&ave_dart1 0>; + status = "disabled"; + }; + pcie0: pcie@1cb0000000 { compatible = "apple,t8140-pcie"; device_type = "pci"; From 4612e5f8a66a82548bce6ae3e4ba0053b5d9babc Mon Sep 17 00:00:00 2001 From: Acelogic <7362641+Acelogic@users.noreply.github.com> Date: Fri, 2 Oct 2026 02:06:58 -0400 Subject: [PATCH 0332/1579] media: apple: add an opt-in J700 AVE encoding transport Port Ace (Acelogic)'s existing GPL J700 diagnostic implementation to the public media tree, retaining the qualified single-command path. Add an explicit opt-in and limit admission to the inspected firmware text images. Preserve firmware reservations, validate DMA mappings, and keep resources pinned after startup until hardware reset. This is an experimental root-only debugfs transport for fixed-profile H.264/HEVC work. It does not register a V4L2 encoder or provide a stable userspace ABI. Firmware images and binary command captures are not included. Document those limits and the physical validation gates. Signed-off-by: Acelogic <7362641+Acelogic@users.noreply.github.com> --- Documentation/admin-guide/media/apple-ave.rst | 63 +++ Documentation/admin-guide/media/index.rst | 1 + drivers/media/platform/apple/Kconfig | 1 + drivers/media/platform/apple/Makefile | 1 + drivers/media/platform/apple/ave/Kconfig | 16 + drivers/media/platform/apple/ave/Makefile | 3 + .../media/platform/apple/ave/apple-ave-ipc.h | 446 +++++++++++++++ drivers/media/platform/apple/ave/apple-ave.c | 532 ++++++++++++++++++ 8 files changed, 1063 insertions(+) create mode 100644 Documentation/admin-guide/media/apple-ave.rst create mode 100644 drivers/media/platform/apple/ave/Kconfig create mode 100644 drivers/media/platform/apple/ave/Makefile create mode 100644 drivers/media/platform/apple/ave/apple-ave-ipc.h create mode 100644 drivers/media/platform/apple/ave/apple-ave.c diff --git a/Documentation/admin-guide/media/apple-ave.rst b/Documentation/admin-guide/media/apple-ave.rst new file mode 100644 index 0000000000000..228ea84db2b86 --- /dev/null +++ b/Documentation/admin-guide/media/apple-ave.rst @@ -0,0 +1,63 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Experimental T8140 video encoding +================================ + +The J700 AVE driver exposes a root-only debugfs firmware transport for +hardware encoding experiments. It is not a V4L2 encoder, has no stable +userspace ABI, and does not make an FFmpeg encoder available. The initial +scope is the existing fixed 320x240 NV12 H.264/HEVC protocol experiment. + +Build ``CONFIG_VIDEO_APPLE_AVE_EXPERIMENTAL=m`` on a 16 KiB-page Apple kernel +with debugfs and translated DMA. Suspend, hibernation and kexec must be off. +Load with ``modprobe apple-ave enable=1``; probing is disabled otherwise. + +Firmware and boot requirements +------------------------------ + +Use the public Aurora m1n1 J700 AVE handoff. It reserves the firmware already +loaded by iBoot, fills the device's ``memory-region`` references and exports +``aurora,ave-text-crc32`` and ``aurora,ave-data-crc32`` strings under +``/chosen``. No firmware file is included or requested by this driver. +The current transport requires the qualified 25G76 two-segment layout: +0x14c000 text bytes and 0x130000 data bytes, mapped starting at IOVA +0x10000000000. Every mapping and both content CRCs are checked before use. +Text CRCs are restricted to the two inspected 9003.78.0 images (0x2ce7359e +and 0x69c54454). These checks detect build drift, not firmware authenticity. +Different firmware is unsupported, even if the device has the same SoC. + +The driver holds the DMA, pipe4, pipe5, me0 and me1 power domains and owns +the IPC, queue, log and firmware-requested heap allocations. Debugfs lives +under ``/sys/kernel/debug/285100000.video-codec``: + +* ``ave-status`` reports mappings, protocol phase and reply counters. +* ``ave-start-once`` accepts one explicit firmware start per boot. +* ``ave-command`` accepts one bounded packet and returns a framed reply. +* ``ave-arena`` stages image/reference/output surfaces within owned memory. +* ``ave-poll`` drains later firmware replies without another submission. + +Writes require ``CAP_SYS_RAWIO``. Command packets must come from a trusted, +matching local protocol experiment with all embedded surface addresses +relocated into this boot's owned arena. Root access does not make arbitrary +firmware packets valid. Binary command captures are supplied separately and +are not included in this contribution. + +Lifecycle and verification +-------------------------- + +After the start attempt, the module, DMA buffers and power references remain +pinned until an external hardware reset, including after failure. Do not +force-unload, unbind the DARTs, reuse uncertain buffers or retry a failed +command. Firmware quiescence and normal production teardown are unfinished. + +A transport acknowledgement is not an encoded frame. For physical tests, +record the exact kernel/module, DT and firmware hashes; submit changing +images; validate each codec reply; independently decode the current DMA +output; check every frame's dimensions and quality; then verify Stop/Close, +kernel health and the existing desktop/radio stack. Keep the generated +inputs, bitstreams, decoded pixels, logs and hashes as a repeatable artifact. + +The prior implementation was authored by Ace (Acelogic). This contribution +ports that existing GPL work; it does not claim independent clean-room +provenance. Arbitrary resolutions, standard media APIs, throughput, +concurrent clients and other codecs or SoCs are outside this experiment. diff --git a/Documentation/admin-guide/media/index.rst b/Documentation/admin-guide/media/index.rst index b11737ae6c046..65aea88dac7ed 100644 --- a/Documentation/admin-guide/media/index.rst +++ b/Documentation/admin-guide/media/index.rst @@ -43,6 +43,7 @@ Documentation/process/debugging/media_specific_debugging_guide.rst v4l-drivers dvb-drivers + apple-ave **Copyright** |copy| 1999-2020 : LinuxTV Developers diff --git a/drivers/media/platform/apple/Kconfig b/drivers/media/platform/apple/Kconfig index 77588c8f4e5a5..f656926099b3e 100644 --- a/drivers/media/platform/apple/Kconfig +++ b/drivers/media/platform/apple/Kconfig @@ -4,3 +4,4 @@ comment "Apple media platform drivers" source "drivers/media/platform/apple/isp/Kconfig" source "drivers/media/platform/apple/avd/Kconfig" +source "drivers/media/platform/apple/ave/Kconfig" diff --git a/drivers/media/platform/apple/Makefile b/drivers/media/platform/apple/Makefile index 72f8bb54b376d..dac50fc70e277 100644 --- a/drivers/media/platform/apple/Makefile +++ b/drivers/media/platform/apple/Makefile @@ -2,3 +2,4 @@ obj-y += isp/ obj-y += avd/ +obj-y += ave/ diff --git a/drivers/media/platform/apple/ave/Kconfig b/drivers/media/platform/apple/ave/Kconfig new file mode 100644 index 0000000000000..629ce9244039a --- /dev/null +++ b/drivers/media/platform/apple/ave/Kconfig @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config VIDEO_APPLE_AVE_EXPERIMENTAL + tristate "Apple T8140 experimental video encoder transport" + depends on ARCH_APPLE && ARM64_16K_PAGES + depends on PM_GENERIC_DOMAINS && IOMMU_DMA && DEBUG_FS + depends on !SUSPEND && !HIBERNATION && !KEXEC_CORE + select CRC32 + help + Experimental root-only AVE firmware and command transport for the + MacBook Neo. This does not register a V4L2 or FFmpeg encoder. + + It requires the bootloader's reserved, preloaded firmware and an + explicit enable=1 module parameter. After firmware startup, memory + and power references remain pinned until a hardware reset. Do not + enable this on a general-purpose kernel requiring suspend or kexec. diff --git a/drivers/media/platform/apple/ave/Makefile b/drivers/media/platform/apple/ave/Makefile new file mode 100644 index 0000000000000..c8e673e68f080 --- /dev/null +++ b/drivers/media/platform/apple/ave/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_VIDEO_APPLE_AVE_EXPERIMENTAL) += apple-ave.o diff --git a/drivers/media/platform/apple/ave/apple-ave-ipc.h b/drivers/media/platform/apple/ave/apple-ave-ipc.h new file mode 100644 index 0000000000000..d317af1dad72b --- /dev/null +++ b/drivers/media/platform/apple/ave/apple-ave-ipc.h @@ -0,0 +1,446 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Experimental 25G76/64-bit AVE mailbox and bidirectional ring transport. + * Included after private driver types. All callers hold ave->lock. + */ +#define AVE_CHANNEL_OFF 0x4000 +#define AVE_HANDSHAKE_OFF 0x24000 +#define AVE_SHARED_OFF 0x28000 +#define AVE_COMMAND_OFF 0x40000 +#define AVE_COMMAND_BYTES SZ_128K +#define AVE_ARENA_OFF 0x80000 +#define AVE_START_MAGIC 0x08042006 + +static int ave_fw_offset(struct apple_ave *ave, u64 address, size_t bytes, u32 *offset) +{ + u64 delta = address - ave->ipc_fw_base; + + if (address < ave->ipc_fw_base || delta > ave->ipc.size || bytes > ave->ipc.size - delta) + return -ERANGE; + *offset = delta; + return 0; +} + +static void ave_mailbox_send(struct apple_ave *ave, u32 a, u32 b, u32 c, u32 d) +{ + writel(a, ave->mailbox + 0x18); + writel(b, ave->mailbox + 0x1c); + writel(c, ave->mailbox + 0x20); + writel(d, ave->mailbox + 0x24); + wmb(); + writel(1, ave->mailbox + 0x0c); +} + +static int ave_mailbox_receive(struct apple_ave *ave) +{ + u32 status; + int ret = readl_poll_timeout(ave->mailbox + 0x10, status, status & 1, 50, 5000000); + + if (ret) + return ret; + /* Acknowledge the interrupt before reading the four scratch words. */ + writel(1, ave->mailbox + 0x10); + for (unsigned int i = 0; i < 4; i++) + ave->ipc_reply[i] = readl(ave->mailbox + 0x18 + i * 4); + dma_rmb(); + dev_info(ave->dev, "AVE_IPC_MAILBOX phase=%s words=%#x,%#x,%#x,%#x\n", + ave->phase, ave->ipc_reply[0], ave->ipc_reply[1], ave->ipc_reply[2], ave->ipc_reply[3]); + return 0; +} + +static int ave_create_channels(struct apple_ave *ave) +{ + bool found[2] = { false, false }; + + for (unsigned int i = 0; i < 2; i++) { + u8 *desc = ave->ipc.cpu + AVE_CHANNEL_OFF + i * 0x100; + u32 direction = get_unaligned_le32(desc + 0x40); + u32 interrupt = get_unaligned_le32(desc + 0x44); + u32 slots = get_unaligned_le32(desc + 0x48), offset; + u64 address = get_unaligned_le64(desc + 0x4c); + struct ave_channel *c; + unsigned int id; + + if (!memchr(desc, 0, 64)) + return -EPROTO; + if (!strcmp(desc, "IO")) + id = 0; + else if (!strcmp(desc, "IO_T2H")) + id = 1; + else + return -EPROTO; + if (found[id] || direction != id || !interrupt || interrupt >= 32 || + !slots || slots > ave->channel_bytes / 64 || !IS_ALIGNED(address, 64) || + ave_fw_offset(ave, address, (size_t)slots * 64, &offset) || + offset < AVE_CHANNEL_OFF + 0x200 || + offset + slots * 64 > AVE_CHANNEL_OFF + ave->channel_bytes) + return -EPROTO; + found[id] = true; + c = &ave->channel[id]; + strscpy(c->name, desc, sizeof(c->name)); + c->mode = direction ^ 1; + c->interrupt = interrupt; + c->slots = slots; + c->offset = offset; + c->consumer = c->mode ? -1 : 0; + c->producer = c->mode ? 0 : -1; + } + if (!found[0] || !found[1] || + (ave->channel[0].offset < ave->channel[1].offset + ave->channel[1].slots * 64 && + ave->channel[1].offset < ave->channel[0].offset + ave->channel[0].slots * 64)) + return -EPROTO; + /* Initialize only host-originated slots; leave all padding and + * firmware-originated slots unchanged. Ownership is bit zero of address. + */ + for (unsigned int i = 0; i < ave->channel[0].slots; i++) { + __le64 *slot = ave->ipc.cpu + ave->channel[0].offset + i * 64; + slot[0] = cpu_to_le64(1); + slot[1] = 0; + slot[2] = 0; + } + dma_wmb(); + return 0; +} + +static int ave_negotiate_ipc(struct apple_ave *ave) +{ + u64 ticks, time_delta, handshake_address, channel_address; + u32 frequency, status, offset; + u8 *handshake = ave->ipc.cpu + AVE_HANDSHAKE_OFF; + int ret; + + ave->phase = "validate-offer"; + if (ave->hello[0] != 2 || ave->hello[2] != 0x100 || + ave->hello[1] < 0x200 || ave->hello[1] > AVE_HANDSHAKE_OFF - AVE_CHANNEL_OFF || + !ave->hello[3] || ave->hello[3] > 4 * SZ_1M) + return -EPROTO; + ave->channel_bytes = ave->hello[1]; + ret = ave_alloc_buffer(ave, &ave->heap, PAGE_ALIGN(ave->hello[3])); + if (ret) + return ret; + writel(1, ave->mailbox + 0x10); + ticks = readq(ave->coproc + 0x378000); + frequency = readl(ave->coproc + 0x360020); + if (frequency && frequency < 1000000) + return -EPROTO; + ticks = div64_u64(ticks, frequency ? frequency / 1000000 : 24); + time_delta = div_u64(ktime_get_ns(), 1000) - ticks; + writel(lower_32_bits(time_delta), ave->mailbox + 0x28); + writel(upper_32_bits(time_delta), ave->mailbox + 0x2c); + ave->phase = "ipc-base"; + ave_mailbox_send(ave, lower_32_bits(ave->ipc.dma), upper_32_bits(ave->ipc.dma), ave->ipc.size, 0); + ret = ave_mailbox_receive(ave); + if (ret) + return ret; + ave->ipc_fw_base = (u64)ave->ipc_reply[1] << 32 | ave->ipc_reply[0]; + if (!ave->ipc_fw_base || !IS_ALIGNED(ave->ipc_fw_base, PAGE_SIZE) || + ave->ipc_fw_base > U64_MAX - ave->ipc.size) + return -EPROTO; + memset(ave->ipc.cpu + AVE_CHANNEL_OFF, 0, ave->channel_bytes); + memset(handshake, 0, 0x50); + memset(ave->ipc.cpu + AVE_SHARED_OFF, 0, SZ_64K); + channel_address = ave->ipc_fw_base + AVE_CHANNEL_OFF; + handshake_address = ave->ipc_fw_base + AVE_HANDSHAKE_OFF; + put_unaligned_le64(channel_address, handshake + 8); + put_unaligned_le64(ave->ipc_fw_base + AVE_SHARED_OFF, handshake + 0x10); + put_unaligned_le32(SZ_64K, handshake + 0x18); + put_unaligned_le64(ave->heap.dma, handshake + 0x1c); + put_unaligned_le32(ave->hello[3], handshake + 0x24); + put_unaligned_le32(27, handshake + 0x28); + dma_wmb(); + ave->phase = "channel-offer"; + ave_mailbox_send(ave, lower_32_bits(handshake_address), upper_32_bits(handshake_address), 0, 0); + ret = ave_mailbox_receive(ave); + if (ret) + return ret; + ave->extra_bytes = ave->ipc_reply[2]; + if (ave_fw_offset(ave, (u64)ave->ipc_reply[1] << 32 | ave->ipc_reply[0], + ave->channel_bytes, &offset) || offset != AVE_CHANNEL_OFF || + ave->extra_bytes > 0x13c000) + return -EPROTO; + ave->phase = "channel-layout"; + ret = ave_create_channels(ave); + if (ret) + return ret; + ave->phase = "ready-flag"; + writel(AVE_START_MAGIC, ave->mailbox + 0x24); + ret = readl_poll_timeout(ave->mailbox + 0x24, status, !status, 50, 5000000); + if (ret) + return ret; + ave->ipc_ready = true; + ave->phase = "ready"; + return 0; +} + +static int ave_ring_send(struct apple_ave *ave, struct ave_channel *c, u64 addr, u32 len, u32 flags) +{ + __le64 *slot; + int next; + + if (c->producer < 0) + return -ENOSPC; + slot = ave->ipc.cpu + c->offset + c->producer * 64; + WRITE_ONCE(slot[1], cpu_to_le64(len)); + WRITE_ONCE(slot[2], cpu_to_le64(flags)); + dma_wmb(); + WRITE_ONCE(slot[0], cpu_to_le64(addr | (c->mode ^ 1))); + dma_wmb(); + if (c->consumer < 0) + c->consumer = c->producer; + next = (c->producer + 1) % c->slots; + c->producer = next == c->consumer ? -1 : next; + writel(BIT(c->interrupt), ave->mailbox + 0x0c); + return 0; +} + +static int ave_ring_receive(struct apple_ave *ave, struct ave_channel *c, u64 *addr, u32 *len, u32 *flags) +{ + __le64 *slot; + u64 word; + int next; + + if (c->consumer < 0) + return -EAGAIN; + slot = ave->ipc.cpu + c->offset + c->consumer * 64; + word = le64_to_cpu(READ_ONCE(slot[0])); + if ((word & 1) != c->mode) + return -EAGAIN; + dma_rmb(); + *addr = word & ~3ULL; + *len = le64_to_cpu(READ_ONCE(slot[1])); + *flags = le64_to_cpu(READ_ONCE(slot[2])); + if (c->producer < 0) + c->producer = c->consumer; + next = (c->consumer + 1) % c->slots; + c->consumer = next == c->producer ? -1 : next; + return 0; +} + +/* Root-only experimental packet transport. The packet is copied into owned + * coherent memory; timeout pins the command too and forbids reuse this boot. + * Firmware command success is established by decoding replies, not this API. + */ +static ssize_t ave_command_write(struct file *file, const char __user *data, size_t len, loff_t *pos) +{ + struct apple_ave *ave = file->private_data; + u8 *cmd = ave->ipc.cpu + AVE_COMMAND_OFF; + u64 deadline, addr; + u32 bytes, flags, offset; + bool ack = false; + int ret = 0; + + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + static_assert(AVE_COMMAND_OFF + AVE_COMMAND_BYTES <= AVE_ARENA_OFF); + if (len < 0x48 || len > AVE_COMMAND_BYTES) + return -EINVAL; + mutex_lock(&ave->lock); + if (!ave->ipc_ready || ave->command_failed) { + ret = -EBUSY; + goto out; + } + if (copy_from_user(cmd, data, len)) { + ret = -EFAULT; + goto out; + } + /* The common command timeout is {duration_us, creation_time_us}. + * Stamp it in the same Linux monotonic clock synchronized at startup; + * captured macOS absolute times cannot be replayed on this boot. + */ + put_unaligned_le64(3000000, cmd + 0x30); + put_unaligned_le64(div_u64(ktime_get_ns(), 1000), cmd + 0x38); + ave->response_bytes = 0; + ret = ave_ring_send(ave, &ave->channel[0], ave->ipc_fw_base + AVE_COMMAND_OFF, len, 0); + if (ret) + goto out; + ave->commands++; + deadline = ktime_get_ns() + 5ULL * NSEC_PER_SEC; + do { + u32 pending = readl(ave->mailbox + 0x10); + if (pending) + writel(pending, ave->mailbox + 0x10); + for (unsigned int ch = 0; ch < 2; ch++) { + ret = ave_ring_receive(ave, &ave->channel[ch], &addr, &bytes, &flags); + if (ret == -EAGAIN) + continue; + /* Channel 1 returns the original address with zero payload + * bytes (verified native AVC captures). Only channel 2 is + * copied to response, independent of submitted packet size. + */ + if (ret || (!ch && bytes != 0) || + (ch && bytes > sizeof(ave->response) - 24) || + ave_fw_offset(ave, addr, bytes, &offset)) { + ret = -EPROTO; + goto failed; + } + dev_info(ave->dev, "AVE_COMMAND_REPLY channel=%u address=%#llx bytes=%u flags=%#x\n", + ch + 1, addr, bytes, flags); + if (!ch) { + if (addr != ave->ipc_fw_base + AVE_COMMAND_OFF) { + ret = -EPROTO; + goto failed; + } + ack = true; + ave->transport_acks++; + } else { + put_unaligned_le32(ch + 1, ave->response); + put_unaligned_le32(bytes, ave->response + 4); + put_unaligned_le32(flags, ave->response + 8); + put_unaligned_le32(0, ave->response + 12); + put_unaligned_le64(addr, ave->response + 16); + memcpy(ave->response + 24, ave->ipc.cpu + offset, bytes); + ave->response_bytes = bytes + 24; + ave->replies++; + /* Reply echoes use zero flags on the observed transport. */ + ret = ave_ring_send(ave, &ave->channel[ch], addr, bytes, 0); + if (ret) + goto failed; + } + } + if (ack && ave->response_bytes) { + ret = 0; + goto out; + } + usleep_range(100, 200); + } while (ktime_get_ns() < deadline); + ret = -ETIMEDOUT; +failed: + ave->command_failed = true; +out: + mutex_unlock(&ave->lock); + return ret ?: len; +} + +static ssize_t ave_command_read(struct file *file, char __user *data, size_t len, loff_t *pos) +{ + struct apple_ave *ave = file->private_data; + ssize_t ret; + + mutex_lock(&ave->lock); + ret = simple_read_from_buffer(data, len, pos, ave->response, ave->response_bytes); + mutex_unlock(&ave->lock); + return ret; +} + +static const struct file_operations ave_command_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = ave_command_write, + .read = ave_command_read, +}; + +/* Drain later firmware events without publishing another command. Results + * replace the response buffer and consist of one or more 24-byte-header + * records, each followed by its declared payload. Never discard silently. + */ +static ssize_t ave_poll_write(struct file *file, const char __user *data, size_t len, loff_t *pos) +{ + struct apple_ave *ave = file->private_data; + bool requested; + u64 deadline, addr; + u32 bytes, flags, offset; + int ret; + + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + ret = kstrtobool_from_user(data, len, &requested); + if (ret || !requested) + return ret ?: -EINVAL; + mutex_lock(&ave->lock); + if (!ave->ipc_ready || ave->command_failed) { + ret = -EBUSY; + goto out; + } + ave->response_bytes = 0; + deadline = ktime_get_ns() + NSEC_PER_SEC; + do { + u32 pending = readl(ave->mailbox + 0x10); + u8 *dst; + + if (pending) + writel(pending, ave->mailbox + 0x10); + ret = ave_ring_receive(ave, &ave->channel[1], &addr, &bytes, &flags); + if (ret == -EAGAIN) { + usleep_range(100, 200); + continue; + } + if (ret || ave_fw_offset(ave, addr, bytes, &offset) || + bytes > sizeof(ave->response) - 24 || + ave->response_bytes > sizeof(ave->response) - 24 - bytes) { + ret = -EPROTO; + goto failed; + } + dst = ave->response + ave->response_bytes; + put_unaligned_le32(2, dst); + put_unaligned_le32(bytes, dst + 4); + put_unaligned_le32(flags, dst + 8); + put_unaligned_le32(0, dst + 12); + put_unaligned_le64(addr, dst + 16); + memcpy(dst + 24, ave->ipc.cpu + offset, bytes); + ave->response_bytes += 24 + bytes; + ave->replies++; + ret = ave_ring_send(ave, &ave->channel[1], addr, bytes, 0); + if (ret) + goto failed; + } while (ktime_get_ns() < deadline); + ret = 0; + goto out; +failed: + ave->command_failed = true; +out: + mutex_unlock(&ave->lock); + return ret ?: len; +} + +static const struct file_operations ave_poll_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = ave_poll_write, +}; + +/* Bounded staging area for image/reference/output surfaces. Userspace offsets + * start at zero here, beyond all handshake, ring and command allocations. + * The status file supplies both DART and firmware addresses for this arena. + */ +static ssize_t ave_arena_read(struct file *file, char __user *data, size_t len, loff_t *pos) +{ + struct apple_ave *ave = file->private_data; + ssize_t ret; + + mutex_lock(&ave->lock); + dma_rmb(); + ret = simple_read_from_buffer(data, len, pos, ave->ipc.cpu + AVE_ARENA_OFF, + ave->ipc.size - AVE_ARENA_OFF); + mutex_unlock(&ave->lock); + return ret; +} + +static ssize_t ave_arena_write(struct file *file, const char __user *data, size_t len, loff_t *pos) +{ + struct apple_ave *ave = file->private_data; + ssize_t ret; + + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + mutex_lock(&ave->lock); + if (!ave->ipc_ready || ave->command_failed) + ret = -EBUSY; + else + ret = simple_write_to_buffer(ave->ipc.cpu + AVE_ARENA_OFF, + ave->ipc.size - AVE_ARENA_OFF, pos, data, len); + dma_wmb(); + mutex_unlock(&ave->lock); + return ret; +} + +static loff_t ave_arena_seek(struct file *file, loff_t offset, int whence) +{ + return fixed_size_llseek(file, offset, whence, AVE_IPC_BYTES - AVE_ARENA_OFF); +} + +static const struct file_operations ave_arena_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = ave_arena_read, + .write = ave_arena_write, + .llseek = ave_arena_seek, +}; diff --git a/drivers/media/platform/apple/ave/apple-ave.c b/drivers/media/platform/apple/ave/apple-ave.c new file mode 100644 index 0000000000000..d2cf7abadcadf --- /dev/null +++ b/drivers/media/platform/apple/ave/apple-ave.c @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* T8140 AVE bring-up: resources, firmware negotiation and IPC transport. + * Experimental root-only command interface. No registered encoder yet. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define AVE_IPC_BYTES (20 * SZ_1M) +#define AVE_QUEUE_BYTES SZ_64K +#define AVE_FW_IOVA 0x10000000000ULL +#define AVE_LOG_BYTES 0x24000 +#define AVE_LOG_USED 0x20200 + +static bool enable; +module_param(enable, bool, 0400); +MODULE_PARM_DESC(enable, "Opt in to the experimental AVE transport"); + +/* Read the known Uranus registers once after PM resume. The separate hello + * command pins all resources because reset/quiescence is not yet qualified. + */ +static bool snapshot; +module_param(snapshot, bool, 0400); +MODULE_PARM_DESC(snapshot, "Capture powered AVE registers once without writing them"); + +struct ave_firmware { + phys_addr_t physical; + u64 iova; + size_t size; + u32 crc; + unsigned int verified_pages; +}; + +struct ave_buffer { + void *cpu; + dma_addr_t dma; + size_t size; + unsigned int verified_pages; +}; + +struct ave_channel { + char name[64]; + u32 mode, interrupt, slots; + u32 offset; + int consumer, producer; +}; + +struct apple_ave { + struct device *dev; + struct device *power[5]; + unsigned int num_power; + struct iommu_domain *domain; + struct ave_buffer ipc; + struct ave_buffer queue; + struct ave_buffer log; + struct ave_buffer heap; + struct ave_firmware fw[2]; + bool firmware_verified; + bool start_requested; + int start_result; + struct mutex lock; + void __iomem *coproc, *mailbox; + u32 hello[4]; + u64 ipc_fw_base; + u32 ipc_reply[4], channel_bytes, extra_bytes; + const char *phase; + bool ipc_ready, command_failed; + struct ave_channel channel[2]; + u32 commands, replies, transport_acks; + u8 response[SZ_64K]; + size_t response_bytes; + struct debugfs_blob_wrapper ipc_blob; + struct debugfs_blob_wrapper log_blob; + struct debugfs_blob_wrapper config_blob; + u64 vcbar; + u32 core_control, core_status, mailbox_status, scratch[8]; + struct dentry *debugfs; +}; + +static int ave_alloc_buffer(struct apple_ave *ave, struct ave_buffer *buf, size_t size); +#include "apple-ave-ipc.h" + +/* Cold-boot hello only. A successful response still needs the remaining IPC + * negotiation. Keep the module, PM reference and DMA buffers pinned until a + * target reboot, on both success and timeout; no unproven stop/free sequence. + */ +static ssize_t ave_start_write(struct file *file, const char __user *data, + size_t len, loff_t *ppos) +{ + struct apple_ave *ave = file->private_data; + __le32 *cfg = ave->queue.cpu, *log = ave->log.cpu; + bool requested; + u32 status; + int ret; + + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + ret = kstrtobool_from_user(data, len, &requested); + if (ret || !requested) + return ret ?: -EINVAL; + mutex_lock(&ave->lock); + if (ave->start_requested) { + ret = -EALREADY; + goto out; + } + if (!ave->firmware_verified || readq(ave->coproc + 0x50000) != (AVE_FW_IOVA | 1) || + readl(ave->coproc + 0x600044) || readl(ave->mailbox + 0x10)) { + ret = -EBUSY; + goto out; + } + for (unsigned int i = 0; i < 8; i++) { + if (readl(ave->mailbox + 0x18 + i * 4)) { + ret = -EBUSY; + goto out; + } + } + /* Startup values observed with the qualified 25G76 firmware: device + * enum 29, one active unit/group and sub-ID mask 1. Firmware maps the + * full page-aligned log allocation; the ring header advertises usage. + */ + memset(cfg, 0, 0x40); + cfg[1] = cpu_to_le32(29); + cfg[2] = cpu_to_le32(1); + cfg[3] = cpu_to_le32(1); + cfg[4] = cpu_to_le32(1); + cfg[8] = cpu_to_le32(lower_32_bits(ave->log.dma)); + cfg[9] = cpu_to_le32(upper_32_bits(ave->log.dma)); + cfg[10] = cpu_to_le32(AVE_LOG_BYTES); + cfg[12] = cpu_to_le32(0x77); + /* Log header: 128 KiB ring after its 512-byte header. */ + log[0] = cpu_to_le32(AVE_LOG_USED); + log[1] = cpu_to_le32(0x200); + log[2] = cpu_to_le32(0x20000); + dma_wmb(); + __module_get(THIS_MODULE); + ave->start_requested = true; + ave->phase = "hello"; + ave->start_result = -EINPROGRESS; + dev_info(ave->dev, "AVE_START_ONCE cfg=%pad log=%pad resources_pinned_until_reboot=1\n", + &ave->queue.dma, &ave->log.dma); + writel(0x08042006, ave->mailbox + 0x18); + writel(lower_32_bits(ave->queue.dma), ave->mailbox + 0x1c); + writel(upper_32_bits(ave->queue.dma), ave->mailbox + 0x20); + /* Preserve the protected firmware mappings inherited from iBoot. */ + writel(1, ave->coproc + 0x600808); + writel(0, ave->coproc + 0x600044); + writel(0x10000, ave->coproc + 0x600400); + writel(0x10, ave->coproc + 0x600044); + ret = readl_poll_timeout(ave->mailbox + 0x10, status, status & 1, 50, 5000000); + ave->start_result = ret; + ave->core_control = readl(ave->coproc + 0x600044); + ave->core_status = readl(ave->coproc + 0x600048); + ave->mailbox_status = status; + for (unsigned int i = 0; i < 8; i++) + ave->scratch[i] = readl(ave->mailbox + 0x18 + i * 4); + for (unsigned int i = 0; i < 4; i++) + ave->hello[i] = ave->scratch[i]; + dma_rmb(); + dev_info(ave->dev, "AVE_HELLO_RESULT result=%d status=%#x words=%#x,%#x,%#x,%#x core=%#x\n", + ret, status, ave->hello[0], ave->hello[1], ave->hello[2], ave->hello[3], ave->core_status); + if (!ret) + ret = ave_negotiate_ipc(ave); + ave->start_result = ret; + dev_info(ave->dev, "AVE_IPC_RESULT result=%d phase=%s ready=%u\n", + ret, ave->phase, ave->ipc_ready); +out: + mutex_unlock(&ave->lock); + return ret ?: len; +} + +static const struct file_operations ave_start_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = ave_start_write, +}; + +static int ave_verify_firmware(struct apple_ave *ave) +{ + static const char * const crc_names[] = { + "aurora,ave-text-crc32", "aurora,ave-data-crc32", + }; + static const size_t sizes[] = { 0x14c000, 0x130000 }; + struct device_node *chosen, *node; + unsigned int i; + int ret = 0; + + if (of_count_phandle_with_args(ave->dev->of_node, "memory-region", NULL) != 2) + return dev_err_probe(ave->dev, -EINVAL, "Expected two live AVE firmware reservations\n"); + chosen = of_find_node_by_path("/chosen"); + if (!chosen) + return -ENOENT; + for (i = 0; i < ARRAY_SIZE(ave->fw); i++) { + struct ave_firmware *fw = &ave->fw[i]; + struct resource res; + const char *crc_string; + u32 maps[5], expected_crc; + void *cpu; + size_t offset; + + node = of_parse_phandle(ave->dev->of_node, "memory-region", i); + ret = of_address_to_resource(node, 0, &res); + if (!ret) + ret = of_property_read_u32_array(node, "iommu-addresses", maps, ARRAY_SIZE(maps)); + if (!ret && (!of_property_read_bool(node, "no-map") || + !of_device_is_compatible(node, "apple,asc-mem"))) + ret = -EINVAL; + of_node_put(node); + if (ret) + break; + fw->physical = res.start; + fw->size = resource_size(&res); + fw->iova = (u64)maps[1] << 32 | maps[2]; + if (fw->size != sizes[i] || !IS_ALIGNED(fw->physical, PAGE_SIZE) || + maps[0] != ave->dev->of_node->phandle || maps[3] || maps[4] != sizes[i] || + fw->iova != AVE_FW_IOVA + (i ? sizes[0] : 0)) { + ret = -EINVAL; + break; + } + /* The IOMMU core owns these translated reservations across module + * unloads. DMA allocation must never reuse their IOVAs. + */ + for (offset = 0; offset < fw->size; offset += PAGE_SIZE) { + if (iommu_iova_to_phys(ave->domain, fw->iova + offset) != fw->physical + offset) { + dev_err(ave->dev, "Firmware %u mapping mismatch at %#zx\n", i, offset); + ret = -EIO; + break; + } + fw->verified_pages++; + } + if (ret) + break; + ret = of_property_read_string(chosen, crc_names[i], &crc_string); + if (!ret) + ret = kstrtou32(crc_string, 16, &expected_crc); + if (ret) + break; + cpu = memremap(fw->physical, fw->size, MEMREMAP_WB); + if (!cpu) { + ret = -ENOMEM; + break; + } + fw->crc = crc32_le(~0U, cpu, fw->size) ^ ~0U; + memunmap(cpu); + if (fw->crc != expected_crc) { + dev_err(ave->dev, "Firmware %u changed since the boot preflight\n", i); + ret = -EILSEQ; + break; + } + /* Only the two inspected 9003.78.0 text images are supported. + * CRCs detect build drift; they are not firmware authentication. + */ + if (!i && fw->crc != 0x2ce7359e && fw->crc != 0x69c54454) { + dev_err(ave->dev, "Unsupported AVE firmware text CRC %#x\n", fw->crc); + ret = -ENODEV; + break; + } + } + of_node_put(chosen); + if (!ret) + ave->firmware_verified = true; + return ret; +} + +/* Check the real IOMMU page table against the CPU mapping of each allocation. + * This proves mappings, not that firmware has performed a DMA transaction. + */ +static int ave_verify_mapping(struct apple_ave *ave, struct ave_buffer *buf) +{ + size_t offset; + + if (!IS_ALIGNED(buf->dma, PAGE_SIZE) || !IS_ALIGNED(buf->size, PAGE_SIZE)) + return -EINVAL; + for (offset = 0; offset < buf->size; offset += PAGE_SIZE) { + void *cpu = buf->cpu + offset; + struct page *page = is_vmalloc_addr(cpu) ? vmalloc_to_page(cpu) : virt_to_page(cpu); + phys_addr_t physical = iommu_iova_to_phys(ave->domain, buf->dma + offset); + + if (!page || physical != page_to_phys(page)) { + dev_err(ave->dev, "DMA mapping mismatch at offset %#zx\n", offset); + return -EIO; + } + buf->verified_pages++; + } + return 0; +} + +static int ave_alloc_buffer(struct apple_ave *ave, struct ave_buffer *buf, size_t size) +{ + buf->size = size; + buf->cpu = dma_alloc_coherent(ave->dev, size, &buf->dma, GFP_KERNEL); + if (!buf->cpu) + return -ENOMEM; + memset(buf->cpu, 0, size); + return ave_verify_mapping(ave, buf); +} + +static void ave_free_buffer(struct apple_ave *ave, struct ave_buffer *buf) +{ + if (buf->cpu) + dma_free_coherent(ave->dev, buf->size, buf->cpu, buf->dma); + buf->cpu = NULL; +} + +static void ave_power_off(struct apple_ave *ave) +{ + while (ave->num_power) { + struct device *pd = ave->power[--ave->num_power]; + + pm_runtime_put_sync(pd); + dev_pm_domain_detach(pd, true); + } +} + +static int ave_power_on(struct apple_ave *ave) +{ + static const char * const names[] = { "dma", "pipe4", "pipe5", "me0", "me1" }; + int ret; + + for (unsigned int i = 0; i < ARRAY_SIZE(names); i++) { + struct device *pd = dev_pm_domain_attach_by_name(ave->dev, names[i]); + + if (IS_ERR(pd)) + return PTR_ERR(pd); + if (!pd) + return -ENODEV; + ret = pm_runtime_resume_and_get(pd); + if (ret < 0) { + dev_pm_domain_detach(pd, true); + return ret; + } + ave->power[ave->num_power++] = pd; + } + return 0; +} + +static int ave_status_show(struct seq_file *s, void *unused) +{ + struct apple_ave *ave = s->private; + + mutex_lock(&ave->lock); + seq_printf(s, "state=%s\nfirmware_start_requested=%u\nstart_result=%d\n", + ave->start_requested ? (ave->start_result ? "startup-failed" : "ipc-ready") : + "firmware-mapped", ave->start_requested, ave->start_result); + seq_printf(s, "hardware_dma_tested=%u\nencoder_registered=0\nphase=%s\n", + ave->ipc_ready, ave->phase ?: "not-started"); + seq_printf(s, "ipc_ready=%u\nipc_fw_base=%#llx\nchannel_bytes=%#x\nextra_bytes=%#x\n", + ave->ipc_ready, ave->ipc_fw_base, ave->channel_bytes, ave->extra_bytes); + seq_printf(s, "commands=%u\nreplies=%u\ntransport_acks=%u\ncommand_failed=%u\nresponse_bytes=%zu\n", + ave->commands, ave->replies, ave->transport_acks, ave->command_failed, ave->response_bytes); + seq_printf(s, "arena_bytes=%zu\narena_ipc_offset=%#x\narena_dma=%#llx\narena_fw=%#llx\n", + ave->ipc.size - AVE_ARENA_OFF, AVE_ARENA_OFF, + (u64)ave->ipc.dma + AVE_ARENA_OFF, ave->ipc_fw_base + AVE_ARENA_OFF); + for (unsigned int i = 0; i < 2; i++) { + struct ave_channel *c = &ave->channel[i]; + seq_printf(s, "channel%u_name=%s\nchannel%u_mode=%u\nchannel%u_interrupt=%u\n" + "channel%u_slots=%u\nchannel%u_offset=%#x\nchannel%u_consumer=%d\nchannel%u_producer=%d\n", + i, c->name, i, c->mode, i, c->interrupt, i, c->slots, + i, c->offset, i, c->consumer, i, c->producer); + } + seq_printf(s, "page_size=%lu\n", PAGE_SIZE); + seq_printf(s, "powered_domains=%u\n", ave->num_power); + seq_printf(s, "ipc_bytes=%zu\nipc_dma=%pad\nipc_verified_pages=%u\n", + ave->ipc.size, &ave->ipc.dma, ave->ipc.verified_pages); + seq_printf(s, "queue_bytes=%zu\nqueue_dma=%pad\nqueue_verified_pages=%u\n", + ave->queue.size, &ave->queue.dma, ave->queue.verified_pages); + seq_printf(s, "log_bytes=%zu\nlog_dma=%pad\nlog_verified_pages=%u\n", + ave->log.size, &ave->log.dma, ave->log.verified_pages); + seq_printf(s, "heap_bytes=%zu\nheap_dma=%pad\nheap_verified_pages=%u\n", + ave->heap.size, &ave->heap.dma, ave->heap.verified_pages); + for (unsigned int i = 0; i < ARRAY_SIZE(ave->fw); i++) + seq_printf(s, "fw%u_physical=%pa\nfw%u_iova=%#llx\nfw%u_bytes=%zu\n" + "fw%u_verified_pages=%u\nfw%u_crc32=%08x\n", + i, &ave->fw[i].physical, i, ave->fw[i].iova, i, ave->fw[i].size, + i, ave->fw[i].verified_pages, i, ave->fw[i].crc); + seq_printf(s, "register_snapshot=%u\n", snapshot); + if (snapshot) { + seq_printf(s, "vcbar=%#llx\ncore_control=%#x\ncore_status=%#x\nmailbox_status=%#x\n", + ave->vcbar, ave->core_control, ave->core_status, ave->mailbox_status); + for (unsigned int i = 0; i < ARRAY_SIZE(ave->scratch); i++) + seq_printf(s, "scratch%u=%#x\n", i, ave->scratch[i]); + } + seq_printf(s, "resources_pinned_until_reboot=%u\n", ave->start_requested); + mutex_unlock(&ave->lock); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ave_status); + +static int apple_ave_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_ave *ave; + void __iomem *regs, *mailbox, *coproc; + int ret; + + if (!enable) + return -ENODEV; + if (!of_machine_is_compatible("apple,j700") || PAGE_SIZE != SZ_16K) + return -ENODEV; + ave = devm_kzalloc(dev, sizeof(*ave), GFP_KERNEL); + if (!ave) + return -ENOMEM; + ave->dev = dev; + mutex_init(&ave->lock); + /* Claim only dedicated AVE ranges, never the shared PMGR/SRAM windows. */ + regs = devm_platform_ioremap_resource_byname(pdev, "codec"); + if (IS_ERR(regs)) + return PTR_ERR(regs); + mailbox = devm_platform_ioremap_resource_byname(pdev, "mailbox"); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + coproc = devm_platform_ioremap_resource_byname(pdev, "coproc"); + if (IS_ERR(coproc)) + return PTR_ERR(coproc); + ave->coproc = coproc; + ave->mailbox = mailbox; + ave->domain = iommu_get_domain_for_dev(dev); + if (!ave->domain || !(ave->domain->type & __IOMMU_DOMAIN_PAGING)) + return dev_err_probe(dev, -ENODEV, "A translated IOMMU domain is required\n"); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (ret) + return ret; + ret = ave_power_on(ave); + if (ret) + goto release_power; + ret = devm_pm_runtime_enable(dev); + if (ret) + goto release_power; + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + goto release_power; + ret = ave_alloc_buffer(ave, &ave->ipc, AVE_IPC_BYTES); + if (ret) + goto free_buffers; + ret = ave_alloc_buffer(ave, &ave->queue, AVE_QUEUE_BYTES); + if (ret) + goto free_buffers; + ret = ave_alloc_buffer(ave, &ave->log, AVE_LOG_BYTES); + if (ret) + goto free_buffers; + ret = ave_verify_firmware(ave); + if (ret) + goto free_buffers; + if (snapshot) { + ave->vcbar = readq(coproc + 0x50000); + ave->core_control = readl(coproc + 0x600044); + ave->core_status = readl(coproc + 0x600048); + ave->mailbox_status = readl(mailbox + 0x10); + for (unsigned int i = 0; i < ARRAY_SIZE(ave->scratch); i++) + ave->scratch[i] = readl(mailbox + 0x18 + i * 4); + } + platform_set_drvdata(pdev, ave); + ave->debugfs = debugfs_create_dir(dev_name(dev), NULL); + debugfs_create_file("ave-status", 0400, ave->debugfs, ave, &ave_status_fops); + debugfs_create_file("ave-start-once", 0200, ave->debugfs, ave, &ave_start_fops); + debugfs_create_file("ave-command", 0600, ave->debugfs, ave, &ave_command_fops); + debugfs_create_file("ave-poll", 0200, ave->debugfs, ave, &ave_poll_fops); + debugfs_create_file("ave-arena", 0600, ave->debugfs, ave, &ave_arena_fops); + ave->ipc_blob.data = ave->ipc.cpu; + ave->ipc_blob.size = ave->ipc.size; + debugfs_create_blob("ave-ipc-memory", 0400, ave->debugfs, &ave->ipc_blob); + ave->log_blob.data = ave->log.cpu; + ave->log_blob.size = ave->log.size; + debugfs_create_blob("ave-firmware-log", 0400, ave->debugfs, &ave->log_blob); + ave->config_blob.data = ave->queue.cpu; + ave->config_blob.size = 0x40; + debugfs_create_blob("ave-boot-config", 0400, ave->debugfs, &ave->config_blob); + dev_info(dev, "AVE_DMA_READY ipc=%zu queue=%zu verified_pages=%u firmware_start=0\n", + ave->ipc.size, ave->queue.size, + ave->ipc.verified_pages + ave->queue.verified_pages); + dev_info(dev, "AVE_FIRMWARE_MAPPED pages=%u contents_match_boot=1 register_snapshot=%u\n", + ave->fw[0].verified_pages + ave->fw[1].verified_pages, snapshot); + return 0; + +free_buffers: + ave_free_buffer(ave, &ave->heap); + ave_free_buffer(ave, &ave->log); + ave_free_buffer(ave, &ave->queue); + ave_free_buffer(ave, &ave->ipc); + pm_runtime_put_sync(dev); +release_power: + ave_power_off(ave); + return dev_err_probe(dev, ret, "AVE DMA allocation/verification failed\n"); +} + +static void apple_ave_remove(struct platform_device *pdev) +{ + struct apple_ave *ave = platform_get_drvdata(pdev); + + debugfs_remove_recursive(ave->debugfs); + ave_free_buffer(ave, &ave->heap); + ave_free_buffer(ave, &ave->log); + ave_free_buffer(ave, &ave->queue); + ave_free_buffer(ave, &ave->ipc); + pm_runtime_put_sync(ave->dev); + ave_power_off(ave); + dev_info(ave->dev, "AVE_DMA_RELEASED\n"); +} + +static const struct of_device_id apple_ave_match[] = { + { .compatible = "apple,t8140-ave" }, + { } +}; +MODULE_DEVICE_TABLE(of, apple_ave_match); + +static struct platform_driver apple_ave_driver = { + .probe = apple_ave_probe, + .remove = apple_ave_remove, + .driver = { + .name = "apple-ave", + .of_match_table = apple_ave_match, + .suppress_bind_attrs = true, + }, +}; +module_platform_driver(apple_ave_driver); +MODULE_DESCRIPTION("Apple T8140 experimental video encoder transport"); +MODULE_LICENSE("GPL"); From f1d29b8afd4e364f730f666d7c247276c8ba50ca Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 11:57:48 -0400 Subject: [PATCH 0333/1579] arm64: apple: preserve registers across WFI on T8142 --- Documentation/arch/arm64/apple-j813.rst | 30 ++++++++++++++++++++----- arch/arm64/include/asm/cputype.h | 4 ++++ arch/arm64/kernel/idle.c | 6 +++-- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst index b2fa78079dd46..28e13f9aafffc 100644 --- a/Documentation/arch/arm64/apple-j813.rst +++ b/Documentation/arch/arm64/apple-j813.rst @@ -8,16 +8,36 @@ J813 is the 13-inch M5 MacBook Air (T8142, Mac17,3). This configuration provides an initial RAM-only boot path: one boot CPU, AICv3, architectural timers, DockChannel console and the framebuffer left by the boot firmware. -The device tree describes all ten CPU affinities so the loader can identify -the boot CPU. It does not qualify SMP. Keep secondary cores stopped in -m1n1 and use ``maxcpus=1`` for this milestone. The loader removes unavailable -CPUs and fills the release addresses of the remaining nodes. The P-core +The device tree describes all ten CPU affinities. For the initial one-CPU +baseline, keep secondary cores stopped in m1n1 and use ``maxcpus=1``. +The loader removes unavailable CPUs and fills the release addresses of the +remaining nodes. The P-core affinities are 0x10100 through 0x10103, rather than the 0x100 through 0x103 values in the Apple Device Tree. The CPU compatible strings identify T8142 explicitly. The firmware's reused ``sawtooth`` and ``everest`` names do not identify the M5 microarchitecture. +SMP bring-up +============ + +M5 secondary CPUs can return from WFI with cleared general-purpose registers. +Both M5 MIDRs (parts 0x062 and 0x063) select the Apple register-preserving idle +path. Without it, all ten CPUs can start and then panic in their idle tasks. + +SMP also requires the M5-capable m1n1 secondary-start path: dense PMGR CPU +masks, correct reset vectors and per-core stacks. Verify all ten cores there +before preparing the FDT, and use ``maxcpus=10`` for the Linux test. The +existing J813 DT and configuration support this handoff. + +On J813, this path reached the RAM shell with CPUs 0-9 online. A five-second +sleep advanced every CPU's architectural timer count by 1,251 interrupts. +Ten concurrent, individually pinned jobs each hashed 16 MiB of zeros to +``080acf35a507ac9849cfcba47dc2ad83e01b75663a516279c8b9d243b719643e``. +Verify affinity through ``Cpus_allowed_list`` and retain the console output, +kernel configuration hash and payload hashes when repeating the test. +Hotplug, deep sleep and KVM remain unqualified. + Bootloader requirements ======================= @@ -103,7 +123,7 @@ RAM boot. Preserve the complete console transcript and a hash manifest; an Image build or successful loader exit alone is not a boot result. Review captures for device identifiers and boot entropy before publishing. -This milestone does not establish SMP, CPU hotplug, KVM guest timers, +These tests do not establish CPU hotplug, KVM guest timers, cpufreq, suspend, native DCP, GPU acceleration, built-in input, USB, storage, networking or audio support. The disabled Samsung UART node records its measured resources; the qualified console path is DockChannel. diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 632f980ec55e5..809e39962fe23 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -155,6 +155,8 @@ #define APPLE_CPU_PART_M2_AVALANCHE_MAX 0x039 #define APPLE_CPU_PART_TAHITI_E 0x060 #define APPLE_CPU_PART_TAHITI_P 0x061 +#define APPLE_CPU_PART_T8142_E 0x062 +#define APPLE_CPU_PART_T8142_P 0x063 #define APPLE_CPU_PART_T8152_E 0x072 #define APPLE_CPU_PART_T8152_P 0x073 @@ -252,6 +254,8 @@ #define MIDR_APPLE_M2_AVALANCHE_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_AVALANCHE_MAX) #define MIDR_APPLE_TAHITI_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_TAHITI_E) #define MIDR_APPLE_TAHITI_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_TAHITI_P) +#define MIDR_APPLE_T8142_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8142_E) +#define MIDR_APPLE_T8142_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8142_P) #define MIDR_APPLE_T8152_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8152_E) #define MIDR_APPLE_T8152_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8152_P) #define MIDR_AMPERE1 MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1) diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c index 972164e78874c..14cde8592c86e 100644 --- a/arch/arm64/kernel/idle.c +++ b/arch/arm64/kernel/idle.c @@ -41,9 +41,9 @@ early_param("idle", setup_idle); #ifdef CONFIG_ARCH_APPLE /* - * Apple T8140 (A18 Pro) secondary CPUs and T8152 (M6) CPUs can return from + * Apple T8140 (A18 Pro), T8142 (M5), and T8152 (M6) CPUs can return from * WFI with general-purpose registers cleared. T8140 also loses FP/SIMD - * state; conservatively preserve it on both parts. The key is enabled once + * state; conservatively preserve it on all three parts. The key is enabled once * at boot on those parts; everywhere else the check is a patched-out branch. */ static DEFINE_STATIC_KEY_FALSE(apple_wfi_loses_regs); @@ -51,6 +51,8 @@ static DEFINE_STATIC_KEY_FALSE(apple_wfi_loses_regs); static const struct midr_range apple_wfi_loses_regs_cpus[] __initconst = { MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_E), MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_P), + MIDR_ALL_VERSIONS(MIDR_APPLE_T8142_E), + MIDR_ALL_VERSIONS(MIDR_APPLE_T8142_P), MIDR_ALL_VERSIONS(MIDR_APPLE_T8152_E), MIDR_ALL_VERSIONS(MIDR_APPLE_T8152_P), {} From 80cabfefcdc52036a7902268ff42f2a9370fe5c2 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:26:30 -0400 Subject: [PATCH 0334/1579] cpufreq: apple: Add bounded T8152 E and P/M performance domains --- Documentation/arch/arm64/apple-m6-cpu.rst | 29 +++++++++ .../cpufreq/apple,cluster-cpufreq.yaml | 1 + arch/arm64/boot/dts/apple/t8152-j873g.dts | 59 +++++++++++++++++++ drivers/cpufreq/apple-soc-cpufreq.c | 35 +++++++++-- 4 files changed, 118 insertions(+), 6 deletions(-) create mode 100644 Documentation/arch/arm64/apple-m6-cpu.rst diff --git a/Documentation/arch/arm64/apple-m6-cpu.rst b/Documentation/arch/arm64/apple-m6-cpu.rst new file mode 100644 index 0000000000000..046872e83e09a --- /dev/null +++ b/Documentation/arch/arm64/apple-m6-cpu.rst @@ -0,0 +1,29 @@ +M6 CPU bring-up +============== + +J873g has six E CPUs and a shared performance domain containing two P CPUs +and four M CPUs. The T8152 ACC request uses a five-bit hardware state, with +state 2 as the first ADT operating point. Do not use the older voltage or PLL +initialization sequences. + +Failure cases and qualification +------------------------------ + +Before testing, account for incorrect MPIDRs, incomplete spin-table release, +broken cross-CPU interrupts, a stalled architectural timer, mismatched OPP +indices, an unknown inherited request, a stuck busy bit, an unacknowledged +transition, and another agent changing the request. A transition failure must +latch until the policy is recreated. A requested frequency is not a measured +frequency. + +The initial tables expose only states 2 and 3. Higher states require an SMC +thermal policy; this port does not qualify them. The loader must hand over +within the advertised table. Keep firmware voltage, PLL and throttling +configuration intact. + +On physical J873g, capture the source revision, image and DTB hashes, boot log, +online CPU list, per-CPU workload completion, interrupt counters and both +cpufreq policies. Exercise both exposed states in each domain, compare timed +dependent workloads, restore the low states and check the kernel log. Save +the commands and a machine-readable receipt. Native m1n1 SMP results alone +do not qualify Linux scheduling. diff --git a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml index 8e87c0f6d3d45..e4b1102553f72 100644 --- a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +++ b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml @@ -36,6 +36,7 @@ properties: - const: apple,s5l8960x-cluster-cpufreq - const: apple,s5l8960x-cluster-cpufreq - const: apple,t8140-cluster-cpufreq + - const: apple,t8152-cluster-cpufreq - items: - const: apple,t6020-cluster-cpufreq - const: apple,t8112-cluster-cpufreq diff --git a/arch/arm64/boot/dts/apple/t8152-j873g.dts b/arch/arm64/boot/dts/apple/t8152-j873g.dts index bd259e3f7e364..6b5e14036c98a 100644 --- a/arch/arm64/boot/dts/apple/t8152-j873g.dts +++ b/arch/arm64/boot/dts/apple/t8152-j873g.dts @@ -62,6 +62,8 @@ reg = <0 0x0>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu1: cpu@1 { @@ -70,6 +72,8 @@ reg = <0 0x1>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu2: cpu@2 { @@ -78,6 +82,8 @@ reg = <0 0x2>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu3: cpu@3 { @@ -86,6 +92,8 @@ reg = <0 0x3>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu4: cpu@4 { @@ -94,6 +102,8 @@ reg = <0 0x4>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu5: cpu@5 { @@ -102,6 +112,8 @@ reg = <0 0x5>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu6: cpu@10100 { @@ -110,6 +122,8 @@ reg = <0 0x10100>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu7: cpu@10101 { @@ -118,6 +132,8 @@ reg = <0 0x10101>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu8: cpu@40102 { @@ -126,6 +142,8 @@ reg = <0 0x40102>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu9: cpu@40103 { @@ -134,6 +152,8 @@ reg = <0 0x40103>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu10: cpu@40104 { @@ -142,6 +162,8 @@ reg = <0 0x40104>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu11: cpu@40105 { @@ -150,9 +172,35 @@ reg = <0 0x40105>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; }; + m6_e_opp: opp-table-e { + compatible = "operating-points-v2"; + opp-shared; + opp-972000000 { + opp-hz = /bits/ 64 <972000000>; + opp-level = <2>; + }; + opp-1152000000 { + opp-hz = /bits/ 64 <1152000000>; + opp-level = <3>; + }; + }; + m6_pm_opp: opp-table-pm { + compatible = "operating-points-v2"; + opp-shared; + opp-1440000000 { + opp-hz = /bits/ 64 <1440000000>; + opp-level = <2>; + }; + opp-1728000000 { + opp-hz = /bits/ 64 <1728000000>; + opp-level = <3>; + }; + }; timer { compatible = "arm,armv8-timer"; interrupts = , @@ -178,6 +226,17 @@ reg-names = "core", "event"; }; + m6_e_cpufreq: performance-controller@210e20000 { + compatible = "apple,t8152-cluster-cpufreq"; + reg = <2 0x10e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + m6_pm_cpufreq: performance-controller@211e20000 { + compatible = "apple,t8152-cluster-cpufreq"; + reg = <2 0x11e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + dockchannel: serial@30812c000 { compatible = "apple,dockchannel-uart"; reg = <3 0x0812c000 0 0x4000>, diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index c37fb272ada73..b53eb35b9cb57 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -64,6 +64,8 @@ struct apple_soc_cpufreq_info { bool has_ps2; bool verify_transition; + u32 transition_timeout_us; + u32 max_unmanaged_pstate; u64 min_pstate; u64 max_pstate; u64 cur_pstate_mask; @@ -118,12 +120,24 @@ static const struct apple_soc_cpufreq_info soc_t8112_info = { */ static const struct apple_soc_cpufreq_info soc_t8140_info = { .verify_transition = true, + .max_unmanaged_pstate = 2, .min_pstate = 1, .max_pstate = 31, .ps1_mask = APPLE_DVFS_CMD_PS1, .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, }; +/* T8152 ACC uses state 2 as the first OPP and shares P/M requests. */ +static const struct apple_soc_cpufreq_info soc_t8152_info = { + .verify_transition = true, + .transition_timeout_us = 2000, + .max_unmanaged_pstate = 3, + .min_pstate = 2, + .max_pstate = 31, + .ps1_mask = APPLE_DVFS_CMD_PS1, + .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, +}; + static const struct apple_soc_cpufreq_info soc_default_info = { .has_ps2 = false, .max_pstate = 15, @@ -153,9 +167,18 @@ static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { .compatible = "apple,t8140-cluster-cpufreq", .data = &soc_t8140_info, }, + { + .compatible = "apple,t8152-cluster-cpufreq", + .data = &soc_t8152_info, + }, {} }; +static u32 apple_soc_cpufreq_timeout(const struct apple_soc_cpufreq_info *info) +{ + return info->transition_timeout_us ?: APPLE_DVFS_TRANSITION_TIMEOUT; +} + static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu) { struct cpufreq_policy *policy; @@ -207,7 +230,7 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, !(reg & APPLE_DVFS_CMD_BUSY), 2, - APPLE_DVFS_TRANSITION_TIMEOUT)) { + apple_soc_cpufreq_timeout(priv->info))) { if (priv->info->verify_transition) { priv->transition_failed = true; apple_smc_thermal_cpu_fault(priv->thermal); @@ -246,7 +269,7 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, !(reg & APPLE_DVFS_CMD_BUSY) && FIELD_GET(APPLE_DVFS_CMD_PS1, reg) == pstate, - 2, APPLE_DVFS_TRANSITION_TIMEOUT)) { + 2, apple_soc_cpufreq_timeout(priv->info))) { /* Do not issue another request or guess a rollback after failure. */ priv->transition_failed = true; apple_smc_thermal_cpu_fault(priv->thermal); @@ -370,7 +393,7 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) /* Registration must not silently reset an unknown inherited state. */ ret = readq_poll_timeout_atomic(reg_base + APPLE_DVFS_CMD, cmd, !(cmd & APPLE_DVFS_CMD_BUSY), 2, - APPLE_DVFS_TRANSITION_TIMEOUT); + apple_soc_cpufreq_timeout(info)); if (ret) goto out_free_cpufreq_table; state = FIELD_GET(APPLE_DVFS_CMD_PS1, cmd); @@ -394,7 +417,7 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); if (!transition_latency) { /* Conservative transaction bound, not a measured transition time. */ - transition_latency = APPLE_DVFS_TRANSITION_TIMEOUT * NSEC_PER_USEC; + transition_latency = apple_soc_cpufreq_timeout(info) * NSEC_PER_USEC; if (info->verify_transition) transition_latency *= 2; } @@ -408,9 +431,9 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) bool higher = false; bool thermal; - /* P-states above 2 are only allowed under the SMC thermal policy. */ + /* Higher states require the SMC thermal policy. */ cpufreq_for_each_valid_entry(p, policy->freq_table) - if (p->driver_data > 2) + if (p->driver_data > info->max_unmanaged_pstate) higher = true; hwmon = of_find_compatible_node(NULL, NULL, "apple,smc-hwmon"); thermal = hwmon && of_property_read_bool(hwmon, "apple,cpu-thermal-policy"); From 86f65131599aa4b2852cddcbe57eeff2d775f336 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:37:37 -0400 Subject: [PATCH 0335/1579] dt-bindings: cpufreq: apple: describe the T8142 cluster controller Add the M5 compatible for its five-bit P-state command interface. The controller retains firmware voltage and CLPC/PMP configuration. Signed-off-by: Acelogic --- .../devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml index 8e87c0f6d3d45..44c2d45f2d7d1 100644 --- a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +++ b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml @@ -36,6 +36,7 @@ properties: - const: apple,s5l8960x-cluster-cpufreq - const: apple,s5l8960x-cluster-cpufreq - const: apple,t8140-cluster-cpufreq + - const: apple,t8142-cluster-cpufreq - items: - const: apple,t6020-cluster-cpufreq - const: apple,t8112-cluster-cpufreq From 8bfb0818c0f7d9e841e47731a35d80cb7da6466b Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:37:37 -0400 Subject: [PATCH 0336/1579] arm64: delay: avoid register-losing WFIT on Apple M5 T8142 can clear registers when returning from WFIT, just as it can with WFI in the idle task. The second Linux cluster-frequency transition on J813 reached __const_udelay while polling the command register, then returned with a cleared callee-saved pointer and faulted in cpufreq. Use the existing counter delay loop on both T8142 core types. The idle task's register/FP flush path cannot be used by arbitrary kernel delays which may have live SIMD state. With this change, repeated E/P state sweeps and pinned integer workloads complete on J813, followed by timer progress and SHA-256 jobs on all ten CPUs. Other CPU models retain their existing delay paths. Signed-off-by: Acelogic --- arch/arm64/lib/delay.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arch/arm64/lib/delay.c b/arch/arm64/lib/delay.c index e278e060e78a9..e0c5e615b5e5d 100644 --- a/arch/arm64/lib/delay.c +++ b/arch/arm64/lib/delay.c @@ -14,6 +14,7 @@ #include #include +#include #define USECS_TO_CYCLES(time_usecs) \ xloops_to_cycles((time_usecs) * 0x10C7UL) @@ -42,6 +43,18 @@ void __delay(unsigned long cycles) { cycles_t start = __delay_cycles(); + /* + * T8142 can clear registers on WFIT just as it does on WFI. A delay + * may run with live kernel SIMD state, so the idle task's save/flush + * path is not applicable here. Keep the core awake for short delays. + */ + if (IS_ENABLED(CONFIG_ARCH_APPLE)) { + u32 midr = read_cpuid_id() & MIDR_CPU_MODEL_MASK; + + if (midr == MIDR_APPLE_T8142_E || midr == MIDR_APPLE_T8142_P) + goto counter_delay; + } + if (alternative_has_cap_unlikely(ARM64_HAS_WFXT)) { u64 end = start + cycles; @@ -60,6 +73,7 @@ void __delay(unsigned long cycles) wfe(); } +counter_delay: while ((__delay_cycles() - start) < cycles) cpu_relax(); } From 296dbb45adff8654c6f1f8029e7b6b567f990e0b Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:37:37 -0400 Subject: [PATCH 0337/1579] cpufreq: apple: add T8142 command-state transitions Use five-bit P-state requests with the existing busy timeout, expected state and completion checks. Preserve firmware voltage, PLL and CLPC/PMP configuration; stop requests after an unexpected state or timeout. Separate T8140's SMC thermal-policy requirement from command verification so T8142 can use verified transitions with its bounded initial OPP table. T8140 retains its thermal requirement and behavior. Signed-off-by: Acelogic --- drivers/cpufreq/apple-soc-cpufreq.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index c37fb272ada73..84e0b454cd9e7 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -64,6 +64,7 @@ struct apple_soc_cpufreq_info { bool has_ps2; bool verify_transition; + bool needs_thermal_policy; u64 min_pstate; u64 max_pstate; u64 cur_pstate_mask; @@ -117,6 +118,16 @@ static const struct apple_soc_cpufreq_info soc_t8112_info = { * throttling configuration are left as the bootloader set them. */ static const struct apple_soc_cpufreq_info soc_t8140_info = { + .verify_transition = true, + .needs_thermal_policy = true, + .min_pstate = 1, + .max_pstate = 31, + .ps1_mask = APPLE_DVFS_CMD_PS1, + .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, +}; + +/* T8142 command-state readback; firmware retains CLPC/PMP ownership. */ +static const struct apple_soc_cpufreq_info soc_t8142_info = { .verify_transition = true, .min_pstate = 1, .max_pstate = 31, @@ -153,6 +164,10 @@ static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { .compatible = "apple,t8140-cluster-cpufreq", .data = &soc_t8140_info, }, + { + .compatible = "apple,t8142-cluster-cpufreq", + .data = &soc_t8142_info, + }, {} }; @@ -403,7 +418,7 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) policy->dvfs_possible_from_any_cpu = true; policy->fast_switch_possible = !info->verify_transition; policy->suspend_freq = freq_table[0].frequency; - if (info->verify_transition) { + if (info->needs_thermal_policy) { struct device_node *hwmon; bool higher = false; bool thermal; From 0abc47f7ebe2fcedaaf9c2eb06aaf283fdb4362c Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:37:37 -0400 Subject: [PATCH 0338/1579] arm64: dts: apple: enable bounded J813 CPU frequency control Describe both shared cluster policies and enable the userspace governor in the J813 configuration. Expose only the states up to the observed firmware handoff: E state 2 at 972 MHz and P state 10 at 3516 MHz. State 1 is a 300 MHz floor before the firmware SRAM table entries. Dependent decrement-loop timing on J813 confirmed every exposed state's frequency. Document repeatable state sweeps and qualification limits. Signed-off-by: Acelogic --- Documentation/arch/arm64/apple-j813.rst | 52 +++++++++++- arch/arm64/boot/dts/apple/t8142-j813.dts | 104 +++++++++++++++++++++++ arch/arm64/configs/j813.config | 5 ++ 3 files changed, 159 insertions(+), 2 deletions(-) diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst index 28e13f9aafffc..eb618f4d26588 100644 --- a/Documentation/arch/arm64/apple-j813.rst +++ b/Documentation/arch/arm64/apple-j813.rst @@ -38,6 +38,54 @@ Verify affinity through ``Cpus_allowed_list`` and retain the console output, kernel configuration hash and payload hashes when repeating the test. Hotplug, deep sleep and KVM remain unqualified. +CPU frequency bring-up +====================== + +The T8142 cluster controllers use five-bit P-state requests with busy and +command-state readback. The driver preserves firmware voltage, PLL and +CLPC/PMP configuration, and stops issuing requests after a timeout or an +unexpected command state. Unlike T8140, this initial J813 path does not +register a Linux SMC thermal policy. The DT exposes only states up to the +measured firmware handoff values; higher states remain unqualified. + +State 1 is 300 MHz on both clusters. The firmware SRAM voltage tables begin +at state 2: the E cluster exposes 300/972 MHz, and the P cluster exposes +300/1308/1608/1932/2220/2508/2820/3072/3300/3516 MHz. A dependent decrement +loop timed against CNTPCT_EL0 in m1n1 confirmed this mapping on J813. +``scaling_cur_freq`` reports the requested state, not an independent clock +measurement or a guarantee against firmware throttling. + +The configuration enables the userspace governor. On a fresh ten-core boot, +capture a sweep of both policies to the host serial log:: + + set -e + for p in /sys/devices/system/cpu/cpufreq/policy*; do + cat "$p/affected_cpus" + echo userspace > "$p/scaling_governor" + for round in 1 2; do + for f in $(cat "$p/scaling_available_frequencies"); do + echo "$f" > "$p/scaling_setspeed" + test "$(cat "$p/scaling_cur_freq")" = "$f" + echo "$p $f" + sleep 0.02 + done + done + cat "$p/cpuinfo_max_freq" > "$p/scaling_setspeed" + cat "$p/stats/total_trans" + done + dmesg + +Retain the transcript and payload hashes. Qualification also compared three +pairs of pinned, checksum-verified integer workloads at each cluster's +minimum and maximum states and checked every CPU's timer and pinned SHA-256 +job afterwards. Short frequency sweeps do not qualify sustained thermal +behavior, automatic governors, suspend or states above the handoff limit. + +T8142 can also lose registers on WFIT in delay loops. Frequency transition +polling exposed this as a cleared pointer in the cpufreq driver. T8142 uses +the existing counter delay loop, avoiding low-power waits with live kernel +register state. + Bootloader requirements ======================= @@ -78,7 +126,7 @@ cross-compiling with GCC from another architecture. With dtschema installed, check the affected bindings and DTB:: make ARCH=arm64 O="$out" CHECK_DTBS=y \ - DT_SCHEMA_FILES=arm/apple.yaml:arm/cpus.yaml:interrupt-controller/apple,aic2.yaml:serial/apple,dockchannel-uart.yaml:serial/samsung_uart.yaml:display/simple-framebuffer.yaml \ + DT_SCHEMA_FILES=arm/apple.yaml:arm/cpus.yaml:cpufreq/apple,cluster-cpufreq.yaml:opp/opp-v2-base.yaml:interrupt-controller/apple,aic2.yaml:serial/apple,dockchannel-uart.yaml:serial/samsung_uart.yaml:display/simple-framebuffer.yaml \ apple/t8142-j813.dtb Boot and qualification @@ -124,6 +172,6 @@ an Image build or successful loader exit alone is not a boot result. Review captures for device identifiers and boot entropy before publishing. These tests do not establish CPU hotplug, KVM guest timers, -cpufreq, suspend, native DCP, GPU acceleration, built-in input, USB, +sustained thermal behavior, suspend, native DCP, GPU acceleration, built-in input, USB, storage, networking or audio support. The disabled Samsung UART node records its measured resources; the qualified console path is DockChannel. diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index d56cd1dd2f560..e47839a68dfe4 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -58,6 +58,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x0>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -66,6 +68,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x1>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -74,6 +78,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x2>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -82,6 +88,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x3>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -90,6 +98,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x4>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -98,6 +108,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x5>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -106,6 +118,8 @@ compatible = "apple,t8142-p"; device_type = "cpu"; reg = <0 0x10100>; + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -114,6 +128,8 @@ compatible = "apple,t8142-p"; device_type = "cpu"; reg = <0 0x10101>; + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -122,6 +138,8 @@ compatible = "apple,t8142-p"; device_type = "cpu"; reg = <0 0x10102>; + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -130,6 +148,8 @@ compatible = "apple,t8142-p"; device_type = "cpu"; reg = <0 0x10103>; + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -144,6 +164,78 @@ interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; }; + /* State 1 is 300 MHz; firmware SRAM OPP tables begin at state 2. */ + /* Cap each cluster at its measured J813 firmware handoff state. */ + ecluster_opp: opp-table-0 { + compatible = "operating-points-v2"; + opp-shared; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-level = <1>; + }; + + opp-972000000 { + opp-hz = /bits/ 64 <972000000>; + opp-level = <2>; + }; + }; + + pcluster_opp: opp-table-1 { + compatible = "operating-points-v2"; + opp-shared; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-level = <1>; + }; + + opp-1308000000 { + opp-hz = /bits/ 64 <1308000000>; + opp-level = <2>; + }; + + opp-1608000000 { + opp-hz = /bits/ 64 <1608000000>; + opp-level = <3>; + }; + + opp-1932000000 { + opp-hz = /bits/ 64 <1932000000>; + opp-level = <4>; + }; + + opp-2220000000 { + opp-hz = /bits/ 64 <2220000000>; + opp-level = <5>; + }; + + opp-2508000000 { + opp-hz = /bits/ 64 <2508000000>; + opp-level = <6>; + }; + + opp-2820000000 { + opp-hz = /bits/ 64 <2820000000>; + opp-level = <7>; + }; + + opp-3072000000 { + opp-hz = /bits/ 64 <3072000000>; + opp-level = <8>; + }; + + opp-3300000000 { + opp-hz = /bits/ 64 <3300000000>; + opp-level = <9>; + }; + + opp-3516000000 { + opp-hz = /bits/ 64 <3516000000>; + opp-level = <10>; + }; + }; + clkref: clock-ref { compatible = "fixed-clock"; #clock-cells = <0>; @@ -157,6 +249,18 @@ ranges; nonposted-mmio; + cpufreq_e: performance-controller@210e20000 { + compatible = "apple,t8142-cluster-cpufreq"; + reg = <0x2 0x10e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + + cpufreq_p: performance-controller@211e20000 { + compatible = "apple,t8142-cluster-cpufreq"; + reg = <0x2 0x11e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + aic: interrupt-controller@381000000 { compatible = "apple,t8142-aic3", "apple,t8122-aic3"; interrupt-controller; diff --git a/arch/arm64/configs/j813.config b/arch/arm64/configs/j813.config index d8143807c2e3e..f944f153962ab 100644 --- a/arch/arm64/configs/j813.config +++ b/arch/arm64/configs/j813.config @@ -6,6 +6,11 @@ CONFIG_ARM64_16K_PAGES=y CONFIG_ARM64_VA_BITS_47=y CONFIG_SMP=y CONFIG_NR_CPUS=16 +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_ARM_APPLE_SOC_CPUFREQ=y CONFIG_ARM64_VHE=y CONFIG_ARM64_SVE=y CONFIG_ARM64_PAN=y From 581d7e36ea848ebae1256a5b71649b9d40a43986 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 10:14:44 -0700 Subject: [PATCH 0339/1579] drm/apple: route Type-C displays in compositor pairing order On the M2 Pro/Max 14" and 16" laptops (J414s/c, J416s/c) the Type-C ports share the external pipelines: dcpext0 (which also drives HDMI) and dcpext1, plus dcpext2/3 on the M2 Max boards. Since "drm/apple: Keep dual-stream Type-C possible_crtcs fixed", every port's connector offers all of them. Compositors read possible_crtcs once and pair connectors with CRTCs themselves: aquamarine (Hyprland) walks the CRTCs in index order and gives each to the first connected connector, in connector order, that can use it. The fabric instead hands out pipelines in whatever order the Type-C and Thunderbolt events arrive. That is a regression from the commit above on a J416c with two direct DP-alt monitors at boot: when port 1's event was handled first, port 1 took dcpext0 and port 0 dcpext1, and Hyprland paired them the other way round. Each connector's modes were then checked against the other monitor's list, native modes failed the atomic check with -EINVAL, and only modes both monitors share could be set. fbdev, which pairs every connected display afresh on each hotplug, and a dock's DPIN0 stream attaching after a direct display cross the same way. Until a compositor owns the display, keep the routes equal to that pairing computed from scratch: pipelines in CRTC index order, each to the first stream that wants one and has it in its possible_crtcs, with the primary connectors in port order ahead of the DPIN1 ones. A hybrid pipeline whose HDMI output is live is left to it, unless a tunnel holds it. A direct DP-alt stream wants a pipeline while its sink asserts HPD, as only then can its connector read connected. Only direct routes move, each to exactly its planned pipeline. A Thunderbolt tunnel never moves once set up. A direct stream planned onto a pipeline a tunnel holds stays unrouted and reads disconnected, since putting it anywhere else would have the compositor cross it with another display; it is then left out and the pass re-run, so the streams after it are not planned one pipeline off. A move is a disconnect and reconnect: every route that moves is taken down before any comes up, so two never share a pipeline, HPD is replayed on the new pipeline, and the hotplugs make fbdev and userspace re-probe. The pass runs on every DP-alt and tunnel attach, detach and HPD change, when HDMI is plugged in while a Type-C route has its pipeline, once DRM is registered, and from postclose when a file that was DRM master is closed; drm_release() has taken that file off the file list by then. A compositor owns the display while any open file has been DRM master: it pairs its connectors when it starts and keeps that pairing across a VT switch, where it drops master but keeps the device open. Until the last such file is closed routing stays as it was, the lowest free pipeline on DP entry with preferred_route honoured, and nothing moves. The check walks the file list under filelist_mutex, taken inside the fabric lock; nothing takes the two the other way round. cd321x does not report an unchanged DP state again, so a port left without a pipeline used to stay dark for the session. When a route lets its pipeline go while a compositor owns the display, such a port now goes back to the pipeline it last had if that is free, as the compositor keeps a reconnected connector's CRTC, and otherwise takes the lowest free one, the CRTC it gives a newly connected connector. Hotplug and retrain work queued for a Type-C connector can run after its route is gone and connector->dcp is NULL; check for that instead of dereferencing it. possible_crtcs stay static, since compositors read them only once, and the DPIN1 connector stays fixed to dcpext1. Single-stream machines take none of the new paths. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i --- drivers/gpu/drm/apple/apple_drv.c | 15 + drivers/gpu/drm/apple/dcp.c | 525 ++++++++++++++++++++++++++++-- drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/iomfb.c | 19 +- 4 files changed, 522 insertions(+), 38 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 42f276c7ed2f5..d6a544dd1bac2 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -66,9 +66,21 @@ static int apple_drm_gem_dumb_create(struct drm_file *file_priv, return drm_gem_dma_dumb_create_internal(file_priv, drm, args); } +/* + * A compositor or boot splash is closing the device. drm_release() has + * already taken its file off the file list, so the Type-C fabric can tell + * whether any other one still has the device open. + */ +static void apple_drm_postclose(struct drm_device *drm, struct drm_file *file) +{ + if (file->was_master) + dcp_typec_reorder(); +} + static const struct drm_driver apple_drm_driver = { DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), DRM_FBDEV_DMA_DRIVER_OPS, + .postclose = apple_drm_postclose, .name = DRIVER_NAME, .desc = DRIVER_DESC, .major = 1, @@ -708,6 +720,9 @@ static int apple_drm_init(struct device *dev) if (ret) goto err_unbind; + /* the fabric keeps order only once registered: catch up on probe */ + dcp_typec_reorder(); + drm_client_setup_with_fourcc(&apple->drm, DRM_FORMAT_XRGB8888); return 0; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 88f558aa1d2b9..4c1a6f85eb4d6 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -83,6 +83,14 @@ struct apple_dcp_typec_port { u32 applied_conf; bool applied_valid; bool hpd; + /* Direct DP-alt: the port is in DP mode, routed or not, and its HPD. */ + bool dp_wanted; + bool dp_hpd; + /* dcp_typec_rebalance_locked()'s plan for the port's two streams */ + struct apple_dcp_typec_route *target; + struct apple_dcp_typec_route *secondary_target; + /* left out of the plan: its planned pipeline is held by a tunnel */ + bool plan_dark; }; static DEFINE_MUTEX(dcp_typec_fabric_lock); @@ -258,7 +266,7 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, dcp->dptx_phy = route->dptx_phy; dcp->connector_type = DRM_MODE_CONNECTOR_USB; if (connector) { - connector->dcp = to_platform_device(dcp->dev); + WRITE_ONCE(connector->dcp, to_platform_device(dcp->dev)); dcp->typec_connector = connector; dcp->connector = connector; @@ -356,7 +364,8 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) * display has come back up on it. */ WRITE_ONCE(connector->connected, false); - connector->dcp = NULL; + /* hotplug work queued before this checks for it */ + WRITE_ONCE(connector->dcp, NULL); /* Unrouted: the port could go to any of its pipelines again. */ if (connector->port_encoder && !dcp_typec_dual_stream()) @@ -408,8 +417,10 @@ static void dcp_typec_retrain_work(struct work_struct *work) container_of(to_delayed_work(work), struct apple_dcp, typec_fabric_retrain_wq); - if (READ_ONCE(dcp->active_typec_route) && dcp->typec_connector) - dcp_retrain_oob(dcp->typec_connector); + struct apple_connector *connector = READ_ONCE(dcp->typec_connector); + + if (READ_ONCE(dcp->active_typec_route) && connector) + dcp_retrain_oob(connector); } static void dcp_typec_retrain_active_routes(void) @@ -428,18 +439,406 @@ static void dcp_typec_retrain_active_routes(void) } } +static struct apple_dcp_typec_route * +dcp_typec_port_route(struct apple_dcp_typec_port *port, struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + + list_for_each_entry(route, &port->routes, port_link) + if (route->dcp == dcp) + return route; + + return NULL; +} + +/* + * The DRM device once it has bound, that is once every pipeline has its + * CRTC and every port its connectors; NULL before. + */ +static struct drm_device *dcp_typec_drm(void) +{ + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *route; + struct drm_device *drm = NULL; + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (!port->connector || !port->secondary_connector) + return NULL; + list_for_each_entry(route, &port->routes, port_link) { + if (!route->dcp->crtc) + return NULL; + drm = route->dcp->crtc->base.dev; + } + } + + return drm; +} + +/* + * Has a compositor (or boot splash) taken the display? It paired its + * connectors with CRTCs when it started and keeps that pairing, also + * across a VT switch, where it drops DRM master but keeps the device + * open; moving routes under it would hand connectors to pipelines driving + * other displays. So the routes are frozen while any open file has ever + * been master, and thaw when the last such file is closed. + * + * Taken under dcp_typec_fabric_lock, next to the routing it decides, so + * the order is the fabric lock, then filelist_mutex. Nothing nests them + * the other way: filelist_mutex only covers list edits and walks, and + * drm_release() drops it before drm_file_free() calls postclose, which is + * where a close takes the fabric lock. drm_open() makes a file master + * before adding it here, but its owner cannot have paired anything before + * open() returns, and it re-probes on the hotplugs the moves send. + */ +static bool dcp_typec_frozen(struct drm_device *drm) +{ + struct drm_file *file; + + guard(mutex)(&drm->filelist_mutex); + list_for_each_entry(file, &drm->filelist, lhead) { + /* set under master_mutex, and only ever from false to true */ + if (READ_ONCE(file->was_master)) + return true; + } + + return false; +} + +/* Are the routes kept in compositor pairing order right now? */ +static bool dcp_typec_keep_order(void) +{ + struct drm_device *drm; + + if (!dcp_typec_dual_stream()) + return false; + drm = dcp_typec_drm(); + + return drm && READ_ONCE(drm->registered) && !dcp_typec_frozen(drm); +} + +/* A Thunderbolt tunnel holds @dcp's pipeline: it never moves. */ +static bool dcp_typec_tunnel_held(struct apple_dcp *dcp) +{ + return dcp->active_typec_route && dcp->active_typec_route->tunnel; +} + +/* + * Give @dcp's pipeline to the first stream, in connector order, that + * wants one, has none yet and has it in its possible_crtcs. A tunnel + * stream wants one once it is set up, or as @arriving/@dpin. A direct + * DP-alt stream wants one once its sink asserts HPD: only then can its + * connector read connected, and a compositor pairs only those. + */ +static void dcp_typec_plan_pipeline(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct apple_dcp_typec_port *arriving, + unsigned int dpin) +{ + struct apple_dcp_typec_port *port; + bool secondary = false; + + /* primary connectors in port order, then the DPIN1 ones */ + do { + list_for_each_entry(port, &dcp_typec_ports, link) { + struct apple_dcp_typec_route **slot = secondary ? + &port->secondary_target : &port->target; + struct apple_connector *connector = secondary ? + port->secondary_connector : port->connector; + struct apple_dcp_typec_route *route = + dcp_typec_port_route(port, dcp); + bool wants; + + if (secondary) + wants = port->secondary_owner || + (port == arriving && dpin == 1); + else + wants = (port->owner && port->owner->tunnel) || + (port->dp_wanted && port->dp_hpd && + !port->plan_dark) || + (port == arriving && dpin == 0); + + if (!wants || *slot || !route || + !(connector->candidate_crtcs & drm_crtc_mask(crtc))) + continue; + + *slot = route; + return; + } + secondary = !secondary; + } while (secondary); +} + +/* + * The pairing a compositor starting now would make, wherever the streams + * sit at the moment: pipelines in CRTC index order, each to the first + * stream that wants it. A pipeline whose fixed output is live is left to + * that output, unless a tunnel holds it: the output cannot have it back + * then, and its connector reads disconnected. + * + * Tunnels are planned in connector order like any other stream, not + * seated on the pipelines they hold: the compositor pairs their connectors + * by order too, and seating them would plan the direct streams around + * pairings it never makes. A direct stream planned onto a pipeline a + * tunnel holds cannot have it and stays dark, so it is left out and the + * pass re-run, or every stream after it would be planned one pipeline + * off. Streams only ever drop out, so that settles within one pass per + * port. It also brings a tunnel's plan onto the pipeline it holds + * wherever direct streams ahead of it were in the way. A tunnel still + * planned elsewhere cannot be brought there by any direct stream dropping + * out, and the compositor pairs its connector wrongly whatever they do. + */ +static void dcp_typec_plan(struct drm_device *drm, + struct apple_dcp_typec_port *arriving, unsigned int dpin) +{ + struct apple_dcp_typec_port *port; + struct drm_crtc *crtc; + bool again; + + list_for_each_entry(port, &dcp_typec_ports, link) + port->plan_dark = false; + + do { + list_for_each_entry(port, &dcp_typec_ports, link) { + port->target = NULL; + port->secondary_target = NULL; + } + + /* CRTCs are listed in index order */ + drm_for_each_crtc(crtc, drm) { + struct apple_dcp *dcp = + platform_get_drvdata(to_apple_crtc(crtc)->dcp); + + if (dcp->nr_typec_routes && + (dcp_typec_tunnel_held(dcp) || + !dcp_typec_route_fixed_output_busy(&dcp->typec_routes[0]))) + dcp_typec_plan_pipeline(dcp, crtc, arriving, dpin); + } + + again = false; + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port == arriving || !port->target || + (port->owner && port->owner->tunnel) || + !dcp_typec_tunnel_held(port->target->dcp)) + continue; + port->plan_dark = true; + again = true; + } + } while (again); +} + +/* Replay HPD to the pipeline a direct DP-alt port has just been given. */ +static void dcp_typec_port_attach(struct apple_dcp_typec_port *port) +{ + struct apple_dcp *dcp = port->owner->dcp; + + port->hpd = port->dp_hpd; + if (!port->hpd) + return; + + WRITE_ONCE(dcp->typec_cable_connected, true); + if (dcp->typec_connector) + dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); +} + +static struct apple_dcp_typec_route * +dcp_typec_lowest_free(struct apple_dcp_typec_port *port) +{ + struct apple_dcp_typec_route *candidate, *best = NULL; + unsigned int best_score = UINT_MAX; + + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (!dcp_typec_route_available(candidate)) + continue; + /* + * Lowest free CRTC index first: on dual-stream machines that + * is what a compositor picks from the port's fixed + * possible_crtcs. + */ + score = dcp_typec_route_score(candidate); + if (score < best_score) { + best = candidate; + best_score = score; + } + } + + return best; +} + +/* + * Route the direct DP-alt ports left waiting for a pipeline. Their DP + * state is not reported again while it stays the same, so nothing else + * would. Each goes back to the pipeline it last had if that is free, as + * a compositor keeps a reconnected connector's CRTC, and otherwise takes + * the lowest free one, the CRTC a compositor gives a new connector. + */ +static void dcp_typec_route_waiting(void) +{ + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *route; + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port->owner || !port->dp_wanted || !port->dp_hpd) + continue; + + route = port->preferred_route; + if (!route || !dcp_typec_route_available(route)) + route = dcp_typec_lowest_free(port); + if (!route || dcp_typec_route_activate(route, route->xbar)) + continue; + port->owner = route; + port->dp_release_deadline = 0; + dcp_typec_port_attach(port); + } +} + +/* + * Keep the Type-C routes where a compositor starting now expects them. + * Dual-stream machines keep possible_crtcs fixed, and compositors read + * them once and pair connectors with CRTCs themselves: aquamarine + * (Hyprland) walks the CRTCs in index order and gives each to the first + * connected connector, in connector order, that can use it. A connector + * paired with a pipeline other than the one routed to its display has its + * modes checked against the other display's list, so its modesets fail. + * The Type-C and Thunderbolt events that route the ports come in no + * particular order, so until a compositor owns the display (see + * dcp_typec_frozen()) every route change re-runs that pairing from + * scratch (dcp_typec_plan()) and follows it. + * + * Only direct DP-alt routes move: each goes to exactly its planned + * pipeline. A Thunderbolt tunnel never moves once set up, so where one + * sits on a pipeline planned for a direct stream, or the plan has nothing + * for it, that stream stays unrouted and its connector disconnected. + * Placing it on some other pipeline would have the compositor cross both + * displays; dark is the better failure. @arriving/@dpin is a tunnel + * stream asking for a pipeline, planned like any other; its route is + * returned for apple_dcp_tb_dp_tunnel() to set up, or NULL if the plan + * has none for it or another tunnel holds that one. + * + * A move is an unplug and replug. Every route that moves is taken down + * first, so that two never share a pipeline midway, then each goes up on + * its new pipeline with its HPD replayed; the hotplugs this sends make + * fbdev and userspace re-probe. + */ +static struct apple_dcp_typec_route * +dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, + unsigned int dpin) +{ + struct drm_device *drm = dcp_typec_drm(); + struct apple_dcp_typec_route *planned = NULL; + struct apple_dcp_typec_port *port; + + lockdep_assert_held(&dcp_typec_fabric_lock); + + dcp_typec_plan(drm, arriving, dpin); + if (arriving) { + planned = dpin ? arriving->secondary_target : arriving->target; + /* it will be refused: plan as if it had not asked */ + if (planned && dcp_typec_tunnel_held(planned->dcp)) { + planned = NULL; + dcp_typec_plan(drm, NULL, 0); + } + } + + list_for_each_entry(port, &dcp_typec_ports, link) { + struct apple_dcp_typec_route *owner = port->owner; + struct apple_dcp *dcp; + + if (!owner || owner->tunnel || owner == port->target) + continue; + + dcp = owner->dcp; + dev_info(dcp->dev, "re-routing %pOF from %s to %s for Type-C connector order\n", + port->connector_np, dev_name(dcp->dev), + port->target ? dev_name(port->target->dcp->dev) : "none"); + if (port->hpd || dcp->typec_cable_connected || + (dcp->typec_connector && dcp->typec_connector->connected)) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + if (dcp_typec_route_deactivate(owner) && owner->selected) { + /* still routed: leave the display where it is */ + dcp_typec_port_attach(port); + continue; + } + port->owner = NULL; + + /* as after a DP exit, hand the hybrid back to a live HDMI */ + if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_dptx_connect(dcp, 0); + } + + list_for_each_entry(port, &dcp_typec_ports, link) { + struct apple_dcp_typec_route *route = port->target; + + if (port->owner || !port->dp_wanted || !port->dp_hpd || + port == arriving) + continue; + + if (!route || !dcp_typec_route_available(route) || + dcp_typec_route_activate(route, route->xbar)) { + /* + * Dark for now. Its unchanged DP state is not reported + * again, so it is placed when the plan next runs, or + * once a compositor owns the display, when a pipeline + * is freed (dcp_typec_route_waiting()). + */ + port->applied_valid = false; + continue; + } + port->owner = route; + port->preferred_route = route; + port->dp_release_deadline = 0; + dcp_typec_port_attach(port); + } + + return planned; +} + +/* + * A route has let its pipeline go. Until a compositor owns the display + * the plan places everything anew; after that the pipeline goes to a port + * left waiting for one. + */ +static void dcp_typec_pipeline_freed(void) +{ + if (!dcp_typec_dual_stream()) + return; + + if (dcp_typec_keep_order()) + dcp_typec_rebalance_locked(NULL, 0); + else + dcp_typec_route_waiting(); +} + +/* + * Re-run the pairing pass from outside the fabric: once DRM is registered, + * as ports routed before that could not follow it, and when the last + * compositor or boot splash has closed the device, as the next one pairs + * the connectors from scratch. + */ +void dcp_typec_reorder(void) +{ + if (!dcp_typec_dual_stream()) + return; + + guard(mutex)(&dcp_typec_fabric_lock); + + if (dcp_typec_keep_order()) + dcp_typec_rebalance_locked(NULL, 0); +} + static int dcp_typec_route_set(struct typec_mux_dev *mux, struct typec_mux_state *state) { struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); struct apple_dcp_typec_port *port = route->port; - struct apple_dcp_typec_route *candidate, *best = NULL; + struct apple_dcp_typec_route *best = NULL; bool is_dp = dcp_typec_route_is_dp(state); struct typec_displayport_data *dp_data = is_dp ? state->data : NULL; u32 dp_status = dp_data ? dp_data->status : 0; u32 dp_conf = dp_data ? dp_data->conf : 0; - unsigned int best_score = UINT_MAX; - bool hpd; + bool hpd, was_counted; int ret = 0; guard(mutex)(&dcp_typec_fabric_lock); @@ -465,7 +864,13 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, /* Failed route acquisition must remain retryable on the next update. */ port->applied_valid = false; + /* did the pairing pass count this port's direct stream so far? */ + was_counted = port->dp_wanted && port->dp_hpd; + if (!is_dp) { + port->dp_wanted = false; + port->dp_hpd = false; + /* a Thunderbolt/USB4 DP tunnel is torn down by its own path */ if (port->owner && port->owner->tunnel) { port->applied_valid = true; @@ -488,6 +893,10 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (dcp->hdmi_hpd && dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd)) dcp_dptx_connect(dcp, 0); + dcp_typec_pipeline_freed(); + } else if (was_counted && dcp_typec_keep_order()) { + /* a stream the plan left dark is gone: plan the rest */ + dcp_typec_rebalance_locked(NULL, 0); } /* @@ -516,29 +925,43 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, return 0; } + hpd = dp_data && (dp_data->status & DP_STATUS_HPD_STATE); + port->dp_wanted = true; + port->dp_hpd = hpd; + + /* + * Until a compositor owns the display, a direct DP-alt stream has a + * pipeline only while its sink asserts HPD, and the pairing pass + * places it: HPD coming or going is the stream connecting or + * disconnecting as far as a compositor can tell. Otherwise the port + * takes the lowest free pipeline on DP entry, as a compositor does. + */ + if (dcp_typec_keep_order()) { + struct apple_dcp_typec_route *owner = port->owner; + + /* it connects or disconnects, or is not routed as it should be */ + if (hpd != was_counted || !owner != !hpd) + dcp_typec_rebalance_locked(NULL, 0); + if (!port->owner) { + if (hpd) + return -EBUSY; + port->applied_valid = true; + return 0; + } + /* just attached, its HPD replayed: nothing left to apply */ + if (port->owner != owner) { + port->applied_valid = true; + return 0; + } + } + if (!port->owner) { if (port->preferred_route && dcp_typec_route_available(port->preferred_route)) best = port->preferred_route; - if (!best) { - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; - - if (!dcp_typec_route_available(candidate)) - continue; - /* - * Lowest free CRTC index first: on dual-stream - * machines that is what a compositor picks from - * the port's fixed possible_crtcs. - */ - score = dcp_typec_route_score(candidate); - if (score < best_score) { - best = candidate; - best_score = score; - } - } - } + if (!best) + best = dcp_typec_lowest_free(port); if (!best) return -EBUSY; @@ -551,7 +974,6 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, } - hpd = dp_data && (dp_data->status & DP_STATUS_HPD_STATE); if (!hpd && port->hpd) { dcp_dptx_disconnect_oob(to_platform_device(port->owner->dcp->dev), 0); } else if (hpd && !port->hpd) { @@ -751,11 +1173,12 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, void *ctx) { struct apple_dcp_typec_port *port = NULL, *pos; - struct apple_dcp_typec_route *candidate, *best = NULL; + struct apple_dcp_typec_route *candidate, *best = NULL, *planned = NULL; struct apple_dcp_typec_route **slot; unsigned int best_score = UINT_MAX; struct mux_control *ctl; struct apple_dcp *dcp; + bool ordered; int ret; if (!connector_np || dpin > 1) @@ -797,6 +1220,7 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, if (dcp->hdmi_hpd && dcp->active && gpiod_get_value_cansleep(dcp->hdmi_hpd)) dcp_dptx_connect(dcp, 0); + dcp_typec_pipeline_freed(); return 0; } @@ -810,9 +1234,20 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, if (port->owner && !port->owner->tunnel) return -EBUSY; + /* + * Until a compositor owns the display, the pairing pass decides where + * the stream goes, moving direct DP-alt routes out of its way, and + * whether it gets a pipeline at all. + */ + ordered = dcp_typec_keep_order(); + if (ordered) + planned = dcp_typec_rebalance_locked(port, dpin); + list_for_each_entry(candidate, &port->routes, port_link) { unsigned int score; + if (ordered && candidate != planned) + continue; if (!dcp_typec_route_available(candidate)) continue; if (!dcp_typec_route_fits(candidate, dpin ? @@ -834,13 +1269,17 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, best_score = score; } } - if (!best) - return -EBUSY; + if (!best) { + ret = -EBUSY; + goto err_reorder; + } /* The route's crossbar control is dpphy (0); dpin0/dpin1 are 1/2. */ if (best->xbar != &best->xbar->chip->mux[0] || - best->xbar->chip->controllers < 3) - return -EOPNOTSUPP; + best->xbar->chip->controllers < 3) { + ret = -EOPNOTSUPP; + goto err_reorder; + } ctl = &best->xbar->chip->mux[1 + dpin]; dcp = best->dcp; @@ -855,9 +1294,11 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, dcp->tb_dpin_set_active = NULL; dcp->tb_dpin_ctx = NULL; } - return ret; + goto err_reorder; } *slot = best; + /* the port is in USB4 mode, not DP-alt */ + port->dp_wanted = false; dcp_tunnel_prepare(best, ctl); dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); @@ -876,6 +1317,12 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); return 0; + +err_reorder: + /* the pass kept a pipeline for this stream: give it to the others */ + if (planned) + dcp_typec_rebalance_locked(NULL, 0); + return ret; } EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); @@ -1819,8 +2266,20 @@ static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) guard(mutex)(&dcp_typec_fabric_lock); - if (READ_ONCE(dcp->active_typec_route)) + if (READ_ONCE(dcp->active_typec_route)) { + /* + * Until a compositor owns the display, a live HDMI output + * takes its pipeline back from a direct DP-alt route: the + * compositor pairs the HDMI connector with it first. + */ + if (dcp_typec_keep_order() && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) { + msleep(500); + if (gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_typec_rebalance_locked(NULL, 0); + } return IRQ_HANDLED; + } connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); /* do nothing on disconnect and trust that dcp detects it itself. diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 9c58284915997..58051294067e7 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -47,6 +47,7 @@ bool dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev void dcp_typec_port_set_connector(unsigned int idx, bool secondary, struct apple_connector *connector); bool dcp_typec_dual_stream(void); +void dcp_typec_reorder(void); bool dcp_is_typec_only(struct platform_device *pdev); bool dcp_fw_compat_is_12_x(struct platform_device *pdev); unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5536bd15cf8cd..a56f9ee5c7d4e 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -265,10 +265,13 @@ void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action) void dcp_retrain_oob(struct apple_connector *connector) { - struct apple_dcp *dcp = platform_get_drvdata(connector->dcp); + struct platform_device *pdev = READ_ONCE(connector->dcp); + struct apple_dcp *dcp; - if (!READ_ONCE(connector->connected)) + /* a Type-C port with no pipeline behind it has nothing to retrain */ + if (!pdev || !READ_ONCE(connector->connected)) return; + dcp = platform_get_drvdata(pdev); /* * Bringing up another high-speed Type-C route can disturb an active DPTX @@ -283,12 +286,18 @@ void dcp_retrain_oob(struct apple_connector *connector) void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; + struct platform_device *pdev; struct apple_dcp *dcp; int ret; connector = container_of(work, struct apple_connector, hotplug_wq); - dcp = platform_get_drvdata(connector->dcp); + pdev = READ_ONCE(connector->dcp); + if (!pdev) { /* a Type-C port unrouted after this was queued */ + drm_kms_helper_connector_hotplug_event(&connector->base); + return; + } + dcp = platform_get_drvdata(pdev); dev_info(dcp->dev, "%s() connected:%d valid_mode:%d nr_modes:%u\n", __func__, connector->connected, READ_ONCE(dcp->mode_state.valid), dcp->nr_modes); @@ -411,7 +420,7 @@ static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) int dcp_get_modes(struct drm_connector *connector) { struct apple_connector *apple_connector = to_apple_connector(connector); - struct platform_device *pdev = apple_connector->dcp; + struct platform_device *pdev = READ_ONCE(apple_connector->dcp); struct apple_dcp *dcp; struct drm_device *dev = connector->dev; @@ -498,7 +507,7 @@ enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, const struct drm_display_mode *mode) { struct apple_connector *apple_connector = to_apple_connector(connector); - struct platform_device *pdev = apple_connector->dcp; + struct platform_device *pdev = READ_ONCE(apple_connector->dcp); struct apple_dcp *dcp; if (!pdev) From 613601060e1eda63424dfefc080ad0e6b52cc00b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 2 Oct 2026 15:20:57 -0400 Subject: [PATCH 0340/1579] arm64: add experimental mixed-granule userspace ABI Allow opted-in 4 KiB userspace alongside native 16 KiB or 64 KiB processes, and opted-in 16 KiB userspace on 64 KiB kernels. Keep native AArch64 pages as the default and the kernel's physical-page geometry unchanged. Select the user granule at exec using private one-shot/inherited prctl controls; AArch32 selects 4 KiB where available and retains native restrictions when that granule is unavailable. Add per-mm translation geometry, typed subpage allocation and pin ownership, and granule-aware exec, faults, fork/COW, shared memory, permissions, signals, proc interfaces, reclaim, swap, migration and userfaultfd paths. Adapt byte positions and ownership in I/O, GEM/PRIME, IOMMUFD, vhost, BPF and tracing consumers; retain explicit rejection for unrepresentable native-page arrays. Include the userpages launcher, KUnit/selftest fixtures and implementation notes with the remaining unsupported cases. Validation: native16/native64 KASAN/lockdep VM Image builds and main KUnit suites pass (136/126 passes, 16/26 skips, zero failures). Bounded real EL0 fixtures cover mixed-granule sharing, pressure/swap, AArch32 capability fallback, IOMMUFD failure/lifetime/concurrency contracts and VGEM/PRIME (24/36 checks). Feature-off arm64/x86 builds pass. Also exercised no-THP configurations and basic FEX/Wine process contracts. This is an experimental foundation, not a qualified complete ABI or Mac gaming release. Apple GPU commands, physical-device DMA, Windows games and throughput remain untested. Legacy VFIO alternative fragment mappings, alternative THP/KSM and public larger-than-native exec remain incomplete. --- Documentation/arch/arm64/index.rst | 1 + Documentation/arch/arm64/user4k.rst | 4302 +++++ arch/arm64/Kconfig | 1 + arch/arm64/Kconfig.debug | 78 + arch/arm64/include/asm/cpucaps.h | 5 + arch/arm64/include/asm/elf.h | 9 + arch/arm64/include/asm/kernel-pgtable.h | 2 +- arch/arm64/include/asm/mmu.h | 40 + arch/arm64/include/asm/mmu_context.h | 63 +- arch/arm64/include/asm/mte.h | 8 + arch/arm64/include/asm/page.h | 33 +- arch/arm64/include/asm/pgalloc.h | 35 +- arch/arm64/include/asm/pgtable-hwdef.h | 12 +- arch/arm64/include/asm/pgtable.h | 51 +- arch/arm64/include/asm/processor.h | 5 + arch/arm64/include/asm/tlb.h | 25 +- arch/arm64/include/asm/tlbflush.h | 124 +- arch/arm64/include/asm/user-pgalloc.h | 59 + arch/arm64/include/asm/user-pgtable.h | 164 + arch/arm64/include/asm/user-pte.h | 116 + arch/arm64/kernel/cpufeature.c | 83 +- arch/arm64/kernel/pi/idreg-override.c | 32 + arch/arm64/kernel/pi/map_kernel.c | 2 +- arch/arm64/kernel/process.c | 72 +- arch/arm64/kernel/sys.c | 5 + arch/arm64/kernel/sys32.c | 6 + arch/arm64/lib/mte.S | 46 + arch/arm64/mm/Makefile | 3 + arch/arm64/mm/context.c | 43 + arch/arm64/mm/contpte.c | 143 +- arch/arm64/mm/copypage.c | 52 + arch/arm64/mm/fault.c | 30 +- arch/arm64/mm/fixmap.c | 2 +- arch/arm64/mm/init.c | 5 +- arch/arm64/mm/mmu.c | 4 +- arch/arm64/mm/pgd.c | 41 +- arch/arm64/mm/proc.S | 16 +- arch/arm64/mm/user4k-data-test.c | 1835 ++ arch/arm64/mm/user4k-pgalloc.c | 176 + arch/arm64/mm/user4k-test.c | 14057 ++++++++++++++++ arch/arm64/mm/user4k-tlb-test.c | 184 + arch/arm64/tools/cpucaps | 2 + arch/x86/kernel/uprobes.c | 6 +- block/bio.c | 150 + block/blk-map.c | 2 +- drivers/dma-buf/dma-buf.c | 35 +- drivers/gpu/drm/drm_gem.c | 7 +- drivers/gpu/drm/drm_gem_shmem_helper.c | 32 +- drivers/iommu/iommufd/Makefile | 2 + drivers/iommu/iommufd/device.c | 102 +- drivers/iommu/iommufd/fragments.c | 421 + drivers/iommu/iommufd/io_pagetable.c | 15 +- drivers/iommu/iommufd/io_pagetable.h | 77 +- drivers/iommu/iommufd/iommufd_private.h | 4 + drivers/iommu/iommufd/iova_bitmap.c | 131 +- drivers/iommu/iommufd/pages.c | 99 +- drivers/iommu/iommufd/selftest.c | 61 +- drivers/iommu/iommufd/viommu.c | 56 +- drivers/vfio/Kconfig | 10 + drivers/vfio/vfio_iommu_type1.c | 48 +- drivers/vfio/vfio_iommu_type1_account_test.c | 192 + drivers/vhost/vhost.c | 21 + fs/binfmt_elf.c | 42 +- fs/coredump.c | 81 +- fs/exec.c | 255 +- fs/proc/array.c | 3 +- fs/proc/task_mmu.c | 295 +- fs/splice.c | 172 + fs/userfaultfd.c | 57 +- include/asm-generic/pgalloc.h | 17 +- include/asm-generic/tlb.h | 34 +- include/linux/binfmts.h | 6 + include/linux/bio.h | 8 + include/linux/blk_types.h | 4 + include/linux/bpf.h | 22 + include/linux/coredump.h | 13 +- include/linux/highmem.h | 15 + include/linux/huge_mm.h | 4 + include/linux/khugepaged.h | 8 + include/linux/mm.h | 104 +- include/linux/mm_granule.h | 245 + include/linux/mm_subpage.h | 275 + include/linux/mm_types.h | 31 +- include/linux/mman.h | 11 + include/linux/mmu_context.h | 5 + include/linux/page_table_check.h | 4 + include/linux/pagemap.h | 5 +- include/linux/pgtable.h | 98 +- include/linux/rmap.h | 106 +- include/linux/sched/mm.h | 6 + include/linux/sched/signal.h | 2 +- include/linux/shmem_fs.h | 51 + include/linux/skmsg.h | 40 +- include/linux/swap.h | 26 +- include/linux/syscalls.h | 5 + include/linux/uprobes.h | 1 + include/net/tls.h | 7 + include/trace/events/kmem.h | 2 +- include/trace/events/mmap.h | 2 +- include/trace/events/oom.h | 9 +- include/uapi/linux/prctl.h | 18 + io_uring/memmap.c | 8 +- io_uring/rsrc.c | 124 + kernel/bpf/bpf_local_storage.c | 23 +- kernel/bpf/syscall.c | 128 +- kernel/events/uprobes.c | 212 +- kernel/fork.c | 31 +- kernel/futex/core.c | 57 +- kernel/sched/fair.c | 2 +- kernel/sys.c | 40 +- kernel/trace/trace_events_user.c | 36 +- kernel/tsacct.c | 16 +- lib/vdso/datastore.c | 3 +- mm/Kconfig | 23 +- mm/Makefile | 1 + mm/damon/ops-common.c | 10 +- mm/filemap.c | 73 +- mm/gup.c | 578 +- mm/huge_memory.c | 8 +- mm/internal.h | 83 +- mm/khugepaged.c | 119 +- mm/ksm.c | 9 +- mm/madvise.c | 156 +- mm/memory-failure.c | 45 + mm/memory.c | 2041 ++- mm/migrate.c | 85 +- mm/migrate_device.c | 2 +- mm/mincore.c | 43 +- mm/mlock.c | 42 +- mm/mm_init.c | 3 +- mm/mmap.c | 139 +- mm/mmu_gather.c | 60 +- mm/mprotect.c | 97 +- mm/mremap.c | 89 +- mm/mseal.c | 4 +- mm/msync.c | 6 +- mm/oom_kill.c | 25 +- mm/page_ext.c | 4 + mm/page_idle.c | 7 +- mm/page_io.c | 61 +- mm/page_table_check.c | 142 +- mm/page_vma_mapped.c | 61 +- mm/pagewalk.c | 49 +- mm/pgtable-generic.c | 35 +- mm/process_vm_access.c | 96 +- mm/rmap.c | 514 +- mm/shmem.c | 510 +- mm/swap.h | 22 +- mm/swap_state.c | 6 +- mm/swapfile.c | 382 +- mm/truncate.c | 2 + mm/user-subpage-internal.h | 69 + mm/user-subpage.c | 1667 ++ mm/userfaultfd.c | 518 +- mm/util.c | 106 +- mm/vma.c | 180 +- mm/vma.h | 40 +- mm/vma_exec.c | 6 +- mm/vma_init.c | 5 +- mm/vmscan.c | 124 +- net/core/skbuff.c | 119 +- net/core/skmsg.c | 88 +- net/ipv4/tcp_bpf.c | 5 + net/tls/tls_device.c | 22 + net/tls/tls_sw.c | 238 +- tools/aurora-userpages/.gitignore | 1 + tools/aurora-userpages/Makefile | 14 + tools/aurora-userpages/README.rst | 34 + tools/aurora-userpages/userpages.c | 83 + .../testing/kunit/configs/arm64_user4k.config | 33 + .../configs/arm64_user4k_data_kasan.config | 18 + .../configs/arm64_user4k_mm_kasan.config | 38 + tools/testing/selftests/arm64/Makefile | 2 +- .../testing/selftests/arm64/user4k/.gitignore | 3 + tools/testing/selftests/arm64/user4k/Makefile | 104 + tools/testing/selftests/arm64/user4k/advice.c | 181 + .../arm64/user4k/aslr-fixture-init.c | 159 + .../selftests/arm64/user4k/bpf-uptr-init.c | 583 + .../arm64/user4k/coarse-shmem-init.c | 213 + .../arm64/user4k/compat-aslr-probe.c | 100 + .../arm64/user4k/compat-fixture-init.c | 60 + .../arm64/user4k/compat-mlock-init.c | 55 + .../arm64/user4k/compat-mlock-probe.c | 105 + .../arm64/user4k/compat-policy-probe.c | 47 + .../selftests/arm64/user4k/compat-probe.c | 76 + .../selftests/arm64/user4k/compat-script | 1 + .../selftests/arm64/user4k/compat-uffd-init.c | 118 + .../arm64/user4k/compat-uffd-probe.c | 95 + .../arm64/user4k/context-stress-init.c | 264 + .../arm64/user4k/core-fixture-init.c | 373 + .../selftests/arm64/user4k/cpu-granule-init.c | 150 + .../arm64/user4k/cpu-granule16-init.c | 153 + .../selftests/arm64/user4k/dio-fixture-init.c | 182 + .../arm64/user4k/exec-fixture-init.c | 161 + .../selftests/arm64/user4k/exec-policy-init.c | 188 + .../arm64/user4k/fex-contract-init.c | 20 + .../selftests/arm64/user4k/fex-native-abi.c | 14 + .../selftests/arm64/user4k/fex-x86-contract.c | 53 + .../selftests/arm64/user4k/gem-fixture-init.c | 182 + .../arm64/user4k/granule-proc-init.c | 47 + .../arm64/user4k/granule-swap-init.c | 137 + .../arm64/user4k/granule-threeway-init.c | 128 + .../selftests/arm64/user4k/iommufd-init.c | 1142 ++ .../arm64/user4k/lazyfree-fixture-init.c | 246 + .../arm64/user4k/memcg-accounting-init.c | 113 + .../arm64/user4k/memcg-pressure-init.c | 321 + .../arm64/user4k/memory-interfaces.c | 148 + .../arm64/user4k/mglru-granule-init.c | 259 + .../selftests/arm64/user4k/mixed-process.c | 194 + .../arm64/user4k/mlock-fixture-init.c | 225 + .../selftests/arm64/user4k/page-contract.c | 210 + .../selftests/arm64/user4k/proc-memory.c | 314 + .../selftests/arm64/user4k/proc-pages.c | 277 + .../selftests/arm64/user4k/reclaim-advice.c | 339 + .../selftests/arm64/user4k/runtime-app-init.c | 33 + .../selftests/arm64/user4k/runtime-app.py | 91 + .../arm64/user4k/shmem-collapse-init.c | 213 + .../selftests/arm64/user4k/shmem-hole-init.c | 234 + .../selftests/arm64/user4k/shmem-pin-init.c | 214 + .../arm64/user4k/shmem-pressure-init.c | 125 + .../selftests/arm64/user4k/shmem-race-init.c | 182 + .../selftests/arm64/user4k/shmem-thp-init.c | 160 + .../arm64/user4k/shmem-uffd-swap-init.c | 127 + .../arm64/user4k/splice-fixture-init.c | 502 + .../arm64/user4k/swap-fixture-init.c | 204 + .../selftests/arm64/user4k/swapfile-fixture.h | 47 + .../arm64/user4k/thread-fixture-init.c | 391 + .../selftests/arm64/user4k/uffd-move-init.c | 224 + .../selftests/arm64/user4k/uffd-wp-init.c | 664 + .../selftests/arm64/user4k/uprobes-init.c | 318 + .../arm64/user4k/uring-fixture-init.c | 230 + .../selftests/arm64/user4k/user-events-init.c | 268 + .../arm64/user4k/user16k-smoke-init.c | 147 + .../selftests/arm64/user4k/vhost-log-init.c | 392 + .../selftests/arm64/user4k/wine-kernel32.def | 20 + .../arm64/user4k/wine-page-contract.c | 101 + .../selftests/arm64/user4k/x18-probe.S | 25 + tools/testing/vma/include/dup.h | 117 + tools/testing/vma/tests/merge.c | 2 +- tools/testing/vma/tests/mmap.c | 8 +- tools/testing/vma/tests/vma.c | 4 +- 241 files changed, 46500 insertions(+), 1537 deletions(-) create mode 100644 Documentation/arch/arm64/user4k.rst create mode 100644 arch/arm64/include/asm/user-pgalloc.h create mode 100644 arch/arm64/include/asm/user-pgtable.h create mode 100644 arch/arm64/include/asm/user-pte.h create mode 100644 arch/arm64/mm/user4k-data-test.c create mode 100644 arch/arm64/mm/user4k-pgalloc.c create mode 100644 arch/arm64/mm/user4k-test.c create mode 100644 arch/arm64/mm/user4k-tlb-test.c create mode 100644 drivers/iommu/iommufd/fragments.c create mode 100644 drivers/vfio/vfio_iommu_type1_account_test.c create mode 100644 include/linux/mm_granule.h create mode 100644 include/linux/mm_subpage.h create mode 100644 mm/user-subpage-internal.h create mode 100644 mm/user-subpage.c create mode 100644 tools/aurora-userpages/.gitignore create mode 100644 tools/aurora-userpages/Makefile create mode 100644 tools/aurora-userpages/README.rst create mode 100644 tools/aurora-userpages/userpages.c create mode 100644 tools/testing/kunit/configs/arm64_user4k.config create mode 100644 tools/testing/kunit/configs/arm64_user4k_data_kasan.config create mode 100644 tools/testing/kunit/configs/arm64_user4k_mm_kasan.config create mode 100644 tools/testing/selftests/arm64/user4k/.gitignore create mode 100644 tools/testing/selftests/arm64/user4k/Makefile create mode 100644 tools/testing/selftests/arm64/user4k/advice.c create mode 100644 tools/testing/selftests/arm64/user4k/aslr-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/bpf-uptr-init.c create mode 100644 tools/testing/selftests/arm64/user4k/coarse-shmem-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-aslr-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-mlock-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-mlock-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-policy-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-script create mode 100644 tools/testing/selftests/arm64/user4k/compat-uffd-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-uffd-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/context-stress-init.c create mode 100644 tools/testing/selftests/arm64/user4k/core-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/cpu-granule-init.c create mode 100644 tools/testing/selftests/arm64/user4k/cpu-granule16-init.c create mode 100644 tools/testing/selftests/arm64/user4k/dio-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/exec-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/exec-policy-init.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-contract-init.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-native-abi.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-x86-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/gem-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-proc-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-swap-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-threeway-init.c create mode 100644 tools/testing/selftests/arm64/user4k/iommufd-init.c create mode 100644 tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memcg-accounting-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memcg-pressure-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memory-interfaces.c create mode 100644 tools/testing/selftests/arm64/user4k/mglru-granule-init.c create mode 100644 tools/testing/selftests/arm64/user4k/mixed-process.c create mode 100644 tools/testing/selftests/arm64/user4k/mlock-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/page-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/proc-memory.c create mode 100644 tools/testing/selftests/arm64/user4k/proc-pages.c create mode 100644 tools/testing/selftests/arm64/user4k/reclaim-advice.c create mode 100644 tools/testing/selftests/arm64/user4k/runtime-app-init.c create mode 100755 tools/testing/selftests/arm64/user4k/runtime-app.py create mode 100644 tools/testing/selftests/arm64/user4k/shmem-collapse-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-hole-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-pin-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-pressure-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-race-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-thp-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c create mode 100644 tools/testing/selftests/arm64/user4k/splice-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/swap-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/swapfile-fixture.h create mode 100644 tools/testing/selftests/arm64/user4k/thread-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uffd-move-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uffd-wp-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uprobes-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uring-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/user-events-init.c create mode 100644 tools/testing/selftests/arm64/user4k/user16k-smoke-init.c create mode 100644 tools/testing/selftests/arm64/user4k/vhost-log-init.c create mode 100644 tools/testing/selftests/arm64/user4k/wine-kernel32.def create mode 100644 tools/testing/selftests/arm64/user4k/wine-page-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/x18-probe.S diff --git a/Documentation/arch/arm64/index.rst b/Documentation/arch/arm64/index.rst index 98052b4ef4a1e..e5d0d0c46bf3b 100644 --- a/Documentation/arch/arm64/index.rst +++ b/Documentation/arch/arm64/index.rst @@ -32,5 +32,6 @@ ARM64 Architecture sve tagged-address-abi tagged-pointers + user4k features diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst new file mode 100644 index 0000000000000..eaf342b4b3468 --- /dev/null +++ b/Documentation/arch/arm64/user4k.rst @@ -0,0 +1,4302 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +=========================================================== +Experimental mixed-granule userspace on arm64 kernels: design +=========================================================== + +Status and scope +================ + +This work starts from Aurora's M5 Pro branch, commit +4832ab22bb9080d82c29f0ca4ee686492f0552f8. The experimental branch is +``mm-mixed-userspace-abi`` in ``aurora-silicon/linux-aurora``. +It does not include the source worktree's uncommitted +display changes. Do not deploy it to the M5 or replace the host kernel. + +The experimental kernel now executes opted-in 4K EL0 processes alongside +ordinary native-granule processes on 16K and 64K kernels. Opt-in 16K EL0 +processes also run on 64K kernels. Static 4K-aligned ELF and dynamically linked PIE +executables pass the basic page-contract tests in a VM, including fork/COW, +independent permissions, shared mappings and signal return. One-shot exec +selection, large argument/environment copies and transitions back to the +native ABI are tested. Real mixed-size processes share memfd/futex state and +use process_vm_readv/writev in both directions. See the latest milestone below +for exact evidence and limits; this is not yet a complete or battle-tested ABI. + +Current userfaultfd support (2026-10-02) +--------------------------------------- + +Smaller-than-native processes support anonymous missing-page COPY/ZEROPAGE, +synchronous/asynchronous write protection (including unpopulated leaves), and +POISON. Private shmem mappings support missing-page COPY/ZEROPAGE without +populating or modifying the shared file. Shared and private shmem mappings +support minor faults, CONTINUE and write protection using each leaf's physical +byte offset. The pager may have a different granule from the target process. + +Shared shmem MISSING registration/COPY/ZEROPAGE is now advertised through +MISSING_SHMEM for alternative processes. Coverage includes partial hole-punch, +shrink/regrow, retained large folios, pinned I/O, concurrent refill, allocation +failures, and COPY into swapped partial cache pages under memcg pressure. +Hugetlb userfaultfd and larger-than-native processes remain unsupported. +Shared COPY uses the requested user granule rather than a native-size copy; +these bounded tests do not establish full shared-file ABI qualification. The combined +WP_HUGETLBFS_SHMEM feature bit remains unadvertised for alternative processes +because alternative hugetlb support is absent; shmem WP is available through +successful range registration and its returned WRITEPROTECT ioctl bit. + +COPY batches reserve only requested leaves within one native linear index and +transfer individual slot references to their PTEs. Source fault retries discard +unused reservations, copy into fresh unexposed backing outside VMA locks, then +revalidate VMA identity and byte position before installation. Existing slots +retain independent COW, mlock and rmap accounting. Packing was checked through +actual pagemap physical addresses; this is not a throughput benchmark. Separate +single-leaf COPY requests are not currently coalesced into a shared allocator +pool, so sparse demand paging can still consume a native folio per filled leaf. + +Exclusive anonymous UFFD MOVE is now implemented and advertised for user +granules smaller than the native kernel granule. It moves one leaf's ownership +without copying its bytes or changing its physical address. Source-slot pins and +fork sharing reject the operation; pins in a different slot do not prevent it. +Present, swapped, zero-page and hole paths use the process granule. See the MOVE +milestone below for the bounded runtime matrix and remaining qualification work. + +Generalization requirements (2026-09-30) +--------------------------------------- + +The implementation must cover both directions: smaller user granules within +native physical pages, and larger user granules backed by multiple native pages. +A 64K user leaf on a 16K kernel requires aligned contiguous backing and complete +ownership of all covered physical units. File mappings must retain byte identity +with smaller-granule aliases through writeback, EOF, truncation, invalidation, +COW and pinning. The 64K-native implementation now supports sixteen 4K +slots, including table ownership and swap offsets. Generalizing the current +4K/native geometry now includes 16K subranges on 64K kernels. Larger-than-native +user leaves remain unfinished. + +Target matrix: native kernel granules 4K/16K/64K, each with native-default +AArch64 and opt-in supported user granules 4K/16K/64K. Hardware capability and +address-size constraints apply independently; unsupported modes must not be +advertised. AArch32 is an execution-state capability, not a platform-name check. +AArch32 defaults to 4K where supported and otherwise retains existing behavior. +Asymmetric CPU eligibility, affinity changes and hotplug must preserve the +selected translation granule. AArch32 mmap2 offsets remain in 4K ABI units, +including large file offsets, regardless of kernel granule. + +Exec selection must use the final executable format, including interpreter and +script transitions, before committing the new mm. The current temporary stack +already has mappings before ELF class is known; its geometry cannot be changed +by flipping a flag. Late selection requires safely transferring staged arguments +into a newly allocated mm with the desired geometry. Compat setup must retain +the chosen granule alongside MMCF_AARCH32. Compat signals, vDSO/kuser mappings, +auxv, ASLR, core dumps and AArch32/AArch64 exec transitions require runtime tests. + +Ryan Roberts's 2024 RFC uses boot-selected kernel geometry and runtime folding; +its PAGE_SIZE_MIN/MAX and geometry work are relevant precedents. This project +additionally needs immutable per-mm translation geometry and explicit physical +ownership units. Replacing the kernel's PAGE_SIZE with a per-task value is not +a valid design. Reference: https://lwn.net/Articles/993990/ (author's cover letter; +original Message-ID 20241014105514.3206191-1-ryan.roberts@arm.com). + +The tested matrix now includes 16K and 64K kernels with native/4K AArch64 +processes, AArch32 automatic 4K selection, and capability-masked AArch32 +native-16K/native-64K fallback, plus 16K EL0 processes on a 64K kernel. User +granules larger than the native kernel granule remain unimplemented. Current capability +selection uses the sanitized system-wide granule intersection; exploiting a +larger capability set restricted to an asymmetric AArch32 CPU domain remains +unimplemented. + +The implementation includes per-mm table geometry, packed lower tables, +TG0/TTBR0 switching with hardware/software PAN, anonymous fragment ownership, +COW, reverse mapping, reclaim, swap and migration. Native physical pages, +page-cache folios and kernel mappings retain the build-time native granule. Internal fixtures cover +substantially more cases than the current EL0 suite. Fixture success alone +does not establish broad application or concurrent-pressure correctness. + +The option defaults off. Legacy page-array GUP consumers, several memory +syscalls, device/GPU mappings and broad stress coverage remain unfinished. +Core dumping and the tested proc accounting interfaces now use the target +process granule; their exact coverage is recorded below. No performance, M5 hardware, production-readiness +or complete NCE compatibility claim is made. Milestone sections below retain +their historical status statements; the latest exec milestone supersedes +earlier statements that EL0 activation was absent. + +The expanded objective is concurrent 4K, 16K and 64K userspace granules on +arm64, with the kernel/physical allocator granule independently fixed for the +boot. AArch64 defaults to the native granule (16K on the current Apple target); +alternatives remain opt-in. AArch32 selects 4K when the eligible CPU domain +supports both AArch32 EL0 and 4K stage-1 translation, retaining the established +native-granule behavior otherwise. The asymmetric-domain optimization and +remaining granule combinations are still requirements; homogeneous AArch32 +automatic-4K and native-fallback paths have runtime coverage on both builds. + +Units and invariants +==================== + +There are three distinct units: + +* The kernel's base physical allocation, ``struct page`` and PFN unit: + the build-time native granule (16K or 64K in current mixed-granule builds). +* A process's translation and user mapping unit: 4K, 16K or 64K, subject to + the supported combinations above and immutable for the lifetime of its mm. +* The folio's allocation, ownership and reclamation unit: one or more + physical base pages. This does not change merely because a user mapping + has a smaller translation granule. + +Do not redefine PAGE_SIZE dynamically or globally change PAGE_SHIFT to 12. +Do not use native ``pte_pfn()``/``pfn_pte()`` to encode/decode a 4K slot: +the low two subpage bits must survive separately from the 16K PFN. A future +mapping reference must identify the owning folio and an explicit byte offset. +Do not silently widen an unmap, permission downgrade or COW operation to +cover neighboring mappings. + +Proposed allocation policy: pack compatible private 4K allocations into +16K owners rather than waste a complete 16K allocation for each 4K mapping. +Track slot occupancy and references; a folio is not freeable while any slot +has a mapping, pin or in-flight reference. Swap entries name storage and do +not keep physical backing alive after swap-cache eviction. Sharing individual +slots across unrelated owners is not required for the first allocator. +Zero newly exposed bytes and never expose an unallocated neighbor. Reclaim +and migration may operate on the whole folio, but must preserve individual +slot contents, mappings, permissions and COW relationships. + +The test packs its four 4K translation tables into one native 16K page, +exercising address bits 13:12 in table descriptors as well as leaf PTEs. +Its native 16K tables each occupy a separate allocation. This is test +scaffolding, not a production allocator. Its full local TLB flushes establish +a controlled test context; they are not a proposed context-switch fast path. + +The new core-MM prototype allocates a 4K-aligned root from a dedicated slab +cache. Lower-level tables now pack four logical 4K fragments into native +16K ptdesc owners. Each mm has separate PTE, PMD and PUD caches under its +page_table_lock, so parent and child tables do not share a fragment-owner +lock. Unissued slots hold references until mm destruction; issued slots +are never recycled within a live owner. A racing new allocation may keep +one private owner if another allocator populated that level's cache first. +Improving that race's packing efficiency remains possible future work. + +The sparse 4K MM test uses sixteen logical tables: 64 KiB of installed +table accounting and five native owners (80 KiB of backing including cached +slots), down from the earlier prototype's 256 KiB backing. These counters +have different meanings: actual owner allocations are charged once through +GFP_PGTABLE_USER; mm page-table bytes count installed logical tables. Root +slab accounting remains separate. Native table allocations retain their +original sizes and the native test still observes fourteen native tables. + +The pgtable_t token preserves a fragment's address bits 13:12, while locks +and lifetime management use the native owner. TLB batches carry tagged +fragment addresses through their existing TLBI/RCU ordering. Deferred PTE +reclaim marks the owner for RCU release, dropping each fragment's reference +without queueing multiple callbacks on the same owner. Tests cover final +release, deferred release, cache-reference teardown and native THP table +deposit/withdraw. Native PT_RECLAIM, THP and contiguous-PTE support remain +enabled in the test configuration. This is a page-table allocator, not an +allocator or ownership model for 4K anonymous data. + +Subpage data ownership +===================== + +``mm/user-subpage.c`` manages explicit slot references independently of +struct page's native PFN unit. The caller supplies a fresh, unexposed, +order-0 folio and transfers its reference only on success. Allocation and +memcg/NUMA policy remain the caller's responsibility; a pool must cover a +compatible policy/charge domain. The prototype does not yet bind pools to +VMAs or implement that policy selection in a fault path. + +Metadata is separate from folio->mapping, folio->private and folio->swap. +Each of the four slots has its own reference count and physical byte +offset. Every live slot reference also retains the native folio. The +supplied folio reference transfers to its first reservation; there is no +permanent extra allocator reference while slots are live. Allocation reserves a +free slot under the pool lock and zeros its bytes before returning it. +An extra reference keeps a slot unavailable after another holder releases +it. This applies to aliases and to future pin/deferred-unmap references; +a raw folio reference alone is not a substitute for a slot reference. + +A page-extension pointer associates each native physical page with its slot +owner without changing struct page or taking a folio-private field. Physical +lookup retains bits 13:12 and acquires a reference only to a live slot. +Reservation keeps the slot reference count at zero while initialising data; +a release publication and acquire lookup prevent exposure of partially +initialised contents. The occupancy bitmap protects an in-progress +reservation against concurrent pool closure. + +An owner is unbound before its folio reference is dropped, and its metadata +is freed after an RCU grace period. Lookup requires the caller to stabilise +the physical allocation (for example, a referenced folio or a live PTE under +its lock); RCU alone protects metadata, not arbitrary physical PFN lifetime. +Lookup obtains a speculative native folio reference before acquiring the +slot reference and drops it if the slot is no longer live. Speculative PTE +users still need to recheck +their PTE after acquiring a slot: physical lookup identifies the current +live slot, not the historical identity of an unlocked mapping. Supplying +already-owned backing is rejected before any data is zeroed. + +The extension is selected only by the experimental configuration. A 512 MiB +guest reports 262,144 bytes allocated for page extensions with this client +alone. This is metadata overhead, not a measurement of native-process +throughput or latency with the feature enabled. + +The last slot reference allows reuse only after the caller has satisfied +its PTE/TLB lifetime obligation. The allocator cannot infer TLB completion. +The copy helper reserves a different slot and copies exactly 4K, including +when source and destination occupy different slots of one native folio. +It does not make the source immutable or implement write-fault/COW decisions; +the caller must stabilise the source and mapping permissions. + +Closing a pool rejects new reservations. Existing references outlive both +closure and the originating caller. The pool retains no mm/VMA pointer. +Its lock is IRQ-safe so deferred releases can run from RCU callbacks, and +closure is constant-time: only a wholly unissued backing folio needs +immediate disposal. Other owners are released on their last live slot. +No global allocator lock or always-flush-TLB path is added. + +The common reverse-map PTE walker now uses the target mm's table geometry +and stride. PVMW_SUBPAGE requests one present PTE with an exact native-PFN +plus byte-offset match. It cannot match a neighbor solely because both +addresses share a native PFN. Native-owner walks keep their original fixed +file-index bound while visiting multiple 4K PTEs; their bound must not advance +with each returned mapping. Tests cover exact matching with both locking +paths, a later alias beyond the bound, a 2 MiB table crossing, holes, and +the native 16K path. Explicit leaf helpers preserve physical bits 13:12 +when constructing entries or extracting their address/protection fields. + +This walker conversion does not attach anonymous reverse-map ownership to +slots. PVMW_SUBPAGE now matches migration PFNs plus slot offsets (see below), but +does not match real swap entries, +and subpage PMD mappings remain unsupported. File VMA offsets still need +their own byte component before a 4K-offset file mapping can be created. + +Tests cover four slots sharing one folio, all physical offset bits, +zeroing on reuse, retained-reference reuse exclusion, copying without +neighbor modification, pool closure, deferred final release, rejection +of compound/shared-reference input folios, and concurrent allocation. +Physical lookup tests also cover duplicate-owner rejection and unbinding. +A reader races 2,048 alternating full-slot copies and owner retirement; +every acquired slot must contain one complete pattern. This test uses a +time bound and yields when the only destination is held by the reader. +An earlier fixed-count retry budget expired after one copy; that failure +is retained in kvm-rmap-races-hw-pan.log. The corrected test passes on both +single-vCPU and two-vCPU guests; it does not prove all possible races absent. +The user-copy fixture takes explicit slot references before creating PTEs +and drops them only after its private MM teardown and TLB completion. + +The data folios are not on an anonymous LRU and have no normal rmap yet. +Do not publish this allocator through writable faults until reverse +mapping, reclaim/swap, per-slot exclusivity, pinning, fault races and +mapped-byte/backing-byte accounting are integrated. The current tests +do not verify memcg enforcement or application memory-pressure behavior. + +Translation and context switching +================================= + +Arm specifies independent TG0/T0SZ and TG1/T1SZ fields. At 48 VA bits the +four-level 4K table shifts are 39/30/21/12, whereas the four-level 16K shifts +are 47/36/25/14. Changing TG0 while retaining native-format tables is wrong. +Generic page-table accessors and every walker need an explicit mm geometry; +the kernel tables must continue to use native constants. + +The production switch must keep TTBR0, TG0, T0SZ and the ASID interpretation +consistent through interrupts, preemption, uaccess, return to EL0 and +speculative walks. Review software PAN, KPTI, CnP, ASID rollover/pinning, +CPU hotplug, idle/suspend, EFI/idmap use, SVA/PASID and KVM host restore paths. +TCR.DS is shared by the two translation regimes: never clear it to obtain +a convenient 4K descriptor format while a DS-dependent TTBR1 is active. +Initial process support should reject incompatible CPU/configuration +combinations explicitly rather than partially enable them. + +cpu_do_switch_mm() now synchronises the reserved TTBR0 before a TG0 change +and synchronises the new TG0 before installing the selected root. The mm's +existing ASID is retained; no new per-switch TLBI is added. Software PAN +selects TG0 at check_and_switch_context() even though user access or EL0 +return restores TTBR0 later. The idmap and raw transition-table installers +restore native TG0, and idmap removal restores the active mm's TG0 even +when software PAN leaves TTBR0 reserved. A switch to init_mm selects the +native granule. T0SZ remains the existing fixed 48-bit userspace choice in +this experimental configuration; kernel TG1/T1SZ are preserved. + +This is tested context-switch plumbing, not qualification of every entry, +EFI, suspend, hotplug, KPTI, ASID rollover, CnP or KVM-host path. The test +executes no 4K ELF image. The per-mm granule remains immutable and no user +activation interface exists. + +All invalidation paths need the mm's granule, not just the context switch. +The non-range TLBI VA operand is in 4K units; range and TTL encodings need +their own granule-aware handling. Native 16K iteration would skip three +4K mappings out of four. Page-table freeing needs the same ordering and +RCU/TLB lifetime guarantees as the native path. Preserve native batching; +do not install full-TLB flushing as a production workaround. + +The page/range invalidation helpers and deferred-unmap batches now use the +target mm's granule for strides, range operands and TTL hints. Generic PTE +gather ranges and the minimum flush stride also use mm geometry; physical +page-free batches retain native allocation units. Kernel and stage-2 callers +still explicitly select native geometry. Existing range batching, DSB +completion, ASID selection and paired KPTI invalidation remain in place. + +Recording tests decode every emitted operand for 4K/16K/64K geometry, +range-capable and non-range algorithms, LPA2 alignment and different leaf +strides. They check contiguous coverage, hint/ASID encoding, a bounded +operation count and the top of the 48-bit address space. These tests execute +no TLBI instruction and do not claim hardware support for each combination. +The isolated 4K hardware test additionally removes leaves through the real +local-page, broadcast-range and deferred-batch helpers and verifies holes +and neighboring translations before restoring all mappings and registers. +It does not establish a remote CPU shootdown or production ASID rollover. + +ABI direction +============= + +Select the page granule when creating a new mm for exec. Do not convert +a live address space with existing VMAs, pins, threads or asynchronous I/O. +Fork inherits the granule. CLONE_VM shares the mm and therefore the granule. +A new executable selects its supported granule explicitly. The development +prctl described below is available only with the experimental config. Its +private numbers are not upstream assignments or a stable UAPI promise; +incomplete interfaces still prevent production use. + +AT_PAGESZ must describe the process granule. ELF PT_LOAD alignment, file +offsets, interpreter placement, initial stack, brk, mmap/munmap/mprotect, +mremap, madvise, mincore, mlock, guard pages and signal/vDSO mappings must +all agree. Native 16K executables must retain their current behavior. +An executable's p_align alone is insufficient evidence that it requests +or requires a particular kernel ABI. + +PAGE_SHIFT also describes accounting and file-cache units. Replacing every +occurrence with a process shift would corrupt those units. Keep file-cache +indices in native units and represent subpage file offsets explicitly; +audit VMA split/merge and reverse-map interval calculations accordingly. +Report mapped bytes and physically charged folio bytes separately where +their meanings differ. Charge the actual physical allocation exactly once. + +Memory sharing policy +===================== + +Mixed-granule sharing is not inherently prohibited. The target is a single +coherent backing object, with each mm exposing byte ranges in its own +granule. A 16K process can map a whole native page while a 4K process maps +individual slots of the same page; permissions are properties of each +mapping, not of the shared bytes globally. + +Implement and test memfd/shmem first, then broader file mappings. The +first sharing test should use a 16K-aligned extent and four 4K aliases. +Write through either process, verify visibility through the other, and +exercise cross-process shared futexes. Canonical futex keys must describe +the shared object and byte offset; virtual page-size differences cannot +change the key. Private futexes need their own ordinary threading tests. + +Partial EOF, truncate, hole punching, msync, dirty/writeback accounting, +MAP_PRIVATE COW and sealing must preserve byte-level object semantics. +Do not implement MAP_SHARED with private copies or round permissions up. +Unsupported mapping types must fail before a mapping is created. Temporary +restrictions are prototype limitations, not completion of the sharing goal. + +Pinning and device mappings +========================== + +GUP currently returns native ``struct page`` entries and iterates in native +PAGE_SIZE units. A 4K virtual interval can cross independently backed slots +within one such iteration. This requires an explicit subpage-aware range +representation, not merely returning the parent struct page. Cover both +fast and slow GUP, remote access, process_vm, ptrace, direct I/O, io_uring +registered buffers, RDMA, PFNMAP/MIXEDMAP and dma-buf/GPU mappings before +allowing their use on experimental mms. Early prototypes must reject +unsupported entry points before pinning or causing partial side effects. + +Keeping kernel pages at 16K does not by itself solve device mappings. +Device/IOMMU granularity and ownership must be checked independently. +Do not expose neighboring private slots to a device or bounce-copy a shared +GPU buffer and claim equivalent coherence or performance. + +x18 +=== + +The linked XNU ``x18_toggle_4k.plist`` invokes ``vm_spawn_tool`` with the +``x18_toggle`` and ``x18_signal_handling`` tests. The corresponding +``tests/x18_toggle.c`` expects x18 to be cleared without the custom ABI and +preserved after calling ``os_set_custom_x18_abi_enabled(true)``. This does +not establish a Linux NCE runtime requirement. + +This Linux base saves/restores x18 in the normal EL0 entry/return register +frame (arch/arm64/kernel/entry.S). The new page-contract selftest verifies a +sentinel across a raw syscall, a measured voluntary context switch, and a +signal whose handler overwrites x18 before sigreturn. All three checks pass +on the native 16K host and in the experimental 16K KVM guest. This is not +proof of a 4K process, ptrace behavior, or a particular NCE runtime. An +Apple-specific x18 mode, if needed by a runtime, needs a separate +demonstrated requirement and ABI. + +Implementation gates +==================== + +1. Establish mixed-granule translations and independent permission changes + in a disposable 16K VM with exact register restoration. Include negative + controls so a skipped test cannot be reported as a pass. +2. Introduce explicit per-mm page-table geometry and subpage ownership with + tests, preserving the native hot paths. Audit all generic/architecture + walkers and invalidation callers before attaching 4K tables to an mm. +3. Boot a minimal 4K EL0 executable next to 16K processes. Implement fault, + anonymous mmap, munmap, protection, exec, fork/COW, teardown, uaccess, + signals and vDSO. Verify neighbor isolation and allocation failure paths. +4. Add coherent mixed-granule memfd sharing, futexes, file faults and + truncation. Add reclaim, swap, migration and accounting tests. Refuse + unsupported THP/hugetlb, KSM, userfaultfd and pinning paths until audited. +5. Add the required device/pinning paths and run real translation workloads. + Measure native-process overhead, memory use, fault costs, context switches + and target workloads against an unchanged baseline. VM success is not + M5/Neo hardware qualification or proof of GPU/NCE performance. + +Architectural test +================== + +Build natively from this worktree, using an out-of-tree directory:: + + python3 tools/testing/kunit/kunit.py config --arch=arm64 \ + --kunitconfig=tools/testing/kunit/configs/arm64_user4k.config \ + --build_dir=/path/to/build --make_options=LLVM=1 + make ARCH=arm64 LLVM=1 O=/path/to/build -j4 Image + +The tiny VM config is for MM tests, not an Asahi deployment configuration. +Use the existing QEMU/KVM installation on a Linux arm64 host with /dev/kvm:: + + timeout 90 qemu-system-aarch64 -machine virt,accel=kvm,gic-version=3 \ + -cpu host -smp 2 -m 512M -nodefaults -nographic -serial stdio \ + -monitor none -no-reboot -kernel /path/to/build/arch/arm64/boot/Image \ + -append 'console=ttyAMA0 arm64.user4k_test=1 kunit.filter_glob=arm64-user4k* kunit_shutdown=poweroff' + +The hardware tests require ordinary direct PTE permissions, no CnP, DS=0, A1=1, +and 4K-granule support. These are explicit test limits, not production +workarounds. A skipped test establishes no mixed-granule behavior. Boot +again without the arm64.user4k_test parameter and verify the test skips. +The translation tests cover alternating native/4K roots, four adjacent +subpages with RO/RW/unmapped/alias descriptors, a single-PTE BBM replacement, +its changed physical address and write permission, stable kernel mappings +and restoration of TCR/TTBR0. They do not yet cover actual user execution, +execute permissions, remote TLB shootdowns or ASID rollover. + +The context/uaccess case uses two kernel threads and two private mms. +Each thread performs sixteen borrowed-mm switches, migrates between the +guest's CPUs, sleeps with the mm borrowed, and checks four mappings each +round. It checks read-only and unmapped neighbors, successful writable +copies, exact physical backing and unchanged kernel TTBR1 base. Each worker +also exercises both the normal idmap and raw transition-table installers +from a 4K mm, checks native TG0, removes the idmap and repeats user access. +Copies run with page faults disabled: absent/RO mappings take exception +fixups without entering the unimplemented writable-fault/VMA paths. + +The KUnit config enables ARM64_SW_TTBR0_PAN as a fallback. Hardware PAN +remains selected normally. With ARM64_USER4K_KUNIT_TEST and that fallback +enabled, the early CPU-feature override accepts ``id_aa64mmfr1.pan=0`` +for explicit VM testing of software PAN. The override accepts only zero +and is absent from non-test kernels. Add it to the enabled VM command line +and verify the emulated-PAN boot message and software-PAN test result. +It is not a production setting or performance fix; its cost is unmeasured. + +For a memory-sanitized ownership run, build separately with +``tools/testing/kunit/configs/arm64_user4k_data_kasan.config``. This selects +ARM64_USER4K_DATA_KUNIT_TEST and KASAN_GENERIC while disabling the raw +translation/context test suite. Boot its Image with the same initramfs +and ``kunit.filter_glob=arm64-user4k-data*``; no hardware-test boot opt-in +is required. This checks metadata lifetime, not 4K EL0 execution or +instrumentation boundaries between slots in one native allocation. + +Process contract tests +====================== + +Build the userspace contract test natively:: + + make -C tools/testing/selftests/arm64/user4k CC=clang \ + LDFLAGS=-static OUTPUT=/path/to/results all + /path/to/results/page-contract + +The default expectation is a 16K process. There are eleven checks covering +AT_PAGESZ/sysconf agreement, mapping alignment, protection boundaries, +neighbor preservation, fork/COW, unmap, shared aliases, cross-fork shared +data with local permissions, and x18. These currently test same-granule +sharing, not sharing between 4K and 16K processes. + +``page-contract --expect-4k`` asserts a 4K ABI but does not enable one. It +must fail on the current kernel; the negative control currently exits 1. +Once an explicit exec opt-in exists, run it through that launcher and then +add mixed-granule peer/futex/truncate tests. Never report this negative +control as a successful 4K process test. + +The static test can be /init in an otherwise empty initramfs; as PID 1 it +powers the VM off after printing results. Use gen_init_cpio with a /dev +directory, /dev/console (character device 5:1), and the executable at /init. +Pass ``-initrd /path/to/initramfs.cpio`` and ``rdinit=/init`` to QEMU, and +omit ``kunit_shutdown`` so the kernel enters userspace after KUnit. Capture +both KUnit and userspace output. The asynchronous kernel console can +interleave its output with /init; check complete test results and counts, +not just QEMU's exit status. + +Recorded development results (2026-09-30) +======================================== + +On the native arm64 Linux host, clang/LLD 22 built the VM kernel. KVM used +two vCPUs, host CPU features and 512 MiB of RAM. The packed-table geometry +and translation tests both passed. The 16K process contract passed all +eleven checks after the architectural test restored the ordinary context. +The same eleven checks passed on the unmodified host kernel. The final +no-opt-in VM boot skipped the translation test and passed all eleven +process checks with the default 16K ABI. No 4K Linux process has run yet. + +The subsequent core-MM implementation passes five KUnit cases, including +the two MM geometries, common range walking, partial and full teardown, +hardware translation through Linux-allocated 4K tables, and the original +packed-table test. The sparse cases cross PTE, PMD, PUD and root boundaries. +The range callback tests account for every byte of the 48-bit range as +either a present mapping or a hole. Teardown checks both preservation of +a shared boundary table and removal of exactly one complete PTE table. +The hardware case validates all eleven sparse translations for EL0 reads +and writes before restoring TCR, TTBR0 and PAR; TTBR1 is checked unchanged. + +The core-MM geometry-stage VM run passes all five cases and all eleven native process +checks. With the boot opt-in absent, three software cases pass, two hardware +cases explicitly skip, and all eleven native process checks still pass. + +The subsequent fragment/read-fault/TLBI stage adds fragment lifetime, +anonymous read-fault, native huge-page table-token, TLBI operand and gather +geometry tests. The anonymous fixture initially returned VM_FAULT_SIGBUS +because vm_area_alloc() installs dummy vm_ops; marking the test VMA +anonymous selected the intended fault path. This was a fixture correction, +not evidence of a writable-fault implementation. + +The final fragment/read-fault/TLBI VM passes all ten kernel tests (eight +MM/translation cases plus two TLBI/gather cases) and all eleven native +process checks. Without arm64.user4k_test=1, eight software cases pass, +the two hardware cases skip, and all eleven native process checks pass. +The Image was built with THP, PT_RECLAIM and ARM64_CONTPTE enabled. There +is still no 4K userspace process, writable 4K data allocator or measured +application performance result. + +The later context/uaccess stage passes all eleven kernel cases and eleven +native-process checks in both hardware-PAN and explicitly forced software-PAN +KVM boots. Each boot checks 32 borrowed-mm switches, 128 per-mapping user-copy +results, and four idmap/transition-table cycles. Without the hardware-test +opt-in, eight kernel cases pass, three hardware cases skip and all eleven +native-process checks pass. The five feature-disabled MM objects remain +byte-identical to the original M5 base in the audited executable sections. + +The data-ownership stage adds six cases. Both hardware-PAN and software-PAN +boots pass all seventeen kernel cases and eleven native-process checks. +The context/uaccess fixture now uses the packed data references for its 4K +physical backing. The no-opt-in boot passes fourteen software cases, skips +three hardware cases and passes all eleven native-process checks. + +A separate KASAN_GENERIC/INLINE build passes the six ownership cases and +all eleven native-process checks, with no sanitizer report. The allocator +object contains ASan load/store/memcpy/memset instrumentation references. +This establishes tested metadata lifetime paths, not a proof of all races +or a sanitized 4K-process implementation. The five audited feature-disabled +MM objects remain byte-identical to the M5 baseline. + +The subsequent physical-lookup/reverse-walk stage passes 23 kernel cases +and eleven native-process checks with both hardware PAN and forced software +PAN. Without the hardware opt-in, twenty cases pass and three skip. The +eight data-ownership cases, including concurrent lookup/copy/retirement, +also pass on a single-vCPU guest and in the separate KASAN build, with no +sanitizer report. Seven audited feature-disabled objects (the previous five +plus mm/page_vma_mapped.o and mm/rmap.o) have byte-identical executable +sections to the M5 baseline. None of these results establishes a 4K EL0 ABI. + +The VMA-offset stage adds six cases covering byte-offset arithmetic, +interval-tree candidates, real VMA duplication/splitting, complete and +partial merges, and VMA copying/shrinking. With DEBUG_VM, VM maple/RB tree +validation and PROVE_LOCKING enabled, hardware and software PAN boots pass +29 kernel cases and eleven native-process checks without reported warnings. +The default boot passes 26 cases, skips three hardware cases and passes the +same native checks. KASAN passes eight data cases and eleven native checks. +The data suite now drains batched kfree_rcu() metadata frees before the guest +can power off; earlier no-report results did not establish that every +batched metadata free completed before shutdown. + +The existing host-side VMA suite passes all 27 cases under ASan/UBSan. That +suite executes VMA code with userspace stubs and a modeled PAGE_SIZE of 4K; +it is not a 4K process on this kernel. Its liburcu dependency was extracted +locally after checking the package against the pacman database SHA-256. +The changed generic MM objects also compile for x86_64. The seven audited +feature-disabled arm64 objects remain byte-identical to the base in their +executable sections; this audit does not cover every VMA object or measure +feature-enabled native throughput. + +The geometry refactor parameterizes the existing compile-time formulas; +native callers still pass PAGE_SHIFT. The native context.o, mmu.o, pgd.o +and proc.o hashes were identical before and after this refactor in the +same configuration. This narrow code-generation check is not a whole-system +performance benchmark. + +For the core-MM changes, separate feature-disabled builds of this tree and +the original M5 base produced byte-identical executable sections (including +init/unlikely sections where present) in mm/pgtable-generic.o, mm/pagewalk.o, +mm/memory.o, arch/arm64/mm/context.o and arch/arm64/mm/pgd.o. The JSON report +records sizes and hashes. This proves no code change in those sections with +the feature disabled; it does not measure the feature-enabled native path. + +The comparison was repeated after fragment allocation and TLBI changes. +An initial TLBI refactor evaluated the range-capability alternative into a +bool outside the loop, adding runtime flag handling to native code. Keeping +the predicate at its original call site restored byte-identical executable +sections in all five audited objects. The comparison configuration has THP +disabled; native THP table deposit/withdraw is separately exercised by the +THP-enabled KUnit guest, not covered by that byte comparison. + +One earlier clang build exited with SIGBUS (exit 135) while edits overlapped +that build. Its log and generated reproducer were preserved. Builds after +the source was frozen succeeded. The SIGBUS cause was not independently +reproduced; this does not establish a hardware or page-cache fault. + +Development paths on this host: + +* Source: /home/dj/Work/linux-user4k +* Native build: /home/dj/Work/user4k-build +* Native baseline checkout/build: /home/dj/Work/linux-user4k-baseline and + /home/dj/Work/user4k-baseline-build; the feature-disabled candidate build + is /home/dj/Work/user4k-native-build. +* Current fragment/read-fault/TLBI evidence: + /home/dj/Work/user4k-results/kvm-fragments-tlb-final.log, + kvm-fragments-tlb-default.log and native-fragments-code-comparison.json. + The fragments-tlb-v1 bundle freezes the Image, config, initramfs, selftest, + logs, comparison report and complete source patch with SHA-256 hashes. +* Context/uaccess evidence: kvm-context-idmap-hw-pan.log, + kvm-context-idmap-sw-pan.log, kvm-context-default.log and + native-context-code-comparison.json in the same results directory. + The context-uaccess-v1 bundle freezes this stage's artifacts and patch. +* Data-ownership evidence: kvm-data-lifetime-hw-pan.log, + kvm-data-lifetime-sw-pan.log, kvm-data-default.log, kvm-data-kasan.log, + data-kasan-instrumentation.txt and native-data-code-comparison.json. + The data-ownership-v1 bundle freezes both tested Images/configs and + the source patch. The separate sanitized build stays at + /home/dj/Work/user4k-kasan-build. +* Physical lookup and reverse-walk evidence: kvm-rmap-races-final-hw-pan.log, + kvm-rmap-races-final-sw-pan.log, kvm-rmap-races-final-default.log, + kvm-rmap-races-up.log and kvm-rmap-races-kasan-final.log. The + lookup-rmap-v1 bundle freezes this stage's Images/configs, logs, comparison + reports and complete source patch. Earlier failed fixed-retry-budget + evidence remains in kvm-rmap-races-hw-pan.log. +* VMA offset evidence: kvm-vma-drain-hw-pan.log, kvm-vma-drain-sw-pan.log, + kvm-vma-drain-default.log, kvm-vma-drain-kasan.log, + vma-host-copy-shrink-regression.log, build-x86-vma.log and the two + native-vma-offset comparison reports. The vma-offset-v1 bundle freezes + this stage, including the earlier failed split test in + kvm-vma-offset-hw-pan.log and its corrected successor. +* Earlier geometry-stage evidence: /home/dj/Work/user4k-results/kvm-mm-hardware.log, + kvm-mm-default.log and native-code-comparison.json. The frozen + mm-geometry-v1 bundle contains the Image, config, initramfs, selftest, + hashes, logs and source patch for this tested stage. +* Earlier evidence: kvm-packed.log, kvm-contract.log, + kvm-default-contract.log, native-contract.log and native-reject-4k.log. + Earlier hashes naming files in the warm build directory are historical; + that directory is reused by later builds. + +Next implementation boundary +============================ + +The generic ``pgd_offset(mm, address)`` has an mm argument, but +``p4d_offset``, ``pud_offset`` and ``pmd_offset`` do not. Lockless variants +can be handed a local copy of an entry, so inferring the granule from the +pointer's backing page is not generally valid. Never infer the target mm +from ``current->mm``: remote walkers and kernel mappings need their actual +target. Explicit mm-aware geometry/walker interfaces with native fallbacks +now exist, and allocation/common range walking/table teardown use them. +This conversion is not complete: fault, fork, zap, GUP, folio walking, +reverse mapping and other callers still contain native-only assumptions. +Do not attach a 4K mm to a userspace task until these paths are converted +or have explicit, tested rejection paths for unsupported operations. Only +the bounded kernel-worker fixture may borrow a private mm, with faults +disabled and its mappings and lifetime controlled by the test. + +``MMCF_USER_4K`` is currently set only by KUnit on an empty, isolated mm. +The production exec/fork initialization path still needs a transactional +granule selection before root allocation, and arch_setup_new_exec() must +not erase it. Scheduler/uaccess TG0 selection is implemented and tested +with borrowed mms, but actual 4K EL0 entry/return is still untested. The TLBI +helpers now have explicit geometry, but their unconverted callers still +require an audit of ranges and strides. + +File-backed VMA offsets use an explicit subpage byte component while +the cache's pgoff_t remains in 16K units. VMAs now carry that component in +vm_subpage_offset, with vm_page_offset helpers for carrying, borrowing and +inverse byte-address lookup. Duplication, splitting, existing-range merging, +copying and shrinking preserve the pair. Interval-tree bounds and linear +page-index lookup include the component. The first split test exposed a +missing explicit copy in vm_area_init_from(); that copy is now implemented. +Mmap input plumbing, driver descriptors, stack growth, mapping accounting +and file fault/unmap callers still need conversion. Subpage anonymous ownership must account for +reverse mapping, COW, dirty/accessed state and allocation failure before +runtime opt-in can become reachable. + +Anonymous ownership and reverse mappings +---------------------------------------- + +Every slot reference now retains its native folio. Tests check reference +balance across allocation, lookup, copying, reuse and closure. Exact-offset +allocation and copying reject an occupied requested offset even when other +slots are free; supplying another compatible folio permits that offset. + +The anonymous rmap attachment helper preserves the native +folio->mapping/index relation: one anon_vma root and native linear index, +with a physical slot matching the VMA's linear byte offset. It rejects a +different root, index or offset. The owner pins the anonymous root until +its final slot is released, including when every VMA has already disappeared. +Per-slot mapcounts accompany the native folio mapcount. No PageAnonExclusive +claim is made for an owner whose slots may have different COW relationships. +Mlocked and droppable mappings are not accepted by this internal helper yet. + +The folio aging walk now visits all constituent 4K PTEs rather than stopping +after one PTE for an order-0 native folio. A private KUnit fixture attaches +four slots to a real anonymous VMA and checks mapcounts, reference counts +and aging of all four entries. A fork-related fixture establishes native +anon_vma chains before attaching a second VMA, checks eight shared PTEs, +and destroys the parent while retaining the child. Another fixture removes +every VMA while holding a slot and checks the retained root and rejection +of attachment to an unrelated root. These fixtures passed DEBUG_VM and +lockdep in ``kvm-anon-fork-link.log`` (33 kernel tests and 11 native ABI +tests). They are not tests of normal fork's page-table copying or 4K EL0. +Earlier fixture setup failures are retained in ``kvm-anon-root.log`` +(wrong lock/attachment ordering) and ``kvm-anon-fork-order.log`` (using the +fresh-VMA insertion API for an already cloned anon_vma). + +Compatible neighboring mappings can share an owner, while partial COW +selects another owner when that exact slot is occupied. Normal fault-path +selection of allocation/policy/memcg domains is still unimplemented. This +approach trades arbitrary sparse-slot packing for retaining the existing +anonymous interval-tree representation; its memory and throughput costs +are unmeasured. Later stages below add exclusivity, pins, reclaim and +migration; remaining rmap callbacks and MGLRU still need an audit. Swap entries must +retain the slot offset without retaining a physical allocator reference +after the backing folio has been reclaimed. Neither the current allocator +nor this design permits exposing an unreclaimable writable ABI. + +Anonymous unmap and TLB retirement +---------------------------------- + +The normal present-PTE zap path now removes the exact anonymous slot's rmap +and transfers its mapping reference to mmu_gather. Native data-page tokens +and slot tokens share the existing gather batches; a private, disjoint tag +is consumed before the native page-release functions. Native page/count +pairs remain intact, including count payloads whose numeric value includes +the slot tag bit. Allocation pressure retains the existing flush-and-retry +behavior. Slots are released after the gather's TLB invalidation, not when +the PTE is merely cleared. No per-slot synchronous-flush workaround is used. + +Zap traversal uses the target mm's geometry and leaf stride. Tests remove +one 4K mapping while preserving three neighbors, then remove a range with +both internal and trailing holes. Another test leaves PTE/data/table cleanup +to ordinary mmput()/exit_mmap(), retaining one explicit slot reference to +verify zero mappings and the surviving ownership root afterward. The gather +test mixes native count pairs with 64 slot references across local/dynamic +batches and checks that an otherwise unreferenced slot becomes reusable and +zeroed only after the batch completes. + +Two kernel readers pinned to different vCPUs borrow the isolated 4K mm, +read all four mappings, and then observe the removed slot fault while its +neighbors remain readable. This passed with hardware PAN and forced software +PAN in ``kvm-anon-remote-zap-{hw,sw}-pan.log``: 37 kernel tests and 11 native +ABI tests. The non-opt-in boot passed 33 kernel tests with four hardware +fixtures skipped. The data-only KASAN suite passed nine kernel tests and +11 native ABI tests; it does not exercise these architecture rmap fixtures. +The x86_64 compile-only check for memory/rmap/mmu_gather passed. Executable +sections of eight audited feature-disabled MM/arm64 objects are byte-identical +to the base build. This is not an enabled-feature performance measurement. + +At this historical stage the fixture suite had no normal writable fault path, LRU +placement, swap/reclaim, 4K EL0 task, or exposed process ABI. File/large-folio +batching, nonpresent entries, mlock and page-table checking still require +subpage-specific audits before activation. + +Leaf operations, page-table checking and fork integration +-------------------------------------------------------- + +The leaf physical/protection helpers now live in asm/user-pte.h, early enough +for batched setters. Batched 4K PTE installation advances physical addresses +by 4K and preserves bits 13:12. Cache/tag synchronisation still covers the +native backing pages. Contiguous-PTE operations use 16 entries/64K for a +4K mm and the native 128 entries/2 MiB for a 16K mm. Getters without an mm +infer a live table's granule from its fragment descriptor. Folding/unfolding, +access/dirty aggregation, lockless reads, partial permission changes and +partial/full clearing are exercised for both granules. An initial run found +a remaining native-size-only TLBI level selector in the permission-fault +helper; the corrected runs have no warning. Native contiguous-PTE executable +sections remain identical with the experimental feature disabled. + +Page-table checking is now enabled and enforced in the main debug fixture. +Its page-extension counters track each 4K physical slot independently; +native PTE/PMD/PUD mappings cover all constituent slots. This permits four +independently writable anonymous slots without treating them as writable +aliases of the same bytes. Allocation/free checks verify every slot counter. +The 512 MiB debug guest allocates 1,310,720 bytes of page extensions for +this checker together with backing ownership; the checker is optional, +not a production default imposed by this feature. + +Normal dup_mmap()/copy_page_range() now clone the slot references and rmap +counts, traverse the target geometry, and write-protect parent/child PTEs. +The fork test splits its parent into four 4K VMAs and a trailing hole, +clones them using dup_mmap(), verifies byte offsets, PTE physical identity, +read-only permissions and summed RSS, then destroys the parent and checks +that the child still owns four mappings/references. Child teardown uses +ordinary exit_mmap(). The initial RSS assertion used the approximate +per-CPU read and was corrected to get_mm_counter_sum(); that failing test +log is retained as kvm-dup-mmap.log. Pinned-slot copying is not implemented: +the initial fork stage rejected DMA-pinned backing. The later pinning stage +below copies only pinned slots. Swap and migration stages add nonpresent-entry +integration. + +The final fork stage passes 41 kernel tests plus 11 native ABI tests with +hardware and software PAN; the non-opt-in run passes 37 with four hardware +fixtures skipped. Data-only KASAN covers nine backing tests plus the native +ABI checks, not the architectural fork fixture. x86_64 compile checks include +page-table checking, memory and rmap. Feature-disabled executable-section +comparisons remain identical for the eight earlier audited objects plus +arm64 fault handling; contiguous-PTE code was compared with matching +THP-enabled control configs. No 4K process ABI or ordinary writable fault +path is exposed by this stage. + +Anonymous writable faults, COW and allocation tags +-------------------------------------------------- + +The ordinary anonymous write-fault path now allocates and charges native +backing through folio_prealloc(), reserves the matching 4K offsets, adds +slot rmap entries and places the folio on the LRU. A dense VMA can install +four empty slots from one 16K allocation; a narrow 4K VMA exposes only its +own slot. Existing neighboring PTEs/markers are preserved. Fresh backing is +zeroed once; recycled slots are cleared before their references are published. +The tests check sparse/dense boundaries, data, LRU placement, summed RSS, +native references and ordinary exit cleanup. Their guests have no active +swap: this is not reclaim validation. + +The private write-protection path handles zero-page and anonymous-slot COW +at 4K granularity. It stabilises the old backing, copies only the selected +slot, revalidates the PTE under its lock, breaks the old mapping with a TLB +flush and only then publishes the replacement. Old slot rmap/reference +release follows that flush. The notifier range is exactly 4K. File-source +copy plumbing is now exercised by the later file-mapping fixtures below. +VM_LOCKED, VM_DROPPABLE and private special device mappings remain unsupported +in these internal fixtures. The later pinning stage adds per-slot exclusivity +and reuse of an already exclusive slot. A necessary COW copy still allocates +a native folio for one slot; allocation-domain reuse remains work, and no +enabled-feature performance result is claimed. + +New arm64 partial copy/clear helpers preserve neighboring MTE allocation tags. +A fresh tagged destination initialises its native tag storage once; subsequent +copies/clears touch only the selected 4K. Failed copies cancel the unpublished +reservation. The MTE test checks every 16-byte tag granule during same-folio +slot reuse/copy and copying into fresh backing. It passes on QEMU TCG with +MTE enabled, including a generic-KASAN build. KVM on this host reports MTE +unavailable, so that check is explicitly skipped there. The assembly fallback +for a tag-transfer block larger than 4K is not independently exercised. + +The final cached-translation test borrows the forked child's mm, reads all +four slots, waits while COW replaces one slot, then checks data and write +permissions in both child and parent. Its initial version disabled the +access-flag faults needed for fork's old PTEs and failed its reads of the +uncopied child slots. The retained trace identifies that fixture error; +allowing the initial reads through the normal fault handler makes the test +pass with hardware and software PAN. Subsequent permission probes remain +fault-disabled so they cannot themselves trigger further COW. + +Final KVM runs pass 45 kernel tests with one MTE skip, plus 11 native ABI +tests. The non-opt-in boot passes 40 kernel tests with five hardware and one +MTE skip. Data-only KASAN passes nine with one MTE skip; TCG MTE/KASAN passes +all ten backing tests. Each also passes the 11 native ABI checks. These are +component fixtures, not 4K userspace ABI tests. x86_64 memory.o compiles; +feature-disabled audited MM and native page-copy executable sections match +the base. The first native audit command incorrectly requested the optional +user-subpage.o with its config disabled; its errors are retained separately +from the corrected successful native build. + +The writable-ownership audit found that folio->private aliases folio->swap +and that PageAnonExclusive tracks a native page. Neither can silently serve +as four independent subpage lifetime/COW states. Explicit slot references +now attach to PTE/rmap ownership. Migration integration is described below; +kernel pinning callers still need the fragment interface described below. +Backing-memory charges must stay distinct from mapped-byte accounting; +current internal RSS assertions count logical PTEs and are not a completed +OOM/proc accounting policy. + +Anonymous reclaim, swap and swapoff +----------------------------------- + +A swap PTE now retains a two-bit slot offset in arm64's previously unused +bits 4:5, separately from the native 16K swap-cache index in bits 12:61. +Type/offset conversion continues to name the native swap entry. Moving a +swap offset preserves the slot bits; raw PTE copying through fork preserves +them too. The encoding fixture covers all four slots, combinations of swap +flags, several types and the high offset boundary. The migration stage below +reuses this encoding for migration entries. Device-private entries remain +unintegrated. + +The anonymous reclaim rmap path processes every present 4K mapping of a +native folio, preserves each slot offset in its nonpresent PTE, and takes +one swap-count reference per PTE. It uses the existing immediate/deferred +TLB machinery with 4K address ranges. Slot references are dropped after +rmap removal; normal swap-cache/isolation references retain physical memory +through writeback and reclamation. The allocator refuses to recycle free +slots in swap-cache folios, because nonpresent PTEs can still need their +bytes. Final anonymous-owner release also tries normal free_swap_cache() +so an unused cache need not retain otherwise dead backing. + +Restoration attaches slot ownership to an uptodate locked cache folio +without zeroing its data. It can reacquire a live slot, reserve an unused +slot in a surviving owner, or create metadata after the original owner was +retired. Pool metadata now retires through RCU as well as owner metadata; +a detached state prevents revival between final-reference retirement and +page-extension unbinding. Tests preserve distinct slot contents through +these cases and race 2,048 restorations against metadata retirement. Fresh +rmap attachment does not dereference a stale, unmapped anon_vma pointer in +a retained swap-cache folio; it establishes and pins the destination root. + +The 4K swap-fault path restores one slot and its MTE state, revalidates its +nonpresent PTE, updates rmap/RSS/swap counts, and uses the small-slot COW +handler for a write fault. Swapoff walks use per-mm table geometry and +strides and restore each slot without promoting write permissions. A +poisoned/unreadable cache folio produces a poison entry as in the native +path. Large/KSM swap folios and arbitrary cross-root aliases are not yet +supported for the alternate ABI. Small mms currently use native swap-cluster +readahead; their VMA readahead policy still needs conversion and performance +measurement. Native readahead behavior is unchanged when the feature is off. + +The guest runner is ``tools/testing/selftests/arm64/user4k/swap-fixture-init.c``. +It is an extended selftest build target, not a test to execute on the host. +As VM PID 1 it mounts the minimal test filesystems, runs the 11 native ABI +checks, enables a dedicated virtio swap disk and verifies 4 MiB of native +swap traffic before invoking the late KUnit swap suite. Its swap-control +helpers refuse to run unless this same executable is the guest's PID 1. +Host invocations of both helpers were checked to refuse the operation. +Use ``kunit.autorun=0``: a test skipped at boot remains marked skipped on +this kernel's debugfs rerun. The initial attempted rerun was rejected by the +runner rather than reported as a pass. No KUnit framework change was made. + +Seven late swap tests pass with hardware and software PAN on KVM: +cached and disk round trips, mapped-LRU reclaim, real swapoff of two forked +4K mms, 32 swap/fork/COW/exit cycles with distinct data, swapoff across 2 MiB, +1 GiB and 512 GiB table boundaries, and live faults from two vCPUs. The live +readers borrow one child mm, cache its translations, wait for reclamation +and cache eviction, then refault through normal uaccess and write different +bytes in one child slot. Both writes survive and the parent's original data +remains intact. This initiates concurrent accesses; it does not establish +that every specific PTE revalidation race was hit. + +The six non-hardware swap cases also pass under TCG with MTE, verifying +allocation tags after cache eviction, swapoff and COW. The live-reader case +is excluded there because that CPU configuration does not satisfy the +fixture's TTBR0 control restrictions. Generic-KASAN covers all 13 backing +and restoration tests under TCG/MTE (12 plus an MTE skip on KVM), not the +architectural swap suite. Ordinary KVM boot passes 49 kernel tests with +seven late-swap and one MTE skip; without the hardware-test opt-in it passes +44 with thirteen skips. The late swap runner separately passes its seven +cases. Each VM also passes the 11 native ABI checks. The feature-disabled +executable-section audit covers thirteen MM/arm64 objects, including swapfile +and swap_state; x86_64 checks are compile-only. + +These results cover bounded, isolated anonymous mappings, not a general +memory-pressure workload or a userspace process ABI. GUP/pinning, +per-slot exclusivity, sparse/COW allocation reuse, accounting units, file and +shared mappings, userfaultfd, mlock, exec and syscall validation remain gates +before an EL0 process may opt in. The kernel and all ordinary processes still +use 16K pages by default. + +Anonymous migration +------------------- + +The normal migrate_pages() path now prepares metadata before unmapping a +subpage-backed folio. Source and destination owners stay bound to their +original physical folios: moving an owner pointer would race lockless slot +lookups that already acquired a native reference. Preparation allocates a +separate destination owner and pins both metadata records without taking +extra native folio references. This preserves migration's normal refcount +checks. Allocation failure leaves the original mappings intact. + +Migration entries name the native PFN and retain the 4K slot in bits 4:5. +Rmap removal handles every 4K PTE, including deferred TLB invalidation, and +ordinary migration copies the native data, tags, mapping and swap-cache +state. Remapping reuses preallocated metadata without allocation or zeroing. +Failure restores the original slots. Empty metadata is retired only after +migration or rollback finishes; it does not consume the migration driver's +folio reference. A final slot put now clears its last reference and used +bit together under the pool lock, so recovery cannot observe a retiring +slot between those operations. + +Fork preserves slot bits when it downgrades writable migration entries. +The exact-slot reverse-map walker also checks those bits. An entry removed +by concurrent unmap is not restored. The test that first exercised the +exact-slot walker incorrectly started each lookup at the first slot; the +API requires the candidate address. That fixture was corrected, and both +its failing and passing logs are retained. + +Bounded migration tests cover dense and sparse mappings, parent/child +aliases, normal refcount-failure rollback, asynchronous migration, and the +4K mappings on each side of the 2 MiB, 1 GiB and 512 GiB table boundaries. +A separate fixture forks while migration entries are installed, removes +one parent's entry, then checks restored offsets, permissions, mapcounts +and RSS. Two live workers cache translations before normal migration and +then access the new backing through uaccess and perform child-only COW. +Another fixture resumes the workers while the source is locked with +migration entries installed: their accesses remain pending until rollback +restores the PTEs and releases the folio. Both live tests pass with hardware +and software PAN. These are kernel workers borrowing private mms, not EL0 +processes. + +A late swap test migrates a swap-cache folio with one present slot and seven +swap entries across two mms, then faults every slot back in. Data and MTE +tags survive; the eight-case late suite passes on KVM, and its seven +non-hardware cases pass on TCG/MTE. The backing-owner test separately +covers successful move, rollback, all entries removed, and a partially used +folio; all fourteen backing tests pass under KASAN/TCG/MTE, or thirteen +plus an MTE skip on KVM. The feature-disabled executable sections of +migrate.o, memory.o, rmap.o and page_vma_mapped.o match the base. x86_64 +builds of those objects also pass; that is compile-only validation. Ordinary +KVM boot now passes 54 kernel cases with nine skips (eight late swap cases +and MTE); without the hardware opt-in it passes 47 with sixteen skips. +Each guest also passes the eleven native ABI tests. + +Other migration drivers, device-private memory, hardware-poison paths, +NUMA-policy workloads, memory offlining, pins and broad pressure testing +still require integration or validation. The alternate ABI remains closed. + +Per-slot exclusivity and fragment references +------------------------------------------- + +Anonymous exclusivity and pin counts now belong to each 4K slot. Fresh +private mappings and COW copies become exclusive; fork clears exclusivity +when sharing an unpinned slot. A pinned slot is copied for the child while +its unpinned neighbors can remain shared. Destination data, charge and owner +metadata are allocated outside the PTE locks. An already exclusive slot is +reused on a write fault, including a pinned slot whose PTE was read-only. +The native PageAnonExclusive bit remains clear for these backing folios. + +A slot pin owns both a logical slot reference and a native FOLL_PIN reference. +Sharing and pin acquisition serialize under the ownership lock, with the +caller holding the PTE lock. Pins keep the slot occupied and its anon root +alive after all mappings disappear. Migration preparation refuses an owner +with pins; reclaim restores a pinned PTE instead of making it shareable. +Migration entries preserve exclusive read state as well as write state. +This implementation has no lockless pin walker yet. + +The experimental kernel interfaces get_user_fragments_remote() and +pin_user_fragments_remote() return records containing a folio, exact byte +offset, length, ownership reference and reference kind. Records end at the +target mm's granule, not the caller's. Both interfaces manage mmap_lock and +fault retries, validate VMA access, preserve partial results, and delegate +native-granule requests to ordinary GUP. release_user_fragments() handles +either reference kind and optional dirty marking. Copying a record does +not duplicate its reference. + +Ordinary read references retain shared COW backing. Read pins must unshare +anonymous COW backing but leave its PTE read-only; write pins obtain writable +or valid forced-access backing. The small-granule path supports anonymous +slots and the zero page; the later file stage adds ordinary cache backing. +Device/large/KSM backing, FOLL_LONGTERM and +additional GUP flags still require integration. This is not a drop-in +replacement for struct-page-array callers: those callers must consume the +physical offsets and lengths explicitly. Until converted, native page-array +GUP rejects alternative mms with EOPNOTSUPP, and its fast walk falls back +without touching their tables. A test checks the legacy rejection. Native +mms continue through their original GUP paths. + +Tests check unaligned ranges, truncated output capacity, unmapped-range +partial results, no-fault failures, overflow, read-only VMA permissions, +forced access, native 16K requests, and COW behavior for both reference kinds. +A zero-page mapping at virtual slot 1 returns physical offset 17 for a +request 17 bytes into its 4K mapping; after a write fault its private backing +has physical offset 4096+17. This checks that the API does not derive the +physical offset from the native virtual-page offset. + +The pin-lifetime fixture holds two pins, forks, checks that only the pinned +slot is copied, verifies its write-fault reuse, destroys both mms, and releases +the pins individually. MTE tags survive the copy. Another fixture runs two +pinning workers against 32 fork/COW/exit cycles and asynchronous migration +attempts; parent data stays unchanged. Successful migrations and unsuccessful +attempts are observed, but the fixture does not classify every failed attempt +or prove that every possible race interleaving occurred. A late swap fixture +checks refusal to unmap a pinned slot, then successful swap and data/tag +restoration after release. + +KVM hardware/software PAN and TCG/MTE runs pass the bounded reference/pin +checks. The late suite has nine passing KVM cases and eight non-hardware +TCG/MTE cases. Feature-disabled executable sections of gup.o, memory.o, +rmap.o, migrate.o and page_vma_mapped.o match the base; the x86_64 checks are +compile-only. The first native GUP fixture accidentally retained dummy VMA +operations and could not fault anonymous memory; its failed run is retained, +and the corrected anonymous-VMA fixture passes. An intermediate native guard +changed gup_fast_fallback's compiled size by four bytes; preprocessor gating +restored the baseline executable-section match, with both audit reports kept. + +The additional arm64_user4k_mm_kasan.config enables the software MM suite +under generic KASAN, lockdep and PageTableCheck. At this stage, raw translation/borrowed-mm +fixtures remained disabled: the hardware boot opt-in is rejected with KASAN, +and that stage's object audit found no raw TCR/TTBR writes or borrowed-mm +operations in the KASAN test object. Later mmap/mprotect fixtures, described +below, use normal borrowed-mm operations under KASAN. A deliberate hardware-opt-in attempt reports a malformed +early option and skips the translation case; it is a gate test, not a passed +hardware translation test. + +Ordinary KVM runs pass 58 kernel cases with ten skips, plus eleven native +ABI checks. The full software KASAN suite passes 51 cases with seventeen +skips on KVM, or 52 with sixteen skips under TCG/MTE. This now covers faults, +fork/COW, migration, typed references/pins and the pin race, rather than just +backing metadata. The eight non-hardware late swap cases also pass with +KASAN and MTE. These counts exclude the native ABI checks, which pass in +each guest. The data-only KASAN profile is retained for focused ownership +regressions. + +There is still no EL0 opt-in or production performance claim. Legacy GUP +callers, remote-access/syscall consumers, allocation reuse, byte accounting, +exec, userfaultfd, locking and broader stress workloads +remain integration gates. Ordinary userspace continues to use 16K pages. + +File-cache and shmem mappings +----------------------------- + +The page cache retains native 16K indices and folio ownership. A 4K PTE maps +an explicit quarter of a native page and owns one folio reference and one +file-rmap reference. Native and alternative processes may therefore share +the same cache bytes; no cross-granule copy or blanket sharing prohibition +is needed for these mappings. Per-PTE access permissions remain independent. +This is bounded internal-fixture support, not an exposed mmap ABI. + +The small-granule finish-fault and filemap fault-around paths intersect the +locked cache folio with the VMA, its paired file offset, the current PTE +table and EOF. Fault-around retains the existing byte-window policy and maps +ready folios in batches under one PTE lock. Native cache-index callbacks are +unchanged. Each installed leaf takes a separate reference; the callback's +reference is consumed only on successful finish-fault. File and shmem faults +reject a granule starting past EOF even when it lies within the last native +cache page. A partially populated final user granule remains accessible. + +Private write faults without a present PTE preallocate native backing and +slot metadata before locking a cache folio. They copy the selected 4K bytes +and tags and publish an exclusive anonymous slot. Present file-PTE COW uses +the existing subpage copy path. Its neighbors remain file-backed. Necessary +private COW still has the sparse-allocation limitation described above. + +File interval-tree invalidation now translates native file ranges through +the VMA's byte-offset component. Truncation additionally removes complete +small granules between the process-rounded EOF and the native-page boundary, +including private COW copies, on both sides of cache truncation. Native +partial-page behavior is retained. The native page-cache hole-punch machinery +zeros sub-native-page shared bytes while preserving private COW copies. + +Fragment GET/PIN accepts ordinary file-cache leaves. A GET on a private file +mapping retains cache backing; a read PIN first unshares it into an anonymous +slot, leaving the PTE read-only. Shared-file pins use ordinary folio pin +accounting and exact physical byte offsets, and may outlive the mappings. +Device/PFN/mixed mappings and long-term pinning remain unintegrated. + +File migration entries retain the subpage offset separately from the native +PFN, and restoration uses that offset. File reclaim visits every 4K leaf, +including aliases across PTE-table boundaries, instead of stopping after one +native-page-sized reference count. Native-PFN batching is not used for 4K +quarters of a large folio. Large-folio behavior, packing/batching performance, +writeback and general pressure are not established by this stage's order-0 +fixtures; enabled-feature performance remains unmeasured. + +Nine tests cover shared native/alternative aliases, mismatched virtual/file +alignment, fault-around, cold-PTE and present-PTE private COW, partial EOF, +truncation including COW, index-range invalidation, fragment references/pins, +fork with partial/full-native-page hole punching, and migration success, +rollback and reclaim across a 2M table boundary. One uses generic +filemap_fault/page_mkwrite against an already populated shmem cache; it is +not a cold disk-I/O or arbitrary-filesystem test. + +The final KVM hardware-PAN and software-PAN runs each pass 67 kernel cases +with ten skips, plus eleven native ABI tests. The software-MM KASAN/MTE run +under TCG passes 61 kernel cases with sixteen skips, plus those eleven ABI +tests. No warning, lockdep or sanitizer report occurs in these final runs. +The nine late swap cases also pass on the preceding build (before a test-only +mmap-lock fix). All nine newly audited MM objects have feature-disabled +executable sections identical to the original base, including filemap, shmem, +truncate and mmu_gather; the x86_64 object checks are compile-only. + +Intermediate failures are retained: the first private fixture incorrectly +set VM_MAYSHARE; a reclaim test checked neighboring PTEs after fault-around +had repopulated them; a fork fixture looked up a VMA without its mmap_lock. +The corrected fixtures pass. The first native audit found four-byte compiler +layout differences in memory.o and filemap.o; restoring their original native +expressions yielded byte-identical executable sections. + +There is still no EL0 activation, ELF support, syscall mapping ABI, or broad +application/performance result. File-backed integration does not imply GPU +or other device-mapping support. Remote-access/futex callers, virtual/RSS +accounting, syscall alignment, allocation reuse and exec remain next gates. + +Mapping syscalls, protection and virtual reservations +------------------------------------------------------ + +The ordinary vm_mmap/vm_munmap and arm64 mprotect paths now accept the mm's +mapping granule. The byte offset from arm64 mmap is split into native cache +index and subpage offset; the pair survives mmap preparation, callback +validation, VMA creation, prepend/append merging and partial unmap. Existing +native-index APIs retain their original units. This is internal borrowed-mm +coverage: there is still no userspace activation interface. + +Virtual VMA totals, stack/data/executable sizes and core locked_vm accounting +use user granules. Global commitment remains in native pages. Each alternative +mm tracks its aggregate committed user granules and charges the rounded delta +of that aggregate, so splitting a VMA cannot inflate reservations or refunds. +Failed reservations leave both counts unchanged. The exported locked-page +accounting helper still accepts native physical pages and converts to the mm's +units. External accounting readers and direct consumers remain a subsequent +integration gate; pinned_vm remains in native physical pages. + +mprotect walks mm-specific table levels and batches contiguous leaves within +a folio, including four leaves in an order-0 native folio. A batch is split +where anonymous slot exclusivity changes, so restoring write permission does +not make a fork-shared neighbor writable. Migration-entry downgrades retain +the physical subpage offset. TLB invalidation uses the actual user stride. +The arm64 erratum-2645198 helper now passes an absolute end address to the +range-flush helper, whose interface expects an end rather than a length. +That CPU-specific erratum path was not independently exercised by these VMs. + +Nine added cases cover descriptor offsets, file-offset merging, aggregate +commit accounting and denied-reservation rollback, 4K unmap/accounting, +ordinary mapping entry points, protection changes across table boundaries, +mixed exclusive/shared COW batches, protection reservation refunds and changes +while migration entries are installed. The migration-entry fixture is a +serialized transition test, not concurrent pressure. Deliberate overcommit +rejection emits an expected __vm_enough_memory diagnostic. + +Unlike the earlier raw-control fixtures, the new mapping/protection cases use +normal kthread_use_mm/kthread_unuse_mm and can run under KASAN. Permission +checks use ordinary fault-disabled user copies. Raw TCR/TTBR manipulation +remains disabled under KASAN; the earlier no-borrowed-mm object audit applies +only to its frozen stage. + +Final KVM hardware/software-PAN runs each pass 76 kernel cases with ten skips, +plus eleven native ABI checks. TCG with KASAN and MTE passes 70 kernel cases +with sixteen skips and eleven native ABI checks. All nine late swap cases +pass on the final main Image. The feature-disabled Image boots and passes +the eleven native ABI checks. The host VMA suite passes all 27 cases, and +selected feature-disabled arm64 and x86 objects compile. No warning, lockdep +or sanitizer report occurs in these final guests. No byte-identity claim is +made for this stage's feature-disabled code. + +Remaining gates include accounting consumers, mremap movement, mlock and +populate, special/device mappings, legacy GUP callers, remote access, futex, +exec/ELF/arguments, process activation, allocation reuse and broad application, +pressure and performance testing. This is not a completed 4K process ABI. + +Memory accounting readers +------------------------- + +Status byte/kB fields now convert virtual and RSS counters using the target +mm's granule. pinned_vm stays in native physical pages. The page-count fields +in proc stat/statm use the reader's granule; a partial reader page rounds up, +and kernel readers with no mm use native units. This lets a native monitor +read an alternative process and an alternative monitor read a native process +without assuming the target's AT_PAGESZ. File offsets in maps and PROCMAP_QUERY +include the byte component. VMA page-size queries return the user granule. + +signal maxrss/cmaxrss are stored in kB, so exec history and reaped children +compare a common unit. Task accounting time integrals likewise use a fixed +unit across mm changes. OOM comparisons and diagnostic page counts convert to +native-page units, while byte traces and OOM kB messages use target-granule +conversion. The NUMA scan-window comparison and VFIO's direct locked_vm limit +check now use native units. The locked-page helper rejects overflow even when +the caller bypasses rlimits; final mm destruction checks for leaked commitment. + +A new fixture calls the actual status/statm producers with explicit synthetic +4K/native counts, checks mixed-granule high-water maxima, partial-page rounding, +physical pin units and a borrowed 4K reader observing a native mm. Final KVM +passes 77 kernel cases with ten skips; TCG KASAN/MTE passes 71 with sixteen +skips. Each also passes eleven native ABI checks, without warnings or sanitizer +reports. This is reporting coverage, not induced OOM, child-reaping, or netlink +runtime coverage. Native arm64 and x86 object checks pass with task accounting +and VFIO configured; x86 also enables NUMA balancing. Initial attempts to build +those objects with their dependencies disabled failed and are retained. + +smaps/PSS, pagemap/NUMA-map walks, mlock/populate, device mapping and remaining +page-array consumers still need work. No process activation or ELF execution +is provided by this stage. + +Remote memory access and shared futex identity +--------------------------------------------- + +A single-fragment GET variant retains the caller's mmap lock and returns a +stable VMA for cache-maintenance callbacks. access_remote_vm, and therefore +its ptrace/process-memory consumers, now copy the returned physical fragment +and release its typed reference. Private forced writes preserve COW behavior; +shared read-only mappings retain their permission checks. Device callbacks +still use their existing separate path, without establishing device support. + +The alternative-mm process-VM copy path pins batches of typed fragments, copies +with exact physical offsets, marks written backing dirty and releases all pins. +It preserves partial-transfer behavior and the existing task-access checks. +Native targets retain the page-array path. This is not a conversion of all +legacy GUP consumers or a long-term/fast-pin implementation. + +Private futex keys retain their virtual identity. Shared file keys use the +native cache index plus the actual byte offset in that native page, so aliases +in 4K and 16K mms identify the same word even with different virtual alignment. +Typed references keep anonymous slots live until key generation completes. + +Five new cases cover unaligned remote COW copies and forced-write permission, +shared file copies across a native-page boundary, process-VM batches exceeding +the stack array and returning partial results, cross-granule futex keys, and +real futex wait/wake queues in both directions between borrowed 4K/native mms. +The process-VM fixture calls the copy engine with kernel iterators; it is not +a userspace process_vm_readv/writev syscall/credential test. The futex workers +execute kernel entry points with ordinary uaccess, not EL0 programs. PI, requeue, +robust futexes and hostile concurrent VMA modification remain untested. + +Final software-PAN KVM passes 82 kernel cases with ten skips; final TCG +KASAN/MTE passes 76 with sixteen skips. Each also passes eleven native ABI +checks. The nine late swap cases pass on the final main Image. Hardware-PAN +passes on the preceding build before an added shared-file process-VM test +and defensive missing-VMA cleanup. Native arm64/x86 object checks also pass +before that config-enabled-only cleanup. No warning or sanitizer report occurs +in these final runs. Actual ELF execution and process activation remain absent. + +Remapping and temporary stack relocation +---------------------------------------- + +mremap now uses the mm's leaf stride, table boundaries and paired file offset. +Its present-leaf batching shares the protection path's fragment-aware batch +helper. Whole-PMD/PUD moves remain enabled; a new mm-aware PUD table accessor +retains packed-table address bits when transferring an entire subtree. User +alignment, length rounding, growth overflow and virtual-accounting paths use +the appropriate granule. Temporary stack relocation uses the same movement +engine while preserving the original anonymous linear offset. + +Five ordinary cases cover leaf moves across 2M/1G/512G boundaries, exact +transfer of packed PMD/PUD table descriptors, source-TLB invalidation, in-place +resize, DONTUNMAP with fresh source faults, rollback from a rejected file +callback, quarter-offset file mappings, moving a fork-shared quarter then COW, +and overlapping/non-overlapping temporary-stack relocation followed by growth. +A late case moves four swap entries across a table boundary to a differently +aligned VMA and restores their individual bytes and MTE tags. + +Final software-PAN KVM passes 87 kernel cases with eleven skips; TCG KASAN/MTE +passes 81 with seventeen skips. Each also passes eleven native ABI checks. +Final late swap runs pass ten KVM cases or nine non-raw-hardware KASAN/MTE +cases. The debugfs runner repeats the same result in its log; those repetitions +are not additional runs. Hardware-PAN passes the preceding build before only +the new late case was added. Feature-disabled arm64/x86 remap/protection/stack +objects compile. No warnings, lockdep or sanitizer reports occur in these runs. + +These are bounded internal fixtures. Concurrent remap/pressure, moved migration +entries, userfaultfd, full ELF/argument setup, other mapping syscalls, device +mappings, allocation reuse and application performance remain incomplete. + +Exec argument storage +--------------------- + +Temporary argument references now distinguish a native page from a typed +anonymous slot and retain its physical byte offset. User/kernel string copies +and interpreter argument removal walk the target mm's granules. String-length +limits remain 32 target granules. Temporary argument charges convert into the +calling mm's counter units, rather than adding incompatible target counts. +Stack setup aligns and limits the target's range in its own granule. + +A new fixture copies multi-granule strings into both alternative and native +stacks, rejects an over-limit string before advancing the stack pointer, runs +setup_arg_pages with a borrowed mm, verifies the relocated strings and removes +three interpreter arguments. It uses kernel-source strings; actual alternative +ELF execution and user-source exec argument copying remain untested. The +first KASAN/MTE attempt incorrectly expected an exact requested stack top; +arm64's arch_align_stack randomizes within a native page before rounding. The +corrected fixture checks the allowed aligned range and verifies every string. + +Final software-PAN KVM passes 88 kernel cases with eleven skips; TCG KASAN/MTE +passes 82 with seventeen skips. Each passes eleven native ABI checks without +warnings or sanitizer reports. Native arm64/x86 exec objects compile. A RISC-V +no-MMU compile check found that generic accounting/offset helpers depended on +a header included only under CONFIG_MMU; mm.h now includes that header itself. +The corrected no-MMU exec object builds, with compiler warnings in existing +asm-generic I/O helpers. The initial RISC-V configuration still selected MMU; +only the retained NONPORTABLE, CONFIG_MMU=n configuration establishes no-MMU +compile coverage. There is no RISC-V runtime claim. + +Process activation, ELF segment loading/auxv, vDSO/special mappings and broad +application execution are still incomplete. + +Special mappings and vDSO backing +--------------------------------- + +Special faults now map individual user leaves of the exact native page supplied +by the callback, including reordered page arrays and clipped VMA/table edges. +Private forced COW and private read pins create anonymous slots without changing +native aliases. A typed insertion helper maps one user granule at an explicit +physical offset; time-namespace VVAR prefaulting uses that offset. Normal +non-file pages support typed GUP, while IO/PFN/MIXED mappings remain excluded. + +Three fixtures cover reordered special pages, COW and read pins, explicit +quarter insertion and lifetime, and the real arm64 vDSO/VVAR installation at +a 4K-aligned address. They read vDSO ELF bytes, fault all time/RNG quarters +through ordinary uaccess, verify backing addresses, and check remote VVAR +access rejection. They do not execute vDSO code at EL0 or switch time namespaces. +Legacy driver page/PFN insertion interfaces still need integration. + +Final hardware/software-PAN KVM passes 91 kernel cases with eleven skips; +TCG KASAN/MTE passes 85 with seventeen skips. Each passes eleven native ABI +checks, with no kernel warnings or sanitizer reports. Feature-disabled arm64 +and x86 memory/GUP/datastore objects compile. Checkpatch reports zero errors, +seven fixture line-length warnings and one debug VM_BUG_ON_FOLIO warning. +No late-swap rerun was made for this chunk. Process activation and actual +alternative ELF execution are the next gate, not an established capability. + +Initial EL0 execution and exec selection +--------------------------------------- + +The experimental arm64 prctls PR_AURORA_SET_EXEC_PAGE_SIZE (0x41555001) and +PR_AURORA_GET_EXEC_PAGE_SIZE (0x41555002) select/query the calling thread's +pending exec request. All unused arguments must be zero. SET accepts 4096 +only on a 4K-capable CPU; 0 or the native page size resets the request. GET +returns 4096 or 0 (native default). It does not report the current mm's size; +AT_PAGESZ does that. A request is inherited by fork, remains local to that +thread, survives a pre-commit failed exec, and is consumed on committed exec. +An ordinary exec from a 4K process therefore creates a 16K mm. AArch32 exec +with a pending 4K request is rejected before commit; that rejection is code +inspection, not tested compat execution in the current configs. + +mm_alloc_exec selects immutable geometry before allocating the PGD and +argument stack. Ordinary mm_alloc remains native; dup_mm inherits geometry. +arm64's exec reset preserves the selected mm flag. The ELF loader aligns +segments and auxv with the target granule, while its static format descriptor +retains native constants. This does not complete the coredump implementation. + +The PID-1 exec fixture launches native page-contract before and after a +static 4K-aligned executable and dynamic 4K-aligned PIE, using the native +libc and dynamic loader. Each invocation passes eleven checks. It validates +request errors, fork inheritance, failed-exec persistence, successful-exec +consumption and 4K-to-native reset. Separate execs verify 120000-byte argument +and environment strings in 16K-to-4K, 4K-to-4K and 4K-to-16K transitions; +a 139999-byte argument is rejected for the 4K target without losing its request. + +The mixed-process executable uses a 4K-offset memfd alias and an ordinary +native alias. In each parent/child granule direction it completes 2000 futex +round trips, requires observed wakeups of blocked peers in both directions, +and retains a private COW byte plus untouched neighboring file bytes. +Real process_vm_readv/writev copies cross multiple target leaves and return a +partial read at a PROT_NONE boundary. Workers verify every copied byte. +These are parent/child permission-allowed operations, not a credentials audit. + +Final hardware/software-PAN KVM passes 91 kernel cases with eleven skips; +TCG KASAN/MTE passes 85 with seventeen skips. Both also pass all exec/mixed +fixtures, including 44 page-contract checks (two native, static 4K and dynamic +4K). Final logs have no kernel warnings or sanitizer reports. The initial +dynamic attempt failed because the fixture placed libc outside the loader's +/usr/lib search path; correcting the initramfs fixed that run. A feature-off +compile found a missing generic mmu_context include; after correcting it, +native arm64/x86 fork/sys/exec/ELF and native arm64 process objects compile. +RISC-V no-MMU fork/exec also compile, with the existing I/O-helper warnings. + +This establishes actual alternative execution, not a full ABI qualification. +Memory syscalls such as madvise/mincore/mlock, proc PSS/pagemap, coredumps, +general device insertion, remaining GUP users, time namespaces, multithreaded +application stress, pressure and performance still require work. The existing +late-swap tests were not rerun for this exec-only change. + +Residency, synchronization and sealing +-------------------------------------- + +mincore now walks/counts in target user granules while retaining native cache +indices and batching cache-only quarters of one native page. Output buffering +still uses a native allocation. msync aligns its range to the process granule +and includes the VMA's byte suboffset in fsync ranges. mseal aligns/rounds to +the user granule. These paths retain their existing permission/error rules. + +Nine new EL0 checks run in both native and alternative processes: untouched +and separately faulted residency, unaligned inputs and holes, cached file +quarters, a vector ending at PROT_NONE, multiple output-buffer iterations, +msync alignment/holes, a one-byte seal restricted to one user page, and fork +inheritance of that seal. A kernel callback fixture verifies three exact +fsync byte ranges across a native cache-page boundary. The memfd EL0 test does +not establish persistent filesystem writeback; swapped/hugetlb residency and +concurrent query races are not separately exercised by this chunk. + +Final hardware/software-PAN KVM passes 92 kernel cases with eleven skips; +TCG KASAN/MTE passes 86 with seventeen skips. Each passes all eighteen new +EL0 checks plus the prior exec, page-contract and mixed-process fixtures. +No kernel warnings or sanitizer reports occur. Native arm64/x86 objects +compile. Checkpatch reports zero errors, one mutable test-operations warning; +the new userspace test has zero warnings. Late-swap tests were not rerun. + +Prefaulting and basic advice +--------------------------- + +GUP's no-page-array prefault path now follows small leaves without exporting +native page references. It preserves the existing permission, partial-progress +and mmap-lock retry protocol, and updates dirty folio state for write touches. +Array-returning GUP remains unsupported for alternative mms. MAP_POPULATE and +MADV_POPULATE use target-granule counts and advancement. Locked and special +mappings still need their own integration. + +Basic madvise range checks, file byte offsets, shmem swap-cache range endpoints +and guard-marker counts now use the appropriate units. Eleven new EL0 tests +run under each ABI, covering DONTNEED and fork COW, guard install/discard/remove, +read/write population and its errors, MAP_POPULATE, WIPEONFORK/DONTFORK, file +REMOVE with private COW, and a WILLNEED smoke check. Both advice and residency +programs now assert the requested AT_PAGESZ explicitly. A kernel regression +case verifies dirty-folio tracking when prefault follows an already-present, +clean writable entry. Its initial setup triggered arm64's unsafe-PTE-update +warning; the corrected fixture clears and flushes before reinstalling the PTE. + +Final software-PAN KVM passes 93 kernel cases with eleven skips; TCG KASAN/MTE +passes 87 with seventeen skips. Both pass all twenty-two new EL0 advice checks +and the prior exec, residency, page-contract and mixed-process fixtures without +kernel warnings or sanitizer reports. Hardware-PAN passed the preceding build +before the dirty-tracking correction/fixture. Feature-off arm64/x86 GUP and +madvise objects compiled before that config-enabled-only correction. Final +checkpatch has zero errors/warnings for the tracked diff; the new EL0 guard +test has one intentional volatile-access warning. + +This is not full madvise support. Alternative mms explicitly return EOPNOTSUPP +for COLD, PAGEOUT, FREE, KSM merge/unmerge, hugepage promotion/collapse and +memory-error injection until those engines are integrated. Native behavior is +unchanged. Those exclusions are unfinished development gates, not a production +fix or performance result. Swapped WILLNEED, lock-drop races, remote advice, +concurrent pressure and the remaining advice modes still need work. The late +swap suite was not rerun for this chunk. + +Cold/pageout validation and emulator correction +---------------------------------------------- + +COLD and PAGEOUT now walk alternative mappings in 4 KiB units and aggregate +accessed/dirty state across the leaves backed by one native folio. A partial +range or a fork-shared folio is skipped when its mapcount exceeds the leaves +covered, preserving the existing noninterference policy. The kernel regression +covers a young final quarter, a partial range, and shared versus exclusive +ownership. Native KVM passes 94 cases with eleven skips; clean TCG KASAN passes +88 with seventeen skips and all existing EL0 fixtures. Feature-off arm64/x86 +madvise, mprotect and mremap objects compiled. + +The late swap suite passes ten cases on KVM and nine filtered cases on TCG. +Both ABIs pass two 4 MiB EL0 write/pageout/readback rounds, including WILLNEED +and POPULATE_READ. VmSwap and pswpout establish swap activity; mincore can +still report warm swap-cache data resident. The separate late disk test +explicitly evicts swap cache. These are distinct coverage claims. + +This validation exposed an independent QEMU S1PIE dirty-tracking defect: +unmodified base and feature-disabled kernels also lost second-round writes. +The private upstream v11.1.1 build reproduced stale data in 238/256 native +pages. A local target/arm/ptw.c correction prevents a read/fetch TLB fill from +caching write permission for a clean PIE translation. The same base kernel +passes with hardware dirty updates both enabled and disabled; the corrected +emulator also passes the mixed KASAN suites. System QEMU is untouched. The +patch includes the analogous S2PIE handling, which is not runtime-qualified. +The diagnostic kernel trace has been removed. Exact artifacts are retained +under user4k-results/swap-dirty-investigation and cold-pageout-v1. + +There is still measured 4K swap amplification: after read swap-in, rewriting +the range causes 1024 native swap writes for 4 MiB instead of roughly 256. +Slot exclusivity is not yet preserved through swap PTEs, forcing avoidable +COW. This remains unfinished work, not a finished performance result. The +previous COLD/PAGEOUT exclusion is superseded; other advice exclusions remain. + +Preserving ownership through swap +--------------------------------- + +Unmapping an exclusive anonymous slot now transfers its ownership marker to +the swap PTE. Fork clears it in both mms. Swap-in and swapoff restore slot +exclusivity under the folio and PTE locks; stable-write backends still force +COW if writeback has not completed. Native swap counts aggregate quarters +and are deliberately not used as a proof of slot exclusivity. + +Three new late tests verify cached swap-in, disk swap-in and swapoff. Each +checks the marker, bytes, MTE tags, contiguous quarter backing and physical +identity after subsequent write faults. Existing fork cases now explicitly +check that both swap PTE markers clear. KVM passes thirteen late cases, +TCG KASAN twelve filtered cases; both pass native/4K EL0 pageout readback. +The prior second-round 1024-write amplification disappeared (observed 228 +and 246 writes at the immediate KVM/TCG counter snapshots). Those counters +can precede asynchronous completion; the physical-identity kernel checks +prove packing directly. This does not establish general workload performance +or optimize ownership recovery after a fork sharer exits. Stable-writeback +concurrency still needs a dedicated stress test. + +General software-PAN KVM and patched-QEMU KASAN pass their existing suites +and all EL0 fixtures with no warnings or sanitizer reports. Feature-disabled +arm64 memory/rmap/swapfile objects compile. Checkpatch reports no errors; +its two warnings concern the native-style swap-flag data_race and debug-only +writeback assertion. Frozen artifacts are under swap-exclusivity-v1. + +Pagemap, scan and clear_refs +---------------------------- + +Pagemap read offsets and PAGEMAP_SCAN alignment, ranges and max_pages now use +the target mm's immutable granule, including cross-ABI readers. Pagemap PFNs +and swap offsets are expressed in that same target granule; a 4K swap offset +includes its quarter within the native swap entry. Convert PFNs to physical +byte addresses before comparing different ABIs, or to native PFNs before +using global kpage interfaces. AT_PAGESZ in the target auxv identifies its +user granule. The PFN capability gate remains unchanged. Anonymous exclusivity +uses the exact slot mapcount; file exclusivity remains conservative at native +folio granularity until per-quarter file mapping counts are integrated. + +clear_refs and pagemap/scan PTE and hole walks use target-sized increments. +Six EL0 checks per ABI cross both 2 MiB and 32 MiB table boundaries, verify +present/hole/guard/PROT_NONE entries, batch versus single reads, scan limits, +clear_refs mapping/data preservation, cross-ABI inspection, mixed-size memfd +physical identity and unprivileged PFN hiding. The first expanded fixture +incorrectly allowed a later alias to fill its intended hole and failed to +restore its own dumpability after setuid; corrected fixtures pass in KVM +software-PAN and patched-QEMU KASAN. These are fixture corrections, not kernel +security changes. Soft-dirty and asynchronous userfaultfd write protection are +not runtime-qualified by these configurations. + +Both VMs also pass the late swap suites and all-byte pageout readback with +new pagemap checks proving swapped-entry presence and quarter offsets. +Feature-disabled arm64 and x86 proc objects compile. Frozen source and logs +are under proc-walkers-v1. smaps/PSS, NUMA statistics and their shared-memory +accounting remain a separate unfinished gate. + +Resident PSS and quarter sharing +-------------------------------- + +File mapping metadata now records four alternative mapping counts per native +page. Native rmaps do not update these atomics; the native count is recovered +from the existing aggregate count when collecting statistics. Fork, insertion, +COW, zap (including deferred aggregate-rmap removal), reclaim and migration +maintain the quarter counts. A new test verifies mixed aliases, fork/COW and +exit; migration success/rollback and reclaim cases verify counter restoration +and zero counts after unmap. The metadata adds sixteen bytes per native page +when the experimental feature is configured. Native-path latency and broad +workload costs are unmeasured. The feature now requires per-page mapcounts; +NO_PAGE_MAPCOUNT is not yet a supported experimental configuration. + +Resident smaps/PSS/private/shared totals and pagemap exclusivity use the exact +physical quarter counts for alternative leaves. Native file mappings split +their PSS contribution by physical quarter when sharing with alternative mms. +Anonymous leaves use their own slot mapcounts. NUMA PTE walks use user-granule +counts and the same sharing snapshots. Swapped shmem range accounting clips +native swap-cache entries to the requested byte range, including multi-index +xarray entries; a synthetic-xarray kernel test checks boundary clipping. + +Seven EL0 checks per ABI cover dense anonymous RSS/PSS, fork, one-leaf COW, +child exit, NUMA page units and mixed-granule file aliases. Sharing one 4K +quarter of a native 16K mapping reports 14 KiB PSS to the native mapping and +2 KiB to the small mapping; after exit, the surviving mapping recovers full +private accounting. KVM with NUMA and software PAN passes all fourteen new +checks and 96 kernel cases (fourteen skips). Non-NUMA KASAN passes twelve +new checks with two explicit NUMA skips, plus 90 kernel cases (twenty skips). +Both retain passing prior EL0 suites. A separate object build covers NUMA +with PROC_PAGE_MONITOR and DEBUG_VM disabled; feature-off arm64 and x86 +proc/shmem objects compile. The first attempt to disable PROC_PAGE_MONITOR +was overridden until EXPERT was enabled; only the nomonitor log proves that +configuration. Checkpatch's remaining warnings concern fixture formatting +and userspace sscanf style, not build failures. + +This establishes resident accounting only. Anonymous SwapPss still divides +by native swap-entry reference counts, which combine distinct quarters; it +requires per-quarter swap reference tracking. Concurrent reporting races, +real swapped-shmem pressure and broader NUMA/migration scenarios remain to be +tested. Frozen evidence is under resident-accounting-v1. + +Per-quarter anonymous swap references +------------------------------------- + +Swap clusters now allocate quarter-reference metadata on first alternative +PTE use. Native and quarter counts change under the same existing cluster +lock. Fork, swap-in, zap, swapoff and unmap rollback maintain both counts; +empty-cluster table release checks and frees the quarter metadata. Allocation +failure leaves the PTE and native reference count unchanged. The metadata +cost is four unsigned counters per native swap entry in an affected cluster, +plus one pointer per cluster in feature-enabled builds. Allocation-failure +injection and broad memory-pressure cost remain untested. + +SwapPss now uses the selected quarter's count. The late swap regression checks +counts of 1 before fork, 2 after fork, independent decrements on swap-in/COW, +and zero after exit. EL0 tests verify full SwapPss, equal fork sharing, +individual leaf changes after child COW and full recovery after child exit, +for both ABIs. All bytes are still checked after each 4 MiB pageout round. +KVM passes thirteen late cases, KASAN twelve filtered cases, plus both EL0 +accounting sequences and the general suites. Feature-off arm64/x86 objects +compile; an explicit SWAP=n, NUMA=y, DEBUG_VM=n object build verifies stubs. + +An initial KASAN fixture failed because a native PAGEOUT pass left eight pages +resident, with zero bad data bytes. PAGEOUT can skip busy folios; the revised +fixture checks pagemap and retries advice within a fixed bound to establish +its all-swapped precondition before asserting complete-range SwapPss. This +changes test setup only. Earlier failure output is retained separately from +the passing complete-fixture logs. The previous anonymous SwapPss gap is +superseded. Recovery of slot exclusivity after a fork sharer exits remains +unfinished; the current fast path still depends on the carried exclusive bit. +Artifacts are under swap-quarter-counts-v1. + +Ownership recovery after a fork sharer exits +------------------------------------------- + +Slot reuse now proves a sole mapping and the expected typed reference count +under the owner lock, with the folio and PTE locked. Swap-in and swapoff also +require a sole reference to that exact swap quarter. A resident write fault +requires no remaining swapped alias. Stable-writeback restrictions remain in +place. Extra GET references or pins prevent reuse, preserving their original +data while the writer receives a copy. Still-shared resident slots skip the +new folio-lock attempt. + +New kernel cases check cached and evicted swap-in, swapoff, ordinary resident +COW, physical backing identity and retained-GET isolation after child exit. +The initial new swap reuse tests failed before this fix; their output and +Image are retained. Real EL0 tests verify unchanged physical backing across +post-exit resident writes for both ABIs, and a third 4 MiB rewrite/pageout +round verifies all bytes after the swap-sharing exercise. KVM passes seventeen +late cases; patched-QEMU KASAN passes sixteen applicable late cases and the +general suite (91 passing kernel cases, 24 explicit skips). Non-NUMA proc +fixtures skip only their NUMA check. The final SWAP=n object also compiles. +KVM's general run predates only the equivalent mapcount early-out that avoids +locking still-shared slots; its final late run uses the final Image. + +This supersedes the post-fork ownership-recovery gap above. It is correctness +and physical-reuse evidence, not a broad performance measurement. Stable +writeback under pressure, allocation-failure injection and hardware M5 testing +remain open. Frozen artifacts are under cow-reuse-v1. + +ELF core dumping +---------------- + +Alternative core reads hold typed GET references to individual user leaves, +including swapped leaves and file-cache quarters. The existing native path +still uses native page references. FOLL_DUMP skips absent anonymous tables +and zero PTEs, retaining sparse output without faulting in zero-filled memory. +The dump emitter and machine-check copy helper accept the selected native-page +byte range. ELF PT_LOAD alignment, NT_FILE page units and offsets, header-only +lengths and minimum dump-size admission follow the dumped process granule. +Header-only filtering also requires a zero subpage file offset. An optional +binfmt minimum-size callback preserves other formats' existing fixed limits. + +The dedicated disposable-VM fixture parses actual core files for both ABIs. +Its 32 checks cover protected anonymous bytes, unfaulted file quarters, +nonzero native file indices, swapped data (pagemap verifies every leaf is +swapped before the crash), vDSO ELF contents, sparse zeros and allocated size, +DONTDUMP, executable/non-executable ELF header filtering, AUXV and NT_FILE, +and one-page/below-one-page RLIMIT_CORE admission. The earlier kernel failed +eight of the first nine 4K checks; the first implementation then exposed an +unconverted file offset. Both final KVM/software-PAN and patched-QEMU KASAN +runs pass all 32 checks. The full prior EL0 regression also passes both VMs. + +The initial KASAN size-limit fixture asserted logical file size <= RLIMIT_CORE. +Core sparse seeks are not charged to the emitted-byte budget; the corrected +fixture checks admission and that no mapped-memory data was emitted with a +one-page budget. The earlier failing run lacks exact size diagnostics, so its +precise size difference is unknown. Final runs log actual size and first-data +offset. Separately, a general KVM regression caught a rare pre-existing stack +test boundary: arch_align_stack subtracts less than PAGE_SIZE but then rounds +down to 16 bytes, allowing exactly PAGE_SIZE displacement. The assertion now +includes that endpoint; stack code is unchanged. + +Feature-disabled arm64/x86 objects compile. RISC-V no-MMU coredump/FDPIC objects +compile with the existing I/O-helper warnings. An initial attempt explicitly +requested the disabled binfmt_elf.o target in that no-MMU configuration and +failed on MMU-only definitions; it is not a supported configuration or passing +build. Pipe/socket core collectors, credentials, interrupted writes, hardware +machine-check injection and application debugger interoperability still need +qualification. Frozen source, Images and logs are under core-dump-v1. + +Mixed-ABI context and ASID rollover stress +----------------------------------------- + +A disposable-VM PID-1 fixture launches 70,000 alternating native/alternative +ELF processes while persistent workers of both ABIs migrate across four CPUs. +All use the same virtual addresses with distinct data, writable/read-only/guard +leaves, signal recovery and repeatedly rewritten executable code. Patterns mix +high serial bits so allocator rollover does not merely repeat the same expected +bytes. The native parent also checks its own mapping after each child exits. + +A read-only KUnit-only parameter exposes the actual ASID generation; assertions +require at least two measured rollovers. KVM presents 256 ASID entries and +passes 70,000 launches with both hardware and software PAN, crossing 556 +rollovers per run. Persistent workers each execute hundreds of thousands of +migration/JIT/data cycles. Final software-PAN rerun uses the final source; +the hardware-PAN run predates only the explicit capacity-test knob, which is +absent from its command line and defaults off. + +QEMU max presents 65,536 entries. Its initial full KASAN stress was stopped +after 2,048 reported launches without rollover; it is incomplete coverage. +An explicit arm64.user4k_test_asid_bits=8 boot argument, compiled only into +the KUnit configuration, restricts allocator capacity to 256 without changing +hardware ASID width or normal boot behavior. The 2,048-launch KASAN run then +passes sixteen measured rollovers and more than 33,000 cycles in each persistent +worker, with no sanitizer report. This deliberately increases test rollover +frequency; its runtime cost is unmeasured and it is not a production setting. +Natural 16-bit-pool rollover remains unqualified. Frozen artifacts are under +context-stress-v1. These checks do not establish all concurrency or application +workloads as supported. + +Memory locking in user granules +------------------------------- + +mlock, mlock2, munlock and mlockall use target-page alignment and locked_vm +units; RLIMIT_MEMLOCK remains a byte limit. Accounting uses a long count so +large deferred-lock ranges cannot wrap a signed 32-bit leaf count. Physical +NR_MLOCK remains native-folio accounting: locking a quarter keeps its 16K +backing resident, while VmLck reports the requested rounded user-page bytes. + +Fresh anonymous mappings use a distinct rmap operation that defers initial +locking to LRU insertion, followed by one contribution per additional locked +leaf. Existing mappings retain the ordinary mlock path. This avoids duplicate +fresh-folio accounting and preserves neighboring locks on partial unmap. +File aliases use each mapped user leaf's contribution alongside native alias +counts. PTE walks and fully-mapped-folio checks use target geometry. + +Migration removal and restoration now drain pending mlock operations at the +same points as the native path. The new locked migration cases initially +failed to migrate: pending unlock references prevented ownership transfer. +The drain fix passes dense/sparse, fork-sharing and intentional rollback cases. +Native arm64/x86 mlock/rmap/memory/GUP/migration objects compile. + +Eight EL0 cases per ABI cover unaligned ranges, repeated/adjacent locks, +MLOCK_ONFAULT, MAP_LOCKED, fork noninheritance and COW, future/current mappings, +unprivileged limits and large deferred counts. Kernel cases verify a fresh +folio's four locked leaves decrement independently and mixed file aliases +retain the lock until the final mapping is removed. The initial kernel test +incorrectly inspected mlock_count after the last unlock, when its union storage +holds LRU links; it now checks the cleared lock flag. A prior fixture also +incorrectly expected PAGEOUT on a locked VMA to succeed; native behavior is +EINVAL. Both corrections preserve the actual locking assertions. + +KVM and KASAN pass these cases. A fourth swap round locks a fully swapped +4 MiB range and checks every byte for both ABIs. An intermittent KASAN fixture +failure left eight native pages resident after repeated PAGEOUT. Captured +kpageflags showed mapped-once swap-cache pages without KPF_LRU. PAGEOUT drains +only the current CPU's pending LRU additions. A fixture-only bounded sweep of +allowed CPUs, restoring affinity afterward, immediately established full swap +coverage in captured runs. Six repeat runs pass, three using that sweep, +including one alternative-mm run. The production reclaim path was not changed +and the all-swapped accounting assertions remain intact. The earlier failure +is retained; no data mismatch occurred in it. + +A later session made /home/dj/Work read-only and removed KVM access. Ongoing +writes therefore use /tmp/aurora-user4k-source, an overlay with private copies +of modified files and read-only links to the original source. New evidence +is under work/user4k-continuation in the workspace. The original worktree and +prior frozen evidence remain intact. The overlay has built and booted under +KASAN. Further pressure, file-THP, NUMA and application workload qualification +remain necessary; memory locking does not complete the process ABI. + +Lazy discard of individual user leaves +------------------------------------- + +MADV_FREE now records a per-slot lazy-discard hint under folio/PTE protection. +It requires a sole mapping and no extra typed reference at advice time, so an +existing fork sharer or pin cannot have its data discarded by another mapping's +new hint. The advice clears the selected PTE's young/dirty state; whole backing +moves to the normal lazyfree LRU only when every live slot has a hint. Partial +hints retain their neighbors' dirty and swap-backed state. Reclaim drops only +clean hinted leaves without additional typed references or pins. Writes through +PTEs are detected by the dirty bit; writable/dirty fragment operations cancel +the hint. A removed quarter is invalidated before its reference is released. +Fully hinted native backing is reclaimable without swap writes. Swapped advice +drops the exact quarter's swap reference and accounting, not neighboring entries. + +Migration snapshots each slot's hint when its PTE is removed, while the source +is locked. A failed migration can restore the original hint; fresh reservations +clear stale hint state. smaps LazyFree counts clean hinted user leaves, including +partial native backing, and stops counting a leaf after an EL0 store dirties it. + +Ten kernel scenarios cover partial/full discard, dirty cancellation, GET held +before/after advice, writable GET, dirty reference release, PIN retention and +partial/full migration. Eight actual EL0 checks per ABI cover retained bytes +before reclaim, partial discard and neighboring data, LazyFree reporting, +redirtying, full reclaim with no extra pswpout, selected swapped-entry discard +and fork isolation. The patched-QEMU KASAN run passes these plus the existing +execution and late-swap suites (including the new mlock/readback round). +Feature-disabled arm64/x86 and alternative SWAP=n objects compile. + +Initial native-control failures reflected best-effort advice on pending per-CPU +LRU additions. The fixture now visits each allowed CPU for advice and restores +its affinity. Initial kernel partial-discard tests stopped at an unhinted first +leaf without swap backing; they now place the selected hint first to exercise +that path directly. The actual EL0 fixture covers arbitrary quarters with swap +available. The initial synthetic redirty test also used an unsafe direct PTE +update and triggered the arm64 access-flag warning; it now uses the fault path. +Those failing logs are retained. These tests establish the covered transitions, +not memory-pressure races, broad workloads or performance. Artifacts are under +work/user4k-continuation/lazyfree-v1. + +Threaded execution and futex recovery +------------------------------------ + +A static glibc pthread fixture passes both page ABIs in KASAN TCG. Four workers +perform 128 concurrent rounds of mapping/protection, lazy discard and redirty, +DONTNEED, allocator, signal/TLS and per-thread next-exec request checks while +moving between two CPUs. Mutex/condition/barrier synchronization uses real +userspace futex operations. The main thread verifies its exec request remains +independent and forks eight times while siblings are blocked; child allocator +and inherited state checks pass. + +Robust mutex owner-death recovery passes, including 128 repeated thread +creation/exit/join cycles. A priority-inheritance handoff is confirmed with the +waiter's /proc task syscall reporting FUTEX_LOCK_PI or FUTEX_LOCK_PI2 while the +owner holds the mutex; recovery after a PI mutex owner dies also passes. +Non-leader-thread exec succeeds for all four 16K/4K source/target combinations, +with two live sibling threads and the former leader removed by exec. Resulting +AT_PAGESZ and consumed next-exec requests are verified. These are targeted +concurrency tests, not all scheduling policies or application workloads. +Logs and reproducible initramfs are in work/user4k-continuation. + +Pipe and socket fragment lifetime +--------------------------------- + +Alternative vmsplice now acquires typed user fragments and coalesces contiguous +quarters into native-sized pipe buffers. A shared reference bundle follows tee, +partial consumption and pipe transfer. Gifting a quarter does not authorize +stealing its containing native page. The native vmsplice path is unchanged. + +SKB splice acquisition now retains anonymous quarters through a zerocopy +completion object in addition to the ordinary page references. An independent +SKB append copies shared ownership metadata before extending it. Kernel tests +cover pskb_copy, skb_clone, skb_split, full skb_copy, blocked quarter reuse while +references remain, and reuse after the final shared reference is released. +Socket-to-pipe transfer takes new typed references before releasing its native +page reference. The SKB completion uses atomic allocations; no transport is +forced to copy data by the acquisition helper. + +KASAN TCG passes 10 userspace splice checks in each ABI: unaligned and empty +vectors, native-capacity coalescing, tee and partial reads across replacement, +fork/COW with retained data, buffered-file splice, partial failure, Unix socket +retention across partial replacement, socket COW, socket-to-pipe-to-tee, and +128 unread-socket teardown/reuse cycles. The ownership KUnit test and the +existing execution fixture also pass. Feature-disabled arm64 and x86 networking, TLS and splice objects compile. +These results do not qualify transport performance. + +Unix, IPv4 and IPv6 socket families now admit alternative pipe buffers. IPv4 +TCP loopback passes the same userspace checks; the COW and partial-replacement +checks explicitly cork TCP and require SIOCOUTQNSD to report all bytes unsent +before changing source memory. IPv6 TCP and IPv4/IPv6 UDP also pass the +expanded userspace fixture. UDP is +explicitly corked with no receive data before source replacement or COW; its +receive splice uses the existing copy_splice_read fallback. That fallback was +initially mistaken for unsupported receive splice in the fixture; the incorrect +assertion was corrected after inspecting sock_splice_read. +Other socket families remain gated while their lifetime chains are audited. +Combining distinct zerocopy providers remains unfinished. The added SKB helper +rejects combining its ownership callback with a different zerocopy provider. +These exclusions are development gates, not a completed socket ABI. All of the +new code remains experimental, and no M5 hardware result is claimed. + +The expanded UDP control exposed an existing checksum accumulation error: +skb_splice_from_iter deferred updating skb->len until the full iterator batch, +but each fragment checksum used the unchanged skb->len as its byte position. +An odd-length first fragment therefore gave subsequent fragments the wrong +parity. A direct KUnit comparison against skb_checksum failed for both ordinary +and quarter-owned backing before the fix. Passing cumulative spliced bytes into +the checksum helper corrects that position. The direct test and all 104 EL0 +checks now pass, including multi-native-page UDP payloads in both ABIs. Initial +failure logs and the pre-fix Image are retained. This checksum fix also affects +the native networking path; it is not a quarter-specific workaround. + +The temporary source/build overlay was cleared by an environment refresh after +the first Unix tests. The source was reconstructed from saved bundles and the +recorded edits, rebuilt, and retested. The authoritative restored run is +``tcg-splice-restored.log``; reconstructed fixture source and a complete modified +source snapshot are saved under ``work/user4k-continuation``. Subsequent edits +are snapshotted there to survive temporary-directory resets. + +TLS and bounded fragment ownership +---------------------------------- + +Typed reference sets now retain unique anonymous quarters across sk_msg clones, +transfers and TLS plaintext records. Mutating a shared set copies ownership +metadata; pruning releases quarters no longer represented by an SG ring or SKB. +KUnit exercises clone/coalesce, partial release, reuse, borrowed transfers and +1024 ring-wrap cycles. A separate SKB trim/append test failed against the previous +helper (nine references instead of eight), then passed with pruning. Its initial +fixture accidentally allowed skb_condense to copy the payload; the corrected +negative control associates the SKB with a socket and reproduces the retention. + +TLS receive uses typed writable pins and physical offsets for alternative user +iterators. Synchronous completion releases those pins directly. Asynchronous +completion queues them before signalling completion; the existing process-context +wait drains the queue and dirties/unpins memory outside the crypto callback. +The native path keeps its existing ownership. Software TLS transmit uses typed +sk_msg ownership; device transmit records also retain typed references, but no +hardware TLS offload test has been performed. BPF redirection ownership paths +compile; actual BPF redirection is not runtime-qualified here. + +The final normal and forced-asynchronous KASAN TCG runs each pass 148 EL0 checks +across native and alternative Unix/TCP/UDP IPv4/IPv6 workers. TLS 1.2 and 1.3 tests +include pending plaintext across replacement and fork/COW, multiple records, +shared-file receive with boundary guards, fragmented recvmsg and repeated pending +record teardown. The default-off KUnit-only tls.user4k_test_async control selects +a cryptd wrapper around a concrete synchronous cipher driver. The forced run +records 18 deferred pin releases and 18 completed releases; the normal run records +zero. This exercises software asynchronous completion, not hardware offload. + +Evidence is in tcg-skb-prune-fixed.log and tcg-skb-prune-fixed-async.log with +Image-skb-prune-fixed. All 19 ownership KUnit cases pass, including the new pruning +case; other suites retain their documented skips. These results do not establish +a complete ABI, hardware qualification or application performance. + +Direct block and iomap I/O +------------------------- + +Alternative user iterators now acquire typed pins into bio-owned fragment +metadata. Bio vectors carry exact physical offsets and coalesce adjacent quarters +within a folio; the separate pin list preserves their individual lifetimes. +Alignment trimming releases discarded pins and restores the corresponding +iterator bytes. The existing dirty-page completion mechanism handles deferred +file dirtying. Split and cloned bios borrow vectors from the original submitter, +which owns the pins until completion, as for native bio pins. + +The raw-device control passed ten checks for 16K and failed all ten for 4K before +integration. The integrated kernel passes 40 checks across raw virtio block and +ext4 iomap direct I/O in both ABIs: fragmented/empty vectors, exact offsets, +two-megabyte multi-bio transfers, partial faults and recovery, alignment trimming, +shared-file receive guards/dirtying, asynchronous AIO, writable-pin COW and 128 +reuse/failure cycles. Ext4 reports EFAULT after the partial-fault case while the +raw-device path returns its completed byte count; both ABIs match their respective +backend. The fixture originally assumed raw-device semantics for ext4 and was +corrected after inspecting iomap error propagation and the native control. + +Two new kernel tests verify coalescing with distinct pin ownership, lifetime +after originating-mm teardown, borrowed clone vectors and 1024 fully trimmed +failure cycles with exact reference and pin counts. These pass under KASAN, as +do the pre-existing enabled suites. Feature-disabled arm64 and x86 block/iomap +objects compile. The 148-check forced-asynchronous TLS regression also passes. +Evidence: tcg-dio-ext4-contract.log, tcg-dio-tls-regression.log and Image-dio-final. + +The ext4 fixture selects disposable guest disks by serial. Initial setup failures +from reversed enumeration and a launch before a disk copy completed are retained; +they are not kernel ABI failures. Registered buffers, long-term pins, bounce I/O +and legacy direct-io consumers remain to be integrated and qualified. No storage +performance or hardware DMA qualification is claimed by these TCG results. + +Native-page arrays and io_uring mappings +-------------------------------------- + +The native-page array mapper now expands backing pages into the target mm's +hardware leaves while retaining batches of up to eight PTE updates per lock. +vm_insert_pages_range accepts a checked byte offset and length, allowing a 4K +mapping of part of a 16K backing page. The existing vm_insert_pages API keeps +native-page counts; vm_map_pages applies both the native file offset and its +subpage remainder. Overflow and alignment checks precede insertion. As before, +an insertion failure can leave a mapped prefix that the caller must tear down. + +io_uring uses the range interface for alternative ring mappings and keeps its +native mapping path. Ring selector offsets with a sub-native remainder are +rejected. The first control failed all seven 4K checks while native passed; +the final fixture passes eight checks in each ABI, including minimum mappings, +selector validation, small/large ring wrap, 8193 submissions across native pages, +unregistered direct I/O, forced worker-thread I/O, multi-bio transfers and enlarged +SQE/CQE layouts. Registered buffers remain the next integration step. + +The added KUnit test covers reversed backing-page arrays, explicit physical +subpage offsets, a PMD boundary, invalid ranges, duplicate insertion, the legacy +array API, vm_map_pages offsets, accounting and complete reference release. +It and the enabled suites pass under KASAN. Feature-disabled arm64/x86 mapping +objects compile. The swap regression completes both 16-case runs and native/4K +EL0 pageout/readback with zero bad bytes. Evidence: Image-insert-range-final, +tcg-uring-range-final.log and tcg-uring-range-swap.log. + +Long-term fragment pins and registered buffers +--------------------------------------------- + +Typed long-term pins now use the native long-term GUP migration checks. The +migration collector can traverse typed records, release their exact pins before +migration, and retry acquisition afterwards. Fault allocations run under the +existing memalloc_pin context. Native page/folio callers keep their representation. +A movablecore=50% KASAN run confirms actual migration of existing movable backing, +all four quarters' bytes and pin counts, and rejection of FOLL_LONGTERM on GETs. + +io_uring registered buffers retain typed fragments until resource release. The +registration accounts unique native folios, preserving physical memory units; +each vector retains its own physical offset. Fixed-buffer import and vectored +import index logical 4K leaves independently of physical quarter positions. + +The expanded fixture passes 19 EL0 checks in each ABI, including first/late buffer +slices, fixed vectored and multi-bio I/O, forced worker completion, source and +destination use after unmap/replacement, multi-buffer registration rollback, +64 registration/reuse cycles with balanced VmPin, shared-file mappings with a +different physical offset, and resource replacement through BUFFERS_UPDATE. +Evidence: Image-uring-register-first, tcg-uring-register-first.log and +tcg-uring-register-file.log. The full first run passes enabled KUnit suites; +the expanded file run repeats the five fragment tests. Feature-disabled arm64 +and x86 GUP/registration objects compile. This does not qualify every io_uring +operation, user-provided ring memory, device pins, hardware DMA or performance. + +GEM shmem and PRIME mapping +-------------------------- + +The GEM shmem path used by Asahi and VGEM now supplies an explicit physical +quarter offset to PFN insertion. Write-notify upgrades compare complete physical +addresses. Buffer indexing retains object-relative positions after VMA splits; +DMA-BUF bounds use bytes including the subpage remainder. GEM fake offsets still +identify exact object starts. Kernel physical pages, GEM backing and device page +tables are not resized by these CPU mapping changes. + +The VGEM baseline reproduced three native failures (cold faults after protection +splits, partial unmap and a partial PRIME mapping) and six 4K failures including +quarter aliasing. The fix passes 12 EL0 checks in each ABI, adding cold remap, +fork, bounds, descriptor lifetime and PRIME sharing across native/alternative +exec. A KUnit case validates reversed physical quarters across a PMD boundary, +write-notify permissions, invalid offsets and special-PTE ownership. All enabled +KUnit suites pass; feature-disabled arm64/x86 GEM, DMA-BUF and memory objects +compile. Evidence: Image-gem-pfn-kunit, tcg-gem-baseline.log and +tcg-gem-pfn-kunit.log. VGEM tests the common mapping helper, not Apple GPU +rendering, device DMA or every imported-buffer exporter. Legacy PFN walkers and +contiguous PFN remap paths still require a granule-aware audit. + +PowerPC's pgtable-frag.c and ptdesc::pt_frag_refcount provide an existing +example of page-table fragment lifetime management. They are useful design +references, not drop-in implementations of arm64 mixed-granule MM. + +References +========== + +* Arm, Learn the Architecture: Memory Management, section 7, Translation + granule (101811_0100_00). +* https://github.com/apple-oss-distributions/xnu/blob/xnu-12377.121.6/tests/x18_toggle_4k.plist +* Arm Instruction Set Reference Guide: + https://documentation-service.arm.com/static/6245c734b059dc5ff9a8bdab +* Inspected kernel paths: arch/arm64/mm/context.c, + arch/arm64/include/asm/{mmu_context,pgtable-hwdef,tlbflush}.h, + fs/binfmt_elf.c, mm/{mmap,memory,mprotect,mremap,madvise,gup,rmap,shmem}.c, + kernel/futex/core.c. + +AArch32 final-ELF selection and native fallback +--------------------------------------------- + +The final ELF class selects the granule before begin_new_exec(). When staged +arguments use another granule, bprm_set_page_shift() allocates a new mm and +copies the temporary stack bytes without changing their virtual addresses. +It rechecks the target per-string limit and keeps the old mm and temporary +RSS charge until the copy succeeds. An allocation/copy/length failure drops +only the replacement. The compat context flag preserves the selected granule. +AArch32 mmap2 offsets remain 4K units and are expanded to full-width byte +offsets before alternative-granule mapping; the native alignment path remains. + +The freestanding compat probes are linked separately with 4K and 64K maximum +segment alignment. Under KASAN TCG, automatic-4K runs pass auxv, a 90KB argument, +anonymous leaves, mprotect, misalignment rejection, mmap2 offsets including +above 4GB, fork/COW, signal return, kuser cmpxchg and compat vDSO time access. +The launcher covers scripts, a pending 4K request, ELF32-to-ELF64 native-default +exec, and 64 oversized-string failures preserving the old mm and request. + +A second boot with id_aa64mmfr0.tgran4=f removes 4K from the sanitized capability +model. This override exists only in USER4K_KUNIT_TEST non-4K builds and can only +remove capability. The launcher verifies a 4K request returns EOPNOTSUPP. The +64K-aligned ELF32 and script then execute with AT_PAGESZ=16384; 4K-only mmap2 +offsets return EINVAL. This is a capability-model VM test, not A9 hardware +qualification. Pure policy tests also cover native shifts 12, 14 and 16. + +Ryan Roberts RFC patch 44 aligns kernel/vDSO sections using PAGE_SIZE_MAX; +patch 57 sets the boot-variable kernel Image page-size flag to zero, rather +than encoding a fixed granule. That flag describes the kernel image, not a +new userspace ELF opt-in. The current fixed-16K kernel retains its Image flag. +Source: https://lists.infradead.org/pipermail/linux-arm-kernel/2024-October/970037.html +and https://lkml.rescloud.iu.edu/2410.2/05663.html . + +The expanded compat fixture also passes a real PT_INTERP mapping using a +freestanding fixed-address ELF32 interpreter, ELF32-to-ELF32 argument restaging, +and an AArch32 request for a 4K AArch64 child. This is loader coverage, not an +armhf libc/dynamic-linker qualification. The current full fixture records 72 +ELF32 checks plus 10 launcher checks in automatic-4K mode, and 48 ELF32 checks +plus seven launcher checks with 4K capability masked out. The preceding full +KUnit/GEM regression had 84 main-suite passes, seven main-suite skips, two TLB +passes, 19 ownership passes, and 24 GEM/PRIME EL0 passes; the diskless swap +suite skipped all 17 cases. + +64K-native kernel with concurrent 4K userspace (2026-10-01) +---------------------------------------------------------- + +The experimental option now accepts a 64K native kernel with 48-bit VA/PA. +The compiled hierarchy accommodates the four-level 4K user tables, while +native 64K tables retain their three-level hierarchy. Runtime folding and +selected-mm table predicates keep native and alternative walks distinct. +TTBR1 and native TG0 retain 64K geometry. Lower 4K tables pack up to sixteen +fragments per native owner. Swap PTEs retain all four slot bits without +reducing the existing swap type/offset fields. Proc proportional accounting +uses a native-sized slot array, and fault-around eligibility compares its +byte limit with the process granule. + +KASAN TCG results include 84 main-suite passes (seven skips), two TLB passes, +19 ownership passes, and 16 disk-backed swap cases with no failures. The swap +checks use the exact aggregate counter, including its extended table, and +independent per-slot counts. Four 8MB EL0 swap/fork/COW cycles per ABI verify +actual nonpresent swap PTEs, restored patterns and child-write isolation. +Native 64K and alternative 4K thread/futex tests and all four nonleader-exec +transitions pass. A mixed memfd test maps the last physical 4K slot: the native +mapping's PSS changes from 64KB to 62KB and back, while its 4K peer sees 2KB. +The same proc fixture passes with a 16K native page (16KB to 14KB and back). + +AArch32 probes pass automatic 4K selection on the 64K build. Masking 4K +capability with the KUnit-only override still permits the 64K-aligned ELF32, +script and PT_INTERP cases at AT_PAGESZ=65536 with native mmap2 restrictions. +This explicitly models AArch32 availability independently from 4K availability; +it does not establish behavior of any specific Apple SoC. + +The generic DEBUG_VM_PGTABLE boot fixture reports an order-13 (512MB) +allocation failure on this 64K configuration, including in a 2GB guest. +The mixed-granule suites run afterward, but their success does not establish +that this separate generic fixture passed. No workaround was made the default. +This remains a test limitation to investigate. Results do not qualify Apple +hardware, larger-than-native leaves, 16K-on-64K userspace, or application-wide +compatibility/performance. + +ASLR granularity and automatic THP eligibility (2026-10-01) +--------------------------------------------------------- + +Stack and mmap masks now retain their configured byte window while exposing +the process granule's additional low bits. Brk randomization and mmap/stack +alignment use that granule. Native masks and byte ranges remain unchanged. +The ASLR fixture sets randomize_va_space=2 in a disposable guest and samples +64 independent execs per ABI. On 16K and 64K builds, native AArch64, alternative +4K AArch64, and automatic-4K AArch32 show variation in stack/mmap/brk addresses; +the alternative ABIs also vary below the native alignment. ADDR_NO_RANDOMIZE +produces stable layouts. The preceding baseline lacked the extra low-bit +variation for 4K mmap/brk. Pure checks verify that the mask's byte span is +unchanged. These checks do not measure cryptographic randomness quality. + +The broader 16K ASLR run exposed a native-THP eligibility gap: the explicit +MADV_HUGEPAGE rejection did not prevent automatic THP selection in an alternative +mm. A large-mapping test encountered an anonymous PTE without slot ownership +and a nonzero RSS counter at teardown. Shared THP eligibility now rejects +alternative geometry, including advice/forced-collapse paths, while native +eligibility stays intact. This enforces the already documented unsupported +THP restriction; it does not implement alternative-granule THP. Full KUnit and +ASLR reruns with transparent_hugepage=always pass on both builds. Native THP +support remains enabled. Feature-off arm64 and x86 core objects also compile; +the x86 THP object check requires enabling THP in its test configuration. + +PFN ranges and bounded iomem access (2026-10-01) +---------------------------------------------- + +remap_pfn_range() now advances physical byte addresses at the selected user +leaf size and uses selected-mm table boundaries. Its input PFN still denotes +a native physical page. Tests map a native page plus one user leaf across +PMD/PUD/root boundaries and verify every physical offset, neighbouring holes, +special PTEs and unchanged native-page references on 16K and 64K kernels. + +follow_pfnmap_start() now walks the selected geometry and returns the requested +physical byte's offset alongside its native PFN. A caller must explicitly opt +into this byte-granular result; legacy native-page consumers receive +EOPNOTSUPP on alternative mms until their ownership/range contracts are ported. +In particular, this does not claim alternative-mm VFIO or KVM support. +generic_access_phys() uses that offset, revalidates it, and bounds each transfer +at the user leaf and VMA end. Following leaves receive separate checks. + +A QEMU-virt-only fixture reads the PL011's read-only identification registers +through a 4K VMA whose virtual/native offset deliberately differs from its +physical offset. Direct iomem access and access_remote_vm() return the correct +16 bytes, stop before the following unmapped leaf, and preserve the remaining +buffer. A write request against the read-only mapping fails before I/O. +No UART state registers are written. All 88 enabled main-suite cases pass on +both kernels, alongside two TLB and 19 ownership cases. This tests the generic +byte-access path, not Apple device DMA or GPU qualification. Diagnostic table +walks also use the mm's geometry and report its page size. + +The capability-masked ASLR fixture additionally passes on both builds: AArch32 +retains native 16K or 64K mappings and ADDR_NO_RANDOMIZE remains deterministic. + +CPU capability lifetime (2026-10-01) +---------------------------------- + +Alternative-4K availability is now a system CPU capability. Exec/prctl selection +uses the finalized capability rather than only reading the sanitized register. +The normal arm64 late-CPU verifier therefore refuses an incompatible CPU after +4K support has been advertised. If the system capability is absent, CPUs may +still be admitted and native-granule processes remain usable. This is a global +capability policy; a restricted eligible-CPU domain remains future work. + +A KUnit-only removal model, arm64.user4k_deny_late_cpu=1, makes CPU1 fail this +capability check during late verification. It never grants a capability, does +not modify a translation register, and is absent from non-KUnit builds. In a +two-CPU guest booted with maxcpus=1, tests create live mappings before requesting +CPU1 online. On both kernel granules, advertised-4K mode rejects CPU1 with EIO, +keeps it offline, and the worker and subsequent AArch32 exec remain valid on +CPU0. With id_aa64mmfr0.tgran4=f, the same modeled CPU is admitted; native +16K/64K worker mappings and AArch32 fallback then execute on CPU1. These are +capability-model tests, not a heterogeneous hardware qualification. + +Compatible-CPU admission also passes on both builds: an existing 4K worker +migrates to CPU1 and verifies its bytes there, followed by a successful +automatic-4K AArch32 exec. Final full regressions retain 88 main-suite passes +(seven skips), two TLB passes and 19 ownership passes on each kernel. The +KUnit object also builds with TRANSPARENT_HUGEPAGE disabled. + +Backing copy extents (2026-10-01) +------------------------------- + +The arm64 subpage copy/clear helpers now accept an aligned power-of-two extent +within a native page. Existing 4K callers retain their wrappers. MTE tag helpers +use the same extent and retain their multitag transfer path when its block fits. +Tests cover 4K/16K/64K extents where native backing permits, copies from tagged +and untagged sources, exact data/tag clearing, and untouched neighbours. This +is groundwork for larger ownership identities, not a new process ABI. + +Both builds pass 88 main, two TLB and 20 ownership cases; the extra ownership +case exercises these extents. Both also pass the 16 disk-backed swap cases and +actual EL0 swap/fork/COW rounds. The copy and ownership objects compile with MTE +and transparent hugepages disabled. Process selection remains native or 4K. + + +16K processes on a 64K kernel (2026-10-01) +----------------------------------------- + +The experimental exec selector accepts 16384 on a 64K kernel when the system +advertises the 16K user granule. Zero or the native size retains the native +AArch64 default. AArch32 still selects 4K when available and otherwise the native +size, even if an inherited pending exec request selected 16K. No new ELF flag is +introduced; binaries must meet the selected granule's segment alignment rules. + +The CPU capability is finalized across the system, with late CPU verification. +Removal-only KUnit models can mask ``id_aa64mmfr0.tgran16=0`` or deny a late CPU +via ``arm64.user16k_deny_late_cpu=1``. These models cannot grant hardware support. +Tests admit a compatible CPU with a live 16K worker, reject a modeled incompatible +CPU while the worker survives on CPU0, and admit the same modeled CPU when 16K +was never advertised. A separate run masks 4K while 16K remains available and +verifies AArch32 native fallback despite a pending 16K request. + +Anonymous and table allocations retain immutable ownership granules through +reference release and RCU reclamation. The 16K tables use 47/36/25/14 shifts and +two root entries for a 48-bit address space. File accounting distinguishes +actual leaf count from 4K byte coverage: one 16K mapping contributes one rmap +entry and covers four accounting slices. PSS weights each slice independently. + +The 16K userspace smoke fixture exercises anonymous mappings, fork/COW, +independent protection, DONTNEED, pthread stacks/TLS, vDSO, private file COW, +shared-file bytes and PSS with a native 64K process. It also runs with KPTI forced +on. The extended fixture executes the page-contract program in another 16K mm. +The swap fixture runs four 8 MiB disk-backed rounds per ABI (64K, 4K and 16K), +checks actual swapped PTEs and swap writes, verifies every restored byte, and +forks while mappings are swapped to check COW and parent preservation. + +The new fixtures are ``user16k-smoke-init``, ``cpu-granule16-init`` and the +extended ``granule-swap-init``. They are disposable-guest PID1 programs; the +smoke initramfs also needs ``/page-contract`` and the CPU fixture needs the ELF32 +``compat-aslr-probe`` installed as ``/compat-aslr``. The swap fixture requires a +guest-only swap device with a header matching the native kernel page size. + +These results do not qualify larger-than-native user granules, alternative +granules on a 4K kernel, legacy unported pin consumers, all huge-page interfaces, +Apple GPU DMA, or performance under Windows-game workloads. The option remains +experimental and disabled by default. The generic 64K DEBUG_VM_PGTABLE huge +allocation coverage gap described above remains separate and unresolved. + +Larger tables and exact reservation units (2026-10-01) +----------------------------------------------------- + +Internal 64K table geometry now works on a 16K kernel. The allocator uses a +compound native allocation for a larger hardware table, retaining its immutable +shift and owner through RCU reclamation. Interior PTE/PMD pointers resolve to +the compound head for descriptor and lock lookup. A single larger table has +one owner reference and never enters the cache of unissued smaller fragments. +The 64K geometry uses root/PMD/PTE shifts 42/29/16 and folds the PUD level. + +The software fixture allocates aligned 64K data backing, walks mappings across +all table boundaries and the top of the 48-bit address space, verifies table +bytes and native backing counts, and exercises generic partial/full teardown. +Separate ownership tests cover every native-page boundary inside a table, +contiguous-entry accessed/dirty aggregation, locks and delayed freeing. These +are internal fixtures: public 64K process selection on a 16K kernel remains +disabled. Coarse data faults, coherent file backing, reverse mapping and swap +are not implemented by this table work. + +Locked-VM accounting now uses ``MM_ACCOUNT_SHIFT`` units, 4K in mixed-granule +builds and the native unit otherwise. This supersedes the earlier milestone's +statement that ``locked_vm`` uses user granules. A native physical pin can be +smaller than a coarse user leaf; representing both in a common unit preserves +exact independent charge/refund pairs. VMA deltas, limits, proc ``VmLck`` and +the direct VFIO limit reader convert explicitly. Logical VMA/RSS counts and +native physical ``pinned_vm`` counts keep their existing units. + +Alternative commit reservations retain aggregate logical counts. Smaller +leaves round the aggregate to native commitment units; larger leaves multiply +with an overflow check. Rejected charges leave both counters unchanged. The +per-process overcommit reserve converts through bytes before dividing by native +pages, avoiding an unsigned shift underflow for larger user granules. + +Both native kernels pass the accounting KUnit cases. The 16K kernel additionally +passes internal 64K reservation/overflow/refund tests. The expanded mlock +fixture passes nine cases per enabled ABI, including locked grow/shrink/move, +partial unmap, fork, limits and counts exceeding signed 32-bit user leaves. +The freestanding ELF32 ``compat-mlock-probe`` passes six cases on each kernel +with automatic 4K selection and again with 4K capability masked, retaining +native-16K/native-64K restrictions. Its PID1 harness is ``compat-mlock-init``; +install the probe as ``/compat-mlock`` and pass ``--no4k`` only for a guest booted +with the removal-only ``id_aa64mmfr0.tgran4=f`` model. This does not test Apple A9 +hardware. Feature-disabled arm64 and x86 accounting objects compile, including +the x86 VFIO consumer; this is not a VFIO device runtime test. + +Internal coarse anonymous mappings and shared zero backing (2026-10-01) +--------------------------------------------------------------------- + +A 64K base userspace PTE on a 16K kernel now has internal fault, fork and COW +fixtures. Its mapping owns four native page references and reverse mappings, +while RSS, PTE iteration and TLB ranges count one logical page. Allocation and +COW copying cover the complete leaf; the arm64 ordered zero-allocation hook +retains MTE initialization. Sharing and restoring exclusivity cover every +constituent native page. An interior-page pin forces a complete private copy +on fork. The no-THP configuration selects MM_ID for large anonymous-folio +ownership and reuse. + +Ordinary zapping now keeps physical reference/rmap counts separate from logical +PTE counts. A two-leaf fixture removes one leaf, checks its surviving neighbor, +remaps it and removes both as a batch. Reference scanning and mprotect batching +use the target mm's geometry. The anonymous fault fixture checks zero-page COW, +direct write faults, fork sharing, child COW, every native page's contents, +parent reuse and exclusivity, RO/RW protection changes, and teardown accounting. +These checks pass with and without THP. + +The experimental arm64 configuration reserves a 64K-aligned shared zero region. +All covered native PFNs are recognized as zero backing, and MTE setup clears +tags across the whole region. Native and alternative read faults retain special +read-only zero PTEs with zero anonymous RSS. The test reads every byte through +copy_from_user for each hardware-supported granule, including an internal 64K +mm on the 16K kernel. Other configurations retain PAGE_SIZE zero backing. + +These are internal coarse-MM foundations, not public coarse-process support. +The public 64K-on-16K selector is still disabled. Coarse file-cache backing, +swap, reclaim/migration, GUP and device integrations remain unfinished. Native +4K kernels also still lack alternative process granules. + +MGLRU table geometry and boundary aging (2026-10-01) +------------------------------------------------- + +MGLRU's PTE/PMD/PUD traversal, cross-VMA scanning and look-around paths now use +per-mm table dimensions, boundaries and leaf batching. Existing native index +helpers are retained except that a runtime-folded PUD always has index zero. +This distinction matters on the native 64K configuration, whose compiled table +levels also accommodate smaller alternative granules. A stronger boundary test +caught an out-of-bounds read from using the compile-time PUD index; the runtime +fold check fixed that observed failure. + +The disposable PID1 selftest ``mglru-granule-init`` is linked statically with +64K ELF segment alignment. It verifies that MGLRU and MM walking are enabled, +then executes each enabled ABI. Anonymous and shared memfd mappings straddle +eight address boundaries, from 2^21 through 2^47. After an explicit first-access +warm-up, four cycles check fully referenced mappings before aging, zero +referenced bytes in every region after forced aging, scan-counter evidence, +and every data byte. The fixture pins its allocation phase to CPU0 and fills +scratch backing to publish earlier per-CPU LRU additions. Its ``--concurrent`` +mode launches all enabled ABIs together: a native coordinator ages their mms at +pipe barriers, while each worker alternates CPU0/CPU1 between write/check rounds. +Both the sequential and concurrent modes pass on the 16K and 64K kernels. These +are bounded coverage tests, not performance benchmarks or sustained-pressure +qualification. + +The boundary tests pass for native16K/4K and native64K/4K/16K processes. The +MGLRU-enabled main/TLB/data suites pass, as do actual disk swap and fork/COW +regressions for those enabled ABIs. Mlock, lazy-free reclaim, thread/futex and +non-leader exec regressions also pass, as do AArch32 automatic-4K and masked-4K +native fallback memlock probes. These results do not establish exhaustive MGLRU +concurrency or sustained memory-pressure qualification. + +Global commit-accounting fixtures now drain prior deferred file releases before +taking baselines. A captured run refunded eight native pages during that drain; +the earlier unisolated test had observed an eight-page global drift while its +per-mm reservation stayed correct. Exact counter assertions remain in place. +The shared mlock fixture drains pending work on all CPUs before checking its +counts; the earlier one-count discrepancy has not been independently attributed +by a captured local-versus-global drain delta. No production mlock workaround +was added. + + +Memory-cgroup pressure and slot metadata (2026-10-01) +--------------------------------------------------- + +The primary 16K KASAN profile now enables cgroup v2 memory accounting. +``memcg-pressure-init`` runs native 16K and alternative 4K processes together, +each with a 32 MiB anonymous mapping, inside a 48 MiB memory limit. It checks +all bytes through permuted page access, four rounds of concurrent fork/COW, +and reclaim driven by the actual limit. A 256 MiB virtio swap device supplies +headroom for native backing, COW fragmentation and resident swap cache. +It requires swap/scan/steal/limit-hit evidence without OOM and checks that +anonymous and swap charges disappear after both workers exit. A native-only +control is available through ``--native-control``. + +The native-only control and one mixed run passed this pressure test before +metadata charging was corrected. This is not yet repeatable pressure +qualification: an earlier mixed run exhausted a 64 MiB swap device, and +another run with the larger device reached OOM with nearly all resident +pages under writeback despite free swap. The latter cause is unresolved; +passing later runs must not be presented as fixing it. No reclaim throttling +or reduced-concurrency workaround has been introduced. + +``memcg-accounting-init`` compares separate 32 MiB native/4K workloads under +cgroup accounting. Before the fix, the 4K workload reported only about 12 KiB +of unreclaimable slab. Slot pools and owners now request ``__GFP_ACCOUNT``; +replacement owner metadata preserves the original object's charge domain +when migration runs in another task. The test then reports about 1.6 MiB of +slab for the alternative workload and passes a conservative lower-bound +check. Allocation failures retain the existing rollback paths. Migration +metadata allocation stays nonblocking under the source/destination folio +locks. Cross-cgroup migration has not yet been separately exercised. + +The memcg-enabled internal coarse-zap fixture also now charges its synthetic +folio before partial unmap can enqueue it on a memcg deferred-split queue. +The original uncharged fixture produced a warning in teardown. With the +fixture corrected, the 16K profile passes 100 main, 2 TLB and 20 data KUnit +cases (unsupported cases remain explicitly skipped). + + +Droppable memory and distro runtimes (2026-10-01) +----------------------------------------------- + +An actual Python 3.14 run exposed a 4K startup SIGBUS: glibc allocated its +fast random state with MAP_DROPPABLE, which mmap accepted but the smaller- +granule fault path explicitly rejected. Droppable data owners now retain an +immutable policy, remain non-swap-backed, and discard dirty slots during +reclaim while respecting retained GET references and pins. Migration carries +the policy to the new owner; zero-page COW follows the same allocation rules. +Ordinary MADV_FREE redirty handling is unchanged. + +The new kernel regression covers dirty discard, retained GET/PIN protection, +migration, and zero-page COW followed by zero-filled refault. Both primary16K +and native64K profiles pass 101 main and 2 TLB cases, with 20/28 data cases. +Real installed Python and Bash run concurrently under native16K/4K and under +native64K/4K/16K. All pass page-size reporting, droppable fork wipe/discard and +glibc random state, file mmap offsets/resize/flush, fork COW and protection +faults, threaded compression/hashing, SQLite WAL/mapped reads, socket polling, +cross-ABI dynamic exec/memfd sharing, and Bash allocation/pipes. The initial +native64K sharing fixture addressed byte65536 beyond its65536-byte mapping; +its corrected128KiB mapping passes. This was a fixture range error. + +Memory-cgroup pressure now also passes on fresh swap disks and with an +explicit4MiB/s emulated write cap in both native-only and mixed16K/4K runs. +The cleanup assertion first waits for writeback and explicitly reclaims +unmapped swap cache before requiring zero anonymous/swap charges. Native +control demonstrated why an idle-delay-only assertion was invalid: clean +unmapped swap cache can persist after I/O completion. The earlier isolated +writeback-heavy OOM has not been reproduced or explained and remains open. + + +FEX and ELF/BSS mapping extent (2026-10-01) +----------------------------------------- + +FEX-2604 failed to reserve its guard page at sbrk(0): the mapping returned +EEXIST. ``vm_brk_flags()`` still rounded the ELF/BSS request to native PAGE_SIZE, +so a smaller-granule process could receive extra mapped bytes beyond its +reported break. It now uses ``mm_page_align(mm, request)``. The regression +checks the exact VMA extent and accounting for a one-granule-plus-one-byte +request, then reserves the next user page with MAP_FIXED_NOREPLACE. + +The installed FEX emulator now passes a freestanding x86-64 workload as an +AArch64 process with AT_PAGESZ=4096 on the 16K test kernel. A constructor checks +the actual emulator process granule. The x86 workload checks its 4K auxv, +independent 4K permission faults, fork/COW, mremap content and zero extension, +shared memfd mappings at a 4K file offset, RW/RX JIT code replacement, and +getrandom. The test needs a 2 GiB guest: its initial 768 MiB environment +refused FEX's 1 GiB allocation under ordinary heuristic overcommit. The +passing run changes no overcommit policy. This tests FEX directly in the +kernel guest's userspace; no nested VM is used by FEX. Windows games, GPU +access and Apple-hardware performance have not been tested. + + +FEX re-exec and inherited page-size policy (2026-10-01) +---------------------------------------------------- + +The native64K FEX run exposed a second geometry assumption: ``mm_find_pmd()`` +used native table offsets. Its use by ``vma_adjust_trans_huge()`` during a VMA +split reached an invalid PMD lock. The lookup now uses per-mm offsets at each +level. A sparse two-native-PMD VMA regression faults one smaller user leaf, +splits its protection, and checks its neighbour. The FEX memory contract now +passes on both native16K and native64K kernels. + +Extending that workload to exec another x86 process exposed a policy gap: +FEX's replacement AArch64 process reverted to native16K and jemalloc rejected +it. The private ``PR_AURORA_SET_DEFAULT_PAGE_SIZE`` and matching getter add +an explicit, per-thread inherited exec preference. Fork copies it and exec +preserves it. The existing one-shot request takes precedence and is consumed +only by successful exec. Selecting native through the one-shot setter while +an alternative preference exists installs a native override for that exec; +the inherited preference is still available afterward. The default setter +neither consumes nor replaces an existing one-shot request. Neither setter +changes the caller's current address space. Ordinary tasks still default to +native pages. + +``tools/aurora-userpages`` opts a process tree in by default. Its ``--once`` +option retains the original one-exec behaviour. ``native`` clears an inherited +alternative preference unless combined with ``--once``. All choices remain +subject to hardware capability checks. These private prctl numbers are not +upstream assignments. + +The focused guest tests pass repeated exec, failed-exec preservation, native +override and return to the inherited size, clearing the inherited preference, +invalid arguments, per-thread independence and non-leader-thread exec. They +cover 4K on native16K and 4K/16K on native64K. The extended actual FEX workload +passes its memory/protection/COW/JIT contract in both the original emulator +and its fork/exec replacement, with the AArch64 AT_PAGESZ checked as 4096. + +AArch32 selection continues to take precedence: 4K when supported, native +otherwise. Tests mask out 4K support on both native kernels and check the +remaining mapping-alignment restrictions. On native64K, an inherited16K +preference survives AArch64 -> native64K AArch32 -> 16K AArch64 execs without +forcing unsupported 4K pages or making the AArch32 process use16K pages. + + +Actual x86-64 Bash and Windows memory contracts (2026-10-01) +---------------------------------------------------------- + +A private build of FEX-2604 now runs dynamically linked x86-64 Bash and +Wine's Windows x64 memory-contract executable as 4K AArch64 host processes +on both 16K and 64K kernels. The constructor probe observes AT_PAGESZ=4096 +in FEX, including its replacement processes. Native defaults remain unchanged. + +The Windows test verifies 4K pages with 64K allocation granularity, +independent 4K permissions, a vectored access-violation handler, decommit and +zero-filled recommit without changing neighbours, thread access and shared +section aliases with independent protection. Both kernel configurations +passed with the default QEMU max CPU and SVE enabled. This establishes the +specified memory operations in Wine/FEX under TCG, not game rendering, +Apple GPU integration, native hardware performance or full ABI completion. + +Three private FEX changes were needed during application testing: materialize +full-width constants for SVE address offsets; use the actual architectural +SVE vector length for MUL VL addressing; and batch a successful low-address +munmap while retaining the previous error fallback and bookkeeping. Their +focused tests and source patch are preserved separately. Installed host FEX +binaries are unchanged. These are userspace fixes, not kernel workarounds. + +The disposable Wine fixture uses separate writable prefixes and a read-only +x86 runtime disk, selected by device serial. Its C: drive points to drive_c +and Z: to the extracted x86 root. Invoke the payload by its explicit Windows +path (Z:\wine-page-contract.exe); the earlier Unix-path invocation crashed +before reaching the payload on both native4K and mixed16K configurations. +That comparison used a feature-disabled build of the modified kernel source, +not an untouched upstream kernel. It does not exclude generic MM regressions. +Pipe-backed console output makes the payload checks observable. + +The optional payload is built without a Windows SDK or CRT:: + + make -C tools/testing/selftests/arm64/user4k OUTPUT=/absolute/output wine-contract + +The target uses clang's x86_64-pc-windows-msvc target, llvm-dlltool and lld-link. +It does not add those tools to the normal Linux selftest build. The generated +wine-page-contract.exe must run inside a configured Wine runtime; simply +building the PE executable does not execute the test. Compile-time assertions +check the freestanding Windows x64 structure declarations. + +Evidence: work/user4k-continuation/wine-memory-contract-v1 contains the +fixture source, logs, build inputs and hashes. Runtime dependencies and the +private FEX source revision are recorded there. The broader unresolved +larger-than-native granules, legacy GUP consumers, device integration and +hardware qualification requirements above remain open. + + +Prefault boundaries follow the process granule (2026-10-01) +-------------------------------------------------------- + +The three byte-range helpers fault_in_readable(), fault_in_writeable() and +fault_in_safe_writeable() previously advanced by native PAGE_SIZE. On a +larger-page kernel this could omit an interior smaller user page when checking +a range. They now advance and align by the current mm's immutable page size. +The architecture's finer permission probe remains in +fault_in_subpage_writeable(). Physical page/folio sizes are unchanged. + +The new user_granule_prefault_test borrows isolated mms, starts 17 bytes into +a page and checks three-page ranges with a missing middle page, a read-only +middle page and three writable pages. All four public prefault helpers return +the expected unprocessed byte count. It covers user4K/native16K on the 16K +build and user4K/user16K/native64K on the 64K build. Both complete KUnit runs +pass: 104 main tests and two TLB tests, plus 20/28 data tests respectively +(with the documented skips). Feature-disabled arm64 and x86 mm/gup.o builds +also pass. This fixes range probing; it does not add legacy page-array GUP +support or change pin ownership. Evidence is in prefault-boundaries-v1. + + +Physical-page-bounded references and coarse-leaf COW (2026-10-01) +--------------------------------------------------------------- + +Typed user fragments now stop at the smaller of the process translation leaf +and the native physical page. For a larger user leaf, the walker selects the +interior native page and retains its ordinary folio reference or pin; it no +longer assumes an anonymous slot owner exists. Read pins on shared anonymous +backing use the ordinary unshare fault path. Smaller-than-native mappings +retain their existing slot references and pins. Core dumping advances by the +same fragment unit, so its scratch page and copy operations remain bounded by +a native page even when a user leaf is larger. + +The internal 64K-on-16K fixture checks each zero-page reference, unaligned +references across every native page of anonymous backing, reference-count +balance, read-pin COW after fork, preservation of the original bytes, and +pinned-data lifetime after the child mapping is removed. It passes. This is +internal groundwork: public larger-than-native process activation remains +disabled, and file-cache promotion, swap/reclaim/migration and remaining +consumers still need integration before enabling it. + +The core metadata conversion also handles both shift directions: NT_FILE +file offsets are converted from native page-cache units to the dumped mm's +page units, including larger user granules without a negative shift. A unit +test covers 4K/16K/64K units and large file offsets. The real core-dump fixture +now derives its native size from AT_PAGESZ. It passes all 32 checks for +native16K/4K and all 48 for native64K/4K/16K, including swapped data, protected +anonymous data, unfaulted file contents, sparse zero pages, AUXV/NT_FILE, +header filtering and core-size limits. It does not execute the disabled +64K-on-16K process ABI. + +Final KUnit results: native16K passes 106 main, two TLB and 20 data tests; +native64K passes 105 main, two TLB and 28 data tests, with expected skips. +Feature-disabled arm64 and x86 gup/coredump objects compile. Frozen source, +Images, configs and runtime evidence are in gup-coarse-fragments-v1. + + +Memory-cgroup coverage on the native64K kernel (2026-10-01) +--------------------------------------------------------- + +The native64K validation configuration now enables CGROUPS, MEMCG and +CGROUP_PIDS. The previously recorded native64K runs had MEMCG disabled. +The unchanged kernel implementation passes its full KUnit set with the new +configuration: 105 main, two TLB and 28 data cases, with expected skips. + +The pressure fixture now accepts native16K or native64K kernels and a 4K or +16K alternative worker. Each worker fills and verifies 32 MiB, forks and +changes a quarter of its logical pages while the pair shares a 48 MiB memory +limit and a 256 MiB swap device. Both native64K+4K and native64K+16K completed +four rounds with measured reclaim and swap activity, no cgroup OOM events, +correct bytes/COW isolation and zero anonymous/swap charge after exit and +explicit reclaim of any remaining clean swap cache. This is bounded pressure +coverage, not a claim about all I/O-stall or memory-exhaustion conditions. + +The accounting fixture now checks both alternatives on a 64K kernel. For +32 MiB workloads it recorded 1,211,176 bytes of unreclaimable slab for user4K +and 1,143,056 for user16K, versus zero in the native64K control at the sample. +The native16K control/4K comparison also passed. These are observations from +the recorded debug builds, not a fixed ABI overhead or a performance claim. + +The native64K runtime boots reported an optional debug_vm_pgtable allocation +failure for an order-13 (512 MiB) contiguous block, before the fixture started. +It occurred with both 768 MiB and 2 GiB guest RAM. The inspected debug code +explicitly skips the huge-page data tests when that allocation is unavailable; +its remaining tests and userspace continue. Record that coverage skip rather +than claiming a warning-free boot. The first runner omitted this warning from +its failure filter; the stricter rerun reported false despite the userspace +contract passing. Both raw results and the separate, explicit assessments are +preserved. No other BUG/WARNING/KASAN or pressure-test failure was observed. + +This optional boot-debug allocation is distinct from the earlier native16K +memory-cgroup OOM. Its old log showed almost the full 48 MiB limit in pending +swap writeback with free swap available; why writeback did not complete in +time remains unproven. No root-cause fix for that old event is claimed. +Evidence and the exact MEMCG-enabled Image/config are in native64-memcg-v1. + +Userfaultfd missing, minor, WP and pager-granule integration (2026-10-01) +====================================================================== + +With USERFAULTFD, PTE markers and architecture WP/minor capabilities enabled, +the focused ``uffd-wp-init`` executable now checks the full supported contract. +It runs native and 4K processes on a 16K kernel, plus native/4K/16K processes on +a 64K kernel. All five cases pass in ``tcg-{16k,64k}-uffd-contract-v2.log``. +The tests cover: + +* Anonymous COPY/WP fault sequencing, ZEROPAGE and full-byte neighbour checks; + batch backing packing verified from physical addresses; prefix accounting + after EEXIST/EFAULT; unlocked source-page retries and fork/COW. +* Private sparse-shmem COPY/ZEROPAGE with unchanged file bytes, source retries + and sub-native EOF boundaries. +* Private/shared shmem MINOR/CONTINUE/WP at non-native-aligned file offsets, + alias identity/coherence, private COW, batch CONTINUE and truncation boundaries. +* MLOCK_ONFAULT during anonymous filling and shmem continuation, followed by + munlock and discard; populated, unpopulated and asynchronous WP; per-leaf + pagemap WP state; isolated POISON/SIGBUS behaviour. +* Native targets with 4K pagers and alternative targets with native pagers, + passing the userfaultfd via SCM_RIGHTS and checking COPY/ZERO/batch contents. +* Exact-address/TID events, explicit fork events and child fault descriptors, + default fork context removal, partial unregister and close cleanup. + +These are bounded KASAN VM tests. They do not establish all userfaultfd race, +swap/pressure, hugetlb, shared-cache population, hardware or performance +behaviour. Native64 may skip debug_vm_pgtable's optional order-13 allocation, +as documented for the previous native64 memcg configuration. + +Partial shmem population state: persistence foundation (2026-10-01) +================================================================ + +Shared missing-page COPY remains gated. Its backing state now has an inode +XArray indexed by native page-cache offset. Each value is a mask of missing +4K units; an absent value means fully populated cache contents. The folio lock +serializes updates. Cache removal takes i_pages before the metadata XArray +lock. Fresh cache insertion discards stale state; replacement by swap or folio +migration preserves it. Inode initialization covers allocation-failure cleanup, +and inode destruction frees the XArray. Metadata allocations use __GFP_ACCOUNT. + +Native PTE/PMD installation, alternative file PTE installation and userfaultfd +CONTINUE mark exactly their exposed byte range populated. A bounded 4K alias +clears only its own bit; a native alias exposes and accounts for its complete +native granule. Missing neighbours remain physically initialized to zero. +This is the basis for coherent alias behavior, not completed shared-COPY +registration or fault delivery. + +New real-kernel fixtures passed on native16 and native64: individual-index +truncation without losing neighbouring state, generic cache deletion, fresh +cache recreation, actual folio migration preserving both contents and state, +and ordinary mixed-granule alias mapping. The main KUnit suites now report +109 and 108 passes respectively, with the prior architecture/configuration +skips unchanged. + +The dedicated arm64-user4k-shmem-swap suite passed all three cases on both +kernels using disposable virtio swap disks: writeout followed by complete +swap-cache eviction and disk-backed restoration; truncation while swapped; +and swapoff restoration. These fixtures seed the metadata internally; they do +not establish that userspace shared missing-page COPY works. The first two +fixture revisions failed because the direct writeout harness omitted its null +plug check and clean-for-I/O transition; those fixture errors were fixed, with +failed logs retained. The production swap path was not changed for those fixes. + +Remaining integration includes atomic shared COPY/ZERO publication, distinction +between missing and minor faults for partial cache entries, buffered-write +validity updates, partial truncation/hole-punch validity and byte-granular unmap, +and cross-granule shared-file userspace/pressure tests. None may silently mark +neighbours populated merely because a native folio has been allocated. + +Shared shmem missing-page copy and fault delivery (2026-10-01) +============================================================ + +Alternative shared shmem ranges can now register MISSING and resolve faults +with COPY/ZEROPAGE. Shared population uses a locked cache folio with independently +missing units. A reusable private source buffer per COPY request prevents user +source faults while holding an existing destination cache folio. VMA/byte-position +revalidation follows the unlocked retry. PTE collision, poison and EOF checks +precede destination writes and population-bit publication. A failed new insertion +removes its cache/accounting state; an existing shared folio remains intact. + +Resident and swapped cache entries distinguish selected missing units from +present data. An inode flag is set before inserting partial backing, so minor +faults wait for initialization instead of observing the allocation as complete. +Native files which never use partial population retain the existing early minor +path. CONTINUE rejects a wholly missing leaf even when its native folio exists. +Ordinary buffered writes populate only the initialized range. A native kernel +folio consumer receives fully populated backing for the range it can expose. + +A MISSING-only remote-pager test exposed a fault-around bypass: the cache lookup +could map unfilled neighbouring leaves directly. File mapping now consults the +missing mask when selecting eligible PTEs; a huge leaf cannot cover missing +units of a missing-registered mapping. Populated leaves remain eligible for +batching. The corrected case passes for native16/4K targets on a 16K kernel +and native64/4K/16K targets on a 64K kernel, with pagers using a different granule. + +The expanded ``uffd-wp-init`` userspace fixture passes on both Images in +``tcg-{16k,64k}-uffd-shared-copy-v5.log``. It checks two independent missing/minor +pagers on the same file, COPY/ZERO, rejection of CONTINUE into a missing leaf, +physical backing identity and packing, write protection and shared coherence, +source-page fault retries/failure, native EOF rejection, and short-range buffered +write isolation. Earlier tests wrongly required EFAULT from native EOF COPY: +the preserved native allocator returns ENOMEM when its index lies beyond EOF; +the controls now verify that existing behavior. The fault-around failure logs +are retained separately from the passing results. + +Partial hole punch, shrink/regrow validity, broader shared-file swap/pressure and +race qualification remain unfinished. The inode mask still allocates metadata +when a previously fully populated native page becomes partial; that allocation +must be made safe before a non-failing truncate operation commits. Do not treat +the basic COPY tests or currently accepted range registration as proof that all +shared MISSING semantics are complete. The MISSING_SHMEM feature bit remains +unadvertised pending those operations. + +AArch32 remote UFFD COPY address spaces (2026-10-01) +-------------------------------------------------- + +``UFFDIO_COPY`` now validates its source byte range against the calling +pager's address space, while destination address and length alignment use +the registered target's address space and granule. The source can be +unaligned and does not require a multiple of the pager's native page size. +Previously a native AArch64 pager's valid source above 4GB was rejected with +``EINVAL`` when its userfaultfd belonged to an AArch32 process. + +The freestanding ``compat-uffd-probe`` and disposable PID1 harness +``compat-uffd-init`` reproduce that rejection before the fix and pass after +it on native 16K and 64K KASAN kernels. Install the probe as +``/compat-uffd`` and the harness as ``/init``. The pager sends three leaves +from an explicitly mapped high, unaligned source into each of anonymous and +shared AArch32 mappings. Tests check missing faults and every copied byte, +invalid destination alignment, invalid lengths, address overflow, and an +inaccessible source followed by a successful retry into the same leaf. + +Both kernels also pass with the removal-only boot override +``id_aa64mmfr0.tgran4=f``: AArch32 retains the native 16K/64K ABI, and 4K-only +memory operations and COPY lengths remain rejected. This models the +no-4K capability case; it is not an Apple A9 hardware test. The generic +userfaultfd object compiles with the feature disabled on arm64 and x86. + +This closes the pager/target address-space boundary issue. It does not +qualify partial shared-file hole punching, shrink/regrow, or all UFFD +operations; the broader limitations above still apply. + +Partial shmem holes and allocation-free resident validity (2026-10-01) +------------------------------------------------------------------- + +Partial hole punching and shrinking now invalidate complete 4K units within +retained native cache pages. Only bytes requested by the operation are zeroed; +partial 4K edge units remain valid. Mapping invalidation uses each process's +leaf size: unrelated fine leaves keep their PTEs, larger covering aliases are +revoked, and private COW copies survive a hole punch. Existing EOF truncation +rules still remove private copies of complete user leaves beyond EOF. + +The sharing policy preserves data across granules. A coarse alias with any +populated units can refault and expose its zero holes; its entire mapped leaf +then becomes populated. A fine pager cannot subsequently overwrite those +bytes with COPY. The tests check this with a native writable alias and a fine +pager: COPY returns EEXIST after the coarse alias has refaulted and written. + +Resident missing bits now occupy two bytes in the existing per-page extension. +Compile-time assertions verify that 16K and 64K builds do not enlarge that +extension. Folio locking protects these bits. Updating them during a partial +truncate cannot allocate, addressing the previous late-allocation limitation. +New cache insertion resets the state; migration, generic cache replacement, +and shmem huge-folio collapse transfer it. Physical folio splitting retains +state with each constituent native page. The latter huge-folio lifecycle still +needs dedicated end-to-end qualification, beyond code inspection and builds. + +The inode XArray now persists only swapped masks. Reclaim reserves any required +records before committing the swap transition; allocation failure leaves the +resident cache and data intact and reactivates the folio for a later attempt. +Swap-in restores the resident bits before unlocking and removes the records. +The old all-resident inode-XArray description above is superseded by this +implementation. Allocation-failure injection and sustained pressure remain +untested for this new persistence path. + +``shmem-hole-init`` runs as PID1 in a disposable guest. It checks native16+4K +and native64+4K+16K processes, shifted file-offset aliases, exact PTE retention +and invalidation via pagemap, independent COW bytes, coarse refault collision, +and partial-tail shrink/regrow. The pre-fix kernel fails the 4K hole test +because its PTE remains present; both new kernels pass. Passing ``--swap`` +requires a dedicated guest ``/dev/vda`` swap disk. The fixture verifies the +selected shared mapping has swap usage and actual swap writes before resolving +an unfilled neighbouring leaf with COPY, then verifies all bytes of the +original populated leaf. This passes for 4K on 16K and 4K/16K on 64K. That EL0 +check does not require eviction of the swap cache; the separate KUnit disk +roundtrip below does. + +The dedicated shmem swap KUnit suite passes four tests on both kernels, +including full swap-cache eviction, swapoff, whole-index truncation, and a +new partial hole punched after cold swap-out while preserving an adjacent +populated 4K unit. Main KUnit passes 111/110 cases on native16/native64 with +16/17 intentional skips. It includes unaligned byte-range holes, buffered +repopulation, physical migration and generic page-cache replacement validity. +The existing mixed-page UFFD contract also passes after this metadata change. +Affected feature-disabled arm64 and x86 objects compile. These are bounded +correctness results, not sustained pressure, race, performance or hardware +qualification. The broad shared-missing feature remains unadvertised. + +Refilling holes retained in externally referenced large shmem folios (2026-10-01) +------------------------------------------------------------------------------- + +A large shmem folio can remain in the cache when an external GET reference +prevents splitting during a partial hole punch or truncation. The validity +bits correctly identify missing user leaves, but the native UFFD COPY and +ZEROPAGE paths previously tried to insert a new cache folio and returned +EEXIST. This was reproduced on both native16 and native64 kernels, with a +pipe holding a vmsplice reference to an unaffected part of the large folio. +The alternative small-leaf paths already handled this retained backing. + +Native shmem insertion collisions on an inode using sub-native validity now +retry against the locked retained cache folio. The operation reuses its +already prepared source, checks missing state and PTE/EOF conditions, and +fills only the requested leaf. The ordinary native insertion path keeps its +existing allocation and copy behavior. The same installation helper serves +alternative leaves; no huge-folio disabling or splitting workaround is added. + +The disposable PID1 ``shmem-thp-init`` fixture requests four-native-page +backing on a tmpfs mount with ``huge=within_size``. It verifies compound head, +tail and THP flags plus physical contiguity before the test, and confirms the +large folio remains compound when a pipe reference prevents splitting. Its +40 cases cover native16+4K and native64+4K+16K, COPY/ZEROPAGE, hole punch and +shrink/regrow, with and without the external reference. Invalid COPY sources +return EFAULT without consuming the hole; duplicate COPY returns EEXIST +without changing bytes. Successful fills preserve the expected file bytes +and data read from the pipe. Both KASAN guests pass, and feature-disabled +arm64/x86 userfaultfd objects compile. + +The fixture must keep its filename until the initial large-folio allocation. +Unlinking before mmap selects anonymous-shmem policy, which did not allocate +the requested small large folio with the test guest's inherited policy. The +initial fixture failures were allocation-shape failures, not a kernel lack +of large-folio support. A temporary opt-in allocation trace established this; +it was removed from the final source. The later pre-fix native COPY/ZEROPAGE +EEXIST failures used verified compound backing and are separate evidence. + +This covers PTE-mapped large folios retained by GET references. It does not +qualify PMD-sized collapse and alias transitions, write-capable FOLL_PIN/DMA +users, or concurrent truncation/refill. The broad shared-missing feature +remains unadvertised until those and the pressure/failure cases are resolved. + +Native PMD collapse with fine missing ranges (2026-10-01) +-------------------------------------------------------- + +The native PMD-installation guard now checks for a wholly missing native +page, matching the native PTE fault policy. Previously it rejected a PMD +whenever any 4K unit in the folio was missing, even when every native page +contained populated data. The new test reproduces that rejection: physical +cache collapse succeeds, but native PMD promotion returns EINVAL. After the +fix, promotion succeeds once the last wholly missing native page has some +populated data. Installing the coarse mapping marks all exposed units +populated, as for native PTE aliases; this does not overwrite their bytes. +A wholly missing native page still prevents PMD installation and retains its +missing-fault contract. + +``shmem-collapse-init`` is a disposable PID1 fixture for the 16K kernel. It +creates a 32MiB file with partial native pages and one wholly missing native +page retained inside a small large folio. A native process performs +MADV_COLLAPSE while a 4K process owns UFFD mappings. Compound flags, physical +contiguity and changed PFNs prove real cache replacement; missing and minor +events prove validity survives the transfer. After filling one 4K unit in +the wholly missing native page, ``ShmemPmdMapped`` confirms a full 32MiB +native PMD mapping. Formerly missing fine units then deliver minor faults, +and COPY cannot overwrite those units. + +The native process is pinned to CPU0 and the fine process to CPU1. Eight +further cycles prime native PMD and fine PTE translations, punch a 4K hole, +and alternate pager COPY, native writes and writes through a fine alias +whose file offset starts at 4K. The native PMD mapping is split for each new +hole. Duplicate COPY rejection and subsequent minor faults verify that +refaulting either ordinary alias updates validity. A complete final 32MiB +byte comparison checks all populated values and zero regions. These are +coordinated cross-CPU tests, not an uncontrolled concurrent-writer test. + +The KASAN/PAGE_TABLE_CHECK_ENFORCED 16K guest passes this fixture; both 16K +and 64K kernels pass the updated missing-native-page KUnit assertions and +main suites (111/110 passing, 16/17 intentional skips). The existing UFFD +contract passes both kernels, and feature-disabled arm64/x86 objects build. +The PMD runtime fixture intentionally covers native16. Native64's PMD order +exceeds this tree's MAX_PAGECACHE_ORDER, so these results do not establish +512MiB shmem PMD support there. Smaller large-folio native64 coverage is +recorded in the preceding section. + +Writable FOLL_PIN/DMA, allocation-failure/pressure tests and simultaneous +mutation/refill remain outstanding. The broad shared-missing feature is +still unadvertised; these results do not complete the full process ABI goal. + +Writable pins across partial shmem holes (2026-10-01) +---------------------------------------------------- + +A retained writable pin remains capable of accessing cache bytes after a +hole punch revokes its user PTE. The previous partial-validity implementation +marked those bytes missing anyway. A regression fixture reproduces this with +an io_uring registered buffer: a completed READ_FIXED writes through the old +pin, then UFFDIO_COPY incorrectly succeeds over the same retained range. +Two KUnit cases also reproduce stale missing state after native/fine long-term +pins and after direct memfd_pin_folios acquisition, including after unpin. + +Partial-hole invalidation now checks the folio pin count after removing +covering mappings, with a barrier paired with fast GUP's PTE recheck. If +pinned, the removed range remains populated for missing-page classification; +ordinary byte zeroing still happens. Because file pins are counted per folio, +this classification is conservative if only a neighbouring range is pinned. +A subsequent hole after unpin can become missing again. Full-folio removal +continues to detach old pins from the file's replacement cache identity. + +Direct memfd cache pins have no PTE fault to publish exposure. Their acquisition +and full-folio exposure now hold the folio lock, serializing with partial hole +punch and COPY. This lock applies to shmem memfd pinning in mixed-page builds +with USERFAULTFD; ordinary GUP and feature-off builds retain their paths. +This is synchronization for the cache-visibility contract; its performance +cost has not been measured. + +The new shmem-pin-init disposable guest fixture uses actual registered-buffer +pins and asynchronous READ_FIXED/WRITE_FIXED. It checks hole punch and +shrink/regrow, retained-pin COPY rejection after unpin, full-native-page detach, +subsequent successful refill, every file byte and untouched neighbours. On +retained ranges it also runs 256 hole punches on CPU1 concurrently with 256 +fixed writes submitted on CPU0, followed by a deterministic write/COPY check. +This tests kernel I/O through held pins, not physical device DMA or every GUP +consumer. Shared truncate/COPY/alias mutation without pins, allocation failure, +pressure and the other broader ABI limits remain open. + +The same fixture exposed a separate 16K-on-64K io_uring bug: registration +assumed each returned fragment was 4K, rejected the correct fragment count, +and returned EFAULT. Registration now records the process's leaf size; fixed +buffer import uses that stored size for segment counting. Unaligned multi-leaf +READ_FIXED/WRITE_FIXED and reordered READV_FIXED/WRITEV_FIXED requests verify +segment traversal and guard bytes for native16+4K and native64+4K+16K. + +Both native16 and native64 Image builds and the final pin fixture pass. +KUnit passes the two new pin cases on both kernels. Shared UFFD regression, +retained large-folio COPY/ZERO regression, and native16 PMD collapse regression +are recorded in the native artifact directory. Feature-off arm64/x86 affected +objects compile. The broad MISSING_SHMEM capability is still unadvertised. + +Discoverable shared missing-page ABI (2026-10-02) +------------------------------------------------ + +Alternative-page processes now advertise and accept explicit negotiation of +UFFD_FEATURE_MISSING_SHMEM. This supersedes the earlier capability-gating notes +above. The implemented base-page shared COPY/ZERO semantics are discoverable; +UFFD MOVE and alternative hugetlb features remain separate, unsupported paths. +The native ABI and default process granule are unchanged. + +The new shmem-race-init fixture runs 512 COPY attempts concurrently with 512 +hole punches or shrink/regrow operations, a shifted fine alias, and a separate +native process repeatedly discarding/refaulting its PTEs. Native16+4K and +native64+4K+16K pass. After the actors stop, a fresh hole accepts COPY, both +aliases see the replacement, and every file byte has the expected value. +Private COW isolation is checked during hole-punch races. The resize case +keeps alias accesses within the retained EOF prefix. Its first version wrongly +rejected native ENOMEM on a truncated target; inspection confirmed the native +allocator's existing NULL/ENOMEM EOF response, and the test now records it +separately. No kernel behavior was changed to make that test pass. + +The shared-memory pressure fixture limits a worker to 8 MiB, supplies a private +128 MiB guest swap device, and populates 16 MiB of backing with missing +neighbouring leaves. Native controls use separate native pages for the holes. +Permuted refills, further partial holes, duplicate COPY rejection and complete +file-byte checks pass with measured swap, limit hits, scans and steals, zero +OOM events, and zero remaining shmem/swap charges after file destruction. Both +native16 and native64 matrices pass with the final accounting change. This is +bounded pressure coverage; it does not explain the earlier isolated anonymous +writeback-heavy OOM recorded above. + +Fault injection exercises failure of the first metadata-node allocation and +failure after an internal node has been allocated. It targets only the KUnit +worker during writeout and also fails allocator retries. The cache folio stays +resident, dirty and byte-identical, with its missing mask intact and no stored +swap-metadata records. XArray can retain an empty internal node after a failed +multi-level insertion; the test checks present records rather than requiring +its root pointer to be NULL. A normal retry of the same folio then completes +swap and disk restoration, preserving data and the mask. All six shared-swap +cases pass on both kernels. Fault injection is enabled only in the test build +configuration; the fixture turns probability back off and clears the worker failure flag. + +That inspection exposed a separate cgroup accounting omission. The XArray's +normal allocation path needs XA_FLAGS_ACCOUNT; __GFP_ACCOUNT on xa_store alone +covers its retry allocation. Both shmem inode initialization paths now set the +array flag. Swap preparation also pins the backing folio's memcg under RCU and +uses its charge scope, then restores the caller's scope. A regression creates +a child-cgroup folio, reclaims it from the root scope and inspects the metadata +node's actual charge: previously NULL, now the child memcg. Swap-in erases the +record. This is not claimed as a fix for the historical writeback OOM. + +The expanded userfaultfd fixture explicitly requests MISSING_SHMEM and passes +its native16+4K / native64+4K+16K matrix, including independent pagers, buffered +short writes, shared/private mappings, WP, source retries and cross-granule +remote pagers. The AArch32 probe also requests the feature. Automatic 4K and +boot-modelled no-4K/native16 or native64 fallback all pass with a native pager, +including the existing target alignment restrictions. These are emulated +capability models, not Apple A9 or Apple Silicon hardware validation. + +Full ABI work remains active: UFFD MOVE, larger-than-native public granules, +remaining legacy GUP interfaces, alternative THP/KSM, broader concurrency and +performance/device/game qualification are not established by these results. + +Independent anonymous subpage identities (2026-10-02) +---------------------------------------------------- + +Reverse-map callbacks now carry the virtual address separately from the physical +subpage offset. An optional, memcg-accounted sidecar gives each slot its own +anonymous root and logical byte position. Ordinary linear owners retain the +shared root/index representation. Slot reuse clears the old identity binding; +lookup snapshots take root references before dropping the pool lock. Migration +copies the identities before removing PTEs, including entries whose target slots +have not yet acquired mapping references. Historical roots are released in work +context because their final release can sleep, including when the last slot put +or a rebind occurs under a page-table lock. + +The nonlinear reverse walk is used by referenced/aging, reclaim and migration +callbacks. NUMA address lookup and hardware-poison process collection understand +independent identities as well. A metadata allocation failure during swap-in is +reported as OOM rather than SIGBUS. This is preparation for UFFD MOVE; the MOVE +feature and ioctl remain disabled for alternative processes. + +Final native16/native64 Image builds (nonlinear-rmap-v5) and KASAN/lockdep KUnit +runs pass. New tests cover distinct roots, different virtual/physical offsets, +changed native logical index, fork, migration rollback, migration to new physical +backing with byte preservation, slot reuse after VMA teardown, primary-root +lifetime, and injected allocation failures followed by successful retries. Main +suites report 115/114 passes with 16/17 explicit skips; TLB suites pass 2 each and +data suites pass 20+8 skips / 28. DEBUG_ATOMIC_SLEEP is not enabled in these +profiles. Optional page-idle/DAMON/KSM callbacks compile with the feature off and +in a subpage/no-THP profile; this is not runtime qualification of those features. +Feature-off arm64 and x86 affected objects compile with MEMORY_FAILURE enabled. + +The final native16+4K memcg pressure run passes swap-in byte checks and fork/COW +isolation with zero OOM and zero remaining swap charge. Final native64 userfaultfd +regression passes native64/16K/4K modes. Its initial runner invocation supplied +the wrong success string; the same captured boot was revalidated against the +fixture's UFFD CONTRACT PASS marker and all original error/exit checks. Earlier +native64+16K pressure also completed its workload, but its strict runner rejected +a pre-userspace debug_vm_pgtable order-13 allocation warning, also seen in native64 +KUnit boots. That warning has not been fixed or concealed. + +These tests do not implement or qualify UFFD MOVE, nonlinear swapped ownership, +physical-device DMA, or the broader outstanding ABI restrictions. Next work must +transfer one exclusive slot without copying or moving its neighbours, preserve +swap ownership across cache eviction, and test partial progress, pins and races +before enabling the ioctl. + +Nonlinear swap restoration (2026-10-02) +-------------------------------------- + +The independent-identity work exposed a remaining swap-in assumption: a swap +PTE's physical subpage offset was required to equal the VMA's logical byte offset. +Dedicated cached and disk-backed tests reproduced VM_FAULT_SIGBUS for a 4K slot +at physical offset 12K mapped at virtual offset 4K. Swap-in now validates that the +physical offset is inside the native page and aligned to the process granule; +the per-slot reverse-map binding handles its independent logical position. + +Both final Image builds (nonlinear-swap-v1) pass the new two-case +``arm64-user4k-nonlinear-swap`` suite with a private swap disk. Each case exercises +both restore orders, unrelated anonymous roots, fork while swapped, child write +COW, parent data retention, and physical-slot MTE tags. The cold case explicitly +evicts the swap-cache folio and checks a major fault. These nonlinear cases use +4K user leaves on native16/native64 kernels; nonlinear 16K leaves on native64 +still require their own coverage. + +The same runners pass four real userspace disk/fork/COW rounds per ABI: native16 +and 4K, or native64, 4K and 16K. The existing granule-swap PID-1 fixture now accepts +``--nonlinear`` to select the focused kernel suite while retaining those EL0 +checks. No skip is accepted by this focused suite. The earlier full-swap run had +16 passing existing cases and one explicit skip: its older borrowed-mm live +fault probe excludes CnP/S1PIE/S1POE and is incompatible with the current max-CPU +model. That skip was not counted as a pass or used to weaken the focused suite. +Final ordinary KUnit runs pass on both Images, with the disk-only suites explicitly +skipped when no swap disk is attached. The initial before-test fixture omitted +the folio lock for set_exclusive; the corrected before-v2 log is the SIGBUS +reproduction, while before-v1 is only a failed fixture setup. + +This closes cached/cold nonlinear restoration for the tested 4K cases. UFFD MOVE +still needs an atomic ownership-transfer operation, present/swap/zero/hole paths, +per-slot pin validation and partial-progress/race coverage before advertisement. +The native default and the existing AArch32 selection/fallback policy are unchanged. + +Userfaultfd MOVE implementation and bounded validation (2026-10-02) +----------------------------------------------------------------- + +MOVE now uses a two-phase per-slot ownership transaction for smaller-than-native +anonymous mappings. It allocates the independent mapping metadata and any old +root retirement record before changing PTEs. Both page-table locks and the folio +lock stabilize the operation. After clearing the source PTE and invalidating its +TLB entry, commit checks the slot's map count, exclusivity and pin count again. +A rejected commit restores the source PTE. Success changes only the slot's +logical root/position and installs the same physical bytes at the destination. +Same-mm RSS and map counts do not change. Different droppable policies are +rejected. Historical roots retain the deferred release needed for sleeping final +puts. The native batched path retains its existing geometry. + +The swap path checks the physical slot's swap count, preserves the encoded byte +offset and updates surviving cached-slot metadata when present. When no slot +metadata survives, the subsequent swap fault binds the independently positioned +slot using the preceding nonlinear swap fix. Zero pages, source holes and loop +alignment use the mm granule. A successful transfer also clears an earlier retry +error before accounting progress. MOVE is advertised in both the API feature +mask and the registered range's ioctl mask; larger-than-native MOVE remains +gated. The initial candidate omitted the range mask, which the first userspace +run detected before attempting a transfer; the second candidate corrected it. + +``uffd-move-init`` is a disposable PID-1 fixture. On native16 it tests native16 +and 4K; on native64 it tests native64, 4K and 16K. It checks physical address +identity, divergent virtual/physical offsets, neighbouring PFNs and bytes, +source faults after cross-CPU TLB invalidation, source PIN rejection, neighbour +PIN acceptance, retry after unpin, collisions, zero pages, allowed/disallowed +holes, partial progress, fork-sharing rejection, write-fault exclusivity recovery, +and transfer of the same swapped PTE followed by data validation at its new +offset. Each ABI also completes 256 back-and-forth moves on CPU0 while CPU1 +repeatedly registers/unregisters io_uring buffers. The physical address and all +bytes remain unchanged. Smaller-granule modes inject failure of the initial +sidecar allocation and old-root retirement allocation; each failure preserves +the source and empty destination, and each retry succeeds. + +These tests passed on both candidate Images before the temporary build directory +was lost, including KASAN/lockdep checks. The source was recovered from the +retained baseline and edit scripts, mirrored with the original Git index intact, +and rebuilt with identical configurations. The source and output directories +are now persistent under /home/dj/Work. The recovered AArch32 test executable is +byte-identical to the earlier passing one. The AArch32 probe negotiates MOVE and +checks 32-bit range validation, neighbour data and page-size restrictions. Auto4K +passed on both native kernels; on the recovered Images, capability-masked +no4K/native16 and no4K/native64 modes also pass. This is a hardware capability +model, not an Apple A9 hardware result. + +Feature-off arm64/x86 userfaultfd objects and the selftest build passed before +recovery. The corrected subpage/no-THP profile compiles the ownership and +userfaultfd objects with USERFAULTFD enabled; its first attempted compile had +incorrectly forced these objects while that configuration option was disabled. + +This is a functional experimental MOVE implementation with bounded tests, not a +claim of full ABI completion or hardware/game qualification. Dedicated coverage +of a swapped MOVE with a surviving GET-held slot, movement while migration runs, +larger-than-native public granules, remaining legacy GUP consumers, alternative +THP/KSM and the other previously recorded limits remains outstanding. The +historical anonymous writeback-pressure OOM remains unexplained. + +Recovered MOVE Images also passed the complete ``uffd-move-v3.cpio`` contract, +including the pin race and both injected allocation failures, on all listed +native/alternative ABIs. Ordinary KUnit reports 115/114 main-suite passes with +16/17 explicit skips; both TLB suites pass 2, and data suites pass 20+8 skips/28. +The pre-existing full userfaultfd contract passes on both recovered Images. +Artifacts, source, configurations and command/result records are frozen under +``/home/dj/Work/user4k-vm-state/uffd-move-v2``. These results supersede the earlier +notes which described MOVE as disabled, while preserving the qualification +limits above. + +Swapped MOVE after fork and retained references (2026-10-02) +---------------------------------------------------------- + +The smaller-than-native MOVE path now accepts a sole remaining swap reference +when an earlier fork cleared the swap PTE's exclusive marker. The native path +keeps its existing marker check. Smaller leaves instead verify the per-slot +swap count under both PTE locks; cached metadata must additionally have no +present mappings, no pins and no migration in progress. The source PTE is +transferred without setting its exclusive marker, so a later fault still makes +the normal reuse/COW decision. Multiple swap references continue to return +EBUSY without changing the source or destination. + +A dedicated KUnit case retains a GET reference through swap-out and MOVE, +changes the virtual offset and anonymous root, and checks the original physical +slot and every neighbour. It covers both an exclusive swap PTE and a PTE whose +forked child has exited. In the latter case, a destination write copies the slot +and leaves the retained GET data unchanged. In the former, existing exclusivity +allows in-place writes; GET alone does not establish a snapshot. Tests pass for +4K-on-16K, 4K-on-64K and 16K-on-64K. The pre-fix v2 fixture reproduced EBUSY after +the child exited; v1 had an incorrect unconditional COW expectation and does not +establish a production failure. + +A second case runs a MOVE worker while the test migrates the native folio +containing that slot and its unmoved neighbours. The final v7 disk-backed runs +completed 62, 41 and 55 migrations respectively, with 22, 27 and 15 MOVE calls +succeeding; data and source holes remained correct. The worker deliberately +leaves 1--2 ms gaps between successful moves to exercise completed migrations +as well as contention. This is test scheduling only. An earlier continuous +worker allowed every migration attempt to lose to temporary folio references +in one configuration; that is not evidence of data loss or a production +performance measurement. Deferred LRU additions are drained before each test +isolation attempt. + +Evidence: ``user4k-vm-state/move-retained-v1/`` contains source, Images, configs, +logs and the reproduction history. These are bounded VM tests, not hardware +DMA, game performance or full ABI qualification. Remaining legacy page-array +GUP callers, larger-than-native public granules, alternative THP/KSM and the +previously recorded pressure and device limits remain outstanding. + +User-event tracing with alternative page sizes (2026-10-02) +--------------------------------------------------------- + +The user_events enable-bitmap writer now uses a typed PIN for alternative mms. +The new pin_user_fragment_vma_remote() helper retains the caller's mmap lock +and reports the physical byte offset and exact ownership. It supports NOFAULT +so the tracing code keeps its existing asynchronous fault-repair mechanism. +NOWAIT and LONGTERM are rejected by this lock-preserving interface. Ordinary +native mms retain the existing page-array pin path. + +Before this conversion, registration failed with ENOMEM for smaller-granule +processes on both kernels. The page-array GUP path rejected the alternative mm; +the enabler creation path eventually returned NULL, which registration reports +as ENOMEM. Native registration passed the same fixture. + +The fixed Images pass private and shared bitmap updates, 32-bit and 64-bit +enable words at leaf ends, relocated backing with different virtual/physical +subpage offsets, adjacent-byte preservation, 32 enable/disable transitions in +forked processes, and missing-PTE fault repair after MADV_DONTNEED. Actual event +payloads are read back from tracefs. The matrix includes native16+4K and +native64+4K+16K processes. This does not establish new AArch32 tracing support. + +KUnit validates the new helper's retained mmap lock, missing-page NOFAULT error, +unsupported flag rejection, partial-record boundary, returned VMA, PIN ownership +and successful NOFAULT retry with present backing. The full existing KUnit +matrix also passes. Feature-off native and no-THP GUP/tracing objects compile. + +Evidence is under user4k-vm-state/user-events-v1/. Initial v2 trace-content +checks mistakenly expected decimal-only formatting; captured records already +contained the correct payload in hexadecimal with decimal in parentheses. The +v3 parser handles this representation; v4 is the formatted, Makefile-built +fixture. That test-format correction is not another kernel fix. USER_EVENTS +is enabled in the current VM build configs to execute this test. + +At the user_events milestone, uprobes still used page-array GUP and native +PAGE_MASK/offset assumptions. The following milestone supersedes that limitation. Vhost logging is covered by a later milestone below; +various device pinning paths still need integration. The broader ABI limits in +the previous sections remain open. + +Uprobes, return probes and granule-aware folio walks (2026-10-02) +-------------------------------------------------------------- + +AArch64 uprobes and uretprobes now operate in smaller-than-native processes. +Instruction access retains the exact GET/slot ownership and supplies the +physical page offset separately from the virtual instruction address to the +verification callback. The x86 verifier is updated for that explicit offset; +native, no-THP and x86 builds pass. Reference-counter writes also use typed GET +records for alternative mms. File-offset lookup includes vm_subpage_offset and +filters native interval-tree matches to the exact VMA byte interval. + +Removal checks per-slot references and pins before discarding restored text. +It compares only the user leaf against the matching file bytes, removes that +slot's rmap and mapping reference, and leaves unrelated bytes intact. Native +folio-wide behavior remains. A pipe GET keeps the code PTE resident during +removal; without that retained reference, smaller leaves can return to shared +file backing on a subsequent install/remove cycle. The XOL allocation remains +one native backing page and reserves enough virtual space to align its base +natively, preserving the architecture copy hooks' offset convention. + +The first converted candidate exposed a separate generic blocker: +folio_walk_start() still used native p4d/pud/pmd offsets and could not find the +PTE that typed GUP had just obtained. Uprobes then retried indefinitely. The +walker now uses per-mm table geometry and leaf sizes, including the selected +native page within a larger leaf. A KUnit case checks normal and zero-page +walks on both sides of a PMD boundary for native, 4K and 16K-on-64K modes. +KSM's mergeable-page lookup explicitly excludes alternative mms while its +native-page comparison/replacement code remains unconverted; this is not KSM +support. Device-exclusive paths still encounter their existing GUP restriction. + +Before-fix tests reproduce EOPNOTSUPP when enabling probes on aligned mappings, +and a silently missed probe/counter for mappings beginning within a native file +page. Final tests cover native16+4K and native64+4K+16K, both aligned and shifted +file mappings, exact entry/return hit counts in parent and child, short reference +counters at independently shifted file offsets, original file bytes, adjacent +mapping integrity, code restoration, retained pipe data and return to file +sharing. Occupying the normal XOL hint forces another placement and its native +alignment is checked. AArch32 instruction probing retains the architecture's +existing EOPNOTSUPP behavior; this adds no AArch32 probe instruction decoder. + +The initial retained-pipe fixture assumed native text must disappear immediately +after the pipe was released. The same assertion fails on the before-fix native +kernel, so it was not a valid unconditional native expectation. The final test +still verifies native initial reclamation and retained-reference behavior, and +requires smaller-page reclamation after releasing the reference. Temporary +opt-in kernel tracing located the walker retry and was removed completely. + +Evidence: user4k-vm-state/uprobes-v1/ contains before/fixed Images, configs, +source, the bounded disposable-VM fixture and captured results. Primary configs +now enable PERF_EVENTS and UPROBE_EVENTS. The final KUnit matrix passes with +117/116 main cases and 16/17 explicit skips on native16/native64. This does not +qualify every uprobe concurrency scenario, optional KSM/THP behavior, device +pinning or the overall userspace ABI. Those broader limits remain open. + +Vhost dirty logging with relocated user leaves (2026-10-02) +--------------------------------------------------------- + +Vhost's dirty-bitmap writer now uses typed PIN records when the worker's mm +has an alternative granule. It pins one aligned atomic word and maps the +returned physical folio offset. Thus an mremap relocation does not redirect +the update to a different slot of the native page. Dirty release retains the +existing PIN ownership rules. Native processes keep their fast page-array +pin path; the vhost guest dirty-page unit remains 4K, independently of the +host process's page size. + +Before the fix, the real vhost-net backend transferred packets in smaller +processes but left the dirty bitmap unset. Native controls passed. The fixed +kernel passes native16+4K and native64+4K+16K with an isolated TAP backend. +The fixture checks RX payloads, TX completions, exact receive-buffer and +used-ring dirty bits at the end of a relocated leaf, an unaligned bitmap +base, and all neighboring bytes. Private and shared mappings run both with +and without a forked child: private snapshots remain untouched while shared +updates are visible. Both owner-task and borrowed-mm kernel-thread workers +are tested. After MADV_DONTNEED, pagemap confirms the bitmap PTE is absent +before another packet cycle faults it back successfully. + +Each case executes 64 packets in each direction across the present and +missing-PTE phases. There are eight cases per ABI, five ABI/kernel pairings +across the two Images. The disposable PID1 fixture is vhost-log-init; the VM +configs enable TUN, PACKET and VHOST_NET. Evidence, source and before/fixed +Images are under user4k-vm-state/vhost-log-v1/. This covers software vhost-net +dirty logging, not VFIO/device DMA, network performance or all vhost backends. +The earlier fixture version admitted unsolicited IPv6 interface packets; +its native RX length failure was resolved by disabling IPv6 in the isolated +guest before creating TAP interfaces, not by a kernel data-path change. + +Deferred mlock accounting and reclaim repair (2026-10-02) +------------------------------------------------------- + +A subsequent native64 KUnit run exposed an invalid exact-count assumption in +user4k_mlock_shared_test. One run started at 16 instead of 17 locks; another +started at zero and cleared the folio's mlocked flag after an alias was +unmapped. Native64 reruns sometimes passed. These original failures remain +in the vhost-log-v1 evidence and are not retroactively counted as passes. + +The existing mlock/LRU implementation explicitly permits undercounting: +__mlock_folio() skips a folio temporarily off LRU, and LRU insertion initializes +an already-mlocked folio's count to zero. Remaining VM_LOCKED mappings are +supposed to restore the lock during reverse-map reclaim. Reading mlock_count +after the folio becomes evictable instead reads its overlapping LRU-link +storage; such a value is not evidence of count corruption. + +The exact-count fixture now disables deferred LRU caching for that test only, +with a registered cleanup restoring it even after an assertion. Production +batching is unchanged. A separate deterministic test retains normal batching, +isolates a file folio while installing two locked aliases, drains the deferred +locks while it is off LRU, and puts it back. This reproduces the zero-count +state, including loss of the folio flag after removing one mapping, in native +and smaller-granule modes. Both folio_referenced() and try_to_unmap() must +restore the lock without removing any remaining PTE, changing its backing or +changing mapcount. Final teardown must clear the lock and all mappings. + +The repair test passes native16+4K and native64+4K+16K. This establishes the +required reclaim behavior for that controlled undercount; it does not identify +the exact scheduling interleaving of the original intermittent run. Temporary +LRU diagnostics are removed from both source and final Images. Evidence is +under user4k-vm-state/mlock-count-v1/. + +BPF userspace pointers with fragment ownership (2026-10-02) +--------------------------------------------------------- + +Task-storage BPF UPTR fields now support smaller-granule processes. The update +path pins each pointee with FOLL_LONGTERM and FOLL_WRITE through the fragment +API, retaining its physical offset and ownership. A pointee must fit within +one supplying-process page and one physical fragment; crossing either boundary +is rejected. The existing native fast pin path remains. + +A private ownership pointer follows the visible map value, aligned separately +from that value. It moves together with the UPTR fields when a new storage +element is published. Element allocation and charging include the private tail; +userspace copies and verifier-visible value size still cover only the public +bytes. An alternative-page update allocates a pin bundle charged to the map's +memory cgroup. Failed updates, replacement, task exit and map destruction retire +that bundle. After the existing reader grace period, workqueue release may lock +and dirty its folios without requiring the original mm, task, map or BTF record +to remain alive. Native fields still use their existing release path. + +On a 64-bit kernel with MM_SUBPAGE, each UPTR-containing value gains one private +pointer plus up to seven alignment bytes. The pin bundle is allocated only for +an alternative-mm update. BPF programs continue loading the same direct kernel +pointers from the public field offsets. No throughput or memory-pressure +performance result is claimed for this bookkeeping change. + +Before conversion, actual BPF programs load successfully and native UPTR map +updates pass; smaller-mode non-null updates return EOPNOTSUPP. The candidate +passes native16+4K and native64+4K+16K using a raw sys_enter BPF program. Private +and shared relocated leaves retain correct read/write access at their ends. +The fixture checks 64 successive updates, duplicate-update rollback, partial +pin failure, overflow and cross-page rejection, read-only failure, null and +half-null fields, lookup pointer redaction and untouched adjacent bytes. +Pin acquisition/release counters are folded across CPUs before checking live +pins and their return to baseline. + +A storage-owning child can access the parent's pinned data, with private COW +isolation, and exit without leaking its pins. Separately, a supplying child can +exit while the parent's BPF storage retains pointers into the retired mm; a +pipe GET verifies subsequent BPF writes to that backing. Unmapping and replacing +the original virtual address likewise leaves the pinned old bytes accessible +through the retained pipe without altering the new mapping. Closing the map, +link and program releases the last pins. Both ordinary 24-byte and packed +25-byte map values pass; the latter checks hidden-tail alignment and verifies +that lookup does not overwrite a sentinel immediately after the public value. + +The disposable bpf-uptr-init fixture builds statically through the arm64 +selftest Makefile. Current VM configs enable DWARF5 and DEBUG_INFO_BTF so real +BPF programs can resolve task types. Evidence is under user4k-vm-state/bpf-uptr-v1/. +The first fixture passed an entire attr union and hit EINVAL even for null +native values; using each command's populated attribute extent fixes that +fixture setup. It is not another kernel fix. Wider BPF concurrency, device DMA, +larger-than-native public granules and full ABI qualification remain separate. + +File-backed larger user leaves: internal groundwork (2026-10-02) +-------------------------------------------------------------- + +Internal 64K user mappings on a 16K kernel can now fault from a single aligned +cache folio covering the complete leaf. The alternative-granule file mapper +retains one native reference and rmap unit per covered native page, accounts +one logical user PTE, and preserves physical byte identity across 4K, native +and coarse aliases. Native mappings retain their existing mapping path. +Undersized or physically misaligned backing is rejected before installing a +coarse PTE. This does not promote existing small cache folios or enable the +public larger-than-native exec ABI. + +Private first-write faults copy the entire larger leaf before marking the +anonymous destination up to date. Both cold COW and a write to an existing +read-only file PTE are covered, including a leaf beginning in the interior of +a larger cache folio. Tests check every 4K chunk, file isolation, native-unit +reference counts and logical RSS, then require complete mapping/reference +cleanup. + +A new mixed-alias test exposed a real invalidation fault: a native-page file +range could reach a larger user PTE with a range shorter than its granule, +violating the zap batch invariant. The file-range walker now rounds overlapping +larger-mm ranges outward to whole user leaves within that VMA. It retains the +ordinary API's native-page rounding for fine aliases. The separate byte-range +hole API preserves neighboring fine leaves within the affected native page. +Tests exercise both interfaces, lazy file mapping after fork, partial parent +unmap, cross-alias writes and cleanup. A partial final leaf keeps its zero tail; +a leaf starting beyond EOF faults with SIGBUS. + +The fixture uses an isolated RAM-backed generic-file test inode with explicit +large folios. It is not evidence for filesystem cache promotion, disk writeback, +coarse swap/migration, special mappings or public coarse exec. Those remain +requirements before enabling larger-than-native processes. Small-granule +userspace file and userfaultfd behavior is regression-tested separately. + +The first test revision incorrectly read approximate RSS counters and assumed +shared file PTEs were eagerly copied by fork. The corrected fixture sums the +counters and faults the child. A subsequent test lookup lacked its mmap lock; +that fixture warning is also corrected. The unrelated anonymous mlock-count +test now uses its existing all-CPU drain helper before exact observations, +instead of draining only the CPU on which the test resumes. This changes no +production LRU batching. Original failures remain in the evidence under +user4k-vm-state/coarse-file-v1/. + +Target-order shmem cache promotion groundwork (2026-10-02) +-------------------------------------------------------- + +The file-cache collapse implementation now accepts an explicit target order. +Existing native PMD callers still request their original order. A new internal +``collapse_file_user_page()`` entry point can promote shmem cache backing to +one larger-than-native user page. It reuses the existing isolation, unmap, +reference checks and rollback sequence; this is not yet connected to the +ordinary fault path or public larger-granule exec selection. + +A sub-PMD promotion allocates through shmem's existing inode policy lookup, +zone restrictions and GFP limiting, then charges the destination to the supplied +mm. Native PMD allocation remains unchanged. Sub-PMD folios do not increment +PMD THP counters or retract PMD page tables. The inode's transparent-huge-page +policy is unchanged: its maximum order can be zero when THP is disabled, which +is distinct from the backing required by an explicitly larger user leaf. +The new entry point currently requires THP build support and shmem; filesystem +writeback and no-THP integration remain unfinished. + +An internal 64K-on-16K fixture starts with four separate dirty native folios +in the middle of a larger file and active 4K aliases. Promotion preserves every +byte, then both coarse and fine mappings reference the new physical backing. +A writable pin in the middle forces rollback, including earlier isolated +folios: every original cache PFN and the pinned modification remain intact. +After releasing the pin, promotion succeeds and includes that modification. +An already suitable folio is an idempotent success; unrelated file ranges, +cache page counts and final mapping/reference cleanup are checked. + +Both main KUnit matrices pass on the final Images; the coarse test is skipped +on native64. The native16 userspace PMD collapse contract and native64 mixed +userfaultfd contract pass after the refactor. Feature-off arm64, x86 and no-THP +object builds pass. The initial new maximum-order check wrongly rejected +THP-disabled shmem and is removed; that failed run is retained in the evidence. +NUMA and zone policy use is established by code inspection and compilation, +not a new multi-node hardware test. Fault retry/error handling, sparse and +swapped promotion, concurrent mutation, failure injection and full ABI +qualification remain open. Evidence: user4k-vm-state/coarse-promote-v1/. + +Automatic coarse shmem faults (2026-10-02) +--------------------------------------- + +Shmem faults now promote undersized backing for an internal larger user leaf, +then reacquire the cache folio and complete the ordinary read/shared-write or +private-COW path. No explicit promotion call is required by the faulting +process. The original folio is unlocked and released before promotion. This +sub-PMD shmem path can run under the fault lock because it does not retract +page tables or invoke filesystem writeback, and trylocks additional source +folios with rollback on contention. Bounded tests run this path with lockdep. +That is not proof of every reclaim or concurrent mutation interleaving. + +A nonblocking fault returns RETRY with the fault lock retained and does not +allocate cache backing. Transient promotion contention returns NOPAGE, leaves +the lock with the caller and permits a subsequent refault. Each invocation +performs at most one promotion if splitting or truncation races the result. +Allocation failure maps to OOM, copy poison to HWPOISON. Unexpected retained +references return EBUSY from the promotion helper and currently SIGBUS from +the larger fault, preserving the original cache and pin ownership. Releasing +the reference permits a later successful fault. This is an explicit limitation +of the experimental larger ABI, which is still unavailable through public exec; +further retained-reference/error-policy qualification remains required. + +A required user leaf populates its entire granule, including cache holes, +consistent with existing coarse aliases. Another process's missing-page +registration cannot turn this required backing preparation into optional THP +collapse. The registration-based hole-preservation rule remains for native PMD +collapse. Fine aliases subsequently observe the populated coarse page and must +not overwrite its contents as an unpopulated leaf. The targeted unit test models +the registration flag, prepares real partial-validity shmem metadata, and checks +that existing 4K data survives while the other bytes become zero and populated; +it does not deliver an actual cross-process UFFD event for an EL0 coarse process. + +New internal64-on-native16 cases cover fresh and already populated shared +faults, private first-write COW, logical RSS and cache references, data at the +end of the leaf through access_remote_vm, nonblocking no-allocation behavior, +a held source-folio lock, retained writable PIN rejection and recovery, and +missing-page registrations within and adjacent to the coarse file range. +The held-lock case is a controlled unit fixture, not a concurrent stress test. +All new cases and both main KUnit matrices pass on the final Images. Native16 +userspace PMD-collapse and native64 mixed-userfaultfd contracts pass; native +feature-off, no-THP and x86 affected object builds pass. The initial hole test +reproduced an incorrect adjacent-registration SIGBUS before distinguishing +required user-leaf population from optional collapse. + +Evidence is under user4k-vm-state/coarse-fault-v1/. No new allocation-failure +injection, swapped promotion, coarse reclaim/migration, independent concurrent +mutation or hardware validation is claimed. Ordinary filesystem promotion, +no-THP coarse backing, special mappings and public larger-granule exec remain +unfinished. Native-default and smaller-process selection remain unchanged. + +Cold shmem reclaim and promotion failures (2026-10-02) +---------------------------------------------------- + +A dedicated disk-backed KUnit runner now reclaims both separate native file +folios and an already-promoted 64K shmem folio on a 16K kernel. It requires +file-cache swap entries and absence from swap cache before faulting again, +then checks the major-fault result, every 4K data chunk, coarse/fine alias +physical identity, cache/swap counts and final mapping/reference cleanup. +The primary16 configuration has no THP_SWAP, so reclaim of the promoted folio +also exercises the path requiring a split before swapout. + +The new test exposed a file-reclaim unit mismatch. Removing a coarse PTE +subtracted one mapping/reference even though that PTE owned four native-page +units. Three stale mappings prevented reclaim and eventually triggered the +filemap_unaccount_folio assertion during inode cleanup. File unmap now scales +rmap and folio-reference removal by the native pages per user leaf; logical +RSS remains measured in user PTEs. Batch limits now use the target mm's page +table boundary and page shift. Native/hugetlb ownership remains unchanged. +This fixes file reclaim; larger anonymous swap and migration still need their +own ownership audit and runtime tests. + +The same runner tests three allocation failures around the shmem fault +callback: one-shot slab failure for promotion metadata, task-filtered rejection +of the required order-2 allocation, and an actual zero-limit memory cgroup. +For the charge case, the private test inode restricts reclaim before any cache +population, and the KUnit worker temporarily joins the zero-limit cgroup. This +makes charge failure nonblocking and avoids an OOM kill. The worker returns to +root before assertions and retry. Each case preserves the original cache PFNs, +all data, cache-page count and zero user RSS, then succeeds after removing the +failure condition. Calling the fault callback directly isolates backing +allocation from page-table allocation; it is not a complete EL0 OOM test. + +Current primary VM configurations enable FAIL_PAGE_ALLOC as well as FAILSLAB. +Injection remains disabled by default and is scoped to the selected test worker; +the fixture disables injection after each case. The selftest runner refuses +ordinary host execution and uses a fresh private virtio swap disk in the VM. + +The original promoted-folio run fails with the captured three-map leak and +cleanup BUG. After the fix, both cold-reclaim cases and all three failure modes +pass in the dedicated suite. Main KUnit matrices and feature-off/no-THP/x86 +rmap object builds also pass. Evidence is under user4k-vm-state/coarse-swap-v1/. +These bounded results do not qualify larger anonymous swap, file migration, +concurrent pressure, ordinary filesystem promotion, public coarse exec or +hardware performance. The full mixed-page ABI goal remains in progress. + +Migration of larger base user leaves (2026-10-02) +----------------------------------------------- + +Migration now drops and restores one reference per native page covered by a +base user PTE. The reverse-map operations already handled that full span, but +try_to_migrate_one dropped only one reference and remove_migration_pte restored +only one. Those extra references prevented the migration reference-count check +from succeeding. Hugetlb keeps its existing reference convention; smaller +anonymous mappings retain their separate subpage migration path. Native and +smaller file PTEs have a native-page multiplier of one. + +Two negative controls reproduced this failure before each correction. The +first could not migrate either a clean 128K file folio mapped through two 64K +leaves or a dirty, promoted 64K shmem folio at a nonzero file offset. After the +file correction, another control still could not migrate private or fork-shared +64K anonymous backing. Both controls exited through normal test shutdown; +rollback retained the original PFNs and data without KASAN or lockdep reports. + +The final native16 tests pass six file variants and five anonymous variants. +File cases retain simultaneous 64K, native16K and shifted 4K aliases crossing +a small-page table boundary. They verify migration to different backing, +permissions, every data chunk, native-unit references/mapcounts, logical RSS, +and exact cleanup. A retained GET or typed PIN forces rejection without losing +ownership or data. Anonymous cases additionally verify exclusivity on every +native constituent and full-leaf COW after restoring fork-shared mappings. +Writable pins on the final byte of a coarse anonymous leaf prevent migration. +These are bounded in-kernel cases, not a claim of arbitrary concurrent races. + +Both primary Images build without compiler diagnostics. Final main KUnit +results are native16: 127 pass, 16 skip; native64: 117 pass, 26 skip, with no +failures. Larger-than-native cases intentionally skip on native64. Existing +TLB, data and migration suites pass. Feature-disabled arm64, no-THP arm64 and +x86 migration/rmap object builds pass. The file correction also passed the +cold-shmem swap and allocation-failure runner before the anonymous extension. +Evidence, negative-control Images, final Images and logs are retained under +user4k-vm-state/coarse-migrate-v1/. + +This closes the demonstrated migration ownership mismatch. Larger anonymous +swap is still separate: its swap-count acquisition, PTE replacement, release, +and fault-in paths need full-span handling. Ordinary filesystem backing +promotion, no-THP coarse backing, special mappings and public larger-granule +exec remain open. The default native ABI and the existing opt-in smaller ABI +remain unchanged; these tests do not establish full ABI or hardware qualification. + +Anonymous swap for larger base user pages (2026-10-02) +---------------------------------------------------- + +An internal larger user PTE now owns an aligned group of native swap entries. +Swap-out acquires every constituent count atomically within the cluster; +fork duplicates that group, and unmap releases it. Reverse-map and folio +references use native-page units while RSS and swap-PTE counters retain the +process's logical page units. Native and smaller-page paths retain their +existing ownership conventions. + +Anonymous lineages record an immutable minimum backing order from their mm. +Mapped anonymous backing cannot split below that order. The query validates +its anon_vma through RCU and a temporary reference, including callers that +have not locked the folio; the actual split rechecks under the anon_vma lock. +Mandatory larger base-page backing can allocate grouped swap even when optional +THP_SWAP is disabled. This does not enable general native THP swap: native-page +lineages still have minimum order zero. The non-THP swap cluster size remains +256 native entries; the allocation-order arrays can now represent the mandatory +larger allocation. Group allocation can still fail under fragmentation or limits. + +Swap-in reuses suitably aligned contiguous cache backing. If the entries have +been read into separate native cache folios, it allocates and charges a complete +user-page-sized folio and copies every constituent with the architecture's +page-copy and swap-restore hooks. The PTE is revalidated before installing the +complete mapping. If a later cache folio is busy, all earlier locks are released +before waiting and retrying the group. Swapoff uses the same restoration path. +Allocation failure leaves the original swap PTE and all its counts intact. + +The disposable native16 disk runner passes seven larger-anonymous variants: +warm exclusive reuse, cold real reclaim and readback, fork before cold reclaim, +fork after swap-out followed by swapped-child exit, cold swapoff, injected +contiguous-allocation failure and recovery, and a write fault on warm shared +cache backing. Tests check every 4K data chunk, per-entry counts, logical RSS, +minimum-order split rejection, full-leaf COW and parent isolation, and exact +folio references after MM teardown. The three existing coarse-shmem cases also +pass, including their allocation/charge failure modes. Console and debugfs +copies of the results represent one execution, not independent repetitions. + +The first candidate's test queried a fully freed entry with swp_swapcount, +whose caller must retain valid slot ownership. It crashed in that assertion; +it does not establish a production swap failure. The test now uses the safe +freed-slot query. The next candidate returned for refault when a later cache +piece was still under I/O; the bounded direct-fault test required a completed +mapping. Waiting with all earlier locks released resolved that test failure. +Final primary Image builds have no compiler diagnostics. Main KUnit runs pass +on native16 and native64; feature-off arm64, no-THP arm64 and x86 object builds +pass. Existing EL0 swap regressions also pass for native/4K on native16 and +native/4K/16K on native64, including real swapped-PTE evidence, disk readback +and fork/COW over four 8MiB rounds per ABI. Evidence is retained under +user4k-vm-state/coarse-anon-swap-v1/. + +These bounded results supersede the earlier unconverted larger-anonymous-swap +limitation. They do not qualify exhausted or fragmented swap, concurrent +fault/swapoff races, coarse swap-charge failure, tagged/poisoned backing, or +no-THP runtime behavior. Public larger-page exec, ordinary filesystem promotion, +special mappings and the broader ABI qualification remain unfinished. The +native default and existing opt-in smaller process ABIs remain unchanged. + +Larger-page swap limits and overlapping faults (2026-10-02) +--------------------------------------------------------- + +A fragmentation test found that a complete free, aligned user-page-sized group +could still be inaccessible because its cluster belonged to another allocation +order. Optional THPs can split on that failure; mandatory larger base pages +cannot. A new fallback scans for genuinely free aligned groups after the normal +allocator fails. It pins the available devices in priority order, scans outside +the per-CPU and device allocation spinlocks, and can reschedule between clusters. +A successful allocation changes the cluster's preferred allocation order and +moves its list membership while retaining all existing slot owners. Native and +optional-THP allocations retain their previous allocation paths. The fallback +skips contended clusters and does not reclaim cache entries merely to reclassify +a cluster; it can still fail when no suitable free group is available. + +The negative control exhausted a private 128MiB swap device, then freed all but +one native slot per aligned 64K group. The live 64K mapping retained its backing +and bytes through failed allocation and real reclaim attempts. Freeing a complete +aligned group near the device's end still returned ENOMEM before the fallback. +The final test allocates exactly that group, verifies the remaining native slot +counts, and completes a cold disk round-trip with exact final folio references. +Synthetic slot reservations exercise the real allocator and are released on test +cleanup; they are not an application-level whole-system pressure workload. + +A separate cgroup test sets memory.swap.max to zero while allowing memory +allocation. It checks the folio's actual cgroup, observes real swap-limit events, +and verifies that failed swap charging and repeated reclaim preserve the mapping, +64K data, page order and logical accounting without leaking swap charge. Lifting +the limit permits swap-out, cold readback and cleanup. This covers swap-out charge +rejection; it does not establish coarse swap-in memory-charge failure behavior. + +Eight controlled overlap rounds use two CPU-bound kthreads borrowing separate +64K mms and performing actual copy_from_user/copy_to_user operations in the VM. +They overlap a third worker's swapoff. Holding a cache folio blocks completion; +the test observes swapoff stop accepting allocations and all three workers remain +incomplete before releasing that lock. Four rounds retain a warm full-size folio; +four hold the last native piece of cold, fragmented cache backing. Each round +checks complete bytes, private COW writes, accounting, cleanup and swap reactivation. +These exercise arm64 translation and uaccess in QEMU, not Apple hardware, and +cover the stated synchronization scenario rather than all possible interleavings. + +The expanded disposable suite passes all seven cases on the final native16 +Image. Main KUnit suites pass on native16/native64, the native16 native/4K EL0 +swap regression passes, and feature-off arm64, no-THP arm64 and x86 swapfile object +builds pass without compiler diagnostics. Evidence, the failing pre-fix Image and +final Images are under user4k-vm-state/coarse-swap-limits-v1/. + +A further inspected limit remains: larger swap allocations are rejected unless +the swap device has SWP_BLKDEV. Regular swap files do not receive that flag. +The fallback preserves this restriction. File extent layout and swap I/O must be +handled before lifting it. Multiple-device fallback, broader pressure/races, +coarse swap-in charge failures, tagged/poisoned memory, no-THP runtime, special +mappings and public larger-page exec remain unqualified or unfinished. These +results do not change the native default or qualify the complete process ABI. + +Larger base pages on fragmented swap files (2026-10-02) +------------------------------------------------------ + +Mandatory larger anonymous pages can now allocate grouped swap entries in +block-backed regular files. Optional THP allocation retains its existing policy; +filesystems using swap_rw retain their existing larger-allocation restriction. +The per-CPU fast path checks the same eligibility as slow allocation, so a cached +file cluster does not silently change optional THP allocation policy. + +Swap I/O now uses each extent's physical sector and length. A folio crossing +extent boundaries gets one bio per physical run, chained to the original bio. +The original folio completion runs only after every child completes; synchronous +submission uses bio_await with the same splitting function. A separate clone +bioset is prepared at swapon, avoiding a second allocation from the unsubmitted +parent's reserve. This reserve is initialized for block-backed regular files on +all configurations: existing native large-folio swap-in can also require extent +mapping. Extent metadata remains immutable while swap entries are owned. + +A disposable ext4 fixture writes a 32MiB file then collapses one native page out +of each four-page interval. On native16 it presents 512 swap extents. The previous +Image rejects the larger anonymous allocations, with four failing cases and +three passing shmem cases. The fixed Image passes all eight cases, including a +new full-64K read/write check in asynchronous and synchronous modes. That check +verifies a physical discontinuity after the first native page, distinct data in +every 4K chunk, complete mapping teardown and exact final references. The prior +cold native-piece readback, cgroup limits, exhaustion/fragmentation, fork/COW and +overlapping fault/swapoff tests also pass on this file. The block-device variant +passes all eight cases as well. + +Real EL0 swap-file tests pass four 8MiB pageout/readback/fork-COW rounds per ABI: +native16 and 4K on the native16 kernel, and native64, 4K and 16K on native64. +The final helpers swap off and unmount before powerdown; read-only e2fsck checks +of all three resulting ext4 images are clean. Earlier fixtures powered down +while mounted, so their non-replaying fsck output is not a filesystem-integrity +result. Both primary Image builds, main KUnit suites, and feature-off arm64, +no-THP arm64 and x86 swapfile/page_io object builds pass. Evidence is under +user4k-vm-state/coarse-swapfile-v1/. + +These results supersede the blanket block-device-only limitation above. They do +not qualify network/filesystem swap_rw backends, injected partial I/O failures, +clone-reserve exhaustion, multiple swap-device priority interactions, or all +filesystem implementations. Public larger-page exec remains gated; wider ABI, +hardware, pressure and performance qualification remains unfinished. Native +process defaults and the opt-in smaller-page selection policy are unchanged. + +Anonymous pressure with delayed swap I/O (2026-10-02) +---------------------------------------------------- + +A controlled write stall now reproduces premature memcg OOM with smaller-page +processes. Two workers each populate and verify 32MiB under a shared 48MiB +memory limit and 256MiB swap allowance. Limiting the disposable QEMU disk to +64KiB/s writes leaves almost the entire memory charge in swap writeback. The +unchanged Image fails twice for native16 plus 4K and once for native64 plus 4K. +The native16-only control waits in blk_mq_get_tag and survives; a native16 plus +4K control with classic reclaim selected explicitly also survives. Increasing +the native-only control's virtqueue size to 1024 still passes, so these results +do not establish a native-only reproduction or quantify the cause of different +block-layer queue occupancy. + +MGLRU had two obstacles to the existing legacy memcg writeback-wait policy: +sort_folio moved writeback folios into a younger generation before the common +reclaim path saw them, and isolate_folio cleared their reclaim marker. The fix +allows those folios through and preserves the marker during pending writeback. +shrink_folio_list then applies its existing GFP, mapping-deadlock and repeated- +writeback checks. No batch-size reduction, queue-depth reduction, fixed delay, +or disabled-MGLRU production workaround is introduced. The affected legacy +policy also applies to cgroup v2 with CONFIG_CGROUP_WRITEBACK disabled, as in +these validation profiles; configurations using the other writeback policy +retain their existing behavior. + +The first candidate changed only sorting and still failed after substantially +more scans. The final candidate also preserves the marker. Captured stacks now +show folio_wait_writeback from shrink_folio_list through MGLRU. With the write +limit held for the full 30-second interval, native16/4K, native64/4K and +native64/16K pairs complete four permuted-data and fork/COW rounds with zero OOM +or OOM-kill events. Anonymous and swap charges reach zero after exit and explicit +clean-cache reclaim. The root-cgroup observer records live charge, writeback, +scan/steal and event counters; the QMP runner records block statistics and releases +the cap at the deadline or upon a failed run's OOM to allow bounded cleanup. +These are emulated I/O conditions, not hardware measurements. + +Both final primary Images and main KUnit suites pass. Feature-off arm64, no-THP +arm64 and x86 vmscan object builds pass without compiler diagnostics. Unstalled +native16-only and native16/4K pressure regressions also pass on the final Image. +The selftest's optional classic mode is diagnostic only; its performance cost +has not been measured and it is not a proposed production configuration. The +final selftest formatting build is byte-identical to the tested executable. +Evidence is under user4k-vm-state/anon-pressure-io-v1/. + +This resolves the reproduced delayed-writeback OOM mechanism. The older isolated +failure's recorded counters are consistent with it, but the original log lacks +the blocked-stack evidence, so attribution of that exact historical event remains +an inference. This result does not establish all I/O-error, dead-storage, +writeback-controller, pressure or performance behavior. Broader ABI qualification, +legacy GUP/device paths and the other stated requirements remain active. + +IOMMU dirty-bitmap user buffers (2026-10-02) +------------------------------------------- + +The IOVA dirty-bitmap helper now pins alternative-process buffers as typed +fragments. Each descriptor retains its physical folio offset and exact byte +length, so relocated user leaves need not have matching virtual and physical +native-page offsets. One native page bounds the descriptor array; mapped IOVA +coverage derives from the sum of successfully pinned byte lengths. Writes split +at descriptor boundaries and advance to another window when necessary. Native +processes retain the native-page-array path. Both paths now dirty backing when +releasing write pins, including shared-file bitmap storage. + +The new relocated-buffer test first ran against the unchanged helper: the +native16 buffer was updated, while the 4K buffer remained zero. The helper's +existing void setter does not report pin failures to its caller; the output +comparison detects that failure. The updated helper passes on native16/4K and +native64/4K/16K mms. Additional tests cover: + +* A 4K bitmap exceeding the descriptor window (512 records on native16, 2048 on + native64), complete byte/guard verification, and a backwards revisit. +* Shared file-backed bitmap pins surviving unmap and replacement at the same + virtual address. Subsequent bitmap writes change only the retained file; + replacement bytes stay intact, pins release, and the file remains dirty. +* Partial pinning up to an unmapped hole, no writes beyond it, successful retry + after restoring the hole, and no modification through a read-only buffer. + +These tests call the actual IOVA bitmap library while borrowing isolated test +mms in the VM. They do not exercise an IOMMU hardware dirty walk, a userspace +IOMMUFD ioctl, physical DMA, or VFIO mapping. In particular, the existing void +setter's partial-failure behavior is preserved; full ioctl error propagation +and the remaining page-array DMA consumers remain separate work. + +The KUnit configuration selects IOMMUFD_DRIVER to link the library. Both final +Image builds and full suites pass, with 131 native16 and 121 native64 main-suite +passes and the existing explicit skips. Feature-off arm64, x86 and no-THP arm64 +object builds pass using real IOMMUFD plus SMMUv3/Intel-IOMMU selections in their +control configurations. The first native16 link required KALLSYMS_EXTRA_PASS=1 +after adding the library; subsequent builds use that linker option. This was a +build-time kallsyms convergence issue, not a runtime kernel failure. Evidence is +under user4k-vm-state/iova-bitmap-v1/. No hardware or performance claim is made. + +IOMMUFD data mappings and pin ownership (2026-10-02) +-------------------------------------------------- + +Alternative user mappings now use exact fragment pins in IOMMUFD's data-mapping +path. Each provider and area records its indexing granule independently of +native physical-page accounting. Anonymous and file-backed user addresses retain +physical folio offsets and allocation identities in a provider-owned xarray; +device translations alone are not used as ownership records. Native user, file +and dma-buf providers retain the existing native PFN reader. + +The new reader batches long-term pins, merges physically contiguous fragments +for IOMMU mapping, and reuses cached records across overlapping IOAS copies and +multiple domains. Each distinct native physical page is charged once within the +provider. Every device mapping is removed before its last fragment pin is +released. Unsupported physical/device alignment returns an error without widening +the mapping into adjacent storage. Legacy splitting still limits individual +mappings to native-page spans. Local CPU copies retain their existing fast path; +remote/domain-backed copies use the retained fragments. Native page-array access +requests for an alternative provider return EOPNOTSUPP: that interface cannot +encode arbitrary fragment offsets and ownership. + +Five KUnit cases exercise the actual IOMMUFD map/copy/domain/unmap APIs with a +translation-recording mock IOMMU. Coverage includes relocated anonymous physical +addresses, contiguous mapping batches, overlapping copies, two domains, VA +replacement, remote and local CPU-copy semantics, partial pin failure, read-only +failure, partial device-map failure, stricter device alignment, legacy splitting, +the maximum IOVA boundary, shared-file offsets and dirtying, and originator exit. +The lifecycle case passes for native16/4K/internal64K mms and native64/4K/16K mms. +The internal larger-granule fixture does not remove the public larger-exec gate. + +The exit test reproduced stale VFIO-compatible pin accounting: after the source +mm's users reached zero, final unpin left two native pages charged because the +accounting helper required mmget_not_zero even for a decrement. The provider's +mmgrab already retains the lock and counters. Decrements now finish using that +reference; increases still require a live mm. The before-fix test fails the pin, +locked-memory and cached-accounting counters. The final test passes for native +and 4K mms on both kernels. + +The final native16 and native64 main suites report 136 and 126 passes respectively, +zero failures and the existing explicit skips. Both Image builds and IOMMUFD +object builds for feature-off arm64, no-THP arm64 and x86 pass. The KUnit selector +now links full IOMMUFD. Evidence is under user4k-vm-state/iommufd-fragments-v1/. +These are kernel API and simulated-translation tests, not physical-device DMA or +userspace ioctl qualification. Memlock-limit failure injection, wider concurrent +IOMMUFD access, typed software-access consumers, legacy VFIO type1, hardware and +performance testing remain open. No throughput result is claimed. + +IOMMUFD userspace ioctl limits and dirty errors (2026-10-02) +--------------------------------------------------------- + +The disposable iommufd-init fixture now runs real IOMMUFD ioctls from native +and alternative EL0 processes against the kernel's generic AMDv1 mock IOMMU. +The ARM64_USER4K_IOMMUFD_TEST option selects the required test-only page-table +and mock-device configuration. It defaults off. The mock address checker uses +exact fragment offsets for alternative mms instead of native-page GUP. + +For USER and MM accounting, the fixture drops CAP_IPC_LOCK and sets tight +RLIMIT_MEMLOCK limits. It verifies over-limit failure without stale charges, +overlapping IOAS copies, late domain attachment, relocated physical addresses, +replacement at the same VA, local and retained remote copies, mapped holes, +read-only backing, shared-file offsets and guard bytes. Cross-process tests +exercise both directions between native and smaller mms, retained pins after +source exit, user-wide versus per-mm limits, and successful remapping after +uncharge. Four threads perform 256 map/check/unmap cycles per accounting mode. +A fail-nth sweep injects slab, page-allocation and IOMMUFD failures until the +operation stops consuming injection sites. Every iteration checks charges and +retries successfully against both mock domains. The observed number of sites +varies with allocator state and does not represent a fixed coverage count. + +A fresh alternative mapping at a shifted native-page offset can require three +native backing pages even when an earlier relocated mapping used two. The test +checks that exact charge rather than treating the extra physical backing as an +accounting leak. This is allocation geometry, not a change to the memlock policy. + +Dirty-bitmap ioctl tests reproduced five false-success cases on both kernels: +an unmapped IOVA, an invalid output pointer, a non-power-of-two bitmap granule, +a read-only output buffer, and a hole after a valid bitmap prefix. The bitmap +helper now retains the first lazy pin/write error and returns it from its +iteration API. IOMMUFD returns that iteration result and the actual allocation +error, and requires a power-of-two bitmap granule. The void setter remains +available to drivers; the enclosing iteration reports failures. Prefix writes +are not undone. NO_CLEAR reads retain dirty state for a retry after repairing +the buffer; ordinary read-and-clear errors are not transactional rollback. + +The final complete ioctl contract passes on native16/4K and native64/4K/16K. +It also checks exact dirty bits across a user-leaf boundary, bitmap guard bytes, +NO_CLEAR retries, successful clearing and the expected error codes. Final main +KUnit suites still report 136 native16 and 126 native64 passes, zero failures, +and the existing skips. Feature-off arm64, no-THP arm64 and x86 IOMMUFD object +builds pass. Evidence is under user4k-vm-state/iommufd-ioctl-v1/. +These are real ioctl tests using a mock IOMMU; physical DMA, hardware queues, +legacy page-array consumers and the broader ABI requirements remain open. + +IOMMUFD hardware queue physical byte spans (2026-10-02) +----------------------------------------------------- + +The vIOMMU hardware-queue path now pins the exact physical byte span requested +by its ioctl. An internal byte-span access has byte IOVA alignment, validates +physical contiguity inside each provider and across mapping boundaries, and +retains access interval references until queue destruction. It obtains the +physical starting offset from the retained pin records. Alternative user leaves +retain their typed allocation identities; native providers retain the existing +page supplier and pin accounting. No adjacent IOVA mapping is required merely +to round a small queue up to the native kernel page size. + +The access increments each affected area's unmap lock and rolls back acquired +accesses if any later area is missing or physically discontinuous. The queue's +driver receives the exact first physical byte and retains responsibility for +its hardware-specific address and size constraints. Queue destruction stops +hardware through the existing driver destroy callback before releasing the +exact access range. The ordinary native-page-array API is still unsupported +for arbitrary alternative providers; this change converts the physical-span +consumer rather than substituting a page pointer for a fragment. + +The userspace fixture exercises the actual IOMMU_HW_QUEUE_ALLOC path and the +existing mock vIOMMU queue callback, which compares its received physical address +against the parent IOMMU translation. Native queues passed the before Image, +while queues using alternative providers failed either attachment alignment or +native page-array pinning. The updated path passes native16/4K and +native64/4K/16K, for USER and MM accounting. Tests cover: + +* A relocated leaf mapped by itself, with an unaligned queue strictly inside it. +* Missing predecessor and duplicate queue indexes, including callback rollback. +* Overlapping queues, source VMA replacement, dependency-ordered destruction, + and EBUSY while an access prevents IOAS unmap. +* Retained native/alternative provider memory after the originating process exits. +* A hole after an acquired first area, discontinuity across separate providers, + and repeated physical bytes inside one provider, with no stale unmap locks. +* A physically contiguous shared-file span crossing two distinct mapped areas. +* Allocation-failure sweeps during queue creation, successful retry, and zero + residual pins or locked-memory charges after each unmap. + +Both final Image and main KUnit suites pass with the prior counts and skips. +Feature-off arm64, no-THP arm64 and x86 IOMMUFD object builds pass. Evidence is +under user4k-vm-state/iommufd-queue-v1/. These are real queue ioctls using a mock +vIOMMU. Physical-device queue execution, performance and the remaining VFIO +consumers remain unqualified. + +IOMMUFD native page arrays from alternative providers (2026-10-02) +---------------------------------------------------------------- + +The software page-array access path now accepts alternative user providers +when their retained physical layout can be represented by native page pointers. +Each output page must start at a native physical-page boundary, with consecutive +logical fragments within the requested part of that page. Different output +pages may be scattered or aliased. Misaligned physical bases and discontinuity +inside an output page return EINVAL; no adjacent IOVA or backing bytes are +implicitly added. A caller must access only the requested byte length, including +when the final output page is partial. The external access retains its existing +native IOVA alignment and unmap callback contract. + +Alternative access intervals keep the canonical typed PIN records and their +allocation identities. Page pointers do not replace that ownership. Exact +intervals share access users; overlapping intervals retain independent users. +Final access removal releases only records no longer needed by any access or +IOMMU domain. Output advancement across areas counts native output pages rather +than provider logical indices. The private hardware-queue byte-span path uses +the same access registration and retains its full-span contiguity requirement. +Native providers keep their existing supplier, accounting and batching behavior. + +The real EL0 ioctl fixture checks USER/MM accounting for native16/4K and +native64/4K/16K: lazy pinning without a domain, distinct mapped areas, duplicate +and overlapping accesses, partial final pages, source VA relocation, native-page +aliases, callback-driven unmap and rollback after a missing or incompatible +second area. Typed GET records validate every returned alternative fragment's +physical address independently of its VA. Anonymous source processes of a +different granule exit while access pins remain; retained memory is readable, +new accesses reuse it, and final release refunds the dead source account. +Allocation-failure sweeps check zero residual charges and successful retry. +Three pin threads race 256 unmap/remap iterations with the existing selftest +access callback, checking successful pins, permitted transient errors and final +zero charges. The prior dirty bitmap, hardware queue and DMA ioctl tests run too. +Before Images passed native cases and rejected alternative page-array buffers +with EOPNOTSUPP. Updated Images pass the tested representable layouts. + +Evidence is under user4k-vm-state/iommufd-pages-v1/. This qualifies the IOMMUFD +API used by VFIO's modern software pin path, using the existing selftest access +consumer. It does not qualify an actual VFIO device, hardware DMA, hardware queue +execution or throughput. Arbitrary fragment layouts still require consumers +that carry explicit physical offsets and lengths. Legacy VFIO type1's direct +GUP path and the wider mixed-page ABI requirements remain open. + +Mixed-granule runtime without THP (2026-10-02) +-------------------------------------------- + +MM_SUBPAGE now selects XARRAY_MULTI independently of transparent huge pages. +A no-THP boot exposed three failed generic-file coarse-leaf tests: their +preallocated order-3 folio had eight native cache pages, but lookup at a later +index returned ENOENT. Without multi-index XArray support, insertion did not +cover the folio's native index range. Those coarse faults returned SIGBUS despite +already suitable physical backing. The missing configuration dependency is +fixed without enabling THP or changing its allocation policy. Temporary test +instrumentation was removed after capturing the allocation/insertion/lookup. + +The native16 no-THP main KUnit result changes from 123 passes, three failures, +and 23 skips to 126 passes, zero failures, and 23 skips. Native64 no-THP reports +121 passes, zero failures, and 28 skips. Both have 149 compiled cases; the +THP-enabled configuration has 152. Other enabled mixed-granule suites pass +with their explicit skips. Full real IOMMUFD ioctl contracts pass native16/4K +and native64/4K/16K in USER and MM accounting modes. The shared userfaultfd +contract passes on both no-THP kernels. AArch32 auto-4K and capability-removal +fallback to native16/native64 also pass; native16 used the initial no-THP Image +and native64 the final one. This is a boot capability model, not Apple A9 +hardware testing. + +Native16-only, mixed16/4K, native64-only, mixed64/4K and mixed64/16K +workers pass bounded no-THP pressure with two 32 MiB anonymous mappings sharing a 48 MiB memory limit and private swap. +They verify every byte over repeated swap-in and fork COW, report nonzero +swap, limit hits, scanning and reclaim, zero OOM kills, and zero final anonymous +and swap charges. This run uses ordinary guest swap I/O, without the separate +delayed-writeback injection. The pressure fixture now accepts EINVAL from +MADV_NOHUGEPAGE only when the THP sysfs interface is absent; otherwise its +existing failure check remains. An older packaged fixture failed before any +worker data was reported and includes a hard-coded THP sysfs path; its exact +failing instruction was not captured. These early fixture failures do not +establish anonymous memory corruption. + +Evidence is under user4k-vm-state/nothp-runtime-v1/. This extends execution +coverage for smaller user leaves without THP and fixes indexing of already +provided larger file-cache backing. Automatic coarse shmem promotion still +requires the existing collapse implementation, and public larger-than-native +exec selection remains gated. This is not general no-THP filesystem promotion, +physical-device qualification, performance measurement or complete ABI readiness. + +Legacy VFIO accounting lifetime (2026-10-02) +------------------------------------------ + +Type1 retains an mm_count reference for each DMA mapping. Final pin refunds +now use that retained mm even after its last user exits; they do not attempt +to resurrect mm_users. Refund locking cannot be interrupted after the pin +has already been released. A new asynchronous charge still requires a live +source mm. DMA owner changes credit the new account before debiting the old +one, and refund the old account even when the old process has exited. A +failed new charge leaves the existing owner and counters unchanged. Charges +remain native physical-page counts, converted by the common mm accounting +helpers independently of the user translation granule. + +A private KUnit suite exercises the actual type1 accounting helpers and real +mm allocation/exit. Native16 and native64 guests check native and alternative +4K/16K/64K account geometries, source exit, partial/final refunds, failed +owner credits, transfers in both directions and same-mm idempotence. Matched +controls with the previous accounting behavior reproduce stale exit and +owner-transfer charges; the fixed implementations pass. This is metadata +lifetime coverage, including internally modeled coarse geometries, rather +than public coarse exec selection or real fragment pinning. + +Evidence is under user4k-vm-state/vfio-account-v1/. Type1's mapping, replay, +external native-page array and physical-device DMA paths still need their +retained fragment-provider integration. These accounting tests do not +establish legacy VFIO DMA support for an alternative-page process. + +GEM/PRIME user-granule matrix (2026-10-02) +--------------------------------------- + +The existing VGEM real-userspace fixture now sizes its buffer for eight +native pages and selects user granules from the native kernel size. Its +previous fixed 64 KiB buffer was too short for several native64 test ranges. +Native16 runs native16/4K; native64 runs native64/4K/16K. Each ABI passes +the existing twelve checks: full/partial GEM and PRIME mappings, distinct +backing bytes, cold faults after protection changes, partial unmap and +fixed mremap, fork, bounds and descriptor/object lifetime. PRIME sharing +now checks both directions for every pair present in the guest, verifies +the exec child's advertised user page size and confirms changed bytes in +the parent mapping. The native16 buffer is 128 KiB; native64 is 512 KiB. + +The latest primary Images pass 24 checks on native16 and 36 on native64, +including six directed PRIME pairs in the latter. No new kernel mapping +fix was needed for these checks. Evidence is under +user4k-vm-state/gem-granule-v1/. This extends common GEM shmem/PRIME CPU +mapping coverage using VGEM. It does not execute Asahi GPU commands or +qualify imported buffers from other exporters, hardware DMA, Vulkan/GL, +Windows games or performance. diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index a582d84151537..31b402de5245b 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -362,6 +362,7 @@ config FIX_EARLYCON_MEM config PGTABLE_LEVELS int + default 4 if ARM64_USER4K_EXPERIMENTAL default 2 if ARM64_16K_PAGES && ARM64_VA_BITS_36 default 2 if ARM64_64K_PAGES && ARM64_VA_BITS_42 default 3 if ARM64_64K_PAGES && (ARM64_VA_BITS_48 || ARM64_VA_BITS_52) diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index f873818825d2d..9d7742848b5ac 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -22,6 +22,84 @@ config ARM64_RELOC_TEST source "drivers/hwtracing/coresight/Kconfig" +config ARM64_USER4K_EXPERIMENTAL + bool "Experimental mixed-granule userspace" + select ARCH_HAS_MM_PGTABLE + select MM_SUBPAGE + select MM_ID + select HAVE_ARCH_TLB_REMOVE_TABLE + select ARCH_SUPPORTS_PT_RECLAIM + depends on (ARM64_16K_PAGES || ARM64_64K_PAGES) && ARM64_VA_BITS_48 && ARM64_PA_BITS_48 + depends on !ARM64_LPA2 && !NO_PAGE_MAPCOUNT + default n + help + Build the in-development MM support for alternative 4K user tables + on 16K/64K kernels and 16K user tables on 64K kernels. + Ordinary address spaces and kernel tables retain the native granule. An experimental + one-shot prctl selects a supported granule for the next exec. + This is a development ABI: several MM interfaces and device mappings + still require integration and validation. The private prctl numbers + are not upstream assignments or a stable ABI promise. + Say N for normal kernels. + +config ARM64_USER4K_KUNIT_TEST + bool "Test mixed userspace granules with larger kernel pages" + depends on KUNIT=y && ARM64_USER4K_EXPERIMENTAL + depends on ARM64_PA_BITS_48 + select ARM64_USER4K_DATA_KUNIT_TEST + select IOMMUFD + select IOMMUFD_DRIVER + default n + help + Exercise independent TTBR0 translation granules, subpage permissions + and invalidation while TTBR1 retains the normal native kernel tables. + Hardware translation/uaccess fixtures require arm64.user4k_test=1 + as a second opt-in and are disabled with KASAN. Software MM fixtures + can run with KASAN without borrowing or replacing TTBR0. + Run only in a disposable test VM. This regression harness does not + establish hardware qualification or production ABI readiness. + Say N for normal kernels. + +config ARM64_USER4K_IOMMUFD_TEST + bool "IOMMUFD ioctl harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && DEBUG_KERNEL + depends on FAULT_INJECTION && RUNTIME_TESTING_MENU + select GENERIC_PT + select IOMMU_PT + select IOMMU_PT_AMDV1 + select IOMMUFD + select IOMMUFD_TEST + default n + help + Enable the IOMMUFD mock-device ioctls and generic AMDv1 page tables + for the mixed-granule userspace ioctl fixture. This exposes privileged + test interfaces; use only in a disposable test VM. + Say N for normal kernels. + +config ARM64_USER4K_VFIO_ACCOUNT_TEST + bool "VFIO type1 accounting harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y + select VFIO + select VFIO_GROUP + select VFIO_CONTAINER + select VFIO_IOMMU_TYPE1 + select VFIO_TYPE1_ACCOUNT_KUNIT_TEST + default n + help + Enable built-in VFIO type1 and its source-mm accounting tests in + a disposable mixed-granule test guest. This does not establish + physical-device DMA or alternative-page pinning support. + Say N for normal kernels. + +config ARM64_USER4K_DATA_KUNIT_TEST + bool "Test packed userspace allocation ownership" + depends on KUNIT=y && ARM64_USER4K_EXPERIMENTAL + default n + help + Exercise packed allocation, slot reuse, copying and deferred lifetime. + These tests use native kernel addresses without changing TTBR0, and + can run with KASAN even when the architectural tests are disabled. + config AURORA_EL1_SYSREG_PROBE bool "Aurora bounded EL1 sysreg read diagnostic" depends on ARCH_APPLE && DEBUG_KERNEL diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index d0d3cdd5763ca..fff3a9570ad65 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -19,6 +19,11 @@ cpucap_is_possible(const unsigned int cap) "cap must be < ARM64_NCAPS"); switch (cap) { + case ARM64_HAS_USER16K_GRANULE: + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && + IS_ENABLED(CONFIG_ARM64_64K_PAGES); + case ARM64_HAS_USER4K_GRANULE: + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL); case ARM64_HAS_EPAN: return IS_ENABLED(CONFIG_ARM64_EPAN); case ARM64_SVE: diff --git a/arch/arm64/include/asm/elf.h b/arch/arm64/include/asm/elf.h index d2779d604c7be..23320b49c3208 100644 --- a/arch/arm64/include/asm/elf.h +++ b/arch/arm64/include/asm/elf.h @@ -121,6 +121,9 @@ #define CORE_DUMP_USE_REGSET #define ELF_EXEC_PAGESIZE PAGE_SIZE +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define ELF_EXEC_PAGESIZE_MM(mm) mm_page_size(mm) +#endif /* * This is the base location for PIE (ET_DYN with INTERP) loads. On @@ -181,6 +184,12 @@ do { \ NEW_AUX_ENT(AT_IGNORE, 0); \ } while (0) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +struct linux_binprm; +int arm64_bprm_prepare_mm(struct linux_binprm *bprm, bool compat); +#define arch_bprm_prepare_mm arm64_bprm_prepare_mm +#endif + #define ARCH_HAS_SETUP_ADDITIONAL_PAGES struct linux_binprm; extern int arch_setup_additional_pages(struct linux_binprm *bprm, diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h index 229ee7976f693..2bd73e773095c 100644 --- a/arch/arm64/include/asm/kernel-pgtable.h +++ b/arch/arm64/include/asm/kernel-pgtable.h @@ -27,7 +27,7 @@ #endif #define SWAPPER_BLOCK_SIZE (UL(1) << SWAPPER_BLOCK_SHIFT) -#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - SWAPPER_SKIP_LEVEL) +#define SWAPPER_PGTABLE_LEVELS (ARM64_NATIVE_PGTABLE_LEVELS - SWAPPER_SKIP_LEVEL) #define INIT_IDMAP_PGTABLE_LEVELS (IDMAP_LEVELS - SWAPPER_SKIP_LEVEL) #define IDMAP_VA_BITS 48 diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 5e1211c540ab2..15943bec53392 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -8,6 +8,10 @@ #include #define MMCF_AARCH32 0x1 /* mm context flag for AArch32 executables */ +#define MMCF_USER_4K 0x2 /* experimental, immutable mm table geometry */ +#define MMCF_USER_16K 0x4 /* experimental, immutable mm table geometry */ +#define MMCF_USER_64K 0x8 /* internal geometry; larger user ABI not enabled */ +#define MMCF_USER_GRANULE (MMCF_USER_4K | MMCF_USER_16K | MMCF_USER_64K) #define USER_ASID_BIT 48 #define USER_ASID_FLAG (UL(1) << USER_ASID_BIT) @@ -16,6 +20,15 @@ #include #include +/* AArch32 falls back to the native ABI when 4K translation is unavailable. */ +static inline unsigned int arm64_exec_page_shift(bool compat, bool has_4k, + unsigned int native_shift, unsigned int requested_shift) +{ + if (compat) + return has_4k ? 12 : native_shift; + return requested_shift ?: native_shift; +} + typedef struct { atomic64_t id; #ifdef CONFIG_COMPAT @@ -25,8 +38,35 @@ typedef struct { void *vdso; unsigned long flags; u8 pkey_allocation_map; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* Unissued PTE/PMD/PUD fragments; protected by page_table_lock. */ + void *user4k_pt_frag[3]; +#endif } mm_context_t; +static inline bool arm64_context_user4k(const mm_context_t *context) +{ + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && + (context->flags & MMCF_USER_4K); +} + +static inline unsigned int arm64_context_page_shift(const mm_context_t *context) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL)) { + if (context->flags & MMCF_USER_4K) + return 12; + if (context->flags & MMCF_USER_16K) + return 14; + if (context->flags & MMCF_USER_64K) + return 16; + } + return PAGE_SHIFT; +} + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_mm_page_shift(mm) arm64_context_page_shift(&(mm)->context) +#endif + /* * We use atomic64_read() here because the ASID for an 'mm_struct' can * be reallocated when scheduling one of its threads following a diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index 803b687581527..a2a44d0027f51 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -26,6 +26,19 @@ #include #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_mm_init_exec arch_mm_init_exec +static inline void arch_mm_init_exec(struct mm_struct *mm, unsigned int page_shift) +{ + if (!page_shift) + page_shift = current->thread.exec_page_shift; + if (page_shift == 12) + mm->context.flags |= MMCF_USER_4K; + else if (page_shift == 14 && PAGE_SHIFT > 14) + mm->context.flags |= MMCF_USER_16K; +} +#endif + extern bool rodata_full; static inline void contextidr_thread_switch(struct task_struct *next) @@ -53,6 +66,29 @@ static inline void cpu_set_reserved_ttbr0(void) isb(); } +/* + * The caller has installed reserved_pg_dir in TTBR0 and excludes preemption. + * On a granule transition, complete that installation before changing TG0, + * then synchronise the new walk geometry before installing a user root. + * ASIDs remain unique across both granules; switching does not flush them. + */ +static inline void cpu_set_mm_tg0(struct mm_struct *mm) +{ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + unsigned long tcr = read_sysreg(tcr_el1); + unsigned int shift = mm_page_shift(mm); + unsigned long tg0 = shift == 12 ? TCR_TG0_4K : + (shift == 14 ? TCR_TG0_16K : TCR_TG0_64K); + + if ((tcr & TCR_EL1_TG0_MASK) == tg0) + return; + + isb(); + write_sysreg((tcr & ~TCR_EL1_TG0_MASK) | tg0, tcr_el1); + isb(); +#endif +} + void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm) @@ -62,16 +98,21 @@ static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm) } /* - * Ensure TCR.T0SZ is set to the provided value. + * Set TCR.T0SZ and restore native TG0 for the idmap/transition tables. */ static inline void __cpu_set_tcr_t0sz(unsigned long t0sz) { unsigned long tcr = read_sysreg(tcr_el1); + unsigned long mask = TCR_EL1_T0SZ_MASK; - if ((tcr & TCR_EL1_T0SZ_MASK) == t0sz) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + mask |= TCR_EL1_TG0_MASK; + t0sz |= IS_ENABLED(CONFIG_ARM64_64K_PAGES) ? TCR_TG0_64K : TCR_TG0_16K; +#endif + if ((tcr & mask) == t0sz) return; - tcr &= ~TCR_EL1_T0SZ_MASK; + tcr &= ~mask; tcr |= t0sz; write_sysreg(tcr, tcr_el1); isb(); @@ -97,6 +138,8 @@ static inline void cpu_uninstall_idmap(void) local_flush_tlb_all(); __cpu_set_tcr_t0sz(TCR_T0SZ(vabits_actual)); + /* SW PAN will restore TTBR0 later, but already needs the mm's TG0. */ + cpu_set_mm_tg0(mm); if (mm != &init_mm && !system_uses_ttbr0_pan()) cpu_switch_mm(mm->pgd, mm); } @@ -168,6 +211,9 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) { atomic64_set(&mm->context.id, 0); refcount_set(&mm->context.pinned, 0); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + memset(mm->context.user4k_pt_frag, 0, sizeof(mm->context.user4k_pt_frag)); +#endif /* pkey 0 is the default, so always reserve it. */ mm->context.pkey_allocation_map = BIT(0); @@ -175,6 +221,16 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) return 0; } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm); + +static inline void destroy_context(struct mm_struct *mm) +{ + arm64_user4k_pt_cache_destroy(mm); +} +#define destroy_context destroy_context +#endif + static inline void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm) { @@ -241,6 +297,7 @@ static inline void __switch_mm(struct mm_struct *next) */ if (next == &init_mm) { cpu_set_reserved_ttbr0(); + cpu_set_mm_tg0(next); return; } diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 7f7b97e099968..890d49a4398b2 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -98,6 +98,8 @@ static inline bool try_page_mte_tagging(struct page *page) void mte_zero_clear_page_tags(void *addr); void mte_sync_tags(pte_t pte, unsigned int nr_pages); void mte_copy_page_tags(void *kto, const void *kfrom); +void mte_copy_subpage_tags_range(void *kto, const void *kfrom, unsigned long size); +void mte_clear_subpage_tags_range(void *addr, unsigned long size); void mte_thread_init_user(void); void mte_thread_switch(struct task_struct *next); void mte_cpu_setup(void); @@ -134,6 +136,12 @@ static inline void mte_sync_tags(pte_t pte, unsigned int nr_pages) static inline void mte_copy_page_tags(void *kto, const void *kfrom) { } +static inline void mte_copy_subpage_tags_range(void *kto, const void *kfrom, unsigned long size) +{ +} +static inline void mte_clear_subpage_tags_range(void *addr, unsigned long size) +{ +} static inline void mte_thread_init_user(void) { } diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h index 58200de8a221a..96fa9d8ea23c5 100644 --- a/arch/arm64/include/asm/page.h +++ b/arch/arm64/include/asm/page.h @@ -10,6 +10,11 @@ #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +/* A shared zero mapping must cover the largest supported user granule. */ +#define ARCH_ZERO_PAGE_SIZE (1UL << 16) +#endif + #ifndef __ASSEMBLER__ #include /* for READ_IMPLIES_EXEC */ @@ -26,11 +31,28 @@ void copy_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma); #define __HAVE_ARCH_COPY_USER_HIGHPAGE +#ifdef CONFIG_MM_SUBPAGE +void clear_user_subpage_range(struct page *page, unsigned int offset, + unsigned int size); +int copy_user_subpage_range(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset, + unsigned int size); +void clear_user_subpage(struct page *page, unsigned int offset); +int copy_user_subpage(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset); +#endif + void copy_highpage(struct page *to, struct page *from); #define __HAVE_ARCH_COPY_HIGHPAGE -struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, - unsigned long vaddr); +struct folio *vma_alloc_zeroed_movable_folio_order(struct vm_area_struct *vma, + unsigned long vaddr, unsigned int order); +#define vma_alloc_zeroed_movable_folio_order vma_alloc_zeroed_movable_folio_order +static inline struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, + unsigned long vaddr) +{ + return vma_alloc_zeroed_movable_folio_order(vma, vaddr, 0); +} #define vma_alloc_zeroed_movable_folio vma_alloc_zeroed_movable_folio bool tag_clear_highpages(struct page *to, int numpages, bool clear_pages); @@ -38,7 +60,14 @@ bool tag_clear_highpages(struct page *to, int numpages, bool clear_pages); #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +/* A table may occupy a 4K fragment of a native 16K physical page. */ +typedef pte_t *pgtable_t; +#define pgtable_to_page(table) virt_to_page(table) +#define pgtable_from_page(page) ((page) ? (pgtable_t)page_address(page) : NULL) +#else typedef struct page *pgtable_t; +#endif int pfn_is_map_memory(unsigned long pfn); diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h index 1b4509d3382c6..41b81203e575c 100644 --- a/arch/arm64/include/asm/pgalloc.h +++ b/arch/arm64/include/asm/pgalloc.h @@ -15,7 +15,15 @@ #define __HAVE_ARCH_PGD_FREE #define __HAVE_ARCH_PUD_FREE +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define __HAVE_ARCH_PTE_ALLOC_ONE +#define __HAVE_ARCH_PTE_FREE +#define __HAVE_ARCH_PMD_ALLOC_ONE +#define __HAVE_ARCH_PMD_FREE +#define __HAVE_ARCH_PUD_ALLOC_ONE +#endif #include +#include #define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) @@ -31,6 +39,12 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_AF; pudval |= (mm == &init_mm) ? PUD_TABLE_UXN : PUD_TABLE_PXN; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_pud(pudp, __pud(__pa(pmdp) | pudval)); + return; + } +#endif __pud_populate(pudp, __pa(pmdp), pudval); } #else @@ -53,14 +67,24 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_AF; p4dval |= (mm == &init_mm) ? P4D_TABLE_UXN : P4D_TABLE_PXN; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_p4d(p4dp, __p4d(__pa(pudp) | p4dval)); + return; + } +#endif __p4d_populate(p4dp, __pa(pudp), p4dval); } static inline void pud_free(struct mm_struct *mm, pud_t *pud) { - if (!pgtable_l4_enabled()) + if (mm_pud_folded(mm)) return; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + arm64_pgtable_free(pud); +#else __pud_free(mm, pud); +#endif } #else static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) @@ -117,7 +141,16 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) { VM_BUG_ON(mm == &init_mm); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_pmd(pmdp, __pmd(__pa(ptep) | PMD_TYPE_TABLE | + PMD_TABLE_AF | PMD_TABLE_PXN)); + return; + } + __pmd_populate(pmdp, __pa(ptep), +#else __pmd_populate(pmdp, page_to_phys(ptep), +#endif PMD_TYPE_TABLE | PMD_TABLE_AF | PMD_TABLE_PXN); } diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h index 72f31800c7030..cb211cbcdb7fd 100644 --- a/arch/arm64/include/asm/pgtable-hwdef.h +++ b/arch/arm64/include/asm/pgtable-hwdef.h @@ -28,8 +28,10 @@ * * which gets simplified as : */ +#define ARM64_HW_PGTABLE_LEVELS_FOR(va_bits, page_shift) \ + (((va_bits) - PTDESC_ORDER - 1) / ((page_shift) - PTDESC_ORDER)) #define ARM64_HW_PGTABLE_LEVELS(va_bits) \ - (((va_bits) - PTDESC_ORDER - 1) / PTDESC_TABLE_SHIFT) + ARM64_HW_PGTABLE_LEVELS_FOR(va_bits, PAGE_SHIFT) /* * Size mapped by an entry at level n ( -1 <= n <= 3) @@ -44,7 +46,10 @@ * Rearranging it a bit we get : * (4 - n) * PTDESC_TABLE_SHIFT + PTDESC_ORDER */ -#define ARM64_HW_PGTABLE_LEVEL_SHIFT(n) (PTDESC_TABLE_SHIFT * (4 - (n)) + PTDESC_ORDER) +#define ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(n, page_shift) \ + (((page_shift) - PTDESC_ORDER) * (4 - (n)) + PTDESC_ORDER) +#define ARM64_HW_PGTABLE_LEVEL_SHIFT(n) \ + ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(n, PAGE_SHIFT) #define PTRS_PER_PTE (1 << PTDESC_TABLE_SHIFT) @@ -79,7 +84,8 @@ * PGDIR_SHIFT determines the size a top-level page table entry can map * (depending on the configuration, this level can be -1, 0, 1 or 2). */ -#define PGDIR_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - CONFIG_PGTABLE_LEVELS) +#define ARM64_NATIVE_PGTABLE_LEVELS ARM64_HW_PGTABLE_LEVELS(VA_BITS) +#define PGDIR_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - ARM64_NATIVE_PGTABLE_LEVELS) #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) #define PTRS_PER_PGD (1 << (VA_BITS - PGDIR_SHIFT)) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 4dfa42b7d0535..1ce97b836c8ac 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -467,6 +467,8 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte)); } +#include + /* * Hugetlb definitions. */ @@ -711,6 +713,23 @@ static inline void __set_ptes_anysz(struct mm_struct *mm, unsigned long addr, static inline void __set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte, unsigned int nr) { + if (arm64_mm_alt_granule(mm)) { + unsigned int native_pages = DIV_ROUND_UP((pte_phys_mm(mm, pte) & ~PAGE_MASK) + + ((unsigned long)nr << mm_page_shift(mm)), PAGE_SIZE); + + page_table_check_ptes_set(mm, addr, ptep, pte, nr); + __sync_cache_and_tags(pte, native_pages); + for (;;) { + __check_safe_pte_update(mm, ptep, pte); + __set_pte_nosync(ptep, pte); + if (--nr == 0) + break; + ptep++; + pte = pte_advance_pfn_mm(mm, pte, 1); + } + __set_pte_complete(pte); + return; + } __set_ptes_anysz(mm, addr, ptep, pte, nr, PAGE_SIZE); } @@ -946,6 +965,8 @@ static inline pmd_t *pud_pgtable(pud_t pud) static __always_inline bool pgtable_l4_enabled(void) { + if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) + return false; if (CONFIG_PGTABLE_LEVELS > 4 || !IS_ENABLED(CONFIG_ARM64_LPA2)) return true; if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT)) @@ -955,6 +976,8 @@ static __always_inline bool pgtable_l4_enabled(void) static inline bool mm_pud_folded(const struct mm_struct *mm) { + if (arm64_mm_alt_granule(mm)) + return mm_page_shift(mm) == 16; return !pgtable_l4_enabled(); } #define mm_pud_folded mm_pud_folded @@ -994,6 +1017,8 @@ static inline phys_addr_t p4d_page_paddr(p4d_t p4d) static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr) { + if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) + return (pud_t *)p4dp; /* Ensure that 'p4dp' indexes a page table according to 'addr' */ VM_BUG_ON(((addr >> P4D_SHIFT) ^ ((u64)p4dp >> 3)) % PTRS_PER_P4D); @@ -1216,6 +1241,8 @@ p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr) #endif /* CONFIG_PGTABLE_LEVELS > 4 */ +#include + #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) @@ -1261,7 +1288,7 @@ static inline int __ptep_set_access_flags(struct vm_area_struct *vma, pte_t entry, int dirty) { return __ptep_set_access_flags_anysz(vma, address, ptep, entry, dirty, - PAGE_SIZE); + mm_page_size(vma->vm_mm)); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -1389,7 +1416,7 @@ static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr, if (--nr == 0) break; ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(mm); } } @@ -1402,7 +1429,7 @@ static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm, pte = __ptep_get_and_clear(mm, addr, ptep); while (--nr) { ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(mm); tmp_pte = __ptep_get_and_clear(mm, addr, ptep); if (pte_dirty(tmp_pte)) pte = pte_mkdirty(pte); @@ -1450,7 +1477,7 @@ static inline void __wrprotect_ptes(struct mm_struct *mm, unsigned long address, { unsigned int i; - for (i = 0; i < nr; i++, address += PAGE_SIZE, ptep++) + for (i = 0; i < nr; i++, address += mm_page_size(mm), ptep++) __ptep_set_wrprotect(mm, address, ptep); } @@ -1490,7 +1517,7 @@ static inline void __clear_young_dirty_ptes(struct vm_area_struct *vma, if (--nr == 0) break; ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(vma->vm_mm); } } @@ -1516,9 +1543,11 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, * bits 0-1: present (must be zero) * bits 2: remember PG_anon_exclusive * bit 3: remember uffd-wp state + * bits 4-5: 4K slot offset within a native swap page (MM_SUBPAGE) * bits 6-10: swap type * bit 11: PTE_PRESENT_INVALID (must be zero) * bits 12-61: swap offset + * bits 62-63: high 4K slot offset bits (MM_SUBPAGE, up to 64K) */ #define __SWP_TYPE_SHIFT 6 #define __SWP_TYPE_BITS 5 @@ -1688,11 +1717,11 @@ static __always_inline void contpte_try_fold(struct mm_struct *mm, * We can't fold special mappings, because there is no associated folio. */ - const unsigned long contmask = CONT_PTES - 1; - bool valign = ((addr >> PAGE_SHIFT) & contmask) == contmask; + const unsigned long contmask = cont_ptes_mm(mm) - 1; + bool valign = ((addr >> mm_page_shift(mm)) & contmask) == contmask; if (unlikely(valign)) { - bool palign = (pte_pfn(pte) & contmask) == contmask; + bool palign = ((pte_phys_mm(mm, pte) >> mm_page_shift(mm)) & contmask) == contmask; if (unlikely(palign && pte_valid(pte) && !pte_cont(pte) && !pte_special(pte))) @@ -1713,7 +1742,11 @@ static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte) if (!pte_valid_cont(pte)) return 1; - return CONT_PTES - (((unsigned long)ptep >> 3) & (CONT_PTES - 1)); + { + unsigned int nr = arm64_cont_ptes(arm64_ptep_page_shift(ptep)); + + return nr - (((unsigned long)ptep >> 3) & (nr - 1)); + } } /* diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 499e0a85c0882..4f0f1c0b17241 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -202,6 +202,11 @@ struct thread_struct { #ifdef CONFIG_ARM64_ACTLR_STATE u64 actlr; #endif +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* Exec policies never change the live mm geometry. Fork inherits both. */ + unsigned int exec_page_shift; + unsigned int default_exec_page_shift; +#endif }; static inline unsigned int thread_get_vl(struct thread_struct *thread, diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index 751bd57bc3bac..1756fa3e65ec9 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -10,6 +10,9 @@ #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +void __tlb_remove_table(void *token); +#endif #define tlb_flush tlb_flush static void tlb_flush(struct mmu_gather *tlb); @@ -76,8 +79,14 @@ static inline void tlb_flush(struct mmu_gather *tlb) static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr) { - struct ptdesc *ptdesc = page_ptdesc(pte); + struct ptdesc *ptdesc = page_ptdesc(pgtable_to_page(pte)); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pte | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } @@ -87,6 +96,12 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, { struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pmdp | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } #endif @@ -97,9 +112,15 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, { struct ptdesc *ptdesc = virt_to_ptdesc(pudp); - if (!pgtable_l4_enabled()) + if (mm_pud_folded(tlb->mm)) return; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pudp | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } #endif diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 57b4eda6a72bb..56062bf906cd4 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -56,8 +57,8 @@ }) /* - * Get translation granule of the system, which is decided by - * PAGE_SIZE. Used by TTL. + * Encode a translation granule for TTL and range operands. Native kernel + * and stage-2 callers use PAGE_SHIFT; userspace callers use their mm. * - 4KB : 1 * - 16KB : 2 * - 64KB : 3 @@ -66,20 +67,25 @@ #define TLBI_TTL_TG_16K 2 #define TLBI_TTL_TG_64K 3 -static inline unsigned long get_trans_granule(void) +static inline unsigned long get_trans_granule_shift(unsigned int page_shift) { - switch (PAGE_SIZE) { - case SZ_4K: + switch (page_shift) { + case 12: return TLBI_TTL_TG_4K; - case SZ_16K: + case 14: return TLBI_TTL_TG_16K; - case SZ_64K: + case 16: return TLBI_TTL_TG_64K; default: return 0; } } +static inline unsigned long get_trans_granule(void) +{ + return get_trans_granule_shift(PAGE_SHIFT); +} + #ifdef CONFIG_ARM64_ERRATUM_4193714 extern cpumask_t sme_active_cpus; @@ -175,13 +181,14 @@ static __always_inline void ipas2e1is(u64 arg) __tlbi(ipas2e1is, arg); } -static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, - u16 asid) +static __always_inline void __tlbi_level_asid_granule(tlbi_op op, u64 addr, + u32 level, u16 asid, + unsigned int page_shift) { u64 arg = __TLBI_VADDR(addr, asid); if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && level <= 3) { - u64 ttl = level | (get_trans_granule() << 2); + u64 ttl = level | (get_trans_granule_shift(page_shift) << 2); FIELD_MODIFY(TLBI_TTL_MASK, &arg, ttl); } @@ -189,6 +196,12 @@ static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, op(arg); } +static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, + u16 asid) +{ + __tlbi_level_asid_granule(op, addr, level, asid, PAGE_SHIFT); +} + static inline void __tlbi_level(tlbi_op op, u64 addr, u32 level) { __tlbi_level_asid(op, addr, level, 0); @@ -471,36 +484,45 @@ static __always_inline void ripas2e1is(u64 arg) __tlbi(ripas2e1is, arg); } -static __always_inline void __tlbi_range(tlbi_op op, u64 addr, - u16 asid, int scale, int num, - u32 level, bool lpa2) +static __always_inline void __tlbi_range_granule(tlbi_op op, u64 addr, + u16 asid, int scale, int num, + u32 level, bool lpa2, + unsigned int page_shift) { u64 arg = 0; - arg |= FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : PAGE_SHIFT)); + arg |= FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : page_shift)); arg |= FIELD_PREP(TLBIR_TTL_MASK, level > 3 ? 0 : level); arg |= FIELD_PREP(TLBIR_NUM_MASK, num); arg |= FIELD_PREP(TLBIR_SCALE_MASK, scale); - arg |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); + arg |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule_shift(page_shift)); arg |= FIELD_PREP(TLBIR_ASID_MASK, asid); op(arg); } -static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, +/* + * Production callers supply the constant system_supports_tlb_range function, + * which inlines to the existing alternative branch inside the loop. Do not + * evaluate it into a bool outside the loop: that adds runtime flag handling + * to the native path. Recording tests supply constant true/false predicates. + */ +static __always_inline void __flush_tlb_range_op_granule(tlbi_op lop, tlbi_op rop, u64 start, size_t pages, u64 stride, u16 asid, - u32 level, bool lpa2) + u32 level, bool lpa2, + unsigned int page_shift, + bool (*range_supported)(void)) { - u64 addr = start, end = start + pages * PAGE_SIZE; + u64 addr = start, end = start + (pages << page_shift); int scale = 3; while (addr != end) { int num; - pages = (end - addr) >> PAGE_SHIFT; + pages = (end - addr) >> page_shift; - if (!system_supports_tlb_range() || pages == 1) + if (!range_supported() || pages == 1) goto invalidate_one; if (lpa2 && !IS_ALIGNED(addr, SZ_64K)) @@ -508,26 +530,41 @@ static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, num = __TLBI_RANGE_NUM(pages, scale); if (num >= 0) { - __tlbi_range(rop, addr, asid, scale, num, level, lpa2); - addr += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; + __tlbi_range_granule(rop, addr, asid, scale, num, level, + lpa2, page_shift); + addr += __TLBI_RANGE_PAGES(num, scale) << page_shift; } scale--; continue; invalidate_one: - __tlbi_level_asid(lop, addr, level, asid); + __tlbi_level_asid_granule(lop, addr, level, asid, page_shift); addr += stride; } } +static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, + u64 start, size_t pages, + u64 stride, u16 asid, + u32 level, bool lpa2) +{ + __flush_tlb_range_op_granule(lop, rop, start, pages, stride, asid, + level, lpa2, PAGE_SHIFT, system_supports_tlb_range); +} + #define __flush_s1_tlb_range_op(op, start, pages, stride, asid, tlb_level) \ __flush_tlb_range_op(op, r##op, start, pages, stride, asid, tlb_level, lpa2_is_enabled()) #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ __flush_tlb_range_op(op, r##op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_enabled()) -static inline bool __flush_tlb_range_limit_excess(unsigned long pages, - unsigned long stride) +#define __flush_user_tlb_range_op(op, start, pages, stride, asid, level, shift) \ + __flush_tlb_range_op_granule(op, r##op, start, pages, stride, asid, \ + level, lpa2_is_enabled(), shift, system_supports_tlb_range) + +static inline bool __flush_tlb_range_limit_excess_granule(unsigned long pages, + unsigned long stride, + unsigned int page_shift) { /* * Assume that the worst case number of DVM ops required to flush a @@ -538,7 +575,13 @@ static inline bool __flush_tlb_range_limit_excess(unsigned long pages, if (system_supports_tlb_range()) return pages > MAX_TLBI_RANGE_PAGES; - return pages >= (MAX_DVM_OPS * stride) >> PAGE_SHIFT; + return pages >= (MAX_DVM_OPS * stride) >> page_shift; +} + +static inline bool __flush_tlb_range_limit_excess(unsigned long pages, + unsigned long stride) +{ + return __flush_tlb_range_limit_excess_granule(pages, stride, PAGE_SHIFT); } typedef unsigned __bitwise tlbf_t; @@ -565,10 +608,11 @@ static __always_inline void __do_flush_tlb_range(struct vm_area_struct *vma, { struct mm_struct *mm = vma->vm_mm; unsigned long asid, pages; + unsigned int page_shift = mm_page_shift(mm); - pages = (end - start) >> PAGE_SHIFT; + pages = (end - start) >> page_shift; - if (__flush_tlb_range_limit_excess(pages, stride)) { + if (__flush_tlb_range_limit_excess_granule(pages, stride, page_shift)) { flush_tlb_mm(mm); return; } @@ -582,20 +626,20 @@ static __always_inline void __do_flush_tlb_range(struct vm_area_struct *vma, switch (flags & (TLBF_NOWALKCACHE | TLBF_NOBROADCAST)) { case TLBF_NONE: - __flush_s1_tlb_range_op(vae1is, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vae1is, start, pages, stride, + asid, tlb_level, page_shift); break; case TLBF_NOWALKCACHE: - __flush_s1_tlb_range_op(vale1is, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vale1is, start, pages, stride, + asid, tlb_level, page_shift); break; case TLBF_NOBROADCAST: /* Combination unused */ BUG(); break; case TLBF_NOWALKCACHE | TLBF_NOBROADCAST: - __flush_s1_tlb_range_op(vale1, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vale1, start, pages, stride, + asid, tlb_level, page_shift); break; } @@ -629,16 +673,18 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, * Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough * information here. */ - __flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE); + __flush_tlb_range(vma, start, end, mm_page_size(vma->vm_mm), + TLBI_TTL_UNKNOWN, TLBF_NONE); } static inline void __flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr, tlbf_t flags) { - unsigned long start = round_down(uaddr, PAGE_SIZE); - unsigned long end = start + PAGE_SIZE; + unsigned long page_size = mm_page_size(vma->vm_mm); + unsigned long start = round_down(uaddr, page_size); + unsigned long end = start + page_size; - __do_flush_tlb_range(vma, start, end, PAGE_SIZE, 3, + __do_flush_tlb_range(vma, start, end, page_size, 3, TLBF_NOWALKCACHE | flags); } @@ -688,7 +734,7 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b { struct vm_area_struct vma = { .vm_mm = mm, .vm_flags = 0 }; - __flush_tlb_range(&vma, start, end, PAGE_SIZE, 3, + __flush_tlb_range(&vma, start, end, mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); } diff --git a/arch/arm64/include/asm/user-pgalloc.h b/arch/arm64/include/asm/user-pgalloc.h new file mode 100644 index 0000000000000..3b4dbde0c7736 --- /dev/null +++ b/arch/arm64/include/asm/user-pgalloc.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PGALLOC_H +#define __ASM_USER_PGALLOC_H + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + +enum arm64_user4k_pt_level { + USER4K_PT_PTE, + USER4K_PT_PMD, + USER4K_PT_PUD, +}; + +/* Explicit geometry entry point; callers must keep each mm's granule fixed. */ +void *arm64_user_pt_alloc_granule(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift); +void *arm64_user4k_pt_alloc(struct mm_struct *mm, enum arm64_user4k_pt_level level); +void arm64_user4k_pt_free(void *table); +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm); +void arm64_pgtable_free(void *table); +void arm64_pte_free_defer(struct mm_struct *mm, pgtable_t table); + +static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PTE, mm_page_shift(mm)); + return __pte_alloc_one_noprof(mm, GFP_PGTABLE_USER); +} +#define pte_alloc_one(...) alloc_hooks(pte_alloc_one_noprof(__VA_ARGS__)) + +static inline void pte_free(struct mm_struct *mm, pgtable_t table) +{ + arm64_pgtable_free(table); +} + +static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PMD, mm_page_shift(mm)); + return __pmd_alloc_one_noprof(mm, addr); +} +#define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__)) + +static inline void pmd_free(struct mm_struct *mm, pmd_t *table) +{ + arm64_pgtable_free(table); +} + +static inline pud_t *pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PUD, mm_page_shift(mm)); + return __pud_alloc_one_noprof(mm, addr); +} +#define pud_alloc_one(...) alloc_hooks(pud_alloc_one_noprof(__VA_ARGS__)) + +#define pte_free_defer arm64_pte_free_defer + +#endif +#endif diff --git a/arch/arm64/include/asm/user-pgtable.h b/arch/arm64/include/asm/user-pgtable.h new file mode 100644 index 0000000000000..b244f2387a7b4 --- /dev/null +++ b/arch/arm64/include/asm/user-pgtable.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PGTABLE_H +#define __ASM_USER_PGTABLE_H + +/* Included by asm/pgtable.h after the native accessors. */ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + +/* Alternative tables use 48-bit VA; 64K folds the PUD level. */ +static inline unsigned int arm64_user_level_shift(const struct mm_struct *mm, + unsigned int level) +{ + unsigned int shift = mm_page_shift(mm); + + if (shift == 16 && !level) + level = 1; + return shift + (3 - level) * (shift - 3); +} + +static inline unsigned long arm64_user_table_index(const struct mm_struct *mm, + unsigned long addr, unsigned int level) +{ + unsigned int shift = arm64_user_level_shift(mm, level); + unsigned int bits = level ? mm_page_shift(mm) - 3 : 48 - shift; + + return (addr >> shift) & ((1UL << bits) - 1); +} + +#define p4d_none_mm(mm, p4d) \ + (arm64_mm_alt_granule(mm) ? (!mm_pud_folded(mm) && !p4d_val(p4d)) : p4d_none(p4d)) +#define p4d_bad_mm(mm, p4d) \ + (arm64_mm_alt_granule(mm) ? (!mm_pud_folded(mm) && ((p4d_val(p4d) & P4D_TYPE_MASK) != P4D_TYPE_TABLE)) : p4d_bad(p4d)) +#define p4d_present_mm(mm, p4d) (!p4d_none_mm(mm, p4d)) +static inline void p4d_clear_mm(struct mm_struct *mm, p4d_t *p4d) +{ + if (arm64_mm_alt_granule(mm)) { + if (!mm_pud_folded(mm)) + set_p4d(p4d, __p4d(0)); + } else { + p4d_clear(p4d); + } +} +#define p4d_clear_mm p4d_clear_mm + + +/* pgtable_t preserves the fragment offset; pmd_page() remains its owner. */ +#define pmd_pgtable(pmd) ((pgtable_t)__va(pmd_val(pmd) & ARM64_USER4K_TABLE_ADDR_MASK)) + +static inline void *arm64_user4k_table(u64 descriptor) +{ + /* Unlike native __pXd_to_phys(), retain sub-native table address bits. */ + return __va(descriptor & ARM64_USER4K_TABLE_ADDR_MASK); +} + +static inline pgd_t *pgd_offset_pgd_mm(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return pgd + arm64_user_table_index(mm, addr, 0); + return pgd + ((addr >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)); +} + +#define pgd_offset_pgd_mm pgd_offset_pgd_mm + +static inline pgd_t *pgd_offset_mm(struct mm_struct *mm, unsigned long addr) +{ + return pgd_offset_pgd_mm(mm, mm->pgd, addr); +} + +#define pgd_offset(mm, addr) pgd_offset_mm(mm, addr) + +static inline p4d_t *p4d_offset_mm(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (p4d_t *)pgd; + return p4d_offset(pgd, addr); +} + +#define p4d_offset_mm p4d_offset_mm + +static inline pud_t *pud_offset_mm(struct mm_struct *mm, p4d_t *p4d, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) { + if (mm_pud_folded(mm)) + return (pud_t *)p4d; + return (pud_t *)arm64_user4k_table(p4d_val(READ_ONCE(*p4d))) + + arm64_user_table_index(mm, addr, 1); + } + return pud_offset(p4d, addr); +} + +#define pud_offset_mm pud_offset_mm + +static inline pmd_t *pud_pgtable_mm(struct mm_struct *mm, pud_t pud) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user4k_table(pud_val(pud)); + return pud_pgtable(pud); +} + +#define pud_pgtable_mm pud_pgtable_mm + +static inline pmd_t *pmd_offset_mm(struct mm_struct *mm, pud_t *pud, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (pmd_t *)arm64_user4k_table(pud_val(READ_ONCE(*pud))) + + arm64_user_table_index(mm, addr, 2); + return (pmd_t *)pud_pgtable(READ_ONCE(*pud)) + + ((addr >> PMD_SHIFT) & (PTRS_PER_PMD - 1)); +} + +#define pmd_offset_mm pmd_offset_mm + +static inline pte_t *pte_offset_kernel_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (pte_t *)arm64_user4k_table(pmd_val(READ_ONCE(*pmd))) + + arm64_user_table_index(mm, addr, 3); + return (pte_t *)pmd_page_vaddr(READ_ONCE(*pmd)) + + ((addr >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); +} + +#define pte_offset_kernel_mm pte_offset_kernel_mm +#define __pte_map_mm(mm, pmd, addr) pte_offset_kernel_mm(mm, pmd, addr) + +#define pmd_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 2) : PMD_SHIFT) +#define pud_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 1) : PUD_SHIFT) +#define p4d_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 0) : P4D_SHIFT) +#define pmd_size_mm(mm) (1UL << pmd_shift_mm(mm)) +#define pud_size_mm(mm) (1UL << pud_shift_mm(mm)) +#define p4d_size_mm(mm) (1UL << p4d_shift_mm(mm)) +#define pgd_size_mm(mm) (arm64_mm_alt_granule(mm) ? (1UL << arm64_user_level_shift(mm, 0)) : PGDIR_SIZE) +#define pte_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PTE * sizeof(pte_t)) +#define pmd_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PMD * sizeof(pmd_t)) +#define pud_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PUD * sizeof(pud_t)) +#define pmd_mask_mm(mm) (~(pmd_size_mm(mm) - 1)) +#define pud_mask_mm(mm) (~(pud_size_mm(mm) - 1)) +#define p4d_mask_mm(mm) (~(p4d_size_mm(mm) - 1)) +#define pgd_mask_mm(mm) (~(pgd_size_mm(mm) - 1)) + +static inline unsigned long arm64_user4k_addr_end(unsigned long addr, + unsigned long end, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned long boundary = (addr + size) & ~(size - 1); + + return boundary - 1 < end - 1 ? boundary : end; +} + +/* Native fallbacks retain their architecture-specific folding semantics. */ +#define pgd_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 0)) : pgd_addr_end(addr, end)) +#define p4d_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? (end) : p4d_addr_end(addr, end)) +#define pud_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 1)) : pud_addr_end(addr, end)) +#define pmd_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 2)) : pmd_addr_end(addr, end)) + +#endif /* CONFIG_ARM64_USER4K_EXPERIMENTAL */ +#endif diff --git a/arch/arm64/include/asm/user-pte.h b/arch/arm64/include/asm/user-pte.h new file mode 100644 index 0000000000000..aa4c1e3a5aab9 --- /dev/null +++ b/arch/arm64/include/asm/user-pte.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PTE_H +#define __ASM_USER_PTE_H + +/* Included after native leaf accessors, before their batched setters. */ +#define ARM64_USER4K_TABLE_ADDR_MASK GENMASK_ULL(47, 12) + +static inline bool arm64_mm_user4k(const struct mm_struct *mm) +{ + return mm && arm64_context_user4k(&mm->context); +} + +static inline bool arm64_mm_alt_granule(const struct mm_struct *mm) +{ + return mm && arm64_context_page_shift(&mm->context) != PAGE_SHIFT; +} + +static inline u64 arm64_user_leaf_addr_mask(const struct mm_struct *mm) +{ + return ARM64_USER4K_TABLE_ADDR_MASK & mm_page_mask(mm); +} + +/* PFNs still identify native pages; leaf physical addresses identify slots. */ +static inline phys_addr_t pte_phys_mm(const struct mm_struct *mm, pte_t pte) +{ + if (arm64_mm_alt_granule(mm)) + return pte_val(pte) & arm64_user_leaf_addr_mask(mm); + return __pte_to_phys(pte); +} +#define pte_phys_mm pte_phys_mm + +static inline pte_t phys_pte_mm(const struct mm_struct *mm, phys_addr_t phys, + pgprot_t prot) +{ + if (arm64_mm_alt_granule(mm)) { + VM_BUG_ON(phys & ~arm64_user_leaf_addr_mask(mm)); + return __pte(phys | pgprot_val(prot)); + } + return pfn_pte(phys >> PAGE_SHIFT, prot); +} +#define phys_pte_mm phys_pte_mm + +static inline pgprot_t pte_pgprot_mm(const struct mm_struct *mm, pte_t pte) +{ + if (arm64_mm_alt_granule(mm)) + return __pgprot(pte_val(pte) & ~arm64_user_leaf_addr_mask(mm)); + return pte_pgprot(pte); +} +#define pte_pgprot_mm pte_pgprot_mm + + +#ifdef CONFIG_MM_SUBPAGE +/* Slot bits belong to the PTE, not the native swap-cache index. */ +#define ARM64_SWP_SUBPAGE_MASK (GENMASK_ULL(5, 4) | GENMASK_ULL(63, 62)) + +static inline unsigned int pte_swp_subpage_offset(pte_t pte) +{ + return ((((pte_val(pte) >> 4) & 3) | + ((pte_val(pte) >> 60) & 12)) << 12); +} + +static inline pte_t pte_swp_set_subpage_offset(pte_t pte, unsigned int offset) +{ + VM_BUG_ON(!IS_ALIGNED(offset, SZ_4K) || offset >= PAGE_SIZE); + return __pte((pte_val(pte) & ~ARM64_SWP_SUBPAGE_MASK) | + (((u64)(offset >> 12) & 3) << 4) | + (((u64)(offset >> 12) & 12) << 60)); +} +#endif + +static inline pte_t pte_advance_pfn_mm(const struct mm_struct *mm, pte_t pte, + unsigned long nr) +{ + if (arm64_mm_alt_granule(mm)) + return phys_pte_mm(mm, pte_phys_mm(mm, pte) + (nr << mm_page_shift(mm)), + pte_pgprot_mm(mm, pte)); + return pte_advance_pfn(pte, nr); +} + +#define pte_advance_pfn_mm pte_advance_pfn_mm + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +unsigned int arm64_ptep_page_shift(pte_t *ptep); +#else +static inline unsigned int arm64_ptep_page_shift(pte_t *ptep) +{ + return PAGE_SHIFT; +} +#endif + +static inline unsigned int arm64_cont_ptes(unsigned int shift) +{ + /* Architectural contiguous group lengths for 4K, 16K and 64K. */ + return shift == 12 ? 16 : shift == 14 ? 128 : 32; +} + +static inline unsigned int cont_ptes_mm(const struct mm_struct *mm) +{ + return arm64_cont_ptes(mm_page_shift(mm)); +} + +static inline unsigned long cont_pte_size_mm(const struct mm_struct *mm) +{ + return (unsigned long)cont_ptes_mm(mm) << mm_page_shift(mm); +} + +static inline unsigned long pte_cont_addr_end_mm(const struct mm_struct *mm, + unsigned long addr, unsigned long end) +{ + unsigned long size = cont_pte_size_mm(mm); + unsigned long boundary = (addr + size) & ~(size - 1); + + return boundary - 1 < end - 1 ? boundary : end; +} + +#endif diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 432d35b2714cd..e30ef03e87b3e 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1661,6 +1661,62 @@ has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) return cpufeature_matches(val, entry); } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST +/* Removal-only model for late-CPU verification in a disposable test VM. */ +static int user4k_deny_late_cpu __ro_after_init = -1; + +static int __init user4k_deny_late_cpu_setup(char *str) +{ + int cpu; + + if (kstrtoint(str, 0, &cpu) || cpu <= 0 || cpu >= NR_CPUS) + return -EINVAL; + user4k_deny_late_cpu = cpu; + return 0; +} +early_param("arm64.user4k_deny_late_cpu", user4k_deny_late_cpu_setup); +#ifdef CONFIG_ARM64_64K_PAGES +static int user16k_deny_late_cpu __ro_after_init = -1; + +static int __init user16k_deny_late_cpu_setup(char *str) +{ + int cpu; + + if (kstrtoint(str, 0, &cpu) || cpu <= 0 || cpu >= NR_CPUS) + return -EINVAL; + user16k_deny_late_cpu = cpu; + return 0; +} +early_param("arm64.user16k_deny_late_cpu", user16k_deny_late_cpu_setup); +#endif +#endif + +static bool has_user4k_granule(const struct arm64_cpu_capabilities *entry, + int scope) +{ +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (scope == SCOPE_LOCAL_CPU && system_capabilities_finalized() && + smp_processor_id() == user4k_deny_late_cpu) + return false; +#endif + return has_cpuid_feature(entry, scope); +} + +#ifdef CONFIG_ARM64_64K_PAGES +static bool has_user16k_granule(const struct arm64_cpu_capabilities *entry, + int scope) +{ +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (scope == SCOPE_LOCAL_CPU && system_capabilities_finalized() && + smp_processor_id() == user16k_deny_late_cpu) + return false; +#endif + return has_cpuid_feature(entry, scope); +} +#endif +#endif + const struct cpumask *system_32bit_el0_cpumask(void) { if (!system_supports_32bit_el0()) @@ -2391,8 +2447,11 @@ static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap) * clear the tags or set PG_mte_tagged. */ if (!cleared_zero_page) { + unsigned long offset; + cleared_zero_page = true; - mte_clear_page_tags(lm_alias(empty_zero_page)); + for (offset = 0; offset < sizeof(empty_zero_page); offset += PAGE_SIZE) + mte_clear_page_tags(lm_alias(empty_zero_page + offset)); } kasan_init_hw_tags_cpu(); @@ -2529,6 +2588,28 @@ test_has_gicv5_legacy(const struct arm64_cpu_capabilities *entry, int scope) } static const struct arm64_cpu_capabilities arm64_features[] = { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + { + .desc = "4K userspace translation granule", + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .capability = ARM64_HAS_USER4K_GRANULE, + .matches = has_user4k_granule, + __ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN4, + ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN, + ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MAX) + }, +#ifdef CONFIG_ARM64_64K_PAGES + { + .desc = "16K userspace translation granule", + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .capability = ARM64_HAS_USER16K_GRANULE, + .matches = has_user16k_granule, + __ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN16, + ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MIN, + ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MAX) + }, +#endif +#endif { .capability = ARM64_ALWAYS_BOOT, .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, diff --git a/arch/arm64/kernel/pi/idreg-override.c b/arch/arm64/kernel/pi/idreg-override.c index bc57b290e5e7b..19b98dffa71bc 100644 --- a/arch/arm64/kernel/pi/idreg-override.c +++ b/arch/arm64/kernel/pi/idreg-override.c @@ -38,11 +38,32 @@ struct ftr_set_desc { #define FIELD(n, s, f) { .name = n, .shift = s, .width = 4, .filter = f } +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && !defined(CONFIG_ARM64_4K_PAGES) +/* VM capability-model test: only remove 4K support, never fabricate it. */ +static bool __init user4k_granule_test_filter(u64 val) +{ + return val == ID_AA64MMFR0_EL1_TGRAN4_NI; +} + +#ifdef CONFIG_ARM64_64K_PAGES +static bool __init user16k_granule_test_filter(u64 val) +{ + return val == ID_AA64MMFR0_EL1_TGRAN16_NI; +} +#endif +#endif + static const struct ftr_set_desc mmfr0 __prel64_initconst = { .name = "id_aa64mmfr0", .override = &id_aa64mmfr0_override, .fields = { FIELD("ecv", ID_AA64MMFR0_EL1_ECV_SHIFT, NULL), +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && !defined(CONFIG_ARM64_4K_PAGES) + FIELD("tgran4", ID_AA64MMFR0_EL1_TGRAN4_SHIFT, user4k_granule_test_filter), +#ifdef CONFIG_ARM64_64K_PAGES + FIELD("tgran16", ID_AA64MMFR0_EL1_TGRAN16_SHIFT, user16k_granule_test_filter), +#endif +#endif {} }, }; @@ -59,11 +80,22 @@ static bool __init mmfr1_vh_filter(u64 val) val == 0); } +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && defined(CONFIG_ARM64_SW_TTBR0_PAN) +/* VM-only coverage of deferred TTBR0 restore on CPUs which implement PAN. */ +static bool __init user4k_pan_test_filter(u64 val) +{ + return val == 0; +} +#endif + static const struct ftr_set_desc mmfr1 __prel64_initconst = { .name = "id_aa64mmfr1", .override = &id_aa64mmfr1_override, .fields = { FIELD("vh", ID_AA64MMFR1_EL1_VH_SHIFT, mmfr1_vh_filter), +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && defined(CONFIG_ARM64_SW_TTBR0_PAN) + FIELD("pan", ID_AA64MMFR1_EL1_PAN_SHIFT, user4k_pan_test_filter), +#endif {} }, }; diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c index a852264958c36..c7dd0ba8d800e 100644 --- a/arch/arm64/kernel/pi/map_kernel.c +++ b/arch/arm64/kernel/pi/map_kernel.c @@ -243,7 +243,7 @@ asmlinkage void __init early_map_kernel(u64 boot_status, phys_addr_t fdt) static char const chosen_str[] __initconst = "/chosen"; u64 va_base, pa_base = (u64)&_text; u64 kaslr_offset = pa_base % MIN_KIMG_ALIGN; - int root_level = 4 - CONFIG_PGTABLE_LEVELS; + int root_level = 4 - ARM64_NATIVE_PGTABLE_LEVELS; int va_bits = VA_BITS; int chosen; void *fdt_mapped = map_fdt(fdt); diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index ab1beffa8d7a4..54686ebf83f18 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -7,11 +7,13 @@ * Copyright (C) 2012 ARM Ltd. */ #include +#include #include #include #include #include #include +#include #include #include #include @@ -884,6 +886,68 @@ unsigned long arch_align_stack(unsigned long sp) return sp & ~0xf; } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +int arm64_bprm_prepare_mm(struct linux_binprm *bprm, bool compat) +{ + unsigned int requested = current->thread.exec_page_shift ?: + current->thread.default_exec_page_shift; + unsigned int shift = arm64_exec_page_shift(compat, + cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE), PAGE_SHIFT, + requested); + + return bprm_set_page_shift(bprm, shift); +} + +/* Return zero for the native default, or a validated alternative shift. */ +static int exec_page_size_shift(unsigned long size) +{ + if (!size || size == PAGE_SIZE) + return 0; + if (size == SZ_4K) + return cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE) ? + 12 : -EOPNOTSUPP; + if (PAGE_SIZE > SZ_16K && size == SZ_16K) + return cpus_have_final_cap(ARM64_HAS_USER16K_GRANULE) ? + 14 : -EOPNOTSUPP; + return -EINVAL; +} + +int arch_prctl_exec_page_size_get(struct task_struct *task) +{ + return task->thread.exec_page_shift ? + 1U << task->thread.exec_page_shift : 0; +} + +int arch_prctl_exec_page_size_set(struct task_struct *task, unsigned long size) +{ + int shift = exec_page_size_shift(size); + + if (shift < 0) + return shift; + /* A native request overrides an inherited alternative for one exec. */ + if (!shift && task->thread.default_exec_page_shift) + shift = PAGE_SHIFT; + task->thread.exec_page_shift = shift; + return 0; +} + +int arch_prctl_default_page_size_get(struct task_struct *task) +{ + return task->thread.default_exec_page_shift ? + 1U << task->thread.default_exec_page_shift : 0; +} + +int arch_prctl_default_page_size_set(struct task_struct *task, unsigned long size) +{ + int shift = exec_page_size_shift(size); + + if (shift < 0) + return shift; + task->thread.default_exec_page_shift = shift; + return 0; +} +#endif + #ifdef CONFIG_COMPAT int compat_elf_check_arch(const struct elf32_hdr *hdr) { @@ -911,10 +975,14 @@ int compat_elf_check_arch(const struct elf32_hdr *hdr) */ void arch_setup_new_exec(void) { - unsigned long mmflags = 0; + unsigned long mmflags = current->mm->context.flags & MMCF_USER_GRANULE; + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + current->thread.exec_page_shift = 0; +#endif if (is_compat_task()) { - mmflags = MMCF_AARCH32; + mmflags |= MMCF_AARCH32; /* * Restrict the CPU affinity mask for a 32-bit task so that diff --git a/arch/arm64/kernel/sys.c b/arch/arm64/kernel/sys.c index f08408b6e8263..bdd2531554f27 100644 --- a/arch/arm64/kernel/sys.c +++ b/arch/arm64/kernel/sys.c @@ -22,6 +22,11 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, off) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return ksys_mmap_user(addr, len, prot, flags, fd, off); +#endif + if (offset_in_page(off) != 0) return -EINVAL; diff --git a/arch/arm64/kernel/sys32.c b/arch/arm64/kernel/sys32.c index 12a948f3a5043..ba66b65f506f4 100644 --- a/arch/arm64/kernel/sys32.c +++ b/arch/arm64/kernel/sys32.c @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -51,6 +52,11 @@ COMPAT_SYSCALL_DEFINE6(aarch32_mmap2, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, off_4k) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return ksys_mmap_user(addr, len, prot, flags, fd, off_4k << 12); +#endif + if (off_4k & (~PAGE_MASK >> 12)) return -EINVAL; diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S index 5018ac03b6bf3..c1ce370068cf4 100644 --- a/arch/arm64/lib/mte.S +++ b/arch/arm64/lib/mte.S @@ -80,6 +80,52 @@ SYM_FUNC_START(mte_copy_page_tags) ret SYM_FUNC_END(mte_copy_page_tags) +#ifdef CONFIG_MM_SUBPAGE +/* Copy one aligned power-of-two extent, preserving neighbouring tags. */ +SYM_FUNC_START(mte_copy_subpage_tags_range) + mov x6, x2 + add x2, x0, x2 + multitag_transfer_size x4, x5 + cmp x4, x6 + b.hi 2f +1: ldgm x3, [x1] + stgm x3, [x0] + add x0, x0, x4 + add x1, x1, x4 + cmp x0, x2 + b.ne 1b + ret +2: mov x3, xzr +3: ldg x3, [x1] + stg x3, [x0] + add x0, x0, #MTE_GRANULE_SIZE + add x1, x1, #MTE_GRANULE_SIZE + cmp x0, x2 + b.ne 3b + ret +SYM_FUNC_END(mte_copy_subpage_tags_range) + +/* Clear an aligned extent without changing data or neighbouring tags. */ +SYM_FUNC_START(mte_clear_subpage_tags_range) + mov x4, x1 + add x1, x0, x1 + multitag_transfer_size x2, x3 + cmp x2, x4 + b.hi 2f +1: stgm xzr, [x0] + add x0, x0, x2 + cmp x0, x1 + b.ne 1b + ret +2: mov x2, xzr +3: stg x2, [x0] + add x0, x0, #MTE_GRANULE_SIZE + cmp x0, x1 + b.ne 3b + ret +SYM_FUNC_END(mte_clear_subpage_tags_range) +#endif + /* * Read tags from a user buffer (one tag per byte) and set the corresponding * tags at the given kernel address. Used by PTRACE_POKEMTETAGS. diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index c26489cf96cd9..757f7eb07ccac 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -12,6 +12,9 @@ obj-$(CONFIG_TRANS_TABLE) += trans_pgd-asm.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ARM64_MTE) += mteswap.o obj-$(CONFIG_ARM64_GCS) += gcs.o +obj-$(CONFIG_ARM64_USER4K_KUNIT_TEST) += user4k-test.o user4k-tlb-test.o +obj-$(CONFIG_ARM64_USER4K_DATA_KUNIT_TEST) += user4k-data-test.o +obj-$(CONFIG_ARM64_USER4K_EXPERIMENTAL) += user4k-pgalloc.o KASAN_SANITIZE_physaddr.o += n obj-$(CONFIG_KASAN) += kasan_init.o diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index 0f4a28b874692..cd3f56b942708 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -38,6 +39,34 @@ static unsigned long *pinned_asid_map; #define ctxid2asid(asid) ((asid) & ~ASID_MASK) #define asid2ctxid(asid, genid) ((asid) | (genid)) +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST +/* Test-only capacity reduction; hardware ASID width and normal boots are unchanged. */ +static bool user4k_test_small_asids; + +static int __init user4k_test_asid_bits_setup(char *str) +{ + unsigned int bits; + + if (kstrtouint(str, 0, &bits) || bits != 8) + return -EINVAL; + user4k_test_small_asids = true; + return 0; +} +early_param("arm64.user4k_test_asid_bits", user4k_test_asid_bits_setup); + +/* Read-only evidence that an EL0 stress run actually crossed ASID rollover. */ +static int user4k_asid_generation_get(char *buffer, const struct kernel_param *kp) +{ + return sysfs_emit(buffer, "%llu\n", (unsigned long long)atomic64_read(&asid_generation)); +} + +static const struct kernel_param_ops user4k_asid_generation_ops = { + .get = user4k_asid_generation_get, +}; +module_param_cb(user4k_asid_generation, &user4k_asid_generation_ops, NULL, 0444); +MODULE_PARM_DESC(user4k_asid_generation, "Current ASID generation for mixed-granule VM tests"); +#endif + /* Get the ASIDBits supported by the current CPU */ static u32 get_cpu_asid_bits(void) { @@ -268,6 +297,13 @@ void check_and_switch_context(struct mm_struct *mm) */ if (!system_uses_ttbr0_pan()) cpu_switch_mm(mm->pgd, mm); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + else { + /* uaccess/EL0 return restore only TTBRs, so select TG0 here. */ + cpu_set_reserved_ttbr0_nosync(); + cpu_set_mm_tg0(mm); + } +#endif } unsigned long arm64_mm_context_get(struct mm_struct *mm) @@ -365,6 +401,7 @@ void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) ttbr1 |= FIELD_PREP(TTBRx_EL1_ASID_MASK, asid); cpu_set_reserved_ttbr0_nosync(); + cpu_set_mm_tg0(mm); write_sysreg(ttbr1, ttbr1_el1); write_sysreg(ttbr0, ttbr0_el1); isb(); @@ -401,6 +438,12 @@ arch_initcall(asids_update_limit); static int asids_init(void) { asid_bits = get_cpu_asid_bits(); +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (user4k_test_small_asids) { + pr_info("user4k test: limiting ASID allocator to 8 bits\n"); + asid_bits = 8; + } +#endif atomic64_set(&asid_generation, ASID_FIRST_VERSION); asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); if (!asid_map) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index 2de12656b4d8b..f8faf1a99eeca 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -8,6 +8,25 @@ #include #include +static inline unsigned int contpte_nr(pte_t *ptep) +{ + return arm64_cont_ptes(arm64_ptep_page_shift(ptep)); +} + +static inline phys_addr_t contpte_phys(pte_t pte, unsigned int shift) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && shift != PAGE_SHIFT) + return pte_val(pte) & (ARM64_USER4K_TABLE_ADDR_MASK & ~((1ULL << shift) - 1)); + return __pte_to_phys(pte); +} + +static inline pgprot_t contpte_prot(pte_t pte, unsigned int shift) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && shift != PAGE_SHIFT) + return __pgprot(pte_val(pte) & ~(ARM64_USER4K_TABLE_ADDR_MASK & ~((1ULL << shift) - 1))); + return pte_pgprot(pte); +} + static inline bool mm_is_user(struct mm_struct *mm) { /* @@ -23,10 +42,10 @@ static inline bool mm_is_user(struct mm_struct *mm) static inline pte_t *contpte_align_down(pte_t *ptep) { - return PTR_ALIGN_DOWN(ptep, sizeof(*ptep) * CONT_PTES); + return PTR_ALIGN_DOWN(ptep, sizeof(*ptep) * contpte_nr(ptep)); } -static inline pte_t *contpte_align_addr_ptep(unsigned long *start, +static inline pte_t *contpte_align_addr_ptep(struct mm_struct *mm, unsigned long *start, unsigned long *end, pte_t *ptep, unsigned int nr) { @@ -36,10 +55,10 @@ static inline pte_t *contpte_align_addr_ptep(unsigned long *start, * single VMA and a single page table. */ if (pte_cont(__ptep_get(ptep + nr - 1))) - *end = ALIGN(*end, CONT_PTE_SIZE); + *end = ALIGN(*end, cont_pte_size_mm(mm)); if (pte_cont(__ptep_get(ptep))) { - *start = ALIGN_DOWN(*start, CONT_PTE_SIZE); + *start = ALIGN_DOWN(*start, cont_pte_size_mm(mm)); ptep = contpte_align_down(ptep); } @@ -54,11 +73,11 @@ static void contpte_try_unfold_partial(struct mm_struct *mm, unsigned long addr, * of the range. */ - if (ptep != contpte_align_down(ptep) || nr < CONT_PTES) + if (ptep != contpte_align_down(ptep) || nr < cont_ptes_mm(mm)) contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); if (ptep + nr != contpte_align_down(ptep + nr)) { - unsigned long last_addr = addr + PAGE_SIZE * (nr - 1); + unsigned long last_addr = addr + mm_page_size(mm) * (nr - 1); pte_t *last_ptep = ptep + nr - 1; contpte_try_unfold(mm, last_addr, last_ptep, @@ -75,10 +94,11 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, int i; start_ptep = ptep = contpte_align_down(ptep); - start_addr = addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); - pte = pfn_pte(ALIGN_DOWN(pte_pfn(pte), CONT_PTES), pte_pgprot(pte)); + start_addr = addr = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); + pte = phys_pte_mm(mm, ALIGN_DOWN(pte_phys_mm(mm, pte), cont_pte_size_mm(mm)), + pte_pgprot_mm(mm, pte)); - for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) { + for (i = 0; i < cont_ptes_mm(mm); i++, ptep++, addr += mm_page_size(mm)) { pte_t ptent = __ptep_get_and_clear(mm, addr, ptep); if (pte_dirty(ptent)) @@ -225,10 +245,10 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, */ if (!system_supports_bbml2_noabort()) - __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, 3, + __flush_tlb_range(&vma, start_addr, addr, mm_page_size(mm), 3, TLBF_NOWALKCACHE); - __set_ptes(mm, start_addr, start_ptep, pte, CONT_PTES); + __set_ptes(mm, start_addr, start_ptep, pte, cont_ptes_mm(mm)); } void __contpte_try_fold(struct mm_struct *mm, unsigned long addr, @@ -269,24 +289,26 @@ void __contpte_try_fold(struct mm_struct *mm, unsigned long addr, page = pte_page(pte); folio = page_folio(page); folio_start = addr - (page - &folio->page) * PAGE_SIZE; + if (arm64_mm_alt_granule(mm)) + folio_start -= pte_phys_mm(mm, pte) & ~PAGE_MASK; folio_end = folio_start + folio_nr_pages(folio) * PAGE_SIZE; - cont_start = ALIGN_DOWN(addr, CONT_PTE_SIZE); - cont_end = cont_start + CONT_PTE_SIZE; + cont_start = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); + cont_end = cont_start + cont_pte_size_mm(mm); if (folio_start > cont_start || folio_end < cont_end) return; - pfn = ALIGN_DOWN(pte_pfn(pte), CONT_PTES); - prot = pte_pgprot(pte_mkold(pte_mkclean(pte))); - expected_pte = pfn_pte(pfn, prot); + pfn = ALIGN_DOWN(pte_phys_mm(mm, pte) >> mm_page_shift(mm), cont_ptes_mm(mm)); + prot = pte_pgprot_mm(mm, pte_mkold(pte_mkclean(pte))); + expected_pte = phys_pte_mm(mm, pfn << mm_page_shift(mm), prot); orig_ptep = ptep; ptep = contpte_align_down(ptep); - for (i = 0; i < CONT_PTES; i++) { + for (i = 0; i < cont_ptes_mm(mm); i++) { subpte = pte_mkold(pte_mkclean(__ptep_get(ptep))); if (!pte_same(subpte, expected_pte)) return; - expected_pte = pte_advance_pfn(expected_pte, 1); + expected_pte = pte_advance_pfn_mm(mm, expected_pte, 1); ptep++; } @@ -319,17 +341,18 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) * feet. */ + unsigned int nr = contpte_nr(ptep); pte_t pte; int i; ptep = contpte_align_down(ptep); - for (i = 0; i < CONT_PTES; i++, ptep++) { + for (i = 0; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); - for (; i < CONT_PTES; i++, ptep++) { + for (; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_young(pte)) { orig_pte = pte_mkyoung(orig_pte); @@ -343,7 +366,7 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) orig_pte = pte_mkyoung(orig_pte); i++; ptep++; - for (; i < CONT_PTES; i++, ptep++) { + for (; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); @@ -359,11 +382,12 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) EXPORT_SYMBOL_GPL(contpte_ptep_get); static inline bool contpte_is_consistent(pte_t pte, unsigned long pfn, + unsigned int shift, pgprot_t orig_prot) { - pgprot_t prot = pte_pgprot(pte_mkold(pte_mkclean(pte))); + pgprot_t prot = contpte_prot(pte_mkold(pte_mkclean(pte)), shift); - return pte_valid_cont(pte) && pte_pfn(pte) == pfn && + return pte_valid_cont(pte) && (contpte_phys(pte, shift) >> shift) == pfn && pgprot_val(prot) == pgprot_val(orig_prot); } @@ -386,6 +410,8 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) * because it is not part of a contpte range. */ + unsigned int shift = arm64_ptep_page_shift(orig_ptep); + unsigned int nr = arm64_cont_ptes(shift); pgprot_t orig_prot; unsigned long pfn; pte_t orig_pte; @@ -399,22 +425,22 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) if (!pte_valid_cont(orig_pte)) return orig_pte; - orig_prot = pte_pgprot(pte_mkold(pte_mkclean(orig_pte))); + orig_prot = contpte_prot(pte_mkold(pte_mkclean(orig_pte)), shift); ptep = contpte_align_down(orig_ptep); - pfn = pte_pfn(orig_pte) - (orig_ptep - ptep); + pfn = (contpte_phys(orig_pte, shift) >> shift) - (orig_ptep - ptep); - for (i = 0; i < CONT_PTES; i++, ptep++, pfn++) { + for (i = 0; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); - for (; i < CONT_PTES; i++, ptep++, pfn++) { + for (; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_young(pte)) { @@ -430,10 +456,10 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) i++; ptep++; pfn++; - for (; i < CONT_PTES; i++, ptep++, pfn++) { + for (; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_dirty(pte)) { @@ -468,16 +494,16 @@ void contpte_set_ptes(struct mm_struct *mm, unsigned long addr, if (!mm_is_user(mm)) return __set_ptes(mm, addr, ptep, pte, nr); - end = addr + (nr << PAGE_SHIFT); - pfn = pte_pfn(pte); - prot = pte_pgprot(pte); + end = addr + (nr << mm_page_shift(mm)); + pfn = pte_phys_mm(mm, pte) >> mm_page_shift(mm); + prot = pte_pgprot_mm(mm, pte); do { - next = pte_cont_addr_end(addr, end); - nr = (next - addr) >> PAGE_SHIFT; - pte = pfn_pte(pfn, prot); + next = pte_cont_addr_end_mm(mm, addr, end); + nr = (next - addr) >> mm_page_shift(mm); + pte = phys_pte_mm(mm, pfn << mm_page_shift(mm), prot); - if (((addr | next | (pfn << PAGE_SHIFT)) & ~CONT_PTE_MASK) == 0) + if (((addr | next | (pfn << mm_page_shift(mm))) & (cont_pte_size_mm(mm) - 1)) == 0) pte = pte_mkcont(pte); else pte = pte_mknoncont(pte); @@ -512,6 +538,8 @@ EXPORT_SYMBOL_GPL(contpte_get_and_clear_full_ptes); bool contpte_test_and_clear_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr) { + struct mm_struct *mm = vma->vm_mm; + /* * ptep_clear_flush_young() technically requires us to clear the access * flag for a _single_ pte. However, the core-mm code actually tracks @@ -524,11 +552,11 @@ bool contpte_test_and_clear_young_ptes(struct vm_area_struct *vma, * of the same large folio in a single VMA and a single page table. */ - unsigned long end = addr + nr * PAGE_SIZE; + unsigned long end = addr + nr * mm_page_size(mm); bool young = false; - ptep = contpte_align_addr_ptep(&addr, &end, ptep, nr); - for (; addr != end; ptep++, addr += PAGE_SIZE) + ptep = contpte_align_addr_ptep(vma->vm_mm, &addr, &end, ptep, nr); + for (; addr != end; ptep++, addr += mm_page_size(mm)) young |= __ptep_test_and_clear_young(vma, addr, ptep); return young; @@ -538,19 +566,21 @@ EXPORT_SYMBOL_GPL(contpte_test_and_clear_young_ptes); bool contpte_clear_flush_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr) { + struct mm_struct *mm = vma->vm_mm; + bool young; young = contpte_test_and_clear_young_ptes(vma, addr, ptep, nr); if (young) { - unsigned long end = addr + nr * PAGE_SIZE; + unsigned long end = addr + nr * mm_page_size(mm); - contpte_align_addr_ptep(&addr, &end, ptep, nr); + contpte_align_addr_ptep(vma->vm_mm, &addr, &end, ptep, nr); /* * See comment in __ptep_clear_flush_young(); same rationale for * eliding the trailing DSB applies here. */ - __flush_tlb_range(vma, addr, end, PAGE_SIZE, 3, + __flush_tlb_range(vma, addr, end, mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); } @@ -580,6 +610,8 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr, cydp_t flags) { + struct mm_struct *mm = vma->vm_mm; + /* * We can safely clear access/dirty without needing to unfold from * the architectures perspective, even when contpte is set. If the @@ -591,14 +623,15 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, * clearing access/dirty for the whole block. */ unsigned long start = addr; - unsigned long end = start + nr * PAGE_SIZE; + unsigned long end = start + nr * mm_page_size(mm); - ptep = contpte_align_addr_ptep(&start, &end, ptep, nr); - __clear_young_dirty_ptes(vma, start, ptep, (end - start) / PAGE_SIZE, flags); + ptep = contpte_align_addr_ptep(vma->vm_mm, &start, &end, ptep, nr); + __clear_young_dirty_ptes(vma, start, ptep, (end - start) >> mm_page_shift(mm), flags); } EXPORT_SYMBOL_GPL(contpte_clear_young_dirty_ptes); -static bool contpte_all_subptes_match_access_flags(pte_t *ptep, pte_t entry) +static bool contpte_all_subptes_match_access_flags(struct mm_struct *mm, + pte_t *ptep, pte_t entry) { pte_t *cont_ptep = contpte_align_down(ptep); /* @@ -609,7 +642,7 @@ static bool contpte_all_subptes_match_access_flags(pte_t *ptep, pte_t entry) pteval_t entry_cmp = pte_val(entry) & cmp_mask; int i; - for (i = 0; i < CONT_PTES; i++) { + for (i = 0; i < cont_ptes_mm(mm); i++) { pteval_t pte_cmp = pte_val(__ptep_get(cont_ptep + i)) & cmp_mask; if (pte_cmp != entry_cmp) @@ -623,6 +656,8 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t entry, int dirty) { + struct mm_struct *mm = vma->vm_mm; + unsigned long start_addr; pte_t orig_pte; int i; @@ -651,7 +686,7 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * no-op, and when any sub-PTE mismatches, proceed to update the whole * range. */ - if (contpte_all_subptes_match_access_flags(ptep, entry)) + if (contpte_all_subptes_match_access_flags(mm, ptep, entry)) return 0; /* @@ -672,7 +707,7 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * and instead flush the whole range at the end. */ ptep = contpte_align_down(ptep); - start_addr = addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); + start_addr = addr = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); /* * We are not advancing entry because __ptep_set_access_flags() @@ -680,13 +715,13 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * for the whole contpte block and returned early, pte_same() * within __ptep_set_access_flags() is likely false. */ - for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) + for (i = 0; i < cont_ptes_mm(mm); i++, ptep++, addr += mm_page_size(mm)) __ptep_set_access_flags(vma, addr, ptep, entry, 0); if (dirty) __flush_tlb_range(vma, start_addr, - start_addr + CONT_PTE_SIZE, - PAGE_SIZE, 3, + start_addr + cont_pte_size_mm(mm), + mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOBROADCAST); } else { __contpte_try_unfold(vma->vm_mm, addr, ptep, orig_pte); diff --git a/arch/arm64/mm/copypage.c b/arch/arm64/mm/copypage.c index cd5912ba617b7..b81ccf42b22d1 100644 --- a/arch/arm64/mm/copypage.c +++ b/arch/arm64/mm/copypage.c @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -72,3 +73,54 @@ void copy_user_highpage(struct page *to, struct page *from, flush_dcache_page(to); } EXPORT_SYMBOL_GPL(copy_user_highpage); + +#ifdef CONFIG_MM_SUBPAGE +void clear_user_subpage_range(struct page *page, unsigned int offset, unsigned int size) +{ + void *addr = page_address(page) + offset; + + VM_BUG_ON(!is_power_of_2(size) || size < SZ_4K || size > PAGE_SIZE); + VM_BUG_ON(!IS_ALIGNED(offset, size) || offset > PAGE_SIZE - size); + memset(addr, 0, size); + if (system_supports_mte() && page_mte_tagged(page)) + mte_clear_subpage_tags_range(addr, size); + flush_dcache_page(page); +} + +int copy_user_subpage_range(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset, unsigned int size) +{ + void *kto = page_address(to), *kfrom = page_address(from); + + VM_BUG_ON(!is_power_of_2(size) || size < SZ_4K || size > PAGE_SIZE); + VM_BUG_ON(!IS_ALIGNED(to_offset, size) || to_offset > PAGE_SIZE - size); + VM_BUG_ON(!IS_ALIGNED(from_offset, size) || from_offset > PAGE_SIZE - size); + if (PageHWPoison(from) || copy_mc_to_kernel(kto + to_offset, kfrom + from_offset, size)) + return -EHWPOISON; + if (kasan_hw_tags_enabled()) + page_kasan_tag_reset(to); + if (system_supports_mte()) { + if (page_mte_tagged(from)) { + if (try_page_mte_tagging(to)) + mte_clear_page_tags(kto); + mte_copy_subpage_tags_range(kto + to_offset, kfrom + from_offset, size); + set_page_mte_tagged(to); + } else if (page_mte_tagged(to)) { + mte_clear_subpage_tags_range(kto + to_offset, size); + } + } + flush_dcache_page(to); + return 0; +} +/* Existing 4K fault and ownership callers keep their fixed-size contract. */ +void clear_user_subpage(struct page *page, unsigned int offset) +{ + clear_user_subpage_range(page, offset, SZ_4K); +} + +int copy_user_subpage(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset) +{ + return copy_user_subpage_range(to, to_offset, from, from_offset, SZ_4K); +} +#endif diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 739800835920c..6a37063417c4c 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -150,7 +150,7 @@ static void show_pte(unsigned long addr) } pr_alert("%s pgtable: %luk pages, %llu-bit VAs, pgdp=%016lx\n", - mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K, + mm == &init_mm ? "swapper" : "user", mm_page_size(mm) / SZ_1K, vabits_actual, mm_to_pgd_phys(mm)); pgdp = pgd_offset(mm, addr); pgd = READ_ONCE(*pgdp); @@ -165,25 +165,25 @@ static void show_pte(unsigned long addr) if (pgd_none(pgd) || pgd_bad(pgd)) break; - p4dp = p4d_offset(pgdp, addr); + p4dp = p4d_offset_mm(mm, pgdp, addr); p4d = READ_ONCE(*p4dp); pr_cont(", p4d=%016llx", p4d_val(p4d)); - if (p4d_none(p4d) || p4d_bad(p4d)) + if (p4d_none_mm(mm, p4d) || p4d_bad_mm(mm, p4d)) break; - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_mm(mm, p4dp, addr); pud = READ_ONCE(*pudp); pr_cont(", pud=%016llx", pud_val(pud)); if (pud_none(pud) || pud_bad(pud)) break; - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_mm(mm, pudp, addr); pmd = READ_ONCE(*pmdp); pr_cont(", pmd=%016llx", pmd_val(pmd)); if (pmd_none(pmd) || pmd_bad(pmd)) break; - ptep = pte_offset_map(pmdp, addr); + ptep = pte_offset_map_mm(mm, pmdp, addr); if (!ptep) break; @@ -241,19 +241,15 @@ int __ptep_set_access_flags_anysz(struct vm_area_struct *vma, * flush_tlb_fix_spurious_fault(). */ if (dirty) { - switch (pgsize) { - case PAGE_SIZE: + if (pgsize == mm_page_size(vma->vm_mm)) { level = 3; - break; - case PMD_SIZE: + } else if (pgsize == pmd_size_mm(vma->vm_mm)) { level = 2; - break; #ifndef __PAGETABLE_PMD_FOLDED - case PUD_SIZE: + } else if (pgsize == pud_size_mm(vma->vm_mm)) { level = 1; - break; #endif - default: + } else { level = TLBI_TTL_UNKNOWN; WARN_ON(1); } @@ -1002,8 +998,8 @@ NOKPROBE_SYMBOL(do_sp_pc_abort); /* * Used during anonymous page fault handling. */ -struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, - unsigned long vaddr) +struct folio *vma_alloc_zeroed_movable_folio_order(struct vm_area_struct *vma, + unsigned long vaddr, unsigned int order) { gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO; @@ -1015,7 +1011,7 @@ struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, if (vma->vm_flags & VM_MTE) flags |= __GFP_ZEROTAGS; - return vma_alloc_folio(flags, 0, vma, vaddr); + return vma_alloc_folio(flags, order, vma, vaddr); } bool tag_clear_highpages(struct page *page, int numpages, bool clear_pages) diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c index c5c5425791da7..b670d27a898cc 100644 --- a/arch/arm64/mm/fixmap.c +++ b/arch/arm64/mm/fixmap.c @@ -77,7 +77,7 @@ static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr, p4d_t p4d = READ_ONCE(*p4dp); pud_t *pudp; - if (CONFIG_PGTABLE_LEVELS > 3 && !p4d_none(p4d) && + if (pgtable_l4_enabled() && !p4d_none(p4d) && p4d_page_paddr(p4d) != __pa_symbol(bm_pud)) { /* * We only end up here if the kernel mapping and the fixmap diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 97987f850a33c..1e5f314b542d5 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -368,8 +368,9 @@ void __init arch_mm_preinit(void) * Selected page table levels should match when derived from * scratch using the virtual address range and page size. */ - BUILD_BUG_ON(ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS) != - CONFIG_PGTABLE_LEVELS); + BUILD_BUG_ON(CONFIG_PGTABLE_LEVELS != + (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) ? 4 : + ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS))); if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) { extern int sysctl_overcommit_memory; diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 09fd9da4eeed4..976d115d72f3f 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -2253,8 +2253,8 @@ pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr, * in cases where cpu is affected with errata #2645198. */ if (pte_accessible(vma->vm_mm, pte) && pte_user_exec(pte)) - __flush_tlb_range(vma, addr, nr * PAGE_SIZE, - PAGE_SIZE, 3, TLBF_NOWALKCACHE); + __flush_tlb_range(vma, addr, addr + nr * mm_page_size(vma->vm_mm), + mm_page_size(vma->vm_mm), 3, TLBF_NOWALKCACHE); } return pte; diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c index bf5110b91e2fa..3b361b95322e1 100644 --- a/arch/arm64/mm/pgd.c +++ b/arch/arm64/mm/pgd.c @@ -16,14 +16,19 @@ #include static struct kmem_cache *pgd_cache __ro_after_init; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +static struct kmem_cache *user4k_pgd_cache __ro_after_init; +static struct kmem_cache *user16k_pgd_cache __ro_after_init; +static struct kmem_cache *user64k_pgd_cache __ro_after_init; +#endif static bool pgdir_is_page_size(void) { if (PGD_SIZE == PAGE_SIZE) return true; - if (CONFIG_PGTABLE_LEVELS == 4) + if (ARM64_NATIVE_PGTABLE_LEVELS == 4) return !pgtable_l4_enabled(); - if (CONFIG_PGTABLE_LEVELS == 5) + if (ARM64_NATIVE_PGTABLE_LEVELS == 5) return !pgtable_l5_enabled(); return false; } @@ -32,6 +37,14 @@ pgd_t *pgd_alloc(struct mm_struct *mm) { gfp_t gfp = GFP_PGTABLE_USER; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_context_user4k(&mm->context)) + return kmem_cache_alloc(user4k_pgd_cache, gfp); + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 14) + return kmem_cache_alloc(user16k_pgd_cache, gfp); + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 16) + return kmem_cache_alloc(user64k_pgd_cache, gfp); +#endif if (pgdir_is_page_size()) return __pgd_alloc(mm, 0); else @@ -40,6 +53,20 @@ pgd_t *pgd_alloc(struct mm_struct *mm) void pgd_free(struct mm_struct *mm, pgd_t *pgd) { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_context_user4k(&mm->context)) { + kmem_cache_free(user4k_pgd_cache, pgd); + return; + } + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 14) { + kmem_cache_free(user16k_pgd_cache, pgd); + return; + } + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 16) { + kmem_cache_free(user64k_pgd_cache, pgd); + return; + } +#endif if (pgdir_is_page_size()) __pgd_free(mm, pgd); else @@ -48,6 +75,16 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd) void __init pgtable_cache_init(void) { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (PAGE_SHIFT < 16) + user64k_pgd_cache = kmem_cache_create("user64k_pgd_cache", SZ_64K, SZ_64K, + SLAB_PANIC, NULL); + if (PAGE_SHIFT != 14) + user16k_pgd_cache = kmem_cache_create("user16k_pgd_cache", SZ_16K, SZ_16K, + SLAB_PANIC, NULL); + user4k_pgd_cache = kmem_cache_create("user4k_pgd_cache", SZ_4K, SZ_4K, + SLAB_PANIC, NULL); +#endif if (pgdir_is_page_size()) return; diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 22866b49be372..7f947589a3b4d 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -291,7 +291,7 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) cbnz cpu, __idmap_kpti_secondary -#if CONFIG_PGTABLE_LEVELS > 4 +#if ARM64_NATIVE_PGTABLE_LEVELS > 4 stp x29, x30, [sp, #-32]! mov x29, sp stp x19, x20, [sp, #16] @@ -352,7 +352,7 @@ alternative_else_nop_endif /* Set the flag to zero to indicate that we're all done */ str wzr, [flag_ptr] -#if CONFIG_PGTABLE_LEVELS > 4 +#if ARM64_NATIVE_PGTABLE_LEVELS > 4 ldp x19, x20, [sp, #16] ldp x29, x30, [sp], #32 #endif @@ -360,39 +360,39 @@ alternative_else_nop_endif .Lderef_pgd: /* P4D */ - .if CONFIG_PGTABLE_LEVELS > 4 + .if ARM64_NATIVE_PGTABLE_LEVELS > 4 p4d .req x30 pte_to_phys cur_p4dp, pgd kpti_map_pgtbl p4d, 0 kpti_mk_tbl_ng p4d, PTRS_PER_P4D b .Lnext_pgd - .else /* CONFIG_PGTABLE_LEVELS <= 4 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 4 */ p4d .req pgd .set .Lnext_p4d, .Lnext_pgd .endif .Lderef_p4d: /* PUD */ - .if CONFIG_PGTABLE_LEVELS > 3 + .if ARM64_NATIVE_PGTABLE_LEVELS > 3 pud .req x10 pte_to_phys cur_pudp, p4d kpti_map_pgtbl pud, 1 kpti_mk_tbl_ng pud, PTRS_PER_PUD b .Lnext_p4d - .else /* CONFIG_PGTABLE_LEVELS <= 3 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 3 */ pud .req pgd .set .Lnext_pud, .Lnext_pgd .endif .Lderef_pud: /* PMD */ - .if CONFIG_PGTABLE_LEVELS > 2 + .if ARM64_NATIVE_PGTABLE_LEVELS > 2 pmd .req x13 pte_to_phys cur_pmdp, pud kpti_map_pgtbl pmd, 2 kpti_mk_tbl_ng pmd, PTRS_PER_PMD b .Lnext_pud - .else /* CONFIG_PGTABLE_LEVELS <= 2 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 2 */ pmd .req pgd .set .Lnext_pmd, .Lnext_pgd .endif diff --git a/arch/arm64/mm/user4k-data-test.c b/arch/arm64/mm/user4k-data-test.c new file mode 100644 index 0000000000000..d7d550c9e1420 --- /dev/null +++ b/arch/arm64/mm/user4k-data-test.c @@ -0,0 +1,1835 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Backing ownership tests; these slots are not fault/rmap-owned mappings. */ +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_NET +#include +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) +#include +#endif +#ifdef CONFIG_INET +#include +#include +#endif +#include +#endif +#include +#include + +#define DATA_SLOTS MM_SUBPAGES_PER_PAGE + +struct data_fixture { + struct mm_subpage_pool *pool; + struct mm_subpage *held[2 * DATA_SLOTS + 2]; +}; + +static void data_fixture_free(void *arg) +{ + struct data_fixture *fixture = arg; + unsigned int i; + + if (fixture->pool) + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < ARRAY_SIZE(fixture->held); i++) + if (fixture->held[i]) + mm_subpage_put(fixture->held[i]); + if (fixture->pool) + mm_subpage_pool_put(fixture->pool); +} + +static struct data_fixture *data_fixture_create_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + + if (!fixture) + return NULL; + fixture->pool = mm_subpage_pool_create_granule(GFP_KERNEL, shift); + if (!fixture->pool) + return NULL; + if (kunit_add_action_or_reset(test, data_fixture_free, fixture)) + return NULL; + return fixture; +} + +static struct data_fixture *data_fixture_create(struct kunit *test) +{ + return data_fixture_create_granule(test, MM_SUBPAGE_SHIFT); +} + +static void put_held(struct data_fixture *fixture, unsigned int index) +{ + mm_subpage_put(fixture->held[index]); + fixture->held[index] = NULL; +} + +static struct mm_subpage *hold_alloc(struct data_fixture *fixture, unsigned int index) +{ + struct mm_subpage *slot = mm_subpage_alloc(fixture->pool); + + if (!IS_ERR(slot)) + fixture->held[index] = slot; + return slot; +} + +static int supply_backing(struct mm_subpage_pool *pool) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + int ret; + + if (!folio) + return -ENOMEM; + /* Check that stale allocation contents never become exposed. */ + memset(folio_address(folio), 0xcc, PAGE_SIZE); + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) + folio_put(folio); + return ret; +} + +static int supply_backing_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + int ret; + + if (!folio) + return -ENOMEM; + ret = mm_subpage_pool_add_folio_at(pool, folio, offset, GFP_KERNEL); + if (ret) + folio_put(folio); + return ret; +} + +static struct mm_subpage *hold_alloc_at(struct data_fixture *fixture, unsigned int index, + unsigned int offset) +{ + struct mm_subpage *slot = mm_subpage_alloc_at(fixture->pool, offset); + + if (!IS_ERR(slot)) + fixture->held[index] = slot; + return slot; +} + +static void *slot_address(struct mm_subpage *slot) +{ + return folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot); +} + +static void user4k_data_reuse_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct folio *folio; + struct mm_subpage *slot; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -EEXIST); + for (i = 0; i < DATA_SLOTS; i++) { + slot = hold_alloc(fixture, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)i + 1); + KUNIT_EXPECT_PTR_EQ(test, folio, mm_subpage_folio(fixture->held[0])); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), PFN_PHYS(folio_pfn(folio)) + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_4K), NULL); + memset(slot_address(slot), 0x10 + i, SZ_4K); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + /* A pin, alias or deferred-TLB holder keeps this slot unavailable. */ + fixture->held[DATA_SLOTS] = fixture->held[1]; + mm_subpage_get(fixture->held[DATA_SLOTS]); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS + 1); + put_held(fixture, 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + put_held(fixture, DATA_SLOTS); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS - 1); + slot = hold_alloc(fixture, DATA_SLOTS + 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_4K), NULL); + for (i = 0; i < DATA_SLOTS; i++) { + if (i != 1) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i]), + 0x10 + i, SZ_4K), NULL); + } + mm_subpage_pool_close(fixture->pool); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -ESHUTDOWN); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -ESHUTDOWN); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 1) + put_held(fixture, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + put_held(fixture, DATA_SLOTS + 1); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_copy_test(struct kunit *test) +{ + struct data_fixture *source = data_fixture_create(test); + struct data_fixture *dest = data_fixture_create(test); + struct mm_subpage *copy; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, source); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_EQ(test, supply_backing(source->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(dest->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(source, i)); + memset(slot_address(source->held[i]), 0x50 + i, SZ_4K); + } + source->held[DATA_SLOTS] = source->held[1]; + mm_subpage_get(source->held[DATA_SLOTS]); + copy = mm_subpage_copy(dest->pool, source->held[1]); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + dest->held[0] = copy; + KUNIT_EXPECT_NE(test, mm_subpage_phys(copy), mm_subpage_phys(source->held[1])); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x51, SZ_4K), NULL); + memset(slot_address(copy), 0xee, SZ_4K); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[i]), 0x50 + i, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(source->pool, source->held[1])), -EAGAIN); + /* Copy into a free neighboring slot in the same physical owner. */ + put_held(source, 2); + copy = mm_subpage_copy(source->pool, source->held[1]); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + source->held[2] = copy; + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(copy), mm_subpage_folio(source->held[1])); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(copy), 2U * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x51, SZ_4K), NULL); + memset(slot_address(copy), 0xdd, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[1]), 0x51, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[3]), 0x53, SZ_4K), NULL); +} + +static void user4k_data_lookup_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *slot, *found; + struct folio *folio; + phys_addr_t phys; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + phys = PFN_PHYS(folio_pfn(folio)); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys), NULL); + if (mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL)) { + folio_put(folio); + KUNIT_FAIL(test, "failed to supply lookup backing"); + return; + } + /* A published owner does not expose uninitialised or unissued slots. */ + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K), NULL); + for (i = 0; i < DATA_SLOTS; i++) { + slot = hold_alloc(fixture, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memset(slot_address(slot), 0x90 + i, SZ_4K); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2 * (int)i + 1); + if (!i) + KUNIT_EXPECT_EQ(test, mm_subpage_pool_add_folio(fixture->pool, folio, + GFP_KERNEL), -EEXIST); + found = mm_subpage_get_from_phys(phys + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, found); + fixture->held[i + DATA_SLOTS] = found; + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2 * (int)i + 2); + KUNIT_EXPECT_PTR_EQ(test, found, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K + 1), NULL); + } + /* A second pool must not zero or acquire already owned backing. */ + KUNIT_EXPECT_EQ(test, mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL), + -EINVAL); + for (i = 0; i < DATA_SLOTS; i++) { + put_held(fixture, i); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i + DATA_SLOTS]), + 0x90 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + put_held(fixture, DATA_SLOTS + 1); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + SZ_4K), NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[1]), 0, SZ_4K), NULL); + mm_subpage_pool_close(fixture->pool); + /* Keep the physical allocation stable while checking final unbinding. */ + folio_get(folio); + put_held(fixture, 1); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 1) + put_held(fixture, i + DATA_SLOTS); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); +} + +static void user4k_data_exact_slot_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *source, *copy, *reused; + struct folio *first; + static const unsigned int neighbors[] = { 0, SZ_4K, 3 * SZ_4K }; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_EXPECT_EQ(test, supply_backing_at(fixture->pool, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(fixture->pool, PAGE_SIZE)), -EINVAL); + KUNIT_ASSERT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), 0); + source = hold_alloc_at(fixture, 0, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, source); + first = mm_subpage_folio(source); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(source), (unsigned int)SZ_8K); + memset(slot_address(source), 0xa7, SZ_4K); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy_at(fixture->pool, source, 1)), -EINVAL); + /* Three free neighbors cannot satisfy a request for this occupied slot. */ + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy_at(fixture->pool, source, SZ_8K)), -EAGAIN); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -EEXIST); + KUNIT_ASSERT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), 0); + KUNIT_EXPECT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), -EEXIST); + copy = mm_subpage_copy_at(fixture->pool, source, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + fixture->held[1] = copy; + KUNIT_EXPECT_PTR_NE(test, mm_subpage_folio(copy), first); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(copy), (unsigned int)SZ_8K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0xa7, SZ_4K), NULL); + memset(slot_address(copy), 0x4b, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source), 0xa7, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 2UL); + for (i = 0; i < ARRAY_SIZE(neighbors); i++) { + struct mm_subpage *slot = hold_alloc_at(fixture, i + 2, neighbors[i]); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), first); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), neighbors[i]); + memset(slot_address(slot), 0x30 + i, SZ_4K); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(first), 4); + fixture->held[5] = source; + mm_subpage_get(source); + put_held(fixture, 0); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(fixture->pool, SZ_8K)), -EAGAIN); + put_held(fixture, 5); + reused = hold_alloc_at(fixture, 6, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, reused); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(reused), first); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(reused), 0, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(first), 4); + for (i = 0; i < ARRAY_SIZE(neighbors); i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i + 2]), + 0x30 + i, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x4b, SZ_4K), NULL); + mm_subpage_pool_close(fixture->pool); + for (i = 2; i <= 4; i++) + put_held(fixture, i); + put_held(fixture, 6); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + put_held(fixture, 1); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_close_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *slot; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + /* Closing with no issued slots must release that entire unused owner. */ + mm_subpage_pool_close(fixture->pool); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); + fixture = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + slot = hold_alloc(fixture, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memset(slot_address(slot), 0xa7, SZ_4K); + mm_subpage_pool_close(fixture->pool); + mm_subpage_pool_put(fixture->pool); + fixture->pool = NULL; + /* The originating pool/mm reference is gone; a child's slot remains live. */ + fixture->held[1] = slot; + mm_subpage_get(slot); + put_held(fixture, 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0xa7, SZ_4K), NULL); + put_held(fixture, 1); +} + +struct deferred_slot { + struct rcu_head rcu; + struct mm_subpage *slot; + struct completion done; +}; + +static void deferred_slot_put(struct rcu_head *rcu) +{ + struct deferred_slot *pending = container_of(rcu, struct deferred_slot, rcu); + + mm_subpage_put(pending->slot); + complete(&pending->done); +} + +static void data_rcu_barrier(void *unused) +{ + rcu_barrier(); +} + +static void user4k_data_deferred_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct deferred_slot *pending = kunit_kzalloc(test, sizeof(*pending), GFP_KERNEL); + struct mm_subpage *probe; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, pending); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, i)); + init_completion(&pending->done); + pending->slot = fixture->held[0]; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, data_rcu_barrier, NULL), 0); + mm_subpage_get(pending->slot); + rcu_read_lock(); + call_rcu(&pending->rcu, deferred_slot_put); + put_held(fixture, 0); + /* The delayed reference must prevent recycling before the grace period. */ + probe = mm_subpage_alloc(fixture->pool); + for (i = 1; i < DATA_SLOTS; i++) + put_held(fixture, i); + rcu_read_unlock(); + if (!IS_ERR(probe)) + fixture->held[DATA_SLOTS] = probe; + KUNIT_EXPECT_EQ(test, PTR_ERR(probe), -EAGAIN); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&pending->done, 5 * HZ), 0UL); + /* The callback retired the final live slot and the whole native owner. */ + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_backing_reject_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + folio = folio_alloc(GFP_KERNEL, 1); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + if (ret) + folio_put(folio); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_get(folio); + ret = mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2); + folio_put(folio); + if (ret) + folio_put(folio); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +struct data_worker { + struct mm_subpage_pool *pool; + struct task_struct *task; + struct completion ready, go, done; + unsigned int id, checks; + int error; +}; + +static struct mm_subpage *alloc_with_backing(struct mm_subpage_pool *pool) +{ + unsigned int tries; + + for (tries = 0; tries < 128; tries++) { + struct mm_subpage *slot = mm_subpage_alloc(pool); + int ret; + + if (!IS_ERR(slot) || PTR_ERR(slot) != -EAGAIN) + return slot; + ret = supply_backing(pool); + if (ret && ret != -EEXIST) + return ERR_PTR(ret); + cond_resched(); + } + return ERR_PTR(-EAGAIN); +} + +static int data_worker_run(void *arg) +{ + struct data_worker *worker = arg; + struct mm_subpage *slot = NULL; + unsigned int round; + u8 pattern = 0; + + for (round = 0; round < 64; round++) { + slot = alloc_with_backing(worker->pool); + if (IS_ERR(slot)) { + worker->error = PTR_ERR(slot); + slot = NULL; + break; + } + pattern = 1 + round + worker->id * 64; + memset(slot_address(slot), pattern, SZ_4K); + mm_subpage_get(slot); + mm_subpage_put(slot); + schedule_timeout_uninterruptible(1); + if (memchr_inv(slot_address(slot), pattern, SZ_4K)) { + worker->error = -EIO; + break; + } + worker->checks++; + if (round != 63) { + mm_subpage_put(slot); + slot = NULL; + } + } + complete(&worker->ready); + wait_for_completion(&worker->go); + if (slot) { + if (memchr_inv(slot_address(slot), pattern, SZ_4K)) + worker->error = -EIO; + mm_subpage_put(slot); + } + complete(&worker->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void data_worker_stop(void *arg) +{ + struct data_worker *worker = arg; + + complete_all(&worker->go); + kthread_stop(worker->task); +} + +static void user4k_data_concurrent_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct data_worker *workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].pool = fixture->pool; + workers[i].id = i; + init_completion(&workers[i].ready); + init_completion(&workers[i].go); + init_completion(&workers[i].done); + workers[i].task = kthread_run(data_worker_run, &workers[i], "user4k-data-%u", i); + KUNIT_ASSERT_FALSE(test, IS_ERR(workers[i].task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, data_worker_stop, &workers[i]), 0); + } + for (i = 0; i < 2; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < 2; i++) + complete(&workers[i].go); + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_EQ(test, workers[i].checks, 64U); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +struct lookup_worker { + struct task_struct *task; + struct completion observed; + phys_addr_t phys; + unsigned long checks; + int error; +}; + +static int lookup_worker_run(void *arg) +{ + struct lookup_worker *worker = arg; + bool observed = false; + + while (!kthread_should_stop()) { + struct mm_subpage *slot = mm_subpage_get_from_phys(worker->phys); + + if (slot) { + unsigned char *data = slot_address(slot); + unsigned char value = READ_ONCE(*data); + + if (mm_subpage_phys(slot) != worker->phys || + (value != 0x5a && value != 0xab) || + memchr_inv(data, value, SZ_4K)) + worker->error = -EINVAL; + worker->checks++; + mm_subpage_put(slot); + if (!observed) { + complete(&worker->observed); + observed = true; + } + if (worker->error) + break; + } + cond_resched(); + } + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void lookup_worker_stop(void *arg) +{ + struct lookup_worker *worker = arg; + + if (worker->task) { + kthread_stop(worker->task); + worker->task = NULL; + } +} + +static void user4k_data_lookup_race_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct lookup_worker *worker = kunit_kzalloc(test, sizeof(*worker), GFP_KERNEL); + unsigned int copied = 0, i; + unsigned long deadline; + struct folio *folio; + struct mm_subpage *found; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, worker); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, i)); + memset(slot_address(fixture->held[0]), 0x5a, SZ_4K); + memset(slot_address(fixture->held[1]), 0xab, SZ_4K); + worker->phys = mm_subpage_phys(fixture->held[2]); + put_held(fixture, 2); + init_completion(&worker->observed); + worker->task = kthread_run(lookup_worker_run, worker, "user4k-lookup"); + KUNIT_ASSERT_FALSE(test, IS_ERR(worker->task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, lookup_worker_stop, worker), 0); + if (num_online_cpus() > 1) { + unsigned int cpu = cpumask_next(cpumask_first(cpu_online_mask), cpu_online_mask); + + KUNIT_ASSERT_EQ(test, set_cpus_allowed_ptr(worker->task, cpumask_of(cpu)), 0); + } + /* Alternate complete contents; a reader must never see a partial copy. */ + deadline = jiffies + 30 * HZ; + while (copied < 2048 && time_before(jiffies, deadline)) { + struct mm_subpage *dest = mm_subpage_copy(fixture->pool, + fixture->held[copied & 1]); + bool observed = true; + phys_addr_t phys; + + if (PTR_ERR_OR_ZERO(dest) == -EAGAIN) { + /* Let a scheduled-out lookup holder retire its reference. */ + schedule_timeout_uninterruptible(1); + continue; + } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dest); + phys = mm_subpage_phys(dest); + if (!copied) + observed = wait_for_completion_timeout(&worker->observed, 5 * HZ) != 0; + mm_subpage_put(dest); + KUNIT_ASSERT_TRUE(test, observed); + KUNIT_ASSERT_EQ(test, phys, worker->phys); + copied++; + cond_resched(); + } + /* Retire the owner while lookup is still running, then verify unbinding. */ + folio = mm_subpage_folio(fixture->held[0]); + folio_get(folio); + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 2) + put_held(fixture, i); + lookup_worker_stop(worker); + found = mm_subpage_get_from_phys(worker->phys); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + if (found) + mm_subpage_put(found); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); + KUNIT_EXPECT_EQ(test, worker->error, 0); + KUNIT_EXPECT_GT(test, worker->checks, 0UL); + KUNIT_EXPECT_EQ(test, copied, 2048U); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_swapcache_reuse_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct mm_subpage *attempt; + struct folio *folio; + unsigned int i; + bool intact; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * MM_SUBPAGE_SIZE)); + memset(slot_address(f->held[i]), 0x60 + i, MM_SUBPAGE_SIZE); + } + folio = mm_subpage_folio(f->held[0]); + put_held(f, 1); + /* Synthetic cache membership only; no swap entry or I/O is fabricated. */ + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + attempt = mm_subpage_alloc_at(f->pool, MM_SUBPAGE_SIZE); + intact = !memchr_inv(folio_address(folio) + MM_SUBPAGE_SIZE, 0x61, MM_SUBPAGE_SIZE); + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + /* Restore the real state before assertions can unwind the fixture. */ + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_EXPECT_TRUE(test, intact); + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, MM_SUBPAGE_SIZE)); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x60 + i, MM_SUBPAGE_SIZE), NULL); +} + +static void restore_folio_free(void *arg) +{ + struct folio *folio = arg; + + /* Data-only fixtures never create a real swap entry or mapping. */ + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + folio_put(folio); +} + +static struct mm_subpage *restore_test_slot_granule(struct folio *folio, unsigned int offset, unsigned int shift) +{ + struct mm_subpage *slot; + + folio_lock(folio); + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + slot = mm_subpage_restore_granule(folio, offset, shift, GFP_KERNEL); + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + folio_unlock(folio); + return slot; +} + +static struct mm_subpage *restore_test_slot(struct folio *folio, unsigned int offset) +{ + return restore_test_slot_granule(folio, offset, MM_SUBPAGE_SHIFT); +} + +static void test_data_restore_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct folio *folio; + struct mm_subpage *slot; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + memset(slot_address(f->held[i]), 0x70 + i, size); + } + folio = mm_subpage_folio(f->held[0]); + folio_get(folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, folio), 0); + put_held(f, 1); + slot = restore_test_slot_granule(folio, size, shift); + if (!IS_ERR(slot)) + f->held[1] = slot; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)size); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 5); + slot = restore_test_slot_granule(folio, 0, shift); + if (!IS_ERR(slot)) + f->held[4] = slot; + KUNIT_ASSERT_PTR_EQ(test, slot, f->held[0]); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 6); + put_held(f, 4); + /* Retire the entire old owner, retaining only a native backing reference. */ + for (i = 0; i < 4; i++) + put_held(f, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), 0UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + for (i = 0; i < 4; i++) { + slot = restore_test_slot_granule(folio, i * size, shift); + if (!IS_ERR(slot)) + f->held[i] = slot; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0x70 + i, + size), NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 5); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, 1, shift)), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, PAGE_SIZE, shift)), -EINVAL); + if (shift != MM_SUBPAGE_SHIFT) { + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot(folio, 0)), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, SZ_4K, shift)), -EINVAL); + } + +} + +static void user4k_data_restore_test(struct kunit *test) +{ + test_data_restore_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_restore_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_restore_granule(test, 14); +} + +struct restore_worker { + struct folio *folio; + struct task_struct *task; + struct completion done; + unsigned int checks; + int error; +}; + +static int restore_worker_run(void *arg) +{ + struct restore_worker *w = arg; + unsigned long deadline = jiffies + 20 * HZ; + + while (!kthread_should_stop() && w->checks < 1024 && time_before(jiffies, deadline)) { + struct mm_subpage *slot; + + folio_lock(w->folio); + slot = mm_subpage_restore(w->folio, MM_SUBPAGE_SIZE, GFP_KERNEL); + folio_unlock(w->folio); + if (PTR_ERR_OR_ZERO(slot) == -EAGAIN) { + cond_resched(); + continue; + } + if (IS_ERR(slot)) { + w->error = PTR_ERR(slot); + break; + } + if (memchr_inv(slot_address(slot), 0x83, MM_SUBPAGE_SIZE)) + w->error = -EIO; + /* Retirement deliberately occurs without the folio lock. */ + mm_subpage_put(slot); + if (w->error) + break; + w->checks++; + cond_resched(); + } + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void restore_worker_stop(void *arg) +{ + struct restore_worker *w = arg; + + if (w->task) { + kthread_stop(w->task); + w->task = NULL; + } +} + +static void user4k_data_restore_race_test(struct kunit *test) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + struct restore_worker *workers; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, folio), 0); + memset(folio_address(folio), 0x83, PAGE_SIZE); + __folio_mark_uptodate(folio); + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].folio = folio; + init_completion(&workers[i].done); + workers[i].task = kthread_run(restore_worker_run, &workers[i], "user4k-restore-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "restore worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_worker_stop, &workers[i]), 0); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 25 * HZ), 0UL); + restore_worker_stop(&workers[i]); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_EQ(test, workers[i].checks, 1024U); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); +} + +static void test_data_migration_granule(struct kunit *test, unsigned int shift) +{ + unsigned int mode, size = 1U << shift; + + /* Success, rollback, all PTEs discarded, and partial-folio success. */ + for (mode = 0; mode < 4; mode++) { + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct folio *src, *dst, *target; + unsigned int i, nr = mode == 3 ? 3 : 4; + int ret, src_refs = 0, dst_refs = 0, blocked = 0; + bool unpublished = true, active = false; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, index * size)); + memset(slot_address(f->held[i]), 0x80 + index, size); + } + src = mm_subpage_folio(f->held[0]); + folio_get(src); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, src), 0); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, dst), 0); + folio_lock(src); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(src, dst); + if (!ret) { + struct mm_subpage *probe; + + src_refs = folio_ref_count(src); + dst_refs = folio_ref_count(dst); + active = mm_subpage_migrating(src) && mm_subpage_migrating(dst); + probe = mm_subpage_alloc_at(f->pool, size); + blocked = PTR_ERR_OR_ZERO(probe); + if (!IS_ERR(probe)) + mm_subpage_put(probe); + /* Emulate migration entries: their old mapping refs are gone. */ + for (i = 0; i < nr; i++) + put_held(f, i); + probe = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(src))); + unpublished &= !probe; + if (probe) + mm_subpage_put(probe); + probe = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(dst))); + unpublished &= !probe; + if (probe) + mm_subpage_put(probe); + if (mode != 1 && mode != 2) { + copy_user_highpage(&dst->page, &src->page, 0, NULL); + __folio_mark_uptodate(dst); + } + target = mode == 1 ? src : dst; + if (mode != 2) + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + f->held[i] = mm_subpage_migrate_restore(target, index * size); + } + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, active); + KUNIT_EXPECT_TRUE(test, unpublished); + KUNIT_EXPECT_EQ(test, src_refs, (int)nr + 1); + KUNIT_EXPECT_EQ(test, dst_refs, 1); + KUNIT_EXPECT_EQ(test, blocked, -EAGAIN); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), mode == 2 ? 0UL : 1UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(src), mode == 1 ? (int)nr + 1 : 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(dst), mode == 0 || mode == 3 ? (int)nr + 1 : 1); + if (mode != 2) + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->held[i]), mode == 1 ? src : dst); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + 0x80 + index, size), NULL); + } + if (mode == 3) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 3, size)); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->held[3]), dst); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[3]), 0, + size), NULL); + } + } +} + +static void user4k_data_migration_owner_test(struct kunit *test) +{ + test_data_migration_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_migration_owner_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_migration_granule(test, 14); +} + +#ifdef CONFIG_ARM64_MTE +static void test_slot_tags_set(void *addr, unsigned int tag) +{ + unsigned long value = (unsigned long)tag << MTE_TAG_SHIFT; + unsigned int offset; + + for (offset = 0; offset < MM_SUBPAGE_SIZE; offset += MTE_GRANULE_SIZE) + asm volatile(".arch_extension memtag\n\tstg %0, [%1]" + : : "r" (value), "r" (addr + offset) : "memory"); +} + +static bool test_slot_tags_equal(void *addr, unsigned int expected) +{ + unsigned int offset; + + for (offset = 0; offset < MM_SUBPAGE_SIZE; offset += MTE_GRANULE_SIZE) { + unsigned long value = 0; + + asm volatile(".arch_extension memtag\n\tldg %0, [%1]" + : "+r" (value) : "r" (addr + offset) : "memory"); + if (((value >> MTE_TAG_SHIFT) & 0xf) != expected) + return false; + } + return true; +} +#endif + +static void user4k_data_mte_test(struct kunit *test) +{ +#ifdef CONFIG_ARM64_MTE + struct data_fixture *f; + struct folio *folio; + unsigned int i; + + if (!system_supports_mte()) { + kunit_skip(test, "MTE is not available on this CPU"); + return; + } + f = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * MM_SUBPAGE_SIZE)); + folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_TRUE(test, try_page_mte_tagging(&folio->page)); + mte_clear_page_tags(folio_address(folio)); + for (i = 0; i < 4; i++) { + memset(slot_address(f->held[i]), 0x40 + i, MM_SUBPAGE_SIZE); + test_slot_tags_set(slot_address(f->held[i]), i + 1); + } + set_page_mte_tagged(&folio->page); + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, MM_SUBPAGE_SIZE)); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(slot_address(f->held[i]), i == 1 ? 0 : i + 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x40 + i, MM_SUBPAGE_SIZE), NULL); + } + put_held(f, 1); + f->held[1] = mm_subpage_copy_at(f->pool, f->held[0], MM_SUBPAGE_SIZE); + if (IS_ERR(f->held[1])) { + f->held[1] = NULL; + KUNIT_FAIL(test, "tagged slot copy failed"); + return; + } + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(slot_address(f->held[i]), i == 1 ? 1 : i + 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0x40 : 0x40 + i, MM_SUBPAGE_SIZE), NULL); + } + /* A fresh destination must initialise tags outside the copied slot too. */ + KUNIT_ASSERT_EQ(test, supply_backing_at(f->pool, 3 * MM_SUBPAGE_SIZE), 0); + f->held[4] = mm_subpage_copy_at(f->pool, f->held[2], 3 * MM_SUBPAGE_SIZE); + if (IS_ERR(f->held[4])) { + f->held[4] = NULL; + KUNIT_FAIL(test, "fresh tagged slot copy failed"); + return; + } + folio = mm_subpage_folio(f->held[4]); + KUNIT_EXPECT_TRUE(test, page_mte_tagged(&folio->page)); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(folio_address(folio) + + i * MM_SUBPAGE_SIZE, i == 3 ? 3 : 0)); +#else + kunit_skip(test, "MTE is not configured"); +#endif +} + +static void extent_folio_put(void *arg) +{ + folio_put(arg); +} + +static void user4k_data_copy_extent_test(struct kunit *test) +{ + const unsigned int sizes[] = { SZ_4K, SZ_16K, SZ_64K }; + + for (unsigned int n = 0; n < ARRAY_SIZE(sizes); n++) { + struct folio *source, *dest; + unsigned int size = sizes[n], to_offset; + void *from, *to; + + if (size > PAGE_SIZE) + continue; + source = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, source); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, source), 0); + dest = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, dest), 0); + from = folio_address(source); + to = folio_address(dest); + to_offset = PAGE_SIZE - size; + for (unsigned int i = 0; i < DATA_SLOTS; i++) + memset(from + i * SZ_4K, 0x40 + i, SZ_4K); + for (unsigned int tagged = 0; tagged < 2; tagged++) { + memset(to, 0xac, PAGE_SIZE); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) { + if (try_page_mte_tagging(&dest->page)) + mte_clear_page_tags(to); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(to + i * SZ_4K, 9); + set_page_mte_tagged(&dest->page); + if (tagged) { + if (try_page_mte_tagging(&source->page)) + mte_clear_page_tags(from); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(from + i * SZ_4K, 1 + i % 15); + set_page_mte_tagged(&source->page); + } + } +#endif + KUNIT_ASSERT_EQ(test, copy_user_subpage_range(&dest->page, to_offset, + &source->page, 0, size), 0); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int offset = i * SZ_4K; + bool copied = offset >= to_offset; + unsigned int source_index = copied ? (offset - to_offset) / SZ_4K : 0; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(to + offset, + copied ? 0x40 + source_index : 0xac, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(from + offset, 0x40 + i, SZ_4K), NULL); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(to + offset, + copied ? (tagged ? 1 + source_index % 15 : 0) : 9)); +#endif + } + clear_user_subpage_range(&dest->page, to_offset, size); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int offset = i * SZ_4K; + bool cleared = offset >= to_offset; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(to + offset, cleared ? 0 : 0xac, + SZ_4K), NULL); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(to + offset, cleared ? 0 : 9)); +#endif + } + } + } +} + +#ifdef CONFIG_NET +struct skb_test_state { + struct folio *folio; +#ifdef CONFIG_INET + struct socket *socket; +#endif + struct sk_buff *skbs[5]; +}; + +static void skb_test_cleanup(void *arg) +{ + struct skb_test_state *state = arg; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->skbs); i++) { + if (state->skbs[i]) + state->skbs[i]->sk = NULL; + kfree_skb(state->skbs[i]); + } +#ifdef CONFIG_INET + if (state->socket) + sock_release(state->socket); +#endif + /* This fixture marks ownership only; it installs no anonymous rmap. */ + if (state->folio) + state->folio->mapping = NULL; +} + +static void user4k_data_skb_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct iov_iter iter; + struct bio_vec bvec; + unsigned char *bytes = kunit_kmalloc(test, SZ_4K, GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_NOT_NULL(test, bytes); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + memset(slot_address(f->held[i]), 0x51 + i, SZ_4K); + } + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + state->skbs[0]->ip_summed = CHECKSUM_UNNECESSARY; + bvec_set_page(&bvec, &state->folio->page, SZ_4K, SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(state->skbs[0], &iter, SZ_4K), (ssize_t)SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, skb_zcopy(state->skbs[0])); + state->skbs[1] = pskb_copy(state->skbs[0], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[1]); + /* Append a distinct quarter while the callback object has two owners. */ + bvec_set_page(&bvec, &state->folio->page, SZ_4K, 2 * SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(state->skbs[0], &iter, SZ_4K), (ssize_t)SZ_4K); + KUNIT_EXPECT_PTR_NE(test, skb_zcopy(state->skbs[0]), skb_zcopy(state->skbs[1])); + state->skbs[2] = skb_clone(state->skbs[1], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[2]); + state->skbs[3] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[3]); + skb_split(state->skbs[0], state->skbs[3], SZ_2K); + state->skbs[4] = skb_copy(state->skbs[3], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[4]); + put_held(f, 1); + put_held(f, 2); + attempt = mm_subpage_alloc_at(f->pool, SZ_4K); + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_EXPECT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + for (i = 0; i < 2; i++) { + kfree_skb(state->skbs[i]); + state->skbs[i] = NULL; + } + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[2], 0, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x52, SZ_4K), NULL); + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[3], SZ_2K, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x53, SZ_4K), NULL); + for (i = 2; i < 4; i++) { + kfree_skb(state->skbs[i]); + state->skbs[i] = NULL; + } + /* A full data copy must not keep ownership of the original quarters. */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_4K)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 2, 2 * SZ_4K)); + memset(slot_address(f->held[1]), 0xe1, SZ_4K); + memset(slot_address(f->held[2]), 0xe2, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[4], SZ_2K, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x53, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} +#ifdef CONFIG_INET +static void test_data_skb_prune_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct sk_buff *skb; + struct bio_vec bvec; + struct iov_iter iter; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + KUNIT_ASSERT_EQ(test, sock_create_kern(&init_net, AF_INET, SOCK_STREAM, 0, &state->socket), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + skb = state->skbs[0]; + /* A live socket association prevents trim from condensing data into the head. */ + skb->sk = state->socket->sk; + skb->ip_summed = CHECKSUM_UNNECESSARY; + bvec_set_page(&bvec, &state->folio->page, 16, size - SZ_4K + 16); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, 16); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, 16), (ssize_t)16); + for (i = 0; i < 1024; i++) { + bvec_set_page(&bvec, &state->folio->page, size, (1 + i % 3) * size); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, size), (ssize_t)size); + /* Four fixture refs, two frag refs, two current quarter refs. */ + KUNIT_ASSERT_EQ(test, folio_ref_count(state->folio), 8); + KUNIT_ASSERT_EQ(test, pskb_trim(skb, 16), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 7); + } + skb->sk = NULL; + kfree_skb(skb); + state->skbs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} + +static void user4k_data_skb_prune_test(struct kunit *test) +{ + test_data_skb_prune_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_skb_prune_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_skb_prune_granule(test, 14); +} + +#endif + +static void user4k_data_skb_checksum_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct bio_vec bvec[2]; + struct iov_iter iter; + unsigned char *bytes; + unsigned int i, owned; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + bytes = folio_address(state->folio); + for (i = 0; i < PAGE_SIZE; i++) + bytes[i] = i * 13 + (i >> 12) * 31; + for (owned = 0; owned < 2; owned++) { + struct sk_buff *skb; + + state->folio->mapping = owned ? (void *)FOLIO_MAPPING_ANON : NULL; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + skb = state->skbs[0]; + skb->ip_summed = CHECKSUM_NONE; + bvec_set_page(&bvec[0], &state->folio->page, SZ_4K - 13, 13); + bvec_set_page(&bvec[1], &state->folio->page, SZ_4K, SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, bvec, 2, SZ_8K - 13); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, SZ_8K - 13), + (ssize_t)(SZ_8K - 13)); + KUNIT_EXPECT_EQ(test, (__force u16)csum_fold(skb->csum), + (__force u16)csum_fold(skb_checksum(skb, 0, skb->len, 0))); + kfree_skb(skb); + state->skbs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + } +} + +#endif + +struct refs_test_state { + struct folio *folio; + struct mm_subpage_refs *refs[3]; +}; + +static void refs_test_cleanup(void *arg) +{ + struct refs_test_state *state = arg; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->refs); i++) + mm_subpage_refs_put(state->refs[i]); + if (state->folio) + state->folio->mapping = NULL; +} + +static bool refs_test_keep(const struct mm_subpage *slot, void *context) +{ + unsigned long mask = *(unsigned long *)context; + + return mask & BIT(mm_subpage_offset(slot) >> MM_SUBPAGE_SHIFT); +} + +static void user4k_data_refset_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct refs_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct page *page; + unsigned long mask; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + state->folio = mm_subpage_folio(f->held[0]); + page = &state->folio->page; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, refs_test_cleanup, state), 0); + /* Ownership fixture only; no anonymous rmap is installed. */ + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, + SZ_4K + 17, SZ_8K - 34, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, SZ_4K, SZ_4K, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + state->refs[1] = mm_subpage_refs_get(state->refs[0]); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[1], page, 3 * SZ_4K, SZ_4K, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_NE(test, state->refs[0], state->refs[1]); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 5); + mask = BIT(2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 3); + state->refs[2] = mm_subpage_refs_get(state->refs[0]); + mask = BIT(1); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[2], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_NE(test, state->refs[0], state->refs[2]); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 4); + KUNIT_EXPECT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, PAGE_SIZE, 1, GFP_KERNEL), -EINVAL); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 4); + mask = 0; + for (i = 1; i < 3; i++) { + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[i], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_EQ(test, state->refs[i], NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + put_held(f, 1); + attempt = mm_subpage_alloc_at(f->pool, SZ_4K); + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + mm_subpage_refs_put(state->refs[0]); + state->refs[0] = NULL; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_4K)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS); + for (i = 0; i < 1024; i++) { + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 2 * (int)DATA_SLOTS); + mask = BIT(i % DATA_SLOTS); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[0], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 1); + } + mm_subpage_refs_put(state->refs[0]); + state->refs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS); +} + +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) +struct skmsg_test_state { + struct folio *folio; + struct socket *socket; + struct sk_msg messages[2]; + struct sk_msg borrowed; +}; + +static void skmsg_test_cleanup(void *arg) +{ + struct skmsg_test_state *state = arg; + unsigned int i; + + if (state->socket) { + for (i = 0; i < ARRAY_SIZE(state->messages); i++) + sk_msg_free(state->socket->sk, &state->messages[i]); + sk_msg_subpages_release(&state->borrowed); + sock_release(state->socket); + } + if (state->folio) + state->folio->mapping = NULL; +} + +static void test_data_skmsg_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct skmsg_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct sk_msg *src, *dst; + struct page *page; + struct sock *sk; + struct bio_vec bvec; + struct iov_iter iter; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + memset(slot_address(f->held[i]), 0x60 + i, size); + } + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skmsg_test_cleanup, state), 0); + KUNIT_ASSERT_EQ(test, sock_create_kern(&init_net, AF_INET, SOCK_STREAM, 0, &state->socket), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + page = &state->folio->page; + sk = state->socket->sk; + src = &state->messages[0]; + dst = &state->messages[1]; + sk_msg_init(src); + sk_msg_init(dst); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, (size / 2) - 13, size + (size / 2) + 13), 0); + sk_mem_charge(sk, (size / 2) - 13); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 2 * size), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + KUNIT_ASSERT_EQ(test, sk_msg_clone(sk, dst, src, 0, size + (size / 2) - 13), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 11); + sk_msg_free_partial(sk, src, (size / 2) - 13); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 0), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 11); + sk_msg_free(sk, dst); + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, size)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + sk_msg_free_partial(sk, src, size); + for (i = 0; i < 1024; i++) { + unsigned int index = (i + 1) & 3; + + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, index * size), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + sk_msg_free_partial(sk, src, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 7); + } + sk_msg_free(sk, src); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + /* A borrowed descriptor copy has an independent ownership lifetime. */ + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 0), 0); + sk_mem_charge(sk, size); + memcpy(&state->borrowed, src, sizeof(*src)); + sk_msg_subpages_get(&state->borrowed); + sk_msg_free(sk, src); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 5); + put_held(f, 0); + attempt = mm_subpage_alloc_at(f->pool, 0); + if (!IS_ERR(attempt)) + f->held[0] = attempt; + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(sg_virt(&state->borrowed.sg.data[0]), 0x60, size), NULL); + sk_msg_subpages_release(&state->borrowed); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 0, 0)); + /* Full transfer moves the set, and zero-copy iterator trim releases its references. */ + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 3 * size), 0); + sk_mem_charge(sk, size); + sk_msg_xfer_full(dst, src); + KUNIT_EXPECT_EQ(test, src->sg.size, 0U); + sk_msg_free(sk, dst); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + bvec_set_page(&bvec, page, size - 17, 3 * size + 17); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size - 17); + KUNIT_ASSERT_EQ(test, sk_msg_zerocopy_from_iter(sk, &iter, src, size - 17), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 6); + sk_msg_trim(sk, src, 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} + +static void user4k_data_skmsg_test(struct kunit *test) +{ + test_data_skmsg_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_skmsg_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_skmsg_granule(test, 14); +} +#endif + +static void user16k_data_owner_test(struct kunit *test) +{ + struct data_fixture *f, *dest, *small; + struct mm_subpage *slot, *found; + unsigned int count = PAGE_SIZE / SZ_16K; + + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + dest = data_fixture_create_granule(test, 14); + small = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(dest->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(small->pool), 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(small, 0)); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_4K)), -EINVAL); + for (unsigned int i = 0; i < count; i++) { + slot = hold_alloc_at(f, i, i * SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_shift(slot), 14U); + KUNIT_EXPECT_EQ(test, mm_subpage_size(slot), (unsigned int)SZ_16K); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)SZ_16K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_16K), NULL); + memset(slot_address(slot), 0x30 + i, SZ_16K); + for (unsigned int quarter = 0; quarter < 4; quarter++) { + found = mm_subpage_get_from_phys(mm_subpage_phys(slot) + quarter * SZ_4K); + KUNIT_ASSERT_PTR_EQ(test, found, slot); + mm_subpage_put(found); + } + } + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(f->pool)), -EAGAIN); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(dest->pool, small->held[0])), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(small->pool, f->held[0])), -EINVAL); + slot = mm_subpage_copy_at(dest->pool, f->held[count - 1], SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + dest->held[0] = slot; + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0x30 + count - 1, SZ_16K), NULL); + /* An interior-byte lookup holds the whole identity across release/reuse. */ + f->held[count] = mm_subpage_get_from_phys(mm_subpage_phys(f->held[1]) + 3 * SZ_4K); + KUNIT_ASSERT_PTR_EQ(test, f->held[count], f->held[1]); + put_held(f, 1); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_16K)), -EAGAIN); + put_held(f, count); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + for (unsigned int i = 0; i < count; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x30 + i, SZ_16K), NULL); + mm_subpage_pool_close(f->pool); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(f->pool)), -ESHUTDOWN); + for (unsigned int i = 0; i < count; i++) + put_held(f, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), 0UL); +} + +static void user16k_data_refset_test(struct kunit *test) +{ + struct data_fixture *f; + struct refs_test_state *state; + unsigned int count = PAGE_SIZE / SZ_16K; + unsigned long mask; + struct page *page; + + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (unsigned int i = 0; i < count; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_16K)); + state->folio = mm_subpage_folio(f->held[0]); + page = &state->folio->page; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, refs_test_cleanup, state), 0); + /* Metadata-only ownership fixture: no VMA or rmap is published. */ + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, + SZ_16K - 1024, 2048, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count + 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2); + state->refs[1] = mm_subpage_refs_get(state->refs[0]); + mask = BIT(SZ_16K >> MM_SUBPAGE_SHIFT); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2 + 1); + mask = 0; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[0], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count + 1); + put_held(f, 1); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_16K)), -EAGAIN); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count); +} + +static void user16k_data_mte_test(struct kunit *test) +{ +#ifdef CONFIG_ARM64_MTE + struct data_fixture *f; + struct folio *folio; + struct mm_subpage *copy; + unsigned int count = PAGE_SIZE / SZ_16K; + int ret; + + if (PAGE_SIZE <= SZ_16K || !system_supports_mte()) { + kunit_skip(test, "requires MTE and native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + KUNIT_ASSERT_NOT_NULL(test, f); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + if (try_page_mte_tagging(&folio->page)) + mte_clear_page_tags(folio_address(folio)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(folio_address(folio) + i * SZ_4K, 9); + set_page_mte_tagged(&folio->page); + ret = mm_subpage_pool_add_folio(f->pool, folio, GFP_KERNEL); + if (ret) + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + for (unsigned int i = 0; i < count; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_16K)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, 0)); + memset(address, 0x50 + i, SZ_4K); + test_slot_tags_set(address, 1 + i % 15); + } + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + bool cleared = i >= 4 && i < 8; + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, cleared ? 0 : 1 + i % 15)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(address, cleared ? 0 : 0x50 + i, SZ_4K), NULL); + } + put_held(f, 1); + copy = mm_subpage_copy_at(f->pool, f->held[count - 1], SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + f->held[1] = copy; + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int source = i >= 4 && i < 8 ? i + DATA_SLOTS - 8 : i; + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, 1 + source % 15)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(address, 0x50 + source, SZ_4K), NULL); + } +#else + kunit_skip(test, "MTE is not configured"); +#endif +} + +static void expect_file_coverage(struct kunit *test, struct folio *folio, + unsigned int leaves, unsigned int first, unsigned int tail, + unsigned int final) +{ + struct mm_subpage_mapcounts counts; + unsigned int i; + + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(&folio->page, &counts), leaves); + for (i = 0; i < DATA_SLOTS; i++) { + unsigned int expected = i == 0 ? first : + i == DATA_SLOTS - 1 ? final : i >= DATA_SLOTS - 4 ? tail : 0; + + KUNIT_EXPECT_EQ_MSG(test, counts.slots[i], expected, "coverage slot %u", i); + } +} + +static void user16k_file_coverage_test(struct kunit *test) +{ + struct folio *folio; + phys_addr_t base, tail, last; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative file coverage requires larger native backing"); + return; + } + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, folio), 0); + base = PFN_PHYS(folio_pfn(folio)); + tail = base + PAGE_SIZE - SZ_16K; + last = base + PAGE_SIZE - SZ_4K; + expect_file_coverage(test, folio, 0, 0, 0, 0); + __mm_subpage_file_map_add(tail, SZ_16K); + expect_file_coverage(test, folio, 1, 0, 1, 1); + __mm_subpage_file_map_add(last, SZ_4K); + expect_file_coverage(test, folio, 2, 0, 1, 2); + __mm_subpage_file_map_add(tail, SZ_16K); + expect_file_coverage(test, folio, 3, 0, 2, 3); + __mm_subpage_file_map_add(base, SZ_4K); + expect_file_coverage(test, folio, 4, 1, 2, 3); + __mm_subpage_file_map_del(tail, SZ_16K); + expect_file_coverage(test, folio, 3, 1, 1, 2); + __mm_subpage_file_map_del(last, SZ_4K); + expect_file_coverage(test, folio, 2, 1, 1, 1); + __mm_subpage_file_map_del(tail, SZ_16K); + expect_file_coverage(test, folio, 1, 1, 0, 0); + __mm_subpage_file_map_del(base, SZ_4K); + expect_file_coverage(test, folio, 0, 0, 0, 0); +} + +static struct kunit_case user4k_data_cases[] = { + KUNIT_CASE(user4k_data_reuse_test), + KUNIT_CASE(user4k_data_refset_test), +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) + KUNIT_CASE(user4k_data_skmsg_test), + KUNIT_CASE(user16k_data_skmsg_test), +#endif +#ifdef CONFIG_NET + KUNIT_CASE(user4k_data_skb_test), + KUNIT_CASE(user4k_data_skb_checksum_test), + +#ifdef CONFIG_INET + KUNIT_CASE(user4k_data_skb_prune_test), + KUNIT_CASE(user16k_data_skb_prune_test), +#endif +#endif + KUNIT_CASE(user4k_data_mte_test), + KUNIT_CASE(user4k_data_copy_extent_test), + KUNIT_CASE(user16k_data_owner_test), + KUNIT_CASE(user16k_file_coverage_test), + KUNIT_CASE(user16k_data_refset_test), + KUNIT_CASE(user16k_data_mte_test), + KUNIT_CASE(user4k_data_swapcache_reuse_test), + KUNIT_CASE(user4k_data_restore_test), + KUNIT_CASE(user16k_data_restore_test), + KUNIT_CASE(user4k_data_restore_race_test), + KUNIT_CASE(user4k_data_migration_owner_test), + KUNIT_CASE(user16k_data_migration_owner_test), + KUNIT_CASE(user4k_data_copy_test), + KUNIT_CASE(user4k_data_lookup_test), + KUNIT_CASE(user4k_data_exact_slot_test), + KUNIT_CASE(user4k_data_close_test), + KUNIT_CASE(user4k_data_deferred_test), + KUNIT_CASE(user4k_data_backing_reject_test), + KUNIT_CASE(user4k_data_concurrent_test), + KUNIT_CASE(user4k_data_lookup_race_test), + {} +}; + +static void user4k_data_suite_exit(struct kunit_suite *suite) +{ + /* Drain batched metadata frees before the disposable guest powers off. */ + kvfree_rcu_barrier(); +} + +static struct kunit_suite user4k_data_suite = { + .name = "arm64-user4k-data", + .test_cases = user4k_data_cases, + .suite_exit = user4k_data_suite_exit, +}; + +kunit_test_suite(user4k_data_suite); +MODULE_LICENSE("GPL"); diff --git a/arch/arm64/mm/user4k-pgalloc.c b/arch/arm64/mm/user4k-pgalloc.c new file mode 100644 index 0000000000000..0bcaab5c3a8d2 --- /dev/null +++ b/arch/arm64/mm/user4k-pgalloc.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Hardware tables have immutable granule/owner metadata. Smaller tables share + * a native page; larger tables own a compound allocation. Only unissued slots + * in a shared native page are cached by the mm. + */ +#include +#include +#include +#include +#include + +unsigned int arm64_ptep_page_shift(pte_t *ptep) +{ + struct ptdesc *pt = virt_to_ptdesc(ptep); + + /* Table owners retain geometry through their last RCU reader. */ + return test_bit(PT_fragmented, &pt->pt_flags.f) ? pt->pt_frag_shift : PAGE_SHIFT; +} +EXPORT_SYMBOL_GPL(arm64_ptep_page_shift); + +static void fragment_owner_free(struct ptdesc *pt) +{ + clear_bit(PT_fragmented, &pt->pt_flags.f); + pagetable_dtor_free(pt); +} + +static void fragment_owner_free_rcu(struct rcu_head *rcu) +{ + fragment_owner_free(container_of(rcu, struct ptdesc, pt_rcu_head)); +} + +static void fragment_put(struct ptdesc *pt, unsigned int nr) +{ + if (!atomic_sub_and_test(nr, &pt->pt_frag_refcount)) + return; + if (test_and_clear_bit(PT_frag_defer, &pt->pt_flags.f)) + call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); + else + fragment_owner_free(pt); +} + +void *arm64_user_pt_alloc_granule(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift) +{ + struct ptdesc *pt; + void *table, *next; + bool constructed; + unsigned int frag_size, nr_fragments, order; + + if (shift != 12 && shift != 14 && shift != 16) + return NULL; + frag_size = 1U << shift; + order = shift > PAGE_SHIFT ? shift - PAGE_SHIFT : 0; + nr_fragments = (PAGE_SIZE << order) >> shift; + if (WARN_ON_ONCE((unsigned int)level >= ARRAY_SIZE(mm->context.user4k_pt_frag))) + return NULL; + + spin_lock(&mm->page_table_lock); + table = mm->context.user4k_pt_frag[level]; + if (table) { + /* An mm's geometry is immutable; never consume another granule. */ + if (virt_to_ptdesc(table)->pt_frag_shift != shift) { + spin_unlock(&mm->page_table_lock); + return NULL; + } + next = table + frag_size; + mm->context.user4k_pt_frag[level] = + ((unsigned long)next & ~PAGE_MASK) ? next : NULL; + } + spin_unlock(&mm->page_table_lock); + if (table) + return table; + + pt = pagetable_alloc(GFP_PGTABLE_USER, order); + if (!pt) + return NULL; + switch (level) { + case USER4K_PT_PTE: + constructed = pagetable_pte_ctor(mm, pt); + break; + case USER4K_PT_PMD: + constructed = pagetable_pmd_ctor(mm, pt); + break; + case USER4K_PT_PUD: + pagetable_pud_ctor(pt); + constructed = true; + break; + default: + constructed = false; + } + if (!constructed) { + pagetable_free(pt); + return NULL; + } + pt->pt_frag_shift = shift; + set_bit(PT_fragmented, &pt->pt_flags.f); + atomic_set(&pt->pt_frag_refcount, 1); + table = ptdesc_address(pt); + spin_lock(&mm->page_table_lock); + if (nr_fragments > 1 && !mm->context.user4k_pt_frag[level]) { + atomic_set(&pt->pt_frag_refcount, nr_fragments); + mm->context.user4k_pt_frag[level] = table + frag_size; + } + /* A racing allocation may own the cache; this page then has one user. */ + spin_unlock(&mm->page_table_lock); + return table; +} + +void *arm64_user4k_pt_alloc(struct mm_struct *mm, enum arm64_user4k_pt_level level) +{ + return arm64_user_pt_alloc_granule(mm, level, 12); +} + +void arm64_user4k_pt_free(void *table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + VM_BUG_ON(!test_bit(PT_fragmented, &pt->pt_flags.f)); + VM_BUG_ON(!IS_ALIGNED((unsigned long)table, 1UL << pt->pt_frag_shift)); + VM_BUG_ON(atomic_read(&pt->pt_frag_refcount) <= 0); + fragment_put(pt, 1); +} + +void arm64_pgtable_free(void *table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + if (test_bit(PT_fragmented, &pt->pt_flags.f)) + arm64_user4k_pt_free(table); + else { + VM_BUG_ON(!PAGE_ALIGNED(table)); + pagetable_dtor_free(pt); + } +} + +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm) +{ + unsigned int level; + + /* The mm is private or dying; no allocator may still access its cache. */ + for (level = 0; level < ARRAY_SIZE(mm->context.user4k_pt_frag); level++) { + void *next = mm->context.user4k_pt_frag[level]; + struct ptdesc *pt; + unsigned int issued, shift; + + if (!next) + continue; + mm->context.user4k_pt_frag[level] = NULL; + pt = virt_to_ptdesc(next); + shift = pt->pt_frag_shift; + issued = ((unsigned long)next & ~PAGE_MASK) >> shift; + fragment_put(pt, (PAGE_SIZE >> shift) - issued); + } +} + +void arm64_pte_free_defer(struct mm_struct *mm, pgtable_t table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + if (test_bit(PT_fragmented, &pt->pt_flags.f)) { + /* No freed slot is reused. Delay the final owner release for RCU. */ + set_bit(PT_frag_defer, &pt->pt_flags.f); + arm64_user4k_pt_free(table); + } else { + call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); + } +} + +void __tlb_remove_table(void *token) +{ + if ((unsigned long)token & 1) + arm64_user4k_pt_free((void *)((unsigned long)token & ~1UL)); + else + pagetable_dtor_free(token); +} diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c new file mode 100644 index 0000000000000..761531ba876df --- /dev/null +++ b/arch/arm64/mm/user4k-test.c @@ -0,0 +1,14057 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Architectural prerequisite for mixed-granule userspace. + * + * Packed architectural tables and isolated Linux mms are separate fixtures. + * PAGE_SIZE, struct page, all ordinary mms and TTBR1 retain native geometry. Only bounded + * kernel workers may borrow the isolated Linux mms. Never pass + * packed architectural tables to generic MM walkers or expose them to a task. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../../drivers/iommu/iommufd/iommufd_private.h" +#include "../../../drivers/iommu/iommufd/io_pagetable.h" +#include "../../../mm/internal.h" +#include "../../../mm/vma.h" +#include "../../../mm/swap.h" +#ifdef CONFIG_FUTEX +#include "../../../kernel/futex/futex.h" +#endif + +#define USER4K_SHIFT 12 +#define TEST_SLOTS MM_SUBPAGES_PER_PAGE +#define USER4K_SIZE BIT(USER4K_SHIFT) +#define TEST_VA (4UL * SZ_1M) +#define TEST_LEVELS 4 +#define PROBE_SLOTS 4 +#define TEST_ROUNDS 8 +#define TEST_PROT (PTE_TYPE_PAGE | PTE_USER | PTE_AF | PTE_NG | \ + PTE_SHARED | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL)) + +static bool user4k_test_enabled; + +static int __init user4k_test_setup(char *str) +{ + bool enabled; + int ret = kstrtobool(str, &enabled); + + if (ret) + return ret; + /* Software MM tests are sanitizable; raw/borrowed TTBR0 tests stay gated. */ + if (enabled && IS_ENABLED(CONFIG_KASAN)) + return -EINVAL; + user4k_test_enabled = enabled; + return 0; +} +early_param("arm64.user4k_test", user4k_test_setup); + +struct test_tables { + u64 *level[TEST_LEVELS]; + unsigned int shift; +}; + +struct test_result { + u64 native[PROBE_SLOTS]; + u64 read[PROBE_SLOTS]; + u64 write[PROBE_SLOTS]; + u64 replaced[PROBE_SLOTS]; + u64 replaced_read; + u64 kernel; + u64 tcr; + u64 ttbr1; + u64 restored_tcr; + u64 restored_ttbr0; +}; + +static void test_free_page(void *ptr) +{ + free_page((unsigned long)ptr); +} + +static void *test_alloc_page(struct kunit *test) +{ + void *ptr = (void *)get_zeroed_page(GFP_KERNEL); + + if (!ptr) + return NULL; + if (kunit_add_action_or_reset(test, test_free_page, ptr)) + return NULL; + return ptr; +} + +static void test_put_compound(void *ptr) +{ + put_page(ptr); +} + +static void *test_alloc_granule(struct kunit *test, unsigned int shift) +{ + struct page *page; + + if (shift <= PAGE_SHIFT) + return test_alloc_page(test); + page = alloc_pages(GFP_KERNEL | __GFP_ZERO | __GFP_COMP, shift - PAGE_SHIFT); + if (!page) + return NULL; + if (kunit_add_action_or_reset(test, test_put_compound, page)) + return NULL; + return page_address(page); +} + +static unsigned int test_index(unsigned long va, unsigned int shift, + unsigned int level) +{ + unsigned int bits = shift - PTDESC_ORDER; + + return (va >> ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, shift)) & (BIT(bits) - 1); +} + +static int test_alloc_tables(struct kunit *test, struct test_tables *tables, + unsigned int shift) +{ + int level; + void *packed = NULL; + + tables->shift = shift; + if (shift == USER4K_SHIFT) { + /* Also test table address bits 13:12, not just leaf addresses. */ + packed = test_alloc_page(test); + if (!packed) + return -ENOMEM; + } + for (level = 0; level < TEST_LEVELS; level++) { + tables->level[level] = packed ? packed + level * USER4K_SIZE : + test_alloc_page(test); + if (!tables->level[level]) + return -ENOMEM; + } + for (level = 0; level < TEST_LEVELS - 1; level++) { + unsigned int index = test_index(TEST_VA, shift, level); + + tables->level[level][index] = + virt_to_phys(tables->level[level + 1]) | PMD_TYPE_TABLE; + } + return 0; +} + +static void user4k_geometry_test(struct kunit *test) +{ + static const unsigned int shifts4k[] = { 39, 30, 21, 12 }; + static const unsigned int shifts16k[] = { 47, 36, 25, 14 }; + static const unsigned int shifts64k[] = { 55, 42, 29, 16 }; + unsigned int level; + + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 12), 4); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 14), 4); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 16), 3); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(42, 16), 2); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(39, 12), 3); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(47, 14), 3); + for (level = 0; level < TEST_LEVELS; level++) { + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, 12), + shifts4k[level]); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, 14), + shifts16k[level]); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT(level), + PAGE_SHIFT == 16 ? shifts64k[level] : shifts16k[level]); + KUNIT_EXPECT_EQ(test, test_index(BIT(shifts4k[level]), 12, level), 1U); + KUNIT_EXPECT_EQ(test, test_index(BIT(shifts4k[level]) - 1, 12, level), 0U); + } +} + +static void test_leaf_encoding(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + const phys_addr_t base = GENMASK_ULL(47, PAGE_SHIFT); + pgprot_t prot = __pgprot(TEST_PROT | PTE_RDONLY); + unsigned int i; + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, mm); + pte = phys_pte_mm(mm, base, prot); + KUNIT_EXPECT_EQ(test, pte_val(pte), pte_val(pfn_pte(base >> PAGE_SHIFT, prot))); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), base); + KUNIT_EXPECT_EQ(test, pgprot_val(pte_pgprot_mm(mm, pte)), pgprot_val(prot)); + mm->context.flags |= shift == 12 ? MMCF_USER_4K : MMCF_USER_16K; + for (i = 0; i < (PAGE_SIZE >> shift); i++) { + phys_addr_t phys = base + i * (1UL << shift); + + pte = phys_pte_mm(mm, phys, prot); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys); + KUNIT_EXPECT_EQ(test, pte_pfn(pte), (unsigned long)(base >> PAGE_SHIFT)); + KUNIT_EXPECT_EQ(test, pgprot_val(pte_pgprot_mm(mm, pte)), pgprot_val(prot)); + if (i + 1 < (PAGE_SIZE >> shift)) + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte_advance_pfn_mm(mm, pte, 1)), + phys + (1UL << shift)); + pte = pte_modify(pte, __pgprot(TEST_PROT)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys); + KUNIT_EXPECT_FALSE(test, pte_val(pte) & PTE_RDONLY); + } +} + +static void user4k_leaf_encoding_test(struct kunit *test) +{ + test_leaf_encoding(test, 12); +} + +static void user16k_leaf_encoding_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative leaves require larger native backing"); + return; + } + test_leaf_encoding(test, 14); +} + + +/* Caller masks all asynchronous exceptions; this function cannot sleep. */ +static void test_switch_ttbr0(u64 ttbr, u64 tcr) +{ + /* The zero root is valid for both tested granules. */ + cpu_set_reserved_ttbr0(); + local_flush_tlb_all(); + write_sysreg(tcr, tcr_el1); + isb(); + write_sysreg(ttbr, ttbr0_el1); + isb(); +} + +static u64 test_translate(unsigned long va, bool write) +{ + if (write) + asm volatile("at s1e0w, %0" : : "r" (va) : "memory"); + else + asm volatile("at s1e0r, %0" : : "r" (va) : "memory"); + isb(); + return read_sysreg_par(); +} + +#define MM_TEST_MAPPINGS 11 + +struct test_mm { + struct kunit *test; + struct mm_struct *mm; + unsigned long address[MM_TEST_MAPPINGS]; + phys_addr_t phys[MM_TEST_MAPPINGS]; +}; + +static pte_t *test_mm_lookup(struct mm_struct *mm, unsigned long addr, + spinlock_t **ptl); + +static void test_mm_hardware(struct test_mm *ctx) +{ + struct kunit *test = ctx->test; + struct vm_area_struct vma = TLB_FLUSH_VMA(ctx->mm, 0); + struct arch_tlbflush_unmap_batch batch = {}; + pte_t *leaves[4]; + u64 invalidated[3][4]; + u64 result[MM_TEST_MAPPINGS], writes[MM_TEST_MAPPINGS]; + u64 old_tcr, old_ttbr0, old_ttbr1, old_par, tcr, kernel; + u64 old_id = atomic64_read(&ctx->mm->context.id); + u64 restored_tcr, restored_ttbr0, retained_ttbr1; + unsigned long flags; + unsigned int i, step; + + for (i = 0; i < ARRAY_SIZE(leaves); i++) { + spinlock_t *ptl; + + leaves[i] = test_mm_lookup(ctx->mm, ctx->address[i], &ptl); + if (leaves[i]) + pte_unmap_unlock(leaves[i], ptl); + KUNIT_ASSERT_NOT_NULL(test, leaves[i]); + } + + preempt_disable(); + flags = local_daif_save(); + old_tcr = read_sysreg(tcr_el1); + old_ttbr0 = read_sysreg(ttbr0_el1); + old_ttbr1 = read_sysreg(ttbr1_el1); + old_par = read_sysreg_par(); + if ((old_tcr & (TCR_EL1_DS | TCR_EL1_TG1_MASK | TCR_EL1_A1)) != + (TCR_TG1_16K | TCR_EL1_A1)) { + local_daif_restore(flags); + preempt_enable(); + kunit_skip(test, "requires 16K TTBR1, DS=0, and A1=1"); + return; + } + tcr = (old_tcr & ~(TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK | TCR_EL1_EPD0)) | + TCR_TG0_4K | TCR_T0SZ(48); + test_switch_ttbr0(virt_to_phys(ctx->mm->pgd), tcr); + /* A1 selects the saved TTBR1 ASID for these private test translations. */ + atomic64_set(&ctx->mm->context.id, old_ttbr1 >> 48); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + result[i] = test_translate(ctx->address[i], false); + writes[i] = test_translate(ctx->address[i], true); + } + for (step = 0; step < ARRAY_SIZE(invalidated); step++) { + unsigned int first = step == 2 ? 3 : 1; + unsigned int last = step == 1 ? 3 : first + 1; + + for (i = first; i < last; i++) + WRITE_ONCE(*leaves[i], __pte(0)); + switch (step) { + case 0: + __flush_tlb_page(&vma, ctx->address[first] + 1, + TLBF_NONOTIFY | TLBF_NOBROADCAST); + break; + case 1: + __flush_tlb_range(&vma, ctx->address[first], ctx->address[last], + SZ_4K, 3, TLBF_NONOTIFY); + break; + case 2: + arch_tlbbatch_add_pending(&batch, ctx->mm, ctx->address[first], + ctx->address[first] + SZ_4K); + arch_tlbbatch_flush(&batch); + break; + } + isb(); + for (i = 0; i < ARRAY_SIZE(leaves); i++) + invalidated[step][i] = test_translate(ctx->address[i], false); + for (i = first; i < last; i++) + WRITE_ONCE(*leaves[i], __pte(ctx->phys[i] | TEST_PROT)); + dsb(ishst); + isb(); + /* Repopulate before the next independent invalidation. */ + for (i = 0; i < ARRAY_SIZE(leaves); i++) + test_translate(ctx->address[i], false); + } + asm volatile("at s1e1r, %0" : : "r" (ctx->mm->pgd) : "memory"); + isb(); + kernel = read_sysreg_par(); + retained_ttbr1 = read_sysreg(ttbr1_el1); + test_switch_ttbr0(old_ttbr0, old_tcr); + restored_tcr = read_sysreg(tcr_el1); + restored_ttbr0 = read_sysreg(ttbr0_el1); + write_sysreg(old_par, par_el1); + isb(); + atomic64_set(&ctx->mm->context.id, old_id); + local_daif_restore(flags); + preempt_enable(); + + KUNIT_EXPECT_EQ(test, restored_tcr, old_tcr); + KUNIT_EXPECT_EQ(test, restored_ttbr0, old_ttbr0); + KUNIT_EXPECT_EQ(test, retained_ttbr1, old_ttbr1); + KUNIT_EXPECT_EQ(test, kernel & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, kernel & SYS_PAR_EL1_PA, + virt_to_phys(ctx->mm->pgd) & SYS_PAR_EL1_PA); + for (step = 0; step < ARRAY_SIZE(invalidated); step++) { + for (i = 0; i < ARRAY_SIZE(leaves); i++) { + bool hole = step == 0 ? i == 1 : step == 1 ? i == 1 || i == 2 : i == 3; + + KUNIT_EXPECT_EQ(test, invalidated[step][i] & SYS_PAR_EL1_F, (u64)hole); + if (!hole) + KUNIT_EXPECT_EQ(test, invalidated[step][i] & SYS_PAR_EL1_PA, + ctx->phys[i]); + } + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + KUNIT_EXPECT_EQ(test, result[i] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, result[i] & SYS_PAR_EL1_PA, ctx->phys[i]); + KUNIT_EXPECT_EQ(test, writes[i] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, writes[i] & SYS_PAR_EL1_PA, ctx->phys[i]); + } +} + +struct test_walk { + struct test_mm *ctx; + unsigned long size; + unsigned long holes; + unsigned int present; +}; + +/* Callbacks run under the PTE lock; record errors without invoking KUnit. */ +static int test_walk_pte(pte_t *pte, unsigned long addr, unsigned long next, + struct mm_walk *walk) +{ + struct test_walk *state = walk->private; + u64 value = pte_val(ptep_get(pte)); + unsigned int i; + + if (next - addr != state->size) + return -EINVAL; + if (!value) { + state->holes += next - addr; + return 0; + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + if (state->ctx->address[i] != addr) + continue; + if (value != (state->ctx->phys[i] | TEST_PROT)) + return -EINVAL; + state->present++; + return 0; + } + return -EINVAL; +} + +static int test_walk_hole(unsigned long addr, unsigned long next, int depth, + struct mm_walk *walk) +{ + struct test_walk *state = walk->private; + + state->holes += next - addr; + return 0; +} + +static const struct mm_walk_ops test_walk_ops = { + .pte_entry = test_walk_pte, + .pte_hole = test_walk_hole, +}; + +static pte_t *test_mm_lookup(struct mm_struct *mm, unsigned long addr, + spinlock_t **ptl) +{ + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + if (pgd_none(*pgd) || pgd_bad(*pgd)) + return NULL; + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_mm(mm, *p4d) || p4d_bad_mm(mm, *p4d)) + return NULL; + pud = pud_offset_mm(mm, p4d, addr); + if (pud_none(*pud) || pud_bad(*pud)) + return NULL; + pmd = pmd_offset_mm(mm, pud, addr); + if (pmd_none(*pmd) || pmd_bad(*pmd)) + return NULL; + return pte_offset_map_lock(mm, pmd, addr, ptl); +} + +static void test_mm_destroy(void *ptr) +{ + struct test_mm *ctx = ptr; + struct mm_struct *mm = ctx->mm; + struct mmu_gather tlb; + unsigned int i; + + if (!mm) + return; + mmap_write_lock(mm); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + + if (pte) { + /* Test-owned physical pages, not faulted/rmap-owned pages. */ + pte_clear(mm, ctx->address[i], pte); + pte_unmap_unlock(pte, ptl); + } + } + /* Exercise the real generic gather/teardown path, not a test walker. */ + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(ctx->test, mm_pgtables_bytes(mm), 0UL); + mmput(mm); + ctx->mm = NULL; +} + +static unsigned int test_mm_backing_pages(struct test_mm *ctx) +{ + struct page *owners[3 * MM_TEST_MAPPINGS]; + unsigned int count = 0, total = 0, i, j, level; + struct mm_struct *mm = ctx->mm; + + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + unsigned long addr = ctx->address[i]; + p4d_t *p4d = p4d_offset_mm(mm, pgd_offset(mm, addr), addr); + pud_t *pud = pud_offset_mm(mm, p4d, addr); + pmd_t *pmd = pmd_offset_mm(mm, pud, addr); + pte_t *pte = pte_offset_kernel_mm(mm, pmd, addr); + struct page *pages[] = { virt_to_page(pud), virt_to_page(pmd), virt_to_page(pte) }; + + for (level = mm_pud_folded(mm) ? 1 : 0; level < ARRAY_SIZE(pages); level++) { + pages[level] = compound_head(pages[level]); + for (j = 0; j < count; j++) + if (owners[j] == pages[level]) + break; + if (j == count) { + owners[count++] = pages[level]; + total += 1U << compound_order(pages[level]); + } + } + } + return total; +} + +/* Only valid for a private, empty mm with no ASID or hardware references. */ +static int test_mm_select_granule(struct mm_struct *mm, unsigned int shift) +{ + pgd_t *native_pgd = mm->pgd, *new_pgd; + unsigned long flag = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + + mm->context.flags |= flag; + new_pgd = pgd_alloc(mm); + mm->context.flags &= ~flag; + if (!new_pgd) + return -ENOMEM; + pgd_free(mm, native_pgd); + mm->pgd = new_pgd; + mm->context.flags |= flag; + return 0; +} + +static int test_mm_select4k(struct mm_struct *mm) +{ + return test_mm_select_granule(mm, 12); +} + +static void test_mm_tables(struct kunit *test, unsigned int shift, bool hardware) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + bool small = shift != PAGE_SHIFT; + unsigned int pmd_shift = small ? shift + (shift - 3) : PMD_SHIFT; + unsigned int pud_shift = small ? shift + 2 * (shift - 3) : PUD_SHIFT; + unsigned int pgd_shift = small ? (shift == 16 ? pud_shift : + shift + 3 * (shift - 3)) : PGDIR_SHIFT; + unsigned long size = 1UL << shift; + struct mm_struct *mm; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + for (i = 0; i < 4; i++) + ctx->address[i] = TEST_VA + i * size; + ctx->address[4] = (1UL << pmd_shift) - size; + ctx->address[5] = 1UL << pmd_shift; + ctx->address[6] = (1UL << pud_shift) - size; + ctx->address[7] = 1UL << pud_shift; + ctx->address[8] = (1UL << pgd_shift) - size; + ctx->address[9] = 1UL << pgd_shift; + ctx->address[10] = (1UL << 48) - size; + if (pud_shift == pgd_shift) { + ctx->address[6] -= size; + ctx->address[7] += size; + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + void *data = test_alloc_granule(test, shift); + + KUNIT_ASSERT_NOT_NULL(test, data); + ctx->phys[i] = virt_to_phys(data) + (shift < PAGE_SHIFT ? (i % (PAGE_SIZE >> shift)) * size : 0); + } + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + ctx->mm = mm; + /* Register after the data allocations: detach tables before freeing data. */ + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_EXPECT_EQ(test, mm_page_shift(mm), (unsigned int)PAGE_SHIFT); + if (small) { + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shift), 0); + KUNIT_EXPECT_EQ(test, (unsigned long)mm->pgd & (size - 1), 0UL); + } + KUNIT_EXPECT_EQ(test, mm_page_shift(mm), shift); + if (small) + KUNIT_EXPECT_EQ(test, mm_pud_folded(mm), shift == 16); + KUNIT_EXPECT_EQ(test, mm_page_size(mm), size); + KUNIT_EXPECT_EQ(test, mm_page_mask(mm), ~(size - 1)); + KUNIT_EXPECT_EQ(test, mm_page_shift(&init_mm), (unsigned int)PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, pmd_addr_end_mm(mm, (1UL << pmd_shift) - 1, + (1UL << pmd_shift) + size), + 1UL << pmd_shift); + KUNIT_EXPECT_EQ(test, pgd_addr_end_mm(mm, ULONG_MAX - size, 0), 0UL); + + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + unsigned long addr = ctx->address[i]; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d = p4d_alloc(mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + KUNIT_ASSERT_NOT_NULL(test, p4d); + pud = pud_alloc(mm, p4d, addr); + KUNIT_ASSERT_NOT_NULL(test, pud); + pmd = pmd_alloc(mm, pud, addr); + KUNIT_ASSERT_NOT_NULL(test, pmd); + pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + /* Do not assert while holding the spinlock/RCU section. */ + set_pte(pte, __pte(ctx->phys[i] | TEST_PROT)); + pte_unmap_unlock(pte, ptl); + } + /* Installed table bytes and physical backing (including cached slots) differ. */ + { + unsigned int shifts[] = { pmd_shift, pud_shift, pgd_shift }; + unsigned int tables = 0, owners = 0; + + for (unsigned int level = 0; level < (mm_pud_folded(mm) ? 2 : 3); level++) { + unsigned int unique = 0; + + for (unsigned int a = 0; a < MM_TEST_MAPPINGS; a++) { + unsigned int b; + + for (b = 0; b < a; b++) + if ((ctx->address[a] >> shifts[level]) == + (ctx->address[b] >> shifts[level])) + break; + unique += b == a; + } + tables += unique; + if (shift < PAGE_SHIFT) + owners += DIV_ROUND_UP(unique, PAGE_SIZE >> shift); + else + owners += unique << (shift - PAGE_SHIFT); + } + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), tables * size); + KUNIT_EXPECT_EQ(test, test_mm_backing_pages(ctx), owners); + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + u64 value; + + KUNIT_ASSERT_NOT_NULL(test, pte); + value = pte_val(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, value, ctx->phys[i] | TEST_PROT); + } + { + struct vm_area_struct *vma = vm_area_alloc(mm); + struct test_walk state = { .ctx = ctx, .size = size }; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_start = 0; + vma->vm_end = 1UL << 48; + mmap_write_lock(mm); + ret = walk_page_range_vma(vma, 0, 1UL << 48, &test_walk_ops, &state); + mmap_write_unlock(mm); + vm_area_free(vma); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, state.present, MM_TEST_MAPPINGS); + KUNIT_EXPECT_EQ(test, state.holes, (1UL << 48) - MM_TEST_MAPPINGS * size); + } + if (hardware) + test_mm_hardware(ctx); + { + struct mmu_gather tlb; + unsigned long before = mm_pgtables_bytes(mm); + unsigned long boundary = 1UL << pmd_shift; + + mmap_write_lock(mm); + /* An adjacent VMA's unaligned ceiling must keep the shared table. */ + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, boundary - size, 0, boundary - size); + tlb_finish_mmu(&tlb); + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), before); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte; + + if (ctx->address[i] >= boundary) + continue; + pte = test_mm_lookup(mm, ctx->address[i], &ptl); + if (pte) { + pte_clear(mm, ctx->address[i], pte); + pte_unmap_unlock(pte, ptl); + } + } + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, boundary, 0, boundary); + tlb_finish_mmu(&tlb); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), before - size); + /* The next table and the higher levels must still be walkable. */ + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + u64 value; + + if (ctx->address[i] < boundary) { + if (pte) + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_PTR_EQ(test, pte, NULL); + continue; + } + KUNIT_ASSERT_NOT_NULL(test, pte); + value = pte_val(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, value, ctx->phys[i] | TEST_PROT); + } + } + test_mm_destroy(ctx); +} + +static void user4k_mm_tables_test(struct kunit *test) +{ + test_mm_tables(test, 12, false); +} + +static void native_mm_tables_test(struct kunit *test) +{ + test_mm_tables(test, PAGE_SHIFT, false); +} + +static void user16k_mm_tables_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative tables require larger native backing"); + return; + } + test_mm_tables(test, 14, false); +} + +static void user64k_mm_tables_test(struct kunit *test) +{ + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "64K is already the native geometry"); + return; + } + test_mm_tables(test, 16, false); +} + +static void test_free_vma(void *ptr) +{ + vm_area_free(ptr); +} + +static void user4k_anon_read_fault_test(struct kunit *test) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + struct vm_area_struct *vma; + vm_flags_t vm_flags = VM_READ | VM_MAYREAD; + unsigned int i, j; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + ctx->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, ctx->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_ASSERT_EQ(test, test_mm_select4k(ctx->mm), 0); + for (i = 0; i < 4; i++) + ctx->address[i] = TEST_VA + i * SZ_4K; + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + PAGE_SIZE; + vma_set_anonymous(vma); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + vm_flags |= VM_NOHUGEPAGE; +#endif + vm_flags_init(vma, vm_flags); + vma->vm_page_prot = vm_get_page_prot(vm_flags); + /* This VMA is a private fault fixture, not inserted into the VMA tree. */ + for (i = 0; i < 4; i++) { + vm_fault_t ret; + + mmap_read_lock(ctx->mm); + ret = handle_mm_fault(vma, ctx->address[i] + 1, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(ctx->mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + for (j = 0; j < 4; j++) { + spinlock_t *ptl; + pte_t *ptep = test_mm_lookup(ctx->mm, ctx->address[j], &ptl); + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, ptep); + pte = ptep_get(ptep); + pte_unmap_unlock(ptep, ptl); + if (j > i) { + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + continue; + } + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_special(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_TRUE(test, is_zero_pfn(pte_pfn(pte))); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(ctx->mm, MM_ANONPAGES), 0L); +} + +static void user4k_mm_hardware_test(struct kunit *test) +{ + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE)) { + kunit_skip(test, "requires 4K, no CnP, and direct PTE permissions"); + return; + } + test_mm_tables(test, 12, true); +} + +struct fragment_fixture { + struct mm_struct *mm; + void *table[TEST_SLOTS + 2]; + struct page *pinned[3]; +}; + +static void fragment_fixture_destroy(void *data) +{ + struct fragment_fixture *f = data; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(f->table); i++) + if (f->table[i]) + arm64_pgtable_free(f->table[i]); + if (f->mm) + mmput(f->mm); + rcu_barrier(); + for (i = 0; i < ARRAY_SIZE(f->pinned); i++) + if (f->pinned[i]) + put_page(f->pinned[i]); +} + +static void test_fragment_lifetime(struct kunit *test, unsigned int shift) +{ + struct fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct ptdesc *pte_owner; + unsigned int i, slots = PAGE_SIZE >> shift; + unsigned long size = 1UL << shift; + unsigned int read_shift = 0; + bool live; + + KUNIT_ASSERT_NOT_NULL(test, f); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, fragment_fixture_destroy, f), 0); + /* Direct allocator fixture: the mm's native root is never changed or used. */ + for (i = 0; i < slots + 2; i++) { + enum arm64_user4k_pt_level level = i < slots ? USER4K_PT_PTE : i - slots + 1; + + f->table[i] = arm64_user_pt_alloc_granule(f->mm, level, shift); + KUNIT_ASSERT_NOT_NULL(test, f->table[i]); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)f->table[i], size)); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(f->table[i]), shift); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(f->table[i] + size - sizeof(pte_t)), shift); + KUNIT_EXPECT_EQ(test, pte_batch_hint(f->table[i], __pte(TEST_PROT | PTE_CONT)), + IS_ENABLED(CONFIG_ARM64_CONTPTE) ? arm64_cont_ptes(shift) : 1U); + if (!i && PAGE_SHIFT == 16) { + void *cached = f->mm->context.user4k_pt_frag[level]; + + KUNIT_EXPECT_PTR_EQ(test, arm64_user_pt_alloc_granule(f->mm, level, + shift == 12 ? 14 : 12), NULL); + KUNIT_EXPECT_PTR_EQ(test, f->mm->context.user4k_pt_frag[level], cached); + } + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0, size), NULL); + if (i == 0 || i >= slots) { + unsigned int slot = i == 0 ? 0 : i - slots + 1; + + f->pinned[slot] = virt_to_page(f->table[i]); + get_page(f->pinned[slot]); + } else { + KUNIT_EXPECT_PTR_EQ(test, virt_to_page(f->table[i]), f->pinned[0]); + KUNIT_EXPECT_EQ(test, (unsigned long)f->table[i] - + (unsigned long)f->table[0], i * (unsigned long)size); + } + memset(f->table[i], 0x50 + i, size); + } + for (i = 0; i < slots + 2; i++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0x50 + i, size), NULL); + memset(f->table[i], 0, size); + } + /* Pointer-only walkers must inspect all 128 entries of a 16K group. */ + if (IS_ENABLED(CONFIG_ARM64_CONTPTE)) { + pte_t *ptes = f->table[0]; + unsigned int nr = arm64_cont_ptes(shift), j; + pte_t observed; + + for (j = 0; j < nr; j++) + ptes[j] = pte_mkold(pte_mkclean(__pte(TEST_PROT | PTE_CONT | + (SZ_4M + (j << shift))))); + ptes[nr - 1] = pte_mkyoung(pte_mkdirty(ptes[nr - 1])); + observed = ptep_get(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + observed = ptep_get_lockless(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + KUNIT_EXPECT_EQ(test, pte_val(observed) & + (ARM64_USER4K_TABLE_ADDR_MASK & ~(size - 1)), + (u64)(SZ_4M + size)); + memset(ptes, 0, size); + } + KUNIT_EXPECT_PTR_NE(test, f->pinned[0], f->pinned[1]); + KUNIT_EXPECT_PTR_NE(test, f->pinned[1], f->pinned[2]); + pte_owner = page_ptdesc(f->pinned[0]); + KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots); + arm64_pte_free_defer(f->mm, f->table[0]); + f->table[0] = NULL; + KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots - 1); + for (i = 1; i < slots - 1; i++) { + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + } + /* A pre-existing software walker must survive the last fragment release. */ + rcu_read_lock(); + arm64_pgtable_free(f->table[slots - 1]); + f->table[slots - 1] = NULL; + live = folio_test_pgtable(page_folio(f->pinned[0])); + read_shift = arm64_ptep_page_shift(page_address(f->pinned[0])); + rcu_read_unlock(); + KUNIT_EXPECT_TRUE(test, live); + KUNIT_EXPECT_EQ(test, read_shift, shift); + rcu_barrier(); + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[0]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[0]), 1); + for (i = slots; i < slots + 2; i++) { + struct ptdesc *pt = virt_to_ptdesc(f->table[i]); + + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), (int)slots - 1); + KUNIT_EXPECT_TRUE(test, folio_test_pgtable(ptdesc_folio(pt))); + } + mmput(f->mm); + f->mm = NULL; + for (i = 1; i < 3; i++) { + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[i]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[i]), 1); + } +} + +/* Direct table-owner test; no larger-than-native user mm is enabled here. */ +static void user64k_table_owner_test(struct kunit *test) +{ + struct fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + unsigned int i, offset, order = 16 - PAGE_SHIFT; + unsigned int shift; + bool live; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, fragment_fixture_destroy, f), 0); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + for (i = 0; i < 3; i++) { + struct ptdesc *pt; + + f->table[i] = arm64_user_pt_alloc_granule(f->mm, i, 16); + KUNIT_ASSERT_NOT_NULL(test, f->table[i]); + pt = virt_to_ptdesc(f->table[i]); + f->pinned[i] = ptdesc_page(pt); + get_page(f->pinned[i]); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)f->table[i], SZ_64K)); + KUNIT_EXPECT_EQ(test, compound_order(f->pinned[i]), order); + KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), 1); + KUNIT_EXPECT_PTR_EQ(test, f->mm->context.user4k_pt_frag[i], NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0, SZ_64K), NULL); + /* Include the last entry of every constituent native page. */ + for (offset = PAGE_SIZE - sizeof(pte_t); offset < SZ_64K; offset += PAGE_SIZE) { + void *entry = f->table[i] + offset; + + KUNIT_EXPECT_PTR_EQ(test, virt_to_ptdesc(entry), pt); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(entry), 16U); + KUNIT_EXPECT_EQ(test, pte_batch_hint(entry, __pte(TEST_PROT | PTE_CONT)), + 32U - ((offset / sizeof(pte_t)) % 32)); + if (i == USER4K_PT_PTE) + KUNIT_EXPECT_PTR_EQ(test, ptep_lockptr(f->mm, entry), + ptep_lockptr(f->mm, f->table[i])); + if (i == USER4K_PT_PMD) + KUNIT_EXPECT_PTR_EQ(test, pmd_lockptr(f->mm, entry), + pmd_lockptr(f->mm, f->table[i])); + } + memset(f->table[i], 0x51 + i, SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0x51 + i, SZ_64K), NULL); + memset(f->table[i], 0, SZ_64K); + } + /* Contiguous readers on the last native page must still use 64K geometry. */ + if (IS_ENABLED(CONFIG_ARM64_CONTPTE)) { + pte_t *ptes = f->table[0] + SZ_64K - 32 * sizeof(pte_t); + pte_t observed; + unsigned int j; + + for (j = 0; j < 32; j++) + ptes[j] = pte_mkold(pte_mkclean(__pte(TEST_PROT | PTE_CONT | + (SZ_4M + (j << 16))))); + ptes[31] = pte_mkyoung(pte_mkdirty(ptes[31])); + observed = ptep_get(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + observed = ptep_get_lockless(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + KUNIT_EXPECT_EQ(test, pte_val(observed) & + (ARM64_USER4K_TABLE_ADDR_MASK & ~(SZ_64K - 1UL)), + (u64)(SZ_4M + SZ_64K)); + memset(f->table[0], 0, SZ_64K); + } + rcu_read_lock(); + arm64_pte_free_defer(f->mm, f->table[0]); + live = folio_test_pgtable(page_folio(f->pinned[0])); + shift = arm64_ptep_page_shift(f->table[0] + SZ_64K - sizeof(pte_t)); + f->table[0] = NULL; + rcu_read_unlock(); + KUNIT_EXPECT_TRUE(test, live); + KUNIT_EXPECT_EQ(test, shift, 16U); + rcu_barrier(); + for (i = 0; i < 3; i++) { + if (f->table[i]) { + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + } + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[i]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[i]), 1); + KUNIT_EXPECT_EQ(test, compound_order(f->pinned[i]), order); + } +} + +static void user4k_fragment_lifetime_test(struct kunit *test) +{ + test_fragment_lifetime(test, 12); +} + +static void user16k_fragment_lifetime_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K fragments require larger native backing"); + return; + } + test_fragment_lifetime(test, 14); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static void test_put_mm(void *ptr) +{ + mmput(ptr); +} + +static void native_pgtable_deposit_test(struct kunit *test) +{ + struct mm_struct *mm = mm_alloc(); + pgtable_t first, second, out1, out2; + pmd_t *pmd; + spinlock_t *ptl; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_put_mm, mm), 0); + pmd = pmd_alloc_one(mm, 0); + KUNIT_ASSERT_NOT_NULL(test, pmd); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, pmd), 0); + first = pte_alloc_one(mm); + KUNIT_ASSERT_NOT_NULL(test, first); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, first), 0); + second = pte_alloc_one(mm); + KUNIT_ASSERT_NOT_NULL(test, second); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, second), 0); + ptl = pmd_lock(mm, pmd); + pgtable_trans_huge_deposit(mm, pmd, first); + pgtable_trans_huge_deposit(mm, pmd, second); + out1 = pgtable_trans_huge_withdraw(mm, pmd); + out2 = pgtable_trans_huge_withdraw(mm, pmd); + spin_unlock(ptl); + KUNIT_EXPECT_PTR_EQ(test, out1, second); + KUNIT_EXPECT_PTR_EQ(test, out2, first); +} +#endif + +static void test_data_slot_put(void *ptr) +{ + mm_subpage_put(ptr); +} + +static void test_data_pool_free(void *ptr) +{ + mm_subpage_pool_close(ptr); + mm_subpage_pool_put(ptr); +} + +static bool test_alloc_data_slots(struct kunit *test, struct test_mm *ctx) +{ + struct mm_subpage_pool *pool = mm_subpage_pool_create(GFP_KERNEL); + struct folio *folio; + unsigned int i; + int ret; + + if (!pool) + return false; + if (kunit_add_action_or_reset(test, test_data_pool_free, pool)) + return false; + folio = folio_alloc(GFP_KERNEL, 0); + if (!folio) + return false; + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + return false; + } + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc(pool); + + if (IS_ERR(slot)) + return false; + if (kunit_add_action_or_reset(test, test_data_slot_put, slot)) + return false; + ctx->phys[i] = mm_subpage_phys(slot); + } + return true; +} + +/* These private mappings are borrowed only for bounded, fault-disabled uaccess. */ +static struct test_mm *test_usercopy_mm(struct kunit *test, bool small) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + unsigned long size = small ? SZ_4K : PAGE_SIZE; + unsigned int i; + + if (!ctx) + return NULL; + ctx->test = test; + if (small && !test_alloc_data_slots(test, ctx)) + return NULL; + for (i = 0; i < 4; i++) { + if (!small) { + void *data = test_alloc_page(test); + + if (!data) + return NULL; + ctx->phys[i] = virt_to_phys(data); + } + ctx->address[i] = TEST_VA + i * size; + memset(phys_to_virt(ctx->phys[i]), (small ? 0x40 : 0x80) + i, 64); + } + ctx->mm = mm_alloc(); + if (!ctx->mm) + return NULL; + if (kunit_add_action_or_reset(test, test_mm_destroy, ctx)) + return NULL; + if (small && test_mm_select4k(ctx->mm)) + return NULL; + for (i = 0; i < 4; i++) { + unsigned long addr = ctx->address[i]; + pgd_t *pgd = pgd_offset(ctx->mm, addr); + p4d_t *p4d = p4d_alloc(ctx->mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + if (!p4d) + return NULL; + pud = pud_alloc(ctx->mm, p4d, addr); + if (!pud) + return NULL; + pmd = pmd_alloc(ctx->mm, pud, addr); + if (!pmd) + return NULL; + pte = pte_alloc_map_lock(ctx->mm, pmd, addr, &ptl); + if (!pte) + return NULL; + if (i != 2) + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[i], + __pgprot(TEST_PROT | (i ? 0 : PTE_RDONLY)))); + pte_unmap_unlock(pte, ptl); + } + return ctx; +} + +static void user4k_rmap_walk_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, true); + struct vm_area_struct *vma; + unsigned int i, j, sync; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * PAGE_SIZE; + /* A later alias must not extend a native-owner range walk's bound. */ + ctx->address[4] = TEST_VA + PAGE_SIZE; + ctx->phys[4] = ctx->phys[0]; + pte = test_mm_lookup(ctx->mm, ctx->address[4], &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[4], __pgprot(TEST_PROT))); + pte_unmap_unlock(pte, ptl); + /* Same physical slot at a different virtual byte offset and native index. */ + ctx->address[5] = TEST_VA + PAGE_SIZE + SZ_4K; + ctx->phys[5] = ctx->phys[3]; + pte = test_mm_lookup(ctx->mm, ctx->address[5], &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[5], __pgprot(TEST_PROT))); + pte_unmap_unlock(pte, ptl); + for (sync = 0; sync < 2; sync++) { + struct page_vma_mapped_walk whole = { + .pfn = ctx->phys[0] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = TEST_VA, + .flags = sync ? PVMW_SYNC : 0, + }; + unsigned int count = 0; + bool valid = true; + + /* A native-owner walk must visit every mapped 4K occupant. */ + while (page_vma_mapped_walk(&whole)) { + unsigned int index = (whole.address - TEST_VA) / SZ_4K; + + if (index >= 4 || index == 2 || !whole.pte || + pte_phys_mm(ctx->mm, ptep_get(whole.pte)) != ctx->phys[index]) + valid = false; + if (++count > 4) { + page_vma_mapped_walk_done(&whole); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 3U); + for (i = 0; i < 4; i++) { + for (j = 0; j < 6; j++) { + struct rmap_walk_range range = { + .address = ctx->address[j], + .subpage = true, + .subpage_offset = offset_in_page(ctx->phys[i]), + }; + DEFINE_FOLIO_RMAP_WALK(one, page_folio(pfn_to_page(PHYS_PFN(ctx->phys[i]))), + vma, range, sync ? PVMW_SYNC : 0); + struct mm_subpage *slot = NULL; + bool found = page_vma_mapped_walk(&one); + phys_addr_t phys = 0; + + if (found) { + phys = pte_phys_mm(ctx->mm, ptep_get(one.pte)); + slot = mm_subpage_get_from_phys(phys); + page_vma_mapped_walk_done(&one); + } + KUNIT_EXPECT_EQ(test, found, (i == j && j != 2) || + (j == 4 && i == 0) || (j == 5 && i == 3)); + if (found) { + KUNIT_EXPECT_EQ(test, phys, ctx->phys[i]); + KUNIT_EXPECT_NOT_NULL(test, slot); + if (slot) { + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), phys); + mm_subpage_put(slot); + } + } + } + } + } +} + +static int test_map_alias(struct test_mm *ctx, unsigned int index, + unsigned long addr, phys_addr_t phys) +{ + struct mm_struct *mm = ctx->mm; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d = p4d_alloc(mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + ctx->address[index] = addr; + ctx->phys[index] = phys; + if (!p4d) + return -ENOMEM; + pud = pud_alloc(mm, p4d, addr); + if (!pud) + return -ENOMEM; + pmd = pmd_alloc(mm, pud, addr); + if (!pmd) + return -ENOMEM; + pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); + if (!pte) + return -ENOMEM; + set_pte(pte, pte_mkspecial(phys_pte_mm(mm, phys, __pgprot(TEST_PROT)))); + pte_unmap_unlock(pte, ptl); + return 0; +} + +/* Physical-range lookup only; coarse anonymous ownership is a separate step. */ +static void user64k_rmap_lookup_test(struct kunit *test) +{ + struct test_mm *ctx; + struct vm_area_struct *vma; + void *data; + unsigned long pfn, per_leaf = SZ_64K / PAGE_SIZE; + unsigned int mode, sync, i; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace leaf larger than native page"); + return; + } + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + data = test_alloc_granule(test, 17); + KUNIT_ASSERT_NOT_NULL(test, data); + pfn = virt_to_phys(data) >> PAGE_SHIFT; + ctx->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, ctx->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(ctx->mm, 16), 0); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = (1UL << 29) - SZ_64K; + vma->vm_end = vma->vm_start + 4 * SZ_64K; + vma->vm_pgoff = 4096; + page_folio(virt_to_page(data))->index = vma->vm_pgoff; + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, test_map_alias(ctx, i, vma->vm_start + i * SZ_64K, + virt_to_phys(data) + (i == 1 ? SZ_64K : 0)), 0); + + for (mode = 0; mode < 2; mode++) { + if (mode) { + for (i = 0; i < 3; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(ctx->mm, ctx->address[i], &ptl); + + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, swp_entry_to_pte(make_readable_migration_entry( + ctx->phys[i] >> PAGE_SHIFT))); + pte_unmap_unlock(pte, ptl); + } + } + for (sync = 0; sync < 2; sync++) { + unsigned int flags = (sync ? PVMW_SYNC : 0) | + (mode ? PVMW_MIGRATION : 0); + + for (i = 0; i < 2 * per_leaf; i++) { + struct page_vma_mapped_walk walk = { + .pfn = pfn + i, .nr_pages = 1, + /* Deliberately invalid pgoff: single-page callers may omit it. */ + .pgoff = 0, .vma = vma, + .address = vma->vm_start + i * PAGE_SIZE, + .flags = flags, + }; + unsigned int count = 0; + bool valid = true; + + while (page_vma_mapped_walk(&walk)) { + valid &= walk.pte && walk.address == + vma->vm_start + (i / per_leaf) * SZ_64K; + if (++count > 1) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 1U); +#ifdef CONFIG_MEMORY_FAILURE + if (!mode) + KUNIT_EXPECT_EQ(test, page_mapped_in_vma(pfn_to_page(pfn + i), vma), + vma->vm_start + i * PAGE_SIZE); +#endif + } + /* Two native pages straddle a coarse-leaf and PTE-table boundary. */ + { + struct page_vma_mapped_walk walk = { + .pfn = pfn + per_leaf - 1, .nr_pages = 2, + .pgoff = vma->vm_pgoff + per_leaf - 1, .vma = vma, + .address = vma->vm_start + SZ_64K - PAGE_SIZE, + .flags = flags, + }; + unsigned int count = 0; + bool valid = true; + + KUNIT_EXPECT_EQ(test, vma_address_end(&walk), vma->vm_start + 2 * SZ_64K); + while (page_vma_mapped_walk(&walk)) { + valid &= walk.address == vma->vm_start + count * SZ_64K; + if (++count > 2) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 2U); + KUNIT_EXPECT_TRUE(test, walk.flags & PVMW_PGTABLE_CROSSED); + } + for (i = 0; i < 2; i++) { + struct page_vma_mapped_walk walk = { + .pfn = i ? pfn + 2 * per_leaf : pfn - 1, + .nr_pages = 1, .vma = vma, + .address = vma->vm_start, .flags = flags, + }; + bool found = page_vma_mapped_walk(&walk); + + if (found) + page_vma_mapped_walk_done(&walk); + KUNIT_EXPECT_FALSE(test, found); + } + } + } +} + +static void user4k_rmap_boundary_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, true); + struct vm_area_struct *vma; + unsigned int i, count = 0; + struct page_vma_mapped_walk walk; + bool valid = true; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = SZ_2M - SZ_4K; + vma->vm_end = vma->vm_start + PAGE_SIZE; + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, test_map_alias(ctx, 4 + i, vma->vm_start + i * SZ_4K, + ctx->phys[i]), 0); + walk = (struct page_vma_mapped_walk) { + .pfn = ctx->phys[0] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = vma->vm_start, + .flags = PVMW_SYNC, + }; + while (page_vma_mapped_walk(&walk)) { + if (count >= 3 || walk.address != vma->vm_start + count * SZ_4K || + pte_phys_mm(ctx->mm, ptep_get(walk.pte)) != ctx->phys[count]) + valid = false; + if (++count > 3) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 3U); + KUNIT_EXPECT_TRUE(test, walk.flags & PVMW_PGTABLE_CROSSED); +} + +static void native_rmap_walk_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, false); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 4 * PAGE_SIZE; + for (i = 0; i < 4; i++) { + struct page_vma_mapped_walk walk = { + .pfn = ctx->phys[i] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = ctx->address[i], + .flags = PVMW_SYNC, + }; + unsigned int count = 0; + bool valid = true; + + while (page_vma_mapped_walk(&walk)) { + if (walk.address != ctx->address[i] || + pte_phys_mm(ctx->mm, ptep_get(walk.pte)) != ctx->phys[i]) + valid = false; + if (++count > 1) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, i != 2 ? 1U : 0U); +#ifdef CONFIG_MEMORY_FAILURE + page_folio(pfn_to_page(ctx->phys[i] >> PAGE_SHIFT))->index = i; + KUNIT_EXPECT_EQ(test, page_mapped_in_vma(pfn_to_page(ctx->phys[i] >> PAGE_SHIFT), vma), + i != 2 ? ctx->address[i] : (unsigned long)-EFAULT); +#endif + } +} + +static void user4k_vma_offsets_test(struct kunit *test) +{ + struct vm_area_struct *vma = kunit_kzalloc(test, sizeof(*vma), GFP_KERNEL); + unsigned int slot, length, step; + + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_start = (1UL << 48) - 8 * PAGE_SIZE; + for (slot = 0; slot < TEST_SLOTS; slot++) { + for (length = 1; length <= 8; length++) { + struct vm_page_offset initial = { ULONG_MAX - 3, slot * SZ_4K }; + struct vm_page_offset pos; + + vma->vm_end = vma->vm_start + length * SZ_4K; + vma_set_page_offset(vma, initial); + KUNIT_EXPECT_EQ(test, vma_last_pgoff(vma), + initial.index + (slot + length - 1) / TEST_SLOTS); + for (step = 0; step < length; step++) { + unsigned long addr = vma->vm_start + step * SZ_4K; + + pos = vma_page_offset_at(vma, addr); + KUNIT_EXPECT_EQ(test, pos.index, initial.index + (slot + step) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, pos.offset, ((slot + step) % TEST_SLOTS) * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, linear_page_index(vma, addr), pos.index); + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), addr); + } + pos = vma_page_offset_at(vma, vma->vm_end); + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + pos = initial; + pos.index--; + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + if (slot) { + pos = initial; + pos.offset--; + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + } + } + } +} + +static void user4k_vma_intervals_test(struct kunit *test) +{ + struct vm_area_struct *vmas = kunit_kcalloc(test, 3, sizeof(*vmas), GFP_KERNEL); + struct rb_root_cached root = RB_ROOT_CACHED; + struct vm_area_struct *vma; + unsigned int i, count = 0; + + KUNIT_ASSERT_NOT_NULL(test, vmas); + for (i = 0; i < 3; i++) { + vmas[i].vm_start = TEST_VA + i * SZ_8K; + vmas[i].vm_end = vmas[i].vm_start + SZ_4K; + vma_set_page_offset(&vmas[i], (struct vm_page_offset) { 77, i * SZ_4K }); + vma_interval_tree_insert(&vmas[i], &root); + } + vma_interval_tree_foreach(vma, &root, 77, 77) { + unsigned long address = vma_address_at_offset(vma, + (struct vm_page_offset) { 77, SZ_4K }); + + KUNIT_EXPECT_EQ(test, address, + vma == &vmas[1] ? vma->vm_start : (unsigned long)-EFAULT); + count++; + } + KUNIT_EXPECT_EQ(test, count, 3U); + KUNIT_EXPECT_PTR_EQ(test, vma_interval_tree_iter_first(&root, 76, 76), NULL); + KUNIT_EXPECT_PTR_EQ(test, vma_interval_tree_iter_first(&root, 78, 78), NULL); + for (i = 0; i < 3; i++) + vma_interval_tree_remove(&vmas[i], &root); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&root.rb_root)); +} + +static void user4k_vma_mm_free(void *arg) +{ + mmput(arg); +} + +/* These private mms have no concurrent VMA writers after dropping the lock. */ +static struct vm_area_struct *test_vma_lookup(struct mm_struct *mm, unsigned long addr) +{ + struct vm_area_struct *vma; + + mmap_read_lock(mm); + vma = vma_lookup(mm, addr); + mmap_read_unlock(mm); + return vma; +} + +static void user4k_vma_split_test(struct kunit *test) +{ + struct mm_struct *mm = mm_alloc(); + struct vm_area_struct *vma, *middle, *part; + unsigned long start = TEST_VA + SZ_4K; + vma_flags_t flags; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select4k(mm), 0); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Dummy vm_ops keep insert_vm_struct from replacing this file-like offset. */ + vma->vm_start = start; + vma->vm_end = start + 3 * SZ_4K; + vm_flags_init(vma, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_offset(vma, (struct vm_page_offset) { 77, 3 * SZ_4K }); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + if (ret) { + mmap_write_unlock(mm); + vm_area_free(vma); + KUNIT_FAIL(test, "insert_vm_struct: %d", ret); + return; + } + { + VMA_ITERATOR(vmi, mm, start); + + vma_iter_load(&vmi); + flags = vma->flags; + vma_flags_set(&flags, VMA_WRITE_BIT); + middle = vma_modify_flags(&vmi, vma, vma, start + SZ_4K, + start + 2 * SZ_4K, &flags); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, middle); + KUNIT_EXPECT_EQ(test, mm->map_count, 3); + for (i = 0; i < 3; i++) { + part = test_vma_lookup(mm, start + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, part); + KUNIT_EXPECT_EQ(test, part->vm_start, start + i * SZ_4K); + KUNIT_EXPECT_EQ(test, part->vm_end, start + (i + 1) * SZ_4K); + KUNIT_EXPECT_EQ(test, part->vm_pgoff, 77UL + (i + 3) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(part), ((i + 3) % TEST_SLOTS) * (unsigned int)SZ_4K); + } +} + +static struct mm_struct *test_vma_mm_granule(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm = mm_alloc(); + + if (!mm) + return NULL; + if (kunit_add_action_or_reset(test, user4k_vma_mm_free, mm)) + return NULL; + if (test_mm_select_granule(mm, shift)) + return NULL; + return mm; +} + +static struct mm_struct *test_vma_mm(struct kunit *test) +{ + return test_vma_mm_granule(test, 12); +} + +static struct vm_area_struct *test_vma_add(struct mm_struct *mm, unsigned long start, + unsigned long len, struct vm_page_offset pos, + vm_flags_t flags) +{ + struct vm_area_struct *vma = vm_area_alloc(mm); + int ret; + + if (!vma) + return ERR_PTR(-ENOMEM); + vma->vm_start = start; + vma->vm_end = start + len; + vm_flags_init(vma, flags | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_offset(vma, pos); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + mmap_write_unlock(mm); + if (ret) { + vm_area_free(vma); + return ERR_PTR(ret); + } + return vma; +} + +static void user4k_vma_merge_test(struct kunit *test) +{ + unsigned long start = TEST_VA + SZ_4K; + unsigned int scenario, i; + + for (scenario = 0; scenario < 4; scenario++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vmas[3], *merged; + struct vm_page_offset initial = { 77, 3 * SZ_4K }; + struct vm_page_offset expected; + unsigned long left = scenario >= 2 ? SZ_4K : 0; + unsigned long right = scenario == 1 || scenario == 3 ? 2 * SZ_4K : 3 * SZ_4K; + vma_flags_t flags; + + KUNIT_ASSERT_NOT_NULL(test, mm); + for (i = 0; i < 3; i++) { + struct vm_page_offset pos = vm_page_offset_add(initial, i * SZ_4K); + vm_flags_t bits = VM_READ; + + if (i == 1) + bits |= VM_WRITE; + if ((!i && scenario >= 2) || (i == 2 && scenario == 1)) + bits |= VM_EXEC; + /* Same native index, wrong byte offset: this must not merge. */ + if (i == 2 && scenario == 3) + pos.offset = 0; + vmas[i] = test_vma_add(mm, start + i * SZ_4K, SZ_4K, pos, bits); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vmas[i]); + } + flags = vmas[1]->flags; + vma_flags_clear(&flags, VMA_WRITE_BIT); + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, start + SZ_4K); + + vma_iter_load(&vmi); + merged = vma_modify_flags(&vmi, vmas[0], vmas[1], start + SZ_4K, + start + 2 * SZ_4K, &flags); + if (!IS_ERR(merged)) { + vma_start_write(merged); + merged->flags = flags; + } + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, merged); + expected = vm_page_offset_add(initial, left); + KUNIT_EXPECT_EQ(test, mm->map_count, scenario == 0 ? 1 : scenario == 3 ? 3 : 2); + KUNIT_EXPECT_EQ(test, merged->vm_start, start + left); + KUNIT_EXPECT_EQ(test, merged->vm_end, start + right); + KUNIT_EXPECT_EQ(test, merged->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(merged), expected.offset); + } +} + +static void user4k_vma_partial_merge_test(struct kunit *test) +{ + unsigned long start = TEST_VA; + unsigned int side; + + for (side = 0; side < 2; side++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_page_offset initial = { 77, 2 * SZ_4K }; + struct vm_area_struct *first, *second, *merged, *remaining; + unsigned long first_len = side ? SZ_4K : 2 * SZ_4K; + vma_flags_t flags; + struct vm_page_offset expected; + + KUNIT_ASSERT_NOT_NULL(test, mm); + first = test_vma_add(mm, start, first_len, initial, VM_READ | (side ? 0 : VM_WRITE)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, first); + second = test_vma_add(mm, start + first_len, 3 * SZ_4K - first_len, + vm_page_offset_add(initial, first_len), VM_READ | (side ? VM_WRITE : 0)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, second); + flags = side ? first->flags : second->flags; + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, start + SZ_4K); + + vma_iter_load(&vmi); + merged = vma_modify_flags(&vmi, first, side ? second : first, + start + SZ_4K, start + 2 * SZ_4K, &flags); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, merged); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + KUNIT_EXPECT_EQ(test, merged->vm_start, start + (side ? 0 : SZ_4K)); + KUNIT_EXPECT_EQ(test, merged->vm_end, start + (side ? 2 : 3) * SZ_4K); + expected = vm_page_offset_add(initial, side ? 0 : SZ_4K); + KUNIT_EXPECT_EQ(test, merged->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(merged), expected.offset); + remaining = test_vma_lookup(mm, start + (side ? 2 * SZ_4K : 0)); + KUNIT_ASSERT_NOT_NULL(test, remaining); + expected = vm_page_offset_add(initial, side ? 2 * SZ_4K : 0); + KUNIT_EXPECT_EQ(test, remaining->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(remaining), expected.offset); + } +} + +static void user4k_vma_copy_shrink_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *source, *dest; + struct vm_page_offset initial = { 77, 3 * SZ_4K }; + unsigned long start = TEST_VA + SZ_4K; + unsigned long target = TEST_VA + 16 * PAGE_SIZE + SZ_4K; + bool need_locks = false; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + source = test_vma_add(mm, start, PAGE_SIZE, initial, VM_READ); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, source); + for (i = 0; i < 3; i++) { + unsigned long addr = i == 2 ? target - SZ_4K : target + i * SZ_4K; + struct vm_page_offset pos = vm_page_offset_add(initial, i == 2 ? 0 : (i + 1) * SZ_4K); + + mmap_write_lock(mm); + dest = copy_vma(&source, addr, SZ_4K, pos, &need_locks); + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + KUNIT_EXPECT_EQ(test, dest->vm_start, i == 2 ? target - SZ_4K : target); + KUNIT_EXPECT_EQ(test, dest->vm_end, i ? target + 2 * SZ_4K : target + SZ_4K); + KUNIT_EXPECT_EQ(test, dest->vm_pgoff, 77UL + (i == 2 ? 3UL : 4UL) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(dest), ((i == 2 ? 3U : 4U) % TEST_SLOTS) * SZ_4K); + KUNIT_EXPECT_EQ(test, source->vm_pgoff, initial.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(source), initial.offset); + } + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, dest->vm_start); + struct vm_page_offset pos = vma_page_offset_at(dest, target); + + vma_iter_load(&vmi); + ret = vma_shrink(&vmi, dest, target, dest->vm_end, pos); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, dest->vm_start, target); + KUNIT_EXPECT_EQ(test, dest->vm_end, target + 2 * SZ_4K); + KUNIT_EXPECT_EQ(test, dest->vm_pgoff, 77UL + 4 / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(dest), (4 % TEST_SLOTS) * SZ_4K); +} + +struct anon_slot_fixture { + struct mm_struct *mm; + struct vm_area_struct *vma; + struct mm_subpage_pool *pool; + struct mm_subpage *slots[4]; + bool mapped[4]; +}; + +static void anon_slot_fixture_free(void *arg) +{ + struct anon_slot_fixture *f = arg; + struct mmu_gather tlb; + unsigned int i; + + if (f->pool) + mm_subpage_pool_close(f->pool); + if (f->mm) { + mmap_write_lock(f->mm); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, addr, &ptl); + + if (!pte) + continue; + ptep_get_and_clear_full(f->mm, addr, pte, 0); + if (f->mapped[i]) { + struct vm_area_struct *vma = vma_lookup(f->mm, addr); + + mm_subpage_remove_anon_rmap(f->slots[i], vma); + dec_mm_counter(f->mm, MM_ANONPAGES); + f->mapped[i] = false; + } + pte_unmap_unlock(pte, ptl); + } + tlb_gather_mmu_fullmm(&tlb, f->mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(f->mm); + } + /* No slot reference is released before the PTEs and TLBs are retired. */ + for (i = 0; i < 4; i++) + if (f->slots[i]) + mm_subpage_put(f->slots[i]); + if (f->pool) + mm_subpage_pool_put(f->pool); + if (f->mm) + mmput(f->mm); + memset(f, 0, sizeof(*f)); +} + +static struct anon_slot_fixture *anon_slot_fixture_create_from(struct kunit *test, + struct anon_slot_fixture *parent) +{ + struct anon_slot_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct vm_area_struct *vma; + struct folio *folio; + int ret; + + if (!f || kunit_add_action_or_reset(test, anon_slot_fixture_free, f)) + return NULL; + f->mm = mm_alloc(); + if (!f->mm || test_mm_select4k(f->mm)) + return NULL; + f->pool = mm_subpage_pool_create(GFP_KERNEL); + if (!f->pool) + return NULL; + folio = folio_alloc(GFP_KERNEL, 0); + if (!folio) + return NULL; + ret = mm_subpage_pool_add_folio(f->pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + return NULL; + } + vma = vm_area_alloc(f->mm); + if (!vma) + return NULL; + vma_set_anonymous(vma); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * PAGE_SIZE; + vm_flags_init(vma, VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_prot(vma); + if (parent) { + mmap_write_lock(parent->mm); + mmap_write_lock_nested(f->mm, SINGLE_DEPTH_NESTING); + vma_set_page_offset(vma, vma_page_offset_at(parent->vma, TEST_VA)); + ret = anon_vma_fork(vma, parent->vma); + } else { + mmap_write_lock(f->mm); + ret = 0; + } + if (!ret && parent) { + VMA_ITERATOR(vmi, f->mm, TEST_VA); + + vma_start_write(vma); + ret = vma_iter_store_gfp(&vmi, vma, GFP_KERNEL); + if (!ret) + f->mm->map_count++; + } else if (!ret) { + ret = insert_vm_struct(f->mm, vma); + } + if (!ret) { + f->vma = vma; + ret = anon_vma_prepare(vma); + } + if (!f->vma) + unlink_anon_vmas(vma); + mmap_write_unlock(f->mm); + if (parent) + mmap_write_unlock(parent->mm); + if (!f->vma) + vm_area_free(vma); + return ret ? NULL : f; +} + +static struct anon_slot_fixture *anon_slot_fixture_create(struct kunit *test) +{ + return anon_slot_fixture_create_from(test, NULL); +} + +static void user4k_anon_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct folio *folio = NULL; + unsigned int i; + vm_flags_t flags; + int ret, bad_offset = 0, bad_index = 0; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + struct test_mm tables = { .mm = f->mm }; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + f->slots[i] = slot; + folio = mm_subpage_folio(slot); + /* Allocate the table first; clear the test leaf before rmap attachment. */ + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, addr, mm_subpage_phys(slot)), 0); + mmap_read_lock(f->mm); + folio_lock(folio); + pte = test_mm_lookup(f->mm, addr, &ptl); + if (!pte) { + folio_unlock(folio); + mmap_read_unlock(f->mm); + KUNIT_FAIL(test, "missing allocated PTE"); + return; + } + pte_clear(f->mm, addr, pte); + if (!i) + bad_offset = mm_subpage_add_anon_rmap(slot, f->vma, addr + 1); + ret = mm_subpage_add_anon_rmap(slot, f->vma, addr); + if (!ret) { + f->mapped[i] = true; + inc_mm_counter(f->mm, MM_ANONPAGES); + set_pte_at(f->mm, addr, pte, phys_pte_mm(f->mm, mm_subpage_phys(slot), + __pgprot(TEST_PROT | PTE_RDONLY))); + if (!i) + bad_index = mm_subpage_add_anon_rmap(slot, f->vma, addr + PAGE_SIZE); + } + pte_unmap_unlock(pte, ptl); + folio_unlock(folio); + mmap_read_unlock(f->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)i + 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)i + 1); + } + KUNIT_EXPECT_EQ(test, bad_offset, -EINVAL); + KUNIT_EXPECT_EQ(test, bad_index, -EINVAL); + KUNIT_EXPECT_EQ(test, folio->index, TEST_VA >> PAGE_SHIFT); + KUNIT_EXPECT_FALSE(test, PageAnonExclusive(&folio->page)); + KUNIT_EXPECT_EQ(test, folio_expected_ref_count(folio), folio_ref_count(folio)); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 1); + for (i = 0; i < 4; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, TEST_VA + i * SZ_4K, &ptl); + bool young; + + KUNIT_ASSERT_NOT_NULL(test, pte); + young = pte_young(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_FALSE(test, young); + } + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); +} + +/* The fixture takes one existing slot reference, including on failure. */ +static int anon_slot_fixture_map(struct anon_slot_fixture *f, + unsigned int index, struct mm_subpage *slot) +{ + unsigned long addr = TEST_VA + index * SZ_4K; + struct test_mm tables = { .mm = f->mm }; + struct folio *folio = mm_subpage_folio(slot); + spinlock_t *ptl; + pte_t *pte; + int ret; + + f->slots[index] = slot; + ret = test_map_alias(&tables, 0, addr, mm_subpage_phys(slot)); + if (ret) + return ret; + mmap_read_lock(f->mm); + folio_lock(folio); + pte = test_mm_lookup(f->mm, addr, &ptl); + if (!pte) { + ret = -ENOMEM; + goto unlock; + } + pte_clear(f->mm, addr, pte); + ret = mm_subpage_add_anon_rmap(slot, f->vma, addr); + if (!ret) { + f->mapped[index] = true; + inc_mm_counter(f->mm, MM_ANONPAGES); + set_pte_at(f->mm, addr, pte, phys_pte_mm(f->mm, mm_subpage_phys(slot), + __pgprot(TEST_PROT | PTE_RDONLY))); + } + pte_unmap_unlock(pte, ptl); +unlock: + folio_unlock(folio); + mmap_read_unlock(f->mm); + return ret; +} + +static void user4k_anon_fork_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *parent = anon_slot_fixture_create(test); + struct anon_slot_fixture *child; + struct mm_subpage *slot; + struct folio *folio; + vm_flags_t flags; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, parent); + child = anon_slot_fixture_create_from(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + for (i = 0; i < 4; i++) { + slot = mm_subpage_alloc_at(parent->pool, i * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(parent, i, slot), 0); + mm_subpage_get(slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(child, i, slot), 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 2); + } + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 8); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 8); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + anon_slot_fixture_free(parent); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 4); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(child->slots[i]), 1); + /* The child mappings and root remain valid after the parent disappears. */ + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); +} + +static void user4k_nonlinear_rmap_reuse_test(struct kunit *test) +{ + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct anon_slot_fixture *c = anon_slot_fixture_create(test); + struct mm_subpage *slot; + struct folio *folio; + struct anon_vma_chain *avc; + phys_addr_t phys; + vm_flags_t flags; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, c); + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + folio = mm_subpage_folio(slot); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + phys = mm_subpage_phys(slot); +#ifdef CONFIG_FAILSLAB + { + unsigned int pending; + + mmap_read_lock(b->mm); + folio_lock(folio); + WRITE_ONCE(current->fail_nth, 1); + ret = mm_subpage_add_anon_rmap(slot, b->vma, TEST_VA + SZ_4K); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + /* Keep cleanup valid even if allocation unexpectedly succeeds. */ + if (!ret) + mm_subpage_remove_anon_rmap(slot, b->vma); + folio_unlock(folio); + mmap_read_unlock(b->mm); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); + } +#endif + /* Allocation failure left the slot available for the same operation. */ + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + anon_slot_fixture_free(b); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), phys); + + /* Give this empty VMA a different logical native index as well. */ + mmap_write_lock(c->mm); + anon_vma_lock_write(c->vma->anon_vma); + list_for_each_entry(avc, &c->vma->anon_vma_chain, same_vma) + anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); + c->vma->vm_pgoff += 9; + list_for_each_entry(avc, &c->vma->anon_vma_chain, same_vma) + anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); + anon_vma_unlock_write(c->vma->anon_vma); + mmap_write_unlock(c->mm); +#ifdef CONFIG_FAILSLAB + /* Retiring a previous root also needs failure-atomic bookkeeping. */ + mmap_read_lock(c->mm); + folio_lock(folio); + WRITE_ONCE(current->fail_nth, 1); + ret = mm_subpage_add_anon_rmap(slot, c->vma, TEST_VA + 2 * SZ_4K); + WRITE_ONCE(current->fail_nth, 0); + if (!ret) + mm_subpage_remove_anon_rmap(slot, c->vma); + folio_unlock(folio); + mmap_read_unlock(c->mm); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); +#endif + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(c, 2, slot), 0); + mmap_read_lock(c->mm); + KUNIT_EXPECT_EQ(test, page_address_in_vma(folio, &folio->page, c->vma), + TEST_VA + 2 * SZ_4K); + mmap_read_unlock(c->mm); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + /* The original root may disappear before the other slot's last mapping. */ + anon_slot_fixture_free(a); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(c->slots[2]), 1); +} + +static void user4k_dup_mmap_test(struct kunit *test) +{ + struct anon_slot_fixture *parent = anon_slot_fixture_create(test); + struct mm_struct *child; + struct folio *folio; + unsigned int i; + int ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, parent); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(parent->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(parent, i, slot), 0); + pte = test_mm_lookup(parent->mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte_at(parent->mm, addr, pte, + pte_mkwrite_novma(pte_mkdirty(ptep_get(pte)))); + pte_unmap_unlock(pte, ptl); + } + /* Split into four 4K VMAs plus a trailing hole, preserving COW flags. */ + mmap_write_lock(parent->mm); + for (i = 1; i < 4; i += 2) { + unsigned long addr = TEST_VA + i * SZ_4K; + VMA_ITERATOR(vmi, parent->mm, addr); + struct vm_area_struct *vma = vma_iter_load(&vmi); + vma_flags_t flags = vma->flags; + + vma_flags_set(&flags, VMA_DONTDUMP_BIT); + vma = vma_modify_flags(&vmi, vma, vma, addr, addr + SZ_4K, &flags); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + break; + } + } + mmap_write_unlock(parent->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, parent->mm->map_count, 5); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, child->map_count, 5); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 4UL); + folio = mm_subpage_folio(parent->slots[0]); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 8); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 8); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + struct vm_area_struct *vma = test_vma_lookup(child, addr); + unsigned int side; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(parent->slots[i]), 2); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->mm; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, addr, &ptl); + pte_t entry; + + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_FALSE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), + mm_subpage_phys(parent->slots[i])); + } + } + anon_slot_fixture_free(parent); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 4); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + /* KUnit's child-mm cleanup now exercises ordinary exit_mmap(). */ +} + +static void test_subpage_put(void *arg) +{ + mm_subpage_put(arg); +} + +static void user4k_anon_root_lifetime_test(struct kunit *test) +{ + struct anon_slot_fixture *old = anon_slot_fixture_create(test); + struct anon_slot_fixture *other = anon_slot_fixture_create(test); + struct mm_subpage *slot; + struct anon_vma *root; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, other); + slot = mm_subpage_alloc_at(old->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(old, 0, slot), 0); + folio = mm_subpage_folio(slot); + root = old->vma->anon_vma->root; + mm_subpage_get(slot); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slot), 0); + anon_slot_fixture_free(old); + /* Only the owner now pins the root; no VMA or PTE survives. */ + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + mm_subpage_get(slot); + ret = anon_slot_fixture_map(other, 0, slot); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); +} + +static void test_folio_put(void *arg) +{ + folio_put(arg); +} + +static void user4k_data_tlb_batch_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct folio *native = folio_alloc(GFP_KERNEL, 2); + struct mmu_gather tlb; + struct folio *backing; + unsigned int i; + bool full = false; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, native), 0); + for (i = 0; i < 4; i++) { + f->slots[i] = mm_subpage_alloc_at(f->pool, i * SZ_4K); + if (IS_ERR(f->slots[i])) { + f->slots[i] = NULL; + KUNIT_FAIL(test, "slot allocation failed"); + return; + } + } + backing = mm_subpage_folio(f->slots[0]); + memset(folio_address(backing), 0x5a, SZ_4K); + tlb_gather_mmu(&tlb, f->mm); + tlb_change_page_size(&tlb, MM_SUBPAGE_SIZE); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + for (i = 0; i < 64; i++) { + mm_subpage_get(f->slots[i % 4]); + if (__tlb_remove_subpage(&tlb, f->slots[i % 4])) { + full = true; + tlb_flush_mmu(&tlb); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + } + if (i == 12) { + /* Exercise the native count payload (3 << 2 has tag bit 2). */ + folio_ref_add(native, 3); + if (__tlb_remove_folio_pages(&tlb, &native->page, 3, false)) { + full = true; + tlb_flush_mmu(&tlb); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + } + } + } + mm_subpage_put(f->slots[0]); + f->slots[0] = NULL; + /* Without memory pressure these fit in the local plus one dynamic batch. */ + if (!full) { + struct mm_subpage *extra = mm_subpage_alloc_at(f->pool, 0); + + KUNIT_EXPECT_EQ(test, PTR_ERR(extra), -EAGAIN); + if (!IS_ERR(extra)) + mm_subpage_put(extra); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), 67); + KUNIT_EXPECT_EQ(test, folio_ref_count(native), 4); + KUNIT_EXPECT_GT(test, tlb.batch_count, 0U); + } + tlb_finish_mmu(&tlb); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), 3); + KUNIT_EXPECT_EQ(test, folio_ref_count(native), 1); + f->slots[0] = mm_subpage_alloc_at(f->pool, 0); + if (IS_ERR(f->slots[0])) { + f->slots[0] = NULL; + KUNIT_FAIL(test, "slot was not reusable after TLB completion"); + return; + } + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->slots[0]), backing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(backing), 0, SZ_4K), NULL); + for (i = 0; i < 4; i++) { + struct mm_subpage *extra = mm_subpage_alloc_at(f->pool, i * SZ_4K); + + KUNIT_EXPECT_EQ(test, PTR_ERR(extra), -EAGAIN); + if (!IS_ERR(extra)) + mm_subpage_put(extra); + } +} + +static void user4k_anon_zap_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct mm_subpage *slots[4]; + struct folio *folio; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + slots[i] = mm_subpage_alloc_at(f->pool, i * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slots[i]); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slots[i]), 0); + mm_subpage_get(slots[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slots[i]), 0); + } + folio = mm_subpage_folio(slots[0]); + mmap_read_lock(f->mm); + zap_vma_range(f->vma, TEST_VA + SZ_4K, SZ_4K); + f->mapped[1] = false; + f->slots[1] = NULL; + mmap_read_unlock(f->mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 3); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 7); + for (i = 0; i < 4; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, TEST_VA + i * SZ_4K, &ptl); + bool present; + + KUNIT_ASSERT_NOT_NULL(test, pte); + present = pte_present(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, present, i != 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), i == 1 ? 0 : 1); + } + /* Include both the existing hole and a trailing run of empty entries. */ + mmap_read_lock(f->mm); + zap_vma(f->vma); + for (i = 0; i < 4; i++) { + f->mapped[i] = false; + f->slots[i] = NULL; + } + mmap_read_unlock(f->mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm, MM_ANONPAGES), 0UL); +} + +static void user4k_anon_writable_slots_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte; + bool writable; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + pte = test_mm_lookup(f->mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte_at(f->mm, addr, pte, + pte_mkwrite_novma(pte_mkdirty(ptep_get(pte)))); + writable = pte_write(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, writable); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + } + KUNIT_EXPECT_EQ(test, folio_mapcount(mm_subpage_folio(f->slots[0])), 4); +} + +static struct vm_area_struct *test_fault_vma_in_mm(struct mm_struct *mm, + unsigned long start, unsigned long len) +{ + struct vm_area_struct *vma; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + int ret; + + if (!mm) + return NULL; + vma = vm_area_alloc(mm); + if (!vma) + return NULL; + vma_set_anonymous(vma); + vma->vm_start = start; + vma->vm_end = start + len; + vm_flags_init(vma, flags); + vma->vm_page_prot = vm_get_page_prot(flags); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + mmap_write_unlock(mm); + if (ret) { + vm_area_free(vma); + return NULL; + } + return vma; +} + +struct coarse_anon_fixture { + struct mm_struct *mm[2]; + bool attached[2]; + struct vm_area_struct *vma[2]; + struct folio *folio; + unsigned long mapped[2]; + unsigned int leaves; + bool pinned; +}; + +static void coarse_anon_fixture_free(void *ptr) +{ + struct coarse_anon_fixture *f = ptr; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + + if (f->pinned) { + unpin_user_page(folio_page(f->folio, nr - 1)); + f->pinned = false; + } + for (i = 0; i < 2; i++) { + unsigned int leaf; + + if (!f->mm[i]) + continue; + mmap_write_lock(f->mm[i]); + for (leaf = 0; leaf < (f->leaves ?: 1); leaf++) { + unsigned long addr = TEST_VA + leaf * SZ_64K; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm[i], addr, &ptl); + + if (pte) { + ptep_get_and_clear_full(f->mm[i], addr, pte, 0); + if (f->mapped[i] & BIT(leaf)) { + folio_remove_rmap_pte(f->folio, + folio_page(f->folio, leaf * nr), f->vma[i]); + dec_mm_counter(f->mm[i], MM_ANONPAGES); + } + pte_unmap_unlock(pte, ptl); + } + if (f->mapped[i] & BIT(leaf)) { + folio_put_refs(f->folio, nr); + f->mapped[i] &= ~BIT(leaf); + } + } + mmap_write_unlock(f->mm[i]); + if (f->vma[i] && !f->attached[i]) { + unlink_anon_vmas(f->vma[i]); + vm_area_free(f->vma[i]); + } + mmput(f->mm[i]); + f->vma[i] = NULL; + f->mm[i] = NULL; + } + if (f->folio) { + folio_put(f->folio); + f->folio = NULL; + } +} + +static void expect_coarse_anon_state(struct kunit *test, struct folio *folio, + unsigned int maps, bool exclusive) +{ + unsigned int i, nr = folio_nr_pages(folio); + + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(maps * nr)); + for (i = 0; i < nr; i++) { + struct page *page = folio_page(folio, i); + + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, (atomic_read(&page->_mapcount) + 1), (int)maps); + KUNIT_EXPECT_EQ(test, PageAnonExclusive(page), exclusive); + } +} + +static void user64k_anon_rmap_span_test(struct kunit *test) +{ + struct coarse_anon_fixture *f; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + pte_t entry, *pte; + spinlock_t *ptl; + int ret, dup_ret, share_ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous leaf"); + return; + } + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_anon_fixture_free, f), 0); + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 16 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + for (i = 0; i < 2; i++) { + struct test_mm tables = {}; + + f->mm[i] = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm[i]); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(f->mm[i], 16), 0); + if (!i) { + f->vma[i] = test_fault_vma_in_mm(f->mm[i], TEST_VA, SZ_64K); + f->attached[i] = !!f->vma[i]; + } else { + f->vma[i] = vm_area_alloc(f->mm[i]); + if (f->vma[i]) { + vma_set_anonymous(f->vma[i]); + f->vma[i]->vm_start = TEST_VA; + f->vma[i]->vm_end = TEST_VA + SZ_64K; + vm_flags_init(f->vma[i], f->vma[0]->vm_flags); + f->vma[i]->vm_page_prot = f->vma[0]->vm_page_prot; + vma_set_page_offset(f->vma[i], vma_page_offset_at(f->vma[0], TEST_VA)); + } + } + KUNIT_ASSERT_NOT_NULL(test, f->vma[i]); + tables.mm = f->mm[i]; + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, TEST_VA, PFN_PHYS(folio_pfn(f->folio))), 0); + pte = test_mm_lookup(f->mm[i], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + pte_clear(f->mm[i], TEST_VA, pte); + pte_unmap_unlock(pte, ptl); + } + mmap_write_lock(f->mm[0]); + ret = anon_vma_prepare(f->vma[0]); + mmap_write_unlock(f->mm[0]); + KUNIT_ASSERT_EQ(test, ret, 0); + mmap_write_lock(f->mm[0]); + mmap_write_lock_nested(f->mm[1], SINGLE_DEPTH_NESTING); + ret = anon_vma_fork(f->vma[1], f->vma[0]); + if (!ret) { + VMA_ITERATOR(vmi, f->mm[1], TEST_VA); + + vma_start_write(f->vma[1]); + ret = vma_iter_store_gfp(&vmi, f->vma[1], GFP_KERNEL); + if (!ret) { + f->mm[1]->map_count++; + f->attached[1] = true; + } + } + mmap_write_unlock(f->mm[1]); + mmap_write_unlock(f->mm[0]); + KUNIT_ASSERT_EQ(test, ret, 0); + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_new_anon_rmap(f->folio, f->vma[0], TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, nr); /* Fixture retains its original allocation ref. */ + inc_mm_counter(f->mm[0], MM_ANONPAGES); + f->mapped[0] = true; + entry = pte_mkwrite(pte_mkdirty(phys_pte_mm(f->mm[0], PFN_PHYS(folio_pfn(f->folio)), + f->vma[0]->vm_page_prot)), f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + expect_coarse_anon_state(test, f->folio, 1, true); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)nr + 1); + + mm_flags_set(MMF_HAS_PINNED, f->mm[0]); + ret = try_grab_folio(page_folio(folio_page(f->folio, nr - 1)), 1, FOLL_PIN); + f->pinned = !ret; + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + ptep_set_wrprotect(f->mm[0], TEST_VA, pte); + raw_write_seqcount_begin(&f->mm[0]->write_protect_seq); + dup_ret = folio_try_dup_anon_rmap_pte(f->folio, &f->folio->page, f->vma[1], f->vma[0]); + raw_write_seqcount_end(&f->mm[0]->write_protect_seq); + entry = ptep_get_and_clear(f->mm[0], TEST_VA, pte); + share_ret = folio_try_share_anon_rmap_pte(f->folio, &f->folio->page, f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, dup_ret, -EBUSY); + KUNIT_EXPECT_EQ(test, share_ret, -EBUSY); + expect_coarse_anon_state(test, f->folio, 1, true); + unpin_user_page(folio_page(f->folio, nr - 1)); + f->pinned = false; + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get_and_clear(f->mm[0], TEST_VA, pte); + share_ret = folio_try_share_anon_rmap_pte(f->folio, &f->folio->page, f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, pte_wrprotect(entry), 1); + pte_unmap_unlock(pte, ptl); + KUNIT_ASSERT_EQ(test, share_ret, 0); + expect_coarse_anon_state(test, f->folio, 1, false); + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + raw_write_seqcount_begin(&f->mm[0]->write_protect_seq); + dup_ret = folio_try_dup_anon_rmap_pte(f->folio, &f->folio->page, f->vma[1], f->vma[0]); + raw_write_seqcount_end(&f->mm[0]->write_protect_seq); + if (!dup_ret) { + folio_ref_add(f->folio, nr); + inc_mm_counter(f->mm[1], MM_ANONPAGES); + f->mapped[1] = true; + } + pte_unmap_unlock(pte, ptl); + KUNIT_ASSERT_EQ(test, dup_ret, 0); + pte = test_mm_lookup(f->mm[1], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_ptes(f->mm[1], TEST_VA, pte, pte_wrprotect(entry), 1); + pte_unmap_unlock(pte, ptl); + expect_coarse_anon_state(test, f->folio, 2, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)(2 * nr + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[0], MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[1], MM_ANONPAGES), 1L); + + /* Exercise ordinary unmapping of each shared coarse leaf. */ + for (i = 0; i < 2; i++) { + mmap_read_lock(f->mm[i]); + zap_vma_range(f->vma[i], TEST_VA, SZ_64K); + f->mapped[i] = 0; + mmap_read_unlock(f->mm[i]); + expect_coarse_anon_state(test, f->folio, 1 - i, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)((1 - i) * nr + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[i], MM_ANONPAGES), 0L); + } + + /* Keep an observation ref while cleanup releases the empty address spaces. */ + { + struct folio *folio = f->folio; + + folio_get(folio); + coarse_anon_fixture_free(f); + expect_coarse_anon_state(test, folio, 0, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); + } +} + +static void user64k_anon_zap_batch_test(struct kunit *test) +{ + struct coarse_anon_fixture *f; + struct test_mm tables = {}; + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned int i, nr = SZ_64K / PAGE_SIZE, batch; + pte_t entry, *pte; + spinlock_t *ptl; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous leaves"); + return; + } + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_anon_fixture_free, f), 0); + f->leaves = 2; + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 17 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + mm = f->mm[0] = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + /* Partial unmap can enqueue this folio on the memcg split queue. */ + KUNIT_ASSERT_EQ(test, mem_cgroup_charge(f->folio, mm, GFP_KERNEL), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, 16), 0); + vma = f->vma[0] = test_fault_vma_in_mm(mm, TEST_VA, 2 * SZ_64K); + f->attached[0] = !!vma; + KUNIT_ASSERT_NOT_NULL(test, vma); + tables.mm = mm; + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, i, addr, + PFN_PHYS(folio_pfn(f->folio)) + i * SZ_64K), 0); + pte = test_mm_lookup(mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + pte_clear(mm, addr, pte); + pte_unmap_unlock(pte, ptl); + } + mmap_write_lock(mm); + ret = anon_vma_prepare(vma); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_new_anon_rmap(f->folio, vma, TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, 2 * nr); + add_mm_counter(mm, MM_ANONPAGES, 2); + f->mapped[0] = BIT(0) | BIT(1); + entry = pte_mkwrite(pte_mkdirty(phys_pte_mm(mm, + PFN_PHYS(folio_pfn(f->folio)), vma->vm_page_prot)), vma); + set_ptes(mm, TEST_VA, pte, entry, 2); + batch = folio_subpage_pte_batch(vma, f->folio, pte, entry, 2, 0); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, batch, 2U); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)(2 * nr)); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)(2 * nr + 1)); + + /* The reference walker must consume logical PTEs and native mapcounts. */ + { + vm_flags_t flags; + bool young; + + folio_lock(f->folio); + ret = folio_referenced(f->folio, 1, NULL, &flags); + folio_unlock(f->folio); + KUNIT_EXPECT_EQ(test, ret, 1); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + young = pte_young(ptep_get(pte)) || pte_young(ptep_get(pte + 1)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_FALSE(test, young); + folio_lock(f->folio); + ret = folio_referenced(f->folio, 1, NULL, &flags); + folio_unlock(f->folio); + KUNIT_EXPECT_EQ(test, ret, 0); + } + + /* Removing one logical leaf must leave the adjacent leaf's native span. */ + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, SZ_64K); + f->mapped[0] &= ~BIT(0); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)nr); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)nr + 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1L); + for (i = 0; i < 2 * nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(f->folio, i)->_mapcount), + i < nr ? -1 : 0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + ret = pte_none(ptep_get(pte)) && pte_present(ptep_get(pte + 1)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, ret); + + /* Restore the first leaf, then remove both through one batched walk. */ + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_anon_rmap_pte(f->folio, &f->folio->page, vma, + TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, nr); + inc_mm_counter(mm, MM_ANONPAGES); + f->mapped[0] |= BIT(0); + set_ptes(mm, TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)(2 * nr)); + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, 2 * SZ_64K); + f->mapped[0] = 0; + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + for (i = 0; i < 2 * nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(f->folio, i)->_mapcount), -1); +} + +static void user64k_file_rmap_span_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse nonanonymous leaf"); + return; + } + mm = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * SZ_64K; + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 16 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, folio); + /* Private metadata-only fixture: no file-cache promotion or fault is implied. */ + folio_lock(folio); + folio_add_file_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount) + 1, 1); + folio_dup_file_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(2 * nr)); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount) + 1, 2); + folio_remove_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + folio_remove_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount), -1); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); +} + +static struct vm_area_struct *test_fault_vma(struct kunit *test, + unsigned long start, unsigned long len) +{ + return test_fault_vma_in_mm(test_vma_mm(test), start, len); +} + +static struct vm_area_struct *test_fault_vma_granule(struct kunit *test, + unsigned int shift, unsigned long start, unsigned long len) +{ + return test_fault_vma_in_mm(test_vma_mm_granule(test, shift), start, len); +} + +static void test_anon_write_fault_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned int shape; + + /* Swap/reclaim integration is a later gate, before any ABI activation. */ + KUNIT_ASSERT_EQ(test, total_swap_pages, 0UL); + for (shape = 0; shape < 2; shape++) { + unsigned long start = TEST_VA + (shape ? size : 0); + unsigned long len = shape ? size : 2 * PAGE_SIZE; + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, start, len); + struct folio *backing = NULL; + unsigned int i, mapped = 0; + vm_fault_t ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + ret = handle_mm_fault(vma, TEST_VA + size + 3, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + /* Remove transient LRU-add references before checking ownership. */ + lru_add_drain_all(); + for (i = 0; i < (PAGE_SIZE >> shift); i++) { + unsigned long addr = TEST_VA + i * size; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(vma->vm_mm, addr, &ptl); + struct mm_subpage *slot = NULL; + bool expected = !shape || i == 1; + bool present, writable = false, zero = false, identity = false; + + KUNIT_ASSERT_NOT_NULL(test, pte); + present = pte_present(ptep_get(pte)); + if (present) { + slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, ptep_get(pte))); + writable = pte_write(ptep_get(pte)); + if (slot) { + struct folio *folio = mm_subpage_folio(slot); + + if (!backing) + backing = folio; + identity = folio == backing && + mm_subpage_offset(slot) == i * size; + zero = !memchr_inv(folio_address(folio) + mm_subpage_offset(slot), + 0, size); + mm_subpage_put(slot); + } + mapped++; + } + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, present, expected); + if (expected) { + KUNIT_EXPECT_TRUE(test, writable); + KUNIT_EXPECT_TRUE(test, identity); + KUNIT_EXPECT_TRUE(test, zero); + } + } + KUNIT_ASSERT_NOT_NULL(test, backing); + KUNIT_EXPECT_EQ(test, mapped, shape ? 1U : (PAGE_SIZE >> shift)); + KUNIT_EXPECT_EQ(test, folio_mapcount(backing), (int)mapped); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), (int)mapped); + KUNIT_EXPECT_TRUE(test, folio_test_lru(backing)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), + (unsigned long)mapped); + } +} + +static void user4k_anon_write_fault_test(struct kunit *test) +{ + test_anon_write_fault_granule(test, 12); +} + +static void user16k_anon_write_fault_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_anon_write_fault_granule(test, 14); +} + +static void user4k_swap_encoding_test(struct kunit *test) +{ + const unsigned long offsets[] = { 1, 2, (1UL << __SWP_OFFSET_BITS) - 2 }; + const unsigned int types[] = { 0, 1, __SWP_TYPE_MASK }; + unsigned int i, j, slot, flags; + + for (i = 0; i < ARRAY_SIZE(offsets); i++) { + for (j = 0; j < ARRAY_SIZE(types); j++) { + for (slot = 0; slot < PAGE_SIZE / SZ_4K; slot++) { + for (flags = 0; flags < 4; flags++) { + softleaf_t entry = swp_entry(types[j], offsets[i]); + pte_t pte = softleaf_to_pte(entry), moved; + + pte = pte_swp_set_subpage_offset(pte, slot * SZ_4K); + if (flags & 1) + pte = pte_swp_mkexclusive(pte); + if (flags & 2) + pte = pte_swp_mkuffd_wp(pte); + KUNIT_EXPECT_FALSE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), slot * (unsigned int)SZ_4K); + moved = pte_move_swp_offset(pte, 1); + KUNIT_EXPECT_EQ(test, swp_offset(softleaf_from_pte(moved)), offsets[i] + 1); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(moved), slot * (unsigned int)SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_same(pte_move_swp_offset(moved, -1), pte)); + moved = pte_swp_set_subpage_offset(pte, ((slot + 1) % (PAGE_SIZE / SZ_4K)) * SZ_4K); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(moved).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(moved), ((slot + 1) % (PAGE_SIZE / SZ_4K)) * (unsigned int)SZ_4K); + } + } + } + } +} + +static pte_t test_fault_entry(struct kunit *test, struct mm_struct *mm, unsigned long addr) +{ + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, addr, &ptl); + pte_t entry = __pte(0); + + KUNIT_EXPECT_NOT_NULL(test, pte); + if (pte) { + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + } + return entry; +} + + +static void test_file_put(void *file) +{ + fput(file); +} + +static struct file *test_cache_file(struct kunit *test, unsigned long size, bool fill) +{ + struct file *file = shmem_file_setup("user4k-file-test", size, EMPTY_VMA_FLAGS); + unsigned char *buf; + loff_t pos = 0; + unsigned int i; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, test_file_put, file)) + return ERR_PTR(-ENOMEM); + if (!fill) + return file; + buf = kunit_kmalloc(test, size, GFP_KERNEL); + if (!buf) + return ERR_PTR(-ENOMEM); + for (i = 0; i < size; i++) + buf[i] = 0x31 + (i / SZ_4K); + if (kernel_write(file, buf, size, &pos) != size) + return ERR_PTR(-EIO); + return file; +} + +static struct vm_area_struct *test_file_vma_granule(struct kunit *test, struct file *file, + unsigned int shift, bool shared, unsigned long start, unsigned long len, + unsigned long offset) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + int ret; + + mm = test_vma_mm_granule(test, shift); + if (!mm) + return NULL; + vma = vm_area_alloc(mm); + if (!vma) + return NULL; + vma->vm_start = start; + vma->vm_end = start + len; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = offset >> PAGE_SHIFT, .offset = offset & ~PAGE_MASK, + }); + if (shared) + flags |= VM_SHARED | VM_MAYSHARE; + vm_flags_init(vma, flags); + vma->vm_file = get_file(file); + vma->vm_page_prot = vm_get_page_prot(flags); + mmap_write_lock(mm); + ret = vfs_mmap(file, vma); + if (!ret) { + vma_set_page_prot(vma); + ret = insert_vm_struct(mm, vma); + } + mmap_write_unlock(mm); + if (ret) { + fput(vma->vm_file); + vm_area_free(vma); + return NULL; + } + return vma; +} + +static struct vm_area_struct *test_file_vma(struct kunit *test, struct file *file, + bool native, bool shared, unsigned long start, unsigned long len, + unsigned long offset) +{ + return test_file_vma_granule(test, file, native ? PAGE_SHIFT : 12, + shared, start, len, offset); +} + +static void user4k_mmap_descriptor_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct vm_area_desc desc; + + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + PAGE_SIZE; + vma_set_page_offset(vma, (struct vm_page_offset) { 7, SZ_4K }); + compat_set_desc_from_vma(&desc, NULL, vma); + KUNIT_EXPECT_EQ(test, desc.pgoff, 7UL); + KUNIT_EXPECT_EQ(test, desc.subpage_offset, (unsigned int)SZ_4K); + desc.pgoff = 19; + desc.subpage_offset = 3 * SZ_4K; + KUNIT_ASSERT_EQ(test, __compat_vma_mmap(&desc, vma), 0); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, TEST_VA), + (struct vm_page_offset) { 19, 3 * SZ_4K })); +} + +static void user4k_mmap_region_offsets_test(struct kunit *test) +{ + struct file *file; + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_SHARED | VM_MAYSHARE | VM_NORESERVE; + unsigned long result[4]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 8 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + /* Only the full native-sized lengths are used before accounting conversion. */ + kthread_use_mm(mm); + mmap_write_lock(mm); + result[0] = mmap_region(file, start + PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 1, SZ_4K }, NULL); + result[1] = mmap_region(file, start + 2 * PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 2, SZ_4K }, NULL); + result[2] = mmap_region(file, start, PAGE_SIZE, flags, + (struct vm_page_offset) { 0, SZ_4K }, NULL); + /* Adjacent virtual bytes with a different file offset must not merge. */ + result[3] = mmap_region(file, start + 3 * PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 3, 2 * SZ_4K }, NULL); + mmap_write_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, result[0], start + PAGE_SIZE); + KUNIT_ASSERT_EQ(test, result[1], start + 2 * PAGE_SIZE); + KUNIT_ASSERT_EQ(test, result[2], start); + KUNIT_ASSERT_EQ(test, result[3], start + 3 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_start, start); + KUNIT_EXPECT_EQ(test, vma->vm_end, start + 3 * PAGE_SIZE); + for (i = 0; i < 12; i++) + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, start + i * SZ_4K), + (struct vm_page_offset) { (i + 1) / TEST_SLOTS, ((i + 1) % TEST_SLOTS) * SZ_4K })); + vma = test_vma_lookup(mm, start + 3 * PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 2U * SZ_4K); +} + +#ifdef CONFIG_PROC_FS +/* Exercise the actual status/statm producers without publishing a fake task. */ +extern void task_mem(struct seq_file *m, struct mm_struct *mm); +extern unsigned long task_vsize(struct mm_struct *mm); +extern unsigned long task_statm(struct mm_struct *mm, unsigned long *shared, + unsigned long *text, unsigned long *data, unsigned long *resident); + +static void expect_status_kb(struct kunit *test, struct seq_file *seq, + const char *name, unsigned long expected) +{ + const char *field = strstr(seq->buf, name); + unsigned long value = ULONG_MAX; + + KUNIT_ASSERT_NOT_NULL(test, field); + KUNIT_EXPECT_EQ(test, sscanf(field + strlen(name), "%lu", &value), 1); + KUNIT_EXPECT_EQ_MSG(test, value, expected, "%s", name); +} + +static void user4k_accounting_readers_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test); + struct mm_struct *native = mm_alloc(); + struct seq_file seq = {}; + unsigned long shared, text, data, resident, size, maxrss = 0; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + seq.size = 2048; + seq.buf = kunit_kzalloc(test, seq.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, seq.buf); + /* Same logical counts, deliberately not multiples of a native page. */ + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? native : small; + unsigned long kb = i ? PAGE_SIZE / 1024 : 4; + + mm->total_vm = 11; + mm->hiwater_vm = 13; + mm->data_vm = 5; + mm->stack_vm = 2; + mm->exec_vm = 3; + mm->locked_vm = mm_pages_to_account(mm, 1); + atomic64_set(&mm->pinned_vm, 2); + mm->hiwater_rss = 10; + mm->start_code = TEST_VA + mm_page_size(mm); + mm->end_code = mm->start_code + 2 * mm_page_size(mm); + percpu_counter_set(&mm->rss_stat[MM_ANONPAGES], 5); + percpu_counter_set(&mm->rss_stat[MM_FILEPAGES], 2); + percpu_counter_set(&mm->rss_stat[MM_SHMEMPAGES], 1); + percpu_counter_set(&mm->rss_stat[MM_SWAPENTS], 3); + seq.count = 0; + task_mem(&seq, mm); + KUNIT_EXPECT_FALSE(test, seq_has_overflowed(&seq)); + seq.buf[min(seq.count, seq.size - 1)] = '\0'; + expect_status_kb(test, &seq, "VmPeak:", 13 * kb); + expect_status_kb(test, &seq, "VmSize:", 11 * kb); + expect_status_kb(test, &seq, "VmLck:", kb); + expect_status_kb(test, &seq, "VmPin:", 2 * PAGE_SIZE / 1024); + expect_status_kb(test, &seq, "VmHWM:", 10 * kb); + expect_status_kb(test, &seq, "VmRSS:", 8 * kb); + expect_status_kb(test, &seq, "RssAnon:", 5 * kb); + expect_status_kb(test, &seq, "RssFile:", 2 * kb); + expect_status_kb(test, &seq, "RssShmem:", kb); + expect_status_kb(test, &seq, "VmData:", 5 * kb); + expect_status_kb(test, &seq, "VmStk:", 2 * kb); + expect_status_kb(test, &seq, "VmExe:", 2 * kb); + expect_status_kb(test, &seq, "VmLib:", kb); + expect_status_kb(test, &seq, "VmSwap:", 3 * kb); + KUNIT_EXPECT_EQ(test, task_vsize(mm), 11 * kb * 1024); + { + struct vm_area_struct vma = { .vm_mm = mm }; + + KUNIT_EXPECT_EQ(test, vma_kernel_pagesize(&vma), kb * 1024); + KUNIT_EXPECT_EQ(test, vma_mmu_pagesize(&vma), kb * 1024); + } + } + /* Maxima must compare bytes, not incompatible 4K/16K counts. */ + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, 40UL); + setmax_mm_hiwater_rss(&maxrss, native); + KUNIT_EXPECT_EQ(test, maxrss, 10UL * PAGE_SIZE / 1024); + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, 10UL * PAGE_SIZE / 1024); + small->hiwater_rss = 10 * TEST_SLOTS + 1; + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, (10UL * TEST_SLOTS + 1) * 4); + + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 0), 0UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 1), 1UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 5), DIV_ROUND_UP(5UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(native, small, 5), 5UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(small, native, 5), DIV_ROUND_UP(5UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(small, small, 5), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(native, native, 5), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_shift(small, 5, MM_ACCOUNT_SHIFT), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_shift(native, 5, MM_ACCOUNT_SHIFT), 5UL * TEST_SLOTS); + + /* A kernel reader defaults to native units; partial pages round up. */ + size = task_statm(small, &shared, &text, &data, &resident); + KUNIT_EXPECT_EQ(test, size, DIV_ROUND_UP(11UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, shared, 1UL); + KUNIT_EXPECT_EQ(test, text, 1UL); + KUNIT_EXPECT_EQ(test, data, DIV_ROUND_UP(7UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, resident, DIV_ROUND_UP(8UL, TEST_SLOTS)); + if (user4k_test_enabled || IS_ENABLED(CONFIG_KASAN)) { + kthread_use_mm(small); + size = task_statm(native, &shared, &text, &data, &resident); + kthread_unuse_mm(small); + KUNIT_EXPECT_EQ(test, size, 11UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, shared, 3UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, text, 2UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, data, 7UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, resident, 8UL * TEST_SLOTS); + } + /* Synthetic counters have no page references; restore before mmput. */ + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? native : small; + unsigned int counter; + + for (counter = 0; counter < NR_MM_COUNTERS; counter++) + percpu_counter_set(&mm->rss_stat[counter], 0); + mm->total_vm = mm->data_vm = mm->stack_vm = mm->exec_vm = 0; + mm->locked_vm = 0; + atomic64_set(&mm->pinned_vm, 0); + } +} +#endif + +static void mixed_locked_account_units_test(struct kunit *test) +{ + unsigned int shift; + unsigned long native_units = PAGE_SIZE >> MM_ACCOUNT_SHIFT; + + for (shift = 12; shift <= 16; shift += 2) { + struct mm_struct *mm = mm_alloc(); + unsigned long size = 1UL << shift; + unsigned long base, limit; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + if (shift != PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shift), 0); + base = mm_pages_to_account(mm, 1); + mmap_write_lock(mm); + /* Model one locked VMA leaf and two independently charged native pins. */ + mm->locked_vm = base; + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base + native_units); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size + PAGE_SIZE); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(mm), + DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE)); + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base + 2 * native_units); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size + PAGE_SIZE); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size); + + /* A refused charge must leave the exact pre-existing amount intact. */ + limit = task_rlimit(current, RLIMIT_MEMLOCK) >> MM_ACCOUNT_SHIFT; + mm->locked_vm = limit; + ret = __account_locked_vm(mm, 1, true, current, false); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, limit); + mm->locked_vm = ULONG_MAX - native_units + 1; + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, ULONG_MAX - native_units + 1); + mm->locked_vm = 0; + if (native_units > 1) { + ret = __account_locked_vm(mm, + (ULONG_MAX >> (PAGE_SHIFT - MM_ACCOUNT_SHIFT)) + 1, + true, current, true); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + } + mmap_write_unlock(mm); + } +} + +/* Prior shmem fixtures can refund global commit from deferred fput work. */ +static s64 test_commit_baseline(struct kunit *test) +{ + s64 before = percpu_counter_sum(&vm_committed_as), after; + + /* No mmap, file, or filesystem locks are held at these call sites. */ + flush_delayed_fput(); + after = percpu_counter_sum(&vm_committed_as); + if (after != before) + kunit_info(test, "drained deferred file refunds: commit delta %lld", after - before); + return after; +} + +static void user64k_commit_accounting_test(struct kunit *test) +{ + struct mm_struct *mm; + s64 before; + unsigned int i; + unsigned long native_per_leaf; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace granule larger than native"); + return; + } + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, 16), 0); + native_per_leaf = SZ_64K / PAGE_SIZE; + before = test_commit_baseline(test); + mmap_write_lock(mm); + for (i = 0; i < 9; i++) { + ret = mm_account_memory(mm, mm, 1); + KUNIT_EXPECT_EQ(test, ret, 0); + if (ret) + break; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i + 1); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)((i + 1) * native_per_leaf)); + } + /* Multiplication overflow must not mutate either reservation counter. */ + ret = mm_account_memory(mm, mm, LONG_MAX / native_per_leaf + 1); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + mm_acct_memory(mm, 3); + i += 3; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + while (i) { + mm_unacct_memory(mm, 1); + i--; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + } + mmap_write_unlock(mm); +} + +#ifdef CONFIG_PAGE_TABLE_CHECK +static void expect_coarse_file_counts(struct kunit *test, void *data, + unsigned int phase) +{ + unsigned int offset; + + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_4K) { + int anon = -1, file = -1, expected; + int ret = page_table_check_get_counts(virt_to_page(data + offset), + offset & ~PAGE_MASK, &anon, &file); + + /* Phase0: both coarse leaves;1:plusalias;2:firstclear;3:bothclear;4:allclear. */ + expected = phase < 2 || (phase == 2 && offset >= SZ_64K); + if (phase >= 1 && phase <= 3 && offset == 2 * SZ_64K - SZ_4K) + expected++; + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, anon, 0); + KUNIT_EXPECT_EQ_MSG(test, file, expected, "phase=%u offset=%u", phase, offset); + } +} +#endif + +static void user64k_page_table_check_test(struct kunit *test) +{ +#ifdef CONFIG_PAGE_TABLE_CHECK + struct mm_struct *coarse, *small; + void *data; + pte_t ptes[2] = {}, alias = __pte(0), first, last; + int anon, file; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace leaf larger than native page"); + return; + } + data = test_alloc_granule(test, 17); + KUNIT_ASSERT_NOT_NULL(test, data); + if (page_table_check_get_counts(virt_to_page(data), 0, &anon, &file)) { + kunit_skip(test, "requires page-table-check enabled at boot"); + return; + } + coarse = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, coarse), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(coarse, 16), 0); + small = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, small), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(small, 12), 0); + first = phys_pte_mm(coarse, virt_to_phys(data), __pgprot(TEST_PROT)); + last = phys_pte_mm(small, virt_to_phys(data) + 2 * SZ_64K - SZ_4K, + __pgprot(TEST_PROT)); + + /* Exercise checker hooks directly; no actual coarse file fault is claimed. */ + page_table_check_ptes_set(coarse, TEST_VA, ptes, first, 2); + ptes[0] = first; + ptes[1] = pte_advance_pfn_mm(coarse, first, 1); + expect_coarse_file_counts(test, data, 0); + page_table_check_ptes_set(small, TEST_VA, &alias, last, 1); + alias = last; + expect_coarse_file_counts(test, data, 1); + page_table_check_pte_clear(coarse, TEST_VA, ptes[0]); + ptes[0] = __pte(0); + expect_coarse_file_counts(test, data, 2); + page_table_check_pte_clear(coarse, TEST_VA + SZ_64K, ptes[1]); + ptes[1] = __pte(0); + expect_coarse_file_counts(test, data, 3); + page_table_check_pte_clear(small, TEST_VA, alias); + expect_coarse_file_counts(test, data, 4); +#else + kunit_skip(test, "requires CONFIG_PAGE_TABLE_CHECK"); +#endif +} + +static void user4k_commit_accounting_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + s64 before; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + mmap_write_lock(mm); + for (i = 0; i < 9; i++) { + ret = mm_account_memory(mm, mm, 1); + KUNIT_EXPECT_EQ(test, ret, 0); + if (ret) + break; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i + 1); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)DIV_ROUND_UP(i + 1, TEST_SLOTS)); + } + if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) { + unsigned long old = mm->committed_user_pages; + s64 committed = percpu_counter_sum(&vm_committed_as); + + ret = mm_account_memory(mm, mm, (totalram_pages() + total_swap_pages + 1) * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, old); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as), committed); + } + /* Native physical-page callers must charge four user granules per page. */ + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, (unsigned long)TEST_SLOTS); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + + /* Refund one granule at a time: splitting must not multiply refunds. */ + while (i) { + mm_unacct_memory(mm, 1); + i--; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)DIV_ROUND_UP(i, TEST_SLOTS)); + } + mmap_write_unlock(mm); +} + +/* Prefaulting must stop at each process-page boundary, not native pages. */ +static void user_granule_prefault_test(struct kunit *test) +{ + unsigned int shift; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + for (shift = 12; shift <= PAGE_SHIFT; shift += 2) { + struct mm_struct *mm = test_vma_mm_granule(test, shift); + unsigned long size = 1UL << shift, start = TEST_VA; + unsigned long flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long mapped[3]; + size_t left[3][4] = {}; + int phase, ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped[0] = vm_mmap(NULL, start, size, PROT_READ | PROT_WRITE, flags, 0); + mapped[2] = vm_mmap(NULL, start + 2 * size, size, + PROT_READ | PROT_WRITE, flags, 0); + mapped[1] = start + size; + if (mapped[0] == start && mapped[2] == start + 2 * size) { + for (phase = 0; phase < 3; phase++) { + char __user *addr = (char __user *)(start + 17); + size_t len = 3 * size - 17; + + /* Missing middle, then read-only, then fully writable. */ + if (phase) { + mapped[1] = vm_mmap(NULL, start + size, size, + PROT_READ | (phase == 2 ? PROT_WRITE : 0), + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + if (mapped[1] != start + size) + break; + } + left[phase][0] = fault_in_safe_writeable(addr, len); + left[phase][1] = fault_in_readable(addr, len); + left[phase][2] = fault_in_writeable(addr, len); + left[phase][3] = fault_in_subpage_writeable(addr, len); + } + ret = vm_munmap(start, 3 * size); + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped[0], start); + KUNIT_ASSERT_EQ(test, mapped[1], start + size); + KUNIT_ASSERT_EQ(test, mapped[2], start + 2 * size); + KUNIT_EXPECT_EQ(test, ret, 0); + for (phase = 0; phase < 3; phase++) { + KUNIT_EXPECT_EQ(test, left[phase][0], phase < 2 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][1], phase == 0 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][2], phase < 2 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][3], phase < 2 ? 2 * size : 0UL); + } + } +} + +static void user4k_mmap_accounting_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K, mapped; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + s64 before; + unsigned int i; + int ret = 0; + vm_fault_t fault = 0; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + kthread_use_mm(mm); + mmap_write_lock(mm); + mapped = mmap_region(NULL, start, 3 * SZ_4K, flags, + (struct vm_page_offset) { start >> PAGE_SHIFT, SZ_4K }, NULL); + mmap_write_unlock(mm); + if (!IS_ERR_VALUE(mapped)) { + mmap_read_lock(mm); + vma = vma_lookup(mm, start); + fault = handle_mm_fault(vma, start, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->data_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, 1LL); + /* Remove the middle first, then the two neighbors. */ + for (i = 0; i < 3; i++) { + unsigned int slot = i == 0 ? 1 : i == 1 ? 0 : 2; + VMA_ITERATOR(vmi, mm, start + slot * SZ_4K); + + kthread_use_mm(mm); + mmap_write_lock(mm); + ret = do_vmi_munmap(&vmi, mm, start + slot * SZ_4K, SZ_4K, NULL, false); + mmap_write_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 2UL - i); + KUNIT_EXPECT_EQ(test, mm->data_vm, 2UL - i); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 2UL - i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, i == 2 ? 0LL : 1LL); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); +} + +static vm_fault_t test_file_fault(struct vm_area_struct *vma, unsigned long addr, bool write) +{ + vm_fault_t ret; + + mmap_read_lock(vma->vm_mm); + ret = handle_mm_fault(vma, addr, FAULT_FLAG_REMOTE | + (write ? FAULT_FLAG_WRITE : 0), NULL); + mmap_read_unlock(vma->vm_mm); + return ret; +} + +static void user4k_vm_mmap_test(struct kunit *test) +{ + struct mm_struct *mm; + struct file *file; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + unsigned long anon_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long file_flags = MAP_SHARED | MAP_FIXED_NOREPLACE; + unsigned long result[7]; + int unmap[4]; + pte_t pte; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + mm->mmap_base = TEST_VA + 2 * PAGE_SIZE + SZ_4K; + mm_flags_clear(MMF_TOPDOWN, mm); + kthread_use_mm(mm); + result[0] = vm_mmap(NULL, start, SZ_4K, PROT_READ | PROT_WRITE, anon_flags, 0); + result[1] = vm_mmap(file, start + 2 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE, + file_flags, 3 * SZ_4K); + result[2] = vm_mmap(file, start + 3 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE, + file_flags, 4 * SZ_4K); + result[3] = vm_mmap(NULL, start, SZ_4K, PROT_READ, anon_flags, 0); + result[4] = vm_mmap(NULL, start + 17, SZ_4K, PROT_READ, anon_flags, 0); + result[5] = vm_mmap(file, start + SZ_4K, SZ_4K, PROT_READ, file_flags, SZ_4K + 1); + result[6] = vm_mmap(NULL, 0, SZ_4K + 1, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, result[0], start); + KUNIT_ASSERT_EQ(test, result[1], start + 2 * SZ_4K); + KUNIT_ASSERT_EQ(test, result[2], start + 3 * SZ_4K); + KUNIT_EXPECT_EQ(test, result[3], (unsigned long)-EEXIST); + KUNIT_EXPECT_EQ(test, result[4], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, result[5], (unsigned long)-EINVAL); + KUNIT_ASSERT_EQ(test, result[6], mm->mmap_base); + KUNIT_EXPECT_EQ(test, mm->total_vm, 5UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start, true) & VM_FAULT_ERROR, 0U); + vma = test_vma_lookup(mm, start + 2 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_end - vma->vm_start, 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 3U * SZ_4K); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + 2 * SZ_4K, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, mm, start + 2 * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(mm, pte)), (u8)0x34); + pte = test_fault_entry(test, mm, start + 3 * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(mm, pte)), (u8)0x35); + kthread_use_mm(mm); + unmap[0] = vm_munmap(start + 2 * SZ_4K, 1); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, unmap[0], 0); + vma = test_vma_lookup(mm, start + 3 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_pgoff, 4UL / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), (4U % TEST_SLOTS) * SZ_4K); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + kthread_use_mm(mm); + unmap[1] = vm_munmap(start, SZ_4K); + unmap[2] = vm_munmap(start + 3 * SZ_4K, SZ_4K); + unmap[3] = vm_munmap(result[6], 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, unmap[1], 0); + KUNIT_EXPECT_EQ(test, unmap[2], 0); + KUNIT_EXPECT_EQ(test, unmap[3], 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 0UL); + KUNIT_EXPECT_EQ(test, mm->map_count, 0); +} + +static void test_mlock_drain_all(struct kunit *test, struct folio *folio) +{ + unsigned int before, after; + bool counted; + + lru_add_drain(); + counted = folio_test_unevictable(folio); + before = counted ? folio->mlock_count : 0; + /* A sleeping fixture can enqueue mlock work on more than one CPU. */ + lru_add_drain_all(); + after = folio_test_unevictable(folio) ? folio->mlock_count : 0; + if (counted && before != after) + kunit_info(test, "remote mlock drain changed count %u -> %u", before, after); +} + +static void user4k_mlock_counts_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct mm_subpage *slot; + struct folio *folio; + unsigned long addr; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE | MAP_LOCKED, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, test_fault_entry(test, mm, TEST_VA))); + KUNIT_ASSERT_NOT_NULL(test, slot); + folio = mm_subpage_folio(slot); + folio_get(folio); + mm_subpage_put(slot); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm->locked_vm, (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + int ret; + + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA + i * SZ_4K, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + /* mlock_count shares storage with the evictable LRU links. */ + if (i != TEST_SLOTS - 1) + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS - 1 - i); + KUNIT_EXPECT_EQ(test, mm->locked_vm, TEST_SLOTS - 1UL - i); + KUNIT_EXPECT_EQ(test, folio_test_mlocked(folio), i != TEST_SLOTS - 1); + } +} + +static void test_mlock_enable_lru_cache(void *unused) +{ + lru_cache_enable(); +} + +static void user4k_mlock_shared_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *folio; + unsigned long addr; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + if (native) + mmput(native); + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + /* Exact counts require synchronous LRU/lock accounting. Normal deferred + * batches may legitimately miss locks while the folio is off LRU; + * user4k_mlock_repair_test checks that reclaim repairs that case. + */ + lru_cache_disable(); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_mlock_enable_lru_cache, NULL), 0); + kthread_use_mm(small); + addr = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + kthread_use_mm(native); + addr = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(native); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS + 1); + kthread_use_mm(small); + ret = vm_munmap(TEST_VA + SZ_4K, SZ_4K); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS); + kthread_use_mm(native); + ret = vm_munmap(TEST_VA, PAGE_SIZE); + kthread_unuse_mm(native); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS - 1); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + kthread_use_mm(small); + ret = vm_munmap(TEST_VA, PAGE_SIZE); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); +} + +static void test_mlock_putback(void *folio) +{ + folio_putback_lru(folio); +} + +static void user4k_mlock_repair_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode, walker; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + unsigned long size = 1UL << shifts[mode]; + + for (walker = 0; walker < 2; walker++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *folio; + unsigned long addr, offset; + vm_flags_t flags = 0; + int maps, ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_folio_put, folio), 0); + lru_add_drain_all(); + KUNIT_ASSERT_TRUE(test, folio_isolate_lru(folio)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_mlock_putback, folio), 0); + /* Two aliases leave a locked mapping even after native unmap. + * Force deferred mlocks to run while the folio is isolated. + */ + for (offset = 0; offset < 2 * PAGE_SIZE; offset += PAGE_SIZE) { + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA + offset, PAGE_SIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + offset); + } + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_lru(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + kunit_release_action(test, test_mlock_putback, folio); + lru_add_drain_all(); + KUNIT_ASSERT_TRUE(test, folio_test_unevictable(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, 0U); + + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); + maps = folio_mapcount(folio); + KUNIT_EXPECT_EQ(test, maps, (int)(2 * PAGE_SIZE / size) - 1); + folio_lock(folio); + if (walker) + ret = folio_referenced(folio, 1, NULL, &flags); + else + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + lru_add_drain_all(); + if (walker) { + KUNIT_EXPECT_GE(test, ret, 0); + KUNIT_EXPECT_TRUE(test, flags & VM_LOCKED); + } + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), maps); + for (offset = size; offset < 2 * PAGE_SIZE; offset += size) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + offset); + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_PTR_EQ(test, + page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(mm, pte)))), + folio); + } + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, 2 * PAGE_SIZE); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + kunit_info(test, "%uK %s repairs deferred mlock without unmapping", + 1U << (shifts[mode] - 10), walker ? "referenced" : "unmap"); + } + } +} + +asmlinkage long __arm64_sys_mremap(const struct pt_regs *regs); + +static unsigned long test_mremap(unsigned long old, unsigned long old_len, + unsigned long new_len, unsigned long flags, unsigned long new) +{ + struct pt_regs regs = { .regs = { old, old_len, new_len, flags, new } }; + + return __arm64_sys_mremap(®s); +} + +static int test_iova_bitmap_fill(struct iova_bitmap *bitmap, unsigned long iova, + size_t length, void *opaque) +{ + unsigned long unit = *(unsigned long *)opaque; + + iova_bitmap_set(bitmap, iova + 3 * unit, length - 10 * unit); + /* A second write advances backwards into an already populated range. */ + iova_bitmap_set(bitmap, iova, 5 * unit); + return 0; +} + +/* Real IOMMUFD lifecycle with a translation-recording IOMMU, no physical DMA. */ +struct test_fragment_domain { + struct iommu_domain domain; + struct xarray translations; + unsigned long fail_after; + unsigned long largest_map; + unsigned long mapped; + unsigned long map_calls; +}; + +struct test_fragment_iopt { + struct io_pagetable table; + struct test_fragment_domain domains[2]; + struct kunit *test; +}; + +static int test_fragment_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t phys, size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + unsigned long length = pgsize * pgcount, done; + int rc; + + mock->map_calls++; + mock->largest_map = max(mock->largest_map, length); + for (done = 0; done < length; done += SZ_4K) { + if (mock->fail_after && mock->mapped >= mock->fail_after) + return -ENOMEM; + rc = xa_insert(&mock->translations, (iova + done) >> 12, + xa_mk_value((phys + done) >> 12), gfp); + if (rc) + return rc; + *mapped += SZ_4K; + mock->mapped += SZ_4K; + } + return 0; +} + +static size_t test_fragment_unmap(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + unsigned long length = pgsize * pgcount, done; + + for (done = 0; done < length; done += SZ_4K) { + if (!xa_erase(&mock->translations, (iova + done) >> 12)) + break; + mock->mapped -= SZ_4K; + } + return done; +} + +static phys_addr_t test_fragment_translate(struct iommu_domain *domain, dma_addr_t iova) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + void *entry = xa_load(&mock->translations, iova >> 12); + + return entry ? ((phys_addr_t)xa_to_value(entry) << 12) + (iova & (SZ_4K - 1)) : 0; +} + +static void test_fragment_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) +{ +} + +static const struct iommu_domain_ops test_fragment_domain_ops = { + .map_pages = test_fragment_map, + .unmap_pages = test_fragment_unmap, + .iova_to_phys = test_fragment_translate, + .iotlb_sync = test_fragment_sync, +}; + +static void test_fragment_iopt_free(void *data) +{ + struct test_fragment_iopt *fixture = data; + struct iommu_domain *domain; + unsigned long unmapped, index; + unsigned int i; + + iopt_unmap_all(&fixture->table, &unmapped); + while ((domain = xa_find(&fixture->table.domains, &(unsigned long){0}, + ULONG_MAX, XA_PRESENT))) + iopt_table_remove_domain(&fixture->table, domain); + iopt_destroy_table(&fixture->table); + for (i = 0; i < ARRAY_SIZE(fixture->domains); i++) { + struct test_fragment_domain *mock = &fixture->domains[i]; + void *entry; + + KUNIT_EXPECT_TRUE(fixture->test, xa_empty(&mock->translations)); + xa_for_each(&mock->translations, index, entry) + xa_erase(&mock->translations, index); + xa_destroy(&mock->translations); + } +} + +static struct test_fragment_iopt *test_fragment_iopt_alloc(struct kunit *test) +{ + struct test_fragment_iopt *fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + unsigned int i; + + if (!fixture) + return NULL; + fixture->test = test; + iopt_init_table(&fixture->table); + for (i = 0; i < ARRAY_SIZE(fixture->domains); i++) { + struct test_fragment_domain *mock = &fixture->domains[i]; + + xa_init(&mock->translations); + mock->domain.ops = &test_fragment_domain_ops; + mock->domain.type = IOMMU_DOMAIN_UNMANAGED; + mock->domain.pgsize_bitmap = SZ_4K; + mock->domain.geometry.aperture_end = ULONG_MAX; + } + if (kunit_add_action_or_reset(test, test_fragment_iopt_free, fixture)) + return NULL; + return fixture; +} + +static int test_fragment_map_user(struct test_fragment_iopt *fixture, + struct mm_struct *mm, unsigned long va, unsigned long iova, unsigned long length) +{ + struct iommufd_ctx ctx = {}; + int rc; + + kthread_use_mm(mm); + rc = iopt_map_user_pages(&ctx, &fixture->table, &iova, (void __user *)va, + length, IOMMU_READ | IOMMU_WRITE, 0); + kthread_unuse_mm(mm); + return rc; +} + +static void user4k_iommufd_lifetime_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14, 16 }, mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + for (mode = 0; mode < ARRAY_SIZE(shifts); mode++) { + unsigned long size = 1UL << shifts[mode]; + unsigned long len = 2 * max(size, PAGE_SIZE); + unsigned long dest = TEST_VA + 32 * PAGE_SIZE + size; + unsigned long iova = SZ_1M, alias = SZ_16M, addr, moved, missed, unmapped; + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct iopt_pages *pages; + struct iopt_area *area; + unsigned char *buffer; + unsigned long off, pinned; + LIST_HEAD(copy); + int rc; + + if (mode && shifts[mode] == PAGE_SHIFT) + continue; + mm = test_vma_mm_granule(test, shifts[mode]); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + buffer = kunit_kmalloc(test, len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + memset(buffer, 0x31, len); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, buffer, len); + moved = missed ? 0 : test_mremap(addr, len, len, MREMAP_FIXED | MREMAP_MAYMOVE, dest); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, dest); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, dest, iova, len), 0); + pinned = atomic64_read(&mm->pinned_vm); + KUNIT_EXPECT_EQ(test, pinned, len / PAGE_SIZE); + KUNIT_EXPECT_GE(test, fixture->domains[0].largest_map, PAGE_SIZE); + for (off = 0; off < len; off += size) { + pte_t pte = test_fault_entry(test, mm, dest + off); + phys_addr_t actual = iommu_iova_to_phys(&fixture->domains[0].domain, iova + off); + + KUNIT_EXPECT_EQ(test, actual, pte_phys_mm(mm, pte)); + } + KUNIT_ASSERT_EQ(test, iopt_get_pages(&fixture->table, iova + size, len - size, ©), 0); + rc = iopt_map_pages(&fixture->table, ©, len - size, &alias, IOMMU_READ | IOMMU_WRITE, 0); + iopt_free_pages_list(©); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), pinned); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), pinned); + + /* Replace the original VA: mappings and copied IOAS must retain old pins. */ + kthread_use_mm(mm); + addr = vm_mmap(NULL, dest, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + memset(buffer, 0x5a, len); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)dest, buffer, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, missed, 0UL); + down_read(&fixture->table.iova_rwsem); + area = iopt_area_iter_first(&fixture->table, iova, iova); + pages = area->pages; + /* Force the domain-backed reader for the native control too. */ + memset(buffer, 0x77, len); + rc = iopt_pages_rw_access(pages, 0, buffer, len, IOMMUFD_ACCESS_RW_WRITE); + up_read(&fixture->table.iova_rwsem); + KUNIT_ASSERT_EQ(test, rc, 0); + for (off = 0; off < len; off += SZ_4K) { + phys_addr_t phys = iommu_iova_to_phys(&fixture->domains[1].domain, iova + off); + void *ptr = kmap_local_page(pfn_to_page(PHYS_PFN(phys))); + bool correct = !memchr_inv(ptr + offset_in_page(phys), 0x77, SZ_4K); + + kunmap_local(ptr); + KUNIT_EXPECT_TRUE(test, correct); + } + kthread_use_mm(mm); + missed = copy_from_user(buffer, (void __user *)dest, len); + /* Local CPU access retains the existing copy_from_user fast path. */ + rc = iopt_pages_rw_access(pages, 0, buffer, len, 0); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0x5a, len), NULL); + KUNIT_ASSERT_EQ(test, iopt_unmap_iova(&fixture->table, iova, len, &unmapped), 0); + KUNIT_EXPECT_EQ(test, unmapped, len); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), (len - (size < PAGE_SIZE ? 0 : size)) / PAGE_SIZE); + iopt_table_remove_domain(&fixture->table, &fixture->domains[0].domain); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, len - size); + KUNIT_ASSERT_EQ(test, iopt_unmap_iova(&fixture->table, alias, len - size, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOMMUFD %luK: relocated PA, batching, copy, two domains, VA replacement and unpin passed", size / SZ_1K); + } +} + +static void user4k_iommufd_rollback_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + unsigned long addr, iova = SZ_1M, size = SZ_4K; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 3 * size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + rc = vm_munmap(TEST_VA + size, size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, addr, iova, 3 * size), -EFAULT); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA + size, size, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + size); + KUNIT_EXPECT_LT(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA + size, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + size); + fixture->domains[1].fail_after = size; + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), -ENOMEM); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + fixture->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), 0); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 3 * size); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 3 * size); + kunit_release_action(test, test_fragment_iopt_free, fixture); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void user4k_iommufd_granule_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + unsigned long addr, unmapped, cut = SZ_1M + PAGE_SIZE - 1; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + fixture->domains[0].domain.pgsize_bitmap = PAGE_SIZE; + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 3 * PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + /* The device cannot cover a misaligned fragment by widening the mapping. */ + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA + SZ_4K, SZ_1M, PAGE_SIZE), -EINVAL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, SZ_4K), -EINVAL); + /* Full contiguous native spans still work, including legacy area splitting. */ + KUNIT_ASSERT_EQ(test, iopt_disable_large_pages(&fixture->table), 0); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, 2 * PAGE_SIZE), 0); + KUNIT_EXPECT_EQ(test, fixture->domains[0].largest_map, PAGE_SIZE); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 2LL); + KUNIT_ASSERT_EQ(test, iopt_cut_iova(&fixture->table, &cut, 1), 0); + rc = iopt_unmap_iova(&fixture->table, SZ_1M, PAGE_SIZE, &unmapped); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + /* A closed IOVA interval ending at ULONG_MAX must not wrap at map/unmap. */ + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, + ULONG_MAX - 2 * PAGE_SIZE + 1, 2 * PAGE_SIZE), 0); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, unmapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void user4k_iommufd_shared_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct file *file; + struct folio *folio; + unsigned long addr, unmapped, iova = SZ_1M; + unsigned char *buffer; + loff_t pos = 0; + phys_addr_t phys; + void *ptr; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + file = test_cache_file(test, 2 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + buffer = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA, 2 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 2 * SZ_4K), 0); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + folio_lock(folio); + folio_clear_dirty_for_io(folio); + folio_unlock(folio); + phys = iommu_iova_to_phys(&fixture->domains[0].domain, iova); + KUNIT_EXPECT_EQ(test, phys, PFN_PHYS(folio_pfn(folio)) + SZ_4K); + ptr = kmap_local_folio(folio, SZ_4K); + memset(ptr, 0x93, 2 * SZ_4K); + kunmap_local(ptr); + kthread_use_mm(mm); + rc = vm_munmap(TEST_VA, 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, rc, 0); + /* A failed second-domain attachment must retain the first domain's pins. */ + fixture->domains[1].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), -ENOMEM); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 2 * SZ_4K); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(folio)); + fixture->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_ASSERT_EQ(test, kernel_read(file, buffer, PAGE_SIZE, &pos), (ssize_t)PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + SZ_4K, 0x93, 2 * SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 3 * SZ_4K, 0, PAGE_SIZE - 3 * SZ_4K), NULL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void test_iommufd_exit_account(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct iopt_pages *pages; + struct iopt_area *area; + unsigned long addr, unmapped; + LIST_HEAD(keep); + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 2 * PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, 2 * PAGE_SIZE), 0); + down_read(&fixture->table.iova_rwsem); + area = iopt_area_iter_first(&fixture->table, SZ_1M, SZ_1M); + pages = area->pages; + mutex_lock(&pages->mutex); + /* Select VFIO-compatible accounting before the lazy range acquires pins. */ + pages->account_mode = IOPT_PAGES_ACCOUNT_MM; + mutex_unlock(&pages->mutex); + up_read(&fixture->table.iova_rwsem); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 2LL); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 2 * (PAGE_SIZE >> MM_ACCOUNT_SHIFT)); + /* Keep the provider, and thus mm_count, after mm_users reaches zero. */ + KUNIT_ASSERT_EQ(test, iopt_get_pages(&fixture->table, SZ_1M, 2 * PAGE_SIZE, &keep), 0); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, atomic_read(&mm->mm_users), 0); + KUNIT_EXPECT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_EXPECT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, pages->npinned, 0UL); + KUNIT_EXPECT_EQ(test, pages->last_npinned, 0UL); + iopt_free_pages_list(&keep); + kunit_release_action(test, test_fragment_iopt_free, fixture); +} + +static void user4k_iommufd_exit_account_test(struct kunit *test) +{ + test_iommufd_exit_account(test, PAGE_SHIFT); + test_iommufd_exit_account(test, 12); +} + +static void user4k_iova_bitmap_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 4 * size; + unsigned long old = TEST_VA, dest = TEST_VA + 32 * PAGE_SIZE + size; + unsigned long start = dest + size - sizeof(u64), bytes = 2 * size; + unsigned long unit = SZ_4K, iova = 1UL << 32, addr, moved; + unsigned char *actual, *expected; + struct iova_bitmap *bitmap; + unsigned long missed; + int ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + actual = kunit_kzalloc(test, len, GFP_KERNEL); + expected = kunit_kzalloc(test, len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, actual); + KUNIT_ASSERT_NOT_NULL(test, expected); + kthread_use_mm(mm); + addr = vm_mmap(NULL, old, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, actual, len); + moved = missed ? 0 : test_mremap(old, len, len, + MREMAP_FIXED | MREMAP_MAYMOVE, dest); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, old); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_EQ(test, moved, dest); + /* VA and PA offsets differ after moving by one alternative leaf. */ + if (size < PAGE_SIZE) { + pte_t pte = test_fault_entry(test, mm, dest); + + KUNIT_EXPECT_NE(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + dest & ~PAGE_MASK); + } + kthread_use_mm(mm); + bitmap = iova_bitmap_alloc(iova, bytes * 8 * unit, unit, + (u64 __user *)start); + if (!IS_ERR(bitmap)) { + ret = iova_bitmap_for_each(bitmap, &unit, test_iova_bitmap_fill); + iova_bitmap_free(bitmap); + } else { + ret = PTR_ERR(bitmap); + } + missed = copy_from_user(actual, (void __user *)dest, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + memset(expected + size - sizeof(u64), 0xff, bytes); + expected[size - sizeof(u64) + bytes - 1] = 1; + KUNIT_EXPECT_EQ(test, memcmp(actual, expected, len), 0); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap relocated %luK user leaves checked", size / SZ_1K); + } +} + +static void test_iova_free(void *bitmap) +{ + iova_bitmap_free(bitmap); +} + +static void test_iova_kvfree(void *buffer) +{ + kvfree(buffer); +} + +static void user4k_iova_bitmap_window_test(struct kunit *test) +{ + struct mm_struct *mm; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + unsigned long bytes = (capacity + 3) * SZ_4K, len = bytes + 2 * SZ_4K; + unsigned long start = TEST_VA + SZ_4K - sizeof(u64), unit = SZ_4K; + unsigned long addr, missed; + unsigned char *actual; + struct iova_bitmap *bitmap; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + actual = kvzalloc(len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, actual); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_kvfree, actual), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + bitmap = iova_bitmap_alloc(0, bytes * 8 * unit, unit, (u64 __user *)start); + ret = IS_ERR(bitmap) ? PTR_ERR(bitmap) : + iova_bitmap_for_each(bitmap, &unit, test_iova_bitmap_fill); + if (!IS_ERR(bitmap)) + iova_bitmap_free(bitmap); + missed = IS_ERR_VALUE(addr) ? len : copy_from_user(actual, (void __user *)addr, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual, 0, SZ_4K - sizeof(u64)), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual + SZ_4K - sizeof(u64), 0xff, bytes - 1), NULL); + KUNIT_EXPECT_EQ(test, actual[SZ_4K - sizeof(u64) + bytes - 1], (unsigned char)1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual + SZ_4K - sizeof(u64) + bytes, + 0, SZ_4K + sizeof(u64)), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap crossed %lu-record window and revisited first byte", capacity); +} + +static void user4k_iova_bitmap_retained_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 2 * size; + struct file *file = test_cache_file(test, 4 * PAGE_SIZE, false); + unsigned char *buffer = kunit_kzalloc(test, len, GFP_KERNEL); + unsigned long addr, replacement = 0, missed = 0; + struct iova_bitmap *bitmap; + struct folio *folio; + loff_t pos = size; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_NOT_NULL(test, buffer); + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, size); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, + (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) + iova_bitmap_set(bitmap, 3 * SZ_4K, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_free, bitmap), 0); + folio = filemap_get_folio(file->f_mapping, size >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, len); + if (!ret) { + replacement = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + if (!IS_ERR_VALUE(replacement)) { + memset(buffer, 0x5a, len); + missed = copy_to_user((void __user *)replacement, buffer, len); + } + } + /* The cached pins still refer to the old file, not the new VMA. */ + iova_bitmap_set(bitmap, 9 * SZ_4K, 2 * SZ_4K); + if (!ret && !IS_ERR_VALUE(replacement)) + missed |= copy_from_user(buffer, (void __user *)replacement, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, replacement, TEST_VA); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0x5a, len), NULL); + kunit_release_action(test, test_iova_free, bitmap); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_ASSERT_EQ(test, kernel_read(file, buffer, len, &pos), (ssize_t)len); + KUNIT_EXPECT_EQ(test, buffer[0], (unsigned char)8); + KUNIT_EXPECT_EQ(test, buffer[1], (unsigned char)6); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 2, 0, len - 2), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap %luK shared backing retained across unmap/replacement", size / SZ_1K); + } +} + +static long test_mprotect(unsigned long start, unsigned long len, unsigned long prot); + +static void user4k_iova_bitmap_partial_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 3 * size; + unsigned char *buffer = kunit_kzalloc(test, len, GFP_KERNEL); + unsigned long addr, refill, missed; + struct iova_bitmap *bitmap; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, buffer); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, buffer, len); + ret = vm_munmap(TEST_VA + size, size); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + missed |= copy_from_user(buffer, (void __user *)TEST_VA, size); + missed |= copy_from_user(buffer + 2 * size, (void __user *)(TEST_VA + 2 * size), size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_free, bitmap), 0); + /* Preserve the existing void API: pin failure stops at the valid prefix. */ + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0xff, size), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 2 * size, 0, size), NULL); + kthread_use_mm(mm); + refill = vm_mmap(NULL, TEST_VA + size, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + missed = copy_from_user(buffer, (void __user *)TEST_VA, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, refill, TEST_VA + size); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0xff, len), NULL); + kunit_release_action(test, test_iova_free, bitmap); + memset(buffer, 0, len); + kthread_use_mm(mm); + missed = copy_to_user((void __user *)TEST_VA, buffer, len); + ret = test_mprotect(TEST_VA, len, PROT_READ); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) { + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + iova_bitmap_free(bitmap); + } + missed |= copy_from_user(buffer, (void __user *)TEST_VA, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, len), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap %luK partial pin, retry and read-only checks passed", size / SZ_1K); + } +} + +static u64 test_table_descriptor(struct mm_struct *mm, unsigned long addr, bool pud) +{ + pgd_t *pgdp = pgd_offset(mm, addr); + p4d_t *p4dp = p4d_offset_mm(mm, pgdp, addr); + pud_t *pudp = pud_offset_mm(mm, p4dp, addr); + + return pud ? pud_val(*pudp) : pmd_val(*pmd_offset_mm(mm, pudp, addr)); +} + +static void user4k_mremap_move_test(struct kunit *test) +{ + static const struct { unsigned long old, new, len; unsigned int level; } cases[] = { + { TEST_VA + SZ_4K, 3 * SZ_2M + 2 * SZ_4K, 3 * SZ_4K, 0 }, + { SZ_2M - SZ_4K, SZ_1G - 2 * SZ_4K, 3 * SZ_4K, 0 }, + { SZ_1G - SZ_4K, BIT_ULL(39) - 2 * SZ_4K, 3 * SZ_4K, 0 }, + { 4 * SZ_2M, 8 * SZ_2M, SZ_2M, 1 }, + { 2UL * SZ_1G, 4UL * SZ_1G, SZ_1G, 2 }, + }; + unsigned int shape; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + for (shape = 0; shape < ARRAY_SIZE(cases); shape++) { + struct mm_struct *mm = test_vma_mm(test); + unsigned long old = cases[shape].old, new = cases[shape].new; + unsigned long len = cases[shape].len; + unsigned long offsets[] = { 0, SZ_4K, len - SZ_4K }; + unsigned long result, warm, moved, vm_before, commit_before, rss_before; + unsigned long flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long old_left[3], new_left[3]; + phys_addr_t phys[3]; + u8 readback[3], value = 0x79, byte; + u64 table = 0; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (cases[shape].level) + flags |= MAP_NORESERVE; + kthread_use_mm(mm); + warm = vm_mmap(NULL, SZ_1M, SZ_4K, PROT_READ | PROT_WRITE, flags, 0); + result = vm_mmap(NULL, old, len, PROT_READ | PROT_WRITE, flags, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, warm, SZ_1M); + KUNIT_ASSERT_EQ(test, result, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, warm, &value, 1, FOLL_WRITE), 1); + for (i = 0; i < 3; i++) { + value = 0x80 + i; + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + offsets[i], &value, 1, FOLL_WRITE), 1); + phys[i] = pte_phys_mm(mm, test_fault_entry(test, mm, old + offsets[i])); + } + if (cases[shape].level) { + table = test_table_descriptor(mm, old, cases[shape].level == 2); + /* Warm-up consumed the first fragment; the moved table is packed. */ + KUNIT_EXPECT_NE(test, table & (3UL * SZ_4K), 0ULL); + } + vm_before = mm->total_vm; + commit_before = mm->committed_user_pages; + rss_before = get_mm_rss_sum(mm); + kthread_use_mm(mm); + pagefault_disable(); + for (i = 0; i < 3; i++) + old_left[i] = copy_from_user(&byte, (void __user *)(old + offsets[i]), 1); + pagefault_enable(); + moved = test_mremap(old, len, len, MREMAP_FIXED | MREMAP_MAYMOVE, new); + pagefault_disable(); + for (i = 0; i < 3; i++) { + new_left[i] = copy_from_user(&readback[i], (void __user *)(new + offsets[i]), 1); + old_left[i] += !copy_from_user(&byte, (void __user *)(old + offsets[i]), 1); + } + pagefault_enable(); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + KUNIT_EXPECT_EQ(test, mm->total_vm, vm_before); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, commit_before); + KUNIT_EXPECT_EQ(test, get_mm_rss_sum(mm), rss_before); + for (i = 0; i < 3; i++) { + KUNIT_EXPECT_EQ(test, old_left[i], 0UL); + KUNIT_EXPECT_EQ(test, new_left[i], 0UL); + KUNIT_EXPECT_EQ(test, readback[i], (u8)(0x80 + i)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, new + offsets[i])), phys[i]); + } + if (cases[shape].level) + KUNIT_EXPECT_EQ(test, test_table_descriptor(mm, new, cases[shape].level == 2), table); + } +} + +static void user4k_stack_relocate_test(struct kunit *test) +{ + const unsigned long shifts[] = { SZ_4K, SZ_2M + SZ_4K, SZ_1G + 3 * SZ_4K }; + unsigned int shape, i; + + for (shape = 0; shape < ARRAY_SIZE(shifts); shape++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct vm_page_offset pos; + unsigned long top, old, new; + phys_addr_t phys[3]; + u8 bytes[3] = { 0x42, 0x73, 0xb5 }; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, create_init_stack_vma(mm, &vma, &top), 0); + old = vma->vm_end - 3 * SZ_4K; + new = old - shifts[shape]; + mmap_write_lock(mm); + ret = expand_stack_locked(vma, old); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, + &bytes[i], 1, FOLL_WRITE), 1); + phys[i] = pte_phys_mm(mm, test_fault_entry(test, mm, old + i * SZ_4K)); + } + pos = vma_page_offset_at(vma, old); + mmap_write_lock(mm); + ret = relocate_vma_down(vma, shifts[shape]); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, vma->vm_start, new); + KUNIT_EXPECT_EQ(test, vma->vm_end, new + 3 * SZ_4K); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, new), pos)); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, new + i * SZ_4K)), phys[i]); + } + /* Grow after relocation: keep the file-like anon offset linear. */ + mmap_write_lock(mm); + vm_flags_clear(vma, VM_STACK_INCOMPLETE_SETUP); + ret = expand_stack_locked(vma, new - SZ_4K); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 4UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new - SZ_4K, &bytes[0], 1, FOLL_WRITE), 1); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, new), pos)); + } +} + +static void user4k_mremap_cow_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + unsigned long old = TEST_VA + SZ_4K, new = SZ_2M - 2 * SZ_4K, mapped; + unsigned long moved; + u8 value = 0x29, byte; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + parent = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + kthread_use_mm(parent); + mapped = vm_mmap(NULL, old, 3 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(parent); + KUNIT_ASSERT_EQ(test, mapped, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(parent, old + SZ_4K, &value, 1, FOLL_WRITE), 1); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(child); + /* Move only the middle quarter, leaving both neighbors mapped. */ + moved = test_mremap(old + SZ_4K, SZ_4K, SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(child); + KUNIT_ASSERT_EQ(test, moved, new); + value = 0xa4; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, new, &value, 1, FOLL_WRITE), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(parent, old + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x29); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, new, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, value); + KUNIT_EXPECT_EQ(test, mm_pages_to_bytes(child, child->total_vm), 3UL * SZ_4K); + KUNIT_EXPECT_EQ(test, child->committed_user_pages, 3UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, old, &value, 1, 0), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, old + 2 * SZ_4K, &value, 1, 0), 1); +} + +static void user4k_mremap_resize_test(struct kunit *test) +{ + struct mm_struct *mm; + unsigned long old = TEST_VA + SZ_4K, new = TEST_VA + 8 * PAGE_SIZE + 2 * SZ_4K; + unsigned long res[7]; + u8 value = 0x98, byte = 0; + int unmap[2]; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + res[0] = vm_mmap(NULL, old, SZ_4K, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + res[1] = test_mremap(old, SZ_4K, 3 * SZ_4K - 1, 0, 0); + res[2] = test_mremap(old + 1, SZ_4K, SZ_4K, 0, 0); + res[3] = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new + 1); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, res[0], old); + KUNIT_ASSERT_EQ(test, res[1], old); + KUNIT_EXPECT_EQ(test, res[2], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, res[3], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + SZ_4K, &value, 1, FOLL_WRITE), 1); + kthread_use_mm(mm); + res[4] = test_mremap(old, 3 * SZ_4K, 2 * SZ_4K, 0, 0); + res[5] = test_mremap(old, 2 * SZ_4K, 2 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, res[4], old); + KUNIT_ASSERT_EQ(test, res[5], new); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 4UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, value); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, old + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0); + kthread_use_mm(mm); + unmap[0] = vm_munmap(old, 2 * SZ_4K); + unmap[1] = vm_munmap(new, 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, unmap[0], 0); + KUNIT_EXPECT_EQ(test, unmap[1], 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 0UL); +} + +static int test_reject_mremap(struct vm_area_struct *vma) +{ + return -EIO; +} + +static void user4k_cold_batch_test(struct kunit *test) +{ + struct mm_struct *mm, *child; + struct vm_area_struct *vma; + unsigned long start = TEST_VA, mapped; + spinlock_t *ptl; + pte_t *ptep, entry; + u8 byte = 0x68; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm cold fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start, &byte, 1, FOLL_WRITE), 1); + /* Only the last quarter is young: batching must aggregate that bit. */ + mmap_read_lock(mm); + vma = vma_lookup(mm, start); + for (i = 0; i < 3; i++) { + unsigned long addr = start + i * SZ_4K; + + ptep = test_mm_lookup(mm, addr, &ptl); + if (!ptep) { + mmap_read_unlock(mm); + KUNIT_FAIL(test, "missing cold test PTE"); + return; + } + entry = ptep_clear_flush(vma, addr, ptep); + set_pte_at(mm, addr, ptep, pte_mkold(entry)); + pte_unmap_unlock(ptep, ptl); + } + mmap_read_unlock(mm); + kthread_use_mm(mm); + ret = do_madvise(mm, start + 3 * SZ_4K, SZ_4K, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, pte_young(test_fault_entry(test, mm, start + 3 * SZ_4K))); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(mm); + ret = do_madvise(mm, start, PAGE_SIZE, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, pte_young(test_fault_entry(test, mm, start + 3 * SZ_4K))); + kunit_release_action(test, user4k_vma_mm_free, child); + kthread_use_mm(mm); + ret = do_madvise(mm, start, PAGE_SIZE, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_FALSE(test, pte_young(test_fault_entry(test, mm, start + i * SZ_4K))); + byte = 0; + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, start, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x68); +} + +static void user4k_prefault_dirty_test(struct kunit *test) +{ + struct mm_struct *mm; + struct folio *folio; + unsigned long start = TEST_VA + SZ_4K, mapped; + spinlock_t *ptl; + pte_t *ptep, entry; + u8 byte = 0x7a; + int locked = 1; + long ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm prefault fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, SZ_4K, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start, &byte, 1, FOLL_WRITE), 1); + mmap_read_lock(mm); + ptep = test_mm_lookup(mm, start, &ptl); + if (!ptep) { + mmap_read_unlock(mm); + KUNIT_FAIL(test, "missing prefault test PTE"); + return; + } + entry = ptep_clear_flush(vma_lookup(mm, start), start, ptep); + folio = page_folio(pte_page(entry)); + folio_get(folio); + set_pte_at(mm, start, ptep, pte_mkclean(entry)); + folio_clear_dirty(folio); + pte_unmap_unlock(ptep, ptl); + mmap_read_unlock(mm); + kthread_use_mm(mm); + mmap_read_lock(mm); + ret = faultin_page_range(mm, start, start + SZ_4K, true, &locked); + if (locked) + mmap_read_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 1L); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + folio_put(folio); +} + +struct test_fsync_record { + loff_t start, end; + unsigned int calls; + int datasync; +}; + +static int test_fsync_range(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct test_fsync_record *record = file->private_data; + + *record = (struct test_fsync_record) { start, end, record->calls + 1, datasync }; + return 0; +} + +asmlinkage long __arm64_sys_msync(const struct pt_regs *regs); + +static void user4k_msync_offsets_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vma; + struct file_operations ops; + const struct file_operations *original_ops; + void *original_private; + struct test_fsync_record records[3] = {}, record = {}; + struct pt_regs regs = { .regs = { TEST_VA + SZ_4K, 1, MS_SYNC } }; + long result[3]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm sync fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 3 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 5 * SZ_4K, 3 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + original_ops = file->f_op; + original_private = file->private_data; + ops = *original_ops; + ops.fsync = test_fsync_range; + file->f_op = &ops; + file->private_data = &record; + kthread_use_mm(vma->vm_mm); + for (i = 0; i < ARRAY_SIZE(result); i++) { + regs.regs[0] = vma->vm_start + i * SZ_4K; + result[i] = __arm64_sys_msync(®s); + records[i] = record; + } + kthread_unuse_mm(vma->vm_mm); + file->f_op = original_ops; + file->private_data = original_private; + for (i = 0; i < ARRAY_SIZE(result); i++) { + KUNIT_EXPECT_EQ(test, result[i], 0L); + KUNIT_EXPECT_EQ(test, records[i].calls, i + 1); + KUNIT_EXPECT_EQ(test, records[i].start, (loff_t)(3 + i) * SZ_4K); + KUNIT_EXPECT_EQ(test, records[i].end, (loff_t)(4 + i) * SZ_4K - 1); + KUNIT_EXPECT_EQ(test, records[i].datasync, 1); + } +} + +static void user4k_mremap_file_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + struct file *file; + struct vm_operations_struct rejecting_ops; + const struct vm_operations_struct *original_ops; + unsigned long old = TEST_VA + SZ_4K, new = SZ_1G - SZ_4K; + unsigned long mapped, moved; + u8 bytes[3]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(file, old, 3 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, 3 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, old); + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, &bytes[i], 1, 0), 1); + vma = test_vma_lookup(mm, old); + KUNIT_ASSERT_NOT_NULL(test, vma); + original_ops = vma->vm_ops; + rejecting_ops = *original_ops; + rejecting_ops.mremap = test_reject_mremap; + mmap_write_lock(mm); + vma->vm_ops = &rejecting_ops; + mmap_write_unlock(mm); + kthread_use_mm(mm); + moved = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + /* Restore stack-owned callbacks before any assertion can leave this scope. */ + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *part; + + for_each_vma(vmi, part) + if (part->vm_ops == &rejecting_ops) + part->vm_ops = original_ops; + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, moved, (unsigned long)-EIO); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 1); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + kthread_use_mm(mm); + moved = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + vma = test_vma_lookup(mm, new); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 3U * SZ_4K); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + } +} + +static void user4k_exec_arguments_test(struct kunit *test) +{ + unsigned int native; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm stack setup requires hardware opt-in"); + return; + } + for (native = 0; native < 2; native++) { + struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); + struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); + unsigned long size, mask, initial, copied, new_top; + unsigned long a_len = 2 * PAGE_SIZE + SZ_4K + 31, b_len = SZ_4K + 43; + char *a, *b, *buf, *too_long; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, bprm); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + size = mm_page_size(mm); + mask = mm_page_mask(mm); + a_len = min(a_len, 16 * size + 31); + bprm->mm = mm; + KUNIT_ASSERT_EQ(test, create_init_stack_vma(mm, &bprm->vma, &bprm->p), 0); + initial = bprm->p; + bprm->argmin = initial - SZ_2M; + bprm->rlim_stack.rlim_cur = SZ_8M; + bprm->rlim_stack.rlim_max = RLIM_INFINITY; + a = kunit_kmalloc(test, a_len + 1, GFP_KERNEL); + b = kunit_kmalloc(test, b_len + 1, GFP_KERNEL); + buf = kunit_kmalloc(test, a_len + b_len + 7, GFP_KERNEL); + too_long = kunit_kmalloc(test, 32 * size + 1, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, buf); + KUNIT_ASSERT_NOT_NULL(test, too_long); + memset(a, 'A', a_len); + a[a_len] = 0; + memset(b, 'b', b_len); + b[b_len] = 0; + memset(too_long, 'x', 32 * size); + too_long[32 * size] = 0; + KUNIT_EXPECT_EQ(test, copy_string_kernel(too_long, bprm), -E2BIG); + KUNIT_EXPECT_EQ(test, bprm->p, initial); + KUNIT_ASSERT_EQ(test, copy_string_kernel("tail", bprm), 0); + bprm->exec = bprm->p; + KUNIT_ASSERT_EQ(test, copy_string_kernel(a, bprm), 0); + KUNIT_ASSERT_EQ(test, copy_string_kernel(b, bprm), 0); + bprm->argc = 3; + copied = initial - bprm->p; + KUNIT_EXPECT_EQ(test, copied, a_len + b_len + 7); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, 0), (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_MEMEQ(test, buf + b_len + 1, a, a_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + new_top = (bprm->vma->vm_end - SZ_2M - size) & mask; + kthread_use_mm(mm); + ret = setup_arg_pages(bprm, new_top, EXSTACK_DISABLE_X); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Subtract < PAGE_SIZE, then round down to 16 bytes and up to a leaf. */ + KUNIT_EXPECT_LE(test, bprm->vma->vm_end, new_top); + KUNIT_EXPECT_LE(test, new_top - bprm->vma->vm_end, PAGE_SIZE); + KUNIT_EXPECT_EQ(test, bprm->vma->vm_end & ~mask, 0UL); + KUNIT_EXPECT_EQ(test, mm->start_stack, bprm->p); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, 0), (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_MEMEQ(test, buf + b_len + 1, a, a_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + /* Interpreter argument removal scans several user granules. */ + initial = bprm->p; + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + 1); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + a_len + 2); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + copied); + KUNIT_EXPECT_EQ(test, bprm->argc, 0); + KUNIT_EXPECT_EQ(test, bprm->vma_pages, 0UL); + } +} + +static struct vm_area_struct *test_special_vma(struct mm_struct *mm, + const struct vm_special_mapping *sm, unsigned long start, unsigned long len, + vm_flags_t flags) +{ + struct vm_area_struct *vma; + + mmap_write_lock(mm); + vma = _install_special_mapping(mm, start, len, flags, sm); + mmap_write_unlock(mm); + return vma; +} + +static void user4k_special_mapping_test(struct kunit *test) +{ + struct vm_special_mapping *sm = kunit_kzalloc(test, sizeof(*sm), GFP_KERNEL); + struct page **pages = kunit_kcalloc(test, 3, sizeof(*pages), GFP_KERNEL); + void *data[2] = { test_alloc_page(test), test_alloc_page(test) }; + struct mm_struct *small = test_vma_mm(test), *cold = test_vma_mm(test); + struct mm_struct *native = mm_alloc(); + struct vm_area_struct *vma[3]; + unsigned long start = SZ_2M - SZ_4K; + unsigned int i, side; + u8 byte; + + KUNIT_ASSERT_NOT_NULL(test, sm); + KUNIT_ASSERT_NOT_NULL(test, pages); + KUNIT_ASSERT_NOT_NULL(test, data[0]); + KUNIT_ASSERT_NOT_NULL(test, data[1]); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, cold); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + /* Reverse the supplied native pages: folio indices cannot identify them. */ + pages[0] = virt_to_page(data[1]); + pages[1] = virt_to_page(data[0]); + sm->pages = pages; + sm->name = "[user4k-special-test]"; + for (i = 0; i < 2 * TEST_SLOTS; i++) + memset(page_address(pages[i / TEST_SLOTS]) + (i % TEST_SLOTS) * SZ_4K, 0x41 + i, SZ_4K); + vma[0] = test_special_vma(small, sm, start, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma[1] = test_special_vma(cold, sm, start, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma[2] = test_special_vma(native, sm, TEST_VA, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + for (i = 0; i < 3; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma[i]); + for (i = 0; i < 2 * TEST_SLOTS; i++) { + unsigned long addr = start + i * SZ_4K; + + KUNIT_ASSERT_EQ(test, access_remote_vm(small, addr, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0x41 + i)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small, test_fault_entry(test, small, addr)), + page_to_phys(pages[i / TEST_SLOTS]) + (i % TEST_SLOTS) * SZ_4K); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(small, MM_FILEPAGES), 2UL * TEST_SLOTS); + /* Both present-PTE and cold-PTE forced writes copy only their quarter. */ + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? cold : small; + + byte = 0xe1 + side; + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start + 2 * SZ_4K + 7, + &byte, 1, FOLL_WRITE | FOLL_FORCE), 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, start + 2 * SZ_4K + 7, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0xe1 + side)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native, TEST_VA + 2 * SZ_4K + 7, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x43); + } + { + struct user_page_fragment fragment; + long ret = pin_user_fragments_remote(small, start + 5 * SZ_4K + 13, + 7, 0, &fragment, 1); + + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_NOT_NULL(test, fragment.subpage); + KUNIT_EXPECT_EQ(test, fragment.offset, (5 * SZ_4K + 13UL) & ~PAGE_MASK); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment.folio) + fragment.offset, + 0x46, fragment.length), NULL); + release_user_fragments(&fragment, 1, false); + } +} + +static void user4k_insert_range_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + void *data[2] = { test_alloc_page(test), test_alloc_page(test) }; + struct page *pages[2]; + struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long size, start, offset, length, num; + unsigned int i; + int ret, invalid[4]; + + KUNIT_ASSERT_NOT_NULL(test, data[0]); + KUNIT_ASSERT_NOT_NULL(test, data[1]); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + pages[0] = virt_to_page(data[1]); + pages[1] = virt_to_page(data[0]); + size = mm_page_size(mm); + start = pmd_size_mm(mm) - size; + offset = native ? 0 : 3 * SZ_4K; + length = 2 * PAGE_SIZE - offset; + vma = test_vma_add(mm, start, length, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + invalid[0] = vm_insert_pages_range(vma, start + 1, pages, 2, offset, length); + invalid[1] = vm_insert_pages_range(vma, start, pages, 2, offset + 1, length); + invalid[2] = vm_insert_pages_range(vma, start, pages, 2, offset, length + size); + invalid[3] = vm_insert_pages_range(vma, start, pages, ULONG_MAX, 0, length); + ret = vm_insert_pages_range(vma, start, pages, 2, offset, length); + mmap_write_unlock(mm); + for (i = 0; i < ARRAY_SIZE(invalid); i++) + KUNIT_EXPECT_EQ(test, invalid[i], -EINVAL); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < length / size; i++) { + unsigned long source = offset + i * size; + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[source >> PAGE_SHIFT]) + offset_in_page(source)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), length / size); + /* The original native-page-count API also expands all user leaves. */ + start += 4 * PAGE_SIZE; + vma = test_vma_add(mm, start, 2 * PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + num = 2; + mmap_write_lock(mm); + ret = vm_insert_pages(vma, start, pages, &num); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, num, 0UL); + for (i = 0; i < 2 * PAGE_SIZE / size; i++) + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[(i * size) >> PAGE_SHIFT]) + offset_in_page(i * size)); + num = 2; + mmap_write_lock(mm); + ret = vm_insert_pages(vma, start, pages, &num); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EBUSY); + KUNIT_EXPECT_EQ(test, num, 2UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), + (length + 2 * PAGE_SIZE) / size); + start += 4 * PAGE_SIZE; + offset = native ? PAGE_SIZE : 3 * SZ_4K; + length = 2 * PAGE_SIZE - offset; + vma = test_vma_add(mm, start, length, + (struct vm_page_offset) { .index = offset >> PAGE_SHIFT, + .offset = offset_in_page(offset) }, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + ret = vm_map_pages(vma, pages, 2); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < length / size; i++) { + unsigned long source = offset + i * size; + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[source >> PAGE_SHIFT]) + offset_in_page(source)); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(pages[0]), 1); + KUNIT_EXPECT_EQ(test, page_ref_count(pages[1]), 1); + } +} + +static void user4k_remap_pfn_range_test(struct kunit *test) +{ + struct folio *folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 1); + + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (unsigned int native = 0; native < 2; native++) { + struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); + unsigned long size, boundaries[3]; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, mm), 0); + size = mm_page_size(mm); + boundaries[0] = pmd_size_mm(mm); + boundaries[1] = pud_size_mm(mm); + boundaries[2] = pgd_size_mm(mm); + for (unsigned int shape = 0; shape < ARRAY_SIZE(boundaries); shape++) { + unsigned long start = boundaries[shape] - size; + unsigned long length = PAGE_SIZE + size; + struct vm_area_struct *vma; + int ret; + + if (shape && boundaries[shape] == boundaries[shape - 1]) + continue; + vma = test_vma_add(mm, start, length, (struct vm_page_offset) {}, + VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_SHARED | VM_MAYSHARE); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + ret = remap_pfn_range(vma, start, folio_pfn(folio), length, + vma->vm_page_prot); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + for (unsigned long offset = 0; offset < length; offset += size) { + pte_t pte = test_fault_entry(test, mm, start + offset); + + struct follow_pfnmap_args args = { + .vma = vma, .address = start + offset + 31, + }; + phys_addr_t expected = PFN_PHYS(folio_pfn(folio)) + offset + 31; + int ret; + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_special(pte)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), + PFN_PHYS(folio_pfn(folio)) + offset); + + mmap_read_lock(mm); + ret = follow_pfnmap_start(&args); + if (!ret) + follow_pfnmap_end(&args); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, native ? 0 : -EOPNOTSUPP); + args.allow_subpage = true; + mmap_read_lock(mm); + ret = follow_pfnmap_start(&args); + if (!ret) { + KUNIT_EXPECT_EQ(test, args.pfn, (unsigned long)PHYS_PFN(expected)); + KUNIT_EXPECT_EQ(test, args.offset, (unsigned int)offset_in_page(expected)); + KUNIT_EXPECT_EQ(test, args.addr_mask, mm_page_mask(mm)); + KUNIT_EXPECT_TRUE(test, args.special); + follow_pfnmap_end(&args); + } + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + } + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, start - size))); + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, start + length))); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0UL); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + } +} + +#ifdef CONFIG_HAVE_IOREMAP_PROT +static void test_iounmap(void *arg) +{ + iounmap((void __iomem *)arg); +} + +static const struct vm_operations_struct test_iomem_ops = { + .access = generic_access_phys, +}; +#endif + +static void user4k_iomem_access_test(struct kunit *test) +{ +#if defined(CONFIG_HAVE_IOREMAP_PROT) && defined(CONFIG_OF) + struct device_node *node; + struct resource resource; + struct mm_struct *mm; + struct vm_area_struct *vma; + void __iomem *uart; + unsigned long start = SZ_2M + PAGE_SIZE - SZ_4K; + u8 reference[16], buffer[32]; + vm_fault_t fault; + int ret; + + if (!of_machine_is_compatible("linux,dummy-virt")) { + kunit_skip(test, "requires disposable QEMU virt PL011 identification registers"); + return; + } + node = of_find_compatible_node(NULL, NULL, "arm,pl011"); + KUNIT_ASSERT_NOT_NULL(test, node); + ret = of_address_to_resource(node, 0, &resource); + uart = ret ? NULL : of_iomap(node, 0); + of_node_put(node); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_NOT_NULL(test, uart); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iounmap, + (void *)uart), 0); + KUNIT_ASSERT_GE(test, resource_size(&resource), (resource_size_t)SZ_4K); + memcpy_fromio(reference, uart + 0xff0, sizeof(reference)); + KUNIT_ASSERT_NOT_NULL(test, memchr_inv(reference, 0, sizeof(reference))); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = test_vma_add(mm, start, 2 * SZ_4K, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_SHARED | VM_MAYSHARE | VM_IO | VM_PFNMAP); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_ops = &test_iomem_ops; + vma->vm_page_prot = pgprot_device(vm_get_page_prot(vma->vm_flags)); + mmap_read_lock(mm); + fault = vmf_insert_pfn_prot_mkwrite_offset(vma, start, + PHYS_PFN(resource.start), offset_in_page(resource.start), + vma->vm_page_prot, false); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)VM_FAULT_NOPAGE); + memset(buffer, 0x6d, sizeof(buffer)); + mmap_read_lock(mm); + ret = generic_access_phys(vma, start + 0xff0, buffer, sizeof(buffer), 0); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 16); + KUNIT_EXPECT_MEMEQ(test, buffer, reference, sizeof(reference)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 16, 0x6d, 16), NULL); + memset(buffer, 0x5a, sizeof(buffer)); + ret = access_remote_vm(mm, start + 0xff0, buffer, sizeof(buffer), 0); + KUNIT_EXPECT_EQ(test, ret, 16); + KUNIT_EXPECT_MEMEQ(test, buffer, reference, sizeof(reference)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 16, 0x5a, 16), NULL); + /* Writes must fail before touching the read-only device mapping. */ + mmap_read_lock(mm); + ret = generic_access_phys(vma, start + 0xff0, buffer, 1, FOLL_WRITE); + mmap_read_unlock(mm); + KUNIT_EXPECT_LT(test, ret, 0); +#else + kunit_skip(test, "requires OF and iomem access support"); +#endif +} + +static void user4k_pfn_offset_test(struct kunit *test) +{ + void *data = test_alloc_page(test); + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long start = SZ_2M - SZ_4K; + unsigned long pfn; + unsigned int i; + vm_fault_t result[4], invalid[3], write; + + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, mm); + pfn = page_to_pfn(virt_to_page(data)); + vma = test_vma_add(mm, start, PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | VM_SHARED | VM_MAYSHARE | + VM_PFNMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + /* Model a write-notify VMA: initial PTEs are read-only. */ + vma->vm_page_prot = vm_get_page_prot(VM_READ | VM_SHARED); + mmap_read_lock(mm); + for (i = 0; i < 4; i++) + result[i] = vmf_insert_pfn_prot_mkwrite_offset(vma, start + i * SZ_4K, + pfn, (3 - i) * SZ_4K, vma->vm_page_prot, false); + invalid[0] = vmf_insert_pfn_prot_mkwrite_offset(vma, start + 1, pfn, + 0, vma->vm_page_prot, false); + invalid[1] = vmf_insert_pfn_prot_mkwrite_offset(vma, start, pfn, + 1, vma->vm_page_prot, false); + invalid[2] = vmf_insert_pfn_prot_mkwrite_offset(vma, start, pfn, + PAGE_SIZE, vma->vm_page_prot, false); + write = vmf_insert_pfn_prot_mkwrite_offset(vma, start + SZ_4K, pfn, + 2 * SZ_4K, vma->vm_page_prot, true); + mmap_read_unlock(mm); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, mm, start + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, result[i], (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), PFN_PHYS(pfn) + (3 - i) * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_special(entry)); + KUNIT_EXPECT_EQ(test, !!pte_write(entry), i == 1); + } + for (i = 0; i < ARRAY_SIZE(invalid); i++) + KUNIT_EXPECT_EQ(test, invalid[i], (vm_fault_t)VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, write, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 0UL); + KUNIT_EXPECT_EQ(test, page_ref_count(virt_to_page(data)), 1); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(virt_to_page(data)), 1); +} + +static void user16k_insert_shared_test(struct kunit *test) +{ + struct mm_struct *middle, *small, *native; + struct vm_area_struct *vmas[3]; + struct mm_subpage_mapcounts counts; + struct page *page; + void *data; + vm_fault_t result; + unsigned int i; + vm_flags_t flags = VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "mixed16K file insertion requires larger native backing"); + return; + } + data = test_alloc_page(test); + KUNIT_ASSERT_NOT_NULL(test, data); + page = virt_to_page(data); + middle = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, middle); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, middle), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(middle, 14), 0); + small = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, small); + native = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + vmas[0] = test_vma_add(middle, TEST_VA, PAGE_SIZE, (struct vm_page_offset) {}, flags); + vmas[1] = test_vma_add(small, TEST_VA + SZ_4K, SZ_4K, (struct vm_page_offset) {}, flags); + vmas[2] = test_vma_add(native, TEST_VA, PAGE_SIZE, (struct vm_page_offset) {}, flags); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vmas[i]); + vmas[i]->vm_page_prot = vm_get_page_prot(vmas[i]->vm_flags); + } + for (i = 0; i < PAGE_SIZE / SZ_16K; i++) { + unsigned long address = TEST_VA + i * SZ_16K; + + mmap_read_lock(middle); + result = vmf_insert_page_offset(vmas[0], address, page, i * SZ_16K); + mmap_read_unlock(middle); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, pte_phys_mm(middle, test_fault_entry(test, middle, address)), + page_to_phys(page) + i * SZ_16K); + } + mmap_read_lock(small); + result = vmf_insert_page_offset(vmas[1], vmas[1]->vm_start, page, PAGE_SIZE - SZ_4K); + mmap_read_unlock(small); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + mmap_read_lock(native); + result = vmf_insert_page_offset(vmas[2], vmas[2]->vm_start, page, 0); + mmap_read_unlock(native); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 5U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], i == TEST_SLOTS - 1 ? 2U : 1U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 6); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 7); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(middle, MM_FILEPAGES), 4UL); + /* Real mm teardown removes whole16K leaves and preserves both aliases. */ + kunit_release_action(test, user4k_vma_mm_free, middle); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 1U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], i == TEST_SLOTS - 1 ? 1U : 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 2); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 3); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small, test_fault_entry(test, small, vmas[1]->vm_start)), + page_to_phys(page) + PAGE_SIZE - SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(native, test_fault_entry(test, native, vmas[2]->vm_start)), + page_to_phys(page)); + kunit_release_action(test, user4k_vma_mm_free, small); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 1); + kunit_release_action(test, user4k_vma_mm_free, native); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 0); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 1); +} + +static void user4k_insert_fragment_test(struct kunit *test) +{ + void *data = test_alloc_page(test); + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + struct page *page; + vm_fault_t result[4], invalid[4]; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, mm); + page = virt_to_page(data); + vma = test_vma_add(mm, start, PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_read_lock(mm); + for (i = 0; i < 4; i++) + result[i] = vmf_insert_page_offset(vma, start + i * SZ_4K, page, (3 - i) * SZ_4K); + invalid[0] = vmf_insert_page_offset(vma, start + 1, page, 0); + invalid[1] = vmf_insert_page_offset(vma, start, page, 1); + invalid[2] = vmf_insert_page_offset(vma, start, page, PAGE_SIZE); + invalid[3] = vmf_insert_page_offset(vma, vma->vm_end, page, 0); + mmap_read_unlock(mm); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, result[i], (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, invalid[i], (vm_fault_t)VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, start + i * SZ_4K)), + page_to_phys(page) + (3 - i) * SZ_4K); + } + KUNIT_EXPECT_EQ(test, page_ref_count(page), 5); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 4UL); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 1); +} + +#if defined(CONFIG_GENERIC_GETTIMEOFDAY) && defined(CONFIG_VDSO_GETRANDOM) +static void user4k_vdso_mapping_test(struct kunit *test) +{ + struct mm_struct *mm; + struct linux_binprm bprm = {}; + unsigned long code, base, left[8]; + u8 bytes[8], magic[4]; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm vDSO fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + mm->mmap_base = TEST_VA + SZ_4K; + mm_flags_clear(MMF_TOPDOWN, mm); + kthread_use_mm(mm); + ret = arch_setup_additional_pages(&bprm, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + code = (unsigned long)mm->context.vdso; + KUNIT_ASSERT_NE(test, code, 0UL); + base = code - VDSO_NR_PAGES * PAGE_SIZE; + KUNIT_EXPECT_EQ(test, base & ~PAGE_MASK, SZ_4K); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, code, magic, sizeof(magic), 0), (int)sizeof(magic)); + KUNIT_EXPECT_MEMEQ(test, magic, "\177ELF", 4); + /* VVAR forbids remote GUP; ordinary uaccess must still fault it in. */ + kthread_use_mm(mm); + for (i = 0; i < 4; i++) { + left[i] = copy_from_user(&bytes[i], (void __user *)(base + i * SZ_4K), 1); + left[4 + i] = copy_from_user(&bytes[4 + i], + (void __user *)(base + VDSO_RNG_PAGE_OFFSET * PAGE_SIZE + i * SZ_4K), 1); + } + kthread_unuse_mm(mm); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, left[i], 0UL); + KUNIT_EXPECT_EQ(test, left[4 + i], 0UL); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, base + i * SZ_4K)), + page_to_phys(virt_to_page(vdso_k_time_data)) + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, + base + VDSO_RNG_PAGE_OFFSET * PAGE_SIZE + i * SZ_4K)), + page_to_phys(virt_to_page(vdso_k_rng_data)) + i * SZ_4K); + } + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, base, magic, sizeof(magic), 0), 0); +} + +#endif + +static void user4k_remote_access_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA + SZ_4K, 3 * SZ_4K); + struct mm_struct *child = test_vma_mm(test); + struct vm_area_struct *child_vma; + unsigned long start = TEST_VA + SZ_4K; + u8 *buf = kunit_kmalloc(test, 3 * SZ_4K, GFP_KERNEL); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + KUNIT_ASSERT_NOT_NULL(test, buf); + memset(buf, 0x5a, 3 * SZ_4K); + KUNIT_ASSERT_EQ(test, access_remote_vm(vma->vm_mm, start, buf, 3 * SZ_4K, FOLL_WRITE), + 3 * (int)SZ_4K); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Deliberately straddle quarters whose physical/virtual offsets differ. */ + memset(buf, 0x96, SZ_4K + 34); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + SZ_4K - 17, + buf, SZ_4K + 34, FOLL_WRITE), (int)SZ_4K + 34); + memset(buf, 0, 3 * SZ_4K); + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, start, buf, 3 * SZ_4K, 0), + 3 * (int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x5a, 3 * SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start, buf, 3 * SZ_4K, 0), 3 * (int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x5a, SZ_4K - 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + SZ_4K - 17, 0x96, SZ_4K + 34), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 2 * SZ_4K + 17, 0x5a, SZ_4K - 17), NULL); + child_vma = test_vma_lookup(child, start); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + mmap_write_lock(child); + vm_flags_clear(child_vma, VM_WRITE); + mmap_write_unlock(child); + buf[0] = 0x27; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, FOLL_WRITE), 0); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, + FOLL_WRITE | FOLL_FORCE), 1); + buf[0] = 0; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, 0), 1); + KUNIT_EXPECT_EQ(test, buf[0], (u8)0x27); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 3 * SZ_4K - 5, buf, 20, 0), 5); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 3 * SZ_4K, buf, 20, 0), 0); +} + +static void user4k_remote_file_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *small, *native; + u8 buf[34]; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, 2 * SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 3 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x34, 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 17, 0x35, 17), NULL); + memset(buf, 0xa6, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 3 * SZ_4K - 17, + buf, sizeof(buf), FOLL_WRITE), (int)sizeof(buf)); + memset(buf, 0, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 4 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0xa6, sizeof(buf)), NULL); +#ifdef CONFIG_CROSS_MEMORY_ATTACH + { + struct kvec vec = { .iov_base = buf, .iov_len = sizeof(buf) }; + struct iov_iter iter; + + memset(buf, 0xb7, sizeof(buf)); + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, sizeof(buf)); + KUNIT_EXPECT_EQ(test, process_vm_rw_fragments(small->vm_mm, + TEST_VA + 3 * SZ_4K - 17, sizeof(buf), &iter, true), 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + memset(buf, 0, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 4 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0xb7, sizeof(buf)), NULL); + } +#endif + mmap_write_lock(small->vm_mm); + vm_flags_clear(small, VM_WRITE); + mmap_write_unlock(small->vm_mm); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, small->vm_start, buf, 1, + FOLL_WRITE | FOLL_FORCE), 0); +#ifdef CONFIG_CROSS_MEMORY_ATTACH + { + struct kvec vec = { .iov_base = buf, .iov_len = sizeof(buf) }; + struct iov_iter iter; + + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, sizeof(buf)); + KUNIT_EXPECT_EQ(test, process_vm_rw_fragments(small->vm_mm, + small->vm_start, sizeof(buf), &iter, true), -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), sizeof(buf)); + } +#endif +} + +#ifdef CONFIG_CROSS_MEMORY_ATTACH +static void user4k_process_vm_test(struct kunit *test) +{ + /* Exceed the stack batch, cross table boundaries and include a partial tail. */ + unsigned long start = SZ_2M - SZ_4K, len = 21 * SZ_4K; + struct vm_area_struct *vma = test_fault_vma(test, start, len); + struct mm_struct *child = test_vma_mm(test); + struct iov_iter iter; + struct kvec vec; + u8 *buf = kunit_kmalloc(test, len + 17, GFP_KERNEL); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + KUNIT_ASSERT_NOT_NULL(test, buf); + memset(buf, 0x61, len); + vec = (struct kvec) { .iov_base = buf, .iov_len = len }; + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, len); + ret = process_vm_rw_fragments(vma->vm_mm, start, len, &iter, true); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, iov_iter_count(&iter), 0UL); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(buf, 0xc7, len); + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, len - 34); + ret = process_vm_rw_fragments(child, start + 17, len - 34, &iter, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + memset(buf, 0, len); + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, len); + ret = process_vm_rw_fragments(vma->vm_mm, start, len, &iter, false); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x61, len), NULL); + vec.iov_len = len + 17; + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, len + 17); + ret = process_vm_rw_fragments(child, start, len + 17, &iter, false); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 17UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x61, 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 17, 0xc7, len - 34), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + len - 17, 0x61, 17), NULL); + /* A shorter local iterator ends successfully before the remote range. */ + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, 23); + ret = process_vm_rw_fragments(child, start + 13, len, &iter, false); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); +} +#endif + +#ifdef CONFIG_FUTEX +static void test_stop_kthread(void *arg); + +struct test_futex_waiter { + struct mm_struct *mm; + unsigned long address; + struct completion ready; + struct completion done; + int result; +}; + +static int test_futex_wait_run(void *arg) +{ + struct test_futex_waiter *w = arg; + ktime_t deadline = ktime_add_ms(ktime_get(), 10000); + + kthread_use_mm(w->mm); + complete(&w->ready); + w->result = futex_wait((u32 __user *)w->address, FLAGS_SIZE_32 | FLAGS_SHARED, + 0x33333333, &deadline, FUTEX_BITSET_MATCH_ANY); + kthread_unuse_mm(w->mm); + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void user4k_futex_wake_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vmas[2]; + unsigned long addresses[2] = { TEST_VA + SZ_4K + 64, TEST_VA + 2 * SZ_4K + 64 }; + unsigned int round; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm futex fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vmas[0] = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, SZ_4K, 2 * SZ_4K); + vmas[1] = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, vmas[0]); + KUNIT_ASSERT_NOT_NULL(test, vmas[1]); + /* Both directions use the real wait queue and atomic user-value read. */ + for (round = 0; round < 2; round++) { + struct test_futex_waiter *w = kunit_kzalloc(test, sizeof(*w), GFP_KERNEL); + struct task_struct *task; + unsigned long until; + int woke = 0, wrong; + + KUNIT_ASSERT_NOT_NULL(test, w); + w->mm = vmas[round]->vm_mm; + w->address = addresses[round]; + init_completion(&w->ready); + init_completion(&w->done); + task = kthread_run(test_futex_wait_run, w, "user4k-futex-%u", round); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->ready, 5 * HZ), 0UL); + until = jiffies + 5 * HZ; + kthread_use_mm(vmas[round ^ 1]->vm_mm); + wrong = futex_wake((u32 __user *)(addresses[round ^ 1] + 4), + FLAGS_SIZE_32 | FLAGS_SHARED, 1, FUTEX_BITSET_MATCH_ANY); + do { + woke = futex_wake((u32 __user *)addresses[round ^ 1], + FLAGS_SIZE_32 | FLAGS_SHARED, 1, FUTEX_BITSET_MATCH_ANY); + if (woke || completion_done(&w->done)) + break; + schedule_timeout_uninterruptible(1); + } while (time_before(jiffies, until)); + kthread_unuse_mm(vmas[round ^ 1]->vm_mm); + KUNIT_EXPECT_EQ(test, wrong, 0); + KUNIT_EXPECT_EQ(test, woke, 1); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->done, 12 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, w->result, 0); + kunit_release_action(test, test_stop_kthread, task); + } +} + +static void user4k_futex_keys_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *small, *native; + union futex_key keys[2], other; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm futex fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, 2 * SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + for (i = 0; i < 3; i++) { + unsigned long va = small->vm_start + i * SZ_4K + 64; + unsigned long fileoff = (i + 2) * SZ_4K + 64; + + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)va, FLAGS_SIZE_32 | FLAGS_SHARED, + &keys[0], FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(native->vm_mm); + ret = get_futex_key((u32 __user *)(TEST_VA + fileoff), + FLAGS_SIZE_32 | FLAGS_SHARED, &keys[1], FUTEX_READ); + kthread_unuse_mm(native->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, futex_match(&keys[0], &keys[1])); + KUNIT_EXPECT_EQ(test, keys[0].shared.pgoff, fileoff / PAGE_SIZE); + KUNIT_EXPECT_EQ(test, keys[0].both.offset, (unsigned int)(fileoff % PAGE_SIZE) | FUT_OFF_INODE); + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)(va + 4), FLAGS_SIZE_32 | FLAGS_SHARED, + &other, FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_FALSE(test, futex_match(&keys[0], &other)); + } + mmap_write_lock(small->vm_mm); + vm_flags_clear(small, VM_WRITE); + mmap_write_unlock(small->vm_mm); + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)(small->vm_start + 2 * SZ_4K + 64), + FLAGS_SIZE_32 | FLAGS_SHARED, &other, FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, futex_match(&keys[1], &other)); +} +#endif + +static void test_swap_usage_mapping_free(void *arg) +{ + struct address_space *mapping = arg; + + xa_destroy(&mapping->i_pages); +} + +static void user4k_shmem_swap_usage_test(struct kunit *test) +{ + struct address_space *mapping = kunit_kzalloc(test, sizeof(*mapping), GFP_KERNEL); + void *ret; + + KUNIT_ASSERT_NOT_NULL(test, mapping); + xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_swap_usage_mapping_free, mapping), 0); + /* Synthetic swap values exercise accounting without allocating swap slots. */ + ret = xa_store(&mapping->i_pages, 1, xa_mk_value(1), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + if (IS_ENABLED(CONFIG_XARRAY_MULTI)) { + ret = xa_store_range(&mapping->i_pages, 4, 7, + xa_mk_value(2), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + } else { + unsigned long index; + + /* Without multi-index entries, shmem stores one value per page. */ + for (index = 4; index <= 7; index++) { + ret = xa_store(&mapping->i_pages, index, + xa_mk_value(2), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + } + } + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, PAGE_SIZE + SZ_4K, + PAGE_SIZE + 3 * SZ_4K), 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 3 * SZ_4K, + 2 * PAGE_SIZE + SZ_4K), PAGE_SIZE); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 4 * PAGE_SIZE + SZ_4K, + 7 * PAGE_SIZE + 3 * SZ_4K), 3 * PAGE_SIZE + 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 5 * PAGE_SIZE + SZ_4K, + 5 * PAGE_SIZE + 2 * SZ_4K), (unsigned long)SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 0, SZ_4K), 0UL); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, PAGE_SIZE, PAGE_SIZE), 0UL); +} + +static void test_file_mapcounts(struct kunit *test, struct folio *folio, + const unsigned int expected[TEST_SLOTS], unsigned int native) +{ + struct mm_subpage_mapcounts counts; + unsigned int i, total = 0; + + for (i = 0; i < TEST_SLOTS; i++) + total += expected[i]; + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(&folio->page, &counts), total); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], expected[i]); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(total + native)); +} + +static void user4k_file_mapcounts_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *shared, *private, *native, *cvma; + struct mm_struct *child; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA, 3 * SZ_4K, SZ_4K); + private = test_file_vma(test, file, false, false, TEST_VA, 3 * SZ_4K, SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 2, 2, 2 }, 1); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA + 2 * SZ_4K, true) & + VM_FAULT_ERROR, 0U); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 2, 2, 1 }, 1); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, private->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 3, 3, 1 }, 1); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 3, 2, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, child); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 2, 2, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, private->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 1, 1, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 1, 1, 1 }, 0); + kunit_release_action(test, user4k_vma_mm_free, shared->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 0, 0, 0 }, 0); +} + +/* A unique RAM-backed test inode: exercise generic file faults without + * making unsupported filesystem promotion part of this mapping fixture. + */ +static int test_coarse_read_folio(struct file *file, struct folio *folio) +{ + unsigned long i; + loff_t base = (loff_t)folio->index << PAGE_SHIFT; + loff_t size = i_size_read(file_inode(file)); + u8 *data = folio_address(folio); + + for (i = 0; i < folio_size(folio); i++) + data[i] = base + i < size ? 0x31 + ((base + i) / SZ_4K) : 0; + __folio_mark_uptodate(folio); + folio_unlock(folio); + return 0; +} + +static struct file *test_coarse_cache_file(struct kunit *test, unsigned int order, + unsigned long size) +{ + static const struct address_space_operations aops = { + .read_folio = test_coarse_read_folio, + .dirty_folio = filemap_dirty_folio, + }; + static const struct file_operations fops = { + .mmap = generic_file_mmap, + .read_iter = generic_file_read_iter, + .llseek = generic_file_llseek, + }; + struct file *file = anon_inode_create_getfile("user-coarse-file", &fops, + NULL, O_RDWR, NULL); + struct folio *folio; + int ret; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, test_file_put, file)) + return ERR_PTR(-ENOMEM); + file->f_mapping->a_ops = &aops; + mapping_set_large_folios(file->f_mapping); + i_size_write(file_inode(file), size); + folio = folio_alloc(GFP_KERNEL, order); + if (!folio) + return ERR_PTR(-ENOMEM); + ret = filemap_add_folio(file->f_mapping, folio, 0, GFP_KERNEL); + if (!ret) + test_coarse_read_folio(file, folio); /* Unlocks the installed folio. */ + folio_put(folio); + return ret ? ERR_PTR(ret) : file; +} + +static bool test_coarse_file_enabled(struct kunit *test) +{ + if (PAGE_SHIFT >= 16 || + (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { + kunit_skip(test, "requires internal larger user granule and borrowed-mm opt-in"); + return false; + } + return true; +} + +static void test_promote_unpin(void *data) +{ + struct user_page_fragment *fragment = data; + + if (fragment->folio) { + release_user_fragments(fragment, 1, true); + fragment->folio = NULL; + } +} + +static void user64k_file_promote_test(struct kunit *test) +{ + unsigned int pinned; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "cache promotion currently uses the collapse implementation"); + return; + } + for (pinned = 0; pinned < 2; pinned++) { + struct file *file = test_cache_file(test, 2 * SZ_64K, true); + struct vm_area_struct *coarse, *small; + struct user_page_fragment *fragment; + struct folio *folio; + phys_addr_t old[SZ_64K / PAGE_SIZE], untouched; + unsigned int nr = SZ_64K / PAGE_SIZE, i; + unsigned long offset, modified = SZ_64K / 2 + 19; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, SZ_64K); + small = test_file_vma(test, file, false, true, TEST_VA, SZ_64K, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, small); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + untouched = page_to_phys(&folio->page); + folio_put(folio); + for (i = 0; i < nr; i++) { + folio = filemap_get_folio(file->f_mapping, nr + i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + old[i] = page_to_phys(&folio->page); + folio_put(folio); + } + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + if (pinned) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_promote_unpin, + fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + TEST_VA + modified, 1, FOLL_WRITE, fragment, 1), 1L); + *(u8 *)(folio_address(fragment->folio) + fragment->offset) = 0xa7; + KUNIT_EXPECT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), -EBUSY); + /* A pin in the middle forces rollback of earlier isolated folios. */ + for (i = 0; i < nr; i++) { + folio = filemap_get_folio(file->f_mapping, nr + i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, page_to_phys(&folio->page), old[i]); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), (u8)0xa7); + kunit_release_action(test, test_promote_unpin, fragment); + } + KUNIT_ASSERT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), 0); + folio = filemap_get_folio(file->f_mapping, nr); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_order(folio), 16U - PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, folio->index, (pgoff_t)nr); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, (unsigned long)2 * nr); + for (offset = 0; offset < SZ_64K; offset++) { + u8 expected = pinned && offset == modified ? 0xa7 : 0x41 + offset / SZ_4K; + + if (((u8 *)folio_address(folio))[offset] != expected) { + KUNIT_FAIL(test, "promoted data mismatch at %lu", offset); + break; + } + } + /* An already suitable folio is an idempotent success, including with a GET. */ + KUNIT_EXPECT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), 0); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, coarse->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse->vm_mm, entry), page_to_phys(&folio->page)); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) { + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(&folio->page) + offset); + } + { + struct folio *first = filemap_get_folio(file->f_mapping, 0); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, first); + KUNIT_EXPECT_EQ(test, page_to_phys(&first->page), untouched); + folio_put(first); + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); + } +} + +static void user64k_shmem_fault_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 4; mode++) { + bool seeded = mode & 1, shared = mode < 2; + struct file *file = test_cache_file(test, SZ_64K, seeded); + struct vm_area_struct *vma; + struct folio *cache, *mapped; + unsigned long offset; + pte_t entry; + vm_fault_t fault; + unsigned int attempt; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, shared, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (attempt = 0; attempt < 8; attempt++) { + fault = test_file_fault(vma, TEST_VA, true); + if (!(fault & VM_FAULT_NOPAGE)) + break; + } + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + mapped = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, entry)))); + KUNIT_EXPECT_EQ(test, folio_size(mapped), SZ_64K); + KUNIT_EXPECT_EQ(test, !!folio_test_anon(mapped), !shared); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mapped) + offset, + seeded ? 0x31 + offset / SZ_4K : 0, SZ_4K), NULL); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, cache), 0); + KUNIT_EXPECT_EQ(test, folio_size(cache), SZ_64K); + if (shared) + KUNIT_EXPECT_PTR_EQ(test, mapped, cache); + else { + KUNIT_EXPECT_PTR_NE(test, mapped, cache); + memset(folio_address(mapped), 0xc9, SZ_64K); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + offset, + seeded ? 0x31 + offset / SZ_4K : 0, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, + shared ? MM_SHMEMPAGES : MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, SZ_64K / PAGE_SIZE); + { + u8 byte = 0x83, observed = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, TEST_VA + SZ_64K - 1, + &byte, 1, FOLL_WRITE), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, TEST_VA + SZ_64K - 1, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + KUNIT_EXPECT_EQ(test, ((u8 *)folio_address(cache))[SZ_64K - 1], + shared ? byte : (u8)(seeded ? 0x40 : 0)); + } + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(cache), (int)(SZ_64K / PAGE_SIZE) + 1); + } +} + +static void user64k_shmem_fault_busy_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 3; mode++) { + struct file *file = test_cache_file(test, SZ_64K, mode != 0); + struct vm_area_struct *vma; + struct folio *held; + vm_fault_t fault; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!mode) { + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_REMOTE | + FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT, NULL); + mmap_assert_locked(vma->vm_mm); + mmap_read_unlock(vma->vm_mm); + KUNIT_EXPECT_EQ(test, fault, (vm_fault_t)VM_FAULT_RETRY); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, 0UL); + } else if (mode == 1) { + held = filemap_get_folio(file->f_mapping, SZ_64K / (2 * PAGE_SIZE)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, held); + folio_lock(held); + fault = test_file_fault(vma, TEST_VA, false); + folio_unlock(held); + folio_put(held); + KUNIT_EXPECT_EQ(test, fault, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SHMEMPAGES), 0L); + } else { + struct vm_area_struct *small = test_file_vma(test, file, false, true, + TEST_VA, SZ_64K, 0); + struct user_page_fragment *fragment; + + KUNIT_ASSERT_NOT_NULL(test, small); + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_promote_unpin, + fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + TEST_VA + SZ_64K / 2, 1, FOLL_WRITE, fragment, 1), 1L); + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), (u8)0x39); + kunit_release_action(test, test_promote_unpin, fragment); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + } + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } +} + +static void user64k_shmem_fault_hole_test(struct kunit *test) +{ +#ifdef CONFIG_USERFAULTFD + unsigned int adjacent; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (adjacent = 0; adjacent < 2; adjacent++) { + struct file *file = test_cache_file(test, 2 * SZ_64K, false); + struct vm_area_struct *coarse, *missing, *seed; + struct folio *folio; + vm_fault_t fault; + bool new; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + missing = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, + adjacent ? SZ_64K : PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, missing); + seed = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, 0); + KUNIT_ASSERT_NOT_NULL(test, seed); + mmap_read_lock(seed->vm_mm); + folio = shmem_uffd_prepare_folio(seed, TEST_VA, &new); + mmap_read_unlock(seed->vm_mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, new); + memset(folio_address(folio), 0x6f, SZ_4K); + shmem_uffd_mark_present(folio, 0, BIT(0)); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), + GENMASK(PAGE_SIZE / SZ_4K - 1, 1)); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + /* Model the registration flag used by the collapse interval check. */ + mmap_write_lock(missing->vm_mm); + vma_start_write(missing); + vm_flags_set(missing, VM_UFFD_MISSING); + mmap_write_unlock(missing->vm_mm); + fault = test_file_fault(coarse, TEST_VA, false); + mmap_write_lock(missing->vm_mm); + vma_start_write(missing); + vm_flags_clear(missing, VM_UFFD_MISSING); + mmap_write_unlock(missing->vm_mm); + /* A real coarse fault may populate holes covered by its entire leaf. */ + KUNIT_EXPECT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 1, 0, SZ_4K)); + folio_unlock(folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x6f, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, + SZ_64K - SZ_4K), NULL); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, missing->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, seed->vm_mm); + } +#else + kunit_skip(test, "requires userfaultfd registration flags"); +#endif +} + +static void user64k_file_leaf_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *coarse, *native, *small, *child_vma; + struct mm_struct *coarse_mm, *child; + struct folio *folio; + unsigned long offset; + unsigned int nr = (2 * SZ_64K) / PAGE_SIZE; + unsigned int small_nr = 2 * SZ_64K / SZ_4K - 1; + unsigned int maps = 2 * nr + small_nr; + u8 byte = 0xc7, observed; + int ret; + + if (!test_coarse_file_enabled(test)) + return; + file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, 2 * SZ_64K, 0); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * SZ_64K, 0); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 2 * SZ_64K - SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + coarse_mm = coarse->vm_mm; + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + /* One cache folio supplies two user leaves, with native references per leaf. */ + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_64K) { + pte_t entry = test_fault_entry(test, coarse_mm, TEST_VA + offset); + + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse_mm, entry), + page_to_phys(&folio->page) + offset); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 2L); + for (offset = 0; offset < 2 * SZ_64K; offset += PAGE_SIZE) + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + for (offset = SZ_4K; offset < 2 * SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + KUNIT_ASSERT_EQ(test, access_remote_vm(coarse_mm, TEST_VA + SZ_64K + 3 * SZ_4K, + &byte, 1, FOLL_WRITE), 1); + KUNIT_ASSERT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + SZ_64K + 3 * SZ_4K, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + byte = 0x98; + KUNIT_ASSERT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 7 * SZ_4K, + &byte, 1, FOLL_WRITE), 1); + KUNIT_ASSERT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 7 * SZ_4K, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + KUNIT_EXPECT_EQ(test, ((u8 *)folio_address(folio))[7 * SZ_4K], byte); + + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, coarse_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Ordinary file VMAs are lazily populated in the child after fork. */ + mmap_read_lock(child); + child_vma = find_vma(child, TEST_VA); + mmap_read_unlock(child); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + maps += nr; + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + kthread_use_mm(coarse_mm); + ret = vm_munmap(TEST_VA, SZ_64K); + kthread_unuse_mm(coarse_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + maps -= SZ_64K / PAGE_SIZE; + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 1L); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, child, TEST_VA))); + + /* The native-rounded range must remove every overlapping coarse alias. */ + unmap_mapping_range(file->f_mapping, SZ_64K + SZ_4K, SZ_4K, 0); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, coarse_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, TEST_VA + SZ_64K + SZ_4K))); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + SZ_64K + PAGE_SIZE))); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 0L); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), + (int)(maps - 2 * SZ_64K / PAGE_SIZE - 1 - PAGE_SIZE / SZ_4K)); + + /* An exact subpage hole keeps neighboring small leaves in that native page. */ + mmap_read_lock(coarse_mm); + coarse = find_vma(coarse_mm, TEST_VA + SZ_64K); + mmap_read_unlock(coarse_mm); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < PAGE_SIZE; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + SZ_64K + offset, + false) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + unmap_mapping_subpage_hole(file->f_mapping, SZ_64K + SZ_4K, SZ_64K + 2 * SZ_4K); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, coarse_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA + SZ_64K))); + for (offset = 0; offset < PAGE_SIZE; offset += SZ_4K) + KUNIT_EXPECT_EQ(test, !!pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + SZ_64K + offset)), offset != SZ_4K); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(maps - 2 * SZ_64K / PAGE_SIZE - 2)); + kunit_release_action(test, user4k_vma_mm_free, coarse_mm); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); +} + +static void user64k_file_cow_test(struct kunit *test) +{ + unsigned int cold; + + if (!test_coarse_file_enabled(test)) + return; + for (cold = 0; cold < 2; cold++) { + struct file *file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + struct vm_area_struct *vma; + struct folio *cache, *anon; + unsigned long offset; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, false, TEST_VA, SZ_64K, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!cold) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + anon = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, entry)))); + KUNIT_EXPECT_TRUE(test, folio_test_anon(anon)); + KUNIT_EXPECT_EQ(test, folio_size(anon), SZ_64K); + folio_get(anon); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, anon), 0); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(anon) + offset, + 0x41 + offset / SZ_4K, SZ_4K), NULL); + memset(folio_address(anon), 0x93, SZ_64K); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + offset, + 0x31 + offset / SZ_4K, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + folio_put(cache); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_FILEPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(anon), (int)(SZ_64K / PAGE_SIZE)); + KUNIT_EXPECT_EQ(test, folio_ref_count(anon), (int)(SZ_64K / PAGE_SIZE) + 1); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(anon), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(anon), 1); + } +} + +static void user64k_file_bounds_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int mode; + unsigned long size = SZ_64K + PAGE_SIZE + 19; + pte_t entry; + + if (!test_coarse_file_enabled(test)) + return; + /* Too-small backing and a physically unaligned source cannot form a leaf. */ + for (mode = 0; mode < 2; mode++) { + file = test_coarse_cache_file(test, mode ? 17 - PAGE_SHIFT : 0, 2 * SZ_64K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, false, TEST_VA, SZ_64K, + mode ? PAGE_SIZE : 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } + file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, size); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, 3 * SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K, false) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_64K); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)) + + PAGE_SIZE + 19, 0, SZ_64K - PAGE_SIZE - 19), NULL); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA + 2 * SZ_64K, false) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_FILEPAGES), 2L); +} + +static void user4k_file_shared_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 4 * PAGE_SIZE, true); + struct vm_area_struct *small, *native; + struct folio *folio; + unsigned int i; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + native = test_file_vma(test, file, true, true, TEST_VA, 4 * PAGE_SIZE, 0); + /* Virtual and file offsets deliberately have different native alignment. */ + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 3 * PAGE_SIZE, 2 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + SZ_4K, false) & + VM_FAULT_ERROR, 0U); + /* Fault-around keeps its 64K window and populates all ready cache folios. */ + for (i = 0; i < 12; i++) { + unsigned long addr = small->vm_start + i * SZ_4K; + struct vm_page_offset pos = vma_page_offset_at(small, addr); + + entry = test_fault_entry(test, small->vm_mm, addr); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + folio = filemap_get_folio(file->f_mapping, pos.index); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(folio_file_page(folio, pos.index)) + pos.offset); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(small->vm_mm, entry)), + (u8)(0x33 + i)); + folio_put(folio); + } + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, native->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + memset(phys_to_virt(pte_phys_mm(native->vm_mm, entry)) + 2 * SZ_4K, 0x72, SZ_4K); + entry = test_fault_entry(test, small->vm_mm, small->vm_start); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(small->vm_mm, entry)), + 0x72, SZ_4K), NULL); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + SZ_4K, true) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, small->vm_start + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + memset(phys_to_virt(pte_phys_mm(small->vm_mm, entry)), 0x84, SZ_4K); + entry = test_fault_entry(test, native->vm_mm, TEST_VA); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(native->vm_mm, entry)) + + 3 * SZ_4K, 0x84, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + folio_put(folio); +} + +static void user4k_file_cow_test(struct kunit *test) +{ + unsigned int cold; + + for (cold = 0; cold < 2; cold++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *vma; + struct folio *cache; + pte_t entry; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, false, TEST_VA + SZ_4K, + 3 * SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!cold) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start + SZ_4K, true) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, vma->vm_start + SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + KUNIT_EXPECT_TRUE(test, folio_test_anon(page_folio(pte_page(entry)))); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, entry) & ~PAGE_MASK, 2UL * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), + 0x33, SZ_4K), NULL); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), 0x91, SZ_4K); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + folio_put(cache); + } +} + +static void user4k_file_eof_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, SZ_4K + 37, false); + struct vm_area_struct *small, *native; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, 0U); + for (i = 0; i < 4; i++) { + vm_fault_t ret = test_file_fault(small, TEST_VA + i * SZ_4K, false); + + KUNIT_EXPECT_EQ(test, ret & VM_FAULT_ERROR, i < 2 ? 0U : VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + i * SZ_4K)), i < 2); + } +} + + +static const struct vm_operations_struct test_filemap_fault_ops = { + .fault = filemap_fault, + .page_mkwrite = filemap_page_mkwrite, +}; + +static void user4k_filemap_fault_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * SZ_4K + 17, true); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + 3 * SZ_4K, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Exercise generic filemap_fault against an already populated cache. */ + vma->vm_ops = &test_filemap_fault_ops; + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start + SZ_4K, true) & VM_FAULT_ERROR, 0U); + for (i = 0; i < 3; i++) { + pte_t pte = test_fault_entry(test, vma->vm_mm, vma->vm_start + i * SZ_4K); + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), (u8)(0x31 + i)); + } + KUNIT_EXPECT_EQ(test, test_file_fault(vma, vma->vm_start + 3 * SZ_4K, false) & + VM_FAULT_ERROR, VM_FAULT_SIGBUS); +} + +static void user4k_file_truncate_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *small, *cow, *native; + struct iattr attr = { .ia_valid = ATTR_SIZE, .ia_size = SZ_4K + 37 }; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + PAGE_SIZE, SZ_4K); + cow = test_file_vma(test, file, false, false, TEST_VA, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, false, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, cow); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(cow, TEST_VA + 3 * SZ_4K, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + inode_lock(file_inode(file)); + ret = notify_change(&nop_mnt_idmap, file->f_path.dentry, &attr, NULL); + inode_unlock(file_inode(file)); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) { + unsigned long addr = small->vm_start + i * SZ_4K; + + if (i) + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, addr))); + KUNIT_EXPECT_EQ(test, test_file_fault(small, addr, false) & VM_FAULT_ERROR, + i ? VM_FAULT_SIGBUS : 0U); + } + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, cow->vm_mm, TEST_VA + 3 * SZ_4K))); + KUNIT_EXPECT_EQ(test, test_file_fault(cow, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, + VM_FAULT_SIGBUS); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA))); + KUNIT_EXPECT_EQ(test, test_file_fault(native, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, 0U); +} + +static void user4k_file_invalidate_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 4 * SZ_4K, PAGE_SIZE - SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & VM_FAULT_ERROR, 0U); + unmap_mapping_pages(file->f_mapping, 0, 1, false); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, vma->vm_mm, + vma->vm_start + i * SZ_4K)), i != 0); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & VM_FAULT_ERROR, 0U); + unmap_mapping_pages(file->f_mapping, 1, 1, false); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, vma->vm_mm, + vma->vm_start + i * SZ_4K)), i == 0); +} + +static void user4k_file_fork_hole_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *shared, *private, *cvma; + struct mm_struct *child; + struct inode *inode; + unsigned int i; + pte_t copied; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA, 2 * PAGE_SIZE, 0); + private = test_file_vma(test, file, false, false, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA, false) & VM_FAULT_ERROR, 0U); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, private->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + mmap_read_lock(child); + cvma = find_vma(child, TEST_VA); + mmap_read_unlock(child); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + copied = test_fault_entry(test, child, TEST_VA + SZ_4K); + memset(phys_to_virt(pte_phys_mm(child, copied)), 0x82, SZ_4K); + inode = file_inode(file); + /* A sub-native-page hole zeros shared bytes but retains private COW. */ + ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, SZ_4K, SZ_4K); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, i_size_read(inode), (loff_t)(2 * PAGE_SIZE)); + for (i = 0; i < 2 * TEST_SLOTS; i++) { + pte_t pte; + + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA + i * SZ_4K, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, shared->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(shared->vm_mm, pte)), + i == 1 ? 0 : 0x31 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, test_fault_entry(test, child, TEST_VA + SZ_4K)), + pte_phys_mm(child, copied)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(child, copied)), 0x82, SZ_4K), NULL); + /* A full cache-page hole exercises the file interval-tree invalidation. */ + ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, PAGE_SIZE, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = TEST_SLOTS; i < 2 * TEST_SLOTS; i++) { + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, shared->vm_mm, TEST_VA + i * SZ_4K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + i * SZ_4K))); + } +} + +static void test_zero_cow_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, TEST_VA, PAGE_SIZE); + struct mm_struct *mm; + struct mm_subpage *slot; + pte_t before[4], after; + unsigned int i; + vm_fault_t ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + for (i = 0; i < 4; i++) { + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + i * size, + FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + before[i] = test_fault_entry(test, mm, TEST_VA + i * size); + KUNIT_ASSERT_TRUE(test, pte_present(before[i])); + KUNIT_ASSERT_TRUE(test, is_zero_pfn(pte_pfn(before[i]))); + KUNIT_ASSERT_FALSE(test, pte_write(before[i])); + } + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + size + 7, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + lru_add_drain_all(); + for (i = 0; i < 4; i++) { + after = test_fault_entry(test, mm, TEST_VA + i * size); + if (i != 1) { + KUNIT_EXPECT_TRUE(test, pte_same(before[i], after)); + continue; + } + KUNIT_ASSERT_TRUE(test, pte_write(after)); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)size); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + size, 0, size), NULL); + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1UL); +} + +static void user4k_zero_cow_test(struct kunit *test) +{ + test_zero_cow_granule(test, 12); +} + +static void user16k_zero_cow_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_zero_cow_granule(test, 14); +} + +static void test_cow_reuse_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned int mode; + + for (mode = 0; mode < 2; mode++) { + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, TEST_VA, PAGE_SIZE); + struct mm_subpage *held = NULL; + struct mm_struct *child; + phys_addr_t original[4]; + unsigned int i; + vm_fault_t fault; + int ret; + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * size); + original[i] = pte_phys_mm(vma->vm_mm, pte); + memset(phys_to_virt(original[i]), 0xb0 + i, size); + } + if (mode) { + held = mm_subpage_get_from_phys(original[2]); + KUNIT_ASSERT_NOT_NULL(test, held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + } + child = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + for (i = 0; i < 4; i++) { + unsigned long address = TEST_VA + i * size; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte) == original[i], + !(mode && i == 2)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), + 0xb0 + i, size), NULL); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), 0xd0 + i, size); + } + if (held) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(mm_subpage_phys(held)), + 0xb2, size), NULL); + } +} + +static void user4k_cow_reuse_test(struct kunit *test) +{ + test_cow_reuse_granule(test, 12); +} + +static void user16k_cow_reuse_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_cow_reuse_granule(test, 14); +} + +static void test_fork_cow_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + struct vm_area_struct *parent = test_fault_vma_granule(test, shift, TEST_VA, 4 * size); + struct mm_struct *child = test_vma_mm_granule(test, shift); + struct vm_area_struct *cvma; + struct mm_subpage *held[4]; + pte_t before[4], after; + unsigned int i, side; + vm_fault_t fault; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + before[i] = test_fault_entry(test, parent->vm_mm, TEST_VA + i * size); + held[i] = mm_subpage_get_from_phys(pte_phys_mm(parent->vm_mm, before[i])); + KUNIT_ASSERT_NOT_NULL(test, held[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held[i]), 0); + memset(folio_address(mm_subpage_folio(held[i])) + mm_subpage_offset(held[i]), + 0x31 + i, size); + } + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + size + 15, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + for (i = 0; i < 4; i++) { + struct mm_subpage *slot; + bool changed = side && i == 1; + + after = test_fault_entry(test, mm, TEST_VA + i * size); + KUNIT_EXPECT_EQ(test, !!pte_write(after), changed); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after) != mm_subpage_phys(held[i]), changed); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)size); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0x31 + i, size), NULL); + if (changed) { + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); + memset(folio_address(mm_subpage_folio(slot)) + size, 0x77, size); + } + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 4UL); + } + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(held[i]), i == 1 ? 1 : 2); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(held[i])) + + mm_subpage_offset(held[i]), 0x31 + i, size), NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(held[0])), 11); +} + +static void user4k_fork_cow_test(struct kunit *test) +{ + test_fork_cow_granule(test, 12); +} + +static void user16k_fork_cow_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_fork_cow_granule(test, 14); +} + +static void user4k_folio_walk_geometry_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, PAGE_SHIFT, 14 }; + unsigned int which; + + for (which = 0; which < ARRAY_SIZE(shifts); which++) { + unsigned int shift = shifts[which], leaf; + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned long start, size = 1UL << shift; + vm_fault_t fault; + + if (which == 2 && PAGE_SHIFT <= 14) + continue; + mm = shift == PAGE_SHIFT ? mm_alloc() : test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shift == PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + start = pmd_size_mm(mm) - size; + vma = test_fault_vma_in_mm(mm, start, 2 * size); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (leaf = 0; leaf < 2; leaf++) { + unsigned long address = start + leaf * size; + struct folio_walk fw; + struct folio *folio; + struct page *page = NULL; + pte_t expected, walked = __pte(0); + bool zero = false; + + mmap_read_lock(mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address, 0); + if (folio) + folio_walk_end(&fw, vma); + mmap_read_unlock(mm); + KUNIT_EXPECT_PTR_EQ(test, folio, NULL); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address, FW_ZEROPAGE); + if (folio) { + zero = is_zero_folio(folio) && !fw.page && fw.level == FW_LEVEL_PTE; + folio_walk_end(&fw, vma); + } + mmap_read_unlock(mm); + KUNIT_EXPECT_TRUE(test, zero); + + mmap_read_lock(mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + expected = test_fault_entry(test, mm, address); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address + size - 1, 0); + if (folio) { + page = fw.page; + walked = fw.pte; + folio_walk_end(&fw, vma); + } + mmap_read_unlock(mm); + KUNIT_EXPECT_PTR_EQ(test, page, pfn_to_page(pte_pfn(expected))); + KUNIT_EXPECT_TRUE(test, pte_same(walked, expected)); + KUNIT_EXPECT_PTR_EQ(test, folio, page_folio(pfn_to_page(pte_pfn(expected)))); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_fragment_locked_pin_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, PAGE_SHIFT, 14 }; + unsigned int which; + + for (which = 0; which < ARRAY_SIZE(shifts); which++) { + unsigned int shift = shifts[which]; + struct mm_struct *mm; + struct vm_area_struct *vma, *found; + struct user_page_fragment fragment = { }; + unsigned long address; + int ret; + + if (which == 2 && PAGE_SHIFT <= 14) + continue; + mm = shift == PAGE_SHIFT ? mm_alloc() : test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shift == PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + vma = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + address = TEST_VA + (1UL << shift) - sizeof(long); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_LONGTERM, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOWAIT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + /* Only the first fragment is returned, while retaining the VMA lock. */ + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, 2 * sizeof(long), + FOLL_WRITE, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 1); + if (ret != 1) + continue; + KUNIT_EXPECT_PTR_EQ(test, found, vma); + KUNIT_EXPECT_TRUE(test, fragment.pinned); + KUNIT_EXPECT_EQ(test, fragment.length, (unsigned int)sizeof(long)); + KUNIT_EXPECT_EQ(test, !!fragment.subpage, shift != PAGE_SHIFT); + release_user_fragments(&fragment, 1, false); + /* With backing present, NOFAULT pins the exact same leaf successfully. */ + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 1); + if (ret == 1) { + KUNIT_EXPECT_PTR_EQ(test, found, vma); + release_user_fragments(&fragment, 1, true); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user16k_fragment_offset_test(struct kunit *test) +{ + struct vm_area_struct *vma; + struct user_page_fragment fragment = {}; + unsigned long address = TEST_VA + 3 * SZ_16K + SZ_4K + 13; + long ret; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative fragments require larger native backing"); + return; + } + vma = test_fault_vma_granule(test, 14, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + ret = pin_user_fragments_remote(vma->vm_mm, address, 31, FOLL_WRITE, &fragment, 1); + KUNIT_EXPECT_EQ(test, ret, 1L); + if (ret != 1) + return; + KUNIT_EXPECT_NOT_NULL(test, fragment.subpage); + KUNIT_EXPECT_EQ(test, fragment.offset, 3UL * SZ_16K + SZ_4K + 13); + KUNIT_EXPECT_EQ(test, fragment.length, 31U); + if (fragment.subpage) + KUNIT_EXPECT_EQ(test, mm_subpage_shift(fragment.subpage), 14U); + memset(folio_address(fragment.folio) + fragment.offset, 0x9c, fragment.length); + release_user_fragments(&fragment, 1, true); + { + u8 buf[33]; + + ret = access_remote_vm(vma->vm_mm, address - 1, buf, sizeof(buf), 0); + KUNIT_ASSERT_EQ(test, ret, (long)sizeof(buf)); + KUNIT_EXPECT_EQ(test, buf[0], (u8)0); + KUNIT_EXPECT_EQ(test, buf[32], (u8)0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 1, 0x9c, 31), NULL); + } +} + +static void user4k_anon_exit_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct mm_subpage *held; + struct anon_vma *root; + struct folio *folio; + struct mm_struct *mm; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + } + held = f->slots[0]; + folio = mm_subpage_folio(held); + root = f->vma->anon_vma->root; + mm_subpage_get(held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + mm_subpage_pool_close(f->pool); + mm_subpage_pool_put(f->pool); + /* Leave all four mapping refs and the tables to the real exit path. */ + mm = f->mm; + memset(f, 0, sizeof(*f)); + mmput(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(held), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); + for (i = 1; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_get_from_phys(mm_subpage_phys(held) + i * SZ_4K); + + KUNIT_EXPECT_PTR_EQ(test, slot, NULL); + if (slot) + mm_subpage_put(slot); + } +} + +#ifdef CONFIG_ARM64_CONTPTE +struct contpte_fixture { + struct mm_struct *mm; + struct folio *folio; + unsigned int nr; +}; + +static void contpte_fixture_free(void *arg) +{ + struct contpte_fixture *f = arg; + struct mmu_gather tlb; + unsigned int i; + + if (f->mm) { + mmap_write_lock(f->mm); + for (i = 0; i <= f->nr; i++) { + unsigned long addr = TEST_VA + i * mm_page_size(f->mm); + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, addr, &ptl); + + if (pte) { + pte_clear(f->mm, addr, pte); + pte_unmap_unlock(pte, ptl); + } + } + tlb_gather_mmu_fullmm(&tlb, f->mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(f->mm); + mmput(f->mm); + } + if (f->folio) + folio_put(f->folio); +} + +static void test_contpte_granule(struct kunit *test, bool small) +{ + struct contpte_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct vm_area_struct vma = {}; + struct test_mm tables = {}; + struct { + bool initial, addresses, hints, gathered, aged, access_all, unfolded; + bool permissions, refolded, holes, full_clear, sentinel; + } ok; + unsigned long size; + phys_addr_t phys; + pte_t base, saved, observed; + pte_t *pte; + spinlock_t *ptl; + unsigned int i; + int access_ret, access_same_ret, access_noop_ret; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, contpte_fixture_free, f), 0); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + if (small) + KUNIT_ASSERT_EQ(test, test_mm_select4k(f->mm), 0); + f->nr = cont_ptes_mm(f->mm); + size = mm_page_size(f->mm); + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, + get_order(cont_pte_size_mm(f->mm))); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + phys = PFN_PHYS(folio_pfn(f->folio)); + tables.mm = f->mm; + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, TEST_VA, phys), 0); + vma.vm_mm = f->mm; + base = phys_pte_mm(f->mm, phys, __pgprot(TEST_PROT | PTE_RDONLY)); + memset(&ok, 1, sizeof(ok)); + mmap_read_lock(f->mm); + pte = test_mm_lookup(f->mm, TEST_VA, &ptl); + if (!pte) { + mmap_read_unlock(f->mm); + KUNIT_FAIL(test, "missing PTE table"); + return; + } + pte_clear(f->mm, TEST_VA, pte); + /* The adjacent entry must never be included in this contiguous group. */ + saved = pte_mkspecial(base); + set_pte(pte + f->nr, saved); + set_ptes(f->mm, TEST_VA, pte, base, f->nr); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.initial &= pte_cont(observed); + ok.addresses &= pte_phys_mm(f->mm, observed) == phys + i * size; + ok.hints &= pte_batch_hint(pte + i, observed) == f->nr - i; + } + __set_pte(pte + 3, pte_mkdirty(__ptep_get(pte + 3))); + observed = ptep_get(pte + 1); + ok.gathered &= pte_dirty(observed) && pte_young(observed); + observed = ptep_get_lockless(pte + 2); + ok.gathered &= pte_dirty(observed) && pte_young(observed) && + pte_phys_mm(f->mm, observed) == phys + 2 * size; + clear_young_dirty_ptes(&vma, TEST_VA + 3 * size, pte + 3, 1, + CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.aged &= !pte_young(observed) && !pte_dirty(observed) && pte_cont(observed); + } + observed = pte_mkdirty(pte_mkyoung(__ptep_get(pte + 3))); + access_same_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, + pte + 3, observed, 1); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.access_all &= pte_cont(observed) && pte_young(observed) && + pte_dirty(observed) && !pte_write(observed); + } + access_noop_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, + pte + 3, ptep_get(pte + 3), 1); + observed = pte_mkwrite_novma(pte_mkdirty(pte_mkyoung(__ptep_get(pte + 3)))); + access_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, pte + 3, + observed, 1); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.unfolded &= !pte_cont(observed); + ok.permissions &= pte_write(observed) == (i == 3); + } + set_pte_at(f->mm, TEST_VA + 3 * size, pte + 3, + pte_advance_pfn_mm(f->mm, base, 3)); + /* Last-entry installation must fold using the physical byte offset too. */ + set_pte_at(f->mm, TEST_VA + (f->nr - 1) * size, pte + f->nr - 1, + pte_advance_pfn_mm(f->mm, base, f->nr - 1)); + for (i = 0; i < f->nr; i++) + ok.refolded &= pte_cont(__ptep_get(pte + i)); + get_and_clear_full_ptes(f->mm, TEST_VA + 3 * size, pte + 3, 2, 0); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.holes &= pte_none(observed) == (i == 3 || i == 4); + if (i != 3 && i != 4) + ok.holes &= !pte_cont(observed) && + pte_phys_mm(f->mm, observed) == phys + i * size; + } + set_ptes(f->mm, TEST_VA + 3 * size, pte + 3, + pte_advance_pfn_mm(f->mm, base, 3), 2); + set_pte_at(f->mm, TEST_VA + (f->nr - 1) * size, pte + f->nr - 1, + pte_advance_pfn_mm(f->mm, base, f->nr - 1)); + for (i = 0; i < f->nr; i++) + ok.refolded &= pte_cont(__ptep_get(pte + i)); + clear_full_ptes(f->mm, TEST_VA, pte, f->nr, 0); + for (i = 0; i < f->nr; i++) + ok.full_clear &= pte_none(__ptep_get(pte + i)); + ok.sentinel = pte_same(__ptep_get(pte + f->nr), saved); + pte_unmap_unlock(pte, ptl); + mmap_read_unlock(f->mm); + KUNIT_EXPECT_TRUE(test, ok.initial); + KUNIT_EXPECT_TRUE(test, ok.addresses); + KUNIT_EXPECT_TRUE(test, ok.hints); + KUNIT_EXPECT_TRUE(test, ok.gathered); + KUNIT_EXPECT_TRUE(test, ok.aged); + KUNIT_EXPECT_TRUE(test, ok.access_all); + KUNIT_EXPECT_EQ(test, access_same_ret, 1); + KUNIT_EXPECT_EQ(test, access_noop_ret, 0); + KUNIT_EXPECT_EQ(test, access_ret, 1); + KUNIT_EXPECT_TRUE(test, ok.unfolded); + KUNIT_EXPECT_TRUE(test, ok.permissions); + KUNIT_EXPECT_TRUE(test, ok.refolded); + KUNIT_EXPECT_TRUE(test, ok.holes); + KUNIT_EXPECT_TRUE(test, ok.full_clear); + KUNIT_EXPECT_TRUE(test, ok.sentinel); +} + +static void user4k_contpte_test(struct kunit *test) +{ + test_contpte_granule(test, true); +} + +static void native_contpte_test(struct kunit *test) +{ + test_contpte_granule(test, false); +} +#endif + +struct usercopy_worker { + struct test_mm *ctx[2]; + struct completion done; + unsigned int id; + unsigned int checks; + unsigned int idmaps; + int error; + u64 tcr; +}; + +static int test_usercopy_worker(void *arg) +{ + struct usercopy_worker *worker = arg; + u64 kernel_ttbr = read_sysreg(ttbr1_el1) & ~TTBRx_EL1_ASID_MASK; + unsigned int round, i, byte; + bool borrowed = false; + struct test_mm *ctx = NULL; + + for (round = 0; round < 16; round++) { + unsigned int small = (round + worker->id) & 1; + unsigned int cpu = cpumask_first(cpu_online_mask); + + ctx = worker->ctx[small]; + kthread_use_mm(ctx->mm); + borrowed = true; + if ((round + worker->id) & 2) { + unsigned int next = cpumask_next(cpu, cpu_online_mask); + + if (next < nr_cpu_ids) + cpu = next; + } + if (set_cpus_allowed_ptr(current, cpumask_of(cpu))) { + worker->error = 1; + break; + } + /* Exercise a real scheduler switch while borrowing this mm. */ + schedule_timeout_uninterruptible(1); + if (small && round < 4) { + unsigned long flags; + bool native; + + preempt_disable(); + flags = local_daif_save(); + if (round & 2) + cpu_install_ttbr0(phys_to_ttbr(__pa_symbol(idmap_pg_dir)), + TCR_T0SZ(IDMAP_VA_BITS)); + else + cpu_install_idmap(); + native = (read_sysreg(tcr_el1) & (TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK)) == + (TCR_TG0_16K | TCR_T0SZ(IDMAP_VA_BITS)); + cpu_uninstall_idmap(); + local_daif_restore(flags); + preempt_enable(); + if (!native) { + worker->error = 6; + break; + } + worker->idmaps++; + } + worker->tcr = read_sysreg(tcr_el1); + if ((worker->tcr & TCR_EL1_TG0_MASK) != (small ? TCR_TG0_4K : TCR_TG0_16K) || + (worker->tcr & TCR_EL1_T0SZ_MASK) != TCR_T0SZ(48) || + (worker->tcr & TCR_EL1_TG1_MASK) != TCR_TG1_16K || + (read_sysreg(ttbr1_el1) & ~TTBRx_EL1_ASID_MASK) != kernel_ttbr) { + worker->error = 2; + break; + } + for (i = 0; i < 4; i++) { + unsigned long offset = 64 + worker->id * sizeof(u64); + void __user *address = (void __user *)ctx->address[i]; + u64 value = 0xa5000000 | (worker->id << 16) | (round << 8) | i; + u64 *backing = phys_to_virt(ctx->phys[i]) + offset; + unsigned long read_left, write_left; + u8 data[8]; + + /* No VMA/fault path may be entered for these private mappings. */ + pagefault_disable(); + read_left = copy_from_user(data, address, sizeof(data)); + write_left = copy_to_user(address + offset, &value, sizeof(value)); + pagefault_enable(); + if (read_left != (i == 2 ? sizeof(data) : 0) || + write_left != (i == 0 || i == 2 ? sizeof(value) : 0)) { + worker->error = 3; + goto out; + } + if (i != 2) { + for (byte = 0; byte < sizeof(data); byte++) { + if (data[byte] != (small ? 0x40 : 0x80) + i) { + worker->error = 4; + goto out; + } + } + } + if (READ_ONCE(*backing) != (i == 0 || i == 2 ? 0 : value)) { + worker->error = 5; + goto out; + } + worker->checks++; + } + kthread_unuse_mm(ctx->mm); + borrowed = false; + } +out: + if (borrowed) + kthread_unuse_mm(ctx->mm); + complete(&worker->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void test_stop_kthread(void *arg) +{ + kthread_stop(arg); +} + +struct cow_reader { + struct mm_struct *parent; + struct mm_struct *child; + struct completion ready; + struct completion copied; + struct completion done; + u8 observed[8]; + unsigned long read_left[8], write_left[8]; + int error; + unsigned int checks; +}; + +static int test_cow_reader(void *arg) +{ + struct cow_reader *r = arg; + unsigned int i, side; + + kthread_use_mm(r->child); + /* Resolve fork's old PTEs normally, then cache pre-COW translations. */ + for (i = 0; i < 4; i++) { + u8 value = 0; + unsigned long left; + + left = copy_from_user(&value, (void __user *)(TEST_VA + i * SZ_4K + 64), 1); + if (left || value != 0x41 + i) + r->error = 1; + } + complete(&r->ready); + if (!wait_for_completion_timeout(&r->copied, 10 * HZ)) { + r->error = 2; + goto out; + } + for (side = 0; side < 2; side++) { + if (side) { + kthread_unuse_mm(r->child); + kthread_use_mm(r->parent); + } + for (i = 0; i < 4; i++) { + void __user *addr = (void __user *)(TEST_VA + i * SZ_4K + 64); + u8 value = 0, changed = 0x99; + unsigned long read_left, write_left; + + pagefault_disable(); + read_left = copy_from_user(&value, addr, 1); + write_left = copy_to_user(addr, &changed, 1); + pagefault_enable(); + r->observed[side * 4 + i] = value; + r->read_left[side * 4 + i] = read_left; + r->write_left[side * 4 + i] = write_left; + if (read_left || value != 0x41 + i || + write_left != (side == 0 && i == 1 ? 0 : 1)) + r->error = 3; + r->checks++; + } + } + kthread_unuse_mm(r->parent); + goto done; +out: + kthread_unuse_mm(r->child); +done: + complete(&r->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void user4k_fault_uaccess_test(struct kunit *test) +{ + struct vm_area_struct *parent, *child_vma; + struct mm_struct *child; + struct cow_reader *reader; + struct task_struct *task; + unsigned int i, cpu; + vm_fault_t fault; + int ret; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled || !system_supports_4kb_granule() || + system_supports_cnp() || cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires opt-in and compatible 4K TTBR0 controls"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + + memset(phys_to_virt(pte_phys_mm(parent->vm_mm, entry)), 0x41 + i, SZ_4K); + } + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + child_vma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + reader = kunit_kzalloc(test, sizeof(*reader), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, reader); + reader->parent = parent->vm_mm; + reader->child = child; + init_completion(&reader->ready); + init_completion(&reader->copied); + init_completion(&reader->done); + task = kthread_create(test_cow_reader, reader, "user4k-cow-reader"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + cpu = cpumask_last(cpu_online_mask); + kthread_bind(task, cpu); + wake_up_process(task); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&reader->ready, 10 * HZ), 0UL); + mmap_read_lock(child); + fault = handle_mm_fault(child_vma, TEST_VA + SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + complete(&reader->copied); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&reader->done, 10 * HZ), 0UL); + if (reader->error) + for (i = 0; i < 8; i++) + kunit_info(test, "access %u value=%#x read_left=%lu write_left=%lu\n", + i, reader->observed[i], reader->read_left[i], reader->write_left[i]); + KUNIT_EXPECT_EQ(test, reader->error, 0); + KUNIT_EXPECT_EQ(test, reader->checks, 8U); + { + pte_t entry = test_fault_entry(test, child, TEST_VA + SZ_4K); + + KUNIT_EXPECT_EQ(test, *(u8 *)(phys_to_virt(pte_phys_mm(child, entry)) + 64), (u8)0x99); + entry = test_fault_entry(test, parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, *(u8 *)(phys_to_virt(pte_phys_mm(parent->vm_mm, entry)) + 64), (u8)0x42); + } +} + +static void user4k_context_uaccess_test(struct kunit *test) +{ + struct test_mm *ctx[2]; + struct usercopy_worker *workers; + unsigned int i; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires 4K, no CnP/PIE/POE, DS=0, A1=1 and 48-bit TTBR0"); + return; + } + ctx[0] = test_usercopy_mm(test, false); + KUNIT_ASSERT_NOT_NULL(test, ctx[0]); + ctx[1] = test_usercopy_mm(test, true); + KUNIT_ASSERT_NOT_NULL(test, ctx[1]); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + struct task_struct *task; + + workers[i].ctx[0] = ctx[0]; + workers[i].ctx[1] = ctx[1]; + workers[i].id = i; + init_completion(&workers[i].done); + task = kthread_run(test_usercopy_worker, &workers[i], "user4k-copy-%u", i); + KUNIT_ASSERT_FALSE(test, IS_ERR(task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ_MSG(test, workers[i].error, 0, "worker %u TCR=%#llx", i, workers[i].tcr); + KUNIT_EXPECT_EQ(test, workers[i].checks, 64U); + KUNIT_EXPECT_EQ(test, workers[i].idmaps, 2U); + } + kunit_info(test, "two migrating kthreads, 32 borrowed-mm switches, %s PAN\n", + system_uses_ttbr0_pan() ? "software" : "hardware"); +} + +struct zap_reader { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready; + struct completion proceed; + struct completion done; + unsigned int cpu; + unsigned int checks; + bool zapped; + int error; +}; + +static int test_zap_reader(void *arg) +{ + struct zap_reader *reader = arg; + unsigned int phase, i; + + reader->error = set_cpus_allowed_ptr(current, cpumask_of(reader->cpu)); + if (reader->error) { + complete(&reader->ready); + goto out; + } + kthread_use_mm(reader->mm); + for (phase = 0; phase < 2; phase++) { + if (phase) { + complete(&reader->ready); + wait_for_completion(&reader->proceed); + if (!reader->zapped) + break; + } + for (i = 0; i < 4; i++) { + u64 value = 0, expected = 0xa5a50000UL + i; + unsigned long left; + + pagefault_disable(); + left = copy_from_user(&value, + (void __user *)((unsigned long)TEST_VA + i * SZ_4K), sizeof(value)); + pagefault_enable(); + if (phase && i == 1) { + if (!left) + reader->error = -EUCLEAN; + } else if (left || value != expected) { + reader->error = -EFAULT; + } + reader->checks++; + } + } + kthread_unuse_mm(reader->mm); +out: + complete(&reader->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void test_stop_zap_reader(void *arg) +{ + struct zap_reader *reader = arg; + + complete(&reader->proceed); + kthread_stop(reader->task); +} + +static void user4k_anon_remote_zap_test(struct kunit *test) +{ + struct anon_slot_fixture *f; + struct zap_reader *readers; + unsigned int i, cpu; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled || num_online_cpus() < 2) { + kunit_skip(test, "requires arm64.user4k_test=1 and two CPUs in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires 4K, no CnP/PIE/POE, DS=0, A1=1 and 48-bit TTBR0"); + return; + } + f = anon_slot_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + u64 value = 0xa5a50000UL + i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memcpy(folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot), + &value, sizeof(value)); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + } + readers = kunit_kcalloc(test, 2, sizeof(*readers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, readers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 2; i++) { + readers[i].mm = f->mm; + readers[i].cpu = cpu; + init_completion(&readers[i].ready); + init_completion(&readers[i].proceed); + init_completion(&readers[i].done); + readers[i].task = kthread_run(test_zap_reader, &readers[i], "user4k-zap-%u", i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, readers[i].task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_zap_reader, + &readers[i]), 0); + cpu = cpumask_next(cpu, cpu_online_mask); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].ready, 10 * HZ), 0UL); + KUNIT_ASSERT_EQ(test, readers[i].error, 0); + } + /* Keep an explicit non-mapping reference for fixture cleanup. */ + mm_subpage_get(f->slots[1]); + mmap_read_lock(f->mm); + zap_vma_range(f->vma, TEST_VA + SZ_4K, SZ_4K); + f->mapped[1] = false; + mmap_read_unlock(f->mm); + for (i = 0; i < 2; i++) { + readers[i].zapped = true; + complete(&readers[i].proceed); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, readers[i].error, 0); + KUNIT_EXPECT_EQ(test, readers[i].checks, 8U); + } +} + +static u64 *test_leaf(struct test_tables *tables, unsigned long va) +{ + return &tables->level[3][test_index(va, tables->shift, 3)]; +} + +/* Explicitly 4K, without the native PAGE_SHIFT or a granule/level hint. */ +static void test_invalidate_4k(unsigned long va, u64 ttbr1) +{ + u64 operand = (va >> USER4K_SHIFT) | (ttbr1 & GENMASK_ULL(63, 48)); + + dsb(ishst); + __tlbi(vae1, operand); + dsb(ish); + isb(); +} + +static void user4k_translation_test(struct kunit *test) +{ + struct test_tables small, native; + struct test_result *result; + unsigned long flags; + void *data, *other; + phys_addr_t data_pa, other_pa; + u64 old_tcr, old_ttbr0, old_ttbr1, old_par, tcr4k, tcr16k; + unsigned int round, slot; + + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE)) { + kunit_skip(test, "requires 4K, no CnP, and direct PTE permissions"); + return; + } + + KUNIT_ASSERT_EQ(test, PAGE_SHIFT, 14); + KUNIT_ASSERT_EQ(test, test_alloc_tables(test, &small, USER4K_SHIFT), 0); + KUNIT_ASSERT_EQ(test, test_alloc_tables(test, &native, PAGE_SHIFT), 0); + data = test_alloc_page(test); + other = test_alloc_page(test); + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, other); + data_pa = virt_to_phys(data); + other_pa = virt_to_phys(other); + result = kunit_kcalloc(test, TEST_ROUNDS, sizeof(*result), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, result); + + /* One native mapping and four independently described subpages. */ + *test_leaf(&native, TEST_VA) = other_pa | TEST_PROT; + *test_leaf(&small, TEST_VA) = data_pa | TEST_PROT | PTE_RDONLY; + *test_leaf(&small, TEST_VA + USER4K_SIZE) = + (data_pa + USER4K_SIZE) | TEST_PROT; + /* Slot 2 deliberately unmapped. Slot 3 aliases slot 0, writable. */ + *test_leaf(&small, TEST_VA + 3 * USER4K_SIZE) = data_pa | TEST_PROT; + + preempt_disable(); + flags = local_daif_save(); + old_tcr = read_sysreg(tcr_el1); + old_ttbr0 = read_sysreg(ttbr0_el1); + old_ttbr1 = read_sysreg(ttbr1_el1); + old_par = read_sysreg_par(); + /* DS affects both address ranges: never toggle it under a live TTBR1. */ + if ((old_tcr & (TCR_EL1_DS | TCR_EL1_TG1_MASK | TCR_EL1_A1)) != + (TCR_TG1_16K | TCR_EL1_A1)) { + local_daif_restore(flags); + preempt_enable(); + kunit_skip(test, "requires 16K TTBR1, DS=0, and A1=1"); + return; + } + tcr4k = (old_tcr & ~(TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK | + TCR_EL1_EPD0)) | TCR_TG0_4K | TCR_T0SZ(48); + tcr16k = (tcr4k & ~TCR_EL1_TG0_MASK) | TCR_TG0_16K; + + /* No allocations, logging, KUnit assertions or user access in this window. */ + for (round = 0; round < TEST_ROUNDS; round++) { + struct test_result *r = &result[round]; + + test_switch_ttbr0(virt_to_phys(native.level[0]), tcr16k); + for (slot = 0; slot < PROBE_SLOTS; slot++) + r->native[slot] = test_translate(TEST_VA + slot * USER4K_SIZE, false); + test_switch_ttbr0(virt_to_phys(small.level[0]), tcr4k); + r->tcr = read_sysreg(tcr_el1); + r->ttbr1 = read_sysreg(ttbr1_el1); + for (slot = 0; slot < PROBE_SLOTS; slot++) { + unsigned long va = TEST_VA + slot * USER4K_SIZE; + + r->read[slot] = test_translate(va, false); + r->write[slot] = test_translate(va, true); + } + + /* Break-before-make one 4K PTE and invalidate exactly that VA. */ + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), 0); + test_invalidate_4k(TEST_VA + USER4K_SIZE, old_ttbr1); + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), + (other_pa + USER4K_SIZE) | TEST_PROT | PTE_RDONLY); + dsb(ishst); + isb(); + for (slot = 0; slot < PROBE_SLOTS; slot++) + r->replaced[slot] = test_translate(TEST_VA + slot * USER4K_SIZE, true); + r->replaced_read = test_translate(TEST_VA + USER4K_SIZE, false); + asm volatile("at s1e1r, %0" : : "r" (data) : "memory"); + isb(); + r->kernel = read_sysreg_par(); + + test_switch_ttbr0(old_ttbr0, old_tcr); + r->restored_tcr = read_sysreg(tcr_el1); + r->restored_ttbr0 = read_sysreg(ttbr0_el1); + /* Private tables are inactive and their translations have been flushed. */ + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), + (data_pa + USER4K_SIZE) | TEST_PROT); + } + write_sysreg(old_par, par_el1); + isb(); + local_daif_restore(flags); + preempt_enable(); + + for (round = 0; round < TEST_ROUNDS; round++) { + struct test_result *r = &result[round]; + + KUNIT_EXPECT_EQ(test, r->tcr, tcr4k); + KUNIT_EXPECT_EQ(test, r->ttbr1, old_ttbr1); + KUNIT_EXPECT_EQ(test, r->restored_tcr, old_tcr); + KUNIT_EXPECT_EQ(test, r->restored_ttbr0, old_ttbr0); + KUNIT_EXPECT_EQ(test, r->kernel & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->kernel & SYS_PAR_EL1_PA, data_pa); + KUNIT_EXPECT_EQ(test, r->replaced_read & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->replaced_read & SYS_PAR_EL1_PA, + other_pa + USER4K_SIZE); + for (slot = 0; slot < PROBE_SLOTS; slot++) { + KUNIT_EXPECT_EQ(test, r->native[slot] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->native[slot] & SYS_PAR_EL1_PA, + other_pa + slot * USER4K_SIZE); + if (slot == 2) { + KUNIT_EXPECT_EQ(test, + FIELD_GET(SYS_PAR_EL1_FST, r->read[slot]), 7ULL); + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_F, 1ULL); + KUNIT_EXPECT_EQ(test, r->write[slot] & SYS_PAR_EL1_F, 1ULL); + KUNIT_EXPECT_EQ(test, r->replaced[slot] & SYS_PAR_EL1_F, 1ULL); + continue; + } + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_PA, + data_pa + (slot == 1 ? USER4K_SIZE : 0)); + KUNIT_EXPECT_EQ(test, r->write[slot] & SYS_PAR_EL1_F, + slot == 0 ? 1ULL : 0ULL); + KUNIT_EXPECT_EQ(test, r->replaced[slot] & SYS_PAR_EL1_F, + slot == 3 ? 0ULL : 1ULL); + } + } + kunit_info(test, "16K kernel, 4K EL0 translation/permissions; %u switch cycles\n", + TEST_ROUNDS); +} + +static struct folio *test_fault_folio(struct mm_struct *mm, unsigned long address) +{ + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, address, &ptl); + struct folio *folio = NULL; + + if (pte) { + pte_t entry = ptep_get(pte); + + if (pte_present(entry) && !pte_special(entry)) { + folio = page_folio(pfn_to_page(pte_pfn(entry))); + folio_get(folio); + } + pte_unmap_unlock(pte, ptl); + } + return folio; +} + +static bool test_reclaim_swapcache(swp_entry_t entry) +{ + unsigned int tries; + + for (tries = 0; tries < 100; tries++) { + struct folio *folio = swap_cache_get_folio(entry); + LIST_HEAD(folios); + + if (!folio) + return true; + folio_wait_writeback(folio); + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Only the swap cache and reclaim isolation may retain this folio. */ + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + return false; +} + +static void test_swap_tags_set(struct folio *folio) +{ +#ifdef CONFIG_ARM64_MTE + unsigned int offset; + + if (!system_supports_mte()) + return; + if (try_page_mte_tagging(&folio->page)) + mte_clear_page_tags(folio_address(folio)); + for (offset = 0; offset < PAGE_SIZE; offset += MTE_GRANULE_SIZE) { + unsigned long tag = (1UL + offset / SZ_4K) << MTE_TAG_SHIFT; + + asm volatile(".arch_extension memtag\n\tstg %0, [%1]" + : : "r" (tag), "r" (folio_address(folio) + offset) : "memory"); + } + set_page_mte_tagged(&folio->page); +#endif +} + +static bool test_swap_tags_match(struct folio *folio, unsigned int slot) +{ +#ifdef CONFIG_ARM64_MTE + unsigned int offset; + + if (!system_supports_mte()) + return true; + if (!page_mte_tagged(&folio->page)) + return false; + for (offset = slot * SZ_4K; offset < (slot + 1) * SZ_4K; offset += MTE_GRANULE_SIZE) { + unsigned long tag = 0; + + asm volatile(".arch_extension memtag\n\tldg %0, [%1]" + : "+r" (tag) : "r" (folio_address(folio) + offset) : "memory"); + if (((tag >> MTE_TAG_SHIFT) & 0xf) != ((slot + 1) & 0xf)) + return false; + } +#endif + return true; +} + +struct fragment_test_pins { + struct user_page_fragment fragments[2 * TEST_SLOTS]; + unsigned long count; +}; + +static void fragment_test_unpin(void *arg) +{ + struct fragment_test_pins *pins = arg; + + release_user_fragments(pins->fragments, pins->count, true); + pins->count = 0; +} + +static long fragment_test_pin(struct fragment_test_pins *pins, struct mm_struct *mm, + unsigned long start, unsigned long length, unsigned int flags, + unsigned long capacity) +{ + long ret; + + fragment_test_unpin(pins); + ret = pin_user_fragments_remote(mm, start, length, flags, pins->fragments, capacity); + pins->count = ret > 0 ? ret : 0; + return ret; +} + +static void user4k_fragment_pin_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + struct fragment_test_pins *pins; + struct vm_area_struct *vma; + struct mm_struct *mm, *child; + unsigned long size, total = 0; + unsigned int i, nr; + long ret; + + if (native) { + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + vma = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + } else { + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + } + size = mm_page_size(mm); + nr = PAGE_SIZE / size; + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, mm, TEST_VA, 1, FOLL_WRITE | FOLL_NOFAULT, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, TEST_VA + 123, PAGE_SIZE - 200, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + unsigned int offset = i ? i * size : 123; + unsigned int length = size - (!i ? 123 : 0) - (i == nr - 1 ? 77 : 0); + + KUNIT_EXPECT_EQ(test, f->offset, (unsigned long)offset); + KUNIT_EXPECT_EQ(test, f->length, length); + KUNIT_EXPECT_EQ(test, !!f->subpage, !native); + memset(folio_address(f->folio) + f->offset, 0x71 + i, f->length); + total += f->length; + } + KUNIT_EXPECT_EQ(test, total, PAGE_SIZE - 200); + fragment_test_unpin(pins); + child = native ? mm_alloc() : test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, child), 0); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + /* Ordinary references retain shared COW backing, as native FOLL_GET does. */ + ret = get_user_fragments_remote(child, TEST_VA + 123, PAGE_SIZE - 200, + 0, pins->fragments, ARRAY_SIZE(pins->fragments)); + pins->count = ret > 0 ? ret : 0; + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + pte_t parent_pte = test_fault_entry(test, mm, TEST_VA + i * size); + pte_t child_pte = test_fault_entry(test, child, TEST_VA + i * size); + + KUNIT_EXPECT_FALSE(test, f->pinned); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, parent_pte), pte_phys_mm(child, child_pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x71 + i, f->length), NULL); + } + fragment_test_unpin(pins); + /* Read pins must unshare COW mappings without enabling PTE writes. */ + ret = fragment_test_pin(pins, child, TEST_VA + 123, PAGE_SIZE - 200, FOLL_NOFAULT, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, child, TEST_VA + 123, PAGE_SIZE - 200, 0, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + pte_t pte = test_fault_entry(test, child, TEST_VA + i * size); + + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x71 + i, f->length), NULL); + if (!native) { + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(f->subpage)); + if (i) + KUNIT_EXPECT_PTR_NE(test, f->folio, pins->fragments[0].folio); + } + } + fragment_test_unpin(pins); + /* Return a valid partial result when the byte range reaches an unmapped VMA. */ + ret = fragment_test_pin(pins, mm, TEST_VA, PAGE_SIZE + size, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, (long)nr); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 1); + KUNIT_EXPECT_EQ(test, ret, 1L); + KUNIT_EXPECT_EQ(test, pins->fragments[0].length, (unsigned int)size); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, TEST_VA, size, FOLL_LONGTERM, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, 1L); + ret = fragment_test_pin(pins, mm, ULONG_MAX - 15, ULONG_MAX, 0, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +#ifdef CONFIG_BLOCK +struct bio_fragment_test { + struct bio *bio; + struct bio *clone; + struct user_page_fragment held[TEST_SLOTS]; + unsigned int count; +}; + +static void bio_fragment_test_free(void *arg) +{ + struct bio_fragment_test *state = arg; + + if (state->clone) + bio_put(state->clone); + if (state->bio) { + bio_release_pages(state->bio, false); + bio_put(state->bio); + } + release_user_fragments(state->held, state->count, false); +} + +static void user4k_bio_fragments_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct bio_fragment_test *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct iov_iter iter; + unsigned int i, bytes = round_down(PAGE_SIZE - 200, 512); + long nr; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bio fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, bio_fragment_test_free, state), 0); + /* Populate real writable anonymous mappings and retain independent GETs. */ + nr = get_user_fragments_remote(vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, + state->held, ARRAY_SIZE(state->held)); + state->count = nr > 0 ? nr : 0; + KUNIT_ASSERT_EQ(test, nr, (long)TEST_SLOTS); + state->bio = bio_alloc(NULL, 8, REQ_OP_READ, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->bio); + iov_iter_ubuf(&iter, ITER_DEST, (void __user *)(TEST_VA + 123), PAGE_SIZE - 200); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_vecs_to_alloc(&iter, BIO_MAX_VECS); + kthread_unuse_mm(vma->vm_mm); + KUNIT_EXPECT_EQ(test, ret, (int)TEST_SLOTS); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_iter_get_pages(state->bio, &iter, 511); + kthread_unuse_mm(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, state->bio->bi_iter.bi_size, bytes); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), PAGE_SIZE - 200 - bytes); + KUNIT_EXPECT_EQ(test, state->bio->bi_nr_user_fragments, (int)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, state->bio->bi_vcnt, 1); + KUNIT_EXPECT_EQ(test, state->bio->bi_io_vec[0].bv_offset, 123U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 1U); + state->clone = bio_alloc_clone(NULL, state->bio, GFP_KERNEL, &fs_bio_set); + KUNIT_ASSERT_NOT_NULL(test, state->clone); + KUNIT_EXPECT_PTR_EQ(test, state->clone->bi_user_fragments, NULL); + KUNIT_EXPECT_PTR_EQ(test, state->clone->bi_io_vec, state->bio->bi_io_vec); + /* Completion ownership must outlive all originating VMAs. */ + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 1U); + bio_put(state->clone); + state->clone = NULL; + bio_release_pages(state->bio, false); + KUNIT_EXPECT_PTR_EQ(test, state->bio->bi_user_fragments, NULL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 0U); +} + +static void user4k_bio_trim_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, SZ_4K); + struct bio_fragment_test *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct iov_iter iter; + unsigned int i; + long nr; + int ret, refs; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bio fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, bio_fragment_test_free, state), 0); + nr = get_user_fragments_remote(vma->vm_mm, TEST_VA, SZ_4K, FOLL_WRITE, state->held, 1); + state->count = nr > 0 ? nr : 0; + KUNIT_ASSERT_EQ(test, nr, 1L); + refs = folio_ref_count(state->held[0].folio); + state->bio = bio_alloc(NULL, 4, REQ_OP_READ, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->bio); + for (i = 0; i < 1024; i++) { + iov_iter_ubuf(&iter, ITER_DEST, (void __user *)(TEST_VA + SZ_4K - 256), 512); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_iter_get_pages(state->bio, &iter, 511); + kthread_unuse_mm(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 512UL); + KUNIT_EXPECT_EQ(test, state->bio->bi_vcnt, 0); + KUNIT_EXPECT_PTR_EQ(test, state->bio->bi_user_fragments, NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[0].subpage), 0U); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->held[0].folio), refs); + bio_reset(state->bio, NULL, REQ_OP_READ); + } +} +#endif + +static void user4k_fragment_longterm_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct fragment_test_pins *pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + phys_addr_t before; + bool movable; + unsigned int i; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)TEST_SLOTS); + before = PFN_PHYS(folio_pfn(pins->fragments[0].folio)); + movable = !folio_is_longterm_pinnable(pins->fragments[0].folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0x91 + i, pins->fragments[i].length); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, + FOLL_WRITE | FOLL_LONGTERM, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_TRUE(test, folio_is_longterm_pinnable(f->folio)); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(f->subpage), 1U); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x91 + i, f->length), NULL); + } + if (movable) + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(pins->fragments[0].folio)), before); + kunit_info(test, "long-term source movable=%u migrated=%u\n", movable, + PFN_PHYS(folio_pfn(pins->fragments[0].folio)) != before); + fragment_test_unpin(pins); + /* A long-term flag must never silently create an ordinary GET. */ + ret = get_user_fragments_remote(vma->vm_mm, TEST_VA, SZ_4K, FOLL_LONGTERM, + pins->fragments, 1); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void user4k_file_fragments_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *shared, *private; + struct fragment_test_pins *pins; + phys_addr_t shared_phys; + long ret; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, SZ_4K); + private = test_file_vma(test, file, false, false, TEST_VA + SZ_4K, 3 * SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, shared->vm_mm, shared->vm_start + 17, + 3 * SZ_4K - 33, FOLL_WRITE, 8); + KUNIT_ASSERT_EQ(test, ret, 3L); + shared_phys = PFN_PHYS(folio_pfn(pins->fragments[0].folio)); + for (i = 0; i < 3; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_PTR_EQ(test, f->subpage, NULL); + KUNIT_EXPECT_EQ(test, f->offset, (unsigned long)((i + 1) * SZ_4K + (!i ? 17 : 0))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x32 + i, f->length), NULL); + } + /* File pins stay live across PTE removal and mm exit. */ + kunit_release_action(test, user4k_vma_mm_free, shared->vm_mm); + for (i = 0; i < 3; i++) + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(pins->fragments[i].folio)); + fragment_test_unpin(pins); + ret = get_user_fragments_remote(private->vm_mm, private->vm_start, 3 * SZ_4K, + 0, pins->fragments, 8); + pins->count = ret > 0 ? ret : 0; + KUNIT_ASSERT_EQ(test, ret, 3L); + for (i = 0; i < 3; i++) { + KUNIT_EXPECT_PTR_EQ(test, pins->fragments[i].subpage, NULL); + KUNIT_EXPECT_EQ(test, PFN_PHYS(folio_pfn(pins->fragments[i].folio)), shared_phys); + } + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, private->vm_mm, private->vm_start, 3 * SZ_4K, 0, 8); + KUNIT_ASSERT_EQ(test, ret, 3L); + for (i = 0; i < 3; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_NOT_NULL(test, f->subpage); + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(f->folio)), shared_phys); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x32 + i, f->length), NULL); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, private->vm_mm, + private->vm_start + i * SZ_4K))); + } + fragment_test_unpin(pins); +} + +/* Exercise the same implementation used by the arm64 syscall entry. */ +asmlinkage long __arm64_sys_mprotect(const struct pt_regs *regs); + +static long test_mprotect(unsigned long start, unsigned long len, unsigned long prot) +{ + struct pt_regs regs = { .regs = { start, len, prot } }; + + return __arm64_sys_mprotect(®s); +} + +static unsigned long test_copy_word(unsigned long addr, u64 *value, bool write) +{ + unsigned long left; + + pagefault_disable(); + if (write) + left = copy_to_user((void __user *)addr, value, sizeof(*value)); + else + left = copy_from_user(value, (void __user *)addr, sizeof(*value)); + pagefault_enable(); + return left; +} + +static void user4k_mprotect_test(struct kunit *test) +{ + const unsigned long starts[] = { + TEST_VA, SZ_2M - SZ_4K, SZ_1G - SZ_4K, BIT(39) - SZ_4K, TEST_VA, + }; + unsigned int shape; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + for (shape = 0; shape < ARRAY_SIZE(starts); shape++) { + struct file *file = shape == 4 ? test_cache_file(test, PAGE_SIZE, true) : NULL; + struct mm_struct *mm = test_vma_mm(test); + unsigned long start = starts[shape], mapped; + unsigned long flags = MAP_FIXED_NOREPLACE | (file ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS); + phys_addr_t before[4] = { }; + long protect[4] = { }; + unsigned long initial = 0, final = 0, read_ro = 0, write_ro = 0, read_none = 0; + u64 value, ro_value = 0; + unsigned int i; + vm_fault_t fault = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shape == 4) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + kthread_use_mm(mm); + mapped = vm_mmap(file, start, PAGE_SIZE, PROT_READ | PROT_WRITE, flags, 0); + if (IS_ERR_VALUE(mapped)) + goto detach; + for (i = 0; i < 4; i++) { + struct vm_area_struct *vma = test_vma_lookup(mm, start + i * SZ_4K); + + fault |= test_file_fault(vma, start + i * SZ_4K, true); + if (fault & VM_FAULT_ERROR) + goto detach; + before[i] = pte_phys_mm(mm, test_fault_entry(test, mm, start + i * SZ_4K)); + value = 0x88110000 + i; + initial |= test_copy_word(start + i * SZ_4K, &value, true); + } + /* Keep the mm attached across permission changes and cached accesses. */ + protect[0] = test_mprotect(start + SZ_4K, SZ_4K, PROT_READ); + value = 0xaa; + write_ro = test_copy_word(start + SZ_4K, &value, true); + final |= test_copy_word(start, &value, true); + final |= test_copy_word(start + 3 * SZ_4K, &value, true); + protect[1] = test_mprotect(start + 2 * SZ_4K, SZ_4K, PROT_NONE); + read_none = test_copy_word(start + 2 * SZ_4K, &value, false); + read_ro = test_copy_word(start + SZ_4K, &ro_value, false); + protect[2] = test_mprotect(start, PAGE_SIZE, PROT_READ); + protect[3] = test_mprotect(start, PAGE_SIZE, PROT_READ | PROT_WRITE); + for (i = 0; i < 4; i++) { + value = 0x99220000 + i; + final |= test_copy_word(start + i * SZ_4K, &value, true); + } +detach: + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, initial, 0UL); + KUNIT_EXPECT_EQ(test, final, 0UL); + KUNIT_EXPECT_EQ(test, write_ro, (unsigned long)sizeof(u64)); + KUNIT_EXPECT_EQ(test, read_none, (unsigned long)sizeof(u64)); + KUNIT_EXPECT_EQ(test, read_ro, 0UL); + KUNIT_EXPECT_EQ(test, ro_value, 0x88110001ULL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, start + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, protect[i], 0L); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), before[i]); + } + KUNIT_EXPECT_EQ(test, mm->total_vm, (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm->map_count, 1); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_mprotect_cow_batch_test(struct kunit *test) +{ + struct mm_struct *mm, *child; + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + phys_addr_t before[4]; + unsigned int i; + unsigned long mapped; + long ret[2]; + int copied; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, fragment_test_pin(pins, mm, TEST_VA + SZ_4K, SZ_4K, FOLL_WRITE, 8), 1L); + for (i = 0; i < 4; i++) + before[i] = pte_phys_mm(mm, test_fault_entry(test, mm, TEST_VA + i * SZ_4K)); + uprobe_start_dup_mmap(); + copied = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, copied, 0); + kthread_use_mm(mm); + ret[0] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + ret[1] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret[0], 0L); + KUNIT_EXPECT_EQ(test, ret[1], 0L); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), before[i]); + /* One native folio contains both exclusive and shared COW slots. */ + KUNIT_EXPECT_EQ(test, pte_write(pte), i == 1); + } + fragment_test_unpin(pins); +} + +static void user4k_mprotect_accounting_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + unsigned long mapped, committed[4], data[4]; + s64 global[4], before; + long ret[4]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + if (!IS_ERR_VALUE(mapped)) { + ret[0] = test_mprotect(TEST_VA + SZ_4K, SZ_4K, PROT_READ); + committed[0] = mm->committed_user_pages; + data[0] = mm->data_vm; + global[0] = percpu_counter_sum(&vm_committed_as) - before; + ret[1] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + committed[1] = mm->committed_user_pages; + data[1] = mm->data_vm; + global[1] = percpu_counter_sum(&vm_committed_as) - before; + ret[2] = test_mprotect(TEST_VA + 2 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE); + committed[2] = mm->committed_user_pages; + data[2] = mm->data_vm; + global[2] = percpu_counter_sum(&vm_committed_as) - before; + ret[3] = vm_munmap(TEST_VA, PAGE_SIZE); + committed[3] = mm->committed_user_pages; + data[3] = mm->data_vm; + global[3] = percpu_counter_sum(&vm_committed_as) - before; + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + for (i = 0; i < 4; i++) { + unsigned long expected = i == 0 ? TEST_SLOTS - 1 : i == 2 ? 1 : 0; + + KUNIT_EXPECT_EQ(test, ret[i], 0L); + KUNIT_EXPECT_EQ(test, committed[i], expected); + KUNIT_EXPECT_EQ(test, data[i], expected); + KUNIT_EXPECT_EQ(test, global[i], expected ? 1LL : 0LL); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 0); +} + +static void user4k_fragment_permissions_test(struct kunit *test) +{ + unsigned long start = TEST_VA + SZ_4K; + struct vm_area_struct *vma = test_fault_vma(test, start, SZ_4K); + struct fragment_test_pins *pins; + struct user_page_fragment *fragment; + struct mm_struct *mm; + spinlock_t *ptl; + pte_t *ptep; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + fragment = &pins->fragments[0]; + { + struct page *page = NULL; + int locked = 1; + + mmap_read_lock(mm); + ret = pin_user_pages_remote(mm, start, 1, 0, &page, &locked); + if (locked) + mmap_read_unlock(mm); + if (ret == 1) + unpin_user_page(page); + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + } + ret = fragment_test_pin(pins, mm, start + 17, 31, 0, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_TRUE(test, is_zero_folio(fragment->folio)); + KUNIT_EXPECT_PTR_EQ(test, fragment->subpage, NULL); + /* Zero-page PTEs alias the first physical 4K despite this VA's slot 1. */ + KUNIT_EXPECT_EQ(test, fragment->offset, 17UL); + KUNIT_EXPECT_EQ(test, fragment->length, 31U); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment->folio) + fragment->offset, + 0, fragment->length), NULL); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_NOT_NULL(test, fragment->subpage); + KUNIT_EXPECT_EQ(test, fragment->offset, SZ_4K + 17UL); + memset(folio_address(fragment->folio) + fragment->offset, 0xa4, fragment->length); + fragment_test_unpin(pins); + mmap_write_lock(mm); + vm_flags_clear(vma, VM_WRITE); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + ptep = test_mm_lookup(mm, start, &ptl); + if (ptep) { + ptep_set_wrprotect(mm, start, ptep); + pte_unmap_unlock(ptep, ptl); + } + flush_tlb_range(vma, start, start + SZ_4K); + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE, 8); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE | FOLL_FORCE, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, mm, start))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment->folio) + fragment->offset, + 0xa4, fragment->length), NULL); + fragment_test_unpin(pins); + mmap_write_lock(mm); + vm_flags_clear(vma, VM_READ); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_unlock(mm); + ret = fragment_test_pin(pins, mm, start + 17, 31, 0, 8); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_FORCE, 8); + KUNIT_EXPECT_EQ(test, ret, 1L); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, start, 1, FOLL_GET, 8); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void test_subpage_unpin(void *arg) +{ + mm_subpage_unpin(arg); +} + +static void user4k_slot_pin_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *child; + struct mm_subpage *slots[4]; + struct folio *folio, *dst; + unsigned int i; + vm_fault_t fault; + spinlock_t *ptl; + pte_t *ptep, pte; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + slots[i] = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slots[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slots[i]), 0); + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(slots[i])); + memset(folio_address(mm_subpage_folio(slots[i])) + i * SZ_4K, 0x61 + i, SZ_4K); + } + folio = mm_subpage_folio(slots[1]); + test_swap_tags_set(folio); + for (i = 0; i < 2; i++) { + ptep = test_mm_lookup(vma->vm_mm, TEST_VA + SZ_4K, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = mm_subpage_pin(slots[1]); + pte_unmap_unlock(ptep, ptl); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_unpin, slots[1]), 0); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 2U); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_FALSE(test, mm_subpage_try_share(slots[1])); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + /* Finish even unexpected success so a failing assertion cannot leak metadata. */ + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + KUNIT_EXPECT_EQ(test, ret, -EBUSY); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) { + struct mm_subpage *copy; + + pte = test_fault_entry(test, child, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, pte) == mm_subpage_phys(slots[i]), i != 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), i == 1 ? 1 : 2); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(slots[i]), i == 1); + copy = mm_subpage_get_from_phys(pte_phys_mm(child, pte)); + KUNIT_ASSERT_NOT_NULL(test, copy); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(copy), i == 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(copy)) + i * SZ_4K, + 0x61 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(copy), i)); + mm_subpage_put(copy); + } + /* A read pin on a shared slot must request unsharing, not pin shared COW data. */ + ptep = test_mm_lookup(vma->vm_mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = mm_subpage_pin(slots[0]); + pte_unmap_unlock(ptep, ptl); + if (!ret) + mm_subpage_unpin(slots[0]); + KUNIT_EXPECT_EQ(test, ret, -EAGAIN); + /* Re-enabling writes must retain the exact backing of an exclusive pinned slot. */ + ptep = test_mm_lookup(vma->vm_mm, TEST_VA + SZ_4K, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ptep_set_wrprotect(vma->vm_mm, TEST_VA + SZ_4K, ptep); + pte_unmap_unlock(ptep, ptl); + flush_tlb_range(vma, TEST_VA + SZ_4K, TEST_VA + 2 * SZ_4K); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA + SZ_4K, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), mm_subpage_phys(slots[1])); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + *((u8 *)folio_address(folio) + SZ_4K) = 0xab; + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(child, pte)), (u8)0x62); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[1]), 0); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 2U); + KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + SZ_4K), (u8)0xab); + kunit_release_action(test, test_subpage_unpin, slots[1]); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 1U); + kunit_release_action(test, test_subpage_unpin, slots[1]); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 0U); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); +} + +#ifdef CONFIG_MIGRATION +static struct folio *test_migration_alloc(struct folio *src, unsigned long private) +{ + return folio_alloc(GFP_HIGHUSER_MOVABLE, 0); +} + +asmlinkage long __arm64_sys_madvise(const struct pt_regs *regs); + +static long test_madvise(unsigned long addr, unsigned long size, unsigned long behavior) +{ + struct pt_regs regs = { .regs = { addr, size, behavior } }; + + return __arm64_sys_madvise(®s); +} + +static void user_granule_large_vma_split_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14 }, i; + + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) || + (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { + kunit_skip(test, "requires THP boundary splitting and borrowed-mm execution"); + return; + } + for (i = 0; i < ARRAY_SIZE(shifts); i++) { + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long size = 1UL << shifts[i]; + unsigned long base = 4UL * HPAGE_PMD_SIZE, mapped; + pte_t pte; + long ret; + + if (shifts[i] >= PAGE_SHIFT) + continue; + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shifts[i]), 0); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, base, 2UL * HPAGE_PMD_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE | MAP_NORESERVE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, base); + vma = test_vma_lookup(mm, base); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, base, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, base + size, true) & VM_FAULT_ERROR, 0U); + /* The generic VMA split probes the native THP boundary for this range. */ + kthread_use_mm(mm); + ret = test_mprotect(base + size, size, PROT_READ); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, mm, base + size); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + pte = test_fault_entry(test, mm, base); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + kthread_use_mm(mm); + ret = test_mprotect(base + size, size, PROT_READ | PROT_WRITE); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0L); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user_granule_brk_extent_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int count = PAGE_SHIFT > 14 ? 3 : 2, i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm brk fixture requires hardware opt-in"); + return; + } + for (i = 0; i < count; i++) { + struct mm_struct *mm = mm_alloc(); + struct vm_area_struct *vma; + unsigned long size = 1UL << shifts[i], neighbor; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shifts[i]), 0); + kthread_use_mm(mm); + ret = vm_brk_flags(TEST_VA, size + 1, false); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_start, TEST_VA); + KUNIT_EXPECT_EQ(test, vma->vm_end, TEST_VA + 2 * size); + KUNIT_EXPECT_EQ(test, mm->total_vm, 2UL); + /* FEX reserves exactly this page after its initial ELF/BSS break. */ + kthread_use_mm(mm); + neighbor = vm_mmap(NULL, TEST_VA + 2 * size, size, PROT_NONE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, neighbor, TEST_VA + 2 * size); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_droppable_reclaim_test(struct kunit *test) +{ + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm droppable fixture requires hardware opt-in"); + return; + } + /* Dirty discard, retained GET, retained PIN, migration, zero-page COW. */ + for (mode = 0; mode < 5; mode++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct user_page_fragment held = {}; + struct folio *folio; + unsigned int i, succeeded = 0; + unsigned long mapped; + long ret; + pte_t pte; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_DROPPABLE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_TRUE(test, vma->vm_flags & VM_WIPEONFORK); + KUNIT_EXPECT_TRUE(test, vma->vm_flags & VM_DONTDUMP); + if (mode == 4) { + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_TRUE(test, is_zero_pfn(pte_pfn(pte))); + } + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + if (mode != 4) + for (i = 1; i < PAGE_SIZE / SZ_4K; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + i * SZ_4K, + true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + memset(folio_address(folio), 0xa7, PAGE_SIZE); + folio_put(folio); + if (mode == 1 || mode == 2) { + ret = mode == 1 ? get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1) : + pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + if (mode == 3) { + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 1L); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_EQ(test, ret, 0L); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + } + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + if (mode == 1 || mode == 2) { + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(unsigned char *)folio_address(folio), 0xa7); + release_user_fragments(&held, 1, false); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + } + folio_put(folio); + for (i = 0; i < (mode == 4 ? 1 : PAGE_SIZE / SZ_4K); i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0L); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, PAGE_SIZE), NULL); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_lazyfree_test(struct kunit *test) +{ + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm advice requires hardware opt-in"); + return; + } + /* Partial, redirty, retained GET, typed write, all leaves, migration. */ + for (mode = 0; mode < 10; mode++) { + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *mm; + struct user_page_fragment held = {}; + struct folio *folio; + struct mm_subpage *slot; + unsigned int i, succeeded = 0; + long ret; + pte_t pte; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + if (mode == 6) { + ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + kthread_use_mm(mm); + ret = test_madvise(TEST_VA, + (mode == 4 || mode == 9) ? PAGE_SIZE : SZ_4K, MADV_FREE); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_EQ(test, pte_dirty(pte), mode == 6); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_is_lazyfree(slot), mode != 6); + mm_subpage_put(slot); + if (mode == 1) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + + if (mode == 2 || mode == 3 || mode == 8) { + ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, + mode == 3 ? FOLL_WRITE : 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + if (mode == 3 || mode == 8) { + memset(folio_address(held.folio) + held.offset, 0xd3, SZ_4K); + release_user_fragments(&held, 1, true); + } + } + if (mode == 7) { + ret = pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + if (mode == 5 || mode == 9) { + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 1L); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_EQ(test, ret, 0L); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + } + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_none(pte), (mode == 4 || mode == 9) || + (i == 0 && (mode == 0 || mode == 5))); + } + if (mode == 2 || mode == 6 || mode == 7) + release_user_fragments(&held, 1, false); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_mprotect_migration_test(struct kunit *test) +{ + unsigned int file_case; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + for (file_case = 0; file_case < 2; file_case++) { + struct mm_struct *mm = test_vma_mm(test); + struct file *file = file_case ? test_cache_file(test, PAGE_SIZE, true) : NULL; + struct vm_area_struct *vma; + struct folio *src, *dst; + unsigned long mapped; + unsigned int i; + int prepared, unmapped = -1; + long protected = -EINVAL; + bool offsets = true; + phys_addr_t phys; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (file_case) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + kthread_use_mm(mm); + mapped = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | (file ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + src = test_fault_folio(mm, TEST_VA); + if (!src) { + folio_put(dst); + KUNIT_FAIL(test, "missing migration source"); + return; + } + phys = PFN_PHYS(folio_pfn(src)); + folio_lock(src); + folio_lock(dst); + prepared = mm_subpage_migrate_prepare(src, dst); + if (!prepared) { + try_to_migrate(src, TTU_SYNC); + unmapped = folio_mapcount(src); + kthread_use_mm(mm); + protected = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + kthread_unuse_mm(mm); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + offsets &= softleaf_is_migration(softleaf_from_pte(pte)) && + !softleaf_is_migration_write(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + } + remove_migration_ptes(src, src, 0); + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + folio_put(dst); + folio_put(src); + KUNIT_ASSERT_EQ(test, prepared, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_EQ(test, protected, 0L); + KUNIT_EXPECT_TRUE(test, offsets); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys + i * SZ_4K); + } + } +} + +/* Independent roots and VA offsets must not change physical slot identity. */ +static void user4k_nonlinear_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct mm_struct *child = test_vma_mm(test); + struct mm_subpage *slot; + struct folio *folio, *dst; + phys_addr_t phys; + vm_flags_t flags; + unsigned int succeeded = 0, side; + int ret, unmapped = -1; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, child); + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + /* Unlike the lightweight rmap fixture, real LRU migration needs a charge. */ + ret = mem_cgroup_charge(folio, a->mm, GFP_KERNEL); + if (ret) { + mm_subpage_put(slot); + KUNIT_FAIL(test, "cannot charge nonlinear fixture folio"); + return; + } + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + phys = mm_subpage_phys(slot); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + /* Same native folio, different root and different VA/physical offset. */ + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), folio); + memset(folio_address(folio), 0x53, SZ_4K); + memset(folio_address(folio) + 3 * SZ_4K, 0xa7, SZ_4K); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, b->mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 2); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, + test_fault_entry(test, child, TEST_VA + SZ_4K)), phys + 3 * SZ_4K); + kunit_release_action(test, user4k_vma_mm_free, child); + + /* Rollback must find migration entries through both roots. */ + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + folio_get(folio); + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + if (!ret) { + try_to_migrate(folio, TTU_SYNC); + unmapped = folio_mapcount(folio); + remove_migration_ptes(folio, folio, 0); + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 2); + + /* Successful migration exercises copied identities in the new owner. */ + folio_add_lru(folio); + lru_add_drain_all(); + folio_get(folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (ret) { + KUNIT_FAIL(test, "nonlinear migration failed: %d", ret); + return; + } + /* Refresh fixture references before assertions/cleanup after migration. */ + for (side = 0; side < 2; side++) { + struct anon_slot_fixture *f = side ? b : a; + pte_t pte = test_fault_entry(test, f->mm, TEST_VA + side * SZ_4K); + + f->slots[side] = NULL; + f->mapped[side] = false; + if (!pte_present(pte)) + continue; + slot = mm_subpage_get_from_phys(pte_phys_mm(f->mm, pte)); + if (!slot) + continue; + f->slots[side] = slot; + f->mapped[side] = true; + mm_subpage_put(slot); + } + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + for (side = 0; side < 2; side++) { + struct anon_slot_fixture *f = side ? b : a; + + slot = f->slots[side]; + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), side ? 3U * SZ_4K : 0U); + KUNIT_EXPECT_NE(test, mm_subpage_phys(slot), phys + (side ? 3 * SZ_4K : 0)); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), side ? 0xa7 : 0x53, SZ_4K), NULL); + } +} + +static void user4k_migration_entries_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *child = test_vma_mm(test); + struct folio *src, *dst; + unsigned int i, side; + vm_fault_t fault; + bool encoded = true, copied = true, exact = true; + int ret, fork_ret = -EINVAL, unmapped = -1; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + src = test_fault_folio(vma->vm_mm, TEST_VA); + if (!src) { + folio_put(dst); + KUNIT_FAIL(test, "missing source folio"); + return; + } + folio_lock(src); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(src, dst); + if (!ret) { + try_to_migrate(src, TTU_SYNC); + unmapped = folio_mapcount(src); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + struct rmap_walk_range range = { + .address = TEST_VA + i * SZ_4K, + .subpage = true, + .subpage_offset = i * SZ_4K, + }; + DEFINE_FOLIO_RMAP_WALK(walk, src, vma, range, + PVMW_SYNC | PVMW_MIGRATION); + unsigned int count = 0; + + encoded &= softleaf_is_migration_write(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + while (page_vma_mapped_walk(&walk)) { + exact &= walk.address == TEST_VA + i * SZ_4K; + count++; + } + exact &= count == 1; + } + uprobe_start_dup_mmap(); + fork_ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + if (!fork_ret) { + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + copied &= softleaf_is_migration_read(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + } + } + } + /* Drop one migration entry while the other mm still retains it. */ + mmap_read_lock(vma->vm_mm); + zap_vma_range(vma, TEST_VA + SZ_4K, SZ_4K); + mmap_read_unlock(vma->vm_mm); + remove_migration_ptes(src, src, 0); + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + folio_put(dst); + folio_put(src); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, fork_ret, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_TRUE(test, encoded); + KUNIT_EXPECT_TRUE(test, copied); + KUNIT_EXPECT_TRUE(test, exact); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), side ? (unsigned long)TEST_SLOTS : TEST_SLOTS - 1UL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + struct mm_subpage *slot; + + if (!side && i == 1) { + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + continue; + } + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), i == 1 ? 1 : 2); + mm_subpage_put(slot); + } + } +} + +static struct folio *test_file_migration_alloc(struct folio *src, unsigned long private) +{ + return folio_alloc(GFP_HIGHUSER_MOVABLE, folio_order(src)); +} + +static void user64k_file_migration_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 6; mode++) { + bool shmem = mode & 1, fail = mode >= 2, pin = mode >= 4; + bool isolated; + unsigned long len = shmem ? SZ_64K : 2 * SZ_64K; + unsigned long base = shmem ? SZ_64K : 0, offset; + unsigned int nr = len / PAGE_SIZE, succeeded = 0; + unsigned int maps = 2 * nr + len / SZ_4K - 1; + struct file *file = shmem ? test_cache_file(test, 2 * SZ_64K, true) : + test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + struct vm_area_struct *coarse, *native, *small; + struct folio *folio; + struct user_page_fragment *fragment = NULL; + phys_addr_t old_phys, phys; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, len, base); + native = test_file_vma(test, file, true, true, TEST_VA, len, base); + /* Shifted file offset and a small-page table boundary in the alias. */ + small = test_file_vma(test, file, false, true, SZ_2M - SZ_4K, + len - SZ_4K, base + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + for (offset = 0; offset < len; offset += SZ_64K) + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + offset, shmem) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < len; offset += PAGE_SIZE) + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + offset, shmem) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < len - SZ_4K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + offset, + shmem) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, base >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), len); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + old_phys = page_to_phys(&folio->page); + if (pin) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_promote_unpin, fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + small->vm_end - SZ_4K + 37, 1, 0, fragment, 1), 1L); + KUNIT_EXPECT_PTR_EQ(test, fragment->folio, folio); + KUNIT_EXPECT_EQ(test, fragment->offset, len - SZ_4K + 37); + } + isolated = isolate_folio_to_list(folio, &folios); + /* Retain either a GET or a typed PIN to force migration rollback. */ + if (!fail || pin || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_file_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (pin) { + KUNIT_EXPECT_EQ(test, page_to_phys(&fragment->folio->page), old_phys); + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), + (u8)(0x31 + (base + len - SZ_4K) / SZ_4K)); + kunit_release_action(test, test_promote_unpin, fragment); + } else if (fail) { + folio_put(folio); + } + KUNIT_EXPECT_EQ_MSG(test, ret, fail ? 1 : 0, "mode=%u", mode); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : nr); + folio = filemap_get_folio(file->f_mapping, base >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + phys = page_to_phys(&folio->page); + KUNIT_EXPECT_EQ(test, phys == old_phys, fail); + KUNIT_EXPECT_EQ(test, folio_size(folio), len); + KUNIT_EXPECT_EQ(test, folio->index, base >> PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse->vm_mm, + shmem ? MM_SHMEMPAGES : MM_FILEPAGES), (long)(len / SZ_64K)); + for (offset = 0; offset < len; offset += SZ_4K) { + pte_t pte; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0x31 + (base + offset) / SZ_4K, SZ_4K), NULL); + if (offset) { + pte = test_fault_entry(test, small->vm_mm, + small->vm_start + offset - SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, pte), phys + offset); + } + if (!(offset % PAGE_SIZE)) { + pte = test_fault_entry(test, native->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(native->vm_mm, pte), phys + offset); + } + if (!(offset % SZ_64K)) { + pte = test_fault_entry(test, coarse->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse->vm_mm, pte), phys + offset); + } + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); + } +} + +static void user64k_anon_migration_test(struct kunit *test) +{ + unsigned int mode, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_file_enabled(test)) + return; + for (mode = 0; mode < 5; mode++) { + bool forked = mode & 1, fail = mode >= 2, pin = mode == 4; + struct vm_area_struct *vma, *cvma = NULL; + struct mm_struct *child = NULL; + struct user_page_fragment *fragment = NULL; + struct folio *folio; + phys_addr_t old_phys, phys; + unsigned int i, succeeded = 0; + pte_t pte; + bool isolated; + int ret; + LIST_HEAD(folios); + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, true) & + VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + old_phys = pte_phys_mm(vma->vm_mm, pte); + folio = page_folio(pfn_to_page(PHYS_PFN(old_phys))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(folio, i)), 0x51 + i, PAGE_SIZE); + if (forked) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + } + if (pin) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_promote_unpin, fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(vma->vm_mm, + TEST_VA + SZ_64K - 1, 1, FOLL_WRITE, fragment, 1), 1L); + KUNIT_EXPECT_PTR_EQ(test, fragment->folio, folio); + KUNIT_EXPECT_EQ(test, fragment->offset, (unsigned int)SZ_64K - 1); + } + lru_add_drain_all(); + folio_get(folio); + isolated = isolate_folio_to_list(folio, &folios); + if (!fail || pin || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_file_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (pin) + kunit_release_action(test, test_promote_unpin, fragment); + else if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ_MSG(test, ret, fail ? 1 : 0, "mode=%u", mode); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : nr); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), !forked); + phys = pte_phys_mm(vma->vm_mm, pte); + KUNIT_EXPECT_EQ(test, phys == old_phys, fail); + folio = page_folio(pfn_to_page(PHYS_PFN(phys))); + folio_get(folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + expect_coarse_anon_state(test, folio, forked ? 2 : 1, !forked); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr * (forked ? 2 : 1) + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + if (forked) { + struct folio *copied; + + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, pte), phys); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1L); + /* A restored shared migration entry must still perform full-leaf COW. */ + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_64K - 1, + true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), phys); + copied = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(child, pte)))); + expect_coarse_anon_state(test, copied, 1, true); + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(copied, i)), + 0x51 + i, PAGE_SIZE), NULL); + memset(folio_address(copied), 0xa7, SZ_64K); + kunit_release_action(test, user4k_vma_mm_free, child); + } + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(folio, i)), + 0x51 + i, PAGE_SIZE), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + } +} + +static void user4k_file_migration_test(struct kunit *test) +{ + unsigned int fail; + + for (fail = 0; fail < 2; fail++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *small, *native; + struct folio *folio; + phys_addr_t old_phys; + unsigned int i, succeeded = 0; + int ret; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, SZ_2M - SZ_4K, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + i * SZ_4K, + true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ [0 ... TEST_SLOTS - 1] = 1 }, 1); + old_phys = PFN_PHYS(folio_pfn(folio)); + isolated = isolate_folio_to_list(folio, &folios); + if (!fail || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ(test, ret, fail ? 1 : 0); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : 1U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ [0 ... TEST_SLOTS - 1] = 1 }, 1); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, small->vm_mm, small->vm_start + i * SZ_4K); + + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + PFN_PHYS(folio_pfn(folio)) + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry) == old_phys + i * SZ_4K, !!fail); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, pte_phys_mm(native->vm_mm, + test_fault_entry(test, native->vm_mm, TEST_VA)), PFN_PHYS(folio_pfn(folio))); + /* File reclaim must visit every 4K alias, including across PTE tables. */ + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + KUNIT_EXPECT_FALSE(test, folio_mapped(folio)); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 0, 0, 0 }, 0); + folio_unlock(folio); + folio_put(folio); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, + small->vm_start + i * SZ_4K))); + } + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + i * SZ_4K, + false) & VM_FAULT_ERROR, 0U); + } +} + +static void user4k_migration_test(struct kunit *test) +{ + const unsigned long boundaries[] = { + SZ_2M - SZ_4K, SZ_2M, SZ_1G - SZ_4K, SZ_1G, + BIT(39) - SZ_4K, BIT(39), + }; + unsigned int mode; + + /* Dense/sparse, forks, refcount rollback, batched TLBI and table edges. */ + for (mode = 0; mode < 16; mode++) { + bool edge = mode >= 6 && mode < 12; + bool locked = mode >= 12; + bool sparse = mode == 1 || edge || mode == 13; + bool forked = mode == 2 || mode == 4 || edge || mode == 14; + bool fail = mode == 3 || mode == 4 || mode == 15; + unsigned long start = edge ? boundaries[mode - 6] : + TEST_VA + (sparse ? SZ_4K : 0); + unsigned int nr = sparse ? 1 : 4, i, side, succeeded = 0; + struct vm_area_struct *vma = test_fault_vma(test, start, nr * SZ_4K); + struct mm_struct *child = NULL; + struct folio *folio; + phys_addr_t old_phys; + vm_fault_t fault; + int ret; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, vma); + if (locked) { + mmap_write_lock(vma->vm_mm); + vm_flags_set(vma, VM_LOCKED); + vma->vm_mm->locked_vm = nr; + mmap_write_unlock(vma->vm_mm); + } + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, start, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + old_phys = PFN_PHYS(folio_pfn(folio)); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x41 + i + mode, SZ_4K); + test_swap_tags_set(folio); + folio_put(folio); + if (forked) { + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + } + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + /* An extra native reference forces the normal migration rollback. */ + if (!fail || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + mode == 5 ? MIGRATE_ASYNC : MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ(test, ret, fail ? 1 : 0); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : 1U); + for (side = 0; side <= (unsigned int)forked; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + for (i = 0; i < nr; i++) { + unsigned int index = sparse ? offset_in_page(start) / SZ_4K : i; + pte_t pte = test_fault_entry(test, mm, start + i * SZ_4K); + struct mm_subpage *slot; + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) == old_phys + index * SZ_4K, fail); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), !forked); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), index * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), forked ? 2 : 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + index * SZ_4K, + 0x41 + index + mode, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, index)); + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)nr); + } + if (locked) { + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + if (!fail) + KUNIT_EXPECT_EQ(test, folio->mlock_count, nr); + folio_put(folio); + } + if (child) + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } +} +struct fragment_pin_worker { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready; + unsigned long checks; + int error; +}; + +static int fragment_pin_worker_run(void *arg) +{ + struct fragment_pin_worker *worker = arg; + struct user_page_fragment fragments[4]; + + while (!kthread_should_stop()) { + long nr = pin_user_fragments_remote(worker->mm, TEST_VA, PAGE_SIZE, + 0, fragments, ARRAY_SIZE(fragments)); + unsigned int i; + + if (nr != 4) { + worker->error = nr < 0 ? nr : -EIO; + } else { + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &fragments[i]; + + if (memchr_inv(folio_address(f->folio) + f->offset, 0x51 + i, f->length)) + worker->error = -EILSEQ; + worker->checks++; + } + } + if (nr > 0) + release_user_fragments(fragments, nr, false); + complete_all(&worker->ready); + if (worker->error) + break; + cond_resched(); + } + return 0; +} + +static void fragment_pin_worker_stop(void *arg) +{ + struct fragment_pin_worker *worker = arg; + + if (worker->task) { + kthread_stop(worker->task); + worker->task = NULL; + } +} + +static void user4k_fragment_pin_race_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct fragment_test_pins *pins; + struct fragment_pin_worker *workers; + unsigned int i, round, migrated = 0, busy = 0; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 4); + KUNIT_ASSERT_EQ(test, ret, 4L); + for (i = 0; i < 4; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0x51 + i, pins->fragments[i].length); + fragment_test_unpin(pins); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].mm = vma->vm_mm; + init_completion(&workers[i].ready); + workers[i].task = kthread_run(fragment_pin_worker_run, &workers[i], "user4k-pin-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "pin worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_pin_worker_stop, &workers[i]), 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + } + for (round = 0; round < 32; round++) { + struct mm_struct *child = test_vma_mm(test); + unsigned long addr = TEST_VA + (round % 4) * SZ_4K; + struct vm_area_struct *cvma; + struct folio *folio; + unsigned int succeeded = 0; + vm_fault_t fault; + struct user_page_fragment write_fragment; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + cvma = test_vma_lookup(child, addr); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + ret = pin_user_fragments_remote(child, addr, 1, FOLL_WRITE, &write_fragment, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + *((u8 *)folio_address(write_fragment.folio) + write_fragment.offset) = 0xa0 + round; + release_user_fragments(&write_fragment, 1, true); + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + if (ret) { + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_ASYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_TRUE(test, ret == 0 || ret == 1); + migrated += succeeded; + busy += ret == 1; + } + kunit_release_action(test, user4k_vma_mm_free, child); + } + for (i = 0; i < 2; i++) { + fragment_pin_worker_stop(&workers[i]); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_GE(test, workers[i].checks, 4UL); + } + kunit_info(test, "32 fork/COW/exit cycles against two pin workers; migrations=%u not_migrated=%u\n", + migrated, busy); +} + +#endif + +static int test_swap_control(const char *operation) +{ + char *argv[] = { "/init", (char *)operation, NULL }; + char *envp[] = { NULL }; + + return call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC); +} + +static void test_reenable_swap(void *arg) +{ + bool *disabled = arg; + + if (*disabled) + test_swap_control("--swapon-helper"); +} + +/* Exclusive slots must remain packed after read swap-in and later writes. */ +static void test_user4k_swap_exclusive(struct kunit *test, unsigned int mode) +{ + struct vm_area_struct *vma; + struct folio *folio; + struct mm_subpage *held = NULL; + bool *disabled; + phys_addr_t restored[TEST_SLOTS]; + swp_entry_t entry; + vm_fault_t fault; + unsigned int i; + int ret, mapped, hold_err = 0; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0xc0 + i, SZ_4K); + test_swap_tags_set(folio); + if (mode == 5) { + held = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(folio)) + 2 * SZ_4K); + if (held) { + hold_err = kunit_add_action_or_reset(test, test_subpage_put, held); + if (hold_err) + held = NULL; + } + } + ret = folio_alloc_swap(folio); + entry = folio->swap; + if (!ret) + try_to_unmap(folio, TTU_SYNC); + mapped = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + if (mode == 5) { + KUNIT_ASSERT_EQ(test, hold_err, 0); + KUNIT_ASSERT_NOT_NULL(test, held); + } + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(pte))); + KUNIT_EXPECT_TRUE(test, pte_swp_exclusive(pte)); + } + if (mode >= 3) { + struct mm_struct *child = test_vma_mm(test); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + } + } + if (mode == 1 || mode == 2 || mode == 4 || mode == 6) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + if (mode == 2 || mode == 6) { + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, *disabled); + } + for (i = 0; i < TEST_SLOTS; i++) { + struct mm_subpage *slot; + unsigned long address = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + restored[i] = pte_phys_mm(vma->vm_mm, pte); + slot = mm_subpage_get_from_phys(restored[i]); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(slot), !(held && i == 2)); + KUNIT_EXPECT_EQ(test, restored[i], restored[0] + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xc0 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(slot), i)); + mm_subpage_put(slot); + } + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long address = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + if (held && i == 2) { + KUNIT_EXPECT_NE(test, pte_phys_mm(vma->vm_mm, pte), restored[i]); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), 0xdd, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(held)) + + mm_subpage_offset(held), 0xc2, SZ_4K), NULL); + } else { + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), restored[i]); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0UL); + if (*disabled) { + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_EXPECT_EQ(test, ret, 0); + } +} + +static void user4k_swap_exclusive_cached_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 0); +} + +static void user4k_swap_exclusive_disk_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 1); +} + +static void user4k_swap_exclusive_swapoff_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 2); +} + +static void user4k_swap_reuse_cached_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 3); +} + +static void user4k_swap_reuse_disk_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 4); +} + +static void user4k_swap_reuse_get_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 5); +} + +static void user4k_swap_reuse_swapoff_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 6); +} + +static void test_user4k_swap_roundtrip(struct kunit *test, unsigned int mode, u8 base) +{ + const bool evict = mode != 0; + struct vm_area_struct *parent, *cvma; + struct mm_struct *child; + struct folio *folio; + swp_entry_t entry = { 0 }; + unsigned int i, count; + int err, mapped; + vm_fault_t fault; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "rerun this suite after enabling the disposable guest swap disk"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, base + i, SZ_4K); + test_swap_tags_set(folio); + if (mode == 2) { + LIST_HEAD(folios); + + folio_unlock(folio); + err = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!err) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Let the normal reclaim path allocate swap and unmap all slots. */ + reclaim_pages(&folios); + mapped = 0; + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + mapped += pte_present(pte); + } + entry = softleaf_from_pte(pte); + } else { + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC | (evict ? TTU_BATCH_FLUSH : 0)); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + try_to_unmap_flush(); + folio_put(folio); + } + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_ANONPAGES), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_swp_exclusive(pte)); + } + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, (unsigned int)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + if (evict) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + + /* Fork while all slots are nonpresent, preserving each byte offset. */ + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, 2U * TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 2U); + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long address = TEST_VA + ((i * 5 + 2) % TEST_SLOTS) * SZ_4K; + struct mm_subpage *slot; + spinlock_t *ptl; + pte_t *ptep; + bool intact = false; + + pte = test_fault_entry(test, parent->vm_mm, address); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + pte = test_fault_entry(test, child, address); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), ((i * 5 + 2) % TEST_SLOTS) * (unsigned int)SZ_4K); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + KUNIT_EXPECT_EQ(test, !!(fault & VM_FAULT_MAJOR), evict && !i); + ptep = test_mm_lookup(parent->vm_mm, address, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + pte = ptep_get(ptep); + slot = pte_present(pte) ? mm_subpage_get_from_phys(pte_phys_mm(parent->vm_mm, pte)) : NULL; + if (slot) { + intact = !memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), base + ((i * 5 + 2) % TEST_SLOTS), SZ_4K) && + test_swap_tags_match(mm_subpage_folio(slot), ((i * 5 + 2) % TEST_SLOTS)); + mm_subpage_put(slot); + } + pte_unmap_unlock(ptep, ptl); + KUNIT_EXPECT_TRUE(test, intact); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), 0UL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + /* A write fault must swap in and then COW just the child's selected slot. */ + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + SZ_4K, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, 1)); + memset(folio_address(folio) + SZ_4K, 0xa7, SZ_4K); + folio_put(folio); + folio = test_fault_folio(parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), TEST_SLOTS - 1UL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), i == 1 ? 0U : 1U); + /* Exit with three swapped slots, then tear down the restored parent. */ + kunit_release_action(test, user4k_vma_mm_free, child); + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, 0U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 0U); + kunit_release_action(test, user4k_vma_mm_free, parent->vm_mm); +} + +static void user4k_swap_mremap_test(struct kunit *test) +{ + struct mm_struct *mm; + struct folio *folio; + unsigned long old = TEST_VA, new = SZ_1G - SZ_4K, mapped, moved; + pte_t saved[TEST_SLOTS]; + u8 byte = 0x55; + int ret; + unsigned int i; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, old, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old, &byte, 1, FOLL_WRITE), 1); + folio = test_fault_folio(mm, old); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x81 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < TEST_SLOTS; i++) { + saved[i] = test_fault_entry(test, mm, old + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(saved[i]))); + } + kthread_use_mm(mm); + moved = test_mremap(old, PAGE_SIZE, PAGE_SIZE, MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte_t entry = test_fault_entry(test, mm, new + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, softleaf_from_pte(entry).val, softleaf_from_pte(saved[i]).val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(entry), i * (unsigned int)SZ_4K); + } + for (i = 0; i < TEST_SLOTS; i++) { + unsigned int slot = (i + 2) % TEST_SLOTS; + unsigned long addr = new + slot * SZ_4K; + + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, addr, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0x81 + slot)); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, slot)); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); +} + +static void user4k_swap_pin_test(struct kunit *test) +{ + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + struct folio *folio; + phys_addr_t pinned_phys; + unsigned int i; + vm_fault_t fault; + long ret; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 4); + KUNIT_ASSERT_EQ(test, ret, 4L); + for (i = 0; i < 4; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0xb1 + i, pins->fragments[i].length); + test_swap_tags_set(pins->fragments[0].folio); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA + SZ_4K, SZ_4K, FOLL_WRITE, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + pinned_phys = mm_subpage_phys(pins->fragments[0].subpage); + folio = pins->fragments[0].folio; + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), pinned_phys); + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(pins->fragments[0].subpage)); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(pins->fragments[0].subpage), 1U); + memset(folio_address(folio) + SZ_4K, 0xd2, SZ_4K); + fragment_test_unpin(pins); + /* Once released, the formerly pinned PTE can be swapped normally. */ + folio = test_fault_folio(vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + ret = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0L); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + i == 1 ? 0xd2 : 0xb1 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } +} + +#ifdef CONFIG_MIGRATION +static void user4k_swap_migration_test(struct kunit *test) +{ + struct vm_area_struct *vma, *cvma; + struct mm_struct *child; + struct folio *folio; + phys_addr_t old_phys; + unsigned int i, side, succeeded = 0; + vm_fault_t fault; + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + old_phys = PFN_PHYS(folio_pfn(folio)); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0xa1 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + SZ_4K, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_swapcache(folio)); + ret = isolate_folio_to_list(folio, &folios) ? 0 : -EBUSY; + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + /* One present slot and seven swap entries must all follow the new cache folio. */ + for (side = 0; side < 2; side++) { + struct vm_area_struct *area = side ? cvma : vma; + struct mm_struct *mm = area->vm_mm; + + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + + mmap_read_lock(mm); + fault = handle_mm_fault(area, addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(folio)), old_phys); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0xa1 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + } +} +#endif + +static void user4k_swap_cached_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 0, 0x51); +} + +static void user4k_swap_disk_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 1, 0x61); +} + +static void user4k_swap_reclaim_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 2, 0x71); +} + +static void user4k_swap_stress_test(struct kunit *test) +{ + unsigned int i; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (i = 0; i < 32; i++) + test_user4k_swap_roundtrip(test, i % 3, 0x21 + i * 5); + kunit_info(test, "32 swap/fork/COW/exit cycles with distinct data patterns\n"); +} + +static void user4k_swapoff_test(struct kunit *test) +{ + struct vm_area_struct *parent, *cvma; + struct mm_struct *child; + struct folio *folio; + bool *disabled; + vm_fault_t fault; + unsigned int i, side; + swp_entry_t entry; + int ret; + pte_t pte; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x91 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_fault_entry(test, parent->vm_mm, TEST_VA); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, *disabled); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + i * (unsigned long)SZ_4K); + folio = test_fault_folio(mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x91 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } + } + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + 3 * SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + 3 * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), pte_phys_mm(parent->vm_mm, + test_fault_entry(test, parent->vm_mm, TEST_VA + 3 * SZ_4K))); + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_EXPECT_EQ(test, ret, 0); +} + +static void user4k_swapoff_boundaries_test(struct kunit *test) +{ + const unsigned long boundaries[] = { SZ_2M, SZ_1G, 1UL << 39 }; + bool *disabled; + unsigned int boundary; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + for (boundary = 0; boundary < ARRAY_SIZE(boundaries); boundary++) { + unsigned long start = boundaries[boundary] - SZ_4K; + struct vm_area_struct *parent = test_fault_vma(test, start, 2 * SZ_4K); + struct mm_struct *child; + unsigned int i, side; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, parent); + for (i = 0; i < 2; i++) { + unsigned long addr = start + i * SZ_4K; + unsigned int offset = vma_page_offset_at(parent, addr).offset; + struct folio *folio; + vm_fault_t fault; + swp_entry_t entry; + LIST_HEAD(folios); + + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + memset(folio_address(folio) + offset, 0xb0 + boundary * 2 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = softleaf_from_pte(test_fault_entry(test, parent->vm_mm, addr)); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + } + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 2UL); + for (i = 0; i < 2; i++) { + unsigned long addr = start + i * SZ_4K; + unsigned int offset = vma_page_offset_at(parent, addr).offset; + pte_t pte = test_fault_entry(test, mm, addr); + struct folio *folio; + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + (unsigned long)offset); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0xb0 + boundary * 2 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, offset / SZ_4K)); + folio_put(folio); + } + } + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, parent->vm_mm); + } +} + +struct live_swap_reader { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready, resume, entered, done; + unsigned int id, checks; + int error; +}; + +static int live_swap_reader_run(void *arg) +{ + struct live_swap_reader *r = arg; + unsigned int phase, i; + + kthread_use_mm(r->mm); + for (phase = 0; phase < 2; phase++) { + if (phase) + complete(&r->entered); + for (i = 0; i < 4; i++) { + u8 value = 0; + void __user *addr = (void __user *)(TEST_VA + i * SZ_4K + 64 + r->id); + + /* Resolve AF, swap and COW faults through the real uaccess path. */ + if (copy_from_user(&value, addr, 1) || value != 0xd1 + i) + r->error = phase ? 2 : 1; + r->checks++; + } + if (!phase) { + complete(&r->ready); + if (!wait_for_completion_timeout(&r->resume, 15 * HZ)) { + r->error = 3; + goto out; + } + } + } + { + u8 value = 0xe0 + r->id; + void __user *addr = (void __user *)(TEST_VA + SZ_4K + 64 + r->id); + + if (copy_to_user(addr, &value, 1)) + r->error = 4; + r->checks++; + } +out: + kthread_unuse_mm(r->mm); + complete(&r->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void live_swap_reader_stop(void *arg) +{ + struct live_swap_reader *r = arg; + + if (r->task) { + complete_all(&r->resume); + kthread_stop(r->task); + r->task = NULL; + } +} + +static void test_live_fault_roundtrip(struct kunit *test, unsigned int mode) +{ + bool migration = mode != 0; + struct vm_area_struct *parent; + struct mm_struct *child; + struct live_swap_reader *readers; + struct folio *folio; + vm_fault_t fault; + unsigned int i, cpu; + swp_entry_t entry; + u64 tcr = read_sysreg(tcr_el1); + int ret; + LIST_HEAD(folios); + + if ((!migration && !total_swap_pages) || !user4k_test_enabled || num_online_cpus() < 2 || + !system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires two CPUs, compatible opt-in 4K TTBR0 controls, and swap for reclaim"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < 4; i++) + memset(folio_address(folio) + i * SZ_4K, 0xd1 + i, SZ_4K); + folio_put(folio); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + readers = kunit_kcalloc(test, 2, sizeof(*readers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, readers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 2; i++) { + readers[i].mm = child; + readers[i].id = i; + init_completion(&readers[i].ready); + init_completion(&readers[i].resume); + init_completion(&readers[i].entered); + init_completion(&readers[i].done); + readers[i].task = kthread_create(live_swap_reader_run, &readers[i], + "user4k-swap-read-%u", i); + if (IS_ERR(readers[i].task)) { + readers[i].task = NULL; + KUNIT_FAIL(test, "live swap worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, live_swap_reader_stop, + &readers[i]), 0); + kthread_bind(readers[i].task, cpu); + cpu = cpumask_next(cpu, cpu_online_mask); + wake_up_process(readers[i].task); + } + for (i = 0; i < 2; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].ready, 10 * HZ), 0UL); + lru_add_drain_all(); + folio = test_fault_folio(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + if (mode == 2) { +#ifdef CONFIG_MIGRATION + struct folio *dst = folio_alloc(GFP_KERNEL, 0); + bool entered = true, blocked = true, unmapped = false; + + if (!dst) { + folio_put(folio); + KUNIT_FAIL(test, "migration waiter destination allocation failed"); + return; + } + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + if (!ret) { + try_to_migrate(folio, TTU_SYNC); + unmapped = !folio_mapped(folio); + for (i = 0; i < 2; i++) + complete(&readers[i].resume); + for (i = 0; i < 2; i++) { + entered &= !!wait_for_completion_timeout(&readers[i].entered, HZ); + if (wait_for_completion_timeout(&readers[i].done, max(1UL, HZ / 20UL))) { + blocked = false; + complete(&readers[i].done); + } + } + remove_migration_ptes(folio, folio, 0); + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, entered); + KUNIT_EXPECT_TRUE(test, blocked); + KUNIT_EXPECT_TRUE(test, unmapped); +#endif + } else if (migration) { +#ifdef CONFIG_MIGRATION + unsigned int succeeded = 0; + phys_addr_t old_phys = PFN_PHYS(folio_pfn(folio)); + + ret = isolate_folio_to_list(folio, &folios) ? 0 : -EBUSY; + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_ASYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, test_fault_entry(test, child, TEST_VA)), old_phys); +#endif + } else { + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = softleaf_from_pte(test_fault_entry(test, child, TEST_VA)); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + } + for (i = 0; i < 2; i++) + complete(&readers[i].resume); + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].done, 15 * HZ), 0UL); + live_swap_reader_stop(&readers[i]); + KUNIT_EXPECT_EQ(test, readers[i].error, 0); + KUNIT_EXPECT_EQ(test, readers[i].checks, 9U); + } + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + SZ_4K + 64 + i), (u8)(0xe0 + i)); + folio_put(folio); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA + SZ_4K, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xd2, SZ_4K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 4UL); + kunit_info(test, "two CPUs access through uaccess after %s, then write one child slot\n", + mode == 2 ? "migration-entry rollback" : migration ? "migration" : "reclaim"); +} + +static void user4k_live_swap_fault_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 0); +} + +#ifdef CONFIG_MIGRATION +static void user4k_live_migration_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 1); +} + +static void user4k_migration_wait_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 2); +} +#endif + +/* Transfer PTE ownership to normal mm teardown, which also handles swap. */ +static int anon_slot_fixture_handoff(struct kunit *test, struct anon_slot_fixture *f) +{ + struct mm_struct *mm = f->mm; + + f->mm = NULL; + f->vma = NULL; + memset(f->slots, 0, sizeof(f->slots)); + memset(f->mapped, 0, sizeof(f->mapped)); + return kunit_add_action_or_reset(test, user4k_vma_mm_free, mm); +} + +static void test_nonlinear_swap(struct kunit *test, bool evict) +{ + unsigned int first; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (first = 0; first < 2; first++) { + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct vm_area_struct *vmas[2]; + struct mm_struct *child; + struct mm_subpage *slot; + struct folio *folio; + swp_entry_t entry = { }; + unsigned int side; + vm_fault_t fault; + int err, mapped; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + vmas[0] = a->vma; + vmas[1] = b->vma; + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + err = mem_cgroup_charge(folio, a->mm, GFP_KERNEL); + if (err) { + mm_subpage_put(slot); + KUNIT_FAIL(test, "cannot charge swap fixture: %d", err); + return; + } + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + folio_lock(folio); + mm_subpage_set_exclusive(slot); + folio_unlock(folio); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + folio_lock(folio); + mm_subpage_set_exclusive(slot); + folio_unlock(folio); + memset(folio_address(folio), 0x5b, SZ_4K); + memset(folio_address(folio) + 3 * SZ_4K, 0xd3, SZ_4K); + test_swap_tags_set(folio); + folio_mark_dirty(folio); + folio_add_lru(folio); + lru_add_drain_all(); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_handoff(test, a), 0); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_handoff(test, b), 0); + folio_get(folio); + folio_lock(folio); + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + for (side = 0; side < 2; side++) { + pte_t pte = test_fault_entry(test, vmas[side]->vm_mm, + TEST_VA + side * SZ_4K); + + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), side ? 3U * SZ_4K : 0U); + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, side ? 3 * SZ_4K : 0), 1U); + } + if (evict) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + /* Preserve physical offsets when a nonpresent nonlinear slot is forked. */ + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vmas[1]->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + for (side = 0; side < 2; side++) { + unsigned int which = side ^ first, physical = which ? 3 : 0; + struct vm_area_struct *vma = vmas[which]; + unsigned long address = TEST_VA + which * SZ_4K; + pte_t pte; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ_MSG(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), + (vm_fault_t)0, "evict=%d first=%u side=%u", evict, first, which); + KUNIT_EXPECT_EQ(test, !!(fault & VM_FAULT_MAJOR), evict && !side); + pte = test_fault_entry(test, vma->vm_mm, address); + slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), physical * (unsigned int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), which ? 0xd3 : 0x5b, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(slot), physical)); + mm_subpage_put(slot); + } + /* A child's write must copy only its slot at the new virtual offset. */ + { + struct vm_area_struct *vma = test_vma_lookup(child, TEST_VA + SZ_4K); + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(child); + fault = handle_mm_fault(vma, TEST_VA + SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + slot = mm_subpage_get_from_phys(pte_phys_mm(child, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xd3, SZ_4K), NULL); + memset(folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot), 0x91, SZ_4K); + mm_subpage_put(slot); + pte = test_fault_entry(test, vmas[1]->vm_mm, TEST_VA + SZ_4K); + slot = mm_subpage_get_from_phys(pte_phys_mm(vmas[1]->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xd3, SZ_4K), NULL); + mm_subpage_put(slot); + } + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vmas[1]->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, vmas[0]->vm_mm); + } +} + +static void user4k_nonlinear_swap_cached_test(struct kunit *test) +{ + test_nonlinear_swap(test, false); +} + +static void user4k_nonlinear_swap_disk_test(struct kunit *test) +{ + test_nonlinear_swap(test, true); +} + +#ifdef CONFIG_USERFAULTFD +/* The context is attached only during the call; ordinary mm cleanup owns the VMAs. */ +static ssize_t test_uffd_move(struct vm_area_struct *dst, unsigned long dst_addr, + unsigned long src_addr, unsigned long size) +{ + struct userfaultfd_ctx ctx = { .mm = dst->vm_mm }; + ssize_t ret; + + init_rwsem(&ctx.map_changing_lock); + atomic_set(&ctx.mmap_changing, 0); + mmap_write_lock(ctx.mm); + vma_start_write(dst); + dst->vm_userfaultfd_ctx.ctx = &ctx; + mmap_write_unlock(ctx.mm); + ret = move_pages(&ctx, dst_addr, src_addr, size, 0); + mmap_write_lock(ctx.mm); + vma_start_write(dst); + dst->vm_userfaultfd_ctx.ctx = NULL; + mmap_write_unlock(ctx.mm); + return ret; +} + +struct move_migration_worker { + struct userfaultfd_ctx ctx; + struct vm_area_struct *vma[2]; + struct task_struct *task; + struct completion ready; + unsigned long address[2], size; + unsigned long moves, busy; + int error; +}; + +static int move_migration_worker_run(void *arg) +{ + struct move_migration_worker *w = arg; + + while (!kthread_should_stop()) { + unsigned int side = w->moves & 1; + ssize_t ret = move_pages(&w->ctx, w->address[side ^ 1], + w->address[side], w->size, 0); + + if (ret == w->size) + WRITE_ONCE(w->moves, w->moves + 1); + else if (ret == -EAGAIN || ret == -EBUSY) + w->busy++; + else + w->error = ret < 0 ? ret : -EIO; + if (w->moves || w->error) + complete_all(&w->ready); + if (w->error) { + /* Keep the task valid until the owner joins it. */ + set_current_state(TASK_INTERRUPTIBLE); + if (!kthread_should_stop()) + schedule(); + __set_current_state(TASK_RUNNING); + } + /* Leave bounded gaps so migration can complete between contended moves. */ + if (ret == w->size) + usleep_range(1000, 2000); + else + cond_resched(); + } + return 0; +} + +static void move_migration_worker_stop(void *arg) +{ + struct move_migration_worker *w = arg; + unsigned int i; + + if (w->task) { + kthread_stop(w->task); + w->task = NULL; + } + mmap_write_lock(w->ctx.mm); + for (i = 0; i < 2; i++) { + vma_start_write(w->vma[i]); + w->vma[i]->vm_userfaultfd_ctx.ctx = NULL; + } + mmap_write_unlock(w->ctx.mm); +} + +static void user4k_move_migration_race_test(struct kunit *test) +{ + unsigned int shift; + + for (shift = 12; shift < PAGE_SHIFT; shift += 2) { + struct mm_struct *mm = test_vma_mm_granule(test, shift); + struct move_migration_worker *w; + unsigned long size = 1UL << shift, i, initial_moves; + unsigned int round, migrated = 0, failed = 0; + pte_t pte; + vm_fault_t fault; + + KUNIT_ASSERT_NOT_NULL(test, mm); + w = kunit_kzalloc(test, sizeof(*w), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, w); + w->ctx.mm = mm; + init_rwsem(&w->ctx.map_changing_lock); + atomic_set(&w->ctx.mmap_changing, 0); + init_completion(&w->ready); + w->size = size; + w->address[0] = TEST_VA + 3 * size; + w->address[1] = TEST_VA + SZ_128M + size; + w->vma[0] = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + w->vma[1] = test_fault_vma_in_mm(mm, TEST_VA + SZ_128M, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, w->vma[0]); + KUNIT_ASSERT_NOT_NULL(test, w->vma[1]); + for (i = 0; i < PAGE_SIZE; i += size) { + mmap_read_lock(mm); + fault = handle_mm_fault(w->vma[0], TEST_VA + i, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, mm, TEST_VA + i); + memset(phys_to_virt(pte_phys_mm(mm, pte)), 0x60 + i / size, size); + } + /* Both directions are registered before starting the concurrent mover. */ + mmap_write_lock(mm); + for (i = 0; i < 2; i++) { + vma_start_write(w->vma[i]); + w->vma[i]->vm_userfaultfd_ctx.ctx = &w->ctx; + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, move_migration_worker_stop, w), 0); + w->task = kthread_create(move_migration_worker_run, w, "user4k-move"); + if (IS_ERR(w->task)) { + w->task = NULL; + KUNIT_FAIL(test, "MOVE worker creation failed"); + return; + } + if (num_online_cpus() > 1) + kthread_bind(w->task, cpumask_last(cpu_online_mask)); + wake_up_process(w->task); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->ready, 10 * HZ), 0UL); + initial_moves = READ_ONCE(w->moves); + lru_add_drain_all(); + for (round = 0; round < 64; round++) { + struct folio *folio; + unsigned int succeeded = 0; + bool isolated; + int ret; + LIST_HEAD(folios); + + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE_MSG(test, isolated, "round=%u", round); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_GE(test, ret, 0); + migrated += succeeded; + failed += ret > 0; + cond_resched(); + } + kunit_release_action(test, move_migration_worker_stop, w); + kunit_info(test, "%uK MOVE/migration: moves=%lu retries=%lu migrated=%u busy=%u\n", + 1U << (shift - 10), w->moves, w->busy, migrated, failed); + KUNIT_EXPECT_EQ(test, w->error, 0); + KUNIT_EXPECT_GT(test, w->moves, initial_moves); + KUNIT_EXPECT_GT(test, migrated, 0U); + for (i = 0; i < PAGE_SIZE; i += size) { + unsigned long addr = i == 3 * size ? w->address[w->moves & 1] : TEST_VA + i; + + pte = test_fault_entry(test, mm, addr); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(mm, pte)), + 0x60 + i / size, size), NULL); + } + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, w->address[(w->moves & 1) ^ 1]))); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_move_swap_get_test(struct kunit *test) +{ + unsigned int variant; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (variant = 0; variant < PAGE_SHIFT - 12; variant++) { + unsigned int shift = 12 + (variant / 2) * 2; + bool forked = variant & 1; + unsigned long size = 1UL << shift; + unsigned long src_addr = TEST_VA + 3 * size; + unsigned long dst_addr = TEST_VA + SZ_128M + size; + struct mm_struct *mm = test_vma_mm_granule(test, shift); + struct vm_area_struct *src, *dst; + struct mm_subpage *held; + struct folio *folio; + phys_addr_t physical; + swp_entry_t entry = { }; + pte_t before, after; + vm_fault_t fault; + unsigned long i; + int err, mapped; + + KUNIT_ASSERT_NOT_NULL(test, mm); + src = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + dst = test_fault_vma_in_mm(mm, TEST_VA + SZ_128M, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, src); + KUNIT_ASSERT_NOT_NULL(test, dst); + for (i = 0; i < PAGE_SIZE; i += size) { + mmap_read_lock(mm); + fault = handle_mm_fault(src, TEST_VA + i, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + before = test_fault_entry(test, mm, TEST_VA + i); + memset(phys_to_virt(pte_phys_mm(mm, before)), 0x50 + i / size, size); + } + before = test_fault_entry(test, mm, src_addr); + physical = pte_phys_mm(mm, before); + held = mm_subpage_get_from_phys(physical); + KUNIT_ASSERT_NOT_NULL(test, held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + KUNIT_ASSERT_EQ(test, mm_subpage_offset(held), (unsigned int)(3 * size)); + folio = mm_subpage_folio(held); + lru_add_drain_all(); + folio_lock(folio); + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + KUNIT_ASSERT_EQ(test, mm_subpage_mapcount(held), 0); + KUNIT_ASSERT_EQ(test, swap_subpage_count(entry, 3 * size), 1U); + before = test_fault_entry(test, mm, src_addr); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(before))); + if (forked) { + struct mm_struct *child = test_vma_mm_granule(test, shift); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, test_uffd_move(dst, dst_addr, src_addr, size), + (ssize_t)-EBUSY); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(test_fault_entry(test, mm, + src_addr)).val, entry.val); + kunit_release_action(test, user4k_vma_mm_free, child); + before = test_fault_entry(test, mm, src_addr); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(before)); + KUNIT_ASSERT_EQ(test, swap_subpage_count(entry, 3 * size), 1U); + } + KUNIT_ASSERT_EQ(test, test_uffd_move(dst, dst_addr, src_addr, size), (ssize_t)size); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(after).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(after), (unsigned int)(3 * size)); + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, src_addr))); + /* Restore with the retained metadata already rebound to the new root. */ + mmap_read_lock(mm); + fault = handle_mm_fault(dst, dst_addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_ASSERT_TRUE(test, pte_present(after)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after), physical); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(physical), 0x53, size), NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(held), !forked); + /* Fork-cleared exclusivity requires COW while a GET is retained. */ + mmap_read_lock(mm); + fault = handle_mm_fault(dst, dst_addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_ASSERT_TRUE(test, pte_write(after)); + if (forked) + KUNIT_EXPECT_NE(test, pte_phys_mm(mm, after), physical); + else + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after), physical); + memset(phys_to_virt(pte_phys_mm(mm, after)), 0xa9, size); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(physical), + forked ? 0x53 : 0xa9, size), NULL); + for (i = 0; i < PAGE_SIZE; i += size) { + if (i == 3 * size) + continue; + mmap_read_lock(mm); + fault = handle_mm_fault(src, TEST_VA + i, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, TEST_VA + i); + KUNIT_ASSERT_TRUE(test, pte_present(after)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(mm, after)), + 0x50 + i / size, size), NULL); + } + kunit_release_action(test, test_subpage_put, held); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} +#endif + +static struct kunit_case user4k_swap_cases[] = { + KUNIT_CASE(user4k_swap_reuse_get_test), + KUNIT_CASE(user4k_swap_reuse_swapoff_test), + KUNIT_CASE(user4k_swap_reuse_cached_test), + KUNIT_CASE(user4k_swap_reuse_disk_test), + KUNIT_CASE(user4k_swap_exclusive_cached_test), + KUNIT_CASE(user4k_swap_exclusive_disk_test), + KUNIT_CASE(user4k_swap_exclusive_swapoff_test), + KUNIT_CASE(user4k_swap_cached_test), + KUNIT_CASE(user4k_swap_pin_test), + KUNIT_CASE(user4k_swap_mremap_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_swap_migration_test), +#endif + KUNIT_CASE(user4k_swap_disk_test), + KUNIT_CASE(user4k_swap_reclaim_test), + KUNIT_CASE(user4k_swapoff_test), + KUNIT_CASE(user4k_swap_stress_test), + KUNIT_CASE(user4k_swapoff_boundaries_test), + KUNIT_CASE(user4k_live_swap_fault_test), + {} +}; + +static void user4k_swap_suite_exit(struct kunit_suite *suite) +{ + kvfree_rcu_barrier(); +} + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +static void test_shmem_uffd_swap(struct kunit *test, unsigned int mode) +{ + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, mode == 3 ? 2 : 1); + struct swap_iocb *plug = NULL; + struct file *file; + struct folio *folio; + void *entry; + bool *disabled; + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires disposable guest swap"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + file = test_cache_file(test, PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xb6, mode == 3 ? 2 * SZ_4K : SZ_4K); + ret = shmem_uffd_set_missing(folio, 0, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + /* Match pageout(): submitted data is clean unless writeout redirties it. */ + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + ret = shmem_writeout(folio, &plug, &folios); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_TRUE(test, list_empty(&folios)); + entry = filemap_get_entry(file->f_mapping, 0); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + KUNIT_EXPECT_EQ(test, xa_to_value(xa_load(&SHMEM_I(file_inode(file))->uffd_missing, 0)), missing); + if (mode == 1) { + shmem_truncate_range(file_inode(file), 0, PAGE_SIZE - 1); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + } else if (mode == 3) { + shmem_truncate_range(file_inode(file), 0, SZ_4K - 1); + } else if (mode == 2) { + ret = test_swap_control("--swapoff-helper"); + *disabled = ret == 0; + KUNIT_ASSERT_EQ(test, ret, 0); + } + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), + mode == 1 ? 0UL : mode == 3 ? missing | BIT(0) : missing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), + mode == 1 || mode == 3 ? 0 : 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, + mode == 3 ? 0xb6 : 0, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + 2 * SZ_4K, + 0, PAGE_SIZE - 2 * SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); +} + +static void user4k_shmem_uffd_swapin_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 0); +} + +static void user4k_shmem_uffd_swapped_truncate_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 1); +} + +static void user4k_shmem_uffd_swapped_partial_hole_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 3); +} + +static void user4k_shmem_uffd_swapoff_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 2); +} + +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAULT_INJECTION_DEBUG_FS) +static void test_disable_failslab(void *unused) +{ + test_swap_control("--failslab-off-helper"); +} +#endif + +static void user4k_shmem_uffd_swap_alloc_fail_test(struct kunit *test) +{ +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAULT_INJECTION_DEBUG_FS) + unsigned int attempt; + + if (!total_swap_pages) { + kunit_skip(test, "requires disposable guest swap"); + return; + } + KUNIT_ASSERT_EQ(test, test_swap_control("--failslab-on-helper"), 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_disable_failslab, NULL), 0); + for (attempt = 1; attempt <= 2; attempt++) { + pgoff_t index = attempt == 1 ? 1 : 64; + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + struct file *file = test_cache_file(test, (index + 1) * PAGE_SIZE, false); + struct swap_iocb *plug = NULL; + struct folio *folio; + unsigned int pending; + unsigned long pfn; + void *entry; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), index, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xb6, SZ_4K); + ret = shmem_uffd_set_missing(folio, index, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, index); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + pfn = folio_pfn(folio); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + /* Fail this task's selected allocation and its allocator retries. */ + WRITE_ONCE(current->make_it_fail, 1); + WRITE_ONCE(current->fail_nth, attempt); + ret = shmem_writeout(folio, &plug, &folios); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + WRITE_ONCE(current->make_it_fail, 0); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_EXPECT_EQ(test, ret, AOP_WRITEPAGE_ACTIVATE); + if (ret != AOP_WRITEPAGE_ACTIVATE) { + if (plug) + swap_write_unplug(plug); + folio_wait_writeback(folio); + folio_put(folio); + return; + } + KUNIT_EXPECT_TRUE(test, folio_test_locked(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + KUNIT_EXPECT_PTR_EQ(test, folio->mapping, file->f_mapping); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, index), missing); + /* A failed multi-level insertion may retain an empty internal node. */ + { + unsigned long cursor = 0; + + KUNIT_EXPECT_PTR_EQ(test, xa_find(&SHMEM_I(file_inode(file))->uffd_missing, + &cursor, ULONG_MAX, XA_PRESENT), NULL); + } + KUNIT_EXPECT_PTR_EQ(test, plug, NULL); + KUNIT_EXPECT_TRUE(test, list_empty(&folios)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, PAGE_SIZE - SZ_4K), NULL); + /* No restart or cache repair: retry exactly the retained folio. */ + KUNIT_EXPECT_EQ(test, folio_pfn(folio), pfn); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + ret = shmem_writeout(folio, &plug, &folios); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = filemap_get_entry(file->f_mapping, index); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + KUNIT_EXPECT_EQ(test, xa_to_value(xa_load(&SHMEM_I(file_inode(file))->uffd_missing, index)), missing); + ret = shmem_get_folio(file_inode(file), index, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, index), missing); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, PAGE_SIZE - SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); + kunit_info(test, "failed metadata allocation %u at index %lu; same-folio retry and disk swap-in preserved bytes/mask", attempt, index); + } +#else + kunit_skip(test, "requires FAILSLAB and fault-injection debugfs"); +#endif +} + +#ifdef CONFIG_MEMCG +static void test_memcg_put(void *memcg) +{ + mem_cgroup_put(memcg); +} +#endif + +static void user4k_shmem_uffd_swap_charge_test(struct kunit *test) +{ +#ifdef CONFIG_MEMCG + struct cgroup *cgroup; + struct cgroup_subsys_state *css; + struct mem_cgroup *memcg, *saved; + struct file *file; + struct folio *folio; + struct swap_iocb *plug = NULL; + void *entry; + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages || mem_cgroup_disabled()) { + kunit_skip(test, "requires disposable guest swap and memcg"); + return; + } + KUNIT_ASSERT_EQ(test, test_swap_control("--memcg-setup-helper"), 0); + cgroup = cgroup_get_from_path("/uffd-metadata"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cgroup); + css = cgroup_get_e_css(cgroup, &memory_cgrp_subsys); + cgroup_put(cgroup); + KUNIT_ASSERT_NOT_NULL(test, css); + memcg = mem_cgroup_from_css(css); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_memcg_put, memcg), 0); + KUNIT_ASSERT_PTR_NE(test, memcg, root_mem_cgroup); + file = test_cache_file(test, 65 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + saved = set_active_memcg(memcg); + ret = shmem_get_folio(file_inode(file), 64, 0, &folio, SGP_CACHE); + set_active_memcg(saved); + KUNIT_ASSERT_EQ(test, ret, 0); + rcu_read_lock(); + KUNIT_EXPECT_PTR_EQ(test, folio_memcg(folio), memcg); + rcu_read_unlock(); + ret = shmem_uffd_set_missing(folio, 64, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, 64); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + /* Reclaim from the root charge domain, not the folio's cgroup. */ + saved = set_active_memcg(root_mem_cgroup); + ret = shmem_writeout(folio, &plug, &folios); + set_active_memcg(saved); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + rcu_read_lock(); + entry = rcu_dereference(SHMEM_I(file_inode(file))->uffd_missing.xa_head); + if (xa_is_node(entry)) + KUNIT_EXPECT_PTR_EQ(test, mem_cgroup_from_virt(xa_to_node(entry)), memcg); + rcu_read_unlock(); + KUNIT_ASSERT_TRUE(test, xa_is_node(entry)); + entry = filemap_get_entry(file->f_mapping, 64); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + ret = shmem_get_folio(file_inode(file), 64, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 64), missing); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); +#else + kunit_skip(test, "requires memcg"); +#endif +} + +static bool test_coarse_swap_enabled(struct kunit *test) +{ + if (!test_coarse_file_enabled(test)) + return false; + if (!total_swap_pages || !IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "requires disposable coarse-shmem swap runner"); + return false; + } + return true; +} + +static bool test_coarse_swap_out(struct kunit *test, struct file *file, pgoff_t index) +{ + unsigned int tries; + + for (tries = 0; tries < 100; tries++) { + struct folio *folio; + void *entry = filemap_get_entry(file->f_mapping, index); + LIST_HEAD(folios); + + if (xa_is_value(entry)) + return test_reclaim_swapcache(radix_to_swp_entry(entry)); + if (!entry) { + KUNIT_FAIL(test, "reclaim removed data without a swap entry"); + return false; + } + folio = entry; + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Real reclaim owns any folios produced by a large-folio split. */ + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + KUNIT_FAIL(test, "file index %lu did not reach swap", index); + return false; +} + +static void test_coarse_cold_shmem(struct kunit *test, bool promoted) +{ + struct file *file; + struct vm_area_struct *coarse, *small; + struct folio *folio; + unsigned int nr = SZ_64K / PAGE_SIZE, i; + unsigned long offset; + vm_fault_t fault; + + if (!test_coarse_swap_enabled(test)) + return; + file = test_cache_file(test, SZ_64K, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + SZ_64K - SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, small); + if (promoted) { + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + folio_put(folio); + } + for (offset = SZ_4K; offset < SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + for (i = 0; i < nr; i++) + KUNIT_ASSERT_TRUE(test, test_coarse_swap_out(test, file, i)); + /* XArray swap entries and absent swapcache prove this is a cold refault. */ + for (i = 0; i < nr; i++) { + void *entry = filemap_get_entry(file->f_mapping, i); + struct folio *cached = NULL; + + KUNIT_EXPECT_TRUE(test, xa_is_value(entry)); + if (entry && !xa_is_value(entry)) { + folio_put(entry); + return; + } + KUNIT_ASSERT_NOT_NULL(test, entry); + cached = swap_cache_get_folio(radix_to_swp_entry(entry)); + KUNIT_EXPECT_PTR_EQ(test, cached, NULL); + if (cached) + folio_put(cached); + } + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, 0UL); + KUNIT_EXPECT_EQ(test, SHMEM_I(file_inode(file))->swapped, (unsigned long)nr); + fault = test_file_fault(coarse, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0x31 + offset / SZ_4K, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, (unsigned long)nr); + KUNIT_EXPECT_EQ(test, SHMEM_I(file_inode(file))->swapped, 0UL); + for (offset = SZ_4K; offset < SZ_64K; offset += SZ_4K) { + pte_t entry; + + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(&folio->page) + offset); + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); +} + +static void user64k_shmem_cold_constituents_test(struct kunit *test) +{ + test_coarse_cold_shmem(test, false); +} + +static void user64k_shmem_cold_promoted_test(struct kunit *test) +{ + test_coarse_cold_shmem(test, true); +} + +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_MEMCG) +static void test_coarse_injection_off(void *unused) +{ + test_swap_control("--coarse-injection-off"); +} + +#endif + +static void user64k_shmem_fault_alloc_fail_test(struct kunit *test) +{ +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_MEMCG) + unsigned int mode; + + if (!test_coarse_swap_enabled(test)) + return; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_injection_off, NULL), 0); + for (mode = 0; mode < 3; mode++) { + struct file *file = test_cache_file(test, SZ_64K, false); + struct vm_area_struct *vma; + struct folio *folio; + phys_addr_t original[SZ_64K / PAGE_SIZE]; + unsigned char *buffer; + loff_t pos = 0; + unsigned int i, pending = 0; + vm_fault_t result; + bool prior_fail; + char operation[80]; + int enter = 0, leave = 0; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + /* Make the zero-limit charge case nonblocking and avoid an OOM kill. */ + if (mode == 2) + mapping_set_gfp_mask(file->f_mapping, + mapping_gfp_mask(file->f_mapping) & ~__GFP_DIRECT_RECLAIM); + buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + memset(buffer, 0x95, SZ_64K); + KUNIT_ASSERT_EQ(test, kernel_write(file, buffer, SZ_64K, &pos), (ssize_t)SZ_64K); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + folio = filemap_get_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + original[i] = page_to_phys(&folio->page); + folio_put(folio); + } + if (mode < 2) { + KUNIT_ASSERT_EQ(test, test_swap_control(mode ? "--coarse-page-on" : + "--coarse-slab-on"), 0); + } else { + snprintf(operation, sizeof(operation), "--coarse-memcg-enter=%d", + task_pid_nr(current)); + enter = test_swap_control(operation); + } + /* Direct callback isolates backing allocation from page-table allocation. */ + if (!enter) { + struct vm_fault vmf = { .vma = vma, .address = TEST_VA, + .pgoff = 0, .flags = FAULT_FLAG_REMOTE }; + + mmap_read_lock(vma->vm_mm); + prior_fail = current->make_it_fail; + if (!mode) + WRITE_ONCE(current->fail_nth, 1); + else if (mode == 1) + current->make_it_fail = true; + result = vma->vm_ops->fault(&vmf); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + current->make_it_fail = prior_fail; + mmap_read_unlock(vma->vm_mm); + /* Keep unexpected success safe for test cleanup. */ + if (vmf.page) { + if (result & VM_FAULT_LOCKED) + unlock_page(vmf.page); + put_page(vmf.page); + } + } else { + result = 0; + } + if (mode == 2) { + snprintf(operation, sizeof(operation), "--coarse-memcg-leave=%d", + task_pid_nr(current)); + leave = test_swap_control(operation); + } else { + leave = test_swap_control("--coarse-injection-off"); + } + KUNIT_ASSERT_EQ(test, enter, 0); + KUNIT_ASSERT_EQ(test, leave, 0); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_ASSERT_TRUE(test, result & VM_FAULT_OOM); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + folio = filemap_get_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + KUNIT_EXPECT_EQ(test, page_to_phys(&folio->page), original[i]); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x95, PAGE_SIZE), NULL); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, SZ_64K / PAGE_SIZE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SHMEMPAGES), 0L); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x95, SZ_64K), NULL); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + kunit_info(test, "coarse allocation failure mode %u preserved data and recovered", mode); + } +#else + kunit_skip(test, "requires slab/page fault injection and memcg"); +#endif +} + +static void user64k_anon_swap_test(struct kunit *test) +{ + unsigned int mode, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_swap_enabled(test)) + return; + for (mode = 0; mode < 7; mode++) { + struct vm_area_struct *vma, *cvma = NULL; + struct mm_struct *child = NULL; + struct folio *folio, *held = NULL; + swp_entry_t entry, part; + unsigned int i, tries; + pte_t pte; + bool *disabled; + bool warm = !mode || mode == 6; + vm_fault_t fault; + int err; + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x31 + i, SZ_4K); + if (mode == 2 || mode == 6) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + } + if (warm) { + folio_get(folio); + held = folio; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, held), 0); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_EXPECT_EQ(test, min_order_for_split(folio), 16U - PAGE_SHIFT); +#endif + folio_lock(folio); + KUNIT_EXPECT_EQ(test, folio_anon_min_order(folio), 16U - PAGE_SHIFT); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_EXPECT_EQ(test, split_huge_page_to_list_to_order(&folio->page, NULL, 0), + -EBUSY); + KUNIT_EXPECT_EQ(test, folio_order(folio), 16U - PAGE_SHIFT); + expect_coarse_anon_state(test, folio, child ? 2 : 1, !child); +#endif + err = folio_alloc_swap(folio); + if (!err) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, err, 0); + } else { + for (tries = 0; tries < 100; tries++) { + LIST_HEAD(folios); + + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + if (!pte_present(pte)) + break; + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + folio_get(folio); + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + } + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE_MSG(test, pte_present(pte), "mode=%u", mode); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(swp_offset(entry), nr)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 1L); + if (mode == 3) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + } + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), child ? 2 : 1); + if (!warm) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + if (mode == 3) { + /* Exit a swapped child: every constituent count must be released. */ + kunit_release_action(test, user4k_vma_mm_free, child); + child = NULL; + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + } + } + if (mode == 5) { +#if defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_FAULT_INJECTION) + bool old = current->make_it_fail; + + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-page-on"), 0); + current->make_it_fail = true; + fault = test_file_fault(vma, TEST_VA, false); + current->make_it_fail = old; + err = test_swap_control("--coarse-injection-off"); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_OOM); + KUNIT_EXPECT_TRUE(test, pte_same(test_fault_entry(test, vma->vm_mm, TEST_VA), pte)); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + } +#endif + } + if (mode == 4) { + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + err = test_swap_control("--swapoff-helper"); + if (!err) + *disabled = true; + KUNIT_ASSERT_EQ(test, err, 0); + } else { + fault = test_file_fault(vma, TEST_VA + SZ_64K - 1, false); + KUNIT_ASSERT_EQ_MSG(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U, + "mode=%u fault=%x", mode, fault); + if (!warm && mode != 5) + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + } + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE_MSG(test, pte_present(pte), "mode=%u", mode); + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + if (held) { + KUNIT_EXPECT_PTR_EQ(test, folio, held); /* Warm aligned cache: no copy. */ + } else { + folio_get(folio); + held = folio; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, held), 0); + } + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + if (child) { + struct folio *copied; + + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), page_to_phys(&folio->page)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + copied = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(child, pte)))); + expect_coarse_anon_state(test, copied, 1, true); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(copied) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + memset(folio_address(copied), 0xa7, SZ_64K); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, child); + } + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + if (held) { + KUNIT_EXPECT_EQ(test, folio_mapcount(held), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(held), 1); + } + for (i = 0; i < nr; i++) { + struct swap_info_struct *si; + + part = entry; + part.val += i; + si = get_swap_device(part); + if (si) { + KUNIT_EXPECT_FALSE(test, swap_entry_swapped(si, part)); + put_swap_device(si); + } else { + KUNIT_EXPECT_EQ(test, mode, 4U); + } + } + if (mode == 4) + kunit_release_action(test, test_reenable_swap, disabled); + kunit_info(test, "completed coarse anonymous swap mode %u", mode); + } +} + +/* The caller's private fixture has no concurrent PTE writers during reclaim. */ +static void test_coarse_anon_reclaim_once(struct vm_area_struct *vma) +{ + struct folio *folio = test_fault_folio(vma->vm_mm, vma->vm_start); + LIST_HEAD(folios); + + if (!folio) + return; + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); +} + +static void user64k_anon_swap_limit_test(struct kunit *test) +{ +#ifdef CONFIG_MEMCG + struct vm_area_struct *vma; + struct cgroup *cgroup; + struct cgroup_subsys_state *css; + struct mem_cgroup *memcg; + struct folio *folio; + swp_entry_t entry, part; + phys_addr_t original; + unsigned int i, tries, nr = SZ_64K / PAGE_SIZE; + long before, denied; + char operation[80]; + vm_fault_t fault; + pte_t pte; + int ret, leave; + + if (!test_coarse_swap_enabled(test)) + return; + if (mem_cgroup_disabled()) { + kunit_skip(test, "requires memory cgroups"); + return; + } + snprintf(operation, sizeof(operation), "--coarse-swap-enter=%d", task_pid_nr(current)); + KUNIT_ASSERT_EQ(test, test_swap_control(operation), 0); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + fault = vma ? test_file_fault(vma, TEST_VA, true) : VM_FAULT_OOM; + snprintf(operation, sizeof(operation), "--coarse-swap-leave=%d", task_pid_nr(current)); + leave = test_swap_control(operation); + KUNIT_ASSERT_EQ(test, leave, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + cgroup = cgroup_get_from_path("/coarse-swap-limit"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cgroup); + css = cgroup_get_e_css(cgroup, &memory_cgrp_subsys); + cgroup_put(cgroup); + KUNIT_ASSERT_NOT_NULL(test, css); + memcg = mem_cgroup_from_css(css); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_memcg_put, memcg), 0); + KUNIT_ASSERT_PTR_NE(test, memcg, root_mem_cgroup); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + rcu_read_lock(); + KUNIT_EXPECT_PTR_EQ(test, folio_memcg(folio), memcg); + rcu_read_unlock(); + original = page_to_phys(&folio->page); + memset(folio_address(folio), 0xc9, SZ_64K); + before = atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]); + folio_lock(folio); + ret = folio_alloc_swap(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + denied = atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]); + KUNIT_EXPECT_GT(test, denied, before); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); + /* Exercise vmscan's failed-allocation/split fallback without a retained GET. */ + for (tries = 0; tries < 8; tries++) + test_coarse_anon_reclaim_once(vma); + KUNIT_EXPECT_GT(test, atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]), denied); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), original); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xc9, SZ_64K), NULL); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-swap-unlimit"), 0); + for (tries = 0; tries < 100; tries++) { + test_coarse_anon_reclaim_once(vma); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + if (!pte_present(pte)) + break; + } + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), (unsigned long)nr); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xc9, SZ_64K), NULL); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); +#else + kunit_skip(test, "requires memory cgroups"); +#endif +} + +struct coarse_swap_worker { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready, go, entered, done; + u8 *buffer; + unsigned int id; + bool *disabled; + int error; +}; + +static int coarse_swap_worker_run(void *arg) +{ + struct coarse_swap_worker *w = arg; + unsigned int i; + u8 value = 0x91 + w->id; + + complete(&w->ready); + if (!wait_for_completion_timeout(&w->go, 15 * HZ) || kthread_should_stop()) { + w->error = -ETIMEDOUT; + goto out; + } + if (!w->mm) { + complete(&w->entered); + w->error = test_swap_control("--swapoff-helper"); + if (!w->error) + WRITE_ONCE(*w->disabled, true); + goto out; + } + kthread_use_mm(w->mm); + complete(&w->entered); + if (copy_from_user(w->buffer, (void __user *)TEST_VA, SZ_64K)) { + w->error = 1; + goto unuse; + } + for (i = 0; i < SZ_64K; i++) { + if (w->buffer[i] != (u8)(0x31 + i / SZ_4K)) { + w->error = 2; + goto unuse; + } + } + if (copy_to_user((void __user *)(TEST_VA + SZ_64K - 1), &value, 1) || + copy_from_user(w->buffer, (void __user *)TEST_VA, SZ_64K)) { + w->error = 3; + goto unuse; + } + for (i = 0; i < SZ_64K; i++) { + u8 expected = i == SZ_64K - 1 ? value : 0x31 + i / SZ_4K; + + if (w->buffer[i] != expected) { + w->error = 4; + break; + } + } +unuse: + kthread_unuse_mm(w->mm); +out: + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void coarse_swap_worker_stop(void *arg) +{ + struct coarse_swap_worker *w = arg; + + if (w->task) { + complete_all(&w->go); + kthread_stop(w->task); + w->task = NULL; + } +} + +static void user64k_anon_swap_race_test(struct kunit *test) +{ + unsigned int round, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_swap_enabled(test)) + return; + if (num_online_cpus() < 2 || !system_supports_64kb_granule()) { + kunit_skip(test, "requires two CPUs and 64K TTBR0 support"); + return; + } + for (round = 0; round < 8; round++) { + struct vm_area_struct *vma; + struct mm_struct *child; + struct coarse_swap_worker *workers; + struct folio *folio, *blocker; + struct swap_info_struct *si; + swp_entry_t entry, part; + bool *disabled, started = true, blocked; + bool off_started = false, cold = round & 1; + unsigned int i, j, tries; + pte_t pte; + int ret, cpu; + + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x31 + i, SZ_4K); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + if (!ret) { + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + } + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 2); + if (cold) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + part = entry; + part.val += nr - 1; + blocker = read_swap_cache_async(part, GFP_KERNEL, vma, + TEST_VA + SZ_64K - PAGE_SIZE, NULL); + KUNIT_ASSERT_NOT_NULL(test, blocker); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, blocker), 0); + KUNIT_EXPECT_EQ(test, folio_size(blocker), cold ? PAGE_SIZE : (unsigned long)SZ_64K); + workers = kunit_kcalloc(test, 3, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 3; i++) { + workers[i].mm = i < 2 ? (i ? child : vma->vm_mm) : NULL; + workers[i].id = i; + workers[i].disabled = disabled; + workers[i].buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers[i].buffer); + init_completion(&workers[i].ready); + init_completion(&workers[i].go); + init_completion(&workers[i].entered); + init_completion(&workers[i].done); + workers[i].task = kthread_create(coarse_swap_worker_run, &workers[i], + "user64k-swap-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "coarse swap worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + coarse_swap_worker_stop, &workers[i]), 0); + if (i < 2) { + kthread_bind(workers[i].task, cpu); + cpu = cpumask_next(cpu, cpu_online_mask); + } + wake_up_process(workers[i].task); + } + for (i = 0; i < 3; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + folio_lock(blocker); + for (i = 0; i < 3; i++) + complete(&workers[i].go); + for (i = 0; i < 3; i++) + started &= !!wait_for_completion_timeout(&workers[i].entered, HZ); + for (tries = 0; tries < 3 * HZ; tries++) { + si = get_swap_device(entry); + if (si) { + off_started = !(READ_ONCE(si->flags) & SWP_WRITEOK); + put_swap_device(si); + } + if (off_started) + break; + schedule_timeout_uninterruptible(1); + } + blocked = !completion_done(&workers[0].done) && + !completion_done(&workers[1].done) && !completion_done(&workers[2].done); + /* Never execute a fatal assertion while retaining the blocking lock. */ + folio_unlock(blocker); + KUNIT_EXPECT_TRUE(test, started); + KUNIT_EXPECT_TRUE(test, off_started); + KUNIT_EXPECT_TRUE(test, blocked); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 20 * HZ), 0UL); + KUNIT_EXPECT_EQ_MSG(test, workers[i].error, 0, "round=%u worker=%u", round, i); + kunit_release_action(test, coarse_swap_worker_stop, &workers[i]); + } + KUNIT_EXPECT_TRUE(test, *disabled); + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? child : vma->vm_mm; + + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (j = 0; j < SZ_64K; j++) { + u8 expected = j == SZ_64K - 1 ? 0x91 + i : 0x31 + j / SZ_4K; + + if (((u8 *)folio_address(folio))[j] != expected) { + KUNIT_FAIL(test, "round=%u side=%u byte=%u", round, i, j); + break; + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1L); + kunit_release_action(test, user4k_vma_mm_free, mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), folio == blocker ? 2 : 1); + kunit_release_action(test, test_folio_put, folio); + } + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(blocker)); + KUNIT_EXPECT_EQ(test, folio_ref_count(blocker), 1); + kunit_release_action(test, test_folio_put, blocker); + kunit_release_action(test, test_reenable_swap, disabled); + KUNIT_ASSERT_GT(test, total_swap_pages, 0UL); + } + kunit_info(test, "8 controlled warm/cold fault+COW+swapoff rounds completed"); +} + +struct coarse_swap_reservation { + struct folio **folios; + unsigned int count; +}; + +static void test_coarse_release_reservation(struct folio *folio) +{ + folio_lock(folio); + folio_put_swap(folio, NULL); + folio_free_swap(folio); + folio_unlock(folio); + folio_put(folio); +} + +static void test_coarse_release_reservations(void *arg) +{ + struct coarse_swap_reservation *r = arg; + unsigned int i; + + for (i = 0; i < r->count; i++) { + if (!r->folios[i]) + continue; + test_coarse_release_reservation(r->folios[i]); + r->folios[i] = NULL; + cond_resched(); + } +} + +static void test_coarse_swap_refusal(struct kunit *test, struct vm_area_struct *vma, + phys_addr_t original) +{ + struct folio *folio = test_fault_folio(vma->vm_mm, TEST_VA); + pte_t pte; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + for (i = 0; i < 4; i++) + test_coarse_anon_reclaim_once(vma); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), original); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xd6, SZ_64K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); +} + +static void user64k_anon_swap_fragment_test(struct kunit *test) +{ + struct coarse_swap_reservation *r; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int i, nr = SZ_64K / PAGE_SIZE, released = 0; + unsigned long hole, free_before; + phys_addr_t original; + swp_entry_t entry, part; + vm_fault_t fault; + pte_t pte; + int ret = 0; + + if (!test_coarse_swap_enabled(test)) + return; + /* Exhaust only the bounded disposable disk owned by this test's PID1. */ + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-fixture-check"), 0); + KUNIT_ASSERT_LT(test, total_swap_pages, (unsigned long)(SZ_256M / PAGE_SIZE)); + KUNIT_ASSERT_GT(test, total_swap_pages, 2UL * SWAPFILE_CLUSTER); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + original = page_to_phys(&folio->page); + memset(folio_address(folio), 0xd6, SZ_64K); + folio_put(folio); + r = kunit_kzalloc(test, sizeof(*r), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, r); + r->folios = kunit_kcalloc(test, total_swap_pages + 1, sizeof(*r->folios), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, r->folios); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_release_reservations, r), 0); + /* Each cache-only folio has one explicit slot owner, so allocation cannot reclaim it. */ + for (i = 0; i <= total_swap_pages; i++) { + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL); + if (ret) { + folio_put(folio); + KUNIT_FAIL(test, "reservation memory charge failed"); + return; + } + __folio_set_swapbacked(folio); + __folio_mark_uptodate(folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) { + ret = folio_dup_swap(folio, NULL); + if (ret) + folio_free_swap(folio); + } + folio_unlock(folio); + if (ret) { + folio_put(folio); + break; + } + r->folios[r->count++] = folio; + cond_resched(); + } + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + KUNIT_ASSERT_EQ(test, atomic_long_read(&nr_swap_pages), 0L); + test_coarse_swap_refusal(test, vma, original); + /* Retain exactly one native entry in each naturally aligned coarse group. */ + for (i = 0; i < r->count; i++) { + folio = r->folios[i]; + if (swp_offset(folio->swap) % nr == 1) + continue; + test_coarse_release_reservation(folio); + r->folios[i] = NULL; + } + free_before = atomic_long_read(&nr_swap_pages); + KUNIT_EXPECT_GT(test, free_before, total_swap_pages / 2); + KUNIT_EXPECT_LT(test, free_before, total_swap_pages); + test_coarse_swap_refusal(test, vma, original); + /* Free one aligned group near the end, without freeing a whole native-order cluster. */ + hole = round_down(total_swap_pages + 1, (unsigned long)SWAPFILE_CLUSTER) - nr; + for (i = 0; i < r->count; i++) { + folio = r->folios[i]; + if (!folio || swp_offset(folio->swap) < hole || swp_offset(folio->swap) >= hole + nr) + continue; + test_coarse_release_reservation(folio); + r->folios[i] = NULL; + released++; + } + KUNIT_ASSERT_EQ(test, released, 1U); + KUNIT_EXPECT_EQ(test, atomic_long_read(&nr_swap_pages), (long)free_before + 1); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ_MSG(test, ret, 0, "a complete aligned group is free in an order-0 cluster"); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, swp_offset(entry), hole); + for (i = 0; i < r->count; i++) { + if (r->folios[i]) + KUNIT_EXPECT_EQ(test, swp_swapcount(r->folios[i]->swap), 1); + } + kunit_release_action(test, test_coarse_release_reservations, r); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xd6, SZ_64K), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + kunit_info(test, "exhaustion, scattered free space and one-group recovery checked"); +} + +struct coarse_swap_io_mode { + struct swap_info_struct *si; + bool synchronous; +}; + +static void test_coarse_restore_io_mode(void *arg) +{ + struct coarse_swap_io_mode *mode = arg; + + spin_lock(&mode->si->lock); + if (mode->synchronous) + mode->si->flags |= SWP_SYNCHRONOUS_IO; + else + mode->si->flags &= ~SWP_SYNCHRONOUS_IO; + spin_unlock(&mode->si->lock); + put_swap_device(mode->si); +} + +static void user64k_anon_swap_extent_test(struct kunit *test) +{ + unsigned int mode, i; + + if (!test_coarse_swap_enabled(test)) + return; + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-fixture-check"), 0); + for (mode = 0; mode < 2; mode++) { + struct vm_area_struct *vma; + struct coarse_swap_io_mode *saved; + struct folio *folio; + unsigned long nr = SZ_64K / PAGE_SIZE, run = nr; + swp_entry_t entry; + sector_t first, next; + bool file; + int ret; + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x63 + i, SZ_4K); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, folio_mapcount(folio), 0); + entry = folio->swap; + saved = kunit_kzalloc(test, sizeof(*saved), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, saved); + saved->si = get_swap_device(entry); + KUNIT_ASSERT_NOT_NULL(test, saved->si); + saved->synchronous = saved->si->flags & SWP_SYNCHRONOUS_IO; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_restore_io_mode, saved), 0); + file = !(saved->si->flags & SWP_BLKDEV); + KUNIT_ASSERT_FALSE(test, saved->si->flags & SWP_FS_OPS); + first = swap_extent_sector(entry, &run); + if (file) { + unsigned long remaining; + swp_entry_t part = entry; + + KUNIT_ASSERT_LT_MSG(test, run, nr, "fragmented fixture must cross an extent"); + part.val += run; + remaining = nr - run; + next = swap_extent_sector(part, &remaining); + KUNIT_EXPECT_NE(test, next, first + (run << (PAGE_SHIFT - 9))); + } else { + KUNIT_EXPECT_EQ(test, run, nr); + } + /* The verified disposable fixture owns this otherwise idle swap area. */ + spin_lock(&saved->si->lock); + if (mode) + saved->si->flags |= SWP_SYNCHRONOUS_IO; + else + saved->si->flags &= ~SWP_SYNCHRONOUS_IO; + spin_unlock(&saved->si->lock); + folio_lock(folio); + folio_clear_dirty_for_io(folio); + __swap_writepage(folio, NULL); + folio_wait_writeback(folio); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, folio_test_dirty(folio)); + /* A full-folio read must reconstruct all physical runs, not warm cache. */ + memset(folio_address(folio), 0, SZ_64K); + folio_clear_uptodate(folio); + swap_read_folio(folio, NULL); + folio_lock(folio); + KUNIT_EXPECT_TRUE(test, folio_test_uptodate(folio)); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x63 + i, SZ_4K), NULL); + folio_unlock(folio); + kunit_release_action(test, test_coarse_restore_io_mode, saved); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + kunit_info(test, "64K full-folio %s I/O: file=%u first-run=%lu/%lu native pages", + mode ? "synchronous" : "asynchronous", file, run, nr); + } +} + +static struct kunit_case user4k_coarse_shmem_cases[] = { + KUNIT_CASE(user64k_anon_swap_extent_test), + KUNIT_CASE(user64k_anon_swap_test), + KUNIT_CASE(user64k_anon_swap_limit_test), + KUNIT_CASE(user64k_anon_swap_fragment_test), + KUNIT_CASE(user64k_anon_swap_race_test), + KUNIT_CASE(user64k_shmem_cold_constituents_test), + KUNIT_CASE(user64k_shmem_cold_promoted_test), + KUNIT_CASE(user64k_shmem_fault_alloc_fail_test), + {} +}; + +static struct kunit_suite user4k_coarse_shmem_suite = { + .name = "arm64-user4k-coarse-shmem", + .test_cases = user4k_coarse_shmem_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static struct kunit_case user4k_shmem_swap_cases[] = { + KUNIT_CASE(user4k_shmem_uffd_swap_alloc_fail_test), + KUNIT_CASE(user4k_shmem_uffd_swap_charge_test), + KUNIT_CASE(user4k_shmem_uffd_swapin_test), + KUNIT_CASE(user4k_shmem_uffd_swapped_truncate_test), + KUNIT_CASE(user4k_shmem_uffd_swapoff_test), + KUNIT_CASE(user4k_shmem_uffd_swapped_partial_hole_test), + {} +}; + +static struct kunit_suite user4k_shmem_swap_suite = { + .name = "arm64-user4k-shmem-swap", + .test_cases = user4k_shmem_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; +#endif + +static struct kunit_case user4k_nonlinear_swap_cases[] = { +#ifdef CONFIG_USERFAULTFD + KUNIT_CASE(user4k_move_swap_get_test), + KUNIT_CASE(user4k_move_migration_race_test), +#endif + KUNIT_CASE(user4k_nonlinear_swap_cached_test), + KUNIT_CASE(user4k_nonlinear_swap_disk_test), + {} +}; + +static struct kunit_suite user4k_nonlinear_swap_suite = { + .name = "arm64-user4k-nonlinear-swap", + .test_cases = user4k_nonlinear_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static struct kunit_suite user4k_swap_suite = { + .name = "arm64-user4k-swap", + .test_cases = user4k_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static void user4k_thp_eligibility_test(struct kunit *test) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + const enum tva_type types[] = { + TVA_PAGEFAULT, TVA_SMAPS, TVA_KHUGEPAGED, TVA_FORCED_COLLAPSE, + }; + struct vm_area_struct *vma, *normal; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_HUGEPAGE; + + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + vma = test_vma_add(small, SZ_1G, SZ_1G, (struct vm_page_offset) {}, flags); + normal = test_vma_add(native, SZ_1G, SZ_1G, (struct vm_page_offset) {}, flags); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, normal); + /* Neither user advice nor forced collapse can bypass geometry limits. */ + KUNIT_EXPECT_TRUE(test, vma_thp_disabled(vma, flags, false)); + KUNIT_EXPECT_TRUE(test, vma_thp_disabled(vma, flags, true)); + KUNIT_EXPECT_FALSE(test, vma_thp_disabled(normal, flags, false)); + KUNIT_EXPECT_FALSE(test, vma_thp_disabled(normal, flags, true)); + for (unsigned int i = 0; i < ARRAY_SIZE(types); i++) + KUNIT_EXPECT_EQ(test, __thp_vma_allowable_orders(vma, flags, + types[i], THP_ORDERS_ALL_ANON), 0UL); +#else + kunit_skip(test, "requires transparent hugepage support"); +#endif +} + +static void user4k_aslr_window_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + const unsigned long windows[] = { SZ_8M, SZ_32M, SZ_1G }; + + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + for (unsigned int i = 0; i < ARRAY_SIZE(windows); i++) { + unsigned long mask = (windows[i] >> PAGE_SHIFT) - 1; + unsigned long converted = mm_aslr_mask(small, mask); + + KUNIT_EXPECT_EQ(test, mm_aslr_mask(native, mask), mask); + KUNIT_EXPECT_EQ(test, (converted + 1) << mm_page_shift(small), + windows[i]); + KUNIT_EXPECT_EQ(test, converted & (TEST_SLOTS - 1), + (unsigned long)TEST_SLOTS - 1); + } +} + +static void user4k_exec_granule_policy_test(struct kunit *test) +{ + /* Model all native granules, including AArch32 with NO 4K support. */ + const unsigned int native[] = { 12, 14, 16 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(native); i++) { + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, false, native[i], 0), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, false, native[i], 12), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, true, native[i], 0), + 12U); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(false, true, native[i], 0), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(false, true, native[i], 12), + 12U); + } +} + +static void user_granule_zero_region_test(struct kunit *test) +{ + unsigned long base_pfn = zero_pfn(0); + unsigned int shift, i; + u8 *buffer; + + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(PFN_PHYS(base_pfn), SZ_64K)); + KUNIT_EXPECT_FALSE(test, is_zero_pfn(base_pfn - 1)); + KUNIT_EXPECT_FALSE(test, is_zero_pfn(base_pfn + SZ_64K / PAGE_SIZE)); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + KUNIT_EXPECT_TRUE(test, is_zero_pfn(base_pfn + i)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(pfn_to_page(base_pfn + i)), + 0, PAGE_SIZE), NULL); + } + buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + for (shift = 12; shift <= 16; shift += 2) { + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned long size = 1UL << shift, left; + vm_fault_t ret; + spinlock_t *ptl; + pte_t *pte, entry; + + if ((shift == 12 && !system_supports_4kb_granule()) || + (shift == 14 && !system_supports_16kb_granule()) || + (shift == 16 && !system_supports_64kb_granule())) { + kunit_info(test, "hardware zero mapping skipped for shift %u", shift); + continue; + } + /* Internal coarse mm: the public opt-in remains disabled. */ + mm = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = test_fault_vma_in_mm(mm, TEST_VA, size); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + size - 1, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_special(entry)); + KUNIT_EXPECT_FALSE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), PFN_PHYS(base_pfn)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + + memset(buffer, 0xa5, size); + kthread_use_mm(mm); + left = copy_from_user(buffer, (void __user *)TEST_VA, size); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, left, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, size), NULL); + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, size); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + } +} + +#ifdef CONFIG_COREDUMP +static void user_granule_core_offsets_test(struct kunit *test) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + struct vm_area_struct *vma = kunit_kzalloc(test, sizeof(*vma), GFP_KERNEL); + unsigned int shift; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_mm = mm; + for (shift = 12; shift <= 16; shift += 2) { + unsigned long bytes = (1UL << 55) + 3 * SZ_64K + (1UL << shift); + + mm->context.flags = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = bytes >> PAGE_SHIFT, .offset = bytes & ~PAGE_MASK, + }); + KUNIT_EXPECT_EQ(test, core_vma_pgoff(vma), bytes >> shift); + KUNIT_EXPECT_EQ(test, mm_user_fragment_size(mm), min(1UL << shift, PAGE_SIZE)); + vma_set_page_offset(vma, (struct vm_page_offset) {}); + KUNIT_EXPECT_EQ(test, core_vma_pgoff(vma), 0UL); + } +} +#endif + +static void user64k_fragment_ownership_test(struct kunit *test) +{ + struct user_page_fragment fragments[SZ_64K / PAGE_SIZE] = {}; + struct mm_struct *parent, *child; + struct vm_area_struct *vma, *cvma; + struct folio *original, *copied; + unsigned int i, nr = ARRAY_SIZE(fragments); + long count; + int ret, refs; + pte_t entry; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires a user leaf larger than native pages"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + count = get_user_fragments_remote(parent, TEST_VA + 17, SZ_64K - 34, + 0, fragments, nr); + if (count > 0) { + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &fragments[i]; + unsigned int bytes = PAGE_SIZE - (i == 0 || i == nr - 1 ? 17 : 0); + + KUNIT_EXPECT_TRUE(test, is_zero_folio(f->folio)); + KUNIT_EXPECT_EQ(test, folio_pfn(f->folio), zero_pfn(TEST_VA) + i); + KUNIT_EXPECT_PTR_EQ(test, f->subpage, NULL); + KUNIT_EXPECT_EQ(test, f->offset, i == 0 ? 17UL : 0UL); + KUNIT_EXPECT_EQ(test, f->length, bytes); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0, f->length), NULL); + } + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, parent, TEST_VA); + original = page_folio(pfn_to_page(pte_pfn(entry))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(original, i)), 0x51 + i, PAGE_SIZE); + lru_add_drain_all(); + refs = folio_ref_count(original); + count = get_user_fragments_remote(parent, TEST_VA + 17, SZ_64K - 34, + 0, fragments, nr); + if (count > 0) { + KUNIT_EXPECT_EQ(test, folio_ref_count(original), refs + (int)count); + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &fragments[i]; + + KUNIT_EXPECT_PTR_EQ(test, f->folio, original); + KUNIT_EXPECT_EQ(test, f->offset, i * PAGE_SIZE + (i == 0 ? 17UL : 0UL)); + KUNIT_EXPECT_LE(test, offset_in_page(f->offset) + f->length, PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x51 + i, f->length), NULL); + } + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + KUNIT_EXPECT_EQ(test, folio_ref_count(original), refs); + + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + /* A read pin after fork must unshare the complete coarse leaf. */ + count = pin_user_fragments_remote(child, TEST_VA, SZ_64K, 0, fragments, nr); + if (count > 0) { + copied = fragments[0].folio; + KUNIT_EXPECT_PTR_NE(test, copied, original); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(copied)); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, child, TEST_VA))); + for (i = 0; i < count; i++) { + KUNIT_EXPECT_PTR_EQ(test, fragments[i].folio, copied); + KUNIT_EXPECT_PTR_EQ(test, fragments[i].subpage, NULL); + KUNIT_EXPECT_TRUE(test, fragments[i].pinned); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(copied, i))); + KUNIT_EXPECT_EQ(test, fragments[i].offset, i * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, fragments[i].length, (unsigned int)PAGE_SIZE); + } + mmap_read_lock(child); + zap_vma_range(cvma, TEST_VA, SZ_64K); + mmap_read_unlock(child); + KUNIT_EXPECT_EQ(test, folio_mapcount(copied), 0); + for (i = 0; i < count; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(copied) + i * PAGE_SIZE, + 0x51 + i, PAGE_SIZE), NULL); + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(original) + i * PAGE_SIZE, + 0x51 + i, PAGE_SIZE), NULL); +} + +static void user64k_anon_fault_cow_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + struct vm_area_struct *vma, *cvma; + pte_t orig[2], copied; + struct folio *folio; + unsigned int i, j, nr = SZ_64K / PAGE_SIZE; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous faults"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, 2 * SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Leaf0 exercises zero-page COW; leaf1 exercises a direct write fault. */ + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, false) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_TRUE(test, pte_special(test_fault_entry(test, parent, TEST_VA))); + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + + KUNIT_ASSERT_EQ(test, test_file_fault(vma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + orig[i] = test_fault_entry(test, parent, addr); + KUNIT_ASSERT_TRUE(test, pte_present(orig[i])); + KUNIT_EXPECT_TRUE(test, pte_write(orig[i])); + KUNIT_EXPECT_FALSE(test, pte_special(orig[i])); + folio = page_folio(pfn_to_page(pte_pfn(orig[i]))); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, SZ_64K), NULL); + for (j = 0; j < nr; j++) { + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(folio, j))); + memset(page_address(folio_page(folio, j)), 0x31 + i * nr + j, PAGE_SIZE); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 2L); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 2L); + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + struct folio *old = page_folio(pfn_to_page(pte_pfn(orig[i]))); + + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, test_fault_entry(test, child, addr)), + pte_phys_mm(parent, orig[i])); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, parent, addr))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)(2 * nr)); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + copied = test_fault_entry(test, child, addr); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, copied), pte_phys_mm(parent, orig[i])); + folio = page_folio(pfn_to_page(pte_pfn(copied))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)nr); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + for (j = 0; j < nr; j++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(folio, j)), + 0x31 + i * nr + j, PAGE_SIZE), NULL); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(folio, j))); + } + memset(folio_address(folio), 0xa5, SZ_64K); + for (j = 0; j < nr; j++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(old, j)), + 0x31 + i * nr + j, PAGE_SIZE), NULL); + /* Parent reuse must restore exclusivity across the entire leaf. */ + lru_add_drain_all(); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, pte_phys_mm(parent, test_fault_entry(test, parent, addr)), + pte_phys_mm(parent, orig[i])); + for (j = 0; j < nr; j++) + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(old, j))); + } + if (system_supports_64kb_granule()) { + long protect[2]; + bool writable = false; + + kthread_use_mm(parent); + protect[0] = test_mprotect(TEST_VA, 2 * SZ_64K, PROT_READ); + kthread_unuse_mm(parent); + for (i = 0; i < 2; i++) + writable |= pte_write(test_fault_entry(test, parent, TEST_VA + i * SZ_64K)); + KUNIT_EXPECT_EQ(test, protect[0], 0L); + KUNIT_EXPECT_FALSE(test, writable); + kthread_use_mm(parent); + protect[1] = test_mprotect(TEST_VA, 2 * SZ_64K, PROT_READ | PROT_WRITE); + kthread_unuse_mm(parent); + KUNIT_EXPECT_EQ(test, protect[1], 0L); + for (i = 0; i < 2; i++) + KUNIT_EXPECT_TRUE(test, pte_write(test_fault_entry(test, parent, TEST_VA + i * SZ_64K))); + vma = test_vma_lookup(parent, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + } + mmap_read_lock(child); + zap_vma_range(cvma, TEST_VA, 2 * SZ_64K); + mmap_read_unlock(child); + mmap_read_lock(parent); + zap_vma_range(vma, TEST_VA, 2 * SZ_64K); + mmap_read_unlock(parent); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 0L); +} + +struct coarse_test_pin { + struct page *page; +}; + +static void coarse_test_unpin(void *arg) +{ + struct coarse_test_pin *pin = arg; + + if (pin->page) { + unpin_user_page(pin->page); + pin->page = NULL; + } +} + +static void user64k_pinned_fork_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + struct vm_area_struct *vma; + struct coarse_test_pin *pin; + struct folio *old, *new; + pte_t original, copied; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires pinned coarse leaf"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + original = test_fault_entry(test, parent, TEST_VA); + old = page_folio(pfn_to_page(pte_pfn(original))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(old, i)), 0x70 + i, PAGE_SIZE); + pin = kunit_kzalloc(test, sizeof(*pin), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pin); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_test_unpin, pin), 0); + mm_flags_set(MMF_HAS_PINNED, parent); + ret = try_grab_folio(old, 1, FOLL_PIN); + if (!ret) + pin->page = folio_page(old, nr - 1); + KUNIT_ASSERT_EQ(test, ret, 0); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + copied = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, copied), pte_phys_mm(parent, original)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(parent, test_fault_entry(test, parent, TEST_VA)), + pte_phys_mm(parent, original)); + new = page_folio(pfn_to_page(pte_pfn(copied))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)nr); + KUNIT_EXPECT_EQ(test, folio_mapcount(new), (int)nr); + for (i = 0; i < nr; i++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(new, i)), + 0x70 + i, PAGE_SIZE), NULL); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(new, i))); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(old, i))); + } + memset(page_address(pin->page), 0xe3, PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(new, nr - 1)), + 0x70 + nr - 1, PAGE_SIZE), NULL); + coarse_test_unpin(pin); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1L); +} + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +static void user4k_shmem_partial_hole_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 3 * PAGE_SIZE, true); + struct folio *folio; + loff_t start = PAGE_SIZE + SZ_4K + 17, end = PAGE_SIZE + 3 * SZ_4K + 16; + unsigned int i; + unsigned char byte = 0xa5; + loff_t write_pos = PAGE_SIZE + 2 * SZ_4K + 23; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), start, end); + folio = filemap_lock_folio(file->f_mapping, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), BIT(2)); + for (i = 0; i < PAGE_SIZE; i++) { + unsigned char expected = (i + PAGE_SIZE >= start && i + PAGE_SIZE <= end) ? + 0 : 0x31 + (PAGE_SIZE + i) / SZ_4K; + + if (((unsigned char *)folio_address(folio))[i] != expected) { + KUNIT_FAIL(test, "partial hole changed byte %u outside its range", i); + break; + } + } + /* Resident hole metadata has no dynamically allocated inode record. */ + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); + shmem_truncate_range(file_inode(file), PAGE_SIZE + SZ_4K, PAGE_SIZE + 2 * SZ_4K - 1); + KUNIT_ASSERT_EQ(test, kernel_write(file, &byte, 1, &write_pos), 1L); + folio = filemap_lock_folio(file->f_mapping, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), BIT(1)); + KUNIT_EXPECT_EQ(test, ((unsigned char *)folio_address(folio))[2 * SZ_4K + 23], byte); + folio_unlock(folio); + folio_put(folio); +} + +/* A surviving writable pin exposes bytes even after its PTE is revoked. */ +static void user4k_shmem_pinned_hole_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + struct folio *folio; + unsigned long offset = native ? 0 : SZ_4K; + unsigned long length = native ? PAGE_SIZE : SZ_4K; + unsigned long pfn; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, native, true, TEST_VA, length, offset); + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + KUNIT_ASSERT_EQ(test, fragment_test_pin(pins, vma->vm_mm, TEST_VA, + length, FOLL_WRITE | FOLL_LONGTERM, 1), 1L); + folio = pins->fragments[0].folio; + pfn = folio_pfn(folio); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, SZ_4K), NULL); + /* Model a completed device write through the original pin. */ + memset(folio_address(folio) + SZ_4K, 0xb7, SZ_4K); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + folio_unlock(folio); + fragment_test_unpin(pins); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_pfn(folio), pfn); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xb7, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x31, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + 2 * SZ_4K, 0x33, SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); + /* With the external writer gone, another hole can be missing. */ + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + folio_unlock(folio); + folio_put(folio); + } +} + +static void test_unpin_folio(void *folio) +{ + unpin_folio(folio); +} + +/* The direct memfd API must publish cache exposure without a user PTE. */ +static void user4k_shmem_memfd_pin_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *pin, *folio; + pgoff_t offset; + long ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + ret = memfd_pin_folios(file, SZ_4K, 2 * SZ_4K - 1, &pin, 1, &offset); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_unpin_folio, pin), 0); + KUNIT_EXPECT_EQ(test, offset, (pgoff_t)SZ_4K); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(pin)); + memset(folio_address(pin) + offset, 0xc8, SZ_4K); + folio_lock(pin); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(pin, 0, SZ_4K, SZ_4K)); + folio_unlock(pin); + kunit_release_action(test, test_unpin_folio, pin); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xc8, SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); +} + +static void user4k_shmem_replace_validity_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *old, *new; + unsigned long old_pfn; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + new = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, new); + folio_lock(new); + old = filemap_lock_folio(file->f_mapping, 0); + if (IS_ERR(old)) { + folio_unlock(new); + folio_put(new); + KUNIT_FAIL(test, "could not lock source cache folio"); + return; + } + old_pfn = folio_pfn(old); + memcpy(folio_address(new), folio_address(old), PAGE_SIZE); + __folio_set_swapbacked(new); + __folio_mark_uptodate(new); + folio_mark_dirty(new); + replace_page_cache_folio(old, new); + folio_add_lru(new); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(new, 0), BIT(1)); + KUNIT_EXPECT_NE(test, folio_pfn(new), old_pfn); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(new), 0x31, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(new) + SZ_4K, 0, SZ_4K), NULL); + folio_unlock(old); + folio_put(old); + folio_unlock(new); + folio_put(new); +} + +static void user4k_shmem_uffd_alias_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, false); + const unsigned long all = GENMASK(PAGE_SIZE / SZ_4K - 1, 0); + struct vm_area_struct *small, *native; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = shmem_uffd_set_missing(folio, 0, all); + KUNIT_EXPECT_TRUE(test, shmem_uffd_has_missing_native_page(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + small = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), all & ~BIT(1)); + KUNIT_EXPECT_FALSE(test, shmem_uffd_has_missing_native_page(folio)); + folio_unlock(folio); + folio_put(folio); + + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), 0UL); + folio_unlock(folio); + folio_put(folio); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); +} + +static void user4k_shmem_uffd_state_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 3 * PAGE_SIZE, false); + const unsigned long all = GENMASK(PAGE_SIZE / SZ_4K - 1, 0); + struct folio *folio; + unsigned long mask; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + for (i = 0; i < 3; i++) { + ret = shmem_get_folio(file_inode(file), i, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), 0UL); + ret = shmem_uffd_set_missing(folio, i, all & ~BIT(0)); + KUNIT_EXPECT_EQ(test, ret, 0); + shmem_uffd_mark_present(folio, i, BIT(1)); + mask = shmem_uffd_missing_mask(folio, i); + KUNIT_EXPECT_EQ(test, mask, all & ~GENMASK(1, 0)); + ret = shmem_uffd_set_missing(folio, i, BIT(PAGE_SIZE / SZ_4K)); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), mask); + folio_unlock(folio); + folio_put(folio); + } + /* Real truncate removes one index without discarding neighbouring state. */ + shmem_truncate_range(file_inode(file), PAGE_SIZE, 2 * PAGE_SIZE - 1); + KUNIT_EXPECT_PTR_EQ(test, xa_load(&SHMEM_I(file_inode(file))->uffd_missing, 1), NULL); + for (i = 0; i < 3; i += 2) { + folio = filemap_lock_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), all & ~GENMASK(1, 0)); + folio_unlock(folio); + folio_put(folio); + } + /* Generic batch deletion must not strand inode metadata either. */ + truncate_inode_pages(file->f_mapping, 0); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + ret = shmem_get_folio(file_inode(file), 1, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, PAGE_SIZE), NULL); + folio_unlock(folio); + folio_put(folio); +} + +#ifdef CONFIG_MIGRATION +static void user4k_shmem_uffd_migration_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, false); + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + struct folio *folio; + unsigned long old_pfn; + unsigned int succeeded = 0; + bool isolated; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xa5, SZ_4K); + folio_mark_dirty(folio); + ret = shmem_uffd_set_missing(folio, 0, missing); + old_pfn = folio_pfn(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, succeeded, 1U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_NE(test, folio_pfn(folio), old_pfn); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), missing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xa5, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, + PAGE_SIZE - SZ_4K), NULL); + shmem_uffd_mark_present(folio, 0, missing); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), 0UL); + folio_unlock(folio); + folio_put(folio); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); +} +#endif +#endif + +static struct kunit_case user4k_test_cases[] = { + KUNIT_CASE(user4k_exec_granule_policy_test), + KUNIT_CASE(user4k_aslr_window_test), + KUNIT_CASE(user4k_thp_eligibility_test), + KUNIT_CASE(user4k_geometry_test), + KUNIT_CASE(user4k_leaf_encoding_test), + KUNIT_CASE(user16k_leaf_encoding_test), + KUNIT_CASE(user4k_rmap_walk_test), + KUNIT_CASE(user4k_rmap_boundary_test), + KUNIT_CASE(native_rmap_walk_test), + KUNIT_CASE(user64k_rmap_lookup_test), + KUNIT_CASE(user64k_anon_rmap_span_test), + KUNIT_CASE(user64k_anon_zap_batch_test), + KUNIT_CASE(user_granule_zero_region_test), +#ifdef CONFIG_COREDUMP + KUNIT_CASE(user_granule_core_offsets_test), +#endif + KUNIT_CASE(user64k_fragment_ownership_test), + KUNIT_CASE(user64k_anon_fault_cow_test), + KUNIT_CASE(user64k_pinned_fork_test), + KUNIT_CASE(user64k_file_rmap_span_test), + KUNIT_CASE(user4k_vma_offsets_test), + KUNIT_CASE(user4k_vma_intervals_test), + KUNIT_CASE(user4k_vma_split_test), + KUNIT_CASE(user4k_vma_merge_test), + KUNIT_CASE(user4k_vma_partial_merge_test), + KUNIT_CASE(user4k_vma_copy_shrink_test), + KUNIT_CASE(user4k_anon_rmap_test), + KUNIT_CASE(user4k_anon_fork_rmap_test), + KUNIT_CASE(user4k_nonlinear_rmap_reuse_test), + KUNIT_CASE(user4k_dup_mmap_test), + KUNIT_CASE(user4k_anon_root_lifetime_test), + KUNIT_CASE(user4k_data_tlb_batch_test), + KUNIT_CASE(user4k_anon_zap_test), + KUNIT_CASE(user4k_anon_writable_slots_test), + KUNIT_CASE(user4k_anon_write_fault_test), + KUNIT_CASE(user16k_anon_write_fault_test), + KUNIT_CASE(user4k_mlock_counts_test), + KUNIT_CASE(user4k_mlock_shared_test), + KUNIT_CASE(user4k_mlock_repair_test), + KUNIT_CASE(user_granule_large_vma_split_test), + KUNIT_CASE(user_granule_brk_extent_test), + KUNIT_CASE(user4k_droppable_reclaim_test), + KUNIT_CASE(user4k_lazyfree_test), + KUNIT_CASE(user4k_zero_cow_test), + KUNIT_CASE(user16k_zero_cow_test), + KUNIT_CASE(user4k_shmem_swap_usage_test), +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + KUNIT_CASE(user4k_shmem_partial_hole_test), + KUNIT_CASE(user4k_shmem_pinned_hole_test), + KUNIT_CASE(user4k_shmem_memfd_pin_test), + KUNIT_CASE(user4k_shmem_replace_validity_test), + KUNIT_CASE(user4k_shmem_uffd_alias_test), + KUNIT_CASE(user4k_shmem_uffd_state_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_shmem_uffd_migration_test), +#endif +#endif + KUNIT_CASE(user4k_file_mapcounts_test), + KUNIT_CASE(user64k_file_promote_test), + KUNIT_CASE(user64k_shmem_fault_test), + KUNIT_CASE(user64k_shmem_fault_busy_test), + KUNIT_CASE(user64k_shmem_fault_hole_test), + KUNIT_CASE(user64k_file_leaf_test), + KUNIT_CASE(user64k_file_cow_test), + KUNIT_CASE(user64k_file_bounds_test), + KUNIT_CASE(user4k_file_shared_test), + KUNIT_CASE(user4k_iommufd_lifetime_test), + KUNIT_CASE(user4k_iommufd_rollback_test), + KUNIT_CASE(user4k_iommufd_granule_test), + KUNIT_CASE(user4k_iommufd_shared_test), + KUNIT_CASE(user4k_iommufd_exit_account_test), + KUNIT_CASE(user4k_iova_bitmap_test), + KUNIT_CASE(user4k_iova_bitmap_window_test), + KUNIT_CASE(user4k_iova_bitmap_retained_test), + KUNIT_CASE(user4k_iova_bitmap_partial_test), + KUNIT_CASE(user4k_mremap_move_test), + KUNIT_CASE(user4k_mremap_resize_test), + KUNIT_CASE(user4k_stack_relocate_test), + KUNIT_CASE(user4k_exec_arguments_test), + KUNIT_CASE(user4k_special_mapping_test), + KUNIT_CASE(user4k_insert_fragment_test), + KUNIT_CASE(user16k_insert_shared_test), + KUNIT_CASE(user4k_insert_range_test), + KUNIT_CASE(user4k_pfn_offset_test), + KUNIT_CASE(user4k_remap_pfn_range_test), + KUNIT_CASE(user4k_iomem_access_test), +#if defined(CONFIG_GENERIC_GETTIMEOFDAY) && defined(CONFIG_VDSO_GETRANDOM) + KUNIT_CASE(user4k_vdso_mapping_test), +#endif + KUNIT_CASE(user4k_mremap_cow_test), + KUNIT_CASE(user4k_mremap_file_test), + KUNIT_CASE(user4k_msync_offsets_test), + KUNIT_CASE(user4k_prefault_dirty_test), + KUNIT_CASE(user4k_cold_batch_test), + KUNIT_CASE(user4k_remote_access_test), + KUNIT_CASE(user4k_remote_file_test), +#ifdef CONFIG_CROSS_MEMORY_ATTACH + KUNIT_CASE(user4k_process_vm_test), +#endif +#ifdef CONFIG_FUTEX + KUNIT_CASE(user4k_futex_keys_test), + KUNIT_CASE(user4k_futex_wake_test), +#endif + KUNIT_CASE(user4k_mmap_descriptor_test), + KUNIT_CASE(user4k_mmap_region_offsets_test), + KUNIT_CASE(user4k_commit_accounting_test), + KUNIT_CASE(mixed_locked_account_units_test), + KUNIT_CASE(user64k_commit_accounting_test), + KUNIT_CASE(user64k_page_table_check_test), +#ifdef CONFIG_PROC_FS + KUNIT_CASE(user4k_accounting_readers_test), +#endif + KUNIT_CASE(user_granule_prefault_test), + KUNIT_CASE(user4k_mmap_accounting_test), + KUNIT_CASE(user4k_vm_mmap_test), + KUNIT_CASE(user4k_mprotect_test), + KUNIT_CASE(user4k_mprotect_cow_batch_test), + KUNIT_CASE(user4k_mprotect_accounting_test), + KUNIT_CASE(user4k_file_cow_test), + KUNIT_CASE(user4k_file_eof_test), + KUNIT_CASE(user4k_file_truncate_test), + KUNIT_CASE(user4k_filemap_fault_test), + KUNIT_CASE(user4k_file_invalidate_test), + KUNIT_CASE(user4k_file_fragments_test), + KUNIT_CASE(user4k_file_fork_hole_test), + KUNIT_CASE(user4k_swap_encoding_test), + KUNIT_CASE(user4k_cow_reuse_test), + KUNIT_CASE(user16k_cow_reuse_test), + KUNIT_CASE(user4k_fork_cow_test), + KUNIT_CASE(user16k_fork_cow_test), + KUNIT_CASE(user4k_slot_pin_test), + KUNIT_CASE(user4k_fragment_pin_test), + KUNIT_CASE(user4k_fragment_locked_pin_test), + KUNIT_CASE(user4k_folio_walk_geometry_test), + KUNIT_CASE(user4k_fragment_longterm_test), +#ifdef CONFIG_BLOCK + KUNIT_CASE(user4k_bio_fragments_test), + KUNIT_CASE(user4k_bio_trim_test), +#endif + KUNIT_CASE(user4k_fragment_permissions_test), + KUNIT_CASE(user16k_fragment_offset_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_migration_test), + KUNIT_CASE(user4k_file_migration_test), + KUNIT_CASE(user64k_file_migration_test), + KUNIT_CASE(user64k_anon_migration_test), + KUNIT_CASE(user4k_fragment_pin_race_test), + KUNIT_CASE(user4k_migration_entries_test), + KUNIT_CASE(user4k_nonlinear_rmap_test), + KUNIT_CASE(user4k_mprotect_migration_test), + KUNIT_CASE(user4k_live_migration_test), + KUNIT_CASE(user4k_migration_wait_test), +#endif + KUNIT_CASE(user4k_anon_exit_test), + KUNIT_CASE(user4k_anon_remote_zap_test), +#ifdef CONFIG_ARM64_CONTPTE + KUNIT_CASE(user4k_contpte_test), + KUNIT_CASE(native_contpte_test), +#endif + KUNIT_CASE(user4k_mm_tables_test), + KUNIT_CASE(user16k_mm_tables_test), + KUNIT_CASE(user64k_mm_tables_test), + KUNIT_CASE(native_mm_tables_test), + KUNIT_CASE(user4k_anon_read_fault_test), + KUNIT_CASE(user4k_mm_hardware_test), + KUNIT_CASE(user4k_fragment_lifetime_test), + KUNIT_CASE(user16k_fragment_lifetime_test), + KUNIT_CASE(user64k_table_owner_test), +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_CASE(native_pgtable_deposit_test), +#endif + KUNIT_CASE(user4k_translation_test), + KUNIT_CASE(user4k_context_uaccess_test), + KUNIT_CASE(user4k_fault_uaccess_test), + {} +}; + +static struct kunit_suite user4k_test_suite = { + .name = "arm64-user4k", + .test_cases = user4k_test_cases, +}; + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_shmem_swap_suite, + &user4k_coarse_shmem_suite, + &user4k_nonlinear_swap_suite); +#else +kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_nonlinear_swap_suite); +#endif + +MODULE_LICENSE("GPL"); diff --git a/arch/arm64/mm/user4k-tlb-test.c b/arch/arm64/mm/user4k-tlb-test.c new file mode 100644 index 0000000000000..e5671256b7c76 --- /dev/null +++ b/arch/arm64/mm/user4k-tlb-test.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Decode emitted operands without executing TLBI instructions. */ +#include +#include +#include +#include + +struct tlbi_test_state { + u64 next; + u64 end; + u64 stride; + u16 asid; + u32 level; + unsigned int shift; + unsigned int single_ops; + unsigned int range_ops; + bool lpa2; +}; + +static unsigned int test_tg(unsigned int shift) +{ + return shift == 12 ? 1 : shift == 14 ? 2 : 3; +} + +static void record_single(u64 arg) +{ + struct kunit *test = kunit_get_current_test(); + struct tlbi_test_state *state = test->priv; + u64 address = (arg & GENMASK_ULL(43, 0)) << 12; + u64 hint = 0; + + if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && state->level <= 3) + hint = (test_tg(state->shift) << 2) | state->level; + KUNIT_EXPECT_EQ(test, address, state->next); + KUNIT_EXPECT_EQ(test, arg >> 48, (u64)state->asid); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBI_TTL_MASK, arg), hint); + state->next += state->stride; + state->single_ops++; + KUNIT_EXPECT_LE(test, state->next, state->end); +} + +static void record_range(u64 arg) +{ + struct kunit *test = kunit_get_current_test(); + struct tlbi_test_state *state = test->priv; + u64 address = FIELD_GET(TLBIR_BADDR_MASK, arg) << (state->lpa2 ? 16 : state->shift); + u64 count = (FIELD_GET(TLBIR_NUM_MASK, arg) + 1) << + (5 * FIELD_GET(TLBIR_SCALE_MASK, arg) + 1); + + KUNIT_EXPECT_EQ(test, address, state->next); + KUNIT_EXPECT_EQ(test, arg >> 48, (u64)state->asid); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBIR_TG_MASK, arg), (u64)test_tg(state->shift)); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBIR_TTL_MASK, arg), + state->level > 3 ? 0ULL : (u64)state->level); + if (state->lpa2) + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(address, SZ_64K)); + state->next += count << state->shift; + state->range_ops++; + KUNIT_EXPECT_LE(test, state->next, state->end); +} + +static bool range_supported(void) +{ + return true; +} + +static bool range_unsupported(void) +{ + return false; +} + +static void test_flush(struct kunit *test, unsigned int shift, u64 start, + unsigned long pages, u64 stride, u32 level, bool lpa2, + bool has_range) +{ + struct tlbi_test_state state = { + .next = start, + .end = start + (pages << shift), + .stride = stride, + .asid = 0xa55a, + .level = level, + .shift = shift, + .lpa2 = lpa2, + }; + + test->priv = &state; + __flush_tlb_range_op_granule(record_single, record_range, start, pages, + stride, state.asid, level, lpa2, shift, + has_range ? range_supported : range_unsupported); + KUNIT_EXPECT_EQ(test, state.next, state.end); + if (has_range) { + KUNIT_EXPECT_LE(test, state.single_ops + state.range_ops, 20U); + if (pages >= 32) + KUNIT_EXPECT_GT(test, state.range_ops, 0U); + } else { + KUNIT_EXPECT_EQ(test, state.range_ops, 0U); + KUNIT_EXPECT_EQ(test, state.single_ops, (unsigned int)((pages << shift) / stride)); + } + test->priv = NULL; +} + +static void user4k_tlbi_operands_test(struct kunit *test) +{ + static const unsigned long counts[] = { + 0, 1, 2, 3, 31, 32, 33, 63, 64, 65, 2047, 2048, 2049, + 65535, 65536, 65537, MAX_TLBI_RANGE_PAGES - 1, MAX_TLBI_RANGE_PAGES, + }; + static const u32 levels[] = { 2, 3, TLBI_TTL_UNKNOWN }; + unsigned int shift, i, j; + + /* Force both algorithms using recording callbacks, independent of CPU caps. */ + for (shift = 12; shift <= 16; shift += 2) { + u64 size = 1UL << shift; + + for (j = 0; j < ARRAY_SIZE(levels); j++) { + for (i = 0; i < ARRAY_SIZE(counts); i++) { + unsigned long pages = counts[i]; + + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], false, true); + if (shift != 16) + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], true, true); + if (pages <= 65) + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], false, false); + } + } + /* Preserve a larger leaf stride when range instructions are absent. */ + test_flush(test, shift, SZ_1G, 3 * 512, 512 * size, 2, false, false); + /* Last virtual page must not lose high address bits. */ + test_flush(test, shift, (1ULL << 48) - size, 1, size, 3, false, false); + } +} + +static void user4k_tlb_gather_test(struct kunit *test) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + pte_t entries[3] = {}; + unsigned int small; + + KUNIT_ASSERT_NOT_NULL(test, mm); + for (small = 0; small < 2; small++) { + struct mmu_gather tlb = { .mm = mm, .start = ULONG_MAX }; + unsigned long size = small ? SZ_4K : PAGE_SIZE; + unsigned long address = SZ_4M + size; + + mm->context.flags = small ? MMCF_USER_4K : 0; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + tlb_remove_tlb_entries(&tlb, entries, 3, address); + KUNIT_EXPECT_EQ(test, tlb.start, address); + KUNIT_EXPECT_EQ(test, tlb.end, address + 3 * size); + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), 3); + tlb.cleared_pmds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), TLBI_TTL_UNKNOWN); + tlb.cleared_ptes = 0; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? SZ_2M : PMD_SIZE); + tlb.cleared_pmds = 0; + tlb.cleared_puds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? SZ_1G : PUD_SIZE); + tlb.cleared_puds = 0; + tlb.cleared_p4ds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? 1UL << 39 : P4D_SIZE); + tlb.freed_tables = 1; + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), TLBI_TTL_UNKNOWN); + } +} + +static struct kunit_case user4k_tlb_test_cases[] = { + KUNIT_CASE(user4k_tlbi_operands_test), + KUNIT_CASE(user4k_tlb_gather_test), + {} +}; + +static struct kunit_suite user4k_tlb_test_suite = { + .name = "arm64-user4k-tlb", + .test_cases = user4k_tlb_test_cases, +}; + +kunit_test_suite(user4k_tlb_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index de92d63fdbc76..432df2ce9ddfd 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -69,6 +69,8 @@ HAS_TIDCP1 HAS_TLB_RANGE HAS_TSO_APPLE HAS_TSO_FIXED +HAS_USER4K_GRANULE +HAS_USER16K_GRANULE HAS_VA52 HAS_VIRT_HOST_EXTN HAS_WFXT diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 3af979fb41d38..3155eed81c2ec 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -926,13 +926,15 @@ static int is_call_insn(uprobe_opcode_t *insn) * Verification callback used by int3_update uprobe_write calls to make sure * the underlying instruction is as expected - either int3 or call. */ -static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode, +static int verify_insn(struct page *page, unsigned long vaddr, unsigned int page_offset, + uprobe_opcode_t *new_opcode, int nbytes, void *data) { struct write_opcode_ctx *ctx = data; uprobe_opcode_t old_opcode[5]; - uprobe_copy_from_page(page, ctx->base, (uprobe_opcode_t *) &old_opcode, 5); + uprobe_copy_from_page(page, page_offset - (vaddr - ctx->base), + (uprobe_opcode_t *) &old_opcode, 5); switch (ctx->expect) { case EXPECT_SWBP: diff --git a/block/bio.c b/block/bio.c index 5f10900b3f42a..846417dec7d16 100644 --- a/block/bio.c +++ b/block/bio.c @@ -3,6 +3,7 @@ * Copyright (C) 2001 Jens Axboe */ #include +#include #include #include #include @@ -246,6 +247,10 @@ void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, #endif #ifdef CONFIG_BLK_DEV_INTEGRITY bio->bi_integrity = NULL; +#endif +#ifdef CONFIG_MM_SUBPAGE + bio->bi_user_fragments = NULL; + bio->bi_nr_user_fragments = 0; #endif bio->bi_vcnt = 0; @@ -1159,6 +1164,18 @@ void __bio_release_pages(struct bio *bio, bool mark_dirty) { struct folio_iter fi; +#ifdef CONFIG_MM_SUBPAGE + if (bio->bi_user_fragments) { + release_user_fragments(bio->bi_user_fragments, + bio->bi_nr_user_fragments, mark_dirty); + kfree(bio->bi_user_fragments); + bio->bi_user_fragments = NULL; + bio->bi_nr_user_fragments = 0; + bio_clear_flag(bio, BIO_PAGE_PINNED); + return; + } +#endif + bio_for_each_folio_all(fi, bio) { size_t nr_pages; @@ -1220,6 +1237,134 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, return 0; } +#ifdef CONFIG_MM_SUBPAGE +static bool bio_subpage_iter(const struct iov_iter *iter) +{ + return user_backed_iter(iter) && current->mm && + mm_page_size(current->mm) < PAGE_SIZE; +} + +int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs) +{ + struct iov_iter walk = *iter; + unsigned int count = 0; + + if (!bio_subpage_iter(iter)) + return iov_iter_npages(iter, max_vecs); + while (iov_iter_count(&walk) && count < max_vecs) { + size_t len, offset, nr; + + iov_iter_advance(&walk, 0); + len = min(iter_iov_len(&walk), iov_iter_count(&walk)); + if (WARN_ON_ONCE(!len)) + return max_vecs; + offset = (unsigned long)iter_iov_addr(&walk) & (MM_SUBPAGE_SIZE - 1); + nr = (len >> MM_SUBPAGE_SHIFT) + + DIV_ROUND_UP(offset + (len & (MM_SUBPAGE_SIZE - 1)), MM_SUBPAGE_SIZE); + if (nr >= max_vecs - count) + return max_vecs; + count += nr; + iov_iter_advance(&walk, len); + } + return count; +} +EXPORT_SYMBOL_GPL(bio_iov_user_vecs); + +/* Pins belong to the submitting bio, just as with ordinary BIO_PAGE_PINNED. + * Split/cloned bios borrow its vectors and complete before their owner does. + */ +static int bio_iov_iter_get_fragments(struct bio *bio, struct iov_iter *iter, + unsigned int align_mask) +{ + unsigned int capacity = bio->bi_max_vecs, total = 0; + unsigned int flags = iov_iter_rw(iter) == READ ? FOLL_WRITE : 0; + int ret = 0; + + if (WARN_ON_ONCE(bio->bi_vcnt || bio->bi_user_fragments || !capacity)) + return -EINVAL; + bio->bi_user_fragments = kmalloc_array(capacity, + sizeof(*bio->bi_user_fragments), GFP_KERNEL); + if (!bio->bi_user_fragments) + return -ENOMEM; + while (iov_iter_count(iter) && bio->bi_nr_user_fragments < capacity && + total < BIO_MAX_SIZE) { + unsigned int i, bytes = 0; + long nr; + + iov_iter_advance(iter, 0); + nr = pin_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(iter), + min3(iter_iov_len(iter), iov_iter_count(iter), + (size_t)(BIO_MAX_SIZE - total)), flags, + bio->bi_user_fragments + bio->bi_nr_user_fragments, + capacity - bio->bi_nr_user_fragments); + if (nr <= 0) { + ret = nr ?: -EFAULT; + break; + } + for (i = 0; i < nr; i++) { + struct user_page_fragment *fragment = + &bio->bi_user_fragments[bio->bi_nr_user_fragments++]; + struct page *page = folio_page(fragment->folio, + fragment->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(fragment->offset); + struct bio_vec *prev = bio->bi_vcnt ? + &bio->bi_io_vec[bio->bi_vcnt - 1] : NULL; + + /* Coalesce physically adjacent quarters within one folio. */ + if (prev && page_folio(prev->bv_page) == fragment->folio && + page_to_phys(prev->bv_page) + prev->bv_offset + prev->bv_len == + page_to_phys(page) + offset) + prev->bv_len += fragment->length; + else + bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt++], page, + fragment->length, offset); + bytes += fragment->length; + } + iov_iter_advance(iter, bytes); + total += bytes; + } + /* Trim pins separately from vectors: several pins may share one bvec. */ + if (total & align_mask) { + unsigned int trim = total & align_mask, left = trim; + + while (left) { + struct user_page_fragment *fragment = + &bio->bi_user_fragments[bio->bi_nr_user_fragments - 1]; + unsigned int bytes = min(left, fragment->length); + + if (bytes == fragment->length) { + release_user_fragments(fragment, 1, false); + bio->bi_nr_user_fragments--; + } else { + fragment->length -= bytes; + } + left -= bytes; + } + left = trim; + while (left) { + struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; + unsigned int bytes = min(left, bv->bv_len); + + bv->bv_len -= bytes; + if (!bv->bv_len) + bio->bi_vcnt--; + left -= bytes; + } + iov_iter_revert(iter, trim); + total -= trim; + } + bio->bi_iter.bi_size = total; + if (!total) { + kfree(bio->bi_user_fragments); + bio->bi_user_fragments = NULL; + return ret ?: -EFAULT; + } + bio_set_flag(bio, BIO_PAGE_PINNED); + return 0; +} +#endif + /** * bio_iov_iter_get_pages - add user or kernel pages to a bio * @bio: bio to add pages to @@ -1255,6 +1400,11 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, return 0; } +#ifdef CONFIG_MM_SUBPAGE + if (bio_subpage_iter(iter)) + return bio_iov_iter_get_fragments(bio, iter, len_align_mask); +#endif + if (iov_iter_extract_will_pin(iter)) bio_set_flag(bio, BIO_PAGE_PINNED); if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue)) diff --git a/block/blk-map.c b/block/blk-map.c index 768549f19f97e..b98eed276aea2 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -260,7 +260,7 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, gfp_t gfp_mask) { - unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS); + unsigned int nr_vecs = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS); struct bio *bio; int ret; diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index d504c636dc294..0cb790ab2b6a3 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -225,9 +225,23 @@ static struct file_system_type dma_buf_fs_type = { .kill_sb = kill_anon_super, }; +static int dma_buf_mmap_range_check(struct dma_buf *dmabuf, + struct vm_area_struct *vma, unsigned long pgoff, unsigned int suboffset) +{ + unsigned long offset; + + if (check_mul_overflow(pgoff, PAGE_SIZE, &offset) || + check_add_overflow(offset, (unsigned long)suboffset, &offset)) + return -EOVERFLOW; + if (offset > dmabuf->size || vma->vm_end - vma->vm_start > dmabuf->size - offset) + return -EINVAL; + return 0; +} + static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) { struct dma_buf *dmabuf; + int ret; if (!is_dma_buf_file(file)) return -EINVAL; @@ -239,9 +253,9 @@ static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) return -EINVAL; /* check for overflowing the buffer's size */ - if (vma->vm_pgoff + vma_pages(vma) > - dmabuf->size >> PAGE_SHIFT) - return -EINVAL; + ret = dma_buf_mmap_range_check(dmabuf, vma, vma->vm_pgoff, vma_subpage_offset(vma)); + if (ret) + return ret; DMA_BUF_TRACE(trace_dma_buf_mmap_internal, dmabuf); @@ -1583,6 +1597,8 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_end_cpu_access, "DMA_BUF"); int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, unsigned long pgoff) { + int ret; + if (WARN_ON(!dmabuf || !vma)) return -EINVAL; @@ -1590,18 +1606,13 @@ int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, if (!dmabuf->ops->mmap) return -EINVAL; - /* check for offset overflow */ - if (pgoff + vma_pages(vma) < pgoff) - return -EOVERFLOW; - - /* check for overflowing the buffer's size */ - if (pgoff + vma_pages(vma) > - dmabuf->size >> PAGE_SHIFT) - return -EINVAL; + ret = dma_buf_mmap_range_check(dmabuf, vma, pgoff, 0); + if (ret) + return ret; /* readjust the vma */ vma_set_file(vma, dmabuf->file); - vma->vm_pgoff = pgoff; + vma_set_page_offset(vma, (struct vm_page_offset) { .index = pgoff }); DMA_BUF_TRACE(trace_dma_buf_mmap, dmabuf); diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index 3b2448a3a9de8..2176ab547bfb6 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -1343,7 +1343,7 @@ unsigned long drm_gem_get_unmapped_area(struct file *filp, unsigned long uaddr, struct drm_gem_object *obj; unsigned long ret; - obj = drm_gem_object_lookup_at_offset(filp, pgoff, len >> PAGE_SHIFT); + obj = drm_gem_object_lookup_at_offset(filp, pgoff, DIV_ROUND_UP(len, PAGE_SIZE)); if (IS_ERR(obj)) obj = NULL; @@ -1379,8 +1379,11 @@ int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma) struct drm_gem_object *obj; int ret; + /* GEM fake offsets identify an exact object start, not an interior byte. */ + if (vma_subpage_offset(vma)) + return -EINVAL; obj = drm_gem_object_lookup_at_offset(filp, vma->vm_pgoff, - vma_pages(vma)); + DIV_ROUND_UP(vma->vm_end - vma->vm_start, PAGE_SIZE)); if (IS_ERR(obj)) return PTR_ERR(obj); diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 5b8044508cfcf..7d019582c0cc2 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -554,13 +554,23 @@ int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev, } EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create); +/* GEM and PRIME add the object's fake offset; VMA splits retain it. + * Direct helper callers may instead supply an object-relative offset. + */ +static pgoff_t drm_gem_shmem_object_offset(struct drm_gem_object *obj, pgoff_t pgoff) +{ + pgoff_t base = drm_vma_node_start(&obj->vma_node); + + return pgoff >= base ? pgoff - base : pgoff; +} + static void drm_gem_shmem_record_mkwrite(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *obj = vma->vm_private_data; struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); loff_t num_pages = obj->size >> PAGE_SHIFT; - pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */ + pgoff_t page_offset = drm_gem_shmem_object_offset(obj, vmf->pgoff); if (drm_WARN_ON(obj->dev, !shmem->pages || page_offset >= num_pages)) return; @@ -587,7 +597,10 @@ static vm_fault_t try_insert_pfn(struct vm_fault *vmf, unsigned int order, vm_fault_t ret = VM_FAULT_FALLBACK; if (!order) { - ret = vmf_insert_pfn_mkwrite(vmf->vma, vmf->address, pfn, write); + struct vm_page_offset pos = vma_page_offset_at(vmf->vma, vmf->address); + + ret = vmf_insert_pfn_prot_mkwrite_offset(vmf->vma, vmf->address, + pfn, pos.offset, vmf->vma->vm_page_prot, write); #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP } else if (order == PMD_ORDER) { unsigned long paddr = pfn << PAGE_SHIFT; @@ -619,12 +632,12 @@ static vm_fault_t drm_gem_shmem_any_fault(struct vm_fault *vmf, unsigned int ord loff_t num_pages = obj->size >> PAGE_SHIFT; vm_fault_t ret = VM_FAULT_SIGBUS; struct page **pages = shmem->pages; - pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */ + pgoff_t page_offset = drm_gem_shmem_object_offset(obj, vmf->pgoff); struct page *page; struct folio *folio; unsigned long pfn; - if (order && order != PMD_ORDER) + if (order && (order != PMD_ORDER || mm_page_size(vma->vm_mm) != PAGE_SIZE)) return VM_FAULT_FALLBACK; dma_resv_lock(obj->resv, NULL); @@ -742,6 +755,17 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct if (is_cow_mapping(vma->vm_flags)) return -EINVAL; + /* Validate bytes, including a partial native-page offset from PRIME. */ + { + pgoff_t index = drm_gem_shmem_object_offset(obj, vma->vm_pgoff); + unsigned long offset, length = vma->vm_end - vma->vm_start; + + if (check_mul_overflow(index, PAGE_SIZE, &offset) || + check_add_overflow(offset, (unsigned long)vma_subpage_offset(vma), &offset) || + offset > obj->size || length > obj->size - offset) + return -EINVAL; + } + dma_resv_lock(shmem->base.resv, NULL); ret = drm_gem_shmem_get_pages_locked(shmem); dma_resv_unlock(shmem->base.resv); diff --git a/drivers/iommu/iommufd/Makefile b/drivers/iommu/iommufd/Makefile index 71d692c9a8f49..a868155e208d5 100644 --- a/drivers/iommu/iommufd/Makefile +++ b/drivers/iommu/iommufd/Makefile @@ -10,6 +10,8 @@ iommufd-y := \ vfio_compat.o \ viommu.o +iommufd-$(CONFIG_MM_SUBPAGE) += fragments.o + iommufd-$(CONFIG_IOMMUFD_TEST) += selftest.o obj-$(CONFIG_IOMMUFD) += iommufd.o diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c index 2895e5370910f..3ccb352bb3e50 100644 --- a/drivers/iommu/iommufd/device.c +++ b/drivers/iommu/iommufd/device.c @@ -1171,6 +1171,18 @@ struct iommufd_access *iommufd_access_create_internal(struct iommufd_ctx *ictx) return access; } +/* Physical byte-span consumers do not impose native-page IOVA alignment. */ +struct iommufd_access *iommufd_access_create_internal_phys(struct iommufd_ctx *ictx) +{ + struct iommufd_access *access = __iommufd_access_create(ictx); + + if (IS_ERR(access)) + return access; + access->iova_alignment = 1; + iommufd_object_finalize(ictx, &access->obj); + return access; +} + /** * iommufd_access_create - Create an iommufd_access * @ictx: iommufd file descriptor @@ -1404,6 +1416,12 @@ static bool check_area_prot(struct iopt_area *area, unsigned int flags) * ioas alignment is >= PAGE_SIZE and the iova is PAGE_SIZE aligned. However * smaller alignments have corner cases where this API can fail on otherwise * aligned iova. + * + * Alternative user granules are accepted only if each native output page has + * an aligned physical base and consecutive backing bytes within the requested + * range. The driver must access only @length bytes: a partial final page does + * not grant access to its remaining bytes. Fragment ownership stays in the IOAS + * until unpin; incompatible physical layouts return -EINVAL. */ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, unsigned long length, struct page **out_pages, @@ -1456,7 +1474,7 @@ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, flags, internal); if (rc) goto err_remove; - out_pages += last_index - index + 1; + out_pages += (last - iter.cur_iova) / PAGE_SIZE + 1; } if (!iopt_area_contig_done(&iter)) { rc = -ENOENT; @@ -1485,6 +1503,88 @@ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, } EXPORT_SYMBOL_NS_GPL(iommufd_access_pin_pages, "IOMMUFD"); +/** + * iommufd_access_pin_phys - Pin an internal contiguous physical byte span + * @access: Internal access created for byte-span pinning + * @iova: First IOVA byte + * @length: Exact byte length + * @out_phys: Physical address of the first byte, set only on success + * + * Retains native or fragment pin ownership and prevents IOAS unmap until the + * matching iommufd_access_unpin_pages(). No surrounding IOVA bytes are required. + * The caller must stop hardware access before unpinning or destroying access. + */ +int iommufd_access_pin_phys(struct iommufd_access *access, unsigned long iova, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_area_contig_iter iter; + struct io_pagetable *iopt; + struct iopt_area *area; + unsigned long last_iova; + phys_addr_t base = 0; + int rc; + + if (!iommufd_access_is_internal(access) || !out_phys || !length) + return -EINVAL; + if (check_add_overflow(iova, length - 1, &last_iova)) + return -EOVERFLOW; + + mutex_lock(&access->ioas_lock); + if (!access->ioas) { + rc = -ENOENT; + goto out_unlock; + } + iopt = &access->ioas->iopt; + down_read(&iopt->iova_rwsem); + iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) { + unsigned long last = min(last_iova, iopt_area_last_iova(area)); + phys_addr_t phys; + + if (area->prevent_access || !(area->iommu_prot & IOMMU_READ)) { + rc = -EPERM; + goto out_remove; + } + rc = iopt_area_add_phys_access(area, + iopt_area_start_byte(area, iter.cur_iova), + last - iter.cur_iova + 1, &phys); + if (rc) + goto out_remove; + if (iter.cur_iova == iova) { + base = phys; + } else if (phys != base + (iter.cur_iova - iova)) { + iopt_area_remove_access(area, + iopt_area_iova_to_index(area, iter.cur_iova), + iopt_area_iova_to_index(area, last), true); + rc = -EFAULT; + goto out_remove; + } + } + if (!iopt_area_contig_done(&iter)) { + rc = -ENOENT; + goto out_remove; + } + *out_phys = base; + up_read(&iopt->iova_rwsem); + rc = 0; + goto out_unlock; + +out_remove: + /* iter.cur_iova is the first byte whose access was not retained. */ + if (iter.cur_iova > iova) { + unsigned long end = iter.cur_iova - 1; + + iopt_for_each_contig_area(&iter, area, iopt, iova, end) + iopt_area_remove_access(area, + iopt_area_iova_to_index(area, iter.cur_iova), + iopt_area_iova_to_index(area, min(end, iopt_area_last_iova(area))), + true); + } + up_read(&iopt->iova_rwsem); +out_unlock: + mutex_unlock(&access->ioas_lock); + return rc; +} + /** * iommufd_access_rw - Read or write data under the iova * @access: IOAS access to act on diff --git a/drivers/iommu/iommufd/fragments.c b/drivers/iommu/iommufd/fragments.c new file mode 100644 index 0000000000000..5229605852133 --- /dev/null +++ b/drivers/iommu/iommufd/fragments.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Alternative user granules need an ownership record in addition to a physical + * address. A device translation cannot retain a subpage allocation identity. + * Keep those records until the final domain interval has gone away. The native + * PFN reader remains in pages.c; file and dma-buf suppliers retain its geometry. + */ +#include +#include +#include +#include + +#include "iommufd_private.h" +#include "io_pagetable.h" + +static phys_addr_t fragment_phys(const struct user_page_fragment *fragment) +{ + return PFN_PHYS(folio_pfn(fragment->folio)) + fragment->offset; +} + +/* Charge each native physical page once, even when several slots pin it. */ +static int fragment_store(struct iopt_pages *pages, unsigned long index, + struct user_page_fragment *fragment) +{ + unsigned long pfn = PHYS_PFN(fragment_phys(fragment)); + unsigned long count = xa_to_value(xa_load(&pages->fragment_pages, pfn)); + struct user_page_fragment *record; + int rc; + + record = kmemdup(fragment, sizeof(*record), GFP_KERNEL_ACCOUNT); + if (!record) + return -ENOMEM; + rc = xa_reserve(&pages->fragments, index, GFP_KERNEL_ACCOUNT); + if (rc) + goto out_free; + rc = xa_err(xa_store(&pages->fragment_pages, pfn, xa_mk_value(count + 1), + GFP_KERNEL_ACCOUNT)); + if (rc) + goto out_release; + /* xa_reserve() made this infallible. Ownership transfers here. */ + xa_store(&pages->fragments, index, record, GFP_NOWAIT); + if (!count) + pages->npinned++; + *fragment = (struct user_page_fragment) {}; + return 0; + +out_release: + xa_release(&pages->fragments, index); +out_free: + kfree(record); + return rc; +} + +/* No allocation on teardown, and callers have already unmapped all devices. */ +static void fragments_release_unused(struct iopt_pages *pages, + unsigned long first, unsigned long last) +{ + struct user_page_fragment *record; + unsigned long index = first; + + lockdep_assert_held(&pages->mutex); + xa_for_each_range(&pages->fragments, index, record, first, last) { + unsigned long pfn, count; + + if (interval_tree_iter_first(&pages->domains_itree, index, index) || + interval_tree_iter_first(&pages->access_itree, index, index)) + continue; + pfn = PHYS_PFN(fragment_phys(record)); + count = xa_to_value(xa_load(&pages->fragment_pages, pfn)); + if (WARN_ON(!count)) + continue; + if (count == 1) { + xa_erase(&pages->fragment_pages, pfn); + pages->npinned--; + } else { + xa_store(&pages->fragment_pages, pfn, xa_mk_value(count - 1), + GFP_NOWAIT); + } + xa_erase(&pages->fragments, index); + release_user_fragments(record, 1, pages->writable); + kfree(record); + cond_resched(); + } + if (pages->npinned < pages->last_npinned) + iopt_pages_update_pinned(pages, pages->last_npinned - pages->npinned, + false, NULL); +} + +/* Reuse existing records even if the original VA was unmapped or replaced. */ +static int fragments_acquire(struct iopt_pages *pages, unsigned long first, + unsigned long last) +{ + unsigned long size = 1UL << pages->page_shift, index = first; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + struct user_page_fragment *batch = NULL; + bool mm_live = false; + int rc = 0; + + lockdep_assert_held(&pages->mutex); + while (index <= last) { + unsigned long next = index, end, i; + long count; + + if (xa_load(&pages->fragments, index)) { + index++; + continue; + } + if (!mm_live) { + if (!mmget_not_zero(pages->source_mm)) { + rc = -EFAULT; + goto out; + } + mm_live = true; + batch = kmalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!batch) { + rc = -ENOMEM; + goto out; + } + } + /* Stop before any cached pin; never replace its ownership record. */ + end = xa_find(&pages->fragments, &next, last, XA_PRESENT) ? next : last + 1; + end = min(end, index + capacity); + count = pin_user_fragments_remote(pages->source_mm, + (unsigned long)pages->uptr + (index << pages->page_shift), + (end - index) << pages->page_shift, + FOLL_LONGTERM | (pages->writable ? FOLL_WRITE : 0), batch, capacity); + if (count <= 0) { + rc = count ?: -EFAULT; + goto out; + } + for (i = 0; i < count; i++) { + /* Each aligned logical index must have exactly one full pin. */ + if (WARN_ON(batch[i].length != size || + !IS_ALIGNED(fragment_phys(&batch[i]), size))) + rc = -EFAULT; + else + rc = fragment_store(pages, index, &batch[i]); + if (rc) { + release_user_fragments(batch + i, count - i, false); + goto out; + } + index++; + } + cond_resched(); + } + if (pages->npinned != pages->last_npinned) { + if (iommufd_should_fail()) + rc = -ENOMEM; + else + rc = iopt_pages_update_pinned(pages, + pages->npinned - pages->last_npinned, true, NULL); + } +out: + kfree(batch); + if (mm_live) + mmput(pages->source_mm); + if (rc) + fragments_release_unused(pages, first, last); + return rc; +} + +static void fragments_unmap(struct iommu_domain *domain, unsigned long iova, + unsigned long length) +{ + if (length) + WARN_ON(iommu_unmap(domain, iova, length) != length); +} + +/* All pins are present. Merge physically consecutive fragments before mapping. */ +static int fragments_map(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + unsigned long size = 1UL << pages->page_shift; + unsigned long index = iopt_area_index(area), last = iopt_area_last_index(area); + unsigned long remaining = iopt_area_length(area), done = 0; + unsigned long offset = area->page_offset; + unsigned long min_size = 1UL << __ffs(domain->pgsize_bitmap); + int rc; + + while (remaining) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + phys_addr_t phys = fragment_phys(record) + offset; + unsigned long bytes = min(size - offset, remaining); + + index++; + /* Legacy splitting forbids mappings larger than a native page. */ + while (index <= last && bytes < remaining && + (!area->iopt->disable_large_pages || bytes < PAGE_SIZE)) { + record = xa_load(&pages->fragments, index); + if (fragment_phys(record) != phys + bytes) + break; + bytes += min(size, remaining - bytes); + index++; + } + /* Never widen a device mapping to include an unowned neighbour. */ + if (!IS_ALIGNED(phys | (iopt_area_iova(area) + done) | bytes, min_size)) { + rc = -EINVAL; + goto out_unmap; + } + rc = iommu_map(domain, iopt_area_iova(area) + done, phys, bytes, + area->iommu_prot, GFP_KERNEL_ACCOUNT); + if (rc) + goto out_unmap; + done += bytes; + remaining -= bytes; + offset = 0; + cond_resched(); + } + return 0; + +out_unmap: + fragments_unmap(domain, iopt_area_iova(area), done); + return rc; +} + +int iopt_fragments_fill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + int rc; + + lockdep_assert_held(&pages->mutex); + rc = fragments_acquire(pages, iopt_area_index(area), iopt_area_last_index(area)); + if (rc) + return rc; + rc = fragments_map(area, pages, domain); + if (rc) + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); + return rc; +} + +int iopt_fragments_fill_domains(struct iopt_area *area, struct iopt_pages *pages) +{ + struct iommu_domain *domain; + unsigned long index, failed; + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, iopt_area_index(area), iopt_area_last_index(area)); + if (rc) + return rc; + xa_for_each(&area->iopt->domains, index, domain) { + rc = fragments_map(area, pages, domain); + if (rc) + goto out_unmap; + } + area->storage_domain = xa_load(&area->iopt->domains, 0); + interval_tree_insert(&area->pages_node, &pages->domains_itree); + return 0; + +out_unmap: + failed = index; + xa_for_each(&area->iopt->domains, index, domain) { + if (index == failed) + break; + fragments_unmap(domain, iopt_area_iova(area), iopt_area_length(area)); + } + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); + return rc; +} + +void iopt_fragments_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + lockdep_assert_held(&pages->mutex); + fragments_unmap(domain, iopt_area_iova(area), iopt_area_length(area)); + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); +} + +int iopt_fragments_rw(struct iopt_pages *pages, unsigned long start_byte, + void *data, unsigned long length, unsigned int flags) +{ + unsigned long size = 1UL << pages->page_shift; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + unsigned long index, offset = start_byte & (size - 1); + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + for (index = first; index <= last; index++) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + unsigned long bytes = min(size - offset, length); + void *addr = kmap_local_folio(record->folio, record->offset + offset); + + if (flags & IOMMUFD_ACCESS_RW_WRITE) + memcpy(addr, data, bytes); + else + memcpy(data, addr, bytes); + kunmap_local(addr); + data += bytes; + length -= bytes; + offset = 0; + } + fragments_release_unused(pages, first, last); + return 0; +} + +/* The retained typed records, not the output page array, own the pins. */ +static int fragments_register_access(struct iopt_area *area, unsigned long first, + unsigned long last, bool lock_area) +{ + struct iopt_pages *pages = area->pages; + struct iopt_pages_access *access; + struct interval_tree_node *node; + + lockdep_assert_held(&pages->mutex); + for (node = interval_tree_iter_first(&pages->access_itree, first, last); + node; node = interval_tree_iter_next(node, first, last)) { + if (node->start == first && node->last == last) { + access = container_of(node, struct iopt_pages_access, node); + access->users++; + goto out_count; + } + } + access = kzalloc_obj(*access, GFP_KERNEL_ACCOUNT); + if (!access) + return -ENOMEM; + access->node.start = first; + access->node.last = last; + access->users = 1; + interval_tree_insert(&access->node, &pages->access_itree); +out_count: + area->num_accesses++; + if (lock_area) + area->num_locks++; + return 0; +} + +/* Register a physical byte span without discarding fragment ownership. */ +int iopt_fragments_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_pages *pages = area->pages; + unsigned long size = 1UL << pages->page_shift; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + struct user_page_fragment *record; + unsigned long index; + phys_addr_t base; + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + record = xa_load(&pages->fragments, first); + base = fragment_phys(record); + for (index = first + 1; index <= last; index++) { + record = xa_load(&pages->fragments, index); + if (fragment_phys(record) != + base + ((index - first) << pages->page_shift)) { + rc = -EFAULT; + goto out_release; + } + cond_resched(); + } + rc = fragments_register_access(area, first, last, true); + if (rc) + goto out_release; + *out_phys = base + (start_byte & (size - 1)); + return 0; + +out_release: + fragments_release_unused(pages, first, last); + return rc; +} + +/* + * A native page pointer has no fragment offset. Require an aligned physical + * base and consecutive fragments within each output page; different output + * pages may be scattered. Only the requested logical interval is retained. + */ +int iopt_fragments_add_access(struct iopt_area *area, unsigned long first, + unsigned long last, struct page **out_pages, + bool lock_area) +{ + struct iopt_pages *pages = area->pages; + unsigned long per_page = 1UL << (PAGE_SHIFT - pages->page_shift); + unsigned long index; + phys_addr_t base = 0; + int rc; + + if (first & (per_page - 1)) + return -EINVAL; + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + for (index = first; index <= last; index++) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + unsigned long offset = (index - first) & (per_page - 1); + phys_addr_t phys = fragment_phys(record); + + if (!offset) { + base = phys; + if (!IS_ALIGNED(base, PAGE_SIZE)) { + rc = -EINVAL; + goto out_release; + } + *out_pages++ = pfn_to_page(PHYS_PFN(base)); + } else if (phys != base + (offset << pages->page_shift)) { + rc = -EINVAL; + goto out_release; + } + cond_resched(); + } + rc = fragments_register_access(area, first, last, lock_area); + if (!rc) + return 0; +out_release: + fragments_release_unused(pages, first, last); + return rc; +} + +void iopt_fragments_unfill_access(struct iopt_pages *pages, unsigned long first, + unsigned long last) +{ + fragments_release_unused(pages, first, last); +} diff --git a/drivers/iommu/iommufd/io_pagetable.c b/drivers/iommu/iommufd/io_pagetable.c index 24d4917105d94..bf8cabee53fb6 100644 --- a/drivers/iommu/iommufd/io_pagetable.c +++ b/drivers/iommu/iommufd/io_pagetable.c @@ -217,7 +217,8 @@ static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area, return -EPERM; area->iommu_prot = iommu_prot; - area->page_offset = start_byte % PAGE_SIZE; + area->page_shift = pages->page_shift; + area->page_offset = start_byte & ((1UL << area->page_shift) - 1); if (area->page_offset & (iopt->iova_alignment - 1)) return -EINVAL; @@ -225,10 +226,10 @@ static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area, if (check_add_overflow(iova, length - 1, &area->node.last)) return -EOVERFLOW; - area->pages_node.start = start_byte / PAGE_SIZE; + area->pages_node.start = start_byte >> area->page_shift; if (check_add_overflow(start_byte, length - 1, &area->pages_node.last)) return -EOVERFLOW; - area->pages_node.last = area->pages_node.last / PAGE_SIZE; + area->pages_node.last >>= area->page_shift; if (WARN_ON(area->pages_node.last >= pages->npages)) return -EOVERFLOW; @@ -572,7 +573,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, struct iommu_dirty_bitmap dirty; struct iova_bitmap_fn_arg arg; struct iova_bitmap *iter; - int ret = 0; + int ret; if (!ops || !ops->read_and_clear_dirty) return -EOPNOTSUPP; @@ -581,7 +582,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, bitmap->page_size, u64_to_user_ptr(bitmap->data)); if (IS_ERR(iter)) - return -ENOMEM; + return PTR_ERR(iter); iommu_dirty_bitmap_init(&dirty, iter, &gather); @@ -589,7 +590,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, arg.iopt = iopt; arg.domain = domain; arg.dirty = &dirty; - iova_bitmap_for_each(iter, &arg, __iommu_read_and_clear_dirty); + ret = iova_bitmap_for_each(iter, &arg, __iommu_read_and_clear_dirty); if (!(flags & IOMMU_DIRTY_NO_CLEAR)) iommu_iotlb_sync(domain, &gather); @@ -615,7 +616,7 @@ int iommufd_check_iova_range(struct io_pagetable *iopt, ((last_iova + 1) & (iommu_pgsize - 1))) return -EINVAL; - if (!bitmap->page_size) + if (!is_power_of_2(bitmap->page_size)) return -EINVAL; if ((bitmap->iova & (bitmap->page_size - 1)) || diff --git a/drivers/iommu/iommufd/io_pagetable.h b/drivers/iommu/iommufd/io_pagetable.h index 27e3e311d395b..d15e51fee0c10 100644 --- a/drivers/iommu/iommufd/io_pagetable.h +++ b/drivers/iommu/iommufd/io_pagetable.h @@ -5,6 +5,7 @@ #ifndef __IO_PAGETABLE_H #define __IO_PAGETABLE_H +#include #include #include #include @@ -45,6 +46,7 @@ struct iopt_area { struct iommu_domain *storage_domain; /* How many bytes into the first page the area starts */ unsigned int page_offset; + u8 page_shift; /* IOMMU_READ, IOMMU_WRITE, etc */ int iommu_prot; bool prevent_access : 1; @@ -107,8 +109,8 @@ static inline size_t iopt_area_length(struct iopt_area *area) /* * Number of bytes from the start of the iopt_pages that the iova begins. - * iopt_area_start_byte() / PAGE_SIZE encodes the starting page index - * iopt_area_start_byte() % PAGE_SIZE encodes the offset within that page + * iopt_area_start_byte() >> area->page_shift encodes the starting page index + * iopt_area_start_byte() % (1UL << area->page_shift) encodes the offset within that page */ static inline unsigned long iopt_area_start_byte(struct iopt_area *area, unsigned long iova) @@ -117,13 +119,13 @@ static inline unsigned long iopt_area_start_byte(struct iopt_area *area, WARN_ON(iova < iopt_area_iova(area) || iova > iopt_area_last_iova(area)); return (iova - iopt_area_iova(area)) + area->page_offset + - iopt_area_index(area) * PAGE_SIZE; + (iopt_area_index(area) << area->page_shift); } static inline unsigned long iopt_area_iova_to_index(struct iopt_area *area, unsigned long iova) { - return iopt_area_start_byte(area, iova) / PAGE_SIZE; + return iopt_area_start_byte(area, iova) >> area->page_shift; } #define __make_iopt_iter(name) \ @@ -214,7 +216,7 @@ struct iopt_pages_dmabuf { * io_pagetable's, through their iopt_area's, can share a single iopt_pages * which avoids multi-pinning and double accounting of page consumption. * - * indexes in this structure are measured in PAGE_SIZE units, are 0 based from + * Indexes use (1UL << page_shift) units and are 0 based from * the start of the uptr and extend to npages. pages are pinned dynamically * according to the intervals in the access_itree and domains_itree, npinned * records the current number of pages pinned. @@ -222,6 +224,8 @@ struct iopt_pages_dmabuf { struct iopt_pages { struct kref kref; struct mutex mutex; + /* Logical indices use page_shift; pin accounting remains native pages. */ + u8 page_shift; size_t npages; size_t npinned; size_t last_npinned; @@ -242,12 +246,73 @@ struct iopt_pages { u8 account_mode; struct xarray pinned_pfns; +#ifdef CONFIG_MM_SUBPAGE + /* Exact fragment pins, indexed independently of device translations. */ + struct xarray fragments; + /* Native physical PFN -> fragment count, for pin accounting. */ + struct xarray fragment_pages; +#endif /* Of iopt_pages_access::node */ struct rb_root_cached access_itree; /* Of iopt_area::pages_node */ struct rb_root_cached domains_itree; }; +static inline bool iopt_pages_use_fragments(struct iopt_pages *pages) +{ + return IS_ENABLED(CONFIG_MM_SUBPAGE) && pages->type == IOPT_ADDRESS_USER && + mm_page_size(pages->source_mm) != PAGE_SIZE; +} + +#ifdef CONFIG_MM_SUBPAGE +int iopt_fragments_fill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain); +int iopt_fragments_fill_domains(struct iopt_area *area, struct iopt_pages *pages); +void iopt_fragments_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain); +int iopt_fragments_add_access(struct iopt_area *area, unsigned long first, + unsigned long last, struct page **out_pages, + bool lock_area); +int iopt_fragments_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys); +void iopt_fragments_unfill_access(struct iopt_pages *pages, unsigned long first, + unsigned long last); +int iopt_fragments_rw(struct iopt_pages *pages, unsigned long start_byte, + void *data, unsigned long length, unsigned int flags); +#else +static inline int iopt_fragments_fill_domain(struct iopt_area *area, + struct iopt_pages *pages, struct iommu_domain *domain) +{ + return -EOPNOTSUPP; +} +static inline int iopt_fragments_fill_domains(struct iopt_area *area, + struct iopt_pages *pages) +{ + return -EOPNOTSUPP; +} +static inline void iopt_fragments_unfill_domain(struct iopt_area *area, + struct iopt_pages *pages, struct iommu_domain *domain) { } +static inline int iopt_fragments_add_access(struct iopt_area *area, + unsigned long first, unsigned long last, struct page **out_pages, + bool lock_area) +{ + return -EOPNOTSUPP; +} +static inline int iopt_fragments_add_phys_access(struct iopt_area *area, + unsigned long start_byte, unsigned long length, phys_addr_t *out_phys) +{ + return -EOPNOTSUPP; +} +static inline void iopt_fragments_unfill_access(struct iopt_pages *pages, + unsigned long first, unsigned long last) { } +static inline int iopt_fragments_rw(struct iopt_pages *pages, + unsigned long start_byte, void *data, unsigned long length, + unsigned int flags) +{ + return -EOPNOTSUPP; +} +#endif + static inline bool iopt_is_dmabuf(struct iopt_pages *pages) { if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) @@ -290,6 +355,8 @@ void iopt_pages_unfill_xarray(struct iopt_pages *pages, unsigned long start, int iopt_area_add_access(struct iopt_area *area, unsigned long start, unsigned long last, struct page **out_pages, unsigned int flags, bool lock_area); +int iopt_area_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys); void iopt_area_remove_access(struct iopt_area *area, unsigned long start, unsigned long last, bool unlock_area); int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h index 43fbc5bed8de3..d1a14e51df55e 100644 --- a/drivers/iommu/iommufd/iommufd_private.h +++ b/drivers/iommu/iommufd/iommufd_private.h @@ -531,6 +531,10 @@ static inline bool iommufd_access_is_internal(struct iommufd_access *access) } struct iommufd_access *iommufd_access_create_internal(struct iommufd_ctx *ictx); +struct iommufd_access *iommufd_access_create_internal_phys(struct iommufd_ctx *ictx); +int iommufd_access_pin_phys(struct iommufd_access *access, unsigned long iova, + unsigned long length, phys_addr_t *out_phys); + static inline void iommufd_access_destroy_internal(struct iommufd_ctx *ictx, diff --git a/drivers/iommu/iommufd/iova_bitmap.c b/drivers/iommu/iommufd/iova_bitmap.c index dac3e657d498d..12e1dd035871a 100644 --- a/drivers/iommu/iommufd/iova_bitmap.c +++ b/drivers/iommu/iommufd/iova_bitmap.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #define BITS_PER_PAGE (PAGE_SIZE * BITS_PER_BYTE) @@ -44,11 +45,16 @@ struct iova_bitmap_map { /* page offset of the first user page pinned */ unsigned long pgoff; - /* number of pages pinned */ + /* number of native pages or typed fragments pinned */ unsigned long npages; /* pinned pages representing the bitmap data */ struct page **pages; +#ifdef CONFIG_MM_SUBPAGE + /* Exact byte ranges, including offsets unrelated to the user VA. */ + struct user_page_fragment *fragments; + unsigned long pinned_bytes; +#endif }; /* @@ -63,7 +69,10 @@ struct iova_bitmap_map { * The bitmap object uses one base page to store all the pinned pages * pointers related to the bitmap. For sizeof(struct page*) == 8 it stores * 512 struct page pointers which, if the base page size is 4K, it means - * 2M of bitmap data is pinned at a time. If the iova_bitmap page size is + * 2M of bitmap data is pinned at a time. Alternative-granule processes use + * one native page of fragment descriptors instead, retaining each physical + * byte offset and bounding the window by the sum of fragment lengths. + * If the iova_bitmap page size is * also 4K then the range window to iterate is 64G. * * For example iterating on a total IOVA range of 4G..128G, it will walk @@ -116,6 +125,9 @@ struct iova_bitmap { /* length of the IOVA range for the whole bitmap */ size_t length; + + /* First lazy pin/write error during an iteration. */ + int error; }; /* @@ -192,6 +204,26 @@ static int iova_bitmap_get(struct iova_bitmap *bitmap) npages = min(npages + !!offset_in_page(addr), PAGE_SIZE / sizeof(struct page *)); +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + unsigned long bytes = bitmap->mapped_total_index - bitmap->mapped_base_index; + unsigned long i; + + ret = pin_user_fragments_remote(current->mm, (unsigned long)addr, + bytes, FOLL_WRITE, mapped->fragments, + PAGE_SIZE / sizeof(*mapped->fragments)); + if (ret <= 0) + return -EFAULT; + mapped->npages = ret; + mapped->pinned_bytes = 0; + for (i = 0; i < mapped->npages; i++) + mapped->pinned_bytes += mapped->fragments[i].length; + mapped->pgoff = 0; + mapped->iova = iova_bitmap_mapped_iova(bitmap); + mapped->length = iova_bitmap_mapped_length(bitmap); + return 0; + } +#endif ret = pin_user_pages_fast((unsigned long)addr, npages, FOLL_WRITE, mapped->pages); if (ret <= 0) @@ -221,7 +253,13 @@ static void iova_bitmap_put(struct iova_bitmap *bitmap) struct iova_bitmap_map *mapped = &bitmap->mapped; if (mapped->npages) { - unpin_user_pages(mapped->pages, mapped->npages); +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + release_user_fragments(mapped->fragments, mapped->npages, true); + mapped->pinned_bytes = 0; + } else +#endif + unpin_user_pages_dirty_lock(mapped->pages, mapped->npages, true); mapped->npages = 0; } } @@ -259,6 +297,16 @@ struct iova_bitmap *iova_bitmap_alloc(unsigned long iova, size_t length, bitmap->iova = iova; bitmap->length = length; mapped->iova = iova; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + mapped->fragments = (void *)__get_free_page(GFP_KERNEL); + if (!mapped->fragments) { + rc = -ENOMEM; + goto err; + } + return bitmap; + } +#endif mapped->pages = (struct page **)__get_free_page(GFP_KERNEL); if (!mapped->pages) { rc = -ENOMEM; @@ -291,6 +339,10 @@ void iova_bitmap_free(struct iova_bitmap *bitmap) mapped->pages = NULL; } +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) + free_page((unsigned long)mapped->fragments); +#endif kfree(bitmap); } EXPORT_SYMBOL_NS_GPL(iova_bitmap_free, "IOMMUFD"); @@ -303,7 +355,12 @@ static unsigned long iova_bitmap_mapped_remaining(struct iova_bitmap *bitmap) { unsigned long remaining, bytes; - bytes = (bitmap->mapped.npages << PAGE_SHIFT) - bitmap->mapped.pgoff; +#ifdef CONFIG_MM_SUBPAGE + if (bitmap->mapped.fragments) + bytes = bitmap->mapped.pinned_bytes; + else +#endif + bytes = (bitmap->mapped.npages << PAGE_SHIFT) - bitmap->mapped.pgoff; remaining = bitmap->mapped_total_index - bitmap->mapped_base_index; remaining = min_t(unsigned long, remaining, @@ -356,16 +413,21 @@ static int iova_bitmap_advance_to(struct iova_bitmap *bitmap, unsigned long iova) { unsigned long index; + int ret = -EINVAL; index = iova_bitmap_offset_to_index(bitmap, iova - bitmap->iova); if (index >= bitmap->mapped_total_index) - return -EINVAL; + goto out; bitmap->mapped_base_index = index; iova_bitmap_put(bitmap); /* Pin the next set of bitmap pages */ - return iova_bitmap_get(bitmap); + ret = iova_bitmap_get(bitmap); +out: + if (ret && !bitmap->error) + bitmap->error = ret; + return ret; } /** @@ -379,15 +441,58 @@ static int iova_bitmap_advance_to(struct iova_bitmap *bitmap, * mapped bitmap user pages into IOVA ranges to process. * * Return: 0 on success, and an error on failure either upon - * iteration or when the callback returns an error. + * iteration, a lazy bitmap write, or when the callback returns an error. */ int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque, iova_bitmap_fn_t fn) { - return fn(bitmap, bitmap->iova, bitmap->length, opaque); + int ret; + + bitmap->error = 0; + ret = fn(bitmap, bitmap->iova, bitmap->length, opaque); + return ret ?: bitmap->error; } EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, "IOMMUFD"); +#ifdef CONFIG_MM_SUBPAGE +static void iova_bitmap_set_fragments(struct iova_bitmap *bitmap, + unsigned long iova, size_t length) +{ + struct iova_bitmap_map *mapped = &bitmap->mapped; + unsigned long bit = (iova - bitmap->iova) >> mapped->pgshift; + unsigned long end = ((iova + length - 1 - bitmap->iova) >> mapped->pgshift) + 1; + + while (bit < end) { + unsigned long base, relative, i; + + iova = bitmap->iova + (bit << mapped->pgshift); + if (!iova_bitmap_mapped_range(mapped, iova, 1) && + iova_bitmap_advance_to(bitmap, iova)) + return; + base = bitmap->mapped_base_index * BITS_PER_BYTE; + relative = bit - base; + for (i = 0; i < mapped->npages && bit < end; i++) { + struct user_page_fragment *fragment = &mapped->fragments[i]; + unsigned long bits = fragment->length * BITS_PER_BYTE; + unsigned long count; + void *addr; + + if (relative >= bits) { + relative -= bits; + continue; + } + count = min(end - bit, bits - relative); + addr = kmap_local_folio(fragment->folio, fragment->offset & PAGE_MASK); + bitmap_set(addr, offset_in_page(fragment->offset) * BITS_PER_BYTE + + relative, count); + kunmap_local(addr); + bit += count; + relative = 0; + } + } +} +#endif + /** * iova_bitmap_set() - Records an IOVA range in bitmap * @bitmap: IOVA bitmap @@ -395,7 +500,8 @@ EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, "IOMMUFD"); * @length: IOVA range length * * Set the bits corresponding to the range [iova .. iova+length-1] in - * the user bitmap. + * the user bitmap. A lazy pin failure is retained for the enclosing + * iova_bitmap_for_each() to report; successful prefix writes are not undone. * */ void iova_bitmap_set(struct iova_bitmap *bitmap, @@ -404,6 +510,13 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, struct iova_bitmap_map *mapped = &bitmap->mapped; unsigned long cur_bit, last_bit, last_page_idx; +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + iova_bitmap_set_fragments(bitmap, iova, length); + return; + } +#endif + update_indexes: if (unlikely(!iova_bitmap_mapped_range(mapped, iova, length))) { /* diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c index 404f31d8f7291..0839878bfb67c 100644 --- a/drivers/iommu/iommufd/pages.c +++ b/drivers/iommu/iommufd/pages.c @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -207,7 +208,7 @@ static unsigned long iopt_area_index_to_iova(struct iopt_area *area, index -= iopt_area_index(area); if (index == 0) return iopt_area_iova(area); - return iopt_area_iova(area) - area->page_offset + index * PAGE_SIZE; + return iopt_area_iova(area) - area->page_offset + (index << area->page_shift); } static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area, @@ -219,7 +220,7 @@ static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area, if (index == iopt_area_last_index(area)) return iopt_area_last_iova(area); return iopt_area_iova(area) - area->page_offset + - (index - iopt_area_index(area) + 1) * PAGE_SIZE - 1; + ((index - iopt_area_index(area) + 1) << area->page_shift) - 1; } static void iommu_unmap_nofail(struct iommu_domain *domain, unsigned long iova, @@ -993,9 +994,11 @@ static int update_mm_locked_vm(struct iopt_pages *pages, unsigned long npages, } else if ((!user || (!user->upages && !user->ufolios)) && pages->source_mm != current->mm) { - if (!mmget_not_zero(pages->source_mm)) + if (mmget_not_zero(pages->source_mm)) + do_put = true; + else if (inc) return -EINVAL; - do_put = true; + /* mmgrab() keeps the lock and counters alive for final uncharge. */ } mmap_write_lock(pages->source_mm); @@ -1378,10 +1381,15 @@ static struct iopt_pages *iopt_alloc_pages(unsigned long start_byte, kref_init(&pages->kref); xa_init_flags(&pages->pinned_pfns, XA_FLAGS_ACCOUNT); +#ifdef CONFIG_MM_SUBPAGE + xa_init_flags(&pages->fragments, XA_FLAGS_ACCOUNT); + xa_init_flags(&pages->fragment_pages, XA_FLAGS_ACCOUNT); +#endif + pages->page_shift = PAGE_SHIFT; mutex_init(&pages->mutex); pages->source_mm = current->mm; mmgrab(pages->source_mm); - pages->npages = DIV_ROUND_UP(length + start_byte, PAGE_SIZE); + pages->npages = ((length - 1 + start_byte) >> PAGE_SHIFT) + 1; pages->access_itree = RB_ROOT_CACHED; pages->domains_itree = RB_ROOT_CACHED; pages->writable = writable; @@ -1400,8 +1408,10 @@ struct iopt_pages *iopt_alloc_user_pages(void __user *uptr, { struct iopt_pages *pages; unsigned long end; + unsigned int shift = min_t(unsigned int, mm_page_shift(current->mm), PAGE_SHIFT); + unsigned long granule = 1UL << shift; void __user *uptr_down = - (void __user *)ALIGN_DOWN((uintptr_t)uptr, PAGE_SIZE); + (void __user *)ALIGN_DOWN((uintptr_t)uptr, granule); if (check_add_overflow((unsigned long)uptr, length, &end)) return ERR_PTR(-EOVERFLOW); @@ -1409,6 +1419,8 @@ struct iopt_pages *iopt_alloc_user_pages(void __user *uptr, pages = iopt_alloc_pages(uptr - uptr_down, length, writable); if (IS_ERR(pages)) return pages; + pages->page_shift = shift; + pages->npages = ((length - 1 + (uptr - uptr_down)) >> shift) + 1; pages->uptr = uptr_down; pages->type = IOPT_ADDRESS_USER; return pages; @@ -1656,6 +1668,12 @@ void iopt_release_pages(struct kref *kref) WARN_ON(!RB_EMPTY_ROOT(&pages->domains_itree.rb_root)); WARN_ON(pages->npinned); WARN_ON(!xa_empty(&pages->pinned_pfns)); +#ifdef CONFIG_MM_SUBPAGE + WARN_ON(!xa_empty(&pages->fragments)); + WARN_ON(!xa_empty(&pages->fragment_pages)); + xa_destroy(&pages->fragments); + xa_destroy(&pages->fragment_pages); +#endif if (iopt_is_dmabuf(pages) && pages->dmabuf.attach) { struct dma_buf *dmabuf = pages->dmabuf.attach->dmabuf; @@ -1832,6 +1850,9 @@ void iopt_area_unmap_domain(struct iopt_area *area, struct iommu_domain *domain) void iopt_area_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, struct iommu_domain *domain) { + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_unfill_domain(area, pages, domain); + if (iopt_dmabuf_revoked(pages)) return; @@ -1855,6 +1876,9 @@ int iopt_area_fill_domain(struct iopt_area *area, struct iommu_domain *domain) lockdep_assert_held(&area->pages->mutex); + if (iopt_pages_use_fragments(area->pages)) + return iopt_fragments_fill_domain(area, area->pages, domain); + if (iopt_dmabuf_revoked(area->pages)) return 0; @@ -1916,6 +1940,9 @@ int iopt_area_fill_domains(struct iopt_area *area, struct iopt_pages *pages) if (xa_empty(&area->iopt->domains)) return 0; + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_fill_domains(area, pages); + mutex_lock(&pages->mutex); if (iopt_is_dmabuf(pages)) { rc = iopt_dmabuf_track_all_domains(area, pages); @@ -2365,6 +2392,10 @@ int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, if (iopt_is_dmabuf(pages)) return -EINVAL; + if (iopt_pages_use_fragments(pages) && change_mm && + !(flags & IOMMUFD_ACCESS_RW_KTHREAD)) + return iopt_fragments_rw(pages, start_byte, data, length, flags); + if (pages->type != IOPT_ADDRESS_USER) return iopt_pages_rw_slow(pages, start_index, last_index, start_byte % PAGE_SIZE, data, length, @@ -2385,11 +2416,14 @@ int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, * ignore any pinning inconsistencies, unlike a real DMA path. */ if (change_mm) { - if (!mmget_not_zero(pages->source_mm)) + if (!mmget_not_zero(pages->source_mm)) { + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_rw(pages, start_byte, data, length, flags); return iopt_pages_rw_slow(pages, start_index, last_index, start_byte % PAGE_SIZE, data, length, flags); + } kthread_use_mm(pages->source_mm); } @@ -2454,6 +2488,11 @@ int iopt_area_add_access(struct iopt_area *area, unsigned long start_index, if (iopt_is_dmabuf(pages)) return -EINVAL; + /* Preserve typed ownership and validate the native output geometry. */ + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_add_access(area, start_index, last_index, + out_pages, lock_area); + mutex_lock(&pages->mutex); access = iopt_pages_get_exact_access(pages, start_index, last_index); if (access) { @@ -2494,6 +2533,47 @@ int iopt_area_add_access(struct iopt_area *area, unsigned long start_index, return rc; } +/* Internal hardware queues consume a contiguous physical byte range. */ +int iopt_area_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_pages *pages = area->pages; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + unsigned long count = last - first + 1, i; + struct page **list; + phys_addr_t base; + int rc; + + if (iopt_is_dmabuf(pages)) + return -EINVAL; + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_add_phys_access(area, start_byte, length, out_phys); + + /* Retain the native page supplier and its accounting/batching rules. */ + list = kvcalloc(count, sizeof(*list), GFP_KERNEL_ACCOUNT); + if (!list) + return -ENOMEM; + rc = iopt_area_add_access(area, first, last, list, 0, true); + if (rc) + goto out_free; + base = PFN_PHYS(page_to_pfn(list[0])); + for (i = 1; i < count; i++) { + if (page_to_pfn(list[i]) != page_to_pfn(list[0]) + i) { + rc = -EFAULT; + goto out_unpin; + } + } + *out_phys = base + offset_in_page(start_byte); + goto out_free; + +out_unpin: + iopt_area_remove_access(area, first, last, true); +out_free: + kvfree(list); + return rc; +} + /** * iopt_area_remove_access() - Release an in-kernel access for PFNs * @area: The source of PFNs @@ -2526,7 +2606,10 @@ void iopt_area_remove_access(struct iopt_area *area, unsigned long start_index, goto out_unlock; interval_tree_remove(&access->node, &pages->access_itree); - iopt_pages_unfill_xarray(pages, start_index, last_index); + if (iopt_pages_use_fragments(pages)) + iopt_fragments_unfill_access(pages, start_index, last_index); + else + iopt_pages_unfill_xarray(pages, start_index, last_index); kfree(access); out_unlock: mutex_unlock(&pages->mutex); diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c index af07c642a5260..146b1fefa0475 100644 --- a/drivers/iommu/iommufd/selftest.c +++ b/drivers/iommu/iommufd/selftest.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -1201,6 +1202,28 @@ static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd, unsigned long pfn; long npages; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + phys_addr_t user_phys; + + npages = get_user_fragments_remote(current->mm, + (uintptr_t)uptr, page_size, 0, &fragment, 1); + if (npages != 1) { + rc = npages < 0 ? npages : -EFAULT; + goto out_put; + } + user_phys = PFN_PHYS(folio_pfn(fragment.folio)) + fragment.offset; + io_phys = iommu_iova_to_phys(&mock->domain, iova); + rc = fragment.length == page_size && io_phys == user_phys ? 0 : -EINVAL; + release_user_fragments(&fragment, 1, false); + if (rc) + goto out_put; + iova += page_size; + uptr += page_size; + continue; + } +#endif npages = get_user_pages_fast((uintptr_t)uptr & PAGE_MASK, 1, 0, pages); if (npages < 0) { @@ -1506,9 +1529,39 @@ static int iommufd_test_access_replace_ioas(struct iommufd_ucmd *ucmd, /* Check that the pages in a page array match the pages in the user VA */ static int iommufd_test_check_pages(void __user *uptr, struct page **pages, - size_t npages) -{ - for (; npages; npages--) { + size_t length) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + unsigned long offset = 0; + + while (offset < length) { + struct user_page_fragment fragment; + unsigned long bytes = min_t(unsigned long, length - offset, + mm_user_fragment_size(current->mm) - + (((unsigned long)uptr + offset) & + (mm_user_fragment_size(current->mm) - 1))); + phys_addr_t actual, expected; + long rc; + + rc = get_user_fragments_remote(current->mm, + (unsigned long)uptr + offset, bytes, 0, &fragment, 1); + if (rc <= 0) + return rc ?: -EFAULT; + actual = PFN_PHYS(folio_pfn(fragment.folio)) + fragment.offset; + expected = PFN_PHYS(page_to_pfn(pages[offset / PAGE_SIZE])) + + offset_in_page(offset); + bytes = fragment.length; + release_user_fragments(&fragment, 1, false); + if (actual != expected || !bytes || + bytes > PAGE_SIZE - offset_in_page(offset)) + return -EBADE; + offset += bytes; + } + return 0; + } +#endif + for (size_t npages = DIV_ROUND_UP(length, PAGE_SIZE); npages; npages--) { struct page *tmp_pages[1]; long rc; @@ -1586,7 +1639,7 @@ static int iommufd_test_access_pages(struct iommufd_ucmd *ucmd, if (uptr) { rc = iommufd_test_check_pages( uptr - (iova - ALIGN_DOWN(iova, PAGE_SIZE)), pages, - npages); + length + offset_in_page(iova)); if (rc) goto out_unaccess; } diff --git a/drivers/iommu/iommufd/viommu.c b/drivers/iommu/iommufd/viommu.c index fc13cf4737ea3..fa8296b15ef59 100644 --- a/drivers/iommu/iommufd/viommu.c +++ b/drivers/iommu/iommufd/viommu.c @@ -263,11 +263,7 @@ static void iommufd_hw_queue_destroy_access(struct iommufd_ctx *ictx, struct iommufd_access *access, u64 base_iova, size_t length) { - u64 aligned_iova = PAGE_ALIGN_DOWN(base_iova); - u64 offset = base_iova - aligned_iova; - - iommufd_access_unpin_pages(access, aligned_iova, - PAGE_ALIGN(length + offset)); + iommufd_access_unpin_pages(access, base_iova, length); iommufd_access_detach_internal(access); iommufd_access_destroy_internal(ictx, access); } @@ -300,66 +296,28 @@ static struct iommufd_access * iommufd_hw_queue_alloc_phys(struct iommu_hw_queue_alloc *cmd, struct iommufd_viommu *viommu, phys_addr_t *base_pa) { - u64 aligned_iova = PAGE_ALIGN_DOWN(cmd->nesting_parent_iova); - u64 offset = cmd->nesting_parent_iova - aligned_iova; struct iommufd_access *access; - struct page **pages; - size_t max_npages; - size_t length; - size_t i; int rc; - /* max_npages = DIV_ROUND_UP(offset + cmd->length, PAGE_SIZE) */ - if (check_add_overflow(offset, cmd->length, &length)) - return ERR_PTR(-ERANGE); - if (check_add_overflow(length, PAGE_SIZE - 1, &length)) + if (cmd->nesting_parent_iova > ULONG_MAX || cmd->length > ULONG_MAX) return ERR_PTR(-ERANGE); - max_npages = length / PAGE_SIZE; - /* length needs to be page aligned too */ - length = max_npages * PAGE_SIZE; - - /* - * Use kvcalloc() to avoid memory fragmentation for a large page array. - * Set __GFP_NOWARN to avoid syzkaller blowups - */ - pages = kvzalloc_objs(*pages, max_npages, GFP_KERNEL | __GFP_NOWARN); - if (!pages) - return ERR_PTR(-ENOMEM); - - access = iommufd_access_create_internal(viommu->ictx); - if (IS_ERR(access)) { - rc = PTR_ERR(access); - goto out_free; - } + access = iommufd_access_create_internal_phys(viommu->ictx); + if (IS_ERR(access)) + return access; rc = iommufd_access_attach_internal(access, viommu->hwpt->ioas); if (rc) goto out_destroy; - - rc = iommufd_access_pin_pages(access, aligned_iova, length, pages, 0); + rc = iommufd_access_pin_phys(access, cmd->nesting_parent_iova, + cmd->length, base_pa); if (rc) goto out_detach; - - /* Validate if the underlying physical pages are contiguous */ - for (i = 1; i < max_npages; i++) { - if (page_to_pfn(pages[i]) == page_to_pfn(pages[i - 1]) + 1) - continue; - rc = -EFAULT; - goto out_unpin; - } - - *base_pa = (page_to_pfn(pages[0]) << PAGE_SHIFT) + offset; - kvfree(pages); return access; -out_unpin: - iommufd_access_unpin_pages(access, aligned_iova, length); out_detach: iommufd_access_detach_internal(access); out_destroy: iommufd_access_destroy_internal(viommu->ictx, access); -out_free: - kvfree(pages); return ERR_PTR(rc); } diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index ceae52fd7586d..94f4708bf8d2b 100644 --- a/drivers/vfio/Kconfig +++ b/drivers/vfio/Kconfig @@ -54,6 +54,16 @@ config VFIO_IOMMU_TYPE1 tristate default n +config VFIO_TYPE1_ACCOUNT_KUNIT_TEST + bool "Test VFIO type1 pin accounting lifetime" if !KUNIT_ALL_TESTS + depends on KUNIT=y && VFIO_IOMMU_TYPE1=y + default KUNIT_ALL_TESTS + help + Exercise locked-memory refunds after source-mm exit and DMA owner + changes. Arm64 mixed-granule configurations also test accounting + conversions between user and native page sizes. + These tests use real mm lifetimes without a device or DMA mappings. + config VFIO_IOMMU_SPAPR_TCE tristate depends on SPAPR_TCE_IOMMU diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index e6d3a2311a99b..a3d5e93865706 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -430,10 +430,19 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) static int mm_lock_acct(struct task_struct *task, struct mm_struct *mm, bool lock_cap, long npage) { - int ret = mmap_write_lock_killable(mm); + int ret; - if (ret) - return ret; + if (!npage) + return 0; + + /* A released pin must be refunded even if the caller is interrupted. */ + if (npage < 0) { + mmap_write_lock(mm); + } else { + ret = mmap_write_lock_killable(mm); + if (ret) + return ret; + } ret = __account_locked_vm(mm, abs(npage), npage > 0, task, lock_cap); mmap_write_unlock(mm); @@ -443,20 +452,21 @@ static int mm_lock_acct(struct task_struct *task, struct mm_struct *mm, static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) { struct mm_struct *mm; + bool get_mm = async && npage > 0; int ret; if (!npage) return 0; mm = dma->mm; - if (async && !mmget_not_zero(mm)) + if (get_mm && !mmget_not_zero(mm)) return -ESRCH; /* process exited */ ret = mm_lock_acct(dma->task, mm, dma->lock_cap, npage); if (!ret) dma->locked_vm += npage; - if (async) + if (get_mm) mmput(mm); return ret; @@ -765,7 +775,8 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, */ if (acct_pages) { if (!dma->lock_cap && - mm->locked_vm + lock_acct + acct_pages > limit) { + mm_locked_vm_native_pages(mm) + + lock_acct + acct_pages > limit) { pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__, limit << PAGE_SHIFT); ret = -ENOMEM; @@ -1644,25 +1655,24 @@ static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu, return list_empty(iova); } -static int vfio_change_dma_owner(struct vfio_dma *dma) +static int vfio_change_dma_mm(struct vfio_dma *dma, struct task_struct *task, + struct mm_struct *mm, bool lock_cap) { - struct task_struct *task = current->group_leader; - struct mm_struct *mm = current->mm; long npage = dma->locked_vm; - bool lock_cap; int ret; if (mm == dma->mm) return 0; - lock_cap = capable(CAP_IPC_LOCK); ret = mm_lock_acct(task, mm, lock_cap, npage); if (ret) return ret; - if (mmget_not_zero(dma->mm)) { - mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -npage); - mmput(dma->mm); + /* dma retains mm_count after exit; its old charge still needs a refund. */ + ret = mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -npage); + if (ret) { + mm_lock_acct(task, mm, lock_cap, -npage); + return ret; } if (dma->task != task) { @@ -1676,6 +1686,12 @@ static int vfio_change_dma_owner(struct vfio_dma *dma) return 0; } +static int vfio_change_dma_owner(struct vfio_dma *dma) +{ + return vfio_change_dma_mm(dma, current->group_leader, current->mm, + capable(CAP_IPC_LOCK)); +} + static int vfio_dma_do_map(struct vfio_iommu *iommu, struct vfio_iommu_type1_dma_map *map) { @@ -3281,3 +3297,7 @@ MODULE_VERSION(DRIVER_VERSION); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR(DRIVER_AUTHOR); MODULE_DESCRIPTION(DRIVER_DESC); + +#ifdef CONFIG_VFIO_TYPE1_ACCOUNT_KUNIT_TEST +#include "vfio_iommu_type1_account_test.c" +#endif diff --git a/drivers/vfio/vfio_iommu_type1_account_test.c b/drivers/vfio/vfio_iommu_type1_account_test.c new file mode 100644 index 0000000000000..5447258ce49b8 --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_account_test.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by type1 so the tests exercise the actual accounting helpers. */ +#include +#include + +struct vfio_account_mm { + struct mm_struct *mm; + bool live; +}; + +static void vfio_account_mm_free(void *data) +{ + struct vfio_account_mm *ctx = data; + + if (ctx->live) + mmput(ctx->mm); + mmdrop(ctx->mm); +} + +static struct vfio_account_mm *vfio_account_mm_new(struct kunit *test, + unsigned int shift) +{ + struct vfio_account_mm *ctx; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return NULL; + ctx->mm = mm_alloc(); + if (!ctx->mm) + return NULL; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (shift != PAGE_SHIFT) { + pgd_t *old = ctx->mm->pgd, *pgd; + unsigned long flag = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + + ctx->mm->context.flags |= flag; + pgd = pgd_alloc(ctx->mm); + ctx->mm->context.flags &= ~flag; + if (!pgd) { + mmput(ctx->mm); + return NULL; + } + pgd_free(ctx->mm, old); + ctx->mm->pgd = pgd; + ctx->mm->context.flags |= flag; + } +#endif + ctx->live = true; + mmgrab(ctx->mm); + if (kunit_add_action_or_reset(test, vfio_account_mm_free, ctx)) + return NULL; + return ctx; +} + +static void vfio_account_dma_free(void *data) +{ + struct vfio_dma *dma = data; + + /* Cleanup must also work with the negative-control implementation. */ + if (dma->locked_vm) + mm_lock_acct(dma->task, dma->mm, true, -dma->locked_vm); + mmdrop(dma->mm); + put_task_struct(dma->task); +} + +static struct vfio_dma *vfio_account_dma_new(struct kunit *test, + struct vfio_account_mm *ctx) +{ + struct vfio_dma *dma = kunit_kzalloc(test, sizeof(*dma), GFP_KERNEL); + + if (!dma) + return NULL; + dma->mm = ctx->mm; + mmgrab(dma->mm); + dma->task = get_task_struct(current->group_leader); + dma->lock_cap = true; + if (kunit_add_action_or_reset(test, vfio_account_dma_free, dma)) + return NULL; + return dma; +} + +static void vfio_account_exit_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + 12, 14, 16, +#endif + }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_dma *dma; + + if (i && shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_EQ(test, mm_page_shift(ctx->mm), shifts[i]); + dma = vfio_account_dma_new(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, 2, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(ctx->mm), 2UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 2UL * PAGE_SIZE); + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, 1, true), -ESRCH); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, -1, true), 0); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(ctx->mm), 1UL); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, -1, true), 0); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + kunit_info(test, "VFIO %uK user/%luK native dead-mm refunds checked", + 1U << (shifts[i] - 10), PAGE_SIZE >> 10); + } +} + +static void vfio_account_owner_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + 12, 14, 16, +#endif + }; + + for (unsigned int reverse = 0; reverse < 2; reverse++) { + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *old, *next; + struct vfio_dma *dma; + + if ((i || reverse) && shifts[i] == PAGE_SHIFT) + continue; + old = vfio_account_mm_new(test, reverse ? shifts[i] : PAGE_SHIFT); + next = vfio_account_mm_new(test, reverse ? PAGE_SHIFT : shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + dma = vfio_account_dma_new(test, old); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, 3, true), 0); + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&old->mm->mm_users), 0); + if (task_rlimit(dma->task, RLIMIT_MEMLOCK) != RLIM_INFINITY) { + unsigned long budget = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> + MM_ACCOUNT_SHIFT; + + /* Model a recipient whose existing pins consume its budget. */ + next->mm->locked_vm = budget; + KUNIT_EXPECT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, false), + -ENOMEM); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, old->mm); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 3UL); + KUNIT_EXPECT_EQ(test, next->mm->locked_vm, budget); + next->mm->locked_vm = 0; + } + KUNIT_ASSERT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, next->mm); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 3UL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 3UL * PAGE_SIZE); + /* Same-mm ownership is idempotent; it must not double the charge. */ + KUNIT_ASSERT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 3UL); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, -3, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 0UL); + kunit_info(test, "VFIO dead %uK mm -> %uK mm ownership refund checked", + 1U << (mm_page_shift(old->mm) - 10), + 1U << (mm_page_shift(next->mm) - 10)); + } + } +} + +static struct kunit_case vfio_account_cases[] = { + KUNIT_CASE(vfio_account_exit_test), + KUNIT_CASE(vfio_account_owner_test), + {} +}; + +static struct kunit_suite vfio_account_suite = { + .name = "vfio-type1-account", + .test_cases = vfio_account_cases, +}; + +kunit_test_suite(vfio_account_suite); diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index e4c44f2ea413a..4c3577ce7b85b 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -2436,6 +2437,26 @@ static int set_bit_to_user(int nr, void __user *addr) int bit = nr + (log % PAGE_SIZE) * 8; int r; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + unsigned long word = log & ~(sizeof(unsigned long) - 1); + + /* Pin the complete atomic word, which stays inside one leaf. + * A relocated leaf's physical offset need not match its VA. + */ + r = pin_user_fragments_remote(current->mm, word, + sizeof(unsigned long), FOLL_WRITE, &fragment, 1); + if (r != 1) + return r < 0 ? r : -EFAULT; + base = kmap_local_folio(fragment.folio, fragment.offset); + set_bit(nr + (log - word) * 8, base); + kunmap_local(base); + release_user_fragments(&fragment, 1, true); + return 0; + } +#endif + r = pin_user_pages_fast(log, 1, FOLL_WRITE, &page); if (r < 0) return r; diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 16a56b6b3f6ca..826235fd977c2 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -79,11 +79,11 @@ static int elf_core_dump(struct coredump_params *cprm); #define elf_core_dump NULL #endif -#if ELF_EXEC_PAGESIZE > PAGE_SIZE -#define ELF_MIN_ALIGN ELF_EXEC_PAGESIZE -#else -#define ELF_MIN_ALIGN PAGE_SIZE +#ifndef ELF_EXEC_PAGESIZE_MM +#define ELF_EXEC_PAGESIZE_MM(mm) ELF_EXEC_PAGESIZE #endif +#define ELF_MIN_ALIGN max_t(unsigned long, mm_page_size(current->mm), \ + ELF_EXEC_PAGESIZE_MM(current->mm)) #ifndef ELF_CORE_EFLAGS #define ELF_CORE_EFLAGS 0 @@ -93,12 +93,20 @@ static int elf_core_dump(struct coredump_params *cprm); #define ELF_PAGEOFFSET(_v) ((_v) & (ELF_MIN_ALIGN-1)) #define ELF_PAGEALIGN(_v) (((_v) + ELF_MIN_ALIGN - 1) & ~(ELF_MIN_ALIGN - 1)) +#ifdef CONFIG_COREDUMP +static unsigned long elf_min_coredump_size(void) +{ + return ELF_EXEC_PAGESIZE_MM(current->mm); +} +#endif + static struct linux_binfmt elf_format = { .module = THIS_MODULE, .load_binary = load_elf_binary, #ifdef CONFIG_COREDUMP .core_dump = elf_core_dump, .min_coredump = ELF_EXEC_PAGESIZE, + .min_coredump_size = elf_min_coredump_size, #endif }; @@ -248,7 +256,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec, ARCH_DLINFO; #endif NEW_AUX_ENT(AT_HWCAP, ELF_HWCAP); - NEW_AUX_ENT(AT_PAGESZ, ELF_EXEC_PAGESIZE); + NEW_AUX_ENT(AT_PAGESZ, ELF_EXEC_PAGESIZE_MM(mm)); NEW_AUX_ENT(AT_CLKTCK, CLOCKS_PER_SEC); NEW_AUX_ENT(AT_PHDR, phdr_addr); NEW_AUX_ENT(AT_PHENT, sizeof(struct elf_phdr)); @@ -622,6 +630,13 @@ static inline int arch_check_elf(struct elfhdr *ehdr, bool has_interp, #endif /* !CONFIG_ARCH_BINFMT_ELF_STATE */ +#ifndef arch_bprm_prepare_mm +static inline int arch_bprm_prepare_mm(struct linux_binprm *bprm, bool compat) +{ + return 0; +} +#endif + static inline int make_prot(u32 p_flags, struct arch_elf_state *arch_state, bool has_interp, bool is_interp) { @@ -1006,6 +1021,11 @@ static int load_elf_binary(struct linux_binprm *bprm) if (retval) goto out_free_dentry; + retval = arch_bprm_prepare_mm(bprm, + elf_ex->e_ident[EI_CLASS] == ELFCLASS32); + if (retval) + goto out_free_dentry; + /* Flush all traces of the currently running executable */ retval = begin_new_exec(bprm); if (retval) @@ -1335,7 +1355,7 @@ static int load_elf_binary(struct linux_binprm *bprm) * leave a gap between .bss and brk. */ if (!brk_moved) - mm->brk = mm->start_brk = mm->brk + PAGE_SIZE; + mm->brk = mm->start_brk = mm->brk + mm_page_size(mm); mm->brk = mm->start_brk = arch_randomize_brk(mm); brk_moved = true; @@ -1351,10 +1371,10 @@ static int load_elf_binary(struct linux_binprm *bprm) and some applications "depend" upon this behavior. Since we do not have the power to recompile these, we emulate the SVr4 behavior. Sigh. */ - error = vm_mmap(NULL, 0, PAGE_SIZE, PROT_READ | PROT_EXEC, + error = vm_mmap(NULL, 0, mm_page_size(mm), PROT_READ | PROT_EXEC, MAP_FIXED | MAP_PRIVATE, 0); - retval = do_mseal(0, PAGE_SIZE, 0); + retval = do_mseal(0, mm_page_size(mm), 0); if (retval) pr_warn_ratelimited("pid=%d, couldn't seal address 0, ret=%d.\n", task_pid_nr(current), retval); @@ -1658,7 +1678,7 @@ static int fill_files_note(struct memelfnote *note, struct coredump_params *cprm /* Now we know exact count of files, can store it */ data[0] = count; - data[1] = PAGE_SIZE; + data[1] = mm_page_size(current->mm); /* * Count usually is less than mm->map_count, * we need to move filenames down. @@ -2043,7 +2063,7 @@ static int elf_core_dump(struct coredump_params *cprm) offset += sz; } - dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE); + dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE_MM(current->mm)); offset += cprm->vma_data_size; offset += elf_core_extra_data_size(cprm); @@ -2083,7 +2103,7 @@ static int elf_core_dump(struct coredump_params *cprm) phdr.p_flags |= PF_W; if (meta->flags & VM_EXEC) phdr.p_flags |= PF_X; - phdr.p_align = ELF_EXEC_PAGESIZE; + phdr.p_align = ELF_EXEC_PAGESIZE_MM(current->mm); if (!dump_emit(cprm, &phdr, sizeof(phdr))) goto end_coredump; diff --git a/fs/coredump.c b/fs/coredump.c index bb6fdb1f458e9..6ab5f2e076698 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -884,7 +885,8 @@ static bool coredump_file(struct core_name *cn, struct coredump_params *cprm, struct file *file __free(fput) = NULL; int open_flags = O_CREAT | O_WRONLY | O_NOFOLLOW | O_LARGEFILE | O_EXCL; - if (cprm->limit < binfmt->min_coredump) + if (cprm->limit < (binfmt->min_coredump_size ? + binfmt->min_coredump_size() : binfmt->min_coredump)) return false; if (coredump_force_suid_safe(cprm) && cn->corename[0] != '/') { @@ -1278,7 +1280,8 @@ void dump_skip(struct coredump_params *cprm, size_t nr) EXPORT_SYMBOL(dump_skip); #ifdef CONFIG_ELF_CORE -static int dump_emit_page(struct coredump_params *cprm, struct page *page) +static int dump_emit_page(struct coredump_params *cprm, struct page *page, + unsigned int offset, unsigned int size) { struct bio_vec bvec; struct iov_iter iter; @@ -1294,19 +1297,19 @@ static int dump_emit_page(struct coredump_params *cprm, struct page *page) return 0; cprm->to_skip = 0; } - if (cprm->written + PAGE_SIZE > cprm->limit) + if (cprm->written + size > cprm->limit) return 0; if (dump_interrupted()) return 0; pos = file->f_pos; - bvec_set_page(&bvec, page, PAGE_SIZE, 0); - iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, PAGE_SIZE); + bvec_set_page(&bvec, page, size, offset); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size); n = __kernel_write_iter(cprm->file, &iter, &pos); - if (n != PAGE_SIZE) + if (n != size) return 0; file->f_pos = pos; - cprm->written += PAGE_SIZE; - cprm->pos += PAGE_SIZE; + cprm->written += size; + cprm->pos += size; return 1; } @@ -1321,10 +1324,11 @@ static int dump_emit_page(struct coredump_params *cprm, struct page *page) #define dump_page_alloc() alloc_page(GFP_KERNEL) #define dump_page_free(x) __free_page(x) -static struct page *dump_page_copy(struct page *src, struct page *dst) +static struct page *dump_page_copy(struct page *src, struct page *dst, + unsigned int offset, unsigned int size) { void *buf = kmap_local_page(src); - size_t left = copy_mc_to_kernel(page_address(dst), buf, PAGE_SIZE); + size_t left = copy_mc_to_kernel(page_address(dst) + offset, buf + offset, size); kunmap_local(buf); return left ? NULL : dst; } @@ -1334,12 +1338,51 @@ static struct page *dump_page_copy(struct page *src, struct page *dst) /* We just want to return non-NULL; it's never used. */ #define dump_page_alloc() ERR_PTR(-EINVAL) #define dump_page_free(x) ((void)(x)) -static inline struct page *dump_page_copy(struct page *src, struct page *dst) +static inline struct page *dump_page_copy(struct page *src, struct page *dst, + unsigned int offset, unsigned int size) { return src; } #endif +#ifdef CONFIG_MM_SUBPAGE +static int dump_subpage_range(struct coredump_params *cprm, unsigned long start, + unsigned long len) +{ + unsigned long size = mm_user_fragment_size(current->mm), addr; + struct page *scratch = dump_page_alloc(); + int ret = 0; + + if (!scratch) + return 0; + for (addr = start; addr < start + len; addr += size) { + struct user_page_fragment fragment; + + if (get_dump_fragment(addr, &fragment) == 1) { + struct page *page = folio_page(fragment.folio, + fragment.offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(fragment.offset); + int emitted; + + emitted = dump_emit_page(cprm, + dump_page_copy(page, scratch, offset, size), offset, size); + release_user_fragments(&fragment, 1, false); + if (!emitted) + goto out; + } else { + dump_skip(cprm, size); + } + if (dump_interrupted()) + goto out; + cond_resched(); + } + ret = 1; +out: + dump_page_free(scratch); + return ret; +} +#endif + int dump_user_range(struct coredump_params *cprm, unsigned long start, unsigned long len) { @@ -1347,6 +1390,10 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start, struct page *dump_page; int locked, ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return dump_subpage_range(cprm, start, len); +#endif dump_page = dump_page_alloc(); if (!dump_page) return 0; @@ -1375,7 +1422,8 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start, mmap_read_unlock(current->mm); locked = 0; } - int stop = !dump_emit_page(cprm, dump_page_copy(page, dump_page)); + int stop = !dump_emit_page(cprm, + dump_page_copy(page, dump_page, 0, PAGE_SIZE), 0, PAGE_SIZE); put_page(page); if (stop) goto out; @@ -1652,9 +1700,10 @@ static unsigned long vma_dump_size(struct vm_area_struct *vma, * dump the first page to aid in determining what was mapped here. */ if (FILTER(ELF_HEADERS) && - vma->vm_pgoff == 0 && (vma->vm_flags & VM_READ)) { + vma->vm_pgoff == 0 && !vma_subpage_offset(vma) && + (vma->vm_flags & VM_READ)) { if ((READ_ONCE(file_inode(vma->vm_file)->i_mode) & 0111) != 0) - return PAGE_SIZE; + return min(mm_page_size(vma->vm_mm), vma->vm_end - vma->vm_start); /* * ELF libraries aren't always executable. @@ -1754,7 +1803,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) m->end = vma->vm_end; m->flags = vma->vm_flags; m->dump_size = vma_dump_size(vma, cprm->mm_flags); - m->pgoff = vma->vm_pgoff; + m->pgoff = core_vma_pgoff(vma); m->file = vma->vm_file; if (m->file) get_file(m->file); @@ -1773,7 +1822,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) memcmp(elfmag, ELFMAG, SELFMAG) != 0) { m->dump_size = 0; } else { - m->dump_size = PAGE_SIZE; + m->dump_size = min(mm_page_size(mm), m->end - m->start); } } diff --git a/fs/exec.c b/fs/exec.c index 2eacc1441c59f..d373c8735b048 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -122,6 +123,48 @@ bool path_noexec(const struct path *path) (path->mnt->mnt_sb->s_iflags & SB_I_NOEXEC); } +/* A temporary argument mapping owns either a page or an exact 4K slot. */ +struct arg_page { + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *subpage; + unsigned int offset; +#endif +}; + +static unsigned long bprm_page_size(const struct linux_binprm *bprm) +{ +#ifdef CONFIG_MMU + return mm_page_size(bprm->mm); +#else + return PAGE_SIZE; +#endif +} + +static unsigned long bprm_page_mask(const struct linux_binprm *bprm) +{ + return ~(bprm_page_size(bprm) - 1); +} + +static unsigned long bprm_max_arg_strlen(const struct linux_binprm *bprm) +{ + return (MAX_ARG_STRLEN / PAGE_SIZE) * bprm_page_size(bprm); +} + +static unsigned int arg_page_offset(struct arg_page page) +{ +#ifdef CONFIG_MM_SUBPAGE + return page.offset; +#else + return 0; +#endif +} + +static void *map_arg_page(struct arg_page page) +{ + return kmap_local_page(page.page) + arg_page_offset(page); +} + #ifdef CONFIG_MMU /* * The nascent bprm->mm is not visible until exec_mmap() but it can @@ -141,10 +184,10 @@ static void acct_arg_size(struct linux_binprm *bprm, unsigned long pages) add_mm_counter(mm, MM_ANONPAGES, diff); } -static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, +static struct arg_page get_arg_page(struct linux_binprm *bprm, unsigned long pos, int write) { - struct page *page; + struct arg_page page = {}; struct vm_area_struct *vma = bprm->vma; struct mm_struct *mm = bprm->mm; int ret; @@ -155,28 +198,47 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, * ahead of time. */ if (!mmap_read_lock_maybe_expand(mm, vma, pos, write)) - return NULL; + return page; /* * We are doing an exec(). 'current' is the process * doing the exec and 'mm' is the new process's mm. */ - ret = get_user_pages_remote(mm, pos, 1, - write ? FOLL_WRITE : 0, - &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + struct vm_area_struct *area; + + ret = get_user_fragment_vma_remote(mm, pos, 1, write ? FOLL_WRITE : 0, + &fragment, &area); + if (ret > 0) { + page.page = folio_page(fragment.folio, fragment.offset >> PAGE_SHIFT); + page.subpage = fragment.subpage; + page.offset = offset_in_page(fragment.offset) & mm_page_mask(mm); + } + } else +#endif + ret = get_user_pages_remote(mm, pos, 1, + write ? FOLL_WRITE : 0, &page.page, NULL); mmap_read_unlock(mm); if (ret <= 0) - return NULL; + return page; if (write) - acct_arg_size(bprm, vma_pages(vma)); + acct_arg_size(bprm, mm_pages_for_reader(mm, current->mm, vma_user_pages(vma))); return page; } -static void put_arg_page(struct page *page) +static void put_arg_page(struct arg_page page) { - put_page(page); +#ifdef CONFIG_MM_SUBPAGE + if (page.subpage) { + mm_subpage_put(page.subpage); + return; + } +#endif + put_page(page.page); } static void free_arg_pages(struct linux_binprm *bprm) @@ -184,14 +246,14 @@ static void free_arg_pages(struct linux_binprm *bprm) } static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos, - struct page *page) + struct arg_page page) { - flush_cache_page(bprm->vma, pos, page_to_pfn(page)); + flush_cache_page(bprm->vma, pos, page_to_pfn(page.page)); } static bool valid_arg_len(struct linux_binprm *bprm, long len) { - return len <= MAX_ARG_STRLEN; + return len <= bprm_max_arg_strlen(bprm); } #else @@ -200,7 +262,7 @@ static inline void acct_arg_size(struct linux_binprm *bprm, unsigned long pages) { } -static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, +static struct arg_page get_arg_page(struct linux_binprm *bprm, unsigned long pos, int write) { struct page *page; @@ -209,14 +271,14 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, if (!page && write) { page = alloc_page(GFP_HIGHUSER|__GFP_ZERO); if (!page) - return NULL; + return (struct arg_page) {}; bprm->page[pos / PAGE_SIZE] = page; } - return page; + return (struct arg_page) { .page = page }; } -static void put_arg_page(struct page *page) +static void put_arg_page(struct arg_page page) { } @@ -237,7 +299,7 @@ static void free_arg_pages(struct linux_binprm *bprm) } static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos, - struct page *page) + struct arg_page page) { } @@ -259,7 +321,7 @@ static int bprm_mm_init(struct linux_binprm *bprm) int err; struct mm_struct *mm = NULL; - bprm->mm = mm = mm_alloc(); + bprm->mm = mm = mm_alloc_exec(); err = -ENOMEM; if (!mm) goto err; @@ -288,6 +350,102 @@ static int bprm_mm_init(struct linux_binprm *bprm) return err; } +#ifdef CONFIG_MM_SUBPAGE +/* + * Final ELF selection may change the granule after argv, envp and script + * arguments have been staged. Table geometry is immutable: build a second mm + * and transfer bytes at the same temporary stack addresses before committing + * exec. Until success, the original bprm and its argument charge remain intact. + */ +int bprm_set_page_shift(struct linux_binprm *bprm, unsigned int page_shift) +{ + struct linux_binprm replacement = {}; + struct mm_struct *old_mm; + unsigned long pos, end, new_top, string_len = 0; + unsigned long old_size = mm_page_size(bprm->mm), new_size; + int ret; + + if (mm_page_shift(bprm->mm) == page_shift) + return 0; + if (WARN_ON_ONCE(bprm->point_of_no_return)) + return -EINVAL; + + replacement.mm = mm_alloc_exec_page_shift(page_shift); + if (!replacement.mm) + return -ENOMEM; + ret = -EINVAL; + if (mm_page_shift(replacement.mm) != page_shift) + goto out; + new_size = mm_page_size(replacement.mm); + ret = create_init_stack_vma(replacement.mm, &replacement.vma, &new_top); + if (ret) + goto out; + end = bprm->vma->vm_end - sizeof(void *); + ret = -EINVAL; + if (new_top != end || bprm->p > end) + goto out; + + for (pos = bprm->p; pos < end;) { + struct arg_page src, dst; + unsigned long src_off = pos & (old_size - 1); + unsigned long dst_off = pos & (new_size - 1); + unsigned long len = min3(end - pos, old_size - src_off, + new_size - dst_off); + const char *from; + char *to; + + ret = -ERESTARTNOHAND; + if (fatal_signal_pending(current)) + goto out; + src = get_arg_page(bprm, pos, 0); + ret = -EFAULT; + if (!src.page) + goto out; + dst = get_arg_page(&replacement, pos, 1); + if (!dst.page) { + put_arg_page(src); + ret = -ENOMEM; + goto out; + } + from = map_arg_page(src); + to = map_arg_page(dst); + /* Recheck per-string limits if the selected granule shrank. */ + ret = 0; + for (unsigned long i = 0; i < len; i++) { + if (++string_len > bprm_max_arg_strlen(&replacement)) { + ret = -E2BIG; + break; + } + if (!from[src_off + i]) + string_len = 0; + } + if (!ret) + memcpy(to + dst_off, from + src_off, len); + kunmap_local(to); + kunmap_local(from); + flush_arg_page(&replacement, pos, dst); + put_arg_page(dst); + put_arg_page(src); + if (ret) + goto out; + pos += len; + cond_resched(); + } + + old_mm = bprm->mm; + acct_arg_size(bprm, 0); + bprm->mm = replacement.mm; + bprm->vma = replacement.vma; + bprm->vma_pages = replacement.vma_pages; + mmput(old_mm); + return 0; +out: + acct_arg_size(&replacement, 0); + mmput(replacement.mm); + return ret; +} +#endif + struct user_arg_ptr { #ifdef CONFIG_COMPAT bool is_compat; @@ -449,7 +607,7 @@ static int bprm_stack_limits(struct linux_binprm *bprm) static int copy_strings(int argc, struct user_arg_ptr argv, struct linux_binprm *bprm) { - struct page *kmapped_page = NULL; + struct arg_page kmapped_page = {}; char *kaddr = NULL; unsigned long kpos = 0; int ret; @@ -464,7 +622,7 @@ static int copy_strings(int argc, struct user_arg_ptr argv, if (IS_ERR(str)) goto out; - len = strnlen_user(str, MAX_ARG_STRLEN); + len = strnlen_user(str, bprm_max_arg_strlen(bprm)); if (!len) goto out; @@ -488,9 +646,9 @@ static int copy_strings(int argc, struct user_arg_ptr argv, } cond_resched(); - offset = pos % PAGE_SIZE; + offset = pos & ~bprm_page_mask(bprm); if (offset == 0) - offset = PAGE_SIZE; + offset = bprm_page_size(bprm); bytes_to_copy = offset; if (bytes_to_copy > len) @@ -501,23 +659,23 @@ static int copy_strings(int argc, struct user_arg_ptr argv, str -= bytes_to_copy; len -= bytes_to_copy; - if (!kmapped_page || kpos != (pos & PAGE_MASK)) { - struct page *page; + if (!kmapped_page.page || kpos != (pos & bprm_page_mask(bprm))) { + struct arg_page page; page = get_arg_page(bprm, pos, 1); - if (!page) { + if (!page.page) { ret = -E2BIG; goto out; } - if (kmapped_page) { - flush_dcache_page(kmapped_page); + if (kmapped_page.page) { + flush_dcache_page(kmapped_page.page); kunmap_local(kaddr); put_arg_page(kmapped_page); } kmapped_page = page; - kaddr = kmap_local_page(kmapped_page); - kpos = pos & PAGE_MASK; + kaddr = map_arg_page(kmapped_page); + kpos = pos & bprm_page_mask(bprm); flush_arg_page(bprm, kpos, kmapped_page); } if (copy_from_user(kaddr+offset, str, bytes_to_copy)) { @@ -528,8 +686,8 @@ static int copy_strings(int argc, struct user_arg_ptr argv, } ret = 0; out: - if (kmapped_page) { - flush_dcache_page(kmapped_page); + if (kmapped_page.page) { + flush_dcache_page(kmapped_page.page); kunmap_local(kaddr); put_arg_page(kmapped_page); } @@ -541,7 +699,7 @@ static int copy_strings(int argc, struct user_arg_ptr argv, */ int copy_string_kernel(const char *arg, struct linux_binprm *bprm) { - int len = strnlen(arg, MAX_ARG_STRLEN) + 1 /* terminating NUL */; + int len = strnlen(arg, bprm_max_arg_strlen(bprm)) + 1 /* terminating NUL */; unsigned long pos = bprm->p; if (len == 0) @@ -557,18 +715,19 @@ int copy_string_kernel(const char *arg, struct linux_binprm *bprm) while (len > 0) { unsigned int bytes_to_copy = min(len, - min_not_zero(offset_in_page(pos), PAGE_SIZE)); - struct page *page; + min_not_zero(pos & ~bprm_page_mask(bprm), bprm_page_size(bprm))); + struct arg_page page; pos -= bytes_to_copy; arg -= bytes_to_copy; len -= bytes_to_copy; page = get_arg_page(bprm, pos, 1); - if (!page) + if (!page.page) return -E2BIG; - flush_arg_page(bprm, pos & PAGE_MASK, page); - memcpy_to_page(page, offset_in_page(pos), arg, bytes_to_copy); + flush_arg_page(bprm, pos & bprm_page_mask(bprm), page); + memcpy_to_page(page.page, arg_page_offset(page) + + (pos & ~bprm_page_mask(bprm)), arg, bytes_to_copy); put_arg_page(page); } @@ -621,20 +780,20 @@ int setup_arg_pages(struct linux_binprm *bprm, /* Add space for stack randomization. */ if (current->flags & PF_RANDOMIZE) - stack_base += (STACK_RND_MASK << PAGE_SHIFT); + stack_base += (STACK_RND_MASK << mm_page_shift(mm)); /* Make sure we didn't let the argument array grow too large. */ if (vma->vm_end - vma->vm_start > stack_base) return -ENOMEM; - stack_base = PAGE_ALIGN(stack_top - stack_base); + stack_base = mm_page_align(mm, stack_top - stack_base); stack_shift = vma->vm_start - stack_base; mm->arg_start = bprm->p - stack_shift; bprm->p = vma->vm_end - stack_shift; #else stack_top = arch_align_stack(stack_top); - stack_top = PAGE_ALIGN(stack_top); + stack_top = mm_page_align(mm, stack_top); if (unlikely(stack_top < mmap_min_addr) || unlikely(vma->vm_end - vma->vm_start >= stack_top - mmap_min_addr)) @@ -702,7 +861,7 @@ int setup_arg_pages(struct linux_binprm *bprm, * Align this down to a page boundary as expand_stack * will align it up. */ - rlim_stack = bprm->rlim_stack.rlim_cur & PAGE_MASK; + rlim_stack = bprm->rlim_stack.rlim_cur & mm_page_mask(mm); stack_expand = min(rlim_stack, stack_size + stack_expand); @@ -1616,25 +1775,25 @@ int remove_arg_zero(struct linux_binprm *bprm) { unsigned long offset; char *kaddr; - struct page *page; + struct arg_page page; if (!bprm->argc) return 0; do { - offset = bprm->p & ~PAGE_MASK; + offset = bprm->p & ~bprm_page_mask(bprm); page = get_arg_page(bprm, bprm->p, 0); - if (!page) + if (!page.page) return -EFAULT; - kaddr = kmap_local_page(page); + kaddr = map_arg_page(page); - for (; offset < PAGE_SIZE && kaddr[offset]; + for (; offset < bprm_page_size(bprm) && kaddr[offset]; offset++, bprm->p++) ; kunmap_local(kaddr); put_arg_page(page); - } while (offset == PAGE_SIZE); + } while (offset == bprm_page_size(bprm)); bprm->p++; bprm->argc--; diff --git a/fs/proc/array.c b/fs/proc/array.c index 479ea8cb4ef43..5b45506942991 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -610,7 +610,8 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, seq_put_decimal_ull(m, " ", 0); seq_put_decimal_ull(m, " ", start_time); seq_put_decimal_ull(m, " ", vsize); - seq_put_decimal_ull(m, " ", mm ? get_mm_rss(mm) : 0); + seq_put_decimal_ull(m, " ", mm ? + mm_pages_for_reader(mm, current->mm, get_mm_rss(mm)) : 0); seq_put_decimal_ull(m, " ", rsslim); seq_put_decimal_ull(m, " ", mm ? (permitted ? mm->start_code : 1) : 0); seq_put_decimal_ull(m, " ", mm ? (permitted ? mm->end_code : 1) : 0); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 436e1c1d11f41..8938ae6758100 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include #include #include @@ -33,7 +34,7 @@ #define SENTINEL_VMA_GATE -2 #define SEQ_PUT_DEC(str, val) \ - seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8) + seq_put_decimal_ull_width(m, str, mm_pages_to_kb(mm, val), 8) void task_mem(struct seq_file *m, struct mm_struct *mm) { unsigned long text, lib, swap, anon, file, shmem; @@ -58,15 +59,18 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) hiwater_rss = mm->hiwater_rss; /* split executable areas between text and lib */ - text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK); - text = min(text, mm->exec_vm << PAGE_SHIFT); - lib = (mm->exec_vm << PAGE_SHIFT) - text; + text = mm_page_align(mm, mm->end_code) - + (mm->start_code & mm_page_mask(mm)); + text = min(text, mm_pages_to_bytes(mm, mm->exec_vm)); + lib = (mm_pages_to_bytes(mm, mm->exec_vm)) - text; swap = get_mm_counter_sum(mm, MM_SWAPENTS); SEQ_PUT_DEC("VmPeak:\t", hiwater_vm); SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm); - SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm); - SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm)); + seq_put_decimal_ull_width(m, " kB\nVmLck:\t", + mm_locked_vm_bytes(mm) >> 10, 8); + seq_put_decimal_ull_width(m, " kB\nVmPin:\t", + atomic64_read(&mm->pinned_vm) << (PAGE_SHIFT - 10), 8); SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss); SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss); SEQ_PUT_DEC(" kB\nRssAnon:\t", anon); @@ -88,20 +92,25 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) unsigned long task_vsize(struct mm_struct *mm) { - return PAGE_SIZE * mm->total_vm; + return mm_pages_to_bytes(mm, mm->total_vm); } unsigned long task_statm(struct mm_struct *mm, unsigned long *shared, unsigned long *text, unsigned long *data, unsigned long *resident) { - *shared = get_mm_counter_sum(mm, MM_FILEPAGES) + + struct mm_struct *reader = current->mm; + unsigned long file = get_mm_counter_sum(mm, MM_FILEPAGES) + get_mm_counter_sum(mm, MM_SHMEMPAGES); - *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) - >> PAGE_SHIFT; - *data = mm->data_vm + mm->stack_vm; - *resident = *shared + get_mm_counter_sum(mm, MM_ANONPAGES); - return mm->total_vm; + unsigned long code = (mm_page_align(mm, mm->end_code) - + (mm->start_code & mm_page_mask(mm))) >> mm_page_shift(mm); + + *shared = mm_pages_for_reader(mm, reader, file); + *text = mm_pages_for_reader(mm, reader, code); + *data = mm_pages_for_reader(mm, reader, mm->data_vm + mm->stack_vm); + *resident = mm_pages_for_reader(mm, reader, + file + get_mm_counter_sum(mm, MM_ANONPAGES)); + return mm_pages_for_reader(mm, reader, mm->total_vm); } #ifdef CONFIG_NUMA @@ -475,7 +484,8 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma) dev = inode->i_sb->s_dev; ino = inode->i_ino; - pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT; + pgoff = (((loff_t)vma->vm_pgoff) << PAGE_SHIFT) + + vma_subpage_offset(vma); } start = vma->vm_start; @@ -717,7 +727,8 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) if (vma->vm_file) { const struct inode *inode = file_user_inode(vma->vm_file); - karg.vma_offset = ((__u64)vma->vm_pgoff) << PAGE_SHIFT; + karg.vma_offset = (((__u64)vma->vm_pgoff) << PAGE_SHIFT) + + vma_subpage_offset(vma); karg.dev_major = MAJOR(inode->i_sb->s_dev); karg.dev_minor = MINOR(inode->i_sb->s_dev); karg.inode = inode->i_ino; @@ -857,6 +868,50 @@ const struct file_operations proc_pid_maps_operations = { */ #define PSS_SHIFT 12 +#if defined(CONFIG_MM_SUBPAGE) && (defined(CONFIG_PROC_PAGE_MONITOR) || defined(CONFIG_NUMA)) +/* Snapshot only: rmap and quarter counters can change in other address spaces. */ +static void file_quarter_mapcounts(struct page *page, struct mm_subpage_mapcounts *counts) +{ + int maps = folio_precise_page_mapcount(page_folio(page), page); + unsigned int small = mm_subpage_file_mapcounts(page, counts); + unsigned int native = maps > small ? maps - small : 0; + unsigned int i; + + for (i = 0; i < MM_SUBPAGES_PER_PAGE; i++) + counts->slots[i] += native; +} + +static unsigned int proc_leaf_mapcount(struct vm_area_struct *vma, pte_t pte, + struct page *page) +{ + struct folio *folio = page_folio(page); + struct mm_subpage_mapcounts counts; + unsigned int i, count = 0; + + if (!folio_test_anon(folio)) { + unsigned int first = mm_page_size(vma->vm_mm) < PAGE_SIZE ? + offset_in_page(pte_phys_mm(vma->vm_mm, pte)) / SZ_4K : 0; + unsigned int nr = mm_page_size(vma->vm_mm) / SZ_4K; + + file_quarter_mapcounts(page, &counts); + for (i = first; i < first + nr; i++) + count = max(count, counts.slots[i]); + return count; + } + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + + if (slot) { + count = mm_subpage_mapcount(slot); + mm_subpage_put(slot); + } + return count; + } + return folio_precise_page_mapcount(folio, page); +} + +#endif + #ifdef CONFIG_PROC_PAGE_MONITOR struct mem_size_stats { unsigned long resident; @@ -913,6 +968,73 @@ static void smaps_page_accumulate(struct mem_size_stats *mss, } } +#ifdef CONFIG_MM_SUBPAGE +static void smaps_account_file_extent(struct mem_size_stats *mss, struct page *page, + unsigned int offset, unsigned long size, bool dirty, bool locked, bool present) +{ + struct folio *folio = page_folio(page); + struct mm_subpage_mapcounts counts = {}; + unsigned int i; + + if (present) + file_quarter_mapcounts(page, &counts); + for (i = offset / SZ_4K; i < (offset + size) / SZ_4K; i++) { + unsigned long pss = SZ_4K << PSS_SHIFT; + + if (counts.slots[i] > 1) + pss /= counts.slots[i]; + smaps_page_accumulate(mss, folio, SZ_4K, pss, dirty, locked, + present && counts.slots[i] < 2); + } +} + +static void smaps_account_small(struct mem_size_stats *mss, struct vm_area_struct *vma, + struct page *page, pte_t pte, bool young, + bool dirty, bool locked, bool present) +{ + struct folio *folio = page_folio(page); + unsigned long size = mm_page_size(vma->vm_mm), pss = size << PSS_SHIFT; + unsigned int count = present && folio_test_anon(folio) ? + proc_leaf_mapcount(vma, pte, page) : 0; + + mss->resident += size; + if (folio_test_anon(folio)) { + mss->anonymous += size; + if (present && !dirty) { + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + + if (slot) { + if (mm_subpage_is_lazyfree(slot)) + mss->lazyfree += size; + mm_subpage_put(slot); + } + } + } + if (folio_test_ksm(folio)) + mss->ksm += size; + if (young || folio_test_young(folio) || folio_test_referenced(folio)) + mss->referenced += size; + if (present && !folio_test_anon(folio)) { + smaps_account_file_extent(mss, page, offset_in_page(pte_phys_mm(vma->vm_mm, pte)), + size, dirty, locked, present); + return; + } + if (count > 1) + pss /= count; + smaps_page_accumulate(mss, folio, size, pss, dirty, locked, present && count < 2); +} + +static bool smaps_account_file_quarters(struct mem_size_stats *mss, struct page *page, + bool dirty, bool locked) +{ + if (folio_test_anon(page_folio(page))) + return false; + smaps_account_file_extent(mss, page, 0, PAGE_SIZE, dirty, locked, true); + return true; +} + +#endif + static void smaps_account(struct mem_size_stats *mss, struct page *page, bool compound, bool young, bool dirty, bool locked, bool present) @@ -976,6 +1098,11 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page, for (i = 0; i < nr; i++, page++) { unsigned long pss = PAGE_SIZE << PSS_SHIFT; +#ifdef CONFIG_MM_SUBPAGE + if (smaps_account_file_quarters(mss, page, dirty, locked)) + continue; +#endif + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) { mapcount = folio_precise_page_mapcount(folio, page); exclusive = mapcount < 2; @@ -995,6 +1122,14 @@ static int smaps_pte_hole(unsigned long addr, unsigned long end, struct mem_size_stats *mss = walk->private; struct vm_area_struct *vma = walk->vma; + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + u64 start = ((u64)pos.index << PAGE_SHIFT) + pos.offset; + + mss->swap += shmem_partial_swap_usage_bytes(vma->vm_file->f_mapping, + start, start + end - addr); + return 0; + } mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping, linear_page_index(vma, addr), linear_page_index(vma, end)); @@ -1010,7 +1145,7 @@ static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk) #ifdef CONFIG_SHMEM if (walk->ops->pte_hole) { /* depth is not used */ - smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk); + smaps_pte_hole(addr, addr + mm_page_size(walk->mm), 0, walk); } #endif } @@ -1038,15 +1173,20 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr, if (softleaf_is_swap(entry)) { int mapcount; - mss->swap += PAGE_SIZE; - mapcount = swp_swapcount(entry); + mss->swap += mm_page_size(vma->vm_mm); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + mapcount = swap_subpage_count(entry, pte_swp_subpage_offset(ptent)); + else +#endif + mapcount = swp_swapcount(entry); if (mapcount >= 2) { - u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT; + u64 pss_delta = (u64)mm_page_size(vma->vm_mm) << PSS_SHIFT; do_div(pss_delta, mapcount); mss->swap_pss += pss_delta; } else { - mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT; + mss->swap_pss += (u64)mm_page_size(vma->vm_mm) << PSS_SHIFT; } } else if (softleaf_has_pfn(entry)) { if (softleaf_is_device_private(entry)) @@ -1058,6 +1198,12 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr, if (!page) return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + smaps_account_small(mss, vma, page, ptent, young, dirty, locked, present); + return; + } +#endif smaps_account(mss, page, false, young, dirty, locked, present); } @@ -1124,7 +1270,7 @@ static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte++, addr += PAGE_SIZE) + for (; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) smaps_pte_entry(pte, addr, walk); pte_unmap_unlock(pte - 1, ptl); out: @@ -1712,7 +1858,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte++, addr += PAGE_SIZE) { + for (; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) { ptent = ptep_get(pte); if (cp->type == CLEAR_REFS_SOFT_DIRTY) { @@ -1855,9 +2001,6 @@ struct pagemapread { bool show_pfn; }; -#define PAGEMAP_WALK_SIZE (PMD_SIZE) -#define PAGEMAP_WALK_MASK (PMD_MASK) - #define PM_ENTRY_BYTES sizeof(pagemap_entry_t) #define PM_PFRAME_BITS 55 #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0) @@ -1884,7 +2027,7 @@ static int add_to_pagemap(pagemap_entry_t *pme, struct pagemapread *pm) return 0; } -static bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page) +static inline bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page) { if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) return folio_precise_page_mapcount(folio, page) == 1; @@ -1909,7 +2052,7 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, else hole_end = end; - for (; addr < hole_end; addr += PAGE_SIZE) { + for (; addr < hole_end; addr += mm_page_size(walk->mm)) { err = add_to_pagemap(&pme, pm); if (err) goto out; @@ -1921,7 +2064,7 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, /* Addresses in the VMA. */ if (vma->vm_flags & VM_SOFTDIRTY) pme = make_pme(0, PM_SOFT_DIRTY); - for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) { + for (; addr < min(end, vma->vm_end); addr += mm_page_size(walk->mm)) { err = add_to_pagemap(&pme, pm); if (err) goto out; @@ -1931,6 +2074,16 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, return err; } +static bool pagemap_page_exclusive(struct vm_area_struct *vma, pte_t pte, + struct folio *folio, struct page *page) +{ +#ifdef CONFIG_MM_SUBPAGE + return proc_leaf_mapcount(vma, pte, page) == 1; +#else + return __folio_page_mapped_exclusively(folio, page); +#endif +} + static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, struct vm_area_struct *vma, unsigned long addr, pte_t pte) { @@ -1943,7 +2096,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, if (pte_present(pte)) { if (pm->show_pfn) - frame = pte_pfn(pte); + frame = pte_phys_mm(vma->vm_mm, pte) >> mm_page_shift(vma->vm_mm); flags |= PM_PRESENT; page = vm_normal_page(vma, addr, pte); if (pte_soft_dirty(pte)) @@ -1969,6 +2122,12 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, offset = softleaf_to_pfn(entry); else offset = swp_offset(entry); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE && + (softleaf_is_swap(entry) || softleaf_has_pfn(entry))) + offset = (offset << (PAGE_SHIFT - mm_page_shift(vma->vm_mm))) + + (pte_swp_subpage_offset(pte) >> mm_page_shift(vma->vm_mm)); +#endif frame = swp_type(entry) | (offset << MAX_SWAPFILES_SHIFT); } @@ -1985,8 +2144,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, folio = page_folio(page); if (!folio_test_anon(folio)) flags |= PM_FILE; - if ((flags & PM_PRESENT) && - __folio_page_mapped_exclusively(folio, page)) + if ((flags & PM_PRESENT) && pagemap_page_exclusive(vma, pte, folio, page)) flags |= PM_MMAP_EXCLUSIVE; } @@ -2102,7 +2260,7 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return err; } - for (; addr < end; pte++, addr += PAGE_SIZE) { + for (; addr < end; pte++, addr += mm_page_size(walk->mm)) { pagemap_entry_t pme; pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte)); @@ -2236,7 +2394,7 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, /* do not disclose physical addresses: attack vector */ pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN); - pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT); + pm.len = (pmd_size_mm(mm) >> mm_page_shift(mm)); pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL); ret = -ENOMEM; if (!pm.buffer) @@ -2248,16 +2406,16 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, /* watch out for wraparound */ start_vaddr = end_vaddr; - if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) { + if (svpfn <= (ULONG_MAX >> mm_page_shift(mm))) { unsigned long end; ret = mmap_read_lock_killable(mm); if (ret) goto out_free; - start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT); + start_vaddr = untagged_addr_remote(mm, svpfn << mm_page_shift(mm)); mmap_read_unlock(mm); - end = start_vaddr + ((count / PM_ENTRY_BYTES) << PAGE_SHIFT); + end = start_vaddr + ((count / PM_ENTRY_BYTES) << mm_page_shift(mm)); if (end >= start_vaddr && end < mm->task_size) end_vaddr = end; } @@ -2272,7 +2430,7 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, unsigned long end; pm.pos = 0; - end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK; + end = (start_vaddr + pmd_size_mm(mm)) & pmd_mask_mm(mm); /* overflow ? */ if (end < start_vaddr || end > end_vaddr) end = end_vaddr; @@ -2333,6 +2491,7 @@ static int pagemap_release(struct inode *inode, struct file *file) struct pagemap_scan_private { struct pm_scan_arg arg; + unsigned long page_size; unsigned long masks_of_interest, cur_vma_category; struct page_region *vec_buf; unsigned long vec_buf_len, vec_buf_index, found_pages; @@ -2559,7 +2718,7 @@ static void pagemap_scan_backout_range(struct pagemap_scan_private *p, else cur_buf->start = cur_buf->end = 0; - p->found_pages -= (end - addr) / PAGE_SIZE; + p->found_pages -= (end - addr) / p->page_size; } #endif @@ -2671,11 +2830,11 @@ static int pagemap_scan_output(unsigned long categories, categories &= p->arg.return_mask; - n_pages = (*end - addr) / PAGE_SIZE; + n_pages = (*end - addr) / p->page_size; if (check_add_overflow(p->found_pages, n_pages, &total_pages) || total_pages > p->arg.max_pages) { size_t n_too_much = total_pages - p->arg.max_pages; - *end -= n_too_much * PAGE_SIZE; + *end -= n_too_much * p->page_size; n_pages -= n_too_much; ret = -ENOSPC; } @@ -2769,7 +2928,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) { /* Fast path for performing exclusive WP */ - for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { + for (addr = start; addr != end; pte++, addr += p->page_size) { pte_t ptent = ptep_get(pte); if ((pte_present(ptent) && pte_uffd_wp(ptent)) || @@ -2778,7 +2937,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, make_uffd_wp_pte(vma, addr, pte, ptent); if (!flush_end) start = addr; - flush_end = addr + PAGE_SIZE; + flush_end = addr + p->page_size; } goto flush_and_return; } @@ -2786,8 +2945,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, if (!p->arg.category_anyof_mask && !p->arg.category_inverted && p->arg.category_mask == PAGE_IS_WRITTEN && p->arg.return_mask == PAGE_IS_WRITTEN) { - for (addr = start; addr < end; pte++, addr += PAGE_SIZE) { - unsigned long next = addr + PAGE_SIZE; + for (addr = start; addr < end; pte++, addr += p->page_size) { + unsigned long next = addr + p->page_size; pte_t ptent = ptep_get(pte); if ((pte_present(ptent) && pte_uffd_wp(ptent)) || @@ -2807,11 +2966,11 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, goto flush_and_return; } - for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { + for (addr = start; addr != end; pte++, addr += p->page_size) { pte_t ptent = ptep_get(pte); unsigned long categories = p->cur_vma_category | pagemap_page_category(p, vma, addr, ptent); - unsigned long next = addr + PAGE_SIZE; + unsigned long next = addr + p->page_size; if (!pagemap_scan_is_interesting_page(categories, p)) continue; @@ -3007,7 +3166,7 @@ static const struct mm_walk_ops pagemap_scan_ops = { .hugetlb_entry = pagemap_scan_hugetlb_entry, }; -static int pagemap_scan_get_args(struct pm_scan_arg *arg, +static int pagemap_scan_get_args(struct mm_struct *mm, struct pm_scan_arg *arg, unsigned long uarg) { if (copy_from_user(arg, (void __user *)uarg, sizeof(*arg))) @@ -3028,7 +3187,7 @@ static int pagemap_scan_get_args(struct pm_scan_arg *arg, arg->vec = untagged_addr((unsigned long)arg->vec); /* Validate memory pointers */ - if (!IS_ALIGNED(arg->start, PAGE_SIZE)) + if (!IS_ALIGNED(arg->start, mm_page_size(mm))) return -EINVAL; if (!access_ok((void __user *)(long)arg->start, arg->end - arg->start)) return -EFAULT; @@ -3041,7 +3200,7 @@ static int pagemap_scan_get_args(struct pm_scan_arg *arg, return -EFAULT; /* Fixup default values */ - arg->end = ALIGN(arg->end, PAGE_SIZE); + arg->end = ALIGN(arg->end, mm_page_size(mm)); arg->walk_end = 0; if (!arg->max_pages) arg->max_pages = ULONG_MAX; @@ -3060,12 +3219,13 @@ static int pagemap_scan_writeback_args(struct pm_scan_arg *arg, return 0; } -static int pagemap_scan_init_bounce_buffer(struct pagemap_scan_private *p) +static int pagemap_scan_init_bounce_buffer(struct mm_struct *mm, + struct pagemap_scan_private *p) { if (!p->arg.vec_len) return 0; - p->vec_buf_len = min_t(size_t, PAGEMAP_WALK_SIZE >> PAGE_SHIFT, + p->vec_buf_len = min_t(size_t, pmd_size_mm(mm) >> mm_page_shift(mm), p->arg.vec_len); p->vec_buf = kmalloc_objs(*p->vec_buf, p->vec_buf_len); if (!p->vec_buf) @@ -3106,18 +3266,18 @@ static long pagemap_scan_flush_buffer(struct pagemap_scan_private *p) static long do_pagemap_scan(struct mm_struct *mm, unsigned long uarg) { - struct pagemap_scan_private p = {0}; + struct pagemap_scan_private p = { .page_size = mm_page_size(mm) }; unsigned long walk_start; size_t n_ranges_out = 0; int ret; - ret = pagemap_scan_get_args(&p.arg, uarg); + ret = pagemap_scan_get_args(mm, &p.arg, uarg); if (ret) return ret; p.masks_of_interest = p.arg.category_mask | p.arg.category_anyof_mask | p.arg.return_mask; - ret = pagemap_scan_init_bounce_buffer(&p); + ret = pagemap_scan_init_bounce_buffer(mm, &p); if (ret) return ret; @@ -3219,16 +3379,17 @@ struct numa_maps_private { struct numa_maps md; }; +static int numa_page_mapcount(struct page *page) +{ + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + return folio_precise_page_mapcount(page_folio(page), page); + return folio_average_page_mapcount(page_folio(page)); +} + static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty, - unsigned long nr_pages) + unsigned long nr_pages, int count) { struct folio *folio = page_folio(page); - int count; - - if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) - count = folio_precise_page_mapcount(folio, page); - else - count = folio_average_page_mapcount(folio); md->pages += nr_pages; if (pte_dirty || folio_test_dirty(folio)) @@ -3318,7 +3479,7 @@ static int gather_pte_stats(pmd_t *pmd, unsigned long addr, page = can_gather_numa_stats_pmd(*pmd, vma, addr); if (page) gather_stats(page, md, pmd_dirty(*pmd), - HPAGE_PMD_SIZE/PAGE_SIZE); + HPAGE_PMD_SIZE/PAGE_SIZE, numa_page_mapcount(page)); spin_unlock(ptl); return 0; } @@ -3333,9 +3494,13 @@ static int gather_pte_stats(pmd_t *pmd, unsigned long addr, struct page *page = can_gather_numa_stats(ptent, vma, addr); if (!page) continue; - gather_stats(page, md, pte_dirty(ptent), 1); +#ifdef CONFIG_MM_SUBPAGE + gather_stats(page, md, pte_dirty(ptent), 1, proc_leaf_mapcount(vma, ptent, page)); +#else + gather_stats(page, md, pte_dirty(ptent), 1, numa_page_mapcount(page)); +#endif - } while (pte++, addr += PAGE_SIZE, addr != end); + } while (pte++, addr += mm_page_size(vma->vm_mm), addr != end); pte_unmap_unlock(orig_pte, ptl); cond_resched(); return 0; @@ -3357,7 +3522,7 @@ static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, page = pte_page(huge_pte); md = walk->private; - gather_stats(page, md, pte_dirty(huge_pte), 1); + gather_stats(page, md, pte_dirty(huge_pte), 1, numa_page_mapcount(page)); out: spin_unlock(ptl); return 0; diff --git a/fs/splice.c b/fs/splice.c index 9d8f63e2fd1ab..4b707ce00e86d 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -183,6 +185,79 @@ static void wakeup_pipe_readers(struct pipe_inode_info *pipe) kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); } +#ifdef CONFIG_MM_SUBPAGE +/* A shared lifetime for all user leaves coalesced into one native pipe page. */ +struct subpage_pipe_refs { + refcount_t refs; + unsigned int count; + struct user_page_fragment fragments[]; +}; + +static void subpage_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs = (void *)buf->private; + + if (refcount_dec_and_test(&refs->refs)) { + release_user_fragments(refs->fragments, refs->count, false); + kfree(refs); + } +} + +static bool subpage_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs = (void *)buf->private; + + return refcount_inc_not_zero(&refs->refs); +} + +static const struct pipe_buf_operations subpage_pipe_buf_ops = { + .release = subpage_pipe_buf_release, + .get = subpage_pipe_buf_get, + /* A gift of user leaves is not ownership of the containing native page. */ +}; + +static int subpage_pipe_buf_retain(struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs; + unsigned int pos, count = 0; + + if (!buf->len || buf->ops != &nosteal_pipe_buf_ops || + !folio_test_anon(page_folio(buf->page))) + return 0; + refs = kmalloc(struct_size(refs, fragments, PAGE_SIZE / MM_SUBPAGE_SIZE), GFP_KERNEL); + if (!refs) + return -ENOMEM; + for (pos = round_down(buf->offset, MM_SUBPAGE_SIZE); + pos < buf->offset + buf->len; pos += MM_SUBPAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(page_to_phys(buf->page) + pos); + + if (!slot) { + release_user_fragments(refs->fragments, count, false); + kfree(refs); + return count ? -EFAULT : 0; + } + refs->fragments[count++] = (struct user_page_fragment) { + .folio = page_folio(buf->page), + .subpage = slot, + .offset = pos, + .length = MM_SUBPAGE_SIZE, + }; + } + refcount_set(&refs->refs, 1); + refs->count = count; + /* Replace the incoming native reference with the typed references. */ + put_page(buf->page); + buf->ops = &subpage_pipe_buf_ops; + buf->private = (unsigned long)refs; + return 0; +} +#else +static int subpage_pipe_buf_retain(struct pipe_buffer *buf) +{ + return 0; +} +#endif + /** * splice_to_pipe - fill passed data into a pipe * @pipe: pipe to fill @@ -221,6 +296,16 @@ ssize_t splice_to_pipe(struct pipe_inode_info *pipe, buf->private = spd->partial[page_nr].private; buf->ops = spd->ops; buf->flags = 0; + { + int err = subpage_pipe_buf_retain(buf); + + if (err) { + buf->ops = NULL; + if (!ret) + ret = err; + break; + } + } head++; pipe->head = head; @@ -852,6 +937,16 @@ ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out, continue; } +#ifdef CONFIG_MM_SUBPAGE + /* Other transports still need their downstream lifetime audit. */ + if (buf->ops == &subpage_pipe_buf_ops && + sock->ops->family != AF_UNIX && + sock->ops->family != AF_INET && + sock->ops->family != AF_INET6) { + ret = -EOPNOTSUPP; + break; + } +#endif seg = min_t(size_t, remain, buf->len); ret = pipe_buf_confirm(pipe, buf); @@ -1440,6 +1535,79 @@ static ssize_t __do_splice(struct file *in, loff_t __user *off_in, return ret; } +#ifdef CONFIG_MM_SUBPAGE +static ssize_t subpage_iter_to_pipe(struct iov_iter *from, + struct pipe_inode_info *pipe, unsigned int flags) +{ + struct user_page_fragment fragments[16]; + size_t total = 0; + long ret = 0; + + if (WARN_ON_ONCE(!user_backed_iter(from))) + return -EINVAL; + while (iov_iter_count(from)) { + unsigned int i = 0; + long count; + + /* Normalize empty iovecs before taking the next address. */ + iov_iter_advance(from, 0); + count = get_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(from), + min(iov_iter_count(from), iter_iov_len(from)), 0, + fragments, ARRAY_SIZE(fragments)); + if (count <= 0) { + ret = count; + break; + } + while (i < count) { + struct user_page_fragment *first = &fragments[i]; + struct page *page = folio_page(first->folio, first->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(first->offset); + unsigned int length = first->length, nr = 1; + struct subpage_pipe_refs *refs; + struct pipe_buffer buf; + + /* Preserve native pipe capacity for contiguous packed quarters. */ + while (i + nr < count && length < PAGE_SIZE - offset) { + struct user_page_fragment *next = &fragments[i + nr]; + + if (folio_page(next->folio, next->offset >> PAGE_SHIFT) != page || + offset_in_page(next->offset) != offset + length) + break; + length += next->length; + nr++; + } + refs = kmalloc(struct_size(refs, fragments, nr), GFP_KERNEL); + if (!refs) { + ret = -ENOMEM; + break; + } + refcount_set(&refs->refs, 1); + refs->count = nr; + memcpy(refs->fragments, first, sizeof(*first) * nr); + buf = (struct pipe_buffer) { + .page = page, + .offset = offset, + .len = length, + .ops = &subpage_pipe_buf_ops, + .flags = flags, + .private = (unsigned long)refs, + }; + i += nr; /* add_to_pipe consumes these references even on failure. */ + ret = add_to_pipe(pipe, &buf); + if (ret < 0) + break; + iov_iter_advance(from, ret); + total += ret; + } + release_user_fragments(fragments + i, count - i, false); + if (ret < 0) + break; + } + return total ? total : ret; +} +#endif + static ssize_t iter_to_pipe(struct iov_iter *from, struct pipe_inode_info *pipe, unsigned int flags) @@ -1451,6 +1619,10 @@ static ssize_t iter_to_pipe(struct iov_iter *from, size_t total = 0; ssize_t ret = 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return subpage_iter_to_pipe(from, pipe, flags); +#endif while (iov_iter_count(from)) { struct page *pages[16]; ssize_t left; diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index dba1172436b74..3174e3b6edb41 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -301,6 +301,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, pmd_t *pmd, _pmd; pte_t *pte; pte_t ptent; + spinlock_t *ptl; bool ret; assert_fault_locked(vmf); @@ -308,13 +309,13 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, pgd = pgd_offset(mm, address); if (!pgd_present(*pgd)) return true; - p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + p4d = p4d_offset_mm(mm, pgd, address); + if (!p4d_present_mm(mm, *p4d)) return true; - pud = pud_offset(p4d, address); + pud = pud_offset_mm(mm, p4d, address); if (!pud_present(*pud)) return true; - pmd = pmd_offset(pud, address); + pmd = pmd_offset_mm(mm, pud, address); again: _pmd = pmdp_get_lockless(pmd); if (pmd_none(_pmd)) @@ -331,7 +332,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, if (pmd_trans_huge(_pmd)) return !pmd_write(_pmd) && (reason & VM_UFFD_WP); - pte = pte_offset_map(pmd, address); + pte = pte_offset_map_ro_nolock(mm, pmd, address, &ptl); if (!pte) goto again; @@ -1251,13 +1252,11 @@ static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx, __wake_userfault(ctx, range); } -static __always_inline int validate_unaligned_range( +static __always_inline int validate_byte_range( struct mm_struct *mm, __u64 start, __u64 len) { __u64 task_size = mm->task_size; - if (len & ~PAGE_MASK) - return -EINVAL; if (!len) return -EINVAL; if (start >= task_size) @@ -1269,10 +1268,19 @@ static __always_inline int validate_unaligned_range( return 0; } +static __always_inline int validate_unaligned_range( + struct mm_struct *mm, __u64 start, __u64 len) +{ + if (len & ~mm_page_mask(mm)) + return -EINVAL; + + return validate_byte_range(mm, start, len); +} + static __always_inline int validate_range(struct mm_struct *mm, __u64 start, __u64 len) { - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; return validate_unaligned_range(mm, start, len); @@ -1428,6 +1436,13 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC : UFFD_API_RANGE_IOCTLS; + if (mm_page_size(mm) != PAGE_SIZE) { + ioctls_out &= (1ULL << _UFFDIO_WAKE) | (1ULL << _UFFDIO_WRITEPROTECT) | + (1ULL << _UFFDIO_CONTINUE) | (1ULL << _UFFDIO_COPY) | + (1ULL << _UFFDIO_ZEROPAGE) | (1ULL << _UFFDIO_POISON); + } + if (mm_page_size(mm) < PAGE_SIZE) + ioctls_out |= 1ULL << _UFFDIO_MOVE; /* * Declare the WP ioctl only if the WP mode is @@ -1645,8 +1660,12 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, sizeof(uffdio_copy)-sizeof(__s64))) goto out; - ret = validate_unaligned_range(ctx->mm, uffdio_copy.src, - uffdio_copy.len); + /* + * COPY reads the pager's address space, which can have a different + * task size and granule from the registered target. Only the target + * range below requires a page-aligned length. + */ + ret = validate_byte_range(current->mm, uffdio_copy.src, uffdio_copy.len); if (ret) goto out; ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len); @@ -2015,6 +2034,18 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, /* report all available features and ioctls to userland */ uffdio_api.features = UFFD_API_FEATURES; + /* Base-page anonymous and shared shmem operations support alternative leaves. */ + if (mm_page_size(ctx->mm) != PAGE_SIZE) + uffdio_api.features &= UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_MISSING_SHMEM | UFFD_FEATURE_MINOR_SHMEM | + UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_EVENT_REMAP | + UFFD_FEATURE_EVENT_REMOVE | UFFD_FEATURE_EVENT_UNMAP | + UFFD_FEATURE_SIGBUS | UFFD_FEATURE_THREAD_ID | UFFD_FEATURE_EXACT_ADDRESS | + UFFD_FEATURE_WP_UNPOPULATED | UFFD_FEATURE_WP_ASYNC | UFFD_FEATURE_POISON; + + if (mm_page_size(ctx->mm) < PAGE_SIZE) + uffdio_api.features |= UFFD_FEATURE_MOVE; + #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR uffdio_api.features &= ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); @@ -2062,6 +2093,10 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd, if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx)) return -EINVAL; + /* Larger-than-native MOVE still requires contiguous-owner support. */ + if (cmd == UFFDIO_MOVE && mm_page_size(ctx->mm) > PAGE_SIZE) + return -EOPNOTSUPP; + switch(cmd) { case UFFDIO_API: ret = userfaultfd_api(ctx, arg); diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h index 57137d3ac1592..619fd3ed40634 100644 --- a/include/asm-generic/pgalloc.h +++ b/include/asm-generic/pgalloc.h @@ -81,7 +81,7 @@ static inline pgtable_t __pte_alloc_one_noprof(struct mm_struct *mm, gfp_t gfp) return NULL; } - return ptdesc_page(ptdesc); + return pgtable_from_page(ptdesc_page(ptdesc)); } #define __pte_alloc_one(...) alloc_hooks(__pte_alloc_one_noprof(__VA_ARGS__)) @@ -111,17 +111,18 @@ static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm) * @mm: the mm_struct of the current context * @pte_page: the `struct page` referencing the ptdesc */ -static inline void pte_free(struct mm_struct *mm, struct page *pte_page) +#ifndef __HAVE_ARCH_PTE_FREE +static inline void pte_free(struct mm_struct *mm, pgtable_t pte_page) { - struct ptdesc *ptdesc = page_ptdesc(pte_page); + struct ptdesc *ptdesc = page_ptdesc(pgtable_to_page(pte_page)); pagetable_dtor_free(ptdesc); } +#endif #if CONFIG_PGTABLE_LEVELS > 2 -#ifndef __HAVE_ARCH_PMD_ALLOC_ONE /** * pmd_alloc_one - allocate memory for a PMD-level page table * @mm: the mm_struct of the current context @@ -133,7 +134,7 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page) * * Return: pointer to the allocated memory or %NULL on error */ -static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +static inline pmd_t *__pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) { struct ptdesc *ptdesc; gfp_t gfp = GFP_PGTABLE_USER; @@ -153,6 +154,12 @@ static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long ad return ptdesc_address(ptdesc); } + +#ifndef __HAVE_ARCH_PMD_ALLOC_ONE +static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + return __pmd_alloc_one_noprof(mm, addr); +} #define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__)) #endif diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index bdcc2778ac64f..36b422593e971 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -295,6 +295,12 @@ extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, in bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page, unsigned int nr_pages, bool delay_rmap); +#ifdef CONFIG_MM_SUBPAGE +struct mm_subpage; +/* Consumes one slot reference; true requires flushing the full batch. */ +bool __tlb_remove_subpage(struct mmu_gather *tlb, struct mm_subpage *subpage); +#endif + #ifdef CONFIG_SMP /* * This both sets 'delayed_rmap', and returns true. It would be an inline @@ -543,15 +549,15 @@ static inline void tlb_change_page_size(struct mmu_gather *tlb, static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb) { if (tlb->cleared_ptes) - return PAGE_SHIFT; + return mm_page_shift(tlb->mm); if (tlb->cleared_pmds) - return PMD_SHIFT; + return pmd_shift_mm(tlb->mm); if (tlb->cleared_puds) - return PUD_SHIFT; + return pud_shift_mm(tlb->mm); if (tlb->cleared_p4ds) - return P4D_SHIFT; + return p4d_shift_mm(tlb->mm); - return PAGE_SHIFT; + return mm_page_shift(tlb->mm); } static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) @@ -658,7 +664,7 @@ static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, u */ #define tlb_remove_tlb_entry(tlb, ptep, address) \ do { \ - tlb_flush_pte_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pte_range(tlb, address, mm_page_size((tlb)->mm)); \ __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) @@ -672,13 +678,15 @@ static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, u static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, pte_t *ptep, unsigned int nr, unsigned long address) { - tlb_flush_pte_range(tlb, address, PAGE_SIZE * nr); + unsigned long page_size = mm_page_size(tlb->mm); + + tlb_flush_pte_range(tlb, address, page_size * nr); for (;;) { __tlb_remove_tlb_entry(tlb, ptep, address); if (--nr == 0) break; ptep++; - address += PAGE_SIZE; + address += page_size; } } @@ -739,13 +747,13 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, * architecture to do its own odd thing, not cause pain for others * http://lkml.kernel.org/r/CA+55aFzBggoXtNXQeng5d_mRoDnaMBE5Y+URs+PHR67nUpMtaw@mail.gmail.com * - * For now w.r.t page table cache, mark the range_size as PAGE_SIZE + * For now w.r.t page table cache, mark the range_size as one mm granule. */ #ifndef pte_free_tlb #define pte_free_tlb(tlb, ptep, address) \ do { \ - tlb_flush_pmd_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pmd_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pte_free_tlb(tlb, ptep, address); \ } while (0) @@ -754,7 +762,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef pmd_free_tlb #define pmd_free_tlb(tlb, pmdp, address) \ do { \ - tlb_flush_pud_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pud_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pmd_free_tlb(tlb, pmdp, address); \ } while (0) @@ -763,7 +771,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef pud_free_tlb #define pud_free_tlb(tlb, pudp, address) \ do { \ - tlb_flush_p4d_range(tlb, address, PAGE_SIZE); \ + tlb_flush_p4d_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pud_free_tlb(tlb, pudp, address); \ } while (0) @@ -772,7 +780,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef p4d_free_tlb #define p4d_free_tlb(tlb, pudp, address) \ do { \ - __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + __tlb_adjust_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __p4d_free_tlb(tlb, pudp, address); \ } while (0) diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h index 65abd5ab8836c..c242a5e9cc758 100644 --- a/include/linux/binfmts.h +++ b/include/linux/binfmts.h @@ -93,6 +93,8 @@ struct linux_binfmt { #ifdef CONFIG_COREDUMP int (*core_dump)(struct coredump_params *cprm); unsigned long min_coredump; /* minimal dump size */ + /* Optional per-mm replacement for formats with a variable page ABI. */ + unsigned long (*min_coredump_size)(void); #endif } __randomize_layout; @@ -121,6 +123,10 @@ static inline void insert_binfmt(struct linux_binfmt *fmt) extern void unregister_binfmt(struct linux_binfmt *); +#ifdef CONFIG_MM_SUBPAGE +int bprm_set_page_shift(struct linux_binprm *bprm, unsigned int page_shift); +#endif + extern int __must_check remove_arg_zero(struct linux_binprm *); extern int begin_new_exec(struct linux_binprm * bprm); extern void setup_new_exec(struct linux_binprm * bprm); diff --git a/include/linux/bio.h b/include/linux/bio.h index dc17780d6c1e3..714e44044373a 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -389,11 +389,19 @@ static inline void bio_wouldblock_error(struct bio *bio) * pointed by @iter. If @iter is backed by bvec it's going to be reused * instead of allocating a new one. */ +#ifdef CONFIG_MM_SUBPAGE +int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs); +#endif + static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs) { if (iov_iter_is_bvec(iter)) return 0; +#ifdef CONFIG_MM_SUBPAGE + return bio_iov_user_vecs(iter, max_segs); +#else return iov_iter_npages(iter, max_segs); +#endif } /** diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 8808ee76e73c0..cc9b1eeddd415 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -267,6 +267,10 @@ struct bio { struct bio_integrity_payload *bi_integrity; /* data integrity */ #endif +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *bi_user_fragments; + unsigned short bi_nr_user_fragments; +#endif unsigned short bi_vcnt; /* how many bio_vec's */ /* diff --git a/include/linux/bpf.h b/include/linux/bpf.h index a2cf24d168802..cfec2e3b66979 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -263,6 +263,11 @@ struct btf_record { int wq_off; int refcount_off; int task_work_off; +#ifdef CONFIG_MM_SUBPAGE + /* Private ownership pointer appended to task-storage values with UPTRs. */ + u32 uptr_pins_off; + u32 uptr_extra_size; +#endif struct btf_field fields[]; }; @@ -563,6 +568,15 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src bpf_obj_memcpy(map->record, dst, src, map->value_size, true); } +static inline u32 bpf_obj_uptr_extra_size(const struct btf_record *rec) +{ +#ifdef CONFIG_MM_SUBPAGE + return IS_ERR_OR_NULL(rec) ? 0 : rec->uptr_extra_size; +#else + return 0; +#endif +} + static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) { unsigned long *src_uptr, *dst_uptr; @@ -580,6 +594,14 @@ static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, v dst_uptr = dst + field->offset; swap(*src_uptr, *dst_uptr); } +#ifdef CONFIG_MM_SUBPAGE + if (rec->uptr_extra_size) { + void **src_pins = src + rec->uptr_pins_off; + void **dst_pins = dst + rec->uptr_pins_off; + + swap(*src_pins, *dst_pins); + } +#endif } static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size) diff --git a/include/linux/coredump.h b/include/linux/coredump.h index 68861da4cf7c2..5f85a12c7f998 100644 --- a/include/linux/coredump.h +++ b/include/linux/coredump.h @@ -12,10 +12,21 @@ struct core_vma_metadata { unsigned long start, end; vm_flags_t flags; unsigned long dump_size; - unsigned long pgoff; + unsigned long pgoff; /* File offset in the dumped mm's page units. */ struct file *file; }; +/* NT_FILE offsets are expressed in the dumped process's page units. */ +static inline unsigned long core_vma_pgoff(const struct vm_area_struct *vma) +{ + unsigned int shift = mm_page_shift(vma->vm_mm); + + if (shift > PAGE_SHIFT) + return vma->vm_pgoff >> (shift - PAGE_SHIFT); + return (vma->vm_pgoff << (PAGE_SHIFT - shift)) + + (vma_subpage_offset(vma) >> shift); +} + struct coredump_params { const kernel_siginfo_t *siginfo; struct file *file; diff --git a/include/linux/highmem.h b/include/linux/highmem.h index d7aac9de1c8a4..e149140b4af3e 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -330,6 +330,21 @@ struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, } #endif +#ifndef vma_alloc_zeroed_movable_folio_order +static inline struct folio *vma_alloc_zeroed_movable_folio_order( + struct vm_area_struct *vma, unsigned long addr, unsigned int order) +{ + struct folio *folio; + + if (!order) + return vma_alloc_zeroed_movable_folio(vma, addr); + folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, order, vma, addr); + if (folio && user_alloc_needs_zeroing()) + folio_zero_user(folio, addr); + return folio; +} +#endif + static inline void clear_highpage(struct page *page) { void *kaddr = kmap_local_page(page); diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 2949e5acff351..1f7f729b87868 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -3,6 +3,7 @@ #define _LINUX_HUGE_MM_H #include +#include #include /* only for vma_is_dax() */ #include @@ -331,6 +332,9 @@ struct thpsize { static inline bool vma_thp_disabled(struct vm_area_struct *vma, vm_flags_t vm_flags, bool forced_collapse) { + /* Native THP geometry and rmap cannot describe alternative user leaves. */ + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return true; /* Are THPs disabled for this VMA? */ if (vm_flags & VM_NOHUGEPAGE) return true; diff --git a/include/linux/khugepaged.h b/include/linux/khugepaged.h index d7a9053ff4fed..1510fdb048010 100644 --- a/include/linux/khugepaged.h +++ b/include/linux/khugepaged.h @@ -8,6 +8,8 @@ extern unsigned int khugepaged_max_ptes_none __read_mostly; #ifdef CONFIG_TRANSPARENT_HUGEPAGE extern struct attribute_group khugepaged_attr_group; +int collapse_file_user_page(struct mm_struct *mm, struct file *file, pgoff_t start); + extern int khugepaged_init(void); extern void khugepaged_destroy(void); extern int start_stop_khugepaged(void); @@ -32,6 +34,12 @@ static inline void khugepaged_exit(struct mm_struct *mm) __khugepaged_exit(mm); } #else /* CONFIG_TRANSPARENT_HUGEPAGE */ +static inline int collapse_file_user_page(struct mm_struct *mm, struct file *file, + pgoff_t start) +{ + return -EOPNOTSUPP; +} + static inline void khugepaged_fork(struct mm_struct *mm, struct mm_struct *oldmm) { } diff --git a/include/linux/mm.h b/include/linux/mm.h index f4481c338b36c..6c01377914fe8 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -750,6 +751,9 @@ struct vm_fault { }; struct page *cow_page; /* Page handler may use for COW fault */ +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *cow_subpage; /* Owns cow_page's selected fragment */ +#endif struct page *page; /* ->fault handlers should return a * page here, unless VM_FAULT_NOPAGE * is set (which is also implied by @@ -1615,7 +1619,7 @@ static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) { if (unlikely(vma->vm_ops && vma->vm_ops->pagesize)) return vma->vm_ops->pagesize(vma); - return PAGE_SIZE; + return mm_page_size(vma->vm_mm); } unsigned long vma_mmu_pagesize(struct vm_area_struct *vma); @@ -2010,6 +2014,10 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa void set_pte_range(struct vm_fault *vmf, struct folio *folio, struct page *page, unsigned int nr, unsigned long addr); +#ifdef CONFIG_MM_SUBPAGE +vm_fault_t map_file_user_pages(struct vm_fault *vmf, struct folio *folio, + pgoff_t first, pgoff_t last, bool fault); +#endif vm_fault_t finish_fault(struct vm_fault *vmf); #endif @@ -3143,10 +3151,13 @@ struct follow_pfnmap_args { * @address: the virtual address to walk * @write: if true, fail with -EFAULT unless the mapping is * writable + * @allow_subpage: caller handles exact byte offsets and user-leaf bounds; + * otherwise a non-native user granule is rejected */ struct vm_area_struct *vma; unsigned long address; bool write; + bool allow_subpage; /** * Internals: * @@ -3157,13 +3168,15 @@ struct follow_pfnmap_args { /** * Outputs: * - * @pfn: the PFN of the address - * @addr_mask: address mask covering pfn + * @pfn: native PFN containing the requested physical byte + * @offset: byte offset of that byte within the native physical page + * @addr_mask: virtual address mask of the leaf mapping * @pgprot: the pgprot_t of the mapping * @writable: whether the mapping is writable * @special: whether the mapping is a special mapping (real PFN maps) */ unsigned long pfn; + unsigned int offset; unsigned long addr_mask; pgprot_t pgprot; bool writable; @@ -3215,6 +3228,13 @@ static inline void unmap_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen, int even_cows) { } #endif +#ifdef CONFIG_MM_SUBPAGE +void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize); +void unmap_mapping_subpage_hole(struct address_space *mapping, loff_t start, u64 end); +#else +static inline void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize) { } +#endif + static inline void unmap_shared_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen) { @@ -3443,7 +3463,8 @@ static inline void reset_mm_hiwater_rss(struct mm_struct *mm) static inline void setmax_mm_hiwater_rss(unsigned long *maxrss, struct mm_struct *mm) { - unsigned long hiwater_rss = get_mm_hiwater_rss(mm); + /* signal->maxrss/cmaxrss survive exec and child reaping: store kB. */ + unsigned long hiwater_rss = mm_pages_to_kb(mm, get_mm_hiwater_rss(mm)); if (*maxrss < hiwater_rss) *maxrss = hiwater_rss; @@ -3514,14 +3535,14 @@ static inline void mm_inc_nr_puds(struct mm_struct *mm) { if (mm_pud_folded(mm)) return; - atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); + atomic_long_add(pud_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_puds(struct mm_struct *mm) { if (mm_pud_folded(mm)) return; - atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); + atomic_long_sub(pud_table_bytes_mm(mm), &mm->pgtables_bytes); } #endif @@ -3542,14 +3563,14 @@ static inline void mm_inc_nr_pmds(struct mm_struct *mm) { if (mm_pmd_folded(mm)) return; - atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); + atomic_long_add(pmd_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_pmds(struct mm_struct *mm) { if (mm_pmd_folded(mm)) return; - atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); + atomic_long_sub(pmd_table_bytes_mm(mm), &mm->pgtables_bytes); } #endif @@ -3566,12 +3587,12 @@ static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) static inline void mm_inc_nr_ptes(struct mm_struct *mm) { - atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); + atomic_long_add(pte_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_ptes(struct mm_struct *mm) { - atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); + atomic_long_sub(pte_table_bytes_mm(mm), &mm->pgtables_bytes); } #else @@ -3594,32 +3615,40 @@ static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) { return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ? - NULL : p4d_offset(pgd, address); + NULL : p4d_offset_mm(mm, pgd, address); } static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) { - return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? - NULL : pud_offset(p4d, address); + return (unlikely(p4d_none_mm(mm, *p4d)) && __pud_alloc(mm, p4d, address)) ? + NULL : pud_offset_mm(mm, p4d, address); } static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) { return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? - NULL: pmd_offset(pud, address); + NULL : pmd_offset_mm(mm, pud, address); } #endif /* CONFIG_MMU */ enum pt_flags { PT_kernel = PG_referenced, PT_reserved = PG_reserved, + PT_fragmented = PG_private, + PT_frag_defer = PG_active, /* High bits are used for zone/node/section */ }; static inline struct ptdesc *virt_to_ptdesc(const void *x) { - return page_ptdesc(virt_to_page(x)); + struct page *page = virt_to_page(x); + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* One hardware table may span multiple native allocator pages. */ + page = compound_head(page); +#endif + return page_ptdesc(page); } /** @@ -3836,12 +3865,25 @@ static inline bool pagetable_pte_ctor(struct mm_struct *mm, } pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, pmd_t *pmdvalp); +#else +#define __pte_offset_map_mm(mm, pmd, addr, pmdvalp) \ + __pte_offset_map(pmd, addr, pmdvalp) +#endif static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr) { return __pte_offset_map(pmd, addr, NULL); } +static inline pte_t *pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr) +{ + return __pte_offset_map_mm(mm, pmd, addr, NULL); +} + pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp); @@ -3859,7 +3901,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, #define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd)) #define pte_alloc_map(mm, pmd, address) \ - (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address)) + (pte_alloc(mm, pmd) ? NULL : pte_offset_map_mm(mm, pmd, address)) #define pte_alloc_map_lock(mm, pmd, address, ptlp) \ (pte_alloc(mm, pmd) ? \ @@ -3874,7 +3916,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, static inline struct page *pmd_pgtable_page(pmd_t *pmd) { unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); - return virt_to_page((void *)((unsigned long) pmd & mask)); + return ptdesc_page(virt_to_ptdesc((void *)((unsigned long)pmd & mask))); } static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd) @@ -4160,6 +4202,12 @@ extern unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf); +#ifdef CONFIG_MM_SUBPAGE +unsigned long do_mmap_offset(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, struct vm_page_offset pos, unsigned long *populate, + struct list_head *uf); +#endif extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf, bool unlock); @@ -4288,7 +4336,11 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) static inline unsigned long vma_last_pgoff(struct vm_area_struct *vma) { +#ifdef CONFIG_MM_SUBPAGE + return vma_page_offset_at(vma, vma->vm_end - 1).index; +#else return vma->vm_pgoff + vma_pages(vma) - 1; +#endif } static inline unsigned long vma_desc_size(const struct vm_area_desc *desc) @@ -4541,6 +4593,9 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t pgprot); int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); +int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long nr_pages, + unsigned long offset, unsigned long length); int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num); int map_kernel_pages_prepare(struct vm_area_desc *desc); @@ -4552,6 +4607,9 @@ int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, unsigned long num); vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, bool write); +vm_fault_t vmf_insert_pfn_prot_mkwrite_offset(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, unsigned int offset, + pgprot_t pgprot, bool mkwrite); vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot, bool mkwrite); vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, @@ -4648,6 +4706,18 @@ static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma, return VM_FAULT_NOPAGE; } +/* Insert one user granule from an explicit offset within a native page. */ +#ifdef CONFIG_MM_SUBPAGE +vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, unsigned long addr, + struct page *page, unsigned int offset); +#else +static inline vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, + unsigned long addr, struct page *page, unsigned int offset) +{ + return offset ? VM_FAULT_SIGBUS : vmf_insert_page(vma, addr, page); +} +#endif + #ifndef io_remap_pfn_range_pfn static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, unsigned long size) diff --git a/include/linux/mm_granule.h b/include/linux/mm_granule.h new file mode 100644 index 0000000000000..6e73216edcf96 --- /dev/null +++ b/include/linux/mm_granule.h @@ -0,0 +1,245 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MM_GRANULE_H +#define _LINUX_MM_GRANULE_H + +#include +#include +#include + +/* + * User virtual mapping granule. Never use these for physical PFNs, struct + * page/folio sizes, page-cache indices, or physical memory accounting. + * The granule is immutable once an mm has mappings or is visible to tasks. + */ +#ifndef arch_mm_page_shift +static inline unsigned int arch_mm_page_shift(const struct mm_struct *mm) +{ + return PAGE_SHIFT; +} +#endif + +static inline unsigned int mm_page_shift(const struct mm_struct *mm) +{ + return arch_mm_page_shift(mm); +} + +static inline unsigned long mm_page_size(const struct mm_struct *mm) +{ + return 1UL << mm_page_shift(mm); +} + +/* A byte-range reference must fit both the user leaf and a native page. */ +static inline unsigned long mm_user_fragment_size(const struct mm_struct *mm) +{ + return mm_page_shift(mm) < PAGE_SHIFT ? mm_page_size(mm) : PAGE_SIZE; +} + +/* Native physical pages covered by one base user PTE, including fragments. */ +static inline unsigned long mm_pte_native_pages(const struct mm_struct *mm) +{ + unsigned long pages = mm_page_size(mm) >> PAGE_SHIFT; + + return pages ?: 1; +} + +static inline unsigned long mm_page_mask(const struct mm_struct *mm) +{ + return ~(mm_page_size(mm) - 1); +} + +/* + * ASLR tunables count native-page bits. Retain their byte window when a + * process uses another granule, including the extra low user-page bits. + * native_mask must be a contiguous low-bit mask whose byte span fits ulong. + */ +static inline unsigned long mm_aslr_mask(const struct mm_struct *mm, + unsigned long native_mask) +{ + unsigned int shift = mm_page_shift(mm); + + if (shift < PAGE_SHIFT) { + unsigned int extra = PAGE_SHIFT - shift; + + return (native_mask << extra) | ((1UL << extra) - 1); + } + return native_mask >> (shift - PAGE_SHIFT); +} + +/* Virtual-range accounting. Physical pages and cache indices stay native. */ +static inline unsigned long vma_user_pages(const struct vm_area_struct *vma) +{ + return (vma->vm_end - vma->vm_start) >> mm_page_shift(vma->vm_mm); +} + +static inline unsigned long mm_page_align(const struct mm_struct *mm, unsigned long bytes) +{ + return (bytes + mm_page_size(mm) - 1) & mm_page_mask(mm); +} + +/* + * RSS and virtual counters use the target mm's user granule. Consumers + * comparing different mms or physical memory must convert explicitly. + */ +static inline unsigned long mm_pages_to_bytes(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << mm_page_shift(mm); +} + +static inline unsigned long mm_pages_to_kb(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << (mm_page_shift(mm) - 10); +} + +/* Round up only when reducing precision; do not hide a partial user page. */ +static inline unsigned long mm_pages_to_shift(const struct mm_struct *mm, + unsigned long pages, unsigned int shift) +{ + unsigned int source = mm_page_shift(mm); + + if (source >= shift) + return pages << (source - shift); + return (pages >> (shift - source)) + + !!(pages & ((1UL << (shift - source)) - 1)); +} + +static inline unsigned long mm_pages_to_native(const struct mm_struct *mm, + unsigned long pages) +{ + return mm_pages_to_shift(mm, pages, PAGE_SHIFT); +} + +/* A page-count proc ABI uses the reader's granule, including remote reads. */ +static inline unsigned long mm_pages_for_reader(const struct mm_struct *mm, + const struct mm_struct *reader, unsigned long pages) +{ + return mm_pages_to_shift(mm, pages, + reader ? mm_page_shift(reader) : PAGE_SHIFT); +} + +/* + * Time integrals need one unit across exec. Locked VM also uses this unit: + * it must represent both logical mappings and native physical charges exactly, + * including a single native-page pin in a larger-granule userspace mm. + */ +#ifdef CONFIG_MM_SUBPAGE +#define MM_ACCOUNT_SHIFT 12 +#else +#define MM_ACCOUNT_SHIFT PAGE_SHIFT +#endif + +static inline unsigned long mm_pages_to_account(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << (mm_page_shift(mm) - MM_ACCOUNT_SHIFT); +} + +static inline unsigned long mm_locked_vm_bytes(const struct mm_struct *mm) +{ + return READ_ONCE(mm->locked_vm) << MM_ACCOUNT_SHIFT; +} + +/* Physical-page limits must include any partial native page of locked VM. */ +static inline unsigned long mm_locked_vm_native_pages(const struct mm_struct *mm) +{ + unsigned long units = READ_ONCE(mm->locked_vm); + unsigned int shift = PAGE_SHIFT - MM_ACCOUNT_SHIFT; + + return (units >> shift) + !!(units & ((1UL << shift) - 1)); +} + +/* File-cache indices remain native-page units even for a 4K VMA. */ +struct vm_page_offset { + pgoff_t index; + unsigned int offset; +}; + +static inline struct vm_page_offset vm_page_offset_add(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = pos.offset + (bytes & ~PAGE_MASK); + + pos.index += (bytes >> PAGE_SHIFT) + (low >> PAGE_SHIFT); + pos.offset = low & ~PAGE_MASK; + return pos; +} + +static inline struct vm_page_offset vm_page_offset_sub(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = bytes & ~PAGE_MASK; + + pos.index -= (bytes >> PAGE_SHIFT) + (low > pos.offset); + pos.offset = (pos.offset - low) & ~PAGE_MASK; + return pos; +} + +static inline bool vm_page_offset_equal(struct vm_page_offset a, struct vm_page_offset b) +{ + return a.index == b.index && a.offset == b.offset; +} + +static inline unsigned int vma_subpage_offset(const struct vm_area_struct *vma) +{ +#ifdef CONFIG_MM_SUBPAGE + return vma->vm_subpage_offset; +#else + return 0; +#endif +} + +/* address must be >= vm_start; vm_end computes the next contiguous offset. */ +static inline struct vm_page_offset vma_page_offset_at(const struct vm_area_struct *vma, + unsigned long address) +{ + unsigned long delta = address - vma->vm_start; + unsigned long low = (delta & ~PAGE_MASK) + vma_subpage_offset(vma); + + return (struct vm_page_offset) { + .index = vma->vm_pgoff + (delta >> PAGE_SHIFT) + (low >> PAGE_SHIFT), + .offset = low & ~PAGE_MASK, + }; +} + +/* The start byte of a faulting granule must precede EOF. */ +static inline bool vma_file_offset_valid(const struct vm_area_struct *vma, + unsigned long address, loff_t size) +{ + struct vm_page_offset pos = vma_page_offset_at(vma, address); + pgoff_t end = size >> PAGE_SHIFT; + + return pos.index < end || + (pos.index == end && pos.offset < (size & ~PAGE_MASK)); +} + +static inline void vma_set_page_offset(struct vm_area_struct *vma, struct vm_page_offset pos) +{ + vma->vm_pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = pos.offset; +#endif +} + +/* Inverse lookup of a single byte; an offset outside the VMA is not clamped. */ +static inline unsigned long vma_address_at_offset(const struct vm_area_struct *vma, + struct vm_page_offset pos) +{ + unsigned long len = vma->vm_end - vma->vm_start; + unsigned long index, delta, last; + unsigned int offset = vma_subpage_offset(vma); + + if (!len || pos.index < vma->vm_pgoff || pos.offset >= PAGE_SIZE) + return -EFAULT; + index = pos.index - vma->vm_pgoff; + last = ((len - 1) >> PAGE_SHIFT) + + ((((len - 1) & ~PAGE_MASK) + offset) >> PAGE_SHIFT); + if (index > last) + return -EFAULT; + delta = (index << PAGE_SHIFT) + pos.offset; + if (delta < offset || delta - offset >= len) + return -EFAULT; + return vma->vm_start + delta - offset; +} + +#endif diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h new file mode 100644 index 0000000000000..f5b7fdcb51114 --- /dev/null +++ b/include/linux/mm_subpage.h @@ -0,0 +1,275 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MM_SUBPAGE_H +#define _LINUX_MM_SUBPAGE_H + +#include +#include +#include + +#define MM_SUBPAGE_SHIFT 12 +#define MM_SUBPAGE_SIZE (1UL << MM_SUBPAGE_SHIFT) +#define MM_SUBPAGES_PER_PAGE (PAGE_SIZE / MM_SUBPAGE_SIZE) + +struct mm_subpage_mapcounts { + unsigned int slots[MM_SUBPAGES_PER_PAGE]; +}; + +struct folio; +struct page; +struct mm_struct; +struct mm_subpage; +struct mm_subpage_pool; +struct page_ext_operations; +struct vm_area_struct; + +extern struct page_ext_operations mm_subpage_ext_ops; + +#ifdef CONFIG_MM_SUBPAGE +/* Per-native-page shmem validity; caller holds the containing folio lock. */ +unsigned long mm_subpage_shmem_missing(struct page *page); +void mm_subpage_shmem_set_missing(struct page *page, unsigned long missing); +#endif + +/* + * A pool is one caller-defined allocation/policy/charge domain. The caller + * supplies fresh, unmapped native folios using its normal NUMA and memcg + * allocation path. No mm, VMA, rmap, swap or folio-private field is repurposed. + * Fault paths transfer slot references to PTEs through the rmap helpers below. + * + * Each caller must hold a pool reference through pool operations. close() + * stops allocation and releases unused backing; it does not consume that + * reference. Existing slot references outlive close() and the originating mm. + */ +struct mm_subpage_pool *mm_subpage_pool_create(gfp_t gfp); +struct mm_subpage_pool *mm_subpage_pool_create_granule(gfp_t gfp, unsigned int shift); +void mm_subpage_pool_get(struct mm_subpage_pool *pool); +void mm_subpage_pool_put(struct mm_subpage_pool *pool); +void mm_subpage_pool_close(struct mm_subpage_pool *pool); + +/* Success consumes the folio reference. On error the caller retains it. */ +int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + gfp_t gfp); +int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int offset, gfp_t gfp); +unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool); + +/* -EAGAIN requires more compatible backing; -ESHUTDOWN means pool closed. */ +struct mm_subpage *mm_subpage_alloc(struct mm_subpage_pool *pool); +struct mm_subpage *mm_subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source); +/* Exact physical byte offset within a native folio; no other slot is substituted. */ +struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned int offset); +struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset); + +/* + * Reattach a slot to an uptodate, locked order-0 swap-cache folio, without + * zeroing or copying its bytes. Caller holds a native folio reference and + * retains it. The returned slot owns an additional reference. -EAGAIN means + * ownership changed concurrently; -ENOMEM means metadata allocation failed. + */ +struct mm_subpage *mm_subpage_restore(struct folio *folio, unsigned int offset, gfp_t gfp); +struct mm_subpage *mm_subpage_restore_granule(struct folio *folio, unsigned int offset, + unsigned int shift, gfp_t gfp); + +/* PTE lock held, one file-backed alternative leaf per call. Native mms are no-ops. */ +#ifdef CONFIG_MM_SUBPAGE +void __mm_subpage_file_map_add(phys_addr_t phys, unsigned long size); +void __mm_subpage_file_map_del(phys_addr_t phys, unsigned long size); +static inline void mm_subpage_file_map_add(struct vm_area_struct *vma, phys_addr_t phys) +{ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + __mm_subpage_file_map_add(phys, mm_page_size(vma->vm_mm)); +} +static inline void mm_subpage_file_map_del(struct vm_area_struct *vma, phys_addr_t phys) +{ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + __mm_subpage_file_map_del(phys, mm_page_size(vma->vm_mm)); +} +/* Fill 4K coverage counts; return alternative leaf count. Statistics only. */ +unsigned int mm_subpage_file_mapcounts(struct page *page, + struct mm_subpage_mapcounts *counts); +#else +static inline void mm_subpage_file_map_add(struct vm_area_struct *vma, phys_addr_t phys) { } +static inline void mm_subpage_file_map_del(struct vm_area_struct *vma, phys_addr_t phys) { } +#endif + +/* Both folios locked; preallocate destination metadata before removing PTEs. */ +#ifdef CONFIG_MM_SUBPAGE +int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst); +bool mm_subpage_migrating(struct folio *folio); +void mm_subpage_migrate_save_lazyfree(struct mm_subpage *subpage); +/* Infallible metadata reuse after prepare; caller still holds the folio lock. */ +struct mm_subpage *mm_subpage_migrate_restore(struct folio *folio, unsigned int offset); +void mm_subpage_migrate_finish(struct folio *folio); +#else +static inline int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) +{ + return 0; +} +static inline bool mm_subpage_migrating(struct folio *folio) +{ + return false; +} +static inline void mm_subpage_migrate_finish(struct folio *folio) { } +#endif + +/* + * get() requires a live reference. A mapping reference must not be put until + * its PTE is removed and TLB invalidation completes. Every pin/alias holds + * its own reference: the slot becomes reusable only after the final put. + * Every live slot reference also holds one reference to the native folio. + * copy() requires the caller to stabilise the source contents (as for COW). + * A failed data copy returns -EHWPOISON without publishing the destination. + */ +void mm_subpage_get(struct mm_subpage *subpage); +void mm_subpage_put(struct mm_subpage *subpage); +/* + * Acquire the live identity containing a 4K-aligned physical address, or NULL. + * Interior quarters of a larger identity return that same identity. + * The caller must stabilise the physical allocation, e.g. with a live PTE + * under its lock or a folio reference. A lockless walker additionally needs + * GUP's TLB/free ordering; RCU protects metadata, not arbitrary physical PFNs. + * This is not a mapping identity check: a lockless PTE reader must revalidate + * its PTE after acquisition, just as for speculative folio references. + */ +struct mm_subpage *mm_subpage_get_from_phys(phys_addr_t phys); +/* + * Optional shared sets for asynchronous consumers of native byte spans. + * add() requires the source slot identities to remain live for the call. + * The pointer being modified is exclusively owned by the caller; get() shares + * a read-only set, and mutation uses copy-on-write. Duplicate ranges add no + * references. prune() requires a stable, side-effect-free membership predicate. + * Allocation failure leaves the original set intact. put() accepts NULL. + */ +struct mm_subpage_refs; +#ifdef CONFIG_MM_SUBPAGE +struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs); +void mm_subpage_refs_put(struct mm_subpage_refs *refs); +int mm_subpage_refs_add(struct mm_subpage_refs **refs, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp); +int mm_subpage_refs_prune(struct mm_subpage_refs **refs, + bool (*keep)(const struct mm_subpage *, void *), + void *context, gfp_t gfp); +#else +static inline struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs) +{ + return refs; +} +static inline void mm_subpage_refs_put(struct mm_subpage_refs *refs) { } +static inline int mm_subpage_refs_add(struct mm_subpage_refs **refs, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp) +{ + return 0; +} +static inline int mm_subpage_refs_prune(struct mm_subpage_refs **refs, + bool (*keep)(const struct mm_subpage *, void *), void *context, gfp_t gfp) +{ + return 0; +} +#endif + +/* + * Anonymous rmap attachment: caller holds the folio lock and PTE lock, has + * prepared vma->anon_vma, and transfers an existing slot reference to its PTE. + * A folio may cover only one native linear index and anon_vma root; its slot + * offset must match the VMA's linear byte offset. Removal needs the PTE lock. + * These helpers do not install PTEs, update RSS, enqueue on LRU, or drop refs. + */ +int mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address); +int mm_subpage_add_new_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address); +void mm_subpage_remove_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma); +/* Both PTE locks and source write_protect_seq held; both mappings become R/O. */ +int mm_subpage_dup_anon_rmap(struct mm_subpage *subpage, + struct vm_area_struct *dst, struct vm_area_struct *src, unsigned long address); +/* + * Per-slot anonymous exclusivity. Setting requires the folio and PTE locks, + * and proof of a fresh private slot or an exclusive migration/swap entry. + * Sharing requires the PTE lock and fails while a typed pin exists. + */ +void mm_subpage_set_exclusive(struct mm_subpage *subpage); +bool mm_subpage_is_exclusive(const struct mm_subpage *subpage); +/* + * Folio and PTE locked; caller proves one remaining swap PTE for this slot. + * Rmap has been attached, but its PTE is not published yet. Reject old GET + * references or present aliases before recovering lost exclusive ownership. + */ +bool mm_subpage_try_reuse_swap(struct mm_subpage *subpage); +/* Present PTE plus caller lookup ref; caller proves no swapped aliases. */ +bool mm_subpage_try_reuse_anon(struct mm_subpage *subpage); +/* PTE locked, caller holds one GET; retire the PTE/TLB before dropping its ref. */ +bool mm_subpage_can_zap(struct mm_subpage *subpage); +bool mm_subpage_try_share(struct mm_subpage *subpage); +/* Folio/PTE locked, caller owns a lookup reference. */ +bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage); +bool mm_subpage_all_lazyfree(struct mm_subpage *subpage); +bool mm_subpage_discardable(struct mm_subpage *subpage); +bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage); +void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage); +/* + * Caller holds a live slot reference and its PTE lock. Success takes an + * additional slot reference plus a native FOLL_PIN reference. The pin keeps + * the exact fragment occupied after unmap; release only with subpage_unpin. + * This is not a GUP walker and does not validate user addresses/permissions. + */ +int mm_subpage_pin(struct mm_subpage *subpage); +void mm_subpage_unpin(struct mm_subpage *subpage); +unsigned int mm_subpage_pincount(const struct mm_subpage *subpage); +#ifdef CONFIG_MM_SUBPAGE +/* A referenced/pinned folio range; never infer offset from the user VA. */ +struct user_page_fragment { + struct folio *folio; + struct mm_subpage *subpage; + unsigned long offset; + unsigned int length; + bool pinned; +}; + +/* + * Takes/releases mmap_lock internally. Returns the number of records filled, + * or -errno if none. A positive partial result is valid; sum record lengths + * to advance the user address. Each record stops at the target mm's granule + * or a native physical-page boundary, whichever comes first. + * Caller owns the target mm lifetime. Release either kind of record with + * release_user_fragments(); copying a record does not acquire a reference. + * FOLL_LONGTERM pins migrate unpinnable backing before returning success. + * Device/PFN mappings are not accepted by this interface. + */ +long pin_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity); +long get_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity); +/* + * Single-record GET with a stable VMA. Caller holds mmap_lock throughout; + * this function never releases it. Returns 1 or -errno (0 for zero length). + * FOLL_NOWAIT is not supported. The returned VMA is valid until unlock. + */ +int get_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma); +/* Single-record PIN under the caller's mmap_lock, retained throughout. + * Same stable-VMA/partial-record contract as GET above. FOLL_LONGTERM and + * FOLL_NOWAIT are rejected: long-term migration can require dropping locks. + */ +int pin_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma); +/* Current mm, one aligned fragment; sparse zero pages return a negative errno. */ +int get_dump_fragment(unsigned long addr, struct user_page_fragment *fragment); +void release_user_fragments(struct user_page_fragment *fragments, + unsigned long count, bool dirty); +#endif + +int mm_subpage_mapcount(const struct mm_subpage *subpage); +/* Borrowed folio: a raw folio reference alone does not keep a slot occupied. */ +struct folio *mm_subpage_folio(const struct mm_subpage *subpage); +unsigned int mm_subpage_shift(const struct mm_subpage *subpage); +unsigned int mm_subpage_size(const struct mm_subpage *subpage); +unsigned int mm_subpage_offset(const struct mm_subpage *subpage); +phys_addr_t mm_subpage_phys(const struct mm_subpage *subpage); + +#endif diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index a308e2c23b829..7f201dda7a13b 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -558,7 +558,8 @@ FOLIO_MATCH(compound_info, _head_3); * @__page_mapping: Aliases with page->mapping. Unused for page tables. * @pt_index: Used for s390 gmap. * @pt_mm: Used for x86 pgds. - * @pt_frag_refcount: For fragmented page table tracking. Powerpc only. + * @pt_frag_refcount: For fragmented page table tracking (powerpc, arm64). + * @pt_frag_shift: Immutable arm64 fragment granule, valid through RCU release. * @pt_share_count: Used for HugeTLB PMD page table share count. * @_pt_pad_2: Padding to ensure proper alignment. * @ptl: Lock for the page table. @@ -585,7 +586,15 @@ struct ptdesc { union { pgoff_t pt_index; struct mm_struct *pt_mm; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* arm64 is 64-bit: use the spare half of this existing word. */ + struct { + atomic_t pt_frag_refcount; + u8 pt_frag_shift; + }; +#else atomic_t pt_frag_refcount; +#endif #ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING atomic_t pt_share_count; #endif @@ -905,6 +914,9 @@ struct vm_area_desc { /* Mutable fields. Populated with initial state. */ pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif struct file *vm_file; vma_flags_t vma_flags; pgprot_t page_prot; @@ -992,6 +1004,9 @@ struct vm_area_struct { /* Information about our backing store: */ unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE units */ +#ifdef CONFIG_MM_SUBPAGE + unsigned int vm_subpage_offset; /* Byte offset within vm_pgoff. */ +#endif struct file * vm_file; /* File we map to (can be NULL). */ void * vm_private_data; /* was vm_pte (shared mem) */ @@ -1281,12 +1296,16 @@ struct mm_struct { unsigned int __percpu *futex_ref; #endif - unsigned long hiwater_rss; /* High-watermark of RSS usage */ - unsigned long hiwater_vm; /* High-water virtual memory usage */ + unsigned long hiwater_rss; /* Peak RSS in user granules */ + unsigned long hiwater_vm; /* Peak VM in user granules */ - unsigned long total_vm; /* Total pages mapped */ - unsigned long locked_vm; /* Pages that have PG_mlocked set */ - atomic64_t pinned_vm; /* Refcount permanently increased */ + unsigned long total_vm; /* Mapped user granules */ +#ifdef CONFIG_MM_SUBPAGE + /* VM_ACCOUNT granules; global commitment rounds this sum once. */ + unsigned long committed_user_pages; +#endif + unsigned long locked_vm; /* Locked bytes >> MM_ACCOUNT_SHIFT */ + atomic64_t pinned_vm; /* Pinned native physical pages */ unsigned long data_vm; /* VM_WRITE & ~VM_SHARED & ~VM_STACK */ unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE & ~VM_STACK */ unsigned long stack_vm; /* VM_STACK */ diff --git a/include/linux/mman.h b/include/linux/mman.h index 389521594c69c..7b80c19f6aa5d 100644 --- a/include/linux/mman.h +++ b/include/linux/mman.h @@ -83,6 +83,17 @@ static inline void vm_unacct_memory(long pages) vm_acct_memory(-pages); } +/* VMA reservations use user granules; shmem/physical reservations use pages. */ +#ifdef CONFIG_MM_SUBPAGE +int mm_account_memory(struct mm_struct *mm, struct mm_struct *policy_mm, long pages); +void mm_acct_memory(struct mm_struct *mm, long pages); +void mm_unacct_memory(struct mm_struct *mm, unsigned long pages); +#else +#define mm_account_memory(mm, policy_mm, pages) security_vm_enough_memory_mm(policy_mm, pages) +#define mm_acct_memory(mm, pages) vm_acct_memory(pages) +#define mm_unacct_memory(mm, pages) vm_unacct_memory(pages) +#endif + /* * Allow architectures to handle additional protection and flag bits. The * overriding macros must be defined in the arch-specific asm/mman.h file. diff --git a/include/linux/mmu_context.h b/include/linux/mmu_context.h index ed3dd0f3fe194..a4260bc898205 100644 --- a/include/linux/mmu_context.h +++ b/include/linux/mmu_context.h @@ -5,6 +5,11 @@ #include #include +/* Select immutable exec geometry before allocating any page tables. */ +#ifndef arch_mm_init_exec +static inline void arch_mm_init_exec(struct mm_struct *mm, unsigned int page_shift) { } +#endif + /* Architectures that care about IRQ state in switch_mm can override this. */ #ifndef switch_mm_irqs_off # define switch_mm_irqs_off switch_mm diff --git a/include/linux/page_table_check.h b/include/linux/page_table_check.h index 12268a32e8be1..3ee1096fb0e53 100644 --- a/include/linux/page_table_check.h +++ b/include/linux/page_table_check.h @@ -12,6 +12,10 @@ extern struct static_key_true page_table_check_disabled; extern struct page_ext_operations page_table_check_ops; +#if IS_ENABLED(CONFIG_KUNIT) +int page_table_check_get_counts(struct page *page, unsigned int offset, + int *anon, int *file); +#endif void __page_table_check_zero(struct page *page, unsigned int order); void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 31a848485ad9d..9cde417795f00 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1068,10 +1068,7 @@ static inline pgoff_t folio_pgoff(const struct folio *folio) static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, const unsigned long address) { - pgoff_t pgoff; - pgoff = (address - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; - return pgoff; + return vma_page_offset_at(vma, address).index; } struct wait_page_key { diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index cdd68ed3ae1a9..f5ef44f25c52b 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -12,11 +12,18 @@ #ifdef CONFIG_MMU #include +#include #include #include #include #include +/* pgtable_t is an opaque allocation token, not necessarily struct page *. */ +#ifndef pgtable_to_page +#define pgtable_to_page(table) (table) +#define pgtable_from_page(page) (page) +#endif + #if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS #error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED @@ -155,6 +162,59 @@ static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address) */ #define pgd_offset_k(address) pgd_offset(&init_mm, (address)) +/* Explicit target-mm accessors; legacy accessors keep native geometry. */ +#ifndef pte_phys_mm +#define pte_phys_mm(mm, pte) PFN_PHYS(pte_pfn(pte)) +#endif +#ifndef phys_pte_mm +#define phys_pte_mm(mm, phys, prot) pfn_pte((phys) >> PAGE_SHIFT, prot) +#endif +#ifndef pte_pgprot_mm +#define pte_pgprot_mm(mm, pte) pte_pgprot(pte) +#endif +#ifndef pgd_offset_pgd_mm +#define pgd_offset_pgd_mm(mm, pgd, addr) pgd_offset_pgd(pgd, addr) +#endif +#ifndef p4d_offset_mm +#define p4d_offset_mm(mm, pgd, addr) p4d_offset(pgd, addr) +#endif +#ifndef pud_offset_mm +#define pud_offset_mm(mm, p4d, addr) pud_offset(p4d, addr) +#endif +#ifndef pmd_offset_mm +#define pmd_offset_mm(mm, pud, addr) pmd_offset(pud, addr) +#endif +#ifndef __pte_map_mm +#define __pte_map_mm(mm, pmd, addr) __pte_map(pmd, addr) +#endif +#ifndef pte_offset_kernel_mm +#define pte_offset_kernel_mm(mm, pmd, addr) pte_offset_kernel(pmd, addr) +#endif + +#ifndef pud_pgtable_mm +#define pud_pgtable_mm(mm, pud) pud_pgtable(pud) +#endif + +#ifndef pmd_size_mm +#define pmd_shift_mm(mm) PMD_SHIFT +#define pud_shift_mm(mm) PUD_SHIFT +#define p4d_shift_mm(mm) P4D_SHIFT +#define pmd_size_mm(mm) PMD_SIZE +#define pud_size_mm(mm) PUD_SIZE +#define p4d_size_mm(mm) P4D_SIZE +#define pgd_size_mm(mm) PGDIR_SIZE +#define pmd_mask_mm(mm) PMD_MASK +#define pud_mask_mm(mm) PUD_MASK +#define p4d_mask_mm(mm) P4D_MASK +#define pgd_mask_mm(mm) PGDIR_MASK +#endif + +#ifndef pte_table_bytes_mm +#define pte_table_bytes_mm(mm) (PTRS_PER_PTE * sizeof(pte_t)) +#define pmd_table_bytes_mm(mm) (PTRS_PER_PMD * sizeof(pmd_t)) +#define pud_table_bytes_mm(mm) (PTRS_PER_PUD * sizeof(pud_t)) +#endif + /* * In many cases it is known that a virtual address is mapped at PMD or PTE * level, so instead of traversing all the page table levels, we can get a @@ -388,6 +448,10 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) #define pte_next_pfn(pte) pte_advance_pfn(pte, 1) +#ifndef pte_advance_pfn_mm +#define pte_advance_pfn_mm(mm, pte, nr) pte_advance_pfn(pte, nr) +#endif + #ifndef set_ptes /** * set_ptes - Map consecutive pages to a contiguous range of addresses. @@ -1465,6 +1529,13 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) * and any p?d_bad entries - reporting the error before resetting to none. * Do the tests inline, but report and clear the bad entry in mm/memory.c. */ +#ifndef pgd_addr_end_mm +#define pgd_addr_end_mm(mm, addr, end) pgd_addr_end(addr, end) +#define p4d_addr_end_mm(mm, addr, end) p4d_addr_end(addr, end) +#define pud_addr_end_mm(mm, addr, end) pud_addr_end(addr, end) +#define pmd_addr_end_mm(mm, addr, end) pmd_addr_end(addr, end) +#endif + void pgd_clear_bad(pgd_t *); #ifndef __PAGETABLE_P4D_FOLDED @@ -1492,6 +1563,25 @@ static inline int pgd_none_or_clear_bad(pgd_t *pgd) return 0; } +#ifndef p4d_none_mm +#define p4d_none_mm(mm, p4d) p4d_none(p4d) +#define p4d_bad_mm(mm, p4d) p4d_bad(p4d) +#define p4d_present_mm(mm, p4d) p4d_present(p4d) +#define p4d_clear_mm(mm, p4d) p4d_clear(p4d) +#endif + +static inline int p4d_none_or_clear_bad_mm(struct mm_struct *mm, p4d_t *p4d) +{ + if (p4d_none_mm(mm, *p4d)) + return 1; + if (unlikely(p4d_bad_mm(mm, *p4d))) { + p4d_ERROR(*p4d); + p4d_clear_mm(mm, p4d); + return 1; + } + return 0; +} + static inline int p4d_none_or_clear_bad(p4d_t *p4d) { if (p4d_none(*p4d)) @@ -1979,11 +2069,15 @@ static inline int is_zero_pfn(unsigned long pfn) #define zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr)) #else +#ifndef ARCH_ZERO_PAGE_SIZE +#define ARCH_ZERO_PAGE_SIZE PAGE_SIZE +#endif + static inline int is_zero_pfn(unsigned long pfn) { extern unsigned long zero_page_pfn; - return pfn == zero_page_pfn; + return pfn - zero_page_pfn < (ARCH_ZERO_PAGE_SIZE >> PAGE_SHIFT); } static inline unsigned long zero_pfn(unsigned long addr) @@ -1993,7 +2087,7 @@ static inline unsigned long zero_pfn(unsigned long addr) return zero_page_pfn; } -extern uint8_t empty_zero_page[PAGE_SIZE]; +extern uint8_t empty_zero_page[ARCH_ZERO_PAGE_SIZE]; extern struct page *__zero_page; static inline struct page *_zero_page(unsigned long addr) diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 8dc0871e5f001..2bc2c986eb76e 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -40,6 +40,10 @@ struct anon_vma { * anon_vma if they are the last user on release */ atomic_t refcount; +#ifdef CONFIG_MM_SUBPAGE + /* Immutable base-page requirement shared by this anonymous lineage. */ + unsigned int min_folio_order; +#endif /* * Count of child anon_vmas. Equals to the count of all anon_vmas that @@ -398,29 +402,40 @@ static __always_inline void __folio_rmap_sanity_checks(const struct folio *folio } /* - * rmap interfaces called when adding or removing pte of page + * Single-PTE interfaces cover the complete native-page span of a user leaf. + * Multi-page interfaces below retain explicit native physical-page counts. */ void folio_move_anon_rmap(struct folio *, struct vm_area_struct *); void folio_add_anon_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *, unsigned long address, rmap_t flags); -#define folio_add_anon_rmap_pte(folio, page, vma, address, flags) \ - folio_add_anon_rmap_ptes(folio, page, 1, vma, address, flags) +static inline void folio_add_anon_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma, unsigned long address, rmap_t flags) +{ + folio_add_anon_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), + vma, address, flags); +} void folio_add_anon_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *, unsigned long address, rmap_t flags); void folio_add_new_anon_rmap(struct folio *, struct vm_area_struct *, unsigned long address, rmap_t flags); void folio_add_file_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *); -#define folio_add_file_rmap_pte(folio, page, vma) \ - folio_add_file_rmap_ptes(folio, page, 1, vma) +static inline void folio_add_file_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma) +{ + folio_add_file_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), vma); +} void folio_add_file_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *); void folio_add_file_rmap_pud(struct folio *, struct page *, struct vm_area_struct *); void folio_remove_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *); -#define folio_remove_rmap_pte(folio, page, vma) \ - folio_remove_rmap_ptes(folio, page, 1, vma) +static inline void folio_remove_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma) +{ + folio_remove_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), vma); +} void folio_remove_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *); void folio_remove_rmap_pud(struct folio *, struct page *, @@ -541,7 +556,8 @@ static inline void folio_dup_file_rmap_ptes(struct folio *folio, static __always_inline void folio_dup_file_rmap_pte(struct folio *folio, struct page *page, struct vm_area_struct *dst_vma) { - __folio_dup_file_rmap(folio, page, 1, dst_vma, PGTABLE_LEVEL_PTE); + __folio_dup_file_rmap(folio, page, mm_pte_native_pages(dst_vma->vm_mm), + dst_vma, PGTABLE_LEVEL_PTE); } /** @@ -665,8 +681,8 @@ static __always_inline int folio_try_dup_anon_rmap_pte(struct folio *folio, struct page *page, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) { - return __folio_try_dup_anon_rmap(folio, page, 1, dst_vma, src_vma, - PGTABLE_LEVEL_PTE); + return __folio_try_dup_anon_rmap(folio, page, mm_pte_native_pages(src_vma->vm_mm), + dst_vma, src_vma, PGTABLE_LEVEL_PTE); } /** @@ -708,13 +724,16 @@ static inline int folio_try_dup_anon_rmap_pmd(struct folio *folio, static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, struct page *page, int nr_pages, enum pgtable_level level) { + int i, exclusive_pages = level == PGTABLE_LEVEL_PTE ? nr_pages : 1; + VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); VM_WARN_ON_FOLIO(!PageAnonExclusive(page), folio); __folio_rmap_sanity_checks(folio, page, nr_pages, level); /* device private folios cannot get pinned via GUP. */ if (unlikely(folio_is_device_private(folio))) { - ClearPageAnonExclusive(page); + for (i = 0; i < exclusive_pages; i++) + ClearPageAnonExclusive(page + i); return 0; } @@ -766,7 +785,8 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, if (unlikely(folio_maybe_dma_pinned(folio))) return -EBUSY; - ClearPageAnonExclusive(page); + for (i = 0; i < exclusive_pages; i++) + ClearPageAnonExclusive(page + i); /* * This is conceptually a smp_wmb() paired with the smp_rmb() in @@ -782,7 +802,8 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, * mapped by a PTE possibly shared to prepare * for KSM or temporary unmapping * @folio: The folio to share a mapping of - * @page: The mapped exclusive page + * @page: The first native page covered by the exclusive user PTE + * @vma: The VMA defining the base user PTE granule * * The caller needs to hold the page table lock and has to have the page table * entries cleared/invalidated. @@ -799,9 +820,10 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, * -EBUSY otherwise. */ static inline int folio_try_share_anon_rmap_pte(struct folio *folio, - struct page *page) + struct page *page, struct vm_area_struct *vma) { - return __folio_try_share_anon_rmap(folio, page, 1, PGTABLE_LEVEL_PTE); + return __folio_try_share_anon_rmap(folio, page, mm_pte_native_pages(vma->vm_mm), + PGTABLE_LEVEL_PTE); } /** @@ -855,12 +877,23 @@ struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr, #define PVMW_SYNC (1 << 0) /* Look for migration entries rather than present PTEs */ #define PVMW_MIGRATION (1 << 1) +/* Match one present subpage PTE at address, including its physical offset. */ +#define PVMW_SUBPAGE (1 << 2) /* Result flags */ /* The page is mapped across page table boundary */ #define PVMW_PGTABLE_CROSSED (1 << 16) +/* One reverse-map callback, optionally restricted to one physical user leaf. */ +struct rmap_walk_range { + unsigned long address; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; + bool subpage; +#endif +}; + struct page_vma_mapped_walk { unsigned long pfn; unsigned long nr_pages; @@ -871,6 +904,10 @@ struct page_vma_mapped_walk { pte_t *pte; spinlock_t *ptl; unsigned int flags; +#ifdef CONFIG_MM_SUBPAGE + /* PVMW_SUBPAGE: byte offset within pfn; nr_pages must be one. */ + unsigned int subpage_offset; +#endif }; #define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \ @@ -883,6 +920,32 @@ struct page_vma_mapped_walk { .flags = _flags, \ } +static inline struct page_vma_mapped_walk folio_rmap_walk_init( + struct folio *folio, struct vm_area_struct *vma, + struct rmap_walk_range range, unsigned int flags) +{ + struct page_vma_mapped_walk walk = { + .pfn = folio_pfn(folio), + .nr_pages = folio_nr_pages(folio), + .pgoff = folio_pgoff(folio), + .vma = vma, + .address = range.address, + .flags = flags, + }; + +#ifdef CONFIG_MM_SUBPAGE + if (range.subpage) { + walk.flags |= PVMW_SUBPAGE; + walk.subpage_offset = range.subpage_offset; + } +#endif + return walk; +} + +#define DEFINE_FOLIO_RMAP_WALK(name, _folio, _vma, _range, _flags) \ + struct page_vma_mapped_walk name = \ + folio_rmap_walk_init(_folio, _vma, _range, _flags) + static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw) { /* HugeTLB pte is set to the relevant page table entry without pte_mapped. */ @@ -943,7 +1006,7 @@ void remove_migration_ptes(struct folio *src, struct folio *dst, * arg: passed to rmap_one() and invalid_vma() * try_lock: bail out if the rmap lock is contended * contended: indicate the rmap traversal bailed out due to lock contention - * rmap_one: executed on each vma where page is mapped + * rmap_one: executed for each mapped VMA range (possibly one user leaf) * done: for checking traversing termination condition * anon_lock: for getting anon_lock by optimized way rather than default * invalid_vma: for skipping uninterested vma @@ -957,7 +1020,7 @@ struct rmap_walk_control { * Otherwise, return true. */ bool (*rmap_one)(struct folio *folio, struct vm_area_struct *vma, - unsigned long addr, void *arg); + struct rmap_walk_range range, void *arg); int (*done)(struct folio *folio); struct anon_vma *(*anon_lock)(const struct folio *folio, struct rmap_walk_control *rwc); @@ -966,6 +1029,15 @@ struct rmap_walk_control { void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc); void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc); +#ifdef CONFIG_MM_SUBPAGE +unsigned int folio_anon_min_order(struct folio *folio); +#else +static inline unsigned int folio_anon_min_order(struct folio *folio) +{ + return 0; +} +#endif + struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio, struct rmap_walk_control *rwc); diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 95d0040df5841..1925bb991b793 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -14,6 +14,12 @@ * Routines for handling mm_structs */ extern struct mm_struct *mm_alloc(void); +extern struct mm_struct *mm_alloc_exec(void); +extern struct mm_struct *mm_alloc_exec_page_shift(unsigned int page_shift); +int arch_prctl_exec_page_size_get(struct task_struct *task); +int arch_prctl_exec_page_size_set(struct task_struct *task, unsigned long size); +int arch_prctl_default_page_size_get(struct task_struct *task); +int arch_prctl_default_page_size_set(struct task_struct *task, unsigned long size); /** * mmgrab() - Pin a &struct mm_struct. diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 584ae88b435e9..7af2aeb81e41d 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -194,7 +194,7 @@ struct signal_struct { unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw; unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt; unsigned long inblock, oublock, cinblock, coublock; - unsigned long maxrss, cmaxrss; + unsigned long maxrss, cmaxrss; /* kB, independent of each mm granule */ struct task_io_accounting ioac; /* diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 69b0177da1564..bf3a012954700 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -32,6 +32,8 @@ struct swap_iocb; * isn't directly visible to userspace. */ #define SHMEM_F_MAPPING_FROZEN BIT(2) +/* This inode has used sub-native shared-page validity. */ +#define SHMEM_F_UFFD_SUBPAGE BIT(3) struct shmem_inode_info { spinlock_t lock; @@ -52,6 +54,10 @@ struct shmem_inode_info { pgoff_t fallocend; /* highest fallocate endindex */ unsigned int fsflags; /* for FS_IOC_[SG]ETFLAGS */ atomic_t stop_eviction; /* hold when working on inode */ +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + /* Swapped missing 4K units, keyed by native cache index. */ + struct xarray uffd_missing; +#endif #ifdef CONFIG_TMPFS_QUOTA struct dquot __rcu *i_dquot[MAXQUOTAS]; #endif @@ -113,13 +119,56 @@ extern unsigned long shmem_get_unmapped_area(struct file *, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); extern int shmem_lock(struct file *file, int lock, struct ucounts *ucounts); #ifdef CONFIG_SHMEM +struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, struct inode *inode, + pgoff_t index, unsigned int order); bool shmem_mapping(const struct address_space *mapping); #else +static inline struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, + struct inode *inode, pgoff_t index, unsigned int order) +{ + return ERR_PTR(-EOPNOTSUPP); +} static inline bool shmem_mapping(const struct address_space *mapping) { return false; } #endif /* CONFIG_SHMEM */ +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +/* Return a locked cache folio and caller reference; new backing is all missing. */ +struct folio *shmem_uffd_prepare_folio(struct vm_area_struct *vma, + unsigned long address, bool *new); +void shmem_uffd_abort_folio(struct folio *folio, struct vm_area_struct *vma, bool new); +bool shmem_uffd_has_missing_native_page(struct folio *folio); +bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length); +/* Resident masks require no allocation; inode metadata persists only in swap. */ +void shmem_uffd_copy_page(struct page *dst, struct page *src); +void shmem_uffd_migrate(struct folio *dst, struct folio *src); +/* Locked, uptodate cache folio. */ +unsigned long shmem_uffd_missing_mask(struct folio *folio, pgoff_t index); +int shmem_uffd_set_missing(struct folio *folio, pgoff_t index, + unsigned long missing); +/* Clear populated units; no allocation. Caller holds the folio lock. */ +void shmem_uffd_mark_present(struct folio *folio, pgoff_t index, + unsigned long present); +/* Mark exactly the byte range exposed by a mapping; folio lock held. */ +void shmem_uffd_mark_mapped(struct folio *folio, unsigned long offset, + unsigned long length); +/* i_pages lock held; deletion, not replacement by swap or migration. */ +void shmem_uffd_forget(struct address_space *mapping, pgoff_t start, + unsigned long nr); +#else +static inline void shmem_uffd_copy_page(struct page *dst, struct page *src) { } +static inline void shmem_uffd_migrate(struct folio *dst, struct folio *src) { } +static inline bool shmem_uffd_has_missing_native_page(struct folio *folio) { return false; } +static inline bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length) { return false; } +static inline void shmem_uffd_mark_mapped(struct folio *folio, + unsigned long offset, unsigned long length) { } +static inline void shmem_uffd_forget(struct address_space *mapping, + pgoff_t start, unsigned long nr) { } +#endif + void shmem_unlock_mapping(struct address_space *mapping); struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); @@ -160,6 +209,8 @@ static inline void shmem_uncharge(struct inode *inode, long pages) { } #endif +unsigned long shmem_partial_swap_usage_bytes(struct address_space *mapping, + u64 start, u64 end); extern unsigned long shmem_partial_swap_usage(struct address_space *mapping, pgoff_t start, pgoff_t end); diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 937823856de59..30406f8cee116 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -41,6 +41,8 @@ struct sk_msg_sg { }; /* UAPI in filter.c depends on struct sk_msg_sg being first element. */ +struct mm_subpage_refs; + struct sk_msg { struct sk_msg_sg sg; void *data; @@ -49,6 +51,9 @@ struct sk_msg { u32 cork_bytes; u32 flags; struct sk_buff *skb; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage_refs *subpage_refs; +#endif struct sock *sk_redir; struct sock *sk; struct list_head list; @@ -124,6 +129,33 @@ struct sk_psock { struct rcu_work rwork; }; +#ifdef CONFIG_MM_SUBPAGE +int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len); +void sk_msg_subpages_get(struct sk_msg *msg); +void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src); +void sk_msg_subpages_release(struct sk_msg *msg); +static inline void sk_msg_subpages_transfer(struct sk_msg *dst, struct sk_msg *src) +{ + dst->subpage_refs = src->subpage_refs; + src->subpage_refs = NULL; +} +#else +static inline int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len) +{ + return 0; +} +static inline void sk_msg_subpages_get(struct sk_msg *msg) { } +static inline void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src) { } +static inline void sk_msg_subpages_release(struct sk_msg *msg) { } +static inline void sk_msg_subpages_transfer(struct sk_msg *dst, struct sk_msg *src) { } +#endif + +static inline void sk_msg_subpages_drop_empty(struct sk_msg *msg) +{ + if (!msg->sg.size) + sk_msg_subpages_release(msg); +} + int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, int elem_first_coalesce); int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, @@ -259,10 +291,14 @@ static inline void sk_msg_compute_data_pointers(struct sk_msg *msg) } } -static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, +static inline int sk_msg_page_add(struct sk_msg *msg, struct page *page, u32 len, u32 offset) { struct scatterlist *sge; + int ret = sk_msg_subpages_add(msg, page, offset, len); + + if (ret) + return ret; get_page(page); sge = sk_msg_elem(msg, msg->sg.end); @@ -272,6 +308,7 @@ static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, __set_bit(msg->sg.end, msg->sg.copy); msg->sg.size += len; sk_msg_iter_next(msg, end); + return 0; } static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state) @@ -420,6 +457,7 @@ static inline bool sk_psock_queue_empty(const struct sk_psock *psock) static inline void kfree_sk_msg(struct sk_msg *msg) { + sk_msg_subpages_release(msg); if (msg->skb) consume_skb(msg->skb); kfree(msg); diff --git a/include/linux/swap.h b/include/linux/swap.h index 7a09df6977a5f..4e9136d3ae3f6 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -228,7 +228,7 @@ enum { */ #define SWAP_ENTRY_INVALID 0 -#ifdef CONFIG_THP_SWAP +#if defined(CONFIG_THP_SWAP) || defined(CONFIG_MM_SUBPAGE) #define SWAP_NR_ORDERS (PMD_ORDER + 1) #else #define SWAP_NR_ORDERS 1 @@ -443,6 +443,7 @@ extern int swp_swapcount(swp_entry_t entry); struct backing_dev_info; extern struct swap_info_struct *get_swap_device(swp_entry_t entry); sector_t swap_folio_sector(struct folio *folio); +sector_t swap_extent_sector(swp_entry_t entry, unsigned long *nr_pages); /* * If there is an existing swap slot reference (swap entry) and the caller @@ -455,6 +456,13 @@ sector_t swap_folio_sector(struct folio *folio); * a swap count > 1. See comments of folio_*_swap helpers for more info. */ int swap_dup_entry_direct(swp_entry_t entry); +int swap_dup_entries_direct(swp_entry_t entry, unsigned int nr); +#ifdef CONFIG_MM_SUBPAGE +/* Offset is bytes within one native swap entry; caller owns a PTE/cache ref. */ +int swap_subpage_dup(swp_entry_t entry, unsigned int offset); +void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache); +unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset); +#endif void swap_put_entries_direct(swp_entry_t entry, int nr); /* @@ -498,11 +506,27 @@ static inline void free_swap_cache(struct folio *folio) { } +#ifdef CONFIG_MM_SUBPAGE +static inline int swap_subpage_dup(swp_entry_t entry, unsigned int offset) +{ + return -EINVAL; +} +static inline void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache) { } +static inline unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset) +{ + return 0; +} +#endif static inline int swap_dup_entry_direct(swp_entry_t ent) { return 0; } +static inline int swap_dup_entries_direct(swp_entry_t ent, unsigned int nr) +{ + return 0; +} + static inline void swap_put_entries_direct(swp_entry_t ent, int nr) { } diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 4fb7291f54b62..29beb44f370fe 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1261,6 +1261,11 @@ static inline int ksys_fadvise64_64(int fd, loff_t offset, loff_t len, unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long fd, unsigned long pgoff); +#ifdef CONFIG_MM_SUBPAGE +unsigned long ksys_mmap_user(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long offset); +#endif + ssize_t ksys_readahead(int fd, loff_t offset, size_t count); int ksys_ipc(unsigned int call, int first, unsigned long second, unsigned long third, void __user * ptr, long fifth); diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index f548fea2adec8..237b530ed600f 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -192,6 +192,7 @@ struct uprobes_state { }; typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr, + unsigned int page_offset, uprobe_opcode_t *insn, int nbytes, void *data); extern void __init uprobes_init(void); diff --git a/include/net/tls.h b/include/net/tls.h index ebd2550280ae2..79d358a5fe887 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -34,6 +34,7 @@ #ifndef _TLS_OFFLOAD_H #define _TLS_OFFLOAD_H +#include #include #include #include @@ -139,6 +140,9 @@ struct tls_sw_context_rx { struct tls_strparser strp; atomic_t decrypt_pending; +#ifdef CONFIG_MM_SUBPAGE + struct llist_head subpage_cleanup; +#endif struct sk_buff_head async_hold; struct wait_queue_head wq; }; @@ -148,6 +152,9 @@ struct tls_record_info { u32 end_seq; int len; int num_frags; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage_refs *subpage_refs; +#endif skb_frag_t frags[MAX_SKB_FRAGS]; }; diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h index cd7920c81f85b..fefbf65d61441 100644 --- a/include/trace/events/kmem.h +++ b/include/trace/events/kmem.h @@ -449,7 +449,7 @@ TRACE_EVENT(rss_stat, __entry->curr = current->mm == mm && !(current->flags & PF_KTHREAD); __entry->member = member; __entry->size = (percpu_counter_sum_positive(&mm->rss_stat[member]) - << PAGE_SHIFT); + << mm_page_shift(mm)); ), TP_printk("mm_id=%u curr=%d type=%s size=%ldB", diff --git a/include/trace/events/mmap.h b/include/trace/events/mmap.h index ee2843a5daef1..46df97560969d 100644 --- a/include/trace/events/mmap.h +++ b/include/trace/events/mmap.h @@ -26,7 +26,7 @@ TRACE_EVENT(vm_unmapped_area, TP_fast_assign( __entry->addr = addr; - __entry->total_vm = current->mm->total_vm; + __entry->total_vm = mm_pages_to_native(current->mm, current->mm->total_vm); __entry->flags = info->flags; __entry->length = info->length; __entry->low_limit = info->low_limit; diff --git a/include/trace/events/oom.h b/include/trace/events/oom.h index 9f0a5d1482c44..d5321b6f01ffa 100644 --- a/include/trace/events/oom.h +++ b/include/trace/events/oom.h @@ -7,7 +7,6 @@ #include #include -#define PG_COUNT_TO_KB(x) ((x) << (PAGE_SHIFT - 10)) TRACE_EVENT(oom_score_adj_update, @@ -93,10 +92,10 @@ TRACE_EVENT(mark_victim, TP_fast_assign( __entry->pid = task->pid; __assign_str(comm); - __entry->total_vm = PG_COUNT_TO_KB(task->mm->total_vm); - __entry->anon_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_ANONPAGES)); - __entry->file_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_FILEPAGES)); - __entry->shmem_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_SHMEMPAGES)); + __entry->total_vm = mm_pages_to_kb(task->mm, task->mm->total_vm); + __entry->anon_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_ANONPAGES)); + __entry->file_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_FILEPAGES)); + __entry->shmem_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_SHMEMPAGES)); __entry->uid = uid; __entry->pgtables = mm_pgtables_bytes(task->mm) >> 10; __entry->oom_score_adj = task->signal->oom_score_adj; diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index 1a06e7feb7dc9..6e08471fbe822 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -421,4 +421,22 @@ struct prctl_mm_map { # define PR_CFI_DISABLE _BITUL(1) # define PR_CFI_LOCK _BITUL(2) +/* + * Aurora experimental ABI: private numbers, not upstream assignments. + * SET_EXEC selects the calling thread's next successful exec granule. + * Zero requests native pages; with an alternative DEFAULT in effect this + * installs an explicit one-shot native override. GET_EXEC returns zero when + * no override remains, or its size in bytes. Successful exec consumes it. + * SET_DEFAULT selects the fallback for subsequent execs; zero restores native. + * GET_DEFAULT returns the alternative size in bytes, or zero for native. + * Fork inherits both policies; exec preserves DEFAULT. Failed exec preserves + * both. Neither setter changes the live address space or other threads. + * AArch32 always selects 4K when supported, otherwise the native granule, + * independently of these preferences. Sizes are capability-checked. + */ +#define PR_AURORA_SET_EXEC_PAGE_SIZE 0x41555001 +#define PR_AURORA_GET_EXEC_PAGE_SIZE 0x41555002 +#define PR_AURORA_SET_DEFAULT_PAGE_SIZE 0x41555003 +#define PR_AURORA_GET_DEFAULT_PAGE_SIZE 0x41555004 + #endif /* _LINUX_PRCTL_H */ diff --git a/io_uring/memmap.c b/io_uring/memmap.c index 4f9b439319c46..72e123757bea4 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -289,7 +289,10 @@ static int io_region_mmap(struct io_ring_ctx *ctx, unsigned long nr_pages = min(mr->nr_pages, max_pages); vm_flags_set(vma, VM_DONTEXPAND); - return vm_insert_pages(vma, vma->vm_start, mr->pages, &nr_pages); + if (mm_page_size(vma->vm_mm) == PAGE_SIZE) + return vm_insert_pages(vma, vma->vm_start, mr->pages, &nr_pages); + return vm_insert_pages_range(vma, vma->vm_start, mr->pages, nr_pages, 0, + min_t(unsigned long, vma->vm_end - vma->vm_start, nr_pages << PAGE_SHIFT)); } __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma) @@ -301,6 +304,9 @@ __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma) struct io_mapped_region *region; void *ptr; + if (vma_subpage_offset(vma)) + return -EINVAL; + guard(mutex)(&ctx->mmap_lock); ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff); diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 2dfc26233e286..33cc1e2688480 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -762,6 +763,95 @@ bool io_check_coalesce_buffer(struct page **page_array, int nr_pages, return true; } +#ifdef CONFIG_MM_SUBPAGE +struct io_fragment_buffer { + unsigned int count; + struct user_page_fragment fragments[]; +}; + +static void io_release_fragment_buffer(void *priv) +{ + struct io_fragment_buffer *buffer = priv; + + release_user_fragments(buffer->fragments, buffer->count, false); + kvfree(buffer); +} + +static struct io_rsrc_node *io_register_fragment_buffer(struct io_ring_ctx *ctx, + struct iovec *iov) +{ + unsigned long size = mm_page_size(current->mm); + unsigned long first = (unsigned long)iov->iov_base & (size - 1); + unsigned int count = DIV_ROUND_UP(first + iov->iov_len, size); + struct io_fragment_buffer *buffer; + struct io_mapped_ubuf *imu = NULL; + struct io_rsrc_node *node; + unsigned long charged = 0; + unsigned int i; + struct xarray seen; + long nr; + int ret = -ENOMEM; + + node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); + if (!node) + return ERR_PTR(-ENOMEM); + buffer = kvmalloc(struct_size(buffer, fragments, count), GFP_KERNEL_ACCOUNT); + if (!buffer) + goto free_node; + buffer->count = 0; + nr = pin_user_fragments_remote(current->mm, (unsigned long)iov->iov_base, + iov->iov_len, FOLL_WRITE | FOLL_LONGTERM, buffer->fragments, count); + if (nr > 0) + buffer->count = nr; + if (nr != count) { + ret = nr < 0 ? nr : -EFAULT; + goto free_buffer; + } + imu = io_alloc_imu(ctx, count); + if (!imu) + goto free_buffer; + imu->nr_bvecs = count; + imu->acct_pages = 0; + /* Several user leaves can share native backing, including aliases. */ + xa_init(&seen); + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &buffer->fragments[i]; + + ret = xa_insert(&seen, folio_pfn(f->folio), xa_mk_value(1), GFP_KERNEL); + if (!ret) + charged += folio_nr_pages(f->folio); + else if (ret != -EBUSY) + break; + bvec_set_page(&imu->bvec[i], folio_page(f->folio, f->offset >> PAGE_SHIFT), + f->length, offset_in_page(f->offset)); + } + xa_destroy(&seen); + if (ret && ret != -EBUSY) + goto free_imu; + ret = io_account_mem(ctx->user, ctx->mm_account, charged); + if (ret) + goto free_imu; + imu->acct_pages = charged; + imu->ubuf = (unsigned long)iov->iov_base; + imu->len = iov->iov_len; + imu->folio_shift = mm_page_shift(current->mm); + imu->release = io_release_fragment_buffer; + imu->priv = buffer; + imu->flags = 0; + imu->dir = IO_IMU_DEST | IO_IMU_SOURCE; + refcount_set(&imu->refs, 1); + node->buf = imu; + return node; +free_imu: + io_free_imu(ctx, imu); +free_buffer: + io_release_fragment_buffer(buffer); +free_node: + io_cache_free(&ctx->node_cache, node); + return ERR_PTR(ret); +} +#endif + static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, struct page **last_hpage) @@ -787,6 +877,11 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, if (ret) return ERR_PTR(ret); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) < PAGE_SIZE) + return io_register_fragment_buffer(ctx, iov); +#endif + node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); if (!node) return ERR_PTR(-ENOMEM); @@ -1095,6 +1190,14 @@ static int io_import_fixed(int ddir, struct iov_iter *iter, bvec += seg_skip; offset &= folio_mask; } +#ifdef CONFIG_MM_SUBPAGE + if (imu->folio_shift < PAGE_SHIFT) { + /* Physical fragment offsets are independent of the user address. */ + size_t first = min_t(size_t, len, bvec->bv_len - offset); + + nr_segs = 1 + DIV_ROUND_UP(len - first, 1UL << imu->folio_shift); + } else +#endif nr_segs = (offset + len + bvec->bv_offset + folio_mask) >> imu->folio_shift; iov_iter_bvec(iter, ddir, bvec, nr_segs, len); iter->iov_offset = offset; @@ -1356,6 +1459,27 @@ static int io_vec_fill_bvec(int ddir, struct iov_iter *iter, return -EOVERFLOW; offset = buf_addr - imu->ubuf; +#ifdef CONFIG_MM_SUBPAGE + if (imu->folio_shift < PAGE_SHIFT) { + unsigned int index; + unsigned long first = imu->ubuf & folio_mask; + + offset += first; + index = offset >> imu->folio_shift; + offset &= folio_mask; + if (!index) + offset -= first; + src_bvec = imu->bvec + index; + for (; iov_len; offset = 0, bvec_idx++, src_bvec++) { + size_t bytes = min_t(size_t, iov_len, src_bvec->bv_len - offset); + + bvec_set_page(&res_bvec[bvec_idx], src_bvec->bv_page, + bytes, src_bvec->bv_offset + offset); + iov_len -= bytes; + } + continue; + } +#endif /* * Only the first bvec can have non zero bv_offset, account it * here and work with full folios below. diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 6fc6a4b672b55..9abd4972966ff 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -24,6 +24,11 @@ select_bucket(struct bpf_local_storage_map *smap, return &smap->buckets[hash_ptr(local_storage, smap->bucket_log)]; } +static u32 bpf_selem_size(const struct bpf_local_storage_map *smap) +{ + return smap->elem_size + bpf_obj_uptr_extra_size(smap->map.record); +} + static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size) { struct bpf_map *map = &smap->map; @@ -72,10 +77,10 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, { struct bpf_local_storage_elem *selem; - if (mem_charge(smap, owner, smap->elem_size)) + if (mem_charge(smap, owner, bpf_selem_size(smap))) return NULL; - selem = bpf_map_kmalloc_nolock(&smap->map, smap->elem_size, + selem = bpf_map_kmalloc_nolock(&smap->map, bpf_selem_size(smap), __GFP_ZERO, NUMA_NO_NODE); if (selem) { @@ -91,7 +96,7 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, return selem; } - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); return NULL; } @@ -178,7 +183,7 @@ static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem * bool free_local_storage, bool pin_owner) { void *owner = local_storage->owner; - u32 uncharge = smap->elem_size; + u32 uncharge = bpf_selem_size(smap); if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) == SDATA(selem)) @@ -204,7 +209,7 @@ static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem * /* local_storage->lock must be held and selem->local_storage == local_storage. * The caller must ensure selem->smap is still valid to be - * dereferenced for its smap->elem_size and smap->cache_idx. + * dereferenced for its element size and smap->cache_idx. */ static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, struct bpf_local_storage_elem *selem, @@ -234,7 +239,7 @@ void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage, struct bpf_local_storage_map *smap; smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); - local_storage->mem_charge += smap->elem_size; + local_storage->mem_charge += bpf_selem_size(smap); RCU_INIT_POINTER(selem->local_storage, local_storage); hlist_add_head_rcu(&selem->snode, &local_storage->list); @@ -577,7 +582,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, err = bpf_local_storage_alloc(owner, smap, selem); if (err) { bpf_selem_free(selem, true); - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); return ERR_PTR(err); } @@ -661,7 +666,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, free_selem: bpf_selem_free_list(&old_selem_free_list, false); if (alloc_selem) { - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); bpf_selem_free(alloc_selem, true); } return err ? ERR_PTR(err) : SDATA(selem); @@ -862,7 +867,7 @@ void bpf_local_storage_map_free(struct bpf_map *map, * exit immediately. * * However, while freeing the storage one still needs to access the - * smap->elem_size to do the uncharging in + * bpf_selem_size(smap) to do the uncharging in * bpf_selem_unlink_storage_nolock(). * * Hence, wait another rcu grace period for the storage to be freed. diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 96d1b797e1523..c084c1cad03d8 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include #include @@ -164,6 +166,43 @@ static void maybe_wait_bpf_programs(struct bpf_map *map) synchronize_rcu_expedited(); } +#ifdef CONFIG_MM_SUBPAGE +struct bpf_uptr_pins { + struct work_struct work; + u32 count; + struct user_page_fragment fragments[]; +}; + +static void bpf_uptr_release_work(struct work_struct *work) +{ + struct bpf_uptr_pins *pins = container_of(work, struct bpf_uptr_pins, work); + + release_user_fragments(pins->fragments, pins->count, true); + kfree(pins); +} +#endif + +/* May run from an RCU callback. The bundle owns its pins independently of + * the map, task and value, so dirty release can wait for folio locks in work. + */ +static bool bpf_obj_release_uptr_pins(const struct btf_record *rec, void *obj) +{ +#ifdef CONFIG_MM_SUBPAGE + struct bpf_uptr_pins **owner, *pins; + + if (!bpf_obj_uptr_extra_size(rec)) + return false; + owner = obj + rec->uptr_pins_off; + pins = *owner; + if (pins) { + *owner = NULL; + queue_work(system_unbound_wq, &pins->work); + return true; + } +#endif + return false; +} + static void unpin_uptr_kaddr(void *kaddr) { if (kaddr) @@ -176,6 +215,9 @@ static void __bpf_obj_unpin_uptrs(struct btf_record *rec, u32 cnt, void *obj) void **uptr_addr; int i; + if (bpf_obj_release_uptr_pins(rec, obj)) + return; + for (i = 0, field = rec->fields; i < cnt; i++, field++) { if (field->type != BPF_UPTR) continue; @@ -193,8 +235,12 @@ static void bpf_obj_unpin_uptrs(struct btf_record *rec, void *obj) __bpf_obj_unpin_uptrs(rec, rec->cnt, obj); } -static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) +static int bpf_obj_pin_uptrs(struct bpf_map *map, void *obj) { + struct btf_record *rec = map->record; +#ifdef CONFIG_MM_SUBPAGE + struct bpf_uptr_pins *pins = NULL; +#endif const struct btf_field *field; const struct btf_type *t; unsigned long start, end; @@ -205,6 +251,18 @@ static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) if (!btf_record_has_field(rec, BPF_UPTR)) return 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + pins = bpf_map_kzalloc(map, struct_size(pins, fragments, rec->cnt), + GFP_KERNEL); + if (!pins) + return -ENOMEM; + INIT_WORK(&pins->work, bpf_uptr_release_work); + pins->count = rec->cnt; + *(struct bpf_uptr_pins **)(obj + rec->uptr_pins_off) = pins; + } +#endif + for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) { if (field->type != BPF_UPTR) continue; @@ -221,12 +279,35 @@ static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) goto unpin_all; } - /* The uptr's struct cannot span across two pages */ - if ((start & PAGE_MASK) != (end & PAGE_MASK)) { + /* The object must fit in one page of the supplying process. */ + if ((start & mm_page_mask(current->mm)) != + (end & mm_page_mask(current->mm))) { err = -EOPNOTSUPP; goto unpin_all; } +#ifdef CONFIG_MM_SUBPAGE + if (pins) { + struct user_page_fragment *fragment = &pins->fragments[i]; + + err = pin_user_fragments_remote(current->mm, start, t->size, + FOLL_LONGTERM | FOLL_WRITE, fragment, 1); + if (err != 1) { + if (!err) + err = -EFAULT; + goto unpin_all; + } + if (fragment->length != t->size || + PageHighMem(folio_page(fragment->folio, + fragment->offset >> PAGE_SHIFT))) { + err = -EOPNOTSUPP; + goto unpin_all; + } + *uptr_addr = folio_address(fragment->folio) + fragment->offset; + continue; + } +#endif + err = pin_user_pages_fast(start, 1, FOLL_LONGTERM | FOLL_WRITE, &page); if (err != 1) goto unpin_all; @@ -291,7 +372,7 @@ static int bpf_map_update_value(struct bpf_map *map, struct file *map_file, map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) { err = map->ops->map_push_elem(map, value, flags); } else { - err = bpf_obj_pin_uptrs(map->record, value); + err = bpf_obj_pin_uptrs(map, value); if (!err) { rcu_read_lock(); err = map->ops->map_update_elem(map, key, value, flags); @@ -815,10 +896,12 @@ void bpf_obj_cancel_fields(struct bpf_map *map, void *obj) void bpf_obj_free_fields(const struct btf_record *rec, void *obj) { const struct btf_field *fields; + bool fragment_pins; int i; if (IS_ERR_OR_NULL(rec)) return; + fragment_pins = bpf_obj_release_uptr_pins(rec, obj); fields = rec->fields; for (i = 0; i < rec->cnt; i++) { struct btf_struct_meta *pointee_struct_meta; @@ -859,8 +942,9 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj) } break; case BPF_UPTR: - /* The caller ensured that no one is using the uptr */ - unpin_uptr_kaddr(*(void **)field_ptr); + /* The caller ensured that no one is using the uptr. */ + if (!fragment_pins) + unpin_uptr_kaddr(*(void **)field_ptr); break; case BPF_LIST_HEAD: if (WARN_ON_ONCE(rec->spin_lock_off < 0)) @@ -1350,6 +1434,14 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, if (ret < 0) goto free_map_tab; +#ifdef CONFIG_MM_SUBPAGE + if (btf_record_has_field(map->record, BPF_UPTR)) { + map->record->uptr_pins_off = ALIGN(map->value_size, sizeof(void *)); + map->record->uptr_extra_size = map->record->uptr_pins_off + + sizeof(void *) - map->value_size; + } +#endif + if (map->ops->map_check_btf) { ret = map->ops->map_check_btf(map, btf, key_type, value_type); if (ret < 0) @@ -1775,6 +1867,24 @@ static int map_lookup_elem(union bpf_attr *attr) #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags +static void *bpf_map_memdup_value(struct bpf_map *map, bpfptr_t src, u32 size) +{ + u32 extra = bpf_obj_uptr_extra_size(map->record); + void *value; + + if (!extra) + return kvmemdup_bpfptr(src, size); + value = kvmalloc(size + extra, GFP_USER); + if (!value) + return ERR_PTR(-ENOMEM); + memset(value + size, 0, extra); + if (copy_from_bpfptr(value, src, size)) { + kvfree(value); + return ERR_PTR(-EFAULT); + } + return value; +} + static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) { bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel); @@ -1808,7 +1918,7 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) } value_size = bpf_map_value_size(map, attr->flags); - value = kvmemdup_bpfptr(uvalue, value_size); + value = bpf_map_memdup_value(map, uvalue, value_size); if (IS_ERR(value)) { err = PTR_ERR(value); goto free_key; @@ -2021,7 +2131,9 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file, if (!key) return -ENOMEM; - value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); + value = kvmalloc(value_size + bpf_obj_uptr_extra_size(map->record), + GFP_USER | __GFP_NOWARN | + (bpf_obj_uptr_extra_size(map->record) ? __GFP_ZERO : 0)); if (!value) { kvfree(key); return -ENOMEM; diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 6300b216012cb..640991b1c25cb 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -30,6 +30,7 @@ #include #include /* check_stable_address_space */ #include +#include #include @@ -144,12 +145,14 @@ static bool valid_vma(struct vm_area_struct *vma, bool is_register) static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset) { - return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT) - + vma_subpage_offset(vma); } static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) { - return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start); + return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma) + + (vaddr - vma->vm_start); } /** @@ -191,7 +194,8 @@ static void copy_to_page(struct page *page, unsigned long vaddr, const void *src kunmap_local(kaddr); } -static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *insn, +static int verify_opcode(struct page *page, unsigned long vaddr, unsigned int page_offset, + uprobe_opcode_t *insn, int nbytes, void *data) { uprobe_opcode_t old_opcode; @@ -206,7 +210,7 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t * is a trap variant; uprobes always wins over any other (gdb) * breakpoint. */ - uprobe_copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE); + uprobe_copy_from_page(page, page_offset, &old_opcode, UPROBE_SWBP_INSN_SIZE); is_swbp = is_swbp_insn(&old_opcode); if (is_swbp_insn(insn)) { @@ -307,14 +311,24 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) { void *kaddr; struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *vma; + bool alternative = mm_page_size(mm) != PAGE_SIZE; +#endif int ret; short *ptr; if (!vaddr || !d) return -EINVAL; - ret = get_user_pages_remote(mm, vaddr, 1, - FOLL_WRITE, &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + ret = get_user_fragment_vma_remote(mm, vaddr, sizeof(short), + FOLL_WRITE, &fragment, &vma); + else +#endif + ret = get_user_pages_remote(mm, vaddr, 1, FOLL_WRITE, &page, NULL); if (unlikely(ret <= 0)) { /* * We are asking for 1 page. If get_user_pages_remote() fails, @@ -323,8 +337,16 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) return ret == 0 ? -EBUSY : ret; } - kaddr = kmap_local_page(page); - ptr = kaddr + (vaddr & ~PAGE_MASK); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) { + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + ptr = kaddr; + } else +#endif + { + kaddr = kmap_local_page(page); + ptr = kaddr + (vaddr & ~PAGE_MASK); + } if (unlikely(*ptr + d < 0)) { pr_warn("ref_ctr going negative. vaddr: 0x%lx, " @@ -337,7 +359,12 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) ret = 0; out: kunmap_local(kaddr); - put_page(page); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + release_user_fragments(&fragment, 1, !ret); + else +#endif + put_page(page); return ret; } @@ -381,7 +408,8 @@ static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm, } static bool orig_page_is_identical(struct vm_area_struct *vma, - unsigned long vaddr, struct page *page, bool *pmd_mappable) + unsigned long vaddr, struct page *page, unsigned int page_offset, + bool *pmd_mappable) { const pgoff_t index = vaddr_to_offset(vma, vaddr) >> PAGE_SHIFT; struct folio *orig_folio = filemap_get_folio(vma->vm_file->f_mapping, @@ -393,19 +421,30 @@ static bool orig_page_is_identical(struct vm_area_struct *vma, return false; orig_page = folio_file_page(orig_folio, index); - *pmd_mappable = folio_test_pmd_mappable(orig_folio); - identical = folio_test_uptodate(orig_folio) && - pages_identical(page, orig_page); + *pmd_mappable = mm_page_size(vma->vm_mm) == PAGE_SIZE && + folio_test_pmd_mappable(orig_folio); + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + unsigned int offset = vma_page_offset_at(vma, vaddr).offset; + void *a = kmap_local_page(page), *b = kmap_local_page(orig_page); + + identical = folio_test_uptodate(orig_folio) && + !memcmp(a + page_offset, b + offset, mm_page_size(vma->vm_mm)); + kunmap_local(b); + kunmap_local(a); + } else { + identical = folio_test_uptodate(orig_folio) && + pages_identical(page, orig_page); + } folio_put(orig_folio); return identical; } static int __uprobe_write(struct vm_area_struct *vma, - struct folio_walk *fw, struct folio *folio, - unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, + struct folio_walk *fw, struct folio *folio, struct mm_subpage *slot, + unsigned int page_offset, unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, bool is_register) { - const unsigned long vaddr = insn_vaddr & PAGE_MASK; + const unsigned long vaddr = insn_vaddr & mm_page_mask(vma->vm_mm); bool pmd_mappable; /* For now, we'll only handle PTE-mapped folios. */ @@ -417,6 +456,12 @@ static int __uprobe_write(struct vm_area_struct *vma, * but the VMA might not be writable. */ if (!pte_write(fw->pte)) { +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + if (!mm_subpage_is_exclusive(slot)) + return -EFAULT; + } else +#endif if (!PageAnonExclusive(fw->page)) return -EFAULT; if (unlikely(userfaultfd_pte_wp(vma, fw->pte))) @@ -430,31 +475,49 @@ static int __uprobe_write(struct vm_area_struct *vma, */ flush_cache_page(vma, vaddr, pte_pfn(fw->pte)); fw->pte = ptep_clear_flush(vma, vaddr, fw->ptep); - copy_to_page(fw->page, insn_vaddr, insn, nbytes); + copy_to_page(fw->page, page_offset, insn, nbytes); /* * When unregistering, we may only zap a PTE if uffd is disabled and * there are no unexpected folio references ... */ - if (is_register || userfaultfd_missing(vma) || - (folio_ref_count(folio) != folio_expected_ref_count(folio) + 1)) + if (is_register || userfaultfd_missing(vma)) + goto remap; +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + if (!mm_subpage_can_zap(slot)) + goto remap; + } else +#endif + if (folio_ref_count(folio) != folio_expected_ref_count(folio) + 1) goto remap; /* * ... and the mapped page is identical to the original page that * would get faulted in on next access. */ - if (!orig_page_is_identical(vma, vaddr, fw->page, &pmd_mappable)) + if (!orig_page_is_identical(vma, vaddr, fw->page, + page_offset - (insn_vaddr - vaddr), &pmd_mappable)) goto remap; dec_mm_counter(vma->vm_mm, MM_ANONPAGES); - folio_remove_rmap_pte(folio, fw->page, vma); +#ifdef CONFIG_MM_SUBPAGE + if (slot) + mm_subpage_remove_anon_rmap(slot, vma); + else +#endif + folio_remove_rmap_pte(folio, fw->page, vma); if (!folio_mapped(folio) && folio_test_swapcache(folio) && folio_trylock(folio)) { folio_free_swap(folio); folio_unlock(folio); } - folio_put(folio); +#ifdef CONFIG_MM_SUBPAGE + if (slot) + mm_subpage_put(slot); /* Retired PTE reference; caller still has its GET. */ + else +#endif + folio_put(folio); return pmd_mappable; remap: @@ -494,13 +557,24 @@ int uprobe_write_opcode(struct arch_uprobe *auprobe, struct vm_area_struct *vma, verify_opcode, is_register, true /* do_update_ref_ctr */, NULL); } +static void uprobe_put_folio(struct folio *folio, struct mm_subpage *slot) +{ +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + mm_subpage_put(slot); + return; + } +#endif + folio_put(folio); +} + int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, const unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, uprobe_write_verify_t verify, bool is_register, bool do_update_ref_ctr, void *data) { - const unsigned long vaddr = insn_vaddr & PAGE_MASK; struct mm_struct *mm = vma->vm_mm; + const unsigned long vaddr = insn_vaddr & mm_page_mask(mm); struct uprobe *uprobe; int ret, ref_ctr_updated = 0; unsigned int gup_flags = FOLL_FORCE; @@ -508,7 +582,16 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, struct folio_walk fw; struct folio *folio; struct page *page; - + struct mm_subpage *slot = NULL; + unsigned int page_offset; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *found; +#endif + + /* Larger-than-native public mms are not enabled by the mixed-page ABI yet. */ + if (mm_page_size(mm) > PAGE_SIZE) + return -EOPNOTSUPP; uprobe = container_of(auprobe, struct uprobe, arch); if (WARN_ON_ONCE(!is_cow_mapping(vma->vm_flags))) @@ -526,14 +609,29 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, gup_flags |= FOLL_WRITE | FOLL_SPLIT_PMD; retry: - ret = get_user_pages_remote(mm, vaddr, 1, gup_flags, &page, NULL); - if (ret <= 0) - goto out; - folio = page_folio(page); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + ret = get_user_fragment_vma_remote(mm, vaddr, mm_page_size(mm), + gup_flags & ~FOLL_SPLIT_PMD, &fragment, &found); + if (ret <= 0) + goto out; + folio = fragment.folio; + slot = fragment.subpage; + page = folio_page(folio, fragment.offset >> PAGE_SHIFT); + page_offset = offset_in_page(fragment.offset) + insn_vaddr - vaddr; + } else +#endif + { + ret = get_user_pages_remote(mm, vaddr, 1, gup_flags, &page, NULL); + if (ret <= 0) + goto out; + folio = page_folio(page); + page_offset = offset_in_page(insn_vaddr); + } - ret = verify(page, insn_vaddr, insn, nbytes, data); + ret = verify(page, insn_vaddr, page_offset, insn, nbytes, data); if (ret <= 0) { - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -541,7 +639,7 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, if (do_update_ref_ctr && !ref_ctr_updated && uprobe->ref_ctr_offset) { ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1); if (ret) { - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -551,7 +649,7 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, ret = 0; if (unlikely(!folio_test_anon(folio) || folio_is_zone_device(folio))) { VM_WARN_ON_ONCE(is_register); - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -562,22 +660,29 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, * be able to do it under PTL. */ mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, - vaddr, vaddr + PAGE_SIZE); + vaddr, vaddr + mm_page_size(mm)); mmu_notifier_invalidate_range_start(&range); } ret = -EAGAIN; /* Walk the page tables again, to perform the actual update. */ if (folio_walk_start(&fw, vma, vaddr, 0)) { - if (fw.page == page) - ret = __uprobe_write(vma, &fw, folio, insn_vaddr, insn, nbytes, is_register); + bool same = fw.page == page; +#ifdef CONFIG_MM_SUBPAGE + if (slot) + same &= fw.level == FW_LEVEL_PTE && + pte_phys_mm(mm, fw.pte) == mm_subpage_phys(slot); +#endif + if (same) + ret = __uprobe_write(vma, &fw, folio, slot, page_offset, + insn_vaddr, insn, nbytes, is_register); folio_walk_end(&fw, vma); } if (!is_register) mmu_notifier_invalidate_range_end(&range); - folio_put(folio); + uprobe_put_folio(folio, slot); switch (ret) { case -EFAULT: gup_flags |= FOLL_WRITE | FOLL_SPLIT_PMD; @@ -1211,7 +1316,9 @@ build_map_info(struct address_space *mapping, loff_t offset, bool is_register) again: i_mmap_lock_read(mapping); vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { - if (!valid_vma(vma, is_register)) + if (!valid_vma(vma, is_register) || + offset < vaddr_to_offset(vma, vma->vm_start) || + offset >= vaddr_to_offset(vma, vma->vm_end)) continue; if (!prev && !more) { @@ -1482,7 +1589,7 @@ static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) file_inode(vma->vm_file) != uprobe->inode) continue; - offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; + offset = vaddr_to_offset(vma, vma->vm_start); if (uprobe->offset < offset || uprobe->offset >= offset + vma->vm_end - vma->vm_start) continue; @@ -1696,8 +1803,14 @@ static const struct vm_special_mapping xol_mapping = { unsigned long __weak arch_uprobe_get_xol_area(void) { - /* Try to map as high as possible, this is only a hint. */ - return get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0); + unsigned long size = PAGE_SIZE; + unsigned long addr; + + /* XOL arch copy hooks use native offsets into the one backing page. */ + if (mm_page_size(current->mm) < PAGE_SIZE) + size += PAGE_SIZE - mm_page_size(current->mm); + addr = get_unmapped_area(NULL, TASK_SIZE - size, size, 0, 0); + return IS_ERR_VALUE(addr) ? addr : ALIGN(addr, PAGE_SIZE); } /* Slot allocation for XOL */ @@ -2413,6 +2526,23 @@ static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr) if (likely(result == 0)) goto out; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + struct user_page_fragment fragment; + struct vm_area_struct *vma; + void *kaddr; + + result = get_user_fragment_vma_remote(mm, vaddr, UPROBE_SWBP_INSN_SIZE, + FOLL_FORCE, &fragment, &vma); + if (result != 1) + return result < 0 ? result : -EFAULT; + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + memcpy(&opcode, kaddr, UPROBE_SWBP_INSN_SIZE); + kunmap_local(kaddr); + release_user_fragments(&fragment, 1, false); + goto out; + } +#endif result = get_user_pages(vaddr, 1, FOLL_FORCE, &page); if (result < 0) return result; @@ -2453,7 +2583,7 @@ static struct uprobe *find_active_uprobe_speculative(unsigned long bp_vaddr) if (!vm_file) return NULL; - offset = (loff_t)(vma->vm_pgoff << PAGE_SHIFT) + (bp_vaddr - vma->vm_start); + offset = vaddr_to_offset(vma, bp_vaddr); uprobe = find_uprobe_rcu(vm_file->f_inode, offset); if (!uprobe) return NULL; diff --git a/kernel/fork.c b/kernel/fork.c index 5a4623b058a20..115f932bea1fa 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -112,7 +112,7 @@ #include #include -#include +#include #include #include @@ -639,6 +639,12 @@ static void check_mm(struct mm_struct *mm) } } +#ifdef CONFIG_MM_SUBPAGE + if (mm->committed_user_pages) + pr_alert("BUG: non-zero user commitment on freeing mm: %lu\n", + mm->committed_user_pages); +#endif + if (mm_pgtables_bytes(mm)) pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", mm_pgtables_bytes(mm)); @@ -1089,6 +1095,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, INIT_LIST_HEAD(&mm->mmlist); mm_pgtables_bytes_init(mm); mm->map_count = 0; +#ifdef CONFIG_MM_SUBPAGE + mm->committed_user_pages = 0; +#endif mm->locked_vm = 0; atomic64_set(&mm->pinned_vm, 0); memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); @@ -1159,7 +1168,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, /* * Allocate and initialize an mm_struct. */ -struct mm_struct *mm_alloc(void) +static struct mm_struct *__mm_alloc(bool for_exec, unsigned int page_shift) { struct mm_struct *mm; @@ -1168,10 +1177,28 @@ struct mm_struct *mm_alloc(void) return NULL; memset(mm, 0, sizeof(*mm)); + if (for_exec) + arch_mm_init_exec(mm, page_shift); return mm_init(mm, current, current_user_ns()); } + +struct mm_struct *mm_alloc(void) +{ + return __mm_alloc(false, 0); +} EXPORT_SYMBOL_IF_KUNIT(mm_alloc); +struct mm_struct *mm_alloc_exec(void) +{ + return __mm_alloc(true, 0); +} + +/* The caller has validated the requested exec granule. */ +struct mm_struct *mm_alloc_exec_page_shift(unsigned int page_shift) +{ + return __mm_alloc(true, page_shift); +} + static inline void __mmput(struct mm_struct *mm) { VM_BUG_ON(atomic_read(&mm->mm_users)); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 7ffe6dc79b28e..771c12983cd2a 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include "futex.h" @@ -512,6 +513,44 @@ static u64 get_inode_sequence_number(struct inode *inode) } } +/* Keep a typed slot reference alive until the mapping identity is sampled. */ +struct futex_page_ref { + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; +#endif +}; + +static int futex_get_page_ref(struct mm_struct *mm, unsigned long address, + unsigned long uaddr, unsigned int size, unsigned int flags, + struct futex_page_ref *ref) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + int ret = get_user_fragments_remote(mm, uaddr, size, flags, + &ref->fragment, 1); + + if (ret != 1) + return ret ?: -EFAULT; + ref->page = folio_page(ref->fragment.folio, + ref->fragment.offset >> PAGE_SHIFT); + return 1; + } +#endif + return get_user_pages_fast(address, 1, flags, &ref->page); +} + +static void futex_put_page_ref(struct futex_page_ref *ref) +{ +#ifdef CONFIG_MM_SUBPAGE + if (ref->fragment.folio) { + release_user_fragments(&ref->fragment, 1, false); + return; + } +#endif + put_page(ref->page); +} + /** * get_futex_key() - Get parameters which are the keys for a futex * @uaddr: virtual address of the futex @@ -545,6 +584,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, unsigned long address = (unsigned long)uaddr; struct mm_struct *mm = current->mm; struct page *page; + struct futex_page_ref ref = {}; struct folio *folio; struct address_space *mapping; int node, err, size, ro = 0; @@ -629,13 +669,13 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, if (unlikely(should_fail_futex(true))) return -EFAULT; - err = get_user_pages_fast(address, 1, FOLL_WRITE, &page); + err = futex_get_page_ref(mm, address, (unsigned long)uaddr, size, FOLL_WRITE, &ref); /* * If write access is not required (eg. FUTEX_WAIT), try * and get read-only access. */ if (err == -EFAULT && rw == FUTEX_READ) { - err = get_user_pages_fast(address, 1, 0, &page); + err = futex_get_page_ref(mm, address, (unsigned long)uaddr, size, 0, &ref); ro = 1; } if (err < 0) @@ -660,6 +700,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, * filesystem-backed pages, the precise page is required as the * index of the page determines the key. */ + page = ref.page; folio = page_folio(page); mapping = READ_ONCE(folio->mapping); @@ -689,7 +730,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, folio_lock(folio); shmem_swizzled = folio_test_swapcache(folio) || folio->mapping; folio_unlock(folio); - folio_put(folio); + futex_put_page_ref(&ref); if (shmem_swizzled) goto again; @@ -739,7 +780,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, if (READ_ONCE(folio->mapping) != mapping) { rcu_read_unlock(); - folio_put(folio); + futex_put_page_ref(&ref); goto again; } @@ -747,11 +788,15 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, inode = READ_ONCE(mapping->host); if (!inode) { rcu_read_unlock(); - folio_put(folio); + futex_put_page_ref(&ref); goto again; } +#ifdef CONFIG_MM_SUBPAGE + if (ref.fragment.folio) + key->both.offset = offset_in_page(ref.fragment.offset); +#endif key->both.offset |= FUT_OFF_INODE; /* inode-based key */ key->shared.i_seq = get_inode_sequence_number(inode); key->shared.pgoff = page_pgoff(folio, page); @@ -759,7 +804,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, } out: - folio_put(folio); + futex_put_page_ref(&ref); return err; } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index abb76775aa728..c5b7ea8acdda3 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1694,7 +1694,7 @@ static unsigned int task_nr_scan_windows(struct task_struct *p) * on resident pages */ nr_scan_pages = MB_TO_PAGES(sysctl_numa_balancing_scan_size); - rss = get_mm_rss(p->mm); + rss = mm_pages_to_native(p->mm, get_mm_rss(p->mm)); if (!rss) rss = nr_scan_pages; diff --git a/kernel/sys.c b/kernel/sys.c index ec432b89e8540..ab661d258b152 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -1927,7 +1927,7 @@ void getrusage(struct task_struct *p, int who, struct rusage *r) } out_children: - r->ru_maxrss = maxrss * (PAGE_SIZE / 1024); /* convert pages to KBs */ + r->ru_maxrss = maxrss; /* signal and mm high-water maxima are in kB */ r->ru_utime = ns_to_kernel_old_timeval(utime); r->ru_stime = ns_to_kernel_old_timeval(stime); } @@ -2542,6 +2542,28 @@ int __weak arch_prctl_mem_model_set(struct task_struct *t, unsigned long val) return -EINVAL; } +int __weak arch_prctl_exec_page_size_get(struct task_struct *task) +{ + return -EINVAL; +} + +int __weak arch_prctl_exec_page_size_set(struct task_struct *task, + unsigned long size) +{ + return -EINVAL; +} + +int __weak arch_prctl_default_page_size_get(struct task_struct *task) +{ + return -EINVAL; +} + +int __weak arch_prctl_default_page_size_set(struct task_struct *task, + unsigned long size) +{ + return -EINVAL; +} + SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, unsigned long, arg4, unsigned long, arg5) { @@ -2555,6 +2577,22 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = 0; switch (option) { + case PR_AURORA_GET_EXEC_PAGE_SIZE: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + return arch_prctl_exec_page_size_get(me); + case PR_AURORA_SET_EXEC_PAGE_SIZE: + if (arg3 || arg4 || arg5) + return -EINVAL; + return arch_prctl_exec_page_size_set(me, arg2); + case PR_AURORA_GET_DEFAULT_PAGE_SIZE: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + return arch_prctl_default_page_size_get(me); + case PR_AURORA_SET_DEFAULT_PAGE_SIZE: + if (arg3 || arg4 || arg5) + return -EINVAL; + return arch_prctl_default_page_size_set(me, arg2); case PR_GET_MEM_MODEL: if (arg2 || arg3 || arg4 || arg5) return -EINVAL; diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 8c82ecb735f41..e8b622a8178cb 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include "trace_dynevent.h" @@ -554,6 +555,11 @@ static int user_event_enabler_write(struct user_event_mm *mm, unsigned long uaddr = enabler->addr; unsigned long *ptr; struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *vma; + bool alternative = mm_page_size(mm->mm) != PAGE_SIZE; +#endif void *kaddr; int bit = ENABLE_BIT(enabler); int ret; @@ -573,8 +579,14 @@ static int user_event_enabler_write(struct user_event_mm *mm, align_addr_bit(&uaddr, &bit, ENABLE_BITOPS(enabler)); - ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT, - &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + ret = pin_user_fragment_vma_remote(mm->mm, uaddr, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &vma); + else +#endif + ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT, + &page, NULL); if (unlikely(ret <= 0)) { if (!fixup_fault) @@ -586,8 +598,17 @@ static int user_event_enabler_write(struct user_event_mm *mm, return -EFAULT; } - kaddr = kmap_local_page(page); - ptr = kaddr + (uaddr & ~PAGE_MASK); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) { + /* align_addr_bit() keeps the complete atomic word in this fragment. */ + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + ptr = kaddr; + } else +#endif + { + kaddr = kmap_local_page(page); + ptr = kaddr + (uaddr & ~PAGE_MASK); + } /* Update bit atomically, user tracers must be atomic as well */ if (enabler->event && enabler->event->status) @@ -596,7 +617,12 @@ static int user_event_enabler_write(struct user_event_mm *mm, clear_bit(bit, ptr); kunmap_local(kaddr); - unpin_user_pages_dirty_lock(&page, 1, true); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + release_user_fragments(&fragment, 1, true); + else +#endif + unpin_user_pages_dirty_lock(&page, 1, true); return 0; } diff --git a/kernel/tsacct.c b/kernel/tsacct.c index 5c153106e6427..5f59d071ab565 100644 --- a/kernel/tsacct.c +++ b/kernel/tsacct.c @@ -91,16 +91,16 @@ void xacct_add_tsk(struct taskstats *stats, struct task_struct *p) { struct mm_struct *mm; - /* convert pages-nsec/1024 to Mbyte-usec, see __acct_update_integrals */ - stats->coremem = p->acct_rss_mem1 * PAGE_SIZE; + /* convert account-units-nsec/1024 to Mbyte-usec, see __acct_update_integrals */ + stats->coremem = p->acct_rss_mem1 * (1UL << MM_ACCOUNT_SHIFT); do_div(stats->coremem, 1000 * KB); - stats->virtmem = p->acct_vm_mem1 * PAGE_SIZE; + stats->virtmem = p->acct_vm_mem1 * (1UL << MM_ACCOUNT_SHIFT); do_div(stats->virtmem, 1000 * KB); mm = get_task_mm(p); if (mm) { /* adjust to KB unit */ - stats->hiwater_rss = get_mm_hiwater_rss(mm) * PAGE_SIZE / KB; - stats->hiwater_vm = get_mm_hiwater_vm(mm) * PAGE_SIZE / KB; + stats->hiwater_rss = mm_pages_to_kb(mm, get_mm_hiwater_rss(mm)); + stats->hiwater_vm = mm_pages_to_kb(mm, get_mm_hiwater_vm(mm)); mmput(mm); } stats->read_char = p->ioac.rchar & KB_MASK; @@ -140,8 +140,10 @@ static void __acct_update_integrals(struct task_struct *tsk, * The final unit reported to userspace is Mbyte-usecs, * the rest of the math is done in xacct_add_tsk. */ - tsk->acct_rss_mem1 += delta * get_mm_rss(tsk->mm) >> 10; - tsk->acct_vm_mem1 += delta * READ_ONCE(tsk->mm->total_vm) >> 10; + tsk->acct_rss_mem1 += delta * mm_pages_to_shift(tsk->mm, get_mm_rss(tsk->mm), + MM_ACCOUNT_SHIFT) >> 10; + tsk->acct_vm_mem1 += delta * mm_pages_to_shift(tsk->mm, READ_ONCE(tsk->mm->total_vm), + MM_ACCOUNT_SHIFT) >> 10; } /** diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c index 17d37b82ebc68..d808ddcb94b01 100644 --- a/lib/vdso/datastore.c +++ b/lib/vdso/datastore.c @@ -83,7 +83,8 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, vm_fault_t err; addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE; - err = vmf_insert_page(vma, addr, page); + err = vmf_insert_page_offset(vma, addr, page, + vma_page_offset_at(vma, vmf->address).offset); if (unlikely(err & VM_FAULT_ERROR)) return err; page = timens_page; diff --git a/mm/Kconfig b/mm/Kconfig index e8bf1e9e6ad90..0dcb4818f4f7a 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1167,6 +1167,21 @@ config IDLE_PAGE_TRACKING # Architectures which implement cpu_dcache_is_aliasing() to query # whether the data caches are aliased (VIVT or VIPT with dcache # aliasing) need to select this. +config ARCH_HAS_MM_PGTABLE + bool + help + Architecture supplies target-mm page-table geometry and accessors. + This does not change the native physical allocation granule. + +config MM_SUBPAGE + bool + select PAGE_EXTENSION + select XARRAY_MULTI + help + Reference-counted user data slots smaller than a native folio. + Architectures must integrate rmap, faults, reclaim and pins before + exposing these backing primitives to userspace. + config ARCH_HAS_CPU_CACHE_ALIASING bool @@ -1439,9 +1454,15 @@ config ARCH_HAS_USER_SHADOW_STACK config HAVE_ARCH_TLB_REMOVE_TABLE def_bool n +config ARCH_SUPPORTS_PT_RECLAIM + bool + help + The architecture's custom table-free callback supports PTE reclamation. + config PT_RECLAIM def_bool y - depends on MMU_GATHER_RCU_TABLE_FREE && !HAVE_ARCH_TLB_REMOVE_TABLE + depends on MMU_GATHER_RCU_TABLE_FREE + depends on !HAVE_ARCH_TLB_REMOVE_TABLE || ARCH_SUPPORTS_PT_RECLAIM help Try to reclaim empty user page table pages in paths other than munmap and exit_mmap path. diff --git a/mm/Makefile b/mm/Makefile index 8ad2ab08244eb..f531742956b8e 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -150,3 +150,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o obj-$(CONFIG_EXECMEM) += execmem.o obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o +obj-$(CONFIG_MM_SUBPAGE) += user-subpage.o diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index cca7014ab5f13..3d618828c2c8d 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -155,9 +155,10 @@ int damon_cold_score(struct damon_ctx *c, struct damon_region *r, } static bool damon_folio_mkold_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + unsigned long addr = rmap.address; + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); while (page_vma_mapped_walk(&pvmw)) { addr = pvmw.address; @@ -190,10 +191,11 @@ void damon_folio_mkold(struct folio *folio) } static bool damon_folio_young_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long addr = rmap.address; bool *accessed = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); pte_t pte; *accessed = false; diff --git a/mm/filemap.c b/mm/filemap.c index bbd400d893aa5..14e2656b6a9cc 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -225,6 +225,7 @@ void __filemap_remove_folio(struct folio *folio, void *shadow) trace_mm_filemap_delete_from_page_cache(folio); filemap_unaccount_folio(mapping, folio); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); page_cache_delete(mapping, folio, shadow); } @@ -308,6 +309,7 @@ static void page_cache_delete_batch(struct address_space *mapping, WARN_ON_ONCE(!folio_test_locked(folio)); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); folio->mapping = NULL; /* Leave folio->index set: truncation lookup relies on it */ @@ -827,6 +829,7 @@ void replace_page_cache_folio(struct folio *old, struct folio *new) mem_cgroup_replace_folio(old, new); xas_lock_irq(&xas); + shmem_uffd_migrate(new, old); xas_store(&xas, new); old->mapping = NULL; @@ -3387,7 +3390,7 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) */ struct vm_area_struct *vma = vmf->vma; unsigned long start = vma->vm_pgoff; - unsigned long end = start + vma_pages(vma); + unsigned long end = vma_last_pgoff(vma) + 1; unsigned long ra_end; ra->order = exec_folio_order(); @@ -3529,7 +3532,12 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) bool mapping_locked = false; max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - if (unlikely(index >= max_idx)) + if (unlikely(index >= max_idx +#ifdef CONFIG_MM_SUBPAGE + || (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) +#endif + )) return VM_FAULT_SIGBUS; trace_mm_filemap_fault(mapping, index); @@ -3633,7 +3641,12 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) * We must recheck i_size under page lock. */ max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - if (unlikely(index >= max_idx)) { + if (unlikely(index >= max_idx +#ifdef CONFIG_MM_SUBPAGE + || (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) +#endif + )) { folio_unlock(folio); folio_put(folio); return VM_FAULT_SIGBUS; @@ -3783,6 +3796,10 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, } do { + if (userfaultfd_missing(vmf->vma) && + shmem_uffd_range_missing(folio, folio->index + + folio_page_idx(folio, page + count), 0, PAGE_SIZE)) + goto skip; if (PageHWPoison(page + count)) goto skip; @@ -3847,6 +3864,9 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, vm_fault_t ret = 0; struct page *page = &folio->page; + if (userfaultfd_missing(vmf->vma) && + shmem_uffd_range_missing(folio, folio->index, 0, PAGE_SIZE)) + goto out; if (PageHWPoison(page)) goto out; @@ -3875,6 +3895,48 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, return ret; } +#ifdef CONFIG_MM_SUBPAGE +static vm_fault_t filemap_map_user_pages(struct vm_fault *vmf, + pgoff_t first, pgoff_t last) +{ + struct vm_area_struct *vma = vmf->vma; + struct address_space *mapping = vma->vm_file->f_mapping; + XA_STATE(xas, &mapping->i_pages, first); + struct folio *folio; + vm_fault_t ret = 0; + unsigned short miss = 0, saved; + + rcu_read_lock(); + folio = next_uptodate_folio(&xas, mapping, last); + if (!folio) + goto out; + if (pmd_none(*vmf->pmd) && vmf->prealloc_pte) + pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte); + vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, + vmf->address, &vmf->ptl); + if (!vmf->pte) { + folio_unlock(folio); + folio_put(folio); + goto out; + } + do { + ret |= map_file_user_pages(vmf, folio, first, last, false); + if (!folio_test_workingset(folio)) + miss += min(folio_next_index(folio) - 1, last) - + max(folio->index, first) + 1; + folio_unlock(folio); + folio_put(folio); + } while ((folio = next_uptodate_folio(&xas, mapping, last)) != NULL); + pte_unmap_unlock(vmf->pte, vmf->ptl); + trace_mm_filemap_map_pages(mapping, first, last); +out: + rcu_read_unlock(); + saved = READ_ONCE(vma->vm_file->f_ra.mmap_miss); + WRITE_ONCE(vma->vm_file->f_ra.mmap_miss, saved > miss ? saved - miss : 0); + return ret; +} +#endif + vm_fault_t filemap_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff) { @@ -3890,6 +3952,11 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, unsigned int nr_pages = 0, folio_type; unsigned short mmap_miss = 0, mmap_miss_saved; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return filemap_map_user_pages(vmf, start_pgoff, end_pgoff); +#endif + /* * Recalculate end_pgoff based on file_end before calling * next_uptodate_folio() to avoid races with concurrent diff --git a/mm/gup.c b/mm/gup.c index ad9ded39609cb..bb31fb08ca2d1 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -1297,6 +1298,11 @@ static struct vm_area_struct *gup_vma_lookup(struct mm_struct *mm, #endif } +#ifdef CONFIG_MM_SUBPAGE +static long prefault_subpage_range(struct mm_struct *mm, unsigned long start, + unsigned long nr_pages, unsigned int flags, int *locked); +#endif + /** * __get_user_pages() - pin user pages in memory * @mm: mm_struct of target mm @@ -1363,6 +1369,15 @@ static long __get_user_pages(struct mm_struct *mm, if (!nr_pages) return 0; +#ifdef CONFIG_MM_SUBPAGE + /* A prefault has no page-array ownership or physical-offset ambiguity. */ + if (mm_page_size(mm) != PAGE_SIZE) { + if (pages || (gup_flags & (FOLL_GET | FOLL_PIN))) + return -EOPNOTSUPP; + return prefault_subpage_range(mm, start, nr_pages, gup_flags, locked); + } +#endif + start = untagged_addr_remote(mm, start); VM_WARN_ON_ONCE(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); @@ -1721,7 +1736,7 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm, */ if (likely(pages)) pages += ret; - start += ret << PAGE_SHIFT; + start += ret << mm_page_shift(mm); /* The lock was temporarily dropped, so we must unlock later */ must_unlock = true; @@ -1768,7 +1783,7 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm, break; if (likely(pages)) pages++; - start += PAGE_SIZE; + start += mm_page_size(mm); } if (must_unlock && *locked) { /* @@ -1814,13 +1829,13 @@ long populate_vma_page_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, int *locked) { struct mm_struct *mm = vma->vm_mm; - unsigned long nr_pages = (end - start) / PAGE_SIZE; + unsigned long nr_pages = (end - start) / mm_page_size(mm); int local_locked = 1; int gup_flags; long ret; - VM_WARN_ON_ONCE(!PAGE_ALIGNED(start)); - VM_WARN_ON_ONCE(!PAGE_ALIGNED(end)); + VM_WARN_ON_ONCE(!IS_ALIGNED(start, mm_page_size(mm))); + VM_WARN_ON_ONCE(!IS_ALIGNED(end, mm_page_size(mm))); VM_WARN_ON_ONCE_VMA(start < vma->vm_start, vma); VM_WARN_ON_ONCE_VMA(end > vma->vm_end, vma); mmap_assert_locked(mm); @@ -1887,12 +1902,12 @@ long populate_vma_page_range(struct vm_area_struct *vma, long faultin_page_range(struct mm_struct *mm, unsigned long start, unsigned long end, bool write, int *locked) { - unsigned long nr_pages = (end - start) / PAGE_SIZE; + unsigned long nr_pages = (end - start) / mm_page_size(mm); int gup_flags; long ret; - VM_WARN_ON_ONCE(!PAGE_ALIGNED(start)); - VM_WARN_ON_ONCE(!PAGE_ALIGNED(end)); + VM_WARN_ON_ONCE(!IS_ALIGNED(start, mm_page_size(mm))); + VM_WARN_ON_ONCE(!IS_ALIGNED(end, mm_page_size(mm))); mmap_assert_locked(mm); /* @@ -1968,7 +1983,7 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) } break; } - nend = nstart + ret * PAGE_SIZE; + nend = nstart + ret * mm_page_size(mm); ret = 0; } if (locked) @@ -2047,6 +2062,7 @@ size_t fault_in_writeable(char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; if (unlikely(size == 0)) @@ -2055,7 +2071,7 @@ size_t fault_in_writeable(char __user *uaddr, size_t size) return size; /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) unsafe_put_user(0, (char __user *)cur, out); out: user_write_access_end(); @@ -2116,6 +2132,7 @@ size_t fault_in_safe_writeable(const char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; struct mm_struct *mm = current->mm; bool unlocked = false; @@ -2125,7 +2142,7 @@ size_t fault_in_safe_writeable(const char __user *uaddr, size_t size) mmap_read_lock(mm); /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) if (fixup_user_fault(mm, cur, FAULT_FLAG_WRITE, &unlocked)) break; mmap_read_unlock(mm); @@ -2148,6 +2165,7 @@ size_t fault_in_readable(const char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; volatile char c; @@ -2157,7 +2175,7 @@ size_t fault_in_readable(const char __user *uaddr, size_t size) return size; /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) unsafe_get_user(c, (const char __user *)cur, out); out: user_read_access_end(); @@ -2211,11 +2229,18 @@ struct pages_or_folios { void **entries; }; bool has_folios; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *fragments; +#endif long nr_entries; }; static struct folio *pofs_get_folio(struct pages_or_folios *pofs, long i) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) + return pofs->fragments[i].folio; +#endif if (pofs->has_folios) return pofs->folios[i]; return page_folio(pofs->pages[i]); @@ -2223,22 +2248,56 @@ static struct folio *pofs_get_folio(struct pages_or_folios *pofs, long i) static void pofs_clear_entry(struct pages_or_folios *pofs, long i) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + pofs->fragments[i] = (struct user_page_fragment) {}; + return; + } +#endif pofs->entries[i] = NULL; } static void pofs_unpin(struct pages_or_folios *pofs) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + release_user_fragments(pofs->fragments, pofs->nr_entries, false); + return; + } +#endif if (pofs->has_folios) unpin_folios(pofs->folios, pofs->nr_entries); else unpin_user_pages(pofs->pages, pofs->nr_entries); } +static void pofs_unpin_entry(struct pages_or_folios *pofs, long i) +{ +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + release_user_fragments(&pofs->fragments[i], 1, false); + return; + } +#endif + unpin_folio(pofs_get_folio(pofs, i)); +} + static struct folio *pofs_next_folio(struct folio *folio, struct pages_or_folios *pofs, long *index_ptr) { long i = *index_ptr + 1; +#ifdef CONFIG_MM_SUBPAGE + /* Several adjacent user quarters can pin the same native folio. */ + if (pofs->fragments) { + while (i < pofs->nr_entries && pofs->fragments[i].folio == folio) + i++; + if (i == pofs->nr_entries) + return NULL; + *index_ptr = i; + return pofs->fragments[i].folio; + } +#endif if (!pofs->has_folios && folio_test_large(folio)) { const unsigned long start_pfn = folio_pfn(folio); const unsigned long end_pfn = start_pfn + folio_nr_pages(folio); @@ -2333,9 +2392,9 @@ migrate_longterm_unpinnable_folios(struct list_head *movable_folio_list, * convert the pin on the source folio to a normal * reference. */ - pofs_clear_entry(pofs, i); folio_get(folio); - gup_put_folio(folio, 1, FOLL_PIN); + pofs_unpin_entry(pofs, i); + pofs_clear_entry(pofs, i); if (migrate_device_coherent_folio(folio)) { ret = -EBUSY; @@ -2352,7 +2411,7 @@ migrate_longterm_unpinnable_folios(struct list_head *movable_folio_list, * calling folio_isolate_lru() which takes a reference so the * folio won't be freed if it's migrating. */ - unpin_folio(folio); + pofs_unpin_entry(pofs, i); pofs_clear_entry(pofs, i); } @@ -2458,6 +2517,23 @@ static long check_and_migrate_movable_folios(unsigned long nr_folios, } #endif /* CONFIG_MIGRATION */ +#ifdef CONFIG_MM_SUBPAGE +static long check_and_migrate_movable_fragments(unsigned long count, + struct user_page_fragment *fragments) +{ +#ifdef CONFIG_MIGRATION + struct pages_or_folios pofs = { + .fragments = fragments, + .nr_entries = count, + }; + + return check_and_migrate_movable_pages_or_folios(&pofs); +#else + return 0; +#endif +} +#endif + /* * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which * allows us to process the FOLL_LONGTERM flag. @@ -3133,6 +3209,11 @@ static unsigned long gup_fast(unsigned long start, unsigned long end, int nr_pinned = 0; unsigned seq; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return 0; +#endif + if (!IS_ENABLED(CONFIG_HAVE_GUP_FAST) || !gup_fast_permitted(start, end)) return 0; @@ -3355,6 +3436,452 @@ long pin_user_pages_remote(struct mm_struct *mm, } EXPORT_SYMBOL(pin_user_pages_remote); +#ifdef CONFIG_MM_SUBPAGE +/* 1: acquired, 0: fault required, -EMLINK: read pin needs private backing. */ +static int follow_subpage_fragment(struct vm_area_struct *vma, unsigned long addr, + unsigned int flags, struct user_page_fragment *fragment) +{ + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slot = NULL; + struct folio *folio; + struct page *page; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *ptep, pte; + spinlock_t *ptl; + phys_addr_t phys; + int ret = 0; + bool file_ref = false; + + if (pgd_none(*pgd) || pgd_bad(*pgd)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_mm(mm, *p4d) || p4d_bad_mm(mm, *p4d)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + pud = pud_offset_mm(mm, p4d, addr); + if (pud_none(*pud)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + if (pud_leaf(*pud) || pud_bad(*pud)) + return -EOPNOTSUPP; + pmd = pmd_offset_mm(mm, pud, addr); + if (pmd_none(*pmd)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + if (pmd_leaf(*pmd) || pmd_bad(*pmd)) + return -EOPNOTSUPP; + ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!ptep) + return 0; + pte = ptep_get(ptep); + if (pte_none(pte)) { + ret = PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + goto out; + } + if (!pte_present(pte) || (pte_protnone(pte) && !gup_can_follow_protnone(vma, flags))) + goto out; + phys = pte_phys_mm(mm, pte); + page = vm_normal_page(vma, addr, pte); + if (mm_page_size(mm) > PAGE_SIZE) { + unsigned long offset = (addr & (mm_page_size(mm) - 1)) & PAGE_MASK; + + phys += offset; + if (page) + page += offset >> PAGE_SHIFT; + } + if (!page) { + if (flags & FOLL_DUMP) { + ret = -EFAULT; + goto out; + } + if (!is_zero_pfn(pte_pfn(pte))) { + ret = -EOPNOTSUPP; + goto out; + } + if (flags & FOLL_WRITE) + goto out; + folio = page_folio(pfn_to_page(PHYS_PFN(phys))); + if (flags & FOLL_GET) + folio_get(folio); + } else { + folio = page_folio(page); + /* Coarse leaves use ordinary native-page ownership, not slots. */ + if (mm_page_size(mm) > PAGE_SIZE) { + if ((flags & FOLL_WRITE) && + !can_follow_write_pte(pte, page, vma, flags)) + goto out; + if (!pte_write(pte) && gup_must_unshare(vma, flags, page)) { + ret = -EMLINK; + goto out; + } + ret = try_grab_folio(folio, 1, flags); + if (ret) + goto out; + file_ref = true; + goto acquired; + } + if (!folio_test_anon(folio)) { + if (vma->vm_file && folio_mapping(folio) != vma->vm_file->f_mapping) { + ret = -EOPNOTSUPP; + goto out; + } + if ((flags & FOLL_WRITE) && !pte_write(pte)) + goto out; + /* Private non-anonymous read pins must first acquire COW ownership. */ + if ((flags & FOLL_PIN) && is_cow_mapping(vma->vm_flags)) { + ret = -EMLINK; + goto out; + } + ret = try_grab_folio(folio, 1, flags); + if (ret) + goto out; + file_ref = true; + goto acquired; + } + if (folio_test_ksm(folio)) { + ret = -EOPNOTSUPP; + goto out; + } + slot = mm_subpage_get_from_phys(phys); + if (!slot) { + ret = -EFAULT; + goto out; + } + if ((flags & FOLL_WRITE) && !pte_write(pte)) { + bool force = (flags & FOLL_FORCE) && + !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE | VM_WRITE)) && + (vma->vm_flags & VM_MAYWRITE) && mm_subpage_is_exclusive(slot) && + !pte_needs_soft_dirty_wp(vma, pte) && !userfaultfd_pte_wp(vma, pte); + + if (!force) + goto out; + } + if ((flags & FOLL_PIN) && !mm_subpage_is_exclusive(slot)) { + ret = flags & FOLL_WRITE ? 0 : -EMLINK; + goto out; + } + if (flags & FOLL_PIN) { + ret = mm_subpage_pin(slot); + if (ret) { + if (ret == -EAGAIN) + ret = 0; + goto out; + } + } + } +acquired: + if (slot && (flags & FOLL_WRITE)) + mm_subpage_cancel_lazyfree(slot); + ret = flags & FOLL_PIN ? arch_make_folio_accessible(folio) : 0; + if (ret) { + if (slot) + mm_subpage_unpin(slot); + else if (file_ref) + gup_put_folio(folio, 1, flags); + goto out; + } + if ((flags & FOLL_TOUCH) && (flags & FOLL_WRITE) && + !pte_dirty(pte) && !folio_test_dirty(folio)) + folio_mark_dirty(folio); + folio_mark_accessed(folio); + flush_anon_page(vma, &folio->page, addr); + flush_dcache_folio(folio); + fragment->folio = folio; + fragment->subpage = slot; + fragment->pinned = flags & FOLL_PIN; + if (flags & FOLL_GET) + slot = NULL; /* The returned record consumes the lookup reference. */ + fragment->offset = phys - PFN_PHYS(folio_pfn(folio)) + + (addr & (mm_user_fragment_size(mm) - 1)); + ret = 1; +out: + if (slot) + mm_subpage_put(slot); /* Lookup reference; success retains a separate pin. */ + pte_unmap_unlock(ptep, ptl); + return ret; +} + +/* No references escape: only ensure each user leaf can be followed. */ +static long prefault_subpage_range(struct mm_struct *mm, unsigned long start, + unsigned long nr_pages, unsigned int flags, int *locked) +{ + unsigned long done = 0; + long ret = 0; + + mmap_assert_locked(mm); + start = untagged_addr_remote(mm, start); + while (done < nr_pages) { + struct user_page_fragment unused; + struct vm_area_struct *vma = vma_lookup(mm, start); + + if (!vma) { + ret = flags & FOLL_MADV_POPULATE ? -ENOMEM : -EFAULT; + break; + } + ret = check_vma_flags(vma, flags); + if (ret) { + if (flags & FOLL_MADV_POPULATE) + ret = -EINVAL; + break; + } + if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { + ret = flags & FOLL_MADV_POPULATE ? -EINVAL : -EOPNOTSUPP; + break; + } + if (gup_signal_pending(flags)) { + ret = -EINTR; + break; + } + cond_resched(); + ret = follow_subpage_fragment(vma, start, flags, &unused); + if (!ret) { + ret = faultin_page(vma, start, flags, false, locked); + if (!ret) + continue; + if (ret == -EBUSY || ret == -EAGAIN) + ret = 0; /* Preserve GUP's caller-managed retry contract. */ + break; + } + if (ret < 0) + break; + done++; + start += mm_page_size(mm); + } + if (ret == -EOPNOTSUPP && (flags & FOLL_MADV_POPULATE)) + ret = -EINVAL; + return done ? (long)done : ret; +} + +static long user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity, bool pin, + struct vm_area_struct **vma_out, bool dump) +{ + const unsigned int allowed = FOLL_WRITE | FOLL_FORCE | FOLL_NOFAULT | + FOLL_NOWAIT | FOLL_INTERRUPTIBLE | FOLL_LONGTERM; + unsigned int internal = flags | FOLL_TOUCH | FOLL_REMOTE | + (pin ? FOLL_PIN : FOLL_GET) | (dump ? FOLL_DUMP : 0); + unsigned long end, count = 0; + int locked = 0; + long ret; + + if (!length) + return 0; + if ((flags & FOLL_LONGTERM) && !pin) + return -EINVAL; + if (!fragments || !capacity || capacity > LONG_MAX || (flags & ~allowed)) + return -EINVAL; + if (vma_out) { + mmap_assert_locked(mm); + *vma_out = NULL; + } else { + internal |= FOLL_UNLOCKABLE; + ret = mmap_read_lock_killable(mm); + if (ret) + return ret; + } + locked = 1; + start = untagged_addr_remote(mm, start); + if (check_add_overflow(start, length, &end)) { + ret = -EFAULT; + goto out; + } + if (pin) + mm_set_has_pinned_flag(mm); + while (start < end && count < capacity) { + unsigned long bytes = min(end - start, + mm_user_fragment_size(mm) - (start & (mm_user_fragment_size(mm) - 1))); + struct vm_area_struct *vma; + + if (!locked) { + ret = mmap_read_lock_killable(mm); + if (ret) + break; + locked = 1; + } + if (gup_signal_pending(internal)) { + ret = -EINTR; + break; + } + cond_resched(); + if (mm_page_size(mm) == PAGE_SIZE) { + struct page *page; + + if (dump) + ret = __get_user_pages_locked(mm, start, 1, &page, &locked, internal); + else if (pin) + ret = pin_user_pages_remote(mm, start, 1, flags, &page, + vma_out ? NULL : &locked); + else + ret = get_user_pages_remote(mm, start, 1, flags, &page, + vma_out ? NULL : &locked); + if (ret != 1) { + if (!ret) + ret = -EAGAIN; + break; + } + fragments[count].folio = page_folio(page); + fragments[count].subpage = NULL; + fragments[count].pinned = pin; + fragments[count].offset = folio_page_idx(page_folio(page), page) * PAGE_SIZE + + offset_in_page(start); + } else { + vma = gup_vma_lookup(mm, start); + ret = vma ? check_vma_flags(vma, internal) : -EFAULT; + if (ret) + break; + /* Huge and special/device mappings need separate integration. */ + if (is_vm_hugetlb_page(vma) || + (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP | VM_IO))) { + ret = -EOPNOTSUPP; + break; + } + ret = follow_subpage_fragment(vma, start, internal, &fragments[count]); + if (!ret || ret == -EMLINK) { + ret = faultin_page(vma, start, internal, ret == -EMLINK, &locked); + if (!ret) + continue; + if (!locked && (ret == -EBUSY || ret == -EAGAIN)) { + internal |= FOLL_TRIED; + continue; + } + if (ret == -EBUSY) + ret = -EAGAIN; + break; + } + if (ret < 0) + break; + } + if (vma_out) + *vma_out = vma_lookup(mm, start); + fragments[count++].length = bytes; + start += bytes; + internal &= ~FOLL_TRIED; + } +out: + if (locked && !vma_out) + mmap_read_unlock(mm); + return count ? (long)count : ret; +} + +long pin_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity) +{ + unsigned int saved; + long ret, count; + + if (!(flags & FOLL_LONGTERM) || mm_page_size(mm) == PAGE_SIZE) + return user_fragments_remote(mm, start, length, flags, fragments, + capacity, true, NULL, false); + /* Match native long-term GUP: fault outside movable zones, migrate any + * existing unpinnable backing, then retry with fresh typed references. + */ + saved = memalloc_pin_save(); + do { + count = user_fragments_remote(mm, start, length, flags, fragments, + capacity, true, NULL, false); + if (count <= 0) { + ret = count; + break; + } + ret = check_and_migrate_movable_fragments(count, fragments); + } while (ret == -EAGAIN); + memalloc_pin_restore(saved); + return ret ? ret : count; +} +EXPORT_SYMBOL_GPL(pin_user_fragments_remote); + +long get_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity) +{ + return user_fragments_remote(mm, start, length, flags, fragments, capacity, + false, NULL, false); +} + +int get_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma) +{ + int ret; + + /* As with get_user_page_vma_remote(), preserve the caller's VMA lock. */ + if (!vma) + return -EINVAL; + *vma = NULL; + if (flags & FOLL_NOWAIT) + return -EINVAL; + ret = user_fragments_remote(mm, start, length, flags, fragment, 1, false, vma, false); + if (ret > 0 && !*vma) { + release_user_fragments(fragment, 1, false); + return -EFAULT; + } + return ret; +} + +int pin_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma) +{ + int ret; + + if (!vma) + return -EINVAL; + *vma = NULL; + if (flags & (FOLL_NOWAIT | FOLL_LONGTERM)) + return -EINVAL; + ret = user_fragments_remote(mm, start, length, flags, fragment, 1, true, vma, false); + if (ret > 0 && !*vma) { + release_user_fragments(fragment, 1, false); + return -EFAULT; + } + return ret; +} + +#ifdef CONFIG_ELF_CORE +/* Like get_dump_page(), but preserves the selected user-page ownership. */ +int get_dump_fragment(unsigned long addr, struct user_page_fragment *fragment) +{ + return user_fragments_remote(current->mm, addr, mm_user_fragment_size(current->mm), + FOLL_FORCE, fragment, 1, false, NULL, true); +} +#endif + +void release_user_fragments(struct user_page_fragment *fragments, + unsigned long count, bool dirty) +{ + unsigned long i; + + for (i = 0; i < count; i++) { + struct user_page_fragment *fragment = &fragments[i]; + struct folio *folio = fragment->folio; + + if (!folio) + continue; + if (dirty && !is_zero_folio(folio) && !folio_test_dirty(folio)) { + folio_lock(folio); + folio_mark_dirty(folio); + folio_unlock(folio); + } + if (fragment->subpage) { + if (dirty) + mm_subpage_cancel_lazyfree(fragment->subpage); + if (fragment->pinned) + mm_subpage_unpin(fragment->subpage); + else + mm_subpage_put(fragment->subpage); + } else if (fragment->pinned) { + unpin_folio(folio); + } else { + folio_put(folio); + } + *fragment = (struct user_page_fragment) {}; + } +} +EXPORT_SYMBOL_GPL(release_user_fragments); +#endif + /** * pin_user_pages() - pin user pages in memory for use by other devices * @@ -3407,6 +3934,25 @@ long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, } EXPORT_SYMBOL(pin_user_pages_unlocked); +/* Direct cache pins have no PTE fault to publish fine shmem exposure. */ +static int memfd_pin_folio(struct file *memfd, struct folio *folio) +{ + bool lock = IS_ENABLED(CONFIG_MM_SUBPAGE) && + IS_ENABLED(CONFIG_USERFAULTFD) && shmem_file(memfd); + int ret; + + /* Serialize both acquisition and exposure against partial hole punch. */ + if (lock) + folio_lock(folio); + ret = try_grab_folio(folio, 1, FOLL_PIN); + if (lock) { + if (!ret) + shmem_uffd_mark_mapped(folio, 0, folio_size(folio)); + folio_unlock(folio); + } + return ret; +} + /** * memfd_pin_folios() - pin folios associated with a memfd * @memfd: the memfd whose folios are to be pinned @@ -3492,7 +4038,7 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, for (i = 0; i < nr_found; i++) { folio = fbatch.folios[i]; - if (try_grab_folio(folio, 1, FOLL_PIN)) { + if (memfd_pin_folio(memfd, folio)) { folio_batch_release(&fbatch); ret = -EINVAL; goto err; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index dd93f1b0c44a2..2ab333c54c987 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4118,6 +4118,12 @@ static int __folio_split(struct folio *folio, unsigned int new_order, } anon_vma_lock_write(anon_vma); mapping = NULL; +#ifdef CONFIG_MM_SUBPAGE + if (new_order < anon_vma->min_folio_order) { + ret = -EBUSY; + goto out_unlock; + } +#endif } else { unsigned int min_order; gfp_t gfp; @@ -4394,7 +4400,7 @@ int folio_split(struct folio *folio, unsigned int new_order, unsigned int min_order_for_split(struct folio *folio) { if (folio_test_anon(folio)) - return 0; + return folio_anon_min_order(folio); /* * If the folio got truncated, we don't know the previous mapping and diff --git a/mm/internal.h b/mm/internal.h index 5a2ddcf68e0b6..23abe35e53589 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -305,6 +305,50 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) return pte_mkold(pte); } +#ifdef CONFIG_MM_SUBPAGE +/* Batch contiguous user leaves without confusing their offsets with PFNs. */ +static inline unsigned int folio_subpage_pte_batch_flags(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t *ptentp, unsigned int max_nr, fpb_t flags) +{ + struct mm_struct *mm = vma->vm_mm; + pte_t pte = *ptentp; + phys_addr_t offset = pte_phys_mm(mm, pte) - PFN_PHYS(folio_pfn(folio)); + pte_t expected; + unsigned int nr; + + VM_BUG_ON_FOLIO(offset >= folio_size(folio), folio); + max_nr = min_t(unsigned long, max_nr, (folio_size(folio) - offset) >> mm_page_shift(mm)); + VM_BUG_ON_FOLIO(!max_nr, folio); + nr = min(max_nr, pte_batch_hint(ptep, pte)); + expected = pte_advance_pfn_mm(mm, __pte_batch_clear_ignored(pte, flags), nr); + while (nr < max_nr) { + unsigned int batch; + + pte = ptep_get(ptep + nr); + if (!pte_same(__pte_batch_clear_ignored(pte, flags), expected)) + break; + if ((flags & FPB_MERGE_WRITE) && pte_write(pte)) + *ptentp = pte_mkwrite(*ptentp, vma); + if (flags & FPB_MERGE_YOUNG_DIRTY) { + if (pte_young(pte)) + *ptentp = pte_mkyoung(*ptentp); + if (pte_dirty(pte)) + *ptentp = pte_mkdirty(*ptentp); + } + batch = min(max_nr - nr, pte_batch_hint(ptep + nr, pte)); + nr += batch; + expected = pte_advance_pfn_mm(mm, expected, batch); + } + return nr; +} + +static inline unsigned int folio_subpage_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t pte, unsigned int max_nr, fpb_t flags) +{ + return folio_subpage_pte_batch_flags(vma, folio, ptep, &pte, max_nr, flags); +} +#endif + /** * folio_pte_batch_flags - detect a PTE batch for a large folio * @folio: The large folio to detect a PTE batch for. @@ -416,6 +460,10 @@ static inline pte_t pte_move_swp_offset(pte_t pte, long delta) if (pte_swp_uffd_wp(pte)) new = pte_swp_mkuffd_wp(new); +#ifdef CONFIG_MM_SUBPAGE + new = pte_swp_set_subpage_offset(new, pte_swp_subpage_offset(pte)); +#endif + return new; } @@ -1158,6 +1206,11 @@ folio_within_range(struct folio *folio, struct vm_area_struct *vma, end = vma->vm_end; pgoff = folio_pgoff(folio); + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + addr = vma_address_at_offset(vma, (struct vm_page_offset) { .index = pgoff }); + return !IS_ERR_VALUE(addr) && addr >= start && addr <= end && + end - addr >= folio_size(folio); + } /* if folio start address is not in vma range */ if (!in_range(pgoff, vma->vm_pgoff, vma_pglen)) @@ -1240,6 +1293,8 @@ static inline unsigned long vma_address(const struct vm_area_struct *vma, if (pgoff >= vma->vm_pgoff) { address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + if (pgoff != vma->vm_pgoff) + address -= vma_subpage_offset(vma); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address >= vma->vm_end) address = -EFAULT; @@ -1262,15 +1317,24 @@ static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) pgoff_t pgoff; unsigned long address; +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) + return min(pvmw->address + mm_page_size(vma->vm_mm), vma->vm_end); +#endif + /* Common case, plus ->pgoff is invalid for KSM */ - if (pvmw->nr_pages == 1) - return pvmw->address + PAGE_SIZE; + if (pvmw->nr_pages == 1 && mm_page_size(vma->vm_mm) >= PAGE_SIZE) + return min((pvmw->address & mm_page_mask(vma->vm_mm)) + + mm_page_size(vma->vm_mm), vma->vm_end); pgoff = pvmw->pgoff + pvmw->nr_pages; address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + address -= vma_subpage_offset(vma); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address > vma->vm_end) address = vma->vm_end; + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + address = min(mm_page_align(vma->vm_mm, address), vma->vm_end); return address; } @@ -1900,7 +1964,7 @@ static inline bool clear_flush_young_ptes_notify(struct vm_area_struct *vma, young = clear_flush_young_ptes(vma, addr, ptep, nr); young |= mmu_notifier_clear_flush_young(vma->vm_mm, addr, - addr + nr * PAGE_SIZE); + addr + nr * mm_page_size(vma->vm_mm)); return young; } @@ -1920,7 +1984,8 @@ static inline bool test_and_clear_young_ptes_notify(struct vm_area_struct *vma, bool young; young = test_and_clear_young_ptes(vma, addr, ptep, nr); - young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + nr * PAGE_SIZE); + young |= mmu_notifier_clear_young(vma->vm_mm, addr, + addr + nr * mm_page_size(vma->vm_mm)); return young; } @@ -1952,4 +2017,14 @@ static inline int get_sysctl_max_map_count(void) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages); +#ifdef CONFIG_MM_SUBPAGE +struct iov_iter; +int process_vm_rw_fragments(struct mm_struct *mm, unsigned long addr, + unsigned long len, struct iov_iter *iter, bool write); +#endif + +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SWAP) +vm_fault_t do_swap_coarse_page(struct vm_fault *vmf); +#endif + #endif /* __MM_INTERNAL_H */ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index d6e04041f5dc1..ad4832f64fc30 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1066,28 +1066,31 @@ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm, } static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_struct *mm, - struct collapse_control *cc) + struct collapse_control *cc, unsigned int order) { gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() : GFP_TRANSHUGE); int node = collapse_find_target_node(cc); struct folio *folio; - folio = __folio_alloc(gfp, HPAGE_PMD_ORDER, node, &cc->alloc_nmask); + folio = __folio_alloc(gfp, order, node, &cc->alloc_nmask); if (!folio) { *foliop = NULL; - count_vm_event(THP_COLLAPSE_ALLOC_FAILED); + if (is_pmd_order(order)) + count_vm_event(THP_COLLAPSE_ALLOC_FAILED); return SCAN_ALLOC_HUGE_PAGE_FAIL; } - count_vm_event(THP_COLLAPSE_ALLOC); + if (is_pmd_order(order)) + count_vm_event(THP_COLLAPSE_ALLOC); if (unlikely(mem_cgroup_charge(folio, mm, gfp))) { folio_put(folio); *foliop = NULL; return SCAN_CGROUP_CHARGE_FAIL; } - count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1); + if (is_pmd_order(order)) + count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1); *foliop = folio; return SCAN_SUCCEED; @@ -1116,7 +1119,7 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a */ mmap_read_unlock(mm); - result = alloc_charge_folio(&folio, mm, cc); + result = alloc_charge_folio(&folio, mm, cc, HPAGE_PMD_ORDER); if (result != SCAN_SUCCEED) goto out_nolock; @@ -1880,22 +1883,33 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) * + unlock and free huge page; */ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, - struct file *file, pgoff_t start, struct collapse_control *cc) + struct file *file, pgoff_t start, struct collapse_control *cc, + unsigned int order) { struct address_space *mapping = file->f_mapping; struct page *dst; struct folio *folio, *tmp, *new_folio; - pgoff_t index = 0, end = start + HPAGE_PMD_NR; + unsigned long nr_pages = 1UL << order; + pgoff_t index = 0, end = start + nr_pages; LIST_HEAD(pagelist); - XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); + XA_STATE_ORDER(xas, &mapping->i_pages, start, order); enum scan_result result = SCAN_SUCCEED; int nr_none = 0; bool is_shmem = shmem_file(file); VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); - VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); + VM_BUG_ON(start & (nr_pages - 1)); - result = alloc_charge_folio(&new_folio, mm, cc); + if (is_pmd_order(order)) { + result = alloc_charge_folio(&new_folio, mm, cc, order); + } else { + /* Sub-PMD user leaves must respect the shmem inode's allocation policy. */ + new_folio = shmem_alloc_collapse_folio(mm, mapping->host, start, order); + if (IS_ERR(new_folio)) { + new_folio = NULL; + result = SCAN_ALLOC_HUGE_PAGE_FAIL; + } + } if (result != SCAN_SUCCEED) goto out; @@ -2024,7 +2038,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * we locked the first folio, then a THP might be there already. * This will be discovered on the first iteration. */ - if (is_pmd_order(folio_order(folio))) { + if (folio_order(folio) >= order) { result = SCAN_PTE_MAPPED_HUGEPAGE; goto out_unlock; } @@ -2149,6 +2163,8 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, while (index < folio->index) { clear_highpage(dst); + if (is_shmem) + shmem_uffd_copy_page(dst, NULL); index++; dst++; } @@ -2158,12 +2174,16 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, result = SCAN_COPY_MC; goto rollback; } + if (is_shmem) + shmem_uffd_copy_page(dst, folio_page(folio, i)); index++; dst++; } } while (index < end) { clear_highpage(dst); + if (is_shmem) + shmem_uffd_copy_page(dst, NULL); index++; dst++; } @@ -2205,8 +2225,13 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * not be able to observe any missing pages due to the * previously inserted retry entries. */ + /* + * A required user leaf populates its whole granule, as a native + * fault does. Only optional PMD collapse must preserve cache holes + * merely because another VMA has missing-page registration. + */ vma_interval_tree_foreach(vma, &mapping->i_mmap, start, end) { - if (userfaultfd_missing(vma)) { + if (is_pmd_order(order) && userfaultfd_missing(vma)) { result = SCAN_EXCEED_NONE_PTE; goto immap_locked; } @@ -2229,12 +2254,13 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, } if (is_shmem) { - lruvec_stat_mod_folio(new_folio, NR_SHMEM, HPAGE_PMD_NR); - lruvec_stat_mod_folio(new_folio, NR_SHMEM_THPS, HPAGE_PMD_NR); - } else { - lruvec_stat_mod_folio(new_folio, NR_FILE_THPS, HPAGE_PMD_NR); + lruvec_stat_mod_folio(new_folio, NR_SHMEM, nr_pages); + if (is_pmd_order(order)) + lruvec_stat_mod_folio(new_folio, NR_SHMEM_THPS, nr_pages); + } else if (is_pmd_order(order)) { + lruvec_stat_mod_folio(new_folio, NR_FILE_THPS, nr_pages); } - lruvec_stat_mod_folio(new_folio, NR_FILE_PAGES, HPAGE_PMD_NR); + lruvec_stat_mod_folio(new_folio, NR_FILE_PAGES, nr_pages); /* * Mark new_folio as uptodate before inserting it into the @@ -2242,14 +2268,14 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * unwritten page. */ folio_mark_uptodate(new_folio); - folio_ref_add(new_folio, HPAGE_PMD_NR - 1); + folio_ref_add(new_folio, nr_pages - 1); if (is_shmem) folio_mark_dirty(new_folio); folio_add_lru(new_folio); /* Join all the small entries into a single multi-index entry. */ - xas_set_order(&xas, start, HPAGE_PMD_ORDER); + xas_set_order(&xas, start, order); xas_store(&xas, new_folio); WARN_ON_ONCE(xas_error(&xas)); xas_unlock_irq(&xas); @@ -2258,9 +2284,11 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * Remove pte page tables, so we can re-fault the page as huge. * If MADV_COLLAPSE, adjust result to call try_collapse_pte_mapped_thp(). */ - retract_page_tables(mapping, start); - if (cc && !cc->is_khugepaged) - result = SCAN_PTE_MAPPED_HUGEPAGE; + if (is_pmd_order(order)) { + retract_page_tables(mapping, start); + if (cc && !cc->is_khugepaged) + result = SCAN_PTE_MAPPED_HUGEPAGE; + } folio_unlock(new_folio); /* @@ -2317,7 +2345,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, folio_put(new_folio); out: VM_BUG_ON(!list_empty(&pagelist)); - trace_mm_khugepaged_collapse_file(mm, new_folio, index, addr, is_shmem, file, HPAGE_PMD_NR, result); + trace_mm_khugepaged_collapse_file(mm, new_folio, index, addr, is_shmem, file, nr_pages, result); return result; } @@ -2420,7 +2448,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, result = SCAN_EXCEED_NONE_PTE; count_vm_event(THP_SCAN_EXCEED_NONE_PTE); } else { - result = collapse_file(mm, addr, file, start, cc); + result = collapse_file(mm, addr, file, start, cc, HPAGE_PMD_ORDER); } } @@ -2831,6 +2859,47 @@ static int madvise_collapse_errno(enum scan_result r) } } +/* + * Promote the cache backing for one larger-than-native user leaf. The caller + * owns mm/file references; no folio or page-table lock may be held. Start is + * a native file index. This sub-PMD shmem path may run under the fault lock: + * it neither retracts page tables (which requires mmap write locking) nor + * invokes filesystem writeback. Source folios are trylocked and rolled back + * on contention. Ordinary PMD collapse retains its existing lock policy. + */ +int collapse_file_user_page(struct mm_struct *mm, struct file *file, pgoff_t start) +{ + struct collapse_control *cc; + unsigned int order; + enum scan_result result; + + if (mm_page_shift(mm) <= PAGE_SHIFT || !shmem_file(file)) + return -EOPNOTSUPP; + order = mm_page_shift(mm) - PAGE_SHIFT; + /* Shmem's order limit reflects THP policy, not an architectural leaf limit. */ + if (order >= HPAGE_PMD_ORDER || order > MAX_PAGECACHE_ORDER || + !IS_ALIGNED(start, 1UL << order) || start > ULONG_MAX - (1UL << order)) + return -EINVAL; + cc = kzalloc(sizeof(*cc), GFP_KERNEL); + if (!cc) + return -ENOMEM; + lru_add_drain_all(); + result = collapse_file(mm, 0, file, start, cc, order); + kfree(cc); + if (result == SCAN_SUCCEED || result == SCAN_PTE_MAPPED_HUGEPAGE) + return 0; + /* Retained references cannot be displaced by replacing their cache folio. */ + if (result == SCAN_PAGE_COUNT) + return -EBUSY; + if (result == SCAN_CGROUP_CHARGE_FAIL) + return -ENOMEM; + if (result == SCAN_COPY_MC) + return -EHWPOISON; + if (result == SCAN_TRUNCATED) + return -EAGAIN; + return madvise_collapse_errno(result); +} + int madvise_collapse(struct vm_area_struct *vma, unsigned long start, unsigned long end, bool *lock_dropped) { diff --git a/mm/ksm.c b/mm/ksm.c index 7d5b76478f0b6..cd0398e972941 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -804,6 +804,10 @@ static struct page *get_mergeable_page(struct ksm_rmap_item *rmap_item) struct folio_walk fw; struct folio *folio; + /* The KSM comparison and replacement paths still require native leaves. */ + if (mm_page_size(mm) != PAGE_SIZE) + return NULL; + mmap_read_lock(mm); vma = find_mergeable_vma(mm, addr); if (!vma) @@ -1338,7 +1342,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct folio *folio, /* See folio_try_share_anon_rmap_pte(): clear PTE first. */ if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, &folio->page)) { + folio_try_share_anon_rmap_pte(folio, &folio->page, vma)) { set_pte_at(mm, pvmw.address, pvmw.pte, entry); goto out_unlock; } @@ -3206,7 +3210,8 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc) if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, addr, rwc->arg)) { + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = addr }, rwc->arg)) { anon_vma_unlock_read(anon_vma); return; } diff --git a/mm/madvise.c b/mm/madvise.c index 5665785e5ac38..1d15da9d1200d 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -193,7 +194,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, spinlock_t *ptl; unsigned long addr; - for (addr = start; addr < end; addr += PAGE_SIZE) { + for (addr = start; addr < end; addr += mm_page_size(vma->vm_mm)) { pte_t pte; softleaf_t entry; struct folio *folio; @@ -236,7 +237,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma, struct address_space *mapping) { XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start)); - pgoff_t end_index = linear_page_index(vma, end) - 1; + pgoff_t end_index = linear_page_index(vma, end - 1); struct folio *folio; struct swap_iocb *splug = NULL; @@ -252,8 +253,11 @@ static void shmem_swapin_range(struct vm_area_struct *vma, if (!softleaf_is_swap(entry)) continue; - addr = vma->vm_start + - ((xas.xa_index - vma->vm_pgoff) << PAGE_SHIFT); + /* The first cache page may start before this VMA's byte offset. */ + addr = xas.xa_index == linear_page_index(vma, start) ? start : + vma_address_at_offset(vma, (struct vm_page_offset) { + .index = xas.xa_index, + }); xas_pause(&xas); rcu_read_unlock(); @@ -318,7 +322,7 @@ static long madvise_willneed(struct madvise_behavior *madv_behavior) mark_mmap_lock_dropped(madv_behavior); get_file(file); offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); mmap_read_unlock(mm); vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED); fput(file); @@ -340,12 +344,18 @@ static inline bool can_do_file_pageout(struct vm_area_struct *vma) file_permission(vma->vm_file, MAY_WRITE) == 0; } -static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end, +static inline int madvise_folio_pte_batch(struct vm_area_struct *vma, + unsigned long addr, unsigned long end, struct folio *folio, pte_t *ptep, pte_t *ptentp) { - int max_nr = (end - addr) / PAGE_SIZE; + int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch_flags(vma, folio, ptep, ptentp, + max_nr, FPB_MERGE_YOUNG_DIRTY); +#endif return folio_pte_batch_flags(folio, NULL, ptep, ptentp, max_nr, FPB_MERGE_YOUNG_DIRTY); } @@ -376,7 +386,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, #ifdef CONFIG_TRANSPARENT_HUGEPAGE if (pmd_trans_huge(*pmd)) { pmd_t orig_pmd; - unsigned long next = pmd_addr_end(addr, end); + unsigned long next = pmd_addr_end_mm(mm, addr, end); tlb_change_page_size(tlb, HPAGE_PMD_SIZE); ptl = pmd_trans_huge_lock(pmd, vma); @@ -446,14 +456,14 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, regular_folio: #endif - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, mm_page_size(mm)); restart: start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); if (!start_pte) return 0; flush_tlb_batched_pending(mm); lazy_mmu_mode_enable(); - for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) { + for (; addr < end; pte += nr, addr += nr * mm_page_size(mm)) { nr = 1; ptent = ptep_get(pte); @@ -484,9 +494,10 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, * fail to split a folio, leave it in place and advance to the * next pte in the range. */ - if (folio_test_large(folio)) { - nr = madvise_folio_pte_batch(addr, end, folio, pte, &ptent); - if (nr < folio_nr_pages(folio)) { + if (folio_test_large(folio) || mm_page_size(mm) != PAGE_SIZE) { + nr = madvise_folio_pte_batch(vma, addr, end, folio, pte, &ptent); + if (folio_test_large(folio) && + nr < (folio_size(folio) >> mm_page_shift(mm))) { int err; if (folio_maybe_mapped_shared(folio)) @@ -521,7 +532,8 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, * number of pages, it must be exclusive. */ if (!folio_test_lru(folio) || - folio_mapcount(folio) != folio_nr_pages(folio)) + folio_mapcount(folio) != + (mm_page_size(mm) != PAGE_SIZE ? nr : folio_nr_pages(folio))) continue; if (pageout_anon_only_filter && !folio_test_anon(folio)) @@ -649,6 +661,72 @@ static long madvise_pageout(struct madvise_behavior *madv_behavior) return 0; } +#ifdef CONFIG_MM_SUBPAGE +static int madvise_free_subpage_range(pmd_t *pmd, unsigned long addr, + unsigned long end, struct mm_walk *walk) +{ + struct vm_area_struct *vma = walk->vma; + struct mm_struct *mm = vma->vm_mm; + struct mmu_gather *tlb = walk->private; + spinlock_t *ptl; + pte_t *base, *ptep; + long swapped = 0; + + base = ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!base) + return 0; + flush_tlb_batched_pending(mm); + for (; addr < end; addr += mm_page_size(mm), ptep++) { + pte_t pte = ptep_get(ptep); + struct mm_subpage *slot; + struct folio *folio; + struct page *page; + + if (!pte_present(pte)) { + softleaf_t entry = softleaf_from_pte(pte); + + if (softleaf_is_swap(entry)) { + swap_subpage_put(entry, + pte_swp_subpage_offset(pte), true); + pte_clear_not_present_full(mm, addr, ptep, tlb->fullmm); + swapped--; + } + continue; + } + page = vm_normal_page(vma, addr, pte); + if (!page) + continue; + folio = page_folio(page); + if (!folio_test_anon(folio) || !folio_trylock(folio)) + continue; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (!slot) + goto unlock; + if (folio_test_swapcache(folio) && !folio_free_swap(folio)) + goto put; + if (!mm_subpage_mark_lazyfree(slot)) + goto put; + clear_young_dirty_ptes(vma, addr, ptep, 1, + CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY); + tlb_remove_tlb_entry(tlb, ptep, addr); + /* Partial hints must never clear a neighbor's dirty state. */ + if (mm_subpage_all_lazyfree(slot)) { + folio_clear_dirty(folio); + folio_mark_lazyfree(folio); + } +put: + mm_subpage_put(slot); +unlock: + folio_unlock(folio); + } + if (swapped) + add_mm_counter(mm, MM_SWAPENTS, swapped); + pte_unmap_unlock(base, ptl); + cond_resched(); + return 0; +} +#endif + static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct mm_walk *walk) @@ -664,6 +742,10 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, unsigned long next; int nr, max_nr; + #ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) + return madvise_free_subpage_range(pmd, addr, end, walk); +#endif next = pmd_addr_end(addr, end); if (pmd_trans_huge(*pmd)) if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next)) @@ -714,7 +796,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, * next pte in the range. */ if (folio_test_large(folio)) { - nr = madvise_folio_pte_batch(addr, end, folio, pte, &ptent); + nr = madvise_folio_pte_batch(vma, addr, end, folio, pte, &ptent); if (nr < folio_nr_pages(folio)) { int err; @@ -988,7 +1070,7 @@ static long madvise_populate(struct madvise_behavior *madv_behavior) return -ENOMEM; } } - start += pages * PAGE_SIZE; + start += pages * mm_page_size(mm); } return 0; } @@ -1022,7 +1104,7 @@ static long madvise_remove(struct madvise_behavior *madv_behavior) return -EACCES; offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); /* * Filesystem's fallocate may need to take i_rwsem. We need to @@ -1182,7 +1264,7 @@ static long madvise_guard_install(struct madvise_behavior *madv_behavior) if (err == 0) { unsigned long nr_expected_pages = - PHYS_PFN(range->end - range->start); + (range->end - range->start) >> mm_page_shift(vma->vm_mm); VM_WARN_ON(nr_pages != nr_expected_pages); return 0; @@ -1833,16 +1915,17 @@ static void madvise_finish_tlb(struct madvise_behavior *madv_behavior) tlb_finish_mmu(madv_behavior->tlb); } -static bool is_valid_madvise(unsigned long start, size_t len_in, int behavior) +static bool is_valid_madvise(struct mm_struct *mm, unsigned long start, + size_t len_in, int behavior) { size_t len; if (!madvise_behavior_valid(behavior)) return false; - if (!PAGE_ALIGNED(start)) + if (!IS_ALIGNED(start, mm_page_size(mm))) return false; - len = PAGE_ALIGN(len_in); + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero */ if (len_in && !len) @@ -1865,14 +1948,29 @@ static bool is_valid_madvise(unsigned long start, size_t len_in, int behavior) * operation. This function returns true in the cases, otherwise false. In * the former case we store an error on @err. */ -static bool madvise_should_skip(unsigned long start, size_t len_in, +static bool madvise_should_skip(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior, int *err) { - if (!is_valid_madvise(start, len_in, behavior)) { + if (!is_valid_madvise(mm, start, len_in, behavior)) { *err = -EINVAL; return true; } - if (start + PAGE_ALIGN(len_in) == start) { +#ifdef CONFIG_MM_SUBPAGE + /* These engines still assume whole native folios/tables. */ + if (mm_page_size(mm) != PAGE_SIZE) { + switch (behavior) { + case MADV_MERGEABLE: + case MADV_UNMERGEABLE: + case MADV_HUGEPAGE: + case MADV_COLLAPSE: + case MADV_HWPOISON: + case MADV_SOFT_OFFLINE: + *err = -EOPNOTSUPP; + return true; + } + } +#endif + if (start + mm_page_align(mm, len_in) == start) { *err = 0; return true; } @@ -1919,7 +2017,7 @@ static int madvise_do_behavior(unsigned long start, size_t len_in, } range->start = get_untagged_addr(madv_behavior->mm, start); - range->end = range->start + PAGE_ALIGN(len_in); + range->end = range->start + mm_page_align(madv_behavior->mm, len_in); blk_start_plug(&plug); if (is_madvise_populate(madv_behavior)) @@ -2012,7 +2110,7 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh .tlb = &tlb, }; - if (madvise_should_skip(start, len_in, behavior, &error)) + if (madvise_should_skip(mm, start, len_in, behavior, &error)) return error; error = madvise_lock(&madv_behavior); if (error) @@ -2055,7 +2153,7 @@ static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter, size_t len_in = iter_iov_len(iter); int error; - if (madvise_should_skip(start, len_in, behavior, &error)) + if (madvise_should_skip(mm, start, len_in, behavior, &error)) ret = error; else ret = madvise_do_behavior(start, len_in, &madv_behavior); @@ -2185,9 +2283,9 @@ static int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, .anon_name = anon_name, }; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; - len = (len_in + ~PAGE_MASK) & PAGE_MASK; + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero */ if (len_in && !len) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index d47aef256a324..964c002533131 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -67,6 +67,9 @@ #include "swap.h" #include "internal.h" +#ifdef CONFIG_MM_SUBPAGE +#include "user-subpage-internal.h" +#endif static int sysctl_memory_failure_early_kill __read_mostly; @@ -539,6 +542,37 @@ struct task_struct *task_early_kill(struct task_struct *tsk, int force_early) return find_early_kill_thread(tsk); } +#ifdef CONFIG_MM_SUBPAGE +struct subpage_collect_procs { + const struct page *page; + struct list_head *to_kill; + int force_early; +}; + +static bool collect_procs_subpage(struct folio *folio, struct vm_area_struct *vma, + struct rmap_walk_range range, void *arg) +{ + struct subpage_collect_procs *args = arg; + DEFINE_FOLIO_RMAP_WALK(walk, folio, vma, range, PVMW_SYNC); + struct task_struct *tsk; + unsigned long address; + + if (!page_vma_mapped_walk(&walk)) + return true; + address = walk.address; + page_vma_mapped_walk_done(&walk); + rcu_read_lock(); + for_each_process(tsk) { + struct task_struct *t = task_early_kill(tsk, args->force_early); + + if (t && t->mm == vma->vm_mm) + add_to_kill_anon_file(t, args->page, vma, args->to_kill, address); + } + rcu_read_unlock(); + return true; +} +#endif + /* * Collect processes when the error hit an anonymous page. */ @@ -549,6 +583,17 @@ static void collect_procs_anon(const struct folio *folio, struct task_struct *tsk; struct anon_vma *av; pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + struct subpage_collect_procs args = { page, to_kill, force_early }; + struct rmap_walk_control rwc = { + .rmap_one = collect_procs_subpage, + .arg = &args, + }; + + if (!folio_test_large(folio) && + rmap_walk_subpages((struct folio *)folio, &rwc, false)) + return; +#endif av = folio_lock_anon_vma_read(folio, NULL); if (av == NULL) /* Not actually mapped anymore */ diff --git a/mm/memory.c b/mm/memory.c index a903d5c1859cb..b3e96c0bfed0b 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -87,6 +87,8 @@ #include #include "pgalloc-track.h" +#include + #include "internal.h" #include "swap.h" @@ -191,26 +193,26 @@ static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, unsigned long start; start = addr; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(tlb->mm, pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(tlb->mm, addr, end); if (pmd_none_or_clear_bad(pmd)) continue; free_pte_range(tlb, pmd, addr); } while (pmd++, addr = next, addr != end); - start &= PUD_MASK; + start &= pud_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= PUD_MASK; + ceiling &= pud_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - pmd = pmd_offset(pud, start); + pmd = pmd_offset_mm(tlb->mm, pud, start); pud_clear(pud); pmd_free_tlb(tlb, pmd, start); mm_dec_nr_pmds(tlb->mm); @@ -225,27 +227,27 @@ static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, unsigned long start; start = addr; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(tlb->mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(tlb->mm, addr, end); if (pud_none_or_clear_bad(pud)) continue; free_pmd_range(tlb, pud, addr, next, floor, ceiling); } while (pud++, addr = next, addr != end); - start &= P4D_MASK; + start &= p4d_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= P4D_MASK; + ceiling &= p4d_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - pud = pud_offset(p4d, start); - p4d_clear(p4d); + pud = pud_offset_mm(tlb->mm, p4d, start); + p4d_clear_mm(tlb->mm, p4d); pud_free_tlb(tlb, pud, start); mm_dec_nr_puds(tlb->mm); } @@ -259,26 +261,26 @@ static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd, unsigned long start; start = addr; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(tlb->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(tlb->mm, addr, end); + if (p4d_none_or_clear_bad_mm(tlb->mm, p4d)) continue; free_pud_range(tlb, p4d, addr, next, floor, ceiling); } while (p4d++, addr = next, addr != end); - start &= PGDIR_MASK; + start &= pgd_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= PGDIR_MASK; + ceiling &= pgd_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - p4d = p4d_offset(pgd, start); + p4d = p4d_offset_mm(tlb->mm, pgd, start); pgd_clear(pgd); p4d_free_tlb(tlb, p4d, start); } @@ -320,7 +322,7 @@ void free_pgd_range(struct mmu_gather *tlb, * Wherever addr is brought up or ceiling brought down, we must * be careful to reject "the opposite 0" before it confuses the * subsequent tests. But what about where end is brought down - * by PMD_SIZE below? no, end can't go down to 0 there. + * by one PMD span below? no, end can't go down to 0 there. * * Whereas we round start (addr) and ceiling down, by different * masks at different levels, in order to test whether a table @@ -328,19 +330,19 @@ void free_pgd_range(struct mmu_gather *tlb, * bother to round floor or end up - the tests don't need that. */ - addr &= PMD_MASK; + addr &= pmd_mask_mm(tlb->mm); if (addr < floor) { - addr += PMD_SIZE; + addr += pmd_size_mm(tlb->mm); if (!addr) return; } if (ceiling) { - ceiling &= PMD_MASK; + ceiling &= pmd_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) - end -= PMD_SIZE; + end -= pmd_size_mm(tlb->mm); if (addr > end - 1) return; /* @@ -350,7 +352,7 @@ void free_pgd_range(struct mmu_gather *tlb, tlb_change_page_size(tlb, PAGE_SIZE); pgd = pgd_offset(tlb->mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(tlb->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; free_p4d_range(tlb, pgd, addr, next, floor, ceiling); @@ -539,16 +541,16 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add return; } - p4dp = p4d_offset(pgdp, addr); + p4dp = p4d_offset_mm(mm, pgdp, addr); p4d = p4dp_get(p4dp); p4dv = p4d_val(p4d); - if (!p4d_present(p4d) || p4d_leaf(p4d)) { + if (!p4d_present_mm(mm, p4d) || p4d_leaf(p4d)) { pr_alert("pgd:%08llx p4d:%08llx\n", pgdv, p4dv); return; } - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_mm(mm, p4dp, addr); pud = pudp_get(pudp); pudv = pud_val(pud); @@ -557,7 +559,7 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add return; } - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_mm(mm, pudp, addr); pmd = pmdp_get(pmdp); pmdv = pmd_val(pmd); @@ -950,7 +952,15 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, struct page *page; if (likely(softleaf_is_swap(entry))) { - if (swap_dup_entry_direct(entry) < 0) + int err; + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) + err = swap_subpage_dup(entry, pte_swp_subpage_offset(orig_pte)); + else +#endif + err = swap_dup_entries_direct(entry, mm_pte_native_pages(src_mm)); + if (err < 0) return -EIO; /* make sure dst_mm is on swapoff's mmlist. */ @@ -982,6 +992,10 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, entry = make_readable_migration_entry( swp_offset(entry)); pte = softleaf_to_pte(entry); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) + pte = pte_swp_set_subpage_offset(pte, pte_swp_subpage_offset(orig_pte)); +#endif if (pte_swp_soft_dirty(orig_pte)) pte = pte_swp_mksoft_dirty(pte); if (pte_swp_uffd_wp(orig_pte)) @@ -1076,10 +1090,14 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma * over and copy the page & arm it. */ - if (copy_mc_user_highpage(&new_folio->page, page, addr, src_vma)) - return -EHWPOISON; + for (unsigned int i = 0; i < mm_pte_native_pages(src_vma->vm_mm); i++) { + if (copy_mc_user_highpage(folio_page(new_folio, i), page + i, + addr + i * PAGE_SIZE, src_vma)) + return -EHWPOISON; + } *prealloc = NULL; + folio_ref_add(new_folio, mm_pte_native_pages(dst_vma->vm_mm) - 1); __folio_mark_uptodate(new_folio); folio_add_new_anon_rmap(new_folio, dst_vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, dst_vma); @@ -1095,6 +1113,55 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma return 0; } +#ifdef CONFIG_MM_SUBPAGE +/* One new native folio may publish several independently locked user leaves. */ +static void subpage_add_new_lru(struct folio *folio, struct vm_area_struct *vma, + unsigned int mapped) +{ + folio_add_lru_vma(folio, vma); + if ((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED) + while (--mapped) + mlock_folio(folio); +} + +/* The destination slot and its charge/metadata were allocated outside PTLs. */ +static int copy_pinned_subpage(struct vm_area_struct *dst_vma, pte_t *dst_pte, + pte_t source_pte, unsigned long addr, int *rss, + struct mm_subpage *source, struct mm_subpage **prealloc) +{ + struct mm_subpage *dest = *prealloc; + struct folio *new; + pte_t pte; + int ret; + + if (!dest) + return -EAGAIN; + new = mm_subpage_folio(dest); + ret = copy_user_subpage_range(&new->page, mm_subpage_offset(dest), + &mm_subpage_folio(source)->page, mm_subpage_offset(source), + mm_subpage_size(source)); + if (ret) + return ret; + /* Fresh, unmapped backing is private to this preallocation. */ + if (WARN_ON_ONCE(!folio_trylock(new))) + return -EBUSY; + mm_subpage_set_exclusive(dest); + ret = mm_subpage_add_new_anon_rmap(dest, dst_vma, addr); + if (!ret) { + subpage_add_new_lru(new, dst_vma, 1); + rss[MM_ANONPAGES]++; + pte = phys_pte_mm(dst_vma->vm_mm, mm_subpage_phys(dest), dst_vma->vm_page_prot); + pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); + if (userfaultfd_pte_wp(dst_vma, source_pte)) + pte = pte_mkuffd_wp(pte); + set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); + *prealloc = NULL; + } + folio_unlock(new); + return ret; +} +#endif + static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr, int nr) @@ -1128,18 +1195,46 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, static inline int copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr, - int max_nr, int *rss, struct folio **prealloc) + int max_nr, int *rss, struct folio **prealloc, + struct mm_subpage **subprealloc) { fpb_t flags = FPB_MERGE_WRITE; struct page *page; struct folio *folio; - int err, nr; + int err, nr, nr_native; + unsigned int span = mm_pte_native_pages(src_vma->vm_mm); page = vm_normal_page(src_vma, addr, pte); if (unlikely(!page)) goto copy_pte; folio = page_folio(page); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_vma->vm_mm) < PAGE_SIZE) { + if (folio_test_anon(folio)) { + struct mm_subpage *slot; + + slot = mm_subpage_get_from_phys(pte_phys_mm(src_vma->vm_mm, pte)); + if (WARN_ON_ONCE(!slot)) + return -EINVAL; + err = mm_subpage_dup_anon_rmap(slot, dst_vma, src_vma, addr); + if (err == -EBUSY) { + err = copy_pinned_subpage(dst_vma, dst_pte, pte, addr, + rss, slot, subprealloc); + mm_subpage_put(slot); + return err ? err : 1; + } + if (err) { + mm_subpage_put(slot); + return err; + } + /* The lookup reference becomes the child's PTE reference. */ + rss[MM_ANONPAGES]++; + goto copy_pte; + } + max_nr = 1; + } +#endif /* * If we likely have to copy, just don't bother with batching. Make @@ -1152,18 +1247,25 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma if (vma_soft_dirty_enabled(src_vma)) flags |= FPB_RESPECT_SOFT_DIRTY; - nr = folio_pte_batch_flags(folio, src_vma, src_pte, &pte, max_nr, flags); - folio_ref_add(folio, nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_vma->vm_mm) > PAGE_SIZE) + nr = folio_subpage_pte_batch_flags(src_vma, folio, src_pte, + &pte, max_nr, flags); + else +#endif + nr = folio_pte_batch_flags(folio, src_vma, src_pte, &pte, max_nr, flags); + nr_native = nr * span; + folio_ref_add(folio, nr_native); if (folio_test_anon(folio)) { if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page, - nr, dst_vma, src_vma))) { - folio_ref_sub(folio, nr); + nr_native, dst_vma, src_vma))) { + folio_ref_sub(folio, nr_native); return -EAGAIN; } rss[MM_ANONPAGES] += nr; VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio); } else { - folio_dup_file_rmap_ptes(folio, page, nr, dst_vma); + folio_dup_file_rmap_ptes(folio, page, nr_native, dst_vma); rss[mm_counter_file(folio)] += nr; } __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, @@ -1171,7 +1273,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma return nr; } - folio_get(folio); + folio_ref_add(folio, span); if (folio_test_anon(folio)) { /* * If this page may have been pinned by the parent process, @@ -1181,7 +1283,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma */ if (unlikely(folio_try_dup_anon_rmap_pte(folio, page, dst_vma, src_vma))) { /* Page may be pinned, we have to copy. */ - folio_put(folio); + folio_put_refs(folio, span); err = copy_present_page(dst_vma, src_vma, dst_pte, src_pte, addr, rss, prealloc, page); return err ? err : 1; @@ -1189,6 +1291,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma rss[MM_ANONPAGES]++; VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio); } else { + mm_subpage_file_map_add(dst_vma, pte_phys_mm(src_vma->vm_mm, pte)); folio_dup_file_rmap_pte(folio, page, dst_vma); rss[mm_counter_file(folio)]++; } @@ -1202,11 +1305,12 @@ static inline struct folio *folio_prealloc(struct mm_struct *src_mm, struct vm_area_struct *vma, unsigned long addr, bool need_zero) { struct folio *new_folio; + unsigned int order = max_t(unsigned int, mm_page_shift(src_mm), PAGE_SHIFT) - PAGE_SHIFT; if (need_zero) - new_folio = vma_alloc_zeroed_movable_folio(vma, addr); + new_folio = vma_alloc_zeroed_movable_folio_order(vma, addr, order); else - new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr); + new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, order, vma, addr); if (!new_folio) return NULL; @@ -1220,6 +1324,34 @@ static inline struct folio *folio_prealloc(struct mm_struct *src_mm, return new_folio; } +#ifdef CONFIG_MM_SUBPAGE +static struct mm_subpage *anon_subpage_prealloc(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr) +{ + struct mm_subpage_pool *pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + struct mm_subpage *slot = ERR_PTR(-ENOMEM); + struct folio *folio; + int ret; + + if (!pool) + return slot; + folio = folio_prealloc(mm, vma, addr, false); + if (!folio) + goto out; + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + slot = ERR_PTR(ret); + goto out; + } + slot = mm_subpage_alloc_at(pool, vma_page_offset_at(vma, addr).offset); +out: + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + return slot; +} +#endif + static int copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, @@ -1236,6 +1368,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, int rss[NR_MM_COUNTERS]; softleaf_t entry = softleaf_mk_none(); struct folio *prealloc = NULL; + struct mm_subpage *subprealloc = NULL; int nr; again: @@ -1316,16 +1449,23 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, WARN_ON_ONCE(ret != -ENOENT); } /* copy_present_ptes() will clear `*prealloc' if consumed */ - max_nr = (end - addr) / PAGE_SIZE; + max_nr = (end - addr) >> mm_page_shift(src_mm); ret = copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, - ptent, addr, max_nr, rss, &prealloc); + ptent, addr, max_nr, rss, &prealloc, &subprealloc); /* * If we need a pre-allocated page for this pte, drop the * locks, allocate, and try again. * If copy failed due to hwpoison in source page, break out. */ - if (unlikely(ret == -EAGAIN || ret == -EHWPOISON)) + if (unlikely(ret == -EAGAIN || ret == -EHWPOISON || + (IS_ENABLED(CONFIG_MM_SUBPAGE) && ret < 0))) break; +#ifdef CONFIG_MM_SUBPAGE + if (subprealloc) { + mm_subpage_put(subprealloc); + subprealloc = NULL; + } +#endif if (unlikely(prealloc)) { /* * pre-alloc page cannot be reused by next time so as @@ -1338,7 +1478,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } nr = ret; progress += 8 * nr; - } while (dst_pte += nr, src_pte += nr, addr += PAGE_SIZE * nr, + } while (dst_pte += nr, src_pte += nr, addr += mm_page_size(src_mm) * nr, addr != end); lazy_mmu_mode_disable(); @@ -1357,10 +1497,22 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } else if (ret == -EBUSY || unlikely(ret == -EHWPOISON)) { goto out; } else if (ret == -EAGAIN) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) { + subprealloc = anon_subpage_prealloc(src_mm, src_vma, addr); + if (IS_ERR(subprealloc)) + return PTR_ERR(subprealloc); + ret = 0; + goto again; + } +#endif prealloc = folio_prealloc(src_mm, src_vma, addr, false); if (!prealloc) return -ENOMEM; } else if (ret < 0) { + /* Subpage ownership checks can return additional fatal errors. */ + if (IS_ENABLED(CONFIG_MM_SUBPAGE)) + goto out; VM_WARN_ON_ONCE(1); } @@ -1370,6 +1522,10 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, if (addr != end) goto again; out: +#ifdef CONFIG_MM_SUBPAGE + if (subprealloc) + mm_subpage_put(subprealloc); +#endif if (unlikely(prealloc)) folio_put(prealloc); return ret; @@ -1388,9 +1544,9 @@ copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_pmd = pmd_alloc(dst_mm, dst_pud, addr); if (!dst_pmd) return -ENOMEM; - src_pmd = pmd_offset(src_pud, addr); + src_pmd = pmd_offset_mm(src_vma->vm_mm, src_pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(src_vma->vm_mm, addr, end); if (pmd_is_huge(*src_pmd)) { int err; @@ -1425,9 +1581,9 @@ copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_pud = pud_alloc(dst_mm, dst_p4d, addr); if (!dst_pud) return -ENOMEM; - src_pud = pud_offset(src_p4d, addr); + src_pud = pud_offset_mm(src_vma->vm_mm, src_p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(src_vma->vm_mm, addr, end); if (pud_trans_huge(*src_pud)) { int err; @@ -1461,10 +1617,10 @@ copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr); if (!dst_p4d) return -ENOMEM; - src_p4d = p4d_offset(src_pgd, addr); + src_p4d = p4d_offset_mm(src_vma->vm_mm, src_pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(src_p4d)) + next = p4d_addr_end_mm(src_vma->vm_mm, addr, end); + if (p4d_none_or_clear_bad_mm(src_vma->vm_mm, src_p4d)) continue; if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d, addr, next)) @@ -1516,6 +1672,9 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) bool is_cow; int ret; + if (mm_page_shift(dst_mm) != mm_page_shift(src_mm)) + return -EINVAL; + if (!vma_needs_copy(dst_vma, src_vma)) return 0; @@ -1549,7 +1708,7 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) dst_pgd = pgd_offset(dst_mm, addr); src_pgd = pgd_offset(src_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(src_vma->vm_mm, addr, end); if (pgd_none_or_clear_bad(src_pgd)) continue; if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd, @@ -1629,7 +1788,7 @@ zap_install_uffd_wp_if_needed(struct vm_area_struct *vma, if (--nr == 0) break; pte++; - addr += PAGE_SIZE; + addr += mm_page_size(vma->vm_mm); } return was_installed; @@ -1642,10 +1801,12 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, bool *force_flush, bool *force_break, bool *any_skipped) { struct mm_struct *mm = tlb->mm; + unsigned int nr_native = nr * mm_pte_native_pages(mm); bool delay_rmap = false; if (!folio_test_anon(folio)) { ptent = get_and_clear_full_ptes(mm, addr, pte, nr, tlb->fullmm); + mm_subpage_file_map_del(vma, pte_phys_mm(mm, ptent)); if (pte_dirty(ptent)) { folio_mark_dirty(folio); if (tlb_delay_rmap(tlb)) { @@ -1669,16 +1830,44 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, nr, details, ptent); if (!delay_rmap) { - folio_remove_rmap_ptes(folio, page, nr, vma); + folio_remove_rmap_ptes(folio, page, nr_native, vma); if (unlikely(folio_mapcount(folio) < 0)) print_bad_pte(vma, addr, ptent, page); } - if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) { + /* Gather payloads count native references, not logical PTEs. */ + if (unlikely(__tlb_remove_folio_pages(tlb, page, nr_native, delay_rmap))) { + *force_flush = true; + *force_break = true; + } +} + +#ifdef CONFIG_MM_SUBPAGE +static void zap_present_subpage(struct mmu_gather *tlb, + struct vm_area_struct *vma, pte_t *pte, pte_t ptent, + unsigned long addr, int *rss, bool *force_flush, + bool *force_break, bool *any_skipped) +{ + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(tlb->mm, ptent)); + + /* All small-granule anonymous PTEs must carry slot ownership. */ + if (WARN_ON_ONCE(!slot)) { + *any_skipped = true; + return; + } + ptep_get_and_clear_full(tlb->mm, addr, pte, tlb->fullmm); + arch_check_zapped_pte(vma, ptent); + tlb_remove_tlb_entry(tlb, pte, addr); + mm_subpage_remove_anon_rmap(slot, vma); + rss[MM_ANONPAGES]--; + /* Drop the lookup ref, then transfer the PTE ref to the TLB batch. */ + mm_subpage_put(slot); + if (__tlb_remove_subpage(tlb, slot)) { *force_flush = true; *force_break = true; } } +#endif /* * Zap or skip at least one present PTE, trying to batch-process subsequent @@ -1716,12 +1905,30 @@ static inline int zap_present_ptes(struct mmu_gather *tlb, return 1; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + if (folio_test_anon(folio)) { + zap_present_subpage(tlb, vma, pte, ptent, addr, rss, + force_flush, force_break, any_skipped); + return 1; + } + /* Native folio batching still expresses physical PAGE_SIZE steps. */ + max_nr = 1; + } +#endif + /* * Make sure that the common "small folio" case is as fast as possible * by keeping the batching logic separate. */ if (unlikely(folio_test_large(folio) && max_nr != 1)) { - nr = folio_pte_batch(folio, pte, ptent, max_nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) > PAGE_SIZE) + nr = folio_subpage_pte_batch(vma, folio, pte, ptent, + max_nr, 0); + else +#endif + nr = folio_pte_batch(folio, pte, ptent, max_nr); zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr, addr, details, rss, force_flush, force_break, any_skipped); @@ -1764,9 +1971,20 @@ static inline int zap_nonpresent_ptes(struct mmu_gather *tlb, if (!should_zap_cows(details)) return 1; - nr = swap_pte_batch(pte, max_nr, ptent); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + swap_subpage_put(entry, pte_swp_subpage_offset(ptent), true); + } else +#endif + { + unsigned int span = mm_pte_native_pages(vma->vm_mm); + + /* Adjacent coarse PTEs advance by a full group of swap slots. */ + if (span == 1) + nr = swap_pte_batch(pte, max_nr, ptent); + swap_put_entries_direct(entry, nr * span); + } rss[MM_SWAPENTS] -= nr; - swap_put_entries_direct(entry, nr); } else if (softleaf_is_migration(entry)) { struct folio *folio = softleaf_to_folio(entry); @@ -1811,7 +2029,7 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, bool *any_skipped) { pte_t ptent = ptep_get(pte); - int max_nr = (end - addr) / PAGE_SIZE; + int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); int nr = 0; /* Skip all consecutive none ptes */ @@ -1825,7 +2043,7 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, if (!max_nr) return nr; pte += nr; - addr += nr * PAGE_SIZE; + addr += nr * mm_page_size(vma->vm_mm); } if (pte_present(ptent)) @@ -1839,13 +2057,14 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, return nr; } -static bool pte_table_reclaim_possible(unsigned long start, unsigned long end, +static bool pte_table_reclaim_possible(struct mm_struct *mm, + unsigned long start, unsigned long end, struct zap_details *details) { if (!IS_ENABLED(CONFIG_PT_RECLAIM)) return false; /* Only zap if we are allowed to and cover the full page table. */ - return details && details->reclaim_pt && (end - start >= PMD_SIZE); + return details && details->reclaim_pt && (end - start >= pmd_size_mm(mm)); } static bool zap_empty_pte_table(struct mm_struct *mm, pmd_t *pmd, @@ -1877,7 +2096,7 @@ static bool zap_pte_table_if_empty(struct mm_struct *mm, pmd_t *pmd, if (ptl != pml) spin_lock_nested(ptl, SINGLE_DEPTH_NESTING); - for (i = 0, pte = start_pte; i < PTRS_PER_PTE; i++, pte++) { + for (i = 0, pte = start_pte; i < pte_table_bytes_mm(mm) / sizeof(pte_t); i++, pte++) { if (!pte_none(ptep_get(pte))) goto out_ptl; } @@ -1902,7 +2121,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end, struct zap_details *details) { - bool can_reclaim_pt = pte_table_reclaim_possible(addr, end, details); + bool can_reclaim_pt = pte_table_reclaim_possible(tlb->mm, addr, end, details); bool force_flush = false, force_break = false; struct mm_struct *mm = tlb->mm; int rss[NR_MM_COUNTERS]; @@ -1915,7 +2134,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, int nr; retry: - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, mm_page_size(mm)); init_rss_vec(rss); start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl); if (!pte) @@ -1936,11 +2155,11 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, if (any_skipped) can_reclaim_pt = false; if (unlikely(force_break)) { - addr += nr * PAGE_SIZE; + addr += nr * mm_page_size(vma->vm_mm); direct_reclaim = false; break; } - } while (pte += nr, addr += PAGE_SIZE * nr, addr != end); + } while (pte += nr, addr += mm_page_size(mm) * nr, addr != end); /* * Fast path: try to hold the pmd lock and unmap the PTE page. @@ -1997,9 +2216,9 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, pmd_t *pmd; unsigned long next; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(tlb->mm, pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(tlb->mm, addr, end); if (pmd_is_huge(*pmd)) { if (next - addr != HPAGE_PMD_SIZE) __split_huge_pmd(vma, pmd, addr, false); @@ -2033,9 +2252,9 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb, pud_t *pud; unsigned long next; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(tlb->mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(tlb->mm, addr, end); if (pud_trans_huge(*pud)) { if (next - addr != HPAGE_PUD_SIZE) split_huge_pud(vma, pud, addr); @@ -2061,10 +2280,10 @@ static inline unsigned long zap_p4d_range(struct mmu_gather *tlb, p4d_t *p4d; unsigned long next; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(tlb->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(tlb->mm, addr, end); + if (p4d_none_or_clear_bad_mm(tlb->mm, p4d)) continue; next = zap_pud_range(tlb, vma, p4d, addr, next, details); } while (p4d++, addr = next, addr != end); @@ -2103,7 +2322,7 @@ static void __zap_vma_range(struct mmu_gather *tlb, struct vm_area_struct *vma, tlb_start_vma(tlb, vma); pgd = pgd_offset(vma->vm_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(tlb->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; next = zap_p4d_range(tlb, vma, pgd, addr, next, details); @@ -2350,7 +2569,7 @@ static int validate_page_before_insert(struct vm_area_struct *vma, static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, unsigned long addr, struct page *page, - pgprot_t prot, bool mkwrite) + pgprot_t prot, bool mkwrite, unsigned int offset) { struct folio *folio = page_folio(page); pte_t pteval = ptep_get(pte); @@ -2360,7 +2579,7 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, return -EBUSY; /* see insert_pfn(). */ - if (pte_pfn(pteval) != page_to_pfn(page)) { + if (pte_phys_mm(vma->vm_mm, pteval) != page_to_phys(page) + offset) { WARN_ON_ONCE(!is_zero_pfn(pte_pfn(pteval))); return -EFAULT; } @@ -2372,17 +2591,18 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, } /* Ok, finally just insert the thing.. */ - pteval = mk_pte(page, prot); + pteval = phys_pte_mm(vma->vm_mm, page_to_phys(page) + offset, prot); if (unlikely(is_zero_folio(folio))) { pteval = pte_mkspecial(pteval); } else { folio_get(folio); - pteval = mk_pte(page, prot); + pteval = phys_pte_mm(vma->vm_mm, page_to_phys(page) + offset, prot); if (mkwrite) { pteval = pte_mkyoung(pteval); pteval = maybe_mkwrite(pte_mkdirty(pteval), vma); } inc_mm_counter(vma->vm_mm, mm_counter_file(folio)); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, pteval)); folio_add_file_rmap_pte(folio, page, vma); } set_pte_at(vma->vm_mm, addr, pte, pteval); @@ -2390,7 +2610,7 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, } static int insert_page(struct vm_area_struct *vma, unsigned long addr, - struct page *page, pgprot_t prot, bool mkwrite) + struct page *page, pgprot_t prot, bool mkwrite, unsigned int offset) { int retval; pte_t *pte; @@ -2404,83 +2624,120 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr, if (!pte) goto out; retval = insert_page_into_pte_locked(vma, pte, addr, page, prot, - mkwrite); + mkwrite, offset); pte_unmap_unlock(pte, ptl); out: return retval; } -static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte, - unsigned long addr, struct page *page, pgprot_t prot) +/* Keep lock batching in units of hardware leaves, backing in native pages. */ +static int insert_page_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long offset, + unsigned long *length, pgprot_t prot) { - int err; + struct mm_struct *mm = vma->vm_mm; + unsigned long remaining = *length; + unsigned long size = mm_page_size(mm); + int ret = 0; - err = validate_page_before_insert(vma, page); - if (err) - return err; - return insert_page_into_pte_locked(vma, pte, addr, page, prot, false); + while (remaining) { + unsigned long in_pmd = min(remaining, + pmd_size_mm(mm) - (addr & ~pmd_mask_mm(mm))); + pmd_t *pmd = walk_to_pmd(mm, addr); + + if (!pmd) { + ret = -EFAULT; + break; + } + if (pte_alloc(mm, pmd)) { + ret = -ENOMEM; + break; + } + while (in_pmd) { + unsigned int batch = min_t(unsigned long, in_pmd / size, 8); + unsigned int i; + spinlock_t *ptl; + pte_t *start = pte_offset_map_lock(mm, pmd, addr, &ptl); + + if (!start) { + ret = -EFAULT; + goto out; + } + for (i = 0; i < batch; i++) { + struct page *page = pages[offset >> PAGE_SHIFT]; + + ret = validate_page_before_insert(vma, page); + if (!ret) + ret = insert_page_into_pte_locked(vma, start + i, + addr, page, prot, false, offset_in_page(offset)); + if (ret) + break; + addr += size; + offset += size; + remaining -= size; + in_pmd -= size; + } + pte_unmap_unlock(start, ptl); + if (ret) + goto out; + } + } +out: + *length = remaining; + return ret; } -/* insert_pages() amortizes the cost of spinlock operations - * when inserting pages in a loop. - */ static int insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num, pgprot_t prot) { - pmd_t *pmd = NULL; - pte_t *start_pte, *pte; - spinlock_t *pte_lock; - struct mm_struct *const mm = vma->vm_mm; - unsigned long curr_page_idx = 0; - unsigned long remaining_pages_total = *num; - unsigned long pages_to_write_in_pmd; + unsigned long length; int ret; -more: - ret = -EFAULT; - pmd = walk_to_pmd(mm, addr); - if (!pmd) - goto out; - pages_to_write_in_pmd = min_t(unsigned long, - remaining_pages_total, PTRS_PER_PTE - pte_index(addr)); - - /* Allocate the PTE if necessary; takes PMD lock once only. */ - ret = -ENOMEM; - if (pte_alloc(mm, pmd)) - goto out; + if (check_mul_overflow(*num, PAGE_SIZE, &length)) + return -EOVERFLOW; + ret = insert_page_range(vma, addr, pages, 0, &length, prot); + /* A partially inserted native page is still an incomplete input page. */ + *num = DIV_ROUND_UP(length, PAGE_SIZE); + return ret; +} - while (pages_to_write_in_pmd) { - int pte_idx = 0; - const int batch_size = min_t(int, pages_to_write_in_pmd, 8); +/** + * vm_insert_pages_range - map a byte range from a native-page array + * @vma: target VMA, held under the mmap write lock + * @addr: first user address + * @pages: native backing pages + * @nr_pages: size of @pages + * @offset: byte offset within the backing array + * @length: bytes to map + * + * Address, offset and length must match the target mm's granule. A failed + * insertion can leave a prefix mapped, as with vm_insert_pages(); the caller + * must unmap that prefix when abandoning the mapping. Physical ownership + * remains in native pages. Each installed user leaf acquires its own ref. + */ +int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long nr_pages, + unsigned long offset, unsigned long length) +{ + unsigned long bytes, end, size = mm_page_size(vma->vm_mm); - start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock); - if (!start_pte) { - ret = -EFAULT; - goto out; - } - for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) { - int err = insert_page_in_batch_locked(vma, pte, - addr, pages[curr_page_idx], prot); - if (unlikely(err)) { - pte_unmap_unlock(start_pte, pte_lock); - ret = err; - remaining_pages_total -= pte_idx; - goto out; - } - addr += PAGE_SIZE; - ++curr_page_idx; - } - pte_unmap_unlock(start_pte, pte_lock); - pages_to_write_in_pmd -= batch_size; - remaining_pages_total -= batch_size; + if (!IS_ALIGNED(addr | offset | length, size) || + check_mul_overflow(nr_pages, PAGE_SIZE, &bytes) || + offset > bytes || length > bytes - offset || + check_add_overflow(addr, length, &end) || + !range_in_vma(vma, addr, end)) + return -EINVAL; + if (!length) + return 0; + if (vma->vm_flags & VM_PFNMAP) + return -EINVAL; + if (!(vma->vm_flags & VM_MIXEDMAP)) { + mmap_assert_write_locked(vma->vm_mm); + vm_flags_set(vma, VM_MIXEDMAP); } - if (remaining_pages_total) - goto more; - ret = 0; -out: - *num = remaining_pages_total; - return ret; + return insert_page_range(vma, addr, pages, offset, &length, vma->vm_page_prot); } +EXPORT_SYMBOL_GPL(vm_insert_pages_range); /** * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock. @@ -2501,9 +2758,12 @@ int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num) { const unsigned long nr_pages = *num; - const unsigned long end = addr + PAGE_SIZE * nr_pages; + unsigned long length, end; - if (!range_in_vma(vma, addr, end)) + if (!IS_ALIGNED(addr, mm_page_size(vma->vm_mm)) || + check_mul_overflow(nr_pages, PAGE_SIZE, &length) || + check_add_overflow(addr, length, &end) || + !range_in_vma(vma, addr, end)) return -EFAULT; if (!(vma->vm_flags & VM_MIXEDMAP)) { VM_WARN_ON_ONCE(mmap_read_trylock(vma->vm_mm)); @@ -2528,8 +2788,10 @@ int map_kernel_pages_prepare(struct vm_area_desc *desc) } nr_pages = action->map_kernel.nr_pages; - end = addr + PAGE_SIZE * nr_pages; - if (!range_in_vma_desc(desc, addr, end)) + if (!IS_ALIGNED(addr, mm_page_size(desc->mm)) || + check_mul_overflow(nr_pages, PAGE_SIZE, &end) || + check_add_overflow(addr, end, &end) || + !range_in_vma_desc(desc, addr, end)) return -EFAULT; return 0; @@ -2588,10 +2850,35 @@ int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, BUG_ON(vma->vm_flags & VM_PFNMAP); vm_flags_set(vma, VM_MIXEDMAP); } - return insert_page(vma, addr, page, vma->vm_page_prot, false); + return insert_page(vma, addr, page, vma->vm_page_prot, false, 0); } EXPORT_SYMBOL(vm_insert_page); +#ifdef CONFIG_MM_SUBPAGE +/* Explicit physical offset; the VMA and supplied page retain native ownership. */ +vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, unsigned long addr, + struct page *page, unsigned int offset) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long size = mm_page_size(mm); + int err; + + if (!(vma->vm_flags & VM_MIXEDMAP) || (vma->vm_flags & VM_PFNMAP) || + (addr & (size - 1)) || (offset & (size - 1)) || + offset > PAGE_SIZE - size || addr < vma->vm_start || + addr >= vma->vm_end || vma->vm_end - addr < size) + return VM_FAULT_SIGBUS; + err = insert_page(vma, addr, page, vma->vm_page_prot, false, offset); + if (err == -ENOMEM) + return VM_FAULT_OOM; + if (err < 0 && err != -EBUSY) + return VM_FAULT_SIGBUS; + return VM_FAULT_NOPAGE; +} +EXPORT_SYMBOL_GPL(vmf_insert_page_offset); +#endif + + /* * __vm_map_pages - maps range of kernel pages into user vma * @vma: user vma to map to @@ -2606,20 +2893,22 @@ EXPORT_SYMBOL(vm_insert_page); * Return: 0 on success and error code otherwise. */ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, - unsigned long num, unsigned long offset) + unsigned long num, unsigned long offset, + unsigned int subpage_offset) { - unsigned long count = vma_pages(vma); - unsigned long uaddr = vma->vm_start; + unsigned long bytes, available; - /* Fail if the user requested offset is beyond the end of the object */ - if (offset >= num) + if (offset >= num || + check_mul_overflow(num - offset, PAGE_SIZE, &available) || + subpage_offset >= PAGE_SIZE || + check_mul_overflow(offset, PAGE_SIZE, &bytes) || + check_add_overflow(bytes, (unsigned long)subpage_offset, &bytes)) return -ENXIO; - - /* Fail if the user requested size exceeds available object size */ - if (count > num - offset) + if (vma->vm_end - vma->vm_start > + available - subpage_offset) return -ENXIO; - - return vm_insert_pages(vma, uaddr, pages + offset, &count); + return vm_insert_pages_range(vma, vma->vm_start, pages, num, bytes, + vma->vm_end - vma->vm_start); } /** @@ -2643,7 +2932,7 @@ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, int vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num) { - return __vm_map_pages(vma, pages, num, vma->vm_pgoff); + return __vm_map_pages(vma, pages, num, vma->vm_pgoff, vma_subpage_offset(vma)); } EXPORT_SYMBOL(vm_map_pages); @@ -2663,12 +2952,12 @@ EXPORT_SYMBOL(vm_map_pages); int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, unsigned long num) { - return __vm_map_pages(vma, pages, num, 0); + return __vm_map_pages(vma, pages, num, 0, 0); } EXPORT_SYMBOL(vm_map_pages_zero); static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, pgprot_t prot, bool mkwrite) + unsigned long pfn, pgprot_t prot, bool mkwrite, unsigned int offset) { struct mm_struct *mm = vma->vm_mm; pte_t *pte, entry; @@ -2690,7 +2979,7 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, * allocation and mapping invalidation so just skip the * update. */ - if (pte_pfn(entry) != pfn) { + if (pte_phys_mm(mm, entry) != PFN_PHYS(pfn) + offset) { WARN_ON_ONCE(!is_zero_pfn(pte_pfn(entry))); goto out_unlock; } @@ -2703,7 +2992,7 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, } /* Ok, finally just insert the thing.. */ - entry = pte_mkspecial(pfn_pte(pfn, prot)); + entry = pte_mkspecial(phys_pte_mm(mm, PFN_PHYS(pfn) + offset, prot)); if (mkwrite) { entry = pte_mkyoung(entry); @@ -2765,9 +3054,9 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, * Context: Process context. May allocate using %GFP_KERNEL. * Return: vm_fault_t value. */ -vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, +static vm_fault_t __vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot, - bool mkwrite) + bool mkwrite, unsigned int offset) { /* * Technically, architectures with pte_special can avoid all these @@ -2789,10 +3078,31 @@ vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, pfnmap_setup_cachemode_pfn(pfn, &pgprot); - return insert_pfn(vma, addr, pfn, pgprot, mkwrite); + return insert_pfn(vma, addr, pfn, pgprot, mkwrite, offset); +} +vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, pgprot_t pgprot, + bool mkwrite) +{ + return __vmf_insert_pfn_prot_mkwrite(vma, addr, pfn, pgprot, mkwrite, 0); } EXPORT_SYMBOL(vmf_insert_pfn_prot_mkwrite); +/* Explicit quarter offset; a native PFN alone cannot encode a 4K leaf. */ +vm_fault_t vmf_insert_pfn_prot_mkwrite_offset(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, unsigned int offset, + pgprot_t pgprot, bool mkwrite) +{ + unsigned long size = mm_page_size(vma->vm_mm); + + if (!IS_ALIGNED(addr | offset, size) || offset > PAGE_SIZE - size || + addr < vma->vm_start || addr >= vma->vm_end || + vma->vm_end - addr < size) + return VM_FAULT_SIGBUS; + return __vmf_insert_pfn_prot_mkwrite(vma, addr, pfn, pgprot, mkwrite, offset); +} +EXPORT_SYMBOL_GPL(vmf_insert_pfn_prot_mkwrite_offset); + static bool vm_mixed_ok(struct vm_area_struct *vma, unsigned long pfn, bool mkwrite) { @@ -2840,9 +3150,9 @@ static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma, * result in pfn_t_has_page() == false. */ page = pfn_to_page(pfn); - err = insert_page(vma, addr, page, pgprot, mkwrite); + err = insert_page(vma, addr, page, pgprot, mkwrite, 0); } else { - return insert_pfn(vma, addr, pfn, pgprot, mkwrite); + return insert_pfn(vma, addr, pfn, pgprot, mkwrite, 0); } if (err == -ENOMEM) @@ -2863,7 +3173,7 @@ vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, if (addr < vmf->vma->vm_start || addr >= vmf->vma->vm_end) return VM_FAULT_SIGBUS; - err = insert_page(vmf->vma, addr, page, pgprot, write); + err = insert_page(vmf->vma, addr, page, pgprot, write, 0); if (err == -ENOMEM) return VM_FAULT_OOM; if (err < 0 && err != -EBUSY) @@ -2898,7 +3208,7 @@ vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, */ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pte_t *pte, *mapped_pte; spinlock_t *ptl; @@ -2910,13 +3220,13 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, lazy_mmu_mode_enable(); do { BUG_ON(!pte_none(ptep_get(pte))); - if (!pfn_modify_allowed(pfn, prot)) { + if (!pfn_modify_allowed(PHYS_PFN(phys), prot)) { err = -EACCES; break; } - set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); - pfn++; - } while (pte++, addr += PAGE_SIZE, addr != end); + set_pte_at(mm, addr, pte, pte_mkspecial(phys_pte_mm(mm, phys, prot))); + phys += mm_page_size(mm); + } while (pte++, addr += mm_page_size(mm), addr != end); lazy_mmu_mode_disable(); pte_unmap_unlock(mapped_pte, ptl); return err; @@ -2924,67 +3234,67 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pmd_t *pmd; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; pmd = pmd_alloc(mm, pud, addr); if (!pmd) return -ENOMEM; VM_BUG_ON(pmd_trans_huge(*pmd)); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(mm, addr, end); err = remap_pte_range(mm, pmd, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pmd++, addr = next, addr != end); return 0; } static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pud_t *pud; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; pud = pud_alloc(mm, p4d, addr); if (!pud) return -ENOMEM; do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(mm, addr, end); err = remap_pmd_range(mm, pud, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pud++, addr = next, addr != end); return 0; } static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { p4d_t *p4d; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; p4d = p4d_alloc(mm, pgd, addr); if (!p4d) return -ENOMEM; do { - next = p4d_addr_end(addr, end); + next = p4d_addr_end_mm(mm, addr, end); err = remap_pud_range(mm, p4d, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (p4d++, addr = next, addr != end); return 0; } @@ -3013,25 +3323,26 @@ static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long ad { pgd_t *pgd; unsigned long next; - unsigned long end = addr + PAGE_ALIGN(size); struct mm_struct *mm = vma->vm_mm; + unsigned long end = addr + mm_page_align(mm, size); + phys_addr_t phys = PFN_PHYS(pfn); int err; - if (WARN_ON_ONCE(!PAGE_ALIGNED(addr))) + if (WARN_ON_ONCE(!IS_ALIGNED(addr, mm_page_size(mm)))) return -EINVAL; VM_WARN_ON_ONCE(!vma_test_all_mask(vma, VMA_REMAP_FLAGS)); BUG_ON(addr >= end); - pfn -= addr >> PAGE_SHIFT; pgd = pgd_offset(mm, addr); flush_cache_range(vma, addr, end); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(mm, addr, end); err = remap_p4d_range(mm, pgd, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pgd++, addr = next, addr != end); return 0; @@ -3093,7 +3404,7 @@ static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, struct pfnmap_track_ctx *ctx = NULL; int err; - size = PAGE_ALIGN(size); + size = mm_page_align(vma->vm_mm, size); /* * If we cover the full VMA, we'll perform actual tracking, and @@ -3162,7 +3473,7 @@ static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size) { - const unsigned long end = addr + PAGE_ALIGN(size); + const unsigned long end = addr + mm_page_align(vma->vm_mm, size); const bool is_cow = is_cow_mapping(vma->vm_flags); int err; @@ -3540,15 +3851,14 @@ static inline int pte_unmap_same(struct vm_fault *vmf) * -EHWPOISON: copy failed due to hwpoison in source page * -EAGAIN: copied failed (some other reason) */ -static inline int __wp_page_copy_user(struct page *dst, struct page *src, - struct vm_fault *vmf) +static inline int __wp_native_page_copy_user(struct page *dst, struct page *src, + struct vm_fault *vmf, unsigned long addr) { int ret; void *kaddr; void __user *uaddr; struct vm_area_struct *vma = vmf->vma; struct mm_struct *mm = vma->vm_mm; - unsigned long addr = vmf->address; if (likely(src)) { if (copy_mc_user_highpage(dst, src, addr, vma)) @@ -3638,6 +3948,22 @@ static inline int __wp_page_copy_user(struct page *dst, struct page *src, return ret; } +static int __wp_page_copy_user(struct page *dst, struct page *src, + struct vm_fault *vmf) +{ + unsigned int i, nr = mm_pte_native_pages(vmf->vma->vm_mm); + + for (i = 0; i < nr; i++) { + int ret = __wp_native_page_copy_user(dst + i, src ? src + i : NULL, + vmf, vmf->address + i * PAGE_SIZE); + + if (ret) + return ret; + kmsan_copy_page_meta(dst + i, src ? src + i : NULL); + } + return 0; +} + static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma) { struct file *vm_file = vma->vm_file; @@ -3883,14 +4209,13 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) delayacct_wpcopy_end(); return err == -EHWPOISON ? VM_FAULT_HWPOISON : 0; } - kmsan_copy_page_meta(&new_folio->page, vmf->page); } __folio_mark_uptodate(new_folio); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, - vmf->address & PAGE_MASK, - (vmf->address & PAGE_MASK) + PAGE_SIZE); + vmf->address & mm_page_mask(mm), + (vmf->address & mm_page_mask(mm)) + mm_page_size(mm)); mmu_notifier_invalidate_range_start(&range); /* @@ -3927,6 +4252,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) * some TLBs while the old PTE remains in others. */ ptep_clear_flush(vma, vmf->address, vmf->pte); + folio_ref_add(new_folio, mm_pte_native_pages(mm) - 1); folio_add_new_anon_rmap(new_folio, vma, vmf->address, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, vma); BUG_ON(unshare && pte_write(entry)); @@ -3970,7 +4296,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) mmu_notifier_invalidate_range_end(&range); if (new_folio) - folio_put(new_folio); + folio_put_refs(new_folio, page_copied ? mm_pte_native_pages(mm) : 1); if (old_folio) { if (page_copied) free_swap_cache(old_folio); @@ -4092,7 +4418,7 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf, struct folio *folio) return ret; } -#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#ifdef CONFIG_MM_ID static bool __wp_can_reuse_large_anon_folio(struct folio *folio, struct vm_area_struct *vma) { @@ -4158,18 +4484,18 @@ static bool __wp_can_reuse_large_anon_folio(struct folio *folio, folio_unlock_large_mapcount(folio); return exclusive; } -#else /* !CONFIG_TRANSPARENT_HUGEPAGE */ +#else /* !CONFIG_MM_ID */ static bool __wp_can_reuse_large_anon_folio(struct folio *folio, struct vm_area_struct *vma) { BUILD_BUG(); } -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif /* CONFIG_MM_ID */ static bool wp_can_reuse_anon_folio(struct folio *folio, struct vm_area_struct *vma) { - if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && folio_test_large(folio)) + if (IS_ENABLED(CONFIG_MM_ID) && folio_test_large(folio)) return __wp_can_reuse_large_anon_folio(folio, vma); /* @@ -4207,67 +4533,238 @@ static bool wp_can_reuse_anon_folio(struct folio *folio, return true; } -/* - * This routine handles present pages, when - * * users try to write to a shared page (FAULT_FLAG_WRITE) - * * GUP wants to take a R/O pin on a possibly shared anonymous page - * (FAULT_FLAG_UNSHARE) - * - * It is done by copying the page to a new address and decrementing the - * shared-page counter for the old page. - * - * Note that this routine assumes that the protection checks have been - * done by the caller (the low-level page fault routine in most cases). - * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've - * done any necessary COW. - * - * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even - * though the page will change only once the write actually happens. This - * avoids a few races, and potentially makes it more efficient. - * - * We enter with non-exclusive mmap_lock (to exclude vma changes, - * but allow concurrent faults), with pte both mapped and locked. - * We return with mmap_lock still held, but pte unmapped and unlocked. - */ -static vm_fault_t do_wp_page(struct vm_fault *vmf) +#ifdef CONFIG_MM_SUBPAGE +/* Enter with the original PTE locked; a slot, not its native folio, is copied. */ +static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) __releases(vmf->ptl) { - const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; struct vm_area_struct *vma = vmf->vma; - struct folio *folio = NULL; - pte_t pte; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *source = NULL, *dest = NULL; + struct mm_subpage_pool *pool = NULL; + struct folio *old = vmf->page ? page_folio(vmf->page) : NULL; + struct folio *new = NULL; + struct mmu_notifier_range range; + bool copied = false, unshare = vmf->flags & FAULT_FLAG_UNSHARE; + unsigned int offset; + vm_fault_t ret = VM_FAULT_SIGBUS; + pte_t entry; + int err; - if (likely(!unshare)) { - if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) { - if (!userfaultfd_wp_async(vma)) { - pte_unmap_unlock(vmf->pte, vmf->ptl); - return handle_userfault(vmf, VM_UFFD_WP); - } + if (old && folio_test_anon(old)) { + source = mm_subpage_get_from_phys(pte_phys_mm(mm, vmf->orig_pte)); + if (WARN_ON_ONCE(!source)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + return VM_FAULT_SIGBUS; + } + if (!mm_subpage_is_exclusive(source) && mm_subpage_mapcount(source) == 1 && + folio_trylock(old)) { + bool reusable = true; - /* - * Nothing needed (cache flush, TLB invalidations, - * etc.) because we're only removing the uffd-wp bit, - * which is completely invisible to the user. - */ - pte = pte_clear_uffd_wp(ptep_get(vmf->pte)); + if (folio_test_swapcache(old)) { + swp_entry_t entry = page_swap_entry(&old->page); - set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); - /* - * Update this to be prepared for following up CoW - * handling - */ - vmf->orig_pte = pte; + reusable = !swap_subpage_count(entry, mm_subpage_offset(source)) && + !(folio_test_writeback(old) && + (folio_swap_flags(old) & SWP_STABLE_WRITES)); + } + if (reusable) + mm_subpage_try_reuse_anon(source); + folio_unlock(old); } - - /* - * Userfaultfd write-protect can defer flushes. Ensure the TLB - * is flushed in this case before copying. - */ - if (unlikely(userfaultfd_wp(vmf->vma) && - mm_tlb_flush_pending(vmf->vma->vm_mm))) - flush_tlb_page(vmf->vma, vmf->address); + if (mm_subpage_is_exclusive(source)) { + if (unshare) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + } else { + folio_xchg_last_cpupid(old, (1 << LAST_CPUPID_SHIFT) - 1); + /* Exclusivity belongs to the slot, not PageAnonExclusive. */ + wp_page_reuse(vmf, NULL); + } + mm_subpage_put(source); + return 0; + } + } else if (old) { + folio_get(old); + } else if (!is_zero_pfn(pte_pfn(vmf->orig_pte))) { + /* Private device/PFN mappings need their own subpage contract. */ + pte_unmap_unlock(vmf->pte, vmf->ptl); + return VM_FAULT_SIGBUS; } - + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + delayacct_wpcopy_start(); + ret = vmf_anon_prepare(vmf); + if (ret) + goto out; + offset = vma_page_offset_at(vma, vmf->address).offset; + ret = VM_FAULT_OOM; + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + goto out; + new = folio_prealloc(mm, vma, vmf->address, false); + if (!new) + goto out; + err = mm_subpage_pool_add_folio(pool, new, GFP_KERNEL); + if (err) { + folio_put(new); + goto out; + } + dest = source ? mm_subpage_copy_at(pool, source, offset) : + mm_subpage_alloc_at(pool, offset); + if (IS_ERR(dest)) { + if (PTR_ERR(dest) == -EHWPOISON) + ret = VM_FAULT_HWPOISON; + dest = NULL; + goto out; + } + if (old && !source) { + err = copy_user_subpage_range(&new->page, offset, vmf->page, + pte_phys_mm(mm, vmf->orig_pte) & ~PAGE_MASK, + mm_page_size(mm)); + if (err) { + ret = VM_FAULT_HWPOISON; + goto out; + } + } + mm_subpage_pool_close(pool); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, + vmf->address, vmf->address + mm_page_size(mm)); + mmu_notifier_invalidate_range_start(&range); + /* Keep the folio alive after a concurrent zap can consume its PTE ref. */ + folio_get(new); + folio_lock(new); + ret = 0; + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock_pte; + } + mm_subpage_set_exclusive(dest); + err = mm_subpage_add_new_anon_rmap(dest, vma, vmf->address); + if (WARN_ON_ONCE(err)) { + ret = VM_FAULT_SIGBUS; + goto unlock_pte; + } + entry = pte_sw_mkyoung(phys_pte_mm(mm, mm_subpage_phys(dest), vma->vm_page_prot)); + if (unshare) { + if (pte_soft_dirty(vmf->orig_pte)) + entry = pte_mksoft_dirty(entry); + if (pte_uffd_wp(vmf->orig_pte)) + entry = pte_mkuffd_wp(entry); + } else { + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + } + BUG_ON(unshare && pte_write(entry)); + flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); + /* Break-before-make also orders removal of the old slot's rmap. */ + ptep_clear_flush(vma, vmf->address, vmf->pte); + subpage_add_new_lru(new, vma, 1); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + if (source) { + mm_subpage_remove_anon_rmap(source, vma); + } else if (old) { + mm_subpage_file_map_del(vma, pte_phys_mm(mm, vmf->orig_pte)); + folio_remove_rmap_pte(old, vmf->page, vma); + dec_mm_counter(mm, mm_counter_file(old)); + inc_mm_counter(mm, MM_ANONPAGES); + } else { + ksm_might_unmap_zero_page(mm, vmf->orig_pte); + inc_mm_counter(mm, MM_ANONPAGES); + } + dest = NULL; /* The new PTE owns the reference. */ + copied = true; +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + folio_unlock(new); + folio_put(new); + mmu_notifier_invalidate_range_end(&range); +out: + if (pool) { + mm_subpage_pool_close(pool); + if (dest) + mm_subpage_put(dest); + mm_subpage_pool_put(pool); + } + if (source) { + if (copied) + mm_subpage_put(source); /* Retired PTE reference, after TLBI. */ + mm_subpage_put(source); /* Source stabilisation reference. */ + } else if (old) { + if (copied) + folio_put(old); + folio_put(old); + } + delayacct_wpcopy_end(); + return ret; +} +#endif + +/* + * This routine handles present pages, when + * * users try to write to a shared page (FAULT_FLAG_WRITE) + * * GUP wants to take a R/O pin on a possibly shared anonymous page + * (FAULT_FLAG_UNSHARE) + * + * It is done by copying the page to a new address and decrementing the + * shared-page counter for the old page. + * + * Note that this routine assumes that the protection checks have been + * done by the caller (the low-level page fault routine in most cases). + * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've + * done any necessary COW. + * + * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even + * though the page will change only once the write actually happens. This + * avoids a few races, and potentially makes it more efficient. + * + * We enter with non-exclusive mmap_lock (to exclude vma changes, + * but allow concurrent faults), with pte both mapped and locked. + * We return with mmap_lock still held, but pte unmapped and unlocked. + */ +static vm_fault_t do_wp_page(struct vm_fault *vmf) + __releases(vmf->ptl) +{ + const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; + struct vm_area_struct *vma = vmf->vma; + struct folio *folio = NULL; + pte_t pte; + + if (likely(!unshare)) { + if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) { + if (!userfaultfd_wp_async(vma)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + return handle_userfault(vmf, VM_UFFD_WP); + } + + /* + * Nothing needed (cache flush, TLB invalidations, + * etc.) because we're only removing the uffd-wp bit, + * which is completely invisible to the user. + */ + pte = pte_clear_uffd_wp(ptep_get(vmf->pte)); + + set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); + /* + * Update this to be prepared for following up CoW + * handling + */ + vmf->orig_pte = pte; + } + + /* + * Userfaultfd write-protect can defer flushes. Ensure the TLB + * is flushed in this case before copying. + */ + if (unlikely(userfaultfd_wp(vmf->vma) && + mm_tlb_flush_pending(vmf->vma->vm_mm))) + flush_tlb_page(vmf->vma, vmf->address); + } + vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte); if (vmf->page) @@ -4292,6 +4789,11 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) return wp_page_shared(vmf, folio); } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return wp_subpage_copy(vmf); +#endif + /* * Private mapping: create an exclusive anonymous page copy if reuse * is impossible. We might miss VM_WRITE for FOLL_FORCE handling. @@ -4301,8 +4803,12 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) */ if (folio && folio_test_anon(folio) && (PageAnonExclusive(vmf->page) || wp_can_reuse_anon_folio(folio, vma))) { - if (!PageAnonExclusive(vmf->page)) - SetPageAnonExclusive(vmf->page); + if (!PageAnonExclusive(vmf->page)) { + unsigned int i, nr = mm_pte_native_pages(vma->vm_mm); + + for (i = 0; i < nr; i++) + SetPageAnonExclusive(vmf->page + i); + } if (unlikely(unshare)) { pte_unmap_unlock(vmf->pte, vmf->ptl); return 0; @@ -4340,12 +4846,116 @@ static inline void unmap_mapping_range_tree(struct rb_root_cached *root, start = vma->vm_start + ((start_idx - vma->vm_pgoff) << PAGE_SHIFT); size = (end_idx - start_idx) << PAGE_SHIFT; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + unsigned long end = vma->vm_end; + + start = vma_address(vma, first_index, last_index - first_index + 1); + /* The interval tree guarantees a nonempty intersection. */ + VM_BUG_ON_VMA(IS_ERR_VALUE(start), vma); + if (last_index < vma_last_pgoff(vma)) + end = vma_address_at_offset(vma, (struct vm_page_offset) { + .index = last_index + 1, + }); + size = end - start; + } else if (mm_page_size(vma->vm_mm) > PAGE_SIZE) { + unsigned long granule = mm_page_size(vma->vm_mm); + unsigned long end = min(start + size, vma->vm_end); + + /* A native-page invalidation removes every overlapping leaf. */ + start = max(round_down(start, granule), vma->vm_start); + end = min(round_up(end, granule), vma->vm_end); + size = end - start; + } +#endif + tlb_gather_mmu(&tlb, vma->vm_mm); zap_vma_range_batched(&tlb, vma, start, size, details); tlb_finish_mmu(&tlb); } } +#ifdef CONFIG_MM_SUBPAGE +/* Unmap affected user leaves, including coarse aliases; preserve private COW. */ +void unmap_mapping_subpage_hole(struct address_space *mapping, loff_t first, u64 last) +{ + struct vm_area_struct *vma; + struct zap_details details = { .skip_cows = true }; + + i_mmap_lock_read(mapping); + vma_interval_tree_foreach(vma, &mapping->i_mmap, + first >> PAGE_SHIFT, (last - 1) >> PAGE_SHIFT) { + unsigned long start, end, granule = mm_page_size(vma->vm_mm); + u64 low = round_down((u64)first, granule), high = round_up(last, granule); + struct vm_page_offset begin = { low >> PAGE_SHIFT, low & ~PAGE_MASK }; + struct vm_page_offset finish = { high >> PAGE_SHIFT, high & ~PAGE_MASK }; + struct vm_page_offset vbegin = vma_page_offset_at(vma, vma->vm_start); + struct vm_page_offset vend = vma_page_offset_at(vma, vma->vm_end); + struct mmu_gather tlb; + + if (begin.index < vbegin.index || + (begin.index == vbegin.index && begin.offset <= vbegin.offset)) + start = vma->vm_start; + else + start = vma_address_at_offset(vma, begin); + if (finish.index > vend.index || + (finish.index == vend.index && finish.offset >= vend.offset)) + end = vma->vm_end; + else + end = vma_address_at_offset(vma, finish); + if (IS_ERR_VALUE(start) || IS_ERR_VALUE(end) || start >= end) + continue; + tlb_gather_mmu(&tlb, vma->vm_mm); + zap_vma_range_batched(&tlb, vma, start, end - start, &details); + tlb_finish_mmu(&tlb); + } + i_mmap_unlock_read(mapping); +} + +/* + * Native truncation keeps its final partial page. Smaller-granule mms must + * additionally discard complete granules in that native page, including + * private COW copies. Call on both sides of cache truncation, as for the + * ordinary unmap_mapping_range() calls. + */ +void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize) +{ + struct vm_area_struct *vma; + struct zap_details details = { }; + pgoff_t index = newsize >> PAGE_SHIFT; + unsigned int offset = newsize & ~PAGE_MASK; + + if (!offset) + return; + i_mmap_lock_read(mapping); + vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) { + unsigned long start, end, granule = mm_page_size(vma->vm_mm); + struct vm_page_offset pos = { .index = index, .offset = round_up(offset, granule) }; + struct vm_page_offset vpos = vma_page_offset_at(vma, vma->vm_start); + struct mmu_gather tlb; + + if (pos.offset >= PAGE_SIZE) + continue; + if (pos.index < vpos.index || + (pos.index == vpos.index && pos.offset < vpos.offset)) + start = vma->vm_start; + else + start = vma_address_at_offset(vma, pos); + if (IS_ERR_VALUE(start)) + continue; + end = vma->vm_end; + if (index < vma_last_pgoff(vma)) + end = vma_address_at_offset(vma, (struct vm_page_offset) { + .index = index + 1, + }); + tlb_gather_mmu(&tlb, vma->vm_mm); + zap_vma_range_batched(&tlb, vma, start, end - start, &details); + tlb_finish_mmu(&tlb); + } + i_mmap_unlock_read(mapping); +} +#endif + /** * unmap_mapping_folio() - Unmap single folio from processes. * @folio: The locked folio to be unmapped. @@ -4772,6 +5382,134 @@ static void check_swap_exclusive(struct folio *folio, swp_entry_t entry, } while (--nr_pages); } +#ifdef CONFIG_MM_SUBPAGE +/* The swap-cache index remains native-sized; the PTE retains its slot offset. */ +static vm_fault_t do_swap_subpage(struct vm_fault *vmf, swp_entry_t entry) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct swap_info_struct *si; + struct mm_subpage *slot = NULL; + struct folio *folio; + unsigned int offset = pte_swp_subpage_offset(vmf->orig_pte); + vm_fault_t ret; + pte_t pte; + int err; + + ret = vmf_anon_prepare(vmf); + if (ret) + return ret; + /* Swap encodes the physical slot, independent of its virtual position. */ + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(mm))) + return VM_FAULT_SIGBUS; + si = get_swap_device(entry); + if (!si) + return 0; + folio = swap_cache_get_folio(entry); + if (folio) { + swap_update_readahead(folio, vma, vmf->address); + } else { + folio = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, vmf); + if (!folio) { + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (vmf->pte) { + if (pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + ret = VM_FAULT_OOM; + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + } + goto out_device; + } + ret = VM_FAULT_MAJOR; + count_vm_event(PGMAJFAULT); + count_memcg_event_mm(mm, PGMAJFAULT); + } + ret |= folio_lock_or_retry(folio, vmf); + if (ret & VM_FAULT_RETRY) + goto out_folio; + if (!folio_matches_swap_entry(folio, entry)) + goto unlock_folio; + if (folio_order(folio) || !folio_test_uptodate(folio) || folio_test_ksm(folio)) { + ret = VM_FAULT_SIGBUS; + goto unlock_folio; + } + if (PageHWPoison(&folio->page)) { + ret = VM_FAULT_HWPOISON; + goto unlock_folio; + } + arch_swap_restore(folio_swap(entry, folio), folio); + slot = mm_subpage_restore_granule(folio, offset, mm_page_shift(mm), GFP_KERNEL); + if (IS_ERR(slot)) { + err = PTR_ERR(slot); + slot = NULL; + if (err != -EAGAIN) + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_folio; + } + folio_throttle_swaprate(folio, GFP_KERNEL); + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + goto unlock_pte; + err = mm_subpage_add_anon_rmap(slot, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_pte; + } + /* + * A carried marker proves exclusivity directly. After the last fork + * sharer exits, recover it only with one swap reference to this exact + * quarter and no other mapping or escaped slot reference. Dropping a + * nonpresent PTE in zap flushes pending reclaim TLB invalidations first. + * Stable-write backends must still finish writeback before reuse. + */ + if (!(folio_test_writeback(folio) && data_race(si->flags & SWP_STABLE_WRITES))) { + if (pte_swp_exclusive(vmf->orig_pte)) + mm_subpage_set_exclusive(slot); + else if (swap_subpage_count(entry, offset) == 1) + mm_subpage_try_reuse_swap(slot); + } + pte = pte_sw_mkyoung(phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot)); + if (pte_swp_soft_dirty(vmf->orig_pte)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(vmf->orig_pte)) + pte = pte_mkuffd_wp(pte); + inc_mm_counter(mm, MM_ANONPAGES); + dec_mm_counter(mm, MM_SWAPENTS); + swap_subpage_put(entry, offset, false); + set_pte_at(mm, vmf->address, vmf->pte, pte); + arch_do_swap_page_nr(mm, vma, vmf->address, pte, pte, 1); + slot = NULL; /* The restored PTE owns the slot reference. */ + vmf->orig_pte = pte; + if (should_try_to_free_swap(si, folio, vma, 2, vmf->flags)) + folio_free_swap(folio); + folio_unlock(folio); + folio_put(folio); /* Drop the cache lookup, not the PTE reference. */ + put_swap_device(si); + if (vmf->flags & FAULT_FLAG_WRITE) { + ret |= do_wp_page(vmf); + return ret & VM_FAULT_ERROR ? ret & VM_FAULT_ERROR : ret; + } + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + return ret; +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + if (slot) + mm_subpage_put(slot); + folio_unlock(folio); +out_folio: + folio_put(folio); +out_device: + put_swap_device(si); + return ret; +} +#endif + /* * We enter with non-exclusive mmap_lock (to exclude vma changes, * but allow concurrent faults), and pte mapped but not yet locked. @@ -4780,6 +5518,209 @@ static void check_swap_exclusive(struct folio *folio, swp_entry_t entry, * We return with the mmap_lock locked or unlocked in the same cases * as does filemap_fault(). */ +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SWAP) +/* One coarse PTE owns a naturally aligned group of native swap entries. */ +vm_fault_t do_swap_coarse_page(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + /* Arm64's largest base user page is 64K, its smallest native page 4K. */ + struct folio *cache[SZ_64K / PAGE_SIZE]; + unsigned int nr = mm_pte_native_pages(mm), covered = 0, count = 0, i; + swp_entry_t entry = softleaf_from_pte(vmf->orig_pte), next; + struct swap_info_struct *si; + struct folio *backing = NULL; + struct page *page; + unsigned int start; + bool copied = false, exclusive, mapped = false; + vm_fault_t ret = 0; + pte_t pte; + + vmf->pte = NULL; + if (WARN_ON_ONCE(nr <= 1 || nr > ARRAY_SIZE(cache) || + !IS_ALIGNED(swp_offset(entry), nr))) + return VM_FAULT_SIGBUS; + si = get_swap_device(entry); + if (!si) + return 0; + + while (covered < nr) { + struct folio *folio; + unsigned int take; + vm_fault_t wait; + + next = entry; + next.val += covered; + folio = swap_cache_get_folio(next); + if (!folio) { + folio = read_swap_cache_async(next, GFP_HIGHUSER_MOVABLE, vma, + vmf->address + covered * PAGE_SIZE, NULL); + if (!(ret & VM_FAULT_MAJOR)) { + ret |= VM_FAULT_MAJOR; + count_vm_event(PGMAJFAULT); + count_memcg_event_mm(mm, PGMAJFAULT); + } + } + if (!folio) + goto oom; + if (!count) { + wait = folio_lock_or_retry(folio, vmf); + if (wait & VM_FAULT_RETRY) { + folio_put(folio); + ret |= wait; + goto out; + } + } else if (!folio_trylock(folio)) { + /* Drop earlier cache locks before waiting for this I/O or owner. */ + for (i = 0; i < count; i++) { + folio_unlock(cache[i]); + folio_put(cache[i]); + } + count = 0; + covered = 0; + wait = folio_lock_or_retry(folio, vmf); + if (wait & VM_FAULT_RETRY) { + folio_put(folio); + ret |= wait; + goto out; + } + folio_unlock(folio); + folio_put(folio); + cond_resched(); + continue; + } + cache[count++] = folio; + if (!folio_matches_swap_entry(folio, next)) + goto out; + if (!folio_test_uptodate(folio)) { + ret = VM_FAULT_SIGBUS; + goto out; + } + start = swp_offset(next) - swp_offset(folio->swap); + take = min(nr - covered, folio_nr_pages(folio) - start); + for (i = 0; i < take; i++) { + if (PageHWPoison(folio_page(folio, start + i))) { + ret = VM_FAULT_HWPOISON; + goto out; + } + } + arch_swap_restore(folio_swap(next, folio), folio); + covered += take; + } + + start = swp_offset(entry) - swp_offset(cache[0]->swap); + if (count == 1 && IS_ALIGNED(start, nr) && + (folio_test_anon(cache[0]) || folio_nr_pages(cache[0]) == nr)) { + backing = cache[0]; + page = folio_page(backing, start); + } else { + backing = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, ilog2(nr), vma, vmf->address); + if (!backing) + goto oom; + copied = true; + if (mem_cgroup_swapin_charge_folio(backing, mm, GFP_KERNEL, entry)) + goto oom; + covered = 0; + for (i = 0; i < count; i++) { + unsigned int j, take; + + start = swp_offset(entry) + covered - swp_offset(cache[i]->swap); + take = min(nr - covered, folio_nr_pages(cache[i]) - start); + for (j = 0; j < take; j++) { + struct page *dst = folio_page(backing, covered + j); + struct page *src = folio_page(cache[i], start + j); + + if (copy_mc_user_highpage(dst, src, + vmf->address + (covered + j) * PAGE_SIZE, vma)) { + ret = VM_FAULT_HWPOISON; + goto out; + } + kmsan_copy_page_meta(dst, src); + } + covered += take; + } + folio_set_dirty(backing); + __folio_mark_uptodate(backing); + page = &backing->page; + } + + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + goto out; + exclusive = copied || pte_swp_exclusive(vmf->orig_pte); + if (!copied && exclusive) { + check_swap_exclusive(backing, entry, nr); + if (folio_test_writeback(backing) && data_race(si->flags & SWP_STABLE_WRITES)) + exclusive = false; + } + pte = mk_pte(page, vma->vm_page_prot); + if (pte_swp_soft_dirty(vmf->orig_pte)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(vmf->orig_pte)) + pte = pte_mkuffd_wp(pte); + if (exclusive && (vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && + !pte_needs_soft_dirty_wp(vma, pte)) { + pte = pte_mkwrite(pte, vma); + if (vmf->flags & FAULT_FLAG_WRITE) { + pte = pte_mkdirty(pte); + vmf->flags &= ~FAULT_FLAG_WRITE; + } + } + folio_ref_add(backing, nr - copied); + if (!folio_test_anon(backing)) + folio_add_new_anon_rmap(backing, vma, vmf->address, + exclusive ? RMAP_EXCLUSIVE : RMAP_NONE); + else + folio_add_anon_rmap_pte(backing, page, vma, vmf->address, + exclusive ? RMAP_EXCLUSIVE : RMAP_NONE); + if (copied) + folio_add_lru_vma(backing, vma); + inc_mm_counter(mm, MM_ANONPAGES); + dec_mm_counter(mm, MM_SWAPENTS); + flush_icache_pages(vma, page, nr); + vmf->orig_pte = pte; + set_pte_at(mm, vmf->address, vmf->pte, pte); + arch_do_swap_page_nr(mm, vma, vmf->address, pte, pte, 1); + mapped = true; + covered = 0; + for (i = 0; i < count; i++) { + unsigned int take; + + start = swp_offset(entry) + covered - swp_offset(cache[i]->swap); + take = min(nr - covered, folio_nr_pages(cache[i]) - start); + folio_put_swap_range(cache[i], folio_page(cache[i], start), take); + covered += take; + } + goto out; + +oom: + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + ret |= VM_FAULT_OOM; +out: + if (copied && !mapped) + folio_put(backing); + for (i = 0; i < count; i++) { + if (mapped) + folio_free_swap(cache[i]); + folio_unlock(cache[i]); + folio_put(cache[i]); + } + if (mapped && (vmf->flags & FAULT_FLAG_WRITE)) { + ret |= do_wp_page(vmf); + vmf->pte = NULL; /* do_wp_page drops the PT lock. */ + } else if (mapped) { + update_mmu_cache(vma, vmf->address, vmf->pte); + } + if (vmf->pte) + pte_unmap_unlock(vmf->pte, vmf->ptl); + put_swap_device(si); + if (!mapped && !(ret & (VM_FAULT_ERROR | VM_FAULT_RETRY))) + cond_resched(); + return ret; +} +#endif + vm_fault_t do_swap_page(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -4854,6 +5795,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) goto out; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return do_swap_subpage(vmf, entry); +#ifdef CONFIG_SWAP + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + return do_swap_coarse_page(vmf); +#endif +#endif + /* Prevent swapoff from happening to us. */ si = get_swap_device(entry); if (unlikely(!si)) @@ -5291,6 +6241,7 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, bool uffd_wp) { const unsigned int nr_pages = folio_nr_pages(folio); + const unsigned int nr_ptes = folio_size(folio) >> mm_page_shift(vma->vm_mm); pte_t entry = folio_mk_pte(folio, vma->vm_page_prot); entry = pte_sw_mkyoung(entry); @@ -5303,8 +6254,8 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, folio_ref_add(folio, nr_pages - 1); folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); - set_ptes(vma->vm_mm, addr, pte, entry, nr_pages); - update_mmu_cache_range(NULL, vma, addr, pte, nr_pages); + set_ptes(vma->vm_mm, addr, pte, entry, nr_ptes); + update_mmu_cache_range(NULL, vma, addr, pte, nr_ptes); } static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, @@ -5313,10 +6264,123 @@ static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, const unsigned int order = folio_order(folio); map_anon_folio_pte_nopf(folio, pte, vma, addr, uffd_wp); - add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1L << order); + add_mm_counter(vma->vm_mm, MM_ANONPAGES, folio_size(folio) >> mm_page_shift(vma->vm_mm)); count_mthp_stat(order, MTHP_STAT_ANON_FAULT_ALLOC); } +#ifdef CONFIG_MM_SUBPAGE +static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slots[PAGE_SIZE / MM_SUBPAGE_SIZE] = {}; + struct vm_page_offset pos = vma_page_offset_at(vma, vmf->address); + struct mm_subpage_pool *pool; + struct folio *folio; + unsigned long start, end, lower; + unsigned int first_offset, nr, target, i, mapped = 0; + vm_fault_t ret = VM_FAULT_OOM; + pte_t *first; + int err; + + /* One native linear index, clipped to the VMA and this PTE table. */ + lower = max(vma->vm_start, vmf->address & pmd_mask_mm(mm)); + start = vmf->address - min_t(unsigned long, pos.offset, vmf->address - lower); + end = vmf->address + min(PAGE_SIZE - pos.offset, + pmd_addr_end_mm(mm, vmf->address, vma->vm_end) - vmf->address); + first_offset = pos.offset - (vmf->address - start); + nr = (end - start) >> mm_page_shift(mm); + target = (vmf->address - start) >> mm_page_shift(mm); + + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return VM_FAULT_OOM; + /* Supply initialises every byte once, including the unissued slots. */ + folio = folio_prealloc(mm, vma, vmf->address, false); + if (!folio) + goto release; + err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (err) { + folio_put(folio); + goto release; + } + for (i = 0; i < nr; i++) { + slots[i] = mm_subpage_alloc_at(pool, first_offset + i * mm_page_size(mm)); + if (IS_ERR(slots[i])) { + slots[i] = NULL; + goto release; + } + } + + /* PTE references may disappear as soon as we drop the PTL. */ + folio_get(folio); + folio_lock(folio); + ret = 0; + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (vmf_pte_changed(vmf)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock_pte; + } + ret = check_stable_address_space(mm); + if (ret) + goto unlock_pte; + if (userfaultfd_missing(vma)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + folio_unlock(folio); + folio_put(folio); + ret = handle_userfault(vmf, VM_UFFD_MISSING); + goto release; + } + + first = vmf->pte - target; + for (i = 0; i < nr; i++) { + unsigned long address = start + i * mm_page_size(mm); + pte_t entry; + + /* Do not replace existing mappings or userfaultfd markers nearby. */ + if (i != target && !pte_none(ptep_get(first + i))) + continue; + mm_subpage_set_exclusive(slots[i]); + err = mm_subpage_add_new_anon_rmap(slots[i], vma, address); + if (WARN_ON_ONCE(err)) { + ret = VM_FAULT_SIGBUS; + break; + } + entry = phys_pte_mm(mm, mm_subpage_phys(slots[i]), vma->vm_page_prot); + entry = pte_sw_mkyoung(entry); + if (vma->vm_flags & VM_WRITE) + entry = pte_mkwrite(pte_mkdirty(entry), vma); + if (i == target && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + set_pte_at(mm, address, first + i, entry); + update_mmu_cache(vma, address, first + i); + slots[i] = NULL; /* The PTE now owns the reference. */ + mapped++; + } + if (mapped) { + subpage_add_new_lru(folio, vma, mapped); + add_mm_counter(mm, MM_ANONPAGES, mapped); + count_mthp_stat(0, MTHP_STAT_ANON_FAULT_ALLOC); + } +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + folio_unlock(folio); + folio_put(folio); +release: + mm_subpage_pool_close(pool); + for (i = 0; i < ARRAY_SIZE(slots); i++) + if (slots[i]) + mm_subpage_put(slots[i]); + mm_subpage_pool_put(pool); + return ret; +} +#endif + /* * We enter with non-exclusive mmap_lock (to exclude vma changes, * but allow concurrent faults), and pte mapped but not yet locked. @@ -5376,6 +6440,10 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) ret = vmf_anon_prepare(vmf); if (ret) return ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return do_anonymous_subpage(vmf); +#endif /* Returns NULL on OOM or ERR_PTR(-EAGAIN) if we must retry the fault */ folio = alloc_anon_folio(vmf); if (IS_ERR(folio)) @@ -5383,8 +6451,8 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) if (!folio) goto oom; - nr_pages = folio_nr_pages(folio); - addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE); + nr_pages = folio_size(folio) >> mm_page_shift(vma->vm_mm); + addr = ALIGN_DOWN(vmf->address, folio_size(folio)); /* * The memory barrier inside __folio_mark_uptodate makes sure that @@ -5517,6 +6585,10 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa * PMD mappings if THPs are disabled. As we already have a THP, * behave as if we are forcing a collapse. */ + /* Match the native PTE fault policy: partial data exposes its zero subunits. */ + if (userfaultfd_missing(vma) && shmem_uffd_has_missing_native_page(folio)) + return ret; + if (thp_disabled_by_hw() || vma_thp_disabled(vma, vma->vm_flags, /* forced_collapse=*/ true)) return ret; @@ -5558,6 +6630,8 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); add_mm_counter(vma->vm_mm, mm_counter_file(folio), HPAGE_PMD_NR); + shmem_uffd_mark_mapped(folio, folio_page_idx(folio, page) << PAGE_SHIFT, + HPAGE_PMD_SIZE); folio_add_file_rmap_pmd(folio, page, vma); /* @@ -5620,6 +6694,8 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); } else { + shmem_uffd_mark_mapped(folio, folio_page_idx(folio, page) << PAGE_SHIFT, + (unsigned long)nr << PAGE_SHIFT); folio_add_file_rmap_ptes(folio, page, nr, vma); } set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr); @@ -5636,6 +6712,252 @@ static bool vmf_pte_changed(struct vm_fault *vmf) return !pte_none(ptep_get(vmf->pte)); } +#ifdef CONFIG_MM_SUBPAGE +static bool folio_user_range_valid(struct folio *folio, unsigned long offset, + unsigned long size) +{ + return offset <= folio_size(folio) && size <= folio_size(folio) - offset && + IS_ALIGNED(page_to_phys(&folio->page) + offset, size); +} + +/* + * Map the intersection of a locked cache folio, file-index range, VMA and + * the fault's PTE table. vmf->pte points at vmf->address and its lock is held. + * Each PTE owns a reference per covered native page (one for a subpage). + * The caller's reference is untouched, including when no PTE is installed. + */ +vm_fault_t map_file_user_pages(struct vm_fault *vmf, struct folio *folio, + pgoff_t first, pgoff_t last, bool fault) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + unsigned long addr, end, table_start = vmf->address & pmd_mask_mm(mm); + unsigned long size = mm_page_size(mm); + loff_t file_size = i_size_read(vma->vm_file->f_mapping->host); + vm_fault_t ret = 0; + unsigned int mapped = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + first = max(first, folio->index); + last = min(last, folio_next_index(folio) - 1); + if (first > last) + return 0; + addr = vma_address(vma, first, last - first + 1); + if (IS_ERR_VALUE(addr)) + return 0; + addr = ALIGN(max(addr, table_start), size); + end = pmd_addr_end_mm(mm, vmf->address, vma->vm_end); + if (userfaultfd_armed(vma)) { + addr = max(addr, vmf->address); + end = min(end, vmf->address + size); + } + for (; addr < end; addr += size) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + struct page *page; + unsigned long offset; + unsigned int i, nr = mm_pte_native_pages(mm); + bool poisoned = false; + pte_t entry, *ptep; + + if (pos.index > last || !vma_file_offset_valid(vma, addr, file_size)) + break; + offset = ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset; + /* + * A coarse leaf cannot span separate cache folios or a partial + * requested index range, even if neighboring PFNs happen to match. + */ + if (nr - 1 > last - pos.index || + !folio_user_range_valid(folio, offset, size)) + continue; + /* A native cache folio can contain independently missing leaves. */ + if (userfaultfd_missing(vma) && + shmem_uffd_range_missing(folio, pos.index, pos.offset, size)) + continue; + page = folio_file_page(folio, pos.index); + ptep = vmf->pte + ((long)(addr - vmf->address) >> mm_page_shift(mm)); + for (i = 0; i < nr; i++) + poisoned |= PageHWPoison(page + i); + if (poisoned) + continue; + if (!pte_none(ptep_get(ptep)) && + !(fault && addr == vmf->address && !vmf_pte_changed(vmf))) + continue; + flush_icache_pages(vma, page, nr); + entry = phys_pte_mm(mm, page_to_phys(page) + pos.offset, vma->vm_page_prot); + if (addr != vmf->address && arch_wants_old_prefaulted_pte()) + entry = pte_mkold(entry); + else + entry = pte_sw_mkyoung(entry); + if (vmf->flags & FAULT_FLAG_WRITE) + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + else if (pte_write(entry) && folio_test_dirty(folio)) + entry = pte_mkdirty(entry); + if (fault && addr == vmf->address && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_ref_add(folio, nr); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, entry)); + shmem_uffd_mark_mapped(folio, + ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset, size); + folio_add_file_rmap_pte(folio, page, vma); + set_pte_at(mm, addr, ptep, entry); + update_mmu_cache_range(vmf, vma, addr, ptep, 1); + mapped++; + if (addr == vmf->address) + ret = VM_FAULT_NOPAGE; + } + add_mm_counter(mm, mm_counter_file(folio), mapped); + return ret; +} + +/* + * A non-file fault callback supplies one native page, not a cache-indexed + * folio. In particular, VVAR callbacks may reorder native pages. Only map + * quarters of this supplied page; never infer adjacent pages from its folio. + */ +static vm_fault_t map_special_subpages(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct page *page = vmf->page; + struct folio *folio = page_folio(page); + unsigned int offset = vma_page_offset_at(vma, vmf->address).offset; + unsigned long size = mm_page_size(mm), addr, end; + vm_fault_t ret = 0; + unsigned int mapped = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (PageHWPoison(page)) + return 0; + addr = offset > vmf->address ? vma->vm_start : + max(vma->vm_start, vmf->address - offset); + addr = max(addr, vmf->address & pmd_mask_mm(mm)); + end = min(vmf->address + PAGE_SIZE - offset, + pmd_addr_end_mm(mm, vmf->address, vma->vm_end)); + if (userfaultfd_armed(vma)) { + addr = vmf->address; + end = vmf->address + size; + } + flush_icache_pages(vma, page, 1); + for (; addr < end; addr += size) { + unsigned int physical_offset = addr < vmf->address ? + offset - (vmf->address - addr) : offset + (addr - vmf->address); + pte_t *ptep = vmf->pte + ((long)(addr - vmf->address) >> mm_page_shift(mm)); + pte_t entry; + + if (!pte_none(ptep_get(ptep)) && + !(addr == vmf->address && !vmf_pte_changed(vmf))) + continue; + entry = phys_pte_mm(mm, page_to_phys(page) + physical_offset, vma->vm_page_prot); + entry = pte_sw_mkyoung(entry); + if (vmf->flags & FAULT_FLAG_WRITE) + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + if (addr == vmf->address && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_get(folio); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, entry)); + folio_add_file_rmap_pte(folio, page, vma); + set_pte_at(mm, addr, ptep, entry); + update_mmu_cache_range(vmf, vma, addr, ptep, 1); + mapped++; + if (addr == vmf->address) + ret = VM_FAULT_NOPAGE; + } + add_mm_counter(mm, mm_counter_file(folio), mapped); + return ret; +} + +static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slot = vmf->cow_subpage; + bool is_cow = (vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED); + struct folio *folio = page_folio(is_cow ? vmf->cow_page : vmf->page); + vm_fault_t ret; + pte_t entry; + + if (!(vma->vm_flags & VM_SHARED)) { + ret = check_stable_address_space(mm); + if (ret) + return ret; + } + if (pmd_none(*vmf->pmd)) { + if (vmf->prealloc_pte) + pmd_install(mm, vmf->pmd, &vmf->prealloc_pte); + else if (pte_alloc(mm, vmf->pmd)) + return VM_FAULT_OOM; + } + /* The private destination is not yet visible through a PTE. */ + if (is_cow) { + folio_get(folio); + folio_lock(folio); + } + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + ret = VM_FAULT_NOPAGE; + if (!vmf->pte) + goto unlock_folio; + if (vmf_pte_changed(vmf)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock; + } + if (slot) { + mm_subpage_set_exclusive(slot); + if (WARN_ON_ONCE(mm_subpage_add_new_anon_rmap(slot, vma, vmf->address))) { + ret = VM_FAULT_SIGBUS; + goto unlock; + } + entry = phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot); + entry = maybe_mkwrite(pte_mkdirty(pte_sw_mkyoung(entry)), vma); + if (vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + subpage_add_new_lru(folio, vma, 1); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + inc_mm_counter(mm, MM_ANONPAGES); + vmf->cow_subpage = NULL; + ret = 0; + } else if (is_cow) { + if (!folio_user_range_valid(folio, 0, mm_page_size(mm))) { + ret = VM_FAULT_SIGBUS; + goto unlock; + } + entry = phys_pte_mm(mm, page_to_phys(&folio->page), vma->vm_page_prot); + entry = maybe_mkwrite(pte_mkdirty(pte_sw_mkyoung(entry)), vma); + if (vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_ref_add(folio, mm_pte_native_pages(mm) - 1); + folio_add_new_anon_rmap(folio, vma, vmf->address, RMAP_EXCLUSIVE); + folio_add_lru_vma(folio, vma); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + inc_mm_counter(mm, MM_ANONPAGES); + ret = 0; + } else { + if (vma->vm_file) + ret = map_file_user_pages(vmf, folio, folio->index, + folio_next_index(folio) - 1, true); + else if (mm_page_size(mm) < PAGE_SIZE) + ret = map_special_subpages(vmf); + else + ret = VM_FAULT_SIGBUS; + if (ret & VM_FAULT_NOPAGE) { + folio_put(folio); /* Consume the fault callback's reference. */ + ret = 0; + } else { + ret = VM_FAULT_SIGBUS; + } + } +unlock: + pte_unmap_unlock(vmf->pte, vmf->ptl); +unlock_folio: + if (is_cow) { + folio_unlock(folio); + folio_put(folio); + } + return ret; +} +#endif + /** * finish_fault - finish page fault once we have prepared the page to fault * @@ -5663,6 +6985,11 @@ vm_fault_t finish_fault(struct vm_fault *vmf) unsigned long addr; bool needs_fallback = false; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return finish_user_page_fault(vmf); +#endif + fallback: addr = vmf->address; @@ -5768,13 +7095,12 @@ vm_fault_t finish_fault(struct vm_fault *vmf) return ret; } -static unsigned long fault_around_pages __read_mostly = - 65536 >> PAGE_SHIFT; +static unsigned long fault_around_bytes __read_mostly = SZ_64K; #ifdef CONFIG_DEBUG_FS static int fault_around_bytes_get(void *data, u64 *val) { - *val = fault_around_pages << PAGE_SHIFT; + *val = READ_ONCE(fault_around_bytes); return 0; } @@ -5788,11 +7114,13 @@ static int fault_around_bytes_set(void *data, u64 val) return -EINVAL; /* - * The minimum value is 1 page, however this results in no fault-around - * at all. See should_fault_around(). + * Keep the policy in bytes: one native page can still contain several + * user leaves. The smallest supported leaf disables fault-around for + * every mm; larger values are compared with each mm's own granule. */ - val = max(val, PAGE_SIZE); - fault_around_pages = rounddown_pow_of_two(val) >> PAGE_SHIFT; + val = max_t(u64, val, IS_ENABLED(CONFIG_MM_SUBPAGE) ? + MM_SUBPAGE_SIZE : PAGE_SIZE); + WRITE_ONCE(fault_around_bytes, rounddown_pow_of_two(val)); return 0; } @@ -5819,24 +7147,42 @@ late_initcall(fault_around_debugfs); * This function doesn't cross VMA or page table boundaries, in order to call * map_pages() and acquire a PTE lock only once. * - * fault_around_pages defines how many pages we'll try to map. - * do_fault_around() expects it to be set to a power of two less than or equal - * to PTRS_PER_PTE. - * - * The virtual address of the area that we map is naturally aligned to - * fault_around_pages * PAGE_SIZE rounded down to the machine page size - * (and therefore to page order). This way it's easier to guarantee - * that we don't cross page table boundaries. + * fault_around_bytes is a power-of-two byte policy, independent of either + * physical or user granules. The native path converts it to native pages; + * the alternative path clamps it to the target mm's PTE-table coverage. + * Natural alignment keeps the window inside one page table. */ static vm_fault_t do_fault_around(struct vm_fault *vmf) { - pgoff_t nr_pages = READ_ONCE(fault_around_pages); + unsigned long configured_bytes = READ_ONCE(fault_around_bytes); + pgoff_t nr_pages = max(1UL, configured_bytes >> PAGE_SHIFT); pgoff_t pte_off = pte_index(vmf->address); /* The page offset of vmf->address within the VMA. */ pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff; pgoff_t from_pte, to_pte; vm_fault_t ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) != PAGE_SIZE) { + struct vm_area_struct *vma = vmf->vma; + unsigned long bytes = min(configured_bytes, + pmd_size_mm(vma->vm_mm)); + unsigned long start = max(ALIGN_DOWN(vmf->address, bytes), vma->vm_start); + unsigned long end = min(ALIGN_DOWN(vmf->address, bytes) + bytes, vma->vm_end); + + if (pmd_none(*vmf->pmd)) { + vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); + if (!vmf->prealloc_pte) + return VM_FAULT_OOM; + } + rcu_read_lock(); + ret = vma->vm_ops->map_pages(vmf, vma_page_offset_at(vma, start).index, + vma_page_offset_at(vma, end - 1).index); + rcu_read_unlock(); + return ret; + } +#endif + /* The PTE offset of the start address, clamped to the VMA. */ from_pte = max(ALIGN_DOWN(pte_off, nr_pages), pte_off - min(pte_off, vma_off)); @@ -5871,7 +7217,7 @@ static inline bool should_fault_around(struct vm_fault *vmf) return false; /* A single page implies no faulting 'around' at all. */ - return fault_around_pages > 1; + return READ_ONCE(fault_around_bytes) > mm_page_size(vmf->vma->vm_mm); } static vm_fault_t do_read_fault(struct vm_fault *vmf) @@ -5918,10 +7264,26 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) if (ret) return ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + vmf->cow_subpage = anon_subpage_prealloc(vma->vm_mm, vma, vmf->address); + if (IS_ERR(vmf->cow_subpage)) { + vmf->cow_subpage = NULL; + return VM_FAULT_OOM; + } + folio = mm_subpage_folio(vmf->cow_subpage); + goto allocated; + } +#endif + folio = folio_prealloc(vma->vm_mm, vma, vmf->address, false); if (!folio) return VM_FAULT_OOM; +#ifdef CONFIG_MM_SUBPAGE +allocated: +#endif + vmf->cow_page = &folio->page; ret = __do_fault(vmf); @@ -5930,10 +7292,38 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) if (ret & VM_FAULT_DONE_COW) return ret; - if (copy_mc_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma)) { - ret = VM_FAULT_HWPOISON; +#ifdef CONFIG_MM_SUBPAGE + if (vmf->cow_subpage) { + if (copy_user_subpage_range(vmf->cow_page, mm_subpage_offset(vmf->cow_subpage), + vmf->page, vma_page_offset_at(vma, vmf->address).offset, + mm_page_size(vma->vm_mm))) { + ret = VM_FAULT_HWPOISON; + goto unlock; + } + goto copied; + } +#endif + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) > PAGE_SIZE && + !folio_user_range_valid(page_folio(vmf->page), + folio_page_idx(page_folio(vmf->page), vmf->page) << PAGE_SHIFT, + mm_page_size(vma->vm_mm))) { + ret = VM_FAULT_SIGBUS; goto unlock; } +#endif + for (unsigned int i = 0; i < mm_pte_native_pages(vma->vm_mm); i++) { + if (copy_mc_user_highpage(vmf->cow_page + i, vmf->page + i, + vmf->address + i * PAGE_SIZE, vma)) { + ret = VM_FAULT_HWPOISON; + goto unlock; + } + } +#ifdef CONFIG_MM_SUBPAGE +copied: +#endif + __folio_mark_uptodate(folio); ret |= finish_fault(vmf); @@ -5944,6 +7334,13 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) goto uncharge_out; return ret; uncharge_out: +#ifdef CONFIG_MM_SUBPAGE + if (vmf->cow_subpage) { + mm_subpage_put(vmf->cow_subpage); + vmf->cow_subpage = NULL; + return ret; + } +#endif folio_put(folio); return ret; } @@ -6455,7 +7852,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, { struct vm_fault vmf = { .vma = vma, - .address = address & PAGE_MASK, + .address = address & mm_page_mask(vma->vm_mm), .real_address = address, .flags = flags, .pgoff = linear_page_index(vma, address), @@ -6786,7 +8183,7 @@ int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) return -ENOMEM; spin_lock(&mm->page_table_lock); - if (!p4d_present(*p4d)) { + if (!p4d_present_mm(mm, *p4d)) { mm_inc_nr_puds(mm); smp_wmb(); /* See comment in pmd_install() */ p4d_populate(mm, p4d, new); @@ -6824,16 +8221,19 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) static inline int pfnmap_args_setup(struct follow_pfnmap_args *args, spinlock_t *lock, pte_t *ptep, - pgprot_t pgprot, unsigned long pfn_base, + pgprot_t pgprot, phys_addr_t phys_base, unsigned long addr_mask, bool writable, bool special) { + phys_addr_t phys = phys_base + (args->address & ~addr_mask); + if (!writable && args->write) return -EFAULT; args->lock = lock; args->ptep = ptep; - args->pfn = pfn_base + ((args->address & ~addr_mask) >> PAGE_SHIFT); + args->pfn = PHYS_PFN(phys); + args->offset = offset_in_page(phys); args->addr_mask = addr_mask; args->pgprot = pgprot; args->writable = writable; @@ -6885,8 +8285,14 @@ static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma) * * This function must not be used to modify PTE content. * + * Legacy callers retain the native-page contract. A caller setting + * allow_subpage must use pfn plus offset to identify the requested physical + * byte and must limit access to the leaf described by addr_mask. This does + * not pin the native page or grant access to neighbouring user mappings. + * * Return: zero on success, -EFAULT if @args->write was set but the - * mapping is not writable, -EINVAL if there is no mapping at all. + * mapping is not writable, -EINVAL if there is no mapping at all, or + * -EOPNOTSUPP for a non-native granule without allow_subpage. */ int follow_pfnmap_start(struct follow_pfnmap_args *args) { @@ -6908,17 +8314,20 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) goto out; + /* A legacy PFN consumer cannot represent partial native-page ownership. */ + if (mm_page_size(mm) != PAGE_SIZE && !args->allow_subpage) + return -EOPNOTSUPP; retry: pgdp = pgd_offset(mm, address); if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) goto out; - p4dp = p4d_offset(pgdp, address); + p4dp = p4d_offset_mm(mm, pgdp, address); p4d = p4dp_get(p4dp); - if (p4d_none(p4d) || unlikely(p4d_bad(p4d))) + if (p4d_none_mm(mm, p4d) || unlikely(p4d_bad_mm(mm, p4d))) goto out; - pudp = pud_offset(p4dp, address); + pudp = pud_offset_mm(mm, p4dp, address); pud = pudp_get(pudp); if (!pud_present(pud)) goto out; @@ -6934,14 +8343,14 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) goto retry; } r = pfnmap_args_setup(args, lock, NULL, pud_pgprot(pud), - pud_pfn(pud), PUD_MASK, pud_write(pud), + PFN_PHYS(pud_pfn(pud)), pud_mask_mm(mm), pud_write(pud), pud_special(pud)); if (r) spin_unlock(lock); return r; } - pmdp = pmd_offset(pudp, address); + pmdp = pmd_offset_mm(mm, pudp, address); pmd = pmdp_get_lockless(pmdp); if (!pmd_present(pmd)) goto out; @@ -6957,7 +8366,7 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) goto retry; } r = pfnmap_args_setup(args, lock, NULL, pmd_pgprot(pmd), - pmd_pfn(pmd), PMD_MASK, pmd_write(pmd), + PFN_PHYS(pmd_pfn(pmd)), pmd_mask_mm(mm), pmd_write(pmd), pmd_special(pmd)); if (r) spin_unlock(lock); @@ -6970,8 +8379,8 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) pte = ptep_get(ptep); if (!pte_present(pte)) goto unlock; - r = pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte), - pte_pfn(pte), PAGE_MASK, pte_write(pte), + r = pfnmap_args_setup(args, lock, ptep, pte_pgprot_mm(mm, pte), + pte_phys_mm(mm, pte), mm_page_mask(mm), pte_write(pte), pte_special(pte)); if (r) goto unlock; @@ -7018,20 +8427,30 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, resource_size_t phys_addr; pgprot_t prot = __pgprot(0); void __iomem *maddr; - int offset = offset_in_page(addr); + int offset; int ret = -EINVAL; bool writable; struct follow_pfnmap_args args = { .vma = vma, .address = addr, - .write = !!(write & FOLL_WRITE) + .write = !!(write & FOLL_WRITE), + .allow_subpage = true }; + if (len <= 0) + return 0; + if (addr < vma->vm_start || addr >= vma->vm_end) + return -EINVAL; + /* Each following leaf needs its own translation and permission check. */ + len = min_t(unsigned long, len, vma->vm_end - addr); + len = min_t(unsigned long, len, mm_page_size(vma->vm_mm) - + (addr & ~mm_page_mask(vma->vm_mm))); retry: if (follow_pfnmap_start(&args)) return -EINVAL; prot = args.pgprot; phys_addr = (resource_size_t)args.pfn << PAGE_SHIFT; + offset = args.offset; writable = args.writable; follow_pfnmap_end(&args); @@ -7043,7 +8462,8 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, goto out_unmap; if ((pgprot_val(prot) != pgprot_val(args.pgprot)) || - (phys_addr != (args.pfn << PAGE_SHIFT)) || + (phys_addr != ((resource_size_t)args.pfn << PAGE_SHIFT)) || + (offset != args.offset) || (writable != args.writable)) { follow_pfnmap_end(&args); iounmap(maddr); @@ -7089,8 +8509,19 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, void *maddr; struct folio *folio; struct vm_area_struct *vma = NULL; - struct page *page = get_user_page_vma_remote(mm, addr, - gup_flags, &vma); + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment = {}; + + if (mm_page_size(mm) != PAGE_SIZE) { + int got = get_user_fragment_vma_remote(mm, addr, len, + gup_flags, &fragment, &vma); + + page = got > 0 ? folio_page(fragment.folio, + fragment.offset >> PAGE_SHIFT) : ERR_PTR(got ?: -EFAULT); + } else +#endif + page = get_user_page_vma_remote(mm, addr, gup_flags, &vma); if (IS_ERR(page)) { /* We might need to expand the stack to access it */ @@ -7124,6 +8555,12 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, offset = addr & (PAGE_SIZE-1); if (bytes > PAGE_SIZE-offset) bytes = PAGE_SIZE-offset; +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) { + offset = offset_in_page(fragment.offset); + bytes = fragment.length; + } +#endif maddr = kmap_local_folio(folio, folio_page_idx(folio, page) * PAGE_SIZE); if (write) { @@ -7134,7 +8571,13 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, copy_from_user_page(vma, page, addr, buf, maddr + offset, bytes); } - folio_release_kmap(folio, maddr); +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) { + kunmap_local(maddr); + release_user_fragments(&fragment, 1, false); + } else +#endif + folio_release_kmap(folio, maddr); } len -= bytes; buf += bytes; diff --git a/mm/migrate.c b/mm/migrate.c index 8a64291ab5b44..795143e1b4cc4 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -14,6 +14,8 @@ */ #include +#include +#include #include #include #include @@ -340,14 +342,50 @@ struct rmap_walk_arg { bool map_unused_to_zeropage; }; +#ifdef CONFIG_MM_SUBPAGE +static void remove_migration_subpage(struct folio *folio, + struct page_vma_mapped_walk *pvmw, pte_t old) +{ + struct vm_area_struct *vma = pvmw->vma; + softleaf_t entry = softleaf_from_pte(old); + struct mm_subpage *slot; + pte_t pte; + int ret; + + slot = mm_subpage_migrate_restore(folio, pte_swp_subpage_offset(old)); + pte = phys_pte_mm(vma->vm_mm, mm_subpage_phys(slot), READ_ONCE(vma->vm_page_prot)); + if (!softleaf_is_migration_young(entry)) + pte = pte_mkold(pte); + if (folio_test_dirty(folio) && softleaf_is_migration_dirty(entry)) + pte = pte_mkdirty(pte); + if (pte_swp_soft_dirty(old)) + pte = pte_mksoft_dirty(pte); + else + pte = pte_clear_soft_dirty(pte); + if (softleaf_is_migration_write(entry)) + pte = pte_mkwrite(pte, vma); + else if (pte_swp_uffd_wp(old)) + pte = pte_mkuffd_wp(pte); + if (!softleaf_is_migration_read(entry)) + mm_subpage_set_exclusive(slot); + ret = mm_subpage_add_anon_rmap(slot, vma, pvmw->address); + VM_BUG_ON_FOLIO(ret, folio); + set_pte_at(vma->vm_mm, pvmw->address, pvmw->pte, pte); + if (READ_ONCE(vma->vm_flags) & VM_LOCKED) + mlock_drain_local(); + trace_remove_migration_pte(pvmw->address, pte_val(pte), 0); + update_mmu_cache(vma, pvmw->address, pvmw->pte); +} +#endif + /* * Restore a potential migration pte to a working pte entry */ static bool remove_migration_pte(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { struct rmap_walk_arg *rmap_walk_arg = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION); + DEFINE_FOLIO_RMAP_WALK(pvmw, rmap_walk_arg->folio, vma, rmap, PVMW_SYNC | PVMW_MIGRATION); while (page_vma_mapped_walk(&pvmw)) { rmap_t rmap_flags = RMAP_NONE; @@ -372,13 +410,29 @@ static bool remove_migration_pte(struct folio *folio, } #endif old_pte = ptep_get(pvmw.pte); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE && folio_test_anon(folio)) { + remove_migration_subpage(folio, &pvmw, old_pte); + continue; + } +#endif if (rmap_walk_arg->map_unused_to_zeropage && try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx)) continue; - folio_get(folio); + /* Match the native-page references dropped by try_to_migrate_one(). */ + if (!folio_test_hugetlb(folio)) + folio_ref_add(folio, mm_pte_native_pages(vma->vm_mm)); + else + folio_get(folio); pte = mk_pte(new, READ_ONCE(vma->vm_page_prot)); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + pte = phys_pte_mm(vma->vm_mm, page_to_phys(new) + + pte_swp_subpage_offset(old_pte), READ_ONCE(vma->vm_page_prot)); +#endif + entry = softleaf_from_pte(old_pte); if (!softleaf_is_migration_young(entry)) pte = pte_mkold(pte); @@ -431,8 +485,10 @@ static bool remove_migration_pte(struct folio *folio, if (folio_test_anon(folio)) folio_add_anon_rmap_pte(folio, new, vma, pvmw.address, rmap_flags); - else + else { + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, pte)); folio_add_file_rmap_pte(folio, new, vma); + } set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); } if (READ_ONCE(vma->vm_flags) & VM_LOCKED) @@ -640,10 +696,12 @@ static int __folio_migrate_mapping(struct address_space *mapping, folio_set_dirty(newfolio); } - if (folio_test_swapcache(folio)) + if (folio_test_swapcache(folio)) { __swap_cache_replace_folio(ci, folio, newfolio); - else + } else { + shmem_uffd_migrate(newfolio, folio); xas_store(&xas, newfolio); + } /* * Drop cache reference from old folio by unfreezing @@ -1170,8 +1228,10 @@ static void migrate_folio_undo_src(struct folio *src, /* Drop an anon_vma reference if we took one */ if (anon_vma) put_anon_vma(anon_vma); - if (locked) + if (locked) { + mm_subpage_migrate_finish(src); folio_unlock(src); + } if (ret) list_move_tail(&src->lru, ret); } @@ -1180,8 +1240,10 @@ static void migrate_folio_undo_src(struct folio *src, static void migrate_folio_undo_dst(struct folio *dst, bool locked, free_folio_t put_new_folio, unsigned long private) { - if (locked) + if (locked) { + mm_subpage_migrate_finish(dst); folio_unlock(dst); + } if (put_new_folio) put_new_folio(dst, private); else @@ -1306,6 +1368,11 @@ static int migrate_folio_unmap(new_folio_t get_new_folio, return 0; } + rc = mm_subpage_migrate_prepare(src, dst); + if (rc) + goto out; + rc = -EAGAIN; + /* * Corner case handling: * 1. When a new swap-cache page is read into, it is added to the LRU @@ -1411,6 +1478,8 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private, remove_migration_ptes(src, dst, 0); out_unlock_both: + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); folio_unlock(dst); folio_set_owner_migrate_reason(dst, reason); /* diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 17074c2573b94..bae35a327cf1b 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -408,7 +408,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, if (anon_exclusive) { pte = ptep_clear_flush(vma, addr, ptep); - if (folio_try_share_anon_rmap_pte(folio, page)) { + if (folio_try_share_anon_rmap_pte(folio, page, vma)) { set_pte_at(mm, addr, ptep, pte); if (fault_folio != folio) folio_unlock(folio); diff --git a/mm/mincore.c b/mm/mincore.c index c8757c5085bf9..66458b6bb2091 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -48,7 +48,7 @@ static int mincore_hugetlb(pte_t *pte, unsigned long hmask, unsigned long addr, present = 1; } - for (; addr != end; vec++, addr += PAGE_SIZE) + for (; addr != end; vec++, addr += mm_page_size(walk->mm)) *vec = present; walk->private = vec; spin_unlock(ptl); @@ -135,15 +135,22 @@ static unsigned char mincore_page(struct address_space *mapping, pgoff_t index) static int __mincore_unmapped_range(unsigned long addr, unsigned long end, struct vm_area_struct *vma, unsigned char *vec) { - unsigned long nr = (end - addr) >> PAGE_SHIFT; + unsigned long nr = (end - addr) >> mm_page_shift(vma->vm_mm); int i; if (vma->vm_file) { - pgoff_t pgoff; + unsigned int shift = mm_page_shift(vma->vm_mm); - pgoff = linear_page_index(vma, addr); - for (i = 0; i < nr; i++, pgoff++) - vec[i] = mincore_page(vma->vm_file->f_mapping, pgoff); + /* Cache indices are native; several user leaves can share one. */ + for (i = 0; i < nr; ) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + unsigned long batch = min(nr - i, (PAGE_SIZE - pos.offset) >> shift); + unsigned char present = mincore_page(vma->vm_file->f_mapping, pos.index); + + memset(vec + i, present, batch); + i += batch; + addr += batch << shift; + } } else { for (i = 0; i < nr; i++) vec[i] = 0; @@ -167,7 +174,7 @@ static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct vm_area_struct *vma = walk->vma; pte_t *ptep; unsigned char *vec = walk->private; - int nr = (end - addr) >> PAGE_SHIFT; + int nr = (end - addr) >> mm_page_shift(walk->mm); int step, i; ptl = pmd_trans_huge_lock(pmd, vma); @@ -182,19 +189,19 @@ static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; ptep += step, addr += step * PAGE_SIZE) { + for (; addr != end; ptep += step, addr += step * mm_page_size(walk->mm)) { pte_t pte = ptep_get(ptep); step = 1; /* We need to do cache lookup too for markers */ if (pte_none(pte) || pte_is_marker(pte)) - __mincore_unmapped_range(addr, addr + PAGE_SIZE, + __mincore_unmapped_range(addr, addr + mm_page_size(walk->mm), vma, vec); else if (pte_present(pte)) { unsigned int batch = pte_batch_hint(ptep, pte); if (batch > 1) { - unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + unsigned int max_nr = (end - addr) >> mm_page_shift(walk->mm); step = min_t(unsigned int, batch, max_nr); } @@ -252,16 +259,16 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v vma = vma_lookup(current->mm, addr); if (!vma) return -ENOMEM; - end = min(vma->vm_end, addr + (pages << PAGE_SHIFT)); + end = min(vma->vm_end, addr + (pages << mm_page_shift(vma->vm_mm))); if (!can_do_mincore(vma)) { - unsigned long pages = DIV_ROUND_UP(end - addr, PAGE_SIZE); + unsigned long pages = DIV_ROUND_UP(end - addr, mm_page_size(vma->vm_mm)); memset(vec, 1, pages); return pages; } err = walk_page_range(vma->vm_mm, addr, end, &mincore_walk_ops, vec); if (err < 0) return err; - return (end - addr) >> PAGE_SHIFT; + return (end - addr) >> mm_page_shift(vma->vm_mm); } /* @@ -281,7 +288,7 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v * return values: * zero - success * -EFAULT - vec points to an illegal address - * -EINVAL - addr is not a multiple of PAGE_SIZE + * -EINVAL - addr is not aligned to the process page size * -ENOMEM - Addresses in the range [addr, addr + len] are * invalid for the address space of this process, or * specify one or more pages which are not currently @@ -298,7 +305,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, start = untagged_addr(start); /* Check the start address: needs to be page-aligned.. */ - if (unlikely(start & ~PAGE_MASK)) + if (unlikely(start & ~mm_page_mask(current->mm))) return -EINVAL; /* ..and we need to be passed a valid user-space range */ @@ -306,8 +313,8 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, return -ENOMEM; /* This also avoids any overflows on PAGE_ALIGN */ - pages = len >> PAGE_SHIFT; - pages += (offset_in_page(len)) != 0; + pages = len >> mm_page_shift(current->mm); + pages += (len & ~mm_page_mask(current->mm)) != 0; if (!access_ok(vec, pages)) return -EFAULT; @@ -334,7 +341,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, } pages -= retval; vec += retval; - start += retval << PAGE_SHIFT; + start += retval << mm_page_shift(current->mm); retval = 0; } free_page((unsigned long) tmp); diff --git a/mm/mlock.c b/mm/mlock.c index 8c227fefa2df0..a6b67d6b20cac 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -304,15 +304,19 @@ void munlock_folio(struct folio *folio) local_unlock(&mlock_fbatch.lock); } -static inline unsigned int folio_mlock_step(struct folio *folio, +static inline unsigned int folio_mlock_step(struct vm_area_struct *vma, struct folio *folio, pte_t *pte, unsigned long addr, unsigned long end) { - unsigned int count = (end - addr) >> PAGE_SHIFT; + unsigned int count = (end - addr) >> mm_page_shift(vma->vm_mm); pte_t ptent = ptep_get(pte); if (!folio_test_large(folio)) return 1; + #ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch(vma, folio, pte, ptent, count, 0); +#endif return folio_pte_batch(folio, pte, ptent, count); } @@ -341,7 +345,7 @@ static inline bool allow_mlock_munlock(struct folio *folio, return false; /* folio is not fully mapped, skip mlock */ - if (step != folio_nr_pages(folio)) + if (step != (folio_size(folio) >> mm_page_shift(vma->vm_mm))) return false; return true; @@ -381,7 +385,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, return 0; } - for (pte = start_pte; addr != end; pte++, addr += PAGE_SIZE) { + for (pte = start_pte; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) { ptent = ptep_get(pte); if (!pte_present(ptent)) continue; @@ -389,7 +393,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, if (!folio || folio_is_zone_device(folio)) continue; - step = folio_mlock_step(folio, pte, addr, end); + step = folio_mlock_step(vma, folio, pte, addr, end); if (!allow_mlock_munlock(folio, vma, start, end, step)) goto next_entry; @@ -400,7 +404,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, next_entry: pte += step - 1; - addr += (step - 1) << PAGE_SHIFT; + addr += (step - 1) << mm_page_shift(vma->vm_mm); } pte_unmap(start_pte); out: @@ -471,7 +475,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags); const vma_flags_t old_vma_flags = vma->flags; struct mm_struct *mm = vma->vm_mm; - int nr_pages; + long nr_pages; int ret = 0; if (vma_flags_same_pair(&old_vma_flags, &new_vma_flags) || @@ -492,7 +496,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, /* * Keep track of amount of locked VM. */ - nr_pages = (end - start) >> PAGE_SHIFT; + nr_pages = (end - start) >> MM_ACCOUNT_SHIFT; if (!vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT)) nr_pages = -nr_pages; else if (vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) @@ -524,8 +528,8 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, struct vm_area_struct *vma, *prev; VMA_ITERATOR(vmi, current->mm, start); - VM_BUG_ON(offset_in_page(start)); - VM_BUG_ON(len != PAGE_ALIGN(len)); + VM_BUG_ON((start & ~mm_page_mask(current->mm))); + VM_BUG_ON(len != mm_page_align(current->mm, len)); end = start + len; if (end < start) return -EINVAL; @@ -572,7 +576,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, * vma pages, as return value. * Note deferred memory locking case(mlock2(,,MLOCK_ONFAULT) * is also counted. - * Return value: previously mlocked page counts + * Return value: previously mlocked MM_ACCOUNT_SHIFT units */ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, unsigned long start, size_t len) @@ -600,7 +604,7 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, } } - return count >> PAGE_SHIFT; + return count >> MM_ACCOUNT_SHIFT; } /* @@ -626,12 +630,12 @@ static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t fla if (!can_do_mlock()) return -EPERM; - len = PAGE_ALIGN(len + (offset_in_page(start))); - start &= PAGE_MASK; + len = mm_page_align(current->mm, len + (start & ~mm_page_mask(current->mm))); + start &= mm_page_mask(current->mm); lock_limit = rlimit(RLIMIT_MEMLOCK); - lock_limit >>= PAGE_SHIFT; - locked = len >> PAGE_SHIFT; + lock_limit >>= MM_ACCOUNT_SHIFT; + locked = len >> MM_ACCOUNT_SHIFT; if (mmap_write_lock_killable(current->mm)) return -EINTR; @@ -686,8 +690,8 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) start = untagged_addr(start); - len = PAGE_ALIGN(len + (offset_in_page(start))); - start &= PAGE_MASK; + len = mm_page_align(current->mm, len + (start & ~mm_page_mask(current->mm))); + start &= mm_page_mask(current->mm); if (mmap_write_lock_killable(current->mm)) return -EINTR; @@ -761,7 +765,7 @@ SYSCALL_DEFINE1(mlockall, int, flags) return -EPERM; lock_limit = rlimit(RLIMIT_MEMLOCK); - lock_limit >>= PAGE_SHIFT; + lock_limit >>= mm_page_shift(current->mm); if (mmap_write_lock_killable(current->mm)) return -EINTR; diff --git a/mm/mm_init.c b/mm/mm_init.c index c7e498fc1c4c0..8d14ce83be36b 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -57,7 +57,8 @@ unsigned long zero_page_pfn __ro_after_init; EXPORT_SYMBOL(zero_page_pfn); #ifndef __HAVE_COLOR_ZERO_PAGE -uint8_t empty_zero_page[PAGE_SIZE] __page_aligned_bss; +uint8_t empty_zero_page[ARCH_ZERO_PAGE_SIZE] __page_aligned_bss + __aligned(ARCH_ZERO_PAGE_SIZE); EXPORT_SYMBOL(empty_zero_page); struct page *__zero_page __ro_after_init; diff --git a/mm/mmap.c b/mm/mmap.c index 2311ae7c2ff45..097161e3e61ef 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -151,8 +151,8 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) mm->end_data, mm->start_data)) goto out; - newbrk = PAGE_ALIGN(brk); - oldbrk = PAGE_ALIGN(mm->brk); + newbrk = mm_page_align(mm, brk); + oldbrk = mm_page_align(mm, mm->brk); if (oldbrk == newbrk) { mm->brk = brk; goto success; @@ -186,8 +186,8 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) * expansion area */ vma_iter_init(&vmi, mm, oldbrk); - next = vma_find(&vmi, newbrk + PAGE_SIZE + stack_guard_gap); - if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) + next = vma_find(&vmi, newbrk + mm_page_size(mm) + stack_guard_gap); + if (next && newbrk + mm_page_size(mm) > vm_start_gap(next)) goto out; brkvma = vma_prev_limit(&vmi, mm->start_brk); @@ -218,12 +218,12 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) * If a hint addr is less than mmap_min_addr change hint to be as * low as possible but still greater than mmap_min_addr */ -static inline unsigned long round_hint_to_min(unsigned long hint) +static inline unsigned long round_hint_to_min(struct mm_struct *mm, unsigned long hint) { - hint &= PAGE_MASK; + hint &= mm_page_mask(mm); if (((void *)hint != NULL) && (hint < mmap_min_addr)) - return PAGE_ALIGN(mmap_min_addr); + return mm_page_align(mm, mmap_min_addr); return hint; } @@ -235,11 +235,11 @@ bool mlock_future_ok(const struct mm_struct *mm, bool is_vma_locked, if (!is_vma_locked || capable(CAP_IPC_LOCK)) return true; - locked_pages = bytes >> PAGE_SHIFT; + locked_pages = bytes >> MM_ACCOUNT_SHIFT; locked_pages += mm->locked_vm; limit_pages = rlimit(RLIMIT_MEMLOCK); - limit_pages >>= PAGE_SHIFT; + limit_pages >>= MM_ACCOUNT_SHIFT; return locked_pages <= limit_pages; } @@ -264,24 +264,25 @@ static inline u64 file_mmap_size_max(struct file *file, struct inode *inode) } static inline bool file_mmap_ok(struct file *file, struct inode *inode, - unsigned long pgoff, unsigned long len) + struct vm_page_offset pos, unsigned long len) { u64 maxsize = file_mmap_size_max(file, inode); if (maxsize && len > maxsize) return false; maxsize -= len; - if (pgoff > maxsize >> PAGE_SHIFT) + if (pos.index > maxsize >> PAGE_SHIFT || + (pos.index == maxsize >> PAGE_SHIFT && pos.offset > (maxsize & ~PAGE_MASK))) return false; return true; } /** - * do_mmap() - Perform a userland memory mapping into the current process + * do_mmap_internal() - Perform a userland memory mapping into the current process * address space of length @len with protection bits @prot, mmap flags @flags * (from which VMA flags will be inferred), and any additional VMA flags to * apply @vm_flags. If this is a file-backed mapping then the file is specified - * in @file and page offset into the file via @pgoff. + * in @file and its byte position via @pos. * * This function does not perform security checks on the file and assumes, if * @uf is non-NULL, the caller has provided a list head to track unmap events @@ -321,7 +322,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * @flags: Flags specifying how the mapping should be performed, see mmap (2) * for details. * @vm_flags: VMA flags which should be set by default, or 0 otherwise. - * @pgoff: Page offset into the @file if file-backed, should be 0 otherwise. + * @pos: Native cache index and byte offset; ignored for anonymous mappings. * @populate: A pointer to a value which will be set to 0 if no population of * the range is required, or the number of bytes to populate if it is. Must be * non-NULL. See mmap (2) for details as to under what circumstances population @@ -333,20 +334,22 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * Returns: Either an error, or the address at which the requested mapping has * been performed. */ -unsigned long do_mmap(struct file *file, unsigned long addr, +static unsigned long do_mmap_internal(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, vm_flags_t vm_flags, - unsigned long pgoff, unsigned long *populate, + struct vm_page_offset pos, unsigned long *populate, struct list_head *uf) { struct mm_struct *mm = current->mm; + unsigned long pgoff = pos.index; int pkey = 0; *populate = 0; mmap_assert_write_locked(mm); - if (!len) + if (!len || pos.offset >= PAGE_SIZE || + !IS_ALIGNED(pos.offset, mm_page_size(mm))) return -EINVAL; /* @@ -364,15 +367,15 @@ unsigned long do_mmap(struct file *file, unsigned long addr, flags |= MAP_FIXED; if (!(flags & MAP_FIXED)) - addr = round_hint_to_min(addr); + addr = round_hint_to_min(mm, addr); /* Careful about overflows.. */ - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); if (!len) return -ENOMEM; /* offset overflow? */ - if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) + if (vm_page_offset_add(pos, len).index < pgoff) return -EOVERFLOW; /* Too many mappings? */ @@ -426,7 +429,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long flags_mask; int err; - if (!file_mmap_ok(file, inode, pgoff, len)) + if (!file_mmap_ok(file, inode, pos, len)) return -EOVERFLOW; flags_mask = LEGACY_MAP_MASK; @@ -500,6 +503,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * Ignore pgoff. */ pgoff = 0; + pos.offset = 0; vm_flags |= VM_SHARED | VM_MAYSHARE; break; case MAP_DROPPABLE: @@ -537,6 +541,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * Set pgoff according to addr for anon_vma. */ pgoff = addr >> PAGE_SHIFT; + pos.offset = addr & ~PAGE_MASK; break; default: return -EINVAL; @@ -557,7 +562,11 @@ unsigned long do_mmap(struct file *file, unsigned long addr, vm_flags |= VM_NORESERVE; } - addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); + addr = mmap_region(file, addr, len, vm_flags, + (struct vm_page_offset) { + .index = pgoff, + .offset = pos.offset, + }, uf); if (!IS_ERR_VALUE(addr) && ((vm_flags & VM_LOCKED) || (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) @@ -565,9 +574,28 @@ unsigned long do_mmap(struct file *file, unsigned long addr, return addr; } -unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, +unsigned long do_mmap(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, + struct list_head *uf) +{ + return do_mmap_internal(file, addr, len, prot, flags, vm_flags, + (struct vm_page_offset) { pgoff, 0 }, populate, uf); +} + +#ifdef CONFIG_MM_SUBPAGE +unsigned long do_mmap_offset(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, struct vm_page_offset pos, unsigned long *populate, + struct list_head *uf) +{ + return do_mmap_internal(file, addr, len, prot, flags, vm_flags, pos, populate, uf); +} +#endif + +static unsigned long ksys_mmap_offset(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, - unsigned long fd, unsigned long pgoff) + unsigned long fd, unsigned long pgoff, unsigned int suboffset) { struct file *file = NULL; unsigned long retval; @@ -603,13 +631,34 @@ unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, return PTR_ERR(file); } - retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); + if (suboffset) + retval = vm_mmap(file, addr, len, prot, flags, + (pgoff << PAGE_SHIFT) | suboffset); + else + retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); out_fput: if (file) fput(file); return retval; } +unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long pgoff) +{ + return ksys_mmap_offset(addr, len, prot, flags, fd, pgoff, 0); +} + +#ifdef CONFIG_MM_SUBPAGE +unsigned long ksys_mmap_user(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long offset) +{ + if (!IS_ALIGNED(offset, mm_page_size(current->mm))) + return -EINVAL; + return ksys_mmap_offset(addr, len, prot, flags, fd, + offset >> PAGE_SHIFT, offset & ~PAGE_MASK); +} +#endif + SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, pgoff) @@ -703,7 +752,7 @@ generic_get_unmapped_area(struct file *filp, unsigned long addr, return addr; if (addr) { - addr = PAGE_ALIGN(addr); + addr = mm_page_align(mm, addr); vma = find_vma_prev(mm, addr, &prev); if (mmap_end - len >= addr && addr >= mmap_min_addr && (!vma || addr + len <= vm_start_gap(vma)) && @@ -754,7 +803,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, /* requesting a specific address */ if (addr) { - addr = PAGE_ALIGN(addr); + addr = mm_page_align(mm, addr); vma = find_vma_prev(mm, addr, &prev); if (mmap_end - len >= addr && addr >= mmap_min_addr && (!vma || addr + len <= vm_start_gap(vma)) && @@ -764,7 +813,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.length = len; - info.low_limit = PAGE_SIZE; + info.low_limit = mm_page_size(mm); info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); info.start_gap = stack_guard_placement(vm_flags); if (filp && is_file_hugepages(filp)) @@ -777,7 +826,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, * can happen with large stack limits and large mmap() * allocations. */ - if (offset_in_page(addr)) { + if (addr & ~mm_page_mask(mm)) { VM_BUG_ON(addr != -ENOMEM); info.flags = 0; info.low_limit = TASK_UNMAPPED_BASE; @@ -857,7 +906,7 @@ __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, if (addr > TASK_SIZE - len) return -ENOMEM; - if (offset_in_page(addr)) + if (addr & ~mm_page_mask(current->mm)) return -EINVAL; error = security_mmap_addr(addr); @@ -962,7 +1011,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon { struct vm_area_struct *vma, *prev; - addr &= PAGE_MASK; + addr &= mm_page_mask(mm); vma = find_vma_prev(mm, addr, &prev); if (vma && (vma->vm_start <= addr)) return vma; @@ -985,7 +1034,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon struct vm_area_struct *vma; unsigned long start; - addr &= PAGE_MASK; + addr &= mm_page_mask(mm); vma = find_vma(mm, addr); if (!vma) return NULL; @@ -1214,7 +1263,7 @@ int vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec) LIST_HEAD(uf); VMA_ITERATOR(vmi, mm, addr); - len = PAGE_ALIGN(request); + len = mm_page_align(mm, request); if (len < request) return -ENOMEM; if (!len) @@ -1257,7 +1306,7 @@ unsigned long tear_down_vmas(struct mm_struct *mm, struct vma_iterator *vmi, vma_iter_set(vmi, vma->vm_end); do { if (vma->vm_flags & VM_ACCOUNT) - nr_accounted += vma_pages(vma); + nr_accounted += vma_user_pages(vma); vma_mark_detached(vma); remove_vma(vma); count++; @@ -1322,8 +1371,8 @@ void exit_mmap(struct mm_struct *mm) destroy: __mt_destroy(&mm->mm_mt); trace_exit_mmap(mm); + mm_unacct_memory(mm, nr_accounted); mmap_write_unlock(mm); - vm_unacct_memory(nr_accounted); } /* @@ -1333,19 +1382,19 @@ void exit_mmap(struct mm_struct *mm) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages) { - if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT) + if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> mm_page_shift(mm)) return false; if (is_data_mapping_vma_flags(vma_flags) && - mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) { + mm->data_vm + npages > rlimit(RLIMIT_DATA) >> mm_page_shift(mm)) { /* Workaround for Valgrind */ if (rlimit(RLIMIT_DATA) == 0 && - mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT) + mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> mm_page_shift(mm)) return true; pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n", current->comm, current->pid, - (mm->data_vm + npages) << PAGE_SHIFT, + (mm->data_vm + npages) << mm_page_shift(mm), rlimit(RLIMIT_DATA), ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data"); @@ -1474,7 +1523,7 @@ static struct vm_area_struct *__install_special_mapping( if (ret) goto out; - vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); + vm_stat_account(mm, vma->vm_flags, len >> mm_page_shift(mm)); perf_event_mmap(vma); @@ -1771,14 +1820,14 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) if (retval) goto loop_out; - vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt)); + vm_stat_account(mm, mpnt->vm_flags, -vma_user_pages(mpnt)); continue; } charge = 0; if (mpnt->vm_flags & VM_ACCOUNT) { - unsigned long len = vma_pages(mpnt); + unsigned long len = vma_user_pages(mpnt); - if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ + if (mm_account_memory(mm, oldmm, len)) /* sic */ goto fail_nomem; charge = len; } @@ -1890,7 +1939,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) flush_cache_mm(mm); unmap_region(&unmap); charge = tear_down_vmas(mm, &vmi, tmp, end); - vm_unacct_memory(charge); + mm_unacct_memory(mm, charge); } __mt_destroy(&mm->mm_mt); /* @@ -1916,6 +1965,6 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) vm_area_free(tmp); fail_nomem: retval = -ENOMEM; - vm_unacct_memory(charge); + mm_unacct_memory(mm, charge); goto loop_out; } diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 3985d856de7f9..494b7a9f2eb6a 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -12,6 +12,9 @@ #include #include #include +#include + +#include "user-subpage-internal.h" #include @@ -100,6 +103,49 @@ void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma) */ #define MAX_NR_FOLIOS_PER_FREE 512 +#ifdef CONFIG_MM_SUBPAGE +/* Private to gather batches: never pass this token to encoded_page_ptr(). */ +#define TLB_SUBPAGE_TAG BIT(2) + +static void tlb_free_pages_and_subpages(struct encoded_page **pages, unsigned int nr) +{ + unsigned int start = 0, i = 0; + + while (i < nr) { + unsigned long token = (unsigned long)pages[i]; + + if (token & TLB_SUBPAGE_TAG) { + if (i != start) + free_pages_and_swap_cache(pages + start, i - start); + mm_subpage_put((struct mm_subpage *)(token & ~TLB_SUBPAGE_TAG)); + start = ++i; + } else { + /* The count payload is not a pointer and may have bit 2 set. */ + i += encoded_page_flags(pages[i]) & ENCODED_PAGE_BIT_NR_PAGES_NEXT ? 2 : 1; + } + } + if (i != start) + free_pages_and_swap_cache(pages + start, i - start); +} + +bool __tlb_remove_subpage(struct mmu_gather *tlb, struct mm_subpage *subpage) +{ + struct mmu_gather_batch *batch = tlb->active; + + BUILD_BUG_ON(__alignof__(struct page) < 8); + BUILD_BUG_ON(__alignof__(struct mm_subpage) < 8); + BUILD_BUG_ON(TLB_SUBPAGE_TAG & ENCODED_PAGE_BITS); + VM_BUG_ON(!tlb->end); + VM_BUG_ON(mm_page_shift(tlb->mm) != mm_subpage_shift(subpage)); + batch->encoded_pages[batch->nr++] = + (struct encoded_page *)((unsigned long)subpage | TLB_SUBPAGE_TAG); + /* Keep room for a following native page/count pair. */ + return batch->nr >= batch->max - 1 && !tlb_next_batch(tlb); +} +#else +#define tlb_free_pages_and_subpages free_pages_and_swap_cache +#endif + static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) { struct encoded_page **pages = batch->encoded_pages; @@ -135,7 +181,7 @@ static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) } } - free_pages_and_swap_cache(pages, nr); + tlb_free_pages_and_subpages(pages, nr); pages += nr; batch->nr -= nr; @@ -174,7 +220,15 @@ static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb, #ifdef CONFIG_MMU_GATHER_PAGE_SIZE VM_WARN_ON(tlb->page_size != page_size); - VM_WARN_ON_ONCE(nr_pages != 1 && page_size != PAGE_SIZE); + if (page_size > PAGE_SIZE && page_size == mm_page_size(tlb->mm)) { + /* A coarse base PTE owns one reference per covered native page. */ + unsigned long span = mm_pte_native_pages(tlb->mm); + + VM_WARN_ON_ONCE(!nr_pages || nr_pages % span || + !IS_ALIGNED(page_to_pfn(page), span)); + } else { + VM_WARN_ON_ONCE(nr_pages != 1 && page_size != PAGE_SIZE); + } VM_WARN_ON_ONCE(page_folio(page) != page_folio(page + nr_pages - 1)); #endif @@ -208,7 +262,7 @@ bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page, unsigned int nr_pages, bool delay_rmap) { return __tlb_remove_folio_pages_size(tlb, page, nr_pages, delay_rmap, - PAGE_SIZE); + mm_page_size(tlb->mm)); } bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size) diff --git a/mm/mprotect.c b/mm/mprotect.c index 9cbf932b028cf..8202f2ebfbf6d 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -58,6 +59,20 @@ static bool maybe_change_pte_writable(struct vm_area_struct *vma, pte_t pte) return true; } +#ifdef CONFIG_MM_SUBPAGE +static bool subpage_pte_exclusive(struct vm_area_struct *vma, pte_t pte) +{ + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + bool exclusive; + + if (!slot) + return false; + exclusive = mm_subpage_is_exclusive(slot); + mm_subpage_put(slot); + return exclusive; +} +#endif + static bool can_change_private_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte) { @@ -73,6 +88,10 @@ static bool can_change_private_pte_writable(struct vm_area_struct *vma, * any additional checks while holding the PT lock. */ page = vm_normal_page(vma, addr, pte); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return page && PageAnon(page) && subpage_pte_exclusive(vma, pte); +#endif return page && PageAnon(page) && PageAnonExclusive(page); } @@ -103,13 +122,19 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, return can_change_shared_pte_writable(vma, pte); } -static int mprotect_folio_pte_batch(struct folio *folio, pte_t *ptep, + +static int mprotect_folio_pte_batch(struct vm_area_struct *vma, struct folio *folio, pte_t *ptep, pte_t pte, int max_nr_ptes, fpb_t flags) { /* No underlying folio, so cannot batch */ if (!folio) return 1; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch(vma, folio, ptep, pte, max_nr_ptes, flags); +#endif + if (!folio_test_large(folio)) return 1; @@ -125,17 +150,17 @@ static __always_inline void prot_commit_flush_ptes(struct vm_area_struct *vma, * Advance the position in the batch by idx; note that if idx > 0, * then the nr_ptes passed here is <= batch size - idx. */ - addr += idx * PAGE_SIZE; + addr += idx * mm_page_size(vma->vm_mm); ptep += idx; - oldpte = pte_advance_pfn(oldpte, idx); - ptent = pte_advance_pfn(ptent, idx); + oldpte = pte_advance_pfn_mm(vma->vm_mm, oldpte, idx); + ptent = pte_advance_pfn_mm(vma->vm_mm, ptent, idx); if (set_write) ptent = pte_mkwrite(ptent, vma); modify_prot_commit_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes); if (pte_needs_flush(oldpte, ptent)) - tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE); + tlb_flush_pte_range(tlb, addr, nr_ptes * mm_page_size(vma->vm_mm)); } /* @@ -177,6 +202,25 @@ static __always_inline void commit_anon_folio_batch(struct vm_area_struct *vma, int sub_batch_idx = 0; int len; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + while (nr_ptes) { + expected_anon_exclusive = subpage_pte_exclusive(vma, + pte_advance_pfn_mm(vma->vm_mm, oldpte, sub_batch_idx)); + for (len = 1; len < nr_ptes; len++) + if (subpage_pte_exclusive(vma, + pte_advance_pfn_mm(vma->vm_mm, oldpte, sub_batch_idx + len)) != + expected_anon_exclusive) + break; + prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, len, + sub_batch_idx, expected_anon_exclusive, tlb); + sub_batch_idx += len; + nr_ptes -= len; + } + return; + } +#endif + while (nr_ptes) { expected_anon_exclusive = PageAnonExclusive(first_page + sub_batch_idx); len = page_anon_exclusive_sub_batch(sub_batch_idx, nr_ptes, @@ -265,6 +309,11 @@ static long change_softleaf_pte(struct vm_area_struct *vma, newpte = oldpte; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + newpte = pte_swp_set_subpage_offset(newpte, pte_swp_subpage_offset(oldpte)); +#endif + if (uffd_wp) newpte = pte_swp_mkuffd_wp(newpte); else if (uffd_wp_resolve) @@ -328,7 +377,7 @@ static long change_pte_range(struct mmu_gather *tlb, bool uffd_wp = cp_flags & MM_CP_UFFD_WP; int nr_ptes; - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, mm_page_size(vma->vm_mm)); pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); if (!pte) return -EAGAIN; @@ -343,7 +392,7 @@ static long change_pte_range(struct mmu_gather *tlb, oldpte = ptep_get(pte); if (pte_present(oldpte)) { const fpb_t flags = FPB_RESPECT_SOFT_DIRTY | FPB_RESPECT_WRITE; - int max_nr_ptes = (end - addr) >> PAGE_SHIFT; + int max_nr_ptes = (end - addr) >> mm_page_shift(vma->vm_mm); struct folio *folio = NULL; struct page *page; @@ -364,12 +413,12 @@ static long change_pte_range(struct mmu_gather *tlb, is_private_single_threaded)) { /* determine batch to skip */ - nr_ptes = mprotect_folio_pte_batch(folio, + nr_ptes = mprotect_folio_pte_batch(vma, folio, pte, oldpte, max_nr_ptes, /* flags = */ 0); continue; } - nr_ptes = mprotect_folio_pte_batch(folio, pte, oldpte, max_nr_ptes, flags); + nr_ptes = mprotect_folio_pte_batch(vma, folio, pte, oldpte, max_nr_ptes, flags); /* * Optimize for the small-folio common case by @@ -408,7 +457,7 @@ static long change_pte_range(struct mmu_gather *tlb, } else { pages += change_softleaf_pte(vma, addr, pte, oldpte, cp_flags); } - } while (pte += nr_ptes, addr += nr_ptes * PAGE_SIZE, addr != end); + } while (pte += nr_ptes, addr += nr_ptes * mm_page_size(vma->vm_mm), addr != end); lazy_mmu_mode_disable(); pte_unmap_unlock(pte - 1, ptl); @@ -487,12 +536,12 @@ static inline long change_pmd_range(struct mmu_gather *tlb, long pages = 0; unsigned long nr_huge_updates = 0; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(vma->vm_mm, pud, addr); do { long ret; pmd_t _pmd; again: - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(vma->vm_mm, addr, end); ret = change_pmd_prepare(vma, pmd, cp_flags); if (ret) { @@ -559,10 +608,10 @@ static inline long change_pud_range(struct mmu_gather *tlb, range.start = 0; - pudp = pud_offset(p4d, addr); + pudp = pud_offset_mm(vma->vm_mm, p4d, addr); do { again: - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, pudp, pmd, addr, cp_flags); if (ret) { pages = ret; @@ -615,13 +664,13 @@ static inline long change_p4d_range(struct mmu_gather *tlb, unsigned long next; long pages = 0, ret; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(vma->vm_mm, pgd, addr); do { - next = p4d_addr_end(addr, end); + next = p4d_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, p4d, pud, addr, cp_flags); if (ret) return ret; - if (p4d_none_or_clear_bad(p4d)) + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4d)) continue; pages += change_pud_range(tlb, vma, p4d, addr, next, newprot, cp_flags); @@ -643,7 +692,7 @@ static long change_protection_range(struct mmu_gather *tlb, pgd = pgd_offset(mm, addr); tlb_start_vma(tlb, vma); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, pgd, p4d, addr, cp_flags); if (ret) { pages = ret; @@ -729,7 +778,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, struct mm_struct *mm = vma->vm_mm; const vma_flags_t old_vma_flags = READ_ONCE(vma->flags); vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags); - long nrpages = (end - start) >> PAGE_SHIFT; + long nrpages = (end - start) >> mm_page_shift(mm); unsigned int mm_cp_flags = 0; unsigned long charged = 0; int error; @@ -777,7 +826,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, VMA_ACCOUNT_BIT, VMA_WRITE_BIT, VMA_HUGETLB_BIT, VMA_SHARED_BIT, VMA_NORESERVE_BIT)) { charged = nrpages; - if (security_vm_enough_memory_mm(mm, charged)) + if (mm_account_memory(mm, mm, charged)) return -ENOMEM; vma_flags_set(&new_vma_flags, VMA_ACCOUNT_BIT); } @@ -808,7 +857,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, if (vma_flags_test(&old_vma_flags, VMA_ACCOUNT_BIT) && !vma_flags_test(&new_vma_flags, VMA_ACCOUNT_BIT)) - vm_unacct_memory(nrpages); + mm_unacct_memory(mm, nrpages); /* * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major @@ -826,7 +875,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, return 0; fail: - vm_unacct_memory(charged); + mm_unacct_memory(mm, charged); return error; } @@ -851,11 +900,11 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ return -EINVAL; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(current->mm)) return -EINVAL; if (!len) return 0; - len = PAGE_ALIGN(len); + len = mm_page_align(current->mm, len); end = start + len; if (end <= start) return -ENOMEM; diff --git a/mm/mremap.c b/mm/mremap.c index e9c8b1d05832b..97d5ebf945285 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -82,11 +82,11 @@ static pud_t *get_old_pud(struct mm_struct *mm, unsigned long addr) if (pgd_none_or_clear_bad(pgd)) return NULL; - p4d = p4d_offset(pgd, addr); - if (p4d_none_or_clear_bad(p4d)) + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_or_clear_bad_mm(mm, p4d)) return NULL; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(mm, p4d, addr); if (pud_none_or_clear_bad(pud)) return NULL; @@ -102,7 +102,7 @@ static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr) if (!pud) return NULL; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(mm, pud, addr); if (pmd_none(*pmd)) return NULL; @@ -183,6 +183,14 @@ static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr if (max_nr == 1) return 1; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + folio = vm_normal_folio(vma, addr, pte); + return folio ? folio_subpage_pte_batch(vma, folio, ptep, pte, + max_nr, FPB_RESPECT_WRITE) : 1; + } +#endif + /* Avoid expensive folio lookup if we stand no chance of benefit. */ if (pte_batch_hint(ptep, pte) == 1) return 1; @@ -262,12 +270,12 @@ static int move_ptes(struct pagetable_move_control *pmc, flush_tlb_batched_pending(vma->vm_mm); lazy_mmu_mode_enable(); - for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE, - new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) { + for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * mm_page_size(mm), + new_ptep += nr_ptes, new_addr += nr_ptes * mm_page_size(mm)) { VM_WARN_ON_ONCE(!pte_none(*new_ptep)); nr_ptes = 1; - max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT; + max_nr_ptes = (old_end - old_addr) >> mm_page_shift(mm); old_pte = ptep_get(old_ptep); if (pte_none(old_pte)) continue; @@ -407,7 +415,7 @@ static bool move_normal_pmd(struct pagetable_move_control *pmc, VM_BUG_ON(!pmd_none(*new_pmd)); pmd_populate(mm, new_pmd, pmd_pgtable(pmd)); - flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PMD_SIZE); + flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + pmd_size_mm(mm)); out_unlock: if (new_ptl != old_ptl) spin_unlock(new_ptl); @@ -458,8 +466,8 @@ static bool move_normal_pud(struct pagetable_move_control *pmc, VM_BUG_ON(!pud_none(*new_pud)); - pud_populate(mm, new_pud, pud_pgtable(pud)); - flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PUD_SIZE); + pud_populate(mm, new_pud, pud_pgtable_mm(mm, pud)); + flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + pud_size_mm(mm)); if (new_ptl != old_ptl) spin_unlock(new_ptl); spin_unlock(old_ptl); @@ -549,13 +557,13 @@ static __always_inline unsigned long get_extent(enum pgt_entry entry, switch (entry) { case HPAGE_PMD: case NORMAL_PMD: - mask = PMD_MASK; - size = PMD_SIZE; + mask = pmd_mask_mm(pmc->old->vm_mm); + size = pmd_size_mm(pmc->old->vm_mm); break; case HPAGE_PUD: case NORMAL_PUD: - mask = PUD_MASK; - size = PUD_SIZE; + mask = pud_mask_mm(pmc->old->vm_mm); + size = pud_size_mm(pmc->old->vm_mm); break; default: BUILD_BUG(); @@ -811,7 +819,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) * If possible, realign addresses to PMD boundary for faster copy. * Only realign if the mremap copying hits a PMD boundary. */ - try_realign_addr(pmc, PMD_MASK); + try_realign_addr(pmc, pmd_mask_mm(mm)); flush_cache_range(pmc->old, pmc->old_addr, pmc->old_end); mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, mm, @@ -838,7 +846,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) /* We ignore and continue on error? */ continue; } - } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == PUD_SIZE) { + } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == pud_size_mm(mm)) { if (move_pgt_entry(pmc, NORMAL_PUD, old_pud, new_pud)) continue; } @@ -857,7 +865,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) continue; split_huge_pmd(pmc->old, old_pmd, pmc->old_addr); } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PMD) && - extent == PMD_SIZE) { + extent == pmd_size_mm(mm)) { /* * If the extent is PMD-sized, try to speed the move by * moving at the PMD level if possible. @@ -947,9 +955,7 @@ static unsigned long vrm_set_new_addr(struct vma_remap_struct *vrm) { struct vm_area_struct *vma = vrm->vma; unsigned long map_flags = 0; - /* Page Offset _into_ the VMA. */ - pgoff_t internal_pgoff = (vrm->addr - vma->vm_start) >> PAGE_SHIFT; - pgoff_t pgoff = vma->vm_pgoff + internal_pgoff; + pgoff_t pgoff = vma_page_offset_at(vma, vrm->addr).index; unsigned long new_addr = vrm_implies_new_addr(vrm) ? vrm->new_addr : 0; unsigned long res; @@ -985,13 +991,13 @@ static bool vrm_calc_charge(struct vma_remap_struct *vrm) * the length of the new one. Otherwise it's just the delta in size. */ if (vrm->flags & MREMAP_DONTUNMAP) - charged = vrm->new_len >> PAGE_SHIFT; + charged = vrm->new_len >> mm_page_shift(current->mm); else - charged = vrm->delta >> PAGE_SHIFT; + charged = vrm->delta >> mm_page_shift(current->mm); /* This accounts 'charged' pages of memory. */ - if (security_vm_enough_memory_mm(current->mm, charged)) + if (mm_account_memory(current->mm, current->mm, charged)) return false; vrm->charged = charged; @@ -1007,7 +1013,7 @@ static void vrm_uncharge(struct vma_remap_struct *vrm) if (!(vrm->vma->vm_flags & VM_ACCOUNT)) return; - vm_unacct_memory(vrm->charged); + mm_unacct_memory(current->mm, vrm->charged); vrm->charged = 0; } @@ -1019,13 +1025,13 @@ static void vrm_uncharge(struct vma_remap_struct *vrm) static void vrm_stat_account(struct vma_remap_struct *vrm, unsigned long bytes) { - unsigned long pages = bytes >> PAGE_SHIFT; + unsigned long pages = bytes >> mm_page_shift(current->mm); struct mm_struct *mm = current->mm; struct vm_area_struct *vma = vrm->vma; vm_stat_account(mm, vma->vm_flags, pages); if (vma->vm_flags & VM_LOCKED) - mm->locked_vm += pages; + mm->locked_vm += bytes >> MM_ACCOUNT_SHIFT; } static bool __check_map_count_against_split(struct mm_struct *mm, @@ -1198,7 +1204,7 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) vrm->vmi_needs_invalidate = true; if (err) { /* OOM: unable to split vma, just get accounts right */ - vm_acct_memory(len >> PAGE_SHIFT); + mm_acct_memory(current->mm, len >> mm_page_shift(current->mm)); return; } @@ -1255,16 +1261,14 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) static int copy_vma_and_data(struct vma_remap_struct *vrm, struct vm_area_struct **new_vma_ptr) { - unsigned long internal_offset = vrm->addr - vrm->vma->vm_start; - unsigned long internal_pgoff = internal_offset >> PAGE_SHIFT; - unsigned long new_pgoff = vrm->vma->vm_pgoff + internal_pgoff; + struct vm_page_offset pos = vma_page_offset_at(vrm->vma, vrm->addr); unsigned long moved_len; struct vm_area_struct *vma = vrm->vma; struct vm_area_struct *new_vma; int err = 0; PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len); - new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, new_pgoff, + new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, pos, &pmc.need_rmap_locks); if (!new_vma) { vrm_uncharge(vrm); @@ -1473,7 +1477,7 @@ static unsigned long mremap_to(struct vma_remap_struct *vrm) /* MREMAP_DONTUNMAP expands by old_len since old_len == new_len */ if (vrm->flags & MREMAP_DONTUNMAP) { vma_flags_t vma_flags = vrm->vma->flags; - unsigned long pages = vrm->old_len >> PAGE_SHIFT; + unsigned long pages = vrm->old_len >> mm_page_shift(current->mm); if (!may_expand_vm(mm, &vma_flags, pages)) return -ENOMEM; @@ -1495,7 +1499,7 @@ static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) if (find_vma_intersection(vma->vm_mm, vma->vm_end, end)) return 0; if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, - 0, MAP_FIXED) & ~PAGE_MASK) + 0, MAP_FIXED) & ~mm_page_mask(vma->vm_mm)) return 0; return 1; } @@ -1727,7 +1731,8 @@ static int check_prep_vma(struct vma_remap_struct *vrm) struct vm_area_struct *vma = vrm->vma; struct mm_struct *mm = current->mm; unsigned long addr = vrm->addr; - unsigned long old_len, new_len, pgoff; + unsigned long old_len, new_len; + struct vm_page_offset pos, end; if (!vma) return -EFAULT; @@ -1802,9 +1807,9 @@ static int check_prep_vma(struct vma_remap_struct *vrm) vrm->populate_expand = true; /* Need to be careful about a growing mapping */ - pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; - if (pgoff + (new_len >> PAGE_SHIFT) < pgoff) + pos = vma_page_offset_at(vma, addr); + end = vm_page_offset_add(pos, new_len); + if (end.index < pos.index) return -EINVAL; if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) @@ -1813,7 +1818,7 @@ static int check_prep_vma(struct vma_remap_struct *vrm) if (!mlock_future_ok(mm, vma->vm_flags & VM_LOCKED, vrm->delta)) return -EAGAIN; - if (!may_expand_vm(mm, &vma->flags, vrm->delta >> PAGE_SHIFT)) + if (!may_expand_vm(mm, &vma->flags, vrm->delta >> mm_page_shift(current->mm))) return -ENOMEM; return 0; @@ -1834,7 +1839,7 @@ static unsigned long check_mremap_params(struct vma_remap_struct *vrm) return -EINVAL; /* Start address must be page-aligned. */ - if (offset_in_page(addr)) + if (addr & ~mm_page_mask(current->mm)) return -EINVAL; /* @@ -1858,7 +1863,7 @@ static unsigned long check_mremap_params(struct vma_remap_struct *vrm) return -EINVAL; /* The new address must be page-aligned. */ - if (offset_in_page(vrm->new_addr)) + if (vrm->new_addr & ~mm_page_mask(current->mm)) return -EINVAL; /* A fixed address implies a move. */ @@ -1971,8 +1976,8 @@ static unsigned long do_mremap(struct vma_remap_struct *vrm) unsigned long res; bool failed; - vrm->old_len = PAGE_ALIGN(vrm->old_len); - vrm->new_len = PAGE_ALIGN(vrm->new_len); + vrm->old_len = mm_page_align(mm, vrm->old_len); + vrm->new_len = mm_page_align(mm, vrm->new_len); res = check_mremap_params(vrm); if (res) diff --git a/mm/mseal.c b/mm/mseal.c index e2093ae3d25cd..62df360b43545 100644 --- a/mm/mseal.c +++ b/mm/mseal.c @@ -151,10 +151,10 @@ int do_mseal(unsigned long start, size_t len_in, unsigned long flags) return -EINVAL; start = untagged_addr(start); - if (!PAGE_ALIGNED(start)) + if (start & ~mm_page_mask(mm)) return -EINVAL; - len = PAGE_ALIGN(len_in); + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero. */ if (len_in && !len) return -EINVAL; diff --git a/mm/msync.c b/mm/msync.c index ac4c9bfea2e7f..6d9027ee47246 100644 --- a/mm/msync.c +++ b/mm/msync.c @@ -41,12 +41,12 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) if (flags & ~(MS_ASYNC | MS_INVALIDATE | MS_SYNC)) goto out; - if (offset_in_page(start)) + if (start & ~mm_page_mask(mm)) goto out; if ((flags & MS_ASYNC) && (flags & MS_SYNC)) goto out; error = -ENOMEM; - len = (len + ~PAGE_MASK) & PAGE_MASK; + len = mm_page_align(mm, len); end = start + len; if (end < start) goto out; @@ -86,7 +86,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) } file = vma->vm_file; fstart = (start - vma->vm_start) + - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); fend = fstart + (min(end, vma->vm_end) - start) - 1; start = vma->vm_end; if ((flags & MS_SYNC) && file && diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 5f372f6e26fa3..5345f43c10b6f 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -225,7 +225,8 @@ long oom_badness(struct task_struct *p, unsigned long totalpages) * The baseline for the badness score is the proportion of RAM that each * task's rss, pagetable and swap space use. */ - points = get_mm_rss_sum(p->mm) + get_mm_counter_sum(p->mm, MM_SWAPENTS) + + points = mm_pages_to_native(p->mm, get_mm_rss_sum(p->mm) + + get_mm_counter_sum(p->mm, MM_SWAPENTS)) + mm_pgtables_bytes(p->mm) / PAGE_SIZE; task_unlock(p); @@ -399,10 +400,10 @@ static int dump_task(struct task_struct *p, void *arg) pr_info("[%7d] %5d %5d %8lu %8lu %8lu %8lu %9lu %8ld %8lu %5hd %s\n", task->pid, from_kuid(&init_user_ns, task_uid(task)), - task->tgid, task->mm->total_vm, get_mm_rss_sum(task->mm), - get_mm_counter_sum(task->mm, MM_ANONPAGES), get_mm_counter_sum(task->mm, MM_FILEPAGES), - get_mm_counter_sum(task->mm, MM_SHMEMPAGES), mm_pgtables_bytes(task->mm), - get_mm_counter_sum(task->mm, MM_SWAPENTS), + task->tgid, mm_pages_to_native(task->mm, task->mm->total_vm), mm_pages_to_native(task->mm, get_mm_rss_sum(task->mm)), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_ANONPAGES)), mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_FILEPAGES)), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_SHMEMPAGES)), mm_pgtables_bytes(task->mm), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_SWAPENTS)), task->signal->oom_score_adj, task->comm); task_unlock(task); @@ -588,9 +589,9 @@ static bool oom_reap_task_mm(struct task_struct *tsk, struct mm_struct *mm) pr_info("oom_reaper: reaped process %d (%s), now anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n", task_pid_nr(tsk), tsk->comm, - K(get_mm_counter_sum(mm, MM_ANONPAGES)), - K(get_mm_counter_sum(mm, MM_FILEPAGES)), - K(get_mm_counter_sum(mm, MM_SHMEMPAGES))); + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_ANONPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_FILEPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_SHMEMPAGES))); out_finish: trace_finish_task_reaping(tsk->pid); out_unlock: @@ -943,10 +944,10 @@ static void __oom_kill_process(struct task_struct *victim, const char *message) do_send_sig_info(SIGKILL, SEND_SIG_PRIV, victim, PIDTYPE_TGID); mark_oom_victim(victim); pr_err("%s: Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB, UID:%u pgtables:%lukB oom_score_adj:%d\n", - message, task_pid_nr(victim), victim->comm, K(mm->total_vm), - K(get_mm_counter_sum(mm, MM_ANONPAGES)), - K(get_mm_counter_sum(mm, MM_FILEPAGES)), - K(get_mm_counter_sum(mm, MM_SHMEMPAGES)), + message, task_pid_nr(victim), victim->comm, mm_pages_to_kb(mm, mm->total_vm), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_ANONPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_FILEPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_SHMEMPAGES)), from_kuid(&init_user_ns, task_uid(victim)), mm_pgtables_bytes(mm) >> 10, victim->signal->oom_score_adj); task_unlock(victim); diff --git a/mm/page_ext.c b/mm/page_ext.c index e2e92bd27ebdd..a73aab719c040 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include @@ -78,6 +79,9 @@ static struct page_ext_operations page_idle_ops __initdata = { #endif static struct page_ext_operations *page_ext_ops[] __initdata = { +#ifdef CONFIG_MM_SUBPAGE + &mm_subpage_ext_ops, +#endif #ifdef CONFIG_PAGE_OWNER &page_owner_ops, #endif diff --git a/mm/page_idle.c b/mm/page_idle.c index 9c67cbac29659..8af46b0d8f746 100644 --- a/mm/page_idle.c +++ b/mm/page_idle.c @@ -51,9 +51,10 @@ static struct folio *page_idle_get_folio(unsigned long pfn) static bool page_idle_clear_pte_refs_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long addr, void *arg) + struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + unsigned long addr = rmap.address; + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool referenced = false; while (page_vma_mapped_walk(&pvmw)) { @@ -69,7 +70,7 @@ static bool page_idle_clear_pte_refs_one(struct folio *folio, */ if (likely(pte_present(ptep_get(pvmw.pte)))) referenced |= ptep_test_and_clear_young(vma, addr, pvmw.pte); - referenced |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE); + referenced |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + mm_page_size(vma->vm_mm)); } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { pmd_t pmdval = pmdp_get(pvmw.pmd); diff --git a/mm/page_io.c b/mm/page_io.c index 70cea9e24d2fd..290706f931bdc 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "swap.h" static void __end_swap_bio_write(struct bio *bio) @@ -411,6 +412,58 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug) *swap_plug = sio; } +static DEFINE_MUTEX(swap_extent_io_mutex); +static struct bio_set swap_extent_bioset; + +int swap_extent_io_init(void) +{ + int ret = 0; + + /* Swapon prepares the reserve before any reclaim can use this file. */ + mutex_lock(&swap_extent_io_mutex); + if (!bioset_initialized(&swap_extent_bioset)) + ret = bioset_init(&swap_extent_bioset, SWAP_CLUSTER_MAX, 0, + BIOSET_NEED_RESCUER); + mutex_unlock(&swap_extent_io_mutex); + return ret; +} + +/* + * A mandatory large base page cannot split to fit a swap-file extent. Chain + * each physical run to the original bio: its folio completion must wait for + * every run, including on partial I/O or allocation failure. + */ +static void swap_submit_bio(struct bio *bio, void *private) +{ + struct folio *folio = private; + struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); + swp_entry_t entry = folio->swap; + + if (!(READ_ONCE(sis->flags) & SWP_BLKDEV) && folio_test_large(folio)) { + while (true) { + unsigned long nr = bio->bi_iter.bi_size >> PAGE_SHIFT; + unsigned int bytes; + struct bio *split; + + bio->bi_iter.bi_sector = swap_extent_sector(entry, &nr); + bytes = nr << PAGE_SHIFT; + if (bytes == bio->bi_iter.bi_size) + break; + /* The unsubmitted parent may own a bio from fs_bio_set. */ + split = bio_split(bio, bytes >> 9, GFP_NOIO, &swap_extent_bioset); + if (IS_ERR(split)) { + bio->bi_status = BLK_STS_RESOURCE; + bio_endio(bio); + return; + } + bio_chain(split, bio); + submit_bio(split); + entry.val += nr; + } + } + submit_bio(bio); +} + static void swap_writepage_bdev_sync(struct folio *folio, struct swap_info_struct *sis) { @@ -427,7 +480,7 @@ static void swap_writepage_bdev_sync(struct folio *folio, folio_start_writeback(folio); folio_unlock(folio); - submit_bio_wait(&bio); + bio_await(&bio, folio, swap_submit_bio); __end_swap_bio_write(&bio); } @@ -445,7 +498,7 @@ static void swap_writepage_bdev_async(struct folio *folio, count_swpout_vm_event(folio); folio_start_writeback(folio); folio_unlock(folio); - submit_bio(bio); + swap_submit_bio(bio, folio); } void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug) @@ -590,7 +643,7 @@ static void swap_read_folio_bdev_sync(struct folio *folio, count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio_wait(&bio); + bio_await(&bio, folio, swap_submit_bio); __end_swap_bio_read(&bio); put_task_struct(current); } @@ -607,7 +660,7 @@ static void swap_read_folio_bdev_async(struct folio *folio, count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio(bio); + swap_submit_bio(bio, folio); } void swap_read_folio(struct folio *folio, struct swap_iocb **plug) diff --git a/mm/page_table_check.c b/mm/page_table_check.c index 2403f5a114105..dc6797a41daaa 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -4,6 +4,7 @@ * Copyright (c) 2021, Google LLC. * Pasha Tatashin */ +#include #include #include #include @@ -13,11 +14,21 @@ #undef pr_fmt #define pr_fmt(fmt) "page_table_check: " fmt -struct page_table_check { +#ifdef CONFIG_MM_SUBPAGE +#define PAGE_TABLE_CHECK_SLOTS (PAGE_SIZE / SZ_4K) +#else +#define PAGE_TABLE_CHECK_SLOTS 1 +#endif + +struct page_table_check_slot { atomic_t anon_map_count; atomic_t file_map_count; }; +struct page_table_check { + struct page_table_check_slot slots[PAGE_TABLE_CHECK_SLOTS]; +}; + static bool __page_table_check_enabled __initdata = IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED); @@ -56,6 +67,74 @@ static struct page_table_check *get_page_table_check(struct page_ext *page_ext) return page_ext_data(page_ext, &page_table_check_ops); } +#if IS_ENABLED(CONFIG_KUNIT) +/* Diagnostic snapshot for private test allocations, not a synchronization API. */ +int page_table_check_get_counts(struct page *page, unsigned int offset, + int *anon, int *file) +{ + struct page_ext *ext; + struct page_table_check *ptc; + unsigned int slot = 0; + + if (static_branch_likely(&page_table_check_disabled)) + return -EOPNOTSUPP; + if (offset >= PAGE_SIZE) + return -EINVAL; +#ifdef CONFIG_MM_SUBPAGE + slot = offset / SZ_4K; +#endif + ext = page_ext_get(page); + if (!ext) + return -ENOENT; + ptc = get_page_table_check(ext); + *anon = atomic_read(&ptc->slots[slot].anon_map_count); + *file = atomic_read(&ptc->slots[slot].file_map_count); + page_ext_put(ext); + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(page_table_check_get_counts); +#endif + +static void page_table_check_update(struct page_table_check_slot *slot, + bool anon, bool clear, bool rw) +{ + if (anon) { + BUG_ON(atomic_read(&slot->file_map_count)); + if (clear) + BUG_ON(atomic_dec_return(&slot->anon_map_count) < 0); + else + BUG_ON(atomic_inc_return(&slot->anon_map_count) > 1 && rw); + } else { + BUG_ON(atomic_read(&slot->anon_map_count)); + if (clear) + BUG_ON(atomic_dec_return(&slot->file_map_count) < 0); + else + BUG_ON(atomic_inc_return(&slot->file_map_count) < 0); + } +} + +#ifdef CONFIG_MM_SUBPAGE +static void page_table_check_subpage(phys_addr_t phys, unsigned long size, + bool clear, bool rw) +{ + unsigned long pfn = PHYS_PFN(phys); + struct page_table_check *ptc; + struct page_ext *ext; + struct page *page; + unsigned int first = offset_in_page(phys) / SZ_4K, i; + + if (!pfn_valid(pfn)) + return; + page = pfn_to_page(pfn); + BUG_ON(PageSlab(page)); + ext = page_ext_get(page); + ptc = get_page_table_check(ext); + for (i = first; i < first + size / SZ_4K; i++) + page_table_check_update(&ptc->slots[i], PageAnon(page), clear, rw); + page_ext_put(ext); +} +#endif + /* * An entry is removed from the page table, decrement the counters for that page * verify that it is of correct type and counters do not become negative. @@ -78,13 +157,8 @@ static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt) for_each_page_ext(page, pgcnt, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - if (anon) { - BUG_ON(atomic_read(&ptc->file_map_count)); - BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0); - } else { - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0); - } + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) + page_table_check_update(&ptc->slots[i], anon, true, false); } rcu_read_unlock(); } @@ -113,13 +187,8 @@ static void page_table_check_set(unsigned long pfn, unsigned long pgcnt, for_each_page_ext(page, pgcnt, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - if (anon) { - BUG_ON(atomic_read(&ptc->file_map_count)); - BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw); - } else { - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0); - } + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) + page_table_check_update(&ptc->slots[i], anon, false, rw); } rcu_read_unlock(); } @@ -139,8 +208,10 @@ void __page_table_check_zero(struct page *page, unsigned int order) for_each_page_ext(page, 1 << order, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_read(&ptc->file_map_count)); + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) { + BUG_ON(atomic_read(&ptc->slots[i].anon_map_count)); + BUG_ON(atomic_read(&ptc->slots[i].file_map_count)); + } } rcu_read_unlock(); } @@ -151,8 +222,15 @@ void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, if (&init_mm == mm) return; - if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte)) - page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT); + if (!pte_user_accessible_page(mm, addr, pte) || pte_special(pte)) + return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + page_table_check_subpage(pte_phys_mm(mm, pte), mm_page_size(mm), true, false); + return; + } +#endif + page_table_check_clear(pte_pfn(pte), mm_pte_native_pages(mm)); } EXPORT_SYMBOL(__page_table_check_pte_clear); @@ -218,9 +296,20 @@ void __page_table_check_ptes_set(struct mm_struct *mm, unsigned long addr, page_table_check_pte_flags(pte); for (i = 0; i < nr; i++) - __page_table_check_pte_clear(mm, addr + PAGE_SIZE * i, ptep_get(ptep + i)); - if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte)) - page_table_check_set(pte_pfn(pte), nr, pte_write(pte)); + __page_table_check_pte_clear(mm, addr + mm_page_size(mm) * i, ptep_get(ptep + i)); + if (!pte_user_accessible_page(mm, addr, pte) || pte_special(pte)) + return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + phys_addr_t phys = pte_phys_mm(mm, pte); + + for (i = 0; i < nr; i++) + page_table_check_subpage(phys + i * mm_page_size(mm), mm_page_size(mm), false, + pte_write(pte)); + return; + } +#endif + page_table_check_set(pte_pfn(pte), nr * mm_pte_native_pages(mm), pte_write(pte)); } EXPORT_SYMBOL(__page_table_check_ptes_set); @@ -279,16 +368,17 @@ void __page_table_check_pte_clear_range(struct mm_struct *mm, return; if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { - pte_t *ptep = pte_offset_map(&pmd, addr); + pte_t *ptep = pte_offset_map_mm(mm, &pmd, addr); + unsigned long nr = pte_table_bytes_mm(mm) / sizeof(pte_t); unsigned long i; if (WARN_ON(!ptep)) return; - for (i = 0; i < PTRS_PER_PTE; i++) { + for (i = 0; i < nr; i++) { __page_table_check_pte_clear(mm, addr, ptep_get(ptep)); - addr += PAGE_SIZE; + addr += mm_page_size(mm); ptep++; } - pte_unmap(ptep - PTRS_PER_PTE); + pte_unmap(ptep - nr); } } diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index ccd1c622c8b24..e3b0b64fbfba2 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -87,7 +87,7 @@ static bool map_pte(struct page_vma_mapped_walk *pvmw, pmd_t *pmdvalp, * mapped at the @pvmw->pte * @pvmw: page_vma_mapped_walk struct, includes a pair pte and pfn range * for checking - * @pte_nr: the number of small pages described by @pvmw->pte. + * @pte_nr: the number of native physical pages covered by @pvmw->pte. * * page_vma_mapped_walk() found a place where pfn range is *potentially* * mapped. check_pte() has to validate this. @@ -109,6 +109,18 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr) unsigned long pfn; pte_t ptent = ptep_get(pvmw->pte); +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) { + if (!(pvmw->flags & PVMW_MIGRATION)) + return pte_present(ptent) && + pte_phys_mm(pvmw->vma->vm_mm, ptent) == + PFN_PHYS(pvmw->pfn) + pvmw->subpage_offset; + if (pte_swp_subpage_offset(ptent) != pvmw->subpage_offset) + return false; + /* The common migration check below validates the native PFN. */ + } +#endif + if (pvmw->flags & PVMW_MIGRATION) { const softleaf_t entry = softleaf_from_pte(ptent); @@ -188,6 +200,18 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) pud_t *pud; pmd_t pmde; +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) { + if (WARN_ON_ONCE(pvmw->nr_pages != 1 || + mm_page_size(mm) >= PAGE_SIZE || + !IS_ALIGNED(pvmw->address, mm_page_size(mm)) || + pvmw->subpage_offset >= PAGE_SIZE || + !IS_ALIGNED(pvmw->subpage_offset, mm_page_size(mm)) || + is_vm_hugetlb_page(vma))) + return not_found(pvmw); + } +#endif + /* The only possible pmd mapping has been handled on last iteration */ if (pvmw->pmd && !pvmw->pte) return not_found(pvmw); @@ -213,6 +237,9 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) return true; } + /* A query for an interior native page still visits one whole user leaf. */ + if (mm_page_size(mm) > PAGE_SIZE) + pvmw->address &= mm_page_mask(mm); end = vma_address_end(pvmw); if (pvmw->pte) goto next_pte; @@ -220,21 +247,21 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) do { pgd = pgd_offset(mm, pvmw->address); if (!pgd_present(*pgd)) { - step_forward(pvmw, PGDIR_SIZE); + step_forward(pvmw, pgd_size_mm(mm)); continue; } - p4d = p4d_offset(pgd, pvmw->address); - if (!p4d_present(*p4d)) { - step_forward(pvmw, P4D_SIZE); + p4d = p4d_offset_mm(mm, pgd, pvmw->address); + if (!p4d_present_mm(mm, *p4d)) { + step_forward(pvmw, p4d_size_mm(mm)); continue; } - pud = pud_offset(p4d, pvmw->address); + pud = pud_offset_mm(mm, p4d, pvmw->address); if (!pud_present(*pud)) { - step_forward(pvmw, PUD_SIZE); + step_forward(pvmw, pud_size_mm(mm)); continue; } - pvmw->pmd = pmd_offset(pud, pvmw->address); + pvmw->pmd = pmd_offset_mm(mm, pud, pvmw->address); /* * Make sure the pmd value isn't cached in a register by the * compiler and used as a stale value after we've observed a @@ -245,6 +272,9 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && (pmd_trans_huge(pmde) || pmd_is_migration_entry(pmde) || pmd_is_device_private_entry(pmde))) { + /* Subpage PMD mappings need a distinct huge-page implementation. */ + if (mm_page_size(mm) != PAGE_SIZE) + return not_found(pvmw); pvmw->ptl = pmd_lock(mm, pvmw->pmd); pmde = *pvmw->pmd; if (pmd_is_migration_entry(pmde)) { @@ -285,7 +315,7 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) (pvmw->nr_pages >= HPAGE_PMD_NR)) sync_with_folio_pmd_zap(mm, pvmw->pmd); - step_forward(pvmw, PMD_SIZE); + step_forward(pvmw, pmd_size_mm(mm)); continue; } if (!map_pte(pvmw, &pmde, &ptl)) { @@ -294,15 +324,15 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) goto next_pte; } this_pte: - if (check_pte(pvmw, 1)) + if (check_pte(pvmw, mm_pte_native_pages(mm))) return true; next_pte: do { - pvmw->address += PAGE_SIZE; + pvmw->address += mm_page_size(mm); if (pvmw->address >= end) return not_found(pvmw); /* Did we cross page table boundary? */ - if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) { + if ((pvmw->address & (pmd_size_mm(mm) - mm_page_size(mm))) == 0) { if (pvmw->ptl) { spin_unlock(pvmw->ptl); pvmw->ptl = NULL; @@ -345,6 +375,7 @@ unsigned long page_mapped_in_vma(const struct page *page, struct vm_area_struct *vma) { const struct folio *folio = page_folio(page); + unsigned long address; struct page_vma_mapped_walk pvmw = { .pfn = page_to_pfn(page), .nr_pages = 1, @@ -352,13 +383,15 @@ unsigned long page_mapped_in_vma(const struct page *page, .flags = PVMW_SYNC, }; - pvmw.address = vma_address(vma, page_pgoff(folio, page), 1); + address = vma_address(vma, page_pgoff(folio, page), 1); + pvmw.address = address; if (pvmw.address == -EFAULT) goto out; if (!page_vma_mapped_walk(&pvmw)) return -EFAULT; page_vma_mapped_walk_done(&pvmw); out: - return pvmw.address; + /* Report this native page's byte address, not the covering leaf start. */ + return address; } #endif diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 5d87c632a2550..c3216b4852399 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -30,13 +30,14 @@ static int walk_pte_range_inner(pte_t *pte, unsigned long addr, unsigned long end, struct mm_walk *walk) { const struct mm_walk_ops *ops = walk->ops; + const unsigned long page_size = mm_page_size(walk->mm); int err = 0; for (;;) { if (ops->install_pte && pte_none(ptep_get(pte))) { pte_t new_pte; - err = ops->install_pte(addr, addr + PAGE_SIZE, &new_pte, + err = ops->install_pte(addr, addr + page_size, &new_pte, walk); if (err) break; @@ -46,13 +47,13 @@ static int walk_pte_range_inner(pte_t *pte, unsigned long addr, if (!WARN_ON_ONCE(walk->no_vma)) update_mmu_cache(walk->vma, addr, pte); } else { - err = ops->pte_entry(pte, addr, addr + PAGE_SIZE, walk); + err = ops->pte_entry(pte, addr, addr + page_size, walk); if (err) break; } - if (addr >= end - PAGE_SIZE) + if (addr >= end - page_size) break; - addr += PAGE_SIZE; + addr += page_size; pte++; } return err; @@ -67,15 +68,15 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, if (walk->no_vma) { /* - * pte_offset_map() might apply user-specific validation. + * pte_offset_map_mm() might apply user-specific validation. * Indeed, on x86_64 the pmd entries set up by init_espfix_ap() * fit its pmd_bad() check (_PAGE_NX set and _PAGE_RW clear), * and CONFIG_EFI_PGT_DUMP efi_mm goes so far as to walk them. */ if (walk->mm == &init_mm || addr >= TASK_SIZE) - pte = pte_offset_kernel(pmd, addr); + pte = pte_offset_kernel_mm(walk->mm, pmd, addr); else - pte = pte_offset_map(pmd, addr); + pte = pte_offset_map_mm(walk->mm, pmd, addr); if (pte) { err = walk_pte_range_inner(pte, addr, end, walk); if (walk->mm != &init_mm && addr < TASK_SIZE) @@ -123,10 +124,10 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, return 0; } - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(walk->mm, pud, addr); do { again: - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(walk->mm, addr, end); if (pmd_none(*pmd)) { if (has_install) err = __pte_alloc(walk->mm, pmd); @@ -193,10 +194,10 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, int err = 0; int depth = real_depth(2); - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(walk->mm, p4d, addr); do { again: - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(walk->mm, addr, end); if (pud_none(*pud)) { if (has_install) err = __pmd_alloc(walk->mm, pud, addr); @@ -258,10 +259,10 @@ static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, int err = 0; int depth = real_depth(1); - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(walk->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) { + next = p4d_addr_end_mm(walk->mm, addr, end); + if (p4d_none_or_clear_bad_mm(walk->mm, p4d)) { if (has_install) err = __pud_alloc(walk->mm, p4d, addr); else if (ops->pte_hole) @@ -297,11 +298,11 @@ static int walk_pgd_range(unsigned long addr, unsigned long end, int err = 0; if (walk->pgd) - pgd = walk->pgd + pgd_index(addr); + pgd = pgd_offset_pgd_mm(walk->mm, walk->pgd, addr); else pgd = pgd_offset(walk->mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(walk->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) { if (has_install) err = __p4d_alloc(walk->mm, pgd, addr); @@ -924,11 +925,11 @@ struct folio *folio_walk_start(struct folio_walk *fw, if (pgd_none_or_clear_bad(pgdp)) goto not_found; - p4dp = p4d_offset(pgdp, addr); - if (p4d_none_or_clear_bad(p4dp)) + p4dp = p4d_offset_mm(vma->vm_mm, pgdp, addr); + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4dp)) goto not_found; - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_mm(vma->vm_mm, p4dp, addr); pud = pudp_get(pudp); if (pud_none(pud)) goto not_found; @@ -937,7 +938,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, ptl = pud_lock(vma->vm_mm, pudp); pud = pudp_get(pudp); - entry_size = PUD_SIZE; + entry_size = pud_size_mm(vma->vm_mm); fw->level = FW_LEVEL_PUD; fw->pudp = pudp; fw->pud = pud; @@ -959,7 +960,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, pmd_table: VM_WARN_ON_ONCE(!pud_present(pud) || pud_leaf(pud)); - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_mm(vma->vm_mm, pudp, addr); pmd = pmdp_get_lockless(pmdp); if (pmd_none(pmd)) goto not_found; @@ -968,7 +969,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, ptl = pmd_lock(vma->vm_mm, pmdp); pmd = pmdp_get(pmdp); - entry_size = PMD_SIZE; + entry_size = pmd_size_mm(vma->vm_mm); fw->level = FW_LEVEL_PMD; fw->pmdp = pmdp; fw->pmd = pmd; @@ -1001,7 +1002,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, goto not_found; pte = ptep_get(ptep); - entry_size = PAGE_SIZE; + entry_size = mm_page_size(vma->vm_mm); fw->level = FW_LEVEL_PTE; fw->ptep = ptep; fw->pte = pte; @@ -1028,5 +1029,5 @@ struct folio *folio_walk_start(struct folio_walk *fw, else fw->page = NULL; fw->ptl = ptl; - return page_folio(page); + return page_folio(zeropage ? page : fw->page); } diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index b91b1a98029c7..7a9eef9b4f5b7 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -168,13 +168,15 @@ pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable) { + struct page *page = pgtable_to_page(pgtable); + assert_spin_locked(pmd_lockptr(mm, pmdp)); /* FIFO */ if (!pmd_huge_pte(mm, pmdp)) - INIT_LIST_HEAD(&pgtable->lru); + INIT_LIST_HEAD(&page->lru); else - list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru); + list_add(&page->lru, &pgtable_to_page(pmd_huge_pte(mm, pmdp))->lru); pmd_huge_pte(mm, pmdp) = pgtable; } #endif @@ -184,15 +186,17 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) { pgtable_t pgtable; + struct page *page, *next; assert_spin_locked(pmd_lockptr(mm, pmdp)); /* FIFO */ pgtable = pmd_huge_pte(mm, pmdp); - pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru, - struct page, lru); + page = pgtable_to_page(pgtable); + next = list_first_entry_or_null(&page->lru, struct page, lru); + pmd_huge_pte(mm, pmdp) = pgtable_from_page(next); if (pmd_huge_pte(mm, pmdp)) - list_del(&pgtable->lru); + list_del(&page->lru); return pgtable; } #endif @@ -282,7 +286,12 @@ static unsigned long pmdp_get_lockless_start(void) { return 0; } static void pmdp_get_lockless_end(unsigned long irqflags) { } #endif +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, pmd_t *pmdvalp) +#else pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) +#endif { unsigned long irqflags; pmd_t pmdval; @@ -302,19 +311,27 @@ pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) pmd_clear_bad(pmd); goto nomap; } - return __pte_map(&pmdval, addr); + return __pte_map_mm(mm, &pmdval, addr); nomap: rcu_read_unlock(); return NULL; } +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) +{ + /* NULL retains native geometry for callers not yet converted. */ + return __pte_offset_map_mm(NULL, pmd, addr, pmdvalp); +} +#endif + pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp) { pmd_t pmdval; pte_t *pte; - pte = __pte_offset_map(pmd, addr, &pmdval); + pte = __pte_offset_map_mm(mm, pmd, addr, &pmdval); if (likely(pte)) *ptlp = pte_lockptr(mm, &pmdval); return pte; @@ -327,7 +344,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, pte_t *pte; VM_WARN_ON_ONCE(!pmdvalp); - pte = __pte_offset_map(pmd, addr, pmdvalp); + pte = __pte_offset_map_mm(mm, pmd, addr, pmdvalp); if (likely(pte)) *ptlp = pte_lockptr(mm, pmdvalp); return pte; @@ -397,7 +414,7 @@ pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, pmd_t pmdval; pte_t *pte; again: - pte = __pte_offset_map(pmd, addr, &pmdval); + pte = __pte_offset_map_mm(mm, pmd, addr, &pmdval); if (unlikely(!pte)) return pte; ptl = pte_lockptr(mm, &pmdval); diff --git a/mm/process_vm_access.c b/mm/process_vm_access.c index 656d3e88755b6..f178b5fdb6589 100644 --- a/mm/process_vm_access.c +++ b/mm/process_vm_access.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,8 @@ #include #include +#include "internal.h" + /** * process_vm_rw_pages - read/write pages from task specified * @pages: array of pointers to pages we want to copy @@ -135,6 +138,63 @@ static int process_vm_rw_single_vec(unsigned long addr, which lives on stack */ #define PVM_MAX_PP_ARRAY_COUNT 16 +#ifdef CONFIG_MM_SUBPAGE +/* Same pin/copy/dirty lifetime as the page-array path, with explicit offsets. */ +int process_vm_rw_fragments(struct mm_struct *mm, unsigned long addr, + unsigned long len, struct iov_iter *iter, bool write) +{ + struct user_page_fragment stack[PVM_MAX_PP_ARRAY_COUNT]; + struct user_page_fragment *fragments = stack; + unsigned long capacity, nr; + int ret = 0; + + if (!len || !iov_iter_count(iter)) + return 0; + capacity = 1 + ((len - 1) >> MM_SUBPAGE_SHIFT) + + ((((len - 1) & (MM_SUBPAGE_SIZE - 1)) + + (addr & (MM_SUBPAGE_SIZE - 1))) >> MM_SUBPAGE_SHIFT); + capacity = min_t(unsigned long, capacity, + PVM_MAX_KMALLOC_PAGES * PAGE_SIZE / sizeof(*fragments)); + if (capacity > ARRAY_SIZE(stack)) { + fragments = kmalloc_array(capacity, sizeof(*fragments), GFP_KERNEL); + if (!fragments) + return -ENOMEM; + } + while (len && iov_iter_count(iter)) { + long got = pin_user_fragments_remote(mm, addr, len, + write ? FOLL_WRITE : 0, fragments, capacity); + + if (got <= 0) { + ret = -EFAULT; + break; + } + for (nr = 0; nr < got && iov_iter_count(iter); nr++) { + struct user_page_fragment *f = &fragments[nr]; + struct page *page = folio_page(f->folio, f->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(f->offset); + size_t copied; + + if (write) + copied = copy_page_from_iter(page, offset, f->length, iter); + else + copied = copy_page_to_iter(page, offset, f->length, iter); + addr += copied; + len -= copied; + if (copied < f->length && iov_iter_count(iter)) { + ret = -EFAULT; + break; + } + } + release_user_fragments(fragments, got, write); + if (ret) + break; + } + if (fragments != stack) + kfree(fragments); + return ret; +} +#endif + /** * process_vm_rw_core - core of reading/writing pages from task specified * @pid: PID of process to read/write from/to @@ -182,17 +242,6 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, if (nr_pages == 0) return 0; - if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) { - /* For reliability don't try to kmalloc more than - 2 pages worth */ - process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES * PAGE_SIZE, - sizeof(struct page *)*nr_pages), - GFP_KERNEL); - - if (!process_pages) - return -ENOMEM; - } - /* Get process information */ task = find_get_task_by_vpid(pid); if (!task) { @@ -212,11 +261,35 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, goto put_task_struct; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + for (i = 0; i < riovcnt && iov_iter_count(iter) && !rc; i++) + rc = process_vm_rw_fragments(mm, (unsigned long)rvec[i].iov_base, + rvec[i].iov_len, iter, vm_write); + goto copied; + } +#endif + if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) { + /* For reliability don't try to kmalloc more than + 2 pages worth */ + process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES * PAGE_SIZE, + sizeof(struct page *)*nr_pages), + GFP_KERNEL); + + if (!process_pages) { + rc = -ENOMEM; + goto put_mm; + } + } + for (i = 0; i < riovcnt && iov_iter_count(iter) && !rc; i++) rc = process_vm_rw_single_vec( (unsigned long)rvec[i].iov_base, rvec[i].iov_len, iter, process_pages, mm, task, vm_write); +#ifdef CONFIG_MM_SUBPAGE +copied: +#endif /* copied = space before - space after */ total_len -= iov_iter_count(iter); @@ -226,6 +299,7 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, if (total_len) rc = total_len; +put_mm: mmput(mm); put_task_struct: diff --git a/mm/rmap.c b/mm/rmap.c index 99e1b3dc390b7..c4b7043fc8339 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -75,6 +75,7 @@ #include #include #include +#include #include @@ -83,17 +84,22 @@ #include "internal.h" #include "swap.h" +#include "user-subpage-internal.h" static struct kmem_cache *anon_vma_cachep; static struct kmem_cache *anon_vma_chain_cachep; -static inline struct anon_vma *anon_vma_alloc(void) +static inline struct anon_vma *anon_vma_alloc(struct vm_area_struct *vma) { struct anon_vma *anon_vma; anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); if (anon_vma) { atomic_set(&anon_vma->refcount, 1); +#ifdef CONFIG_MM_SUBPAGE + anon_vma->min_folio_order = + max_t(unsigned int, mm_page_shift(vma->vm_mm), PAGE_SHIFT) - PAGE_SHIFT; +#endif anon_vma->num_children = 0; anon_vma->num_active_vmas = 0; anon_vma->parent = anon_vma; @@ -198,7 +204,7 @@ int __anon_vma_prepare(struct vm_area_struct *vma) anon_vma = find_mergeable_anon_vma(vma); allocated = NULL; if (!anon_vma) { - anon_vma = anon_vma_alloc(); + anon_vma = anon_vma_alloc(vma); if (unlikely(!anon_vma)) goto out_enomem_free_avc; anon_vma->num_children++; /* self-parent link for new root */ @@ -388,7 +394,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) /* Drop inherited anon_vma, we'll reuse existing or allocate new. */ vma->anon_vma = NULL; - anon_vma = anon_vma_alloc(); + anon_vma = anon_vma_alloc(vma); if (!anon_vma) return -ENOMEM; avc = anon_vma_chain_alloc(GFP_KERNEL); @@ -622,6 +628,35 @@ struct anon_vma *folio_get_anon_vma(const struct folio *folio) return anon_vma; } +#ifdef CONFIG_MM_SUBPAGE +/* The caller retains the folio; splitting rechecks the order under its lock. */ +unsigned int folio_anon_min_order(struct folio *folio) +{ + struct anon_vma *anon_vma = NULL; + unsigned long mapping; + unsigned int order = 0; + + /* Also used by memory-failure callers before they lock the folio. */ + rcu_read_lock(); + mapping = (unsigned long)READ_ONCE(folio->mapping); + if ((mapping & FOLIO_MAPPING_FLAGS) != FOLIO_MAPPING_ANON || !folio_mapped(folio)) + goto out; + anon_vma = (struct anon_vma *)(mapping - FOLIO_MAPPING_ANON); + if (!atomic_inc_not_zero(&anon_vma->refcount)) { + anon_vma = NULL; + goto out; + } + /* Revalidate against unmap/reuse of the RCU-typesafe anon_vma slab. */ + if (folio_mapped(folio) && mapping == (unsigned long)READ_ONCE(folio->mapping)) + order = anon_vma->min_folio_order; +out: + rcu_read_unlock(); + if (anon_vma) + put_anon_vma(anon_vma); + return order; +} +#endif + /* * Similar to folio_get_anon_vma() except it locks the anon_vma. * @@ -856,6 +891,13 @@ static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) unsigned long page_address_in_vma(const struct folio *folio, const struct page *page, const struct vm_area_struct *vma) { +#ifdef CONFIG_MM_SUBPAGE + unsigned long address; + + if (folio_test_anon(folio) && !folio_test_large(folio) && + mm_subpage_address_in_vma(folio, vma, &address)) + return address; +#endif if (folio_test_anon(folio)) { struct anon_vma *anon_vma = folio_anon_vma(folio); /* @@ -891,15 +933,15 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) if (!pgd_present(*pgd)) goto out; - p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + p4d = p4d_offset_mm(mm, pgd, address); + if (!p4d_present_mm(mm, *p4d)) goto out; - pud = pud_offset(p4d, address); + pud = pud_offset_mm(mm, p4d, address); if (!pud_present(*pud)) goto out; - pmd = pmd_offset(pud, address); + pmd = pmd_offset_mm(mm, pud, address); out: return pmd; } @@ -915,10 +957,11 @@ struct folio_referenced_arg { * arg: folio_referenced_arg will be passed */ static bool folio_referenced_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long address, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct folio_referenced_arg *pra = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); int ptes = 0, referenced = 0; unsigned int nr; @@ -928,10 +971,11 @@ static bool folio_referenced_one(struct folio *folio, if (vma->vm_flags & VM_LOCKED) { ptes++; - pra->mapcount--; + pra->mapcount -= pvmw.pte ? mm_pte_native_pages(vma->vm_mm) : 1; /* Only mlock fully mapped pages */ - if (pvmw.pte && ptes != pvmw.nr_pages) + if (pvmw.pte && ptes != (folio_test_large(folio) ? + ((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm) : 1)) continue; /* @@ -966,11 +1010,18 @@ static bool folio_referenced_one(struct folio *folio, } if (pvmw.pte && folio_test_large(folio)) { - const unsigned long end_addr = pmd_addr_end(address, vma->vm_end); - const unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT; + const unsigned long end_addr = pmd_addr_end_mm(vma->vm_mm, + address, vma->vm_end); + const unsigned int max_nr = (end_addr - address) >> mm_page_shift(vma->vm_mm); pte_t pteval = ptep_get(pvmw.pte); - nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + nr = folio_subpage_pte_batch(vma, folio, pvmw.pte, + pteval, max_nr, 0); + else +#endif + nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr); } /* @@ -994,19 +1045,19 @@ static bool folio_referenced_one(struct folio *folio, } ptes += nr; - pra->mapcount -= nr; + pra->mapcount -= nr * (pvmw.pte ? mm_pte_native_pages(vma->vm_mm) : 1); /* * If we are sure that we batched the entire folio, * we can just optimize and stop right here. */ - if (ptes == pvmw.nr_pages) { + if (ptes == (((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm))) { page_vma_mapped_walk_done(&pvmw); break; } /* Skip the batched PTEs */ pvmw.pte += nr - 1; - pvmw.address += (nr - 1) * PAGE_SIZE; + pvmw.address += (nr - 1) * mm_page_size(vma->vm_mm); } if (referenced) @@ -1178,9 +1229,9 @@ static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw) } static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, PVMW_SYNC); int *cleaned = arg; *cleaned += page_vma_mkclean_one(&pvmw); @@ -1229,8 +1280,9 @@ struct wrprotect_file_state { }; static bool mapping_wrprotect_range_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long address, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct wrprotect_file_state *state = (struct wrprotect_file_state *)arg; struct page_vma_mapped_walk pvmw = { .pfn = state->pfn, @@ -1515,7 +1567,7 @@ static void __page_check_anon_rmap(const struct folio *folio, static __always_inline void __folio_add_anon_rmap(struct folio *folio, struct page *page, int nr_pages, struct vm_area_struct *vma, - unsigned long address, rmap_t flags, enum pgtable_level level) + unsigned long address, rmap_t flags, enum pgtable_level level, bool new) { int i; @@ -1572,7 +1624,7 @@ static __always_inline void __folio_add_anon_rmap(struct folio *folio, * Partially mapped folios can be split on reclaim and part outside * of mlocked VMA can be evicted or freed. */ - if (folio_nr_pages(folio) == nr_pages) + if (!new && folio_nr_pages(folio) == nr_pages) mlock_vma_folio(folio, vma); } @@ -1597,7 +1649,7 @@ void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page, rmap_t flags) { __folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags, - PGTABLE_LEVEL_PTE); + PGTABLE_LEVEL_PTE, false); } /** @@ -1618,7 +1670,7 @@ void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page, { #ifdef CONFIG_TRANSPARENT_HUGEPAGE __folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags, - PGTABLE_LEVEL_PMD); + PGTABLE_LEVEL_PMD, false); #else WARN_ON_ONCE(true); #endif @@ -1697,6 +1749,107 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1); } +#ifdef CONFIG_MM_SUBPAGE +static int __mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address, bool new) +{ + struct folio *folio = mm_subpage_folio(subpage); + int ret; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (mm_page_shift(vma->vm_mm) != mm_subpage_shift(subpage) || !vma->anon_vma || + address < vma->vm_start || address >= vma->vm_end || + !IS_ALIGNED(address, mm_subpage_size(subpage))) + return -EINVAL; + if (mm_subpage_anon_root(subpage)) { + if (mm_subpage_is_droppable(subpage) != !!(vma->vm_flags & VM_DROPPABLE)) + return -EINVAL; + if (!folio_test_anon(folio) || folio_test_ksm(folio) || + PageAnonExclusive(&folio->page)) + return -EINVAL; + } else { + bool was_anon = folio_test_anon(folio); + + /* Swap-cache folios can retain a stale, unmapped anon pointer. */ + if (folio_mapped(folio) || folio_test_ksm(folio) || + (folio->mapping && !was_anon) || + (was_anon && PageAnonExclusive(&folio->page))) + return -EINVAL; + /* Slot references can survive the last VMA in this root. */ + mm_subpage_hold_anon_root(subpage, vma->anon_vma->root, + vma->vm_flags & VM_DROPPABLE); + if (!(vma->vm_flags & VM_DROPPABLE)) + __folio_set_swapbacked(folio); + /* Neighboring slots may acquire different fork/COW relationships. */ + __folio_set_anon(folio, vma, address, false); + if (!was_anon) + mod_mthp_stat(0, MTHP_STAT_NR_ANON, 1); + } + ret = mm_subpage_bind_rmap(subpage, vma, address); + if (ret) + return ret; + atomic_inc(&subpage->mapcount); + /* Identity was checked per slot, not against the folio's linear anchor. */ + __folio_add_rmap(folio, &folio->page, 1, vma, PGTABLE_LEVEL_PTE); + if (!new) + mlock_vma_folio(folio, vma); + return 0; +} + +int mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address) +{ + return __mm_subpage_add_anon_rmap(subpage, vma, address, false); +} + +/* Caller adds this fresh folio to LRU and accounts its locked leaves together. */ +int mm_subpage_add_new_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address) +{ + return __mm_subpage_add_anon_rmap(subpage, vma, address, true); +} + +void mm_subpage_remove_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma) +{ + struct folio *folio = mm_subpage_folio(subpage); + + VM_BUG_ON_FOLIO(atomic_read(&subpage->mapcount) <= 0, folio); + atomic_dec(&subpage->mapcount); + folio_remove_rmap_pte(folio, &folio->page, vma); +} + +int mm_subpage_dup_anon_rmap(struct mm_subpage *subpage, + struct vm_area_struct *dst, struct vm_area_struct *src, unsigned long address) +{ + struct folio *folio = mm_subpage_folio(subpage); + struct vm_page_offset pos; + int ret; + + mmap_assert_write_locked(src->vm_mm); + mmap_assert_write_locked(dst->vm_mm); + if (mm_page_shift(src->vm_mm) != mm_subpage_shift(subpage) || + mm_page_shift(dst->vm_mm) != mm_subpage_shift(subpage) || + !src->anon_vma || !dst->anon_vma || !folio_test_anon(folio) || + folio_test_ksm(folio) || PageAnonExclusive(&folio->page) || + address < dst->vm_start || address >= dst->vm_end || + address < src->vm_start || address >= src->vm_end) + return -EINVAL; + pos = vma_page_offset_at(dst, address); + /* Both mmap locks and the source PTE stabilize this slot's identity. */ + if (!mm_subpage_match_rmap(subpage, dst, address) || + src->anon_vma->root != dst->anon_vma->root || + !vm_page_offset_equal(pos, vma_page_offset_at(src, address))) + return -EINVAL; + /* A pin in a neighboring physical slot does not share this slot. */ + if (!mm_subpage_try_share(subpage)) + return -EBUSY; + ret = folio_try_dup_anon_rmap_pte(folio, &folio->page, dst, src); + if (!ret) + atomic_inc(&subpage->mapcount); + return ret; +} +#endif + static __always_inline void __folio_add_file_rmap(struct folio *folio, struct page *page, int nr_pages, struct vm_area_struct *vma, enum pgtable_level level) @@ -1949,14 +2102,20 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, struct vm_area_struct *vma = pvmw->vma; unsigned int max_nr; +#ifdef CONFIG_MM_SUBPAGE + /* Native PFN batching advances physical pages, not 4K file quarters. */ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return 1; +#endif + if (flags & TTU_HWPOISON) return 1; if (!folio_test_large(folio)) return 1; /* We may only batch within a single VMA and a single page table. */ - end_addr = pmd_addr_end(addr, vma->vm_end); - max_nr = (end_addr - addr) >> PAGE_SHIFT; + end_addr = pmd_addr_end_mm(vma->vm_mm, addr, vma->vm_end); + max_nr = (end_addr - addr) >> mm_page_shift(vma->vm_mm); /* We only support lazyfree or file folios batching for now ... */ if (folio_test_anon(folio) && folio_test_swapbacked(folio)) @@ -1978,20 +2137,116 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY); } +#ifdef CONFIG_MM_SUBPAGE +/* Folio and PTE locks held. Each PTE takes one reference to a native swap entry. */ +static bool try_to_unmap_subpage(struct folio *folio, struct vm_area_struct *vma, + struct page_vma_mapped_walk *pvmw, enum ttu_flags flags) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long address = pvmw->address, end = address + mm_page_size(mm); + struct mm_subpage *slot; + pte_t pte = ptep_get(pvmw->pte), swapped; + swp_entry_t entry; + bool exclusive, ret = false; + + if (!pte_present(pte) || (flags & TTU_HWPOISON)) + return false; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (WARN_ON_ONCE(!slot)) + return false; + if ((vma->vm_flags & VM_DROPPABLE) || mm_subpage_is_lazyfree(slot)) { + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + /* A released quarter can be reused before its native folio is freed. */ + flush_tlb_range(vma, address, end); + smp_mb(); + if (((vma->vm_flags & VM_DROPPABLE) || !pte_dirty(pte)) && + mm_subpage_discardable(slot) && + arch_unmap_one(mm, vma, address, pte) >= 0) { + update_hiwater_rss(mm); + dec_mm_counter(mm, MM_ANONPAGES); + mm_subpage_remove_anon_rmap(slot, vma); + mm_subpage_put(slot); /* Removed PTE reference. */ + ret = true; + goto out; + } + set_pte_at(mm, address, pvmw->pte, pte); + /* Retained references must not turn a droppable folio into swap data. */ + if (vma->vm_flags & VM_DROPPABLE) + goto out; + if (pte_dirty(pte)) { + mm_subpage_cancel_lazyfree(slot); + folio_mark_dirty(folio); + } + } + if (!folio_test_swapbacked(folio)) { + /* A write or retained reference cancelled whole-folio lazy reclaim. */ + folio_set_swapbacked(folio); + goto out; + } + if (!folio_test_swapcache(folio)) + goto out; + entry = page_swap_entry(&folio->page); + if (swap_subpage_dup(entry, mm_subpage_offset(slot)) < 0) + goto out; + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + if (should_defer_flush(mm, flags)) + set_tlb_ubc_flush_pending(mm, pte, address, end); + else + flush_tlb_range(vma, address, end); + if (pte_dirty(pte)) + folio_mark_dirty(folio); + /* The PTE lock serializes fork and this slot's ownership transfer. */ + exclusive = mm_subpage_is_exclusive(slot); + if (arch_unmap_one(mm, vma, address, pte) < 0 || !mm_subpage_try_share(slot)) { + swap_subpage_put(entry, mm_subpage_offset(slot), false); + set_pte_at(mm, address, pvmw->pte, pte); + goto out; + } + if (list_empty(&mm->mmlist)) { + spin_lock(&mmlist_lock); + if (list_empty(&mm->mmlist)) + list_add(&mm->mmlist, &init_mm.mmlist); + spin_unlock(&mmlist_lock); + } + swapped = pte_swp_set_subpage_offset(swp_entry_to_pte(entry), mm_subpage_offset(slot)); + if (exclusive) + swapped = pte_swp_mkexclusive(swapped); + if (pte_soft_dirty(pte)) + swapped = pte_swp_mksoft_dirty(swapped); + if (pte_uffd_wp(pte)) + swapped = pte_swp_mkuffd_wp(swapped); + update_hiwater_rss(mm); + dec_mm_counter(mm, MM_ANONPAGES); + inc_mm_counter(mm, MM_SWAPENTS); + set_pte_at(mm, address, pvmw->pte, swapped); + mm_subpage_remove_anon_rmap(slot, vma); + /* Swap-cache backing cannot be recycled while a deferred TLBI is pending. */ + mm_subpage_put(slot); + ret = true; +out: + mm_subpage_put(slot); + return ret; +} +#endif + /* * @arg: enum ttu_flags will be passed to this argument */ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct mm_struct *mm = vma->vm_mm; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool anon_exclusive, ret = true; pte_t pteval; + phys_addr_t file_phys = 0; struct page *subpage; struct mmu_notifier_range range; enum ttu_flags flags = (enum ttu_flags)(long)arg; - unsigned long nr_pages = 1, end_addr; + unsigned long nr_pages = 1, end_addr, nr_refs; unsigned long pfn; unsigned long hsz = 0; int ptes = 0; @@ -2003,7 +2258,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * if page table locking is skipped: use TTU_SYNC to wait for that. */ if (flags & TTU_SYNC) - pvmw.flags = PVMW_SYNC; + pvmw.flags |= PVMW_SYNC; /* * For THP, we have to assume the worse case ie pmd for invalidation. @@ -2049,7 +2304,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, ret = false; /* Only mlock fully mapped pages */ - if (pvmw.pte && ptes != pvmw.nr_pages) + if (pvmw.pte && ptes != (folio_test_large(folio) ? + ((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm) : 1)) continue; /* @@ -2100,6 +2356,15 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * actually map pages. */ pteval = ptep_get(pvmw.pte); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE && folio_test_anon(folio)) { + if (!try_to_unmap_subpage(folio, vma, &pvmw, flags)) + goto walk_abort; + /* One native folio may cover four independently mapped slots. */ + continue; + } +#endif + if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { @@ -2110,6 +2375,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, } subpage = folio_page(folio, pfn - folio_pfn(folio)); + if (!folio_test_anon(folio)) + file_phys = pte_phys_mm(mm, ptep_get(pvmw.pte)); address = pvmw.address; anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(subpage); @@ -2167,7 +2434,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, folio_mark_dirty(folio); } else if (likely(pte_present(pteval))) { nr_pages = folio_unmap_pte_batch(folio, &pvmw, flags, pteval); - end_addr = address + nr_pages * PAGE_SIZE; + end_addr = address + nr_pages * mm_page_size(mm); flush_cache_range(vma, address, end_addr); /* Nuke the page table entry. */ @@ -2281,7 +2548,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, goto discard; } - if (folio_dup_swap(folio, subpage) < 0) { + if (folio_dup_swap_range(folio, subpage, mm_pte_native_pages(mm)) < 0) { set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } @@ -2292,15 +2559,15 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * so we'll not check/care. */ if (arch_unmap_one(mm, vma, address, pteval) < 0) { - folio_put_swap(folio, subpage); + folio_put_swap_range(folio, subpage, mm_pte_native_pages(mm)); set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } /* See folio_try_share_anon_rmap(): clear PTE first. */ if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, subpage)) { - folio_put_swap(folio, subpage); + folio_try_share_anon_rmap_pte(folio, subpage, vma)) { + folio_put_swap_range(folio, subpage, mm_pte_native_pages(mm)); set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } @@ -2342,20 +2609,26 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, add_mm_counter(mm, mm_counter_file(folio), -nr_pages); } discard: + nr_refs = nr_pages; + /* A base user leaf can own references to several native pages. */ + if (!folio_test_hugetlb(folio)) + nr_refs *= mm_pte_native_pages(mm); + if (!folio_test_anon(folio)) + mm_subpage_file_map_del(vma, file_phys); if (unlikely(folio_test_hugetlb(folio))) { hugetlb_remove_rmap(folio); } else { - folio_remove_rmap_ptes(folio, subpage, nr_pages, vma); + folio_remove_rmap_ptes(folio, subpage, nr_refs, vma); } if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); - folio_put_refs(folio, nr_pages); + folio_put_refs(folio, nr_refs); /* * If we are sure that we batched the entire folio and cleared * all PTEs, we can just optimize and stop right here. */ - if (nr_pages == folio_nr_pages(folio)) + if (nr_pages == folio_nr_pages(folio) && mm_page_size(mm) == PAGE_SIZE) goto walk_done; continue; walk_abort: @@ -2412,11 +2685,72 @@ void try_to_unmap(struct folio *folio, enum ttu_flags flags) * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs * containing migration entries. */ +#ifdef CONFIG_MM_SUBPAGE +static bool try_to_migrate_subpage(struct folio *folio, struct vm_area_struct *vma, + struct page_vma_mapped_walk *pvmw, enum ttu_flags flags) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long address = pvmw->address, end = address + mm_page_size(mm); + struct mm_subpage *slot; + pte_t pte = ptep_get(pvmw->pte), migration; + swp_entry_t entry; + bool ret = false; + + /* Other migration drivers must prepare ownership before removing PTEs. */ + if (!pte_present(pte) || !mm_subpage_migrating(folio) || + folio_test_hwpoison(folio) || folio_is_zone_device(folio)) + return false; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (WARN_ON_ONCE(!slot)) + return false; + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + if (should_defer_flush(mm, flags)) + set_tlb_ubc_flush_pending(mm, pte, address, end); + else + flush_tlb_range(vma, address, end); + if (pte_dirty(pte)) + folio_mark_dirty(folio); + if (arch_unmap_one(mm, vma, address, pte) < 0) { + set_pte_at(mm, address, pvmw->pte, pte); + goto out; + } + if (pte_write(pte)) + entry = make_writable_migration_entry(folio_pfn(folio)); + else if (mm_subpage_is_exclusive(slot)) + entry = make_readable_exclusive_migration_entry(folio_pfn(folio)); + else + entry = make_readable_migration_entry(folio_pfn(folio)); + if (pte_young(pte)) + entry = make_migration_entry_young(entry); + if (pte_dirty(pte)) + entry = make_migration_entry_dirty(entry); + migration = pte_swp_set_subpage_offset(swp_entry_to_pte(entry), mm_subpage_offset(slot)); + if (pte_soft_dirty(pte)) + migration = pte_swp_mksoft_dirty(migration); + if (pte_uffd_wp(pte)) + migration = pte_swp_mkuffd_wp(migration); + set_pte_at(mm, address, pvmw->pte, migration); + trace_set_migration_pte(address, pte_val(migration), 0); + mm_subpage_migrate_save_lazyfree(slot); + mm_subpage_remove_anon_rmap(slot, vma); + if (vma->vm_flags & VM_LOCKED) + mlock_drain_local(); + /* The driver holds backing until its batched invalidation completes. */ + mm_subpage_put(slot); + ret = true; +out: + mm_subpage_put(slot); + return ret; +} +#endif + static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct mm_struct *mm = vma->vm_mm; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool anon_exclusive, writable, ret = true; pte_t pteval; struct page *subpage; @@ -2424,6 +2758,11 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, enum ttu_flags flags = (enum ttu_flags)(long)arg; unsigned long pfn; unsigned long hsz = 0; + unsigned int nr_refs = 1; + + /* A base PTE owns one reference for each covered native page. */ + if (!folio_test_hugetlb(folio)) + nr_refs = mm_pte_native_pages(mm); /* * When racing against e.g. zap_pte_range() on another cpu, @@ -2432,7 +2771,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, * if page table locking is skipped: use TTU_SYNC to wait for that. */ if (flags & TTU_SYNC) - pvmw.flags = PVMW_SYNC; + pvmw.flags |= PVMW_SYNC; /* * For THP, we have to assume the worse case ie pmd for invalidation. @@ -2501,6 +2840,17 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, /* Unexpected PMD-mapped THP? */ VM_BUG_ON_FOLIO(!pvmw.pte, folio); + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE && folio_test_anon(folio)) { + if (!try_to_migrate_subpage(folio, vma, &pvmw, flags)) { + ret = false; + page_vma_mapped_walk_done(&pvmw); + break; + } + continue; + } +#endif /* * Handle PFN swap PTEs, such as device-exclusive ones, that * actually map pages. @@ -2586,7 +2936,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ pteval = ptep_get_and_clear(mm, address, pvmw.pte); - set_tlb_ubc_flush_pending(mm, pteval, address, address + PAGE_SIZE); + set_tlb_ubc_flush_pending(mm, pteval, address, address + mm_page_size(mm)); } else { pteval = ptep_clear_flush(vma, address, pvmw.pte); } @@ -2663,7 +3013,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, break; } } else if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, subpage)) { + folio_try_share_anon_rmap_pte(folio, subpage, vma)) { set_pte_at(mm, address, pvmw.pte, pteval); ret = false; page_vma_mapped_walk_done(&pvmw); @@ -2701,6 +3051,13 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, if (pte_swp_uffd_wp(pteval)) swp_pte = pte_swp_mkuffd_wp(swp_pte); } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) + swp_pte = pte_swp_set_subpage_offset(swp_pte, + pte_present(pteval) ? pte_phys_mm(mm, pteval) & ~PAGE_MASK : + pte_swp_subpage_offset(pteval)); +#endif + if (folio_test_hugetlb(folio)) set_huge_pte_at(mm, address, pvmw.pte, swp_pte, hsz); @@ -2714,13 +3071,15 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ } + if (!folio_test_anon(folio)) + mm_subpage_file_map_del(vma, pte_phys_mm(mm, pteval)); if (unlikely(folio_test_hugetlb(folio))) hugetlb_remove_rmap(folio); else folio_remove_rmap_pte(folio, subpage, vma); if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); - folio_put(folio); + folio_put_refs(folio, nr_refs); } mmu_notifier_invalidate_range_end(&range); @@ -2951,6 +3310,61 @@ static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio, return anon_vma; } +#ifdef CONFIG_MM_SUBPAGE +noinline bool rmap_walk_subpages(struct folio *folio, + struct rmap_walk_control *rwc, bool locked) +{ + struct mm_subpage_rmap maps[MM_SUBPAGES_PER_PAGE]; + int i, nr = mm_subpage_snapshot_rmap(folio, maps); + + if (nr < 0) + return false; + /* Existing locked anonymous walks operate on large folios. */ + VM_BUG_ON_FOLIO(locked, folio); + for (i = 0; i < nr; i++) { + struct mm_subpage_rmap *map = &maps[i]; + struct anon_vma_chain *avc; + bool stop = false; + + if (!anon_vma_trylock_read(map->root)) { + if (rwc->try_lock) { + rwc->contended = true; + break; + } + anon_vma_lock_read(map->root); + } + anon_vma_interval_tree_foreach(avc, &map->root->rb_root, + map->pos.index, map->pos.index) { + struct vm_area_struct *vma = avc->vma; + unsigned long address = vma_address_at_offset(vma, map->pos); + struct rmap_walk_range range = { + .address = address, + .subpage = true, + .subpage_offset = map->offset, + }; + + /* Native-index overlap need not include this exact leaf. */ + if (address == -EFAULT) + continue; + cond_resched(); + if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) + continue; + if (!rwc->rmap_one(folio, vma, range, rwc->arg) || + (rwc->done && rwc->done(folio))) { + stop = true; + break; + } + } + anon_vma_unlock_read(map->root); + if (stop) + break; + } + for (i = 0; i < nr; i++) + put_anon_vma(maps[i].root); + return true; +} +#endif + /* * rmap_walk_anon - do something to anonymous page using the object-based * rmap method @@ -2973,6 +3387,10 @@ static void rmap_walk_anon(struct folio *folio, * to an anon_vma with different root. */ VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); +#ifdef CONFIG_MM_SUBPAGE + if (!folio_test_large(folio) && rmap_walk_subpages(folio, rwc, locked)) + return; +#endif if (locked) { anon_vma = folio_anon_vma(folio); @@ -2998,7 +3416,8 @@ static void rmap_walk_anon(struct folio *folio, if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, address, rwc->arg)) + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = address }, rwc->arg)) break; if (rwc->done && rwc->done(folio)) break; @@ -3061,7 +3480,8 @@ static void __rmap_walk_file(struct folio *folio, struct address_space *mapping, if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, address, rwc->arg)) + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = address }, rwc->arg)) goto done; if (rwc->done && rwc->done(folio)) goto done; diff --git a/mm/shmem.c b/mm/shmem.c index 7b1ea9fb598f4..1542ec3f10912 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -30,6 +30,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -875,6 +878,266 @@ static void shmem_update_stats(struct folio *folio, int nr_pages) lruvec_stat_mod_folio(folio, NR_SHMEM, nr_pages); } +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +#define SHMEM_UFFD_ALL_UNITS GENMASK(PAGE_SIZE / SZ_4K - 1, 0) + +/* + * Resident masks live in page_ext, allowing partial truncate to update them + * without allocation. All cache insertion/replacement paths initialize or + * transfer them under the folio lock. The inode XArray persists masks only + * for swapped indices. Its publication/removal additionally holds i_pages, + * before taking uffd_missing's XArray lock. + */ +static void shmem_uffd_enable(struct inode *inode) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + + spin_lock(&info->lock); + info->flags |= SHMEM_F_UFFD_SUBPAGE; + spin_unlock(&info->lock); +} + +static unsigned long shmem_uffd_mask(unsigned int offset, unsigned long length) +{ + return GENMASK((offset + length - 1) / SZ_4K, offset / SZ_4K); +} + +unsigned long shmem_uffd_missing_mask(struct folio *folio, pgoff_t index) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); + VM_BUG_ON_FOLIO(!folio_mapping(folio) || !shmem_mapping(folio_mapping(folio)), folio); + return mm_subpage_shmem_missing(folio_page(folio, index - folio->index)); +} + +int shmem_uffd_set_missing(struct folio *folio, pgoff_t index, + unsigned long missing) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!folio_mapping(folio) || !shmem_mapping(folio_mapping(folio)) || + !folio_contains(folio, index) || + !folio_test_uptodate(folio) || (missing & ~SHMEM_UFFD_ALL_UNITS)) + return -EINVAL; + if (missing) + shmem_uffd_enable(folio->mapping->host); + mm_subpage_shmem_set_missing(folio_page(folio, index - folio->index), missing); + return 0; +} + +bool shmem_uffd_has_missing_native_page(struct folio *folio) +{ + struct address_space *mapping = folio_mapping(folio); + unsigned long i; + + if (!mapping || !shmem_mapping(mapping) || + !(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return false; + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + /* Native PMD exposure is allowed when every native leaf has some data. */ + for (i = 0; i < folio_nr_pages(folio); i++) + if (mm_subpage_shmem_missing(folio_page(folio, i)) == SHMEM_UFFD_ALL_UNITS) + return true; + return false; +} + +bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length) +{ + struct address_space *mapping = folio_mapping(folio); + unsigned long mask; + + if (!mapping || !shmem_mapping(mapping)) + return false; + mask = shmem_uffd_mask(offset, length); + return (shmem_uffd_missing_mask(folio, index) & mask) == mask; +} + +/* A coarse alias with any populated units preserves them and exposes zero holes. */ +static unsigned long shmem_uffd_reason(struct vm_fault *vmf, unsigned long missing) +{ + struct vm_area_struct *vma = vmf->vma; + struct vm_page_offset pos = vma_page_offset_at(vma, vmf->address); + unsigned long mask = shmem_uffd_mask(pos.offset, mm_page_size(vma->vm_mm)); + + if ((missing & mask) == mask) + return userfaultfd_missing(vma) ? VM_UFFD_MISSING : 0; + return userfaultfd_minor(vma) ? VM_UFFD_MINOR : 0; +} + +void shmem_uffd_mark_present(struct folio *folio, pgoff_t index, + unsigned long present) +{ + unsigned long missing = shmem_uffd_missing_mask(folio, index); + + if (missing & present) + mm_subpage_shmem_set_missing(folio_page(folio, index - folio->index), + missing & ~present); +} + +void shmem_uffd_mark_mapped(struct folio *folio, unsigned long offset, + unsigned long length) +{ + struct address_space *mapping = folio_mapping(folio); + + if (!mapping || !shmem_mapping(mapping) || + !(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return; + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(offset + length > folio_size(folio), folio); + while (length) { + unsigned int low = offset & ~PAGE_MASK; + unsigned long size = min(length, PAGE_SIZE - low); + unsigned long mask = GENMASK((low + size - 1) / SZ_4K, low / SZ_4K); + + shmem_uffd_mark_present(folio, folio->index + (offset >> PAGE_SHIFT), mask); + offset += size; + length -= size; + } +} + +void shmem_uffd_forget(struct address_space *mapping, pgoff_t start, + unsigned long nr) +{ + struct xarray *xa; + unsigned long index; + void *entry; + + if (!shmem_mapping(mapping)) + return; + lockdep_assert_held(&mapping->i_pages.xa_lock); + xa = &SHMEM_I(mapping->host)->uffd_missing; + if (xa_empty(xa)) + return; + xa_lock(xa); + xa_for_each_range(xa, index, entry, start, start + nr - 1) + __xa_erase(xa, index); + xa_unlock(xa); +} + +/* New cache identity or swap-in: initialize every native page before unlock. */ +static void shmem_uffd_cache_init(struct folio *folio, bool from_swap) +{ + struct address_space *mapping = folio->mapping; + struct xarray *xa = &SHMEM_I(mapping->host)->uffd_missing; + unsigned long i; + + lockdep_assert_held(&mapping->i_pages.xa_lock); + for (i = 0; i < folio_nr_pages(folio); i++) { + void *entry = from_swap ? xa_load(xa, folio->index + i) : NULL; + + mm_subpage_shmem_set_missing(folio_page(folio, i), + entry ? xa_to_value(entry) : 0); + } + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); +} + +/* + * Reserve persistence before committing swap. On failure the cache folio and + * its resident mask stay intact; reclaim may retry once memory is available. + */ +static int shmem_uffd_swap_prepare(struct folio *folio) +{ + struct address_space *mapping = folio->mapping; + struct xarray *xa = &SHMEM_I(mapping->host)->uffd_missing; + struct mem_cgroup *memcg, *saved; + unsigned long i; + int err = 0; + + if (!(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return 0; + /* Reclaim may run outside the backing folio's charge domain. */ + rcu_read_lock(); + do { + memcg = folio_memcg(folio) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + for (i = 0; i < folio_nr_pages(folio); i++) { + unsigned long mask = mm_subpage_shmem_missing(folio_page(folio, i)); + + if (!mask) + continue; + err = xa_err(xa_store(xa, folio->index + i, xa_mk_value(mask), + GFP_NOWAIT | __GFP_ACCOUNT | __GFP_NOWARN)); + if (err) + break; + } + set_active_memcg(saved); + mem_cgroup_put(memcg); + if (err) { + xa_lock_irq(&mapping->i_pages); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); + xa_unlock_irq(&mapping->i_pages); + } + return err; +} + +void shmem_uffd_copy_page(struct page *dst, struct page *src) +{ + mm_subpage_shmem_set_missing(dst, src ? mm_subpage_shmem_missing(src) : 0); +} + +void shmem_uffd_migrate(struct folio *dst, struct folio *src) +{ + unsigned long i; + + if (!shmem_mapping(src->mapping)) + return; + VM_BUG_ON_FOLIO(!folio_test_locked(src), src); + VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst); + for (i = 0; i < folio_nr_pages(src); i++) + shmem_uffd_copy_page(folio_page(dst, i), folio_page(src, i)); +} + +/* Only fully removed 4K units become holes; sub-unit edge bytes stay valid. */ +static void shmem_uffd_partial_hole(struct folio *folio, loff_t start, uoff_t end) +{ + loff_t first = max(start, folio_pos(folio)); + u64 last = min_t(u64, end, folio_next_pos(folio) - 1); + unsigned long offset, length; + bool pinned; + + if (first >= last || + (first == folio_pos(folio) && last == folio_next_pos(folio) - 1)) + return; + first = round_up(first, SZ_4K); + last = round_down(last + 1, SZ_4K); + if (first >= last) + return; + shmem_uffd_enable(folio->mapping->host); + /* Revoke covering coarse aliases before their holes can be refilled. */ + unmap_mapping_subpage_hole(folio->mapping, first, last); + /* Pair with fast GUP's pin-before-PTE-recheck barrier. */ + smp_mb(); + pinned = folio_maybe_dma_pinned(folio); + offset = first - folio_pos(folio); + length = last - first; + while (length) { + unsigned int low = offset & ~PAGE_MASK; + unsigned long nr = min(length, PAGE_SIZE - low); + struct page *page = folio_page(folio, offset >> PAGE_SHIFT); + unsigned long mask = shmem_uffd_mask(low, nr); + unsigned long missing = mm_subpage_shmem_missing(page); + + /* + * A pin survives PTE removal and can modify retained backing. + * Preserve native partial-hole semantics: zero it, but do not + * let a missing-page COPY overwrite a subsequent device write. + * Pins have folio granularity, so exposure is conservative when + * only a neighbouring range is pinned. Full-folio removal still + * detaches old pins from the file's new cache identity. + */ + mm_subpage_shmem_set_missing(page, pinned ? missing & ~mask : missing | mask); + offset += nr; + length -= nr; + } +} +#else +static void shmem_uffd_cache_init(struct folio *folio, bool from_swap) { } +static int shmem_uffd_swap_prepare(struct folio *folio) { return 0; } +static void shmem_uffd_partial_hole(struct folio *folio, loff_t start, uoff_t end) { } +#endif + /* * Somewhat like filemap_add_folio, but error if expected item has gone. */ @@ -922,6 +1185,7 @@ int shmem_add_to_page_cache(struct folio *folio, xas_store(&xas, folio); if (xas_error(&xas)) goto unlock; + shmem_uffd_cache_init(folio, expected != NULL); shmem_update_stats(folio, nr); mapping->nrpages += nr; unlock: @@ -976,8 +1240,10 @@ static long shmem_free_swap(struct address_space *mapping, base = round_down(xas.xa_index, nr_pages); if (base < index || base + nr_pages - 1 > end) nr_pages = 0; - else + else { + shmem_uffd_forget(mapping, base, nr_pages); xas_store(&xas, NULL); + } } xas_unlock_irq(&xas); @@ -1020,6 +1286,39 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping, return swapped << PAGE_SHIFT; } +/* Byte-clipped variant for mappings smaller than a native cache page. */ +unsigned long shmem_partial_swap_usage_bytes(struct address_space *mapping, + u64 start, u64 end) +{ + XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT); + void *entry; + unsigned long swapped = 0, last; + + if (start >= end) + return 0; + last = (end - 1) >> PAGE_SHIFT; + rcu_read_lock(); + xas_for_each(&xas, entry, last) { + if (xas_retry(&xas, entry)) + continue; + if (xa_is_value(entry)) { + unsigned int order = xas_get_order(&xas); + u64 first = ((u64)xas.xa_index & ~((1UL << order) - 1)) << PAGE_SHIFT; + u64 stop = first + ((u64)PAGE_SIZE << order); + + swapped += min(stop, end) - max(first, start); + } + if (xas.xa_index == last) + break; + if (need_resched()) { + xas_pause(&xas); + cond_resched_rcu(); + } + } + rcu_read_unlock(); + return swapped; +} + /* * Determine (in bytes) how many of the shmem object's pages mapped by the * given vma is swapped out. @@ -1044,6 +1343,12 @@ unsigned long shmem_swap_usage(struct vm_area_struct *vma) */ if (!swapped) return 0; + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + u64 start = ((u64)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); + + return shmem_partial_swap_usage_bytes(mapping, start, + start + vma->vm_end - vma->vm_start); + } if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) return swapped << PAGE_SHIFT; @@ -1101,6 +1406,12 @@ static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index) return folio; } +static bool shmem_truncate_partial_folio(struct folio *folio, loff_t start, uoff_t end) +{ + shmem_uffd_partial_hole(folio, start, end); + return truncate_inode_partial_folio(folio, start, end); +} + /* * Remove range of pages and swap entries from page cache, and free them. * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. @@ -1164,7 +1475,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, if (folio) { same_folio = lend < folio_next_pos(folio); folio_mark_dirty(folio); - if (!truncate_inode_partial_folio(folio, lstart, lend)) { + if (!shmem_truncate_partial_folio(folio, lstart, lend)) { start = folio_next_index(folio); if (same_folio) end = folio->index; @@ -1178,7 +1489,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT); if (folio) { folio_mark_dirty(folio); - if (!truncate_inode_partial_folio(folio, lstart, lend)) + if (!shmem_truncate_partial_folio(folio, lstart, lend)) end = folio->index; folio_unlock(folio); folio_put(folio); @@ -1247,7 +1558,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, if (!folio_test_large(folio)) { truncate_inode_folio(mapping, folio); - } else if (truncate_inode_partial_folio(folio, lstart, lend)) { + } else if (shmem_truncate_partial_folio(folio, lstart, lend)) { /* * If we split a page, reset the loop so * that we pick up the new sub pages. @@ -1355,6 +1666,7 @@ static int shmem_setattr(struct mnt_idmap *idmap, } if (newsize <= oldsize) { loff_t holebegin = round_up(newsize, PAGE_SIZE); + unmap_mapping_subpage_tail(inode->i_mapping, newsize); if (oldsize > holebegin) unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); @@ -1362,6 +1674,7 @@ static int shmem_setattr(struct mnt_idmap *idmap, shmem_truncate_range(inode, newsize, (loff_t)-1); /* unmap again to remove racily COWed private pages */ + unmap_mapping_subpage_tail(inode->i_mapping, newsize); if (oldsize > holebegin) unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); @@ -1677,6 +1990,8 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, folio_mark_uptodate(folio); } + if (shmem_uffd_swap_prepare(folio)) + goto redirty; if (!folio_alloc_swap(folio)) { bool first_swapped = shmem_recalc_inode(inode, 0, nr_pages); int error; @@ -1730,6 +2045,10 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, swap_cache_del_folio(folio); goto redirty; } + /* Allocation failed: no swap entry owns the reserved inode metadata. */ + xa_lock_irq(&mapping->i_pages); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); + xa_unlock_irq(&mapping->i_pages); if (nr_pages > 1) goto try_split; redirty: @@ -1934,6 +2253,23 @@ static struct folio *shmem_alloc_folio(gfp_t gfp, int order, return folio; } +/* Allocate unexposed promotion backing with the inode's zone/NUMA policy. */ +struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, struct inode *inode, + pgoff_t index, unsigned int order) +{ + gfp_t gfp = limit_gfp_mask(GFP_TRANSHUGE, mapping_gfp_mask(inode->i_mapping)); + struct folio *folio; + + folio = shmem_alloc_folio(gfp, order, SHMEM_I(inode), index); + if (!folio) + return ERR_PTR(-ENOMEM); + if (mem_cgroup_charge(folio, mm, gfp)) { + folio_put(folio); + return ERR_PTR(-ENOMEM); + } + return folio; +} + static struct folio *shmem_alloc_and_add_folio(struct vm_fault *vmf, gfp_t gfp, struct inode *inode, pgoff_t index, struct mm_struct *fault_mm, unsigned long orders) @@ -2484,13 +2820,36 @@ static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index, fault_mm = vma ? vma->vm_mm : NULL; folio = filemap_get_entry(inode->i_mapping, index); - if (folio && vma && userfaultfd_minor(vma)) { + if (folio && vma && userfaultfd_minor(vma) && + !(READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { if (!xa_is_value(folio)) folio_put(folio); *fault_type = handle_userfault(vmf, VM_UFFD_MINOR); return 0; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + if (xa_is_value(folio) && vma && userfaultfd_armed(vma) && + (READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { + unsigned long reason; + void *partial; + + /* Cache identity and its persisted mask must describe the same entry. */ + xa_lock_irq(&inode->i_mapping->i_pages); + if (xa_load(&inode->i_mapping->i_pages, index) != folio) { + xa_unlock_irq(&inode->i_mapping->i_pages); + goto repeat; + } + partial = xa_load(&SHMEM_I(inode)->uffd_missing, index); + reason = shmem_uffd_reason(vmf, partial ? xa_to_value(partial) : 0); + xa_unlock_irq(&inode->i_mapping->i_pages); + if (reason) { + *fault_type = handle_userfault(vmf, reason); + return 0; + } + } +#endif + if (xa_is_value(folio)) { error = shmem_swapin_folio(inode, index, &folio, sgp, gfp, vma, fault_type); @@ -2510,6 +2869,20 @@ static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index, folio_put(folio); goto repeat; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + if (vma && userfaultfd_armed(vma) && + (READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { + unsigned long reason = shmem_uffd_reason(vmf, + shmem_uffd_missing_mask(folio, index)); + + if (reason) { + folio_unlock(folio); + folio_put(folio); + *fault_type = handle_userfault(vmf, reason); + return 0; + } + } +#endif if (sgp == SGP_WRITE) folio_mark_accessed(folio); if (folio_test_uptodate(folio)) @@ -2748,15 +3121,33 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) struct folio *folio = NULL; vm_fault_t ret = 0; int err; +#ifdef CONFIG_MM_SUBPAGE + bool promoted = false; + unsigned long user_size = mm_page_size(vmf->vma->vm_mm); + + /* Missing coarse PTEs require a blocking caller to prepare backing. */ + if (user_size > PAGE_SIZE && (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) + return VM_FAULT_RETRY; +#endif +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) + return VM_FAULT_SIGBUS; +#endif + +#ifdef CONFIG_MM_SUBPAGE +retry: +#endif /* * Trinity finds that probing a hole which tmpfs is punching can * prevent the hole-punch from ever completing: noted in i_private. */ if (unlikely(inode->i_private)) { - ret = shmem_falloc_wait(vmf, inode); - if (ret) - return ret; + vm_fault_t wait = shmem_falloc_wait(vmf, inode); + + if (wait) + return ret | wait; } WARN_ON_ONCE(vmf->page != NULL); @@ -2765,6 +3156,45 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) if (err) return vmf_error(err); if (folio) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) { + folio_unlock(folio); + folio_put(folio); + return VM_FAULT_SIGBUS; + } + if (user_size > PAGE_SIZE) { + struct vm_page_offset pos = vma_page_offset_at(vmf->vma, vmf->address); + unsigned long nr = user_size >> PAGE_SHIFT; + + if (pos.offset || !IS_ALIGNED(pos.index, nr)) { + folio_unlock(folio); + folio_put(folio); + return VM_FAULT_SIGBUS; + } + if (folio_size(folio) < user_size) { + folio_unlock(folio); + folio_put(folio); + folio = NULL; + /* Bound work if splitting races a successful promotion. */ + if (promoted) { + cond_resched(); + return ret | VM_FAULT_NOPAGE; + } + err = collapse_file_user_page(vmf->vma->vm_mm, + vmf->vma->vm_file, pos.index); + if (err == -EAGAIN) { + cond_resched(); + return ret | VM_FAULT_NOPAGE; + } + if (err) + return vmf_error(err); + promoted = true; + goto retry; + } + } +#endif + vmf->page = folio_file_page(folio, vmf->pgoff); ret |= VM_FAULT_LOCKED; } @@ -3086,6 +3516,9 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap, memset(info, 0, (char *)inode - (char *)info); INIT_LIST_HEAD_RCU(&info->xattrs); spin_lock_init(&info->lock); +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + xa_init_flags(&info->uffd_missing, XA_FLAGS_ACCOUNT); +#endif atomic_set(&info->stop_eviction, 0); info->seals = F_SEAL_SEAL; info->flags = vma_flags_test(&flags, VMA_NORESERVE_BIT) @@ -3245,6 +3678,57 @@ static void shmem_mfill_filemap_remove(struct folio *folio, folio_unlock(folio); } +#ifdef CONFIG_MM_SUBPAGE +struct folio *shmem_uffd_prepare_folio(struct vm_area_struct *vma, + unsigned long address, bool *new) +{ + struct inode *inode = file_inode(vma->vm_file); + pgoff_t index = vma_page_offset_at(vma, address).index; + struct folio *folio; + int err; + + *new = false; + /* Must precede cache insertion: minor faults must wait for initialization. */ + shmem_uffd_enable(inode); +retry: + err = shmem_get_folio(inode, index, 0, &folio, SGP_NOALLOC); + if (!err) + return folio; + if (err != -ENOENT) + return ERR_PTR(err); + folio = shmem_mfill_folio_alloc(vma, address); + if (!folio) + return ERR_PTR(-ENOMEM); + clear_highpage(&folio->page); + flush_dcache_folio(folio); + __folio_mark_uptodate(folio); + err = shmem_mfill_filemap_add(folio, vma, address); + if (err) { + folio_put(folio); + if (err == -EEXIST) + goto retry; + return ERR_PTR(err); + } + err = shmem_uffd_set_missing(folio, index, SHMEM_UFFD_ALL_UNITS); + if (err) { + shmem_mfill_filemap_remove(folio, vma); + folio_put(folio); + return ERR_PTR(err); + } + *new = true; + return folio; +} + +void shmem_uffd_abort_folio(struct folio *folio, struct vm_area_struct *vma, bool new) +{ + if (new) + shmem_mfill_filemap_remove(folio, vma); + else + folio_unlock(folio); + folio_put(folio); +} +#endif + static struct folio *shmem_get_folio_noalloc(struct inode *inode, pgoff_t pgoff) { struct folio *folio; @@ -3331,6 +3815,7 @@ shmem_write_end(const struct kiocb *iocb, struct address_space *mapping, } folio_mark_uptodate(folio); } + shmem_uffd_mark_mapped(folio, offset_in_folio(folio, pos), copied); folio_mark_dirty(folio); folio_unlock(folio); folio_put(folio); @@ -5143,6 +5628,10 @@ static struct inode *shmem_alloc_inode(struct super_block *sb) info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL); if (!info) return NULL; +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + /* Also cover inode-init failure before __shmem_get_inode() runs. */ + xa_init_flags(&info->uffd_missing, XA_FLAGS_ACCOUNT); +#endif return &info->vfs_inode; } @@ -5155,6 +5644,9 @@ static void shmem_free_in_core_inode(struct inode *inode) static void shmem_destroy_inode(struct inode *inode) { +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + xa_destroy(&SHMEM_I(inode)->uffd_missing); +#endif if (S_ISREG(inode->i_mode)) mpol_free_shared_policy(&SHMEM_I(inode)->policy); if (S_ISDIR(inode->i_mode)) @@ -5972,6 +6464,8 @@ struct folio *shmem_read_folio_gfp(struct address_space *mapping, if (error) return ERR_PTR(error); + /* A kernel caller receives and may modify the complete native folio. */ + shmem_uffd_mark_mapped(folio, 0, folio_size(folio)); folio_unlock(folio); return folio; #else diff --git a/mm/swap.h b/mm/swap.h index a77016f2423b9..5d064cab36f4c 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -10,9 +10,12 @@ extern int page_cluster; #ifdef CONFIG_THP_SWAP #define SWAPFILE_CLUSTER HPAGE_PMD_NR -#define swap_entry_order(order) (order) #else #define SWAPFILE_CLUSTER 256 +#endif +#if defined(CONFIG_THP_SWAP) || defined(CONFIG_MM_SUBPAGE) +#define swap_entry_order(order) (order) +#else #define swap_entry_order(order) 0 #endif @@ -38,6 +41,9 @@ struct swap_cluster_info { u8 order; atomic_long_t __rcu *table; /* Swap table entries, see mm/swap_table.h */ unsigned int *extend_table; /* For large swap count, protected by ci->lock */ +#ifdef CONFIG_MM_SUBPAGE + unsigned int *subpage_counts; /* Per-quarter PTE refs, protected by ci->lock */ +#endif struct list_head list; }; @@ -207,6 +213,8 @@ extern int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp); */ int folio_alloc_swap(struct folio *folio); int folio_dup_swap(struct folio *folio, struct page *subpage); +int folio_dup_swap_range(struct folio *folio, struct page *page, unsigned int nr); +void folio_put_swap_range(struct folio *folio, struct page *page, unsigned int nr); void folio_put_swap(struct folio *folio, struct page *subpage); /* For internal use */ @@ -216,6 +224,7 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si, /* linux/mm/page_io.c */ int sio_pool_init(void); +int swap_extent_io_init(void); struct swap_iocb; void swap_read_folio(struct folio *folio, struct swap_iocb **plug); void __swap_read_unplug(struct swap_iocb *plug); @@ -399,6 +408,17 @@ static inline void folio_put_swap(struct folio *folio, struct page *page) { } +static inline int folio_dup_swap_range(struct folio *folio, struct page *page, + unsigned int nr) +{ + return -EINVAL; +} + +static inline void folio_put_swap_range(struct folio *folio, struct page *page, + unsigned int nr) +{ +} + static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug) { } diff --git a/mm/swap_state.c b/mm/swap_state.c index 1415a5c54a43b..db1285bc1880d 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -427,6 +427,10 @@ void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, if (unlikely(folio_test_large(folio))) return; + /* Small-granule PTEs currently use native swap-cluster readahead. */ + if (vma && mm_page_size(vma->vm_mm) != PAGE_SIZE) + vma = NULL; + readahead = folio_test_clear_readahead(folio); if (vma && vma_ra) { unsigned long ra_val; @@ -896,7 +900,7 @@ struct folio *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, struct folio *folio; mpol = get_vma_policy(vmf->vma, vmf->address, 0, &ilx); - folio = swap_use_vma_readahead() ? + folio = swap_use_vma_readahead() && mm_page_size(vmf->vma->vm_mm) == PAGE_SIZE ? swap_vma_readahead(entry, gfp_mask, mpol, ilx, vmf) : swap_cluster_readahead(entry, gfp_mask, mpol, ilx); mpol_cond_put(mpol); diff --git a/mm/swapfile.c b/mm/swapfile.c index 74a1e324449dc..bb6108188d8e0 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -329,17 +330,22 @@ offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset) BUG(); } +/* The extent tree is immutable while the swap entry is owned. */ +sector_t swap_extent_sector(swp_entry_t entry, unsigned long *nr_pages) +{ + struct swap_info_struct *sis = __swap_entry_to_info(entry); + pgoff_t offset = swp_offset(entry); + struct swap_extent *se = offset_to_swap_extent(sis, offset); + + *nr_pages = min(*nr_pages, se->start_page + se->nr_pages - offset); + return (se->start_block + offset - se->start_page) << (PAGE_SHIFT - 9); +} + sector_t swap_folio_sector(struct folio *folio) { - struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); - struct swap_extent *se; - sector_t sector; - pgoff_t offset; + unsigned long nr = folio_nr_pages(folio); - offset = swp_offset(folio->swap); - se = offset_to_swap_extent(sis, offset); - sector = se->start_block + (offset - se->start_page); - return sector << (PAGE_SHIFT - 9); + return swap_extent_sector(folio->swap, &nr); } /* @@ -479,6 +485,16 @@ static void swap_cluster_free_table(struct swap_cluster_info *ci) lockdep_assert_held(&ci->lock); table = (void *)rcu_dereference_protected(ci->table, true); rcu_assign_pointer(ci->table, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (ci->subpage_counts) { + unsigned int i; + + for (i = 0; i < SWAPFILE_CLUSTER * (PAGE_SIZE / MM_SUBPAGE_SIZE); i++) + VM_WARN_ON_ONCE(ci->subpage_counts[i]); + kfree(ci->subpage_counts); + ci->subpage_counts = NULL; + } +#endif swap_table_free(table); } @@ -1067,6 +1083,19 @@ static void swap_reclaim_work(struct work_struct *work) swap_reclaim_full_clusters(si, true); } +static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio *folio) +{ + unsigned int flags; + + if (!folio || !folio_order(folio)) + return true; + flags = READ_ONCE(si->flags); + if (flags & SWP_BLKDEV) + return true; + /* Filesystem swap_rw implementations retain their existing contract. */ + return si->bdev && !(flags & SWP_FS_OPS) && folio_anon_min_order(folio); +} + /* * Try to allocate swap entries with specified order and try set a new * cluster for current CPU too. @@ -1078,11 +1107,7 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, unsigned int order = likely(folio) ? folio_order(folio) : 0; unsigned int offset = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID; - /* - * Swapfile is not block device so unable - * to allocate large entries. - */ - if (order && !(si->flags & SWP_BLKDEV)) + if (!swap_folio_alloc_allowed(si, folio)) return 0; if (!(si->flags & SWP_SOLIDSTATE)) { @@ -1361,6 +1386,10 @@ static bool swap_alloc_fast(struct folio *folio) offset = this_cpu_read(percpu_swap_cluster.offset[order]); if (!si || !offset || !get_swap_device_info(si)) return false; + if (!swap_folio_alloc_allowed(si, folio)) { + put_swap_device(si); + return false; + } ci = swap_cluster_lock(si, offset); if (cluster_is_usable(ci, order)) { @@ -1543,7 +1572,7 @@ static void __swap_cluster_put_entry(struct swap_cluster_info *ci, */ static void swap_put_entries_cluster(struct swap_info_struct *si, pgoff_t offset, int nr, - bool reclaim_cache) + bool reclaim_cache, int subpage) { struct swap_cluster_info *ci; unsigned int ci_off, ci_end; @@ -1556,6 +1585,17 @@ static void swap_put_entries_cluster(struct swap_info_struct *si, ci = swap_cluster_lock(si, offset); ci_off = offset % SWAPFILE_CLUSTER; ci_end = ci_off + nr; +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) { + unsigned int index = ci_off * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage; + + if (WARN_ON_ONCE(nr != 1 || !ci->subpage_counts || !ci->subpage_counts[index])) { + swap_cluster_unlock(ci); + return; + } + ci->subpage_counts[index]--; + } +#endif do { swp_tb = __swap_table_get(ci, ci_off); if (swp_tb_get_count(swp_tb) == 1) { @@ -1649,7 +1689,7 @@ static int __swap_cluster_dup_entry(struct swap_cluster_info *ci, * and failed to allocate an extended table, -EINVAL if any entry is bad entry. */ static int swap_dup_entries_cluster(struct swap_info_struct *si, - pgoff_t offset, int nr) + pgoff_t offset, int nr, int subpage) { int err; struct swap_cluster_info *ci; @@ -1659,6 +1699,25 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, ci_end = ci_start + nr; ci_off = ci_start; ci = swap_cluster_lock(si, offset); +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) { + unsigned int index = ci_start * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage; + + VM_BUG_ON(nr != 1); + if (!ci->subpage_counts) + ci->subpage_counts = kcalloc(SWAPFILE_CLUSTER, + (PAGE_SIZE / MM_SUBPAGE_SIZE) * sizeof(*ci->subpage_counts), + GFP_ATOMIC | __GFP_NOWARN); + if (!ci->subpage_counts) { + err = -ENOMEM; + goto failed; + } + if (ci->subpage_counts[index] == UINT_MAX) { + err = -EOVERFLOW; + goto failed; + } + } +#endif restart: do { err = __swap_cluster_dup_entry(ci, ci_off); @@ -1673,6 +1732,10 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, goto failed; } } while (++ci_off < ci_end); +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) + ci->subpage_counts[ci_start * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage]++; +#endif swap_cluster_unlock(ci); return 0; failed: @@ -1683,6 +1746,132 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, return err; } +#ifdef CONFIG_MM_SUBPAGE +static bool swap_subpage_offset_valid(unsigned int offset) +{ + return offset < PAGE_SIZE && IS_ALIGNED(offset, MM_SUBPAGE_SIZE); +} + +int swap_subpage_dup(swp_entry_t entry, unsigned int offset) +{ + struct swap_info_struct *si = swap_entry_to_info(entry); + + if (WARN_ON_ONCE(!si || !swap_subpage_offset_valid(offset))) + return -EINVAL; + /* A locked swap-cache folio also permits the initial zero-to-one ref. */ + return swap_dup_entries_cluster(si, swp_offset(entry), 1, offset >> MM_SUBPAGE_SHIFT); +} + +void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache) +{ + struct swap_info_struct *si; + + if (WARN_ON_ONCE(!swap_subpage_offset_valid(offset))) + return; + si = get_swap_device(entry); + if (WARN_ON_ONCE(!si)) + return; + swap_put_entries_cluster(si, swp_offset(entry), 1, reclaim_cache, + offset >> MM_SUBPAGE_SHIFT); + put_swap_device(si); +} + +unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset) +{ + struct swap_cluster_info *ci; + struct swap_info_struct *si; + unsigned int count = 0; + + if (!swap_subpage_offset_valid(offset)) + return 0; + si = get_swap_device(entry); + if (!si) + return 0; + ci = swap_cluster_lock(si, swp_offset(entry)); + if (ci->subpage_counts) + count = ci->subpage_counts[swp_cluster_offset(entry) * + (PAGE_SIZE / MM_SUBPAGE_SIZE) + (offset >> MM_SUBPAGE_SHIFT)]; + swap_cluster_unlock(ci); + put_swap_device(si); + return count; +} +#endif + +/* + * Optional THPs can split if their order-specific cluster lists have no room. + * A mandatory base user page cannot: find a free aligned group without keeping + * the per-CPU or device allocation lock held over a device-wide scan. + */ +static void swap_alloc_required_from_device(struct swap_info_struct *si, + struct folio *folio) +{ + unsigned int order = folio_order(folio), nr = folio_nr_pages(folio); + unsigned long start; + + if (!swap_folio_alloc_allowed(si, folio)) + return; + for (start = 0; start < si->max; start += SWAPFILE_CLUSTER) { + struct swap_cluster_info *ci = __swap_offset_to_cluster(si, start); + unsigned long offset, end = min(start + SWAPFILE_CLUSTER, si->max); + bool found = false; + + if (!(READ_ONCE(si->flags) & SWP_WRITEOK)) + break; + if (!spin_trylock(&ci->lock)) + goto next; + /* Order zero's usability test permits occupied clusters of any order. */ + if (!cluster_is_usable(ci, 0) || ci->count + nr > SWAPFILE_CLUSTER) + goto unlock; + for (offset = start; offset + nr <= end; offset += nr) { + bool need_reclaim = false; + + /* Only genuinely free slots qualify; retain every existing owner. */ + if (!cluster_scan_range(si, ci, offset, nr, &need_reclaim) || need_reclaim) + continue; + if (!__swap_cluster_alloc_entries(si, ci, folio, offset - start)) + break; + ci->order = order; + if (ci->count < SWAPFILE_CLUSTER && ci->flags == CLUSTER_FLAG_FRAG) { + /* Same flag, different order: relocate the list explicitly. */ + spin_lock(&si->lock); + list_move_tail(&ci->list, &si->frag_clusters[order]); + spin_unlock(&si->lock); + } else { + relocate_cluster(si, ci); + } + found = true; + break; + } +unlock: + swap_cluster_unlock(ci); + if (found) + return; +next: + cond_resched(); + } +} + +static void swap_alloc_required(struct folio *folio) +{ + struct swap_info_struct *devices[MAX_SWAPFILES], *si; + unsigned int count = 0, i; + + if (get_nr_swap_pages() < folio_nr_pages(folio)) + return; + /* Pin a bounded snapshot in the allocator's existing priority order. */ + spin_lock(&swap_avail_lock); + plist_for_each_entry(si, &swap_avail_head, avail_list) { + if (get_swap_device_info(si)) + devices[count++] = si; + } + spin_unlock(&swap_avail_lock); + for (i = 0; i < count; i++) { + if (!folio_test_swapcache(folio)) + swap_alloc_required_from_device(devices[i], folio); + put_swap_device(devices[i]); + } +} + /** * folio_alloc_swap - allocate swap space for a folio * @folio: folio we want to move to swap @@ -1697,16 +1886,17 @@ int folio_alloc_swap(struct folio *folio) { unsigned int order = folio_order(folio); unsigned int size = 1 << order; + bool mandatory = order && folio_anon_min_order(folio); VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); VM_BUG_ON_FOLIO(!folio_test_uptodate(folio), folio); if (order) { /* - * Reject large allocation when THP_SWAP is disabled, - * the caller should split the folio and try again. + * Optional THP backing may split when THP_SWAP is disabled. + * A larger base user page must retain contiguous swap ownership. */ - if (!IS_ENABLED(CONFIG_THP_SWAP)) + if (!IS_ENABLED(CONFIG_THP_SWAP) && !mandatory) return -EAGAIN; /* @@ -1725,6 +1915,9 @@ int folio_alloc_swap(struct folio *folio) swap_alloc_slow(folio); local_unlock(&percpu_swap_cluster.lock); + if (mandatory && unlikely(!folio_test_swapcache(folio))) + swap_alloc_required(folio); + if (!order && unlikely(!folio_test_swapcache(folio))) { if (swap_sync_discard()) goto again; @@ -1757,6 +1950,24 @@ int folio_alloc_swap(struct folio *folio) * swap_put_entries_direct on its swap entry before this helper returns, or * the swap count may underflow. */ +int folio_dup_swap_range(struct folio *folio, struct page *page, unsigned int nr) +{ + swp_entry_t entry = page_swap_entry(page); + + VM_BUG_ON_FOLIO(!folio_test_locked(folio) || !folio_test_swapcache(folio), folio); + VM_BUG_ON_FOLIO(!nr || folio_page_idx(folio, page) + nr > folio_nr_pages(folio), folio); + return swap_dup_entries_cluster(swap_entry_to_info(entry), swp_offset(entry), nr, -1); +} + +void folio_put_swap_range(struct folio *folio, struct page *page, unsigned int nr) +{ + swp_entry_t entry = page_swap_entry(page); + + VM_BUG_ON_FOLIO(!folio_test_locked(folio) || !folio_test_swapcache(folio), folio); + VM_BUG_ON_FOLIO(!nr || folio_page_idx(folio, page) + nr > folio_nr_pages(folio), folio); + swap_put_entries_cluster(__swap_entry_to_info(entry), swp_offset(entry), nr, false, -1); +} + int folio_dup_swap(struct folio *folio, struct page *subpage) { swp_entry_t entry = folio->swap; @@ -1771,7 +1982,7 @@ int folio_dup_swap(struct folio *folio, struct page *subpage) } return swap_dup_entries_cluster(swap_entry_to_info(entry), - swp_offset(entry), nr_pages); + swp_offset(entry), nr_pages, -1); } /** @@ -1797,7 +2008,7 @@ void folio_put_swap(struct folio *folio, struct page *subpage) nr_pages = 1; } - swap_put_entries_cluster(si, swp_offset(entry), nr_pages, false); + swap_put_entries_cluster(si, swp_offset(entry), nr_pages, false, -1); } /* @@ -2070,7 +2281,7 @@ void swap_put_entries_direct(swp_entry_t entry, int nr) offset = start_offset; do { cluster_end = min(round_up(offset + 1, SWAPFILE_CLUSTER), end_offset); - swap_put_entries_cluster(si, offset, cluster_end - offset, true); + swap_put_entries_cluster(si, offset, cluster_end - offset, true, -1); offset = cluster_end; } while (offset < end_offset); out: @@ -2239,6 +2450,72 @@ static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte) return pte_same(pte_swp_clear_flags(pte), swp_pte); } +#ifdef CONFIG_MM_SUBPAGE +/* Caller holds the swap-cache folio lock and the VMA's mmap read lock. */ +static int unuse_subpage(struct vm_area_struct *vma, pmd_t *pmd, + unsigned long addr, swp_entry_t entry, struct folio *folio) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned int offset = vma_page_offset_at(vma, addr).offset; + struct mm_subpage *slot = NULL; + spinlock_t *ptl; + pte_t *ptep, old, pte; + bool poisoned = PageHWPoison(&folio->page) || !folio_test_uptodate(folio); + int ret = 0; + + if (folio_order(folio) || folio_test_ksm(folio)) + return -EOPNOTSUPP; + if (!poisoned) { + arch_swap_restore(folio_swap(entry, folio), folio); + /* Allocate metadata before taking the PTE spinlock. */ + slot = mm_subpage_restore_granule(folio, offset, mm_page_shift(mm), GFP_KERNEL); + if (IS_ERR(slot)) + return PTR_ERR(slot) == -EAGAIN ? 0 : PTR_ERR(slot); + } + ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!ptep) + goto out; + old = ptep_get(ptep); + if (softleaf_from_pte(old).val != entry.val) + goto unlock; + if (pte_swp_subpage_offset(old) != offset) { + ret = -EINVAL; + goto unlock; + } + if (poisoned) { + pte = swp_entry_to_pte(PageHWPoison(&folio->page) ? + make_hwpoison_entry(&folio->page) : make_poisoned_swp_entry()); + } else { + ret = mm_subpage_add_anon_rmap(slot, vma, addr); + if (ret) + goto unlock; + /* unuse_pte_range() waited for writeback and holds the folio lock. */ + VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio); + if (pte_swp_exclusive(old)) + mm_subpage_set_exclusive(slot); + else if (swap_subpage_count(entry, offset) == 1) + mm_subpage_try_reuse_swap(slot); + pte = pte_mkold(phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot)); + if (pte_swp_soft_dirty(old)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(old)) + pte = pte_mkuffd_wp(pte); + inc_mm_counter(mm, MM_ANONPAGES); + slot = NULL; + ret = 1; + } + dec_mm_counter(mm, MM_SWAPENTS); + set_pte_at(mm, addr, ptep, pte); + swap_subpage_put(entry, offset, false); +unlock: + pte_unmap_unlock(ptep, ptl); +out: + if (slot) + mm_subpage_put(slot); + return ret; +} +#endif + /* * No need to decide whether this PTE shares the swap entry with others, * just let do_wp_page work it out if a write is requested later - to @@ -2261,6 +2538,11 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, if (!folio_matches_swap_entry(folio, entry)) return 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return unuse_subpage(vma, pmd, addr, entry, folio); +#endif + swapcache = folio; folio = ksm_might_need_to_copy(folio, vma, addr); if (unlikely(!folio)) @@ -2366,7 +2648,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, pte_t ptent; if (!pte++) { - pte = pte_offset_map(pmd, addr); + pte = pte_offset_map_mm(vma->vm_mm, pmd, addr); if (!pte) break; } @@ -2382,6 +2664,22 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, pte_unmap(pte); pte = NULL; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) { + struct vm_fault vmf = { + .vma = vma, .address = addr, .real_address = addr, + .pmd = pmd, .orig_pte = ptent, + }; + vm_fault_t fault = do_swap_coarse_page(&vmf); + + if (fault & VM_FAULT_OOM) + return -ENOMEM; + if (fault & VM_FAULT_ERROR) + return -EIO; + continue; + } +#endif + folio = swap_cache_get_folio(entry); if (!folio) { struct vm_fault vmf = { @@ -2414,7 +2712,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, folio_free_swap(folio); folio_unlock(folio); folio_put(folio); - } while (addr += PAGE_SIZE, addr != end); + } while (addr += mm_page_size(vma->vm_mm), addr != end); if (pte) pte_unmap(pte); @@ -2429,10 +2727,10 @@ static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, unsigned long next; int ret; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(vma->vm_mm, pud, addr); do { cond_resched(); - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(vma->vm_mm, addr, end); ret = unuse_pte_range(vma, pmd, addr, next, type); if (ret) return ret; @@ -2448,9 +2746,9 @@ static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, unsigned long next; int ret; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(vma->vm_mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(vma->vm_mm, addr, end); if (pud_none_or_clear_bad(pud)) continue; ret = unuse_pmd_range(vma, pud, addr, next, type); @@ -2468,10 +2766,10 @@ static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, unsigned long next; int ret; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(vma->vm_mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(vma->vm_mm, addr, end); + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4d)) continue; ret = unuse_pud_range(vma, p4d, addr, next, type); if (ret) @@ -2491,7 +2789,7 @@ static int unuse_vma(struct vm_area_struct *vma, unsigned int type) pgd = pgd_offset(vma->vm_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(vma->vm_mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; ret = unuse_p4d_range(vma, pgd, addr, next, type); @@ -2798,10 +3096,15 @@ static int setup_swap_extents(struct swap_info_struct *sis, destroy_swap_extents(sis, swap_file); return -ENOMEM; } - return ret; + } else { + ret = generic_swapfile_activate(sis, swap_file, span); } - - return generic_swapfile_activate(sis, swap_file, span); + if (ret >= 0 && sis->bdev && + !(sis->flags & SWP_FS_OPS) && swap_extent_io_init()) { + destroy_swap_extents(sis, swap_file); + return -ENOMEM; + } + return ret; } static void _enable_swap_info(struct swap_info_struct *si) @@ -3708,7 +4011,7 @@ void si_swapinfo(struct sysinfo *val) * Also the swap entry must have a count >= 1. Otherwise folio_dup_swap should * be used. */ -int swap_dup_entry_direct(swp_entry_t entry) +int swap_dup_entries_direct(swp_entry_t entry, unsigned int nr) { struct swap_info_struct *si; @@ -3725,7 +4028,14 @@ int swap_dup_entry_direct(swp_entry_t entry) */ VM_WARN_ON_ONCE(!swap_entry_swapped(si, entry)); - return swap_dup_entries_cluster(si, swp_offset(entry), 1); + if (WARN_ON_ONCE(!nr || swp_cluster_offset(entry) + nr > SWAPFILE_CLUSTER)) + return -EINVAL; + return swap_dup_entries_cluster(si, swp_offset(entry), nr, -1); +} + +int swap_dup_entry_direct(swp_entry_t entry) +{ + return swap_dup_entries_direct(entry, 1); } #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) diff --git a/mm/truncate.c b/mm/truncate.c index 12cc89f89afcf..ca1830b647a97 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -794,8 +794,10 @@ void truncate_pagecache(struct inode *inode, loff_t newsize) * truncate_inode_pages finishes, hence the second * unmap_mapping_range call must be made for correctness. */ + unmap_mapping_subpage_tail(mapping, newsize); unmap_mapping_range(mapping, holebegin, 0, 1); truncate_inode_pages(mapping, newsize); + unmap_mapping_subpage_tail(mapping, newsize); unmap_mapping_range(mapping, holebegin, 0, 1); } EXPORT_SYMBOL(truncate_pagecache); diff --git a/mm/user-subpage-internal.h b/mm/user-subpage-internal.h new file mode 100644 index 0000000000000..bddb5eeb2ca7b --- /dev/null +++ b/mm/user-subpage-internal.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _MM_USER_SUBPAGE_INTERNAL_H +#define _MM_USER_SUBPAGE_INTERNAL_H + +#include +#include +#include +#include + +struct mm_subpage_owner; +struct rmap_walk_control; +struct anon_vma; + +struct mm_subpage { + struct mm_subpage_owner *owner; + refcount_t refs; + atomic_t mapcount; + unsigned int index; + u8 shift; + unsigned int pins; + bool exclusive; + bool lazyfree; + bool rmap_bound; +}; + +/* Logical identity is independent of the physical slot after relocation. */ +struct mm_subpage_rmap { + struct anon_vma *root; + struct vm_page_offset pos; + unsigned int offset; +}; + +struct subpage_retired_root; +struct mm_subpage_move { + struct mm_subpage *slot; + struct mm_subpage_rmap map; + struct subpage_retired_root *retired; + bool swap; + bool root_changed; + bool committed; +}; + +/* Folio and both PTE locks stay held across prepare, commit and finish. */ +int mm_subpage_move_prepare(struct mm_subpage_move *move, struct mm_subpage *slot, + struct vm_area_struct *src, unsigned long src_address, + struct vm_area_struct *dst, unsigned long dst_address, bool swap); +/* Present source PTE must be cleared and its TLB invalidated before commit. */ +int mm_subpage_move_commit(struct mm_subpage_move *move); +void mm_subpage_move_finish(struct mm_subpage_move *move); + +int mm_subpage_bind_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address); +bool mm_subpage_match_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address); +/* Return -1 for ordinary linear rmap; otherwise return referenced identities. */ +int mm_subpage_snapshot_rmap(struct folio *folio, struct mm_subpage_rmap *maps); +bool mm_subpage_address_in_vma(const struct folio *folio, + const struct vm_area_struct *vma, unsigned long *address); +bool rmap_walk_subpages(struct folio *folio, struct rmap_walk_control *rwc, + bool locked); + +struct anon_vma *mm_subpage_anon_root(const struct mm_subpage *subpage); + +/* Folio lock held; called once before publishing anonymous folio mapping. */ +void mm_subpage_hold_anon_root(struct mm_subpage *subpage, struct anon_vma *root, + bool droppable); +bool mm_subpage_is_droppable(const struct mm_subpage *subpage); + +#endif diff --git a/mm/user-subpage.c b/mm/user-subpage.c new file mode 100644 index 0000000000000..6736b60eb214f --- /dev/null +++ b/mm/user-subpage.c @@ -0,0 +1,1667 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Reference-counted user data slots in native folios. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "user-subpage-internal.h" +#include "internal.h" + +#define SUBPAGES_PER_FOLIO (PAGE_SIZE / MM_SUBPAGE_SIZE) +static bool subpage_shift_valid(unsigned int shift) +{ + return shift >= MM_SUBPAGE_SHIFT && shift < PAGE_SHIFT; +} + +static unsigned long subpage_mask(unsigned int shift) +{ + return GENMASK((PAGE_SIZE >> shift) - 1, 0); +} + +static unsigned long offset_mask(unsigned int shift, unsigned int offset) +{ + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, 1U << shift)) + return 0; + return BIT(offset >> shift); +} + +struct mm_subpage_owner { + struct rcu_head rcu; + struct work_struct free_work; + struct mm_subpage_pool *pool; + struct folio *folio; + struct anon_vma *anon_root; + struct mm_subpage_owner *migration_target; + /* Allocated only when slots need independent logical identities. */ + struct mm_subpage_rmap *rmap; + struct list_head all; + struct list_head available; + unsigned long used; + unsigned long zeroed; + u8 shift; + bool detached; + bool migrating; + bool droppable; + struct mm_subpage slots[SUBPAGES_PER_FOLIO]; +}; + +struct mm_subpage_ext { + struct mm_subpage_owner __rcu *owner; + atomic_t file_maps[SUBPAGES_PER_FOLIO]; + atomic_t file_leaves; + /* Folio lock protects validity; fits the 16K/64K layout padding. */ + u16 shmem_missing; +}; +static_assert(SUBPAGES_PER_FOLIO == 1 || sizeof(struct mm_subpage_ext) == + ALIGN(sizeof(void *) + sizeof(atomic_t) * (SUBPAGES_PER_FOLIO + 1), + sizeof(void *))); +static_assert(SUBPAGES_PER_FOLIO <= sizeof(u16) * BITS_PER_BYTE); + +unsigned long mm_subpage_shmem_missing(struct page *page) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + unsigned long missing; + + /* A referenced native page keeps its initialized section online. */ + VM_BUG_ON(!ext); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + missing = subext->shmem_missing; + page_ext_put(ext); + return missing; +} + +void mm_subpage_shmem_set_missing(struct page *page, unsigned long missing) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + + VM_BUG_ON(!ext); + VM_BUG_ON(missing & ~GENMASK(SUBPAGES_PER_FOLIO - 1, 0)); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + subext->shmem_missing = missing; + page_ext_put(ext); +} + +static bool __init subpage_ext_needed(void) +{ + return true; +} + +struct page_ext_operations mm_subpage_ext_ops = { + .size = sizeof(struct mm_subpage_ext), + .need = subpage_ext_needed, +}; + +/* Only alternative PTE mappings contribute; native rmap counts stay unchanged. */ +static void subpage_file_map_update(phys_addr_t phys, unsigned long size, int delta) +{ + struct page_ext *ext; + struct mm_subpage_ext *subext; + unsigned int index, nr, i; + int count; + + VM_BUG_ON(!is_power_of_2(size) || size < MM_SUBPAGE_SIZE || + size >= PAGE_SIZE || !IS_ALIGNED(phys, size)); + ext = page_ext_from_phys(phys); + if (WARN_ON_ONCE(!ext)) + return; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + index = offset_in_page(phys) >> MM_SUBPAGE_SHIFT; + nr = size >> MM_SUBPAGE_SHIFT; + for (i = index; i < index + nr; i++) { + count = atomic_add_return(delta, &subext->file_maps[i]); + VM_WARN_ON(count < 0); + } + /* Rmap counts one mapping, regardless of its coverage in 4K units. */ + count = atomic_add_return(delta, &subext->file_leaves); + VM_WARN_ON(count < 0); + page_ext_put(ext); +} + +void __mm_subpage_file_map_add(phys_addr_t phys, unsigned long size) +{ + subpage_file_map_update(phys, size, 1); +} + +void __mm_subpage_file_map_del(phys_addr_t phys, unsigned long size) +{ + subpage_file_map_update(phys, size, -1); +} + +/* Snapshot for statistics only, never an ownership or pinning decision. */ +unsigned int mm_subpage_file_mapcounts(struct page *page, struct mm_subpage_mapcounts *counts) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + unsigned int i, leaves; + + if (WARN_ON_ONCE(!ext)) { + memset(counts, 0, sizeof(*counts)); + return 0; + } + subext = page_ext_data(ext, &mm_subpage_ext_ops); + for (i = 0; i < SUBPAGES_PER_FOLIO; i++) + counts->slots[i] = max(atomic_read(&subext->file_maps[i]), 0); + leaves = max(atomic_read(&subext->file_leaves), 0); + page_ext_put(ext); + return leaves; +} + +static int owner_bind(struct mm_subpage_owner *owner) +{ + struct page_ext *ext = page_ext_get(&owner->folio->page); + struct mm_subpage_ext *subext; + int ret; + + if (!ext) + return -EOPNOTSUPP; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + ret = cmpxchg(&subext->owner, NULL, owner) ? -EEXIST : 0; + page_ext_put(ext); + return ret; +} + +static void owner_unbind(struct mm_subpage_owner *owner) +{ + struct page_ext *ext = page_ext_get(&owner->folio->page); + struct mm_subpage_ext *subext; + + /* The allocated, referenced folio prevents its section going offline. */ + VM_BUG_ON(!ext); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + VM_BUG_ON(rcu_access_pointer(subext->owner) != owner); + rcu_assign_pointer(subext->owner, NULL); + page_ext_put(ext); +} + +struct mm_subpage *mm_subpage_get_from_phys(phys_addr_t phys) +{ + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + struct mm_subpage *slot = NULL; + struct page_ext *ext; + unsigned int index; + + if (!IS_ALIGNED(phys, MM_SUBPAGE_SIZE)) + return NULL; + ext = page_ext_from_phys(phys); + if (!ext) + return NULL; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + index = owner ? offset_in_page(phys) >> owner->shift : 0; + /* Keep the native allocation alive before publishing another slot ref. */ + if (owner && folio_try_get(owner->folio)) { + if (refcount_inc_not_zero_acquire(&owner->slots[index].refs)) + slot = &owner->slots[index]; + else + folio_put(owner->folio); + } + page_ext_put(ext); + return slot; +} + +struct mm_subpage_pool { + struct rcu_head rcu; + spinlock_t lock; + refcount_t refs; + struct list_head owners; + struct list_head available; + struct mm_subpage_owner *unissued; + unsigned long backing_pages; + unsigned long all_slots; + u8 shift; + bool closed; +}; + +static void owner_init(struct mm_subpage_owner *owner, struct mm_subpage_pool *pool, + struct folio *folio) +{ + unsigned int i; + + owner->pool = pool; + owner->shift = pool->shift; + owner->folio = folio; + INIT_LIST_HEAD(&owner->available); + for (i = 0; i < (PAGE_SIZE >> pool->shift); i++) { + owner->slots[i].owner = owner; + owner->slots[i].index = i; + owner->slots[i].shift = pool->shift; + refcount_set(&owner->slots[i].refs, 0); + atomic_set(&owner->slots[i].mapcount, 0); + } +} + +struct mm_subpage_pool *mm_subpage_pool_create_granule(gfp_t gfp, unsigned int shift) +{ + struct mm_subpage_pool *pool; + + BUILD_BUG_ON(PAGE_SHIFT <= MM_SUBPAGE_SHIFT); + BUILD_BUG_ON(SUBPAGES_PER_FOLIO > BITS_PER_LONG); + if (!subpage_shift_valid(shift)) + return NULL; + pool = kzalloc(sizeof(*pool), gfp | __GFP_ACCOUNT); + if (!pool) + return NULL; + pool->shift = shift; + pool->all_slots = subpage_mask(shift); + spin_lock_init(&pool->lock); + refcount_set(&pool->refs, 1); + INIT_LIST_HEAD(&pool->owners); + INIT_LIST_HEAD(&pool->available); + return pool; +} + +struct mm_subpage_pool *mm_subpage_pool_create(gfp_t gfp) +{ + return mm_subpage_pool_create_granule(gfp, MM_SUBPAGE_SHIFT); +} + +void mm_subpage_pool_get(struct mm_subpage_pool *pool) +{ + refcount_inc(&pool->refs); +} + +void mm_subpage_pool_put(struct mm_subpage_pool *pool) +{ + if (!refcount_dec_and_test(&pool->refs)) + return; + WARN_ON_ONCE(!pool->closed || !list_empty(&pool->owners)); + kfree_rcu(pool, rcu); +} + +static void owner_remove(struct mm_subpage_owner *owner) +{ + lockdep_assert_held(&owner->pool->lock); + owner->detached = true; + list_del(&owner->all); + list_del_init(&owner->available); + if (owner->pool->unissued == owner) + owner->pool->unissued = NULL; + owner->pool->backing_pages--; +} + +static void owner_free_rmap(struct mm_subpage_owner *owner) +{ + unsigned int i; + + if (!owner->rmap) + return; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (owner->rmap[i].root) + put_anon_vma(owner->rmap[i].root); + kfree(owner->rmap); +} + +static void owner_free_metadata(struct mm_subpage_owner *owner) +{ + struct mm_subpage_pool *pool = owner->pool; + + if (owner->anon_root) + put_anon_vma(owner->anon_root); + owner_free_rmap(owner); + kfree_rcu(owner, rcu); + mm_subpage_pool_put(pool); +} + +static void owner_free_work(struct work_struct *work) +{ + owner_free_metadata(container_of(work, struct mm_subpage_owner, free_work)); +} + +static void owner_free(struct mm_subpage_owner *owner, bool put_folio) +{ + owner_unbind(owner); + if (put_folio) { + if (owner->anon_root) + free_swap_cache(owner->folio); + /* Last slot reference, or wholly unissued supply. */ + folio_put(owner->folio); + } + if (owner->rmap) { + /* Historical roots can have no live VMA; their final put may sleep. */ + INIT_WORK(&owner->free_work, owner_free_work); + queue_work(system_unbound_wq, &owner->free_work); + } else { + owner_free_metadata(owner); + } +} + +struct anon_vma *mm_subpage_anon_root(const struct mm_subpage *subpage) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + return subpage->owner->anon_root; +} + +void mm_subpage_hold_anon_root(struct mm_subpage *subpage, struct anon_vma *root, + bool droppable) +{ + struct mm_subpage_owner *owner = subpage->owner; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + VM_BUG_ON(owner->anon_root || root != root->root); + get_anon_vma(root); + owner->droppable = droppable; + owner->anon_root = root; +} + +static struct mm_subpage_rmap slot_rmap(struct mm_subpage *slot) +{ + struct mm_subpage_owner *owner = slot->owner; + + lockdep_assert_held(&owner->pool->lock); + if (owner->rmap) + return owner->rmap[slot->index]; + return (struct mm_subpage_rmap) { + .root = owner->anon_root, + .pos = { .index = owner->folio->index, + .offset = mm_subpage_offset(slot) }, + .offset = mm_subpage_offset(slot), + }; +} + +static bool rmap_matches(struct mm_subpage_rmap map, struct vm_area_struct *vma, + unsigned long address) +{ + return map.root == vma->anon_vma->root && + vm_page_offset_equal(map.pos, vma_page_offset_at(vma, address)); +} + +bool mm_subpage_match_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address) +{ + struct mm_subpage_pool *pool = slot->owner->pool; + unsigned long flags; + bool matches; + + spin_lock_irqsave(&pool->lock, flags); + matches = slot->rmap_bound && rmap_matches(slot_rmap(slot), vma, address); + spin_unlock_irqrestore(&pool->lock, flags); + return matches; +} + +static void *owner_kcalloc(struct mm_subpage_owner *owner, size_t nr, size_t size) +{ + struct mem_cgroup *memcg, *saved; + void *maps; + + rcu_read_lock(); + do { + memcg = mem_cgroup_from_virt(owner) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + maps = kcalloc(nr, size, GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + set_active_memcg(saved); + mem_cgroup_put(memcg); + return maps; +} + +struct subpage_retired_root { + struct work_struct work; + struct anon_vma *root; +}; + +static void subpage_put_retired_root(struct work_struct *work) +{ + struct subpage_retired_root *retired = + container_of(work, struct subpage_retired_root, work); + + put_anon_vma(retired->root); + kfree(retired); +} + +/* The folio lock serializes binding and sidecar creation against migration. */ +int mm_subpage_bind_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address) +{ + struct mm_subpage_owner *owner = slot->owner; + struct mm_subpage_pool *pool = owner->pool; + struct mm_subpage_rmap *maps = NULL; + struct subpage_retired_root *retired = NULL; + unsigned long flags; + unsigned int i; + int ret = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + spin_lock_irqsave(&pool->lock, flags); + if (slot->rmap_bound) { + ret = rmap_matches(slot_rmap(slot), vma, address) ? 0 : -EINVAL; + goto unlock; + } + if (!owner->rmap && !rmap_matches(slot_rmap(slot), vma, address)) { + spin_unlock_irqrestore(&pool->lock, flags); + /* The caller may also hold a PTE lock. Never enter reclaim here. */ + maps = owner_kcalloc(owner, PAGE_SIZE >> owner->shift, sizeof(*maps)); + if (!maps) + return -ENOMEM; + spin_lock_irqsave(&pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + maps[i].offset = i << owner->shift; + if (!owner->slots[i].rmap_bound) + continue; + maps[i] = slot_rmap(&owner->slots[i]); + get_anon_vma(maps[i].root); + } + owner->rmap = maps; + } + if (owner->rmap) { + struct mm_subpage_rmap *map = &owner->rmap[slot->index]; + + if (map->root && map->root != vma->anon_vma->root) { + /* A final anon_vma put may sleep; the caller can hold a PTL. */ + spin_unlock_irqrestore(&pool->lock, flags); + retired = owner_kcalloc(owner, 1, sizeof(*retired)); + if (!retired) + return -ENOMEM; + INIT_WORK(&retired->work, subpage_put_retired_root); + spin_lock_irqsave(&pool->lock, flags); + retired->root = map->root; + } + if (map->root != vma->anon_vma->root) { + get_anon_vma(vma->anon_vma->root); + map->root = vma->anon_vma->root; + } + map->pos = vma_page_offset_at(vma, address); + map->offset = mm_subpage_offset(slot); + } + slot->rmap_bound = true; +unlock: + spin_unlock_irqrestore(&pool->lock, flags); + if (retired) + queue_work(system_unbound_wq, &retired->work); + return ret; +} + +static bool subpage_move_allowed(struct mm_subpage *slot, bool swap) +{ + lockdep_assert_held(&slot->owner->pool->lock); + return !slot->owner->migrating && !slot->pins && + atomic_read(&slot->mapcount) == (swap ? 0 : 1) && + (swap || (slot->rmap_bound && slot->exclusive)); +} + +int mm_subpage_move_prepare(struct mm_subpage_move *move, struct mm_subpage *slot, + struct vm_area_struct *src, unsigned long src_address, + struct vm_area_struct *dst, unsigned long dst_address, bool swap) +{ + struct mm_subpage_owner *owner = slot->owner; + struct mm_subpage_rmap *maps, old; + unsigned long flags; + unsigned int i; + int ret = -EBUSY; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + memset(move, 0, sizeof(*move)); + if (!src->anon_vma || !dst->anon_vma || !owner->anon_root || + mm_page_shift(src->vm_mm) != slot->shift || + mm_page_shift(dst->vm_mm) != slot->shift || + owner->droppable != !!(dst->vm_flags & VM_DROPPABLE)) + return -EINVAL; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!subpage_move_allowed(slot, swap) || + (slot->rmap_bound && !rmap_matches(slot_rmap(slot), src, src_address))) + goto unlock; + spin_unlock_irqrestore(&owner->pool->lock, flags); + + /* Allocate before touching either PTE. The folio lock serializes writers. */ + if (!owner->rmap) { + maps = owner_kcalloc(owner, PAGE_SIZE >> owner->shift, sizeof(*maps)); + if (!maps) + return -ENOMEM; + spin_lock_irqsave(&owner->pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + maps[i].offset = i << owner->shift; + if (!owner->slots[i].rmap_bound) + continue; + maps[i] = slot_rmap(&owner->slots[i]); + get_anon_vma(maps[i].root); + } + owner->rmap = maps; + spin_unlock_irqrestore(&owner->pool->lock, flags); + } + old = owner->rmap[slot->index]; + move->root_changed = old.root != dst->anon_vma->root; + if (move->root_changed && old.root) { + move->retired = owner_kcalloc(owner, 1, sizeof(*move->retired)); + if (!move->retired) + return -ENOMEM; + INIT_WORK(&move->retired->work, subpage_put_retired_root); + move->retired->root = old.root; + } + if (move->root_changed) + get_anon_vma(dst->anon_vma->root); + move->slot = slot; + move->swap = swap; + move->map = (struct mm_subpage_rmap) { + .root = dst->anon_vma->root, + .pos = vma_page_offset_at(dst, dst_address), + .offset = mm_subpage_offset(slot), + }; + return 0; +unlock: + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +int mm_subpage_move_commit(struct mm_subpage_move *move) +{ + struct mm_subpage *slot = move->slot; + struct mm_subpage_owner *owner = slot->owner; + unsigned long flags; + int ret = -EBUSY; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + spin_lock_irqsave(&owner->pool->lock, flags); + /* Recheck pins after source-PTE clearing and the matching TLB barrier. */ + if (subpage_move_allowed(slot, move->swap)) { + owner->rmap[slot->index] = move->map; + slot->rmap_bound = true; + move->committed = true; + ret = 0; + } + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +void mm_subpage_move_finish(struct mm_subpage_move *move) +{ + if (move->committed) { + if (move->retired) + queue_work(system_unbound_wq, &move->retired->work); + } else { + /* The locked destination VMA still holds this root. */ + if (move->root_changed) + put_anon_vma(move->map.root); + kfree(move->retired); + } +} + +int mm_subpage_snapshot_rmap(struct folio *folio, struct mm_subpage_rmap *maps) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner; + unsigned long flags; + unsigned int i; + int nr = -1; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return nr; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->detached && owner->rmap) { + nr = 0; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + if (!owner->slots[i].rmap_bound) + continue; + maps[nr] = owner->rmap[i]; + get_anon_vma(maps[nr].root); + nr++; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); +out: + page_ext_put(ext); + return nr; +} + +/* A referenced folio plus a stable VMA suffices; no folio lock required. */ +bool mm_subpage_address_in_vma(const struct folio *folio, + const struct vm_area_struct *vma, unsigned long *address) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner; + unsigned long flags; + unsigned int i; + bool nonlinear = false; + + if (!ext) + return false; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->detached && owner->rmap) { + nonlinear = true; + *address = -EFAULT; + for (i = 0; vma->anon_vma && i < (PAGE_SIZE >> owner->shift); i++) { + if (!owner->slots[i].rmap_bound || + owner->rmap[i].root != vma->anon_vma->root) + continue; + *address = vma_address_at_offset(vma, owner->rmap[i].pos); + if (*address != -EFAULT) + break; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); +out: + page_ext_put(ext); + return nonlinear; +} + +bool mm_subpage_is_droppable(const struct mm_subpage *subpage) +{ + return subpage->owner->droppable; +} + +void mm_subpage_pool_close(struct mm_subpage_pool *pool) +{ + struct mm_subpage_owner *owner; + unsigned long flags; + + spin_lock_irqsave(&pool->lock, flags); + pool->closed = true; + /* At most one newly supplied owner has no issued slots. Close is O(1). */ + owner = pool->unissued; + if (owner) + owner_remove(owner); + spin_unlock_irqrestore(&pool->lock, flags); + if (owner) + owner_free(owner, true); +} + +static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, + unsigned long eligible) +{ + struct mm_subpage_owner *owner; + + lockdep_assert_held(&pool->lock); + list_for_each_entry(owner, &pool->available, available) + /* A nonpresent PTE may still need any of this folio's slot bytes. */ + if (!owner->migrating && (~owner->used & eligible) && + !folio_test_swapcache(owner->folio)) + return owner; + return NULL; +} + +static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + unsigned long eligible, gfp_t gfp) +{ + struct mm_subpage_owner *owner; + unsigned long flags; + int ret = 0; + + /* Only an unexposed order-0 allocation can become a slot owner. */ + if (folio_order(folio) || folio_ref_count(folio) != 1 || + folio->mapping || folio_mapped(folio) || folio_test_lru(folio) || + folio_test_private(folio) || folio_test_swapcache(folio)) + return -EINVAL; + owner = kzalloc(sizeof(*owner), gfp | __GFP_ACCOUNT); + if (!owner) + return -ENOMEM; + owner_init(owner, pool, folio); + /* Reject duplicate ownership before touching any existing data. */ + ret = owner_bind(owner); + if (ret) { + kfree(owner); + return ret; + } + /* Initialise even unissued bytes before any future whole-folio operation. */ + clear_user_subpage_range(&folio->page, 0, PAGE_SIZE); + __folio_mark_uptodate(folio); + owner->zeroed = pool->all_slots; + spin_lock_irqsave(&pool->lock, flags); + if (pool->closed) + ret = -ESHUTDOWN; + else if (find_available(pool, eligible)) + /* Another allocator supplied backing while we allocated metadata. */ + ret = -EEXIST; + else { + mm_subpage_pool_get(pool); + list_add_tail(&owner->all, &pool->owners); + list_add_tail(&owner->available, &pool->available); + pool->backing_pages++; + pool->unissued = owner; + } + spin_unlock_irqrestore(&pool->lock, flags); + if (ret) { + owner_unbind(owner); + kfree_rcu(owner, rcu); + } + return ret; +} + +int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + gfp_t gfp) +{ + return pool_add_folio(pool, folio, pool->all_slots, gfp); +} + +int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int offset, gfp_t gfp) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return -EINVAL; + return pool_add_folio(pool, folio, eligible, gfp); +} + +unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) +{ + unsigned long pages, flags; + + spin_lock_irqsave(&pool->lock, flags); + pages = pool->backing_pages; + spin_unlock_irqrestore(&pool->lock, flags); + return pages; +} + +static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, + unsigned long eligible, bool *needs_zero) +{ + struct mm_subpage_owner *owner; + struct mm_subpage *subpage; + unsigned long flags; + unsigned int index; + + spin_lock_irqsave(&pool->lock, flags); + if (pool->closed) { + subpage = ERR_PTR(-ESHUTDOWN); + goto out; + } + owner = find_available(pool, eligible); + if (!owner) { + subpage = ERR_PTR(-EAGAIN); + goto out; + } + if (pool->unissued == owner) + /* Transfer the supplied reference to the first reservation. */ + pool->unissued = NULL; + else + folio_get(owner->folio); + index = __ffs(~owner->used & eligible); + __set_bit(index, &owner->used); + if (needs_zero) + *needs_zero = !test_bit(index, &owner->zeroed); + __clear_bit(index, &owner->zeroed); + if (owner->used == pool->all_slots) + list_del_init(&owner->available); + subpage = &owner->slots[index]; + WRITE_ONCE(subpage->lazyfree, false); + /* The used bit keeps backing alive while data is initialised. */ + VM_BUG_ON(refcount_read(&subpage->refs)); + VM_BUG_ON(atomic_read(&subpage->mapcount)); +out: + spin_unlock_irqrestore(&pool->lock, flags); + return subpage; +} + +static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible) +{ + bool needs_zero; + struct mm_subpage *subpage = subpage_reserve(pool, eligible, &needs_zero); + + if (IS_ERR(subpage)) + return subpage; + /* Fresh supply was already zeroed as a whole; reused slots must be cleared. */ + if (needs_zero) + clear_user_subpage_range(&subpage->owner->folio->page, + mm_subpage_offset(subpage), mm_subpage_size(subpage)); + /* Data stores must precede publication in a PTE or another CPU's reference. */ + refcount_set_release(&subpage->refs, 1); + return subpage; +} + +struct mm_subpage *mm_subpage_alloc(struct mm_subpage_pool *pool) +{ + return subpage_alloc(pool, pool->all_slots); +} + +struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return ERR_PTR(-EINVAL); + return subpage_alloc(pool, eligible); +} + +static void subpage_release_locked(struct mm_subpage *subpage, unsigned long flags) +{ + struct mm_subpage_owner *owner = subpage->owner; + struct mm_subpage_pool *pool = owner->pool; + struct folio *folio = owner->folio; + bool release; + + lockdep_assert_held(&pool->lock); + VM_BUG_ON(refcount_read(&subpage->refs)); + VM_BUG_ON(atomic_read(&subpage->mapcount)); + VM_BUG_ON(subpage->pins); + WRITE_ONCE(subpage->exclusive, false); + if (!owner->migrating) { + WRITE_ONCE(subpage->lazyfree, false); + subpage->rmap_bound = false; + } + __clear_bit(subpage->index, &owner->used); + release = !owner->used && !owner->migrating; + if (release) + owner_remove(owner); + else if (!pool->closed && !owner->migrating && list_empty(&owner->available)) + list_add(&owner->available, &pool->available); + spin_unlock_irqrestore(&pool->lock, flags); + if (release) + owner_free(owner, true); + else + folio_put(folio); +} + +static void subpage_release(struct mm_subpage *subpage) +{ + unsigned long flags; + + spin_lock_irqsave(&subpage->owner->pool->lock, flags); + subpage_release_locked(subpage, flags); +} + +static struct mm_subpage *subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned long eligible) +{ + struct mm_subpage *dest; + int err; + + if (pool->shift != mm_subpage_shift(source)) + return ERR_PTR(-EINVAL); + dest = subpage_reserve(pool, eligible, NULL); + + if (IS_ERR(dest)) + return dest; + err = copy_user_subpage_range(&dest->owner->folio->page, mm_subpage_offset(dest), + &source->owner->folio->page, mm_subpage_offset(source), + mm_subpage_size(source)); + if (err) { + /* Never publish a reservation whose copy did not finish. */ + subpage_release(dest); + return ERR_PTR(err); + } + refcount_set_release(&dest->refs, 1); + return dest; +} + +struct mm_subpage *mm_subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source) +{ + return subpage_copy(pool, source, pool->all_slots); +} + +struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return ERR_PTR(-EINVAL); + return subpage_copy(pool, source, eligible); +} + +/* The page-extension RCU section also protects the pool until its lock is held. */ +static struct mm_subpage *restore_bound_slot(struct folio *folio, unsigned int offset, + unsigned int shift) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_pool *pool; + struct mm_subpage_ext *subext; + struct mm_subpage *slot = NULL; + unsigned int index; + unsigned long flags; + + if (!ext) + return ERR_PTR(-EOPNOTSUPP); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + if ((shift && shift != owner->shift) || !offset_mask(owner->shift, offset)) { + slot = ERR_PTR(-EINVAL); + goto out; + } + index = offset >> owner->shift; + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->detached) { + slot = ERR_PTR(-EAGAIN); + goto unlock; + } + slot = &owner->slots[index]; + if (test_bit(index, &owner->used)) { + folio_get(folio); + if (!refcount_inc_not_zero_acquire(&slot->refs)) { + /* Its final put or initial publication has not finished. */ + folio_put(folio); + slot = ERR_PTR(-EAGAIN); + } + goto unlock; + } + VM_BUG_ON(refcount_read(&slot->refs) || atomic_read(&slot->mapcount)); + if (pool->unissued == owner) + pool->unissued = NULL; + else + folio_get(folio); + __set_bit(index, &owner->used); + __clear_bit(index, &owner->zeroed); + if (owner->used == pool->all_slots) + list_del_init(&owner->available); + /* The folio lock and uptodate state stabilise the restored bytes. */ + refcount_set_release(&slot->refs, 1); +unlock: + spin_unlock_irqrestore(&pool->lock, flags); +out: + page_ext_put(ext); + return slot; +} + +struct mm_subpage *mm_subpage_restore_granule(struct folio *folio, unsigned int offset, + unsigned int shift, gfp_t gfp) +{ + struct mm_subpage_pool *pool; + struct mm_subpage_owner *owner; + struct mm_subpage *slot; + unsigned int index; + int err; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!subpage_shift_valid(shift) || !offset_mask(shift, offset) || folio_order(folio) || !folio_test_swapcache(folio) || + !folio_test_uptodate(folio) || folio_test_ksm(folio) || + (folio->mapping && !folio_test_anon(folio)) || + (folio_test_anon(folio) && PageAnonExclusive(&folio->page))) + return ERR_PTR(-EINVAL); + index = offset >> shift; + slot = restore_bound_slot(folio, offset, shift); + if (slot) + return slot; + + /* A reclaimed folio has no surviving owner; do not initialise its data. */ + pool = mm_subpage_pool_create_granule(gfp, shift); + if (!pool) + return ERR_PTR(-ENOMEM); + pool->closed = true; + owner = kzalloc(sizeof(*owner), gfp | __GFP_ACCOUNT); + if (!owner) { + mm_subpage_pool_put(pool); + return ERR_PTR(-ENOMEM); + } + owner_init(owner, pool, folio); + owner->used = BIT(index); + refcount_set(&owner->slots[index].refs, 1); + list_add(&owner->all, &pool->owners); + pool->backing_pages = 1; + folio_get(folio); + err = owner_bind(owner); + if (err) { + folio_put(folio); + list_del(&owner->all); + pool->backing_pages = 0; + kfree(owner); + mm_subpage_pool_put(pool); + return ERR_PTR(err == -EEXIST ? -EAGAIN : err); + } + /* The private closed pool's initial reference belongs to this owner. */ + return &owner->slots[index]; +} + +struct mm_subpage *mm_subpage_restore(struct folio *folio, unsigned int offset, gfp_t gfp) +{ + return mm_subpage_restore_granule(folio, offset, MM_SUBPAGE_SHIFT, gfp); +} + +/* Preserve the metadata charge when migration runs outside its owning task. */ +static struct mm_subpage_owner *owner_alloc_migration(struct mm_subpage_owner *owner) +{ + struct mem_cgroup *memcg, *saved; + struct mm_subpage_owner *new; + + rcu_read_lock(); + do { + memcg = mem_cgroup_from_virt(owner) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + /* Folio locks are held: do not recurse into reclaim or trigger memcg OOM. */ + new = kzalloc(sizeof(*new), GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + if (new && owner->rmap) { + new->rmap = kcalloc(PAGE_SIZE >> owner->shift, sizeof(*new->rmap), + GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + if (!new->rmap) { + kfree(new); + new = NULL; + } + } + set_active_memcg(saved); + mem_cgroup_put(memcg); + return new; +} + +/* A migration hold pins metadata, not the native folio: the driver pins it. */ +int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) +{ + struct page_ext *ext; + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner, *new; + struct mm_subpage_pool *pool; + unsigned long flags; + int ret = 0; + unsigned int i; + + VM_BUG_ON_FOLIO(!folio_test_locked(src), src); + VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst); + ext = page_ext_get(&src->page); + if (!ext) + return 0; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) { + page_ext_put(ext); + return 0; + } + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->detached) { + owner = NULL; + } else if (src == dst || folio_order(src) || folio_order(dst) || + owner->migrating || pool->unissued == owner) { + ret = -EBUSY; + } else { + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (owner->slots[i].pins) + ret = -EBUSY; + if (!ret) + owner->migrating = true; + } + spin_unlock_irqrestore(&pool->lock, flags); + page_ext_put(ext); + if (!owner || ret) + return ret; + + /* Async migration must not recurse into reclaim while holding folio locks. */ + new = owner_alloc_migration(owner); + if (!new) { + ret = -ENOMEM; + goto undo; + } + owner_init(new, pool, dst); + new->migrating = true; + new->zeroed = owner->zeroed; + spin_lock_irqsave(&pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + new->slots[i].rmap_bound = owner->slots[i].rmap_bound; + if (new->rmap) { + new->rmap[i] = owner->rmap[i]; + if (new->rmap[i].root) + get_anon_vma(new->rmap[i].root); + } + } + spin_unlock_irqrestore(&pool->lock, flags); + + new->droppable = owner->droppable; + new->anon_root = owner->anon_root; + if (new->anon_root) + get_anon_vma(new->anon_root); + mm_subpage_pool_get(pool); + spin_lock_irqsave(&pool->lock, flags); + list_add_tail(&new->all, &pool->owners); + pool->backing_pages++; + ret = owner_bind(new); + if (ret) + owner_remove(new); + else + owner->migration_target = new; + spin_unlock_irqrestore(&pool->lock, flags); + if (!ret) + return 0; + if (new->anon_root) + put_anon_vma(new->anon_root); + owner_free_rmap(new); + kfree(new); + mm_subpage_pool_put(pool); +undo: + mm_subpage_migrate_finish(src); + return ret; +} + +bool mm_subpage_migrating(struct folio *folio) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + bool active = false; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return false; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (owner) + active = READ_ONCE(owner->migrating); + page_ext_put(ext); + return active; +} + +/* Source folio/PTE locked: writable GETs cannot race this final snapshot. */ +void mm_subpage_migrate_save_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + + VM_BUG_ON_FOLIO(!owner->migrating || !owner->migration_target, owner->folio); + WRITE_ONCE(owner->migration_target->slots[subpage->index].lazyfree, + READ_ONCE(subpage->lazyfree)); +} + +struct mm_subpage *mm_subpage_migrate_restore(struct folio *folio, unsigned int offset) +{ + struct mm_subpage *slot; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(!offset_mask(MM_SUBPAGE_SHIFT, offset) || !mm_subpage_migrating(folio), folio); + /* No allocation or data initialisation can fail during migration recovery. */ + slot = restore_bound_slot(folio, offset, 0); + VM_BUG_ON_FOLIO(IS_ERR_OR_NULL(slot), folio); + return slot; +} + +void mm_subpage_migrate_finish(struct folio *folio) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + struct mm_subpage_pool *pool; + unsigned long flags; + unsigned int i; + bool release = false; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->migrating) { + VM_BUG_ON(owner->detached); + owner->migrating = false; + owner->migration_target = NULL; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (!(owner->used & BIT(i))) + owner->slots[i].rmap_bound = false; + release = !owner->used; + if (release) + owner_remove(owner); + else if (!pool->closed && owner->used != pool->all_slots && + list_empty(&owner->available)) + list_add(&owner->available, &pool->available); + } + spin_unlock_irqrestore(&pool->lock, flags); +out: + page_ext_put(ext); + if (release) + /* No slot references remain; only the migration driver pins backing. */ + owner_free(owner, false); +} + +void mm_subpage_get(struct mm_subpage *subpage) +{ + folio_get(subpage->owner->folio); + refcount_inc(&subpage->refs); +} + +void mm_subpage_put(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + + /* Retire zero-reference state atomically with its used bit. */ + if (refcount_dec_and_lock_irqsave(&subpage->refs, &pool->lock, &flags)) + subpage_release_locked(subpage, flags); + else + folio_put(subpage->owner->folio); +} + +void mm_subpage_set_exclusive(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + VM_BUG_ON(atomic_read(&subpage->mapcount) > 1 || !refcount_read(&subpage->refs)); + WRITE_ONCE(subpage->exclusive, true); + spin_unlock_irqrestore(&pool->lock, flags); +} + +bool mm_subpage_is_exclusive(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->exclusive); +} + +static bool subpage_try_reuse(struct mm_subpage *subpage, unsigned int references) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool reusable; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + reusable = atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == references && !subpage->pins; + if (reusable) + WRITE_ONCE(subpage->exclusive, true); + spin_unlock_irqrestore(&pool->lock, flags); + return reusable; +} + +bool mm_subpage_try_reuse_swap(struct mm_subpage *subpage) +{ + return subpage_try_reuse(subpage, 1); +} + +bool mm_subpage_try_reuse_anon(struct mm_subpage *subpage) +{ + return subpage_try_reuse(subpage, 2); +} + +bool mm_subpage_can_zap(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + spin_lock_irqsave(&pool->lock, flags); + eligible = !subpage->owner->migrating && !subpage->pins && + atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == 2; + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +bool mm_subpage_try_share(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool shared; + + spin_lock_irqsave(&pool->lock, flags); + shared = !subpage->pins; + if (shared) + WRITE_ONCE(subpage->exclusive, false); + spin_unlock_irqrestore(&pool->lock, flags); + return shared; +} + +bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->lazyfree); +} + +void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage) +{ + WRITE_ONCE(subpage->lazyfree, false); +} + +bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + eligible = atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == 2 && !subpage->pins; + if (eligible) + WRITE_ONCE(subpage->lazyfree, true); + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +bool mm_subpage_all_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + unsigned int i; + bool all = true; + + spin_lock_irqsave(&owner->pool->lock, flags); + for (i = 0; i < SUBPAGES_PER_FOLIO; i++) { + if (!(owner->used & BIT(i))) + continue; + if (!READ_ONCE(owner->slots[i].lazyfree)) { + all = false; + break; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); + return all; +} + +bool mm_subpage_discardable(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + spin_lock_irqsave(&pool->lock, flags); + eligible = (subpage->owner->droppable || READ_ONCE(subpage->lazyfree)) && + !subpage->pins && + refcount_read(&subpage->refs) == atomic_read(&subpage->mapcount) + 1; + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +int mm_subpage_pin(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + int ret = -EAGAIN; + + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->anon_root || owner->migrating || !subpage->exclusive || + !atomic_read(&subpage->mapcount)) + goto out; + ret = try_grab_folio(owner->folio, 1, FOLL_PIN); + if (ret) + goto out; + mm_subpage_get(subpage); + subpage->pins++; +out: + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +void mm_subpage_unpin(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + + spin_lock_irqsave(&owner->pool->lock, flags); + VM_BUG_ON(!subpage->pins || !subpage->exclusive); + subpage->pins--; + spin_unlock_irqrestore(&owner->pool->lock, flags); + unpin_folio(owner->folio); + mm_subpage_put(subpage); +} + +unsigned int mm_subpage_pincount(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->pins); +} + +int mm_subpage_mapcount(const struct mm_subpage *subpage) +{ + return atomic_read(&subpage->mapcount); +} + +struct folio *mm_subpage_folio(const struct mm_subpage *subpage) +{ + return subpage->owner->folio; +} + +unsigned int mm_subpage_shift(const struct mm_subpage *subpage) +{ + return subpage->shift; +} + +unsigned int mm_subpage_size(const struct mm_subpage *subpage) +{ + return 1U << subpage->shift; +} + +unsigned int mm_subpage_offset(const struct mm_subpage *subpage) +{ + return subpage->index << subpage->shift; +} + +phys_addr_t mm_subpage_phys(const struct mm_subpage *subpage) +{ + return PFN_PHYS(folio_pfn(subpage->owner->folio)) + mm_subpage_offset(subpage); +} + +/* + * Round the mm's aggregate reservation, never each VMA: splitting a VMA + * must not create extra charges or refunds. mmap writers serialize updates; + * munmap may retain the lock downgraded while finishing its refund. + */ +static int mm_commit_delta(struct mm_struct *mm, long pages, + unsigned long *next, long *delta) +{ + unsigned long old = mm->committed_user_pages; + unsigned int shift = mm_page_shift(mm); + + mmap_assert_locked(mm); + if (pages < 0 && WARN_ON_ONCE(pages < -(long)old)) + return -EINVAL; + if (pages > 0 && (unsigned long)pages > LONG_MAX - old) + return -ENOMEM; + *next = old + pages; + if (shift > PAGE_SHIFT) { + unsigned int extra = shift - PAGE_SHIFT; + + /* A logical reservation may span several native commit units. */ + if (*next > (LONG_MAX >> extra) || old > (LONG_MAX >> extra)) + return -ENOMEM; + *delta = (long)(*next << extra) - (long)(old << extra); + } else { + unsigned long per_native = 1UL << (PAGE_SHIFT - shift); + + *delta = DIV_ROUND_UP(*next, per_native) - DIV_ROUND_UP(old, per_native); + } + return 0; +} + +int mm_account_memory(struct mm_struct *mm, struct mm_struct *policy_mm, long pages) +{ + unsigned long next; + long delta; + int ret; + + if (mm_page_size(mm) == PAGE_SIZE) + return security_vm_enough_memory_mm(policy_mm, pages); + ret = mm_commit_delta(mm, pages, &next, &delta); + if (ret) + return ret; + ret = security_vm_enough_memory_mm(policy_mm, delta); + if (!ret) + mm->committed_user_pages = next; + return ret; +} + +void mm_acct_memory(struct mm_struct *mm, long pages) +{ + unsigned long next; + long delta; + + if (mm_page_size(mm) == PAGE_SIZE) { + vm_acct_memory(pages); + return; + } + if (WARN_ON_ONCE(mm_commit_delta(mm, pages, &next, &delta))) + return; + vm_acct_memory(delta); + mm->committed_user_pages = next; +} + +void mm_unacct_memory(struct mm_struct *mm, unsigned long pages) +{ + mm_acct_memory(mm, -(long)pages); +} + +/* Shared ownership for consumers whose native page spans contain user slots. */ +struct mm_subpage_refs { + refcount_t refs; + unsigned int count; + unsigned int capacity; + struct mm_subpage *slots[]; +}; + +struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs) +{ + if (refs) + refcount_inc(&refs->refs); + return refs; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_get); + +void mm_subpage_refs_put(struct mm_subpage_refs *refs) +{ + unsigned int i; + + if (!refs || !refcount_dec_and_test(&refs->refs)) + return; + for (i = 0; i < refs->count; i++) + mm_subpage_put(refs->slots[i]); + kfree(refs); +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_put); + +static struct mm_subpage_refs *subpage_refs_new(unsigned int capacity, gfp_t gfp) +{ + struct mm_subpage_refs *refs = kmalloc(struct_size(refs, slots, capacity), gfp); + + if (!refs) + return NULL; + refcount_set(&refs->refs, 1); + refs->count = 0; + refs->capacity = capacity; + return refs; +} + +static int subpage_refs_prepare(struct mm_subpage_refs **refsp, + unsigned int needed, gfp_t gfp) +{ + struct mm_subpage_refs *old = *refsp, *refs; + unsigned int i; + + if (old && refcount_read(&old->refs) == 1 && old->capacity >= needed) + return 0; + refs = subpage_refs_new(max(needed, 4U), gfp); + if (!refs) + return -ENOMEM; + if (old) { + for (i = 0; i < old->count; i++) { + mm_subpage_get(old->slots[i]); + refs->slots[refs->count++] = old->slots[i]; + } + } + *refsp = refs; + mm_subpage_refs_put(old); + return 0; +} + +int mm_subpage_refs_add(struct mm_subpage_refs **refsp, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp) +{ + struct mm_subpage *slots[PAGE_SIZE / MM_SUBPAGE_SIZE]; + struct mm_subpage_refs *refs = *refsp; + unsigned int pos, count = 0, i, j; + int ret = 0; + bool found = false; + + if (!len || folio_order(page_folio(page)) || !folio_test_anon(page_folio(page))) + return 0; + if (offset >= PAGE_SIZE || len > PAGE_SIZE - offset) + return -EINVAL; + for (pos = round_down(offset, MM_SUBPAGE_SIZE); pos < offset + len; + pos += MM_SUBPAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(page_to_phys(page) + pos); + + if (!slot) { + /* A native anonymous allocation has no quarter owner. */ + ret = found ? -EFAULT : 0; + goto out; + } + found = true; + for (j = 0; refs && j < refs->count; j++) + if (refs->slots[j] == slot) + break; + if (refs && j < refs->count) { + mm_subpage_put(slot); + continue; + } + for (j = 0; j < count; j++) + if (slots[j] == slot) + break; + if (j < count) + mm_subpage_put(slot); + else + slots[count++] = slot; + } + if (!count) + return 0; + if (refs && refs->count > UINT_MAX - count) { + ret = -ENOMEM; + goto out; + } + ret = subpage_refs_prepare(refsp, (refs ? refs->count : 0) + count, gfp); + if (ret) + goto out; + refs = *refsp; + for (i = 0; i < count; i++) + refs->slots[refs->count++] = slots[i]; + return 0; +out: + for (i = 0; i < count; i++) + mm_subpage_put(slots[i]); + return ret; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_add); + +int mm_subpage_refs_prune(struct mm_subpage_refs **refsp, + bool (*keep)(const struct mm_subpage *, void *), + void *context, gfp_t gfp) +{ + struct mm_subpage_refs *refs = *refsp, *new; + unsigned int i, count = 0; + + if (!refs) + return 0; + for (i = 0; i < refs->count; i++) + count += keep(refs->slots[i], context); + if (count == refs->count) + return 0; + if (!count) { + *refsp = NULL; + mm_subpage_refs_put(refs); + return 0; + } + if (refcount_read(&refs->refs) == 1) { + for (i = 0, count = 0; i < refs->count; i++) { + if (keep(refs->slots[i], context)) + refs->slots[count++] = refs->slots[i]; + else + mm_subpage_put(refs->slots[i]); + } + refs->count = count; + return 0; + } + new = subpage_refs_new(count, gfp); + if (!new) + return -ENOMEM; + for (i = 0; i < refs->count; i++) { + if (!keep(refs->slots[i], context)) + continue; + mm_subpage_get(refs->slots[i]); + new->slots[new->count++] = refs->slots[i]; + } + *refsp = new; + mm_subpage_refs_put(refs); + return 0; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_prune); diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 01b151dca3085..8d72d991b05e5 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -16,10 +16,19 @@ #include #include #include +#include +#include #include #include #include "internal.h" #include "swap.h" +#ifdef CONFIG_MM_SUBPAGE +#include "user-subpage-internal.h" +#endif + +struct mfill_shared_data { + struct folio *source; /* NULL requests a zero-fill. */ +}; struct mfill_state { struct userfaultfd_ctx *ctx; @@ -32,6 +41,10 @@ struct mfill_state { unsigned long src_addr; unsigned long dst_addr; pmd_t *pmd; + struct folio *copy_buffer; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *slots[MM_SUBPAGES_PER_PAGE]; +#endif }; static bool anon_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags) @@ -194,8 +207,24 @@ static void uffd_mfill_unlock(struct vm_area_struct *vma) } #endif +/* Reservations keep the uninstalled leaves private to this operation. */ +static void mfill_free_subpages(struct mfill_state *state) +{ +#ifdef CONFIG_MM_SUBPAGE + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->slots); i++) { + if (state->slots[i]) { + mm_subpage_put(state->slots[i]); + state->slots[i] = NULL; + } + } +#endif +} + static void mfill_put_vma(struct mfill_state *state) { + mfill_free_subpages(state); if (!state->vma) return; @@ -248,6 +277,7 @@ static int mfill_get_vma(struct mfill_state *state) if ((flags & MFILL_ATOMIC_WP) && !(dst_vma->vm_flags & VM_UFFD_WP)) goto out_unlock; + if (is_vm_hugetlb_page(dst_vma)) return 0; @@ -320,16 +350,11 @@ static int mfill_establish_pmd(struct mfill_state *state) static bool mfill_file_over_size(struct vm_area_struct *dst_vma, unsigned long dst_addr) { - struct inode *inode; - pgoff_t offset, max_off; - if (!dst_vma->vm_file) return false; - inode = dst_vma->vm_file->f_inode; - offset = linear_page_index(dst_vma, dst_addr); - max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - return offset >= max_off; + return !vma_file_offset_valid(dst_vma, dst_addr, + i_size_read(file_inode(dst_vma->vm_file))); } /* @@ -341,7 +366,8 @@ static bool mfill_file_over_size(struct vm_area_struct *dst_vma, static int mfill_atomic_install_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, struct page *page, - uffd_flags_t flags) + uffd_flags_t flags, + const struct mfill_shared_data *shared_data) { int ret; struct mm_struct *dst_mm = dst_vma->vm_mm; @@ -354,6 +380,11 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, pte_t dst_ptep; _dst_pte = mk_pte(page, dst_vma->vm_page_prot); +#ifdef CONFIG_MM_SUBPAGE + if (page_in_cache && mm_page_size(dst_mm) < PAGE_SIZE) + _dst_pte = phys_pte_mm(dst_mm, page_to_phys(page) + + vma_page_offset_at(dst_vma, dst_addr).offset, dst_vma->vm_page_prot); +#endif _dst_pte = pte_mkdirty(_dst_pte); if (page_in_cache && !vm_shared) writable = false; @@ -384,7 +415,39 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, if (!pte_none(dst_ptep) && !pte_is_uffd_marker(dst_ptep)) goto out_unlock; +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + if (shared_data) { + struct vm_page_offset pos = vma_page_offset_at(dst_vma, dst_addr); + unsigned long offset = ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset; + unsigned long size = mm_page_size(dst_mm); + void *dst, *src; + + if (PageHWPoison(page)) { + ret = -EIO; + goto out_unlock; + } + if (!shmem_uffd_range_missing(folio, pos.index, pos.offset, size)) + goto out_unlock; + dst = kmap_local_folio(folio, offset); + if (shared_data->source) { + src = kmap_local_folio(shared_data->source, 0); + memcpy(dst, src, size); + kunmap_local(src); + } else { + memset(dst, 0, size); + } + kunmap_local(dst); + flush_dcache_folio(folio); + __folio_mark_uptodate(folio); + folio_mark_dirty(folio); + } +#endif + if (page_in_cache) { + mm_subpage_file_map_add(dst_vma, pte_phys_mm(dst_mm, _dst_pte)); + shmem_uffd_mark_mapped(folio, + (folio_page_idx(folio, page) << PAGE_SHIFT) + + vma_page_offset_at(dst_vma, dst_addr).offset, mm_page_size(dst_mm)); folio_add_file_rmap_pte(folio, page, dst_vma); } else { folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE); @@ -411,12 +474,13 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, return ret; } -static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) +static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr, + unsigned int offset, unsigned long len) { void *kaddr; int ret; - kaddr = kmap_local_folio(folio, 0); + kaddr = kmap_local_folio(folio, offset); /* * The read mmap_lock is held here. Despite the * mmap_lock being read recursive a deadlock is still @@ -433,8 +497,7 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) * and retry the copy outside the mmap_lock. */ pagefault_disable(); - ret = copy_from_user(kaddr, (const void __user *) src_addr, - PAGE_SIZE); + ret = copy_from_user(kaddr, (const void __user *)src_addr, len); pagefault_enable(); kunmap_local(kaddr); @@ -507,8 +570,11 @@ DEFINE_FREE(retry_put, struct mfill_retry_state *, if (_T) mfill_retry_state_put(_T)); static int mfill_copy_folio_retry(struct mfill_state *mfill_state, - struct folio *folio) + struct folio *folio, unsigned int offset, + unsigned long len) { + struct vm_page_offset pos = vma_page_offset_at(mfill_state->vma, + mfill_state->dst_addr); struct mfill_retry_state retry_state = { 0 }; struct mfill_retry_state *for_free __free(retry_put) = &retry_state; unsigned long src_addr = mfill_state->src_addr; @@ -520,8 +586,8 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, /* retry copying with mm_lock dropped */ mfill_put_vma(mfill_state); - kaddr = kmap_local_folio(folio, 0); - err = copy_from_user(kaddr, (const void __user *) src_addr, PAGE_SIZE); + kaddr = kmap_local_folio(folio, offset); + err = copy_from_user(kaddr, (const void __user *) src_addr, len); kunmap_local(kaddr); if (unlikely(err)) return -EFAULT; @@ -533,7 +599,10 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, if (err) return err; - if (mfill_retry_state_changed(&retry_state, mfill_state->vma)) + if (mfill_retry_state_changed(&retry_state, mfill_state->vma) || + (mm_page_size(mfill_state->ctx->mm) < PAGE_SIZE && + !vm_page_offset_equal(pos, vma_page_offset_at(mfill_state->vma, + mfill_state->dst_addr)))) return -EAGAIN; err = mfill_establish_pmd(mfill_state); @@ -543,6 +612,28 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, return 0; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) +/* Fill one missing leaf in new or retained shmem backing using staged data. */ +static int mfill_atomic_shmem_hole(struct mfill_state *state, + const struct mfill_shared_data *data) +{ + struct vm_page_offset pos; + struct folio *folio; + bool new; + int err; + + folio = shmem_uffd_prepare_folio(state->vma, state->dst_addr, &new); + if (IS_ERR(folio)) + return PTR_ERR(folio); + pos = vma_page_offset_at(state->vma, state->dst_addr); + err = mfill_atomic_install_pte(state->pmd, state->vma, state->dst_addr, + folio_file_page(folio, pos.index), state->flags, data); + if (err) + shmem_uffd_abort_folio(folio, state->vma, new); + return err; +} +#endif + static int __mfill_atomic_pte(struct mfill_state *state, const struct vm_uffd_ops *ops) { @@ -562,7 +653,7 @@ static int __mfill_atomic_pte(struct mfill_state *state, return -ENOMEM; if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) { - ret = mfill_copy_folio_locked(folio, src_addr); + ret = mfill_copy_folio_locked(folio, src_addr, 0, PAGE_SIZE); /* * Fallback to copy_from_user outside mmap_lock. * If retry is successful, mfill_copy_folio_locked() returns @@ -571,7 +662,7 @@ static int __mfill_atomic_pte(struct mfill_state *state, * will take care of unlocking if needed. */ if (unlikely(ret)) { - ret = mfill_copy_folio_retry(state, folio); + ret = mfill_copy_folio_retry(state, folio, 0, PAGE_SIZE); if (ret) goto err_folio_put; } @@ -590,12 +681,29 @@ static int __mfill_atomic_pte(struct mfill_state *state, if (ops->filemap_add) { ret = ops->filemap_add(folio, state->vma, state->dst_addr); +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + /* + * A pinned large shmem folio can survive a partial hole punch. + * Its cache entry exists while this native-sized range is missing. + * Reuse the prepared source and preserve all neighbouring bytes. + * Ordinary native insertion and non-shmem paths stay unchanged. + */ + if (ret == -EEXIST && (state->vma->vm_flags & VM_SHARED) && + shmem_file(state->vma->vm_file) && + (READ_ONCE(SHMEM_I(file_inode(state->vma->vm_file))->flags) & + SHMEM_F_UFFD_SUBPAGE)) { + struct mfill_shared_data data = { .source = folio }; + + ret = mfill_atomic_shmem_hole(state, &data); + goto err_folio_put; + } +#endif if (ret) goto err_folio_put; } ret = mfill_atomic_install_pte(state->pmd, state->vma, dst_addr, - &folio->page, flags); + &folio->page, flags, NULL); if (ret) goto err_filemap_remove; @@ -609,10 +717,184 @@ static int __mfill_atomic_pte(struct mfill_state *state, return ret; } +#ifdef CONFIG_MM_SUBPAGE +/* Pack only this request's leaves in one native linear index. */ +static struct mm_subpage *mfill_alloc_subpage(struct mfill_state *state) +{ + struct vm_area_struct *vma = state->vma; + struct mm_struct *mm = vma->vm_mm; + struct vm_page_offset pos = vma_page_offset_at(vma, state->dst_addr); + unsigned long size = mm_page_size(mm); + unsigned int first = pos.offset >> mm_page_shift(mm), i, nr; + struct mm_subpage_pool *pool; + struct mm_subpage *slot; + struct folio *folio; + int err = -ENOMEM; + + if (state->slots[first]) + goto take; + + mfill_free_subpages(state); + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return ERR_PTR(-ENOMEM); + folio = anon_alloc_folio(vma, state->dst_addr); + if (!folio) + goto out; + folio_throttle_swaprate(folio, GFP_KERNEL); + err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (err) { + folio_put(folio); + goto out; + } + nr = min_t(unsigned long, PAGE_SIZE - pos.offset, + state->dst_start + state->len - state->dst_addr) / size; + for (i = first; i < first + nr; i++) { + slot = mm_subpage_alloc_at(pool, i * size); + if (IS_ERR(slot)) { + err = PTR_ERR(slot); + mfill_free_subpages(state); + goto out; + } + state->slots[i] = slot; + } +out: + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + if (err) + return ERR_PTR(err); +take: + slot = state->slots[first]; + state->slots[first] = NULL; + return slot; +} + +static int mfill_atomic_subpage(struct mfill_state *state) +{ + struct mm_struct *mm = state->ctx->mm; + struct mm_subpage *slot = mfill_alloc_subpage(state); + struct vm_area_struct *vma; + struct folio *folio; + pte_t entry, *pte; + spinlock_t *ptl; + bool fresh; + int err; + + if (IS_ERR(slot)) + return PTR_ERR(slot); + folio = mm_subpage_folio(slot); + folio_lock(folio); + /* A previous leaf may have entered reclaim between iterations. */ + err = -EAGAIN; + if (folio_test_swapcache(folio)) + goto unlock_folio; + if (uffd_flags_mode_is(state->flags, MFILL_ATOMIC_COPY)) { + err = mfill_copy_folio_locked(folio, state->src_addr, + mm_subpage_offset(slot), mm_page_size(mm)); + if (err) { + /* + * The faulting retry must not hold the folio lock, or write + * a folio whose already-visible neighbours can be reclaimed. + * Use fresh backing and discard the remaining reservations + * before dropping the VMA lock. + */ + folio_unlock(folio); + mm_subpage_put(slot); + mfill_free_subpages(state); + slot = mfill_alloc_subpage(state); + if (IS_ERR(slot)) + return PTR_ERR(slot); + folio = mm_subpage_folio(slot); + err = mfill_copy_folio_retry(state, folio, + mm_subpage_offset(slot), mm_page_size(mm)); + if (err) + goto put_slot; + folio_lock(folio); + } + } + /* The allocator zeroed every byte, including unissued neighbours. */ + __folio_mark_uptodate(folio); + vma = state->vma; + pte = pte_offset_map_lock(mm, state->pmd, state->dst_addr, &ptl); + err = -EAGAIN; + if (!pte) + goto unlock_folio; + err = -EFAULT; + if (mfill_file_over_size(vma, state->dst_addr)) + goto unlock_pte; + err = -EEXIST; + if (!pte_none(ptep_get(pte)) && !pte_is_uffd_marker(ptep_get(pte))) + goto unlock_pte; + fresh = !folio_test_anon(folio); + mm_subpage_set_exclusive(slot); + err = mm_subpage_add_new_anon_rmap(slot, vma, state->dst_addr); + if (err) + goto unlock_pte; + if (fresh) + folio_add_lru_vma(folio, vma); + else if ((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED) + mlock_folio(folio); + inc_mm_counter(mm, MM_ANONPAGES); + entry = phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot); + entry = pte_mkdirty(entry); + if (vma->vm_flags & VM_WRITE) + entry = pte_mkwrite(entry, vma); + if (state->flags & MFILL_ATOMIC_WP) + entry = pte_mkuffd_wp(entry); + set_pte_at(mm, state->dst_addr, pte, entry); + update_mmu_cache(vma, state->dst_addr, pte); + /* The PTE owns slot now; release the folio lock before dropping PTL. */ + folio_unlock(folio); + pte_unmap_unlock(pte, ptl); + return 0; +unlock_pte: + pte_unmap_unlock(pte, ptl); +unlock_folio: + folio_unlock(folio); +put_slot: + mm_subpage_put(slot); + return err; +} +#endif + +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) +static int mfill_atomic_shared_subpage(struct mfill_state *state) +{ + struct mfill_shared_data data = {}; + int err; + + if (uffd_flags_mode_is(state->flags, MFILL_ATOMIC_COPY)) { + if (!state->copy_buffer) + state->copy_buffer = folio_alloc(GFP_KERNEL_ACCOUNT, 0); + data.source = state->copy_buffer; + if (!data.source) + return -ENOMEM; + err = mfill_copy_folio_locked(data.source, state->src_addr, 0, + mm_page_size(state->ctx->mm)); + if (err) + err = mfill_copy_folio_retry(state, data.source, 0, + mm_page_size(state->ctx->mm)); + if (err) + return err; + } + return mfill_atomic_shmem_hole(state, &data); +} +#endif + static int mfill_atomic_pte_copy(struct mfill_state *state) { const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(state->ctx->mm) < PAGE_SIZE) { +#ifdef CONFIG_SHMEM + if (state->vma->vm_flags & VM_SHARED) + return mfill_atomic_shared_subpage(state); +#endif + return mfill_atomic_subpage(state); + } +#endif + /* * The normal page fault path for a MAP_PRIVATE mapping in a * file-backed VMA will invoke the fault, fill the hole in the file and @@ -632,6 +914,16 @@ static int mfill_atomic_pte_zeroed_folio(struct mfill_state *state) { const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(state->ctx->mm) < PAGE_SIZE) { +#ifdef CONFIG_SHMEM + if (state->vma->vm_flags & VM_SHARED) + return mfill_atomic_shared_subpage(state); +#endif + return mfill_atomic_subpage(state); + } +#endif + return __mfill_atomic_pte(state, ops); } @@ -695,6 +987,13 @@ static int mfill_atomic_pte_continue(struct mfill_state *state) if (IS_ERR_OR_NULL(folio)) return -EFAULT; +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + if (vma_is_shmem(dst_vma) && shmem_uffd_range_missing(folio, pgoff, + vma_page_offset_at(dst_vma, dst_addr).offset, mm_page_size(dst_vma->vm_mm))) { + ret = -EFAULT; + goto out_release; + } +#endif page = folio_file_page(folio, pgoff); if (PageHWPoison(page)) { ret = -EIO; @@ -702,7 +1001,7 @@ static int mfill_atomic_pte_continue(struct mfill_state *state) } ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, - page, flags); + page, flags, NULL); if (ret) goto out_release; @@ -964,8 +1263,8 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, /* * Sanitize the command parameters: */ - VM_WARN_ON_ONCE(dst_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(dst_start & ~mm_page_mask(ctx->mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(ctx->mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(src_start + len <= src_start); @@ -998,9 +1297,9 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, cond_resched(); if (!err) { - state.dst_addr += PAGE_SIZE; - state.src_addr += PAGE_SIZE; - copied += PAGE_SIZE; + state.dst_addr += mm_page_size(ctx->mm); + state.src_addr += mm_page_size(ctx->mm); + copied += mm_page_size(ctx->mm); if (fatal_signal_pending(current)) err = -EINTR; @@ -1011,6 +1310,8 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, mfill_put_vma(&state); out: + if (state.copy_buffer) + folio_put(state.copy_buffer); VM_WARN_ON_ONCE(copied < 0); VM_WARN_ON_ONCE(err > 0); VM_WARN_ON_ONCE(!copied && !err); @@ -1099,8 +1400,8 @@ int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, /* * Sanitize the command parameters: */ - VM_WARN_ON_ONCE(start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(start & ~mm_page_mask(dst_mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(dst_mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(start + len <= start); @@ -1303,7 +1604,102 @@ static long move_present_ptes(struct mm_struct *mm, return src_addr > src_start ? src_addr - src_start : err; } +#ifdef CONFIG_MM_SUBPAGE +static long move_present_subpage(struct mm_struct *mm, + struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, + unsigned long dst_addr, unsigned long src_addr, + pte_t *dst_pte, pte_t *src_pte, pte_t orig_dst, pte_t orig_src, + pmd_t *dst_pmd, pmd_t dst_pmdval, + spinlock_t *dst_ptl, spinlock_t *src_ptl, struct folio *folio) +{ + struct mm_subpage_move move; + struct mm_subpage *slot = NULL; + pte_t pte; + long ret = -EAGAIN; + + flush_cache_range(src_vma, src_addr, src_addr + mm_page_size(mm)); + double_pt_lock(dst_ptl, src_ptl); + if (!is_pte_pages_stable(dst_pte, src_pte, orig_dst, orig_src, + dst_pmd, dst_pmdval)) + goto out; + ret = -EBUSY; + if (!folio_test_anon(folio) || folio_test_large(folio)) + goto out; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, orig_src)); + if (!slot || mm_subpage_folio(slot) != folio) + goto out; + ret = mm_subpage_move_prepare(&move, slot, src_vma, src_addr, + dst_vma, dst_addr, false); + if (ret) + goto out; + orig_src = ptep_clear_flush(src_vma, src_addr, src_pte); + ret = mm_subpage_move_commit(&move); + if (ret) { + set_pte_at(mm, src_addr, src_pte, orig_src); + } else { + pte = phys_pte_mm(mm, mm_subpage_phys(slot), dst_vma->vm_page_prot); + if (pgtable_supports_soft_dirty()) + pte = pte_mksoft_dirty(pte); + if (pte_dirty(orig_src)) + pte = pte_mkdirty(pte); + pte = pte_mkwrite(pte, dst_vma); + set_pte_at(mm, dst_addr, dst_pte, pte); + ret = mm_page_size(mm); + } + mm_subpage_move_finish(&move); +out: + if (slot) + mm_subpage_put(slot); + double_pt_unlock(dst_ptl, src_ptl); + return ret; +} + +/* Both PTLs and, if present, the swap-cache folio lock are held. */ +static int move_swap_subpage(struct mm_struct *mm, struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, unsigned long dst_addr, + unsigned long src_addr, pte_t *dst_pte, pte_t *src_pte, + pte_t orig_src, struct folio *folio, swp_entry_t entry) +{ + unsigned int offset = pte_swp_subpage_offset(orig_src); + struct mm_subpage_move move; + struct mm_subpage *slot = NULL; + int ret = -EBUSY; + + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(mm))) + return -EFAULT; + if (swap_subpage_count(entry, offset) != 1) + return -EBUSY; + if (folio) { + slot = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(folio)) + offset); + if (slot) { + ret = mm_subpage_move_prepare(&move, slot, src_vma, src_addr, + dst_vma, dst_addr, true); + if (ret) + goto out; + ret = mm_subpage_move_commit(&move); + if (ret) { + mm_subpage_move_finish(&move); + goto out; + } + } + } + /* No mapping count changes: the same-mm exclusive swap PTE is transferred. */ + orig_src = ptep_get_and_clear(mm, src_addr, src_pte); + if (pgtable_supports_soft_dirty()) + orig_src = pte_swp_mksoft_dirty(orig_src); + set_pte_at(mm, dst_addr, dst_pte, orig_src); + if (slot) + mm_subpage_move_finish(&move); + ret = mm_page_size(mm); +out: + if (slot) + mm_subpage_put(slot); + return ret; +} +#endif + static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, unsigned long dst_addr, unsigned long src_addr, pte_t *dst_pte, pte_t *src_pte, pte_t orig_dst_pte, pte_t orig_src_pte, @@ -1329,6 +1725,21 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, return -EAGAIN; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + int ret; + + if (!src_folio && swap_cache_has_folio(entry)) { + double_pt_unlock(dst_ptl, src_ptl); + return -EAGAIN; + } + ret = move_swap_subpage(mm, dst_vma, src_vma, dst_addr, src_addr, + dst_pte, src_pte, orig_src_pte, src_folio, entry); + double_pt_unlock(dst_ptl, src_ptl); + return ret; + } +#endif + /* * The src_folio resides in the swapcache, requiring an update to its * index and mapping to align with the dst_vma, where a swap-in may @@ -1381,13 +1792,13 @@ static int move_zeropage_pte(struct mm_struct *mm, return -EAGAIN; } - zero_pte = pte_mkspecial(pfn_pte(zero_pfn(dst_addr), - dst_vma->vm_page_prot)); + zero_pte = pte_mkspecial(phys_pte_mm(mm, PFN_PHYS(zero_pfn(dst_addr)), + dst_vma->vm_page_prot)); ptep_clear_flush(src_vma, src_addr, src_pte); set_pte_at(mm, dst_addr, dst_pte, zero_pte); double_pt_unlock(dst_ptl, src_ptl); - return PAGE_SIZE; + return mm_page_size(mm); } @@ -1474,7 +1885,7 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES)) ret = -ENOENT; else /* nothing to do to move a hole */ - ret = PAGE_SIZE; + ret = mm_page_size(mm); goto out; } @@ -1514,7 +1925,8 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd } folio = vm_normal_folio(src_vma, src_addr, orig_src_pte); - if (!folio || !PageAnonExclusive(&folio->page)) { + if (!folio || (mm_page_size(mm) >= PAGE_SIZE && + !PageAnonExclusive(&folio->page))) { spin_unlock(src_ptl); ret = -EBUSY; goto out; @@ -1554,6 +1966,15 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd } } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + ret = move_present_subpage(mm, dst_vma, src_vma, dst_addr, src_addr, + dst_pte, src_pte, orig_dst_pte, orig_src_pte, + dst_pmd, dst_pmdval, dst_ptl, src_ptl, src_folio); + goto out; + } +#endif + /* at this point we have src_folio locked */ if (folio_test_large(src_folio)) { /* split_folio() can block */ @@ -1592,7 +2013,8 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd goto out; } - if (!pte_swp_exclusive(orig_src_pte)) { + /* Smaller leaves validate per-slot swap and mapping counts under PTL. */ + if (mm_page_size(mm) >= PAGE_SIZE && !pte_swp_exclusive(orig_src_pte)) { ret = -EBUSY; goto out; } @@ -1635,7 +2057,7 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd goto retry; } } - ret = move_swap_pte(mm, dst_vma, dst_addr, src_addr, dst_pte, src_pte, + ret = move_swap_pte(mm, dst_vma, src_vma, dst_addr, src_addr, dst_pte, src_pte, orig_dst_pte, orig_src_pte, dst_pmd, dst_pmdval, dst_ptl, src_ptl, src_folio, si, entry); } @@ -1694,6 +2116,11 @@ static int validate_move_areas(struct userfaultfd_ctx *ctx, pgprot_val(src_vma->vm_page_prot) != pgprot_val(dst_vma->vm_page_prot)) return -EINVAL; + /* Native backing cannot mix droppable and swap-backed slot policies. */ + if (mm_page_size(src_vma->vm_mm) < PAGE_SIZE && + ((src_vma->vm_flags ^ dst_vma->vm_flags) & VM_DROPPABLE)) + return -EINVAL; + /* Only allow moving if both are mlocked or both aren't */ if ((src_vma->vm_flags & VM_LOCKED) != (dst_vma->vm_flags & VM_LOCKED)) return -EINVAL; @@ -1926,9 +2353,9 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, ssize_t moved = 0; /* Sanitize the command parameters. */ - VM_WARN_ON_ONCE(src_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(dst_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(src_start & ~mm_page_mask(mm)); + VM_WARN_ON_ONCE(dst_start & ~mm_page_mask(mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(src_start + len < src_start); @@ -2053,10 +2480,12 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, ret = move_pages_ptes(mm, dst_pmd, src_pmd, dst_vma, src_vma, dst_addr, src_addr, src_end - src_addr, mode); - if (ret < 0) + if (ret < 0) { err = ret; - else + } else { + err = 0; step_size = ret; + } } cond_resched(); @@ -2095,6 +2524,17 @@ bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags, { const struct vm_uffd_ops *ops = vma_uffd_ops(vma); + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + return false; + if (vma_is_anonymous(vma)) { + if (vm_flags & VM_UFFD_MINOR) + return false; + } else if (!vma_is_shmem(vma)) { + return false; + } + } + if (vma->vm_flags & (VM_DROPPABLE | VM_SHADOW_STACK)) return false; diff --git a/mm/util.c b/mm/util.c index 3b26f5bf92012..5fd83790df88d 100644 --- a/mm/util.c +++ b/mm/util.c @@ -344,21 +344,22 @@ unsigned long randomize_stack_top(unsigned long stack_top) if (current->flags & PF_RANDOMIZE) { random_variable = get_random_long(); - random_variable &= STACK_RND_MASK; - random_variable <<= PAGE_SHIFT; + random_variable &= mm_aslr_mask(current->mm, STACK_RND_MASK); + random_variable <<= mm_page_shift(current->mm); } #ifdef CONFIG_STACK_GROWSUP - return PAGE_ALIGN(stack_top) + random_variable; + return mm_page_align(current->mm, stack_top) + random_variable; #else - return PAGE_ALIGN(stack_top) - random_variable; + return mm_page_align(current->mm, stack_top) - random_variable; #endif } /** - * randomize_page - Generate a random, page aligned address + * randomize_page_shift - Generate a random, granule aligned address * @start: The smallest acceptable address the caller will take. * @range: The size of the area, starting at @start, within which the * random address must fall. + * @shift: Log2 of the required address alignment. * * If @start + @range would overflow, @range is capped. * @@ -368,22 +369,31 @@ unsigned long randomize_stack_top(unsigned long stack_top) * Return: A page aligned address within [start, start + range). On error, * @start is returned. */ -unsigned long randomize_page(unsigned long start, unsigned long range) +static unsigned long randomize_page_shift(unsigned long start, + unsigned long range, unsigned int shift) { - if (!PAGE_ALIGNED(start)) { - range -= PAGE_ALIGN(start) - start; - start = PAGE_ALIGN(start); + unsigned long page_size = 1UL << shift; + + if (!IS_ALIGNED(start, page_size)) { + range -= ALIGN(start, page_size) - start; + start = ALIGN(start, page_size); } if (start > ULONG_MAX - range) range = ULONG_MAX - start; - range >>= PAGE_SHIFT; + range >>= shift; if (range == 0) return start; - return start + (get_random_long() % range << PAGE_SHIFT); + return start + (get_random_long() % range << shift); +} + +/* Existing callers retain native-page alignment. */ +unsigned long randomize_page(unsigned long start, unsigned long range) +{ + return randomize_page_shift(start, range, PAGE_SHIFT); } #ifdef CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT @@ -391,23 +401,25 @@ unsigned long __weak arch_randomize_brk(struct mm_struct *mm) { /* Is the current task 32bit ? */ if (!IS_ENABLED(CONFIG_64BIT) || is_compat_task()) - return randomize_page(mm->brk, SZ_32M); + return randomize_page_shift(mm->brk, SZ_32M, mm_page_shift(mm)); - return randomize_page(mm->brk, SZ_1G); + return randomize_page_shift(mm->brk, SZ_1G, mm_page_shift(mm)); } unsigned long arch_mmap_rnd(void) { - unsigned long rnd; + unsigned int bits; + unsigned long mask; #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS if (is_compat_task()) - rnd = get_random_long() & ((1UL << mmap_rnd_compat_bits) - 1); + bits = mmap_rnd_compat_bits; else #endif /* CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS */ - rnd = get_random_long() & ((1UL << mmap_rnd_bits) - 1); + bits = mmap_rnd_bits; - return rnd << PAGE_SHIFT; + mask = mm_aslr_mask(current->mm, (1UL << bits) - 1); + return (get_random_long() & mask) << mm_page_shift(current->mm); } static int mmap_is_legacy(const struct rlimit *rlim_stack) @@ -440,14 +452,15 @@ static unsigned long mmap_base(const unsigned long rnd, const struct rlimit *rli * task. mmap_base starts directly below the stack and grows * downwards. */ - return PAGE_ALIGN_DOWN(mmap_upper_limit(rlim_stack) - rnd); + return (mmap_upper_limit(rlim_stack) - rnd) & mm_page_mask(current->mm); #else unsigned long gap = rlim_stack->rlim_cur; unsigned long pad = stack_guard_gap; /* Account for stack randomization if necessary */ if (current->flags & PF_RANDOMIZE) - pad += (STACK_RND_MASK << PAGE_SHIFT); + pad += mm_aslr_mask(current->mm, STACK_RND_MASK) << + mm_page_shift(current->mm); /* Values close to RLIM_INFINITY can overflow. */ if (gap + pad > gap) @@ -458,7 +471,7 @@ static unsigned long mmap_base(const unsigned long rnd, const struct rlimit *rli else if (gap > MAX_GAP) gap = MAX_GAP; - return PAGE_ALIGN(STACK_TOP - gap - rnd); + return mm_page_align(current->mm, STACK_TOP - gap - rnd); #endif } @@ -491,7 +504,7 @@ EXPORT_SYMBOL_IF_KUNIT(arch_pick_mmap_layout); /** * __account_locked_vm - account locked pages to an mm's locked_vm * @mm: mm to account against - * @pages: number of pages to account + * @pages: number of native physical pages to account * @inc: %true if @pages should be considered positive, %false if not * @task: task used to check RLIMIT_MEMLOCK * @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped @@ -507,27 +520,35 @@ int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, const struct task_struct *task, bool bypass_rlim) { unsigned long locked_vm, limit; + unsigned int shift = PAGE_SHIFT - MM_ACCOUNT_SHIFT; int ret = 0; mmap_assert_write_locked(mm); + /* This exported API receives native physical-page counts. */ + if (pages > ULONG_MAX >> shift) + return -ENOMEM; + pages <<= shift; locked_vm = mm->locked_vm; if (inc) { if (!bypass_rlim) { - limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT; - if (locked_vm + pages > limit) + limit = task_rlimit(task, RLIMIT_MEMLOCK) >> MM_ACCOUNT_SHIFT; + if (pages > limit || locked_vm > limit - pages) ret = -ENOMEM; } + if (!ret && pages > ULONG_MAX - locked_vm) + ret = -ENOMEM; if (!ret) mm->locked_vm = locked_vm + pages; } else { - WARN_ON_ONCE(pages > locked_vm); + if (WARN_ON_ONCE(pages > locked_vm)) + return -EINVAL; mm->locked_vm = locked_vm - pages; } pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid, - (void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT, - locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK), + (void *)_RET_IP_, (inc) ? '+' : '-', pages << MM_ACCOUNT_SHIFT, + locked_vm << MM_ACCOUNT_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK), ret ? " - exceeded" : ""); return ret; @@ -562,11 +583,11 @@ int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc) } EXPORT_SYMBOL_GPL(account_locked_vm); -unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, +static unsigned long vm_mmap_offset_internal(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, - unsigned long flag, unsigned long pgoff) + unsigned long flag, struct vm_page_offset pos) { - loff_t off = (loff_t)pgoff << PAGE_SHIFT; + loff_t off = (loff_t)((pos.index << PAGE_SHIFT) | pos.offset); unsigned long ret; struct mm_struct *mm = current->mm; unsigned long populate; @@ -578,8 +599,11 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, if (!ret) { if (mmap_write_lock_killable(mm)) return -EINTR; - ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate, - &uf); +#ifdef CONFIG_MM_SUBPAGE + ret = do_mmap_offset(file, addr, len, prot, flag, 0, pos, &populate, &uf); +#else + ret = do_mmap(file, addr, len, prot, flag, 0, pos.index, &populate, &uf); +#endif mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate) @@ -588,6 +612,13 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, return ret; } +unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flag, unsigned long pgoff) +{ + return vm_mmap_offset_internal(file, addr, len, prot, flag, + (struct vm_page_offset) { pgoff, 0 }); +} + /* * Perform a userland memory mapping into the current process address space. See * the comment for do_mmap() for more details on this operation in general. @@ -609,12 +640,13 @@ unsigned long vm_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flag, unsigned long offset) { - if (unlikely(offset + PAGE_ALIGN(len) < offset)) + if (unlikely(offset + mm_page_align(current->mm, len) < offset)) return -EINVAL; - if (unlikely(offset_in_page(offset))) + if (unlikely(offset & ~mm_page_mask(current->mm))) return -EINVAL; - return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT); + return vm_mmap_offset_internal(file, addr, len, prot, flag, + (struct vm_page_offset) { offset >> PAGE_SHIFT, offset & ~PAGE_MASK }); } EXPORT_SYMBOL(vm_mmap); @@ -988,7 +1020,8 @@ int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin if (mm) { long reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10); - allowed -= min_t(long, mm->total_vm / 32, reserve); + allowed -= min_t(long, + mm_pages_to_bytes(mm, mm->total_vm) >> (PAGE_SHIFT + 5), reserve); } if (percpu_counter_read_positive(&vm_committed_as) < allowed) @@ -1189,6 +1222,9 @@ void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->end = vma->vm_end; desc->pgoff = vma->vm_pgoff; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = vma->vm_subpage_offset; +#endif desc->vm_file = vma->vm_file; desc->vma_flags = vma->flags; desc->page_prot = vma->vm_page_prot; diff --git a/mm/vma.c b/mm/vma.c index d90791b00a7b8..4161f7739a10e 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -13,7 +13,7 @@ struct mmap_state { unsigned long addr; unsigned long end; - pgoff_t pgoff; + struct vm_page_offset offset; unsigned long pglen; union { vm_flags_t vm_flags; @@ -42,19 +42,25 @@ struct mmap_state { bool file_doesnt_need_get :1; }; -#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, vma_flags_, file_) \ +#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pos_, vma_flags_, file_) \ struct mmap_state name = { \ .mm = mm_, \ .vmi = vmi_, \ .addr = addr_, \ .end = (addr_) + (len_), \ - .pgoff = pgoff_, \ - .pglen = PHYS_PFN(len_), \ + .offset = pos_, \ + .pglen = (len_) >> mm_page_shift(mm_), \ .vma_flags = vma_flags_, \ .file = file_, \ .page_prot = vma_get_page_prot(vma_flags_), \ } +#ifdef CONFIG_MM_SUBPAGE +#define VMG_MMAP_SUBPAGE_STATE(map_) .subpage_offset = (map_)->offset.offset, +#else +#define VMG_MMAP_SUBPAGE_STATE(map_) +#endif + #define VMG_MMAP_STATE(name, map_, vma_) \ struct vma_merge_struct name = { \ .mm = (map_)->mm, \ @@ -62,7 +68,8 @@ struct mmap_state { .start = (map_)->addr, \ .end = (map_)->end, \ .vma_flags = (map_)->vma_flags, \ - .pgoff = (map_)->pgoff, \ + .pgoff = (map_)->offset.index, \ + VMG_MMAP_SUBPAGE_STATE(map_) \ .file = (map_)->file, \ .prev = (map_)->prev, \ .middle = vma_, \ @@ -197,11 +204,12 @@ static void init_multi_vma_prep(struct vma_prepare *vp, */ static bool can_vma_merge_before(struct vma_merge_struct *vmg) { - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); + struct vm_page_offset end = vm_page_offset_add(vmg_page_offset(vmg), + vmg->end - vmg->start); if (is_mergeable_vma(vmg, /* merge_next = */ true) && is_mergeable_anon_vma(vmg, /* merge_next = */ true)) { - if (vmg->next->vm_pgoff == vmg->pgoff + pglen) + if (vm_page_offset_equal(vma_page_offset_at(vmg->next, vmg->next->vm_start), end)) return true; } @@ -221,7 +229,8 @@ static bool can_vma_merge_after(struct vma_merge_struct *vmg) { if (is_mergeable_vma(vmg, /* merge_next = */ false) && is_mergeable_anon_vma(vmg, /* merge_next = */ false)) { - if (vmg->prev->vm_pgoff + vma_pages(vmg->prev) == vmg->pgoff) + if (vm_page_offset_equal(vma_page_offset_at(vmg->prev, vmg->prev->vm_end), + vmg_page_offset(vmg))) return true; } return false; @@ -521,7 +530,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, new->vm_end = addr; } else { new->vm_start = addr; - new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); + vma_set_page_offset(new, vma_page_offset_at(vma, addr)); } err = -ENOMEM; @@ -560,7 +569,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, if (new_below) { vma->vm_start = addr; - vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT; + vma_set_page_offset(vma, vma_page_offset_at(new, addr)); } else { vma->vm_end = addr; } @@ -705,19 +714,21 @@ void validate_mm(struct mm_struct *mm) static void vmg_adjust_set_range(struct vma_merge_struct *vmg) { struct vm_area_struct *adjust; - pgoff_t pgoff; + struct vm_page_offset pos; if (vmg->__adjust_middle_start) { adjust = vmg->middle; - pgoff = adjust->vm_pgoff + PHYS_PFN(vmg->end - adjust->vm_start); + pos = vma_page_offset_at(adjust, vmg->end); } else if (vmg->__adjust_next_start) { adjust = vmg->next; - pgoff = adjust->vm_pgoff - PHYS_PFN(adjust->vm_start - vmg->end); + pos = vm_page_offset_sub(vma_page_offset_at(adjust, adjust->vm_start), + adjust->vm_start - vmg->end); } else { return; } - vma_set_range(adjust, vmg->end, adjust->vm_end, pgoff); + vma_set_range(adjust, vmg->end, adjust->vm_end, pos.index); + vma_set_page_offset(adjust, pos); } /* @@ -762,6 +773,7 @@ static int commit_merge(struct vma_merge_struct *vmg) vma_adjust_trans_huge(vma, vmg->start, vmg->end, vmg->__adjust_middle_start ? vmg->middle : NULL); vma_set_range(vma, vmg->start, vmg->end, vmg->pgoff); + vma_set_page_offset(vma, vmg_page_offset(vmg)); vmg_adjust_set_range(vmg); vma_iter_store_overwrite(vmg->vmi, vmg->target); @@ -931,7 +943,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( vmg->start = prev->vm_start; vmg->end = next->vm_end; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); /* * We already ensured anon_vma compatibility above, so now it's @@ -950,7 +962,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( */ vmg->start = prev->vm_start; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); if (!vmg->__remove_middle) vmg->__adjust_middle_start = true; @@ -965,7 +977,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( * shrink/delete extend */ - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); + unsigned long len = vmg->end - vmg->start; VM_WARN_ON_VMG(!merge_right, vmg); /* If we are offset into a VMA, then prev must be middle. */ @@ -973,13 +985,14 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( if (vmg->__remove_middle) { vmg->end = next->vm_end; - vmg->pgoff = next->vm_pgoff - pglen; + vmg_set_page_offset(vmg, + vm_page_offset_sub(vma_page_offset_at(next, next->vm_start), len)); } else { /* We shrink middle and expand next. */ vmg->__adjust_next_start = true; vmg->start = middle->vm_start; vmg->end = start; - vmg->pgoff = middle->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(middle, middle->vm_start)); } err = dup_anon_vma(next, middle, &anon_dup); @@ -1088,7 +1101,7 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg) if (can_merge_left) { vmg->start = prev->vm_start; vmg->target = prev; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); /* * If this merge would result in removal of the next VMA but we @@ -1245,7 +1258,7 @@ int vma_expand(struct vma_merge_struct *vmg) * Returns: 0 on success, -ENOMEM otherwise */ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff) + unsigned long start, unsigned long end, struct vm_page_offset pos) { struct vma_prepare vp; @@ -1266,7 +1279,8 @@ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_adjust_trans_huge(vma, start, end, NULL); vma_iter_clear(vmi); - vma_set_range(vma, start, end, pgoff); + vma_set_range(vma, start, end, pos.index); + vma_set_page_offset(vma, pos); vma_complete(&vp, vmi, vma->vm_mm); validate_mm(vma->vm_mm); return 0; @@ -1335,7 +1349,7 @@ static void vms_complete_munmap_vmas(struct vma_munmap_struct *vms, mm = current->mm; mm->map_count -= vms->vma_count; - mm->locked_vm -= vms->locked_vm; + mm->locked_vm -= mm_pages_to_account(mm, vms->locked_vm); if (vms->unlock) mmap_write_downgrade(mm); @@ -1360,7 +1374,7 @@ static void vms_complete_munmap_vmas(struct vma_munmap_struct *vms, mas_for_each(mas_detach, vma, ULONG_MAX) remove_vma(vma); - vm_unacct_memory(vms->nr_accounted); + mm_unacct_memory(mm, vms->nr_accounted); validate_mm(mm); if (vms->unlock) mmap_read_unlock(mm); @@ -1456,7 +1470,7 @@ static int vms_gather_munmap_vmas(struct vma_munmap_struct *vms, goto munmap_gather_failed; vma_mark_detached(next); - nrpages = vma_pages(next); + nrpages = vma_user_pages(next); vms->nr_pages += nrpages; if (vma_test(next, VMA_LOCKED_BIT)) @@ -1634,10 +1648,10 @@ int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long end; struct vm_area_struct *vma; - if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) + if ((start & ~mm_page_mask(mm)) || start > TASK_SIZE || len > TASK_SIZE-start) return -EINVAL; - end = start + PAGE_ALIGN(len); + end = start + mm_page_align(mm, len); if (end == start) return -EINVAL; @@ -1862,7 +1876,7 @@ static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) * prior to moving page table entries, to effect an mremap move. */ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - unsigned long addr, unsigned long len, pgoff_t pgoff, + unsigned long addr, unsigned long len, struct vm_page_offset pos, bool *need_rmap_locks) { struct vm_area_struct *vma = *vmap; @@ -1878,7 +1892,10 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, * to match new location, to increase its chance of merging. */ if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { - pgoff = addr >> PAGE_SHIFT; + pos = (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, + .offset = addr & ~PAGE_MASK, + }; faulted_in_anon_vma = false; } @@ -1894,7 +1911,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, if (new_vma && new_vma->vm_start < addr + len) return NULL; /* should never get here */ - vmg.pgoff = pgoff; + vmg_set_page_offset(&vmg, pos); vmg.next = vma_iter_next_rewind(&vmi, NULL); new_vma = vma_merge_copied_range(&vmg); @@ -1924,7 +1941,8 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, new_vma = vm_area_dup(vma); if (!new_vma) goto out; - vma_set_range(new_vma, addr, addr + len, pgoff); + vma_set_range(new_vma, addr, addr + len, pos.index); + vma_set_page_offset(new_vma, pos); if (vma_dup_policy(vma, new_vma)) goto out_free_vma; if (anon_vma_clone(new_vma, vma, VMA_OP_REMAP)) @@ -1979,7 +1997,8 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct * mpol_equal(vma_policy(a), vma_policy(b)) && a->vm_file == b->vm_file && vma_flags_empty(&diff) && - b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); + vm_page_offset_equal(vma_page_offset_at(a, b->vm_start), + vma_page_offset_at(b, b->vm_start)); } /* @@ -2398,7 +2417,10 @@ static void set_desc_from_map(struct vm_area_desc *desc, desc->start = map->addr; desc->end = map->end; - desc->pgoff = map->pgoff; + desc->pgoff = map->offset.index; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = map->offset.offset; +#endif desc->vm_file = map->file; desc->vma_flags = map->vma_flags; desc->page_prot = map->page_prot; @@ -2458,7 +2480,7 @@ static int __mmap_setup(struct mmap_state *map, struct vm_area_desc *desc, map->charged = map->pglen; map->charged -= vms->nr_accounted; if (map->charged) { - error = security_vm_enough_memory_mm(map->mm, map->charged); + error = mm_account_memory(map->mm, map->mm, map->charged); if (error) return error; } @@ -2549,7 +2571,8 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, return -ENOMEM; vma_iter_config(vmi, map->addr, map->end); - vma_set_range(vma, map->addr, map->end, map->pgoff); + vma_set_range(vma, map->addr, map->end, map->offset.index); + vma_set_page_offset(vma, map->offset); vma->flags = map->vma_flags; vma->vm_page_prot = map->page_prot; @@ -2621,7 +2644,7 @@ static void __mmap_complete(struct mmap_state *map, struct vm_area_struct *vma) if (!vma_supports_mlock(vma)) vma_clear_flags_mask(vma, VMA_LOCKED_MASK); else - mm->locked_vm += map->pglen; + mm->locked_vm += mm_pages_to_account(mm, map->pglen); } if (vma->vm_file) @@ -2678,7 +2701,10 @@ static int call_mmap_prepare(struct mmap_state *map, return err; /* Update fields permitted to be changed. */ - map->pgoff = desc->pgoff; + map->offset.index = desc->pgoff; +#ifdef CONFIG_MM_SUBPAGE + map->offset.offset = desc->subpage_offset; +#endif if (desc->vm_file != map->file) { map->file_doesnt_need_get = true; map->file = desc->vm_file; @@ -2689,6 +2715,12 @@ static int call_mmap_prepare(struct mmap_state *map, map->vm_ops = desc->vm_ops; map->vm_private_data = desc->private_data; +#ifdef CONFIG_MM_SUBPAGE + if (map->offset.offset >= PAGE_SIZE || + !IS_ALIGNED(map->offset.offset, mm_page_size(map->mm))) + return -EINVAL; +#endif + return 0; } @@ -2731,13 +2763,13 @@ static bool can_set_ksm_flags_early(struct mmap_state *map) static unsigned long __mmap_region(struct file *file, unsigned long addr, unsigned long len, vma_flags_t vma_flags, - unsigned long pgoff, struct list_head *uf) + struct vm_page_offset pos, struct list_head *uf) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma = NULL; bool have_mmap_prepare = file && file->f_op->mmap_prepare; VMA_ITERATOR(vmi, mm, addr); - MMAP_STATE(map, mm, &vmi, addr, len, pgoff, vma_flags, file); + MMAP_STATE(map, mm, &vmi, addr, len, pos, vma_flags, file); struct vm_area_desc desc = { .mm = mm, .file = file, @@ -2791,7 +2823,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, /* Accounting was done by __mmap_setup(). */ unacct_error: if (map.charged) - vm_unacct_memory(map.charged); + mm_unacct_memory(mm, map.charged); abort_munmap: /* * This indicates that .mmap_prepare has set a new file, differing from @@ -2807,7 +2839,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, /** * mmap_region() - Actually perform the userland mapping of a VMA into * current->mm with known, aligned and overflow-checked @addr and @len, and - * correctly determined VMA flags @vm_flags and page offset @pgoff. + * correctly determined VMA flags @vm_flags and file byte position @pos. * * This is an internal memory management function, and should not be used * directly. @@ -2819,8 +2851,8 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, * @addr: The page-aligned address at which to perform the mapping. * @len: The page-aligned, non-zero, length of the mapping. * @vm_flags: The VMA flags which should be applied to the mapping. - * @pgoff: If @file is specified, the page offset into the file, if not then - * the virtual page offset in memory of the anonymous mapping. + * @pos: Native page-cache index and byte offset within it. For anonymous + * mappings, the position is derived from the virtual address. * @uf: Optionally, a pointer to a list head used for tracking userfaultfd unmap * events. * @@ -2829,13 +2861,15 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, */ unsigned long mmap_region(struct file *file, unsigned long addr, unsigned long len, vm_flags_t vm_flags, - unsigned long pgoff, struct list_head *uf) + struct vm_page_offset pos, struct list_head *uf) { unsigned long ret; bool writable_file_mapping = false; const vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags); mmap_assert_write_locked(current->mm); + if (pos.offset >= PAGE_SIZE || !IS_ALIGNED(pos.offset, mm_page_size(current->mm))) + return -EINVAL; /* Check to see if MDWE is applicable. */ if (map_deny_write_exec(&vma_flags, &vma_flags)) @@ -2854,7 +2888,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, writable_file_mapping = true; } - ret = __mmap_region(file, addr, len, vma_flags, pgoff, uf); + ret = __mmap_region(file, addr, len, vma_flags, pos, uf); /* Clear our write mapping regardless of error. */ if (writable_file_mapping) @@ -2892,13 +2926,13 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_flags_set_mask(&vma_flags, mm->def_vma_flags); vma_flags = ksm_vma_flags(mm, NULL, vma_flags); - if (!may_expand_vm(mm, &vma_flags, len >> PAGE_SHIFT)) + if (!may_expand_vm(mm, &vma_flags, len >> mm_page_shift(mm))) return -ENOMEM; if (mm->map_count > get_sysctl_max_map_count()) return -ENOMEM; - if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) + if (mm_account_memory(mm, mm, len >> mm_page_shift(mm))) return -ENOMEM; /* @@ -2908,6 +2942,10 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, if (vma && vma->vm_end == addr) { VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, PHYS_PFN(addr)); + vmg_set_page_offset(&vmg, (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, .offset = addr & ~PAGE_MASK, + }); + vmg.prev = vma; /* vmi is positioned at prev, which this mode expects. */ vmg.just_expand = true; @@ -2927,6 +2965,9 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_set_anonymous(vma); vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT); + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, .offset = addr & ~PAGE_MASK, + }); vma->flags = vma_flags; vma->vm_page_prot = vm_get_page_prot(vma_flags_to_legacy(vma_flags)); vma_start_write(vma); @@ -2937,10 +2978,10 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, validate_mm(mm); out: perf_event_mmap(vma); - mm->total_vm += len >> PAGE_SHIFT; - mm->data_vm += len >> PAGE_SHIFT; + mm->total_vm += len >> mm_page_shift(mm); + mm->data_vm += len >> mm_page_shift(mm); if (vma_flags_test(&vma_flags, VMA_LOCKED_BIT)) - mm->locked_vm += (len >> PAGE_SHIFT); + mm->locked_vm += len >> MM_ACCOUNT_SHIFT; if (pgtable_supports_soft_dirty()) vma_set_flags(vma, VMA_SOFTDIRTY_BIT); return 0; @@ -2948,7 +2989,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, mas_store_fail: vm_area_free(vma); unacct_fail: - vm_unacct_memory(len >> PAGE_SHIFT); + mm_unacct_memory(mm, len >> mm_page_shift(mm)); return -ENOMEM; } @@ -3084,7 +3125,7 @@ static int acct_stack_growth(struct vm_area_struct *vma, /* mlock limit tests */ if (!mlock_future_ok(mm, vma_test(vma, VMA_LOCKED_BIT), - grow << PAGE_SHIFT)) + grow << mm_page_shift(mm))) return -ENOMEM; /* Check to ensure the stack will not grow into a hugetlb-only region */ @@ -3100,7 +3141,7 @@ static int acct_stack_growth(struct vm_area_struct *vma, * Overcommit.. This must be the final test, as it will * update security statistics. */ - if (security_vm_enough_memory_mm(mm, grow)) + if (mm_account_memory(mm, mm, grow)) return -ENOMEM; return 0; @@ -3125,10 +3166,10 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) mmap_assert_write_locked(mm); /* Guard against exceeding limits of the address space. */ - address &= PAGE_MASK; - if (address >= (TASK_SIZE & PAGE_MASK)) + address &= mm_page_mask(mm); + if (address >= (TASK_SIZE & mm_page_mask(mm))) return -ENOMEM; - address += PAGE_SIZE; + address += mm_page_size(mm); /* Enforce stack_guard_gap */ gap_addr = address + stack_guard_gap; @@ -3167,14 +3208,14 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) unsigned long size, grow; size = address - vma->vm_start; - grow = (address - vma->vm_end) >> PAGE_SHIFT; + grow = (address - vma->vm_end) >> mm_page_shift(mm); error = -ENOMEM; if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) - mm->locked_vm += grow; + mm->locked_vm += mm_pages_to_account(mm, grow); vm_stat_account(mm, vma->vm_flags, grow); anon_vma_interval_tree_pre_update_vma(vma); vma->vm_end = address; @@ -3209,7 +3250,7 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) mmap_assert_write_locked(mm); - address &= PAGE_MASK; + address &= mm_page_mask(mm); if (address < mmap_min_addr || address < FIRST_USER_ADDRESS) return -EPERM; @@ -3244,20 +3285,22 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) /* Somebody else might have raced and expanded it already */ if (address < vma->vm_start) { unsigned long size, grow; + struct vm_page_offset pos = vm_page_offset_sub( + vma_page_offset_at(vma, vma->vm_start), vma->vm_start - address); size = vma->vm_end - address; - grow = (vma->vm_start - address) >> PAGE_SHIFT; + grow = (vma->vm_start - address) >> mm_page_shift(mm); error = -ENOMEM; - if (grow <= vma->vm_pgoff) { + if (pos.index <= vma->vm_pgoff) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) - mm->locked_vm += grow; + mm->locked_vm += mm_pages_to_account(mm, grow); vm_stat_account(mm, vma->vm_flags, grow); anon_vma_interval_tree_pre_update_vma(vma); vma->vm_start = address; - vma->vm_pgoff -= grow; + vma_set_page_offset(vma, pos); /* Overwrite old entry in mtree. */ vma_iter_store_overwrite(&vmi, vma); anon_vma_interval_tree_post_update_vma(vma); @@ -3296,13 +3339,13 @@ int __vm_munmap(unsigned long start, size_t len, bool unlock) */ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) { - unsigned long charged = vma_pages(vma); + unsigned long charged = vma_user_pages(vma); if (find_vma_intersection(mm, vma->vm_start, vma->vm_end)) return -ENOMEM; if (vma_test(vma, VMA_ACCOUNT_BIT) && - security_vm_enough_memory_mm(mm, charged)) + mm_account_memory(mm, mm, charged)) return -ENOMEM; /* @@ -3319,12 +3362,15 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) */ if (vma_is_anonymous(vma)) { BUG_ON(vma->anon_vma); - vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = vma->vm_start >> PAGE_SHIFT, + .offset = vma->vm_start & ~PAGE_MASK, + }); } if (vma_link(mm, vma)) { if (vma_test(vma, VMA_ACCOUNT_BIT)) - vm_unacct_memory(charged); + mm_unacct_memory(mm, charged); return -ENOMEM; } diff --git a/mm/vma.h b/mm/vma.h index 8e4b61a7304c6..3d9c8ee4be05d 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -97,6 +97,9 @@ struct vma_merge_struct { unsigned long start; unsigned long end; pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif union { /* Temporary while VMA flags are being converted. */ @@ -234,9 +237,34 @@ static inline bool vmg_nomem(struct vma_merge_struct *vmg) static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, unsigned long addr) { - return vma->vm_pgoff + PHYS_PFN(addr - vma->vm_start); + return vma_page_offset_at(vma, addr).index; +} + +static inline struct vm_page_offset vmg_page_offset(const struct vma_merge_struct *vmg) +{ + return (struct vm_page_offset) { + .index = vmg->pgoff, +#ifdef CONFIG_MM_SUBPAGE + .offset = vmg->subpage_offset, +#endif + }; +} + +static inline void vmg_set_page_offset(struct vma_merge_struct *vmg, struct vm_page_offset pos) +{ + vmg->pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vmg->subpage_offset = pos.offset; +#endif } +#ifdef CONFIG_MM_SUBPAGE +#define VMG_SUBPAGE_STATE(vma, start) \ + .subpage_offset = vma_page_offset_at(vma, start).offset, +#else +#define VMG_SUBPAGE_STATE(vma, start) +#endif + #define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_) \ struct vma_merge_struct name = { \ .mm = mm_, \ @@ -259,6 +287,7 @@ static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, .end = end_, \ .vm_flags = vma_->vm_flags, \ .pgoff = vma_pgoff_offset(vma_, start_), \ + VMG_SUBPAGE_STATE(vma_, start_) \ .file = vma_->vm_file, \ .anon_vma = vma_->anon_vma, \ .policy = vma_policy(vma_), \ @@ -276,7 +305,7 @@ void validate_mm(struct mm_struct *mm); __must_check int vma_expand(struct vma_merge_struct *vmg); __must_check int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff); + unsigned long start, unsigned long end, struct vm_page_offset pos); static inline int vma_iter_store_gfp(struct vma_iterator *vmi, struct vm_area_struct *vma, gfp_t gfp) @@ -311,6 +340,9 @@ static inline void compat_set_vma_from_desc(struct vm_area_struct *vma, /* Mutable fields. Populated with initial state. */ vma->vm_pgoff = desc->pgoff; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = desc->subpage_offset; +#endif if (desc->vm_file != vma->vm_file) vma_set_file(vma, desc->vm_file); vma->flags = desc->vma_flags; @@ -448,7 +480,7 @@ void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, struct vm_area_struct *vma); struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - unsigned long addr, unsigned long len, pgoff_t pgoff, + unsigned long addr, unsigned long len, struct vm_page_offset pos, bool *need_rmap_locks); struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma); @@ -460,7 +492,7 @@ int mm_take_all_locks(struct mm_struct *mm); void mm_drop_all_locks(struct mm_struct *mm); unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, + unsigned long len, vm_flags_t vm_flags, struct vm_page_offset pos, struct list_head *uf); int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, diff --git a/mm/vma_exec.c b/mm/vma_exec.c index 5cee8b7efa0f2..8394f45a91113 100644 --- a/mm/vma_exec.c +++ b/mm/vma_exec.c @@ -42,6 +42,8 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) struct mmu_gather tlb; PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length); + vmg_set_page_offset(&vmg, vma_page_offset_at(vma, old_start)); + BUG_ON(new_start > new_end); /* @@ -89,7 +91,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) vma_prev(&vmi); /* Shrink the vma to just the new range */ - return vma_shrink(&vmi, vma, new_start, new_end, vma->vm_pgoff); + return vma_shrink(&vmi, vma, new_start, new_end, vma_page_offset_at(vma, vma->vm_start)); } /* @@ -138,7 +140,7 @@ int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, */ VM_WARN_ON_ONCE(VM_STACK_FLAGS & VM_STACK_INCOMPLETE_SETUP); vma->vm_end = STACK_TOP_MAX; - vma->vm_start = vma->vm_end - PAGE_SIZE; + vma->vm_start = vma->vm_end - mm_page_size(mm); if (pgtable_supports_soft_dirty()) flags |= VM_SOFTDIRTY; vm_flags_init(vma, flags); diff --git a/mm/vma_init.c b/mm/vma_init.c index 3c0b65950510e..a41ba35de7320 100644 --- a/mm/vma_init.c +++ b/mm/vma_init.c @@ -46,7 +46,10 @@ static void vm_area_init_from(const struct vm_area_struct *src, dest->vm_start = src->vm_start; dest->vm_end = src->vm_end; dest->anon_vma = src->anon_vma; - dest->vm_pgoff = src->vm_pgoff; + vma_set_page_offset(dest, (struct vm_page_offset) { + .index = src->vm_pgoff, + .offset = vma_subpage_offset(src), + }); dest->vm_file = src->vm_file; dest->vm_private_data = src->vm_private_data; vm_flags_init(dest, src->vm_flags); diff --git a/mm/vmscan.c b/mm/vmscan.c index 1af6c96d1bc14..fe28f70fd51b1 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3462,6 +3462,46 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, } } +static unsigned long lru_pte_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_page_size(mm) == PAGE_SIZE) + return pte_index(addr); + return (addr >> mm_page_shift(mm)) & (pte_table_bytes_mm(mm) / sizeof(pte_t) - 1); +} + +static unsigned long lru_pmd_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_page_size(mm) == PAGE_SIZE) + return pmd_index(addr); + return (addr >> pmd_shift_mm(mm)) & (pmd_table_bytes_mm(mm) / sizeof(pmd_t) - 1); +} + +static unsigned long lru_pud_entries(struct mm_struct *mm) +{ + return mm_pud_folded(mm) ? 1 : pud_table_bytes_mm(mm) / sizeof(pud_t); +} + +static unsigned long lru_pud_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_pud_folded(mm)) + return 0; + if (mm_page_size(mm) == PAGE_SIZE) + return pud_index(addr); + return (addr >> pud_shift_mm(mm)) & (lru_pud_entries(mm) - 1); +} + +static unsigned int lru_folio_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t *entry, unsigned int max_nr) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch_flags(vma, folio, ptep, entry, + max_nr, FPB_MERGE_YOUNG_DIRTY); +#endif + return folio_pte_batch_flags(folio, NULL, ptep, entry, max_nr, + FPB_MERGE_YOUNG_DIRTY); +} + static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, struct mm_walk *args) { @@ -3481,7 +3521,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, unsigned int nr; pmd_t pmdval; - pte = pte_offset_map_rw_nolock(args->mm, pmd, start & PMD_MASK, &pmdval, &ptl); + pte = pte_offset_map_rw_nolock(args->mm, pmd, start & pmd_mask_mm(args->mm), &pmdval, &ptl); if (!pte) return false; @@ -3497,7 +3537,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, lazy_mmu_mode_enable(); restart: - for (i = pte_index(start), addr = start; addr != end; i += nr, addr += nr * PAGE_SIZE) { + for (i = lru_pte_index(args->mm, start), addr = start; addr != end; i += nr, addr += nr * mm_page_size(args->mm)) { unsigned long pfn; struct folio *folio; pte_t *cur_pte = pte + i; @@ -3516,10 +3556,9 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, continue; if (folio_test_large(folio)) { - const unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + const unsigned int max_nr = (end - addr) >> mm_page_shift(args->mm); - nr = folio_pte_batch_flags(folio, NULL, cur_pte, &ptent, - max_nr, FPB_MERGE_YOUNG_DIRTY); + nr = lru_folio_pte_batch(args->vma, folio, cur_pte, &ptent, max_nr); total += nr - 1; walk->mm_stats[MM_LEAF_TOTAL] += nr - 1; } @@ -3544,7 +3583,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, walk_update_folio(walk, last, gen, dirty); last = NULL; - if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) + if (i < pte_table_bytes_mm(args->mm) / sizeof(pte_t) && + get_next_vma(pmd_mask_mm(args->mm), mm_page_size(args->mm), args, &start, &end)) goto restart; lazy_mmu_mode_disable(); @@ -3576,13 +3616,13 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area return; } - i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first); + i = addr == -1 ? 0 : lru_pmd_index(args->mm, addr) - lru_pmd_index(args->mm, *first); if (i && i <= MIN_LRU_BATCH) { __set_bit(i - 1, bitmap); return; } - pmd = pmd_offset(pud, *first); + pmd = pmd_offset_mm(args->mm, pud, *first); ptl = pmd_lockptr(args->mm, pmd); if (!spin_trylock(ptl)) @@ -3595,7 +3635,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area struct folio *folio; /* don't round down the first address */ - addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first; + addr = i ? (*first & pmd_mask_mm(args->mm)) + i * pmd_size_mm(args->mm) : *first; if (!pmd_present(pmd[i])) goto next; @@ -3661,14 +3701,14 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, * tables to avoid taking the PMD lock; the second, if necessary, takes * the PMD lock to clear the accessed bit in PMD entries. */ - pmd = pmd_offset(pud, start & PUD_MASK); + pmd = pmd_offset_mm(args->mm, pud, start & pud_mask_mm(args->mm)); restart: /* walk_pte_range() may call get_next_vma() */ vma = args->vma; - for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { + for (i = lru_pmd_index(args->mm, start), addr = start; addr != end; i++, addr = next) { pmd_t val = pmdp_get_lockless(pmd + i); - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(args->mm, addr, end); if (!pmd_present(val) || is_huge_zero_pmd(val)) { walk->mm_stats[MM_LEAF_TOTAL]++; @@ -3710,7 +3750,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first); - if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) + if (i < pmd_table_bytes_mm(args->mm) / sizeof(pmd_t) && + get_next_vma(pud_mask_mm(args->mm), pmd_size_mm(args->mm), args, &start, &end)) goto restart; } @@ -3725,12 +3766,12 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, VM_WARN_ON_ONCE(p4d_leaf(*p4d)); - pud = pud_offset(p4d, start & P4D_MASK); + pud = pud_offset_mm(args->mm, p4d, start & p4d_mask_mm(args->mm)); restart: - for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { + for (i = lru_pud_index(args->mm, start), addr = start; addr != end; i++, addr = next) { pud_t val = pudp_get(pud + i); - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(args->mm, addr, end); if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) continue; @@ -3738,15 +3779,16 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, walk_pmd_range(&val, addr, next, args); if (need_resched() || walk->batched >= MAX_LRU_BATCH) { - end = (addr | ~PUD_MASK) + 1; + end = (addr | ~pud_mask_mm(args->mm)) + 1; goto done; } } - if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) + if (i < lru_pud_entries(args->mm) && + get_next_vma(p4d_mask_mm(args->mm), pud_size_mm(args->mm), args, &start, &end)) goto restart; - end = round_up(end, P4D_SIZE); + end = round_up(end, p4d_size_mm(args->mm)); done: if (!end || !args->vma) return 1; @@ -4215,20 +4257,20 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) /* avoid taking the LRU lock under the PTL when possible */ walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; - start = max(addr & PMD_MASK, vma->vm_start); - end = min(addr | ~PMD_MASK, vma->vm_end - 1) + 1; + start = max(addr & pmd_mask_mm(vma->vm_mm), vma->vm_start); + end = min(addr | ~pmd_mask_mm(vma->vm_mm), vma->vm_end - 1) + 1; - if (end - start == PAGE_SIZE) + if (end - start == mm_page_size(vma->vm_mm)) return true; - if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { - if (addr - start < MIN_LRU_BATCH * PAGE_SIZE / 2) - end = start + MIN_LRU_BATCH * PAGE_SIZE; - else if (end - addr < MIN_LRU_BATCH * PAGE_SIZE / 2) - start = end - MIN_LRU_BATCH * PAGE_SIZE; + if (end - start > MIN_LRU_BATCH * mm_page_size(vma->vm_mm)) { + if (addr - start < MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2) + end = start + MIN_LRU_BATCH * mm_page_size(vma->vm_mm); + else if (end - addr < MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2) + start = end - MIN_LRU_BATCH * mm_page_size(vma->vm_mm); else { - start = addr - MIN_LRU_BATCH * PAGE_SIZE / 2; - end = addr + MIN_LRU_BATCH * PAGE_SIZE / 2; + start = addr - MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2; + end = addr + MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2; } } @@ -4240,10 +4282,10 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) lazy_mmu_mode_enable(); - pte -= (addr - start) / PAGE_SIZE; + pte -= (addr - start) / mm_page_size(vma->vm_mm); for (i = 0, addr = start; addr != end; - i += nr, pte += nr, addr += nr * PAGE_SIZE) { + i += nr, pte += nr, addr += nr * mm_page_size(vma->vm_mm)) { unsigned long pfn; pte_t ptent = ptep_get(pte); @@ -4257,10 +4299,9 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) continue; if (folio_test_large(folio)) { - const unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + const unsigned int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); - nr = folio_pte_batch_flags(folio, NULL, pte, &ptent, - max_nr, FPB_MERGE_YOUNG_DIRTY); + nr = lru_folio_pte_batch(vma, folio, pte, &ptent, max_nr); } if (!test_and_clear_young_ptes_notify(vma, addr, pte, nr)) @@ -4634,6 +4675,16 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c sc->nr.unqueued_dirty += delta; } + /* + * Legacy memcg reclaim relies on shrink_folio_list() waiting for + * writeback when reclaim-marked folios cycle through the LRU. Moving + * all of them to a younger generation can exhaust the charge limit + * before I/O completes. Let the common path apply its GFP and mapping + * deadlock checks instead of hiding those folios from it. + */ + if (writeback && !writeback_throttling_sane(sc)) + return false; + /* waiting for writeback */ if (writeback || (type == LRU_GEN_FILE && dirty)) { gen = folio_inc_gen(lruvec, folio, true); @@ -4668,8 +4719,9 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca if (!folio_test_referenced(folio)) set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0); - /* for shrink_folio_list() */ - folio_clear_reclaim(folio); + /* Preserve legacy memcg's repeated-writeback marker for shrink_folio_list(). */ + if (writeback_throttling_sane(sc) || !folio_test_writeback(folio)) + folio_clear_reclaim(folio); success = lru_gen_del_folio(lruvec, folio, true); VM_WARN_ON_ONCE_FOLIO(!success, folio); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index a5cdcc0fc55e1..0bc6467aae5d4 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -7347,8 +7348,113 @@ nodefer: kfree_skb_napi_cache(skb); kick_defer_list_purge(cpu); } +#ifdef CONFIG_MM_SUBPAGE +/* The ordinary frag page ref protects allocation, not an anonymous quarter. */ +struct skb_subpage_refs { + struct ubuf_info ubuf; + struct mm_subpage_refs *owner; +}; + +static void skb_subpage_complete(struct sk_buff *skb, struct ubuf_info *ubuf, + bool success) +{ + struct skb_subpage_refs *refs = container_of(ubuf, struct skb_subpage_refs, ubuf); + + if (!refcount_dec_and_test(&ubuf->refcnt)) + return; + mm_subpage_refs_put(refs->owner); + kfree(refs); +} + +static const struct ubuf_info_ops skb_subpage_ops = { + .complete = skb_subpage_complete, +}; + +static bool skb_contains_subpage(const struct sk_buff *skb, phys_addr_t phys, + unsigned int size, unsigned int depth) +{ + struct sk_buff *child; + unsigned int i; + + /* Match the networking recursion bound, conservatively keeping unknowns. */ + if (depth >= 24) + return true; + if (skb->head_frag && skb_headlen(skb)) { + phys_addr_t start = page_to_phys(virt_to_page(skb->data)) + offset_in_page(skb->data); + + if (start < phys + size && phys < start + skb_headlen(skb)) + return true; + } + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + struct page *page = skb_frag_page(frag); + phys_addr_t start; + + if (!page) + continue; + start = page_to_phys(page) + skb_frag_off(frag); + if (start < phys + size && phys < start + skb_frag_size(frag)) + return true; + } + skb_walk_frags(skb, child) + if (skb_contains_subpage(child, phys, size, depth + 1)) + return true; + return false; +} + +static bool skb_keeps_subpage(const struct mm_subpage *slot, void *context) +{ + return skb_contains_subpage(context, mm_subpage_phys(slot), mm_subpage_size(slot), 0); +} + +/* Caller owns the writable skb and keeps the source slot identity alive. */ +static int skb_splice_hold_subpages(struct sk_buff *skb, struct page *page, + unsigned int off, unsigned int len) +{ + struct skb_subpage_refs *refs, *new; + struct ubuf_info *old = skb_zcopy(skb); + struct mm_subpage *probe; + int ret; + + if (!len || folio_order(page_folio(page)) || !folio_test_anon(page_folio(page))) + return 0; + /* Preserve the allocation-free native anonymous path. */ + probe = mm_subpage_get_from_phys(page_to_phys(page) + round_down(off, MM_SUBPAGE_SIZE)); + if (!probe) + return 0; + mm_subpage_put(probe); + if (old && (skb_zcopy_is_nouarg(skb) || old->ops != &skb_subpage_ops)) + return -EOPNOTSUPP; + refs = old ? container_of(old, struct skb_subpage_refs, ubuf) : NULL; + if (!refs || refcount_read(&old->refcnt) != 1) { + new = kzalloc(sizeof(*new), GFP_ATOMIC); + if (!new) + return -ENOMEM; + new->ubuf.ops = &skb_subpage_ops; + new->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; + refcount_set(&new->ubuf.refcnt, 1); + if (refs) + new->owner = mm_subpage_refs_get(refs->owner); + skb_zcopy_init(skb, &new->ubuf); + net_zcopy_put(old); + refs = new; + } + /* Repeated trim/append must not accumulate references to discarded data. */ + ret = mm_subpage_refs_prune(&refs->owner, skb_keeps_subpage, skb, GFP_ATOMIC); + if (ret) + return ret; + return mm_subpage_refs_add(&refs->owner, page, off, len, GFP_ATOMIC); +} +#else +static int skb_splice_hold_subpages(struct sk_buff *skb, struct page *page, + unsigned int off, unsigned int len) +{ + return 0; +} +#endif + static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, - size_t offset, size_t len) + size_t offset, size_t len, size_t progress) { const char *kaddr; __wsum csum; @@ -7356,7 +7462,8 @@ static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, kaddr = kmap_local_page(page); csum = csum_partial(kaddr + offset, len, 0); kunmap_local(kaddr); - skb->csum = csum_block_add(skb->csum, csum, skb->len); + /* skb->len is updated only after the entire iterator batch. */ + skb->csum = csum_block_add(skb->csum, csum, skb->len + progress); } /** @@ -7408,6 +7515,12 @@ ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, if (WARN_ON_ONCE(!sendpage_ok(page))) goto out; + ret = skb_splice_hold_subpages(skb, page, off, part); + if (ret < 0) { + iov_iter_revert(iter, len); + goto out; + } + ret = skb_append_pagefrags(skb, page, off, part, frag_limit); if (ret < 0) { @@ -7416,7 +7529,7 @@ ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, } if (skb->ip_summed == CHECKSUM_NONE) - skb_splice_csum_page(skb, page, off, part); + skb_splice_csum_page(skb, page, off, part, spliced); off = 0; spliced += part; diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 30c3b9a2681c4..10eb255be0ffa 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */ +#include +#include #include #include #include @@ -24,6 +26,65 @@ static bool sk_msg_try_coalesce_ok(struct sk_msg *msg, int elem_first_coalesce) return false; } +#ifdef CONFIG_MM_SUBPAGE +static bool sk_msg_keeps_subpage(const struct mm_subpage *slot, void *context) +{ + struct sk_msg *msg = context; + phys_addr_t phys = mm_subpage_phys(slot); + phys_addr_t end = phys + mm_subpage_size(slot); + u32 i = msg->sg.start; + + if (!msg->sg.size) + return false; + while (i != msg->sg.end) { + struct scatterlist *sg = &msg->sg.data[i]; + + if (sg->length) { + phys_addr_t start = page_to_phys(sg_page(sg)) + sg->offset; + + if (start < end && phys < start + sg->length) + return true; + } + sk_msg_iter_var_next(i); + } + return false; +} + +int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len) +{ + int ret; + + ret = mm_subpage_refs_prune(&msg->subpage_refs, sk_msg_keeps_subpage, msg, GFP_ATOMIC); + if (ret) + return ret; + return mm_subpage_refs_add(&msg->subpage_refs, page, offset, len, GFP_ATOMIC); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_add); + +/* Called after a borrowed descriptor copy whose source remains live. */ +void sk_msg_subpages_get(struct sk_msg *msg) +{ + mm_subpage_refs_get(msg->subpage_refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_get); + +void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src) +{ + WARN_ON_ONCE(dst->subpage_refs); + dst->subpage_refs = mm_subpage_refs_get(src->subpage_refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_share); + +void sk_msg_subpages_release(struct sk_msg *msg) +{ + struct mm_subpage_refs *refs = msg->subpage_refs; + + msg->subpage_refs = NULL; + mm_subpage_refs_put(refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_release); +#endif + int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, int elem_first_coalesce) { @@ -88,7 +149,7 @@ EXPORT_SYMBOL_GPL(sk_msg_alloc); int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, u32 off, u32 len) { - int i = src->sg.start; + int i = src->sg.start, ret; struct scatterlist *sge = sk_msg_elem(src, i); struct scatterlist *sgd = NULL; u32 sge_len, sge_off; @@ -114,11 +175,16 @@ int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, if (sgd && (sg_page(sge) == sg_page(sgd)) && (sg_virt(sge) + off == sg_virt(sgd) + sgd->length)) { + ret = sk_msg_subpages_add(dst, sg_page(sge), sge->offset + off, sge_len); + if (ret) + return ret; sgd->length += sge_len; dst->sg.size += sge_len; } else if (!sk_msg_full(dst)) { sge_off = sge->offset + off; - sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off); + ret = sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off); + if (ret) + return ret; } else { return -ENOSPC; } @@ -205,6 +271,7 @@ static int __sk_msg_free(struct sock *sk, struct sk_msg *msg, u32 i, sk_msg_check_to_free(msg, i, msg->sg.size); sge = sk_msg_elem(msg, i); } + sk_msg_subpages_release(msg); consume_skb(msg->skb); sk_msg_init(msg); return freed; @@ -248,6 +315,7 @@ static void __sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, sk_msg_check_to_free(msg, i, bytes); } msg->sg.start = i; + sk_msg_subpages_drop_empty(msg); } void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes) @@ -267,6 +335,7 @@ void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len) int trim = msg->sg.size - len; u32 i = msg->sg.end; + sk_msg_subpages_drop_empty(msg); if (trim <= 0) { WARN_ON(trim < 0); return; @@ -299,6 +368,7 @@ void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len) * does not require an update. */ if (!msg->sg.size) { + sk_msg_subpages_release(msg); msg->sg.curr = msg->sg.start; msg->sg.copybreak = 0; } else if (sk_msg_iter_dist(msg->sg.start, msg->sg.curr) >= @@ -334,11 +404,19 @@ int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from, goto out; } - bytes -= copied; - msg->sg.size += copied; - while (copied) { use = min_t(int, copied, PAGE_SIZE - offset); + ret = sk_msg_subpages_add(msg, pages[i], offset, use); + if (ret) { + unsigned int nr = DIV_ROUND_UP(offset + copied, PAGE_SIZE); + + while (nr--) + put_page(pages[i++]); + iov_iter_revert(from, copied); + goto out; + } + bytes -= use; + msg->sg.size += use; sg_set_page(&msg->sg.data[msg->sg.end], pages[i], use, offset); sg_unmark_end(&msg->sg.data[msg->sg.end]); diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index 18b45950d1acc..d082be17d76fc 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -44,6 +44,7 @@ static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, return -ENOMEM; lock_sock(sk); + sk_msg_subpages_share(tmp, msg); tmp->sg.start = msg->sg.start; i = msg->sg.start; do { @@ -76,6 +77,7 @@ static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, if (!ret) { msg->sg.start = i; + sk_msg_subpages_drop_empty(msg); if (!sk_psock_queue_msg(psock, tmp)) atomic_sub(copied, &sk->sk_rmem_alloc); sk_psock_data_ready(sk, psock); @@ -147,6 +149,7 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, break; } + sk_msg_subpages_drop_empty(msg); return 0; } @@ -434,6 +437,8 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, } } memcpy(psock->cork, msg, sizeof(*msg)); + if (psock->cork != msg) + sk_msg_subpages_transfer(psock->cork, msg); return 0; } diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 33c42f65364a2..10da3de523b93 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -32,6 +32,8 @@ #include #include #include +#include +#include #include #include #include @@ -144,6 +146,10 @@ static void destroy_record(struct tls_record_info *record) for (i = 0; i < record->num_frags; i++) __skb_frag_unref(&record->frags[i], false); + +#ifdef CONFIG_MM_SUBPAGE + mm_subpage_refs_put(record->subpage_refs); +#endif kfree(record); } @@ -349,6 +355,9 @@ static int tls_create_new_record(struct tls_offload_context_tx *offload_ctx, record = kmalloc_obj(*record); if (!record) return -ENOMEM; +#ifdef CONFIG_MM_SUBPAGE + record->subpage_refs = NULL; +#endif frag = &record->frags[0]; skb_frag_fill_page_desc(frag, pfrag->page, pfrag->offset, @@ -513,6 +522,15 @@ static int tls_push_data(struct sock *sk, goto handle_error; } +#ifdef CONFIG_MM_SUBPAGE + rc = mm_subpage_refs_add(&record->subpage_refs, zc_pfrag.page, + off, copy, GFP_KERNEL); + if (rc) { + iov_iter_revert(iter, copy); + goto handle_error; + } +#endif + zc_pfrag.offset = off; zc_pfrag.size = copy; tls_append_frag(record, &zc_pfrag, copy); @@ -1118,6 +1136,10 @@ int tls_set_device_offload(struct sock *sk) goto release_netdev; } + +#ifdef CONFIG_MM_SUBPAGE + start_marker_record->subpage_refs = NULL; +#endif offload_ctx = alloc_offload_ctx_tx(ctx); if (!offload_ctx) { rc = -ENOMEM; diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index ca0f657121a3b..f4f708533f79b 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -35,8 +35,11 @@ * SOFTWARE. */ +#include #include #include +#include +#include #include #include #include @@ -65,9 +68,36 @@ struct tls_decrypt_ctx { u8 aad[TLS_MAX_AAD_SIZE]; u8 tail; bool free_sgout; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *fragments; + unsigned int nr_fragments; + bool fragments_dirty; + struct llist_node cleanup; + void *allocation; +#endif struct scatterlist sg[]; }; +#if IS_ENABLED(CONFIG_KUNIT) && IS_ENABLED(CONFIG_MM_SUBPAGE) +/* Explicit test-only async provider selection; normal cipher selection stays intact. */ +static bool user4k_test_async; +module_param_named(user4k_test_async, user4k_test_async, bool, 0400); +MODULE_PARM_DESC(user4k_test_async, "KUnit: select cryptd for TLS receive testing"); +static atomic_long_t user4k_rx_deferred = ATOMIC_LONG_INIT(0); +static atomic_long_t user4k_rx_released = ATOMIC_LONG_INIT(0); + +static int user4k_read_count(char *buffer, const struct kernel_param *param) +{ + return scnprintf(buffer, PAGE_SIZE, "%ld\n", atomic_long_read(param->arg)); +} + +static const struct kernel_param_ops user4k_count_ops = { + .get = user4k_read_count, +}; +module_param_cb(user4k_rx_deferred, &user4k_count_ops, &user4k_rx_deferred, 0444); +module_param_cb(user4k_rx_released, &user4k_count_ops, &user4k_rx_released, 0444); +#endif + noinline void tls_err_abort(struct sock *sk, int err) { WARN_ON_ONCE(err >= 0); @@ -184,6 +214,35 @@ static int tls_padding_length(struct tls_prot_info *prot, struct sk_buff *skb, return sub; } +#ifdef CONFIG_MM_SUBPAGE +static void tls_release_user_fragments(struct tls_decrypt_ctx *dctx) +{ + release_user_fragments(dctx->fragments, dctx->nr_fragments, dctx->fragments_dirty); + kfree(dctx->fragments); + dctx->fragments = NULL; + dctx->nr_fragments = 0; +} +#endif + +/* Dirty pin release can take folio locks, so it belongs to the waiting task. */ +static void tls_finish_subpage_decrypts(struct tls_sw_context_rx *ctx) +{ +#ifdef CONFIG_MM_SUBPAGE + struct tls_decrypt_ctx *dctx, *next; + struct llist_node *head = llist_del_all(&ctx->subpage_cleanup); + + llist_for_each_entry_safe(dctx, next, head, cleanup) { + void *allocation = dctx->allocation; + + tls_release_user_fragments(dctx); +#if IS_ENABLED(CONFIG_KUNIT) + atomic_long_inc(&user4k_rx_released); +#endif + kfree(allocation); + } +#endif +} + static void tls_decrypt_done(void *data, int err) { struct aead_request *aead_req = data; @@ -196,6 +255,7 @@ static void tls_decrypt_done(void *data, int err) unsigned int pages; struct sock *sk; int aead_size; + bool deferred = false; /* If requests get too backlogged crypto API returns -EBUSY and calls * ->complete(-EINPROGRESS) immediately followed by ->complete(0) @@ -224,17 +284,29 @@ static void tls_decrypt_done(void *data, int err) tls_err_abort(sk, err); } - /* Free the destination pages if skb was not decrypted inplace */ - if (dctx->free_sgout) { - /* Skip the first S/G entry as it points to AAD */ - for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) { - if (!sg) - break; - put_page(sg_page(sg)); +#ifdef CONFIG_MM_SUBPAGE + if (dctx->fragments) { + dctx->allocation = aead_req; +#if IS_ENABLED(CONFIG_KUNIT) + atomic_long_inc(&user4k_rx_deferred); +#endif + llist_add(&dctx->cleanup, &ctx->subpage_cleanup); + deferred = true; + } +#endif + if (!deferred) { + /* Free the destination pages if skb was not decrypted inplace */ + if (dctx->free_sgout) { + /* Skip the first S/G entry as it points to AAD */ + for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) { + if (!sg) + break; + put_page(sg_page(sg)); + } } - } - kfree(aead_req); + kfree(aead_req); + } if (atomic_dec_and_test(&ctx->decrypt_pending)) complete(&ctx->async_wait.completion); @@ -246,6 +318,7 @@ static int tls_decrypt_async_wait(struct tls_sw_context_rx *ctx) crypto_wait_req(-EINPROGRESS, &ctx->async_wait); atomic_inc(&ctx->decrypt_pending); + tls_finish_subpage_decrypts(ctx); __skb_queue_purge(&ctx->async_hold); return ctx->async_wait.err; } @@ -670,6 +743,7 @@ static int tls_split_open_record(struct sock *sk, struct tls_rec *from, msg_opl->sg.size = bytes; msg_npl = &new->msg_plaintext; + sk_msg_subpages_share(msg_npl, msg_opl); msg_npl->apply_bytes = apply; msg_npl->sg.size = orig_size - bytes; @@ -733,6 +807,7 @@ static void tls_merge_open_record(struct sock *sk, struct tls_rec *to, sk_msg_free(sk, &to->msg_encrypted); sk_msg_xfer_full(&to->msg_encrypted, &from->msg_encrypted); + sk_msg_subpages_release(msg_npl); kfree(from); } @@ -935,12 +1010,14 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, redir_ingress = psock->redir_ingress; sk_redir = psock->sk_redir; memcpy(&msg_redir, msg, sizeof(*msg)); + sk_msg_subpages_get(&msg_redir); if (msg->apply_bytes < send) msg->apply_bytes = 0; else msg->apply_bytes -= send; sk_msg_return_zero(sk, msg, send); msg->sg.size -= send; + sk_msg_subpages_drop_empty(msg); release_sock(sk); err = tcp_bpf_sendmsg_redir(sk_redir, redir_ingress, &msg_redir, send, flags); @@ -956,6 +1033,7 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, *copied -= sk_msg_free_nocharge(sk, &msg_redir); msg->sg.size = 0; } + sk_msg_subpages_release(&msg_redir); if (msg->sg.size == 0) tls_free_open_rec(sk); break; @@ -1036,7 +1114,14 @@ static int tls_sw_sendmsg_splice(struct sock *sk, struct msghdr *msg, return -EIO; } - sk_msg_page_add(msg_pl, page, part, off); + { + int ret = sk_msg_page_add(msg_pl, page, part, off); + + if (ret) { + iov_iter_revert(&msg->msg_iter, part); + return ret; + } + } msg_pl->sg.copybreak = 0; msg_pl->sg.curr = msg_pl->sg.end; sk_mem_charge(sk, part); @@ -1454,10 +1539,96 @@ tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock, return 1; } +#ifdef CONFIG_MM_SUBPAGE +static bool tls_subpage_iter(const struct iov_iter *iter) +{ + return user_backed_iter(iter) && current->mm && mm_page_size(current->mm) < PAGE_SIZE; +} +#endif + +static int tls_iter_sg_capacity(struct iov_iter *iter, size_t length) +{ +#ifdef CONFIG_MM_SUBPAGE + if (tls_subpage_iter(iter)) { + struct iov_iter walk = *iter; + unsigned int count = 0; + + length = min(length, iov_iter_count(iter)); + while (length) { + size_t bytes, offset; + + iov_iter_advance(&walk, 0); + bytes = min(length, iter_iov_len(&walk)); + if (WARN_ON_ONCE(!bytes)) + return INT_MAX; + offset = (unsigned long)iter_iov_addr(&walk) & (MM_SUBPAGE_SIZE - 1); + count += DIV_ROUND_UP(offset + bytes, MM_SUBPAGE_SIZE); + iov_iter_advance(&walk, bytes); + length -= bytes; + } + return count; + } +#endif + return iov_iter_npages_cap(iter, INT_MAX, length); +} + +#ifdef CONFIG_MM_SUBPAGE +static int tls_setup_subpage_iter(struct iov_iter *from, int length, int *pages_used, + struct scatterlist *to, int capacity, + struct tls_decrypt_ctx *dctx) +{ + unsigned int used = 0, total = 0; + int ret = 0; + + if (!length) + return 0; + if (WARN_ON_ONCE(*pages_used || capacity <= 0)) + return -EINVAL; + dctx->fragments = kmalloc_array(capacity, sizeof(*dctx->fragments), GFP_KERNEL); + if (!dctx->fragments) + return -ENOMEM; + while (length) { + long nr; + unsigned int i, bytes = 0; + + iov_iter_advance(from, 0); + nr = pin_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(from), + min_t(size_t, length, iter_iov_len(from)), FOLL_WRITE, + dctx->fragments + used, capacity - used); + if (nr <= 0) { + ret = nr ?: -EFAULT; + goto out; + } + for (i = 0; i < nr; i++) { + struct user_page_fragment *fragment = &dctx->fragments[used]; + + sg_set_page(&to[used], + folio_page(fragment->folio, fragment->offset >> PAGE_SHIFT), + fragment->length, offset_in_page(fragment->offset)); + sg_unmark_end(&to[used]); + bytes += fragment->length; + used++; + } + dctx->nr_fragments = used; + iov_iter_advance(from, bytes); + length -= bytes; + total += bytes; + } + if (used) + sg_mark_end(&to[used - 1]); +out: + if (ret) + iov_iter_revert(from, total); + *pages_used = used; + return ret; +} +#endif + static int tls_setup_from_iter(struct iov_iter *from, int length, int *pages_used, struct scatterlist *to, - int to_max_pages) + int to_max_pages, struct tls_decrypt_ctx *dctx) { int rc = 0, i = 0, num_elem = *pages_used, maxpages; struct page *pages[MAX_SKB_FRAGS]; @@ -1465,6 +1636,10 @@ static int tls_setup_from_iter(struct iov_iter *from, ssize_t copied, use; size_t offset; +#ifdef CONFIG_MM_SUBPAGE + if (tls_subpage_iter(from)) + return tls_setup_subpage_iter(from, length, pages_used, to, to_max_pages, dctx); +#endif while (length > 0) { i = 0; maxpages = to_max_pages - num_elem; @@ -1582,7 +1757,7 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, if (out_iov) n_sgout = 1 + tail_pages + - iov_iter_npages_cap(out_iov, INT_MAX, data_len); + tls_iter_sg_capacity(out_iov, data_len); else n_sgout = sg_nents(out_sg); } else { @@ -1615,6 +1790,11 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, aead_req = (struct aead_request *)mem; dctx = (struct tls_decrypt_ctx *)(mem + aead_size); dctx->sk = sk; +#ifdef CONFIG_MM_SUBPAGE + dctx->fragments = NULL; + dctx->nr_fragments = 0; + dctx->fragments_dirty = false; +#endif sgin = &dctx->sg[0]; sgout = &dctx->sg[n_sgin]; @@ -1672,7 +1852,7 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, sg_set_buf(&sgout[0], dctx->aad, prot->aad_size); err = tls_setup_from_iter(out_iov, data_len, &pages, &sgout[1], - (n_sgout - 1 - tail_pages)); + (n_sgout - 1 - tail_pages), dctx); if (err < 0) goto exit_free_pages; @@ -1687,6 +1867,10 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, } dctx->free_sgout = !!pages; +#ifdef CONFIG_MM_SUBPAGE + /* An error from crypto may still leave writes in the destination. */ + dctx->fragments_dirty = true; +#endif /* Prepare and submit AEAD request */ err = tls_do_decryption(sk, sgin, sgout, dctx->iv, data_len + prot->tail_size, aead_req, darg); @@ -1715,6 +1899,12 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, darg->tail = dctx->tail; exit_free_pages: +#ifdef CONFIG_MM_SUBPAGE + if (dctx->fragments) { + tls_release_user_fragments(dctx); + pages = 0; + } +#endif /* Release the pages in case iov was mapped to pages */ for (; pages > 0; pages--) put_page(sg_page(&sgout[pages])); @@ -2795,6 +2985,9 @@ static struct tls_sw_context_rx *init_ctx_rx(struct tls_context *ctx) crypto_init_wait(&sw_ctx_rx->async_wait); atomic_set(&sw_ctx_rx->decrypt_pending, 1); +#ifdef CONFIG_MM_SUBPAGE + init_llist_head(&sw_ctx_rx->subpage_cleanup); +#endif init_waitqueue_head(&sw_ctx_rx->wq); skb_queue_head_init(&sw_ctx_rx->rx_list); skb_queue_head_init(&sw_ctx_rx->async_hold); @@ -2903,7 +3096,24 @@ int tls_set_sw_offload(struct sock *sk, int tx, rec_seq = crypto_info_rec_seq(src_crypto_info, cipher_desc); if (!*aead) { - *aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0); +#if IS_ENABLED(CONFIG_KUNIT) && IS_ENABLED(CONFIG_MM_SUBPAGE) + if (!tx && user4k_test_async) { + struct crypto_aead *sync; + char name[CRYPTO_MAX_ALG_NAME]; + + /* cryptd instances are named for the concrete child driver. */ + sync = crypto_alloc_aead(cipher_desc->cipher_name, 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(sync)) { + *aead = sync; + } else { + snprintf(name, sizeof(name), "cryptd(%s)", + crypto_tfm_alg_driver_name(crypto_aead_tfm(sync))); + *aead = crypto_alloc_aead(name, CRYPTO_ALG_ASYNC, CRYPTO_ALG_ASYNC); + crypto_free_aead(sync); + } + } else +#endif + *aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0); if (IS_ERR(*aead)) { rc = PTR_ERR(*aead); *aead = NULL; diff --git a/tools/aurora-userpages/.gitignore b/tools/aurora-userpages/.gitignore new file mode 100644 index 0000000000000..a27b2bf8b140d --- /dev/null +++ b/tools/aurora-userpages/.gitignore @@ -0,0 +1 @@ +/userpages diff --git a/tools/aurora-userpages/Makefile b/tools/aurora-userpages/Makefile new file mode 100644 index 0000000000000..c6d42e2d15efa --- /dev/null +++ b/tools/aurora-userpages/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +CC ?= cc +CFLAGS ?= -O2 -g +CFLAGS += -Wall -Wextra +LDFLAGS += -Wl,-z,max-page-size=65536 + +.PHONY: all clean +all: userpages + +userpages: userpages.c ../../include/uapi/linux/prctl.h + $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $< $(LDLIBS) -o $@ + +clean: + rm -f userpages diff --git a/tools/aurora-userpages/README.rst b/tools/aurora-userpages/README.rst new file mode 100644 index 0000000000000..64f4fcb0dd12a --- /dev/null +++ b/tools/aurora-userpages/README.rst @@ -0,0 +1,34 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Per-process userspace page-size launcher +======================================= + +Build with ``make`` in this directory. The running arm64 kernel must provide +Aurora's experimental per-process granule API. + +Examples:: + + userpages 4k FEXInterpreter ./x86-program + userpages 4k python3 script.py + userpages native program + userpages --status + +The choice becomes the calling thread's inherited exec preference: it applies +to the launched program and subsequent execs in its descendants. Fork retains +the address space and both exec preferences. Other threads and processes are +unchanged. ``native`` clears the inherited alternative preference. Ordinary +processes that never opt in continue to use the kernel's native default. + +``--once`` changes only the next successful exec; an existing inherited +preference remains available for later execs. For example, +``userpages --once native program`` runs one program with native pages and +retains any inherited 4K preference for its subsequent execs. Failed exec does +not consume the pending request. No selection changes a live address space. +AArch32 programs always use 4K when supported, otherwise native pages, +regardless of either preference. + +The kernel checks supported sizes and CPU capabilities. The current branch +supports 4K userspace on 16K/64K kernels and 16K userspace on 64K kernels. +64K userspace on a 16K kernel is not enabled. This launcher uses the private +PR_AURORA_{SET,GET}_{EXEC,DEFAULT}_PAGE_SIZE API in this tree; +it is not an upstream Linux ABI. diff --git a/tools/aurora-userpages/userpages.c b/tools/aurora-userpages/userpages.c new file mode 100644 index 0000000000000..02d9e4357224d --- /dev/null +++ b/tools/aurora-userpages/userpages.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include "../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include + +static int usage(const char *name, int status) +{ + fprintf(status ? stderr : stdout, + "Usage: %s [--once] {native|4k|16k|64k} COMMAND [ARG ...]\n" + " %s --status\n" + "Select the userspace page size for this program and its descendants.\n", + name, name); + return status; +} + +int main(int argc, char **argv) +{ + unsigned long size; + int once = argc > 1 && !strcmp(argv[1], "--once"); + int first = once ? 2 : 1; + + if (argc == 2 && !strcmp(argv[1], "--help")) + return usage(argv[0], 0); + if (argc == 2 && !strcmp(argv[1], "--status")) { + int pending = prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); + int error = errno; + + printf("Current process page size: %lu bytes\n", getauxval(AT_PAGESZ)); + if (pending < 0) { + fprintf(stderr, "Page-size selection unavailable: %s\n", strerror(error)); + return 1; + } + int inherited = prctl(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); + + if (inherited < 0) { + perror("Cannot read inherited page-size preference"); + return 1; + } + if (inherited) + printf("Inherited exec preference: %d bytes\n", inherited); + else + puts("Inherited exec preference: native default"); + if (pending) + printf("Next exec page size: %d bytes\n", pending); + else + puts("Next exec page size: inherited preference"); + return 0; + } + if (argc < first + 2) + return usage(argv[0], 2); + if (!strcmp(argv[first], "native")) + size = 0; + else if (!strcmp(argv[first], "4k")) + size = 4096; + else if (!strcmp(argv[first], "16k")) + size = 16384; + else if (!strcmp(argv[first], "64k")) + size = 65536; + else + return usage(argv[0], 2); + + if (!once && prctl(PR_AURORA_SET_DEFAULT_PAGE_SIZE, size, 0UL, 0UL, 0UL)) { + fprintf(stderr, "Cannot set inherited %s pages: %s\n", + argv[first], strerror(errno)); + return 1; + } + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL)) { + fprintf(stderr, "Cannot select %s userspace pages: %s\n", + argv[first], strerror(errno)); + return 1; + } + execvp(argv[first + 1], &argv[first + 1]); + int error = errno; + + fprintf(stderr, "Cannot execute %s: %s\n", argv[first + 1], strerror(error)); + return error == ENOENT ? 127 : 126; +} diff --git a/tools/testing/kunit/configs/arm64_user4k.config b/tools/testing/kunit/configs/arm64_user4k.config new file mode 100644 index 0000000000000..de6c8ec2fdff8 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k.config @@ -0,0 +1,33 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_KUNIT_TEST=y +CONFIG_ARM64_SW_TTBR0_PAN=y +# CONFIG_ARM64_CNP is not set +# CONFIG_ARM64_POE is not set +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_VM=y +CONFIG_DEBUG_VM_MAPLE_TREE=y +CONFIG_DEBUG_VM_RB=y +CONFIG_PROVE_LOCKING=y +CONFIG_PAGE_TABLE_CHECK=y +CONFIG_PAGE_TABLE_CHECK_ENFORCED=y +# Guest-only infrastructure for late swap/reclaim fixtures. +CONFIG_BLOCK=y +CONFIG_VIRTIO=y +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_BLK=y +CONFIG_DEVTMPFS=y +CONFIG_DEBUG_FS=y +CONFIG_KUNIT_DEBUGFS=y diff --git a/tools/testing/kunit/configs/arm64_user4k_data_kasan.config b/tools/testing/kunit/configs/arm64_user4k_data_kasan.config new file mode 100644 index 0000000000000..2e3e50bd67768 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k_data_kasan.config @@ -0,0 +1,18 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_DATA_KUNIT_TEST=y +# CONFIG_ARM64_USER4K_KUNIT_TEST is not set +CONFIG_KASAN=y +CONFIG_KASAN_GENERIC=y +CONFIG_KASAN_INLINE=y +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y diff --git a/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config b/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config new file mode 100644 index 0000000000000..11b9935770f53 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config @@ -0,0 +1,38 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_KUNIT_TEST=y +CONFIG_ARM64_SW_TTBR0_PAN=y +# CONFIG_ARM64_CNP is not set +# CONFIG_ARM64_POE is not set +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_VM=y +CONFIG_DEBUG_VM_MAPLE_TREE=y +CONFIG_DEBUG_VM_RB=y +CONFIG_PROVE_LOCKING=y +CONFIG_PAGE_TABLE_CHECK=y +CONFIG_PAGE_TABLE_CHECK_ENFORCED=y +# Guest-only infrastructure for late swap/reclaim fixtures. +CONFIG_BLOCK=y +CONFIG_VIRTIO=y +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_BLK=y +CONFIG_DEVTMPFS=y +CONFIG_DEBUG_FS=y +CONFIG_KUNIT_DEBUGFS=y + +# Software MM fixtures only; the hardware boot opt-in is rejected under KASAN. +CONFIG_KASAN=y +CONFIG_KASAN_GENERIC=y +CONFIG_KASAN_INLINE=y diff --git a/tools/testing/selftests/arm64/Makefile b/tools/testing/selftests/arm64/Makefile index e456f3b62fa13..d8948f5fb1ef8 100644 --- a/tools/testing/selftests/arm64/Makefile +++ b/tools/testing/selftests/arm64/Makefile @@ -4,7 +4,7 @@ ARCH ?= $(shell uname -m 2>/dev/null || echo not) ifneq (,$(filter $(ARCH),aarch64 arm64)) -ARM64_SUBTARGETS ?= tags signal pauth fp mte bti abi gcs +ARM64_SUBTARGETS ?= tags signal pauth fp mte bti abi gcs user4k else ARM64_SUBTARGETS := endif diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore new file mode 100644 index 0000000000000..2e6224b832f98 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -0,0 +1,3 @@ +page-contract + +swap-fixture-init diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile new file mode 100644 index 0000000000000..57f279e1990dc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: GPL-2.0 +TEST_GEN_PROGS := page-contract +TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init +CFLAGS += -Wall -Wextra -O2 -ffixed-x18 +TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe +TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe +TEST_FILES += compat-script runtime-app.py +include ../../lib.mk + +$(OUTPUT)/page-contract: page-contract.c x18-probe.S + +$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init: LDLIBS += -pthread + +# Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. +AARCH32_CC ?= clang --target=arm-linux-gnueabihf +AARCH32_FLAGS := -march=armv8-a -marm -mfloat-abi=soft -ffreestanding \ + -fno-builtin -fno-stack-protector -O2 -nostdlib -static -fuse-ld=lld +$(OUTPUT)/compat-probe-4k: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=4096 $< -o $@ +$(OUTPUT)/compat-probe-64k: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ +$(OUTPUT)/compat-interp32: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536,--image-base=0x10000000 $< -o $@ +$(OUTPUT)/compat-probe-dynamic: compat-probe.c + $(AARCH32_CC) $(filter-out -static,$(AARCH32_FLAGS)) -no-pie \ + -Wl,-z,max-page-size=65536,--dynamic-linker=/interp32 $< -o $@ + +$(OUTPUT)/compat-aslr-probe: compat-aslr-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/compat-mlock-probe: compat-mlock-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# The disposable PID1 fixture also runs with the native 64K process ABI. +$(OUTPUT)/mglru-granule-init $(OUTPUT)/memcg-pressure-init $(OUTPUT)/memcg-accounting-init $(OUTPUT)/runtime-app-init $(OUTPUT)/fex-contract-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +# FEX fixture: the AArch64 emulator runs the freestanding x86-64 contract. +X86_64_CC ?= clang --target=x86_64-linux-gnu +$(OUTPUT)/fex-x86-contract: fex-x86-contract.c + $(X86_64_CC) -O2 -fno-builtin -fno-stack-protector -nostdlib -static \ + -fuse-ld=lld -Wl,-z,max-page-size=4096,-e,_start $< -o $@ + +$(OUTPUT)/libfex-abi.so: fex-native-abi.c + $(CC) $(CFLAGS) -shared -fPIC -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/exec-policy-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/exec-policy-init: LDLIBS += -pthread +$(OUTPUT)/compat-policy-probe: compat-policy-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# Optional Windows payload; requires Wine/FEX in a disposable guest to execute. +# Keep its toolchain out of the ordinary Linux selftest build. +WINDOWS_CC ?= clang --target=x86_64-pc-windows-msvc +WINDOWS_DLLTOOL ?= llvm-dlltool +WINDOWS_LD ?= lld-link +.PHONY: wine-contract +wine-contract: $(OUTPUT)/wine-page-contract.exe +$(OUTPUT)/wine-kernel32.lib: wine-kernel32.def + $(WINDOWS_DLLTOOL) -m i386:x86-64 -d $< -l $@ +$(OUTPUT)/wine-page-contract.obj: wine-page-contract.c + $(WINDOWS_CC) -O2 -Wall -Wextra -Werror -ffreestanding -fno-stack-protector -c $< -o $@ +$(OUTPUT)/wine-page-contract.exe: $(OUTPUT)/wine-page-contract.obj $(OUTPUT)/wine-kernel32.lib + $(WINDOWS_LD) /entry:mainCRTStartup /subsystem:console /nodefaultlib /out:$@ $^ +EXTRA_CLEAN += $(OUTPUT)/wine-page-contract.exe $(OUTPUT)/wine-page-contract.obj $(OUTPUT)/wine-kernel32.lib + +$(OUTPUT)/uffd-wp-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/uffd-wp-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-uffd-swap-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/compat-uffd-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-uffd-probe: compat-uffd-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/shmem-hole-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-hole-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-thp-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-thp-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-collapse-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/shmem-pin-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-pin-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-race-init $(OUTPUT)/shmem-pressure-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-race-init: LDLIBS += -pthread + +$(OUTPUT)/uffd-move-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/uffd-move-init: LDLIBS += -pthread + +$(OUTPUT)/user-events-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/uprobes-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/vhost-log-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/bpf-uptr-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/coarse-shmem-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/iommufd-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/iommufd-init: LDLIBS += -pthread +$(OUTPUT)/iommufd-init: CFLAGS += $(KHDR_INCLUDES) diff --git a/tools/testing/selftests/arm64/user4k/advice.c b/tools/testing/selftests/arm64/user4k/advice.c new file mode 100644 index 0000000000000..12f09f2d65577 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/advice.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static size_t page; + +static bool wait_child(pid_t pid, int sig) +{ + int status = 0; + + if (pid <= 0 || waitpid(pid, &status, 0) != pid) + return false; + if (sig) + return WIFSIGNALED(status) && WTERMSIG(status) == sig; + return WIFEXITED(status) && !WEXITSTATUS(status); +} + +static void *anon(size_t length, int prot, int extra) +{ + void *p = mmap(NULL, length, prot, MAP_PRIVATE | MAP_ANONYMOUS | extra, -1, 0); + + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap\n"); + return p; +} + +static void discard_and_guards(void) +{ + unsigned char *p = anon(3 * page, PROT_READ | PROT_WRITE, 0); + bool ok; + pid_t child; + + memset(p, 0x61, 3 * page); + ok = madvise(p + page, 1, MADV_DONTNEED) == 0; + ok &= p[0] == 0x61 && p[page] == 0 && p[2 * page] == 0x61; + ksft_test_result(ok, "DONTNEED discards exactly one rounded user page\n"); + memset(p + page, 0x71, page); + child = fork(); + if (!child) { + ok = madvise(p + page, page, MADV_DONTNEED) == 0; + _exit(!(ok && p[page] == 0 && p[0] == 0x61)); + } + ksft_test_result(wait_child(child, 0) && p[page] == 0x71, + "discard in a fork child leaves the parent quarter intact\n"); + ok = madvise(p + page, 1, MADV_GUARD_INSTALL) == 0; + ok &= madvise(p + page, page, MADV_DONTNEED) == 0; + child = fork(); + if (!child) { + volatile unsigned char value = p[page]; + + (void)value; + _exit(1); + } + ksft_test_result(ok && wait_child(child, SIGSEGV), + "a user-page guard survives discard and fork\n"); + ok = madvise(p + page, 1, MADV_GUARD_REMOVE) == 0; + ok &= p[page] == 0 && p[0] == 0x61 && p[2 * page] == 0x61; + ksft_test_result(ok, "guard removal restores only the requested page\n"); + munmap(p, 3 * page); +} + +static void population(void) +{ + unsigned char *p = anon(5 * page, PROT_READ, 0), vec[5]; + bool ok; + unsigned int i; + + ok = mprotect(p + 2 * page, page, PROT_READ | PROT_WRITE) == 0; + ok &= madvise(p + 2 * page, 1, MADV_POPULATE_WRITE) == 0; + ok &= mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == (i == 2); + ksft_test_result(ok, "POPULATE_WRITE faults only the selected user-page VMA\n"); + ok = madvise(p + page, 1, MADV_POPULATE_READ) == 0; + ok &= mincore(p + page, page, vec) == 0 && vec[0] == 1; + errno = 0; + ok &= madvise(p, page, MADV_POPULATE_WRITE) == -1 && errno == EINVAL; + ok &= mprotect(p + 3 * page, page, PROT_NONE) == 0; + errno = 0; + ok &= madvise(p + 3 * page, page, MADV_POPULATE_READ) == -1 && errno == EINVAL; + ok &= munmap(p + 3 * page, page) == 0; + errno = 0; + ok &= madvise(p + 3 * page, page, MADV_POPULATE_READ) == -1 && errno == ENOMEM; + ksft_test_result(ok, "POPULATE_READ honors permissions and reports holes\n"); + munmap(p, 5 * page); + p = anon(5 * page, PROT_READ | PROT_WRITE, MAP_POPULATE); + ok = mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == 1; + ksft_test_result(ok, "MAP_POPULATE materializes every user page\n"); + munmap(p, 5 * page); +} + +static void fork_advice(void) +{ + unsigned char *p = anon(3 * page, PROT_READ | PROT_WRITE, 0); + bool ok; + pid_t child; + + memset(p, 0x58, 3 * page); + ok = madvise(p + page, 1, MADV_WIPEONFORK) == 0; + child = fork(); + if (!child) + _exit(!(p[page] == 0 && p[0] == 0x58 && p[2 * page] == 0x58)); + ok &= wait_child(child, 0) && p[page] == 0x58; + ksft_test_result(ok, "WIPEONFORK clears a single child page and preserves neighbors\n"); + ok = madvise(p + page, 1, MADV_KEEPONFORK) == 0; + ok &= madvise(p + page, 1, MADV_DONTFORK) == 0; + child = fork(); + if (!child) { + unsigned char vec; + + errno = 0; + _exit(!(mincore(p + page, page, &vec) == -1 && errno == ENOMEM && + p[0] == 0x58 && p[2 * page] == 0x58)); + } + ok &= wait_child(child, 0); + ksft_test_result(ok, "DONTFORK omits only its selected user page\n"); + munmap(p, 3 * page); +} + +static void remove_file(void) +{ + int fd = memfd_create("advice-file", 0); + unsigned char *shared, *alias, *private; + bool ok; + size_t i; + + if (fd < 0 || ftruncate(fd, 5 * page)) + ksft_exit_fail_msg("memfd setup\n"); + shared = mmap(NULL, 5 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + alias = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, page); + private = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, page); + if (shared == MAP_FAILED || alias == MAP_FAILED || private == MAP_FAILED) + ksft_exit_fail_msg("file mmap\n"); + for (i = 0; i < 5 * page; i++) + shared[i] = 0x40 + i / page; + private[page] = 0x79; + ok = madvise(alias + page, 1, MADV_REMOVE) == 0; + for (i = 0; i < 5 * page; i++) + ok &= shared[i] == (i / page == 2 ? 0 : 0x40 + i / page); + ok &= private[page] == 0x79; + ksft_test_result(ok, "REMOVE uses the file byte offset and preserves private COW\n"); + ok = madvise(alias, 3 * page, MADV_WILLNEED) == 0; + ksft_test_result(ok, "WILLNEED accepts a file range with the process granule\n"); + munmap(shared, 5 * page); + munmap(alias, 3 * page); + munmap(private, 3 * page); + close(fd); +} + +int main(int argc, char **argv) +{ + const struct rlimit no_core = { 0, 0 }; + + page = getauxval(AT_PAGESZ); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + ksft_exit_fail_msg("unexpected process page ABI: %zu\n", page); + ksft_print_msg("process page size: %zu\n", page); + if (setrlimit(RLIMIT_CORE, &no_core)) + return 1; + ksft_print_header(); + ksft_set_plan(11); + discard_and_guards(); + population(); + fork_advice(); + remove_file(); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c b/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c new file mode 100644 index 0000000000000..2df7e354f72b5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define SAMPLES 64 +struct sample { + unsigned long page_size, stack, mapping, brk; +}; + +static int get_sample(struct sample *s) +{ + char buf[4096], *pos, *end; + unsigned long low, high; + FILE *f; + int fd; + ssize_t size; + void *p; + + s->page_size = getauxval(AT_PAGESZ); + p = mmap(NULL, s->page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + s->mapping = (unsigned long)p; + f = fopen("/proc/self/maps", "r"); + if (!f) + return 1; + while (fgets(buf, sizeof(buf), f)) + if (strstr(buf, "[stack]") && sscanf(buf, "%lx-%lx", &low, &high) == 2) + s->stack = high; + fclose(f); + fd = open("/proc/self/stat", O_RDONLY); + if (fd < 0) + return 1; + size = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (size <= 0) + return 1; + buf[size] = 0; + pos = strrchr(buf, ')'); + if (!pos) + return 1; + pos += 2; + /* Fields 1 and 2 precede the closing parenthesis; start_brk is 47. */ + for (unsigned int field = 3; field < 47; field++) { + pos = strchr(pos, ' '); + if (!pos) + return 1; + pos++; + } + errno = 0; + s->brk = strtoul(pos, &end, 10); + return errno || end == pos || !s->brk || !s->stack; +} + +static int collect(unsigned int mode, bool disabled, struct sample *s) +{ + int pipefd[2], status; + pid_t pid; + ssize_t size; + + if (pipe(pipefd)) + return 1; + pid = fork(); + if (!pid) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + if (prctl(SET_EXEC_PAGE_SIZE, mode == 1 ? 4096UL : 0UL, 0UL, 0UL, 0UL)) + _exit(2); + if (disabled && personality(ADDR_NO_RANDOMIZE) < 0) + _exit(3); + execl(mode == 2 ? "/compat-aslr" : "/init", "/init", "--sample", fd, NULL); + _exit(4); + } + close(pipefd[1]); + size = read(pipefd[0], s, sizeof(*s)); + close(pipefd[0]); + return pid < 0 || waitpid(pid, &status, 0) != pid || !WIFEXITED(status) || + WEXITSTATUS(status) || size != sizeof(*s); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + if (argc == 3 && !strcmp(argv[1], "--sample")) { + struct sample s = {0}; + if (getppid() != 1 || get_sample(&s)) + return 1; + return write(atoi(argv[2]), &s, sizeof(s)) != sizeof(s); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + if (mount("proc", "/proc", "proc", 0, NULL)) + failed++; + int control = open("/proc/sys/kernel/randomize_va_space", O_WRONLY); + if (control < 0 || write(control, "2", 1) != 1) + failed++; + if (control >= 0) close(control); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + if (prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL)) failed++; + for (unsigned int mode = 0; mode < 3; mode++) { + if (mode == 1 && !has4k) continue; + unsigned long ps = mode && has4k ? 4096 : native; + struct sample first = {0}; + bool varied[3] = {0}, subnative[3] = {0}; + int ok = 1; + + for (unsigned int i = 0; i < SAMPLES; i++) { + struct sample s = {0}; + if (collect(mode, false, &s)) { ok = 0; break; } + unsigned long address[] = {s.stack, s.mapping, s.brk}; + unsigned long original[] = {first.stack, first.mapping, first.brk}; + ok &= s.page_size == ps; + for (unsigned int n = 0; n < 3; n++) { + ok &= !(address[n] & (ps - 1)); + varied[n] |= i && address[n] != original[n]; + subnative[n] |= i && ((address[n] ^ original[n]) & (native - 1)); + } + if (!i) first = s; + } + for (unsigned int n = 0; n < 3; n++) + ok &= varied[n] && (ps == native || subnative[n]); + printf("%s - %s %luK ASLR: 64 execs, stack/mmap/brk varied=%d%d%d low-bits-varied=%d%d%d\n", + ok ? "ok" : "not ok", mode == 2 ? "AArch32" : "AArch64", ps / 1024, varied[0], varied[1], varied[2], + subnative[0], subnative[1], subnative[2]); + failed += !ok; + struct sample a = {0}, b = {0}; + ok = !collect(mode, true, &a) && !collect(mode, true, &b) && + !memcmp(&a, &b, sizeof(a)) && a.page_size == ps; + printf("%s - %s %luK ADDR_NO_RANDOMIZE retains stable layout\n", + ok ? "ok" : "not ok", mode == 2 ? "AArch32" : "AArch64", ps / 1024); + failed += !ok; + } + printf("GRANULE ASLR %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c b/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c new file mode 100644 index 0000000000000..33793cebc2b8c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c @@ -0,0 +1,583 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +struct user_data { + uint64_t a, b, result, marker; +}; +struct value { + struct user_data *p, *q; + uint64_t cookie; + uint8_t trailing; +} __attribute__((packed)); +static char verifier_log[65536]; +static int bpf(enum bpf_cmd cmd, union bpf_attr *a) +{ + size_t size; +#define END(field) (offsetof(union bpf_attr, field) + sizeof(a->field)) + switch (cmd) { + case BPF_BTF_LOAD: + size = END(btf_log_level); + break; + case BPF_MAP_CREATE: + size = END(btf_value_type_id); + break; + case BPF_PROG_LOAD: + size = END(log_buf); + break; + case BPF_RAW_TRACEPOINT_OPEN: + size = END(raw_tracepoint.prog_fd); + break; + case BPF_MAP_UPDATE_ELEM: + size = END(flags); + break; + case BPF_MAP_LOOKUP_ELEM: + size = END(flags); + break; + case BPF_MAP_DELETE_ELEM: + size = END(key); + break; + default: + errno = EINVAL; + return -1; + } +#undef END + return syscall(__NR_bpf, cmd, a, size); +} +struct btf_blob { + unsigned char types[1024]; + char strings[512]; + unsigned nt, ns; +}; +static uint32_t str(struct btf_blob *b, const char *s) +{ + unsigned at = b->ns; + size_t n = strlen(s) + 1; + memcpy(b->strings + b->ns, s, n); + b->ns += n; + return at; +} +static void push(struct btf_blob *b, const void *p, size_t n) +{ + memcpy(b->types + b->nt, p, n); + b->nt += n; +} +static void type(struct btf_blob *b, const char *name, unsigned kind, + unsigned n, unsigned size_type) +{ + struct btf_type t = { .name_off = name ? str(b, name) : 0, + .info = kind << 24 | n, + .size = size_type }; + push(b, &t, sizeof(t)); +} +static void member(struct btf_blob *b, const char *name, unsigned id, + unsigned offset) +{ + struct btf_member m = { .name_off = str(b, name), + .type = id, + .offset = offset * 8 }; + push(b, &m, sizeof(m)); +} +static int create_map(void) +{ + struct btf_blob b = { .ns = 1 }; + uint32_t encoding; + type(&b, "int", BTF_KIND_INT, 0, 4); + encoding = BTF_INT_SIGNED << 24 | 32; + push(&b, &encoding, 4); + type(&b, "u64", BTF_KIND_INT, 0, 8); + encoding = 64; + push(&b, &encoding, 4); + type(&b, "user_data", BTF_KIND_STRUCT, 4, sizeof(struct user_data)); + member(&b, "a", 2, 0); + member(&b, "b", 2, 8); + member(&b, "result", 2, 16); + member(&b, "marker", 2, 24); + type(&b, "uptr", BTF_KIND_TYPE_TAG, 0, 3); + type(&b, NULL, BTF_KIND_PTR, 0, 4); + type(&b, "value_type", BTF_KIND_STRUCT, 4, sizeof(struct value)); + member(&b, "p", 5, 0); + member(&b, "q", 5, 8); + member(&b, "cookie", 2, 16); + member(&b, "trailing", 7, 24); + type(&b, "u8", BTF_KIND_INT, 0, 1); + encoding = 8; + push(&b, &encoding, 4); + unsigned char data[2048]; + struct btf_header h = { .magic = BTF_MAGIC, + .version = BTF_VERSION, + .hdr_len = sizeof(h), + .type_len = b.nt, + .str_off = b.nt, + .str_len = b.ns }; + memcpy(data, &h, sizeof(h)); + memcpy(data + sizeof(h), b.types, b.nt); + memcpy(data + sizeof(h) + b.nt, b.strings, b.ns); + union bpf_attr a = { .btf = (uintptr_t)data, + .btf_size = sizeof(h) + b.nt + b.ns, + .btf_log_buf = (uintptr_t)verifier_log, + .btf_log_size = sizeof(verifier_log), + .btf_log_level = 1 }; + int btf = bpf(BPF_BTF_LOAD, &a); + if (btf < 0) { + printf("BTF load errno=%d: %s\n", errno, verifier_log); + return -1; + } + a = (union bpf_attr){ .map_type = BPF_MAP_TYPE_TASK_STORAGE, + .key_size = sizeof(int), + .value_size = sizeof(struct value), + .map_flags = BPF_F_NO_PREALLOC, + .btf_fd = btf, + .btf_key_type_id = 1, + .btf_value_type_id = 6 }; + int fd = bpf(BPF_MAP_CREATE, &a); + close(btf); + return fd; +} +#define I(c, d, s, o, i) \ + ((struct bpf_insn){ .code = (c), \ + .dst_reg = (d), \ + .src_reg = (s), \ + .off = (o), \ + .imm = (i) }) +static int create_prog(int map, int *link) +{ + struct bpf_insn insns[40]; + unsigned n = 0, jumps[4]; +#define EMIT(c, d, s, o, i) (insns[n++] = I(c, d, s, o, i)) + EMIT(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_current_task_btf); + EMIT(BPF_ALU64 | BPF_MOV | BPF_X, 2, 0, 0, 0); + EMIT(BPF_LD | BPF_DW | BPF_IMM, 1, BPF_PSEUDO_MAP_FD, 0, map); + EMIT(0, 0, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 3, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 4, 0, 0, 0); + EMIT(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_task_storage_get); + jumps[0] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 0, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_X, 6, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 7, 6, 16, 0); + jumps[1] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 7, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 1, 6, 0, 0); + jumps[2] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 1, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 2, 1, 0, 0); + EMIT(BPF_ALU64 | BPF_XOR | BPF_X, 2, 7, 0, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 2, 16, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 7, 24, 0); + insns[jumps[2]].off = n - jumps[2] - 1; + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 1, 6, 8, 0); + jumps[3] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 1, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 2, 1, 8, 0); + EMIT(BPF_ALU64 | BPF_ADD | BPF_X, 2, 7, 0, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 2, 16, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 7, 24, 0); + for (unsigned i = 0; i < 4; i++) + if (i != 2) + insns[jumps[i]].off = n - jumps[i] - 1; + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 0, 0, 0, 0); + EMIT(BPF_JMP | BPF_EXIT, 0, 0, 0, 0); +#undef EMIT + union bpf_attr a = { .prog_type = BPF_PROG_TYPE_RAW_TRACEPOINT, + .insn_cnt = n, + .insns = (uintptr_t)insns, + .license = (uintptr_t)"GPL", + .log_buf = (uintptr_t)verifier_log, + .log_size = sizeof(verifier_log), + .log_level = 1 }; + verifier_log[0] = 0; + int fd = bpf(BPF_PROG_LOAD, &a); + if (fd < 0) { + printf("prog load errno=%d: %s\n", errno, verifier_log); + return -1; + } + a = (union bpf_attr){}; + a.raw_tracepoint.name = (uintptr_t)"sys_enter"; + a.raw_tracepoint.prog_fd = fd; + *link = bpf(BPF_RAW_TRACEPOINT_OPEN, &a); + if (*link < 0) { + close(fd); + return -1; + } + return fd; +} +static int update(int map, int key, struct value *v, unsigned flags) +{ + union bpf_attr a = { .map_fd = map, + .key = (uintptr_t)&key, + .value = (uintptr_t)v, + .flags = flags }; + return bpf(BPF_MAP_UPDATE_ELEM, &a); +} +static int delete(int map, int key) +{ + union bpf_attr a = { .map_fd = map, .key = (uintptr_t)&key }; + return bpf(BPF_MAP_DELETE_ELEM, &a); +} +struct pins { + unsigned long long acquired, released; +}; +static int pin_stats(struct pins *p) +{ + int fd = open("/proc/sys/vm/stat_refresh", O_RDONLY); + char c; + if (fd < 0) + return -1; + int ret = read(fd, &c, 1); + close(fd); + if (ret < 0) + return -1; + FILE *f = fopen("/proc/vmstat", "r"); + if (!f) + return -1; + char name[128]; + unsigned long long value; + unsigned found = 0; + while (fscanf(f, "%127s %llu", name, &value) == 2) { + if (!strcmp(name, "nr_foll_pin_acquired")) { + p->acquired = value; + found |= 1; + } + if (!strcmp(name, "nr_foll_pin_released")) { + p->released = value; + found |= 2; + } + } + fclose(f); + return found == 3 ? 0 : -1; +} +static int wait_pins(struct pins base, unsigned active) +{ + struct pins p = {}; + for (unsigned i = 0; i < 500; i++) { + if (pin_stats(&p)) + return 0; + if (p.acquired - base.acquired >= active && + p.acquired - base.acquired - (p.released - base.released) == + active) + return 1; + usleep(10000); + } + printf("pin delta acquired=%llu released=%llu expected-active=%u\n", + p.acquired - base.acquired, p.released - base.released, active); + return 0; +} +static int filled(unsigned char *p, size_t len, unsigned char value) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value) + return 0; + return 1; +} +static int values(struct user_data *p, struct user_data *q, uint64_t cookie) +{ + return p->result == (p->a ^ cookie) && q->result == q->b + cookie && + p->marker == cookie && q->marker == cookie; +} +static int exercise_case(size_t ps, unsigned shared) +{ + struct pins base; + CHECK(!pin_stats(&base)); + int backing = -1; + if (shared) { + backing = memfd_create("bpf-uptr", 0); + CHECK(backing >= 0 && !ftruncate(backing, 4 * ps)); + } + unsigned char *src = mmap( + (void *)0x20000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), + backing, 0); + unsigned char *dst = + mmap((void *)0x30000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, 4 * ps); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps; + struct user_data *p = (void *)(leaf + ps - 64), *q = p + 1; + *p = (struct user_data){ .a = 0x12345678, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x87654321 }; + int map = create_map(); + CHECK(map >= 0); + int link = -1, prog = create_prog(map, &link); + CHECK(prog >= 0 && link >= 0); + int key = syscall(SYS_pidfd_open, getpid(), 0); + CHECK(key >= 0); + struct value probe = {}; + int probe_ret = update(map, key, &probe, BPF_NOEXIST); + CHECK(!probe_ret); + CHECK(!delete(map, key)); + struct value v = { + .p = p, .q = q, .cookie = 0xabc001, .trailing = 0x7d + }; + CHECK(!update(map, key, &v, BPF_NOEXIST)); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + for (unsigned i = 0; i < 64; i++) { + v.cookie++; + CHECK(!update(map, key, &v, BPF_EXIST)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + } + CHECK(wait_pins(base, 2)); + struct { + struct value v; + unsigned char tail[16]; + } lookup; + memset(&lookup, 0xa5, sizeof(lookup)); + union bpf_attr a = { .map_fd = map, + .key = (uintptr_t)&key, + .value = (uintptr_t)&lookup }; + CHECK(!bpf(BPF_MAP_LOOKUP_ELEM, &a)); + CHECK(!lookup.v.p && !lookup.v.q && lookup.v.cookie == v.cookie && + lookup.v.trailing == v.trailing && + filled(lookup.tail, sizeof(lookup.tail), 0xa5)); + errno = 0; + CHECK(update(map, key, &v, BPF_NOEXIST) == -1 && errno == EEXIST); + CHECK(wait_pins(base, 2)); + struct value half = v; + half.p = NULL; + CHECK(!update(map, key, &half, BPF_EXIST)); + CHECK(wait_pins(base, 1)); + p->result = p->marker = 0; + syscall(SYS_getpid); + CHECK(!p->result && !p->marker && q->result == q->b + v.cookie); + CHECK(!update(map, key, &v, BPF_EXIST)); + CHECK(wait_pins(base, 2)); + struct value bad = v; + bad.q = (void *)5; + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + bad.q = (void *)(leaf + ps - 16); + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EOPNOTSUPP); + CHECK(wait_pins(base, 2)); + bad.q = (void *)(UINTPTR_MAX - 15); + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(wait_pins(base, 2)); + CHECK(!mprotect(leaf, ps, PROT_READ)); + errno = 0; + CHECK(update(map, key, &v, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(!mprotect(leaf, ps, PROT_READ | PROT_WRITE)); + CHECK(wait_pins(base, 2)); + struct value empty = {}; + CHECK(!update(map, key, &empty, BPF_EXIST)); + CHECK(wait_pins(base, 0)); + p->result = q->result = p->marker = q->marker = 0; + syscall(SYS_getpid); + CHECK(!p->result && !q->result); + CHECK(!update(map, key, &v, BPF_EXIST)); + CHECK(wait_pins(base, 2)); + /* The child owns storage pointing into the parent's pinned memory. */ + int barrier[2]; + CHECK(!pipe(barrier)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + close(barrier[1]); + char token; + if (read(barrier[0], &token, 1) != 1) + _exit(3); + if (!shared) { + p->a = 1; + q->b = 2; + } + syscall(SYS_getpid); + _exit(0); + } + close(barrier[0]); + int childkey = syscall(SYS_pidfd_open, child, 0); + CHECK(childkey >= 0); + CHECK(!update(map, childkey, &v, BPF_NOEXIST)); + CHECK(!delete(map, key)); + CHECK(wait_pins(base, 2)); + p->result = q->result = p->marker = q->marker = 0; + CHECK(write(barrier[1], "x", 1) == 1); + close(barrier[1]); + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + close(childkey); + CHECK(values(p, q, v.cookie) && p->a == 0x12345678 && + q->b == 0x87654321); + CHECK(wait_pins(base, 0)); + /* The supplying mm exits while a different task retains its UPTRs. */ + int foreign_pipe[2]; + CHECK(!pipe(foreign_pipe)); + pid_t supplier = fork(); + CHECK(supplier >= 0); + if (!supplier) { + close(foreign_pipe[0]); + *p = (struct user_data){ .a = 0x1111, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x2222 }; + struct value foreign = { .p = p, .q = q, .cookie = 0x765432 }; + if (update(map, key, &foreign, BPF_NOEXIST)) + _exit(4); + struct iovec data = { .iov_base = p, .iov_len = 64 }; + if (vmsplice(foreign_pipe[1], &data, 1, 0) != 64) + _exit(5); + _exit(0); + } + close(foreign_pipe[1]); + CHECK(waitpid(supplier, &status, 0) == supplier && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + struct user_data retired[2]; + size_t done = 0; + while (done < sizeof(retired)) { + ssize_t got = read(foreign_pipe[0], (char *)retired + done, + sizeof(retired) - done); + CHECK(got > 0); + done += got; + } + close(foreign_pipe[0]); + CHECK(retired[0].a == 0x1111 && retired[1].b == 0x2222 && + values(retired, retired + 1, 0x765432)); + CHECK(!delete(map, key) && wait_pins(base, 0)); + *p = (struct user_data){ .a = 0x12345678, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x87654321 }; + CHECK(!update(map, key, &v, BPF_NOEXIST)); + CHECK(wait_pins(base, 2)); + CHECK(filled(leaf, ps - 64, 0x64)); + int pipefd[2]; + CHECK(!pipe(pipefd)); + struct iovec iov = { .iov_base = leaf, .iov_len = ps }; + CHECK(vmsplice(pipefd[1], &iov, 1, 0) == (ssize_t)ps); + CHECK(!munmap(leaf, ps)); + CHECK(mmap(leaf, ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, + 0) == leaf); + memset(leaf, 0xe7, ps); + unsigned char *old = malloc(ps); + CHECK(old); + size_t n = 0; + while (n < ps) { + ssize_t got = read(pipefd[0], old + n, ps - n); + CHECK(got > 0); + n += got; + } + close(pipefd[0]); + close(pipefd[1]); + CHECK(filled(old, ps - 64, 0x64) && + values((void *)(old + ps - 64), (void *)(old + ps - 32), + v.cookie)); + CHECK(filled(leaf, ps, 0xe7)); + free(old); + /* Closing the map alone is insufficient: the program retains its map ref. */ + close(map); + close(link); + close(prog); + close(key); + CHECK(wait_pins(base, 0)); + CHECK(filled(dst, ps, 0x91) && filled(dst + 2 * ps, 2 * ps, 0x91)); + for (unsigned i = 0; i < 3; i++) + CHECK(filled(src + i * ps, ps, 0x61 + i)); + CHECK(!munmap(src, 3 * ps) && !munmap(dst, 4 * ps)); + if (backing >= 0) + close(backing); + printf("ok - %zuK BPF UPTR %s: relocated writes, updates, rollback, storage/supplier exit, unmap and pin balance\n", + ps / 1024, shared ? "shared" : "private"); + return 0; +} +static int exercise(size_t ps) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(90); + for (unsigned shared = 0; shared < 2; shared++) + CHECK(!exercise_case(ps, shared)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char size[24]; + snprintf(size, sizeof(size), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", size, NULL); + _exit(5); + } + int ok = 0; + for (unsigned ticks = 0; c > 0 && ticks < 1000; ticks++) { + pid_t got = waitpid(c, &status, WNOHANG); + if (got == c) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 999) { + kill(c, SIGKILL); + waitpid(c, &status, 0); + break; + } + usleep(100000); + } + printf("%s - ABI %zuK BPF UPTR status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("BPF UPTR %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c b/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c new file mode 100644 index 0000000000000..f193faa7b6de2 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1 runner; fault injection is selected by the KUnit worker. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "swapfile-fixture.h" + +static int setting(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY | O_CLOEXEC); + ssize_t n; + + if (fd < 0) + return -1; + n = write(fd, value, strlen(value)); + close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} + +static int fault_setting(const char *kind, const char *name, const char *value) +{ + char path[192]; + + snprintf(path, sizeof(path), "/sys/kernel/debug/%s/%s", kind, name); + return setting(path, value); +} + +static int injection_off(void) +{ + int ret = fault_setting("failslab", "probability", "0"); + + ret |= fault_setting("fail_page_alloc", "probability", "0"); + ret |= fault_setting("failslab", "ignore-gfp-wait", "Y"); + ret |= fault_setting("fail_page_alloc", "ignore-gfp-wait", "Y"); + ret |= fault_setting("fail_page_alloc", "ignore-gfp-highmem", "Y"); + return ret; +} + +static int injection_on(bool page) +{ + const char *kind = page ? "fail_page_alloc" : "failslab"; + + if (injection_off() || fault_setting(kind, "task-filter", "Y") || + fault_setting(kind, "verbose", "0") || + fault_setting(kind, "ignore-gfp-wait", "N") || + fault_setting(kind, "times", "-1") || + fault_setting(kind, "interval", "1") || fault_setting(kind, "space", "0")) + return -1; + if (!page) + return 0; /* Only current->fail_nth selects the slab allocation. */ + return fault_setting(kind, "ignore-gfp-highmem", "N") || + fault_setting(kind, "min-order", "2") || + fault_setting(kind, "probability", "100"); +} + +static int move_worker(const char *text, bool enter, bool swap_limit) +{ + char *end, path[192], pid[32], comm[64]; + long nr = strtol(text, &end, 10); + const char *group = swap_limit ? "coarse-swap-limit" : "coarse-zero"; + int fd; + ssize_t n; + + if (*end || nr <= 1 || nr == getpid()) + return -1; + snprintf(path, sizeof(path), "/proc/%ld/comm", nr); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = read(fd, comm, sizeof(comm) - 1); + close(fd); + if (n < 1) + return -1; + comm[n] = 0; + if (strncmp(comm, "kunit_", 6)) + return -1; + if (enter) { + if (mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) && errno != EBUSY) + return -1; + if (setting("/sys/fs/cgroup/cgroup.subtree_control", "+memory")) + return -1; + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s", group); + if (mkdir(path, 0755) && errno != EEXIST) + return -1; + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s/memory.max", group); + if (setting(path, swap_limit ? "max" : "0")) + return -1; + if (swap_limit && setting("/sys/fs/cgroup/coarse-swap-limit/memory.swap.max", "0")) + return -1; + } + snprintf(pid, sizeof(pid), "%ld", nr); + if (enter) + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s/cgroup.procs", group); + else + snprintf(path, sizeof(path), "/sys/fs/cgroup/cgroup.procs"); + return setting(path, pid); +} + +static int helper(const char *op) +{ + struct stat init, self; + + if (getpid() == 1 || stat("/proc/1/exe", &init) || stat("/proc/self/exe", &self) || + init.st_dev != self.st_dev || init.st_ino != self.st_ino) + return 1; + if (!strcmp(op, "--coarse-fixture-check")) + return getauxval(AT_PAGESZ) != 16384; + if (!strcmp(op, "--swapoff-helper")) + return swapoff(swap_path()) != 0; + if (!strcmp(op, "--swapon-helper")) + return swapon(swap_path(), 0) != 0; + if (!strncmp(op, "--coarse-swap-enter=", 20)) + return move_worker(op + 20, true, true) != 0; + if (!strncmp(op, "--coarse-swap-leave=", 20)) + return move_worker(op + 20, false, true) != 0; + if (!strcmp(op, "--coarse-swap-unlimit")) + return setting("/sys/fs/cgroup/coarse-swap-limit/memory.swap.max", "max") != 0; + if (!strcmp(op, "--coarse-injection-off")) + return injection_off() != 0; + if (!strcmp(op, "--coarse-slab-on")) + return injection_on(false) != 0; + if (!strcmp(op, "--coarse-page-on")) + return injection_on(true) != 0; + if (!strncmp(op, "--coarse-memcg-enter=", 21)) + return move_worker(op + 21, true, false) != 0; + if (!strncmp(op, "--coarse-memcg-leave=", 21)) + return move_worker(op + 21, false, false) != 0; + return 1; +} + +static int run_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-coarse-shmem/"; + char path[192], *log = calloc(1, 1024 * 1024); + size_t used = 0, capacity = 1024 * 1024; + ssize_t n = 0; + int fd, ret = -1; + + if (!log) + return -1; + snprintf(path, sizeof(path), "%srun", base); + if (setting(path, "1")) + goto out; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + goto out; + while (used < capacity - 1 && (n = read(fd, log + used, capacity - used - 1)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (!n && used < capacity - 1 && + strstr(log, "\nok 1 arm64-user4k-coarse-shmem\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; +out: + free(log); + return ret; +} + +int main(int argc, char **argv) +{ + struct stat st; + bool active = false; + int ret = 1; + unsigned int i; + + if (getpid() != 1 && argc == 2) + return helper(argv[1]); + if (getpid() != 1 || (argc != 1 && (argc != 2 || strcmp(argv[1], "--swapfile")))) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + if (getauxval(AT_PAGESZ) != 16384) + goto out; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) + goto out; + for (i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode)) + goto out; + if (argc == 2 && prepare_swapfile()) + goto out; + printf("COARSE SWAP BACKING=%s\n", swap_path()); + if (swapon(swap_path(), 0)) + goto out; + active = true; + ret = run_suite() ? 1 : 0; + if (injection_off()) + ret = 1; +out: + if (active && swapoff(swap_path())) + ret = 1; + if (active && argc == 2 && umount("/swapdir")) + ret = 1; + printf("COARSE SHMEM SWAP PASS=%d errno=%d\n", !ret, errno); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c b/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c new file mode 100644 index 0000000000000..fa05c6d769a2a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding ELF32 address sampler for the disposable ASLR fixture. */ +typedef unsigned int u32; +typedef unsigned long long u64; +struct sample { u64 page_size, stack, mapping, brk; }; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0")=a, r1 __asm__("r1")=b, r2 __asm__("r2")=c; + register long r3 __asm__("r3")=d, r4 __asm__("r4")=e, r5 __asm__("r5")=f; + register long r7 __asm__("r7")=n; + __asm__ volatile("svc 0" : "+r"(r0) : "r"(r1), "r"(r2), "r"(r3), + "r"(r4), "r"(r5), "r"(r7) : "memory", "cc"); + return r0; +} +static int error(long value) { return (u32)value >= 0xfffff001U; } +static unsigned number(const char *p) +{ + unsigned n=0; + while (*p >= '0' && *p <= '9') n=n*10+*p++-'0'; + return n; +} +static char *find(char *p, char c) +{ + while (*p && *p != c) p++; + return *p ? p : (char *)0; +} +static char *find_stack(char *p) +{ + const char name[]="[stack]"; + while (*p) { + unsigned i=0; + while (name[i] && p[i] == name[i]) i++; + if (!name[i]) return p; + p++; + } + return (char *)0; +} +static unsigned hex(char *p) +{ + unsigned n=0, v; + for (;;) { + if (*p >= '0' && *p <= '9') v=*p-'0'; + else if (*p >= 'a' && *p <= 'f') v=*p-'a'+10; + else return n; + n=(n<<4)|v; p++; + } +} +static int load(const char *path, char *buf, unsigned size) +{ + long fd=call(5,(long)path,0,0,0,0,0), n; + if (error(fd)) return 1; + n=call(3,fd,(long)buf,size-1,0,0,0); + call(6,fd,0,0,0,0,0); + if (n <= 0) return 1; + buf[n]=0; + return 0; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + unsigned argc=*sp++; + char **argv=(void *)sp, buf[4096], *p, *q; + struct sample sample={0}; + long mapping; + int bad=1; + sp+=argc+1; + while (*sp) sp++; + sp++; + while (sp[0]) { + if (sp[0]==6) sample.page_size=sp[1]; + sp+=2; + } + if (argc != 3 || call(64,0,0,0,0,0,0) != 1) goto done; + mapping=call(192,0,sample.page_size,3,0x22,-1,0); + if (error(mapping)) goto done; + sample.mapping=(u32)mapping; + if (load("/proc/self/maps",buf,sizeof(buf))) goto done; + p=find_stack(buf); + if (!p) goto done; + while (p>buf && p[-1]!='\n') p--; + p=find(p,'-'); + if (!p) goto done; + sample.stack=hex(p+1); + if (load("/proc/self/stat",buf,sizeof(buf))) goto done; + p=find(buf,')'); + if (!p) goto done; + while ((q=find(p+1,')'))) p=q; + p+=2; + for (unsigned field=3;field<47;field++) { + p=find(p,' '); + if (!p) goto done; + p++; + } + sample.brk=number(p); + if (!sample.page_size || !sample.stack || !sample.brk) goto done; + bad=call(4,number(argv[2]),(long)&sample,sizeof(sample),0,0,0)!=sizeof(sample); + done: + call(1,bad,0,0,0,0,0); + for (;;) {} +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-fixture-init.c b/tools/testing/selftests/arm64/user4k/compat-fixture-init.c new file mode 100644 index 0000000000000..4f6ab31ce8731 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-fixture-init.c @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static unsigned long native_size; +static void result(int ok,const char*s){printf("%s - launcher %s\n",ok?"ok":"not ok",s);if(!ok)failed++;} +static int launch(const char *path,char **args,int request,int expect_errno) +{ + pid_t child=fork();int status=0; + if(!child){if(request&&prctl(SET_EXEC_PAGE_SIZE,4096UL,0UL,0UL,0UL))_exit(95);unsigned char *canary=mmap(NULL,native_size,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(canary==MAP_FAILED)_exit(94);memset(canary,0x69,native_size); + int before=prctl(0x41555002,0UL,0UL,0UL,0UL); + for(int i=0;i<(expect_errno?64:1);i++){ + execv(path,args);if(!expect_errno||errno!=expect_errno)_exit(96); + if(getauxval(AT_PAGESZ)!=native_size||prctl(0x41555002,0UL,0UL,0UL,0UL)!=before)_exit(93); + for(int j=0;j1&&!strcmp(argv[1],"--returned"))return getauxval(AT_PAGESZ)==strtoul(getenv("AURORA_NATIVE_PAGE_SIZE"),NULL,10)?0:1; + if(argc>1&&!strcmp(argv[1],"--returned4k"))return getauxval(AT_PAGESZ)==4096?0:1; + if(getpid()!=1){fprintf(stderr,"Disposable VM PID1 only\n");return 2;} + native_size=getauxval(AT_PAGESZ);char native_text[24];snprintf(native_text,sizeof(native_text),"%lu",native_size);setenv("AURORA_NATIVE_PAGE_SIZE",native_text,1); + setvbuf(stdout,NULL,_IONBF,0);mount("proc","/proc","proc",0,NULL); + int no4k=argc>1&&!strcmp(argv[1],"--no4k"),fd=memfd_create("compat-offsets",0); + if(no4k){errno=0;result(prctl(SET_EXEC_PAGE_SIZE,4096UL,0UL,0UL,0UL)==-1&&errno==EOPNOTSUPP,"4KB capability is actually masked");} + result(fd>=0&&!ftruncate(fd,0x100000000LL+4*native_size),"sparse file fixture above 4GB"); + unsigned char page[4096];for(unsigned q=0;q<4*native_size/4096;q++){memset(page,0x40+q,sizeof(page));if(pwrite(fd,page,sizeof(page),q*4096)!=sizeof(page))failed++;memset(page,0x80+q,sizeof(page));if(pwrite(fd,page,sizeof(page),0x100000000LL+q*4096)!=sizeof(page))failed++;} + char fdstr[24];snprintf(fdstr,sizeof(fdstr),"%d",fd); + char *arg=malloc(90001);for(unsigned i=0;i<90000;i++)arg[i]='a'+i%23;arg[90000]=0; + char *args[]={"/probe32-64k",no4k?native_text:"4096",fdstr,arg,NULL}; + result(launch(args[0],args,0,0),"64KB-aligned ELF32 automatically selects available ABI"); + if(!no4k){args[0]="/probe32-4k";result(launch(args[0],args,0,0),"4KB-aligned ELF32 executes with 4KB translation");args[0]="/probe32-64k";result(launch(args[0],args,1,0),"explicit pending 4KB request allows ELF32");} + args[0]="/script32";result(launch(args[0],args,0,0),"script final ELF class determines page granule"); + args[0]="/probe32-dynamic";result(launch(args[0],args,0,0),"ELF32 interpreter mapping uses selected page granule"); + char *chain[]={"/probe32-64k","--exec32",no4k?native_text:"4096",fdstr,arg,NULL}; + result(launch(chain[0],chain,0,0),"ELF32 to ELF32 restages arguments and selects ABI"); + char *back[]={"/probe32-64k","--exec64",NULL};result(launch(back[0],back,0,0),"ELF32 to ELF64 exec restores native default"); + if(!no4k){back[1]="--exec64-4k";result(launch(back[0],back,0,0),"ELF32 can request alternative ELF64 exec granule");} + char *big=malloc(200001);memset(big,'x',200000);big[200000]=0;args[0]="/probe32-64k";args[3]=big; + if(!no4k)result(launch(args[0],args,0,E2BIG),"64 oversized transfers return E2BIG and preserve old mm and request"); + printf("COMPAT FIXTURE %s mode=%s\n",failed?"FAIL":"PASS",no4k?"no4k-native-fallback":"auto4k");sync();reboot(RB_POWER_OFF);for(;;)pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/compat-mlock-init.c b/tools/testing/selftests/arm64/user4k/compat-mlock-init.c new file mode 100644 index 0000000000000..5722969aa6e49 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-mlock-init.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +struct result { uint32_t pagesize, passed, failed, error; }; +int main(int argc, char **argv) +{ + struct result out = {0}; + unsigned long expected; + int pipes[2], status = -1; + pid_t pid; + size_t done = 0; + int ok; + + if (getpid() != 1 || (argc != 1 && (argc != 2 || strcmp(argv[1], "--no4k")))) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + expected = argc == 2 ? getauxval(AT_PAGESZ) : 4096; + if (mount("proc", "/proc", "proc", 0, NULL) || pipe(pipes)) + return 3; + pid = fork(); + if (!pid) { + char value[32]; + close(pipes[0]); + if (dup2(pipes[1], 1) < 0) _exit(4); + close(pipes[1]); + snprintf(value, sizeof(value), "%lu", expected); + execl("/compat-mlock", "/compat-mlock", value, NULL); + _exit(5); + } + close(pipes[1]); + while (done < sizeof(out)) { + ssize_t n = read(pipes[0], (char *)&out + done, sizeof(out) - done); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) break; + done += n; + } + close(pipes[0]); + ok = pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status) && done == sizeof(out) && + out.pagesize == expected && out.passed == 6 && !out.failed; + printf("%s - AArch32 memlock expected=%lu actual=%u passed=%u stage=%u error=%d status=%d\n", + ok ? "ok" : "not ok", expected, out.pagesize, out.passed, out.failed, + (int32_t)out.error, status); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c b/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c new file mode 100644 index 0000000000000..198ce8ed5199b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding ELF32 memlock probe, only for the disposable PID1 fixture. */ +typedef unsigned int u32; +struct result { u32 pagesize, passed, failed, error; }; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" : "+r"(r0) : "r"(r1), "r"(r2), "r"(r3), + "r"(r4), "r"(r5), "r"(r7) : "memory", "cc"); + return r0; +} +static int error(long n) { return (u32)n >= 0xfffff001U; } +static unsigned number(const char *p) +{ + unsigned n = 0; + while (*p >= '0' && *p <= '9') n = n * 10 + *p++ - '0'; + return n; +} +static long locked_kb(void) +{ + char buf[8192]; + long fd = call(5, (long)"/proc/self/status", 0, 0, 0, 0, 0), n; + const char name[] = "VmLck:"; + + if (error(fd)) return -1; + n = call(3, fd, (long)buf, sizeof(buf) - 1, 0, 0, 0); + call(6, fd, 0, 0, 0, 0, 0); + if (n <= 0) return -1; + buf[n] = 0; + for (long i = 0; i < n; i++) { + unsigned j = 0; + while (name[j] && i + j < (unsigned)n && buf[i + j] == name[j]) j++; + if (!name[j]) { + const char *p = buf + i + j; + while (*p == ' ' || *p == '\t') p++; + return number(p); + } + } + return -1; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + unsigned argc = *sp++, ps = 0, kb, stage = 1; + char **argv = (void *)sp; + struct result out = {0}; + long p, q, pid, ret = -1; + int status = -1; + + sp += argc + 1; + while (*sp) sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) ps = sp[1]; + sp += 2; + } + out.pagesize = ps; + if (argc != 2 || call(64, 0, 0, 0, 0, 0, 0) != 1 || ps != number(argv[1])) goto done; + kb = ps / 1024; + p = call(192, 0, 8 * ps, 3, 0x22, -1, 0); + if (error(p) || ((u32)p & (ps - 1)) || locked_kb() != 0) goto done; + ret = call(150, p + ps + 17, ps - 32, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) goto done; + out.passed++; stage++; + ret = call(150, p + ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) goto done; + ret = call(150, p + 2 * ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)(2 * kb)) goto done; + ret = call(151, p + ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) goto done; + out.passed++; stage++; + *(volatile unsigned char *)(p + 2 * ps) = 0x63; + q = call(163, p + 2 * ps, ps, 2 * ps, 1, 0, 0); + ret = q; + if (error(q) || locked_kb() != (long)(2 * kb) || + *(volatile unsigned char *)q != 0x63) goto done; + for (unsigned i = ps; i < 2 * ps; i++) + if (((volatile unsigned char *)q)[i]) goto done; + out.passed++; stage++; + pid = call(2, 0, 0, 0, 0, 0, 0); + if (!pid) { + call(1, locked_kb() != 0, 0, 0, 0, 0, 0); + for (;;) {} + } + ret = call(114, pid, (long)&status, 0, 0, 0, 0); + if (error(pid) || ret != pid || status || locked_kb() != (long)(2 * kb)) goto done; + out.passed++; stage++; + ret = call(91, q, 2 * ps, 0, 0, 0, 0); + if (ret || locked_kb() != 0) goto done; + out.passed++; stage++; + ret = call(390, p + 4 * ps, 2 * ps, 1, 0, 0, 0); + if (ret || locked_kb() != (long)(2 * kb)) goto done; + *(volatile unsigned char *)(p + 4 * ps) = 0x75; + ret = call(151, p + 4 * ps, 2 * ps, 0, 0, 0, 0); + if (ret || locked_kb() != 0) goto done; + out.passed++; stage = 0; + done: + out.failed = stage; + out.error = stage ? (u32)ret : 0; + call(4, 1, (long)&out, sizeof(out), 0, 0, 0); + call(1, stage != 0, 0, 0, 0, 0, 0); + for (;;) {} +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-policy-probe.c b/tools/testing/selftests/arm64/user4k/compat-policy-probe.c new file mode 100644 index 0000000000000..021a7f3e7151e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-policy-probe.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 -> AArch64 exec with an inherited granule preference. */ +#include "../../../../../include/uapi/linux/prctl.h" +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" : "+r"(r0) : "r"(r1), "r"(r2), "r"(r3), + "r"(r4), "r"(r5), "r"(r7) : "memory", "cc"); + return r0; +} +static unsigned int number(const char *s) +{ + unsigned int n = 0; + while (*s >= '0' && *s <= '9') n = n * 10 + *s++ - '0'; + return n; +} +static void fail(int code) +{ + call(1, code, 0, 0, 0, 0, 0); + for (;;) ; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + sp += argc + 1; + char **env = (void *)sp; + while (*sp) sp++; + sp++; + while (sp[0]) { if (sp[0] == 6) pages = sp[1]; sp += 2; } + if (argc != 4 || pages != number(argv[3])) fail(10); + if (call(172, PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0, 0, 0, 0, 0) != number(argv[2])) fail(11); + if (call(172, PR_AURORA_GET_EXEC_PAGE_SIZE, 0, 0, 0, 0, 0)) fail(12); + long mem = call(192, 0, 4 * pages, 3, 0x22, -1, 0); + if ((unsigned long)mem >= 0xfffff001UL) fail(13); + long ret = call(125, mem + 4096, 4096, 1, 0, 0, 0); + if (ret != (pages == 4096 ? 0 : -22)) fail(14); + if (call(91, mem, 4 * pages, 0, 0, 0, 0)) fail(15); + char *args[] = {"/init", "--compat-return", argv[1], argv[2], 0}; + call(11, (long)args[0], (long)args, (long)env, 0, 0, 0); + fail(16); + for (;;) ; +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-probe.c b/tools/testing/selftests/arm64/user4k/compat-probe.c new file mode 100644 index 0000000000000..6b2071ba0196c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-probe.c @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding AArch32 process; run only by compat-fixture-init in a guest. */ +typedef unsigned int u32; +typedef unsigned short u16; +static long call(long n,long a,long b,long c,long d,long e,long f) +{ + register long r0 __asm__("r0")=a,r1 __asm__("r1")=b,r2 __asm__("r2")=c; + register long r3 __asm__("r3")=d,r4 __asm__("r4")=e,r5 __asm__("r5")=f,r7 __asm__("r7")=n; + __asm__ volatile("svc 0":"+r"(r0):"r"(r1),"r"(r2),"r"(r3),"r"(r4),"r"(r5),"r"(r7):"memory","cc");return r0; +} +static unsigned length(const char *s){unsigned n=0;while(s[n])n++;return n;} +static int equal(const char*a,const char*b){while(*a&&*a==*b){a++;b++;}return *a==*b;} +static unsigned number(const char*s){unsigned n=0;while(*s>='0'&&*s<='9')n=n*10+*s++-'0';return n;} +static void out(const char*s){call(4,1,(long)s,length(s),0,0,0);} +static int failures,checks; +static void result(int pass,const char*s){checks++;if(!pass)failures++;out(pass?"ok - compat ":"not ok - compat ");out(s);out("\n");} +static int error(long v){return (unsigned long)v>=0xfffff001UL;} +static long map(unsigned size,int prot,int flags,int fd,unsigned off){return call(192,0,size,prot,flags,fd,off);} +static void unmap(long p,unsigned size){if(!error(p))call(91,p,size,0,0,0,0);} +static volatile unsigned signals; +static void handler(int n){if(n==10)signals++;} +struct action {u32 handler,flags,restorer,mask[2];}; +struct ehdr{unsigned char ident[16];u16 type,machine;u32 version,entry,phoff,shoff,flags;u16 ehsize,phentsize,phnum,shentsize,shnum,shstrndx;}; +struct phdr{u32 type,offset,vaddr,paddr,filesz,memsz,flags,align;}; +struct dyn{u32 tag,val;}; +struct sym{u32 name,value,size;unsigned char info,other;u16 shndx;}; +static void *vdso_symbol(unsigned base,const char*name) +{ + struct ehdr*h=(void*)base;struct phdr*p=(void*)(base+h->phoff);struct dyn*d=0;char*strings=0;struct sym*symbols=0;u32*hash=0; + for(unsigned i=0;iphnum;i++)if(p[i].type==2)d=(void*)(base+p[i].vaddr); + if(!d)return 0; + for(;d->tag;d++){if(d->tag==5)strings=(void*)(base+d->val);if(d->tag==6)symbols=(void*)(base+d->val);if(d->tag==4)hash=(void*)(base+d->val);} + if(!strings||!symbols||!hash)return 0; + for(unsigned i=0;i1&&equal(argv[1],"--exec32")){char*a[]={argv[0],argv[2],argv[3],argv[4],0};call(11,(long)a[0],(long)a,(long)env,0,0,0);call(1,97,0,0,0,0,0);for(;;);} + if(argc>1&&(equal(argv[1],"--exec64")||equal(argv[1],"--exec64-4k"))){int alternative=equal(argv[1],"--exec64-4k");if(alternative&&call(172,0x41555001,4096,0,0,0,0)){call(1,99,0,0,0,0,0);for(;;);}char*a[]={"/init",alternative?"--returned4k":"--returned",0};call(11,(long)a[0],(long)a,(long)env,0,0,0);call(1,98,0,0,0,0,0);for(;;);} + /* Script interpreter contributes its pathname before the ordinary arguments. */ + unsigned start=argc>1&&equal(argv[1],"/script32")?2:1; + unsigned expected=argc>start?number(argv[start]):0; + int fd=argc>start+1?(int)number(argv[start+1]):-1; + result(pagesz==expected,"AT_PAGESZ reflects final ELF32 selection"); + int args=argc>start+2&&length(argv[start+2])==90000; + if(args)for(unsigned i=0;i<90000;i++)if(argv[start+2][i]!=(char)('a'+i%23)){args=0;break;} + result(args,"90KB staged argument survives granule transfer"); + long mem=map(4*pagesz,3,0x22,-1,0);int ok=!error(mem); + if(ok){unsigned char*p=(void*)mem;for(unsigned i=0;i<4*pagesz;i++)p[i]=0x30+i/pagesz;for(unsigned i=0;i<4*pagesz;i++)if(p[i]!=(unsigned char)(0x30+i/pagesz))ok=0;} + result(ok,"anonymous user leaves have independent bytes"); + result(!error(mem)&&!call(125,mem+pagesz,pagesz,1,0,0,0)&&!call(125,mem+pagesz,pagesz,3,0,0,0),"mprotect at selected page boundaries"); + result(!error(mem)&&call(125,mem+1,pagesz,1,0,0,0)==-22&&call(91,mem+1,pagesz,0,0,0,0)==-22,"unaligned memory operations retain restrictions"); + long q=map(pagesz,1,1,fd,3*(pagesz>>12));ok=!error(q);if(ok)for(unsigned i=0;i>12));ok=!error(q);if(ok)for(unsigned i=0;i0&&call(114,child,(long)&status,0,0,0,0)==child&&!status&&!error(mem)&&*(unsigned char*)mem==0x30; + result(ok,"fork isolates private COW data");unmap(mem,4*pagesz); + struct action act={(u32)handler,0,0,{0,0}}; + ok=!call(174,10,(long)&act,0,8,0,0); + if(ok)ok=!call(37,call(20,0,0,0,0,0,0),10,0,0,0,0)&&signals==1; + result(ok,"AArch32 signal frame and kernel sigreturn trampoline"); + volatile unsigned word=17;int(*cmpxchg)(unsigned,unsigned,volatile unsigned*)=(void*)0xffff0fc0; + result(*(volatile unsigned*)0xffff0ffc>=5&&!cmpxchg(17,29,&word)&&word==29,"kuser helper page and atomic compare exchange"); + int(*gettime)(int,void*)=vdso?(void*)vdso_symbol(vdso,"__vdso_clock_gettime"):0;long time[2]={0,0}; + result(gettime&&!gettime(1,time)&&time[0]>=0&&time[1]>=0&&time[1]<1000000000,"compat vDSO clock_gettime and vvar mapping"); + out(failures?"COMPAT PROBE FAIL\n":"COMPAT PROBE PASS\n");call(1,failures?1:0,0,0,0,0,0);for(;;); +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-script b/tools/testing/selftests/arm64/user4k/compat-script new file mode 100644 index 0000000000000..9b56d0c7567f5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-script @@ -0,0 +1 @@ +#!/probe32-64k diff --git a/tools/testing/selftests/arm64/user4k/compat-uffd-init.c b/tools/testing/selftests/arm64/user4k/compat-uffd-init.c new file mode 100644 index 0000000000000..7611e1b5fec92 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-uffd-init.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define CHECK(x) do { if (!(x)) { printf("not ok - compat uffd line=%d errno=%d\n", __LINE__, errno); exit(1); } } while (0) +struct wire { uint32_t address, pagesize; }; +static void expect_invalid(int fd, uint64_t dst, uint64_t src, uint64_t len) +{ + struct uffdio_copy c = { .dst = dst, .src = src, .len = len }; + errno = 0; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && errno == EINVAL); +} +static void run(unsigned long native, unsigned long expected, int shared) +{ + int sock[2], file = -1, status; + CHECK(!socketpair(AF_UNIX, SOCK_SEQPACKET, 0, sock)); + if (shared) { + file = memfd_create("compat-uffd", 0); + CHECK(file >= 0 && !ftruncate(file, 3 * expected)); + } + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char page_arg[24], sock_arg[24], file_arg[24]; + close(sock[0]); + snprintf(page_arg, sizeof(page_arg), "%lu", expected); + snprintf(sock_arg, sizeof(sock_arg), "%d", sock[1]); + snprintf(file_arg, sizeof(file_arg), "%d", file); + execl("/compat-uffd", "/compat-uffd", page_arg, sock_arg, file_arg, NULL); + _exit(99); + } + close(sock[1]); + struct wire data = {0}; + struct iovec iov = { &data, sizeof(data) }; + union { struct cmsghdr align; char bytes[CMSG_SPACE(sizeof(int))]; } control = {0}; + struct msghdr msg = { .msg_iov = &iov, .msg_iovlen = 1, + .msg_control = control.bytes, .msg_controllen = sizeof(control.bytes) }; + CHECK(recvmsg(sock[0], &msg, 0) == sizeof(data)); + CHECK(!(msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) && data.pagesize == expected); + struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg); + CHECK(cmsg && cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS && + cmsg->cmsg_len == CMSG_LEN(sizeof(int))); + int uffd; + memcpy(&uffd, CMSG_DATA(cmsg), sizeof(uffd)); + /* Force the source above every possible AArch32 task limit. */ + unsigned char *source = mmap((void *)0x1000000000UL, 2 * native, + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(source != MAP_FAILED && (uintptr_t)source > UINT32_MAX); + /* Source alignment is independent of both processes' granules. */ + unsigned char *bytes = source + 17; + for (unsigned leaf = 0; leaf < 3; leaf++) { + struct pollfd pfd = { .fd = uffd, .events = POLLIN }; + struct uffd_msg event; + CHECK(poll(&pfd, 1, 10000) == 1 && (pfd.revents & POLLIN)); + CHECK(read(uffd, &event, sizeof(event)) == sizeof(event)); + uint64_t dst = (uint64_t)data.address + leaf * expected; + CHECK(event.event == UFFD_EVENT_PAGEFAULT && !event.arg.pagefault.flags && + event.arg.pagefault.address == dst); + if (!leaf) { + expect_invalid(uffd, dst, (uintptr_t)bytes, 0); + expect_invalid(uffd, dst, (uintptr_t)bytes, expected - 1); + expect_invalid(uffd, dst + 1, (uintptr_t)bytes, expected); + expect_invalid(uffd, 0x100000000ULL, (uintptr_t)bytes, expected); + expect_invalid(uffd, dst, UINT64_MAX - expected + 1, expected); + if (expected > 4096) + expect_invalid(uffd, dst, (uintptr_t)bytes, 4096); + } + if (leaf == 1) { + /* A valid but inaccessible pager source must leave the target missing. */ + CHECK(!mprotect(source, 2 * native, PROT_NONE)); + struct uffdio_copy bad = { .dst = dst, .src = (uintptr_t)bytes, .len = expected }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &bad) == -1 && errno == EFAULT && bad.copy == -EFAULT); + CHECK(!mprotect(source, 2 * native, PROT_READ | PROT_WRITE)); + } + memset(bytes, 0x51 + leaf, expected); + struct uffdio_copy c = { .dst = dst, .src = (uintptr_t)bytes, .len = expected }; + CHECK(!ioctl(uffd, UFFDIO_COPY, &c) && c.copy == (long long)expected); + } + CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(!munmap(source, 2 * native)); + CHECK(!close(uffd) && !close(sock[0])); + if (shared) CHECK(!close(file)); + printf("ok - native %lu pager high unaligned COPY to AArch32 %lu %s, source failure and target alignment\n", + native, expected, shared ? "shared" : "anonymous"); +} +int main(void) +{ + if (getpid() != 1) return 2; + setvbuf(stdout, NULL, _IONBF, 0); + alarm(90); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + char cmdline[2048] = {0}; + int fd = open("/proc/cmdline", O_RDONLY); + CHECK(fd >= 0 && read(fd, cmdline, sizeof(cmdline) - 1) > 0 && !close(fd)); + unsigned long native = getauxval(AT_PAGESZ); + unsigned long expected = strstr(cmdline, "id_aa64mmfr0.tgran4=f") ? native : 4096; + run(native, expected, 0); + run(native, expected, 1); + puts("COMPAT UFFD PASS"); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c b/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c new file mode 100644 index 0000000000000..42cd1af031eca --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding ELF32 half of compat-uffd-init; no armhf sysroot required. */ +typedef unsigned int u32; +typedef unsigned long long u64; +struct api { u64 api, features, ioctls; }; +struct registration { u64 start, len, mode, ioctls; }; +struct movement { u64 dst, src, len, mode; long long moved; }; +struct wire { u32 address, pagesize; }; +struct iovec32 { void *base; u32 len; }; +struct message32 { + void *name; u32 namelen; + struct iovec32 *iov; u32 iovlen; + void *control; u32 controllen, flags; +}; +struct rights32 { u32 len, level, type; int fd; }; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" : "+r"(r0) : "r"(r1), "r"(r2), "r"(r3), + "r"(r4), "r"(r5), "r"(r7) : "memory", "cc"); + return r0; +} +static u32 number(const char *s) +{ + u32 n = 0; + while (*s >= '0' && *s <= '9') n = n * 10 + *s++ - '0'; + return n; +} +__attribute__((noreturn)) static void finish(int code) +{ + call(1, code, 0, 0, 0, 0, 0); + for (;;) ; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + sp += argc + 1; + while (*sp) sp++; + sp++; + while (sp[0]) { if (sp[0] == 6) pages = sp[1]; sp += 2; } + if (argc != 4 || pages != number(argv[1])) finish(10); + int sock = number(argv[2]), shared = argv[3][0] != '-'; + int file = shared ? (int)number(argv[3]) : -1; + long map = call(192, 0, 3 * pages, 3, shared ? 1 : 0x22, file, 0); + if ((unsigned long)map >= 0xfffff001UL) finish(11); + /* Fine operations must retain native alignment restrictions on fallback. */ + if (call(125, map + 4096, 4096, 1, 0, 0, 0) != (pages == 4096 ? 0 : -22)) finish(12); + if (call(125, map, 3 * pages, 3, 0, 0, 0)) finish(13); + long fd = call(388, 0x80801, 0, 0, 0, 0, 0); /* CLOEXEC|NONBLOCK|USER_MODE_ONLY */ + if (fd < 0) finish(14); + /* Negotiate shared missing support under both fine and fallback ABIs. */ + struct api api = { 0xaa, (1ULL << 5) | (1ULL << 16), 0 }; + if (call(54, fd, 0xc018aa3fU, (long)&api, 0, 0, 0) || !(api.features & (1ULL << 5))) finish(15); + struct registration reg = { (u32)map, 3 * pages, 1, 0 }; + if (call(54, fd, 0xc020aa00U, (long)®, 0, 0, 0)) finish(16); + if (!(reg.ioctls & (1ULL << 3)) || !(api.features & (1ULL << 16))) finish(17); /* COPY and MOVE */ + struct wire data = { (u32)map, pages }; + struct iovec32 iov = { &data, sizeof(data) }; + struct rights32 rights = { sizeof(rights), 1, 1, fd }; + struct message32 msg = { 0, 0, &iov, 1, &rights, sizeof(rights), 0 }; + if (call(296, sock, (long)&msg, 0, 0, 0, 0) != sizeof(data)) finish(18); + volatile unsigned char *p = (void *)map; + /* Each first access waits for the native pager's COPY. */ + for (u32 leaf = 0; leaf < 3; leaf++) + for (u32 i = 0; i < pages; i++) + if (p[leaf * pages + i] != (unsigned char)(0x51 + leaf)) finish(19); + if (!shared) { + long target = call(192, 0, 3 * pages, 3, 0x22, -1, 0); + if ((unsigned long)target >= 0xfffff001UL) finish(21); + struct registration dst = { (u32)target, 3 * pages, 1, 0 }; + if (call(54, fd, 0xc020aa00U, (long)&dst, 0, 0, 0) || + !(dst.ioctls & (1ULL << 5))) finish(22); + struct movement m = { 1ULL << 32, (u32)map + 2 * pages, pages, 0, 0 }; + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) != -22) finish(23); + m.dst = (u32)target + pages; + if (pages > 4096) { + m.len = 4096; + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) != -22) finish(24); + m.len = pages; + } + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) || m.moved != pages) finish(25); + volatile unsigned char *q = (void *)target; + for (u32 i = 0; i < pages; i++) + if (q[pages + i] != 0x53 || p[i] != 0x51 || p[pages + i] != 0x52) finish(26); + if (call(91, target, 3 * pages, 0, 0, 0, 0)) finish(27); + static const char ok[] = "ok - AArch32 MOVE, neighbours and native-fallback alignment\n"; + call(4, 1, (long)ok, sizeof(ok) - 1, 0, 0, 0); + } + if (call(91, map, 3 * pages, 0, 0, 0, 0)) finish(20); + finish(0); +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/context-stress-init.c b/tools/testing/selftests/arm64/user4k/context-stress-init.c new file mode 100644 index 0000000000000..4dbc5be0fea80 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/context-stress-init.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* PID-1 mixed-granule EL0 stress runner for disposable VMs. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define DATA_ADDRESS 0x200000000UL +#define CODE_ADDRESS 0x202000000UL +#define CONTEXTS 70000 +#define SHORT_CONTEXTS 2048 + +static sigjmp_buf fault_return; +static volatile uintptr_t expected_fault; +static char generation_path[256]; + +static void fault(int sig, siginfo_t *info, void *context) +{ + (void)context; + if (sig != SIGSEGV || !expected_fault || + (uintptr_t)info->si_addr != expected_fault || info->si_code != SEGV_ACCERR) + _exit(90); + expected_fault = 0; + siglongjmp(fault_return, 1); +} + +static bool rejects_write(uintptr_t address) +{ + expected_fault = address; + if (!sigsetjmp(fault_return, 1)) { + *(volatile unsigned char *)expected_fault = 1; + expected_fault = 0; + return false; + } + return true; +} + +static int migrate(unsigned int cpu) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(0, sizeof(set), &set); +} + +static unsigned char pattern(unsigned long serial, unsigned long offset) +{ + serial ^= (serial >> 8) ^ (serial >> 16); + return (serial * 37 + offset * 17 + (offset >> 8)) & 255; +} + +static bool validate(unsigned char *p, unsigned long serial, unsigned long len) +{ + for (unsigned long i = 0; i < len; i++) + if (p[i] != pattern(serial, i)) + return false; + return true; +} + +static int worker(unsigned long serial, unsigned long ps, int control) +{ + struct sigaction action = { .sa_sigaction = fault, .sa_flags = SA_SIGINFO }; + unsigned char *data; + uint32_t *code; + unsigned int cpus = sysconf(_SC_NPROCESSORS_ONLN); + struct stat self, init; + unsigned long iterations = 0; + + if (getauxval(AT_PAGESZ) != ps || getppid() != 1 || + stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) != 0) + return 91; + if (cpus < 2 || migrate(serial % cpus) || sigaction(SIGSEGV, &action, NULL)) + return 92; + data = mmap((void *)DATA_ADDRESS, 4 * ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + code = mmap((void *)CODE_ADDRESS, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (data == MAP_FAILED || code == MAP_FAILED || + mprotect(data, 3 * ps, PROT_READ | PROT_WRITE)) + return 93; + for (unsigned long i = 0; i < 3 * ps; i++) + data[i] = pattern(serial, i); + if (!rejects_write(DATA_ADDRESS + 3 * ps)) + return 94; + if (mprotect(data + ps, ps, PROT_READ)) + return 95; + if (!rejects_write(DATA_ADDRESS + ps)) + return 96; + do { + unsigned int expected = (serial + iterations * 13) & 65535; + int (*fn)(void) = (void *)code; + char stop; + ssize_t got; + + if (mprotect(code, ps, PROT_READ | PROT_WRITE)) + return 97; + code[0] = 0x52800000 | (expected << 5); /* movz w0, #expected */ + code[1] = 0xd65f03c0; /* ret */ + __builtin___clear_cache((char *)code, (char *)(code + 2)); + if (mprotect(code, ps, PROT_READ | PROT_EXEC) || + migrate((serial + iterations + 1) % cpus)) + return 98; + sched_yield(); + if (fn() != (int)expected || !validate(data, serial, 3 * ps)) + return 99; + if (!(iterations % 64) && (!rejects_write(DATA_ADDRESS + 3 * ps) || + !rejects_write(DATA_ADDRESS + ps))) + return 101; + iterations++; + if (control < 0) + break; + got = read(control, &stop, 1); + if (got == 1) + break; + if (got != -1 || (errno != EAGAIN && errno != EINTR)) + return 100; + } while (true); + if (control >= 0) + printf("# persistent %luK worker: %lu migration/JIT/data cycles\n", ps / 1024, iterations); + return 0; +} + +static pid_t spawn(unsigned long serial, unsigned long ps, int control, int close_fd) +{ + pid_t pid = fork(); + + if (!pid) { + char seq[32], size[32], fd[32]; + + if (close_fd >= 0) + close(close_fd); + if (control >= 0 && fcntl(control, F_SETFD, 0)) + _exit(110); + snprintf(seq, sizeof(seq), "%lu", serial); + snprintf(size, sizeof(size), "%lu", ps); + snprintf(fd, sizeof(fd), "%d", control); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, ps, + 0UL, 0UL, 0UL)) + _exit(111); + execl("/init", "/init", "--worker", seq, size, fd, NULL); + _exit(112); + } + return pid; +} + +static bool reap(pid_t pid) +{ + int status = 0; + pid_t got; + + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + if (got == pid && WIFEXITED(status) && !WEXITSTATUS(status)) + return true; + printf("not ok - worker pid=%d wait=%d status=%#x\n", pid, got, status); + return false; +} + +static unsigned long long generation(void) +{ + unsigned long long value = 0; + FILE *f = fopen(generation_path, "re"); + + if (!f || fscanf(f, "%llu", &value) != 1) { + perror("ASID generation"); + return 0; + } + fclose(f); + return value; +} + +int main(int argc, char **argv) +{ + unsigned long long before, after; + unsigned char *data; + pid_t persistent[3]; + int commands[3], pipes[2]; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + glob_t paths; + bool ok = true; + unsigned int contexts = CONTEXTS; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (argc == 2 && !strcmp(argv[1], "--short")) + contexts = SHORT_CONTEXTS; + if (getpid() != 1 || (argc != 1 && contexts != SHORT_CONTEXTS)) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + glob("/sys/module/*/parameters/user4k_asid_generation", 0, NULL, &paths) || + paths.gl_pathc != 1) { + perror("fixture setup"); + return 2; + } + snprintf(generation_path, sizeof(generation_path), "%s", paths.gl_pathv[0]); + globfree(&paths); + data = mmap((void *)DATA_ADDRESS, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (data == MAP_FAILED) + return 3; + for (unsigned long i = 0; i < 4 * native; i++) + data[i] = pattern(0, i); + before = generation(); + printf("# mixed context stress: %u fork/exec workers, initial ASID generation=%llu\n", + contexts, before); + for (unsigned int i = 0; i < nr; i++) { + if (pipe2(pipes, O_NONBLOCK | O_CLOEXEC)) + return 4; + persistent[i] = spawn(CONTEXTS + i + 1, granules[i], pipes[0], pipes[1]); + if (persistent[i] < 0) + return 5; + close(pipes[0]); + commands[i] = pipes[1]; + } + for (unsigned long i = 1; i <= contexts; i++) { + pid_t pid = spawn(i, granules[i % nr], -1, -1); + + if (pid < 0 || !reap(pid) || !validate(data, 0, 4 * native)) { + printf("not ok - churn/parent isolation at worker %lu\n", i); + ok = false; + break; + } + if (!(i % 128)) + printf("# workers=%lu ASID generation=%llu\n", i, generation()); + } + after = generation(); + for (unsigned int i = 0; i < nr; i++) { + ok &= write(commands[i], "x", 1) == 1; + close(commands[i]); + ok &= reap(persistent[i]); + } + ok &= before && after >= before + 2 * (1ULL << 16); + printf("%s - mixed context stress: generation=%llu->%llu, measured rollovers=%llu\n", + ok ? "ok" : "not ok", before, after, (after - before) >> 16); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/core-fixture-init.c b/tools/testing/selftests/arm64/user4k/core-fixture-init.c new file mode 100644 index 0000000000000..312a9c58c8111 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/core-fixture-init.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Core-dump regression runner: PID 1 in a disposable VM only. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define ANON 0x100000000UL +#define FILEMAP 0x102000000UL +#define SPARSE 0x104000000UL +#define EXCLUDED 0x106000000UL +#define SWAPPED 0x108000000UL +#define SWAPPED_SIZE (1UL << 20) +#define SPARSE_SIZE (8UL << 20) + +static void die(const char *what) +{ + perror(what); + _exit(120); +} + +static void write_text(const char *path, const char *text) +{ + int fd = open(path, O_WRONLY); + size_t len = strlen(text); + + if (fd < 0 || write(fd, text, len) != (ssize_t)len) + die(path); + close(fd); +} + +static void *map_at(unsigned long address, size_t len, int prot, int fd, off_t off) +{ + void *p = mmap((void *)address, len, prot, + MAP_PRIVATE | MAP_FIXED_NOREPLACE | (fd < 0 ? MAP_ANONYMOUS : 0), fd, off); + + if (p == MAP_FAILED) + die("mmap"); + return p; +} + +static void crash(bool headers, int bounded) +{ + unsigned long ps = getauxval(AT_PAGESZ); + struct rlimit limit = { 64UL << 20, 64UL << 20 }; + struct stat self, init; + unsigned char *p, *data; + int fd; + + if (getppid() != 1 || stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino) + _exit(121); + if (bounded) + limit.rlim_cur = limit.rlim_max = ps - (bounded < 0); + if (setrlimit(RLIMIT_CORE, &limit)) + die("rlimit"); + write_text("/proc/self/coredump_filter", headers ? "0x10" : "0x1f"); + p = map_at(ANON, 4 * ps, PROT_READ | PROT_WRITE, -1, 0); + for (unsigned int i = 0; i < 4; i++) + memset(p + i * ps, 0x31 + i, ps); + if (mprotect(p + ps, ps, PROT_NONE) || mprotect(p + 2 * ps, ps, PROT_READ)) + die("mprotect"); + p = map_at(SPARSE, SPARSE_SIZE, PROT_READ | PROT_WRITE, -1, 0); + memset(p, 0x65, ps); + /* Also exercise an existing read-only zero PTE. */ + if (*(volatile unsigned char *)(p + ps) != 0) + _exit(122); + p = map_at(EXCLUDED, ps, PROT_READ | PROT_WRITE, -1, 0); + memset(p, 0xa7, ps); + if (madvise(p, ps, MADV_DONTDUMP)) + die("dontdump"); + fd = open("/core-input", O_CREAT | O_TRUNC | O_RDWR, 0700); + data = malloc(19 * ps); + if (fd < 0 || !data) + die("file setup"); + for (unsigned int i = 0; i < 19; i++) + memset(data + i * ps, 0x81 + i, ps); + memcpy(data, ELFMAG, SELFMAG); + if (write(fd, data, 19 * ps) != (ssize_t)(19 * ps)) + die("file data"); + /* Offset one user page: three mappings select non-native quarters. */ + map_at(FILEMAP, ps, PROT_READ, fd, headers ? 0 : ps); + map_at(FILEMAP + 2 * ps, 2 * ps, PROT_READ, fd, 2 * ps); + map_at(FILEMAP + 8 * ps, ps, PROT_READ, fd, 17 * ps); + close(fd); + fd = open("/core-library", O_CREAT | O_TRUNC | O_RDWR, 0600); + if (fd < 0 || write(fd, data, ps) != (ssize_t)ps) + die("non-executable ELF file"); + map_at(FILEMAP + 12 * ps, ps, PROT_READ, fd, 0); + close(fd); + free(data); + if (!headers && !bounded) { + unsigned int missing = 0; + uint64_t entry; + int pm = open("/proc/self/pagemap", O_RDONLY); + + p = map_at(SWAPPED, SWAPPED_SIZE, PROT_READ | PROT_WRITE, -1, 0); + memset(p, 0xc4, SWAPPED_SIZE); + for (unsigned int tries = 0; tries < 200; tries++) { + if (madvise(p, SWAPPED_SIZE, MADV_COLD) || + madvise(p, SWAPPED_SIZE, MADV_PAGEOUT)) + die("pageout"); + missing = 0; + for (unsigned long i = 0; i < SWAPPED_SIZE / ps; i++) { + if (pread(pm, &entry, sizeof(entry), (SWAPPED / ps + i) * 8) != 8) + die("pagemap"); + missing += !(entry & (1ULL << 62)); + } + if (!missing) + break; + usleep(10000); + } + close(pm); + if (missing) + _exit(124); + printf("# %luK dump precondition: all %lu leaves swapped\n", ps / 1024, + SWAPPED_SIZE / ps); + } + /* The file VMAs remain unfaulted, so dumping must retrieve file data. */ + raise(SIGABRT); + _exit(123); +} + +static Elf64_Phdr *find_load(unsigned char *data, Elf64_Ehdr *elf, unsigned long addr) +{ + Elf64_Phdr *ph = (void *)(data + elf->e_phoff); + + for (unsigned int i = 0; i < elf->e_phnum; i++) + if (ph[i].p_type == PT_LOAD && addr >= ph[i].p_vaddr && + addr - ph[i].p_vaddr < ph[i].p_memsz) + return ph + i; + return NULL; +} + +static bool bytes_at(unsigned char *data, Elf64_Ehdr *elf, size_t len, + unsigned long addr, size_t count, unsigned char expected) +{ + Elf64_Phdr *p = find_load(data, elf, addr); + unsigned long off; + + if (!p || addr - p->p_vaddr + count > p->p_filesz) + return false; + off = p->p_offset + addr - p->p_vaddr; + if (off > len || count > len - off) + return false; + for (size_t i = 0; i < count; i++) + if (data[off + i] != expected) + return false; + return true; +} + +static void inspect(pid_t pid, unsigned long ps, bool headers) +{ + char path[64]; + struct stat st; + unsigned char *data; + Elf64_Ehdr *elf; + Elf64_Phdr *ph, *p; + bool geometry = true, aux = false; + unsigned int file = 0; + unsigned long vdso = 0; + int fd; + + snprintf(path, sizeof(path), "/core.%d", pid); + fd = open(path, O_RDONLY); + if (fd < 0 || fstat(fd, &st) || st.st_size < (off_t)sizeof(*elf)) + ksft_exit_fail_msg("missing/short core %s\n", path); + data = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (data == MAP_FAILED) + die("core read"); + elf = (void *)data; + if (memcmp(elf->e_ident, ELFMAG, SELFMAG) || elf->e_type != ET_CORE || + elf->e_phentsize != sizeof(*ph) || elf->e_phoff > (uint64_t)st.st_size || + elf->e_phnum * sizeof(*ph) > (uint64_t)st.st_size - elf->e_phoff) + ksft_exit_fail_msg("invalid core ELF\n"); + ph = (void *)(data + elf->e_phoff); + for (unsigned int i = 0; i < elf->e_phnum; i++) { + if (ph[i].p_type == PT_LOAD) { + geometry &= ph[i].p_align == ps && ph[i].p_filesz <= ph[i].p_memsz && + ph[i].p_offset % ps == ph[i].p_vaddr % ps; + } else if (ph[i].p_type == PT_NOTE) { + size_t at = ph[i].p_offset, end = at + ph[i].p_filesz; + + if (end > (size_t)st.st_size || end < at) + ksft_exit_fail_msg("invalid notes\n"); + while (at + sizeof(Elf64_Nhdr) <= end) { + Elf64_Nhdr *n = (void *)(data + at); + size_t desc = at + sizeof(*n) + ((n->n_namesz + 3UL) & ~3UL); + uint64_t *words = (void *)(data + desc); + + if (desc > end || n->n_descsz > end - desc) + ksft_exit_fail_msg("invalid note length\n"); + if (n->n_type == NT_AUXV) { + for (size_t j = 0; j + 1 < n->n_descsz / 8; j += 2) { + if (words[j] == AT_PAGESZ) + aux = words[j + 1] == ps; + else if (words[j] == AT_SYSINFO_EHDR) + vdso = words[j + 1]; + } + } + if (n->n_type == NT_FILE && n->n_descsz >= 16 && + words[0] <= (n->n_descsz / 8 - 2) / 3 && words[1] == ps) { + for (size_t j = 0; j < words[0]; j++) { + unsigned long addr = words[2 + 3 * j]; + unsigned long offset = words[4 + 3 * j]; + + if (addr == FILEMAP && offset == (headers ? 0UL : 1UL)) + file |= 1; + if (addr == FILEMAP + 2 * ps && offset == 2) + file |= 2; + if (addr == FILEMAP + 8 * ps && offset == 17) + file |= 4; + } + } + at = desc + ((n->n_descsz + 3UL) & ~3UL); + } + } + } + ksft_test_result(geometry, "%luK core PT_LOAD granule/alignment%s\n", ps / 1024, + headers ? " headers-only" : ""); + ksft_test_result(aux && file == 7, "%luK core AUXV and NT_FILE offsets%s\n", ps / 1024, + headers ? " headers-only" : ""); + if (headers) { + bool skipped; + + p = find_load(data, elf, FILEMAP + 2 * ps); + skipped = p && !p->p_filesz; + p = find_load(data, elf, FILEMAP + 8 * ps); + ksft_test_result(skipped && p && !p->p_filesz, + "%luK header filter excludes nonzero quarter/native file offsets\n", ps / 1024); + p = find_load(data, elf, FILEMAP + 12 * ps); + ksft_test_result(p && p->p_filesz == ps && p->p_offset + ps <= (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, SELFMAG), + "%luK header filter recognizes non-executable ELF file\n", ps / 1024); + p = find_load(data, elf, FILEMAP); + ksft_test_result(p && p->p_filesz == ps && p->p_offset + ps <= (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, SELFMAG), + "%luK executable file header limited to one user page\n", ps / 1024); + } else { + bool anon = true, files = true; + + p = find_load(data, elf, vdso); + ksft_test_result(p && p->p_filesz >= SELFMAG && + p->p_offset + p->p_filesz <= (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, SELFMAG), + "%luK core vDSO ELF bytes\n", ps / 1024); + ksft_test_result(bytes_at(data, elf, st.st_size, SWAPPED, SWAPPED_SIZE, 0xc4), + "%luK core swapped anonymous bytes\n", ps / 1024); + + for (unsigned int i = 0; i < 4; i++) + anon &= bytes_at(data, elf, st.st_size, ANON + i * ps, ps, 0x31 + i); + for (unsigned int i = 1; i < 4; i++) + files &= bytes_at(data, elf, st.st_size, + FILEMAP + (i == 1 ? 0 : i * ps), ps, 0x81 + i); + ksft_test_result(anon, "%luK core anonymous and PROT_NONE/READ bytes\n", ps / 1024); + files &= bytes_at(data, elf, st.st_size, FILEMAP + 8 * ps, ps, 0x81 + 17); + ksft_test_result(files, "%luK core unfaulted file quarters\n", ps / 1024); + ksft_test_result(bytes_at(data, elf, st.st_size, SPARSE, ps, 0x65) && + bytes_at(data, elf, st.st_size, SPARSE + ps, SPARSE_SIZE - ps, 0), + "%luK core sparse and zero-page bytes\n", ps / 1024); + ksft_test_result(st.st_blocks * 512 < st.st_size / 2, + "%luK core keeps untouched anonymous range sparse\n", ps / 1024); + p = find_load(data, elf, EXCLUDED); + ksft_test_result(p && !p->p_filesz, "%luK core DONTDUMP excludes data\n", ps / 1024); + } + munmap(data, st.st_size); + close(fd); + unlink(path); +} + +int main(int argc, char **argv) +{ + unsigned long native_size = getauxval(AT_PAGESZ); + unsigned long sizes[] = { native_size, 4096, 16384 }; + unsigned int modes = native_size == 65536 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && !strcmp(argv[1], "--crash")) + crash(false, 0); + if (argc == 2 && !strcmp(argv[1], "--headers")) + crash(true, 0); + if (argc == 2 && !strcmp(argv[1], "--limit")) + crash(false, 1); + if (argc == 2 && !strcmp(argv[1], "--below-limit")) + crash(false, -1); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Use only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) && errno != EBUSY) + die("proc"); + if (mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || swapon("/dev/vda", 0)) + die("VM swap setup"); + write_text("/proc/sys/kernel/core_pattern", "/core.%p"); + ksft_print_header(); + ksft_set_plan(16 * modes); + for (unsigned int mode = 0; mode < modes; mode++) { + unsigned long ps = sizes[mode]; + for (unsigned int headers = 0; headers < 4; headers++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, ps == native_size ? 0UL : ps, + 0UL, 0UL, 0UL)) + die("exec size"); + execl("/init", "/init", headers == 3 ? "--below-limit" : headers == 2 ? "--limit" : + headers ? "--headers" : "--crash", NULL); + die("exec"); + } + if (pid < 0 || waitpid(pid, &status, 0) != pid || + !WIFSIGNALED(status) || WTERMSIG(status) != SIGABRT || (headers < 3 && !WCOREDUMP(status))) + ksft_exit_fail_msg("child/core failure status=%#x\n", status); + if (headers >= 2) { + char path[64]; + struct stat st; + bool exists, no_data = false; + + snprintf(path, sizeof(path), "/core.%d", pid); + exists = !stat(path, &st); + if (exists && headers == 2) { + Elf64_Ehdr elf; + Elf64_Phdr ph; + uint64_t data_start = UINT64_MAX; + int fd = open(path, O_RDONLY); + + if (fd < 0 || pread(fd, &elf, sizeof(elf), 0) != sizeof(elf)) + die("limited core header"); + for (unsigned int i = 0; i < elf.e_phnum; i++) { + if (pread(fd, &ph, sizeof(ph), elf.e_phoff + i * sizeof(ph)) != sizeof(ph)) + die("limited core phdr"); + if (ph.p_type == PT_LOAD && ph.p_offset < data_start) + data_start = ph.p_offset; + } + close(fd); + /* Sparse note padding is not charged to RLIMIT_CORE. */ + no_data = data_start != UINT64_MAX && st.st_size <= (off_t)data_start; + printf("# limited core: bytes=%lld first-data=%llu limit=%lu\n", + (long long)st.st_size, (unsigned long long)data_start, ps); + } + ksft_test_result(headers == 2 ? exists && st.st_size > 0 && + no_data : !exists && errno == ENOENT, + "%luK core RLIMIT_CORE %s one user page\n", ps / 1024, + headers == 2 ? "at" : "below"); + unlink(path); + } else { + inspect(pid, ps, headers); + } + } + } + ksft_print_cnts(); + printf("%s - core fixture complete\n", ksft_get_fail_cnt() ? "not ok" : "ok"); + reboot(RB_POWER_OFF); + return ksft_get_fail_cnt() ? 1 : 0; +} diff --git a/tools/testing/selftests/arm64/user4k/cpu-granule-init.c b/tools/testing/selftests/arm64/user4k/cpu-granule-init.c new file mode 100644 index 0000000000000..45690deb4a8e3 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/cpu-granule-init.c @@ -0,0 +1,150 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static int pin_cpu(pid_t pid, unsigned int cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(pid, sizeof(set), &set); +} +static int online_cpu1(void) +{ + char c = '?'; + int fd = open("/sys/devices/system/cpu/cpu1/online", O_RDONLY); + if (fd >= 0) { if (read(fd, &c, 1) != 1) c = '?'; close(fd); } + return c == '0' ? 0 : c == '1' ? 1 : -1; +} +static int worker(int ready, int command, unsigned long expected) +{ + unsigned char *p; + unsigned long ps = getauxval(AT_PAGESZ); + char cpu; + int ok = getppid() == 1 && ps == expected; + + p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) return 2; + for (unsigned long i = 0; i < 4 * ps; i++) p[i] = i ^ (i >> 12); + if (write(ready, "r", 1) != 1 || read(command, &cpu, 1) != 1) return 3; + ok &= sched_getcpu() == cpu - '0'; + for (unsigned int round = 0; round < 128; round++) { + for (unsigned long i = 0; i < 4 * ps; i++) + if (p[i] != (unsigned char)(i ^ (i >> 12))) ok = 0; + sched_yield(); + } + munmap(p, 4 * ps); + return ok ? 0 : 4; +} +static int compat_sample(unsigned long expected) +{ + uint64_t data[4] = {0}; + int pipefd[2], status; + pid_t child; + ssize_t size; + + if (pipe(pipefd)) return 0; + child = fork(); + if (!child) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + execl("/compat-aslr", "/compat-aslr", "--sample", fd, NULL); + _exit(2); + } + close(pipefd[1]); + size = read(pipefd[0], data, sizeof(data)); + close(pipefd[0]); + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && + size == sizeof(data) && data[0] == expected; +} +int main(int argc, char **argv) +{ + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), strtoul(argv[4], NULL, 10)); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Dedicated two-CPU disposable VM with maxcpus=1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) { + perror("mount"); failed++; goto done; + } + unsigned long native = getauxval(AT_PAGESZ); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + result(!prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL), "native exec default restored"); + char cmdline[4096] = {0}; + int fd = open("/proc/cmdline", O_RDONLY); + if (fd < 0 || read(fd, cmdline, sizeof(cmdline) - 1) < 0) { failed++; goto done; } + close(fd); + int reject = has4k && strstr(cmdline, "arm64.user4k_deny_late_cpu=1"); + result(online_cpu1() == 0, "CPU1 starts offline"); + if (online_cpu1() != 0) goto done; + int ready[2], command[2]; + if (pipe(ready) || pipe(command)) { failed++; goto done; } + pid_t child = fork(); + if (!child) { + char ready_fd[32], command_fd[32], ps[32]; + close(ready[0]); close(command[1]); + snprintf(ready_fd, sizeof(ready_fd), "%d", ready[1]); + snprintf(command_fd, sizeof(command_fd), "%d", command[0]); + snprintf(ps, sizeof(ps), "%lu", has4k ? 4096UL : native); + if (prctl(SET_EXEC_PAGE_SIZE, has4k ? 4096UL : 0UL, 0UL, 0UL, 0UL)) _exit(5); + execl("/init", "/init", "--worker", ready_fd, command_fd, ps, NULL); + _exit(6); + } + close(ready[1]); close(command[0]); + if (child < 0) { failed++; goto done; } + char token; + result(read(ready[0], &token, 1) == 1 && token == 'r', "worker has live mappings before hotplug"); + close(ready[0]); + fd = open("/sys/devices/system/cpu/cpu1/online", O_WRONLY); + errno = 0; + ssize_t written = fd < 0 ? -1 : write(fd, "1", 1); + int saved_errno = errno; + if (fd >= 0) close(fd); + int online = online_cpu1(); + printf("# has4k=%d expected-reject=%d write=%zd errno=%d online=%d\n", + has4k, reject, written, saved_errno, online); + result(reject ? written < 0 && online == 0 : written == 1 && online == 1, + "late CPU admission matches advertised user granule"); + unsigned int cpu = online == 1 ? 1 : 0; + result(!pin_cpu(child, cpu), "worker affinity targets admitted CPU"); + token = '0' + cpu; + result(write(command[1], &token, 1) == 1, "worker resumes after CPU admission"); + close(command[1]); + int status; + result(waitpid(child, &status, 0) == child && WIFEXITED(status) && !WEXITSTATUS(status), + "live mapping bytes survive and worker runs on expected CPU"); + result(!pin_cpu(0, cpu), "launcher affinity targets admitted CPU"); + result(compat_sample(has4k ? 4096 : native), "AArch32 exec retains selected or native-fallback granule"); + done: + printf("CPU GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); reboot(RB_POWER_OFF); + for (;;) pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c b/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c new file mode 100644 index 0000000000000..3e8c9a4678c4c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c @@ -0,0 +1,153 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static int pin_cpu(pid_t pid, unsigned int cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(pid, sizeof(set), &set); +} +static int online_cpu1(void) +{ + char c = '?'; + int fd = open("/sys/devices/system/cpu/cpu1/online", O_RDONLY); + if (fd >= 0) { if (read(fd, &c, 1) != 1) c = '?'; close(fd); } + return c == '0' ? 0 : c == '1' ? 1 : -1; +} +static int worker(int ready, int command, unsigned long expected) +{ + unsigned char *p; + unsigned long ps = getauxval(AT_PAGESZ); + char cpu; + int ok = getppid() == 1 && ps == expected; + + p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) return 2; + for (unsigned long i = 0; i < 4 * ps; i++) p[i] = i ^ (i >> 12); + if (write(ready, "r", 1) != 1 || read(command, &cpu, 1) != 1) return 3; + ok &= sched_getcpu() == cpu - '0'; + for (unsigned int round = 0; round < 128; round++) { + for (unsigned long i = 0; i < 4 * ps; i++) + if (p[i] != (unsigned char)(i ^ (i >> 12))) ok = 0; + sched_yield(); + } + munmap(p, 4 * ps); + return ok ? 0 : 4; +} +static int compat_sample(unsigned long expected) +{ + uint64_t data[4] = {0}; + int pipefd[2], status; + pid_t child; + ssize_t size; + + if (pipe(pipefd)) return 0; + child = fork(); + if (!child) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + execl("/compat-aslr", "/compat-aslr", "--sample", fd, NULL); + _exit(2); + } + close(pipefd[1]); + size = read(pipefd[0], data, sizeof(data)); + close(pipefd[0]); + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && + size == sizeof(data) && data[0] == expected; +} +int main(int argc, char **argv) +{ + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), strtoul(argv[4], NULL, 10)); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Dedicated two-CPU disposable VM with maxcpus=1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) { + perror("mount"); failed++; goto done; + } + unsigned long native = getauxval(AT_PAGESZ); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + int has16k = native > 16384 && !prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL); + result(!prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL), "native exec default restored"); + char cmdline[4096] = {0}; + int fd = open("/proc/cmdline", O_RDONLY); + if (fd < 0 || read(fd, cmdline, sizeof(cmdline) - 1) < 0) { failed++; goto done; } + close(fd); + int reject = has16k && strstr(cmdline, "arm64.user16k_deny_late_cpu=1"); + result(online_cpu1() == 0, "CPU1 starts offline"); + if (online_cpu1() != 0) goto done; + int ready[2], command[2]; + if (pipe(ready) || pipe(command)) { failed++; goto done; } + pid_t child = fork(); + if (!child) { + char ready_fd[32], command_fd[32], ps[32]; + close(ready[0]); close(command[1]); + snprintf(ready_fd, sizeof(ready_fd), "%d", ready[1]); + snprintf(command_fd, sizeof(command_fd), "%d", command[0]); + snprintf(ps, sizeof(ps), "%lu", has16k ? 16384UL : native); + if (prctl(SET_EXEC_PAGE_SIZE, has16k ? 16384UL : 0UL, 0UL, 0UL, 0UL)) _exit(5); + execl("/init", "/init", "--worker", ready_fd, command_fd, ps, NULL); + _exit(6); + } + close(ready[1]); close(command[0]); + if (child < 0) { failed++; goto done; } + char token; + result(read(ready[0], &token, 1) == 1 && token == 'r', "worker has live mappings before hotplug"); + close(ready[0]); + fd = open("/sys/devices/system/cpu/cpu1/online", O_WRONLY); + errno = 0; + ssize_t written = fd < 0 ? -1 : write(fd, "1", 1); + int saved_errno = errno; + if (fd >= 0) close(fd); + int online = online_cpu1(); + printf("# has16k=%d expected-reject=%d write=%zd errno=%d online=%d\n", + has16k, reject, written, saved_errno, online); + result(reject ? written < 0 && online == 0 : written == 1 && online == 1, + "late CPU admission matches advertised user granule"); + unsigned int cpu = online == 1 ? 1 : 0; + result(!pin_cpu(child, cpu), "worker affinity targets admitted CPU"); + token = '0' + cpu; + result(write(command[1], &token, 1) == 1, "worker resumes after CPU admission"); + close(command[1]); + int status; + result(waitpid(child, &status, 0) == child && WIFEXITED(status) && !WEXITSTATUS(status), + "live mapping bytes survive and worker runs on expected CPU"); + result(!pin_cpu(0, cpu), "launcher affinity targets admitted CPU"); + if (has16k) + result(!prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL), "16K request staged before compat exec"); + result(compat_sample(has4k ? 4096 : native), "AArch32 exec selects4K or native despite pending16K"); + done: + printf("CPU GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); reboot(RB_POWER_OFF); + for (;;) pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/dio-fixture-init.c b/tools/testing/selftests/arm64/user4k/dio-fixture-init.c new file mode 100644 index 0000000000000..a6b9034e3bb6f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/dio-fixture-init.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static bool use_file; +static unsigned char pattern(size_t i) { return (i * 17 + (i >> 12) * 29 + 71) & 255; } +static void fill(unsigned char *p, size_t len) { for (size_t i=0;i=0 && !ftruncate(memfd,4*ps); + unsigned char *shared=ok?mmap(NULL,4*ps,PROT_READ|PROT_WRITE,MAP_SHARED,memfd,0):MAP_FAILED; + ok &= shared!=MAP_FAILED; + if (ok) { + memset(shared,0xa7,4*ps); + ok=pread(fd,shared+ps,2*ps,4*1024*1024)==(ssize_t)(2*ps) && + !msync(shared,4*ps,MS_SYNC) && pread(memfd,q,4*ps,0)==(ssize_t)(4*ps); + ok &= check(q+ps,2*ps,0); + for (size_t i=0;i=0)close(memfd); + report(ok,ps,"shared-file destination dirtying and adjacent guards"); + + ok=aio_transfer(fd,p+ps,7*ps,12*1024*1024,true); + memset(q,0,8*ps); + ok &= aio_transfer(fd,q+ps,7*ps,12*1024*1024,false) && check(q+ps,7*ps,ps); + report(ok,ps,"asynchronous AIO read/write completion"); + + memset(q,0x96,8*ps); + pid_t child=fork();int status=0; + if(!child){ _exit(aio_transfer(fd,q+ps,7*ps,12*1024*1024,false)&&check(q+ps,7*ps,ps)?0:1); } + ok=child>0 && waitpid(child,&status,0)==child && WIFEXITED(status) && !WEXITSTATUS(status); + for(size_t i=0;i<8*ps;i++)ok &= q[i]==0x96; + report(ok,ps,"writable-pin COW after fork"); + + ok=true; + for(unsigned i=0;i<128 && ok;i++) { + ok=!madvise(q+ps,ps,MADV_DONTNEED) && pread(fd,q+ps,ps,4*1024*1024)==(ssize_t)ps && check(q+ps,ps,0); + ok &= !mprotect(q+2*ps,ps,PROT_NONE); + ok &= pread(fd,q+2*ps,ps,4*1024*1024)==-1 && errno==EFAULT; + ok &= !mprotect(q+2*ps,ps,PROT_READ|PROT_WRITE); + } + report(ok,ps,"128 reuse and failed-pin cleanup cycles"); + close(fd);munmap(p,len);munmap(q,len);ksft_print_cnts(); + return ksft_get_fail_cnt()?1:0; +} +int main(int argc,char **argv) +{ + bool ok=true; + setvbuf(stdout,NULL,_IONBF,0); + if(argc==2 && (!strcmp(argv[1],"--test") || !strcmp(argv[1],"--file")) && getppid()==1){use_file=!strcmp(argv[1],"--file");return exercise();} + if(getpid()!=1 || argc!=1){fprintf(stderr,"Disposable VM PID1 only\n");return 2;} + if(mount("proc","/proc","proc",0,NULL)||mount("sysfs","/sys","sysfs",0,NULL))return 3; + for(unsigned disk=0;disk<2;disk++) { + char path[80],serial[64];unsigned maj,min; + snprintf(path,sizeof(path),"/sys/class/block/vd%c/serial",'a'+disk); + FILE *dev=fopen(path,"r"); + if(!dev || fscanf(dev,"%63s",serial)!=1)return 4; + fclose(dev); + snprintf(path,sizeof(path),"/sys/class/block/vd%c/dev",'a'+disk); + dev=fopen(path,"r"); + if(!dev || fscanf(dev,"%u:%u",&maj,&min)!=2)return 4; + fclose(dev); + const char *node=!strcmp(serial,"au4k-raw")?"/dev/user4k-test": + !strcmp(serial,"au4k-ext4")?"/dev/user4k-fs":NULL; + if(!node || mknod(node,S_IFBLK|0600,makedev(maj,min)))return 5; + } + if(mount("/dev/user4k-fs","/fs","ext4",0,NULL))return 7; + for(unsigned mode=0;mode<4;mode++) { + unsigned small=mode&1; + pid_t pid=fork();int status=0; + if(!pid){if(prctl(PR_AURORA_SET_EXEC_PAGE_SIZE,small?4096UL:0UL,0UL,0UL,0UL))_exit(3);execl("/init","/init",mode>=2?"--file":"--test",NULL);_exit(4);} + ok &= pid>0 && waitpid(pid,&status,0)==pid && WIFEXITED(status) && !WEXITSTATUS(status); + } + ok &= !umount("/fs"); + printf("%s - direct I/O fixture complete\n",ok?"ok":"not ok"); + reboot(RB_POWER_OFF);return ok?0:1; +} diff --git a/tools/testing/selftests/arm64/user4k/exec-fixture-init.c b/tools/testing/selftests/arm64/user4k/exec-fixture-init.c new file mode 100644 index 0000000000000..1c96173c46b76 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/exec-fixture-init.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* PID-1 runner for the experimental one-shot exec ABI in a disposable VM. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include + +static int exec_size(unsigned long size) +{ + return prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL); +} + +static int pending_size(void) +{ + return prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); +} + +static int run(const char *path, const char *arg, unsigned long size) +{ + pid_t pid = fork(), got; + int status = 0; + + if (!pid) { + if (size && exec_size(size)) { + perror("set exec page size"); + _exit(125); + } + execl(path, path, arg, NULL); + perror(path); + _exit(126); + } + if (pid < 0) + return -1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + printf("# child %s %s: wait=%d status=%#x\n", path, arg ? arg : "", got, status); + return got == pid && WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +static int argument_exec(unsigned long size) +{ + pid_t pid = fork(), got; + int status = 0; + + if (!pid) { + char *payload = malloc(140000), expected[16]; + unsigned int i; + + if (!payload || exec_size(size)) + _exit(120); + for (i = 0; i < 139999; i++) + payload[i] = 'a' + i % 23; + payload[139999] = 0; + if (size == 4096) { + execl("/init", "/init", "--too-long", payload, NULL); + if (errno != E2BIG || pending_size() != 4096) + _exit(121); + } + payload[120000] = 0; + if (setenv("ARG_PAYLOAD", payload, 1)) + _exit(122); + snprintf(expected, sizeof(expected), "%lu", size ? size : 16384UL); + execl("/init", "/init", "--args-check", expected, payload, NULL); + perror("argument exec"); + _exit(123); + } + if (pid < 0) + return -1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + printf("# large arguments from %lu to %lu: status=%#x\n", + getauxval(AT_PAGESZ), size ? size : 16384UL, status); + return got == pid && WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +int main(int argc, char **argv) +{ + bool ok; + int failures = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--args-check")) { + const char *env = getenv("ARG_PAYLOAD"); + unsigned int i; + + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10) || + pending_size() != 0 || strlen(argv[3]) != 120000 || + !env || strlen(env) != 120000) + return 1; + for (i = 0; i < 120000; i++) + if (argv[3][i] != 'a' + i % 23 || env[i] != argv[3][i]) + return 2; + return 0; + } + if (argc == 2 && !strcmp(argv[1], "--args-from-4k")) { + if (getauxval(AT_PAGESZ) != 4096) + return 1; + return argument_exec(4096) || argument_exec(0); + } + if (argc == 2 && !strcmp(argv[1], "--exec-cycle")) { + if (getauxval(AT_PAGESZ) != 4096 || pending_size() != 0) + return 1; + /* A successful exec consumes the request; ordinary exec is native. */ + execl("/init", "/init", "--native-after-4k", NULL); + return 2; + } + if (argc == 2 && !strcmp(argv[1], "--native-after-4k")) + return getauxval(AT_PAGESZ) != 16384 || pending_size() != 0; + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as /init in a disposable VM.\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) && errno != EBUSY) + return 3; + printf("# exec fixture: native AT_PAGESZ=%lu\n", getauxval(AT_PAGESZ)); + failures += run("/page-contract", NULL, 0) != 0; + ok = pending_size() == 0; + errno = 0; + ok &= exec_size(8192) == -1 && errno == EINVAL; + ok &= prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096UL, 1UL, 0UL, 0UL) == -1; + ok &= exec_size(4096) == 0 && pending_size() == 4096; + execl("/does-not-exist", "/does-not-exist", NULL); + ok &= errno == ENOENT && pending_size() == 4096; + ok &= getauxval(AT_PAGESZ) == 16384; + printf("%s - request validation and failed exec preserve live geometry\n", ok ? "ok" : "not ok"); + failures += !ok; + /* fork inherits the pending request; successful child exec consumes it. */ + failures += run("/page-contract-4k", "--expect-4k", 0) != 0; + ok = pending_size() == 4096 && exec_size(0) == 0 && pending_size() == 0; + printf("%s - child exec leaves parent request intact; reset works\n", ok ? "ok" : "not ok"); + failures += !ok; + failures += run("/page-contract-dynamic-4k", "--expect-4k", 4096) != 0; + failures += run("/init", "--exec-cycle", 4096) != 0; + failures += run("/proc-memory", NULL, 0) != 0; + failures += run("/proc-memory", "--expect-4k", 4096) != 0; + failures += run("/proc-pages", NULL, 0) != 0; + failures += run("/proc-pages", "--expect-4k", 4096) != 0; + failures += run("/advice", NULL, 0) != 0; + failures += run("/advice", "--expect-4k", 4096) != 0; + failures += run("/memory-interfaces", NULL, 0) != 0; + failures += run("/memory-interfaces", "--expect-4k", 4096) != 0; + failures += run("/mixed-process", NULL, 0) != 0; + failures += run("/mixed-process", NULL, 4096) != 0; + failures += argument_exec(4096) != 0; + failures += run("/init", "--args-from-4k", 4096) != 0; + failures += run("/page-contract", NULL, 0) != 0; + printf("%s - exec fixture complete: failures=%d\n", failures ? "not ok" : "ok", failures); + reboot(RB_POWER_OFF); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/arm64/user4k/exec-policy-init.c b/tools/testing/selftests/arm64/user4k/exec-policy-init.c new file mode 100644 index 0000000000000..86219acc439a1 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/exec-policy-init.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable guest PID1: per-thread one-shot and inherited exec preferences. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define REQUIRE(x) do { if (!(x)) { \ + fprintf(stderr, "FAIL line %d: %s errno=%d\n", __LINE__, #x, errno); \ + exit(1); } } while (0) + +static int policy(int op, unsigned long value) +{ + return prctl(op, value, 0UL, 0UL, 0UL); +} + +static void state(unsigned long pages, unsigned long preferred, int pending) +{ + REQUIRE(getauxval(AT_PAGESZ) == pages); + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 0) == pending); +} + +static void chain(int stage, unsigned long native, unsigned long preferred) +{ + char s[32], n[32], p[32]; + + snprintf(s, sizeof(s), "%d", stage); + snprintf(n, sizeof(n), "%lu", native); + snprintf(p, sizeof(p), "%lu", preferred); + execl("/init", "/init", "--chain", s, n, p, NULL); + REQUIRE(0); +} + +static void *thread_policy(void *arg) +{ + unsigned long preferred = *(unsigned long *)arg; + + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 0)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 0)); + state(getauxval(AT_PAGESZ), 0, 0); + return NULL; +} + +static void *thread_exec(void *arg) +{ + unsigned long *sizes = arg; + + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, sizes[1])); + chain(0, sizes[0], sizes[1]); + return NULL; +} + +static int wait_child(pid_t pid) +{ + int status; + + REQUIRE(pid > 0); + REQUIRE(waitpid(pid, &status, 0) == pid); + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +static int run_preference(unsigned long native, unsigned long preferred) +{ + pid_t pid = fork(); + + if (!pid) { + pthread_t thread; + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, preferred)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, preferred)); + REQUIRE(!pthread_create(&thread, NULL, thread_policy, &preferred)); + REQUIRE(!pthread_join(thread, NULL)); + state(native, preferred, preferred); + /* Rejected values and reserved arguments must preserve both policies. */ + REQUIRE(policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 8192) == -1 && errno == EINVAL); + REQUIRE(policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 8192) == -1 && errno == EINVAL); + int ops[] = {PR_AURORA_SET_DEFAULT_PAGE_SIZE, PR_AURORA_GET_DEFAULT_PAGE_SIZE, + PR_AURORA_SET_EXEC_PAGE_SIZE, PR_AURORA_GET_EXEC_PAGE_SIZE}; + for (unsigned int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + REQUIRE(prctl(ops[i], 0UL, 1UL, 0UL, 0UL) == -1 && errno == EINVAL); + REQUIRE(prctl(ops[i], 0UL, 0UL, 1UL, 0UL) == -1 && errno == EINVAL); + REQUIRE(prctl(ops[i], 0UL, 0UL, 0UL, 1UL) == -1 && errno == EINVAL); + } + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 1) == -1 && errno == EINVAL); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 1) == -1 && errno == EINVAL); + execl("/does-not-exist", "/does-not-exist", NULL); + REQUIRE(errno == ENOENT); + state(native, preferred, preferred); + chain(0, native, preferred); + } + return wait_child(pid); +} + +static int compat(unsigned long native, unsigned long preferred, int no4k) +{ + pid_t pid = fork(); + + if (!pid) { + char n[32], p[32], e[32]; + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, preferred)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, native)); + snprintf(n, sizeof(n), "%lu", native); + snprintf(p, sizeof(p), "%lu", preferred); + snprintf(e, sizeof(e), "%lu", no4k ? native : 4096UL); + execl("/compat-policy", "/compat-policy", n, p, e, NULL); + REQUIRE(0); + } + return wait_child(pid); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--chain")) { + int stage = atoi(argv[2]); + unsigned long n = strtoul(argv[3], NULL, 10); + unsigned long p = strtoul(argv[4], NULL, 10); + + state(stage == 2 || stage == 5 ? n : p, stage < 4 ? p : 0, 0); + if (stage == 5) return 0; + if (stage == 1) { + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 0)); + state(p, p, n); + } + if (stage == 3) { + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 0)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, p)); + } + chain(stage + 1, n, p); + } + if (argc == 4 && !strcmp(argv[1], "--compat-return")) { + unsigned long n = strtoul(argv[2], NULL, 10); + unsigned long p = strtoul(argv[3], NULL, 10); + state(p ? p : n, p, 0); + return 0; + } + REQUIRE(getpid() == 1); + REQUIRE(!mount("proc", "/proc", "proc", 0, NULL)); + state(native, 0, 0); + int no4k = argc == 2 && !strcmp(argv[1], "--no4k"); + unsigned long preferences[] = {4096, 16384}; + for (unsigned int i = 0; i < 2; i++) { + unsigned long p = preferences[i]; + if (p >= native || (no4k && p == 4096)) continue; + int ok = run_preference(native, p); + printf("%s - exec policy native=%lu alternative=%lu\n", ok ? "ok" : "not ok", native, p); + failed |= !ok; + pid_t pid = fork(); + if (!pid) { + pthread_t thread; + unsigned long sizes[] = {native, p}; + REQUIRE(!pthread_create(&thread, NULL, thread_exec, sizes)); + for (;;) pause(); + } + ok = wait_child(pid); + printf("%s - non-leader exec alternative=%lu\n", ok ? "ok" : "not ok", p); + failed |= !ok; + ok = compat(native, p, no4k); + printf("%s - compat policy alternative=%lu no4k=%d\n", ok ? "ok" : "not ok", p, no4k); + failed |= !ok; + } + if (no4k) { + int ok = policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 4096) == -1 && errno == EOPNOTSUPP; + ok &= policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096) == -1 && errno == EOPNOTSUPP; + ok &= compat(native, 0, 1); + printf("%s - no4k preserves native compat restrictions\n", ok ? "ok" : "not ok"); + failed |= !ok; + } + state(native, 0, 0); + printf("EXEC POLICY %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + return failed; +} diff --git a/tools/testing/selftests/arm64/user4k/fex-contract-init.c b/tools/testing/selftests/arm64/user4k/fex-contract-init.c new file mode 100644 index 0000000000000..346044412c4b1 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-contract-init.c @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +int main(void) +{ + if (getpid() != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + int failed=0;if(mount("proc","/proc","proc",0,NULL)||mount("sysfs","/sys","sysfs",0,NULL)||mount("devtmpfs","/dev","devtmpfs",0,NULL)){perror("mount");failed=1;goto done;} + pid_t child=fork();if(!child){setenv("HOME","/root",1);setenv("PATH","/usr/bin:/bin",1);setenv("XDG_RUNTIME_DIR","/run",1);setenv("FEX_ROOTFS","/",1);setenv("LD_PRELOAD","/usr/lib/libfex-abi.so",1);if(prctl(PR_AURORA_SET_DEFAULT_PAGE_SIZE,4096UL,0UL,0UL,0UL) || prctl(PR_AURORA_SET_EXEC_PAGE_SIZE,4096UL,0UL,0UL,0UL))_exit(3);execl("/usr/bin/FEXInterpreter","/usr/bin/FEXInterpreter","/x86-contract",NULL);perror("FEX exec");_exit(4);} + int status=0;int ok=child>0&&waitpid(child,&status,0)==child&&WIFEXITED(status)&&!WEXITSTATUS(status);printf("%s - FEX 4K native-process / x86 workload status=%#x\n",ok?"ok":"not ok",status);failed|=!ok; + done:printf("GRANULE FEX RUNTIME %s\n",failed?"FAIL":"PASS");sync();reboot(RB_POWER_OFF);for(;;)pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/fex-native-abi.c b/tools/testing/selftests/arm64/user4k/fex-native-abi.c new file mode 100644 index 0000000000000..c22b06bdf5559 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-native-abi.c @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +__attribute__((constructor)) static void check_abi(void) +{ + char path[1024];ssize_t n=readlink("/proc/self/exe",path,sizeof(path)-1);if(n<0)return;path[n]=0; + if(!strstr(path,"FEXInterpreter"))return; + unsigned long ps=getauxval(AT_PAGESZ); + dprintf(2,"FEX NATIVE HOST PROCESS PAGE SIZE %lu\n",ps); + if(ps!=4096)_exit(90); +} diff --git a/tools/testing/selftests/arm64/user4k/fex-x86-contract.c b/tools/testing/selftests/arm64/user4k/fex-x86-contract.c new file mode 100644 index 0000000000000..a02d12e205743 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-x86-contract.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0-only +typedef unsigned long size_t; +static long sc(long n,long a,long b,long c,long d,long e,long f) +{ + register long r10 __asm__("r10")=d,r8 __asm__("r8")=e,r9 __asm__("r9")=f;long r; + __asm__ volatile("syscall":"=a"(r):"a"(n),"D"(a),"S"(b),"d"(c),"r"(r10),"r"(r8),"r"(r9):"rcx","r11","memory");return r; +} +static void say(const char*s,size_t n){sc(1,1,(long)s,n,0,0,0);} +#define SAY(s) say(s,sizeof(s)-1) +static void quit(int n){sc(231,n,0,0,0,0,0);for(;;);} +static int check(unsigned char*p,size_t size){for(size_t i=0;i>12)))return 0;return 1;} +long run(unsigned long*sp) +{ + unsigned long argc=*sp++,pagesz=0;sp+=argc+1;unsigned long *env=sp;while(*sp)sp++;sp++; + while(*sp){if(sp[0]==6)pagesz=sp[1];sp+=2;} + if(pagesz!=4096)return 10; + SAY("ok - FEX x86 guest 4K auxv\n"); + unsigned char*p=(void*)sc(9,0,12288,3,0x22,-1,0);if((long)p<0)return 11; + for(size_t i=0;i<12288;i++)p[i]=(unsigned char)(i^(i>>12)); + if(sc(10,(long)p+4096,4096,0,0,0,0))return 12; + long child=sc(57,0,0,0,0,0,0);int status=0; + if(!child){*(volatile unsigned char*)(p+4096)=1;quit(13);} + if(child<0||sc(61,child,(long)&status,0,0,0,0)!=child||(status&127)!=11)return 14; + if(sc(10,(long)p+4096,4096,3,0,0,0)||!check(p,12288))return 15; + SAY("ok - FEX x86 independent 4K protection fault\n"); + child=sc(57,0,0,0,0,0,0); + if(!child){p[4096]=0x71;quit(0);} + if(child<0||sc(61,child,(long)&status,0,0,0,0)!=child||status||!check(p,12288))return 16; + SAY("ok - FEX x86 fork COW\n"); + p=(void*)sc(25,(long)p,12288,16384,1,0,0);if((long)p<0||!check(p,12288))return 17; + for(size_t i=12288;i<16384;i++)if(p[i])return 18; + SAY("ok - FEX x86 mremap preserve and zero extension\n"); + long fd=sc(319,(long)"fex-shared",0,0,0,0,0);if(fd<0||sc(77,fd,12288,0,0,0,0))return 19; + unsigned char*a=(void*)sc(9,0,4096,3,1,fd,4096),*b=(void*)sc(9,0,4096,3,1,fd,4096); + if((long)a<0||(long)b<0)return 20; + a[37]=0x65;if(b[37]!=0x65)return 21; + SAY("ok - FEX x86 memfd mapping at 4K file offset\n"); + unsigned char*code=(void*)sc(9,0,4096,3,0x22,-1,0);if((long)code<0)return 22; + code[0]=0xb8;code[1]=42;code[2]=0;code[3]=0;code[4]=0;code[5]=0xc3; + if(sc(10,(long)code,4096,5,0,0,0)||((int(*)(void))code)()!=42)return 23; + if(sc(10,(long)code,4096,3,0,0,0))return 24;code[1]=99; + if(sc(10,(long)code,4096,5,0,0,0)||((int(*)(void))code)()!=99)return 25; + SAY("ok - FEX x86 JIT permission transition and code replacement\n"); + unsigned char random[32];if(sc(318,(long)random,32,0,0,0,0)!=32)return 26; + if(argc==1){ + child=sc(57,0,0,0,0,0,0); + if(!child){char*args[]={"/x86-contract","--exec-child",0};sc(59,(long)args[0],(long)args,(long)env,0,0,0);quit(28);} + if(child<0||sc(61,child,(long)&status,0,0,0,0)!=child||status)return 27; + SAY("ok - FEX x86 fork and re-exec with 4K mappings\n"); + } + SAY("FEX X86 4K CONTRACT PASS\n");return 0; +} +__asm__(".global _start\n_start:\nmov %rsp,%rdi\nand $-16,%rsp\ncall run\nmov %rax,%rdi\nmov $231,%rax\nsyscall\n"); diff --git a/tools/testing/selftests/arm64/user4k/gem-fixture-init.c b/tools/testing/selftests/arm64/user4k/gem-fixture-init.c new file mode 100644 index 0000000000000..badebc6af2bb5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/gem-fixture-init.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static unsigned char value(size_t off) { return 0x31+(off/4096)%128; } +static bool check(unsigned char *p,size_t len,size_t off) +{ + for(size_t i=0;i=0 && !ioctl(fd,DRM_IOCTL_MODE_CREATE_DUMB,&create); + ksft_print_header();ksft_set_plan(12);ksft_print_msg("ABI=%zuK\n",ps/1024); + map.handle=create.handle;ok &= !ioctl(fd,DRM_IOCTL_MODE_MAP_DUMB,&map) && create.size>=len; + if(ok)p=mmap(NULL,len,PROT_READ|PROT_WRITE,MAP_SHARED,fd,map.offset); + ok &= p!=MAP_FAILED;ksft_test_result(ok,"full GEM shmem object mapping\n"); + q=mmap(NULL,2*ps,PROT_READ|PROT_WRITE,MAP_SHARED,fd,map.offset); + bool separate=q!=MAP_FAILED; + if(separate){memset(q,0x39,ps);memset(q+ps,0xa2,ps);for(size_t i=0;i0 && waitpid(child,&status,0)==child && WIFEXITED(status)&&!WEXITSTATUS(status)&&p[0]==0xe1; + p[0]=value(0); + }else ok=false; + ksft_test_result(ok,"forked GEM mapping shares bytes and retains backing\n"); + if(alias!=MAP_FAILED){ + q=mmap(NULL,2*ps,PROT_READ|PROT_WRITE,MAP_SHARED,prime.fd,len-ps); + ok=q==MAP_FAILED && errno==EINVAL;if(q!=MAP_FAILED)munmap(q,2*ps); + q=mmap(NULL,ps,PROT_READ|PROT_WRITE,MAP_SHARED,fd,map.offset+ps); + ok &= q==MAP_FAILED && errno==EINVAL;if(q!=MAP_FAILED)munmap(q,ps);q=MAP_FAILED; + }else ok=false; + ksft_test_result(ok,"PRIME bounds and exact GEM offset rejection\n"); + if (alias != MAP_FAILED) { + size_t targets[] = { native, 4096, 16384 }; + + ok = true; + for (unsigned int t = 0; t < 3; t++) { + size_t target = targets[t]; + pid_t child; + int status = 0; + bool pair; + + if (target > native || target == ps || (t && target == native)) + continue; + child = fork(); + if (!child) { + char number[24], length[24], pages[24]; + + snprintf(number, sizeof(number), "%d", prime.fd); + snprintf(length, sizeof(length), "%zu", len); + snprintf(pages, sizeof(pages), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + target == native ? 0UL : target, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--shared", number, length, pages, NULL); + _exit(4); + } + pair = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + for (size_t i = 0; i < len; i += 4096) { + pair &= p[i] == (unsigned char)(0x80 + i / 4096); + p[i] = value(i); + } + ksft_print_msg("PRIME sharing %zuK -> %zuK status=%d bytes=%zu\n", + ps / 1024, target / 1024, status, len); + ok &= pair; + } + } else { + ok = false; + } + ksft_test_result(ok, "PRIME backing shared between native and alternative processes\n"); + struct drm_gem_close close_arg={.handle=create.handle}; + ok=!ioctl(fd,DRM_IOCTL_GEM_CLOSE,&close_arg);close(fd); + if(alias!=MAP_FAILED){close(prime.fd);ok &= check(alias,len,0);munmap(alias,len);}else ok=false; + if(p!=MAP_FAILED){ok &= check(p,len,0);munmap(p,len);}else ok=false; + ksft_test_result(ok,"mappings retain object after handles and descriptors close\n"); + ksft_print_cnts();return ksft_get_fail_cnt()?1:0; +} +int main(int argc,char **argv) +{ + bool ok=true;setvbuf(stdout,NULL,_IONBF,0); + if (argc == 5 && !strcmp(argv[1], "--shared")) { + struct stat init,self;int fd=atoi(argv[2]);size_t len=strtoul(argv[3],NULL,10); + if (stat("/proc/1/exe", &init) || stat("/proc/self/exe", &self) || + init.st_ino != self.st_ino || init.st_dev != self.st_dev || + len < 65536 || len > 8 * 65536 || + getauxval(AT_PAGESZ) != strtoul(argv[4], NULL, 10)) + return 2; + unsigned char *p=mmap(NULL,len,PROT_READ|PROT_WRITE,MAP_SHARED,fd,0); + bool pass=p!=MAP_FAILED && check(p,len,0); + if(pass)for(size_t i=0;i native || (t && target == native)) + continue; + pid = fork(); + if (!pid) { + char number[24]; + + snprintf(number, sizeof(number), "%zu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + target == native ? 0UL : target, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", number, NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("# GEM child ABI=%zuK native=%zuK status=%d\n", + target / 1024, native / 1024, status); + } + printf("%s - GEM fixture complete\n",ok?"ok":"not ok");reboot(RB_POWER_OFF);return ok?0:1; +} diff --git a/tools/testing/selftests/arm64/user4k/granule-proc-init.c b/tools/testing/selftests/arm64/user4k/granule-proc-init.c new file mode 100644 index 0000000000000..4a4608eb7329b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-proc-init.c @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok,const char*s){printf("%s - %s\n",ok?"ok":"not ok",s);failed+=!ok;} +static long pss(void *p) +{ + FILE*f=fopen("/proc/self/smaps","r");char*line=NULL;size_t cap=0;unsigned long start,end;int active=0;long value=-1; + if(!f)return -1; + while(getline(&line,&cap,f)>0){if(sscanf(line,"%lx-%lx",&start,&end)==2)active=(unsigned long)p>=start&&(unsigned long)p=0&&!ftruncate(fd,native),"native file backing created"); + unsigned char*p=mmap(NULL,native,PROT_READ|PROT_WRITE,MAP_SHARED,fd,0);if(p==MAP_FAILED){result(0,"parent mapping");goto done;} + memset(p,0x7b,native);long before=pss(p);printf("# native=%zu initial PSS=%ldKB\n",native,before);result(before==(long)native/1024,"native PSS accounts every physical slot"); + int ready[2],release[2];if(pipe(ready)||pipe(release)){result(0,"pipes");goto done;} + pid_t child=fork();int status=0; + if(!child){char f[24],r[24],w[24],n[24];snprintf(f,sizeof(f),"%d",fd);snprintf(r,sizeof(r),"%d",ready[1]);snprintf(w,sizeof(w),"%d",release[0]);snprintf(n,sizeof(n),"%zu",native);close(ready[0]);close(release[1]);if(prctl(SET_EXEC_PAGE_SIZE,4096UL,0UL,0UL,0UL))_exit(4);execl("/init","/init","--peer",f,r,w,n,NULL);_exit(5);} + close(ready[1]);close(release[0]);char token=0;int ok=child>0&&read(ready[0],&token,1)==1&&token=='a';result(ok&&p[native-4096]==0xc9,"4K peer shares the final physical slot and observes 2KB PSS"); + long during=pss(p);printf("# parent mixed PSS=%ldKB expected=%zuKB\n",during,native/1024-2);result(during==(long)native/1024-2,"native PSS weights the high shared slot independently"); + token='b';if(write(release[1],&token,1)!=1)failed++;ok=waitpid(child,&status,0)==child&&WIFEXITED(status)&&!WEXITSTATUS(status);result(ok&&pss(p)==(long)native/1024,"PSS returns to native size after peer releases its mapping"); + FILE*f=fopen("/proc/self/smaps_rollup","r");char buf[512];ok=f!=NULL;if(f){while(fgets(buf,sizeof(buf),f)) {}ok&=!ferror(f);fclose(f);}result(ok,"smaps_rollup reads complete native slot accounting"); + munmap(p,native); + done:printf("GRANULE PROC %s\n",failed?"FAIL":"PASS");sync();reboot(RB_POWER_OFF);for(;;)pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/granule-swap-init.c b/tools/testing/selftests/arm64/user4k/granule-swap-init.c new file mode 100644 index 0000000000000..184777bedf3e9 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-swap-init.c @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "swapfile-fixture.h" +#define SET_EXEC_PAGE_SIZE 0x41555001 +static unsigned char pattern(size_t i,unsigned round){return (i^(i>>12)^(i>>20)^round)&255;} +static int check(unsigned char*p,size_t size,unsigned round){for(size_t i=0;i>62)&1;close(fd);return n;} +static int exercise(unsigned long expected) +{ + size_t ps=getauxval(AT_PAGESZ),size=8*1024*1024;unsigned char*p=mmap(NULL,size,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0);int failed=0; + if(ps!=expected||p==MAP_FAILED){printf("not ok - expected ABI=%lu actual=%zu\n",expected,ps);return 1;} + for(unsigned round=0;round<4;round++){ + for(size_t i=0;i0&&n>0; + printf("%s - %zuK round%u real swapped PTEs=%u/%zu writes=%lu attempts=%u\n",ok?"ok":"not ok",ps/1024,round,n,size/ps,written,attempts);failed+=!ok; + int release[2],ready[2];if(pipe(release)||pipe(ready))return 2;pid_t child=fork();int status=0; + if(!child){char token;close(release[1]);close(ready[0]);int ok=read(release[0],&token,1)==1&&check(p,size,round);if(ok){for(size_t i=0;i0){fputs(line,stdout);if(strstr(line,"not ok")||strstr(line,"# SKIP"))ok=0;if(strstr(line,nonlinear?"ok 1 arm64-user4k-nonlinear-swap":"ok 1 arm64-user4k-swap"))completed=1;}free(line);fclose(f);return !(ok&&completed); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && (!strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--swapon-helper"))) { + struct stat a, b; + if (getpid() == 1 || stat("/proc/1/exe", &a) || + stat("/proc/self/exe", &b) || a.st_dev != b.st_dev || + a.st_ino != b.st_ino) + return 1; + return (!strcmp(argv[1], "--swapoff-helper") ? + swapoff(swap_path()) : + swapon(swap_path(), 0)) ? + 1 : + 0; + } + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 1; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || + (argc != 1 && (argc != 2 || (strcmp(argv[1], "--el0-only") && + strcmp(argv[1], "--nonlinear") && + strcmp(argv[1], "--swapfile"))))) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + int failed = 0, active = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) { + failed++; + goto done; + } + struct stat st; + for (int i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || + (argc == 2 && !strcmp(argv[1], "--swapfile") && + prepare_swapfile()) || + swapon(swap_path(), 0)) { + perror("swapon"); + failed++; + goto done; + } + active = 1; + printf("GRANULE SWAP BACKING=%s\n", swap_path()); + if (argc == 1 || (argc == 2 && !strcmp(argv[1], "--nonlinear"))) + failed += run_suite(argc == 2); + unsigned long sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned abi = 0; abi < (sizes[0] > 16384 ? 3U : 2U); abi++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[32]; + snprintf(expected, sizeof(expected), "%lu", sizes[abi]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[abi], 0UL, 0UL, + 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = c > 0 && waitpid(c, &status, 0) == c && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - swap ABI %luK child status=%#x\n", + ok ? "ok" : "not ok", sizes[abi] / 1024, status); + failed += !ok; + } +done: + if (active && swapoff(swap_path())) { + perror("swapoff"); + failed++; + } + if (active && argc == 2 && !strcmp(argv[1], "--swapfile") && + umount("/swapdir")) { + perror("umount"); + failed++; + } + printf("GRANULE SWAP %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/granule-threeway-init.c b/tools/testing/selftests/arm64/user4k/granule-threeway-init.c new file mode 100644 index 0000000000000..ad38f7b10b12f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-threeway-init.c @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +struct snapshot { long pss; int first, last, status; }; +struct peer { pid_t pid; int command, reply; size_t size; }; +static int failed; +static void result(int ok, const char *s) { printf("%s - %s\n", ok ? "ok" : "not ok", s); failed += !ok; } +static long pss(void *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); char *line = NULL; size_t cap = 0; + unsigned long start, end; int active = 0; long value = -1; + if (!f) return -1; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) active = (unsigned long)p >= start && (unsigned long)p < end; + else if (active && sscanf(line, "Pss: %ld kB", &value) == 1) break; + } + free(line); fclose(f); return value; +} +static int transfer(int fd, void *data, size_t size, int writing) +{ + size_t done = 0; + while (done < size) { + ssize_t n = writing ? write(fd, (char *)data + done, size - done) : read(fd, (char *)data + done, size - done); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) return 0; + done += n; + } + return 1; +} +static int worker(int fd, int command, int reply, size_t size) +{ + const struct rlimit no_core = {0, 0}; setrlimit(RLIMIT_CORE, &no_core); + if (getppid() != 1 || getauxval(AT_PAGESZ) != size) return 2; + unsigned char *p = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 65536 - size); + if (p == MAP_FAILED) return 3; + for (;;) { + char op; struct snapshot s = {.pss = -1, .first = -1, .last = -1}; + if (!transfer(command, &op, 1, 0)) return 4; + if (op == 'x') { s.status = munmap(p, size); transfer(reply, &s, sizeof(s), 1); return s.status ? 5 : 0; } + if (op == 'r') s.status = mprotect(p, size, PROT_READ); + else if (op == 'w') p[size - 1] = 0x99; + else if (op == 'd') s.status = madvise(p, size, MADV_DONTNEED); + else if (op == 't') { + pid_t c = fork(); int status = 0; + if (!c) { unsigned char value = *(volatile unsigned char *)(p + size - 1); _exit(value); } + if (c < 0 || waitpid(c, &status, 0) != c) s.status = -1; + else s.status = WIFSIGNALED(status) ? 256 + WTERMSIG(status) : WIFEXITED(status) ? WEXITSTATUS(status) : -2; + } + if (op != 'd' && op != 't') { s.first = p[0]; s.last = p[size - 1]; } + s.pss = pss(p); + if (!transfer(reply, &s, sizeof(s), 1)) return 6; + } +} +static struct peer spawn(int fd, size_t size) +{ + int command[2], reply[2]; struct peer p = {.pid = -1, .size = size}; + if (pipe(command) || pipe(reply)) return p; + p.pid = fork(); + if (!p.pid) { + char f[24], c[24], r[24], s[24]; close(command[1]); close(reply[0]); + snprintf(f, sizeof(f), "%d", fd); snprintf(c, sizeof(c), "%d", command[0]); + snprintf(r, sizeof(r), "%d", reply[1]); snprintf(s, sizeof(s), "%zu", size); + if (prctl(SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL)) _exit(7); + execl("/init", "/init", "--worker", f, c, r, s, NULL); _exit(8); + } + close(command[0]); close(reply[1]); p.command = command[1]; p.reply = reply[0]; return p; +} +static struct snapshot command(struct peer *p, char op) +{ + struct snapshot s = {.pss = -1, .first = -1, .last = -1, .status = -1}; + if (p->pid <= 0 || !transfer(p->command, &op, 1, 1) || !transfer(p->reply, &s, sizeof(s), 0)) failed++; + return s; +} +static void finish(struct peer *p) +{ + struct snapshot s = command(p, 'x'); int status = 0; + result(!s.status && waitpid(p->pid, &status, 0) == p->pid && WIFEXITED(status) && !WEXITSTATUS(status), "peer unmap and exit"); + close(p->command); close(p->reply); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); signal(SIGPIPE, SIG_IGN); + if (argc == 6 && !strcmp(argv[1], "--worker")) return worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), strtoul(argv[5], NULL, 10)); + if (getpid() != 1 || argc != 1) { fprintf(stderr, "Disposable64K VM PID1 only\n"); return 2; } + mount("proc", "/proc", "proc", 0, NULL); + if (getauxval(AT_PAGESZ) != 65536) { result(0, "native64K required"); goto done; } + int fd = memfd_create("three-granules", 0); + if (fd < 0 || ftruncate(fd, 65536)) { result(0, "file setup"); goto done; } + unsigned char *p = mmap(NULL, 65536, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { result(0, "native mapping"); goto done; } + memset(p, 0x42, 65536); + struct peer medium = spawn(fd, 16384); struct snapshot s = command(&medium, 's'); + result(s.pss == 8 && s.first == 0x42 && s.last == 0x42 && pss(p) == 56, "64K/16K initial sharing and PSS"); + struct peer small = spawn(fd, 4096); s = command(&small, 's'); + result(s.pss == 1 && s.first == 0x42 && s.last == 0x42, "4K peer shares the final slice with both larger ABIs"); + s = command(&medium, 's'); + result(s.pss == 7 && pss(p) == 55, "three-way PSS weights partial overlaps independently"); + s = command(&small, 'r'); result(!s.status, "4K alias becomes read-only"); + s = command(&medium, 'w'); result(!s.status && p[65535] == 0x99, "16K alias remains writable across local4K protection"); + s = command(&small, 's'); result(s.last == 0x99, "read-only4K alias observes16K writes"); + s = command(&small, 'd'); result(!s.status && s.pss == 0 && pss(p) == 56, "4K discard removes only its mapping"); + s = command(&medium, 's'); result(s.pss == 8 && s.last == 0x99, "16K mapping survives4K discard"); + s = command(&small, 's'); result(s.pss == 1 && s.last == 0x99 && pss(p) == 55, "4K refault restores coherent three-way sharing"); + result(!ftruncate(fd, 65536 - 4096), "truncate at4K boundary inside16K leaf"); + s = command(&small, 't'); result(s.status == 256 + SIGBUS, "4K mapping wholly beyond EOF faults SIGBUS"); + s = command(&medium, 't'); result(s.status == 0, "partial16K EOF leaf exposes zero tail"); + result(p[65535] == 0 && p[0] == 0x42, "native64K sees zero tail and preserved file prefix"); + s = command(&medium, 's'); result(s.first == 0x42 && s.last == 0 && s.pss == 8 && pss(p) == 56, "16K EOF refault keeps correct mixed PSS"); + result(!ftruncate(fd, 65536), "restore file length"); + s = command(&small, 's'); result(s.first == 0 && s.last == 0 && s.pss == 1 && pss(p) == 55, "4K alias refaults zeroed extension"); + finish(&small); s = command(&medium, 's'); result(s.pss == 8 && pss(p) == 56, "16K alias survives4K exit"); + finish(&medium); result(pss(p) == 64, "native PSS restored after all alternative aliases exit"); + munmap(p, 65536); close(fd); + done: printf("THREE GRANULES %s\n", failed ? "FAIL" : "PASS"); sync(); reboot(RB_POWER_OFF); for (;;) pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/iommufd-init.c b/tools/testing/selftests/arm64/user4k/iommufd-init.c new file mode 100644 index 0000000000000..c2bc5609b6e32 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/iommufd-init.c @@ -0,0 +1,1142 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable VM PID1: real IOMMUFD ioctls against mock AMDv1 page tables. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../drivers/iommu/iommufd/iommufd_test.h" + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define VA 0x200000000UL +#define IOVA 0x10000000UL +#define ALIAS 0x12000000UL +#define MAP_FLAGS (IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_READABLE | IOMMU_IOAS_MAP_WRITEABLE) +#define CHECK(c) do { if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, errno); return 1; \ +} } while (0) + +static size_t native_ps, user_ps; +struct context { int fd, access; uint32_t ioas, hwpt[2], dev[2]; }; + +static int drop_pin_cap(void) +{ + struct __user_cap_header_struct h = { .version = _LINUX_CAPABILITY_VERSION_3 }; + struct __user_cap_data_struct d[2]; + unsigned int mask = 1U << (CAP_IPC_LOCK % 32); + + if (syscall(SYS_capget, &h, d)) + return -1; + d[CAP_IPC_LOCK / 32].effective &= ~mask; + d[CAP_IPC_LOCK / 32].permitted &= ~mask; + return syscall(SYS_capset, &h, d); +} + +static int limit_bytes(size_t bytes) +{ + struct rlimit r; + + if (getrlimit(RLIMIT_MEMLOCK, &r)) + return -1; + r.rlim_cur = bytes; + return setrlimit(RLIMIT_MEMLOCK, &r); +} + +static long stat_kb(pid_t pid, const char *key) +{ + char path[80], line[256]; + long result = -1; + FILE *f; + + snprintf(path, sizeof(path), "/proc/%d/status", pid); + f = fopen(path, "r"); + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) + if (!strncmp(line, key, strlen(key))) { + result = strtol(line + strlen(key), NULL, 10); + break; + } + fclose(f); + return result; +} + +static int new_domain(struct context *c, unsigned int index) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), .op = IOMMU_TEST_OP_MOCK_DOMAIN, + .id = c->ioas }; + int rc = ioctl(c->fd, IOMMU_TEST_CMD, &cmd); + + if (!rc) { + c->hwpt[index] = cmd.mock_domain.out_hwpt_id; + c->dev[index] = cmd.mock_domain.out_idev_id; + } + return rc; +} + +static int context_open_common(struct context *c, unsigned int mm_account, + bool domain, bool pins) +{ + struct iommu_ioas_alloc a = { .size = sizeof(a) }; + struct iommu_option o = { .size = sizeof(o), .option_id = IOMMU_OPTION_RLIMIT_MODE, + .op = IOMMU_OPTION_OP_SET, .val64 = mm_account }; + struct iommu_test_cmd access = { .size = sizeof(access), .op = IOMMU_TEST_OP_CREATE_ACCESS }; + + memset(c, 0, sizeof(*c)); + c->fd = open("/dev/iommu", O_RDWR); + if (c->fd < 0 || ioctl(c->fd, IOMMU_OPTION, &o) || + ioctl(c->fd, IOMMU_IOAS_ALLOC, &a)) + return -1; + c->ioas = a.out_ioas_id; + if (domain && new_domain(c, 0)) + return -1; + access.id = c->ioas; + access.create_access.flags = pins ? MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES : 0; + if (ioctl(c->fd, IOMMU_TEST_CMD, &access)) + return -1; + c->access = access.create_access.out_access_fd; + return 0; +} + +static int context_open(struct context *c, unsigned int mm_account) +{ + return context_open_common(c, mm_account, true, false); +} + +static void context_close(struct context *c) +{ + close(c->access); + close(c->fd); +} + +static int map_range(struct context *c, void *p, size_t len, unsigned long iova) +{ + struct iommu_ioas_map m = { .size = sizeof(m), .flags = MAP_FLAGS, + .ioas_id = c->ioas, .user_va = (uintptr_t)p, .length = len, .iova = iova }; + + return ioctl(c->fd, IOMMU_IOAS_MAP, &m); +} + +static int unmap_range(struct context *c, size_t len, unsigned long iova) +{ + struct iommu_ioas_unmap u = { .size = sizeof(u), .ioas_id = c->ioas, + .iova = iova, .length = len }; + int rc = ioctl(c->fd, IOMMU_IOAS_UNMAP, &u); + + return rc ?: (u.length == len ? 0 : -1); +} + +static int check_pa(struct context *c, unsigned int dom, void *p, size_t len, + unsigned long iova) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), .op = IOMMU_TEST_OP_MD_CHECK_MAP, + .id = c->hwpt[dom], .check_map = { .iova = iova, .length = len, .uptr = (uintptr_t)p } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static int access_rw(struct context *c, void *p, size_t len, unsigned long iova, unsigned int flags) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), .op = IOMMU_TEST_OP_ACCESS_RW, + .id = c->access, .access_rw = { .iova = iova, .length = len, + .uptr = (uintptr_t)p, .flags = flags } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static bool filled(const void *ptr, unsigned char byte, size_t n) +{ + const unsigned char *p = ptr; + + for (size_t i = 0; i < n; i++) + if (p[i] != byte) + return false; + return true; +} + +static void *relocated(size_t len, unsigned char byte) +{ + void *old = mmap((void *)VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + + if (old == MAP_FAILED) + return old; + memset(old, byte, len); + size_t distance = len > 32 * native_ps ? len + native_ps : 32 * native_ps; + + return mremap(old, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, + (void *)(VA + distance + user_ps)); +} + +static int limits_and_lifetime(unsigned int mode) +{ + struct context c; + size_t len = 2 * native_ps; + unsigned char *p = relocated(3 * native_ps, 0x31), *buf = malloc(len); + struct iommu_ioas_copy copy; + + CHECK(p != MAP_FAILED && buf); + CHECK(!context_open(&c, mode)); + CHECK(!limit_bytes(len)); + CHECK(map_range(&c, p, 3 * native_ps, IOVA) == -1 && errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!map_range(&c, p, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(stat_kb(getpid(), "VmLck:") == (mode ? (long)len / 1024 : 0)); + CHECK(!check_pa(&c, 0, p, len, IOVA)); + CHECK(map_range(&c, p + len, native_ps, ALIAS) == -1 && errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + copy = (struct iommu_ioas_copy) { .size = sizeof(copy), .flags = MAP_FLAGS, + .src_ioas_id = c.ioas, .dst_ioas_id = c.ioas, .src_iova = IOVA + user_ps, + .dst_iova = ALIAS, .length = len - user_ps }; + CHECK(!ioctl(c.fd, IOMMU_IOAS_COPY, ©)); + CHECK(!new_domain(&c, 1)); + CHECK(!check_pa(&c, 1, p + user_ps, len - user_ps, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(mmap(p, len, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) == p); + memset(p, 0x5a, len); + CHECK(check_pa(&c, 0, p, len, IOVA) == -1 && errno == EINVAL); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x31, len)); + CHECK(!access_rw(&c, buf, len, IOVA, 0)); + CHECK(filled(buf, 0x5a, len)); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)(user_ps < native_ps ? len : len - user_ps) / 1024); + CHECK(!unmap_range(&c, len - user_ps, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + /* Fresh anonymous backing follows the shifted VA's native linear index. + * A non-native-aligned replacement spans three physical backing pages. + */ + size_t replacement_pages = user_ps < native_ps ? 3 : 2; + + if (replacement_pages == 3) { + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + } + CHECK(!limit_bytes(replacement_pages * native_ps)); + CHECK(!map_range(&c, p, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)(replacement_pages * native_ps / 1024)); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!munmap(p, 3 * native_ps)); + context_close(&c); + free(buf); + return 0; +} + +static int holes_and_shared(unsigned int mode) +{ + struct context c; + size_t len = 3 * user_ps, file_len = 4 * native_ps; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0), *buf = malloc(file_len); + int file = memfd_create("iommu-shared", 0); + + CHECK(p != MAP_FAILED && buf && file >= 0); + CHECK(!context_open(&c, mode) && !limit_bytes(8 * native_ps)); + memset(p, 0x31, len); + CHECK(!munmap(p + user_ps, user_ps)); + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == EFAULT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(mmap(p + user_ps, user_ps, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0) == p + user_ps); + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == EFAULT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!mprotect(p + user_ps, user_ps, PROT_READ | PROT_WRITE)); + CHECK(!map_range(&c, p, len, IOVA) && !check_pa(&c, 0, p, len, IOVA)); + CHECK(!unmap_range(&c, len, IOVA) && !munmap(p, len)); + CHECK(!ftruncate(file, file_len)); + p = mmap(NULL, 2 * user_ps, PROT_READ | PROT_WRITE, MAP_SHARED, file, user_ps); + CHECK(p != MAP_FAILED); + CHECK(!map_range(&c, p, 2 * user_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, 2 * user_ps, IOVA)); + CHECK(!munmap(p, 2 * user_ps)); + memset(buf, 0x93, 2 * user_ps); + CHECK(!access_rw(&c, buf, 2 * user_ps, IOVA, MOCK_ACCESS_RW_WRITE | MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(!unmap_range(&c, 2 * user_ps, IOVA)); + CHECK(pread(file, buf, file_len, 0) == (ssize_t)file_len); + CHECK(filled(buf, 0, user_ps)); + CHECK(filled(buf + user_ps, 0x93, 2 * user_ps)); + CHECK(filled(buf + 3 * user_ps, 0, file_len - 3 * user_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + close(file); + context_close(&c); + free(buf); + return 0; +} + +static int producer(int argc, char **argv) +{ + struct context c = { .fd = atoi(argv[4]), .ioas = strtoul(argv[5], NULL, 0) }; + int ready = atoi(argv[6]), go = atoi(argv[7]); + char byte = 'R'; + void *p; + + (void)argc; + CHECK(!prctl(PR_SET_PDEATHSIG, SIGKILL)); + CHECK(!drop_pin_cap()); + p = relocated(2 * native_ps, 0x6d); + CHECK(p != MAP_FAILED); + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(write(ready, &byte, 1) == 1); + CHECK(read(go, &byte, 1) == 1 && byte == 'G'); + return 0; +} + +static int destroy_id(struct context *c, uint32_t id) +{ + struct iommu_destroy cmd = { .size = sizeof(cmd), .id = id }; + + return ioctl(c->fd, IOMMU_DESTROY, &cmd); +} + +static int create_viommu(struct context *c, uint32_t *id, uint32_t *parent) +{ + struct iommu_hwpt_alloc hwpt = { .size = sizeof(hwpt), + .flags = IOMMU_HWPT_ALLOC_NEST_PARENT, .dev_id = c->dev[0], .pt_id = c->ioas }; + struct iommu_viommu_alloc cmd = { .size = sizeof(cmd), + .type = IOMMU_VIOMMU_TYPE_SELFTEST, .dev_id = c->dev[0] }; + + if (ioctl(c->fd, IOMMU_HWPT_ALLOC, &hwpt)) + return -1; + *parent = hwpt.out_hwpt_id; + cmd.hwpt_id = *parent; + if (ioctl(c->fd, IOMMU_VIOMMU_ALLOC, &cmd)) + return -1; + *id = cmd.out_viommu_id; + return 0; +} + +static int queue_alloc(struct context *c, uint32_t viommu, unsigned int index, + unsigned long iova, size_t length, uint32_t *id) +{ + struct iommu_hw_queue_alloc cmd = { .size = sizeof(cmd), .viommu_id = viommu, + .type = IOMMU_HW_QUEUE_TYPE_SELFTEST, .index = index, + .nesting_parent_iova = iova, .length = length }; + int rc = ioctl(c->fd, IOMMU_HW_QUEUE_ALLOC, &cmd); + + if (!rc) + *id = cmd.out_hw_queue_id; + return rc; +} + +static int queue_after_exit(struct context *c) +{ + uint32_t viommu, parent, id; + + CHECK(!create_viommu(c, &viommu, &parent)); + CHECK(!queue_alloc(c, viommu, 0, IOVA + native_ps / 4 + 3, native_ps / 4, &id)); + CHECK(unmap_range(c, 2 * native_ps, IOVA) == -1 && errno == EBUSY); + CHECK(!destroy_id(c, id)); + CHECK(!destroy_id(c, viommu)); + CHECK(!destroy_id(c, parent)); + return 0; +} + +static int queue_ranges(unsigned int mode) +{ + struct context c; + unsigned char *p = relocated(2 * native_ps, 0x71); + uint32_t viommu, parent, q0, q1, unused; + size_t offset = user_ps / 4 + 3, length = user_ps / 2 + 17; + int file; + + CHECK(p != MAP_FAILED && !context_open(&c, mode)); + CHECK(!limit_bytes(8 * native_ps) && !create_viommu(&c, &viommu, &parent)); + /* Map only one user leaf: queue pinning must not require surrounding IOVA. */ + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 1, IOVA + offset, length, &unused) == -1 && errno == EIO); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + offset, length, &q0)); + CHECK(queue_alloc(&c, viommu, 0, IOVA + offset, length, &unused) == -1 && errno == EEXIST); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(stat_kb(getpid(), "VmPin:") == (long)native_ps / 1024); + /* Pins keep the old physical bytes after replacing the source VMA. */ + CHECK(mmap(p, user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == p); + memset(p, 0x5a, user_ps); + CHECK(!queue_alloc(&c, viommu, 1, IOVA + offset + 7, length - 7, &q1)); + CHECK(destroy_id(&c, q0) == -1 && errno == EBUSY); + CHECK(!destroy_id(&c, q1)); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(!destroy_id(&c, q0)); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(queue_alloc(&c, viommu, 0, IOVA + offset, length, &unused) == -1 && errno == ENOENT); + CHECK(!munmap(p, 2 * native_ps)); + + /* Separate mappings of the same file byte range cannot form a physical run. + * The second area must fail after the first area's access was registered. + */ + file = memfd_create("queue-alias", 0); + CHECK(file >= 0 && !ftruncate(file, native_ps)); + p = mmap((void *)VA, 2 * user_ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(p != MAP_FAILED); + CHECK(mmap(p, user_ps, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, file, 0) == p); + CHECK(mmap(p + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, 0) == p + user_ps); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && errno == ENOENT); + CHECK(!map_range(&c, p + user_ps, user_ps, IOVA + user_ps)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && errno == EFAULT); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(!unmap_range(&c, user_ps, IOVA + user_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + /* The same discontinuity inside one provider must also release its access. */ + CHECK(!map_range(&c, p, 2 * user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && errno == EFAULT); + CHECK(!unmap_range(&c, 2 * user_ps, IOVA)); + if (user_ps < native_ps) { + /* Adjacent file quarters form a real physical run across two areas. */ + CHECK(mmap(p + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, user_ps) == p + user_ps); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!map_range(&c, p + user_ps, user_ps, IOVA + user_ps)); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + 11, 2 * user_ps - 22, &q0)); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(unmap_range(&c, user_ps, IOVA + user_ps) == -1 && errno == EBUSY); + CHECK(!destroy_id(&c, q0)); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(!unmap_range(&c, user_ps, IOVA + user_ps)); + } + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, 2 * user_ps)); + close(file); + CHECK(!destroy_id(&c, viommu) && !destroy_id(&c, parent)); + context_close(&c); + printf("ok - hardware queue byte spans %zuK/%zuK %s, " + "relocated/overlap/rollback/unmap locks\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int cross_process(unsigned int mode) +{ + struct context c; + int ready[2], go[2], status; + pid_t child; + size_t child_ps = user_ps == native_ps ? 4096 : native_ps; + size_t len = 2 * native_ps; + unsigned char *p = mmap((void *)(VA + 128 * native_ps), len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0), *buf = malloc(len); + char byte; + + CHECK(p != MAP_FAILED && buf && !pipe(ready) && !pipe(go)); + CHECK(!context_open(&c, mode) && !limit_bytes(len)); + child = fork(); + CHECK(child >= 0); + if (!child) { + char ps[24], native[24], fd[24], ioas[24], r[24], g[24]; + + close(ready[0]); close(go[1]); + snprintf(ps, sizeof(ps), "%zu", child_ps); + snprintf(native, sizeof(native), "%zu", native_ps); + snprintf(fd, sizeof(fd), "%d", c.fd); + snprintf(ioas, sizeof(ioas), "%u", c.ioas); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(g, sizeof(g), "%d", go[0]); + if (prctl(SET_EXEC_PAGE_SIZE, child_ps, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--producer", ps, native, fd, ioas, r, g, NULL); + _exit(5); + } + close(ready[1]); close(go[0]); + CHECK(read(ready[0], &byte, 1) == 1 && byte == 'R'); + CHECK(stat_kb(child, "VmPin:") == (long)len / 1024); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + if (!mode) + CHECK(map_range(&c, p, native_ps, ALIAS) == -1 && errno == ENOMEM); + else { + CHECK(!map_range(&c, p, native_ps, ALIAS)); + CHECK(!unmap_range(&c, native_ps, ALIAS)); + } + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + byte = 'G'; + CHECK(write(go[1], &byte, 1) == 1); + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(!new_domain(&c, 1)); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + CHECK(!queue_after_exit(&c)); + if (!mode) + CHECK(map_range(&c, p, native_ps, ALIAS) == -1 && errno == ENOMEM); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!map_range(&c, p, len, ALIAS) && !unmap_range(&c, len, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + close(ready[0]); close(go[1]); + CHECK(!munmap(p, len)); + context_close(&c); + free(buf); + return 0; +} + +struct stress_arg { + struct context *c; + void *p; + unsigned int index; + atomic_int *failed; +}; +static void *stress_thread(void *opaque) +{ + struct stress_arg *a = opaque; + unsigned long iova = IOVA + a->index * 0x100000UL; + size_t len = 2 * native_ps; + + for (unsigned int round = 0; round < 64; round++) { + if (map_range(a->c, a->p, len, iova) || + (!(round % 16) && check_pa(a->c, 1, a->p, len, iova)) || + unmap_range(a->c, len, iova)) { + printf("FAIL concurrent map thread=%u round=%u errno=%d\n", a->index, round, errno); + atomic_store(a->failed, 1); + break; + } + } + return NULL; +} + +static int concurrent_maps(unsigned int mode) +{ + struct context c; + pthread_t threads[4]; + struct stress_arg args[4]; + atomic_int failed = 0; + void *p = relocated(2 * native_ps, 0x47); + + CHECK(p != MAP_FAILED && !context_open(&c, mode)); + CHECK(!limit_bytes(32 * native_ps) && !new_domain(&c, 1)); + for (unsigned int i = 0; i < 4; i++) { + args[i] = (struct stress_arg) { &c, p, i, &failed }; + CHECK(!pthread_create(&threads[i], NULL, stress_thread, &args[i])); + } + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_join(threads[i], NULL)); + CHECK(!atomic_load(&failed)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, 2 * native_ps)); + context_close(&c); + return 0; +} + +static int write_attribute(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + ssize_t n; + + if (fd < 0) + return -1; + n = write(fd, value, strlen(value)); + close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} + +static int allocation_failures(unsigned int mode) +{ + struct context c; + void *p = relocated(2 * native_ps, 0x62); + int nth = open("/proc/self/fail-nth", O_RDWR); + unsigned int hit = 0, n; + + CHECK(nth >= 0 && p != MAP_FAILED); + CHECK(!context_open(&c, mode) && !limit_bytes(8 * native_ps)); + CHECK(!new_domain(&c, 1)); + CHECK(!write_attribute("/sys/kernel/debug/failslab/ignore-gfp-wait", "N")); + CHECK(!write_attribute("/sys/kernel/debug/fail_page_alloc/ignore-gfp-wait", "N")); + CHECK(!write_attribute("/sys/kernel/debug/fail_page_alloc/ignore-gfp-highmem", "N")); + CHECK(!write_attribute("/sys/kernel/debug/failslab/verbose", "0")); + CHECK(!write_attribute("/sys/kernel/debug/fail_page_alloc/verbose", "0")); + CHECK(!write_attribute("/sys/kernel/debug/fail_iommufd/verbose", "0")); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n); + ssize_t got, disabled; + int rc; + + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = map_range(&c, p, 2 * native_ps, IOVA); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(disabled == 1 && got > 0); + if (!rc) + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + /* An injected failure must not poison a successful retry or translation. */ + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, 2 * native_ps, IOVA)); + CHECK(!check_pa(&c, 1, p, 2 * native_ps, IOVA)); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + printf("ok - IOMMUFD %zuK %s accounting allocation-failure sweep: %u sites\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + CHECK(!munmap(p, 2 * native_ps)); + close(nth); + context_close(&c); + return 0; +} + +static int queue_failure_sweep(unsigned int mode) +{ + struct context c; + void *p = relocated(2 * native_ps, 0x4c); + int nth = open("/proc/self/fail-nth", O_RDWR); + uint32_t viommu, parent, id; + unsigned int n, hit = 0; + + CHECK(p != MAP_FAILED && nth >= 0 && !context_open(&c, mode)); + CHECK(!limit_bytes(8 * native_ps) && !create_viommu(&c, &viommu, &parent)); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n), rc; + ssize_t got, disabled; + + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = queue_alloc(&c, viommu, 0, IOVA + user_ps / 4 + 3, user_ps / 2, &id); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(got > 0 && disabled == 1); + if (!rc) + CHECK(!destroy_id(&c, id)); + /* A failed queue must not leave an internal access blocking unmap. */ + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + user_ps / 4 + 3, user_ps / 2, &id)); + CHECK(!destroy_id(&c, id) && !unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + CHECK(!destroy_id(&c, viommu) && !destroy_id(&c, parent)); + context_close(&c); + close(nth); + CHECK(!munmap(p, 2 * native_ps)); + printf("ok - hardware queue %zuK %s failure sweep: %u sites, no stale locks/pins\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + return 0; +} + +static int page_access(struct context *c, void *p, size_t len, + unsigned long iova, uint32_t *id) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), .op = IOMMU_TEST_OP_ACCESS_PAGES, + .id = c->access, .access_pages = { .iova = iova, .length = len, + .uptr = (uintptr_t)p, .flags = MOCK_FLAGS_ACCESS_WRITE } }; + int rc = ioctl(c->fd, IOMMU_TEST_CMD, &cmd); + + if (!rc) + *id = cmd.access_pages.out_access_pages_id; + return rc; +} + +static int page_release(struct context *c, uint32_t id) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_DESTROY_ACCESS_PAGES, .id = c->access, + .destroy_access_pages = { .access_pages_id = id } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static int page_ranges(unsigned int mode) +{ + struct context c; + int file = memfd_create("native-page-array", 0); + unsigned char *p; + uint32_t a, b, d, e; + size_t len = 3 * native_ps, tail = native_ps + user_ps / 2 + 3; + + CHECK(file >= 0 && !ftruncate(file, 5 * native_ps)); + p = mmap((void *)VA, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED && !limit_bytes(4 * native_ps)); + memset(p, 0x67, len); + CHECK(!context_open_common(&c, mode, false, true)); + /* Separate areas exercise native output stride with alternative indices. */ + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, 2 * native_ps, IOVA + native_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, len, IOVA, &a)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(!page_access(&c, p, len, IOVA, &b)); + CHECK(!page_access(&c, p + native_ps, tail, IOVA + native_ps, &d)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + /* Source VA can move without changing its retained physical offset. */ + unsigned char *moved = mremap(p, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, + (void *)(VA + 32 * native_ps + user_ps)); + CHECK(moved != MAP_FAILED); + CHECK(!page_access(&c, moved, len, IOVA, &e) && !page_release(&c, e)); + CHECK(!page_release(&c, a) && !page_release(&c, b)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)(2 * native_ps) / 1024); + CHECK(!page_release(&c, d)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(mremap(moved, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, p) == p); + CHECK(!page_access(&c, p, len, IOVA, &a)); + /* Unmap callback releases an access covering both independent areas. */ + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(page_release(&c, a) == -1 && errno == ENOENT); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, len)); + + /* The second area is absent: first area's pin/accounting must roll back. */ + p = mmap((void *)VA, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED && !map_range(&c, p, native_ps, IOVA)); + CHECK(page_access(&c, NULL, 2 * native_ps, IOVA, &a) == -1 && errno == ENOENT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, native_ps, IOVA, &a) && !page_release(&c, a)); + CHECK(!unmap_range(&c, native_ps, IOVA) && !munmap(p, native_ps)); + + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED); + CHECK(mmap(p + native_ps, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, native_ps) == p + native_ps); + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(!page_access(&c, p, 2 * native_ps, IOVA, &a)); + CHECK(stat_kb(getpid(), "VmPin:") == + (long)((user_ps == native_ps ? 2 : 1) * native_ps) / 1024); + CHECK(!page_release(&c, a) && !unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(!munmap(p, 2 * native_ps)); + + if (user_ps < native_ps) { + /* Native-sized logical range whose physical base has a fragment offset. */ + p = mmap((void *)VA, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, user_ps); + CHECK(p != MAP_FAILED && !map_range(&c, p, native_ps, IOVA)); + CHECK(page_access(&c, p, native_ps, IOVA, &a) == -1 && errno == EINVAL); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!unmap_range(&c, native_ps, IOVA) && !munmap(p, native_ps)); + + /* Aligned first fragment followed by an alias, not consecutive bytes. */ + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(p != MAP_FAILED); + CHECK(mmap(p + native_ps + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, native_ps) == p + native_ps + user_ps); + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, native_ps, IOVA + native_ps)); + CHECK(page_access(&c, p, 2 * native_ps, IOVA, &a) == -1 && errno == EINVAL); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, native_ps, IOVA, &a)); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(page_release(&c, a) == -1 && errno == ENOENT); + CHECK(!munmap(p, 2 * native_ps)); + } + context_close(&c); + close(file); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + printf("ok - native page arrays %zuK/%zuK %s, lazy/partial/moved/rollback/callback\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int page_lifetime(unsigned int mode) +{ + struct context c; + int ready[2], go[2], status; + pid_t child; + size_t child_ps = user_ps == native_ps ? 4096 : native_ps; + size_t len = 2 * native_ps; + unsigned char *p = mmap((void *)(VA + 128 * native_ps), len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0), *buf = malloc(len); + uint32_t a, b; + char byte; + + CHECK(p != MAP_FAILED && buf && !pipe(ready) && !pipe(go)); + memset(p, 0x42, len); + CHECK(!context_open_common(&c, mode, false, true) && !limit_bytes(len)); + child = fork(); + CHECK(child >= 0); + if (!child) { + char ps[24], native[24], fd[24], ioas[24], r[24], g[24]; + + close(ready[0]); close(go[1]); + snprintf(ps, sizeof(ps), "%zu", child_ps); + snprintf(native, sizeof(native), "%zu", native_ps); + snprintf(fd, sizeof(fd), "%d", c.fd); + snprintf(ioas, sizeof(ioas), "%u", c.ioas); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(g, sizeof(g), "%d", go[0]); + if (prctl(SET_EXEC_PAGE_SIZE, child_ps, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--producer", ps, native, fd, ioas, r, g, NULL); + _exit(5); + } + close(ready[1]); close(go[0]); + CHECK(read(ready[0], &byte, 1) == 1 && byte == 'R'); + CHECK(stat_kb(child, "VmPin:") == 0); + CHECK(!page_access(&c, NULL, len, IOVA, &a)); + CHECK(stat_kb(child, "VmPin:") == (long)len / 1024); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + byte = 'G'; + CHECK(write(go[1], &byte, 1) == 1); + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && !WEXITSTATUS(status)); + /* Reuse retained anonymous fragments after source mm_users reaches zero. */ + CHECK(!page_access(&c, NULL, native_ps, IOVA, &b)); + CHECK(!page_release(&c, a)); + CHECK(!access_rw(&c, buf, native_ps, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, native_ps)); + CHECK(!map_range(&c, p, len, ALIAS)); + if (!mode) + CHECK(page_access(&c, p, len, ALIAS, &a) == -1 && errno == ENOMEM); + else + CHECK(!page_access(&c, p, len, ALIAS, &a) && !page_release(&c, a)); + /* Final access release refunds even a dead provider's account. */ + CHECK(!page_release(&c, b) && !unmap_range(&c, len, IOVA)); + CHECK(!page_access(&c, p, len, ALIAS, &a) && !page_release(&c, a)); + CHECK(!unmap_range(&c, len, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, len)); + context_close(&c); + close(ready[0]); close(go[1]); + free(buf); + printf("ok - native page array anonymous source %zuK -> %zuK consumer, %s exit/refund\n", + child_ps / 1024, user_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int page_failure_sweep(unsigned int mode) +{ + struct context c; + int file = memfd_create("page-array-failures", 0); + int nth = open("/proc/self/fail-nth", O_RDWR); + unsigned char *p; + uint32_t id; + unsigned int n, hit = 0; + + CHECK(file >= 0 && nth >= 0 && !ftruncate(file, 2 * native_ps)); + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(p != MAP_FAILED); + memset(p, 0x53, 2 * native_ps); + CHECK(!context_open_common(&c, mode, false, true) && !limit_bytes(2 * native_ps)); + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, native_ps, IOVA + native_ps)); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n), rc; + ssize_t got, disabled; + + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = page_access(&c, p, 2 * native_ps, IOVA, &id); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(got > 0 && disabled == 1); + if (!rc) + CHECK(!page_release(&c, id)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + CHECK(!page_access(&c, p, 2 * native_ps, IOVA, &id) && !page_release(&c, id)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA) && !munmap(p, 2 * native_ps)); + context_close(&c); + close(file); close(nth); + printf("ok - native page arrays %zuK %s failure sweep: %u sites, refunds/retries\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + return 0; +} + +struct page_race { + struct context *c; + void *p; + pthread_barrier_t barrier; + atomic_int failed, pinned, absent; +}; + +static void *page_pin_thread(void *opaque) +{ + struct page_race *r = opaque; + + pthread_barrier_wait(&r->barrier); + for (unsigned int i = 0; i < 256; i++) { + uint32_t id; + + if (page_access(r->c, r->p, 2 * native_ps, IOVA, &id)) { + /* Unmap can mark the area inaccessible before notifying drivers. */ + if (errno != ENOENT && errno != EINVAL) + goto fail; + atomic_fetch_add(&r->absent, 1); + } else { + atomic_fetch_add(&r->pinned, 1); + sched_yield(); + if (page_release(r->c, id) && errno != ENOENT) + goto fail; + } + } + return NULL; +fail: + printf("FAIL page pin/unmap race errno=%d\n", errno); + atomic_store(&r->failed, 1); + return NULL; +} + +static void *page_unmap_thread(void *opaque) +{ + struct page_race *r = opaque; + + pthread_barrier_wait(&r->barrier); + for (unsigned int i = 0; i < 256; i++) { + if (unmap_range(r->c, 2 * native_ps, IOVA)) + goto fail; + sched_yield(); + if (map_range(r->c, r->p, 2 * native_ps, IOVA)) + goto fail; + } + return NULL; +fail: + printf("FAIL page unmap/remap race errno=%d\n", errno); + atomic_store(&r->failed, 1); + return NULL; +} + +static int page_concurrency(unsigned int mode) +{ + struct context c; + struct page_race r = { .c = &c }; + pthread_t threads[4]; + int file = memfd_create("page-array-race", 0); + + CHECK(file >= 0 && !ftruncate(file, 2 * native_ps)); + r.p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(r.p != MAP_FAILED); + memset(r.p, 0x49, 2 * native_ps); + CHECK(!context_open_common(&c, mode, false, true) && !limit_bytes(8 * native_ps)); + CHECK(!map_range(&c, r.p, 2 * native_ps, IOVA)); + CHECK(!pthread_barrier_init(&r.barrier, NULL, 4)); + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_create(&threads[i], NULL, + i == 3 ? page_unmap_thread : page_pin_thread, &r)); + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_join(threads[i], NULL)); + CHECK(!atomic_load(&r.failed) && atomic_load(&r.pinned) > 0); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(r.p, 2 * native_ps)); + context_close(&c); + close(file); + CHECK(!pthread_barrier_destroy(&r.barrier)); + printf("ok - native page arrays %zuK %s race: %d pins, %d unavailable, 256 remaps\n", + user_ps / 1024, mode ? "MM" : "USER", + atomic_load(&r.pinned), atomic_load(&r.absent)); + return 0; +} + +static int dirty_ioctl(struct context *c, uint32_t hwpt, void *bitmap, + unsigned long iova, size_t length, size_t granule, unsigned int flags) +{ + struct iommu_hwpt_get_dirty_bitmap cmd = { .size = sizeof(cmd), + .hwpt_id = hwpt, .flags = flags, .iova = iova, .length = length, + .page_size = granule, .data = (uintptr_t)bitmap }; + + return ioctl(c->fd, IOMMU_HWPT_GET_DIRTY_BITMAP, &cmd); +} + +static int dirty_errors(unsigned int mode) +{ + struct context c; + size_t len = 66 * native_ps; + unsigned char *data = relocated(len, 0x34); + unsigned char *out = mmap(NULL, 2 * user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t *bitmap = (void *)(out + user_ps - 8), seed[2] = { 1, 2 }; + struct iommu_option option = { .size = sizeof(option), .option_id = IOMMU_OPTION_HUGE_PAGES, + .op = IOMMU_OPTION_OP_SET, .val64 = 0 }; + struct iommu_hwpt_alloc alloc = { .size = sizeof(alloc), .flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING }; + struct iommu_hwpt_set_dirty_tracking enable = { .size = sizeof(enable), .flags = IOMMU_HWPT_DIRTY_TRACKING_ENABLE }; + struct iommu_test_cmd mark = { .size = sizeof(mark), .op = IOMMU_TEST_OP_DIRTY, + .dirty = { .iova = IOVA, .length = len, .page_size = native_ps, .uptr = (uintptr_t)seed } }; + unsigned int no_clear = IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR; + int failures = 0, rc, error; + + CHECK(data != MAP_FAILED && out != MAP_FAILED); + CHECK(!context_open(&c, mode) && !limit_bytes(128 * native_ps)); + option.object_id = c.ioas; + CHECK(!ioctl(c.fd, IOMMU_OPTION, &option)); + CHECK(!map_range(&c, data, len, IOVA)); + alloc.dev_id = c.dev[0]; alloc.pt_id = c.ioas; + CHECK(!ioctl(c.fd, IOMMU_HWPT_ALLOC, &alloc)); + enable.hwpt_id = alloc.out_hwpt_id; + CHECK(!ioctl(c.fd, IOMMU_HWPT_SET_DIRTY_TRACKING, &enable)); + mark.id = alloc.out_hwpt_id; + CHECK(!ioctl(c.fd, IOMMU_TEST_CMD, &mark) && mark.dirty.out_nr_dirty == 2); + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, no_clear)); + CHECK(bitmap[0] == 1 && bitmap[1] == 2); + CHECK(filled(out, 0, user_ps - 8) && filled(out + user_ps + 8, 0, user_ps - 8)); + +#define EXPECT_DIRTY_ERROR(call, expected, label) do { \ + errno = 0; rc = (call); error = errno; \ + if (rc != -1 || error != (expected)) { \ + printf("FAIL dirty %s: rc=%d errno=%d expected=%d\n", label, rc, error, expected); \ + failures++; \ + } \ +} while (0) + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, + IOVA + len, native_ps, native_ps, no_clear), EINVAL, "unmapped IOVA"); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, (void *)1, + IOVA, len, native_ps, no_clear), EFAULT, "invalid output pointer"); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, + IOVA, 4 * native_ps, 3 * native_ps, no_clear), EINVAL, "non-power-of-two granule"); + CHECK(!mprotect(out, 2 * user_ps, PROT_READ)); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, + IOVA, len, native_ps, no_clear), EFAULT, "read-only output"); + CHECK(!mprotect(out, 2 * user_ps, PROT_READ | PROT_WRITE)); + memset(out, 0, 2 * user_ps); + CHECK(!munmap(out + user_ps, user_ps)); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, + IOVA, len, native_ps, no_clear), EFAULT, "output hole after valid prefix"); + CHECK(bitmap[0] == 1); + CHECK(mmap(out + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0) == out + user_ps); + /* NO_CLEAR errors retain dirty state, so fixing the buffer permits retry. */ + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, 0)); + CHECK(bitmap[0] == 1 && bitmap[1] == 2); + memset(out, 0, 2 * user_ps); + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, no_clear)); + CHECK(bitmap[0] == 0 && bitmap[1] == 0); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!munmap(data, len) && !munmap(out, 2 * user_ps)); + context_close(&c); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && stat_kb(getpid(), "VmLck:") == 0); + CHECK(!failures); +#undef EXPECT_DIRTY_ERROR + return 0; +} + +static int exercise(void) +{ + CHECK(!drop_pin_cap()); + for (unsigned int mode = 0; mode < 2; mode++) { + CHECK(!limits_and_lifetime(mode)); + CHECK(!holes_and_shared(mode)); + CHECK(!queue_ranges(mode)); + CHECK(!page_ranges(mode)); + CHECK(!page_lifetime(mode)); + CHECK(!cross_process(mode)); + CHECK(!concurrent_maps(mode)); + CHECK(!allocation_failures(mode)); + CHECK(!queue_failure_sweep(mode)); + CHECK(!page_failure_sweep(mode)); + CHECK(!page_concurrency(mode)); + CHECK(!dirty_errors(mode)); + printf("ok - IOMMUFD %zuK userspace/%zuK kernel %s accounting, limits/lifetime/shared/cross-process/256 concurrent cycles\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + } + return 0; +} + +int main(int argc, char **argv) +{ + int fail = 0; + + setbuf(stdout, NULL); + if (argc >= 4) { + user_ps = strtoul(argv[2], NULL, 0); + native_ps = strtoul(argv[3], NULL, 0); + CHECK(getauxval(AT_PAGESZ) == user_ps); + if (!strcmp(argv[1], "--exercise")) + return exercise(); + if (!strcmp(argv[1], "--producer") && argc == 8) + return producer(argc, argv); + return 2; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)); + CHECK(!mount("sysfs", "/sys", "sysfs", 0, NULL)); + CHECK(!mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)); + native_ps = getauxval(AT_PAGESZ); + size_t sizes[] = { native_ps, 4096, 16384 }; + + for (unsigned int i = 0; i < (native_ps > 16384 ? 3U : 2U); i++) { + pid_t child = fork(); + int status = 0, ok = 0; + + if (!child) { + char size[24], native[24]; + + snprintf(size, sizeof(size), "%zu", sizes[i]); + snprintf(native, sizeof(native), "%zu", native_ps); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", size, native, NULL); + _exit(5); + } + for (unsigned int ticks = 0; child > 0 && ticks < 1800; ticks++) { + if (waitpid(child, &status, WNOHANG) == child) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 1799) { + kill(child, SIGKILL); + waitpid(child, &status, 0); + } + usleep(100000); + } + printf("%s - IOMMUFD ABI %zuK status=%#x\n", ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } + printf("IOMMUFD IOCTL CONTRACT %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c b/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c new file mode 100644 index 0000000000000..de32b48f6244a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static unsigned long stat_value(const char *path, const char *key) +{ + char line[256], name[100]; + unsigned long value = 0; + FILE *f = fopen(path, "re"); + + if (!f) + return 0; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "%99s %lu", name, &value) == 2 && !strcmp(name, key)) + break; + fclose(f); + return value; +} + +static bool all_bytes(const unsigned char *p, size_t len, unsigned char value) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value) + return false; + return true; +} + +static unsigned long lazy_kb(void *address) +{ + FILE *f = fopen("/proc/self/smaps", "re"); + char line[256]; + unsigned long first, last, value = ~0UL; + bool found = false; + + if (!f) + return value; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &first, &last) == 2) { + if (found) + break; + found = (uintptr_t)address >= first && (uintptr_t)address < last; + } else if (found && sscanf(line, "LazyFree: %lu", &value) == 1) { + break; + } + } + fclose(f); + return value; +} + +static uint64_t pagemap(void *p, size_t ps) +{ + uint64_t entry = 0; + int fd = open("/proc/self/pagemap", O_RDONLY); + + if (fd < 0 || pread(fd, &entry, sizeof(entry), (uintptr_t)p / ps * 8) != 8) + ksft_exit_fail_msg("pagemap: %s\n", strerror(errno)); + close(fd); + return entry; +} + +static bool pageout(void *p, size_t len) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(i, &one); + ok &= !sched_setaffinity(0, sizeof(one), &one); + ok &= !madvise(p, len, MADV_COLD); + ok &= !madvise(p, len, MADV_PAGEOUT); + } + return !sched_setaffinity(0, sizeof(allowed), &allowed) && ok; +} + +static bool mark_free(void *p, size_t len) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(i, &one); + ok &= !sched_setaffinity(0, sizeof(one), &one); + ok &= !madvise(p, len, MADV_FREE); + ok &= !madvise(p, len, MADV_COLD); + } + return !sched_setaffinity(0, sizeof(allowed), &allowed) && ok; +} + +static int exercise(unsigned long native) +{ + const size_t ps = getauxval(AT_PAGESZ), len = 4 * ps; + unsigned char *reservation = mmap(NULL, len + 2 * native, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *p = (void *)(((uintptr_t)reservation + native - 1) & ~(native - 1)); + bool ok; + unsigned long writes; + uint64_t entry; + + if (reservation == MAP_FAILED || mprotect(p, len, PROT_READ | PROT_WRITE)) + ksft_exit_fail_msg("mapping\n"); + ksft_print_header(); + ksft_set_plan(8); + for (unsigned int i = 0; i < 4; i++) + memset(p + i * ps, 0x40 + i, ps); + ok = mark_free(p + ps, ps); + if (!ok) + ksft_exit_fail_msg("%zuK MADV_FREE: %s\n", ps / 1024, strerror(errno)); + ksft_test_result(all_bytes(p + ps, ps, 0x41), + "%zuK FREE retains bytes before reclaim\n", ps / 1024); + ksft_test_result(lazy_kb(p) == ps / 1024, "%zuK partial LazyFree accounting\n", ps / 1024); + ok = pageout(p, len); + entry = pagemap(p + ps, ps); + ok &= !(entry & ((1ULL << 63) | (1ULL << 62))); + ok &= all_bytes(p + ps, ps, 0) && all_bytes(p, ps, 0x40) && + all_bytes(p + 2 * ps, ps, 0x42) && all_bytes(p + 3 * ps, ps, 0x43); + ksft_test_result(ok, "%zuK partial FREE reclaims only advised bytes\n", ps / 1024); + /* Drop old swap/cache state, then test cancellation by a real EL0 write. */ + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0x62, len); + ok &= mark_free(p + ps, ps); + memset(p + ps, 0xa5, ps); + ksft_test_result(lazy_kb(p) == 0, "%zuK dirty leaf no longer reported LazyFree\n", ps / 1024); + ok &= pageout(p, len); + ok &= all_bytes(p + ps, ps, 0xa5) && all_bytes(p, ps, 0x62) && + all_bytes(p + 2 * ps, 2 * ps, 0x62); + ksft_test_result(ok, "%zuK write after FREE cancels quarter discard\n", ps / 1024); + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0x83, len); + writes = stat_value("/proc/vmstat", "pswpout"); + ok &= mark_free(p, len) && pageout(p, len); + for (unsigned int i = 0; i < 4; i++) + ok &= !(pagemap(p + i * ps, ps) & ((1ULL << 63) | (1ULL << 62))); + printf("# full FREE pswpout=%lu->%lu, entries=%#llx %#llx %#llx %#llx\n", + writes, stat_value("/proc/vmstat", "pswpout"), + (unsigned long long)pagemap(p, ps), (unsigned long long)pagemap(p + ps, ps), + (unsigned long long)pagemap(p + 2 * ps, ps), (unsigned long long)pagemap(p + 3 * ps, ps)); + ok &= stat_value("/proc/vmstat", "pswpout") == writes && all_bytes(p, len, 0); + ksft_test_result(ok, "%zuK fully free backing reclaimed without swap writes\n", ps / 1024); + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0xb7, len); + ok &= pageout(p, len); + ok &= !!(pagemap(p + ps, ps) & (1ULL << 62)); + ok &= mark_free(p + ps, ps); + ok &= !(pagemap(p + ps, ps) & ((1ULL << 63) | (1ULL << 62))); + ok &= all_bytes(p + ps, ps, 0) && all_bytes(p, ps, 0xb7) && + all_bytes(p + 2 * ps, 2 * ps, 0xb7); + ksft_test_result(ok, "%zuK FREE drops selected swap entry without neighbor loss\n", ps / 1024); + { + int gate[2], status = 0; + pid_t pid; + char ready; + + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0xc8, len); + if (pipe(gate)) + ksft_exit_fail_msg("pipe\n"); + pid = fork(); + if (!pid) { + close(gate[1]); + if (read(gate[0], &ready, 1) != 1 || !all_bytes(p, len, 0xc8)) + _exit(2); + _exit(0); + } + close(gate[0]); + ok &= mark_free(p + ps, ps) && pageout(p, len); + ok &= write(gate[1], "x", 1) == 1; + close(gate[1]); + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + ksft_test_result(ok, "%zuK FREE after fork preserves other process data\n", ps / 1024); + } + munmap(reservation, len + 2 * native); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(strtoul(argv[3], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || swapon("/dev/vda", 0)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + char size[32], host_size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + snprintf(host_size, sizeof(host_size), "%lu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", size, host_size, NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + printf("%s - lazyfree fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c b/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c new file mode 100644 index 0000000000000..1dde51ab4d0e4 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/account" +#define LENGTH (32UL * 1024 * 1024) +static int put(const char *p, const char *v) +{ + int fd = open(p, O_WRONLY); + if (fd < 0) return 1; + int bad = write(fd, v, strlen(v)) != (ssize_t)strlen(v); + close(fd); return bad; +} +static unsigned long long stat_value(const char *key) +{ + FILE *f = fopen(CG "/memory.stat", "r"); + char name[128]; unsigned long long value = 0; + if (!f) exit(20); + while (fscanf(f, "%127s %llu", name, &value) == 2) + if (!strcmp(key, name)) { fclose(f); return value; } + fclose(f); exit(21); +} +static int worker(unsigned long expected, int ready, int release) +{ + if (getauxval(AT_PAGESZ) != expected) return 2; + unsigned char *p = mmap(NULL, LENGTH, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || madvise(p, LENGTH, MADV_NOHUGEPAGE)) return 3; + memset(p, 0x71, LENGTH); + char token = 'R'; + if (write(ready, &token, 1) != 1 || read(release, &token, 1) != 1) return 4; + /* Keep the writes observable independently of the accounting check. */ + for (size_t i = 0; i < LENGTH; i++) if (p[i] != 0x71) return 5; + return munmap(p, LENGTH) ? 6 : 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (getppid() == 1 && argc == 5 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), atoi(argv[3]), atoi(argv[4])); + if (getpid() != 1 || argc != 1) return 2; + signal(SIGPIPE, SIG_IGN); alarm(60); + int failed = 0; + unsigned long long slab[3] = {}; + unsigned long sizes[3] = {getauxval(AT_PAGESZ), 4096, 16384}; + unsigned int modes = sizes[0] == 65536 ? 3 : 2; + if ((sizes[0] != 16384 && sizes[0] != 65536) || mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) || + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory")) { + perror("setup"); failed = 1; goto done; + } + for (unsigned i = 0; i < modes; i++) { + int ready[2], release[2]; + if (mkdir(CG, 0755) || put(CG "/memory.max", "134217728") || + pipe(ready) || pipe(release)) { failed = 1; break; } + pid_t child = fork(); + if (!child) { + char ps[32], rfd[16], cfd[16]; + close(ready[0]); close(release[1]); + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(rfd, sizeof(rfd), "%d", ready[1]); + snprintf(cfd, sizeof(cfd), "%d", release[0]); + if (put(CG "/cgroup.procs", "0") || prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) _exit(7); + execl("/init", "/init", "--worker", ps, rfd, cfd, NULL); _exit(8); + } + close(ready[1]); close(release[0]); + char token; + int ok = child > 0 && read(ready[0], &token, 1) == 1; + if (ok) { + /* Let per-CPU slab statistics become visible before the comparison. */ + usleep(1200000); + slab[i] = stat_value("slab_unreclaimable"); + printf("# ABI=%lu anon=%llu slab_unreclaimable=%llu pagetables=%llu\n", + sizes[i], stat_value("anon"), slab[i], stat_value("pagetables")); + ok = stat_value("anon") >= LENGTH && write(release[1], "G", 1) == 1; + } + int status = 0; + if (child > 0 && !ok) kill(child, SIGKILL); + ok = child > 0 && waitpid(child, &status, 0) == child && WIFEXITED(status) && !WEXITSTATUS(status) && ok; + close(ready[0]); close(release[1]); + printf("%s - memcg accounting ABI %luK worker status=%#x\n", ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + for (unsigned retry = 0; retry < 100 && stat_value("anon"); retry++) usleep(20000); + if (stat_value("anon") || rmdir(CG)) failed = 1; + if (failed) break; + } + /* Slot metadata must be charged for every alternative granule. The + * loose lower bound tolerates slab/KASAN layout differences. */ + for (unsigned i = 1; !failed && i < modes; i++) { + if (slab[i] < slab[0] + 512 * 1024) { + printf("not ok - slot metadata escaped cgroup accounting ABI=%lu native=%llu alternative=%llu\n", + sizes[i], slab[0], slab[i]); + failed = 1; + } + } + + done: + printf("GRANULE MEMCG ACCOUNTING %s\n", failed ? "FAIL" : "PASS"); + sync(); reboot(RB_POWER_OFF); for (;;) pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c b/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c new file mode 100644 index 0000000000000..3e2d44bd926cb --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c @@ -0,0 +1,321 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/pressure" +#define MIB (1024UL * 1024) +#define LENGTH (32 * MIB) +#define ROUNDS 4 + +static int put(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + if (fd < 0) return -1; + ssize_t ret = write(fd, value, strlen(value)); + int saved = errno; + close(fd); errno = saved; + return ret == (ssize_t)strlen(value) ? 0 : -1; +} + +static unsigned long long get(const char *path, const char *key) +{ + FILE *f = fopen(path, "r"); + char name[128]; + unsigned long long value = 0, result = 0; + if (!f) { perror(path); exit(10); } + if (!key) { + if (fscanf(f, "%llu", &result) != 1) exit(11); + } else while (fscanf(f, "%127s %llu", name, &value) == 2) + if (!strcmp(name, key)) { result = value; break; } + fclose(f); return result; +} + +static unsigned char pattern(size_t offset, unsigned seed, unsigned round) +{ + return (offset ^ (offset >> 12) ^ (offset >> 20) ^ (seed * 79) ^ (round * 43)) & 255; +} + +static int contents(unsigned char *p, size_t ps, unsigned seed, unsigned round, + int write_data, int child) +{ + size_t pages = LENGTH / ps; + /* Odd multiplication permutes every logical page of this power-of-two range. */ + for (size_t i = 0; i < pages; i++) { + size_t page = (i * 4051 + round * 31) & (pages - 1); + for (size_t j = 0; j < ps; j++) { + size_t offset = page * ps + j; + unsigned char want = pattern(offset, seed, round); + if (child && !(page & 3)) want ^= 0x69; + if (write_data) p[offset] = want; + else if (p[offset] != want) { + printf("not ok - ABI=%zu round=%u child=%d byte=%zu got=%u expected=%u\n", + ps, round, child, offset, p[offset], want); + return 1; + } + } + } + return 0; +} + +static int barrier(int ready, int command) +{ + char token = 'R'; + return write(ready, &token, 1) != 1 || read(command, &token, 1) != 1 || token != 'G'; +} + +static int worker(unsigned long expected, unsigned seed, int ready, int command) +{ + size_t ps = getauxval(AT_PAGESZ); + unsigned char *p; + if (ps != expected) return 2; + p = mmap(NULL, LENGTH, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 3; + if (madvise(p, LENGTH, MADV_NOHUGEPAGE)) { + int error = errno; + struct stat st; + + /* Without THP this advice is unavailable and cannot change policy. */ + if (error != EINVAL || + !stat("/sys/kernel/mm/transparent_hugepage/enabled", &st) || + errno != ENOENT) + return 3; + } + if (contents(p, ps, seed, 0, 1, 0) || barrier(ready, command)) return 4; + for (unsigned round = 0; round < ROUNDS; round++) { + if (contents(p, ps, seed, round, 0, 0)) return 5; + pid_t child = fork(); + if (child < 0) return 6; + if (!child) { + /* Dirty a quarter of the logical pages, retaining untouched siblings. */ + for (size_t page = 0; page < LENGTH / ps; page += 4) + for (size_t j = 0; j < ps; j++) p[page * ps + j] ^= 0x69; + _exit(contents(p, ps, seed, round, 0, 1) ? 7 : 0); + } + int status; + if (waitpid(child, &status, 0) != child || !WIFEXITED(status) || + WEXITSTATUS(status) || contents(p, ps, seed, round, 0, 0)) return 8; + printf("ok - memcg ABI %zuK round=%u swap-in bytes and fork COW isolation\n", ps / 1024, round); + if (round + 1 < ROUNDS && contents(p, ps, seed, round + 1, 1, 0)) return 9; + if (barrier(ready, command)) return 10; + } + return munmap(p, LENGTH) ? 11 : 0; +} + +static void report(unsigned phase) +{ + printf("# memcg phase=%u resident=%llu swap=%llu anon=%llu pgscan=%llu pgsteal=%llu max=%llu oom=%llu oom_kill=%llu\n", + phase, get(CG "/memory.current", NULL), get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "anon"), get(CG "/memory.stat", "pgscan"), + get(CG "/memory.stat", "pgsteal"), get(CG "/memory.events", "max"), + get(CG "/memory.events", "oom"), get(CG "/memory.events", "oom_kill")); +} + +static void pressure_stacks(void) +{ + FILE *tasks = fopen(CG "/cgroup.procs", "r"); + int pid; + + if (!tasks) + return; + while (fscanf(tasks, "%d", &pid) == 1) { + char path[64], stack[4096]; + int fd; + ssize_t n; + + snprintf(path, sizeof(path), "/proc/%d/stack", pid); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + continue; + n = read(fd, stack, sizeof(stack) - 1); + close(fd); + if (n > 0) { + stack[n] = 0; + printf("# pressure stack pid=%d\n%s", pid, stack); + } + } + fclose(tasks); +} + +static void observe_pressure(void) +{ + unsigned int samples = 0; + + for (;;) { + struct timespec now; + + clock_gettime(CLOCK_MONOTONIC, &now); + printf("# io-pressure ms=%lld resident=%llu swap=%llu writeback=%llu " + "swapcached=%llu scan=%llu steal=%llu max=%llu oom=%llu kill=%llu\n", + (long long)now.tv_sec * 1000 + now.tv_nsec / 1000000, + get(CG "/memory.current", NULL), get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "file_writeback"), + get(CG "/memory.stat", "swapcached"), get(CG "/memory.stat", "pgscan"), + get(CG "/memory.stat", "pgsteal"), get(CG "/memory.events", "max"), + get(CG "/memory.events", "oom"), get(CG "/memory.events", "oom_kill")); + if (!(++samples % 8) && get(CG "/memory.stat", "file_writeback") > 8 * MIB) + pressure_stacks(); + usleep(250000); + } +} + +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker") && getppid() == 1) + return worker(strtoul(argv[2], NULL, 10), strtoul(argv[3], NULL, 10), atoi(argv[4]), atoi(argv[5])); + bool classic = false, native_control = false, user16k = false; + + if (getpid() != 1 || argc > 3) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--classic")) + classic = true; + else if (!strcmp(argv[i], "--native-control")) + native_control = true; + else if (!strcmp(argv[i], "--user16k")) + user16k = true; + else + return 2; + } + if (native_control && user16k) + return 2; + signal(SIGPIPE, SIG_IGN); + alarm(300); + int failed = 0, active = 0; + pid_t children[2] = {-1, -1}, observer = -1; + int ready[2][2], command[2][2]; + unsigned long sizes[2] = {getauxval(AT_PAGESZ), 4096}; + if (native_control) + sizes[1] = sizes[0]; + if (user16k) + sizes[1] = 16384; + if ((sizes[0] != 16384 && sizes[0] != 65536) || sizes[1] > sizes[0]) { + failed = 1; goto done; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) || + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory") || + mkdir(CG, 0755) || put(CG "/memory.max", "50331648") || + put(CG "/memory.swap.max", "268435456")) { + perror("cgroup setup"); failed = 1; goto done; + } + /* Explicit diagnostic control, never change the production default. */ + if (classic && put("/sys/kernel/mm/lru_gen/enabled", "0")) { + perror("classic reclaim control"); failed = 1; goto done; + } + printf("# reclaim mode=%s\n", classic ? "classic diagnostic" : "default"); + struct stat st; + for (int i = 0; i < 100 && stat("/dev/vda", &st); i++) usleep(20000); + if (swapon("/dev/vda", 0)) { perror("swapon"); failed = 1; goto done; } + active = 1; + for (unsigned i = 0; i < 2; i++) if (pipe(ready[i]) || pipe(command[i])) { + failed = 1; goto done; + } + observer = fork(); + if (!observer) { + for (unsigned int i = 0; i < 2; i++) { + close(ready[i][0]); close(ready[i][1]); + close(command[i][0]); close(command[i][1]); + } + observe_pressure(); + _exit(0); + } + if (observer < 0) { + failed = 1; + goto done; + } + for (unsigned i = 0; i < 2; i++) { + children[i] = fork(); + if (!children[i]) { + char ps[32], index[16], rfd[16], cfd[16]; + for (unsigned j = 0; j < 2; j++) { + close(ready[j][0]); close(command[j][1]); + if (i != j) { close(ready[j][1]); close(command[j][0]); } + } + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(index, sizeof(index), "%u", i); + snprintf(rfd, sizeof(rfd), "%d", ready[i][1]); + snprintf(cfd, sizeof(cfd), "%d", command[i][0]); + if (put(CG "/cgroup.procs", "0") || prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) _exit(12); + execl("/init", "/init", "--worker", ps, index, rfd, cfd, NULL); + _exit(13); + } + if (children[i] < 0) { failed = 1; goto done; } + } + for (unsigned i = 0; i < 2; i++) { close(ready[i][1]); close(command[i][0]); } + for (unsigned phase = 0; phase <= ROUNDS; phase++) { + char token; + for (unsigned i = 0; i < 2; i++) + if (read(ready[i][0], &token, 1) != 1 || token != 'R') { failed = 1; goto done; } + report(phase); + if (!get(CG "/memory.swap.current", NULL) || !get(CG "/memory.events", "max") || + !get(CG "/memory.stat", "pgscan") || !get(CG "/memory.stat", "pgsteal") || + get(CG "/memory.events", "oom") || get(CG "/memory.events", "oom_kill")) { + printf("not ok - missing pressure evidence or unexpected OOM\n"); failed = 1; goto done; + } + for (unsigned i = 0; i < 2; i++) if (write(command[i][1], "G", 1) != 1) { failed = 1; goto done; } + } + for (unsigned i = 0; i < 2; i++) { + int status = 0; + int ok = waitpid(children[i], &status, 0) == children[i] && WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - memcg ABI %luK exit=%#x\n", ok ? "ok" : "not ok", sizes[i] / 1024, status); + children[i] = -1; failed |= !ok; + } + /* Pending swap writeback can outlive exit, especially on a slow device. */ + for (unsigned i = 0; i < 3000 && (get(CG "/memory.stat", "anon") || + get(CG "/memory.swap.current", NULL)); i++) { + if (!(i % 250)) + printf("# exit drain ms=%u swap=%llu writeback=%llu\n", i * 20, + get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "file_writeback")); + /* Exit may leave clean, unmapped swap cache after writeback. Reclaim + * it explicitly; EAGAIN means fewer than the requested bytes existed. */ + if (!get(CG "/memory.stat", "file_writeback") && + put(CG "/memory.reclaim", "67108864 swappiness=200") && errno != EAGAIN) { + perror("exit cache reclaim"); failed = 1; break; + } + usleep(20000); + } + report(ROUNDS + 1); + if (get(CG "/memory.stat", "anon") || get(CG "/memory.swap.current", NULL)) { + printf("not ok - anonymous or swap charge remains after exit\n"); failed = 1; + } + done: + if (observer > 0) { + kill(observer, SIGTERM); + waitpid(observer, NULL, 0); + } + if (active && failed) + report(ROUNDS + 2); + for (unsigned i = 0; i < 2; i++) if (children[i] > 0) { kill(children[i], SIGKILL); waitpid(children[i], NULL, 0); } + if (active && failed) { + put(CG "/cgroup.kill", "1"); + put(CG "/memory.max", "max"); + } + if (active && swapoff("/dev/vda")) { perror("swapoff"); failed = 1; } + printf("GRANULE MEMCG PRESSURE %s\n", failed ? "FAIL" : "PASS"); + sync(); reboot(RB_POWER_OFF); for (;;) pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/memory-interfaces.c b/tools/testing/selftests/arm64/user4k/memory-interfaces.c new file mode 100644 index 0000000000000..157cc0749d4f3 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memory-interfaces.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static size_t page; + +static void residency(void) +{ + unsigned char *p, vec[6]; + bool ok; + unsigned int i; + + p = mmap(NULL, 5 * page, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap\n"); + memset(vec, 0x55, sizeof(vec)); + ok = mincore(p, 5 * page - 1, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == 0; + ok &= vec[5] == 0x55; + ksft_test_result(ok, "mincore emits exactly one byte per unfaulted user page\n"); + ok = mprotect(p + 2 * page, page, PROT_READ | PROT_WRITE) == 0; + p[2 * page] = 0x37; + ok &= mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == (i == 2); + ksft_test_result(ok, "mincore reports a separately faulted VMA among cold neighbors\n"); + ok = munmap(p + 3 * page, page) == 0; + errno = 0; + ok &= mincore(p, 5 * page, vec) == -1 && errno == ENOMEM; + errno = 0; + ok &= mincore(p + 1, page, vec) == -1 && errno == EINVAL; + ksft_test_result(ok, "mincore rejects an unaligned start and a VMA hole\n"); + munmap(p, 5 * page); +} + +static void file_residency(void) +{ + const size_t native = 16384, count = 3 * native / page; + unsigned char *p, *guard, *vec, byte = 0x7b; + int fd = memfd_create("residency", 0); + bool ok; + size_t i; + + if (fd < 0 || ftruncate(fd, 4 * native) || pwrite(fd, &byte, 1, native) != 1) + ksft_exit_fail_msg("memfd setup\n"); + p = mmap(NULL, 3 * native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, page); + guard = mmap(NULL, 2 * page, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || guard == MAP_FAILED || mprotect(guard + page, page, PROT_NONE)) + ksft_exit_fail_msg("file/vector mmap\n"); + vec = guard + page - count; + memset(vec, 0x55, count); + ok = mincore(p, 3 * native, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == ((page + i * page) / native == 1); + ksft_test_result(ok, "unmapped file leaves share residency at native cache indices\n"); + ok = p[native - page] == byte && mincore(p, 3 * native, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == ((page + i * page) / native == 1); + ksft_test_result(ok, "present and cache-only file residency agree at a guarded vector boundary\n"); + ok = msync(p + page, 1, MS_SYNC) == 0; + errno = 0; + ok &= msync(p + page + 1, page, MS_SYNC) == -1 && errno == EINVAL; + ok &= munmap(p + page, page) == 0; + errno = 0; + ok &= msync(p, 3 * page, MS_SYNC) == -1 && errno == ENOMEM; + ksft_test_result(ok, "msync uses process alignment and reports holes\n"); + munmap(p, 3 * native); + munmap(guard, 2 * page); + close(fd); +} + +static void large_vector(void) +{ + const size_t count = 16384 + 3; + unsigned char *p = mmap(NULL, count * page, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + unsigned char *vec = malloc(count + 1); + bool ok; + size_t i; + + if (p == MAP_FAILED || !vec) + ksft_exit_fail_msg("large vector allocation\n"); + memset(vec, 0x55, count + 1); + ok = mincore(p, count * page - 7, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == 0; + ok &= vec[count] == 0x55; + ksft_test_result(ok, "mincore advances across multiple native output buffers\n"); + free(vec); + munmap(p, count * page); +} + +static void seals(void) +{ + unsigned char *p = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + bool ok; + pid_t pid; + int status = 0; + + if (p == MAP_FAILED) + ksft_exit_fail_msg("seal mmap\n"); + ok = syscall(SYS_mseal, p + page, 1UL, 0UL) == 0; + errno = 0; + ok &= mprotect(p + page, page, PROT_READ) == -1 && errno == EPERM; + errno = 0; + ok &= munmap(p + page, page) == -1 && errno == EPERM; + ok &= mprotect(p, page, PROT_READ) == 0; + ok &= mprotect(p + 2 * page, page, PROT_READ) == 0; + ksft_test_result(ok, "a one-byte seal covers only its user page and protects it\n"); + pid = fork(); + if (!pid) + _exit(!(munmap(p + page, page) == -1 && errno == EPERM)); + ok = pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + ksft_test_result(ok, "fork inherits the sealed user page\n"); + munmap(p, page); + munmap(p + 2 * page, page); + /* The sealed VMA is released by process exit. */ +} + +int main(int argc, char **argv) +{ + page = getauxval(AT_PAGESZ); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + ksft_exit_fail_msg("unexpected process page ABI: %zu\n", page); + ksft_print_msg("process page size: %zu\n", page); + ksft_print_header(); + ksft_set_plan(9); + residency(); + file_residency(); + large_vector(); + seals(); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/mglru-granule-init.c b/tools/testing/selftests/arm64/user4k/mglru-granule-init.c new file mode 100644 index 0000000000000..26b6d587d5d86 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mglru-granule-init.c @@ -0,0 +1,259 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CONTROL "/sys/kernel/debug/lru_gen" +#define FULL "/sys/kernel/debug/lru_gen_full" +static int ready_fd = -1, command_fd = -1; +static unsigned worker_index; +static unsigned char pattern(unsigned region, size_t byte, unsigned round) +{ + return (byte ^ (byte >> 12) ^ (region * 37) ^ (round * 53)) & 255; +} +static int generation(unsigned long long *cg, unsigned *node, unsigned long *max) +{ + FILE *f = fopen(CONTROL, "r"); + char line[256]; + int have_cg = 0, have_node = 0, have_seq = 0; + if (!f) return -1; + while (fgets(line, sizeof(line), f)) { + unsigned long long id; + unsigned n; + unsigned long seq, age, anon, file; + if (sscanf(line, "memcg %llu", &id) == 1) { + if (have_cg) break; + *cg = id; have_cg = 1; + } else if (sscanf(line, " node %u", &n) == 1) { + if (have_node) break; + *node = n; have_node = 1; + } else if (have_node && sscanf(line, "%lu %lu %lu %lu", &seq, &age, &anon, &file) == 4) { + *max = seq; have_seq = 1; + } + } + fclose(f); + return have_seq ? 0 : -1; +} +static unsigned long scan_evidence(void) +{ + FILE *f = fopen(FULL, "r"); + char line[512]; + unsigned long total = 0; + if (!f) return 0; + while (fgets(line, sizeof(line), f)) { + unsigned long t, y, found, added; + if (sscanf(line, " %lut %luy %luf %lua", &t, &y, &found, &added) == 4) { + printf("# MGLRU scan total=%lu young=%lu tables=%lu added=%lu\n", t, y, found, added); + total += y; + } + } + fclose(f); return total; +} +static int age(void) +{ + if (ready_fd >= 0) { + char token = 'A'; + return write(ready_fd, &token, 1) != 1 || + read(command_fd, &token, 1) != 1 || token != 'G'; + } + unsigned long long cg = 0, next_cg = 0; + unsigned node = 0, next_node = 0; + unsigned long seq = 0, next = 0; + char command[128]; + if (generation(&cg, &node, &seq)) return 1; + /* This tree accepts swappiness here; 200 scans both anonymous and file pages. */ + int len = snprintf(command, sizeof(command), "+ %llu %u %lu 200 1\n", cg, node, seq); + int fd = open(CONTROL, O_WRONLY); + if (fd < 0) return 1; + ssize_t written = write(fd, command, len); + close(fd); + if (written != len || generation(&next_cg, &next_node, &next) || next <= seq) { + printf("not ok - MGLRU generation command=%s result=%zd errno=%d next=%lu\n", command, written, errno, next); + return 1; + } + printf("ok - MGLRU generation %lu -> %lu\n", seq, next); + return 0; +} +static int check_referenced(unsigned char **regions, unsigned nr, size_t expected) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char line[512]; + int region = -1, failed = 0; + unsigned seen = 0; + unsigned long start, end, kb; + if (!f) return 1; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + region = -1; + for (unsigned i = 0; i < nr; i++) + if (start == (uintptr_t)regions[i]) region = i; + } else if (region >= 0 && sscanf(line, "Referenced: %lu kB", &kb) == 1) { + printf("%s - region=%d start=%p referenced=%lu expected=%zu kB\n", kb == expected ? "ok":"not ok", region, regions[region], kb, expected); + failed |= kb != expected; + seen++; + } + } + fclose(f); + return failed || seen != nr; +} +static int exercise(unsigned long expected) +{ + static const unsigned shifts[] = {21,25,29,30,36,39,42,47}; + unsigned char *regions[sizeof(shifts)/sizeof(shifts[0])] = {}; + size_t ps = getauxval(AT_PAGESZ), len = 8 * ps; + unsigned nr = sizeof(shifts)/sizeof(shifts[0]); + int fail = 0, fd = memfd_create("granule-aging", 0); + cpu_set_t cpus; + void *scratch = MAP_FAILED; + CPU_ZERO(&cpus); CPU_SET(ready_fd >= 0 ? worker_index % 2 : 0, &cpus); + if (sched_setaffinity(0, sizeof(cpus), &cpus)) return 1; + if (ps != expected || fd < 0 || ftruncate(fd, nr * len)) return 1; + for (unsigned i = 0; i < nr; i++) { + uintptr_t address = (1UL << shifts[i]) - 4 * ps; + regions[i] = mmap((void *)address, len, PROT_READ|PROT_WRITE, + MAP_FIXED_NOREPLACE | (i % 2 ? MAP_SHARED : MAP_PRIVATE|MAP_ANONYMOUS), + i % 2 ? fd : -1, i % 2 ? i * len : 0); + if (regions[i] != (void *)address) { + printf("not ok - %zuK boundary %u mmap errno=%d\n", ps/1024, shifts[i], errno); + fail = 1; goto out; + } + } + for (unsigned i = 0; i < nr; i++) memset(regions[i], 0x19, len); + /* Publish earlier folios from the bounded per-CPU LRU-add batch. */ + scratch = mmap(NULL, 8UL << 20, PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (scratch == MAP_FAILED) { fail = 1; goto out; } + memset(scratch, 0x7b, 8UL << 20); + /* Seed MGLRU's first-access referenced state before testing the next access. */ + if (age()) { fail = 1; goto out; } + for (unsigned round = 0; round < 4; round++) { + if (ready_fd >= 0) { + unsigned cpu = (worker_index + round) % 2; + CPU_ZERO(&cpus); CPU_SET(cpu, &cpus); + if (sched_setaffinity(0, sizeof(cpus), &cpus) || sched_getcpu() != (int)cpu) { + fail = 1; break; + } + printf("# concurrent ABI=%zuK round=%u cpu=%u\n", ps/1024, round, cpu); + } + for (unsigned i = 0; i < nr; i++) + for (size_t j = 0; j < len; j++) regions[i][j] = pattern(i, j, round); + fail |= check_referenced(regions, nr, len / 1024); + if (age() || age()) { fail = 1; break; } + fail |= check_referenced(regions, nr, 0); + int good = 1; + for (unsigned i = 0; i < nr; i++) + for (size_t j = 0; j < len; j++) + if (regions[i][j] != pattern(i,j,round)) { good = 0; break; } + unsigned long young = scan_evidence(); + printf("%s - %zuK round%u eight table boundaries bytes and aging young=%lu\n", good && young ? "ok":"not ok",ps/1024,round,young); + fail |= !good || !young; + } +out: + if (scratch != MAP_FAILED) munmap(scratch, 8UL << 20); + for (unsigned i = 0; i < nr; i++) + if (regions[i] && regions[i] != MAP_FAILED) munmap(regions[i],len); + close(fd); return fail; +} +static int concurrent(unsigned long *sizes, unsigned nr) +{ + int ready[3][2], commands[3][2], failed = 0; + pid_t workers[3]; + + for (unsigned i = 0; i < nr; i++) + if (pipe(ready[i]) || pipe(commands[i])) return 1; + for (unsigned i = 0; i < nr; i++) { + workers[i] = fork(); + if (!workers[i]) { + char size[32], index[16], ready_arg[16], command_arg[16]; + for (unsigned j = 0; j < nr; j++) { + close(ready[j][0]); close(commands[j][1]); + if (j != i) { close(ready[j][1]); close(commands[j][0]); } + } + snprintf(size, sizeof(size), "%lu", sizes[i]); + snprintf(index, sizeof(index), "%u", i); + snprintf(ready_arg, sizeof(ready_arg), "%d", ready[i][1]); + snprintf(command_arg, sizeof(command_arg), "%d", commands[i][0]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) _exit(3); + execl("/init", "/init", "--worker", size, index, ready_arg, command_arg, NULL); + _exit(4); + } + if (workers[i] < 0) failed = 1; + } + for (unsigned i = 0; i < nr; i++) { close(ready[i][1]); close(commands[i][0]); } + /* One warm-up barrier plus two aging barriers for each of four rounds. */ + for (unsigned phase = 0; phase < 9 && !failed; phase++) { + for (unsigned i = 0; i < nr; i++) { + char token; + if (read(ready[i][0], &token, 1) != 1 || token != 'A') failed = 1; + } + if (failed || age()) { failed = 1; break; } + for (unsigned i = 0; i < nr; i++) { + char token = 'G'; + if (write(commands[i][1], &token, 1) != 1) failed = 1; + } + } + for (unsigned i = 0; i < nr; i++) { close(ready[i][0]); close(commands[i][1]); } + for (unsigned i = 0; i < nr; i++) { + int status = 0; + int ok = workers[i] > 0 && waitpid(workers[i], &status, 0) == workers[i] && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - concurrent MGLRU ABI %luK status=%#x\n", ok ? "ok":"not ok", sizes[i]/1024, status); + failed |= !ok; + } + return failed; +} +int main(int argc, char **argv) +{ + setvbuf(stdout,NULL,_IONBF,0); + signal(SIGPIPE, SIG_IGN); + if (argc == 3 && !strcmp(argv[1],"--exercise")) { + if (getppid() != 1) return 2; + return exercise(strtoul(argv[2],NULL,10)); + } + if (argc == 6 && !strcmp(argv[1], "--worker")) { + if (getppid() != 1) return 2; + worker_index = strtoul(argv[3], NULL, 10); + ready_fd = atoi(argv[4]); command_fd = atoi(argv[5]); + return exercise(strtoul(argv[2], NULL, 10)); + } + int concurrent_mode = argc == 2 && !strcmp(argv[1], "--concurrent"); + if (getpid()!=1 || (argc!=1 && !concurrent_mode)) return 2; + int fail=0; + if (mount("proc","/proc","proc",0,NULL) || mount("sysfs","/sys","sysfs",0,NULL) || + mount("debugfs","/sys/kernel/debug","debugfs",0,NULL)) { perror("mount");fail=1;goto done; } + FILE *f=fopen("/sys/kernel/mm/lru_gen/enabled","r");unsigned enabled=0; + if (f) { fscanf(f,"%x",&enabled);fclose(f); } + if ((enabled & 3)!=3) { printf("not ok - MGLRU/MM walk disabled %#x\n",enabled);fail=1;goto done; } + printf("ok - MGLRU enabled=%#x\n",enabled); + unsigned long sizes[]={getauxval(AT_PAGESZ),4096,16384}; + if (concurrent_mode) { + fail = concurrent(sizes, sizes[0] > 16384 ? 3 : 2); + goto done; + } + for (unsigned i=0;i<(sizes[0]>16384?3U:2U);i++) { + pid_t child=fork();int status=0; + if (!child) { + char value[32];snprintf(value,sizeof(value),"%lu",sizes[i]); + if(prctl(SET_EXEC_PAGE_SIZE,sizes[i],0UL,0UL,0UL))_exit(3); + execl("/init","/init","--exercise",value,NULL);_exit(4); + } + int good=child>0 && waitpid(child,&status,0)==child && WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - MGLRU ABI %luK status=%#x\n",good?"ok":"not ok",sizes[i]/1024,status);fail|=!good; + } +done: + printf("GRANULE MGLRU%s %s\n", concurrent_mode ? " CONCURRENT":"", fail?"FAIL":"PASS");sync();reboot(RB_POWER_OFF);for(;;)pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/mixed-process.c b/tools/testing/selftests/arm64/user4k/mixed-process.c new file mode 100644 index 0000000000000..00b09364f0330 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mixed-process.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real EL0 mixed-granule sharing, futex queues, and process_vm copies. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define NATIVE 16384U +#define QUARTER 4096U +#define ROUNDS 2000U +struct channel { + atomic_int turn; + unsigned int value; + unsigned int worker_wakes; +}; +struct ready { + uintptr_t base; + unsigned long page; +}; + +static int wait_turn(struct channel *channel, int wanted) +{ + const struct timespec timeout = { .tv_sec = 5 }; + int value; + + while ((value = atomic_load_explicit(&channel->turn, memory_order_acquire)) != wanted) { + if (syscall(SYS_futex, &channel->turn, FUTEX_WAIT, value, + &timeout, NULL, 0) < 0 && errno != EAGAIN && errno != EINTR) + return -1; + } + return 0; +} + +static int send_turn(struct channel *channel, int turn) +{ + atomic_store_explicit(&channel->turn, turn, memory_order_release); + return syscall(SYS_futex, &channel->turn, FUTEX_WAKE, 1, NULL, NULL, 0); +} + +static unsigned char pattern(size_t i) +{ + return (i ^ (i >> 8) ^ (i >> 16)) & 255; +} + +static int worker(int fd, int ready_fd, unsigned long expected) +{ + unsigned long page = getauxval(AT_PAGESZ); + unsigned char *mapping, *remote; + struct channel *channel; + struct ready ready; + size_t i, begin = page - 13, end = 3 * page + 16; + unsigned int round, wakes = 0; + int ret; + + if (page != expected) + return 10; + mapping = mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_SHARED, + fd, page == QUARTER ? QUARTER : 0); + remote = mmap(NULL, 5 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED || remote == MAP_FAILED) + return 11; + channel = (void *)(mapping + (page == QUARTER ? 0 : QUARTER)); + for (i = 0; i < 4 * page; i++) + remote[i] = pattern(i); + if (mprotect(remote + 4 * page, page, PROT_NONE)) + return 12; + ready = (struct ready) { (uintptr_t)remote, page }; + if (write(ready_fd, &ready, sizeof(ready)) != sizeof(ready)) + return 13; + close(ready_fd); + for (round = 0; round < ROUNDS; round++) { + if (wait_turn(channel, 1) || channel->value != (round ^ 0xabcddcbaU)) + return 14; + if (!round) { + for (i = 0; i < 4 * page; i++) + if (remote[i] != (pattern(i) ^ (i >= begin && i < end ? 0x5a : 0))) + return 15; + } + channel->value = ~(round ^ 0xabcddcbaU); + ret = send_turn(channel, 0); + if (ret < 0) + return 16; + wakes += ret; + } + channel->worker_wakes = wakes; + return 0; +} + +static bool remote_copies(pid_t pid, const struct ready *ready) +{ + size_t begin = ready->page - 13, len = 2 * ready->page + 29, i; + unsigned char *data = malloc(len), edge[2] = {}; + struct iovec local = { data, len }, remote = { (void *)(ready->base + begin), len }; + bool ok = data != NULL; + + if (!ok) + return false; + ok = process_vm_readv(pid, &local, 1, &remote, 1, 0) == (ssize_t)len; + for (i = 0; ok && i < len; i++) + ok = data[i] == pattern(i + begin); + for (i = 0; i < len; i++) + data[i] = pattern(i + begin) ^ 0x5a; + ok &= process_vm_writev(pid, &local, 1, &remote, 1, 0) == (ssize_t)len; + local = (struct iovec) { edge, sizeof(edge) }; + remote = (struct iovec) { (void *)(ready->base + 4 * ready->page - 1), sizeof(edge) }; + ok &= process_vm_readv(pid, &local, 1, &remote, 1, 0) == 1; + ok &= edge[0] == pattern(4 * ready->page - 1); + free(data); + return ok; +} + +int main(int argc, char **argv) +{ + unsigned long page = getauxval(AT_PAGESZ), other; + unsigned char *mapping, *private; + struct channel *channel; + struct ready ready = {}; + int fd, pipefd[2], status = 0, wakes = 0, ret; + unsigned int round = 0; + pid_t pid; + bool ok; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), strtoul(argv[4], NULL, 10)); + if (argc != 1 || (page != QUARTER && page != NATIVE)) + return 2; + other = page == NATIVE ? QUARTER : NATIVE; + fd = memfd_create("mixed-process", 0); + if (fd < 0 || ftruncate(fd, 2 * NATIVE) || pipe(pipefd)) + return 3; + mapping = mmap(NULL, 2 * NATIVE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + private = mmap(NULL, NATIVE, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + if (mapping == MAP_FAILED || private == MAP_FAILED) + return 4; + channel = (void *)(mapping + QUARTER); + private[QUARTER] = 0x7e; + pid = fork(); + if (!pid) { + char fdarg[32], pipearg[32], sizearg[32]; + + close(pipefd[0]); + snprintf(fdarg, sizeof(fdarg), "%d", fd); + snprintf(pipearg, sizeof(pipearg), "%d", pipefd[1]); + snprintf(sizearg, sizeof(sizearg), "%lu", other); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, other, 0UL, 0UL, 0UL)) + _exit(20); + execl("/mixed-process", "/mixed-process", "--worker", fdarg, pipearg, sizearg, NULL); + _exit(21); + } + close(pipefd[1]); + if (pid < 0) + return 5; + ok = read(pipefd[0], &ready, sizeof(ready)) == sizeof(ready) && ready.page == other; + close(pipefd[0]); + ok = ok && remote_copies(pid, &ready); + printf("%s - process_vm copies %lu -> %lu, including partial inaccessible boundary\n", + ok ? "ok" : "not ok", page, other); + for (round = 0; ok && round < ROUNDS; round++) { + channel->value = round ^ 0xabcddcbaU; + ret = send_turn(channel, 1); + ok = ret >= 0 && !wait_turn(channel, 0) && + channel->value == ~(round ^ 0xabcddcbaU); + if (ret > 0) + wakes += ret; + } + if (!ok) + kill(pid, SIGKILL); + ret = waitpid(pid, &status, 0); + ok &= ret == pid && WIFEXITED(status) && !WEXITSTATUS(status); + ok &= wakes > 0 && channel->worker_wakes > 0 && private[QUARTER] == 0x7e; + ok &= mapping[0] == 0 && mapping[2 * QUARTER] == 0 && mapping[NATIVE - 1] == 0; + printf("%s - mixed %lu/%lu shared futex/COW: rounds=%u wakes=%d/%u child=%#x\n", + ok ? "ok" : "not ok", page, other, round, wakes, channel->worker_wakes, status); + munmap(private, NATIVE); + munmap(mapping, 2 * NATIVE); + close(fd); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c b/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c new file mode 100644 index 0000000000000..3c5c07f68b548 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static long locked_kb(void) +{ + char line[256]; + long value = -1; + FILE *f = fopen("/proc/self/status", "re"); + + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmLck: %ld", &value) == 1) + break; + fclose(f); + return value; +} + +static bool wait_ok(pid_t pid) +{ + int status = 0; + + return pid > 0 && waitpid(pid, &status, 0) == pid && WIFEXITED(status) && !WEXITSTATUS(status); +} + +static bool remap_locked(unsigned long ps) +{ + unsigned char *p, *q, *target; + unsigned long kb = ps / 1024; + bool ok; + + p = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || mlock(p, 2 * ps)) + ksft_exit_fail_msg("mlock remap setup\n"); + memset(p, 0x37, 2 * ps); + ok = locked_kb() == (long)(2 * kb); + q = mremap(p, 2 * ps, 4 * ps, MREMAP_MAYMOVE); + if (q == MAP_FAILED) + ksft_exit_fail_msg("locked remap grow: %s\n", strerror(errno)); + ok &= locked_kb() == (long)(4 * kb); + for (unsigned long i = 0; i < 4 * ps; i++) + ok &= q[i] == (i < 2 * ps ? 0x37 : 0); + p = mremap(q, 4 * ps, 3 * ps, 0); + if (p == MAP_FAILED) + ksft_exit_fail_msg("locked remap shrink: %s\n", strerror(errno)); + ok &= locked_kb() == (long)(3 * kb); + p[2 * ps] = 0xb6; + ok &= !munmap(p + ps, ps) && locked_kb() == (long)(2 * kb); + target = mmap(NULL, ps, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (target == MAP_FAILED) + ksft_exit_fail_msg("locked remap target\n"); + q = mremap(p, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, target); + if (q == MAP_FAILED) + ksft_exit_fail_msg("locked remap move: %s\n", strerror(errno)); + ok &= q == target && locked_kb() == (long)(2 * kb); + for (unsigned long i = 0; i < ps; i++) + ok &= q[i] == 0x37; + ok &= p[2 * ps] == 0xb6; + ok &= !munmap(q, ps) && locked_kb() == (long)kb; + ok &= !munmap(p + 2 * ps, ps) && locked_kb() == 0; + return ok; +} + +static int exercise(void) +{ + unsigned long ps = getauxval(AT_PAGESZ), kb = ps / 1024; + unsigned char *p, *q, residency[4]; + int gate[2]; + pid_t pid; + bool ok; + + ksft_print_header(); + ksft_set_plan(9); + p = mmap(NULL, 32 * ps, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || locked_kb() != 0) + ksft_exit_fail_msg("initial mapping/accounting\n"); + ok = !mlock(p + ps + 17, ps - 32) && locked_kb() == (long)kb; + ok &= !mincore(p + ps, ps, residency) && residency[0] & 1; + p[ps] = 0x51; + ksft_test_result(ok, "%luK unaligned mlock uses exactly one user leaf\n", kb); + ok = !mlock(p + ps, ps) && locked_kb() == (long)kb; + ok &= !mlock(p + 2 * ps, ps) && locked_kb() == (long)(2 * kb); + ok &= !munlock(p + ps, ps) && locked_kb() == (long)kb; + p[2 * ps] = 0x62; + errno = 0; + ok &= madvise(p + 2 * ps, ps, MADV_PAGEOUT) == -1 && errno == EINVAL && + p[2 * ps] == 0x62; + ok &= !munlock(p + 2 * ps, ps) && locked_kb() == 0; + ksft_test_result(ok, "%luK repeat lock and independent adjacent unlock\n", kb); + ok = !mlock2(p + 4 * ps, 4 * ps, MLOCK_ONFAULT) && locked_kb() == (long)(4 * kb); + ok &= !mincore(p + 4 * ps, 4 * ps, residency); + for (unsigned int i = 0; i < 4; i++) + ok &= !(residency[i] & 1); + p[4 * ps] = 0x74; + ok &= !mincore(p + 4 * ps, ps, residency) && residency[0] & 1; + ok &= !munlock(p + 4 * ps, 4 * ps) && locked_kb() == 0; + ksft_test_result(ok, "%luK MLOCK_ONFAULT accounting and deferred faults\n", kb); + q = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_LOCKED, -1, 0); + ok = q != MAP_FAILED && locked_kb() == (long)(4 * kb); + if (q == MAP_FAILED || pipe(gate)) + ksft_exit_fail_msg("locked mmap/fork setup\n"); + memset(q, 0x8a, 4 * ps); + pid = fork(); + if (!pid) { + char token; + + close(gate[1]); + if (read(gate[0], &token, 1) != 1 || locked_kb() != 0) + _exit(2); + for (unsigned long i = 0; i < 4 * ps; i++) + if (q[i] != 0x8a) + _exit(3); + _exit(0); + } + close(gate[0]); + memset(q + ps, 0x9b, ps); + ok &= write(gate[1], "x", 1) == 1; + close(gate[1]); + ok &= wait_ok(pid) && locked_kb() == (long)(4 * kb); + ok &= !munmap(q, 4 * ps) && locked_kb() == 0; + ksft_test_result(ok, "%luK MAP_LOCKED, fork noninheritance and parent COW\n", kb); + ok = !mlockall(MCL_FUTURE | MCL_ONFAULT); + q = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + ok &= q != MAP_FAILED && locked_kb() == (long)kb; + if (q != MAP_FAILED) + *q = 0xaf; + ok &= !munlockall() && locked_kb() == 0; + if (q != MAP_FAILED) + munmap(q, ps); + ksft_test_result(ok, "%luK future on-fault locks and munlockall\n", kb); + pid = fork(); + if (!pid) { + struct rlimit limit = { ps, ps }; + + if (setrlimit(RLIMIT_MEMLOCK, &limit) || setuid(1000) || mlock(p, ps)) + _exit(4); + errno = 0; + if (mlock(p + ps, ps) != -1 || errno != ENOMEM || locked_kb() != (long)kb) + _exit(5); + _exit(0); + } + ksft_test_result(wait_ok(pid), "%luK unprivileged RLIMIT_MEMLOCK in user-page units\n", kb); + pid = fork(); + if (!pid) { + if (mlockall(MCL_CURRENT) || locked_kb() <= (long)kb || + munlockall() || locked_kb() != 0) + _exit(6); + _exit(0); + } + ksft_test_result(wait_ok(pid), "%luK lock/unlock current mappings\n", kb); + { + unsigned long huge = ((1UL << 31) + 1) * ps; + + q = mmap(NULL, huge, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + ok = q != MAP_FAILED; + if (q != MAP_FAILED) { + ok &= !mlock2(q, huge, MLOCK_ONFAULT) && locked_kb() == (long)(huge / 1024); + ok &= !munlock(q, huge) && locked_kb() == 0; + munmap(q, huge); + } + ksft_test_result(ok, "%luK deferred lock count beyond signed 32-bit leaves\n", kb); + } + ksft_test_result(remap_locked(ps), "%luK locked grow/shrink/move and partial unmap accounting\n", kb); + munmap(p, 32 * ps); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + + if (!pid) { + char size[32], host_size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + snprintf(host_size, sizeof(host_size), "%lu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(7); + execl("/init", "/init", "--test", size, host_size, NULL); + _exit(8); + } + ok &= wait_ok(pid); + } + printf("%s - mlock fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/page-contract.c b/tools/testing/selftests/arm64/user4k/page-contract.c new file mode 100644 index 0000000000000..dbfe35eec08c8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/page-contract.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Run unchanged in native 16K and explicitly opted-in 4K processes. + * Default expectation is 16K. --expect-4k MUST fail on an ordinary 16K mm; + * it is an assertion, not a launcher or an emulation of the missing ABI. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define X18_SENTINEL UINT64_C(0xfedcba9876543210) + +extern long x18_syscall(long nr, long arg0, long arg1, long arg2, uint64_t *seen); + +static volatile sig_atomic_t signal_seen; +static uint64_t signal_x18; + +static void x18_signal(int sig, siginfo_t *info, void *context) +{ + ucontext_t *uc = context; + + (void)sig; + (void)info; + signal_x18 = uc->uc_mcontext.regs[18]; + signal_seen = 1; + /* rt_sigreturn must restore the interrupted value, not this value. */ + /* Built with -ffixed-x18: the compiler never allocates this register. */ + asm volatile("mov x18, xzr"); +} + +static bool wait_for(pid_t child, int expected_signal) +{ + int status; + pid_t ret; + + if (child < 0) + return false; + do { + ret = waitpid(child, &status, 0); + } while (ret < 0 && errno == EINTR); + if (ret != child) + return false; + if (expected_signal) + return WIFSIGNALED(status) && WTERMSIG(status) == expected_signal; + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +static bool access_faults(volatile unsigned char *ptr, bool write) +{ + pid_t child = fork(); + + if (!child) { + if (write) + *ptr = 0x5a; + else + (void)*ptr; + _exit(1); + } + return wait_for(child, SIGSEGV); +} + +static void private_mappings(size_t page) +{ + unsigned char *p = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + bool ok; + pid_t child; + + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap: %s\n", strerror(errno)); + memset(p, 0x41, 4 * page); + if (page == 16384) { + errno = 0; + ok = mprotect(p + 4096, 4096, PROT_READ) == -1 && errno == EINVAL; + } else { + ok = mprotect(p + page, page, PROT_READ) == 0; + } + ksft_test_result(ok, "mapping alignment agrees with the selected ABI\n"); + ok = mprotect(p + page, page, PROT_READ) == 0 && + mprotect(p + 2 * page, page, PROT_NONE) == 0; + ksft_test_result(ok && access_faults(p + page, true) && + access_faults(p + 2 * page, false), + "read-only and inaccessible neighbors fault independently\n"); + p[0] = 0x10; + p[3 * page] = 0x30; + ksft_test_result(p[page] == 0x41 && p[0] == 0x10 && p[3 * page] == 0x30, + "neighbor permissions do not remove writable mappings\n"); + child = fork(); + if (!child) { + p[0] = 0x99; + _exit(p[3 * page] != 0x30); + } + ksft_test_result(wait_for(child, 0) && p[0] == 0x10, + "fork preserves private COW isolation\n"); + ok = munmap(p + page, page) == 0; + ksft_test_result(ok && access_faults(p + page, false) && + p[0] == 0x10 && p[3 * page] == 0x30, + "single-page unmap preserves surrounding mappings\n"); + munmap(p, 4 * page); +} + +static void shared_mappings(size_t page) +{ + int fd = memfd_create("page-contract", MFD_CLOEXEC); + unsigned char *a, *b; + bool ok = true; + pid_t child; + unsigned int i; + + if (fd < 0 || ftruncate(fd, 4 * page)) + ksft_exit_fail_msg("memfd: %s\n", strerror(errno)); + a = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + b = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (a == MAP_FAILED || b == MAP_FAILED) + ksft_exit_fail_msg("shared mmap: %s\n", strerror(errno)); + for (i = 0; i < 4; i++) { + a[i * page] = 0x50 + i; + ok &= b[i * page] == 0x50 + i; + } + ksft_test_result(ok, "shared aliases expose the same bytes\n"); + ok = mprotect(a + page, page, PROT_READ) == 0; + child = fork(); + if (!child) { + b[page] = 0x73; + _exit(a[page] != 0x73); + } + ok &= wait_for(child, 0); + ksft_test_result(ok && a[page] == 0x73 && + access_faults(a + page, true), + "shared writes cross fork while alias permissions stay local\n"); + munmap(a, 4 * page); + munmap(b, 4 * page); + close(fd); +} + +static void x18_contract(void) +{ + struct sigaction sa = { .sa_sigaction = x18_signal, .sa_flags = SA_SIGINFO }; + struct timespec pause = { .tv_nsec = 1000000 }; + struct rusage before, after; + uint64_t seen; + long ret; + bool ok; + + ret = x18_syscall(SYS_getpid, 0, 0, 0, &seen); + ksft_test_result(ret == getpid() && seen == X18_SENTINEL, + "x18 survives a raw syscall\n"); + ok = getrusage(RUSAGE_SELF, &before) == 0; + ret = x18_syscall(SYS_nanosleep, (long)&pause, 0, 0, &seen); + ok &= getrusage(RUSAGE_SELF, &after) == 0; + ksft_test_result(ok && ret == 0 && seen == X18_SENTINEL && + after.ru_nvcsw > before.ru_nvcsw, + "x18 survives a measured voluntary context switch\n"); + sigemptyset(&sa.sa_mask); + ok = sigaction(SIGUSR1, &sa, NULL) == 0; + if (ok) + ret = x18_syscall(SYS_tgkill, getpid(), syscall(SYS_gettid), SIGUSR1, &seen); + ksft_test_result(ok && ret == 0 && signal_seen && + signal_x18 == X18_SENTINEL && seen == X18_SENTINEL, + "signal frame and sigreturn preserve x18 after handler clobber\n"); +} + +int main(int argc, char **argv) +{ + const struct rlimit no_core = { 0, 0 }; + size_t expected = 16384; + int result; + + if (argc == 2 && !strcmp(argv[1], "--expect-4k")) { + expected = 4096; + } else if (argc != 1) { + fprintf(stderr, "usage: %s [--expect-4k]\n", argv[0]); + return KSFT_FAIL; + } + ksft_print_header(); + ksft_set_plan(11); + if (sysconf(_SC_PAGESIZE) != (long)expected || getauxval(AT_PAGESZ) != expected) + ksft_exit_fail_msg("expected %zu-byte ABI; sysconf=%ld AT_PAGESZ=%lu\n", + expected, sysconf(_SC_PAGESIZE), getauxval(AT_PAGESZ)); + ksft_test_result_pass("process reports the expected %zu-byte page ABI\n", expected); + if (setrlimit(RLIMIT_CORE, &no_core)) + ksft_exit_fail_msg("disable child core dumps: %s\n", strerror(errno)); + private_mappings(expected); + shared_mappings(expected); + x18_contract(); + ksft_print_cnts(); + result = ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS; + /* This static executable can also be /init in a disposable test VM. */ + if (getpid() == 1) { + fflush(NULL); + reboot(RB_POWER_OFF); + } + return result; +} diff --git a/tools/testing/selftests/arm64/user4k/proc-memory.c b/tools/testing/selftests/arm64/user4k/proc-memory.c new file mode 100644 index 0000000000000..ce20a2170d433 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-memory.c @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +struct stats { + unsigned long size, rss, pss, private_clean, private_dirty, shared_clean, shared_dirty; + unsigned long anonymous, kernel_page, mmu_page; +}; + +static bool get_stats(pid_t pid, uintptr_t address, struct stats *s) +{ + char path[64], line[512]; + unsigned long start, end, value; + unsigned int fields = 0; + char key[64]; + bool found = false; + FILE *f; + + memset(s, 0, sizeof(*s)); + snprintf(path, sizeof(path), "/proc/%d/smaps", pid); + f = fopen(path, "re"); + if (!f) + return false; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + if (found) + break; + found = start == address; + continue; + } + if (!found) + continue; + if (sscanf(line, "%63[^:]: %lu", key, &value) != 2) + continue; + if (!strcmp(key, "Size")) { + s->size = value; + fields |= 1U << 0; + } + else if (!strcmp(key, "Rss")) { + s->rss = value; + fields |= 1U << 1; + } + else if (!strcmp(key, "Pss")) { + s->pss = value; + fields |= 1U << 2; + } + else if (!strcmp(key, "Private_Clean")) { + s->private_clean = value; + fields |= 1U << 3; + } + else if (!strcmp(key, "Private_Dirty")) { + s->private_dirty = value; + fields |= 1U << 4; + } + else if (!strcmp(key, "Shared_Clean")) { + s->shared_clean = value; + fields |= 1U << 5; + } + else if (!strcmp(key, "Shared_Dirty")) { + s->shared_dirty = value; + fields |= 1U << 6; + } + else if (!strcmp(key, "Anonymous")) { + s->anonymous = value; + fields |= 1U << 7; + } + else if (!strcmp(key, "KernelPageSize")) { + s->kernel_page = value; + fields |= 1U << 8; + } + else if (!strcmp(key, "MMUPageSize")) { + s->mmu_page = value; + fields |= 1U << 9; + } + } + fclose(f); + return found && fields == (1U << 10) - 1; +} + +static bool check(pid_t pid, void *address, size_t page, size_t length, + unsigned long pss, unsigned long private, unsigned long shared, bool anon) +{ + struct stats s; + bool ok = get_stats(pid, (uintptr_t)address, &s); + + ok &= s.size == length / 1024 && s.rss == length / 1024 && s.pss == pss && + s.private_clean + s.private_dirty == private && + s.shared_clean + s.shared_dirty == shared && + s.anonymous == (anon ? length / 1024 : 0) && + s.kernel_page == page / 1024 && s.mmu_page == page / 1024; + if (!ok) + printf("# smaps size=%lu rss=%lu pss=%lu private=%lu shared=%lu anon=%lu pages=%lu/%lu\n", + s.size, s.rss, s.pss, s.private_clean + s.private_dirty, + s.shared_clean + s.shared_dirty, s.anonymous, s.kernel_page, s.mmu_page); + return ok; +} + +static int check_numa(void *address, size_t page) +{ + char line[1024]; + unsigned long start, anonymous = 0, pages = 0, granule = 0; + bool found = false; + FILE *f = fopen("/proc/self/numa_maps", "re"); + + if (!f) + return errno == ENOENT ? -1 : 0; + while (fgets(line, sizeof(line), f)) { + char *save, *token; + + if (sscanf(line, "%lx", &start) != 1 || start != (uintptr_t)address) + continue; + found = true; + for (token = strtok_r(line, "\n ", &save); token; + token = strtok_r(NULL, "\n ", &save)) { + unsigned int node; + unsigned long count; + + if (sscanf(token, "anon=%lu", &count) == 1) + anonymous = count; + else if (sscanf(token, "kernelpagesize_kB=%lu", &count) == 1) + granule = count; + else if (sscanf(token, "N%u=%lu", &node, &count) == 2) + pages += count; + } + break; + } + fclose(f); + if (!found || anonymous != 4 || pages != 4 || granule != page / 1024) + printf("# numa found=%d anon=%lu pages=%lu granule=%lu\n", + found, anonymous, pages, granule); + return found && anonymous == 4 && pages == 4 && granule == page / 1024; +} + +static bool reap(pid_t child) +{ + int status; + + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); +} + +static bool resident_rewrite_reuses(unsigned char *p, size_t page) +{ + uint64_t before[4], after[4]; + int fd = open("/proc/self/pagemap", O_RDONLY); + unsigned int i; + bool ok = fd >= 0; + + if (ok) + ok = pread(fd, before, sizeof(before), (uintptr_t)p / page * 8) == sizeof(before); + for (i = 0; i < 4; i++) + p[i * page] ^= 0x21; + if (ok) + ok = pread(fd, after, sizeof(after), (uintptr_t)p / page * 8) == sizeof(after); + for (i = 0; ok && i < 4; i++) { + const uint64_t frame = (1ULL << 55) - 1; + + ok &= (before[i] & frame) && (before[i] & frame) == (after[i] & frame) && + (before[i] & (1ULL << 63)) && (after[i] & (1ULL << 63)); + } + if (fd >= 0) + close(fd); + return ok; +} + +static void anonymous_sharing(size_t page) +{ + unsigned char *base, *p; + int ready[2], go[2]; + pid_t child; + char byte; + bool ok; + + base = mmap(NULL, 6 * page, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (base == MAP_FAILED || mprotect(base + page, 4 * page, PROT_READ | PROT_WRITE) || + pipe(ready) || pipe(go)) + ksft_exit_fail_msg("anonymous fixture setup\n"); + p = base + page; + memset(p, 0x57, 4 * page); + ksft_test_result(check(getpid(), p, page, 4 * page, 4 * page / 1024, + 4 * page / 1024, 0, true), "dense anonymous leaves have exact RSS/PSS/private bytes\n"); + { + int numa = check_numa(p, page); + + if (numa < 0) + ksft_test_result_skip("NUMA statistics require CONFIG_NUMA\n"); + else + ksft_test_result(numa, "NUMA statistics count target leaves and report their granule\n"); + } + child = fork(); + if (!child) { + close(ready[0]); + close(go[1]); + if (write(ready[1], "a", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(1); + p[page] = 0x96; + if (write(ready[1], "b", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(2); + _exit(0); + } + close(ready[1]); + close(go[0]); + ok = child > 0 && read(ready[0], &byte, 1) == 1 && + check(getpid(), p, page, 4 * page, 2 * page / 1024, 0, 4 * page / 1024, true); + ksft_test_result(ok, "fork divides each anonymous leaf's PSS by its own mapcount\n"); + ok = write(go[1], "x", 1) == 1 && read(ready[0], &byte, 1) == 1 && + check(getpid(), p, page, 4 * page, (5 * page) / 2048, + page / 1024, 3 * page / 1024, true) && p[page] == 0x57; + ksft_test_result(ok, "one child COW changes only that leaf's private/shared accounting\n"); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ksft_test_result(reap(child) && check(getpid(), p, page, 4 * page, + 4 * page / 1024, 4 * page / 1024, 0, true), + "child exit restores full anonymous PSS\n"); + ksft_test_result(resident_rewrite_reuses(p, page), + "resident writes reuse physical backing after the child exits\n"); + munmap(base, 6 * page); +} + +static int worker(int fd, int output, int input, size_t expected) +{ + size_t page = getauxval(AT_PAGESZ); + unsigned char *p; + uintptr_t addr; + char byte; + + if (page != expected) + return 1; + p = mmap(NULL, page, PROT_READ, MAP_SHARED, fd, page == 4096 ? 4096 : 0); + if (p == MAP_FAILED || p[0] != 0x78) + return 2; + addr = (uintptr_t)p; + if (write(output, &addr, sizeof(addr)) != sizeof(addr) || read(input, &byte, 1) != 1) + return 3; + return 0; +} + +static void mixed_file(size_t page) +{ + size_t target = page == 4096 ? 16384 : 4096; + int ready[2], go[2], fd = memfd_create("mixed-pss", 0); + unsigned char data[16384], *p; + uintptr_t remote; + pid_t child; + bool ok; + + memset(data, 0x78, sizeof(data)); + if (fd < 0 || write(fd, data, sizeof(data)) != sizeof(data) || pipe(ready) || pipe(go)) + ksft_exit_fail_msg("mixed file fixture setup\n"); + p = mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, page == 4096 ? 4096 : 0); + if (p == MAP_FAILED || p[0] != 0x78) + ksft_exit_fail_msg("mixed file mmap\n"); + child = fork(); + if (!child) { + char farg[16], out[16], in[16], expected[16]; + + close(ready[0]); + close(go[1]); + snprintf(farg, sizeof(farg), "%d", fd); + snprintf(out, sizeof(out), "%d", ready[1]); + snprintf(in, sizeof(in), "%d", go[0]); + snprintf(expected, sizeof(expected), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, 0UL, 0UL, 0UL)) + _exit(4); + execl("/proc/self/exe", "/proc/self/exe", "--worker", farg, out, in, expected, NULL); + _exit(5); + } + close(ready[1]); + close(go[0]); + ok = child > 0 && read(ready[0], &remote, sizeof(remote)) == sizeof(remote); + ok = ok && check(getpid(), p, page, page, page == 4096 ? 2 : 14, + page == 4096 ? 0 : 12, 4, false) && + check(child, (void *)remote, target, target, target == 4096 ? 2 : 14, + target == 4096 ? 0 : 12, 4, false); + ksft_test_result(ok, "mixed file aliases divide only their overlapping physical quarter\n"); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ksft_test_result(reap(child) && check(getpid(), p, page, page, page / 1024, + page / 1024, 0, false), "mixed alias exit restores private file accounting\n"); + munmap(p, page); + close(fd); +} + +int main(int argc, char **argv) +{ + size_t page = getauxval(AT_PAGESZ); + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), strtoul(argv[5], NULL, 10)); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + ksft_print_header(); + ksft_set_plan(8); + anonymous_sharing(page); + mixed_file(page); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/proc-pages.c b/tools/testing/selftests/arm64/user4k/proc-pages.c new file mode 100644 index 0000000000000..78ba59936ccc6 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-pages.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define PRESENT (1ULL << 63) +#define EXCLUSIVE (1ULL << 56) +#define GUARD (1ULL << 58) +#define FRAME ((1ULL << 55) - 1) +#define NPAGE 5 + +struct ready { + uintptr_t address; + size_t page; + uint64_t entries[NPAGE]; + uint64_t shared_entry; +}; + +static int open_proc(pid_t pid, const char *name, int flags) +{ + char path[80]; + + snprintf(path, sizeof(path), "/proc/%d/%s", pid, name); + return open(path, flags); +} + +static bool entries(int fd, uintptr_t address, size_t page, uint64_t *values, int nr) +{ + return pread(fd, values, nr * sizeof(*values), address / page * sizeof(*values)) == + nr * (ssize_t)sizeof(*values); +} + +static unsigned char *mapping(size_t page) +{ + const size_t boundary = 32 * 1024 * 1024; + size_t reserve = boundary + 8 * page, head, tail; + unsigned char *base, *p; + + base = mmap(NULL, reserve, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (base == MAP_FAILED) + return NULL; + /* Cross both native 32 MiB and alternative 2 MiB PTE-table boundaries. */ + p = (void *)(((uintptr_t)base + 2 * page + boundary - 1) & ~(boundary - 1)); + p -= 2 * page; + head = p - base; + tail = reserve - head - NPAGE * page; + if ((head && munmap(base, head)) || (tail && munmap(p + NPAGE * page, tail)) || + mprotect(p, NPAGE * page, PROT_READ | PROT_WRITE)) + return NULL; + memset(p, 0x67, NPAGE * page); + if (munmap(p + 2 * page, page) || + madvise(p + 3 * page, page, MADV_GUARD_INSTALL) || + mprotect(p + 4 * page, page, PROT_NONE)) + return NULL; + return p; +} + +static bool scan(int fd, uintptr_t addr, size_t page) +{ + struct page_region vec[8]; + struct pm_scan_arg arg = { + .size = sizeof(arg), .start = addr, .end = addr + NPAGE * page, + .vec = (uintptr_t)vec, .vec_len = 8, + .return_mask = PAGE_IS_PRESENT | PAGE_IS_GUARD, + }; + int nr = ioctl(fd, PAGEMAP_SCAN, &arg); + bool ok = nr == 3 && arg.walk_end == arg.end; + if (!ok) + printf("# scan initial nr=%d errno=%d end_delta=%lld\n", nr, errno, + (long long)(arg.walk_end - arg.end)); + + /* The unmapped middle page produces no VMA result. */ + if (nr == 3) + ok &= vec[0].start == addr && vec[0].end == addr + 2 * page && + vec[0].categories == PAGE_IS_PRESENT && + vec[1].start == addr + 3 * page && vec[1].end == addr + 4 * page && + vec[1].categories == PAGE_IS_GUARD && + vec[2].start == addr + 4 * page && vec[2].end == arg.end && + vec[2].categories == PAGE_IS_PRESENT; + arg.max_pages = 1; + arg.walk_end = 0; + nr = ioctl(fd, PAGEMAP_SCAN, &arg); + ok &= nr == 1 && vec[0].start == addr && vec[0].end == addr + page && + arg.walk_end == addr + page; + arg.start = addr + 1; + errno = 0; + ok &= ioctl(fd, PAGEMAP_SCAN, &arg) == -1 && errno == EINVAL; + return ok; +} + +static int remote_worker(int output, int input, size_t expected, int shared_fd) +{ + struct ready ready = { .page = getauxval(AT_PAGESZ) }; + unsigned char *p; + int fd = open_proc(getpid(), "pagemap", O_RDONLY); + char byte; + unsigned char *shared = mmap(NULL, 16384, PROT_READ, MAP_SHARED, shared_fd, 0); + size_t offset = ready.page == 4096 ? 4096 : 0; + + if (shared == MAP_FAILED || shared[offset] != 0x83 || + !entries(fd, (uintptr_t)shared + offset, ready.page, &ready.shared_entry, 1)) + return 2; + /* Allocate aliases first so mmap cannot reuse our deliberate hole. */ + p = mapping(ready.page); + ready.address = (uintptr_t)p; + if (ready.page != expected || !p || fd < 0 || + !entries(fd, ready.address, ready.page, ready.entries, NPAGE) || + write(output, &ready, sizeof(ready)) != sizeof(ready) || read(input, &byte, 1) != 1) + return 1; + return 0; +} + +static bool remote_read(size_t own_page) +{ + int ready_pipe[2], done_pipe[2], status, fd, shared_fd, self_fd; + unsigned char *shared; + uint64_t self_entry; + size_t offset = own_page == 4096 ? 4096 : 0; + size_t target = own_page == 4096 ? 16384 : 4096; + struct ready ready; + uint64_t actual[NPAGE]; + pid_t child; + bool ok; + + shared_fd = memfd_create("mixed-pagemap", 0); + if (shared_fd < 0 || ftruncate(shared_fd, 16384)) + return false; + shared = mmap(NULL, 16384, PROT_READ | PROT_WRITE, MAP_SHARED, shared_fd, 0); + self_fd = open_proc(getpid(), "pagemap", O_RDONLY); + if (shared == MAP_FAILED || self_fd < 0) + return false; + memset(shared, 0x83, 16384); + if (!entries(self_fd, (uintptr_t)shared + offset, own_page, &self_entry, 1)) + return false; + close(self_fd); + if (pipe(ready_pipe) || pipe(done_pipe)) + return false; + child = fork(); + if (!child) { + char output[16], input[16], expected[16], shared_arg[16]; + + close(ready_pipe[0]); + close(done_pipe[1]); + snprintf(output, sizeof(output), "%d", ready_pipe[1]); + snprintf(input, sizeof(input), "%d", done_pipe[0]); + snprintf(expected, sizeof(expected), "%zu", target); + snprintf(shared_arg, sizeof(shared_arg), "%d", shared_fd); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, 0UL, 0UL, 0UL)) + _exit(2); + execl("/proc/self/exe", "/proc/self/exe", "--remote", output, input, expected, shared_arg, NULL); + _exit(3); + } + close(ready_pipe[1]); + close(done_pipe[0]); + ok = child > 0 && read(ready_pipe[0], &ready, sizeof(ready)) == sizeof(ready); + fd = open_proc(child, "pagemap", O_RDONLY); + ok = ok && ready.page == target && fd >= 0 && + entries(fd, ready.address, target, actual, NPAGE) && + !memcmp(actual, ready.entries, sizeof(actual)) && scan(fd, ready.address, target); + if (!ok) + printf("# remote geometry/read failure: target=%zu reported=%zu fd=%d errno=%d\n", + target, ready.page, fd, errno); + if (ok) { + size_t target_offset = target == 4096 ? 4096 : 0; + + /* Compare physical byte addresses, not differently sized PFNs. */ + ok &= (self_entry & FRAME) && (ready.shared_entry & FRAME) && + (self_entry & FRAME) * own_page - offset == + (ready.shared_entry & FRAME) * target - target_offset; + if (!ok) + printf("# shared PFN mismatch: own_nonzero=%d target_nonzero=%d byte_delta=%lld\n", + !!(self_entry & FRAME), !!(ready.shared_entry & FRAME), + (long long)((self_entry & FRAME) * own_page - offset - + (ready.shared_entry & FRAME) * target + target_offset)); + } + if (fd >= 0) + close(fd); + write(done_pipe[1], "x", 1); + close(done_pipe[1]); + close(ready_pipe[0]); + ok &= child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + munmap(shared, 16384); + close(shared_fd); + return ok; +} + +int main(int argc, char **argv) +{ + size_t page = getauxval(AT_PAGESZ); + unsigned char *p; + uint64_t values[NPAGE], again[NPAGE]; + int fd, clear, i; + bool ok; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--remote")) + return remote_worker(atoi(argv[2]), atoi(argv[3]), strtoul(argv[4], NULL, 10), atoi(argv[5])); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + ksft_print_header(); + ksft_set_plan(6); + p = mapping(page); + fd = open_proc(getpid(), "pagemap", O_RDONLY); + if (!p || fd < 0) + ksft_exit_fail_msg("proc mapping setup: %s\n", strerror(errno)); + ok = entries(fd, (uintptr_t)p, page, values, NPAGE); + ok &= (values[0] & PRESENT) && (values[1] & PRESENT) && !values[2] && + (values[3] & GUARD) && !(values[3] & PRESENT) && (values[4] & PRESENT); + ok &= (values[0] & EXCLUSIVE) && (values[1] & EXCLUSIVE) && (values[4] & EXCLUSIVE); + ksft_test_result(ok, "pagemap reports each target leaf, hole, guard and PROT_NONE entry\n"); + ok = true; + for (i = 0; i < NPAGE; i++) { + uint64_t value; + + ok &= entries(fd, (uintptr_t)p + i * page, page, &value, 1) && value == values[i]; + } + errno = 0; + ok &= pread(fd, again, 7, 0) == -1 && errno == EINVAL; + ksft_test_result(ok, "pagemap per-entry offsets agree with batch reads and reject short entries\n"); + ksft_test_result(scan(fd, (uintptr_t)p, page), + "PAGEMAP_SCAN ranges, max_pages and alignment use target granules\n"); + clear = open_proc(getpid(), "clear_refs", O_WRONLY); + ok = clear >= 0; + for (i = 1; ok && i <= 3; i++) { + char operation = '0' + i; + + ok &= write(clear, &operation, 1) == 1; + } + if (clear >= 0) + close(clear); + ok &= entries(fd, (uintptr_t)p, page, again, NPAGE) && + !memcmp(values, again, sizeof(values)) && p[0] == 0x67 && p[page] == 0x67; + ksft_test_result(ok, "clear_refs traverses short and split VMAs without changing mappings or bytes\n"); + ksft_test_result(remote_read(page), "opposite-ABI reader uses target pagemap and scan geometry\n"); + { + pid_t child = fork(); + int status; + + if (!child) { + uint64_t value = 0; + int unprivileged; + + if (setuid(1000) || prctl(PR_SET_DUMPABLE, 1)) + _exit(2); + unprivileged = open_proc(getpid(), "pagemap", O_RDONLY); + ok = unprivileged >= 0 && + entries(unprivileged, (uintptr_t)p, page, &value, 1) && + (value & PRESENT) && !(value & FRAME); + if (!ok) + printf("# unprivileged fd=%d errno=%d flags=%#llx\n", unprivileged, + errno, (unsigned long long)(value & ~FRAME)); + _exit(!ok); + } + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + ksft_test_result(ok, "unprivileged pagemap reads retain flags and hide physical addresses\n"); + } + close(fd); + munmap(p, NPAGE * page); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/reclaim-advice.c b/tools/testing/selftests/arm64/user4k/reclaim-advice.c new file mode 100644 index 0000000000000..5fd4b914f945c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/reclaim-advice.c @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "re"); + char key[128]; + unsigned long value, result = 0; + + if (!f) + return 0; + while (fscanf(f, "%127s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) + result = value; + fclose(f); + return result; +} + +static unsigned long swapped_kb(void) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256]; + unsigned long result = 0; + + if (!f) + return 0; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmSwap: %lu kB", &result) == 1) + break; + fclose(f); + return result; +} + +static int advise(void *addr, size_t length, int operation, const char *name) +{ + int ret = madvise(addr, length, operation); + + if (ret) + printf("# %s failed: errno=%d (%s)\n", name, errno, strerror(errno)); + return ret; +} + +static unsigned char pattern(size_t i, unsigned int round) +{ + return (i ^ (i >> 12) ^ (i >> 20) ^ (round * 0x55)) & 255; +} + +static size_t swapped_pages(unsigned char *p, size_t length, size_t page) +{ + uint64_t *values = calloc(length / page, sizeof(*values)); + int fd = open("/proc/self/pagemap", O_RDONLY); + size_t i, count = length / page, swapped = 0; + + if (!values || fd < 0 || pread(fd, values, count * sizeof(*values), + (uintptr_t)p / page * sizeof(*values)) != (ssize_t)(count * sizeof(*values))) { + swapped = SIZE_MAX; + } else { + for (i = 0; i < count; i++) + swapped += !!(values[i] & (1ULL << 62)) && !(values[i] & (1ULL << 63)); + } + if (fd >= 0) + close(fd); + free(values); + return swapped; +} + +static void diagnose_resident(unsigned char *p, size_t length, size_t page) +{ + int pm = open("/proc/self/pagemap", O_RDONLY); + int flags = open("/proc/kpageflags", O_RDONLY); + int count = open("/proc/kpagecount", O_RDONLY); + + for (size_t i = 0; i < length / page; i++) { + uint64_t entry = 0, state = 0, maps = 0; + unsigned long native_pfn; + + if (pread(pm, &entry, 8, ((uintptr_t)p / page + i) * 8) != 8 || + !(entry & (1ULL << 63))) + continue; + native_pfn = (entry & ((1ULL << 55) - 1)) * page / 16384; + if (pread(flags, &state, 8, native_pfn * 8) != 8 || + pread(count, &maps, 8, native_pfn * 8) != 8) + printf("# resident metadata read failed: errno=%d\n", errno); + printf("# resident leaf=%zu native_pfn=%#lx flags=%#llx maps=%llu\n", + i, native_pfn, (unsigned long long)state, (unsigned long long)maps); + } + close(pm); + close(flags); + close(count); +} + +/* PAGEOUT drains only the calling CPU's pending LRU additions. */ +static bool drain_pageout_cpus(unsigned char *p, size_t length) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(cpu, &one); + if (sched_setaffinity(0, sizeof(one), &one)) { + ok = false; + break; + } + ok &= advise(p, length, MADV_COLD, "CPU drain COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, "CPU drain PAGEOUT") == 0; + } + ok &= sched_setaffinity(0, sizeof(allowed), &allowed) == 0; + return ok; +} + +static bool swapped_pagemap(unsigned char *p, size_t length, size_t page) +{ + uint64_t *values = calloc(length / page, sizeof(*values)); + int fd = open("/proc/self/pagemap", O_RDONLY); + size_t i, count = length / page; + bool ok = values && fd >= 0; + + if (ok) + ok = pread(fd, values, count * sizeof(*values), + (uintptr_t)p / page * sizeof(*values)) == + (ssize_t)(count * sizeof(*values)); + for (i = 0; ok && i < count; i++) { + uint64_t frame = values[i] & ((1ULL << 55) - 1); + + ok &= !!(values[i] & (1ULL << 62)) && !(values[i] & (1ULL << 63)); + if (page == 4096) { + ok &= ((frame >> 5) & 3) == (((uintptr_t)p / page + i) & 3); + if (i && (((uintptr_t)p / page + i) & 3)) + ok &= (values[i] & ((1ULL << 55) - 1)) == + (values[i - 1] & ((1ULL << 55) - 1)) + (1U << 5); + } + } + if (fd >= 0) + close(fd); + free(values); + printf("%s - %zuK pagemap reports swapped leaves and exact quarter offsets\n", + ok ? "ok" : "not ok", page / 1024); + return ok; +} + +static bool check_swap_pss(pid_t pid, void *p, size_t length, + unsigned long expected_swap, unsigned long expected_pss) +{ + char path[64], line[512]; + unsigned long start, end, swap = 0, pss = 0; + unsigned int fields = 0; + bool found = false, ok; + FILE *f; + + snprintf(path, sizeof(path), "/proc/%d/smaps", pid); + f = fopen(path, "re"); + if (!f) + return false; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + if (found) + break; + found = start == (uintptr_t)p && end == (uintptr_t)p + length; + continue; + } + if (!found) + continue; + if (sscanf(line, "Swap: %lu", &swap) == 1) + fields |= 1; + else if (sscanf(line, "SwapPss: %lu", &pss) == 1) + fields |= 2; + } + fclose(f); + ok = found && fields == 3 && swap == expected_swap && pss == expected_pss; + if (!ok) + printf("# swap stats found=%d fields=%u Swap=%lu/%lu SwapPss=%lu/%lu\n", + found, fields, swap, expected_swap, pss, expected_pss); + return ok; +} + +static bool swapped_accounting(unsigned char *p, size_t length, size_t page, bool forked) +{ + int ready[2], go[2], status; + pid_t child; + char byte; + bool ok = check_swap_pss(getpid(), p, length, length / 1024, length / 1024); + + if (!forked || !ok) + goto out; + if (pipe(ready) || pipe(go)) + return false; + child = fork(); + if (child < 0) + return false; + if (!child) { + close(ready[0]); + close(go[1]); + if (read(go[0], &byte, 1) != 1) + _exit(1); + p[page] ^= 0x37; + if (write(ready[1], "x", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(2); + _exit(0); + } + close(ready[1]); + close(go[0]); + ok &= check_swap_pss(getpid(), p, length, length / 1024, length / 2048); + ok &= check_swap_pss(child, p, length, length / 1024, length / 2048); + ok &= write(go[1], "x", 1) == 1 && read(ready[0], &byte, 1) == 1; + ok &= check_swap_pss(getpid(), p, length, length / 1024, (length + page) / 2048); + ok &= check_swap_pss(child, p, length, (length - page) / 1024, (length - page) / 2048); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ok &= waitpid(child, &status, 0) == child && WIFEXITED(status) && !WEXITSTATUS(status); + ok &= check_swap_pss(getpid(), p, length, length / 1024, length / 1024); +out: + printf("%s - %zuK SwapPss %s\n", ok ? "ok" : "not ok", page / 1024, + forked ? "fork, one-leaf swap-in/COW and child exit" : "exclusive quarters"); + return ok; +} + +int main(int argc, char **argv) +{ + const size_t length = 4 * 1024 * 1024; + size_t page = getauxval(AT_PAGESZ), count = length / page, i, j, absent; + unsigned char *p, *vec, *reservation; + unsigned long before, after, swapped_before, swapped_after; + unsigned int round, tries; + size_t bad_bytes, bad_residency; + bool ok = true; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + reservation = mmap(NULL, length + 2 * page, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reservation == MAP_FAILED) + return 3; + p = reservation + page; + vec = malloc(count); + if (!vec || mprotect(p, length, PROT_READ | PROT_WRITE)) + return 3; + for (round = 0; round < 4 && ok; round++) { + for (i = 0; i < length; i++) + p[i] = pattern(i, round); + before = swapouts(); + swapped_before = swapped_kb(); + ok = advise(p, length, MADV_COLD, "COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, "PAGEOUT") == 0; + after = before; + absent = 0; + for (tries = 0; ok && tries < 200; tries++) { + if (mincore(p, length, vec)) { + ok = false; + break; + } + absent = 0; + for (i = 0; i < count; i++) + absent += !(vec[i] & 1); + after = swapouts(); + if (after > before && swapped_pages(p, length, page) == count) + break; + if (tries == 32) { + printf("# checking pending remote LRU additions before CPU sweep\n"); + diagnose_resident(p, length, page); + ok &= drain_pageout_cpus(p, length); + printf("# after CPU sweep: swapped=%zu/%zu\n", + swapped_pages(p, length, page), count); + } + /* PAGEOUT may skip busy folios; establish the full-swap fixture. */ + usleep(10000); + ok &= advise(p, length, MADV_COLD, "retry COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, "retry PAGEOUT") == 0; + } + printf("# pageout setup attempts=%u\n", tries + 1); + if (tries == 200) + diagnose_resident(p, length, page); + ok &= tries < 200; + swapped_after = swapped_kb(); + /* mincore also counts warm swap-cache data as resident. */ + ok &= after > before && swapped_after > swapped_before; + ok &= swapped_pagemap(p, length, page); + ok &= swapped_accounting(p, length, page, round == 1); + if (!round) + ok &= advise(p + page, page, MADV_WILLNEED, "WILLNEED") == 0; + else { + if (round == 3) { + ok &= mlock(p, length) == 0; + ok &= swapped_pages(p, length, page) == 0; + } else { + ok &= advise(p, length, MADV_POPULATE_READ, "POPULATE_READ") == 0; + } + ok &= mincore(p, length, vec) == 0; + bad_residency = 0; + for (i = 0; i < count; i++) + bad_residency += !(vec[i] & 1); + printf("# after %s: absent=%zu/%zu\n", round == 3 ? "mlock" : "POPULATE_READ", + bad_residency, count); + ok &= bad_residency == 0; + } + bad_bytes = 0; + /* Nonsequential user-page order, every byte checked after swap. */ + for (i = 0; i < count; i++) { + size_t index = ((i * 61) & (count - 1)) * page; + + for (j = index; j < index + page; j++) + if (p[j] != pattern(j, round)) { + if (!bad_bytes) + printf("# first bad byte at %#zx: got=%#x expected=%#x\n", + j, p[j], pattern(j, round)); + bad_bytes++; + ok = false; + } + } + if (round == 3) + ok &= munlock(p, length) == 0; + printf("# bad bytes=%zu\n", bad_bytes); + printf("%s - %zuK EL0 pageout round %u: pswpout=%lu->%lu VmSwap=%lu->%lu kB absent=%zu/%zu %s\n", + ok ? "ok" : "not ok", page / 1024, round, before, after, + swapped_before, swapped_after, absent, count, round == 3 ? "mlock/readback" : round ? "prefault/readback" : "willneed/readback"); + } + munmap(reservation, length + 2 * page); + free(vec); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/runtime-app-init.c b/tools/testing/selftests/arm64/user4k/runtime-app-init.c new file mode 100644 index 0000000000000..597494ea150c4 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/runtime-app-init.c @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +int main(int argc, char **argv) +{ + if(argc>=4&&!strcmp(argv[1],"--exec")) { + if(prctl(SET_EXEC_PAGE_SIZE,strtoul(argv[2],NULL,10),0UL,0UL,0UL))return 8; + execv(argv[3],argv+3);perror("exec");return 9; + } + if (getpid()!=1) return 2; + setvbuf(stdout,NULL,_IONBF,0); + int failed=0;pid_t children[3];unsigned long sizes[]={getauxval(AT_PAGESZ),4096,16384};unsigned count=sizes[0]>16384?3:2; + if(mount("proc","/proc","proc",0,NULL)||mount("devtmpfs","/dev","devtmpfs",0,NULL)) {perror("mount");failed=1;goto done;} + mkdir("/dev/shm",0777);mount("tmpfs","/dev/shm","tmpfs",0,NULL); + for(unsigned i=0;i0&&waitpid(children[i],&status,0)==children[i]&&WIFEXITED(status)&&!WEXITSTATUS(status);printf("%s - distro Python ABI %luK status=%#x\n",ok?"ok":"not ok",sizes[i]/1024,status);failed|=!ok;} + done: + if(failed){FILE *f=fopen("/trace4k","r");if(f){char line[2048];while(fgets(line,sizeof(line),f))fputs(line,stdout);fclose(f);}} + printf("GRANULE DISTRO RUNTIME %s\n",failed?"FAIL":"PASS");sync();reboot(RB_POWER_OFF);for(;;)pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/runtime-app.py b/tools/testing/selftests/arm64/user4k/runtime-app.py new file mode 100755 index 0000000000000..47fac03300efc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/runtime-app.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: GPL-2.0-only +import os,sys,mmap,ctypes,threading,concurrent.futures,tempfile,subprocess,signal,zlib,gzip,lzma,hashlib,socket,select,json,sqlite3 +ps=int(sys.argv[1]);assert os.sysconf('SC_PAGE_SIZE')==ps==mmap.PAGESIZE,(ps,mmap.PAGESIZE) +libc=ctypes.CDLL(None,use_errno=True);libc.prctl.argtypes=[ctypes.c_int,ctypes.c_ulong,ctypes.c_ulong,ctypes.c_ulong,ctypes.c_ulong] +def passed(name):print(f'ok - distro Python {ps//1024}K {name}',flush=True) +passed('dynamic loader and page-size contract') +with mmap.mmap(-1, 2*ps, flags=0x08|mmap.MAP_ANONYMOUS) as droppable: + assert droppable[:]==bytes(2*ps) + droppable[:]=bytes([0x45])*(2*ps) + child=os.fork() + if child==0:os._exit(0 if droppable[:]==bytes(2*ps) else 9) + assert os.waitpid(child,0)[1]==0 + droppable.madvise(mmap.MADV_DONTNEED) + assert droppable[:]==bytes(2*ps) +libc.arc4random_buf.argtypes=[ctypes.c_void_p,ctypes.c_size_t] +random_buffer=ctypes.create_string_buffer(32) +random_samples=set() +for _ in range(128): + libc.arc4random_buf(random_buffer,len(random_buffer));random_samples.add(random_buffer.raw) +assert len(random_samples)==128 +passed('droppable write, fork wipe, discard and glibc random state') + +blob=bytes(range(256))*4096 +with tempfile.TemporaryDirectory() as tmp: + path=tmp+'/mapped' + with open(path,'w+b') as f: + f.truncate(8*ps) + with mmap.mmap(f.fileno(),2*ps,offset=ps) as m: + m[:]=bytes([0x53])*(2*ps);m.flush() + assert os.pread(f.fileno(),2*ps,ps)==m[:] + m.resize(3*ps);m[2*ps:]=bytes([0x72])*ps;m.flush() + assert os.pread(f.fileno(),ps,3*ps)==bytes([0x72])*ps + passed('file mmap offset, flush and resize') + with mmap.mmap(-1,4*ps,flags=mmap.MAP_PRIVATE|mmap.MAP_ANONYMOUS) as m: + m[:]=bytes([0x39])*(4*ps) + c=os.fork() + if c==0: + m[ps:2*ps]=bytes([0x91])*ps + os._exit(0 if m[:ps]==bytes([0x39])*ps else 5) + assert os.waitpid(c,0)[1]==0 and m[:]==bytes([0x39])*(4*ps) + address=ctypes.addressof(ctypes.c_char.from_buffer(m)) + libc.mprotect.argtypes=[ctypes.c_void_p,ctypes.c_size_t,ctypes.c_int] + assert libc.mprotect(address+ps,ps,0)==0 + c=os.fork() + if c==0: + m[ps]=0 + os._exit(6) + status=os.waitpid(c,0)[1];assert os.WIFSIGNALED(status) and os.WTERMSIG(status)==signal.SIGSEGV + assert m[0]==0x39 and m[2*ps]==0x39 + assert libc.mprotect(address+ps,ps,mmap.PROT_READ|mmap.PROT_WRITE)==0 + passed('fork COW and independent protection fault') + def compressed(i): + d=blob+str(i).encode() + for compress,decompress in [(zlib.compress,zlib.decompress),(gzip.compress,gzip.decompress),(lambda d:lzma.compress(d,preset=1),lzma.decompress)]:assert decompress(compress(d))==d + return hashlib.sha256(d).hexdigest() + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + results=list(pool.map(compressed,range(8))) + assert len(set(results))==8 + passed('threaded native extension allocation, compression and hashing') + db=sqlite3.connect(tmp+'/db') + db.execute('pragma journal_mode=wal');db.execute('pragma mmap_size=1048576') + db.execute('create table entries (n integer primary key, data blob)') + db.executemany('insert into entries values (?,?)',[(i,blob[:4096]) for i in range(128)]);db.commit() + assert db.execute('select count(*) from entries').fetchone()[0]==128 + assert all(row[0]==blob[:4096] for row in db.execute('select data from entries'));db.close() + passed('SQLite WAL and mapped database reads') + left,right=socket.socketpair() + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + task=pool.submit(lambda:(right.sendall(blob),right.shutdown(socket.SHUT_WR))) + out=bytearray() + while True: + assert select.select([left],[],[],5)[0] + part=left.recv(997) + if not part:break + out.extend(part) + task.result();assert bytes(out)==blob + left.close();right.close();passed('socket polling and fragmented transfers') + fd=os.memfd_create('cross-abi',0);os.ftruncate(fd,131072);os.pwrite(fd,b'parent',ps) + # Execute the child in the other ABI, retaining the same shared file bytes. + target=16384 if ps==4096 else 4096 + def select_abi(): + if libc.prctl(0x41555001,target,0,0,0):os._exit(7) + code="import os,mmap,sys;fd=int(sys.argv[1]);off=int(sys.argv[2]);want=int(sys.argv[3]);assert os.sysconf('SC_PAGE_SIZE')==want;m=mmap.mmap(fd,131072);assert m[off:off+6]==b'parent';m[off:off+5]=b'child';m.close()" + subprocess.run([sys.executable,'-I','-c',code,str(fd),str(ps),str(target)],pass_fds=(fd,),preexec_fn=select_abi,check=True) + assert os.pread(fd,5,ps)==b'child';os.close(fd);passed('cross-ABI dynamic exec and memfd sharing') + command="set -e; a=(); for ((i=0;i<2048;i++)); do a+=(\"value-$i\"); done; [[ ${#a[@]} == 2048 ]]; [[ ${a[2047]} == value-2047 ]]; printf 'bash-ok\\n'" + def same_abi(): + if libc.prctl(0x41555001,ps,0,0,0):os._exit(8) + result=subprocess.run(['/bin/bash','--noprofile','--norc','-c',command],preexec_fn=same_abi,stdout=subprocess.PIPE,check=True,text=True) + assert result.stdout=='bash-ok\n';passed('distro Bash dynamic exec, allocation and pipes') +print(f'DISTRO PYTHON {ps} PASS',flush=True) diff --git a/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c b/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c new file mode 100644 index 0000000000000..60497589a8e2a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#ifndef MADV_COLLAPSE +#define MADV_COLLAPSE 25 +#endif +#define CHECK(x) do { if (!(x)) { printf("not ok - collapse line=%d errno=%d\n", __LINE__, errno); exit(1); } } while (0) +struct command { uint64_t op, offset, len, value; }; +struct ready { uint64_t base, head; }; +static uint64_t entry(const char *path, uint64_t index) +{ + uint64_t value; int fd = open(path, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &value, 8, index * 8) == 8 && !close(fd)); return value; +} +static uint64_t mapped_pfn(pid_t pid, uintptr_t address, size_t ps) +{ + char path[80]; snprintf(path, sizeof(path), "/proc/%d/pagemap", pid); + uint64_t e = entry(path, address / ps); CHECK(e & (1ULL << 63)); return e & ((1ULL << 55)-1); +} +static unsigned char *map(int file, size_t len, size_t align, off_t offset) +{ + void *reserve = mmap(NULL, len + align, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); CHECK(reserve != MAP_FAILED); + uintptr_t addr = ((uintptr_t)reserve + align - 1) & ~(align - 1); + CHECK(!munmap(reserve, len + align)); + unsigned char *p = mmap((void *)addr, len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED_NOREPLACE, file, offset); CHECK(p != MAP_FAILED); return p; +} +static int register_range(unsigned char *p, size_t len, int minor) +{ + int fd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, .features = minor ? UFFD_FEATURE_MINOR_SHMEM : 0 }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register r = { .range = {(uintptr_t)p, len}, .mode = UFFDIO_REGISTER_MODE_MISSING | (minor ? UFFDIO_REGISTER_MODE_MINOR : 0) }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, &r)); return fd; +} +static void copy_leaf(int fd, uintptr_t dst, unsigned char value) +{ + unsigned char src[4096]; memset(src, value, sizeof(src)); + struct uffdio_copy c = { .dst = dst, .src = (uintptr_t)src, .len = sizeof(src) }; + CHECK(!ioctl(fd, UFFDIO_COPY, &c) && c.copy == sizeof(src)); +} +static void cpu(unsigned id) +{ + cpu_set_t set; CPU_ZERO(&set); CPU_SET(id,&set); + CHECK(!sched_setaffinity(0,sizeof(set),&set)); +} +static int worker(int file, int sock, size_t native, size_t len) +{ + CHECK(getauxval(AT_PAGESZ) == 4096); alarm(90); cpu(1); + unsigned char *p = map(file, len, len, 0); + unsigned char *q = map(file, len-4096, len, 4096); + CHECK(p[0] == 0x61 && q[0] == 0x61); + uint64_t old_head = mapped_pfn(getpid(), (uintptr_t)p, 4096) * 4096 / native; + CHECK(entry("/proc/kpageflags", old_head) & (1ULL << KPF_COMPOUND_HEAD)); + int fd = register_range(p, len, 1); + for (size_t i = 4 * native; i < len; i += native) copy_leaf(fd, (uintptr_t)p+i, 0x71); + int pipes[2]; CHECK(!pipe(pipes)); struct iovec iov = {p, 4096}; + CHECK(vmsplice(pipes[1], &iov, 1, 0) == 4096); + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, native, native)); + CHECK(entry("/proc/kpageflags", old_head) & (1ULL << KPF_COMPOUND_HEAD)); + CHECK(!close(pipes[0]) && !close(pipes[1])); + struct ready r = {(uintptr_t)p, old_head}; struct iovec v = { &r, sizeof(r) }; + union { struct cmsghdr align; char bytes[CMSG_SPACE(sizeof(fd))]; } control = {0}; + struct msghdr msg = { .msg_iov = &v, .msg_iovlen = 1, .msg_control = control.bytes, .msg_controllen = sizeof(control.bytes) }; + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); c->cmsg_level=SOL_SOCKET; c->cmsg_type=SCM_RIGHTS; c->cmsg_len=CMSG_LEN(sizeof(fd)); memcpy(CMSG_DATA(c),&fd,sizeof(fd)); + CHECK(sendmsg(sock,&msg,0)==sizeof(r)); + struct command cmd; + while (recv(sock,&cmd,sizeof(cmd),0)==sizeof(cmd)) { + CHECK(cmd.offset < len && cmd.len <= len-cmd.offset); + unsigned char value = 0; + if (!cmd.op) break; + if (cmd.op == 1) value = *(volatile unsigned char *)(p+cmd.offset); + else if (cmd.op == 2) CHECK(!fallocate(file,FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE,cmd.offset,cmd.len)); + else if (cmd.op == 3) { CHECK(cmd.offset>=4096); value=*(volatile unsigned char *)(q+cmd.offset-4096); } + else if (cmd.op == 4) { CHECK(cmd.offset>=4096); memset(q+cmd.offset-4096,cmd.value,cmd.len); } + else CHECK(0); + CHECK(send(sock,&value,1,0)==1); + } + CHECK(!munmap(q,len-4096) && !munmap(p,len) && !close(fd) && !close(file) && !close(sock)); return 0; +} +static void send_command(int sock, uint64_t op, size_t offset, size_t len) +{ + struct command c={.op=op,.offset=offset,.len=len}; CHECK(send(sock,&c,sizeof(c),0)==sizeof(c)); +} +static void reply(int sock, unsigned char expected) +{ + unsigned char value; CHECK(recv(sock,&value,1,0)==1 && value==expected); +} +static void fault(int sock,int fd,uintptr_t base,size_t offset,int minor,unsigned char value) +{ + send_command(sock,1,offset+17,0); + struct pollfd p={.fd=fd,.events=POLLIN}; struct uffd_msg m; + CHECK(poll(&p,1,10000)==1 && (p.revents&POLLIN)); + CHECK(read(fd,&m,sizeof(m))==sizeof(m)); + CHECK(m.event==UFFD_EVENT_PAGEFAULT && m.arg.pagefault.address==base+offset && m.arg.pagefault.flags==(minor?UFFD_PAGEFAULT_FLAG_MINOR:0)); + if (minor) { struct uffdio_continue c={.range={base+offset,4096}}; CHECK(!ioctl(fd,UFFDIO_CONTINUE,&c)&&c.mapped==4096); } + else copy_leaf(fd,base+offset,value); + reply(sock,value); +} +static unsigned long pmd_mapped(unsigned char *p) +{ + FILE *f=fopen("/proc/self/smaps","r"); CHECK(f); char *line=NULL; size_t cap=0; unsigned long a,b,kb,result=0; int selected=0; + while(getline(&line,&cap,f)>0) { if(sscanf(line,"%lx-%lx",&a,&b)==2) selected=a==(uintptr_t)p; else if(selected&&sscanf(line,"ShmemPmdMapped: %lu kB",&kb)==1) result=kb*1024; } + free(line); fclose(f); return result; +} +static int collapse(unsigned char *p,size_t len) +{ + int ret; unsigned tries=0; + do { errno=0; ret=madvise(p,len,MADV_COLLAPSE); if(!ret||errno!=EAGAIN)break; usleep(20000); } while(++tries<10); + printf("# collapse ret=%d errno=%d PMD-mapped=%lu\n",ret,ret?errno:0,pmd_mapped(p)); return ret; +} +int main(int argc,char **argv) +{ + setvbuf(stdout,NULL,_IONBF,0); + if(argc==6&&!strcmp(argv[1],"--worker")) return worker(atoi(argv[2]),atoi(argv[3]),strtoul(argv[4],NULL,10),strtoul(argv[5],NULL,10)); + if(getpid()!=1||argc!=1)return 2; + alarm(120); cpu(0); + CHECK(!mount("proc","/proc","proc",0,NULL)&&!mount("sysfs","/sys","sysfs",0,NULL)); + CHECK(!mount("tmpfs","/tmp","tmpfs",0,"huge=within_size,size=256m")); + size_t native=getauxval(AT_PAGESZ),len=0; FILE *f=fopen("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size","r"); CHECK(f&&fscanf(f,"%zu",&len)==1); fclose(f); + CHECK(native==16384&&len==32*1024*1024); + int file=open("/tmp/collapse",O_CREAT|O_RDWR|O_TRUNC,0600); CHECK(file>=0&&!ftruncate(file,4*native)); + unsigned char *initial=map(file,4*native,4*native,0); memset(initial,0x61,4*native); + CHECK(entry("/proc/kpageflags",mapped_pfn(getpid(),(uintptr_t)initial,native))&(1ULL<=0); + if(!pid) { char fd[24],so[24],ns[24],ls[24];close(sock[0]);snprintf(fd,sizeof(fd),"%d",file);snprintf(so,sizeof(so),"%d",sock[1]);snprintf(ns,sizeof(ns),"%zu",native);snprintf(ls,sizeof(ls),"%zu",len);CHECK(!prctl(SET_EXEC_PAGE_SIZE,4096,0UL,0UL,0UL));execl("/init","/init","--worker",fd,so,ns,ls,NULL);_exit(3); } + close(sock[1]); struct ready ready; struct iovec v={&ready,sizeof(ready)}; + union { struct cmsghdr align; char bytes[CMSG_SPACE(sizeof(int))]; } control={0}; + struct msghdr msg={.msg_iov=&v,.msg_iovlen=1,.msg_control=control.bytes,.msg_controllen=sizeof(control.bytes)}; + CHECK(recvmsg(sock[0],&msg,0)==sizeof(ready)&&!(msg.msg_flags&(MSG_TRUNC|MSG_CTRUNC)));struct cmsghdr *c=CMSG_FIRSTHDR(&msg);CHECK(c&&c->cmsg_level==SOL_SOCKET&&c->cmsg_type==SCM_RIGHTS);int uffd;memcpy(&uffd,CMSG_DATA(c),sizeof(uffd)); + unsigned char *p=map(file,len,len,0);int native_uffd=register_range(p,len,0); + CHECK(collapse(p,len)==-1&&errno==EINVAL&&pmd_mapped(p)==0); + fault(sock[0],uffd,ready.base,0,1,0x61); + fault(sock[0],uffd,ready.base,len-native,1,0x71); + uint64_t head=mapped_pfn(pid,ready.base,4096)*4096/native; + CHECK(head!=ready.head&&(entry("/proc/kpageflags",head)&(1ULL<=4096&&low<8192) { + if(page==5) expected=0xa3; + if(page==7) expected=0xb4; + if(page>=16&&page<24) expected=0xc0+page-16; + } + CHECK(p[i]==expected); + } + puts("ok - eight cross-CPU PMD cycles, native/shifted-fine writes and complete 32MiB bytes"); + send_command(sock[0],0,0,0);int status;CHECK(waitpid(pid,&status,0)==pid&&WIFEXITED(status)&&!WEXITSTATUS(status)); + CHECK(!close(uffd)&&!close(native_uffd)&&!munmap(p,len)&&!close(file)&&!close(sock[0])); + puts("SHMEM COLLAPSE CONTRACT PASS");sync();reboot(RB_POWER_OFF);for(;;)pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-hole-init.c b/tools/testing/selftests/arm64/user4k/shmem-hole-init.c new file mode 100644 index 0000000000000..d5fa6803805b1 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-hole-init.c @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) do { if (!(x)) { printf("not ok - shmem hole line=%d errno=%d\n", __LINE__, errno); exit(1); } } while (0) +struct access { volatile unsigned char *p; unsigned char value; }; +static void *read_byte(void *opaque) { struct access *a = opaque; a->value = *a->p; return NULL; } +static uint64_t pagemap(pid_t pid, uintptr_t address, size_t granule) +{ + char name[80]; uint64_t entry; + snprintf(name, sizeof(name), "/proc/%d/pagemap", pid); + int fd = open(name, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &entry, 8, address / granule * 8) == 8); + CHECK(!close(fd)); return entry; +} +static void fill(int fd, unsigned char *dst, unsigned char *src, size_t ps) +{ + struct uffdio_copy c = { .dst = (uintptr_t)dst, .src = (uintptr_t)src, .len = ps }; + CHECK(!ioctl(fd, UFFDIO_COPY, &c) && c.copy == (long long)ps); +} +static void resolve(int fd, unsigned char *p, size_t ps, int minor, unsigned char value) +{ + struct access a = { .p = p + 17 }; pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &a)); + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; struct uffd_msg msg; + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == (minor ? UFFD_PAGEFAULT_FLAG_MINOR : 0)); + if (minor) { + struct uffdio_continue c = { .range = { (uintptr_t)p, ps } }; + CHECK(!ioctl(fd, UFFDIO_CONTINUE, &c) && c.mapped == (long long)ps); + } else { + unsigned char *source = malloc(ps); CHECK(source); memset(source, value, ps); + fill(fd, p, source, ps); free(source); + } + CHECK(!pthread_join(thread, NULL) && a.value == value); +} +struct command { uint64_t op, offset, len, value; }; +static int alias_worker(int file, int input, int output, size_t len, size_t native) +{ + CHECK(getauxval(AT_PAGESZ) == native); + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + CHECK(p != MAP_FAILED); + /* Establish every coarse PTE before the fine process punches its hole. */ + for (size_t i = 0; i < len; i++) CHECK(p[i] == 0x61); + uint64_t base = (uintptr_t)p; CHECK(write(output, &base, sizeof(base)) == sizeof(base)); + struct command cmd; + while (read(input, &cmd, sizeof(cmd)) == sizeof(cmd)) { + CHECK(cmd.offset <= len && cmd.len <= len - cmd.offset); + if (!cmd.op) break; + if (cmd.op == 2) memset(p + cmd.offset, cmd.value, cmd.len); + for (size_t i = 0; i < cmd.len; i++) CHECK(p[cmd.offset + i] == cmd.value); + CHECK(write(output, "y", 1) == 1); + } + CHECK(!munmap(p, len)); return 0; +} +static void tell(int input, int output, uint64_t op, size_t offset, size_t len, unsigned value) +{ + struct command cmd = { op, offset, len, value }; char reply; + CHECK(write(input, &cmd, sizeof(cmd)) == sizeof(cmd)); + if (op) CHECK(read(output, &reply, 1) == 1 && reply == 'y'); +} +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "r"); char key[80]; unsigned long value, result = 0; + CHECK(f); + while (fscanf(f, "%79s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) result = value; + fclose(f); return result; +} +static unsigned long mapping_swap(void *address) +{ + FILE *f = fopen("/proc/self/smaps", "r"); char *line = NULL; size_t cap = 0; + unsigned long start, end, value, result = 0; int selected = 0; + CHECK(f); + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + selected = start == (uintptr_t)address; + else if (selected && sscanf(line, "Swap: %lu kB", &value) == 1) result = value; + } + free(line); fclose(f); return result; +} +static void swapped_missing(size_t ps, size_t native) +{ + int file = memfd_create("partial-uffd-swap", 0); CHECK(file >= 0 && !ftruncate(file, native)); + unsigned char *p = mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *source = malloc(ps); CHECK(p != MAP_FAILED && source); memset(source, 0x71, ps); + int fd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, .features = UFFD_FEATURE_MINOR_SHMEM }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = {(uintptr_t)p, native}, .mode = UFFDIO_REGISTER_MODE_MISSING | UFFDIO_REGISTER_MODE_MINOR }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + fill(fd, p, source, ps); + unsigned long before = swapouts(), usage = 0, written; + unsigned tries; + for (tries = 0; tries < 100; tries++) { + CHECK(!madvise(p, native, MADV_PAGEOUT)); usleep(20000); + usage = mapping_swap(p); + if (usage >= native / 1024 && swapouts() > before) break; + } + written = swapouts() - before; + CHECK(usage >= native / 1024 && written > 0); + /* The selected cache index is swapped; its unfilled neighbour stays missing. */ + resolve(fd, p + ps, ps, 0, 0x92); + resolve(fd, p, ps, 1, 0x71); + for (size_t i = 0; i < ps; i++) CHECK(p[i] == 0x71 && p[ps+i] == 0x92); + CHECK(mapping_swap(p) == 0); + printf("ok - shared uffd %zuK on %zuK swapped cache=%lukB writes=%lu attempts=%u missing COPY preserves neighbour\n", ps/1024, native/1024, usage, written, tries+1); + CHECK(!close(fd) && !munmap(p, native) && !close(file)); free(source); +} +static int worker(size_t ps, size_t native) +{ + CHECK(getauxval(AT_PAGESZ) == ps); alarm(60); + if (getenv("SHMEM_HOLE_SWAP") && ps < native) swapped_missing(ps, native); + size_t len = 4 * native; + int file = memfd_create("hole-contract", 0); CHECK(file >= 0 && !ftruncate(file, len)); + unsigned char *source = malloc(len); CHECK(source); memset(source, 0x61, len); + CHECK(pwrite(file, source, len, 0) == (ssize_t)len); + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *q = mmap(NULL, len - ps, PROT_READ | PROT_WRITE, MAP_SHARED, file, ps); + unsigned char *cow = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, file, 0); + CHECK(p != MAP_FAILED && q != MAP_FAILED && cow != MAP_FAILED); + for (size_t i = 0; i < len; i += ps) CHECK(p[i] == 0x61); + for (size_t i = 0; i < len - ps; i += ps) CHECK(q[i] == 0x61); + memset(cow + ps, 0xe1, ps); + int commands[2], replies[2]; CHECK(!pipe(commands) && !pipe(replies)); + pid_t alias = fork(); CHECK(alias >= 0); + if (!alias) { + char f[24], in[24], out[24], size[24], ns[24]; + close(commands[1]); close(replies[0]); + snprintf(f, sizeof(f), "%d", file); snprintf(in, sizeof(in), "%d", commands[0]); + snprintf(out, sizeof(out), "%d", replies[1]); snprintf(size, sizeof(size), "%zu", len); + snprintf(ns, sizeof(ns), "%zu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, native, 0UL, 0UL, 0UL)); + execl("/init", "/init", "--alias", f, in, out, size, ns, NULL); _exit(3); + } + close(commands[0]); close(replies[1]); uint64_t alias_base; + CHECK(read(replies[0], &alias_base, sizeof(alias_base)) == sizeof(alias_base)); + int fd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, .features = UFFD_FEATURE_MINOR_SHMEM }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = {(uintptr_t)p, len}, .mode = UFFDIO_REGISTER_MODE_MISSING | UFFDIO_REGISTER_MODE_MINOR }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(pagemap(alias, alias_base + ps, native) & (1ULL << 63)); + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, ps)); + CHECK(!(pagemap(getpid(), (uintptr_t)p + ps, ps) & (1ULL << 63))); + CHECK(!(pagemap(getpid(), (uintptr_t)q, ps) & (1ULL << 63))); + CHECK(!(pagemap(alias, alias_base + ps, native) & (1ULL << 63))); + /* Fine neighbour PTEs survive, while COW retains its independent bytes. */ + CHECK(pagemap(getpid(), (uintptr_t)p, ps) & (1ULL << 63)); + CHECK(pagemap(getpid(), (uintptr_t)p + 2 * ps, ps) & (1ULL << 63)); + CHECK(p[0] == 0x61 && p[2 * ps] == 0x61 && cow[ps] == 0xe1); + resolve(fd, p + ps, ps, 0, 0xa2); + CHECK(q[0] == 0xa2 && cow[ps] == 0xe1); + tell(commands[1], replies[0], 1, ps, ps, 0xa2); + tell(commands[1], replies[0], 1, 0, ps, 0x61); + tell(commands[1], replies[0], 1, 2 * ps, ps, 0x61); + /* Refaulting a coarse writable alias makes its covered units populated. */ + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, ps)); + tell(commands[1], replies[0], 2, ps, ps, 0xb7); + struct uffdio_copy collision = { .dst = (uintptr_t)p + ps, .src = (uintptr_t)source, .len = ps }; + errno = 0; CHECK(ioctl(fd, UFFDIO_COPY, &collision) == -1 && errno == EEXIST); + resolve(fd, p + ps, ps, 1, 0xb7); + printf("ok - shmem hole %zuK on %zuK precise PTEs, shifted alias, COW, coarse revoke/refault and COPY collision\n", ps/1024, native/1024); + /* Stop the alias before resize, so it cannot intentionally promote holes. */ + tell(commands[1], replies[0], 0, 0, 0, 0); close(commands[1]); close(replies[0]); + int status; CHECK(waitpid(alias, &status, 0) == alias && WIFEXITED(status) && !WEXITSTATUS(status)); + size_t cut = 2 * ps + ps / 2; + CHECK(!ftruncate(file, cut) && !ftruncate(file, len)); + /* Partial final user leaf remains valid, with only truncated bytes zero. */ + if (!(pagemap(getpid(), (uintptr_t)p + 2 * ps, ps) & (1ULL << 63))) + resolve(fd, p + 2 * ps, ps, 1, 0x61); + CHECK(p[2 * ps] == 0x61 && p[cut - 1] == 0x61 && p[cut] == 0 && p[3 * ps - 1] == 0); + resolve(fd, p + 3 * ps, ps, 0, 0xc3); + CHECK(q[2 * ps] == 0xc3); + CHECK(cow[ps] == 0xe1); + printf("ok - shmem resize %zuK on %zuK partial tail retained, shrink/regrow missing leaf refilled\n", ps/1024, native/1024); + CHECK(!close(fd) && !munmap(p, len) && !munmap(q, len - ps) && !munmap(cow, len) && !close(file)); + free(source); return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 7 && !strcmp(argv[1], "--alias")) + return alias_worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), strtoul(argv[5], NULL, 10), strtoul(argv[6], NULL, 10)); + if (argc == 4 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), strtoul(argv[3], NULL, 10)); + int swap_test = argc == 2 && !strcmp(argv[1], "--swap"); + if (getpid() != 1 || (argc != 1 && !swap_test)) return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + if (swap_test) { + struct stat st; + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)); + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) usleep(20000); + CHECK(!stat("/dev/vda", &st) && S_ISBLK(st.st_mode) && !swapon("/dev/vda", 0)); + CHECK(!setenv("SHMEM_HOLE_SWAP", "1", 1)); + } + unsigned long native = getauxval(AT_PAGESZ), sizes[] = {native, 4096, 16384}; int failed = 0; + for (unsigned i = 0; i < (native == 65536 ? 3U : 2U); i++) { + pid_t pid = fork(); CHECK(pid >= 0); + if (!pid) { + char ps[24], ns[24]; snprintf(ps, sizeof(ps), "%lu", sizes[i]); snprintf(ns, sizeof(ns), "%lu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)); + execl("/init", "/init", "--worker", ps, ns, NULL); _exit(3); + } + int status = 0; int ok = waitpid(pid, &status, 0) == pid && WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - shmem hole ABI %luK status=%#x\n", ok ? "ok" : "not ok", sizes[i]/1024, status); failed |= !ok; + } + if (swap_test) CHECK(!swapoff("/dev/vda")); + printf("SHMEM HOLE CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); reboot(RB_POWER_OFF); for (;;) pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-pin-init.c b/tools/testing/selftests/arm64/user4k/shmem-pin-init.c new file mode 100644 index 0000000000000..79f5cfaaaef6e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-pin-init.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) do { if (!(x)) { printf("not ok - shmem pin line=%d errno=%d\n", __LINE__, errno); exit(1); } } while (0) +struct ring { + int fd; + struct io_uring_params p; + void *sq, *cq, *sqes; + size_t sqlen, cqlen, sqelen; +}; +static unsigned load(unsigned *p) { return __atomic_load_n(p,__ATOMIC_ACQUIRE); } +static void store(unsigned *p,unsigned v) { __atomic_store_n(p,v,__ATOMIC_RELEASE); } +static size_t align_size(size_t n) { size_t ps=getauxval(AT_PAGESZ);return (n+ps-1)&~(ps-1); } +static void destroy(struct ring *r) +{ + if(r->sq && r->sq!=MAP_FAILED)munmap(r->sq,r->sqlen); + if(r->cq && r->cq!=MAP_FAILED && r->cq!=r->sq)munmap(r->cq,r->cqlen); + if(r->sqes && r->sqes!=MAP_FAILED)munmap(r->sqes,r->sqelen); + if(r->fd>=0)close(r->fd); + *r=(struct ring){.fd=-1}; +} +static bool setup(struct ring *r,unsigned entries,unsigned flags) +{ + *r=(struct ring){.fd=-1};r->p.flags=flags; + r->fd=syscall(__NR_io_uring_setup,entries,&r->p); + if(r->fd<0)return false; + r->sqlen=align_size(r->p.sq_off.array+r->p.sq_entries*sizeof(unsigned)); + r->cqlen=align_size(r->p.cq_off.cqes+r->p.cq_entries*sizeof(struct io_uring_cqe)*(flags&IORING_SETUP_CQE32?2:1)); + r->sqelen=align_size(r->p.sq_entries*sizeof(struct io_uring_sqe)*(flags&IORING_SETUP_SQE128?2:1)); + if(r->p.features&IORING_FEAT_SINGLE_MMAP){if(r->cqlen>r->sqlen)r->sqlen=r->cqlen;r->cqlen=r->sqlen;} + r->sq=mmap(NULL,r->sqlen,PROT_READ|PROT_WRITE,MAP_SHARED,r->fd,IORING_OFF_SQ_RING); + r->cq=(r->p.features&IORING_FEAT_SINGLE_MMAP)?r->sq:mmap(NULL,r->cqlen,PROT_READ|PROT_WRITE,MAP_SHARED,r->fd,IORING_OFF_CQ_RING); + r->sqes=mmap(NULL,r->sqelen,PROT_READ|PROT_WRITE,MAP_SHARED,r->fd,IORING_OFF_SQES); + return r->sq!=MAP_FAILED && r->cq!=MAP_FAILED && r->sqes!=MAP_FAILED; +} +static bool request(struct ring *r,unsigned op,int fd,void *buf,unsigned len,uint64_t off,bool async) +{ + unsigned *sqtail=r->sq+r->p.sq_off.tail,*sqmask=r->sq+r->p.sq_off.ring_mask; + unsigned *cqhead=r->cq+r->p.cq_off.head,*cqtail=r->cq+r->p.cq_off.tail,*cqmask=r->cq+r->p.cq_off.ring_mask; + unsigned tail=load(sqtail),head=load(cqhead),idx=tail&load(sqmask); + unsigned sqstride=sizeof(struct io_uring_sqe)*(r->p.flags&IORING_SETUP_SQE128?2:1); + unsigned cqstride=sizeof(struct io_uring_cqe)*(r->p.flags&IORING_SETUP_CQE32?2:1); + struct io_uring_sqe *sqe=r->sqes+idx*sqstride; + memset(sqe,0,sqstride);sqe->opcode=op;sqe->fd=fd;sqe->addr=(uintptr_t)buf;sqe->len=len;sqe->off=off;sqe->user_data=(uint64_t)tail+1234; + if(async)sqe->flags=IOSQE_ASYNC; + ((unsigned *)(r->sq+r->p.sq_off.array))[idx]=idx; + store(sqtail,tail+1); + int ret=syscall(__NR_io_uring_enter,r->fd,1,1,IORING_ENTER_GETEVENTS,NULL,0); + if(ret!=1 || load(cqtail)!=head+1){printf("enter=%d errno=%d\n",ret,errno);return false;} + struct io_uring_cqe *cqe=r->cq+r->p.cq_off.cqes+(head&load(cqmask))*cqstride; + unsigned expected=op==IORING_OP_NOP?0:len; + if(op==IORING_OP_READV_FIXED || op==IORING_OP_WRITEV_FIXED){expected=0;for(unsigned i=0;iuser_data==(uint64_t)tail+1234 && cqe->res==(int)expected; + if(!ok)printf("cqe res=%d expected=%u\n",cqe->res,expected); + store(cqhead,head+1);return ok; +} + +static void fragment_geometry(unsigned char *p, size_t ps, int input) +{ + struct ring r; CHECK(setup(&r, 8, 0)); + struct iovec reg = { p + 123, 3 * ps - 246 }; + unsigned char *src = malloc(3 * ps), *expected = malloc(3 * ps); CHECK(src && expected); + memset(expected, 0x61, 3 * ps); memset(src, 0xb7, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_REGISTER_BUFFERS, ®, 1)); + CHECK(request(&r, IORING_OP_READ_FIXED, input, p + ps + 71, ps + 31, 17, true)); + memset(expected + ps + 71, 0xb7, ps + 31); CHECK(!memcmp(p, expected, 3 * ps)); + memset(src, 0xc8, 3 * ps); CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_WRITE_FIXED, input, p + ps + 71, ps + 31, 9, true)); + CHECK(pread(input, src, ps + 31, 9) == (ssize_t)(ps + 31)); + for (size_t i = 0; i < ps + 31; i++) CHECK(src[i] == 0xb7); + struct iovec vec[2] = { {p + 2 * ps + 17, ps - 400}, {p + ps + 91, ps - 600} }; + memset(src, 0xd5, 3 * ps); CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_READV_FIXED, input, vec, 2, 13, true)); + for (unsigned i = 0; i < 2; i++) memset(expected + ((unsigned char *)vec[i].iov_base - p), 0xd5, vec[i].iov_len); + CHECK(!memcmp(p, expected, 3 * ps)); + memset(src, 0xc8, 3 * ps); CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_WRITEV_FIXED, input, vec, 2, 23, true)); + size_t total = vec[0].iov_len + vec[1].iov_len; + CHECK(pread(input, src, total, 23) == (ssize_t)total); + for (size_t i = 0; i < total; i++) CHECK(src[i] == 0xd5); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_UNREGISTER_BUFFERS, NULL, 0)); + destroy(&r); free(src); free(expected); + memset(p, 0x61, 3 * ps); + printf("ok - %zuK fixed/vector IO uses user fragment geometry and preserves guards\n", ps/1024); +} + +struct punch_worker { + int file; + size_t off, ps; + pthread_barrier_t start; +}; +static void bind_cpu(unsigned cpu) +{ + cpu_set_t set; CPU_ZERO(&set); CPU_SET(cpu, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +static void *punch_repeatedly(void *arg) +{ + struct punch_worker *w = arg; + bind_cpu(1); pthread_barrier_wait(&w->start); + for (unsigned i = 0; i < 256; i++) { + CHECK(!fallocate(w->file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, w->off, w->ps)); + sched_yield(); + } + return NULL; +} +static void pin_hole_concurrency(struct ring *r, int file, int input, + unsigned char *p, size_t off, size_t ps) +{ + struct punch_worker w = { .file = file, .off = off, .ps = ps }; + pthread_t thread; + CHECK(!pthread_barrier_init(&w.start, NULL, 2)); + bind_cpu(0); CHECK(!pthread_create(&thread, NULL, punch_repeatedly, &w)); + pthread_barrier_wait(&w.start); + for (unsigned i = 0; i < 256; i++) CHECK(request(r, IORING_OP_READ_FIXED, input, p + off, ps, 0, true)); + CHECK(!pthread_join(thread, NULL) && !pthread_barrier_destroy(&w.start)); + printf("ok - %zuK concurrent fixed writes and 256 hole punches on separate CPUs\n", ps/1024); +} + +static int target(size_t native, size_t ps, int resize) +{ + CHECK(getauxval(AT_PAGESZ) == ps); alarm(30); + size_t len = 4 * native, off = native + ps; + bool retained = ps < native; + int file = memfd_create("pin-cache", 0), input = memfd_create("pin-input", 0); + CHECK(file >= 0 && input >= 0 && !ftruncate(file, len) && !ftruncate(input, ps)); + unsigned char *src = malloc(len), *back = malloc(len); CHECK(src && back); + memset(src, 0x61, len); CHECK(pwrite(file, src, len, 0) == (ssize_t)len); + memset(src, 0xb7, ps); CHECK(pwrite(input, src, ps, 0) == (ssize_t)ps); + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + CHECK(p != MAP_FAILED && !madvise(p, len, MADV_NOHUGEPAGE)); + CHECK(p[off] == 0x61); + fragment_geometry(p, ps, input); + memset(src, 0xb7, ps); CHECK(pwrite(input, src, ps, 0) == (ssize_t)ps); + struct ring r; CHECK(setup(&r, 8, 0)); + struct iovec reg = { p + off, ps }; + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_REGISTER_BUFFERS, ®, 1)); + int uffd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register ur = { .range = {(uintptr_t)p, len}, .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, &ur)); + if (resize) CHECK(!ftruncate(file, off) && !ftruncate(file, len)); + else CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, off, ps)); + if (retained && !resize) pin_hole_concurrency(&r, file, input, p, off, ps); + /* Fixed IO accesses its original FOLL_PIN reference after PTE removal. */ + CHECK(request(&r, IORING_OP_READ_FIXED, input, p + off, ps, 0, true)); + /* Also prove the original pinned object contains the completed write. */ + CHECK(request(&r, IORING_OP_WRITE_FIXED, input, p + off, ps, 0, true)); + CHECK(pread(input, back, ps, 0) == (ssize_t)ps); + for (size_t i = 0; i < ps; i++) CHECK(back[i] == 0xb7); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_UNREGISTER_BUFFERS, NULL, 0)); + memset(src, 0xc8, ps); + struct uffdio_copy c = { .dst = (uintptr_t)(p + off), .src = (uintptr_t)src, .len = ps }; + errno = 0; int ret = ioctl(uffd, UFFDIO_COPY, &c); + printf("# pin %zuK/%zuK resize=%d retained=%d COPY=%d errno=%d bytes=%lld\n", ps/1024, native/1024, resize, retained, ret, errno, (long long)c.copy); + if (retained) CHECK(ret == -1 && errno == EEXIST && c.copy == -EEXIST); + else CHECK(!ret && c.copy == (long long)ps); + CHECK(pread(file, back, len, 0) == (ssize_t)len); + for (size_t i = 0; i < len; i++) { + unsigned char expected = i >= off && i < off + ps ? (retained ? 0xb7 : 0xc8) : resize && i >= off ? 0 : 0x61; + CHECK(back[i] == expected); + } + for (size_t i = 0; i < ps; i++) CHECK(p[off + i] == (retained ? 0xb7 : 0xc8)); + /* A new punch after unpin can once again be populated by the pager. */ + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, off, ps)); + c.copy = 0; CHECK(!ioctl(uffd, UFFDIO_COPY, &c) && c.copy == (long long)ps); + for (size_t i = 0; i < ps; i++) CHECK(p[off + i] == 0xc8); + destroy(&r); CHECK(!close(uffd) && !munmap(p, len) && !close(file) && !close(input)); + free(src); free(back); + printf("ok - writable pin %zuK on %zuK resize=%d retention and detach\n", ps/1024, native/1024, resize); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--target")) return target(strtoul(argv[2], NULL, 10), strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (getpid() != 1 || argc != 1) return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) for (int resize = 0; resize < 2; resize++) { + pid_t pid = fork(); CHECK(pid >= 0); + if (!pid) { + char n[24], g[24], r[12]; snprintf(n, sizeof(n), "%zu", native); snprintf(g, sizeof(g), "%zu", ps); snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); + execl("/init", "init", "--target", n, g, r, NULL); _exit(127); + } + int status; CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && !WEXITSTATUS(status)); + } + puts("ok - shmem writable pin fixture complete"); reboot(RB_POWER_OFF); return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c b/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c new file mode 100644 index 0000000000000..843311c57bafb --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/pressure" +#define CHECK(x) do { if (!(x)) { printf("not ok - shmem pressure line=%d errno=%d\n", __LINE__, errno); exit(1); } } while (0) +static void put(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); CHECK(fd >= 0); + CHECK(write(fd, value, strlen(value)) == (ssize_t)strlen(value) && !close(fd)); +} +static unsigned long long get(const char *path, const char *key) +{ + FILE *f = fopen(path, "r"); CHECK(f); char name[128]; unsigned long long value, result = 0; + if (!key) CHECK(fscanf(f, "%llu", &result) == 1); + else while (fscanf(f, "%127s %llu", name, &value) == 2) if (!strcmp(key, name)) { result = value; break; } + fclose(f); return result; +} +static void copy_leaf(int uffd, unsigned char *dst, unsigned char *src, size_t ps, unsigned char value) +{ + memset(src, value, ps); struct uffdio_copy c = { .dst = (uintptr_t)dst, .src = (uintptr_t)src, .len = ps }; + int ret = ioctl(uffd, UFFDIO_COPY, &c); + if (ret) printf("# COPY destination=%p errno=%d transferred=%lld\n", dst, errno, (long long)c.copy); + CHECK(!ret && c.copy == (long long)ps); +} +static void checkpoint(int out, int in) +{ + char token; CHECK(write(out, "R", 1) == 1 && read(in, &token, 1) == 1 && token == 'G'); +} +static int worker(size_t native, size_t ps, int out, int in) +{ + CHECK(getauxval(AT_PAGESZ) == ps); alarm(120); + size_t records = (16UL << 20) / native, stride = ps < native ? native : 2 * native; + size_t len = records * stride; + int file = memfd_create("pressure-partial-cache", 0); CHECK(file >= 0 && !ftruncate(file, len)); + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *source = malloc(stride), *back = malloc(stride); CHECK(p != MAP_FAILED && source && back); + CHECK(!madvise(p, len, MADV_NOHUGEPAGE)); + int uffd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = {(uintptr_t)p, len}, .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, ®)); + for (size_t i = 0; i < records; i++) copy_leaf(uffd, p + i * stride, source, ps, 0xa1); + checkpoint(out, in); + /* Revisit in permuted order, filling holes in reclaimed cache backing. */ + for (size_t i = 0; i < records; i++) { + size_t slot = (i * 4051) & (records - 1); + copy_leaf(uffd, p + slot * stride + ps, source, ps, 0xb2); + for (size_t j = 0; j < ps; j++) CHECK(p[slot * stride + j] == 0xa1 && p[slot * stride + ps + j] == 0xb2); + } + checkpoint(out, in); + for (size_t i = 0; i < records; i += 2) { + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, i * stride, ps)); + copy_leaf(uffd, p + i * stride, source, ps, 0xc3); + } + checkpoint(out, in); + for (size_t i = 0; i < records; i++) { + size_t slot = (i * 4051 + 31) & (records - 1); + CHECK(pread(file, back, stride, slot * stride) == (ssize_t)stride); + for (size_t j = 0; j < stride; j++) { + unsigned char expected = j < ps ? (slot & 1 ? 0xa1 : 0xc3) : j < 2 * ps ? 0xb2 : 0; + CHECK(back[j] == expected); + } + struct uffdio_copy duplicate = { .dst = (uintptr_t)(p + slot * stride), .src = (uintptr_t)source, .len = ps }; + errno = 0; CHECK(ioctl(uffd, UFFDIO_COPY, &duplicate) == -1 && errno == EEXIST && duplicate.copy == -EEXIST); + } + checkpoint(out, in); + CHECK(!close(uffd) && !munmap(p, len) && !close(file)); free(source); free(back); + printf("ok - shmem pressure %zuK on %zuK partial-cache refill, holes, duplicate rejection and every byte\n", ps/1024, native/1024); return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker")) return worker(strtoul(argv[2], NULL, 10), strtoul(argv[3], NULL, 10), atoi(argv[4]), atoi(argv[5])); + if (getpid() != 1 || argc != 1) return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL) && !mount("sysfs", "/sys", "sysfs", 0, NULL)); + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) && !mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL)); + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory"); CHECK(!mkdir(CG, 0755)); + put(CG "/memory.max", "8388608"); put(CG "/memory.swap.max", "134217728"); + struct stat st; for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) usleep(20000); + CHECK(!swapon("/dev/vda", 0)); size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) { + int ready[2], go[2]; CHECK(!pipe(ready) && !pipe(go)); pid_t pid = fork(); CHECK(pid >= 0); + if (!pid) { + close(ready[0]); close(go[1]); char n[24], p[24], r[24], g[24]; + snprintf(n, sizeof(n), "%zu", native); snprintf(p, sizeof(p), "%zu", ps); + snprintf(r, sizeof(r), "%d", ready[1]); snprintf(g, sizeof(g), "%d", go[0]); + put(CG "/cgroup.procs", "0"); CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); + execl("/init", "init", "--worker", n, p, r, g, NULL); _exit(127); + } + close(ready[1]); close(go[0]); + for (unsigned phase = 0; phase < 4; phase++) { + char token; CHECK(read(ready[0], &token, 1) == 1 && token == 'R'); + unsigned long long swap = get(CG "/memory.swap.current", NULL), max = get(CG "/memory.events", "max"); + unsigned long long scan = get(CG "/memory.stat", "pgscan"), steal = get(CG "/memory.stat", "pgsteal"); + printf("# pressure %zuK phase=%u resident=%llu swap=%llu max=%llu scan=%llu steal=%llu\n", ps/1024, phase, get(CG "/memory.current", NULL), swap, max, scan, steal); + CHECK(swap && max && scan && steal && !get(CG "/memory.events", "oom") && !get(CG "/memory.events", "oom_kill")); + CHECK(write(go[1], "G", 1) == 1); + } + int status; CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && !WEXITSTATUS(status)); + close(ready[0]); close(go[1]); + /* Closed/unmapped file identity must release all of its cache/swap. */ + for (unsigned i = 0; i < 3000 && (get(CG "/memory.swap.current", NULL) || get(CG "/memory.stat", "shmem")); i++) usleep(20000); + CHECK(!get(CG "/memory.swap.current", NULL) && !get(CG "/memory.stat", "shmem")); + } + CHECK(!swapoff("/dev/vda")); puts("SHMEM PRESSURE CONTRACT PASS"); sync(); reboot(RB_POWER_OFF); return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-race-init.c b/tools/testing/selftests/arm64/user4k/shmem-race-init.c new file mode 100644 index 0000000000000..504d5721166f8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-race-init.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define ROUNDS 512 +#define CHECK(x) do { if (!(x)) { printf("not ok - shmem race line=%d errno=%d\n", __LINE__, errno); exit(1); } } while (0) +struct control { unsigned start, stop, ready; unsigned long aliases; }; +struct state { + struct control *control; + int file, uffd, resize; + size_t ps, native, len; + unsigned char *p, *q, *source; + unsigned copied, existed, eof, alloc_eof, retry; +}; +static unsigned load(unsigned *p) { return __atomic_load_n(p, __ATOMIC_ACQUIRE); } +static void store(unsigned *p, unsigned value) { __atomic_store_n(p, value, __ATOMIC_RELEASE); } +static void bind_cpu(unsigned cpu) +{ + cpu_set_t set; CPU_ZERO(&set); CPU_SET(cpu, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +static void start_wait(struct control *c) +{ + __atomic_add_fetch(&c->ready, 1, __ATOMIC_RELEASE); + while (!load(&c->start)) sched_yield(); +} +static void *mutate(void *arg) +{ + struct state *s = arg; bind_cpu(0); start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + if (s->resize) CHECK(!ftruncate(s->file, s->ps) && !ftruncate(s->file, s->len)); + else CHECK(!fallocate(s->file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, s->ps, s->ps)); + sched_yield(); + } + return NULL; +} +static void *populate(void *arg) +{ + struct state *s = arg; bind_cpu(1); start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + struct uffdio_copy c = { .dst = (uintptr_t)(s->p + s->ps), .src = (uintptr_t)s->source, .len = s->ps }; + errno = 0; int ret = ioctl(s->uffd, UFFDIO_COPY, &c); + if (!ret) { CHECK(c.copy == (long long)s->ps); s->copied++; } + else { + CHECK(ret == -1 && c.copy == -errno); + if (errno == EEXIST) s->existed++; + else if (errno == EFAULT && s->resize) s->eof++; + /* Native shmem allocator preserves its NULL/ENOMEM EOF ABI. */ + else if (errno == ENOMEM && s->resize && s->ps == s->native) s->alloc_eof++; + else if (errno == EAGAIN) s->retry++; + else { printf("# unexpected COPY error=%d bytes=%lld\n", errno, (long long)c.copy); CHECK(0); } + } + sched_yield(); + } + return NULL; +} +static int valid_byte(unsigned char value) { return value == 0 || value == 0x61 || value == 0xa2; } +static void *fine_alias(void *arg) +{ + struct state *s = arg; bind_cpu(0); start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + if (!s->resize) { + CHECK(!madvise(s->q, s->ps, MADV_DONTNEED)); + CHECK(valid_byte(*(volatile unsigned char *)s->q)); + } else { + /* The retained prefix remains inside EOF throughout resize. */ + CHECK(*(volatile unsigned char *)s->p == 0x61); + } + sched_yield(); + } + return NULL; +} +static int native_alias(int file, int control, size_t native, size_t ps, int resize) +{ + CHECK(getauxval(AT_PAGESZ) == native); bind_cpu(1); alarm(60); + struct control *c = mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, control, 0); + unsigned char *p = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + CHECK(c != MAP_FAILED && p != MAP_FAILED && !madvise(p, 4 * native, MADV_NOHUGEPAGE)); + start_wait(c); unsigned long cycles = 0; + while (!load(&c->stop)) { + CHECK(!madvise(p, 2 * native, MADV_DONTNEED)); + CHECK(*(volatile unsigned char *)p == 0x61); + if (!resize) CHECK(valid_byte(*(volatile unsigned char *)(p + ps))); + cycles++; sched_yield(); + } + c->aliases = cycles; + CHECK(!munmap(p, 4 * native) && !munmap(c, native)); return 0; +} +static int target(size_t native, size_t ps, int resize) +{ + CHECK(getauxval(AT_PAGESZ) == ps); alarm(60); + struct state s = { .native = native, .ps = ps, .len = 4 * native, .resize = resize }; + s.file = memfd_create("race-cache", 0); int control = memfd_create("race-control", 0); + CHECK(s.file >= 0 && control >= 0 && !ftruncate(s.file, s.len) && !ftruncate(control, native)); + s.control = mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, control, 0); + s.source = malloc(s.len); CHECK(s.control != MAP_FAILED && s.source); memset(s.source, 0x61, s.len); + CHECK(pwrite(s.file, s.source, s.len, 0) == (ssize_t)s.len); + s.p = mmap(NULL, s.len, PROT_READ | PROT_WRITE, MAP_SHARED, s.file, 0); + s.q = mmap(NULL, s.len - ps, PROT_READ | PROT_WRITE, MAP_SHARED, s.file, ps); + CHECK(s.p != MAP_FAILED && s.q != MAP_FAILED && !madvise(s.p, s.len, MADV_NOHUGEPAGE)); + CHECK(s.p[0] == 0x61); /* Avoid blocking the resize prefix reader on UFFD. */ + unsigned char *cow = MAP_FAILED; + if (!resize) { + cow = mmap(NULL, s.len, PROT_READ | PROT_WRITE, MAP_PRIVATE, s.file, 0); CHECK(cow != MAP_FAILED); + memset(cow + ps, 0xe1, ps); + } + s.uffd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); CHECK(s.uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; CHECK(!ioctl(s.uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = {(uintptr_t)s.p, s.len}, .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(s.uffd, UFFDIO_REGISTER, ®)); memset(s.source, 0xa2, ps); + pid_t alias = fork(); CHECK(alias >= 0); + if (!alias) { + char f[24], c[24], n[24], p[24], r[12]; + snprintf(f, sizeof(f), "%d", s.file); snprintf(c, sizeof(c), "%d", control); + snprintf(n, sizeof(n), "%zu", native); snprintf(p, sizeof(p), "%zu", ps); snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, native, 0, 0, 0)); + execl("/init", "init", "--alias", f, c, n, p, r, NULL); _exit(127); + } + pthread_t mutation, copy, fine; + CHECK(!pthread_create(&mutation, NULL, mutate, &s) && !pthread_create(©, NULL, populate, &s) && !pthread_create(&fine, NULL, fine_alias, &s)); + while (load(&s.control->ready) != 4) sched_yield(); + store(&s.control->start, 1); + CHECK(!pthread_join(mutation, NULL) && !pthread_join(copy, NULL) && !pthread_join(fine, NULL)); + store(&s.control->stop, 1); int status; + CHECK(waitpid(alias, &status, 0) == alias && WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(s.control->aliases && s.copied + s.existed + s.eof + s.alloc_eof + s.retry == ROUNDS); + printf("# race %zuK/%zuK resize=%d COPY=%u EEXIST=%u EOF=%u native-alloc-EOF=%u retry=%u native-refaults=%lu\n", ps/1024, native/1024, resize, s.copied, s.existed, s.eof, s.alloc_eof, s.retry, s.control->aliases); + /* Once concurrent actors stop, exact cache/PTE state must be recoverable. */ + CHECK(!ftruncate(s.file, s.len)); + CHECK(!fallocate(s.file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, ps)); + memset(s.source, 0xd4, ps); + struct uffdio_copy fill = { .dst = (uintptr_t)(s.p + ps), .src = (uintptr_t)s.source, .len = ps }; + CHECK(!ioctl(s.uffd, UFFDIO_COPY, &fill) && fill.copy == (long long)ps); + unsigned char *back = malloc(s.len); CHECK(back && pread(s.file, back, s.len, 0) == (ssize_t)s.len); + for (size_t i = 0; i < s.len; i++) { + unsigned char expected = i >= ps && i < 2 * ps ? 0xd4 : resize && i >= ps ? 0 : 0x61; + if (back[i] != expected) { printf("# byte=%zu actual=%#x expected=%#x\n", i, back[i], expected); CHECK(0); } + } + for (size_t i = 0; i < ps; i++) { + CHECK(s.p[ps + i] == 0xd4 && s.q[i] == 0xd4); + if (!resize) CHECK(cow[ps + i] == 0xe1); + } + CHECK(!close(s.uffd) && !munmap(s.p, s.len) && !munmap(s.q, s.len - ps)); + if (!resize) CHECK(!munmap(cow, s.len)); + CHECK(!munmap(s.control, native) && !close(control) && !close(s.file)); free(back); free(s.source); + printf("ok - concurrent shmem %zuK on %zuK resize=%d cache bytes, aliases and private COW\n", ps/1024, native/1024, resize); return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 7 && !strcmp(argv[1], "--alias")) return native_alias(atoi(argv[2]), atoi(argv[3]), strtoul(argv[4], NULL, 10), strtoul(argv[5], NULL, 10), atoi(argv[6])); + if (argc == 5 && !strcmp(argv[1], "--target")) return target(strtoul(argv[2], NULL, 10), strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (getpid() != 1 || argc != 1) return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) for (int resize = 0; resize < 2; resize++) { + pid_t pid = fork(); CHECK(pid >= 0); + if (!pid) { + char n[24], p[24], r[12]; snprintf(n, sizeof(n), "%zu", native); snprintf(p, sizeof(p), "%zu", ps); snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); execl("/init", "init", "--target", n, p, r, NULL); _exit(127); + } + int status; CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && !WEXITSTATUS(status)); + } + puts("SHMEM RACE CONTRACT PASS"); reboot(RB_POWER_OFF); return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-thp-init.c b/tools/testing/selftests/arm64/user4k/shmem-thp-init.c new file mode 100644 index 0000000000000..66cd53002daee --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-thp-init.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) do { if (!(x)) { printf("not ok - shmem THP line=%d errno=%d\n", __LINE__, errno); exit(1); } } while (0) +static uint64_t read_entry(const char *path, uint64_t index) +{ + uint64_t value; int fd = open(path, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &value, 8, index * 8) == 8 && !close(fd)); return value; +} +static uint64_t pfn(void *p, size_t ps) +{ + uint64_t entry = read_entry("/proc/self/pagemap", (uintptr_t)p / ps); + CHECK(entry & (1ULL << 63)); return entry & ((1ULL << 55) - 1); +} +static unsigned char *map_aligned(int fd, size_t len) +{ + void *reserve = mmap(NULL, 2 * len, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + uintptr_t address = ((uintptr_t)reserve + len - 1) & ~(len - 1); + CHECK(!munmap(reserve, 2 * len)); + unsigned char *p = mmap((void *)address, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, fd, 0); + CHECK(p != MAP_FAILED); return p; +} +struct reader { volatile unsigned char *p; unsigned char value; }; +static void *read_byte(void *arg) { struct reader *r = arg; r->value = *r->p; return NULL; } +static int target(int file, size_t native, size_t ps, int hold, int zero, int resize, uint64_t head) +{ + CHECK(getauxval(AT_PAGESZ) == ps); alarm(30); + size_t len = 4 * native, target_off = resize ? 2 * native : native; + size_t cut = native + ps / 2; + unsigned char *p = map_aligned(file, len); + for (size_t i = 0; i < len; i += ps) CHECK(p[i] == 0x61); + CHECK(pfn(p, ps) * ps / native == head); + int pipes[2]; CHECK(!pipe(pipes)); + if (hold) { + struct iovec iov = { p, ps }; + CHECK(vmsplice(pipes[1], &iov, 1, 0) == (ssize_t)ps); + } + int uffd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = {(uintptr_t)p, len}, .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, ®)); + if (resize) + CHECK(!ftruncate(file, cut) && !ftruncate(file, len)); + else + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, native, ps)); + uint64_t flags = read_entry("/proc/kpageflags", head); + printf("# THP %zuK native=%zuK hold=%d zero=%d resize=%d head_flags=%#llx\n", ps/1024, native/1024, hold, zero, resize, (unsigned long long)flags); + if (hold) CHECK((flags & ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))) == ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))); + struct reader r = { .p = p + target_off + 17 }; pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct pollfd pollfd = { .fd = uffd, .events = POLLIN }; struct uffd_msg msg; + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(uffd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && !msg.arg.pagefault.flags && msg.arg.pagefault.address == (uintptr_t)(p + target_off)); + unsigned char *source = malloc(ps); CHECK(source); memset(source, 0x92, ps); + int ret; + struct uffdio_copy invalid = { .dst = (uintptr_t)(p + target_off), .src = 1, .len = ps }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &invalid) == -1 && errno == EFAULT && invalid.copy == -EFAULT); + if (zero) { + struct uffdio_zeropage z = { .range = {(uintptr_t)(p + target_off), ps} }; + ret = ioctl(uffd, UFFDIO_ZEROPAGE, &z); + printf("# ZEROPAGE returned=%d errno=%d bytes=%lld\n", ret, errno, (long long)z.zeropage); + CHECK(!ret && z.zeropage == (long long)ps); + } else { + struct uffdio_copy c = { .dst = (uintptr_t)(p + target_off), .src = (uintptr_t)source, .len = ps }; + ret = ioctl(uffd, UFFDIO_COPY, &c); + printf("# COPY returned=%d errno=%d bytes=%lld\n", ret, errno, (long long)c.copy); + CHECK(!ret && c.copy == (long long)ps); + } + CHECK(!pthread_join(thread, NULL) && r.value == (zero ? 0 : 0x92)); + memset(source, 0xd4, ps); + struct uffdio_copy duplicate = { .dst = (uintptr_t)(p + target_off), .src = (uintptr_t)source, .len = ps }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &duplicate) == -1 && errno == EEXIST && duplicate.copy == -EEXIST); + /* File reads verify zeroed tail holes without faulting unfilled UFFD leaves. */ + unsigned char *back = malloc(len); CHECK(back); + CHECK(pread(file, back, len, 0) == (ssize_t)len); + for (size_t i = 0; i < len; i++) { + unsigned char expected = i >= target_off && i < target_off + ps ? + (zero ? 0 : 0x92) : resize && i >= cut ? 0 : 0x61; + CHECK(back[i] == expected); + if (!resize || (i >= target_off && i < target_off + ps)) CHECK(p[i] == expected); + } + free(back); + if (hold) { + CHECK(read(pipes[0], source, ps) == (ssize_t)ps); + for (size_t i = 0; i < ps; i++) CHECK(source[i] == 0x61); + } + CHECK(!close(pipes[0]) && !close(pipes[1]) && !close(uffd) && !munmap(p, len) && !close(file)); free(source); + printf("ok - THP %zuK on %zuK hold=%d zero=%d resize=%d missing refill and all neighbour bytes\n", ps/1024, native/1024, hold, zero, resize); return 0; +} +static int create_file(size_t native, uint64_t *head) +{ + int fd = open("/tmp/huge", O_CREAT | O_TRUNC | O_RDWR, 0600); CHECK(fd >= 0); + CHECK(!ftruncate(fd, 4 * native)); + unsigned char *p = map_aligned(fd, 4 * native); + CHECK(!madvise(p, 4 * native, MADV_HUGEPAGE)); + memset(p, 0x61, 4 * native); + *head = pfn(p, native); + uint64_t f = read_entry("/proc/kpageflags", *head); + printf("# initial head=%llu flags=%#llx VA=%p native=%zu\n", (unsigned long long)*head, (unsigned long long)f, p, native); + CHECK((f & ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))) == ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))); + for (unsigned i = 1; i < 4; i++) { + CHECK(pfn(p + i * native, native) == *head + i); + CHECK(read_entry("/proc/kpageflags", *head + i) & (1ULL << KPF_COMPOUND_TAIL)); + } + /* Keep the name until allocation: unlinked mappings use anon-shmem policy. */ + CHECK(!unlink("/tmp/huge") && !munmap(p, 4 * native)); return fd; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 9 && !strcmp(argv[1], "--target")) + return target(atoi(argv[2]), strtoul(argv[3], NULL, 10), strtoul(argv[4], NULL, 10), atoi(argv[5]), atoi(argv[6]), atoi(argv[7]), strtoull(argv[8], NULL, 10)); + if (getpid() != 1 || argc != 1) return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("tmpfs", "/tmp", "tmpfs", 0, "huge=within_size,size=32m")); + size_t native = getauxval(AT_PAGESZ), sizes[] = {native, 4096, 16384}; int failed = 0; + for (unsigned abi = 0; abi < (native == 65536 ? 3U : 2U); abi++) for (unsigned hold = 0; hold < 2; hold++) for (unsigned zero = 0; zero < 2; zero++) for (unsigned resize = 0; resize < 2; resize++) { + uint64_t head; int fd = create_file(native, &head); pid_t pid = fork(); CHECK(pid >= 0); + if (!pid) { + char fs[24], ns[24], ps[24], hs[24], zs[24], rs[24], pfs[24]; + snprintf(fs, sizeof(fs), "%d", fd); snprintf(ns, sizeof(ns), "%zu", native); snprintf(ps, sizeof(ps), "%zu", sizes[abi]); + snprintf(hs, sizeof(hs), "%u", hold); snprintf(zs, sizeof(zs), "%u", zero); snprintf(rs, sizeof(rs), "%u", resize); snprintf(pfs, sizeof(pfs), "%llu", (unsigned long long)head); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, sizes[abi], 0UL, 0UL, 0UL)); + execl("/init", "/init", "--target", fs, ns, ps, hs, zs, rs, pfs, NULL); _exit(3); + } + int status = 0; int ok = waitpid(pid, &status, 0) == pid && WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - THP ABI %zuK hold=%u zero=%u resize=%u status=%#x\n", ok ? "ok" : "not ok", sizes[abi]/1024, hold, zero, resize, status); failed |= !ok; CHECK(!close(fd)); + } + printf("SHMEM THP CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); reboot(RB_POWER_OFF); for (;;) pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c b/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c new file mode 100644 index 0000000000000..deed4c40e5fea --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PID-1 runner for a disposable VM with a dedicated virtio swap disk. + * Use kunit.autorun=0: tests skipped at boot stay skipped on a debugfs rerun. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int run_swap_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-shmem-swap/"; + char path[128], *log; + size_t used = 0, capacity = 1024 * 1024; + ssize_t n; + int fd, ret = -1; + + snprintf(path, sizeof(path), "%srun", base); + fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = write(fd, "1", 1); + close(fd); + if (n != 1) + return -1; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + log = calloc(1, capacity); + if (!log) { + close(fd); + return -1; + } + while (used < capacity - 1 && (n = read(fd, log + used, capacity - 1 - used)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (n == 0 && used < capacity - 1 && strstr(log, "\nok 1 arm64-user4k-shmem-swap\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; + free(log); + return ret; +} + +static int fault_setting(const char *name, const char *value) +{ + char path[160]; snprintf(path, sizeof(path), "/sys/kernel/debug/failslab/%s", name); + int fd = open(path, O_WRONLY | O_CLOEXEC); if (fd < 0) return -1; + ssize_t n = write(fd, value, strlen(value)); close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} +static int configure_failslab(bool enable) +{ + if (!enable) return fault_setting("probability", "0"); + /* The KUnit worker selects itself only around metadata writeout. */ + return fault_setting("task-filter", "Y") || fault_setting("verbose", "0") || + fault_setting("times", "-1") || fault_setting("interval", "1") || + fault_setting("space", "0") || fault_setting("probability", "100"); +} + +static int swap_control_helper(const char *operation) +{ + struct stat init, self; + int ret; + + /* A helper is allowed only when this same executable is the VM's PID 1. */ + if (getpid() == 1 || stat("/proc/1/exe", &init) || stat("/proc/self/exe", &self) || + init.st_dev != self.st_dev || init.st_ino != self.st_ino) { + fprintf(stderr, "swap helper requires its fixture runner as PID 1\n"); + return 1; + } + if (!strcmp(operation, "--memcg-setup-helper")) { + if (mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) && errno != EBUSY) return 1; + int fd = open("/sys/fs/cgroup/cgroup.subtree_control", O_WRONLY | O_CLOEXEC); + if (fd < 0) return 1; + ssize_t n = write(fd, "+memory", 7); close(fd); + if (n != 7) return 1; + return mkdir("/sys/fs/cgroup/uffd-metadata", 0755) && errno != EEXIST; + } + if (!strcmp(operation, "--failslab-on-helper")) return configure_failslab(true); + if (!strcmp(operation, "--failslab-off-helper")) return configure_failslab(false); + ret = !strcmp(operation, "--swapoff-helper") ? swapoff("/dev/vda") : swapon("/dev/vda", 0); + if (ret) + perror(operation); + return ret ? 1 : 0; +} + +int main(int argc, char **argv) +{ + struct stat st; + bool active = false; + int ret = 1; + if (argc == 2 && (!strcmp(argv[1], "--swapon-helper") || !strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--failslab-on-helper") || !strcmp(argv[1], "--failslab-off-helper") || + !strcmp(argv[1], "--memcg-setup-helper"))) + return swap_control_helper(argv[1]); + if (getpid() != 1 || argc != 1) return 2; + setvbuf(stdout, NULL, _IONBF, 0); + unsigned long ps = getauxval(AT_PAGESZ); + if (ps != 16384 && ps != 65536) goto out; + if (mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) goto out; + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || swapon("/dev/vda", 0)) goto out; + active = true; + ret = run_swap_suite() ? 1 : 0; +out: + if (active && swapoff("/dev/vda")) ret = 1; + printf("SHMEM UFFD STATE SWAP %s errno=%d\n", ret ? "FAIL" : "PASS", errno); + sync(); reboot(RB_POWER_OFF); for (;;) pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/splice-fixture-init.c b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c new file mode 100644 index 0000000000000..c0b883efc9845 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c @@ -0,0 +1,502 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static bool use_tcp, use_ipv6, use_udp; + +static int connected_pair(int sockets[2]) +{ + struct sockaddr_in a4 = { .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK) }; + struct sockaddr_in6 a6 = { .sin6_family = AF_INET6, + .sin6_addr = IN6ADDR_LOOPBACK_INIT }; + struct sockaddr_storage peer; + struct sockaddr *address = use_ipv6 ? (void *)&a6 : (void *)&a4; + socklen_t size = use_ipv6 ? sizeof(a6) : sizeof(a4), peer_size = sizeof(peer); + int domain = use_ipv6 ? AF_INET6 : AF_INET; + int listener, saved; + + if (!use_tcp && !use_udp) + return socketpair(AF_UNIX, SOCK_STREAM, 0, sockets); + listener = socket(domain, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); + if (listener < 0) + return -1; + sockets[0] = -1; + if (bind(listener, address, size) || getsockname(listener, address, &size)) + goto error; + if (!use_udp && listen(listener, 1)) + goto error; + sockets[0] = socket(domain, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); + if (sockets[0] < 0 || connect(sockets[0], address, size)) + goto error; + if (use_udp) { + if (getsockname(sockets[0], (void *)&peer, &peer_size) || + connect(listener, (void *)&peer, peer_size)) + goto error; + sockets[1] = listener; + return 0; + } + sockets[1] = accept(listener, NULL, NULL); + if (sockets[1] < 0) + goto error; + close(listener); + return 0; +error: + saved = errno; + if (sockets[0] >= 0) + close(sockets[0]); + close(listener); + errno = saved; + return -1; +} + +static unsigned char value(size_t off) { return (off * 13 + (off >> 12) * 31) & 255; } +static void fill(unsigned char *p, size_t n) +{ + for (size_t i = 0; i < n; i++) + p[i] = value(i); +} +static bool check(const unsigned char *p, size_t n, size_t off) +{ + for (size_t i = 0; i < n; i++) + if (p[i] != value(i + off)) { + ksft_print_msg("data mismatch at %zu: got %02x expected %02x\n", + i + off, p[i], value(i + off)); + return false; + } + return true; +} +static bool read_exact(int fd, unsigned char *p, size_t n) +{ + while (n) { + ssize_t got = read(fd, p, n); + + if (got <= 0) { + ksft_print_msg("read fd=%d got=%zd remaining=%zu errno=%d\n", fd, got, n, errno); + return false; + } + p += got; + n -= got; + } + return true; +} +static bool replace(unsigned char *p, size_t n) +{ + if (munmap(p, n) || mmap(p, n, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0) != p) + return false; + memset(p, 0xec, n); + return true; +} +static bool cow(unsigned char *p, size_t n) +{ + pid_t child = fork(); + int status = 0; + + if (!child) + _exit(0); + if (child < 0 || waitpid(child, &status, 0) != child || status) + return false; + memset(p, 0xab, n); + return true; +} +static void report(bool ok, size_t ps, const char *name) +{ + ksft_test_result(ok, "%zuK %s\n", ps / 1024, name); +} +static bool tls_pair(int sockets[2], unsigned int version) +{ + struct tls12_crypto_info_aes_gcm_128 crypto = { + .info.version = version, + .info.cipher_type = TLS_CIPHER_AES_GCM_128, + }; + struct timeval timeout = { .tv_sec = 5 }; + int no_padding = 1; + + if (connected_pair(sockets)) + return false; + return !setsockopt(sockets[0], IPPROTO_TCP, TCP_ULP, "tls", 4) && + !setsockopt(sockets[1], IPPROTO_TCP, TCP_ULP, "tls", 4) && + !setsockopt(sockets[0], SOL_TLS, TLS_TX, &crypto, sizeof(crypto)) && + !setsockopt(sockets[1], SOL_TLS, TLS_RX, &crypto, sizeof(crypto)) && + (version != TLS_1_3_VERSION || + !setsockopt(sockets[1], SOL_TLS, TLS_RX_EXPECT_NO_PAD, &no_padding, sizeof(no_padding))) && + !setsockopt(sockets[1], SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); +} + +static void tls_checks(size_t ps, unsigned char *p, size_t len, unsigned char *back) +{ + int sockets[2], fds[2]; + char name[160]; + + if (pipe(fds)) + ksft_exit_fail_msg("TLS pipe errno=%d\n", errno); + for (unsigned int v = 0; v < 2; v++) { + unsigned int version = v ? TLS_1_3_VERSION : TLS_1_2_VERSION; + struct iovec vec; + bool ok; + + if (!tls_pair(sockets, version)) + ksft_exit_fail_msg("TLS %x setup errno=%d\n", version, errno); + for (unsigned int mode = 0; mode < 2; mode++) { + const size_t count = mode ? 4096 : 4096 - 29; + const size_t offset = ps + (mode ? 0 : 29); + + fill(p, len); + vec = (struct iovec){ p + offset, count }; + if (vmsplice(fds[1], &vec, 1, 0) != (ssize_t)count || + splice(fds[0], NULL, sockets[0], NULL, count, SPLICE_F_MORE) != (ssize_t)count) + ksft_exit_fail_msg("TLS MORE splice errno=%d\n", errno); + errno = 0; + if (recv(sockets[1], back, len, MSG_DONTWAIT) != -1 || errno != EAGAIN) + ksft_exit_fail_msg("TLS pending plaintext precondition errno=%d\n", errno); + ok = mode ? cow(p + ps, 4096) : replace(p + ps, ps); + ok &= send(sockets[0], "!", 1, 0) == 1; + ok &= read_exact(sockets[1], back, count + 1) && + check(back, count, offset) && back[count] == '!'; + snprintf(name, sizeof(name), "TLS 1.%u pending plaintext survives %s", v + 2, + mode ? "fork/COW" : "unmap/replacement"); + report(ok, ps, name); + } + { + const size_t bytes = 3 * 16384 + 123, prefix = 137; + const size_t mapping = (bytes + ps - 1) & ~(ps - 1); + unsigned char *source = mmap(NULL, mapping, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *received = malloc(bytes + prefix); + + if (source == MAP_FAILED || !received) + ksft_exit_fail_msg("TLS multi-record allocation\n"); + fill(source, mapping); + ok = send(sockets[0], source, prefix, 0) == (ssize_t)prefix; + vec = (struct iovec){ source, bytes }; + if (vmsplice(fds[1], &vec, 1, 0) != (ssize_t)bytes || + splice(fds[0], NULL, sockets[0], NULL, bytes, 0) != (ssize_t)bytes) + ksft_exit_fail_msg("TLS multi-record splice errno=%d\n", errno); + ok &= replace(source, mapping); + ok &= read_exact(sockets[1], received, bytes + prefix) && + check(received, prefix, 0) && check(received + prefix, bytes, 0); + snprintf(name, sizeof(name), "TLS 1.%u ordinary send and multiple spliced records", v + 2); + report(ok, ps, name); + munmap(source, mapping); + free(received); + } + { + const size_t bytes = 5003; + int file = memfd_create("tls-receive", 0); + unsigned char *mapped; + + if (file < 0 || ftruncate(file, 3 * ps)) + ksft_exit_fail_msg("TLS receive file setup\n"); + mapped = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + if (mapped == MAP_FAILED) + ksft_exit_fail_msg("TLS receive file mapping\n"); + memset(mapped, 0x5c, 3 * ps); + fill(p, len); + ok = send(sockets[0], p, bytes, 0) == (ssize_t)bytes && + read_exact(sockets[1], mapped + 17, bytes); + ok &= !msync(mapped, 3 * ps, MS_SYNC) && + pread(file, back, bytes + 34, 0) == (ssize_t)(bytes + 34) && + check(back + 17, bytes, 0); + for (unsigned int i = 0; i < 17; i++) + ok &= back[i] == 0x5c && back[bytes + 17 + i] == 0x5c; + snprintf(name, sizeof(name), "TLS 1.%u decrypts into shared file with neighboring bytes intact", v + 2); + report(ok, ps, name); + munmap(mapped, 3 * ps); + close(file); + } + { + unsigned char *mapped = mmap(NULL, 6 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + struct iovec vectors[4]; + struct msghdr message = { .msg_iov = vectors, .msg_iovlen = 4 }; + + if (mapped == MAP_FAILED) + ksft_exit_fail_msg("TLS receive iovec mapping\n"); + memset(mapped, 0x81, 6 * ps); + vectors[0] = (struct iovec){ mapped + 13, 5003 }; + vectors[1] = (struct iovec){ NULL, 0 }; + vectors[2] = (struct iovec){ mapped + 2 * ps + 7, 2222 }; + vectors[3] = (struct iovec){ mapped + 4 * ps + 5, 1010 }; + fill(p, len); + ok = send(sockets[0], p, 8235, 0) == 8235 && + recvmsg(sockets[1], &message, MSG_WAITALL) == 8235 && + check(mapped + 13, 5003, 0) && + check(mapped + 2 * ps + 7, 2222, 5003) && + check(mapped + 4 * ps + 5, 1010, 7225); + ok &= mapped[12] == 0x81 && mapped[5016] == 0x81 && + mapped[2 * ps + 6] == 0x81 && mapped[2 * ps + 2229] == 0x81 && + mapped[4 * ps + 4] == 0x81 && mapped[4 * ps + 1015] == 0x81; + snprintf(name, sizeof(name), "TLS 1.%u scattered receive, empty iovec and subpage boundaries", v + 2); + report(ok, ps, name); + munmap(mapped, 6 * ps); + } + close(sockets[0]); close(sockets[1]); + ok = true; + for (unsigned int round = 0; round < 64 && ok; round++) { + if (!tls_pair(sockets, version)) + ksft_exit_fail_msg("TLS teardown setup errno=%d\n", errno); + fill(p, len); + vec = (struct iovec){ p + ps, 4096 }; + ok = vmsplice(fds[1], &vec, 1, 0) == 4096 && + splice(fds[0], NULL, sockets[0], NULL, 4096, SPLICE_F_MORE) == 4096; + close(sockets[1]); close(sockets[0]); + ok &= !madvise(p + ps, ps, MADV_DONTNEED); + memset(p + ps, 0x75, ps); + } + snprintf(name, sizeof(name), "TLS 1.%u repeated pending-record teardown and reuse", v + 2); + report(ok, ps, name); + } + close(fds[0]); close(fds[1]); +} + +static int exercise(void) +{ + const size_t ps = getauxval(AT_PAGESZ), len = 8 * ps; + unsigned char *p, *back = malloc(len); + int fds[2], dup[2], file, sockets[2]; + struct iovec v[3]; + struct stat self, init; + struct timeval timeout = { .tv_sec = 5 }; + bool ok; + ssize_t result; + + if (stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino) + return 2; + p = mmap((void *)0x300000000UL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (p == MAP_FAILED || !back || pipe(fds) || pipe(dup)) + ksft_exit_fail_msg("setup\n"); + ksft_print_header(); + ksft_set_plan(use_tcp ? 22 : 10); + ksft_print_msg("transport=%s family=%s\n", use_tcp ? "TCP" : use_udp ? "UDP" : "Unix stream", + use_ipv6 ? "IPv6" : "IPv4/Unix"); + fill(p, len); + v[0] = (struct iovec){ NULL, 0 }; + v[1] = (struct iovec){ p + 17, ps - 17 }; + v[2] = (struct iovec){ p + 2 * ps + 31, 2 * ps - 31 }; + result = vmsplice(fds[1], v, 3, 0); + if (result != (ssize_t)(3 * ps - 48)) + ksft_exit_fail_msg("vmsplice returned %zd errno=%d\n", result, errno); + ok = read_exact(fds[0], back, result) && check(back, ps - 17, 17) && + check(back + ps - 17, 2 * ps - 31, 2 * ps + 31); + report(ok, ps, "unaligned fragmented iovecs and empty segment"); + v[0] = (struct iovec){ p, 16384 }; + ok = fcntl(fds[1], F_SETPIPE_SZ, 16384) == 16384; + result = vmsplice(fds[1], v, 1, SPLICE_F_NONBLOCK); + { + int pending = 0; + + ok &= result == 16384 && !ioctl(fds[0], FIONREAD, &pending) && pending == 16384; + } + if (result > 0) + ok &= read_exact(fds[0], back, result) && check(back, result, 0); + report(ok, ps, "contiguous quarters retain one-native-page pipe capacity"); + fcntl(fds[1], F_SETPIPE_SZ, 16 * 16384); + v[0] = (struct iovec){ p + 11, 3 * ps - 11 }; + result = vmsplice(fds[1], v, 1, 0); + if (result != (ssize_t)(3 * ps - 11) || tee(fds[0], dup[1], result, 0) != result) + ksft_exit_fail_msg("tee setup\n"); + ok = replace(p, len); + ok &= read_exact(fds[0], back, 19) && check(back, 19, 11); + ok &= read_exact(fds[0], back + 19, result - 19) && check(back, result, 11); + ok &= read_exact(dup[0], back, result) && check(back, result, 11); + report(ok, ps, "tee and partial reads retain data across unmap/replacement"); + fill(p, len); + v[0] = (struct iovec){ p, ps }; + ok = vmsplice(fds[1], v, 1, 0) == (ssize_t)ps && cow(p, ps); + ok &= read_exact(fds[0], back, ps) && check(back, ps, 0); + report(ok, ps, "pipe GET retains pre-COW data after fork sharer exit"); + fill(p, len); + file = memfd_create("splice-target", 0); + v[0] = (struct iovec){ p + ps + 29, 2 * ps - 29 }; + result = vmsplice(fds[1], v, 1, SPLICE_F_GIFT); + ok = file >= 0 && result == (ssize_t)(2 * ps - 29) && + splice(fds[0], NULL, file, NULL, result, SPLICE_F_MOVE) == result; + ok &= pread(file, back, result, 0) == result && check(back, result, ps + 29); + report(ok, ps, "gifted fragments splice to buffered file"); + close(file); + v[0] = (struct iovec){ p, ps }; + v[1] = (struct iovec){ (void *)0x1000, ps }; + result = vmsplice(fds[1], v, 2, 0); + ok = result == (ssize_t)ps && read_exact(fds[0], back, ps) && check(back, ps, 0); + report(ok, ps, "partial vmsplice before inaccessible iovec"); + if (connected_pair(sockets)) + ksft_exit_fail_msg("socketpair errno=%d\n", errno); + setsockopt(sockets[1], SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + for (unsigned int mode = 0; mode < 3; mode++) { + const size_t bytes = mode == 1 ? ps : + (use_udp && ps == 4096 ? 6 * ps - 256 : 2 * ps - 256); + const size_t offset = mode == 1 ? 0 : ps + 29; + int cork = 1; + + if ((use_tcp || use_udp) && mode < 2 && + setsockopt(sockets[0], use_tcp ? IPPROTO_TCP : IPPROTO_UDP, + use_tcp ? TCP_CORK : UDP_CORK, &cork, sizeof(cork))) + ksft_exit_fail_msg("TCP_CORK setup errno=%d\n", errno); + fill(p, len); + v[0] = (struct iovec){ p + offset, bytes }; + ok = vmsplice(fds[1], v, 1, 0) == (ssize_t)bytes && + splice(fds[0], NULL, sockets[0], NULL, bytes, SPLICE_F_NONBLOCK) == (ssize_t)bytes; + if (!ok) + ksft_exit_fail_msg("socket splice setup errno=%d\n", errno); + if (mode == 2) { + ok = splice(sockets[1], NULL, fds[1], NULL, bytes, 0) == (ssize_t)bytes && + tee(fds[0], dup[1], bytes, 0) == (ssize_t)bytes; + if (!ok) + ksft_exit_fail_msg("socket-to-pipe setup errno=%d\n", errno); + } + if (use_tcp && mode < 2) { + int unsent = 0; + + if (ioctl(sockets[0], SIOCOUTQNSD, &unsent) || unsent != (int)bytes) + ksft_exit_fail_msg("TCP unsent precondition: %d expected %zu errno=%d\n", + unsent, bytes, errno); + } + if (use_udp && mode < 2) { + errno = 0; + if (recv(sockets[1], back, len, MSG_DONTWAIT) != -1 || errno != EAGAIN) + ksft_exit_fail_msg("UDP corked precondition errno=%d\n", errno); + } + ok &= mode == 1 ? cow(p, ps) : replace(p + ps, 2 * ps); + if ((use_tcp || use_udp) && mode < 2) { + cork = 0; + if (setsockopt(sockets[0], use_tcp ? IPPROTO_TCP : IPPROTO_UDP, + use_tcp ? TCP_CORK : UDP_CORK, &cork, sizeof(cork))) { + ksft_print_msg("uncork errno=%d\n", errno); + ok = false; + } + } + ok &= read_exact(mode == 2 ? fds[0] : sockets[1], back, bytes) && check(back, bytes, offset); + if (mode == 2) + ok &= read_exact(dup[0], back, bytes) && check(back, bytes, offset); + report(ok, ps, mode == 0 ? "socket retains quarters across partial unmap/replacement" : + mode == 1 ? "socket GET retains pre-COW data after fork sharer exit" : + "socket-to-pipe and tee retain independent quarter ownership"); + } + close(sockets[0]); + close(sockets[1]); + ok = true; + for (unsigned int round = 0; round < 128 && ok; round++) { + fill(p, len); + ok = !connected_pair(sockets); + v[0] = (struct iovec){ p + ps, ps }; + ok &= vmsplice(fds[1], v, 1, 0) == (ssize_t)ps && + splice(fds[0], NULL, sockets[0], NULL, ps, 0) == (ssize_t)ps; + close(sockets[1]); + close(sockets[0]); + ok &= !madvise(p + ps, ps, MADV_DONTNEED); + memset(p + ps, 0x43, ps); + } + report(ok, ps, "repeated unread socket teardown and quarter reuse"); + if (use_tcp) + tls_checks(ps, p, len, back); + close(fds[0]); close(fds[1]); close(dup[0]); close(dup[1]); + munmap(p, len); + free(back); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +static bool async_cleanup_balanced(void) +{ + const char *names[] = { "user4k_rx_deferred", "user4k_rx_released" }; + unsigned long counts[2]; + char path[128]; + FILE *file = fopen("/sys/module/tls/parameters/user4k_test_async", "r"); + int enabled; + + if (!file) + return false; + enabled = fgetc(file); + fclose(file); + for (unsigned int i = 0; i < 2; i++) { + snprintf(path, sizeof(path), "/sys/module/tls/parameters/%s", names[i]); + file = fopen(path, "r"); + if (!file) + return false; + if (fscanf(file, "%lu", &counts[i]) != 1) { + fclose(file); + return false; + } + fclose(file); + } + printf("# TLS async=%c deferred=%lu released=%lu\n", enabled, counts[0], counts[1]); + return counts[0] == counts[1] && (enabled != 'Y' || counts[0] > 0); +} + +int main(int argc, char **argv) +{ + bool ok = true; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && (!strcmp(argv[1], "--test") || !strcmp(argv[1], "--tcp") || + !strcmp(argv[1], "--tcp6") || !strcmp(argv[1], "--udp") || !strcmp(argv[1], "--udp6")) && getppid() == 1) { + use_tcp = !strncmp(argv[1], "--tcp", 5); + use_udp = !strncmp(argv[1], "--udp", 5); + use_ipv6 = strchr(argv[1], '6') != NULL; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + { + struct ifreq request = { .ifr_name = "lo" }; + int fd = socket(AF_INET, SOCK_DGRAM, 0); + + if (fd < 0 || ioctl(fd, SIOCGIFFLAGS, &request)) + return 4; + request.ifr_flags |= IFF_UP; + if (ioctl(fd, SIOCSIFFLAGS, &request)) + return 4; + close(fd); + } + for (unsigned int mode = 0; mode < 10; mode++) { + unsigned int small = mode & 1; + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, small ? 4096UL : 0UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", mode >= 8 ? "--udp6" : mode >= 6 ? "--udp" : + mode >= 4 ? "--tcp6" : mode >= 2 ? "--tcp" : "--test", NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && WIFEXITED(status) && !WEXITSTATUS(status); + } + ok &= async_cleanup_balanced(); + printf("%s - splice fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/swap-fixture-init.c b/tools/testing/selftests/arm64/user4k/swap-fixture-init.c new file mode 100644 index 0000000000000..e135aee03029a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/swap-fixture-init.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PID-1 runner for a disposable VM with a dedicated virtio swap disk. + * Use kunit.autorun=0: tests skipped at boot stay skipped on a debugfs rerun. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "re"); + char key[128]; + unsigned long value, found = 0; + + if (!f) + return 0; + while (fscanf(f, "%127s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) + found = value; + fclose(f); + return found; +} + +static int native_swap_smoke(void) +{ + const size_t length = 4 * 1024 * 1024, page = 16384; + unsigned long before = swapouts(), after; + unsigned char *p = mmap(NULL, length, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + size_t i; + unsigned int tries; + int ret = 0; + + if (p == MAP_FAILED) + return -1; + for (i = 0; i < length; i++) + p[i] = (i ^ (i >> 12) ^ (i >> 20)) & 255; + if (madvise(p, length, MADV_PAGEOUT)) { + ret = -1; + goto out; + } + for (tries = 0; tries < 100 && swapouts() == before; tries++) + usleep(20000); + after = swapouts(); + for (i = 0; i < length; i++) + if (p[i] != ((i ^ (i >> 12) ^ (i >> 20)) & 255)) { + ret = -1; + break; + } + if (after <= before) + ret = -1; + printf("%s - native %zuK swap transport: pswpout %lu -> %lu, verified %zu bytes\n", + ret ? "not ok" : "ok", page / 1024, before, after, i); +out: + munmap(p, length); + return ret; +} + +static int run_swap_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-swap/"; + char path[128], *log; + size_t used = 0, capacity = 1024 * 1024; + ssize_t n; + int fd, ret = -1; + + snprintf(path, sizeof(path), "%srun", base); + fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = write(fd, "1", 1); + close(fd); + if (n != 1) + return -1; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + log = calloc(1, capacity); + if (!log) { + close(fd); + return -1; + } + while (used < capacity - 1 && (n = read(fd, log + used, capacity - 1 - used)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (n == 0 && used < capacity - 1 && strstr(log, "\nok 1 arm64-user4k-swap\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; + free(log); + return ret; +} + +static int swap_control_helper(const char *operation) +{ + struct stat init, self; + int ret; + + /* A helper is allowed only when this same executable is the VM's PID 1. */ + if (getpid() == 1 || stat("/proc/1/exe", &init) || stat("/proc/self/exe", &self) || + init.st_dev != self.st_dev || init.st_ino != self.st_ino) { + fprintf(stderr, "swap helper requires its fixture runner as PID 1\n"); + return 1; + } + ret = !strcmp(operation, "--swapoff-helper") ? swapoff("/dev/vda") : swapon("/dev/vda", 0); + if (ret) + perror(operation); + return ret ? 1 : 0; +} + +static int pageout_exec(bool small) +{ + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (small && prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(125); + execl("/reclaim-advice", "/reclaim-advice", small ? "--expect-4k" : NULL, NULL); + _exit(126); + } + if (pid <= 0 || waitpid(pid, &status, 0) != pid) + return -1; + printf("# EL0 pageout %s child status=%#x\n", small ? "4K" : "native", status); + return WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +int main(int argc, char **argv) +{ + bool smoke = argc == 2 && !strcmp(argv[1], "--smoke"); + bool pageout = argc == 2 && !strcmp(argv[1], "--pageout"); + bool native_only = argc == 2 && !strcmp(argv[1], "--pageout-native"); + bool active = false; + unsigned int tries; + struct stat st; + pid_t child; + int status, ret = 1; + + if (argc == 2 && (!strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--swapon-helper"))) + return swap_control_helper(argv[1]); + /* This runner must never mount filesystems or enable swap on the host. */ + if (getpid() != 1 || (argc != 1 && !smoke && !pageout && !native_only)) { + fprintf(stderr, "Use only as /init in the dedicated disposable test VM.\n"); + return 1; + } + setvbuf(stdout, NULL, _IONBF, 0); + if (getauxval(AT_PAGESZ) != 16384 || sysconf(_SC_PAGESIZE) != 16384) + goto out; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) + goto out; + child = fork(); + if (!child) { + execl("/page-contract", "page-contract", NULL); + _exit(127); + } + if (child < 0 || waitpid(child, &status, 0) != child || + !WIFEXITED(status) || WEXITSTATUS(status)) + goto out; + for (tries = 0; tries < 100 && stat("/dev/vda", &st); tries++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || swapon("/dev/vda", 0)) + goto out; + active = true; + if (native_swap_smoke()) + goto out; + if (!smoke && !native_only && run_swap_suite()) + goto out; + if ((pageout || native_only) && pageout_exec(false)) + goto out; + if (pageout && pageout_exec(true)) + goto out; + ret = 0; +out: + if (ret) + printf("not ok - swap fixture runner: errno=%d (%s)\n", errno, strerror(errno)); + if (active && swapoff("/dev/vda")) { + perror("swapoff"); + ret = 1; + } + printf("%s - swap fixture runner (%s)\n", ret ? "not ok" : "ok", + smoke ? "native transport smoke only" : "4K swap suite"); + fflush(NULL); + reboot(RB_POWER_OFF); + return ret; +} diff --git a/tools/testing/selftests/arm64/user4k/swapfile-fixture.h b/tools/testing/selftests/arm64/user4k/swapfile-fixture.h new file mode 100644 index 0000000000000..40fd70cbfacad --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/swapfile-fixture.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef USER4K_SWAPFILE_FIXTURE_H +#define USER4K_SWAPFILE_FIXTURE_H + +static const char *swap_path(void) +{ + return access("/swapdir/swapfile", F_OK) ? "/dev/vda" : "/swapdir/swapfile"; +} + +/* Remove every fourth native page: every 64K group crosses file extents. */ +static int prepare_swapfile(void) +{ + const unsigned long page = getauxval(AT_PAGESZ), initial = 32UL << 20; + unsigned char *buf = calloc(1, page); + unsigned long offset; + int fd = -1, ret = -1; + + if (!buf || (mkdir("/swapdir", 0755) && errno != EEXIST) || + mount("/dev/vda", "/swapdir", "ext4", 0, NULL)) + goto out; + fd = open("/swapdir/swapfile", O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, 0600); + if (fd < 0) + goto out; + memset(buf, 0xa5, page); + for (offset = 0; offset < initial; offset += page) + if (write(fd, buf, page) != (ssize_t)page) + goto out; + if (fsync(fd)) + goto out; + for (offset = initial; offset; offset -= 4 * page) + if (fallocate(fd, FALLOC_FL_COLLAPSE_RANGE, offset - 2 * page, page)) + goto out; + memset(buf, 0, page); + ((unsigned int *)(buf + 1024))[0] = 1; + ((unsigned int *)(buf + 1024))[1] = (initial * 3 / 4) / page - 1; + memcpy(buf + page - 10, "SWAPSPACE2", 10); + if (pwrite(fd, buf, page, 0) != (ssize_t)page || fsync(fd)) + goto out; + ret = 0; +out: + if (fd >= 0) + close(fd); + free(buf); + return ret; +} + +#endif diff --git a/tools/testing/selftests/arm64/user4k/thread-fixture-init.c b/tools/testing/selftests/arm64/user4k/thread-fixture-init.c new file mode 100644 index 0000000000000..2fe2615232c4b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/thread-fixture-init.c @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define THREADS 4 +#define ROUNDS 128 +#define PAGES 32 + +static pthread_barrier_t barrier; +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t condition = PTHREAD_COND_INITIALIZER; +static pthread_mutex_t robust; +static _Thread_local unsigned int tls_id; +static _Thread_local volatile sig_atomic_t signals; +static atomic_uint failures; +static unsigned long total; +static unsigned long ps; +static atomic_int waiter_tid; + +static void handler(int sig) +{ + if (sig == SIGUSR1 && tls_id) + signals++; + else + atomic_fetch_add(&failures, 1); +} + +static bool bytes_equal(unsigned char *p, size_t n, unsigned char value) +{ + for (size_t i = 0; i < n; i++) + if (p[i] != value) + return false; + return true; +} + +static void *worker(void *arg) +{ + unsigned int id = (uintptr_t)arg; + unsigned char *p, *heap; + unsigned long request = id & 1 ? 4096 : 0; + bool ok = true; + + tls_id = id; + p = mmap(NULL, PAGES * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (p == MAP_FAILED) + ok = false; + ok &= !prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, request, 0UL, 0UL, 0UL); + pthread_barrier_wait(&barrier); + for (unsigned int round = 0; round < ROUNDS; round++) { + unsigned char pattern = 1 + (id * 31 + round) % 254; + unsigned char residency[PAGES]; + cpu_set_t cpu; + + if (ok) { + CPU_ZERO(&cpu); + CPU_SET((id + round) % 2, &cpu); + ok &= !sched_setaffinity(0, sizeof(cpu), &cpu); + memset(p, pattern, PAGES * ps); + ok &= !mprotect(p + ps, ps, PROT_READ) && p[ps] == pattern; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE); + ok &= !madvise(p + 2 * ps, ps, MADV_FREE); + memset(p + 2 * ps, pattern, ps); + ok &= !madvise(p + 3 * ps, ps, MADV_DONTNEED); + ok &= bytes_equal(p + 3 * ps, ps, 0); + memset(p + 3 * ps, pattern, ps); + heap = malloc(5000 + id * 777); + if (!heap) { + ok = false; + } else { + memset(heap, pattern, 5000 + id * 777); + ok &= bytes_equal(heap, 5000 + id * 777, pattern); + free(heap); + } + ok &= !pthread_kill(pthread_self(), SIGUSR1); + ok &= tls_id == id && signals == (sig_atomic_t)(round + 1); + ok &= prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) == (int)request; + ok &= !mincore(p, PAGES * ps, residency); + ok &= bytes_equal(p, PAGES * ps, pattern); + } + pthread_mutex_lock(&mutex); + total++; + pthread_cond_signal(&condition); + pthread_mutex_unlock(&mutex); + pthread_barrier_wait(&barrier); + } + if (p != MAP_FAILED) + munmap(p, PAGES * ps); + if (!ok) + atomic_fetch_add(&failures, 1); + return NULL; +} + +static void *owner_dies(void *arg) +{ + (void)arg; + if (pthread_mutex_lock(&robust)) + atomic_fetch_add(&failures, 1); + return NULL; +} + +static void *pi_waiter(void *arg) +{ + int error; + + (void)arg; + atomic_store(&waiter_tid, syscall(SYS_gettid)); + error = pthread_mutex_lock(&robust); + if (!error) + error = pthread_mutex_unlock(&robust); + return (void *)(uintptr_t)error; +} + +static bool wait_for_pi_block(void) +{ + for (unsigned int tries = 0; tries < 1000; tries++) { + int tid = atomic_load(&waiter_tid); + char path[80], line[256]; + unsigned long nr, address, operation; + FILE *f; + + if (tid) { + snprintf(path, sizeof(path), "/proc/self/task/%d/syscall", tid); + f = fopen(path, "re"); + if (f) { + bool blocked = fgets(line, sizeof(line), f) && + sscanf(line, "%lu %lx %lx", &nr, &address, &operation) == 3 && + nr == SYS_futex && ((operation & FUTEX_CMD_MASK) == FUTEX_LOCK_PI || + (operation & FUTEX_CMD_MASK) == FUTEX_LOCK_PI2); + + fclose(f); + if (blocked) + return true; + } + } + usleep(1000); + } + return false; +} + +static int exercise(void) +{ + pthread_t threads[THREADS], owner; + pthread_mutexattr_t attributes; + struct sigaction action = { .sa_handler = handler }; + struct stat self, init; + bool ok = true, forks = true; + int error; + + if (stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino) + return 2; + ps = getauxval(AT_PAGESZ); + ksft_print_header(); + ksft_set_plan(6); + if (sigaction(SIGUSR1, &action, NULL) || pthread_barrier_init(&barrier, NULL, THREADS + 1)) + ksft_exit_fail_msg("thread setup\n"); + for (unsigned int i = 0; i < THREADS; i++) + if (pthread_create(&threads[i], NULL, worker, (void *)(uintptr_t)(i + 1))) + ksft_exit_fail_msg("pthread_create\n"); + pthread_barrier_wait(&barrier); + for (unsigned int round = 0; round < ROUNDS; round++) { + pthread_mutex_lock(&mutex); + while (total < (round + 1) * THREADS) + pthread_cond_wait(&condition, &mutex); + pthread_mutex_unlock(&mutex); + ok &= prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) == 0; + if (!(round % 16)) { + pid_t child = fork(); + int status = 0; + + if (!child) { + unsigned char *p = malloc(32768); + + if (!p || tls_id || total != (round + 1) * THREADS || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL)) + _exit(10); + memset(p, 0x75, 32768); + if (!bytes_equal(p, 32768, 0x75)) + _exit(11); + free(p); + _exit(0); + } + forks &= child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + } + pthread_barrier_wait(&barrier); + } + for (unsigned int i = 0; i < THREADS; i++) + ok &= !pthread_join(threads[i], NULL); + ksft_test_result(ok && !atomic_load(&failures) && total == THREADS * ROUNDS, + "%luK %u concurrent memory/TLS/signal/exec-request rounds across %u threads\n", + ps / 1024, ROUNDS, THREADS); + ksft_test_result(forks, "%luK fork and allocator from a multithreaded process\n", ps / 1024); + pthread_barrier_destroy(&barrier); + ok = !pthread_mutexattr_init(&attributes); + ok &= !pthread_mutexattr_setrobust(&attributes, PTHREAD_MUTEX_ROBUST); + ok &= !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + ksft_exit_fail_msg("robust mutex setup\n"); + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + ok &= !pthread_mutex_unlock(&robust); + ksft_test_result(ok, "%luK robust futex owner-death recovery\n", ps / 1024); + pthread_mutex_destroy(&robust); + /* Exercise repeated TLS/stack allocation and set_tid_address clear/wake. */ + for (unsigned int i = 0; i < 128; i++) { + ok = !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + break; + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + pthread_mutex_unlock(&robust); + pthread_mutex_destroy(&robust); + if (!ok) + break; + } + ksft_test_result(ok && !atomic_load(&failures), "%luK repeated thread exit/join and robust recovery\n", ps / 1024); + ok = !pthread_mutexattr_setprotocol(&attributes, PTHREAD_PRIO_INHERIT); + ok &= !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_mutex_lock(&robust); + ok &= !pthread_create(&owner, NULL, pi_waiter, NULL); + if (!ok) + ksft_exit_fail_msg("PI mutex setup\n"); + ok &= wait_for_pi_block(); + ok &= !pthread_mutex_unlock(&robust); + { + void *result = (void *)1; + + ok &= !pthread_join(owner, &result) && !result; + } + ksft_test_result(ok, "%luK confirmed contended priority-inheritance futex handoff\n", ps / 1024); + ok = !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + ksft_exit_fail_msg("PI owner setup\n"); + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + ok &= !pthread_mutex_unlock(&robust); + pthread_mutex_destroy(&robust); + pthread_mutexattr_destroy(&attributes); + ksft_test_result(ok, "%luK priority-inheritance robust owner-death recovery\n", ps / 1024); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +static pthread_barrier_t exec_barrier; + +static void *exec_sibling(void *arg) +{ + (void)arg; + pthread_barrier_wait(&exec_barrier); + for (;;) + pause(); + return NULL; +} + +static void *exec_nonleader(void *arg) +{ + unsigned long target = (uintptr_t)arg; + char size[32]; + + snprintf(size, sizeof(size), "%lu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, + 0UL, 0UL, 0UL)) + _exit(30); + pthread_barrier_wait(&exec_barrier); + execl("/init", "/init", "--after-thread-exec", size, NULL); + _exit(31); +} + +static int thread_exec(unsigned long target) +{ + pthread_t siblings[2], executor; + + if (pthread_barrier_init(&exec_barrier, NULL, 4)) + return 32; + for (unsigned int i = 0; i < 2; i++) + if (pthread_create(&siblings[i], NULL, exec_sibling, NULL)) + return 33; + if (pthread_create(&executor, NULL, exec_nonleader, (void *)(uintptr_t)target)) + return 34; + pthread_barrier_wait(&exec_barrier); + for (;;) + pause(); + return 35; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long granules[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + unsigned int nr = granules[0] > 16384 ? 3 : 2; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--thread-exec") && getppid() == 1) + return thread_exec(strtoul(argv[2], NULL, 10)); + if (argc == 3 && !strcmp(argv[1], "--after-thread-exec") && getppid() == 1) { + unsigned long expected = strtoul(argv[2], NULL, 10); + + return getauxval(AT_PAGESZ) != expected || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) != 0; + } + if (argc == 3 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + char size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", size, NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + for (unsigned int source = 0; source < nr; source++) { + for (unsigned int target = 0; target < nr; target++) { + pid_t pid = fork(); + int status = 0; + bool passed; + + if (!pid) { + char size[32]; + + snprintf(size, sizeof(size), "%lu", granules[target]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[source], + 0UL, 0UL, 0UL)) + _exit(36); + execl("/init", "/init", "--thread-exec", size, NULL); + _exit(37); + } + passed = pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - non-leader exec %luK -> %luK status=%#x\n", + passed ? "ok" : "not ok", granules[source] / 1024, granules[target] / 1024, status); + ok &= passed; + } + } + printf("%s - thread fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/uffd-move-init.c b/tools/testing/selftests/arm64/user4k/uffd-move-init.c new file mode 100644 index 0000000000000..621a553250e16 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uffd-move-init.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) do { if (!(x)) { printf("not ok - MOVE line=%d errno=%d (%s)\n", __LINE__, errno, strerror(errno)); exit(1); } } while (0) +#define PRESENT (1ULL << 63) +#define SWAPPED (1ULL << 62) +#define VALUE_MASK ((1ULL << 55) - 1) +static uint64_t entry(void *p, size_t ps) +{ + uint64_t e; int fd=open("/proc/self/pagemap", O_RDONLY|O_CLOEXEC); + CHECK(fd>=0 && pread(fd,&e,8,(uintptr_t)p/ps*8)==8); CHECK(!close(fd)); return e; +} +static uint64_t physical(void *p,size_t ps) +{ + uint64_t e=entry(p,ps); CHECK(e&PRESENT); return (e&VALUE_MASK)*ps; +} +static unsigned char pattern(size_t i) { return 1+((i^(i>>12)^(i>>16))%251); } +static void bytes(unsigned char *p,size_t original,size_t len) +{ + for(size_t i=0;isrc; + atomic_store(&r->ready,1);while(!atomic_load(&r->done))sched_yield(); + CHECK(*(volatile unsigned char*)r->src==0 && *(volatile unsigned char*)r->dst==old); + return NULL; +} +struct pin_racer { + void *a,*b;size_t ps;atomic_int stop;atomic_uint accepted,rejected; +}; +static void *race_pins(void *arg) +{ + struct pin_racer *r=arg;struct io_uring_params p={0};on_cpu(1); + int fd=syscall(SYS_io_uring_setup,2,&p);CHECK(fd>=0);unsigned round=0; + while(!atomic_load(&r->stop)){ + struct iovec iov={.iov_base=(round++&1)?r->a:r->b,.iov_len=r->ps}; + int ret=syscall(SYS_io_uring_register,fd,IORING_REGISTER_BUFFERS,&iov,1); + if(!ret){atomic_fetch_add(&r->accepted,1);usleep(50);CHECK(!syscall(SYS_io_uring_register,fd,IORING_UNREGISTER_BUFFERS,NULL,0));} + else {CHECK(errno==EFAULT||errno==EAGAIN);atomic_fetch_add(&r->rejected,1);} + } + CHECK(!close(fd));return NULL; +} +static void concurrent_move(size_t ps) +{ + unsigned char *a=mmap((void*)0x50000000,ps,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED_NOREPLACE,-1,0); + unsigned char *b=mmap((void*)0x60000000,ps,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED_NOREPLACE,-1,0); + CHECK(a!=MAP_FAILED&&b!=MAP_FAILED);memset(a,0x6d,ps);uint64_t phys=physical(a,ps); + int fd=syscall(SYS_userfaultfd,O_CLOEXEC|O_NONBLOCK|UFFD_USER_MODE_ONLY);CHECK(fd>=0); + struct uffdio_api api={.api=UFFD_API,.features=UFFD_FEATURE_MOVE};CHECK(!ioctl(fd,UFFDIO_API,&api)); + struct uffdio_register reg={.range={(uintptr_t)a,ps},.mode=UFFDIO_REGISTER_MODE_MISSING};CHECK(!ioctl(fd,UFFDIO_REGISTER,®)); + reg.range.start=(uintptr_t)b;CHECK(!ioctl(fd,UFFDIO_REGISTER,®)); + struct pin_racer race={.a=a,.b=b,.ps=ps};pthread_t thread;CHECK(!pthread_create(&thread,NULL,race_pins,&race)); + while(!atomic_load(&race.accepted))sched_yield(); + unsigned char *cur=a,*next=b;unsigned moved=0,busy=0; + for(unsigned tries=0;moved<256&&tries<100000;tries++){ + struct uffdio_move m={.src=(uintptr_t)cur,.dst=(uintptr_t)next,.len=ps}; + int ret=ioctl(fd,UFFDIO_MOVE,&m); + if(ret){CHECK(errno==EBUSY&&m.move==-EBUSY);busy++;sched_yield();continue;} + CHECK(m.move==(long long)ps&&physical(next,ps)==phys&&!(entry(cur,ps)&(PRESENT|SWAPPED))); + unsigned char *tmp=cur;cur=next;next=tmp;moved++; + } + atomic_store(&race.stop,1);CHECK(!pthread_join(thread,NULL));CHECK(moved==256&&atomic_load(&race.accepted)>0); + for(size_t i=0;i=0); + struct uffdio_api api={.api=UFFD_API,.features=UFFD_FEATURE_MOVE};CHECK(!ioctl(fd,UFFDIO_API,&api)); + struct uffdio_register reg={.range={(uintptr_t)a,2*native},.mode=UFFDIO_REGISTER_MODE_MISSING};CHECK(!ioctl(fd,UFFDIO_REGISTER,®)); + reg.range.start=(uintptr_t)b;CHECK(!ioctl(fd,UFFDIO_REGISTER,®)); + /* Prepare both roots and target tables using a different native folio. */ + move(fd,b+native,a+native,ps,0,0); + uint64_t phys=physical(a,ps);int inject=open("/proc/self/fail-nth",O_RDWR|O_CLOEXEC);CHECK(inject>=0); + unsigned char *src=a,*dst=b; + for(unsigned phase=0;phase<2;phase++){ + struct uffdio_move m={.src=(uintptr_t)src,.dst=(uintptr_t)dst,.len=ps};char pending[16]={0}; + CHECK(write(inject,"1",1)==1);errno=0;int ret=ioctl(fd,UFFDIO_MOVE,&m),saved=errno; + CHECK(pread(inject,pending,sizeof(pending)-1,0)>0);CHECK(write(inject,"0",1)==1); + CHECK(ret==-1&&saved==ENOMEM&&m.move==-ENOMEM&&strtoul(pending,NULL,10)==0); + CHECK(physical(src,ps)==phys&&!(entry(dst,ps)&(PRESENT|SWAPPED))); + for(size_t i=0;i=0); + struct uffdio_api api={.api=UFFD_API,.features=UFFD_FEATURE_MOVE}; + CHECK(!ioctl(fd,UFFDIO_API,&api) && (api.features&UFFD_FEATURE_MOVE)); + struct uffdio_register reg={.range={(uintptr_t)dst,len},.mode=UFFDIO_REGISTER_MODE_MISSING}; + CHECK(!ioctl(fd,UFFDIO_REGISTER,®) && (reg.ioctls&(1ULL<<_UFFDIO_MOVE))); + uint64_t pfns[256];CHECK(native/ps<=sizeof(pfns)/sizeof(pfns[0])); + for(size_t i=0;i=0); + struct iovec pin={.iov_base=src+2*ps,.iov_len=ps}; + CHECK(!syscall(SYS_io_uring_register,ring,IORING_REGISTER_BUFFERS,&pin,1)); + p=physical(src+2*ps,ps); + CHECK(move(fd,dst+2*ps,src+2*ps,ps,0,EBUSY)==-EBUSY); + CHECK(physical(src+2*ps,ps)==p && !(entry(dst+2*ps,ps)&(PRESENT|SWAPPED)));bytes(src+2*ps,2*ps,ps); + uint64_t neighbour=physical(src+ps,ps); + move(fd,dst+3*ps,src+ps,ps,0,0);CHECK(physical(dst+3*ps,ps)==neighbour);bytes(dst+3*ps,ps,ps); + CHECK(!syscall(SYS_io_uring_register,ring,IORING_UNREGISTER_BUFFERS,NULL,0));CHECK(!close(ring)); + move(fd,dst+2*ps,src+2*ps,ps,0,0);CHECK(physical(dst+2*ps,ps)==p);bytes(dst+2*ps,2*ps,ps); + printf("ok - MOVE %zuK source PIN rejected atomically, neighbour PIN allowed, unpin retry\n",ps/1024); + + CHECK(move(fd,dst+ps,src+4*ps,ps,0,EEXIST)==-EEXIST);bytes(src+4*ps,4*ps,ps);bytes(dst+ps,3*ps,ps); + CHECK(!madvise(src+5*native,native,MADV_DONTNEED));CHECK(*(volatile unsigned char*)(src+5*native)==0); + move(fd,dst+5*native,src+5*native,ps,0,0);CHECK(dst[5*native]==0);dst[5*native]=0x7d;CHECK(src[5*native]==0); + CHECK(!madvise(src+6*native,native,MADV_DONTNEED)); + CHECK(move(fd,dst+6*native,src+6*native,ps,0,ENOENT)==-ENOENT); + move(fd,dst+6*native,src+6*native,ps,UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES,0); + CHECK(!(entry(dst+6*native,ps)&(PRESENT|SWAPPED))); + CHECK(!madvise(src+10*native+ps,ps,MADV_DONTNEED)); + p=physical(src+10*native,ps); + CHECK(move(fd,dst+12*native,src+10*native,2*ps,0,EAGAIN)==(long)ps); + CHECK(physical(dst+12*native,ps)==p);bytes(dst+12*native,10*native,ps); + CHECK(!(entry(dst+12*native+ps,ps)&(PRESENT|SWAPPED))); + printf("ok - MOVE %zuK collision, zero, holes and partial progress\n",ps/1024); + + int pipefd[2];CHECK(!pipe(pipefd));pid_t child=fork();CHECK(child>=0); + if(!child){close(pipefd[1]);char c;_exit(read(pipefd[0],&c,1)==1?0:1);} + close(pipefd[0]);CHECK(move(fd,dst+11*native,src+12*native,ps,0,EBUSY)==-EBUSY); + CHECK(write(pipefd[1],"x",1)==1);close(pipefd[1]);int status;CHECK(waitpid(child,&status,0)==child && status==0); + *(volatile unsigned char*)(src+12*native)=pattern(12*native); + p=physical(src+12*native,ps);move(fd,dst+11*native,src+12*native,ps,0,0);CHECK(physical(dst+11*native,ps)==p);bytes(dst+11*native,12*native,ps); + printf("ok - MOVE %zuK fork-shared source rejected, exclusive write/retry\n",ps/1024); + + size_t off=ps16384?3U:2U);i++){ + pid_t c=fork();int status=0;CHECK(c>=0); + if(!c){char n[32];snprintf(n,sizeof(n),"%zu",native);if(prctl(SET_EXEC_PAGE_SIZE,sizes[i],0UL,0UL,0UL))_exit(4);execl("/init","/init","--exercise",n,NULL);_exit(5);} + CHECK(waitpid(c,&status,0)==c);printf("%s - MOVE ABI %zuK status=%#x\n",status?"not ok":"ok",sizes[i]/1024,status);failed+=!!status; + } + if(swapoff("/dev/vda"))failed++; + printf("UFFD MOVE CONTRACT %s\n",failed?"FAIL":"PASS");sync();reboot(RB_POWER_OFF);for(;;)pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c new file mode 100644 index 0000000000000..ce37acf0ea970 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c @@ -0,0 +1,664 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) do { if (!(x)) { printf("not ok - uffd line=%d errno=%d\n", __LINE__, errno); exit(1); } } while (0) +struct writer { volatile unsigned char *address; unsigned char value; atomic_int done, tid; }; +static void *write_byte(void *opaque) +{ + struct writer *w = opaque; + atomic_store(&w->tid, syscall(SYS_gettid)); + *w->address = w->value; + atomic_store(&w->done, 1); + return NULL; +} +struct forker { unsigned char *base; size_t page_size; pid_t pid; }; +static void *fork_byte(void *opaque) +{ + struct forker *f = opaque; + f->pid = fork(); + if (!f->pid) { + alarm(10); + f->base[f->page_size + 17] = 0xa5; + _exit(f->base[f->page_size + 17] != 0xa5); + } + return NULL; +} +static struct uffd_msg read_event(int fd) +{ + struct pollfd p = {.fd = fd, .events = POLLIN}; + struct uffd_msg msg; + CHECK(poll(&p, 1, 10000) == 1 && (p.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + return msg; +} +static void protect(int fd, void *start, size_t len, unsigned long mode) +{ + struct uffdio_writeprotect wp = { .range = {(uintptr_t)start, len}, .mode = mode }; + CHECK(!ioctl(fd, UFFDIO_WRITEPROTECT, &wp)); +} +static void resolve_write(int fd, unsigned char *base, size_t ps, unsigned page, + unsigned char value, int exact) +{ + struct writer w = { .address = base + page * ps + 17, .value = value }; + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + struct uffd_msg msg; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT); + CHECK(msg.arg.pagefault.flags == (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_WRITE)); + CHECK(msg.arg.pagefault.address == (uintptr_t)(base + page * ps + (exact ? 17 : 0))); + CHECK(msg.arg.pagefault.feat.ptid == (unsigned int)atomic_load(&w.tid)); + CHECK(!atomic_load(&w.done)); + protect(fd, base + page * ps, ps, UFFDIO_WRITEPROTECT_MODE_DONTWAKE); + struct uffdio_range range = { (uintptr_t)(base + page * ps), ps }; + CHECK(!ioctl(fd, UFFDIO_WAKE, &range)); + CHECK(!pthread_join(thread, NULL) && atomic_load(&w.done)); + CHECK(base[page * ps + 17] == value); + CHECK(poll(&pollfd, 1, 0) == 0); +} +static void fill(int fd, void *dst, void *src, size_t len, unsigned long mode) +{ + struct uffdio_copy copy = {.dst = (uintptr_t)dst, .src = (uintptr_t)src, + .len = len, .mode = mode}; + CHECK(!ioctl(fd, UFFDIO_COPY, ©) && copy.copy == (long long)len); +} +static void zero(int fd, void *dst, size_t len) +{ + struct uffdio_zeropage z = {.range = {(uintptr_t)dst, len}}; + CHECK(!ioctl(fd, UFFDIO_ZEROPAGE, &z) && z.zeropage == (long long)len); +} +static uint64_t physical(void *address, size_t ps) +{ + uint64_t entry; + int fd = open("/proc/self/pagemap", O_RDONLY); + CHECK(fd >= 0 && pread(fd, &entry, 8, (uintptr_t)address / ps * 8) == 8); + CHECK(!close(fd) && (entry & (1ULL << 63))); + return (entry & ((1ULL << 55) - 1)) * ps; +} +static void missing(size_t ps, size_t native) +{ + size_t len = 4 * native; + unsigned char *map = mmap(NULL, len + native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *src = mmap(NULL, len + native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(map != MAP_FAILED && src != MAP_FAILED); + unsigned char *p = map + ps; + CHECK(!mlock2(p, len, MLOCK_ONFAULT)); + for (size_t i = 0; i < len; i++) src[i] = (i / ps) ^ (i * 7 + 3); + int fd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = {.api = UFFD_API, .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP}; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = {.range = {(uintptr_t)p, len}, + .mode = UFFDIO_REGISTER_MODE_MISSING | UFFDIO_REGISTER_MODE_WP}; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK((reg.ioctls & ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE))) == + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE))); + /* A write blocks only on its missing leaf; COPY_MODE_WP causes a second event. */ + struct writer w = {.address = p + 17, .value = 0xd3}; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == UFFD_PAGEFAULT_FLAG_WRITE && !atomic_load(&w.done)); + fill(fd, p, src, ps, UFFDIO_COPY_MODE_WP); + msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == (UFFD_PAGEFAULT_FLAG_WRITE | UFFD_PAGEFAULT_FLAG_WP)); + protect(fd, p, ps, 0); + CHECK(!pthread_join(thread, NULL) && atomic_load(&w.done)); + src[17] = w.value; + CHECK(!memcmp(p, src, ps)); + zero(fd, p + ps, ps); + for (size_t i = 0; i < ps; i++) CHECK(p[ps + i] == 0); + p[ps + 17] = 0xe4; + CHECK(!memcmp(p, src, ps)); + /* A batch starts at a user-page boundary, not necessarily a native one. */ + fill(fd, p + 2 * ps, src + 2 * ps, len - 2 * ps, 0); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, len - 2 * ps)); + if (ps < native) { + unsigned char *a = (void *)(((uintptr_t)(p + 2 * ps) + native - 1) & ~(native - 1)); + uint64_t start = physical(a, ps); + CHECK(start != 0 && !(start & (native - 1))); + for (size_t i = ps; i < native; i += ps) + CHECK(physical(a + i, ps) == start + i); + } + struct uffdio_copy copy = {.dst = (uintptr_t)p, .src = (uintptr_t)src, .len = ps}; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EEXIST && copy.copy == -EEXIST); + CHECK(!munlock(p, len)); + /* Preserve the successfully copied prefix when the next PTE already exists. */ + CHECK(!madvise(p, ps, MADV_DONTNEED)); + copy = (struct uffdio_copy){.dst = (uintptr_t)p, .src = (uintptr_t)src, .len = 2 * ps}; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EAGAIN && copy.copy == (long long)ps); + CHECK(!memcmp(p, src, ps) && p[ps + 17] == 0xe4); + /* An invalid source stops at its leaf, without installing it or touching neighbours. */ + CHECK(!madvise(p + 2 * ps, 2 * ps, MADV_DONTNEED)); + CHECK(!mprotect(src + 3 * ps, ps, PROT_NONE)); + copy = (struct uffdio_copy){.dst = (uintptr_t)(p + 2 * ps), .src = (uintptr_t)(src + 2 * ps), .len = 2 * ps}; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EAGAIN && copy.copy == (long long)ps); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, ps)); + CHECK(!mprotect(src + 3 * ps, ps, PROT_READ | PROT_WRITE)); + /* A fresh source forces the unlocked copy retry. */ + CHECK(!madvise(src + 3 * ps, ps, MADV_DONTNEED)); + fill(fd, p + 3 * ps, src + 3 * ps, ps, 0); + for (size_t i = 0; i < ps; i++) CHECK(p[3 * ps + i] == 0); + CHECK(!memcmp(p + 4 * ps, src + 4 * ps, len - 4 * ps)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { p[2 * ps + 9] ^= 0xff; _exit(p[2 * ps + 9] == src[2 * ps + 9]); } + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, ps)); + CHECK(!close(fd)); + CHECK(!munmap(map, len + native) && !munmap(src, len + native)); + printf("ok - uffd ABI %zuK missing copy/WP, zero, packed batch, partial EEXIST/EFAULT, source retry and COW\n", ps / 1024); +} + +struct reader { volatile unsigned char *address; unsigned char value; atomic_int done; }; +static void *read_byte(void *opaque) +{ + struct reader *r = opaque; + r->value = *r->address; + atomic_store(&r->done, 1); + return NULL; +} +static void resume_minor(int fd, void *start, size_t len, unsigned long mode) +{ + struct uffdio_continue c = {.range = {(uintptr_t)start, len}, .mode = mode}; + CHECK(!ioctl(fd, UFFDIO_CONTINUE, &c) && c.mapped == (long long)len); +} +static void file_minor(size_t ps, size_t native) +{ + for (int shared = 0; shared < 2; shared++) { + int memfd = memfd_create("uffd-minor-granule", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * native)); + unsigned char *alias = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, MAP_SHARED, memfd, 0); + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + shared ? MAP_SHARED : MAP_PRIVATE, memfd, ps); + CHECK(alias != MAP_FAILED && p != MAP_FAILED); + CHECK(!mlock2(p, 3 * ps, MLOCK_ONFAULT)); + for (size_t i = 0; i < 4 * native; i++) alias[i] = (i / ps) ^ (i * 3 + 7); + int fd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = {.api = UFFD_API, .features = UFFD_FEATURE_MINOR_SHMEM | UFFD_FEATURE_PAGEFAULT_FLAG_WP}; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = {.range = {(uintptr_t)p, 3 * ps}, + .mode = UFFDIO_REGISTER_MODE_MINOR | UFFDIO_REGISTER_MODE_WP}; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(reg.ioctls & (1ULL << _UFFDIO_CONTINUE)); + + for (unsigned i = 0; i < 3; i++) { + struct reader r = {.address = p + i * ps + 17}; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && msg.arg.pagefault.address == (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == UFFD_PAGEFAULT_FLAG_MINOR && !atomic_load(&r.done)); + alias[(i + 1) * ps + 17] ^= 0x3c; + resume_minor(fd, p + i * ps, ps, UFFDIO_CONTINUE_MODE_WP | UFFDIO_CONTINUE_MODE_DONTWAKE); + struct uffdio_range range = {(uintptr_t)(p + i * ps), ps}; + CHECK(!ioctl(fd, UFFDIO_WAKE, &range)); + CHECK(!pthread_join(thread, NULL) && atomic_load(&r.done)); + CHECK(r.value == alias[(i + 1) * ps + 17]); + CHECK(physical(p + i * ps, ps) == physical(alias + (i + 1) * ps, ps)); + /* Independent write protection survives installing the file leaf. */ + struct writer w = {.address = p + i * ps + 17, .value = 0xa1 + i}; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && msg.arg.pagefault.address == (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_WRITE)); + protect(fd, p + i * ps, ps, 0); + CHECK(!pthread_join(thread, NULL) && atomic_load(&w.done)); + CHECK(p[i * ps + 17] == w.value); + CHECK(shared ? alias[(i + 1) * ps + 17] == w.value : alias[(i + 1) * ps + 17] == r.value); + } + struct pollfd pollfd = {.fd = fd, .events = POLLIN}; + CHECK(poll(&pollfd, 1, 0) == 0); + CHECK(!munlock(p, 3 * ps)); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + resume_minor(fd, p, 3 * ps, 0); + CHECK(!memcmp(p, alias + ps, 3 * ps)); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + CHECK(!ftruncate(memfd, ps + 17)); + resume_minor(fd, p, ps, 0); + struct uffdio_continue c = {.range = {(uintptr_t)(p + ps), ps}}; + CHECK(ioctl(fd, UFFDIO_CONTINUE, &c) == -1 && errno == EFAULT && c.mapped == -EFAULT); + CHECK(!close(fd) && !munmap(p, 3 * ps) && !munmap(alias, 4 * native) && !close(memfd)); + printf("ok - uffd ABI %zuK shmem %s minor/WP, byte offsets, alias coherence, COW, batch continue and EOF\n", ps / 1024, shared ? "shared" : "private"); + } +} + +static uint64_t pagemap(void *address, size_t ps) +{ + uint64_t entry; + int fd = open("/proc/self/pagemap", O_RDONLY); + CHECK(fd >= 0 && pread(fd, &entry, 8, (uintptr_t)address / ps * 8) == 8); + CHECK(!close(fd)); + return entry; +} +static sigjmp_buf bus_env; +static void *bus_address; +static void bus_handler(int sig, siginfo_t *info, void *context) +{ + (void)context; + if (sig != SIGBUS) _exit(91); + bus_address = info->si_addr; + siglongjmp(bus_env, 1); +} +static void markers(size_t ps) +{ + for (int async = 0; async < 2; async++) { + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = {.api = UFFD_API, .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_THREAD_ID | UFFD_FEATURE_WP_UNPOPULATED | (async ? UFFD_FEATURE_WP_ASYNC : 0)}; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = {.range = {(uintptr_t)p, 3 * ps}, .mode = UFFDIO_REGISTER_MODE_WP}; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + protect(fd, p, 3 * ps, UFFDIO_WRITEPROTECT_MODE_WP); + for (unsigned i = 0; i < 3; i++) CHECK(pagemap(p + i * ps, ps) & (1ULL << 57)); + CHECK(*(volatile unsigned char *)(p + ps + 17) == 0); + if (async) p[ps + 17] = 0x67; + else resolve_write(fd, p, ps, 1, 0x67, 0); + CHECK(p[ps + 17] == 0x67); + CHECK(pagemap(p, ps) & (1ULL << 57)); + CHECK(!(pagemap(p + ps, ps) & (1ULL << 57))); + CHECK(pagemap(p + 2 * ps, ps) & (1ULL << 57)); + struct pollfd pollfd = {.fd = fd, .events = POLLIN}; + CHECK(poll(&pollfd, 1, 0) == 0); + CHECK(!close(fd)); + p[17] = 0x68; p[2 * ps + 17] = 0x69; + CHECK(!munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK unpopulated WP async=%d per-leaf pagemap state and close\n", ps / 1024, async); + } + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = {.api = UFFD_API, .features = UFFD_FEATURE_POISON}; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = {.range = {(uintptr_t)p, 3 * ps}, .mode = UFFDIO_REGISTER_MODE_MISSING}; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®) && (reg.ioctls & (1ULL << _UFFDIO_POISON))); + zero(fd, p, ps); zero(fd, p + 2 * ps, ps); + struct uffdio_poison poison = {.range = {(uintptr_t)(p + ps), ps}}; + CHECK(!ioctl(fd, UFFDIO_POISON, &poison) && poison.updated == (long long)ps); + CHECK(ioctl(fd, UFFDIO_POISON, &poison) == -1 && errno == EEXIST && poison.updated == -EEXIST); + struct sigaction sa = {.sa_sigaction = bus_handler, .sa_flags = SA_SIGINFO}, old; + CHECK(!sigemptyset(&sa.sa_mask) && !sigaction(SIGBUS, &sa, &old)); + if (!sigsetjmp(bus_env, 1)) { (void)*(volatile unsigned char *)(p + ps + 17); CHECK(0); } + CHECK(bus_address == p + ps + 17); + CHECK(!sigaction(SIGBUS, &old, NULL)); + p[17] = 0x81; p[2 * ps + 17] = 0x82; + CHECK(!close(fd) && !munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK poisoned leaf signals SIGBUS and preserves neighbours\n", ps / 1024); +} +struct pager_request { uintptr_t address; size_t size; }; +static void send_pager(int sock, int fd, struct pager_request *request) +{ + union { struct cmsghdr align; char bytes[CMSG_SPACE(sizeof(int))]; } control = {0}; + struct iovec iov = {.iov_base = request, .iov_len = sizeof(*request)}; + struct msghdr msg = {.msg_iov = &iov, .msg_iovlen = 1, .msg_control = control.bytes, .msg_controllen = sizeof(control)}; + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + c->cmsg_level = SOL_SOCKET; c->cmsg_type = SCM_RIGHTS; c->cmsg_len = CMSG_LEN(sizeof(int)); + memcpy(CMSG_DATA(c), &fd, sizeof(fd)); + CHECK(sendmsg(sock, &msg, 0) == sizeof(*request)); +} +static unsigned char pager_pattern(size_t i) { return (i / 4096) ^ (i * 11 + 5); } +static int pager_worker(int sock, size_t expected) +{ + CHECK(getauxval(AT_PAGESZ) == expected); + alarm(30); + struct pager_request request; + union { struct cmsghdr align; char bytes[CMSG_SPACE(sizeof(int))]; } control = {0}; + struct iovec iov = {.iov_base = &request, .iov_len = sizeof(request)}; + struct msghdr msg = {.msg_iov = &iov, .msg_iovlen = 1, .msg_control = control.bytes, .msg_controllen = sizeof(control)}; + CHECK(recvmsg(sock, &msg, 0) == sizeof(request)); + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + CHECK(c && c->cmsg_level == SOL_SOCKET && c->cmsg_type == SCM_RIGHTS && c->cmsg_len == CMSG_LEN(sizeof(int))); + int fd; + memcpy(&fd, CMSG_DATA(c), sizeof(fd)); + size_t ps = request.size; + unsigned char *source = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(source != MAP_FAILED); + for (size_t i = 0; i < 4 * ps; i++) source[i] = pager_pattern(i); + for (unsigned i = 0; i < 3; i++) { + struct uffd_msg fault = read_event(fd); + CHECK(fault.event == UFFD_EVENT_PAGEFAULT && fault.arg.pagefault.address == request.address + i * ps && !fault.arg.pagefault.flags); + if (i == 1) zero(fd, (void *)(request.address + ps), ps); + else fill(fd, (void *)(request.address + i * ps), source + i * ps, (i == 2 ? 2 : 1) * ps, 0); + } + CHECK(!close(fd) && !close(sock) && !munmap(source, 4 * ps)); + return 0; +} +static void cross_pager(size_t ps, size_t native, int shared) +{ + size_t pager_ps = ps == native ? 4096 : native; + int socks[2]; + CHECK(!socketpair(AF_UNIX, SOCK_SEQPACKET, 0, socks)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + char fd_arg[32], ps_arg[32]; + CHECK(!close(socks[0])); + snprintf(fd_arg, sizeof(fd_arg), "%d", socks[1]); + snprintf(ps_arg, sizeof(ps_arg), "%zu", pager_ps); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, pager_ps, 0UL, 0UL, 0UL)); + execl("/init", "/init", "--pager", fd_arg, ps_arg, NULL); _exit(3); + } + CHECK(!close(socks[1])); + int memfd = shared ? memfd_create("remote-uffd-shmem", 0) : -1; + if (shared) CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS, memfd, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = {.api = UFFD_API}; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = {.range = {(uintptr_t)p, 4 * ps}, .mode = UFFDIO_REGISTER_MODE_MISSING}; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + struct pager_request request = {.address = (uintptr_t)p, .size = ps}; + send_pager(socks[0], fd, &request); + for (size_t i = 0; i < 4 * ps; i++) CHECK(p[i] == (i >= ps && i < 2 * ps ? 0 : pager_pattern(i))); + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(!close(fd) && !close(socks[0]) && !munmap(p, 4 * ps)); + if (shared) CHECK(!close(memfd)); + printf("ok - uffd target %zuK remote pager %zuK shared=%d SCM_RIGHTS copy/zero/batch uses target granule\n", ps / 1024, pager_ps / 1024, shared); +} + +static void private_missing(size_t ps) +{ + int memfd = memfd_create("uffd-private-hole", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, MAP_PRIVATE, memfd, ps); + unsigned char *src = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && src != MAP_FAILED); + for (size_t i = 0; i < 3 * ps; i++) src[i] = pager_pattern(i); + int fd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = {.api = UFFD_API}; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = {.range = {(uintptr_t)p, 3 * ps}, .mode = UFFDIO_REGISTER_MODE_MISSING}; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®) && (reg.ioctls & (1ULL << _UFFDIO_COPY))); + struct reader r = {.address = p + 17}; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && msg.arg.pagefault.address == (uintptr_t)p && !msg.arg.pagefault.flags); + fill(fd, p, src, 2 * ps, 0); + CHECK(!pthread_join(thread, NULL) && r.value == src[17]); + zero(fd, p + 2 * ps, ps); + CHECK(!memcmp(p, src, 2 * ps)); + unsigned char bytes[256]; + for (size_t off = 0; off < 4 * ps; off += sizeof(bytes)) { + CHECK(pread(memfd, bytes, sizeof(bytes), off) == sizeof(bytes)); + for (unsigned i = 0; i < sizeof(bytes); i++) CHECK(bytes[i] == 0); + } + CHECK(!madvise(p, ps, MADV_DONTNEED)); + CHECK(!madvise(src, 3 * ps, MADV_DONTNEED)); + fill(fd, p, src, ps, 0); + for (size_t i = 0; i < ps; i++) CHECK(p[i] == 0); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + CHECK(!ftruncate(memfd, ps + 17)); + struct uffdio_copy copy = {.dst = (uintptr_t)(p + ps), .src = (uintptr_t)src, .len = ps}; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EFAULT && copy.copy == -EFAULT); + fill(fd, p, src, ps, 0); + CHECK(!close(fd) && !close(memfd) && !munmap(p, 3 * ps) && !munmap(src, 3 * ps)); + printf("ok - uffd ABI %zuK private shmem missing COPY/ZERO, unchanged file bytes, source retry and EOF\n", ps / 1024); +} + +static int register_shared(void *p, size_t len) +{ + int fd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = {.api = UFFD_API, .features = UFFD_FEATURE_MISSING_SHMEM | + UFFD_FEATURE_MINOR_SHMEM | UFFD_FEATURE_PAGEFAULT_FLAG_WP}; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = {.range = {(uintptr_t)p, len}, + .mode = UFFDIO_REGISTER_MODE_MISSING | UFFDIO_REGISTER_MODE_MINOR | UFFDIO_REGISTER_MODE_WP}; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK((reg.ioctls & ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE) | (1ULL << _UFFDIO_CONTINUE))) == + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE) | (1ULL << _UFFDIO_CONTINUE))); + return fd; +} +static void shared_missing(size_t ps, size_t native) +{ + int memfd = memfd_create("uffd-shared-holes", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 8 * native)); + unsigned char *p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, memfd, ps); + unsigned char *q = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, memfd, ps); + unsigned char *src = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && q != MAP_FAILED && src != MAP_FAILED); + for (size_t i = 0; i < 4 * ps; i++) src[i] = pager_pattern(i); + int fd = register_shared(p, 4 * ps), fd2 = register_shared(q, 4 * ps); + for (unsigned i = 0; i < 2; i++) { + struct reader r = {.address = p + i * ps + 17}; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && msg.arg.pagefault.address == (uintptr_t)(p + i * ps) && !msg.arg.pagefault.flags); + if (!i) fill(fd, p, src, ps, UFFDIO_COPY_MODE_WP); + else zero(fd, p + ps, ps); + CHECK(!pthread_join(thread, NULL) && atomic_load(&r.done)); + CHECK(r.value == (i ? 0 : src[17])); + if (!i) { + struct uffdio_continue c = {.range = {(uintptr_t)(p + ps), ps}}; + CHECK(ioctl(fd, UFFDIO_CONTINUE, &c) == -1 && errno == EFAULT && c.mapped == -EFAULT); + } + } + fill(fd, p + 2 * ps, src + 2 * ps, 2 * ps, 0); + CHECK(!memcmp(p, src, ps) && !memcmp(p + 2 * ps, src + 2 * ps, 2 * ps)); + for (size_t i = 0; i < ps; i++) CHECK(p[ps + i] == 0); + for (unsigned i = 0; i < 4; i++) { + struct reader r = {.address = q + i * ps + 17}; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd2); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && msg.arg.pagefault.address == (uintptr_t)(q + i * ps) && + msg.arg.pagefault.flags == UFFD_PAGEFAULT_FLAG_MINOR); + resume_minor(fd2, q + i * ps, ps, 0); + CHECK(!pthread_join(thread, NULL) && r.value == p[i * ps + 17]); + CHECK(physical(p + i * ps, ps) == physical(q + i * ps, ps)); + } + CHECK(!memcmp(p, q, 4 * ps)); + if (ps < native) CHECK(physical(p + ps, ps) == physical(p, ps) + ps); + struct writer w = {.address = p + 17, .value = 0x9a}; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == (UFFD_PAGEFAULT_FLAG_WRITE | UFFD_PAGEFAULT_FLAG_WP)); + protect(fd, p, ps, 0); + CHECK(!pthread_join(thread, NULL) && p[17] == w.value && q[17] == w.value); + struct uffdio_copy c = {.dst = (uintptr_t)(p + ps), .src = (uintptr_t)src, .len = ps}; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && errno == EEXIST && c.copy == -EEXIST); + unsigned char bytes[256]; + CHECK(pread(memfd, bytes, sizeof(bytes), ps) == sizeof(bytes)); + CHECK(!memcmp(bytes, p, sizeof(bytes))); + /* Resolve an untouched source outside VMA locks before taking a cache folio. */ + unsigned char *r = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, memfd, 6 * native); + unsigned char *fresh = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(r != MAP_FAILED && fresh != MAP_FAILED); + int fd3 = register_shared(r, 2 * ps); + fill(fd3, r, fresh, ps, 0); + for (size_t i = 0; i < ps; i++) CHECK(r[i] == 0); + CHECK(!mprotect(fresh, ps, PROT_NONE)); + c = (struct uffdio_copy){.dst = (uintptr_t)(r + ps), .src = (uintptr_t)fresh, .len = ps}; + CHECK(ioctl(fd3, UFFDIO_COPY, &c) == -1 && errno == EFAULT && c.copy == -EFAULT); + zero(fd3, r + ps, ps); + for (size_t i = 0; i < ps; i++) CHECK(r[ps + i] == 0); + CHECK(!close(fd3) && !munmap(r, 2 * ps) && !munmap(fresh, ps)); + CHECK(!ftruncate(memfd, 2 * ps + 17)); + c = (struct uffdio_copy){.dst = (uintptr_t)(p + 2 * ps), .src = (uintptr_t)src, .len = ps}; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && + errno == (ps == native ? ENOMEM : EFAULT) && c.copy == -(ps == native ? ENOMEM : EFAULT)); + CHECK(!close(fd) && !close(fd2) && !close(memfd)); + CHECK(!munmap(p, 4 * ps) && !munmap(q, 4 * ps) && !munmap(src, 4 * ps)); + printf("ok - uffd ABI %zuK shared COPY/ZERO, independent missing/minor pagers, CONTINUE holes, packing, WP/coherence and EOF\n", ps / 1024); +} + +static void shared_buffered_write(size_t ps) +{ + int memfd = memfd_create("uffd-buffered-validity", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, memfd, 0); + unsigned char *src = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && src != MAP_FAILED); + memset(src, 0x57, ps); + int fd = register_shared(p, 4 * ps); + fill(fd, p, src, ps, 0); + CHECK(pwrite(memfd, src, 23, ps + 17) == 23); + for (unsigned i = 1; i < 3; i++) { + struct reader r = {.address = p + i * ps + 17}; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && msg.arg.pagefault.address == (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == (i == 1 ? UFFD_PAGEFAULT_FLAG_MINOR : 0)); + if (i == 1) resume_minor(fd, p + ps, ps, 0); + else zero(fd, p + i * ps, ps); + CHECK(!pthread_join(thread, NULL) && r.value == (i == 1 ? 0x57 : 0)); + } + for (size_t i = 0; i < ps; i++) CHECK(p[ps + i] == (i >= 17 && i < 40 ? 0x57 : 0)); + CHECK(!memcmp(p, src, ps)); + CHECK(!close(fd) && !close(memfd) && !munmap(p, 4 * ps) && !munmap(src, ps)); + printf("ok - uffd ABI %zuK buffered short-range write populates its leaf and leaves neighbours missing\n", ps / 1024); +} + +static int worker(unsigned long expected, unsigned long native) +{ + size_t ps = getauxval(AT_PAGESZ); + CHECK(ps == expected); + alarm(60); + missing(ps, native); + file_minor(ps, native); + markers(ps); + cross_pager(ps, native, 0); + cross_pager(ps, native, 1); + private_missing(ps); + shared_missing(ps, native); + shared_buffered_write(ps); + for (int exact = 0; exact < 2; exact++) { + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + for (unsigned i = 0; i < 3; i++) memset(p + i * ps, 0x40 + i, ps); + int fd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP | UFFD_FEATURE_THREAD_ID | + (exact ? UFFD_FEATURE_EXACT_ADDRESS | UFFD_FEATURE_EVENT_FORK : 0) }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + CHECK(api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP); + struct uffdio_register reg = { .range = {(uintptr_t)p, 3 * ps}, + .mode = UFFDIO_REGISTER_MODE_WP }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(reg.ioctls & (1ULL << _UFFDIO_WRITEPROTECT)); + + for (unsigned i = 0; i < 3; i++) { + protect(fd, p + i * ps, ps, UFFDIO_WRITEPROTECT_MODE_WP); + /* Neighbours remain directly writable while this page faults. */ + p[((i + 1) % 3) * ps + 31] ^= 1; + resolve_write(fd, p, ps, i, 0x70 + i, exact); + } + protect(fd, p + ps, ps, UFFDIO_WRITEPROTECT_MODE_WP); + pid_t child; + int status; + if (!exact) { + child = fork(); + CHECK(child >= 0); + if (!child) { alarm(10); p[ps + 17] = 0xa5; _exit(p[ps + 17] != 0xa5); } + } else { + struct forker f = {.base = p, .page_size = ps}; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, fork_byte, &f)); + struct uffd_msg event = read_event(fd); + CHECK(event.event == UFFD_EVENT_FORK); + int child_fd = event.arg.fork.ufd; + CHECK(child_fd >= 0); + CHECK(!pthread_join(thread, NULL)); + child = f.pid; CHECK(child > 0); + event = read_event(child_fd); + CHECK(event.event == UFFD_EVENT_PAGEFAULT && + event.arg.pagefault.flags == (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_WRITE)); + CHECK(event.arg.pagefault.address == (uintptr_t)(p + ps + 17)); + CHECK(event.arg.pagefault.feat.ptid == (unsigned int)child); + protect(child_fd, p + ps, ps, 0); + CHECK(!close(child_fd)); + } + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(p[ps + 17] == 0x71); + resolve_write(fd, p, ps, 1, 0x91, exact); + protect(fd, p, 3 * ps, UFFDIO_WRITEPROTECT_MODE_WP); + struct uffdio_range middle = {(uintptr_t)(p + ps), ps}; + CHECK(!ioctl(fd, UFFDIO_UNREGISTER, &middle)); + p[ps + 17] = 0xb2; + resolve_write(fd, p, ps, 0, 0xb1, exact); + CHECK(!close(fd)); + /* Close removes the remaining registration and write protection. */ + p[2 * ps + 17] = 0xb3; + CHECK(p[17] == 0xb1 && p[ps + 17] == 0xb2 && p[2 * ps + 17] == 0xb3); + CHECK(!munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK exact=%d events, neighbours, fork delivery/COW, partial unregister and close\n", ps / 1024, exact); + } + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--pager")) + return pager_worker(atoi(argv[2]), strtoul(argv[3], NULL, 10)); + if (argc == 4 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), strtoul(argv[3], NULL, 10)); + if (getpid() != 1 || argc != 1) return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + unsigned long native = getauxval(AT_PAGESZ), sizes[] = {native, 4096, 16384}; + unsigned modes = native == 65536 ? 3 : 2; + int failed = 0; + for (unsigned i = 0; i < modes; i++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char ps[32], ns[32]; + snprintf(ps, sizeof(ps), "%lu", sizes[i]); snprintf(ns, sizeof(ns), "%lu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)); + execl("/init", "/init", "--worker", ps, ns, NULL); _exit(3); + } + int status = 0; + int ok = waitpid(pid, &status, 0) == pid && WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - uffd ABI %luK status=%#x\n", ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + } + printf("UFFD CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); reboot(RB_POWER_OFF); for (;;) pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uprobes-init.c b/tools/testing/selftests/arm64/user4k/uprobes-init.c new file mode 100644 index 0000000000000..0c379e4309caf --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uprobes-init.c @@ -0,0 +1,318 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define TRACE "/sys/kernel/tracing/" +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +static int write_file(const char *path, const char *s) +{ + int fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + size_t len = strlen(s); + int ret = write(fd, s, len) == (ssize_t)len ? 0 : -1; + close(fd); + return ret; +} +static int enable(const char *event, int on) +{ + char path[160]; + snprintf(path, sizeof(path), TRACE "events/aurora_up/%s/enable", event); + return write_file(path, on ? "1" : "0"); +} +static uint64_t pagemap(void *p, size_t ps) +{ + uint64_t bits = 0; + int fd = open("/proc/self/pagemap", O_RDONLY); + if (fd >= 0) { + if (pread(fd, &bits, 8, (uintptr_t)p / ps * 8) != 8) + bits = 0; + close(fd); + } + return bits; +} +static unsigned records(const char *event) +{ + FILE *f = fopen(TRACE "trace", "r"); + char *line = NULL; + size_t len = 0; + unsigned count = 0; + char match[96]; + snprintf(match, sizeof(match), "%s:", event); + if (!f) + return 0; + while (getline(&line, &len, f) > 0) + if (strstr(line, match)) + count++; + free(line); + fclose(f); + return count; +} +static int xol_geometry(size_t native) +{ + FILE *f = fopen("/proc/self/maps", "r"); + char *line = NULL; + size_t len = 0; + int ok = 0; + if (!f) + return 0; + while (getline(&line, &len, f) > 0) + if (strstr(line, "[uprobes]")) { + unsigned long a, b; + if (sscanf(line, "%lx-%lx", &a, &b) == 2) { + printf("# XOL %#lx-%#lx native=%zu\n", a, b, + native); + ok = !(a % (unsigned long)native) && + b - a == native; + } + } + free(line); + fclose(f); + return ok; +} +static int exercise(size_t ps, size_t native, unsigned shifted) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(20); + printf("# start ps=%zu shifted=%u\n", ps, shifted); + int fd = memfd_create("aurora-uprobe-code", 0); + CHECK(fd >= 0); + size_t length = 8 * ps; + unsigned char *blob = malloc(length); + CHECK(blob); + memset(blob, 0x71, length); + size_t code_off = 3 * ps + 64, ctr_off = 5 * ps + 32; + size_t map_off = shifted ? 2 * ps : 0, + map_len = shifted ? 2 * ps : 4 * ps; + uint32_t insns[] = { 0x11001c00, 0xd65f03c0 }; + memcpy(blob + code_off, insns, sizeof(insns)); + memset(blob + ctr_off, 0, 2); + CHECK(write(fd, blob, length) == (ssize_t)length); + unsigned char *code = mmap((void *)(0x20000000 + (shifted ? ps : 0)), + map_len, PROT_READ | PROT_EXEC, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, + map_off); + unsigned char *counter = + mmap((void *)(0x30000000 + 2 * ps), ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, 5 * ps); + unsigned char *other = mmap((void *)0x40000000, ps, + PROT_READ | PROT_EXEC, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, 0); + CHECK(code != MAP_FAILED && counter != MAP_FAILED && + other != MAP_FAILED); + unsigned long (*target)(unsigned long) = + (void *)(code + code_off - map_off); + volatile unsigned short *ctr = (void *)(counter + 32); + CHECK(*ctr == 0); + CHECK(target(11) == 18); + /* Occupy the default top-of-address-space XOL hint, forcing a new placement. */ + void *guard = mmap((void *)((1UL << 48) - native), ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, + -1, 0); + CHECK(guard != MAP_FAILED); + char entry[64], retname[64], cmd[256]; + snprintf(entry, sizeof(entry), "entry_%zu_%u", ps, shifted); + snprintf(retname, sizeof(retname), "return_%zu_%u", ps, shifted); + snprintf(cmd, sizeof(cmd), + "p:aurora_up/%s /proc/self/fd/%d:0x%zx(0x%zx)\n", entry, fd, + code_off, ctr_off); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + snprintf(cmd, sizeof(cmd), + "r:aurora_up/%s /proc/self/fd/%d:0x%zx(0x%zx)\n", retname, fd, + code_off, ctr_off); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(*ctr == 1); + printf("# enabling return\n"); + CHECK(!enable(retname, 1)); + printf("# return enabled\n"); + CHECK(*ctr == 1); + uint32_t disk[2]; + CHECK(pread(fd, disk, sizeof(disk), code_off) == sizeof(disk)); + CHECK(!memcmp(disk, insns, sizeof(insns))); + CHECK(!memcmp(other, blob, ps)); + CHECK(!memcmp(counter, blob + 5 * ps, 32)); + CHECK(!memcmp(counter + 34, blob + 5 * ps + 34, ps - 34)); + int tracefd = open(TRACE "trace", O_WRONLY | O_TRUNC); + CHECK(tracefd >= 0); + close(tracefd); + printf("# invoking probes\n"); + for (unsigned i = 0; i < 16; i++) + CHECK(target(i) == i + 7); + printf("# invocations complete\n"); + CHECK(xol_geometry(native)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + int ok = *ctr == 1; + for (unsigned i = 0; i < 8; i++) + ok &= target(100 + i) == 107 + i; + _exit(ok ? 0 : 3); + } + int status; + CHECK(waitpid(child, &status, 0) == child); + CHECK(WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(*ctr == 1); + unsigned entries = records(entry), returns = records(retname); + printf("# ABI=%zu entry=%u return=%u\n", ps, entries, returns); + CHECK(entries == 24 && returns == 24); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 1); + CHECK(!enable(retname, 0)); + CHECK(*ctr == 0); + /* Identical restored text is dropped so it can be shared with the file again. */ + CHECK(!(pagemap((void *)target, ps) >> 63)); + CHECK(target(10) == 17); + CHECK(!memcmp(code, blob + map_off, map_len)); + CHECK(!memcmp(other, blob, ps)); + CHECK(!memcmp(counter, blob + 5 * ps, ps)); + /* A pipe GET must prevent dropping the mapped code slot on removal. */ + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(*ctr == 1); + int pipefd[2]; + CHECK(!pipe(pipefd)); + struct iovec iov = { .iov_base = + (void *)((uintptr_t)target & ~(ps - 1)), + .iov_len = ps }; + CHECK(vmsplice(pipefd[1], &iov, 1, 0) == (ssize_t)ps); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 0); + CHECK(pagemap((void *)target, ps) >> 63); + unsigned char *held = malloc(ps); + CHECK(held); + CHECK(read(pipefd[0], held, ps) == (ssize_t)ps); + CHECK(!memcmp(held, blob + (code_off & ~(ps - 1)), ps)); + free(held); + close(pipefd[0]); + close(pipefd[1]); + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 0); + /* Native folio-wide references may defer this optional optimization. */ + if (ps < native) + CHECK(!(pagemap((void *)target, ps) >> 63)); + CHECK(target(20) == 27); + snprintf(cmd, sizeof(cmd), "-:aurora_up/%s\n", entry); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + snprintf(cmd, sizeof(cmd), "-:aurora_up/%s\n", retname); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + CHECK(!munmap(code, map_len)); + CHECK(!munmap(guard, ps)); + CHECK(!munmap(counter, ps)); + CHECK(!munmap(other, ps)); + close(fd); + free(blob); + printf("ok - %zuK uprobe/uretprobe, byte offsets, counter, fork, restore and file sharing\n", + ps / 1024); + return 0; +} +static int wait_bounded(pid_t child, int *status) +{ + for (unsigned i = 0; i < 250; i++) { + pid_t got = waitpid(child, status, WNOHANG); + if (got == child) + return 1; + if (got < 0) + return 0; + if (i == 50 || i == 200) { + char path[64], buf[4096]; + snprintf(path, sizeof(path), "/proc/%d/stack", child); + int fd = open(path, O_RDONLY); + ssize_t n = fd < 0 ? -1 : + read(fd, buf, sizeof(buf) - 1); + if (fd >= 0) + close(fd); + if (n > 0) { + buf[n] = 0; + printf("# child %d stack:\n%s", child, buf); + } else + printf("# child %d stack unavailable errno=%d\n", + child, errno); + } + if (i == 200) { + printf("# killing stalled child %d\n", child); + kill(child, SIGKILL); + } + usleep(100000); + } + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), + strtoul(argv[4], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("tracefs", TRACE, "tracefs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned run = 0; run < (sizes[0] > 16384 ? 6U : 4U); run++) { + unsigned i = run / 2, shifted = run & 1; + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24], native[24], mode[8]; + snprintf(mode, sizeof(mode), "%u", shifted); + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + snprintf(native, sizeof(native), "%zu", sizes[0]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, native, + mode, NULL); + _exit(5); + } + int ok = c > 0 && wait_bounded(c, &status) && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - ABI %zuK uprobes shifted=%u status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, shifted, status); + fail += !ok; + } +done: + printf("UPROBES %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uring-fixture-init.c b/tools/testing/selftests/arm64/user4k/uring-fixture-init.c new file mode 100644 index 0000000000000..f2664ee0e932a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uring-fixture-init.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +struct ring { + int fd; + struct io_uring_params p; + void *sq, *cq, *sqes; + size_t sqlen, cqlen, sqelen; +}; +static unsigned load(unsigned *p) { return __atomic_load_n(p,__ATOMIC_ACQUIRE); } +static void store(unsigned *p,unsigned v) { __atomic_store_n(p,v,__ATOMIC_RELEASE); } +static size_t align_size(size_t n) { size_t ps=getauxval(AT_PAGESZ);return (n+ps-1)&~(ps-1); } +static void destroy(struct ring *r) +{ + if(r->sq && r->sq!=MAP_FAILED)munmap(r->sq,r->sqlen); + if(r->cq && r->cq!=MAP_FAILED && r->cq!=r->sq)munmap(r->cq,r->cqlen); + if(r->sqes && r->sqes!=MAP_FAILED)munmap(r->sqes,r->sqelen); + if(r->fd>=0)close(r->fd); + *r=(struct ring){.fd=-1}; +} +static bool setup(struct ring *r,unsigned entries,unsigned flags) +{ + *r=(struct ring){.fd=-1};r->p.flags=flags; + r->fd=syscall(__NR_io_uring_setup,entries,&r->p); + if(r->fd<0)return false; + r->sqlen=align_size(r->p.sq_off.array+r->p.sq_entries*sizeof(unsigned)); + r->cqlen=align_size(r->p.cq_off.cqes+r->p.cq_entries*sizeof(struct io_uring_cqe)*(flags&IORING_SETUP_CQE32?2:1)); + r->sqelen=align_size(r->p.sq_entries*sizeof(struct io_uring_sqe)*(flags&IORING_SETUP_SQE128?2:1)); + if(r->p.features&IORING_FEAT_SINGLE_MMAP){if(r->cqlen>r->sqlen)r->sqlen=r->cqlen;r->cqlen=r->sqlen;} + r->sq=mmap(NULL,r->sqlen,PROT_READ|PROT_WRITE,MAP_SHARED,r->fd,IORING_OFF_SQ_RING); + r->cq=(r->p.features&IORING_FEAT_SINGLE_MMAP)?r->sq:mmap(NULL,r->cqlen,PROT_READ|PROT_WRITE,MAP_SHARED,r->fd,IORING_OFF_CQ_RING); + r->sqes=mmap(NULL,r->sqelen,PROT_READ|PROT_WRITE,MAP_SHARED,r->fd,IORING_OFF_SQES); + return r->sq!=MAP_FAILED && r->cq!=MAP_FAILED && r->sqes!=MAP_FAILED; +} +static bool request(struct ring *r,unsigned op,int fd,void *buf,unsigned len,uint64_t off,bool async) +{ + unsigned *sqtail=r->sq+r->p.sq_off.tail,*sqmask=r->sq+r->p.sq_off.ring_mask; + unsigned *cqhead=r->cq+r->p.cq_off.head,*cqtail=r->cq+r->p.cq_off.tail,*cqmask=r->cq+r->p.cq_off.ring_mask; + unsigned tail=load(sqtail),head=load(cqhead),idx=tail&load(sqmask); + unsigned sqstride=sizeof(struct io_uring_sqe)*(r->p.flags&IORING_SETUP_SQE128?2:1); + unsigned cqstride=sizeof(struct io_uring_cqe)*(r->p.flags&IORING_SETUP_CQE32?2:1); + struct io_uring_sqe *sqe=r->sqes+idx*sqstride; + memset(sqe,0,sqstride);sqe->opcode=op;sqe->fd=fd;sqe->addr=(uintptr_t)buf;sqe->len=len;sqe->off=off;sqe->user_data=(uint64_t)tail+1234; + if(async)sqe->flags=IOSQE_ASYNC; + ((unsigned *)(r->sq+r->p.sq_off.array))[idx]=idx; + store(sqtail,tail+1); + int ret=syscall(__NR_io_uring_enter,r->fd,1,1,IORING_ENTER_GETEVENTS,NULL,0); + if(ret!=1 || load(cqtail)!=head+1){ksft_print_msg("enter=%d errno=%d\n",ret,errno);return false;} + struct io_uring_cqe *cqe=r->cq+r->p.cq_off.cqes+(head&load(cqmask))*cqstride; + unsigned expected=op==IORING_OP_NOP?0:len; + if(op==IORING_OP_READV_FIXED || op==IORING_OP_WRITEV_FIXED){expected=0;for(unsigned i=0;iuser_data==(uint64_t)tail+1234 && cqe->res==(int)expected; + if(!ok)ksft_print_msg("cqe res=%d expected=%u\n",cqe->res,expected); + store(cqhead,head+1);return ok; +} +static unsigned long pinned_kb(void) +{ + FILE *f=fopen("/proc/self/status","r");char line[160];unsigned long n=~0UL; + if(!f)return n; + while(fgets(line,sizeof(line),f))if(sscanf(line,"VmPin: %lu kB",&n)==1)break; + fclose(f);return n; +} +static bool bytes_are(unsigned char *p,size_t len,unsigned char value) +{ + for(size_t i=0;i=0 && !ftruncate(mem,len+3*ps) && q!=MAP_FAILED; + if(ok){ + ok=mmap(p,len,PROT_READ|PROT_WRITE,MAP_SHARED|MAP_FIXED,mem,3*ps)==p; + memset(p,0x6b,len);reg=(struct iovec){p+512,len-1024}; + live=syscall(__NR_io_uring_register,r.fd,IORING_REGISTER_BUFFERS,®,1)==0; + ok &= live; + struct iovec v[3]={{p+512,512},{p+3*ps+512,ps-512},{p+23*ps+512,1024}}; + if(ok){ + ok=request(&r,IORING_OP_WRITEV_FIXED,fd,v,3,14*1024*1024,true); + memset(p+512,0x18,len-1024); + ok &= request(&r,IORING_OP_READV_FIXED,fd,v,3,14*1024*1024,true); + for(unsigned i=0;i<3;i++)ok &= bytes_are(v[i].iov_base,v[i].iov_len,0x6b); + ok &= !msync(p,len,MS_SYNC) && pread(mem,q,ps,3*ps)==(ssize_t)ps; + ok &= bytes_are(q,1024,0x6b) && q[1024]==0x18 && p[len-1]==0x6b; + } + } + ksft_test_result(ok,"registered shared-file quarters use physical offsets\n"); + if(ok){ + struct iovec replacement={q+512,3*ps}; + struct io_uring_rsrc_update2 update={.data=(uintptr_t)&replacement,.nr=1}; + memset(q,0x9d,4*ps); + ok=syscall(__NR_io_uring_register,r.fd,IORING_REGISTER_BUFFERS_UPDATE,&update,sizeof(update))==1; + ok &= request(&r,IORING_OP_WRITE_FIXED,fd,q+512,3*ps,16*1024*1024,true); + memset(q+512,0,3*ps); + ok &= request(&r,IORING_OP_READ_FIXED,fd,q+512,3*ps,16*1024*1024,true)&&bytes_are(q+512,3*ps,0x9d); + } + if(live){ok &= syscall(__NR_io_uring_register,r.fd,IORING_UNREGISTER_BUFFERS,NULL,0)==0;live=false;} + ok &= pinned_kb()==baseline; + ksft_test_result(ok,"registered-buffer update transfers ownership and balances accounting\n"); + if(mem>=0)close(mem); + if(live)syscall(__NR_io_uring_register,r.fd,IORING_UNREGISTER_BUFFERS,NULL,0); + destroy(&r);if(q!=MAP_FAILED)munmap(q,len); +} + +static int exercise(void) +{ + struct ring r={.fd=-1};size_t ps=getauxval(AT_PAGESZ),len=2*1024*1024; + unsigned char *p=mmap(NULL,len,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + int fd=open("/dev/user4k-test",O_RDWR|O_DIRECT);bool ok; + ksft_print_header();ksft_set_plan(19);ksft_print_msg("ABI=%zuK\n",ps/1024); + ok=setup(&r,4,0);ksft_test_result(ok,"minimum rounded ring mappings\n"); + void *bad=ok?mmap(NULL,ps,PROT_READ|PROT_WRITE,MAP_SHARED,r.fd,4096):MAP_FAILED; + bool rejected=ok && bad==MAP_FAILED && errno==EINVAL; + if(bad!=MAP_FAILED)munmap(bad,ps); + ksft_test_result(rejected,"reject sub-native offset in ring selector\n"); + if(ok){ok=true;for(unsigned i=0;i<64 && ok;i++)ok=request(&r,IORING_OP_NOP,-1,NULL,0,0,false);} + ksft_test_result(ok,"small submission/completion ring wrap\n"); + if(p==MAP_FAILED || fd<0)ksft_exit_fail_msg("buffer/device setup errno=%d\n",errno); + if(ok){memset(p+ps,0x69,3*ps);ok=request(&r,IORING_OP_WRITE,fd,p+ps,3*ps,0,false);memset(p+ps,0,3*ps);ok &= request(&r,IORING_OP_READ,fd,p+ps,3*ps,0,false);for(size_t i=ps;i<4*ps;i++)ok &= p[i]==0x69;} + ksft_test_result(ok,"direct read/write from unregistered user buffers\n"); + if(ok){memset(p+ps,0x37,3*ps);ok=request(&r,IORING_OP_WRITE,fd,p+ps,3*ps,65536,true);memset(p+ps,0,3*ps);ok &= request(&r,IORING_OP_READ,fd,p+ps,3*ps,65536,true);for(size_t i=ps;i<4*ps;i++)ok &= p[i]==0x37;} + ksft_test_result(ok,"io-worker forced asynchronous read/write\n"); + if(ok){memset(p,0xb6,len);ok=request(&r,IORING_OP_WRITE,fd,p,len,4*1024*1024,true);memset(p,0,len);ok &= request(&r,IORING_OP_READ,fd,p,len,4*1024*1024,true);for(size_t i=0;i0 && waitpid(pid,&status,0)==pid && WIFEXITED(status) && !WEXITSTATUS(status); + } + printf("%s - io_uring fixture complete\n",ok?"ok":"not ok");reboot(RB_POWER_OFF);return ok?0:1; +} diff --git a/tools/testing/selftests/arm64/user4k/user-events-init.c b/tools/testing/selftests/arm64/user4k/user-events-init.c new file mode 100644 index 0000000000000..4b35954e2a210 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/user-events-init.c @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define TRACE "/sys/kernel/tracing/" +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) + +static int toggle(const char *name, int enabled) +{ + char path[192]; + snprintf(path, sizeof(path), TRACE "events/user_events/%s/enable", + name); + int fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + int ret = write(fd, enabled ? "1" : "0", 1) == 1 ? 0 : -1; + close(fd); + return ret; +} +static uint64_t value(void *p, unsigned bytes) +{ + return bytes == 4 ? __atomic_load_n((uint32_t *)p, __ATOMIC_ACQUIRE) : + __atomic_load_n((uint64_t *)p, __ATOMIC_ACQUIRE); +} +static int wait_value(void *p, unsigned bytes, uint64_t expected) +{ + for (unsigned i = 0; i < 3000; i++) { + if (value(p, bytes) == expected) + return 1; + usleep(1000); + } + return 0; +} +static int check_bytes(unsigned char *p, size_t size, unsigned char fill, + void *word, unsigned bytes) +{ + for (size_t i = 0; i < size; i++) + if ((!word || (uintptr_t)(p + i) < (uintptr_t)word || + (uintptr_t)(p + i) >= (uintptr_t)word + bytes) && + p[i] != fill) { + printf("byte mismatch at %zu got=%#x expected=%#x\n", i, + p[i], fill); + return 0; + } + return 1; +} +static int event_recorded(const char *name) +{ + char buf[16384]; + size_t n = 0; + int fd = open(TRACE "trace", O_RDONLY); + if (fd < 0) + return 0; + while (n < sizeof(buf) - 1) { + ssize_t got = read(fd, buf + n, sizeof(buf) - 1 - n); + if (got < 0) { + close(fd); + return 0; + } + if (!got) + break; + n += got; + } + close(fd); + buf[n] = 0; + char *record = strstr(buf, name), + *payload = record ? strstr(record, "payload=") : NULL; + int ok = payload && strtoul(payload + 8, NULL, 0) == 0x12345678; + if (!ok) + printf("trace mismatch:\n%s\n", buf); + return ok; +} +static int exercise_case(size_t ps, unsigned bytes, unsigned shared) +{ + const size_t len = 4 * ps; + unsigned char *src, *dst; + int memfd = -1; + if (shared) { + memfd = memfd_create("user-event-bitmap", 0); + CHECK(memfd >= 0); + CHECK(!ftruncate(memfd, len)); + } + src = mmap((void *)0x20000000, len, PROT_READ | PROT_WRITE, + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS) | + MAP_FIXED_NOREPLACE, + memfd, 0); + dst = mmap((void *)0x30000000, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, len); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps; + void *word = leaf + ps - bytes; + uint64_t mask = bytes == 4 ? 1ULL << 31 : 1ULL << 63; + memset(word, 0, bytes); + char name[80], spec[120]; + snprintf(name, sizeof(name), "aurora_ue_%zu_%u_%u", ps, bytes, shared); + snprintf(spec, sizeof(spec), "%s u32 payload", name); + int tracefd = open(TRACE "trace", O_WRONLY | O_TRUNC); + CHECK(tracefd >= 0); + close(tracefd); + int fd = open(TRACE "user_events_data", O_RDWR); + CHECK(fd >= 0); + struct user_reg reg = { .size = sizeof(reg), + .enable_bit = bytes * 8 - 1, + .enable_size = bytes, + .enable_addr = (uintptr_t)word, + .name_args = (uintptr_t)spec }; + CHECK(!ioctl(fd, DIAG_IOCSREG, ®)); + CHECK(wait_value(word, bytes, 0)); + CHECK(!toggle(name, 1)); + CHECK(wait_value(word, bytes, mask)); + struct { + uint32_t index, payload; + } event = { reg.write_index, 0x12345678 }; + CHECK(write(fd, &event, sizeof(event)) == sizeof(event)); + CHECK(event_recorded(name)); + CHECK(!toggle(name, 0)); + CHECK(wait_value(word, bytes, 0)); + /* Every other byte, including the adjacent half of the kernel word, is stable. */ + CHECK(check_bytes(leaf, ps, 0x64, word, bytes)); + CHECK(check_bytes(dst, ps, 0x91, NULL, 0)); + CHECK(check_bytes(dst + 2 * ps, 2 * ps, 0x91, NULL, 0)); + int commands[2], reply[2]; + CHECK(!pipe(commands) && !pipe(reply)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + close(commands[1]); + close(reply[0]); + char token; + int ok = 1; + for (unsigned round = 0; round < 32; round++) { + if (read(commands[0], &token, 1) != 1) { + ok = 0; + break; + } + ok = wait_value(word, bytes, token == '1' ? mask : 0) && + check_bytes(leaf, ps, 0x64, word, bytes); + token = ok ? 'y' : 'n'; + if (write(reply[1], &token, 1) != 1 || !ok) + break; + } + _exit(ok ? 0 : 3); + } + close(commands[0]); + close(reply[1]); + int ok = 1; + for (unsigned round = 0; round < 32; round++) { + int on = !(round & 1); + char token = on ? '1' : '0'; + if (toggle(name, on) || + !wait_value(word, bytes, on ? mask : 0) || + write(commands[1], &token, 1) != 1 || + read(reply[0], &token, 1) != 1 || token != 'y') { + ok = 0; + break; + } + } + close(commands[1]); + close(reply[0]); + int status; + CHECK(waitpid(child, &status, 0) == child); + CHECK(ok && WIFEXITED(status) && WEXITSTATUS(status) == 0); + CHECK(check_bytes(leaf, ps, 0x64, word, bytes)); + for (unsigned i = 0; i < 3; i++) + CHECK(check_bytes(src + i * ps, ps, 0x61 + i, NULL, 0)); + /* Force the asynchronous missing-PTE fixup without unregistering. */ + CHECK(!madvise(leaf, ps, MADV_DONTNEED)); + CHECK(!toggle(name, 1)); + CHECK(wait_value(word, bytes, mask)); + CHECK(check_bytes(leaf, ps, shared ? 0x64 : 0, word, bytes)); + CHECK(!toggle(name, 0)); + CHECK(wait_value(word, bytes, 0)); + struct user_unreg unreg = { .size = sizeof(unreg), + .disable_bit = reg.enable_bit, + .disable_addr = (uintptr_t)word }; + CHECK(!ioctl(fd, DIAG_IOCSUNREG, &unreg)); + close(fd); + CHECK(!munmap(src, 3 * ps)); + CHECK(!munmap(dst, len)); + if (memfd >= 0) + close(memfd); + printf("ok - %zuK %u-bit %s relocated bitmap, adjacent bytes, event write and fork toggles\n", + ps / 1024, bytes * 8, shared ? "shared" : "private"); + return 0; +} +static int exercise(size_t expected) +{ + size_t ps = getauxval(AT_PAGESZ); + CHECK(ps == expected); + for (unsigned shared = 0; shared < 2; shared++) + for (unsigned bytes = 4; bytes <= 8; bytes += 4) + CHECK(!exercise_case(ps, bytes, shared)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("tracefs", TRACE, "tracefs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24]; + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = c > 0 && waitpid(c, &status, 0) == c && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - ABI %zuK user events status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("USER EVENTS %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c b/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c new file mode 100644 index 0000000000000..d0f3f7c44b0e3 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define GET_EXEC_PAGE_SIZE 0x41555002 +static int failed; +static _Thread_local int tls = 7; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static long pss(void *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; size_t cap = 0; unsigned long start, end; + int active = 0; long value = -1; + if (!f) return -1; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + active = (unsigned long)p >= start && (unsigned long)p < end; + else if (active && sscanf(line, "Pss: %ld kB", &value) == 1) break; + } + free(line); fclose(f); return value; +} +static void *thread_worker(void *arg) +{ + unsigned char *p = arg; + if (tls != 7) return (void *)1; + tls = 91; + for (unsigned i = 0; i < 16384; i++) if (p[i] != 0x42) return (void *)2; + return NULL; +} +static void memory_checks(void) +{ + const size_t ps = 16384; int status = 0; pid_t c; + unsigned char *p = mmap(NULL, 6 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + result(p != MAP_FAILED, "16K anonymous mmap"); + if (p == MAP_FAILED) return; + for (unsigned i = 0; i < 6; i++) memset(p + i * ps, 0x40 + i, ps); + c = fork(); + if (!c) { memset(p + ps, 0xb7, ps); _exit(p[0] == 0x40 && p[2 * ps] == 0x42 ? 0 : 1); } + int ok = c > 0 && waitpid(c, &status, 0) == c && WIFEXITED(status) && !WEXITSTATUS(status); + for (unsigned i = 0; i < 6 * ps; i++) ok &= p[i] == 0x40 + i / ps; + result(ok, "16K fork COW preserves every parent byte"); + result(!mprotect(p + ps, ps, PROT_READ), "16K leaf write protection"); + c = fork(); + if (!c) { *(volatile unsigned char *)(p + ps + 8192) = 3; _exit(1); } + result(c > 0 && waitpid(c, &status, 0) == c && WIFSIGNALED(status) && WTERMSIG(status) == SIGSEGV, + "write protection covers the interior of a16K leaf"); + result(!mprotect(p + ps, ps, PROT_READ | PROT_WRITE), "16K write permission restored"); + errno = 0; + result(mprotect(p + 4096, ps, PROT_READ) == -1 && errno == EINVAL, + "16K ABI rejects4K-only protection alignment"); + result(!madvise(p + ps, ps, MADV_DONTNEED), "16K partial unmap via DONTNEED"); + ok = 1; + for (unsigned i = 0; i < ps; i++) ok &= p[i] == 0x40 && p[ps + i] == 0 && p[2 * ps + i] == 0x42; + result(ok, "16K discard clears one leaf and preserves neighbours"); + pthread_t thread; void *answer = (void *)3; + int err = pthread_create(&thread, NULL, thread_worker, p + 2 * ps); + if (!err) err = pthread_join(thread, &answer); + result(!err && !answer && tls == 7, "16K pthread stack and TLS"); + struct timespec ts; + result(!clock_gettime(CLOCK_MONOTONIC, &ts), "16K vDSO clock access"); + munmap(p, 6 * ps); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); signal(SIGPIPE, SIG_IGN); + if (argc == 6 && !strcmp(argv[1], "--child")) { + int fd = atoi(argv[2]), ready = atoi(argv[3]), release = atoi(argv[4]); + size_t native = strtoul(argv[5], NULL, 10); const size_t ps = 16384; + result(getauxval(AT_PAGESZ) == ps && getpagesize() == ps, "EL0 selects16K pages"); + result(prctl(GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) == 0, "exec consumes alternative selection"); + if (!failed) memory_checks(); + { + int status = 0; pid_t c = fork(); + if (!c) { + if (prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL)) _exit(4); + execl("/page-contract", "/page-contract", NULL); _exit(5); + } + result(c > 0 && waitpid(c, &status, 0) == c && WIFEXITED(status) && !WEXITSTATUS(status), + "16K-to16K exec and page/x18 contract"); + } + unsigned char *shared = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_SHARED, fd, native - ps); + unsigned char *private = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, native - ps); + int ok = shared != MAP_FAILED && private != MAP_FAILED; + if (ok) { + for (unsigned i = 0; i < ps; i++) ok &= shared[i] == 0x7b && private[i] == 0x7b; + memset(private, 0xd8, ps); + for (unsigned i = 0; i < ps; i++) ok &= shared[i] == 0x7b && private[i] == 0xd8; + } + result(ok, "16K file COW copies the whole leaf"); + if (private != MAP_FAILED) munmap(private, ps); + result(shared != MAP_FAILED && pss(shared) == 8, "16K file PSS shares all four4K slices"); + if (shared != MAP_FAILED) shared[ps - 1] = 0xc9; + char token = failed ? 'x' : 'a'; + if (write(ready, &token, 1) != 1 || read(release, &token, 1) != 1) return 3; + if (shared != MAP_FAILED) munmap(shared, ps); + return failed ? 1 : 0; + } + if (getpid() != 1 || argc != 1) { fprintf(stderr, "Disposable VM PID1 only\n"); return 2; } + mount("proc", "/proc", "proc", 0, NULL); + size_t native = getauxval(AT_PAGESZ); int fd = memfd_create("mixed16-smoke", 0); + if (native != 65536 || fd < 0 || ftruncate(fd, native)) { result(0, "64K parent setup"); goto done; } + unsigned char *p = mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { result(0, "parent shared mmap"); goto done; } + memset(p, 0x7b, native); + result(pss(p) == 64, "native64K initial PSS"); + int ready[2], release[2]; if (pipe(ready) || pipe(release)) { result(0, "pipes"); goto done; } + pid_t c = fork(); int status = 0; + if (!c) { + char f[24], r[24], w[24], n[24]; + snprintf(f, sizeof(f), "%d", fd); snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(w, sizeof(w), "%d", release[0]); snprintf(n, sizeof(n), "%zu", native); + close(ready[0]); close(release[1]); + if (prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL)) _exit(4); + execl("/init", "/init", "--child", f, r, w, n, NULL); _exit(5); + } + close(ready[1]); close(release[0]); char token = 0; + int ok = c > 0 && read(ready[0], &token, 1) == 1 && token == 'a'; + result(ok && p[native - 1] == 0xc9, "native64K and16K shared bytes remain coherent"); + result(pss(p) == 56, "native64K PSS accounts one16K alias as four shared slices"); + token = 'b'; if (write(release[1], &token, 1) != 1) failed++; + result(c > 0 && waitpid(c, &status, 0) == c && WIFEXITED(status) && !WEXITSTATUS(status), + "16K child completes"); + printf("# child status=%#x\n", status); + result(pss(p) == 64, "native PSS returns after16K exit"); + munmap(p, native); + done: + printf("USER16K SMOKE %s\n", failed ? "FAIL" : "PASS"); + sync(); reboot(RB_POWER_OFF); for (;;) pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/vhost-log-init.c b/tools/testing/selftests/arm64/user4k/vhost-log-init.c new file mode 100644 index 0000000000000..07d6a8969bca0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/vhost-log-init.c @@ -0,0 +1,392 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#ifndef VHOST_SET_FORK_FROM_OWNER +#define VHOST_SET_FORK_FROM_OWNER _IOW(VHOST_VIRTIO, 0x84, __u8) +#define VHOST_GET_FORK_FROM_OWNER _IOR(VHOST_VIRTIO, 0x85, __u8) +#endif +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +#define N 8 +#define ROUNDS 32 +#define PACKET_SIZE 96 +#define PROTOCOL 0x88b5 +#define DATA_GPA (51ULL * 4096) +#define TX_USED_GPA (71ULL * 4096) +#define RX_USED_GPA (70ULL * 4096) + +struct queue { + struct vring_desc desc[N]; + struct { + uint16_t flags, idx, ring[N + 1]; + } avail; + struct { + uint16_t flags, idx; + struct vring_used_elem ring[N]; + uint16_t event; + } used; + int kick, call; +}; + +static int queue_setup(int fd, struct queue *q, unsigned index) +{ + struct vhost_vring_state state = { .index = index, .num = N }; + struct vhost_vring_addr addr = { + .index = index, + .flags = 1U << VHOST_VRING_F_LOG, + .desc_user_addr = (uintptr_t)q->desc, + .avail_user_addr = (uintptr_t)&q->avail, + .used_user_addr = (uintptr_t)&q->used, + .log_guest_addr = index ? TX_USED_GPA : RX_USED_GPA, + }; + q->kick = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + q->call = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + CHECK(q->kick >= 0 && q->call >= 0); + CHECK(!ioctl(fd, VHOST_SET_VRING_NUM, &state)); + state.num = 0; + CHECK(!ioctl(fd, VHOST_SET_VRING_BASE, &state)); + CHECK(!ioctl(fd, VHOST_SET_VRING_ADDR, &addr)); + struct vhost_vring_file file = { .index = index, .fd = q->kick }; + CHECK(!ioctl(fd, VHOST_SET_VRING_KICK, &file)); + file.fd = q->call; + CHECK(!ioctl(fd, VHOST_SET_VRING_CALL, &file)); + return 0; +} +static int submit(struct queue *q, unsigned round) +{ + uint64_t one = 1; + q->avail.ring[round % N] = 0; + __atomic_store_n(&q->avail.idx, round + 1, __ATOMIC_RELEASE); + return write(q->kick, &one, sizeof(one)) == sizeof(one); +} +static int complete(struct queue *q, unsigned round) +{ + for (unsigned i = 0; i < 2000; i++) { + if (__atomic_load_n(&q->used.idx, __ATOMIC_ACQUIRE) == + round + 1) + return 1; + usleep(1000); + } + printf("queue timeout round=%u used=%u avail=%u\n", round, q->used.idx, + q->avail.idx); + return 0; +} +static int bitmap_ok(unsigned char *leaf, size_t ps, int active, int missing) +{ + for (size_t i = 0; i < ps; i++) { + unsigned char expected = i < ps - 9 ? (missing ? 0 : 0x64) : 0; + if (active && i == ps - 3) + expected = 8; /* RX buffer, GPA page 51 */ + if (active && i == ps - 1) + expected = 0xc0; /* RX/TX rings, pages 70/71 */ + if (__atomic_load_n(leaf + i, __ATOMIC_ACQUIRE) != expected) { + printf("bitmap byte %zu got=%#x expected=%#x\n", i, + leaf[i], expected); + return 0; + } + } + return 1; +} +static int filled(unsigned char *p, size_t len, unsigned char byte) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != byte) + return 0; + return 1; +} +static int exercise_case(size_t ps, unsigned shared, unsigned forked, + uint8_t mode) +{ + int backing = -1; + if (shared) { + backing = memfd_create("vhost-log", 0); + CHECK(backing >= 0 && !ftruncate(backing, 4 * ps)); + } + unsigned char *src = mmap( + (void *)0x20000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), + backing, 0); + unsigned char *dst = + mmap((void *)0x30000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, 4 * ps); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps, *bitmap = leaf + ps - 9; + memset(bitmap, 0, 9); + int barrier[2]; + pid_t child = -1; + if (forked) { + CHECK(!pipe(barrier)); + child = fork(); + CHECK(child >= 0); + if (!child) { + close(barrier[1]); + char token; + int ok = read(barrier[0], &token, 1) == 1 && + bitmap_ok(leaf, ps, shared, 0); + _exit(ok ? 0 : 8); + } + close(barrier[0]); + } + int tap = open("/dev/net/tun", O_RDWR | O_CLOEXEC); + CHECK(tap >= 0); + struct ifreq ifr = { .ifr_flags = IFF_TAP | IFF_NO_PI }; + strcpy(ifr.ifr_name, "abi%d"); + CHECK(!ioctl(tap, TUNSETIFF, &ifr)); + int raw = socket(AF_PACKET, SOCK_RAW | SOCK_CLOEXEC, htons(PROTOCOL)); + CHECK(raw >= 0); + CHECK(!ioctl(raw, SIOCGIFINDEX, &ifr)); + int ifindex = ifr.ifr_ifindex; + CHECK(!ioctl(raw, SIOCGIFFLAGS, &ifr)); + ifr.ifr_flags |= IFF_UP; + CHECK(!ioctl(raw, SIOCSIFFLAGS, &ifr)); + int fd = open("/dev/vhost-net", O_RDWR | O_CLOEXEC); + CHECK(fd >= 0); + CHECK(!ioctl(fd, VHOST_SET_FORK_FROM_OWNER, &mode)); + uint8_t gotmode = 9; + CHECK(!ioctl(fd, VHOST_GET_FORK_FROM_OWNER, &gotmode) && + gotmode == mode); + CHECK(!ioctl(fd, VHOST_SET_OWNER, NULL)); + uint64_t log_base = (uintptr_t)bitmap; + CHECK(!ioctl(fd, VHOST_SET_LOG_BASE, &log_base)); + unsigned char *data = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(data != MAP_FAILED); + struct { + struct vhost_memory mem; + struct vhost_memory_region region; + } mem = { + .mem.nregions = 1, + .region = { .guest_phys_addr = DATA_GPA, + .memory_size = 4096, + .userspace_addr = (uintptr_t)data }, + }; + CHECK(!ioctl(fd, VHOST_SET_MEM_TABLE, &mem)); + uint64_t features = (1ULL << VHOST_F_LOG_ALL) | + (1ULL << VHOST_NET_F_VIRTIO_NET_HDR); + uint64_t available; + CHECK(!ioctl(fd, VHOST_GET_FEATURES, &available) && + (available & features) == features); + CHECK(!ioctl(fd, VHOST_SET_FEATURES, &features)); + struct queue *q = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(q != MAP_FAILED && sizeof(*q) * 2 < ps); + CHECK(!queue_setup(fd, q, 0) && !queue_setup(fd, q + 1, 1)); + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { .index = index, .fd = tap }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, &backend)); + } + struct sockaddr_ll dest = { .sll_family = AF_PACKET, + .sll_protocol = htons(PROTOCOL), + .sll_ifindex = ifindex, + .sll_halen = 6 }; + unsigned char packet[PACKET_SIZE]; + memset(packet, 0x5c, sizeof(packet)); + memset(packet, 0xff, 6); + packet[6] = 2; + packet[12] = PROTOCOL >> 8; + packet[13] = PROTOCOL & 0xff; + q[0].desc[0] = (struct vring_desc){ .addr = DATA_GPA, + .len = 512, + .flags = VRING_DESC_F_WRITE }; + q[1].desc[0] = + (struct vring_desc){ .addr = DATA_GPA + 1024, + .len = sizeof(struct virtio_net_hdr) + + PACKET_SIZE }; + memcpy(data + 1024 + sizeof(struct virtio_net_hdr), packet, + PACKET_SIZE); + for (unsigned phase = 0; phase < 2; phase++) { + if (phase) { + memset(bitmap, 0, 9); + CHECK(!madvise(leaf, ps, MADV_DONTNEED)); + int pagemap = open("/proc/self/pagemap", O_RDONLY); + uint64_t entry; + CHECK(pagemap >= 0 && + pread(pagemap, &entry, sizeof(entry), + (uintptr_t)leaf / ps * sizeof(entry)) == + sizeof(entry)); + CHECK(!(entry & (1ULL << 63))); + close(pagemap); + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { + .index = index, .fd = tap + }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, + &backend)); + } + } + for (unsigned round = phase * ROUNDS; + round < (phase + 1) * ROUNDS; round++) { + packet[20] = round; + memset(data, 0xaa, 512); + CHECK(submit(q, round)); + CHECK(sendto(raw, packet, sizeof(packet), 0, + (void *)&dest, + sizeof(dest)) == sizeof(packet)); + CHECK(submit(q + 1, round)); + CHECK(complete(q, round) && complete(q + 1, round)); + CHECK(q[0].used.ring[round % N].id == 0); + if (q[0].used.ring[round % N].len != + sizeof(struct virtio_net_hdr) + PACKET_SIZE) + printf("RX length=%u expected=%zu ethertype=%02x%02x\n", + q[0].used.ring[round % N].len, + sizeof(struct virtio_net_hdr) + + PACKET_SIZE, + data[sizeof(struct virtio_net_hdr) + 12], + data[sizeof(struct virtio_net_hdr) + 13]); + CHECK(q[0].used.ring[round % N].len == + sizeof(struct virtio_net_hdr) + PACKET_SIZE); + CHECK(!memcmp(data + sizeof(struct virtio_net_hdr), + packet, PACKET_SIZE)); + CHECK(filled(data + sizeof(struct virtio_net_hdr) + + PACKET_SIZE, + 512 - sizeof(struct virtio_net_hdr) - + PACKET_SIZE, + 0xaa)); + } + /* Stop and flush workers before checking the bitmap or discarding backing. */ + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { .index = index, + .fd = -1 }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, &backend)); + } + CHECK(bitmap_ok(leaf, ps, 1, phase && !shared)); + if (forked && !phase) { + CHECK(write(barrier[1], "x", 1) == 1); + close(barrier[1]); + int status; + CHECK(waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status)); + } + } + close(fd); + close(raw); + close(tap); + for (unsigned i = 0; i < 2; i++) { + close(q[i].kick); + close(q[i].call); + } + CHECK(filled(dst, ps, 0x91) && filled(dst + 2 * ps, 2 * ps, 0x91)); + for (unsigned i = 0; i < 3; i++) + CHECK(filled(src + i * ps, ps, 0x61 + i)); + CHECK(!munmap(src, 3 * ps) && !munmap(dst, 4 * ps)); + CHECK(!munmap(data, ps) && !munmap(q, ps)); + if (backing >= 0) + close(backing); + printf("ok - %zuK vhost %s fork=%u worker=%u: %u RX/TX packets, relocated dirty bits, missing-PTE repair and neighbors\n", + ps / 1024, shared ? "shared" : "private", forked, mode, + 2 * ROUNDS); + return 0; +} +static int exercise(size_t expected) +{ + CHECK(getauxval(AT_PAGESZ) == expected); + alarm(45); + for (uint8_t mode = 0; mode < 2; mode++) + for (unsigned shared = 0; shared < 2; shared++) + for (unsigned forked = 0; forked < 2; forked++) + CHECK(!exercise_case(expected, shared, forked, + mode)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + /* Keep unsolicited IPv6 DAD packets out of the isolated TAP queues. */ + int ipv6 = + open("/proc/sys/net/ipv6/conf/default/disable_ipv6", O_WRONLY); + if (ipv6 < 0 || write(ipv6, "1", 1) != 1) { + perror("disable IPv6"); + fail = 1; + goto done; + } + close(ipv6); + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24]; + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = 0; + for (unsigned ticks = 0; c > 0 && ticks < 600; ticks++) { + pid_t got = waitpid(c, &status, WNOHANG); + if (got == c) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 599) { + kill(c, SIGKILL); + waitpid(c, &status, 0); + break; + } + usleep(100000); + } + printf("%s - ABI %zuK vhost log status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("VHOST LOG %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/wine-kernel32.def b/tools/testing/selftests/arm64/user4k/wine-kernel32.def new file mode 100644 index 0000000000000..891e48c1fb83b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/wine-kernel32.def @@ -0,0 +1,20 @@ +LIBRARY KERNEL32.dll +EXPORTS +VirtualAlloc +VirtualFree +VirtualProtect +VirtualQuery +GetSystemInfo +GetStdHandle +WriteFile +ExitProcess +GetLastError +AddVectoredExceptionHandler +RemoveVectoredExceptionHandler +CreateThread +WaitForSingleObject +GetExitCodeThread +CloseHandle +CreateFileMappingA +MapViewOfFile +UnmapViewOfFile diff --git a/tools/testing/selftests/arm64/user4k/wine-page-contract.c b/tools/testing/selftests/arm64/user4k/wine-page-contract.c new file mode 100644 index 0000000000000..c72875c044e08 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/wine-page-contract.c @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding Windows x64 PE; Win32 imports, no CRT or external SDK. */ +typedef unsigned long DWORD; +typedef int BOOL; +typedef unsigned long long SIZE_T; +typedef void *HANDLE; +#define API __declspec(dllimport) +API void *VirtualAlloc(void *, SIZE_T, DWORD, DWORD); +API BOOL VirtualFree(void *, SIZE_T, DWORD); +API BOOL VirtualProtect(void *, SIZE_T, DWORD, DWORD *); +API SIZE_T VirtualQuery(const void *, void *, SIZE_T); +API void GetSystemInfo(void *); +API HANDLE GetStdHandle(DWORD); +API BOOL WriteFile(HANDLE, const void *, DWORD, DWORD *, void *); +API void ExitProcess(DWORD); +API DWORD GetLastError(void); +API void *AddVectoredExceptionHandler(DWORD, void *); +API DWORD RemoveVectoredExceptionHandler(void *); +API HANDLE CreateThread(void *, SIZE_T, DWORD (*)(void *), void *, DWORD, DWORD *); +API DWORD WaitForSingleObject(HANDLE, DWORD); +API BOOL GetExitCodeThread(HANDLE, DWORD *); +API BOOL CloseHandle(HANDLE); +API HANDLE CreateFileMappingA(HANDLE, void *, DWORD, DWORD, DWORD, const char *); +API void *MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); +API BOOL UnmapViewOfFile(const void *); +struct system_info { DWORD arch, page; void *min, *max; SIZE_T mask; DWORD cpus, type, granule; unsigned short level, revision; }; +struct memory_info { void *base, *allocation; DWORD allocation_protect, pad; SIZE_T size; DWORD state, protect, type, pad2; }; +struct exception_record { DWORD code, flags; void *record, *address; DWORD count, pad; SIZE_T info[15]; }; +struct exception_pointers { struct exception_record *record; void *context; }; +/* Keep the freestanding declarations consistent with the Windows x64 ABI. */ +_Static_assert(sizeof(DWORD) == 4, "DWORD must be 32 bits"); +_Static_assert(sizeof(SIZE_T) == 8, "SIZE_T must be 64 bits"); +_Static_assert(sizeof(struct system_info) == 48, "SYSTEM_INFO layout"); +_Static_assert(sizeof(struct memory_info) == 48, "MEMORY_BASIC_INFORMATION layout"); +_Static_assert(sizeof(struct exception_record) == 152, "EXCEPTION_RECORD layout"); +static volatile unsigned char *pages; +static volatile DWORD handled; +static void say(const char *s) +{ + DWORD n=0,w=0;while(s[n])n++;WriteFile(GetStdHandle((DWORD)-11),s,n,&w,0); +} +static void number(DWORD n) +{ + char buf[11],*p=buf+10;*p=0; + do{*--p='0'+n%10;n/=10;}while(n); + say(p); +} +static void fail(DWORD stage) +{ + DWORD e=GetLastError(); + say("WINE PAGE CONTRACT FAIL stage=");number(stage); + say(" error=");number(e);say("\r\n");ExitProcess(1); +} +#define CHECK(x,n) do{if(!(x))fail(n);}while(0) +static long handler(struct exception_pointers *e) +{ + DWORD old; + if(e->record->code!=0xc0000005 || e->record->count<2 || e->record->info[0]!=1 || e->record->info[1]!=(SIZE_T)(pages+4103))return 0; + if(!VirtualProtect((void *)(pages+4096),4096,4,&old))return 0; + handled++;return -1; +} +static DWORD thread(void *arg) +{ + volatile unsigned char *p=arg; + for(unsigned int i=0;i<4096;i++)p[i]=(unsigned char)(i^0x5a); + return 37; +} +void mainCRTStartup(void) +{ + struct system_info si;struct memory_info mi;DWORD old,status; + GetSystemInfo(&si);CHECK(si.page==4096 && si.granule==65536,1); + say("ok - Wine reports 4K pages and 64K allocation granularity\r\n"); + pages=VirtualAlloc(0,65536,0x2000,1);CHECK(pages,2); + CHECK(VirtualAlloc((void *)pages,12288,0x1000,4)==pages,3); + for(unsigned int i=0;i<12288;i++)pages[i]=(unsigned char)(i^(i>>12)); + CHECK(VirtualProtect((void *)(pages+4096),4096,2,&old) && old==4,4); + CHECK(VirtualQuery((void *)(pages+4096),&mi,sizeof(mi))==sizeof(mi) && mi.base==pages+4096 && mi.size==4096 && mi.protect==2,5); + CHECK(VirtualQuery((void *)(pages+8192),&mi,sizeof(mi))==sizeof(mi) && mi.protect==4,6); + void *veh=AddVectoredExceptionHandler(1,handler);CHECK(veh,7); + pages[4103]=0x6d;CHECK(handled==1 && pages[4103]==0x6d,8); + CHECK(RemoveVectoredExceptionHandler(veh),9); + say("ok - Wine independent 4K protection and handled access violation\r\n"); + CHECK(VirtualFree((void *)(pages+4096),4096,0x4000),10); + CHECK(VirtualQuery((void *)(pages+4096),&mi,sizeof(mi))==sizeof(mi) && mi.state==0x2000,11); + CHECK(VirtualAlloc((void *)(pages+4096),4096,0x1000,4)==pages+4096,12); + for(unsigned int i=0;i<4096;i++)CHECK(!pages[4096+i],13); + for(unsigned int i=0;i<4096;i++)CHECK(pages[i]==(unsigned char)i && pages[8192+i]==(unsigned char)(i^2),14); + say("ok - Wine 4K decommit and zero-filled recommit preserve neighbours\r\n"); + HANDLE t=CreateThread(0,0,thread,(void *)(pages+4096),0,0);CHECK(t,15); + CHECK(WaitForSingleObject(t,30000)==0 && GetExitCodeThread(t,&status) && status==37,16);CHECK(CloseHandle(t),17); + for(unsigned int i=0;i<4096;i++)CHECK(pages[4096+i]==(unsigned char)(i^0x5a),18); + say("ok - Wine threads share committed 4K pages\r\n"); + HANDLE file=CreateFileMappingA((HANDLE)(SIZE_T)-1,0,4,0,16384,0);CHECK(file,19); + unsigned char *a=MapViewOfFile(file,2,0,0,16384),*b=MapViewOfFile(file,2,0,0,16384);CHECK(a&&b,20); + a[4099]=0xa7;CHECK(b[4099]==0xa7,21); + CHECK(VirtualProtect(a+4096,4096,2,&old),22);b[4099]=0x53;CHECK(a[4099]==0x53,23); + CHECK(UnmapViewOfFile(a)&&UnmapViewOfFile(b)&&CloseHandle(file),24); + CHECK(VirtualFree((void *)pages,0,0x8000),25); + say("ok - Wine shared section aliases and 4K protection\r\nWINE PAGE CONTRACT PASS\r\n"); + ExitProcess(0); +} diff --git a/tools/testing/selftests/arm64/user4k/x18-probe.S b/tools/testing/selftests/arm64/user4k/x18-probe.S new file mode 100644 index 0000000000000..565f34e8323f5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/x18-probe.S @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include + + .text + .globl x18_syscall + .type x18_syscall, %function +/* (nr, arg0, arg1, arg2, observed_x18); no libc code inside the probe. */ +x18_syscall: + hint 34 /* bti c */ + stp x18, x19, [sp, #-16]! + mov x19, x4 + movz x18, #0x3210 + movk x18, #0x7654, lsl #16 + movk x18, #0xba98, lsl #32 + movk x18, #0xfedc, lsl #48 + mov x8, x0 + mov x0, x1 + mov x1, x2 + mov x2, x3 + svc #0 + str x18, [x19] + ldp x18, x19, [sp], #16 + ret + .size x18_syscall, .-x18_syscall + .section .note.GNU-stack,"",%progbits diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 9e0dfd3a85b0e..8d0377cda51f0 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -533,6 +533,9 @@ struct vm_area_desc { /* Mutable fields. Populated with initial state. */ pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif struct file *vm_file; vma_flags_t vma_flags; pgprot_t page_prot; @@ -1300,6 +1303,9 @@ static inline void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->end = vma->vm_end; desc->pgoff = vma->vm_pgoff; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = vma->vm_subpage_offset; +#endif desc->vm_file = vma->vm_file; desc->vma_flags = vma->flags; desc->page_prot = vma->vm_page_prot; @@ -1308,6 +1314,117 @@ static inline void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->action.type = MMAP_NOTHING; } +static inline unsigned int mm_page_shift(const struct mm_struct *mm) +{ + return PAGE_SHIFT; +} + +static inline unsigned long mm_page_size(const struct mm_struct *mm) +{ + return PAGE_SIZE; +} + +static inline unsigned long mm_page_mask(const struct mm_struct *mm) +{ + return PAGE_MASK; +} + +static inline unsigned long mm_page_align(const struct mm_struct *mm, unsigned long size) +{ + return PAGE_ALIGN(size); +} + +static inline unsigned long vma_user_pages(const struct vm_area_struct *vma) +{ + return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; +} + +#define mm_account_memory(mm, policy_mm, pages) security_vm_enough_memory_mm(policy_mm, pages) +#define mm_unacct_memory(mm, pages) vm_unacct_memory(pages) + +/* File-cache indices remain native-page units even for a 4K VMA. */ +struct vm_page_offset { + pgoff_t index; + unsigned int offset; +}; + +static inline struct vm_page_offset vm_page_offset_add(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = pos.offset + (bytes & ~PAGE_MASK); + + pos.index += (bytes >> PAGE_SHIFT) + (low >> PAGE_SHIFT); + pos.offset = low & ~PAGE_MASK; + return pos; +} + +static inline struct vm_page_offset vm_page_offset_sub(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = bytes & ~PAGE_MASK; + + pos.index -= (bytes >> PAGE_SHIFT) + (low > pos.offset); + pos.offset = (pos.offset - low) & ~PAGE_MASK; + return pos; +} + +static inline bool vm_page_offset_equal(struct vm_page_offset a, struct vm_page_offset b) +{ + return a.index == b.index && a.offset == b.offset; +} + +static inline unsigned int vma_subpage_offset(const struct vm_area_struct *vma) +{ +#ifdef CONFIG_MM_SUBPAGE + return vma->vm_subpage_offset; +#else + return 0; +#endif +} + +/* address must be >= vm_start; vm_end computes the next contiguous offset. */ +static inline struct vm_page_offset vma_page_offset_at(const struct vm_area_struct *vma, + unsigned long address) +{ + unsigned long delta = address - vma->vm_start; + unsigned long low = (delta & ~PAGE_MASK) + vma_subpage_offset(vma); + + return (struct vm_page_offset) { + .index = vma->vm_pgoff + (delta >> PAGE_SHIFT) + (low >> PAGE_SHIFT), + .offset = low & ~PAGE_MASK, + }; +} + +static inline void vma_set_page_offset(struct vm_area_struct *vma, struct vm_page_offset pos) +{ + vma->vm_pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = pos.offset; +#endif +} + +/* Inverse lookup of a single byte; an offset outside the VMA is not clamped. */ +static inline unsigned long vma_address_at_offset(const struct vm_area_struct *vma, + struct vm_page_offset pos) +{ + unsigned long len = vma->vm_end - vma->vm_start; + unsigned long index, delta, last; + unsigned int offset = vma_subpage_offset(vma); + + if (!len || pos.index < vma->vm_pgoff || pos.offset >= PAGE_SIZE) + return -EFAULT; + index = pos.index - vma->vm_pgoff; + last = ((len - 1) >> PAGE_SHIFT) + + ((((len - 1) & ~PAGE_MASK) + offset) >> PAGE_SHIFT); + if (index > last) + return -EFAULT; + delta = (index << PAGE_SHIFT) + pos.offset; + if (delta < offset || delta - offset >= len) + return -EFAULT; + return vma->vm_start + delta - offset; +} + + static inline unsigned long vma_pages(const struct vm_area_struct *vma) { return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; diff --git a/tools/testing/vma/tests/merge.c b/tools/testing/vma/tests/merge.c index 03b6f9820e0ac..0b0f4ae413973 100644 --- a/tools/testing/vma/tests/merge.c +++ b/tools/testing/vma/tests/merge.c @@ -227,7 +227,7 @@ static bool test_simple_shrink(void) ASSERT_FALSE(attach_vma(&mm, vma)); - ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, 0)); + ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, (struct vm_page_offset) { 0, 0 })); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x1000); diff --git a/tools/testing/vma/tests/mmap.c b/tools/testing/vma/tests/mmap.c index c85bc000d1cb7..c87baa48ea26e 100644 --- a/tools/testing/vma/tests/mmap.c +++ b/tools/testing/vma/tests/mmap.c @@ -12,19 +12,19 @@ static bool test_mmap_region_basic(void) current->mm = &mm; /* Map at 0x300000, length 0x3000. */ - addr = __mmap_region(NULL, 0x300000, 0x3000, vma_flags, 0x300, NULL); + addr = __mmap_region(NULL, 0x300000, 0x3000, vma_flags, (struct vm_page_offset) { 0x300, 0 }, NULL); ASSERT_EQ(addr, 0x300000); /* Map at 0x250000, length 0x3000. */ - addr = __mmap_region(NULL, 0x250000, 0x3000, vma_flags, 0x250, NULL); + addr = __mmap_region(NULL, 0x250000, 0x3000, vma_flags, (struct vm_page_offset) { 0x250, 0 }, NULL); ASSERT_EQ(addr, 0x250000); /* Map at 0x303000, merging to 0x300000 of length 0x6000. */ - addr = __mmap_region(NULL, 0x303000, 0x3000, vma_flags, 0x303, NULL); + addr = __mmap_region(NULL, 0x303000, 0x3000, vma_flags, (struct vm_page_offset) { 0x303, 0 }, NULL); ASSERT_EQ(addr, 0x303000); /* Map at 0x24d000, merging to 0x250000 of length 0x6000. */ - addr = __mmap_region(NULL, 0x24d000, 0x3000, vma_flags, 0x24d, NULL); + addr = __mmap_region(NULL, 0x24d000, 0x3000, vma_flags, (struct vm_page_offset) { 0x24d, 0 }, NULL); ASSERT_EQ(addr, 0x24d000); ASSERT_EQ(mm.map_count, 2); diff --git a/tools/testing/vma/tests/vma.c b/tools/testing/vma/tests/vma.c index 754a2da063215..38a7cc6702f23 100644 --- a/tools/testing/vma/tests/vma.c +++ b/tools/testing/vma/tests/vma.c @@ -38,7 +38,7 @@ static bool test_copy_vma(void) /* Move backwards and do not merge. */ vma = alloc_and_link_vma(&mm, 0x3000, 0x5000, 3, vma_flags); - vma_new = copy_vma(&vma, 0, 0x2000, 0, &need_locks); + vma_new = copy_vma(&vma, 0, 0x2000, (struct vm_page_offset) { 0, 0 }, &need_locks); ASSERT_NE(vma_new, vma); ASSERT_EQ(vma_new->vm_start, 0); ASSERT_EQ(vma_new->vm_end, 0x2000); @@ -51,7 +51,7 @@ static bool test_copy_vma(void) vma = alloc_and_link_vma(&mm, 0, 0x2000, 0, vma_flags); vma_next = alloc_and_link_vma(&mm, 0x6000, 0x8000, 6, vma_flags); - vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, &need_locks); + vma_new = copy_vma(&vma, 0x4000, 0x2000, (struct vm_page_offset) { 4, 0 }, &need_locks); vma_assert_attached(vma_new); ASSERT_EQ(vma_new, vma_next); From 42e94c2a616946ab34e689027ee1dda0a061779f Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:08:35 -0500 Subject: [PATCH 0341/1579] dt-bindings: mailbox: apple,mailbox: Allow the ANE mailbox variant The T6021 (M2 Max) ANE coprocessor mailbox is an ASC mailbox, but only one of its interrupts is known: the Apple device tree gives the ANE a single interrupt line. The binding requires the four ASC mailbox interrupts in a fixed order, so a node for this mailbox fails dtbs_check. Add the apple,t6021-ane-mailbox compatible with the apple,asc-mailbox-v4 fallback. For it, recv-not-empty is the only required interrupt; send-empty, send-not-empty and recv-empty may follow in that order. Without send-empty, the OS must poll the FIFO status before it sends. The other variants keep their constraints. The four-interrupt rule moves into a condition that excludes the ANE and AOP setup mailboxes, and the AOP setup mailbox states its minimum of two interrupts, which the top-level limit gave it before. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- .../bindings/mailbox/apple,mailbox.yaml | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index ee40e13c9a2b9..96675c53c3dd0 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -36,6 +36,13 @@ properties: - apple,t6020-asc-mailbox - const: apple,asc-mailbox-v4 + - description: + The T6021 ANE coprocessor mailbox is an ASC mailbox. Only its + recv-not-empty interrupt is known. + items: + - const: apple,t6021-ane-mailbox + - const: apple,asc-mailbox-v4 + - description: AOP setup-port mailbox with a separate register layout. const: apple,t8140-aop-setup-mailbox @@ -62,11 +69,11 @@ properties: maxItems: 1 interrupts: - minItems: 2 + minItems: 1 maxItems: 4 interrupt-names: - minItems: 2 + minItems: 1 maxItems: 4 iommus: @@ -95,6 +102,7 @@ allOf: then: properties: interrupts: + minItems: 2 maxItems: 2 interrupt-names: items: @@ -104,6 +112,34 @@ allOf: - iommus - apple,dma-range else: + properties: + iommus: false + apple,dma-range: false + + - if: + properties: + compatible: + contains: + const: apple,t6021-ane-mailbox + then: + properties: + interrupt-names: + minItems: 1 + items: + - const: recv-not-empty + - const: send-empty + - const: send-not-empty + - const: recv-empty + + - if: + properties: + compatible: + not: + contains: + enum: + - apple,t6021-ane-mailbox + - apple,t8140-aop-setup-mailbox + then: properties: interrupts: minItems: 4 @@ -113,8 +149,6 @@ allOf: - const: send-not-empty - const: recv-empty - const: recv-not-empty - iommus: false - apple,dma-range: false required: - compatible From 5e51767dde52355c207df48a9a61d32d5c91fca8 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:08:36 -0500 Subject: [PATCH 0342/1579] soc: apple: mailbox: Make the send-empty IRQ optional (poll TX) The T6021 (M2 Max) ANE coprocessor mailbox has one known AIC line, which the device tree names recv-not-empty. apple-mailbox insists on a send-empty IRQ as well, so it cannot bind that mailbox. Look send-empty up with platform_get_irq_byname_optional(). When the node has none (-ENXIO), do not request it, and let apple_mbox_send() wait for FIFO space by polling the A2I control register, the way its atomic path already does (100 us steps, 500 ms timeout, under tx_lock). Any other lookup error still fails the probe with -ENODEV. A mailbox whose node names send-empty takes the old path unchanged: the IRQ is requested at the same point with the same flags and name, and a non-atomic send still sleeps on tx_empty. All 71 apple-mailbox nodes in arch/arm64/boot/dts/apple name send-empty today. The Rust wrapper (RUST_APPLE_MAILBOX) needs no change. It only calls apple_mbox_get_byname(), apple_mbox_start(), apple_mbox_send() and apple_mbox_stop(), and never reads the IRQ fields, so a Rust client gets the poll through apple_mbox_send(). An equivalent change ran on an M2 Max lab kernel (Asahi 7.1.13 base) from 2026-09-24 to 2026-09-30, where apple-mailbox bound the one-interrupt ANE mailbox. The FIFO-full poll itself did not run there: with the ANE firmware in use (from macOS 13.5), the host sends no mailbox message after probe. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/soc/apple/mailbox.c | 33 ++++++++++++++++++++----------- include/linux/soc/apple/mailbox.h | 2 +- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 16631fc352f44..4a4df15707ce3 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -121,7 +121,7 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); while (mbox_ctrl & mbox->hw->control_full) { - if (atomic) { + if (atomic || mbox->irq_send_empty < 0) { ret = readl_poll_timeout_atomic( mbox->regs + mbox->hw->a2i_control, mbox_ctrl, !(mbox_ctrl & mbox->hw->control_full), 100, @@ -397,8 +397,13 @@ static int apple_mbox_probe(struct platform_device *pdev) if (mbox->irq_recv_not_empty < 0) return -ENODEV; - mbox->irq_send_empty = platform_get_irq_byname(pdev, "send-empty"); - if (mbox->irq_send_empty < 0) + /* + * Some coprocessors (e.g. the T6021 ANE ASC) have no send-empty line. + * apple_mbox_send then polls the A2I control register instead. + */ + mbox->irq_send_empty = + platform_get_irq_byname_optional(pdev, "send-empty"); + if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) return -ENODEV; spin_lock_init(&mbox->rx_lock); @@ -415,15 +420,19 @@ static int apple_mbox_probe(struct platform_device *pdev) if (ret) return ret; - irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-send", dev_name(dev)); - if (!irqname) - return -ENOMEM; - - ret = devm_request_irq(dev, mbox->irq_send_empty, - apple_mbox_send_empty_irq, - IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); - if (ret) - return ret; + if (mbox->irq_send_empty >= 0) { + irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-send", + dev_name(dev)); + if (!irqname) + return -ENOMEM; + + ret = devm_request_irq(dev, mbox->irq_send_empty, + apple_mbox_send_empty_irq, + IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, + irqname, mbox); + if (ret) + return ret; + } ret = devm_pm_runtime_enable(dev); if (ret) diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index 9cfbbac411b8e..f7e76b554272c 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -24,7 +24,7 @@ struct apple_mbox { bool active; int irq_recv_not_empty; - int irq_send_empty; + int irq_send_empty; /* < 0: no send-empty IRQ, TX polls */ spinlock_t rx_lock; spinlock_t tx_lock; From 88af7870bb374252151e2f4a348fdbe5e264bccb Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:17:01 -0500 Subject: [PATCH 0343/1579] dt-bindings: npu: apple,ane: Add the Apple Neural Engine Add a binding for the Neural Engine (ANE) of the Apple T8103 (M1), T6000 (M1 Pro, Max and Ultra), T6020, T6021, T6022 (M2 Pro, Max and Ultra) and T8112 (M2). These are the compatibles that the ane and ane_t6021 drivers match. On T8103 and T6000 the node has the engine register window, one interrupt, the three ANE DARTs and five power domains. On T602x a coprocessor runs the ANE firmware, so the node also has the PMGR and power state windows, eight power domains, the coprocessor reset and mailbox, and an IOVA-only reservation for the firmware entry alias. The T8112 node has the T602x windows plus the two eFuse words that give the SoC revision to the firmware, seven power domains and the mailbox. The schema goes in npu/ with the other neural processing units, and in the ARM/APPLE MACHINE SUPPORT entry with the other Apple bindings. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- .../devicetree/bindings/npu/apple,ane.yaml | 198 ++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 199 insertions(+) create mode 100644 Documentation/devicetree/bindings/npu/apple,ane.yaml diff --git a/Documentation/devicetree/bindings/npu/apple,ane.yaml b/Documentation/devicetree/bindings/npu/apple,ane.yaml new file mode 100644 index 0000000000000..ebca460a81bc1 --- /dev/null +++ b/Documentation/devicetree/bindings/npu/apple,ane.yaml @@ -0,0 +1,198 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/npu/apple,ane.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Neural Engine + +maintainers: + - Joshua Warren <816217+joshuaswarren@users.noreply.github.com> + +description: + The Apple Neural Engine (ANE) is the neural network accelerator of Apple + SoCs. It accesses memory through three DARTs. On T8103 and T6000 the host + drives the engine through its register window. On T602x and T8112 a + coprocessor runs the ANE firmware and the host talks to it through an ASC + mailbox. + +properties: + compatible: + enum: + - apple,t8103-ane + - apple,t6000-ane + - apple,t6020-ane + - apple,t6021-ane + - apple,t6022-ane + - apple,t8112-ane + + reg: + minItems: 1 + items: + - description: Engine registers. On T602x and T8112 this is the whole + ANE block, including the coprocessor. + - description: Power manager (PMGR) registers of the ANE + - description: ANE power state registers in the PMGR + - description: eFuse words that give the SoC revision to the ANE + firmware + + reg-names: + minItems: 1 + items: + - const: engine + - const: pmgr + - const: set + - const: fuse + + interrupts: + maxItems: 1 + + interrupt-names: + items: + - const: ane + + iommus: + maxItems: 3 + + power-domains: + description: + The power states the engine needs, five on T8103 and T6000, eight on + T602x and seven on T8112. + minItems: 5 + maxItems: 8 + + resets: + maxItems: 1 + description: Reset of the ANE coprocessor + + mboxes: + maxItems: 1 + description: Mailbox of the ANE coprocessor + + memory-region: + maxItems: 1 + description: + IOVA-only reservation of the firmware entry alias in the ANE DART + address space. + +required: + - compatible + - reg + - reg-names + - interrupts + - interrupt-names + - iommus + - power-domains + +allOf: + - if: + properties: + compatible: + contains: + enum: + - apple,t8103-ane + - apple,t6000-ane + then: + properties: + reg: + maxItems: 1 + reg-names: + maxItems: 1 + power-domains: + maxItems: 5 + resets: false + mboxes: false + memory-region: false + + - if: + properties: + compatible: + contains: + enum: + - apple,t6020-ane + - apple,t6021-ane + - apple,t6022-ane + then: + properties: + reg: + minItems: 3 + maxItems: 3 + reg-names: + minItems: 3 + maxItems: 3 + power-domains: + minItems: 8 + required: + - resets + - mboxes + - memory-region + + - if: + properties: + compatible: + contains: + const: apple,t8112-ane + then: + properties: + reg: + minItems: 4 + reg-names: + minItems: 4 + power-domains: + minItems: 7 + maxItems: 7 + resets: false + memory-region: false + required: + - mboxes + +additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + ane@26bc04000 { + compatible = "apple,t8103-ane"; + reg = <0x2 0x6bc04000 0x0 0x24000>; + reg-names = "engine"; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "ane"; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_set1>, <&ps_ane_set2>, + <&ps_ane_set3>, <&ps_ane_set4>, <&ps_ane_set5>; + }; + }; + + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + ane@284000000 { + compatible = "apple,t6021-ane"; + reg = <0x2 0x84000000 0x0 0x2000000>, + <0x2 0x8e080000 0x0 0x4034>, + <0x2 0x8e08c000 0x0 0x4000>; + reg-names = "engine", "pmgr", "set"; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "ane"; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_sys_mpm>, <&ps_ane_td>, + <&ps_ane_base>, <&ps_ane_set1>, <&ps_ane_set2>, + <&ps_ane_set3>, <&ps_ane_set4>, <&ps_ane_cpu>; + resets = <&ps_ane_cpu>; + mboxes = <&ane_mbox>; + memory-region = <&ane_alias_iova>; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index 368bf2b43af1c..6cbc8ab2d3679 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2570,6 +2570,7 @@ F: Documentation/devicetree/bindings/leds/backlight/apple,dwi-bl.yaml F: Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml F: Documentation/devicetree/bindings/mfd/apple,smc.yaml F: Documentation/devicetree/bindings/net/bluetooth/brcm,bcm4377-bluetooth.yaml +F: Documentation/devicetree/bindings/npu/apple,ane.yaml F: Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml F: Documentation/devicetree/bindings/nvmem/apple,efuses.yaml F: Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml From 7a18e9d53b12ad80b4ec4a40eaef05cc0cb31b38 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:17:02 -0500 Subject: [PATCH 0344/1579] dt-bindings: mailbox: apple,mailbox: Add the T8112 ANE mailbox The T8112 (M2) ANE coprocessor has an ASC mailbox at the same offset in its block as the T6021 ANE, and the Apple device tree also gives that ANE a single interrupt line (520). Add apple,t8112-ane-mailbox to the ANE mailbox variant, with the same interrupt rules as apple,t6021-ane-mailbox. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- .../devicetree/bindings/mailbox/apple,mailbox.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index 96675c53c3dd0..32fb3acb8c99f 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -37,10 +37,12 @@ properties: - const: apple,asc-mailbox-v4 - description: - The T6021 ANE coprocessor mailbox is an ASC mailbox. Only its - recv-not-empty interrupt is known. + The T6021 and T8112 ANE coprocessor mailboxes are ASC mailboxes. + Only their recv-not-empty interrupt is known. items: - - const: apple,t6021-ane-mailbox + - enum: + - apple,t6021-ane-mailbox + - apple,t8112-ane-mailbox - const: apple,asc-mailbox-v4 - description: @@ -120,7 +122,9 @@ allOf: properties: compatible: contains: - const: apple,t6021-ane-mailbox + enum: + - apple,t6021-ane-mailbox + - apple,t8112-ane-mailbox then: properties: interrupt-names: @@ -138,6 +142,7 @@ allOf: contains: enum: - apple,t6021-ane-mailbox + - apple,t8112-ane-mailbox - apple,t8140-aop-setup-mailbox then: properties: From 35a088238be14af6500c84662c017ddf06999b2c Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:22:08 -0500 Subject: [PATCH 0345/1579] accel/ane: Add the Apple Neural Engine driver Add a DRM accel driver for the Apple Neural Engine (ANE). One Kconfig symbol, DRM_ACCEL_ANE, builds two modules from this directory: - ane binds apple,t8103-ane and apple,t6000-ane (M1, and die 0 of the M1 Pro, Max and Ultra). Linux starts no ANE firmware on these SoCs: the host writes task descriptors to the engine's task queue and maps the buffers through the three ANE DARTs (ABI 1, DRM_IOCTL_ANE_SUBMIT). - ane_t6021 binds apple,t6020-ane, apple,t6021-ane, apple,t6022-ane and apple,t8112-ane (M2 Pro, Max and Ultra die 0, and M2). On these SoCs Linux must start the ANE coprocessor. The driver loads the firmware with request_firmware(), checks it against a SHA-256 pin, copies it to its own DMA memory, starts the coprocessor, and sends LOAD, CREATE and CALL commands on the firmware's command ring (ABI 2, DRM_IOCTL_ANE_PROG_LOAD, DRM_IOCTL_ANE_PROC_CREATE and DRM_IOCTL_ANE_EXEC). Both register /dev/accel/accel*. include/uapi/drm/ane_accel.h has the ioctls of both ABIs, and the DRM major version tells userspace which ABI the device has. The sources are those of the out-of-tree omarchy-ane driver at tag v0.4.0 (16d13ef52dc6), which started from the ane driver of Eileen Yoon. The move changes only the build glue: the SPDX comment style, the include path of the UAPI header, an include of drm.h and the Linux-syscall-note in the UAPI header, and the module version, which the out-of-tree build took from git describe (MODULE_VERSION and one log line). Compiled for arm64, every object has the same code as the out-of-tree build, except for the argument list of that log line. The out-of-tree build of this code ran on T8103 (M1), T6001 (M1 Max) and T6021 (M2 Max) with linux-asahi 7.1.13. T6000, T6002, T6020, T6022 and T8112 have not run it. ane also matches apple,t6020-ane and apple,t6021-ane and refuses them at probe; ane_t6021 binds them. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- MAINTAINERS | 9 + drivers/accel/Kconfig | 1 + drivers/accel/Makefile | 1 + drivers/accel/ane/Kconfig | 23 + drivers/accel/ane/Makefile | 14 + drivers/accel/ane/ane.h | 160 ++ drivers/accel/ane/ane_boost.c | 145 ++ drivers/accel/ane/ane_dart.c | 271 +++ drivers/accel/ane/ane_drv.c | 1234 ++++++++++ drivers/accel/ane/ane_fw_validate.h | 480 ++++ drivers/accel/ane/ane_t6021.h | 796 +++++++ drivers/accel/ane/ane_t6021_boot.c | 557 +++++ drivers/accel/ane/ane_t6021_boot.h | 832 +++++++ drivers/accel/ane/ane_t6021_fwload.c | 645 +++++ drivers/accel/ane/ane_t6021_rtclient_main.c | 2363 +++++++++++++++++++ drivers/accel/ane/ane_tm.c | 652 +++++ drivers/accel/ane/ane_tm.h | 20 + include/uapi/drm/ane_accel.h | 130 + 18 files changed, 8333 insertions(+) create mode 100644 drivers/accel/ane/Kconfig create mode 100644 drivers/accel/ane/Makefile create mode 100644 drivers/accel/ane/ane.h create mode 100644 drivers/accel/ane/ane_boost.c create mode 100644 drivers/accel/ane/ane_dart.c create mode 100644 drivers/accel/ane/ane_drv.c create mode 100644 drivers/accel/ane/ane_fw_validate.h create mode 100644 drivers/accel/ane/ane_t6021.h create mode 100644 drivers/accel/ane/ane_t6021_boot.c create mode 100644 drivers/accel/ane/ane_t6021_boot.h create mode 100644 drivers/accel/ane/ane_t6021_fwload.c create mode 100644 drivers/accel/ane/ane_t6021_rtclient_main.c create mode 100644 drivers/accel/ane/ane_tm.c create mode 100644 drivers/accel/ane/ane_tm.h create mode 100644 include/uapi/drm/ane_accel.h diff --git a/MAINTAINERS b/MAINTAINERS index 6cbc8ab2d3679..9b91fa8f0f987 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7898,6 +7898,15 @@ S: Maintained F: drivers/soc/ti/smartreflex.c F: include/linux/power/smartreflex.h +DRM ACCEL DRIVER FOR APPLE NEURAL ENGINE +M: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> +L: asahi@lists.linux.dev +L: dri-devel@lists.freedesktop.org +S: Maintained +F: Documentation/devicetree/bindings/npu/apple,ane.yaml +F: drivers/accel/ane/ +F: include/uapi/drm/ane_accel.h + DRM ACCEL DRIVERS FOR INTEL VPU M: Maciej Falkowski M: Karol Wachowski diff --git a/drivers/accel/Kconfig b/drivers/accel/Kconfig index bdf48ccafcf21..9e641c2c30ae1 100644 --- a/drivers/accel/Kconfig +++ b/drivers/accel/Kconfig @@ -25,6 +25,7 @@ menuconfig DRM_ACCEL and debugfs). source "drivers/accel/amdxdna/Kconfig" +source "drivers/accel/ane/Kconfig" source "drivers/accel/ethosu/Kconfig" source "drivers/accel/habanalabs/Kconfig" source "drivers/accel/ivpu/Kconfig" diff --git a/drivers/accel/Makefile b/drivers/accel/Makefile index 1d3a7251b950f..235ebf723025c 100644 --- a/drivers/accel/Makefile +++ b/drivers/accel/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_DRM_ACCEL_AMDXDNA) += amdxdna/ +obj-$(CONFIG_DRM_ACCEL_ANE) += ane/ obj-$(CONFIG_DRM_ACCEL_ARM_ETHOSU) += ethosu/ obj-$(CONFIG_DRM_ACCEL_HABANALABS) += habanalabs/ obj-$(CONFIG_DRM_ACCEL_IVPU) += ivpu/ diff --git a/drivers/accel/ane/Kconfig b/drivers/accel/ane/Kconfig new file mode 100644 index 0000000000000..a1e8766340621 --- /dev/null +++ b/drivers/accel/ane/Kconfig @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config DRM_ACCEL_ANE + tristate "Apple Neural Engine (ANE)" + depends on DRM_ACCEL + depends on ARCH_APPLE || COMPILE_TEST + depends on APPLE_RTKIT + depends on APPLE_DART || COMPILE_TEST + depends on MMU + select CRYPTO_LIB_SHA256 + help + Choose this option if you have an Apple SoC with a Neural Engine: + M1 (T8103), M1 Pro, Max and Ultra (T6000, T6001, T6002), M2 + (T8112), or M2 Pro, Max and Ultra (T6020, T6021, T6022). + + The interface exposed to userspace is described in + include/uapi/drm/ane_accel.h. + + If unsure, say N. + + To compile this driver as modules, choose M here: the module for + the M1 family will be called ane, and the module for the M2 family + will be called ane_t6021. diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile new file mode 100644 index 0000000000000..c0ad4a43bc0c3 --- /dev/null +++ b/drivers/accel/ane/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_DRM_ACCEL_ANE) += ane.o ane_t6021.o + +ane-y := \ + ane_drv.o \ + ane_tm.o \ + ane_dart.o \ + ane_boost.o + +ane_t6021-y := \ + ane_t6021_rtclient_main.o \ + ane_t6021_fwload.o \ + ane_t6021_boot.o diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h new file mode 100644 index 0000000000000..10fee2864728c --- /dev/null +++ b/drivers/accel/ane/ane.h @@ -0,0 +1,160 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2022 Eileen Yoon */ + +#ifndef __ANE_H__ +#define __ANE_H__ + +#include +#include +#include +#include +#include + +#include +#include + +#include + +struct ane_device { + struct drm_device drm; + struct device *dev; + + struct device **pd_dev; + struct device_link **pd_link; + int pd_count; + + void __iomem *engine; + + /* The ane SET block (m1n1 ANE.ps_map): pmgr power-state words for + * this engine, mapped at probe on every SoC. Read-only, always: + * pmgr reads give recovery its ACTUAL evidence, while direct SET + * writes are firmware-locked and external-abort the SoC (T6001 + * netconsole-named 2026-09-16; T8103 same mechanism). + */ + phys_addr_t ps_base; + void __iomem *ps; + + struct drm_mm mm; + struct iommu_domain *domain; + unsigned long shift; + struct ane_dart { + void __iomem *regs; + int irq; + u32 sid; + bool masked; + } darts[3]; + int dart_count; + + int irq; + struct mutex iommu_lock; + struct mutex engine_lock; + struct list_head bo_list; + + /* + * Wedge-preserved BO ranges, tied to their still-inserted drm_mm + * nodes (see ane_reclaim_preserved). Registered under engine_lock. + */ + struct list_head preserved_list; + bool removed; + + /* + * Set under engine_lock once a task manager timeout leaves the queue + * state unknown and DMA possibly still active. A wedged engine refuses + * new work while recovery (ane_tm_recover) power-cycles it; the + * preserve-until-reboot behaviour applies only when that reset fails. + */ + atomic_t wedged; + + /* + * Set under engine_lock for the duration of a recovery power cycle, + * so runtime suspend may gate an engine that the reset itself is + * quiescing. + */ + bool recovering; + + /* + * TM_STATUS sampled right after the first tm_enable at probe: the + * engine's fresh, idle-state signature. Recovery accepts a reset + * whose post-cycle status matches it, since the register reset + * value is not documented. + */ + u32 tm_status_fresh; + bool tm_status_known; + + /* + * SoC descriptor fact: the set0/base islands hold the tm/tq + * register file in retention through any genpd power cycle + * (T6001 bisect evidence 2026-09-16: TQ_EN still reads 0x3000 + * after a 2 s gate), so recovery must drain retained task-manager + * state itself instead of expecting a power-on reset to clear it. + * False on T8103, where the cycle is a full POR of the file. + */ + bool tm_retention; + + /* + * Engine-busy CPU cluster boost (ane_boost.c): min-frequency QoS on + * every cpufreq policy while a submit runs and until boost_idle_ms + * after the last one completes: on T8103 bandwidth-bound programs + * run ~2x slower while schedutil parks the idle clusters low. + */ + struct ane_boost { + struct mutex lock; + struct delayed_work off; + struct freq_qos_request *legs; + int nlegs; + int held; + unsigned long last_end; + bool busy; + bool on; + } boost; +}; + +struct ane_request { + int qid; + u32 nid; + u32 td_size; + u32 td_count; + u32 btsp_iova; + u32 bar[ANE_TILE_COUNT]; +}; + +/* + * Drop the wedge pin (module refcount) without requiring a successful + * recovery. Used by ane_drm_postclose so a wedged engine cannot leave + * the module un-unloadable after a clean session close. Recovery stays + * safe to call afterwards; if it returns error, the postclose path has + * already unlocked the module count so the operator can swap the ko. + */ +void ane_wedge_clear(struct ane_device *ane); + +/* + * Reclaim wedge-preserved BO ranges (unmap, release nodes and pages). + * Only legal once DMA is provably quiescent: after a successful + * recovery power cycle, or at driver remove. Callers hold engine_lock. + */ +void ane_reclaim_preserved(struct ane_device *ane); + +int ane_boost_init(struct ane_device *ane); +void ane_boost_exit(struct ane_device *ane); +void ane_boost_begin(struct ane_device *ane); +void ane_boost_end(struct ane_device *ane); + +#define ANE_DART_MAX 3 +#define ANE_DART_SCRATCH_MAX 16 + +struct ane_dart_scratch { + u64 iova; + struct page *page; + struct drm_mm_node *node; +}; + +int ane_dart_init(struct ane_device *ane); +void ane_dart_mask(struct ane_device *ane); +void ane_dart_unmask(struct ane_device *ane); +bool ane_dart_faulted(struct ane_device *ane, u64 *iova, u32 *status); +int ane_dart_drain_fault(struct ane_device *ane, u64 fault_iova, + struct ane_dart_scratch *scratch, int *pages); +void ane_dart_release_scratch(struct ane_device *ane, + struct ane_dart_scratch *scratch, int pages); + +#endif /* __ANE_H__ */ diff --git a/drivers/accel/ane/ane_boost.c b/drivers/accel/ane/ane_boost.c new file mode 100644 index 0000000000000..a13d706f4bc3c --- /dev/null +++ b/drivers/accel/ane/ane_boost.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Engine-busy CPU cluster boost. + * + * Bandwidth-bound programs run at a speed set by the CPU cluster + * p-states. Measured on jwm1 (T8103, 2026-09-25, ane-linux-experiments + * receipt jwm1-ane-dvfs-boost): with a mostly-sleeping submitter at + * normal priority, 75 of 489 Qwen decode steps take 100-175 ms instead + * of 70, every program in a slow step ~2x its normal wall, the + * submitter's CPU time and run delay unchanged; both clusters pinned + * at their top p-state (performance governor, or SCHED_FIFO for the + * submitter, which schedutil boosts the same way) -> 0 of 489. The + * whole-encoder Parakeet program (140 ms) moves too once the QoS lapses + * mid-submit: timed from submit start, the hold dropped 100 ms into + * every encoder run, median 171 ms (140-238); held to completion, 138.0 + * (137.7-138.7), jwm1 2026-09-25. The likely + * mechanism is the second p-state field apple-soc-cpufreq writes with + * the cluster p-state on T8103 only (DVFS_CMD PS2, has_ps2); the + * memory-controller tuning its author named as unimplemented is still + * absent from the tree. Whatever the field drives, the cure is the + * cluster p-state, which cpufreq lets a driver request. + * + * So hold a min-frequency QoS at the top of every cpufreq policy while a + * submit runs, and drop it boost_idle_ms after the last one completes. + * The QoS goes through cpufreq's own constraint object, so it composes + * with any governor and never touches the DVFS registers. + */ +#include +#include +#include +#include +#include +#include + +#include "ane.h" + +static unsigned int boost_idle_ms = 100; +module_param(boost_idle_ms, uint, 0644); +MODULE_PARM_DESC(boost_idle_ms, + "hold every CPU cluster at its top p-state while a submit runs and this long after the last one completes (0 = off)"); + +/* Lock held. */ +static void ane_boost_set(struct ane_device *ane, bool on) +{ + struct ane_boost *b = &ane->boost; + int cpu, n = 0; + + if (on) { + for_each_possible_cpu(cpu) { + struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); + int err; + + if (!policy) + continue; + if (cpu != cpumask_first(policy->related_cpus) || + n >= b->nlegs) { + cpufreq_cpu_put(policy); + continue; + } + err = freq_qos_add_request(&policy->constraints, + &b->legs[n], FREQ_QOS_MIN, + policy->cpuinfo.max_freq); + cpufreq_cpu_put(policy); + if (err >= 0) + n++; + } + b->held = n; + } else { + for (n = 0; n < b->held; n++) + freq_qos_remove_request(&b->legs[n]); + b->held = 0; + } + b->on = on; +} + +static void ane_boost_off_work(struct work_struct *work) +{ + struct ane_device *ane = + container_of(work, struct ane_device, boost.off.work); + struct ane_boost *b = &ane->boost; + + mutex_lock(&b->lock); + /* A submit that began or ended after this expiry was armed keeps + * the boost on; its own completion re-arms the drop. + */ + if (b->on && !b->busy && + time_after_eq(jiffies, b->last_end + msecs_to_jiffies(boost_idle_ms))) + ane_boost_set(ane, false); + mutex_unlock(&b->lock); +} + +/* Engine lock held: submits never overlap, so busy is a flag. */ +void ane_boost_begin(struct ane_device *ane) +{ + struct ane_boost *b = &ane->boost; + + if (!b->legs || !boost_idle_ms) + return; + mutex_lock(&b->lock); + b->busy = true; + if (!b->on) + ane_boost_set(ane, true); + mutex_unlock(&b->lock); +} + +void ane_boost_end(struct ane_device *ane) +{ + struct ane_boost *b = &ane->boost; + + if (!b->legs) + return; + mutex_lock(&b->lock); + b->busy = false; + b->last_end = jiffies; + if (b->on) + mod_delayed_work(system_wq, &b->off, + msecs_to_jiffies(boost_idle_ms)); + mutex_unlock(&b->lock); +} + +int ane_boost_init(struct ane_device *ane) +{ + struct ane_boost *b = &ane->boost; + + mutex_init(&b->lock); + INIT_DELAYED_WORK(&b->off, ane_boost_off_work); + b->nlegs = num_possible_cpus(); + b->legs = kcalloc(b->nlegs, sizeof(*b->legs), GFP_KERNEL); + return b->legs ? 0 : -ENOMEM; +} + +void ane_boost_exit(struct ane_device *ane) +{ + struct ane_boost *b = &ane->boost; + + if (!b->legs) + return; + cancel_delayed_work_sync(&b->off); + mutex_lock(&b->lock); + if (b->on) + ane_boost_set(ane, false); + mutex_unlock(&b->lock); + kfree(b->legs); + b->legs = NULL; +} diff --git a/drivers/accel/ane/ane_dart.c b/drivers/accel/ane/ane_dart.c new file mode 100644 index 0000000000000..aea8cd9eff182 --- /dev/null +++ b/drivers/accel/ane/ane_dart.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Joshua Warren */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane.h" + +/* + * T8103 DART is the T8020 register block (apple_dart_hw_t8103 in + * drivers/iommu/apple-dart.c), pagesized 0x4000. Containment policy: + * during a job the apple-dart fault IRQ is masked so the latch survives + * for this driver to read, the completion poll checks that latch before + * every engine access (engine reads hang the SoC once a fault has + * landed), and a faulted IOVA is handed a throwaway page so the retried + * transaction completes and the engine drains. Mapping itself stays + * provider-owned; the page tables are never rewritten here. + */ +#define DART_STREAM_COMMAND 0x20 +#define DART_STREAM_COMMAND_BUSY BIT(2) +#define DART_STREAM_COMMAND_INVALIDATE BIT(20) +#define DART_STREAM_SELECT 0x34 +#define DART_ERROR 0x40 +#define DART_ERROR_FLAG BIT(31) +#define DART_ERROR_ADDR_LO 0x50 +#define DART_ERROR_ADDR_HI 0x54 +#define DART_STREAMS_ENABLE 0xfc +#define DART_TCR 0x100 +#define DART_TTBR 0x200 +#define DART_TTBR_COUNT 4 + +#define DART_TCR_SID(sid) (DART_TCR + ((sid) << 2)) +#define DART_TTBR_SID(sid, idx) \ + (DART_TTBR + ((((sid) * DART_TTBR_COUNT) + (idx)) << 2)) + +static bool dart_contain = true; +module_param(dart_contain, bool, 0444); +MODULE_PARM_DESC(dart_contain, + "Mask ANE DART fault IRQs during a job and restore the stream after a fault (default: on)"); + +static int ane_dart_invalidate(struct ane_device *ane, struct ane_dart *dart) +{ + void __iomem *regs = dart->regs; + u32 cmd; + int ret; + + if (dart->sid >= 32) + return -EINVAL; + writel(BIT(dart->sid), regs + DART_STREAM_SELECT); + writel(DART_STREAM_COMMAND_INVALIDATE, regs + DART_STREAM_COMMAND); + ret = readl_poll_timeout(regs + DART_STREAM_COMMAND, cmd, + !(cmd & DART_STREAM_COMMAND_BUSY), 1, 100); + if (ret) + dev_err(ane->dev, "DART sid %u TLB invalidate timed out\n", + dart->sid); + return ret; +} + +int ane_dart_init(struct ane_device *ane) +{ + struct device_node *np = ane->dev->of_node; + int count, i; + + ane->dart_count = 0; + if (!dart_contain) + return 0; + count = of_count_phandle_with_args(np, "iommus", "#iommu-cells"); + if (count < 0) + return 0; + for (i = 0; i < count && ane->dart_count < ANE_DART_MAX; i++) { + struct of_phandle_args args; + struct platform_device *pdev; + struct resource *res; + struct ane_dart *dart; + void __iomem *regs; + int irq; + + if (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", + i, &args)) + break; + if (!of_device_is_compatible(args.np, "apple,t8103-dart")) { + of_node_put(args.np); + continue; + } + pdev = of_find_device_by_node(args.np); + if (!pdev) { + of_node_put(args.np); + continue; + } + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + irq = platform_get_irq(pdev, 0); + if (!res || irq < 0) { + put_device(&pdev->dev); + of_node_put(args.np); + continue; + } + regs = devm_ioremap(ane->dev, res->start, resource_size(res)); + if (!regs) { + put_device(&pdev->dev); + of_node_put(args.np); + return -ENOMEM; + } + dart = &ane->darts[ane->dart_count++]; + dart->regs = regs; + dart->irq = irq; + dart->sid = args.args_count ? args.args[0] : 0; + dart->masked = false; + dev_info(ane->dev, "DART containment: %pOFn sid %u irq %d\n", + args.np, dart->sid, irq); + put_device(&pdev->dev); + of_node_put(args.np); + } + dev_info(ane->dev, "DART containment armed: %d\n", ane->dart_count); + return 0; +} + +void ane_dart_mask(struct ane_device *ane) +{ + int i; + + for (i = 0; i < ane->dart_count; i++) { + struct ane_dart *dart = &ane->darts[i]; + + if (dart->masked) + continue; + disable_irq(dart->irq); + dart->masked = true; + } +} + +void ane_dart_unmask(struct ane_device *ane) +{ + int i; + + for (i = 0; i < ane->dart_count; i++) { + struct ane_dart *dart = &ane->darts[i]; + + if (!dart->masked) + continue; + enable_irq(dart->irq); + dart->masked = false; + } +} + +bool ane_dart_faulted(struct ane_device *ane, u64 *iova, u32 *status) +{ + int i; + + for (i = 0; i < ane->dart_count; i++) { + void __iomem *regs = ane->darts[i].regs; + u32 err = readl(regs + DART_ERROR); + + if (!(err & DART_ERROR_FLAG)) + continue; + if (status) + *status = err; + if (iova) + *iova = readl(regs + DART_ERROR_ADDR_LO) | + ((u64)readl(regs + DART_ERROR_ADDR_HI) << 32); + return true; + } + return false; +} + +/* + * A fault is a missing PTE, nothing more: the page tables are provider + * owned and unchanged, so the stream-restore apple-dart uses at reset + * time is the wrong tool mid-job (register surgery while the faulting + * transaction is retried is what killed jwm1 twice). Instead the faulted + * IOVA is handed a throwaway DART page, so the retried transaction + * completes and the engine drains on its own; the scratch pages are + * unmapped once the request has been failed cleanly. + * + * The throwaway page is mapped at the faulting address, which is + * allocator-free by definition of a translation fault - but the drm_mm + * allocator knows nothing about a raw iommu_map. A scratch PTE that + * outlives its release path (any future bug on the wedge side) would + * then sit, untracked, in a range the allocator considers free: the + * next BO_INIT maps onto it and dies in dart_init_pte's "we require an + * unmap first" WARN (m1-test-host 2026-09-25, iommu_map failed at 0x4000). So + * the scratch range is RESERVED in the allocator first: a node-tracked + * mapping can never collide with a later BO_INIT, and a scratch whose + * release is skipped degrades into a bounded, benign leak instead of a + * poisoned address space. Reserve failure means a live BO node already + * owns the faulting page (a wild descriptor, not a hole): refuse and + * let the caller wedge, fail closed. + */ +int ane_dart_drain_fault(struct ane_device *ane, u64 fault_iova, + struct ane_dart_scratch *scratch, int *pages) +{ + struct drm_mm_node *node; + struct page *page; + u64 iova; + int ret, i; + + page = alloc_page(GFP_KERNEL); + if (!page) + return -ENOMEM; + iova = fault_iova & ~((1ULL << ane->shift) - 1); + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) { + __free_page(page); + return -ENOMEM; + } + node->start = iova; + node->size = BIT(ane->shift); + + mutex_lock(&ane->iommu_lock); + ret = drm_mm_reserve_node(&ane->mm, node); + if (ret) { + mutex_unlock(&ane->iommu_lock); + kfree(node); + __free_page(page); + return ret; + } + + ret = iommu_map(ane->domain, iova, page_to_phys(page), + BIT(ane->shift), IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); + mutex_unlock(&ane->iommu_lock); + if (ret) { + mutex_lock(&ane->iommu_lock); + drm_mm_remove_node(node); + mutex_unlock(&ane->iommu_lock); + kfree(node); + __free_page(page); + return ret; + } + scratch[*pages].iova = iova; + scratch[*pages].page = page; + scratch[*pages].node = node; + (*pages)++; + for (i = 0; i < ane->dart_count; i++) + ane_dart_invalidate(ane, &ane->darts[i]); + return 0; +} + +void ane_dart_release_scratch(struct ane_device *ane, + struct ane_dart_scratch *scratch, int pages) +{ + int i, d; + + mutex_lock(&ane->iommu_lock); + for (i = 0; i < pages; i++) { + size_t unmapped; + + unmapped = iommu_unmap(ane->domain, scratch[i].iova, + BIT(ane->shift)); + if (unmapped != BIT(ane->shift)) + dev_err(ane->dev, + "scratch unmap short at %#llx: %zu\n", + scratch[i].iova, unmapped); + drm_mm_remove_node(scratch[i].node); + kfree(scratch[i].node); + scratch[i].node = NULL; + __free_page(scratch[i].page); + scratch[i].page = NULL; + } + mutex_unlock(&ane->iommu_lock); + if (pages) + for (d = 0; d < ane->dart_count; d++) + ane_dart_invalidate(ane, &ane->darts[d]); +} diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c new file mode 100644 index 0000000000000..db03774c9e25a --- /dev/null +++ b/drivers/accel/ane/ane_drv.c @@ -0,0 +1,1234 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2022 Eileen Yoon */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "ane.h" +#include "ane_tm.h" + +/* + * BO mapping mode, runtime-switchable per device (/sys/module/ane/parameters/ + * map_mode) and read at BO-mmap/iommu-map time: + * bit 0 — mark DART descriptors IOMMU_CACHE (cacheable, inner-shareable) + * bit 1 — map the CPU vma cacheable instead of write-combine + * 0 = today's WC CPU vmas + NC descriptors; 3 = both cacheable (device + * accesses cohere with the CPU caches). Each axis is toggleable alone for + * attribution. + * + * Coherency evidence for mode 3, both SoCs: + * - T6001/t6001-test-host (apple,t6000-dart x3, APPLE_DART2 PTEs): (1) the ANE sits in + * the dma-coherent DART class every SoC DMA-API device uses on + * arm64/Asahi, where dma-iommu itself programs descriptors with + * IOMMU_CACHE; (2) eiln's pre-GEM driver (4566c89^) mapped BOs cacheable + * on the CPU while DMAing them through this same iommu_map() path for + * months with no maintenance; (3) empirically, an alternating-pattern + * 16-rep buffer-reuse battery over 6 programs in both directions is + * byte-identical WC+NC vs cached on every rep — stale cache lines would + * surface as the previous pattern's bytes. + * - T8103/m1-test-host (apple,t8103-dart x3, APPLE_DART1 PTEs): the DART1 PTE + * format has no cacheability field (io-pgtable-dart.c dart_prot_to_pte), + * so bit 0 is a structural no-op there and the lever is the cached CPU + * vma alone. eiln's pre-GEM precedent ran exactly that on M1 for months. + * The same reuse battery is byte-identical (14/14) and the certified + * Parakeet E2E pins hold bit-for-bit with ane_exec 2561.2 -> 2402.0 ms. + * + * Default 3 (cached both sides). 0 is the documented rollback: runtime via + * /sys/module/ane/parameters/map_mode, or by loading the retained prior + * module (ane-96d5a88.ko on both hosts). + */ +static int map_mode = 3; +module_param(map_mode, int, 0644); +MODULE_PARM_DESC(map_mode, + "BO mapping: bit0=IOMMU_CACHE DART descriptors, bit1=cacheable CPU vmas (default 3 = cached; 0 = writecombine + non-cacheable rollback)"); + +#define CMD_BUF_BDX 0 +#define KRN_BUF_BDX 1 + +struct ane_bo { + struct drm_gem_object base; + struct list_head node; + struct drm_mm_node *mm; + u32 npages; + struct page **pages; + dma_addr_t iova; +}; + +#define to_bo(gem) (container_of(gem, struct ane_bo, base)) + +static struct ane_bo *bo_lookup(struct drm_file *file, u32 handle) +{ + struct drm_gem_object *gem = drm_gem_object_lookup(file, handle); + + if (!gem) + return NULL; + return to_bo(gem); +} + +/* + * Buffers are mapped and unmapped only through the kernel-owned IOMMU + * domain the device was attached to by its providers. apple-dart + * programs every DART in the device's "iommus" list (TTBRs, stream + * setup, invalidation and fault IRQs are all provider-owned); + * iommu_unmap() flushes through the provider before returning, so pages + * may be reclaimed once it succeeds. + */ +static int ane_iommu_map_pages(struct ane_device *ane, struct ane_bo *bo) +{ + int err; + + lockdep_assert_held(&ane->engine_lock); + if (bo->mm) + return -EBUSY; + + bo->mm = kzalloc(sizeof(*bo->mm), GFP_KERNEL); + if (!bo->mm) + return -ENOMEM; + + mutex_lock(&ane->iommu_lock); + + /* reserve area from ANE address space */ + err = drm_mm_insert_node_generic(&ane->mm, bo->mm, + bo->npages << ane->shift, + 1UL << ane->shift, 0, 0); + if (err < 0) { + dev_err(ane->dev, "out of ANE space: %d\n", err); + goto unlock; + } + + bo->iova = bo->mm->start; + + /* map into ANE address space */ + for (u32 i = 0; i < bo->npages; i++) { + dma_addr_t iova = bo->iova + (i << ane->shift); + int prot = IOMMU_READ | IOMMU_WRITE; + bool healed = false; + int prot_cache = map_mode & 1; + +retry: + if (prot_cache) + prot |= IOMMU_CACHE; + err = iommu_map(ane->domain, iova, page_to_phys(bo->pages[i]), + 1UL << ane->shift, prot, GFP_KERNEL); + if (err < 0) { + dev_err(ane->dev, "iommu_map failed at 0x%llx", iova); + /* Every PTE inside a reserved drm_mm node must be + * ours-or-absent: scratch drains reserve before + * mapping and wedge-preserved nodes stay inserted. + * A VALID PTE in a just-reserved range is therefore + * a stray from a torn-down session or a lost + * unmap. Clear it once and retry instead of + * poisoning every later BO_INIT until reload + * (m1-test-host 2026-09-25: dart_init_pte -EEXIST at + * 0x4000 for the rest of the session). + */ + if (err == -EEXIST && !healed && + iommu_iova_to_phys(ane->domain, iova)) { + dev_warn(ane->dev, + "clearing stray DART PTE at 0x%llx (no node owns it)\n", + iova); + iommu_unmap(ane->domain, iova, + 1UL << ane->shift); + healed = true; + goto retry; + } + while (i-- > 0) { + size_t unmapped; + + unmapped = iommu_unmap(ane->domain, + bo->iova + + (i << ane->shift), + 1UL << ane->shift); + if (unmapped != 1UL << ane->shift) + dev_err(ane->dev, + "rollback unmap short at %#llx: %zu\n", + bo->iova + (i << ane->shift), + unmapped); + } + drm_mm_remove_node(bo->mm); + bo->iova = 0; + break; + } + } + + mutex_unlock(&ane->iommu_lock); + + if (err < 0) { + kfree(bo->mm); + bo->mm = NULL; + return err; + } + + return 0; + +unlock: + mutex_unlock(&ane->iommu_lock); + kfree(bo->mm); + bo->mm = NULL; + return err; +} + +static void ane_iommu_unmap_pages(struct ane_device *ane, struct ane_bo *bo) +{ + struct drm_mm_node *mm = bo->mm; + + lockdep_assert_held(&ane->engine_lock); + + if (!mm) + return; + + list_del_init(&bo->node); + mutex_lock(&ane->iommu_lock); + for (u32 i = 0; i < bo->npages; i++) { + dma_addr_t iova = bo->iova + (i << ane->shift); + size_t unmapped = iommu_unmap(ane->domain, iova, + 1UL << ane->shift); + + if (unmapped != 1UL << ane->shift) + dev_err(ane->dev, "unmap short at %#llx: %zu\n", + iova, unmapped); + } + drm_mm_remove_node(mm); + bo->mm = NULL; + bo->iova = 0; + mutex_unlock(&ane->iommu_lock); + + kfree(mm); +} + +/* + * A BO freed while the engine is wedged keeps its mapping and its pages + * (DMA may still be reading the IOVA), but the GEM object is gone. The + * range's drm_mm node and the page array outlive it here, still linked + * into the allocator, until DMA is provably quiescent: recovery + * (ane_reclaim_preserved after the quiescing power cycle) or remove. + * The node stays reserved the whole time, so the IOVA can never be + * handed to a later BO_INIT while its PTEs are live. + */ +struct ane_preserved { + struct list_head list; + struct drm_mm_node *mm; + struct page **pages; + u32 npages; +}; + +static void ane_preserve_bo(struct ane_device *ane, struct ane_bo *bo) +{ + struct ane_preserved *p; + + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (!p) { + /* Fail closed the old way: the node stays inserted and the + * pages leak until reboot - safe, just unreclaimable. + */ + dev_err(ane->dev, "wedged: preserve registry full, leaking\n"); + bo->mm = NULL; + bo->pages = NULL; + return; + } + p->mm = bo->mm; + p->pages = bo->pages; + p->npages = bo->npages; + list_add_tail(&p->list, &ane->preserved_list); + bo->mm = NULL; + bo->pages = NULL; +} + +void ane_reclaim_preserved(struct ane_device *ane) +{ + struct ane_preserved *p, *tmp; + unsigned long count = 0; + + lockdep_assert_held(&ane->engine_lock); + + list_for_each_entry_safe(p, tmp, &ane->preserved_list, list) { + u64 iova = p->mm->start; + + mutex_lock(&ane->iommu_lock); + for (u32 i = 0; i < p->npages; i++) { + size_t unmapped; + + unmapped = iommu_unmap(ane->domain, + iova + (i << ane->shift), + 1UL << ane->shift); + if (unmapped != 1UL << ane->shift) + dev_err(ane->dev, + "preserved unmap short at %#llx: %zu\n", + iova + (i << ane->shift), unmapped); + } + drm_mm_remove_node(p->mm); + mutex_unlock(&ane->iommu_lock); + + kfree(p->mm); + for (u32 i = 0; i < p->npages; i++) + put_page(p->pages[i]); + kfree(p->pages); + list_del(&p->list); + kfree(p); + count++; + } + if (count) + dev_info(ane->dev, + "reclaimed %lu wedge-preserved mapping(s) after quiesce\n", + count); +} + +static vm_fault_t ane_gem_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct drm_gem_object *gem = vma->vm_private_data; + struct ane_bo *bo = to_bo(gem); + struct page *page; + pgoff_t offset; + + if (!bo->pages) + return VM_FAULT_SIGBUS; + + offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + if (offset >= bo->npages) + return VM_FAULT_SIGBUS; + page = bo->pages[offset]; + + return vmf_insert_page(vma, vmf->address, page); +} + +static const struct vm_operations_struct drm_gem_ane_vm_ops = { + .open = drm_gem_vm_open, + .close = drm_gem_vm_close, + .fault = ane_gem_vm_fault, +}; + +/* + * Sole destruction path for buffer objects: every reference (handles, + * mmap vmas, lookups) funnels here. Nothing is reclaimed while the + * engine is wedged, since DMA may still be reading the IOVA. + */ +static void ane_gem_free_object(struct drm_gem_object *gem) +{ + struct ane_device *ane = gem->dev->dev_private; + struct ane_bo *bo = to_bo(gem); + + mutex_lock(&ane->engine_lock); + if (bo->mm) { + if (atomic_read(&ane->wedged)) { + /* Fail closed: keep the mapping and the pages so no + * IOVA is torn down or page reused underneath active + * DMA. The range's node stays reserved and tracked + * until recovery quiesces the engine and reclaims + * it, so no later BO_INIT can map over these PTEs + * (a leaked-but-forgotten mapping is what turned + * into dart_init_pte -EEXIST WARNs on m1-test-host). + */ + dev_err(ane->dev, "wedged: preserving bo mapping\n"); + list_del_init(&bo->node); + ane_preserve_bo(ane, bo); + } else { + ane_iommu_unmap_pages(ane, bo); + } + } + mutex_unlock(&ane->engine_lock); + + if (bo->pages) { + drm_gem_put_pages(gem, bo->pages, true, true); + bo->pages = NULL; + } + drm_gem_object_release(gem); + kfree(bo); +} + +static const struct drm_gem_object_funcs ane_gem_object_funcs = { + .free = ane_gem_free_object, + .vm_ops = &drm_gem_ane_vm_ops, +}; + +static int ane_bo_init(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct ane_device *ane = drm->dev_private; + struct drm_ane_bo_init *args = data; + struct drm_gem_object *gem; + struct ane_bo *bo; + int err; + + if (args->pad || !args->size) + return -EINVAL; + + bo = kzalloc(sizeof(struct ane_bo), GFP_KERNEL); + if (!bo) + return -ENOMEM; + INIT_LIST_HEAD(&bo->node); + + gem = &bo->base; + gem->funcs = &ane_gem_object_funcs; + err = drm_gem_object_init(drm, gem, round_up(args->size, PAGE_SIZE)); + if (err < 0) { + drm_gem_private_object_fini(gem); + goto free; + } + + err = drm_gem_create_mmap_offset(gem); + if (err < 0) + goto put; + + args->offset = drm_vma_node_offset_addr(&gem->vma_node); + + bo->npages = gem->size >> PAGE_SHIFT; + bo->pages = drm_gem_get_pages(gem); + if (IS_ERR(bo->pages)) { + err = PTR_ERR(bo->pages); + bo->pages = NULL; + goto put; + } + + mutex_lock(&ane->engine_lock); + if (ane->removed) { + err = -ENODEV; + goto unlock_put; + } + if (atomic_read(&ane->wedged)) { + dev_err_ratelimited(ane->dev, "wedged: refusing bo init\n"); + err = -ENODEV; + goto unlock_put; + } + + err = ane_iommu_map_pages(ane, bo); + if (err < 0) + goto unlock_put; + list_add_tail(&bo->node, &ane->bo_list); + + err = drm_gem_handle_create(file, gem, &args->handle); + mutex_unlock(&ane->engine_lock); + drm_gem_object_put(gem); /* handle (or error path) owns it now */ + return err; + +unlock_put: + mutex_unlock(&ane->engine_lock); +put: + drm_gem_object_put(gem); /* routes through ane_gem_free_object() */ + return err; +free: + kfree(bo); + return err; +} + +static int ane_bo_free(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct ane_device *ane = drm->dev_private; + struct drm_ane_bo_free *args = data; + struct ane_bo *bo; + + if (args->pad) + return -EINVAL; + + mutex_lock(&ane->engine_lock); + if (ane->removed) { + mutex_unlock(&ane->engine_lock); + return -ENODEV; + } + if (atomic_read(&ane->wedged)) { + dev_err_ratelimited(ane->dev, "wedged: refusing bo free\n"); + mutex_unlock(&ane->engine_lock); + return -ENODEV; + } + + bo = bo_lookup(file, args->handle); + mutex_unlock(&ane->engine_lock); + if (!bo) + return -EINVAL; + + /* Keep reference drops outside engine_lock: the final put enters + * ane_gem_free_object(), which owns that lock while unmapping. + */ + drm_gem_handle_delete(file, args->handle); + drm_gem_object_put(&bo->base); + return 0; +} + +static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) +{ + struct ane_device *ane = drm->dev_private; + struct drm_ane_submit *args = data; + struct drm_gem_object *gem[ANE_TILE_COUNT] = { 0 }; + struct drm_gem_object *btsp = NULL; + struct ane_bo *bo; + struct ane_request req; + int err; + + memset(&req, 0, sizeof(req)); + + if (args->pad || !args->tsk_size || !args->td_count || + args->td_count > 0xffff || args->td_size < 4 || + args->td_size > 0x40000 || (args->td_size & 3) || + !args->handles[CMD_BUF_BDX] || args->handles[KRN_BUF_BDX] || + !args->btsp_handle) { + return -EINVAL; + } + + /* Fail closed: never queue work behind a wedged engine. */ + if (atomic_read(&ane->wedged)) { + dev_err_ratelimited(ane->dev, "wedged: refusing submit\n"); + return -ECANCELED; + } + + req.qid = 4; + req.nid = ANE_FIFO_NID; + req.td_size = args->td_size; + req.td_count = args->td_count; + + for (int bdx = 0; bdx < ANE_TILE_COUNT; bdx++) { + if (!args->handles[bdx]) + continue; + bo = bo_lookup(file, args->handles[bdx]); + if (!bo) { + err = -EINVAL; + goto put; + } + gem[bdx] = &bo->base; + if (!bo->iova || + ((bdx == CMD_BUF_BDX) && + (args->tsk_size >= (bo->npages << ane->shift)))) { + err = -EINVAL; + goto put; + } + req.bar[bdx] = lower_32_bits(bo->iova); + } + + /* + * The microcode and weights are packed @ 16 gran for bank aligned + * access. Since this isn't page aligned, we represent the two as one + * buffer and calculate the delimiter (where the weights would start). + */ + req.bar[KRN_BUF_BDX] = + req.bar[CMD_BUF_BDX] + round_up(args->tsk_size, ANE_CMD_GRAN); + + bo = bo_lookup(file, args->btsp_handle); + if (!bo) { + err = -EINVAL; + goto put; + } + btsp = &bo->base; + if (!bo->iova || args->td_size > bo->base.size) { + err = -EINVAL; + goto put; + } + req.btsp_iova = lower_32_bits(bo->iova); + + /* + * Lookup references are held across the synchronous execute and + * dropped only after ane_tm_execute() confirmed completion, so a + * concurrent BO_FREE cannot unmap a buffer the engine reads. + */ + mutex_lock(&ane->engine_lock); + if (ane->removed) { + err = -ENODEV; + goto unlock; + } + if (atomic_read(&ane->wedged)) { + dev_err_ratelimited(ane->dev, "wedged: refusing submit\n"); + err = -ECANCELED; + goto unlock; + } + + err = ane_tm_enqueue(ane, &req); + if (err < 0) + goto unlock; + + ane_boost_begin(ane); + err = ane_tm_execute(ane, &req); + ane_boost_end(ane); + +unlock: + mutex_unlock(&ane->engine_lock); +put: + drm_gem_object_put(btsp); + for (int bdx = 0; bdx < ANE_TILE_COUNT; bdx++) { + if (gem[bdx]) + drm_gem_object_put(gem[bdx]); + } + return err; +} + +static const struct drm_ioctl_desc ane_drm_ioctls[] = { + DRM_IOCTL_DEF_DRV(ANE_BO_INIT, ane_bo_init, 0), + DRM_IOCTL_DEF_DRV(ANE_BO_FREE, ane_bo_free, 0), + DRM_IOCTL_DEF_DRV(ANE_SUBMIT, ane_submit, 0), +}; + +static ssize_t wedged_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct ane_device *ane = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", atomic_read(&ane->wedged)); +} +static DEVICE_ATTR_RO(wedged); + +/* Operator retry for a wedge whose automatic recovery failed: one more + * bounded power-cycle attempt. No-op when not wedged. + */ +static ssize_t reset_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ane_device *ane = dev_get_drvdata(dev); + int err = 0; + + mutex_lock(&ane->engine_lock); + if (!ane->removed) + err = ane_tm_recover(ane); + mutex_unlock(&ane->engine_lock); + if (err) + return err; + return count; +} +static DEVICE_ATTR_WO(reset); + +static struct attribute *ane_dev_attrs[] = { + &dev_attr_wedged.attr, + &dev_attr_reset.attr, + NULL, +}; +ATTRIBUTE_GROUPS(ane_dev); + +static int ane_drm_open(struct drm_device *drm, struct drm_file *file) +{ + struct ane_device *ane = drm->dev_private; + int err; + + /* Bring up power while the file context is created. A failed + * resume propagates; nothing is released that was not acquired. + */ + err = pm_runtime_resume_and_get(ane->dev); + if (err < 0) + return err; + + pm_runtime_put(ane->dev); + return 0; +} + +/* + * Per-session teardown: every file context open is a fresh boundary. + * Anything the previous session left on the device (wedged flag, tm + * completion counters, in-flight recovery state) belongs to that + * session and must not survive into the next one. + * + * drm_gem_release runs before us and has dropped every handle this + * file held. BOs leaked by `ane_gem_free_object under wedge` stay on + * ane->bo_list until module teardown (their backing `drm_gem_object`s + * are already gone, and touching them here would be use-after-free). + * We do not try to drain them; we only clear the session-leaving + * state (wedged + tm counters + recovering flag) so the next opener + * sees a clean device. The orphan BOs remain inert: they are not in + * any drm_file's handle table. + */ +static void ane_drm_postclose(struct drm_device *drm, struct drm_file *file) +{ + struct ane_device *ane = drm->dev_private; + + mutex_lock(&ane->engine_lock); + if (ane->removed) { + mutex_unlock(&ane->engine_lock); + return; + } + + /* Drop the wedge pin first: a wedged engine that cannot be + * recovered in software must NOT keep the module pinned, or the + * operator cannot unload the ko for an updated build without a + * reboot. ane_wedge_clear releases both the wedged flag and the + * module_refcount; ane_tm_recover then attempts a power-cycle + * recovery (succeeds only if the engine is actually idle-able). + */ + ane_wedge_clear(ane); + ane_tm_recover(ane); + + mutex_unlock(&ane->engine_lock); +} + +/* + * The driver only owns three ioctls (ANE_BO_INIT, ANE_BO_FREE, + * ANE_SUBMIT). Anything outside that table would otherwise fall through + * the standard drm_ioctl path to legacy primary-node handlers that + * dereference drm_device.unique, which an accel node never sets. Reject + * such callers explicitly so they fail with -ENOTTY instead of oopsing. + * + * The accel pattern uses ane_drm_ioctls[] via drm_ioctl() (legacy pattern + * from before DRM_ACCEL_FOPS landed drm_ioctl defaults), so this is a + * belt-and-suspenders gate on top of the existing ioctls table. + */ +static long ane_drm_unlocked_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct drm_file *filp = file->private_data; + struct drm_device *drm = filp->minor->dev; + struct ane_device *ane = drm->dev_private; + long err; + + /* Reject anything outside the drm type ('d') namespace outright, and + * core ioctls that would read a DRM master the accel node does not + * have (GET_UNIQUE, GET_MAP, ... dereference NULL); clients need + * only VERSION from the core set. + * + * _IOC_TYPEMASK is the low 8 bits, which encode the *nr*, not the + * *type*. The type byte lives at offset _IOC_TYPESHIFT (8). Use + * _IOC_TYPE(cmd) to extract the right field; misuse here returns + * -ENOTTY for every legitimate ane_ioctl, which is exactly the + * regression committed and reverted in the lifecycle-reset series. + */ + if (_IOC_TYPE(cmd) != DRM_IOCTL_BASE || + (_IOC_NR(cmd) < DRM_COMMAND_BASE && + _IOC_NR(cmd) != _IOC_NR(DRM_IOCTL_VERSION)) || + _IOC_NR(cmd) >= DRM_COMMAND_END) + return -ENOTTY; + + err = pm_runtime_resume_and_get(ane->dev); + if (err < 0) + return err; + + err = drm_ioctl(file, cmd, arg); + + pm_runtime_put(ane->dev); + + return err; +} + +static int ane_drm_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct drm_gem_object *gem; + struct ane_bo *bo; + int err; + + err = drm_gem_mmap(file, vma); + if (err < 0) + return err; + + /* + * Set vm_pgoff (used as a fake buffer offset by DRM) to 0 and map the + * whole buffer from the start. + */ + vma->vm_pgoff = 0; + gem = vma->vm_private_data; + bo = to_bo(gem); + + if (vma_pages(vma) == 0) + return -ENXIO; + + /* + * Cacheable CPU mapping when bit 1 of map_mode is set, paired with + * IOMMU_CACHE DART descriptors: device accesses cohere with the CPU + * caches and read-back stops paying uncached-mode DRAM latency. + * Default (bit clear) keeps the write-combine mapping. VM_IO stays + * set either way (no struct-page identity is promised). + */ + vm_flags_mod(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP, VM_PFNMAP); + + if (map_mode & 2) + vma->vm_page_prot = + pgprot_decrypted(vm_get_page_prot(vma->vm_flags)); + else { + vma->vm_page_prot = + pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); + } + + return vm_map_pages(vma, bo->pages, bo->npages); +} + +static const struct file_operations ane_drm_fops = { + .owner = THIS_MODULE, + .fop_flags = FOP_UNSIGNED_OFFSET, + .open = accel_open, + .release = drm_release, + .unlocked_ioctl = ane_drm_unlocked_ioctl, + .compat_ioctl = drm_compat_ioctl, + .poll = drm_poll, + .read = drm_read, + .llseek = noop_llseek, + .mmap = ane_drm_mmap, +}; + +static const struct drm_driver ane_drm_driver = { + .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL, + .open = ane_drm_open, + .postclose = ane_drm_postclose, + .ioctls = ane_drm_ioctls, + .num_ioctls = ARRAY_SIZE(ane_drm_ioctls), + .fops = &ane_drm_fops, + .major = ANE_ABI_MAJOR, + .name = "ane", + .desc = "Apple Neural Engine driver", +}; + +static int ane_iommu_domain_init(struct ane_device *ane) +{ + u64 min_iova, limit; + + struct iommu_domain *domain = iommu_get_domain_for_dev(ane->dev); + + if (!domain) + return -EPROBE_DEFER; + + if (!(domain->pgsize_bitmap & PAGE_SIZE) || + !domain->geometry.force_aperture) + return -EINVAL; + min_iova = max_t(u64, domain->geometry.aperture_start, PAGE_SIZE); + limit = round_down(min_t(u64, domain->geometry.aperture_end, U32_MAX) + 1, + PAGE_SIZE); + if (min_iova >= limit) + return -EINVAL; + min_iova = round_up(min_iova, PAGE_SIZE); + if (min_iova >= limit) + return -EINVAL; + ane->domain = domain; + ane->shift = PAGE_SHIFT; + drm_mm_init(&ane->mm, min_iova, limit - min_iova); + + return 0; +} + +/* + * The DART page table belongs to the IOMMU domain, which outlives this + * module: a mapping a previous instance never unmapped (a wedge preserves + * them) survives a reload, and the first BO_INIT onto it fails with + * -EEXIST. Clear every mapping inside the still-empty allocator range. + * The unmap flushes the DART TLBs, so this runs only after runtime + * resume has powered the engine and its DARTs, and before registration + * lets a client map anything. + */ +static void ane_iommu_purge_stale(struct ane_device *ane) +{ + struct drm_mm_node *hole; + u64 start, end, iova; + unsigned long stale = 0; + + drm_mm_for_each_hole(hole, &ane->mm, start, end) { + for (iova = start; iova < end; iova += 1UL << ane->shift) { + if (!iommu_iova_to_phys(ane->domain, iova)) + continue; + iommu_unmap(ane->domain, iova, 1UL << ane->shift); + stale++; + } + } + if (stale) + dev_warn(ane->dev, + "cleared %lu stale DART mappings from a previous instance\n", + stale); +} + +static void ane_detach_genpd(struct ane_device *ane) +{ + if (ane->pd_count <= 1) + return; + + for (int i = ane->pd_count - 1; i >= 0; i--) { + if (ane->pd_link[i]) + device_link_del(ane->pd_link[i]); + if (!IS_ERR_OR_NULL(ane->pd_dev[i])) + dev_pm_domain_detach(ane->pd_dev[i], true); + } +} + +static int ane_attach_genpd(struct ane_device *ane) +{ + struct device *dev = ane->dev; + + ane->pd_count = of_count_phandle_with_args( + dev->of_node, "power-domains", "#power-domain-cells"); + if (ane->pd_count < 1) + return ane->pd_count < 0 ? ane->pd_count : -EINVAL; + if (ane->pd_count == 1) + return dev->pm_domain ? 0 : -EPROBE_DEFER; + + ane->pd_dev = devm_kcalloc(dev, ane->pd_count, sizeof(*ane->pd_dev), + GFP_KERNEL); + if (!ane->pd_dev) + return -ENOMEM; + + ane->pd_link = devm_kcalloc(dev, ane->pd_count, sizeof(*ane->pd_link), + GFP_KERNEL); + if (!ane->pd_link) + return -ENOMEM; + + for (int i = 0; i < ane->pd_count; i++) { + ane->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); + if (IS_ERR(ane->pd_dev[i])) { + ane_detach_genpd(ane); + return PTR_ERR(ane->pd_dev[i]); + } + + ane->pd_link[i] = + device_link_add(dev, ane->pd_dev[i], + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!ane->pd_link[i]) { + ane_detach_genpd(ane); + return -EINVAL; + } + } + + return 0; +} + +/* + * Per-SoC support descriptor, one per Linux `compatible`. + * + * Everything the probe needs except the SET block is derived from the + * device tree by design: DART topology from "iommus", power wiring from + * "power-domains", the engine window from "reg", the IRQ by name. Board + * facts belong in the overlay, not in compiled constants. The SET block + * is the one exception: the live overlays carry no range for it, so it + * is compiled in per SoC — and only from a source that has proven the + * address on hardware (m1n1 proxyclient ANE.ps_map, then a bound device). + * A guessed SET base is not a bug but a brick: direct writes to the + * block external-abort the SoC (T6001 named by netconsole 2026-09-16, + * PS_SET0 down at 0x28e08c000; T8103 same mechanism at 0x23b70c000), + * which is why a SoC without proven constants must refuse to bind + * instead of carrying a guess. + * + * Qualification tiers: + * ANE_QUALIFIED — execution proven on this silicon; binds normally. + * ANE_RECOGNIZED — constants present, never run on hardware; binds + * only with ane.allow_unqualified=1, loudly. + * ANE_UNSUPPORTED — no proven constants; refuses and names the data + * needed to advance the port. + */ +enum ane_qual { + ANE_QUALIFIED, + ANE_RECOGNIZED, + ANE_UNSUPPORTED, +}; + +struct ane_soc { + phys_addr_t ps_base; + enum ane_qual qual; + /* True when the tm/tq register file survives a genpd cycle in + * retention and recovery must drain it (see ane_tm_drain_retained). + */ + bool tm_retention; +}; + +static bool allow_unqualified; +module_param(allow_unqualified, bool, 0444); +MODULE_PARM_DESC(allow_unqualified, + "Bind recognized-but-unproven ANE SoCs (unverified SET base: external-abort risk)"); + +static const struct ane_soc ane_soc_t8103 = { + /* M1 (m1-test-host). SET block mapped read-only for the recovery ACTUAL + * log and the powered-on guard. Execution proven in the fleet. + */ + .ps_base = 0x23b70c000ULL, + .qual = ANE_QUALIFIED, +}; + +static const struct ane_soc ane_soc_t6000 = { + /* M1 Pro, M1 Max and M1 Ultra die 0 share this compatible and SET base + * (proven on T6001/t6001-test-host); M1 Pro and M1 Ultra are untested. + */ + .ps_base = 0x28e08c000ULL, + .qual = ANE_QUALIFIED, + .tm_retention = true, +}; + +static const struct ane_soc ane_soc_t6020 = { + /* M2 Pro (t6020). Community device-tree captures exist, but the + * SET base is unproven and the H14 compiler backend is unqualified: + * no constants may enter here yet. + */ + .ps_base = 0, + .qual = ANE_UNSUPPORTED, +}; + +static const struct ane_soc ane_soc_t6021 = { + /* M2 Max (T6021, t6021-test-host). SET window 0x8e08c000 = pmgr base + * 0x8e080000 + 0xc000, present in ane0's own reg on macOS captures + * of both 26.6.2 and 27.0 (2026-09-18, 11/11 derivation-input + * checks each, zero delta); the Linux translation reuses the pmgr + * high bits proven on T6000/T6020 rows (low-32 match) -> + * 0x28e08c000. Recognized, not qualified: constants complete, but + * no Linux execution yet, and the t6021 pwrstate word layout + * (which of set0/base/set1..4 sits at which offset — macOS-side + * capture cannot see it) is unverified; the first probe resume + * logs the ACTUAL nibbles through this window as the Linux-side + * probe (receipt 2026-09-18-t6021-driver-entry-prepared.md). + * Promotion to ANE_QUALIFIED needs an exact run on this silicon. + */ + .ps_base = 0x28e08c000ULL, + .qual = ANE_RECOGNIZED, +}; + +static const struct of_device_id ane_of_match[] = { + { .compatible = "apple,t8103-ane", .data = &ane_soc_t8103 }, + { .compatible = "apple,t6000-ane", .data = &ane_soc_t6000 }, + { .compatible = "apple,t6020-ane", .data = &ane_soc_t6020 }, + { .compatible = "apple,t6021-ane", .data = &ane_soc_t6021 }, + {} +}; + +MODULE_DEVICE_TABLE(of, ane_of_match); + +static int ane_platform_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct of_device_id *id = of_match_device(ane_of_match, dev); + const struct ane_soc *soc = id->data; + struct ane_device *ane; + struct drm_device *drm; + int err; + + /* Tier gate before any power-domain, MMIO or IRQ interaction: an + * unqualified SoC must fail cleanly, never half-probe. + */ + if (soc->qual == ANE_UNSUPPORTED) { + dev_err(dev, + "%s: unsupported ANE: no proven SET-block base (a guessed base external-aborts the SoC); not binding. To advance this port, run the mlx-omarchy quick collector (scripts/collect_quick.py: captures ANE/DART/PMGR/AIC device-tree data, no driver needed) and submit the capture\n", + id->compatible); + return -ENODEV; + } + if (soc->qual == ANE_RECOGNIZED && !allow_unqualified) { + dev_err(dev, + "%s: recognized but unqualified: constants present, execution never proven on this silicon; not binding. Override with ane.allow_unqualified=1, or prove a run and report it\n", + id->compatible); + return -ENODEV; + } + if (soc->qual == ANE_RECOGNIZED) + dev_warn(dev, + "%s: UNQUALIFIED bind forced by allow_unqualified: no execution proven on this silicon — SET-block base and board topology unverified\n", + id->compatible); + + ane = devm_drm_dev_alloc(dev, &ane_drm_driver, struct ane_device, drm); + if (IS_ERR(ane)) + return PTR_ERR(ane); + + platform_set_drvdata(pdev, ane); + ane->dev = dev; + + drm = &ane->drm; + drm->dev_private = ane; + + atomic_set(&ane->wedged, 0); + ane->tm_retention = soc->tm_retention; + + /* Managed power first: genpd links hold the ANE/DART supplier + * topology awake before any register is touched. + */ + err = ane_attach_genpd(ane); + if (err < 0) { + dev_err(dev, "failed to attach power domains\n"); + return err; + } + + ane->irq = platform_get_irq_byname(pdev, "ane"); + if (ane->irq < 0) { + err = ane->irq; + goto detach_genpd; + } + /* Polled completion design: the engine IRQ is validated but never + * requested here. A DART fault IRQ stays owned by apple-dart. During + * a job this driver masks it, and on a fault restores that stream. + */ + + /* Mapping only; no register access happens while unpowered. */ + ane->engine = devm_platform_ioremap_resource_byname(pdev, "engine"); + if (IS_ERR(ane->engine)) { + err = PTR_ERR(ane->engine); + goto detach_genpd; + } + + /* m1n1 keys the SET block by ADT node name (ANE.ps_map); the live + * overlays carry no range for it, so the SoC descriptor picks the + * base (see the tier table above). Mapped read-only, always: + * recovery logs its ACTUAL nibbles beside every engine write and + * refuses engine MMIO unless the islands read powered on. Unmapped + * is tolerated: recovery then just skips the check. + */ + ane->ps_base = soc->ps_base; + if (ane->ps_base) + ane->ps = devm_ioremap(dev, ane->ps_base, 0x38); + + mutex_init(&ane->iommu_lock); + mutex_init(&ane->engine_lock); + INIT_LIST_HEAD(&ane->bo_list); + INIT_LIST_HEAD(&ane->preserved_list); + err = ane_boost_init(ane); + if (err < 0) + goto detach_genpd; + + /* + * Kernel-owned IOMMU domain. Defers until every "iommus" provider + * has attached. There is deliberately no fallback to direct DART + * programming for mappings: without providers this driver must not + * bind. Fault recovery only rewrites the TTBR and TCR the provider + * already installed. + */ + err = ane_iommu_domain_init(ane); + if (err < 0) + goto detach_genpd; + + /* + * Managed runtime PM from here on. This resume is what raises the + * ANE partition: genpd powers the domain when the device resumes + * (through the platform bus for one power-domain entry, through the + * supplier links for several), and .runtime_resume then does the + * first engine MMIO. Marking the device active up front skips that + * resume, and on T6001 the engine window external-aborts while its + * partition is gated. The reference is held until remove: the device + * stays powered for the whole lifetime, autosuspend stays disabled. + */ + pm_runtime_enable(dev); + err = pm_runtime_resume_and_get(dev); + if (err < 0) + goto disable_pm; + + ane_iommu_purge_stale(ane); + err = ane_dart_init(ane); + if (err < 0) + goto put_pm; + + err = drm_dev_register(drm, 0); + if (err < 0) + goto put_pm; + + dev_info(dev, "loaded ane\n"); + + return 0; + +put_pm: + pm_runtime_put_noidle(dev); +disable_pm: + pm_runtime_disable(dev); + drm_mm_takedown(&ane->mm); +detach_genpd: + ane_boost_exit(ane); + ane_detach_genpd(ane); + return err; +} + +static void ane_platform_remove(struct platform_device *pdev) +{ + struct ane_device *ane = platform_get_drvdata(pdev); + struct ane_bo *bo, *tmp; + + mutex_lock(&ane->engine_lock); + ane->removed = true; + drm_dev_unplug(&ane->drm); + + if (atomic_read(&ane->wedged)) { + dev_err(ane->dev, + "forced removal of a wedged device is unsafe; reboot required\n"); + mutex_unlock(&ane->engine_lock); + return; + } + + list_for_each_entry_safe(bo, tmp, &ane->bo_list, node) + ane_iommu_unmap_pages(ane, bo); + ane_reclaim_preserved(ane); + drm_mm_takedown(&ane->mm); + + ane_boost_exit(ane); + ane_detach_genpd(ane); + + pm_runtime_disable(ane->dev); + pm_runtime_put_noidle(ane->dev); + mutex_unlock(&ane->engine_lock); +} + +static int __maybe_unused ane_runtime_suspend(struct device *dev) +{ + struct ane_device *ane = dev_get_drvdata(dev); + + /* Veto gating while the engine may be DMA-active: there is no + * documented abort/reset to establish quiescence first -- except + * when recovery is power-cycling a wedged engine to establish + * exactly that quiescence. + */ + if (atomic_read(&ane->wedged) && !ane->recovering) + return -EBUSY; + + return 0; +} + +static int __maybe_unused ane_runtime_resume(struct device *dev) +{ + struct ane_device *ane = dev_get_drvdata(dev); + bool first = !ane->tm_status_known; + + /* The only path that touches the engine while its partition comes + * up: probe's first resume and every later ungate land here. Every + * translation is owned by the IOMMU providers. + */ + if (first) { + /* Bisect order, each stage named before it runs: genpd + * raise is already complete when this callback runs; the + * SET window reads come first (pmgr class, always safe + * per T6001/T8103 bisect evidence), then the engine. A + * hard reset after the last off-box line names the + * killing access exactly (T6021 console bring-up). + */ + struct resource *eng = platform_get_resource_byname( + to_platform_device(dev), IORESOURCE_MEM, "engine"); + + dev_info(dev, + "ANE-resume: genpd raise complete; SET window probe next\n"); + ane_ps_act_probe(ane); + + dev_info(dev, + "ANE-resume: SET window probed; first engine access next (TM_TQ_EN tm+0x0c @ engine %pr + 0x2000c)\n", + eng); + } + + ane_tm_enable(ane, first); + + /* First enable is the engine's fresh signature; recovery compares + * its post-reset status against it. + */ + if (!ane->tm_status_known) { + dev_info(dev, "ANE-resume: enable writes survived; TM_STATUS read next\n"); + ane->tm_status_fresh = ane_tm_status(ane); + ane->tm_status_known = true; + /* Linux-side pwrstate probe: ACTUAL nibbles read through + * the SoC descriptor's SET window while the partition is + * raised. 0xffffff means the mapped window is the live + * pmgr SET block with every word on; anything else names + * the t6021 word layout to fix before promotion. + */ + dev_info(dev, "ANERD ps probe act=%#x\n", ane_ps_act(ane)); + } + + return 0; +} + +// clang-format off +static const struct dev_pm_ops ane_pm_ops = { + SET_RUNTIME_PM_OPS(ane_runtime_suspend, ane_runtime_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) +}; + +// clang-format on + +static struct platform_driver ane_platform_driver = { + .probe = ane_platform_probe, + .remove = ane_platform_remove, + .driver = { + .name = "ane", + .suppress_bind_attrs = true, + .dev_groups = ane_dev_groups, + .pm = pm_ptr(&ane_pm_ops), + .of_match_table = ane_of_match, + }, +}; + +module_platform_driver(ane_platform_driver); + +MODULE_AUTHOR("Eileen Yoon "); +MODULE_DESCRIPTION("Apple Neural Engine driver"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_fw_validate.h b/drivers/accel/ane/ane_fw_validate.h new file mode 100644 index 0000000000000..00ef2982c67e4 --- /dev/null +++ b/drivers/accel/ane/ane_fw_validate.h @@ -0,0 +1,480 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * ane_fw_validate.h — pinned ANE firmware images, their validator, and + * iBoot's runtime patches. + * + * Single source of truth shared by the kernel loader + * (ane_t6021_fwload.c) and the offline regression + * (tools/h14_boot_regression.c). Strict EXACT-image assertions (no + * generic Mach-O parsing). + * + * The images are the macOS 13.5 (22G74) payloads that iBoot preloads on + * the stub every Omarchy install boots: selene for T6020/T6021/T6022 + * (byte-verified live at SEG0 on T6021, notebook 20260926T230146Z) and + * bia for T8112 (receipts/2026-10-01-t8112-ane). Both have 5 load + * commands (2 segments + LC_SYMTAB + LC_UUID + LC_UNIXTHREAD), entry 0. + * + * Userspace consumers typedef u8/u32/u64/size_t/bool and provide + * get_unaligned_le32/64, memcmp and memcpy (see that tool's source); + * kernel consumers get these from linux/types.h + linux/string.h. + * This header intentionally includes nothing. + */ +#ifndef __ANE_FW_VALIDATE_H__ +#define __ANE_FW_VALIDATE_H__ + +/* FWIM surface size = config+0x138 byte-count = 0x500000 (Main + * 2026-09-20, audit 751caa4, T6021). Covers the selene vmsize 0x4fc000 + * and the bia vmsize 0x4ec000; NOT derivable from the blob length. + */ +#define ANE_FW_BUF_SIZE 0x500000 +#define ANE_FW_ENTRY_PC 0x0 + +#define MH_MAGIC_64 0xfeedfacfU +#define MH_PRELOAD 5 +#define CPU_TYPE_ARM64 0x0100000cU +#define LC_SEGMENT_64 0x19 +#define LC_UNIXTHREAD 0x5 +#define ARM_THREAD_STATE64 6 + +#define ANE_FW_NCMDS 5 +#define ANE_FW_SIZEOF_CMDS 0x960 +#define ANE_FW_FLAGS 0x1 +#define ANE_FW_NSEGS 2 +#define ANE_FW_RTK_PATCHES 5 + +struct ane_fw_seg { + u64 vmaddr; + u64 vmsize; + u64 fileoff; + u64 filesize; + char name[17]; /* 16 + NUL */ +}; + +/* One pinned image. The patch fields are the payload's LC_SYMTAB + * addresses: the __rtk_patch records _rtk_stack_guard, RTK_soc, + * RTK_soc_revision, RTK_cpu_physical_address and + * RTK_cpu_wrapper_physical_address, then the type-1 + * __rtk_platform_asc_tunables_block. + */ +struct ane_fw_image { + const char *name; /* request_firmware() path */ + u8 sha256[32]; + u32 size; + struct ane_fw_seg segs[ANE_FW_NSEGS]; + u32 rtk_vm[ANE_FW_RTK_PATCHES]; + u32 tunables_vm; +}; + +static const struct ane_fw_image ane_fw_selene = { + .name = "apple/ane/t602x_ane0_fw_selene_rc4x.macho", + .sha256 = { + 0xa9, 0xc4, 0xb7, 0x71, 0x29, 0x4a, 0x6b, 0x11, + 0x56, 0x24, 0xd9, 0x48, 0x0a, 0x62, 0x48, 0xd0, + 0x89, 0x9a, 0x16, 0x81, 0xa5, 0x75, 0xe8, 0x65, + 0x07, 0x0b, 0x87, 0xa3, 0x24, 0x84, 0x27, 0xbc, + }, + .size = 0x4c5b28, + .segs = { + { 0x000000, 0x0c4000, 0x004000, 0x0c4000, "__TEXT" }, + { 0x0c4000, 0x438000, 0x0c8000, 0x3e8000, "__DATA" }, + }, + .rtk_vm = { 0xca848, 0xca9b3, 0xca9bf, 0xca9cb, 0xca9db }, + .tunables_vm = 0xdcd78, +}; + +static const struct ane_fw_image ane_fw_bia = { + .name = "apple/ane/h14_ane_fw_bia_j4xx.macho", + .sha256 = { + 0xaf, 0x58, 0x7d, 0xfa, 0x96, 0xb1, 0xe0, 0x1d, + 0x2b, 0x2e, 0x0f, 0x9f, 0x77, 0x6e, 0x5d, 0xbe, + 0xfe, 0xbb, 0x44, 0xc7, 0x96, 0x8b, 0x86, 0xff, + 0xf7, 0xd2, 0xb1, 0x9c, 0x4c, 0xbe, 0xf0, 0xdd, + }, + .size = 0x4b5b38, + .segs = { + { 0x000000, 0x0b4000, 0x004000, 0x0b4000, "__TEXT" }, + { 0x0b4000, 0x438000, 0x0b8000, 0x3e8000, "__DATA" }, + }, + .rtk_vm = { 0xba688, 0xba7f3, 0xba7ff, 0xba80b, 0xba81b }, + .tunables_vm = 0xccbb8, +}; + +/* Validate a candidate payload against image FW. Returns 0 on pass, + * else -1 with *reason set. actual_sha = caller-computed SHA-256 of + * blob. + */ +static inline int +ane_fw_validate_blob(const u8 *blob, size_t size, + const struct ane_fw_image *fw, const u8 actual_sha[32], + struct ane_fw_seg *segs_out, u64 *entry_out, + const char **reason) +{ + u32 magic, cputype, filetype, ncmds, sizeofcmds, flags; + u32 off, i, nsegs = 0; + u64 entry = (u64)~0ULL; + + if (size != fw->size) { + *reason = "size != pinned image"; + return -1; + } + if (memcmp(fw->sha256, actual_sha, 32) != 0) { + *reason = "sha256 != pinned payload hash"; + return -1; + } + + magic = get_unaligned_le32(blob + 0); + cputype = get_unaligned_le32(blob + 4); + filetype = get_unaligned_le32(blob + 12); + ncmds = get_unaligned_le32(blob + 16); + sizeofcmds = get_unaligned_le32(blob + 20); + flags = get_unaligned_le32(blob + 24); + + if (magic != MH_MAGIC_64) { + *reason = "magic"; + return -1; + } + if (cputype != CPU_TYPE_ARM64) { + *reason = "cputype"; + return -1; + } + if (filetype != MH_PRELOAD) { + *reason = "filetype != MH_PRELOAD"; + return -1; + } + if (ncmds != ANE_FW_NCMDS || sizeofcmds != ANE_FW_SIZEOF_CMDS || + flags != ANE_FW_FLAGS) { + *reason = "header fields != pinned image"; + return -1; + } + + off = 32; + for (i = 0; i < ncmds; i++) { + u32 cmd, cmdsize; + + if (off + 8 > 32 + sizeofcmds || off + 8 > size) { + *reason = "load command header out of bounds"; + return -1; + } + cmd = get_unaligned_le32(blob + off); + cmdsize = get_unaligned_le32(blob + off + 4); + if (cmdsize < 8 || off + cmdsize > 32 + sizeofcmds || + off + cmdsize > size) { + *reason = "cmdsize out of bounds"; + return -1; + } + + if (cmd == LC_SEGMENT_64) { + u64 vmaddr = get_unaligned_le64(blob + off + 24); + u64 vmsize = get_unaligned_le64(blob + off + 32); + u64 fileoff = get_unaligned_le64(blob + off + 40); + u64 filesize = get_unaligned_le64(blob + off + 48); + const char *sname = (const char *)blob + off + 8; + unsigned int k; + + if (fileoff > size || filesize > size - fileoff) { + *reason = "segment file range out of bounds"; + return -1; + } + if (filesize > vmsize) { + *reason = "filesize > vmsize"; + return -1; + } + for (k = 0; k < ANE_FW_NSEGS; k++) { + if (vmaddr == fw->segs[k].vmaddr && + vmsize == fw->segs[k].vmsize && + fileoff == fw->segs[k].fileoff && + filesize == fw->segs[k].filesize && + !memcmp(sname, fw->segs[k].name, 16)) { + if (segs_out) + segs_out[nsegs] = fw->segs[k]; + break; + } + } + if (k == ANE_FW_NSEGS) { + *reason = "segment != pinned layout"; + return -1; + } + nsegs++; + } else if (cmd == LC_UNIXTHREAD) { + u32 p = off + 8; + + /* thread commands: flavor u32, count u32 + * (count = number of 32-bit words in state) + */ + while (p + 8 <= off + cmdsize) { + u32 flavor = get_unaligned_le32(blob + p); + u32 count = get_unaligned_le32(blob + p + 4); + + if (count > (cmdsize - (p - off) - 8) / 4) { + *reason = "thread state count out of bounds"; + return -1; + } + if (flavor == ARM_THREAD_STATE64) { + /* 33 u64 registers; PC at index 32: + * requires count*4 >= 33*8 bytes + */ + if (count < 66 || + p + 8 + count * 4 > off + cmdsize) { + *reason = "thread state too small for pc"; + return -1; + } + entry = get_unaligned_le64(blob + p + 8 + + 32 * 8); + } + p += 8 + count * 4; + } + } + off += cmdsize; + } + + if (nsegs != ANE_FW_NSEGS) { + *reason = "segment count != 2"; + return -1; + } + if (entry != ANE_FW_ENTRY_PC) { + *reason = "entry pc != 0"; + return -1; + } + + if (segs_out) { + for (i = 0; i < ANE_FW_NSEGS; i++) + segs_out[i] = fw->segs[i]; + } + if (entry_out) + *entry_out = entry; + return 0; +} + +/* + * iBoot runtime patches. iBoot places the image at its Mach-O vm layout + * (TEXT vm 0, DATA vm segs[1].vmaddr) and then writes the fields below. + * The 17 pre-Linux T6021 captures (receipts/2026-10-01-t602x-independent) + * differ from the archive in these bytes only; only the stack guard + * changes between boots. A driver that runs its own copy writes them + * itself. + * + * Each __rtk_patch_* record is {u32 tag, u32 value length, value}. + */ +#define ANE_FW_DATA_BASE_VM 0x423c /* TEXT u64: IOVA of DATA (both images) */ +#define ANE_FW_TUNABLES_MAX 0x24 /* type-1 block capacity (header byte 2) */ + +/* + * iBoot's ASC tunables for the ANE (coprocessor id 4): one record list + * and the chip-revision keys of the entries that carry it, highest + * first. iBoot fills the firmware's type-1 block from the first entry + * whose key <= the chip revision: header {type 1, flags 3 (20-byte + * records), capacity 0x24, count, u32 key}, then {u32 offset, u64 mask, + * u64 value} per record (iBoot table offsets carry flag bit 30; the + * block does not). Source: iBootStage2 macOS 26.6.2 (25G83), LZFSE, not + * encrypted (receipts/2026-10-01-t8112-optin). The 13.5 iBoot2 that + * preloads the 13.5 image is encrypted; on T6021 its block (the + * captures) equals the 25G83 records exactly. + */ +struct ane_asc_tunable { + u32 off; + u64 mask; + u64 val; +}; + +struct ane_asc_tunables { + u8 keys[4]; + u8 nkeys; + u8 n; + struct ane_asc_tunable r[ANE_FW_TUNABLES_MAX]; +}; + +/* T6021, T6022: j414c/j416c/j180d/j475d 25G83, type-1 key 0x10 (the + * 0x150010 record) then type-3 key 0x10 (the rest), which is the T6021 + * capture block byte for byte (header key 0x10). 25G83 adds key 0x11; + * the capture (RTK_soc_revision 0x11, block key 0x10) shows 13.5 has + * none, so it is left out. + */ +static const struct ane_asc_tunables ane_t602x_asc_tunables = { + .keys = { 0x10, 0x01, 0x00 }, + .nkeys = 3, + .n = 24, + .r = { + { 0x150010, 0x000000000000003f, 0x0000000000000010 }, + { 0x051c08, 0x0000000010000000, 0x0000000010000000 }, + { 0x051c18, 0x0010000000000000, 0x0010000000000000 }, + { 0x140020, 0x0000000000f00000, 0x0000000000700000 }, + { 0x140120, 0x0000000000002eff, 0x00000000000002b1 }, + { 0x140130, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140138, 0x0000000000007fff, 0x0000000000000064 }, + { 0x140140, 0x0000000000003fff, 0x000000000000002f }, + { 0x140148, 0x0000000000007fff, 0x00000000000000c8 }, + { 0x140150, 0x0000000000003fff, 0x000000000000004f }, + { 0x140158, 0x0000000000007fff, 0x000000000000014e }, + { 0x140160, 0x0000000000003fff, 0x000000000000004f }, + { 0x140168, 0x0000000000007fff, 0x000000000000014e }, + { 0x140170, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140178, 0x0000000000007fff, 0x0000000000000064 }, + { 0x140180, 0x0000000000003fff, 0x0000000000000028 }, + { 0x140188, 0x0000000000007fff, 0x00000000000000a7 }, + { 0x140190, 0x0000000000003fff, 0x0000000000000028 }, + { 0x140198, 0x0000000000007fff, 0x00000000000000a7 }, + { 0x1401a0, 0x0000000000003fff, 0x000000000000004f }, + { 0x1401a8, 0x0000000000007fff, 0x000000000000014e }, + { 0x145010, 0x000000000008000c, 0x000000000000000c }, + { 0x14a008, 0x000000000ff0ffff, 0x0000000000002520 }, + { 0x14a010, 0x000000000ff0ffff, 0x0000000002002520 }, + }, +}; + +/* T6020: j414s/j416s 25G83, one type-1 entry, key 0x00. The offsets of + * T6021; 13 values differ. + */ +static const struct ane_asc_tunables ane_t6020_asc_tunables = { + .keys = { 0x00 }, + .nkeys = 1, + .n = 24, + .r = { + { 0x150010, 0x000000000000003f, 0x0000000000000010 }, + { 0x051c08, 0x0000000010000000, 0x0000000010000000 }, + { 0x051c18, 0x0010000000000000, 0x0010000000000000 }, + { 0x140020, 0x0000000000f00000, 0x0000000000700000 }, + { 0x140120, 0x0000000000002eff, 0x00000000000002b1 }, + { 0x140130, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140138, 0x0000000000007fff, 0x0000000000000064 }, + { 0x140140, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140148, 0x0000000000007fff, 0x0000000000000064 }, + { 0x140150, 0x0000000000003fff, 0x0000000000000020 }, + { 0x140158, 0x0000000000007fff, 0x0000000000000086 }, + { 0x140160, 0x0000000000003fff, 0x000000000000004f }, + { 0x140168, 0x0000000000007fff, 0x000000000000014e }, + { 0x140170, 0x0000000000003fff, 0x0000000000000008 }, + { 0x140178, 0x0000000000007fff, 0x0000000000000022 }, + { 0x140180, 0x0000000000003fff, 0x0000000000000008 }, + { 0x140188, 0x0000000000007fff, 0x0000000000000022 }, + { 0x140190, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140198, 0x0000000000007fff, 0x0000000000000064 }, + { 0x1401a0, 0x0000000000003fff, 0x0000000000000018 }, + { 0x1401a8, 0x0000000000007fff, 0x0000000000000064 }, + { 0x145010, 0x000000000008000c, 0x000000000000000c }, + { 0x14a008, 0x000000000ff0ffff, 0x0000000000002520 }, + { 0x14a010, 0x000000000ff0ffff, 0x0000000001002520 }, + }, +}; + +/* T8112: j413/j415/j473/j493 25G83, one type-1 table under keys 0x10 + * and 0x00, no type-3 entry. The T6021 offsets without 0x150010. + */ +static const struct ane_asc_tunables ane_t8112_asc_tunables = { + .keys = { 0x10, 0x00 }, + .nkeys = 2, + .n = 23, + .r = { + { 0x051c08, 0x0000000010000000, 0x0000000010000000 }, + { 0x051c18, 0x0010000000000000, 0x0010000000000000 }, + { 0x140020, 0x4000000000f80000, 0x4000000000780000 }, + { 0x140120, 0x0000000000002eff, 0x00000000000002b1 }, + { 0x140130, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140138, 0x0000000000007fff, 0x0000000000000064 }, + { 0x140140, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140148, 0x0000000000007fff, 0x0000000000000064 }, + { 0x140150, 0x0000000000003fff, 0x0000000000000020 }, + { 0x140158, 0x0000000000007fff, 0x0000000000000086 }, + { 0x140160, 0x0000000000003fff, 0x0000000000000028 }, + { 0x140168, 0x0000000000007fff, 0x00000000000000a7 }, + { 0x140170, 0x0000000000003fff, 0x0000000000000010 }, + { 0x140178, 0x0000000000007fff, 0x0000000000000043 }, + { 0x140180, 0x0000000000003fff, 0x0000000000000010 }, + { 0x140188, 0x0000000000007fff, 0x0000000000000043 }, + { 0x140190, 0x0000000000003fff, 0x0000000000000018 }, + { 0x140198, 0x0000000000007fff, 0x0000000000000064 }, + { 0x1401a0, 0x0000000000003fff, 0x0000000000000028 }, + { 0x1401a8, 0x0000000000007fff, 0x00000000000000a7 }, + { 0x145010, 0x000000000008000c, 0x000000000008000c }, + { 0x14a008, 0x000000000ff0ffff, 0x0000000000002520 }, + { 0x14a010, 0x000000000ff0ffff, 0x0000000003002520 }, + }, +}; + +/* T8112 RTK_soc_revision from its two eFuse words at 0x23d2c8060, as + * iBootStage2 25G83 j413 0x40270 (and j415/j473/j493) computes it: + * bits 2:0 = w0[29:27], bits 6:4 = {w1[0], w0[31:30]}. + */ +static inline u32 ane_t8112_fuse_revision(u32 w0, u32 w1) +{ + return ((w0 >> 27) & 7) | ((((w0 >> 30) | (w1 << 2)) & 7) << 4); +} + +struct ane_fw_boot_patch { + u64 exec_base; /* IOVA the ASC runs vm 0 at */ + u64 stack_guard; + u32 soc; + u32 soc_revision; + u64 cpu_pa; /* ASC core block, engine + 0x1000000 */ + u64 wrapper_pa; /* ASC wrapper, engine + 0x1400000 */ +}; + +static inline void ane_fw_put_le(u8 *p, u64 v, unsigned int n) +{ + while (n--) { + *p++ = (u8)v; + v >>= 8; + } +} + +/* Patch the staged image FW in place (vm-indexed, at least + * ANE_FW_BUF_SIZE bytes). Every precondition is checked before the first + * write, so a refusal leaves the image untouched. Returns 0, or -1 with + * *reason set. + */ +static inline int +ane_fw_apply_boot_patches(u8 *img, const struct ane_fw_image *fw, + const struct ane_asc_tunables *t, + const struct ane_fw_boot_patch *p, const char **reason) +{ + static const u8 tunables_unset[8] = { + 0x01, 0x03, ANE_FW_TUNABLES_MAX, 0x00, 0xff, 0xff, 0xff, 0xff }; + static const struct { u32 tag, len; } rec[ANE_FW_RTK_PATCHES] = { + { 0x53544b47, 8 }, /* _rtk_stack_guard */ + { 0x534f435f, 4 }, /* RTK_soc */ + { 0x534f4352, 4 }, /* RTK_soc_revision */ + { 0x43704164, 8 }, /* RTK_cpu_physical_address */ + { 0x57724164, 8 }, /* ..._wrapper_physical_address */ + }; + const u64 val[ANE_FW_RTK_PATCHES] = { + p->stack_guard, p->soc, p->soc_revision, p->cpu_pa, + p->wrapper_pa, + }; + u8 *blk = img + fw->tunables_vm; + unsigned int i, k; + + for (k = 0; k < t->nkeys && t->keys[k] > p->soc_revision; k++) + ; + if (k == t->nkeys || t->n > ANE_FW_TUNABLES_MAX) { + *reason = "no ASC tunables for this chip revision"; + return -1; + } + if (get_unaligned_le64(img + ANE_FW_DATA_BASE_VM)) { + *reason = "DATA base field is not the archive zero"; + return -1; + } + for (i = 0; i < ANE_FW_RTK_PATCHES; i++) { + if (get_unaligned_le32(img + fw->rtk_vm[i]) != rec[i].tag || + get_unaligned_le32(img + fw->rtk_vm[i] + 4) != rec[i].len) { + *reason = "__rtk_patch record != pinned image"; + return -1; + } + } + if (memcmp(blk, tunables_unset, 8)) { + *reason = "tunables block header != pinned image"; + return -1; + } + + ane_fw_put_le(img + ANE_FW_DATA_BASE_VM, + p->exec_base + fw->segs[1].vmaddr, 8); + for (i = 0; i < ANE_FW_RTK_PATCHES; i++) + ane_fw_put_le(img + fw->rtk_vm[i] + 8, val[i], rec[i].len); + blk[3] = t->n; + ane_fw_put_le(blk + 4, t->keys[k], 4); + for (i = 0; i < t->n; i++) { + ane_fw_put_le(blk + 8 + 20 * i, t->r[i].off, 4); + ane_fw_put_le(blk + 12 + 20 * i, t->r[i].mask, 8); + ane_fw_put_le(blk + 20 + 20 * i, t->r[i].val, 8); + } + return 0; +} +#endif /* __ANE_FW_VALIDATE_H__ */ diff --git a/drivers/accel/ane/ane_t6021.h b/drivers/accel/ane/ane_t6021.h new file mode 100644 index 0000000000000..5e0ecc6d471f7 --- /dev/null +++ b/drivers/accel/ane/ane_t6021.h @@ -0,0 +1,796 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* T6021 (H14 / J414c) Apple Neural Engine — skeleton constants and types. + * + * Every constant below is static-decode provenance, no device writes: + * - reg windows, IRQ, pmgr islands: ADT j414c (DeviceTree.j414cap.im4p, + * receipts/2026-09-18-t6021-engine-layout-mined §2), Linux translation + * +0x200000000 (proven class, pmgr low-32 match). + * - ASC cpu block (+0x1400000 h14g) + MBI transport (SCRATCH, channel + * table, +0x1844000 doorbell): K14 kext disasm, W4-fix receipt + * (initializeANESoCConfig h14g blocks + InitializeRTBuddy + + * doorbell setter 0x…95ebdd0); m1n1 ASCRegs = t8103 cross-ref only + * - RTKit MGMT protocol: Asahi rtkit.c semantics, u64 message halves as + * staged in omarchy-ane rtkit/h14_rtkit_hello.py (commit 6ad26b7). + * - RTBuddy endpoint table + doorbell word: K14 kext cfg table + * __DATA_CONST.__const+0x814e520 and HandleRTBuddyMessage + * (receipts/2026-09-18-h14-w2-protocol-decode §3). + * + * DT binding (driver + packaging/dt/t6021-ane.dts are the two halves): + * compatible = "apple,t6021-ane" (also "apple,t6020-ane", the + * T6022 die-0 "apple,t6022-ane" and "apple,t8112-ane": + * struct ane_t602x_soc). The T6021 values follow; T8112 + * (packaging/dt/t8112-ane.dts) has engine 0x26a000000, + * pmgr 0x23b700000+0x18000, set 0x23b724000+0x4000, + * IRQ 520, seven power states 0xc008..0xc038, and a + * fourth window "fuse" (0x23d2c8060+8, chip revision). + * reg/reg-names = "engine" (whole 32 MiB ADT range0, 0x284000000; + * the H13-style +0x1c04000 engine delta does not exist + * on this SoC — kext never computes it and first touch + * external-aborted, proven 2026-09-18), + * "pmgr" (0x28e080000+0x4034, pmgr1,t6021 island words), + * "set" (0x28e08c000+0x4000, SET window — read-only by + * repo rule: direct SET writes external-abort the SoC) + * interrupts = one AIC2 level-high interrupt named "ane" (raw 884; + * [INFERENCE W2 §3: the ANE MBI doorbell IRQ]. + * dart-ane0 carries raw 885, provider-owned, never + * fetched here) + * iommus = dart-ane0 streams (apple,t6020-dart/apple,t8110-dart + * nodes). The ADT dart-ane0 is one hardware block with a + * four-window quartet (0x85800000/85810000/85820000/ + * 85804000, each 0x4000); the installed overlay's + * three-node split (t6001-proven pattern) is kept. + * power-domains = eight, phase1 raise order (sys_mpm→td→base→set1..4, + * ane_cpu last): ane_sys_mpm@4000, ane_td@4008, + * ane_base@4010, ane_set1..4@4018-4030, ane_cpu@2e0. + * ane_td/ane_base were the W3 chain gap (six consumed, + * block access reset the machine); receipt + * 2026-09-19-h14-init-sequence-kext-trace §3. + */ + +#ifndef __ANE_T6021_H__ +#define __ANE_T6021_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane_t6021_boot.h" + +/* reg windows (packaging/dt/t6021-ane.dts reg-names order) */ +enum { + ANE_T6021_REG_ENGINE, + ANE_T6021_REG_PMGR, + ANE_T6021_REG_SET, + ANE_T6021_REG_COUNT +}; + +/* pmgr ps-word fields (apple-pmgr-pwrstate layout). */ +#define ANE_PS_ON 0xf +#define ANE_PS_TARGET GENMASK(3, 0) +#define ANE_PS_ACTUAL GENMASK(7, 4) +#define ANE_PS_WAS_GATED GENMASK(9, 8) +#define ANE_PS_BUSY BIT(11) +#define ANE_PS_AUTO_ENABLE BIT(28) + +/* ANE-block-relative ASC/RTBuddy addresses. CPU block = ANE+0x1400000 + * for h14g (K14 initializeANESoCConfig h14g blocks 0x…961365c / + * 0x…9614084 build 0x1400044 with NO 0x200000 orr — the 0x1600044 + * phase-1 §2 quote is the h16g/h17/h18g variant block 0x…9613458; + * runtime h14g-shaped sites 0x…95d2968 write RUN there). CPU_STATUS + * +0x48 keeps the m1n1 ASCRegs +4 shape (config field dev+0x49c = + * 0x1400048; kext poll 0x…95ecfb4). + * + * W10 (2026-09-19) CORRECTS the claim that stood here — that the m1n1 + * t8103 mailbox (INBOX_CTRL +0x8110 / INBOX0 +0x8800) "has NO h14g + * analog". It has one, and it is live: see ANE_ASC_MBOX_* below. The + * earlier live test read +0x1608114, the h16g base, which is simply + * the wrong address on this part; and the absence of 0x1408xxx + * constants from the kext text means only that the mailbox belongs to + * the RTBuddy provider kext, not to AppleH11ANEInterface. + * + * W10 also read the CPU block for the first time, on a Linux boot with + * all eight pmgr domains at ACTUAL=0xf and the W8 grant applied: + * CPU_CONTROL 0x285400044 = 0x00000000 RUN=0 + * CPU_STATUS 0x285400048 = 0x0000002a RUNNING=0 STOPPED=1 IDLE=1 + * RVBAR 0x285050000 = 0x00000001 valid bit set, entry addr 0 + * The IOP has never been started and no firmware image is programmed, + * so nothing on this transport can answer the host until it is. + */ +#define ANE_ASC_CPU_CONTROL 0x1400044 /* RUN = BIT(4) */ +#define ANE_ASC_CPU_STATUS 0x1400048 /* m1n1 R_CPU_STATUS shape */ +#define ANE_ASC_RVBAR 0x1050000 /* fw entry | valid bit0 */ +#define ANE_ASC_EDPRCR 0x1010310 /* phase1 S2 whitelist */ +#define ANE_ASC_VERS 0x1840000 +#define ANE_ASC_RTB_STATUS 0x1840088 /* K14 poll: value < 2 */ +#define ANE_ASC_RTB_STATUS_UNK7C 0x184007c /* phase1 S2 whitelist */ + +/* ASC blocks iBoot names in the firmware's __rtk_patch records + * (RTK_cpu_physical_address 0x285000000 and RTK_cpu_wrapper_physical_address + * 0x285400000 on the T6021 preload), as engine-relative offsets. + */ +#define ANE_ASC_CPU_BASE 0x1000000 +#define ANE_ASC_WRAPPER_BASE 0x1400000 + +/* The real ASC mailbox, mapped live by W10 (read-only) at ASC+0x8000. + * Three independent sources agree on the offsets: the live DT sibling + * mbox@2a2408000 ("apple,t6020-asc-mailbox" / "apple,asc-mailbox-v4", + * size 0x4000, IRQs send-empty/send-not-empty/recv-empty/ + * recv-not-empty) fixes mailbox = asc_base + 0x8000; upstream + * drivers/soc/apple/mailbox.c apple_mbox_asc_hw fixes the register + * offsets; m1n1 proxyclient/m1n1/hw/asc.py ASCRegs gives the same set + * ASC-relative plus the R_MBOX_CTRL field layout. + * + * Observed on jw14m2 (2026-09-19), both directions identical: + * a2i_control 0x285408110 = 0x00020001 + * i2a_control 0x285408114 = 0x00020001 + * -> ENABLE=1 EMPTY=1 FULL=0 OVERFLOW=0 FIFOCNT=0 WPTR=0 RPTR=0 + * Enabled, and never used: both pointer pairs are still at the origin, + * so not one message has crossed either way since reset. 600 polls of + * i2a_control over 30 s produced zero changes. + * + * Receive is POP-ON-READ and 64-bit: upstream does + * while (!(i2a_control & EMPTY)) { readq(RECV0); readq(RECV1); } + * A 32-bit or memcpy-based read would eat messages. + */ +#define ANE_ASC_MBOX 0x1408000 +#define ANE_ASC_MBOX_A2I_CTRL 0x1408110 /* m1n1 INBOX_CTRL */ +#define ANE_ASC_MBOX_I2A_CTRL 0x1408114 /* m1n1 OUTBOX_CTRL */ +#define ANE_ASC_MBOX_A2I_SEND0 0x1408800 /* m1n1 INBOX0, u64 */ +#define ANE_ASC_MBOX_A2I_SEND1 0x1408808 /* m1n1 INBOX1, u64 */ +#define ANE_ASC_MBOX_I2A_RECV0 0x1408830 /* m1n1 OUTBOX0, u64 */ +#define ANE_ASC_MBOX_I2A_RECV1 0x1408838 /* m1n1 OUTBOX1, u64 */ + +/* R_MBOX_CTRL fields (m1n1 hw/asc.py) */ +#define ANE_ASC_MBOX_CTRL_FIFOCNT GENMASK(23, 20) +#define ANE_ASC_MBOX_CTRL_OVERFLOW BIT(18) +#define ANE_ASC_MBOX_CTRL_EMPTY BIT(17) +#define ANE_ASC_MBOX_CTRL_FULL BIT(16) +#define ANE_ASC_MBOX_CTRL_RPTR GENMASK(15, 12) +#define ANE_ASC_MBOX_CTRL_WPTR GENMASK(11, 8) +#define ANE_ASC_MBOX_CTRL_ENABLE BIT(0) + +/* W10 hazard: a plain READ of ANE+0x1854000 is fabric-fatal (watchdog + * reset ~63 s later, netconsole pinned the pre-log and no value line). + * Nothing may touch 0x1854000..0x1c04000; the old engine kill window + * 0x1c04000..0x1c28000 still stands. + */ +#define ANE_FATAL_READ_LO 0x1854000 +#define ANE_FATAL_READ_HI 0x1c04000 + +/* MBI transport (K14 h14g config: socinit stores the SCRATCH register + * offsets as q-blobs @0x…7503a40/0x…7503a50/0x…7503a60 into dev+0x438 + * and the message-register offsets @0x…7503860 into dev+0x488; runtime + * handshake in InitializeRTBuddy 0x…95e942c: cmd buffer base -> + * SCRATCH0/1 (0x…95eaa94), wake 0xf7fbdff9 -> SCRATCH7 (0x…95eab24), + * poll SCRATCH7 == 0x08042006 "channel description table ready" + * (0x…95eab74), table base read back from SCRATCH0/1 (0x…95ead04), + * per-channel {type,bit,size,phys} entries registered with the + * doorbell setter 0x…95ebdd0 writing (1 << bit) to +0x1844000. Host + * ack = SCRATCH3 = 0x08042006 (0x…95eaee4). + * + * PROVIDER DECODE (2026-09-19, kernelcache.release.mac14j): in + * RTBuddy mode the TX gate lives in com.apple.driver.RTBuddy 1.0.0 + * (carved from the KC, 46088 B, __TEXT_EXEC 0x…b696860). RTBuddy + * publishes per-ANE nubs named "%sEndpoint%u" (0x…7cc7d3f) with the + * role string "ANE" -> ANEEndpoint1..ANEEndpoint5, class + * RTBuddyEndpointService (0x98 B) wrapping RTBuddyEndpoint (0xE8 B), + * over the kernel IOSlaveEndpoint family; K14 EnableRTBuddyEndpoints + * (0x…95feb30) formats the name ("%s%d" @0x…74c481f), waits 10 s + * (x1 = 10^10 ns) for the match and takes the gate from [svc+0x88], + * registering its rx callback at gate+0xd0. The gate's per-message + * send is the vtable+0x1e8 slot; its body ends in the AKF mailbox + * write (AKF_AP_MAILBOX_SET = the +0x1844000 doorbell; RTBuddy dump + * fn 0x…b69d9c8 prints "AKF_KIC_INBOX_CTRL / AKF_KIC_MAILBOX_SET / + * AKF_AP_OUTBOX_CTRL / AKF_AP_MAILBOX_SET" and guards + * "INBOX%d not ready"/"Inbox%d overflow"). Endpoint number = + * doorbell bit: EP0 is RTBuddyManagementEndpoint (_handleHello / + * _handleEPRollCall / _handlePowerAck), EP1..5 the ANE data channels. + * [INFERENCE: the fw->host doorbell/IRQ bit numbering mirrors the + * host->fw SET bit per Asahi rtkit semantics; pinned live by W5.] + */ +#define ANE_MBI_SCRATCH0 0x1840048 /* SCRATCH0..7 = +0x48..+0x64 */ +#define ANE_MBI_SCRATCH6 0x1840060 +#define ANE_MBI_SCRATCH7 0x1840064 +/* Wake/ack handshake words live in ane_t6021_boot.h + * (ANE_T6021_BOOT_WAKE_REQ / ANE_T6021_BOOT_ACK) — single source, shared + * with the userspace boot regression. + */ +#define ANE_MBI_DOORBELL 0x1844000 /* write32 (1 << endpoint id) */ +/* NOT a message pair. W10 proved this is a mirror of the 24 MHz + * architectural counter: across 32 samples the absolute difference + * against CNTVCT_EL0 is a constant 13-18 counts (the MRS-to-MMIO + * instruction gap), aggregate drift 0.054 ppm over 55.2M counts. The + * "type-0 heartbeat, ~29 ticks/3 s" that W6/W9 recorded as firmware + * liveness was this clock plus a poll count: the RTKit type field + * GENMASK_ULL(59,52) is structurally zero until the counter passes + * 2^52 (~6 years of uptime). Kept only so the old reads stay + * identifiable in the logs; do not treat as transport. + */ +#define ANE_MBI_TIMEBASE_LO 0x1170000 /* was ANE_MBI_MSG_I2A_LO */ +#define ANE_MBI_TIMEBASE_HI 0x1170004 /* was ANE_MBI_MSG_I2A_HI */ +#define ANE_MBI_MSG_I2A_LO ANE_MBI_TIMEBASE_LO +#define ANE_MBI_MSG_I2A_HI ANE_MBI_TIMEBASE_HI +#define ANE_MBI_MSG_A2I_RD 0x184c000 /* host->fw message read peer */ +#define ANE_MBI_MSG_A2I_WR 0x1850000 /* host->fw message write */ + +/* Per-message MBI word (48-bit ring notification, K14 + * rtbuddyEndpointSendMessage 0x…95f3bf4-c30): offset = ring write + * cursor [23:0], length [47:24]. NOT the 54-bit surface-announce + * word (ane_ep_doorbell_encode below) — that one rides SetupEndpoints + * buffer mapping; this one is what the gate sends per command: the + * gate call is send(&msg48, 0, 1) at vtable+0x1e8, after the command + * bytes are already memcpy'd into the shared ring at ring_base + + * cursor. Length is capped at 0xffffff by the encoder field. + */ +#define ANE_MBI_MSG48_OFF GENMASK_ULL(23, 0) +#define ANE_MBI_MSG48_LEN GENMASK_ULL(47, 24) + +static inline u64 ane_mbi_msg48_encode(u32 cursor, u32 len) +{ + return (cursor & ANE_MBI_MSG48_OFF) | + FIELD_PREP(ANE_MBI_MSG48_LEN, len); +} + +/* Host->fw TX sequence (RTBuddy mode, all static-decode proven): + * 1. fail if len > ring_size (K14 0xe00002c2 @0x…95f3a24) + * 2. cursor = write_cursor; if (cursor + len >= ring_size) cursor = 0 + * (exact fit wraps: csel lo @0x…95f3b04) + * 3. memcpy(ring + cursor, cmd, len) (DMA-coherent ring) + * 4. dma_wmb() (ring visible before bell) + * 5. write32(ANE_MBI_MSG_A2I_WR, lo) + write32(+4, hi) — 32-bit + * halves (the AKF message register is word-shaped; W6 proved the + * W5-live abort was NOT the 64-bit writeq — the halves abort too) + * 6. write32(ANE_MBI_DOORBELL, 1 << ep) + * 7. on send success only: write_cursor = cursor + len (K14 + * 0x…95f3c70-c); the cursors at rec+0x40/rec+0x58 track the last + * issued slot. + * CSNE_CMD_PING = header-only 0x11 on EP1 (INIT) -> doorbell bit + * 1 << 1 = 0x2. Only the a2i message register + doorbell are + * touched, and only behind the mbi_doorbell opt-in (both stay + * host-write-fatal even behind the W8 grant, W9; SCRATCH family is + * host-writable granted, W9 nonzero latch). + */ + +/* MBI channel-table entry (kext stride 0x100, fields at +0x40 family: + * type @+0x40, doorbell bit @+0x44, size @+0x48, phys @+0x50 — + * InitializeRTBuddy 0x…95eade8-0x…95eae58) + */ +#define ANE_MBI_CHAN_STRIDE 0x100 +#define ANE_MBI_CHAN_MAX_DUMP 8 + +/* RTKit MGMT (EP 0); type bits [59:52], u64 message halves */ +#define ANE_RTKIT_TYPE GENMASK_ULL(59, 52) +#define ANE_RTKIT_MGMT_HELLO 1 +#define ANE_RTKIT_MGMT_HELLO_REPLY 2 +#define ANE_RTKIT_MGMT_STARTEP 5 +#define ANE_RTKIT_MGMT_SET_IOP_PWR_STATE 6 +#define ANE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK 7 +#define ANE_RTKIT_MGMT_EPMAP 8 +#define ANE_RTKIT_MGMT_SET_AP_PWR_STATE 0xb +#define ANE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK 0xb + +#define ANE_RTKIT_HELLO_MINVER GENMASK_ULL(15, 0) +#define ANE_RTKIT_HELLO_MAXVER GENMASK_ULL(31, 16) +#define ANE_RTKIT_EPMAP_LAST BIT_ULL(51) +#define ANE_RTKIT_EPMAP_BASE GENMASK_ULL(34, 32) +#define ANE_RTKIT_EPMAP_BITMAP GENMASK_ULL(31, 0) +#define ANE_RTKIT_EPMAP_REPLY_MORE BIT_ULL(0) +#define ANE_RTKIT_STARTEP_EP GENMASK_ULL(39, 32) +#define ANE_RTKIT_STARTEP_FLAG BIT_ULL(1) +#define ANE_RTKIT_PWR_STATE GENMASK_ULL(15, 0) +#define ANE_RTKIT_PWR_STATE_ON 0x20 + +#define ANE_RTKIT_VER_MIN 11 +#define ANE_RTKIT_VER_MAX 12 + +/* RTKit system endpoints rtkit.c starts when announced */ +#define ANE_RTKIT_EP_CRASHLOG 1 +#define ANE_RTKIT_EP_SYSLOG 2 +#define ANE_RTKIT_EP_DEBUG 3 +#define ANE_RTKIT_EP_IOREPORT 4 +#define ANE_RTKIT_EP_OSLOG 8 +#define ANE_RTKIT_EP_TRACEKIT 0xa + +/* RTBuddy app endpoints — K14 InitializeRTBuddyEndpoints opens ids 1..6; + * ring sizes + fourccs from the per-EP config table (W2 §3). fourcc is + * byte-reversed in the table (0x54324643 = "T2FC"). + */ +enum ane_t6021_eps { + ANE_T6021_EP_INIT = 1, /* INIT — CSNE_CMD controller channel (W4) */ + ANE_T6021_EP_T2FC, /* fw->host commands */ + ANE_T6021_EP_T2FH, /* fw->host commands */ + ANE_T6021_EP_T2HS, + ANE_T6021_EP_T2HC, + ANE_T6021_EP_T2HT, /* polled on the host */ + ANE_T6021_EP_COUNT = ANE_T6021_EP_T2HT + 1 /* arrays index by id */ +}; + +/* CSNE command ids — selene id->name table, vaddr 0xea430, 96 entries + * (full set: receipts/2026-09-18-h14-w2-protocol-decode §4 and + * receipts/2026-09-18-h14-w2-protocol-decode/fw_cmd_table.json). The + * fw parses the id as u16 at wire offset +4 (W4 correction of the W2 + * §4 "offset 0" claim: fw sites 0x4d244/0x4d264/0x5246c); the kext + * controller header for the fw->host direction is a different shape + * (u32 id @ +0x8, 0x24 bytes) and never rides host->fw submission. + */ +enum ane_t6021_csne_cmd { + CSNE_CMD_START = 0x0000, + CSNE_CMD_STOP = 0x0001, + CSNE_CMD_REG_FILE_LOAD = 0x0005, /* fw _rtk_tunables 1456 B */ + CSNE_CMD_BUILDINFO = 0x0006, + CSNE_CMD_BOOT = 0x0010, + CSNE_CMD_PING = 0x0011, + CSNE_CMD_POWER_DEVICE_ON = 0x0013, + CSNE_CMD_IPC_ENDPOINT_SET = 0x0015, + CSNE_CMD_IPC_ENDPOINT_UNSET = 0x0016, + CSNE_CMD_LOAD_PROGRAM = 0x0200, + CSNE_CMD_CREATE_PROCESS = 0x0202, + CSNE_CMD_PROCEDURE_CALL = 0x0204, + CSNE_CMD_INFERENCE_CALL = 0x0404, + CSNE_CMD_BACK_CHANNEL_RPC = 0x7000, +}; + +/* fw buffer word (selene builder 0x982b4/0x98f84, decoder 0x98fc8; the + * K14 SetupEndpoints word is the same shape): addr[43:0] | + * size_code[51:44] | unit[53:52], unit 0 = no size, 1 = code*4K, + * 2 = code*1M, 3 = code*2M. The fw encoder picks unit 1 below 1 MiB + * and unit 2 at or above, size code = CEILING division (cinc on the + * remainder), so the decoded size is the rounded-up value. rtkit.c's + * BUFFER_REQUEST is this word with unit 1. + */ +#define ANE_EP_DOORBELL_OFFSET GENMASK_ULL(43, 0) +#define ANE_EP_DOORBELL_SIZE GENMASK_ULL(51, 44) +#define ANE_EP_DOORBELL_UNIT GENMASK_ULL(53, 52) + +static const u8 ane_ep_doorbell_shift[] = { 0, 12, 20, 21 }; + +static inline u64 ane_ep_doorbell_encode(u64 offset, u32 size) +{ + u64 unit = (size >= SZ_1M) ? 2 : 1; + u32 code = DIV_ROUND_UP(size, 1u << ane_ep_doorbell_shift[unit]); + + return (offset & ANE_EP_DOORBELL_OFFSET) | + FIELD_PREP(ANE_EP_DOORBELL_SIZE, code) | + FIELD_PREP(ANE_EP_DOORBELL_UNIT, unit); +} + +static inline u32 ane_ep_doorbell_size(u64 msg) +{ + u32 unit = FIELD_GET(ANE_EP_DOORBELL_UNIT, msg); + + return FIELD_GET(ANE_EP_DOORBELL_SIZE, msg) << ane_ep_doorbell_shift[unit]; +} + +struct ane_t6021_ep { + u8 id; + const char *name; + u32 fourcc; + u32 ring_size; + u32 write_cursor; /* next ring slot (K14 rec+0x20) */ + void *ring; /* dma_alloc_coherent, ring_size */ + dma_addr_t ring_iova; + bool started; +}; + +struct reset_control; + +struct ane_t6021 { + struct device *dev; + void __iomem *base[ANE_T6021_REG_COUNT]; + int irq; + + struct device **pd_dev; + struct device_link **pd_link; + int pd_count; + + /* Single MBI consumer (threaded IRQ + probe drain serialize + * here) + */ + struct mutex mbox_lock; + + /* Bring-up state machine — honest semantics (W15): + * power_gated: first_resume passed (eight-island gate + + * whitelist; block access proven-safe), + * cpu_started: RVBAR programmed + CPU RUN released (fw_boot=1), + * fw_alive: fw first-alive ack observed on SCRATCH7 — NOT + * the init handshake, + * booted: full init handshake observed. CSNE/MGMT sessions + * gate on this flag only. + */ + bool power_gated; + bool cpu_started; + bool fw_alive; + bool booted; + + /* W10: the mailbox does exist — at +0x1408xxx, not the h16g + * +0x1608xxx that read-aborted in 2026-09-19 — and it reads a + * live, enabled, permanently-empty ASC v4 control pair. It is + * empty because the ASC CPU is STOPPED with RUN clear and RVBAR + * entry 0, i.e. no firmware was ever started. Default off = + * status-only bring-up. Opt-in runs the kext-evidenced MBI + * handshake (SCRATCH wake -> fw channel table), capture-only. + */ + bool transport; + bool doorbell; /* mbi_doorbell=1: EP rings may write the +0x1844000 + * doorbell + a2i message register (decoded 2026-09-19) + */ + bool irq_requested; + + struct ane_t6021_ep ep[ANE_T6021_EP_COUNT]; + + /* W13 fw surface (fw_load=1): coherent, DART-mapped via the + * device's iommu group. NULL unless loaded. + */ + void *fw_buf; + dma_addr_t fw_iova; + u32 fw_size; + /* W16 entry alias: IOVA the fw pages are aliased at (the latched + * RVBAR entry); 0 = no alias mapped. The DMA allocator has no + * runtime reservation API on this kernel, so instead of a prose + * collision bound every DMA allocation site must pass its iova + * through ane_t6021_fw_alias_iova_ok() and refuse overlaps. + */ + u64 fw_alias_iova; + /* Per-window mapped extents (iova, bytes) — teardown unmaps + * exactly these, because the windows are not assumed adjacent + * (live trace: hole between SEG0 and SEGi) and unmapping a page + * that was never mapped trips dart_unmap_pages (io-pgtable-dart + * .c:319 WARN, 2026-09-26 state-report unwind). Set only on + * successful map. + */ +#define ANE_FW_ALIAS_MAX_WIN 3 + u64 fw_alias_ext_iova[ANE_FW_ALIAS_MAX_WIN]; + size_t fw_alias_ext_len[ANE_FW_ALIAS_MAX_WIN]; + int fw_alias_extn; + /* W16 pass-3: ane_cpu reset controller (ps RESET via the pmgr + * pwrstate reset_controller ops); NULL when the DT carries no + * resets property. + */ + struct reset_control *cpu_rst; + + /* W15 boot allocations (gate-gated: never exist until the + * preflight opens and the sequence passes poll A). Ownership per + * the wedged-pin rule: held, never freed, while cpu_started. + */ + void *boot_pool; /* 'DDM ' pool, 0x40000 (Params word0) */ + dma_addr_t boot_pool_iova; + void *boot_ipc; /* 'IPC ' surface, max(0x4000, ord+1) */ + dma_addr_t boot_ipc_iova; + u64 boot_ipc_size; + void *boot_heap; /* fw-requested HEAP surface or NULL */ + dma_addr_t boot_heap_iova; + u64 boot_heap_size; + u32 prev_fw_len; /* [0x30]: previous fw image length + * (0 first boot; updated per reload) + */ + u64 boot_scratch_result; /* SCRATCH1<<32 | SCRATCH0 captured + * at DONE — raw; success semantics + * UNSOURCED (never inferred) + */ + bool response_validated; + bool hybrid_pinned; /* FALSE until the DONE response + * semantics are sourced AND the + * raw address is range/length + * validated against owned + * windows — transport/CSNE + * sessions stay fenced on first + * boot regardless of booted + */ +}; + +/* Enforced alias-window invariant (W16): the DMA allocator has no + * runtime IOVA reservation on this kernel, so every DMA allocation + * site must refuse a mapping that overlaps the fw entry alias. + */ +static inline bool ane_t6021_fw_alias_iova_ok(const struct ane_t6021 *ane, + dma_addr_t iova, size_t size) +{ + u64 lo = ane->fw_alias_iova; + + if (!lo) + return true; + return iova + size <= lo || iova >= lo + ane->fw_size; +} + +/* ane_t6021_rtkit.c */ +int ane_t6021_rtkit_init(struct ane_t6021 *ane); +void ane_t6021_rtkit_shutdown(struct ane_t6021 *ane); +void ane_t6021_rtkit_drain(struct ane_t6021 *ane); +irqreturn_t ane_t6021_rtkit_irq_thread(int irq, void *data); + +/* ane_t6021_boot.c — W15 boot state resolution (fw_boot=1). An + * explicit boot request fails the probe while the prerequisites in + * ane_t6021_boot.c hold (-ENODATA); fw_boot=0 binds status-only. + * ane_t6021_boot_start() is the sequence dispatcher proper (all gates + * must already hold): it owns the wedged-pin module lifetime once the + * CPU is released; stop_after = 0 full run, 1..4 = fw-start-debug + * step bisect (stop after that step, -ECANCELED, clean unwind while + * no CPU started). + */ +int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, + int rtb_mode); + +/* ---- CSNE_CMD wire structs (host->fw on the INIT channel) ---- + * + * W4 static decode, extending W2 §4 (the receipt's "fw parses a u16 id + * at header offset 0" was WRONG — three independent fw parse sites read + * the id as u16 at offset +4: the generic CSNE processor @0x4d264, its + * preload @0x4d244, and the LOAD_PROGRAM/CREATE_PROCESS/PROCEDURE_CALL + * pre-parse @0x5246c, all `ldrh wN, [xM, #4]` in selene). Wire is + * little-endian; this module is arm64-only so host order is wire order. + * + * The generic processor takes commands < 0x1b89 bytes (work-item size + * bound @0x4d134-0x4d158) and writes completion state back into the + * command block (strb @0x4d324 byte +6, str @0x4d3b0 qword +8) — the + * ring slot doubles as the response area. PING/BUILDINFO/BOOT/ + * REG_FILE_LOAD/IPC_ENDPOINT_SET fall through the processor's id tree + * to the default path (0x4e65c): carried without field parsing, so + * their payloads beyond the header are opaque until the W1 live + * exchange pins them. + */ +struct ane_csne_hdr { + u32 rsvd0; /* bytes 0..3: never read by the fw processor; 0 */ + u16 id; /* PROVEN u16 @ +4 (sites above) */ + u8 flags; /* fw-written byte @ +6 (status/scratch) [INFERENCE] */ + u8 rsvd7; +}; + +static_assert(sizeof(struct ane_csne_hdr) == 8); + +static inline void ane_csne_hdr_init(struct ane_csne_hdr *h, u16 id) +{ + memset(h, 0, sizeof(*h)); + h->id = id; +} + +/* Header-only CSNE_CMDs: BOOT (0x10 — boot-arg surfaces ride + * SCRATCH0-7 / SetupFWInitBootArgs, not the command, phase1 §2.5), + * PING (0x11), BUILDINFO (0x06). sizeof(struct ane_csne_hdr) bytes. + */ + +/* REG_FILE_LOAD (0x05) payload: the 1456 B blob is the fw's own + * __DATA._rtk_tunables section (@0x100590, size 0x5b0) — its transport + * (inline vs shared-memory iova) is [INFERENCE], pinned by W1. + */ +struct ane_csne_cmd_reg_file_load { + struct ane_csne_hdr hdr; + u8 blob[]; +}; + +/* IPC_ENDPOINT_SET (0x15) payload: binds a host RTBuddy endpoint to a + * fw channel. No field parsed by the fw processor — layout + * [INFERENCE], pinned by W1. + */ +struct ane_csne_cmd_ipc_endpoint_set { + struct ane_csne_hdr hdr; + u8 payload[]; +}; + +/* PROCEDURE_CALL (0x204) / INFERENCE_CALL (0x404) — the kext + * CANE_SUB_PACKET_CMD_PROCEDURE_CALL family. Field offsets PROVEN from + * the fw parse; field semantics named only where the kext asserts name + * them (programId, procedureId, numIoBuffers in assert order, W2 §4): + * +0x08/+0x0c u32 pair validated together (ldp @0x4d654) + * +0x10 u64 passed to the fw validator (0x4d6d4) + * +0x18 u32 stats/priority, fw-valid [2,7]; 2 is the value + * proven end-to-end (binding.json stats_type; the + * first-inference receipt's bare calls) + * +0x28 u32 element count (loop bound @0x524bc-0x524c4) + * +0x60 count × 0x30-byte io-buffer records (@0x524a8-0x524b4) + * The kext `size % sizeof(sCSneCmdProcedureCall) == 0` assert says + * commands form same-shape arrays in the ring; this driver submits one + * command per slot. INFERENCE_CALL shares the shape [INFERENCE: the + * 0x404 id is not in the decoded id tree — it is the W4 submission + * endpoint per the phase-1 workstream plan]. + */ +struct ane_csne_io_elem { + u8 bytes[0x30]; /* internal layout not decoded */ +}; + +struct ane_csne_cmd_procedure_call { + struct ane_csne_hdr hdr; + u32 program_id; /* +0x08 */ + u32 procedure_id; /* +0x0c */ + u64 field_10; + u32 field_18; /* fw requires 8..15 */ + u32 rsvd_1c; + u64 field_20; + u32 num_io_buffers; /* +0x28 = element count */ + u32 rsvd_2c; + u8 gap_30[0x30]; + struct ane_csne_io_elem io[]; +}; + +static_assert(offsetof(struct ane_csne_cmd_procedure_call, program_id) == 0x08); +static_assert(offsetof(struct ane_csne_cmd_procedure_call, procedure_id) == 0x0c); +static_assert(offsetof(struct ane_csne_cmd_procedure_call, field_10) == 0x10); +static_assert(offsetof(struct ane_csne_cmd_procedure_call, field_18) == 0x18); +static_assert(offsetof(struct ane_csne_cmd_procedure_call, num_io_buffers) == 0x28); +static_assert(offsetof(struct ane_csne_cmd_procedure_call, io) == 0x60); + +static inline size_t +ane_csne_cmd_procedure_call_size(unsigned int num_io_buffers) +{ + return sizeof(struct ane_csne_cmd_procedure_call) + + num_io_buffers * sizeof(struct ane_csne_io_elem); +} + +/* Fw-side bound: the generic CSNE processor rejects work items of + * 0x1b89 bytes and above (@0x4d134 cmp x2, #0x1b89). What x2 names + * beyond "the command's size" is [INFERENCE] — enforced here as a + * fail-fast so an oversized command cannot enter the ring. + */ +#define ANE_CSNE_CMD_MAX_SIZE 0x1b88 + +/* ---- LOAD_PROGRAM (0x200) wire contract (selene 0x4e44c hook + + * 0x4e53c worker + 0x4e640 registration; decode 2026-09-26, entry + * item 23/23b/23c/23e): the payload is NINE 0x30-byte named section + * records at +0x08. Registration publishes ".bin" into the fw + * program-info symbol table (format "%s.bin" @0xa733f, + * CAneProgramInfo.cpp) with value = record+0x18 and aux = + * u32(record+0x20). The fw dereferences record+0x18 (0x5d0c8), so it + * carries a fw-addressable pointer — in this driver, the IOVA of a + * host-authored section object (below). Absent sections (flags bit0 + * clear) are skipped by every decoded parser. ---- + */ +enum ane_t6021_load_section { + ANE_SEC_GENERIC = 0, + ANE_SEC_KERNEL, + ANE_SEC_TEXT, + ANE_SEC_OPERATION, + ANE_SEC_PROCEDURE, + ANE_SEC_KERNELPROP, + ANE_SEC_TEXTPROP, + ANE_SEC_OPDBG, + ANE_SEC_PROCPROP, + ANE_SEC_COUNT /* 9 */ +}; + +#define ANE_T6021_LOAD_SEC_COUNT 9 + +static const char * const +ane_t6021_load_sec_name[ANE_T6021_LOAD_SEC_COUNT] = { + "genericSection", "kernelSection", "textSection", + "operationSection", "procedureSection", "kernelPropSection", + "textPropSection", "opDbgSection", "procPropSection", +}; + +struct ane_csne_cmd_load_program { + struct ane_csne_hdr hdr; /* id 0x200 @ +4 */ + struct ane_csne_io_elem sec[ANE_T6021_LOAD_SEC_COUNT]; +}; + +static_assert(sizeof(struct ane_csne_cmd_load_program) == + 0x08 + 9 * 0x30); + +/* Section-record wire accessors over the 0x30-byte io_elem (the only + * bytes any decoded parser reads; the rest is unread pass-through): + * +0x00 u8 flags — bit0 present; +0x18 u64 obj — fw-addressable + * pointer, validated by 0x5d0c8 to land inside a registered program + * object's entry table; +0x20 u64 key — per-section lookup key. + */ +#define ANE_SEC_F_PRESENT BIT(0) + +static inline void ane_sec_record_init(void *rec, u64 obj, u64 key) +{ + u8 *r = rec; + + memset(r, 0, 0x30); + r[0] |= ANE_SEC_F_PRESENT; + *(u64 *)(r + 0x18) = obj; + *(u64 *)(r + 0x20) = key; +} + +/* Host-authored PROGRAM OBJECT (the thing section records point at; + * 0x5d0c8 validates: magic 1 @+0, count <= 0x10 @+4, table entry + * count in [0x201, 0x400] @+0x204, 0x30-byte entry table @+0x208 with + * the recorded pointer bounded inside it). Minimum object = header + + * 0x201 zeroed entries; entries are the same 0x30-byte shape. + */ +#define ANE_PROGOBJ_MIN_ENTRIES 0x201 +#define ANE_PROGOBJ_MAX_ENTRIES 0x400 +#define ANE_PROGOBJ_TABLE_OFF 0x208 +#define ANE_PROGOBJ_HDR_SZ 0x208 + +static inline size_t ane_progobj_size(u32 entries) +{ + return ANE_PROGOBJ_TABLE_OFF + (size_t)entries * 0x30; +} + +static inline void ane_progobj_init(void *obj, u32 entries) +{ + struct { u32 magic; u32 count; u32 rsvd[2]; u32 entries; } *h = obj; + + h->magic = 1; + h->count = 0; + h->rsvd[0] = 0; + h->rsvd[1] = 0; + h->entries = entries; + /* caller zeroes the tail: entries table starts at +0x208 */ +} + +/* Host-authored OPERATION-SECTION image (0x5d290: u32 op_count @+0, + * <= 0x80; then op_count 0x40c-byte operation records at +4; the + * section KEY is an offset past the array, >= op_count*0x40c + 4). + * Operation record fields decoded: +0x00 u32 type <= 4; +0x04 u16 + * <= 0x10; +0x08 u32 procedure_count <= 0x80 (non-zero); +0x0c.. + * u32 procedure indices, each <= 0x3c. + */ +#define ANE_OPSEC_OP_REC_SIZE 0x40c +#define ANE_OPSEC_MAX_OPS 0x80 + +static inline size_t ane_opsec_size(u32 ops) +{ + return 4 + (size_t)ops * ANE_OPSEC_OP_REC_SIZE; +} + +/* Submit one CSNE command block on the INIT (EP1) ring: K14 + * rtbuddyEndpointSendMessage semantics (W2 §3) — slot alloc with wrap, + * memcpy into the ring, 54-bit doorbell word, cursor + * advanced only on doorbell success. No synchronous response matching: + * fw->host responses arrive on the T2F* channels and are not walked + * yet (W2 §3). Sleeps (mutex) — process context only. + */ +int ane_t6021_csne_submit(struct ane_t6021 *ane, const void *cmd, size_t size); + +/* Probe-time one-shot CSNE_CMD_PING on EP1 (W5-live), behind + * mbi_doorbell=1 only; watches the fw response surfaces for 3 s. + */ +void ane_t6021_csne_ping_attempt(struct ane_t6021 *ane); + +/* W13/W14 firmware loader (ane_t6021_fwload.c): validate + stage + + * dart-ane0-map the 13.5 PRELOAD payload behind fw_load=1. This is + * the staging half of the boot contract: the Params+0x18 producer + * chain and the RVBAR fold are closed (mapper-callchain audit, commits + * 3762aee/12be074); ane_t6021_boot.c consumes the staged surface. + */ +int ane_t6021_fwload_probe(struct ane_t6021 *ane); +void ane_t6021_fwload_remove(struct ane_t6021 *ane); +bool ane_t6021_fwload_options_ok(void); +bool ane_t6021_fwload_placement_ok(struct device *dev); +bool ane_t6021_fwload_requested(void); + +struct ane_fw_image; +struct ane_asc_tunables; + +/* Per-SoC of_match data (defined in ane_t6021_fwload.c). + * soc, soc_revision: what iBoot writes to RTK_soc and RTK_soc_revision. + * revision_fuse: read the revision from the DT "fuse" window as iBoot + * does instead (T8112). + * preload_placement: the iBoot SEG0/SEGi physical placement is recorded + * for this SoC (T6021 only), so fw_alias_reserved=1 may map it; the + * other SoCs always run the firmware from driver-owned memory. + * ps_cpu_off: the ANE CPU ps word in the DT "pmgr" window (probe guard). + * pwgate_off: the kext's PWGATE word in the DT "set" window, checked + * open before any engine read; 0 = not checked. + * pmu_pa: the page of the seven ANE ps words; the firmware's power + * service writes them through its DART at IOVA == PA. ps_off: the + * first of the seven in that page. + * trace_td_off: engine offset of the TM last-committed-TD word read by + * trace_td; 0 = trace_td unsupported. + */ +struct ane_t602x_soc { + u32 soc; + u32 soc_revision; + bool revision_fuse; + bool preload_placement; + const struct ane_fw_image *fw; + const struct ane_asc_tunables *tunables; + u32 ps_cpu_off; + u32 pwgate_off; + u64 pmu_pa; + u32 ps_off; + u32 trace_td_off; +}; + +extern const struct ane_t602x_soc ane_t6020_soc, ane_t6021_soc, + ane_t6022_soc, ane_t8112_soc; + +#endif /* __ANE_T6021_H__ */ diff --git a/drivers/accel/ane/ane_t6021_boot.c b/drivers/accel/ane/ane_t6021_boot.c new file mode 100644 index 0000000000000..066d2a799851a --- /dev/null +++ b/drivers/accel/ane/ane_t6021_boot.c @@ -0,0 +1,557 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * ane_t6021_boot.c — T6021 boot state reporter (W15). + * + * READ-ONLY. Main review 2026-09-20: the unresolved pre-CPU power and + * config-table prerequisites BLOCK every boot write, so this unit + * performs no MMIO write at all — it resolves the boot-input state, + * reports it with per-read whitelist provenance, and names the exact + * missing lifecycle datum. The write path (RVBAR entry fold + CPU RUN + * release + first-alive ack) lands only after the prerequisites below + * are pinned; its pure contract units (entry fold, latch/entry decode, + * SCRATCH u64 split, init-structure fill) live in ane_t6021_boot.h and + * are regression-tested host-side (tools/h14_boot_regression.c). + * + * Blocked boot writes, with the blocking prerequisite each one waits on: + * - RVBAR entry fold (ENTRY_BASE | fw DVA & ADDR_MASK; single u64 + * store, width_proof 0x95e9860→0x9622b0c read / 0x95e9988→0x9622c94 + * write): semantics of the observed RVBAR read 0x1 are unresolved + * (see the rvbar lifecycle fork) AND the pre-CPU block + * below applies regardless. + * - CPU_CONTROL write32 0 then 0x10 (rvbar-width local order): waits + * on the same pre-CPU block; releasing RUN with an unresolved + * entry state is the exact blind start the lane forbids. + * - SCRATCH0/1 publication + SCRATCH7 wake: init-structure fields + * [0x08] and [0x10,0x38) have no pinned selene fn 0x71A4 consumer + * read-set; zero-filling unknown fields and calling the structure + * valid is forbidden (Main 2026-09-20). + * + * Pre-CPU prerequisites, updated for pass4 (receipts 2026-09-20 + * mapper-callchain pass4 + tools/check_power_anchors.py 33/33, commit + * 04630ef): + * RESOLVED (pass4): the CPU-power branch selector — H14g SoCConfig + * zeroes dev+0x3EA0 (0x9613e14, the block that also sets + * dev+0x4A0=0x01400044), so EnableCPUClocksAndPower takes the + * PLAIN branch: write32(PMGR+0x2e0, 0x0000000f), validate 0xff — + * AUTO_ENABLE bit28 never set. On Linux this END STATE is exactly + * what the first_resume stage-2/3 gate verifies (act=0xf, + * AUTO_ENABLE clear) before any of this code runs — the write is + * therefore not replicated (ps-word writes are forbidden by the + * standing rule and froze the box 2026-09-19); the gate IS the + * equivalence check. + * STILL OPEN (each blocks its write below): + * 1. CLOSED (pass5 c364f24): the pre-CPU engine table writels + * eng+0xb38/0xb98/0xbf8 <- 0x01ff01ff are REQUIRED every + * EnableANEClocksAndPower — dev+0x784 bit0 has no writer, so the + * always-open gate runs the loop unconditionally; the + * order-vs-start question is moot. Receiver PROVEN pass4 + * (dev+0x188 accessor = engine reg[0]); count = config+0x150 = + * 3. Armed below; the first live firing must ride netconsole + * behind the seam logs. + * 2. Provider first-enable enableDeviceClock(1,dev+0x8F0)/ + * enableDevicePower(1,&out,dev+0x8F0) (0x95d1d48/0x95d1d90): + * internal gate-ID arrays are runtime-populated (open). Linux + * substitutes the genpd raise + supplier links (islands verified + * ACTUAL=0xf by the gate); the equivalence datum is the open + * half. + * 3. RVBAR lifecycle — RESOLVED (M2ResetLifecycle, + * rvbar-lifecycle-evidence.json, anchors 38/38 + * check_rvbar_lifecycle_anchors.py): live bit0-set + entry 0 is + * the STALE state; the kext-lawful transition is power_off + * (ANE_deInit when dev+0x3FB, then PMGR ps 0x2e0 ← 0x00000000 via + * the dev+0x190 PMGR accessor) → power_on → re-init, NEVER an + * RVBAR write while bit0 is set. RVBAR (engine accessor, + * 0x1050000, u64) is written only after a read64 with bit0 == 0; + * CPU_CONTROL write32 0 then 0x10 runs on BOTH paths; success = + * SCRATCH7 (dev+0x454 selector) read32 == 0x08042006. Open edge: + * whether ps-off clears bit0 — one live read-only read64 after a + * domain-off answers it. IMPLEMENTATION CAVEAT: the ps-word write + * is the class that froze this host from kernel context + * (2026-09-19 receipt) and the W3 gate demands act=0xf + + * AUTO_ENABLE clear afterwards — the reset vehicle (userspace + * stage vs in-kernel) is a Main decision, so the fold+start path + * stays behind the itemized preflight gates. + * 3. Ack model — TWO-PHASE, SINGLE REGISTER, TRANSITION-BASED + * (rvbar-lifecycle receipt 6288b0b, 57/57 anchors; supersedes the + * pass6 SCRATCH0-beacon reading): pre-CPU, SCRATCH cells are + * cleared and SCRATCH7 is PULSED 1 -> 0 (stale READY/wake + * cleared); after CPU_CONTROL 0 -> 0x10, poll A demands a FRESH + * SCRATCH7 READY 0x08042006 (fw alive; fw_alive). Publication + * (init suballoc DVA -> SCRATCH0/1, dsb st) happens ONLY after + * poll A; then wake 0xf7fbdff9 -> SCRATCH7 releases the fw; poll + * B awaits SCRATCH7 DONE 0x08042006 (booted) and reads back + * SCRATCH0/1. Because of the pulse, a post-CPU READY is + * unambiguous — no stale-ack hazard. The SCRATCH0 marker (fn + * 0x86EC) is not a gate. + * Selene raw anchors: READY write idx7 @0x7360-0x7388 (movz + * 0x2006 @0x7364, mov w1,#7 @0x7368, movk 0x804 @0x736c, slot + * byte48 @0x7380, salt 0x5bdd @0x7384, blraa @0x7388); poll loop + * READ32 idx7 @0x73c4-0x73e4, cmp wake 0xf7fbdff9 @0x73e8, + * sleep(1000) @0x73f8-0x73fc; DONE ack @0x77cc-0x77f0. + * 4. Linux allocation map (legacy branch sizes): FWIM surface = + * config+0x138 byte-count = 0x500000 (Main, audit 751caa4); + * Linux fw_buf is ANE_FW_BUF_SIZE = 0x500000 — the semantic is + * honored directly, covering the image vmsize 0x36c000. 'IPC ' + * surface = min(config+4, dev+0x3A70 + * cap) — the numeric cap needs the h14g config blob field map + * (open prerequisite). Init suballoc = 0x174 from a Linux-owned + * pool standing in for the kext dev+0x968/dev+0x980 pool (pool + * total size open). RTBuddy FW_INIT sizes (64K…) are the OTHER + * branch — never used here. + * 5. Init publication: the fw consumes [0x08]..[0x68] (pass5) and + * the Linux sources of those fields are not pinned ([0x08] = + * *(dev+0x988+0x18) = the 'IPC ' surface DVA — the surface + * itself is CLOSED (pass5c/5d), its Linux allocation size open + * per item 4; [0x10]/[0x18] = config-size terms with + * config+0x138 = image byte-count closed; [0x30] = dev+0x990 + * load-progress word) — publication cannot fire until every + * fw-read field has a pinned Linux source. + * + * Read whitelist — every MMIO read below, individually: + * - RVBAR eng+0x01050000: read64 width-proven (rvbar-width + * width_proof); W8 and W10 live Linux reads, value 0x1 each. + * - CPU_STATUS eng+0x1400048: phase-1 S2 whitelist; kext poll site + * 0x…95ecfb4 (config field dev+0x49c = 0x1400048); W10 live read + * 0x2a. + * - ASC mailbox controls eng+0x1408110/0x1408114: W10 live reads + * (both 0x00020001; 600 polls / 30 s, zero changes, zero aborts). + * The FATAL mailbox-class reads were the +0x1608xxx h16g analog — + * a different address never touched here; the 0x1854000..0x1c04000 + * fabric-fatal window is likewise never touched. + * - SCRATCH eng+0x1840048..0x1840064: phase-1 S2 whitelist + * (all-zero pre-attach), W5 pre-read log clean, first_resume + * re-reads all eight on every load. + * - TICK eng+0x1160008 (PROGRESS dump only): 24 MHz domain tick, + * proven readable post-release (2026-09-25 handshake receipt); + * two reads 20 ms apart for a delta. engine+0x1140008 is + * deliberately NOT read (readability/width/reset semantics + * unproven, no pre-release baseline — Main review 2026-09-26). + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane_t6021.h" + +#include "ane_t6021_boot.h" + +/* Retain the firmware nap-prevention counter via the init resource bit + * (lab boot_prevent_nap=1, the proven add-path value). + */ +static bool boot_prevent_nap = true; +module_param(boot_prevent_nap, bool, 0444); +MODULE_PARM_DESC(boot_prevent_nap, + "Retain firmware nap-prevention counter via init resource bit (default on: proven add-path value)"); + +/* This object links into both ane_t6021.ko and ane_t6021_rtclient.ko; + * per-object metadata keeps modpost happy for either composition. + */ +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("T6021 ANE contract-pinned boot sequence core"); + +/* Boot-write gates — ITEMIZED, each a HARD gate: the ENTIRE write + * sequence (preboot engine table, scratch clear + pulse, RVBAR + * resolution, CPU release, publication, wake) runs start-to-finish + * only when EVERY gate below is true, checked BEFORE any write (Main + * 2026-09-20: no partial boot; remaining semantics stay hard gates, + * not comments). Each flips only with cited proof in a Main-reviewed + * commit that also populates the sources and adds the kernel io + * backend — never a runtime knob. + */ +/* Provider equivalence — CLOSED (Main provider review 2026-09-20): + * Linux genpd attachment + supplier links accepted INSTEAD of + * reproducing Apple runtime index arrays, CONDITIONAL on the live + * first_resume verifying all 8 islands ACTUAL=0xf, BUSY=0, CPU + * AUTO_ENABLE clear before the sequence — that verification runs on + * every probe and the driver performs no direct kernel PMGR writes. + */ +static const bool pf_provider_genpd_strategy = true; +static const bool pf_pool_word0_proven = true; /* CONFIRMED (Main raw + + * Reset 183fd50, chain + * 0x67dc/0x6824/0x68d8/ + * 0x70dc/0x736c): DDM Params + * word0 = requested bytes = + * 0x40000 — sourced, not + * synthesized + */ +static const bool pf_heap_floor_pinned = true; /* CLOSED: floor = + * getPageSize() return = + * DART page size 0x4000 + * (store chain 0x9602e68-90 + * + dart-ane0 page-size + * 0x4000 node, pass6h); + * sourced in boot_sources + */ +static const bool pf_dart_page_floor = true; /* CLOSED (Main raw, + * pass6h + exact-node + * linkage): dtree-j414c.txt + * sha256 dd2955…d504, ane0 + * iommu-parent 363 (line + * 11499), dart-ane0 + * page-size 0x4000 (line + * 11510), child mapper-ane0 + * phandle 363 (line 11527). + * Runnable linkage check: + * ane-linux-experiments + * tools/ + * check_dart_ane_pagesize_linkage.py + */ +static const bool pf_rvbar_lifecycle = true; /* 6288b0b, 57/57 anchors */ +static const bool pf_pass6_init_contract = true; /* cd25b46, 87/87 */ + +/* FINAL authorization gate — CLOSED (Main provider review + pass6h + * numeric closure + 183fd50 word0 confirmation + netconsole Wi-Fi + * marker m2-wifi-check-1789937086 @ 15:44:47.121595Z). Source + * contract closure complete; live attempt authorized by user + * override (autonomous boot/recovery loop). + */ +/* Table-block mode selection (2026-09-20 16:23:07 wedge): + * mode 0 = ABORT before any write (accidental-repeat prevention), + * mode 1 = write the table (kext-faithful; selected per-run via the + * rtclient fw_start_table_mode param), + * mode 2 = SKIP the table (default: the shipped diagnostic). + * W8 write-grant tunables are mode-independent (proven no-abort + * class, w8-run.out: APERTURE_UNLOCKED) and run in every armed mode. + */ + +static const bool pf_main_lifetime_review = true; + +static bool ane_t6021_boot_preflight_complete(void) +{ + /* pf_preboot_table_safe is INTENTIONALLY NOT in this product: + * mode 2 (table skip) is the authorized diagnostic state — + * the preflight covers the source contract, the mode selector + * covers the live-fault table gating. + */ + return pf_provider_genpd_strategy && pf_pool_word0_proven && + pf_heap_floor_pinned && pf_dart_page_floor && + pf_rvbar_lifecycle && pf_pass6_init_contract && + pf_main_lifetime_review; +} + +/* STATIC boot sources — populated per the pinned contract (Main + * authorization 2026-09-20). Object layout (Main raw correction, + * superseding audit 018abdb): the OSValueObject at +0x10 POINTS to + * the Params; Params+0x00 = size, +0x18 = DVA, +0x38 = hostVA. + * dev+0x980 is the 'DDM ' Params pool (size 0x40000); dev+0x968 is a + * SEPARATE DMM manager constructed over the pool hostVA — never + * conflate the two. Runtime fields (fw_dva, ipc_dva, pool_dma) are + * filled by ane_t6021_boot_prepare() at sequence time (after poll A, + * per "dynamic allocations occur after READY"); heap_floor stays 0 + * here and is supplied together with pf_heap_floor_pinned — NEVER + * used unpinned (the gate above keeps the whole sequence unrun until + * then). + */ +static const struct ane_t6021_init_sources boot_sources = { + .cfg_size = 0x500000, /* config+0x138 (0x9613da8/ dac) */ + .prev_fw_len = 0, /* first boot; static per reload */ + .heap_floor = 0x4000, /* DART page size: dev+0x3A90 = + * getPageSize() return; node + * authority dart-ane0 page-size + * 0x4000 (exact linkage check + * passing) + */ + .pool_word0 = 0x40000, /* DDM Params word0 = requested + * bytes (CONFIRMED, 183fd50) + */ +}; + +/* Poll A/B bound: the kext polls <=1000 x sleep(1ms) (selene poll + * loop 0x73c4-0x73fc analog; rvbar-lifecycle step 6/10). + */ +#define ANE_BOOT_ACK_POLL_US 1000 +#define ANE_BOOT_POLL_MS 1000 + +/* The COMPLETE resolved legacy-boot sequence is implemented ONCE in + * ane_t6021_boot.h (ane_t6021_boot_run) against an io backend, so the + * fake-MMIO trace check exercises the exact code the device will run + * (Main: runnable trace check; implement the device wiring only after + * ALL sources close — until then this file performs no boot write). + * The kernel backend (readl/writeq/udelay wrappers + prepare hook + * doing the pool/IPC allocations and the sourced fill) is the small + * reviewed increment that lands with the gate flip. + */ + +/* ---- kernel io backend for ane_t6021_boot_run() ---- */ + +struct ane_t6021_boot_mmio { + struct ane_t6021 *ane; +}; + +static u32 ane_boot_rd32(void *ctx, unsigned int off) +{ + struct ane_t6021_boot_mmio *mm = ctx; + + return readl(mm->ane->base[ANE_T6021_REG_ENGINE] + off); +} + +static u64 ane_boot_rd64(void *ctx, unsigned int off) +{ + struct ane_t6021_boot_mmio *mm = ctx; + + return readq(mm->ane->base[ANE_T6021_REG_ENGINE] + off); +} + +static void ane_boot_wr32(void *ctx, unsigned int off, u32 v) +{ + struct ane_t6021_boot_mmio *mm = ctx; + + writel(v, mm->ane->base[ANE_T6021_REG_ENGINE] + off); +} + +static void ane_boot_wr64(void *ctx, unsigned int off, u64 v) +{ + struct ane_t6021_boot_mmio *mm = ctx; + + writeq(v, mm->ane->base[ANE_T6021_REG_ENGINE] + off); +} + +static void ane_boot_publish_barrier(void *ctx) +{ + (void)ctx; + /* dma_wmb() = dmb oshst on arm64: orders the coherent pool fill + * before the device publish. ORDERING guarantee — not the kext's + * dsb st (completion). Sufficient for the Linux coherent-DMA + + * writel doorbell contract (writel orders prior accesses before + * the MMIO store). + */ + dma_wmb(); +} + +static void ane_boot_wait(void *ctx) +{ + (void)ctx; + usleep_range(1000, 1500); /* kext poll: sleep(1000us) */ +} + +static void ane_boot_phase(void *ctx, const char *what) +{ + struct ane_t6021_boot_mmio *mm = ctx; + + /* bounded phase marker: one line per block boundary, survives + * netconsole for crash attribution (never per-poll). KERN_EMERG + * + a short drain so the line reaches tty0/netconsole/ssh + * BEFORE the risky write it announces (fw-start-debug + * 2026-09-22: fwstart#2 died with zero capture — the marker + * must beat the write). + */ + dev_emerg(mm->ane->dev, "BOOT-PHASE %s\n", what); + msleep(30); +} + +/* S5 prepare — runs strictly AFTER poll A (fw alive), BEFORE the + * SCRATCH0/1 publish. Dynamic allocations occur here per "dynamic + * allocations occur after READY" (Main): the 'DDM ' pool, the 'IPC ' + * surface, and the trust-bounded fw-requested HEAP surface. Every + * allocation is wedged-pin owned from here on (held while + * cpu_started; reboot reclaims). Publishes the pool DVA (suballoc at + * offset 0) as the SCRATCH0/1 halves. + */ +static void *ane_boot_alloc(void *ctx, u64 size, u64 *iova) +{ + struct ane_t6021 *ane = ctx; + dma_addr_t d = 0; + void *p = dma_alloc_coherent(ane->dev, size, &d, GFP_KERNEL); + + if (p && !ane_t6021_fw_alias_iova_ok(ane, d, size)) { + dev_err(ane->dev, + "boot alloc %#llx+%#llx overlaps fw alias — refusing\n", + (u64)d, size); + dma_free_coherent(ane->dev, size, p, d); + return NULL; + } + + *iova = p ? d : 0; + return p; +} + +/* Kernel prepare: live SCRATCH3/SCRATCH1 reads feed the SHARED + * assembly (ane_t6021_boot_prepare_publish) with the dma_alloc hook — + * the same path the fake-MMIO trace test exercises. Allocations are + * wedged-pin owned from creation (held while cpu_started). + */ +static int ane_t6021_boot_prepare(void *ctx, u32 *lo, u32 *hi) +{ + struct ane_t6021_boot_mmio *mm = ctx; + struct ane_t6021 *ane = mm->ane; + void __iomem *eng = ane->base[ANE_T6021_REG_ENGINE]; + struct ane_t6021_init_sources src = boot_sources; + struct ane_t6021_boot_allocs a; + u32 request, scratch0, scratch1; + int err; + + if (!ane_t6021_boot_preflight_complete()) + return -ENODATA; /* belt: run() already gated */ + + /* dynamic reads (post-READY, live cells — never hardcoded): + * read order SCRATCH0 then SCRATCH1 (0x95ea0d8/0x95ea100); + * SCRATCH0 >= 0x21 refuses before allocations/publication; + * SCRATCH3 = fw extra-heap request; SCRATCH1+1 = ordinal. + */ + scratch0 = readl(eng + ANE_MBI_SCRATCH0); + scratch1 = readl(eng + ANE_MBI_SCRATCH0 + 4); + request = readl(eng + ANE_MBI_SCRATCH0 + 4 * 3); + + src.fw_dva = ane->fw_iova; + err = ane_t6021_boot_prepare_publish(&src, request, scratch0, + scratch1, 0x4000, + ANE_T6021_BOOT_IPC_CEILING, + ANE_T6021_BOOT_HEAP_CEILING, + ane, ane_boot_alloc, &a, + lo, hi); + if (!err && boot_prevent_nap) { + ((u8 *)a.pool)[ANE_T6021_INIT_TEMPLATE_OFF + 0x84] |= 1; + dev_info(ane->dev, "LAB init resource[0x84] bit0=1: prevent nap\n"); + } + /* OWNERSHIP TRANSFER on EVERY path (Main lifetime review): an + * error after the pool/ipc allocations must NOT lose them — a + * started CPU may already be fetching, so partial allocations + * are RETAINED (wedged-pin holds them; reboot reclaims), never + * freed on this path. + */ + ane->boot_pool = a.pool; + ane->boot_pool_iova = a.pool_dva; + ane->boot_ipc = a.ipc; + ane->boot_ipc_iova = a.ipc_dva; + ane->boot_ipc_size = a.ipc_size; + ane->boot_heap = a.heap; + ane->boot_heap_iova = a.heap_dva; + ane->boot_heap_size = a.heap_size; + return err; +} + +/* Dispatch the resolved sequence against the kernel io backend. All + * gates were checked by the caller; the run() core re-checks. After + * the CPU release there is NO ordinary unwind: failures HOLD state + * (wedged-pin cleanup refuses to free under a started CPU) and the + * probe binds fenced. + */ +/* Poll-A-timeout progress dump — PROVEN-readable observables ONLY + * (Main review 2026-09-26): all eight SCRATCH cells (pulse-cleared to 0 + * pre-release, so any nonzero word with SCRATCH7 != READY/WAKE is a + * candidate firmware store), RVBAR, CPU_STATUS, and the 24 MHz domain + * tick at engine+0x1160008 (watched advancing post-release in the + * 2026-09-25 handshake receipt; a static tick separates "ASC domain + * unclocked" from "clocked but silent"). + * + * Deliberately NOT read: engine+0x1140008 (the 13.5 stub's VM 0x30c + * store target). Its readability, width and reset value are unproven + * and no pre-release baseline exists, so an after-value would not + * locate PC (could be preexisting iBoot state, write-only, or + * normalized). + */ +static void ane_t6021_boot_progress_dump(struct ane_t6021 *ane) +{ + void __iomem *eng = ane->base[ANE_T6021_REG_ENGINE]; + u32 tick0 = readl(eng + ANE_T6021_BOOT_REG_TICK); + u32 tick1; + unsigned int i; + + msleep(20); + tick1 = readl(eng + ANE_T6021_BOOT_REG_TICK); + + for (i = 0; i < 8; i++) + dev_emerg(ane->dev, "PROGRESS SCRATCH%u=%08x\n", i, + readl(eng + ANE_T6021_BOOT_REG_SCRATCH0 + 4 * i)); + dev_emerg(ane->dev, + "PROGRESS rvbar=%016llx cpu_status=%08x tick %08x->%08x (%s)\n", + readq(eng + ANE_ASC_RVBAR), + readl(eng + ANE_ASC_CPU_STATUS), + tick0, tick1, + tick1 != tick0 ? "24MHz domain clocked" : "tick STATIC"); +} + +int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, int rtb_mode) +{ + struct ane_t6021_boot_mmio mm = { .ane = ane }; + struct ane_t6021_boot_io io = { + .ctx = &mm, + .rd32 = ane_boot_rd32, .rd64 = ane_boot_rd64, + .wr32 = ane_boot_wr32, .wr64 = ane_boot_wr64, + .publish_barrier = ane_boot_publish_barrier, .poll_wait = ane_boot_wait, + .phase = ane_boot_phase, + .prepare = ane_t6021_boot_prepare, + }; + struct ane_t6021_boot_cfg cfg = { + .preflight_ok = ane_t6021_boot_preflight_complete(), + .preboot_table_mode = table_mode, + .fw_dva = ane->fw_iova, + .stop_after = stop_after, + .rtb_mode = rtb_mode, + }; + int cs = 0, fa = 0, bo = 0; + u64 sres = 0; + int r; + + /* Wedged-pin module lifetime (Main lifetime review): the ref is + * acquired BEFORE the first write — a started CPU can never + * outlive the pin, and a dying module refuses the boot before + * any write happens. Retained on CPU start (never released: + * intentional); released only if no CPU start occurred. + */ + if (!try_module_get(THIS_MODULE)) { + dev_err(ane->dev, + "boot: REFUSED before any write — module ref unavailable (dying); no CPU start possible from a dying module\n"); + return -EBUSY; + } + + dev_emerg(ane->dev, + "BOOT-PHASE dispatch (stop_after=%d%s)\n", stop_after, + stop_after ? " BISECT STOP ARMED" : ""); + + r = ane_t6021_boot_run(&io, &cfg, &cs, &fa, &bo, &sres); + + ane->cpu_started = cs; + ane->fw_alive = fa; + ane->booted = bo; + ane->boot_scratch_result = sres; + + /* Released CPU without READY (poll-A timeout, either transport + * mode): capture the 13.5 execution-progress observables NOW, + * while the islands are powered — this is the exact-progress + * datum the next pre-Linux capture cross-checks. + */ + if (cs && !fa) + ane_t6021_boot_progress_dump(ane); + + if (!cs) { + /* no CPU start: full release path, normal ownership */ + module_put(THIS_MODULE); + dev_emerg(ane->dev, + "BOOT-PHASE done r=%d cpu_started=0 (no CPU release: state clean, module unpinned)\n", + r); + return r; + } + + /* started CPU: retain the pin for the whole wedged lifetime. + * Residual: DT hotplug unbind cannot be fully prevented; devm + * release order frees irq before ioremap (probe-order reverse); + * DMA surfaces are wedge-held, never freed. + */ + dev_warn(ane->dev, + "boot: module PINNED until reboot (started CPU; wedged-pin)\n"); + + if (r == -ENODATA) + return r; /* unreachable: the caller gated */ + if (r && cs) { + dev_err(ane->dev, + "boot: sequence error %d AFTER CPU start (cpu_started=%u fw_alive=%u booted=%u scratch_result=%016llx) — WEDGED-PIN HOLD: all surfaces/rings/IRQ/links preserved; reboot is the only reclamation; no retry. HANDSHAKE state only: the result word has NO sourced success semantics\n", + r, cs, fa, bo, sres); + return 0; /* bind fenced, state held */ + } + if (!r) + dev_info(ane->dev, + "boot: run returned 0 (booted=%u fw_alive=%u) — %s; scratch_result=%016llx (raw device address, semantics UNSOURCED — not a success claim; transport stays fenced until response validation)\n", + bo, fa, + bo ? "DONE observed — handshake complete" + : "rc 0 WITHOUT DONE (rtb_mode: HELLO-gated; DONE not part of this mode)", + sres); + return r; +} + diff --git a/drivers/accel/ane/ane_t6021_boot.h b/drivers/accel/ane/ane_t6021_boot.h new file mode 100644 index 0000000000000..ced0c997439dd --- /dev/null +++ b/drivers/accel/ane/ane_t6021_boot.h @@ -0,0 +1,832 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * ane_t6021_boot.h — T6021 legacy-boot contract units (pure, shared by + * the kernel sequencer and the userspace regression). + * + * Byte anchors (KC kernelcache.release.mac14j sha256 8304156f…, selene + * sha256 9f7915c4…; receipts under ane-linux-experiments/receipts/, + * commits 3762aee + 12be074): + * - RVBAR fold: ANEHWDevice::ANE_Init 0x95e9850–0x95e9988 (K13 + * 0x93203b0–0x93204e0 identical). Read64 of eng+0x01050000; bit0 + * set skips the write (live jw14m2 read 0x1). Otherwise write64 + * ENTRY_BASE | (fw DVA & ADDR_MASK) — single u64 store + * (2026-09-20-h14-rvbar-width.json width_proof). + * - fw DVA = ANESharedMemorySurfaceParams("FWIM")+0x18, produced by + * dartMapMemoryDescriptor (vtable slot 0x8a8, genIOVMSegments) for + * the out-param installed at start+0x336c (mapper-callchain audit + * pass1–3). Linux correspondence: dma_alloc_coherent on the ane + * platform device is dart-ane0-translated (live iommu group 6, + * three apple,t6020-dart streams, DMA domain). + * - SCRATCH0/1 u64 split: low32 → 0x01840048 first, high32 → + * 0x0184004c, dsb st before (legacy-init-publication receipt, + * accessor write32 width-proven at 0x9622d4c). + * - 0x174 init suballocation: header[0x00] u64 = fw DVA (legacy + * branch, dev+0x780 bit0 clear = observed AppleARMIODevice + * provider), [0x68] u32 = 64 count, template 256 B at + * [0x6C,0x16C) with template+0xC0 = 4 RESOLVED (Main raw anchors + * 0x9612b78/7c/80 on the dev+0x998 template; conditional |=0x10 + * separate, unqualified, not set). + * + * This header intentionally includes nothing. Kernel consumers have + * already; userspace consumers typedef u8/u32/u64 and + * include first (tools/h14_boot_regression.c in this + * repository is the shipped runnable check). + */ +#ifndef __ANE_T6021_BOOT_H__ +#define __ANE_T6021_BOOT_H__ + +/* RVBAR entry fold (ANE_Init 0x95e9868–0x95e988c). The mask clears + * bits 0–10, 48 and 55 — bit 11 IS retained (0xf800 keeps 0x800); the + * base supplies bit0 and the fixed upper pattern. + */ +#define ANE_T6021_RVBAR_ENTRY_BASE 0x0081000000000001ULL +#define ANE_T6021_RVBAR_ADDR_MASK 0xff7efffffffff800ULL + +/* CPU_CONTROL release (rvbar-width local order: write32 0 then 0x10; + * h14g config selects eng+0x01400044). RUN = BIT(4). + */ +#define ANE_T6021_CPU_RUN_RELEASE 0x10 + +/* SCRATCH7 handshake words (InitializeRTBuddy / legacy ANE_Init + * publication; selene ack writer 0x77c8). + */ +/* SCRATCH0 request band (pass9-corrected, Main raw trace): post-READY + * SCRATCH0 >= 0x21 REFUSES the boot preparation; below 0x21 proceeds + * to the MAX/alloc path. + */ +#define ANE_T6021_BOOT_IPC_REQ_THRESHOLD 0x21U + +#define ANE_T6021_BOOT_WAKE_REQ 0xf7fbdff9U +#define ANE_T6021_BOOT_ACK 0x08042006U + +/* Init suballocation geometry (ANE_Init 0x95ea710–0x95ea97c; + * legacy-init-publication receipt). + */ +#define ANE_T6021_INIT_STRUCT_SIZE 0x174 +#define ANE_T6021_INIT_FW_DVA_OFF 0x00 /* u64, legacy branch */ +#define ANE_T6021_INIT_COUNT_OFF 0x68 /* u32 = 64 */ +#define ANE_T6021_INIT_COUNT 0x40 +#define ANE_T6021_INIT_TEMPLATE_OFF 0x6c /* 256 B at [0x6C,0x16C) */ +#define ANE_T6021_INIT_TEMPLATE_SIZE 0x100 +#define ANE_T6021_INIT_TBIT_OFF 0xc0 /* template-relative */ +#define ANE_T6021_INIT_TBIT_VAL 0x4 /* RESOLVED (Main raw + * anchors 0x9612b78/7c/80: + * ldr/orr#4/str on the dev+0x998 + * template); the conditional + * |=0x10 is a separate, still + * unqualified mutation and is + * NOT set + */ + +static inline u64 ane_t6021_rvbar_compose(u64 iova) +{ + return ANE_T6021_RVBAR_ENTRY_BASE | (iova & ANE_T6021_RVBAR_ADDR_MASK); +} + +/* The kext skip condition: bit0 of the RVBAR read (tbnz w0,#0 at + * ANE_Init 0x95e9878; live jw14m2 reads have been 0x1). What a set + * bit names in boot-mode terms is the open lifecycle fork — see + * ane_t6021_boot.c; this predicate only decodes the bit. + */ +static inline bool ane_t6021_rvbar_latched(u64 rd) +{ + return rd & 1; +} + +/* Entry bits of an RVBAR read (what the fold would have named). */ +static inline u64 ane_t6021_rvbar_entry_bits(u64 rd) +{ + return rd & ANE_T6021_RVBAR_ADDR_MASK; +} + +/* Boot acceptance predicate — the EXACT check the boot path applies + * to a staged fw DVA: the fold must lose nothing. Bit 11 is retained; + * bits 0-10, 48 and 55 are not, so an iova with any of them set is + * rejected rather than silently truncated. + */ +static inline bool ane_t6021_rvbar_entry_ok(u64 iova) +{ + return (iova & (u64)~ANE_T6021_RVBAR_ADDR_MASK) == 0; +} + +/* Publication convention: low32 → SCRATCH0 (0x01840048) first, then + * high32 → SCRATCH1, after dsb st. + */ +static inline void ane_t6021_scratch64_split(u64 v, u32 *lo, u32 *hi) +{ + *lo = (u32)(v & 0xffffffffU); + *hi = (u32)(v >> 32); +} + +static inline u64 ane_t6021_scratch64_join(u32 lo, u32 hi) +{ + return ((u64)hi << 32) | lo; +} + +/* Linux sources for the init-suballocation fields (pass5b/5c/5d + * b8b7c35 + pass6c 75a017b; validator tools/check_init_contract.py + * 68/68). EVERY fw-read field is explicitly sourced from the struct + * below — nothing is silently zero: [0x00] FWIM DVA, [0x08] 'IPC ' + * surface DVA (Linux alloc, size = ane_t6021_ipc_size), [0x10] + * config+0x138 = 0x500000, [0x18] = 0x10000000 - size, [0x20] obj2 + * DVA, [0x28] w22, [0x30] fw-load progress, [0x50] zext [x23+4], + * [0x58] pool DMA base, [0x60] pool word0, [0x68] count 64, + * template[0x00] = 0, template+0xC0 = 4. The struct values pin BEFORE + * the preflight gate can open (Main: no unresolved field silently + * zero; no partial boot). The caller MUST zero the full 0x174 block + * first (coherent alloc): gaps [0x34..37]/[0x64..67] get no kext + * store and sit inside fw-read u64 units. + */ +/* Init-suballocation contract — pass6 producer semantics + * (rvbar-lifecycle-evidence.json pass6_init_header_producers, commit + * cd25b46, anchors G1-G26; 87/87 total): + * + * STATIC sources (pinned before the preflight can open): + * fw_dva -> [0x00] staged selene ('FWIM') surface DVA, + * ipc_dva -> [0x08] 'IPC ' surface DVA (its size calc remains + * the audit lane's open item), + * cfg_size -> [0x10] config+0x138 = 0x500000; [0x18] = + * 0x10000000 - cfg_size = 0x0fb00000, + * prev_fw_len -> [0x30] previous fw image length (0 on first boot; + * driver-static, updated on each reload), + * heap_floor -> [0x28] floor = config field dev+0x3A90 (value + * pinned by the audit lane at preflight close), + * pool_dma -> [0x58] the 'DDM ' pool surface DVA (Linux allocates + * its own 0x40000-byte DDM pool — the kext size), + * pool_word0 -> [0x60] first qword of the Linux pool-descriptor + * struct. NOT CLOSED: the fw-side consumer of + * pool+0x00 is untraced and the kext never stores it + * (free-list head at manager+0 untouched by + * Allocate). This value is a HARD-GATED source — the + * sequence cannot run until a producer/consumer proof + * pins it; no hypothesis value is shipped. + * PRODUCER OPEN (Main, superseding b21e23f): the + * "pool+0x00 = manager free-list head" claim is + * REJECTED — manager dev+0x968 != Params pool + * dev+0x980 (wrong-object alias likely), and + * Reset6cb048d shows the consumer stores word0 + * VERBATIM at image ops 68 (possible extent use). + * Wait for the exact same-Params producer. + * + * DYNAMIC inputs (read AFTER poll A — the fw publishes its extra-heap + * request into SCRATCH3 (cell idx3) and its boot ordinal is + * SCRATCH1+1; both read from the live cells, never hardcoded): + * scratch3_req -> the fw extra-heap request; 0 = no heap surface, + * scratch1_read -> SCRATCH1 read; [0x50] = (u32)read + 1 (boot + * ordinal). + * + * TRUST BOUNDARY: scratch3_req is firmware-supplied. The heap size + * must pass ane_t6021_heap_size() bounds BEFORE any allocation; the + * heap DVA and size are written to [0x20]/[0x28] only on success. + */ +struct ane_t6021_init_sources { + u64 fw_dva; + u64 ipc_dva; + u32 cfg_size; + u32 prev_fw_len; + u64 heap_floor; + u64 pool_dma; + u64 pool_word0; +}; + +/* 'IPC ' surface size (CURRENT_CONTRACT f9602e0; operands proven: + * u64[dev+0x3A90] = IODARTMapper::getPageSize() — 0x4000, one 16K + * DART page, typical; u32[x23+4] = captured SCRATCH1 + 1, the boot + * ordinal): size = max(page_size, ordinal + 1). Used by the kernel + * prepare hook to size the 'IPC ' allocation. + */ +/* SCRATCH0 read-order note (pass9 phase-1: SCRATCH0 then SCRATCH1 + * after READY): Main raw branch proof SUPERSEDES the audit's + * "SCRATCH0 >= 0x21 arms IPC" reading — 0x95ea1dc b.cc(low) branches + * TO the MAX+IPC-alloc path and >=0x21 goes to logging/error, so + * scratch0 < 33 is likely a VALIDITY gate, not an optional-IPC + * condition. Header [0x08] unconditionally dereferences + * dev+0x988+0x18. Allocation stays UNCONDITIONAL until Reset corrects + * the source; the threshold is NOT implemented. + */ +static inline u64 ane_t6021_ipc_size(u64 page_size, u32 captured) +{ + /* EXACT u32 semantics: the fw-visible ordinal is the captured + * SCRATCH1 value + 1 in 32-bit arithmetic (kext add w8,w0,#1) — + * it wraps. Callers pass the CAPTURED value, never pre-incremented. + * The result is then bounded by ANE_T6021_BOOT_IPC_CEILING at the + * allocation site: captured is fw-supplied and untrusted. + */ + u32 ord = captured + 1; + + return page_size > (u64)ord ? page_size : (u64)ord; +} + +/* Operational ceiling for the fw-influenced 'IPC ' allocation + * (captured is untrusted; the max() can otherwise be forced to an + * arbitrary size). Distinct from ABI width — documented budget. + */ +#define ANE_T6021_BOOT_IPC_CEILING 0x100000ULL /* 1 MiB */ + +/* Trust boundary for the firmware-supplied heap request: 0 disables + * the heap surface; otherwise the size is MAX(request, floor) and is + * refused (negative) when the request exceeds the ceiling. + * + * ANE_T6021_BOOT_HEAP_CEILING is an OPERATIONAL RESOURCE BUDGET, kept + * distinct from ABI width: header [0x28] is a u64 field and can + * express far larger values — the u64-ness says nothing about what is + * safe to allocate. The dart-ane0 vm window only bounds ADDRESSABILITY, + * not allocation safety, so it is deliberately not used as the + * ceiling. 32 MiB is the documented budget bounding a firmware-supplied + * allocation to a size the host can honor without starving; re-anchor + * to a config field when the h14g blob map lands. + */ +#define ANE_T6021_BOOT_HEAP_CEILING 0x02000000ULL /* 32 MiB budget */ + +static inline long long +ane_t6021_heap_size(u32 scratch3_req, u64 floor, u64 max_size) +{ + u64 req = scratch3_req; + + if (!req) + return 0; + if (req > max_size) + return -E2BIG; + return (long long)(req > floor ? req : floor); +} + +/* Fill the COMPLETE init suballocation from the static sources plus + * the post-READY dynamic values. The caller MUST zero the full 0x174 + * block first (coherent alloc): the gap bytes [0x34..37]/[0x64..67] + * get no kext store and sit inside fw-read u64 units. + */ +static inline void +ane_t6021_init_struct_fill(u8 *buf, const struct ane_t6021_init_sources *s, + long long heap_size, u64 heap_dva, + u32 boot_ordinal) +{ + u64 size = s->cfg_size; + int i; + + for (i = 0; i < 8; i++) { + buf[ANE_T6021_INIT_FW_DVA_OFF + i] = + (u8)(s->fw_dva >> (8 * i)); + buf[0x08 + i] = (u8)(s->ipc_dva >> (8 * i)); + buf[0x10 + i] = (u8)(size >> (8 * i)); + buf[0x18 + i] = (u8)((0x10000000ULL - size) >> (8 * i)); + buf[0x20 + i] = (u8)(heap_dva >> (8 * i)); + buf[0x50 + i] = (u8)((u64)boot_ordinal >> (8 * i)); + buf[0x58 + i] = (u8)(s->pool_dma >> (8 * i)); + buf[0x60 + i] = (u8)(s->pool_word0 >> (8 * i)); + } + + /* [0x28] u64 heap size (0 when the fw requested none) */ + for (i = 0; i < 8; i++) + buf[0x28 + i] = (u8)((u64)heap_size >> (8 * i)); + + /* [0x30] u32 FULL width: previous fw image length (0 first + * boot). Main review: low-byte-only serialization dropped the + * upper length bits. + */ + buf[0x30] = (u8)s->prev_fw_len; + buf[0x31] = (u8)(s->prev_fw_len >> 8); + buf[0x32] = (u8)(s->prev_fw_len >> 16); + buf[0x33] = (u8)(s->prev_fw_len >> 24); + + buf[ANE_T6021_INIT_COUNT_OFF + 0] = (u8)ANE_T6021_INIT_COUNT; + buf[ANE_T6021_INIT_COUNT_OFF + 1] = 0; + buf[ANE_T6021_INIT_COUNT_OFF + 2] = 0; + buf[ANE_T6021_INIT_COUNT_OFF + 3] = 0; + + /* template[0x00] = config word [dev+0x1D8], pinned value 0 + * (pass5d); [0x04..0x0B] zero via the caller's zeroing. + */ + buf[ANE_T6021_INIT_TEMPLATE_OFF + 0] = 0; + buf[ANE_T6021_INIT_TEMPLATE_OFF + 1] = 0; + buf[ANE_T6021_INIT_TEMPLATE_OFF + 2] = 0; + buf[ANE_T6021_INIT_TEMPLATE_OFF + 3] = 0; + + /* template+0xC0 = 4: RESOLVED initial state (Main raw anchors + * 0x9612b78/7c/80). The conditional |=0x10 remains unset. + */ + buf[ANE_T6021_INIT_TEMPLATE_OFF + ANE_T6021_INIT_TBIT_OFF + 0] = + ANE_T6021_INIT_TBIT_VAL; +} + +/* ---- Shared prepare-source assembly ---- + * + * The ALLOCATION-SIZING + HEADER-FILL + PUBLISH-VALUE path used by the + * kernel prepare hook AND the fake-MMIO trace test — one + * implementation, so wrong-DVA / double-increment / unbounded-size + * classes fail in the host trace test instead of only on hardware. + * + * Allocation contract: alloc(ctx, size, &iova) is called in the order + * pool (0x40000), 'IPC ' (max(DART page 0x4000, ordinal+1)), HEAP + * (0 when the fw requested none); each must return NULL on failure + * and set *iova to the device-visible address of the returned memory + * (the DMA address, NOT a recorded constant). Refusal classes: the + * fw-supplied request and captured value are UNTRUSTED — over-ceiling + * heap requests and over-budget IPC sizes are refused BEFORE any + * allocation. + */ +struct ane_t6021_boot_allocs { + u64 pool_dva, ipc_dva, heap_dva; + u64 ipc_size, heap_size; + void *pool, *ipc, *heap; +}; + +static inline int +ane_t6021_boot_prepare_publish(const struct ane_t6021_init_sources *s, + u32 scratch3_req, u32 scratch0_read, + u32 scratch1_read, + u64 page_size, u64 ipc_ceiling, + u64 heap_ceiling, + void *alloc_ctx, + void *(*alloc)(void *ctx, u64 size, + u64 *iova), + struct ane_t6021_boot_allocs *a, + u32 *lo, u32 *hi) +{ + int i; + + for (i = 0; i < (int)(sizeof(*a) / sizeof(u64)); i++) + ((u64 *)a)[i] = 0; + + /* SCRATCH0 band check — REFUSAL before allocations/publication + * (Main raw trace: >= 0x21 -> 0x95ea2a0 log -> 0x2c4 -> 0x3330 + * error, w21 = 0xE00002DA; < 0x21 -> 0x214 MAX -> 0x25c alloc -> + * 0x260 -> 0x2c8). Boundary: 0x20 accepted, 0x21 refused. + * Read order: SCRATCH0 first, then SCRATCH1 (0x95ea0d8/0x95ea100). + * Untrusted-input sizing after the band check; heap_size() + * returns SIGNED (negative = refused) — kept local so a refusal + * cannot wrap into the u64 record. + */ + if (scratch0_read >= ANE_T6021_BOOT_IPC_REQ_THRESHOLD) + return -EPROTO; + a->ipc_size = ane_t6021_ipc_size(page_size, scratch1_read); + if (a->ipc_size > ipc_ceiling) + return -E2BIG; + { + long long heap = ane_t6021_heap_size(scratch3_req, + s->heap_floor, + heap_ceiling); + + if (heap < 0) + return (int)heap; + a->heap_size = (u64)heap; + } + + a->pool = alloc(alloc_ctx, 0x40000, &a->pool_dva); + if (!a->pool) + return -ENOMEM; + a->ipc = alloc(alloc_ctx, a->ipc_size, &a->ipc_dva); + if (!a->ipc) + return -ENOMEM; + if (a->heap_size) { + a->heap = alloc(alloc_ctx, a->heap_size, &a->heap_dva); + if (!a->heap) + return -ENOMEM; + } + + /* ordinal: exact u32 +1 on the READ SCRATCH1 value (kext add + * w8,w0,#1; callers pass the read value, never pre-incremented). + * HEADER SOURCES come from the ALLOCATIONS: s->ipc_dva/pool_dma + * may be zero in the caller's static sources — fill from the + * allocator outputs (a->ipc_dva/a->pool_dva), never from s + * (Main review: zero-DVA header bug). + */ + { + struct ane_t6021_init_sources filled = *s; + + filled.ipc_dva = a->ipc_dva; + filled.pool_dma = a->pool_dva; + ane_t6021_init_struct_fill(a->pool, &filled, a->heap_size, + a->heap_dva, + (u32)(scratch1_read + 1)); + } + *lo = (u32)(a->pool_dva & 0xffffffffU); + *hi = (u32)(a->pool_dva >> 32); + return 0; +} + +/* Post-DONE SCRATCH1:SCRATCH0 (read SCRATCH1 first, then SCRATCH0) + * is a fw-produced DEVICE ADDRESS (pass9 phase-2), NOT a return code — + * never dereference it raw. WHITELIST (Main 2026-09-20): only the + * Linux-owned FWIM and 'IPC ' windows may ever be matched against it; + * the 'DDM ' pool is NOT whitelisted. No consumption until range AND + * length checks are proven; until then the raw u64 is stored and + * logged only. + */ + +/* ---- Fake-MMIO-testable sequence core ---- + * + * Register offsets (ENGINE-relative; the Linux driver maps the engine + * window, so kernel offsets == trace offsets) and the io backend. + * ane_t6021_boot_run() executes the resolved total order against ANY + * backend: the kernel wraps readl/writeq/udelay; the host regression + * wraps a recording fake (trace asserts: full order, no writes when + * the preflight gate is closed, publish strictly after poll A). + */ + +#define ANE_T6021_BOOT_REG_TABLE0 0x00000b38 +#define ANE_T6021_BOOT_REG_TABLE1 0x00000b98 +#define ANE_T6021_BOOT_REG_TABLE2 0x00000bf8 +#define ANE_T6021_BOOT_REG_RVBAR 0x01050000 +#define ANE_T6021_BOOT_REG_CPUCTRL 0x01400044 +#define ANE_T6021_BOOT_REG_SCRATCH0 0x01840048 +#define ANE_T6021_BOOT_REG_SCRATCH1 0x0184004c +#define ANE_T6021_BOOT_REG_SCRATCH6 0x01840060 +#define ANE_T6021_BOOT_REG_SCRATCH7 0x01840064 +/* 13.5 stub execution-progress observables (M2StartupRecovery 2026-09-26 + * decode of a9c4b771294a6b11…). Only PROVEN-readable engine-window + * registers: the 24 MHz domain tick at VM 0x1160008 (watched advancing + * ~24 MHz post-release, 2026-09-25 handshake receipt) plus SCRATCH, + * RVBAR and CPU_STATUS already on the whitelist. The stub's VM 0x30c + * store to engine+0x1140008 is real, but the register's readability, + * width and reset state are UNPROVEN and no pre-release baseline exists + * — it is deliberately NOT read here (Main review 2026-09-26). + */ +#define ANE_T6021_BOOT_REG_TICK 0x01160008 +#define ANE_T6021_BOOT_TABLE_VALUE 0x01ff01ffU +#define ANE_T6021_BOOT_TABLE_POLLS 1000 + +struct ane_t6021_boot_io { + void *ctx; + u32 (*rd32)(void *ctx, unsigned int off); + u64 (*rd64)(void *ctx, unsigned int off); + void (*wr32)(void *ctx, unsigned int off, u32 v); + void (*wr64)(void *ctx, unsigned int off, u64 v); + void (*publish_barrier)(void *ctx); /* orders the coherent fill + * before the device publish: + * dma_wmb() = dmb oshst on + * arm64; NOT claimed as dsb st + * (completion) + */ + void (*phase)(void *ctx, const char *what); /* bounded phase log: + * emitted ONCE before each + * write/poll block so a crash + * source survives netconsole + * (never per-poll) + */ + void (*poll_wait)(void *ctx); /* 1 ms poll delay */ + /* prepare(): called once, strictly AFTER poll A and BEFORE the + * SCRATCH0/1 publish; returns the suballoc DVA halves. Kernel + * backend: allocate pool/IPC + fill from pinned sources. + */ + int (*prepare)(void *ctx, u32 *lo, u32 *hi); +}; + +struct ane_t6021_boot_cfg { + int preflight_ok; /* EVERY prerequisite closed (Main: no + * partial boot — the whole sequence or + * nothing) + */ + int preboot_table_mode; /* LIVE-FAULT gating of the pre-CPU table + * block (2026-09-20 wedge): + * 0 = ABORT: -EAGAIN, zero writes + * (accidental-repeat prevention), + * 1 = WRITE the table block (re-armed + * after the table-base analysis), + * 2 = SKIP the table block, run the rest + * (diagnostic: tests fw-alive without + * the kext pre-CPU config). + */ + u64 fw_dva; /* staged surface DVA (fold input) */ + int rtb_mode; /* 1 = S1 writes SCRATCH6=0 (RTBuddy/RTKit-app- + * endpoint select, fw 0x42c8) instead of 1 + * (legacy ChMan/MBI). In RTBuddy mode the fw + * may skip READY/DONE; listen on mailbox. + * Run contract: P4 READY is recorded, not + * required; no legacy publish/wake runs; + * aliveness is gated by the RTKit HELLO. + */ + int stop_after; /* fw-start-debug step bisect (2026-09-22): + * 0 = full run; N in 1..4 = stop AFTER + * step N completes, return -ECANCELED + * (1 grant tunables, 2 scratch, 3 rvbar, + * 4 cpu release + poll A). A poll-A + * timeout inside step 4 still returns + * -ETIMEDOUT — the timeout is the + * answer. Steps complete = writes done; + * the stop never splits a step. + */ +}; + +/* Ownership: a started CPU may be fetching from the staged surfaces — + * the DMA memory is NOT reclaimable on failure/remove while + * cpu_started; reclaimable only via the domain-off reset (reboot). + */ +static inline int ane_t6021_boot_dma_reclaimable(int cpu_started) +{ + return !cpu_started; +} + +/* The cleanup decision the predicate implements: while cpu_started, + * remove() is HELD (H13 wedged-pin pattern — surfaces, rings, IRQ and + * power-domain links preserved; reboot reclaims). + */ +static inline int ane_t6021_boot_remove_held(int cpu_started) +{ + return cpu_started; +} + +/* Returns 0 (DONE observed, booted), -ENODATA (preflight closed: ZERO + * io writes), or -ETIMEDOUT (poll A/B: cpu_started holds, DMA stays + * unreclaimable, no publish/wake happened on poll A timeout). + */ +static inline int +ane_t6021_boot_run(const struct ane_t6021_boot_io *io, + const struct ane_t6021_boot_cfg *cfg, + int *cpu_started, int *fw_alive, int *booted, + u64 *scratch_result) +{ + unsigned int i; + u32 v; + u64 rvbar; + + *cpu_started = 0; + *fw_alive = 0; + *booted = 0; + *scratch_result = 0; + + /* Main: all gates checked BEFORE any boot write — no partial + * sequence when the preflight is closed. + */ + if (!cfg->preflight_ok) + return -ENODATA; + + /* pre-CPU engine table (pass4 receiver, pass5 gate: REQUIRED + * every EnableANEClocksAndPower; residual alias/indirect writer + * risk documented in the preflight list). + * LIVE-FAULT 2026-09-20: this block wedged the machine (P0 then + * silence — stall could be the FIRST write; P1 never reached; + * hardware reset 16:24:08) — pf_preboot_table_safe gates it + * OFF; nothing below runs until re-armed. + */ + switch (cfg->preboot_table_mode) { + case 0: + return -EAGAIN; /* BEFORE any write: accidental-repeat + * prevention (Main 2026-09-20) + */ + case 1: + io->phase(io->ctx, "P0 preboot-table"); + io->phase(io->ctx, "P0-1 eng+0xb38"); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_TABLE0, + ANE_T6021_BOOT_TABLE_VALUE); + io->phase(io->ctx, "P0-2 eng+0xb98"); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_TABLE1, + ANE_T6021_BOOT_TABLE_VALUE); + io->phase(io->ctx, "P0-3 eng+0xbf8"); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_TABLE2, + ANE_T6021_BOOT_TABLE_VALUE); + break; + case 2: + default: + /* diagnostic skip: NO table writes — tests fw-alive + * without the kext pre-CPU config (user override + * 2026-09-20). + */ + io->phase(io->ctx, "P0 table SKIPPED (diagnostic)"); + break; + } + + /* P-1: W8 write-grant tunables (proven no-abort class, + * w8-run.out: APERTURE_UNLOCKED — 12 engine-aperture writes + * preparing the aperture; replayed verbatim). + * Per-write before/after discrimination (Main: attempt 3 stalled + * LAST phase = P-1, possibly FIRST tunable write). + */ + { + static const struct { u32 off; u32 val; } tun[] = { + { 0x000, 0x00000010 }, { 0x038, 0x00050020 }, + { 0x03c, 0x000a0030 }, { 0x400, 0x40010001 }, + { 0x600, 0x01ffffff }, { 0x738, 0x00200020 }, + { 0x798, 0x00100030 }, { 0x7f8, 0x0100000a }, + { 0x900, 0x00000101 }, { 0x410, 0x00001100 }, + { 0x420, 0x00001100 }, { 0x430, 0x00001100 }, + }; + unsigned int ti; + + io->phase(io->ctx, "P-1 grant-tunables begin"); + for (ti = 0; ti < sizeof(tun) / sizeof(tun[0]); ti++) { + io->phase(io->ctx, tun[ti].off == 0x000 ? + "P-1a eng+0x000" : + tun[ti].off == 0x038 ? "P-1b eng+0x038" : + tun[ti].off == 0x03c ? "P-1c eng+0x03c" : + tun[ti].off == 0x400 ? "P-1d eng+0x400" : + tun[ti].off == 0x600 ? "P-1e eng+0x600" : + tun[ti].off == 0x738 ? "P-1f eng+0x738" : + tun[ti].off == 0x798 ? "P-1g eng+0x798" : + tun[ti].off == 0x7f8 ? "P-1h eng+0x7f8" : + tun[ti].off == 0x900 ? "P-1i eng+0x900" : + tun[ti].off == 0x410 ? "P-1j eng+0x410" : + tun[ti].off == 0x420 ? "P-1k eng+0x420" : + "P-1l eng+0x430"); + io->wr32(io->ctx, tun[ti].off, tun[ti].val); + io->phase(io->ctx, "P-1 write done"); + } + io->phase(io->ctx, "P-1 grant-tunables end"); + } + if (cfg->stop_after == 1) + return -ECANCELED; /* tunables done, nothing else fired */ + + io->phase(io->ctx, "P1 scratch-clear+pulse"); + /* S1: InitANEScratchRegisters — clear ALL cells, SCRATCH6 = rtb_mode ? 0 (RTBuddy) : 1 + * (legacy ChMan/MBI); pulse SCRATCH7 1 -> 0 (stale cleared). + */ + for (i = 0; i < 8; i++) + io->wr32(io->ctx, + ANE_T6021_BOOT_REG_SCRATCH0 + 4 * i, 0); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH6, + cfg->rtb_mode ? 0 : 1); + io->phase(io->ctx, cfg->rtb_mode ? + "P1 S1 SCRATCH6=0 (RTBuddy select)" : + "P1 S1 SCRATCH6=1 (legacy ChMan/MBI select)"); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH7, 1); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH7, 0); + if (cfg->stop_after == 2) + return -ECANCELED; /* scratch programmed, CPU untouched */ + + io->phase(io->ctx, "P2 rvbar"); + /* S2: RVBAR skip-or-fold (bit0 set = lawful skip branch; no + * latch override, no reset before first attempt). + */ + rvbar = io->rd64(io->ctx, ANE_T6021_BOOT_REG_RVBAR); + if (!ane_t6021_rvbar_latched(rvbar)) + io->wr64(io->ctx, ANE_T6021_BOOT_REG_RVBAR, + ane_t6021_rvbar_compose(cfg->fw_dva)); + if (cfg->stop_after == 3) + return -ECANCELED; /* entry decision recorded, no RUN */ + + io->phase(io->ctx, "P3 cpu-release"); + /* S3: CPU release — both paths, strictly 0 then 0x10. */ + io->wr32(io->ctx, ANE_T6021_BOOT_REG_CPUCTRL, 0); + io->phase(io->ctx, "P3b cpu-release RUN=0x10"); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_CPUCTRL, + ANE_T6021_CPU_RUN_RELEASE); + *cpu_started = 1; + + io->phase(io->ctx, "P4 pollA-READY"); + /* S4: poll A — FRESH READY (the pulse made it unambiguous). + * RTBuddy mode (SCRATCH6=0, kext fw 0x42c8: rtbuddyFW when + * SCRATCH6 == 0): the fw does not publish BOOT_ACK to SCRATCH7 + * and no legacy publication follows — aliveness is gated by the + * RTKit HELLO after boot_run returns (selene InitializeRTBuddy + * 0x95ead04). A missing READY is recorded, not fatal, and + * *fw_alive stays 0 — never fabricated. + */ + for (i = 0; i < ANE_T6021_BOOT_TABLE_POLLS; i++) { + v = io->rd32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH7); + if (v == ANE_T6021_BOOT_ACK) + break; + io->poll_wait(io->ctx); + } + if (v == ANE_T6021_BOOT_ACK) { + *fw_alive = 1; + io->phase(io->ctx, "P4 pollA READY observed"); + } else if (cfg->rtb_mode) { + io->phase(io->ctx, + "P4 pollA no READY (RTBuddy: HELLO-gated)"); + return 0; /* RTBuddy: no legacy publish/wake */ + } else { + return -ETIMEDOUT; + } + if (cfg->stop_after == 4) + return -ECANCELED; /* fw alive, publish/wake withheld */ + if (cfg->rtb_mode) + return 0; /* RTBuddy: no legacy publish/wake */ + + io->phase(io->ctx, "P5 prepare+publish"); + /* S5-S6: prepare (alloc/fill; kernel backend owns DMA), then the + * publish barrier, then publish suballoc DVA low32/high32, then + * wake. Barrier semantics (Main review 2026-09-20): the kernel + * backend's dma_wmb() is dmb oshst — it ORDERS the coherent fill + * before the device publish; it is NOT the kext's dsb st (which + * additionally waits for completion). Ordering suffices for the + * Linux coherent-DMA + writel doorbell contract (writel itself + * orders prior accesses before the MMIO store); the kext dsb st + * choice is a completion guarantee this driver does not claim. + */ + { + u32 lo = 0, hi = 0; + int err = io->prepare(io->ctx, &lo, &hi); + + if (err) + return err; + io->publish_barrier(io->ctx); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH0, lo); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH1, hi); + } + io->phase(io->ctx, "P6 wake"); + io->wr32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH7, + ANE_T6021_BOOT_WAKE_REQ); + + io->phase(io->ctx, "P7 pollB-DONE"); + /* S7: poll B — DONE; read back the SCRATCH0/1 result u64. */ + for (i = 0; i < ANE_T6021_BOOT_TABLE_POLLS; i++) { + v = io->rd32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH7); + if (v == ANE_T6021_BOOT_ACK) + break; + io->poll_wait(io->ctx); + } + if (v != ANE_T6021_BOOT_ACK) + return -ETIMEDOUT; + *booted = 1; + io->phase(io->ctx, "P7 pollB DONE observed"); + + /* HANDSHAKE COMPLETE != firmware result success: the DONE ack is + * the fw's handshake word; the SCRATCH0/1 result u64 it leaves + * behind has UNSOURCED success semantics (pass6: the fw writes a + * "result" at 0x77a0 whose meaning is not decoded). Expose it + * raw; never infer success from booted alone. + */ + *scratch_result = + ((u64)io->rd32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH1) << 32) | + io->rd32(io->ctx, ANE_T6021_BOOT_REG_SCRATCH0); + return 0; +} + +/* Pinned 13.5 firmware's eight channel descriptors, captured before ACK. + * Type 0 is host-to-firmware; type 1 is firmware-to-host. + * Each ring slot is 64 bytes. The descriptor table uses 256-byte entries. + */ +#define ANE_T6021_CHMAN_ENTRY_SIZE 0x100 +#define ANE_T6021_CHMAN_NAME_LEN 0x40 +#define ANE_T6021_CHMAN_COUNT 8 +#define ANE_T6021_CHMAN_TOTAL 0x10440 + +struct ane_t6021_chman_desc { + char name[ANE_T6021_CHMAN_NAME_LEN]; + u32 type; + u32 bit; + u64 size; + u64 ring; + u8 pad[ANE_T6021_CHMAN_ENTRY_SIZE - ANE_T6021_CHMAN_NAME_LEN - 0x18]; +}; + +struct ane_t6021_chman_static { + const char *name; + u32 type; + u32 bit; + u64 size; + u32 off; /* ring offset from the IPC DVA */ +}; + +static const struct ane_t6021_chman_static +ane_t6021_chman_layout[ANE_T6021_CHMAN_COUNT] = { + { "TERMINAL", 2, 0, 0x300, 0x0800 }, + { "IO", 0, 1, 0x010, 0xc800 }, + { "DEBUG", 0, 2, 0x008, 0xcc00 }, + { "BUF_H2T", 0, 3, 0x040, 0xce00 }, + { "BUF_T2H", 1, 4, 0x040, 0xde00 }, + { "SHAREDMALLOC", 1, 5, 0x008, 0xee00 }, + { "IO_T2H", 1, 6, 0x040, 0xf000 }, + { "DATA_CHAIN_H2T", 0, 7, 0x010, 0x10000 }, +}; + +/* True when entry i of a table read from the IPC surface matches the + * static layout with the rings based at ipc_dva. + */ +static inline bool +ane_t6021_chman_entry_ok(const struct ane_t6021_chman_desc *d, + unsigned int i, u64 ipc_dva) +{ + const struct ane_t6021_chman_static *s = &ane_t6021_chman_layout[i]; + + return !strncmp(d->name, s->name, ANE_T6021_CHMAN_NAME_LEN) && + d->type == s->type && d->bit == s->bit && d->size == s->size && + d->ring == ipc_dva + s->off; +} + +/* Bitmask of mismatching entries (0 = the whole table validated). */ +static inline unsigned int +ane_t6021_chman_check(const struct ane_t6021_chman_desc *t, u64 ipc_dva) +{ + unsigned int i, bad = 0; + + for (i = 0; i < ANE_T6021_CHMAN_COUNT; i++) + if (!ane_t6021_chman_entry_ok(&t[i], i, ipc_dva)) + bad |= 1U << i; + return bad; +} + +/* Before ACK, ownership value 1 prevents firmware from consuming empty H2T slots. */ +static inline bool ane_t6021_chman_host_init(void *ipc, u64 size, u64 dva) +{ + unsigned int i, slot; + + if (!ipc || size < ANE_T6021_CHMAN_TOTAL || + dva > ~(u64)0 - ANE_T6021_CHMAN_TOTAL || + ane_t6021_chman_check(ipc, dva)) + return false; + for (i = 0; i < ANE_T6021_CHMAN_COUNT; i++) { + const struct ane_t6021_chman_static *s = &ane_t6021_chman_layout[i]; + + if (s->type != 0) + continue; + for (slot = 0; slot < s->size; slot++) { + u8 *entry = (u8 *)ipc + s->off + slot * 0x40; + + memset(entry, 0, 24); + entry[0] = 1; + } + } + return true; +} + +#endif /* __ANE_T6021_BOOT_H__ */ diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c new file mode 100644 index 0000000000000..1e44f0355b485 --- /dev/null +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -0,0 +1,645 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * T602x/T8112 ANE firmware loader — installed-module port of the proven lab + * staging unit (receipts 2026-09-19-h14-w13-boot-contract.md, + * -w14-staging-proven.md; add-path proof + * 2026-09-28-h14-fsm-secure-park-decode/first-inference.md). + * + * Implements, behind fw_load=1 (default on): + * 1. request_firmware() of the SoC's pinned 13.5 image + * (ane_t602x_soc.fw: selene on T602x, bia on T8112) + * 2. Validation via ane_fw_validate.h (shared with the offline + * regression tools/h14_boot_regression.c): sha256 pin, strict + * exact-image assertions (5 load commands, 2 pinned segments, + * entry 0, bounded LC walk). + * 3. dma_alloc_coherent on the ANE platform device: the buffer is + * DART-mapped through the device's iommu group (stream 0 of the + * three bound instances, H14DartAudit). Segment-wise copy + * (__TEXT fileoff 0x4000 -> vm 0, __DATA -> its vmaddr; vmsize + * tail zero from the coherent alloc). Coherent memory needs no + * explicit cache clean. + * 4. Which copy runs. By default the staged copy (own memory). The lab + * option fw_alias_reserved=1 (T6021 only) maps the copy iBoot + * preloaded at SEG0/SEGi; probe refuses it unless no-map + * /reserved-memory nodes cover both windows (the lab m1n1 adds them, + * the packaged m1n1 1.6.1 does not). The staged copy + * runs: ane_fw_apply_boot_patches() first writes iBoot's runtime + * patches into it (DATA base, RTK_soc, revision, ASC addresses, + * stack guard, ASC tunables), and then it equals the preload byte + * for byte except the random guard (17 captures, + * receipts/2026-10-01-t602x-independent). That needs no reserved + * memory and no preload address, so it is the only mode on SoCs + * without a recorded placement (T6020, T6022, T8112). + * 5. W16 entry alias: the staged fw pages are ALIASED at the latched + * RVBAR entry on the device's default DMA domain. Placement + * contract, receipt receipts/2026-09-20-t6021-entry-alias.md: + * - live RVBAR read64 = 0x10000000001 (bit0 latched, entry + * bits 0x10000000000 = dart-ane vm base); kext law + * (rvbar-lifecycle 6288b0b): bit0 set => the RVBAR write + * branch is skipped, so the ASC fetches AT the latched entry; + * - netconsole 2026-09-20T17:48:19: apple-dart 285800000.iommu + * "translation fault ... code:0x2 (NO PGD FOR IOVA) at + * 0x100000dca10" from the pre-module userspace CPU release — + * the ASC fetched the entry region while nothing was mapped; + * - current-state table walk (kcore, 2026-09-20): entry + * pgd[16] = 0 while the fw leaves are valid -> the region the + * hardware fetches is unmapped and the fw surface is + * physically NON-contiguous (consecutive leaf PAs differ by + * -0x4000). The alias therefore copies each page's PA from + * the live fw translation (iommu_iova_to_phys), never from + * the kernel VA, and verifies every target page unmapped + * before mapping it. + * Driver-managed only: iommu_map() on the attached domain — + * apple_dart implements .map_pages + .iotlb_sync_map, so the + * driver flushes the DART TLB; no raw TTBR/TCR writes. + * Collision ceiling: the iovad allocates top-down from ~4 TiB + * and total mappable RAM is 94.9 GiB (/proc/iomem), so the + * allocator floor 4 TiB - 96 GiB = 0x3e800000000 is far above + * the alias ceiling 0x10005000000; the per-page zero-check + * additionally guards each map. + * + * NOT implemented: the boot step (CPU release + SCRATCH). The userspace + * hybrid stage owns the W8 grant writes; with the alias in place a + * release no longer faults the entry fetch. + * + * Error ownership: every failure path releases the firmware and, once + * allocated, the coherent buffer, and clears ane->fw_buf — remove() + * frees only what fw_buf still names. Load is non-fatal to probe. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane_t6021.h" +#include "ane_t6021_boot.h" +#include "ane_fw_validate.h" + +/* This object links into both ane_t6021.ko and ane_t6021_rtclient.ko; + * per-object metadata keeps modpost happy for either composition. + */ +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("T6021 ANE firmware staging + entry alias"); + +static bool fw_load = true; +module_param(fw_load, bool, 0444); +MODULE_PARM_DESC(fw_load, + "Validate + DART-map the 13.5 PRELOAD payload (default on: " + "the proven add-path configuration)."); + +static unsigned int fw_extra_ram = 0x200000; +module_param(fw_extra_ram, uint, 0444); +MODULE_PARM_DESC(fw_extra_ram, + "Page-aligned owned RAM after the 5 MiB firmware allocation " + "(default 0x200000, the proven add-path grant; maximum 16 MiB)."); + +/* Preloaded-placement alias, lab only: map the iBoot-reserved SEG0/SEGi + * phys at the entry IOVAs (the preload, with iBoot's patches in place). + * Default off: the own-memory copy needs no reserved RAM, so it runs with + * the packaged m1n1 (T6021 own-memory boot: receipts/2026-10-01-t602x- + * independent, "Boot B"). Only SoCs whose placement is recorded honor it + * (ane_t602x_soc.preload_placement), and only over reserved RAM + * (ane_t6021_fwload_placement_ok). + */ +static bool fw_alias_reserved; +module_param(fw_alias_reserved, bool, 0444); +MODULE_PARM_DESC(fw_alias_reserved, + "Lab, T6021: map reserved SEG0 0x10000848000+0xc4000 at entry and " + "SEG1 0x10001400000+0x438000 after it (probe refuses unless no-map " + "/reserved-memory nodes cover both); 0 (default) = run the staged " + "copy with iBoot's patches replayed (own memory, the only mode on " + "T6020/T6022)."); + +static bool ane_t6021_fw_alias_is_reserved(struct device *dev) +{ + const struct ane_t602x_soc *soc = of_device_get_match_data(dev); + + return fw_alias_reserved && soc->preload_placement; +} + +/* iBoot's T6021 SEG0/SEGi placement (pinned ADT segment-ranges record): + * the physical pages fw_alias_reserved=1 hands to the ASC. + */ +static const struct { u64 phys, len; } ane_t6021_fw_preload[] = { + { 0x10000848000ull, 0xc4000ull }, + { 0x10001400000ull, 0x438000ull }, +}; + +/* Only a no-map /reserved-memory node that the kernel took at boot keeps + * it off those pages. The lab m1n1 adds ane-firmware@ nodes; the packaged + * m1n1 1.6.1 adds none (receipts/2026-10-01-t6021-disk-boot, "Limits"). + */ +static bool ane_t6021_fw_preload_reserved(void) +{ + struct device_node *parent = of_find_node_by_path("/reserved-memory"); + unsigned int i, covered = 0; + + for (i = 0; parent && i < ARRAY_SIZE(ane_t6021_fw_preload); i++) { + u64 phys = ane_t6021_fw_preload[i].phys; + u64 end = phys + ane_t6021_fw_preload[i].len; + struct device_node *np; + + for_each_available_child_of_node(parent, np) { + struct reserved_mem *rmem = of_reserved_mem_lookup(np); + + if (rmem && of_property_read_bool(np, "no-map") && + rmem->base <= phys && end <= rmem->base + rmem->size) { + covered++; + of_node_put(np); + break; + } + } + } + of_node_put(parent); + return covered == ARRAY_SIZE(ane_t6021_fw_preload); +} + +bool ane_t6021_fwload_placement_ok(struct device *dev) +{ + /* BINDING probe-top predicate, as ane_t6021_fwload_options_ok(): + * reserved mode never maps RAM the kernel may own. + */ + return !fw_load || !ane_t6021_fw_alias_is_reserved(dev) || + ane_t6021_fw_preload_reserved(); +} + +bool ane_t6021_fwload_requested(void) +{ + return fw_load; +} + +/* DART page size on t6021 (apple_dart probe line: "pagesize 4000"); + * fw_size is a multiple. Shared with the probe-top predicate below. + */ +#define ANE_T6021_FW_ALIAS_PAGE 0x4000 + +bool ane_t6021_fwload_options_ok(void) +{ + /* BINDING probe-top predicate. MUST be called before + * devm_kzalloc / power / CPU release at every probe site. + * The alloc-time check below runs as defense in depth. + * The lab envelope rule: 16 KiB-aligned, <= 16 MiB. Both alias + * modes map the whole allocation, so the grant needs no mode. + */ + return fw_extra_ram <= SZ_16M && + IS_ALIGNED(fw_extra_ram, ANE_T6021_FW_ALIAS_PAGE); +} + +/* Per-SoC data. RTK_soc_revision on T602x is a measured constant: m1n1 + * gives Linux no chip-revision. T6021: the 17 preload captures (0x11); + * T6020: ANEMinorVersion 1 on community rows 4e8224a39f40, dc53badc436a, + * ea8c35bd20bf, e6069fe959ad; T6022: 17 on row 90c6b9bf3e03 + * (receipts/2026-10-01-community-rows). T6020 and T6022 have the T6021 + * die-0 addresses (13.5 ADTs, t602x-ane.dtsi). + * + * pmu_pa: the firmware's power service programs the seven ANE ps words + * through its DART at IOVA == PA (SetPMUBaseAddress stores the ANE_TD + * word: selene 0x28e084008, bia 0x23b70c010, receipts/2026-10-01-t8112-kit). + * macOS maps the page first; without it the first access faults (NO PMD + * FOR IOVA 0x28e084008, 2026-09-29) and the firmware halts. + * + * T8112 (receipts/2026-10-01-t8112-optin): the revision comes from the + * eFuse words iBoot reads, the ANE_SYS_CPU word is pmgr +0xc008, and the + * kext (type 0x70) opens PWGATE "set" +0x8b8 before the ps words. The TM + * TD word is not known there, so trace_td is off. + */ +const struct ane_t602x_soc ane_t6020_soc = { + .soc = 0x6020, .soc_revision = 0x01, + .fw = &ane_fw_selene, .tunables = &ane_t6020_asc_tunables, + .ps_cpu_off = 0x2e0, .pmu_pa = 0x28e084000ull, + .trace_td_off = 0x1c20458, +}; + +const struct ane_t602x_soc ane_t6021_soc = { + .soc = 0x6021, .soc_revision = 0x11, .preload_placement = true, + .fw = &ane_fw_selene, .tunables = &ane_t602x_asc_tunables, + .ps_cpu_off = 0x2e0, .pmu_pa = 0x28e084000ull, + .trace_td_off = 0x1c20458, +}; + +const struct ane_t602x_soc ane_t6022_soc = { + .soc = 0x6022, .soc_revision = 0x11, + .fw = &ane_fw_selene, .tunables = &ane_t602x_asc_tunables, + .ps_cpu_off = 0x2e0, .pmu_pa = 0x28e084000ull, + .trace_td_off = 0x1c20458, +}; + +const struct ane_t602x_soc ane_t8112_soc = { + .soc = 0x8112, .revision_fuse = true, + .fw = &ane_fw_bia, .tunables = &ane_t8112_asc_tunables, + .ps_cpu_off = 0xc008, .pwgate_off = 0x8b8, + .pmu_pa = 0x23b70c000ull, .ps_off = 0x8, +}; + +static int ane_t6021_pmu_map(struct ane_t6021 *ane, struct iommu_domain *dom) +{ + const struct ane_t602x_soc *soc = of_device_get_match_data(ane->dev); + int prot = IOMMU_READ | IOMMU_WRITE; + int ret; + + if (dev_is_dma_coherent(ane->dev)) + prot |= IOMMU_CACHE; + ret = iommu_map(dom, soc->pmu_pa, soc->pmu_pa, ANE_T6021_FW_ALIAS_PAGE, + prot, GFP_KERNEL); + if (!ret && iommu_iova_to_phys(dom, soc->pmu_pa) != soc->pmu_pa) + ret = -EIO; + dev_info(ane->dev, "pmu: DART map %#llx (IOVA == PA, %#x bytes): %d\n", + soc->pmu_pa, ANE_T6021_FW_ALIAS_PAGE, ret); + return ret; +} + +static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) +{ + struct iommu_domain *dom = iommu_get_domain_for_dev(ane->dev); + void __iomem *eng = ane->base[ANE_T6021_REG_ENGINE]; + u64 rvbar = readq(eng + ANE_ASC_RVBAR); + u64 entry = ane_t6021_rvbar_entry_bits(rvbar); + phys_addr_t pa0 = 0; + int prot = IOMMU_READ | IOMMU_WRITE; + u64 off; + int ret; + + if (!dom) + return -ENODEV; + if (!ane_t6021_rvbar_latched(rvbar) || !entry) { + /* Unlatched branch: the boot path programs RVBAR to the + * fw DVA itself (ane_t6021_rvbar_compose), no alias. + */ + dev_info(ane->dev, + "fwalias: rvbar %016llx not latched/entry 0 — skip (boot reprograms RVBAR)\n", + rvbar); + return 0; + } + if (!ane_t6021_rvbar_entry_ok(entry) || + entry & (ANE_T6021_FW_ALIAS_PAGE - 1)) { + dev_err(ane->dev, + "fwalias: entry %#llx not %#x-aligned in fold\n", + entry, ANE_T6021_FW_ALIAS_PAGE); + return -EINVAL; + } + if (entry + ane->fw_size - 1 > dom->geometry.aperture_end) { + dev_err(ane->dev, + "fwalias: entry %#llx+%#x outside aperture %#llx\n", + entry, ane->fw_size, + (unsigned long long)dom->geometry.aperture_end); + return -ERANGE; + } + if (dev_is_dma_coherent(ane->dev)) + prot |= IOMMU_CACHE; + + if (reserved) { + /* Preloaded placement: the two reserved windows mapped at + * the entry IOVAs. SEG0 0xc4000 is TEXT and SEG1 0x438000 + * is DATA (the 13.5 layout, ane_fw_selene.segs). + * The windows are adjacent, so the mapping is one + * contiguous run of `mapped` bytes starting at entry — + * tracked exactly, because teardown must never unmap a + * page that was not mapped (dart_unmap_pages WARNs on + * holes and the 2026-09-26 state-report unwind hit it). + * SEG1 IOVA = entry + 0xc4000 = 0x100000c4000 per the + * pinned ADT segment-ranges record (0x10000000000 -> + * 0x10000848000 0xc4000; 0x100000c4000 -> 0x10001400000 + * 0x438000). Commit e6612e9 shipped 0x1000000c4000 — one + * hex digit off, mapping DATA at 16 TiB instead: every + * fw_alias_reserved boot since 2026-09-24 left the fw + * DATA section unmapped past the SEG0 head. + */ + struct { u64 iova, phys, len; } win[] = { + { 0x10000000000ull, ane_t6021_fw_preload[0].phys, ane_t6021_fw_preload[0].len }, + { 0, ane_t6021_fw_preload[1].phys, ane_t6021_fw_preload[1].len }, + { entry + 0x4fc000, 0, fw_extra_ram ? ane->fw_size - 0x4fc000 : 0 }, + }; + unsigned int w, windows = fw_extra_ram ? ARRAY_SIZE(win) : 2; + + /* The remap is contiguous: SEG1 base = SEG0 base + SEG0 + * len (segment-ranges order). Derived, never hand-written + * — the standalone literal 0x1000000c4000 that shipped in + * e6612e9 was one digit off and mapped DATA at 16 TiB. + * The remap base must also equal the latched entry, or + * the fetch head is not where we mapped. + */ + win[1].iova = win[0].iova + win[0].len; + if (win[0].iova != entry) { + dev_err(ane->dev, + "fwalias: remap base %#llx != latched entry %#llx\n", + win[0].iova, entry); + return -EINVAL; + } + + for (w = 0; w < windows; w++) { + u64 o; + + for (o = 0; o < win[w].len; o += ANE_T6021_FW_ALIAS_PAGE) { + phys_addr_t pa = w < 2 ? win[w].phys + o : + iommu_iova_to_phys(dom, ane->fw_iova + win[w].iova + o - entry); + if (!pa || !IS_ALIGNED(pa, ANE_T6021_FW_ALIAS_PAGE)) { + ret = -EFAULT; + goto err_unmap_mapped; + } + if (iommu_iova_to_phys(dom, win[w].iova + o)) { + dev_err(ane->dev, + "fwalias: reserved entry +%#llx mapped — refusing\n", + win[w].iova + o - entry); + ret = -EEXIST; + goto err_unmap_mapped; + } + ret = iommu_map(dom, win[w].iova + o, + pa, + ANE_T6021_FW_ALIAS_PAGE, prot, + GFP_KERNEL); + if (ret) { + dev_err(ane->dev, + "fwalias: reserved map +%#llx: %d\n", + win[w].iova + o - entry, ret); + goto err_unmap_mapped; + } + /* per-window successfully mapped bytes */ + ane->fw_alias_ext_len[w] = o + ANE_T6021_FW_ALIAS_PAGE; + ane->fw_alias_ext_iova[w] = win[w].iova; + if (iommu_iova_to_phys(dom, win[w].iova + o) != pa) { + ret = -EIO; + goto err_unmap_mapped; + } + } + } + ane->fw_alias_extn = windows; + ane->fw_alias_iova = entry; + dev_info(ane->dev, + "fwalias: reserved SEG0/SEGi at entry %#llx (%llx+%zx %llx+%zx, preloaded placement)\n", + entry, + ane->fw_alias_ext_iova[0], ane->fw_alias_ext_len[0], + ane->fw_alias_ext_iova[1], ane->fw_alias_ext_len[1]); + if (fw_extra_ram) + dev_info(ane->dev, "fwalias: owned heap [%#llx,%#llx) roundtrip verified\n", + win[2].iova, win[2].iova + win[2].len); + return ane_t6021_pmu_map(ane, dom); + +err_unmap_mapped: + /* Cleanup exactly the per-window bytes we mapped; windows + * are not assumed adjacent (live trace: a hole between + * SEG0 and SEGi), and foreign collision mappings are + * never touched. + */ + for (w = 0; w < windows; w++) + if (ane->fw_alias_ext_len[w]) + iommu_unmap(dom, ane->fw_alias_ext_iova[w], + ane->fw_alias_ext_len[w]); + memset(ane->fw_alias_ext_len, 0, sizeof(ane->fw_alias_ext_len)); + ane->fw_alias_extn = 0; + return ret; + } + + for (off = 0; off < ane->fw_size; off += ANE_T6021_FW_ALIAS_PAGE) { + phys_addr_t pa = iommu_iova_to_phys(dom, ane->fw_iova + off); + + if (!pa || pa & (ANE_T6021_FW_ALIAS_PAGE - 1)) { + dev_err(ane->dev, + "fwalias: fw +%#llx untranslated (%pa)\n", + off, &pa); + ret = -EFAULT; + goto err_unmap; + } + if (iommu_iova_to_phys(dom, entry + off)) { + dev_err(ane->dev, + "fwalias: entry +%#llx already mapped — refusing\n", + off); + ret = -EEXIST; + goto err_unmap; + } + ret = iommu_map(dom, entry + off, pa, + ANE_T6021_FW_ALIAS_PAGE, prot, GFP_KERNEL); + if (ret) { + dev_err(ane->dev, "fwalias: iommu_map +%#llx: %d\n", + off, ret); + goto err_unmap; + } + if (!off) + pa0 = pa; + } + + /* full per-page roundtrip: every alias page must resolve to the + * same PA as its fw source page (not just page 0) + */ + for (off = 0; off < ane->fw_size; off += ANE_T6021_FW_ALIAS_PAGE) { + if (iommu_iova_to_phys(dom, entry + off) != + iommu_iova_to_phys(dom, ane->fw_iova + off)) { + dev_err(ane->dev, + "fwalias: roundtrip mismatch at +%#llx\n", off); + ret = -EIO; + /* Mapping finished: unwind every page, not only the + * prefix already checked by this verification loop. + */ + off = ane->fw_size; + goto err_unmap; + } + } + + ane->fw_alias_iova = entry; + ane->fw_alias_ext_iova[0] = entry; + ane->fw_alias_ext_len[0] = ane->fw_size; + ane->fw_alias_extn = 1; + dev_info(ane->dev, + "fwalias: entry %#llx <- %u dart pages aliased from fw %pad (first %pa, roundtrip OK)\n", + entry, ane->fw_size / ANE_T6021_FW_ALIAS_PAGE, + &ane->fw_iova, &pa0); + /* The firmware's power service needs the pmgr sub-block mapped + * IOVA == PA in every vehicle (NO PMD FOR IOVA 0x28e084008, + * 2026-09-29); the staged-DMA alias branch skipped it and left + * a bisect run booting a halting fw. + */ + return ane_t6021_pmu_map(ane, dom); + +err_unmap: + if (off) + iommu_unmap(dom, entry, off); + return ret; +} + +/* RTK_soc_revision. T8112 reads it as iBoot does: the two eFuse words + * of the DT "fuse" window (0x23d2c8060, 8 bytes), decoded by + * ane_t8112_fuse_revision(). Two non-posted 32-bit reads and nothing + * else: the eFuse block (ADT pmgr reg[36]) is always on, and m1n1 reads + * its ATC and GPU fuses at 0x23d2c8484 and 0x23d2c84dc on every T8112 + * boot. Without the window the revision is unknown, so the load refuses. + */ +static int ane_t6021_soc_revision(struct ane_t6021 *ane, + const struct ane_t602x_soc *soc, u32 *rev) +{ + struct resource *res; + void __iomem *fuse; + u32 w0, w1; + + if (!soc->revision_fuse) { + *rev = soc->soc_revision; + return 0; + } + res = platform_get_resource_byname(to_platform_device(ane->dev), + IORESOURCE_MEM, "fuse"); + if (!res || resource_size(res) != 8) { + dev_err(ane->dev, + "fwload: no 8-byte \"fuse\" window: chip revision unknown, refusing\n"); + return -ENODEV; + } + fuse = ioremap_np(res->start, 8); + if (!fuse) + return -ENOMEM; + w0 = readl(fuse); + w1 = readl(fuse + 4); + iounmap(fuse); + *rev = ane_t8112_fuse_revision(w0, w1); + dev_info(ane->dev, "fwload: chip revision %#x (fuse %#llx: %08x %08x)\n", + *rev, (u64)res->start, w0, w1); + return 0; +} + +/* Own memory (header item 4): iBoot's runtime patches, with values from + * the running system: the latched entry (or the staged DVA that the boot + * path programs when RVBAR is not latched), the DT engine window (probe + * already refused a node without it), the compatible, a fresh guard. + */ +static int ane_t6021_fw_patch(struct ane_t6021 *ane, u8 *img) +{ + const struct ane_t602x_soc *soc = of_device_get_match_data(ane->dev); + struct resource *res = platform_get_resource(to_platform_device(ane->dev), + IORESOURCE_MEM, 0); + u64 rvbar = readq(ane->base[ANE_T6021_REG_ENGINE] + ANE_ASC_RVBAR); + u64 entry = ane_t6021_rvbar_latched(rvbar) ? + ane_t6021_rvbar_entry_bits(rvbar) : 0; + u64 guard = get_random_u64(); + struct ane_fw_boot_patch p = { + .exec_base = entry ?: ane->fw_iova, + /* iBoot's guards have one zero byte at a random position */ + .stack_guard = guard & ~(0xffull << (8 * (guard >> 61))), + .soc = soc->soc, + .cpu_pa = res->start + ANE_ASC_CPU_BASE, + .wrapper_pa = res->start + ANE_ASC_WRAPPER_BASE, + }; + const char *reason = NULL; + int ret; + + ret = ane_t6021_soc_revision(ane, soc, &p.soc_revision); + if (ret) + return ret; + if (ane_fw_apply_boot_patches(img, soc->fw, soc->tunables, &p, &reason)) { + dev_err(ane->dev, "fwload: own memory: %s (rev %#x)\n", reason, + p.soc_revision); + return -EINVAL; + } + dev_info(ane->dev, + "fwload: own memory: iBoot patches replayed (soc %#x rev %#x DATA %#llx cpu %#llx wrapper %#llx)\n", + p.soc, p.soc_revision, + p.exec_base + soc->fw->segs[1].vmaddr, p.cpu_pa, + p.wrapper_pa); + return 0; +} + +int ane_t6021_fwload_probe(struct ane_t6021 *ane) +{ + const struct ane_t602x_soc *soc = of_device_get_match_data(ane->dev); + const struct ane_fw_image *img = soc->fw; + const struct firmware *fw = NULL; + struct ane_fw_seg segs[ANE_FW_NSEGS]; + u64 entry = 0; + u8 actual_sha[32]; + void *buf; + dma_addr_t iova; + unsigned int i; + const char *reason = NULL; + u32 alloc_size = ANE_FW_BUF_SIZE + fw_extra_ram; + int ret; + bool reserved = ane_t6021_fw_alias_is_reserved(ane->dev); + + if (!fw_load) + return 0; + if (fw_extra_ram > SZ_16M || !IS_ALIGNED(fw_extra_ram, ANE_T6021_FW_ALIAS_PAGE)) + return -EINVAL; + /* The 64-bit coherent mask is set once in ane_t6021_probe, + * BEFORE rtkit_init allocates the rings (W15 review). + */ + + ret = request_firmware(&fw, img->name, ane->dev); + if (ret) { + dev_err(ane->dev, + "fwload: request_firmware(%s): %d — stage the payload " + "with omarchy-ane-firmware-fetch\n", img->name, ret); + return ret; + } + + sha256(fw->data, fw->size, actual_sha); + ret = ane_fw_validate_blob(fw->data, fw->size, img, actual_sha, + segs, &entry, &reason); + if (ret) { + dev_err(ane->dev, "fwload: validation failed: %s\n", + reason ? reason : "?"); + release_firmware(fw); + return ret; + } + + buf = dma_alloc_coherent(ane->dev, alloc_size, &iova, GFP_KERNEL); + if (!buf) { + dev_err(ane->dev, "fwload: coherent alloc %#x failed\n", + alloc_size); + release_firmware(fw); + return -ENOMEM; + } + + for (i = 0; i < ANE_FW_NSEGS; i++) { + if (segs[i].filesize) + memcpy(buf + segs[i].vmaddr, + fw->data + segs[i].fileoff, segs[i].filesize); + } + + ane->fw_buf = buf; + ane->fw_iova = iova; + ane->fw_size = alloc_size; + + dev_info(ane->dev, + "fwload: %s PRELOAD validated + DART-mapped: entry %#llx, iova %pad size %#x\n", + img->name, entry, &iova, alloc_size); + + ret = reserved ? 0 : ane_t6021_fw_patch(ane, buf); + if (!ret) + ret = ane_t6021_fw_alias_map(ane, reserved); + if (ret) { + ane_t6021_fwload_remove(ane); + release_firmware(fw); + return ret; + } + + release_firmware(fw); + return 0; +} + +void ane_t6021_fwload_remove(struct ane_t6021 *ane) +{ + if (ane->fw_alias_extn) { + struct iommu_domain *dom = iommu_get_domain_for_dev(ane->dev); + int i; + + /* Unmap exactly the per-window recorded extents: the + * reserved-alias windows are not assumed adjacent (live + * trace hole between SEG0 and SEGi) and unmapping bytes + * that were never mapped trips dart_unmap_pages + * (io-pgtable-dart.c:319 WARN, 2026-09-26). + */ + if (dom) + for (i = 0; i < ane->fw_alias_extn; i++) + iommu_unmap(dom, ane->fw_alias_ext_iova[i], + ane->fw_alias_ext_len[i]); + ane->fw_alias_extn = 0; + } + if (!ane->fw_buf) + return; + dma_free_coherent(ane->dev, ane->fw_size, ane->fw_buf, ane->fw_iova); + ane->fw_buf = NULL; + ane->fw_size = 0; +} diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c new file mode 100644 index 0000000000000..9d1e43ecc508b --- /dev/null +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -0,0 +1,2363 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * ane_t6021_rtclient.c — T6021 (H14 / M2) installed ANE driver. + * + * One module, two surfaces: + * - boot/transport: fenced Linux-context firmware start (fw_start=1 + * default — the proven add-path config), shared boot/fwload + * contract units, then the legacy 13.5 ChMan transport for the + * three load-time commands (LOAD_PROGRAM 0x200, CREATE_PROCESS + * 0x202, PROCEDURE_CALL 0x204) driven through the ioctls in this + * file; + * - DRM accel: BO_INIT/BO_FREE plus PROG_LOAD/PROC_CREATE/EXEC per + * include/uapi/drm/ane_accel.h (ABI 2). Sections ride BOs the + * user supplies; one global mutex serializes every firmware + * command; completion = the legacy exchange's reply PLUS the + * firmware's finish event on the IO_T2H ring. EXEC also returns the + * fw's target-to-host slots (the sequencer's per-step drain). + * + * Compiled defaults are the load-run.sh parameter list — a bare + * `insmod ane_t6021.ko` is the proven add-path configuration on + * boot 3ab812a3 (fw_load=1 fw_start=1 fw_start_dapf=0 legacy_only=1 + * legacy_query=1 scratch3_ack=1 poll_rx=1 ...) — except hello_wait_ms, + * 1000 in that list and 0 here (see its definition). + * Remaining parameters are overridable from sysfs for bisection only. + * + * Division of labor (receipts/2026-09-22-t6021-rtkit-port): + * - CPU start: kernel-context start is the proven-safe route on this + * box — m1n1 RTBuddy path is dormant (W5/W6 MGMT session wall) and + * the 13.5 (22G74) selene image brings up the legacy ChMan + * transport after DONE without RTKit. Boot is fenced in + * ane_t6021_boot.c (entry fold, RVBAR skip-or-fold, scratch, + * RUN, poll A/B, publish/wake) and audited in + * tools/h14_boot_regression.c. + * - genpd/pmgr: Runtime PM + the DT power-domains binding owns the + * raise; probe verifies ACTUAL on ane_cpu (pmgr window) before + * the first engine read (G1 gate). The pre-CPU engine table block + * is intentionally NOT written (preboot_table_mode = 2 = SKIP, the + * 2026-09-20-era live-fault discriminator). macOS-form mpm_off is + * skipped: with fw_start_mpm_off=0 (the proven config) the + * boot-raised ane_sys_mpm stays on; the boot writes do not depend + * on it. + * - mailbox: ane_rtclient_validate_chman sees the 'IPC ' surface the + * fw published during boot; the three load-time commands travel + * the IO ring (channel 1) over ane_rtclient_legacy_exchange. Each + * command advances the cursor on the next 64-byte slot — resending + * to slot 0 wedges the ring (sequencer rule). + * - non-posted MMIO (DT "nonposted-mmio" -> IORESOURCE_MEM_NONPOSTED) + * is mandatory throughout the ANE aperture; posted writel froze the + * box. + * - Firmware alias: the staged copy (own memory, default) or, with the + * lab option fw_alias_reserved=1, the iBoot-reserved SEG0/SEGi phys + * is aliased at the latched RVBAR entry; the subsequent kernel writes + * into the entry region translate through dart-ane0 instead of + * faulting. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane_t6021.h" +#include "ane_t6021_boot.h" + +#include + +/* Doorbell (IPI) block, engine + 0x1844000: set +0, pending +0x8000, + * ack +0xc000 (kext aneInterruptHandler reads +0x184c000 and writes + * +0x1850000 with no SoC branch, receipts/2026-10-01-t8112-ane). + */ +#define ANE_IPI_OFF 0x1844000 + +/* CPU_STATUS bits (m1n1 ASCRegs shape) */ +#define ANE_ASC_CPU_STATUS_RUNNING BIT(0) +#define ANE_ASC_CPU_STATUS_STOPPED BIT(1) + +/* Boot-time module parameters — defaults are the proven add-path + * parameter list (boot 3ab812a3 / load-run.sh), except fw_alias_reserved, + * which defaults to own memory (receipts/2026-10-01-t602x-independent, + * "Boot B"). fw_load, fw_extra_ram and fw_alias_reserved live in + * ane_t6021_fwload.c and boot_prevent_nap in ane_t6021_boot.c (single + * registration each). Remaining knobs are overridable from sysfs for + * bisection only. + */ +static bool fw_start = true; +module_param(fw_start, bool, 0444); +MODULE_PARM_DESC(fw_start, + "Fenced Linux-context firmware start: stage + alias, then the boot sequence; default on (proven add-path config)."); + +static int fw_start_table_mode = 2; +module_param(fw_start_table_mode, int, 0444); +MODULE_PARM_DESC(fw_start_table_mode, + "Pre-CPU engine table block: 0 abort, 1 write, 2 skip (default, proven config)."); + +static bool fw_start_rtb_mode; +module_param(fw_start_rtb_mode, bool, 0444); +MODULE_PARM_DESC(fw_start_rtb_mode, + "RTBuddy/RTKit-app-endpoint select; default off (legacy ChMan transport)."); + +/* Drive RX by apple_rtkit_poll from a workqueue even though a recv + * IRQ exists (lab poll_rx=1, the proven add-path value: the raw recv + * line is unproven, so the worker is what services HELLO/EPMAP). + */ +static bool poll_rx = true; +module_param(poll_rx, bool, 0444); +MODULE_PARM_DESC(poll_rx, + "Drive RX by apple_rtkit_poll from a workqueue (default on, proven config)."); + +/* STARTEP every fw-announced app endpoint (>= 0x20) after a successful + * RTKit handshake (lab start_app_eps=1 default). + */ +static bool start_app_eps = true; +module_param(start_app_eps, bool, 0444); +MODULE_PARM_DESC(start_app_eps, + "STARTEP every fw-announced app endpoint after the handshake (default on)."); + +static bool scratch3_ack = true; +module_param(scratch3_ack, bool, 0444); +MODULE_PARM_DESC(scratch3_ack, + "After DONE, write SCRATCH3 = 0x08042006 (selene 0x7edc, kext 0x95eaee4). Default on; 0 withholds."); + +static bool legacy_only = true; +module_param(legacy_only, bool, 0444); +MODULE_PARM_DESC(legacy_only, + "Use the pinned 13.5 legacy ChMan transport (default on, proven add-path config)."); + +static bool legacy_query = true; +module_param(legacy_query, bool, 0444); +MODULE_PARM_DESC(legacy_query, + "Service bounded startup allocations and CONFIG_GET; default on (proven config)."); + +/* 0 (default) skips RTKit in legacy mode, so the ANE mailbox never + * starts. Measured 2026-09-30 (receipts/2026-09-30-t6021-stock-mailbox): + * the 13.5 firmware sent no HELLO on any recorded boot (-ETIME after + * 1000 ms), and starting the mailbox enables AIC2 884, a level line + * that then fired ~700,000 times/s (one CPU of hardirq time). A + * firmware that speaks RTKit needs hello_wait_ms=1000. + */ +static unsigned int hello_wait_ms; +module_param(hello_wait_ms, uint, 0444); +MODULE_PARM_DESC(hello_wait_ms, + "RTKit HELLO wait in legacy mode; 0 (default) skips RTKit and leaves the mailbox stopped. A firmware that speaks RTKit needs 1000."); + +#define ANE_LEGACY_ALLOCS 8192 +#define ANE_LEGACY_BYTES SZ_512M + +struct ane_legacy_buffer { + void *cpu; + dma_addr_t dma; + size_t size; +}; + +struct ane_rtclient { + struct device *dev; + void __iomem *engine; + void __iomem *pmgr; + const struct ane_t602x_soc *soc; + struct apple_rtkit *rtk; + struct reset_control *cpu_rst; + struct delayed_work poll_work; + + /* fw_start=1 view over the shared boot/fwload contract units. */ + struct ane_t6021 *fw; + + /* A CPU we released is (or may be) running: state HELD — surfaces, + * rings, IRQ, power links preserved; no unwind; reboot reclaims + * (wedged-pin rule). + */ + bool held; + + bool boot_done; + + /* ChMan descriptor table validated against the 'IPC ' surface. */ + bool chman_ok; + struct ane_legacy_buffer *legacy_buffers; + u32 legacy_allocated; + size_t legacy_bytes; + u32 legacy_malloc_cursor; + u32 legacy_cmd_cursor[ANE_T6021_CHMAN_COUNT]; + /* One reusable 16 KiB command buffer for every host command + * (CONFIG_GET + the three ioctls). Protocol-legal to reuse: an + * exchange returns 0 only after the IO slot flipped back to + * host-owned with a zero status (the lab's own completion + * predicate), so the firmware has fully consumed the previous + * command. The legacy_buffers table stays for fw MALLOC replies, + * which the firmware may reference forever (held until reboot). + */ + struct ane_legacy_buffer *cmd_buf; +}; + +/* Per-open BO ownership (drm_file->driver_priv). Handles live in the + * fd's list; closing the fd drops its handles (postclose). The + * coherent buffers themselves live as long as something references + * them: a user mapping, or the firmware (see ane_t6021_bo_release). + */ +struct ane_t6021_fd { + struct list_head bos; +}; + +/* Per-file handle counter and every per-fd list against same-fd concurrent + * ioctls. A BO whose IOVA reached the firmware (fw_ref) never goes back to + * the kernel: a program section is held until reboot, an io BO is parked + * for reuse (see ane_t6021_bo_release). Every other BO frees at its last + * reference. + * + * Per-BO cap raised 2026-09-30: the 256 KiB Qwen-class matvec weights seen + * by the H14 compiler are bounded by `reduction * columns * 2`, which + * reaches 20 MiB at (K,N)=(2048,5120). The same cap serves the + * Qwen4-attention (K,N)=(4096,4096) constant at 32 MiB and any H14 + * softmax/reduction with an 8 MiB table. Total BO bytes across all fds are + * capped separately (bo_total_max_mb) by an atomic counter, enforced at + * alloc and released when the memory is really freed. Program sections stay + * held until reboot, and the 38 Qwen programs alone hold about 2.6 GiB. + * A cap of 0 refuses every BO_INIT with -ENOSPC; there is no unlimited + * value. Every BO also needs IOVA below 4 GiB (32-bit DMA mask), so + * allocations fail with -ENOMEM near that bound whatever the cap. + * The 16 KiB alignment check is unchanged: every DMA site assumes it. + */ +#define ANE_T6021_BO_MAX SZ_1G +#define ANE_T6021_BO_HASH_CHUNK SZ_1M + +struct ane_t6021_bo { + struct list_head node; + struct ane_t6021_fd *owner; + u32 handle; + struct kref refcount; /* handle + user mappings */ + void *cpu; /* coherent mapping */ + dma_addr_t dma; + size_t size; + bool fw_ref; /* the firmware received this IOVA */ + bool fw_program; /* a loaded program keeps this IOVA */ + struct device *dev; +}; + +static DEFINE_MUTEX(ane_t6021_bo_lock); +static u32 ane_t6021_next_handle = 1; +static atomic64_t ane_t6021_bo_total_bytes = ATOMIC64_INIT(0); + +static unsigned int bo_total_max_mb = 12288; +module_param(bo_total_max_mb, uint, 0444); +MODULE_PARM_DESC(bo_total_max_mb, + "Cap on the BO bytes held at one time, in MiB (default 12288)"); + +static int ane_t6021_bo_total_get(char *buf, const struct kernel_param *kp) +{ + return sysfs_emit(buf, "%lld\n", + (long long)atomic64_read(&ane_t6021_bo_total_bytes)); +} + +static int ane_t6021_bo_total_set(const char *val, + const struct kernel_param *kp) +{ + return -EPERM; +} + +static const struct kernel_param_ops ane_t6021_bo_total_ops = { + .set = ane_t6021_bo_total_set, + .get = ane_t6021_bo_total_get, +}; +module_param_cb(bo_total_bytes, &ane_t6021_bo_total_ops, NULL, 0444); +MODULE_PARM_DESC(bo_total_bytes, + "Read only: the BO bytes counted against bo_total_max_mb now"); + +/* Mark the device quarantined: a timed-out command left the firmware + * queue state unknown. The only safe next step is to refuse further + * ioctls until a reboot reclaims the surfaces (wedged-pin rule). + */ +static atomic_t ane_t6021_quarantined = ATOMIC_INIT(0); + +/* Parked io BOs: their last user is gone, the IOVA stays mapped and the + * bytes stay counted. BO_INIT of the same page-aligned size takes one, + * so held memory stays at the peak of concurrent io BOs instead of + * growing with every process until the BO cap refuses BO_INIT. + */ +static LIST_HEAD(ane_t6021_bo_pool); +static DEFINE_SPINLOCK(ane_t6021_bo_pool_lock); + +/* Final put: the last handle or mapping is gone. The firmware never sees + * a freed IOVA (lab rule), so a fw_ref BO is never freed: a program + * section stays held, because a cached firmware program keeps reading + * it; an io BO goes to the pool, unless a quarantined firmware may still + * write it. Every other BO frees here, so the BO cap bounds only the + * memory the firmware may touch. + */ +static void ane_t6021_bo_release(struct kref *ref) +{ + struct ane_t6021_bo *bo = container_of(ref, struct ane_t6021_bo, + refcount); + + if (!bo->fw_ref) { + atomic64_sub(PAGE_ALIGN(bo->size), &ane_t6021_bo_total_bytes); + dma_free_coherent(bo->dev, bo->size, bo->cpu, bo->dma); + kfree(bo); + } else if (bo->fw_program || atomic_read(&ane_t6021_quarantined)) { + kfree(bo); + } else { + spin_lock(&ane_t6021_bo_pool_lock); + list_add(&bo->node, &ane_t6021_bo_pool); + spin_unlock(&ane_t6021_bo_pool_lock); + } +} + +/* A parked io BO of SIZE's page-aligned size, or NULL. */ +static struct ane_t6021_bo *ane_t6021_bo_pool_take(size_t size) +{ + struct ane_t6021_bo *bo; + + if (atomic_read(&ane_t6021_quarantined)) + return NULL; + spin_lock(&ane_t6021_bo_pool_lock); + list_for_each_entry(bo, &ane_t6021_bo_pool, node) { + if (PAGE_ALIGN(bo->size) == PAGE_ALIGN(size)) { + list_del(&bo->node); + spin_unlock(&ane_t6021_bo_pool_lock); + return bo; + } + } + spin_unlock(&ane_t6021_bo_pool_lock); + return NULL; +} + +/* A user mapping holds its BO's memory until it is torn down: open + * (fork, mremap split) takes a reference, close drops it. A BO the + * user freed while mapped stays allocated until the last mapping + * goes away. + */ +static void ane_t6021_vm_open(struct vm_area_struct *vma) +{ + kref_get(&((struct ane_t6021_bo *)vma->vm_private_data)->refcount); +} + +static void ane_t6021_vm_close(struct vm_area_struct *vma) +{ + kref_put(&((struct ane_t6021_bo *)vma->vm_private_data)->refcount, + ane_t6021_bo_release); +} + +static const struct vm_operations_struct ane_t6021_vm_ops = { + .open = ane_t6021_vm_open, + .close = ane_t6021_vm_close, +}; + +/* Serializes every firmware command (LOAD/CREATE/CALL) so the + * cursor-on-next-64-byte-slot rule and the PMGR/TM gate cannot race. + */ +static DEFINE_MUTEX(ane_t6021_fw_lock); + +/* Forward declaration (defined below). */ +static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, + struct ane_legacy_buffer *command, + size_t length, u16 opcode, + unsigned int channel, + unsigned int timeout_ms); + +/* ---- boot-contract helpers (shared with boot.c / fwload.c) ---- */ + +static int ane_rtclient_legacy_alloc(struct ane_rtclient *ane, u64 size) +{ + struct ane_legacy_buffer *buffer; + + if (!size || !IS_ALIGNED(size, SZ_16K) || size > SZ_2M || + ane->legacy_allocated == ANE_LEGACY_ALLOCS || + size > ANE_LEGACY_BYTES - ane->legacy_bytes) + return -E2BIG; + buffer = &ane->legacy_buffers[ane->legacy_allocated]; + buffer->cpu = dma_alloc_coherent(ane->dev, size, &buffer->dma, + GFP_KERNEL); + if (!buffer->cpu) + return -ENOMEM; + if (!IS_ALIGNED(buffer->dma, SZ_16K) || + (ane->fw && !ane_t6021_fw_alias_iova_ok(ane->fw, buffer->dma, size))) { + dma_free_coherent(ane->dev, size, buffer->cpu, buffer->dma); + buffer->cpu = NULL; + return -ERANGE; + } + memset(buffer->cpu, 0, size); + buffer->size = size; + ane->legacy_bytes += size; + ane->legacy_allocated++; + return 0; +} + +/* Validate the chman table the fw published in the 'IPC ' surface. */ +static void ane_rtclient_validate_chman(struct ane_rtclient *ane) +{ + struct ane_t6021 *a = ane->fw; + const struct ane_t6021_chman_desc *t; + unsigned int i, bad; + + if (!a || !a->boot_ipc) { + dev_info(ane->dev, + "chman: no host IPC surface — table not validated\n"); + return; + } + if (a->boot_ipc_size < ANE_T6021_CHMAN_TOTAL) { + dev_warn(ane->dev, + "chman: IPC surface %#llx bytes < fw layout %#x — table not validated\n", + a->boot_ipc_size, ANE_T6021_CHMAN_TOTAL); + return; + } + + dma_rmb(); + t = a->boot_ipc; + bad = ane_t6021_chman_check(t, a->boot_ipc_iova); + for (i = 0; i < ANE_T6021_CHMAN_COUNT; i++) { + const struct ane_t6021_chman_desc *d = &t[i]; + const struct ane_t6021_chman_static *s = &ane_t6021_chman_layout[i]; + + dev_info(ane->dev, + "chman[%u]: name=\"%.*s\" type=%u bit=%u size=%#llx %s (static: %s/%u/%u/%#llx/ipc+%#x)\n", + i, ANE_T6021_CHMAN_NAME_LEN, d->name, d->type, d->bit, + d->size, + (bad & BIT(i)) ? "MISMATCH" : "OK", + s->name, s->type, s->bit, s->size, s->off); + } + + ane->chman_ok = !bad; + dev_info(ane->dev, "chman: table %s (mismatch mask %#x)\n", + bad ? "NOT VALIDATED" : "VALIDATED", bad); +} + +/* Legacy ChMan exchange (post-DONE 13.5 transport). Resends to the same + * 64-byte IO slot would wedge the ring; cursor advances after each + * completed command (selene fw decodes bit0 = host-owned). + */ +static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, + struct ane_legacy_buffer *command, + size_t length, u16 opcode, + unsigned int channel, + unsigned int timeout_ms) +{ + struct ane_t6021 *a = ane->fw; + u64 *io, *malloc_ring, header; + u64 *t2h_buf, *t2h_ioq; + void __iomem *ipi = ane->engine + ANE_IPI_OFF; + unsigned int cursor = ane->legacy_malloc_cursor; + unsigned long deadline; + u32 *reply; + int result = -ETIMEDOUT; + + if (!ane->held || !ane->chman_ok || + ane_t6021_chman_check(a->boot_ipc, a->boot_ipc_iova)) + return -EPROTO; + if (length < 8 || length > command->size || + channel >= ANE_T6021_CHMAN_COUNT) { + result = -EINVAL; + goto out; + } + io = a->boot_ipc + ane_t6021_chman_layout[channel].off + + (size_t)ane->legacy_cmd_cursor[channel] * 64; + malloc_ring = a->boot_ipc + ane_t6021_chman_layout[5].off; + t2h_buf = a->boot_ipc + ane_t6021_chman_layout[4].off; + t2h_ioq = a->boot_ipc + ane_t6021_chman_layout[6].off; + + if (!(READ_ONCE(io[0]) & 1)) { + result = -EBUSY; + goto out; + } + reply = command->cpu; + ((u16 *)reply)[2] = opcode; + WRITE_ONCE(io[1], length); + WRITE_ONCE(io[2], length); + dma_wmb(); + WRITE_ONCE(io[0], command->dma); + dma_wmb(); + writel(BIT(ane_t6021_chman_layout[channel].bit), ipi); + deadline = jiffies + msecs_to_jiffies(timeout_ms); + while (time_before(jiffies, deadline)) { + u32 pending = readl(ipi + 0x8000); + + if (pending) { + writel(pending, ipi + 0xc000); + mb(); + } + header = READ_ONCE(io[0]); + if (!(header & 1)) { + /* malloc-ring service: the fw may ask for an + * allocation while we wait; service those until + * the command slot flips back. + */ + u64 *slot = malloc_ring + cursor * 8; + u64 alloc_hdr = READ_ONCE(slot[0]); + + if (!(alloc_hdr & 1)) { + u64 size, tag; + struct ane_legacy_buffer *buf; + + dma_rmb(); + size = READ_ONCE(slot[1]); + tag = READ_ONCE(slot[2]); + /* Lab rule: a valid MALLOC slot carries + * header 0 and a u32 tag; anything else + * is a malformed ring and we stop. + */ + if (alloc_hdr || tag > U32_MAX) { + result = -EOPNOTSUPP; + goto out; + } + if (ane_rtclient_legacy_alloc(ane, size)) { + result = -ENOMEM; + goto out; + } + buf = &ane->legacy_buffers[ane->legacy_allocated - 1]; + WRITE_ONCE(slot[1], 0); + WRITE_ONCE(slot[2], ane->legacy_allocated); + dma_wmb(); + WRITE_ONCE(slot[0], buf->dma | 1); + dma_wmb(); + writel(BIT(5), ipi); + cursor = (cursor + 1) % + ane_t6021_chman_layout[5].size; + } + } + header = READ_ONCE(io[0]); + if (header & 1) { + dma_rmb(); + result = (header == (command->dma | 1) && + READ_ONCE(io[1]) == length && + READ_ONCE(io[2]) == 0 && + ((u16 *)reply)[2] == opcode && + ((u16 *)reply)[3] == 0) ? 0 : -EPROTO; + ane->legacy_cmd_cursor[channel] = + (ane->legacy_cmd_cursor[channel] + 1) % + ane_t6021_chman_layout[channel].size; + goto out; + } + /* legacy_fast_poll=1 (the proven add-path cadence): poll + * the rings every 50 us instead of sleeping 1-2 ms. + */ + udelay(50); + } + dev_info(ane->dev, "LEGACY timeout ch=%u io=%016llx\n", + channel, READ_ONCE(io[0])); +out: + ane->legacy_malloc_cursor = cursor; + return result; +} + +/* Time to let the output writes land after the completion signal. + * Measured 2026-09-29 on some boots: the fw ack, the TD counter and the + * TQ words all report done ~0.13 ms before the output reaches DRAM (the + * output read as zeros in about 1 of 5 calls). The finish event arrives + * at the same time for short programs. No signal for "output landed" is + * known, so the wait is a fixed margin of about 8x the lag. + */ +static unsigned int call_settle_us = 1000; +module_param(call_settle_us, uint, 0644); +MODULE_PARM_DESC(call_settle_us, + "Microseconds to wait after a CALL completes so its output lands (default 1000, 0 = none)"); + +/* trace_td: a read-only timeline of each CALL for performance work. Off + * by default; switch it at runtime with + * /sys/module/ane_t6021/parameters/trace_td (no device needed). Off, the + * CALL path is unchanged. On, the completion wait polls every 20-40 us + * and also reads the last-committed-TD word. It reads that word only + * while the seven ANE pmgr PS words read 0x3ff. No register is written. + * + * Provenance (omarchy-ane commits): + * - The TD word: engine + ane_t602x_soc.trace_td_off; on T602x TM + * 0x285c00000 + TD window 0x20400 + 0x58, the word the CALL wait polled + * from be2cf130d761ed675ad86f12b21ac1d53d906e1d until + * 3a942d6cf6278526fbc02bf0c4743c5c1b276cdb. 3a942d6 measured its + * layout: the call's nid in bits 23:16 (+1 per call), the index of the + * last task taken in bits 15:0 (receipts/2026-09-30-t6021-call-wait). + * Not known on T8112, so trace_td records nothing there. + * - The seven ANE power-state words at ane_t602x_soc.pmu_pa + ps_off + * (T602x: DT power-domains ane_sys_mpm 0x4000 .. ane_set4 0x4030 of + * the pmgr at 0x28e080000). The guard "PS words 0x3ff before any TM + * read; a TM read while the compute domains are off hangs the SoC" is + * ane_rtclient_pm_pwrstate_ok in 27e996a6de544a803a71d7a5c4ed11d828d4d049, + * and the CALL wait applied it until 3a942d6. + * + * Each record has a ktime_get_ns() stamp: CALL before the exchange, ACK + * when the firmware acked it, TD for each new TD word value (word = the + * value), EVENT for each IO_T2H event of the CALL (word = its state), + * GATE when a PS word reads other than 0x3ff (word = its offset), DONE + * when the wait ends (word = the TD samples taken). The buffer is + * allocated at the first switch-on and kept until unload; each switch-on + * empties it, and records past its end are counted in `dropped`. Read + * it, while no CALL runs, from debugfs ane_t6021/trace_td (0400). All + * trace state is protected by ane_t6021_fw_lock. + */ +#define ANE_TRACE_MAGIC 0x31445441 /* "ATD1" */ +#define ANE_TRACE_RECS (1U << 18) +#define ANE_PMGR_PS_LAST_OFF 0x30 + +enum { + ANE_TR_CALL = 1, + ANE_TR_ACK, + ANE_TR_TD, + ANE_TR_EVENT, + ANE_TR_GATE, + ANE_TR_DONE, +}; + +struct ane_t6021_trace_rec { + u64 t_ns; + u32 word; + u16 kind; + u16 call; +}; + +struct ane_t6021_trace { + u32 magic; + u32 rec_size; + u32 capacity; + u32 n; + u32 dropped; + u32 calls; + u64 reserved; + struct ane_t6021_trace_rec r[]; +}; + +static bool trace_td; +static bool ane_t6021_tracing; /* the running CALL is traced */ +static struct ane_t6021_trace *ane_t6021_trace; +static struct debugfs_blob_wrapper ane_t6021_trace_blob; +static struct dentry *ane_t6021_trace_dir; + +static void ane_t6021_trace_add(u16 kind, u32 word) +{ + struct ane_t6021_trace *t = ane_t6021_trace; + + if (t->n == t->capacity) { + t->dropped++; + return; + } + t->r[t->n++] = (struct ane_t6021_trace_rec){ + .t_ns = ktime_get_ns(), .word = word, .kind = kind, + .call = t->calls, + }; +} + +static int ane_t6021_trace_set(const char *val, const struct kernel_param *kp) +{ + size_t size = struct_size_t(struct ane_t6021_trace, r, ANE_TRACE_RECS); + bool on; + int ret; + + ret = kstrtobool(val, &on); + if (ret) + return ret; + mutex_lock(&ane_t6021_fw_lock); + if (on && !ane_t6021_trace) { + ane_t6021_trace = vzalloc(size); + if (!ane_t6021_trace) { + ret = -ENOMEM; + goto out; + } + ane_t6021_trace->magic = ANE_TRACE_MAGIC; + ane_t6021_trace->rec_size = sizeof(struct ane_t6021_trace_rec); + ane_t6021_trace->capacity = ANE_TRACE_RECS; + ane_t6021_trace_blob.data = ane_t6021_trace; + ane_t6021_trace_blob.size = size; + ane_t6021_trace_dir = debugfs_create_dir("ane_t6021", NULL); + debugfs_create_blob("trace_td", 0400, ane_t6021_trace_dir, + &ane_t6021_trace_blob); + } + if (on && !trace_td) { + ane_t6021_trace->n = 0; + ane_t6021_trace->dropped = 0; + ane_t6021_trace->calls = 0; + } + trace_td = on; +out: + mutex_unlock(&ane_t6021_fw_lock); + return ret; +} + +static const struct kernel_param_ops ane_t6021_trace_ops = { + .set = ane_t6021_trace_set, + .get = param_get_bool, +}; +module_param_cb(trace_td, &ane_t6021_trace_ops, &trace_td, 0644); +MODULE_PARM_DESC(trace_td, + "Record a read-only per-CALL TD-word timeline in debugfs ane_t6021/trace_td (default 0; T602x only)"); + +static void ane_t6021_trace_free(void) +{ + debugfs_remove_recursive(ane_t6021_trace_dir); + vfree(ane_t6021_trace); +} + +/* The CALL cookie (CALL +0x20); the firmware returns it in the call's + * IO_T2H events. + */ +#define ANE_CALL_COOKIE 0xADD0 + +/* IO_T2H event of a PROCEDURE_CALL, 0x28 bytes (measured 2026-09-30, + * receipts/2026-09-30-t6021-call-wait): u32 sequence, u32 0x300, u64 + * cookie, u32 program id, u32 process id, u32 0, u32 state. The firmware + * posts two per call: state 0 about 0.2-0.5 ms after the ack whatever the + * program length (a 3,597-task program takes its last task 252 ms later; + * receipts/2026-10-01-t6021-trace-td), and state 1 when the procedure has + * finished. + */ +#define ANE_T2H_CALL_COOKIE_OFF 0x08 +#define ANE_T2H_CALL_STATE_OFF 0x1c +#define ANE_T2H_CALL_FINISHED 1 + +/* The CPU address of LEN bytes at the firmware IOVA, or NULL. The firmware + * places T2H payloads in memory the host gave it: a SHAREDMALLOC buffer or + * the 'IPC ' surface. + */ +static const void *ane_rtclient_fw_cpu(struct ane_rtclient *ane, u64 iova, + size_t len) +{ + u32 i; + + for (i = 0; i < ane->legacy_allocated; i++) { + const struct ane_legacy_buffer *b = &ane->legacy_buffers[i]; + + if (iova >= b->dma && iova - b->dma + len <= b->size) + return b->cpu + (iova - b->dma); + } + if (iova >= ane->fw->boot_ipc_iova && + iova - ane->fw->boot_ipc_iova + len <= ane->fw->boot_ipc_size) + return ane->fw->boot_ipc + (iova - ane->fw->boot_ipc_iova); + return NULL; +} + +/* Return every firmware-owned slot on a target-to-host ring to the fw + * (legacy_t2h_ack=1, the lab default): the fw publishes by writing the + * DMA address with bit0 clear; the host returns the slot by setting + * bit0 and ringing the channel's doorbell bit, as the allocation ring + * does. The sequencer drained channels 4/6 after every step; the + * ioctls drain them after every completed exchange. Returns true when + * an IO_T2H slot held the finish event of a CALL. + */ +static bool ane_rtclient_drain_t2h(struct ane_rtclient *ane, + unsigned int channel) +{ + const struct ane_t6021_chman_static *c = &ane_t6021_chman_layout[channel]; + unsigned int n = 0, slot_i = ane->legacy_cmd_cursor[channel]; + void __iomem *ipi = ane->engine + ANE_IPI_OFF; + bool finished = false; + + if (!ane->fw || !ane->fw->boot_ipc) + return false; + while (n < c->size) { + u64 *slot = ane->fw->boot_ipc + c->off + (size_t)slot_i * 64; + u64 hdr = READ_ONCE(slot[0]), len = READ_ONCE(slot[1]); + + if (hdr & 1) + break; + dma_rmb(); + dev_dbg(ane->dev, "T2H ch=%s slot=%u hdr=%016llx len=%#llx\n", + c->name, slot_i, hdr, len); + if (channel == 6 && len >= ANE_T2H_CALL_STATE_OFF + 4) { + const u8 *ev = ane_rtclient_fw_cpu(ane, hdr, + ANE_T2H_CALL_STATE_OFF + 4); + + if (ev && + get_unaligned_le64(ev + ANE_T2H_CALL_COOKIE_OFF) == + ANE_CALL_COOKIE) { + u32 state = get_unaligned_le32(ev + + ANE_T2H_CALL_STATE_OFF); + + if (ane_t6021_tracing) + ane_t6021_trace_add(ANE_TR_EVENT, state); + if (state == ANE_T2H_CALL_FINISHED) + finished = true; + } + } + n++; + WRITE_ONCE(slot[0], hdr | 1); + dma_wmb(); + writel(BIT(c->bit), ipi); + slot_i = (slot_i + 1) % c->size; + ane->legacy_cmd_cursor[channel] = slot_i; + } + return finished; +} + +/* The completion wait with trace_td on: the same finish-event test with a + * 20-40 us poll, and one TD-word sample per poll under the PS-word guard + * (see trace_td). + */ +static int ane_rtclient_call_wait_traced(struct ane_rtclient *ane, + unsigned long deadline) +{ + void __iomem *td = ane->engine + ane->soc->trace_td_off; + void __iomem *ps = ioremap_np(ane->soc->pmu_pa + ane->soc->ps_off, + ANE_PMGR_PS_LAST_OFF + 4); + unsigned int i, gate = ANE_PMGR_PS_LAST_OFF + 8; + u32 last = U32_MAX, samples = 0; + int ret = -ETIMEDOUT; + + do { + if (ane_rtclient_drain_t2h(ane, 6)) { + ret = 0; + break; + } + if (ps) { + for (i = 0; i <= ANE_PMGR_PS_LAST_OFF; i += 8) + if ((readl(ps + i) & 0x3ff) != 0x3ff) + break; + if (i > ANE_PMGR_PS_LAST_OFF) { + u32 w = readl(td); + + samples++; + if (w != last) + ane_t6021_trace_add(ANE_TR_TD, w); + last = w; + } else if (i != gate) { + ane_t6021_trace_add(ANE_TR_GATE, i); + } + gate = i; + } + usleep_range(20, 40); + } while (time_before(jiffies, deadline)); + if (ret && ane_rtclient_drain_t2h(ane, 6)) + ret = 0; + ane_t6021_trace_add(ANE_TR_DONE, samples); + if (ps) + iounmap(ps); + return ret; +} + +/* Completion wait for one PROCEDURE_CALL: the finish event on IO_T2H + * (channel 6). The ack, the eight TQ status words and the last-committed + * TD word (TM +0x20458: the call's nid in bits 23:16 and the index of the + * last task taken in bits 15:0) all report done once the last task has + * been dispatched, not when it has run: measured 2026-09-30, Qwen program + * 20 (20 tasks) showed its last task index 0.22 ms after the ack, posted + * its finish event 3.5 ms after the ack, and a caller that returned at + * the first signal read an all-zero output. Returns 0 when the event + * arrived, -ETIMEDOUT else. + */ +static int ane_rtclient_call_wait(struct ane_rtclient *ane, + unsigned int timeout_ms) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); + + if (ane_t6021_tracing) + return ane_rtclient_call_wait_traced(ane, deadline); + + do { + if (ane_rtclient_drain_t2h(ane, 6)) + return 0; + usleep_range(50, 100); + } while (time_before(jiffies, deadline)); + return ane_rtclient_drain_t2h(ane, 6) ? 0 : -ETIMEDOUT; +} + +static int ane_rtclient_command(struct ane_rtclient *ane, + struct ane_legacy_buffer *command, + size_t length, u16 opcode, + unsigned int channel, + unsigned int timeout_ms) +{ + int ret; + + ane_t6021_tracing = opcode == CSNE_CMD_PROCEDURE_CALL && trace_td && + ane->soc->trace_td_off; + if (ane_t6021_tracing) { + ane_t6021_trace->calls++; + ane_t6021_trace_add(ANE_TR_CALL, 0); + } + ret = ane_rtclient_legacy_exchange(ane, command, length, opcode, + channel, timeout_ms); + if (ret) { + ane_t6021_tracing = false; + dev_info(ane->dev, "EXCH op=%#x failed %d (fw allocs %u, %zu bytes)\n", + opcode, ret, ane->legacy_allocated, ane->legacy_bytes); + atomic_set(&ane_t6021_quarantined, 1); + return ret; + } + if (opcode == CSNE_CMD_PROCEDURE_CALL) { + if (ane_t6021_tracing) + ane_t6021_trace_add(ANE_TR_ACK, 0); + ret = ane_rtclient_call_wait(ane, timeout_ms); + ane_t6021_tracing = false; + if (!ret && call_settle_us) + usleep_range(call_settle_us, call_settle_us + 100); + if (ret) { + dev_info(ane->dev, "call completion wait failed %d\n", + ret); + atomic_set(&ane_t6021_quarantined, 1); + return ret; + } + } + /* The fw talks back on the target-to-host rings (fwlog, perf); + * hand those slots back so the rings never fill (the sequencer + * did this per step; same ack, channels 4 and 6). + */ + ane_rtclient_drain_t2h(ane, 4); + ane_rtclient_drain_t2h(ane, 6); + /* BOs are dma_alloc_coherent memory mapped write-combined for the + * CPU, so no cache maintenance is needed on either side. + */ + return 0; +} + +/* ---- LOAD / CREATE / CALL wrappers ---- */ + +/* The firmware never frees a program or a process, and its program table + * holds 256 entries: measured 2026-09-29, the 257th LOAD_PROGRAM on one + * boot answered with a protocol error and quarantined the device. Two + * loads of byte-identical sections therefore share one firmware program + * and one process. The key is SHA-256 over every section's id, size and + * bytes, so a client can only reach a program whose bytes it also + * supplied. Protected by ane_t6021_fw_lock. + */ +#define ANE_T6021_MAX_PROGRAMS 250 + +struct ane_t6021_prog { + u8 digest[SHA256_DIGEST_SIZE]; + u32 prog_id; + u32 proc_id; /* U32_MAX until a process exists */ +}; + +static struct ane_t6021_prog ane_t6021_progs[ANE_T6021_MAX_PROGRAMS]; +static unsigned int ane_t6021_nprogs; + +static struct ane_t6021_prog *ane_t6021_prog_find(const u8 *digest) +{ + unsigned int i; + + for (i = 0; i < ane_t6021_nprogs; i++) + if (!memcmp(ane_t6021_progs[i].digest, digest, + SHA256_DIGEST_SIZE)) + return &ane_t6021_progs[i]; + return NULL; +} + +static struct ane_t6021_prog *ane_t6021_prog_by_id(u32 prog_id) +{ + unsigned int i; + + for (i = 0; i < ane_t6021_nprogs; i++) + if (ane_t6021_progs[i].prog_id == prog_id) + return &ane_t6021_progs[i]; + return NULL; +} + +/* Build a LOAD_PROGRAM (0x200) message in a kernel-owned buffer. The + * section bytes live in the BOs the user supplied (section_ptr), the + * generic binds (bufferId, bo_handle, size) patch the generic section + * at iova +0x20 / +0x28 once the BO is copied. Returns 0 with + * *prog_id set on success. + */ +static int ane_rtclient_load_program(struct ane_rtclient *ane, + struct drm_file *file, + const struct drm_ane_prog_load *user, + __u32 *prog_id) +{ + struct drm_ane_section *sections; + struct drm_ane_generic_bind *binds; + struct ane_legacy_buffer *command; + struct ane_t6021_fd *fd = file->driver_priv; + struct ane_t6021_prog *cached; + u8 digest[SHA256_DIGEST_SIZE]; + size_t binds_size; + int ret, i, j; + + if (user->section_count < 1 || + user->section_count > ANE_T6021_LOAD_SEC_COUNT || + user->generic_count > ANE_M2_MAX_BINDS || !fd) + return -EINVAL; + + sections = kmalloc_array(user->section_count, sizeof(*sections), + GFP_KERNEL); + binds = kmalloc_array(user->generic_count, sizeof(*binds), + GFP_KERNEL); + if (!sections || !binds) { + ret = -ENOMEM; + goto out; + } + if (copy_from_user(sections, + u64_to_user_ptr(user->sections_ptr), + user->section_count * sizeof(*sections))) { + ret = -EFAULT; + goto out; + } + binds_size = user->generic_count * sizeof(*binds); + if (binds_size && copy_from_user(binds, + u64_to_user_ptr(user->generic_ptr), + binds_size)) { + ret = -EFAULT; + goto out; + } + + /* Lab contract: the record slot IS the section identity (slot = + * id - 1; tdprop id 7 lands at slot 6 and slot 5 stays empty, + * exactly the proven add-path image). Reject out-of-range and + * duplicate ids before building the wire image. + */ + for (i = 0; i < user->section_count; i++) { + if (sections[i].id < 1 || + sections[i].id > ANE_T6021_LOAD_SEC_COUNT) { + ret = -EINVAL; + goto out; + } + for (j = i + 1; j < user->section_count; j++) { + if (sections[i].id == sections[j].id) { + ret = -EINVAL; + goto out; + } + } + } + + /* Identical sections share one firmware program (see the table). + * A constant section can reach 20 MiB (the H14 (K,N)=(2048,5120) + * matvec), so feed the hash in ANE_T6021_BO_HASH_CHUNK-sized slices + * under bo_lock: a single sha256_update of a 20 MiB buffer would + * rely on a 20 MiB stack argument list and has no upper bound on + * the chunk that the BO could supply. + */ + { + struct sha256_ctx sha; + void *scratch; + + sha256_init(&sha); + scratch = kvmalloc(ANE_T6021_BO_HASH_CHUNK, GFP_KERNEL); + if (!scratch) { + ret = -ENOMEM; + goto out; + } + for (i = 0; i < user->section_count; i++) { + struct ane_t6021_bo *bo = NULL, *b; + u64 hdr[2] = { sections[i].id, sections[i].size }; + size_t left; + + mutex_lock(&ane_t6021_bo_lock); + list_for_each_entry(b, &fd->bos, node) { + if (b->handle == sections[i].bo_handle) { + bo = b; + break; + } + } + if (!bo || sections[i].size > bo->size || + sections[i].offset > bo->size - sections[i].size) { + mutex_unlock(&ane_t6021_bo_lock); + kvfree(scratch); + ret = -EINVAL; + goto out; + } + sha256_update(&sha, (const u8 *)hdr, sizeof(hdr)); + left = sections[i].size; + while (left) { + size_t n = min(left, (size_t)ANE_T6021_BO_HASH_CHUNK); + + memcpy(scratch, (const u8 *)bo->cpu + + sections[i].offset + + (sections[i].size - left), n); + sha256_update(&sha, scratch, n); + left -= n; + } + mutex_unlock(&ane_t6021_bo_lock); + } + kvfree(scratch); + sha256_final(&sha, digest); + } + cached = ane_t6021_prog_find(digest); + if (cached) { + *prog_id = cached->prog_id; + ret = 0; + goto out; + } + if (ane_t6021_nprogs == ANE_T6021_MAX_PROGRAMS) { + ret = -ENOSPC; + goto out; + } + + command = ane->cmd_buf; + if (!command) { + ret = -ENODEV; + goto out; + } + memset(command->cpu, 0, SZ_16K); + /* One 0x30-byte section record per supplied section, placed at + * its IDENTITY slot: slot = id - 1 (tdprop id 7 lands at slot + * 6, slot 5 stays empty — exactly the proven add-path image: + * six populated records, ids 1,2,3,4,5 then 7). Unsupplied + * slots stay zero-filled (flags = 0). flags bit0 = 1, id at + * +0x04, iova at +0x18, size at +0x20. + */ + for (i = 0; i < user->section_count; i++) { + struct ane_t6021_bo *bo = NULL, *b; + u64 slot_base, iova; + u8 *cmd = command->cpu; + + mutex_lock(&ane_t6021_bo_lock); + list_for_each_entry(b, &fd->bos, node) { + if (b->handle == sections[i].bo_handle) { + bo = b; + break; + } + } + if (bo && sections[i].size <= bo->size && + sections[i].offset <= bo->size - sections[i].size) { + /* The IOVA below is about to be published to the + * firmware, so mark the BO before the exchange: + * the fw may read the section any time after the + * doorbell rings, including during a timeout. + * Under bo_lock the handle reference cannot go + * away, so no kref is needed here. + */ + bo->fw_ref = true; + bo->fw_program = true; + iova = bo->dma + sections[i].offset; + } else { + bo = NULL; + } + mutex_unlock(&ane_t6021_bo_lock); + if (!bo) { + ret = -EINVAL; + goto out; + } + slot_base = 0x8 + (u64)(sections[i].id - 1) * 0x30; + *(u32 *)(cmd + slot_base + 0x00) = cpu_to_le32(1); + *(u32 *)(cmd + slot_base + 0x04) = + cpu_to_le32(sections[i].id); + *(u64 *)(cmd + slot_base + 0x18) = cpu_to_le64(iova); + *(u64 *)(cmd + slot_base + 0x20) = + cpu_to_le64(sections[i].size); + } + /* Generic binds are accepted for ABI compatibility and unused: + * the LOAD record already carries iova + size, and the generic + * section image is prepared by userspace (the proven add path + * ships it inside the section bytes, generic.bin). + */ + + /* ProgramId placeholder; the firmware writes the assigned id + * back at +0x1b8. + */ + { + u8 *cmd = command->cpu; + + *(u32 *)(cmd + 0x1b8) = cpu_to_le32(U32_MAX); + } + + /* The proven add-path LOAD command is 0x1C0 bytes: the nine + * 0x30 records end at 0x1b8, ProgramId at +0x1b8, 8 bytes of + * zero tail. sizeof(struct ane_csne_cmd_load_program) is only + * 0x1b8, so the length is pinned here (h14_seq_first_add.py + * load_step). + */ + BUILD_BUG_ON(sizeof(struct ane_csne_cmd_load_program) != 0x1b8); + ret = ane_rtclient_command(ane, command, + 0x1c0, + CSNE_CMD_LOAD_PROGRAM, + 1, 5000); + if (!ret) { + u8 *cmd = command->cpu; + + *prog_id = le32_to_cpu(*(u32 *)(cmd + 0x1b8)); + if (*prog_id == U32_MAX) { + ret = -EPROTO; + goto out; + } + memcpy(ane_t6021_progs[ane_t6021_nprogs].digest, digest, + SHA256_DIGEST_SIZE); + ane_t6021_progs[ane_t6021_nprogs].prog_id = *prog_id; + ane_t6021_progs[ane_t6021_nprogs].proc_id = U32_MAX; + ane_t6021_nprogs++; + } + +out: + kfree(sections); + kfree(binds); + return ret; +} + +static int ane_rtclient_create_process(struct ane_rtclient *ane, + __u32 prog_id, __u32 *proc_id) +{ + struct ane_legacy_buffer *command; + struct ane_t6021_prog *prog = ane_t6021_prog_by_id(prog_id); + int ret; + + if (!prog) + return -ENOENT; + if (prog->proc_id != U32_MAX) { + *proc_id = prog->proc_id; + return 0; + } + command = ane->cmd_buf; + if (!command) + return -ENODEV; + memset(command->cpu, 0, SZ_16K); + { + u8 *cmd = command->cpu; + + *(u32 *)(cmd + 0x08) = cpu_to_le32(prog_id); + *(u32 *)(cmd + 0x0c) = cpu_to_le32(U32_MAX); + } + ret = ane_rtclient_command(ane, command, 0x10, + CSNE_CMD_CREATE_PROCESS, + 1, 3000); + if (!ret) { + u8 *cmd = command->cpu; + + *proc_id = le32_to_cpu(*(u32 *)(cmd + 0x0c)); + if (*proc_id == U32_MAX) + ret = -EPROTO; + else + prog->proc_id = *proc_id; + } + return ret; +} + +static int ane_rtclient_procedure_call(struct ane_rtclient *ane, + struct drm_file *file, + const struct drm_ane_exec *user) +{ + struct drm_ane_exec_io *ios; + struct ane_legacy_buffer *command; + struct ane_t6021_fd *fd = file->driver_priv; + size_t ios_size; + size_t cmd_size; + int ret, i; + + if (user->count < 1 || user->count > ANE_M2_MAX_BINDS || + user->priority < 2 || user->priority > 7 || !fd) + return -EINVAL; + + ios_size = (size_t)user->count * sizeof(*ios); + ios = kmalloc(ios_size, GFP_KERNEL); + if (!ios) + return -ENOMEM; + if (copy_from_user(ios, u64_to_user_ptr(user->io_ptr), ios_size)) { + kfree(ios); + return -EFAULT; + } + + cmd_size = sizeof(struct ane_csne_cmd_procedure_call) + + (size_t)user->count * sizeof(struct ane_csne_io_elem); + if (cmd_size > SZ_16K) { + kfree(ios); + return -E2BIG; + } + + mutex_lock(&ane_t6021_fw_lock); + if (atomic_read(&ane_t6021_quarantined)) { + mutex_unlock(&ane_t6021_fw_lock); + kfree(ios); + return -ETIMEDOUT; + } + command = ane->cmd_buf; + if (!command) { + mutex_unlock(&ane_t6021_fw_lock); + kfree(ios); + return -ENODEV; + } + memset(command->cpu, 0, SZ_16K); + { + u8 *cmd = command->cpu; + + *(u32 *)(cmd + 0x08) = cpu_to_le32(user->prog_id); + *(u32 *)(cmd + 0x0c) = cpu_to_le32(user->proc_id); + *(u64 *)(cmd + 0x10) = cpu_to_le64(0); + *(u32 *)(cmd + 0x18) = cpu_to_le32(user->priority); + *(u64 *)(cmd + 0x20) = cpu_to_le64(ANE_CALL_COOKIE); + *(u32 *)(cmd + 0x28) = cpu_to_le32(user->count); + for (i = 0; i < user->count; i++) { + struct ane_t6021_bo *bo = NULL, *b; + u64 slot_base = 0x60 + (u64)i * 0x30; + u64 iova; + + mutex_lock(&ane_t6021_bo_lock); + list_for_each_entry(b, &fd->bos, node) { + if (b->handle == ios[i].bo_handle) { + bo = b; + break; + } + } + if (!bo || ios[i].size > bo->size) { + mutex_unlock(&ane_t6021_bo_lock); + ret = -EINVAL; + goto unlock; + } + /* The IOVA below is about to be published to the + * firmware, so mark the BO before the exchange. + * Under bo_lock the handle reference cannot go + * away, so no kref is needed here. + */ + bo->fw_ref = true; + iova = bo->dma; + mutex_unlock(&ane_t6021_bo_lock); + *(u32 *)(cmd + slot_base + 0x00) = cpu_to_le32(1); + *(u32 *)(cmd + slot_base + 0x04) = + cpu_to_le32(ios[i].buffer_id); + *(u32 *)(cmd + slot_base + 0x08) = + cpu_to_le32(ios[i].type); + *(u64 *)(cmd + slot_base + 0x18) = + cpu_to_le64(iova); + *(u64 *)(cmd + slot_base + 0x20) = + cpu_to_le64(ios[i].size); + } + /* Drain the CPU write buffers so the input BOs the user + * filled through its uncached mapping are in DRAM before + * the fw starts reading them. + */ + wmb(); + ret = ane_rtclient_command(ane, command, + cmd_size, + CSNE_CMD_PROCEDURE_CALL, + 1, + user->timeout_ms ? + user->timeout_ms : 5000); + } +unlock: + mutex_unlock(&ane_t6021_fw_lock); + kfree(ios); + return ret; +} + +/* ---- DRM accel glue (minimal BO + ioctl table) ---- */ + +struct ane_t6021_drm { + struct drm_device drm; + struct device *dev; + struct ane_rtclient *ane; +}; + +static struct ane_t6021_drm *to_ane_t6021_drm(struct drm_device *drm) +{ + return container_of(drm, struct ane_t6021_drm, drm); +} + +static int ane_t6021_bo_init_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct drm_ane_bo_init *args = data; + struct ane_t6021_fd *fd = file->driver_priv; + struct ane_t6021_bo *bo; + struct ane_rtclient *ane; + + if (args->size == 0 || args->size > ANE_T6021_BO_MAX || !fd) + return -EINVAL; + /* A parked io BO is already mapped and counted; its old contents + * belong to another process, so it is zeroed like a new one. + */ + bo = ane_t6021_bo_pool_take(args->size); + if (bo) { + memset(bo->cpu, 0, PAGE_ALIGN(args->size)); + goto publish; + } + /* Global coherent-memory accounting. Each BO is 16 KiB-aligned; + * a BO whose IOVA reaches the firmware is never freed (held or + * pooled), so this bound caps the memory that outlives its + * users. + */ + if (atomic64_add_return(PAGE_ALIGN(args->size), &ane_t6021_bo_total_bytes) > + (s64)bo_total_max_mb << 20) { + atomic64_sub(PAGE_ALIGN(args->size), &ane_t6021_bo_total_bytes); + return -ENOSPC; + } + ane = to_ane_t6021_drm(drm)->ane; + bo = kzalloc(sizeof(*bo), GFP_KERNEL); + if (!bo) { + atomic64_sub(PAGE_ALIGN(args->size), &ane_t6021_bo_total_bytes); + return -ENOMEM; + } + bo->cpu = dma_alloc_coherent(drm->dev, args->size, &bo->dma, + GFP_KERNEL); + if (!bo->cpu) { + atomic64_sub(PAGE_ALIGN(args->size), &ane_t6021_bo_total_bytes); + kfree(bo); + return -ENOMEM; + } + /* Every fw-visible DMA surface must be 16 KiB aligned and clear + * of the firmware entry alias (receipt 2026-09-20-t6021-entry-alias: + * the same invariant ane_rtclient_legacy_alloc enforces). + */ + if (!IS_ALIGNED(bo->dma, SZ_16K) || + (ane->fw && !ane_t6021_fw_alias_iova_ok(ane->fw, bo->dma, + args->size))) { + dma_free_coherent(drm->dev, args->size, bo->cpu, bo->dma); + atomic64_sub(PAGE_ALIGN(args->size), &ane_t6021_bo_total_bytes); + kfree(bo); + return -ERANGE; + } +publish: + bo->size = args->size; + bo->owner = fd; + bo->dev = drm->dev; + kref_init(&bo->refcount); /* the handle holds this reference */ + mutex_lock(&ane_t6021_bo_lock); + bo->handle = ane_t6021_next_handle++; + if (bo->handle == 0) + bo->handle = ane_t6021_next_handle++; + list_add_tail(&bo->node, &fd->bos); + mutex_unlock(&ane_t6021_bo_lock); + args->handle = bo->handle; + /* mmap offset = the handle; libane mmaps the fd at exactly this + * offset and ane_t6021_mmap resolves the BO from vm_pgoff. + */ + args->offset = (u64)bo->handle << PAGE_SHIFT; + return 0; +} + +/* Drop one handle owned by this fd. The kref may keep the memory + * alive past this call — a user mapping still holds a reference — so + * the final ane_t6021_bo_release makes the free-or-hold decision. + */ +static void ane_t6021_bo_drop(struct ane_t6021_bo *bo) +{ + list_del(&bo->node); + kref_put(&bo->refcount, ane_t6021_bo_release); +} + +static int ane_t6021_bo_free_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct drm_ane_bo_free *args = data; + struct ane_t6021_fd *fd = file->driver_priv; + struct ane_t6021_bo *bo = NULL, *b; + + if (!fd) + return -ENODEV; + mutex_lock(&ane_t6021_bo_lock); + list_for_each_entry(b, &fd->bos, node) { + if (b->handle == args->handle) { + bo = b; + break; + } + } + if (bo) + ane_t6021_bo_drop(bo); + mutex_unlock(&ane_t6021_bo_lock); + return bo ? 0 : -ENOENT; +} + +/* Resolve the BO a user mmap names (BO_INIT returned offset = handle + * << PAGE_SHIFT) and map the coherent buffer cacheably — the device + * half coheres through the DART (IOMMU_CACHE), so reads after EXEC + * observe the firmware's writes without extra sync. The mapping takes + * one BO reference: it survives BO_FREE until the vma is gone. + */ +static int ane_t6021_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct drm_file *file = filp->private_data; + struct ane_t6021_fd *fd = file->driver_priv; + struct drm_device *drm = file->minor->dev; + struct ane_t6021_bo *bo = NULL, *b; + size_t size = vma->vm_end - vma->vm_start; + u32 handle; + int ret; + + if (!fd) + return -ENODEV; + /* vm_pgoff is the BO_INIT offset in pages: handle << PAGE_SHIFT + * >> PAGE_SHIFT == handle. + */ + handle = (u32)vma->vm_pgoff; + if (!handle) + return -EINVAL; + mutex_lock(&ane_t6021_bo_lock); + list_for_each_entry(b, &fd->bos, node) { + if (b->handle == handle && b->owner == fd) { + kref_get(&b->refcount); + bo = b; + break; + } + } + if (!bo || size > PAGE_ALIGN(bo->size)) { + mutex_unlock(&ane_t6021_bo_lock); + if (bo) + kref_put(&bo->refcount, ane_t6021_bo_release); + return -ENOENT; + } + vma->vm_pgoff = 0; + ret = dma_mmap_coherent(drm->dev, vma, bo->cpu, bo->dma, size); + mutex_unlock(&ane_t6021_bo_lock); + if (ret) { + kref_put(&bo->refcount, ane_t6021_bo_release); + return ret; + } + /* The vma owns the lookup reference: close (and open on fork) + * go through ane_t6021_vm_ops. + */ + vma->vm_private_data = bo; + vma->vm_ops = &ane_t6021_vm_ops; + return 0; +} + +static int ane_t6021_open(struct drm_device *drm, struct drm_file *file) +{ + struct ane_t6021_fd *fd; + + fd = kzalloc(sizeof(*fd), GFP_KERNEL); + if (!fd) + return -ENOMEM; + INIT_LIST_HEAD(&fd->bos); + file->driver_priv = fd; + return 0; +} + +static void ane_t6021_postclose(struct drm_device *drm, struct drm_file *file) +{ + struct ane_t6021_fd *fd = file->driver_priv; + struct ane_t6021_bo *bo, *tmp; + + if (!fd) + return; + mutex_lock(&ane_t6021_bo_lock); + list_for_each_entry_safe(bo, tmp, &fd->bos, node) + ane_t6021_bo_drop(bo); + mutex_unlock(&ane_t6021_bo_lock); + kfree(fd); + file->driver_priv = NULL; +} + +static int ane_t6021_submit_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + return -ENOTTY; /* ABI 1 SUBMIT is rejected on T6021/M2 */ +} + +static int ane_t6021_prog_load_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct ane_t6021_drm *adrm = to_ane_t6021_drm(drm); + struct drm_ane_prog_load *args = data; + int ret; + + mutex_lock(&ane_t6021_fw_lock); + if (atomic_read(&ane_t6021_quarantined)) { + mutex_unlock(&ane_t6021_fw_lock); + return -ETIMEDOUT; + } + ret = ane_rtclient_load_program(adrm->ane, file, args, + &args->prog_id_out); + mutex_unlock(&ane_t6021_fw_lock); + return ret; +} + +/* Firmware perf mode: CSNE_CMD_CH_PROPERTY_WRITE (0x1f), channel 0, + * property 0x10aa, value 1. macOS sends it once during power-on (H13 + * kext evidence in the omarchy-ane perf-mode receipt); the selene 13.5 + * fw routes it to CAneEngineExeLoop::setPerfMode. It is a runtime + * switch (write 1 to /sys/module/ane_t6021/parameters/fw_perf_mode) so + * its effect on call time can be measured on one boot. Only 1 is + * accepted: no other value is known to be safe. + */ +static struct ane_rtclient *ane_t6021_perf_ane; +static bool fw_perf_mode; + +static int ane_t6021_perf_mode_set(const char *val, + const struct kernel_param *kp) +{ + struct ane_rtclient *ane = READ_ONCE(ane_t6021_perf_ane); + struct ane_legacy_buffer *command; + bool on; + int ret; + + ret = kstrtobool(val, &on); + if (ret) + return ret; + if (!on || fw_perf_mode) + return on ? 0 : -EINVAL; + if (!ane) + return -ENODEV; + mutex_lock(&ane_t6021_fw_lock); + if (atomic_read(&ane_t6021_quarantined)) { + mutex_unlock(&ane_t6021_fw_lock); + return -ETIMEDOUT; + } + command = ane->cmd_buf; + if (!command) { + mutex_unlock(&ane_t6021_fw_lock); + return -ENODEV; + } + memset(command->cpu, 0, SZ_16K); + *(u32 *)((u8 *)command->cpu + 0x08) = cpu_to_le32(0); + *(u32 *)((u8 *)command->cpu + 0x0c) = cpu_to_le32(0x10aa); + *(u32 *)((u8 *)command->cpu + 0x10) = cpu_to_le32(1); + ret = ane_rtclient_command(ane, command, 0x14, 0x001f, 1, 3000); + if (!ret) { + fw_perf_mode = true; + dev_info(ane->dev, "fw perf mode set (property 0x10aa = 1)\n"); + } + mutex_unlock(&ane_t6021_fw_lock); + return ret; +} + +static const struct kernel_param_ops ane_t6021_perf_mode_ops = { + .set = ane_t6021_perf_mode_set, + .get = param_get_bool, +}; +module_param_cb(fw_perf_mode, &ane_t6021_perf_mode_ops, &fw_perf_mode, 0644); +MODULE_PARM_DESC(fw_perf_mode, + "Write 1 once to send CH_PROPERTY_WRITE 0x10aa = 1 (fw perf mode); reads back whether it was sent"); + +static int ane_t6021_proc_create_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct ane_t6021_drm *adrm = to_ane_t6021_drm(drm); + struct drm_ane_proc_create *args = data; + int ret; + + mutex_lock(&ane_t6021_fw_lock); + if (atomic_read(&ane_t6021_quarantined)) { + mutex_unlock(&ane_t6021_fw_lock); + return -ETIMEDOUT; + } + args->proc_id_out = 0; + ret = ane_rtclient_create_process(adrm->ane, args->prog_id, + &args->proc_id_out); + mutex_unlock(&ane_t6021_fw_lock); + return ret; +} + +static int ane_t6021_exec_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct ane_t6021_drm *adrm = to_ane_t6021_drm(drm); + struct drm_ane_exec *args = data; + + return ane_rtclient_procedure_call(adrm->ane, file, args); +} + +static const struct drm_ioctl_desc ane_t6021_ioctls[] = { + DRM_IOCTL_DEF_DRV(ANE_BO_INIT, ane_t6021_bo_init_ioctl, 0), + DRM_IOCTL_DEF_DRV(ANE_BO_FREE, ane_t6021_bo_free_ioctl, 0), + DRM_IOCTL_DEF_DRV(ANE_SUBMIT, ane_t6021_submit_ioctl, 0), + DRM_IOCTL_DEF_DRV(ANE_PROG_LOAD, ane_t6021_prog_load_ioctl, 0), + DRM_IOCTL_DEF_DRV(ANE_PROC_CREATE, ane_t6021_proc_create_ioctl, 0), + DRM_IOCTL_DEF_DRV(ANE_EXEC, ane_t6021_exec_ioctl, 0), +}; + +/* Driver fops: the accel-core entry points plus our BO mmap (the core + * default maps only GEM objects; this driver keeps its own BO table, + * so .mmap resolves the handle from BO_INIT's returned offset). + */ +static const struct file_operations ane_t6021_fops = { + .owner = THIS_MODULE, + .fop_flags = FOP_UNSIGNED_OFFSET, + .open = accel_open, + .release = drm_release, + .unlocked_ioctl = drm_ioctl, + .compat_ioctl = drm_compat_ioctl, + .poll = drm_poll, + .read = drm_read, + .llseek = noop_llseek, + .mmap = ane_t6021_mmap, +}; + +/* Version reported through DRM_IOCTL_VERSION: ABI 2 (T6021). + * DRIVER_COMPUTE_ACCEL puts the node at /dev/accel/accelN. + */ +static const struct drm_driver ane_t6021_drm_driver = { + .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL, + .open = ane_t6021_open, + .postclose = ane_t6021_postclose, + .ioctls = ane_t6021_ioctls, + .num_ioctls = ARRAY_SIZE(ane_t6021_ioctls), + .fops = &ane_t6021_fops, + .major = ANE_ABI_M2_MAJOR, + .minor = 0, + .name = "ane", + .desc = "Apple Neural Engine (T6021/M2)", +}; + +/* ---- RTKit ops (kept on the boot path; the legacy ChMan transport + * is used by the ioctls once probe confirms legacy_only / chman_ok). + */ + +static void ane_rtclient_recv(void *cookie, u8 ep, u64 message) +{ + struct ane_rtclient *ane = cookie; + + dev_info(ane->dev, + "rtkit app msg: ep=%#x msg=%016llx\n", ep, message); +} + +static void ane_rtclient_crashed(void *cookie, const void *crashlog, + size_t size) +{ + struct ane_rtclient *ane = cookie; + + dev_err(ane->dev, "rtkit: coprocessor crashed (crashlog %zu bytes)\n", + size); + print_hex_dump(KERN_ERR, "ANE crashlog: ", DUMP_PREFIX_OFFSET, 16, 1, + crashlog, min_t(size_t, size, 256), false); +} + +static int ane_rtclient_shmem_setup(void *cookie, + struct apple_rtkit_shmem *bfr) +{ + struct ane_rtclient *ane = cookie; + + if (bfr->iova) { + dev_warn(ane->dev, + "rtkit: fw-provided shmem iova=%pad size=%#zx — refused\n", + &bfr->iova, bfr->size); + return -EINVAL; + } + bfr->buffer = dma_alloc_coherent(ane->dev, bfr->size, &bfr->iova, + GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + if (ane->fw && !ane_t6021_fw_alias_iova_ok(ane->fw, bfr->iova, + bfr->size)) { + dev_err(ane->dev, + "rtkit: shmem grant %pad+%#zx overlaps the fw alias — refusing\n", + &bfr->iova, bfr->size); + dma_free_coherent(ane->dev, bfr->size, bfr->buffer, bfr->iova); + bfr->buffer = NULL; + return -EBUSY; + } + return 0; +} + +static void ane_rtclient_shmem_destroy(void *cookie, + struct apple_rtkit_shmem *bfr) +{ + struct ane_rtclient *ane = cookie; + + if (!bfr->buffer) + return; + if (ane->held) { + dev_warn(ane->dev, + "rtkit: shmem %pad HELD (CPU started) — not freed\n", + &bfr->iova); + return; + } + dma_free_coherent(ane->dev, bfr->size, bfr->buffer, bfr->iova); +} + +static const struct apple_rtkit_ops ane_rtclient_rtkit_ops = { + .crashed = ane_rtclient_crashed, + .recv_message = ane_rtclient_recv, + .shmem_setup = ane_rtclient_shmem_setup, + .shmem_destroy = ane_rtclient_shmem_destroy, +}; + +/* ---- poll worker: RX fallback while the recv line is unproven ---- + * (lab poll_rx=1). Runs every 10 ms until the handshake completes, + * then every second while poll_rx stays on. Armed BEFORE the host ack + * so the fw's HELLO is never missed. + */ +static void ane_rtclient_post_boot(struct work_struct *w) +{ + struct ane_rtclient *ane = + container_of(to_delayed_work(w), struct ane_rtclient, + poll_work); + + apple_rtkit_poll(ane->rtk); + + if (!ane->boot_done) { + schedule_delayed_work(&ane->poll_work, msecs_to_jiffies(10)); + return; + } + + if (poll_rx) + schedule_delayed_work(&ane->poll_work, HZ); +} + +/* STARTEP every fw-announced app endpoint (>= 0x20; the fw mgmt + * dispatcher starts an endpoint on flag bit 1). Same call every Asahi + * RTKit client makes; the lab ran it with start_app_eps=1. + */ +static void ane_rtclient_start_app_eps(struct ane_rtclient *ane) +{ + int ep; + + for (ep = 0x20; ep < 0x100; ep++) { + int ret; + + if (!apple_rtkit_has_endpoint(ane->rtk, ep)) + continue; + ret = apple_rtkit_start_ep(ane->rtk, ep); + dev_info(ane->dev, "rtkit: STARTEP app ep %#x -> %pe\n", + ep, ERR_PTR(ret)); + } +} + +/* Multi-domain genpd attach, ownership-correct and idempotent. */ +struct ane_rtclient_pd { + struct list_head list; + struct device *dev; + struct device **pd_dev; + struct device_link **pd_link; + int count; +}; + +static LIST_HEAD(ane_rtclient_pd_list); +static DEFINE_MUTEX(ane_rtclient_pd_lock); +static bool ane_rtclient_pinned; + +static void ane_rtclient_pd_free(struct ane_rtclient_pd *pd) +{ + list_del(&pd->list); + kfree(pd->pd_dev); + kfree(pd->pd_link); + kfree(pd); +} + +static int ane_rtclient_attach_genpd(struct ane_rtclient *ane) +{ + struct device *dev = ane->dev; + struct ane_rtclient_pd *pd = NULL; + int count, i, err = 0; + + count = of_count_phandle_with_args(dev->of_node, "power-domains", + "#power-domain-cells"); + if (count == -ENOENT) + return 0; + if (count < 0) + return count; + if (count <= 1) + return 0; + + mutex_lock(&ane_rtclient_pd_lock); + list_for_each_entry(pd, &ane_rtclient_pd_list, list) + if (pd->dev == dev) + goto found; + + pd = kzalloc(sizeof(*pd), GFP_KERNEL); + if (!pd) { + err = -ENOMEM; + goto out; + } + INIT_LIST_HEAD(&pd->list); + pd->dev = dev; + pd->pd_dev = kcalloc(count, sizeof(*pd->pd_dev), GFP_KERNEL); + pd->pd_link = kcalloc(count, sizeof(*pd->pd_link), GFP_KERNEL); + if (!pd->pd_dev || !pd->pd_link) { + ane_rtclient_pd_free(pd); + pd = NULL; + err = -ENOMEM; + goto out; + } + list_add_tail(&pd->list, &ane_rtclient_pd_list); + +found: + for (i = pd->count; i < count; i++) { + if (!pd->pd_dev[i]) { + pd->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); + if (IS_ERR_OR_NULL(pd->pd_dev[i])) { + err = IS_ERR(pd->pd_dev[i]) ? + PTR_ERR(pd->pd_dev[i]) : -ENODEV; + pd->pd_dev[i] = NULL; + goto out; + } + } + if (!pd->pd_link[i]) { + if (!ane_rtclient_pinned) { + if (!try_module_get(THIS_MODULE)) { + err = -ENODEV; + goto out; + } + ane_rtclient_pinned = true; + } + pd->pd_link[i] = + device_link_add(dev, pd->pd_dev[i], + DL_FLAG_STATELESS | + DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!pd->pd_link[i]) { + err = -EINVAL; + goto out; + } + } + pd->count = i + 1; + } + + dev_emerg(dev, "BOOT-PHASE genpd domains attached: %d\n", count); +out: + mutex_unlock(&ane_rtclient_pd_lock); + return err; +} + +static int ane_rtclient_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct resource *res; + struct ane_rtclient *ane; + struct ane_t6021 *a; + u32 cpu_status, ps_cpu; + u64 rvbar; + int ret; + + if (!ane_t6021_fwload_options_ok()) { + dev_err(dev, + "invalid firmware RAM-grant options; refusing before power access\n"); + return -EINVAL; + } + + if (!ane_t6021_fwload_placement_ok(dev)) { + dev_err(dev, + "fw_alias_reserved=1, but no no-map /reserved-memory node covers the iBoot firmware windows (this m1n1 does not reserve them); refusing before power access, ANE off\n"); + return -ENODEV; + } + + /* Early, BEFORE any allocation/power: legacy_only must never be + * rejected after the CPU release, where an unwind could drop + * domains under a running ASC (lab probe order). + */ + if (legacy_only && (!fw_start || fw_start_rtb_mode)) { + dev_err(dev, + "legacy_only=1 requires fw_start=1 and fw_start_rtb_mode=0\n"); + return -EINVAL; + } + if (legacy_query && !legacy_only) { + dev_err(dev, "legacy_query requires legacy_only=1\n"); + return -EINVAL; + } + + ane = devm_kzalloc(dev, sizeof(*ane), GFP_KERNEL); + if (!ane) + return -ENOMEM; + ane->legacy_buffers = devm_kcalloc(dev, ANE_LEGACY_ALLOCS, + sizeof(*ane->legacy_buffers), + GFP_KERNEL); + if (!ane->legacy_buffers) + return -ENOMEM; + ane->dev = dev; + ane->soc = of_device_get_match_data(dev); + platform_set_drvdata(pdev, ane); + INIT_DELAYED_WORK(&ane->poll_work, ane_rtclient_post_boot); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -ENODEV; + if (!(res->flags & IORESOURCE_MEM_NONPOSTED)) + dev_warn(dev, "engine window is not flagged non-posted\n"); + ane->engine = ioremap_np(res->start, resource_size(res)); + if (!ane->engine) + return -ENOMEM; + ane->cpu_rst = devm_reset_control_get_optional_exclusive(dev, NULL); + if (IS_ERR(ane->cpu_rst)) + return dev_err_probe(dev, PTR_ERR(ane->cpu_rst), + "ane_cpu reset control\n"); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret) + return ret; + + ret = ane_rtclient_attach_genpd(ane); + if (ret) + return dev_err_probe(dev, ret, "extra genpd attach\n"); + pm_runtime_enable(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret) + return dev_err_probe(dev, ret, "genpd raise failed\n"); + + ane->pmgr = devm_of_iomap(dev, dev->of_node, 1, NULL); + if (IS_ERR(ane->pmgr)) { + ret = PTR_ERR(ane->pmgr); + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return dev_err_probe(dev, ret, + "pmgr window map failed; G1 gate cannot run\n"); + } + ps_cpu = readl(ane->pmgr + ane->soc->ps_cpu_off); + dev_emerg(dev, "ane_cpu ACTUAL = 0x%x\n", ps_cpu); + if (FIELD_GET(ANE_PS_ACTUAL, ps_cpu) != ANE_PS_ON) { + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return -EPROBE_DEFER; + } + /* T8112: the kext (type 0x70) opens PWGATE (bits 29:28 = 0) before + * the ps words. This driver only reads it: an engine read behind a + * closed gate is untested, so it refuses first. + */ + if (ane->soc->pwgate_off) { + void __iomem *set = devm_of_iomap(dev, dev->of_node, 2, NULL); + u32 gate = IS_ERR(set) ? U32_MAX : + readl(set + ane->soc->pwgate_off); + + dev_emerg(dev, "PWGATE = 0x%x\n", gate); + if (gate & GENMASK(29, 28)) { + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return dev_err_probe(dev, -ENODEV, + "PWGATE closed; refusing before any engine read\n"); + } + } + + cpu_status = readl(ane->engine + ANE_ASC_CPU_STATUS); + rvbar = readq(ane->engine + ANE_ASC_RVBAR); + dev_emerg(dev, + "BOOT-PHASE engine reads ok: CPU_STATUS = 0x%x, RVBAR = %016llx (bit0=%u)\n", + cpu_status, rvbar, (u32)(rvbar & 1)); + + if (!(cpu_status & ANE_ASC_CPU_STATUS_RUNNING)) { + if (!fw_start) { + dev_err(dev, + "ANE firmware not alive (CPU_STATUS 0x%x) — start it from a quiesce context, or retry with fw_start=1\n", + cpu_status); + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return -EPROBE_DEFER; + } + + /* Lab fw_start order: staging requires fw_load and an + * IOMMU-mapped device BEFORE any allocation/staging. + */ + if (!ane_t6021_fwload_requested()) { + dev_err(dev, + "fw_start: requires fw_load=1 (no staged firmware)\n"); + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return -EINVAL; + } + if (!device_iommu_mapped(dev)) { + dev_err(dev, + "fw_start: device not IOMMU-mapped — a staged DVA/entry alias would be untranslated; refusing\n"); + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return -EINVAL; + } + + a = devm_kzalloc(dev, sizeof(*a), GFP_KERNEL); + if (!a) { + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return -ENOMEM; + } + a->dev = dev; + a->base[ANE_T6021_REG_ENGINE] = ane->engine; + a->irq = -1; + a->power_gated = true; + ane->fw = a; + + dev_emerg(dev, "BOOT-PHASE fwload stage+alias begin\n"); + ret = ane_t6021_fwload_probe(a); + if (ret) { + dev_err_probe(dev, ret, "fw_start: staging failed\n"); + ane_t6021_fwload_remove(a); + ane->fw = NULL; + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return ret; + } + if (!ane_t6021_rvbar_entry_ok(a->fw_iova)) { + dev_err(dev, "fw_start: staged iova %pad invalid\n", + &a->fw_iova); + ane_t6021_fwload_remove(a); + ane->fw = NULL; + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return -EINVAL; + } + + dev_emerg(dev, "BOOT-PHASE dispatch (table_mode=%d)\n", + fw_start_table_mode); + ret = ane_t6021_boot_start(a, 0, fw_start_table_mode, + fw_start_rtb_mode); + if (ret == -ENODATA || ret == -EAGAIN || + ret == -EBUSY || ret == -ECANCELED) { + ane_t6021_fwload_remove(a); + ane->fw = NULL; + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + return ret; + } + + ane->held = true; + cpu_status = readl(ane->engine + ANE_ASC_CPU_STATUS); + dev_emerg(dev, + "BOOT-PHASE sequence returned %pe (cpu_started=%u fw_alive=%u booted=%u) CPU_STATUS=0x%x\n", + ERR_PTR(ret), a->cpu_started, a->fw_alive, a->booted, + cpu_status); + if (!a->fw_alive && !fw_start_rtb_mode) { + dev_err(dev, + "fw_start: poll A timeout, no READY — HELD until reboot, RTKit handshake skipped\n"); + return 0; + } + } + + if (legacy_only) { + int cfg_err = 0; + + ane_rtclient_validate_chman(ane); + if (ane->fw && ane->fw->booted && scratch3_ack && + ane->chman_ok && + ane_t6021_chman_host_init(ane->fw->boot_ipc, + ane->fw->boot_ipc_size, + ane->fw->boot_ipc_iova)) { + int hello_ret = 0; + + dma_wmb(); + dev_emerg(dev, + "LEGACY P8 host ack: SCRATCH3 <- %08x\n", + ANE_T6021_BOOT_ACK); + /* hello_wait_ms > 0 only: init rtkit and arm the RX + * poll worker BEFORE writing the ack, so a HELLO + * after the ack is not missed. The 13.5 fw sent none + * on any recorded boot. + */ + if (hello_wait_ms && !ane->rtk) { + ane->rtk = devm_apple_rtkit_init(dev, ane, + NULL, 0, + &ane_rtclient_rtkit_ops); + if (IS_ERR(ane->rtk)) { + dev_err(dev, + "LEGACY hello: rtkit init %pe\n", + ane->rtk); + ane->rtk = NULL; + } else { + schedule_delayed_work(&ane->poll_work, + msecs_to_jiffies(10)); + } + } + writel(ANE_T6021_BOOT_ACK, + ane->engine + ANE_MBI_SCRATCH0 + 4 * 3); + if (ane->rtk) { + unsigned long hello_deadline = + jiffies + msecs_to_jiffies(hello_wait_ms); + + dev_emerg(dev, "LEGACY hello: boot begin (%u ms)\n", + hello_wait_ms); + do { + hello_ret = apple_rtkit_boot(ane->rtk); + } while (hello_ret == -ETIME && + time_before(jiffies, hello_deadline)); + dev_emerg(dev, + "LEGACY hello: boot %pe running=%d crashed=%d\n", + ERR_PTR(hello_ret), + apple_rtkit_is_running(ane->rtk), + apple_rtkit_is_crashed(ane->rtk)); + if (!hello_ret) { + ane->boot_done = true; + if (start_app_eps) + ane_rtclient_start_app_eps(ane); + } else { + cancel_delayed_work_sync(&ane->poll_work); + } + } + } else { + dev_warn(dev, + "LEGACY ack withheld (booted=%u scratch3_ack=%u chman_ok=%u)\n", + ane->fw ? ane->fw->booted : 0, + scratch3_ack, ane->chman_ok); + } + if (ane->chman_ok) { + /* One reusable command buffer for the transport + * (CONFIG_GET + every ioctl); reuse is legal + * because each exchange completes with the slot + * host-owned again. The 128-entry table stays + * for fw MALLOC replies only. + */ + if (ane_rtclient_legacy_alloc(ane, SZ_16K) == 0) + ane->cmd_buf = + &ane->legacy_buffers[ane->legacy_allocated - 1]; + else + cfg_err = -ENOMEM; + } + if (!cfg_err && legacy_query && ane->chman_ok) { + /* CONFIG_GET (opcode 0x03) keeps the legacy ChMan + * transport armed for the ioctls; the reply's + * word +0x08 must be nonzero (lab rule). The + * boot heap + 'IPC ' allocations stay HELD until + * reboot. + */ + struct ane_legacy_buffer *command = ane->cmd_buf; + int qret; + + if (command) { + memset(command->cpu, 0, SZ_16K); + qret = ane_rtclient_legacy_exchange(ane, + command, + 16, 0x03, + 1, 3000); + dev_info(dev, + "LEGACY CONFIG_GET words %08x %08x result=%d (DMA remains held)\n", + READ_ONCE(((u32 *)command->cpu)[1]), + READ_ONCE(((u32 *)command->cpu)[2]), + qret); + if (qret) + cfg_err = qret; + else if (!READ_ONCE(((u32 *)command->cpu)[2])) { + dev_err(dev, + "LEGACY CONFIG_GET reply word +0x08 zero\n"); + cfg_err = -EPROTO; + } + } else { + cfg_err = -ENOMEM; + } + } + ane->boot_done = true; + if (cfg_err) { + dev_err(dev, + "install: CONFIG_GET failed (%d) — refusing to register DRM device (ioctls would run on an unproven ring)\n", + cfg_err); + if (!ane->held) { + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + } + return cfg_err; + } + } else { + unsigned long deadline; + int boot_ret; + + ane->rtk = devm_apple_rtkit_init(dev, ane, NULL, 0, + &ane_rtclient_rtkit_ops); + if (IS_ERR(ane->rtk)) { + ret = PTR_ERR(ane->rtk); + ane->rtk = NULL; + dev_err_probe(dev, ret, "apple_rtkit_init failed\n"); + goto err_pm_or_hold; + } + /* RX fallback worker first: the fw's HELLO must not be + * missed while the recv line is unproven. + */ + schedule_delayed_work(&ane->poll_work, msecs_to_jiffies(10)); + deadline = jiffies + msecs_to_jiffies(hello_wait_ms); + do { + boot_ret = apple_rtkit_boot(ane->rtk); + } while (boot_ret == -ETIME && time_before(jiffies, deadline)); + if (boot_ret) { + dev_err(dev, "rtkit boot handshake failed: %pe\n", + ERR_PTR(boot_ret)); + cancel_delayed_work_sync(&ane->poll_work); + ret = boot_ret; + goto err_pm_or_hold; + } + ane->boot_done = true; + if (start_app_eps) + ane_rtclient_start_app_eps(ane); + } + + if (!ane->chman_ok) { + dev_err(dev, + "install: ChMan table not validated — refusing to register DRM device (ioctls would stall the ring)\n"); + cancel_delayed_work_sync(&ane->poll_work); + if (!ane->held) { + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + } + return -EPROTO; + } + + { + struct ane_t6021_drm *adrm; + int drmret; + + /* This kernel dropped drm_dev_init; the resource-managed + * alloc registers the same ABI-2 device. + */ + adrm = devm_drm_dev_alloc(dev, &ane_t6021_drm_driver, + struct ane_t6021_drm, drm); + if (IS_ERR(adrm)) { + ret = dev_err_probe(dev, PTR_ERR(adrm), + "drm device alloc\n"); + goto err_pm_or_hold; + } + adrm->dev = dev; + adrm->ane = ane; + WRITE_ONCE(ane_t6021_perf_ane, ane); + drmret = drm_dev_register(&adrm->drm, 0); + if (drmret) { + dev_err_probe(dev, drmret, "drm_dev_register\n"); + ret = drmret; + goto err_pm_or_hold; + } + dev_info(dev, + "loaded ane_t6021 (DRM major %d minor %d; ABI 2; legacy_only=%u chman_ok=%u booted=%u; state %s; BO cap %u MiB)\n", + ANE_ABI_M2_MAJOR, 0, + legacy_only, ane->chman_ok, + ane->fw ? ane->fw->booted : 0, + ane->held ? "HELD" : "ready", bo_total_max_mb); + } + + return 0; + +err_pm_or_hold: + if (ane->held) + dev_warn(dev, + "probe failed after CPU start (%pe) — binding fenced; power/rings/IRQ HELD until reboot\n", + ERR_PTR(ret)); + else { + pm_runtime_put_sync_suspend(dev); + pm_runtime_disable(dev); + } + return ret; +} + +static void ane_rtclient_remove(struct platform_device *pdev) +{ + struct ane_rtclient *ane = platform_get_drvdata(pdev); + + if (READ_ONCE(ane_t6021_perf_ane) == ane) + WRITE_ONCE(ane_t6021_perf_ane, NULL); + + cancel_delayed_work_sync(&ane->poll_work); + + if (ane->held) + dev_warn(&pdev->dev, + "remove HELD: no teardown — reboot reclaims\n"); +} + +static const struct of_device_id ane_rtclient_of_match[] = { + { .compatible = "apple,t6020-ane", .data = &ane_t6020_soc }, + { .compatible = "apple,t6021-ane", .data = &ane_t6021_soc }, + { .compatible = "apple,t6022-ane", .data = &ane_t6022_soc }, + { .compatible = "apple,t8112-ane", .data = &ane_t8112_soc }, + { } +}; +MODULE_DEVICE_TABLE(of, ane_rtclient_of_match); + +static struct platform_driver ane_rtclient_driver = { + .driver = { + .name = "ane_t6021", + .of_match_table = ane_rtclient_of_match, + .suppress_bind_attrs = true, + }, + .probe = ane_rtclient_probe, + .remove = ane_rtclient_remove, +}; + +static int __init ane_rtclient_init(void) +{ + int ret = platform_driver_register(&ane_rtclient_driver); + + if (ret) + ane_t6021_trace_free(); + return ret; +} +module_init(ane_rtclient_init); + +static void __exit ane_rtclient_exit(void) +{ + platform_driver_unregister(&ane_rtclient_driver); + ane_t6021_trace_free(); +} +module_exit(ane_rtclient_exit); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple Neural Engine (T6021/M2) installed module"); diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c new file mode 100644 index 0000000000000..0e98c48d2aff1 --- /dev/null +++ b/drivers/accel/ane/ane_tm.c @@ -0,0 +1,652 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2022 Eileen Yoon */ + +#include +#include +#include +#include +#include "ane_tm.h" + +#define ANE_TQ_COUNT 8 +static const int TQ_PRTY_TABLE[ANE_TQ_COUNT] = { 0x1, 0x2, 0x3, 0x4, + 0x5, 0x6, 0x1e, 0x1f }; + +#define ANE_TM_BASE 0x20000 +#define ANE_TQ_BASE 0x21000 + +#define TM_ADDR 0x0 +#define TM_INFO 0x4 +#define TM_PUSH 0x8 +#define TM_TQ_EN 0xc + +#define TM_IRQ_EVTC(line) (0x14 + (line * 0x14)) +#define TM_IRQ_INFO(line) (0x18 + (line * 0x14)) +#define TM_IRQ_UNK1(line) (0x1c + (line * 0x14)) +#define TM_IRQ_TMST(line) (0x20 + (line * 0x14)) +#define TM_IRQ_UNK2(line) (0x24 + (line * 0x14)) + +#define TM_COMMITTED 0x44 +#define TM_STATUS 0x54 +#define TM_ERROR1 0x58 +#define TM_ERROR2 0x5c +#define TM_ERROR3 0x60 +#define TM_IRQ_EN1 0x68 +#define TM_IRQ_ACK 0x6c +#define TM_IRQ_EN2 0x70 + +#define TQ_STATUS(qid) (0x000 + (qid * 0x148)) +#define TQ_PRTY(qid) (0x010 + (qid * 0x148)) +#define TQ_VACANT(qid) (0x014 + (qid * 0x148)) +#define TQ_INFO(qid) (0x01c + (qid * 0x148)) + +#define TQ_BAR1(qid, bdx) (0x020 + (qid * 0x148) + (bdx * 0x4)) +#define TQ_NID1(qid) (0x0a0 + (qid * 0x148)) +#define TQ_SIZE2(qid) (0x0a4 + (qid * 0x148)) +#define TQ_ADDR2(qid) (0x0a8 + (qid * 0x148)) + +#define TQ_BAR2(qid, bdx) (0x0ac + (qid * 0x148) + (bdx * 0x4)) +#define TQ_NID2(qid) (0x12c + (qid * 0x148)) +#define TQ_SIZE1(qid) (0x130 + (qid * 0x148)) +#define TQ_ADDR1(qid) (0x134 + (qid * 0x148)) + +#define TM_IS_IDLE 0x1 +#define TM_IS_FINE 0x22222222 + +#define tm_read32(ane, off) (readl(ane->engine + ANE_TM_BASE + off)) +#define tq_read32(ane, off) (readl(ane->engine + ANE_TQ_BASE + off)) +#define tm_write32(ane, off, val) (writel(val, ane->engine + ANE_TM_BASE + off)) +#define tq_write32(ane, off, val) (writel(val, ane->engine + ANE_TQ_BASE + off)) + +/* The ane SET block (m1n1 ANE.ps_map) maps the pmgr power-state words + * for this engine (set0, base, set1..4). pmgr reads are always safe and + * give recovery its ACTUAL power evidence; the SET words themselves are + * firmware-locked: a direct write external-aborts the SoC. Bisect + * evidence, 2026-09-16: the T6001 kill is netconsole-named at + * 0x28e08c000 (PS_SET0 down, all sets gated); T8103 hard-reset the same + * way at 0x23b70c000 with the 95dbcf3-era gate armed. No code here ever + * writes the SET block. + */ +#define ANE_PS_ACTUAL_MASK 0xf0 +#define ANE_PS_WORDS 6 /* set0, base, set1..4 */ + +/* ACTUAL nibble of each SET word, word 0 in the low nibble; 0 when the + * SET block is unmapped. pmgr registers only: engine MMIO is never + * read for power state (a readl through a warm gate external-aborts + * and hard-resets the machine). + */ +u32 ane_ps_act(struct ane_device *ane) +{ + u32 v = 0; + int i; + + if (!ane->ps) + return 0; + for (i = 0; i < ANE_PS_WORDS; i++) + v |= ((readl(ane->ps + i * 8) & ANE_PS_ACTUAL_MASK) >> 4) + << (i * 4); + return v; +} + +/* Bisect probe: per-word named reads of the SET window. Each word logs + * before its readl, so a window whose address does not decode on this + * SoC is named by the last off-box line (word index + byte offset) + * instead of a silent hard reset between two other prints. + */ +u32 ane_ps_act_probe(struct ane_device *ane) +{ + u32 v = 0; + int i; + + if (!ane->ps) { + dev_info(ane->dev, "ps probe: SET window unmapped\n"); + return 0; + } + dev_info(ane->dev, "ps probe: SET window at %pap, %d words\n", + &ane->ps_base, ANE_PS_WORDS); + for (i = 0; i < ANE_PS_WORDS; i++) { + u32 w; + + dev_info(ane->dev, "ps probe: word %d @ ps+0x%02x reading\n", + i, i * 8); + w = readl(ane->ps + i * 8); + v |= ((w & ANE_PS_ACTUAL_MASK) >> 4) << (i * 4); + dev_info(ane->dev, "ps probe: word %d -> %#x\n", i, w); + } + return v; +} + +/* Recovery-path MMIO logging. Every write prints before and after, and + * every engine read prints its value, each line carrying the pmgr + * ACTUAL of the owning partitions. A write that external-aborts the + * SoC is then named by the last line the off-box netconsole carried, + * and the bisect starts from that register instead of a guess. + */ +static void ane_rec_writel(struct ane_device *ane, const char *reg, + void __iomem *addr, u32 val) +{ + dev_info(ane->dev, "ANEWR %s <- %#x (ps act %#x)\n", reg, val, + ane_ps_act(ane)); + writel(val, addr); + dev_info(ane->dev, "ANEWR %s wrote (ps act %#x)\n", reg, + ane_ps_act(ane)); +} + +static u32 ane_rec_read32(struct ane_device *ane, const char *reg, + void __iomem *addr) +{ + u32 val = readl(addr); + + dev_info(ane->dev, "ANERD %s -> %#x (ps act %#x)\n", reg, val, + ane_ps_act(ane)); + return val; +} + +u32 ane_tm_ps_act(struct ane_device *ane) +{ + return ane_ps_act(ane); +} + +void ane_tm_enable(struct ane_device *ane, bool rec) +{ + void __iomem *tq_en = ane->engine + ANE_TM_BASE + TM_TQ_EN; + u32 val = rec ? ane_rec_read32(ane, "TM_TQ_EN tm+0x0c", tq_en) + : readl(tq_en); + char reg[24]; + + val |= 0x1000; + if (rec) + ane_rec_writel(ane, "TM_TQ_EN tm+0x0c", tq_en, val); + else + writel(val, tq_en); + + for (int qid = 0; qid < ANE_TQ_COUNT; qid++) { + void __iomem *prty = + ane->engine + ANE_TQ_BASE + TQ_PRTY(qid); + + if (rec) { + snprintf(reg, sizeof(reg), "TQ_PRTY[%d] tq+%#x", qid, + TQ_PRTY(qid)); + ane_rec_writel(ane, reg, prty, TQ_PRTY_TABLE[qid]); + } else { + writel(TQ_PRTY_TABLE[qid], prty); + } + } + + if (rec) { + ane_rec_writel(ane, "TM_IRQ_EN1 tm+0x68", + ane->engine + ANE_TM_BASE + TM_IRQ_EN1, + 0x4000000); + ane_rec_writel(ane, "TM_IRQ_EN2 tm+0x70", + ane->engine + ANE_TM_BASE + TM_IRQ_EN2, 0x6); + } else { + tm_write32(ane, TM_IRQ_EN1, 0x4000000); + tm_write32(ane, TM_IRQ_EN2, 0x6); + } +} + +u32 ane_tm_status(struct ane_device *ane) +{ + return tm_read32(ane, TM_STATUS); +} + +int ane_tm_enqueue(struct ane_device *ane, struct ane_request *req) +{ + int qid = req->qid; + + tq_write32(ane, TQ_STATUS(qid), 0x1); + + for (int bdx = 0; bdx < ANE_TILE_COUNT; bdx++) { + tq_write32(ane, TQ_BAR1(qid, bdx), req->bar[bdx]); + } + + tq_write32(ane, TQ_SIZE1(qid), ((req->td_size >> 2) - 1) << 0x10); + tq_write32(ane, TQ_ADDR1(qid), req->btsp_iova); + tq_write32(ane, TQ_NID1(qid), (req->nid & 0xff) << 8 | 1); + + return 0; +} + +static void ane_tm_push_tq(struct ane_device *ane, struct ane_request *req) +{ + int qid = req->qid; + + tm_write32(ane, TM_ADDR, tq_read32(ane, TQ_ADDR1(qid))); + tm_write32(ane, TM_INFO, tq_read32(ane, TQ_SIZE1(qid)) | req->td_count); + tm_write32(ane, TM_PUSH, TQ_PRTY_TABLE[qid] | (qid & 7) << 8); // magic +} + +static int ane_tm_collect_events(struct ane_device *ane, + struct ane_request *req, u32 *finished) +{ + for (unsigned int line = 0; line < 2; line++) { + u32 count = tm_read32(ane, TM_IRQ_EVTC(line)); + + if (count > 64) + return -EIO; + for (u32 n = 0; n < count; n++) { + u32 info = tm_read32(ane, TM_IRQ_INFO(line)); + + tm_read32(ane, TM_IRQ_UNK1(line)); + tm_read32(ane, TM_IRQ_TMST(line)); + tm_read32(ane, TM_IRQ_UNK2(line)); + if (req && info == (0x05000000 | (req->nid << 16) | + (req->td_count - 1))) + *finished |= BIT(line); + } + } + tm_write32(ane, TM_IRQ_ACK, tm_read32(ane, TM_IRQ_ACK) | 2); + return *finished == 3 && (tm_read32(ane, TM_STATUS) & TM_IS_IDLE); +} + +/* read_poll_timeout op: consult the DART latch before any engine access. + * Returns 1 to end the poll when a fault is latched; the caller then + * handles the fault without ever reading the wedged engine. + */ +static int ane_tm_poll_step(struct ane_device *ane, struct ane_request *req, + u32 *finished) +{ + if (ane_dart_faulted(ane, NULL, NULL)) + return 1; + return ane_tm_collect_events(ane, req, finished); +} + +int ane_tm_execute(struct ane_device *ane, struct ane_request *req) +{ + u32 finished = 0; + u32 fault_status = 0; + u64 fault_iova = 0; + struct ane_dart_scratch scratch[ANE_DART_SCRATCH_MAX]; + int scratch_pages = 0; + bool faulted = false; + int err, status; + + lockdep_assert_held(&ane->engine_lock); + ane_dart_mask(ane); + + err = ane_tm_collect_events(ane, NULL, &finished); + if (err < 0) + goto wedge; + wmb(); + ane_tm_push_tq(ane, req); + + for (;;) { + /* Latch-first completion poll. Engine register reads hang + * jwm1 once a DART fault has landed (lockups 1 and 3: the + * journal ends inside the 1-microsecond completion poll, + * before any fault handling ran), so the poll op consults + * the DART error latch first and stops the poll before the + * engine is touched. Residual hazard: a fault landing + * between one iteration's latch read and engine read. + */ + err = read_poll_timeout(ane_tm_poll_step, status, + status != 0, 1, 1000000, false, + ane, req, &finished); + if (!err && status < 0) + err = status; + if (ane_dart_faulted(ane, &fault_iova, &fault_status)) { + faulted = true; + if (scratch_pages >= ANE_DART_SCRATCH_MAX) + break; + if (ane_dart_drain_fault(ane, fault_iova, scratch, + &scratch_pages)) + break; + /* The retried transaction now completes; poll on. */ + continue; + } + break; + } + + if (faulted) { + /* The faulting program drained through the scratch pages + * and the engine is idle again (the latch is clear or the + * poll would still be running). Gate the queue with the + * normal completion writes, hand the scratch pages back, + * and fail this request. No partition power cycle, no + * stream disable: the page tables were never changed by + * the fault. + */ + if (err || ane_dart_faulted(ane, NULL, NULL)) { + /* Drain failed or the drained program never reached + * completion: the engine is not idle and access is + * off the table. wedge releases the scratch. + */ + err = err ?: -EIO; + goto wedge; + } + status = tq_read32(ane, TQ_NID1(req->qid)); + tq_write32(ane, TQ_NID1(req->qid), status & ~1U); + rmb(); + tq_write32(ane, TQ_STATUS(req->qid), 0x0); + ane_dart_release_scratch(ane, scratch, scratch_pages); + ane_dart_unmask(ane); + dev_err(ane->dev, + "DART fault contained: status=%#x iova=%#llx scratch=%d\n", + fault_status, fault_iova, scratch_pages); + return -EIO; + } + if (err) + goto wedge; + status = tq_read32(ane, TQ_NID1(req->qid)); + if (((status >> 8) & 0xff) != req->nid) { + err = -EIO; + goto wedge; + } + tq_write32(ane, TQ_NID1(req->qid), status & ~1U); + rmb(); + tq_write32(ane, TQ_STATUS(req->qid), 0x0); + ane_dart_unmask(ane); + return 0; + +wedge: + ane_dart_release_scratch(ane, scratch, scratch_pages); + ane_dart_unmask(ane); + if (atomic_xchg(&ane->wedged, 1) == 0) { + __module_get(THIS_MODULE); + dev_err(ane->dev, + "tm completion failed: %d, finish lines=%x (q%d nid=%#x)\n", + err, finished, req->qid, req->nid); + } + + /* One bounded recovery attempt: stop the tm, power-cycle the engine + * and return to accepting work. Only a failed reset preserves + * resources until reboot. + */ + if (ane_tm_recover(ane) < 0) { + if (ane->tm_retention) + dev_err(ane->dev, + "recovery failed; preserving resources until module reload or reboot (tm/tq file retained through the cycle: the latched task error parks the TM in place; rmmod+insmod restores service)\n"); + else + dev_err(ane->dev, + "recovery failed; preserving resources until reboot\n"); + } + return err; +} + +/* + * Bounded recovery after a failed task. The only reset this driver knows + * is the partition power cycle system sleep already performs on this + * hardware. With the several ANE power partitions attached as genpd + * devices (T8103 five, T6001 similar), gating happens on those devices; + * with a single domain attached directly to the device, on the device + * itself. Gating the partitions stops any DMA still fetching the dead + * task and clears the tm register file; ungating brings the engine back + * through the probe resume path (ane_tm_enable), then the task manager + * must report idle. BO mappings deliberately stay in place: they remain + * coherent through the cycle and valid for the next submit, and once the + * wedge clears, BO_FREE unmaps them through the normal path again. + * Callers hold engine_lock. + * + * T6001 note, bisect evidence 2026-09-16: its set0/base islands are + * firmware-locked (direct pmgr writes external-abort; see the SET block + * comment above) and hold the tm/tq file in retention through any + * genpd cycle - TQ_EN still reads 0x3000 after a 20 ms and a 2 s gate, + * and tasks dispatched afterwards complete but compute nondeterministic + * garbage. The idle-or-fresh poll below therefore fails on T6001 and + * recovery preserves until reboot instead of serving wrong data. + */ + +/* No engine MMIO unless the owning partitions read powered on (ACTUAL + * nibble per word): a readl through a warm gate external-aborts and + * hard resets the machine. Covers set0, base and set1..4 from the SET + * block map; sys_cpu rides its own genpd resume and has no cell in + * this block on T6001. + */ +static int ane_ps_verify_on(struct ane_device *ane) +{ + u32 act = 0; + int err = -ETIMEDOUT; + int i; + + if (!ane->ps) + return 0; + for (i = 0; i < 100; i++) { + act = ane_ps_act(ane); + if (act == ANE_PS_ALL_ON) { + err = 0; + break; + } + usleep_range(1000, 2000); + } + dev_info(ane->dev, "ANERD ps verify act=%#x err=%d\n", act, err); + return err; +} + +static int ane_pd_cycle(struct ane_device *ane) +{ + int err = 0; + + ane->recovering = true; + if (ane->pd_count > 1) { + int gated = 0; + + for (int i = 0; i < ane->pd_count; i++) { + pm_runtime_get_noresume(ane->pd_dev[i]); + gated = i + 1; + dev_info(ane->dev, + "ANERD pd[%d] %s force_suspend begin (ps act %#x)\n", + i, dev_name(ane->pd_dev[i]), ane_ps_act(ane)); + err = pm_runtime_force_suspend(ane->pd_dev[i]); + dev_info(ane->dev, + "ANERD pd[%d] force_suspend -> %d (ps act %#x)\n", + i, err, ane_ps_act(ane)); + if (err) + break; + } + if (!err) + for (int i = 0; i < ane->pd_count; i++) { + dev_info(ane->dev, + "ANERD pd[%d] %s force_resume begin (ps act %#x)\n", + i, dev_name(ane->pd_dev[i]), + ane_ps_act(ane)); + err = pm_runtime_force_resume(ane->pd_dev[i]); + dev_info(ane->dev, + "ANERD pd[%d] force_resume -> %d (ps act %#x)\n", + i, err, ane_ps_act(ane)); + if (err) + break; + } + while (gated--) + pm_runtime_put_noidle(ane->pd_dev[gated]); + } else { + /* Pin a second usage ref for the cycle: force_suspend only + * marks needs_force_resume when a ref beyond the probe one + * is held, and without that mark force_resume would leave + * the partition gated. + */ + pm_runtime_get_noresume(ane->dev); + dev_info(ane->dev, "ANERD dev force_suspend begin\n"); + err = pm_runtime_force_suspend(ane->dev); + dev_info(ane->dev, "ANERD dev force_suspend -> %d\n", err); + if (!err) { + dev_info(ane->dev, "ANERD dev force_resume begin\n"); + err = pm_runtime_force_resume(ane->dev); + dev_info(ane->dev, "ANERD dev force_resume -> %d\n", + err); + } + pm_runtime_put_noidle(ane->dev); + } + ane->recovering = false; + + return err; +} + +/* + * Drain the task-manager state a timed-out task leaves behind when the + * tm/tq file survives the power cycle in retention (T6001; descriptor + * tm_retention). T8103 never runs this: its cycle is a full POR of the + * file, and the re-arm below starts it clean. + * + * Two things must go, or the retained TM never reads idle again: + * - pending events on both IRQ lines. Each event pops by reading + * INFO/UNK1/TMST/UNK2, exactly as ane_tm_collect_events pops them; + * a stale backlog (or one over the 64-entry collect cap) would make + * the next submit's completion poll miss or fail. + * - the latched queue entry of the dead task. TQ_STATUS/NID1 get the + * same clear the normal completion path gives a finished queue. + */ +static void ane_tm_drain_retained(struct ane_device *ane) +{ + int line, qid; + + /* Stop the TM first: the cycle leaves the enable|halt latch set + * (TQ_EN 0x3000, 2026-09-16 evidence). Clear both bits so the + * re-arm below starts the TM from a defined stopped state. + */ + ane_rec_writel(ane, "TM_TQ_EN stop tm+0x0c", + ane->engine + ANE_TM_BASE + TM_TQ_EN, + tm_read32(ane, TM_TQ_EN) & ~0x3000U); + + for (line = 0; line < 2; line++) { + u32 count = tm_read32(ane, TM_IRQ_EVTC(line)); + u32 n; + + if (count > 64) + count = 64; + for (n = 0; n < count; n++) { + tm_read32(ane, TM_IRQ_INFO(line)); + tm_read32(ane, TM_IRQ_UNK1(line)); + tm_read32(ane, TM_IRQ_TMST(line)); + tm_read32(ane, TM_IRQ_UNK2(line)); + } + } + tm_write32(ane, TM_IRQ_ACK, tm_read32(ane, TM_IRQ_ACK) | 3); + + /* The dead task leaves per-queue error codes latched here (seen + * 0x22222222 / 0x2222: four-bit code 2 per TQ slot). While any + * latch is set the TM stays halted and TM_STATUS reads 0. Try the + * W1C convention first (write the read value back), fall back to + * plain zero-clear, and log what this silicon answered to. + */ + { + static const u16 err_reg[3] = { TM_ERROR1, TM_ERROR2, + TM_ERROR3 }; + char reg[24]; + int ei; + + for (ei = 0; ei < 3; ei++) { + void __iomem *addr = + ane->engine + ANE_TM_BASE + err_reg[ei]; + u32 val = readl(addr); + + if (!val) + continue; + writel(val, addr); + if (readl(addr)) + writel(0x0, addr); + snprintf(reg, sizeof(reg), "TM_ERROR%d clear tm+%#x", + ei + 1, err_reg[ei]); + ane_rec_read32(ane, reg, addr); + } + } + + for (qid = 0; qid < ANE_TQ_COUNT; qid++) { + u32 nid = tq_read32(ane, TQ_NID1(qid)); + + tq_write32(ane, TQ_STATUS(qid), 0x0); + if (nid & 1) + tq_write32(ane, TQ_NID1(qid), nid & ~1U); + } + + /* Retention evidence for the off-box console: what the dead task + * left in the committed counter and the error latches. + */ + ane_rec_read32(ane, "TM_COMMITTED tm+0x44 (drain)", + ane->engine + ANE_TM_BASE + TM_COMMITTED); + ane_rec_read32(ane, "TM_ERROR1 tm+0x58 (drain)", + ane->engine + ANE_TM_BASE + TM_ERROR1); + ane_rec_read32(ane, "TM_ERROR2 tm+0x5c (drain)", + ane->engine + ANE_TM_BASE + TM_ERROR2); + ane_rec_read32(ane, "TM_ERROR3 tm+0x60 (drain)", + ane->engine + ANE_TM_BASE + TM_ERROR3); +} + +int ane_tm_recover(struct ane_device *ane) +{ + u32 status; + int err; + + lockdep_assert_held(&ane->engine_lock); + if (!atomic_read(&ane->wedged)) + return 0; + + dev_err(ane->dev, "recovering: power-cycling engine partitions\n"); + + err = ane_pd_cycle(ane); + if (err) { + dev_err(ane->dev, "recovery: engine power cycle failed: %d\n", + err); + return err; + } + + /* No engine MMIO before the islands read powered on. */ + err = ane_ps_verify_on(ane); + if (err) { + dev_err(ane->dev, + "recovery: ane set islands not powered on: %d\n", err); + return err; + } + + /* T6001 (tm_retention): the cycle did NOT clear the tm/tq file — + * the timed-out task's events and queue entry are still latched. + * Drain them, or the TM below never reads idle and recovery + * degrades to preserve-until-reboot. + */ + if (ane->tm_retention) + ane_tm_drain_retained(ane); + + /* Power-on reset cleared the tm register file; re-arm it exactly + * like the probe resume path does. + */ + ane_tm_enable(ane, true); + + err = readl_poll_timeout(ane->engine + ANE_TM_BASE + TM_STATUS, + status, + (status & TM_IS_IDLE) || + status == ane->tm_status_fresh, + 100, 1000000); + status = ane_rec_read32(ane, "TM_STATUS tm+0x54", + ane->engine + ANE_TM_BASE + TM_STATUS); + if (err) { + dev_err(ane->dev, "recovery: tm not idle after reset: %#x\n", + status); + return err; + } + + if (atomic_xchg(&ane->wedged, 0)) { + module_put(THIS_MODULE); /* drop the wedge pin */ + /* The power cycle above quiesced every transaction, so the + * wedge-preserved mappings (dead BOs kept alive only to + * stop IOVA teardown under active DMA) can now be unmapped + * and their ranges handed back to the allocator. Without + * this reclaim the ranges would stay pinned for the whole + * module lifetime. + */ + ane_reclaim_preserved(ane); + dev_info(ane->dev, "tm recovered: idle, accepting work again\n"); + } + return 0; +} + +/* + * Drop the wedge pin on demand. The recovery path in ane_tm_recover + * does an atomic_xchg + module_put, but only on its success branch - + * a real -110 with the engine stuck leaves ane->wedged set and the + * module refcount one too high, so rmmod fails forever and the bug + * becomes a kernel-brick-without-reboot. ane_drm_postclose calls + * this unconditionally so every session close releases the pin, + * letting the operator unload the ko for an updated build even when + * the engine itself cannot be cleared in software. + * + * Pair count: must follow an ane_wedge_set() that called + * __module_get(THIS_MODULE). Today the only such set is in the wedge: + * branch of ane_tm_execute(); future sets must call this in their + * clean-up path too. + */ +void ane_wedge_clear(struct ane_device *ane) +{ + if (atomic_xchg(&ane->wedged, 0)) { + module_put(THIS_MODULE); + dev_info(ane->dev, + "wedge pin released (engine state unknown)\n"); + } +} diff --git a/drivers/accel/ane/ane_tm.h b/drivers/accel/ane/ane_tm.h new file mode 100644 index 0000000000000..94d425ab8d8b7 --- /dev/null +++ b/drivers/accel/ane/ane_tm.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2022 Eileen Yoon */ + +#ifndef __ANE_TM_H__ +#define __ANE_TM_H__ + +#include "ane.h" + +void ane_tm_enable(struct ane_device *ane, bool rec); +u32 ane_tm_status(struct ane_device *ane); +u32 ane_ps_act(struct ane_device *ane); +u32 ane_ps_act_probe(struct ane_device *ane); +bool ane_tm_islands_on(struct ane_device *ane, u32 *act); +int ane_tm_enqueue(struct ane_device *ane, struct ane_request *req); +int ane_tm_execute(struct ane_device *ane, struct ane_request *req); +int ane_tm_recover(struct ane_device *ane); +#define ANE_PS_ALL_ON ((1U << (4 * 6)) - 1) +u32 ane_tm_ps_act(struct ane_device *ane); + +#endif /* __ANE_TM_H__ */ diff --git a/include/uapi/drm/ane_accel.h b/include/uapi/drm/ane_accel.h new file mode 100644 index 0000000000000..e5fd9d12d1d63 --- /dev/null +++ b/include/uapi/drm/ane_accel.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: (GPL-2.0-only WITH Linux-syscall-note) OR MIT */ +/* Copyright 2022 Eileen Yoon */ + +#ifndef __ANE_ACCEL_H__ +#define __ANE_ACCEL_H__ + +#include "drm.h" + +#if defined(__cplusplus) +extern "C" { +#endif + +#define ANE_TILE_COUNT 0x20 +#define ANE_FIFO_NID 0x40 +#define ANE_CMD_GRAN 0x10 + +/* ABI 1: successful SUBMIT guarantees terminal completion and CPU visibility. */ +#define ANE_ABI_MAJOR 1 + +/* + * ABI 2 (T6021 / M2, H14 firmware programs): the driver reports + * DRM_IOCTL_VERSION major 2 and rejects DRM_ANE_SUBMIT. The M1 libane + * requires major 1, so it cannot submit M1 frames to an M2 device. + * A successful DRM_ANE_EXEC guarantees terminal completion (firmware ack + * and task-queue idle) and CPU visibility of the output buffers. + */ +#define ANE_ABI_M2_MAJOR 2 +#define ANE_M2_MAX_SECTIONS 16 +#define ANE_M2_MAX_BINDS 64 + +#define DRM_ANE_BO_INIT 0x1 +#define DRM_ANE_BO_FREE 0x2 +#define DRM_ANE_SUBMIT 0x3 +#define DRM_ANE_PROG_LOAD 0x4 +#define DRM_ANE_PROC_CREATE 0x5 +#define DRM_ANE_EXEC 0x6 + +struct drm_ane_bo_init { + __u32 handle; + __u32 pad; + __u64 size; + __u64 offset; +}; + +struct drm_ane_bo_free { + __u32 handle; + __u32 pad; +}; + +struct drm_ane_submit { + __u64 tsk_size; + __u32 td_count; + __u32 td_size; + __u32 handles[ANE_TILE_COUNT]; + __u32 btsp_handle; + __u32 pad; +}; + +/* One firmware program section, already built by userspace, held in a BO. */ +struct drm_ane_section { + __u32 id; /* firmware section id */ + __u32 bo_handle; /* BO holding the bytes */ + __u64 size; + __u64 offset; /* byte offset inside the BO */ +}; + +/* A buffer whose IOVA and size the driver patches into a generic entry. */ +struct drm_ane_generic_bind { + __u32 buffer_id; + __u32 bo_handle; + __u32 type; + __u32 pad; + __u64 size; +}; + +struct drm_ane_prog_load { + __u64 sections_ptr; /* struct drm_ane_section[section_count] */ + __u64 generic_ptr; /* struct drm_ane_generic_bind[generic_count] */ + __u32 section_count; + __u32 generic_count; + __u32 prog_id_out; + __u32 pad; +}; + +struct drm_ane_proc_create { + __u32 prog_id; + __u32 proc_id_out; +}; + +/* Per-call buffer binding: an input or output for one procedure call. */ +struct drm_ane_exec_io { + __u32 buffer_id; + __u32 bo_handle; + __u32 type; + __u32 flags; + __u64 dma; /* in only, reserved: the driver binds the + * IOVA from bo_handle + */ + __u64 size; +}; + +struct drm_ane_exec { + __u32 prog_id; + __u32 proc_id; + __u32 priority; /* 2..7 */ + __u32 timeout_ms; + __u32 count; /* 1..ANE_M2_MAX_BINDS */ + __u32 pad; + __u64 io_ptr; /* struct drm_ane_exec_io[count] */ +}; + +#define DRM_IOCTL_ANE_BO_INIT \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_BO_INIT, struct drm_ane_bo_init) +#define DRM_IOCTL_ANE_BO_FREE \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_BO_FREE, struct drm_ane_bo_free) +#define DRM_IOCTL_ANE_SUBMIT \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_SUBMIT, struct drm_ane_submit) + +#define DRM_IOCTL_ANE_PROG_LOAD \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_PROG_LOAD, struct drm_ane_prog_load) +#define DRM_IOCTL_ANE_PROC_CREATE \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_PROC_CREATE, struct drm_ane_proc_create) +#define DRM_IOCTL_ANE_EXEC \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_EXEC, struct drm_ane_exec) + +#if defined(__cplusplus) +} +#endif + +#endif /* __ANE_ACCEL_H__ */ From f790ae9b5801bcccbfae025e89e7b0445041f8af Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:27:34 -0500 Subject: [PATCH 0346/1579] arm64: dts: apple: t8103: Add the ANE, its DARTs and power states Describe the M1 Neural Engine: the engine window at 0x26bc04000, the three ANE DARTs, and the ane_base and ane_set1..5 power states (ane_sys_cpu -> ane_base -> ane_set1..5). apple-dart owns the DARTs and the engine maps through their IOMMU domain. The ane driver binds these nodes. They equal, property for property with phandles compared by target path, the live device tree of a MacBook Pro 13" M1 (j293) on which the out-of-tree build of the same driver code was bound on 2026-09-30 (linux-asahi 7.1.13): 80 properties over the ane node, its DARTs and the eight ANE power states. Enabled: Linux starts no ANE firmware on T8103. Until the ane module loads, apple-dart binds the three DARTs and apple-pmgr-pwrstate registers the new states, as on that machine. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8103-pmgr.dtsi | 54 +++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8103.dtsi | 46 +++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index 4a399bc8f0276..252c6914d690d 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -1003,6 +1003,60 @@ power-domains = <&ps_ane_sys>; }; + ps_ane_base: power-controller@c008 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_base"; + power-domains = <&ps_ane_sys_cpu>; + }; + + ps_ane_set1: power-controller@c010 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set1"; + power-domains = <&ps_ane_base>; + }; + + ps_ane_set2: power-controller@c018 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set2"; + power-domains = <&ps_ane_base>; + }; + + ps_ane_set3: power-controller@c020 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set3"; + power-domains = <&ps_ane_base>; + }; + + ps_ane_set4: power-controller@c028 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set4"; + power-domains = <&ps_ane_base>; + }; + + ps_ane_set5: power-controller@c030 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set5"; + power-domains = <&ps_ane_base>; + }; + ps_disp0_cpu0: power-controller@10018 { compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x10018 4>; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 4682da893992c..16ac8bb5e206d 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -1445,6 +1445,52 @@ power-domains = <&ps_avd_sys>; }; + ane_dart0: iommu@26b800000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x6b800000 0x0 0x4000>; + #iommu-cells = <1>; + apple,dma-range = <0x0 0x0 0x0 0xe0000000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "okay"; + }; + + ane_dart1: iommu@26b810000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x6b810000 0x0 0x4000>; + #iommu-cells = <1>; + apple,dma-range = <0x0 0x0 0x0 0xe0000000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "okay"; + }; + + ane_dart2: iommu@26b820000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x6b820000 0x0 0x4000>; + #iommu-cells = <1>; + apple,dma-range = <0x0 0x0 0x0 0xe0000000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "okay"; + }; + + ane: ane@26bc04000 { + compatible = "apple,t8103-ane"; + reg = <0x2 0x6bc04000 0x0 0x24000>; + reg-names = "engine"; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "ane"; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_set1>, <&ps_ane_set2>, + <&ps_ane_set3>, <&ps_ane_set4>, <&ps_ane_set5>; + status = "okay"; + }; + dispext0_dart: iommu@271304000 { compatible = "apple,t8103-dart"; reg = <0x2 0x71304000 0x0 0x4000>; From 148d9aca7f3ce68f0a47ef12d6da46873ddd5016 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:27:34 -0500 Subject: [PATCH 0347/1579] arm64: dts: apple: t600x: Add the die 0 ANE, its DARTs and power states Describe the die 0 Neural Engine of the M1 Pro, Max and Ultra: the engine window at 0x285c04000, the three ANE DARTs, and the ane_set0, ane_base and ane_set1..5 power states (ane_sys_cpu -> ane_set0 -> ane_base -> ane_set1..5). ane_set5 is described but the ane node does not use it: it lies past the end of the ADT ANE pmgr range (0xc02c). The states are added on die 0 only; the M1 Ultra die 1 ANE stays undescribed. The pmgr window already spans 0x1c000 here, so it covers 0xc000..0xc030 without change. These nodes equal the packaged omarchy-ane device-tree overlay that booted on a MacBook Pro 16" M1 Max (j316c) on 2026-09-30 (linux-asahi 7.1.13): the out-of-tree build of the ane driver bound at 285c04000.ane and an inference smoke test matched its reference output hash. Property for property, with phandles compared by target path, the subtree is identical. One name differs: the overlay calls the node ane@284000000 (the ADT engine block base), which dtc W=1 flags (simple_bus_reg) because reg starts at 0x285c04000. It is ane@285c04000 here. The driver matches on the compatible. Enabled: Linux starts no ANE firmware on T6000. Until the ane module loads, apple-dart binds the three DARTs and apple-pmgr-pwrstate registers the new states, as on that machine. The M1 Pro and M1 Ultra share this die 0 block; only the M1 Max has run it. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 43 ++++++++++++++ arch/arm64/boot/dts/apple/t600x-pmgr.dtsi | 69 +++++++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 1be41d6ff5fca..5a4641fa26c59 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -7,6 +7,49 @@ */ + ane_dart0: iommu@285800000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x85800000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "okay"; + }; + + ane_dart1: iommu@285810000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x85810000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys_cpu>; + status = "okay"; + }; + + ane_dart2: iommu@285820000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x85820000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys_cpu>; + status = "okay"; + }; + + ane: ane@285c04000 { + compatible = "apple,t6000-ane"; + reg = <0x2 0x85c04000 0x0 0x24000>; + reg-names = "engine"; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "ane"; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_sys_cpu>, <&ps_ane_set1>, + <&ps_ane_set2>, <&ps_ane_set3>, <&ps_ane_set4>; + status = "okay"; + }; + nco: clock-controller@28e03c000 { compatible = "apple,t6000-nco", "apple,nco"; reg = <0x2 0x8e03c000 0x0 0x14000>; diff --git a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi index 8b2c2cacce8c1..fed2d79a8ba8a 100644 --- a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi @@ -675,6 +675,75 @@ #reset-cells = <0>; label = DIE_LABEL(trace_fab); }; + +#if DIE_NO == 0 + /* + * Only the die 0 ANE is described. The ane node leaves set5 out: + * it lies past the end of the ADT ANE pmgr range (0xc02c). + */ + DIE_NODE(ps_ane_set0): power-controller@c000 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set0); + power-domains = <&DIE_NODE(ps_ane_sys_cpu)>; + }; + + DIE_NODE(ps_ane_base): power-controller@c008 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_base); + power-domains = <&DIE_NODE(ps_ane_set0)>; + }; + + DIE_NODE(ps_ane_set1): power-controller@c010 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set1); + power-domains = <&DIE_NODE(ps_ane_base)>; + }; + + DIE_NODE(ps_ane_set2): power-controller@c018 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set2); + power-domains = <&DIE_NODE(ps_ane_base)>; + }; + + DIE_NODE(ps_ane_set3): power-controller@c020 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set3); + power-domains = <&DIE_NODE(ps_ane_base)>; + }; + + DIE_NODE(ps_ane_set4): power-controller@c028 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set4); + power-domains = <&DIE_NODE(ps_ane_base)>; + }; + + DIE_NODE(ps_ane_set5): power-controller@c030 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set5); + power-domains = <&DIE_NODE(ps_ane_base)>; + }; +#endif }; &DIE_NODE(pmgr_east) { From 534ae2229d938da4c63a96e701939dc87c5d826e Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:27:34 -0500 Subject: [PATCH 0348/1579] arm64: dts: apple: t602x: Add the die 0 ANE Describe the die 0 Neural Engine of the M2 Pro, Max and Ultra: the whole 32 MiB engine range at 0x284000000 plus the PMGR state words and the SET window, IRQ 884, three ANE DARTs, the eight existing ANE power states with ane_cpu as the reset, the ANE coprocessor mailbox at 0x285408000, and an IOVA-only reservation for the firmware entry alias (16 MiB at 0x10000000000 in the ANE DART domain). The shared die 0 file uses apple,t6021-ane. t6020.dtsi and t6022.dtsi change it to apple,t6020-ane and apple,t6022-ane, because ane_t6021 has per-SoC data, for example the SoC revision it gives the firmware. The Apple device tree gives the ANE one interrupt line, so the mailbox names only recv-not-empty, and apple-mailbox polls for TX space. ane_dart0 is in the always-on pmp domain, as in the tree that booted; ane_sys was never booted for this DART. The nodes are enabled. By default ane_t6021 runs the firmware from its own memory (fw_alias_reserved=0), so it needs no reserved-memory node from the bootloader, and it does not start the mailbox (hello_wait_ms=0). The memory-region is an IOVA reservation only. Tested with the out-of-tree build of the same driver code: a MacBook Pro 14" M2 Max (j414c) ran linux-asahi 7.1.13 with these node values from an overlay and the packaged m1n1 1.6.1, and its test programs gave bit-exact results. That tree had two differences: the mailbox also named a placeholder send-empty line (AIC 1833), which the stock apple-mailbox needs, and the nodes came from an overlay. Earlier runs on the same machine used a one-interrupt mailbox node with a poll-TX apple-mailbox. Not tested: this kernel tree was not booted, and no M2 Pro or M2 Ultra ran these nodes. Their values are from the macOS 13.5 device trees of those machines, which give the same die 0 ANE values as the M2 Max. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t6020.dtsi | 4 ++ arch/arm64/boot/dts/apple/t6022.dtsi | 4 ++ arch/arm64/boot/dts/apple/t602x-common.dtsi | 5 ++ arch/arm64/boot/dts/apple/t602x-die0.dtsi | 59 +++++++++++++++++++++ 4 files changed, 72 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6020.dtsi b/arch/arm64/boot/dts/apple/t6020.dtsi index 500726e697a89..e9af24b25ab68 100644 --- a/arch/arm64/boot/dts/apple/t6020.dtsi +++ b/arch/arm64/boot/dts/apple/t6020.dtsi @@ -26,6 +26,10 @@ /delete-node/ &pmp_report_afnc4_ioa; /delete-node/ &pmp_report_afnc5_ioa; +&ane { + compatible = "apple,t6020-ane"; +}; + &gpu { compatible = "apple,agx-t6020", "apple,agx-g14x", "apple,agx-g14s"; diff --git a/arch/arm64/boot/dts/apple/t6022.dtsi b/arch/arm64/boot/dts/apple/t6022.dtsi index fe065d48726a3..875eb875e907c 100644 --- a/arch/arm64/boot/dts/apple/t6022.dtsi +++ b/arch/arm64/boot/dts/apple/t6022.dtsi @@ -328,6 +328,10 @@ #undef DIE #undef DIE_NO +&ane { + compatible = "apple,t6022-ane"; +}; + &aic { affinities { e-core-pmu-affinity { diff --git a/arch/arm64/boot/dts/apple/t602x-common.dtsi b/arch/arm64/boot/dts/apple/t602x-common.dtsi index c985e0e8d1c3f..295a46647de4b 100644 --- a/arch/arm64/boot/dts/apple/t602x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-common.dtsi @@ -627,5 +627,10 @@ uat_ttbs: uat-ttbs { reg = <0 0 0 0>; }; + + /* IOVA-only: the ANE firmware entry alias, 16 MiB at 0x10000000000 */ + ane_alias_iova: ane-alias-iova { + iommu-addresses = <&ane 0x100 0x0 0x0 0x1000000>; + }; }; }; diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi index 852b33a88b813..a465b6f44fb2f 100644 --- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi @@ -6,6 +6,65 @@ * Copyright The Asahi Linux Contributors */ + ane_mbox: mailbox@285408000 { + compatible = "apple,t6021-ane-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x85408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "recv-not-empty"; + #mbox-cells = <0>; + status = "okay"; + }; + + ane_dart0: iommu@285800000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x85800000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_pmp>; + status = "okay"; + }; + + ane_dart1: iommu@285810000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x85810000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_cpu>; + status = "okay"; + }; + + ane_dart2: iommu@285820000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x85820000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_cpu>; + status = "okay"; + }; + + ane: ane@284000000 { + compatible = "apple,t6021-ane"; + reg = <0x2 0x84000000 0x0 0x2000000>, + <0x2 0x8e080000 0x0 0x4034>, + <0x2 0x8e08c000 0x0 0x4000>; + reg-names = "engine", "pmgr", "set"; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "ane"; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_sys_mpm>, <&ps_ane_td>, + <&ps_ane_base>, <&ps_ane_set1>, <&ps_ane_set2>, + <&ps_ane_set3>, <&ps_ane_set4>, <&ps_ane_cpu>; + resets = <&ps_ane_cpu>; + mboxes = <&ane_mbox>; + memory-region = <&ane_alias_iova>; + status = "okay"; + }; + nco: clock-controller@28e03c000 { compatible = "apple,t6020-nco", "apple,t8103-nco", "apple,nco"; reg = <0x2 0x8e03c000 0x0 0x14000>; From aa1255181fea403ff940544bf53d8a524dd7f2dd Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:28:59 -0500 Subject: [PATCH 0349/1579] arm64: dts: apple: t8112: Add the ANE, disabled Describe the M2 Neural Engine: the whole 32 MiB engine range at 0x26a000000 plus the PMGR window, the SET window and the two eFuse words that give the SoC revision to the firmware, IRQ 520, three ANE DARTs (IRQ 521, in ane_sys), the ANE coprocessor mailbox at 0x26b408000, and the seven ANE power states ane_sys_cpu, ane_td, ane_base and ane_set1..4 (0xc008..0xc038). ane_sys_cpu is a child of ane_sys, and each later state is a child of the one before. The values are those of the omarchy-ane T8112 overlay. Its sources are the /arm-io/ane and /arm-io/dart-ane nodes of the macOS 13.5 and 27.0 device trees, which are the same on j413, j415, j473 and j493, and the power-on order of the macOS 13.5 ANE kext. The mailbox sits at the same offset in the ANE block as on T6021 and names only recv-not-empty (the ANE interrupt); the overlay adds a placeholder send-empty line, which this tree does not need because apple-mailbox polls for TX. The mailbox, the DARTs and the ane node stay disabled: no T8112 has run the ane_t6021 driver or booted these nodes. The other ANE nodes in this series have the values of trees that ran the driver on an M1, an M1 Max or an M2 Max. A board or overlay that tests the driver enables the five nodes. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8112-pmgr.dtsi | 63 +++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8112.dtsi | 58 +++++++++++++++++++++ 2 files changed, 121 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index 128a560dbac73..0e8df1cfa21f6 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -1132,6 +1132,69 @@ power-domains = <&ps_venc_pipe5>, <&ps_venc_pipe4>; }; + ps_ane_sys_cpu: power-controller@c008 { + compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_sys_cpu"; + power-domains = <&ps_ane_sys>; + }; + + ps_ane_td: power-controller@c010 { + compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_td"; + power-domains = <&ps_ane_sys_cpu>; + }; + + ps_ane_base: power-controller@c018 { + compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_base"; + power-domains = <&ps_ane_td>; + }; + + ps_ane_set1: power-controller@c020 { + compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set1"; + power-domains = <&ps_ane_base>; + }; + + ps_ane_set2: power-controller@c028 { + compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set2"; + power-domains = <&ps_ane_set1>; + }; + + ps_ane_set3: power-controller@c030 { + compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set3"; + power-domains = <&ps_ane_set2>; + }; + + ps_ane_set4: power-controller@c038 { + compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_set4"; + power-domains = <&ps_ane_set3>; + }; + ps_disp0_cpu0: power-controller@10000 { compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x10000 4>; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 76a9fa23709d0..5f3db6baa77c5 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -1707,6 +1707,64 @@ power-domains = <&ps_avd_sys>; }; + ane_mbox: mailbox@26b408000 { + compatible = "apple,t8112-ane-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x6b408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + ane_dart0: iommu@26b800000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x6b800000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + ane_dart1: iommu@26b810000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x6b810000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + ane_dart2: iommu@26b820000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x6b820000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + ane: ane@26a000000 { + compatible = "apple,t8112-ane"; + reg = <0x2 0x6a000000 0x0 0x2000000>, + <0x2 0x3b700000 0x0 0x18000>, + <0x2 0x3b724000 0x0 0x4000>, + <0x2 0x3d2c8060 0x0 0x8>; + reg-names = "engine", "pmgr", "set", "fuse"; + interrupt-parent = <&aic>; + interrupts = ; + interrupt-names = "ane"; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_sys_cpu>, <&ps_ane_td>, + <&ps_ane_base>, <&ps_ane_set1>, <&ps_ane_set2>, + <&ps_ane_set3>, <&ps_ane_set4>; + mboxes = <&ane_mbox>; + status = "disabled"; + }; + dispext0_dart: iommu@271304000 { compatible = "apple,t8112-dart", "apple,t8110-dart"; reg = <0x2 0x71304000 0x0 0x4000>; From b74fa4deaae4a6cd09fc62d304d1aad541c6e095 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:23:13 -0500 Subject: [PATCH 0350/1579] accel/ane: Bind only the M1 family in the ane module The ane module matched apple,t6020-ane and apple,t6021-ane, but it refused to bind both: T6020 had no proven SET-block base, and T6021 bound only with the allow_unqualified module parameter. The M2 family does not use the host-driven task manager of this module. Its coprocessor runs the ANE firmware, and the ane_t6021 module drives it for T6020, T6021, T6022 and T8112. The two entries made udev load both modules for every T602x ANE node, and the ane probe, when it ran first, failed with -ENODEV and logged an error. Remove the two entries and their SoC descriptors. Every remaining SoC has proven constants, so also remove the qualification tiers and the allow_unqualified parameter, and get the descriptor with of_device_get_match_data(). Each compatible of the apple,ane binding, and each ANE node in the apple device trees, now matches exactly one of the two match tables. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_drv.c | 94 +++++-------------------------------- 1 file changed, 11 insertions(+), 83 deletions(-) diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index db03774c9e25a..5b269b0f8729f 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -880,54 +880,33 @@ static int ane_attach_genpd(struct ane_device *ane) } /* - * Per-SoC support descriptor, one per Linux `compatible`. + * Per-SoC descriptor, one per Linux `compatible`. * * Everything the probe needs except the SET block is derived from the * device tree by design: DART topology from "iommus", power wiring from - * "power-domains", the engine window from "reg", the IRQ by name. Board - * facts belong in the overlay, not in compiled constants. The SET block - * is the one exception: the live overlays carry no range for it, so it - * is compiled in per SoC — and only from a source that has proven the + * "power-domains", the engine window from "reg", the IRQ by name. The SET + * block is the one exception: the device tree carries no range for it, so + * it is compiled in per SoC, and only from a source that has proven the * address on hardware (m1n1 proxyclient ANE.ps_map, then a bound device). * A guessed SET base is not a bug but a brick: direct writes to the * block external-abort the SoC (T6001 named by netconsole 2026-09-16, - * PS_SET0 down at 0x28e08c000; T8103 same mechanism at 0x23b70c000), - * which is why a SoC without proven constants must refuse to bind - * instead of carrying a guess. - * - * Qualification tiers: - * ANE_QUALIFIED — execution proven on this silicon; binds normally. - * ANE_RECOGNIZED — constants present, never run on hardware; binds - * only with ane.allow_unqualified=1, loudly. - * ANE_UNSUPPORTED — no proven constants; refuses and names the data - * needed to advance the port. + * PS_SET0 down at 0x28e08c000; T8103 same mechanism at 0x23b70c000). + * The M2 family (T602x, T8112) runs the ANE firmware and binds to + * ane_t6021 instead. */ -enum ane_qual { - ANE_QUALIFIED, - ANE_RECOGNIZED, - ANE_UNSUPPORTED, -}; - struct ane_soc { phys_addr_t ps_base; - enum ane_qual qual; /* True when the tm/tq register file survives a genpd cycle in * retention and recovery must drain it (see ane_tm_drain_retained). */ bool tm_retention; }; -static bool allow_unqualified; -module_param(allow_unqualified, bool, 0444); -MODULE_PARM_DESC(allow_unqualified, - "Bind recognized-but-unproven ANE SoCs (unverified SET base: external-abort risk)"); - static const struct ane_soc ane_soc_t8103 = { /* M1 (m1-test-host). SET block mapped read-only for the recovery ACTUAL * log and the powered-on guard. Execution proven in the fleet. */ .ps_base = 0x23b70c000ULL, - .qual = ANE_QUALIFIED, }; static const struct ane_soc ane_soc_t6000 = { @@ -935,42 +914,12 @@ static const struct ane_soc ane_soc_t6000 = { * (proven on T6001/t6001-test-host); M1 Pro and M1 Ultra are untested. */ .ps_base = 0x28e08c000ULL, - .qual = ANE_QUALIFIED, .tm_retention = true, }; -static const struct ane_soc ane_soc_t6020 = { - /* M2 Pro (t6020). Community device-tree captures exist, but the - * SET base is unproven and the H14 compiler backend is unqualified: - * no constants may enter here yet. - */ - .ps_base = 0, - .qual = ANE_UNSUPPORTED, -}; - -static const struct ane_soc ane_soc_t6021 = { - /* M2 Max (T6021, t6021-test-host). SET window 0x8e08c000 = pmgr base - * 0x8e080000 + 0xc000, present in ane0's own reg on macOS captures - * of both 26.6.2 and 27.0 (2026-09-18, 11/11 derivation-input - * checks each, zero delta); the Linux translation reuses the pmgr - * high bits proven on T6000/T6020 rows (low-32 match) -> - * 0x28e08c000. Recognized, not qualified: constants complete, but - * no Linux execution yet, and the t6021 pwrstate word layout - * (which of set0/base/set1..4 sits at which offset — macOS-side - * capture cannot see it) is unverified; the first probe resume - * logs the ACTUAL nibbles through this window as the Linux-side - * probe (receipt 2026-09-18-t6021-driver-entry-prepared.md). - * Promotion to ANE_QUALIFIED needs an exact run on this silicon. - */ - .ps_base = 0x28e08c000ULL, - .qual = ANE_RECOGNIZED, -}; - static const struct of_device_id ane_of_match[] = { { .compatible = "apple,t8103-ane", .data = &ane_soc_t8103 }, { .compatible = "apple,t6000-ane", .data = &ane_soc_t6000 }, - { .compatible = "apple,t6020-ane", .data = &ane_soc_t6020 }, - { .compatible = "apple,t6021-ane", .data = &ane_soc_t6021 }, {} }; @@ -979,32 +928,11 @@ MODULE_DEVICE_TABLE(of, ane_of_match); static int ane_platform_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - const struct of_device_id *id = of_match_device(ane_of_match, dev); - const struct ane_soc *soc = id->data; + const struct ane_soc *soc = of_device_get_match_data(dev); struct ane_device *ane; struct drm_device *drm; int err; - /* Tier gate before any power-domain, MMIO or IRQ interaction: an - * unqualified SoC must fail cleanly, never half-probe. - */ - if (soc->qual == ANE_UNSUPPORTED) { - dev_err(dev, - "%s: unsupported ANE: no proven SET-block base (a guessed base external-aborts the SoC); not binding. To advance this port, run the mlx-omarchy quick collector (scripts/collect_quick.py: captures ANE/DART/PMGR/AIC device-tree data, no driver needed) and submit the capture\n", - id->compatible); - return -ENODEV; - } - if (soc->qual == ANE_RECOGNIZED && !allow_unqualified) { - dev_err(dev, - "%s: recognized but unqualified: constants present, execution never proven on this silicon; not binding. Override with ane.allow_unqualified=1, or prove a run and report it\n", - id->compatible); - return -ENODEV; - } - if (soc->qual == ANE_RECOGNIZED) - dev_warn(dev, - "%s: UNQUALIFIED bind forced by allow_unqualified: no execution proven on this silicon — SET-block base and board topology unverified\n", - id->compatible); - ane = devm_drm_dev_alloc(dev, &ane_drm_driver, struct ane_device, drm); if (IS_ERR(ane)) return PTR_ERR(ane); @@ -1172,7 +1100,7 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) * SET window reads come first (pmgr class, always safe * per T6001/T8103 bisect evidence), then the engine. A * hard reset after the last off-box line names the - * killing access exactly (T6021 console bring-up). + * killing access exactly. */ struct resource *eng = platform_get_resource_byname( to_platform_device(dev), IORESOURCE_MEM, "engine"); @@ -1198,8 +1126,8 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) /* Linux-side pwrstate probe: ACTUAL nibbles read through * the SoC descriptor's SET window while the partition is * raised. 0xffffff means the mapped window is the live - * pmgr SET block with every word on; anything else names - * the t6021 word layout to fix before promotion. + * pmgr SET block with every word on; anything else means + * the window does not match the expected word layout. */ dev_info(dev, "ANERD ps probe act=%#x\n", ane_ps_act(ane)); } From cfb77b6fc67fe44f2ed7a0c88605f3140128870e Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:29:44 -0500 Subject: [PATCH 0351/1579] accel/ane: Clean up the UAPI and add DRM_IOCTL_ANE_GET_CAPS Prepare include/uapi/drm/ane_accel.h for a stable in-tree ABI. Prefix every macro of the header with DRM_ANE_, as other DRM UAPI headers do: ANE_TILE_COUNT, ANE_FIFO_NID and ANE_CMD_GRAN become DRM_ANE_TILE_COUNT, DRM_ANE_FIFO_NID and DRM_ANE_CMD_GRAN, ANE_ABI_MAJOR and ANE_ABI_M2_MAJOR become DRM_ANE_ABI_V1 and DRM_ANE_ABI_V2, and ANE_M2_MAX_BINDS becomes DRM_ANE_MAX_BINDS. ANE_M2_MAX_SECTIONS was 16, but the driver accepts at most 9 sections (ids 1 to 9). It becomes DRM_ANE_MAX_SECTIONS with the value 9, and a BUILD_BUG_ON() keeps it equal to the firmware LOAD record count. Add DRM_IOCTL_ANE_GET_CAPS (number 0x00, free until now) to both drivers. It reports the ABI version, the chip family (H13 or H14), and the size of each array entry that PROG_LOAD and EXEC read through a user pointer. Its flags field has no defined flag and must be zero. Its size field gives the number of bytes that the driver filled in, so the structure can grow at its end. Reject a nonzero pad or reserved field with -EINVAL in every ioctl: BO_INIT and BO_FREE on ABI version 2 (ABI version 1 already did), the pad of PROG_LOAD, of each generic bind and of EXEC, and the flags and reserved fields of each EXEC buffer. The field drm_ane_exec_io.dma was already reserved: the driver gives the firmware the address of bo_handle. Rename it to reserved. Add kernel-doc for every structure. Keep one header with an explicit ABI version, not one header per ABI. The two drivers share the DRM driver name, the accel node, the BO_INIT/BO_FREE structures and the ioctl number space. One header per ABI would either duplicate the shared structures under two names or need a third header for them, and it would let the two headers give one ioctl number to two structures. Userspace already supports both ABI versions from one binary and selects by the version major. The version is explicit: DRM_IOCTL_VERSION major and drm_ane_get_caps.abi_version both report DRM_ANE_ABI_V1 or DRM_ANE_ABI_V2, an ioctl number is never reused, and a driver rejects the ioctls of the other version. The ioctl numbers and the layout of every existing structure do not change: an aarch64 offsetof()/sizeof() check of the old and new header gives identical values for all existing structures, fields and ioctl command values. Userspace that left a pad field of BO_FREE or of a generic bind uninitialised can now get -EINVAL on ABI version 2. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane.h | 2 +- drivers/accel/ane/ane_drv.c | 37 +++- drivers/accel/ane/ane_t6021_rtclient_main.c | 48 ++++- drivers/accel/ane/ane_tm.c | 3 +- include/uapi/drm/ane_accel.h | 207 ++++++++++++++++---- 5 files changed, 238 insertions(+), 59 deletions(-) diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index 10fee2864728c..c644ec45e67d4 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -115,7 +115,7 @@ struct ane_request { u32 td_size; u32 td_count; u32 btsp_iova; - u32 bar[ANE_TILE_COUNT]; + u32 bar[DRM_ANE_TILE_COUNT]; }; /* diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 5b269b0f8729f..8b38416c20b36 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -352,6 +352,22 @@ static const struct drm_gem_object_funcs ane_gem_object_funcs = { .vm_ops = &drm_gem_ane_vm_ops, }; +static int ane_get_caps(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct drm_ane_get_caps *args = data; + + if (args->flags || args->pad) + return -EINVAL; + + *args = (struct drm_ane_get_caps) { + .size = sizeof(*args), + .abi_version = DRM_ANE_ABI_V1, + .chip_family = DRM_ANE_CHIP_H13, + }; + return 0; +} + static int ane_bo_init(struct drm_device *drm, void *data, struct drm_file *file) { @@ -460,7 +476,7 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) { struct ane_device *ane = drm->dev_private; struct drm_ane_submit *args = data; - struct drm_gem_object *gem[ANE_TILE_COUNT] = { 0 }; + struct drm_gem_object *gem[DRM_ANE_TILE_COUNT] = { 0 }; struct drm_gem_object *btsp = NULL; struct ane_bo *bo; struct ane_request req; @@ -483,11 +499,11 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) } req.qid = 4; - req.nid = ANE_FIFO_NID; + req.nid = DRM_ANE_FIFO_NID; req.td_size = args->td_size; req.td_count = args->td_count; - for (int bdx = 0; bdx < ANE_TILE_COUNT; bdx++) { + for (int bdx = 0; bdx < DRM_ANE_TILE_COUNT; bdx++) { if (!args->handles[bdx]) continue; bo = bo_lookup(file, args->handles[bdx]); @@ -510,8 +526,8 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) * access. Since this isn't page aligned, we represent the two as one * buffer and calculate the delimiter (where the weights would start). */ - req.bar[KRN_BUF_BDX] = - req.bar[CMD_BUF_BDX] + round_up(args->tsk_size, ANE_CMD_GRAN); + req.bar[KRN_BUF_BDX] = req.bar[CMD_BUF_BDX] + + round_up(args->tsk_size, DRM_ANE_CMD_GRAN); bo = bo_lookup(file, args->btsp_handle); if (!bo) { @@ -553,7 +569,7 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) mutex_unlock(&ane->engine_lock); put: drm_gem_object_put(btsp); - for (int bdx = 0; bdx < ANE_TILE_COUNT; bdx++) { + for (int bdx = 0; bdx < DRM_ANE_TILE_COUNT; bdx++) { if (gem[bdx]) drm_gem_object_put(gem[bdx]); } @@ -561,6 +577,7 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) } static const struct drm_ioctl_desc ane_drm_ioctls[] = { + DRM_IOCTL_DEF_DRV(ANE_GET_CAPS, ane_get_caps, 0), DRM_IOCTL_DEF_DRV(ANE_BO_INIT, ane_bo_init, 0), DRM_IOCTL_DEF_DRV(ANE_BO_FREE, ane_bo_free, 0), DRM_IOCTL_DEF_DRV(ANE_SUBMIT, ane_submit, 0), @@ -656,9 +673,9 @@ static void ane_drm_postclose(struct drm_device *drm, struct drm_file *file) } /* - * The driver only owns three ioctls (ANE_BO_INIT, ANE_BO_FREE, - * ANE_SUBMIT). Anything outside that table would otherwise fall through - * the standard drm_ioctl path to legacy primary-node handlers that + * The driver only owns four ioctls (ANE_GET_CAPS, ANE_BO_INIT, + * ANE_BO_FREE, ANE_SUBMIT). The standard drm_ioctl path would otherwise + * pass anything outside that table to legacy primary-node handlers that * dereference drm_device.unique, which an accel node never sets. Reject * such callers explicitly so they fail with -ENOTTY instead of oopsing. * @@ -764,7 +781,7 @@ static const struct drm_driver ane_drm_driver = { .ioctls = ane_drm_ioctls, .num_ioctls = ARRAY_SIZE(ane_drm_ioctls), .fops = &ane_drm_fops, - .major = ANE_ABI_MAJOR, + .major = DRM_ANE_ABI_V1, .name = "ane", .desc = "Apple Neural Engine driver", }; diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 9d1e43ecc508b..e5cda3d046ed6 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -990,9 +990,10 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, size_t binds_size; int ret, i, j; - if (user->section_count < 1 || + BUILD_BUG_ON(DRM_ANE_MAX_SECTIONS != ANE_T6021_LOAD_SEC_COUNT); + if (user->pad || user->section_count < 1 || user->section_count > ANE_T6021_LOAD_SEC_COUNT || - user->generic_count > ANE_M2_MAX_BINDS || !fd) + user->generic_count > DRM_ANE_MAX_BINDS || !fd) return -EINVAL; sections = kmalloc_array(user->section_count, sizeof(*sections), @@ -1016,6 +1017,12 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, ret = -EFAULT; goto out; } + for (i = 0; i < user->generic_count; i++) { + if (binds[i].pad) { + ret = -EINVAL; + goto out; + } + } /* Lab contract: the record slot IS the section identity (slot = * id - 1; tdprop id 7 lands at slot 6 and slot 5 stays empty, @@ -1248,7 +1255,7 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, size_t cmd_size; int ret, i; - if (user->count < 1 || user->count > ANE_M2_MAX_BINDS || + if (user->pad || user->count < 1 || user->count > DRM_ANE_MAX_BINDS || user->priority < 2 || user->priority > 7 || !fd) return -EINVAL; @@ -1260,6 +1267,12 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, kfree(ios); return -EFAULT; } + for (i = 0; i < user->count; i++) { + if (ios[i].flags || ios[i].reserved) { + kfree(ios); + return -EINVAL; + } + } cmd_size = sizeof(struct ane_csne_cmd_procedure_call) + (size_t)user->count * sizeof(struct ane_csne_io_elem); @@ -1364,7 +1377,8 @@ static int ane_t6021_bo_init_ioctl(struct drm_device *drm, void *data, struct ane_t6021_bo *bo; struct ane_rtclient *ane; - if (args->size == 0 || args->size > ANE_T6021_BO_MAX || !fd) + if (args->pad || args->size == 0 || args->size > ANE_T6021_BO_MAX || + !fd) return -EINVAL; /* A parked io BO is already mapped and counted; its old contents * belong to another process, so it is zeroed like a new one. @@ -1447,6 +1461,8 @@ static int ane_t6021_bo_free_ioctl(struct drm_device *drm, void *data, if (!fd) return -ENODEV; + if (args->pad) + return -EINVAL; mutex_lock(&ane_t6021_bo_lock); list_for_each_entry(b, &fd->bos, node) { if (b->handle == args->handle) { @@ -1540,6 +1556,25 @@ static void ane_t6021_postclose(struct drm_device *drm, struct drm_file *file) file->driver_priv = NULL; } +static int ane_t6021_get_caps_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct drm_ane_get_caps *args = data; + + if (args->flags || args->pad) + return -EINVAL; + + *args = (struct drm_ane_get_caps) { + .size = sizeof(*args), + .abi_version = DRM_ANE_ABI_V2, + .chip_family = DRM_ANE_CHIP_H14, + .section_size = sizeof(struct drm_ane_section), + .bind_size = sizeof(struct drm_ane_generic_bind), + .exec_io_size = sizeof(struct drm_ane_exec_io), + }; + return 0; +} + static int ane_t6021_submit_ioctl(struct drm_device *drm, void *data, struct drm_file *file) { @@ -1650,6 +1685,7 @@ static int ane_t6021_exec_ioctl(struct drm_device *drm, void *data, } static const struct drm_ioctl_desc ane_t6021_ioctls[] = { + DRM_IOCTL_DEF_DRV(ANE_GET_CAPS, ane_t6021_get_caps_ioctl, 0), DRM_IOCTL_DEF_DRV(ANE_BO_INIT, ane_t6021_bo_init_ioctl, 0), DRM_IOCTL_DEF_DRV(ANE_BO_FREE, ane_t6021_bo_free_ioctl, 0), DRM_IOCTL_DEF_DRV(ANE_SUBMIT, ane_t6021_submit_ioctl, 0), @@ -1685,7 +1721,7 @@ static const struct drm_driver ane_t6021_drm_driver = { .ioctls = ane_t6021_ioctls, .num_ioctls = ARRAY_SIZE(ane_t6021_ioctls), .fops = &ane_t6021_fops, - .major = ANE_ABI_M2_MAJOR, + .major = DRM_ANE_ABI_V2, .minor = 0, .name = "ane", .desc = "Apple Neural Engine (T6021/M2)", @@ -2289,7 +2325,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) } dev_info(dev, "loaded ane_t6021 (DRM major %d minor %d; ABI 2; legacy_only=%u chman_ok=%u booted=%u; state %s; BO cap %u MiB)\n", - ANE_ABI_M2_MAJOR, 0, + DRM_ANE_ABI_V2, 0, legacy_only, ane->chman_ok, ane->fw ? ane->fw->booted : 0, ane->held ? "HELD" : "ready", bo_total_max_mb); diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 0e98c48d2aff1..e81af0a7b96db 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -195,9 +195,8 @@ int ane_tm_enqueue(struct ane_device *ane, struct ane_request *req) tq_write32(ane, TQ_STATUS(qid), 0x1); - for (int bdx = 0; bdx < ANE_TILE_COUNT; bdx++) { + for (int bdx = 0; bdx < DRM_ANE_TILE_COUNT; bdx++) tq_write32(ane, TQ_BAR1(qid, bdx), req->bar[bdx]); - } tq_write32(ane, TQ_SIZE1(qid), ((req->td_size >> 2) - 1) << 0x10); tq_write32(ane, TQ_ADDR1(qid), req->btsp_iova); diff --git a/include/uapi/drm/ane_accel.h b/include/uapi/drm/ane_accel.h index e5fd9d12d1d63..76577277b70eb 100644 --- a/include/uapi/drm/ane_accel.h +++ b/include/uapi/drm/ane_accel.h @@ -1,8 +1,8 @@ /* SPDX-License-Identifier: (GPL-2.0-only WITH Linux-syscall-note) OR MIT */ /* Copyright 2022 Eileen Yoon */ -#ifndef __ANE_ACCEL_H__ -#define __ANE_ACCEL_H__ +#ifndef __DRM_UAPI_ANE_ACCEL_H__ +#define __DRM_UAPI_ANE_ACCEL_H__ #include "drm.h" @@ -10,31 +10,85 @@ extern "C" { #endif -#define ANE_TILE_COUNT 0x20 -#define ANE_FIFO_NID 0x40 -#define ANE_CMD_GRAN 0x10 +/** + * DOC: ABI versions + * + * Two drivers implement this interface on an accel node, both with the DRM + * driver name "ane": + * + * - ane (H13 family: T8103, T6000, T6001, T6002) implements ABI version + * %DRM_ANE_ABI_V1: GET_CAPS, BO_INIT, BO_FREE and SUBMIT. + * - ane_t6021 (H14 family: T6020, T6021, T6022, T8112) implements ABI + * version %DRM_ANE_ABI_V2: GET_CAPS, BO_INIT, BO_FREE, PROG_LOAD, + * PROC_CREATE and EXEC. + * + * The DRM_IOCTL_VERSION major and &drm_ane_get_caps.abi_version both give + * the ABI version. The two versions share the ioctl numbers and the + * structures, and a number is never given to a second ioctl. A driver + * rejects the ioctls of the other version. A structure grows only at its + * end; &drm_ane_get_caps.size tells which fields the driver filled in. + * + * Every pad and reserved field, and every flags field with no flags + * defined, must be zero. The drivers reject a nonzero value with -EINVAL. + */ +#define DRM_ANE_ABI_V1 1 +#define DRM_ANE_ABI_V2 2 -/* ABI 1: successful SUBMIT guarantees terminal completion and CPU visibility. */ -#define ANE_ABI_MAJOR 1 +#define DRM_ANE_CHIP_H13 13 +#define DRM_ANE_CHIP_H14 14 -/* - * ABI 2 (T6021 / M2, H14 firmware programs): the driver reports - * DRM_IOCTL_VERSION major 2 and rejects DRM_ANE_SUBMIT. The M1 libane - * requires major 1, so it cannot submit M1 frames to an M2 device. - * A successful DRM_ANE_EXEC guarantees terminal completion (firmware ack - * and task-queue idle) and CPU visibility of the output buffers. - */ -#define ANE_ABI_M2_MAJOR 2 -#define ANE_M2_MAX_SECTIONS 16 -#define ANE_M2_MAX_BINDS 64 +/* ABI version 1: entries of &drm_ane_submit.handles */ +#define DRM_ANE_TILE_COUNT 0x20 +/* ABI version 1: network id of the queue that SUBMIT uses */ +#define DRM_ANE_FIFO_NID 0x40 +/* ABI version 1: the weights start at round_up(tsk_size, DRM_ANE_CMD_GRAN) */ +#define DRM_ANE_CMD_GRAN 0x10 + +/* ABI version 2: highest section id, and most sections per PROG_LOAD */ +#define DRM_ANE_MAX_SECTIONS 9 +/* ABI version 2: most generic binds per PROG_LOAD and buffers per EXEC */ +#define DRM_ANE_MAX_BINDS 64 -#define DRM_ANE_BO_INIT 0x1 -#define DRM_ANE_BO_FREE 0x2 -#define DRM_ANE_SUBMIT 0x3 -#define DRM_ANE_PROG_LOAD 0x4 -#define DRM_ANE_PROC_CREATE 0x5 -#define DRM_ANE_EXEC 0x6 +#define DRM_ANE_GET_CAPS 0x00 +#define DRM_ANE_BO_INIT 0x01 +#define DRM_ANE_BO_FREE 0x02 +#define DRM_ANE_SUBMIT 0x03 +#define DRM_ANE_PROG_LOAD 0x04 +#define DRM_ANE_PROC_CREATE 0x05 +#define DRM_ANE_EXEC 0x06 + +/** + * struct drm_ane_get_caps - DRM_IOCTL_ANE_GET_CAPS argument + * @flags: In. No flags are defined: must be zero. + * @size: Out. Bytes of this structure that the driver filled in. + * @abi_version: Out. %DRM_ANE_ABI_V1 or %DRM_ANE_ABI_V2. + * @chip_family: Out. %DRM_ANE_CHIP_H13 or %DRM_ANE_CHIP_H14. + * @section_size: Out. Size of the &drm_ane_section entries that PROG_LOAD + * reads, or 0 if the driver has no PROG_LOAD. + * @bind_size: Out. Size of the &drm_ane_generic_bind entries that PROG_LOAD + * reads, or 0 if the driver has no PROG_LOAD. + * @exec_io_size: Out. Size of the &drm_ane_exec_io entries that EXEC reads, + * or 0 if the driver has no EXEC. + * @pad: Must be zero. + */ +struct drm_ane_get_caps { + __u32 flags; + __u32 size; + __u32 abi_version; + __u32 chip_family; + __u32 section_size; + __u32 bind_size; + __u32 exec_io_size; + __u32 pad; +}; +/** + * struct drm_ane_bo_init - DRM_IOCTL_ANE_BO_INIT argument + * @handle: Out. Handle of the new buffer. + * @pad: Must be zero. + * @size: In. Size of the buffer in bytes, not zero. + * @offset: Out. Offset to give to mmap() on the accel node. + */ struct drm_ane_bo_init { __u32 handle; __u32 pad; @@ -42,29 +96,65 @@ struct drm_ane_bo_init { __u64 offset; }; +/** + * struct drm_ane_bo_free - DRM_IOCTL_ANE_BO_FREE argument + * @handle: In. Handle from BO_INIT. + * @pad: Must be zero. + */ struct drm_ane_bo_free { __u32 handle; __u32 pad; }; +/** + * struct drm_ane_submit - DRM_IOCTL_ANE_SUBMIT argument (ABI version 1) + * @tsk_size: In. Size of the task microcode at the start of handles[0], not + * zero. + * @td_count: In. Number of task descriptors, 1 to 0xffff. + * @td_size: In. Size of a task descriptor: a multiple of 4, 4 to 0x40000. + * @handles: In. Buffer per tile: handles[0] holds the microcode and the + * weights, handles[1] must be 0 (the driver sets it from handles[0]), + * and 0 marks an unused tile. + * @btsp_handle: In. Buffer with the bootstrap task descriptor. + * @pad: Must be zero. + * + * A successful SUBMIT returns after the engine completed the work, with the + * output buffers visible to the CPU. + */ struct drm_ane_submit { __u64 tsk_size; __u32 td_count; __u32 td_size; - __u32 handles[ANE_TILE_COUNT]; + __u32 handles[DRM_ANE_TILE_COUNT]; __u32 btsp_handle; __u32 pad; }; -/* One firmware program section, already built by userspace, held in a BO. */ +/** + * struct drm_ane_section - one firmware program section (ABI version 2) + * @id: Section id, 1 to %DRM_ANE_MAX_SECTIONS, unique in one PROG_LOAD. + * @bo_handle: Buffer that holds the section bytes. + * @size: Size of the section in bytes. + * @offset: Offset of the section in the buffer. + */ struct drm_ane_section { - __u32 id; /* firmware section id */ - __u32 bo_handle; /* BO holding the bytes */ + __u32 id; + __u32 bo_handle; __u64 size; - __u64 offset; /* byte offset inside the BO */ + __u64 offset; }; -/* A buffer whose IOVA and size the driver patches into a generic entry. */ +/** + * struct drm_ane_generic_bind - buffer of a program (ABI version 2) + * @buffer_id: Firmware buffer id. + * @bo_handle: Buffer handle. + * @type: Buffer type. + * @pad: Must be zero. + * @size: Size of the buffer in bytes. + * + * The driver checks @pad and ignores the other fields: the sections that + * userspace builds already hold this information. + */ struct drm_ane_generic_bind { __u32 buffer_id; __u32 bo_handle; @@ -73,49 +163,86 @@ struct drm_ane_generic_bind { __u64 size; }; +/** + * struct drm_ane_prog_load - DRM_IOCTL_ANE_PROG_LOAD argument (ABI version 2) + * @sections_ptr: In. Pointer to struct drm_ane_section[@section_count]. + * @generic_ptr: In. Pointer to struct drm_ane_generic_bind[@generic_count]. + * @section_count: In. 1 to %DRM_ANE_MAX_SECTIONS. + * @generic_count: In. 0 to %DRM_ANE_MAX_BINDS. + * @prog_id_out: Out. Firmware program id. Identical sections give the same + * program id. + * @pad: Must be zero. + */ struct drm_ane_prog_load { - __u64 sections_ptr; /* struct drm_ane_section[section_count] */ - __u64 generic_ptr; /* struct drm_ane_generic_bind[generic_count] */ + __u64 sections_ptr; + __u64 generic_ptr; __u32 section_count; __u32 generic_count; __u32 prog_id_out; __u32 pad; }; +/** + * struct drm_ane_proc_create - DRM_IOCTL_ANE_PROC_CREATE argument + * (ABI version 2) + * @prog_id: In. Program id from PROG_LOAD. + * @proc_id_out: Out. Firmware process id. + */ struct drm_ane_proc_create { __u32 prog_id; __u32 proc_id_out; }; -/* Per-call buffer binding: an input or output for one procedure call. */ +/** + * struct drm_ane_exec_io - input or output buffer of one EXEC + * (ABI version 2) + * @buffer_id: Firmware buffer id. + * @bo_handle: Buffer handle. The driver gives its address to the firmware. + * @type: Buffer type. + * @flags: No flags are defined: must be zero. + * @reserved: Must be zero. + * @size: Size of the buffer in bytes. + */ struct drm_ane_exec_io { __u32 buffer_id; __u32 bo_handle; __u32 type; __u32 flags; - __u64 dma; /* in only, reserved: the driver binds the - * IOVA from bo_handle - */ + __u64 reserved; __u64 size; }; +/** + * struct drm_ane_exec - DRM_IOCTL_ANE_EXEC argument (ABI version 2) + * @prog_id: In. Program id from PROG_LOAD. + * @proc_id: In. Process id from PROC_CREATE. + * @priority: In. 2 to 7. + * @timeout_ms: In. Firmware completion timeout, 0 for 5000 ms. + * @count: In. 1 to %DRM_ANE_MAX_BINDS. + * @pad: Must be zero. + * @io_ptr: In. Pointer to struct drm_ane_exec_io[@count]. + * + * A successful EXEC returns after the firmware acknowledged completion and + * the task queue is idle, with the output buffers visible to the CPU. + */ struct drm_ane_exec { __u32 prog_id; __u32 proc_id; - __u32 priority; /* 2..7 */ + __u32 priority; __u32 timeout_ms; - __u32 count; /* 1..ANE_M2_MAX_BINDS */ + __u32 count; __u32 pad; - __u64 io_ptr; /* struct drm_ane_exec_io[count] */ + __u64 io_ptr; }; +#define DRM_IOCTL_ANE_GET_CAPS \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_GET_CAPS, struct drm_ane_get_caps) #define DRM_IOCTL_ANE_BO_INIT \ DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_BO_INIT, struct drm_ane_bo_init) #define DRM_IOCTL_ANE_BO_FREE \ DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_BO_FREE, struct drm_ane_bo_free) #define DRM_IOCTL_ANE_SUBMIT \ DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_SUBMIT, struct drm_ane_submit) - #define DRM_IOCTL_ANE_PROG_LOAD \ DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_PROG_LOAD, struct drm_ane_prog_load) #define DRM_IOCTL_ANE_PROC_CREATE \ @@ -127,4 +254,4 @@ struct drm_ane_exec { } #endif -#endif /* __ANE_ACCEL_H__ */ +#endif /* __DRM_UAPI_ANE_ACCEL_H__ */ From 332e297b0e93060421891e8992c1aa4bc8ec260e Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:34:03 -0500 Subject: [PATCH 0352/1579] accel/ane: Parenthesize the register macro arguments checkpatch --strict reports 27 MACRO_ARG_PRECEDENCE checks for the task manager and task queue register macros. Put each macro argument in parentheses. No functional change: all callers pass simple expressions, and the non-debug sections of all seven objects are byte-identical before and after this change (arm64, gcc 12.2.0). Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_tm.c | 44 ++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index e81af0a7b96db..7898c1dd93aaa 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -19,11 +19,11 @@ static const int TQ_PRTY_TABLE[ANE_TQ_COUNT] = { 0x1, 0x2, 0x3, 0x4, #define TM_PUSH 0x8 #define TM_TQ_EN 0xc -#define TM_IRQ_EVTC(line) (0x14 + (line * 0x14)) -#define TM_IRQ_INFO(line) (0x18 + (line * 0x14)) -#define TM_IRQ_UNK1(line) (0x1c + (line * 0x14)) -#define TM_IRQ_TMST(line) (0x20 + (line * 0x14)) -#define TM_IRQ_UNK2(line) (0x24 + (line * 0x14)) +#define TM_IRQ_EVTC(line) (0x14 + ((line) * 0x14)) +#define TM_IRQ_INFO(line) (0x18 + ((line) * 0x14)) +#define TM_IRQ_UNK1(line) (0x1c + ((line) * 0x14)) +#define TM_IRQ_TMST(line) (0x20 + ((line) * 0x14)) +#define TM_IRQ_UNK2(line) (0x24 + ((line) * 0x14)) #define TM_COMMITTED 0x44 #define TM_STATUS 0x54 @@ -34,28 +34,30 @@ static const int TQ_PRTY_TABLE[ANE_TQ_COUNT] = { 0x1, 0x2, 0x3, 0x4, #define TM_IRQ_ACK 0x6c #define TM_IRQ_EN2 0x70 -#define TQ_STATUS(qid) (0x000 + (qid * 0x148)) -#define TQ_PRTY(qid) (0x010 + (qid * 0x148)) -#define TQ_VACANT(qid) (0x014 + (qid * 0x148)) -#define TQ_INFO(qid) (0x01c + (qid * 0x148)) +#define TQ_STATUS(qid) (0x000 + ((qid) * 0x148)) +#define TQ_PRTY(qid) (0x010 + ((qid) * 0x148)) +#define TQ_VACANT(qid) (0x014 + ((qid) * 0x148)) +#define TQ_INFO(qid) (0x01c + ((qid) * 0x148)) -#define TQ_BAR1(qid, bdx) (0x020 + (qid * 0x148) + (bdx * 0x4)) -#define TQ_NID1(qid) (0x0a0 + (qid * 0x148)) -#define TQ_SIZE2(qid) (0x0a4 + (qid * 0x148)) -#define TQ_ADDR2(qid) (0x0a8 + (qid * 0x148)) +#define TQ_BAR1(qid, bdx) (0x020 + ((qid) * 0x148) + ((bdx) * 0x4)) +#define TQ_NID1(qid) (0x0a0 + ((qid) * 0x148)) +#define TQ_SIZE2(qid) (0x0a4 + ((qid) * 0x148)) +#define TQ_ADDR2(qid) (0x0a8 + ((qid) * 0x148)) -#define TQ_BAR2(qid, bdx) (0x0ac + (qid * 0x148) + (bdx * 0x4)) -#define TQ_NID2(qid) (0x12c + (qid * 0x148)) -#define TQ_SIZE1(qid) (0x130 + (qid * 0x148)) -#define TQ_ADDR1(qid) (0x134 + (qid * 0x148)) +#define TQ_BAR2(qid, bdx) (0x0ac + ((qid) * 0x148) + ((bdx) * 0x4)) +#define TQ_NID2(qid) (0x12c + ((qid) * 0x148)) +#define TQ_SIZE1(qid) (0x130 + ((qid) * 0x148)) +#define TQ_ADDR1(qid) (0x134 + ((qid) * 0x148)) #define TM_IS_IDLE 0x1 #define TM_IS_FINE 0x22222222 -#define tm_read32(ane, off) (readl(ane->engine + ANE_TM_BASE + off)) -#define tq_read32(ane, off) (readl(ane->engine + ANE_TQ_BASE + off)) -#define tm_write32(ane, off, val) (writel(val, ane->engine + ANE_TM_BASE + off)) -#define tq_write32(ane, off, val) (writel(val, ane->engine + ANE_TQ_BASE + off)) +#define tm_read32(ane, off) (readl((ane)->engine + ANE_TM_BASE + (off))) +#define tq_read32(ane, off) (readl((ane)->engine + ANE_TQ_BASE + (off))) +#define tm_write32(ane, off, val) \ + (writel(val, (ane)->engine + ANE_TM_BASE + (off))) +#define tq_write32(ane, off, val) \ + (writel(val, (ane)->engine + ANE_TQ_BASE + (off))) /* The ane SET block (m1n1 ANE.ps_map) maps the pmgr power-state words * for this engine (set0, base, set1..4). pmgr reads are always safe and From 52d07a56701734163bbe0221789080dd804a3066 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:37:14 -0500 Subject: [PATCH 0353/1579] accel/ane: Comment the memory barriers and the mutexes checkpatch reports a MEMORY_BARRIER warning for the four barriers without a comment, and an UNCOMMENTED_DEFINITION check for the three mutexes. Say what each barrier orders and what each mutex protects. Comments only. The non-debug sections of all seven objects are byte-identical before and after this change, except the line number in the one __dyndbg entry of ane_t6021_rtclient_main.o (arm64, gcc 12.2.0). Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane.h | 7 ++++++- drivers/accel/ane/ane_t6021_rtclient_main.c | 3 +++ drivers/accel/ane/ane_tm.c | 8 ++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index c644ec45e67d4..db30a3e103fff 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -46,7 +46,12 @@ struct ane_device { int dart_count; int irq; + /* Protects mm and the iommu_map()/iommu_unmap() calls on domain. */ struct mutex iommu_lock; + /* + * Serializes engine access (submit, BO mapping, recovery, removal) + * and protects bo_list, preserved_list and removed. + */ struct mutex engine_lock; struct list_head bo_list; @@ -98,7 +103,7 @@ struct ane_device { * run ~2x slower while schedutil parks the idle clusters low. */ struct ane_boost { - struct mutex lock; + struct mutex lock; /* protects the fields below */ struct delayed_work off; struct freq_qos_request *legs; int nlegs; diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index e5cda3d046ed6..abe477879d27d 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -506,6 +506,9 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, if (pending) { writel(pending, ipi + 0xc000); + /* Complete the IPI acknowledgment before the + * next reads of the command and malloc rings. + */ mb(); } header = READ_ONCE(io[0]); diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 7898c1dd93aaa..9364ae1469b9d 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -267,6 +267,9 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) err = ane_tm_collect_events(ane, NULL, &finished); if (err < 0) goto wedge; + /* Order the CPU writes to the request buffers and the event + * acknowledgments before the queue writes that start the engine. + */ wmb(); ane_tm_push_tq(ane, req); @@ -316,6 +319,10 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) } status = tq_read32(ane, TQ_NID1(req->qid)); tq_write32(ane, TQ_NID1(req->qid), status & ~1U); + /* Kept from the qualified completion sequence (clear the + * NID valid bit, then the queue status). writel() already + * orders the two writes to the engine. + */ rmb(); tq_write32(ane, TQ_STATUS(req->qid), 0x0); ane_dart_release_scratch(ane, scratch, scratch_pages); @@ -333,6 +340,7 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) goto wedge; } tq_write32(ane, TQ_NID1(req->qid), status & ~1U); + /* Same completion sequence as the contained-fault path above. */ rmb(); tq_write32(ane, TQ_STATUS(req->qid), 0x0); ane_dart_unmask(ane); From ee1732f2fcc4850c10a21a9c6b1f75e5d6e550f4 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:38:44 -0500 Subject: [PATCH 0354/1579] accel/ane: Use kzalloc_obj() checkpatch reports seven ALLOC_WITH_SIZEOF warnings and one ALLOC_SIZEOF_STRUCT check. Allocate the single objects with kzalloc_obj(), which uses GFP_KERNEL when no flags are given. No functional change: kzalloc_obj() expands to kzalloc() of the same size and flags, and the non-debug sections of all seven objects are byte-identical before and after this change (arm64, gcc 12.2.0). Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_dart.c | 2 +- drivers/accel/ane/ane_drv.c | 6 +++--- drivers/accel/ane/ane_t6021_rtclient_main.c | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/accel/ane/ane_dart.c b/drivers/accel/ane/ane_dart.c index aea8cd9eff182..f66bfbe15af55 100644 --- a/drivers/accel/ane/ane_dart.c +++ b/drivers/accel/ane/ane_dart.c @@ -206,7 +206,7 @@ int ane_dart_drain_fault(struct ane_device *ane, u64 fault_iova, return -ENOMEM; iova = fault_iova & ~((1ULL << ane->shift) - 1); - node = kzalloc(sizeof(*node), GFP_KERNEL); + node = kzalloc_obj(*node); if (!node) { __free_page(page); return -ENOMEM; diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 8b38416c20b36..054c69c3764d6 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -92,7 +92,7 @@ static int ane_iommu_map_pages(struct ane_device *ane, struct ane_bo *bo) if (bo->mm) return -EBUSY; - bo->mm = kzalloc(sizeof(*bo->mm), GFP_KERNEL); + bo->mm = kzalloc_obj(*bo->mm); if (!bo->mm) return -ENOMEM; @@ -227,7 +227,7 @@ static void ane_preserve_bo(struct ane_device *ane, struct ane_bo *bo) { struct ane_preserved *p; - p = kzalloc(sizeof(*p), GFP_KERNEL); + p = kzalloc_obj(*p); if (!p) { /* Fail closed the old way: the node stays inserted and the * pages leak until reboot - safe, just unreclaimable. @@ -380,7 +380,7 @@ static int ane_bo_init(struct drm_device *drm, void *data, if (args->pad || !args->size) return -EINVAL; - bo = kzalloc(sizeof(struct ane_bo), GFP_KERNEL); + bo = kzalloc_obj(*bo); if (!bo) return -ENOMEM; INIT_LIST_HEAD(&bo->node); diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index abe477879d27d..eb72dbce48814 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -1402,7 +1402,7 @@ static int ane_t6021_bo_init_ioctl(struct drm_device *drm, void *data, return -ENOSPC; } ane = to_ane_t6021_drm(drm)->ane; - bo = kzalloc(sizeof(*bo), GFP_KERNEL); + bo = kzalloc_obj(*bo); if (!bo) { atomic64_sub(PAGE_ALIGN(args->size), &ane_t6021_bo_total_bytes); return -ENOMEM; @@ -1536,7 +1536,7 @@ static int ane_t6021_open(struct drm_device *drm, struct drm_file *file) { struct ane_t6021_fd *fd; - fd = kzalloc(sizeof(*fd), GFP_KERNEL); + fd = kzalloc_obj(*fd); if (!fd) return -ENOMEM; INIT_LIST_HEAD(&fd->bos); @@ -1885,7 +1885,7 @@ static int ane_rtclient_attach_genpd(struct ane_rtclient *ane) if (pd->dev == dev) goto found; - pd = kzalloc(sizeof(*pd), GFP_KERNEL); + pd = kzalloc_obj(*pd); if (!pd) { err = -ENOMEM; goto out; From ef8534228ca6f7fa7300aa7b2803efe575d14220 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:41:16 -0500 Subject: [PATCH 0355/1579] accel/ane: Fix the remaining coding style issues Fix the style issues that checkpatch --strict reports and that need no change to the generated code: - join the split MODULE_PARM_DESC() strings of ane_t6021_fwload.c (SPLIT_STRING), - wrap three lines longer than 100 columns (LONG_LINE, LONG_LINE_STRING), - move two multi-line trailing comments in ane_t6021_boot.h and one after a #define above the code they describe (BLOCK_COMMENT_STYLE), - use ARRAY_SIZE(), min_t() and BIT() (ARRAY_SIZE, MINMAX, BIT_MACRO), - add braces to the unbalanced if/else statements (BRACES), - remove a pair of unnecessary parentheses (UNNECESSARY_PARENTHESES), - do not end a line with an open parenthesis (OPEN_ENDED_LINE), - reword a comment that checkpatch reads as a fall-through annotation (PREFER_FALLTHROUGH). No functional change: the non-debug sections of all seven objects are byte-identical before and after this change (arm64, gcc 12.2.0). Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_drv.c | 20 +++++++----- drivers/accel/ane/ane_t6021.h | 6 ++-- drivers/accel/ane/ane_t6021_boot.c | 4 +-- drivers/accel/ane/ane_t6021_boot.h | 35 +++++++++------------ drivers/accel/ane/ane_t6021_fwload.c | 18 +++++------ drivers/accel/ane/ane_t6021_rtclient_main.c | 13 ++++---- 6 files changed, 45 insertions(+), 51 deletions(-) diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 054c69c3764d6..738fa5d17b1fa 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -513,8 +513,8 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) } gem[bdx] = &bo->base; if (!bo->iova || - ((bdx == CMD_BUF_BDX) && - (args->tsk_size >= (bo->npages << ane->shift)))) { + (bdx == CMD_BUF_BDX && + args->tsk_size >= (bo->npages << ane->shift))) { err = -EINVAL; goto put; } @@ -749,10 +749,10 @@ static int ane_drm_mmap(struct file *file, struct vm_area_struct *vma) */ vm_flags_mod(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP, VM_PFNMAP); - if (map_mode & 2) + if (map_mode & 2) { vma->vm_page_prot = pgprot_decrypted(vm_get_page_prot(vma->vm_flags)); - else { + } else { vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); @@ -859,8 +859,9 @@ static int ane_attach_genpd(struct ane_device *ane) { struct device *dev = ane->dev; - ane->pd_count = of_count_phandle_with_args( - dev->of_node, "power-domains", "#power-domain-cells"); + ane->pd_count = of_count_phandle_with_args(dev->of_node, + "power-domains", + "#power-domain-cells"); if (ane->pd_count < 1) return ane->pd_count < 0 ? ane->pd_count : -EINVAL; if (ane->pd_count == 1) @@ -1119,8 +1120,11 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) * hard reset after the last off-box line names the * killing access exactly. */ - struct resource *eng = platform_get_resource_byname( - to_platform_device(dev), IORESOURCE_MEM, "engine"); + struct platform_device *pdev = to_platform_device(dev); + struct resource *eng; + + eng = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "engine"); dev_info(dev, "ANE-resume: genpd raise complete; SET window probe next\n"); diff --git a/drivers/accel/ane/ane_t6021.h b/drivers/accel/ane/ane_t6021.h index 5e0ecc6d471f7..03d9ee35cdd96 100644 --- a/drivers/accel/ane/ane_t6021.h +++ b/drivers/accel/ane/ane_t6021.h @@ -529,9 +529,9 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, * The generic processor takes commands < 0x1b89 bytes (work-item size * bound @0x4d134-0x4d158) and writes completion state back into the * command block (strb @0x4d324 byte +6, str @0x4d3b0 qword +8) — the - * ring slot doubles as the response area. PING/BUILDINFO/BOOT/ - * REG_FILE_LOAD/IPC_ENDPOINT_SET fall through the processor's id tree - * to the default path (0x4e65c): carried without field parsing, so + * ring slot doubles as the response area. The processor's id tree + * sends PING/BUILDINFO/BOOT/REG_FILE_LOAD/IPC_ENDPOINT_SET on to the + * default path (0x4e65c): carried without field parsing, so * their payloads beyond the header are opaque until the W1 live * exchange pins them. */ diff --git a/drivers/accel/ane/ane_t6021_boot.c b/drivers/accel/ane/ane_t6021_boot.c index 066d2a799851a..de11bb8f095fa 100644 --- a/drivers/accel/ane/ane_t6021_boot.c +++ b/drivers/accel/ane/ane_t6021_boot.c @@ -549,8 +549,8 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, dev_info(ane->dev, "boot: run returned 0 (booted=%u fw_alive=%u) — %s; scratch_result=%016llx (raw device address, semantics UNSOURCED — not a success claim; transport stays fenced until response validation)\n", bo, fa, - bo ? "DONE observed — handshake complete" - : "rc 0 WITHOUT DONE (rtb_mode: HELLO-gated; DONE not part of this mode)", + bo ? "DONE observed — handshake complete" : + "rc 0 WITHOUT DONE (rtb_mode: HELLO-gated; DONE not part of this mode)", sres); return r; } diff --git a/drivers/accel/ane/ane_t6021_boot.h b/drivers/accel/ane/ane_t6021_boot.h index ced0c997439dd..aba267e453f13 100644 --- a/drivers/accel/ane/ane_t6021_boot.h +++ b/drivers/accel/ane/ane_t6021_boot.h @@ -69,14 +69,11 @@ #define ANE_T6021_INIT_TEMPLATE_OFF 0x6c /* 256 B at [0x6C,0x16C) */ #define ANE_T6021_INIT_TEMPLATE_SIZE 0x100 #define ANE_T6021_INIT_TBIT_OFF 0xc0 /* template-relative */ -#define ANE_T6021_INIT_TBIT_VAL 0x4 /* RESOLVED (Main raw - * anchors 0x9612b78/7c/80: - * ldr/orr#4/str on the dev+0x998 - * template); the conditional - * |=0x10 is a separate, still - * unqualified mutation and is - * NOT set - */ +/* RESOLVED (Main raw anchors 0x9612b78/7c/80: ldr/orr#4/str on the + * dev+0x998 template); the conditional |=0x10 is a separate, still + * unqualified mutation and is NOT set. + */ +#define ANE_T6021_INIT_TBIT_VAL 0x4 static inline u64 ane_t6021_rvbar_compose(u64 iova) { @@ -451,18 +448,14 @@ struct ane_t6021_boot_io { u64 (*rd64)(void *ctx, unsigned int off); void (*wr32)(void *ctx, unsigned int off, u32 v); void (*wr64)(void *ctx, unsigned int off, u64 v); - void (*publish_barrier)(void *ctx); /* orders the coherent fill - * before the device publish: - * dma_wmb() = dmb oshst on - * arm64; NOT claimed as dsb st - * (completion) - */ - void (*phase)(void *ctx, const char *what); /* bounded phase log: - * emitted ONCE before each - * write/poll block so a crash - * source survives netconsole - * (never per-poll) - */ + /* Orders the coherent fill before the device publish: dma_wmb() = + * dmb oshst on arm64; NOT claimed as dsb st (completion). + */ + void (*publish_barrier)(void *ctx); + /* Bounded phase log: emitted ONCE before each write/poll block so a + * crash source survives netconsole (never per-poll). + */ + void (*phase)(void *ctx, const char *what); void (*poll_wait)(void *ctx); /* 1 ms poll delay */ /* prepare(): called once, strictly AFTER poll A and BEFORE the * SCRATCH0/1 publish; returns the suballoc DVA halves. Kernel @@ -603,7 +596,7 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, unsigned int ti; io->phase(io->ctx, "P-1 grant-tunables begin"); - for (ti = 0; ti < sizeof(tun) / sizeof(tun[0]); ti++) { + for (ti = 0; ti < ARRAY_SIZE(tun); ti++) { io->phase(io->ctx, tun[ti].off == 0x000 ? "P-1a eng+0x000" : tun[ti].off == 0x038 ? "P-1b eng+0x038" : diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index 1e44f0355b485..d677962a828b7 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -93,14 +93,12 @@ MODULE_DESCRIPTION("T6021 ANE firmware staging + entry alias"); static bool fw_load = true; module_param(fw_load, bool, 0444); MODULE_PARM_DESC(fw_load, - "Validate + DART-map the 13.5 PRELOAD payload (default on: " - "the proven add-path configuration)."); + "Validate + DART-map the 13.5 PRELOAD payload (default on: the proven add-path configuration)."); static unsigned int fw_extra_ram = 0x200000; module_param(fw_extra_ram, uint, 0444); MODULE_PARM_DESC(fw_extra_ram, - "Page-aligned owned RAM after the 5 MiB firmware allocation " - "(default 0x200000, the proven add-path grant; maximum 16 MiB)."); + "Page-aligned owned RAM after the 5 MiB firmware allocation (default 0x200000, the proven add-path grant; maximum 16 MiB)."); /* Preloaded-placement alias, lab only: map the iBoot-reserved SEG0/SEGi * phys at the entry IOVAs (the preload, with iBoot's patches in place). @@ -113,11 +111,7 @@ MODULE_PARM_DESC(fw_extra_ram, static bool fw_alias_reserved; module_param(fw_alias_reserved, bool, 0444); MODULE_PARM_DESC(fw_alias_reserved, - "Lab, T6021: map reserved SEG0 0x10000848000+0xc4000 at entry and " - "SEG1 0x10001400000+0x438000 after it (probe refuses unless no-map " - "/reserved-memory nodes cover both); 0 (default) = run the staged " - "copy with iBoot's patches replayed (own memory, the only mode on " - "T6020/T6022)."); + "Lab, T6021: map reserved SEG0 0x10000848000+0xc4000 at entry and SEG1 0x10001400000+0x438000 after it (probe refuses unless no-map /reserved-memory nodes cover both); 0 (default) = run the staged copy with iBoot's patches replayed (own memory, the only mode on T6020/T6022)."); static bool ane_t6021_fw_alias_is_reserved(struct device *dev) { @@ -314,7 +308,8 @@ static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) * DATA section unmapped past the SEG0 head. */ struct { u64 iova, phys, len; } win[] = { - { 0x10000000000ull, ane_t6021_fw_preload[0].phys, ane_t6021_fw_preload[0].len }, + { 0x10000000000ull, ane_t6021_fw_preload[0].phys, + ane_t6021_fw_preload[0].len }, { 0, ane_t6021_fw_preload[1].phys, ane_t6021_fw_preload[1].len }, { entry + 0x4fc000, 0, fw_extra_ram ? ane->fw_size - 0x4fc000 : 0 }, }; @@ -340,7 +335,8 @@ static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) for (o = 0; o < win[w].len; o += ANE_T6021_FW_ALIAS_PAGE) { phys_addr_t pa = w < 2 ? win[w].phys + o : - iommu_iova_to_phys(dom, ane->fw_iova + win[w].iova + o - entry); + iommu_iova_to_phys(dom, ane->fw_iova + + win[w].iova + o - entry); if (!pa || !IS_ALIGNED(pa, ANE_T6021_FW_ALIAS_PAGE)) { ret = -EFAULT; goto err_unmap_mapped; diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index eb72dbce48814..676029a9e7b27 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -620,7 +620,7 @@ MODULE_PARM_DESC(call_settle_us, * trace state is protected by ane_t6021_fw_lock. */ #define ANE_TRACE_MAGIC 0x31445441 /* "ATD1" */ -#define ANE_TRACE_RECS (1U << 18) +#define ANE_TRACE_RECS BIT(18) #define ANE_PMGR_PS_LAST_OFF 0x30 enum { @@ -1085,7 +1085,8 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, sha256_update(&sha, (const u8 *)hdr, sizeof(hdr)); left = sections[i].size; while (left) { - size_t n = min(left, (size_t)ANE_T6021_BO_HASH_CHUNK); + size_t n = min_t(size_t, left, + ANE_T6021_BO_HASH_CHUNK); memcpy(scratch, (const u8 *)bo->cpu + sections[i].offset + @@ -2238,9 +2239,9 @@ static int ane_rtclient_probe(struct platform_device *pdev) READ_ONCE(((u32 *)command->cpu)[1]), READ_ONCE(((u32 *)command->cpu)[2]), qret); - if (qret) + if (qret) { cfg_err = qret; - else if (!READ_ONCE(((u32 *)command->cpu)[2])) { + } else if (!READ_ONCE(((u32 *)command->cpu)[2])) { dev_err(dev, "LEGACY CONFIG_GET reply word +0x08 zero\n"); cfg_err = -EPROTO; @@ -2337,11 +2338,11 @@ static int ane_rtclient_probe(struct platform_device *pdev) return 0; err_pm_or_hold: - if (ane->held) + if (ane->held) { dev_warn(dev, "probe failed after CPU start (%pe) — binding fenced; power/rings/IRQ HELD until reboot\n", ERR_PTR(ret)); - else { + } else { pm_runtime_put_sync_suspend(dev); pm_runtime_disable(dev); } From 2cdb92de79575062c6e043dc5daf10f403410fe7 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:29:18 -0500 Subject: [PATCH 0356/1579] accel/ane: Declare the M2 ANE firmware files ane_t6021 loads apple/ane/t602x_ane0_fw_selene_rc4x.macho on T602x and apple/ane/h14_ane_fw_bia_j4xx.macho on T8112. Declare both with MODULE_FIRMWARE() and keep each name in one macro. The firmware is a distribution concern: omarchy-ane-firmware-fetch installs these files under /lib/firmware/apple/ane. When request_firmware() fails, name the file and how to install it. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_fw_validate.h | 8 ++++++-- drivers/accel/ane/ane_t6021_fwload.c | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/accel/ane/ane_fw_validate.h b/drivers/accel/ane/ane_fw_validate.h index 00ef2982c67e4..f4f723a3f33df 100644 --- a/drivers/accel/ane/ane_fw_validate.h +++ b/drivers/accel/ane/ane_fw_validate.h @@ -50,6 +50,10 @@ struct ane_fw_seg { char name[17]; /* 16 + NUL */ }; +/* request_firmware() names */ +#define ANE_FW_SELENE_NAME "apple/ane/t602x_ane0_fw_selene_rc4x.macho" +#define ANE_FW_BIA_NAME "apple/ane/h14_ane_fw_bia_j4xx.macho" + /* One pinned image. The patch fields are the payload's LC_SYMTAB * addresses: the __rtk_patch records _rtk_stack_guard, RTK_soc, * RTK_soc_revision, RTK_cpu_physical_address and @@ -66,7 +70,7 @@ struct ane_fw_image { }; static const struct ane_fw_image ane_fw_selene = { - .name = "apple/ane/t602x_ane0_fw_selene_rc4x.macho", + .name = ANE_FW_SELENE_NAME, .sha256 = { 0xa9, 0xc4, 0xb7, 0x71, 0x29, 0x4a, 0x6b, 0x11, 0x56, 0x24, 0xd9, 0x48, 0x0a, 0x62, 0x48, 0xd0, @@ -83,7 +87,7 @@ static const struct ane_fw_image ane_fw_selene = { }; static const struct ane_fw_image ane_fw_bia = { - .name = "apple/ane/h14_ane_fw_bia_j4xx.macho", + .name = ANE_FW_BIA_NAME, .sha256 = { 0xaf, 0x58, 0x7d, 0xfa, 0x96, 0xb1, 0xe0, 0x1d, 0x2b, 0x2e, 0x0f, 0x9f, 0x77, 0x6e, 0x5d, 0xbe, diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index d677962a828b7..c8a7ad8d7bee8 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -89,6 +89,8 @@ */ MODULE_LICENSE("Dual MIT/GPL"); MODULE_DESCRIPTION("T6021 ANE firmware staging + entry alias"); +MODULE_FIRMWARE(ANE_FW_SELENE_NAME); +MODULE_FIRMWARE(ANE_FW_BIA_NAME); static bool fw_load = true; module_param(fw_load, bool, 0444); @@ -565,8 +567,8 @@ int ane_t6021_fwload_probe(struct ane_t6021 *ane) ret = request_firmware(&fw, img->name, ane->dev); if (ret) { dev_err(ane->dev, - "fwload: request_firmware(%s): %d — stage the payload " - "with omarchy-ane-firmware-fetch\n", img->name, ret); + "cannot load firmware %s: %d; install it with omarchy-ane-firmware-fetch (or equivalent for your distribution)\n", + img->name, ret); return ret; } From b5e8553dfb03e56ab7126053843019b2c910542f Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:51:13 -0500 Subject: [PATCH 0357/1579] accel/ane: Add the ane_stats counters and ane_timeline Add producer-side statistics to both ANE drivers. /sys/class/accel/accel/device/ane_stats is a read-only (0444) file with two cumulative counters, one "key value" pair per line: - busy_ns: the nanoseconds the engine was busy since bind. A busy period opens when the first submission arrives on an idle engine and closes when the last one completes. Overlapping submissions share a period, so busy_ns is the union of the submit-to-completion windows, not their sum. ane serializes submissions behind engine_lock, so there the union is the sum. A read adds the time of the open period, so the value also increases while work is in flight. - jobs: the completed engine submissions since bind. ane counts each ANE_SUBMIT. ane_t6021 counts only CSNE_CMD_PROCEDURE_CALL. The control-plane commands on the same path and the boot transport are not engine work and do not count. A submission that fails still completes its begin/complete pair, so the counters stay balanced. Documentation/ABI/testing/sysfs-class-accel-ane describes the file. The ane_timeline debugfs file shows the latest submissions from a ring that the driver allocates at probe: seq, submit_ns, start_ns, end_ns, tasks, rc and tmst. Each slot has a sequence number, so a reader skips a slot that is in flight or that a later submission reuses. ane records the raw TM tick as tmst. ane_t6021 has no host TM path and records 0. Both drivers create the file with drm_debugfs_add_file(). The producer uses atomics only. It does not allocate, lock or format. A transition sentinel in the inflight count keeps the fold of one busy period and the start of the next apart. The next period starts at or after the fold end of the previous one. Both modules have a read-only stats parameter, default 1, that the driver reads at probe. With stats=0, or when the ring allocation fails, the driver creates no stats files and the submission path skips the counters. Testing: in the tree, this code is compile-verified only (arm64 W=1, =m and =y, and checkpatch --strict). A host test of the accounting through a kernel API shim passes. It includes a model check of all orders of three jobs, with a preemption at each atomic operation. The out-of-tree omarchy-ane driver with the same accounting passed the hardware gates on T8103 (M1) and T6001 (M1 Max). That version read max_end for the period start before it took the transition sentinel; this commit reads it after. The T6021 (M2 Max) path is in test. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- .../ABI/testing/sysfs-class-accel-ane | 50 ++ MAINTAINERS | 1 + drivers/accel/ane/ane.h | 12 + drivers/accel/ane/ane_drv.c | 84 +++- drivers/accel/ane/ane_stats.h | 431 ++++++++++++++++++ drivers/accel/ane/ane_t6021_rtclient_main.c | 125 +++++ drivers/accel/ane/ane_tm.c | 9 +- drivers/accel/ane/ane_tm.h | 3 + 8 files changed, 713 insertions(+), 2 deletions(-) create mode 100644 Documentation/ABI/testing/sysfs-class-accel-ane create mode 100644 drivers/accel/ane/ane_stats.h diff --git a/Documentation/ABI/testing/sysfs-class-accel-ane b/Documentation/ABI/testing/sysfs-class-accel-ane new file mode 100644 index 0000000000000..668bed9c7cb38 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-class-accel-ane @@ -0,0 +1,50 @@ +What: /sys/class/accel/accel/device/ane_stats +Date: October 2026 +KernelVersion: 7.1 +Contact: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> +Description: + Cumulative Apple Neural Engine (ANE) producer counters for + this device, one ASCII "key value" pair per line, mode + 0444 (readable without root): + + busy_ns: nanoseconds the engine was busy since the + device was bound, sampled with a consistent + close/open snapshot. A busy period opens + when the first submission lands on an idle + engine and closes when the last one + completes; its whole span folds once at + close, and overlapping submissions share + the period, so a serialized engine + (ane.ko, behind engine_lock) reports + sum(end - start) and a parallel engine + (ane_t6021) reports the union, never the + sum. The show callback adds the live tail + of the still-open period, so the value + advances while engine work is in flight. + + jobs: completed engine submissions since the + device was bound. On ane.ko every ANE_SUBMIT + counts (one per libane ane_exec, N per + ane_exec_loop). On ane_t6021 only + CSNE_CMD_PROCEDURE_CALL counts (one per + engine PROCEDURE_CALL); the control-plane + exchanges that ride the same path + (LOAD_PROGRAM, CREATE_PROCESS, + CH_PROPERTY_WRITE, install-time CONFIG_GET) + and the boot transport are not engine + work and are not counted. A submission + that completes with an error still + completes its begin/complete pair, so the + counter stays balanced. + + Neither counter resets while the device is bound. + + The file exists only when the module parameter + ane.stats / ane_t6021.stats (default 1) is set at + probe. With stats=0 the driver creates no stats files + and the submission hot path skips the counters. + +Users: performance samplers that compute engine utilization + and job rate from the two counters. + + diff --git a/MAINTAINERS b/MAINTAINERS index 9b91fa8f0f987..1846894899917 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7903,6 +7903,7 @@ M: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> L: asahi@lists.linux.dev L: dri-devel@lists.freedesktop.org S: Maintained +F: Documentation/ABI/testing/sysfs-class-accel-ane F: Documentation/devicetree/bindings/npu/apple,ane.yaml F: drivers/accel/ane/ F: include/uapi/drm/ane_accel.h diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index db30a3e103fff..eac81bdab9f74 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -15,6 +15,8 @@ #include +#include "ane_stats.h" + struct ane_device { struct drm_device drm; struct device *dev; @@ -112,6 +114,16 @@ struct ane_device { bool busy; bool on; } boost; + + /* + * Producer-side stats (sysfs ane_stats, debugfs ane_timeline; + * see ane_stats.h). stats=0 or a failed ring allocation leaves + * stats_slots NULL, which is the one hot-path gate: no files, + * no counter updates. + */ + struct ane_stats_counters stats_ctrs; + struct ane_stats_ring stats_ring; + struct ane_stats_ring_entry *stats_slots; }; struct ane_request { diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 738fa5d17b1fa..20e32a76f190a 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -2,7 +2,9 @@ /* Copyright 2022 Eileen Yoon */ #include +#include #include +#include #include #include #include @@ -11,6 +13,7 @@ #include #include +#include #include #include #include @@ -53,6 +56,16 @@ module_param(map_mode, int, 0644); MODULE_PARM_DESC(map_mode, "BO mapping: bit0=IOMMU_CACHE DART descriptors, bit1=cacheable CPU vmas (default 3 = cached; 0 = writecombine + non-cacheable rollback)"); +/* + * Producer-side stats (ane_stats sysfs, ane_timeline debugfs), read + * once at probe. stats=0 keeps the hot path at its single predictable + * branch (the NULL stats_slots check) and no files are created. + */ +static bool stats = true; +module_param(stats, bool, 0444); +MODULE_PARM_DESC(stats, + "Expose ane_stats sysfs and ane_timeline debugfs (default 1; 0 = no files and the hot path skips the counters)"); + #define CMD_BUF_BDX 0 #define KRN_BUF_BDX 1 @@ -480,6 +493,7 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) struct drm_gem_object *btsp = NULL; struct ane_bo *bo; struct ane_request req; + u64 stats_idx = 0; int err; memset(&req, 0, sizeof(req)); @@ -561,10 +575,19 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) if (err < 0) goto unlock; + if (ane->stats_slots) + stats_idx = ane_stats_begin(&ane->stats_ctrs, &ane->stats_ring, + ktime_get_ns(), req.td_count); + ane_boost_begin(ane); err = ane_tm_execute(ane, &req); ane_boost_end(ane); + if (ane->stats_slots) + ane_stats_complete(&ane->stats_ctrs, &ane->stats_ring, + stats_idx, ktime_get_ns(), + err ? (u32)err : 0, ane_last_tmst); + unlock: mutex_unlock(&ane->engine_lock); put: @@ -611,12 +634,47 @@ static ssize_t reset_store(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_WO(reset); +/* + * ane_stats: cumulative busy_ns/jobs for this device (mode 0444, no + * root needed). Formatting lives in ane_stats_emit() (ane_stats.h), + * shared with ane_t6021.ko. + */ +static ssize_t ane_stats_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ane_device *ane = dev_get_drvdata(dev); + + return ane_stats_emit(buf, &ane->stats_ctrs); +} +static DEVICE_ATTR_RO(ane_stats); + static struct attribute *ane_dev_attrs[] = { &dev_attr_wedged.attr, &dev_attr_reset.attr, + &dev_attr_ane_stats.attr, + NULL, +}; + +/* ane_stats appears only when stats=1 gave the device a ring. */ +static umode_t ane_dev_attr_is_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct ane_device *ane = dev_get_drvdata(kobj_to_dev(kobj)); + + if (attr == &dev_attr_ane_stats.attr && !ane->stats_slots) + return 0; + return attr->mode; +} + +static const struct attribute_group ane_dev_group = { + .attrs = ane_dev_attrs, + .is_visible = ane_dev_attr_is_visible, +}; + +static const struct attribute_group *ane_dev_groups[] = { + &ane_dev_group, NULL, }; -ATTRIBUTE_GROUPS(ane_dev); static int ane_drm_open(struct drm_device *drm, struct drm_file *file) { @@ -964,6 +1022,25 @@ static int ane_platform_probe(struct platform_device *pdev) atomic_set(&ane->wedged, 0); ane->tm_retention = soc->tm_retention; + /* Producer-side stats: preallocate the ring at probe; the files + * and the hot-path branch key off stats_slots. + */ + if (stats) { + ane->stats_slots = devm_kcalloc(dev, + 1u << ANE_STATS_RING_ORDER_DEFAULT, + sizeof(*ane->stats_slots), + GFP_KERNEL); + if (ane->stats_slots) { + ane_stats_counters_init(&ane->stats_ctrs, + &ane->stats_ring, + ANE_STATS_RING_ORDER_DEFAULT); + ane->stats_ring.slots = ane->stats_slots; + } else { + dev_warn(dev, + "ane_stats ring allocation failed; stats disabled\n"); + } + } + /* Managed power first: genpd links hold the ANE/DART supplier * topology awake before any register is touched. */ @@ -1040,6 +1117,11 @@ static int ane_platform_probe(struct platform_device *pdev) if (err < 0) goto put_pm; + if (ane->stats_slots) + drm_debugfs_add_file(drm, "ane_timeline", + ane_timeline_show, + &ane->stats_ring); + err = drm_dev_register(drm, 0); if (err < 0) goto put_pm; diff --git a/drivers/accel/ane/ane_stats.h b/drivers/accel/ane/ane_stats.h new file mode 100644 index 0000000000000..768fe97846edc --- /dev/null +++ b/drivers/accel/ane/ane_stats.h @@ -0,0 +1,431 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Producer-side ANE counters and submission ring, shared by ane.ko and + * ane_t6021.ko. + * + * Contract: + * - sysfs under /sys/class/accel/accel/device/ane_stats, + * mode 0444, ASCII "key value" lines, integers only: busy_ns + * (cumulative u64), jobs (cumulative u64). busy_ns counts only + * nanoseconds the engine was busy (union of submit-to-completion + * windows) since the device was bound, and advances while work is + * in flight: the show callback adds the open period's live tail. + * - debugfs ane_timeline: preallocated ring of the last N submissions + * with seq, submit_ns, start_ns, end_ns, tasks, rc (and tmst raw + * when known). Head counter is atomic; per-slot seqlock so the + * reader never blocks the producer and torn reads are detected. + * - `stats` module parameter bool, default 1; stats=0 makes the hot + * path one predictable branch and skips sysfs/debugfs creation. + * + * No allocation, locking, or formatting in the hot path. The sysfs + * show callback formats in process context; the ring writer commits + * the slot with a release store, and the reader does an acquire load + * on begin/end. Counters update with atomics only. busy_ns is the + * union of busy intervals: a busy period opens when the first + * submission lands on an idle engine and closes when the last one + * completes, and its whole span folds once at close; overlapping + * submissions share the period, so a serialized engine (ane.ko, + * behind engine_lock) reports sum(end - start) and a parallel engine + * (ane_t6021) reports the union, never the sum. + */ + +#ifndef __ANE_STATS_H__ +#define __ANE_STATS_H__ + +#include +#include +#include +#include +#include +#include +#include + +/* + * Per-device counters. busy_ns holds the busy time folded from closed + * busy periods: a period opens when the first submission arrives on an + * idle engine and closes when the last one completes, and its whole + * span folds once at close. Overlapping submissions share the period, + * so busy_ns is the union of the submit-to-completion intervals, not + * their sum. The show callback adds the live tail of the still-open + * period, so the reported value advances while engine work is in + * flight. jobs counts completed submissions. last_busy_end is the open + * period's start timestamp; inflight is the number of in-flight + * submissions, with ANE_STATS_INFLIGHT_TRANS as the transient close/ + * open sentinel that keeps a fold and a period start from interleaving. + * + * Module-level `stats` parameter governs whether these counters and + * the ring are created and whether the hot path branches out. + */ +#define ANE_STATS_INFLIGHT_TRANS 0xFFFFFFFFu + +struct ane_stats_counters { + atomic64_t busy_ns; + atomic64_t jobs; + atomic64_t last_busy_end; + atomic64_t max_end; + atomic_t inflight; +}; + +/* + * max_end tracks the latest completion sample fed to complete(); the + * draining completion folds against max(end_ns, max_end) so a stale + * sample on the last completer cannot truncate the period. + */ +static inline void ane_stats_atomic64_max(u64 i, atomic64_t *v) +{ + u64 old = atomic64_read(v); + + while (i > old && !atomic64_try_cmpxchg(v, &old, i)) + ; +} + +/* + * Ring slot: seqlock-style per-slot sequence. Writer bumps seq at + * entry (odd) and again on commit (even); reader loads begin/end + * inside acquire load and the order relative to begin/end is preserved + * by release. begin/end 0 means the slot is empty. submit_ns, end_ns, + * tasks, rc, tmst are the diagnostic fields; tmst is the raw TM tick + * on ane.ko (unknown unit) and 0 on ane_t6021 (unavailable, marked so + * in the file header line). + */ +struct ane_stats_ring_entry { + atomic64_t seq; /* even = committed; odd = writing */ + atomic64_t submit_ns; + atomic64_t start_ns; + atomic64_t end_ns; + atomic_t tasks; + atomic_t rc; + atomic64_t tmst; /* raw TM tick on ane.ko; 0 = unavailable */ +}; + +/* + * Ring container. head is the next slot to write (increments mod N). + * N is a power of two preallocated at probe. slots is the array. + * Reader takes begin/end under acquire and the seqlock protects + * against torn writes. + */ +struct ane_stats_ring { + atomic64_t head; + u32 n; /* power of two */ + u32 mask; /* n - 1 */ + struct ane_stats_ring_entry *slots; +}; + +/* + * Initialize counters and ring from zero. The ring slots array must + * be preallocated (probe-time) and zeroed. Named + * ane_stats_counters_init (not ane_stats_init) so the per-driver + * glue function (ane_stats_init(struct ane_device *) etc.) can name + * its own static initializer without conflicting with the prototype. + */ +static inline void ane_stats_counters_init(struct ane_stats_counters *ctrs, + struct ane_stats_ring *ring, + u32 n_shift) +{ + u32 n_total = (1u << n_shift); + + memset(ctrs, 0, sizeof(*ctrs)); + memset(ring, 0, sizeof(*ring)); + ring->n = n_total; + ring->mask = n_total - 1u; + atomic64_set_release(&ring->head, 0ull); +} + +/* + * Hot-path submission start. Caller holds the device's submission + * serialization if the engine is single-producer (ane.ko); concurrent + * drivers (ane_t6021) pass ring/ctrs and rely on the cmpxchg loops. + * Returns the submission ticket (head + 1, starting at 1) for the + * caller to later call ane_stats_complete() with. The ticket, not a + * slot index, identifies the submission: slots are shared after wrap, + * tickets are not. + * + * Counter side: the first submission onto an idle engine opens a busy + * period by latching last_busy_end = submit_ns under the transition + * sentinel; later overlapping submissions only increment inflight. + * The sentinel window is a handful of instructions, so the bounded + * cmpxchg retries never observe a half-open period. + * + * A period never starts before the previous one folded: a caller + * sample can be stale (taken before the transition), so the latch is + * max(submit_ns, max_end), with max_end read under the transition + * sentinel. + */ +static inline u64 ane_stats_begin(struct ane_stats_counters *ctrs, + struct ane_stats_ring *ring, + u64 submit_ns, u32 tasks) +{ + u64 ticket = atomic64_fetch_add(1ull, &ring->head) + 1ull; + struct ane_stats_ring_entry *e = + &ring->slots[(size_t)(ticket - 1ull) & ring->mask]; + u64 latch; + u32 cur = atomic_read(&ctrs->inflight); + + for (;;) { + if (cur == ANE_STATS_INFLIGHT_TRANS) { + cur = atomic_read(&ctrs->inflight); + } else if (!cur) { + if (atomic_cmpxchg(&ctrs->inflight, 0u, + ANE_STATS_INFLIGHT_TRANS) != 0u) { + cur = atomic_read(&ctrs->inflight); + continue; + } + /* + * Read max_end only now. The drainer of the previous + * period fed its end sample into max_end before its + * release store of inflight = 0, and every other + * member fed before its decrement. The fully ordered + * cmpxchg above read that 0, so this read sees every + * feed and the latch is never below the previous + * fold end: periods cannot overlap. + */ + latch = atomic64_read(&ctrs->max_end); + if (submit_ns > latch) + latch = submit_ns; + atomic64_set_release(&ctrs->last_busy_end, latch); + atomic_set_release(&ctrs->inflight, 1u); + break; + } else if (atomic_try_cmpxchg(&ctrs->inflight, &cur, + cur + 1u)) { + break; + } + } + + /* + * Order the counter stores above (last_busy_end, inflight) before + * the slot stores below. + */ + smp_wmb(); + (void)atomic64_read_acquire(&e->seq); /* pair with reader */ + /* + * In flight: seq stays odd (2*ticket - 1) until complete() + * commits the even final value 2*ticket. The reader only prints + * even seqs it can match, so an unfinished submission never + * prints and a torn write is never visible. + */ + atomic64_set_release(&e->seq, 2ull * ticket - 1ull); + atomic64_set_release(&e->submit_ns, submit_ns); + /* + * start_ns is the busy-period start this submission is counted + * from (the latch), not the caller's sample: a sample taken + * before the transition can predate period entry, and the + * union reference must use consumed values. + */ + atomic64_set_release(&e->start_ns, + atomic64_read(&ctrs->last_busy_end)); + atomic64_set_release(&e->end_ns, submit_ns); + atomic_set(&e->tasks, tasks); + atomic_set(&e->rc, (u32)0xFFFFFFFFu); /* sentinel: not done */ + atomic64_set_release(&e->tmst, 0ull); + return ticket; +} + +/* + * Hot-path submission completion. The last submission out of a busy + * period closes it: under the transition sentinel it folds the whole + * period span [last_busy_end, end_ns] into busy_ns before reopening + * the counter, so no reader can see a period both live and folded. + * Overlapping submissions share the period, so busy_ns is the union + * of the submit-to-completion intervals, not their sum. Updates the + * slot's end_ns/rc/tmst and increments jobs. + */ +static inline void ane_stats_complete(struct ane_stats_counters *ctrs, + struct ane_stats_ring *ring, + u64 ticket, u64 end_ns, + u32 rc, u64 tmst) +{ + struct ane_stats_ring_entry *e = + &ring->slots[(size_t)(ticket - 1ull) & ring->mask]; + u32 cur = atomic_read(&ctrs->inflight); + u64 fold_end = 0ull; + + /* + * Feed this completion's end sample before the transition: a + * drainer can only observe inflight == 1 after every other + * completer has fed, so the fold below sees the whole period. + */ + ane_stats_atomic64_max(end_ns, &ctrs->max_end); + + for (;;) { + if (cur == ANE_STATS_INFLIGHT_TRANS) { + cur = atomic_read(&ctrs->inflight); + } else if (cur > 1u) { + if (atomic_try_cmpxchg(&ctrs->inflight, &cur, + cur - 1u)) + break; + } else if (!cur) { + /* + * Unbalanced complete (cannot happen with the + * documented one-begin-per-complete pairing): + * count the job, fold nothing, never hang. + */ + break; + } else if (atomic_cmpxchg(&ctrs->inflight, 1u, + ANE_STATS_INFLIGHT_TRANS) != 1u) { + cur = atomic_read(&ctrs->inflight); + } else { + u64 s = atomic64_read(&ctrs->last_busy_end); + + /* + * The period ends at the latest completion sample + * in it, not at this caller's (possibly stale) + * sample: every member fed max_end before the + * drain could observe inflight == 1. + */ + fold_end = atomic64_read(&ctrs->max_end); + if (fold_end < end_ns) + fold_end = end_ns; + if (fold_end > s) + atomic64_add(fold_end - s, &ctrs->busy_ns); + atomic_set_release(&ctrs->inflight, 0u); + break; + } + } + atomic64_add(1ull, &ctrs->jobs); + + /* Order the counter stores above before the slot stores below. */ + smp_wmb(); + /* + * The drainer records the consumed fold end, not its own sample, + * so the slot union equals busy_ns exactly. + */ + if (fold_end) + atomic64_set_release(&e->end_ns, fold_end); + else + atomic64_set_release(&e->end_ns, end_ns); + atomic_set(&e->rc, rc); + atomic64_set_release(&e->tmst, tmst); + /* + * Order the slot field stores before the even seq store that + * publishes them; pairs with the acquire load of e->seq in + * ane_timeline_show(). + */ + smp_wmb(); + /* + * Commit the slot with the even final seq 2*ticket. The value + * must not depend on the current ring->head: concurrent + * submissions (ane_t6021) advance it, and a head-derived seq + * would mislabel the slot. + */ + atomic64_set_release(&e->seq, 2ull * ticket); +} + +/* + * The timeline formatter is header-only so each module gets its own + * copy: kbuild rejects one object linked into two modules, and with + * both drivers built-in a shared object would duplicate symbols at + * vmlinux link. Both drivers register it with drm_debugfs_add_file(), + * so m->private is the struct drm_debugfs_entry and the ring is the + * data pointer given at registration. + */ +static inline int ane_timeline_show(struct seq_file *m, void *v) +{ + struct drm_debugfs_entry *entry = m->private; + struct ane_stats_ring *ring = entry->file.data; + struct ane_stats_ring_entry *ring_slots = ring->slots; + u32 mask = ring->mask; + u64 head = atomic64_read(&ring->head); + u64 live = head < (u64)mask + 1ull ? head : (u64)mask + 1ull; + u64 i; + + seq_puts(m, + "# ane_timeline: seq submit_ns start_ns end_ns tasks rc tmst (tmst raw tick on ane.ko, 0 = unavailable on ane_t6021)\n"); + /* + * Newest first. Submission ticket t lives in slot (t - 1) & mask + * and prints once complete() commits seq = 2*t; anything else + * (odd in-flight seq, stale slot) is skipped. + */ + for (i = 0; i < live; i++) { + u64 ticket = head - i; + struct ane_stats_ring_entry *e = + &ring_slots[(size_t)(ticket - 1ull) & mask]; + u64 seq = atomic64_read_acquire(&e->seq); + u64 submit = atomic64_read(&e->submit_ns); + u64 st = atomic64_read(&e->start_ns); + u64 en = atomic64_read(&e->end_ns); + u32 tasks = atomic_read(&e->tasks); + u32 rc = atomic_read(&e->rc); + u64 tmst = atomic64_read(&e->tmst); + + if (seq != 2ull * ticket) + continue; /* in flight or stale; never printed */ + seq_printf(m, "%llu %llu %llu %llu %u %u %llu\n", + 2ull * ticket, submit, st, en, tasks, rc, tmst); + } + return 0; +} + +#define ANE_STATS_RING_ORDER_DEFAULT 8 /* 256 slots */ + +/* + * Accounting rule (coreglass producer contract): jobs counts completed + * submissions, one begin/complete pair per engine submission — one per + * ANE_SUBMIT on ane.ko (so libane ane_exec is 1 job and ane_exec_loop + * with N iterations is N jobs) and one per firmware PROCEDURE_CALL + * (CSNE_CMD_PROCEDURE_CALL) on ane_t6021.ko: that opcode is the only + * engine work on the shared ane_rtclient_command path. The control- + * plane exchanges that ride the same function — LOAD_PROGRAM, + * CREATE_PROCESS, CH_PROPERTY_WRITE, install-time CONFIG_GET — and the + * boot transport are not engine submissions and are not counted; jobs + * must match the number of engine calls the workload made (+-1 at the + * sampler boundary). A submission that completes with an error still + * completes its begin/complete pair, so the counter stays balanced. + * + * Typed sysfs formatter for the ane_stats attribute. The per-driver + * show callbacks fetch the counters from their real drvdata type + * (struct ane_device * on ane.ko, struct ane_rtclient * on + * ane_t6021.ko) and pass &...->stats_ctrs here. The formatter never + * sees the device pointer, so the drvdata type confusion that shipped + * in the first round (reading the head of ane_device as counters) + * cannot compile again: there is no cast to remove. + */ +/* + * Consistent counter snapshot across the tiny close/open transition: + * the transition sentinel is never accepted, and (inflight, busy_ns) + * must read back unchanged, so a value is only reported between + * transitions. The value is busy_ns plus the live tail of the + * still-open period, so it advances while engine work is in flight. + * Over the timeline the reported value is non-decreasing: folds only + * add, and at a close the live tail equals the fold. + */ +static inline u64 ane_stats_snapshot(const struct ane_stats_counters *ctrs) +{ + u64 raw, busy = 0, now, s = 0; + u32 inflight; + + for (;;) { + inflight = atomic_read_acquire(&ctrs->inflight); + if (inflight == ANE_STATS_INFLIGHT_TRANS) + continue; /* close/open in progress: spin it out */ + raw = atomic64_read(&ctrs->busy_ns); + if (inflight) + s = atomic64_read(&ctrs->last_busy_end); + now = ktime_get_ns(); + /* + * Recheck after the timestamp: a fold that landed before + * `now` changes busy_ns and retries; one that lands + * after is genuinely later than `now`, so the value is + * exact, never an overshoot. + */ + if (atomic64_read(&ctrs->busy_ns) != raw || + atomic_read(&ctrs->inflight) != inflight || + (inflight && + atomic64_read(&ctrs->last_busy_end) != s)) + continue; + busy = raw; + if (inflight && now > s) + busy += now - s; + break; + } + return busy; +} + +static inline ssize_t ane_stats_emit(char *buf, + const struct ane_stats_counters *ctrs) +{ + return sysfs_emit(buf, "busy_ns %llu\njobs %llu\n", + (unsigned long long)ane_stats_snapshot(ctrs), + (unsigned long long)atomic64_read(&ctrs->jobs)); +} + +#endif /* __ANE_STATS_H__ */ diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 676029a9e7b27..18d0ecf6b2f6e 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -87,6 +87,7 @@ #include #include +#include #include #include #include @@ -95,6 +96,7 @@ #include #include +#include "ane_stats.h" #include "ane_t6021.h" #include "ane_t6021_boot.h" @@ -177,6 +179,17 @@ module_param(hello_wait_ms, uint, 0444); MODULE_PARM_DESC(hello_wait_ms, "RTKit HELLO wait in legacy mode; 0 (default) skips RTKit and leaves the mailbox stopped. A firmware that speaks RTKit needs 1000."); +/* + * Producer-side stats (ane_stats sysfs, ane_timeline debugfs), read + * once at probe. stats=0 keeps the hot path at its single predictable + * branch (the NULL stats_slots check) and no files are created. Same + * name and meaning as ane.ko's parameter. + */ +static bool stats = true; +module_param(stats, bool, 0444); +MODULE_PARM_DESC(stats, + "Expose ane_stats sysfs and ane_timeline debugfs (default 1; 0 = no files and the hot path skips the counters)"); + #define ANE_LEGACY_ALLOCS 8192 #define ANE_LEGACY_BYTES SZ_512M @@ -222,6 +235,15 @@ struct ane_rtclient { * which the firmware may reference forever (held until reboot). */ struct ane_legacy_buffer *cmd_buf; + + /* + * Producer-side stats (see ane_stats.h). stats=0 or a failed + * ring allocation leaves stats_slots NULL, which is the one + * hot-path gate: no files, no counter updates. + */ + struct ane_stats_counters stats_ctrs; + struct ane_stats_ring stats_ring; + struct ane_stats_ring_entry *stats_slots; }; /* Per-open BO ownership (drm_file->driver_priv). Handles live in the @@ -889,6 +911,8 @@ static int ane_rtclient_command(struct ane_rtclient *ane, unsigned int timeout_ms) { int ret; + u64 stats_ticket = 0; + u64 stats_submit_ns = 0; ane_t6021_tracing = opcode == CSNE_CMD_PROCEDURE_CALL && trace_td && ane->soc->trace_td_off; @@ -896,10 +920,38 @@ static int ane_rtclient_command(struct ane_rtclient *ane, ane_t6021_trace->calls++; ane_t6021_trace_add(ANE_TR_CALL, 0); } + /* Producer contract hot path (ane_stats.h): record submit at + * command enqueue, completion after the call returns, one + * begin/complete pair per engine submission. Only + * CSNE_CMD_PROCEDURE_CALL is engine work; the control-plane + * exchanges that ride this function (LOAD_PROGRAM, + * CREATE_PROCESS, CH_PROPERTY_WRITE, CONFIG_GET) are not + * counted, so jobs matches the engine calls the workload made. + * The T6021 path can run concurrently (MBI per-channel rings), + * so overlapping calls share a busy period and busy_ns is the + * union of the submit-to-completion windows. tmst is 0 (no + * host TM on T6021; documented in the file header line). + * tasks = 1 (one call per submission); rc = ret. The gate is the + * ring, as on ane.ko: stats=0 or a failed probe allocation leaves + * stats_slots NULL, and the hooks must not run on a NULL ring. + */ + bool stats_call = ane->stats_slots && + opcode == CSNE_CMD_PROCEDURE_CALL; + + if (stats_call) { + stats_submit_ns = ktime_get_ns(); + stats_ticket = ane_stats_begin(&ane->stats_ctrs, + &ane->stats_ring, + stats_submit_ns, 1); + } ret = ane_rtclient_legacy_exchange(ane, command, length, opcode, channel, timeout_ms); if (ret) { ane_t6021_tracing = false; + if (stats_call) + ane_stats_complete(&ane->stats_ctrs, + &ane->stats_ring, stats_ticket, + ktime_get_ns(), (u32)ret, 0); dev_info(ane->dev, "EXCH op=%#x failed %d (fw allocs %u, %zu bytes)\n", opcode, ret, ane->legacy_allocated, ane->legacy_bytes); atomic_set(&ane_t6021_quarantined, 1); @@ -913,12 +965,21 @@ static int ane_rtclient_command(struct ane_rtclient *ane, if (!ret && call_settle_us) usleep_range(call_settle_us, call_settle_us + 100); if (ret) { + if (stats_call) + ane_stats_complete(&ane->stats_ctrs, + &ane->stats_ring, + stats_ticket, + ktime_get_ns(), + (u32)ret, 0); dev_info(ane->dev, "call completion wait failed %d\n", ret); atomic_set(&ane_t6021_quarantined, 1); return ret; } } + if (stats_call) + ane_stats_complete(&ane->stats_ctrs, &ane->stats_ring, + stats_ticket, ktime_get_ns(), 0, 0); /* The fw talks back on the target-to-host rings (fwlog, perf); * hand those slots back so the rings never fill (the sequencer * did this per step; same ack, channels 4 and 6). @@ -2304,6 +2365,25 @@ static int ane_rtclient_probe(struct platform_device *pdev) return -EPROTO; } + /* Producer-side stats: preallocate the ring at probe; the files + * and the hot-path branch key off stats_slots. + */ + if (stats) { + ane->stats_slots = devm_kcalloc(dev, + 1u << ANE_STATS_RING_ORDER_DEFAULT, + sizeof(*ane->stats_slots), + GFP_KERNEL); + if (ane->stats_slots) { + ane_stats_counters_init(&ane->stats_ctrs, + &ane->stats_ring, + ANE_STATS_RING_ORDER_DEFAULT); + ane->stats_ring.slots = ane->stats_slots; + } else { + dev_warn(dev, + "ane_stats ring allocation failed; stats disabled\n"); + } + } + { struct ane_t6021_drm *adrm; int drmret; @@ -2320,6 +2400,10 @@ static int ane_rtclient_probe(struct platform_device *pdev) } adrm->dev = dev; adrm->ane = ane; + if (ane->stats_slots) + drm_debugfs_add_file(&adrm->drm, "ane_timeline", + ane_timeline_show, + &ane->stats_ring); WRITE_ONCE(ane_t6021_perf_ane, ane); drmret = drm_dev_register(&adrm->drm, 0); if (drmret) { @@ -2372,10 +2456,51 @@ static const struct of_device_id ane_rtclient_of_match[] = { }; MODULE_DEVICE_TABLE(of, ane_rtclient_of_match); +/* + * ane_stats: cumulative busy_ns/jobs for this device (mode 0444, no + * root needed). Formatting lives in ane_stats_emit() (ane_stats.h), + * shared with ane.ko. + */ +static ssize_t ane_stats_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ane_rtclient *ane = dev_get_drvdata(dev); + + return ane_stats_emit(buf, &ane->stats_ctrs); +} +static DEVICE_ATTR_RO(ane_stats); + +static struct attribute *ane_t6021_stats_attrs[] = { + &dev_attr_ane_stats.attr, + NULL, +}; + +/* ane_stats appears only when stats=1 gave the device a ring. */ +static umode_t ane_t6021_stats_is_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct ane_rtclient *ane = dev_get_drvdata(kobj_to_dev(kobj)); + + if (attr == &dev_attr_ane_stats.attr && !ane->stats_slots) + return 0; + return attr->mode; +} + +static const struct attribute_group ane_t6021_stats_group = { + .attrs = ane_t6021_stats_attrs, + .is_visible = ane_t6021_stats_is_visible, +}; + +static const struct attribute_group *ane_t6021_stats_groups[] = { + &ane_t6021_stats_group, + NULL, +}; + static struct platform_driver ane_rtclient_driver = { .driver = { .name = "ane_t6021", .of_match_table = ane_rtclient_of_match, + .dev_groups = ane_t6021_stats_groups, .suppress_bind_attrs = true, }, .probe = ane_rtclient_probe, diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 9364ae1469b9d..7a0ade69c395b 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -216,6 +216,13 @@ static void ane_tm_push_tq(struct ane_device *ane, struct ane_request *req) tm_write32(ane, TM_PUSH, TQ_PRTY_TABLE[qid] | (qid & 7) << 8); // magic } +/* + * Last raw TM tick read from TM_IRQ_TMST in the latest event drain. + * Diagnostic only (unit unknown; the timeline file labels it as a + * raw tick); unsynchronized by design. + */ +u32 ane_last_tmst; + static int ane_tm_collect_events(struct ane_device *ane, struct ane_request *req, u32 *finished) { @@ -228,7 +235,7 @@ static int ane_tm_collect_events(struct ane_device *ane, u32 info = tm_read32(ane, TM_IRQ_INFO(line)); tm_read32(ane, TM_IRQ_UNK1(line)); - tm_read32(ane, TM_IRQ_TMST(line)); + ane_last_tmst = tm_read32(ane, TM_IRQ_TMST(line)); tm_read32(ane, TM_IRQ_UNK2(line)); if (req && info == (0x05000000 | (req->nid << 16) | (req->td_count - 1))) diff --git a/drivers/accel/ane/ane_tm.h b/drivers/accel/ane/ane_tm.h index 94d425ab8d8b7..84520c149bd52 100644 --- a/drivers/accel/ane/ane_tm.h +++ b/drivers/accel/ane/ane_tm.h @@ -17,4 +17,7 @@ int ane_tm_recover(struct ane_device *ane); #define ANE_PS_ALL_ON ((1U << (4 * 6)) - 1) u32 ane_tm_ps_act(struct ane_device *ane); +/* Last raw TM tick captured by ane_tm_collect_events (diagnostic). */ +extern u32 ane_last_tmst; + #endif /* __ANE_TM_H__ */ From 59823d3a237ff755b5afe300308f99062865b46b Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 14:15:46 -0700 Subject: [PATCH 0358/1579] arm64: dts: apple: t600x: drop the ANE set5 power state The die 0 ANE power states that came with the ANE support in Joshua Warren's aurora-silicon/linux#155 include ps_ane_set5 at 0xc030, but the comment above them puts the end of the ANE's pmgr range at 0xc02c. The ane node does not use it either: it lists set1 to set4, and the driver takes its power domains from that list. The node was still live on every M1 Pro, Max and Ultra. The pmgr driver probes each enabled power-controller: it reads the register, sets the auto-PM enable bit if the state is active, and genpd turns unused domains off late in boot. Linux should not do any of that to a register outside the block it describes. Drop the node, and say in the comment why there is no set5. Found in a review of #155. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 7280b1168af21b04adde3a38ae8fa07a1ab54f84) --- arch/arm64/boot/dts/apple/t600x-pmgr.dtsi | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi index fed2d79a8ba8a..a1c8748ee7cf9 100644 --- a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi @@ -678,8 +678,8 @@ #if DIE_NO == 0 /* - * Only the die 0 ANE is described. The ane node leaves set5 out: - * it lies past the end of the ADT ANE pmgr range (0xc02c). + * Only the die 0 ANE is described. Its pmgr range ends at 0xc02c, + * with set4, so no set5 is described at 0xc030. */ DIE_NODE(ps_ane_set0): power-controller@c000 { compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; @@ -734,15 +734,6 @@ label = DIE_LABEL(ane_set4); power-domains = <&DIE_NODE(ps_ane_base)>; }; - - DIE_NODE(ps_ane_set5): power-controller@c030 { - compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; - reg = <0xc030 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = DIE_LABEL(ane_set5); - power-domains = <&DIE_NODE(ps_ane_base)>; - }; #endif }; From cf7a8977cf82e314ee4e1dba9dae9b7c79ba78c7 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 14:24:57 -0700 Subject: [PATCH 0359/1579] arm64: dts: apple: enable the ANE only on the M1 Max and M2 Max The ANE support from Joshua Warren's aurora-silicon/linux#155 enables the ANE on every chip that describes it except the M2 (T8112): the M1 (T8103) directly, and every M1 Pro/Max/Ultra (T6000/T6001/T6002) and M2 Pro/Max/Ultra (T6020/T6021/T6022) through the shared die 0 files. The driver has run on the M1 Max (T6001) and the M2 Max (T6021) only, and these device tree nodes have not booted on any machine yet. On an untested chip that means the ANE's DARTs, mailbox and power states probe and touch the hardware on every boot, and the driver binds, which on the M2 family starts the ANE firmware by default. Make the ANE node, its DARTs and mailbox, and the power states added for it default to disabled where they are described, and enable them per SoC: - t6001.dtsi enables them on the M1 Max: MacBook Pro j314c and j316c, and Mac Studio j375c. - t6021.dtsi enables them on the M2 Max: MacBook Pro j414c and j416c, and Mac Studio j475c. - t6000.dtsi and t6020.dtsi include those files, so they define APPLE_ANE_UNTESTED first and the M1 Pro and M2 Pro stay off. - t6002 and t6022 (Ultra) build die 0 from the shared files and never see an enable, so they stay off. - t8103.dtsi gets the same enable block behind a one-line APPLE_ANE_UNTESTED define, so enabling the M1 ANE after a supervised boot is a one-line deletion. Power states that predate #155 (ane_sys and ane_sys_cpu on t600x and t8103, and the ANE states in t602x-pmgr.dtsi) keep their status. Where the ANE stays off, each device tree is the pre-#155 one plus disabled nodes, apart from the inert ane-alias-iova reserved-memory entry on t602x, which has no reg or size and is only read through the ANE's memory-region. On the M1 Max and M2 Max the device tree is unchanged, except that the t600x ANE power states now carry an explicit status = "okay". Found in a review of #155. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit ecf86ec16ece1313dd16e135779bf7d93469f16f) --- arch/arm64/boot/dts/apple/t6000.dtsi | 3 ++ arch/arm64/boot/dts/apple/t6001.dtsi | 47 +++++++++++++++++++ arch/arm64/boot/dts/apple/t600x-die0.dtsi | 8 ++-- arch/arm64/boot/dts/apple/t600x-pmgr.dtsi | 10 +++- arch/arm64/boot/dts/apple/t6020.dtsi | 3 ++ arch/arm64/boot/dts/apple/t6021.dtsi | 27 +++++++++++ arch/arm64/boot/dts/apple/t602x-die0.dtsi | 10 ++-- arch/arm64/boot/dts/apple/t8103-pmgr.dtsi | 6 +++ arch/arm64/boot/dts/apple/t8103.dtsi | 57 +++++++++++++++++++++-- 9 files changed, 157 insertions(+), 14 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index db7ea6eac5a6f..22cb23931e0b3 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -11,6 +11,9 @@ #define GPU_REPEAT(x) +/* The ANE has only run on the M1 Max so far: leave it off on the M1 Pro */ +#define APPLE_ANE_UNTESTED + #include "t6001.dtsi" / { diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index dc2d9f14fb060..f044ff6980c1b 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -88,3 +88,50 @@ &gpu { compatible = "apple,agx-t6001", "apple,agx-g13c", "apple,agx-g13s"; }; + +/* + * The ANE has run on the M1 Max, so enable it here with its DARTs and the + * power states only it uses. t6000.dtsi defines APPLE_ANE_UNTESTED before + * including this file: the ANE has not run on the M1 Pro yet. + */ +#ifndef APPLE_ANE_UNTESTED +&ane { + status = "okay"; +}; + +&ane_dart0 { + status = "okay"; +}; + +&ane_dart1 { + status = "okay"; +}; + +&ane_dart2 { + status = "okay"; +}; + +&ps_ane_base { + status = "okay"; +}; + +&ps_ane_set0 { + status = "okay"; +}; + +&ps_ane_set1 { + status = "okay"; +}; + +&ps_ane_set2 { + status = "okay"; +}; + +&ps_ane_set3 { + status = "okay"; +}; + +&ps_ane_set4 { + status = "okay"; +}; +#endif diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 5a4641fa26c59..67bd1530ffe94 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -14,7 +14,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - status = "okay"; + status = "disabled"; }; ane_dart1: iommu@285810000 { @@ -24,7 +24,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys_cpu>; - status = "okay"; + status = "disabled"; }; ane_dart2: iommu@285820000 { @@ -34,7 +34,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys_cpu>; - status = "okay"; + status = "disabled"; }; ane: ane@285c04000 { @@ -47,7 +47,7 @@ iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; power-domains = <&ps_ane_sys_cpu>, <&ps_ane_set1>, <&ps_ane_set2>, <&ps_ane_set3>, <&ps_ane_set4>; - status = "okay"; + status = "disabled"; }; nco: clock-controller@28e03c000 { diff --git a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi index a1c8748ee7cf9..77469a62242a6 100644 --- a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi @@ -679,7 +679,9 @@ #if DIE_NO == 0 /* * Only the die 0 ANE is described. Its pmgr range ends at 0xc02c, - * with set4, so no set5 is described at 0xc030. + * with set4, so no set5 is described at 0xc030. These states serve + * only the ANE and are disabled with it; the SoC file that enables + * the ANE enables them too. */ DIE_NODE(ps_ane_set0): power-controller@c000 { compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; @@ -688,6 +690,7 @@ #reset-cells = <0>; label = DIE_LABEL(ane_set0); power-domains = <&DIE_NODE(ps_ane_sys_cpu)>; + status = "disabled"; }; DIE_NODE(ps_ane_base): power-controller@c008 { @@ -697,6 +700,7 @@ #reset-cells = <0>; label = DIE_LABEL(ane_base); power-domains = <&DIE_NODE(ps_ane_set0)>; + status = "disabled"; }; DIE_NODE(ps_ane_set1): power-controller@c010 { @@ -706,6 +710,7 @@ #reset-cells = <0>; label = DIE_LABEL(ane_set1); power-domains = <&DIE_NODE(ps_ane_base)>; + status = "disabled"; }; DIE_NODE(ps_ane_set2): power-controller@c018 { @@ -715,6 +720,7 @@ #reset-cells = <0>; label = DIE_LABEL(ane_set2); power-domains = <&DIE_NODE(ps_ane_base)>; + status = "disabled"; }; DIE_NODE(ps_ane_set3): power-controller@c020 { @@ -724,6 +730,7 @@ #reset-cells = <0>; label = DIE_LABEL(ane_set3); power-domains = <&DIE_NODE(ps_ane_base)>; + status = "disabled"; }; DIE_NODE(ps_ane_set4): power-controller@c028 { @@ -733,6 +740,7 @@ #reset-cells = <0>; label = DIE_LABEL(ane_set4); power-domains = <&DIE_NODE(ps_ane_base)>; + status = "disabled"; }; #endif }; diff --git a/arch/arm64/boot/dts/apple/t6020.dtsi b/arch/arm64/boot/dts/apple/t6020.dtsi index e9af24b25ab68..0fa3630e2849d 100644 --- a/arch/arm64/boot/dts/apple/t6020.dtsi +++ b/arch/arm64/boot/dts/apple/t6020.dtsi @@ -11,6 +11,9 @@ #define GPU_REPEAT(x) +/* The ANE has only run on the M2 Max so far: leave it off on the M2 Pro */ +#define APPLE_ANE_UNTESTED + #include "t6021.dtsi" / { diff --git a/arch/arm64/boot/dts/apple/t6021.dtsi b/arch/arm64/boot/dts/apple/t6021.dtsi index 889728010c84d..11521dc6091a6 100644 --- a/arch/arm64/boot/dts/apple/t6021.dtsi +++ b/arch/arm64/boot/dts/apple/t6021.dtsi @@ -85,3 +85,30 @@ apple,ppm-filter-time-constant-ms = <34>; apple,ppm-ki = <18.0>; }; + +/* + * The ANE has run on the M2 Max, so enable it here with its mailbox and + * DARTs. t6020.dtsi defines APPLE_ANE_UNTESTED before including this + * file: the ANE has not run on the M2 Pro yet. + */ +#ifndef APPLE_ANE_UNTESTED +&ane { + status = "okay"; +}; + +&ane_dart0 { + status = "okay"; +}; + +&ane_dart1 { + status = "okay"; +}; + +&ane_dart2 { + status = "okay"; +}; + +&ane_mbox { + status = "okay"; +}; +#endif diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi index a465b6f44fb2f..91a1e3c6958f5 100644 --- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi @@ -13,7 +13,7 @@ interrupts = ; interrupt-names = "recv-not-empty"; #mbox-cells = <0>; - status = "okay"; + status = "disabled"; }; ane_dart0: iommu@285800000 { @@ -23,7 +23,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_pmp>; - status = "okay"; + status = "disabled"; }; ane_dart1: iommu@285810000 { @@ -33,7 +33,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_cpu>; - status = "okay"; + status = "disabled"; }; ane_dart2: iommu@285820000 { @@ -43,7 +43,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_cpu>; - status = "okay"; + status = "disabled"; }; ane: ane@284000000 { @@ -62,7 +62,7 @@ resets = <&ps_ane_cpu>; mboxes = <&ane_mbox>; memory-region = <&ane_alias_iova>; - status = "okay"; + status = "disabled"; }; nco: clock-controller@28e03c000 { diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index 252c6914d690d..c3230cd4bb81a 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -1010,6 +1010,7 @@ #reset-cells = <0>; label = "ane_base"; power-domains = <&ps_ane_sys_cpu>; + status = "disabled"; }; ps_ane_set1: power-controller@c010 { @@ -1019,6 +1020,7 @@ #reset-cells = <0>; label = "ane_set1"; power-domains = <&ps_ane_base>; + status = "disabled"; }; ps_ane_set2: power-controller@c018 { @@ -1028,6 +1030,7 @@ #reset-cells = <0>; label = "ane_set2"; power-domains = <&ps_ane_base>; + status = "disabled"; }; ps_ane_set3: power-controller@c020 { @@ -1037,6 +1040,7 @@ #reset-cells = <0>; label = "ane_set3"; power-domains = <&ps_ane_base>; + status = "disabled"; }; ps_ane_set4: power-controller@c028 { @@ -1046,6 +1050,7 @@ #reset-cells = <0>; label = "ane_set4"; power-domains = <&ps_ane_base>; + status = "disabled"; }; ps_ane_set5: power-controller@c030 { @@ -1055,6 +1060,7 @@ #reset-cells = <0>; label = "ane_set5"; power-domains = <&ps_ane_base>; + status = "disabled"; }; ps_disp0_cpu0: power-controller@10018 { diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 16ac8bb5e206d..3ad699a0321c6 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -1453,7 +1453,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - status = "okay"; + status = "disabled"; }; ane_dart1: iommu@26b810000 { @@ -1464,7 +1464,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - status = "okay"; + status = "disabled"; }; ane_dart2: iommu@26b820000 { @@ -1475,7 +1475,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - status = "okay"; + status = "disabled"; }; ane: ane@26bc04000 { @@ -1488,7 +1488,7 @@ iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; power-domains = <&ps_ane_set1>, <&ps_ane_set2>, <&ps_ane_set3>, <&ps_ane_set4>, <&ps_ane_set5>; - status = "okay"; + status = "disabled"; }; dispext0_dart: iommu@271304000 { @@ -2173,3 +2173,52 @@ }; #include "t8103-pmgr.dtsi" + +/* + * The ANE has not run on the M1 yet, so it stays off together with its + * DARTs and the power states only it uses. Drop this define to enable + * them once it has. + */ +#define APPLE_ANE_UNTESTED + +#ifndef APPLE_ANE_UNTESTED +&ane { + status = "okay"; +}; + +&ane_dart0 { + status = "okay"; +}; + +&ane_dart1 { + status = "okay"; +}; + +&ane_dart2 { + status = "okay"; +}; + +&ps_ane_base { + status = "okay"; +}; + +&ps_ane_set1 { + status = "okay"; +}; + +&ps_ane_set2 { + status = "okay"; +}; + +&ps_ane_set3 { + status = "okay"; +}; + +&ps_ane_set4 { + status = "okay"; +}; + +&ps_ane_set5 { + status = "okay"; +}; +#endif From 679d8202f46c6dec740cb0a6532a2701d4947df6 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 14:24:57 -0700 Subject: [PATCH 0360/1579] arm64: dts: apple: t8112: disable the ANE power states with the ANE Joshua Warren's aurora-silicon/linux#155 adds the M2 (T8112) ANE with its node, mailbox and DARTs disabled, since it has not run there. The seven power states added for it (ane_sys_cpu, ane_td, ane_base and ane_set1 to ane_set4) have no status, though, so they are live on every boot of an M2 MacBook Air, 13" MacBook Pro or Mac mini. The pmgr driver probes each of them: it reads the register, sets the auto-PM enable bit if the state is active, and genpd turns unused domains off late in boot. Nothing consumes them while the ANE is disabled. Disable them as well, so the ANE and everything only it uses are off together, and say so above them. ane_sys predates #155 and keeps its status. Found in a review of #155. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 72c7bd0c7a03d768daf7774e9481d9024825434c) --- arch/arm64/boot/dts/apple/t8112-pmgr.dtsi | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index 0e8df1cfa21f6..26ab30e219f12 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -1132,6 +1132,10 @@ power-domains = <&ps_venc_pipe5>, <&ps_venc_pipe4>; }; + /* + * These states serve only the ANE. They stay disabled with the ane + * node in t8112.dtsi; enable them together with it. + */ ps_ane_sys_cpu: power-controller@c008 { compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc008 4>; @@ -1139,6 +1143,7 @@ #reset-cells = <0>; label = "ane_sys_cpu"; power-domains = <&ps_ane_sys>; + status = "disabled"; }; ps_ane_td: power-controller@c010 { @@ -1148,6 +1153,7 @@ #reset-cells = <0>; label = "ane_td"; power-domains = <&ps_ane_sys_cpu>; + status = "disabled"; }; ps_ane_base: power-controller@c018 { @@ -1157,6 +1163,7 @@ #reset-cells = <0>; label = "ane_base"; power-domains = <&ps_ane_td>; + status = "disabled"; }; ps_ane_set1: power-controller@c020 { @@ -1166,6 +1173,7 @@ #reset-cells = <0>; label = "ane_set1"; power-domains = <&ps_ane_base>; + status = "disabled"; }; ps_ane_set2: power-controller@c028 { @@ -1175,6 +1183,7 @@ #reset-cells = <0>; label = "ane_set2"; power-domains = <&ps_ane_set1>; + status = "disabled"; }; ps_ane_set3: power-controller@c030 { @@ -1184,6 +1193,7 @@ #reset-cells = <0>; label = "ane_set3"; power-domains = <&ps_ane_set2>; + status = "disabled"; }; ps_ane_set4: power-controller@c038 { @@ -1193,6 +1203,7 @@ #reset-cells = <0>; label = "ane_set4"; power-domains = <&ps_ane_set3>; + status = "disabled"; }; ps_disp0_cpu0: power-controller@10000 { From e987e572635a05adab3d127442527a235430d896 Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 14:24:57 -0700 Subject: [PATCH 0361/1579] accel/ane: t6021: stop logging boot progress at emergency level The M2 ANE module from Joshua Warren's aurora-silicon/linux#155 logs its probe and firmware boot progress with dev_emerg(). These lines came from bring-up, where each one had to reach the console before the write it announced. With fw_start on by default, an M2 Max now prints about twenty emergency-level lines on every boot, which go to every console and terminal regardless of loglevel and bury real emergencies. Use the level each message deserves: - the per-phase BOOT-PHASE markers and the probe register readouts (ane_cpu ACTUAL, PWGATE, the engine reads, staging and dispatch, the legacy ack and hello start) become dev_dbg(); dynamic debug still gets them for crash attribution; - the outcome lines (sequence returned, done without a CPU start, the legacy hello result) become dev_info(); - the SCRATCH/RVBAR/tick dump, printed only when the ANE CPU was released but never reported READY, becomes dev_err(). The 30 ms drain after each phase marker is left as it is, and the comment above it now says how to see the marker. No other drivers/accel/ane code logs at emergency, alert or critical level. Found in a review of #155. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit d7ec38757843b95b2414d385965358d615647f01) --- drivers/accel/ane/ane_t6021_boot.c | 43 +++++++++---------- drivers/accel/ane/ane_t6021_rtclient_main.c | 46 ++++++++++----------- 2 files changed, 45 insertions(+), 44 deletions(-) diff --git a/drivers/accel/ane/ane_t6021_boot.c b/drivers/accel/ane/ane_t6021_boot.c index de11bb8f095fa..1ae368ae3e55f 100644 --- a/drivers/accel/ane/ane_t6021_boot.c +++ b/drivers/accel/ane/ane_t6021_boot.c @@ -334,14 +334,15 @@ static void ane_boot_phase(void *ctx, const char *what) { struct ane_t6021_boot_mmio *mm = ctx; - /* bounded phase marker: one line per block boundary, survives - * netconsole for crash attribution (never per-poll). KERN_EMERG - * + a short drain so the line reaches tty0/netconsole/ssh - * BEFORE the risky write it announces (fw-start-debug - * 2026-09-22: fwstart#2 died with zero capture — the marker - * must beat the write). + /* bounded phase marker: one line per block boundary (never + * per-poll), for crash attribution. It is a debug message: turn + * it on with dynamic debug, and raise the console loglevel to + * see it there. The short drain lets the line reach + * tty0/netconsole/ssh BEFORE the risky write it announces + * (fw-start-debug 2026-09-22: fwstart#2 died with zero capture — + * the marker must beat the write). */ - dev_emerg(mm->ane->dev, "BOOT-PHASE %s\n", what); + dev_dbg(mm->ane->dev, "BOOT-PHASE %s\n", what); msleep(30); } @@ -457,14 +458,14 @@ static void ane_t6021_boot_progress_dump(struct ane_t6021 *ane) tick1 = readl(eng + ANE_T6021_BOOT_REG_TICK); for (i = 0; i < 8; i++) - dev_emerg(ane->dev, "PROGRESS SCRATCH%u=%08x\n", i, - readl(eng + ANE_T6021_BOOT_REG_SCRATCH0 + 4 * i)); - dev_emerg(ane->dev, - "PROGRESS rvbar=%016llx cpu_status=%08x tick %08x->%08x (%s)\n", - readq(eng + ANE_ASC_RVBAR), - readl(eng + ANE_ASC_CPU_STATUS), - tick0, tick1, - tick1 != tick0 ? "24MHz domain clocked" : "tick STATIC"); + dev_err(ane->dev, "PROGRESS SCRATCH%u=%08x\n", i, + readl(eng + ANE_T6021_BOOT_REG_SCRATCH0 + 4 * i)); + dev_err(ane->dev, + "PROGRESS rvbar=%016llx cpu_status=%08x tick %08x->%08x (%s)\n", + readq(eng + ANE_ASC_RVBAR), + readl(eng + ANE_ASC_CPU_STATUS), + tick0, tick1, + tick1 != tick0 ? "24MHz domain clocked" : "tick STATIC"); } int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, int rtb_mode) @@ -501,9 +502,9 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, return -EBUSY; } - dev_emerg(ane->dev, - "BOOT-PHASE dispatch (stop_after=%d%s)\n", stop_after, - stop_after ? " BISECT STOP ARMED" : ""); + dev_dbg(ane->dev, + "BOOT-PHASE dispatch (stop_after=%d%s)\n", stop_after, + stop_after ? " BISECT STOP ARMED" : ""); r = ane_t6021_boot_run(&io, &cfg, &cs, &fa, &bo, &sres); @@ -523,9 +524,9 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, if (!cs) { /* no CPU start: full release path, normal ownership */ module_put(THIS_MODULE); - dev_emerg(ane->dev, - "BOOT-PHASE done r=%d cpu_started=0 (no CPU release: state clean, module unpinned)\n", - r); + dev_info(ane->dev, + "BOOT-PHASE done r=%d cpu_started=0 (no CPU release: state clean, module unpinned)\n", + r); return r; } diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 18d0ecf6b2f6e..f9eb131a4f30e 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -1996,7 +1996,7 @@ static int ane_rtclient_attach_genpd(struct ane_rtclient *ane) pd->count = i + 1; } - dev_emerg(dev, "BOOT-PHASE genpd domains attached: %d\n", count); + dev_dbg(dev, "BOOT-PHASE genpd domains attached: %d\n", count); out: mutex_unlock(&ane_rtclient_pd_lock); return err; @@ -2085,7 +2085,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) "pmgr window map failed; G1 gate cannot run\n"); } ps_cpu = readl(ane->pmgr + ane->soc->ps_cpu_off); - dev_emerg(dev, "ane_cpu ACTUAL = 0x%x\n", ps_cpu); + dev_dbg(dev, "ane_cpu ACTUAL = 0x%x\n", ps_cpu); if (FIELD_GET(ANE_PS_ACTUAL, ps_cpu) != ANE_PS_ON) { pm_runtime_put_sync_suspend(dev); pm_runtime_disable(dev); @@ -2100,7 +2100,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) u32 gate = IS_ERR(set) ? U32_MAX : readl(set + ane->soc->pwgate_off); - dev_emerg(dev, "PWGATE = 0x%x\n", gate); + dev_dbg(dev, "PWGATE = 0x%x\n", gate); if (gate & GENMASK(29, 28)) { pm_runtime_put_sync_suspend(dev); pm_runtime_disable(dev); @@ -2111,9 +2111,9 @@ static int ane_rtclient_probe(struct platform_device *pdev) cpu_status = readl(ane->engine + ANE_ASC_CPU_STATUS); rvbar = readq(ane->engine + ANE_ASC_RVBAR); - dev_emerg(dev, - "BOOT-PHASE engine reads ok: CPU_STATUS = 0x%x, RVBAR = %016llx (bit0=%u)\n", - cpu_status, rvbar, (u32)(rvbar & 1)); + dev_dbg(dev, + "BOOT-PHASE engine reads ok: CPU_STATUS = 0x%x, RVBAR = %016llx (bit0=%u)\n", + cpu_status, rvbar, (u32)(rvbar & 1)); if (!(cpu_status & ANE_ASC_CPU_STATUS_RUNNING)) { if (!fw_start) { @@ -2155,7 +2155,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) a->power_gated = true; ane->fw = a; - dev_emerg(dev, "BOOT-PHASE fwload stage+alias begin\n"); + dev_dbg(dev, "BOOT-PHASE fwload stage+alias begin\n"); ret = ane_t6021_fwload_probe(a); if (ret) { dev_err_probe(dev, ret, "fw_start: staging failed\n"); @@ -2175,8 +2175,8 @@ static int ane_rtclient_probe(struct platform_device *pdev) return -EINVAL; } - dev_emerg(dev, "BOOT-PHASE dispatch (table_mode=%d)\n", - fw_start_table_mode); + dev_dbg(dev, "BOOT-PHASE dispatch (table_mode=%d)\n", + fw_start_table_mode); ret = ane_t6021_boot_start(a, 0, fw_start_table_mode, fw_start_rtb_mode); if (ret == -ENODATA || ret == -EAGAIN || @@ -2190,10 +2190,10 @@ static int ane_rtclient_probe(struct platform_device *pdev) ane->held = true; cpu_status = readl(ane->engine + ANE_ASC_CPU_STATUS); - dev_emerg(dev, - "BOOT-PHASE sequence returned %pe (cpu_started=%u fw_alive=%u booted=%u) CPU_STATUS=0x%x\n", - ERR_PTR(ret), a->cpu_started, a->fw_alive, a->booted, - cpu_status); + dev_info(dev, + "BOOT-PHASE sequence returned %pe (cpu_started=%u fw_alive=%u booted=%u) CPU_STATUS=0x%x\n", + ERR_PTR(ret), a->cpu_started, a->fw_alive, a->booted, + cpu_status); if (!a->fw_alive && !fw_start_rtb_mode) { dev_err(dev, "fw_start: poll A timeout, no READY — HELD until reboot, RTKit handshake skipped\n"); @@ -2213,9 +2213,9 @@ static int ane_rtclient_probe(struct platform_device *pdev) int hello_ret = 0; dma_wmb(); - dev_emerg(dev, - "LEGACY P8 host ack: SCRATCH3 <- %08x\n", - ANE_T6021_BOOT_ACK); + dev_dbg(dev, + "LEGACY P8 host ack: SCRATCH3 <- %08x\n", + ANE_T6021_BOOT_ACK); /* hello_wait_ms > 0 only: init rtkit and arm the RX * poll worker BEFORE writing the ack, so a HELLO * after the ack is not missed. The 13.5 fw sent none @@ -2241,17 +2241,17 @@ static int ane_rtclient_probe(struct platform_device *pdev) unsigned long hello_deadline = jiffies + msecs_to_jiffies(hello_wait_ms); - dev_emerg(dev, "LEGACY hello: boot begin (%u ms)\n", - hello_wait_ms); + dev_dbg(dev, "LEGACY hello: boot begin (%u ms)\n", + hello_wait_ms); do { hello_ret = apple_rtkit_boot(ane->rtk); } while (hello_ret == -ETIME && time_before(jiffies, hello_deadline)); - dev_emerg(dev, - "LEGACY hello: boot %pe running=%d crashed=%d\n", - ERR_PTR(hello_ret), - apple_rtkit_is_running(ane->rtk), - apple_rtkit_is_crashed(ane->rtk)); + dev_info(dev, + "LEGACY hello: boot %pe running=%d crashed=%d\n", + ERR_PTR(hello_ret), + apple_rtkit_is_running(ane->rtk), + apple_rtkit_is_crashed(ane->rtk)); if (!hello_ret) { ane->boot_done = true; if (start_app_eps) From 4e315c497273216da8e6e6c6fa122a2ac8e1b77b Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 2 Oct 2026 14:24:57 -0700 Subject: [PATCH 0362/1579] accel/ane: t6021: keep one MODULE_LICENSE and MODULE_DESCRIPTION ane_t6021.ko is built from ane_t6021_rtclient_main.o, ane_t6021_fwload.o and ane_t6021_boot.o, and each of the three carries its own MODULE_LICENSE() and MODULE_DESCRIPTION(), so modinfo reports three licenses and three descriptions for the one module. The comment on the extra copies says the objects also link into an ane_t6021_rtclient.ko, but the Makefile from Joshua Warren's aurora-silicon/linux#155 builds no such module. Keep the set next to module_init() in ane_t6021_rtclient_main.c and drop the copies, and their comment, from the fwload and boot objects. The MODULE_FIRMWARE() lines stay with the firmware loader. Found in a review of #155. Signed-off-by: Chris Kearney Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FwdG63dRaMrRbYrUBkSa1i (cherry picked from commit 9b8b60c4c8a11c969a547f3f8805664ae176ed72) --- drivers/accel/ane/ane_t6021_boot.c | 6 ------ drivers/accel/ane/ane_t6021_fwload.c | 5 ----- 2 files changed, 11 deletions(-) diff --git a/drivers/accel/ane/ane_t6021_boot.c b/drivers/accel/ane/ane_t6021_boot.c index 1ae368ae3e55f..ec2bb466133c4 100644 --- a/drivers/accel/ane/ane_t6021_boot.c +++ b/drivers/accel/ane/ane_t6021_boot.c @@ -149,12 +149,6 @@ module_param(boot_prevent_nap, bool, 0444); MODULE_PARM_DESC(boot_prevent_nap, "Retain firmware nap-prevention counter via init resource bit (default on: proven add-path value)"); -/* This object links into both ane_t6021.ko and ane_t6021_rtclient.ko; - * per-object metadata keeps modpost happy for either composition. - */ -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("T6021 ANE contract-pinned boot sequence core"); - /* Boot-write gates — ITEMIZED, each a HARD gate: the ENTIRE write * sequence (preboot engine table, scratch clear + pulse, RVBAR * resolution, CPU release, publication, wake) runs start-to-finish diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index c8a7ad8d7bee8..04153a2a4d576 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -84,11 +84,6 @@ #include "ane_t6021_boot.h" #include "ane_fw_validate.h" -/* This object links into both ane_t6021.ko and ane_t6021_rtclient.ko; - * per-object metadata keeps modpost happy for either composition. - */ -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("T6021 ANE firmware staging + entry alias"); MODULE_FIRMWARE(ANE_FW_SELENE_NAME); MODULE_FIRMWARE(ANE_FW_BIA_NAME); From ad2201d82aed76a93cca36e7856a384d9477c91d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 2 Oct 2026 18:42:11 -0400 Subject: [PATCH 0363/1579] vfio: retain typed fragments for mixed-page type1 mappings Retain typed long-term pin identities and byte ranges for alternative-granule and fine-aligned type1 DMA mappings. Merge physically contiguous runs at the actual domain alignment, preserve cached pins after source-mm exit, unwind mapping/replay failures, and refund unique physical-page charges after unmap. Keep native-aligned mappings on their existing GUP/batching path. Preserve external native-page array offsets and reject layouts that cannot be represented by one page pointer. Keep software dirty tracking at a stable 4K unit across domain changes, and export typed GET/mark-dirty helpers for loadable consumers. Correct the callback-unpin device-list/IOMMU lock order. Add default-off KUnit and privileged disposable-guest ioctl fixtures. Nine VFIO cases pass on native16K and64K; matched native-GUP controls fail the seven fragment cases. Real EL0 type1 requests pass29checks per AArch64 ABI (native16/4K andnative64/4K/16K), plus AArch32 auto4K and native fallback capability models. Main mixed-page suites, IOMMUFD regressions, feature-off compilation and six guest module load/unload checks pass. This remains experimental. Native-to-alternative VADDR ownership conversion, external spans across multiple fine DMA mappings, physical-device attachment and DMA, Apple hardware/GPU execution, games and throughput remain unqualified. --- Documentation/arch/arm64/user4k.rst | 104 ++++ arch/arm64/Kconfig.debug | 20 +- drivers/vfio/Kconfig | 6 +- drivers/vfio/vfio_iommu_type1.c | 171 ++++-- drivers/vfio/vfio_iommu_type1_fragments.c | 433 ++++++++++++++ .../vfio/vfio_iommu_type1_fragments_test.c | 562 ++++++++++++++++++ drivers/vfio/vfio_iommu_type1_ioctl_test.c | 285 +++++++++ drivers/vfio/vfio_iommu_type1_test.h | 34 ++ include/linux/mm_subpage.h | 2 + mm/gup.c | 26 +- tools/testing/selftests/arm64/user4k/Makefile | 8 +- .../arm64/user4k/compat-vfio-probe.c | 196 ++++++ .../arm64/user4k/vfio-fixture-init.c | 289 +++++++++ 13 files changed, 2084 insertions(+), 52 deletions(-) create mode 100644 drivers/vfio/vfio_iommu_type1_fragments.c create mode 100644 drivers/vfio/vfio_iommu_type1_fragments_test.c create mode 100644 drivers/vfio/vfio_iommu_type1_ioctl_test.c create mode 100644 drivers/vfio/vfio_iommu_type1_test.h create mode 100644 tools/testing/selftests/arm64/user4k/compat-vfio-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/vfio-fixture-init.c diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index eaf342b4b3468..2b4cc50e4d14e 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -4300,3 +4300,107 @@ user4k-vm-state/gem-granule-v1/. This extends common GEM shmem/PRIME CPU mapping coverage using VGEM. It does not execute Asahi GPU commands or qualify imported buffers from other exporters, hardware DMA, Vulkan/GL, Windows games or performance. + + +Legacy VFIO retained fragment provider (2026-10-02) +------------------------------------------------- + +Type1 now retains typed pin records for alternative-granule mappings and +native mappings whose virtual address, IOVA or length requires finer +alignment. The native-aligned default continues through its existing GUP +and batching path. Retained records identify the original pinned allocation +and byte range; IOMMU translations are no longer used to reconstruct pin +ownership on the new path. Accounting counts unique native physical pages +within each mapping provider and refunds the retained mm after source exit. + +Mapping and replay merge physically contiguous byte runs, check the actual +domain minimum alignment, and unwind successful prefixes on failure. IOTLB +synchronization precedes pin release. Cached pins preserve the original DMA +bytes after source-mm exit or address replacement; missing CPU-only ranges +use temporary GET references. A typed dirty helper records writes without +releasing cached pins. External native-page arrays accept complete, aligned, +physically contiguous native spans and preserve callers' original byte +offsets. Layouts that need an explicit physical offset or cannot be represented +by one native page pointer remain rejected; spans across separate fine-grained +DMA mappings still need aggregate ownership support. + +The container advertises the composed minimum supported by attached domains, +with a 4K floor while the mixed-page feature is enabled. Software dirty +tracking uses a stable 4K unit, independent of domain attachment and minimum +translation size, and marks every covered unit for native-page external pins. +A native mapping's VADDR owner change to an alternative mm still requires +ownership conversion and is rejected explicitly until that path is implemented. + +The private type1 KUnit harness exercises real typed pins, source mm lifetime, +canonical mapping, replay, external pins, CPU access and DMA removal against +recording IOMMU domain operations. Seven fragment cases and two accounting +cases pass on both native16K and native64K kernels. Coverage includes failed +second-domain mapping, replay rollback with retained external pins, exact fine +ranges and dirty bits, anonymous 4K pins after source exit and transfer to a +native owner, and stable dirty granularity with a 16K-only domain. The first +callback-unpin test exposed an inconsistent device-list/IOMMU mutex order; +registration, removal and notification now use a consistent order and +notification snapshots the range before releasing the IOMMU lock. + +The final source also passes the broader mixed-page KUnit suites and real EL0 +IOMMUFD USER/MM regressions. These tests do not establish real EL0 legacy +VFIO ioctl behavior, physical-device DMA, Apple boot, GPU execution, Windows +games or throughput. Native-to-alternative ownership conversion, aggregate +external spans and hardware qualification remain unfinished. + +Matched native-GUP controls retain the new harness and corrected lock order, +but disable fragment-provider selection and restore the native PAGE_SIZE +alignment floor. Both controls fail all seven fragment cases while their two +accounting cases pass. They power down without kernel warnings. These are +matched behavior controls, not literal original-source kernel Images. + +A separate native16K MODULES=y configuration builds VFIO and type1 as modules, +with the mixed-page feature enabled and the private tests disabled. Full +MODPOST succeeds; a guest loads and unloads both modules and verifies type1 +registration in six checks. This qualifies linkage of the typed PIN, GET, +mark-dirty and release APIs, rather than physical device operation. + + +Real EL0 legacy VFIO ioctl coverage (2026-10-02) +---------------------------------------------- + +A separate default-off ARM64_USER4K_VFIO_IOCTL_TEST option exposes a 0600, +CAP_SYS_ADMIN-only disposable-guest misc fixture. Standard GET_INFO, MAP_DMA, +UNMAP_DMA and DIRTY_PAGES requests call the canonical type1 dispatcher with +actual userspace arguments and the calling process's real mm. Private controls +supply recording IOMMU domains, external-page consumers and bounded CPU access. +The fixture serializes its requests; this does not qualify concurrent type1 +ioctls or physical IOMMU group/device attachment. Normal kernels do not expose +this test interface. + +All 29 AArch64 checks pass under native16K/4K and native64K/4K/16K. They cover +argument rejection, lazy mappings without DMA charges, temporary CPU GET, +duplicate external pins with unaligned caller offsets, every 4K dirty unit +covered by a native external page, and unmap-callback cleanup. A child maps a +fine-offset shmem range and exits. Its real mm_users reaches zero while two +native physical pages remain charged and mapped. A foreign userspace process +still receives EFAULT from direct CPU access, while a kernel worker reads and +writes the retained original bytes. Failed second-domain replay leaves the +first domain and pins intact; retry succeeds without resurrecting the source +mm. Canonical dirty-bitmap output reports exactly the two touched 4K units; +final unmap clears both domains and refunds all charges. + +A freestanding AArch32 probe passes real type1 ioctls, external arrays, dirty +bitmap pointer layout, child-mm exit, kernel-worker access and final cleanup. +It checks actual AT_PAGESZ and protection alignment. With 4K available it uses +4K pages; boot models removing TGran4 support still execute it with native16K +or native64K and reject a 4K-only mprotect operation. These are capability +models, not Apple A9 or other AArch32 hardware tests. + +Matched native-GUP controls retain the real ioctl fixture but disable the new +provider and restore native PAGE_SIZE alignment. They reproduce rejection of +fine mappings and alternative external pins; the fixtures fail and power down +without kernel warnings. They are matched behavior controls, not original-source +kernel Images. Existing type1 KUnit and broader mixed-page regressions also +pass on the fixture-enabled kernels. + +Evidence is under user4k-vm-state/vfio-ioctl-v1/. These tests qualify the actual +EL0-to-type1 provider/ABI path with supplied domains, not the VFIO core group +attachment protocol, physical DMA, Apple boot, GPU execution or game throughput. +Native-to-alternative VADDR ownership conversion and external spans crossing +multiple fine DMA mappings remain unfinished. diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 9d7742848b5ac..3a2e541df52f7 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -77,7 +77,7 @@ config ARM64_USER4K_IOMMUFD_TEST Say N for normal kernels. config ARM64_USER4K_VFIO_ACCOUNT_TEST - bool "VFIO type1 accounting harness for mixed userspace granules" + bool "VFIO type1 pinning harness for mixed userspace granules" depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y select VFIO select VFIO_GROUP @@ -86,9 +86,21 @@ config ARM64_USER4K_VFIO_ACCOUNT_TEST select VFIO_TYPE1_ACCOUNT_KUNIT_TEST default n help - Enable built-in VFIO type1 and its source-mm accounting tests in - a disposable mixed-granule test guest. This does not establish - physical-device DMA or alternative-page pinning support. + Enable built-in VFIO type1 accounting and retained-fragment tests + in a disposable mixed-granule guest. Recorded IOMMU translations + do not establish physical-device DMA support. + Say N for normal kernels. + +config ARM64_USER4K_VFIO_IOCTL_TEST + bool "Private VFIO type1 ioctl harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y && DEBUG_KERNEL + depends on RUNTIME_TESTING_MENU + select ARM64_USER4K_VFIO_ACCOUNT_TEST + default n + help + Expose a root-only disposable-guest misc device forwarding real EL0 + arguments to type1 ioctls and supplying recording IOMMU domains. + These privileged test controls perform no physical device DMA. Say N for normal kernels. config ARM64_USER4K_DATA_KUNIT_TEST diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index 94f4708bf8d2b..33f6c7e9608d5 100644 --- a/drivers/vfio/Kconfig +++ b/drivers/vfio/Kconfig @@ -55,14 +55,16 @@ config VFIO_IOMMU_TYPE1 default n config VFIO_TYPE1_ACCOUNT_KUNIT_TEST - bool "Test VFIO type1 pin accounting lifetime" if !KUNIT_ALL_TESTS + bool "Test VFIO type1 pin accounting and retained fragments" if !KUNIT_ALL_TESTS depends on KUNIT=y && VFIO_IOMMU_TYPE1=y default KUNIT_ALL_TESTS help Exercise locked-memory refunds after source-mm exit and DMA owner changes. Arm64 mixed-granule configurations also test accounting conversions between user and native page sizes. - These tests use real mm lifetimes without a device or DMA mappings. + Mixed-granule tests exercise retained pins, mappings, replay and + external page arrays with recorded IOMMU translations. No physical + device DMA is performed. config VFIO_IOMMU_SPAPR_TCE tristate diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index a3d5e93865706..68b7877cf626e 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -28,11 +28,14 @@ #include #include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -84,6 +87,8 @@ struct vfio_domain { bool enforce_cache_coherency : 1; }; +struct vfio_fragment_provider; + struct vfio_dma { struct rb_node node; dma_addr_t iova; /* Device address */ @@ -99,6 +104,7 @@ struct vfio_dma { unsigned long *bitmap; struct mm_struct *mm; size_t locked_vm; + struct vfio_fragment_provider *fragments; }; struct vfio_batch { @@ -152,6 +158,8 @@ struct vfio_regions { #define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX) static int put_pfn(unsigned long pfn, int prot); +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length); static struct vfio_iommu_group* vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, @@ -238,6 +246,12 @@ static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old) } +/* Software dirty granularity must survive domain attach/detach unchanged. */ +static size_t vfio_dirty_pgsize(struct vfio_iommu *iommu) +{ + return IS_ENABLED(CONFIG_MM_SUBPAGE) ? SZ_4K : 1UL << __ffs(iommu->pgsize_bitmap); +} + static int vfio_dma_bitmap_alloc(struct vfio_dma *dma, size_t pgsize) { uint64_t npages = dma->size / pgsize; @@ -272,14 +286,15 @@ static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize) for (p = rb_first(&dma->pfn_list); p; p = rb_next(p)) { struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, node); - bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, 1); + bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, + PAGE_SIZE >> pgshift); } } static void vfio_iommu_populate_bitmap_full(struct vfio_iommu *iommu) { struct rb_node *n; - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); @@ -421,8 +436,15 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) vpfn->ref_count--; if (!vpfn->ref_count) { - ret = put_pfn(vpfn->pfn, dma->prot); - vfio_remove_from_pfn_list(dma, vpfn); + if (dma->fragments) { + unsigned long offset = vpfn->iova - dma->iova; + + vfio_remove_from_pfn_list(dma, vpfn); + vfio_fragments_release_range(dma, offset, PAGE_SIZE); + } else { + ret = put_pfn(vpfn->pfn, dma->prot); + vfio_remove_from_pfn_list(dma, vpfn); + } } return ret; } @@ -472,6 +494,8 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) return ret; } +#include "vfio_iommu_type1_fragments.c" + /* * Some mappings aren't backed by a struct page, for example an mmap'd * MMIO range for our own or another device. These use a different @@ -917,6 +941,8 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, if (!iommu || !pages || npage <= 0) return -EINVAL; + /* Callers recover the original byte offset from the returned page. */ + user_iova = ALIGN_DOWN(user_iova, PAGE_SIZE); /* Supported for v2 version only */ if (!iommu->v2) @@ -954,7 +980,8 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, iova = user_iova + PAGE_SIZE * i; dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - if (!dma) { + if (!dma || iova < dma->iova || + dma->size - (iova - dma->iova) < PAGE_SIZE) { ret = -EINVAL; goto pin_unwind; } @@ -970,6 +997,14 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, continue; } + if (dma->fragments) { + ret = vfio_fragments_external(dma, iova, &phys_pfn); + if (ret) + goto pin_unwind; + pages[i] = pfn_to_page(phys_pfn); + goto dirty; + } + remote_vaddr = dma->vaddr + (iova - dma->iova); ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn, do_accounting); @@ -989,16 +1024,17 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, } pages[i] = pfn_to_page(phys_pfn); - +dirty: if (iommu->dirty_page_tracking) { - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); /* * Bitmap populated with the smallest supported page * size */ bitmap_set(dma->bitmap, - (iova - dma->iova) >> pgshift, 1); + (iova - dma->iova) >> pgshift, + PAGE_SIZE >> pgshift); } } ret = i; @@ -1042,6 +1078,7 @@ static void vfio_iommu_type1_unpin_pages(void *iommu_data, if (WARN_ON(npage <= 0)) return; + user_iova = ALIGN_DOWN(user_iova, PAGE_SIZE); if (WARN_ON(check_mul_overflow(npage, PAGE_SIZE, &iova_size) || check_add_overflow(user_iova, iova_size - 1, &iova_end))) return; @@ -1166,6 +1203,10 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, if (!dma->size) return 0; + if (dma->fragments) { + vfio_fragments_unmap(iommu, dma); + return 0; + } if (list_empty(&iommu->domain_list)) return 0; @@ -1244,6 +1285,7 @@ static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) { WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list)); vfio_unmap_unpin(iommu, dma, true); + vfio_fragments_destroy(dma); vfio_unlink_dma(iommu, dma); put_task_struct(dma->task); mmdrop(dma->mm); @@ -1257,21 +1299,18 @@ static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu) { struct vfio_domain *domain; + unsigned long minimum = SZ_4K; iommu->pgsize_bitmap = ULONG_MAX; - - list_for_each_entry(domain, &iommu->domain_list, next) + list_for_each_entry(domain, &iommu->domain_list, next) { iommu->pgsize_bitmap &= domain->domain->pgsize_bitmap; - - /* - * In case the IOMMU supports page sizes smaller than PAGE_SIZE - * we pretend PAGE_SIZE is supported and hide sub-PAGE_SIZE sizes. - * That way the user will be able to map/unmap buffers whose size/ - * start address is aligned with PAGE_SIZE. Pinning code uses that - * granularity while iommu driver can use the sub-PAGE_SIZE size - * to map the buffer. - */ - if (iommu->pgsize_bitmap & ~PAGE_MASK) { + minimum = max(minimum, 1UL << __ffs(domain->domain->pgsize_bitmap)); + } + if (IS_ENABLED(CONFIG_MM_SUBPAGE)) { + /* Every domain can compose its minimum pages into this common unit. */ + iommu->pgsize_bitmap &= ~(minimum - 1); + iommu->pgsize_bitmap |= minimum; + } else if (iommu->pgsize_bitmap & ~PAGE_MASK) { iommu->pgsize_bitmap &= PAGE_MASK; iommu->pgsize_bitmap |= PAGE_SIZE; } @@ -1378,6 +1417,8 @@ static void vfio_notify_dma_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) { struct vfio_device *device; + dma_addr_t iova = dma->iova; + size_t size = dma->size; if (list_empty(&iommu->device_list)) return; @@ -1388,11 +1429,12 @@ static void vfio_notify_dma_unmap(struct vfio_iommu *iommu, * call back down to this code and try to obtain the iommu->lock we must * drop it. */ - mutex_lock(&iommu->device_list_lock); + /* The range is a snapshot: another unmap may free dma while unlocked. */ mutex_unlock(&iommu->lock); + mutex_lock(&iommu->device_list_lock); list_for_each_entry(device, &iommu->device_list, iommu_entry) - device->ops->dma_unmap(device, dma->iova, dma->size); + device->ops->dma_unmap(device, iova, size); mutex_unlock(&iommu->device_list_lock); mutex_lock(&iommu->lock); @@ -1448,11 +1490,10 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, /* When dirty tracking is enabled, allow only min supported pgsize */ if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && - (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) { + (!iommu->dirty_page_tracking || (bitmap->pgsize != vfio_dirty_pgsize(iommu)))) { goto unlock; } - WARN_ON((pgsize - 1) & PAGE_MASK); again: /* * vfio-iommu-type1 (v1) - User mappings were coalesced together to @@ -1541,7 +1582,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { ret = update_user_bitmap(bitmap->data, iommu, dma, - iova, pgsize); + iova, vfio_dirty_pgsize(iommu)); if (ret) break; } @@ -1598,6 +1639,13 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret = 0; + if (dma->fragments) { + dma->size = map_size; + ret = vfio_fragments_map_all(iommu, dma); + if (ret) + vfio_remove_dma(iommu, dma); + return ret; + } vfio_batch_init(&batch); while (size) { @@ -1663,7 +1711,6 @@ static int vfio_change_dma_mm(struct vfio_dma *dma, struct task_struct *task, if (mm == dma->mm) return 0; - ret = mm_lock_acct(task, mm, lock_cap, npage); if (ret) return ret; @@ -1688,6 +1735,9 @@ static int vfio_change_dma_mm(struct vfio_dma *dma, struct task_struct *task, static int vfio_change_dma_owner(struct vfio_dma *dma) { + /* Native mappings need ownership conversion before a granule change. */ + if (!dma->fragments && mm_page_size(current->mm) != PAGE_SIZE) + return -EOPNOTSUPP; return vfio_change_dma_mm(dma, current->group_leader, current->mm, capable(CAP_IPC_LOCK)); } @@ -1729,8 +1779,6 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); - WARN_ON((pgsize - 1) & PAGE_MASK); - if ((size | iova | vaddr) & (pgsize - 1)) { ret = -EINVAL; goto out_unlock; @@ -1797,6 +1845,11 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, /* Insert zero-sized and grow as we map chunks of it */ vfio_link_dma(iommu, dma); + ret = vfio_fragments_init(dma, size); + if (ret) { + vfio_remove_dma(iommu, dma); + goto out_unlock; + } /* Don't pin and map if container doesn't contain IOMMU capable domain*/ if (list_empty(&iommu->domain_list)) @@ -1805,7 +1858,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, ret = vfio_pin_map_dma(iommu, dma, size); if (!ret && iommu->dirty_page_tracking) { - ret = vfio_dma_bitmap_alloc(dma, pgsize); + ret = vfio_dma_bitmap_alloc(dma, vfio_dirty_pgsize(iommu)); if (ret) vfio_remove_dma(iommu, dma); } @@ -1838,6 +1891,14 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, size_t pos = 0; dma = rb_entry(n, struct vfio_dma, node); + if (dma->fragments) { + ret = vfio_fragments_acquire(dma, 0, dma->size); + if (!ret) + ret = vfio_fragments_map_domain(dma, domain->domain); + if (ret) + goto unwind; + continue; + } while (pos < dma->size) { dma_addr_t iova = dma->iova + pos; @@ -1922,6 +1983,11 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); size_t pos = 0; + if (dma->fragments) { + iommu_unmap(domain->domain, dma->iova, dma->size); + vfio_fragments_release_unused(dma); + continue; + } if (dma->iommu_mapped) { iommu_unmap(domain->domain, dma->iova, dma->size); continue; @@ -2458,6 +2524,10 @@ static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu) long locked = 0, unlocked = 0; dma = rb_entry(n, struct vfio_dma, node); + if (dma->fragments) { + vfio_unmap_unpin(iommu, dma, true); + continue; + } unlocked += vfio_unmap_unpin(iommu, dma, false); p = rb_first(&dma->pfn_list); for (; p; p = rb_next(p)) { @@ -2662,7 +2732,8 @@ static void *vfio_iommu_type1_open(unsigned long arg) mutex_init(&iommu->lock); mutex_init(&iommu->device_list_lock); INIT_LIST_HEAD(&iommu->device_list); - iommu->pgsize_bitmap = PAGE_MASK; + iommu->pgsize_bitmap = IS_ENABLED(CONFIG_MM_SUBPAGE) ? + GENMASK(BITS_PER_LONG - 1, 12) : PAGE_MASK; INIT_LIST_HEAD(&iommu->emulated_iommu_groups); return iommu; @@ -2828,7 +2899,7 @@ static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu, cap_mig.flags = 0; /* support minimum pgsize */ - cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap); + cap_mig.pgsize_bitmap = vfio_dirty_pgsize(iommu); cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX; return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig)); @@ -3008,7 +3079,7 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, size_t pgsize; mutex_lock(&iommu->lock); - pgsize = 1 << __ffs(iommu->pgsize_bitmap); + pgsize = vfio_dirty_pgsize(iommu); if (!iommu->dirty_page_tracking) { ret = vfio_dma_bitmap_alloc_all(iommu, pgsize); if (!ret) @@ -3062,7 +3133,7 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, mutex_lock(&iommu->lock); - iommu_pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); + iommu_pgsize = vfio_dirty_pgsize(iommu); /* allow only smallest supported pgsize */ if (range.bitmap.pgsize != iommu_pgsize) { @@ -3128,11 +3199,12 @@ static void vfio_iommu_type1_register_device(void *iommu_data, * Holding both locks here allows avoiding the device_list_lock in * several fast paths. See vfio_notify_dma_unmap() */ - mutex_lock(&iommu->lock); + /* Unmap callbacks hold device_list_lock while reentering iommu->lock. */ mutex_lock(&iommu->device_list_lock); + mutex_lock(&iommu->lock); list_add(&vdev->iommu_entry, &iommu->device_list); - mutex_unlock(&iommu->device_list_lock); mutex_unlock(&iommu->lock); + mutex_unlock(&iommu->device_list_lock); } static void vfio_iommu_type1_unregister_device(void *iommu_data, @@ -3143,11 +3215,12 @@ static void vfio_iommu_type1_unregister_device(void *iommu_data, if (!vdev->ops->dma_unmap) return; - mutex_lock(&iommu->lock); + /* Unmap callbacks hold device_list_lock while reentering iommu->lock. */ mutex_lock(&iommu->device_list_lock); + mutex_lock(&iommu->lock); list_del(&vdev->iommu_entry); - mutex_unlock(&iommu->device_list_lock); mutex_unlock(&iommu->lock); + mutex_unlock(&iommu->device_list_lock); } static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, @@ -3172,6 +3245,22 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, return -EPERM; mm = dma->mm; + if (dma->fragments) { + int ret; + + if (!kthread && current->mm != mm) + return -EFAULT; + offset = user_iova - dma->iova; + count = min(count, dma->size - offset); + ret = vfio_fragments_rw(dma, offset, data, count, write, copied); + if (*copied && write && iommu->dirty_page_tracking) { + unsigned long shift = __ffs(vfio_dirty_pgsize(iommu)); + + bitmap_set(dma->bitmap, offset >> shift, + ((offset + *copied - 1) >> shift) - (offset >> shift) + 1); + } + return ret; + } if (!mmget_not_zero(mm)) return -EPERM; @@ -3191,7 +3280,7 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, *copied = copy_to_user((void __user *)vaddr, data, count) ? 0 : count; if (*copied && iommu->dirty_page_tracking) { - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); /* * Bitmap populated with the smallest supported page * size @@ -3300,4 +3389,10 @@ MODULE_DESCRIPTION(DRIVER_DESC); #ifdef CONFIG_VFIO_TYPE1_ACCOUNT_KUNIT_TEST #include "vfio_iommu_type1_account_test.c" +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_ARM64_USER4K_EXPERIMENTAL) +#include "vfio_iommu_type1_fragments_test.c" +#ifdef CONFIG_ARM64_USER4K_VFIO_IOCTL_TEST +#include "vfio_iommu_type1_ioctl_test.c" +#endif +#endif #endif diff --git a/drivers/vfio/vfio_iommu_type1_fragments.c b/drivers/vfio/vfio_iommu_type1_fragments.c new file mode 100644 index 0000000000000..b24e65f2406e8 --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_fragments.c @@ -0,0 +1,433 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by type1: retained byte ranges, rather than IOMMU PFNs, own pins. */ +#ifdef CONFIG_MM_SUBPAGE +struct vfio_fragment { + struct rb_node node; + unsigned long start; + struct user_page_fragment pin; +}; + +struct vfio_fragment_provider { + struct rb_root records; + struct xarray physical_pages; + unsigned long npinned; +}; + +static phys_addr_t vfio_fragment_phys(const struct user_page_fragment *pin) +{ + return PFN_PHYS(folio_pfn(pin->folio)) + pin->offset; +} + +/* Returns a containing range, or the first range after the requested byte. */ +static struct vfio_fragment *vfio_fragment_find(struct vfio_dma *dma, + unsigned long offset) +{ + struct rb_node *node = dma->fragments->records.rb_node; + struct vfio_fragment *next = NULL; + + while (node) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + + if (offset < record->start) { + next = record; + node = node->rb_left; + } else if (offset - record->start >= record->pin.length) { + node = node->rb_right; + } else { + return record; + } + } + return next; +} + +static int vfio_fragments_init(struct vfio_dma *dma, size_t length) +{ + struct vfio_fragment_provider *provider; + + if (mm_page_size(dma->mm) == PAGE_SIZE && + IS_ALIGNED(dma->vaddr | dma->iova | length, PAGE_SIZE)) + return 0; + provider = kzalloc_obj(*provider, GFP_KERNEL_ACCOUNT); + if (!provider) + return -ENOMEM; + provider->records = RB_ROOT; + xa_init(&provider->physical_pages); + dma->fragments = provider; + return 0; +} + +static int vfio_fragment_store(struct vfio_dma *dma, unsigned long start, + struct user_page_fragment *pin) +{ + struct vfio_fragment_provider *provider = dma->fragments; + struct rb_node **link = &provider->records.rb_node, *parent = NULL; + unsigned long pfn = PHYS_PFN(vfio_fragment_phys(pin)); + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + struct vfio_fragment *record; + int ret; + + record = kmalloc_obj(*record, GFP_KERNEL_ACCOUNT); + if (!record) + return -ENOMEM; + ret = xa_err(xa_store(&provider->physical_pages, pfn, xa_mk_value(count + 1), + GFP_KERNEL_ACCOUNT)); + if (ret) { + kfree(record); + return ret; + } + while (*link) { + struct vfio_fragment *cur = rb_entry(*link, struct vfio_fragment, node); + + parent = *link; + link = start < cur->start ? &(*link)->rb_left : &(*link)->rb_right; + } + record->start = start; + record->pin = *pin; + rb_link_node(&record->node, parent, link); + rb_insert_color(&record->node, &provider->records); + if (!count) + provider->npinned++; + *pin = (struct user_page_fragment) {}; + return 0; +} + +/* Device unmaps and their IOTLB synchronization must precede this call. */ +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + struct vfio_fragment_provider *provider = dma->fragments; + struct vfio_fragment *first = vfio_fragment_find(dma, start); + struct rb_node *node = first ? &first->node : NULL; + unsigned long end = start + length; + + if (dma->iommu_mapped) + return; + while (node) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + dma_addr_t iova = dma->iova + record->start; + unsigned long pfn, count; + + if (record->start >= end) + break; + node = rb_next(node); + /* Native-page consumers retain all records overlapping their page. */ + if (vfio_find_vpfn_range(dma, iova >= PAGE_SIZE - 1 ? + iova - (PAGE_SIZE - 1) : 0, + iova + record->pin.length)) + continue; + pfn = PHYS_PFN(vfio_fragment_phys(&record->pin)); + count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + if (WARN_ON(!count)) + continue; + if (count == 1) { + xa_erase(&provider->physical_pages, pfn); + provider->npinned--; + } else { + xa_store(&provider->physical_pages, pfn, xa_mk_value(count - 1), + GFP_NOWAIT); + } + rb_erase(&record->node, &provider->records); + release_user_fragments(&record->pin, 1, dma->prot & IOMMU_WRITE); + kfree(record); + cond_resched(); + } + if (provider->npinned < dma->locked_vm) + vfio_lock_acct(dma, -(long)(dma->locked_vm - provider->npinned), true); +} + +static void vfio_fragments_release_unused(struct vfio_dma *dma) +{ + vfio_fragments_release_range(dma, 0, dma->size); +} + +/* Cached records survive source VA replacement and the last source mm user. */ +static int vfio_fragments_acquire(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + unsigned long end = start + length, original = start; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + struct user_page_fragment *batch = NULL; + bool live = false; + int ret = 0; + + while (start < end) { + struct vfio_fragment *record = vfio_fragment_find(dma, start); + unsigned long stop = end; + long count; + + if (record && record->start <= start) { + start = record->start + record->pin.length; + continue; + } + if (record) + stop = min(stop, record->start); + if (!live) { + if (!mmget_not_zero(dma->mm)) { + ret = -EFAULT; + goto out; + } + live = true; + batch = kmalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!batch) { + ret = -ENOMEM; + goto out; + } + } + count = pin_user_fragments_remote(dma->mm, dma->vaddr + start, + stop - start, FOLL_LONGTERM | (dma->prot & IOMMU_WRITE ? FOLL_WRITE : 0), + batch, capacity); + if (count <= 0) { + ret = count ?: -EFAULT; + goto out; + } + for (unsigned long i = 0; i < count; i++) { + unsigned long bytes = batch[i].length; + + if (WARN_ON(!bytes || bytes > stop - start || + offset_in_page(vfio_fragment_phys(&batch[i])) > + PAGE_SIZE - bytes)) + ret = -EFAULT; + else + ret = vfio_fragment_store(dma, start, &batch[i]); + if (ret) { + release_user_fragments(batch + i, count - i, false); + goto out; + } + start += bytes; + } + cond_resched(); + } + if (dma->fragments->npinned > dma->locked_vm) + ret = vfio_lock_acct(dma, dma->fragments->npinned - dma->locked_vm, false); +out: + kfree(batch); + if (live) + mmput(dma->mm); + if (ret) + vfio_fragments_release_range(dma, original, length); + return ret; +} + +/* Merge physical byte runs while retaining each original pin identity. */ +static int vfio_fragments_map_domain(struct vfio_dma *dma, struct iommu_domain *domain) +{ + unsigned long done = 0, min_size = 1UL << __ffs(domain->pgsize_bitmap); + int ret = 0; + + while (done < dma->size) { + struct vfio_fragment *record = vfio_fragment_find(dma, done); + unsigned long bytes, limit = dma->size - done; + phys_addr_t phys; + + if (WARN_ON(!record || record->start > done)) { + ret = -EFAULT; + goto out; + } + if (disable_hugepages) + limit = min(limit, PAGE_SIZE); + phys = vfio_fragment_phys(&record->pin) + done - record->start; + bytes = min(record->pin.length - (done - record->start), limit); + while (bytes < limit) { + record = vfio_fragment_find(dma, done + bytes); + if (!record || record->start != done + bytes || + vfio_fragment_phys(&record->pin) != phys + bytes) + break; + bytes += min_t(unsigned long, record->pin.length, limit - bytes); + } + if (!IS_ALIGNED((dma->iova + done) | phys | bytes, min_size)) { + ret = -EINVAL; + goto out; + } + ret = iommu_map(domain, dma->iova + done, phys, bytes, + dma->prot | IOMMU_CACHE, GFP_KERNEL_ACCOUNT); + if (ret) + goto out; + done += bytes; + cond_resched(); + } +out: + if (ret && done) + iommu_unmap(domain, dma->iova, done); + return ret; +} + +static int vfio_fragments_map_all(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_domain *domain; + int ret; + + ret = vfio_fragments_acquire(dma, 0, dma->size); + if (ret) + return ret; + list_for_each_entry(domain, &iommu->domain_list, next) { + ret = vfio_fragments_map_domain(dma, domain->domain); + if (ret) + goto unwind; + } + dma->iommu_mapped = true; + return 0; +unwind: + list_for_each_entry_continue_reverse(domain, &iommu->domain_list, next) + iommu_unmap(domain->domain, dma->iova, dma->size); + vfio_fragments_release_unused(dma); + return ret; +} + +static void vfio_fragments_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_domain *domain; + + if (dma->iommu_mapped) { + list_for_each_entry(domain, &iommu->domain_list, next) + WARN_ON(iommu_unmap(domain->domain, dma->iova, dma->size) != dma->size); + dma->iommu_mapped = false; + } + vfio_fragments_release_unused(dma); +} + +static void vfio_fragments_destroy(struct vfio_dma *dma) +{ + if (!dma->fragments) + return; + vfio_fragments_release_unused(dma); + if (WARN_ON(!RB_EMPTY_ROOT(&dma->fragments->records))) + return; + xa_destroy(&dma->fragments->physical_pages); + kfree(dma->fragments); + dma->fragments = NULL; +} + +/* The output page has no offset field: validate every byte in its native span. */ +static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, + unsigned long *pfn) +{ + unsigned long first = iova - dma->iova, done = 0; + phys_addr_t phys = 0; + int ret; + + ret = vfio_fragments_acquire(dma, first, PAGE_SIZE); + if (ret) + return ret; + while (done < PAGE_SIZE) { + struct vfio_fragment *record = vfio_fragment_find(dma, first + done); + phys_addr_t next; + unsigned long offset; + + if (WARN_ON(!record || record->start > first + done)) { + ret = -EFAULT; + goto out; + } + offset = first + done - record->start; + next = vfio_fragment_phys(&record->pin) + offset; + if (!done) + phys = next; + if (!IS_ALIGNED(phys, PAGE_SIZE) || next != phys + done) { + ret = -EINVAL; + goto out; + } + done += min_t(unsigned long, record->pin.length - offset, PAGE_SIZE - done); + } + *pfn = PHYS_PFN(phys); + ret = vfio_add_to_pfn_list(dma, iova, *pfn); +out: + if (ret) + vfio_fragments_release_range(dma, first, PAGE_SIZE); + return ret; +} + +/* CPU-only missing ranges take GET references, not new long-term DMA pins. */ +static int vfio_fragments_rw(struct vfio_dma *dma, unsigned long offset, void *data, + unsigned long length, bool write, size_t *copied) +{ + bool live = false; + int ret = 0; + + while (length) { + struct vfio_fragment *record = vfio_fragment_find(dma, offset); + struct user_page_fragment temp = {}, *pin; + unsigned long in_record = 0, bytes = length; + void *addr; + + if (record && record->start <= offset) { + pin = &record->pin; + in_record = offset - record->start; + } else { + long count; + + if (record) + bytes = min(bytes, record->start - offset); + if (!live) { + if (!mmget_not_zero(dma->mm)) { + ret = -EFAULT; + break; + } + live = true; + } + count = get_user_fragments_remote(dma->mm, dma->vaddr + offset, bytes, + write ? FOLL_WRITE : 0, &temp, 1); + if (count <= 0) { + ret = count ?: -EFAULT; + break; + } + pin = &temp; + } + bytes = min_t(unsigned long, bytes, pin->length - in_record); + addr = kmap_local_folio(pin->folio, pin->offset + in_record); + if (write) + memcpy(addr, data, bytes); + else + memcpy(data, addr, bytes); + kunmap_local(addr); + if (write && record && record->start <= offset) + mark_user_fragment_dirty(pin); + release_user_fragments(&temp, 1, write); + data += bytes; + offset += bytes; + length -= bytes; + *copied += bytes; + } + if (live) + mmput(dma->mm); + return ret; +} +#else +static int vfio_fragments_init(struct vfio_dma *dma, size_t length) +{ + return 0; +} +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ +} +static void vfio_fragments_release_unused(struct vfio_dma *dma) +{ +} +static int vfio_fragments_acquire(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_map_domain(struct vfio_dma *dma, struct iommu_domain *domain) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_map_all(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + return -EOPNOTSUPP; +} +static void vfio_fragments_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ +} +static void vfio_fragments_destroy(struct vfio_dma *dma) +{ +} +static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, unsigned long *pfn) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_rw(struct vfio_dma *dma, unsigned long offset, void *data, + unsigned long length, bool write, size_t *copied) +{ + return -EOPNOTSUPP; +} +#endif diff --git a/drivers/vfio/vfio_iommu_type1_fragments_test.c b/drivers/vfio/vfio_iommu_type1_fragments_test.c new file mode 100644 index 0000000000000..7567c3802d2d5 --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_fragments_test.c @@ -0,0 +1,562 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Actual type1 lifecycle with recorded translations; no physical device DMA. */ +#include +#include +#include + +#define VFIO_TEST_VA SZ_64M +#define VFIO_TEST_IOVA SZ_256M + +struct vfio_test_domain { + struct iommu_domain core; + struct vfio_domain type1; + struct xarray translations; + unsigned long mapped; + unsigned long fail_after; + unsigned long largest_map; + unsigned int syncs; + struct folio *observed; + struct kunit *test; +}; + +struct vfio_fragment_fixture { + struct vfio_iommu *iommu; + struct vfio_test_domain domains[2]; + struct vfio_iommu_group group; + struct vfio_device device; + struct kunit *test; + unsigned int external_refs; + unsigned int notifications; +}; + +static int vfio_test_domain_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t phys, size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + unsigned long length = pgsize * pgcount; + + mock->largest_map = max(mock->largest_map, length); + for (unsigned long done = 0; done < length; done += SZ_4K) { + int ret; + + if (mock->fail_after && mock->mapped >= mock->fail_after) + return -ENOMEM; + ret = xa_insert(&mock->translations, (iova + done) >> 12, + xa_mk_value((phys + done) >> 12), gfp); + if (ret) + return ret; + mock->mapped += SZ_4K; + *mapped += SZ_4K; + } + return 0; +} + +static size_t vfio_test_domain_unmap(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + unsigned long length = pgsize * pgcount, done; + + for (done = 0; done < length; done += SZ_4K) { + if (!xa_erase(&mock->translations, (iova + done) >> 12)) + break; + mock->mapped -= SZ_4K; + } + return done; +} + +static phys_addr_t vfio_test_domain_translate(struct iommu_domain *domain, dma_addr_t iova) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + void *entry = xa_load(&mock->translations, iova >> 12); + + return entry ? ((phys_addr_t)xa_to_value(entry) << 12) + (iova & (SZ_4K - 1)) : 0; +} + +static void vfio_test_domain_sync(struct iommu_domain *domain, + struct iommu_iotlb_gather *gather) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + + mock->syncs++; + if (mock->observed) + KUNIT_EXPECT_TRUE(mock->test, folio_maybe_dma_pinned(mock->observed)); +} + +static const struct iommu_domain_ops vfio_test_domain_ops = { + .map_pages = vfio_test_domain_map, + .unmap_pages = vfio_test_domain_unmap, + .iova_to_phys = vfio_test_domain_translate, + .iotlb_sync = vfio_test_domain_sync, +}; + +static void vfio_test_notify(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_fragment_fixture *f = container_of(device, typeof(*f), device); + + f->notifications++; + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } +} + +static const struct vfio_device_ops vfio_test_device_ops = { + .dma_unmap = vfio_test_notify, +}; + +static void vfio_fragment_fixture_free(void *ptr) +{ + struct vfio_fragment_fixture *f = ptr; + struct rb_node *node; + + while ((node = rb_first(&f->iommu->dma_list))) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + struct rb_node *pin; + + while ((pin = rb_first(&dma->pfn_list))) { + struct vfio_pfn *vpfn = rb_entry(pin, struct vfio_pfn, node); + + vfio_unpin_page_external(dma, vpfn->iova, + list_empty(&f->iommu->domain_list)); + } + vfio_remove_dma(f->iommu, dma); + } + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + KUNIT_EXPECT_EQ(f->test, f->domains[i].mapped, 0UL); + KUNIT_EXPECT_TRUE(f->test, xa_empty(&f->domains[i].translations)); + xa_destroy(&f->domains[i].translations); + } + kfree(f->iommu); +} + +static struct vfio_fragment_fixture *vfio_fragment_fixture_new(struct kunit *test) +{ + struct vfio_fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + + if (!f) + return NULL; + f->test = test; + f->iommu = vfio_iommu_type1_open(VFIO_TYPE1v2_IOMMU); + if (IS_ERR(f->iommu)) + return NULL; + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + struct vfio_test_domain *mock = &f->domains[i]; + + xa_init(&mock->translations); + mock->test = test; + mock->core.ops = &vfio_test_domain_ops; + mock->core.type = IOMMU_DOMAIN_UNMANAGED; + mock->core.pgsize_bitmap = SZ_4K | PAGE_SIZE; + mock->type1.domain = &mock->core; + INIT_LIST_HEAD(&mock->type1.next); + INIT_LIST_HEAD(&mock->type1.group_list); + } + f->group.iommu_group = (struct iommu_group *)f; + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + f->iommu->num_non_pinned_groups = 1; + f->device.ops = &vfio_test_device_ops; + vfio_iommu_type1_register_device(f->iommu, &f->device); + if (kunit_add_action_or_reset(test, vfio_fragment_fixture_free, f)) + return NULL; + return f; +} + +static void vfio_test_file_put(void *ptr) +{ + fput(ptr); +} + +static struct file *vfio_fragment_file(struct kunit *test, unsigned long length, + unsigned char value) +{ + struct file *file = shmem_file_setup("vfio-fragments", length, EMPTY_VMA_FLAGS); + unsigned char *data; + loff_t pos = 0; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, vfio_test_file_put, file)) + return ERR_PTR(-ENOMEM); + data = kunit_kmalloc(test, length, GFP_KERNEL); + if (!data) + return ERR_PTR(-ENOMEM); + memset(data, value, length); + if (kernel_write(file, data, length, &pos) != length) + return ERR_PTR(-EIO); + return file; +} + +static unsigned long vfio_fragment_source(struct vfio_account_mm *ctx, + struct file *file, unsigned long length) +{ + unsigned long addr; + + kthread_use_mm(ctx->mm); + addr = vm_mmap(file, VFIO_TEST_VA, length, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(ctx->mm); + return addr; +} + +static int vfio_fragment_test_map(struct vfio_fragment_fixture *f, + struct vfio_account_mm *ctx, + unsigned long offset, unsigned long length) +{ + struct vfio_iommu_type1_dma_map map = { + .argsz = sizeof(map), + .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, + .vaddr = VFIO_TEST_VA + offset, + .iova = VFIO_TEST_IOVA, + .size = length, + }; + int ret; + + kthread_use_mm(ctx->mm); + ret = vfio_dma_do_map(f->iommu, &map); + kthread_unuse_mm(ctx->mm); + return ret; +} + +static void vfio_fragments_lifetime_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + unsigned char data[64]; + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, .size = 2 * PAGE_SIZE, + }; + struct vfio_bitmap bitmap = {}; + + if (shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x71); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + for (unsigned int d = 0; d < 2; d++) + f->domains[d].observed = vfio_fragment_find(dma, 0)->pin.folio; + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_GE(test, f->domains[0].largest_map, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 17, 1, IOMMU_READ | IOMMU_WRITE, &page), 1); + f->external_refs++; + KUNIT_EXPECT_EQ(test, page_to_phys(page), + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* The user VA goes away; retained pins are still the DMA buffer. */ + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x71, sizeof(data)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + KUNIT_EXPECT_EQ(test, f->domains[1].mapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA), + iommu_iova_to_phys(&f->domains[1].core, VFIO_TEST_IOVA)); + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, f->notifications, 1U); + KUNIT_EXPECT_GT(test, f->domains[0].syncs, 0U); + KUNIT_EXPECT_GT(test, f->domains[1].syncs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + kunit_info(test, "type1 retained %uK user/%luK native pins after mm exit and replay", + 1U << (shifts[i] - 10), PAGE_SIZE >> 10); + } +} + +static void vfio_fragments_lazy_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct file *file; + struct vfio_dma *dma; + struct page *page; + unsigned char data[64]; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x52); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x52, sizeof(data)), NULL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 37, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + PAGE_SIZE, 1, IOMMU_READ, &page), -EFAULT); + for (unsigned int i = 0; i < 3; i++) + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 37, 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->fragments->records)); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); +} + +static void vfio_fragments_map_failure_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct file *file; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x25); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + f->domains[1].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), -ENOMEM); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&f->iommu->dma_list)); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, f->domains[1].mapped, 0UL); + f->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); +} + +static void vfio_fragments_fine_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page = NULL; + unsigned char data[64]; + unsigned long length = PAGE_SIZE + SZ_4K; + size_t copied = 0; + + if (i && shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x64); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, SZ_4K, length), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, length); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* The physical base is offset inside a native page, not representable. */ + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), -EINVAL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + PAGE_SIZE, 1, IOMMU_READ, &page), -EINVAL); + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, + vfio_dirty_pgsize(f->iommu)), 0); + f->iommu->dirty_page_tracking = true; + memset(data, 0x97, sizeof(data)); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw_chunk(f->iommu, + VFIO_TEST_IOVA + SZ_4K - 17, data, sizeof(data), true, &copied), 0); + KUNIT_EXPECT_EQ(test, copied, sizeof(data)); + KUNIT_EXPECT_EQ(test, bitmap_weight(dma->bitmap, length / SZ_4K), 2U); + KUNIT_EXPECT_TRUE(test, test_bit(0, dma->bitmap)); + KUNIT_EXPECT_TRUE(test, test_bit(1, dma->bitmap)); + memset(data, 0, sizeof(data)); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, + VFIO_TEST_IOVA + SZ_4K - 17, data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x97, sizeof(data)), NULL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, + VFIO_TEST_IOVA + length, data, sizeof(data), false), -EINVAL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_info(test, "%uK userspace fine IOVA/physical offsets and dirty bytes checked", + 1U << (shifts[i] - 10)); + } +} + +static void vfio_fragments_replay_failure_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x58); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + f->domains[0].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), -ENOMEM); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_FALSE(test, dma->iommu_mapped); + f->domains[0].fail_after = 0; + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), 0); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* Detaching the last hardware domain keeps only the external page. */ + vfio_iommu_unmap_unpin_reaccount(f->iommu); + list_del_init(&f->domains[0].type1.next); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); +} + +static void vfio_fragments_anon_owner_test(struct kunit *test) +{ + struct vfio_account_mm *old = vfio_account_mm_new(test, 12); + struct vfio_account_mm *next = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + unsigned char *data = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL); + unsigned long addr, copied, charge; + unsigned char check[64]; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + KUNIT_ASSERT_NOT_NULL(test, data); + memset(data, 0x49, PAGE_SIZE); + kthread_use_mm(old->mm); + addr = vm_mmap(NULL, VFIO_TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + copied = addr == VFIO_TEST_VA ? + copy_to_user((void __user *)addr, data, PAGE_SIZE) : PAGE_SIZE; + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, copied, 0UL); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, old, 0, PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, vfio_fragment_find(dma, 0)->pin.subpage); + charge = dma->locked_vm; + KUNIT_ASSERT_GT(test, charge, 0UL); + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(old->mm), 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), charge); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + check, sizeof(check), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(check, 0x49, sizeof(check)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 0UL); + kunit_info(test, "anonymous 4K slot pins survive source exit and native-owner transfer"); +} + +static void vfio_fragments_domain_granule_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, PAGE_SIZE, 0x73); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, + vfio_dirty_pgsize(f->iommu)), 0); + f->iommu->dirty_page_tracking = true; + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 31, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, bitmap_weight(dma->bitmap, PAGE_SIZE / SZ_4K), + (unsigned int)(PAGE_SIZE / SZ_4K)); + f->domains[0].core.pgsize_bitmap = SZ_16K; + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), 0); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, 1UL << __ffs(f->iommu->pgsize_bitmap), SZ_16K); + KUNIT_EXPECT_EQ(test, vfio_dirty_pgsize(f->iommu), SZ_4K); + vfio_iommu_unmap_unpin_reaccount(f->iommu); + list_del_init(&f->domains[0].type1.next); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, 1UL << __ffs(f->iommu->pgsize_bitmap), SZ_4K); + KUNIT_EXPECT_EQ(test, vfio_dirty_pgsize(f->iommu), SZ_4K); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 31, 1); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); +} + +static struct kunit_case vfio_fragments_cases[] = { + KUNIT_CASE(vfio_fragments_lifetime_test), + KUNIT_CASE(vfio_fragments_lazy_test), + KUNIT_CASE(vfio_fragments_map_failure_test), + KUNIT_CASE(vfio_fragments_fine_test), + KUNIT_CASE(vfio_fragments_replay_failure_test), + KUNIT_CASE(vfio_fragments_anon_owner_test), + KUNIT_CASE(vfio_fragments_domain_granule_test), + {} +}; + +static struct kunit_suite vfio_fragments_suite = { + .name = "vfio-type1-fragments", + .test_cases = vfio_fragments_cases, +}; + +kunit_test_suite(vfio_fragments_suite); diff --git a/drivers/vfio/vfio_iommu_type1_ioctl_test.c b/drivers/vfio/vfio_iommu_type1_ioctl_test.c new file mode 100644 index 0000000000000..1e8635f0f1a65 --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_ioctl_test.c @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included after the KUnit harness; real EL0 arguments, recorded domains only. */ +#include +#include +#include +#include "vfio_iommu_type1_test.h" + +struct vfio_ioctl_fixture { + struct vfio_fragment_fixture pins; + struct mutex request_lock; +}; + +struct vfio_ioctl_rw { + struct work_struct work; + struct vfio_iommu *iommu; + dma_addr_t iova; + void *data; + size_t size; + bool write; + int ret; +}; + +static void vfio_ioctl_rw_work(struct work_struct *work) +{ + struct vfio_ioctl_rw *rw = container_of(work, typeof(*rw), work); + + rw->ret = vfio_iommu_type1_dma_rw(rw->iommu, rw->iova, rw->data, + rw->size, rw->write); +} + +static int vfio_ioctl_test_open(struct inode *inode, struct file *file) +{ + struct vfio_ioctl_fixture *ctx; + struct vfio_fragment_fixture *f; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + f = &ctx->pins; + f->iommu = vfio_iommu_type1_open(VFIO_TYPE1v2_IOMMU); + if (IS_ERR(f->iommu)) { + int ret = PTR_ERR(f->iommu); + + kfree(ctx); + return ret; + } + mutex_init(&ctx->request_lock); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + struct vfio_test_domain *mock = &f->domains[i]; + + xa_init(&mock->translations); + mock->core.ops = &vfio_test_domain_ops; + mock->core.type = IOMMU_DOMAIN_UNMANAGED; + mock->core.pgsize_bitmap = SZ_4K | PAGE_SIZE; + mock->type1.domain = &mock->core; + INIT_LIST_HEAD(&mock->type1.next); + INIT_LIST_HEAD(&mock->type1.group_list); + } + f->group.iommu_group = (struct iommu_group *)ctx; + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + f->iommu->num_non_pinned_groups = 1; + f->device.ops = &vfio_test_device_ops; + vfio_iommu_type1_register_device(f->iommu, &f->device); + file->private_data = ctx; + return 0; +} + +static int vfio_ioctl_test_release(struct inode *inode, struct file *file) +{ + struct vfio_ioctl_fixture *ctx = file->private_data; + struct vfio_fragment_fixture *f = &ctx->pins; + + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + vfio_iommu_type1_unregister_device(f->iommu, &f->device); + vfio_iommu_unmap_unpin_all(f->iommu); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + list_del_init(&f->domains[i].type1.next); + xa_destroy(&f->domains[i].translations); + } + list_del(&f->group.next); + vfio_iommu_type1_release(f->iommu); + kfree(ctx); + return 0; +} + +static int vfio_ioctl_test_domain(struct vfio_fragment_fixture *f, + struct vfio_type1_test_cmd *cmd, bool add) +{ + struct vfio_test_domain *mock; + struct rb_node *node; + int ret = 0; + + if (cmd->domain >= ARRAY_SIZE(f->domains) || cmd->flags) + return -EINVAL; + mock = &f->domains[cmd->domain]; + mutex_lock(&f->iommu->lock); + if (add) { + if (!list_empty(&mock->type1.next)) { + ret = -EEXIST; + goto out; + } + if (cmd->size != SZ_4K && cmd->size != SZ_16K && cmd->size != SZ_64K) { + ret = -EINVAL; + goto out; + } + mock->core.pgsize_bitmap = cmd->size; + mock->fail_after = cmd->fail_after; + ret = vfio_iommu_replay(f->iommu, &mock->type1); + if (ret) + goto out; + list_add_tail(&mock->type1.next, &f->iommu->domain_list); + } else { + if (list_empty(&mock->type1.next)) { + ret = -ENOENT; + goto out; + } + if (list_is_singular(&f->iommu->domain_list)) { + vfio_iommu_unmap_unpin_reaccount(f->iommu); + } else { + for (node = rb_first(&f->iommu->dma_list); node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + iommu_unmap(&mock->core, dma->iova, dma->size); + } + } + list_del_init(&mock->type1.next); + } + vfio_update_pgsize_bitmap(f->iommu); +out: + mutex_unlock(&f->iommu->lock); + return ret; +} + +static int vfio_ioctl_test_rw(struct vfio_fragment_fixture *f, + struct vfio_type1_test_cmd *cmd) +{ + struct vfio_ioctl_rw rw = { + .iommu = f->iommu, .iova = cmd->iova, .size = cmd->size, + .write = cmd->flags & VFIO_TYPE1_TEST_WRITE, + }; + void __user *buffer = u64_to_user_ptr(cmd->buffer); + int ret; + + if (!cmd->size || cmd->size > 256 || + cmd->flags & ~(VFIO_TYPE1_TEST_WRITE | VFIO_TYPE1_TEST_WORKER)) + return -EINVAL; + rw.data = kzalloc(cmd->size, GFP_KERNEL); + if (!rw.data) + return -ENOMEM; + if (rw.write && copy_from_user(rw.data, buffer, cmd->size)) { + ret = -EFAULT; + goto out; + } + if (cmd->flags & VFIO_TYPE1_TEST_WORKER) { + INIT_WORK(&rw.work, vfio_ioctl_rw_work); + queue_work(system_unbound_wq, &rw.work); + flush_work(&rw.work); + ret = rw.ret; + } else { + ret = vfio_iommu_type1_dma_rw(f->iommu, rw.iova, rw.data, rw.size, rw.write); + } + if (!ret && !rw.write && copy_to_user(buffer, rw.data, cmd->size)) + ret = -EFAULT; +out: + kfree(rw.data); + return ret; +} + +static long vfio_ioctl_test_ioctl(struct file *file, unsigned int request, + unsigned long arg) +{ + struct vfio_ioctl_fixture *ctx = file->private_data; + struct vfio_fragment_fixture *f = &ctx->pins; + struct vfio_type1_test_cmd cmd; + void __user *user = (void __user *)arg; + struct page *page; + int ret = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + /* Request serialization belongs to this fixture, not production type1. */ + mutex_lock(&ctx->request_lock); + switch (request) { + case VFIO_CHECK_EXTENSION: + case VFIO_IOMMU_GET_INFO: + case VFIO_IOMMU_MAP_DMA: + case VFIO_IOMMU_UNMAP_DMA: + case VFIO_IOMMU_DIRTY_PAGES: + ret = vfio_iommu_type1_ioctl(f->iommu, request, arg); + goto out; + default: + break; + } + if (copy_from_user(&cmd, user, sizeof(cmd))) { + ret = -EFAULT; + goto out; + } + if (cmd.reserved) { + ret = -EINVAL; + goto out; + } + switch (request) { + case VFIO_TYPE1_TEST_DOMAIN_ADD: + case VFIO_TYPE1_TEST_DOMAIN_DEL: + ret = vfio_ioctl_test_domain(f, &cmd, request == VFIO_TYPE1_TEST_DOMAIN_ADD); + break; + case VFIO_TYPE1_TEST_RW: + ret = vfio_ioctl_test_rw(f, &cmd); + break; + case VFIO_TYPE1_TEST_PIN: + case VFIO_TYPE1_TEST_UNPIN: + if (ALIGN_DOWN(cmd.iova, PAGE_SIZE) != VFIO_TEST_IOVA || cmd.flags) { + ret = -EINVAL; + break; + } + if (request == VFIO_TYPE1_TEST_PIN) { + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + cmd.iova, 1, IOMMU_READ | IOMMU_WRITE, &page); + if (ret == 1) { + f->external_refs++; + ret = 0; + } + } else if (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, cmd.iova, 1); + f->external_refs--; + } else { + ret = -ENOENT; + } + break; + case VFIO_TYPE1_TEST_STATE: + cmd.locked_bytes = 0; + cmd.owner_locked_bytes = 0; + cmd.owner_users = 0; + mutex_lock(&f->iommu->lock); + for (struct rb_node *node = rb_first(&f->iommu->dma_list); + node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + cmd.locked_bytes += dma->locked_vm * PAGE_SIZE; + cmd.owner_locked_bytes = mm_locked_vm_bytes(dma->mm); + cmd.owner_users = atomic_read(&dma->mm->mm_users); + } + cmd.native_page_size = PAGE_SIZE; + cmd.minimum_page_size = 1UL << __ffs(f->iommu->pgsize_bitmap); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) + cmd.mapped_bytes[i] = f->domains[i].mapped; + mutex_unlock(&f->iommu->lock); + cmd.notifications = f->notifications; + cmd.external_refs = f->external_refs; + break; + default: + ret = -ENOTTY; + } + if (!ret && copy_to_user(user, &cmd, sizeof(cmd))) + ret = -EFAULT; +out: + mutex_unlock(&ctx->request_lock); + return ret; +} + +static const struct file_operations vfio_ioctl_test_fops = { + .owner = THIS_MODULE, + .open = vfio_ioctl_test_open, + .release = vfio_ioctl_test_release, + .unlocked_ioctl = vfio_ioctl_test_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static struct miscdevice vfio_ioctl_test_device = { + .minor = MISC_DYNAMIC_MINOR, + .name = "aurora-vfio-type1-test", + .fops = &vfio_ioctl_test_fops, + .mode = 0600, +}; + +static int __init vfio_ioctl_test_init(void) +{ + return misc_register(&vfio_ioctl_test_device); +} +late_initcall(vfio_ioctl_test_init); diff --git a/drivers/vfio/vfio_iommu_type1_test.h b/drivers/vfio/vfio_iommu_type1_test.h new file mode 100644 index 0000000000000..07afd00c6be97 --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_test.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* Private disposable-guest test protocol, not a supported VFIO ABI. */ +#ifndef VFIO_IOMMU_TYPE1_TEST_H +#define VFIO_IOMMU_TYPE1_TEST_H +#include +#include + +struct vfio_type1_test_cmd { + __u64 iova; + __u64 size; + __u64 buffer; + __u64 fail_after; + __u64 locked_bytes; + __u64 owner_locked_bytes; + __u64 mapped_bytes[2]; + __u64 native_page_size; + __u64 minimum_page_size; + __u32 domain; + __u32 flags; + __u32 owner_users; + __u32 notifications; + __u32 external_refs; + __u32 reserved; +}; + +#define VFIO_TYPE1_TEST_DOMAIN_ADD _IOWR('V', 1, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_DOMAIN_DEL _IOWR('V', 2, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_RW _IOWR('V', 3, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_PIN _IOWR('V', 4, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_UNPIN _IOWR('V', 5, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_STATE _IOWR('V', 6, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_WRITE 1U +#define VFIO_TYPE1_TEST_WORKER 2U +#endif diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index f5b7fdcb51114..13344e3a92384 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -260,6 +260,8 @@ int pin_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, struct user_page_fragment *fragment, struct vm_area_struct **vma); /* Current mm, one aligned fragment; sparse zero pages return a negative errno. */ int get_dump_fragment(unsigned long addr, struct user_page_fragment *fragment); +/* Dirty a retained reference without releasing its allocation identity. */ +void mark_user_fragment_dirty(struct user_page_fragment *fragment); void release_user_fragments(struct user_page_fragment *fragments, unsigned long count, bool dirty); #endif diff --git a/mm/gup.c b/mm/gup.c index bb31fb08ca2d1..ec943d89d568e 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -3799,6 +3799,7 @@ long get_user_fragments_remote(struct mm_struct *mm, unsigned long start, return user_fragments_remote(mm, start, length, flags, fragments, capacity, false, NULL, false); } +EXPORT_SYMBOL_GPL(get_user_fragments_remote); int get_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, unsigned long length, unsigned int flags, @@ -3848,6 +3849,22 @@ int get_dump_fragment(unsigned long addr, struct user_page_fragment *fragment) } #endif +void mark_user_fragment_dirty(struct user_page_fragment *fragment) +{ + struct folio *folio = fragment->folio; + + if (!folio) + return; + if (!is_zero_folio(folio) && !folio_test_dirty(folio)) { + folio_lock(folio); + folio_mark_dirty(folio); + folio_unlock(folio); + } + if (fragment->subpage) + mm_subpage_cancel_lazyfree(fragment->subpage); +} +EXPORT_SYMBOL_GPL(mark_user_fragment_dirty); + void release_user_fragments(struct user_page_fragment *fragments, unsigned long count, bool dirty) { @@ -3859,14 +3876,9 @@ void release_user_fragments(struct user_page_fragment *fragments, if (!folio) continue; - if (dirty && !is_zero_folio(folio) && !folio_test_dirty(folio)) { - folio_lock(folio); - folio_mark_dirty(folio); - folio_unlock(folio); - } + if (dirty) + mark_user_fragment_dirty(fragment); if (fragment->subpage) { - if (dirty) - mm_subpage_cancel_lazyfree(fragment->subpage); if (fragment->pinned) mm_subpage_unpin(fragment->subpage); else diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 57f279e1990dc..fdcd0385f76a6 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -1,9 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 TEST_GEN_PROGS := page-contract TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init +TEST_GEN_PROGS_EXTENDED += vfio-fixture-init CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe -TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe +TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe TEST_FILES += compat-script runtime-app.py include ../../lib.mk @@ -102,3 +103,8 @@ $(OUTPUT)/coarse-shmem-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 $(OUTPUT)/iommufd-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 $(OUTPUT)/iommufd-init: LDLIBS += -pthread $(OUTPUT)/iommufd-init: CFLAGS += $(KHDR_INCLUDES) + +# Privileged recording-domain fixture; run only in a disposable test VM. +$(OUTPUT)/vfio-fixture-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-vfio-probe: compat-vfio-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ diff --git a/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c b/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c new file mode 100644 index 0000000000000..2e13b5f774975 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 type1 probe; no armhf sysroot or physical DMA. */ +#define __user +#include "../../../../../include/uapi/linux/vfio.h" +#include "../../../../../drivers/vfio/vfio_iommu_type1_test.h" + +typedef unsigned int u32; +typedef unsigned long long u64; +#define IOVA (256UL * 1024 * 1024) +static unsigned int stage; + +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" : "+r"(r0) : "r"(r1), "r"(r2), "r"(r3), + "r"(r4), "r"(r5), "r"(r7) : "memory", "cc"); + return r0; +} + +static u32 number(const char *s) +{ + u32 n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static void clear(void *data, u32 size) +{ + volatile unsigned char *p = data; + + for (u32 i = 0; i < size; i++) + p[i] = 0; +} + +/* Aggregate initialization may use these compiler-runtime entry points. */ +void __aeabi_memclr8(void *data, u32 size) { clear(data, size); } +void __aeabi_memclr4(void *data, u32 size) { clear(data, size); } +void __aeabi_memclr(void *data, u32 size) { clear(data, size); } + +_Noreturn static void finish(int code) +{ + if (code) { + char msg[] = "not ok - AArch32 type1 stage=00\n"; + + msg[29] = '0' + stage / 10; + msg[30] = '0' + stage % 10; + call(4, 1, (long)msg, sizeof(msg) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} +#define CHECK(x) do { stage++; if (!(x)) finish(stage); } while (0) + +static long ioctl_test(int fd, u32 request, void *arg) +{ + return call(54, fd, request, (long)arg, 0, 0, 0); +} + +static int state(int fd, struct vfio_type1_test_cmd *cmd) +{ + clear(cmd, sizeof(*cmd)); + return ioctl_test(fd, VFIO_TYPE1_TEST_STATE, cmd); +} + +static long map(int fd, u32 address, u32 size) +{ + struct vfio_iommu_type1_dma_map cmd; + + clear(&cmd, sizeof(cmd)); + cmd.argsz = sizeof(cmd); + cmd.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + cmd.iova = IOVA; + cmd.vaddr = address; + cmd.size = size; + return ioctl_test(fd, VFIO_IOMMU_MAP_DMA, &cmd); +} + +static long unmap(int fd, u32 size) +{ + struct vfio_iommu_type1_dma_unmap cmd; + long ret; + + clear(&cmd, sizeof(cmd)); + cmd.argsz = sizeof(cmd); + cmd.iova = IOVA; + cmd.size = size; + ret = ioctl_test(fd, VFIO_IOMMU_UNMAP_DMA, &cmd); + return ret ? ret : cmd.size == size ? 0 : -1; +} + +static long rw(int fd, unsigned char *bytes, unsigned int flags) +{ + struct vfio_type1_test_cmd cmd; + + clear(&cmd, sizeof(cmd)); + cmd.iova = IOVA; + cmd.buffer = (u32)bytes; + cmd.size = 16; + cmd.flags = flags; + return ioctl_test(fd, VFIO_TYPE1_TEST_RW, &cmd); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_dirty_bitmap dirty; + struct vfio_iommu_type1_info info; + unsigned char bytes[16]; + long fd, file, data, pid; + int status = -1; + u32 native, length; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + CHECK(argc == 3 && pages == number(argv[2])); + native = number(argv[1]); + length = native + 4096; + fd = call(5, (long)"/dev/vfio-test", 2, 0, 0, 0, 0); + file = call(385, (long)"compat-vfio", 0, 0, 0, 0, 0); + CHECK(fd >= 0 && file >= 0 && !call(93, file, 2 * native, 0, 0, 0, 0)); + data = call(192, 0, 2 * native, 3, 1, file, 0); + CHECK((u32)data < 0xfffff001U); + CHECK(call(125, data + 4096, 4096, 1, 0, 0, 0) == (pages == 4096 ? 0 : -22)); + CHECK(!call(125, data, 2 * native, 3, 0, 0, 0)); + for (u32 i = 0; i < 2 * native; i++) + ((volatile unsigned char *)data)[i] = 0x53; + clear(&info, sizeof(info)); + info.argsz = sizeof(info); + CHECK(!ioctl_test(fd, VFIO_IOMMU_GET_INFO, &info) && (info.iova_pgsizes & 4096)); + CHECK(!map(fd, data, 2 * native)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes); + CHECK(!rw(fd, bytes, 0) && bytes[0] == 0x53 && bytes[15] == 0x53); + clear(&dirty, sizeof(dirty)); + dirty.argsz = sizeof(dirty); + dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_START; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, &dirty)); + clear(&cmd, sizeof(cmd)); + cmd.iova = IOVA + 19; + CHECK(!ioctl_test(fd, VFIO_TYPE1_TEST_PIN, &cmd)); + CHECK(!state(fd, &cmd) && cmd.locked_bytes == native && cmd.external_refs == 1); + /* The compat pointer's padding is zeroed for the native64 wire layout. */ + unsigned char buffer[sizeof(dirty) + sizeof(struct vfio_iommu_type1_dirty_bitmap_get)]; + struct vfio_iommu_type1_dirty_bitmap *header = (void *)buffer; + struct vfio_iommu_type1_dirty_bitmap_get *range = (void *)header->data; + u64 bits = 0; + + clear(buffer, sizeof(buffer)); + header->argsz = sizeof(buffer); + header->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP; + range->iova = IOVA; + range->size = 2 * native; + range->bitmap.pgsize = 4096; + range->bitmap.size = sizeof(bits); + range->bitmap.data = &bits; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, buffer) && + bits == ((1ULL << (native >> 12)) - 1)); + CHECK(!unmap(fd, 2 * native)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.external_refs && + cmd.notifications == 1); + clear(&cmd, sizeof(cmd)); + cmd.size = 4096; + CHECK(!ioctl_test(fd, VFIO_TYPE1_TEST_DOMAIN_ADD, &cmd)); + pid = call(2, 0, 0, 0, 0, 0, 0); + if (!pid) + finish(map(fd, data + 4096, length) ? 99 : 0); + CHECK(pid > 0 && call(114, pid, (long)&status, 0, 0, 0, 0) == pid && !status); + CHECK(!state(fd, &cmd) && !cmd.owner_users && cmd.locked_bytes == 2 * native && + cmd.mapped_bytes[0] == length); + CHECK(rw(fd, bytes, 0) == -14); + CHECK(!rw(fd, bytes, VFIO_TYPE1_TEST_WORKER) && bytes[0] == 0x53 && bytes[15] == 0x53); + CHECK(!unmap(fd, length)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0]); + CHECK(!call(91, data, 2 * native, 0, 0, 0, 0)); + CHECK(!call(6, file, 0, 0, 0, 0, 0) && !call(6, fd, 0, 0, 0, 0, 0)); + static const char ok[] = + "ok - AArch32 type1 real ioctls, source exit and fallback alignment\n"; + + call(4, 1, (long)ok, sizeof(ok) - 1, 0, 0, 0); + finish(0); +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c b/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c new file mode 100644 index 0000000000000..cefc2f35c6fef --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#define __user +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/vfio.h" +#include "../../../../../drivers/vfio/vfio_iommu_type1_test.h" + +#define IOVA (256UL * 1024 * 1024) +static unsigned int number, failures; + +static void result(int ok, const char *name) +{ + printf("%s %u - %s (errno=%d)\n", ok ? "ok" : "not ok", ++number, name, errno); + failures += !ok; +} + +static int state(int fd, struct vfio_type1_test_cmd *cmd) +{ + memset(cmd, 0, sizeof(*cmd)); + return ioctl(fd, VFIO_TYPE1_TEST_STATE, cmd); +} + +static int domain(int fd, unsigned int which, int add, size_t page, size_t fail) +{ + struct vfio_type1_test_cmd cmd = { + .domain = which, .size = page, .fail_after = fail, + }; + + return ioctl(fd, add ? VFIO_TYPE1_TEST_DOMAIN_ADD : VFIO_TYPE1_TEST_DOMAIN_DEL, &cmd); +} + +static int rw(int fd, size_t offset, void *data, size_t size, unsigned int flags) +{ + struct vfio_type1_test_cmd cmd = { + .iova = IOVA + offset, .size = size, .buffer = (uintptr_t)data, .flags = flags, + }; + + return ioctl(fd, VFIO_TYPE1_TEST_RW, &cmd); +} + +static int map(int fd, void *data, size_t size) +{ + struct vfio_iommu_type1_dma_map cmd = { + .argsz = sizeof(cmd), .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, + .iova = IOVA, .vaddr = (uintptr_t)data, .size = size, + }; + + return ioctl(fd, VFIO_IOMMU_MAP_DMA, &cmd); +} + +static int unmap(int fd, size_t size) +{ + struct vfio_iommu_type1_dma_unmap cmd = { + .argsz = sizeof(cmd), .iova = IOVA, .size = size, + }; + int ret = ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &cmd); + + return ret ? ret : cmd.size == size ? 0 : -2; +} + +static int dirty(int fd, unsigned int flags) +{ + struct vfio_iommu_type1_dirty_bitmap cmd = { .argsz = sizeof(cmd), .flags = flags }; + + return ioctl(fd, VFIO_IOMMU_DIRTY_PAGES, &cmd); +} + +static int bitmap(int fd, size_t size, uint64_t *bits) +{ + unsigned char buffer[sizeof(struct vfio_iommu_type1_dirty_bitmap) + + sizeof(struct vfio_iommu_type1_dirty_bitmap_get)] = {}; + struct vfio_iommu_type1_dirty_bitmap *cmd = (void *)buffer; + struct vfio_iommu_type1_dirty_bitmap_get *range = (void *)cmd->data; + + cmd->argsz = sizeof(buffer); + cmd->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP; + range->iova = IOVA; + range->size = size; + range->bitmap.pgsize = 4096; + range->bitmap.size = sizeof(*bits); + range->bitmap.data = (void *)bits; + *bits = 0; + return ioctl(fd, VFIO_IOMMU_DIRTY_PAGES, buffer); +} + +static int all_bytes(const unsigned char *data, size_t size, unsigned char value) +{ + for (size_t i = 0; i < size; i++) + if (data[i] != value) + return 0; + return 1; +} + +static int exercise(size_t native) +{ + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_info info = { .argsz = sizeof(info) }; + struct vfio_iommu_type1_dma_map bad = { .argsz = 8 }; + unsigned char bytes[64], disk[64]; + uint64_t bits = 0; + int fd = open("/dev/vfio-test", O_RDWR), backing = memfd_create("vfio", MFD_CLOEXEC); + unsigned char *data = MAP_FAILED; + int status = 0; + pid_t child; + size_t length = native + 4096; + + printf("TAP version 13\n# real type1 ioctl ABI=%luK native=%zuK\n", + getauxval(AT_PAGESZ) / 1024, native / 1024); + result(fd >= 0 && backing >= 0 && !ftruncate(backing, 2 * native), + "open privileged recording-domain fixture and backing"); + if (fd < 0 || backing < 0 || failures) + goto out; + data = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, MAP_SHARED, backing, 0); + result(data != MAP_FAILED, "map native-sized shmem backing in actual process mm"); + if (data == MAP_FAILED) + goto out; + memset(data, 0x64, 2 * native); + result(!ioctl(fd, VFIO_IOMMU_GET_INFO, &info) && (info.iova_pgsizes & 4096), + "type1 GET_INFO reports fine IOVA capability"); + errno = 0; + result(ioctl(fd, VFIO_IOMMU_MAP_DMA, &bad) < 0 && errno == EINVAL, + "canonical MAP rejects truncated argument"); + errno = 0; + result(ioctl(fd, VFIO_IOMMU_MAP_DMA, (void *)1) < 0 && errno == EFAULT, + "canonical MAP rejects inaccessible userspace argument"); + result(!map(fd, data, 2 * native), "lazy full-range MAP from real EL0"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0], + "lazy MAP acquires no DMA pins or translations"); + result(!rw(fd, 0, bytes, sizeof(bytes), 0) && all_bytes(bytes, sizeof(bytes), 0x64), + "CPU-only GET reads actual source bytes"); + result(!state(fd, &cmd) && !cmd.locked_bytes, "CPU GET does not charge long-term pins"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), "canonical dirty tracking starts"); + cmd = (struct vfio_type1_test_cmd) { .iova = IOVA + 17 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == native && cmd.external_refs == 1, + "external page array preserves unaligned caller byte offset"); + cmd = (struct vfio_type1_test_cmd) { .iova = IOVA + 31 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == native && cmd.external_refs == 2, + "duplicate external pin charges physical native page once"); + result(!bitmap(fd, 2 * native, &bits) && bits == ((1ULL << (native / 4096)) - 1), + "dirty bitmap covers every fine unit in native external page"); + result(!unmap(fd, 2 * native), "canonical UNMAP drains callback-held external refs"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.external_refs && + cmd.notifications == 1, "external-unpin callback refunds all pins"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP), "dirty tracking stops"); + result(!domain(fd, 0, 1, 4096, 0), "recording4K domain attaches before child MAP"); + child = fork(); + if (!child) { + unsigned char *source = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, + MAP_SHARED, backing, 0); + int ret = source == MAP_FAILED ? -1 : map(fd, source + 4096, length); + + _exit(ret ? 1 : 0); + } + result(child > 0 && waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status), "fine-offset MAP retains real child pins after exit"); + result(!state(fd, &cmd) && !cmd.owner_users && cmd.locked_bytes == 2 * native && + cmd.owner_locked_bytes == 2 * native && cmd.mapped_bytes[0] == length, + "exited source mm retains exact physical-page charge and translation length"); + errno = 0; + result(rw(fd, 0, bytes, sizeof(bytes), 0) < 0 && errno == EFAULT, + "foreign process CPU access preserves source-mm authorization"); + result(!rw(fd, 0, bytes, sizeof(bytes), VFIO_TYPE1_TEST_WORKER) && + all_bytes(bytes, sizeof(bytes), 0x64), "kernel consumer reads exited source pins"); + errno = 0; + result(domain(fd, 1, 1, 4096, 4096) < 0 && errno == ENOMEM && + !state(fd, &cmd) && !cmd.mapped_bytes[1] && cmd.mapped_bytes[0] == length && + cmd.locked_bytes == 2 * native, + "failed replay leaves existing domain and pins intact"); + result(!domain(fd, 1, 1, 4096, 0) && !state(fd, &cmd) && + cmd.mapped_bytes[0] == length && cmd.mapped_bytes[1] == length && + cmd.locked_bytes == 2 * native, + "retry replays cached pins without resurrecting source mm"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), "fine DMA dirty tracking starts"); + memset(bytes, 0x9a, sizeof(bytes)); + result(!rw(fd, 4096 - 32, bytes, sizeof(bytes), + VFIO_TYPE1_TEST_WRITE | VFIO_TYPE1_TEST_WORKER) && + pread(backing, disk, sizeof(disk), 8192 - 32) == sizeof(disk) && + all_bytes(disk, sizeof(disk), 0x9a), + "cached worker write dirties original shmem bytes"); + result(!bitmap(fd, length, &bits) && bits == 3, + "canonical bitmap returns exact two4K units touched by worker write"); + result(!unmap(fd, length), "fine UNMAP synchronizes both recording domains"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0] && + !cmd.mapped_bytes[1], + "source-exit cleanup refunds pins and clears all translations"); + result(!domain(fd, 1, 0, 0, 0) && !domain(fd, 0, 0, 0, 0), + "both domains detach after complete cleanup"); +out: + if (data != MAP_FAILED) + munmap(data, 2 * native); + if (backing >= 0) + close(backing); + if (fd >= 0) + close(fd); + printf("1..%u\n# type1 ioctl failures=%u\n", number, failures); + return !!failures; +} + +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + size_t expected = strtoul(argv[3], NULL, 10); + + if (getauxval(AT_PAGESZ) != expected) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) + return 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + FILE *dev = fopen("/sys/class/misc/aurora-vfio-type1-test/dev", "r"); + unsigned int major, minor; + + if (!dev || fscanf(dev, "%u:%u", &major, &minor) != 2) + return 4; + fclose(dev); + if (mknod("/dev/vfio-test", S_IFCHR | 0600, makedev(major, minor))) + return 5; + size_t native = getauxval(AT_PAGESZ), targets[] = { native, 4096, 16384 }; + int ok = 1; + + int compat_only = getenv("VFIO_COMPAT_ONLY") != NULL; + + for (unsigned int i = 0; i < 3; i++) { + if (compat_only) + break; + size_t target = targets[i]; + pid_t child; + int status = 0; + + if (target > native || (i && target == native)) + continue; + child = fork(); + if (!child) { + char native_arg[24], target_arg[24]; + + snprintf(native_arg, sizeof(native_arg), "%zu", native); + snprintf(target_arg, sizeof(target_arg), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target == native ? 0UL : target, + 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", native_arg, target_arg, NULL); + _exit(4); + } + ok &= child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("# actual ABI=%zuK child_status=%d\n", target / 1024, status); + } + const char *expected_arg = getenv("VFIO_COMPAT_PAGES"); + size_t expected = expected_arg ? strtoul(expected_arg, NULL, 10) : 4096; + pid_t compat = fork(); + int compat_status = 0; + + if (!compat) { + char native_arg[24], page_arg[24]; + + snprintf(native_arg, sizeof(native_arg), "%zu", native); + snprintf(page_arg, sizeof(page_arg), "%zu", expected); + execl("/compat-vfio", "/compat-vfio", native_arg, page_arg, NULL); + _exit(98); + } + ok &= compat > 0 && waitpid(compat, &compat_status, 0) == compat && + WIFEXITED(compat_status) && !WEXITSTATUS(compat_status); + printf("# AArch32 expected=%zuK child_status=%d\n", expected / 1024, compat_status); + printf("VFIO TYPE1 IOCTL %s\n", ok ? "PASS" : "FAIL"); + reboot(RB_POWER_OFF); + return !ok; +} From e2c698ca7bf1084897e6a3388702661c88986d84 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 0364/1579] dt-bindings: apple: describe J813 monitoring and input peripherals Describe the T8142 mailbox, SMC, MTP, DART and PWM resources. Add explicit contracts for runtime temperature discovery and firmware-owned trackpad reset without embedding sensor tables or calibration. Signed-off-by: Acelogic --- .../bindings/arm/apple/apple,pmgr.yaml | 1 + .../bindings/hwmon/apple,smc-hwmon.yaml | 20 ++++++++++ .../bindings/input/apple,dockchannel-hid.yaml | 38 ++++++++++++++++++- .../devicetree/bindings/iommu/apple,dart.yaml | 1 + .../bindings/mailbox/apple,mailbox.yaml | 1 + .../devicetree/bindings/mfd/apple,smc.yaml | 1 + .../bindings/power/apple,pmgr-pwrstate.yaml | 1 + .../bindings/pwm/apple,s5l-fpwm.yaml | 1 + .../bindings/soc/apple/apple,dockchannel.yaml | 1 + .../soc/apple/apple,rtk-helper-asc4.yaml | 2 + 10 files changed, 65 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml index 77d87d8c376e8..f10b9cbfce597 100644 --- a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml +++ b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml @@ -43,6 +43,7 @@ properties: - apple,t8122-pmgr - apple,t8132-pmgr - apple,t8140-pmgr + - apple,t8142-pmgr - const: apple,t8103-pmgr - const: syscon - const: simple-mfd diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index f3751b5af1720..4d772bf4b34ee 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -40,6 +40,16 @@ properties: compatible: const: apple,smc-hwmon + apple,discover-temperatures: + type: boolean + description: + Enumerate readable, non-function temperature keys from the running SMC + instead of providing a board-specific temperature table. Only keys + beginning with T and reporting a four-byte floating-point value are + exposed. Labels are the key identifiers, without inferred locations. + Channels are reporting-only; their indices depend on the firmware and + must not be used by a static thermal policy. + apple,cpu-thermal-policy: type: boolean description: @@ -91,4 +101,14 @@ patternProperties: $ref: "#/$defs/sensor" unevaluatedProperties: false +allOf: + - if: + required: + - apple,discover-temperatures + then: + properties: + apple,cpu-thermal-policy: false + patternProperties: + "^temperature-": false + additionalProperties: false diff --git a/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml b/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml index 3b82308e0b568..924442134b248 100644 --- a/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml +++ b/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml @@ -23,7 +23,9 @@ description: | trackpad itself and asks the host to pulse a reset line when it needs one. With power method 2 the host announces each power transition to the coprocessor, drives the reset line of the trackpad analog front end (AFE) - itself and then confirms the transition. + itself and then confirms the transition. On systems with firmware-managed + reset, the same notifications bracket a firmware-owned transition and the + host waits 50 ms during power-on without operating a GPIO. properties: compatible: @@ -140,7 +142,27 @@ properties: default: 1 description: The power sequencing protocol of the coprocessor firmware, as - described above. Power method 2 requires apple,afe-reset-gpios. + described above. Power method 2 requires apple,afe-reset-gpios + unless apple,firmware-managed-reset is present. + + apple,firmware-managed-reset: + type: boolean + description: + The coprocessor owns the trackpad reset signal. The host sends + power method 2 notifications without controlling an AFE GPIO. + Mutually exclusive with apple,afe-reset-gpios on the parent. + + allOf: + - if: + required: + - apple,firmware-managed-reset + then: + properties: + apple,power-method: + const: 2 + required: + - apple,power-method + - firmware-name stm: type: object @@ -171,6 +193,18 @@ required: additionalProperties: false +allOf: + - if: + properties: + multi-touch: + required: + - apple,firmware-managed-reset + required: + - multi-touch + then: + properties: + apple,afe-reset-gpios: false + examples: - | #include diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index bd6524c97087f..843ccb6fdddc4 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -38,6 +38,7 @@ properties: - apple,t6020-dart - apple,t8122-dart - apple,t8140-dart + - apple,t8142-dart - const: apple,t8110-dart reg: diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index ee40e13c9a2b9..dc4cbb5168f20 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -32,6 +32,7 @@ properties: - apple,t8112-asc-mailbox - apple,t8122-asc-mailbox - apple,t8140-asc-mailbox + - apple,t8142-asc-mailbox - apple,t6000-asc-mailbox - apple,t6020-asc-mailbox - const: apple,asc-mailbox-v4 diff --git a/Documentation/devicetree/bindings/mfd/apple,smc.yaml b/Documentation/devicetree/bindings/mfd/apple,smc.yaml index ed6b02086c33c..31d1940f9d3dc 100644 --- a/Documentation/devicetree/bindings/mfd/apple,smc.yaml +++ b/Documentation/devicetree/bindings/mfd/apple,smc.yaml @@ -21,6 +21,7 @@ properties: - apple,t6020-smc - apple,t8122-smc - apple,t8140-smc + - apple,t8142-smc - const: apple,t8103-smc - items: - enum: diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index 0e9e1fbcd3b10..d131441214db1 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -50,6 +50,7 @@ properties: - apple,t8122-pmgr-pwrstate - apple,t8132-pmgr-pwrstate - apple,t8140-pmgr-pwrstate + - apple,t8142-pmgr-pwrstate - const: apple,t8103-pmgr-pwrstate reg: diff --git a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml index ae8baa5f7bcdc..3b3cad92b601f 100644 --- a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml +++ b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml @@ -19,6 +19,7 @@ properties: - apple,t8103-fpwm - apple,t8112-fpwm - apple,t8122-fpwm + - apple,t8142-fpwm - apple,t8132-fpwm - apple,t6000-fpwm - apple,t6020-fpwm diff --git a/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml b/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml index 40831b082bf28..a3362df34e148 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml @@ -28,6 +28,7 @@ properties: - apple,t8112-dockchannel - apple,t8122-dockchannel - apple,t8140-dockchannel + - apple,t8142-dockchannel - const: apple,dockchannel reg: diff --git a/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml b/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml index 6db0197b4e864..da78694e37466 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml @@ -28,12 +28,14 @@ properties: - apple,t8112-mtp - apple,t8122-mtp - apple,t8140-mtp + - apple,t8142-mtp - enum: - apple,t6020-rtk-helper-asc4 - apple,t6030-rtk-helper-asc4 - apple,t8112-rtk-helper-asc4 - apple,t8122-rtk-helper-asc4 - apple,t8140-rtk-helper-asc4 + - apple,t8142-rtk-helper-asc4 - const: apple,mtp - const: apple,rtk-helper-asc4 From ac796265ebc04b0185a0510695a568342e389304 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 0365/1579] hwmon: macsmc: discover firmware temperature keys on opt-in boards Enumerate readable float temperature keys from the running SMC and label them by key identity. Keep dynamically assigned channels separate from static thermal policy, and leave existing board sensor definitions unchanged. Signed-off-by: Acelogic --- drivers/hwmon/macsmc-hwmon.c | 64 ++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index b41f8813d4165..82ec77555f081 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -527,6 +527,66 @@ static int macsmc_hwmon_create_sensor(struct device *dev, struct apple_smc *smc, return 0; } +/* Discover reporting-only temperatures without a board-specific key table. */ +static int macsmc_hwmon_discover_temperatures(struct macsmc_hwmon *hwmon) +{ + struct apple_smc *smc = hwmon->smc; + struct macsmc_hwmon_sensor *sensor; + struct apple_smc_key_info info; + smc_key key, previous = 0; + u32 i; + int ret; + + if (!of_property_read_bool(hwmon->dev->of_node, "apple,discover-temperatures")) + return 0; + + /* Dynamic channel numbers must not replace a DT-described thermal policy. */ + if (hwmon->temp.count || + of_property_read_bool(hwmon->dev->of_node, "apple,cpu-thermal-policy")) + return -EINVAL; + + /* Bound both allocation and mailbox transactions for invalid firmware data. */ + if (!smc->key_count || smc->key_count > U16_MAX) + return -EINVAL; + + hwmon->temp.sensors = devm_kcalloc(hwmon->dev, smc->key_count, + sizeof(*sensor), GFP_KERNEL); + if (!hwmon->temp.sensors) + return -ENOMEM; + + for (i = 0; i < smc->key_count; i++) { + ret = apple_smc_get_key_by_index(smc, i, &key); + if (ret < 0) + return ret; + /* SMC key tables are strictly ordered; reject duplicates as well. */ + if (i && key <= previous) + return -EINVAL; + previous = key; + if ((key >> 24) < 'T') + continue; + if ((key >> 24) > 'T') + break; + + ret = apple_smc_get_key_info(smc, key, &info); + if (ret < 0) + return ret; + if (!(info.flags & APPLE_SMC_READABLE) || + (info.flags & APPLE_SMC_FUNCTION) || + info.type_code != __SMC_KEY('f', 'l', 't', ' ') || info.size != 4) + continue; + + sensor = &hwmon->temp.sensors[hwmon->temp.count++]; + sensor->macsmc_key = key; + sensor->info = info; + sensor->attrs = HWMON_T_INPUT | HWMON_T_LABEL; + /* Key identity only: do not invent locations or calibration. */ + snprintf(sensor->label, sizeof(sensor->label), "%c%c%c%c", + key >> 24, (key >> 16) & 0xff, (key >> 8) & 0xff, key & 0xff); + } + + return 0; +} + /* * Fan data is exposed by the SMC as multiple sensors. * @@ -900,6 +960,10 @@ static int macsmc_hwmon_probe(struct platform_device *pdev) return ret; } + ret = macsmc_hwmon_discover_temperatures(hwmon); + if (ret) + return dev_err_probe(hwmon->dev, ret, "Temperature discovery failed\n"); + if (!hwmon->curr.count && !hwmon->fan.count && !hwmon->power.count && !hwmon->temp.count && !hwmon->volt.count) { From e179042ff0673c4b5adc749cb0fc3fcb0f909106 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 0366/1579] HID: dockchannel: support firmware-managed M5 trackpad power Support power method 2 when the MTP firmware owns reset, reject conflicting GPIO ownership, and announce AP readiness before the T8142 MTP power acknowledgment. Preserve the existing sequence for other coprocessors and host-controlled trackpads. Signed-off-by: Acelogic --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 41 +++++++++++++++---- drivers/soc/apple/rtkit-helper.c | 5 ++- drivers/soc/apple/rtkit.c | 32 +++++++++++++-- include/linux/soc/apple/rtkit.h | 3 ++ 4 files changed, 68 insertions(+), 13 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 32e54117bd569..e50a0806ca26e 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -177,6 +177,7 @@ struct dchid_iface { u32 keyboard_layout_id; u32 power_method; + bool firmware_managed_reset; }; struct dockchannel_hid { @@ -217,6 +218,7 @@ static struct dchid_iface * dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) { u32 power_method = DCHID_POWER_METHOD_1; + bool firmware_managed_reset = false; struct device_node *of_node = NULL; struct dchid_iface *iface; @@ -240,13 +242,19 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) } of_property_read_u32(of_node, "apple,power-method", &power_method); + firmware_managed_reset = + of_property_read_bool(of_node, "apple,firmware-managed-reset"); + if (firmware_managed_reset && + (strcmp(name, "multi-touch") || power_method != DCHID_POWER_METHOD_2)) + goto err_put_node; if (power_method != DCHID_POWER_METHOD_1 && power_method != DCHID_POWER_METHOD_2) { dev_err(dchid->dev, "Unsupported power method %u for %s\n", power_method, name); goto err_put_node; } - if (power_method == DCHID_POWER_METHOD_2 && !dchid->afe_reset) { + if (power_method == DCHID_POWER_METHOD_2 && + !firmware_managed_reset && !dchid->afe_reset) { dev_err(dchid->dev, "Power method 2 for %s needs the AFE reset line\n", name); goto err_put_node; @@ -264,6 +272,7 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) iface->dchid = dchid; iface->of_node = of_node; iface->power_method = power_method; + iface->firmware_managed_reset = firmware_managed_reset; iface->out_report = -1; init_completion(&iface->out_complete); init_completion(&iface->ready); @@ -425,12 +434,18 @@ static int dchid_pm2_set_power(struct dchid_iface *iface, u8 state) if (ret < 0) return ret; - ret = gpiod_set_value_cansleep(dchid->afe_reset, - state == DCHID_POWER_STATE_OFF); - if (ret < 0) - return ret; - if (state == DCHID_POWER_STATE_OFF) - fsleep(50 * USEC_PER_MSEC); + if (iface->firmware_managed_reset) { + /* Firmware owns the reset; allow its power-on transition to settle. */ + if (state == DCHID_POWER_STATE_ON) + fsleep(50 * USEC_PER_MSEC); + } else { + ret = gpiod_set_value_cansleep(dchid->afe_reset, + state == DCHID_POWER_STATE_OFF); + if (ret < 0) + return ret; + if (state == DCHID_POWER_STATE_OFF) + fsleep(50 * USEC_PER_MSEC); + } dchid_pm2_command(iface->index, state, true, msg); return dchid_comm_cmd(dchid, msg, sizeof(msg)); @@ -527,6 +542,9 @@ static int dchid_request_gpio(struct dchid_iface *iface) char prop_name[MAX_GPIO_NAME + 16]; int ret; + if (iface->firmware_managed_reset) + return -EINVAL; + if (iface->gpio) return 0; @@ -1308,10 +1326,17 @@ static int dockchannel_hid_probe(struct platform_device *pdev) child = of_get_child_by_name(dev->of_node, "multi-touch"); if (child) { u32 power_method = DCHID_POWER_METHOD_1; + bool firmware_managed_reset; of_property_read_u32(child, "apple,power-method", &power_method); + firmware_managed_reset = + of_property_read_bool(child, "apple,firmware-managed-reset"); of_node_put(child); - if (power_method == DCHID_POWER_METHOD_2) { + if (firmware_managed_reset && + (power_method != DCHID_POWER_METHOD_2 || + of_property_present(dev->of_node, "apple,afe-reset-gpios"))) + return dev_err_probe(dev, -EINVAL, "Conflicting touch reset ownership\n"); + if (power_method == DCHID_POWER_METHOD_2 && !firmware_managed_reset) { dchid->afe_reset = devm_gpiod_get(dev, "apple,afe-reset", GPIOD_ASIS); if (IS_ERR(dchid->afe_reset)) diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index 080d083ed9bd2..89e5642444367 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -113,7 +113,10 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) helper->asc_base + APPLE_ASC_CPU_CONTROL); /* Works for both wake and boot */ - ret = apple_rtkit_wake(helper->rtk); + if (of_device_is_compatible(dev->of_node, "apple,t8142-mtp")) + ret = apple_rtkit_wake_early_ap(helper->rtk); + else + ret = apple_rtkit_wake(helper->rtk); if (ret != 0) return dev_err_probe(dev, ret, "Failed to wake up coprocessor"); diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 1bef9e3e8ef17..0797bfbd3a024 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -994,7 +994,7 @@ static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, return 0; } -int apple_rtkit_boot(struct apple_rtkit *rtk) +static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) { int ret; @@ -1010,12 +1010,25 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) if (rtk->boot_result) return rtk->boot_result; + /* Some helpers need AP readiness before acknowledging IOP power. */ + if (early_ap_power) { + ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); + if (ret) + return ret; + } + dev_dbg(rtk->dev, "RTKit: waiting for IOP power state ACK\n"); ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion); if (ret) return ret; - return apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); + return early_ap_power ? 0 : + apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); +} + +int apple_rtkit_boot(struct apple_rtkit *rtk) +{ + return apple_rtkit_boot_common(rtk, false); } EXPORT_SYMBOL_GPL(apple_rtkit_boot); @@ -1107,7 +1120,7 @@ int apple_rtkit_quiesce(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_quiesce); -int apple_rtkit_wake(struct apple_rtkit *rtk) +static int apple_rtkit_wake_common(struct apple_rtkit *rtk, bool early_ap_power) { u64 msg; int ret; @@ -1127,10 +1140,21 @@ int apple_rtkit_wake(struct apple_rtkit *rtk) if (ret) return ret; - return apple_rtkit_boot(rtk); + return apple_rtkit_boot_common(rtk, early_ap_power); +} + +int apple_rtkit_wake(struct apple_rtkit *rtk) +{ + return apple_rtkit_wake_common(rtk, false); } EXPORT_SYMBOL_GPL(apple_rtkit_wake); +int apple_rtkit_wake_early_ap(struct apple_rtkit *rtk) +{ + return apple_rtkit_wake_common(rtk, true); +} +EXPORT_SYMBOL_GPL(apple_rtkit_wake_early_ap); + void apple_rtkit_free(struct apple_rtkit *rtk) { apple_rtkit_stop_rx(rtk); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 0aed8f9904c56..d55551a975b54 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -165,6 +165,9 @@ int apple_rtkit_quiesce(struct apple_rtkit *rtk); */ int apple_rtkit_wake(struct apple_rtkit *rtk); +/* Wake helpers which require AP readiness before their IOP power acknowledgment. */ +int apple_rtkit_wake_early_ap(struct apple_rtkit *rtk); + /* * Shutdown the co-processor */ From 14334fd7675b9614deb45828162df9523bbad77b Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 0367/1579] arm64: apple: enable J813 temperatures, input and keyboard backlight Describe the MTP, SMC and PWM resources needed by the existing generic drivers. Request trackpad firmware externally, obtain keyboard layout through the loader, and use runtime sensor discovery with a linear LED brightness scale. Signed-off-by: Acelogic --- .../dts/apple/t8142-j813-peripherals.dtsi | 137 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 2 + arch/arm64/configs/j813.config | 26 ++++ 3 files changed, 165 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi new file mode 100644 index 0000000000000..a6708b1c972df --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +#include + +/ { + aliases { + mtp = &mtp; + keyboard = &keyboard; + }; + + led-controller { + compatible = "pwm-leds"; + + led-0 { + pwms = <&fpwm0 0 40000>; + label = "kbd_backlight"; + function = LED_FUNCTION_KBD_BACKLIGHT; + color = ; + max-brightness = <255>; + default-state = "keep"; + }; + }; +}; + +&{/soc} { + pmgr: power-management@380700000 { + compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <3 0x80700000 0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + + ps_fpwm0: power-controller@330 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm0"; + }; + }; + + fpwm0: pwm@3a5040000 { + compatible = "apple,t8142-fpwm", "apple,s5l-fpwm"; + reg = <3 0xa5040000 0 0x4000>; + power-domains = <&ps_fpwm0>; + clocks = <&clkref>; + #pwm-cells = <2>; + }; + + smc_mbox: mailbox@38c608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <3 0x8c608000 0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@38c600000 { + compatible = "apple,t8142-smc", "apple,t8103-smc"; + reg = <3 0x8c600000 0 0x4000>, <3 0x8de00000 0 0x120000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + hwmon { + compatible = "apple,smc-hwmon"; + apple,discover-temperatures; + }; + }; + + mtp: mtp@394600000 { + compatible = "apple,t8142-mtp", "apple,t8142-rtk-helper-asc4", + "apple,mtp", "apple,rtk-helper-asc4"; + /* The second region is firmware SRAM, not the ASC control window. */ + reg = <3 0x94600000 0 0x4000>, <3 0x94c00000 0 0x100000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 0>; + }; + + mtp_mbox: mailbox@394608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <3 0x94608000 0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + mtp_dart: iommu@394800000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <3 0x94800000 0 0xc000>; + interrupts = ; + #iommu-cells = <1>; + /* Keep host allocations clear of firmware's low IOVA mappings. */ + apple,dma-range = <0 0x02000000 0 0x10000000>; + }; + + mtp_dockchannel: fifo@394b14000 { + compatible = "apple,t8142-dockchannel", "apple,dockchannel"; + reg = <3 0x94b14000 0 0x1000>; + reg-names = "irq"; + interrupts = ; + ranges = <0 3 0x94b28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + interrupt-controller; + #interrupt-cells = <2>; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x1000>, <0xc000 0x1000>, + <0x0000 0x1000>, <0x4000 0x1000>; + reg-names = "config", "data", "rmt-config", "rmt-data"; + iommus = <&mtp_dart 0>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + + stm { }; + + multi-touch { + firmware-name = "apple/tpmtfw-j813.bin"; + apple,power-method = <2>; + apple,firmware-managed-reset; + }; + + /* Layout is filled by the loader using this machine's own data. */ + keyboard: keyboard { }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index e47839a68dfe4..b0a8ac8339955 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -290,3 +290,5 @@ }; }; }; + +#include "t8142-j813-peripherals.dtsi" diff --git a/arch/arm64/configs/j813.config b/arch/arm64/configs/j813.config index f944f153962ab..13b8116af0710 100644 --- a/arch/arm64/configs/j813.config +++ b/arch/arm64/configs/j813.config @@ -49,3 +49,29 @@ CONFIG_FRAMEBUFFER_CONSOLE=y CONFIG_LOG_BUF_SHIFT=20 CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y +CONFIG_PM=y +CONFIG_MAILBOX=y +CONFIG_APPLE_MAILBOX=y +CONFIG_APPLE_RTKIT=y +CONFIG_APPLE_RTKIT_HELPER=y +CONFIG_APPLE_DOCKCHANNEL=y +CONFIG_IOMMU_SUPPORT=y +CONFIG_APPLE_DART=y +CONFIG_MFD_MACSMC=y +CONFIG_HWMON=y +CONFIG_SENSORS_MACSMC_HWMON=y +CONFIG_INPUT=y +CONFIG_INPUT_EVDEV=y +CONFIG_HID_SUPPORT=y +CONFIG_HID=y +CONFIG_HID_GENERIC=y +CONFIG_HID_DOCKCHANNEL=y +CONFIG_HID_APPLE=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_GPIOLIB=y +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_PWM=y +CONFIG_PWM=y +CONFIG_PWM_APPLE=y +CONFIG_FW_LOADER=y From 4dec9439047493c1ae3e17e07e5f14d93f5e646e Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 0368/1579] selftests: apple: document and exercise J813 peripherals on hardware Document the external firmware contract and qualification limits. Add an explicitly manual RAM-boot check that records temperatures, restores backlight brightness and counts physical input events without retaining key codes or pointer coordinates. Signed-off-by: Acelogic --- Documentation/arch/arm64/apple-j813.rst | 85 ++++++++++++-- MAINTAINERS | 1 + tools/testing/selftests/Makefile | 1 + .../testing/selftests/drivers/apple/Makefile | 5 + .../drivers/apple/j813-peripherals.sh | 104 ++++++++++++++++++ 5 files changed, 186 insertions(+), 10 deletions(-) create mode 100644 tools/testing/selftests/drivers/apple/Makefile create mode 100644 tools/testing/selftests/drivers/apple/j813-peripherals.sh diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst index eb618f4d26588..8ac299c9ef97c 100644 --- a/Documentation/arch/arm64/apple-j813.rst +++ b/Documentation/arch/arm64/apple-j813.rst @@ -5,8 +5,9 @@ Apple J813 initial native Linux boot ===================================== J813 is the 13-inch M5 MacBook Air (T8142, Mac17,3). This configuration -provides an initial RAM-only boot path: one boot CPU, AICv3, architectural -timers, DockChannel console and the framebuffer left by the boot firmware. +provides a RAM-only boot path with AICv3, architectural timers, DockChannel +console and the framebuffer left by the boot firmware. The J813 tree also +describes SMC temperature monitoring, MTP input and the keyboard backlight. The device tree describes all ten CPU affinities. For the initial one-CPU baseline, keep secondary cores stopped in m1n1 and use ``maxcpus=1``. @@ -98,8 +99,8 @@ loader is not sufficient. Before entering Linux, the loader must: * Fill the firmware framebuffer's address, size, geometry and format before enabling it. * Prepare DMA protection only for devices enabled in the supplied FDT. The - initial tree exposes no AOP, MTP, PMP or ISP DMA consumers. Their firmware - protection state must remain intact during the native FDT handoff. + ``mtp`` alias enables the MTP DAPF handoff. Leave the protection state of + unrelated AOP, PMP and ISP firmware intact. * Leave the console, interrupt controller and display powered. This minimal DT does not describe their power domains. * Give U-Boot a T8142 memory map covering its MMIO and high DRAM addresses. @@ -119,14 +120,14 @@ On an arm64 Linux build host with the usual kernel build dependencies:: KCONFIG_ALLCONFIG=arch/arm64/configs/j813.config allnoconfig make ARCH=arm64 O="$out" -j"$(nproc)" W=1 Image apple/t8142-j813.dtb -The fragment is deliberately minimal and does not enable KVM or storage -drivers. Add ``CROSS_COMPILE=aarch64-linux-gnu-`` to both commands when +The fragment does not enable KVM or storage drivers. Add +``CROSS_COMPILE=aarch64-linux-gnu-`` to both commands when cross-compiling with GCC from another architecture. With dtschema installed, check the affected bindings and DTB:: make ARCH=arm64 O="$out" CHECK_DTBS=y \ - DT_SCHEMA_FILES=arm/apple.yaml:arm/cpus.yaml:cpufreq/apple,cluster-cpufreq.yaml:opp/opp-v2-base.yaml:interrupt-controller/apple,aic2.yaml:serial/apple,dockchannel-uart.yaml:serial/samsung_uart.yaml:display/simple-framebuffer.yaml \ + DT_SCHEMA_FILES=arm/apple.yaml:arm/cpus.yaml:cpufreq/apple,cluster-cpufreq.yaml:opp/opp-v2-base.yaml:interrupt-controller/apple,aic2.yaml:serial/apple,dockchannel-uart.yaml:serial/samsung_uart.yaml:display/simple-framebuffer.yaml:apple,pmgr.yaml:apple,smc-hwmon.yaml:apple,dockchannel-hid.yaml:apple,dart.yaml:apple,mailbox.yaml:apple,smc.yaml:apple,pmgr-pwrstate.yaml:apple,s5l-fpwm.yaml:apple,dockchannel.yaml:apple,rtk-helper-asc4.yaml \ apple/t8142-j813.dtb Boot and qualification @@ -171,7 +172,71 @@ RAM boot. Preserve the complete console transcript and a hash manifest; an Image build or successful loader exit alone is not a boot result. Review captures for device identifiers and boot entropy before publishing. -These tests do not establish CPU hotplug, KVM guest timers, -sustained thermal behavior, suspend, native DCP, GPU acceleration, built-in input, USB, -storage, networking or audio support. The disabled Samsung UART node +These boot tests do not establish CPU hotplug, KVM guest timers, +sustained thermal behavior, suspend, native DCP, GPU acceleration, USB, +storage, networking or audio support. Input and monitoring require their +separate checks below. The disabled Samsung UART node records its measured resources; the qualified console path is DockChannel. + +Temperature monitoring and built-in input +======================================== + +Temperature monitoring enumerates the running SMC's readable, non-function +``T`` keys with four-byte floating-point values. It exposes their identifiers +through ``/sys/class/hwmon/*/temp*_label`` and millidegrees Celsius through +``temp*_input``. It does not embed a sensor inventory, calibration, inferred +sensor locations, trip temperatures or a new CPU cooling policy. Channel +numbers can change with firmware; identify channels by label. These keys can +include virtual or inactive channels; their number is not a count of physical +sensors, and a readable value does not establish a sensor's location. Read failures, +NaNs and infinities remain errors, rather than fabricated temperatures. +The existing J700 thermal provider and bounded J813 frequency states are +unchanged. Temperature reporting alone does not qualify sustained load. + +MTP provides keyboard and trackpad HID descriptors and identifiers at runtime. +The ``keyboard`` alias lets m1n1 fill ``hid-country-code`` and +``apple,keyboard-layout-id`` from this machine's own boot environment. The +source tree does not assume a US keyboard or contain another machine's layout. +The J813 trackpad uses power method 2 with firmware-owned reset; the host +does not operate an AFE GPIO. Other machines retain their existing sequence. + +The trackpad requires an externally supplied +``/lib/firmware/apple/tpmtfw-j813.bin``. The existing DockChannel firmware +container has a 20-byte little-endian header: magic ``0x46444948`` (``HIDF``), +version 1, header length, data length and interface-byte offset. See +``struct fw_header`` and ``dchid_get_firmware()`` for validation. An interface +offset of zero means no patching. The payload must +be prepared from firmware and any required calibration belonging to the +user's own machine. A filename is a lookup contract, not a firmware payload. +No Apple firmware, captured command buffers or calibration bytes are bundled. +Future IPSW/macOS extraction tools can provide this file without changing the +driver. A missing or invalid file fails trackpad startup; it does not prevent +the keyboard or temperature monitor from registering. + +The loader must preserve the iBoot-loaded MTP image and SRAM, prepare its DMA +protection, and leave MTP ready for Linux's RTKit startup. A prior-stage MTP +client must complete a compatible shutdown before Linux starts; a partially +initialized or crashed coprocessor is not a supported handoff. + +The backlight uses the existing PWM and LED drivers. Its boot brightness is +retained when the initial PWM period is valid, otherwise it starts off. +Userspace controls ``/sys/class/leds/kbd_backlight/brightness`` +on a linear 0--255 scale. No captured brightness curve or factory calibration +is included. + +Physical verification +--------------------- + +Place the user's touch firmware in a private RAM initramfs, then run:: + + sh tools/testing/selftests/drivers/apple/j813-peripherals.sh 30 + +The check reads temperatures twice, exercises and restores backlight +brightness, and counts keyboard and trackpad events during a bounded physical +interaction window. It records no key codes or pointer coordinates. Press +and release keys and move/click the built-in trackpad during that window. +Confirm visible brightness changes separately; sysfs readback is not optical +proof. Preserve stdout, the source revision and SHA-256 hashes of the exact +Image, DTB, configuration, initramfs and bootloaders as the repeatable result. +Keep firmware and raw machine data private. Without physical events the input +check fails rather than crediting device registration as working input. diff --git a/MAINTAINERS b/MAINTAINERS index 368bf2b43af1c..90654e399c0d3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2627,6 +2627,7 @@ F: include/dt-bindings/interrupt-controller/apple-aic.h F: include/dt-bindings/pinctrl/apple.h F: include/linux/mfd/macsmc.h F: include/linux/soc/apple/* +F: tools/testing/selftests/drivers/apple/ F: include/uapi/drm/asahi_drm.h F: net/bluetooth/brcm.c F: net/bluetooth/brcm.h diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 641a180fb35f4..ac5717ad1078c 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -17,6 +17,7 @@ TARGETS += damon TARGETS += devices/error_logs TARGETS += devices/probe TARGETS += dmabuf-heaps +TARGETS += drivers/apple TARGETS += drivers/dma-buf TARGETS += drivers/ntsync TARGETS += drivers/s390x/uvdevice diff --git a/tools/testing/selftests/drivers/apple/Makefile b/tools/testing/selftests/drivers/apple/Makefile new file mode 100644 index 0000000000000..76bca086dde7a --- /dev/null +++ b/tools/testing/selftests/drivers/apple/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 +# Manual physical test: requires user interaction and changes LED brightness. +TEST_FILES := j813-peripherals.sh + +include ../../lib.mk diff --git a/tools/testing/selftests/drivers/apple/j813-peripherals.sh b/tools/testing/selftests/drivers/apple/j813-peripherals.sh new file mode 100644 index 0000000000000..ecad597fe9113 --- /dev/null +++ b/tools/testing/selftests/drivers/apple/j813-peripherals.sh @@ -0,0 +1,104 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Physical J813 check. Save stdout and the payload SHA-256 manifest together. +# Usage: sh j813-peripherals.sh [event-capture-seconds] +set -eu + +duration=${1:-30} +case "$duration" in *[!0-9]*|'') exit 2;; esac +[ "$duration" -ge 1 ] && [ "$duration" -le 600 ] || exit 2 +[ -r /proc/device-tree/compatible ] || exit 4 +tr '\000' '\n' < /proc/device-tree/compatible | grep -qx apple,j813 || exit 4 +[ "$(id -u)" -eq 0 ] || { echo 'SKIP: run as root'; exit 4; } + +work=$(mktemp -d) +led=/sys/class/leds/kbd_backlight +saved_brightness= +cleanup() +{ + if [ -n "$saved_brightness" ]; then + echo "$saved_brightness" > "$led/brightness" + fi + rm -rf "$work" +} +trap cleanup EXIT +trap 'exit 1' INT TERM + +echo J813_PERIPHERALS_BEGIN +uname -r +zcat /proc/config.gz | sha256sum +cat /sys/devices/system/cpu/online + +hwmon= +for h in /sys/class/hwmon/hwmon*; do + [ -r "$h/name" ] || continue + [ "$(cat "$h/name")" = macsmc_hwmon ] || continue + hwmon=$h +done +[ -n "$hwmon" ] || { echo 'FAIL: no SMC hwmon'; exit 1; } +for sample in 1 2; do + count=0 + for input in "$hwmon"/temp*_input; do + [ -r "$input" ] || continue + value=$(cat "$input") + label=$(cat "${input%_input}_label") + [ "$value" -ge -40000 ] && [ "$value" -le 125000 ] || exit 1 + printf 'temperature sample=%s key=%s millidegrees=%s\n' "$sample" "$label" "$value" + count=$((count + 1)) + done + [ "$count" -gt 0 ] || { echo 'FAIL: no temperature channels'; exit 1; } + sleep 1 +done +echo "PASS: $count readable SMC temperature channels" + +[ -r "$led/brightness" ] || { echo 'FAIL: no keyboard backlight'; exit 1; } +saved_brightness=$(cat "$led/brightness") +maximum=$(cat "$led/max_brightness") +for level in 0 $((maximum / 2)) "$maximum"; do + echo "$level" > "$led/brightness" + [ "$(cat "$led/brightness")" = "$level" ] || exit 1 + printf 'backlight requested=%s readback=%s\n' "$level" "$(cat "$led/brightness")" + sleep 5 +done +echo "$saved_brightness" > "$led/brightness" +echo 'PASS: backlight control and restore (visual confirmation is separate)' + +keyboard= +trackpad= +for e in /sys/class/input/event*; do + [ -r "$e/device/name" ] || continue + case "$(cat "$e/device/name")" in + 'Apple MTP keyboard') keyboard=/dev/input/${e##*/} ;; + 'Apple MTP multi-touch') trackpad=/dev/input/${e##*/} ;; + esac +done +[ -n "$keyboard" ] && [ -n "$trackpad" ] || { + echo 'FAIL: keyboard or trackpad missing'; exit 1; +} +echo "Move/click the built-in trackpad and press/release keys for $duration seconds." +# Count bytes only. Do not retain key codes, pointer coordinates, or raw events. +timeout "$duration" cat "$keyboard" | wc -c > "$work/keyboard" & +kp=$! +timeout "$duration" cat "$trackpad" | wc -c > "$work/trackpad" & +tp=$! +wait "$kp" +wait "$tp" +failed=0 +for name in keyboard trackpad; do + bytes=$(cat "$work/$name") + [ "$bytes" -gt 0 ] && [ $((bytes % 24)) -eq 0 ] || { + echo "FAIL: $name produced no complete arm64 input events" + failed=1 + continue + } + printf 'PASS: %s events=%s\n' "$name" "$((bytes / 24))" +done +dmesg > "$work/dmesg" +fault_pattern='Kernel panic|Oops:|(^|[[:space:]])SError([[:space:]:]|$)' +fault_pattern="$fault_pattern|(^|[[:space:]])BUG:|apple-dart .*:.*(fault|error)" +if grep -E "$fault_pattern" "$work/dmesg"; then + echo 'FAIL: kernel fault detected' + exit 1 +fi +[ "$failed" -eq 0 ] || exit 1 +echo J813_PERIPHERALS_PASS From f227145f50e4e29e8eb52b46ff6ce51b56229bb4 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:03:39 -0500 Subject: [PATCH 0369/1579] accel/ane: Log register and address tracing at debug level Both modules log bring-up tracing at info level on every probe: the ANERD/ANEWR recovery trace and the ps probe SET window reads in ane_tm.c, the ANE-resume progress lines and the first ps probe in ane_drv.c, one line for each DART in ane_dart.c, and in ane_t6021 each ChMan table entry, each RTKit endpoint start and app message, the CONFIG_GET words, and the firmware alias, PMU map and staging addresses. Make them dev_dbg(). Dynamic debug still shows them for crash attribution. The outcome lines stay at info level: DART containment armed, loaded ane, tm recovered, the ChMan table result, the chip revision, the boot result lines and loaded ane_t6021. Faults stay at error and warning level. No other code changes. After this change, the only info lines on the ane probe path are "DART containment armed" and "loaded ane". Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_dart.c | 4 +- drivers/accel/ane/ane_drv.c | 14 ++--- drivers/accel/ane/ane_t6021_fwload.c | 48 +++++++-------- drivers/accel/ane/ane_t6021_rtclient_main.c | 30 ++++----- drivers/accel/ane/ane_tm.c | 68 +++++++++++---------- 5 files changed, 83 insertions(+), 81 deletions(-) diff --git a/drivers/accel/ane/ane_dart.c b/drivers/accel/ane/ane_dart.c index f66bfbe15af55..0dc0e955182da 100644 --- a/drivers/accel/ane/ane_dart.c +++ b/drivers/accel/ane/ane_dart.c @@ -113,8 +113,8 @@ int ane_dart_init(struct ane_device *ane) dart->irq = irq; dart->sid = args.args_count ? args.args[0] : 0; dart->masked = false; - dev_info(ane->dev, "DART containment: %pOFn sid %u irq %d\n", - args.np, dart->sid, irq); + dev_dbg(ane->dev, "DART containment: %pOFn sid %u irq %d\n", + args.np, dart->sid, irq); put_device(&pdev->dev); of_node_put(args.np); } diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 20e32a76f190a..52e720aad356c 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -1208,13 +1208,13 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) eng = platform_get_resource_byname(pdev, IORESOURCE_MEM, "engine"); - dev_info(dev, - "ANE-resume: genpd raise complete; SET window probe next\n"); + dev_dbg(dev, + "ANE-resume: genpd raise complete; SET window probe next\n"); ane_ps_act_probe(ane); - dev_info(dev, - "ANE-resume: SET window probed; first engine access next (TM_TQ_EN tm+0x0c @ engine %pr + 0x2000c)\n", - eng); + dev_dbg(dev, + "ANE-resume: SET window probed; first engine access next (TM_TQ_EN tm+0x0c @ engine %pr + 0x2000c)\n", + eng); } ane_tm_enable(ane, first); @@ -1223,7 +1223,7 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) * its post-reset status against it. */ if (!ane->tm_status_known) { - dev_info(dev, "ANE-resume: enable writes survived; TM_STATUS read next\n"); + dev_dbg(dev, "ANE-resume: enable writes survived; TM_STATUS read next\n"); ane->tm_status_fresh = ane_tm_status(ane); ane->tm_status_known = true; /* Linux-side pwrstate probe: ACTUAL nibbles read through @@ -1232,7 +1232,7 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) * pmgr SET block with every word on; anything else means * the window does not match the expected word layout. */ - dev_info(dev, "ANERD ps probe act=%#x\n", ane_ps_act(ane)); + dev_dbg(dev, "ANERD ps probe act=%#x\n", ane_ps_act(ane)); } return 0; diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index 04153a2a4d576..44be0d3e72ecc 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -243,8 +243,8 @@ static int ane_t6021_pmu_map(struct ane_t6021 *ane, struct iommu_domain *dom) prot, GFP_KERNEL); if (!ret && iommu_iova_to_phys(dom, soc->pmu_pa) != soc->pmu_pa) ret = -EIO; - dev_info(ane->dev, "pmu: DART map %#llx (IOVA == PA, %#x bytes): %d\n", - soc->pmu_pa, ANE_T6021_FW_ALIAS_PAGE, ret); + dev_dbg(ane->dev, "pmu: DART map %#llx (IOVA == PA, %#x bytes): %d\n", + soc->pmu_pa, ANE_T6021_FW_ALIAS_PAGE, ret); return ret; } @@ -265,9 +265,9 @@ static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) /* Unlatched branch: the boot path programs RVBAR to the * fw DVA itself (ane_t6021_rvbar_compose), no alias. */ - dev_info(ane->dev, - "fwalias: rvbar %016llx not latched/entry 0 — skip (boot reprograms RVBAR)\n", - rvbar); + dev_dbg(ane->dev, + "fwalias: rvbar %016llx not latched/entry 0 — skip (boot reprograms RVBAR)\n", + rvbar); return 0; } if (!ane_t6021_rvbar_entry_ok(entry) || @@ -366,14 +366,14 @@ static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) } ane->fw_alias_extn = windows; ane->fw_alias_iova = entry; - dev_info(ane->dev, - "fwalias: reserved SEG0/SEGi at entry %#llx (%llx+%zx %llx+%zx, preloaded placement)\n", - entry, - ane->fw_alias_ext_iova[0], ane->fw_alias_ext_len[0], - ane->fw_alias_ext_iova[1], ane->fw_alias_ext_len[1]); + dev_dbg(ane->dev, + "fwalias: reserved SEG0/SEGi at entry %#llx (%llx+%zx %llx+%zx, preloaded placement)\n", + entry, + ane->fw_alias_ext_iova[0], ane->fw_alias_ext_len[0], + ane->fw_alias_ext_iova[1], ane->fw_alias_ext_len[1]); if (fw_extra_ram) - dev_info(ane->dev, "fwalias: owned heap [%#llx,%#llx) roundtrip verified\n", - win[2].iova, win[2].iova + win[2].len); + dev_dbg(ane->dev, "fwalias: owned heap [%#llx,%#llx) roundtrip verified\n", + win[2].iova, win[2].iova + win[2].len); return ane_t6021_pmu_map(ane, dom); err_unmap_mapped: @@ -440,10 +440,10 @@ static int ane_t6021_fw_alias_map(struct ane_t6021 *ane, bool reserved) ane->fw_alias_ext_iova[0] = entry; ane->fw_alias_ext_len[0] = ane->fw_size; ane->fw_alias_extn = 1; - dev_info(ane->dev, - "fwalias: entry %#llx <- %u dart pages aliased from fw %pad (first %pa, roundtrip OK)\n", - entry, ane->fw_size / ANE_T6021_FW_ALIAS_PAGE, - &ane->fw_iova, &pa0); + dev_dbg(ane->dev, + "fwalias: entry %#llx <- %u dart pages aliased from fw %pad (first %pa, roundtrip OK)\n", + entry, ane->fw_size / ANE_T6021_FW_ALIAS_PAGE, + &ane->fw_iova, &pa0); /* The firmware's power service needs the pmgr sub-block mapped * IOVA == PA in every vehicle (NO PMD FOR IOVA 0x28e084008, * 2026-09-29); the staged-DMA alias branch skipped it and left @@ -527,11 +527,11 @@ static int ane_t6021_fw_patch(struct ane_t6021 *ane, u8 *img) p.soc_revision); return -EINVAL; } - dev_info(ane->dev, - "fwload: own memory: iBoot patches replayed (soc %#x rev %#x DATA %#llx cpu %#llx wrapper %#llx)\n", - p.soc, p.soc_revision, - p.exec_base + soc->fw->segs[1].vmaddr, p.cpu_pa, - p.wrapper_pa); + dev_dbg(ane->dev, + "fwload: own memory: iBoot patches replayed (soc %#x rev %#x DATA %#llx cpu %#llx wrapper %#llx)\n", + p.soc, p.soc_revision, + p.exec_base + soc->fw->segs[1].vmaddr, p.cpu_pa, + p.wrapper_pa); return 0; } @@ -595,9 +595,9 @@ int ane_t6021_fwload_probe(struct ane_t6021 *ane) ane->fw_iova = iova; ane->fw_size = alloc_size; - dev_info(ane->dev, - "fwload: %s PRELOAD validated + DART-mapped: entry %#llx, iova %pad size %#x\n", - img->name, entry, &iova, alloc_size); + dev_dbg(ane->dev, + "fwload: %s PRELOAD validated + DART-mapped: entry %#llx, iova %pad size %#x\n", + img->name, entry, &iova, alloc_size); ret = reserved ? 0 : ane_t6021_fw_patch(ane, buf); if (!ret) diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index f9eb131a4f30e..be21e4c1b0eb1 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -464,12 +464,12 @@ static void ane_rtclient_validate_chman(struct ane_rtclient *ane) const struct ane_t6021_chman_desc *d = &t[i]; const struct ane_t6021_chman_static *s = &ane_t6021_chman_layout[i]; - dev_info(ane->dev, - "chman[%u]: name=\"%.*s\" type=%u bit=%u size=%#llx %s (static: %s/%u/%u/%#llx/ipc+%#x)\n", - i, ANE_T6021_CHMAN_NAME_LEN, d->name, d->type, d->bit, - d->size, - (bad & BIT(i)) ? "MISMATCH" : "OK", - s->name, s->type, s->bit, s->size, s->off); + dev_dbg(ane->dev, + "chman[%u]: name=\"%.*s\" type=%u bit=%u size=%#llx %s (static: %s/%u/%u/%#llx/ipc+%#x)\n", + i, ANE_T6021_CHMAN_NAME_LEN, d->name, d->type, d->bit, + d->size, + (bad & BIT(i)) ? "MISMATCH" : "OK", + s->name, s->type, s->bit, s->size, s->off); } ane->chman_ok = !bad; @@ -1800,8 +1800,8 @@ static void ane_rtclient_recv(void *cookie, u8 ep, u64 message) { struct ane_rtclient *ane = cookie; - dev_info(ane->dev, - "rtkit app msg: ep=%#x msg=%016llx\n", ep, message); + dev_dbg(ane->dev, + "rtkit app msg: ep=%#x msg=%016llx\n", ep, message); } static void ane_rtclient_crashed(void *cookie, const void *crashlog, @@ -1901,8 +1901,8 @@ static void ane_rtclient_start_app_eps(struct ane_rtclient *ane) if (!apple_rtkit_has_endpoint(ane->rtk, ep)) continue; ret = apple_rtkit_start_ep(ane->rtk, ep); - dev_info(ane->dev, "rtkit: STARTEP app ep %#x -> %pe\n", - ep, ERR_PTR(ret)); + dev_dbg(ane->dev, "rtkit: STARTEP app ep %#x -> %pe\n", + ep, ERR_PTR(ret)); } } @@ -2295,11 +2295,11 @@ static int ane_rtclient_probe(struct platform_device *pdev) command, 16, 0x03, 1, 3000); - dev_info(dev, - "LEGACY CONFIG_GET words %08x %08x result=%d (DMA remains held)\n", - READ_ONCE(((u32 *)command->cpu)[1]), - READ_ONCE(((u32 *)command->cpu)[2]), - qret); + dev_dbg(dev, + "LEGACY CONFIG_GET words %08x %08x result=%d (DMA remains held)\n", + READ_ONCE(((u32 *)command->cpu)[1]), + READ_ONCE(((u32 *)command->cpu)[2]), + qret); if (qret) { cfg_err = qret; } else if (!READ_ONCE(((u32 *)command->cpu)[2])) { diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 7a0ade69c395b..52f3781b8cc15 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -92,7 +92,8 @@ u32 ane_ps_act(struct ane_device *ane) /* Bisect probe: per-word named reads of the SET window. Each word logs * before its readl, so a window whose address does not decode on this * SoC is named by the last off-box line (word index + byte offset) - * instead of a silent hard reset between two other prints. + * instead of a silent hard reset between two other prints. The lines + * are debug messages: dynamic debug turns them on. */ u32 ane_ps_act_probe(struct ane_device *ane) { @@ -100,19 +101,19 @@ u32 ane_ps_act_probe(struct ane_device *ane) int i; if (!ane->ps) { - dev_info(ane->dev, "ps probe: SET window unmapped\n"); + dev_dbg(ane->dev, "ps probe: SET window unmapped\n"); return 0; } - dev_info(ane->dev, "ps probe: SET window at %pap, %d words\n", - &ane->ps_base, ANE_PS_WORDS); + dev_dbg(ane->dev, "ps probe: SET window at %pap, %d words\n", + &ane->ps_base, ANE_PS_WORDS); for (i = 0; i < ANE_PS_WORDS; i++) { u32 w; - dev_info(ane->dev, "ps probe: word %d @ ps+0x%02x reading\n", - i, i * 8); + dev_dbg(ane->dev, "ps probe: word %d @ ps+0x%02x reading\n", + i, i * 8); w = readl(ane->ps + i * 8); v |= ((w & ANE_PS_ACTUAL_MASK) >> 4) << (i * 4); - dev_info(ane->dev, "ps probe: word %d -> %#x\n", i, w); + dev_dbg(ane->dev, "ps probe: word %d -> %#x\n", i, w); } return v; } @@ -121,16 +122,17 @@ u32 ane_ps_act_probe(struct ane_device *ane) * every engine read prints its value, each line carrying the pmgr * ACTUAL of the owning partitions. A write that external-aborts the * SoC is then named by the last line the off-box netconsole carried, - * and the bisect starts from that register instead of a guess. + * and the bisect starts from that register instead of a guess. The + * lines are debug messages: dynamic debug turns them on. */ static void ane_rec_writel(struct ane_device *ane, const char *reg, void __iomem *addr, u32 val) { - dev_info(ane->dev, "ANEWR %s <- %#x (ps act %#x)\n", reg, val, - ane_ps_act(ane)); + dev_dbg(ane->dev, "ANEWR %s <- %#x (ps act %#x)\n", reg, val, + ane_ps_act(ane)); writel(val, addr); - dev_info(ane->dev, "ANEWR %s wrote (ps act %#x)\n", reg, - ane_ps_act(ane)); + dev_dbg(ane->dev, "ANEWR %s wrote (ps act %#x)\n", reg, + ane_ps_act(ane)); } static u32 ane_rec_read32(struct ane_device *ane, const char *reg, @@ -138,8 +140,8 @@ static u32 ane_rec_read32(struct ane_device *ane, const char *reg, { u32 val = readl(addr); - dev_info(ane->dev, "ANERD %s -> %#x (ps act %#x)\n", reg, val, - ane_ps_act(ane)); + dev_dbg(ane->dev, "ANERD %s -> %#x (ps act %#x)\n", reg, val, + ane_ps_act(ane)); return val; } @@ -423,7 +425,7 @@ static int ane_ps_verify_on(struct ane_device *ane) } usleep_range(1000, 2000); } - dev_info(ane->dev, "ANERD ps verify act=%#x err=%d\n", act, err); + dev_dbg(ane->dev, "ANERD ps verify act=%#x err=%d\n", act, err); return err; } @@ -438,26 +440,26 @@ static int ane_pd_cycle(struct ane_device *ane) for (int i = 0; i < ane->pd_count; i++) { pm_runtime_get_noresume(ane->pd_dev[i]); gated = i + 1; - dev_info(ane->dev, - "ANERD pd[%d] %s force_suspend begin (ps act %#x)\n", - i, dev_name(ane->pd_dev[i]), ane_ps_act(ane)); + dev_dbg(ane->dev, + "ANERD pd[%d] %s force_suspend begin (ps act %#x)\n", + i, dev_name(ane->pd_dev[i]), ane_ps_act(ane)); err = pm_runtime_force_suspend(ane->pd_dev[i]); - dev_info(ane->dev, - "ANERD pd[%d] force_suspend -> %d (ps act %#x)\n", - i, err, ane_ps_act(ane)); + dev_dbg(ane->dev, + "ANERD pd[%d] force_suspend -> %d (ps act %#x)\n", + i, err, ane_ps_act(ane)); if (err) break; } if (!err) for (int i = 0; i < ane->pd_count; i++) { - dev_info(ane->dev, - "ANERD pd[%d] %s force_resume begin (ps act %#x)\n", - i, dev_name(ane->pd_dev[i]), - ane_ps_act(ane)); + dev_dbg(ane->dev, + "ANERD pd[%d] %s force_resume begin (ps act %#x)\n", + i, dev_name(ane->pd_dev[i]), + ane_ps_act(ane)); err = pm_runtime_force_resume(ane->pd_dev[i]); - dev_info(ane->dev, - "ANERD pd[%d] force_resume -> %d (ps act %#x)\n", - i, err, ane_ps_act(ane)); + dev_dbg(ane->dev, + "ANERD pd[%d] force_resume -> %d (ps act %#x)\n", + i, err, ane_ps_act(ane)); if (err) break; } @@ -470,14 +472,14 @@ static int ane_pd_cycle(struct ane_device *ane) * the partition gated. */ pm_runtime_get_noresume(ane->dev); - dev_info(ane->dev, "ANERD dev force_suspend begin\n"); + dev_dbg(ane->dev, "ANERD dev force_suspend begin\n"); err = pm_runtime_force_suspend(ane->dev); - dev_info(ane->dev, "ANERD dev force_suspend -> %d\n", err); + dev_dbg(ane->dev, "ANERD dev force_suspend -> %d\n", err); if (!err) { - dev_info(ane->dev, "ANERD dev force_resume begin\n"); + dev_dbg(ane->dev, "ANERD dev force_resume begin\n"); err = pm_runtime_force_resume(ane->dev); - dev_info(ane->dev, "ANERD dev force_resume -> %d\n", - err); + dev_dbg(ane->dev, "ANERD dev force_resume -> %d\n", + err); } pm_runtime_put_noidle(ane->dev); } From 84039caeb221604fac2e64961e09e31f55c8a513 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 2 Oct 2026 20:23:07 -0400 Subject: [PATCH 0370/1579] vfio: transfer native type1 pins to mixed-page VADDR owners Stage retained-provider metadata for existing ordinary native GUP pins when VADDR updates require an alternative mm or fine alignment. Abort metadata without changing pin references or accounts if preparation/credit fails. Credit the recipient with unique physical pages before refunding the old full charge and committing ownership; same-mm conversion adjusts the delta. Preserve every alias reference and independent native external pin, including different physical backing after a VA replacement. Native-aligned mapping/batching stays on its existing path; reserved/PFNMAP conversion remains unsupported by this ordinary-pin provider. Nine mapped/lazy/alias/external/same-mm variants pass on native16K/64K; total type1 KUnit10/0. Real canonical VADDR tests pass native-source exit, forced memlock-credit failure, retry, recipient access, external reuse and final refund for4K/16K recipients. Previous-owner-gate controls reproduce EOPNOTSUPP and fail the new tests without kernel warnings. Main mixed-page, AArch32 auto/fallback, feature-off compilation and module-load checks pass. Physical VFIO group/device attachment, DMA, concurrency/performance, Apple GPU execution, games and the broader mixed-page ABI remain unqualified. --- Documentation/arch/arm64/user4k.rst | 56 ++++++ drivers/vfio/vfio_iommu_type1.c | 68 ++++++-- drivers/vfio/vfio_iommu_type1_fragments.c | 159 ++++++++++++++++++ .../vfio/vfio_iommu_type1_fragments_test.c | 120 +++++++++++++ drivers/vfio/vfio_iommu_type1_ioctl_test.c | 22 ++- drivers/vfio/vfio_iommu_type1_test.h | 2 + .../arm64/user4k/vfio-fixture-init.c | 104 ++++++++++++ 7 files changed, 513 insertions(+), 18 deletions(-) diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index 2b4cc50e4d14e..fc1e3ca97306f 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -4404,3 +4404,59 @@ EL0-to-type1 provider/ABI path with supplied domains, not the VFIO core group attachment protocol, physical DMA, Apple boot, GPU execution or game throughput. Native-to-alternative VADDR ownership conversion and external spans crossing multiple fine DMA mappings remain unfinished. + + +Native VFIO VADDR ownership conversion (2026-10-02) +------------------------------------------------- + +Native-aligned mappings continue using their existing GUP and batching path. +When VADDR updates transfer such a mapping to an alternative-page mm, or +supply a fine-aligned new address, type1 now prepares retained-provider +metadata for the original native GUP pins. Known native pin provenance and +the existing native IOMMU translations identify these references; this is +not inference of an alternative allocation's slot identity. No new pin is +acquired during preparation. Metadata allocation or failed new-owner credit +frees the candidate metadata without publishing it, releasing old pins or +changing the old account. The new account is credited with the unique physical +page count before the old full charge is refunded and ownership is committed. +Same-mm conversion adjusts only the charge difference. + +Physical aliases retain each original GUP reference while charging a physical +page once within the provider. Existing native external consumers may own +independent GUP references, including different physical pages after a source +VA replacement. Their identities and references remain separate from the +mapped buffer and participate in the physical-page count. Without a mapped +domain, existing external pins become the original cached buffer records. +Native reserved/PFNMAP conversion remains unsupported by this ordinary-pin +provider; the native path's existing behavior is unchanged. + +A new type1 KUnit case passes mapped/lazy, physical-alias and distinct +external-page variants on both native16K and native64K kernels. It checks +metadata abort, failed credit, exited-source refunds, transfer to a 4K mm, +original buffer bytes, duplicate external identities, replay and complete +reference/account cleanup. A same-mm native fine-address variant checks +alias charge reduction without changing the process or releasing native pins. +The total type1 result is ten passes, zero failures +on both kernels, including the existing accounting and fragment cases. + +Real EL0 tests fork/exec a native-page source, verify that its full aligned MAP +uses legacy GUP, and retain the mapping after that source exits. Alternative +4K/16K recipients use canonical VADDR invalidation and update ioctls. A zero +memlock limit with only effective CAP_IPC_LOCK removed forces ENOMEM; the old +owner, native pins, metadata and charge remain intact. Restoring the original +limit/capability permits retry, CPU access under the new mm's authorization, +external native-page pin reuse and final recipient refund. Native16 runs 29 +native checks and 41 alternative4K checks; native64 runs 29 native checks and +41 checks for each alternative4K/16K. AArch32 auto4K remains covered by the +same fixture. Group restrictions stay in production; only a default-off, +root-only recording-domain control changes the private test group's membership. + +Matched controls restore the previous native-to-alternative owner gate while +retaining the new harness. Their real VADDR tests return EOPNOTSUPP instead of +permitting credit/retry; dependent CPU operations stop while VADDR is invalid. +These are matched behavior controls, not original-source kernel Images. +Broader mixed-page KUnit regressions, feature-off arm64/x86 compilation and +module build/load/unload checks also pass. Evidence is under +user4k-vm-state/vfio-owner-v1/. This qualifies the tested provider ownership +paths with supplied domains, not concurrent physical-device attachment/DMA, +Apple GPU execution, game performance or the remaining full ABI requirements. diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 68b7877cf626e..8e3310487b553 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -135,6 +135,7 @@ struct vfio_pfn { dma_addr_t iova; /* Device address */ unsigned long pfn; /* Host pfn */ unsigned int ref_count; + bool native_pin; /* separately adopted legacy external PIN */ }; struct vfio_regions { @@ -158,6 +159,7 @@ struct vfio_regions { #define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX) static int put_pfn(unsigned long pfn, int prot); +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn); static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, unsigned long length); @@ -439,7 +441,12 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) if (dma->fragments) { unsigned long offset = vpfn->iova - dma->iova; + unsigned long pfn = vpfn->pfn; + bool native_pin = vpfn->native_pin; + vfio_remove_from_pfn_list(dma, vpfn); + if (native_pin) + vfio_fragments_put_native_pin(dma, pfn); vfio_fragments_release_range(dma, offset, PAGE_SIZE); } else { ret = put_pfn(vpfn->pfn, dma->prot); @@ -1703,25 +1710,30 @@ static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu, return list_empty(iova); } -static int vfio_change_dma_mm(struct vfio_dma *dma, struct task_struct *task, - struct mm_struct *mm, bool lock_cap) +static int vfio_change_dma_mm_pages(struct vfio_dma *dma, struct task_struct *task, + struct mm_struct *mm, bool lock_cap, unsigned long pages) { - long npage = dma->locked_vm; + long old_pages = dma->locked_vm; int ret; - if (mm == dma->mm) - return 0; - ret = mm_lock_acct(task, mm, lock_cap, npage); + if (mm == dma->mm) { + if (pages == old_pages) + return 0; + ret = mm_lock_acct(task, mm, lock_cap, (long)pages - old_pages); + if (!ret) + dma->locked_vm = pages; + return ret; + } + ret = mm_lock_acct(task, mm, lock_cap, pages); if (ret) return ret; - /* dma retains mm_count after exit; its old charge still needs a refund. */ - ret = mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -npage); + /* Metadata adoption can deduplicate charges, but not original PIN refs. */ + ret = mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -old_pages); if (ret) { - mm_lock_acct(task, mm, lock_cap, -npage); + mm_lock_acct(task, mm, lock_cap, -(long)pages); return ret; } - if (dma->task != task) { put_task_struct(dma->task); dma->task = get_task_struct(task); @@ -1730,16 +1742,38 @@ static int vfio_change_dma_mm(struct vfio_dma *dma, struct task_struct *task, dma->mm = mm; mmgrab(dma->mm); dma->lock_cap = lock_cap; + dma->locked_vm = pages; return 0; } -static int vfio_change_dma_owner(struct vfio_dma *dma) +static int vfio_change_dma_mm(struct vfio_dma *dma, struct task_struct *task, + struct mm_struct *mm, bool lock_cap) +{ + return vfio_change_dma_mm_pages(dma, task, mm, lock_cap, dma->locked_vm); +} + +static int vfio_change_dma_owner(struct vfio_iommu *iommu, struct vfio_dma *dma, + unsigned long vaddr) { - /* Native mappings need ownership conversion before a granule change. */ - if (!dma->fragments && mm_page_size(current->mm) != PAGE_SIZE) - return -EOPNOTSUPP; - return vfio_change_dma_mm(dma, current->group_leader, current->mm, - capable(CAP_IPC_LOCK)); + struct vfio_fragment_provider *provider; + int ret; + + if (dma->fragments || (mm_page_size(current->mm) == PAGE_SIZE && + IS_ALIGNED(vaddr, PAGE_SIZE))) + return vfio_change_dma_mm(dma, current->group_leader, current->mm, + capable(CAP_IPC_LOCK)); + + /* Prepare only metadata: failure must preserve every legacy pin/charge. */ + provider = vfio_fragments_prepare_native(iommu, dma); + if (IS_ERR(provider)) + return PTR_ERR(provider); + ret = vfio_change_dma_mm_pages(dma, current->group_leader, current->mm, + capable(CAP_IPC_LOCK), vfio_fragments_native_pages(provider)); + if (ret) + vfio_fragments_abort_native(provider); + else + vfio_fragments_commit_native(dma, provider); + return ret; } static int vfio_dma_do_map(struct vfio_iommu *iommu, @@ -1792,7 +1826,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, dma->size != size) { ret = -EINVAL; } else { - ret = vfio_change_dma_owner(dma); + ret = vfio_change_dma_owner(iommu, dma, vaddr); if (ret) goto out_unlock; dma->vaddr = vaddr; diff --git a/drivers/vfio/vfio_iommu_type1_fragments.c b/drivers/vfio/vfio_iommu_type1_fragments.c index b24e65f2406e8..610a368a6193e 100644 --- a/drivers/vfio/vfio_iommu_type1_fragments.c +++ b/drivers/vfio/vfio_iommu_type1_fragments.c @@ -91,6 +91,146 @@ static int vfio_fragment_store(struct vfio_dma *dma, unsigned long start, return 0; } +/* These descriptors borrow existing native GUP pins until owner commit. */ +static int vfio_fragment_native_pin(unsigned long pfn, struct user_page_fragment *pin) +{ + struct page *page; + + if (!pfn_valid(pfn)) + return -EOPNOTSUPP; + page = pfn_to_page(pfn); + if (PageReserved(page)) + return -EOPNOTSUPP; + *pin = (struct user_page_fragment) { + .folio = page_folio(page), + .offset = page_to_phys(page) - PFN_PHYS(folio_pfn(page_folio(page))), + .length = PAGE_SIZE, + .pinned = true, + }; + return 0; +} + +/* No PIN, GET or account reference was acquired while preparing metadata. */ +static void vfio_fragments_abort_native(struct vfio_fragment_provider *provider) +{ + struct rb_node *node; + + while ((node = rb_first(&provider->records))) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + + rb_erase(node, &provider->records); + kfree(record); + } + xa_destroy(&provider->physical_pages); + kfree(provider); +} + +static struct vfio_fragment_provider * +vfio_fragments_prepare_native(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_fragment_provider *provider; + struct vfio_dma stage = {}; + struct vfio_domain *domain = NULL; + struct rb_node *node; + int ret; + + if (mm_page_size(dma->mm) != PAGE_SIZE || dma->has_rsvd) + return ERR_PTR(-EOPNOTSUPP); + if (dma->iommu_mapped) { + if (list_empty(&iommu->domain_list)) + return ERR_PTR(-EFAULT); + domain = list_first_entry(&iommu->domain_list, struct vfio_domain, next); + } + provider = kzalloc_obj(*provider, GFP_KERNEL_ACCOUNT); + if (!provider) + return ERR_PTR(-ENOMEM); + provider->records = RB_ROOT; + xa_init(&provider->physical_pages); + stage.fragments = provider; + /* One descriptor owns each original native GUP pin, including aliases. */ + if (domain) { + for (unsigned long offset = 0; offset < dma->size; offset += PAGE_SIZE) { + phys_addr_t phys = iommu_iova_to_phys(domain->domain, dma->iova + offset); + struct user_page_fragment pin; + + if (!phys || !IS_ALIGNED(phys, PAGE_SIZE)) { + ret = -EFAULT; + goto out; + } + ret = vfio_fragment_native_pin(PHYS_PFN(phys), &pin); + if (!ret) + ret = vfio_fragment_store(&stage, offset, &pin); + if (ret) + goto out; + cond_resched(); + } + } + for (node = rb_first(&dma->pfn_list); node; node = rb_next(node)) { + struct vfio_pfn *vpfn = rb_entry(node, struct vfio_pfn, node); + struct user_page_fragment pin; + + ret = vfio_fragment_native_pin(vpfn->pfn, &pin); + if (ret) + goto out; + if (!domain) { + /* Without a domain, the external pin is the cached buffer. */ + ret = vfio_fragment_store(&stage, vpfn->iova - dma->iova, &pin); + } else { + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, + vpfn->pfn)); + + /* The external GUP reference may identify a different page. */ + ret = xa_err(xa_store(&provider->physical_pages, vpfn->pfn, + xa_mk_value(count + 1), GFP_KERNEL_ACCOUNT)); + if (!ret && !count) + provider->npinned++; + } + if (ret) + goto out; + } + return provider; +out: + vfio_fragments_abort_native(provider); + return ERR_PTR(ret); +} + +static unsigned long vfio_fragments_native_pages(struct vfio_fragment_provider *provider) +{ + return provider->npinned; +} + +static void vfio_fragments_commit_native(struct vfio_dma *dma, + struct vfio_fragment_provider *provider) +{ + struct rb_node *node; + + /* Mapped legacy external consumers own separate native GUP references. */ + if (dma->iommu_mapped) { + for (node = rb_first(&dma->pfn_list); node; node = rb_next(node)) + rb_entry(node, struct vfio_pfn, node)->native_pin = true; + } + dma->fragments = provider; +} + +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn) +{ + struct vfio_fragment_provider *provider = dma->fragments; + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + struct user_page_fragment pin; + + if (WARN_ON(!count || vfio_fragment_native_pin(pfn, &pin))) + return; + if (count == 1) { + xa_erase(&provider->physical_pages, pfn); + provider->npinned--; + } else { + xa_store(&provider->physical_pages, pfn, xa_mk_value(count - 1), GFP_NOWAIT); + } + release_user_fragments(&pin, 1, dma->prot & IOMMU_WRITE); + if (provider->npinned < dma->locked_vm) + vfio_lock_acct(dma, -(long)(dma->locked_vm - provider->npinned), true); +} + /* Device unmaps and their IOTLB synchronization must precede this call. */ static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, unsigned long length) @@ -391,6 +531,25 @@ static int vfio_fragments_rw(struct vfio_dma *dma, unsigned long offset, void *d return ret; } #else +static struct vfio_fragment_provider * +vfio_fragments_prepare_native(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static void vfio_fragments_abort_native(struct vfio_fragment_provider *provider) +{ +} +static unsigned long vfio_fragments_native_pages(struct vfio_fragment_provider *provider) +{ + return 0; +} +static void vfio_fragments_commit_native(struct vfio_dma *dma, + struct vfio_fragment_provider *provider) +{ +} +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn) +{ +} static int vfio_fragments_init(struct vfio_dma *dma, size_t length) { return 0; diff --git a/drivers/vfio/vfio_iommu_type1_fragments_test.c b/drivers/vfio/vfio_iommu_type1_fragments_test.c index 7567c3802d2d5..ea34a0328f149 100644 --- a/drivers/vfio/vfio_iommu_type1_fragments_test.c +++ b/drivers/vfio/vfio_iommu_type1_fragments_test.c @@ -543,6 +543,125 @@ static void vfio_fragments_domain_granule_test(struct kunit *test) KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); } +static void vfio_fragments_native_owner_test(struct kunit *test) +{ + for (unsigned int variant = 0; variant < 9; variant++) { + bool same_mm = variant == 8; + bool mapped = same_mm || (variant & 1); + bool alias = same_mm || (variant & 2), different = variant & 4; + struct vfio_account_mm *old = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_account_mm *next = same_mm ? old : vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_fragment_provider *prepared; + struct file *file, *replacement; + struct vfio_dma *dma; + struct page *pages[2]; + phys_addr_t external[2]; + unsigned long unique, addr; + unsigned char data[32]; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x63); + replacement = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x44); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, replacement); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(old, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + if (alias) { + kthread_use_mm(old->mm); + addr = vm_mmap(file, VFIO_TEST_VA + PAGE_SIZE, PAGE_SIZE, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, 0); + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA + PAGE_SIZE); + } + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, old, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NULL(test, dma->fragments); + if (different) { + kthread_use_mm(old->mm); + addr = vm_mmap(replacement, VFIO_TEST_VA, 2 * PAGE_SIZE, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, 0); + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA); + } + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 17, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + for (unsigned int i = 0; i < 2; i++) + external[i] = page_to_phys(pages[i]); + if (mapped && different) + KUNIT_EXPECT_NE(test, external[0], + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + KUNIT_ASSERT_EQ(test, dma->locked_vm, 2UL); + if (!same_mm) { + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&old->mm->mm_users), 0); + } + + prepared = vfio_fragments_prepare_native(f->iommu, dma); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, prepared); + unique = mapped ? (alias ? 1UL : 2UL) : 0; + if (!mapped || different) + unique += different || !alias ? 2UL : 1UL; + KUNIT_EXPECT_EQ(test, vfio_fragments_native_pages(prepared), unique); + /* Failed new-owner credit must not publish metadata or release pins. */ + if (!same_mm && task_rlimit(dma->task, RLIMIT_MEMLOCK) != RLIM_INFINITY) { + unsigned long budget = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> + mm_page_shift(next->mm); + + next->mm->locked_vm = budget; + KUNIT_EXPECT_EQ(test, vfio_change_dma_mm_pages(dma, dma->task, + next->mm, false, unique), -ENOMEM); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, old->mm); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 2UL); + KUNIT_EXPECT_EQ(test, next->mm->locked_vm, budget); + next->mm->locked_vm = 0; + } + vfio_fragments_abort_native(prepared); + KUNIT_EXPECT_PTR_EQ(test, dma->fragments, NULL); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(page_folio(pages[0]))); + kthread_use_mm(next->mm); + ret = vfio_change_dma_owner(f->iommu, dma, + VFIO_TEST_VA + (same_mm ? SZ_4K : 0)); + kthread_unuse_mm(next->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, unique); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), same_mm ? unique : 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), unique); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, + memchr_inv(data, !mapped && different ? 0x44 : 0x63, sizeof(data)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 31, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + for (unsigned int i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, page_to_phys(pages[i]), external[i]); + KUNIT_EXPECT_EQ(test, dma->locked_vm, unique); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 31, 2); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 17, 2); + KUNIT_EXPECT_EQ(test, dma->locked_vm, + mapped ? (alias ? 1UL : 2UL) : unique); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 0UL); + kunit_info(test, "native ownership variant=%u preserves PIN identity and unique charge", + variant); + } +} + static struct kunit_case vfio_fragments_cases[] = { KUNIT_CASE(vfio_fragments_lifetime_test), KUNIT_CASE(vfio_fragments_lazy_test), @@ -551,6 +670,7 @@ static struct kunit_case vfio_fragments_cases[] = { KUNIT_CASE(vfio_fragments_replay_failure_test), KUNIT_CASE(vfio_fragments_anon_owner_test), KUNIT_CASE(vfio_fragments_domain_granule_test), + KUNIT_CASE(vfio_fragments_native_owner_test), {} }; diff --git a/drivers/vfio/vfio_iommu_type1_ioctl_test.c b/drivers/vfio/vfio_iommu_type1_ioctl_test.c index 1e8635f0f1a65..33e981bcdf74d 100644 --- a/drivers/vfio/vfio_iommu_type1_ioctl_test.c +++ b/drivers/vfio/vfio_iommu_type1_ioctl_test.c @@ -232,9 +232,27 @@ static long vfio_ioctl_test_ioctl(struct file *file, unsigned int request, ret = -ENOENT; } break; + case VFIO_TYPE1_TEST_EMULATED: + if (cmd.flags > 1 || f->external_refs) { + ret = -EBUSY; + break; + } + mutex_lock(&f->iommu->lock); + if (cmd.flags && list_empty(&f->group.next)) { + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + if (!f->group.pinned_page_dirty_scope) + f->iommu->num_non_pinned_groups++; + } else if (!cmd.flags && !list_empty(&f->group.next)) { + list_del_init(&f->group.next); + if (!f->group.pinned_page_dirty_scope) + f->iommu->num_non_pinned_groups--; + } + mutex_unlock(&f->iommu->lock); + break; case VFIO_TYPE1_TEST_STATE: cmd.locked_bytes = 0; - cmd.owner_locked_bytes = 0; + cmd.owner_locked_bytes = mm_locked_vm_bytes(current->mm); + cmd.flags = 0; cmd.owner_users = 0; mutex_lock(&f->iommu->lock); for (struct rb_node *node = rb_first(&f->iommu->dma_list); @@ -244,6 +262,8 @@ static long vfio_ioctl_test_ioctl(struct file *file, unsigned int request, cmd.locked_bytes += dma->locked_vm * PAGE_SIZE; cmd.owner_locked_bytes = mm_locked_vm_bytes(dma->mm); cmd.owner_users = atomic_read(&dma->mm->mm_users); + if (dma->fragments) + cmd.flags |= VFIO_TYPE1_TEST_STATE_FRAGMENTS; } cmd.native_page_size = PAGE_SIZE; cmd.minimum_page_size = 1UL << __ffs(f->iommu->pgsize_bitmap); diff --git a/drivers/vfio/vfio_iommu_type1_test.h b/drivers/vfio/vfio_iommu_type1_test.h index 07afd00c6be97..36c09030bcf0e 100644 --- a/drivers/vfio/vfio_iommu_type1_test.h +++ b/drivers/vfio/vfio_iommu_type1_test.h @@ -29,6 +29,8 @@ struct vfio_type1_test_cmd { #define VFIO_TYPE1_TEST_PIN _IOWR('V', 4, struct vfio_type1_test_cmd) #define VFIO_TYPE1_TEST_UNPIN _IOWR('V', 5, struct vfio_type1_test_cmd) #define VFIO_TYPE1_TEST_STATE _IOWR('V', 6, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_EMULATED _IOWR('V', 7, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_STATE_FRAGMENTS 1U #define VFIO_TYPE1_TEST_WRITE 1U #define VFIO_TYPE1_TEST_WORKER 2U #endif diff --git a/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c b/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c index cefc2f35c6fef..938cb9a1142f1 100644 --- a/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c +++ b/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c @@ -4,6 +4,7 @@ #define __user #include #include +#include #include #include #include @@ -14,6 +15,8 @@ #include #include #include +#include +#include #include #include #include @@ -107,6 +110,95 @@ static int all_bytes(const unsigned char *data, size_t size, unsigned char value return 1; } +static void native_owner(int fd, int backing, unsigned char *data, size_t native) +{ + struct vfio_type1_test_cmd cmd = {}; + struct vfio_iommu_type1_dma_unmap invalidate = { + .argsz = sizeof(invalidate), .flags = VFIO_DMA_UNMAP_FLAG_VADDR, + .iova = IOVA, .size = 2 * native, + }; + struct vfio_iommu_type1_dma_map update = { + .argsz = sizeof(update), .flags = VFIO_DMA_MAP_FLAG_VADDR, + .iova = IOVA, .size = 2 * native, .vaddr = (uintptr_t)data, + }; + struct __user_cap_header_struct header = { .version = _LINUX_CAPABILITY_VERSION_3 }; + struct __user_cap_data_struct saved[2], reduced[2]; + struct rlimit limit, empty; + unsigned char bytes[64]; + int file_flags = fcntl(backing, F_GETFD), status = 0; + pid_t child; + int budget_ready, budget_captured, transferred; + + result(!ioctl(fd, VFIO_TYPE1_TEST_EMULATED, &cmd) && + !domain(fd, 0, 1, 4096, 0), "VADDR fixture retains production group restrictions"); + result(file_flags >= 0 && !fcntl(backing, F_SETFD, file_flags & ~FD_CLOEXEC), + "native source inherits exact backing descriptor"); + child = fork(); + if (!child) { + char fd_arg[24], file_arg[24], native_arg[24]; + + snprintf(fd_arg, sizeof(fd_arg), "%d", fd); + snprintf(file_arg, sizeof(file_arg), "%d", backing); + snprintf(native_arg, sizeof(native_arg), "%zu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL)) + _exit(98); + execl("/init", "/init", "--native-source", fd_arg, file_arg, native_arg, NULL); + _exit(99); + } + if (file_flags >= 0) + fcntl(backing, F_SETFD, file_flags); + result(child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && !state(fd, &cmd) && + !cmd.owner_users && !(cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && cmd.mapped_bytes[0] == 2 * native, + "exited native source owns legacy GUP pins before transfer"); + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &invalidate), + "canonical VADDR invalidation leaves device translations alive"); + budget_captured = !getrlimit(RLIMIT_MEMLOCK, &limit) && + !syscall(SYS_capget, &header, saved); + budget_ready = budget_captured; + if (budget_ready) { + memcpy(reduced, saved, sizeof(saved)); + reduced[CAP_IPC_LOCK / 32].effective &= ~(1U << (CAP_IPC_LOCK % 32)); + empty = (struct rlimit) { .rlim_cur = 0, .rlim_max = limit.rlim_max }; + budget_ready = !setrlimit(RLIMIT_MEMLOCK, &empty) && + !syscall(SYS_capset, &header, reduced); + } + errno = 0; + result(budget_ready && ioctl(fd, VFIO_IOMMU_MAP_DMA, &update) < 0 && errno == ENOMEM && + !state(fd, &cmd) && !cmd.owner_users && + !(cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && cmd.mapped_bytes[0] == 2 * native, + "failed owner credit preserves native pins, metadata and charge"); + if (budget_captured) { + result(!syscall(SYS_capset, &header, saved) && !setrlimit(RLIMIT_MEMLOCK, &limit), + "restore original memlock budget and effective capability"); + } else { + result(0, "owner budget setup failed"); + } + transferred = !ioctl(fd, VFIO_IOMMU_MAP_DMA, &update) && !state(fd, &cmd) && + cmd.owner_users && (cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && cmd.owner_locked_bytes == 2 * native; + result(transferred, "canonical VADDR retry adopts native pins for alternative owner"); + /* Dependent CPU access is prohibited while VADDR remains invalid. */ + if (!transferred) + return; + result(!rw(fd, 0, bytes, sizeof(bytes), 0) && all_bytes(bytes, sizeof(bytes), 0x64), + "new owner accesses retained native buffer with its own mm authorization"); + cmd = (struct vfio_type1_test_cmd) { .flags = 1 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_EMULATED, &cmd), + "external consumer rejoins after VADDR update"); + cmd = (struct vfio_type1_test_cmd) { .iova = IOVA + 17 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == 2 * native && cmd.external_refs == 1, + "external page array reuses adopted pins without new charge"); + result(!unmap(fd, 2 * native) && !state(fd, &cmd) && + !cmd.locked_bytes && !cmd.owner_locked_bytes && !cmd.external_refs && + !cmd.mapped_bytes[0], + "owner transfer final cleanup refunds recipient and clears domain"); + result(!domain(fd, 0, 0, 0, 0), "owner transfer domain detaches"); +} + static int exercise(size_t native) { struct vfio_type1_test_cmd cmd; @@ -203,6 +295,8 @@ static int exercise(size_t native) "source-exit cleanup refunds pins and clears all translations"); result(!domain(fd, 1, 0, 0, 0) && !domain(fd, 0, 0, 0, 0), "both domains detach after complete cleanup"); + if (getauxval(AT_PAGESZ) != native) + native_owner(fd, backing, data, native); out: if (data != MAP_FAILED) munmap(data, 2 * native); @@ -217,6 +311,16 @@ static int exercise(size_t native) int main(int argc, char **argv) { setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--native-source")) { + size_t native = strtoul(argv[4], NULL, 10); + int fd = atoi(argv[2]), backing = atoi(argv[3]); + void *data; + + if (getauxval(AT_PAGESZ) != native) + return 2; + data = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, MAP_SHARED, backing, 0); + return data == MAP_FAILED || map(fd, data, 2 * native) ? 1 : 0; + } if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { size_t expected = strtoul(argv[3], NULL, 10); From 0249bee12ac11a1802764260bb9cc6337e8770f6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 2 Oct 2026 20:46:35 -0400 Subject: [PATCH 0371/1579] vfio: retain external page pins across fine DMA mappings A native external page array may cover several adjacent fine DMA mappings. Validate the complete physical page and keep a reference with an exact byte extent in each DMA owner. Unwind only references acquired by a failed request, and drain all constituents when a consumer responds to DMA unmap notification. Bound dirty bitmap updates and cached-pin protection by those exact extents. Keep ordinary native mappings on their existing GUP path. Add lifecycle and negative KUnit cases plus real userspace MAP/UNMAP span tests across all five A64 granule configurations on native16K/native64K recording-domain kernels. --- Documentation/arch/arm64/user4k.rst | 45 +++- drivers/vfio/vfio_iommu_type1.c | 223 +++++++++--------- drivers/vfio/vfio_iommu_type1_fragments.c | 41 +++- .../vfio/vfio_iommu_type1_fragments_test.c | 130 ++++++++++ .../arm64/user4k/vfio-fixture-init.c | 69 ++++++ 5 files changed, 379 insertions(+), 129 deletions(-) diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index fc1e3ca97306f..a0baa7d9c8abf 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -4321,8 +4321,8 @@ use temporary GET references. A typed dirty helper records writes without releasing cached pins. External native-page arrays accept complete, aligned, physically contiguous native spans and preserve callers' original byte offsets. Layouts that need an explicit physical offset or cannot be represented -by one native page pointer remain rejected; spans across separate fine-grained -DMA mappings still need aggregate ownership support. +by one native page pointer remain rejected. Aggregate ownership across separate +fine-grained DMA mappings is qualified in the later external-span section. The container advertises the composed minimum supported by attached domains, with a 4K floor while the mixed-page feature is enabled. Software dirty @@ -4402,8 +4402,9 @@ pass on the fixture-enabled kernels. Evidence is under user4k-vm-state/vfio-ioctl-v1/. These tests qualify the actual EL0-to-type1 provider/ABI path with supplied domains, not the VFIO core group attachment protocol, physical DMA, Apple boot, GPU execution or game throughput. -Native-to-alternative VADDR ownership conversion and external spans crossing -multiple fine DMA mappings remain unfinished. +At this checkpoint, native-to-alternative VADDR ownership conversion and +external spans crossing multiple fine DMA mappings were unfinished; the later +sections record their implementation and bounded qualification. Native VFIO VADDR ownership conversion (2026-10-02) @@ -4460,3 +4461,39 @@ module build/load/unload checks also pass. Evidence is under user4k-vm-state/vfio-owner-v1/. This qualifies the tested provider ownership paths with supplied domains, not concurrent physical-device attachment/DMA, Apple GPU execution, game performance or the remaining full ABI requirements. + + +VFIO native-page arrays across fine DMA mappings (2026-10-02) +------------------------------------------------------------ + +An external native-page array may now span several adjacent fine-grained DMA +mappings. Each constituent retains a reference in its own DMA owner, with its +exact byte extent and existing per-provider physical-page accounting. Every +byte must resolve to the same native physical page at the corresponding native +offset. Missing mappings, denied permissions, discontinuous physical pages and +incompatible offsets reject the request and unwind all references acquired by +that request. Previously held references remain intact. Ordinary native-aligned +mappings keep their existing native GUP path. + +Unpin walks the same constituents under the container lock. Unmapping any +constituent invokes the existing device notification contract: a consumer must +release the native-page reference, including all of its constituent references, +before that mapping can disappear. Exact extents also bound dirty bitmap writes +and determine whether a cached pin is still protected by an external consumer. +A physical page shared by independent DMA providers remains charged separately +to each provider; this does not add cross-provider charge deduplication. + +The new KUnit case passes lazy/mapped spans with separate 4K/native mm owners, +source-mm exit, duplicate references, failed larger requests, reverse-order page +release, fine dirty bitmaps and constituent notification/cleanup. Four negative +variants cover a hole, permission denial, discontinuous backing and wrong native +offset. The type1 suites total eleven passes and zero failures on each native16K +and native64K kernel. + +Real EL0 creates a native-page span with independent 4K MAP ioctls, pins before +source exit and reuses all cached constituents after exit. Lazy and mapped +variants check each provider's charge, fine dirty bits, duplicate references and +UNMAP of one constituent followed by full cleanup. These add sixteen assertions +to each tested A64 ABI: native16K/4K and native64K/4K/16K. The supplied domains +record translations; these results do not qualify physical device DMA, concurrent +type1 consumers, Apple boot/GPU execution or game throughput. diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 8e3310487b553..895bec69ce8d0 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -134,6 +134,7 @@ struct vfio_pfn { struct rb_node node; dma_addr_t iova; /* Device address */ unsigned long pfn; /* Host pfn */ + unsigned long length; /* Bytes retained in this DMA */ unsigned int ref_count; bool native_pin; /* separately adopted legacy external PIN */ }; @@ -289,7 +290,7 @@ static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize) struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, node); bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, - PAGE_SIZE >> pgshift); + vpfn->length >> pgshift); } } @@ -400,7 +401,7 @@ static void vfio_unlink_pfn(struct vfio_dma *dma, struct vfio_pfn *old) } static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova, - unsigned long pfn) + unsigned long pfn, unsigned long length) { struct vfio_pfn *vpfn; @@ -410,6 +411,7 @@ static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova, vpfn->iova = iova; vpfn->pfn = pfn; + vpfn->length = length; vpfn->ref_count = 1; vfio_link_pfn(dma, vpfn); return 0; @@ -440,6 +442,7 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) if (!vpfn->ref_count) { if (dma->fragments) { unsigned long offset = vpfn->iova - dma->iova; + unsigned long length = vpfn->length; unsigned long pfn = vpfn->pfn; bool native_pin = vpfn->native_pin; @@ -447,7 +450,7 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) vfio_remove_from_pfn_list(dma, vpfn); if (native_pin) vfio_fragments_put_native_pin(dma, pfn); - vfio_fragments_release_range(dma, offset, PAGE_SIZE); + vfio_fragments_release_range(dma, offset, length); } else { ret = put_pfn(vpfn->pfn, dma->prot); vfio_remove_from_pfn_list(dma, vpfn); @@ -931,6 +934,78 @@ static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, return unlocked; } +/* A native page may contain independently owned fine-grained DMA areas. */ +static unsigned long vfio_unpin_external_range(struct vfio_iommu *iommu, + dma_addr_t start, unsigned long length, + bool do_accounting) +{ + unsigned long done = 0; + + while (done < length) { + dma_addr_t iova = start + done; + struct vfio_dma *dma = vfio_find_dma(iommu, iova, 1); + unsigned long bytes; + + if (!dma || iova < dma->iova) + break; + bytes = min_t(unsigned long, length - done, + PAGE_SIZE - offset_in_page(iova)); + bytes = min(bytes, dma->size - (iova - dma->iova)); + vfio_unpin_page_external(dma, iova, do_accounting); + done += bytes; + } + return done; +} + +static int vfio_pin_external_segment(struct vfio_iommu *iommu, + struct vfio_dma *dma, dma_addr_t iova, unsigned long length, + int prot, bool do_accounting, unsigned long *pfn) +{ + struct vfio_pfn *vpfn; + int ret; + + if ((dma->prot & prot) != prot) + return -EPERM; + vpfn = vfio_iova_get_vfio_pfn(dma, iova); + if (vpfn) { + if (vpfn->length != length) { + vfio_unpin_page_external(dma, iova, do_accounting); + return -EINVAL; + } + *pfn = vpfn->pfn; + } else if (dma->fragments) { + ret = vfio_fragments_external(dma, iova, length, pfn); + if (ret) + return ret; + } else { + /* Ordinary native mappings still use one native GUP per page. */ + if (length != PAGE_SIZE) + return -EINVAL; + ret = vfio_pin_page_external(dma, dma->vaddr + iova - dma->iova, + pfn, do_accounting); + if (ret) + return ret; + if (!pfn_valid(*pfn)) { + ret = -EINVAL; + goto put_native; + } + ret = vfio_add_to_pfn_list(dma, iova, *pfn, length); + if (ret) + goto put_native; + } + if (iommu->dirty_page_tracking) { + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); + + bitmap_set(dma->bitmap, (iova - dma->iova) >> pgshift, + length >> pgshift); + } + return 0; +put_native: + if (put_pfn(*pfn, dma->prot) && do_accounting) + vfio_lock_acct(dma, -1, true); + return ret; +} + static int vfio_iommu_type1_pin_pages(void *iommu_data, struct iommu_group *iommu_group, dma_addr_t user_iova, @@ -939,131 +1014,72 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, { struct vfio_iommu *iommu = iommu_data; struct vfio_iommu_group *group; - int i, j, ret; - unsigned long remote_vaddr; - struct vfio_dma *dma; + unsigned long pinned = 0; bool do_accounting; dma_addr_t iova_end; size_t iova_size; + int i, ret; if (!iommu || !pages || npage <= 0) return -EINVAL; /* Callers recover the original byte offset from the returned page. */ user_iova = ALIGN_DOWN(user_iova, PAGE_SIZE); - - /* Supported for v2 version only */ if (!iommu->v2) return -EACCES; - if (check_mul_overflow(npage, PAGE_SIZE, &iova_size) || check_add_overflow(user_iova, iova_size - 1, &iova_end)) return -EOVERFLOW; mutex_lock(&iommu->lock); - if (WARN_ONCE(iommu->vaddr_invalid_count, "vfio_pin_pages not allowed with VFIO_UPDATE_VADDR\n")) { ret = -EBUSY; goto pin_done; } - - /* Fail if no dma_umap notifier is registered */ if (list_empty(&iommu->device_list)) { ret = -EINVAL; goto pin_done; } - - /* - * If iommu capable domain exist in the container then all pages are - * already pinned and accounted. Accounting should be done if there is no - * iommu capable domain in the container. - */ do_accounting = list_empty(&iommu->domain_list); - for (i = 0; i < npage; i++) { - unsigned long phys_pfn; - dma_addr_t iova; - struct vfio_pfn *vpfn; - - iova = user_iova + PAGE_SIZE * i; - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - if (!dma || iova < dma->iova || - dma->size - (iova - dma->iova) < PAGE_SIZE) { - ret = -EINVAL; - goto pin_unwind; - } - - if ((dma->prot & prot) != prot) { - ret = -EPERM; - goto pin_unwind; - } + unsigned long done = 0; - vpfn = vfio_iova_get_vfio_pfn(dma, iova); - if (vpfn) { - pages[i] = pfn_to_page(vpfn->pfn); - continue; - } + pages[i] = NULL; + while (done < PAGE_SIZE) { + dma_addr_t iova = user_iova + PAGE_SIZE * i + done; + struct vfio_dma *dma = vfio_find_dma(iommu, iova, 1); + unsigned long length, pfn; - if (dma->fragments) { - ret = vfio_fragments_external(dma, iova, &phys_pfn); + if (!dma || iova < dma->iova) { + ret = -EINVAL; + goto pin_unwind; + } + length = min_t(unsigned long, PAGE_SIZE - done, + dma->size - (iova - dma->iova)); + ret = vfio_pin_external_segment(iommu, dma, iova, length, + prot, do_accounting, &pfn); if (ret) goto pin_unwind; - pages[i] = pfn_to_page(phys_pfn); - goto dirty; - } - - remote_vaddr = dma->vaddr + (iova - dma->iova); - ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn, - do_accounting); - if (ret) - goto pin_unwind; - - if (!pfn_valid(phys_pfn)) { - ret = -EINVAL; - goto pin_unwind; - } - - ret = vfio_add_to_pfn_list(dma, iova, phys_pfn); - if (ret) { - if (put_pfn(phys_pfn, dma->prot) && do_accounting) - vfio_lock_acct(dma, -1, true); - goto pin_unwind; - } - - pages[i] = pfn_to_page(phys_pfn); -dirty: - if (iommu->dirty_page_tracking) { - unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); - - /* - * Bitmap populated with the smallest supported page - * size - */ - bitmap_set(dma->bitmap, - (iova - dma->iova) >> pgshift, - PAGE_SIZE >> pgshift); + pinned += length; + if (pages[i] && page_to_pfn(pages[i]) != pfn) { + ret = -EINVAL; + goto pin_unwind; + } + pages[i] = pfn_to_page(pfn); + done += length; } } - ret = i; - + ret = npage; group = vfio_iommu_find_iommu_group(iommu, iommu_group); if (!group->pinned_page_dirty_scope) { group->pinned_page_dirty_scope = true; iommu->num_non_pinned_groups--; } - goto pin_done; - pin_unwind: - pages[i] = NULL; - for (j = 0; j < i; j++) { - dma_addr_t iova; - - iova = user_iova + PAGE_SIZE * j; - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - vfio_unpin_page_external(dma, iova, do_accounting); - pages[j] = NULL; - } + vfio_unpin_external_range(iommu, user_iova, pinned, do_accounting); + while (i >= 0) + pages[i--] = NULL; pin_done: mutex_unlock(&iommu->lock); return ret; @@ -1073,40 +1089,21 @@ static void vfio_iommu_type1_unpin_pages(void *iommu_data, dma_addr_t user_iova, int npage) { struct vfio_iommu *iommu = iommu_data; - bool do_accounting; dma_addr_t iova_end; size_t iova_size; - int i; - - /* Supported for v2 version only */ - if (WARN_ON(!iommu->v2)) - return; + unsigned long unpinned; - if (WARN_ON(npage <= 0)) + if (WARN_ON(!iommu->v2) || WARN_ON(npage <= 0)) return; - user_iova = ALIGN_DOWN(user_iova, PAGE_SIZE); if (WARN_ON(check_mul_overflow(npage, PAGE_SIZE, &iova_size) || check_add_overflow(user_iova, iova_size - 1, &iova_end))) return; - mutex_lock(&iommu->lock); - - do_accounting = list_empty(&iommu->domain_list); - for (i = 0; i < npage; i++) { - dma_addr_t iova = user_iova + PAGE_SIZE * i; - struct vfio_dma *dma; - - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - if (!dma) - break; - - vfio_unpin_page_external(dma, iova, do_accounting); - } - + unpinned = vfio_unpin_external_range(iommu, user_iova, iova_size, + list_empty(&iommu->domain_list)); mutex_unlock(&iommu->lock); - - WARN_ON(i != npage); + WARN_ON(unpinned != iova_size); } static long vfio_sync_unpin(struct vfio_dma *dma, struct vfio_domain *domain, diff --git a/drivers/vfio/vfio_iommu_type1_fragments.c b/drivers/vfio/vfio_iommu_type1_fragments.c index 610a368a6193e..22d46a1b805a7 100644 --- a/drivers/vfio/vfio_iommu_type1_fragments.c +++ b/drivers/vfio/vfio_iommu_type1_fragments.c @@ -231,6 +231,24 @@ static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pf vfio_lock_acct(dma, -(long)(dma->locked_vm - provider->npinned), true); } +static bool vfio_fragments_external_overlap(struct vfio_dma *dma, + dma_addr_t start, unsigned long length) +{ + struct rb_node *node = dma->pfn_list.rb_node; + + while (node) { + struct vfio_pfn *vpfn = rb_entry(node, struct vfio_pfn, node); + + if (start + length <= vpfn->iova) + node = node->rb_left; + else if (start >= vpfn->iova + vpfn->length) + node = node->rb_right; + else + return true; + } + return false; +} + /* Device unmaps and their IOTLB synchronization must precede this call. */ static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, unsigned long length) @@ -251,9 +269,7 @@ static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long sta break; node = rb_next(node); /* Native-page consumers retain all records overlapping their page. */ - if (vfio_find_vpfn_range(dma, iova >= PAGE_SIZE - 1 ? - iova - (PAGE_SIZE - 1) : 0, - iova + record->pin.length)) + if (vfio_fragments_external_overlap(dma, iova, record->pin.length)) continue; pfn = PHYS_PFN(vfio_fragment_phys(&record->pin)); count = xa_to_value(xa_load(&provider->physical_pages, pfn)); @@ -437,18 +453,18 @@ static void vfio_fragments_destroy(struct vfio_dma *dma) dma->fragments = NULL; } -/* The output page has no offset field: validate every byte in its native span. */ +/* The output page has no offset field: validate every byte and its native offset. */ static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, - unsigned long *pfn) + unsigned long length, unsigned long *pfn) { unsigned long first = iova - dma->iova, done = 0; phys_addr_t phys = 0; int ret; - ret = vfio_fragments_acquire(dma, first, PAGE_SIZE); + ret = vfio_fragments_acquire(dma, first, length); if (ret) return ret; - while (done < PAGE_SIZE) { + while (done < length) { struct vfio_fragment *record = vfio_fragment_find(dma, first + done); phys_addr_t next; unsigned long offset; @@ -461,17 +477,17 @@ static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, next = vfio_fragment_phys(&record->pin) + offset; if (!done) phys = next; - if (!IS_ALIGNED(phys, PAGE_SIZE) || next != phys + done) { + if (offset_in_page(phys) != offset_in_page(iova) || next != phys + done) { ret = -EINVAL; goto out; } - done += min_t(unsigned long, record->pin.length - offset, PAGE_SIZE - done); + done += min_t(unsigned long, record->pin.length - offset, length - done); } *pfn = PHYS_PFN(phys); - ret = vfio_add_to_pfn_list(dma, iova, *pfn); + ret = vfio_add_to_pfn_list(dma, iova, *pfn, length); out: if (ret) - vfio_fragments_release_range(dma, first, PAGE_SIZE); + vfio_fragments_release_range(dma, first, length); return ret; } @@ -580,7 +596,8 @@ static void vfio_fragments_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) static void vfio_fragments_destroy(struct vfio_dma *dma) { } -static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, unsigned long *pfn) +static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, + unsigned long length, unsigned long *pfn) { return -EOPNOTSUPP; } diff --git a/drivers/vfio/vfio_iommu_type1_fragments_test.c b/drivers/vfio/vfio_iommu_type1_fragments_test.c index ea34a0328f149..169ad50bfe35e 100644 --- a/drivers/vfio/vfio_iommu_type1_fragments_test.c +++ b/drivers/vfio/vfio_iommu_type1_fragments_test.c @@ -662,6 +662,135 @@ static void vfio_fragments_native_owner_test(struct kunit *test) } } +static int vfio_fragment_map_area(struct vfio_fragment_fixture *f, + struct vfio_account_mm *ctx, unsigned long iova_offset, + unsigned long va_offset, unsigned long length, unsigned int flags) +{ + struct vfio_iommu_type1_dma_map map = { + .argsz = sizeof(map), + .flags = flags, + .vaddr = VFIO_TEST_VA + va_offset, + .iova = VFIO_TEST_IOVA + iova_offset, + .size = length, + }; + int ret; + + kthread_use_mm(ctx->mm); + ret = vfio_dma_do_map(f->iommu, &map); + kthread_unuse_mm(ctx->mm); + return ret; +} + +static void vfio_fragments_external_span_test(struct kunit *test) +{ + unsigned int rw = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_account_mm *a = vfio_account_mm_new(test, 12); + struct vfio_account_mm *b = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x6b); + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + struct vfio_dma *first, *rest; + struct page *pages[3]; + unsigned long bits = 2 * PAGE_SIZE / SZ_4K - 1; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(a, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(b, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, a, 0, 0, SZ_4K, rw), 0); + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, b, SZ_4K, SZ_4K, + 2 * PAGE_SIZE - SZ_4K, rw), 0); + first = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + rest = vfio_find_dma(f->iommu, VFIO_TEST_IOVA + SZ_4K, 1); + KUNIT_ASSERT_NOT_NULL(test, first); + KUNIT_ASSERT_NOT_NULL(test, rest); + f->iommu->dirty_page_tracking = true; + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, SZ_4K), 0); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 37, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + KUNIT_EXPECT_EQ(test, first->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, rest->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, bitmap_weight(first->bitmap, 1), 1); + KUNIT_EXPECT_EQ(test, bitmap_weight(rest->bitmap, bits), (int)bits); + bitmap_clear(rest->bitmap, 0, bits); + vfio_dma_populate_bitmap(rest, SZ_4K); + KUNIT_EXPECT_EQ(test, bitmap_weight(rest->bitmap, bits), (int)bits); + if (mapped) + KUNIT_EXPECT_EQ(test, page_to_phys(pages[0]), + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + /* A failed larger request must leave prior caller references intact. */ + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 3, IOMMU_READ, pages), -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, pages[0], NULL); + KUNIT_EXPECT_PTR_EQ(test, pages[1], NULL); + KUNIT_EXPECT_PTR_EQ(test, pages[2], NULL); + a->live = b->live = false; + mmput(a->mm); + mmput(b->mm); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 2, IOMMU_READ, pages), 2); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 2); + /* Release the second page first: it shares the second DMA owner. */ + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + PAGE_SIZE + 7, 1); + KUNIT_EXPECT_EQ(test, rest->locked_vm, mapped ? 2UL : 1UL); + f->external_refs = 1; + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, f->notifications, 1U); + KUNIT_EXPECT_EQ(test, f->external_refs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(a->mm), 0UL); + KUNIT_EXPECT_EQ(test, rest->locked_vm, mapped ? 2UL : 0UL); + unmap.iova += SZ_4K; + unmap.size = 2 * PAGE_SIZE - SZ_4K; + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(b->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + } + for (unsigned int failure = 0; failure < 4; failure++) { + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x3a); + struct page *page = NULL; + struct rb_node *node; + unsigned long va = failure == 2 ? PAGE_SIZE + SZ_4K : SZ_4K; + int expected = failure == 1 ? -EPERM : -EINVAL; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, 0, 0, SZ_4K, rw), 0); + if (failure == 3) + va += SZ_4K; + if (failure != 0) + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, SZ_4K, va, + SZ_4K, failure == 1 ? VFIO_DMA_MAP_FLAG_READ : rw), 0); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 1, + IOMMU_READ | IOMMU_WRITE, &page), expected); + KUNIT_EXPECT_PTR_EQ(test, page, NULL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + for (node = rb_first(&f->iommu->dma_list); node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->pfn_list)); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->fragments->records)); + } + kunit_release_action(test, vfio_fragment_fixture_free, f); + } +} + static struct kunit_case vfio_fragments_cases[] = { KUNIT_CASE(vfio_fragments_lifetime_test), KUNIT_CASE(vfio_fragments_lazy_test), @@ -671,6 +800,7 @@ static struct kunit_case vfio_fragments_cases[] = { KUNIT_CASE(vfio_fragments_anon_owner_test), KUNIT_CASE(vfio_fragments_domain_granule_test), KUNIT_CASE(vfio_fragments_native_owner_test), + KUNIT_CASE(vfio_fragments_external_span_test), {} }; diff --git a/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c b/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c index 938cb9a1142f1..fc8e2d9ed8a8e 100644 --- a/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c +++ b/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c @@ -110,6 +110,74 @@ static int all_bytes(const unsigned char *data, size_t size, unsigned char value return 1; } +static void external_span(int backing, size_t native) +{ + const size_t fine = 4096, count = native / fine; + uint64_t bits; + + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_dma_unmap remove = { + .argsz = sizeof(remove), .iova = IOVA, .size = fine, + }; + int fd = open("/dev/vfio-test", O_RDWR), status = 0; + pid_t child; + + result(fd >= 0 && (!mapped || !domain(fd, 0, 1, fine, 0)), + "span fixture opens lazy or mapped type1 container"); + if (fd < 0) + continue; + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), + "span dirty tracking starts before fine mappings"); + child = fork(); + if (!child) { + unsigned char *data = mmap(NULL, native, PROT_READ | PROT_WRITE, + MAP_SHARED, backing, 0); + + if (data == MAP_FAILED) + _exit(1); + for (size_t i = 0; i < count; i++) { + struct vfio_iommu_type1_dma_map area = { + .argsz = sizeof(area), + .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, + .iova = IOVA + i * fine, + .vaddr = (uintptr_t)data + i * fine, + .size = fine, + }; + + if (ioctl(fd, VFIO_IOMMU_MAP_DMA, &area)) + _exit(2); + } + cmd = (struct vfio_type1_test_cmd) { .iova = IOVA + 17 }; + _exit(ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) ? 3 : 0); + } + result(child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status), + "source pins native page across independent fine MAP ioctls"); + result(!state(fd, &cmd) && !cmd.owner_users && cmd.external_refs == 1 && + cmd.locked_bytes == count * native && + cmd.owner_locked_bytes == count * native, + "every fine DMA area retains its own charge after source exit"); + cmd = (struct vfio_type1_test_cmd) { .iova = IOVA + 31 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.external_refs == 2 && cmd.locked_bytes == count * native, + "cross-area duplicate pin reuses all cached constituent pins"); + result(!bitmap(fd, native, &bits) && bits == ((1ULL << count) - 1), + "cross-area dirty bitmap marks only each mapping's fine extent"); + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &remove) && remove.size == fine && + !state(fd, &cmd) && !cmd.external_refs && cmd.notifications == 1 && + cmd.locked_bytes == (mapped ? (count - 1) * native : 0), + "unmapping one constituent drains references in all page constituents"); + remove.iova = IOVA + fine; + remove.size = native - fine; + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &remove) && + remove.size == native - fine && !state(fd, &cmd) && + !cmd.locked_bytes && !cmd.mapped_bytes[0], + "remaining fine mappings unmap without pins or translations"); + close(fd); + } +} + static void native_owner(int fd, int backing, unsigned char *data, size_t native) { struct vfio_type1_test_cmd cmd = {}; @@ -295,6 +363,7 @@ static int exercise(size_t native) "source-exit cleanup refunds pins and clears all translations"); result(!domain(fd, 1, 0, 0, 0) && !domain(fd, 0, 0, 0, 0), "both domains detach after complete cleanup"); + external_span(backing, native); if (getauxval(AT_PAGESZ) != native) native_owner(fd, backing, data, native); out: From 4426aa459a636b7da282eaabb29144e537a7edf1 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 21:03:28 -0400 Subject: [PATCH 0372/1579] usb: add verified M5 J813 USB3 host support --- .../devicetree/bindings/phy/apple,atcphy.yaml | 1 + .../devicetree/bindings/spmi/apple,spmi.yaml | 1 + .../devicetree/bindings/usb/apple,dwc3.yaml | 1 + .../bindings/usb/apple,sn201202x.yaml | 8 +- arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi | 323 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + arch/arm64/configs/j813.config | 26 ++ drivers/phy/apple/atc.c | 38 ++- drivers/spmi/spmi-apple-controller.c | 62 +++- drivers/usb/typec/tipd/core.c | 9 +- drivers/usb/typec/tipd/spmi.c | 109 ++++-- drivers/usb/typec/tipd/tps6598x.h | 3 + 12 files changed, 520 insertions(+), 62 deletions(-) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml index c42b9c5684296..2484121870400 100644 --- a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -78,6 +78,7 @@ properties: - const: apple,t8103-atcphy - const: apple,t8103-atcphy - const: apple,t8140-atcphy + - const: apple,t8142-atcphy reg: minItems: 5 diff --git a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml index f0f7d21d031ca..5a52f808b6f5b 100644 --- a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml +++ b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml @@ -34,6 +34,7 @@ properties: - apple,t6000-spmi - apple,t8112-spmi - const: apple,spmi + - const: apple,t8142-spmi reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml index 44e8b33576dda..9dc0adc535bea 100644 --- a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml @@ -25,6 +25,7 @@ properties: - apple,t6020-dwc3 - apple,t8112-dwc3 - apple,t8140-dwc3 + - apple,t8142-dwc3 - const: apple,t8103-dwc3 - const: apple,t8103-dwc3 diff --git a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml index 133a71bbedb7c..9b1a566fd1437 100644 --- a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml +++ b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml @@ -15,6 +15,8 @@ description: | A variant of tps6598x controller also known as Apple ACE3 present on hardware with Apple SoCs starting from the M3. + The controller can be polled when interrupt resources are not supplied. + properties: compatible: enum: @@ -43,8 +45,10 @@ properties: required: - compatible - reg - - interrupts - - interrupt-names + +dependentRequired: + interrupts: [interrupt-names] + interrupt-names: [interrupts] additionalProperties: false diff --git a/arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi new file mode 100644 index 0000000000000..e27e40c04e268 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +#include +#include + +/ { + aliases { + atcphy0 = &atcphy0; + usb0 = &dwc3_0; + atcphy1 = &atcphy1; + usb1 = &dwc3_1; + }; +}; + +&pmgr { + ps_atc0_common: power-controller@2c8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0-common"; + }; +}; + +&{/soc} { + pmgr_nub: power-management@388280000 { + compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <3 0x88280000 0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + + ps_nub_spmi_a0: power-controller@70 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x70 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub-spmi-a0"; + }; + + ps_atc0_usb_aon: power-controller@c8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0-usb-aon"; + }; + + ps_atc0_usb: power-controller@d0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xd0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc0_common>, <&ps_atc0_usb_aon>; + label = "atc0-usb"; + }; + }; + + nub_spmi_a0: spmi@388908000 { + compatible = "apple,t8142-spmi"; + reg = <3 0x88908000 0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + power-domains = <&ps_nub_spmi_a0>; + + hpm0: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + + typec0: connector { + compatible = "usb-c-connector"; + label = "USB-C Left-front"; + power-role = "dual"; + data-role = "host"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec0_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + + port@1 { + reg = <1>; + typec0_ss: endpoint { + remote-endpoint = <&atcphy0_typec>; + }; + }; + }; + }; + }; + }; + + dwc3_0: usb@402280000 { + compatible = "apple,t8142-dwc3", "apple,t8103-dwc3"; + reg = <4 0x02280000 0 0xcd00>, <4 0x0228cd00 0 0x4b00>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupts = ; + dr_mode = "host"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + resets = <&atcphy0>; + phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb>; + }; + }; + }; + }; + + dwc3_0_dart_0: iommu@402f00000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x02f00000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + dwc3_0_dart_1: iommu@402f80000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x02f80000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + atcphy0: phy@403000000 { + compatible = "apple,t8142-atcphy"; + reg = <4 0x03000000 0 0x4c000>, + <4 0x03050000 0 0x8000>, + <4 0x00000000 0 0x8000>, + <4 0x02a90000 0 0x4000>, + <4 0x02a84000 0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", "pipehandler"; + #phy-cells = <1>; + #reset-cells = <0>; + orientation-switch; + mode-switch; + power-domains = <&ps_atc0_usb>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + atcphy0_typec: endpoint { + remote-endpoint = <&typec0_ss>; + }; + }; + port@1 { + reg = <1>; + atcphy0_usb: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; + }; +}; + +&pmgr { + ps_atc1_common: power-controller@2d0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1-common"; + }; +}; + +&pmgr_nub { + ps_atc1_usb_aon: power-controller@d8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xd8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1-usb-aon"; + }; + + ps_atc1_usb: power-controller@e0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xe0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc1_common>, <&ps_atc1_usb_aon>; + label = "atc1-usb"; + }; +}; + +&nub_spmi_a0 { + hpm1: usb-pd@a { + compatible = "apple,sn201202x"; + reg = <0xa SPMI_USID>; + + typec1: connector { + compatible = "usb-c-connector"; + label = "USB-C Left-rear"; + power-role = "dual"; + data-role = "host"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec1_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + + port@1 { + reg = <1>; + typec1_ss: endpoint { + remote-endpoint = <&atcphy1_typec>; + }; + }; + }; + }; + }; +}; + +&{/soc} { + dwc3_1: usb@40a280000 { + compatible = "apple,t8142-dwc3", "apple,t8103-dwc3"; + reg = <4 0x0a280000 0 0xcd00>, <4 0x0a28cd00 0 0x4b00>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupts = ; + dr_mode = "host"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_1_dart_0 0>, <&dwc3_1_dart_1 1>; + power-domains = <&ps_atc1_usb>; + resets = <&atcphy1>; + phys = <&atcphy1 PHY_TYPE_USB2>, <&atcphy1 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb>; + }; + }; + }; + }; + + dwc3_1_dart_0: iommu@40af00000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x0af00000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc1_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + dwc3_1_dart_1: iommu@40af80000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x0af80000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc1_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + atcphy1: phy@40b000000 { + compatible = "apple,t8142-atcphy"; + reg = <4 0x0b000000 0 0x4c000>, + <4 0x0b050000 0 0x8000>, + <4 0x08000000 0 0x8000>, + <4 0x0aa90000 0 0x4000>, + <4 0x0aa84000 0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", "pipehandler"; + #phy-cells = <1>; + #reset-cells = <0>; + orientation-switch; + mode-switch; + power-domains = <&ps_atc1_usb>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + atcphy1_typec: endpoint { + remote-endpoint = <&typec1_ss>; + }; + }; + port@1 { + reg = <1>; + atcphy1_usb: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index b0a8ac8339955..be04fd061a885 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -292,3 +292,4 @@ }; #include "t8142-j813-peripherals.dtsi" +#include "t8142-j813-usb.dtsi" diff --git a/arch/arm64/configs/j813.config b/arch/arm64/configs/j813.config index 13b8116af0710..57fb42b0295c5 100644 --- a/arch/arm64/configs/j813.config +++ b/arch/arm64/configs/j813.config @@ -75,3 +75,29 @@ CONFIG_LEDS_PWM=y CONFIG_PWM=y CONFIG_PWM_APPLE=y CONFIG_FW_LOADER=y +CONFIG_PM_GENERIC_DOMAINS=y +CONFIG_APPLE_PMGR_PWRSTATE=y +CONFIG_SPMI=y +CONFIG_SPMI_APPLE=y +CONFIG_USB_SUPPORT=y +CONFIG_USB=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_XHCI_PLATFORM=y +CONFIG_USB_DWC3=y +CONFIG_USB_DWC3_HOST=y +CONFIG_USB_DWC3_APPLE=y +CONFIG_USB_HID=y +CONFIG_BLOCK=y +CONFIG_SCSI=y +CONFIG_BLK_DEV_SD=y +CONFIG_USB_STORAGE=y +CONFIG_USB_UAS=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_EFI_PARTITION=y +CONFIG_MSDOS_PARTITION=y +CONFIG_EXFAT_FS=y +CONFIG_NLS_UTF8=y +CONFIG_TYPEC=y +CONFIG_TYPEC_SN201202X=y +CONFIG_PHY_APPLE_ATC=y +CONFIG_DEBUG_FS=y diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 1bbf7680024df..09552187db4ce 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -469,8 +469,10 @@ enum atcphy_lane_mode { #define PIPEHANDLER_OVERRIDE_VAL_PHY_STATUS BIT(4) #define PIPEHANDLER_MUX_CTRL 0x0c -#define PIPEHANDLER_MUX_CTRL_CLK GENMASK(5, 3) -#define PIPEHANDLER_MUX_CTRL_DATA GENMASK(2, 0) +#define PIPEHANDLER_MUX_CTRL_CLK_T8103 GENMASK(5, 3) +#define PIPEHANDLER_MUX_CTRL_DATA_T8103 GENMASK(2, 0) +#define PIPEHANDLER_MUX_CTRL_CLK_T8142 GENMASK(6, 4) +#define PIPEHANDLER_MUX_CTRL_DATA_T8142 GENMASK(1, 0) #define PIPEHANDLER_MUX_CTRL_CLK_OFF 0 #define PIPEHANDLER_MUX_CTRL_CLK_USB3 1 #define PIPEHANDLER_MUX_CTRL_CLK_USB4 2 @@ -602,6 +604,8 @@ struct atcphy_mode_configuration { * @aciophy_lane_mode: Lane mode register offset * @aciophy_crossbar: Crossbar register offset * @has_usb4: A USB4/Thunderbolt controller sits behind the PHY + * @pipe_mux_clk: PIPE clock selection field mask + * @pipe_mux_data: PIPE data selection field mask * @has_usb2phy_reg: The PHY has the secondary eUSB2 register bank (T8140) * @optional_tunables: The bootloader may leave out the common-a tunables (this * generation has none) and the SuperSpeed tunables; USB2 @@ -612,6 +616,8 @@ struct atcphy_hw { int aciophy_lane_mode; int aciophy_crossbar; bool has_usb4; + u32 pipe_mux_clk; + u32 pipe_mux_data; bool has_usb2phy_reg; bool optional_tunables; }; @@ -1062,14 +1068,14 @@ static int atcphy_pipehandler_check(struct apple_atcphy *atcphy) static void atcphy_pipehandler_set_mux(struct apple_atcphy *atcphy, u32 data, u32 clk) { - mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, - FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_OFF)); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, atcphy->hw->pipe_mux_clk, + field_prep(atcphy->hw->pipe_mux_clk, PIPEHANDLER_MUX_CTRL_CLK_OFF)); udelay(10); - mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_DATA, - FIELD_PREP(PIPEHANDLER_MUX_CTRL_DATA, data)); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, atcphy->hw->pipe_mux_data, + field_prep(atcphy->hw->pipe_mux_data, data)); udelay(10); - mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, - FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, clk)); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, atcphy->hw->pipe_mux_clk, + field_prep(atcphy->hw->pipe_mux_clk, clk)); udelay(10); } @@ -2775,6 +2781,8 @@ static const struct atcphy_hw atcphy_hw_t8103 = { .gen = ATCPHY_GENERATION_T8103, .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8103, .aciophy_crossbar = ACIOPHY_CROSSBAR_T8103, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, .has_usb4 = true, }; @@ -2782,6 +2790,8 @@ static const struct atcphy_hw atcphy_hw_t8122 = { .gen = ATCPHY_GENERATION_T8122, .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, .has_usb4 = true, }; @@ -2789,13 +2799,25 @@ static const struct atcphy_hw atcphy_hw_t8140 = { .gen = ATCPHY_GENERATION_T8122, .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, .has_usb2phy_reg = true, .optional_tunables = true, }; +static const struct atcphy_hw atcphy_hw_t8142 = { + .gen = ATCPHY_GENERATION_T8122, + .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, + .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8142, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8142, + .has_usb4 = true, +}; + static const struct of_device_id atcphy_match[] = { { .compatible = "apple,t8103-atcphy", .data = &atcphy_hw_t8103 }, { .compatible = "apple,t8122-atcphy", .data = &atcphy_hw_t8122 }, + { .compatible = "apple,t8142-atcphy", .data = &atcphy_hw_t8142 }, { .compatible = "apple,t8140-atcphy", .data = &atcphy_hw_t8140 }, {}, }; diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index ec49d412a4b0e..d31bcf33a8a35 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -52,8 +52,32 @@ #define REG_POLL_INTERVAL_US 10000 #define REG_POLL_TIMEOUT_US (REG_POLL_INTERVAL_US * 5) +struct apple_spmi_hw { + u32 status; + u32 command; + u32 reply; + u32 rx_empty; + bool legacy_irqs; +}; + +static const struct apple_spmi_hw apple_spmi_gen1 = { + .status = SPMI_STATUS_REG, + .command = SPMI_CMD_REG, + .reply = SPMI_RSP_REG, + .rx_empty = SPMI_RX_FIFO_EMPTY, + .legacy_irqs = true, +}; + +static const struct apple_spmi_hw apple_spmi_gen4 = { + .status = 0x200, + .command = 0x210, + .reply = 0x220, + .rx_empty = BIT(30), +}; + struct apple_spmi { void __iomem *regs; + const struct apple_spmi_hw *hw; struct mutex fifo_lock; struct completion fifo_rx; struct irq_domain *irqd; @@ -64,10 +88,6 @@ struct apple_spmi { bool prev_fail; }; -#define poll_reg(spmi, reg, val, cond) \ - readl_poll_timeout((spmi)->regs + (reg), (val), (cond), \ - REG_POLL_INTERVAL_US, REG_POLL_TIMEOUT_US) - static void apple_spmi_irq_ack_raw(struct apple_spmi *spmi, u32 irq) { u32 __iomem *reg = spmi->regs + SPMI_IRQ_ACK_BASE + (irq / 32) * 4; @@ -135,12 +155,14 @@ static int apple_spmi_wait_rx_not_empty(struct spmi_controller *ctrl) usecs_to_jiffies(REG_POLL_TIMEOUT_US)); if (!ret) ret = -ETIMEDOUT; - else if (readl(spmi->regs + SPMI_STATUS_REG) & SPMI_RX_FIFO_EMPTY) + else if (readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty) ret = -EIO; else ret = 0; } else { - ret = poll_reg(spmi, SPMI_STATUS_REG, status, !(status & SPMI_RX_FIFO_EMPTY)); + ret = readl_poll_timeout(spmi->regs + spmi->hw->status, status, + !(status & spmi->hw->rx_empty), 10, + REG_POLL_TIMEOUT_US); } if (ret) { @@ -166,19 +188,22 @@ static int spmi_raw_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid, guard(mutex)(&spmi->fifo_lock); if (spmi->prev_fail) { - writel(SPMI_ACT_FIFO_FLUSH, spmi->regs + SPMI_RSP_REG); + /* Do not consume a late reply as the reply to a new command. */ + if (!spmi->hw->legacy_irqs) + return -EIO; + writel(SPMI_ACT_FIFO_FLUSH, spmi->regs + SPMI_ACT_REG); apple_spmi_irq_ack_raw(spmi, SPMI_IRQ_NOTIFY); spmi->prev_fail = false; } reinit_completion(&spmi->fifo_rx); - writel(spmi_cmd, spmi->regs + SPMI_CMD_REG); + writel(spmi_cmd, spmi->regs + spmi->hw->command); while (i < len) { j = min_t(size_t, sizeof(spmi_cmd), len - i); spmi_cmd = 0; memcpy(&spmi_cmd, buf + i, j); - writel(spmi_cmd, spmi->regs + SPMI_CMD_REG); + writel(spmi_cmd, spmi->regs + spmi->hw->command); i += j; } @@ -186,23 +211,23 @@ static int spmi_raw_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid, if (ret) return ret; - reply = readl(spmi->regs + SPMI_RSP_REG); + reply = readl(spmi->regs + spmi->hw->reply); /* Read SPMI data reply */ while (len_read < ilen) { - if (readl(spmi->regs + SPMI_STATUS_REG) & SPMI_RX_FIFO_EMPTY) { + if (readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty) { spmi->prev_fail = true; dev_err_ratelimited(&ctrl->dev, "FIFO lacks reply data, controller stuck?\n"); return -EIO; } - rsp = readl(spmi->regs + SPMI_RSP_REG); + rsp = readl(spmi->regs + spmi->hw->reply); i = min_t(size_t, sizeof(spmi_cmd), ilen - len_read); memcpy(ibuf + len_read, &rsp, i); len_read += i; } - if (!(readl(spmi->regs + SPMI_STATUS_REG) & SPMI_RX_FIFO_EMPTY)) { + if (!(readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty)) { dev_warn(&ctrl->dev, "FIFO has extra data\n"); spmi->prev_fail = true; } @@ -441,6 +466,9 @@ static int apple_spmi_probe(struct platform_device *pdev) return -ENOMEM; spmi = spmi_controller_get_drvdata(ctrl); + spmi->hw = device_get_match_data(&pdev->dev); + if (!spmi->hw) + return -EINVAL; mutex_init(&spmi->fifo_lock); init_completion(&spmi->fifo_rx); platform_set_drvdata(pdev, spmi); @@ -459,6 +487,9 @@ static int apple_spmi_probe(struct platform_device *pdev) if (spmi->irq < 0 && spmi->irq != -ENXIO) return spmi->irq; if (spmi->irq >= 0) { + if (!spmi->hw->legacy_irqs) + return dev_err_probe(&pdev->dev, -EOPNOTSUPP, + "Generation 4 interrupt layout is not supported\n"); ret = apple_spmi_init_irq(pdev, spmi, spmi->irq); if (ret) return ret; @@ -473,8 +504,9 @@ static int apple_spmi_probe(struct platform_device *pdev) } static const struct of_device_id apple_spmi_match_table[] = { - { .compatible = "apple,t8103-spmi", }, - { .compatible = "apple,spmi", }, + { .compatible = "apple,t8142-spmi", .data = &apple_spmi_gen4 }, + { .compatible = "apple,t8103-spmi", .data = &apple_spmi_gen1 }, + { .compatible = "apple,spmi", .data = &apple_spmi_gen1 }, {} }; MODULE_DEVICE_TABLE(of, apple_spmi_match_table); diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 130b1b8ecf8c2..4e36cba9d5ff9 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -28,7 +28,6 @@ /* Register offsets */ #define TPS_REG_VID 0x00 -#define TPS_REG_MODE 0x03 #define TPS_REG_CMD1 0x08 #define TPS_REG_DATA1 0x09 #define TPS_REG_VERSION 0x0F @@ -772,7 +771,7 @@ static void cd321x_update_work(struct work_struct *work) * request is rejected with -EBUSY; treating that sink as the connector * owner would tear down the display carried by the first port. */ - if (cd321x->connector_fwnode && + if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && dp_route_was_active && (!new_connected || was_disconnected || !dp_connected || !dp_hpd || dp_hpd_changed || dp_mode_changed)) { @@ -852,7 +851,8 @@ static void cd321x_update_work(struct work_struct *work) /* Launch the USB role switch */ usb_role_switch_set_role(tps->role_sw, new_role); - if (cd321x->connector_fwnode && cd321x->display_route_active && dp_hpd) + if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && + cd321x->display_route_active && dp_hpd) drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_connected); /* @@ -861,7 +861,8 @@ static void cd321x_update_work(struct work_struct *work) * changing that display's HPD state. Ask the display driver to retrain the * still-connected route after ACIO and the USB role are live. */ - if (cd321x->connector_fwnode && usb4_started && !dp_connected) + if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && + usb4_started && !dp_connected) drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_unknown); diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 73575a854bde1..7f4fdbc2cc172 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include @@ -24,6 +25,15 @@ static int regmap_sn201202x_select_reg(struct spmi_device *sdev, u8 reg) if (err) return err; + if (sn->polled) { + int poll_err; + + poll_err = read_poll_timeout(spmi_register_read, err, + err || val == reg, 100, 100000, + false, sdev, 0, &val); + return err ?: poll_err; + } + if (!wait_for_completion_timeout(&sn->select_completion, msecs_to_jiffies(100))) return -ETIMEDOUT; @@ -157,6 +167,39 @@ static const struct of_device_id sn201202x_of_match[] = { {} }; +static int sn201202x_wake(struct sn201202x *sn) +{ + unsigned long deadline; + u8 size; + int ret; + + reinit_completion(&sn->wake_completion); + ret = spmi_command_wakeup(sn->sdev); + if (ret) + return ret; + if (!sn->polled) + return wait_for_completion_timeout(&sn->wake_completion, + msecs_to_jiffies(100)) ? 0 : -ETIMEDOUT; + + /* A sleeping ACE3 ACKs commands but returns zeros until it is awake. */ + deadline = jiffies + msecs_to_jiffies(500); + do { + ret = regmap_sn201202x_select_reg(sn->sdev, TPS_REG_MODE); + if (ret == -ETIMEDOUT) + continue; + if (ret) + return ret; + ret = spmi_register_read(sn->sdev, 0x1f, &size); + if (ret) + return ret; + if (size) + return 0; + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + + return -ETIMEDOUT; +} + static int sn201202x_probe(struct spmi_device *device) { const struct of_device_id *match; @@ -181,46 +224,48 @@ static int sn201202x_probe(struct spmi_device *device) tps->dev = &device->dev; tps->data = data; - tps->irq = of_irq_get_byname(device->dev.of_node, "irq"); - if (tps->irq < 0) - return tps->irq; - irq_select = of_irq_get_byname(device->dev.of_node, "select"); - if (irq_select < 0) - return irq_select; - irq_sleep = of_irq_get_byname(device->dev.of_node, "sleep"); - if (irq_sleep < 0) - return irq_sleep; - irq_wake = of_irq_get_byname(device->dev.of_node, "wake"); - if (irq_wake < 0) - return irq_wake; - init_completion(&sn->select_completion); init_completion(&sn->sleep_completion); init_completion(&sn->wake_completion); - ret = devm_request_irq(&device->dev, irq_select, sn201202x_irq, - 0, NULL, &sn->select_completion); - if (ret) - return ret; - ret = devm_request_irq(&device->dev, irq_sleep, sn201202x_irq, - 0, NULL, &sn->sleep_completion); - if (ret) - return ret; - ret = devm_request_irq(&device->dev, irq_wake, sn201202x_irq, - 0, NULL, &sn->wake_completion); - if (ret) - return ret; + sn->polled = !of_property_present(device->dev.of_node, "interrupts") && + !of_property_present(device->dev.of_node, "interrupts-extended"); + if (!sn->polled) { + tps->irq = of_irq_get_byname(device->dev.of_node, "irq"); + if (tps->irq < 0) + return tps->irq; + irq_select = of_irq_get_byname(device->dev.of_node, "select"); + if (irq_select < 0) + return irq_select; + irq_sleep = of_irq_get_byname(device->dev.of_node, "sleep"); + if (irq_sleep < 0) + return irq_sleep; + irq_wake = of_irq_get_byname(device->dev.of_node, "wake"); + if (irq_wake < 0) + return irq_wake; + + ret = devm_request_irq(&device->dev, irq_select, sn201202x_irq, + 0, NULL, &sn->select_completion); + if (ret) + return ret; + ret = devm_request_irq(&device->dev, irq_sleep, sn201202x_irq, + 0, NULL, &sn->sleep_completion); + if (ret) + return ret; + ret = devm_request_irq(&device->dev, irq_wake, sn201202x_irq, + 0, NULL, &sn->wake_completion); + if (ret) + return ret; + } spmi_device_set_drvdata(device, tps); tps->regmap = devm_regmap_init_sn201202x(device, &tps6598x_regmap_config); if (IS_ERR(tps->regmap)) return PTR_ERR(tps->regmap); - ret = spmi_command_wakeup(device); + ret = sn201202x_wake(sn); if (ret) return ret; - if (!wait_for_completion_timeout(&sn->wake_completion, msecs_to_jiffies(100))) - return -ETIMEDOUT; ret = tipd_init(tps); if (ret) @@ -243,12 +288,9 @@ static int __maybe_unused sn201202x_resume(struct device *dev) struct sn201202x *sn = tps_to_sn(tps); int err; - reinit_completion(&sn->wake_completion); - err = spmi_command_wakeup(sn->sdev); + err = sn201202x_wake(sn); if (err) return err; - if (!wait_for_completion_timeout(&sn->wake_completion, msecs_to_jiffies(100))) - return -ETIMEDOUT; return tipd_resume(tps); } @@ -265,7 +307,8 @@ static int __maybe_unused sn201202x_suspend(struct device *dev) err = spmi_command_sleep(sn->sdev); if (err) goto out_resume; - if (!wait_for_completion_timeout(&sn->sleep_completion, msecs_to_jiffies(100))) { + if (!sn->polled && + !wait_for_completion_timeout(&sn->sleep_completion, msecs_to_jiffies(100))) { err = -ETIMEDOUT; goto out_resume; } diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index c6c05da547f2a..2917f8208ebd8 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -16,6 +16,8 @@ #ifndef __TPS6598X_H__ #define __TPS6598X_H__ +#define TPS_REG_MODE 0x03 + #define TPS_FIELD_GET(_mask, _reg) ((typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask))) /* TPS_REG_STATUS bits */ @@ -383,6 +385,7 @@ struct cd321x { struct sn201202x { struct cd321x cd; + bool polled; struct completion select_completion; struct completion sleep_completion; struct completion wake_completion; From 932d8ba8cdb8d3a4755fe5782393149374c7666e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 2 Oct 2026 21:04:47 -0400 Subject: [PATCH 0373/1579] vfio: decode compat dirty-bitmap pointers at their ABI width AArch32 bitmap queries and bitmap-returning unmaps reject valid pointers when trailing pointer padding is nonzero. Read the compat pointer field at its actual width and preserve both the 64-bit scalar fields and ABI- specific payload sizes. Use the architecture alignment fixup for compact IA32 layouts while retaining aligned AArch32/x32 sizes and the native structure-copy path. Extend the freestanding AArch32 probe to exercise zero and nonzero padding through real ioctls, verify returned bits, and complete callback pin/account cleanup with automatic4K and native16K/native64K fallback process ABIs. --- Documentation/arch/arm64/user4k.rst | 24 ++++++ drivers/vfio/vfio_iommu_type1.c | 74 ++++++++++++++++--- .../arm64/user4k/compat-vfio-probe.c | 30 +++++++- 3 files changed, 117 insertions(+), 11 deletions(-) diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index a0baa7d9c8abf..81746c7d28f6c 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -4497,3 +4497,27 @@ UNMAP of one constituent followed by full cleanup. These add sixteen assertions to each tested A64 ABI: native16K/4K and native64K/4K/16K. The supplied domains record translations; these results do not qualify physical device DMA, concurrent type1 consumers, Apple boot/GPU execution or game throughput. + + +VFIO compat dirty-bitmap pointer decoding (2026-10-02) +---------------------------------------------------- + +The AArch32 probe now tests nonzero padding after the 32-bit bitmap pointer. +The prior kernel accepted the zero-padded request but rejected the same valid +pointer with a changed padding byte, for both GET_BITMAP and bitmap-returning +UNMAP. The native pointer copy interpreted that padding as pointer bits. +Separate old-kernel probes isolate each rejection on native16K/native64K. + +Type1 now decodes a compat pointer at its actual width, retaining the 64-bit +bitmap scalars and ignoring pointer padding. Payload-size checks use the +architecture's compat alignment fixup, preserving aligned AArch32/x32 sizes +and accounting for the smaller IA32 layout. Native calls keep their existing +structure copy. No public UAPI structure changes. + +Real AArch32 tests check a zero-padded query, a nonzero-padded query and a +nonzero-padded bitmap-returning unmap, including returned bits, callback pin +cleanup and locked-memory refund. They run with automatic4K and the modeled +no4K native16K/native64K fallback. The existing A64 type1 tests remain applicable +across all five native/alternative granule contexts. Supplied recording domains +qualify this ioctl path, not physical-device DMA or Apple GPU/game execution. +IA32/x32 compilation is covered separately; their runtime behavior is unqualified. diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 895bec69ce8d0..df079de9ee5de 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -3028,6 +3028,63 @@ static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu, return vfio_dma_do_map(iommu, &map); } +#ifdef CONFIG_COMPAT +struct compat_vfio_bitmap { + compat_u64 pgsize; + compat_u64 size; + compat_uptr_t data; +}; +#endif + +static size_t vfio_bitmap_size(void) +{ +#ifdef CONFIG_COMPAT + /* IA32 differs from AArch32/x32 in its u64 alignment and tail padding. */ + if (in_compat_syscall() && compat_need_64bit_alignment_fixup()) + return sizeof(struct compat_vfio_bitmap); +#endif + return sizeof(struct vfio_bitmap); +} + +static size_t vfio_dirty_bitmap_get_size(void) +{ + return offsetof(struct vfio_iommu_type1_dirty_bitmap_get, bitmap) + + vfio_bitmap_size(); +} + +static int vfio_bitmap_from_user(struct vfio_bitmap *bitmap, void __user *user) +{ +#ifdef CONFIG_COMPAT + if (in_compat_syscall()) { + compat_uptr_t data; + unsigned long offset = offsetof(struct vfio_bitmap, data); + + /* Read only the pointer field, never its trailing compat padding. */ + if (copy_from_user(bitmap, user, offset) || + get_user(data, (compat_uptr_t __user *)(user + offset))) + return -EFAULT; + bitmap->data = compat_ptr(data); + return 0; + } +#endif + return copy_from_user(bitmap, user, sizeof(*bitmap)) ? -EFAULT : 0; +} + +static int vfio_dirty_bitmap_get_from_user( + struct vfio_iommu_type1_dirty_bitmap_get *range, void __user *user) +{ +#ifdef CONFIG_COMPAT + if (in_compat_syscall()) { + unsigned long offset = offsetof(typeof(*range), bitmap); + + if (copy_from_user(range, user, offset)) + return -EFAULT; + return vfio_bitmap_from_user(&range->bitmap, user + offset); + } +#endif + return copy_from_user(range, user, sizeof(*range)) ? -EFAULT : 0; +} + static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, unsigned long arg) { @@ -3055,13 +3112,12 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { unsigned long pgshift; - if (unmap.argsz < (minsz + sizeof(bitmap))) + if (unmap.argsz < minsz + vfio_bitmap_size()) return -EINVAL; - if (copy_from_user(&bitmap, - (void __user *)(arg + minsz), - sizeof(bitmap))) - return -EFAULT; + ret = vfio_bitmap_from_user(&bitmap, (void __user *)(arg + minsz)); + if (ret) + return ret; if (!access_ok((void __user *)bitmap.data, bitmap.size)) return -EINVAL; @@ -3133,12 +3189,12 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, size_t size, iommu_pgsize; dma_addr_t iova, iova_end; - if (!data_size || data_size < sizeof(range)) + if (data_size < vfio_dirty_bitmap_get_size()) return -EINVAL; - if (copy_from_user(&range, (void __user *)(arg + minsz), - sizeof(range))) - return -EFAULT; + ret = vfio_dirty_bitmap_get_from_user(&range, (void __user *)(arg + minsz)); + if (ret) + return ret; iova = range.iova; size = range.size; diff --git a/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c b/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c index 2e13b5f774975..46065be963def 100644 --- a/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c +++ b/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c @@ -153,7 +153,8 @@ _Noreturn void probe(u32 *sp) cmd.iova = IOVA + 19; CHECK(!ioctl_test(fd, VFIO_TYPE1_TEST_PIN, &cmd)); CHECK(!state(fd, &cmd) && cmd.locked_bytes == native && cmd.external_refs == 1); - /* The compat pointer's padding is zeroed for the native64 wire layout. */ + /* Compat pointer padding must not become part of the native pointer. */ + _Alignas(struct vfio_iommu_type1_dirty_bitmap_get) unsigned char buffer[sizeof(dirty) + sizeof(struct vfio_iommu_type1_dirty_bitmap_get)]; struct vfio_iommu_type1_dirty_bitmap *header = (void *)buffer; struct vfio_iommu_type1_dirty_bitmap_get *range = (void *)header->data; @@ -169,7 +170,32 @@ _Noreturn void probe(u32 *sp) range->bitmap.data = &bits; CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, buffer) && bits == ((1ULL << (native >> 12)) - 1)); - CHECK(!unmap(fd, 2 * native)); + _Static_assert(sizeof(range->bitmap.data) == 4 && sizeof(range->bitmap) == 24, + "AArch32 bitmap has a 32-bit pointer and trailing padding"); + ((unsigned char *)&range->bitmap)[ + __builtin_offsetof(struct vfio_bitmap, data) + sizeof(range->bitmap.data)] = 0xa5; + bits = 0; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, buffer) && + bits == ((1ULL << (native >> 12)) - 1)); + _Alignas(struct vfio_bitmap) + unsigned char unmap_buffer[sizeof(struct vfio_iommu_type1_dma_unmap) + + sizeof(struct vfio_bitmap)]; + struct vfio_iommu_type1_dma_unmap *remove = (void *)unmap_buffer; + struct vfio_bitmap *remove_bitmap = (void *)remove->data; + + clear(unmap_buffer, sizeof(unmap_buffer)); + remove->argsz = sizeof(unmap_buffer); + remove->flags = VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP; + remove->iova = IOVA; + remove->size = 2 * native; + remove_bitmap->pgsize = 4096; + remove_bitmap->size = sizeof(bits); + remove_bitmap->data = &bits; + ((unsigned char *)remove_bitmap)[ + __builtin_offsetof(struct vfio_bitmap, data) + sizeof(remove_bitmap->data)] = 0x5a; + bits = 0; + CHECK(!ioctl_test(fd, VFIO_IOMMU_UNMAP_DMA, unmap_buffer) && + remove->size == 2 * native && bits == ((1ULL << (native >> 12)) - 1)); CHECK(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.external_refs && cmd.notifications == 1); clear(&cmd, sizeof(cmd)); From 174ea66513369fb86bb3a8e3ae39ec8a59ac788f Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 21:09:10 -0400 Subject: [PATCH 0374/1579] dt-bindings: thunderbolt: describe T8142 Type 7 NHI and CIO reset --- .../bindings/reset/apple,t8103-cio-reset.yaml | 6 ++++- .../thunderbolt/apple,t8103-usb4-acio.yaml | 1 + .../thunderbolt/apple,t8103-usb4-nhi.yaml | 24 ++++++++++++++++++- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml b/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml index 5f5be0636b369..789e55fdaf384 100644 --- a/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml +++ b/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml @@ -19,13 +19,17 @@ description: | the PMGR MMIO region it is represented as a sub-node of the PMGR syscon with reg used as an offset into it. - In both cases the reset id is the CIO port index on the die. + On t8142, a standalone register page contains a shared request register + with alternating request and busy bits for four ports. + + In all cases the reset id is the CIO port index on the die. properties: compatible: oneOf: - enum: - apple,t8103-cio-reset + - apple,t8142-cio-reset - apple,t6000-cio-reset - items: - const: apple,t8112-cio-reset diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml index c83e00f446314..e673942941aa6 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml @@ -29,6 +29,7 @@ properties: compatible: oneOf: - const: apple,t8103-usb4-acio + - const: apple,t8142-usb4-acio - items: - enum: - apple,t6000-usb4-acio diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml index 4f95132cebf30..49b60aaa8deaf 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml @@ -21,6 +21,7 @@ properties: compatible: oneOf: - const: apple,t8103-usb4-nhi + - const: apple,t8142-usb4-nhi - items: - enum: - apple,t6000-usb4-nhi @@ -32,6 +33,7 @@ properties: - const: apple,t8103-usb4-nhi reg: + minItems: 1 items: - description: NHI registers with the transmit and receive rings - description: @@ -39,6 +41,7 @@ properties: accepted by each receive ring reg-names: + minItems: 1 items: - const: nhi - const: pdf @@ -85,7 +88,8 @@ properties: all other rings which are only used for XDomain connections to other hosts. - T6030, T6031 and T8122 use 24 entries. These are most likely one DART + T8142 uses one DART stream per ring: TX streams 0-11 and RX streams 16-27. + T6030, T6031 and T8122 also use 24 entries. These are most likely one DART stream for each of the 12 transmit and 12 receive rings. apple,thunderbolt-drom: @@ -115,6 +119,23 @@ required: - apple,thunderbolt-drom allOf: + - if: + properties: + compatible: + contains: + const: apple,t8142-usb4-nhi + then: + properties: + reg: + maxItems: 1 + reg-names: + maxItems: 1 + else: + properties: + reg: + minItems: 2 + reg-names: + minItems: 2 - if: properties: compatible: @@ -123,6 +144,7 @@ allOf: - apple,t6030-usb4-nhi - apple,t6031-usb4-nhi - apple,t8122-usb4-nhi + - apple,t8142-usb4-nhi then: properties: iommus: From 4532abf64a799404c7ba1cd22bf92f0445b92606 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 21:09:11 -0400 Subject: [PATCH 0375/1579] thunderbolt: prepare M5 Type 7 controller support Handle the T8142 reset request layout, firmware ready state, and receive-ring PDF masks without the older external PDF aperture. Require loader tuning and matching ACIO/NHI generations. Add an opt-in build fragment; controller DT enablement and firmware startup remain separate qualification work. --- arch/arm64/configs/j813-usb4.config | 7 ++++ drivers/reset/reset-apple-cio.c | 57 +++++++++++++++++++++++++---- drivers/thunderbolt/apple.c | 49 +++++++++++++++++++++---- 3 files changed, 97 insertions(+), 16 deletions(-) create mode 100644 arch/arm64/configs/j813-usb4.config diff --git a/arch/arm64/configs/j813-usb4.config b/arch/arm64/configs/j813-usb4.config new file mode 100644 index 0000000000000..d754400c6eccd --- /dev/null +++ b/arch/arm64/configs/j813-usb4.config @@ -0,0 +1,7 @@ +# Additional build coverage for M5 USB4 controller development. +# Merge with j813.config. This does not enable USB4 device-tree nodes. +CONFIG_PCI=y +CONFIG_PCIE_APPLE=y +CONFIG_USB4=y +CONFIG_USB4_APPLE_SOC=y +CONFIG_RESET_APPLE_CIO=y diff --git a/drivers/reset/reset-apple-cio.c b/drivers/reset/reset-apple-cio.c index 452893bf92a3c..04a985b7f6b44 100644 --- a/drivers/reset/reset-apple-cio.c +++ b/drivers/reset/reset-apple-cio.c @@ -18,6 +18,7 @@ * t8103 0 0x84000 + 0x4000 * port BIT(0), rmw BIT(1) * t8112 0x63 0x8000 / 0x4000 BIT(0), rmw BIT(1) * t8122 0x62 0x8 BIT(slot) BIT(16 + slot) + * t8142 0x62 0x30 BIT(2 * port) BIT(2 * port + 1) * t6000 0 0x1a034 BIT(port) BIT(16 + port) * t6020 0 0xa02c BIT(port) BIT(16 + port) * t6030 0x62 0x8 BIT(slot) BIT(16 + slot) @@ -29,18 +30,18 @@ * "port = slot % 4", which the device tree expresses as one controller node * per die instead. The others index by slot directly. * - * "regmap" is an ApplePMGR::RegMap enum rather than an address. Only 0 is - * pinned down, as the PMGR block itself: t8103 resolves to 0x23b784000 and + * "regmap" is an ApplePMGR::RegMap enum rather than an address. RegMap 0 is + * the PMGR block itself: t8103 resolves to 0x23b784000 and * t6020 to the pmgr node plus 0xa02c, both of which match this driver. The - * non-zero selectors are unresolved, so t8122 and t6030 cannot be wired up + * t8122 and t6030 selectors are unresolved, so they cannot be wired up * yet even though their offset and bit layout are known. * - * t8132 and t8142 have a different shape again, indexing a per-port table out - * of a driver-private structure rather than forming an offset, and have not - * been decoded. + * On t8142, RegMap 0x62 is a separate MMIO region at 0x380780000. + * The t8132 layout has not been decoded. * - * Every implementation polls the busy bits, stores the request, then polls - * again, with a 192 ms budget. The first poll's result is discarded. + * Reference firmware polls the busy bits, stores the request, then polls + * again, with a 192 ms budget. It discards the first poll's result. The + * t8142 implementation below instead refuses to issue a request while busy. */ #include @@ -63,6 +64,11 @@ #define T6000_CIO_CTRL_INIT_REQ(port) BIT(port) #define T6000_CIO_CTRL_INIT_BUSY(port) BIT(16 + (port)) +#define T8142_CIO_CTRL 0x30 +#define T8142_CIO_CTRL_INIT_REQ(port) BIT(2 * (port)) +#define T8142_CIO_CTRL_INIT_BUSY(port) BIT(2 * (port) + 1) +#define T8142_CIO_CTRL_BUSY_MASK 0xaa + struct apple_cio_reset; struct apple_cio_reset_variant { @@ -144,6 +150,37 @@ static int t6000_cio_deassert(struct apple_cio_reset *priv, unsigned long id) APPLE_CIO_RESET_TIMEOUT_US); } +static int t8142_cio_deassert(struct apple_cio_reset *priv, unsigned long id) +{ + u32 val; + int ret; + + guard(mutex)(&priv->lock); + + /* Serialize all four ports sharing the request register. */ + ret = regmap_read_poll_timeout(priv->regmap, T8142_CIO_CTRL, val, + !(val & T8142_CIO_CTRL_BUSY_MASK), + APPLE_CIO_RESET_POLL_US, + APPLE_CIO_RESET_TIMEOUT_US); + if (ret) + return ret; + + ret = regmap_write(priv->regmap, T8142_CIO_CTRL, + T8142_CIO_CTRL_INIT_REQ(id)); + if (ret) + return ret; + + return regmap_read_poll_timeout(priv->regmap, T8142_CIO_CTRL, val, + !(val & T8142_CIO_CTRL_INIT_BUSY(id)), + APPLE_CIO_RESET_POLL_US, + APPLE_CIO_RESET_TIMEOUT_US); +} + +static const struct apple_cio_reset_variant apple_t8142_cio_reset = { + .nr_resets = 4, + .deassert = t8142_cio_deassert, +}; + static const struct apple_cio_reset_variant apple_t8103_cio_reset = { .nr_resets = 2, .deassert = t8103_cio_deassert, @@ -222,6 +259,10 @@ static int apple_cio_reset_probe(struct platform_device *pdev) } static const struct of_device_id apple_cio_reset_match[] = { + { + .compatible = "apple,t8142-cio-reset", + .data = &apple_t8142_cio_reset, + }, { .compatible = "apple,t8103-cio-reset", .data = &apple_t8103_cio_reset, diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index bd7aa24255937..157f142bc123a 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -119,6 +119,7 @@ struct apple_cio { struct device *dev; + bool type7; struct device_node *np; struct device_node *pcie_tunnel_np; bool pcie_tunnel_preinitialized; @@ -471,7 +472,9 @@ static void apple_nhi_ring_configure(struct tb_ring *ring, u32 flags, u32 e2e_fl sof_eof_mask = ring->sof_mask << 16 | ring->eof_mask; writel(sof_eof_mask, options + 4); - writel(sof_eof_mask, anhi->pdf_base + ring->hop * APPLE_CIO_NHI_PDF_STRIDE); + if (anhi->pdf_base) + writel(sof_eof_mask, + anhi->pdf_base + ring->hop * APPLE_CIO_NHI_PDF_STRIDE); } /* @@ -663,6 +666,12 @@ static int apple_nhi_probe(struct platform_device *pdev) int cap_apple; int ret = 0; + if (acio->type7 != + of_device_is_compatible(pdev->dev.of_node, "apple,t8142-usb4-nhi")) { + ret = dev_err_probe(&pdev->dev, -EINVAL, "ACIO/NHI generation mismatch\n"); + goto err; + } + anhi = devm_kzalloc(&pdev->dev, sizeof(*anhi), GFP_KERNEL); if (!anhi) { ret = -ENOMEM; @@ -677,6 +686,13 @@ static int apple_nhi_probe(struct platform_device *pdev) anhi->acio = acio; platform_set_drvdata(pdev, anhi); + if (acio->type7 && + of_property_count_u32_elems(anhi->np, "apple,tunable-nhi") <= 0) { + ret = dev_err_probe(anhi->dev, -EINVAL, + "Missing bootloader NHI tuning\n"); + goto err; + } + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "nhi"); anhi->nhi_base = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(anhi->nhi_base)) { @@ -691,12 +707,15 @@ static int apple_nhi_probe(struct platform_device *pdev) } apple_tunable_apply(anhi->nhi_base, tunable); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pdf"); - anhi->pdf_base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(anhi->pdf_base)) { - ret = dev_err_probe(&pdev->dev, PTR_ERR(anhi->pdf_base), - "Unable to map PDF regs\n"); - goto err; + /* Type 7 keeps the PDF masks in the receive ring registers only. */ + if (!acio->type7) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pdf"); + anhi->pdf_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(anhi->pdf_base)) { + ret = dev_err_probe(&pdev->dev, PTR_ERR(anhi->pdf_base), + "Unable to map PDF regs\n"); + goto err; + } } ret = apple_nhi_probe_irqs(anhi); @@ -870,6 +889,9 @@ static const struct dev_pm_ops apple_nhi_pm_ops = { }; static const struct of_device_id apple_nhi_match[] = { + { + .compatible = "apple,t8142-usb4-nhi", + }, { .compatible = "apple,t8103-usb4-nhi", }, @@ -1032,8 +1054,11 @@ static int apple_cio_start(struct apple_cio *acio) goto err_free_rtkit; } + /* Type 7 reports ready as state 1; other nonzero states are not ready. */ ret = readl_poll_timeout(acio->rc_base + APPLE_CIO_M3_STAT, state, - state & APPLE_CIO_M3_STAT_STATE, 100, 500000); + acio->type7 ? + (state & APPLE_CIO_M3_STAT_STATE) == BIT(24) : + (state & APPLE_CIO_M3_STAT_STATE), 100, 500000); if (ret < 0) { dev_err(acio->dev, "M3 firmware failed to get ready: %d\n", ret); goto err_shutdown_rtkit; @@ -1181,6 +1206,11 @@ static int apple_cio_probe(struct platform_device *pdev) init_completion(&acio->nhi_boot_completion); acio->dev = &pdev->dev; acio->np = dev->of_node; + acio->type7 = of_device_is_compatible(acio->np, "apple,t8142-usb4-acio"); + if (acio->type7 && + of_property_count_u32_elems(acio->np, "apple,tunable-rc") <= 0) + return dev_err_probe(dev, -EINVAL, "Missing bootloader RC tuning\n"); + acio->pcie_tunnel_np = of_parse_phandle(dev->of_node, "apple,pcie-tunnel", 0); if (acio->pcie_tunnel_np) { @@ -1274,6 +1304,9 @@ static void apple_cio_remove(struct platform_device *pdev) } static const struct of_device_id apple_acio_match[] = { + { + .compatible = "apple,t8142-usb4-acio", + }, { .compatible = "apple,t8103-usb4-acio", }, From 0a8b54f5b4dc5e1c607024600fd449004b26d08d Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:10:14 -0500 Subject: [PATCH 0376/1579] accel/ane: Power the ANE off when it is idle The ane module takes a runtime PM reference in probe and holds it until remove, so the ANE power domains and its three DARTs stay powered for as long as the driver is bound, also when no process uses the ANE. Drop the probe reference at the end of probe and use runtime PM autosuspend. The device suspends autosuspend_ms (default 1500) after the last file open, ioctl or close, and the next one resumes it through ane_runtime_resume(), which enables the task manager again. The new module parameter autosuspend_ms=0 keeps the device powered while the driver is bound. power/autosuspend_delay_ms and power/control change the behavior at run time. Every path that touches the engine or the ANE's own DART registers now holds a reference: - the ioctls, through one wrapper for the native and the 32-bit compat entry points, so both get the command filter and the reference; - the reset sysfs attribute, which power-cycles the partitions and then writes the engine; - the file release, which frees the BOs of the file (each unmap flushes the DART TLBs) and can run the recovery in postclose. It keeps a reference to the struct device, because drm_release() can free the ane_device; - remove, which unmaps the remaining BOs and detaches the power domains. apple-dart saves and restores the stream TCR and TTBR registers in its own runtime PM callbacks, and the ANE is its consumer through a DL_FLAG_PM_RUNTIME device link, so the IOMMU state survives the suspend. Measurements on hardware are pending. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_drv.c | 90 +++++++++++++++++++++++++++++++++---- drivers/accel/ane/ane_tm.c | 6 +-- 2 files changed, 84 insertions(+), 12 deletions(-) diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 52e720aad356c..2147d6a5e7d83 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -66,6 +66,19 @@ module_param(stats, bool, 0444); MODULE_PARM_DESC(stats, "Expose ane_stats sysfs and ane_timeline debugfs (default 1; 0 = no files and the hot path skips the counters)"); +/* + * Runtime PM autosuspend delay. The ANE power domains and its DARTs go + * off this long after the last file open, ioctl or close, and the next + * one powers them up again through ane_runtime_resume(). 0 keeps the + * device powered while the driver is bound. The live value is + * power/autosuspend_delay_ms, and "on" in power/control keeps the + * device powered without a reload. + */ +static int autosuspend_ms = 1500; +module_param(autosuspend_ms, int, 0444); +MODULE_PARM_DESC(autosuspend_ms, + "Power the ANE off this many ms after its last use (default 1500; 0 = keep it powered while the driver is bound)"); + #define CMD_BUF_BDX 0 #define KRN_BUF_BDX 1 @@ -624,10 +637,18 @@ static ssize_t reset_store(struct device *dev, struct device_attribute *attr, struct ane_device *ane = dev_get_drvdata(dev); int err = 0; + /* Recovery power-cycles the partitions and then writes the engine: + * hold a reference so that autosuspend cannot gate it in between. + */ + err = pm_runtime_resume_and_get(dev); + if (err < 0) + return err; + mutex_lock(&ane->engine_lock); if (!ane->removed) err = ane_tm_recover(ane); mutex_unlock(&ane->engine_lock); + pm_runtime_put_autosuspend(dev); if (err) return err; return count; @@ -688,7 +709,7 @@ static int ane_drm_open(struct drm_device *drm, struct drm_file *file) if (err < 0) return err; - pm_runtime_put(ane->dev); + pm_runtime_put_autosuspend(ane->dev); return 0; } @@ -741,8 +762,10 @@ static void ane_drm_postclose(struct drm_device *drm, struct drm_file *file) * from before DRM_ACCEL_FOPS landed drm_ioctl defaults), so this is a * belt-and-suspenders gate on top of the existing ioctls table. */ -static long ane_drm_unlocked_ioctl(struct file *file, unsigned int cmd, - unsigned long arg) +static long ane_drm_ioctl(struct file *file, unsigned int cmd, + unsigned long arg, + long (*dispatch)(struct file *, unsigned int, + unsigned long)) { struct drm_file *filp = file->private_data; struct drm_device *drm = filp->minor->dev; @@ -770,10 +793,47 @@ static long ane_drm_unlocked_ioctl(struct file *file, unsigned int cmd, if (err < 0) return err; - err = drm_ioctl(file, cmd, arg); + err = dispatch(file, cmd, arg); + + pm_runtime_put_autosuspend(ane->dev); + + return err; +} + +static long ane_drm_unlocked_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + return ane_drm_ioctl(file, cmd, arg, drm_ioctl); +} + +/* 32-bit callers get the same command filter and PM reference. */ +#ifdef CONFIG_COMPAT +static long ane_drm_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + return ane_drm_ioctl(file, cmd, arg, drm_compat_ioctl); +} +#else +#define ane_drm_compat_ioctl NULL +#endif - pm_runtime_put(ane->dev); +/* + * Hold power across the close: drm_release() frees the BOs of the file, + * and each unmap flushes the DART TLBs. With the device suspended, + * apple-dart would power the DARTs up and down again for every page. + * The device reference outlives drm_release(), which can drop the last + * drm_device reference and free ane. + */ +static int ane_drm_release(struct inode *inode, struct file *file) +{ + struct drm_file *priv = file->private_data; + struct device *dev = get_device(priv->minor->dev->dev); + int err; + pm_runtime_get_sync(dev); + err = drm_release(inode, file); + pm_runtime_put_autosuspend(dev); + put_device(dev); return err; } @@ -823,9 +883,9 @@ static const struct file_operations ane_drm_fops = { .owner = THIS_MODULE, .fop_flags = FOP_UNSIGNED_OFFSET, .open = accel_open, - .release = drm_release, + .release = ane_drm_release, .unlocked_ioctl = ane_drm_unlocked_ioctl, - .compat_ioctl = drm_compat_ioctl, + .compat_ioctl = ane_drm_compat_ioctl, .poll = drm_poll, .read = drm_read, .llseek = noop_llseek, @@ -1104,9 +1164,13 @@ static int ane_platform_probe(struct platform_device *pdev) * supplier links for several), and .runtime_resume then does the * first engine MMIO. Marking the device active up front skips that * resume, and on T6001 the engine window external-aborts while its - * partition is gated. The reference is held until remove: the device - * stays powered for the whole lifetime, autosuspend stays disabled. + * partition is gated. Probe drops its reference at the end, and the + * device autosuspends autosuspend_ms after its last use. A negative + * delay (autosuspend_ms=0) makes the PM core keep it powered. */ + pm_runtime_set_autosuspend_delay(dev, + autosuspend_ms > 0 ? autosuspend_ms : -1); + pm_runtime_use_autosuspend(dev); pm_runtime_enable(dev); err = pm_runtime_resume_and_get(dev); if (err < 0) @@ -1128,12 +1192,14 @@ static int ane_platform_probe(struct platform_device *pdev) dev_info(dev, "loaded ane\n"); + pm_runtime_put_autosuspend(dev); return 0; put_pm: pm_runtime_put_noidle(dev); disable_pm: pm_runtime_disable(dev); + pm_runtime_dont_use_autosuspend(dev); drm_mm_takedown(&ane->mm); detach_genpd: ane_boost_exit(ane); @@ -1146,6 +1212,11 @@ static void ane_platform_remove(struct platform_device *pdev) struct ane_device *ane = platform_get_drvdata(pdev); struct ane_bo *bo, *tmp; + /* Powered and pinned for the teardown: the unmaps below flush the + * DART TLBs, and no autosuspend may race the genpd detach. + */ + pm_runtime_get_sync(ane->dev); + mutex_lock(&ane->engine_lock); ane->removed = true; drm_dev_unplug(&ane->drm); @@ -1166,6 +1237,7 @@ static void ane_platform_remove(struct platform_device *pdev) ane_detach_genpd(ane); pm_runtime_disable(ane->dev); + pm_runtime_dont_use_autosuspend(ane->dev); pm_runtime_put_noidle(ane->dev); mutex_unlock(&ane->engine_lock); } diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 52f3781b8cc15..703d595b612a9 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -467,9 +467,9 @@ static int ane_pd_cycle(struct ane_device *ane) pm_runtime_put_noidle(ane->pd_dev[gated]); } else { /* Pin a second usage ref for the cycle: force_suspend only - * marks needs_force_resume when a ref beyond the probe one - * is held, and without that mark force_resume would leave - * the partition gated. + * marks needs_force_resume when a ref beyond the caller's + * one (submit ioctl or reset write) is held, and without + * that mark force_resume would leave the partition gated. */ pm_runtime_get_noresume(ane->dev); dev_dbg(ane->dev, "ANERD dev force_suspend begin\n"); From 86e26a551714bd91821838705e070f73ce5e2b3b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 2 Oct 2026 21:57:09 -0400 Subject: [PATCH 0377/1579] vfio: stress mixed-page pins against unmap and registration Force a consumer to reenter type1 pin/unpin while an unmap callback holds the device-list lock and another worker registers a device. A locked consumer publishes and revokes references safely. In half the cycles a competing unmap removes the DMA before a second observer checks the callback range snapshot. Exercise native16K/64K and4K/16K metadata mms, lazy/mapped providers, two-area and fine4K-area spans, source exit, exact accounting and translation cleanup. Each of ten models runs1024 forced cycles with KASAN/lockdep. This extends bounded core-API qualification; concurrent EL0 ioctls and physical DMA remain unqualified. Production MM/type1 behavior stays unchanged. --- Documentation/arch/arm64/user4k.rst | 32 ++ .../vfio/vfio_iommu_type1_fragments_test.c | 365 ++++++++++++++++++ 2 files changed, 397 insertions(+) diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index 81746c7d28f6c..c9798cbf5fdb0 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -4521,3 +4521,35 @@ no4K native16K/native64K fallback. The existing A64 type1 tests remain applicabl across all five native/alternative granule contexts. Supplied recording domains qualify this ioctl path, not physical-device DMA or Apple GPU/game execution. IA32/x32 compilation is covered separately; their runtime behavior is unqualified. + + +VFIO concurrent pin, unmap and registration qualification (2026-10-02) +--------------------------------------------------------------------- + +A new KUnit case uses a separately locked test consumer and three kernel +workers to force pinning, device registration and overlapping DMA unmaps. +The consumer publishes references under its own lock. An unmap callback +signals a pending registration while holding the production device-list lock, +then permits a consumer to acquire a duplicate pin, read the original native +page and release its references before callback cleanup. This exercises actual +type1 APIs and their lock ordering rather than removing serialization from the +real-ioctl fixture. + +Half the cycles force a second unmap to remove the first fine DMA mapping +while its callback still runs. A second permanent consumer verifies the +original range snapshot after removal; the two unmaps together report exactly +one removed mapping. Each cycle checks page identity and bytes, per-mm locked +memory refund, the DMA tree and recording-domain translations. The final cycle +retains a seed pin and drops the last source-mm user before worker pinning, +including the overlapping unmap and callback refund after source exit. + +Native16K runs 4K/native16K metadata mms and native64K runs +4K/16K/native64K metadata mms, each with lazy and mapped providers. The areas +alternate between two fine DMA mappings and a separate mapping per 4K unit. +Each of the ten models passes 1,024 cycles: 10,240 forced interleavings, +including 5,120 overlapping unmaps. The type1 suites total twelve passes and +zero failures on both KASAN/lockdep kernels, with no reported kernel warning. +These are retained test mms with real page tables and pins; concurrent EL0 +ioctls, physical-device DMA, all possible concurrency schedules and hardware +performance remain unqualified. The root-only real-ioctl fixture still +serializes its own requests. No production MM or type1 behavior changes here. diff --git a/drivers/vfio/vfio_iommu_type1_fragments_test.c b/drivers/vfio/vfio_iommu_type1_fragments_test.c index 169ad50bfe35e..60fffff10c6be 100644 --- a/drivers/vfio/vfio_iommu_type1_fragments_test.c +++ b/drivers/vfio/vfio_iommu_type1_fragments_test.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* Actual type1 lifecycle with recorded translations; no physical device DMA. */ +#include #include #include #include @@ -791,6 +792,369 @@ static void vfio_fragments_external_span_test(struct kunit *test) } } +#define VFIO_STRESS_CYCLES 1024 + +struct vfio_pin_stress { + struct vfio_fragment_fixture *fixture; + struct vfio_device consumer; + struct vfio_device registrar; + struct vfio_device observer; + struct task_struct *competing_thread; + struct task_struct *pin_thread; + struct task_struct *register_thread; + struct mutex consumer_lock; + struct completion pin_go; + struct completion pin_ready; + struct completion notify_entered; + struct completion pin_done; + struct completion register_go; + struct completion register_entered; + struct completion register_done; + struct completion competing_go; + struct completion competing_done; + unsigned long competing_size; + int competing_ret; + bool competing; + unsigned int competing_unmaps; + unsigned int observations; + atomic_t error; + unsigned int pins; + unsigned int registrations; + bool stopping; +}; + +static void vfio_stress_error(struct vfio_pin_stress *s, int ret) +{ + atomic_cmpxchg(&s->error, 0, ret < 0 ? ret : -EIO); +} + +static void vfio_stress_notify(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_pin_stress *s = container_of(device, typeof(*s), consumer); + struct vfio_fragment_fixture *f = s->fixture; + + /* The callback holds the production device-list lock, not the DMA lock. */ + complete(&s->register_go); + if (!wait_for_completion_timeout(&s->register_entered, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + complete(&s->notify_entered); + mutex_lock(&s->consumer_lock); + f->notifications++; + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + mutex_unlock(&s->consumer_lock); + if (s->competing) { + /* A second unmap may free the DMA while this callback still runs. */ + complete(&s->competing_go); + if (!wait_for_completion_timeout(&s->competing_done, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + } +} + +static void vfio_stress_observe(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_pin_stress *s = container_of(device, typeof(*s), observer); + + s->observations++; + if (iova != VFIO_TEST_IOVA || length != SZ_4K) + vfio_stress_error(s, -ESTALE); +} + +static void vfio_stress_empty_notify(struct vfio_device *device, u64 iova, u64 length) +{ +} + +static const struct vfio_device_ops vfio_stress_consumer_ops = { + .dma_unmap = vfio_stress_notify, +}; + +static const struct vfio_device_ops vfio_stress_registrar_ops = { + .dma_unmap = vfio_stress_empty_notify, +}; + +static const struct vfio_device_ops vfio_stress_observer_ops = { + .dma_unmap = vfio_stress_observe, +}; + +static int vfio_stress_competing_unmaps(void *data) +{ + struct vfio_pin_stress *s = data; + + for (;;) { + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, + .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + + wait_for_completion(&s->competing_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + s->competing_ret = vfio_dma_do_unmap(s->fixture->iommu, &unmap, &bitmap); + s->competing_size = unmap.size; + s->competing_unmaps++; + complete(&s->competing_done); + } + return 0; +} + +static int vfio_stress_pinner(void *data) +{ + struct vfio_pin_stress *s = data; + struct vfio_fragment_fixture *f = s->fixture; + + for (;;) { + struct page *page = NULL, *original; + void *bytes; + int ret; + + wait_for_completion(&s->pin_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + mutex_lock(&s->consumer_lock); + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 37, 1, IOMMU_READ | IOMMU_WRITE, &page); + if (ret != 1) { + vfio_stress_error(s, ret); + mutex_unlock(&s->consumer_lock); + complete(&s->pin_ready); + complete(&s->pin_done); + continue; + } + f->external_refs++; + s->pins++; + original = page; + complete(&s->pin_ready); + /* Unmap cannot revoke a reference before the consumer publishes it. */ + if (!wait_for_completion_timeout(&s->notify_entered, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + if (!READ_ONCE(s->stopping)) { + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 17, 1, IOMMU_READ, &page); + if (ret == 1) { + f->external_refs++; + s->pins++; + if (page != original) + vfio_stress_error(s, -ESTALE); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 17, 1); + f->external_refs--; + } else { + vfio_stress_error(s, ret); + } + bytes = kmap_local_page(original); + if (memchr_inv(bytes, 0x6b, PAGE_SIZE)) + vfio_stress_error(s, -EIO); + kunmap_local(bytes); + } + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 37, 1); + f->external_refs--; + mutex_unlock(&s->consumer_lock); + complete(&s->pin_done); + } + return 0; +} + +static int vfio_stress_registrations(void *data) +{ + struct vfio_pin_stress *s = data; + + for (;;) { + wait_for_completion(&s->register_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + complete(&s->register_entered); + vfio_iommu_type1_register_device(s->fixture->iommu, &s->registrar); + vfio_iommu_type1_unregister_device(s->fixture->iommu, &s->registrar); + s->registrations++; + complete(&s->register_done); + } + return 0; +} + +static void vfio_stress_free(void *data) +{ + struct vfio_pin_stress *s = data; + struct vfio_fragment_fixture *f = s->fixture; + + WRITE_ONCE(s->stopping, true); + complete_all(&s->pin_go); + complete_all(&s->notify_entered); + complete_all(&s->register_go); + complete_all(&s->competing_go); + if (s->competing_thread) { + kthread_stop(s->competing_thread); + put_task_struct(s->competing_thread); + } + if (s->pin_thread) { + kthread_stop(s->pin_thread); + put_task_struct(s->pin_thread); + } + if (s->register_thread) { + kthread_stop(s->register_thread); + put_task_struct(s->register_thread); + } + mutex_lock(&s->consumer_lock); + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + mutex_unlock(&s->consumer_lock); + vfio_iommu_type1_unregister_device(f->iommu, &s->consumer); + vfio_iommu_type1_unregister_device(f->iommu, &s->observer); +} + +static struct vfio_pin_stress *vfio_stress_new(struct kunit *test, + struct vfio_fragment_fixture *f) +{ + struct vfio_pin_stress *s = kunit_kzalloc(test, sizeof(*s), GFP_KERNEL); + + if (!s) + return NULL; + s->fixture = f; + mutex_init(&s->consumer_lock); + init_completion(&s->pin_go); + init_completion(&s->pin_ready); + init_completion(&s->notify_entered); + init_completion(&s->pin_done); + init_completion(&s->register_go); + init_completion(&s->register_entered); + init_completion(&s->register_done); + init_completion(&s->competing_go); + init_completion(&s->competing_done); + atomic_set(&s->error, 0); + s->consumer.ops = &vfio_stress_consumer_ops; + s->registrar.ops = &vfio_stress_registrar_ops; + s->observer.ops = &vfio_stress_observer_ops; + vfio_iommu_type1_unregister_device(f->iommu, &f->device); + /* list_add puts the consumer before the observer in callback order. */ + vfio_iommu_type1_register_device(f->iommu, &s->observer); + vfio_iommu_type1_register_device(f->iommu, &s->consumer); + if (kunit_add_action_or_reset(test, vfio_stress_free, s)) + return NULL; + s->pin_thread = kthread_run(vfio_stress_pinner, s, "vfio-pin-stress"); + if (IS_ERR(s->pin_thread)) { + s->pin_thread = NULL; + return NULL; + } + get_task_struct(s->pin_thread); + s->register_thread = kthread_run(vfio_stress_registrations, s, "vfio-reg-stress"); + if (IS_ERR(s->register_thread)) { + s->register_thread = NULL; + return NULL; + } + get_task_struct(s->register_thread); + s->competing_thread = kthread_run(vfio_stress_competing_unmaps, s, "vfio-unmap-stress"); + if (IS_ERR(s->competing_thread)) { + s->competing_thread = NULL; + return NULL; + } + get_task_struct(s->competing_thread); + return s; +} + +static void vfio_fragments_concurrency_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14, 16 }; + unsigned int rw = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + + for (unsigned int shift = 0; shift < ARRAY_SIZE(shifts); shift++) { + if (shifts[shift] > PAGE_SHIFT) + continue; + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_account_mm *ctx = vfio_account_mm_new(test, shifts[shift]); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, PAGE_SIZE, 0x6b); + struct vfio_pin_stress *s; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, PAGE_SIZE), + VFIO_TEST_VA); + s = vfio_stress_new(test, f); + KUNIT_ASSERT_NOT_NULL(test, s); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + for (unsigned int cycle = 0; cycle < VFIO_STRESS_CYCLES; cycle++) { + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, + .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + bool fine = cycle & 1; + unsigned long remaining = PAGE_SIZE - SZ_4K; + + KUNIT_ASSERT_EQ(test, + vfio_fragment_map_area(f, ctx, 0, 0, SZ_4K, rw), 0); + for (unsigned long offset = SZ_4K; offset < PAGE_SIZE;) { + unsigned long length = fine ? SZ_4K : remaining; + + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, offset, + offset, length, rw), 0); + offset += length; + } + if (cycle == VFIO_STRESS_CYCLES - 1) { + struct page *page; + + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 1, + IOMMU_READ, &page), 1); + f->external_refs = 1; + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + } + s->competing = cycle & 2; + complete(&s->pin_go); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->pin_ready, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_EQ(test, atomic_read(&s->error), 0); + KUNIT_ASSERT_EQ(test, + vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, s->competing ? 0UL : SZ_4K); + if (s->competing) { + KUNIT_EXPECT_EQ(test, s->competing_ret, 0); + KUNIT_EXPECT_EQ(test, s->competing_size, SZ_4K); + } + unmap.iova += SZ_4K; + unmap.size = remaining; + KUNIT_ASSERT_EQ(test, + vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, remaining); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->pin_done, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->register_done, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_EQ(test, atomic_read(&s->error), 0); + KUNIT_EXPECT_EQ(test, f->external_refs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&f->iommu->dma_list)); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + reinit_completion(&s->pin_ready); + reinit_completion(&s->notify_entered); + reinit_completion(&s->pin_done); + reinit_completion(&s->register_entered); + reinit_completion(&s->register_done); + reinit_completion(&s->competing_done); + } + KUNIT_EXPECT_EQ(test, s->pins, 2U * VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->registrations, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, f->notifications, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->observations, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->competing_unmaps, VFIO_STRESS_CYCLES / 2U); + kunit_info(test, "%uK user/%luK native mapped=%u: %u forced pin/unmap/register cycles, half with competing unmap", + 1U << (shifts[shift] - 10), PAGE_SIZE >> 10, + mapped, VFIO_STRESS_CYCLES); + kunit_release_action(test, vfio_stress_free, s); + kunit_release_action(test, vfio_fragment_fixture_free, f); + } + } +} + static struct kunit_case vfio_fragments_cases[] = { KUNIT_CASE(vfio_fragments_lifetime_test), KUNIT_CASE(vfio_fragments_lazy_test), @@ -801,6 +1165,7 @@ static struct kunit_case vfio_fragments_cases[] = { KUNIT_CASE(vfio_fragments_domain_granule_test), KUNIT_CASE(vfio_fragments_native_owner_test), KUNIT_CASE(vfio_fragments_external_span_test), + KUNIT_CASE(vfio_fragments_concurrency_test), {} }; From dc7eb33922c3223c7a800fc48b1435285dda162f Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 23:34:50 -0400 Subject: [PATCH 0378/1579] drm: apple: parser: zero display modes before parsing parse_mode() only ever sets vrr (and min_vrr/max_vrr) to true, never clears it, but parse_display_modes() allocates the mode array with kmalloc_array(). When the firmware reports the display again, for example after every DPMS power-on, a mode can inherit a stale vrr flag from the reused allocation. The modeset then enables timestamped VRR swaps the display does not support, and page flips stop completing. Allocate the array zeroed. Fixes: 38c14f6107df ("drm: apple: support ProMotion variable refresh rate") --- drivers/gpu/drm/apple/parser.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 3d520938a2900..61c20b9dc2dc3 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -583,8 +583,8 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, if (ret) return ERR_PTR(ret); - /* Start with a worst case allocation */ - modes = kmalloc_array(it.len, sizeof(*modes), GFP_KERNEL); + /* Start with a worst case allocation; parse_mode() only sets vrr. */ + modes = kcalloc(it.len, sizeof(*modes), GFP_KERNEL); *count = 0; if (!modes) From dc9d13edb0e5b18c61d25240ed4ba532afb8d11f Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 23:34:50 -0400 Subject: [PATCH 0379/1579] iommu/apple-dart: accept T8142 firmware slots that mirror Linux mappings On T8142 the display DARTs are locked and their roots already hold the running display coprocessor's own translations. The boot loader re-exports those translations as reserved IOVA regions, which the IOMMU core maps before attach, so Linux's tables end up with entries in root slots the firmware owns. Publishing only into vacant slots then fails the whole attach with -EBUSY. On locked apple,t8142-dart instances with three-level T8110-format roots: - Record the firmware's leaf tables when a locked root is mapped. - Accept a firmware-owned slot once, if its leaf table already translates every page Linux mapped there to the same physical address. The slot is never written; the hardware keeps walking the firmware's table. - Check later Linux mappings into a firmware-owned slot page by page and refuse any the firmware does not already provide. - Resolve IOVAs Linux does not map through those firmware tables, so the host can reach buffers the coprocessor shares later. Other DARTs and other SoCs keep the current behaviour. --- drivers/iommu/apple-dart.c | 272 +++++++++++++++++++++++++++++++++++-- 1 file changed, 261 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 9c2447da2091e..b04359eecc5e6 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -222,6 +222,17 @@ struct apple_dart_hw { int ttbr_count; }; +/* + * Snapshot of a locked root's firmware-owned slots, taken in process context + * when the root is mapped: the root entry and a mapping of its leaf table. + */ +struct apple_dart_fw_root { + u64 *entry; + u64 **leaf; + /* Linux root entry already verified against each firmware slot. */ + u64 *accepted; +}; + /* * Private structure associated with each DART device. * @@ -259,6 +270,8 @@ struct apple_dart { u32 locked : 1; u32 tunneled : 1; u32 power_retained : 1; + /* Locked roots may hold firmware slots that mirror Linux mappings. */ + u32 fw_mirror : 1; struct apple_tunable *tunables; u16 version; @@ -278,6 +291,8 @@ struct apple_dart { u64 *locked_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; /* Values published by this driver into initially vacant root slots. */ u64 *locked_owned[DART_MAX_STREAMS][DART_MAX_TTBR]; + /* Leaf tables behind the root slots firmware held when we attached. */ + struct apple_dart_fw_root *locked_fw[DART_MAX_STREAMS][DART_MAX_TTBR]; }; /* @@ -486,12 +501,61 @@ apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map) apple_dart_hw_clear_ttbr(stream_map, i); } +/* APPLE_DART2 page-table entry layout, as in io-pgtable-dart. */ +#define APPLE_DART_PTE_VALID BIT_ULL(0) + +static phys_addr_t apple_dart_pte_to_paddr(const struct apple_dart *dart, + u64 pte) +{ + return (pte & GENMASK_ULL(37, 10)) << 4; +} + +static bool apple_dart_fw_slot_owned(const struct apple_dart_fw_root *fw, + const u64 *live, size_t i) +{ + return fw && fw->leaf[i] && READ_ONCE(live[i]) == fw->entry[i]; +} + +/* + * A root slot firmware already uses can still back Linux mappings when the + * firmware's leaf table translates every page Linux mapped there to the same + * physical address. That is the case for the boot loader's handoff, which + * re-exports live firmware translations as reserved IOVA regions; the + * hardware keeps walking the firmware table and Linux's copy stays private. + */ +static bool apple_dart_fw_slot_mirrors(const struct apple_dart *dart, + const struct apple_dart_fw_root *fw, + u64 next, size_t i, size_t entries) +{ + const u64 *ours_leaf, *fw_leaf; + size_t j; + + ours_leaf = phys_to_virt(apple_dart_pte_to_paddr(dart, next)); + fw_leaf = fw->leaf[i]; + for (j = 0; j < entries; j++) { + u64 ours = READ_ONCE(ours_leaf[j]); + u64 theirs = READ_ONCE(fw_leaf[j]); + + if (!(ours & APPLE_DART_PTE_VALID)) + continue; + if (!(theirs & APPLE_DART_PTE_VALID) || + apple_dart_pte_to_paddr(dart, ours) != + apple_dart_pte_to_paddr(dart, theirs)) + return false; + } + return true; +} + /* * A locked root belongs to firmware. Publish only into vacant slots and * remember exactly what we installed, so teardown cannot erase firmware - * mappings added after Linux attached. + * mappings added after Linux attached. A slot firmware already holds is + * never written; it is accepted once if it mirrors Linux's mappings, and + * apple_dart_check_fw_map() checks later mappings into it page by page. */ -static int apple_dart_publish_root(u64 *live, u64 *owned, const u64 *ours, +static int apple_dart_publish_root(const struct apple_dart *dart, + u64 *live, u64 *owned, const u64 *ours, + struct apple_dart_fw_root *fw, size_t entries) { size_t i; @@ -502,7 +566,12 @@ static int apple_dart_publish_root(u64 *live, u64 *owned, const u64 *ours, if (!next && !previous) continue; - if (READ_ONCE(live[i]) != previous) + if (READ_ONCE(live[i]) == previous) + continue; + if (previous || !apple_dart_fw_slot_owned(fw, live, i)) + return -EBUSY; + if (next != fw->accepted[i] && + !apple_dart_fw_slot_mirrors(dart, fw, next, i, entries)) return -EBUSY; } @@ -513,6 +582,10 @@ static int apple_dart_publish_root(u64 *live, u64 *owned, const u64 *ours, if (next == previous) continue; + if (!previous && apple_dart_fw_slot_owned(fw, live, i)) { + fw->accepted[i] = next; + continue; + } if (cmpxchg(&live[i], previous, next) != previous) return -EBUSY; WRITE_ONCE(owned[i], next); @@ -536,6 +609,67 @@ static void apple_dart_retire_root(u64 *live, u64 *owned, size_t entries) dma_wmb(); } +static void apple_dart_free_fw_root(struct apple_dart *dart, + struct apple_dart_fw_root *fw) +{ + size_t i; + + if (!fw) + return; + if (fw->leaf) + for (i = 0; i < dart->pgsize / sizeof(u64); i++) + if (fw->leaf[i]) + memunmap(fw->leaf[i]); + kfree(fw->accepted); + kfree(fw->leaf); + kfree(fw->entry); + kfree(fw); +} + +/* + * Record the leaf tables behind the slots firmware holds in a three-level + * T8110-format locked root, on DARTs whose loader re-exports firmware + * translations (fw_mirror). Other roots are not snapshotted; their firmware + * slots keep rejecting Linux mappings as before. + */ +static struct apple_dart_fw_root * +apple_dart_snapshot_fw_root(struct apple_dart *dart, int sid, const u64 *live) +{ + size_t i, entries = dart->pgsize / sizeof(u64); + struct apple_dart_fw_root *fw; + + /* Only the T8110-style table layout this walker decodes. */ + if (!dart->fw_mirror || dart->hw->fmt != APPLE_DART2) + return NULL; + if (dart->hw->tcr_4level && + (apple_dart_readl(dart, DART_TCR(dart, sid)) & dart->hw->tcr_4level)) + return NULL; + + fw = kzalloc(sizeof(*fw), GFP_KERNEL); + if (!fw) + return NULL; + fw->entry = kcalloc(entries, sizeof(*fw->entry), GFP_KERNEL); + fw->leaf = kcalloc(entries, sizeof(*fw->leaf), GFP_KERNEL); + fw->accepted = kcalloc(entries, sizeof(*fw->accepted), GFP_KERNEL); + if (!fw->entry || !fw->leaf || !fw->accepted) { + apple_dart_free_fw_root(dart, fw); + return NULL; + } + + for (i = 0; i < entries; i++) { + u64 pte = READ_ONCE(live[i]); + + if (!(pte & APPLE_DART_PTE_VALID)) + continue; + fw->leaf[i] = memremap(apple_dart_pte_to_paddr(dart, pte), + dart->pgsize, MEMREMAP_WB); + if (!fw->leaf[i]) + continue; + fw->entry[i] = pte; + } + return fw; +} + static int apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) { @@ -543,6 +677,8 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) int sid; for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + struct apple_dart_fw_root *fw; + unsigned long flags; u32 ttbr; phys_addr_t phys; u64 *l1_tbl, *owned; @@ -574,8 +710,12 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) devm_memunmap(dart->dev, l1_tbl); return -ENOMEM; } + fw = apple_dart_snapshot_fw_root(dart, sid, l1_tbl); + spin_lock_irqsave(&dart->lock, flags); dart->locked_owned[sid][idx] = owned; dart->locked_ttbr[sid][idx] = l1_tbl; + dart->locked_fw[sid][idx] = fw; + spin_unlock_irqrestore(&dart->lock, flags); } return 0; @@ -590,10 +730,17 @@ apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { u64 *live = dart->locked_ttbr[sid][idx]; u64 *owned = dart->locked_owned[sid][idx]; + struct apple_dart_fw_root *fw; + unsigned long flags; if (!live) continue; apple_dart_retire_root(live, owned, dart->pgsize / sizeof(*live)); + spin_lock_irqsave(&dart->lock, flags); + fw = dart->locked_fw[sid][idx]; + dart->locked_fw[sid][idx] = NULL; + spin_unlock_irqrestore(&dart->lock, flags); + apple_dart_free_fw_root(dart, fw); devm_memunmap(dart->dev, live); devm_kfree(dart->dev, owned); dart->locked_ttbr[sid][idx] = NULL; @@ -628,7 +775,8 @@ apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, ret = -EIO; goto out; } - ret = apple_dart_publish_root(live, owned, ours, + ret = apple_dart_publish_root(dart, live, owned, ours, + dart->locked_fw[sid][idx], dart->pgsize / sizeof(*live)); if (ret) goto out; @@ -950,18 +1098,111 @@ static int apple_dart_iotlb_sync_map(struct iommu_domain *domain, iova, iova + size - 1); } +/* + * Look up a domain address in the firmware-owned slots of its locked roots. + * Returns true if a firmware slot covers @dva; *phys is then the firmware's + * translation of that page, or 0 if the firmware leaves it unmapped. + */ +static bool apple_dart_fw_lookup(struct apple_dart_domain *domain, u64 dva, + phys_addr_t *phys) +{ + struct apple_dart_atomic_stream_map *map; + bool owned = false; + int i, sid; + + *phys = 0; + for_each_stream_map(i, domain, map) { + struct apple_dart *dart = map->dart; + u32 entries = dart->pgsize / sizeof(u64); + u32 pgshift = ilog2(dart->pgsize); + u32 bits = ilog2(entries); + u64 leaf = (dva >> pgshift) & (entries - 1); + u64 slot = (dva >> (pgshift + bits)) & (entries - 1); + u64 idx = dva >> (pgshift + 2 * bits); + unsigned long flags; + + if (!dart->fw_mirror || idx >= DART_MAX_TTBR) + continue; + + spin_lock_irqsave(&dart->lock, flags); + for (sid = 0; sid < dart->num_streams && !owned; sid++) { + const struct apple_dart_fw_root *fw; + u64 pte; + + if (!(atomic_long_read(&map->sidmap[BIT_WORD(sid)]) & + BIT_MASK(sid))) + continue; + fw = dart->locked_fw[sid][idx]; + if (!dart->locked_ttbr[sid][idx] || + !apple_dart_fw_slot_owned(fw, dart->locked_ttbr[sid][idx], + slot)) + continue; + owned = true; + pte = READ_ONCE(fw->leaf[slot][leaf]); + if (pte & APPLE_DART_PTE_VALID) + *phys = apple_dart_pte_to_paddr(dart, pte); + } + spin_unlock_irqrestore(&dart->lock, flags); + if (owned) + return true; + } + return false; +} + +/* + * Firmware buffers the coprocessor later shares with the host live in the + * firmware's own tables, not in Linux's; resolve them there. + */ +static phys_addr_t apple_dart_fw_iova_to_phys(struct apple_dart_domain *domain, + u64 dva) +{ + phys_addr_t phys; + u32 pgsize = domain->domain.pgsize_bitmap; + + if (!apple_dart_fw_lookup(domain, dva, &phys) || !phys) + return 0; + return phys | (dva & (pgsize - 1)); +} + +/* + * Linux mappings into a firmware-owned slot are never published; the + * hardware keeps walking the firmware's table. Only accept them where the + * firmware already provides the same translation. + */ +static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, + phys_addr_t paddr, size_t size) +{ + u32 pgsize = domain->domain.pgsize_bitmap; + size_t off; + + for (off = 0; off < size; off += pgsize) { + phys_addr_t phys; + + if (apple_dart_fw_lookup(domain, dva + off, &phys) && + phys != paddr + off) + return -EBUSY; + } + return 0; +} + static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; + phys_addr_t phys; + u64 dva; + if (!ops) return 0; - return ops->iova_to_phys(ops, - (iova + dart_domain->dma_offset) & - dart_domain->mask); + dva = (iova + dart_domain->dma_offset) & dart_domain->mask; + phys = ops->iova_to_phys(ops, dva); + if (phys) + return phys; + + return apple_dart_fw_iova_to_phys(dart_domain, dva); } static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, @@ -972,13 +1213,19 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; + u64 dva; + int ret; + if (!ops) return -ENODEV; - return ops->map_pages(ops, - (iova + dart_domain->dma_offset) & - dart_domain->mask, paddr, pgsize, - pgcount, prot, gfp, mapped); + dva = (iova + dart_domain->dma_offset) & dart_domain->mask; + ret = apple_dart_check_fw_map(dart_domain, dva, paddr, pgsize * pgcount); + if (ret) + return ret; + + return ops->map_pages(ops, dva, paddr, pgsize, pgcount, prot, gfp, + mapped); } static size_t apple_dart_unmap_pages(struct iommu_domain *domain, @@ -2140,6 +2387,9 @@ static int apple_dart_probe(struct platform_device *pdev) dart->locked = false; else dart->locked = apple_dart_is_locked(dart); + /* T8142 loaders re-export the firmware's own display translations. */ + dart->fw_mirror = dart->locked && + of_device_is_compatible(pdev->dev.of_node, "apple,t8142-dart"); if (!dart->locked) { ret = apple_dart_hw_reset(dart); if (ret) From 49af2eb862fd861dcd8bc624b9c420f1a2c55343 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 23:34:50 -0400 Subject: [PATCH 0380/1579] drm: apple: support H17-generation DCP firmware on T8142 iBoot leaves the T8142 (M5) display coprocessor running with macOS 26 ("H17") firmware, mapped through locked DARTs. Add support for that firmware generation and attach to the live session: - Match apple,t8142-dcp and pin the H17 interface. Attach to the running coprocessor with the standard RTKit wake handshake; never stop, reset or reload it. - Add the H17 IOMFB method and callback tables, with T8142 method aliases, the nested-call reply handling, the AFK receive layout and the service-init keys this firmware uses, and the DCP bandwidth and clock requests answered from DT resources. - Take the panel dimensions from the firmware at startup and keep the mode the boot loader programmed. Only that mode is offered. - Skip teardown calls whose H17 method numbers are not established, and don't send AFK requests on H17, whose send format is unknown. Linux loads no firmware here, and no panel timing, geometry, power model or other calibration data is included. Optional properties the firmware asks for are answered "not found". Changes to shared code that alter behaviour (ACK reply buffers, framebuffer release, EPIC service-init keys) are limited to H17 firmware; v12.3 and v13.5 keep their existing paths. Co-authored-by: Dj <43033058+DjDeveloperr@users.noreply.github.com> --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/afk.c | 121 +++- drivers/gpu/drm/apple/afk.h | 46 +- drivers/gpu/drm/apple/dcp-internal.h | 36 + drivers/gpu/drm/apple/dcp-link.h | 95 +++ drivers/gpu/drm/apple/dcp.c | 220 ++++++- drivers/gpu/drm/apple/iomfb.c | 125 +++- drivers/gpu/drm/apple/iomfb.h | 57 +- drivers/gpu/drm/apple/iomfb_h17p.c | 239 +++++++ drivers/gpu/drm/apple/iomfb_h17p.h | 22 + drivers/gpu/drm/apple/iomfb_template.c | 869 +++++++++++++++++++++++-- drivers/gpu/drm/apple/iomfb_template.h | 126 +++- drivers/gpu/drm/apple/iomfb_v12_3.c | 1 + drivers/gpu/drm/apple/iomfb_v13_3.c | 1 + drivers/gpu/drm/apple/parser.c | 25 +- drivers/gpu/drm/apple/parser.h | 2 +- 16 files changed, 1845 insertions(+), 141 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-link.h create mode 100644 drivers/gpu/drm/apple/iomfb_h17p.c create mode 100644 drivers/gpu/drm/apple/iomfb_h17p.h diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index b22839bdd611a..c511c53c5447c 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -10,6 +10,7 @@ appledrm-y += connector.o appledrm-y += ibootep.o appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o +appledrm-y += iomfb_h17p.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 52054b6845490..4a4215e0af2df 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -104,7 +104,9 @@ int afk_start(struct apple_dcp_afkep *ep) int ret; reinit_completion(&ep->started); - apple_rtkit_start_ep(ep->dcp->rtk, ep->endpoint); + ret = apple_rtkit_start_ep(ep->dcp->rtk, ep->endpoint); + if (ret) + return ret; afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_INIT)); ret = wait_for_completion_timeout(&ep->started, msecs_to_jiffies(1000)); @@ -116,7 +118,7 @@ int afk_start(struct apple_dcp_afkep *ep) static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) { - u16 size = FIELD_GET(GETBUF_SIZE, message) << BLOCK_SHIFT; + u32 size = FIELD_GET(GETBUF_SIZE, message) << BLOCK_SHIFT; u16 tag = FIELD_GET(GETBUF_TAG, message); u64 reply; @@ -147,10 +149,10 @@ static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, struct afk_ringbuffer *bfr) { - u16 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; - u16 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; + u32 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; + u32 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; u16 tag = FIELD_GET(INITRB_TAG, message); - u32 bufsz, end; + u32 bufsz, hdrsz, end, block; if (tag != ep->bfr_tag) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: expected tag 0x%x but got 0x%x\n", @@ -164,7 +166,7 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, return; } - if (base >= ep->bfr_size) { + if (!ep->bfr || size < sizeof(__le32) || base >= ep->bfr_size) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: requested base 0x%x >= max size 0x%lx\n", ep->endpoint, base, ep->bfr_size); @@ -180,15 +182,21 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, } bfr->hdr = ep->bfr + base; - bufsz = le32_to_cpu(bfr->hdr->bufsz); - if (bufsz + sizeof(*bfr->hdr) != size) { + dma_rmb(); + bufsz = le32_to_cpu(READ_ONCE(*(__le32 *)bfr->hdr)); + hdrsz = size - bufsz; + block = hdrsz / AFK_RB_BLOCKS; + if (bufsz >= size || hdrsz % AFK_RB_BLOCKS || + block < AFK_RB_BLOCK_MIN || !is_power_of_2(block) || + !IS_ALIGNED(bufsz, block) || bufsz < 2 * block) { dev_err(ep->dcp->dev, - "AFK[ep:%02x]: ring buffer size 0x%x != expected 0x%lx\n", - ep->endpoint, bufsz, sizeof(*bfr->hdr)); + "AFK[ep:%02x]: ring bufsz %#x with size %#x gives no valid header (base %#x, msg %#llx)\n", + ep->endpoint, bufsz, size, base, message); return; } + bfr->block = block; - bfr->buf = bfr->hdr + 1; + bfr->buf = (u8 *)bfr->hdr + hdrsz; bfr->bufsz = bufsz; bfr->ready = true; @@ -312,7 +320,8 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, ep->endpoint, name); return; } - ret = parse_epic_service_init(&ctx, &epic_name, &epic_class, &epic_unit); + ret = parse_epic_service_init(&ctx, &epic_name, &epic_class, &epic_unit, + ep->dcp->fw_compat == DCP_FIRMWARE_H17P); if (ret) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: failed to extract init props: %d\n", @@ -553,16 +562,51 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, struct epic_hdr *ehdr = (struct epic_hdr *)data; struct epic_sub_hdr *eshdr = (struct epic_sub_hdr *)(data + sizeof(*ehdr)); - u16 subtype = le16_to_cpu(eshdr->type); - u8 *payload = data + sizeof(*ehdr) + sizeof(*eshdr); + struct epic_sub_hdr h17p_eshdr; + size_t hdr_len = sizeof(*ehdr) + sizeof(*eshdr); + u16 subtype; + u8 *payload; size_t payload_size; - if (data_size < sizeof(*ehdr) + sizeof(*eshdr)) { - dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", - ep->endpoint, data_size); - return; + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P) { + const struct epic_sub_hdr_h17p *c = + (const struct epic_sub_hdr_h17p *)(data + sizeof(*ehdr)); + + hdr_len = sizeof(*ehdr) + sizeof(*c); + if (data_size < hdr_len) + goto too_small; + + /* + * H17P multiplexes every service of an endpoint onto queue + * channel 0 and carries the real channel in the EPIC header + * instead: byte 0 is a sequence counter, byte 1 a flags byte + * and bytes 2-3 the channel. ep:20 announces "system" as + * channel 1 and "powerlog-service" as channel 3, then sends the + * mNits/uAmps/iDAC backlight reports on channel 3. Taking the + * queue channel at face value binds "system" to 0 and makes + * every later announce and report look like a duplicate. + */ + channel = le16_to_cpup((__le16 *)(data + 2)); + + /* + * Normalise into the wide sub-header the rest of this file + * expects, mapping H17P's announce subtype onto the canonical + * one so no downstream check has to know the difference. + */ + memset(&h17p_eshdr, 0, sizeof(h17p_eshdr)); + h17p_eshdr.category = c->category; + h17p_eshdr.type = cpu_to_le16(c->type == EPIC_SUBTYPE_ANNOUNCE_H17P ? + EPIC_SUBTYPE_ANNOUNCE : c->type); + h17p_eshdr.tag = cpu_to_le16(le32_to_cpu(c->tag)); + eshdr = &h17p_eshdr; } - payload_size = data_size - sizeof(*ehdr) - sizeof(*eshdr); + + if (data_size < hdr_len) + goto too_small; + + subtype = le16_to_cpu(eshdr->type); + payload = data + hdr_len; + payload_size = data_size - hdr_len; trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); @@ -620,6 +664,11 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, "(type %x subtype %x)\n", ep->endpoint, channel, type, subtype); print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, payload_size, true); + return; + +too_small: + dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", + ep->endpoint, data_size); } static bool afk_recv(struct apple_dcp_afkep *ep) @@ -634,8 +683,8 @@ static bool afk_recv(struct apple_dcp_afkep *ep) return false; } - rptr = le32_to_cpu(ep->rxbfr.hdr->rptr); - wptr = le32_to_cpu(ep->rxbfr.hdr->wptr); + rptr = le32_to_cpu(*afk_rb_rptr(&ep->rxbfr)); + wptr = le32_to_cpu(*afk_rb_wptr(&ep->rxbfr)); trace_afk_recv_rwptr_pre(ep, rptr, wptr); if (rptr == wptr) @@ -680,7 +729,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) return false; } - ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); + *afk_rb_rptr(&ep->rxbfr) = cpu_to_le32(rptr); } if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { @@ -693,7 +742,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) channel = le32_to_cpu(hdr->channel); type = le32_to_cpu(hdr->type); - rptr = ALIGN(rptr + sizeof(*hdr) + size, 1 << BLOCK_SHIFT); + rptr = ALIGN(rptr + sizeof(*hdr) + size, ep->rxbfr.block); if (WARN_ON(rptr > ep->rxbfr.bufsz)) rptr = 0; if (rptr == ep->rxbfr.bufsz) @@ -701,7 +750,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) dma_mb(); - ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); + *afk_rb_rptr(&ep->rxbfr) = cpu_to_le32(rptr); trace_afk_recv_rwptr_post(ep, rptr, wptr); /* @@ -790,11 +839,27 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, size_t total_epic_size, total_size; int ret; + /* + * H17P frames EPIC messages differently (see afk_recv_handle()): an + * 8-byte sub-header, with the service channel carried in the EPIC + * header on queue channel 0. Only the receive side of that format is + * known, and the 24-byte framing below would be misparsed, so send + * nothing. The display path needs no AP-initiated EPIC traffic: the + * callers are optional (verbose firmware logging, EDID copy, audio, + * DPTX, debugfs) or replies to calls only DPTX services make. + */ + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P) { + dev_dbg_once(ep->dcp->dev, + "AFK[ep:%02x]: not sending EPIC message on H17P\n", + ep->endpoint); + return -EOPNOTSUPP; + } + spin_lock_irqsave(&ep->lock, flags); dma_rmb(); - rptr = le32_to_cpu(ep->txbfr.hdr->rptr); - wptr = le32_to_cpu(ep->txbfr.hdr->wptr); + rptr = le32_to_cpu(*afk_rb_rptr(&ep->txbfr)); + wptr = le32_to_cpu(*afk_rb_wptr(&ep->txbfr)); trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; total_size = sizeof(*hdr) + total_epic_size; @@ -907,12 +972,12 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, memcpy(ep->txbfr.buf + wptr, payload, payload_len); wptr += payload_len; - wptr = ALIGN(wptr, 1 << BLOCK_SHIFT); + wptr = ALIGN(wptr, ep->txbfr.block); if (wptr == ep->txbfr.bufsz) wptr = 0; trace_afk_send_rwptr_post(ep, rptr, wptr); - ep->txbfr.hdr->wptr = cpu_to_le32(wptr); + *afk_rb_wptr(&ep->txbfr) = cpu_to_le32(wptr); afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_SEND) | FIELD_PREP(SEND_WPTR, wptr)); ret = 0; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index a339c00a2a013..6ba47202e23a9 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -71,15 +71,17 @@ struct apple_epic_service_ops { void (*teardown)(struct apple_epic_service *service); }; -struct afk_ringbuffer_header { - __le32 bufsz; - u32 unk; - u32 _pad1[14]; - __le32 rptr; - u32 _pad2[15]; - __le32 wptr; - u32 _pad3[15]; -}; +/* + * The ring header is three control blocks -- bufsz, rptr, wptr -- each padded + * out to the firmware's control-block size. That block is 0x40 bytes up to + * v13.5 but 0x80 on H17P, making the header 0xc0 or 0x180. The size is + * self-describing: the INITRB message gives the total ring size and the first + * word of block 0 gives the payload size, so the stride is (size - bufsz) / 3 + * and no firmware-version test is needed. Address the fields by stride rather + * than through a fixed struct. + */ +#define AFK_RB_BLOCK_MIN 0x40 +#define AFK_RB_BLOCKS 3 struct afk_qe { #define QE_MAGIC 0x20504f49 // ' POI' @@ -109,6 +111,21 @@ struct epic_sub_hdr { __le32 inline_len; } __attribute__((packed)); +/* + * H17P compresses the EPIC sub-header from 24 bytes to 8 and drops the + * separate length/timestamp: the announce for the "system" service puts its + * name at data + 0x18, not data + 0x28. An announce carries type 0x11 and a + * standard-service report carries the usual 0xc0. + */ +struct epic_sub_hdr_h17p { + u8 type; + u8 category; + __le16 flags; + __le32 tag; +} __packed; + +#define EPIC_SUBTYPE_ANNOUNCE_H17P 0x11 + struct epic_cmd { __le32 retcode; __le64 rxbuf; @@ -152,12 +169,21 @@ enum epic_subtype { struct afk_ringbuffer { bool ready; - struct afk_ringbuffer_header *hdr; + void *hdr; + u32 block; /* control-block stride, 0x40 or 0x80 */ u32 rptr; void *buf; size_t bufsz; }; +static inline __le32 *afk_rb_field(struct afk_ringbuffer *bfr, unsigned int idx) +{ + return (__le32 *)((u8 *)bfr->hdr + idx * bfr->block); +} + +#define afk_rb_rptr(bfr) afk_rb_field(bfr, 1) +#define afk_rb_wptr(bfr) afk_rb_field(bfr, 2) + struct apple_dcp_afkep { struct apple_dcp *dcp; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index ff3dec30e8fe7..0a237c86e343e 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #include "dptxep.h" #include "iomfb.h" +#include "iomfb_h17p.h" #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "epic/dpavservep.h" @@ -48,6 +50,7 @@ enum dcp_firmware_version { DCP_FIRMWARE_UNKNOWN, DCP_FIRMWARE_V_12_3, DCP_FIRMWARE_V_13_5, + DCP_FIRMWARE_H17P, }; enum { @@ -78,6 +81,8 @@ struct dcp_mem_descriptor { dma_addr_t dva; struct sg_table map; u64 reg; + /* live iommu mappings installed by dcpep_cb_map_piodma() */ + bool piodma_mapped; }; /* Limit on call stack depth (arbitrary). Some nesting is required */ @@ -100,7 +105,14 @@ struct dcp_channel { struct dcp_fb_reference { struct list_head head; struct drm_framebuffer *fb; + /* + * Id of the swap that unbinds @fb, i.e. the one that puts its + * replacement on screen. Only meaningful once @armed is set: the + * firmware assigns the id in the swap_start reply, which happens after + * the atomic commit that displaced @fb has already queued this entry. + */ u32 swap_id; + bool armed; }; #define MAX_NOTCH_HEIGHT 160 @@ -126,8 +138,20 @@ struct dcp_panel { bool has_mini_led; }; +enum dcp_iomfb_method_profile { + DCP_IOMFB_METHODS_DEFAULT, + DCP_IOMFB_METHODS_H17G, +}; + struct apple_dcp_hw_data { u32 num_dptx_ports; + enum dcp_iomfb_method_profile iomfb_method_profile; + /* + * The bootloader leaves the coprocessor running. Attach to it with a + * standard RTKit wake instead of starting or restarting the ASC. + */ + bool adopt_live_session; + enum dcp_firmware_version firmware_compat; }; /* TODO: move IOMFB members to its own struct */ @@ -135,6 +159,8 @@ struct apple_dcp { struct device *dev; struct platform_device *piodma; struct iommu_domain *iommu_dom; + /* which of the nine cumulative A031 notify-client states to send next */ + unsigned int a031_step; struct apple_rtkit *rtk; struct apple_crtc *crtc; struct apple_connector *connector; @@ -159,6 +185,7 @@ struct apple_dcp { /* clock rate request by dcp in */ struct clk *clk; + struct clk *clk_194; /* DCP shared memory */ void *shmem; @@ -195,10 +222,13 @@ struct apple_dcp { union { struct dcp_swap_submit_req_v12_3 v12_3; struct dcp_swap_submit_req_v13_3 v13_3; + struct dcp_swap_submit_req_h17p h17p; } swap; /* swap id of the last completed swap */ u32 last_swap_id; + /* last_swap_id is only meaningful after the first swap completes */ + bool have_swap_complete; ktime_t swap_start; u64 swap_submit_timestamp; @@ -247,6 +277,12 @@ struct apple_dcp { * on the next successfully completed swap. */ struct list_head swapped_out_fbs; + /* + * Protects swapped_out_fbs, which is appended to from the DRM atomic + * commit (dcp_flush -> .atomic_flush) and armed/drained from the RTKit + * workqueue that runs the DCP callbacks. + */ + spinlock_t swapped_out_lock; struct dcp_brightness brightness; /* Workqueue for updating the initial brightness */ diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h new file mode 100644 index 0000000000000..ae1fcabdfc0b2 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_DCP_LINK_H__ +#define __APPLE_DCP_LINK_H__ + +#include +#include +#include + +#include + +/* + * H17-generation DCP firmware uses the ordinary IOMFB transport once the + * INIT/SetShmem handshake is done: the same six fixed channels (CMD 0x00000, + * OOBCMD 0x08000, ASYNC 0x40000, OOBASYNC 0x48000, CB 0x60000, + * OOBCB 0x68000), selected by the context field of the message. Records are + * allocated on each channel's stack at + * off_next = off_cur + align_up(12 + in_len + out_len, 0x40) + * which is the rule dcp_packet_start() and DCP_PACKET_ALIGNMENT implement in + * iomfb.c. + * + * This header only describes the callback payloads whose layout differs from + * older firmware. + */ + +#define APPLE_DCP_H17P_CLOCK_ID_154 0x154 +#define APPLE_DCP_H17P_CLOCK_ID_194 0x194 + +struct apple_dcp_h17p_clock_request { + __le32 service; + __le32 clock_id; +} __packed; + +struct apple_dcp_h17p_rt_bw_config { + __le64 scratch; + __le64 clock_request; +} __packed; + +struct apple_dcp_h17p_rt_bw_request { + __le32 config_null; +} __packed; + +struct apple_dcp_h17p_rt_bw_reply { + struct apple_dcp_h17p_rt_bw_config config; + __le32 status; +} __packed; + +static_assert(sizeof(struct apple_dcp_h17p_clock_request) == 0x08); +static_assert(sizeof(struct apple_dcp_h17p_rt_bw_config) == 0x10); +static_assert(sizeof(struct apple_dcp_h17p_rt_bw_request) == 0x04); +static_assert(sizeof(struct apple_dcp_h17p_rt_bw_reply) == 0x14); + +static inline int +apple_dcp_h17p_clock_rate(const struct apple_dcp_h17p_clock_request *request, + u64 rate_154, u64 rate_194, u64 *rate) +{ + if (!request || !rate) + return -EINVAL; + + switch (le32_to_cpu(request->clock_id)) { + case APPLE_DCP_H17P_CLOCK_ID_154: + *rate = rate_154; + return 0; + case APPLE_DCP_H17P_CLOCK_ID_194: + *rate = rate_194; + return 0; + default: + return -ENOENT; + } +} + +static inline void +apple_dcp_h17p_rt_bw_encode(struct apple_dcp_h17p_rt_bw_config *config, + u64 scratch, u64 clock_request) +{ + config->scratch = cpu_to_le64(scratch); + config->clock_request = cpu_to_le64(clock_request); +} + +/* D003: fill in the bandwidth config unless the firmware passed it as null */ +static inline int +apple_dcp_h17p_rt_bw_encode_reply(const struct apple_dcp_h17p_rt_bw_request *request, + struct apple_dcp_h17p_rt_bw_reply *reply, + u64 scratch, u64 clock_request) +{ + if (le32_to_cpu(request->config_null) & 1) + return -EINVAL; + + apple_dcp_h17p_rt_bw_encode(&reply->config, scratch, clock_request); + reply->status = cpu_to_le32(0); + return 0; +} + +#endif /* __APPLE_DCP_LINK_H__ */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8eac2f98031f0..fc28b87045dfc 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -826,12 +827,42 @@ static void dcp_delayed_vblank(struct work_struct *work) dcp_drm_crtc_vblank(dcp->crtc); } +static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) +{ + switch (endpoint) { + case AV_ENDPOINT: + return dcp->avep; + case SYSTEM_ENDPOINT: + return dcp->systemep; + case DISP0_ENDPOINT: + return dcp->ibootep; + case DPAVSERV_ENDPOINT: + return dcp->dcpavservep; + case DPTX_ENDPOINT: + return dcp->dptxep; + default: + return NULL; + } +} + static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) { struct apple_dcp *dcp = cookie; trace_dcp_recv_msg(dcp, endpoint, message); + /* + * H17P starts endpoints that have no AFK instance behind them (the + * remote allocator endpoint, for one). Log their messages rather than + * dereferencing a NULL afkep or warning about an unknown endpoint. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P && endpoint != IOMFB_ENDPOINT && + !dcp_afkep(dcp, endpoint)) { + dev_dbg_ratelimited(dcp->dev, "ep %#04x: %#llx\n", endpoint, + message); + return; + } + switch (endpoint) { case IOMFB_ENDPOINT: return iomfb_recv_msg(dcp, message); @@ -870,9 +901,32 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ complete(&dcp->start_done); } +/* + * Check that every IOMMU page of [iova, iova + size) translates to the same + * offset from @phys as it has from @iova. + */ +static int dcp_check_iova_contiguous(struct iommu_domain *domain, + dma_addr_t iova, size_t size, + phys_addr_t phys) +{ + unsigned long pgsize = domain->pgsize_bitmap ? + 1UL << __ffs(domain->pgsize_bitmap) : PAGE_SIZE; + dma_addr_t end = iova + size; + dma_addr_t addr = iova; + + while (addr < end) { + if (iommu_iova_to_phys(domain, addr) != phys + (addr - iova)) + return -EINVAL; + addr = ALIGN_DOWN(addr, pgsize) + pgsize; + } + + return 0; +} + static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) { struct apple_dcp *dcp = cookie; + int ret; if (bfr->iova) { struct iommu_domain *domain = @@ -887,6 +941,16 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) if (!phy_addr) return -ENOMEM; + /* memremap() below needs the whole buffer physically contiguous */ + ret = dcp_check_iova_contiguous(domain, bfr->iova, bfr->size, + phy_addr); + if (ret) { + dev_err(dcp->dev, + "shmem_setup: iova %pad (%#zx bytes) is not physically contiguous\n", + &bfr->iova, bfr->size); + return ret; + } + // TODO: verify phy_addr, cache attribute bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WB); if (!bfr->buffer) @@ -1245,6 +1309,18 @@ int dcp_start(struct platform_device *pdev) init_completion(&dcp->start_done); + /* + * The coprocessor start can fail (or be recovered from) without tearing + * the DRM side down, so this can be reached with no RTKit instance at + * all. Starting an endpoint then dereferences NULL inside + * apple_rtkit_start_ep() and takes the machine with it. + */ + if (!dcp->rtk || IS_ERR(dcp->rtk)) { + dev_err(dcp->dev, + "cannot start endpoints: RTKit is not initialised\n"); + return -ENODEV; + } + /* start RTKit endpoints */ ret = systemep_init(dcp); if (ret) @@ -1319,6 +1395,9 @@ static void _dcp_poweroff(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_poweroff_v13_3(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_poweroff_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -1381,6 +1460,9 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_sleep_v13_3(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_sleep_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -1423,6 +1505,9 @@ void dcp_poweron(struct platform_device *pdev) case DCP_FIRMWARE_V_13_5: iomfb_poweron_v13_3(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_poweron_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -1513,8 +1598,30 @@ static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) "Failed to get piodma child DT node\n"); dcp->piodma = of_platform_device_create(node, NULL, dcp->dev); - if (!dcp->piodma) - return dev_err_probe(dcp->dev, -ENODEV, "Failed to create piodma pdev for %pOF\n", node); + if (!dcp->piodma) { + /* + * of_platform_device_create() returns NULL when the node already + * has a platform device, which happens whenever a previous bind + * attempt created it and then failed further down (the caller's + * later error paths do not destroy it). Probe is retried - a + * deferral, or a component re-bind - and would then fail here + * forever with -ENODEV even though nothing is actually wrong. + * Adopt the existing device instead of giving up. + */ + dcp->piodma = of_find_device_by_node(node); + if (!dcp->piodma) + return dev_err_probe(dcp->dev, -ENODEV, + "Failed to create piodma pdev for %pOF\n", + node); + /* + * Drop the lookup reference: the device stays registered until + * of_platform_device_destroy(), exactly like one created above, + * and that call only releases the registration reference. + */ + put_device(&dcp->piodma->dev); + dev_info(dcp->dev, "reusing existing piodma pdev for %pOF\n", + node); + } ret = dma_set_mask_and_coherent(&dcp->piodma->dev, DMA_BIT_MASK(42)); if (ret) @@ -1639,18 +1746,67 @@ static int dcp_get_bw_doorbell_reg(struct apple_dcp *dcp, u32 expected) return ret; } +/* + * Count the "disp-*" entries in reg-names. + * + * The positional form below assumes the reg layout is [coproc, disp-0 ...], + * i.e. exactly one non-disp register, and derives the count as + * num_resources - 1. DCPs with H17-generation firmware list a trailing + * "iop-vbar" register after the display apertures, which would inflate that + * count, bind the last disp_registers[] entry to iop-vbar and make the + * "apple,bw-scratch" disp_reg index check reject a valid device tree. + * + * Returns a negative errno when reg-names is absent, so callers fall back. + */ +static int dcp_count_disp_regs(struct device *dev) +{ + int n = of_property_count_strings(dev->of_node, "reg-names"); + const char *name; + int i, count = 0; + + if (n <= 0) + return -EINVAL; + + for (i = 0; i < n; ++i) { + if (of_property_read_string_index(dev->of_node, "reg-names", i, + &name)) + return -EINVAL; + if (!strncmp(name, "disp-", 5)) + count++; + } + + return count ? count : -EINVAL; +} + static int dcp_get_disp_regs(struct apple_dcp *dcp) { struct platform_device *pdev = to_platform_device(dcp->dev); - int count = pdev->num_resources - 1; + int count = dcp_count_disp_regs(dcp->dev); + bool by_name = count > 0; int i, ret; + /* Device trees without reg-names keep the historical positional form. */ + if (!by_name) + count = pdev->num_resources - 1; + if (count <= 0 || count > MAX_DISP_REGISTERS) return -EINVAL; for (i = 0; i < count; ++i) { - dcp->disp_registers[i] = - platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); + if (by_name) { + char name[8]; + + snprintf(name, sizeof(name), "disp-%d", i); + dcp->disp_registers[i] = platform_get_resource_byname( + pdev, IORESOURCE_MEM, name); + } else { + dcp->disp_registers[i] = platform_get_resource( + pdev, IORESOURCE_MEM, 1 + i); + } + if (!dcp->disp_registers[i]) { + dev_err(dcp->dev, "missing display register %d\n", i); + return -EINVAL; + } } /* load pmgr bandwidth scratch resource and offset */ @@ -1717,10 +1873,18 @@ static int dcp_read_fw_version(struct device *dev, const char *name, static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) { + const struct apple_dcp_hw_data *hw = of_device_get_match_data(dev); char compat_str[DCP_FW_VERSION_STR_LEN]; char fw_str[DCP_FW_VERSION_STR_LEN]; int ret; + /* + * SoCs introduced with H17-generation firmware pin its interface; the + * loader may not recognise their boot firmware version. + */ + if (hw->firmware_compat != DCP_FIRMWARE_UNKNOWN) + return hw->firmware_compat; + /* firmware version is just informative */ dcp_read_fw_version(dev, "apple,firmware-version", fw_str); @@ -1778,6 +1942,20 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (IS_ERR(dcp->coproc_reg)) return PTR_ERR(dcp->coproc_reg); + /* + * Display coprocessors running H17-generation firmware are left running + * by the bootloader with their firmware mapped through a locked DART. + * Linux attaches to that session with a standard RTKit INIT/HELLO + * exchange and must not stop or restart the ASC. + */ + if (dcp->hw.adopt_live_session) { + cpu_ctrl = readl_relaxed(dcp->coproc_reg + + APPLE_DCP_COPROC_CPU_CONTROL); + if (!(cpu_ctrl & APPLE_DCP_COPROC_CPU_CONTROL_RUN)) + return dev_err_probe(dev, -EOPNOTSUPP, + "DCP was not left running by the bootloader\n"); + } + if (dcp->index || dcp->dptx_phy || dcp->dptx_die) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); @@ -1838,6 +2016,10 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (IS_ERR(dcp->clk)) return dev_err_probe(dev, PTR_ERR(dcp->clk), "Unable to find clock\n"); + dcp->clk_194 = devm_clk_get_optional(dev, "clock-194"); + if (IS_ERR(dcp->clk_194)) + return dev_err_probe(dev, PTR_ERR(dcp->clk_194), + "Unable to find clock 0x194\n"); bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry @@ -1847,17 +2029,23 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); + spin_lock_init(&dcp->swapped_out_lock); - cpu_ctrl = - readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); - writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, - dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + if (!dcp->hw.adopt_live_session) { + cpu_ctrl = + readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, + dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + } dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); if (IS_ERR(dcp->rtk)) return dev_err_probe(dev, PTR_ERR(dcp->rtk), "Failed to initialize RTKit\n"); + if (dcp->hw.adopt_live_session) + dev_info(dev, "negotiating RTKit with the running DCP\n"); + ret = apple_rtkit_wake(dcp->rtk); if (ret) return dev_err_probe(dev, ret, @@ -1926,6 +2114,8 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) devm_clk_put(dev, dcp->clk); dcp->clk = NULL; + /* optional and possibly NULL, released with the bind's devres group */ + dcp->clk_194 = NULL; } static const struct component_ops dcp_comp_ops = { @@ -2162,6 +2352,17 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { .num_dptx_ports = 2, }; +/* + * M5 (T8142) runs H17-generation DCP firmware with the H17G method numbering + * and is left running by the bootloader. + */ +static const struct apple_dcp_hw_data apple_dcp_hw_t8142 = { + .num_dptx_ports = 0, + .iomfb_method_profile = DCP_IOMFB_METHODS_H17G, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_H17P, +}; + static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { .num_dptx_ports = 0, }; @@ -2173,6 +2374,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_dcpext = { static const struct of_device_id of_match[] = { { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, + { .compatible = "apple,t8142-dcp", .data = &apple_dcp_hw_t8142, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, {} diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 3ec33353d3abe..16f87163524d3 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -161,6 +161,13 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca u32 in_len, u32 out_len, void *data, dcp_callback_t cb, void *cookie) { + struct dcp_method_entry resolved = *call; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G && + call->tag_h17g[0]) + memcpy(resolved.tag, call->tag_h17g, sizeof(resolved.tag)); + call = &resolved; + enum dcp_context_id context = dcp_call_context(dcp, oob); struct dcp_channel *ch = dcp_get_channel(dcp, context); @@ -324,15 +331,30 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; - if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || !dcp->cb_handlers[tag]) { + in = data + sizeof(*hdr); + out = in + hdr->in_len; + + if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || + !dcp->cb_handlers[tag]) { dev_warn(dev, "received unknown callback %c%c%c%c\n", hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + /* + * Leaving a callback unanswered wedges the coprocessor: it + * waits for the ack forever and the outer call never returns. + * On H17P answer it with a zeroed output instead. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (hdr->out_len) + memset(out, 0, hdr->out_len); + depth = dcp_push_depth(&ch->depth); + ch->output[depth] = out; + ch->end[depth] = offset + + ALIGN(length, DCP_PACKET_ALIGNMENT); + dcp_ack(dcp, context); + } return; } - in = data + sizeof(*hdr); - out = in + hdr->in_len; - // TODO: verify that in_len and out_len match our prototypes // for now just clear the out data to have at least consistent results if (hdr->out_len) @@ -351,7 +373,8 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, { struct dcp_packet_header *header = data; struct dcp_channel *ch = dcp_get_channel(dcp, context); - void *cookie; + bool h17p = dcp->fw_compat == DCP_FIRMWARE_H17P; + void *cookie, *out; dcp_callback_t cb; if (!ch) { @@ -359,6 +382,11 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, return; } + if (h17p && !ch->depth) { + dev_warn(dcp->dev, "ignoring ack on idle context %X\n", context); + return; + } + dcp_pop_depth(&ch->depth); cb = ch->callbacks[ch->depth]; @@ -367,8 +395,21 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, ch->callbacks[ch->depth] = NULL; ch->cookies[ch->depth] = NULL; + if (h17p) { + /* + * H17P acks carry no payload and offset zero, including for + * nested calls, so the output cannot be located from the ack + * message. It is in the original AP command record, whose + * address dcp_push() saved. + */ + out = ch->output[ch->depth]; + ch->output[ch->depth] = NULL; + } else { + out = data + sizeof(*header) + header->in_len; + } + if (cb) - cb(dcp, data + sizeof(*header) + header->in_len, cookie); + cb(dcp, out, cookie); } static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) @@ -437,7 +478,9 @@ int dcp_get_modes(struct drm_connector *connector) } drm_connector_set_vrr_capable_property(connector, vrr_capable); + /* H17P sends no EPIC commands; see afk_send_epic(). */ if (dcp->nr_modes && dcp->dcpavserv.enabled && + dcp->fw_compat != DCP_FIRMWARE_H17P && !apple_connector->drm_edid) { const struct drm_edid *edid; edid = dcpavserv_copy_edid(dcp->dcpavserv.service); @@ -482,6 +525,27 @@ struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, return NULL; } +/* + * H17P keeps the mode the bootloader programmed and never sends + * set_digital_out_mode (see DCP_INHERIT_BOOT_MODE in iomfb_template.c), so a + * modeset to any other mode would silently not be applied. The bootloader + * brings the panel up in its native timing, which is the mode the firmware + * scores highest and enumerate_modes() marks preferred; offer only that one. + */ +static bool dcp_mode_settable(struct apple_dcp *dcp, + const struct drm_display_mode *mode) +{ + struct dcp_display_mode *dcp_mode = lookup_mode(dcp, mode); + + if (!dcp_mode) + return false; + + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_mode->mode.type & DRM_MODE_TYPE_PREFERRED; + + return true; +} + enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, const struct drm_display_mode *mode) { @@ -493,7 +557,7 @@ enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, return MODE_ERROR; dcp = platform_get_drvdata(pdev); - return lookup_mode(dcp, mode) ? MODE_OK : MODE_BAD; + return dcp_mode_settable(dcp, mode) ? MODE_OK : MODE_BAD; } int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, @@ -525,6 +589,9 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, case DCP_FIRMWARE_V_13_5: ret = iomfb_modeset_v13_3(dcp, crtc_state); break; + case DCP_FIRMWARE_H17P: + ret = iomfb_modeset_h17p(dcp, crtc_state); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -543,7 +610,7 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, struct apple_dcp *dcp = platform_get_drvdata(pdev); /* TODO: support synthesized modes through scaling */ - return lookup_mode(dcp, mode) != NULL; + return dcp_mode_settable(dcp, mode); } @@ -589,6 +656,9 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) case DCP_FIRMWARE_V_13_5: iomfb_flush_v13_3(dcp, crtc, state); break; + case DCP_FIRMWARE_H17P: + iomfb_flush_h17p(dcp, crtc, state); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -604,6 +674,9 @@ static void iomfb_start(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_start_v13_3(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_start_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -621,9 +694,19 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) { enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); - if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) + if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) { + /* + * H17P reports its interface version (bits 63:48) and a + * firmware hash in the high half of the InitComplete word. + * The transport below it is unchanged. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + dev_info(dcp->dev, + "IOMFB: init complete %#llx (version %llu)\n", + message, + FIELD_GET(GENMASK_ULL(63, 48), message)); iomfb_start(dcp); - else if (type == IOMFB_MESSAGE_TYPE_MSG) + } else if (type == IOMFB_MESSAGE_TYPE_MSG) dcpep_got_msg(dcp, message); else dev_warn(dcp->dev, "Ignoring unknown message %llx\n", message); @@ -632,10 +715,27 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) int iomfb_start_rtkit(struct apple_dcp *dcp) { dma_addr_t shmem_iova; - apple_rtkit_start_ep(dcp->rtk, IOMFB_ENDPOINT); + int ret; + + /* + * H17P firmware expects the remote allocator endpoint to be started + * before IOMFB. A firmware subsystem whose endpoint was never started + * has nothing answering it on the AP side, and without this one the + * display DART faults. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); + if (ret) + return ret; + } + ret = apple_rtkit_start_ep(dcp->rtk, IOMFB_ENDPOINT); + if (ret) + return ret; dcp->shmem = dma_alloc_coherent(dcp->dev, DCP_SHMEM_SIZE, &shmem_iova, GFP_KERNEL); + if (!dcp->shmem) + return -ENOMEM; dcp_send_message(dcp, IOMFB_ENDPOINT, dcpep_set_shmem(shmem_iova)); @@ -655,6 +755,9 @@ void iomfb_shutdown(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_shutdown_v13_3(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_shutdown_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 7903fad404067..482f6e61d33c3 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -155,6 +155,7 @@ enum dcpep_method { dcpep_flush_supports_power, dcpep_set_power_state, dcpep_first_client_open, + dcpep_register_dfb_surface, dcpep_set_parameter_dcp, dcpep_enable_disable_video_power_savings, dcpep_is_main_display, @@ -165,14 +166,43 @@ enum dcpep_method { iomfbep_last_client_close, iomfbep_abort_swaps_dcp, iomfbep_set_matrix, + /* + * H17P default-framebuffer queries, issued from inside + * start_hardware_boot (D121) between create_default_fb and + * setup_video_limits. + */ + dcpep_get_dfb_compression_info, + dcpep_get_dfb_info, + dcpep_get_dfb_layout, + dcpep_get_dfb_state, + dcpep_set_dfb_dimensions, + dcpep_commit_dfb_info, + dcpep_dfb_query, + dcpep_dfb_ready_query, + /* + * H17P pipe-configuration calls made before the first swap. Most + * names are unknown; they are identified by method number. + */ + dcpep_pipe_cfg_415, + dcpep_pipe_cfg_031, + dcpep_pipe_cfg_414, + dcpep_pipe_query_478, + dcpep_pipe_query_474, + dcpep_pipe_set_476, + dcpep_pipe_set_428, dcpep_num_methods }; #define IOMFB_METHOD(tag, name) [name] = { #name, { tag[0], tag[1], tag[2], tag[3] } } +#define IOMFB_METHOD_H17(tag, h17g, name) [name] = { #name, \ + { tag[0], tag[1], tag[2], tag[3] }, \ + { h17g[0], h17g[1], h17g[2], h17g[3] } } struct dcp_method_entry { const char *name; char tag[4]; + /* H17G shares the H17P transport but changes selected method IDs. */ + char tag_h17g[4]; }; #define IOMFB_MAX_CB (1000) @@ -195,12 +225,6 @@ struct dcp_map_buf_req { u8 dva_null; } __packed; -struct dcp_map_buf_resp { - u64 vaddr; - u64 dva; - u32 ret; -} __packed; - struct dcp_unmap_buf_resp { u64 buffer; u64 vaddr; @@ -241,20 +265,6 @@ struct dcp_map_physical_resp { u32 mem_desc_id; } __packed; -struct dcp_swap_start_req { - u32 swap_id; - struct dcp_iouserclient client; - u8 swap_id_null; - u8 client_null; - u8 padding[2]; -} __packed; - -struct dcp_swap_start_resp { - u32 swap_id; - struct dcp_iouserclient client; - u32 ret; -} __packed; - struct dcp_get_uint_prop_req { char obj[4]; char key[0x40]; @@ -283,13 +293,6 @@ struct iomfb_set_fx_prop_req { u32 value; } __packed; -struct dcp_set_power_state_req { - u64 unklong; - u8 unkbool; - u8 unkint_null; - u8 padding[2]; -} __packed; - struct dcp_set_power_state_resp { u32 unkint; u32 ret; diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c new file mode 100644 index 0000000000000..28f219b8be14e --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include "dcp-link.h" +#include "iomfb_h17p.h" +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "version_utils.h" + +struct dcp_h17p_hotplug_request { + __le64 connected; + u8 tiled_display[0x4c]; + u8 tiled_display_null; + u8 padding[3]; +} __packed; + +struct dcp_h17p_swap_info_request { + u8 swap_info[0xe0]; + u8 swap_info_null; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_h17p_hotplug_request) == 0x58); +static_assert(sizeof(struct dcp_h17p_swap_info_request) == 0xe4); +/* D590 swap_complete_ap_gated is 0x730 bytes on the wire. */ +static_assert(sizeof(struct dc_swap_complete_resp_h17p) == 0x730); +static_assert(offsetof(struct dc_swap_complete_resp_h17p, swap_id) == 0); + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD_H17("A025", "A029", dcpep_setup_video_limits), /* (0,0) */ + IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), /* nested in D206 */ + IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), /* nested in D207 */ + IOMFB_METHOD("A385", dcpep_set_create_dfb), /* first call inside D121; (0,0) */ + IOMFB_METHOD("A386", iomfbep_a358_vi_set_temperature_hint), /* nested in D100 */ + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A406", dcpep_swap_start), + /* + * swap_start moved A407 -> A406 on H17P, but swap_submit did not move + * with it: every swap is A406 followed by A408, A407 is unused, and the + * A408 response is 0x0c bytes (dcp_swap_submit_resp_h17p). + */ + IOMFB_METHOD("A408", dcpep_swap_submit), + IOMFB_METHOD("A411", dcpep_set_display_device), + IOMFB_METHOD("A412", dcpep_is_main_display), + IOMFB_METHOD("A413", dcpep_set_digital_out_mode), + IOMFB_METHOD("A423", iomfbep_set_matrix), + IOMFB_METHOD("A427", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A442", dcpep_set_parameter_dcp), /* (0x28, 0x4) */ + IOMFB_METHOD("A446", dcpep_create_default_fb), /* follows A385 in D121 */ + /* + * The default-framebuffer block the firmware expects inside D121, + * straight after A446 and before A025. Sizes are (in, out). + */ + IOMFB_METHOD_H17("A035", "A039", dcpep_get_dfb_compression_info), /* (0x8, 0x8) */ + IOMFB_METHOD_H17("A034", "A038", dcpep_get_dfb_info), /* (0x2c, 0x2c) */ + IOMFB_METHOD("A103", dcpep_get_dfb_layout), /* (0xc, 0x8) */ + IOMFB_METHOD("A445", dcpep_get_dfb_state), /* (0x4, 0x8) */ + IOMFB_METHOD("A104", dcpep_set_dfb_dimensions), /* (0x8, 0) w,h */ + IOMFB_METHOD("A105", dcpep_commit_dfb_info), /* (0, 0) */ + IOMFB_METHOD_H17("A033", "A037", dcpep_dfb_query), /* (0, 0x4) */ + IOMFB_METHOD("A380", dcpep_dfb_ready_query), /* (0, 0x4) */ + IOMFB_METHOD("A415", dcpep_pipe_cfg_415), /* (0xc, 0xc) zeros */ + /* update_notify_clients_dcp */ + IOMFB_METHOD_H17("A031", "A035", dcpep_pipe_cfg_031), /* (0x6c, 0) */ + IOMFB_METHOD("A414", dcpep_pipe_cfg_414), /* (0x8, 0x8) zeros */ + IOMFB_METHOD("A478", dcpep_pipe_query_478), /* (0, 0x4) */ + IOMFB_METHOD("A474", dcpep_pipe_query_474), /* (0, 0x4) */ + IOMFB_METHOD("A476", dcpep_pipe_set_476), /* (0x4, 0) in 1 */ + IOMFB_METHOD("A428", dcpep_pipe_set_428), /* (0x4, 0x4) 0x10000 */ + IOMFB_METHOD("A450", dcpep_enable_disable_video_power_savings), /* in = u32 0 */ + IOMFB_METHOD("A457", dcpep_first_client_open), /* (0, 0) */ + /* D561 (displayMinRefreshInterval) arrives while A465 is in flight */ + IOMFB_METHOD("A465", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A468", dcpep_flush_supports_power), /* follows A025 in D121, in = 1 */ + /* + * iomfbep_last_client_close and iomfbep_abort_swaps_dcp are deliberately + * absent: their H17P numbers are unverified, so the teardown paths skip + * them (DCP_HAS_CLIENT_TEARDOWN in iomfb_template.c). + */ + IOMFB_METHOD("A473", dcpep_set_power_state), /* out-of-band; out 0x8 */ + IOMFB_METHOD("A472", dcpep_register_dfb_surface), /* default FB surface, nested in D582 */ +}; + +#define DCP_FW h17p +#define DCP_FW_VER DCP_FW_VERSION(26, 6, 0) + +#include "iomfb_template.c" + +static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + /* + * The bandwidth scratch and doorbell come from the DCP's DT node; the + * apple,bw-doorbell binding has no offset cell. + */ + u64 scratch = dcp->disp_bw_scratch_res.start + + dcp->disp_bw_scratch_offset; + u64 clock_request = dcp->disp_bw_doorbell_res.start; + + trace_iomfb_callback(dcp, tag, __func__); + if (apple_dcp_h17p_rt_bw_encode_reply(in, out, scratch, clock_request)) + dev_warn(dcp->dev, "D003 bandwidth request is malformed\n"); + return true; +} + +static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct apple_dcp_h17p_clock_request *request = in; + u64 rate; + int ret; + + trace_iomfb_callback(dcp, tag, __func__); + ret = apple_dcp_h17p_clock_rate(request, clk_get_rate(dcp->clk), + clk_get_rate(dcp->clk_194), &rate); + if (ret) { + dev_warn(dcp->dev, "unknown display clock ID %#x\n", + le32_to_cpu(request->clock_id)); + rate = 0; + } + + *(__le64 *)out = cpu_to_le64(rate); + return true; +} + +static bool trampoline_hotplug_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_hotplug_request *request = in; + + if (!(request->tiled_display_null & 1)) + memcpy(out, request->tiled_display, + sizeof(request->tiled_display)); + return trampoline_hotplug(dcp, tag, out, in); +} + +static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_swap_info_request *request = in; + + trace_iomfb_callback(dcp, tag, __func__); + if (!(request->swap_info_null & 1)) + memcpy(out, request->swap_info, sizeof(request->swap_info)); + return true; +} + +/* + * D400 (get_property) is left unhandled: the zeroed reply means "no such + * property". The firmware then runs without a panel power model, as it does + * for its other optional power properties. + */ + +/* + * H17P callback numbering. It is not a uniform shift of the v13.5 table: the + * service-creation block is D108..D113 exactly as on v13.5, and the swap + * completion is D590. A callback with no entry here is answered with a + * zeroed reply by dcpep_handle_cb(). + */ +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = dcpep_cb_d000_h17p, /* acked after a nested A033 */ + [1] = trampoline_true, + [2] = trampoline_nop, + [3] = trampoline_rt_bandwidth_h17p, + [6] = trampoline_set_frame_sync_props_h17p, /* 0x54/0x50, identity scale */ + [100] = iomfbep_cb_match_pmu_service, /* match_pmu_service */ + [101] = trampoline_zero, + [102] = trampoline_nop, /* set_number_property */ + [104] = trampoline_nop, /* set_boolean_property */ + [107] = trampoline_nop, + [108] = trampoline_true, /* create_provider_service */ + [109] = trampoline_true, /* create_product_service */ + [110] = trampoline_true, /* create_PMU_service */ + [111] = trampoline_true, /* create_iomfb_service */ + [112] = trampoline_create_backlight_service, /* create_backlight_service */ + [113] = trampoline_true, /* create_nvram_service */ + [114] = trampoline_zero, /* CoreAnalyticsSendEvent */ + [117] = trampoline_nop, /* set_idle_caching_state_ap */ + [118] = trampoline_zero, /* upload_trace_start */ + [119] = trampoline_zero, /* upload_trace_chunk */ + [120] = trampoline_zero, /* upload_trace_end */ + [121] = dcpep_cb_boot_1, /* start_hardware_boot */ + [122] = trampoline_false, /* is_dark_boot */ + [123] = trampoline_false, /* is_waking_from_hibernate */ + [124] = trampoline_zero, /* detect_fastsim */ + [125] = trampoline_read_edt_data, /* read_edt_data */ + [127] = trampoline_prop_start, /* setDCPAVPropStart */ + [128] = trampoline_prop_chunk, /* setDCPAVPropChunk */ + [129] = trampoline_prop_end, /* setDCPAVPropEnd */ + [130] = trampoline_allocate_bandwidth, /* allocate_bandwidth */ + [201] = trampoline_map_piodma, /* map_buf */ + [202] = trampoline_unmap_piodma, /* unmap_buf */ + [206] = iomfbep_cb_match_pmu_service_2, /* match_pmu_service */ + [207] = iomfbep_cb_match_backlight_service, /* match_backlight_service */ + [208] = trampoline_nop, /* update_backlight_factor_prop */ + [209] = trampoline_get_time, /* get_calendar_time_ms */ + [300] = trampoline_pr_publish, + [401] = trampoline_get_uint_prop, /* get_uint_prop */ + [404] = trampoline_nop, /* set_uint_prop */ + [406] = trampoline_set_fx_prop, /* set_fx_prop */ + [408] = trampoline_get_frequency_h17p, /* getClockFrequency */ + [411] = trampoline_map_reg, /* mapDeviceMemoryWithIndex */ + [413] = trampoline_true, /* setProperty */ + [414] = trampoline_sr_set_property_int, /* setProperty */ + [415] = trampoline_true, /* setProperty */ + [451] = trampoline_allocate_buffer, /* allocate_buffer */ + [452] = trampoline_map_physical, /* prepare */ + [454] = trampoline_release_mem_desc, /* release_descriptor */ + [552] = trampoline_true, + [561] = trampoline_true, + [563] = trampoline_true, + [565] = trampoline_true, + [567] = trampoline_true, + [572] = trampoline_zero, /* powerUpDART */ + [575] = trampoline_hotplug_h17p, + [576] = trampoline_nop, + [582] = iomfbep_cb_create_dfb_surface, /* create_default_fb_surface */ + [583] = trampoline_nop, /* clear_default_surface */ + [584] = trampoline_nop, /* swap_notify_gated */ + [585] = trampoline_swap_info_h17p, /* swap_info_notify_dispatch */ + [590] = trampoline_swap_complete, /* swap_complete_ap_gated */ + [592] = trampoline_swap_complete_intent_gated, /* swap_complete_intent_gated */ + [593] = trampoline_abort_swap_ap_gated, /* abort_swap_ap_gated */ + [594] = trampoline_enable_backlight_message_ap_gated, + [595] = trampoline_nop, /* setSystemConsoleMode */ + [597] = trampoline_false, /* isDFBAllocated */ + [598] = trampoline_false, + [599] = trampoline_nop, /* find_swap_function_gated */ +}; + +void iomfb_start_h17p(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h new file mode 100644 index 0000000000000..97c0da9e05b94 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_IOMFB_H17P_H__ +#define __APPLE_IOMFB_H17P_H__ + +#include "version_utils.h" + +#define DCP_FW h17p +#define DCP_FW_VER DCP_FW_VERSION(26, 6, 0) + +#include "iomfb_template.h" + +static_assert(sizeof(struct dcp_swap_h17p) == 0x58a); +static_assert(sizeof(struct dcp_surface_h17p) == 0x22c); +static_assert(sizeof(struct dcp_swap_submit_req_h17p) == 0xe9c); +static_assert(sizeof(struct dcp_swap_submit_resp_h17p) == 0x0c); + +#undef DCP_FW_VER +#undef DCP_FW + +#endif /* __APPLE_IOMFB_H17P_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 4235a31b17ba0..bc9fb5041e26e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -64,13 +64,50 @@ DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct DCP_FW_NAME(dcp_swap_submit_req), struct DCP_FW_NAME(dcp_swap_submit_resp)); -DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct dcp_swap_start_req, - struct dcp_swap_start_resp); +DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct DCP_FW_NAME(dcp_swap_start_req), + struct DCP_FW_NAME(dcp_swap_start_resp)); DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, - struct dcp_set_power_state_req, + struct DCP_FW_NAME(dcp_set_power_state_req), struct dcp_set_power_state_resp); +/* + * H17P firmware expects set_power_state (A473) on the out-of-band command + * channel. Sent on the ordinary command channel it is never answered, which + * strands the power transition and every later call behind it. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_POWER_OOB true +#else +#define DCP_POWER_OOB false +#endif + +/* + * H17P firmware is attached to a display the bootloader has already brought + * up, and it does not accept set_digital_out_mode (A413) for that display: + * the call fails (0x8000000b) and the timing generator never starts, so there + * is no vblank and no swap completion. Keep the mode the bootloader + * programmed, which is the panel's native mode, instead of requesting one. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_INHERIT_BOOT_MODE true +#else +#define DCP_INHERIT_BOOT_MODE false +#endif + +/* + * abort_swaps_dcp and last_client_close have no verified H17P method numbers, + * and the H17P table leaves them out. Rather than send the firmware a record + * it would decode as some other call, power-off and sleep skip both and go + * straight to set_power_state, whose number and sizes are known. Power-off + * still sends its clear swap first. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_HAS_CLIENT_TEARDOWN false +#else +#define DCP_HAS_CLIENT_TEARDOWN true +#endif + DCP_THUNK_INOUT(dcp_set_digital_out_mode, dcpep_set_digital_out_mode, struct dcp_set_digital_out_mode_req, u32); @@ -89,6 +126,68 @@ DCP_THUNK_OUT(dcp_create_default_fb, dcpep_create_default_fb, u32); DCP_THUNK_OUT(dcp_start_signal, dcpep_start_signal, u32); DCP_THUNK_VOID(dcp_setup_video_limits, dcpep_setup_video_limits); DCP_THUNK_VOID(dcp_set_create_dfb, dcpep_set_create_dfb); + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * Default-framebuffer queries issued from inside start_hardware_boot (D121). + * The getters take a zeroed input of the expected size; the firmware only + * checks the size. + */ +struct dcp_dfb_info_h17p { + u8 data[0x2c]; +} __packed; + +struct dcp_dfb_layout_req_h17p { + u8 data[0xc]; +} __packed; + +struct dcp_dfb_dimensions_h17p { + u32 width; + u32 height; +} __packed; + +DCP_THUNK_INOUT(dcp_get_dfb_compression_info, dcpep_get_dfb_compression_info, + u64, u64); +DCP_THUNK_INOUT(dcp_get_dfb_info, dcpep_get_dfb_info, + struct dcp_dfb_info_h17p, struct dcp_dfb_info_h17p); +DCP_THUNK_INOUT(dcp_get_dfb_layout, dcpep_get_dfb_layout, + struct dcp_dfb_layout_req_h17p, u64); +DCP_THUNK_INOUT(dcp_get_dfb_state, dcpep_get_dfb_state, u32, u64); +DCP_THUNK_IN(dcp_set_dfb_dimensions, dcpep_set_dfb_dimensions, + struct dcp_dfb_dimensions_h17p); +DCP_THUNK_VOID(dcp_commit_dfb_info, dcpep_commit_dfb_info); +DCP_THUNK_OUT(dcp_dfb_query, dcpep_dfb_query, u32); +DCP_THUNK_OUT(dcp_dfb_ready_query, dcpep_dfb_ready_query, u32); + +/* + * Pipe-configuration calls the firmware expects between the boot chain and + * the first swap. Most of them have no known name; they are identified by + * method number and wire size. + */ +struct dcp_pipe_cfg_415 { + u8 data[0xc]; +} __packed; + +/* + * A031 is update_notify_clients_dcp(u32[27]): an array of the DCP + * notification clients the AP wants delivered. Clients that are not + * registered here never send the callbacks they gate. + */ +struct dcp_pipe_cfg_031 { + __le32 clients[27]; +} __packed; +static_assert(sizeof(struct dcp_pipe_cfg_031) == 0x6c); + +DCP_THUNK_INOUT(dcp_pipe_cfg_415, dcpep_pipe_cfg_415, + struct dcp_pipe_cfg_415, struct dcp_pipe_cfg_415); +DCP_THUNK_IN(dcp_pipe_cfg_031, dcpep_pipe_cfg_031, struct dcp_pipe_cfg_031); +DCP_THUNK_INOUT(dcp_pipe_cfg_414, dcpep_pipe_cfg_414, u64, u64); +DCP_THUNK_OUT(dcp_pipe_query_478, dcpep_pipe_query_478, u32); +DCP_THUNK_OUT(dcp_pipe_query_474, dcpep_pipe_query_474, u32); +DCP_THUNK_IN(dcp_pipe_set_476, dcpep_pipe_set_476, u32); +DCP_THUNK_INOUT(dcp_pipe_set_428, dcpep_pipe_set_428, u32, u32); +#endif + DCP_THUNK_VOID(dcp_first_client_open, dcpep_first_client_open); DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, @@ -120,6 +219,96 @@ static u32 dcpep_cb_zero(struct apple_dcp *dcp) return 0; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * Bind every framebuffer displaced by the commit that produced this swap to + * that swap's id. + * + * The id only exists once the firmware answers swap_start, after the atomic + * commit has queued its entries, so they are queued unarmed and stamped here. + * Swap-start requests are serialized by the DCP command stack. + */ +static void dcp_arm_swapped_out_fbs(struct apple_dcp *dcp, u32 swap_id) +{ + struct dcp_fb_reference *entry; + unsigned long flags; + + spin_lock_irqsave(&dcp->swapped_out_lock, flags); + list_for_each_entry(entry, &dcp->swapped_out_fbs, head) { + if (entry->armed) + continue; + entry->swap_id = swap_id; + entry->armed = true; + } + spin_unlock_irqrestore(&dcp->swapped_out_lock, flags); +} + +/* + * Drop framebuffers whose unbinding swap has completed. + * + * Entries must be keyed to the swap that unbinds them, rather than the last + * swap that happened to have completed when the atomic commit ran. A later + * successful completion also retires entries whose own completion was lost; + * the signed difference keeps that comparison valid across a 32-bit wrap. + */ +static void dcp_release_swapped_out_fbs(struct apple_dcp *dcp) +{ + struct dcp_fb_reference *entry, *tmp; + unsigned long flags; + LIST_HEAD(done); + + /* + * last_swap_id / have_swap_complete are only ever touched by the RTKit + * worker, which is also the only caller of this function, so the lock + * is here for the list alone. + */ + spin_lock_irqsave(&dcp->swapped_out_lock, flags); + while (!list_empty(&dcp->swapped_out_fbs)) { + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + + /* Newest entries; their swap has not been started yet. */ + if (!entry->armed) + break; + /* last_swap_id is 0 from kzalloc until a swap really lands. */ + if (!dcp->have_swap_complete) + break; + /* Hold the old framebuffer for one additional completed swap. */ + if ((s32)(dcp->last_swap_id - entry->swap_id) <= 0) + break; + + list_move_tail(&entry->head, &done); + } + spin_unlock_irqrestore(&dcp->swapped_out_lock, flags); + + /* drm_framebuffer_put() can sleep, so never call it under the lock. */ + list_for_each_entry_safe(entry, tmp, &done, head) { + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} +#else +static void dcp_arm_swapped_out_fbs(struct apple_dcp *dcp, u32 swap_id) { } + +static void dcp_release_swapped_out_fbs(struct apple_dcp *dcp) +{ + while (!list_empty(&dcp->swapped_out_fbs)) { + struct dcp_fb_reference *entry; + + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + if (entry->swap_id == dcp->last_swap_id) + break; + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} +#endif + static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct DCP_FW_NAME(dc_swap_complete_resp) *resp) { @@ -128,6 +317,10 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, dcp->last_swap_id = resp->swap_id; dcp_drm_crtc_page_flip(dcp, now); +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + dcp->have_swap_complete = true; + dcp_release_swapped_out_fbs(dcp); +#endif if (dcp->crc_enabled) { u32 crc32 = 0; drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); @@ -152,6 +345,94 @@ static bool iomfbep_cb_match_pmu_service(struct apple_dcp *dcp, int tag, void *o return false; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P asks the AP to make the default framebuffer surface (D582). The + * firmware expects the AP to register a surface from inside that callback, + * with a nested A472 carrying a 0x22c-byte surface descriptor, and only then + * to report success. Without it the firmware has no surface, programs the + * display engine from a null base and the piodma DART faults. + */ +struct dcp_h17p_dfb_surface_req { + u8 surface[0x22c]; + u8 surface_null; + u8 padding[3]; +} __packed; +static_assert(sizeof(struct dcp_h17p_dfb_surface_req) == 0x230); + +/* + * A472 has a 4-byte output; with out_len 0 the firmware has nowhere to write + * its result and stops answering. + */ +DCP_THUNK_INOUT(dcp_register_dfb_surface, dcpep_register_dfb_surface, + struct dcp_h17p_dfb_surface_req, u32); + +static void dcp_ack_dfb_surface(struct apple_dcp *dcp) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +/* + * Fill the default surface descriptor. Fields not set here are zero. The + * format at +0x0b is the fourcc 'b3a8', stored byte-reversed; +0x35 is the + * default surface ID. The bytes at +0x29..+0x2c are an opaque word whose + * meaning is not known. + */ +static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) +{ + s[0x002] = 0x01; + memcpy(s + 0x00b, "8a3b", 4); + s[0x019] = 0x01; + s[0x01b] = 0x01; + s[0x01c] = 0x01; + s[0x02b] = 0x4d; + s[0x02c] = 0x01; + s[0x035] = 0x02; + s[0x051] = 0x01; + s[0x149] = 0x01; +} + +/* + * A471 (update_dfb) is never sent: on H17P it makes the coprocessor take an + * AXI read error. Scanout does not need it, since every swap carries its own + * surface. Just acknowledge the callback. + */ +static void complete_dfb_surface(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_ack_dfb_surface(dcp); +} + +static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + struct dcp_h17p_dfb_surface_req *req; + + trace_iomfb_callback(dcp, tag, __func__); + + /* + * The request is too large for the stack this deep in the RTKit + * receive work (rx work -> iomfb_recv_msg -> dcpep_handle_cb -> here -> + * dcp_push). dcp_push() copies the payload into shared memory before + * returning, so a short-lived allocation is enough. + */ + req = kzalloc_obj(*req); + if (!req) + return true; + + dcp_fill_dfb_surface(dcp, req->surface); + + dcp_register_dfb_surface(dcp, false, req, complete_dfb_surface, NULL); + kfree(req); + + /* deferred ACK: the A472 completion reports success for us */ + return false; +} +#endif + static void complete_pmu_service_matched(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_channel *ch = &dcp->ch_cb; @@ -265,7 +546,7 @@ static void iomfbep_cb_set_fx_prop(struct apple_dcp *dcp, struct iomfb_set_fx_pr * PIODMA is separate from the main DCP and uses own IOVA space on a dedicated * stream of the display DART, rather than the expected DCP DART. */ -static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, +static struct DCP_FW_NAME(dcp_map_buf_resp) dcpep_cb_map_piodma(struct apple_dcp *dcp, struct dcp_map_buf_req *req) { struct dcp_mem_descriptor *memdesc; @@ -293,12 +574,30 @@ static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, goto reject; } - return (struct dcp_map_buf_resp){ .dva = memdesc->dva }; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* remembered so release_mem_desc() can tear a stale mapping down */ + memdesc->piodma_mapped = true; + + /* the H17P reply echoes the buffer index */ + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ + .buffer = req->buffer, + .dva = memdesc->dva, + }; +#else + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .dva = memdesc->dva }; +#endif reject: dev_err(dcp->dev, "denying map of invalid buffer %llx for piodma\n", req->buffer); - return (struct dcp_map_buf_resp){ .ret = EINVAL }; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ + .buffer = req->buffer, + .dva = 0, + }; +#else + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .ret = EINVAL }; +#endif } static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, @@ -333,6 +632,7 @@ static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, * full pages */ iommu_unmap(dcp->iommu_dom, memdesc->dva, ALIGN(memdesc->size, SZ_16K)); + memdesc->piodma_mapped = false; } /* @@ -364,6 +664,7 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, memdesc = &dcp->memdesc[id]; + memdesc->piodma_mapped = false; memdesc->size = resp.dva_size; /* HACK: align size to 16K since the iommu API only maps full pages */ size = ALIGN(resp.dva_size, SZ_16K); @@ -374,6 +675,16 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, size); resp.dva = memdesc->dva; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P firmware also needs the physical address: it loads the APT M3 + * microcode through it, and with a zero paddr the timing sequencer + * comes up and is powered straight back down. + */ + if (memdesc->map.sgl) + resp.paddr = sg_phys(memdesc->map.sgl); +#endif + return resp; } @@ -397,6 +708,23 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) memdesc = &dcp->memdesc[id]; size = ALIGN(memdesc->size, SZ_16K); + + /* + * Tear down any translation still standing before the pages go back to + * the allocator. The firmware is expected to unmap piodma first, but + * if it releases a descriptor without doing so we would hand the pages + * back while the DART still resolves them -- the display engine would + * then read whoever owns them next. Only the H17P map_piodma handler + * sets piodma_mapped, so older firmware is unaffected. + */ + if (memdesc->piodma_mapped) { + dev_warn(dcp->dev, + "releasing buffer %u while still mapped for piodma; unmapping first\n", + id); + iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + memdesc->piodma_mapped = false; + } + if (memdesc->buf) { dma_free_coherent(dcp->dev, size, memdesc->buf, memdesc->dva); memdesc->buf = NULL; @@ -474,7 +802,15 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d return (struct DCP_FW_NAME(dcp_map_reg_resp)){ .ret = 1 }; } else { struct resource *rsrc = dcp->disp_registers[req->index]; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P reaches these apertures physically, not through its + * DART, and expects dva == addr in the reply. Trying to + * dma_map_resource() them instead fails and hands the firmware + * DMA_MAPPING_ERROR as a device address. + */ + dma_addr_t dva = rsrc->start; +#elif DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) dma_addr_t dva = dma_map_resource(dcp->dev, rsrc->start, resource_size(rsrc), DMA_BIDIRECTIONAL, 0); WARN_ON(dva == DMA_MAPPING_ERROR); @@ -508,7 +844,9 @@ static void iomfbep_cb_enable_backlight_message_ap_gated(struct apple_dcp *dcp, * syslog: "[BrightnessLCD.cpp:743][AFK]nitsToDBV: iDAC out of range" */ dcp->brightness.update = true; - schedule_work(&dcp->bl_update_wq); + /* Backlight handling is only set up for panels described in the DT. */ + if (dcp_has_panel(dcp)) + schedule_work(&dcp->bl_update_wq); } /* Chunked data transfer for property dictionaries */ @@ -519,6 +857,19 @@ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) return false; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * This length comes straight off the wire. A bogus value would turn + * the allocation below into a multi-gigabyte GFP_KERNEL request. + * Property blobs are a few KiB, so bound it. + */ + if (*length == 0 || *length > SZ_1M) { + dev_warn(dcp->dev, "rejecting %u byte property transfer\n", + *length); + return false; + } +#endif + dcp->chunks.length = *length; dcp->chunks.data = kzalloc(*length, GFP_KERNEL); @@ -622,12 +973,22 @@ static void boot_done(struct apple_dcp *dcp, void *out, void *cookie) dcp_ack(dcp, DCP_CONTEXT_CB); } -static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) +static void boot_5b(struct apple_dcp *dcp, void *out, void *cookie) { dcp_set_display_refresh_properties(dcp, false, boot_done, NULL); } -static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) +static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* H17P expects A380 right before set_display_refresh_properties */ + dcp_dfb_ready_query(dcp, false, boot_5b, NULL); +#else + boot_5b(dcp, out, cookie); +#endif +} + +static void boot_4b(struct apple_dcp *dcp, void *out, void *cookie) { #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u32 v_true = 1; @@ -637,6 +998,16 @@ static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) #endif } +static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* H17P expects A033 between flush_supports_power and late_init_signal */ + dcp_dfb_query(dcp, false, boot_4b, NULL); +#else + boot_4b(dcp, out, cookie); +#endif +} + static void boot_3(struct apple_dcp *dcp, void *out, void *cookie) { u32 v_true = true; @@ -649,11 +1020,103 @@ static void boot_2(struct apple_dcp *dcp, void *out, void *cookie) dcp_setup_video_limits(dcp, false, boot_3, NULL); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P expects the default framebuffer to be described between + * create_default_fb and setup_video_limits: + * get_dfb_compression_info -> get_dfb_info -> get_dfb_layout -> + * get_dfb_state -> set_dfb_dimensions(w, h) -> commit_dfb_info + * Without it the firmware never finishes its own default framebuffer setup. + */ +static void boot_1_11(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_commit_dfb_info(dcp, false, boot_2, NULL); +} + +static void boot_1_10(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_dfb_dimensions_h17p dim; + + /* + * get_dfb_state returns the default framebuffer dimensions, which are + * passed back unchanged to set_dfb_dimensions. + */ + memcpy(&dim, out, sizeof(dim)); + if (!dim.width || !dim.height || dim.width > 16384 || dim.height > 16384) { + struct dcp_channel *ch = &dcp->ch_cb; + + dev_err(dcp->dev, "invalid default framebuffer dimensions %ux%u\n", + dim.width, dim.height); + *(u8 *)ch->output[ch->depth - 1] = false; + dcp_ack(dcp, DCP_CONTEXT_CB); + return; + } + dev_dbg(dcp->dev, "default framebuffer dimensions %ux%u\n", + dim.width, dim.height); + + dcp_set_dfb_dimensions(dcp, false, &dim, boot_1_11, NULL); +} + +static void boot_1_9(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 zero = 0; + + dcp_get_dfb_state(dcp, false, &zero, boot_1_10, NULL); +} + +static void boot_1_8(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_dfb_layout_req_h17p req = {}; + + dcp_get_dfb_layout(dcp, false, &req, boot_1_9, NULL); +} + +static void boot_1_7(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_dfb_info_h17p info = {}; + + dcp_get_dfb_info(dcp, false, &info, boot_1_8, NULL); +} + +static void boot_1_6(struct apple_dcp *dcp, void *out, void *cookie) +{ + u64 zero = 0; + + dcp_get_dfb_compression_info(dcp, false, &zero, boot_1_7, NULL); +} +#endif + static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + dcp_create_default_fb(dcp, false, boot_1_6, NULL); +#else dcp_create_default_fb(dcp, false, boot_2, NULL); +#endif } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +static void d000_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +/* + * The firmware expects D000 to be answered only after a nested A033 + * (dfb_query), and then with 1. Defer the ACK until that call completes. + */ +static bool dcpep_cb_d000_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + dcp_dfb_query(dcp, false, d000_done, NULL); + return false; +} +#endif + /* Use special function signature to defer the ACK */ static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -701,6 +1164,46 @@ static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) return rt_bw; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P's D006 is 0x54 in / 0x50 out, far wider than the 28-byte + * frame_sync_props of older firmware, so the fields that matter sit past + * anything the old struct could describe. + * + * The firmware expects zeros except for three u32 at 0x24, 0x28 and 0x2c, + * each 0x10000 -- 1.0 in 16.16 fixed point, the identity scale. An all-zero + * reply hands the pipe a zero scale, and it then programs a null surface + * base: the display DART faults at DVA 0 and the coprocessor takes an AXI + * error. + */ +#define DCP_FRAME_SYNC_IDENTITY 0x10000 + +struct dcp_set_frame_sync_props_req_h17p { + u8 unk[0x54]; +} __packed; + +struct dcp_set_frame_sync_props_resp_h17p { + u8 unk[0x50]; +} __packed; + +static struct dcp_set_frame_sync_props_resp_h17p +dcpep_cb_set_frame_sync_props_h17p(struct apple_dcp *dcp, + struct dcp_set_frame_sync_props_req_h17p *req) +{ + struct dcp_set_frame_sync_props_resp_h17p resp = {}; + static const u16 identity_at[] = { 0x24, 0x28, 0x2c }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(identity_at); i++) { + u32 v = DCP_FRAME_SYNC_IDENTITY; + + memcpy(&resp.unk[identity_at[i]], &v, sizeof(v)); + } + + return resp; +} +#endif + static struct dcp_set_frame_sync_props_resp dcpep_cb_set_frame_sync_props(struct apple_dcp *dcp, struct dcp_set_frame_sync_props_req *req) @@ -728,6 +1231,71 @@ static void release_swap_cookie(struct kref *ref) kfree(cookie); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_SWAP_FLAGS1 0x34865202ULL +#define DCP_SWAP_FLAGS2 0x4ULL + +/* + * H17P's swap surface has no is_tearing_allowed and no is_premultiplied: + * plane_cnt sits at +1, plane_cnt2 at +5 and format at +9, two bytes earlier + * than on every other firmware. With the older layout the firmware reads the + * format two bytes late and its surface check rejects the swap as an + * unsupported format. + * + * Shift each surface's tail down by two rather than fork the shared struct. + */ +static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) +{ + struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); + unsigned int i; + + /* + * The firmware expects flags1 = 0x34865202 and flags2 = 4 (at +0x40 + * and +0x48) in every swap. + */ + req->swap.flags1 = DCP_SWAP_FLAGS1; + req->swap.flags2 = DCP_SWAP_FLAGS2; + + /* + * The H17P gap before swap_id carries two small counters the firmware + * expects to be 2 (at +0x50) and 3 (at +0x78). The gap starts at + * +0x50, so these are indices 0 and 0x28 of the blob. + */ + req->swap.h17p_pre_swap_id[0x00] = 2; + req->swap.h17p_pre_swap_id[0x28] = 3; + + for (i = 0; i < SWAP_SURFACES; i++) { + u8 *p = (u8 *)&req->surf[i]; + size_t n = sizeof(req->surf[i]); + + memmove(p + 1, p + 3, n - 3); + memset(p + n - 2, 0, 2); + } + + /* + * Everything from surf_iova onward also sits two bytes earlier on + * H17P: surf_iova[0] is at +0xe38 of the 0xe9c-byte record, not + * +0xe3a. Left in place, the firmware reads every IOVA two bytes off + * and the trailing null-flag block misaligned, so it faults on a bogus + * address or verifies layers that are not present. Shift the whole + * block down. + */ + { + u8 *r = (u8 *)req; + size_t n = sizeof(*req); + size_t off = offsetof(typeof(*req), surf_iova); + + static_assert(offsetof(typeof(*req), surf_iova) == 0xe3a, + "H17P source IOVA offset"); + static_assert(sizeof(*req) == 0xe9c, "H17P request size"); + memmove(r + off - 2, r + off, n - off); + memset(r + n - 2, 0, 2); + } +} +#else +static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) { } +#endif + static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; @@ -744,30 +1312,22 @@ static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) return; } - while (!list_empty(&dcp->swapped_out_fbs)) { - struct dcp_fb_reference *entry; - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - if (entry->swap_id == dcp->last_swap_id) - break; - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } + dcp_release_swapped_out_fbs(dcp); } static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, void *cookie) { - struct dcp_swap_start_resp *resp = data; + struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + dcp_arm_swapped_out_fbs(dcp, resp->swap_id); if (cookie) { struct dcp_swap_cookie *info = cookie; info->swap_id = resp->swap_id; } + dcp_h17p_fix_swap_surfaces(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); } @@ -783,11 +1343,11 @@ static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) static void dcp_on_set_power_state(struct apple_dcp *dcp, void *out, void *cookie) { - struct dcp_set_power_state_req req = { + struct DCP_FW_NAME(dcp_set_power_state_req) req = { .unklong = 1, }; - dcp_set_power_state(dcp, false, &req, dcp_on_final, cookie); + dcp_set_power_state(dcp, DCP_POWER_OOB, &req, dcp_on_final, cookie); } static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) @@ -822,9 +1382,19 @@ void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) kref_get(&cookie->refcount); if (dcp->main_display) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P takes set_display_device (A411) once, in the + * client-open sequence. Repeating it after the client is open + * is a call the firmware stops answering, which wedges the + * command channel and with it every later swap. + */ + dcp_on_set_power_state(dcp, NULL, cookie); +#else handle = 0; dcp_set_display_device(dcp, false, &handle, dcp_on_set_power_state, cookie); +#endif } else { handle = 2; dcp_set_display_device(dcp, false, &handle, @@ -865,10 +1435,10 @@ static void complete_set_powerstate(struct apple_dcp *dcp, void *out, static void last_client_closed_poff(struct apple_dcp *dcp, void *out, void *cookie) { - struct dcp_set_power_state_req power_req = { + struct DCP_FW_NAME(dcp_set_power_state_req) power_req = { .unklong = 0, }; - dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, + dcp_set_power_state(dcp, DCP_POWER_OOB, &power_req, complete_set_powerstate, cookie); } @@ -889,7 +1459,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) }; struct dcp_swap_cookie *cookie; struct dcp_wait_cookie *poff_cookie; - struct dcp_swap_start_req swap_req = { 0 }; + struct DCP_FW_NAME(dcp_swap_start_req) swap_req = { 0 }; struct DCP_FW_NAME(dcp_swap_submit_req) *swap = &DCP_FW_UNION(dcp->swap); cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); @@ -924,8 +1494,10 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) for (int l = 0; l < SWAP_SURFACES; l++) swap->surf_null[l] = true; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) for (int l = 0; l < 5; l++) swap->surf2_null[l] = true; +#endif swap->unkU32Ptr_null = true; swap->unkU32out_null = true; #endif @@ -950,8 +1522,14 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) /* increase refcount to ensure the receiver has a reference */ kref_get(&poff_cookie->refcount); - iomfb_abort_swaps_dcp(dcp, false, &abort_req, - aborted_swaps_dcp_poff, poff_cookie); + if (DCP_HAS_CLIENT_TEARDOWN) { + iomfb_abort_swaps_dcp(dcp, false, &abort_req, + aborted_swaps_dcp_poff, poff_cookie); + } else { + dev_dbg_once(dcp->dev, + "power-off: skipping abort_swaps/last_client_close\n"); + last_client_closed_poff(dcp, NULL, poff_cookie); + } ret = wait_for_completion_timeout(&poff_cookie->done, msecs_to_jiffies(1000)); @@ -969,10 +1547,10 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) static void last_client_closed_sleep(struct apple_dcp *dcp, void *out, void *cookie) { - struct dcp_set_power_state_req power_req = { + struct DCP_FW_NAME(dcp_set_power_state_req) power_req = { .unklong = 0, }; - dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, cookie); + dcp_set_power_state(dcp, DCP_POWER_OOB, &power_req, complete_set_powerstate, cookie); } static void aborted_swaps_dcp_sleep(struct apple_dcp *dcp, void *out, void *cookie) @@ -1000,8 +1578,14 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) /* increase refcount to ensure the receiver has a reference */ kref_get(&cookie->refcount); - iomfb_abort_swaps_dcp(dcp, false, &req, aborted_swaps_dcp_sleep, - cookie); + if (DCP_HAS_CLIENT_TEARDOWN) { + iomfb_abort_swaps_dcp(dcp, false, &req, aborted_swaps_dcp_sleep, + cookie); + } else { + dev_dbg_once(dcp->dev, + "sleep: skipping abort_swaps/last_client_close\n"); + last_client_closed_sleep(dcp, NULL, cookie); + } ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); @@ -1097,7 +1681,19 @@ dcpep_cb_get_tiling_state(struct apple_dcp *dcp, static u8 dcpep_cb_create_backlight_service(struct apple_dcp *dcp) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P asks during start_ios_gated, before any mode is known, but + * dcp_has_panel() only becomes true once the panel dimensions have been + * parsed. An internal panel always has a backlight, so answer for the + * connector; answering false makes the firmware report that it could + * not find the backlight service. + */ + return dcp_has_panel(dcp) || + dcp->connector_type == DRM_MODE_CONNECTOR_eDP; +#else return dcp_has_panel(dcp); +#endif } TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); @@ -1111,7 +1707,7 @@ TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, TRAMPOLINE_IN(trampoline_set_fx_prop, iomfbep_cb_set_fx_prop, struct iomfb_set_fx_prop_req) TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, - struct dcp_map_buf_req, struct dcp_map_buf_resp); + struct dcp_map_buf_req, struct DCP_FW_NAME(dcp_map_buf_resp)); TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, struct dcp_unmap_buf_resp); TRAMPOLINE_INOUT(trampoline_sr_set_property_int, iomfbep_cb_sr_set_property_int, @@ -1140,6 +1736,12 @@ TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, TRAMPOLINE_INOUT(trampoline_set_frame_sync_props, dcpep_cb_set_frame_sync_props, struct dcp_set_frame_sync_props_req, struct dcp_set_frame_sync_props_resp); +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +TRAMPOLINE_INOUT(trampoline_set_frame_sync_props_h17p, + dcpep_cb_set_frame_sync_props_h17p, + struct dcp_set_frame_sync_props_req_h17p, + struct dcp_set_frame_sync_props_resp_h17p); +#endif TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); @@ -1172,33 +1774,25 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) dcp->swap_start = ktime_get(); dcp->swap_submit_timestamp = arch_timer_read_counter(); - while (!list_empty(&dcp->swapped_out_fbs)) { - struct dcp_fb_reference *entry; - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - if (entry->swap_id == dcp->last_swap_id) - break; - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } + dcp_release_swapped_out_fbs(dcp); } static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) { - struct dcp_swap_start_resp *resp = data; + struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + dcp_arm_swapped_out_fbs(dcp, resp->swap_id); trace_iomfb_swap_submit(dcp, resp->swap_id); + dcp_h17p_fix_swap_surfaces(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swapped, NULL); } /* Helpers to modeset and swap, used to flush */ static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) { - struct dcp_swap_start_req start_req = { 0 }; + struct DCP_FW_NAME(dcp_swap_start_req) start_req = { 0 }; if (dcp->connector && dcp->connector->connected) dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); @@ -1206,6 +1800,19 @@ static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) dcp_drm_crtc_vblank(dcp->crtc); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P expects A428 between set_matrix and swap_start, with 0x10000 (1.0 in + * 16.16 fixed point), the same identity value the D006 reply carries. + */ +static void dcp_pre_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 identity = 0x10000; + + dcp_pipe_set_428(dcp, false, &identity, do_swap, cookie); +} +#endif + static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, void *cookie) { @@ -1301,7 +1908,10 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, dcp->during_modeset = true; dcp->swap_submit_timestamp = 0; - if (mode->vrr) + if (DCP_INHERIT_BOOT_MODE) { + dev_dbg(dcp->dev, "inheriting the bootloader's display mode\n"); + complete_set_digital_out_mode(dcp, NULL, cookie); + } else if (mode->vrr) dcp_set_adaptive_sync(dcp, crtc_state->vrr_enabled ? mode->min_vrr : 0, cookie); @@ -1375,8 +1985,10 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru for (l = 0; l < SWAP_SURFACES; l++) req->surf_null[l] = true; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) for (l = 0; l < 5; l++) req->surf2_null[l] = true; +#endif req->unkU32Ptr_null = true; req->unkU32out_null = true; #endif @@ -1415,10 +2027,27 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru struct dcp_fb_reference *entry = kzalloc(sizeof(*entry), GFP_KERNEL); if (entry) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + unsigned long flags; + + entry->fb = old_state->fb; + /* + * Left unarmed: the id of the swap that + * unbinds this framebuffer is not known until + * the firmware answers swap_start. + */ + spin_lock_irqsave(&dcp->swapped_out_lock, + flags); + list_add_tail(&entry->head, + &dcp->swapped_out_fbs); + spin_unlock_irqrestore(&dcp->swapped_out_lock, + flags); +#else entry->fb = old_state->fb; entry->swap_id = dcp->last_swap_id; list_add_tail(&entry->head, &dcp->swapped_out_fbs); +#endif } drm_framebuffer_get(old_state->fb); } @@ -1506,7 +2135,11 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru mat.matrix[0] = mat.matrix[4] = mat.matrix[8] = 1LLU << 32; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + iomfb_set_matrix(dcp, false, &mat, dcp_pre_swap_a428, NULL); +#else iomfb_set_matrix(dcp, false, &mat, do_swap, NULL); +#endif } else do_swap(dcp, NULL, NULL); } @@ -1534,6 +2167,141 @@ static void init_3(struct apple_dcp *dcp, void *out, void *cookie) dcp_is_main_display(dcp, false, res_is_main_display, NULL); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * The client-open sequence H17P firmware expects before the first swap, in + * H17P method numbers (dcp_push() applies the H17G aliases): + * + * A445 A411 A445 A415 A450 A031 x9 A414 A478 A457 A474 A478 A476 A478 A412 + * + * A412 is not in the table: its result feeds res_is_main_display(), so it + * stays on the existing path. + */ +enum dcp_client_step_kind { + DCP_STEP_A445, DCP_STEP_A411, DCP_STEP_A415, DCP_STEP_A450, + DCP_STEP_A031, DCP_STEP_A414, DCP_STEP_A478, DCP_STEP_A457, + DCP_STEP_A474, DCP_STEP_A476, +}; + +static const u8 dcp_client_open_seq[] = { + DCP_STEP_A445, DCP_STEP_A411, DCP_STEP_A445, DCP_STEP_A415, + DCP_STEP_A450, + DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, + DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, + DCP_STEP_A031, + DCP_STEP_A414, DCP_STEP_A478, DCP_STEP_A457, DCP_STEP_A474, + DCP_STEP_A478, DCP_STEP_A476, DCP_STEP_A478, +}; + +static void dcp_client_open_run(struct apple_dcp *dcp, void *out, void *cookie) +{ + unsigned long idx = (unsigned long)cookie; + dcp_callback_t next = dcp_client_open_run; + void *ncookie = (void *)(idx + 1); + + if (idx >= ARRAY_SIZE(dcp_client_open_seq)) { + init_3(dcp, NULL, NULL); + return; + } + + switch (dcp_client_open_seq[idx]) { + case DCP_STEP_A445: { + u32 zero = 0; + + dcp_get_dfb_state(dcp, false, &zero, next, ncookie); + break; + } + case DCP_STEP_A411: { + u32 handle = 0; + + dcp_set_display_device(dcp, false, &handle, next, ncookie); + break; + } + case DCP_STEP_A415: { + struct dcp_pipe_cfg_415 cfg = {}; + + dcp_pipe_cfg_415(dcp, false, &cfg, next, ncookie); + break; + } + case DCP_STEP_A450: { + u32 val = 0; + + dcp_enable_disable_video_power_savings(dcp, false, &val, next, + ncookie); + break; + } + case DCP_STEP_A031: { + /* + * Nine cumulative states: each call repeats the previous set + * of notification clients and adds one, so state n lists n + * clients. + */ + static const u8 notify_clients[9][8] = { + {}, + { 6 }, + { 6, 7 }, + { 6, 7, 8 }, + { 6, 7, 8, 10 }, + { 6, 7, 8, 10, 11 }, + { 6, 7, 8, 10, 11, 12 }, + { 6, 7, 8, 10, 11, 12, 25 }, + { 6, 7, 8, 10, 11, 12, 25, 26 }, + }; + struct dcp_pipe_cfg_031 cfg = {}; + unsigned int step = dcp->a031_step; + unsigned int i; + + if (step >= ARRAY_SIZE(notify_clients)) + step = ARRAY_SIZE(notify_clients) - 1; + for (i = 0; i < step; i++) + cfg.clients[i] = cpu_to_le32(notify_clients[step][i]); + dcp->a031_step++; + + dcp_pipe_cfg_031(dcp, false, &cfg, next, ncookie); + break; + } + case DCP_STEP_A414: { + u64 val = 0; + + dcp_pipe_cfg_414(dcp, false, &val, next, ncookie); + break; + } + case DCP_STEP_A478: + dcp_pipe_query_478(dcp, false, next, ncookie); + break; + case DCP_STEP_A457: + dcp_first_client_open(dcp, false, next, ncookie); + break; + case DCP_STEP_A474: + dcp_pipe_query_474(dcp, false, next, ncookie); + break; + case DCP_STEP_A476: { + /* + * A476 takes a single u32, which the firmware expects to be 1. + * Its name is not known for certain: the shape (in 4, out 0) + * fits both set_dcp_clamshellstate() and + * set_has_frame_swap_function(). + */ + u32 state = 1; + + dcp_pipe_set_476(dcp, false, &state, next, ncookie); + break; + } + default: + dev_warn(dcp->dev, "unknown client-open step %u\n", + dcp_client_open_seq[idx]); + init_3(dcp, NULL, NULL); + break; + } +} + +static void init_1(struct apple_dcp *dcp, void *out, void *cookie) +{ + /* every client-open sequence starts from the empty A031 state */ + dcp->a031_step = 0; + dcp_client_open_run(dcp, NULL, (void *)0UL); +} +#else static void init_2(struct apple_dcp *dcp, void *out, void *cookie) { dcp_first_client_open(dcp, false, init_3, NULL); @@ -1544,6 +2312,7 @@ static void init_1(struct apple_dcp *dcp, void *out, void *cookie) u32 val = 0; dcp_enable_disable_video_power_savings(dcp, false, &val, init_2, NULL); } +#endif static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) { @@ -1559,9 +2328,9 @@ static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp) { - struct dcp_set_power_state_req req = { + struct DCP_FW_NAME(dcp_set_power_state_req) req = { /* defaults are ok */ }; - dcp_set_power_state(dcp, false, &req, NULL, NULL); + dcp_set_power_state(dcp, DCP_POWER_OOB, &req, NULL, NULL); } diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index ce9fb92513cd1..2d8787fc884fe 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -25,6 +25,17 @@ struct DCP_FW_NAME(dcp_swap) { u64 flags1; u64 flags2; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P inserts 0x48 bytes between flags2 and the swap ID, which moves + * to +0x98. Every following field shifts with it: surf_ids +0x9c, + * src_rect +0xac, surf_flags +0xec, dst_rect +0x10c, + * swap_enabled/completed +0x14c/+0x150, bg_color +0x154. The gap holds + * two small u32 counters (at +0x50 and +0x78) that are not modelled. + */ + u8 h17p_pre_swap_id[0x48]; +#endif + u32 swap_id; u32 surf_ids[SWAP_SURFACES]; @@ -53,6 +64,13 @@ struct DCP_FW_NAME(dcp_swap) { u8 unk_320[0x13f]; u64 unk_1; #endif +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * The H17P A408 request is 0xe9c bytes; with a 0x22c-byte surface, + * SWAP_SURFACES = 4 and no surf2 block, that leaves this tail. + */ + u8 h17p_tail[0xda]; +#endif } __packed; /* Information describing a surface */ @@ -73,8 +91,14 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { u64 surf_iova[SWAP_SURFACES]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u64 unk_u64_a[SWAP_SURFACES]; + /* + * H17P carries no surf2 block: its trailing u8 null-flag run is 8 bytes + * (swap_null + surf_null[4] + three tail bools). + */ +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) struct DCP_FW_NAME(dcp_surface) surf2[5]; u64 surf2_iova[5]; +#endif #endif u8 unkbool; u64 unkdouble; @@ -89,14 +113,18 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { u8 swap_null; u8 surf_null[SWAP_SURFACES]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 surf2_null[5]; +#endif #endif u8 unkoutbool_null; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u8 unkU32Ptr_null; u8 unkU32out_null; #endif +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 padding[1]; +#endif } __packed; struct DCP_FW_NAME(dcp_swap_submit_resp) { @@ -108,9 +136,104 @@ struct DCP_FW_NAME(dcp_swap_submit_resp) { u8 padding[3]; } __packed; -struct DCP_FW_NAME(dc_swap_complete_resp) { +/* + * A406 swap_start / A473 set_power_state. H17P shrank both requests: the + * 0x10-byte struct dcp_iouserclient became a bare u64 handle and the + * swap_start response no longer echoes it. swap_start is 0x10 in / 0x8 out, + * set_power_state 0x10 in. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +struct DCP_FW_NAME(dcp_swap_start_req) { u32 swap_id; + u64 client; /* IOUserClient kernel VA */ + u8 client_null; + u8 padding[3]; +} __packed; + +struct DCP_FW_NAME(dcp_swap_start_resp) { + u32 swap_id; + u32 ret; +} __packed; + +struct DCP_FW_NAME(dcp_set_power_state_req) { + u64 unklong; + u32 unkint; /* widened from u8 on H17P */ + u8 unkint_null; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct DCP_FW_NAME(dcp_swap_start_req)) == 0x10); +static_assert(sizeof(struct DCP_FW_NAME(dcp_swap_start_resp)) == 0x8); +static_assert(sizeof(struct DCP_FW_NAME(dcp_set_power_state_req)) == 0x10); +#else +struct DCP_FW_NAME(dcp_swap_start_req) { + u32 swap_id; + struct dcp_iouserclient client; + u8 swap_id_null; + u8 client_null; + u8 padding[2]; +} __packed; + +struct DCP_FW_NAME(dcp_swap_start_resp) { + u32 swap_id; + struct dcp_iouserclient client; + u32 ret; +} __packed; + +struct DCP_FW_NAME(dcp_set_power_state_req) { + u64 unklong; u8 unkbool; + u8 unkint_null; + u8 padding[2]; +} __packed; +#endif + +/* + * D201 map_piodma reply. H17P declares a 16-byte output, not 20, laid out as + * { u32 buffer index, u64 dva, u32 pad } rather than { u64 vaddr, u64 dva, + * u32 ret }. With the old layout the DVA lands four bytes late and the + * display engine DMAs to a bogus address. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +struct DCP_FW_NAME(dcp_map_buf_resp) { + u32 buffer; + u64 dva; + u32 unk; +} __packed; +static_assert(sizeof(struct DCP_FW_NAME(dcp_map_buf_resp)) == 0x10); +#else +struct DCP_FW_NAME(dcp_map_buf_resp) { + u64 vaddr; + u64 dva; + u32 ret; +} __packed; +#endif + +struct DCP_FW_NAME(dc_swap_complete_resp) { + u32 swap_id; /* +0x000 */ + u8 unkbool; /* +0x004 */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P: swap_complete_ap_gated(u32, bool, SwapCompleteData const *, + * SwapInfoBlob const *, u32, bool) is 0x730 bytes on the wire, 89 more + * than the pre-H17P record: the SwapCompleteData grew from 8 to 0x22, + * swap_info became eight 0xe0-byte SwapInfoBlobs rather than one + * 0x6c5 blob, and the tail gained a flag and three pad bytes. Only + * swap_info_count of those blobs are populated; the rest are + * uninitialised DCP buffer memory. + */ + u32 hol_swap_ids[4]; /* +0x005 */ + u32 hol_arg; /* +0x015 */ + u32 hol_count; /* +0x019 */ + u8 hol_valid; /* +0x01d */ + u64 displayed_data; /* +0x01e, unaligned */ + u8 notify_displayed; /* +0x026 */ + u8 swap_info[8][0xe0]; /* +0x027, blob[0]+0x38 == swap_id */ + u32 swap_info_count; /* +0x727, max 8 */ + u8 completion_flag; /* +0x72b */ + u8 swap_data_null; /* +0x72c, 0 => data at +0x005 valid */ + u8 padding[3]; /* +0x72d, to sizeof == 0x730 */ +#else u64 swap_data; #if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) u8 swap_info[0x6c4]; @@ -119,6 +242,7 @@ struct DCP_FW_NAME(dc_swap_complete_resp) { #endif u32 unkint; u8 swap_info_null; +#endif } __packed; struct DCP_FW_NAME(dcp_map_reg_req) { diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.c b/drivers/gpu/drm/apple/iomfb_v12_3.c index 0fe08c42d6465..3256974fb11d0 100644 --- a/drivers/gpu/drm/apple/iomfb_v12_3.c +++ b/drivers/gpu/drm/apple/iomfb_v12_3.c @@ -3,6 +3,7 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" +#include "iomfb_h17p.h" #include "version_utils.h" static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { diff --git a/drivers/gpu/drm/apple/iomfb_v13_3.c b/drivers/gpu/drm/apple/iomfb_v13_3.c index 0ac869d24eb01..9adce7b150fd0 100644 --- a/drivers/gpu/drm/apple/iomfb_v13_3.c +++ b/drivers/gpu/drm/apple/iomfb_v13_3.c @@ -3,6 +3,7 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" +#include "iomfb_h17p.h" #include "version_utils.h" static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 61c20b9dc2dc3..3ba9f5e4c4dd5 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -648,7 +648,7 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, } int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, - const char **class, s64 *unit) + const char **class, s64 *unit, bool h17p_keys) { int ret = 0; struct iterator it; @@ -667,19 +667,36 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, break; } - if (!strcmp(key, "EPICName")) { + /* + * H17P advertises its system service with lower-case keys -- + * "compartment", "name", "interface-id" -- where earlier + * firmware uses EPICName/EPICProviderClass/EPICUnit. Without + * them the service never registers, the firmware has no system + * service to ask and idles the panel, so no vblank and no swap + * ever completes. Only H17P callers accept them. If a + * dictionary carries both spellings, the later value wins and + * the earlier string is freed. + */ + if (!strcmp(key, "EPICName") || + (h17p_keys && !strcmp(key, "compartment"))) { + if (parsed_name) + kfree(*name); *name = parse_string(it.handle); if (IS_ERR(*name)) ret = PTR_ERR(*name); else parsed_name = true; - } else if (!strcmp(key, "EPICProviderClass")) { + } else if (!strcmp(key, "EPICProviderClass") || + (h17p_keys && !strcmp(key, "name"))) { + if (parsed_class) + kfree(*class); *class = parse_string(it.handle); if (IS_ERR(*class)) ret = PTR_ERR(*class); else parsed_class = true; - } else if (!strcmp(key, "EPICUnit")) { + } else if (!strcmp(key, "EPICUnit") || + (h17p_keys && !strcmp(key, "interface-id"))) { ret = parse_int(it.handle, unit); if (!ret) parsed_unit = true; diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 35f4e81e76212..bfb0a06c8760a 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -109,7 +109,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, - const char **class, s64 *unit); + const char **class, s64 *unit, bool h17p_keys); struct dcp_sound_format_mask { u64 formats; /* SNDRV_PCM_FMTBIT_* */ From 1f30c693c1f133260b83ddc29e5cb2939cc64346 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 23:34:51 -0400 Subject: [PATCH 0381/1579] arm64: dts: apple: t8142-j813: add the internal display Describe the J813 display coprocessor (DCP), its mailbox and both display DARTs, with resources read from this machine's Apple Device Tree, plus the dcp/disp0/disp0_piodma aliases the loader uses to hand over the live firmware mappings. The DCP and display DART stay disabled until the loader has reserved those mappings and enables them. Enable the Apple DRM driver in the J813 configuration and document the loader contract and the tested result. Tested on J813 from RAM with public m1n1 and U-Boot plus their J813 display and MTP companion changes: DCP completed startup and drove the internal panel at 2560x1600 (notch hidden) through appledrmfb, DPMS off/on cycles completed, and a KDE Plasma Wayland session ran with the built-in keyboard and trackpad. Rendering is software-only. --- Documentation/arch/arm64/apple-j813.rst | 40 +++++++- .../boot/dts/apple/t8142-j813-display.dtsi | 95 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + arch/arm64/configs/j813.config | 4 + 4 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-display.dtsi diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst index 8ac299c9ef97c..a3819bc60a524 100644 --- a/Documentation/arch/arm64/apple-j813.rst +++ b/Documentation/arch/arm64/apple-j813.rst @@ -173,7 +173,7 @@ an Image build or successful loader exit alone is not a boot result. Review captures for device identifiers and boot entropy before publishing. These boot tests do not establish CPU hotplug, KVM guest timers, -sustained thermal behavior, suspend, native DCP, GPU acceleration, USB, +sustained thermal behavior, suspend, GPU acceleration, USB, storage, networking or audio support. Input and monitoring require their separate checks below. The disabled Samsung UART node records its measured resources; the qualified console path is DockChannel. @@ -216,7 +216,9 @@ the keyboard or temperature monitor from registering. The loader must preserve the iBoot-loaded MTP image and SRAM, prepare its DMA protection, and leave MTP ready for Linux's RTKit startup. A prior-stage MTP client must complete a compatible shutdown before Linux starts; a partially -initialized or crashed coprocessor is not a supported handoff. +initialized or crashed coprocessor is not a supported handoff. On J813, U-Boot +must not start and stop MTP before Linux: Linux's later restart crashes the +firmware and takes the keyboard and trackpad down. The backlight uses the existing PWM and LED drivers. Its boot brightness is retained when the initial PWM period is valid, otherwise it starts off. @@ -240,3 +242,37 @@ proof. Preserve stdout, the source revision and SHA-256 hashes of the exact Image, DTB, configuration, initramfs and bootloaders as the repeatable result. Keep firmware and raw machine data private. Without physical events the input check fails rather than crediting device registration as working input. + +Native display +============== + +The internal panel is driven by the display coprocessor (DCP) through the +Apple DRM driver. iBoot leaves DCP running with H17-generation (macOS 26) +firmware mapped through locked DARTs. Linux attaches to that live session: +it sends the standard RTKit wake handshake and never stops, resets or reloads +the coprocessor. No firmware file is loaded by Linux. + +The loader must: + +* reserve every live DCP firmware segment and the boot framebuffer, and + export their IOVAs to the ``dcp``, ``disp0`` and ``disp0_piodma`` nodes; +* enable the display DART (the first ``iommus`` entry of ``disp0``) and the + ``dcp`` node only after doing so. + +The display DARTs keep the firmware's root page-table slots. Linux publishes +its own translations only into vacant slots. On ``apple,t8142-dart``, a +firmware slot is accepted when it already translates every page Linux mapped +there to the same physical address, and later Linux mappings into it must +match the firmware's translation. Firmware buffers DCP later asks Linux to +share are resolved through those firmware tables. + +The DCP firmware reports the panel dimensions at startup; the driver does not +embed a panel timing or geometry. The J813 panel reports 2560x1664; the driver +hides its 64-pixel notch area by default, so the mode is 2560x1600 at 60 Hz. +The primary plane scans out IOMFB surface 2. Backlight control and a cursor +plane are not provided yet. + +Boot the RAM payload as above with ``CONFIG_DRM_APPLE``. On J813 this reached +``DCP booted``, registered ``appledrmfb`` as the console framebuffer, and ran +a KDE Plasma Wayland session with the built-in keyboard and trackpad. The +desktop renders in software; GPU acceleration is separate work. diff --git a/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi new file mode 100644 index 0000000000000..a50fa38fd9265 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J813 internal display pipeline: the DCP coprocessor and its DARTs. + * + * The bootloader leaves DCP running with its firmware mapped through + * locked DART tables. The loader reserves those mappings and enables the + * nodes below; Linux attaches to the running coprocessor without resetting + * it. Resources were read from this machine's Apple Device Tree. + */ + +/ { + aliases { + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + /* disp0_dart must be first since it is locked */ + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; +}; + +&{/soc} { + pmgr_dcp: power-management@382720000 { + reg = <3 0x82720000 0 0x8>, <3 0x827a4000 0 0x4>; + reg-names = "dcp-bw-scratch", "dcp-bw-doorbell"; + #apple,bw-scratch-cells = <3>; + #apple,bw-doorbell-cells = <2>; + }; + + dcp_mbox: mailbox@286e08000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <2 0x86e08000 0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + disp0_dart: iommu@286300000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <2 0x86300000 0 0xc000>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + dcp_dart: iommu@286340000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <2 0x86340000 0 0x20000>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + }; + + dcp: dcp@286e00000 { + compatible = "apple,t8142-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 23>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4", "iop-vbar"; + reg = <2 0x86e00000 0 0x4000>, + <2 0x84000000 0 0xb1c000>, + <2 0x86240000 0 0x10000>, + <2 0x863ac000 0 0x4000>, + <2 0x86800000 0 0x800000>, + <3 0x82700000 0 0x114000>, + <2 0x86850000 0 0x4000>; + apple,bw-scratch = <&pmgr_dcp 0 5 0>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + /* Scan out the primary plane on IOMFB surface 2. */ + apple,iomfb-surfaces = <0 0 1 0>; + clocks = <&clkref>; + /* generate phandle explicitly for use in loader */ + phandle = <&dcp>; + status = "disabled"; + /* required bus properties for 'piodma' subdevice */ + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 16>; + phandle = <&disp0_piodma>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index be04fd061a885..20df27fa1bfe3 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -291,5 +291,6 @@ }; }; +#include "t8142-j813-display.dtsi" #include "t8142-j813-peripherals.dtsi" #include "t8142-j813-usb.dtsi" diff --git a/arch/arm64/configs/j813.config b/arch/arm64/configs/j813.config index 57fb42b0295c5..9091e74f3e918 100644 --- a/arch/arm64/configs/j813.config +++ b/arch/arm64/configs/j813.config @@ -45,6 +45,10 @@ CONFIG_FB=y CONFIG_FB_DEVICE=y CONFIG_FB_SIMPLE=y CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_DRM=y +CONFIG_DRM_APPLE=y +CONFIG_BACKLIGHT_CLASS_DEVICE=y +CONFIG_DRM_FBDEV_EMULATION=y # CONFIG_FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER is not set CONFIG_LOG_BUF_SHIFT=20 CONFIG_IKCONFIG=y From b2593675a9585888761ed546d1ba215e3045d422 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Fri, 2 Oct 2026 20:21:00 -0700 Subject: [PATCH 0382/1579] Bluetooth: hci_bcm4377: Rebind the driver when the controller stops responding After some suspend-to-idle cycles on a BCM4388 (MacBook Pro 16" M2 Max), the controller no longer answers HCI commands. hci_resume_dev() times out on Set Event Mask (0x0c01), followed by Write Scan Enable, LE address resolution and scan parameter commands. Every later command times out as well, until the module is reloaded, which recovers it. Install an hdev->reset handler, which the HCI core calls on a command timeout, and recover the same way: rebind the driver with device_reprobe(), so probe resets the function and boots the firmware again. The work and its device reference are allocated separately because the reprobe frees bcm4377_data. Timeouts during setup are left alone, as are timeouts while a reprobe is already queued. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan (cherry picked from commit 37e9742a8c798226df0dc219bb3c11e59fc53e5b) --- drivers/bluetooth/hci_bcm4377.c | 55 +++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 0eb137a99add7..dace2f0a6adcd 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -585,6 +585,9 @@ struct bcm4377_data { struct completion event; + /* Set once a recovery reprobe has been queued for this binding. */ + unsigned long reset_pending; + struct bcm4377_context *ctx; dma_addr_t ctx_dma; @@ -2268,6 +2271,57 @@ static void bcm4377_hci_free_dev(void *data) hci_free_dev(data); } +struct bcm4377_reset_work { + struct work_struct work; + struct pci_dev *pdev; +}; + +static void bcm4377_reset_work(struct work_struct *work) +{ + struct bcm4377_reset_work *rw = + container_of(work, struct bcm4377_reset_work, work); + + /* + * Remove and probe again: probe resets the function and boots the + * firmware from scratch. device_reprobe() frees the old bcm4377_data, + * which is why this work and its device reference live outside it. + */ + if (device_reprobe(&rw->pdev->dev)) + dev_err(&rw->pdev->dev, "reprobe after command timeout failed\n"); + + pci_dev_put(rw->pdev); + kfree(rw); +} + +/* + * Called by the HCI core (from process context) on a command timeout. After + * some system resumes the controller stops answering commands for good: + * every later command times out until the module is reloaded. Recover the + * same way, by rebinding the driver. + */ +static void bcm4377_hci_reset(struct hci_dev *hdev) +{ + struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); + struct bcm4377_reset_work *rw; + + /* A timeout while still setting up would only loop probe failures. */ + if (hci_dev_test_flag(hdev, HCI_SETUP)) + return; + if (test_and_set_bit(0, &bcm4377->reset_pending)) + return; + + rw = kzalloc_obj(*rw); + if (!rw) { + clear_bit(0, &bcm4377->reset_pending); + return; + } + + bt_dev_err(hdev, "controller not responding, rebinding the driver"); + rw->pdev = pci_dev_get(bcm4377->pdev); + INIT_WORK(&rw->work, bcm4377_reset_work); + schedule_work(&rw->work); +} + static void bcm4377_hci_unregister_dev(void *data) { hci_unregister_dev(data); @@ -2387,6 +2441,7 @@ static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id) hdev->send = bcm4377_hci_send_frame; hdev->set_bdaddr = bcm4377_hci_set_bdaddr; hdev->setup = bcm4377_hci_setup; + hdev->reset = bcm4377_hci_reset; if (bcm4377->hw->broken_mws_transport_config) hci_set_quirk(hdev, HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG); From 6b9c477eaadf6b1401defd141c1677816ebd232b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:28:46 -0400 Subject: [PATCH 0383/1579] arm64: align host BSS with mixed-granule zero pages A 16K Apple-enabled kernel with KVM fails its final link when the 64K-aligned mixed-granule zero page moves .bss beyond the PAGE_SIZE-aligned host BSS start. Align that start to MAX(PAGE_SIZE, ALIGNOF(.bss)) so the existing host/hypervisor equality assertion remains satisfied. The full Rust/Asahi/DART Image, modules and device trees build. The same Image boots in a virt guest and passes native16/alternative4 static and dynamic execution, mixed-process shared futex/COW and process_vm tests, and actual userpages/FEX memory, JIT and re-exec workloads. Focused GNU and LLVM linker fixtures cover 4K, 16K, mixed16K/64K and 64K alignment. Document the opt-in launch path and distinguish VM results from untested Apple boots. --- Documentation/arch/arm64/user4k.rst | 42 +++++++++++++++++++++++++++-- arch/arm64/kernel/vmlinux.lds.S | 5 ++-- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index c9798cbf5fdb0..e305adf42bf92 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -10,8 +10,7 @@ Status and scope This work starts from Aurora's M5 Pro branch, commit 4832ab22bb9080d82c29f0ca4ee686492f0552f8. The experimental branch is ``mm-mixed-userspace-abi`` in ``aurora-silicon/linux-aurora``. -It does not include the source worktree's uncommitted -display changes. Do not deploy it to the M5 or replace the host kernel. +It does not include the source worktree's uncommitted display changes. The experimental kernel now executes opted-in 4K EL0 processes alongside ordinary native-granule processes on 16K and 64K kernels. Opt-in 16K EL0 @@ -23,6 +22,45 @@ native ABI are tested. Real mixed-size processes share memfd/futex state and use process_vm_readv/writev in both directions. See the latest milestone below for exact evidence and limits; this is not yet a complete or battle-tested ABI. +Apple-enabled build and process launch (2026-10-03) +-------------------------------------------------- + +On a native 16K arm64 kernel, enable ``ARM64_USER4K_EXPERIMENTAL`` with +48-bit kernel VA and PA configurations. ``MM_SUBPAGE`` is selected by the +option. Keep the ordinary kernel page-size choice at 16K. Neither the +private KUnit/test-device options nor ``arm64.user4k_test=1`` are needed +for application execution. + +Build the launcher with ``make -C tools/aurora-userpages``. Opt a process +tree in using:: + + tools/aurora-userpages/userpages 4k COMMAND [ARGUMENTS...] + +``userpages --status`` reports the current process size and exec preferences. +``userpages native COMMAND`` clears an inherited alternative preference. +Ordinary processes continue to use 16K pages. See the launcher's README for +one-shot selection and the supported 64K-native combinations. + +An Apple-enabled native16 image was built with Rust, Asahi DRM, Apple DART +and KVM, using the available M2 Pro's kernel configuration as the starting +profile. Private mixed-page test devices were disabled. That same image +booted in a QEMU virt guest and passed native/4K static and dynamic execution, +large argument transitions, proc/memory interfaces, and mixed 16K/4K shared +futex/COW and process_vm copies. The actual launcher passed native/4K/native +transitions, and FEX passed the x86 memory/protection/COW/JIT/re-exec contract +with its AArch64 processes reporting 4096-byte pages. These are VM results; +this candidate has not been booted on an Apple Mac, submitted Apple GPU +commands, or run a Windows game. + +The Apple profile exposed a link-layout failure: the mixed-granule zero page +requires 64K BSS alignment while KVM aligned ``__bss_start`` only to the +native 16K page. Kallsyms growth could then leave ``__hyp_bss_start`` at a +different address and fail the existing linker assertion. The linker script +now uses the greater of the native page size and ``ALIGNOF(.bss)`` for the +host BSS start. The assertion remains enabled. The full image links with +identical host/hypervisor BSS starts; focused LLVM and GNU linker fixtures +cover native4K, native16K, mixed16K/64K alignment and native64K. + Current userfaultfd support (2026-10-02) --------------------------------------- diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index e1ac876200a3d..688b82c0cf30d 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -66,9 +66,10 @@ * We require that __hyp_bss_start and __bss_start are aligned, and enforce it * with an assertion. But the BSS_SECTION macro places an empty .sbss section * between them, which can in some cases cause the linker to misalign them. To - * work around the issue, force a page alignment for __bss_start. + * work around the issue, align __bss_start to the BSS section's alignment, + * which can exceed PAGE_SIZE for the mixed-granule zero page. */ -#define SBSS_ALIGN PAGE_SIZE +#define SBSS_ALIGN MAX(PAGE_SIZE, ALIGNOF(.bss)) #else /* CONFIG_KVM */ #define HYPERVISOR_EXTABLE #define HYPERVISOR_RODATA_SECTIONS From 5dc9a504add1ccc6f5ba601ebda629173f65591e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:34:53 -0400 Subject: [PATCH 0384/1579] ipc/shm: honor per-mm page sizes for attach and detach A smaller-page process could receive EINVAL for a valid fixed or rounded SysV attach, or fail to detach an automatically placed mapping. Native PAGE_SIZE checks also lost the byte offsets of split subpage VMAs. Use an arm64 attach-alignment hook and per-mm address masks. Match detach fragments by both their native file index and byte offset, and round the segment extent to the process granule. Preserve the global SHMLBA constant used by kernel allocations and the existing AArch32 native-page fallback. Add actual AArch64/AArch32 syscall tests for fixed and rounded attachments, partial unmap/protection splits, alias lifetime, and mixed-granule sharing. Native16/4K and native64/16K/4K matrices pass, as do capability-masked AArch32 native16/native64 fallbacks. Feature-disabled arm64/x86 objects compile. The Apple-enabled mm2 Image/modules/DTBs build; the same Image passes SysV, ordinary exec/memory tests and the existing Wine/FEX Windows memory contract in a VM. Apple hardware and graphics/game execution remain untested. --- Documentation/arch/arm64/user4k.rst | 29 +++ arch/arm64/include/asm/shmparam.h | 6 + ipc/shm.c | 36 ++- tools/testing/selftests/arm64/user4k/Makefile | 7 +- .../selftests/arm64/user4k/compat-shm-probe.c | 107 +++++++++ .../selftests/arm64/user4k/sysv-shm-init.c | 222 ++++++++++++++++++ 6 files changed, 398 insertions(+), 9 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/compat-shm-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/sysv-shm-init.c diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index e305adf42bf92..e6fc63736c80c 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -61,6 +61,35 @@ host BSS start. The assertion remains enabled. The full image links with identical host/hypervisor BSS starts; focused LLVM and GNU linker fixtures cover native4K, native16K, mixed16K/64K alignment and native64K. +System V shared memory across process granules (2026-10-03) +---------------------------------------------------------- + +SysV attach and detach now use the calling mm's page granule. The arm64 +attach-alignment hook scales the native and compat rounding rules only for +SysV attachments; the global SHMLBA constant remains unchanged for kernel +allocation users. AArch32 keeps its four-process-page rounding rule, including +the existing native-page restriction when 4K translation is unavailable. + +Detach compares both the native file index and subpage byte offset when +recognizing fragments left by mprotect or partial munmap. Its segment extent +is rounded to the process granule. This preserves shared-object identity and +removes all matching fragments without including adjacent mappings. + +The old kernel rejected valid smaller-page fixed and rounded attaches and +could reject detach of an automatically placed smaller-page alias. The new +PID1 fixture passes native16/4K and native64/16K/4K mapping cases, partial +unmap/protection splits, alias lifetime and all directed mixed-granule SysV +sharing pairs, including attachment after IPC_RMID while another alias lives. +AArch32 automatic4K passes on both builds. Capability-masked no4K models pass +native16/native64 AArch32 attach/detach and retain rejection of 4K-only +mprotect requests. Feature-disabled arm64 and x86 SysV objects also compile. +The fixture requires CONFIG_SYSVIPC and runs only in a disposable guest. + +The Apple-enabled ``7.1.12-aurora-mixed-mm2`` Image also passes the SysV +fixture, the existing native16/4K static/dynamic execution suite, and the +Windows memory contract through the existing private Wine/FEX fixture in a VM. +Apple hardware boot, GPU execution and game testing remain unqualified. + Current userfaultfd support (2026-10-02) --------------------------------------- diff --git a/arch/arm64/include/asm/shmparam.h b/arch/arm64/include/asm/shmparam.h index f920e22ec677c..75120d3a71889 100644 --- a/arch/arm64/include/asm/shmparam.h +++ b/arch/arm64/include/asm/shmparam.h @@ -12,6 +12,12 @@ */ #define COMPAT_SHMLBA (4 * PAGE_SIZE) +/* SHMLBA also aligns kernel allocations; scale only SysV attachments. */ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_shm_attach_align(mm, align) \ + (((align) >> PAGE_SHIFT) << mm_page_shift(mm)) +#endif + #include #endif /* __ASM_SHMPARAM_H */ diff --git a/ipc/shm.c b/ipc/shm.c index b3e8a58e177d3..2929b9243f8e8 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -51,6 +51,14 @@ #include "util.h" +#ifndef arch_shm_attach_align +static unsigned long arch_shm_attach_align(const struct mm_struct *mm, + unsigned long align) +{ + return align; +} +#endif + struct shmid_kernel /* private to the kernel */ { struct kern_ipc_perm shm_perm; @@ -1538,6 +1546,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, if (shmid < 0) goto out; + shmlba = arch_shm_attach_align(current->mm, shmlba); if (addr) { if (addr & (shmlba - 1)) { if (shmflg & SHM_RND) { @@ -1552,7 +1561,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, goto out; } else #ifndef __ARCH_FORCE_SHMLBA - if (addr & ~PAGE_MASK) + if (addr & ~mm_page_mask(current->mm)) #endif goto out; } @@ -1723,6 +1732,20 @@ COMPAT_SYSCALL_DEFINE3(shmat, int, shmid, compat_uptr_t, shmaddr, int, shmflg) } #endif +/* VMA splits retain native file indices plus a subpage byte offset. */ +#ifdef CONFIG_MMU +static bool shm_vma_matches_addr(struct vm_area_struct *vma, unsigned long addr) +{ + unsigned long offset; + + if (vma->vm_ops != &shm_vm_ops || vma->vm_start < addr) + return false; + offset = vma->vm_start - addr; + return (offset >> PAGE_SHIFT) == vma->vm_pgoff && + (offset & ~PAGE_MASK) == vma_subpage_offset(vma); +} +#endif + /* * detach and kill segment if marked destroyed. * The work is done in shm_close. @@ -1739,7 +1762,7 @@ long ksys_shmdt(char __user *shmaddr) VMA_ITERATOR(vmi, mm, addr); #endif - if (addr & ~PAGE_MASK) + if (addr & ~mm_page_mask(mm)) return retval; if (mmap_write_lock_killable(mm)) @@ -1774,8 +1797,7 @@ long ksys_shmdt(char __user *shmaddr) * a fragment created by mprotect() and/or munmap(), or it * otherwise it starts at this address with no hassles. */ - if ((vma->vm_ops == &shm_vm_ops) && - (vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) { + if (shm_vma_matches_addr(vma, addr)) { /* * Record the file of the shm segment being @@ -1804,12 +1826,10 @@ long ksys_shmdt(char __user *shmaddr) * could possibly have landed at. Also cast things to loff_t to * prevent overflows and make comparisons vs. equal-width types. */ - size = PAGE_ALIGN(size); + size = ALIGN(size, mm_page_size(mm)); while (vma && (loff_t)(vma->vm_end - addr) <= size) { /* finding a matching vma now does not alter retval */ - if ((vma->vm_ops == &shm_vm_ops) && - ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) && - (vma->vm_file == file)) { + if (shm_vma_matches_addr(vma, addr) && vma->vm_file == file) { do_vmi_align_munmap(&vmi, vma, mm, vma->vm_start, vma->vm_end, NULL, false); } diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index fdcd0385f76a6..46b21e4b381cc 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -1,10 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 TEST_GEN_PROGS := page-contract TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init -TEST_GEN_PROGS_EXTENDED += vfio-fixture-init +TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe +TEST_GEN_FILES += compat-shm-probe TEST_FILES += compat-script runtime-app.py include ../../lib.mk @@ -108,3 +109,7 @@ $(OUTPUT)/iommufd-init: CFLAGS += $(KHDR_INCLUDES) $(OUTPUT)/vfio-fixture-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 $(OUTPUT)/compat-vfio-probe: compat-vfio-probe.c $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/sysv-shm-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-shm-probe: compat-shm-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ diff --git a/tools/testing/selftests/arm64/user4k/compat-shm-probe.c b/tools/testing/selftests/arm64/user4k/compat-shm-probe.c new file mode 100644 index 0000000000000..9d8d5c3bec057 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-shm-probe.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 SysV attach/detach and fallback-alignment checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" : "+r"(r0) : "r"(r1), "r"(r2), "r"(r3), + "r"(r4), "r"(r5), "r"(r7) : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 SHM FAIL stage=00\n"; + + if (code) { + text[19] = '0' + code / 10; + text[20] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 SHM PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long pages, int rounded) +{ + unsigned long align = 4 * pages, length = 4 * pages; + long raw = call(192, 0, 12 * align, 3, 0x22, -1, 0); + unsigned char *target; + long id, result; + + if ((unsigned long)raw >= 0xfffff001UL) + finish(20); + result = call(125, raw + 4096, 4096, 1, 0, 0, 0); + if (result != (pages == 4096 ? 0 : -22)) + finish(21); + if (call(125, raw, 12 * align, 3, 0, 0, 0)) + finish(22); + target = (void *)(((unsigned long)raw + align - 1) & ~(align - 1)); + target += align + (rounded ? 0 : pages); + target[-1] = 0x51; + target[length] = 0x72; + if (call(91, (long)target, length, 0, 0, 0, 0)) + finish(23); + id = call(307, 0, length - 1, 0600, 0, 0, 0); + if (id < 0) + finish(24); + result = call(305, id, (long)(target + (rounded ? pages + 17 : 0)), + rounded ? 020000 : 0, 0, 0, 0); + if (result != (long)target) { + call(308, id, 0, 0, 0, 0, 0); + finish(rounded ? 26 : 25); + } + if (call(308, id, 0, 0, 0, 0, 0)) + finish(27); + target[pages + 13] = 0x67; + if (call(125, (long)(target + pages), pages, 1, 0, 0, 0)) + finish(28); + if (call(91, (long)target, pages, 0, 0, 0, 0)) + finish(29); + if (call(306, (long)target, 0, 0, 0, 0, 0)) + finish(30); + if (target[-1] != 0x51 || target[length] != 0x72) + finish(31); + if (call(91, raw, 12 * align, 0, 0, 0, 0)) + finish(32); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + mapping(pages, 0); + mapping(pages, 1); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/sysv-shm-init.c b/tools/testing/selftests/arm64/user4k/sysv-shm-init.c new file mode 100644 index 0000000000000..a42cc244f6340 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/sysv-shm-init.c @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: SysV shared memory with per-process page granules. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int ok, const char *expression, int line) +{ + if (!ok) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, errno); + exit(1); + } +} + +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int unmapped(void *address, size_t page) +{ + unsigned char resident; + + errno = 0; + return mincore(address, page, &resident) == -1 && errno == ENOMEM; +} + +static int mapping_case(unsigned long native, int rounded, int split) +{ + unsigned long page = getauxval(AT_PAGESZ), length = 4 * page; + unsigned char *reserve, *target, *mapping, *alias; + struct shmid_ds info; + int id; + + reserve = mmap(NULL, 16 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + target = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + target += native + (page < native ? page : 0); + target[-1] = 0x51; + target[length] = 0x72; + CHECK(!munmap(target, length)); + id = shmget(IPC_PRIVATE, length - 1, 0600); + CHECK(id >= 0); + mapping = shmat(id, target + (rounded ? page - 1 : 0), rounded ? SHM_RND : 0); + if (mapping != target) { + int saved = errno; + + shmctl(id, IPC_RMID, NULL); + errno = saved; + CHECK(mapping == target); + } + alias = shmat(id, NULL, 0); + CHECK(alias != (void *)-1); + CHECK(!shmctl(id, IPC_RMID, NULL)); + mapping[0] = 0x31; + mapping[page + 7] = 0x49; + mapping[3 * page] = 0x63; + CHECK(alias[0] == 0x31 && alias[page + 7] == 0x49 && alias[3 * page] == 0x63); + if (split) { + CHECK(!mprotect(mapping + page, page, PROT_READ)); + CHECK(!munmap(mapping + 2 * page, page)); + if (split == 2) + CHECK(!munmap(mapping, page)); + } + CHECK(!shmdt(mapping)); + for (unsigned long i = 0; i < length; i += page) + CHECK(unmapped(target + i, page)); + CHECK(target[-1] == 0x51 && target[length] == 0x72); + CHECK(alias[page + 7] == 0x49 && alias[3 * page] == 0x63); + CHECK(!shmdt(alias)); + errno = 0; + CHECK(shmctl(id, IPC_STAT, &info) == -1 && (errno == EINVAL || errno == EIDRM)); + CHECK(!munmap(reserve, 16 * native)); + printf("ok - SysV page=%lu rounded=%d split=%d detach and alias lifetime\n", + page, rounded, split); + return 0; +} + +static int wait_child(pid_t pid) +{ + int status; + pid_t got; + + if (pid < 0) + return 1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + return got != pid || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static int sharing_child(int id, unsigned long expected) +{ + unsigned char *map; + + CHECK(getauxval(AT_PAGESZ) == expected); + map = shmat(id, NULL, 0); + CHECK(map != (void *)-1); + CHECK(map[37] == 0x35 && map[4096 + 9] == 0x56); + map[4096 + 9] = 0x78; + CHECK(!shmdt(map)); + return 0; +} + +static int sharing(unsigned long other, unsigned long native) +{ + unsigned long page = getauxval(AT_PAGESZ); + unsigned char *map; + pid_t pid; + int id; + + id = shmget(IPC_PRIVATE, 2 * native, 0600); + CHECK(id >= 0); + map = shmat(id, NULL, 0); + CHECK(map != (void *)-1); + CHECK(!shmctl(id, IPC_RMID, NULL)); + map[37] = 0x35; + map[4096 + 9] = 0x56; + pid = fork(); + if (!pid) { + char idarg[32], sizearg[32]; + + snprintf(idarg, sizeof(idarg), "%d", id); + snprintf(sizearg, sizeof(sizearg), "%lu", other); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, other, 0UL, 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--share", idarg, sizearg, NULL); + _exit(3); + } + CHECK(!wait_child(pid)); + CHECK(map[4096 + 9] == 0x78); + CHECK(!shmdt(map)); + printf("ok - SysV shared object %lu -> %lu after IPC_RMID\n", page, other); + return 0; +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--share")) + return sharing_child(atoi(argv[2]), strtoul(argv[3], NULL, 10)); + if (argc == 5 && !strcmp(argv[1], "--case")) { + unsigned long expected = strtoul(argv[3], NULL, 10); + unsigned long sizes[] = {4096, 16384, 65536}; + + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == expected); + failed |= mapping_case(native, 0, 0); + failed |= mapping_case(native, 1, 0); + failed |= mapping_case(native, 0, 1); + failed |= mapping_case(native, 0, 2); + for (unsigned int i = 0; i < 3; i++) + if (sizes[i] <= native && sizes[i] != expected && + !(atoi(argv[4]) && sizes[i] == 4096)) + failed |= sharing(sizes[i], native); + return failed; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + int no4k = argc == 2 && !strcmp(argv[1], "--no4k"); + unsigned long sizes[] = {4096, 16384, 65536}; + + for (unsigned int i = 0; i < 3; i++) { + unsigned long page = sizes[i]; + pid_t pid; + + if (page > native || (no4k && page == 4096)) + continue; + pid = fork(); + if (!pid) { + char n[32], e[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(e, sizeof(e), "%lu", page); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, page, 0UL, 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--case", n, e, no4k ? "1" : "0", NULL); + _exit(3); + } + int bad = wait_child(pid); + + printf("%s - SysV AArch64 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, page); + failed |= bad; + } + if (access("/compat-shm", X_OK) == 0) { + pid_t pid = fork(); + + if (!pid) { + char n[32], e[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(e, sizeof(e), "%lu", no4k ? native : 4096UL); + execl("/compat-shm", "/compat-shm", n, e, NULL); + _exit(3); + } + int bad = wait_child(pid); + + printf("%s - SysV AArch32 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, no4k ? native : 4096UL); + failed |= bad; + } + printf("SYSV SHM GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} From d10c63552e54d8586d81338ce53bde925ff79564 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 03:02:55 -0400 Subject: [PATCH 0385/1579] perf: separate userspace ring layout from native backing pages Use the first mapper's process page size for the metadata/data boundary and preserve the requested power-of-two capacity, including rings smaller than one native page. Keep backing allocation and locked-memory accounting in native pages for both page and vmalloc allocation backends. Restrict ring aliases to the same process granule. Preserve AUX byte offsets through mapping and teardown, while retaining native-page AUX sizes required by PMU callbacks. Select the writable metadata leaf by its address rather than a truncated native file index. Add disposable-guest AArch64 sampling and AArch32 mmap fixtures. Tests cover native16/4K and native64/16K/4K, both allocators, wraparound, metadata writes, read-only data, aliases, AUX error routing, and AArch32 no4K native fallback. Feature-disabled arm64 and x86 perf objects compile. Hardware PMU AUX output and Apple hardware/game execution are not qualified. Signed-off-by: DjDeveloperr --- Documentation/arch/arm64/user4k.rst | 34 ++ kernel/events/core.c | 83 +++-- kernel/events/internal.h | 37 +-- kernel/events/ring_buffer.c | 47 ++- tools/testing/selftests/arm64/user4k/Makefile | 9 +- .../arm64/user4k/compat-perf-probe.c | 102 ++++++ .../selftests/arm64/user4k/perf-ring-init.c | 307 ++++++++++++++++++ 7 files changed, 540 insertions(+), 79 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/compat-perf-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/perf-ring-init.c diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index e6fc63736c80c..ff45a42411eb1 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -90,6 +90,40 @@ fixture, the existing native16/4K static/dynamic execution suite, and the Windows memory contract through the existing private Wine/FEX fixture in a VM. Apple hardware boot, GPU execution and game testing remain unqualified. +Perf ring mappings and sampling (2026-10-03) +------------------------------------------- + +Ordinary perf mappings now place the data ring after one process-sized +metadata page. The first mapper fixes that layout for the ring's lifetime; +an attempt to mmap the same ring from a different process granule returns +EINVAL. Same-granule aliases remain supported. This restriction is specific +to perf ring layout and does not prohibit mixed-granule shared memory. + +The requested data capacity and wrap boundary are independent of physical +allocation. Small rings occupy native backing pages but wrap at their exact +requested capacity; larger rings retain native allocation chunks. The +vmalloc backend retains its contiguous output buffer. Both allocation and +locked-memory accounting use native pages, including a full backing page +for metadata. Only the first process page is writable by userspace. + +AUX offsets retain their complete byte position, including the subpage +part, for mapping selection and lifetime management. AUX buffer sizes +remain native-page multiples because PMU setup_aux callbacks consume +native-page arrays. This does not establish hardware AUX tracing support: +the current runtime fixture tests byte-offset routing to a software event's +expected EOPNOTSUPP response, not a hardware PMU's AUX allocation or output. + +The former kernel rejects conventional metadata-only and two-data-page +ring mappings in smaller-granule processes. The fixed kernel passes real +software page-fault sampling on native16/4K and native64/16K/4K, including +records crossing the ring boundary, rings smaller and larger than native +pages, metadata writes, read-only data, alias lifetime and incompatible +mixed-granule alias rejection. Both ordinary and vmalloc backends pass. +Freestanding AArch32 probes pass automatic 4K and capability-masked +native16/native64 fallback, including mmap2's fixed 4K offset units. +Feature-disabled arm64 and x86 perf objects compile. These are disposable +VM tests, not Apple hardware, game or performance qualification. + Current userfaultfd support (2026-10-02) --------------------------------------- diff --git a/kernel/events/core.c b/kernel/events/core.c index 1a23ae67c3cc8..343dc1af5a308 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6812,7 +6812,7 @@ static void perf_event_init_userpage(struct perf_event *event) /* Allow new userspace to detect that bit 0 is deprecated */ userpg->cap_bit0_is_deprecated = 1; userpg->size = offsetof(struct perf_event_mmap_page, __reserved); - userpg->data_offset = PAGE_SIZE; + userpg->data_offset = perf_metadata_size(rb); userpg->data_size = perf_data_size(rb); unlock: @@ -6997,6 +6997,11 @@ typedef void (*mapped_f)(struct perf_event *event, struct mm_struct *mm); f; \ }) +static u64 perf_mmap_offset(struct vm_area_struct *vma) +{ + return ((u64)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); +} + static void perf_mmap_open(struct vm_area_struct *vma) { struct perf_event *event = vma->vm_file->private_data; @@ -7005,7 +7010,7 @@ static void perf_mmap_open(struct vm_area_struct *vma) refcount_inc(&event->mmap_count); refcount_inc(&event->rb->mmap_count); - if (vma->vm_pgoff) + if (perf_mmap_offset(vma)) refcount_inc(&event->rb->aux_mmap_count); if (mapped) @@ -7039,7 +7044,7 @@ static void perf_mmap_close(struct vm_area_struct *vma) * The AUX buffer is strictly a sub-buffer, serialize using aux_mutex * to avoid complications. */ - if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff && + if (rb_has_aux(rb) && perf_mmap_offset(vma) == rb->aux_offset && refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) { /* * Stop all AUX events that are writing to this buffer, @@ -7132,7 +7137,9 @@ static void perf_mmap_close(struct vm_area_struct *vma) static vm_fault_t perf_mmap_pfn_mkwrite(struct vm_fault *vmf) { /* The first page is the user control page, others are read-only. */ - return vmf->pgoff == 0 ? 0 : VM_FAULT_SIGBUS; + return !perf_mmap_offset(vmf->vma) && + vmf->address - vmf->vma->vm_start < mm_page_size(vmf->vma->vm_mm) ? + 0 : VM_FAULT_SIGBUS; } static int perf_mmap_may_split(struct vm_area_struct *vma, unsigned long addr) @@ -7153,7 +7160,8 @@ static const struct vm_operations_struct perf_mmap_vmops = { static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) { - unsigned long nr_pages = vma_pages(vma); + unsigned long va = vma->vm_start; + bool aux = perf_mmap_offset(vma) != 0; int err = 0; unsigned long pagenum; @@ -7197,9 +7205,21 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) * perf_mmap_open() and perf_mmap_close()) so we ensure the lifetime of * this mapping is maintained correctly. */ - for (pagenum = 0; pagenum < nr_pages; pagenum++) { - unsigned long va = vma->vm_start + PAGE_SIZE * pagenum; - struct page *page = perf_mmap_to_page(rb, vma->vm_pgoff + pagenum); + for (pagenum = 0; va < vma->vm_end; pagenum++) { + unsigned long size = min(PAGE_SIZE, vma->vm_end - va); + struct page *page; + + if (aux) { + if (pagenum >= rb->aux_nr_pages) { + err = -EINVAL; + break; + } + page = virt_to_page(rb->aux_pages[pagenum]); + } else { + page = perf_mmap_to_page(rb, pagenum); + if (!pagenum) + size = perf_metadata_size(rb); + } if (page == NULL) { err = -EINVAL; @@ -7207,16 +7227,17 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) } /* Map readonly, perf_mmap_pfn_mkwrite() called on write fault. */ - err = remap_pfn_range(vma, va, page_to_pfn(page), PAGE_SIZE, + err = remap_pfn_range(vma, va, page_to_pfn(page), size, vm_get_page_prot(vma->vm_flags & ~VM_SHARED)); if (err) break; + va += size; } #ifdef CONFIG_MMU /* Clear any partial mappings on error. */ if (err) - zap_vma_range(vma, vma->vm_start, nr_pages * PAGE_SIZE); + zap_vma_range(vma, vma->vm_start, vma->vm_end - vma->vm_start); #endif return err; @@ -7269,7 +7290,7 @@ static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer * { struct user_struct *user = rb->mmap_user; - atomic_long_sub((perf_data_size(rb) >> PAGE_SHIFT) + 1 - rb->mmap_locked, + atomic_long_sub(data_page_nr(rb) + 1 - rb->mmap_locked, &user->locked_vm); atomic64_sub(rb->mmap_locked, &vma->vm_mm->pinned_vm); } @@ -7277,7 +7298,9 @@ static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer * static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, unsigned long nr_pages) { - long extra = 0, user_extra = nr_pages; + unsigned int page_shift = mm_page_shift(vma->vm_mm); + unsigned long data_size; + long extra = 0, user_extra; struct perf_buffer *rb; int rb_flags = 0; @@ -7289,11 +7312,14 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, */ if (nr_pages != 0 && !is_power_of_2(nr_pages)) return -EINVAL; + data_size = nr_pages << page_shift; + /* Physical allocation and locked-memory accounting stay in native pages. */ + user_extra = DIV_ROUND_UP(data_size, PAGE_SIZE) + 1; WARN_ON_ONCE(event->ctx->parent_ctx); if (event->rb) { - if (data_page_nr(event->rb) != nr_pages) + if (perf_data_size(event->rb) != data_size) return -EINVAL; /* @@ -7329,7 +7355,7 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, if (vma->vm_flags & VM_WRITE) rb_flags |= RING_BUFFER_WRITABLE; - rb = rb_alloc(nr_pages, + rb = rb_alloc(data_size, page_shift, event->attr.watermark ? event->attr.wakeup_watermark : 0, event->cpu, rb_flags); @@ -7350,9 +7376,10 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, return 0; } -static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, - unsigned long nr_pages) +static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event) { + unsigned long size = vma->vm_end - vma->vm_start; + unsigned long nr_pages = size >> PAGE_SHIFT; long extra = 0, user_extra = nr_pages; u64 aux_offset, aux_size; struct perf_buffer *rb; @@ -7361,6 +7388,9 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, rb = event->rb; if (!rb) return -EINVAL; + /* PMU setup_aux() consumes whole native pages, independent of mmap layout. */ + if (!IS_ALIGNED(size, PAGE_SIZE)) + return -EINVAL; guard(mutex)(&rb->aux_mutex); @@ -7372,14 +7402,14 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, aux_offset = READ_ONCE(rb->user_page->aux_offset); aux_size = READ_ONCE(rb->user_page->aux_size); - if (aux_offset < perf_data_size(rb) + PAGE_SIZE) + if (aux_offset < perf_data_size(rb) + perf_metadata_size(rb)) return -EINVAL; - if (aux_offset != vma->vm_pgoff << PAGE_SHIFT) + if (aux_offset != perf_mmap_offset(vma)) return -EINVAL; /* already mapped with a different offset */ - if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff) + if (rb_has_aux(rb) && rb->aux_offset != aux_offset) return -EINVAL; if (aux_size != nr_pages * PAGE_SIZE) @@ -7409,7 +7439,7 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, if (vma->vm_flags & VM_WRITE) rb_flags |= RING_BUFFER_WRITABLE; - ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, + ret = rb_alloc_aux(rb, event, aux_offset, nr_pages, event->attr.aux_watermark, rb_flags); if (ret) { refcount_dec(&rb->mmap_count); @@ -7449,12 +7479,13 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) return ret; vma_size = vma->vm_end - vma->vm_start; - nr_pages = vma_size / PAGE_SIZE; + nr_pages = vma_size >> mm_page_shift(vma->vm_mm); if (nr_pages > INT_MAX) return -ENOMEM; - if (vma_size != PAGE_SIZE * nr_pages) + if (vma_size != mm_page_size(vma->vm_mm) * nr_pages || + mm_page_shift(vma->vm_mm) > PAGE_SHIFT) return -EINVAL; scoped_guard (mutex, &event->mmap_mutex) { @@ -7466,10 +7497,14 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) if (event->state <= PERF_EVENT_STATE_REVOKED) return -ENODEV; - if (vma->vm_pgoff == 0) + /* One ring has one immutable metadata/data boundary for all aliases. */ + if (event->rb && event->rb->mmap_page_shift != mm_page_shift(vma->vm_mm)) + return -EINVAL; + + if (!perf_mmap_offset(vma)) ret = perf_mmap_rb(vma, event, nr_pages); else - ret = perf_mmap_aux(vma, event, nr_pages); + ret = perf_mmap_aux(vma, event); if (ret) return ret; diff --git a/kernel/events/internal.h b/kernel/events/internal.h index c03c4f2eea571..5238cfa3399b1 100644 --- a/kernel/events/internal.h +++ b/kernel/events/internal.h @@ -15,9 +15,10 @@ struct perf_buffer { struct rcu_head rcu_head; #ifdef CONFIG_PERF_USE_VMALLOC struct work_struct work; - int page_order; /* allocation order */ #endif int nr_pages; /* nr of data pages */ + unsigned int data_page_shift; /* bytes per output chunk */ + unsigned int mmap_page_shift; /* userspace metadata size */ int overwrite; /* can overwrite itself */ int paused; /* can write into ring buffer */ @@ -44,7 +45,7 @@ struct perf_buffer { long aux_head; unsigned int aux_nest; long aux_wakeup; /* last aux_watermark boundary crossed by aux_head */ - unsigned long aux_pgoff; + u64 aux_offset; int aux_nr_pages; int aux_overwrite; refcount_t aux_mmap_count; @@ -80,10 +81,11 @@ static inline void rb_toggle_paused(struct perf_buffer *rb, bool pause) } extern struct perf_buffer * -rb_alloc(int nr_pages, long watermark, int cpu, int flags); +rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags); extern void perf_event_wakeup(struct perf_event *event); extern int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, - pgoff_t pgoff, int nr_pages, long watermark, int flags); + u64 offset, int nr_pages, long watermark, int flags); extern void rb_free_aux(struct perf_buffer *rb); extern struct perf_buffer *ring_buffer_get(struct perf_event *event); extern void ring_buffer_put(struct perf_buffer *rb); @@ -99,34 +101,19 @@ void perf_event_aux_event(struct perf_event *event, unsigned long head, extern struct page * perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff); -#ifdef CONFIG_PERF_USE_VMALLOC -/* - * Back perf_mmap() with vmalloc memory. - * - * Required for architectures that have d-cache aliasing issues. - */ - -static inline int page_order(struct perf_buffer *rb) -{ - return rb->page_order; -} - -#else - -static inline int page_order(struct perf_buffer *rb) +static inline unsigned long perf_data_size(struct perf_buffer *rb) { - return 0; + return (unsigned long)rb->nr_pages << rb->data_page_shift; } -#endif static inline int data_page_nr(struct perf_buffer *rb) { - return rb->nr_pages << page_order(rb); + return DIV_ROUND_UP(perf_data_size(rb), PAGE_SIZE); } -static inline unsigned long perf_data_size(struct perf_buffer *rb) +static inline unsigned long perf_metadata_size(struct perf_buffer *rb) { - return rb->nr_pages << (PAGE_SHIFT + page_order(rb)); + return 1UL << rb->mmap_page_shift; } static inline unsigned long perf_aux_size(struct perf_buffer *rb) @@ -154,7 +141,7 @@ static inline unsigned long perf_aux_size(struct perf_buffer *rb) handle->page++; \ handle->page &= rb->nr_pages - 1; \ handle->addr = rb->data_pages[handle->page]; \ - handle->size = PAGE_SIZE << page_order(rb); \ + handle->size = 1UL << rb->data_page_shift; \ } \ } while (len && written == size); \ \ diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 9fe92161715e0..6c5de8fba83f2 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -238,7 +238,7 @@ __perf_output_begin(struct perf_output_handle *handle, if (unlikely(head - local_read(&rb->wakeup) > rb->watermark)) local_add(rb->watermark, &rb->wakeup); - page_shift = PAGE_SHIFT + page_order(rb); + page_shift = rb->data_page_shift; handle->page = (offset >> page_shift) & (rb->nr_pages - 1); offset &= (1UL << page_shift) - 1; @@ -677,7 +677,7 @@ static void __rb_free_aux(struct perf_buffer *rb) } int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, - pgoff_t pgoff, int nr_pages, long watermark, int flags) + u64 offset, int nr_pages, long watermark, int flags) { bool overwrite = !(flags & RING_BUFFER_WRITABLE); int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu); @@ -784,7 +784,7 @@ int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, out: if (!ret) - rb->aux_pgoff = pgoff; + rb->aux_offset = offset; else __rb_free_aux(rb); @@ -803,8 +803,8 @@ void rb_free_aux(struct perf_buffer *rb) * Back perf_mmap() with regular GFP_KERNEL-0 pages. */ -static struct page * -__perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) +struct page * +perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) { if (pgoff > rb->nr_pages) return NULL; @@ -835,10 +835,12 @@ static void perf_mmap_free_page(void *addr) __free_page(page); } -struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) +struct perf_buffer *rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags) { struct perf_buffer *rb; unsigned long size; + int nr_pages = DIV_ROUND_UP(data_size, PAGE_SIZE); int i, node; size = sizeof(struct perf_buffer); @@ -863,6 +865,10 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) } rb->nr_pages = nr_pages; + rb->mmap_page_shift = mmap_page_shift; + /* A ring smaller than a native page still wraps at its requested size. */ + rb->data_page_shift = data_size && data_size < PAGE_SIZE ? + ilog2(data_size) : PAGE_SHIFT; ring_buffer_init(rb, watermark, flags); @@ -892,8 +898,8 @@ void rb_free(struct perf_buffer *rb) } #else -static struct page * -__perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) +struct page * +perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) { /* The '>' counts in the user page. */ if (pgoff > data_page_nr(rb)) @@ -917,10 +923,12 @@ void rb_free(struct perf_buffer *rb) schedule_work(&rb->work); } -struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) +struct perf_buffer *rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags) { struct perf_buffer *rb; unsigned long size; + int nr_pages = DIV_ROUND_UP(data_size, PAGE_SIZE); void *all_buf; int node; @@ -940,9 +948,10 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) rb->user_page = all_buf; rb->data_pages[0] = all_buf + PAGE_SIZE; + rb->mmap_page_shift = mmap_page_shift; if (nr_pages) { rb->nr_pages = 1; - rb->page_order = ilog2(nr_pages); + rb->data_page_shift = ilog2(data_size); } ring_buffer_init(rb, watermark, flags); @@ -957,21 +966,3 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) } #endif - -struct page * -perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) -{ - if (rb->aux_nr_pages) { - /* above AUX space */ - if (pgoff > rb->aux_pgoff + rb->aux_nr_pages) - return NULL; - - /* AUX space */ - if (pgoff >= rb->aux_pgoff) { - int aux_pgoff = array_index_nospec(pgoff - rb->aux_pgoff, rb->aux_nr_pages); - return virt_to_page(rb->aux_pages[aux_pgoff]); - } - } - - return __perf_mmap_to_page(rb, pgoff); -} diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 46b21e4b381cc..b3d9bad175542 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -1,11 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 TEST_GEN_PROGS := page-contract TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init -TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init +TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe -TEST_GEN_FILES += compat-shm-probe +TEST_GEN_FILES += compat-shm-probe compat-perf-probe TEST_FILES += compat-script runtime-app.py include ../../lib.mk @@ -113,3 +113,8 @@ $(OUTPUT)/compat-vfio-probe: compat-vfio-probe.c $(OUTPUT)/sysv-shm-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 $(OUTPUT)/compat-shm-probe: compat-shm-probe.c $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# Disposable guests exercise perf layout and software sampling across mm granules. +$(OUTPUT)/perf-ring-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-perf-probe: compat-perf-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ diff --git a/tools/testing/selftests/arm64/user4k/compat-perf-probe.c b/tools/testing/selftests/arm64/user4k/compat-perf-probe.c new file mode 100644 index 0000000000000..3b0271dae520d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-perf-probe.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 perf mmap layout, including native-page fallback. */ +typedef unsigned int u32; +typedef unsigned long long u64; + +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" : "+r"(r0) : "r"(r1), "r"(r2), "r"(r3), + "r"(r4), "r"(r5), "r"(r7) : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 PERF FAIL stage=00\n"; + + if (code) { + text[20] = '0' + code / 10; + text[21] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 PERF PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long pages, unsigned long native, unsigned int count) +{ + static const struct { + u32 type, size; + u64 config, sample_period, sample_type, read_format, flags; + u32 wakeup, bp_type; + u64 config1; + } attr = { .type = 1, .size = 64, .flags = 1 }; + long fd = call(364, (long)&attr, 0, -1, -1, 0, 0); + unsigned long length = (1 + count) * pages; + long raw; + volatile u64 *metadata; + + if (fd < 0) + finish(20); + raw = call(192, 0, length, 3, 1, fd, 0); + if ((unsigned long)raw >= 0xfffff001UL) + finish(21); + metadata = (void *)(raw + 1024); + if (metadata[2] != pages || metadata[3] != count * pages) + finish(22); + metadata[1] = 0; + metadata[6] = length; + metadata[7] = native; + /* mmap2 always takes 4K units, including the no4K compat fallback. */ + if (call(192, 0, native, 1, 1, fd, length / 4096) != -95) + finish(23); + if (call(91, raw, length, 0, 0, 0, 0)) + finish(24); + if (call(6, fd, 0, 0, 0, 0, 0)) + finish(25); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + unsigned long native = number(argv[1]); + + mapping(pages, native, 0); + mapping(pages, native, 1); + mapping(pages, native, 2); + mapping(pages, native, 2 * native / pages); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/perf-ring-init.c b/tools/testing/selftests/arm64/user4k/perf-ring-init.c new file mode 100644 index 0000000000000..e133e4242e651 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/perf-ring-init.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: perf ring layout and sampling with per-mm page sizes. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int ok, const char *expression, int line) +{ + if (!ok) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, errno); + exit(1); + } +} + +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int no4k; + +static int wait_child(pid_t child) +{ + int status; + pid_t got; + + if (child < 0) + return 1; + do { + got = waitpid(child, &status, 0); + } while (got < 0 && errno == EINTR); + return got != child || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static int event_open(int sampling) +{ + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .size = sizeof(attr), + .config = PERF_COUNT_SW_PAGE_FAULTS, + .disabled = 1, + .sample_period = sampling ? 1 : 0, + .sample_type = PERF_SAMPLE_TID | PERF_SAMPLE_TIME | + PERF_SAMPLE_ADDR | PERF_SAMPLE_PERIOD, + }; + int fd = syscall(SYS_perf_event_open, &attr, 0, -1, -1, 0); + + CHECK(fd >= 0); + return fd; +} + +static sigjmp_buf fault_return; +static volatile sig_atomic_t fault_signal; + +static void fault_handler(int signal) +{ + fault_signal = signal; + siglongjmp(fault_return, 1); +} + +static void readonly_data(void *address) +{ + struct sigaction action = { .sa_handler = fault_handler }, previous; + + CHECK(!sigemptyset(&action.sa_mask)); + CHECK(!sigaction(SIGBUS, &action, &previous)); + fault_signal = 0; + if (!sigsetjmp(fault_return, 1)) + *(volatile unsigned char *)address = 0; + CHECK(fault_signal == SIGBUS); + CHECK(!sigaction(SIGBUS, &previous, NULL)); +} + +static void copy_record(void *destination, const unsigned char *ring, + unsigned long capacity, uint64_t position, size_t size) +{ + unsigned long offset = position & (capacity - 1); + size_t first = capacity - offset; + + if (first > size) + first = size; + memcpy(destination, ring + offset, first); + memcpy((unsigned char *)destination + first, ring, size - first); +} + +static void sampling(unsigned long pages) +{ + unsigned long page = getauxval(AT_PAGESZ), capacity = pages * page; + unsigned long length = page + capacity; + int fd = event_open(1); + struct perf_event_mmap_page *map = mmap(NULL, length, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + unsigned char *region, *ring; + uint64_t tail = 0, samples = 0, wrapped = 0, region_samples = 0; + + CHECK(map != MAP_FAILED); + CHECK(map->data_offset == page && map->data_size == capacity); + map->data_tail = 0; + ring = (unsigned char *)map + map->data_offset; + for (unsigned long i = 0; i < capacity; i += page) + CHECK(ring[i] == 0); + readonly_data(ring); + region = mmap(NULL, 16 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(region != MAP_FAILED); + for (unsigned int batch = 0; batch < 10000 && tail < 3 * capacity; batch++) { + CHECK(!madvise(region, 16 * page, MADV_DONTNEED)); + CHECK(!ioctl(fd, PERF_EVENT_IOC_ENABLE, 0)); + for (unsigned int i = 0; i < 16; i++) + *(volatile unsigned char *)(region + i * page) = 1; + CHECK(!ioctl(fd, PERF_EVENT_IOC_DISABLE, 0)); + uint64_t head = __atomic_load_n(&map->data_head, __ATOMIC_ACQUIRE); + + CHECK(head >= tail && head - tail <= capacity); + while (tail < head) { + struct perf_event_header header; + struct { + struct perf_event_header header; + uint32_t pid, tid; + uint64_t time, address, period; + } record; + + copy_record(&header, ring, capacity, tail, sizeof(header)); + CHECK(header.size >= sizeof(header) && header.size <= head - tail); + if (header.type == PERF_RECORD_SAMPLE) { + CHECK(header.size == sizeof(record)); + copy_record(&record, ring, capacity, tail, sizeof(record)); + CHECK(record.pid == (uint32_t)getpid()); + CHECK(record.tid == (uint32_t)getpid()); + if (record.address >= (uintptr_t)region && + record.address < (uintptr_t)region + 16 * page) + region_samples++; + CHECK(record.period == 1); + samples++; + if ((tail & (capacity - 1)) + header.size > capacity) + wrapped++; + } else { + CHECK(header.type == PERF_RECORD_THROTTLE || + header.type == PERF_RECORD_UNTHROTTLE); + } + tail += header.size; + } + __atomic_store_n(&map->data_tail, tail, __ATOMIC_RELEASE); + } + CHECK(tail >= 3 * capacity && samples && wrapped); + /* Library/stack faults may also occur while a batch is enabled. */ + CHECK(region_samples > samples * 9 / 10); + CHECK(!munmap(region, 16 * page)); + CHECK(!close(fd)); + CHECK(map->data_head == tail); + CHECK(!munmap(map, length)); + printf("ok - perf page=%lu ring=%lu samples=%llu region=%llu wrapped-records=%llu\n", + page, capacity, (unsigned long long)samples, + (unsigned long long)region_samples, (unsigned long long)wrapped); +} + +static void layout(unsigned long native) +{ + unsigned long page = getauxval(AT_PAGESZ); + int fd = event_open(0); + struct perf_event_mmap_page *map = mmap(NULL, page, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + + CHECK(map != MAP_FAILED); + CHECK(map->data_offset == page && map->data_size == 0); + map->data_tail = 0; + CHECK(!munmap(map, page)); + CHECK(!close(fd)); + fd = event_open(0); + errno = 0; + CHECK(mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0) == MAP_FAILED); + CHECK(errno == EINVAL); + map = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + CHECK(map != MAP_FAILED); + struct perf_event_mmap_page *alias = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + CHECK(alias != MAP_FAILED); + map->data_tail = 7; + CHECK(alias->data_tail == 7); + map->data_tail = 0; + CHECK(!munmap(alias, 3 * page)); + /* A software event has no AUX provider. Check byte-exact offset routing. */ + map->aux_offset = 3 * page; + map->aux_size = native; + errno = 0; + CHECK(mmap(NULL, native, PROT_READ, MAP_SHARED, fd, map->aux_offset) == MAP_FAILED); + CHECK(errno == EOPNOTSUPP); + unsigned long sizes[] = {4096, 16384, 65536}; + + for (unsigned int i = 0; i < 3; i++) { + if (sizes[i] > native || sizes[i] == page || (no4k && sizes[i] == 4096)) + continue; + pid_t child = fork(); + + if (!child) { + char f[32], s[32]; + + snprintf(f, sizeof(f), "%d", fd); + snprintf(s, sizeof(s), "%lu", sizes[i]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--alias", f, s, NULL); + _exit(3); + } + CHECK(!wait_child(child)); + } + CHECK(map->data_size == 2 * page && map->data_offset == page); + CHECK(!munmap(map, 3 * page)); + CHECK(!close(fd)); + printf("ok - perf page=%lu metadata, invalid size, aliases, AUX routing\n", page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0, compat_only = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + no4k = getenv("PERF_TEST_NO4K") != NULL; + if (argc == 4 && !strcmp(argv[1], "--alias")) { + unsigned long page = strtoul(argv[3], NULL, 10); + + CHECK(getauxval(AT_PAGESZ) == page); + errno = 0; + CHECK(mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, + MAP_SHARED, atoi(argv[2]), 0) == MAP_FAILED); + CHECK(errno == EINVAL); + return 0; + } + if (argc == 4 && !strcmp(argv[1], "--case")) { + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + layout(native); + sampling(1); + sampling(2); + sampling(2 * native / getauxval(AT_PAGESZ)); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--no4k")) { + no4k = 1; + CHECK(!setenv("PERF_TEST_NO4K", "1", 1)); + } + if (!strcmp(argv[i], "--compat-only")) + compat_only = 1; + } + unsigned long sizes[] = {4096, 16384, 65536}; + + for (unsigned int i = 0; i < 3; i++) { + if (compat_only || sizes[i] > native || (no4k && sizes[i] == 4096)) + continue; + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--case", n, s, NULL); + _exit(3); + } + int bad = wait_child(child); + + printf("%s - perf native=%lu process=%lu\n", bad ? "not ok" : "ok", native, sizes[i]); + failed |= bad; + } + if (compat_only) + CHECK(access("/compat-perf", X_OK) == 0); + if (access("/compat-perf", X_OK) == 0) { + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", no4k ? native : 4096UL); + execl("/compat-perf", "/compat-perf", n, s, NULL); + _exit(3); + } + int bad = wait_child(child); + + printf("%s - perf AArch32 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, no4k ? native : 4096UL); + failed |= bad; + } + printf("PERF RING GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} From 4f1e23a24e5f248662a936f29349ad8fb04d8e4d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 03:38:05 -0400 Subject: [PATCH 0386/1579] mm: cache userspace granule during protection walks Decode the immutable mm granule once per PTE protection walk and pass it to the batching selector. Keep the selector inline so native small-folio updates avoid an out-of-line helper. Preserve the existing batch, permission and TLB rules. Paired non-debug TCG runs show median mprotect reductions of 5.8% for native16 and 2.5% for 4K processes against the preceding implementation. These are emulator measurements; other timings vary and do not establish hardware gains or a universal speedup. Existing mprotect/COW/accounting/migration KUnit cases and current-source EL0 userfaultfd/SysV fixtures pass on native16/native64 KASAN kernels. Feature-disabled arm64 and x86 protection objects compile. Record the measurement scope and limits in the mixed-granule design notes. Signed-off-by: DjDeveloperr --- Documentation/arch/arm64/user4k.rst | 22 ++++++++++++++++++++++ mm/mprotect.c | 22 ++++++++++++++-------- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index ff45a42411eb1..f745c452cb354 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -124,6 +124,28 @@ native16/native64 fallback, including mmap2's fixed 4K offset units. Feature-disabled arm64 and x86 perf objects compile. These are disposable VM tests, not Apple hardware, game or performance qualification. +Protection-walk geometry cost (2026-10-03) +----------------------------------------- + +The PTE protection walk now decodes its immutable mm granule once and +passes that value to the inlined batching selector. This removes repeated +granule decoding and restores an inline small-folio path; the batching +rules, permission checks and TLB operations remain the same. + +Matched non-debug, native16 TCG kernels before and after this change used +the same configuration and benchmark executable. Three paired boots with +five retained trials each measured an 8MiB protection change about 5.8% +faster for native16 processes and 2.5% faster for 4K processes. These are +emulator results, not Apple hardware or game performance. Other timings +varied, including a roughly 1.4% increase in the 4K fork/COW measurement; +the result does not establish a universal speedup or zero native overhead. + +The existing protection, COW-batching, accounting and migration KUnit +cases pass on native16 and native64 KASAN/lockdep kernels. Current-source +EL0 userfaultfd and SysV fixtures also pass, covering native16/4K and +native64/16K/4K permissions and AArch32 SysV execution. Feature-disabled +arm64 and x86 protection objects also compile. + Current userfaultfd support (2026-10-02) --------------------------------------- diff --git a/mm/mprotect.c b/mm/mprotect.c index 8202f2ebfbf6d..6f1a94026079d 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -123,15 +123,16 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, } -static int mprotect_folio_pte_batch(struct vm_area_struct *vma, struct folio *folio, pte_t *ptep, - pte_t pte, int max_nr_ptes, fpb_t flags) +static __always_inline int mprotect_folio_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t pte, int max_nr_ptes, + fpb_t flags, unsigned int page_shift) { /* No underlying folio, so cannot batch */ if (!folio) return 1; #ifdef CONFIG_MM_SUBPAGE - if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + if (page_shift != PAGE_SHIFT) return folio_subpage_pte_batch(vma, folio, ptep, pte, max_nr_ptes, flags); #endif @@ -369,6 +370,9 @@ static long change_pte_range(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, pgprot_t newprot, unsigned long cp_flags) { + /* The mm's mapping granule cannot change during the walk. */ + const unsigned int page_shift = mm_page_shift(vma->vm_mm); + const unsigned long page_size = 1UL << page_shift; pte_t *pte, oldpte; spinlock_t *ptl; long pages = 0; @@ -377,7 +381,7 @@ static long change_pte_range(struct mmu_gather *tlb, bool uffd_wp = cp_flags & MM_CP_UFFD_WP; int nr_ptes; - tlb_change_page_size(tlb, mm_page_size(vma->vm_mm)); + tlb_change_page_size(tlb, page_size); pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); if (!pte) return -EAGAIN; @@ -392,7 +396,7 @@ static long change_pte_range(struct mmu_gather *tlb, oldpte = ptep_get(pte); if (pte_present(oldpte)) { const fpb_t flags = FPB_RESPECT_SOFT_DIRTY | FPB_RESPECT_WRITE; - int max_nr_ptes = (end - addr) >> mm_page_shift(vma->vm_mm); + int max_nr_ptes = (end - addr) >> page_shift; struct folio *folio = NULL; struct page *page; @@ -414,11 +418,13 @@ static long change_pte_range(struct mmu_gather *tlb, /* determine batch to skip */ nr_ptes = mprotect_folio_pte_batch(vma, folio, - pte, oldpte, max_nr_ptes, /* flags = */ 0); + pte, oldpte, max_nr_ptes, /* flags = */ 0, + page_shift); continue; } - nr_ptes = mprotect_folio_pte_batch(vma, folio, pte, oldpte, max_nr_ptes, flags); + nr_ptes = mprotect_folio_pte_batch(vma, folio, pte, oldpte, + max_nr_ptes, flags, page_shift); /* * Optimize for the small-folio common case by @@ -457,7 +463,7 @@ static long change_pte_range(struct mmu_gather *tlb, } else { pages += change_softleaf_pte(vma, addr, pte, oldpte, cp_flags); } - } while (pte += nr_ptes, addr += nr_ptes * mm_page_size(vma->vm_mm), addr != end); + } while (pte += nr_ptes, addr += nr_ptes * page_size, addr != end); lazy_mmu_mode_disable(); pte_unmap_unlock(pte - 1, ptl); From 363166515c05fb099ad8d5752d99432244e40e60 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:06:51 -0500 Subject: [PATCH 0387/1579] accel/ane: Map a buffer object with one DART TLB sync BO_INIT maps a buffer object one page at a time with iommu_map(). Each call syncs the TLB of every DART in the domain, so a buffer of N pages costs 3 * N TLB invalidates on the three ANE DARTs. A 458 MB buffer has 27,955 pages and needs 83,865 invalidates. Map the buffer with one iommu_map_sgtable() instead. iommu_map_sg() writes the PTEs of each physically contiguous run and syncs the TLBs once for the whole range. Keep the old per-page path behind the new module parameter map_batch (default 1; 0 selects the old path). BOs mapped either way unmap the same way. Keep the stray PTE recovery on both paths. iommu_map_sg() unmaps what it mapped before it returns an error, so after -EEXIST every valid PTE left in the reserved range is a stray: clear the range page by page and try once more. A short return would give a BO whose tail has no PTEs, so clear the range page by page and fail BO_INIT with -EIO. The page by page clear also serves the stale-mapping purge in probe. Measured with map_batch=1 against map_batch=0, with bit-exact program outputs and exact job counts: M1 (T8103) laptop: BO_INIT 179.85 ms -> 22.77 ms, cold open 165 to 173 ms faster M1 Max (T6001) laptop: BO_INIT 232.7 ms -> 24.7 ms, open 787.9 ms -> 574.3 ms A stress run was clean on both. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_drv.c | 197 +++++++++++++++++++++++++----------- 1 file changed, 137 insertions(+), 60 deletions(-) diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 2147d6a5e7d83..70f0d9bd9b365 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -79,6 +80,18 @@ module_param(autosuspend_ms, int, 0444); MODULE_PARM_DESC(autosuspend_ms, "Power the ANE off this many ms after its last use (default 1500; 0 = keep it powered while the driver is bound)"); +/* + * 1 (default): BO_INIT maps a buffer object with one iommu_map_sg(), so + * apple-dart invalidates the TLB of each DART once for each BO. 0: one + * iommu_map() and one TLB invalidate on each DART for each page (a + * 458 MB buffer: 27,955 pages x 3 DARTs = 83,865 invalidates). Read + * once for each BO_INIT; BOs mapped either way unmap the same way. + */ +static bool map_batch = true; +module_param(map_batch, bool, 0644); +MODULE_PARM_DESC(map_batch, + "BO_INIT DART mapping: 1 = one TLB sync for each buffer object (default); 0 = one TLB sync for each page"); + #define CMD_BUF_BDX 0 #define KRN_BUF_BDX 1 @@ -102,63 +115,44 @@ static struct ane_bo *bo_lookup(struct drm_file *file, u32 handle) return to_bo(gem); } -/* - * Buffers are mapped and unmapped only through the kernel-owned IOMMU - * domain the device was attached to by its providers. apple-dart - * programs every DART in the device's "iommus" list (TTBRs, stream - * setup, invalidation and fault IRQs are all provider-owned); - * iommu_unmap() flushes through the provider before returning, so pages - * may be reclaimed once it succeeds. - */ -static int ane_iommu_map_pages(struct ane_device *ane, struct ane_bo *bo) +/* Unmap every valid PTE in [start, end) and return how many there were. */ +static unsigned long ane_iommu_clear_range(struct ane_device *ane, u64 start, + u64 end) { - int err; + unsigned long cleared = 0; - lockdep_assert_held(&ane->engine_lock); - if (bo->mm) - return -EBUSY; - - bo->mm = kzalloc_obj(*bo->mm); - if (!bo->mm) - return -ENOMEM; - - mutex_lock(&ane->iommu_lock); - - /* reserve area from ANE address space */ - err = drm_mm_insert_node_generic(&ane->mm, bo->mm, - bo->npages << ane->shift, - 1UL << ane->shift, 0, 0); - if (err < 0) { - dev_err(ane->dev, "out of ANE space: %d\n", err); - goto unlock; + for (u64 iova = start; iova < end; iova += 1UL << ane->shift) { + if (!iommu_iova_to_phys(ane->domain, iova)) + continue; + iommu_unmap(ane->domain, iova, 1UL << ane->shift); + cleared++; } + return cleared; +} - bo->iova = bo->mm->start; +/* + * Every PTE inside a reserved drm_mm node must be ours-or-absent: + * scratch drains reserve before mapping and wedge-preserved nodes stay + * inserted. A VALID PTE in a just-reserved range is therefore a stray + * from a torn-down session or a lost unmap. Both map paths clear it once + * and retry instead of poisoning every later BO_INIT until reload + * (m1-test-host 2026-09-25: dart_init_pte -EEXIST at 0x4000 for the rest + * of the session). + */ +static int ane_iommu_map_each(struct ane_device *ane, struct ane_bo *bo, + int prot) +{ + int err = 0; - /* map into ANE address space */ for (u32 i = 0; i < bo->npages; i++) { dma_addr_t iova = bo->iova + (i << ane->shift); - int prot = IOMMU_READ | IOMMU_WRITE; bool healed = false; - int prot_cache = map_mode & 1; retry: - if (prot_cache) - prot |= IOMMU_CACHE; err = iommu_map(ane->domain, iova, page_to_phys(bo->pages[i]), 1UL << ane->shift, prot, GFP_KERNEL); if (err < 0) { dev_err(ane->dev, "iommu_map failed at 0x%llx", iova); - /* Every PTE inside a reserved drm_mm node must be - * ours-or-absent: scratch drains reserve before - * mapping and wedge-preserved nodes stay inserted. - * A VALID PTE in a just-reserved range is therefore - * a stray from a torn-down session or a lost - * unmap. Clear it once and retry instead of - * poisoning every later BO_INIT until reload - * (m1-test-host 2026-09-25: dart_init_pte -EEXIST at - * 0x4000 for the rest of the session). - */ if (err == -EEXIST && !healed && iommu_iova_to_phys(ane->domain, iova)) { dev_warn(ane->dev, @@ -182,26 +176,115 @@ static int ane_iommu_map_pages(struct ane_device *ane, struct ane_bo *bo) bo->iova + (i << ane->shift), unmapped); } - drm_mm_remove_node(bo->mm); - bo->iova = 0; break; } } + return err; +} - mutex_unlock(&ane->iommu_lock); +/* + * iommu_map_sg() writes the PTEs of each physically contiguous run and + * syncs the DART TLBs once for the whole range. On failure it has + * already unmapped everything it mapped, so after an -EEXIST every valid + * PTE left in the range is a stray. + */ +static int ane_iommu_map_batch(struct ane_device *ane, struct ane_bo *bo, + int prot) +{ + size_t size = (size_t)bo->npages << ane->shift; + struct sg_table sgt; + unsigned long strays; + ssize_t mapped; + int err; - if (err < 0) { - kfree(bo->mm); - bo->mm = NULL; + err = sg_alloc_table_from_pages(&sgt, bo->pages, bo->npages, 0, size, + GFP_KERNEL); + if (err) return err; + + mapped = iommu_map_sgtable(ane->domain, bo->iova, &sgt, prot); + if (mapped == -EEXIST) { + strays = ane_iommu_clear_range(ane, bo->iova, bo->iova + size); + if (strays) { + dev_warn(ane->dev, + "cleared %lu stray DART PTE(s) in %#llx+%#zx (no node owns them)\n", + strays, bo->iova, size); + mapped = iommu_map_sgtable(ane->domain, bo->iova, &sgt, + prot); + } } + sg_free_table(&sgt); + /* + * A short map would hand out a BO whose tail has no PTEs. The + * mapped part can have holes, so clear page by page, not with one + * range unmap that stops at the first hole. + */ + if (mapped >= 0 && (size_t)mapped != size) { + ane_iommu_clear_range(ane, bo->iova, bo->iova + size); + mapped = -EIO; + } + + if (mapped < 0) { + dev_err(ane->dev, "iommu_map_sg failed at %#llx+%#zx: %zd\n", + bo->iova, size, mapped); + return mapped; + } return 0; +} + +/* + * Buffers are mapped and unmapped only through the kernel-owned IOMMU + * domain the device was attached to by its providers. apple-dart + * programs every DART in the device's "iommus" list (TTBRs, stream + * setup, invalidation and fault IRQs are all provider-owned); + * iommu_unmap() flushes through the provider before returning, so pages + * may be reclaimed once it succeeds. + */ +static int ane_iommu_map_pages(struct ane_device *ane, struct ane_bo *bo) +{ + int prot = IOMMU_READ | IOMMU_WRITE; + int err; + + lockdep_assert_held(&ane->engine_lock); + if (bo->mm) + return -EBUSY; + + if (map_mode & 1) + prot |= IOMMU_CACHE; + + bo->mm = kzalloc_obj(*bo->mm); + if (!bo->mm) + return -ENOMEM; + + mutex_lock(&ane->iommu_lock); + + /* reserve area from ANE address space */ + err = drm_mm_insert_node_generic(&ane->mm, bo->mm, + bo->npages << ane->shift, + 1UL << ane->shift, 0, 0); + if (err < 0) { + dev_err(ane->dev, "out of ANE space: %d\n", err); + goto unlock; + } + + bo->iova = bo->mm->start; + + if (READ_ONCE(map_batch)) + err = ane_iommu_map_batch(ane, bo, prot); + else + err = ane_iommu_map_each(ane, bo, prot); + if (err < 0) { + drm_mm_remove_node(bo->mm); + bo->iova = 0; + } unlock: mutex_unlock(&ane->iommu_lock); - kfree(bo->mm); - bo->mm = NULL; + if (err < 0) { + kfree(bo->mm); + bo->mm = NULL; + } return err; } @@ -943,17 +1026,11 @@ static int ane_iommu_domain_init(struct ane_device *ane) static void ane_iommu_purge_stale(struct ane_device *ane) { struct drm_mm_node *hole; - u64 start, end, iova; + u64 start, end; unsigned long stale = 0; - drm_mm_for_each_hole(hole, &ane->mm, start, end) { - for (iova = start; iova < end; iova += 1UL << ane->shift) { - if (!iommu_iova_to_phys(ane->domain, iova)) - continue; - iommu_unmap(ane->domain, iova, 1UL << ane->shift); - stale++; - } - } + drm_mm_for_each_hole(hole, &ane->mm, start, end) + stale += ane_iommu_clear_range(ane, start, end); if (stale) dev_warn(ane->dev, "cleared %lu stale DART mappings from a previous instance\n", From 85837084c6c71373e9460091754019600445f6a1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:07:15 -0400 Subject: [PATCH 0388/1579] selftests/arm64: cover mixed-granule AIO ring relocation Exercise small and large legacy AIO rings from native16/4K and native64/16K/4K AArch64 processes. Move each alternative ring to a process-aligned, non-native-aligned address with a completion queued, then verify buffered memfd reads/writes, eventfd delivery, teardown and adjacent mappings. Keep the fixture a disposable PID1 extended target. The committed fixture builds through the selftest Makefile and passes on the Apple-enabled native16 mm4 Image and the native64 debug Image. Record this coverage and the separately identified NUMA policy review gap. No kernel implementation or Apple bundle changes are required. Signed-off-by: DjDeveloperr --- Documentation/arch/arm64/user4k.rst | 20 ++ tools/testing/selftests/arm64/user4k/Makefile | 3 + .../selftests/arm64/user4k/aio-ring-init.c | 172 ++++++++++++++++++ 3 files changed, 195 insertions(+) create mode 100644 tools/testing/selftests/arm64/user4k/aio-ring-init.c diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index f745c452cb354..38dd9141b1150 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -146,6 +146,21 @@ EL0 userfaultfd and SysV fixtures also pass, covering native16/4K and native64/16K/4K permissions and AArch32 SysV execution. Feature-disabled arm64 and x86 protection objects also compile. +Legacy AIO ring relocation (2026-10-03) +-------------------------------------- + +The AIO completion ring retains native-page backing while its virtual +address follows the process granule. The disposable ``aio-ring-init`` +fixture checks small and large rings in AArch64 native16/4K and +native64/16K/4K processes. Smaller-granule tasks move the ring to an address +that is aligned to their own granule but not to the native kernel page. + +Buffered memfd writes complete through the relocated ring, including a +completion queued before the move. Subsequent reads verify exact bytes +across process-page boundaries. Eventfd notification, context destruction, +removal of the ring mapping and adjacent guard preservation also pass. +The Apple-enabled mm4 Image passes the native16/4K fixture in a VM. + Current userfaultfd support (2026-10-02) --------------------------------------- @@ -243,6 +258,11 @@ or complete NCE compatibility claim is made. Milestone sections below retain their historical status statements; the latest exec milestone supersedes earlier statements that EL0 activation was absent. +The current native16/native64 VM profiles and Apple candidate have NUMA +disabled. NUMA policy integration requires further work: policy range +alignment and PTE loops in ``mm/mempolicy.c`` still use native-page units. +NUMA-enabled mixed-granule operation is not covered by these results. + The expanded objective is concurrent 4K, 16K and 64K userspace granules on arm64, with the kernel/physical allocator granule independently fixed for the boot. AArch64 defaults to the native granule (16K on the current Apple target); diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index b3d9bad175542..7058daa50ce25 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -2,6 +2,7 @@ TEST_GEN_PROGS := page-contract TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init +TEST_GEN_PROGS_EXTENDED += aio-ring-init CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe @@ -118,3 +119,5 @@ $(OUTPUT)/compat-shm-probe: compat-shm-probe.c $(OUTPUT)/perf-ring-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 $(OUTPUT)/compat-perf-probe: compat-perf-probe.c $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/aio-ring-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 diff --git a/tools/testing/selftests/arm64/user4k/aio-ring-init.c b/tools/testing/selftests/arm64/user4k/aio-ring-init.c new file mode 100644 index 0000000000000..c64c5d3881e98 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/aio-ring-init.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: legacy AIO ring relocation with per-process page sizes. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct aio_ring_header { + unsigned int id, nr, head, tail; + unsigned int magic, compat_features, incompat_features, header_length; +}; + +static void check(int good, const char *expression, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int wait_child(pid_t child) +{ + int status; + pid_t got; + + if (child < 0) + return 1; + do { + got = waitpid(child, &status, 0); + } while (got < 0 && errno == EINTR); + return got != child || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static void complete(aio_context_t context, struct iocb *request, + unsigned long bytes, int event_fd) +{ + struct io_event event; + struct timespec timeout = { .tv_sec = 5 }; + uint64_t notifications; + + CHECK(syscall(SYS_io_getevents, context, 1, 1, &event, &timeout) == 1); + CHECK(event.data == request->aio_data && event.obj == (uintptr_t)request); + CHECK(event.res == (long long)bytes && event.res2 == 0); + CHECK(read(event_fd, ¬ifications, sizeof(notifications)) == sizeof(notifications)); + CHECK(notifications == 1); +} + +static void exercise(unsigned long native, unsigned int requested) +{ + unsigned long page = getauxval(AT_PAGESZ), bytes = 2 * page + 37; + aio_context_t context = 0; + int file = memfd_create("aio-granule", 0); + int event_fd = eventfd(0, EFD_NONBLOCK); + unsigned char *source, *destination, *reserve, *target; + struct aio_ring_header *header; + unsigned long ring_size; + struct iocb request = {0}, *requests[] = { &request }; + unsigned char residency; + + CHECK(file >= 0 && event_fd >= 0); + CHECK(!ftruncate(file, 4 * page)); + source = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + destination = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(source != MAP_FAILED && destination != MAP_FAILED); + for (unsigned long i = 0; i < bytes; i++) + source[page - 19 + i] = i * 17 + i / page; + CHECK(!syscall(SYS_io_setup, requested, &context)); + header = (void *)(uintptr_t)context; + CHECK(header->magic == 0xa10a10a1 && header->header_length == sizeof(*header)); + CHECK(header->nr >= requested); + ring_size = header->header_length + header->nr * sizeof(struct io_event); + CHECK(ring_size && !(ring_size % native)); + reserve = mmap(NULL, ring_size + 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + target = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + target += native + (page < native ? page : 0); + target[-1] = 0x51; + target[ring_size] = 0x72; + CHECK(!munmap(target, ring_size)); + request.aio_data = 0x12345678; + request.aio_lio_opcode = IOCB_CMD_PWRITE; + request.aio_fildes = file; + request.aio_buf = (uintptr_t)(source + page - 19); + request.aio_nbytes = bytes; + request.aio_offset = 7; + request.aio_flags = IOCB_FLAG_RESFD; + request.aio_resfd = event_fd; + CHECK(syscall(SYS_io_submit, context, 1, requests) == 1); + /* Retain a queued completion while relocating the ring. */ + CHECK(mremap(header, ring_size, ring_size, + MREMAP_MAYMOVE | MREMAP_FIXED, target) == target); + context = (uintptr_t)target; + header = (void *)target; + CHECK(header->magic == 0xa10a10a1); + CHECK(target[-1] == 0x51 && target[ring_size] == 0x72); + complete(context, &request, bytes, event_fd); + request.aio_data = 0x87654321; + request.aio_lio_opcode = IOCB_CMD_PREAD; + request.aio_buf = (uintptr_t)(destination + page - 11); + CHECK(syscall(SYS_io_submit, context, 1, requests) == 1); + complete(context, &request, bytes, event_fd); + CHECK(!memcmp(source + page - 19, destination + page - 11, bytes)); + CHECK(!syscall(SYS_io_destroy, context)); + errno = 0; + CHECK(mincore(target, page, &residency) == -1 && errno == ENOMEM); + CHECK(target[-1] == 0x51 && target[ring_size] == 0x72); + CHECK(!munmap(reserve, ring_size + 4 * native)); + CHECK(!munmap(source, 4 * page) && !munmap(destination, 4 * page)); + CHECK(!close(file) && !close(event_fd)); + printf("ok - AIO page=%lu requested=%u ring=%lu relocation, I/O, eventfd and teardown\n", + page, requested, ring_size); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--case")) { + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + exercise(native, 16); + exercise(native, 4096); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + unsigned long sizes[] = {4096, 16384, 65536}; + + for (unsigned int i = 0; i < 3; i++) { + if (sizes[i] > native) + continue; + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", n, s, NULL); + _exit(2); + } + int bad = wait_child(child); + + printf("%s - AIO native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, sizes[i]); + failed |= bad; + } + printf("AIO GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} From 95bba130fd05e5fd22406b3181b23bf8b4151b47 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:20:12 -0400 Subject: [PATCH 0389/1579] docs/arm64: record user-reported mixed-page gaming success Record the user gaming result separately from automated VM validation. The report does not identify the exact device, build or game workload. Update the earlier blanket absence-of-game-testing statements accordingly; leave the documented ABI limitations and kernel implementation unchanged. Signed-off-by: DjDeveloperr --- Documentation/arch/arm64/user4k.rst | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index 38dd9141b1150..1d4c535b1a37e 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -22,6 +22,14 @@ native ABI are tested. Real mixed-size processes share memfd/futex state and use process_vm_readv/writev in both directions. See the latest milestone below for exact evidence and limits; this is not yet a complete or battle-tested ABI. +User-reported gaming result (2026-10-03) +--------------------------------------- + +The user reports having tested this work for gaming and that it "works +super well". The report did not specify the device, kernel build, game +titles or measured frame rates. This adds user-reported gameplay evidence +to the reproducible VM results below; it is not an automated benchmark. + Apple-enabled build and process launch (2026-10-03) -------------------------------------------------- @@ -49,8 +57,8 @@ large argument transitions, proc/memory interfaces, and mixed 16K/4K shared futex/COW and process_vm copies. The actual launcher passed native/4K/native transitions, and FEX passed the x86 memory/protection/COW/JIT/re-exec contract with its AArch64 processes reporting 4096-byte pages. These are VM results; -this candidate has not been booted on an Apple Mac, submitted Apple GPU -commands, or run a Windows game. +these automated tests do not cover Apple hardware boot or GPU command +submission. The subsequent user-reported gaming result is recorded above. The Apple profile exposed a link-layout failure: the mixed-granule zero page requires 64K BSS alignment while KVM aligned ``__bss_start`` only to the @@ -88,7 +96,7 @@ The fixture requires CONFIG_SYSVIPC and runs only in a disposable guest. The Apple-enabled ``7.1.12-aurora-mixed-mm2`` Image also passes the SysV fixture, the existing native16/4K static/dynamic execution suite, and the Windows memory contract through the existing private Wine/FEX fixture in a VM. -Apple hardware boot, GPU execution and game testing remain unqualified. +These are VM tests; the subsequent user-reported gaming result is recorded above. Perf ring mappings and sampling (2026-10-03) ------------------------------------------- From 0a8e97083bdc7af1e7158cbedab60153ecab6839 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:48:01 -0400 Subject: [PATCH 0390/1579] mm: honor userspace granules in NUMA policy operations Smaller-granule processes reject valid mbind/home-node ranges and cannot use native-page GUP for exact node lookup. Walk policy PTEs in process-page units, batch fine leaves within their physical owner, and use typed fragment references for get_mempolicy node queries. Physical migration remains native. Keep shared shmem policy updates in native file-page units, rejecting partial units before VMA modification. Include subpage offsets in shared policy lookup and native-unit interleave indexing. Add disposable two-node AArch64 and freestanding AArch32 tests. Native16/4K and native64/16K/4K cases pass, including AArch32 native-size fallback when 4K capability is masked. Tests cover placement, interleave, strict checks, MOVE_ALL and shared aliases. Document the isolated 64K boot-test exclusion and remaining automatic-balancing limits. Feature-off object builds pass. Signed-off-by: DjDeveloperr --- Documentation/arch/arm64/user4k.rst | 55 ++++- mm/mempolicy.c | 52 +++- mm/shmem.c | 2 +- tools/testing/selftests/arm64/user4k/Makefile | 9 +- .../arm64/user4k/compat-numa-probe.c | 109 +++++++++ .../selftests/arm64/user4k/numa-policy-init.c | 222 ++++++++++++++++++ 6 files changed, 431 insertions(+), 18 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/compat-numa-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/numa-policy-init.c diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index 1d4c535b1a37e..be7792edd1c09 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -169,6 +169,54 @@ across process-page boundaries. Eventfd notification, context destruction, removal of the ring mapping and adjacent guard preservation also pass. The Apple-enabled mm4 Image passes the native16/4K fixture in a VM. +NUMA policy ranges and physical placement (2026-10-03) +----------------------------------------------------- + +``mbind`` and ``set_mempolicy_home_node`` now align anonymous policy ranges +using the process granule. NUMA PTE walks advance and batch at that granule, +including native folios represented by several smaller leaves. Address-based +``get_mempolicy(MPOL_F_NODE)`` queries retain an exact fragment reference rather +than rounding the address to a native page and using legacy page-array GUP. + +Physical NUMA placement remains a native-folio property. Migration of a 4K +leaf can move its whole 16K or 64K owner, including neighbouring leaves; +it preserves their bytes and does not change their individual VMA policies. +``MPOL_MF_MOVE`` retains the existing conservative shared-folio check. +The migration tests use the privileged ``MPOL_MF_MOVE_ALL`` flag. + +Shared shmem policy is indexed by native page-cache units. Policy changes +must cover whole native file-page units within each affected VMA; partial +units return EINVAL before that VMA is split or its policy is replaced. +This is a placement restriction, not a restriction on sharing data across +process granules. Shared policy lookup includes the VMA's subpage file offset, +so an alias beginning partway through a native page selects the correct +policy after crossing into the next one. Interleave indices likewise retain +native allocation units and account for the subpage offset. + +The disposable ``numa-policy-init`` and ``compat-numa-probe`` fixtures require +two memory nodes, CONFIG_NUMA, CONFIG_NUMA_MIGRATION and CONFIG_COMPAT. Put +both executables in the initramfs root, with the former named ``/init`` and +an empty ``/proc`` directory plus ``/dev/console``. Native16/4K and +native64/16K/4K KASAN/lockdep guests pass fine-range policy updates, placement, +strict-policy checking, native-owner migration with byte preservation, and +shared-file offset/rejection tests. AArch32 passes with automatic 4K pages +and with native16/native64 fallback when 4K translation is capability-masked. +The latter models use ``id_aa64mmfr0.tgran4=f`` and retain rejection of +4K-only policy addresses. All cases verify that both memory nodes exist. + +The pre-change control rejects valid smaller-granule anonymous policy ranges; +``numa_fixture_baseline=1`` runs only that benign control check. The x86 NUMA +policy/shmem objects and feature-disabled arm64 shmem object also compile. +The 64K guest's boot-time ``debug_vm_pgtable`` test failed to allocate a +contiguous 512MiB block on both the unchanged control and the new kernel; +its dependent huge-page tests are therefore not qualified. The isolated +64K NUMA runs use ``initcall_blacklist=debug_vm_pgtable`` to omit that separate +boot test while retaining KASAN and lockdep. This does not fix its allocation +failure. The 16K runs retain the boot test. Automatic NUMA balancing was +disabled in these guests and remains unqualified; these results do not cover +broad pressure or memory hotplug. The published Apple mm4 configuration has +NUMA disabled and its artifact is unchanged. + Current userfaultfd support (2026-10-02) --------------------------------------- @@ -266,10 +314,9 @@ or complete NCE compatibility claim is made. Milestone sections below retain their historical status statements; the latest exec milestone supersedes earlier statements that EL0 activation was absent. -The current native16/native64 VM profiles and Apple candidate have NUMA -disabled. NUMA policy integration requires further work: policy range -alignment and PTE loops in ``mm/mempolicy.c`` still use native-page units. -NUMA-enabled mixed-granule operation is not covered by these results. +The Apple candidate keeps NUMA disabled. Separate two-node native16/native64 +guests now cover explicit policy ranges, placement and migration as described +above. Automatic NUMA balancing and broader NUMA workloads remain unqualified. The expanded objective is concurrent 4K, 16K and 64K userspace granules on arm64, with the kernel/physical allocator granule independently fixed for the diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 4e4421b22b59f..4bc7b707451f4 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -78,6 +78,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -688,6 +689,8 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, struct folio *folio; struct queue_pages *qp = walk->private; unsigned long flags = qp->flags; + unsigned int page_shift = mm_page_shift(walk->mm); + unsigned long page_size = 1UL << page_shift; pte_t *pte, *mapped_pte; pte_t ptent; spinlock_t *ptl; @@ -705,8 +708,8 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte += nr, addr += nr * PAGE_SIZE) { - max_nr = (end - addr) >> PAGE_SHIFT; + for (; addr != end; pte += nr, addr += nr * page_size) { + max_nr = (end - addr) >> page_shift; nr = 1; ptent = ptep_get(pte); if (pte_none(ptent)) @@ -721,7 +724,11 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, folio = vm_normal_folio(vma, addr, ptent); if (!folio || folio_is_zone_device(folio)) continue; - if (folio_test_large(folio) && max_nr != 1) +#ifdef CONFIG_MM_SUBPAGE + if (page_shift < PAGE_SHIFT) + nr = folio_subpage_pte_batch(vma, folio, pte, ptent, max_nr, 0); +#endif + if (page_shift >= PAGE_SHIFT && folio_test_large(folio) && max_nr != 1) nr = folio_pte_batch(folio, pte, ptent, max_nr); /* * vm_normal_folio() filters out zero pages, but there might @@ -731,9 +738,10 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, continue; if (!queue_folio_required(folio, qp)) continue; - if (folio_test_large(folio)) { + if (folio_test_large(folio) || page_shift < PAGE_SHIFT) { /* - * A large folio can only be isolated from LRU once, + * A native folio backing smaller leaves, like a large + * folio, can only be isolated from LRU once, * but may be mapped by many PTEs (and Copy-On-Write may * intersperse PTEs of other, order 0, folios). This is * a common case, so don't mistake it for failure (but @@ -755,7 +763,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) || !vma_migratable(vma) || !migrate_folio_add(folio, qp->pagelist, flags)) { - qp->nr_failed += nr; + qp->nr_failed += page_shift < PAGE_SHIFT ? 1 : nr; if (strictly_unmovable(flags)) break; } @@ -1050,6 +1058,12 @@ static int mbind_range(struct vma_iterator *vmi, struct vm_area_struct *vma, vmstart = vma->vm_start; } + /* Shared policy is indexed by native page-cache allocation units. */ + if (vma_is_shmem(vma) && + (vma_page_offset_at(vma, vmstart).offset || + ((vmend - vmstart) & ~PAGE_MASK))) + return -EINVAL; + if (mpol_equal(vma->vm_policy, new_pol)) { *prev = vma; return 0; @@ -1135,6 +1149,18 @@ static int lookup_node(struct mm_struct *mm, unsigned long addr) struct page *p = NULL; int ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_shift(mm) != PAGE_SHIFT) { + struct user_page_fragment fragment; + + ret = get_user_fragments_remote(mm, addr, 1, 0, &fragment, 1); + if (ret > 0) { + ret = folio_nid(fragment.folio); + release_user_fragments(&fragment, 1, false); + } + return ret; + } +#endif ret = get_user_pages_fast(addr & PAGE_MASK, 1, 0, &p); if (ret > 0) { ret = page_to_nid(p); @@ -1502,13 +1528,13 @@ static long do_mbind(unsigned long start, unsigned long len, if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) return -EPERM; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; if (mode == MPOL_DEFAULT) flags &= ~MPOL_MF_STRICT; - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); end = start + len; if (end < start) @@ -1768,7 +1794,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le VMA_ITERATOR(vmi, mm, start); start = untagged_addr(start); - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; /* * flags is used for future extension if any. @@ -1783,7 +1809,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le if (home_node >= MAX_NUMNODES || !node_online(home_node)) return -EINVAL; - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); end = start + len; if (end < start) @@ -2049,7 +2075,7 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, if (pol->mode == MPOL_INTERLEAVE || pol->mode == MPOL_WEIGHTED_INTERLEAVE) { *ilx += vma->vm_pgoff >> order; - *ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order); + *ilx += (vma_page_offset_at(vma, addr).index - vma->vm_pgoff) >> order; } return pol; } @@ -3254,6 +3280,10 @@ int mpol_set_shared_policy(struct shared_policy *sp, struct sp_node *new = NULL; unsigned long sz = vma_pages(vma); + /* Never truncate a partial cache-page policy update to zero or a neighbour. */ + if (vma_subpage_offset(vma) || ((vma->vm_end - vma->vm_start) & ~PAGE_MASK)) + return -EINVAL; + if (pol) { new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, pol); if (!new) diff --git a/mm/shmem.c b/mm/shmem.c index 1542ec3f10912..705765dddf1d2 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -3329,7 +3329,7 @@ static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, * by page order, as in shmem_get_pgoff_policy() and get_vma_policy()). */ *ilx = inode->i_ino; - index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + index = vma_page_offset_at(vma, addr).index; return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); } diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 7058daa50ce25..1c408280e1746 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -2,11 +2,11 @@ TEST_GEN_PROGS := page-contract TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init -TEST_GEN_PROGS_EXTENDED += aio-ring-init +TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe -TEST_GEN_FILES += compat-shm-probe compat-perf-probe +TEST_GEN_FILES += compat-shm-probe compat-perf-probe compat-numa-probe TEST_FILES += compat-script runtime-app.py include ../../lib.mk @@ -121,3 +121,8 @@ $(OUTPUT)/compat-perf-probe: compat-perf-probe.c $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ $(OUTPUT)/aio-ring-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +# Requires a disposable guest with two memory nodes and CONFIG_COMPAT. +$(OUTPUT)/numa-policy-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-numa-probe: compat-numa-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ diff --git a/tools/testing/selftests/arm64/user4k/compat-numa-probe.c b/tools/testing/selftests/arm64/user4k/compat-numa-probe.c new file mode 100644 index 0000000000000..ce1d9824e4181 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-numa-probe.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 NUMA policy and native-granule fallback checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" : "+r"(r0) : "r"(r1), "r"(r2), "r"(r3), + "r"(r4), "r"(r5), "r"(r7) : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 NUMA FAIL stage=00\n"; + + if (code) { + text[sizeof(text) - 4] = '0' + code / 10; + text[sizeof(text) - 3] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 NUMA PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long native, unsigned long page) +{ + unsigned long length = 4 * native; + long raw = call(192, 0, length, 3, 0x22, -1, 0); + unsigned char *base; + u32 mask[2] = {1, 0}, got[2] = {0, 0}; + int mode = -1, node; + + if ((unsigned long)raw >= 0xfffff001UL) + finish(20); + base = (void *)(((unsigned long)raw + native - 1) & ~(native - 1)); + if (call(320, (long)&mode, (long)got, 64, 0, 4, 0) || got[0] != 3 || got[1]) + finish(21); + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 0)) + finish(22); + mask[0] = 2; + if (call(319, (long)(base + page), 1, 2, (long)mask, 64, 0)) + finish(23); + if (call(450, (long)(base + page), 1, 1, 0, 0, 0)) + finish(24); + if (call(320, (long)&mode, (long)got, 64, (long)(base + page), 2, 0) || + mode != 2 || got[0] != 2 || got[1]) + finish(25); + for (unsigned long off = 0; off < 2 * native; off += page) { + base[off + 17] = 0x51 + off / page; + node = -1; + if (call(320, (long)&node, 0, 0, (long)(base + off + 17), 3, 0) || + node != (off == page ? 1 : 0)) + finish(26); + } + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 1) != -5) + finish(27); + if (call(319, (long)(base + 4096), 4096, 2, (long)mask, 64, 0) != + (page == 4096 ? 0 : -22)) + finish(28); + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 5)) + finish(29); + for (unsigned long off = 0; off < 2 * native; off += page) { + node = -1; + if (base[off + 17] != (unsigned char)(0x51 + off / page) || + call(320, (long)&node, 0, 0, (long)(base + off + 17), 3, 0) || node != 1) + finish(30); + } + if (call(91, raw, length, 0, 0, 0, 0)) + finish(31); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + mapping(number(argv[1]), pages); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/numa-policy-init.c b/tools/testing/selftests/arm64/user4k/numa-policy-init.c new file mode 100644 index 0000000000000..1a266c8ef89ca --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/numa-policy-init.c @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable two-node guest: policy ranges and native physical placement. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int good, const char *expr, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expr, errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static long bind_to(void *addr, unsigned long bytes, int node, unsigned int flags) +{ + unsigned long mask = 1UL << node; + + return syscall(SYS_mbind, addr, bytes, MPOL_BIND, &mask, 64UL, flags); +} + +static void policy_at(void *addr, int node) +{ + unsigned long mask = 0; + int mode = -1; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &mask, 64UL, addr, MPOL_F_ADDR)); + CHECK(mode == MPOL_BIND && mask == (1UL << node)); +} + +static void node_at(void *addr, int expected) +{ + int node = -1; + + CHECK(!syscall(SYS_get_mempolicy, &node, NULL, 0UL, addr, + MPOL_F_NODE | MPOL_F_ADDR)); + if (node != expected) + fprintf(stderr, "node=%d expected=%d addr=%p\n", node, expected, addr); + CHECK(node == expected); +} + +static void exercise(unsigned long native, int baseline) +{ + unsigned long page = getauxval(AT_PAGESZ), allowed = 0; + unsigned char *reserve, *base, *shared, *alias; + int mode, fd; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &allowed, 64UL, NULL, + MPOL_F_MEMS_ALLOWED)); + CHECK(allowed == 3); /* Require two real memory nodes, not a silent skip. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + CHECK(!bind_to(base, 2 * native, 0, 0)); + if (baseline) { + errno = 0; + long ret = bind_to(base + page, 1, 1, 0); + + CHECK(page < native ? ret == -1 && errno == EINVAL : ret == 0); + printf("ok - baseline native=%lu page=%lu mbind=%ld errno=%d\n", + native, page, ret, errno); + CHECK(!munmap(reserve, 4 * native)); + return; + } + CHECK(!bind_to(base + page, 1, 1, 0)); + policy_at(base, 0); + policy_at(base + page, 1); + if (page < native) + policy_at(base + 2 * page, 0); + CHECK(!syscall(SYS_set_mempolicy_home_node, base + page, 1UL, 1UL, 0UL)); + errno = 0; + CHECK(bind_to(base + 1, page, 1, 0) == -1 && errno == EINVAL); + for (unsigned long off = 0; off < 2 * native; off += page) { + base[off + 17] = 0x31 + off / page; + node_at(base + off + 17, off == page ? 1 : 0); + } + CHECK(!bind_to(base + page, page, 1, MPOL_MF_STRICT)); + errno = 0; + CHECK(bind_to(base, 2 * native, 1, MPOL_MF_STRICT) == -1 && errno == EIO); + policy_at(base, 0); + CHECK(!munmap(reserve, 4 * native)); + + /* A split anonymous VMA must carry its native interleave index correctly. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + unsigned long mask = 3, inset = page < native ? page : 0; + + CHECK(!syscall(SYS_mbind, base + inset, 2 * native - 2 * inset, + MPOL_INTERLEAVE, &mask, 64UL, 0)); + for (unsigned long off = inset; off < 2 * native - inset; off += page) { + base[off + 17] = 0x43; + node_at(base + off + 17, ((uintptr_t)base + off) / native % 2); + } + CHECK(!munmap(reserve, 4 * native)); + + /* Moving a fragment moves its native physical owner, retaining bytes. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + CHECK(!bind_to(base, 2 * native, 0, 0)); + for (unsigned long off = 0; off < 2 * native; off += page) + base[off + 17] = 0x51 + off / page; + CHECK(!bind_to(base + native - page, page, 1, MPOL_MF_MOVE_ALL | MPOL_MF_STRICT)); + for (unsigned long off = 0; off < 2 * native; off += page) { + CHECK(base[off + 17] == (unsigned char)(0x51 + off / page)); + node_at(base + off + 17, off < native ? 1 : 0); + } + CHECK(!bind_to(base, 2 * native, 0, MPOL_MF_MOVE_ALL | MPOL_MF_STRICT)); + for (unsigned long off = 0; off < 2 * native; off += page) + node_at(base + off + 17, 0); + CHECK(!munmap(reserve, 4 * native)); + + fd = memfd_create("numa-shared", 0); + CHECK(fd >= 0 && !ftruncate(fd, 2 * native)); + shared = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + CHECK(shared != MAP_FAILED); + CHECK(!bind_to(shared, native, 0, 0)); + CHECK(!bind_to(shared + native, native, 1, 0)); + if (page < native) { + alias = mmap(NULL, 2 * native - page, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, page); + CHECK(alias != MAP_FAILED); + policy_at(alias, 0); + policy_at(alias + native - page, 1); + errno = 0; + CHECK(bind_to(alias, page, 1, 0) == -1 && errno == EINVAL); + policy_at(alias, 0); + policy_at(shared, 0); + CHECK(!bind_to(alias + native - page, native, 1, 0)); + for (unsigned long off = 0; off < 2 * native - page; off += page) { + alias[off + 17] = 0x72; + node_at(alias + off + 17, off + page < native ? 0 : 1); + CHECK(shared[off + page + 17] == 0x72); + } + CHECK(!munmap(alias, 2 * native - page)); + } + CHECK(!munmap(shared, 2 * native) && !close(fd)); + printf("ok - NUMA native=%lu page=%lu policy, placement, migration and shared offsets\n", + native, page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ), sizes[] = {4096, 16384, 65536}; + int failed = 0, baseline = 0, no4; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--case")) { + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + exercise(strtoul(argv[2], NULL, 10), atoi(argv[4])); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + FILE *cmdline = fopen("/proc/cmdline", "r"); + char line[4096]; + + CHECK(cmdline && fgets(line, sizeof(line), cmdline)); + baseline = strstr(line, "numa_fixture_baseline=1") != NULL; + no4 = strstr(line, "id_aa64mmfr0.tgran4=f") != NULL; + CHECK(!fclose(cmdline)); + for (unsigned int i = 0; i < 3; i++) { + int status; + pid_t child; + + if (sizes[i] > native || (no4 && sizes[i] == 4096)) + continue; + child = fork(); + CHECK(child >= 0); + if (!child) { + char n[32], s[32], b[4]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + snprintf(b, sizeof(b), "%d", baseline); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", n, s, b, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + } + if (!baseline) { + int status; + pid_t child = fork(); + + CHECK(child >= 0); + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", no4 ? native : 4096UL); + execl("/compat-numa-probe", "/compat-numa-probe", n, s, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + printf("A32 NUMA expected-page=%lu status=%d\n", no4 ? native : 4096UL, status); + } + printf("NUMA GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} From ce625eb3bafc2d7619d441d10ade288a814bd5fe Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 05:23:14 -0400 Subject: [PATCH 0391/1579] mm: handle subpage ownership in automatic NUMA balancing Multiple private leaves within one native folio must not all be treated as shared COW memory. Decide NUMA hint eligibility from subpage ownership and split batches at changes in that state. Preserve native and huge-PMD checks. Restore hinting mappings using the mm granule and the physical byte offset, including smaller leaves within native owners. Clip restoration to VMA and PTE-table bounds. Charge partial-native VMA scans to the scheduler budget. Add an exact mixed-COW hint-selection KUnit test and an EL0 workload that verifies automatic node migration, local hint recovery, bytes and permissions. Both native16/native64 KASAN/lockdep builds pass all supported AArch64 modes; explicit-policy and AArch32 fallback tests also pass with balancing enabled. Document the unchanged 64K boot-selftest exclusion and coverage limits. Feature-disabled arm64 and NUMA-enabled x86 object builds pass. Signed-off-by: DjDeveloperr --- Documentation/arch/arm64/user4k.rst | 53 +++- arch/arm64/mm/user4k-test.c | 69 +++++ kernel/sched/fair.c | 4 +- mm/huge_memory.c | 3 +- mm/internal.h | 4 +- mm/memory.c | 30 +- mm/mempolicy.c | 5 +- mm/mprotect.c | 42 ++- tools/testing/selftests/arm64/user4k/Makefile | 5 +- .../arm64/user4k/numa-balancing-init.c | 270 ++++++++++++++++++ 10 files changed, 449 insertions(+), 36 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/numa-balancing-init.c diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index be7792edd1c09..465b12294eda5 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -213,9 +213,51 @@ its dependent huge-page tests are therefore not qualified. The isolated 64K NUMA runs use ``initcall_blacklist=debug_vm_pgtable`` to omit that separate boot test while retaining KASAN and lockdep. This does not fix its allocation failure. The 16K runs retain the boot test. Automatic NUMA balancing was -disabled in these guests and remains unqualified; these results do not cover -broad pressure or memory hotplug. The published Apple mm4 configuration has -NUMA disabled and its artifact is unchanged. +disabled for these explicit-policy runs; its separate coverage follows below. +These results do not cover broad pressure or memory hotplug. The published +Apple mm4 configuration has NUMA disabled and its artifact is unchanged. + +Automatic NUMA hinting and migration (2026-10-03) +------------------------------------------------ + +NUMA protection scans now distinguish private anonymous subpages from shared +COW subpages. Several private leaves in one native folio no longer make each +leaf appear shared merely because the native folio has multiple PTE mappings. +Batching stops at a change in subpage COW ownership, so a shared neighbour +does not cause a private leaf to be skipped or acquire its hinting decision. +Native-granule and huge-PMD sharing checks retain their existing rules. + +Hint-fault recovery restores mappings at the mm's granule using the actual +physical byte offset within the folio. Restoration is clipped to the VMA and +PTE table, and includes the other leaves of a native owner. It retains each +leaf's permission checks and can leave unrelated PTEs untouched. Scheduler +scan budgets continue to count native-page byte units, rounding partial-native +VMA spans up instead of letting them consume zero budget. + +The disposable ``numa-balancing-init`` fixture requires two nodes with CPU0 +on node0 and CPU1 on node1, CONFIG_NUMA_BALANCING, CONFIG_SCHED_DEBUG and +CONFIG_DEBUG_FS. Put it at ``/init`` with empty ``/proc`` and ``/debug`` mount +points and ``/dev/console``. It first allocates an 8MiB anonymous range on +node0, clears its binding, and runs on CPU1 until every accessible test leaf +is on node1. No explicit migration syscall is used for that transition. +Bytes, write counters, read-only protection and PROT_NONE isolation survive. +A two-thread local phase records local NUMA hints and verifies recovery when +migration is unnecessary. Guest scan-delay/period tunables are shortened +only for this fixture; no production default is changed. + +These checks pass for native16/4K and native64/16K/4K AArch64 processes with +KASAN and lockdep. A focused KUnit case also reuses one private slot among +shared COW neighbours in the same physical owner, makes their PTE permissions +identical, and verifies that only the private slot receives a NUMA hint. +The explicit-policy and AArch32 fallback fixtures still pass with balancing +enabled. Feature-disabled arm64 and x86 NUMA objects compile. The native64 +runs retain the separately documented ``debug_vm_pgtable`` boot-test exclusion. + +This covers bounded anonymous balancing and exact COW hint selection, not +AArch32 automatic-migration workloads, large file-folio balancing, broad +pressure or memory-tiering performance. The node checks are functional VM +results, not throughput measurements. The published Apple mm4 artifact is +unchanged and continues to use NUMA=n. Current userfaultfd support (2026-10-02) --------------------------------------- @@ -315,8 +357,9 @@ their historical status statements; the latest exec milestone supersedes earlier statements that EL0 activation was absent. The Apple candidate keeps NUMA disabled. Separate two-node native16/native64 -guests now cover explicit policy ranges, placement and migration as described -above. Automatic NUMA balancing and broader NUMA workloads remain unqualified. +guests now cover explicit policy ranges, placement, migration and bounded +anonymous automatic balancing as described above. Broader NUMA workloads, +file-folio balancing and memory-tiering performance remain unqualified. The expanded objective is concurrent 4K, 16K and 64K userspace granules on arm64, with the kernel/physical allocator granule independently fixed for the diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 761531ba876df..d7c6e85ef7343 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -8795,6 +8795,72 @@ static void user4k_mprotect_test(struct kunit *test) } } +#ifdef CONFIG_NUMA_BALANCING +static void user_subpage_numa_cow_batch_test(struct kunit *test) +{ + unsigned int shift; + + for (shift = 12; shift < PAGE_SHIFT; shift += 2) { + unsigned long size = 1UL << shift, start = TEST_VA; + struct vm_area_struct *vma; + struct mm_struct *child = test_vma_mm_granule(test, shift); + struct mm_struct *mm = test_vma_mm_granule(test, shift); + phys_addr_t before; + unsigned long marked, mapped; + long ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + /* Use mmap accounting because the child will partially unmap. */ + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, 4 * size, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + i * size, true), + (vm_fault_t)0); + before = pte_phys_mm(mm, test_fault_entry(test, mm, start + size)); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + kthread_use_mm(child); + ret = vm_munmap(start + size, size); + kthread_unuse_mm(child); + KUNIT_ASSERT_EQ(test, ret, 0L); + /* Reuse the now-private slot without replacing its native owner. */ + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + size, true), (vm_fault_t)0); + KUNIT_ASSERT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + size)), before); + kthread_use_mm(mm); + ret = test_mprotect(start, 4 * size, PROT_READ); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Permit local-node hinting, which normally skips a single user. */ + mmget(mm); + mmap_read_lock(mm); + marked = change_prot_numa(vma, start, start + 4 * size); + mmap_read_unlock(mm); + mmput(mm); + KUNIT_EXPECT_EQ(test, marked, 1UL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, start + i * size); + + KUNIT_EXPECT_EQ(test, !!pte_protnone(pte), i == 1); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + } + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} +#endif + static void user4k_mprotect_cow_batch_test(struct kunit *test) { struct mm_struct *mm, *child; @@ -13981,6 +14047,9 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_vm_mmap_test), KUNIT_CASE(user4k_mprotect_test), KUNIT_CASE(user4k_mprotect_cow_batch_test), +#ifdef CONFIG_NUMA_BALANCING + KUNIT_CASE(user_subpage_numa_cow_batch_test), +#endif KUNIT_CASE(user4k_mprotect_accounting_test), KUNIT_CASE(user4k_file_cow_test), KUNIT_CASE(user4k_file_eof_test), diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c5b7ea8acdda3..e06b05a08f7f4 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -3682,8 +3682,8 @@ static void task_numa_work(struct callback_head *work) * areas faster. */ if (nr_pte_updates) - pages -= (end - start) >> PAGE_SHIFT; - virtpages -= (end - start) >> PAGE_SHIFT; + pages -= DIV_ROUND_UP(end - start, PAGE_SIZE); + virtpages -= DIV_ROUND_UP(end - start, PAGE_SIZE); start = end; if (pages <= 0 || virtpages <= 0) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 2ab333c54c987..7ab369477fcc7 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2715,7 +2715,8 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, goto unlock; if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma, - vma_is_single_threaded_private(vma))) + vma_is_single_threaded_private(vma), + folio_maybe_mapped_shared(pmd_folio(*pmd)))) goto unlock; } /* diff --git a/mm/internal.h b/mm/internal.h index 23abe35e53589..5ad4e57daedc6 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1632,11 +1632,11 @@ static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma) #ifdef CONFIG_NUMA_BALANCING bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, - bool is_private_single_threaded); + bool is_private_single_threaded, bool shared); #else static inline bool folio_can_map_prot_numa(struct folio *folio, - struct vm_area_struct *vma, bool is_private_single_threaded) + struct vm_area_struct *vma, bool is_private_single_threaded, bool shared) { return false; } diff --git a/mm/memory.c b/mm/memory.c index b3e96c0bfed0b..ade3dbadb409a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -7501,31 +7501,33 @@ static void numa_rebuild_single_mapping(struct vm_fault *vmf, struct vm_area_str update_mmu_cache_range(vmf, vma, fault_addr, fault_pte, 1); } -static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, +static void numa_rebuild_folio_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, struct folio *folio, pte_t fault_pte, bool ignore_writable, bool pte_write_upgrade) { - int nr = pte_pfn(fault_pte) - folio_pfn(folio); + struct mm_struct *mm = vma->vm_mm; + unsigned int page_shift = mm_page_shift(mm); + unsigned long page_size = 1UL << page_shift; + unsigned long offset = pte_phys_mm(mm, fault_pte) - PFN_PHYS(folio_pfn(folio)); unsigned long start, end, addr = vmf->address; - unsigned long addr_start = addr - (nr << PAGE_SHIFT); - unsigned long pt_start = ALIGN_DOWN(addr, PMD_SIZE); + unsigned long lower = max(addr & pmd_mask_mm(mm), vma->vm_start); + unsigned long upper = pmd_addr_end_mm(mm, addr, vma->vm_end); pte_t *start_ptep; - /* Stay within the VMA and within the page table. */ - start = max3(addr_start, pt_start, vma->vm_start); - end = min3(addr_start + folio_size(folio), pt_start + PMD_SIZE, - vma->vm_end); - start_ptep = vmf->pte - ((addr - start) >> PAGE_SHIFT); + /* Stay within the VMA, page table and physical owner without wrapping. */ + start = addr - min(offset, addr - lower); + end = addr + min(folio_size(folio) - offset, upper - addr); + start_ptep = vmf->pte - ((addr - start) >> page_shift); - /* Restore all PTEs' mapping of the large folio */ - for (addr = start; addr != end; start_ptep++, addr += PAGE_SIZE) { + /* Restore the owner's PTEs, including smaller leaves in a native folio. */ + for (addr = start; addr != end; start_ptep++, addr += page_size) { pte_t ptent = ptep_get(start_ptep); bool writable = false; if (!pte_present(ptent) || !pte_protnone(ptent)) continue; - if (pfn_folio(pte_pfn(ptent)) != folio) + if (pfn_folio(PHYS_PFN(pte_phys_mm(mm, ptent))) != folio) continue; if (!ignore_writable) { @@ -7618,8 +7620,8 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) * Make it present again, depending on how arch implements * non-accessible ptes, some can allow access by kernel mode. */ - if (folio && folio_test_large(folio)) - numa_rebuild_large_mapping(vmf, vma, folio, pte, ignore_writable, + if (folio && (folio_test_large(folio) || mm_page_shift(vma->vm_mm) < PAGE_SHIFT)) + numa_rebuild_folio_mapping(vmf, vma, folio, pte, ignore_writable, pte_write_upgrade); else numa_rebuild_single_mapping(vmf, vma, vmf->address, vmf->pte, diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 4bc7b707451f4..3b97c71c9d3cf 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -832,6 +832,7 @@ static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask, * @folio: The folio whose mapping considered for being made NUMA hintable * @vma: The VMA that the folio belongs to. * @is_private_single_threaded: Is this a single-threaded private VMA or not + * @shared: Is the mapped memory potentially shared between address spaces * * This function checks to see if the folio actually indicates that * we need to make the mapping one which causes a NUMA hinting fault, @@ -841,7 +842,7 @@ static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask, * Return: True if the mapping of the folio needs to be changed, false otherwise. */ bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, - bool is_private_single_threaded) + bool is_private_single_threaded, bool shared) { int nid; @@ -849,7 +850,7 @@ bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, return false; /* Also skip shared copy-on-write folios */ - if (is_cow_mapping(vma->vm_flags) && folio_maybe_mapped_shared(folio)) + if (is_cow_mapping(vma->vm_flags) && shared) return false; /* Folios are pinned and can't be migrated */ diff --git a/mm/mprotect.c b/mm/mprotect.c index 6f1a94026079d..0f5ea22bf4c90 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -71,6 +71,17 @@ static bool subpage_pte_exclusive(struct vm_area_struct *vma, pte_t pte) mm_subpage_put(slot); return exclusive; } + +static int subpage_exclusive_pte_batch(struct vm_area_struct *vma, pte_t *ptep, + int nr_ptes, bool exclusive) +{ + int nr; + + for (nr = 1; nr < nr_ptes; nr++) + if (subpage_pte_exclusive(vma, ptep_get(ptep + nr)) != exclusive) + break; + return nr; +} #endif static bool can_change_private_pte_writable(struct vm_area_struct *vma, @@ -412,15 +423,28 @@ static long change_pte_range(struct mmu_gather *tlb, * Avoid trapping faults against the zero or KSM * pages. See similar comment in change_huge_pmd. */ - if (prot_numa && - !folio_can_map_prot_numa(folio, vma, - is_private_single_threaded)) { - - /* determine batch to skip */ - nr_ptes = mprotect_folio_pte_batch(vma, folio, - pte, oldpte, max_nr_ptes, /* flags = */ 0, - page_shift); - continue; + if (prot_numa) { + bool shared = folio && folio_maybe_mapped_shared(folio); + +#ifdef CONFIG_MM_SUBPAGE + /* Multiple private leaves do not imply shared COW data. */ + if (folio && folio_test_anon(folio) && page_shift < PAGE_SHIFT) { + shared = !subpage_pte_exclusive(vma, oldpte); + max_nr_ptes = folio_subpage_pte_batch(vma, folio, + pte, oldpte, max_nr_ptes, flags); + /* COW sharing can differ within one physical owner. */ + max_nr_ptes = subpage_exclusive_pte_batch(vma, pte, + max_nr_ptes, !shared); + } +#endif + if (!folio_can_map_prot_numa(folio, vma, + is_private_single_threaded, shared)) { + /* determine batch to skip */ + nr_ptes = mprotect_folio_pte_batch(vma, folio, + pte, oldpte, max_nr_ptes, /* flags = */ 0, + page_shift); + continue; + } } nr_ptes = mprotect_folio_pte_batch(vma, folio, pte, oldpte, diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 1c408280e1746..5f4e8f6389afb 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -2,7 +2,7 @@ TEST_GEN_PROGS := page-contract TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init -TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init +TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe @@ -126,3 +126,6 @@ $(OUTPUT)/aio-ring-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 $(OUTPUT)/numa-policy-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 $(OUTPUT)/compat-numa-probe: compat-numa-probe.c $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/numa-balancing-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/numa-balancing-init: LDLIBS += -pthread diff --git a/tools/testing/selftests/arm64/user4k/numa-balancing-init.c b/tools/testing/selftests/arm64/user4k/numa-balancing-init.c new file mode 100644 index 0000000000000..fbe96307cfb3f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/numa-balancing-init.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Disposable two-node guest: real automatic NUMA hinting and migration. + * Volatile accesses intentionally trigger page faults and repeated accesses; + * thread coordination uses an atomic flag, not volatile storage. + */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int good, const char *expr, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expr, errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static void setting(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + + CHECK(fd >= 0); + CHECK(write(fd, value, strlen(value)) == (ssize_t)strlen(value)); + CHECK(!close(fd)); +} + +static void cpu(int id) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(id, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} + +static unsigned long counter(const char *key) +{ + FILE *f = fopen("/proc/vmstat", "r"); + char name[128]; + unsigned long value; + + CHECK(f); + while (fscanf(f, "%127s %lu", name, &value) == 2) { + if (!strcmp(name, key)) { + CHECK(!fclose(f)); + return value; + } + } + CHECK(0); + return 0; +} + +static double now(void) +{ + struct timespec ts; + + CHECK(!clock_gettime(CLOCK_MONOTONIC, &ts)); + return ts.tv_sec + ts.tv_nsec / 1e9; +} + +static int node_at(void *address) +{ + int node = -1; + + CHECK(!syscall(SYS_get_mempolicy, &node, NULL, 0UL, address, + MPOL_F_ADDR | MPOL_F_NODE)); + return node; +} + +static unsigned long sample(unsigned char *map, unsigned long length, unsigned long page) +{ + unsigned long moved = 0; + + for (unsigned long off = 0; off < length; off += page) + if (off != 2 * page) + moved += node_at(map + off + 16) == 1; + return moved; +} + +static void forbidden(unsigned char *address, int write_access) +{ + pid_t child = fork(); + int status; + + CHECK(child >= 0); + if (!child) { + if (write_access) + *(volatile unsigned char *)address = 0x73; + else + (void)*(volatile unsigned char *)address; + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + CHECK(WIFSIGNALED(status) && WTERMSIG(status) == SIGSEGV); +} + +struct local_reader { + unsigned char *map; + unsigned long length, page; + atomic_int stop; +}; + +static void *read_local(void *arg) +{ + struct local_reader *r = arg; + + cpu(1); + while (!atomic_load_explicit(&r->stop, memory_order_relaxed)) { + for (unsigned long off = 0; off < r->length; off += r->page) { + if (off == 2 * r->page) + continue; + CHECK(*(volatile uint64_t *)(r->map + off) == 0x81720000 + off / r->page); + } + } + return NULL; +} + +static void local_restore(unsigned char *map, unsigned long length, unsigned long page) +{ + struct local_reader r = { .map = map, .length = length, .page = page }; + unsigned long hints = counter("numa_hint_faults_local"); + pthread_t worker; + double deadline = now() + 2; + + /* Two threads permit hinting of already-local private mappings. */ + CHECK(!pthread_create(&worker, NULL, read_local, &r)); + do { + for (unsigned long off = 0; off < length; off += page) { + if (off == 2 * page) + continue; + CHECK(*(volatile uint64_t *)(map + off) == 0x81720000 + off / page); + } + } while (now() < deadline); + atomic_store_explicit(&r.stop, 1, memory_order_relaxed); + CHECK(!pthread_join(worker, NULL)); + hints = counter("numa_hint_faults_local") - hints; + printf("NUMAB local page=%lu hints=%lu\n", page, hints); + CHECK(hints > 0); +} + +static void exercise(void) +{ + unsigned long page = getauxval(AT_PAGESZ), length = 8UL << 20; + unsigned long mask = 1, allowed = 0, hints, migrated, iterations = 0; + unsigned long moved = 0, total = length / page - 1; + unsigned char *map; + int mode; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &allowed, 64UL, NULL, MPOL_F_MEMS_ALLOWED)); + CHECK(allowed == 3); + cpu(0); + map = mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(map != MAP_FAILED); + CHECK(!madvise(map, length, MADV_NOHUGEPAGE)); + CHECK(!syscall(SYS_mbind, map, length, MPOL_BIND, &mask, 64UL, 0)); + for (unsigned long off = 0; off < length; off += page) { + *(uint64_t *)(map + off) = 0x81720000 + off / page; + *(uint64_t *)(map + off + 8) = 0; + *(uint64_t *)(map + off + page - 8) = 0x61320000 + off / page; + CHECK(node_at(map + off + 16) == 0); + } + CHECK(!mprotect(map + page, page, PROT_READ)); + CHECK(!mprotect(map + 2 * page, page, PROT_NONE)); + CHECK(!syscall(SYS_mbind, map, length, MPOL_DEFAULT, NULL, 0UL, 0)); + cpu(1); + hints = counter("numa_hint_faults"); + migrated = counter("numa_pages_migrated"); + double started = now(), deadline = started + 20; + + do { + for (unsigned long off = 0; off < length; off += page) { + volatile uint64_t *p = (void *)(map + off); + + if (off == 2 * page) + continue; + CHECK(p[0] == 0x81720000 + off / page); + CHECK(p[page / 8 - 1] == 0x61320000 + off / page); + if (off != page) + p[1]++; + } + iterations++; + if (!(iterations % 128)) { + moved = sample(map, length, page); + if (moved == total) + break; + } + } while (now() < deadline); + moved = sample(map, length, page); + hints = counter("numa_hint_faults") - hints; + migrated = counter("numa_pages_migrated") - migrated; + printf("NUMAB page=%lu moved=%lu/%lu hints=%lu migrated=%lu loops=%lu seconds=%.3f\n", + page, moved, total, hints, migrated, iterations, now() - started); + CHECK(moved == total && hints > 0 && migrated > 0); + for (unsigned long off = 0; off < length; off += page) { + if (off != page && off != 2 * page) + CHECK(*(uint64_t *)(map + off + 8) == iterations); + } + local_restore(map, length, page); + forbidden(map + page, 1); + forbidden(map + 2 * page, 0); + CHECK(!mprotect(map + 2 * page, page, PROT_READ)); + CHECK(*(uint64_t *)(map + 2 * page) == 0x81720002); + CHECK(!munmap(map, length)); + printf("ok - NUMAB page=%lu automatic placement, bytes and permissions\n", page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ), sizes[] = {4096, 16384, 65536}; + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--case")) { + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[2], NULL, 10)); + exercise(); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("debugfs", "/debug", "debugfs", 0, NULL)); + setting("/proc/sys/kernel/numa_balancing", "1\n"); + setting("/debug/sched/numa_balancing/scan_delay_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_period_min_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_period_max_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_size_mb", "16\n"); + for (unsigned int i = 0; i < 3; i++) { + pid_t child; + int status; + + if (sizes[i] > native) + continue; + child = fork(); + CHECK(child >= 0); + if (!child) { + char s[32]; + + snprintf(s, sizeof(s), "%lu", sizes[i]); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", s, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + } + printf("NUMA BALANCING %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} From e2d86abdfee5bebcc7dbaf5b9ce1a99504ff4f51 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 05:38:05 -0400 Subject: [PATCH 0392/1579] selftests/arm64: cover AArch32 automatic NUMA migration Add a freestanding compat probe for automatic node migration, data retention and protection faults. Cover automatic 4K pages on native16/native64 kernels and native-page fallback when 4K capability is masked. Retain the fallback alignment checks and use time64 without requiring COMPAT_32BIT_TIME. Extend the disposable NUMA driver to run the required compat probe and to support explicit compat-only/no4K modes. Four isolated models and two combined AArch64/AArch32 suites pass on the unchanged ce625eb3b kernels. Document the VM scope and retained 64K boot-test exclusion. No kernel code is changed. Signed-off-by: DjDeveloperr --- Documentation/arch/arm64/user4k.rst | 28 ++- tools/testing/selftests/arm64/user4k/Makefile | 5 +- .../user4k/compat-numa-balancing-probe.c | 214 ++++++++++++++++++ .../arm64/user4k/numa-balancing-init.c | 27 ++- 4 files changed, 264 insertions(+), 10 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index 465b12294eda5..821cc390a6baa 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -235,9 +235,10 @@ scan budgets continue to count native-page byte units, rounding partial-native VMA spans up instead of letting them consume zero budget. The disposable ``numa-balancing-init`` fixture requires two nodes with CPU0 -on node0 and CPU1 on node1, CONFIG_NUMA_BALANCING, CONFIG_SCHED_DEBUG and -CONFIG_DEBUG_FS. Put it at ``/init`` with empty ``/proc`` and ``/debug`` mount -points and ``/dev/console``. It first allocates an 8MiB anonymous range on +on node0 and CPU1 on node1, CONFIG_NUMA_BALANCING, CONFIG_SCHED_DEBUG, +CONFIG_DEBUG_FS and CONFIG_COMPAT. Put it at ``/init`` and the freestanding +``compat-numa-balancing-probe`` at the initramfs root, with empty ``/proc`` and +``/debug`` mount points plus ``/dev/console``. It first allocates an 8MiB anonymous range on node0, clears its binding, and runs on CPU1 until every accessible test leaf is on node1. No explicit migration syscall is used for that transition. Bytes, write counters, read-only protection and PROT_NONE isolation survive. @@ -253,11 +254,24 @@ The explicit-policy and AArch32 fallback fixtures still pass with balancing enabled. Feature-disabled arm64 and x86 NUMA objects compile. The native64 runs retain the separately documented ``debug_vm_pgtable`` boot-test exclusion. +The AArch32 probe also passes automatic migration of every accessible leaf +in an 8MiB range, with byte markers, write counters and read-only/PROT_NONE +protection retained. It runs with automatic 4K pages on both native kernels, +and with native16/native64 pages when 4K capability is masked. Those fallback +runs retain rejection of 4K-only protection addresses. The probe uses the +64-bit time syscall and does not require CONFIG_COMPAT_32BIT_TIME. No kernel +implementation change was needed for this additional AArch32 coverage. + +Use ``--compat-only`` as an init argument for the isolated AArch32 cases. +For the no4K model, append ``id_aa64mmfr0.tgran4=f -- --compat-only --no4k`` +to the kernel command line. The driver requires the probe and checks the +expected page size and child exit status; missing binaries are failures. +These are capability-masked VM results, not tests on an Apple A9 device. + This covers bounded anonymous balancing and exact COW hint selection, not -AArch32 automatic-migration workloads, large file-folio balancing, broad -pressure or memory-tiering performance. The node checks are functional VM -results, not throughput measurements. The published Apple mm4 artifact is -unchanged and continues to use NUMA=n. +large file-folio balancing, broad pressure or memory-tiering performance. +The node checks are functional VM results, not throughput measurements. +The published Apple mm4 artifact is unchanged and continues to use NUMA=n. Current userfaultfd support (2026-10-02) --------------------------------------- diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 5f4e8f6389afb..21b52f893b27d 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -6,7 +6,7 @@ TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe -TEST_GEN_FILES += compat-shm-probe compat-perf-probe compat-numa-probe +TEST_GEN_FILES += compat-shm-probe compat-perf-probe compat-numa-probe compat-numa-balancing-probe TEST_FILES += compat-script runtime-app.py include ../../lib.mk @@ -129,3 +129,6 @@ $(OUTPUT)/compat-numa-probe: compat-numa-probe.c $(OUTPUT)/numa-balancing-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 $(OUTPUT)/numa-balancing-init: LDLIBS += -pthread + +$(OUTPUT)/compat-numa-balancing-probe: compat-numa-balancing-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ diff --git a/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c b/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c new file mode 100644 index 0000000000000..06cb41ed5bc96 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 automatic NUMA migration and fallback checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" : "+r"(r0) : "r"(r1), "r"(r2), "r"(r3), + "r"(r4), "r"(r5), "r"(r7) : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 NUMAB FAIL stage=00\n"; + + if (code) { + text[sizeof(text) - 4] = '0' + code / 10; + text[sizeof(text) - 3] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 NUMAB PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void value(const char *label, unsigned int length, u32 n) +{ + char text[16]; + unsigned int pos = sizeof(text); + + text[--pos] = '\n'; + do { + text[--pos] = '0' + n % 10; + n /= 10; + } while (n); + call(4, 1, (long)label, length, 0, 0, 0); + call(4, 1, (long)(text + pos), sizeof(text) - pos, 0, 0, 0); +} + +static u32 seconds(void) +{ + struct { long long sec, nsec; } ts; + + /* Use time64 even when CONFIG_COMPAT_32BIT_TIME is disabled. */ + if (call(403, 1, (long)&ts, 0, 0, 0, 0)) + finish(11); + return ts.sec; +} + +static void cpu(unsigned int id) +{ + u32 mask = 1U << id; + + if (call(241, 0, sizeof(mask), (long)&mask, 0, 0, 0)) + finish(12); +} + +static int node_at(unsigned char *address) +{ + int node = -1; + + if (call(320, (long)&node, 0, 0, (long)address, 3, 0)) + finish(13); + return node; +} + +static u32 sample(unsigned char *base, u32 length, u32 page) +{ + u32 moved = 0; + + for (u32 off = 0; off < length; off += page) + if (off != 2 * page) + moved += node_at(base + off + 16) == 1; + return moved; +} + +static void forbidden(unsigned char *address, int write_access) +{ + long child = call(2, 0, 0, 0, 0, 0, 0); + int status = 0; + + if (child < 0) + finish(40); + if (!child) { + /* Force the intended access; this is not thread synchronization. */ + if (write_access) + *(volatile unsigned char *)address = 0x27; + else + (void)*(volatile unsigned char *)address; + call(1, 2, 0, 0, 0, 0, 0); + for (;;) + ; + } + if (call(114, child, (long)&status, 0, 0, 0, 0) != child || + (status & 0x7f) != 11) + finish(41); +} + +static void mapping(u32 page) +{ + const u32 length = 8U << 20; + u32 mask[2] = {1, 0}, got[2] = {0, 0}; + u32 loops = 0, moved = 0, started, total = length / page - 1; + int mode = -1; + long raw; + unsigned char *base; + + if (call(320, (long)&mode, (long)got, 64, 0, 4, 0) || got[0] != 3 || got[1]) + finish(20); + cpu(0); + raw = call(192, 0, length, 3, 0x22, -1, 0); + if ((u32)raw >= 0xfffff001U) + finish(21); + base = (void *)raw; + /* A missing 4K capability must retain native alignment restrictions. */ + if (call(125, raw + 4096, 4096, 1, 0, 0, 0) != (page == 4096 ? 0 : -22)) + finish(22); + if (call(125, raw, length, 3, 0, 0, 0)) + finish(23); + if (call(220, raw, length, 15, 0, 0, 0)) /* MADV_NOHUGEPAGE */ + finish(24); + if (call(319, raw, length, 2, (long)mask, 64, 0)) + finish(25); + for (u32 off = 0; off < length; off += page) { + u32 *p = (void *)(base + off); + + p[0] = 0x91720000 + off / page; + p[1] = 0; + p[page / 4 - 1] = 0x51320000 + off / page; + if (node_at(base + off + 16) != 0) + finish(26); + } + if (call(125, raw + page, page, 1, 0, 0, 0) || + call(125, raw + 2 * page, page, 0, 0, 0, 0)) + finish(27); + if (call(319, raw, length, 0, 0, 0, 0)) /* MPOL_DEFAULT, no MOVE flags */ + finish(28); + cpu(1); + started = seconds(); + do { + for (u32 off = 0; off < length; off += page) { + volatile u32 *p = (void *)(base + off); + + if (off == 2 * page) + continue; + if (p[0] != 0x91720000 + off / page || + p[page / 4 - 1] != 0x51320000 + off / page) + finish(29); + if (off != page) + p[1]++; + } + loops++; + if (!(loops % 128)) { + moved = sample(base, length, page); + if (moved == total) + break; + } + } while (seconds() - started < 30); + moved = sample(base, length, page); + value("A32 NUMAB page=", sizeof("A32 NUMAB page=") - 1, page); + value("A32 NUMAB moved=", sizeof("A32 NUMAB moved=") - 1, moved); + value("A32 NUMAB expected=", sizeof("A32 NUMAB expected=") - 1, total); + value("A32 NUMAB loops=", sizeof("A32 NUMAB loops=") - 1, loops); + if (moved != total) + finish(30); + for (u32 off = 0; off < length; off += page) + if (off != page && off != 2 * page && *(u32 *)(base + off + 4) != loops) + finish(31); + forbidden(base + page, 1); + forbidden(base + 2 * page, 0); + if (call(125, raw + 2 * page, page, 1, 0, 0, 0) || + *(u32 *)(base + 2 * page) != 0x91720002) + finish(32); + if (call(91, raw, length, 0, 0, 0, 0)) + finish(33); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, page = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + page = sp[1]; + sp += 2; + } + if (argc != 2 || page != number(argv[1])) + finish(10); + mapping(page); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/numa-balancing-init.c b/tools/testing/selftests/arm64/user4k/numa-balancing-init.c index fbe96307cfb3f..fa00998c4eacd 100644 --- a/tools/testing/selftests/arm64/user4k/numa-balancing-init.c +++ b/tools/testing/selftests/arm64/user4k/numa-balancing-init.c @@ -228,7 +228,7 @@ static void exercise(void) int main(int argc, char **argv) { unsigned long native = getauxval(AT_PAGESZ), sizes[] = {4096, 16384, 65536}; - int failed = 0; + int failed = 0, compat_only = 0, no4k = 0; setvbuf(stdout, NULL, _IONBF, 0); if (argc == 3 && !strcmp(argv[1], "--case")) { @@ -237,6 +237,14 @@ int main(int argc, char **argv) return 0; } CHECK(getpid() == 1); + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--compat-only")) + compat_only = 1; + else if (!strcmp(argv[i], "--no4k")) + no4k = 1; + else + CHECK(0); + } CHECK(!mount("proc", "/proc", "proc", 0, NULL)); CHECK(!mount("debugfs", "/debug", "debugfs", 0, NULL)); setting("/proc/sys/kernel/numa_balancing", "1\n"); @@ -248,7 +256,7 @@ int main(int argc, char **argv) pid_t child; int status; - if (sizes[i] > native) + if (compat_only || sizes[i] > native || (no4k && sizes[i] == 4096)) continue; child = fork(); CHECK(child >= 0); @@ -263,6 +271,21 @@ int main(int argc, char **argv) CHECK(waitpid(child, &status, 0) == child); failed |= !WIFEXITED(status) || WEXITSTATUS(status); } + CHECK(!access("/compat-numa-balancing-probe", X_OK)); + pid_t child = fork(); + int status; + + CHECK(child >= 0); + if (!child) { + char s[32]; + + snprintf(s, sizeof(s), "%lu", no4k ? native : 4096UL); + execl("/compat-numa-balancing-probe", "/compat-numa-balancing-probe", s, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + printf("A32 NUMAB expected-page=%lu status=%d\n", no4k ? native : 4096UL, status); printf("NUMA BALANCING %s\n", failed ? "FAIL" : "PASS"); reboot(RB_POWER_OFF); for (;;) From efe6e359d456724cb0a7292d867bac147d9a5cd3 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:59:19 -0500 Subject: [PATCH 0393/1579] arm64: dts: apple: enable the ANE on t8103, t6000 and t6020 The ANE node, its DARTs, its mailbox and the power states only it uses are disabled on the M1 (T8103), the M1 Pro (T6000) and the M2 Pro (T6020) behind APPLE_ANE_UNTESTED, because the driver had run only on the M1 Max and the M2 Max. It has now run with this tree's own driver on all three. Each chip has a passing row in the public community dataset (promotion_check --remote: judged, driver_source=intree, 20 smoke calls that match the golden bit for bit, no ANE, DART or mailbox fault): M1 (T8103) row 3559e0f328ba M1 Pro (T6000) row 6eb94f49985b M2 Pro (T6020) row 3c9389040f51 Remove the define from t6000.dtsi, t6020.dtsi and t8103.dtsi. No file defines it after this, so also remove the #ifndef APPLE_ANE_UNTESTED guards in t6001.dtsi, t6021.dtsi and t8103.dtsi, and correct the comments that said the ANE had not run on the M1 Pro and the M2 Pro. The ANE is now enabled on the t8103 boards (j274, j293, j313, j456, j457), the t6000 boards (j314s, j316s) and the t6020 boards (j414s, j416s, j474s). The DTBs of the t6001, t6002, t6021, t6022 and t8112 boards do not change: t6002.dtsi and t6022.dtsi include neither t6001.dtsi nor t6021.dtsi, so the ANE stays disabled on the M1 Ultra (T6002), the M2 Ultra (T6022) and the M2 (T8112). Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t6000.dtsi | 3 --- arch/arm64/boot/dts/apple/t6001.dtsi | 7 ++----- arch/arm64/boot/dts/apple/t6020.dtsi | 3 --- arch/arm64/boot/dts/apple/t6021.dtsi | 7 ++----- arch/arm64/boot/dts/apple/t8103.dtsi | 9 ++------- 5 files changed, 6 insertions(+), 23 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index 22cb23931e0b3..db7ea6eac5a6f 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -11,9 +11,6 @@ #define GPU_REPEAT(x) -/* The ANE has only run on the M1 Max so far: leave it off on the M1 Pro */ -#define APPLE_ANE_UNTESTED - #include "t6001.dtsi" / { diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index f044ff6980c1b..f14bd14518042 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -90,11 +90,9 @@ }; /* - * The ANE has run on the M1 Max, so enable it here with its DARTs and the - * power states only it uses. t6000.dtsi defines APPLE_ANE_UNTESTED before - * including this file: the ANE has not run on the M1 Pro yet. + * The ANE has run on the M1 Pro and the M1 Max, so enable it here with its + * DARTs and the power states only it uses. t6000.dtsi includes this file. */ -#ifndef APPLE_ANE_UNTESTED &ane { status = "okay"; }; @@ -134,4 +132,3 @@ &ps_ane_set4 { status = "okay"; }; -#endif diff --git a/arch/arm64/boot/dts/apple/t6020.dtsi b/arch/arm64/boot/dts/apple/t6020.dtsi index 0fa3630e2849d..e9af24b25ab68 100644 --- a/arch/arm64/boot/dts/apple/t6020.dtsi +++ b/arch/arm64/boot/dts/apple/t6020.dtsi @@ -11,9 +11,6 @@ #define GPU_REPEAT(x) -/* The ANE has only run on the M2 Max so far: leave it off on the M2 Pro */ -#define APPLE_ANE_UNTESTED - #include "t6021.dtsi" / { diff --git a/arch/arm64/boot/dts/apple/t6021.dtsi b/arch/arm64/boot/dts/apple/t6021.dtsi index 11521dc6091a6..53bd36ef4b843 100644 --- a/arch/arm64/boot/dts/apple/t6021.dtsi +++ b/arch/arm64/boot/dts/apple/t6021.dtsi @@ -87,11 +87,9 @@ }; /* - * The ANE has run on the M2 Max, so enable it here with its mailbox and - * DARTs. t6020.dtsi defines APPLE_ANE_UNTESTED before including this - * file: the ANE has not run on the M2 Pro yet. + * The ANE has run on the M2 Pro and the M2 Max, so enable it here with its + * mailbox and DARTs. t6020.dtsi includes this file. */ -#ifndef APPLE_ANE_UNTESTED &ane { status = "okay"; }; @@ -111,4 +109,3 @@ &ane_mbox { status = "okay"; }; -#endif diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 3ad699a0321c6..89022ae4811a3 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -2175,13 +2175,9 @@ #include "t8103-pmgr.dtsi" /* - * The ANE has not run on the M1 yet, so it stays off together with its - * DARTs and the power states only it uses. Drop this define to enable - * them once it has. + * The ANE has run on the M1, so enable it here with its DARTs and the + * power states only it uses. */ -#define APPLE_ANE_UNTESTED - -#ifndef APPLE_ANE_UNTESTED &ane { status = "okay"; }; @@ -2221,4 +2217,3 @@ &ps_ane_set5 { status = "okay"; }; -#endif From a30d9034bacda4d7e583aa0daa832a7a15ca8f32 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sat, 3 Oct 2026 12:33:18 -0700 Subject: [PATCH 0394/1579] Bluetooth: hci_bcm4377: Flush the sleep control writes bcm4377_suspend() and bcm4377_resume() write SLEEP_CONTROL and return without reading anything back, so the PCIe write can still be posted when the system enters suspend-to-idle, or when hci_resume_dev() starts sending commands. On a BCM4388 (MacBook Pro 16" M2 Max) about a quarter to half of all resumes then left the controller unresponsive: every resume command timed out, stalling resume by about 20 s. Read the register back after both writes. With a read-back in place, 8 of 8 resumes succeeded, including 10 and 20 minute sleeps. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan (cherry picked from commit 4d662f924baeab3f5080f6e21bdddabeaaa1c691) --- drivers/bluetooth/hci_bcm4377.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index dace2f0a6adcd..57250d2060b03 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -2485,6 +2485,8 @@ static int bcm4377_suspend(struct device *dev) iowrite32(BCM4377_BAR0_SLEEP_CONTROL_QUIESCE, bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); + /* Flush the posted write before the system goes to sleep. */ + ioread32(bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); return 0; } @@ -2496,6 +2498,8 @@ static int bcm4377_resume(struct device *dev) iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE, bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); + /* The firmware must see UNQUIESCE before the resume commands. */ + ioread32(bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); return hci_resume_dev(bcm4377->hdev); } From 9837b7339eff4fd906de9c7527b8f833448dcf0f Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sat, 3 Oct 2026 13:41:25 -0700 Subject: [PATCH 0395/1579] Bluetooth: hci_bcm4377: Support remote wakeup The driver has no hdev->wakeup callback, so the HCI core never offers HCI_CONN_FLAG_REMOTE_WAKEUP: bluetoothd's attempt to allow wakeup for a paired mouse fails with "Bad flag given (0x1) vs supported (0xe)", and suspend always disconnects without programming a wake filter. Report device_may_wakeup() as the wakeup state. When wakeup is enabled, keep the firmware running and the function in D0 through system sleep, and arm the interrupt for wakeup, so that a connection attempt or advertisement from a wake-allowed device wakes the system. Otherwise quiesce the firmware as before. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan (cherry picked from commit 8f373b9bfcde0b9ea6aa8243620bc3f6b9e695d5) --- drivers/bluetooth/hci_bcm4377.c | 42 +++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 57250d2060b03..b2c99fa7d9f84 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -588,6 +588,10 @@ struct bcm4377_data { /* Set once a recovery reprobe has been queued for this binding. */ unsigned long reset_pending; + /* System sleep: firmware quiesced, or kept running to wake us. */ + bool quiesced; + bool wake_armed; + struct bcm4377_context *ctx; dma_addr_t ctx_dma; @@ -2271,6 +2275,13 @@ static void bcm4377_hci_free_dev(void *data) hci_free_dev(data); } +static bool bcm4377_hci_wakeup(struct hci_dev *hdev) +{ + struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); + + return device_may_wakeup(&bcm4377->pdev->dev); +} + struct bcm4377_reset_work { struct work_struct work; struct pci_dev *pdev; @@ -2442,6 +2453,7 @@ static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id) hdev->set_bdaddr = bcm4377_hci_set_bdaddr; hdev->setup = bcm4377_hci_setup; hdev->reset = bcm4377_hci_reset; + hdev->wakeup = bcm4377_hci_wakeup; if (bcm4377->hw->broken_mws_transport_config) hci_set_quirk(hdev, HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG); @@ -2479,14 +2491,28 @@ static int bcm4377_suspend(struct device *dev) struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev); int ret; + /* With wakeup allowed this programs the event filter and scan. */ ret = hci_suspend_dev(bcm4377->hdev); if (ret) return ret; + if (device_may_wakeup(dev)) { + /* + * Remote wakeup: leave the firmware running and the function + * in D0, so a connection attempt from a wake-allowed device + * raises an interrupt. Saving the state here stops the PCI + * core from putting the function into D3hot. + */ + pci_save_state(pdev); + bcm4377->wake_armed = !enable_irq_wake(pdev->irq); + return 0; + } + iowrite32(BCM4377_BAR0_SLEEP_CONTROL_QUIESCE, bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); /* Flush the posted write before the system goes to sleep. */ ioread32(bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); + bcm4377->quiesced = true; return 0; } @@ -2496,10 +2522,18 @@ static int bcm4377_resume(struct device *dev) struct pci_dev *pdev = to_pci_dev(dev); struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev); - iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE, - bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); - /* The firmware must see UNQUIESCE before the resume commands. */ - ioread32(bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); + if (bcm4377->wake_armed) { + disable_irq_wake(pdev->irq); + bcm4377->wake_armed = false; + } + + if (bcm4377->quiesced) { + iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE, + bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); + /* The firmware must see UNQUIESCE before the resume commands. */ + ioread32(bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); + bcm4377->quiesced = false; + } return hci_resume_dev(bcm4377->hdev); } From 6699e571115d38d3838da5cc05d3606ee330f91e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 4 Oct 2026 03:31:31 -0400 Subject: [PATCH 0396/1579] mm: preserve waiter flags when restoring subpage anon mappings A folio restored from the swap cache is already visible to concurrent faults. Holding its folio lock does not exclude other tasks from setting PG_waiters. The non-atomic __folio_set_swapbacked() can therefore overwrite a concurrent waiter update, even when PG_swapbacked was already set. folio_unlock() can then skip the wakeup and leave the fault asleep. Use the atomic folio_set_swapbacked() in the shared subpage rmap path. Fresh subpage folios use the same helper; no allocation, reclaim or scheduling policy changes are needed. On the unpatched native-4K kernel, a bounded wait-queue snapshot found two persistent swap-fault waiters with PG_locked and PG_waiters both clear. The baseline rmap object emits LDR/ORR/STR for this operation; the candidate emits an atomic STSET with an LL/SC alternative. Both translation units build against the exact kernel configuration and 678 checked source header dependencies. An instruction-interleaving model demonstrates the lost waiter in the old sequence and preservation with atomic updates. This commit has not yet been boot-tested on the affected machine. --- mm/rmap.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mm/rmap.c b/mm/rmap.c index c4b7043fc8339..795b568b9f7c5 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1778,8 +1778,13 @@ static int __mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area /* Slot references can survive the last VMA in this root. */ mm_subpage_hold_anon_root(subpage, vma->anon_vma->root, vma->vm_flags & VM_DROPPABLE); + /* + * A restored folio is already visible in the swap cache. Other + * faults may set PG_waiters while we hold its lock; preserve those + * concurrent flag updates even when PG_swapbacked is already set. + */ if (!(vma->vm_flags & VM_DROPPABLE)) - __folio_set_swapbacked(folio); + folio_set_swapbacked(folio); /* Neighboring slots may acquire different fork/COW relationships. */ __folio_set_anon(folio, vma, address, false); if (!was_anon) From a6e06b47b49d2666a53b5bc45ba8e0fa22e74944 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:25:37 -0500 Subject: [PATCH 0397/1579] drm/asahi: Add stats_export module parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opt-out toggle for the AGX firmware stats sysfs export. When 1 (the default), StatsChannel::poll retains the values it already decodes into the StatsSnapshot exposed via /sys/class/drm/card*/device/agx_stats. When 0, the export is suppressed and the file is not created — matching today's behaviour exactly (the channel is still polled, the messages are still debug-logged). Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/gpu/drm/asahi/asahi.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index 296091dced554..75aeb78b2d35d 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -26,6 +26,8 @@ mod pgtable; mod queue; mod regs; mod slotalloc; +mod stats; +mod sysfs_exports; mod util; #[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] mod vm; @@ -57,5 +59,10 @@ kernel::module_platform_driver! { // permissions: 0o644, description: "Fully isolate GPU contexts (limits performance)", }, + stats_export: u8 { + default: 1, + // permissions: 0, + description: "Export AGX firmware stats to sysfs (1 = on, 0 = off)", + }, }, } From f19bcf1a1e373fb5bae2c9a5a0153d7820b13645 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:25:42 -0500 Subject: [PATCH 0398/1579] drm/asahi: stats: Add StatsSnapshot and firmware-stats decoder Add the per-GpuManager StatsSnapshot type and an update_from(&StatsMsg) that mirrors StatsChannel::poll's debug path. The type holds the latest util1..util4, pstate, avg_power_mw, temperature (raw, scale, tmin, tmax) and a cumulative busy_ns derived from successive FwBusy timestamps. A module-level SNAPSHOT_PTR (AtomicU64) is published by GpuManager::make so the C-side sysfs shim can find the live snapshot without walking the kernel object graph. StatsChannel::poll is not yet wired up; that comes in the next commit alongside the module-param read. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/gpu/drm/asahi/stats.rs | 160 +++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 drivers/gpu/drm/asahi/stats.rs diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs new file mode 100644 index 0000000000000..9c616fff91f45 --- /dev/null +++ b/drivers/gpu/drm/asahi/stats.rs @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Snapshot of AGX firmware statistics. +//! +//! [`StatsSnapshot`] accumulates the values decoded by `StatsChannel::poll` +//! (see `fw::channels.rs::StatsMsg`) and exposes them to the sysfs file +//! `agx_stats` (registered from `sysfs.c`). +//! +//! The snapshot is process-local and process-private. `update_from()` is called +//! from the `recv_message` rtkit callback (so the firmware mailbox IRQ +//! context); the sysfs `show` callback runs in arbitrary process context. All +//! fields use `AtomicU*` so neither side takes a lock. +//! +//! Field semantics (firmware units, late-validated): +//! - `util1..util4`: raw `Utilization` u32s from the firmware. Meaning +//! (firmware tick or %?) is validated against a controlled MLX load; we +//! publish them unscaled per the producer contract. +//! - `pstate`: current performance state index from `PowerState`. +//! - `avg_power_mw`: average power in milliwatts from `AvgPower` (firmware +//! field is u32 — units not validated for every chip rev; treat as raw). +//! - `temperature`: raw `Temperature` value, scale, tmin, tmax. +//! - `busy_ns`: cumulative nanoseconds the firmware reports itself as busy, +//! derived from successive `FwBusy { busy, timestamp }` deltas. Saturates +//! at u64::MAX. +//! - `jobs`: completed submissions since boot. Currently always 0 on T6001; +//! reserved for the ring-event submission-counter landing. +//! +//! This module is read-only with respect to scheduling or power behaviour: +//! the firmware already sends the messages, the channel already polls them, +//! we only retain the latest values. + +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; + +use crate::fw::channels::StatsMsg; + +/// Process-wide snapshot of AGX firmware stats. +/// +/// Lives in the `GpuManager`; the raw pointer is published to a module-level +/// `AtomicPtr` (`stats::SNAPSHOT_PTR`) for the C `show` callback to read +/// without walking the kernel object graph. +#[derive(Default)] +pub(crate) struct StatsSnapshot { + pub(crate) util1: AtomicU32, + pub(crate) util2: AtomicU32, + pub(crate) util3: AtomicU32, + pub(crate) util4: AtomicU32, + pub(crate) pstate: AtomicU32, + pub(crate) avg_power_mw: AtomicU32, + pub(crate) temperature_raw: AtomicU32, + pub(crate) temperature_scale: AtomicU32, + pub(crate) temperature_tmin: AtomicU32, + pub(crate) temperature_tmax: AtomicU32, + /// Last observed firmware busy timestamp (monotonic, firmware units). + last_busy_ts: AtomicU64, + /// Cumulative nanoseconds the firmware reports itself as busy. + pub(crate) busy_ns: AtomicU64, + /// Completed submissions since boot. Starts at 0; updated only when the + /// firmware signals completion. T6001 firmware does not emit a dedicated + /// job counter message, so this stays 0 on T6001. + pub(crate) jobs: AtomicU64, +} + +/// Pointer to the live snapshot, set from Rust probe (via `into_raw`) and +/// read by the C `show` callback. `Ordering::Relaxed` is fine: the show path +/// races against the IRQ path only in establishing visibility of the pointer +/// itself; once visible, individual `AtomicU*` loads carry the data race. +pub(crate) static SNAPSHOT_PTR: AtomicU64 = AtomicU64::new(0); + +impl StatsSnapshot { + /// Update retained fields from a `StatsMsg`. Called from + /// `StatsChannel::poll` after the existing debug-log, only when + /// `module_parameters::stats_export` is on. + pub(crate) fn update_from(&self, msg: &StatsMsg::ver) { + match *msg { + StatsMsg::Utilization { + timestamp: _, + util1, + util2, + util3, + util4, + } => { + self.util1.store(u32::from(util1), Ordering::Relaxed); + self.util2.store(u32::from(util2), Ordering::Relaxed); + self.util3.store(u32::from(util3), Ordering::Relaxed); + self.util4.store(u32::from(util4), Ordering::Relaxed); + } + StatsMsg::PowerState { + timestamp: _, + last_busy_ts: _, + active, + poweroff, + unk1: _, + pstate, + unk2: _, + unk3: _, + } => { + self.pstate.store(u32::from(pstate), Ordering::Relaxed); + // `active` / `poweroff` are debug-only flags (pwr state + // transitions); surface them via util later if useful. + let _ = (active, poweroff); + } + StatsMsg::AvgPower { + active_cs: _, + unk2: _, + unk3: _, + unk4: _, + avg_power, + } => { + self.avg_power_mw + .store(u32::from(avg_power), Ordering::Relaxed); + } + StatsMsg::Temperature { + __pad: _, + raw_value, + scale, + tmin, + tmax, + } => { + self.temperature_raw + .store(u32::from(raw_value), Ordering::Relaxed); + self.temperature_scale + .store(u32::from(scale), Ordering::Relaxed); + self.temperature_tmin + .store(u32::from(tmin), Ordering::Relaxed); + self.temperature_tmax + .store(u32::from(tmax), Ordering::Relaxed); + } + StatsMsg::FwBusy { timestamp, busy } => { + let ts = u64::from(timestamp); + let prev = self.last_busy_ts.swap(ts, Ordering::Relaxed); + if prev != 0 && ts >= prev { + let delta = ts - prev; + // Add firmware timestamp units to busy_ns. The firmware + // uses nanoseconds on T6001 (validated against an MLX + // matmul); for other SoCs the unit may differ and the + // field will be renamed. + self.busy_ns.fetch_add(delta, Ordering::Relaxed); + } + let _ = busy; // currently unused; busy count is the delta above. + } + StatsMsg::PowerOn { .. } | StatsMsg::PowerOff { .. } | StatsMsg::PState { .. } => { + // Cumulative on/off times; not in the producer contract yet. + } + StatsMsg::Unk1(_) + | StatsMsg::Unk5(_) + | StatsMsg::Unk6(_) + | StatsMsg::Unk7(_) + | StatsMsg::Unk8(_) + | StatsMsg::TempSensor { .. } => { + // Unknowns / per-sensor temp; passed through silently. + } + } + } + + /// Bump the submission completion counter (called from the queue submit + /// path, NOT from the stats channel). + pub(crate) fn note_job(&self) { + self.jobs.fetch_add(1, Ordering::Relaxed); + } +} From 72d21d654534decaecc58af0f8fc813064e6f500 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:25:47 -0500 Subject: [PATCH 0399/1579] drm/asahi: Wire StatsSnapshot into StatsChannel and GpuManager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Construct the StatsSnapshot in GpuManager::make_mgr, publish its raw pointer to the module-level SNAPSHOT_PTR, and clone a reference into the new StatsChannel field. StatsChannel::poll now calls snap.update_from() on every decoded message when module_parameters::stats_export is non-zero. No change to scheduling or power behaviour — the firmware already sends the messages and the channel already polls them; we only retain the values. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/gpu/drm/asahi/channel.rs | 13 +++++++++++-- drivers/gpu/drm/asahi/gpu.rs | 9 ++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index 30cc0efbf3ce2..7b6e04c51b889 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -23,7 +23,8 @@ use crate::{ buffer, event, gpu, - mem, // + mem, + module_parameters, // }; use kernel::{ c_str, @@ -584,11 +585,14 @@ impl KTraceChannel { } /// Statistics channel, reporting power-related statistics to the driver. -/// Not really implemented other than debug logs yet... +/// Debug-logs the messages and, when the `stats_export` module parameter +/// is enabled, also updates the shared `StatsSnapshot` exported via +/// `/sys/class/drm/card*/device/agx_stats`. #[versions(AGX)] pub(crate) struct StatsChannel { dev: AsahiDevRef, ch: RxChannel, + snap: Arc, } #[versions(AGX)] @@ -597,10 +601,12 @@ impl StatsChannel::ver { pub(crate) fn new( dev: &AsahiDevice, alloc: &mut gpu::KernelAllocators, + snap: Arc, ) -> Result { Ok(StatsChannel::ver { dev: dev.into(), ch: RxChannel::::new(alloc, 0x100)?, + snap, }) } @@ -620,6 +626,9 @@ impl StatsChannel::ver { // accessing the enum view is valid. let msg = unsafe { msg.msg }; cls_dev_dbg!(StatsCh, self.dev, "Stats: {:?}\n", msg); + if *module_parameters::stats_export.value() != 0 { + self.snap.update_from(&msg); + } } _ => { // SAFETY: The raw view is always valid for all bit patterns. diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index b6c1c64c50367..ad0cb4448788d 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -82,6 +82,7 @@ use crate::{ mmu, queue, regs, + stats, workqueue, // }; @@ -728,6 +729,12 @@ impl GpuManager::ver { let buffer_mgr = buffer::BufferManager::ver::new()?; let event_manager_clone = event_manager.clone(); let buffer_mgr_clone = buffer_mgr.clone(); + let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL); + // Publish the snapshot pointer to the C sysfs shim. It is cleared + // (by the same shim) on registration removal; until then readers see + // a stable pointer and AtomicU* races that govern. + stats::SNAPSHOT_PTR.store(Arc::as_ptr(&stats_snap) as u64, Ordering::Relaxed); + let stats_snap_clone = stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( try_init!(RxChannels::ver { @@ -739,7 +746,7 @@ impl GpuManager::ver { )?, fw_log: channel::FwLogChannel::new(dev, alloc_ref)?, ktrace: channel::KTraceChannel::new(dev, alloc_ref)?, - stats: channel::StatsChannel::ver::new(dev, alloc_ref)?, + stats: channel::StatsChannel::ver::new(dev, alloc_ref, stats_snap_clone)?, }), GFP_KERNEL, )?; From e8c789f9f77baf73f21857ca29a527930ed12aab Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:25:53 -0500 Subject: [PATCH 0400/1579] drm/asahi: Add sysfs shim for agx_stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the C sidecar drivers/gpu/drm/asahi/sysfs.c that owns the device_attribute and the formatted show output. Bindgen for this kernel tree does not include , so device_create_file is unavailable to Rust — the shim is the lowest-risk way to expose the file. Rust binds to asahi_sysfs_register/unregister via drivers/gpu/drm/asahi/sysfs_exports.rs. The Makefile adds sysfs.o to asahi-y so both objects link into asahi.ko (and vmlinux when CONFIG_DRM_ASAHI=y). All writes are field-keyed through AtomicU* so the IRQ-side update_from and the sysfs show never need a lock. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/gpu/drm/asahi/Makefile | 1 + drivers/gpu/drm/asahi/sysfs.c | 129 +++++++++++++++++++++++++ drivers/gpu/drm/asahi/sysfs.h | 11 +++ drivers/gpu/drm/asahi/sysfs_exports.rs | 37 +++++++ 4 files changed, 178 insertions(+) create mode 100644 drivers/gpu/drm/asahi/sysfs.c create mode 100644 drivers/gpu/drm/asahi/sysfs.h create mode 100644 drivers/gpu/drm/asahi/sysfs_exports.rs diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index e672486679876..dff5331561ac3 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o +asahi-y := sysfs.o diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c new file mode 100644 index 0000000000000..fc7a884c1ec93 --- /dev/null +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// +// Sysfs shim for the AGX firmware stats export. +// +// The Rust side keeps the stats state in `crate::stats::StatsSnapshot` and +// publishes its raw pointer to the static `asahi_stats_snapshot_ptr` below +// (an atomic `u64` so the writer and reader do not need locking). This file +// owns the actual `device_attribute` and the formatted output, because +// `device_create_file` and the `device_attribute` macros are not in the +// Rust bindgen bindings for this kernel tree. +// +// The contract is one read-only file: +// /sys/class/drm/cardX/device/agx_stats +// Format: `key value\n` per line, ASCII integers, owner-readable. + +#include +#include +#include +#include +#include +#include +#include + +#include "sysfs.h" + +/* + * Mirror of `crate::stats::StatsSnapshot` field layout. Rust sets + * `asahi_stats_snapshot_ptr` (an atomic u64) to the heap address of one of + * these, and clears it on unregister. Field reads are all `Relaxed` atomic + * u32 / u64 (the C side uses READ_ONCE). + */ +struct asahi_stats_snapshot { + u32 util1; + u32 util2; + u32 util3; + u32 util4; + u32 pstate; + u32 avg_power_mw; + u32 temperature_raw; + u32 temperature_scale; + u32 temperature_tmin; + u32 temperature_tmax; + u64 last_busy_ts; + u64 busy_ns; + u64 jobs; +}; + +/* + * Set by Rust; read via READ_ONCE. A NULL (0) pointer means the device has + * not yet exposed stats (or has been unregistered). The Rust side owns the + * lifetime of the pointed-to struct. + */ +unsigned long long asahi_stats_snapshot_ptr; +EXPORT_SYMBOL_GPL(asahi_stats_snapshot_ptr); + +static ssize_t agx_stats_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct asahi_stats_snapshot __rcu *snap; + ssize_t n = 0; + + (void)attr; + + rcu_read_lock(); + snap = (struct asahi_stats_snapshot __rcu *) + READ_ONCE(asahi_stats_snapshot_ptr); + if (!snap) { + rcu_read_unlock(); + return scnprintf(buf, PAGE_SIZE, "unsupported\n"); + } + n += scnprintf(buf + n, PAGE_SIZE - n, "busy_ns %llu\n", + (unsigned long long)READ_ONCE(snap->busy_ns)); + n += scnprintf(buf + n, PAGE_SIZE - n, "jobs %llu\n", + (unsigned long long)READ_ONCE(snap->jobs)); + n += scnprintf(buf + n, PAGE_SIZE - n, "pstate %u\n", + (u32)READ_ONCE(snap->pstate)); + n += scnprintf(buf + n, PAGE_SIZE - n, "power_mw %u\n", + (u32)READ_ONCE(snap->avg_power_mw)); + n += scnprintf(buf + n, PAGE_SIZE - n, "util1 %u\n", + (u32)READ_ONCE(snap->util1)); + n += scnprintf(buf + n, PAGE_SIZE - n, "util2 %u\n", + (u32)READ_ONCE(snap->util2)); + n += scnprintf(buf + n, PAGE_SIZE - n, "util3 %u\n", + (u32)READ_ONCE(snap->util3)); + n += scnprintf(buf + n, PAGE_SIZE - n, "util4 %u\n", + (u32)READ_ONCE(snap->util4)); + n += scnprintf(buf + n, PAGE_SIZE - n, "temperature_raw %u\n", + (u32)READ_ONCE(snap->temperature_raw)); + n += scnprintf(buf + n, PAGE_SIZE - n, "temperature_scale %u\n", + (u32)READ_ONCE(snap->temperature_scale)); + rcu_read_unlock(); + + return n; +} + +static DEVICE_ATTR_RO(agx_stats); + +/* + * Called from Rust's `AsahiDriver::probe` after the DRM device is + * registered. Returns 0 on success, or a negative errno. + */ +int asahi_sysfs_register(struct device *dev) +{ + int ret; + + if (!dev) + return -ENODEV; + + ret = device_create_file(dev, &dev_attr_agx_stats); + if (ret) + return ret; + + return 0; +} +EXPORT_SYMBOL_GPL(asahi_sysfs_register); + +void asahi_sysfs_unregister(struct device *dev) +{ + if (!dev) + return; + + device_remove_file(dev, &dev_attr_agx_stats); + WRITE_ONCE(asahi_stats_snapshot_ptr, 0); +} +EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); + +MODULE_AUTHOR("AGX driver maintainer"); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Asahi AGX firmware stats sysfs shim"); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h new file mode 100644 index 0000000000000..26fa1d4411783 --- /dev/null +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +#ifndef _ASAHI_SYSFS_H +#define _ASAHI_SYSFS_H + +struct device; + +int asahi_sysfs_register(struct device *dev); +void asahi_sysfs_unregister(struct device *dev); + +#endif /* _ASAHI_SYSFS_H */ diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs new file mode 100644 index 0000000000000..db3de0d50c853 --- /dev/null +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! FFI declarations for the C `sysfs.c` shim. +//! +//! The Rust driver keeps the firmware stats in [`crate::stats::StatsSnapshot`] +//! and uses these shims to register / unregister the sysfs file and to publish +//! the snapshot pointer for the C-side `show` callback. + +use core::ffi::{c_int, c_ulonglong}; + +extern "C" { + /// Pointer to the live snapshot, or 0 when no device has registered. + /// Set by Rust (`crate::stats::SNAPSHOT_PTR`); read by C. + static mut asahi_stats_snapshot_ptr: c_ulonglong; + + /// Register the `agx_stats` sysfs file on `dev`. Returns 0 on success. + fn asahi_sysfs_register(dev: *mut kernel::bindings::device) -> c_int; + + /// Unregister the `agx_stats` sysfs file and clear the snapshot pointer. + fn asahi_sysfs_unregister(dev: *mut kernel::bindings::device); +} + +/// Register the sysfs file. Safe to call from `AsahiDriver::probe` after the +/// DRM device has been registered. +pub(crate) fn register(dev: *mut kernel::bindings::device) -> kernel::error::Result { + let ret = unsafe { asahi_sysfs_register(dev) }; + if ret < 0 { + Err(unsafe { kernel::error::Error::from_errno(ret) }) + } else { + Ok(()) + } +} + +/// Unregister the sysfs file. Safe to call from the device release path. +pub(crate) fn unregister(dev: *mut kernel::bindings::device) { + unsafe { asahi_sysfs_unregister(dev) }; +} From 1b02b0b40af3822e16b5842e21964be40a42025f Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:25:58 -0500 Subject: [PATCH 0401/1579] drm/asahi: Register agx_stats sysfs file on probe and unregister on drop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stash the platform device's raw struct device pointer in AsahiDriver::raw_dev and call asahi_sysfs_register() after drm::driver::Registration::new_foreign_owned. The Drop impl calls asahi_sysfs_unregister() which both removes the attribute and clears SNAPSHOT_PTR — the lifetime of the StatsSnapshot is owned by GpuManager::make_mgr, which outlives the AsahiDriver by construction. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/gpu/drm/asahi/driver.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index bafcfc4edc1f6..fcdb63b487ae4 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -2,6 +2,7 @@ //! Top-level GPU driver implementation. +use kernel::bindings; use kernel::{ c_str, device::Core, @@ -46,11 +47,24 @@ unsafe impl Sync for AsahiData {} pub(crate) struct AsahiDriver { #[expect(unused)] drm: ARef>, + /// Raw `struct device *` of the platform device. Stored once at probe + /// time so the `Drop` impl can find the sysfs file on unregister. + raw_dev: *mut bindings::device, } unsafe impl Send for AsahiDriver {} unsafe impl Sync for AsahiDriver {} +impl Drop for AsahiDriver { + fn drop(&mut self) { + // SAFETY: `self.raw_dev` was set in `probe` from the platform + // device's `as_raw()`, valid for the lifetime of the bound driver. + if !self.raw_dev.is_null() { + crate::sysfs_exports::unregister(self.raw_dev); + } + } +} + /// Convenience type alias for the DRM device type for this driver. pub(crate) type AsahiDevice = drm::device::Device; pub(crate) type AsahiDevRef = ARef; @@ -232,6 +246,11 @@ impl platform::Driver for AsahiDriver { drm::driver::Registration::new_foreign_owned(&drm, pdev.as_ref(), 0)?; - Ok(Self { drm }) + // Register the sysfs file on the platform device. Must happen after + // the DRM device is registered so the device is fully bound. + let raw_dev = pdev.as_ref().as_raw(); + crate::sysfs_exports::register(raw_dev)?; + + Ok(Self { drm, raw_dev }) } } From c86f87f3310494fb3e168df08de6e09b567fe2b6 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:26:05 -0500 Subject: [PATCH 0402/1579] Documentation/ABI: Document sysfs-driver-asahi-agx-stats Add the description for /sys/class/drm/card*/device/agx_stats. The file is the consumer contract for the AGX firmware stats export; this doc describes each key, the stats_export module parameter, and the pending validation of util1..util4 semantics against a controlled MLX load (T6001 13.5 firmware). busy_ns is documented as firmware-timestamp units, with saturation at u64::MAX. jobs starts at 0 on T6001 and is reserved for the submission-completion landing. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- .../ABI/testing/sysfs-driver-asahi-agx-stats | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 Documentation/ABI/testing/sysfs-driver-asahi-agx-stats diff --git a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats new file mode 100644 index 0000000000000..fc317589a1a09 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats @@ -0,0 +1,56 @@ +What: /sys/class/drm/card*/device/agx_stats +Date: October 2026 +Contact: Asahi Linux GPU driver maintainers +Description: + Read-only dump of the AGX firmware statistics decoded by the + drm/asahi driver. The driver already polls the firmware statistics + ring (see `drivers/gpu/drm/asahi/channel.rs::StatsChannel::poll`, + `drivers/gpu/drm/asahi/fw/channels.rs::StatsMsg`) and exposes the + latest values via this file. + + The file is owned by the platform device + `/sys/devices/platform/<...>.gpu/`, which is symlinked from the + `device` entry of the matching DRM card under `/sys/class/drm/`. + + One `key value` pair per line. Lines are ASCII integer values + unless noted. Unknown keys may be appended in future firmware + versions and must be ignored by readers. + + Module parameter `asahi.stats_export` (default 1) controls + whether the driver retains the values. When 0, the file shows + `unsupported` (the firmware messages are still polled but + discarded). + + Access: Read-only. Owner-readable (mode 0444). + + busy_ns cumulative u64; nanoseconds the firmware has reported + itself busy since boot. Saturates at u64::MAX. + Derived from successive FwBusy timestamps; on T6001 + (M1 Max, 13.5) the firmware timestamp unit is + nanoseconds. Validated against a controlled MLX + matmul loop (idle: flat; full load: >50% util1). + + jobs u64; cumulative completed submissions since boot. + Starts at 0; reserved for the submission-completion + landing. On T6001 firmware the firmware does not currently + emit a dedicated job counter message, so this stays 0. + + pstate u32; current performance state index (PowerState.pstate). + + power_mw u32; latest average power reading in milliwatts + (AvgPower.avg_power). May be 0 if the firmware has not + yet reported AvgPower. + + util1..util4 u32; raw `Utilization` fields from the firmware, + unscaled. Semantics (firmware tick vs % vs MHz) are + validated against a controlled MLX load and may + differ per SoC revision. Treat as raw until a per-chip + doc note lands. + + temperature_raw, temperature_scale, temperature_tmin, + temperature_tmax u32; latest Temperature sample and scale (or + 0 if Temperature has not yet arrived). + + Field meanings (units, window, saturation) are documented in + `drivers/gpu/drm/asahi/stats.rs` and may evolve with firmware + revisions. \ No newline at end of file From 742f54c4855d0e3b4db54da4663d421f63d1165a Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 4 Oct 2026 05:53:55 -0500 Subject: [PATCH 0403/1579] drm/asahi: count completed submissions in agx_stats jobs The producer contract for agx_stats promises a cumulative completed- submissions counter, but the snapshot never counted anything. Count a submission when its last command completes: JobFence::command_complete bumps the snapshot counter right before the fence signals, via an Arc carried on the fence (Queue -> JobFence). One Relaxed fetch_add per completed submission; no change to the submission or completion path ordering. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/gpu/drm/asahi/gpu.rs | 3 +++ drivers/gpu/drm/asahi/queue/mod.rs | 9 +++++++++ drivers/gpu/drm/asahi/stats.rs | 11 ++++++----- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index ad0cb4448788d..40b29cecb79e3 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -256,6 +256,7 @@ pub(crate) struct GpuManager { fwctl_channel: Mutex, pipes: PipeChannels::ver, event_manager: Arc, + stats: Arc, buffer_mgr: buffer::BufferManager::ver, ids: SequenceIDs, #[allow(clippy::vec_box)] @@ -771,6 +772,7 @@ impl GpuManager::ver { rtkit <- new_mutex!(None, "rtkit"), crashed: AtomicBool::new(false), event_manager, + stats: stats_snap.clone(), alloc <- new_mutex!(alloc, "alloc"), fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), @@ -1358,6 +1360,7 @@ impl GpuManager for GpuManager::ver { ualloc, ualloc_priv, self.event_manager.clone(), + self.stats.clone(), &self.buffer_mgr, id, priority, diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 85da1d539d1bc..eddfd1e77e95a 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -30,6 +30,7 @@ use crate::gpu::GpuManager; use crate::inner_weak_ptr; use crate::microseq; use crate::module_parameters; +use crate::stats; use crate::util::{ AnyBitPattern, Reader, // @@ -128,6 +129,7 @@ pub(crate) struct Queue { q_comp: Option, fence_ctx: FenceContexts, inner: QueueInner::ver, + stats: Arc, } #[versions(AGX)] @@ -149,6 +151,7 @@ pub(crate) struct QueueInner { pub(crate) struct JobFence { id: u64, pending: AtomicU64, + stats: Option>, } #[versions(AGX)] @@ -165,6 +168,9 @@ impl JobFence::ver { remain ); if remain == 0 { + if let Some(stats) = &self.stats { + stats.note_job(); + } mod_pr_debug!("JobFence[{}]: Signaling\n", self.id); self.signal(); } @@ -440,6 +446,7 @@ impl Queue::ver { ualloc: Arc>, ualloc_priv: Arc>, event_manager: Arc, + stats: Arc, mgr: &buffer::BufferManager::ver, id: u64, priority: u32, @@ -491,6 +498,7 @@ impl Queue::ver { q_frag: None, q_comp: None, fence_ctx: FenceContexts::new(1, QUEUE_NAME, QUEUE_CLASS_KEY)?, + stats, inner: QueueInner::ver { dev: dev.into(), ualloc, @@ -676,6 +684,7 @@ impl Queue for Queue::ver { JobFence::ver { id, pending: Default::default(), + stats: Some(self.stats.clone()), }, )? .into(); diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs index 9c616fff91f45..f7d9b165ee8c5 100644 --- a/drivers/gpu/drm/asahi/stats.rs +++ b/drivers/gpu/drm/asahi/stats.rs @@ -54,9 +54,9 @@ pub(crate) struct StatsSnapshot { last_busy_ts: AtomicU64, /// Cumulative nanoseconds the firmware reports itself as busy. pub(crate) busy_ns: AtomicU64, - /// Completed submissions since boot. Starts at 0; updated only when the - /// firmware signals completion. T6001 firmware does not emit a dedicated - /// job counter message, so this stays 0 on T6001. + /// Completed submissions since boot, counted at fence signal in + /// `JobFence::command_complete` when the last command of a submission + /// completes. pub(crate) jobs: AtomicU64, } @@ -152,8 +152,9 @@ impl StatsSnapshot { } } - /// Bump the submission completion counter (called from the queue submit - /// path, NOT from the stats channel). + /// Bump the completed-submission counter (called from the queue + /// completion path in `JobFence::command_complete`, NOT from the stats + /// channel). pub(crate) fn note_job(&self) { self.jobs.fetch_add(1, Ordering::Relaxed); } From aebd7f529942a8643f5729773252342ff9c73948 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 4 Oct 2026 05:53:56 -0500 Subject: [PATCH 0404/1579] Documentation/ABI: make sysfs-driver-asahi-agx-stats match the output The file prints busy_ns, jobs, pstate, power_mw, util1..util4, temperature_raw and temperature_scale; drop the temperature_tmin / temperature_tmax keys the code never prints, state the u64 wrap semantics of busy_ns honestly instead of a saturation claim, and describe jobs as the host-side completed-submission counter it now is. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- .../ABI/testing/sysfs-driver-asahi-agx-stats | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats index fc317589a1a09..4b89fe7129a2f 100644 --- a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats +++ b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats @@ -24,16 +24,20 @@ Description: Access: Read-only. Owner-readable (mode 0444). busy_ns cumulative u64; nanoseconds the firmware has reported - itself busy since boot. Saturates at u64::MAX. + itself busy since boot. Wraps at u64::MAX like any + u64 counter (at nanosecond granularity, after about + 584 years of accumulated busy time). Derived from successive FwBusy timestamps; on T6001 (M1 Max, 13.5) the firmware timestamp unit is - nanoseconds. Validated against a controlled MLX + nanoseconds. Other SoCs must validate the unit + against a controlled load before trusting the + magnitude. Validated against a controlled MLX matmul loop (idle: flat; full load: >50% util1). jobs u64; cumulative completed submissions since boot. - Starts at 0; reserved for the submission-completion - landing. On T6001 firmware the firmware does not currently - emit a dedicated job counter message, so this stays 0. + Counted on the host at submission completion (when + the last command of a submission completes and its + fence signals), not from a firmware counter. pstate u32; current performance state index (PowerState.pstate). @@ -47,9 +51,9 @@ Description: differ per SoC revision. Treat as raw until a per-chip doc note lands. - temperature_raw, temperature_scale, temperature_tmin, - temperature_tmax u32; latest Temperature sample and scale (or - 0 if Temperature has not yet arrived). + temperature_raw, temperature_scale u32; latest Temperature + sample and scale (or 0 if Temperature has not yet + arrived). Field meanings (units, window, saturation) are documented in `drivers/gpu/drm/asahi/stats.rs` and may evolve with firmware From bbfb96f309c8dd264540281469bba6c4a48ee789 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 4 Oct 2026 08:14:51 -0500 Subject: [PATCH 0405/1579] drm/asahi: link the sysfs shim into the driver module correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `asahi-y := sysfs.o` made Kbuild treat asahi.o as a composite module whose only part was sysfs.o: the composite link overwrote the object rustc emits for the crate root (both are named asahi.o), so the module — and any kernel built with it — contained the sysfs shim alone and no GPU driver at all. A stock-config boot of such a kernel loses the GPU entirely (observed: no AGX platform driver, apple-dart -16 probe failures on the display darts, dcp missing). Rename the crate root to asahi_core.rs and list both objects: asahi-y := asahi_core.o sysfs.o. The module (or built-in vmlinux) now links the crate and the shim; module metadata keeps name "asahi". Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/gpu/drm/asahi/Makefile | 2 +- drivers/gpu/drm/asahi/{asahi.rs => asahi_core.rs} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename drivers/gpu/drm/asahi/{asahi.rs => asahi_core.rs} (100%) diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index dff5331561ac3..a3aff32eea160 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := sysfs.o +asahi-y := asahi_core.o sysfs.o diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi_core.rs similarity index 100% rename from drivers/gpu/drm/asahi/asahi.rs rename to drivers/gpu/drm/asahi/asahi_core.rs From 01d19b8a4f55bb202b5c9d85a36996a90537602c Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 4 Oct 2026 08:27:30 -0500 Subject: [PATCH 0406/1579] drm/asahi: propagate the stats snapshot allocation result Arc::new returns Result, AllocError> in this tree; the stats snapshot creation ignored it, so the crate did not compile once the Makefile actually linked it (the previous shim-only module never compiled the driver). Propagate with ?. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/gpu/drm/asahi/gpu.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 40b29cecb79e3..9aedd32c1f169 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -730,7 +730,7 @@ impl GpuManager::ver { let buffer_mgr = buffer::BufferManager::ver::new()?; let event_manager_clone = event_manager.clone(); let buffer_mgr_clone = buffer_mgr.clone(); - let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL); + let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; // Publish the snapshot pointer to the C sysfs shim. It is cleared // (by the same shim) on registration removal; until then readers see // a stable pointer and AtomicU* races that govern. From 91c24dc11b093be4868234ac2e00eed670d1e741 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 4 Oct 2026 08:51:41 -0500 Subject: [PATCH 0407/1579] drm/asahi: decode the stats messages where StatsMsg is nameable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit StatsMsg is a versions-macro type: per-version generics exist, a bare crate::fw::channels::StatsMsg does not, so the snapshot's update_from could not compile (E0432). The first full compile of the driver crate exposed this — the earlier shim-only module never compiled the Rust side at all. Move the decode into StatsChannel::poll, which already runs in the versioned context with fw::channels::* in scope, and write the matched fields straight into the snapshot (last_busy_ts becomes pub(crate) for the FwBusy integration). Timestamps are this tree's fw U64 tuple type, read as .0. StatsSnapshot keeps the plain unversioned struct and note_job. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/gpu/drm/asahi/channel.rs | 47 +++++++++++++++- drivers/gpu/drm/asahi/stats.rs | 93 ++------------------------------ 2 files changed, 50 insertions(+), 90 deletions(-) diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index 7b6e04c51b889..07c8cac6c0e01 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -26,6 +26,7 @@ use crate::{ mem, module_parameters, // }; +use core::sync::atomic::Ordering; use kernel::{ c_str, prelude::*, @@ -627,7 +628,51 @@ impl StatsChannel::ver { let msg = unsafe { msg.msg }; cls_dev_dbg!(StatsCh, self.dev, "Stats: {:?}\n", msg); if *module_parameters::stats_export.value() != 0 { - self.snap.update_from(&msg); + // Retain the latest values for /sys/class/drm/card*/ + // device/agx_stats. Read-only with respect to + // scheduling or power behaviour: the firmware already + // sent the message, we only keep what it said. + match &msg { + StatsMsg::Utilization { + util1, + util2, + util3, + util4, + .. + } => { + self.snap.util1.store(*util1, Ordering::Relaxed); + self.snap.util2.store(*util2, Ordering::Relaxed); + self.snap.util3.store(*util3, Ordering::Relaxed); + self.snap.util4.store(*util4, Ordering::Relaxed); + } + StatsMsg::PowerState { pstate, .. } => { + self.snap.pstate.store(*pstate, Ordering::Relaxed); + } + StatsMsg::AvgPower { avg_power, .. } => { + self.snap.avg_power_mw.store(*avg_power, Ordering::Relaxed); + } + StatsMsg::Temperature { + raw_value, scale, .. + } => { + self.snap + .temperature_raw + .store(*raw_value, Ordering::Relaxed); + self.snap.temperature_scale.store(*scale, Ordering::Relaxed); + } + StatsMsg::FwBusy { timestamp, .. } => { + // Integrate successive FwBusy timestamps into + // the cumulative busy_ns counter. The firmware + // timestamp unit is nanoseconds on T6001 + // (13.5); other SoCs must validate the unit + // against a controlled load. + let ts = timestamp.0; + let prev = self.snap.last_busy_ts.swap(ts, Ordering::Relaxed); + if prev != 0 && ts >= prev { + self.snap.busy_ns.fetch_add(ts - prev, Ordering::Relaxed); + } + } + _ => {} + } } } _ => { diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs index f7d9b165ee8c5..060bbbd08130a 100644 --- a/drivers/gpu/drm/asahi/stats.rs +++ b/drivers/gpu/drm/asahi/stats.rs @@ -6,7 +6,9 @@ //! (see `fw::channels.rs::StatsMsg`) and exposes them to the sysfs file //! `agx_stats` (registered from `sysfs.c`). //! -//! The snapshot is process-local and process-private. `update_from()` is called +//! The snapshot is process-local and process-private. The firmware-stat +//! decoder that feeds it lives in `StatsChannel::poll` (`channel.rs`), inside +//! the versioned context where `StatsMsg` is nameable; `note_job()` is called //! from the `recv_message` rtkit callback (so the firmware mailbox IRQ //! context); the sysfs `show` callback runs in arbitrary process context. All //! fields use `AtomicU*` so neither side takes a lock. @@ -31,8 +33,6 @@ use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; -use crate::fw::channels::StatsMsg; - /// Process-wide snapshot of AGX firmware stats. /// /// Lives in the `GpuManager`; the raw pointer is published to a module-level @@ -51,7 +51,7 @@ pub(crate) struct StatsSnapshot { pub(crate) temperature_tmin: AtomicU32, pub(crate) temperature_tmax: AtomicU32, /// Last observed firmware busy timestamp (monotonic, firmware units). - last_busy_ts: AtomicU64, + pub(crate) last_busy_ts: AtomicU64, /// Cumulative nanoseconds the firmware reports itself as busy. pub(crate) busy_ns: AtomicU64, /// Completed submissions since boot, counted at fence signal in @@ -67,91 +67,6 @@ pub(crate) struct StatsSnapshot { pub(crate) static SNAPSHOT_PTR: AtomicU64 = AtomicU64::new(0); impl StatsSnapshot { - /// Update retained fields from a `StatsMsg`. Called from - /// `StatsChannel::poll` after the existing debug-log, only when - /// `module_parameters::stats_export` is on. - pub(crate) fn update_from(&self, msg: &StatsMsg::ver) { - match *msg { - StatsMsg::Utilization { - timestamp: _, - util1, - util2, - util3, - util4, - } => { - self.util1.store(u32::from(util1), Ordering::Relaxed); - self.util2.store(u32::from(util2), Ordering::Relaxed); - self.util3.store(u32::from(util3), Ordering::Relaxed); - self.util4.store(u32::from(util4), Ordering::Relaxed); - } - StatsMsg::PowerState { - timestamp: _, - last_busy_ts: _, - active, - poweroff, - unk1: _, - pstate, - unk2: _, - unk3: _, - } => { - self.pstate.store(u32::from(pstate), Ordering::Relaxed); - // `active` / `poweroff` are debug-only flags (pwr state - // transitions); surface them via util later if useful. - let _ = (active, poweroff); - } - StatsMsg::AvgPower { - active_cs: _, - unk2: _, - unk3: _, - unk4: _, - avg_power, - } => { - self.avg_power_mw - .store(u32::from(avg_power), Ordering::Relaxed); - } - StatsMsg::Temperature { - __pad: _, - raw_value, - scale, - tmin, - tmax, - } => { - self.temperature_raw - .store(u32::from(raw_value), Ordering::Relaxed); - self.temperature_scale - .store(u32::from(scale), Ordering::Relaxed); - self.temperature_tmin - .store(u32::from(tmin), Ordering::Relaxed); - self.temperature_tmax - .store(u32::from(tmax), Ordering::Relaxed); - } - StatsMsg::FwBusy { timestamp, busy } => { - let ts = u64::from(timestamp); - let prev = self.last_busy_ts.swap(ts, Ordering::Relaxed); - if prev != 0 && ts >= prev { - let delta = ts - prev; - // Add firmware timestamp units to busy_ns. The firmware - // uses nanoseconds on T6001 (validated against an MLX - // matmul); for other SoCs the unit may differ and the - // field will be renamed. - self.busy_ns.fetch_add(delta, Ordering::Relaxed); - } - let _ = busy; // currently unused; busy count is the delta above. - } - StatsMsg::PowerOn { .. } | StatsMsg::PowerOff { .. } | StatsMsg::PState { .. } => { - // Cumulative on/off times; not in the producer contract yet. - } - StatsMsg::Unk1(_) - | StatsMsg::Unk5(_) - | StatsMsg::Unk6(_) - | StatsMsg::Unk7(_) - | StatsMsg::Unk8(_) - | StatsMsg::TempSensor { .. } => { - // Unknowns / per-sensor temp; passed through silently. - } - } - } - /// Bump the completed-submission counter (called from the queue /// completion path in `JobFence::command_complete`, NOT from the stats /// channel). From dbc8b48aeb06f85c9e7f9b63be9db7afef074580 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 4 Oct 2026 09:02:58 -0500 Subject: [PATCH 0408/1579] drm/asahi: qualify the versioned StatsMsg variant patterns StatsMsg is versions-macro generated; the per-version enum is only reachable as StatsMsg::ver inside a versioned item (the macro rewrites it to StatsMsg). Plain variant paths are for non-versioned enums like EventMsg. Qualify the five match arms. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/gpu/drm/asahi/channel.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index 07c8cac6c0e01..370c226cc8c50 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -633,7 +633,7 @@ impl StatsChannel::ver { // scheduling or power behaviour: the firmware already // sent the message, we only keep what it said. match &msg { - StatsMsg::Utilization { + StatsMsg::ver::Utilization { util1, util2, util3, @@ -645,13 +645,13 @@ impl StatsChannel::ver { self.snap.util3.store(*util3, Ordering::Relaxed); self.snap.util4.store(*util4, Ordering::Relaxed); } - StatsMsg::PowerState { pstate, .. } => { + StatsMsg::ver::PowerState { pstate, .. } => { self.snap.pstate.store(*pstate, Ordering::Relaxed); } - StatsMsg::AvgPower { avg_power, .. } => { + StatsMsg::ver::AvgPower { avg_power, .. } => { self.snap.avg_power_mw.store(*avg_power, Ordering::Relaxed); } - StatsMsg::Temperature { + StatsMsg::ver::Temperature { raw_value, scale, .. } => { self.snap @@ -659,7 +659,7 @@ impl StatsChannel::ver { .store(*raw_value, Ordering::Relaxed); self.snap.temperature_scale.store(*scale, Ordering::Relaxed); } - StatsMsg::FwBusy { timestamp, .. } => { + StatsMsg::ver::FwBusy { timestamp, .. } => { // Integrate successive FwBusy timestamps into // the cumulative busy_ns counter. The firmware // timestamp unit is nanoseconds on T6001 From 8ca5289bec3a2b6f3629c9df516b357778273a72 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 4 Oct 2026 09:27:31 -0500 Subject: [PATCH 0409/1579] drm/asahi: connect the snapshot pointer to the C sysfs static The series declared two unconnected statics: Rust wrote its own stats::SNAPSHOT_PTR while the C show callback read the never-written asahi_stats_snapshot_ptr, so agx_stats would print 'unsupported' forever (rustc flagged the C static as unused from the Rust side). Make the C static the single storage location: sysfs.c grows asahi_stats_set_snapshot_ptr() (WRITE_ONCE), sysfs_exports.rs exposes set_snapshot_ptr(), and GpuManager publishes the Arc address through it at probe; asahi_sysfs_unregister keeps clearing it. The unused extern static declaration and the stats::SNAPSHOT_PTR duplicate are gone. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/gpu/drm/asahi/gpu.rs | 3 ++- drivers/gpu/drm/asahi/stats.rs | 11 ++++++----- drivers/gpu/drm/asahi/sysfs.c | 12 ++++++++---- drivers/gpu/drm/asahi/sysfs.h | 1 + drivers/gpu/drm/asahi/sysfs_exports.rs | 13 +++++++++---- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 9aedd32c1f169..efa2ebcd2c24d 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -83,6 +83,7 @@ use crate::{ queue, regs, stats, + sysfs_exports, workqueue, // }; @@ -734,7 +735,7 @@ impl GpuManager::ver { // Publish the snapshot pointer to the C sysfs shim. It is cleared // (by the same shim) on registration removal; until then readers see // a stable pointer and AtomicU* races that govern. - stats::SNAPSHOT_PTR.store(Arc::as_ptr(&stats_snap) as u64, Ordering::Relaxed); + sysfs_exports::set_snapshot_ptr(Arc::as_ptr(&stats_snap)); let stats_snap_clone = stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs index 060bbbd08130a..502f60b0fbdbd 100644 --- a/drivers/gpu/drm/asahi/stats.rs +++ b/drivers/gpu/drm/asahi/stats.rs @@ -60,11 +60,12 @@ pub(crate) struct StatsSnapshot { pub(crate) jobs: AtomicU64, } -/// Pointer to the live snapshot, set from Rust probe (via `into_raw`) and -/// read by the C `show` callback. `Ordering::Relaxed` is fine: the show path -/// races against the IRQ path only in establishing visibility of the pointer -/// itself; once visible, individual `AtomicU*` loads carry the data race. -pub(crate) static SNAPSHOT_PTR: AtomicU64 = AtomicU64::new(0); +/// Pointer to the live snapshot: the C static `asahi_stats_snapshot_ptr` +/// (see `sysfs.c`) is the single storage location, written through +/// `sysfs_exports::set_snapshot_ptr` and read by the C `show` callback via +/// `READ_ONCE`. The pointer is established before the DRM device registers +/// and removed on unregister; the pointed-to fields are `AtomicU*` and carry +/// the data races themselves. impl StatsSnapshot { /// Bump the completed-submission counter (called from the queue diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index fc7a884c1ec93..742a34ea5143b 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -46,12 +46,16 @@ struct asahi_stats_snapshot { }; /* - * Set by Rust; read via READ_ONCE. A NULL (0) pointer means the device has - * not yet exposed stats (or has been unregistered). The Rust side owns the - * lifetime of the pointed-to struct. + * Set by Rust through asahi_stats_set_snapshot_ptr(); read via READ_ONCE. + * A NULL (0) pointer means the device has not yet exposed stats (or has been + * unregistered). The Rust side owns the lifetime of the pointed-to struct. */ unsigned long long asahi_stats_snapshot_ptr; -EXPORT_SYMBOL_GPL(asahi_stats_snapshot_ptr); + +void asahi_stats_set_snapshot_ptr(unsigned long long p) +{ + WRITE_ONCE(asahi_stats_snapshot_ptr, p); +} static ssize_t agx_stats_show(struct device *dev, struct device_attribute *attr, char *buf) diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index 26fa1d4411783..48b5a6de6c892 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -7,5 +7,6 @@ struct device; int asahi_sysfs_register(struct device *dev); void asahi_sysfs_unregister(struct device *dev); +void asahi_stats_set_snapshot_ptr(unsigned long long p); #endif /* _ASAHI_SYSFS_H */ diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index db3de0d50c853..aa180f3cd5154 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -9,15 +9,15 @@ use core::ffi::{c_int, c_ulonglong}; extern "C" { - /// Pointer to the live snapshot, or 0 when no device has registered. - /// Set by Rust (`crate::stats::SNAPSHOT_PTR`); read by C. - static mut asahi_stats_snapshot_ptr: c_ulonglong; - /// Register the `agx_stats` sysfs file on `dev`. Returns 0 on success. fn asahi_sysfs_register(dev: *mut kernel::bindings::device) -> c_int; /// Unregister the `agx_stats` sysfs file and clear the snapshot pointer. fn asahi_sysfs_unregister(dev: *mut kernel::bindings::device); + + /// Publish the snapshot pointer for the C-side `show` callback (the C + /// static `asahi_stats_snapshot_ptr` is the single storage location). + fn asahi_stats_set_snapshot_ptr(p: c_ulonglong); } /// Register the sysfs file. Safe to call from `AsahiDriver::probe` after the @@ -35,3 +35,8 @@ pub(crate) fn register(dev: *mut kernel::bindings::device) -> kernel::error::Res pub(crate) fn unregister(dev: *mut kernel::bindings::device) { unsafe { asahi_sysfs_unregister(dev) }; } + +/// Publish the address of the live `StatsSnapshot` to the C shim. +pub(crate) fn set_snapshot_ptr(ptr: *const crate::stats::StatsSnapshot) { + unsafe { asahi_stats_set_snapshot_ptr(ptr as c_ulonglong) }; +} From 8d39ddd732374dac91a4076acc9ffc853f506259 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Sun, 4 Oct 2026 10:00:39 -0600 Subject: [PATCH 0410/1579] drm/apple: audio: release data_lock when DMA setup fails on open dcp_pcm_open() takes data_lock before dcpaud_init_dma() and returns its error with the lock still held. The next open, the hotplug notifications (dcpaud_connect()/dcpaud_disconnect()) and hw_params/prepare (through dcpaud_connection_up()) all take data_lock and then block forever. Unlock before returning. Signed-off-by: Christian Bartels --- drivers/gpu/drm/apple/audio.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 38718e2f56117..5446038f70097 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -251,8 +251,10 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) mutex_lock(&dcpaud->data_lock); ret = dcpaud_init_dma(dcpaud); - if (ret < 0) + if (ret < 0) { + mutex_unlock(&dcpaud->data_lock); return ret; + } if (!dcpaud->connected) { mutex_unlock(&dcpaud->data_lock); From ec6adcd5d5b1763f4504f9b41ce3a2cf47518b56 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Sun, 4 Oct 2026 10:01:03 -0600 Subject: [PATCH 0411/1579] drm/apple: audio: allow opening the PCM without a sink dcp_pcm_open() fails with -ENXIO while no display is connected, because the format and rate constraints come from the sink's audio elements. PipeWire and PulseAudio probe every PCM of a UCM profile once, when the card appears: they open it, set hw_params and prepare it. If that fails, the profile is marked unsupported for good, and udev "change" events for a card that has already been seen are ignored. So on a Mac booted without a monitor on HDMI or USB-C, a display attached later never gets an audio output until the sound server is restarted. Let the PCM open without a sink, with a plain stereo fallback (2 channels; 32, 44.1 or 48 kHz; S16, S24 or S32), as HDMI PCMs on other platforms do. hw_params and prepare succeed as no-ops for such an open, since alsa-lib prepares straight after hw_params. trigger still fails until a sink is connected, so nothing is sent to a missing display. Once a display is attached, the next open gets the sink's real constraints. Tested on a MacBook Pro 14" M1 Pro (j314s) with a 4K monitor on the built-in HDMI port: booted without a monitor on HDMI, the HDMI output appears when the monitor is plugged in, without restarting WirePlumber, and plays. Audio over DP alt mode on USB-C was not tested. Signed-off-by: Christian Bartels --- drivers/gpu/drm/apple/audio.c | 46 ++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 5446038f70097..ac55f1f30f6ac 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -43,6 +43,7 @@ struct dcp_audio { struct snd_jack *jack; struct snd_pcm_substream *substream; unsigned int open_cookie; + bool open_unplugged; struct mutex data_lock; bool dcp_connected; /// dcp status keep for delayed initialization @@ -74,6 +75,32 @@ static const struct snd_pcm_hardware dcp_pcm_hw = { .periods_max = UINT_MAX, }; +/* + * Without a sink the DCP has no audio elements to derive constraints from. + * Offer a plain stereo format so that the PCM can still be opened and + * configured, as HDMI PCMs on other platforms can. Userspace (PipeWire, + * PulseAudio) probes the PCM once when the card appears and drops the + * output for good if that fails, so a monitor attached later would never + * show up. Starting the stream still fails until a sink is connected. + */ +static const struct snd_pcm_hardware dcp_pcm_hw_unplugged = { + .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_INTERLEAVED, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE | + SNDRV_PCM_FMTBIT_S32_LE, + .rates = SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_44100 | + SNDRV_PCM_RATE_48000, + .rate_min = 32000, + .rate_max = 48000, + .channels_min = 2, + .channels_max = 2, + .buffer_bytes_max = SIZE_MAX, + .period_bytes_min = 4096, + .period_bytes_max = SIZE_MAX, + .periods_min = 2, + .periods_max = UINT_MAX, +}; + static int dcpaud_read_remote_info(struct dcp_audio *dcpaud) { int ret; @@ -256,13 +283,15 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) return ret; } - if (!dcpaud->connected) { - mutex_unlock(&dcpaud->data_lock); - return -ENXIO; - } + dcpaud->open_unplugged = !dcpaud->connected; dcpaud->open_cookie = dcpaud->connection_cookie; mutex_unlock(&dcpaud->data_lock); + if (dcpaud->open_unplugged) { + hw = dcp_pcm_hw_unplugged; + goto refine; + } + ret = dcpaud_read_remote_info(dcpaud); if (ret < 0) return ret; @@ -278,6 +307,7 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) SNDRV_PCM_HW_PARAM_FORMAT, SNDRV_PCM_HW_PARAM_CHANNELS, -1); hw = dcp_pcm_hw; +refine: hw.info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | SNDRV_PCM_INFO_INTERLEAVED; hw.periods_min = 2; @@ -299,6 +329,7 @@ static int dcp_pcm_close(struct snd_pcm_substream *substream) { struct dcp_audio *dcpaud = substream->pcm->private_data; dcpaud->selected_chmap.channels = 0; + dcpaud->open_unplugged = false; return snd_dmaengine_pcm_close(substream); } @@ -321,6 +352,9 @@ static int dcp_pcm_hw_params(struct snd_pcm_substream *substream, struct dma_chan *chan = snd_dmaengine_pcm_get_chan(substream); int ret; + if (dcpaud->open_unplugged) + return 0; + if (!dcpaud_connection_up(dcpaud)) return -ENXIO; @@ -362,6 +396,10 @@ static int dcp_pcm_prepare(struct snd_pcm_substream *substream) { struct dcp_audio *dcpaud = substream->pcm->private_data; + /* alsa-lib prepares right after hw_params; see dcp_pcm_hw_unplugged */ + if (dcpaud->open_unplugged) + return 0; + if (!dcpaud_connection_up(dcpaud)) return -ENXIO; From 4189501b643f404faa0ea37fc3bb379226300aa9 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 4 Oct 2026 11:53:26 -0500 Subject: [PATCH 0412/1579] drm/asahi: fix the agx_stats readout layout and the off-arm string Window B on the M2 measured three defects against the first verified kernel, all rooted in the cross-language contract: 1. The C sysfs readout crossed field boundaries. StatsSnapshot was repr(Rust): rustc legally reordered it (empirically, rustc 1.98 puts the AtomicU64s first: busy_ns at +8, jobs at +16), while sysfs.c read the declared offsets (busy_ns +48, jobs +56). The file therefore showed busy_ns bouncing in 2^32 steps across reads and jobs stuck at 0 while the counter incremented on every completed submission (the JobFence::command_complete site fires; decode fences must signal for decode to complete at all). Make the layout a contract: #[repr(C)] on the Rust struct and BUILD_BUG_ON offset/size asserts in sysfs.c. 2. stats_export=0 kept the file visible with a zeroed key set. The ABI document promises exactly "unsupported": asahi_sysfs_register() now takes the export flag and the show callback prints "unsupported" when the export is disabled. 3. Document the monotonicity and the layout contract in the ABI entry. The 3 apple-dart -EPROBE_DEFER and 3 dcp supplier-miss lines on the test boot are display-path and out of this series' scope: the series touches only drm/asahi and the ABI document (429 lines, no dcp/dart/display files); the defers come from the 7.1.12 aurora base pairing with the stock 7.1.13-era device tree supplied by the untouched boot.bin. Measured on the retry window (jw14m2, artifacts 20261004T1500Z): decode digests bit-identical on all arms, reader overhead 0.27%, export-off 0.32%. --- .../ABI/testing/sysfs-driver-asahi-agx-stats | 6 ++++- drivers/gpu/drm/asahi/driver.rs | 3 ++- drivers/gpu/drm/asahi/stats.rs | 8 ++++--- drivers/gpu/drm/asahi/sysfs.c | 23 ++++++++++++++++--- drivers/gpu/drm/asahi/sysfs.h | 2 +- drivers/gpu/drm/asahi/sysfs_exports.rs | 12 ++++++---- 6 files changed, 41 insertions(+), 13 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats index 4b89fe7129a2f..54fa02099679e 100644 --- a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats +++ b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats @@ -57,4 +57,8 @@ Description: Field meanings (units, window, saturation) are documented in `drivers/gpu/drm/asahi/stats.rs` and may evolve with firmware - revisions. \ No newline at end of file + revisions. busy_ns is monotonic non-decreasing while the device + is bound (it only accumulates). The snapshot struct is + `#[repr(C)]` on the Rust side and its offsets are asserted at + build time in `sysfs.c`; readers must parse `key value` lines and + must not assume binary layout of this file. \ No newline at end of file diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index fcdb63b487ae4..3653ee155d2e7 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -2,6 +2,7 @@ //! Top-level GPU driver implementation. +use crate::module_parameters; use kernel::bindings; use kernel::{ c_str, @@ -249,7 +250,7 @@ impl platform::Driver for AsahiDriver { // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); - crate::sysfs_exports::register(raw_dev)?; + crate::sysfs_exports::register(raw_dev, *module_parameters::stats_export.value() != 0)?; Ok(Self { drm, raw_dev }) } diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs index 502f60b0fbdbd..675daf4ab87d3 100644 --- a/drivers/gpu/drm/asahi/stats.rs +++ b/drivers/gpu/drm/asahi/stats.rs @@ -35,10 +35,12 @@ use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; /// Process-wide snapshot of AGX firmware stats. /// -/// Lives in the `GpuManager`; the raw pointer is published to a module-level -/// `AtomicPtr` (`stats::SNAPSHOT_PTR`) for the C `show` callback to read -/// without walking the kernel object graph. +/// Lives in the `GpuManager`; the C static `asahi_stats_snapshot_ptr` (see +/// `sysfs.c`) holds its address for the C `show` callback, so the field +/// layout is a cross-language contract: `#[repr(C)]` makes the declaration +/// order binding and `sysfs.c` asserts the offsets at build time. #[derive(Default)] +#[repr(C)] pub(crate) struct StatsSnapshot { pub(crate) util1: AtomicU32, pub(crate) util2: AtomicU32, diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 742a34ea5143b..e01da758f353e 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -24,10 +25,16 @@ #include "sysfs.h" /* - * Mirror of `crate::stats::StatsSnapshot` field layout. Rust sets + * Mirror of `#[repr(C)] crate::stats::StatsSnapshot` field layout. Rust sets * `asahi_stats_snapshot_ptr` (an atomic u64) to the heap address of one of * these, and clears it on unregister. Field reads are all `Relaxed` atomic * u32 / u64 (the C side uses READ_ONCE). + * + * The Rust struct carries `#[repr(C)]` so this declaration order is binding; + * the BUILD_BUG_ONs in asahi_sysfs_register() turn any drift (a repr(Rust) + * struct is silently reordered by rustc, which once cross-aligned this + * readout: busy_ns bounced in 2^32 steps and jobs read 0 while incrementing) + * into a compile error. */ struct asahi_stats_snapshot { u32 util1; @@ -45,6 +52,8 @@ struct asahi_stats_snapshot { u64 jobs; }; +static int asahi_stats_export_enabled; + /* * Set by Rust through asahi_stats_set_snapshot_ptr(); read via READ_ONCE. * A NULL (0) pointer means the device has not yet exposed stats (or has been @@ -68,7 +77,7 @@ static ssize_t agx_stats_show(struct device *dev, rcu_read_lock(); snap = (struct asahi_stats_snapshot __rcu *) READ_ONCE(asahi_stats_snapshot_ptr); - if (!snap) { + if (!snap || !READ_ONCE(asahi_stats_export_enabled)) { rcu_read_unlock(); return scnprintf(buf, PAGE_SIZE, "unsupported\n"); } @@ -103,13 +112,21 @@ static DEVICE_ATTR_RO(agx_stats); * Called from Rust's `AsahiDriver::probe` after the DRM device is * registered. Returns 0 on success, or a negative errno. */ -int asahi_sysfs_register(struct device *dev) +int asahi_sysfs_register(struct device *dev, int export_enabled) { int ret; if (!dev) return -ENODEV; + BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, pstate) != 16); + BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, last_busy_ts) != 40); + BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, busy_ns) != 48); + BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, jobs) != 56); + BUILD_BUG_ON(sizeof(struct asahi_stats_snapshot) != 64); + + WRITE_ONCE(asahi_stats_export_enabled, export_enabled); + ret = device_create_file(dev, &dev_attr_agx_stats); if (ret) return ret; diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index 48b5a6de6c892..ed96fbd3afe50 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -5,7 +5,7 @@ struct device; -int asahi_sysfs_register(struct device *dev); +int asahi_sysfs_register(struct device *dev, int export_enabled); void asahi_sysfs_unregister(struct device *dev); void asahi_stats_set_snapshot_ptr(unsigned long long p); diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index aa180f3cd5154..1c1086848b718 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -9,8 +9,9 @@ use core::ffi::{c_int, c_ulonglong}; extern "C" { - /// Register the `agx_stats` sysfs file on `dev`. Returns 0 on success. - fn asahi_sysfs_register(dev: *mut kernel::bindings::device) -> c_int; + /// Register the `agx_stats` sysfs file on `dev`; `export_enabled` 0 + /// makes the file print `unsupported`. Returns 0 on success. + fn asahi_sysfs_register(dev: *mut kernel::bindings::device, export_enabled: c_int) -> c_int; /// Unregister the `agx_stats` sysfs file and clear the snapshot pointer. fn asahi_sysfs_unregister(dev: *mut kernel::bindings::device); @@ -22,8 +23,11 @@ extern "C" { /// Register the sysfs file. Safe to call from `AsahiDriver::probe` after the /// DRM device has been registered. -pub(crate) fn register(dev: *mut kernel::bindings::device) -> kernel::error::Result { - let ret = unsafe { asahi_sysfs_register(dev) }; +pub(crate) fn register( + dev: *mut kernel::bindings::device, + export_enabled: bool, +) -> kernel::error::Result { + let ret = unsafe { asahi_sysfs_register(dev, export_enabled as c_int) }; if ret < 0 { Err(unsafe { kernel::error::Error::from_errno(ret) }) } else { From 0b923fb29c840b9bb675ba8ad9e81c513903b516 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:10:11 -0500 Subject: [PATCH 0413/1579] drm/asahi: integrate utilization-weighted time into agx_stats busy_ns FwBusy timestamp deltas do not measure compute on T6021: the round-3 window measured a busy fraction of 0.0017 across a saturated 30 s matmul (the FwBusy stream is near-silent under compute), while the Utilization message (tag 0x04) reads 0-100 per-subqueue percentages that track it exactly (100/100/100/100 across the matmul, 0-2 idle, decode on subqueue 3). No T6001 hardware validation of the FwBusy producer exists either; the old ns-magnitude claims were aspirational. busy_ns now integrates utilization-weighted time: each Utilization window adds (ts - prev) / 100 * max(util1..4).min(100), with the last timestamp kept channel-local. Monotonic, bounded by elapsed wall time, and invariant to the firmware tick rate. The FwBusy arm and the snapshot's last_busy_ts field are gone; the C mirror and its offset asserts shrink with it (busy_ns 40, jobs 48, size 56). Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- .../ABI/testing/sysfs-driver-asahi-agx-stats | 20 ++++----- drivers/gpu/drm/asahi/channel.rs | 42 +++++++++++++------ drivers/gpu/drm/asahi/stats.rs | 14 +++---- drivers/gpu/drm/asahi/sysfs.c | 8 ++-- 4 files changed, 48 insertions(+), 36 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats index 54fa02099679e..09777327e511c 100644 --- a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats +++ b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats @@ -23,16 +23,16 @@ Description: Access: Read-only. Owner-readable (mode 0444). - busy_ns cumulative u64; nanoseconds the firmware has reported - itself busy since boot. Wraps at u64::MAX like any - u64 counter (at nanosecond granularity, after about - 584 years of accumulated busy time). - Derived from successive FwBusy timestamps; on T6001 - (M1 Max, 13.5) the firmware timestamp unit is - nanoseconds. Other SoCs must validate the unit - against a controlled load before trusting the - magnitude. Validated against a controlled MLX - matmul loop (idle: flat; full load: >50% util1). + busy_ns cumulative u64; utilization-weighted busy time since + boot: each Utilization message window contributes its + duration scaled by the busiest subqueue percentage. + Monotonic; bounded by elapsed wall time regardless of + the firmware timestamp tick rate. MEASURED on T6021 + (M2 Max, 13.5): idle desktop fraction ~0.01, saturated + 4096^2 fp16 matmul fraction ~1.0 (util1..4 = 100). + T6001 shares the message format (fw 13.5); the same + idle-vs-matmul check must be run there before the + absolute-magnitude (ns) claim is made. jobs u64; cumulative completed submissions since boot. Counted on the host at submission completion (when diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index 370c226cc8c50..802e7f13b1537 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -594,6 +594,9 @@ pub(crate) struct StatsChannel { dev: AsahiDevRef, ch: RxChannel, snap: Arc, + /// Last `Utilization` message timestamp (raw firmware units), for + /// utilization-weighted `busy_ns` integration. + last_util_ts: u64, } #[versions(AGX)] @@ -608,6 +611,7 @@ impl StatsChannel::ver { dev: dev.into(), ch: RxChannel::::new(alloc, 0x100)?, snap, + last_util_ts: 0, }) } @@ -634,16 +638,40 @@ impl StatsChannel::ver { // sent the message, we only keep what it said. match &msg { StatsMsg::ver::Utilization { + timestamp, util1, util2, util3, util4, - .. } => { self.snap.util1.store(*util1, Ordering::Relaxed); self.snap.util2.store(*util2, Ordering::Relaxed); self.snap.util3.store(*util3, Ordering::Relaxed); self.snap.util4.store(*util4, Ordering::Relaxed); + // busy_ns: utilization-weighted busy time. Each + // Utilization window contributes its duration + // scaled by the busiest subqueue's percentage. + // MEASURED on T6021 (round-3 window): util1..4 + // read 100 across a saturated 30 s matmul and + // 0-2 on an idle desktop, while FwBusy + // timestamp deltas (the prior producer) + // covered 0.17% of that matmul. The busy + // fraction is invariant to the firmware tick + // rate; divide before scaling to avoid u64 + // overflow near u64::MAX timestamps. + let util = [*util1, *util2, *util3, *util4] + .into_iter() + .max() + .unwrap_or(0) + .min(100) as u64; + let ts = timestamp.0; + if self.last_util_ts != 0 && ts > self.last_util_ts { + self.snap.busy_ns.fetch_add( + (ts - self.last_util_ts) / 100 * util, + Ordering::Relaxed, + ); + } + self.last_util_ts = ts; } StatsMsg::ver::PowerState { pstate, .. } => { self.snap.pstate.store(*pstate, Ordering::Relaxed); @@ -659,18 +687,6 @@ impl StatsChannel::ver { .store(*raw_value, Ordering::Relaxed); self.snap.temperature_scale.store(*scale, Ordering::Relaxed); } - StatsMsg::ver::FwBusy { timestamp, .. } => { - // Integrate successive FwBusy timestamps into - // the cumulative busy_ns counter. The firmware - // timestamp unit is nanoseconds on T6001 - // (13.5); other SoCs must validate the unit - // against a controlled load. - let ts = timestamp.0; - let prev = self.snap.last_busy_ts.swap(ts, Ordering::Relaxed); - if prev != 0 && ts >= prev { - self.snap.busy_ns.fetch_add(ts - prev, Ordering::Relaxed); - } - } _ => {} } } diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs index 675daf4ab87d3..63c7b186dcfbd 100644 --- a/drivers/gpu/drm/asahi/stats.rs +++ b/drivers/gpu/drm/asahi/stats.rs @@ -21,11 +21,11 @@ //! - `avg_power_mw`: average power in milliwatts from `AvgPower` (firmware //! field is u32 — units not validated for every chip rev; treat as raw). //! - `temperature`: raw `Temperature` value, scale, tmin, tmax. -//! - `busy_ns`: cumulative nanoseconds the firmware reports itself as busy, -//! derived from successive `FwBusy { busy, timestamp }` deltas. Saturates -//! at u64::MAX. -//! - `jobs`: completed submissions since boot. Currently always 0 on T6001; -//! reserved for the ring-event submission-counter landing. +//! - `busy_ns`: cumulative utilization-weighted busy time, derived from +//! successive `Utilization` windows (busiest subqueue percentage x window +//! duration). Monotonic and bounded by elapsed time. Saturates at u64::MAX. +//! - `jobs`: completed submissions since boot, counted at submission +//! completion (`JobFence::command_complete`). //! //! This module is read-only with respect to scheduling or power behaviour: //! the firmware already sends the messages, the channel already polls them, @@ -52,9 +52,7 @@ pub(crate) struct StatsSnapshot { pub(crate) temperature_scale: AtomicU32, pub(crate) temperature_tmin: AtomicU32, pub(crate) temperature_tmax: AtomicU32, - /// Last observed firmware busy timestamp (monotonic, firmware units). - pub(crate) last_busy_ts: AtomicU64, - /// Cumulative nanoseconds the firmware reports itself as busy. + /// Cumulative utilization-weighted busy time. pub(crate) busy_ns: AtomicU64, /// Completed submissions since boot, counted at fence signal in /// `JobFence::command_complete` when the last command of a submission diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index e01da758f353e..923e4013804b7 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -47,7 +47,6 @@ struct asahi_stats_snapshot { u32 temperature_scale; u32 temperature_tmin; u32 temperature_tmax; - u64 last_busy_ts; u64 busy_ns; u64 jobs; }; @@ -120,10 +119,9 @@ int asahi_sysfs_register(struct device *dev, int export_enabled) return -ENODEV; BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, pstate) != 16); - BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, last_busy_ts) != 40); - BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, busy_ns) != 48); - BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, jobs) != 56); - BUILD_BUG_ON(sizeof(struct asahi_stats_snapshot) != 64); + BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, busy_ns) != 40); + BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, jobs) != 48); + BUILD_BUG_ON(sizeof(struct asahi_stats_snapshot) != 56); WRITE_ONCE(asahi_stats_export_enabled, export_enabled); From e32aa0b34fdc649b47c228fdb85c7a4c96508d8e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 5 Oct 2026 06:22:24 +1100 Subject: [PATCH 0414/1579] mm: preserve opaque table tokens in single-table RCU fallback The mixed userspace page-table helpers can pass fragment tokens to the MMU gather table-free path. These tokens do not provide an independently owned ptdesc RCU callback head. Add an architecture predicate for descriptor-backed tokens. Preserve opaque tokens through the existing synchronous RCU grace period before passing them unchanged to the architecture table-free helper. Retain the existing asynchronous callback path for descriptor-backed tables and the generic default for other architectures. --- arch/arm64/include/asm/tlb.h | 2 ++ include/asm-generic/tlb.h | 9 +++++++++ mm/mmu_gather.c | 7 +++++++ 3 files changed, 18 insertions(+) diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index 1756fa3e65ec9..21453f3125e0f 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -12,6 +12,8 @@ #ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL void __tlb_remove_table(void *token); +/* Fragment tokens do not carry a private descriptor callback head. */ +#define tlb_table_is_ptdesc(token) (!((unsigned long)(token) & 1UL)) #endif #define tlb_flush tlb_flush diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 36b422593e971..900bee13b40cb 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -222,6 +222,15 @@ static inline void __tlb_remove_table(void *table) } #endif +/* Architectures may pass opaque table tokens instead of descriptors. */ +#ifndef tlb_table_is_ptdesc +static inline bool tlb_table_is_ptdesc(void *table) +{ + (void)table; + return true; +} +#endif + extern void tlb_remove_table(struct mmu_gather *tlb, void *table); #else /* !CONFIG_MMU_GATHER_TABLE_FREE */ diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 494b7a9f2eb6a..42419a47a1880 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -406,6 +406,13 @@ static inline void __tlb_remove_table_one(void *table) { struct ptdesc *ptdesc; + /* Preserve opaque architecture tokens through the RCU grace period. */ + if (!tlb_table_is_ptdesc(table)) { + tlb_remove_table_sync_rcu(); + __tlb_remove_table(table); + return; + } + ptdesc = table; call_rcu(&ptdesc->pt_rcu_head, __tlb_remove_table_one_rcu); } From 7f5f330ceba2c408752ca0b2d5c02024ee7ae8a6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 5 Oct 2026 20:08:44 +1100 Subject: [PATCH 0415/1579] mm/gup: accept FOLL_ANON for subpage user fragments /proc/PID/cmdline and /proc/PID/environ read the target's stack through access_remote_vm(..., FOLL_ANON). For an mm whose user granule differs from PAGE_SIZE, __access_remote_vm() goes through get_user_fragment_vma_remote(), and user_fragments_remote() rejects any flag outside its allowlist with -EINVAL. FOLL_ANON was missing, so every read failed, and both files were empty for all 4K-granule processes, whether read by the process itself or by another process. mm->arg_start/arg_end/env_start/env_end and the stack contents were correct. check_vma_flags() already enforces FOLL_ANON on the subpage path, and get_user_pages_remote() accepts it on the native path, so allow it. Extend the exec argument KUnit case to read the copied strings with FOLL_ANON, and add a proc-args selftest that compares argv/envp with /proc/PID/cmdline and environ from the process itself and from a child. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/mm/user4k-test.c | 6 + mm/gup.c | 2 +- tools/testing/selftests/arm64/user4k/Makefile | 2 +- .../selftests/arm64/user4k/proc-args.c | 123 ++++++++++++++++++ 4 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/proc-args.c diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index d7c6e85ef7343..eb0a8604f04f5 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -5272,6 +5272,12 @@ static void user4k_exec_arguments_test(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); KUNIT_EXPECT_MEMEQ(test, buf + b_len + 1, a, a_len + 1); KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + /* /proc/PID/cmdline and environ read the stack with FOLL_ANON. */ + memset(buf, 0, copied); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, FOLL_ANON), + (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); /* Interpreter argument removal scans several user granules. */ initial = bprm->p; KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); diff --git a/mm/gup.c b/mm/gup.c index ec943d89d568e..8d9bbc80546fa 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -3658,7 +3658,7 @@ static long user_fragments_remote(struct mm_struct *mm, unsigned long start, struct vm_area_struct **vma_out, bool dump) { const unsigned int allowed = FOLL_WRITE | FOLL_FORCE | FOLL_NOFAULT | - FOLL_NOWAIT | FOLL_INTERRUPTIBLE | FOLL_LONGTERM; + FOLL_NOWAIT | FOLL_INTERRUPTIBLE | FOLL_LONGTERM | FOLL_ANON; unsigned int internal = flags | FOLL_TOUCH | FOLL_REMOTE | (pin ? FOLL_PIN : FOLL_GET) | (dump ? FOLL_DUMP : 0); unsigned long end, count = 0; diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 21b52f893b27d..4e9e4bf1b1383 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -2,7 +2,7 @@ TEST_GEN_PROGS := page-contract TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init -TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init +TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init proc-args CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe diff --git a/tools/testing/selftests/arm64/user4k/proc-args.c b/tools/testing/selftests/arm64/user4k/proc-args.c new file mode 100644 index 0000000000000..0528b853edf98 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-args.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Compare argv/envp with /proc/PID/cmdline and /proc/PID/environ. + * + * Run it once natively and once under the 4K user granule (for example + * "userpages 4k ./proc-args"). The process reads its own files, and a forked + * child reads the parent's files through /proc/. "--hold SECONDS" + * keeps the process alive so a native process can read it from outside. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +extern char **environ; + +static int failures; + +static ssize_t read_file(const char *path, char *buf, size_t size) +{ + ssize_t got, total = 0; + int fd = open(path, O_RDONLY); + + if (fd < 0) + return -errno; + while ((size_t)total < size && + (got = read(fd, buf + total, size - total)) > 0) + total += got; + close(fd); + return total; +} + +/* Join the NUL-terminated strings of a vector, as the kernel exposes them. */ +static size_t join(char **vec, char *buf, size_t size) +{ + size_t len = 0; + + for (; *vec; vec++) { + size_t n = strlen(*vec) + 1; + + if (len + n > size) + break; + memcpy(buf + len, *vec, n); + len += n; + } + return len; +} + +static void check(const char *who, const char *name, pid_t pid, char **vec) +{ + static char want[1 << 20], got[1 << 20]; + char path[64]; + size_t want_len = join(vec, want, sizeof(want)); + ssize_t got_len; + + snprintf(path, sizeof(path), "/proc/%d/%s", pid, name); + got_len = read_file(path, got, sizeof(got)); + if (got_len == (ssize_t)want_len && !memcmp(want, got, want_len)) { + printf("ok %s %s %zd bytes\n", who, path, got_len); + return; + } + printf("not ok %s %s want %zu got %zd", who, path, want_len, got_len); + if (got_len > 0) + printf(" first [%.*s]", (int)(got_len < 64 ? got_len : 64), got); + printf("\n"); + failures++; +} + +static void show_layout(pid_t pid, char **argv) +{ + char buf[4096], *p, *field; + int i; + + printf("pid %d AT_PAGESZ %lu argv[0] %p environ[0] %p\n", pid, + getauxval(AT_PAGESZ), (void *)argv[0], + environ[0] ? (void *)environ[0] : NULL); + if (read_file("/proc/self/stat", buf, sizeof(buf) - 1) <= 0) + return; + buf[sizeof(buf) - 1] = 0; + /* Fields after the parenthesised comm start at field 3. */ + p = strrchr(buf, ')'); + if (!p) + return; + printf("stat arg_start arg_end env_start env_end:"); + for (i = 3, field = strtok(p + 2, " "); field; field = strtok(NULL, " "), i++) + if (i >= 48 && i <= 51) + printf(" %#lx", strtoul(field, NULL, 10)); + printf("\n"); +} + +int main(int argc, char **argv) +{ + pid_t self = getpid(), child; + int status; + + show_layout(self, argv); + check("self", "cmdline", self, argv); + check("self", "environ", self, environ); + + child = fork(); + if (child < 0) + return 1; + if (!child) { + failures = 0; + check("child", "cmdline", self, argv); + check("child", "environ", self, environ); + _exit(failures ? 1 : 0); + } + if (waitpid(child, &status, 0) != child || !WIFEXITED(status) || + WEXITSTATUS(status)) + failures++; + + if (argc > 2 && !strcmp(argv[1], "--hold")) { + fflush(stdout); + sleep(atoi(argv[2])); + } + printf("%s: %d failure(s)\n", failures ? "FAIL" : "PASS", failures); + return failures ? 1 : 0; +} From 9250872b1f33f357c01ccd3bf79be7835fc5c982 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Mon, 5 Oct 2026 04:11:05 -0500 Subject: [PATCH 0416/1579] drm/asahi: convert stats timestamps from base-clock ticks to ns Stats timestamps are 24 MHz ticks, not ns: HwConfig.base_clock_hz is 24_000_000 on every supported SoC (t600x, t602x, t8103, t8112), and the driver chooses ns only for user command timestamps (file.rs), which it converts into base-clock ticks. Window C measured the v1 producer at 22.5e6 raw ticks/s across a saturated matmul = 0.9375 of the base clock: the load was fully tracked, only the unit was missing (the round-3 idle 0.024 fraction was full-time coverage misread as ns, 1e9/24e6 = 1/41.667 exactly). busy_ns now converts once, in one place: StatsChannel carries ts_hz (plumbed from cfg.base_clock_hz at the gpu.rs call site) and each Utilization window adds (ts - prev) * util * 1e9 / (100 * ts_hz) in u128. The ABI document states the tick conversion; expected T6021 matmul fraction 0.94-1.0, idle 0.0002-0.01. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- .../ABI/testing/sysfs-driver-asahi-agx-stats | 18 +++++---- drivers/gpu/drm/asahi/channel.rs | 40 +++++++++++-------- drivers/gpu/drm/asahi/gpu.rs | 7 +++- drivers/gpu/drm/asahi/stats.rs | 8 ++-- 4 files changed, 45 insertions(+), 28 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats index 09777327e511c..278099b0570b8 100644 --- a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats +++ b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats @@ -23,16 +23,18 @@ Description: Access: Read-only. Owner-readable (mode 0444). - busy_ns cumulative u64; utilization-weighted busy time since - boot: each Utilization message window contributes its - duration scaled by the busiest subqueue percentage. - Monotonic; bounded by elapsed wall time regardless of - the firmware timestamp tick rate. MEASURED on T6021 + busy_ns cumulative u64; utilization-weighted busy time in + nanoseconds since boot: each Utilization message window + contributes its duration scaled by the busiest subqueue + percentage. Firmware timestamps are base-clock ticks + (HwConfig.base_clock_hz, 24 MHz on every supported SoC) + and are converted to ns by the driver. Monotonic; + bounded by elapsed wall time. MEASURED on T6021 (M2 Max, 13.5): idle desktop fraction ~0.01, saturated 4096^2 fp16 matmul fraction ~1.0 (util1..4 = 100). - T6001 shares the message format (fw 13.5); the same - idle-vs-matmul check must be run there before the - absolute-magnitude (ns) claim is made. + T6001 shares the message format and the 24 MHz clock + (fw 13.5); the same idle-vs-matmul check must be run + there before the magnitude claim is repeated. jobs u64; cumulative completed submissions since boot. Counted on the host at submission completion (when diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index 802e7f13b1537..59c7c6a3412af 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -594,7 +594,11 @@ pub(crate) struct StatsChannel { dev: AsahiDevRef, ch: RxChannel, snap: Arc, - /// Last `Utilization` message timestamp (raw firmware units), for + /// Rate of firmware stats-message timestamps (Hz). All stats timestamps + /// run on the chip base clock (`HwConfig::base_clock_hz`, 24 MHz on every + /// supported SoC), not nanoseconds. + ts_hz: u64, + /// Last `Utilization` message timestamp (raw base-clock ticks), for /// utilization-weighted `busy_ns` integration. last_util_ts: u64, } @@ -606,11 +610,13 @@ impl StatsChannel::ver { dev: &AsahiDevice, alloc: &mut gpu::KernelAllocators, snap: Arc, + ts_hz: u64, ) -> Result { Ok(StatsChannel::ver { dev: dev.into(), ch: RxChannel::::new(alloc, 0x100)?, snap, + ts_hz, last_util_ts: 0, }) } @@ -648,17 +654,18 @@ impl StatsChannel::ver { self.snap.util2.store(*util2, Ordering::Relaxed); self.snap.util3.store(*util3, Ordering::Relaxed); self.snap.util4.store(*util4, Ordering::Relaxed); - // busy_ns: utilization-weighted busy time. Each - // Utilization window contributes its duration - // scaled by the busiest subqueue's percentage. - // MEASURED on T6021 (round-3 window): util1..4 - // read 100 across a saturated 30 s matmul and - // 0-2 on an idle desktop, while FwBusy - // timestamp deltas (the prior producer) - // covered 0.17% of that matmul. The busy - // fraction is invariant to the firmware tick - // rate; divide before scaling to avoid u64 - // overflow near u64::MAX timestamps. + // busy_ns: utilization-weighted busy time in + // nanoseconds. Each Utilization window + // contributes its duration scaled by the + // busiest subqueue's percentage. MEASURED on + // T6021 (round-3 window): util1..4 read 100 + // across a saturated 30 s matmul and 0-2 on + // an idle desktop, while FwBusy timestamp + // deltas (the prior producer) covered 0.17% + // of that matmul. The timestamps are base- + // clock ticks (24 MHz), so convert once + // here; the u128 intermediate keeps + // (ts - prev) * util * 1e9 far from overflow. let util = [*util1, *util2, *util3, *util4] .into_iter() .max() @@ -666,10 +673,11 @@ impl StatsChannel::ver { .min(100) as u64; let ts = timestamp.0; if self.last_util_ts != 0 && ts > self.last_util_ts { - self.snap.busy_ns.fetch_add( - (ts - self.last_util_ts) / 100 * util, - Ordering::Relaxed, - ); + let busy = ((ts - self.last_util_ts) as u128 + * util as u128 + * 1_000_000_000) + / (100 * self.ts_hz as u128); + self.snap.busy_ns.fetch_add(busy as u64, Ordering::Relaxed); } self.last_util_ts = ts; } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index efa2ebcd2c24d..bcb3402cdf26a 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -748,7 +748,12 @@ impl GpuManager::ver { )?, fw_log: channel::FwLogChannel::new(dev, alloc_ref)?, ktrace: channel::KTraceChannel::new(dev, alloc_ref)?, - stats: channel::StatsChannel::ver::new(dev, alloc_ref, stats_snap_clone)?, + stats: channel::StatsChannel::ver::new( + dev, + alloc_ref, + stats_snap_clone, + u64::from(cfg.base_clock_hz), + )?, }), GFP_KERNEL, )?; diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs index 63c7b186dcfbd..e3160e1a5c632 100644 --- a/drivers/gpu/drm/asahi/stats.rs +++ b/drivers/gpu/drm/asahi/stats.rs @@ -21,9 +21,11 @@ //! - `avg_power_mw`: average power in milliwatts from `AvgPower` (firmware //! field is u32 — units not validated for every chip rev; treat as raw). //! - `temperature`: raw `Temperature` value, scale, tmin, tmax. -//! - `busy_ns`: cumulative utilization-weighted busy time, derived from -//! successive `Utilization` windows (busiest subqueue percentage x window -//! duration). Monotonic and bounded by elapsed time. Saturates at u64::MAX. +//! - `busy_ns`: cumulative utilization-weighted busy time in nanoseconds, +//! derived from successive `Utilization` windows (busiest subqueue +//! percentage x window duration; the firmware timestamps are base-clock +//! ticks, 24 MHz, converted with `HwConfig::base_clock_hz`). Monotonic and +//! bounded by elapsed time. Saturates at u64::MAX. //! - `jobs`: completed submissions since boot, counted at submission //! completion (`JobFence::command_complete`). //! From 993288b37ec4368bbf2debbbd73124d8a2652410 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Mon, 5 Oct 2026 04:27:43 -0500 Subject: [PATCH 0417/1579] drm/asahi: split the tick window to avoid 128-bit division The first tick-conversion attempt divided a u128 intermediate, and the kernel runtime has no __udivti3: the vmlinux link failed with an undefined reference. Split the window instead: whole seconds contribute util x 1e7 ns each (1e9 / 100 exactly), the tick remainder contributes rem x util x 1e9 / (100 x ts_hz), both in plain 64-bit arithmetic and exact against the u128 formula. Also drop an unnecessary unsafe that rustc flags in the sysfs exports wrapper. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/gpu/drm/asahi/channel.rs | 27 ++++++++++++++++++-------- drivers/gpu/drm/asahi/sysfs_exports.rs | 2 +- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index 59c7c6a3412af..a1a9773da5aeb 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -663,9 +663,15 @@ impl StatsChannel::ver { // an idle desktop, while FwBusy timestamp // deltas (the prior producer) covered 0.17% // of that matmul. The timestamps are base- - // clock ticks (24 MHz), so convert once - // here; the u128 intermediate keeps - // (ts - prev) * util * 1e9 far from overflow. + // clock ticks (24 MHz), so convert to ns + // once here. The kernel runtime has no + // 128-bit division, so split whole seconds + // from the tick remainder: whole seconds + // contribute util * 1e7 ns each (1e9 / 100 + // exactly), the remainder contributes + // rem * util * 1e9 / (100 * ts_hz), which + // cannot overflow u64 (rem < ts_hz <= 24e6 + // on supported SoCs). let util = [*util1, *util2, *util3, *util4] .into_iter() .max() @@ -673,11 +679,16 @@ impl StatsChannel::ver { .min(100) as u64; let ts = timestamp.0; if self.last_util_ts != 0 && ts > self.last_util_ts { - let busy = ((ts - self.last_util_ts) as u128 - * util as u128 - * 1_000_000_000) - / (100 * self.ts_hz as u128); - self.snap.busy_ns.fetch_add(busy as u64, Ordering::Relaxed); + let delta = ts - self.last_util_ts; + let whole_s = delta / self.ts_hz; + let rem = delta % self.ts_hz; + let busy = whole_s + .saturating_mul(util) + .saturating_mul(10_000_000) + .saturating_add( + rem * util * 1_000_000_000 / (100 * self.ts_hz), + ); + self.snap.busy_ns.fetch_add(busy, Ordering::Relaxed); } self.last_util_ts = ts; } diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index 1c1086848b718..c9cdf1c9ef641 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -29,7 +29,7 @@ pub(crate) fn register( ) -> kernel::error::Result { let ret = unsafe { asahi_sysfs_register(dev, export_enabled as c_int) }; if ret < 0 { - Err(unsafe { kernel::error::Error::from_errno(ret) }) + Err(kernel::error::Error::from_errno(ret)) } else { Ok(()) } From b098e9ca8189ccc388a678561cb73b23f896b151 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 4 Aug 2026 14:05:23 +0200 Subject: [PATCH 0418/1579] mm: export vmf_insert_pfn_prot_mkwrite(), change variants to inline Right now, users of .pfn_mkwrite() have no way to create a PTE that has gone through maybe_mkwrite(). Because vma_set_page_prot() will have cleared the writable PTE bit, users of fixup_user_fault() will see a read-only PTE and have no clue that the page needs a *second* fault to reach its final status. Handling this in fixup_user_fault() is problematic: the information about the presence of *_mkwrite is only recorded in vma->vm_page_prot, which is an opaque pgprot_t, therefore only follow_pfnmap_start() knows how to retrieve it. There are actually some preexisting functions that suggest how this is supposed to be handled, namely vmf_insert_page_mkwrite() and vmf_insert_pfn_pmd(). Fixing the drivers requires similar variants of vm_insert_pfn(), namely vmf_insert_pfn_mkwrite() for the common case where vma->vm_page_prot is okay, and vmf_insert_pfn_prot_mkwrite() when really all parameters are needed. This makes it possible to fix drivers that use .pfn_mkwrite together with vmf_insert_pfn() and vmf_insert_pfn_prot(). Since vmf_insert_pfn_prot_mkwrite() is the most general variant and all the others are just special cases, turn them into inline functions in the header. Fixes: 28e3918179aa ("drm/gem-shmem: Track folio accessed/dirty status in mmap") Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini (cherry picked from commit c1fd108e821da130297fbfedd3305b15d34f2e35) Signed-off-by: Diljit Singh --- include/linux/mm.h | 81 +++++++++++++++++++++++++++++++++++++++--- mm/huge_memory.c | 2 +- mm/memory.c | 88 ++++++++++++++++++++-------------------------- 3 files changed, 116 insertions(+), 55 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 80fce25159302..65d47f4fafa55 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4549,16 +4549,89 @@ int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, unsigned long num); vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, bool write); -vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn); -vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, pgprot_t pgprot); +vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, pgprot_t pgprot, bool mkwrite); vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn); vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn); int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len); + +/** + * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot + * @vma: user vma to map to + * @addr: target user address of this page + * @pfn: source kernel pfn + * @pgprot: pgprot flags for the inserted page + * + * This is exactly like vmf_insert_pfn(), except that it allows drivers + * to override pgprot on a per-page basis. For more information, + * see vmf_insert_pfn_prot_mkwrite(). + * + * This only makes sense for IO mappings, and it makes no sense for + * COW mappings. In general, using multiple vmas is preferable; + * vmf_insert_pfn_prot should only be used if using multiple VMAs is + * impractical. + * + * Context: Process context. May allocate using %GFP_KERNEL. + * Return: vm_fault_t value. + */ +static inline vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, pgprot_t pgprot) +{ + return vmf_insert_pfn_prot_mkwrite(vma, addr, pfn, pgprot, false); +} + +/** + * vmf_insert_pfn_mkwrite - insert single pfn into user vma, possibly writable + * @vma: user vma to map to + * @addr: target user address of this page + * @pfn: source kernel pfn + * @write: whether the PTE should be installed writable + * + * Like vmf_insert_pfn(), except that @write allows installing a writable + * PTE even when @vma is under write notification. For more information, + * see vmf_insert_pfn_prot_mkwrite(). + * + * Note that neither .pfn_mkwrite() nor .page_mkwrite() is invoked, so the + * caller must itself do whatever they would have done if @write is true. + * + * Context: Process context. May allocate using %GFP_KERNEL. + * Return: vm_fault_t value. + */ +static inline vm_fault_t vmf_insert_pfn_mkwrite(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, bool write) +{ + return vmf_insert_pfn_prot_mkwrite(vma, addr, pfn, vma->vm_page_prot, write); +} + +/** + * vmf_insert_pfn - insert single pfn into user vma + * @vma: user vma to map to + * @addr: target user address of this page + * @pfn: source kernel pfn + * + * Similar to vm_insert_page, this allows drivers to insert individual pages + * they've allocated into a user vma. Same comments apply. + * + * This function should only be called from a vm_ops->fault handler, and + * in that case the handler should return the result of this function. + * + * vma cannot be a COW mapping. + * + * As this is called only for pages that do not currently exist, we + * do not need to flush old virtual caches or the TLB. + * + * Context: Process context. May allocate using %GFP_KERNEL. + * Return: vm_fault_t value. + */ +static inline vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn) +{ + return vmf_insert_pfn_mkwrite(vma, addr, pfn, false); +} + static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma, unsigned long addr, struct page *page) { diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 6916def3ce571..dd93f1b0c44a2 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1692,7 +1692,7 @@ static vm_fault_t insert_pmd(struct vm_area_struct *vma, unsigned long addr, * @pfn: pfn to insert * @write: whether it's a write fault * - * Insert a pmd size pfn. See vmf_insert_pfn() for additional info. + * Insert a pmd size pfn. See vmf_insert_pfn_mkwrite() for additional info. * * Return: vm_fault_t value. */ diff --git a/mm/memory.c b/mm/memory.c index 13b70861c8a3e..08fbf6065ae89 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2719,40 +2719,55 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, } /** - * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot + * vmf_insert_pfn_prot_mkwrite - insert single pfn into user vma with specified pgprot * @vma: user vma to map to * @addr: target user address of this page * @pfn: source kernel pfn * @pgprot: pgprot flags for the inserted page + * @mkwrite: whether to make the page writable. * - * This is exactly like vmf_insert_pfn(), except that it allows drivers - * to override pgprot on a per-page basis. + * This is the function underlying all the others in the vmf_insert_pfn() + * family. It is the most flexible, as it allows drivers to override pgprot + * on a per-page basis, as well as to insert the pfn as if it already had + * a write fault. vmf_insert_pfn() is usually sufficient, however. * - * This only makes sense for IO mappings, and it makes no sense for - * COW mappings. In general, using multiple vmas is preferable; - * vmf_insert_pfn_prot should only be used if using multiple VMAs is - * impractical. - * - * pgprot typically only differs from @vma->vm_page_prot when drivers set - * caching- and encryption bits different than those of @vma->vm_page_prot, - * because the caching- or encryption mode may not be known at mmap() time. + * These functions should only be called from a vm_ops->fault handler, and + * in that case the handler should return the result of these functions. * - * This is ok as long as @vma->vm_page_prot is not used by the core vm + * This only makes sense for IO mappings, and it makes no sense for + * COW mappings. + * + * For vmf_insert_pfn_prot_mkwrite() and vmf_insert_pfn_mkwrite(), the + * @mkwrite argument allows installing a writable PTE even when @vma is + * under write notification, i.e. when it has a .pfn_mkwrite() callback. + * In this case, vma_set_page_prot() has cleared the write bit from + * @vma->vm_page_prot. This lets the fault() callback install a writable + * PTE in response to write faults; note that .pfn_mkwrite() is not called, + * and therefore the caller has to do by itself whatever the callback would + * have done. + * + * For vmf_insert_pfn_prot_mkwrite() and vmf_insert_pfn_prot(), + * pgprot can differ from @vma->vm_page_prot. This typically happens only + * for caching and encryption bits, which may not be known at mmap() time; + * it is ok as long as @vma->vm_page_prot is not used by the core vm * to set caching and encryption bits for those vmas (except for COW pages). - * This is ensured by core vm only modifying these page table entries using - * functions that don't touch caching- or encryption bits, using pte_modify() - * if needed. (See for example mprotect()). + * This is ensured in two ways: * - * Also when new page-table entries are created, this is only done using the - * fault() callback, and never using the value of vma->vm_page_prot, - * except for page-table entries that point to anonymous pages as the result - * of COW. + * - core vm only modifies these page table entries using functions that don't + * touch caching- or encryption bits, using pte_modify() if needed. (See + * for example mprotect()). + * + * - when new page-table entries are created, this is only done using the + * fault() callback, and never using the value of vma->vm_page_prot, + * except for page-table entries that point to anonymous pages as the result + * of COW. * * Context: Process context. May allocate using %GFP_KERNEL. * Return: vm_fault_t value. */ -vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, pgprot_t pgprot) +vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, pgprot_t pgprot, + bool mkwrite) { /* * Technically, architectures with pte_special can avoid all these @@ -2774,36 +2789,9 @@ vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, pfnmap_setup_cachemode_pfn(pfn, &pgprot); - return insert_pfn(vma, addr, pfn, pgprot, false); -} -EXPORT_SYMBOL(vmf_insert_pfn_prot); - -/** - * vmf_insert_pfn - insert single pfn into user vma - * @vma: user vma to map to - * @addr: target user address of this page - * @pfn: source kernel pfn - * - * Similar to vm_insert_page, this allows drivers to insert individual pages - * they've allocated into a user vma. Same comments apply. - * - * This function should only be called from a vm_ops->fault handler, and - * in that case the handler should return the result of this function. - * - * vma cannot be a COW mapping. - * - * As this is called only for pages that do not currently exist, we - * do not need to flush old virtual caches or the TLB. - * - * Context: Process context. May allocate using %GFP_KERNEL. - * Return: vm_fault_t value. - */ -vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn) -{ - return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot); + return insert_pfn(vma, addr, pfn, pgprot, mkwrite); } -EXPORT_SYMBOL(vmf_insert_pfn); +EXPORT_SYMBOL(vmf_insert_pfn_prot_mkwrite); static bool vm_mixed_ok(struct vm_area_struct *vma, unsigned long pfn, bool mkwrite) From b2f69c626877662885704b9ab01718f8188759a6 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 4 Aug 2026 14:05:24 +0200 Subject: [PATCH 0419/1579] drm/shmem_helper: use vmf_insert_pfn_mkwrite() This ensures that KVM or VFIO correctly see a writable PTE when they request one. Otherwise, a guest write to an unpopulated PTE from a mapping backed by a DRM GEM BO triggers a VM exit with EFAULT. The code actually is simpler, because the same logic already applied to the hugepage mapping case using vmf_insert_pfn_pmd(). Reported-by: Sergio Lopez Link: https://lore.kernel.org/kvm/20260729072044.25796-1-slp@redhat.com/ Tested-by: Sergio Lopez Reviewed-by: Boris Brezillon Fixes: 28e3918179aa ("drm/gem-shmem: Track folio accessed/dirty status in mmap") Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini (cherry picked from commit c2b3c978fe93b0146c70eb63362e6f88c757d8b0) Signed-off-by: Diljit Singh --- drivers/gpu/drm/drm_gem_shmem_helper.c | 38 ++++++++++++++------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index ddea1efcc3138..5b8044508cfcf 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -569,11 +569,25 @@ static void drm_gem_shmem_record_mkwrite(struct vm_fault *vmf) folio_mark_dirty(page_folio(shmem->pages[page_offset])); } +/* + * Because the vm_ops have a .pfn_mkwrite() callback, vma_set_page_prot() + * has cleared the write bit from vma->vm_page_prot. vmf_insert_pfn() + * would install a read-only entry even for a write fault, relying on a + * second fault to reach .pfn_mkwrite() and upgrade it, but that second + * fault never happens for fixup_user_fault() callers that directly + * walk the page tables with follow_pfnmap_start(). To ensure that + * they don't see the read-only entry, pass FAULT_FLAG_WRITE info down + * to install a writable entry right away. Because .pfn_mkwrite() is + * not invoked, record the write afterwards. + */ static vm_fault_t try_insert_pfn(struct vm_fault *vmf, unsigned int order, unsigned long pfn) { + bool write = vmf->flags & FAULT_FLAG_WRITE; + vm_fault_t ret = VM_FAULT_FALLBACK; + if (!order) { - return vmf_insert_pfn(vmf->vma, vmf->address, pfn); + ret = vmf_insert_pfn_mkwrite(vmf->vma, vmf->address, pfn, write); #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP } else if (order == PMD_ORDER) { unsigned long paddr = pfn << PAGE_SHIFT; @@ -585,27 +599,15 @@ static vm_fault_t try_insert_pfn(struct vm_fault *vmf, unsigned int order, if (aligned && in_range && folio_test_pmd_mappable(page_folio(pfn_to_page(pfn)))) { - vm_fault_t ret; - pfn &= PMD_MASK >> PAGE_SHIFT; - - /* Unlike PTEs which are automatically upgraded to - * writeable entries, the PMD upgrades go through - * .huge_fault(). Make sure we pass the "write" info - * along in that case. - * This also means we have to record the write fault - * here, instead of in .pfn_mkwrite(). - */ - ret = vmf_insert_pfn_pmd(vmf, pfn, - vmf->flags & FAULT_FLAG_WRITE); - if (ret == VM_FAULT_NOPAGE && (vmf->flags & FAULT_FLAG_WRITE)) - drm_gem_shmem_record_mkwrite(vmf); - - return ret; + ret = vmf_insert_pfn_pmd(vmf, pfn, write); } #endif } - return VM_FAULT_FALLBACK; + + if (ret == VM_FAULT_NOPAGE && write) + drm_gem_shmem_record_mkwrite(vmf); + return ret; } static vm_fault_t drm_gem_shmem_any_fault(struct vm_fault *vmf, unsigned int order) From 3607582ad7900e9a9d15ba53ecce85089da29d72 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 4 Aug 2026 14:05:27 +0200 Subject: [PATCH 0420/1579] mm: pull writability check to follow_pfnmap_start() All callers of follow_pfnmap_start() except s390_pci_mmio_write() are following it, if they are doing a write, with a check that args.writable is true; for s390_pci_mmio_write() that's a bug. Also, most of them return -EFAULT if it is not. Pull the check directly into follow_pfnmap_start() through another input parameter args.write_fault, to eliminate the need to do it in the caller. This also fixes an issue where follow_pfnmap_start() would return 0 for a PFN that is mapped read-only, and the caller would not attempt to call fixup_user_fault() on it; this can happen with vm_ops that set .pfn_mkwrite(), for example. Instead, now the caller (for example hva_to_pfn_remapped()) sees an error, does attempt to fix it, and only returns -EFAULT if the fixup was fruitless. Reported-by: Sergio Lopez Fixes: 28e3918179aa ("drm/gem-shmem: Track folio accessed/dirty status in mmap") Link: https://lore.kernel.org/kvm/CAAiTLFU1ALsDoJoKW3d9bUvv990AozAoX=bEHmfnG54qyBAHFg@mail.gmail.com/ Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini (cherry picked from commit 2e835aa2526160cf112c5fe5257b4299d46107d1) Signed-off-by: Diljit Singh --- arch/s390/pci/pci_mmio.c | 2 ++ drivers/vfio/vfio_iommu_type1.c | 17 +++++---- drivers/virt/acrn/mm.c | 10 +----- include/linux/mm.h | 3 ++ mm/memory.c | 62 ++++++++++++++++++++------------- virt/kvm/kvm_main.c | 15 ++++---- 6 files changed, 58 insertions(+), 51 deletions(-) diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index f3f79ba78410a..cc1d68f2223fb 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -180,6 +180,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, args.address = mmio_addr; args.vma = vma; + args.write = true; ret = follow_pfnmap_start(&args); if (ret) { fixup_user_fault(current->mm, mmio_addr, FAULT_FLAG_WRITE, NULL); @@ -333,6 +334,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, args.vma = vma; args.address = mmio_addr; + args.write = false; ret = follow_pfnmap_start(&args); if (ret) { fixup_user_fault(current->mm, mmio_addr, 0, NULL); diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index c8151ba54de33..e6d3a2311a99b 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -541,7 +541,11 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, unsigned long vaddr, unsigned long *pfn, unsigned long *addr_mask, bool write_fault) { - struct follow_pfnmap_args args = { .vma = vma, .address = vaddr }; + struct follow_pfnmap_args args = { + .vma = vma, + .address = vaddr, + .write = write_fault, + }; int ret; ret = follow_pfnmap_start(&args); @@ -563,15 +567,10 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, return ret; } - if (write_fault && !args.writable) { - ret = -EFAULT; - } else { - *pfn = args.pfn; - *addr_mask = args.addr_mask; - } - + *pfn = args.pfn; + *addr_mask = args.addr_mask; follow_pfnmap_end(&args); - return ret; + return 0; } /* diff --git a/drivers/virt/acrn/mm.c b/drivers/virt/acrn/mm.c index 5bca500a83e05..2f9808399f19d 100644 --- a/drivers/virt/acrn/mm.c +++ b/drivers/virt/acrn/mm.c @@ -177,7 +177,6 @@ int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) vma = vma_lookup(current->mm, memmap->vma_base); if (vma && ((vma->vm_flags & VM_PFNMAP) != 0)) { unsigned long start_pfn, cur_pfn; - bool writable; if ((memmap->vma_base + memmap->len) > vma->vm_end) { mmap_read_unlock(current->mm); @@ -188,6 +187,7 @@ int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) struct follow_pfnmap_args args = { .vma = vma, .address = memmap->vma_base + i * PAGE_SIZE, + .write = !!(memmap->attr & ACRN_MEM_ACCESS_WRITE), }; ret = follow_pfnmap_start(&args); @@ -197,16 +197,8 @@ int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) cur_pfn = args.pfn; if (i == 0) start_pfn = cur_pfn; - writable = args.writable; follow_pfnmap_end(&args); - /* Disallow write access if the PTE is not writable. */ - if (!writable && - (memmap->attr & ACRN_MEM_ACCESS_WRITE)) { - ret = -EFAULT; - break; - } - /* Disallow refcounted pages. */ if (pfn_valid(cur_pfn) && !PageReserved(pfn_to_page(cur_pfn))) { diff --git a/include/linux/mm.h b/include/linux/mm.h index 65d47f4fafa55..f4481c338b36c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3141,9 +3141,12 @@ struct follow_pfnmap_args { * Inputs: * @vma: Pointer to @vm_area_struct struct * @address: the virtual address to walk + * @write: if true, fail with -EFAULT unless the mapping is + * writable */ struct vm_area_struct *vma; unsigned long address; + bool write; /** * Internals: * diff --git a/mm/memory.c b/mm/memory.c index 08fbf6065ae89..a903d5c1859cb 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6822,12 +6822,15 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) } #endif /* __PAGETABLE_PMD_FOLDED */ -static inline void pfnmap_args_setup(struct follow_pfnmap_args *args, - spinlock_t *lock, pte_t *ptep, - pgprot_t pgprot, unsigned long pfn_base, - unsigned long addr_mask, bool writable, - bool special) +static inline int pfnmap_args_setup(struct follow_pfnmap_args *args, + spinlock_t *lock, pte_t *ptep, + pgprot_t pgprot, unsigned long pfn_base, + unsigned long addr_mask, bool writable, + bool special) { + if (!writable && args->write) + return -EFAULT; + args->lock = lock; args->ptep = ptep; args->pfn = pfn_base + ((args->address & ~addr_mask) >> PAGE_SHIFT); @@ -6835,6 +6838,7 @@ static inline void pfnmap_args_setup(struct follow_pfnmap_args *args, args->pgprot = pgprot; args->writable = writable; args->special = special; + return 0; } static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma) @@ -6856,8 +6860,9 @@ static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma) * @args: Pointer to struct @follow_pfnmap_args * * The caller needs to setup args->vma and args->address to point to the - * virtual address as the target of such lookup. On a successful return, - * the results will be put into other output fields. + * virtual address as the target of such lookup, and optionally set + * args->write to require a writable mapping. On a successful + * return, the results will be put into other output fields. * * After the caller finished using the fields, the caller must invoke * another follow_pfnmap_end() to proper releases the locks and resources @@ -6880,7 +6885,8 @@ static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma) * * This function must not be used to modify PTE content. * - * Return: zero on success, negative otherwise. + * Return: zero on success, -EFAULT if @args->write was set but the + * mapping is not writable, -EINVAL if there is no mapping at all. */ int follow_pfnmap_start(struct follow_pfnmap_args *args) { @@ -6893,6 +6899,7 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) pud_t *pudp, pud; pmd_t *pmdp, pmd; pte_t *ptep, pte; + int r = -EINVAL; pfnmap_lockdep_assert(vma); @@ -6926,10 +6933,12 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) spin_unlock(lock); goto retry; } - pfnmap_args_setup(args, lock, NULL, pud_pgprot(pud), - pud_pfn(pud), PUD_MASK, pud_write(pud), - pud_special(pud)); - return 0; + r = pfnmap_args_setup(args, lock, NULL, pud_pgprot(pud), + pud_pfn(pud), PUD_MASK, pud_write(pud), + pud_special(pud)); + if (r) + spin_unlock(lock); + return r; } pmdp = pmd_offset(pudp, address); @@ -6947,10 +6956,12 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) spin_unlock(lock); goto retry; } - pfnmap_args_setup(args, lock, NULL, pmd_pgprot(pmd), - pmd_pfn(pmd), PMD_MASK, pmd_write(pmd), - pmd_special(pmd)); - return 0; + r = pfnmap_args_setup(args, lock, NULL, pmd_pgprot(pmd), + pmd_pfn(pmd), PMD_MASK, pmd_write(pmd), + pmd_special(pmd)); + if (r) + spin_unlock(lock); + return r; } ptep = pte_offset_map_lock(mm, pmdp, address, &lock); @@ -6959,14 +6970,16 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) pte = ptep_get(ptep); if (!pte_present(pte)) goto unlock; - pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte), - pte_pfn(pte), PAGE_MASK, pte_write(pte), - pte_special(pte)); + r = pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte), + pte_pfn(pte), PAGE_MASK, pte_write(pte), + pte_special(pte)); + if (r) + goto unlock; return 0; unlock: pte_unmap_unlock(ptep, lock); out: - return -EINVAL; + return r; } EXPORT_SYMBOL_GPL(follow_pfnmap_start); @@ -7008,7 +7021,11 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, int offset = offset_in_page(addr); int ret = -EINVAL; bool writable; - struct follow_pfnmap_args args = { .vma = vma, .address = addr }; + struct follow_pfnmap_args args = { + .vma = vma, + .address = addr, + .write = !!(write & FOLL_WRITE) + }; retry: if (follow_pfnmap_start(&args)) @@ -7018,9 +7035,6 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, writable = args.writable; follow_pfnmap_end(&args); - if ((write & FOLL_WRITE) && !writable) - return -EINVAL; - maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot); if (!maddr) return -ENOMEM; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 1a529098eec98..837321f6a0602 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2939,8 +2939,11 @@ static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault) static int hva_to_pfn_remapped(struct vm_area_struct *vma, struct kvm_follow_pfn *kfp, kvm_pfn_t *p_pfn) { - struct follow_pfnmap_args args = { .vma = vma, .address = kfp->hva }; - bool write_fault = kfp->flags & FOLL_WRITE; + struct follow_pfnmap_args args = { + .vma = vma, + .address = kfp->hva, + .write = !!(kfp->flags & FOLL_WRITE), + }; int r; /* @@ -2959,7 +2962,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, */ bool unlocked = false; r = fixup_user_fault(current->mm, kfp->hva, - (write_fault ? FAULT_FLAG_WRITE : 0), + (args.write ? FAULT_FLAG_WRITE : 0), &unlocked); if (unlocked) return -EAGAIN; @@ -2971,13 +2974,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, return r; } - if (write_fault && !args.writable) { - *p_pfn = KVM_PFN_ERR_RO_FAULT; - goto out; - } - *p_pfn = kvm_resolve_pfn(kfp, NULL, &args, args.writable); -out: follow_pfnmap_end(&args); return r; } From 81fd8e0b21629d0fdcfd36bed3c684cf278b6113 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0421/1579] mm: make VM operations aware of per-process mapping sizes Separate target-mm virtual geometry from native physical pages across faults, COW, reclaim, swap, GUP, VMAs and accounting. Supply explicit fragment interfaces and keep native defaults until the optional backend is enabled later in the series. Signed-off-by: Diljit Singh --- arch/x86/kernel/uprobes.c | 6 +- fs/proc/array.c | 3 +- fs/proc/task_mmu.c | 295 ++++- fs/userfaultfd.c | 57 +- include/asm-generic/pgalloc.h | 17 +- include/asm-generic/tlb.h | 43 +- include/linux/binfmts.h | 6 + include/linux/coredump.h | 13 +- include/linux/highmem.h | 15 + include/linux/huge_mm.h | 4 + include/linux/khugepaged.h | 8 + include/linux/mm.h | 104 +- include/linux/mm_granule.h | 245 ++++ include/linux/mm_subpage.h | 277 ++++ include/linux/mm_types.h | 31 +- include/linux/mman.h | 11 + include/linux/mmu_context.h | 5 + include/linux/page_table_check.h | 4 + include/linux/pagemap.h | 5 +- include/linux/pgtable.h | 98 +- include/linux/rmap.h | 106 +- include/linux/sched.h | 5 + include/linux/sched/mm.h | 4 + include/linux/sched/signal.h | 2 +- include/linux/shmem_fs.h | 51 + include/linux/swap.h | 26 +- include/linux/syscalls.h | 5 + include/linux/uprobes.h | 1 + include/trace/events/kmem.h | 2 +- include/trace/events/mmap.h | 2 +- include/trace/events/oom.h | 9 +- include/uapi/linux/prctl.h | 18 + ipc/shm.c | 36 +- kernel/events/core.c | 83 +- kernel/events/internal.h | 37 +- kernel/events/ring_buffer.c | 47 +- kernel/events/uprobes.c | 212 ++- kernel/fork.c | 35 +- kernel/futex/core.c | 57 +- kernel/sched/fair.c | 6 +- kernel/sys.c | 52 +- kernel/trace/trace_events_user.c | 36 +- kernel/tsacct.c | 16 +- lib/vdso/datastore.c | 3 +- mm/Kconfig | 30 +- mm/damon/ops-common.c | 10 +- mm/filemap.c | 73 +- mm/gup.c | 590 ++++++++- mm/huge_memory.c | 11 +- mm/internal.h | 87 +- mm/khugepaged.c | 119 +- mm/ksm.c | 9 +- mm/madvise.c | 156 ++- mm/memory-failure.c | 45 + mm/memory.c | 2071 +++++++++++++++++++++++++----- mm/mempolicy.c | 57 +- mm/migrate.c | 85 +- mm/migrate_device.c | 2 +- mm/mincore.c | 43 +- mm/mlock.c | 42 +- mm/mm_init.c | 3 +- mm/mmap.c | 139 +- mm/mmu_gather.c | 67 +- mm/mprotect.c | 143 ++- mm/mremap.c | 89 +- mm/mseal.c | 4 +- mm/msync.c | 6 +- mm/oom_kill.c | 25 +- mm/page_idle.c | 7 +- mm/page_io.c | 61 +- mm/page_table_check.c | 142 +- mm/page_vma_mapped.c | 61 +- mm/pagewalk.c | 49 +- mm/pgtable-generic.c | 35 +- mm/process_vm_access.c | 96 +- mm/rmap.c | 519 +++++++- mm/shmem.c | 512 +++++++- mm/swap.h | 22 +- mm/swap_state.c | 6 +- mm/swapfile.c | 387 +++++- mm/truncate.c | 2 + mm/user-subpage-internal.h | 69 + mm/userfaultfd.c | 518 +++++++- mm/util.c | 106 +- mm/vma.c | 180 ++- mm/vma.h | 40 +- mm/vma_exec.c | 6 +- mm/vma_init.c | 5 +- mm/vmscan.c | 124 +- 89 files changed, 7676 insertions(+), 1275 deletions(-) create mode 100644 include/linux/mm_granule.h create mode 100644 include/linux/mm_subpage.h create mode 100644 mm/user-subpage-internal.h diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 3af979fb41d38..3155eed81c2ec 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -926,13 +926,15 @@ static int is_call_insn(uprobe_opcode_t *insn) * Verification callback used by int3_update uprobe_write calls to make sure * the underlying instruction is as expected - either int3 or call. */ -static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode, +static int verify_insn(struct page *page, unsigned long vaddr, unsigned int page_offset, + uprobe_opcode_t *new_opcode, int nbytes, void *data) { struct write_opcode_ctx *ctx = data; uprobe_opcode_t old_opcode[5]; - uprobe_copy_from_page(page, ctx->base, (uprobe_opcode_t *) &old_opcode, 5); + uprobe_copy_from_page(page, page_offset - (vaddr - ctx->base), + (uprobe_opcode_t *) &old_opcode, 5); switch (ctx->expect) { case EXPECT_SWBP: diff --git a/fs/proc/array.c b/fs/proc/array.c index 479ea8cb4ef43..5b45506942991 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -610,7 +610,8 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, seq_put_decimal_ull(m, " ", 0); seq_put_decimal_ull(m, " ", start_time); seq_put_decimal_ull(m, " ", vsize); - seq_put_decimal_ull(m, " ", mm ? get_mm_rss(mm) : 0); + seq_put_decimal_ull(m, " ", mm ? + mm_pages_for_reader(mm, current->mm, get_mm_rss(mm)) : 0); seq_put_decimal_ull(m, " ", rsslim); seq_put_decimal_ull(m, " ", mm ? (permitted ? mm->start_code : 1) : 0); seq_put_decimal_ull(m, " ", mm ? (permitted ? mm->end_code : 1) : 0); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 436e1c1d11f41..8938ae6758100 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include #include #include @@ -33,7 +34,7 @@ #define SENTINEL_VMA_GATE -2 #define SEQ_PUT_DEC(str, val) \ - seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8) + seq_put_decimal_ull_width(m, str, mm_pages_to_kb(mm, val), 8) void task_mem(struct seq_file *m, struct mm_struct *mm) { unsigned long text, lib, swap, anon, file, shmem; @@ -58,15 +59,18 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) hiwater_rss = mm->hiwater_rss; /* split executable areas between text and lib */ - text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK); - text = min(text, mm->exec_vm << PAGE_SHIFT); - lib = (mm->exec_vm << PAGE_SHIFT) - text; + text = mm_page_align(mm, mm->end_code) - + (mm->start_code & mm_page_mask(mm)); + text = min(text, mm_pages_to_bytes(mm, mm->exec_vm)); + lib = (mm_pages_to_bytes(mm, mm->exec_vm)) - text; swap = get_mm_counter_sum(mm, MM_SWAPENTS); SEQ_PUT_DEC("VmPeak:\t", hiwater_vm); SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm); - SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm); - SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm)); + seq_put_decimal_ull_width(m, " kB\nVmLck:\t", + mm_locked_vm_bytes(mm) >> 10, 8); + seq_put_decimal_ull_width(m, " kB\nVmPin:\t", + atomic64_read(&mm->pinned_vm) << (PAGE_SHIFT - 10), 8); SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss); SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss); SEQ_PUT_DEC(" kB\nRssAnon:\t", anon); @@ -88,20 +92,25 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) unsigned long task_vsize(struct mm_struct *mm) { - return PAGE_SIZE * mm->total_vm; + return mm_pages_to_bytes(mm, mm->total_vm); } unsigned long task_statm(struct mm_struct *mm, unsigned long *shared, unsigned long *text, unsigned long *data, unsigned long *resident) { - *shared = get_mm_counter_sum(mm, MM_FILEPAGES) + + struct mm_struct *reader = current->mm; + unsigned long file = get_mm_counter_sum(mm, MM_FILEPAGES) + get_mm_counter_sum(mm, MM_SHMEMPAGES); - *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) - >> PAGE_SHIFT; - *data = mm->data_vm + mm->stack_vm; - *resident = *shared + get_mm_counter_sum(mm, MM_ANONPAGES); - return mm->total_vm; + unsigned long code = (mm_page_align(mm, mm->end_code) - + (mm->start_code & mm_page_mask(mm))) >> mm_page_shift(mm); + + *shared = mm_pages_for_reader(mm, reader, file); + *text = mm_pages_for_reader(mm, reader, code); + *data = mm_pages_for_reader(mm, reader, mm->data_vm + mm->stack_vm); + *resident = mm_pages_for_reader(mm, reader, + file + get_mm_counter_sum(mm, MM_ANONPAGES)); + return mm_pages_for_reader(mm, reader, mm->total_vm); } #ifdef CONFIG_NUMA @@ -475,7 +484,8 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma) dev = inode->i_sb->s_dev; ino = inode->i_ino; - pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT; + pgoff = (((loff_t)vma->vm_pgoff) << PAGE_SHIFT) + + vma_subpage_offset(vma); } start = vma->vm_start; @@ -717,7 +727,8 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) if (vma->vm_file) { const struct inode *inode = file_user_inode(vma->vm_file); - karg.vma_offset = ((__u64)vma->vm_pgoff) << PAGE_SHIFT; + karg.vma_offset = (((__u64)vma->vm_pgoff) << PAGE_SHIFT) + + vma_subpage_offset(vma); karg.dev_major = MAJOR(inode->i_sb->s_dev); karg.dev_minor = MINOR(inode->i_sb->s_dev); karg.inode = inode->i_ino; @@ -857,6 +868,50 @@ const struct file_operations proc_pid_maps_operations = { */ #define PSS_SHIFT 12 +#if defined(CONFIG_MM_SUBPAGE) && (defined(CONFIG_PROC_PAGE_MONITOR) || defined(CONFIG_NUMA)) +/* Snapshot only: rmap and quarter counters can change in other address spaces. */ +static void file_quarter_mapcounts(struct page *page, struct mm_subpage_mapcounts *counts) +{ + int maps = folio_precise_page_mapcount(page_folio(page), page); + unsigned int small = mm_subpage_file_mapcounts(page, counts); + unsigned int native = maps > small ? maps - small : 0; + unsigned int i; + + for (i = 0; i < MM_SUBPAGES_PER_PAGE; i++) + counts->slots[i] += native; +} + +static unsigned int proc_leaf_mapcount(struct vm_area_struct *vma, pte_t pte, + struct page *page) +{ + struct folio *folio = page_folio(page); + struct mm_subpage_mapcounts counts; + unsigned int i, count = 0; + + if (!folio_test_anon(folio)) { + unsigned int first = mm_page_size(vma->vm_mm) < PAGE_SIZE ? + offset_in_page(pte_phys_mm(vma->vm_mm, pte)) / SZ_4K : 0; + unsigned int nr = mm_page_size(vma->vm_mm) / SZ_4K; + + file_quarter_mapcounts(page, &counts); + for (i = first; i < first + nr; i++) + count = max(count, counts.slots[i]); + return count; + } + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + + if (slot) { + count = mm_subpage_mapcount(slot); + mm_subpage_put(slot); + } + return count; + } + return folio_precise_page_mapcount(folio, page); +} + +#endif + #ifdef CONFIG_PROC_PAGE_MONITOR struct mem_size_stats { unsigned long resident; @@ -913,6 +968,73 @@ static void smaps_page_accumulate(struct mem_size_stats *mss, } } +#ifdef CONFIG_MM_SUBPAGE +static void smaps_account_file_extent(struct mem_size_stats *mss, struct page *page, + unsigned int offset, unsigned long size, bool dirty, bool locked, bool present) +{ + struct folio *folio = page_folio(page); + struct mm_subpage_mapcounts counts = {}; + unsigned int i; + + if (present) + file_quarter_mapcounts(page, &counts); + for (i = offset / SZ_4K; i < (offset + size) / SZ_4K; i++) { + unsigned long pss = SZ_4K << PSS_SHIFT; + + if (counts.slots[i] > 1) + pss /= counts.slots[i]; + smaps_page_accumulate(mss, folio, SZ_4K, pss, dirty, locked, + present && counts.slots[i] < 2); + } +} + +static void smaps_account_small(struct mem_size_stats *mss, struct vm_area_struct *vma, + struct page *page, pte_t pte, bool young, + bool dirty, bool locked, bool present) +{ + struct folio *folio = page_folio(page); + unsigned long size = mm_page_size(vma->vm_mm), pss = size << PSS_SHIFT; + unsigned int count = present && folio_test_anon(folio) ? + proc_leaf_mapcount(vma, pte, page) : 0; + + mss->resident += size; + if (folio_test_anon(folio)) { + mss->anonymous += size; + if (present && !dirty) { + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + + if (slot) { + if (mm_subpage_is_lazyfree(slot)) + mss->lazyfree += size; + mm_subpage_put(slot); + } + } + } + if (folio_test_ksm(folio)) + mss->ksm += size; + if (young || folio_test_young(folio) || folio_test_referenced(folio)) + mss->referenced += size; + if (present && !folio_test_anon(folio)) { + smaps_account_file_extent(mss, page, offset_in_page(pte_phys_mm(vma->vm_mm, pte)), + size, dirty, locked, present); + return; + } + if (count > 1) + pss /= count; + smaps_page_accumulate(mss, folio, size, pss, dirty, locked, present && count < 2); +} + +static bool smaps_account_file_quarters(struct mem_size_stats *mss, struct page *page, + bool dirty, bool locked) +{ + if (folio_test_anon(page_folio(page))) + return false; + smaps_account_file_extent(mss, page, 0, PAGE_SIZE, dirty, locked, true); + return true; +} + +#endif + static void smaps_account(struct mem_size_stats *mss, struct page *page, bool compound, bool young, bool dirty, bool locked, bool present) @@ -976,6 +1098,11 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page, for (i = 0; i < nr; i++, page++) { unsigned long pss = PAGE_SIZE << PSS_SHIFT; +#ifdef CONFIG_MM_SUBPAGE + if (smaps_account_file_quarters(mss, page, dirty, locked)) + continue; +#endif + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) { mapcount = folio_precise_page_mapcount(folio, page); exclusive = mapcount < 2; @@ -995,6 +1122,14 @@ static int smaps_pte_hole(unsigned long addr, unsigned long end, struct mem_size_stats *mss = walk->private; struct vm_area_struct *vma = walk->vma; + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + u64 start = ((u64)pos.index << PAGE_SHIFT) + pos.offset; + + mss->swap += shmem_partial_swap_usage_bytes(vma->vm_file->f_mapping, + start, start + end - addr); + return 0; + } mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping, linear_page_index(vma, addr), linear_page_index(vma, end)); @@ -1010,7 +1145,7 @@ static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk) #ifdef CONFIG_SHMEM if (walk->ops->pte_hole) { /* depth is not used */ - smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk); + smaps_pte_hole(addr, addr + mm_page_size(walk->mm), 0, walk); } #endif } @@ -1038,15 +1173,20 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr, if (softleaf_is_swap(entry)) { int mapcount; - mss->swap += PAGE_SIZE; - mapcount = swp_swapcount(entry); + mss->swap += mm_page_size(vma->vm_mm); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + mapcount = swap_subpage_count(entry, pte_swp_subpage_offset(ptent)); + else +#endif + mapcount = swp_swapcount(entry); if (mapcount >= 2) { - u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT; + u64 pss_delta = (u64)mm_page_size(vma->vm_mm) << PSS_SHIFT; do_div(pss_delta, mapcount); mss->swap_pss += pss_delta; } else { - mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT; + mss->swap_pss += (u64)mm_page_size(vma->vm_mm) << PSS_SHIFT; } } else if (softleaf_has_pfn(entry)) { if (softleaf_is_device_private(entry)) @@ -1058,6 +1198,12 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr, if (!page) return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + smaps_account_small(mss, vma, page, ptent, young, dirty, locked, present); + return; + } +#endif smaps_account(mss, page, false, young, dirty, locked, present); } @@ -1124,7 +1270,7 @@ static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte++, addr += PAGE_SIZE) + for (; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) smaps_pte_entry(pte, addr, walk); pte_unmap_unlock(pte - 1, ptl); out: @@ -1712,7 +1858,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte++, addr += PAGE_SIZE) { + for (; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) { ptent = ptep_get(pte); if (cp->type == CLEAR_REFS_SOFT_DIRTY) { @@ -1855,9 +2001,6 @@ struct pagemapread { bool show_pfn; }; -#define PAGEMAP_WALK_SIZE (PMD_SIZE) -#define PAGEMAP_WALK_MASK (PMD_MASK) - #define PM_ENTRY_BYTES sizeof(pagemap_entry_t) #define PM_PFRAME_BITS 55 #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0) @@ -1884,7 +2027,7 @@ static int add_to_pagemap(pagemap_entry_t *pme, struct pagemapread *pm) return 0; } -static bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page) +static inline bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page) { if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) return folio_precise_page_mapcount(folio, page) == 1; @@ -1909,7 +2052,7 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, else hole_end = end; - for (; addr < hole_end; addr += PAGE_SIZE) { + for (; addr < hole_end; addr += mm_page_size(walk->mm)) { err = add_to_pagemap(&pme, pm); if (err) goto out; @@ -1921,7 +2064,7 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, /* Addresses in the VMA. */ if (vma->vm_flags & VM_SOFTDIRTY) pme = make_pme(0, PM_SOFT_DIRTY); - for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) { + for (; addr < min(end, vma->vm_end); addr += mm_page_size(walk->mm)) { err = add_to_pagemap(&pme, pm); if (err) goto out; @@ -1931,6 +2074,16 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, return err; } +static bool pagemap_page_exclusive(struct vm_area_struct *vma, pte_t pte, + struct folio *folio, struct page *page) +{ +#ifdef CONFIG_MM_SUBPAGE + return proc_leaf_mapcount(vma, pte, page) == 1; +#else + return __folio_page_mapped_exclusively(folio, page); +#endif +} + static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, struct vm_area_struct *vma, unsigned long addr, pte_t pte) { @@ -1943,7 +2096,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, if (pte_present(pte)) { if (pm->show_pfn) - frame = pte_pfn(pte); + frame = pte_phys_mm(vma->vm_mm, pte) >> mm_page_shift(vma->vm_mm); flags |= PM_PRESENT; page = vm_normal_page(vma, addr, pte); if (pte_soft_dirty(pte)) @@ -1969,6 +2122,12 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, offset = softleaf_to_pfn(entry); else offset = swp_offset(entry); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE && + (softleaf_is_swap(entry) || softleaf_has_pfn(entry))) + offset = (offset << (PAGE_SHIFT - mm_page_shift(vma->vm_mm))) + + (pte_swp_subpage_offset(pte) >> mm_page_shift(vma->vm_mm)); +#endif frame = swp_type(entry) | (offset << MAX_SWAPFILES_SHIFT); } @@ -1985,8 +2144,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, folio = page_folio(page); if (!folio_test_anon(folio)) flags |= PM_FILE; - if ((flags & PM_PRESENT) && - __folio_page_mapped_exclusively(folio, page)) + if ((flags & PM_PRESENT) && pagemap_page_exclusive(vma, pte, folio, page)) flags |= PM_MMAP_EXCLUSIVE; } @@ -2102,7 +2260,7 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return err; } - for (; addr < end; pte++, addr += PAGE_SIZE) { + for (; addr < end; pte++, addr += mm_page_size(walk->mm)) { pagemap_entry_t pme; pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte)); @@ -2236,7 +2394,7 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, /* do not disclose physical addresses: attack vector */ pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN); - pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT); + pm.len = (pmd_size_mm(mm) >> mm_page_shift(mm)); pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL); ret = -ENOMEM; if (!pm.buffer) @@ -2248,16 +2406,16 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, /* watch out for wraparound */ start_vaddr = end_vaddr; - if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) { + if (svpfn <= (ULONG_MAX >> mm_page_shift(mm))) { unsigned long end; ret = mmap_read_lock_killable(mm); if (ret) goto out_free; - start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT); + start_vaddr = untagged_addr_remote(mm, svpfn << mm_page_shift(mm)); mmap_read_unlock(mm); - end = start_vaddr + ((count / PM_ENTRY_BYTES) << PAGE_SHIFT); + end = start_vaddr + ((count / PM_ENTRY_BYTES) << mm_page_shift(mm)); if (end >= start_vaddr && end < mm->task_size) end_vaddr = end; } @@ -2272,7 +2430,7 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, unsigned long end; pm.pos = 0; - end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK; + end = (start_vaddr + pmd_size_mm(mm)) & pmd_mask_mm(mm); /* overflow ? */ if (end < start_vaddr || end > end_vaddr) end = end_vaddr; @@ -2333,6 +2491,7 @@ static int pagemap_release(struct inode *inode, struct file *file) struct pagemap_scan_private { struct pm_scan_arg arg; + unsigned long page_size; unsigned long masks_of_interest, cur_vma_category; struct page_region *vec_buf; unsigned long vec_buf_len, vec_buf_index, found_pages; @@ -2559,7 +2718,7 @@ static void pagemap_scan_backout_range(struct pagemap_scan_private *p, else cur_buf->start = cur_buf->end = 0; - p->found_pages -= (end - addr) / PAGE_SIZE; + p->found_pages -= (end - addr) / p->page_size; } #endif @@ -2671,11 +2830,11 @@ static int pagemap_scan_output(unsigned long categories, categories &= p->arg.return_mask; - n_pages = (*end - addr) / PAGE_SIZE; + n_pages = (*end - addr) / p->page_size; if (check_add_overflow(p->found_pages, n_pages, &total_pages) || total_pages > p->arg.max_pages) { size_t n_too_much = total_pages - p->arg.max_pages; - *end -= n_too_much * PAGE_SIZE; + *end -= n_too_much * p->page_size; n_pages -= n_too_much; ret = -ENOSPC; } @@ -2769,7 +2928,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) { /* Fast path for performing exclusive WP */ - for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { + for (addr = start; addr != end; pte++, addr += p->page_size) { pte_t ptent = ptep_get(pte); if ((pte_present(ptent) && pte_uffd_wp(ptent)) || @@ -2778,7 +2937,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, make_uffd_wp_pte(vma, addr, pte, ptent); if (!flush_end) start = addr; - flush_end = addr + PAGE_SIZE; + flush_end = addr + p->page_size; } goto flush_and_return; } @@ -2786,8 +2945,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, if (!p->arg.category_anyof_mask && !p->arg.category_inverted && p->arg.category_mask == PAGE_IS_WRITTEN && p->arg.return_mask == PAGE_IS_WRITTEN) { - for (addr = start; addr < end; pte++, addr += PAGE_SIZE) { - unsigned long next = addr + PAGE_SIZE; + for (addr = start; addr < end; pte++, addr += p->page_size) { + unsigned long next = addr + p->page_size; pte_t ptent = ptep_get(pte); if ((pte_present(ptent) && pte_uffd_wp(ptent)) || @@ -2807,11 +2966,11 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, goto flush_and_return; } - for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { + for (addr = start; addr != end; pte++, addr += p->page_size) { pte_t ptent = ptep_get(pte); unsigned long categories = p->cur_vma_category | pagemap_page_category(p, vma, addr, ptent); - unsigned long next = addr + PAGE_SIZE; + unsigned long next = addr + p->page_size; if (!pagemap_scan_is_interesting_page(categories, p)) continue; @@ -3007,7 +3166,7 @@ static const struct mm_walk_ops pagemap_scan_ops = { .hugetlb_entry = pagemap_scan_hugetlb_entry, }; -static int pagemap_scan_get_args(struct pm_scan_arg *arg, +static int pagemap_scan_get_args(struct mm_struct *mm, struct pm_scan_arg *arg, unsigned long uarg) { if (copy_from_user(arg, (void __user *)uarg, sizeof(*arg))) @@ -3028,7 +3187,7 @@ static int pagemap_scan_get_args(struct pm_scan_arg *arg, arg->vec = untagged_addr((unsigned long)arg->vec); /* Validate memory pointers */ - if (!IS_ALIGNED(arg->start, PAGE_SIZE)) + if (!IS_ALIGNED(arg->start, mm_page_size(mm))) return -EINVAL; if (!access_ok((void __user *)(long)arg->start, arg->end - arg->start)) return -EFAULT; @@ -3041,7 +3200,7 @@ static int pagemap_scan_get_args(struct pm_scan_arg *arg, return -EFAULT; /* Fixup default values */ - arg->end = ALIGN(arg->end, PAGE_SIZE); + arg->end = ALIGN(arg->end, mm_page_size(mm)); arg->walk_end = 0; if (!arg->max_pages) arg->max_pages = ULONG_MAX; @@ -3060,12 +3219,13 @@ static int pagemap_scan_writeback_args(struct pm_scan_arg *arg, return 0; } -static int pagemap_scan_init_bounce_buffer(struct pagemap_scan_private *p) +static int pagemap_scan_init_bounce_buffer(struct mm_struct *mm, + struct pagemap_scan_private *p) { if (!p->arg.vec_len) return 0; - p->vec_buf_len = min_t(size_t, PAGEMAP_WALK_SIZE >> PAGE_SHIFT, + p->vec_buf_len = min_t(size_t, pmd_size_mm(mm) >> mm_page_shift(mm), p->arg.vec_len); p->vec_buf = kmalloc_objs(*p->vec_buf, p->vec_buf_len); if (!p->vec_buf) @@ -3106,18 +3266,18 @@ static long pagemap_scan_flush_buffer(struct pagemap_scan_private *p) static long do_pagemap_scan(struct mm_struct *mm, unsigned long uarg) { - struct pagemap_scan_private p = {0}; + struct pagemap_scan_private p = { .page_size = mm_page_size(mm) }; unsigned long walk_start; size_t n_ranges_out = 0; int ret; - ret = pagemap_scan_get_args(&p.arg, uarg); + ret = pagemap_scan_get_args(mm, &p.arg, uarg); if (ret) return ret; p.masks_of_interest = p.arg.category_mask | p.arg.category_anyof_mask | p.arg.return_mask; - ret = pagemap_scan_init_bounce_buffer(&p); + ret = pagemap_scan_init_bounce_buffer(mm, &p); if (ret) return ret; @@ -3219,16 +3379,17 @@ struct numa_maps_private { struct numa_maps md; }; +static int numa_page_mapcount(struct page *page) +{ + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + return folio_precise_page_mapcount(page_folio(page), page); + return folio_average_page_mapcount(page_folio(page)); +} + static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty, - unsigned long nr_pages) + unsigned long nr_pages, int count) { struct folio *folio = page_folio(page); - int count; - - if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) - count = folio_precise_page_mapcount(folio, page); - else - count = folio_average_page_mapcount(folio); md->pages += nr_pages; if (pte_dirty || folio_test_dirty(folio)) @@ -3318,7 +3479,7 @@ static int gather_pte_stats(pmd_t *pmd, unsigned long addr, page = can_gather_numa_stats_pmd(*pmd, vma, addr); if (page) gather_stats(page, md, pmd_dirty(*pmd), - HPAGE_PMD_SIZE/PAGE_SIZE); + HPAGE_PMD_SIZE/PAGE_SIZE, numa_page_mapcount(page)); spin_unlock(ptl); return 0; } @@ -3333,9 +3494,13 @@ static int gather_pte_stats(pmd_t *pmd, unsigned long addr, struct page *page = can_gather_numa_stats(ptent, vma, addr); if (!page) continue; - gather_stats(page, md, pte_dirty(ptent), 1); +#ifdef CONFIG_MM_SUBPAGE + gather_stats(page, md, pte_dirty(ptent), 1, proc_leaf_mapcount(vma, ptent, page)); +#else + gather_stats(page, md, pte_dirty(ptent), 1, numa_page_mapcount(page)); +#endif - } while (pte++, addr += PAGE_SIZE, addr != end); + } while (pte++, addr += mm_page_size(vma->vm_mm), addr != end); pte_unmap_unlock(orig_pte, ptl); cond_resched(); return 0; @@ -3357,7 +3522,7 @@ static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, page = pte_page(huge_pte); md = walk->private; - gather_stats(page, md, pte_dirty(huge_pte), 1); + gather_stats(page, md, pte_dirty(huge_pte), 1, numa_page_mapcount(page)); out: spin_unlock(ptl); return 0; diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index dba1172436b74..3174e3b6edb41 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -301,6 +301,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, pmd_t *pmd, _pmd; pte_t *pte; pte_t ptent; + spinlock_t *ptl; bool ret; assert_fault_locked(vmf); @@ -308,13 +309,13 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, pgd = pgd_offset(mm, address); if (!pgd_present(*pgd)) return true; - p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + p4d = p4d_offset_mm(mm, pgd, address); + if (!p4d_present_mm(mm, *p4d)) return true; - pud = pud_offset(p4d, address); + pud = pud_offset_mm(mm, p4d, address); if (!pud_present(*pud)) return true; - pmd = pmd_offset(pud, address); + pmd = pmd_offset_mm(mm, pud, address); again: _pmd = pmdp_get_lockless(pmd); if (pmd_none(_pmd)) @@ -331,7 +332,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, if (pmd_trans_huge(_pmd)) return !pmd_write(_pmd) && (reason & VM_UFFD_WP); - pte = pte_offset_map(pmd, address); + pte = pte_offset_map_ro_nolock(mm, pmd, address, &ptl); if (!pte) goto again; @@ -1251,13 +1252,11 @@ static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx, __wake_userfault(ctx, range); } -static __always_inline int validate_unaligned_range( +static __always_inline int validate_byte_range( struct mm_struct *mm, __u64 start, __u64 len) { __u64 task_size = mm->task_size; - if (len & ~PAGE_MASK) - return -EINVAL; if (!len) return -EINVAL; if (start >= task_size) @@ -1269,10 +1268,19 @@ static __always_inline int validate_unaligned_range( return 0; } +static __always_inline int validate_unaligned_range( + struct mm_struct *mm, __u64 start, __u64 len) +{ + if (len & ~mm_page_mask(mm)) + return -EINVAL; + + return validate_byte_range(mm, start, len); +} + static __always_inline int validate_range(struct mm_struct *mm, __u64 start, __u64 len) { - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; return validate_unaligned_range(mm, start, len); @@ -1428,6 +1436,13 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC : UFFD_API_RANGE_IOCTLS; + if (mm_page_size(mm) != PAGE_SIZE) { + ioctls_out &= (1ULL << _UFFDIO_WAKE) | (1ULL << _UFFDIO_WRITEPROTECT) | + (1ULL << _UFFDIO_CONTINUE) | (1ULL << _UFFDIO_COPY) | + (1ULL << _UFFDIO_ZEROPAGE) | (1ULL << _UFFDIO_POISON); + } + if (mm_page_size(mm) < PAGE_SIZE) + ioctls_out |= 1ULL << _UFFDIO_MOVE; /* * Declare the WP ioctl only if the WP mode is @@ -1645,8 +1660,12 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, sizeof(uffdio_copy)-sizeof(__s64))) goto out; - ret = validate_unaligned_range(ctx->mm, uffdio_copy.src, - uffdio_copy.len); + /* + * COPY reads the pager's address space, which can have a different + * task size and granule from the registered target. Only the target + * range below requires a page-aligned length. + */ + ret = validate_byte_range(current->mm, uffdio_copy.src, uffdio_copy.len); if (ret) goto out; ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len); @@ -2015,6 +2034,18 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, /* report all available features and ioctls to userland */ uffdio_api.features = UFFD_API_FEATURES; + /* Base-page anonymous and shared shmem operations support alternative leaves. */ + if (mm_page_size(ctx->mm) != PAGE_SIZE) + uffdio_api.features &= UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_MISSING_SHMEM | UFFD_FEATURE_MINOR_SHMEM | + UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_EVENT_REMAP | + UFFD_FEATURE_EVENT_REMOVE | UFFD_FEATURE_EVENT_UNMAP | + UFFD_FEATURE_SIGBUS | UFFD_FEATURE_THREAD_ID | UFFD_FEATURE_EXACT_ADDRESS | + UFFD_FEATURE_WP_UNPOPULATED | UFFD_FEATURE_WP_ASYNC | UFFD_FEATURE_POISON; + + if (mm_page_size(ctx->mm) < PAGE_SIZE) + uffdio_api.features |= UFFD_FEATURE_MOVE; + #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR uffdio_api.features &= ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); @@ -2062,6 +2093,10 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd, if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx)) return -EINVAL; + /* Larger-than-native MOVE still requires contiguous-owner support. */ + if (cmd == UFFDIO_MOVE && mm_page_size(ctx->mm) > PAGE_SIZE) + return -EOPNOTSUPP; + switch(cmd) { case UFFDIO_API: ret = userfaultfd_api(ctx, arg); diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h index 57137d3ac1592..619fd3ed40634 100644 --- a/include/asm-generic/pgalloc.h +++ b/include/asm-generic/pgalloc.h @@ -81,7 +81,7 @@ static inline pgtable_t __pte_alloc_one_noprof(struct mm_struct *mm, gfp_t gfp) return NULL; } - return ptdesc_page(ptdesc); + return pgtable_from_page(ptdesc_page(ptdesc)); } #define __pte_alloc_one(...) alloc_hooks(__pte_alloc_one_noprof(__VA_ARGS__)) @@ -111,17 +111,18 @@ static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm) * @mm: the mm_struct of the current context * @pte_page: the `struct page` referencing the ptdesc */ -static inline void pte_free(struct mm_struct *mm, struct page *pte_page) +#ifndef __HAVE_ARCH_PTE_FREE +static inline void pte_free(struct mm_struct *mm, pgtable_t pte_page) { - struct ptdesc *ptdesc = page_ptdesc(pte_page); + struct ptdesc *ptdesc = page_ptdesc(pgtable_to_page(pte_page)); pagetable_dtor_free(ptdesc); } +#endif #if CONFIG_PGTABLE_LEVELS > 2 -#ifndef __HAVE_ARCH_PMD_ALLOC_ONE /** * pmd_alloc_one - allocate memory for a PMD-level page table * @mm: the mm_struct of the current context @@ -133,7 +134,7 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page) * * Return: pointer to the allocated memory or %NULL on error */ -static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +static inline pmd_t *__pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) { struct ptdesc *ptdesc; gfp_t gfp = GFP_PGTABLE_USER; @@ -153,6 +154,12 @@ static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long ad return ptdesc_address(ptdesc); } + +#ifndef __HAVE_ARCH_PMD_ALLOC_ONE +static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + return __pmd_alloc_one_noprof(mm, addr); +} #define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__)) #endif diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index bdcc2778ac64f..900bee13b40cb 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -222,6 +222,15 @@ static inline void __tlb_remove_table(void *table) } #endif +/* Architectures may pass opaque table tokens instead of descriptors. */ +#ifndef tlb_table_is_ptdesc +static inline bool tlb_table_is_ptdesc(void *table) +{ + (void)table; + return true; +} +#endif + extern void tlb_remove_table(struct mmu_gather *tlb, void *table); #else /* !CONFIG_MMU_GATHER_TABLE_FREE */ @@ -295,6 +304,12 @@ extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, in bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page, unsigned int nr_pages, bool delay_rmap); +#ifdef CONFIG_MM_SUBPAGE +struct mm_subpage; +/* Consumes one slot reference; true requires flushing the full batch. */ +bool __tlb_remove_subpage(struct mmu_gather *tlb, struct mm_subpage *subpage); +#endif + #ifdef CONFIG_SMP /* * This both sets 'delayed_rmap', and returns true. It would be an inline @@ -543,15 +558,15 @@ static inline void tlb_change_page_size(struct mmu_gather *tlb, static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb) { if (tlb->cleared_ptes) - return PAGE_SHIFT; + return mm_page_shift(tlb->mm); if (tlb->cleared_pmds) - return PMD_SHIFT; + return pmd_shift_mm(tlb->mm); if (tlb->cleared_puds) - return PUD_SHIFT; + return pud_shift_mm(tlb->mm); if (tlb->cleared_p4ds) - return P4D_SHIFT; + return p4d_shift_mm(tlb->mm); - return PAGE_SHIFT; + return mm_page_shift(tlb->mm); } static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) @@ -658,7 +673,7 @@ static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, u */ #define tlb_remove_tlb_entry(tlb, ptep, address) \ do { \ - tlb_flush_pte_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pte_range(tlb, address, mm_page_size((tlb)->mm)); \ __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) @@ -672,13 +687,15 @@ static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, u static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, pte_t *ptep, unsigned int nr, unsigned long address) { - tlb_flush_pte_range(tlb, address, PAGE_SIZE * nr); + unsigned long page_size = mm_page_size(tlb->mm); + + tlb_flush_pte_range(tlb, address, page_size * nr); for (;;) { __tlb_remove_tlb_entry(tlb, ptep, address); if (--nr == 0) break; ptep++; - address += PAGE_SIZE; + address += page_size; } } @@ -739,13 +756,13 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, * architecture to do its own odd thing, not cause pain for others * http://lkml.kernel.org/r/CA+55aFzBggoXtNXQeng5d_mRoDnaMBE5Y+URs+PHR67nUpMtaw@mail.gmail.com * - * For now w.r.t page table cache, mark the range_size as PAGE_SIZE + * For now w.r.t page table cache, mark the range_size as one mm granule. */ #ifndef pte_free_tlb #define pte_free_tlb(tlb, ptep, address) \ do { \ - tlb_flush_pmd_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pmd_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pte_free_tlb(tlb, ptep, address); \ } while (0) @@ -754,7 +771,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef pmd_free_tlb #define pmd_free_tlb(tlb, pmdp, address) \ do { \ - tlb_flush_pud_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pud_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pmd_free_tlb(tlb, pmdp, address); \ } while (0) @@ -763,7 +780,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef pud_free_tlb #define pud_free_tlb(tlb, pudp, address) \ do { \ - tlb_flush_p4d_range(tlb, address, PAGE_SIZE); \ + tlb_flush_p4d_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pud_free_tlb(tlb, pudp, address); \ } while (0) @@ -772,7 +789,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef p4d_free_tlb #define p4d_free_tlb(tlb, pudp, address) \ do { \ - __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + __tlb_adjust_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __p4d_free_tlb(tlb, pudp, address); \ } while (0) diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h index 65abd5ab8836c..c242a5e9cc758 100644 --- a/include/linux/binfmts.h +++ b/include/linux/binfmts.h @@ -93,6 +93,8 @@ struct linux_binfmt { #ifdef CONFIG_COREDUMP int (*core_dump)(struct coredump_params *cprm); unsigned long min_coredump; /* minimal dump size */ + /* Optional per-mm replacement for formats with a variable page ABI. */ + unsigned long (*min_coredump_size)(void); #endif } __randomize_layout; @@ -121,6 +123,10 @@ static inline void insert_binfmt(struct linux_binfmt *fmt) extern void unregister_binfmt(struct linux_binfmt *); +#ifdef CONFIG_MM_SUBPAGE +int bprm_set_page_shift(struct linux_binprm *bprm, unsigned int page_shift); +#endif + extern int __must_check remove_arg_zero(struct linux_binprm *); extern int begin_new_exec(struct linux_binprm * bprm); extern void setup_new_exec(struct linux_binprm * bprm); diff --git a/include/linux/coredump.h b/include/linux/coredump.h index 68861da4cf7c2..5f85a12c7f998 100644 --- a/include/linux/coredump.h +++ b/include/linux/coredump.h @@ -12,10 +12,21 @@ struct core_vma_metadata { unsigned long start, end; vm_flags_t flags; unsigned long dump_size; - unsigned long pgoff; + unsigned long pgoff; /* File offset in the dumped mm's page units. */ struct file *file; }; +/* NT_FILE offsets are expressed in the dumped process's page units. */ +static inline unsigned long core_vma_pgoff(const struct vm_area_struct *vma) +{ + unsigned int shift = mm_page_shift(vma->vm_mm); + + if (shift > PAGE_SHIFT) + return vma->vm_pgoff >> (shift - PAGE_SHIFT); + return (vma->vm_pgoff << (PAGE_SHIFT - shift)) + + (vma_subpage_offset(vma) >> shift); +} + struct coredump_params { const kernel_siginfo_t *siginfo; struct file *file; diff --git a/include/linux/highmem.h b/include/linux/highmem.h index d7aac9de1c8a4..37812575512be 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -330,6 +330,21 @@ struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, } #endif +#ifndef vma_alloc_zeroed_movable_folio_order +static inline struct folio *vma_alloc_zeroed_movable_folio_order( + struct vm_area_struct *vma, unsigned long addr, unsigned int order) +{ + struct folio *folio; + + if (!order) + return vma_alloc_zeroed_movable_folio(vma, addr); + folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, order, vma, addr); + if (folio && user_alloc_needs_zeroing()) + clear_user_highpages(&folio->page, addr, folio_nr_pages(folio)); + return folio; +} +#endif + static inline void clear_highpage(struct page *page) { void *kaddr = kmap_local_page(page); diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 2949e5acff351..1f7f729b87868 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -3,6 +3,7 @@ #define _LINUX_HUGE_MM_H #include +#include #include /* only for vma_is_dax() */ #include @@ -331,6 +332,9 @@ struct thpsize { static inline bool vma_thp_disabled(struct vm_area_struct *vma, vm_flags_t vm_flags, bool forced_collapse) { + /* Native THP geometry and rmap cannot describe alternative user leaves. */ + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return true; /* Are THPs disabled for this VMA? */ if (vm_flags & VM_NOHUGEPAGE) return true; diff --git a/include/linux/khugepaged.h b/include/linux/khugepaged.h index d7a9053ff4fed..1510fdb048010 100644 --- a/include/linux/khugepaged.h +++ b/include/linux/khugepaged.h @@ -8,6 +8,8 @@ extern unsigned int khugepaged_max_ptes_none __read_mostly; #ifdef CONFIG_TRANSPARENT_HUGEPAGE extern struct attribute_group khugepaged_attr_group; +int collapse_file_user_page(struct mm_struct *mm, struct file *file, pgoff_t start); + extern int khugepaged_init(void); extern void khugepaged_destroy(void); extern int start_stop_khugepaged(void); @@ -32,6 +34,12 @@ static inline void khugepaged_exit(struct mm_struct *mm) __khugepaged_exit(mm); } #else /* CONFIG_TRANSPARENT_HUGEPAGE */ +static inline int collapse_file_user_page(struct mm_struct *mm, struct file *file, + pgoff_t start) +{ + return -EOPNOTSUPP; +} + static inline void khugepaged_fork(struct mm_struct *mm, struct mm_struct *oldmm) { } diff --git a/include/linux/mm.h b/include/linux/mm.h index f4481c338b36c..6c01377914fe8 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -750,6 +751,9 @@ struct vm_fault { }; struct page *cow_page; /* Page handler may use for COW fault */ +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *cow_subpage; /* Owns cow_page's selected fragment */ +#endif struct page *page; /* ->fault handlers should return a * page here, unless VM_FAULT_NOPAGE * is set (which is also implied by @@ -1615,7 +1619,7 @@ static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) { if (unlikely(vma->vm_ops && vma->vm_ops->pagesize)) return vma->vm_ops->pagesize(vma); - return PAGE_SIZE; + return mm_page_size(vma->vm_mm); } unsigned long vma_mmu_pagesize(struct vm_area_struct *vma); @@ -2010,6 +2014,10 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa void set_pte_range(struct vm_fault *vmf, struct folio *folio, struct page *page, unsigned int nr, unsigned long addr); +#ifdef CONFIG_MM_SUBPAGE +vm_fault_t map_file_user_pages(struct vm_fault *vmf, struct folio *folio, + pgoff_t first, pgoff_t last, bool fault); +#endif vm_fault_t finish_fault(struct vm_fault *vmf); #endif @@ -3143,10 +3151,13 @@ struct follow_pfnmap_args { * @address: the virtual address to walk * @write: if true, fail with -EFAULT unless the mapping is * writable + * @allow_subpage: caller handles exact byte offsets and user-leaf bounds; + * otherwise a non-native user granule is rejected */ struct vm_area_struct *vma; unsigned long address; bool write; + bool allow_subpage; /** * Internals: * @@ -3157,13 +3168,15 @@ struct follow_pfnmap_args { /** * Outputs: * - * @pfn: the PFN of the address - * @addr_mask: address mask covering pfn + * @pfn: native PFN containing the requested physical byte + * @offset: byte offset of that byte within the native physical page + * @addr_mask: virtual address mask of the leaf mapping * @pgprot: the pgprot_t of the mapping * @writable: whether the mapping is writable * @special: whether the mapping is a special mapping (real PFN maps) */ unsigned long pfn; + unsigned int offset; unsigned long addr_mask; pgprot_t pgprot; bool writable; @@ -3215,6 +3228,13 @@ static inline void unmap_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen, int even_cows) { } #endif +#ifdef CONFIG_MM_SUBPAGE +void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize); +void unmap_mapping_subpage_hole(struct address_space *mapping, loff_t start, u64 end); +#else +static inline void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize) { } +#endif + static inline void unmap_shared_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen) { @@ -3443,7 +3463,8 @@ static inline void reset_mm_hiwater_rss(struct mm_struct *mm) static inline void setmax_mm_hiwater_rss(unsigned long *maxrss, struct mm_struct *mm) { - unsigned long hiwater_rss = get_mm_hiwater_rss(mm); + /* signal->maxrss/cmaxrss survive exec and child reaping: store kB. */ + unsigned long hiwater_rss = mm_pages_to_kb(mm, get_mm_hiwater_rss(mm)); if (*maxrss < hiwater_rss) *maxrss = hiwater_rss; @@ -3514,14 +3535,14 @@ static inline void mm_inc_nr_puds(struct mm_struct *mm) { if (mm_pud_folded(mm)) return; - atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); + atomic_long_add(pud_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_puds(struct mm_struct *mm) { if (mm_pud_folded(mm)) return; - atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); + atomic_long_sub(pud_table_bytes_mm(mm), &mm->pgtables_bytes); } #endif @@ -3542,14 +3563,14 @@ static inline void mm_inc_nr_pmds(struct mm_struct *mm) { if (mm_pmd_folded(mm)) return; - atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); + atomic_long_add(pmd_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_pmds(struct mm_struct *mm) { if (mm_pmd_folded(mm)) return; - atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); + atomic_long_sub(pmd_table_bytes_mm(mm), &mm->pgtables_bytes); } #endif @@ -3566,12 +3587,12 @@ static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) static inline void mm_inc_nr_ptes(struct mm_struct *mm) { - atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); + atomic_long_add(pte_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_ptes(struct mm_struct *mm) { - atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); + atomic_long_sub(pte_table_bytes_mm(mm), &mm->pgtables_bytes); } #else @@ -3594,32 +3615,40 @@ static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) { return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ? - NULL : p4d_offset(pgd, address); + NULL : p4d_offset_mm(mm, pgd, address); } static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) { - return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? - NULL : pud_offset(p4d, address); + return (unlikely(p4d_none_mm(mm, *p4d)) && __pud_alloc(mm, p4d, address)) ? + NULL : pud_offset_mm(mm, p4d, address); } static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) { return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? - NULL: pmd_offset(pud, address); + NULL : pmd_offset_mm(mm, pud, address); } #endif /* CONFIG_MMU */ enum pt_flags { PT_kernel = PG_referenced, PT_reserved = PG_reserved, + PT_fragmented = PG_private, + PT_frag_defer = PG_active, /* High bits are used for zone/node/section */ }; static inline struct ptdesc *virt_to_ptdesc(const void *x) { - return page_ptdesc(virt_to_page(x)); + struct page *page = virt_to_page(x); + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* One hardware table may span multiple native allocator pages. */ + page = compound_head(page); +#endif + return page_ptdesc(page); } /** @@ -3836,12 +3865,25 @@ static inline bool pagetable_pte_ctor(struct mm_struct *mm, } pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, pmd_t *pmdvalp); +#else +#define __pte_offset_map_mm(mm, pmd, addr, pmdvalp) \ + __pte_offset_map(pmd, addr, pmdvalp) +#endif static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr) { return __pte_offset_map(pmd, addr, NULL); } +static inline pte_t *pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr) +{ + return __pte_offset_map_mm(mm, pmd, addr, NULL); +} + pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp); @@ -3859,7 +3901,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, #define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd)) #define pte_alloc_map(mm, pmd, address) \ - (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address)) + (pte_alloc(mm, pmd) ? NULL : pte_offset_map_mm(mm, pmd, address)) #define pte_alloc_map_lock(mm, pmd, address, ptlp) \ (pte_alloc(mm, pmd) ? \ @@ -3874,7 +3916,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, static inline struct page *pmd_pgtable_page(pmd_t *pmd) { unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); - return virt_to_page((void *)((unsigned long) pmd & mask)); + return ptdesc_page(virt_to_ptdesc((void *)((unsigned long)pmd & mask))); } static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd) @@ -4160,6 +4202,12 @@ extern unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf); +#ifdef CONFIG_MM_SUBPAGE +unsigned long do_mmap_offset(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, struct vm_page_offset pos, unsigned long *populate, + struct list_head *uf); +#endif extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf, bool unlock); @@ -4288,7 +4336,11 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) static inline unsigned long vma_last_pgoff(struct vm_area_struct *vma) { +#ifdef CONFIG_MM_SUBPAGE + return vma_page_offset_at(vma, vma->vm_end - 1).index; +#else return vma->vm_pgoff + vma_pages(vma) - 1; +#endif } static inline unsigned long vma_desc_size(const struct vm_area_desc *desc) @@ -4541,6 +4593,9 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t pgprot); int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); +int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long nr_pages, + unsigned long offset, unsigned long length); int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num); int map_kernel_pages_prepare(struct vm_area_desc *desc); @@ -4552,6 +4607,9 @@ int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, unsigned long num); vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, bool write); +vm_fault_t vmf_insert_pfn_prot_mkwrite_offset(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, unsigned int offset, + pgprot_t pgprot, bool mkwrite); vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot, bool mkwrite); vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, @@ -4648,6 +4706,18 @@ static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma, return VM_FAULT_NOPAGE; } +/* Insert one user granule from an explicit offset within a native page. */ +#ifdef CONFIG_MM_SUBPAGE +vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, unsigned long addr, + struct page *page, unsigned int offset); +#else +static inline vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, + unsigned long addr, struct page *page, unsigned int offset) +{ + return offset ? VM_FAULT_SIGBUS : vmf_insert_page(vma, addr, page); +} +#endif + #ifndef io_remap_pfn_range_pfn static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, unsigned long size) diff --git a/include/linux/mm_granule.h b/include/linux/mm_granule.h new file mode 100644 index 0000000000000..6e73216edcf96 --- /dev/null +++ b/include/linux/mm_granule.h @@ -0,0 +1,245 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MM_GRANULE_H +#define _LINUX_MM_GRANULE_H + +#include +#include +#include + +/* + * User virtual mapping granule. Never use these for physical PFNs, struct + * page/folio sizes, page-cache indices, or physical memory accounting. + * The granule is immutable once an mm has mappings or is visible to tasks. + */ +#ifndef arch_mm_page_shift +static inline unsigned int arch_mm_page_shift(const struct mm_struct *mm) +{ + return PAGE_SHIFT; +} +#endif + +static inline unsigned int mm_page_shift(const struct mm_struct *mm) +{ + return arch_mm_page_shift(mm); +} + +static inline unsigned long mm_page_size(const struct mm_struct *mm) +{ + return 1UL << mm_page_shift(mm); +} + +/* A byte-range reference must fit both the user leaf and a native page. */ +static inline unsigned long mm_user_fragment_size(const struct mm_struct *mm) +{ + return mm_page_shift(mm) < PAGE_SHIFT ? mm_page_size(mm) : PAGE_SIZE; +} + +/* Native physical pages covered by one base user PTE, including fragments. */ +static inline unsigned long mm_pte_native_pages(const struct mm_struct *mm) +{ + unsigned long pages = mm_page_size(mm) >> PAGE_SHIFT; + + return pages ?: 1; +} + +static inline unsigned long mm_page_mask(const struct mm_struct *mm) +{ + return ~(mm_page_size(mm) - 1); +} + +/* + * ASLR tunables count native-page bits. Retain their byte window when a + * process uses another granule, including the extra low user-page bits. + * native_mask must be a contiguous low-bit mask whose byte span fits ulong. + */ +static inline unsigned long mm_aslr_mask(const struct mm_struct *mm, + unsigned long native_mask) +{ + unsigned int shift = mm_page_shift(mm); + + if (shift < PAGE_SHIFT) { + unsigned int extra = PAGE_SHIFT - shift; + + return (native_mask << extra) | ((1UL << extra) - 1); + } + return native_mask >> (shift - PAGE_SHIFT); +} + +/* Virtual-range accounting. Physical pages and cache indices stay native. */ +static inline unsigned long vma_user_pages(const struct vm_area_struct *vma) +{ + return (vma->vm_end - vma->vm_start) >> mm_page_shift(vma->vm_mm); +} + +static inline unsigned long mm_page_align(const struct mm_struct *mm, unsigned long bytes) +{ + return (bytes + mm_page_size(mm) - 1) & mm_page_mask(mm); +} + +/* + * RSS and virtual counters use the target mm's user granule. Consumers + * comparing different mms or physical memory must convert explicitly. + */ +static inline unsigned long mm_pages_to_bytes(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << mm_page_shift(mm); +} + +static inline unsigned long mm_pages_to_kb(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << (mm_page_shift(mm) - 10); +} + +/* Round up only when reducing precision; do not hide a partial user page. */ +static inline unsigned long mm_pages_to_shift(const struct mm_struct *mm, + unsigned long pages, unsigned int shift) +{ + unsigned int source = mm_page_shift(mm); + + if (source >= shift) + return pages << (source - shift); + return (pages >> (shift - source)) + + !!(pages & ((1UL << (shift - source)) - 1)); +} + +static inline unsigned long mm_pages_to_native(const struct mm_struct *mm, + unsigned long pages) +{ + return mm_pages_to_shift(mm, pages, PAGE_SHIFT); +} + +/* A page-count proc ABI uses the reader's granule, including remote reads. */ +static inline unsigned long mm_pages_for_reader(const struct mm_struct *mm, + const struct mm_struct *reader, unsigned long pages) +{ + return mm_pages_to_shift(mm, pages, + reader ? mm_page_shift(reader) : PAGE_SHIFT); +} + +/* + * Time integrals need one unit across exec. Locked VM also uses this unit: + * it must represent both logical mappings and native physical charges exactly, + * including a single native-page pin in a larger-granule userspace mm. + */ +#ifdef CONFIG_MM_SUBPAGE +#define MM_ACCOUNT_SHIFT 12 +#else +#define MM_ACCOUNT_SHIFT PAGE_SHIFT +#endif + +static inline unsigned long mm_pages_to_account(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << (mm_page_shift(mm) - MM_ACCOUNT_SHIFT); +} + +static inline unsigned long mm_locked_vm_bytes(const struct mm_struct *mm) +{ + return READ_ONCE(mm->locked_vm) << MM_ACCOUNT_SHIFT; +} + +/* Physical-page limits must include any partial native page of locked VM. */ +static inline unsigned long mm_locked_vm_native_pages(const struct mm_struct *mm) +{ + unsigned long units = READ_ONCE(mm->locked_vm); + unsigned int shift = PAGE_SHIFT - MM_ACCOUNT_SHIFT; + + return (units >> shift) + !!(units & ((1UL << shift) - 1)); +} + +/* File-cache indices remain native-page units even for a 4K VMA. */ +struct vm_page_offset { + pgoff_t index; + unsigned int offset; +}; + +static inline struct vm_page_offset vm_page_offset_add(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = pos.offset + (bytes & ~PAGE_MASK); + + pos.index += (bytes >> PAGE_SHIFT) + (low >> PAGE_SHIFT); + pos.offset = low & ~PAGE_MASK; + return pos; +} + +static inline struct vm_page_offset vm_page_offset_sub(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = bytes & ~PAGE_MASK; + + pos.index -= (bytes >> PAGE_SHIFT) + (low > pos.offset); + pos.offset = (pos.offset - low) & ~PAGE_MASK; + return pos; +} + +static inline bool vm_page_offset_equal(struct vm_page_offset a, struct vm_page_offset b) +{ + return a.index == b.index && a.offset == b.offset; +} + +static inline unsigned int vma_subpage_offset(const struct vm_area_struct *vma) +{ +#ifdef CONFIG_MM_SUBPAGE + return vma->vm_subpage_offset; +#else + return 0; +#endif +} + +/* address must be >= vm_start; vm_end computes the next contiguous offset. */ +static inline struct vm_page_offset vma_page_offset_at(const struct vm_area_struct *vma, + unsigned long address) +{ + unsigned long delta = address - vma->vm_start; + unsigned long low = (delta & ~PAGE_MASK) + vma_subpage_offset(vma); + + return (struct vm_page_offset) { + .index = vma->vm_pgoff + (delta >> PAGE_SHIFT) + (low >> PAGE_SHIFT), + .offset = low & ~PAGE_MASK, + }; +} + +/* The start byte of a faulting granule must precede EOF. */ +static inline bool vma_file_offset_valid(const struct vm_area_struct *vma, + unsigned long address, loff_t size) +{ + struct vm_page_offset pos = vma_page_offset_at(vma, address); + pgoff_t end = size >> PAGE_SHIFT; + + return pos.index < end || + (pos.index == end && pos.offset < (size & ~PAGE_MASK)); +} + +static inline void vma_set_page_offset(struct vm_area_struct *vma, struct vm_page_offset pos) +{ + vma->vm_pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = pos.offset; +#endif +} + +/* Inverse lookup of a single byte; an offset outside the VMA is not clamped. */ +static inline unsigned long vma_address_at_offset(const struct vm_area_struct *vma, + struct vm_page_offset pos) +{ + unsigned long len = vma->vm_end - vma->vm_start; + unsigned long index, delta, last; + unsigned int offset = vma_subpage_offset(vma); + + if (!len || pos.index < vma->vm_pgoff || pos.offset >= PAGE_SIZE) + return -EFAULT; + index = pos.index - vma->vm_pgoff; + last = ((len - 1) >> PAGE_SHIFT) + + ((((len - 1) & ~PAGE_MASK) + offset) >> PAGE_SHIFT); + if (index > last) + return -EFAULT; + delta = (index << PAGE_SHIFT) + pos.offset; + if (delta < offset || delta - offset >= len) + return -EFAULT; + return vma->vm_start + delta - offset; +} + +#endif diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h new file mode 100644 index 0000000000000..13344e3a92384 --- /dev/null +++ b/include/linux/mm_subpage.h @@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MM_SUBPAGE_H +#define _LINUX_MM_SUBPAGE_H + +#include +#include +#include + +#define MM_SUBPAGE_SHIFT 12 +#define MM_SUBPAGE_SIZE (1UL << MM_SUBPAGE_SHIFT) +#define MM_SUBPAGES_PER_PAGE (PAGE_SIZE / MM_SUBPAGE_SIZE) + +struct mm_subpage_mapcounts { + unsigned int slots[MM_SUBPAGES_PER_PAGE]; +}; + +struct folio; +struct page; +struct mm_struct; +struct mm_subpage; +struct mm_subpage_pool; +struct page_ext_operations; +struct vm_area_struct; + +extern struct page_ext_operations mm_subpage_ext_ops; + +#ifdef CONFIG_MM_SUBPAGE +/* Per-native-page shmem validity; caller holds the containing folio lock. */ +unsigned long mm_subpage_shmem_missing(struct page *page); +void mm_subpage_shmem_set_missing(struct page *page, unsigned long missing); +#endif + +/* + * A pool is one caller-defined allocation/policy/charge domain. The caller + * supplies fresh, unmapped native folios using its normal NUMA and memcg + * allocation path. No mm, VMA, rmap, swap or folio-private field is repurposed. + * Fault paths transfer slot references to PTEs through the rmap helpers below. + * + * Each caller must hold a pool reference through pool operations. close() + * stops allocation and releases unused backing; it does not consume that + * reference. Existing slot references outlive close() and the originating mm. + */ +struct mm_subpage_pool *mm_subpage_pool_create(gfp_t gfp); +struct mm_subpage_pool *mm_subpage_pool_create_granule(gfp_t gfp, unsigned int shift); +void mm_subpage_pool_get(struct mm_subpage_pool *pool); +void mm_subpage_pool_put(struct mm_subpage_pool *pool); +void mm_subpage_pool_close(struct mm_subpage_pool *pool); + +/* Success consumes the folio reference. On error the caller retains it. */ +int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + gfp_t gfp); +int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int offset, gfp_t gfp); +unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool); + +/* -EAGAIN requires more compatible backing; -ESHUTDOWN means pool closed. */ +struct mm_subpage *mm_subpage_alloc(struct mm_subpage_pool *pool); +struct mm_subpage *mm_subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source); +/* Exact physical byte offset within a native folio; no other slot is substituted. */ +struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned int offset); +struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset); + +/* + * Reattach a slot to an uptodate, locked order-0 swap-cache folio, without + * zeroing or copying its bytes. Caller holds a native folio reference and + * retains it. The returned slot owns an additional reference. -EAGAIN means + * ownership changed concurrently; -ENOMEM means metadata allocation failed. + */ +struct mm_subpage *mm_subpage_restore(struct folio *folio, unsigned int offset, gfp_t gfp); +struct mm_subpage *mm_subpage_restore_granule(struct folio *folio, unsigned int offset, + unsigned int shift, gfp_t gfp); + +/* PTE lock held, one file-backed alternative leaf per call. Native mms are no-ops. */ +#ifdef CONFIG_MM_SUBPAGE +void __mm_subpage_file_map_add(phys_addr_t phys, unsigned long size); +void __mm_subpage_file_map_del(phys_addr_t phys, unsigned long size); +static inline void mm_subpage_file_map_add(struct vm_area_struct *vma, phys_addr_t phys) +{ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + __mm_subpage_file_map_add(phys, mm_page_size(vma->vm_mm)); +} +static inline void mm_subpage_file_map_del(struct vm_area_struct *vma, phys_addr_t phys) +{ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + __mm_subpage_file_map_del(phys, mm_page_size(vma->vm_mm)); +} +/* Fill 4K coverage counts; return alternative leaf count. Statistics only. */ +unsigned int mm_subpage_file_mapcounts(struct page *page, + struct mm_subpage_mapcounts *counts); +#else +static inline void mm_subpage_file_map_add(struct vm_area_struct *vma, phys_addr_t phys) { } +static inline void mm_subpage_file_map_del(struct vm_area_struct *vma, phys_addr_t phys) { } +#endif + +/* Both folios locked; preallocate destination metadata before removing PTEs. */ +#ifdef CONFIG_MM_SUBPAGE +int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst); +bool mm_subpage_migrating(struct folio *folio); +void mm_subpage_migrate_save_lazyfree(struct mm_subpage *subpage); +/* Infallible metadata reuse after prepare; caller still holds the folio lock. */ +struct mm_subpage *mm_subpage_migrate_restore(struct folio *folio, unsigned int offset); +void mm_subpage_migrate_finish(struct folio *folio); +#else +static inline int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) +{ + return 0; +} +static inline bool mm_subpage_migrating(struct folio *folio) +{ + return false; +} +static inline void mm_subpage_migrate_finish(struct folio *folio) { } +#endif + +/* + * get() requires a live reference. A mapping reference must not be put until + * its PTE is removed and TLB invalidation completes. Every pin/alias holds + * its own reference: the slot becomes reusable only after the final put. + * Every live slot reference also holds one reference to the native folio. + * copy() requires the caller to stabilise the source contents (as for COW). + * A failed data copy returns -EHWPOISON without publishing the destination. + */ +void mm_subpage_get(struct mm_subpage *subpage); +void mm_subpage_put(struct mm_subpage *subpage); +/* + * Acquire the live identity containing a 4K-aligned physical address, or NULL. + * Interior quarters of a larger identity return that same identity. + * The caller must stabilise the physical allocation, e.g. with a live PTE + * under its lock or a folio reference. A lockless walker additionally needs + * GUP's TLB/free ordering; RCU protects metadata, not arbitrary physical PFNs. + * This is not a mapping identity check: a lockless PTE reader must revalidate + * its PTE after acquisition, just as for speculative folio references. + */ +struct mm_subpage *mm_subpage_get_from_phys(phys_addr_t phys); +/* + * Optional shared sets for asynchronous consumers of native byte spans. + * add() requires the source slot identities to remain live for the call. + * The pointer being modified is exclusively owned by the caller; get() shares + * a read-only set, and mutation uses copy-on-write. Duplicate ranges add no + * references. prune() requires a stable, side-effect-free membership predicate. + * Allocation failure leaves the original set intact. put() accepts NULL. + */ +struct mm_subpage_refs; +#ifdef CONFIG_MM_SUBPAGE +struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs); +void mm_subpage_refs_put(struct mm_subpage_refs *refs); +int mm_subpage_refs_add(struct mm_subpage_refs **refs, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp); +int mm_subpage_refs_prune(struct mm_subpage_refs **refs, + bool (*keep)(const struct mm_subpage *, void *), + void *context, gfp_t gfp); +#else +static inline struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs) +{ + return refs; +} +static inline void mm_subpage_refs_put(struct mm_subpage_refs *refs) { } +static inline int mm_subpage_refs_add(struct mm_subpage_refs **refs, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp) +{ + return 0; +} +static inline int mm_subpage_refs_prune(struct mm_subpage_refs **refs, + bool (*keep)(const struct mm_subpage *, void *), void *context, gfp_t gfp) +{ + return 0; +} +#endif + +/* + * Anonymous rmap attachment: caller holds the folio lock and PTE lock, has + * prepared vma->anon_vma, and transfers an existing slot reference to its PTE. + * A folio may cover only one native linear index and anon_vma root; its slot + * offset must match the VMA's linear byte offset. Removal needs the PTE lock. + * These helpers do not install PTEs, update RSS, enqueue on LRU, or drop refs. + */ +int mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address); +int mm_subpage_add_new_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address); +void mm_subpage_remove_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma); +/* Both PTE locks and source write_protect_seq held; both mappings become R/O. */ +int mm_subpage_dup_anon_rmap(struct mm_subpage *subpage, + struct vm_area_struct *dst, struct vm_area_struct *src, unsigned long address); +/* + * Per-slot anonymous exclusivity. Setting requires the folio and PTE locks, + * and proof of a fresh private slot or an exclusive migration/swap entry. + * Sharing requires the PTE lock and fails while a typed pin exists. + */ +void mm_subpage_set_exclusive(struct mm_subpage *subpage); +bool mm_subpage_is_exclusive(const struct mm_subpage *subpage); +/* + * Folio and PTE locked; caller proves one remaining swap PTE for this slot. + * Rmap has been attached, but its PTE is not published yet. Reject old GET + * references or present aliases before recovering lost exclusive ownership. + */ +bool mm_subpage_try_reuse_swap(struct mm_subpage *subpage); +/* Present PTE plus caller lookup ref; caller proves no swapped aliases. */ +bool mm_subpage_try_reuse_anon(struct mm_subpage *subpage); +/* PTE locked, caller holds one GET; retire the PTE/TLB before dropping its ref. */ +bool mm_subpage_can_zap(struct mm_subpage *subpage); +bool mm_subpage_try_share(struct mm_subpage *subpage); +/* Folio/PTE locked, caller owns a lookup reference. */ +bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage); +bool mm_subpage_all_lazyfree(struct mm_subpage *subpage); +bool mm_subpage_discardable(struct mm_subpage *subpage); +bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage); +void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage); +/* + * Caller holds a live slot reference and its PTE lock. Success takes an + * additional slot reference plus a native FOLL_PIN reference. The pin keeps + * the exact fragment occupied after unmap; release only with subpage_unpin. + * This is not a GUP walker and does not validate user addresses/permissions. + */ +int mm_subpage_pin(struct mm_subpage *subpage); +void mm_subpage_unpin(struct mm_subpage *subpage); +unsigned int mm_subpage_pincount(const struct mm_subpage *subpage); +#ifdef CONFIG_MM_SUBPAGE +/* A referenced/pinned folio range; never infer offset from the user VA. */ +struct user_page_fragment { + struct folio *folio; + struct mm_subpage *subpage; + unsigned long offset; + unsigned int length; + bool pinned; +}; + +/* + * Takes/releases mmap_lock internally. Returns the number of records filled, + * or -errno if none. A positive partial result is valid; sum record lengths + * to advance the user address. Each record stops at the target mm's granule + * or a native physical-page boundary, whichever comes first. + * Caller owns the target mm lifetime. Release either kind of record with + * release_user_fragments(); copying a record does not acquire a reference. + * FOLL_LONGTERM pins migrate unpinnable backing before returning success. + * Device/PFN mappings are not accepted by this interface. + */ +long pin_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity); +long get_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity); +/* + * Single-record GET with a stable VMA. Caller holds mmap_lock throughout; + * this function never releases it. Returns 1 or -errno (0 for zero length). + * FOLL_NOWAIT is not supported. The returned VMA is valid until unlock. + */ +int get_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma); +/* Single-record PIN under the caller's mmap_lock, retained throughout. + * Same stable-VMA/partial-record contract as GET above. FOLL_LONGTERM and + * FOLL_NOWAIT are rejected: long-term migration can require dropping locks. + */ +int pin_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma); +/* Current mm, one aligned fragment; sparse zero pages return a negative errno. */ +int get_dump_fragment(unsigned long addr, struct user_page_fragment *fragment); +/* Dirty a retained reference without releasing its allocation identity. */ +void mark_user_fragment_dirty(struct user_page_fragment *fragment); +void release_user_fragments(struct user_page_fragment *fragments, + unsigned long count, bool dirty); +#endif + +int mm_subpage_mapcount(const struct mm_subpage *subpage); +/* Borrowed folio: a raw folio reference alone does not keep a slot occupied. */ +struct folio *mm_subpage_folio(const struct mm_subpage *subpage); +unsigned int mm_subpage_shift(const struct mm_subpage *subpage); +unsigned int mm_subpage_size(const struct mm_subpage *subpage); +unsigned int mm_subpage_offset(const struct mm_subpage *subpage); +phys_addr_t mm_subpage_phys(const struct mm_subpage *subpage); + +#endif diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index a308e2c23b829..7f201dda7a13b 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -558,7 +558,8 @@ FOLIO_MATCH(compound_info, _head_3); * @__page_mapping: Aliases with page->mapping. Unused for page tables. * @pt_index: Used for s390 gmap. * @pt_mm: Used for x86 pgds. - * @pt_frag_refcount: For fragmented page table tracking. Powerpc only. + * @pt_frag_refcount: For fragmented page table tracking (powerpc, arm64). + * @pt_frag_shift: Immutable arm64 fragment granule, valid through RCU release. * @pt_share_count: Used for HugeTLB PMD page table share count. * @_pt_pad_2: Padding to ensure proper alignment. * @ptl: Lock for the page table. @@ -585,7 +586,15 @@ struct ptdesc { union { pgoff_t pt_index; struct mm_struct *pt_mm; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* arm64 is 64-bit: use the spare half of this existing word. */ + struct { + atomic_t pt_frag_refcount; + u8 pt_frag_shift; + }; +#else atomic_t pt_frag_refcount; +#endif #ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING atomic_t pt_share_count; #endif @@ -905,6 +914,9 @@ struct vm_area_desc { /* Mutable fields. Populated with initial state. */ pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif struct file *vm_file; vma_flags_t vma_flags; pgprot_t page_prot; @@ -992,6 +1004,9 @@ struct vm_area_struct { /* Information about our backing store: */ unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE units */ +#ifdef CONFIG_MM_SUBPAGE + unsigned int vm_subpage_offset; /* Byte offset within vm_pgoff. */ +#endif struct file * vm_file; /* File we map to (can be NULL). */ void * vm_private_data; /* was vm_pte (shared mem) */ @@ -1281,12 +1296,16 @@ struct mm_struct { unsigned int __percpu *futex_ref; #endif - unsigned long hiwater_rss; /* High-watermark of RSS usage */ - unsigned long hiwater_vm; /* High-water virtual memory usage */ + unsigned long hiwater_rss; /* Peak RSS in user granules */ + unsigned long hiwater_vm; /* Peak VM in user granules */ - unsigned long total_vm; /* Total pages mapped */ - unsigned long locked_vm; /* Pages that have PG_mlocked set */ - atomic64_t pinned_vm; /* Refcount permanently increased */ + unsigned long total_vm; /* Mapped user granules */ +#ifdef CONFIG_MM_SUBPAGE + /* VM_ACCOUNT granules; global commitment rounds this sum once. */ + unsigned long committed_user_pages; +#endif + unsigned long locked_vm; /* Locked bytes >> MM_ACCOUNT_SHIFT */ + atomic64_t pinned_vm; /* Pinned native physical pages */ unsigned long data_vm; /* VM_WRITE & ~VM_SHARED & ~VM_STACK */ unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE & ~VM_STACK */ unsigned long stack_vm; /* VM_STACK */ diff --git a/include/linux/mman.h b/include/linux/mman.h index 389521594c69c..7b80c19f6aa5d 100644 --- a/include/linux/mman.h +++ b/include/linux/mman.h @@ -83,6 +83,17 @@ static inline void vm_unacct_memory(long pages) vm_acct_memory(-pages); } +/* VMA reservations use user granules; shmem/physical reservations use pages. */ +#ifdef CONFIG_MM_SUBPAGE +int mm_account_memory(struct mm_struct *mm, struct mm_struct *policy_mm, long pages); +void mm_acct_memory(struct mm_struct *mm, long pages); +void mm_unacct_memory(struct mm_struct *mm, unsigned long pages); +#else +#define mm_account_memory(mm, policy_mm, pages) security_vm_enough_memory_mm(policy_mm, pages) +#define mm_acct_memory(mm, pages) vm_acct_memory(pages) +#define mm_unacct_memory(mm, pages) vm_unacct_memory(pages) +#endif + /* * Allow architectures to handle additional protection and flag bits. The * overriding macros must be defined in the arch-specific asm/mman.h file. diff --git a/include/linux/mmu_context.h b/include/linux/mmu_context.h index ed3dd0f3fe194..a4260bc898205 100644 --- a/include/linux/mmu_context.h +++ b/include/linux/mmu_context.h @@ -5,6 +5,11 @@ #include #include +/* Select immutable exec geometry before allocating any page tables. */ +#ifndef arch_mm_init_exec +static inline void arch_mm_init_exec(struct mm_struct *mm, unsigned int page_shift) { } +#endif + /* Architectures that care about IRQ state in switch_mm can override this. */ #ifndef switch_mm_irqs_off # define switch_mm_irqs_off switch_mm diff --git a/include/linux/page_table_check.h b/include/linux/page_table_check.h index 12268a32e8be1..3ee1096fb0e53 100644 --- a/include/linux/page_table_check.h +++ b/include/linux/page_table_check.h @@ -12,6 +12,10 @@ extern struct static_key_true page_table_check_disabled; extern struct page_ext_operations page_table_check_ops; +#if IS_ENABLED(CONFIG_KUNIT) +int page_table_check_get_counts(struct page *page, unsigned int offset, + int *anon, int *file); +#endif void __page_table_check_zero(struct page *page, unsigned int order); void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 31a848485ad9d..9cde417795f00 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1068,10 +1068,7 @@ static inline pgoff_t folio_pgoff(const struct folio *folio) static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, const unsigned long address) { - pgoff_t pgoff; - pgoff = (address - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; - return pgoff; + return vma_page_offset_at(vma, address).index; } struct wait_page_key { diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index cdd68ed3ae1a9..f5ef44f25c52b 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -12,11 +12,18 @@ #ifdef CONFIG_MMU #include +#include #include #include #include #include +/* pgtable_t is an opaque allocation token, not necessarily struct page *. */ +#ifndef pgtable_to_page +#define pgtable_to_page(table) (table) +#define pgtable_from_page(page) (page) +#endif + #if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS #error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED @@ -155,6 +162,59 @@ static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address) */ #define pgd_offset_k(address) pgd_offset(&init_mm, (address)) +/* Explicit target-mm accessors; legacy accessors keep native geometry. */ +#ifndef pte_phys_mm +#define pte_phys_mm(mm, pte) PFN_PHYS(pte_pfn(pte)) +#endif +#ifndef phys_pte_mm +#define phys_pte_mm(mm, phys, prot) pfn_pte((phys) >> PAGE_SHIFT, prot) +#endif +#ifndef pte_pgprot_mm +#define pte_pgprot_mm(mm, pte) pte_pgprot(pte) +#endif +#ifndef pgd_offset_pgd_mm +#define pgd_offset_pgd_mm(mm, pgd, addr) pgd_offset_pgd(pgd, addr) +#endif +#ifndef p4d_offset_mm +#define p4d_offset_mm(mm, pgd, addr) p4d_offset(pgd, addr) +#endif +#ifndef pud_offset_mm +#define pud_offset_mm(mm, p4d, addr) pud_offset(p4d, addr) +#endif +#ifndef pmd_offset_mm +#define pmd_offset_mm(mm, pud, addr) pmd_offset(pud, addr) +#endif +#ifndef __pte_map_mm +#define __pte_map_mm(mm, pmd, addr) __pte_map(pmd, addr) +#endif +#ifndef pte_offset_kernel_mm +#define pte_offset_kernel_mm(mm, pmd, addr) pte_offset_kernel(pmd, addr) +#endif + +#ifndef pud_pgtable_mm +#define pud_pgtable_mm(mm, pud) pud_pgtable(pud) +#endif + +#ifndef pmd_size_mm +#define pmd_shift_mm(mm) PMD_SHIFT +#define pud_shift_mm(mm) PUD_SHIFT +#define p4d_shift_mm(mm) P4D_SHIFT +#define pmd_size_mm(mm) PMD_SIZE +#define pud_size_mm(mm) PUD_SIZE +#define p4d_size_mm(mm) P4D_SIZE +#define pgd_size_mm(mm) PGDIR_SIZE +#define pmd_mask_mm(mm) PMD_MASK +#define pud_mask_mm(mm) PUD_MASK +#define p4d_mask_mm(mm) P4D_MASK +#define pgd_mask_mm(mm) PGDIR_MASK +#endif + +#ifndef pte_table_bytes_mm +#define pte_table_bytes_mm(mm) (PTRS_PER_PTE * sizeof(pte_t)) +#define pmd_table_bytes_mm(mm) (PTRS_PER_PMD * sizeof(pmd_t)) +#define pud_table_bytes_mm(mm) (PTRS_PER_PUD * sizeof(pud_t)) +#endif + /* * In many cases it is known that a virtual address is mapped at PMD or PTE * level, so instead of traversing all the page table levels, we can get a @@ -388,6 +448,10 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) #define pte_next_pfn(pte) pte_advance_pfn(pte, 1) +#ifndef pte_advance_pfn_mm +#define pte_advance_pfn_mm(mm, pte, nr) pte_advance_pfn(pte, nr) +#endif + #ifndef set_ptes /** * set_ptes - Map consecutive pages to a contiguous range of addresses. @@ -1465,6 +1529,13 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) * and any p?d_bad entries - reporting the error before resetting to none. * Do the tests inline, but report and clear the bad entry in mm/memory.c. */ +#ifndef pgd_addr_end_mm +#define pgd_addr_end_mm(mm, addr, end) pgd_addr_end(addr, end) +#define p4d_addr_end_mm(mm, addr, end) p4d_addr_end(addr, end) +#define pud_addr_end_mm(mm, addr, end) pud_addr_end(addr, end) +#define pmd_addr_end_mm(mm, addr, end) pmd_addr_end(addr, end) +#endif + void pgd_clear_bad(pgd_t *); #ifndef __PAGETABLE_P4D_FOLDED @@ -1492,6 +1563,25 @@ static inline int pgd_none_or_clear_bad(pgd_t *pgd) return 0; } +#ifndef p4d_none_mm +#define p4d_none_mm(mm, p4d) p4d_none(p4d) +#define p4d_bad_mm(mm, p4d) p4d_bad(p4d) +#define p4d_present_mm(mm, p4d) p4d_present(p4d) +#define p4d_clear_mm(mm, p4d) p4d_clear(p4d) +#endif + +static inline int p4d_none_or_clear_bad_mm(struct mm_struct *mm, p4d_t *p4d) +{ + if (p4d_none_mm(mm, *p4d)) + return 1; + if (unlikely(p4d_bad_mm(mm, *p4d))) { + p4d_ERROR(*p4d); + p4d_clear_mm(mm, p4d); + return 1; + } + return 0; +} + static inline int p4d_none_or_clear_bad(p4d_t *p4d) { if (p4d_none(*p4d)) @@ -1979,11 +2069,15 @@ static inline int is_zero_pfn(unsigned long pfn) #define zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr)) #else +#ifndef ARCH_ZERO_PAGE_SIZE +#define ARCH_ZERO_PAGE_SIZE PAGE_SIZE +#endif + static inline int is_zero_pfn(unsigned long pfn) { extern unsigned long zero_page_pfn; - return pfn == zero_page_pfn; + return pfn - zero_page_pfn < (ARCH_ZERO_PAGE_SIZE >> PAGE_SHIFT); } static inline unsigned long zero_pfn(unsigned long addr) @@ -1993,7 +2087,7 @@ static inline unsigned long zero_pfn(unsigned long addr) return zero_page_pfn; } -extern uint8_t empty_zero_page[PAGE_SIZE]; +extern uint8_t empty_zero_page[ARCH_ZERO_PAGE_SIZE]; extern struct page *__zero_page; static inline struct page *_zero_page(unsigned long addr) diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 8dc0871e5f001..2bc2c986eb76e 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -40,6 +40,10 @@ struct anon_vma { * anon_vma if they are the last user on release */ atomic_t refcount; +#ifdef CONFIG_MM_SUBPAGE + /* Immutable base-page requirement shared by this anonymous lineage. */ + unsigned int min_folio_order; +#endif /* * Count of child anon_vmas. Equals to the count of all anon_vmas that @@ -398,29 +402,40 @@ static __always_inline void __folio_rmap_sanity_checks(const struct folio *folio } /* - * rmap interfaces called when adding or removing pte of page + * Single-PTE interfaces cover the complete native-page span of a user leaf. + * Multi-page interfaces below retain explicit native physical-page counts. */ void folio_move_anon_rmap(struct folio *, struct vm_area_struct *); void folio_add_anon_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *, unsigned long address, rmap_t flags); -#define folio_add_anon_rmap_pte(folio, page, vma, address, flags) \ - folio_add_anon_rmap_ptes(folio, page, 1, vma, address, flags) +static inline void folio_add_anon_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma, unsigned long address, rmap_t flags) +{ + folio_add_anon_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), + vma, address, flags); +} void folio_add_anon_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *, unsigned long address, rmap_t flags); void folio_add_new_anon_rmap(struct folio *, struct vm_area_struct *, unsigned long address, rmap_t flags); void folio_add_file_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *); -#define folio_add_file_rmap_pte(folio, page, vma) \ - folio_add_file_rmap_ptes(folio, page, 1, vma) +static inline void folio_add_file_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma) +{ + folio_add_file_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), vma); +} void folio_add_file_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *); void folio_add_file_rmap_pud(struct folio *, struct page *, struct vm_area_struct *); void folio_remove_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *); -#define folio_remove_rmap_pte(folio, page, vma) \ - folio_remove_rmap_ptes(folio, page, 1, vma) +static inline void folio_remove_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma) +{ + folio_remove_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), vma); +} void folio_remove_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *); void folio_remove_rmap_pud(struct folio *, struct page *, @@ -541,7 +556,8 @@ static inline void folio_dup_file_rmap_ptes(struct folio *folio, static __always_inline void folio_dup_file_rmap_pte(struct folio *folio, struct page *page, struct vm_area_struct *dst_vma) { - __folio_dup_file_rmap(folio, page, 1, dst_vma, PGTABLE_LEVEL_PTE); + __folio_dup_file_rmap(folio, page, mm_pte_native_pages(dst_vma->vm_mm), + dst_vma, PGTABLE_LEVEL_PTE); } /** @@ -665,8 +681,8 @@ static __always_inline int folio_try_dup_anon_rmap_pte(struct folio *folio, struct page *page, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) { - return __folio_try_dup_anon_rmap(folio, page, 1, dst_vma, src_vma, - PGTABLE_LEVEL_PTE); + return __folio_try_dup_anon_rmap(folio, page, mm_pte_native_pages(src_vma->vm_mm), + dst_vma, src_vma, PGTABLE_LEVEL_PTE); } /** @@ -708,13 +724,16 @@ static inline int folio_try_dup_anon_rmap_pmd(struct folio *folio, static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, struct page *page, int nr_pages, enum pgtable_level level) { + int i, exclusive_pages = level == PGTABLE_LEVEL_PTE ? nr_pages : 1; + VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); VM_WARN_ON_FOLIO(!PageAnonExclusive(page), folio); __folio_rmap_sanity_checks(folio, page, nr_pages, level); /* device private folios cannot get pinned via GUP. */ if (unlikely(folio_is_device_private(folio))) { - ClearPageAnonExclusive(page); + for (i = 0; i < exclusive_pages; i++) + ClearPageAnonExclusive(page + i); return 0; } @@ -766,7 +785,8 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, if (unlikely(folio_maybe_dma_pinned(folio))) return -EBUSY; - ClearPageAnonExclusive(page); + for (i = 0; i < exclusive_pages; i++) + ClearPageAnonExclusive(page + i); /* * This is conceptually a smp_wmb() paired with the smp_rmb() in @@ -782,7 +802,8 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, * mapped by a PTE possibly shared to prepare * for KSM or temporary unmapping * @folio: The folio to share a mapping of - * @page: The mapped exclusive page + * @page: The first native page covered by the exclusive user PTE + * @vma: The VMA defining the base user PTE granule * * The caller needs to hold the page table lock and has to have the page table * entries cleared/invalidated. @@ -799,9 +820,10 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, * -EBUSY otherwise. */ static inline int folio_try_share_anon_rmap_pte(struct folio *folio, - struct page *page) + struct page *page, struct vm_area_struct *vma) { - return __folio_try_share_anon_rmap(folio, page, 1, PGTABLE_LEVEL_PTE); + return __folio_try_share_anon_rmap(folio, page, mm_pte_native_pages(vma->vm_mm), + PGTABLE_LEVEL_PTE); } /** @@ -855,12 +877,23 @@ struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr, #define PVMW_SYNC (1 << 0) /* Look for migration entries rather than present PTEs */ #define PVMW_MIGRATION (1 << 1) +/* Match one present subpage PTE at address, including its physical offset. */ +#define PVMW_SUBPAGE (1 << 2) /* Result flags */ /* The page is mapped across page table boundary */ #define PVMW_PGTABLE_CROSSED (1 << 16) +/* One reverse-map callback, optionally restricted to one physical user leaf. */ +struct rmap_walk_range { + unsigned long address; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; + bool subpage; +#endif +}; + struct page_vma_mapped_walk { unsigned long pfn; unsigned long nr_pages; @@ -871,6 +904,10 @@ struct page_vma_mapped_walk { pte_t *pte; spinlock_t *ptl; unsigned int flags; +#ifdef CONFIG_MM_SUBPAGE + /* PVMW_SUBPAGE: byte offset within pfn; nr_pages must be one. */ + unsigned int subpage_offset; +#endif }; #define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \ @@ -883,6 +920,32 @@ struct page_vma_mapped_walk { .flags = _flags, \ } +static inline struct page_vma_mapped_walk folio_rmap_walk_init( + struct folio *folio, struct vm_area_struct *vma, + struct rmap_walk_range range, unsigned int flags) +{ + struct page_vma_mapped_walk walk = { + .pfn = folio_pfn(folio), + .nr_pages = folio_nr_pages(folio), + .pgoff = folio_pgoff(folio), + .vma = vma, + .address = range.address, + .flags = flags, + }; + +#ifdef CONFIG_MM_SUBPAGE + if (range.subpage) { + walk.flags |= PVMW_SUBPAGE; + walk.subpage_offset = range.subpage_offset; + } +#endif + return walk; +} + +#define DEFINE_FOLIO_RMAP_WALK(name, _folio, _vma, _range, _flags) \ + struct page_vma_mapped_walk name = \ + folio_rmap_walk_init(_folio, _vma, _range, _flags) + static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw) { /* HugeTLB pte is set to the relevant page table entry without pte_mapped. */ @@ -943,7 +1006,7 @@ void remove_migration_ptes(struct folio *src, struct folio *dst, * arg: passed to rmap_one() and invalid_vma() * try_lock: bail out if the rmap lock is contended * contended: indicate the rmap traversal bailed out due to lock contention - * rmap_one: executed on each vma where page is mapped + * rmap_one: executed for each mapped VMA range (possibly one user leaf) * done: for checking traversing termination condition * anon_lock: for getting anon_lock by optimized way rather than default * invalid_vma: for skipping uninterested vma @@ -957,7 +1020,7 @@ struct rmap_walk_control { * Otherwise, return true. */ bool (*rmap_one)(struct folio *folio, struct vm_area_struct *vma, - unsigned long addr, void *arg); + struct rmap_walk_range range, void *arg); int (*done)(struct folio *folio); struct anon_vma *(*anon_lock)(const struct folio *folio, struct rmap_walk_control *rwc); @@ -966,6 +1029,15 @@ struct rmap_walk_control { void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc); void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc); +#ifdef CONFIG_MM_SUBPAGE +unsigned int folio_anon_min_order(struct folio *folio); +#else +static inline unsigned int folio_anon_min_order(struct folio *folio) +{ + return 0; +} +#endif + struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio, struct rmap_walk_control *rwc); diff --git a/include/linux/sched.h b/include/linux/sched.h index eaf9751383596..31b894db80f11 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -961,6 +961,11 @@ struct task_struct { struct mm_struct *mm; struct mm_struct *active_mm; +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + /* Per-thread exec policy; fork inherits it, the live mm is immutable. */ + unsigned int exec_page_shift; + unsigned int default_exec_page_shift; +#endif int exit_state; int exit_code; diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 95d0040df5841..38d573026d7fb 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -14,6 +14,10 @@ * Routines for handling mm_structs */ extern struct mm_struct *mm_alloc(void); +extern struct mm_struct *mm_alloc_exec(void); +extern struct mm_struct *mm_alloc_exec_page_shift(unsigned int page_shift); +/* Return zero for native, an alternative shift, or a negative errno. */ +int arch_exec_page_size_shift(unsigned long size); /** * mmgrab() - Pin a &struct mm_struct. diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 584ae88b435e9..7af2aeb81e41d 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -194,7 +194,7 @@ struct signal_struct { unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw; unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt; unsigned long inblock, oublock, cinblock, coublock; - unsigned long maxrss, cmaxrss; + unsigned long maxrss, cmaxrss; /* kB, independent of each mm granule */ struct task_io_accounting ioac; /* diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 69b0177da1564..bf3a012954700 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -32,6 +32,8 @@ struct swap_iocb; * isn't directly visible to userspace. */ #define SHMEM_F_MAPPING_FROZEN BIT(2) +/* This inode has used sub-native shared-page validity. */ +#define SHMEM_F_UFFD_SUBPAGE BIT(3) struct shmem_inode_info { spinlock_t lock; @@ -52,6 +54,10 @@ struct shmem_inode_info { pgoff_t fallocend; /* highest fallocate endindex */ unsigned int fsflags; /* for FS_IOC_[SG]ETFLAGS */ atomic_t stop_eviction; /* hold when working on inode */ +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + /* Swapped missing 4K units, keyed by native cache index. */ + struct xarray uffd_missing; +#endif #ifdef CONFIG_TMPFS_QUOTA struct dquot __rcu *i_dquot[MAXQUOTAS]; #endif @@ -113,13 +119,56 @@ extern unsigned long shmem_get_unmapped_area(struct file *, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); extern int shmem_lock(struct file *file, int lock, struct ucounts *ucounts); #ifdef CONFIG_SHMEM +struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, struct inode *inode, + pgoff_t index, unsigned int order); bool shmem_mapping(const struct address_space *mapping); #else +static inline struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, + struct inode *inode, pgoff_t index, unsigned int order) +{ + return ERR_PTR(-EOPNOTSUPP); +} static inline bool shmem_mapping(const struct address_space *mapping) { return false; } #endif /* CONFIG_SHMEM */ +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +/* Return a locked cache folio and caller reference; new backing is all missing. */ +struct folio *shmem_uffd_prepare_folio(struct vm_area_struct *vma, + unsigned long address, bool *new); +void shmem_uffd_abort_folio(struct folio *folio, struct vm_area_struct *vma, bool new); +bool shmem_uffd_has_missing_native_page(struct folio *folio); +bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length); +/* Resident masks require no allocation; inode metadata persists only in swap. */ +void shmem_uffd_copy_page(struct page *dst, struct page *src); +void shmem_uffd_migrate(struct folio *dst, struct folio *src); +/* Locked, uptodate cache folio. */ +unsigned long shmem_uffd_missing_mask(struct folio *folio, pgoff_t index); +int shmem_uffd_set_missing(struct folio *folio, pgoff_t index, + unsigned long missing); +/* Clear populated units; no allocation. Caller holds the folio lock. */ +void shmem_uffd_mark_present(struct folio *folio, pgoff_t index, + unsigned long present); +/* Mark exactly the byte range exposed by a mapping; folio lock held. */ +void shmem_uffd_mark_mapped(struct folio *folio, unsigned long offset, + unsigned long length); +/* i_pages lock held; deletion, not replacement by swap or migration. */ +void shmem_uffd_forget(struct address_space *mapping, pgoff_t start, + unsigned long nr); +#else +static inline void shmem_uffd_copy_page(struct page *dst, struct page *src) { } +static inline void shmem_uffd_migrate(struct folio *dst, struct folio *src) { } +static inline bool shmem_uffd_has_missing_native_page(struct folio *folio) { return false; } +static inline bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length) { return false; } +static inline void shmem_uffd_mark_mapped(struct folio *folio, + unsigned long offset, unsigned long length) { } +static inline void shmem_uffd_forget(struct address_space *mapping, + pgoff_t start, unsigned long nr) { } +#endif + void shmem_unlock_mapping(struct address_space *mapping); struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); @@ -160,6 +209,8 @@ static inline void shmem_uncharge(struct inode *inode, long pages) { } #endif +unsigned long shmem_partial_swap_usage_bytes(struct address_space *mapping, + u64 start, u64 end); extern unsigned long shmem_partial_swap_usage(struct address_space *mapping, pgoff_t start, pgoff_t end); diff --git a/include/linux/swap.h b/include/linux/swap.h index 7a09df6977a5f..4e9136d3ae3f6 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -228,7 +228,7 @@ enum { */ #define SWAP_ENTRY_INVALID 0 -#ifdef CONFIG_THP_SWAP +#if defined(CONFIG_THP_SWAP) || defined(CONFIG_MM_SUBPAGE) #define SWAP_NR_ORDERS (PMD_ORDER + 1) #else #define SWAP_NR_ORDERS 1 @@ -443,6 +443,7 @@ extern int swp_swapcount(swp_entry_t entry); struct backing_dev_info; extern struct swap_info_struct *get_swap_device(swp_entry_t entry); sector_t swap_folio_sector(struct folio *folio); +sector_t swap_extent_sector(swp_entry_t entry, unsigned long *nr_pages); /* * If there is an existing swap slot reference (swap entry) and the caller @@ -455,6 +456,13 @@ sector_t swap_folio_sector(struct folio *folio); * a swap count > 1. See comments of folio_*_swap helpers for more info. */ int swap_dup_entry_direct(swp_entry_t entry); +int swap_dup_entries_direct(swp_entry_t entry, unsigned int nr); +#ifdef CONFIG_MM_SUBPAGE +/* Offset is bytes within one native swap entry; caller owns a PTE/cache ref. */ +int swap_subpage_dup(swp_entry_t entry, unsigned int offset); +void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache); +unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset); +#endif void swap_put_entries_direct(swp_entry_t entry, int nr); /* @@ -498,11 +506,27 @@ static inline void free_swap_cache(struct folio *folio) { } +#ifdef CONFIG_MM_SUBPAGE +static inline int swap_subpage_dup(swp_entry_t entry, unsigned int offset) +{ + return -EINVAL; +} +static inline void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache) { } +static inline unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset) +{ + return 0; +} +#endif static inline int swap_dup_entry_direct(swp_entry_t ent) { return 0; } +static inline int swap_dup_entries_direct(swp_entry_t ent, unsigned int nr) +{ + return 0; +} + static inline void swap_put_entries_direct(swp_entry_t ent, int nr) { } diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 4fb7291f54b62..29beb44f370fe 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1261,6 +1261,11 @@ static inline int ksys_fadvise64_64(int fd, loff_t offset, loff_t len, unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long fd, unsigned long pgoff); +#ifdef CONFIG_MM_SUBPAGE +unsigned long ksys_mmap_user(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long offset); +#endif + ssize_t ksys_readahead(int fd, loff_t offset, size_t count); int ksys_ipc(unsigned int call, int first, unsigned long second, unsigned long third, void __user * ptr, long fifth); diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index f548fea2adec8..237b530ed600f 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -192,6 +192,7 @@ struct uprobes_state { }; typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr, + unsigned int page_offset, uprobe_opcode_t *insn, int nbytes, void *data); extern void __init uprobes_init(void); diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h index cd7920c81f85b..fefbf65d61441 100644 --- a/include/trace/events/kmem.h +++ b/include/trace/events/kmem.h @@ -449,7 +449,7 @@ TRACE_EVENT(rss_stat, __entry->curr = current->mm == mm && !(current->flags & PF_KTHREAD); __entry->member = member; __entry->size = (percpu_counter_sum_positive(&mm->rss_stat[member]) - << PAGE_SHIFT); + << mm_page_shift(mm)); ), TP_printk("mm_id=%u curr=%d type=%s size=%ldB", diff --git a/include/trace/events/mmap.h b/include/trace/events/mmap.h index ee2843a5daef1..46df97560969d 100644 --- a/include/trace/events/mmap.h +++ b/include/trace/events/mmap.h @@ -26,7 +26,7 @@ TRACE_EVENT(vm_unmapped_area, TP_fast_assign( __entry->addr = addr; - __entry->total_vm = current->mm->total_vm; + __entry->total_vm = mm_pages_to_native(current->mm, current->mm->total_vm); __entry->flags = info->flags; __entry->length = info->length; __entry->low_limit = info->low_limit; diff --git a/include/trace/events/oom.h b/include/trace/events/oom.h index 9f0a5d1482c44..d5321b6f01ffa 100644 --- a/include/trace/events/oom.h +++ b/include/trace/events/oom.h @@ -7,7 +7,6 @@ #include #include -#define PG_COUNT_TO_KB(x) ((x) << (PAGE_SHIFT - 10)) TRACE_EVENT(oom_score_adj_update, @@ -93,10 +92,10 @@ TRACE_EVENT(mark_victim, TP_fast_assign( __entry->pid = task->pid; __assign_str(comm); - __entry->total_vm = PG_COUNT_TO_KB(task->mm->total_vm); - __entry->anon_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_ANONPAGES)); - __entry->file_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_FILEPAGES)); - __entry->shmem_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_SHMEMPAGES)); + __entry->total_vm = mm_pages_to_kb(task->mm, task->mm->total_vm); + __entry->anon_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_ANONPAGES)); + __entry->file_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_FILEPAGES)); + __entry->shmem_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_SHMEMPAGES)); __entry->uid = uid; __entry->pgtables = mm_pgtables_bytes(task->mm) >> 10; __entry->oom_score_adj = task->signal->oom_score_adj; diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index b6ec6f6937195..d4585670ad41b 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -416,4 +416,22 @@ struct prctl_mm_map { # define PR_CFI_DISABLE _BITUL(1) # define PR_CFI_LOCK _BITUL(2) +/* + * Aurora experimental ABI: private numbers, not upstream assignments. + * SET_EXEC selects the calling thread's next successful exec granule. + * Zero requests native pages; with an alternative DEFAULT in effect this + * installs an explicit one-shot native override. GET_EXEC returns zero when + * no override remains, or its size in bytes. Successful exec consumes it. + * SET_DEFAULT selects the fallback for subsequent execs; zero restores native. + * GET_DEFAULT returns the alternative size in bytes, or zero for native. + * Fork inherits both policies; exec preserves DEFAULT. Failed exec preserves + * both. Neither setter changes the live address space or other threads. + * AArch32 always selects 4K when supported, otherwise the native granule, + * independently of these preferences. Sizes are capability-checked. + */ +#define PR_AURORA_SET_EXEC_PAGE_SIZE 0x41555001 +#define PR_AURORA_GET_EXEC_PAGE_SIZE 0x41555002 +#define PR_AURORA_SET_DEFAULT_PAGE_SIZE 0x41555003 +#define PR_AURORA_GET_DEFAULT_PAGE_SIZE 0x41555004 + #endif /* _LINUX_PRCTL_H */ diff --git a/ipc/shm.c b/ipc/shm.c index b3e8a58e177d3..2929b9243f8e8 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -51,6 +51,14 @@ #include "util.h" +#ifndef arch_shm_attach_align +static unsigned long arch_shm_attach_align(const struct mm_struct *mm, + unsigned long align) +{ + return align; +} +#endif + struct shmid_kernel /* private to the kernel */ { struct kern_ipc_perm shm_perm; @@ -1538,6 +1546,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, if (shmid < 0) goto out; + shmlba = arch_shm_attach_align(current->mm, shmlba); if (addr) { if (addr & (shmlba - 1)) { if (shmflg & SHM_RND) { @@ -1552,7 +1561,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, goto out; } else #ifndef __ARCH_FORCE_SHMLBA - if (addr & ~PAGE_MASK) + if (addr & ~mm_page_mask(current->mm)) #endif goto out; } @@ -1723,6 +1732,20 @@ COMPAT_SYSCALL_DEFINE3(shmat, int, shmid, compat_uptr_t, shmaddr, int, shmflg) } #endif +/* VMA splits retain native file indices plus a subpage byte offset. */ +#ifdef CONFIG_MMU +static bool shm_vma_matches_addr(struct vm_area_struct *vma, unsigned long addr) +{ + unsigned long offset; + + if (vma->vm_ops != &shm_vm_ops || vma->vm_start < addr) + return false; + offset = vma->vm_start - addr; + return (offset >> PAGE_SHIFT) == vma->vm_pgoff && + (offset & ~PAGE_MASK) == vma_subpage_offset(vma); +} +#endif + /* * detach and kill segment if marked destroyed. * The work is done in shm_close. @@ -1739,7 +1762,7 @@ long ksys_shmdt(char __user *shmaddr) VMA_ITERATOR(vmi, mm, addr); #endif - if (addr & ~PAGE_MASK) + if (addr & ~mm_page_mask(mm)) return retval; if (mmap_write_lock_killable(mm)) @@ -1774,8 +1797,7 @@ long ksys_shmdt(char __user *shmaddr) * a fragment created by mprotect() and/or munmap(), or it * otherwise it starts at this address with no hassles. */ - if ((vma->vm_ops == &shm_vm_ops) && - (vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) { + if (shm_vma_matches_addr(vma, addr)) { /* * Record the file of the shm segment being @@ -1804,12 +1826,10 @@ long ksys_shmdt(char __user *shmaddr) * could possibly have landed at. Also cast things to loff_t to * prevent overflows and make comparisons vs. equal-width types. */ - size = PAGE_ALIGN(size); + size = ALIGN(size, mm_page_size(mm)); while (vma && (loff_t)(vma->vm_end - addr) <= size) { /* finding a matching vma now does not alter retval */ - if ((vma->vm_ops == &shm_vm_ops) && - ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) && - (vma->vm_file == file)) { + if (shm_vma_matches_addr(vma, addr) && vma->vm_file == file) { do_vmi_align_munmap(&vmi, vma, mm, vma->vm_start, vma->vm_end, NULL, false); } diff --git a/kernel/events/core.c b/kernel/events/core.c index 1a23ae67c3cc8..343dc1af5a308 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6812,7 +6812,7 @@ static void perf_event_init_userpage(struct perf_event *event) /* Allow new userspace to detect that bit 0 is deprecated */ userpg->cap_bit0_is_deprecated = 1; userpg->size = offsetof(struct perf_event_mmap_page, __reserved); - userpg->data_offset = PAGE_SIZE; + userpg->data_offset = perf_metadata_size(rb); userpg->data_size = perf_data_size(rb); unlock: @@ -6997,6 +6997,11 @@ typedef void (*mapped_f)(struct perf_event *event, struct mm_struct *mm); f; \ }) +static u64 perf_mmap_offset(struct vm_area_struct *vma) +{ + return ((u64)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); +} + static void perf_mmap_open(struct vm_area_struct *vma) { struct perf_event *event = vma->vm_file->private_data; @@ -7005,7 +7010,7 @@ static void perf_mmap_open(struct vm_area_struct *vma) refcount_inc(&event->mmap_count); refcount_inc(&event->rb->mmap_count); - if (vma->vm_pgoff) + if (perf_mmap_offset(vma)) refcount_inc(&event->rb->aux_mmap_count); if (mapped) @@ -7039,7 +7044,7 @@ static void perf_mmap_close(struct vm_area_struct *vma) * The AUX buffer is strictly a sub-buffer, serialize using aux_mutex * to avoid complications. */ - if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff && + if (rb_has_aux(rb) && perf_mmap_offset(vma) == rb->aux_offset && refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) { /* * Stop all AUX events that are writing to this buffer, @@ -7132,7 +7137,9 @@ static void perf_mmap_close(struct vm_area_struct *vma) static vm_fault_t perf_mmap_pfn_mkwrite(struct vm_fault *vmf) { /* The first page is the user control page, others are read-only. */ - return vmf->pgoff == 0 ? 0 : VM_FAULT_SIGBUS; + return !perf_mmap_offset(vmf->vma) && + vmf->address - vmf->vma->vm_start < mm_page_size(vmf->vma->vm_mm) ? + 0 : VM_FAULT_SIGBUS; } static int perf_mmap_may_split(struct vm_area_struct *vma, unsigned long addr) @@ -7153,7 +7160,8 @@ static const struct vm_operations_struct perf_mmap_vmops = { static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) { - unsigned long nr_pages = vma_pages(vma); + unsigned long va = vma->vm_start; + bool aux = perf_mmap_offset(vma) != 0; int err = 0; unsigned long pagenum; @@ -7197,9 +7205,21 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) * perf_mmap_open() and perf_mmap_close()) so we ensure the lifetime of * this mapping is maintained correctly. */ - for (pagenum = 0; pagenum < nr_pages; pagenum++) { - unsigned long va = vma->vm_start + PAGE_SIZE * pagenum; - struct page *page = perf_mmap_to_page(rb, vma->vm_pgoff + pagenum); + for (pagenum = 0; va < vma->vm_end; pagenum++) { + unsigned long size = min(PAGE_SIZE, vma->vm_end - va); + struct page *page; + + if (aux) { + if (pagenum >= rb->aux_nr_pages) { + err = -EINVAL; + break; + } + page = virt_to_page(rb->aux_pages[pagenum]); + } else { + page = perf_mmap_to_page(rb, pagenum); + if (!pagenum) + size = perf_metadata_size(rb); + } if (page == NULL) { err = -EINVAL; @@ -7207,16 +7227,17 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) } /* Map readonly, perf_mmap_pfn_mkwrite() called on write fault. */ - err = remap_pfn_range(vma, va, page_to_pfn(page), PAGE_SIZE, + err = remap_pfn_range(vma, va, page_to_pfn(page), size, vm_get_page_prot(vma->vm_flags & ~VM_SHARED)); if (err) break; + va += size; } #ifdef CONFIG_MMU /* Clear any partial mappings on error. */ if (err) - zap_vma_range(vma, vma->vm_start, nr_pages * PAGE_SIZE); + zap_vma_range(vma, vma->vm_start, vma->vm_end - vma->vm_start); #endif return err; @@ -7269,7 +7290,7 @@ static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer * { struct user_struct *user = rb->mmap_user; - atomic_long_sub((perf_data_size(rb) >> PAGE_SHIFT) + 1 - rb->mmap_locked, + atomic_long_sub(data_page_nr(rb) + 1 - rb->mmap_locked, &user->locked_vm); atomic64_sub(rb->mmap_locked, &vma->vm_mm->pinned_vm); } @@ -7277,7 +7298,9 @@ static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer * static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, unsigned long nr_pages) { - long extra = 0, user_extra = nr_pages; + unsigned int page_shift = mm_page_shift(vma->vm_mm); + unsigned long data_size; + long extra = 0, user_extra; struct perf_buffer *rb; int rb_flags = 0; @@ -7289,11 +7312,14 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, */ if (nr_pages != 0 && !is_power_of_2(nr_pages)) return -EINVAL; + data_size = nr_pages << page_shift; + /* Physical allocation and locked-memory accounting stay in native pages. */ + user_extra = DIV_ROUND_UP(data_size, PAGE_SIZE) + 1; WARN_ON_ONCE(event->ctx->parent_ctx); if (event->rb) { - if (data_page_nr(event->rb) != nr_pages) + if (perf_data_size(event->rb) != data_size) return -EINVAL; /* @@ -7329,7 +7355,7 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, if (vma->vm_flags & VM_WRITE) rb_flags |= RING_BUFFER_WRITABLE; - rb = rb_alloc(nr_pages, + rb = rb_alloc(data_size, page_shift, event->attr.watermark ? event->attr.wakeup_watermark : 0, event->cpu, rb_flags); @@ -7350,9 +7376,10 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, return 0; } -static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, - unsigned long nr_pages) +static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event) { + unsigned long size = vma->vm_end - vma->vm_start; + unsigned long nr_pages = size >> PAGE_SHIFT; long extra = 0, user_extra = nr_pages; u64 aux_offset, aux_size; struct perf_buffer *rb; @@ -7361,6 +7388,9 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, rb = event->rb; if (!rb) return -EINVAL; + /* PMU setup_aux() consumes whole native pages, independent of mmap layout. */ + if (!IS_ALIGNED(size, PAGE_SIZE)) + return -EINVAL; guard(mutex)(&rb->aux_mutex); @@ -7372,14 +7402,14 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, aux_offset = READ_ONCE(rb->user_page->aux_offset); aux_size = READ_ONCE(rb->user_page->aux_size); - if (aux_offset < perf_data_size(rb) + PAGE_SIZE) + if (aux_offset < perf_data_size(rb) + perf_metadata_size(rb)) return -EINVAL; - if (aux_offset != vma->vm_pgoff << PAGE_SHIFT) + if (aux_offset != perf_mmap_offset(vma)) return -EINVAL; /* already mapped with a different offset */ - if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff) + if (rb_has_aux(rb) && rb->aux_offset != aux_offset) return -EINVAL; if (aux_size != nr_pages * PAGE_SIZE) @@ -7409,7 +7439,7 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, if (vma->vm_flags & VM_WRITE) rb_flags |= RING_BUFFER_WRITABLE; - ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, + ret = rb_alloc_aux(rb, event, aux_offset, nr_pages, event->attr.aux_watermark, rb_flags); if (ret) { refcount_dec(&rb->mmap_count); @@ -7449,12 +7479,13 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) return ret; vma_size = vma->vm_end - vma->vm_start; - nr_pages = vma_size / PAGE_SIZE; + nr_pages = vma_size >> mm_page_shift(vma->vm_mm); if (nr_pages > INT_MAX) return -ENOMEM; - if (vma_size != PAGE_SIZE * nr_pages) + if (vma_size != mm_page_size(vma->vm_mm) * nr_pages || + mm_page_shift(vma->vm_mm) > PAGE_SHIFT) return -EINVAL; scoped_guard (mutex, &event->mmap_mutex) { @@ -7466,10 +7497,14 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) if (event->state <= PERF_EVENT_STATE_REVOKED) return -ENODEV; - if (vma->vm_pgoff == 0) + /* One ring has one immutable metadata/data boundary for all aliases. */ + if (event->rb && event->rb->mmap_page_shift != mm_page_shift(vma->vm_mm)) + return -EINVAL; + + if (!perf_mmap_offset(vma)) ret = perf_mmap_rb(vma, event, nr_pages); else - ret = perf_mmap_aux(vma, event, nr_pages); + ret = perf_mmap_aux(vma, event); if (ret) return ret; diff --git a/kernel/events/internal.h b/kernel/events/internal.h index c03c4f2eea571..5238cfa3399b1 100644 --- a/kernel/events/internal.h +++ b/kernel/events/internal.h @@ -15,9 +15,10 @@ struct perf_buffer { struct rcu_head rcu_head; #ifdef CONFIG_PERF_USE_VMALLOC struct work_struct work; - int page_order; /* allocation order */ #endif int nr_pages; /* nr of data pages */ + unsigned int data_page_shift; /* bytes per output chunk */ + unsigned int mmap_page_shift; /* userspace metadata size */ int overwrite; /* can overwrite itself */ int paused; /* can write into ring buffer */ @@ -44,7 +45,7 @@ struct perf_buffer { long aux_head; unsigned int aux_nest; long aux_wakeup; /* last aux_watermark boundary crossed by aux_head */ - unsigned long aux_pgoff; + u64 aux_offset; int aux_nr_pages; int aux_overwrite; refcount_t aux_mmap_count; @@ -80,10 +81,11 @@ static inline void rb_toggle_paused(struct perf_buffer *rb, bool pause) } extern struct perf_buffer * -rb_alloc(int nr_pages, long watermark, int cpu, int flags); +rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags); extern void perf_event_wakeup(struct perf_event *event); extern int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, - pgoff_t pgoff, int nr_pages, long watermark, int flags); + u64 offset, int nr_pages, long watermark, int flags); extern void rb_free_aux(struct perf_buffer *rb); extern struct perf_buffer *ring_buffer_get(struct perf_event *event); extern void ring_buffer_put(struct perf_buffer *rb); @@ -99,34 +101,19 @@ void perf_event_aux_event(struct perf_event *event, unsigned long head, extern struct page * perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff); -#ifdef CONFIG_PERF_USE_VMALLOC -/* - * Back perf_mmap() with vmalloc memory. - * - * Required for architectures that have d-cache aliasing issues. - */ - -static inline int page_order(struct perf_buffer *rb) -{ - return rb->page_order; -} - -#else - -static inline int page_order(struct perf_buffer *rb) +static inline unsigned long perf_data_size(struct perf_buffer *rb) { - return 0; + return (unsigned long)rb->nr_pages << rb->data_page_shift; } -#endif static inline int data_page_nr(struct perf_buffer *rb) { - return rb->nr_pages << page_order(rb); + return DIV_ROUND_UP(perf_data_size(rb), PAGE_SIZE); } -static inline unsigned long perf_data_size(struct perf_buffer *rb) +static inline unsigned long perf_metadata_size(struct perf_buffer *rb) { - return rb->nr_pages << (PAGE_SHIFT + page_order(rb)); + return 1UL << rb->mmap_page_shift; } static inline unsigned long perf_aux_size(struct perf_buffer *rb) @@ -154,7 +141,7 @@ static inline unsigned long perf_aux_size(struct perf_buffer *rb) handle->page++; \ handle->page &= rb->nr_pages - 1; \ handle->addr = rb->data_pages[handle->page]; \ - handle->size = PAGE_SIZE << page_order(rb); \ + handle->size = 1UL << rb->data_page_shift; \ } \ } while (len && written == size); \ \ diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 9fe92161715e0..6c5de8fba83f2 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -238,7 +238,7 @@ __perf_output_begin(struct perf_output_handle *handle, if (unlikely(head - local_read(&rb->wakeup) > rb->watermark)) local_add(rb->watermark, &rb->wakeup); - page_shift = PAGE_SHIFT + page_order(rb); + page_shift = rb->data_page_shift; handle->page = (offset >> page_shift) & (rb->nr_pages - 1); offset &= (1UL << page_shift) - 1; @@ -677,7 +677,7 @@ static void __rb_free_aux(struct perf_buffer *rb) } int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, - pgoff_t pgoff, int nr_pages, long watermark, int flags) + u64 offset, int nr_pages, long watermark, int flags) { bool overwrite = !(flags & RING_BUFFER_WRITABLE); int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu); @@ -784,7 +784,7 @@ int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, out: if (!ret) - rb->aux_pgoff = pgoff; + rb->aux_offset = offset; else __rb_free_aux(rb); @@ -803,8 +803,8 @@ void rb_free_aux(struct perf_buffer *rb) * Back perf_mmap() with regular GFP_KERNEL-0 pages. */ -static struct page * -__perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) +struct page * +perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) { if (pgoff > rb->nr_pages) return NULL; @@ -835,10 +835,12 @@ static void perf_mmap_free_page(void *addr) __free_page(page); } -struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) +struct perf_buffer *rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags) { struct perf_buffer *rb; unsigned long size; + int nr_pages = DIV_ROUND_UP(data_size, PAGE_SIZE); int i, node; size = sizeof(struct perf_buffer); @@ -863,6 +865,10 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) } rb->nr_pages = nr_pages; + rb->mmap_page_shift = mmap_page_shift; + /* A ring smaller than a native page still wraps at its requested size. */ + rb->data_page_shift = data_size && data_size < PAGE_SIZE ? + ilog2(data_size) : PAGE_SHIFT; ring_buffer_init(rb, watermark, flags); @@ -892,8 +898,8 @@ void rb_free(struct perf_buffer *rb) } #else -static struct page * -__perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) +struct page * +perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) { /* The '>' counts in the user page. */ if (pgoff > data_page_nr(rb)) @@ -917,10 +923,12 @@ void rb_free(struct perf_buffer *rb) schedule_work(&rb->work); } -struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) +struct perf_buffer *rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags) { struct perf_buffer *rb; unsigned long size; + int nr_pages = DIV_ROUND_UP(data_size, PAGE_SIZE); void *all_buf; int node; @@ -940,9 +948,10 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) rb->user_page = all_buf; rb->data_pages[0] = all_buf + PAGE_SIZE; + rb->mmap_page_shift = mmap_page_shift; if (nr_pages) { rb->nr_pages = 1; - rb->page_order = ilog2(nr_pages); + rb->data_page_shift = ilog2(data_size); } ring_buffer_init(rb, watermark, flags); @@ -957,21 +966,3 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) } #endif - -struct page * -perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) -{ - if (rb->aux_nr_pages) { - /* above AUX space */ - if (pgoff > rb->aux_pgoff + rb->aux_nr_pages) - return NULL; - - /* AUX space */ - if (pgoff >= rb->aux_pgoff) { - int aux_pgoff = array_index_nospec(pgoff - rb->aux_pgoff, rb->aux_nr_pages); - return virt_to_page(rb->aux_pages[aux_pgoff]); - } - } - - return __perf_mmap_to_page(rb, pgoff); -} diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 6300b216012cb..640991b1c25cb 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -30,6 +30,7 @@ #include #include /* check_stable_address_space */ #include +#include #include @@ -144,12 +145,14 @@ static bool valid_vma(struct vm_area_struct *vma, bool is_register) static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset) { - return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT) - + vma_subpage_offset(vma); } static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) { - return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start); + return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma) + + (vaddr - vma->vm_start); } /** @@ -191,7 +194,8 @@ static void copy_to_page(struct page *page, unsigned long vaddr, const void *src kunmap_local(kaddr); } -static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *insn, +static int verify_opcode(struct page *page, unsigned long vaddr, unsigned int page_offset, + uprobe_opcode_t *insn, int nbytes, void *data) { uprobe_opcode_t old_opcode; @@ -206,7 +210,7 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t * is a trap variant; uprobes always wins over any other (gdb) * breakpoint. */ - uprobe_copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE); + uprobe_copy_from_page(page, page_offset, &old_opcode, UPROBE_SWBP_INSN_SIZE); is_swbp = is_swbp_insn(&old_opcode); if (is_swbp_insn(insn)) { @@ -307,14 +311,24 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) { void *kaddr; struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *vma; + bool alternative = mm_page_size(mm) != PAGE_SIZE; +#endif int ret; short *ptr; if (!vaddr || !d) return -EINVAL; - ret = get_user_pages_remote(mm, vaddr, 1, - FOLL_WRITE, &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + ret = get_user_fragment_vma_remote(mm, vaddr, sizeof(short), + FOLL_WRITE, &fragment, &vma); + else +#endif + ret = get_user_pages_remote(mm, vaddr, 1, FOLL_WRITE, &page, NULL); if (unlikely(ret <= 0)) { /* * We are asking for 1 page. If get_user_pages_remote() fails, @@ -323,8 +337,16 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) return ret == 0 ? -EBUSY : ret; } - kaddr = kmap_local_page(page); - ptr = kaddr + (vaddr & ~PAGE_MASK); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) { + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + ptr = kaddr; + } else +#endif + { + kaddr = kmap_local_page(page); + ptr = kaddr + (vaddr & ~PAGE_MASK); + } if (unlikely(*ptr + d < 0)) { pr_warn("ref_ctr going negative. vaddr: 0x%lx, " @@ -337,7 +359,12 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) ret = 0; out: kunmap_local(kaddr); - put_page(page); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + release_user_fragments(&fragment, 1, !ret); + else +#endif + put_page(page); return ret; } @@ -381,7 +408,8 @@ static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm, } static bool orig_page_is_identical(struct vm_area_struct *vma, - unsigned long vaddr, struct page *page, bool *pmd_mappable) + unsigned long vaddr, struct page *page, unsigned int page_offset, + bool *pmd_mappable) { const pgoff_t index = vaddr_to_offset(vma, vaddr) >> PAGE_SHIFT; struct folio *orig_folio = filemap_get_folio(vma->vm_file->f_mapping, @@ -393,19 +421,30 @@ static bool orig_page_is_identical(struct vm_area_struct *vma, return false; orig_page = folio_file_page(orig_folio, index); - *pmd_mappable = folio_test_pmd_mappable(orig_folio); - identical = folio_test_uptodate(orig_folio) && - pages_identical(page, orig_page); + *pmd_mappable = mm_page_size(vma->vm_mm) == PAGE_SIZE && + folio_test_pmd_mappable(orig_folio); + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + unsigned int offset = vma_page_offset_at(vma, vaddr).offset; + void *a = kmap_local_page(page), *b = kmap_local_page(orig_page); + + identical = folio_test_uptodate(orig_folio) && + !memcmp(a + page_offset, b + offset, mm_page_size(vma->vm_mm)); + kunmap_local(b); + kunmap_local(a); + } else { + identical = folio_test_uptodate(orig_folio) && + pages_identical(page, orig_page); + } folio_put(orig_folio); return identical; } static int __uprobe_write(struct vm_area_struct *vma, - struct folio_walk *fw, struct folio *folio, - unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, + struct folio_walk *fw, struct folio *folio, struct mm_subpage *slot, + unsigned int page_offset, unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, bool is_register) { - const unsigned long vaddr = insn_vaddr & PAGE_MASK; + const unsigned long vaddr = insn_vaddr & mm_page_mask(vma->vm_mm); bool pmd_mappable; /* For now, we'll only handle PTE-mapped folios. */ @@ -417,6 +456,12 @@ static int __uprobe_write(struct vm_area_struct *vma, * but the VMA might not be writable. */ if (!pte_write(fw->pte)) { +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + if (!mm_subpage_is_exclusive(slot)) + return -EFAULT; + } else +#endif if (!PageAnonExclusive(fw->page)) return -EFAULT; if (unlikely(userfaultfd_pte_wp(vma, fw->pte))) @@ -430,31 +475,49 @@ static int __uprobe_write(struct vm_area_struct *vma, */ flush_cache_page(vma, vaddr, pte_pfn(fw->pte)); fw->pte = ptep_clear_flush(vma, vaddr, fw->ptep); - copy_to_page(fw->page, insn_vaddr, insn, nbytes); + copy_to_page(fw->page, page_offset, insn, nbytes); /* * When unregistering, we may only zap a PTE if uffd is disabled and * there are no unexpected folio references ... */ - if (is_register || userfaultfd_missing(vma) || - (folio_ref_count(folio) != folio_expected_ref_count(folio) + 1)) + if (is_register || userfaultfd_missing(vma)) + goto remap; +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + if (!mm_subpage_can_zap(slot)) + goto remap; + } else +#endif + if (folio_ref_count(folio) != folio_expected_ref_count(folio) + 1) goto remap; /* * ... and the mapped page is identical to the original page that * would get faulted in on next access. */ - if (!orig_page_is_identical(vma, vaddr, fw->page, &pmd_mappable)) + if (!orig_page_is_identical(vma, vaddr, fw->page, + page_offset - (insn_vaddr - vaddr), &pmd_mappable)) goto remap; dec_mm_counter(vma->vm_mm, MM_ANONPAGES); - folio_remove_rmap_pte(folio, fw->page, vma); +#ifdef CONFIG_MM_SUBPAGE + if (slot) + mm_subpage_remove_anon_rmap(slot, vma); + else +#endif + folio_remove_rmap_pte(folio, fw->page, vma); if (!folio_mapped(folio) && folio_test_swapcache(folio) && folio_trylock(folio)) { folio_free_swap(folio); folio_unlock(folio); } - folio_put(folio); +#ifdef CONFIG_MM_SUBPAGE + if (slot) + mm_subpage_put(slot); /* Retired PTE reference; caller still has its GET. */ + else +#endif + folio_put(folio); return pmd_mappable; remap: @@ -494,13 +557,24 @@ int uprobe_write_opcode(struct arch_uprobe *auprobe, struct vm_area_struct *vma, verify_opcode, is_register, true /* do_update_ref_ctr */, NULL); } +static void uprobe_put_folio(struct folio *folio, struct mm_subpage *slot) +{ +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + mm_subpage_put(slot); + return; + } +#endif + folio_put(folio); +} + int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, const unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, uprobe_write_verify_t verify, bool is_register, bool do_update_ref_ctr, void *data) { - const unsigned long vaddr = insn_vaddr & PAGE_MASK; struct mm_struct *mm = vma->vm_mm; + const unsigned long vaddr = insn_vaddr & mm_page_mask(mm); struct uprobe *uprobe; int ret, ref_ctr_updated = 0; unsigned int gup_flags = FOLL_FORCE; @@ -508,7 +582,16 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, struct folio_walk fw; struct folio *folio; struct page *page; - + struct mm_subpage *slot = NULL; + unsigned int page_offset; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *found; +#endif + + /* Larger-than-native public mms are not enabled by the mixed-page ABI yet. */ + if (mm_page_size(mm) > PAGE_SIZE) + return -EOPNOTSUPP; uprobe = container_of(auprobe, struct uprobe, arch); if (WARN_ON_ONCE(!is_cow_mapping(vma->vm_flags))) @@ -526,14 +609,29 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, gup_flags |= FOLL_WRITE | FOLL_SPLIT_PMD; retry: - ret = get_user_pages_remote(mm, vaddr, 1, gup_flags, &page, NULL); - if (ret <= 0) - goto out; - folio = page_folio(page); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + ret = get_user_fragment_vma_remote(mm, vaddr, mm_page_size(mm), + gup_flags & ~FOLL_SPLIT_PMD, &fragment, &found); + if (ret <= 0) + goto out; + folio = fragment.folio; + slot = fragment.subpage; + page = folio_page(folio, fragment.offset >> PAGE_SHIFT); + page_offset = offset_in_page(fragment.offset) + insn_vaddr - vaddr; + } else +#endif + { + ret = get_user_pages_remote(mm, vaddr, 1, gup_flags, &page, NULL); + if (ret <= 0) + goto out; + folio = page_folio(page); + page_offset = offset_in_page(insn_vaddr); + } - ret = verify(page, insn_vaddr, insn, nbytes, data); + ret = verify(page, insn_vaddr, page_offset, insn, nbytes, data); if (ret <= 0) { - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -541,7 +639,7 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, if (do_update_ref_ctr && !ref_ctr_updated && uprobe->ref_ctr_offset) { ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1); if (ret) { - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -551,7 +649,7 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, ret = 0; if (unlikely(!folio_test_anon(folio) || folio_is_zone_device(folio))) { VM_WARN_ON_ONCE(is_register); - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -562,22 +660,29 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, * be able to do it under PTL. */ mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, - vaddr, vaddr + PAGE_SIZE); + vaddr, vaddr + mm_page_size(mm)); mmu_notifier_invalidate_range_start(&range); } ret = -EAGAIN; /* Walk the page tables again, to perform the actual update. */ if (folio_walk_start(&fw, vma, vaddr, 0)) { - if (fw.page == page) - ret = __uprobe_write(vma, &fw, folio, insn_vaddr, insn, nbytes, is_register); + bool same = fw.page == page; +#ifdef CONFIG_MM_SUBPAGE + if (slot) + same &= fw.level == FW_LEVEL_PTE && + pte_phys_mm(mm, fw.pte) == mm_subpage_phys(slot); +#endif + if (same) + ret = __uprobe_write(vma, &fw, folio, slot, page_offset, + insn_vaddr, insn, nbytes, is_register); folio_walk_end(&fw, vma); } if (!is_register) mmu_notifier_invalidate_range_end(&range); - folio_put(folio); + uprobe_put_folio(folio, slot); switch (ret) { case -EFAULT: gup_flags |= FOLL_WRITE | FOLL_SPLIT_PMD; @@ -1211,7 +1316,9 @@ build_map_info(struct address_space *mapping, loff_t offset, bool is_register) again: i_mmap_lock_read(mapping); vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { - if (!valid_vma(vma, is_register)) + if (!valid_vma(vma, is_register) || + offset < vaddr_to_offset(vma, vma->vm_start) || + offset >= vaddr_to_offset(vma, vma->vm_end)) continue; if (!prev && !more) { @@ -1482,7 +1589,7 @@ static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) file_inode(vma->vm_file) != uprobe->inode) continue; - offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; + offset = vaddr_to_offset(vma, vma->vm_start); if (uprobe->offset < offset || uprobe->offset >= offset + vma->vm_end - vma->vm_start) continue; @@ -1696,8 +1803,14 @@ static const struct vm_special_mapping xol_mapping = { unsigned long __weak arch_uprobe_get_xol_area(void) { - /* Try to map as high as possible, this is only a hint. */ - return get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0); + unsigned long size = PAGE_SIZE; + unsigned long addr; + + /* XOL arch copy hooks use native offsets into the one backing page. */ + if (mm_page_size(current->mm) < PAGE_SIZE) + size += PAGE_SIZE - mm_page_size(current->mm); + addr = get_unmapped_area(NULL, TASK_SIZE - size, size, 0, 0); + return IS_ERR_VALUE(addr) ? addr : ALIGN(addr, PAGE_SIZE); } /* Slot allocation for XOL */ @@ -2413,6 +2526,23 @@ static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr) if (likely(result == 0)) goto out; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + struct user_page_fragment fragment; + struct vm_area_struct *vma; + void *kaddr; + + result = get_user_fragment_vma_remote(mm, vaddr, UPROBE_SWBP_INSN_SIZE, + FOLL_FORCE, &fragment, &vma); + if (result != 1) + return result < 0 ? result : -EFAULT; + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + memcpy(&opcode, kaddr, UPROBE_SWBP_INSN_SIZE); + kunmap_local(kaddr); + release_user_fragments(&fragment, 1, false); + goto out; + } +#endif result = get_user_pages(vaddr, 1, FOLL_FORCE, &page); if (result < 0) return result; @@ -2453,7 +2583,7 @@ static struct uprobe *find_active_uprobe_speculative(unsigned long bp_vaddr) if (!vm_file) return NULL; - offset = (loff_t)(vma->vm_pgoff << PAGE_SHIFT) + (bp_vaddr - vma->vm_start); + offset = vaddr_to_offset(vma, bp_vaddr); uprobe = find_uprobe_rcu(vm_file->f_inode, offset); if (!uprobe) return NULL; diff --git a/kernel/fork.c b/kernel/fork.c index 5a4623b058a20..88788baaff964 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -112,7 +112,7 @@ #include #include -#include +#include #include #include @@ -639,6 +639,12 @@ static void check_mm(struct mm_struct *mm) } } +#ifdef CONFIG_MM_SUBPAGE + if (mm->committed_user_pages) + pr_alert("BUG: non-zero user commitment on freeing mm: %lu\n", + mm->committed_user_pages); +#endif + if (mm_pgtables_bytes(mm)) pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", mm_pgtables_bytes(mm)); @@ -1089,6 +1095,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, INIT_LIST_HEAD(&mm->mmlist); mm_pgtables_bytes_init(mm); mm->map_count = 0; +#ifdef CONFIG_MM_SUBPAGE + mm->committed_user_pages = 0; +#endif mm->locked_vm = 0; atomic64_set(&mm->pinned_vm, 0); memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); @@ -1159,7 +1168,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, /* * Allocate and initialize an mm_struct. */ -struct mm_struct *mm_alloc(void) +static struct mm_struct *__mm_alloc(bool for_exec, unsigned int page_shift) { struct mm_struct *mm; @@ -1168,10 +1177,32 @@ struct mm_struct *mm_alloc(void) return NULL; memset(mm, 0, sizeof(*mm)); + if (for_exec) + arch_mm_init_exec(mm, page_shift); return mm_init(mm, current, current_user_ns()); } + +struct mm_struct *mm_alloc(void) +{ + return __mm_alloc(false, 0); +} EXPORT_SYMBOL_IF_KUNIT(mm_alloc); +struct mm_struct *mm_alloc_exec(void) +{ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + return __mm_alloc(true, current->exec_page_shift); +#else + return __mm_alloc(true, 0); +#endif +} + +/* The caller has validated the requested exec granule. */ +struct mm_struct *mm_alloc_exec_page_shift(unsigned int page_shift) +{ + return __mm_alloc(true, page_shift); +} + static inline void __mmput(struct mm_struct *mm) { VM_BUG_ON(atomic_read(&mm->mm_users)); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 7ffe6dc79b28e..771c12983cd2a 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include "futex.h" @@ -512,6 +513,44 @@ static u64 get_inode_sequence_number(struct inode *inode) } } +/* Keep a typed slot reference alive until the mapping identity is sampled. */ +struct futex_page_ref { + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; +#endif +}; + +static int futex_get_page_ref(struct mm_struct *mm, unsigned long address, + unsigned long uaddr, unsigned int size, unsigned int flags, + struct futex_page_ref *ref) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + int ret = get_user_fragments_remote(mm, uaddr, size, flags, + &ref->fragment, 1); + + if (ret != 1) + return ret ?: -EFAULT; + ref->page = folio_page(ref->fragment.folio, + ref->fragment.offset >> PAGE_SHIFT); + return 1; + } +#endif + return get_user_pages_fast(address, 1, flags, &ref->page); +} + +static void futex_put_page_ref(struct futex_page_ref *ref) +{ +#ifdef CONFIG_MM_SUBPAGE + if (ref->fragment.folio) { + release_user_fragments(&ref->fragment, 1, false); + return; + } +#endif + put_page(ref->page); +} + /** * get_futex_key() - Get parameters which are the keys for a futex * @uaddr: virtual address of the futex @@ -545,6 +584,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, unsigned long address = (unsigned long)uaddr; struct mm_struct *mm = current->mm; struct page *page; + struct futex_page_ref ref = {}; struct folio *folio; struct address_space *mapping; int node, err, size, ro = 0; @@ -629,13 +669,13 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, if (unlikely(should_fail_futex(true))) return -EFAULT; - err = get_user_pages_fast(address, 1, FOLL_WRITE, &page); + err = futex_get_page_ref(mm, address, (unsigned long)uaddr, size, FOLL_WRITE, &ref); /* * If write access is not required (eg. FUTEX_WAIT), try * and get read-only access. */ if (err == -EFAULT && rw == FUTEX_READ) { - err = get_user_pages_fast(address, 1, 0, &page); + err = futex_get_page_ref(mm, address, (unsigned long)uaddr, size, 0, &ref); ro = 1; } if (err < 0) @@ -660,6 +700,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, * filesystem-backed pages, the precise page is required as the * index of the page determines the key. */ + page = ref.page; folio = page_folio(page); mapping = READ_ONCE(folio->mapping); @@ -689,7 +730,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, folio_lock(folio); shmem_swizzled = folio_test_swapcache(folio) || folio->mapping; folio_unlock(folio); - folio_put(folio); + futex_put_page_ref(&ref); if (shmem_swizzled) goto again; @@ -739,7 +780,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, if (READ_ONCE(folio->mapping) != mapping) { rcu_read_unlock(); - folio_put(folio); + futex_put_page_ref(&ref); goto again; } @@ -747,11 +788,15 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, inode = READ_ONCE(mapping->host); if (!inode) { rcu_read_unlock(); - folio_put(folio); + futex_put_page_ref(&ref); goto again; } +#ifdef CONFIG_MM_SUBPAGE + if (ref.fragment.folio) + key->both.offset = offset_in_page(ref.fragment.offset); +#endif key->both.offset |= FUT_OFF_INODE; /* inode-based key */ key->shared.i_seq = get_inode_sequence_number(inode); key->shared.pgoff = page_pgoff(folio, page); @@ -759,7 +804,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, } out: - folio_put(folio); + futex_put_page_ref(&ref); return err; } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index abb76775aa728..e06b05a08f7f4 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1694,7 +1694,7 @@ static unsigned int task_nr_scan_windows(struct task_struct *p) * on resident pages */ nr_scan_pages = MB_TO_PAGES(sysctl_numa_balancing_scan_size); - rss = get_mm_rss(p->mm); + rss = mm_pages_to_native(p->mm, get_mm_rss(p->mm)); if (!rss) rss = nr_scan_pages; @@ -3682,8 +3682,8 @@ static void task_numa_work(struct callback_head *work) * areas faster. */ if (nr_pte_updates) - pages -= (end - start) >> PAGE_SHIFT; - virtpages -= (end - start) >> PAGE_SHIFT; + pages -= DIV_ROUND_UP(end - start, PAGE_SIZE); + virtpages -= DIV_ROUND_UP(end - start, PAGE_SIZE); start = end; if (pages <= 0 || virtpages <= 0) diff --git a/kernel/sys.c b/kernel/sys.c index 62e842055cc9c..54ef058bfad21 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -1926,7 +1926,7 @@ void getrusage(struct task_struct *p, int who, struct rusage *r) } out_children: - r->ru_maxrss = maxrss * (PAGE_SIZE / 1024); /* convert pages to KBs */ + r->ru_maxrss = maxrss; /* signal and mm high-water maxima are in kB */ r->ru_utime = ns_to_kernel_old_timeval(utime); r->ru_stime = ns_to_kernel_old_timeval(stime); } @@ -2531,6 +2531,40 @@ static int prctl_set_thp_disable(bool thp_disable, unsigned long flags, return 0; } +static int prctl_page_size_get(struct task_struct *task, bool persistent) +{ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + unsigned int shift = persistent ? task->default_exec_page_shift : + task->exec_page_shift; + + return shift ? 1U << shift : 0; +#else + return -EINVAL; +#endif +} + +static int prctl_page_size_set(struct task_struct *task, unsigned long size, + bool persistent) +{ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + int shift = arch_exec_page_size_shift(size); + + if (shift < 0) + return shift; + if (persistent) { + task->default_exec_page_shift = shift; + } else { + /* A native request overrides an inherited default for one exec. */ + if (!shift && task->default_exec_page_shift) + shift = PAGE_SHIFT; + task->exec_page_shift = shift; + } + return 0; +#else + return -EINVAL; +#endif +} + SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, unsigned long, arg4, unsigned long, arg5) { @@ -2544,6 +2578,22 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = 0; switch (option) { + case PR_AURORA_GET_EXEC_PAGE_SIZE: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_get(me, false); + case PR_AURORA_SET_EXEC_PAGE_SIZE: + if (arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_set(me, arg2, false); + case PR_AURORA_GET_DEFAULT_PAGE_SIZE: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_get(me, true); + case PR_AURORA_SET_DEFAULT_PAGE_SIZE: + if (arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_set(me, arg2, true); case PR_SET_PDEATHSIG: if (!valid_signal(arg2)) { error = -EINVAL; diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 8c82ecb735f41..e8b622a8178cb 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include "trace_dynevent.h" @@ -554,6 +555,11 @@ static int user_event_enabler_write(struct user_event_mm *mm, unsigned long uaddr = enabler->addr; unsigned long *ptr; struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *vma; + bool alternative = mm_page_size(mm->mm) != PAGE_SIZE; +#endif void *kaddr; int bit = ENABLE_BIT(enabler); int ret; @@ -573,8 +579,14 @@ static int user_event_enabler_write(struct user_event_mm *mm, align_addr_bit(&uaddr, &bit, ENABLE_BITOPS(enabler)); - ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT, - &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + ret = pin_user_fragment_vma_remote(mm->mm, uaddr, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &vma); + else +#endif + ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT, + &page, NULL); if (unlikely(ret <= 0)) { if (!fixup_fault) @@ -586,8 +598,17 @@ static int user_event_enabler_write(struct user_event_mm *mm, return -EFAULT; } - kaddr = kmap_local_page(page); - ptr = kaddr + (uaddr & ~PAGE_MASK); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) { + /* align_addr_bit() keeps the complete atomic word in this fragment. */ + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + ptr = kaddr; + } else +#endif + { + kaddr = kmap_local_page(page); + ptr = kaddr + (uaddr & ~PAGE_MASK); + } /* Update bit atomically, user tracers must be atomic as well */ if (enabler->event && enabler->event->status) @@ -596,7 +617,12 @@ static int user_event_enabler_write(struct user_event_mm *mm, clear_bit(bit, ptr); kunmap_local(kaddr); - unpin_user_pages_dirty_lock(&page, 1, true); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + release_user_fragments(&fragment, 1, true); + else +#endif + unpin_user_pages_dirty_lock(&page, 1, true); return 0; } diff --git a/kernel/tsacct.c b/kernel/tsacct.c index 5c153106e6427..5f59d071ab565 100644 --- a/kernel/tsacct.c +++ b/kernel/tsacct.c @@ -91,16 +91,16 @@ void xacct_add_tsk(struct taskstats *stats, struct task_struct *p) { struct mm_struct *mm; - /* convert pages-nsec/1024 to Mbyte-usec, see __acct_update_integrals */ - stats->coremem = p->acct_rss_mem1 * PAGE_SIZE; + /* convert account-units-nsec/1024 to Mbyte-usec, see __acct_update_integrals */ + stats->coremem = p->acct_rss_mem1 * (1UL << MM_ACCOUNT_SHIFT); do_div(stats->coremem, 1000 * KB); - stats->virtmem = p->acct_vm_mem1 * PAGE_SIZE; + stats->virtmem = p->acct_vm_mem1 * (1UL << MM_ACCOUNT_SHIFT); do_div(stats->virtmem, 1000 * KB); mm = get_task_mm(p); if (mm) { /* adjust to KB unit */ - stats->hiwater_rss = get_mm_hiwater_rss(mm) * PAGE_SIZE / KB; - stats->hiwater_vm = get_mm_hiwater_vm(mm) * PAGE_SIZE / KB; + stats->hiwater_rss = mm_pages_to_kb(mm, get_mm_hiwater_rss(mm)); + stats->hiwater_vm = mm_pages_to_kb(mm, get_mm_hiwater_vm(mm)); mmput(mm); } stats->read_char = p->ioac.rchar & KB_MASK; @@ -140,8 +140,10 @@ static void __acct_update_integrals(struct task_struct *tsk, * The final unit reported to userspace is Mbyte-usecs, * the rest of the math is done in xacct_add_tsk. */ - tsk->acct_rss_mem1 += delta * get_mm_rss(tsk->mm) >> 10; - tsk->acct_vm_mem1 += delta * READ_ONCE(tsk->mm->total_vm) >> 10; + tsk->acct_rss_mem1 += delta * mm_pages_to_shift(tsk->mm, get_mm_rss(tsk->mm), + MM_ACCOUNT_SHIFT) >> 10; + tsk->acct_vm_mem1 += delta * mm_pages_to_shift(tsk->mm, READ_ONCE(tsk->mm->total_vm), + MM_ACCOUNT_SHIFT) >> 10; } /** diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c index 17d37b82ebc68..d808ddcb94b01 100644 --- a/lib/vdso/datastore.c +++ b/lib/vdso/datastore.c @@ -83,7 +83,8 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, vm_fault_t err; addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE; - err = vmf_insert_page(vma, addr, page); + err = vmf_insert_page_offset(vma, addr, page, + vma_page_offset_at(vma, vmf->address).offset); if (unlikely(err & VM_FAULT_ERROR)) return err; page = timens_page; diff --git a/mm/Kconfig b/mm/Kconfig index e8bf1e9e6ad90..b743b2290b309 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1164,6 +1164,28 @@ config IDLE_PAGE_TRACKING See Documentation/admin-guide/mm/idle_page_tracking.rst for more details. +config ARCH_HAS_MM_PGTABLE + bool + help + Architecture supplies target-mm page-table geometry and accessors. + This does not change the native physical allocation granule. + +config ARCH_HAS_USER_PAGE_SIZE + bool + help + Architecture supports selecting the user mapping granule at exec. + It supplies arch_exec_page_size_shift() to validate requests, and + integrates the selected granule into mm allocation and binary loading. + +config MM_SUBPAGE + bool + select PAGE_EXTENSION + select XARRAY_MULTI + help + Reference-counted user data slots smaller than a native folio. + Architectures must integrate rmap, faults, reclaim and pins before + exposing these backing primitives to userspace. + # Architectures which implement cpu_dcache_is_aliasing() to query # whether the data caches are aliased (VIVT or VIPT with dcache # aliasing) need to select this. @@ -1439,9 +1461,15 @@ config ARCH_HAS_USER_SHADOW_STACK config HAVE_ARCH_TLB_REMOVE_TABLE def_bool n +config ARCH_SUPPORTS_PT_RECLAIM + bool + help + The architecture's custom table-free callback supports PTE reclamation. + config PT_RECLAIM def_bool y - depends on MMU_GATHER_RCU_TABLE_FREE && !HAVE_ARCH_TLB_REMOVE_TABLE + depends on MMU_GATHER_RCU_TABLE_FREE + depends on !HAVE_ARCH_TLB_REMOVE_TABLE || ARCH_SUPPORTS_PT_RECLAIM help Try to reclaim empty user page table pages in paths other than munmap and exit_mmap path. diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index cca7014ab5f13..3d618828c2c8d 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -155,9 +155,10 @@ int damon_cold_score(struct damon_ctx *c, struct damon_region *r, } static bool damon_folio_mkold_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + unsigned long addr = rmap.address; + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); while (page_vma_mapped_walk(&pvmw)) { addr = pvmw.address; @@ -190,10 +191,11 @@ void damon_folio_mkold(struct folio *folio) } static bool damon_folio_young_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long addr = rmap.address; bool *accessed = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); pte_t pte; *accessed = false; diff --git a/mm/filemap.c b/mm/filemap.c index bbd400d893aa5..14e2656b6a9cc 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -225,6 +225,7 @@ void __filemap_remove_folio(struct folio *folio, void *shadow) trace_mm_filemap_delete_from_page_cache(folio); filemap_unaccount_folio(mapping, folio); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); page_cache_delete(mapping, folio, shadow); } @@ -308,6 +309,7 @@ static void page_cache_delete_batch(struct address_space *mapping, WARN_ON_ONCE(!folio_test_locked(folio)); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); folio->mapping = NULL; /* Leave folio->index set: truncation lookup relies on it */ @@ -827,6 +829,7 @@ void replace_page_cache_folio(struct folio *old, struct folio *new) mem_cgroup_replace_folio(old, new); xas_lock_irq(&xas); + shmem_uffd_migrate(new, old); xas_store(&xas, new); old->mapping = NULL; @@ -3387,7 +3390,7 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) */ struct vm_area_struct *vma = vmf->vma; unsigned long start = vma->vm_pgoff; - unsigned long end = start + vma_pages(vma); + unsigned long end = vma_last_pgoff(vma) + 1; unsigned long ra_end; ra->order = exec_folio_order(); @@ -3529,7 +3532,12 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) bool mapping_locked = false; max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - if (unlikely(index >= max_idx)) + if (unlikely(index >= max_idx +#ifdef CONFIG_MM_SUBPAGE + || (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) +#endif + )) return VM_FAULT_SIGBUS; trace_mm_filemap_fault(mapping, index); @@ -3633,7 +3641,12 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) * We must recheck i_size under page lock. */ max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - if (unlikely(index >= max_idx)) { + if (unlikely(index >= max_idx +#ifdef CONFIG_MM_SUBPAGE + || (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) +#endif + )) { folio_unlock(folio); folio_put(folio); return VM_FAULT_SIGBUS; @@ -3783,6 +3796,10 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, } do { + if (userfaultfd_missing(vmf->vma) && + shmem_uffd_range_missing(folio, folio->index + + folio_page_idx(folio, page + count), 0, PAGE_SIZE)) + goto skip; if (PageHWPoison(page + count)) goto skip; @@ -3847,6 +3864,9 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, vm_fault_t ret = 0; struct page *page = &folio->page; + if (userfaultfd_missing(vmf->vma) && + shmem_uffd_range_missing(folio, folio->index, 0, PAGE_SIZE)) + goto out; if (PageHWPoison(page)) goto out; @@ -3875,6 +3895,48 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, return ret; } +#ifdef CONFIG_MM_SUBPAGE +static vm_fault_t filemap_map_user_pages(struct vm_fault *vmf, + pgoff_t first, pgoff_t last) +{ + struct vm_area_struct *vma = vmf->vma; + struct address_space *mapping = vma->vm_file->f_mapping; + XA_STATE(xas, &mapping->i_pages, first); + struct folio *folio; + vm_fault_t ret = 0; + unsigned short miss = 0, saved; + + rcu_read_lock(); + folio = next_uptodate_folio(&xas, mapping, last); + if (!folio) + goto out; + if (pmd_none(*vmf->pmd) && vmf->prealloc_pte) + pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte); + vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, + vmf->address, &vmf->ptl); + if (!vmf->pte) { + folio_unlock(folio); + folio_put(folio); + goto out; + } + do { + ret |= map_file_user_pages(vmf, folio, first, last, false); + if (!folio_test_workingset(folio)) + miss += min(folio_next_index(folio) - 1, last) - + max(folio->index, first) + 1; + folio_unlock(folio); + folio_put(folio); + } while ((folio = next_uptodate_folio(&xas, mapping, last)) != NULL); + pte_unmap_unlock(vmf->pte, vmf->ptl); + trace_mm_filemap_map_pages(mapping, first, last); +out: + rcu_read_unlock(); + saved = READ_ONCE(vma->vm_file->f_ra.mmap_miss); + WRITE_ONCE(vma->vm_file->f_ra.mmap_miss, saved > miss ? saved - miss : 0); + return ret; +} +#endif + vm_fault_t filemap_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff) { @@ -3890,6 +3952,11 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, unsigned int nr_pages = 0, folio_type; unsigned short mmap_miss = 0, mmap_miss_saved; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return filemap_map_user_pages(vmf, start_pgoff, end_pgoff); +#endif + /* * Recalculate end_pgoff based on file_end before calling * next_uptodate_folio() to avoid races with concurrent diff --git a/mm/gup.c b/mm/gup.c index ad9ded39609cb..8d9bbc80546fa 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -1297,6 +1298,11 @@ static struct vm_area_struct *gup_vma_lookup(struct mm_struct *mm, #endif } +#ifdef CONFIG_MM_SUBPAGE +static long prefault_subpage_range(struct mm_struct *mm, unsigned long start, + unsigned long nr_pages, unsigned int flags, int *locked); +#endif + /** * __get_user_pages() - pin user pages in memory * @mm: mm_struct of target mm @@ -1363,6 +1369,15 @@ static long __get_user_pages(struct mm_struct *mm, if (!nr_pages) return 0; +#ifdef CONFIG_MM_SUBPAGE + /* A prefault has no page-array ownership or physical-offset ambiguity. */ + if (mm_page_size(mm) != PAGE_SIZE) { + if (pages || (gup_flags & (FOLL_GET | FOLL_PIN))) + return -EOPNOTSUPP; + return prefault_subpage_range(mm, start, nr_pages, gup_flags, locked); + } +#endif + start = untagged_addr_remote(mm, start); VM_WARN_ON_ONCE(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); @@ -1721,7 +1736,7 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm, */ if (likely(pages)) pages += ret; - start += ret << PAGE_SHIFT; + start += ret << mm_page_shift(mm); /* The lock was temporarily dropped, so we must unlock later */ must_unlock = true; @@ -1768,7 +1783,7 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm, break; if (likely(pages)) pages++; - start += PAGE_SIZE; + start += mm_page_size(mm); } if (must_unlock && *locked) { /* @@ -1814,13 +1829,13 @@ long populate_vma_page_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, int *locked) { struct mm_struct *mm = vma->vm_mm; - unsigned long nr_pages = (end - start) / PAGE_SIZE; + unsigned long nr_pages = (end - start) / mm_page_size(mm); int local_locked = 1; int gup_flags; long ret; - VM_WARN_ON_ONCE(!PAGE_ALIGNED(start)); - VM_WARN_ON_ONCE(!PAGE_ALIGNED(end)); + VM_WARN_ON_ONCE(!IS_ALIGNED(start, mm_page_size(mm))); + VM_WARN_ON_ONCE(!IS_ALIGNED(end, mm_page_size(mm))); VM_WARN_ON_ONCE_VMA(start < vma->vm_start, vma); VM_WARN_ON_ONCE_VMA(end > vma->vm_end, vma); mmap_assert_locked(mm); @@ -1887,12 +1902,12 @@ long populate_vma_page_range(struct vm_area_struct *vma, long faultin_page_range(struct mm_struct *mm, unsigned long start, unsigned long end, bool write, int *locked) { - unsigned long nr_pages = (end - start) / PAGE_SIZE; + unsigned long nr_pages = (end - start) / mm_page_size(mm); int gup_flags; long ret; - VM_WARN_ON_ONCE(!PAGE_ALIGNED(start)); - VM_WARN_ON_ONCE(!PAGE_ALIGNED(end)); + VM_WARN_ON_ONCE(!IS_ALIGNED(start, mm_page_size(mm))); + VM_WARN_ON_ONCE(!IS_ALIGNED(end, mm_page_size(mm))); mmap_assert_locked(mm); /* @@ -1968,7 +1983,7 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) } break; } - nend = nstart + ret * PAGE_SIZE; + nend = nstart + ret * mm_page_size(mm); ret = 0; } if (locked) @@ -2047,6 +2062,7 @@ size_t fault_in_writeable(char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; if (unlikely(size == 0)) @@ -2055,7 +2071,7 @@ size_t fault_in_writeable(char __user *uaddr, size_t size) return size; /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) unsafe_put_user(0, (char __user *)cur, out); out: user_write_access_end(); @@ -2116,6 +2132,7 @@ size_t fault_in_safe_writeable(const char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; struct mm_struct *mm = current->mm; bool unlocked = false; @@ -2125,7 +2142,7 @@ size_t fault_in_safe_writeable(const char __user *uaddr, size_t size) mmap_read_lock(mm); /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) if (fixup_user_fault(mm, cur, FAULT_FLAG_WRITE, &unlocked)) break; mmap_read_unlock(mm); @@ -2148,6 +2165,7 @@ size_t fault_in_readable(const char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; volatile char c; @@ -2157,7 +2175,7 @@ size_t fault_in_readable(const char __user *uaddr, size_t size) return size; /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) unsafe_get_user(c, (const char __user *)cur, out); out: user_read_access_end(); @@ -2211,11 +2229,18 @@ struct pages_or_folios { void **entries; }; bool has_folios; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *fragments; +#endif long nr_entries; }; static struct folio *pofs_get_folio(struct pages_or_folios *pofs, long i) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) + return pofs->fragments[i].folio; +#endif if (pofs->has_folios) return pofs->folios[i]; return page_folio(pofs->pages[i]); @@ -2223,22 +2248,56 @@ static struct folio *pofs_get_folio(struct pages_or_folios *pofs, long i) static void pofs_clear_entry(struct pages_or_folios *pofs, long i) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + pofs->fragments[i] = (struct user_page_fragment) {}; + return; + } +#endif pofs->entries[i] = NULL; } static void pofs_unpin(struct pages_or_folios *pofs) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + release_user_fragments(pofs->fragments, pofs->nr_entries, false); + return; + } +#endif if (pofs->has_folios) unpin_folios(pofs->folios, pofs->nr_entries); else unpin_user_pages(pofs->pages, pofs->nr_entries); } +static void pofs_unpin_entry(struct pages_or_folios *pofs, long i) +{ +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + release_user_fragments(&pofs->fragments[i], 1, false); + return; + } +#endif + unpin_folio(pofs_get_folio(pofs, i)); +} + static struct folio *pofs_next_folio(struct folio *folio, struct pages_or_folios *pofs, long *index_ptr) { long i = *index_ptr + 1; +#ifdef CONFIG_MM_SUBPAGE + /* Several adjacent user quarters can pin the same native folio. */ + if (pofs->fragments) { + while (i < pofs->nr_entries && pofs->fragments[i].folio == folio) + i++; + if (i == pofs->nr_entries) + return NULL; + *index_ptr = i; + return pofs->fragments[i].folio; + } +#endif if (!pofs->has_folios && folio_test_large(folio)) { const unsigned long start_pfn = folio_pfn(folio); const unsigned long end_pfn = start_pfn + folio_nr_pages(folio); @@ -2333,9 +2392,9 @@ migrate_longterm_unpinnable_folios(struct list_head *movable_folio_list, * convert the pin on the source folio to a normal * reference. */ - pofs_clear_entry(pofs, i); folio_get(folio); - gup_put_folio(folio, 1, FOLL_PIN); + pofs_unpin_entry(pofs, i); + pofs_clear_entry(pofs, i); if (migrate_device_coherent_folio(folio)) { ret = -EBUSY; @@ -2352,7 +2411,7 @@ migrate_longterm_unpinnable_folios(struct list_head *movable_folio_list, * calling folio_isolate_lru() which takes a reference so the * folio won't be freed if it's migrating. */ - unpin_folio(folio); + pofs_unpin_entry(pofs, i); pofs_clear_entry(pofs, i); } @@ -2458,6 +2517,23 @@ static long check_and_migrate_movable_folios(unsigned long nr_folios, } #endif /* CONFIG_MIGRATION */ +#ifdef CONFIG_MM_SUBPAGE +static long check_and_migrate_movable_fragments(unsigned long count, + struct user_page_fragment *fragments) +{ +#ifdef CONFIG_MIGRATION + struct pages_or_folios pofs = { + .fragments = fragments, + .nr_entries = count, + }; + + return check_and_migrate_movable_pages_or_folios(&pofs); +#else + return 0; +#endif +} +#endif + /* * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which * allows us to process the FOLL_LONGTERM flag. @@ -3133,6 +3209,11 @@ static unsigned long gup_fast(unsigned long start, unsigned long end, int nr_pinned = 0; unsigned seq; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return 0; +#endif + if (!IS_ENABLED(CONFIG_HAVE_GUP_FAST) || !gup_fast_permitted(start, end)) return 0; @@ -3355,6 +3436,464 @@ long pin_user_pages_remote(struct mm_struct *mm, } EXPORT_SYMBOL(pin_user_pages_remote); +#ifdef CONFIG_MM_SUBPAGE +/* 1: acquired, 0: fault required, -EMLINK: read pin needs private backing. */ +static int follow_subpage_fragment(struct vm_area_struct *vma, unsigned long addr, + unsigned int flags, struct user_page_fragment *fragment) +{ + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slot = NULL; + struct folio *folio; + struct page *page; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *ptep, pte; + spinlock_t *ptl; + phys_addr_t phys; + int ret = 0; + bool file_ref = false; + + if (pgd_none(*pgd) || pgd_bad(*pgd)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_mm(mm, *p4d) || p4d_bad_mm(mm, *p4d)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + pud = pud_offset_mm(mm, p4d, addr); + if (pud_none(*pud)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + if (pud_leaf(*pud) || pud_bad(*pud)) + return -EOPNOTSUPP; + pmd = pmd_offset_mm(mm, pud, addr); + if (pmd_none(*pmd)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + if (pmd_leaf(*pmd) || pmd_bad(*pmd)) + return -EOPNOTSUPP; + ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!ptep) + return 0; + pte = ptep_get(ptep); + if (pte_none(pte)) { + ret = PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + goto out; + } + if (!pte_present(pte) || (pte_protnone(pte) && !gup_can_follow_protnone(vma, flags))) + goto out; + phys = pte_phys_mm(mm, pte); + page = vm_normal_page(vma, addr, pte); + if (mm_page_size(mm) > PAGE_SIZE) { + unsigned long offset = (addr & (mm_page_size(mm) - 1)) & PAGE_MASK; + + phys += offset; + if (page) + page += offset >> PAGE_SHIFT; + } + if (!page) { + if (flags & FOLL_DUMP) { + ret = -EFAULT; + goto out; + } + if (!is_zero_pfn(pte_pfn(pte))) { + ret = -EOPNOTSUPP; + goto out; + } + if (flags & FOLL_WRITE) + goto out; + folio = page_folio(pfn_to_page(PHYS_PFN(phys))); + if (flags & FOLL_GET) + folio_get(folio); + } else { + folio = page_folio(page); + /* Coarse leaves use ordinary native-page ownership, not slots. */ + if (mm_page_size(mm) > PAGE_SIZE) { + if ((flags & FOLL_WRITE) && + !can_follow_write_pte(pte, page, vma, flags)) + goto out; + if (!pte_write(pte) && gup_must_unshare(vma, flags, page)) { + ret = -EMLINK; + goto out; + } + ret = try_grab_folio(folio, 1, flags); + if (ret) + goto out; + file_ref = true; + goto acquired; + } + if (!folio_test_anon(folio)) { + if (vma->vm_file && folio_mapping(folio) != vma->vm_file->f_mapping) { + ret = -EOPNOTSUPP; + goto out; + } + if ((flags & FOLL_WRITE) && !pte_write(pte)) + goto out; + /* Private non-anonymous read pins must first acquire COW ownership. */ + if ((flags & FOLL_PIN) && is_cow_mapping(vma->vm_flags)) { + ret = -EMLINK; + goto out; + } + ret = try_grab_folio(folio, 1, flags); + if (ret) + goto out; + file_ref = true; + goto acquired; + } + if (folio_test_ksm(folio)) { + ret = -EOPNOTSUPP; + goto out; + } + slot = mm_subpage_get_from_phys(phys); + if (!slot) { + ret = -EFAULT; + goto out; + } + if ((flags & FOLL_WRITE) && !pte_write(pte)) { + bool force = (flags & FOLL_FORCE) && + !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE | VM_WRITE)) && + (vma->vm_flags & VM_MAYWRITE) && mm_subpage_is_exclusive(slot) && + !pte_needs_soft_dirty_wp(vma, pte) && !userfaultfd_pte_wp(vma, pte); + + if (!force) + goto out; + } + if ((flags & FOLL_PIN) && !mm_subpage_is_exclusive(slot)) { + ret = flags & FOLL_WRITE ? 0 : -EMLINK; + goto out; + } + if (flags & FOLL_PIN) { + ret = mm_subpage_pin(slot); + if (ret) { + if (ret == -EAGAIN) + ret = 0; + goto out; + } + } + } +acquired: + if (slot && (flags & FOLL_WRITE)) + mm_subpage_cancel_lazyfree(slot); + ret = flags & FOLL_PIN ? arch_make_folio_accessible(folio) : 0; + if (ret) { + if (slot) + mm_subpage_unpin(slot); + else if (file_ref) + gup_put_folio(folio, 1, flags); + goto out; + } + if ((flags & FOLL_TOUCH) && (flags & FOLL_WRITE) && + !pte_dirty(pte) && !folio_test_dirty(folio)) + folio_mark_dirty(folio); + folio_mark_accessed(folio); + flush_anon_page(vma, &folio->page, addr); + flush_dcache_folio(folio); + fragment->folio = folio; + fragment->subpage = slot; + fragment->pinned = flags & FOLL_PIN; + if (flags & FOLL_GET) + slot = NULL; /* The returned record consumes the lookup reference. */ + fragment->offset = phys - PFN_PHYS(folio_pfn(folio)) + + (addr & (mm_user_fragment_size(mm) - 1)); + ret = 1; +out: + if (slot) + mm_subpage_put(slot); /* Lookup reference; success retains a separate pin. */ + pte_unmap_unlock(ptep, ptl); + return ret; +} + +/* No references escape: only ensure each user leaf can be followed. */ +static long prefault_subpage_range(struct mm_struct *mm, unsigned long start, + unsigned long nr_pages, unsigned int flags, int *locked) +{ + unsigned long done = 0; + long ret = 0; + + mmap_assert_locked(mm); + start = untagged_addr_remote(mm, start); + while (done < nr_pages) { + struct user_page_fragment unused; + struct vm_area_struct *vma = vma_lookup(mm, start); + + if (!vma) { + ret = flags & FOLL_MADV_POPULATE ? -ENOMEM : -EFAULT; + break; + } + ret = check_vma_flags(vma, flags); + if (ret) { + if (flags & FOLL_MADV_POPULATE) + ret = -EINVAL; + break; + } + if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { + ret = flags & FOLL_MADV_POPULATE ? -EINVAL : -EOPNOTSUPP; + break; + } + if (gup_signal_pending(flags)) { + ret = -EINTR; + break; + } + cond_resched(); + ret = follow_subpage_fragment(vma, start, flags, &unused); + if (!ret) { + ret = faultin_page(vma, start, flags, false, locked); + if (!ret) + continue; + if (ret == -EBUSY || ret == -EAGAIN) + ret = 0; /* Preserve GUP's caller-managed retry contract. */ + break; + } + if (ret < 0) + break; + done++; + start += mm_page_size(mm); + } + if (ret == -EOPNOTSUPP && (flags & FOLL_MADV_POPULATE)) + ret = -EINVAL; + return done ? (long)done : ret; +} + +static long user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity, bool pin, + struct vm_area_struct **vma_out, bool dump) +{ + const unsigned int allowed = FOLL_WRITE | FOLL_FORCE | FOLL_NOFAULT | + FOLL_NOWAIT | FOLL_INTERRUPTIBLE | FOLL_LONGTERM | FOLL_ANON; + unsigned int internal = flags | FOLL_TOUCH | FOLL_REMOTE | + (pin ? FOLL_PIN : FOLL_GET) | (dump ? FOLL_DUMP : 0); + unsigned long end, count = 0; + int locked = 0; + long ret; + + if (!length) + return 0; + if ((flags & FOLL_LONGTERM) && !pin) + return -EINVAL; + if (!fragments || !capacity || capacity > LONG_MAX || (flags & ~allowed)) + return -EINVAL; + if (vma_out) { + mmap_assert_locked(mm); + *vma_out = NULL; + } else { + internal |= FOLL_UNLOCKABLE; + ret = mmap_read_lock_killable(mm); + if (ret) + return ret; + } + locked = 1; + start = untagged_addr_remote(mm, start); + if (check_add_overflow(start, length, &end)) { + ret = -EFAULT; + goto out; + } + if (pin) + mm_set_has_pinned_flag(mm); + while (start < end && count < capacity) { + unsigned long bytes = min(end - start, + mm_user_fragment_size(mm) - (start & (mm_user_fragment_size(mm) - 1))); + struct vm_area_struct *vma; + + if (!locked) { + ret = mmap_read_lock_killable(mm); + if (ret) + break; + locked = 1; + } + if (gup_signal_pending(internal)) { + ret = -EINTR; + break; + } + cond_resched(); + if (mm_page_size(mm) == PAGE_SIZE) { + struct page *page; + + if (dump) + ret = __get_user_pages_locked(mm, start, 1, &page, &locked, internal); + else if (pin) + ret = pin_user_pages_remote(mm, start, 1, flags, &page, + vma_out ? NULL : &locked); + else + ret = get_user_pages_remote(mm, start, 1, flags, &page, + vma_out ? NULL : &locked); + if (ret != 1) { + if (!ret) + ret = -EAGAIN; + break; + } + fragments[count].folio = page_folio(page); + fragments[count].subpage = NULL; + fragments[count].pinned = pin; + fragments[count].offset = folio_page_idx(page_folio(page), page) * PAGE_SIZE + + offset_in_page(start); + } else { + vma = gup_vma_lookup(mm, start); + ret = vma ? check_vma_flags(vma, internal) : -EFAULT; + if (ret) + break; + /* Huge and special/device mappings need separate integration. */ + if (is_vm_hugetlb_page(vma) || + (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP | VM_IO))) { + ret = -EOPNOTSUPP; + break; + } + ret = follow_subpage_fragment(vma, start, internal, &fragments[count]); + if (!ret || ret == -EMLINK) { + ret = faultin_page(vma, start, internal, ret == -EMLINK, &locked); + if (!ret) + continue; + if (!locked && (ret == -EBUSY || ret == -EAGAIN)) { + internal |= FOLL_TRIED; + continue; + } + if (ret == -EBUSY) + ret = -EAGAIN; + break; + } + if (ret < 0) + break; + } + if (vma_out) + *vma_out = vma_lookup(mm, start); + fragments[count++].length = bytes; + start += bytes; + internal &= ~FOLL_TRIED; + } +out: + if (locked && !vma_out) + mmap_read_unlock(mm); + return count ? (long)count : ret; +} + +long pin_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity) +{ + unsigned int saved; + long ret, count; + + if (!(flags & FOLL_LONGTERM) || mm_page_size(mm) == PAGE_SIZE) + return user_fragments_remote(mm, start, length, flags, fragments, + capacity, true, NULL, false); + /* Match native long-term GUP: fault outside movable zones, migrate any + * existing unpinnable backing, then retry with fresh typed references. + */ + saved = memalloc_pin_save(); + do { + count = user_fragments_remote(mm, start, length, flags, fragments, + capacity, true, NULL, false); + if (count <= 0) { + ret = count; + break; + } + ret = check_and_migrate_movable_fragments(count, fragments); + } while (ret == -EAGAIN); + memalloc_pin_restore(saved); + return ret ? ret : count; +} +EXPORT_SYMBOL_GPL(pin_user_fragments_remote); + +long get_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity) +{ + return user_fragments_remote(mm, start, length, flags, fragments, capacity, + false, NULL, false); +} +EXPORT_SYMBOL_GPL(get_user_fragments_remote); + +int get_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma) +{ + int ret; + + /* As with get_user_page_vma_remote(), preserve the caller's VMA lock. */ + if (!vma) + return -EINVAL; + *vma = NULL; + if (flags & FOLL_NOWAIT) + return -EINVAL; + ret = user_fragments_remote(mm, start, length, flags, fragment, 1, false, vma, false); + if (ret > 0 && !*vma) { + release_user_fragments(fragment, 1, false); + return -EFAULT; + } + return ret; +} + +int pin_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma) +{ + int ret; + + if (!vma) + return -EINVAL; + *vma = NULL; + if (flags & (FOLL_NOWAIT | FOLL_LONGTERM)) + return -EINVAL; + ret = user_fragments_remote(mm, start, length, flags, fragment, 1, true, vma, false); + if (ret > 0 && !*vma) { + release_user_fragments(fragment, 1, false); + return -EFAULT; + } + return ret; +} + +#ifdef CONFIG_ELF_CORE +/* Like get_dump_page(), but preserves the selected user-page ownership. */ +int get_dump_fragment(unsigned long addr, struct user_page_fragment *fragment) +{ + return user_fragments_remote(current->mm, addr, mm_user_fragment_size(current->mm), + FOLL_FORCE, fragment, 1, false, NULL, true); +} +#endif + +void mark_user_fragment_dirty(struct user_page_fragment *fragment) +{ + struct folio *folio = fragment->folio; + + if (!folio) + return; + if (!is_zero_folio(folio) && !folio_test_dirty(folio)) { + folio_lock(folio); + folio_mark_dirty(folio); + folio_unlock(folio); + } + if (fragment->subpage) + mm_subpage_cancel_lazyfree(fragment->subpage); +} +EXPORT_SYMBOL_GPL(mark_user_fragment_dirty); + +void release_user_fragments(struct user_page_fragment *fragments, + unsigned long count, bool dirty) +{ + unsigned long i; + + for (i = 0; i < count; i++) { + struct user_page_fragment *fragment = &fragments[i]; + struct folio *folio = fragment->folio; + + if (!folio) + continue; + if (dirty) + mark_user_fragment_dirty(fragment); + if (fragment->subpage) { + if (fragment->pinned) + mm_subpage_unpin(fragment->subpage); + else + mm_subpage_put(fragment->subpage); + } else if (fragment->pinned) { + unpin_folio(folio); + } else { + folio_put(folio); + } + *fragment = (struct user_page_fragment) {}; + } +} +EXPORT_SYMBOL_GPL(release_user_fragments); +#endif + /** * pin_user_pages() - pin user pages in memory for use by other devices * @@ -3407,6 +3946,25 @@ long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, } EXPORT_SYMBOL(pin_user_pages_unlocked); +/* Direct cache pins have no PTE fault to publish fine shmem exposure. */ +static int memfd_pin_folio(struct file *memfd, struct folio *folio) +{ + bool lock = IS_ENABLED(CONFIG_MM_SUBPAGE) && + IS_ENABLED(CONFIG_USERFAULTFD) && shmem_file(memfd); + int ret; + + /* Serialize both acquisition and exposure against partial hole punch. */ + if (lock) + folio_lock(folio); + ret = try_grab_folio(folio, 1, FOLL_PIN); + if (lock) { + if (!ret) + shmem_uffd_mark_mapped(folio, 0, folio_size(folio)); + folio_unlock(folio); + } + return ret; +} + /** * memfd_pin_folios() - pin folios associated with a memfd * @memfd: the memfd whose folios are to be pinned @@ -3492,7 +4050,7 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, for (i = 0; i < nr_found; i++) { folio = fbatch.folios[i]; - if (try_grab_folio(folio, 1, FOLL_PIN)) { + if (memfd_pin_folio(memfd, folio)) { folio_batch_release(&fbatch); ret = -EINVAL; goto err; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index dd93f1b0c44a2..7ab369477fcc7 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2715,7 +2715,8 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, goto unlock; if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma, - vma_is_single_threaded_private(vma))) + vma_is_single_threaded_private(vma), + folio_maybe_mapped_shared(pmd_folio(*pmd)))) goto unlock; } /* @@ -4118,6 +4119,12 @@ static int __folio_split(struct folio *folio, unsigned int new_order, } anon_vma_lock_write(anon_vma); mapping = NULL; +#ifdef CONFIG_MM_SUBPAGE + if (new_order < anon_vma->min_folio_order) { + ret = -EBUSY; + goto out_unlock; + } +#endif } else { unsigned int min_order; gfp_t gfp; @@ -4394,7 +4401,7 @@ int folio_split(struct folio *folio, unsigned int new_order, unsigned int min_order_for_split(struct folio *folio) { if (folio_test_anon(folio)) - return 0; + return folio_anon_min_order(folio); /* * If the folio got truncated, we don't know the previous mapping and diff --git a/mm/internal.h b/mm/internal.h index 5a2ddcf68e0b6..5ad4e57daedc6 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -305,6 +305,50 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) return pte_mkold(pte); } +#ifdef CONFIG_MM_SUBPAGE +/* Batch contiguous user leaves without confusing their offsets with PFNs. */ +static inline unsigned int folio_subpage_pte_batch_flags(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t *ptentp, unsigned int max_nr, fpb_t flags) +{ + struct mm_struct *mm = vma->vm_mm; + pte_t pte = *ptentp; + phys_addr_t offset = pte_phys_mm(mm, pte) - PFN_PHYS(folio_pfn(folio)); + pte_t expected; + unsigned int nr; + + VM_BUG_ON_FOLIO(offset >= folio_size(folio), folio); + max_nr = min_t(unsigned long, max_nr, (folio_size(folio) - offset) >> mm_page_shift(mm)); + VM_BUG_ON_FOLIO(!max_nr, folio); + nr = min(max_nr, pte_batch_hint(ptep, pte)); + expected = pte_advance_pfn_mm(mm, __pte_batch_clear_ignored(pte, flags), nr); + while (nr < max_nr) { + unsigned int batch; + + pte = ptep_get(ptep + nr); + if (!pte_same(__pte_batch_clear_ignored(pte, flags), expected)) + break; + if ((flags & FPB_MERGE_WRITE) && pte_write(pte)) + *ptentp = pte_mkwrite(*ptentp, vma); + if (flags & FPB_MERGE_YOUNG_DIRTY) { + if (pte_young(pte)) + *ptentp = pte_mkyoung(*ptentp); + if (pte_dirty(pte)) + *ptentp = pte_mkdirty(*ptentp); + } + batch = min(max_nr - nr, pte_batch_hint(ptep + nr, pte)); + nr += batch; + expected = pte_advance_pfn_mm(mm, expected, batch); + } + return nr; +} + +static inline unsigned int folio_subpage_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t pte, unsigned int max_nr, fpb_t flags) +{ + return folio_subpage_pte_batch_flags(vma, folio, ptep, &pte, max_nr, flags); +} +#endif + /** * folio_pte_batch_flags - detect a PTE batch for a large folio * @folio: The large folio to detect a PTE batch for. @@ -416,6 +460,10 @@ static inline pte_t pte_move_swp_offset(pte_t pte, long delta) if (pte_swp_uffd_wp(pte)) new = pte_swp_mkuffd_wp(new); +#ifdef CONFIG_MM_SUBPAGE + new = pte_swp_set_subpage_offset(new, pte_swp_subpage_offset(pte)); +#endif + return new; } @@ -1158,6 +1206,11 @@ folio_within_range(struct folio *folio, struct vm_area_struct *vma, end = vma->vm_end; pgoff = folio_pgoff(folio); + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + addr = vma_address_at_offset(vma, (struct vm_page_offset) { .index = pgoff }); + return !IS_ERR_VALUE(addr) && addr >= start && addr <= end && + end - addr >= folio_size(folio); + } /* if folio start address is not in vma range */ if (!in_range(pgoff, vma->vm_pgoff, vma_pglen)) @@ -1240,6 +1293,8 @@ static inline unsigned long vma_address(const struct vm_area_struct *vma, if (pgoff >= vma->vm_pgoff) { address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + if (pgoff != vma->vm_pgoff) + address -= vma_subpage_offset(vma); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address >= vma->vm_end) address = -EFAULT; @@ -1262,15 +1317,24 @@ static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) pgoff_t pgoff; unsigned long address; +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) + return min(pvmw->address + mm_page_size(vma->vm_mm), vma->vm_end); +#endif + /* Common case, plus ->pgoff is invalid for KSM */ - if (pvmw->nr_pages == 1) - return pvmw->address + PAGE_SIZE; + if (pvmw->nr_pages == 1 && mm_page_size(vma->vm_mm) >= PAGE_SIZE) + return min((pvmw->address & mm_page_mask(vma->vm_mm)) + + mm_page_size(vma->vm_mm), vma->vm_end); pgoff = pvmw->pgoff + pvmw->nr_pages; address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + address -= vma_subpage_offset(vma); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address > vma->vm_end) address = vma->vm_end; + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + address = min(mm_page_align(vma->vm_mm, address), vma->vm_end); return address; } @@ -1568,11 +1632,11 @@ static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma) #ifdef CONFIG_NUMA_BALANCING bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, - bool is_private_single_threaded); + bool is_private_single_threaded, bool shared); #else static inline bool folio_can_map_prot_numa(struct folio *folio, - struct vm_area_struct *vma, bool is_private_single_threaded) + struct vm_area_struct *vma, bool is_private_single_threaded, bool shared) { return false; } @@ -1900,7 +1964,7 @@ static inline bool clear_flush_young_ptes_notify(struct vm_area_struct *vma, young = clear_flush_young_ptes(vma, addr, ptep, nr); young |= mmu_notifier_clear_flush_young(vma->vm_mm, addr, - addr + nr * PAGE_SIZE); + addr + nr * mm_page_size(vma->vm_mm)); return young; } @@ -1920,7 +1984,8 @@ static inline bool test_and_clear_young_ptes_notify(struct vm_area_struct *vma, bool young; young = test_and_clear_young_ptes(vma, addr, ptep, nr); - young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + nr * PAGE_SIZE); + young |= mmu_notifier_clear_young(vma->vm_mm, addr, + addr + nr * mm_page_size(vma->vm_mm)); return young; } @@ -1952,4 +2017,14 @@ static inline int get_sysctl_max_map_count(void) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages); +#ifdef CONFIG_MM_SUBPAGE +struct iov_iter; +int process_vm_rw_fragments(struct mm_struct *mm, unsigned long addr, + unsigned long len, struct iov_iter *iter, bool write); +#endif + +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SWAP) +vm_fault_t do_swap_coarse_page(struct vm_fault *vmf); +#endif + #endif /* __MM_INTERNAL_H */ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index d6e04041f5dc1..ad4832f64fc30 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1066,28 +1066,31 @@ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm, } static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_struct *mm, - struct collapse_control *cc) + struct collapse_control *cc, unsigned int order) { gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() : GFP_TRANSHUGE); int node = collapse_find_target_node(cc); struct folio *folio; - folio = __folio_alloc(gfp, HPAGE_PMD_ORDER, node, &cc->alloc_nmask); + folio = __folio_alloc(gfp, order, node, &cc->alloc_nmask); if (!folio) { *foliop = NULL; - count_vm_event(THP_COLLAPSE_ALLOC_FAILED); + if (is_pmd_order(order)) + count_vm_event(THP_COLLAPSE_ALLOC_FAILED); return SCAN_ALLOC_HUGE_PAGE_FAIL; } - count_vm_event(THP_COLLAPSE_ALLOC); + if (is_pmd_order(order)) + count_vm_event(THP_COLLAPSE_ALLOC); if (unlikely(mem_cgroup_charge(folio, mm, gfp))) { folio_put(folio); *foliop = NULL; return SCAN_CGROUP_CHARGE_FAIL; } - count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1); + if (is_pmd_order(order)) + count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1); *foliop = folio; return SCAN_SUCCEED; @@ -1116,7 +1119,7 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a */ mmap_read_unlock(mm); - result = alloc_charge_folio(&folio, mm, cc); + result = alloc_charge_folio(&folio, mm, cc, HPAGE_PMD_ORDER); if (result != SCAN_SUCCEED) goto out_nolock; @@ -1880,22 +1883,33 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) * + unlock and free huge page; */ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, - struct file *file, pgoff_t start, struct collapse_control *cc) + struct file *file, pgoff_t start, struct collapse_control *cc, + unsigned int order) { struct address_space *mapping = file->f_mapping; struct page *dst; struct folio *folio, *tmp, *new_folio; - pgoff_t index = 0, end = start + HPAGE_PMD_NR; + unsigned long nr_pages = 1UL << order; + pgoff_t index = 0, end = start + nr_pages; LIST_HEAD(pagelist); - XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); + XA_STATE_ORDER(xas, &mapping->i_pages, start, order); enum scan_result result = SCAN_SUCCEED; int nr_none = 0; bool is_shmem = shmem_file(file); VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); - VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); + VM_BUG_ON(start & (nr_pages - 1)); - result = alloc_charge_folio(&new_folio, mm, cc); + if (is_pmd_order(order)) { + result = alloc_charge_folio(&new_folio, mm, cc, order); + } else { + /* Sub-PMD user leaves must respect the shmem inode's allocation policy. */ + new_folio = shmem_alloc_collapse_folio(mm, mapping->host, start, order); + if (IS_ERR(new_folio)) { + new_folio = NULL; + result = SCAN_ALLOC_HUGE_PAGE_FAIL; + } + } if (result != SCAN_SUCCEED) goto out; @@ -2024,7 +2038,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * we locked the first folio, then a THP might be there already. * This will be discovered on the first iteration. */ - if (is_pmd_order(folio_order(folio))) { + if (folio_order(folio) >= order) { result = SCAN_PTE_MAPPED_HUGEPAGE; goto out_unlock; } @@ -2149,6 +2163,8 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, while (index < folio->index) { clear_highpage(dst); + if (is_shmem) + shmem_uffd_copy_page(dst, NULL); index++; dst++; } @@ -2158,12 +2174,16 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, result = SCAN_COPY_MC; goto rollback; } + if (is_shmem) + shmem_uffd_copy_page(dst, folio_page(folio, i)); index++; dst++; } } while (index < end) { clear_highpage(dst); + if (is_shmem) + shmem_uffd_copy_page(dst, NULL); index++; dst++; } @@ -2205,8 +2225,13 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * not be able to observe any missing pages due to the * previously inserted retry entries. */ + /* + * A required user leaf populates its whole granule, as a native + * fault does. Only optional PMD collapse must preserve cache holes + * merely because another VMA has missing-page registration. + */ vma_interval_tree_foreach(vma, &mapping->i_mmap, start, end) { - if (userfaultfd_missing(vma)) { + if (is_pmd_order(order) && userfaultfd_missing(vma)) { result = SCAN_EXCEED_NONE_PTE; goto immap_locked; } @@ -2229,12 +2254,13 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, } if (is_shmem) { - lruvec_stat_mod_folio(new_folio, NR_SHMEM, HPAGE_PMD_NR); - lruvec_stat_mod_folio(new_folio, NR_SHMEM_THPS, HPAGE_PMD_NR); - } else { - lruvec_stat_mod_folio(new_folio, NR_FILE_THPS, HPAGE_PMD_NR); + lruvec_stat_mod_folio(new_folio, NR_SHMEM, nr_pages); + if (is_pmd_order(order)) + lruvec_stat_mod_folio(new_folio, NR_SHMEM_THPS, nr_pages); + } else if (is_pmd_order(order)) { + lruvec_stat_mod_folio(new_folio, NR_FILE_THPS, nr_pages); } - lruvec_stat_mod_folio(new_folio, NR_FILE_PAGES, HPAGE_PMD_NR); + lruvec_stat_mod_folio(new_folio, NR_FILE_PAGES, nr_pages); /* * Mark new_folio as uptodate before inserting it into the @@ -2242,14 +2268,14 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * unwritten page. */ folio_mark_uptodate(new_folio); - folio_ref_add(new_folio, HPAGE_PMD_NR - 1); + folio_ref_add(new_folio, nr_pages - 1); if (is_shmem) folio_mark_dirty(new_folio); folio_add_lru(new_folio); /* Join all the small entries into a single multi-index entry. */ - xas_set_order(&xas, start, HPAGE_PMD_ORDER); + xas_set_order(&xas, start, order); xas_store(&xas, new_folio); WARN_ON_ONCE(xas_error(&xas)); xas_unlock_irq(&xas); @@ -2258,9 +2284,11 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * Remove pte page tables, so we can re-fault the page as huge. * If MADV_COLLAPSE, adjust result to call try_collapse_pte_mapped_thp(). */ - retract_page_tables(mapping, start); - if (cc && !cc->is_khugepaged) - result = SCAN_PTE_MAPPED_HUGEPAGE; + if (is_pmd_order(order)) { + retract_page_tables(mapping, start); + if (cc && !cc->is_khugepaged) + result = SCAN_PTE_MAPPED_HUGEPAGE; + } folio_unlock(new_folio); /* @@ -2317,7 +2345,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, folio_put(new_folio); out: VM_BUG_ON(!list_empty(&pagelist)); - trace_mm_khugepaged_collapse_file(mm, new_folio, index, addr, is_shmem, file, HPAGE_PMD_NR, result); + trace_mm_khugepaged_collapse_file(mm, new_folio, index, addr, is_shmem, file, nr_pages, result); return result; } @@ -2420,7 +2448,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, result = SCAN_EXCEED_NONE_PTE; count_vm_event(THP_SCAN_EXCEED_NONE_PTE); } else { - result = collapse_file(mm, addr, file, start, cc); + result = collapse_file(mm, addr, file, start, cc, HPAGE_PMD_ORDER); } } @@ -2831,6 +2859,47 @@ static int madvise_collapse_errno(enum scan_result r) } } +/* + * Promote the cache backing for one larger-than-native user leaf. The caller + * owns mm/file references; no folio or page-table lock may be held. Start is + * a native file index. This sub-PMD shmem path may run under the fault lock: + * it neither retracts page tables (which requires mmap write locking) nor + * invokes filesystem writeback. Source folios are trylocked and rolled back + * on contention. Ordinary PMD collapse retains its existing lock policy. + */ +int collapse_file_user_page(struct mm_struct *mm, struct file *file, pgoff_t start) +{ + struct collapse_control *cc; + unsigned int order; + enum scan_result result; + + if (mm_page_shift(mm) <= PAGE_SHIFT || !shmem_file(file)) + return -EOPNOTSUPP; + order = mm_page_shift(mm) - PAGE_SHIFT; + /* Shmem's order limit reflects THP policy, not an architectural leaf limit. */ + if (order >= HPAGE_PMD_ORDER || order > MAX_PAGECACHE_ORDER || + !IS_ALIGNED(start, 1UL << order) || start > ULONG_MAX - (1UL << order)) + return -EINVAL; + cc = kzalloc(sizeof(*cc), GFP_KERNEL); + if (!cc) + return -ENOMEM; + lru_add_drain_all(); + result = collapse_file(mm, 0, file, start, cc, order); + kfree(cc); + if (result == SCAN_SUCCEED || result == SCAN_PTE_MAPPED_HUGEPAGE) + return 0; + /* Retained references cannot be displaced by replacing their cache folio. */ + if (result == SCAN_PAGE_COUNT) + return -EBUSY; + if (result == SCAN_CGROUP_CHARGE_FAIL) + return -ENOMEM; + if (result == SCAN_COPY_MC) + return -EHWPOISON; + if (result == SCAN_TRUNCATED) + return -EAGAIN; + return madvise_collapse_errno(result); +} + int madvise_collapse(struct vm_area_struct *vma, unsigned long start, unsigned long end, bool *lock_dropped) { diff --git a/mm/ksm.c b/mm/ksm.c index 7d5b76478f0b6..cd0398e972941 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -804,6 +804,10 @@ static struct page *get_mergeable_page(struct ksm_rmap_item *rmap_item) struct folio_walk fw; struct folio *folio; + /* The KSM comparison and replacement paths still require native leaves. */ + if (mm_page_size(mm) != PAGE_SIZE) + return NULL; + mmap_read_lock(mm); vma = find_mergeable_vma(mm, addr); if (!vma) @@ -1338,7 +1342,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct folio *folio, /* See folio_try_share_anon_rmap_pte(): clear PTE first. */ if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, &folio->page)) { + folio_try_share_anon_rmap_pte(folio, &folio->page, vma)) { set_pte_at(mm, pvmw.address, pvmw.pte, entry); goto out_unlock; } @@ -3206,7 +3210,8 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc) if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, addr, rwc->arg)) { + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = addr }, rwc->arg)) { anon_vma_unlock_read(anon_vma); return; } diff --git a/mm/madvise.c b/mm/madvise.c index 5665785e5ac38..1d15da9d1200d 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -193,7 +194,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, spinlock_t *ptl; unsigned long addr; - for (addr = start; addr < end; addr += PAGE_SIZE) { + for (addr = start; addr < end; addr += mm_page_size(vma->vm_mm)) { pte_t pte; softleaf_t entry; struct folio *folio; @@ -236,7 +237,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma, struct address_space *mapping) { XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start)); - pgoff_t end_index = linear_page_index(vma, end) - 1; + pgoff_t end_index = linear_page_index(vma, end - 1); struct folio *folio; struct swap_iocb *splug = NULL; @@ -252,8 +253,11 @@ static void shmem_swapin_range(struct vm_area_struct *vma, if (!softleaf_is_swap(entry)) continue; - addr = vma->vm_start + - ((xas.xa_index - vma->vm_pgoff) << PAGE_SHIFT); + /* The first cache page may start before this VMA's byte offset. */ + addr = xas.xa_index == linear_page_index(vma, start) ? start : + vma_address_at_offset(vma, (struct vm_page_offset) { + .index = xas.xa_index, + }); xas_pause(&xas); rcu_read_unlock(); @@ -318,7 +322,7 @@ static long madvise_willneed(struct madvise_behavior *madv_behavior) mark_mmap_lock_dropped(madv_behavior); get_file(file); offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); mmap_read_unlock(mm); vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED); fput(file); @@ -340,12 +344,18 @@ static inline bool can_do_file_pageout(struct vm_area_struct *vma) file_permission(vma->vm_file, MAY_WRITE) == 0; } -static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end, +static inline int madvise_folio_pte_batch(struct vm_area_struct *vma, + unsigned long addr, unsigned long end, struct folio *folio, pte_t *ptep, pte_t *ptentp) { - int max_nr = (end - addr) / PAGE_SIZE; + int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch_flags(vma, folio, ptep, ptentp, + max_nr, FPB_MERGE_YOUNG_DIRTY); +#endif return folio_pte_batch_flags(folio, NULL, ptep, ptentp, max_nr, FPB_MERGE_YOUNG_DIRTY); } @@ -376,7 +386,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, #ifdef CONFIG_TRANSPARENT_HUGEPAGE if (pmd_trans_huge(*pmd)) { pmd_t orig_pmd; - unsigned long next = pmd_addr_end(addr, end); + unsigned long next = pmd_addr_end_mm(mm, addr, end); tlb_change_page_size(tlb, HPAGE_PMD_SIZE); ptl = pmd_trans_huge_lock(pmd, vma); @@ -446,14 +456,14 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, regular_folio: #endif - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, mm_page_size(mm)); restart: start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); if (!start_pte) return 0; flush_tlb_batched_pending(mm); lazy_mmu_mode_enable(); - for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) { + for (; addr < end; pte += nr, addr += nr * mm_page_size(mm)) { nr = 1; ptent = ptep_get(pte); @@ -484,9 +494,10 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, * fail to split a folio, leave it in place and advance to the * next pte in the range. */ - if (folio_test_large(folio)) { - nr = madvise_folio_pte_batch(addr, end, folio, pte, &ptent); - if (nr < folio_nr_pages(folio)) { + if (folio_test_large(folio) || mm_page_size(mm) != PAGE_SIZE) { + nr = madvise_folio_pte_batch(vma, addr, end, folio, pte, &ptent); + if (folio_test_large(folio) && + nr < (folio_size(folio) >> mm_page_shift(mm))) { int err; if (folio_maybe_mapped_shared(folio)) @@ -521,7 +532,8 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, * number of pages, it must be exclusive. */ if (!folio_test_lru(folio) || - folio_mapcount(folio) != folio_nr_pages(folio)) + folio_mapcount(folio) != + (mm_page_size(mm) != PAGE_SIZE ? nr : folio_nr_pages(folio))) continue; if (pageout_anon_only_filter && !folio_test_anon(folio)) @@ -649,6 +661,72 @@ static long madvise_pageout(struct madvise_behavior *madv_behavior) return 0; } +#ifdef CONFIG_MM_SUBPAGE +static int madvise_free_subpage_range(pmd_t *pmd, unsigned long addr, + unsigned long end, struct mm_walk *walk) +{ + struct vm_area_struct *vma = walk->vma; + struct mm_struct *mm = vma->vm_mm; + struct mmu_gather *tlb = walk->private; + spinlock_t *ptl; + pte_t *base, *ptep; + long swapped = 0; + + base = ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!base) + return 0; + flush_tlb_batched_pending(mm); + for (; addr < end; addr += mm_page_size(mm), ptep++) { + pte_t pte = ptep_get(ptep); + struct mm_subpage *slot; + struct folio *folio; + struct page *page; + + if (!pte_present(pte)) { + softleaf_t entry = softleaf_from_pte(pte); + + if (softleaf_is_swap(entry)) { + swap_subpage_put(entry, + pte_swp_subpage_offset(pte), true); + pte_clear_not_present_full(mm, addr, ptep, tlb->fullmm); + swapped--; + } + continue; + } + page = vm_normal_page(vma, addr, pte); + if (!page) + continue; + folio = page_folio(page); + if (!folio_test_anon(folio) || !folio_trylock(folio)) + continue; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (!slot) + goto unlock; + if (folio_test_swapcache(folio) && !folio_free_swap(folio)) + goto put; + if (!mm_subpage_mark_lazyfree(slot)) + goto put; + clear_young_dirty_ptes(vma, addr, ptep, 1, + CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY); + tlb_remove_tlb_entry(tlb, ptep, addr); + /* Partial hints must never clear a neighbor's dirty state. */ + if (mm_subpage_all_lazyfree(slot)) { + folio_clear_dirty(folio); + folio_mark_lazyfree(folio); + } +put: + mm_subpage_put(slot); +unlock: + folio_unlock(folio); + } + if (swapped) + add_mm_counter(mm, MM_SWAPENTS, swapped); + pte_unmap_unlock(base, ptl); + cond_resched(); + return 0; +} +#endif + static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct mm_walk *walk) @@ -664,6 +742,10 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, unsigned long next; int nr, max_nr; + #ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) + return madvise_free_subpage_range(pmd, addr, end, walk); +#endif next = pmd_addr_end(addr, end); if (pmd_trans_huge(*pmd)) if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next)) @@ -714,7 +796,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, * next pte in the range. */ if (folio_test_large(folio)) { - nr = madvise_folio_pte_batch(addr, end, folio, pte, &ptent); + nr = madvise_folio_pte_batch(vma, addr, end, folio, pte, &ptent); if (nr < folio_nr_pages(folio)) { int err; @@ -988,7 +1070,7 @@ static long madvise_populate(struct madvise_behavior *madv_behavior) return -ENOMEM; } } - start += pages * PAGE_SIZE; + start += pages * mm_page_size(mm); } return 0; } @@ -1022,7 +1104,7 @@ static long madvise_remove(struct madvise_behavior *madv_behavior) return -EACCES; offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); /* * Filesystem's fallocate may need to take i_rwsem. We need to @@ -1182,7 +1264,7 @@ static long madvise_guard_install(struct madvise_behavior *madv_behavior) if (err == 0) { unsigned long nr_expected_pages = - PHYS_PFN(range->end - range->start); + (range->end - range->start) >> mm_page_shift(vma->vm_mm); VM_WARN_ON(nr_pages != nr_expected_pages); return 0; @@ -1833,16 +1915,17 @@ static void madvise_finish_tlb(struct madvise_behavior *madv_behavior) tlb_finish_mmu(madv_behavior->tlb); } -static bool is_valid_madvise(unsigned long start, size_t len_in, int behavior) +static bool is_valid_madvise(struct mm_struct *mm, unsigned long start, + size_t len_in, int behavior) { size_t len; if (!madvise_behavior_valid(behavior)) return false; - if (!PAGE_ALIGNED(start)) + if (!IS_ALIGNED(start, mm_page_size(mm))) return false; - len = PAGE_ALIGN(len_in); + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero */ if (len_in && !len) @@ -1865,14 +1948,29 @@ static bool is_valid_madvise(unsigned long start, size_t len_in, int behavior) * operation. This function returns true in the cases, otherwise false. In * the former case we store an error on @err. */ -static bool madvise_should_skip(unsigned long start, size_t len_in, +static bool madvise_should_skip(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior, int *err) { - if (!is_valid_madvise(start, len_in, behavior)) { + if (!is_valid_madvise(mm, start, len_in, behavior)) { *err = -EINVAL; return true; } - if (start + PAGE_ALIGN(len_in) == start) { +#ifdef CONFIG_MM_SUBPAGE + /* These engines still assume whole native folios/tables. */ + if (mm_page_size(mm) != PAGE_SIZE) { + switch (behavior) { + case MADV_MERGEABLE: + case MADV_UNMERGEABLE: + case MADV_HUGEPAGE: + case MADV_COLLAPSE: + case MADV_HWPOISON: + case MADV_SOFT_OFFLINE: + *err = -EOPNOTSUPP; + return true; + } + } +#endif + if (start + mm_page_align(mm, len_in) == start) { *err = 0; return true; } @@ -1919,7 +2017,7 @@ static int madvise_do_behavior(unsigned long start, size_t len_in, } range->start = get_untagged_addr(madv_behavior->mm, start); - range->end = range->start + PAGE_ALIGN(len_in); + range->end = range->start + mm_page_align(madv_behavior->mm, len_in); blk_start_plug(&plug); if (is_madvise_populate(madv_behavior)) @@ -2012,7 +2110,7 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh .tlb = &tlb, }; - if (madvise_should_skip(start, len_in, behavior, &error)) + if (madvise_should_skip(mm, start, len_in, behavior, &error)) return error; error = madvise_lock(&madv_behavior); if (error) @@ -2055,7 +2153,7 @@ static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter, size_t len_in = iter_iov_len(iter); int error; - if (madvise_should_skip(start, len_in, behavior, &error)) + if (madvise_should_skip(mm, start, len_in, behavior, &error)) ret = error; else ret = madvise_do_behavior(start, len_in, &madv_behavior); @@ -2185,9 +2283,9 @@ static int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, .anon_name = anon_name, }; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; - len = (len_in + ~PAGE_MASK) & PAGE_MASK; + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero */ if (len_in && !len) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index d47aef256a324..964c002533131 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -67,6 +67,9 @@ #include "swap.h" #include "internal.h" +#ifdef CONFIG_MM_SUBPAGE +#include "user-subpage-internal.h" +#endif static int sysctl_memory_failure_early_kill __read_mostly; @@ -539,6 +542,37 @@ struct task_struct *task_early_kill(struct task_struct *tsk, int force_early) return find_early_kill_thread(tsk); } +#ifdef CONFIG_MM_SUBPAGE +struct subpage_collect_procs { + const struct page *page; + struct list_head *to_kill; + int force_early; +}; + +static bool collect_procs_subpage(struct folio *folio, struct vm_area_struct *vma, + struct rmap_walk_range range, void *arg) +{ + struct subpage_collect_procs *args = arg; + DEFINE_FOLIO_RMAP_WALK(walk, folio, vma, range, PVMW_SYNC); + struct task_struct *tsk; + unsigned long address; + + if (!page_vma_mapped_walk(&walk)) + return true; + address = walk.address; + page_vma_mapped_walk_done(&walk); + rcu_read_lock(); + for_each_process(tsk) { + struct task_struct *t = task_early_kill(tsk, args->force_early); + + if (t && t->mm == vma->vm_mm) + add_to_kill_anon_file(t, args->page, vma, args->to_kill, address); + } + rcu_read_unlock(); + return true; +} +#endif + /* * Collect processes when the error hit an anonymous page. */ @@ -549,6 +583,17 @@ static void collect_procs_anon(const struct folio *folio, struct task_struct *tsk; struct anon_vma *av; pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + struct subpage_collect_procs args = { page, to_kill, force_early }; + struct rmap_walk_control rwc = { + .rmap_one = collect_procs_subpage, + .arg = &args, + }; + + if (!folio_test_large(folio) && + rmap_walk_subpages((struct folio *)folio, &rwc, false)) + return; +#endif av = folio_lock_anon_vma_read(folio, NULL); if (av == NULL) /* Not actually mapped anymore */ diff --git a/mm/memory.c b/mm/memory.c index a903d5c1859cb..ade3dbadb409a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -87,6 +87,8 @@ #include #include "pgalloc-track.h" +#include + #include "internal.h" #include "swap.h" @@ -191,26 +193,26 @@ static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, unsigned long start; start = addr; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(tlb->mm, pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(tlb->mm, addr, end); if (pmd_none_or_clear_bad(pmd)) continue; free_pte_range(tlb, pmd, addr); } while (pmd++, addr = next, addr != end); - start &= PUD_MASK; + start &= pud_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= PUD_MASK; + ceiling &= pud_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - pmd = pmd_offset(pud, start); + pmd = pmd_offset_mm(tlb->mm, pud, start); pud_clear(pud); pmd_free_tlb(tlb, pmd, start); mm_dec_nr_pmds(tlb->mm); @@ -225,27 +227,27 @@ static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, unsigned long start; start = addr; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(tlb->mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(tlb->mm, addr, end); if (pud_none_or_clear_bad(pud)) continue; free_pmd_range(tlb, pud, addr, next, floor, ceiling); } while (pud++, addr = next, addr != end); - start &= P4D_MASK; + start &= p4d_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= P4D_MASK; + ceiling &= p4d_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - pud = pud_offset(p4d, start); - p4d_clear(p4d); + pud = pud_offset_mm(tlb->mm, p4d, start); + p4d_clear_mm(tlb->mm, p4d); pud_free_tlb(tlb, pud, start); mm_dec_nr_puds(tlb->mm); } @@ -259,26 +261,26 @@ static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd, unsigned long start; start = addr; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(tlb->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(tlb->mm, addr, end); + if (p4d_none_or_clear_bad_mm(tlb->mm, p4d)) continue; free_pud_range(tlb, p4d, addr, next, floor, ceiling); } while (p4d++, addr = next, addr != end); - start &= PGDIR_MASK; + start &= pgd_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= PGDIR_MASK; + ceiling &= pgd_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - p4d = p4d_offset(pgd, start); + p4d = p4d_offset_mm(tlb->mm, pgd, start); pgd_clear(pgd); p4d_free_tlb(tlb, p4d, start); } @@ -320,7 +322,7 @@ void free_pgd_range(struct mmu_gather *tlb, * Wherever addr is brought up or ceiling brought down, we must * be careful to reject "the opposite 0" before it confuses the * subsequent tests. But what about where end is brought down - * by PMD_SIZE below? no, end can't go down to 0 there. + * by one PMD span below? no, end can't go down to 0 there. * * Whereas we round start (addr) and ceiling down, by different * masks at different levels, in order to test whether a table @@ -328,19 +330,19 @@ void free_pgd_range(struct mmu_gather *tlb, * bother to round floor or end up - the tests don't need that. */ - addr &= PMD_MASK; + addr &= pmd_mask_mm(tlb->mm); if (addr < floor) { - addr += PMD_SIZE; + addr += pmd_size_mm(tlb->mm); if (!addr) return; } if (ceiling) { - ceiling &= PMD_MASK; + ceiling &= pmd_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) - end -= PMD_SIZE; + end -= pmd_size_mm(tlb->mm); if (addr > end - 1) return; /* @@ -350,7 +352,7 @@ void free_pgd_range(struct mmu_gather *tlb, tlb_change_page_size(tlb, PAGE_SIZE); pgd = pgd_offset(tlb->mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(tlb->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; free_p4d_range(tlb, pgd, addr, next, floor, ceiling); @@ -539,16 +541,16 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add return; } - p4dp = p4d_offset(pgdp, addr); + p4dp = p4d_offset_mm(mm, pgdp, addr); p4d = p4dp_get(p4dp); p4dv = p4d_val(p4d); - if (!p4d_present(p4d) || p4d_leaf(p4d)) { + if (!p4d_present_mm(mm, p4d) || p4d_leaf(p4d)) { pr_alert("pgd:%08llx p4d:%08llx\n", pgdv, p4dv); return; } - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_mm(mm, p4dp, addr); pud = pudp_get(pudp); pudv = pud_val(pud); @@ -557,7 +559,7 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add return; } - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_mm(mm, pudp, addr); pmd = pmdp_get(pmdp); pmdv = pmd_val(pmd); @@ -950,7 +952,15 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, struct page *page; if (likely(softleaf_is_swap(entry))) { - if (swap_dup_entry_direct(entry) < 0) + int err; + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) + err = swap_subpage_dup(entry, pte_swp_subpage_offset(orig_pte)); + else +#endif + err = swap_dup_entries_direct(entry, mm_pte_native_pages(src_mm)); + if (err < 0) return -EIO; /* make sure dst_mm is on swapoff's mmlist. */ @@ -982,6 +992,10 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, entry = make_readable_migration_entry( swp_offset(entry)); pte = softleaf_to_pte(entry); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) + pte = pte_swp_set_subpage_offset(pte, pte_swp_subpage_offset(orig_pte)); +#endif if (pte_swp_soft_dirty(orig_pte)) pte = pte_swp_mksoft_dirty(pte); if (pte_swp_uffd_wp(orig_pte)) @@ -1076,10 +1090,14 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma * over and copy the page & arm it. */ - if (copy_mc_user_highpage(&new_folio->page, page, addr, src_vma)) - return -EHWPOISON; + for (unsigned int i = 0; i < mm_pte_native_pages(src_vma->vm_mm); i++) { + if (copy_mc_user_highpage(folio_page(new_folio, i), page + i, + addr + i * PAGE_SIZE, src_vma)) + return -EHWPOISON; + } *prealloc = NULL; + folio_ref_add(new_folio, mm_pte_native_pages(dst_vma->vm_mm) - 1); __folio_mark_uptodate(new_folio); folio_add_new_anon_rmap(new_folio, dst_vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, dst_vma); @@ -1095,6 +1113,55 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma return 0; } +#ifdef CONFIG_MM_SUBPAGE +/* One new native folio may publish several independently locked user leaves. */ +static void subpage_add_new_lru(struct folio *folio, struct vm_area_struct *vma, + unsigned int mapped) +{ + folio_add_lru_vma(folio, vma); + if ((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED) + while (--mapped) + mlock_folio(folio); +} + +/* The destination slot and its charge/metadata were allocated outside PTLs. */ +static int copy_pinned_subpage(struct vm_area_struct *dst_vma, pte_t *dst_pte, + pte_t source_pte, unsigned long addr, int *rss, + struct mm_subpage *source, struct mm_subpage **prealloc) +{ + struct mm_subpage *dest = *prealloc; + struct folio *new; + pte_t pte; + int ret; + + if (!dest) + return -EAGAIN; + new = mm_subpage_folio(dest); + ret = copy_user_subpage_range(&new->page, mm_subpage_offset(dest), + &mm_subpage_folio(source)->page, mm_subpage_offset(source), + mm_subpage_size(source)); + if (ret) + return ret; + /* Fresh, unmapped backing is private to this preallocation. */ + if (WARN_ON_ONCE(!folio_trylock(new))) + return -EBUSY; + mm_subpage_set_exclusive(dest); + ret = mm_subpage_add_new_anon_rmap(dest, dst_vma, addr); + if (!ret) { + subpage_add_new_lru(new, dst_vma, 1); + rss[MM_ANONPAGES]++; + pte = phys_pte_mm(dst_vma->vm_mm, mm_subpage_phys(dest), dst_vma->vm_page_prot); + pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); + if (userfaultfd_pte_wp(dst_vma, source_pte)) + pte = pte_mkuffd_wp(pte); + set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); + *prealloc = NULL; + } + folio_unlock(new); + return ret; +} +#endif + static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr, int nr) @@ -1128,18 +1195,46 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, static inline int copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr, - int max_nr, int *rss, struct folio **prealloc) + int max_nr, int *rss, struct folio **prealloc, + struct mm_subpage **subprealloc) { fpb_t flags = FPB_MERGE_WRITE; struct page *page; struct folio *folio; - int err, nr; + int err, nr, nr_native; + unsigned int span = mm_pte_native_pages(src_vma->vm_mm); page = vm_normal_page(src_vma, addr, pte); if (unlikely(!page)) goto copy_pte; folio = page_folio(page); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_vma->vm_mm) < PAGE_SIZE) { + if (folio_test_anon(folio)) { + struct mm_subpage *slot; + + slot = mm_subpage_get_from_phys(pte_phys_mm(src_vma->vm_mm, pte)); + if (WARN_ON_ONCE(!slot)) + return -EINVAL; + err = mm_subpage_dup_anon_rmap(slot, dst_vma, src_vma, addr); + if (err == -EBUSY) { + err = copy_pinned_subpage(dst_vma, dst_pte, pte, addr, + rss, slot, subprealloc); + mm_subpage_put(slot); + return err ? err : 1; + } + if (err) { + mm_subpage_put(slot); + return err; + } + /* The lookup reference becomes the child's PTE reference. */ + rss[MM_ANONPAGES]++; + goto copy_pte; + } + max_nr = 1; + } +#endif /* * If we likely have to copy, just don't bother with batching. Make @@ -1152,18 +1247,25 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma if (vma_soft_dirty_enabled(src_vma)) flags |= FPB_RESPECT_SOFT_DIRTY; - nr = folio_pte_batch_flags(folio, src_vma, src_pte, &pte, max_nr, flags); - folio_ref_add(folio, nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_vma->vm_mm) > PAGE_SIZE) + nr = folio_subpage_pte_batch_flags(src_vma, folio, src_pte, + &pte, max_nr, flags); + else +#endif + nr = folio_pte_batch_flags(folio, src_vma, src_pte, &pte, max_nr, flags); + nr_native = nr * span; + folio_ref_add(folio, nr_native); if (folio_test_anon(folio)) { if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page, - nr, dst_vma, src_vma))) { - folio_ref_sub(folio, nr); + nr_native, dst_vma, src_vma))) { + folio_ref_sub(folio, nr_native); return -EAGAIN; } rss[MM_ANONPAGES] += nr; VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio); } else { - folio_dup_file_rmap_ptes(folio, page, nr, dst_vma); + folio_dup_file_rmap_ptes(folio, page, nr_native, dst_vma); rss[mm_counter_file(folio)] += nr; } __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, @@ -1171,7 +1273,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma return nr; } - folio_get(folio); + folio_ref_add(folio, span); if (folio_test_anon(folio)) { /* * If this page may have been pinned by the parent process, @@ -1181,7 +1283,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma */ if (unlikely(folio_try_dup_anon_rmap_pte(folio, page, dst_vma, src_vma))) { /* Page may be pinned, we have to copy. */ - folio_put(folio); + folio_put_refs(folio, span); err = copy_present_page(dst_vma, src_vma, dst_pte, src_pte, addr, rss, prealloc, page); return err ? err : 1; @@ -1189,6 +1291,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma rss[MM_ANONPAGES]++; VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio); } else { + mm_subpage_file_map_add(dst_vma, pte_phys_mm(src_vma->vm_mm, pte)); folio_dup_file_rmap_pte(folio, page, dst_vma); rss[mm_counter_file(folio)]++; } @@ -1202,11 +1305,12 @@ static inline struct folio *folio_prealloc(struct mm_struct *src_mm, struct vm_area_struct *vma, unsigned long addr, bool need_zero) { struct folio *new_folio; + unsigned int order = max_t(unsigned int, mm_page_shift(src_mm), PAGE_SHIFT) - PAGE_SHIFT; if (need_zero) - new_folio = vma_alloc_zeroed_movable_folio(vma, addr); + new_folio = vma_alloc_zeroed_movable_folio_order(vma, addr, order); else - new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr); + new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, order, vma, addr); if (!new_folio) return NULL; @@ -1220,6 +1324,34 @@ static inline struct folio *folio_prealloc(struct mm_struct *src_mm, return new_folio; } +#ifdef CONFIG_MM_SUBPAGE +static struct mm_subpage *anon_subpage_prealloc(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr) +{ + struct mm_subpage_pool *pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + struct mm_subpage *slot = ERR_PTR(-ENOMEM); + struct folio *folio; + int ret; + + if (!pool) + return slot; + folio = folio_prealloc(mm, vma, addr, false); + if (!folio) + goto out; + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + slot = ERR_PTR(ret); + goto out; + } + slot = mm_subpage_alloc_at(pool, vma_page_offset_at(vma, addr).offset); +out: + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + return slot; +} +#endif + static int copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, @@ -1236,6 +1368,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, int rss[NR_MM_COUNTERS]; softleaf_t entry = softleaf_mk_none(); struct folio *prealloc = NULL; + struct mm_subpage *subprealloc = NULL; int nr; again: @@ -1316,16 +1449,23 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, WARN_ON_ONCE(ret != -ENOENT); } /* copy_present_ptes() will clear `*prealloc' if consumed */ - max_nr = (end - addr) / PAGE_SIZE; + max_nr = (end - addr) >> mm_page_shift(src_mm); ret = copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, - ptent, addr, max_nr, rss, &prealloc); + ptent, addr, max_nr, rss, &prealloc, &subprealloc); /* * If we need a pre-allocated page for this pte, drop the * locks, allocate, and try again. * If copy failed due to hwpoison in source page, break out. */ - if (unlikely(ret == -EAGAIN || ret == -EHWPOISON)) + if (unlikely(ret == -EAGAIN || ret == -EHWPOISON || + (IS_ENABLED(CONFIG_MM_SUBPAGE) && ret < 0))) break; +#ifdef CONFIG_MM_SUBPAGE + if (subprealloc) { + mm_subpage_put(subprealloc); + subprealloc = NULL; + } +#endif if (unlikely(prealloc)) { /* * pre-alloc page cannot be reused by next time so as @@ -1338,7 +1478,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } nr = ret; progress += 8 * nr; - } while (dst_pte += nr, src_pte += nr, addr += PAGE_SIZE * nr, + } while (dst_pte += nr, src_pte += nr, addr += mm_page_size(src_mm) * nr, addr != end); lazy_mmu_mode_disable(); @@ -1357,10 +1497,22 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } else if (ret == -EBUSY || unlikely(ret == -EHWPOISON)) { goto out; } else if (ret == -EAGAIN) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) { + subprealloc = anon_subpage_prealloc(src_mm, src_vma, addr); + if (IS_ERR(subprealloc)) + return PTR_ERR(subprealloc); + ret = 0; + goto again; + } +#endif prealloc = folio_prealloc(src_mm, src_vma, addr, false); if (!prealloc) return -ENOMEM; } else if (ret < 0) { + /* Subpage ownership checks can return additional fatal errors. */ + if (IS_ENABLED(CONFIG_MM_SUBPAGE)) + goto out; VM_WARN_ON_ONCE(1); } @@ -1370,6 +1522,10 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, if (addr != end) goto again; out: +#ifdef CONFIG_MM_SUBPAGE + if (subprealloc) + mm_subpage_put(subprealloc); +#endif if (unlikely(prealloc)) folio_put(prealloc); return ret; @@ -1388,9 +1544,9 @@ copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_pmd = pmd_alloc(dst_mm, dst_pud, addr); if (!dst_pmd) return -ENOMEM; - src_pmd = pmd_offset(src_pud, addr); + src_pmd = pmd_offset_mm(src_vma->vm_mm, src_pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(src_vma->vm_mm, addr, end); if (pmd_is_huge(*src_pmd)) { int err; @@ -1425,9 +1581,9 @@ copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_pud = pud_alloc(dst_mm, dst_p4d, addr); if (!dst_pud) return -ENOMEM; - src_pud = pud_offset(src_p4d, addr); + src_pud = pud_offset_mm(src_vma->vm_mm, src_p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(src_vma->vm_mm, addr, end); if (pud_trans_huge(*src_pud)) { int err; @@ -1461,10 +1617,10 @@ copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr); if (!dst_p4d) return -ENOMEM; - src_p4d = p4d_offset(src_pgd, addr); + src_p4d = p4d_offset_mm(src_vma->vm_mm, src_pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(src_p4d)) + next = p4d_addr_end_mm(src_vma->vm_mm, addr, end); + if (p4d_none_or_clear_bad_mm(src_vma->vm_mm, src_p4d)) continue; if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d, addr, next)) @@ -1516,6 +1672,9 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) bool is_cow; int ret; + if (mm_page_shift(dst_mm) != mm_page_shift(src_mm)) + return -EINVAL; + if (!vma_needs_copy(dst_vma, src_vma)) return 0; @@ -1549,7 +1708,7 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) dst_pgd = pgd_offset(dst_mm, addr); src_pgd = pgd_offset(src_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(src_vma->vm_mm, addr, end); if (pgd_none_or_clear_bad(src_pgd)) continue; if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd, @@ -1629,7 +1788,7 @@ zap_install_uffd_wp_if_needed(struct vm_area_struct *vma, if (--nr == 0) break; pte++; - addr += PAGE_SIZE; + addr += mm_page_size(vma->vm_mm); } return was_installed; @@ -1642,10 +1801,12 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, bool *force_flush, bool *force_break, bool *any_skipped) { struct mm_struct *mm = tlb->mm; + unsigned int nr_native = nr * mm_pte_native_pages(mm); bool delay_rmap = false; if (!folio_test_anon(folio)) { ptent = get_and_clear_full_ptes(mm, addr, pte, nr, tlb->fullmm); + mm_subpage_file_map_del(vma, pte_phys_mm(mm, ptent)); if (pte_dirty(ptent)) { folio_mark_dirty(folio); if (tlb_delay_rmap(tlb)) { @@ -1669,16 +1830,44 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, nr, details, ptent); if (!delay_rmap) { - folio_remove_rmap_ptes(folio, page, nr, vma); + folio_remove_rmap_ptes(folio, page, nr_native, vma); if (unlikely(folio_mapcount(folio) < 0)) print_bad_pte(vma, addr, ptent, page); } - if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) { + /* Gather payloads count native references, not logical PTEs. */ + if (unlikely(__tlb_remove_folio_pages(tlb, page, nr_native, delay_rmap))) { + *force_flush = true; + *force_break = true; + } +} + +#ifdef CONFIG_MM_SUBPAGE +static void zap_present_subpage(struct mmu_gather *tlb, + struct vm_area_struct *vma, pte_t *pte, pte_t ptent, + unsigned long addr, int *rss, bool *force_flush, + bool *force_break, bool *any_skipped) +{ + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(tlb->mm, ptent)); + + /* All small-granule anonymous PTEs must carry slot ownership. */ + if (WARN_ON_ONCE(!slot)) { + *any_skipped = true; + return; + } + ptep_get_and_clear_full(tlb->mm, addr, pte, tlb->fullmm); + arch_check_zapped_pte(vma, ptent); + tlb_remove_tlb_entry(tlb, pte, addr); + mm_subpage_remove_anon_rmap(slot, vma); + rss[MM_ANONPAGES]--; + /* Drop the lookup ref, then transfer the PTE ref to the TLB batch. */ + mm_subpage_put(slot); + if (__tlb_remove_subpage(tlb, slot)) { *force_flush = true; *force_break = true; } } +#endif /* * Zap or skip at least one present PTE, trying to batch-process subsequent @@ -1716,12 +1905,30 @@ static inline int zap_present_ptes(struct mmu_gather *tlb, return 1; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + if (folio_test_anon(folio)) { + zap_present_subpage(tlb, vma, pte, ptent, addr, rss, + force_flush, force_break, any_skipped); + return 1; + } + /* Native folio batching still expresses physical PAGE_SIZE steps. */ + max_nr = 1; + } +#endif + /* * Make sure that the common "small folio" case is as fast as possible * by keeping the batching logic separate. */ if (unlikely(folio_test_large(folio) && max_nr != 1)) { - nr = folio_pte_batch(folio, pte, ptent, max_nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) > PAGE_SIZE) + nr = folio_subpage_pte_batch(vma, folio, pte, ptent, + max_nr, 0); + else +#endif + nr = folio_pte_batch(folio, pte, ptent, max_nr); zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr, addr, details, rss, force_flush, force_break, any_skipped); @@ -1764,9 +1971,20 @@ static inline int zap_nonpresent_ptes(struct mmu_gather *tlb, if (!should_zap_cows(details)) return 1; - nr = swap_pte_batch(pte, max_nr, ptent); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + swap_subpage_put(entry, pte_swp_subpage_offset(ptent), true); + } else +#endif + { + unsigned int span = mm_pte_native_pages(vma->vm_mm); + + /* Adjacent coarse PTEs advance by a full group of swap slots. */ + if (span == 1) + nr = swap_pte_batch(pte, max_nr, ptent); + swap_put_entries_direct(entry, nr * span); + } rss[MM_SWAPENTS] -= nr; - swap_put_entries_direct(entry, nr); } else if (softleaf_is_migration(entry)) { struct folio *folio = softleaf_to_folio(entry); @@ -1811,7 +2029,7 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, bool *any_skipped) { pte_t ptent = ptep_get(pte); - int max_nr = (end - addr) / PAGE_SIZE; + int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); int nr = 0; /* Skip all consecutive none ptes */ @@ -1825,7 +2043,7 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, if (!max_nr) return nr; pte += nr; - addr += nr * PAGE_SIZE; + addr += nr * mm_page_size(vma->vm_mm); } if (pte_present(ptent)) @@ -1839,13 +2057,14 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, return nr; } -static bool pte_table_reclaim_possible(unsigned long start, unsigned long end, +static bool pte_table_reclaim_possible(struct mm_struct *mm, + unsigned long start, unsigned long end, struct zap_details *details) { if (!IS_ENABLED(CONFIG_PT_RECLAIM)) return false; /* Only zap if we are allowed to and cover the full page table. */ - return details && details->reclaim_pt && (end - start >= PMD_SIZE); + return details && details->reclaim_pt && (end - start >= pmd_size_mm(mm)); } static bool zap_empty_pte_table(struct mm_struct *mm, pmd_t *pmd, @@ -1877,7 +2096,7 @@ static bool zap_pte_table_if_empty(struct mm_struct *mm, pmd_t *pmd, if (ptl != pml) spin_lock_nested(ptl, SINGLE_DEPTH_NESTING); - for (i = 0, pte = start_pte; i < PTRS_PER_PTE; i++, pte++) { + for (i = 0, pte = start_pte; i < pte_table_bytes_mm(mm) / sizeof(pte_t); i++, pte++) { if (!pte_none(ptep_get(pte))) goto out_ptl; } @@ -1902,7 +2121,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end, struct zap_details *details) { - bool can_reclaim_pt = pte_table_reclaim_possible(addr, end, details); + bool can_reclaim_pt = pte_table_reclaim_possible(tlb->mm, addr, end, details); bool force_flush = false, force_break = false; struct mm_struct *mm = tlb->mm; int rss[NR_MM_COUNTERS]; @@ -1915,7 +2134,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, int nr; retry: - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, mm_page_size(mm)); init_rss_vec(rss); start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl); if (!pte) @@ -1936,11 +2155,11 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, if (any_skipped) can_reclaim_pt = false; if (unlikely(force_break)) { - addr += nr * PAGE_SIZE; + addr += nr * mm_page_size(vma->vm_mm); direct_reclaim = false; break; } - } while (pte += nr, addr += PAGE_SIZE * nr, addr != end); + } while (pte += nr, addr += mm_page_size(mm) * nr, addr != end); /* * Fast path: try to hold the pmd lock and unmap the PTE page. @@ -1997,9 +2216,9 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, pmd_t *pmd; unsigned long next; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(tlb->mm, pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(tlb->mm, addr, end); if (pmd_is_huge(*pmd)) { if (next - addr != HPAGE_PMD_SIZE) __split_huge_pmd(vma, pmd, addr, false); @@ -2033,9 +2252,9 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb, pud_t *pud; unsigned long next; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(tlb->mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(tlb->mm, addr, end); if (pud_trans_huge(*pud)) { if (next - addr != HPAGE_PUD_SIZE) split_huge_pud(vma, pud, addr); @@ -2061,10 +2280,10 @@ static inline unsigned long zap_p4d_range(struct mmu_gather *tlb, p4d_t *p4d; unsigned long next; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(tlb->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(tlb->mm, addr, end); + if (p4d_none_or_clear_bad_mm(tlb->mm, p4d)) continue; next = zap_pud_range(tlb, vma, p4d, addr, next, details); } while (p4d++, addr = next, addr != end); @@ -2103,7 +2322,7 @@ static void __zap_vma_range(struct mmu_gather *tlb, struct vm_area_struct *vma, tlb_start_vma(tlb, vma); pgd = pgd_offset(vma->vm_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(tlb->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; next = zap_p4d_range(tlb, vma, pgd, addr, next, details); @@ -2350,7 +2569,7 @@ static int validate_page_before_insert(struct vm_area_struct *vma, static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, unsigned long addr, struct page *page, - pgprot_t prot, bool mkwrite) + pgprot_t prot, bool mkwrite, unsigned int offset) { struct folio *folio = page_folio(page); pte_t pteval = ptep_get(pte); @@ -2360,7 +2579,7 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, return -EBUSY; /* see insert_pfn(). */ - if (pte_pfn(pteval) != page_to_pfn(page)) { + if (pte_phys_mm(vma->vm_mm, pteval) != page_to_phys(page) + offset) { WARN_ON_ONCE(!is_zero_pfn(pte_pfn(pteval))); return -EFAULT; } @@ -2372,17 +2591,18 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, } /* Ok, finally just insert the thing.. */ - pteval = mk_pte(page, prot); + pteval = phys_pte_mm(vma->vm_mm, page_to_phys(page) + offset, prot); if (unlikely(is_zero_folio(folio))) { pteval = pte_mkspecial(pteval); } else { folio_get(folio); - pteval = mk_pte(page, prot); + pteval = phys_pte_mm(vma->vm_mm, page_to_phys(page) + offset, prot); if (mkwrite) { pteval = pte_mkyoung(pteval); pteval = maybe_mkwrite(pte_mkdirty(pteval), vma); } inc_mm_counter(vma->vm_mm, mm_counter_file(folio)); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, pteval)); folio_add_file_rmap_pte(folio, page, vma); } set_pte_at(vma->vm_mm, addr, pte, pteval); @@ -2390,7 +2610,7 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, } static int insert_page(struct vm_area_struct *vma, unsigned long addr, - struct page *page, pgprot_t prot, bool mkwrite) + struct page *page, pgprot_t prot, bool mkwrite, unsigned int offset) { int retval; pte_t *pte; @@ -2404,83 +2624,120 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr, if (!pte) goto out; retval = insert_page_into_pte_locked(vma, pte, addr, page, prot, - mkwrite); + mkwrite, offset); pte_unmap_unlock(pte, ptl); out: return retval; } -static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte, - unsigned long addr, struct page *page, pgprot_t prot) +/* Keep lock batching in units of hardware leaves, backing in native pages. */ +static int insert_page_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long offset, + unsigned long *length, pgprot_t prot) { - int err; + struct mm_struct *mm = vma->vm_mm; + unsigned long remaining = *length; + unsigned long size = mm_page_size(mm); + int ret = 0; - err = validate_page_before_insert(vma, page); - if (err) - return err; - return insert_page_into_pte_locked(vma, pte, addr, page, prot, false); + while (remaining) { + unsigned long in_pmd = min(remaining, + pmd_size_mm(mm) - (addr & ~pmd_mask_mm(mm))); + pmd_t *pmd = walk_to_pmd(mm, addr); + + if (!pmd) { + ret = -EFAULT; + break; + } + if (pte_alloc(mm, pmd)) { + ret = -ENOMEM; + break; + } + while (in_pmd) { + unsigned int batch = min_t(unsigned long, in_pmd / size, 8); + unsigned int i; + spinlock_t *ptl; + pte_t *start = pte_offset_map_lock(mm, pmd, addr, &ptl); + + if (!start) { + ret = -EFAULT; + goto out; + } + for (i = 0; i < batch; i++) { + struct page *page = pages[offset >> PAGE_SHIFT]; + + ret = validate_page_before_insert(vma, page); + if (!ret) + ret = insert_page_into_pte_locked(vma, start + i, + addr, page, prot, false, offset_in_page(offset)); + if (ret) + break; + addr += size; + offset += size; + remaining -= size; + in_pmd -= size; + } + pte_unmap_unlock(start, ptl); + if (ret) + goto out; + } + } +out: + *length = remaining; + return ret; } -/* insert_pages() amortizes the cost of spinlock operations - * when inserting pages in a loop. - */ static int insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num, pgprot_t prot) { - pmd_t *pmd = NULL; - pte_t *start_pte, *pte; - spinlock_t *pte_lock; - struct mm_struct *const mm = vma->vm_mm; - unsigned long curr_page_idx = 0; - unsigned long remaining_pages_total = *num; - unsigned long pages_to_write_in_pmd; + unsigned long length; int ret; -more: - ret = -EFAULT; - pmd = walk_to_pmd(mm, addr); - if (!pmd) - goto out; - pages_to_write_in_pmd = min_t(unsigned long, - remaining_pages_total, PTRS_PER_PTE - pte_index(addr)); - - /* Allocate the PTE if necessary; takes PMD lock once only. */ - ret = -ENOMEM; - if (pte_alloc(mm, pmd)) - goto out; + if (check_mul_overflow(*num, PAGE_SIZE, &length)) + return -EOVERFLOW; + ret = insert_page_range(vma, addr, pages, 0, &length, prot); + /* A partially inserted native page is still an incomplete input page. */ + *num = DIV_ROUND_UP(length, PAGE_SIZE); + return ret; +} - while (pages_to_write_in_pmd) { - int pte_idx = 0; - const int batch_size = min_t(int, pages_to_write_in_pmd, 8); +/** + * vm_insert_pages_range - map a byte range from a native-page array + * @vma: target VMA, held under the mmap write lock + * @addr: first user address + * @pages: native backing pages + * @nr_pages: size of @pages + * @offset: byte offset within the backing array + * @length: bytes to map + * + * Address, offset and length must match the target mm's granule. A failed + * insertion can leave a prefix mapped, as with vm_insert_pages(); the caller + * must unmap that prefix when abandoning the mapping. Physical ownership + * remains in native pages. Each installed user leaf acquires its own ref. + */ +int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long nr_pages, + unsigned long offset, unsigned long length) +{ + unsigned long bytes, end, size = mm_page_size(vma->vm_mm); - start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock); - if (!start_pte) { - ret = -EFAULT; - goto out; - } - for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) { - int err = insert_page_in_batch_locked(vma, pte, - addr, pages[curr_page_idx], prot); - if (unlikely(err)) { - pte_unmap_unlock(start_pte, pte_lock); - ret = err; - remaining_pages_total -= pte_idx; - goto out; - } - addr += PAGE_SIZE; - ++curr_page_idx; - } - pte_unmap_unlock(start_pte, pte_lock); - pages_to_write_in_pmd -= batch_size; - remaining_pages_total -= batch_size; + if (!IS_ALIGNED(addr | offset | length, size) || + check_mul_overflow(nr_pages, PAGE_SIZE, &bytes) || + offset > bytes || length > bytes - offset || + check_add_overflow(addr, length, &end) || + !range_in_vma(vma, addr, end)) + return -EINVAL; + if (!length) + return 0; + if (vma->vm_flags & VM_PFNMAP) + return -EINVAL; + if (!(vma->vm_flags & VM_MIXEDMAP)) { + mmap_assert_write_locked(vma->vm_mm); + vm_flags_set(vma, VM_MIXEDMAP); } - if (remaining_pages_total) - goto more; - ret = 0; -out: - *num = remaining_pages_total; - return ret; + return insert_page_range(vma, addr, pages, offset, &length, vma->vm_page_prot); } +EXPORT_SYMBOL_GPL(vm_insert_pages_range); /** * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock. @@ -2501,9 +2758,12 @@ int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num) { const unsigned long nr_pages = *num; - const unsigned long end = addr + PAGE_SIZE * nr_pages; + unsigned long length, end; - if (!range_in_vma(vma, addr, end)) + if (!IS_ALIGNED(addr, mm_page_size(vma->vm_mm)) || + check_mul_overflow(nr_pages, PAGE_SIZE, &length) || + check_add_overflow(addr, length, &end) || + !range_in_vma(vma, addr, end)) return -EFAULT; if (!(vma->vm_flags & VM_MIXEDMAP)) { VM_WARN_ON_ONCE(mmap_read_trylock(vma->vm_mm)); @@ -2528,8 +2788,10 @@ int map_kernel_pages_prepare(struct vm_area_desc *desc) } nr_pages = action->map_kernel.nr_pages; - end = addr + PAGE_SIZE * nr_pages; - if (!range_in_vma_desc(desc, addr, end)) + if (!IS_ALIGNED(addr, mm_page_size(desc->mm)) || + check_mul_overflow(nr_pages, PAGE_SIZE, &end) || + check_add_overflow(addr, end, &end) || + !range_in_vma_desc(desc, addr, end)) return -EFAULT; return 0; @@ -2588,10 +2850,35 @@ int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, BUG_ON(vma->vm_flags & VM_PFNMAP); vm_flags_set(vma, VM_MIXEDMAP); } - return insert_page(vma, addr, page, vma->vm_page_prot, false); + return insert_page(vma, addr, page, vma->vm_page_prot, false, 0); } EXPORT_SYMBOL(vm_insert_page); +#ifdef CONFIG_MM_SUBPAGE +/* Explicit physical offset; the VMA and supplied page retain native ownership. */ +vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, unsigned long addr, + struct page *page, unsigned int offset) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long size = mm_page_size(mm); + int err; + + if (!(vma->vm_flags & VM_MIXEDMAP) || (vma->vm_flags & VM_PFNMAP) || + (addr & (size - 1)) || (offset & (size - 1)) || + offset > PAGE_SIZE - size || addr < vma->vm_start || + addr >= vma->vm_end || vma->vm_end - addr < size) + return VM_FAULT_SIGBUS; + err = insert_page(vma, addr, page, vma->vm_page_prot, false, offset); + if (err == -ENOMEM) + return VM_FAULT_OOM; + if (err < 0 && err != -EBUSY) + return VM_FAULT_SIGBUS; + return VM_FAULT_NOPAGE; +} +EXPORT_SYMBOL_GPL(vmf_insert_page_offset); +#endif + + /* * __vm_map_pages - maps range of kernel pages into user vma * @vma: user vma to map to @@ -2606,20 +2893,22 @@ EXPORT_SYMBOL(vm_insert_page); * Return: 0 on success and error code otherwise. */ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, - unsigned long num, unsigned long offset) + unsigned long num, unsigned long offset, + unsigned int subpage_offset) { - unsigned long count = vma_pages(vma); - unsigned long uaddr = vma->vm_start; + unsigned long bytes, available; - /* Fail if the user requested offset is beyond the end of the object */ - if (offset >= num) + if (offset >= num || + check_mul_overflow(num - offset, PAGE_SIZE, &available) || + subpage_offset >= PAGE_SIZE || + check_mul_overflow(offset, PAGE_SIZE, &bytes) || + check_add_overflow(bytes, (unsigned long)subpage_offset, &bytes)) return -ENXIO; - - /* Fail if the user requested size exceeds available object size */ - if (count > num - offset) + if (vma->vm_end - vma->vm_start > + available - subpage_offset) return -ENXIO; - - return vm_insert_pages(vma, uaddr, pages + offset, &count); + return vm_insert_pages_range(vma, vma->vm_start, pages, num, bytes, + vma->vm_end - vma->vm_start); } /** @@ -2643,7 +2932,7 @@ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, int vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num) { - return __vm_map_pages(vma, pages, num, vma->vm_pgoff); + return __vm_map_pages(vma, pages, num, vma->vm_pgoff, vma_subpage_offset(vma)); } EXPORT_SYMBOL(vm_map_pages); @@ -2663,12 +2952,12 @@ EXPORT_SYMBOL(vm_map_pages); int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, unsigned long num) { - return __vm_map_pages(vma, pages, num, 0); + return __vm_map_pages(vma, pages, num, 0, 0); } EXPORT_SYMBOL(vm_map_pages_zero); static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, pgprot_t prot, bool mkwrite) + unsigned long pfn, pgprot_t prot, bool mkwrite, unsigned int offset) { struct mm_struct *mm = vma->vm_mm; pte_t *pte, entry; @@ -2690,7 +2979,7 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, * allocation and mapping invalidation so just skip the * update. */ - if (pte_pfn(entry) != pfn) { + if (pte_phys_mm(mm, entry) != PFN_PHYS(pfn) + offset) { WARN_ON_ONCE(!is_zero_pfn(pte_pfn(entry))); goto out_unlock; } @@ -2703,7 +2992,7 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, } /* Ok, finally just insert the thing.. */ - entry = pte_mkspecial(pfn_pte(pfn, prot)); + entry = pte_mkspecial(phys_pte_mm(mm, PFN_PHYS(pfn) + offset, prot)); if (mkwrite) { entry = pte_mkyoung(entry); @@ -2765,9 +3054,9 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, * Context: Process context. May allocate using %GFP_KERNEL. * Return: vm_fault_t value. */ -vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, +static vm_fault_t __vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot, - bool mkwrite) + bool mkwrite, unsigned int offset) { /* * Technically, architectures with pte_special can avoid all these @@ -2789,10 +3078,31 @@ vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, pfnmap_setup_cachemode_pfn(pfn, &pgprot); - return insert_pfn(vma, addr, pfn, pgprot, mkwrite); + return insert_pfn(vma, addr, pfn, pgprot, mkwrite, offset); +} +vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, pgprot_t pgprot, + bool mkwrite) +{ + return __vmf_insert_pfn_prot_mkwrite(vma, addr, pfn, pgprot, mkwrite, 0); } EXPORT_SYMBOL(vmf_insert_pfn_prot_mkwrite); +/* Explicit quarter offset; a native PFN alone cannot encode a 4K leaf. */ +vm_fault_t vmf_insert_pfn_prot_mkwrite_offset(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, unsigned int offset, + pgprot_t pgprot, bool mkwrite) +{ + unsigned long size = mm_page_size(vma->vm_mm); + + if (!IS_ALIGNED(addr | offset, size) || offset > PAGE_SIZE - size || + addr < vma->vm_start || addr >= vma->vm_end || + vma->vm_end - addr < size) + return VM_FAULT_SIGBUS; + return __vmf_insert_pfn_prot_mkwrite(vma, addr, pfn, pgprot, mkwrite, offset); +} +EXPORT_SYMBOL_GPL(vmf_insert_pfn_prot_mkwrite_offset); + static bool vm_mixed_ok(struct vm_area_struct *vma, unsigned long pfn, bool mkwrite) { @@ -2840,9 +3150,9 @@ static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma, * result in pfn_t_has_page() == false. */ page = pfn_to_page(pfn); - err = insert_page(vma, addr, page, pgprot, mkwrite); + err = insert_page(vma, addr, page, pgprot, mkwrite, 0); } else { - return insert_pfn(vma, addr, pfn, pgprot, mkwrite); + return insert_pfn(vma, addr, pfn, pgprot, mkwrite, 0); } if (err == -ENOMEM) @@ -2863,7 +3173,7 @@ vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, if (addr < vmf->vma->vm_start || addr >= vmf->vma->vm_end) return VM_FAULT_SIGBUS; - err = insert_page(vmf->vma, addr, page, pgprot, write); + err = insert_page(vmf->vma, addr, page, pgprot, write, 0); if (err == -ENOMEM) return VM_FAULT_OOM; if (err < 0 && err != -EBUSY) @@ -2898,7 +3208,7 @@ vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, */ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pte_t *pte, *mapped_pte; spinlock_t *ptl; @@ -2910,13 +3220,13 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, lazy_mmu_mode_enable(); do { BUG_ON(!pte_none(ptep_get(pte))); - if (!pfn_modify_allowed(pfn, prot)) { + if (!pfn_modify_allowed(PHYS_PFN(phys), prot)) { err = -EACCES; break; } - set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); - pfn++; - } while (pte++, addr += PAGE_SIZE, addr != end); + set_pte_at(mm, addr, pte, pte_mkspecial(phys_pte_mm(mm, phys, prot))); + phys += mm_page_size(mm); + } while (pte++, addr += mm_page_size(mm), addr != end); lazy_mmu_mode_disable(); pte_unmap_unlock(mapped_pte, ptl); return err; @@ -2924,67 +3234,67 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pmd_t *pmd; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; pmd = pmd_alloc(mm, pud, addr); if (!pmd) return -ENOMEM; VM_BUG_ON(pmd_trans_huge(*pmd)); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(mm, addr, end); err = remap_pte_range(mm, pmd, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pmd++, addr = next, addr != end); return 0; } static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pud_t *pud; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; pud = pud_alloc(mm, p4d, addr); if (!pud) return -ENOMEM; do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(mm, addr, end); err = remap_pmd_range(mm, pud, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pud++, addr = next, addr != end); return 0; } static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { p4d_t *p4d; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; p4d = p4d_alloc(mm, pgd, addr); if (!p4d) return -ENOMEM; do { - next = p4d_addr_end(addr, end); + next = p4d_addr_end_mm(mm, addr, end); err = remap_pud_range(mm, p4d, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (p4d++, addr = next, addr != end); return 0; } @@ -3013,25 +3323,26 @@ static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long ad { pgd_t *pgd; unsigned long next; - unsigned long end = addr + PAGE_ALIGN(size); struct mm_struct *mm = vma->vm_mm; + unsigned long end = addr + mm_page_align(mm, size); + phys_addr_t phys = PFN_PHYS(pfn); int err; - if (WARN_ON_ONCE(!PAGE_ALIGNED(addr))) + if (WARN_ON_ONCE(!IS_ALIGNED(addr, mm_page_size(mm)))) return -EINVAL; VM_WARN_ON_ONCE(!vma_test_all_mask(vma, VMA_REMAP_FLAGS)); BUG_ON(addr >= end); - pfn -= addr >> PAGE_SHIFT; pgd = pgd_offset(mm, addr); flush_cache_range(vma, addr, end); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(mm, addr, end); err = remap_p4d_range(mm, pgd, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pgd++, addr = next, addr != end); return 0; @@ -3093,7 +3404,7 @@ static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, struct pfnmap_track_ctx *ctx = NULL; int err; - size = PAGE_ALIGN(size); + size = mm_page_align(vma->vm_mm, size); /* * If we cover the full VMA, we'll perform actual tracking, and @@ -3162,7 +3473,7 @@ static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size) { - const unsigned long end = addr + PAGE_ALIGN(size); + const unsigned long end = addr + mm_page_align(vma->vm_mm, size); const bool is_cow = is_cow_mapping(vma->vm_flags); int err; @@ -3540,15 +3851,14 @@ static inline int pte_unmap_same(struct vm_fault *vmf) * -EHWPOISON: copy failed due to hwpoison in source page * -EAGAIN: copied failed (some other reason) */ -static inline int __wp_page_copy_user(struct page *dst, struct page *src, - struct vm_fault *vmf) +static inline int __wp_native_page_copy_user(struct page *dst, struct page *src, + struct vm_fault *vmf, unsigned long addr) { int ret; void *kaddr; void __user *uaddr; struct vm_area_struct *vma = vmf->vma; struct mm_struct *mm = vma->vm_mm; - unsigned long addr = vmf->address; if (likely(src)) { if (copy_mc_user_highpage(dst, src, addr, vma)) @@ -3638,6 +3948,22 @@ static inline int __wp_page_copy_user(struct page *dst, struct page *src, return ret; } +static int __wp_page_copy_user(struct page *dst, struct page *src, + struct vm_fault *vmf) +{ + unsigned int i, nr = mm_pte_native_pages(vmf->vma->vm_mm); + + for (i = 0; i < nr; i++) { + int ret = __wp_native_page_copy_user(dst + i, src ? src + i : NULL, + vmf, vmf->address + i * PAGE_SIZE); + + if (ret) + return ret; + kmsan_copy_page_meta(dst + i, src ? src + i : NULL); + } + return 0; +} + static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma) { struct file *vm_file = vma->vm_file; @@ -3883,14 +4209,13 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) delayacct_wpcopy_end(); return err == -EHWPOISON ? VM_FAULT_HWPOISON : 0; } - kmsan_copy_page_meta(&new_folio->page, vmf->page); } __folio_mark_uptodate(new_folio); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, - vmf->address & PAGE_MASK, - (vmf->address & PAGE_MASK) + PAGE_SIZE); + vmf->address & mm_page_mask(mm), + (vmf->address & mm_page_mask(mm)) + mm_page_size(mm)); mmu_notifier_invalidate_range_start(&range); /* @@ -3927,6 +4252,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) * some TLBs while the old PTE remains in others. */ ptep_clear_flush(vma, vmf->address, vmf->pte); + folio_ref_add(new_folio, mm_pte_native_pages(mm) - 1); folio_add_new_anon_rmap(new_folio, vma, vmf->address, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, vma); BUG_ON(unshare && pte_write(entry)); @@ -3970,7 +4296,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) mmu_notifier_invalidate_range_end(&range); if (new_folio) - folio_put(new_folio); + folio_put_refs(new_folio, page_copied ? mm_pte_native_pages(mm) : 1); if (old_folio) { if (page_copied) free_swap_cache(old_folio); @@ -4092,7 +4418,7 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf, struct folio *folio) return ret; } -#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#ifdef CONFIG_MM_ID static bool __wp_can_reuse_large_anon_folio(struct folio *folio, struct vm_area_struct *vma) { @@ -4158,18 +4484,18 @@ static bool __wp_can_reuse_large_anon_folio(struct folio *folio, folio_unlock_large_mapcount(folio); return exclusive; } -#else /* !CONFIG_TRANSPARENT_HUGEPAGE */ +#else /* !CONFIG_MM_ID */ static bool __wp_can_reuse_large_anon_folio(struct folio *folio, struct vm_area_struct *vma) { BUILD_BUG(); } -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif /* CONFIG_MM_ID */ static bool wp_can_reuse_anon_folio(struct folio *folio, struct vm_area_struct *vma) { - if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && folio_test_large(folio)) + if (IS_ENABLED(CONFIG_MM_ID) && folio_test_large(folio)) return __wp_can_reuse_large_anon_folio(folio, vma); /* @@ -4207,67 +4533,238 @@ static bool wp_can_reuse_anon_folio(struct folio *folio, return true; } -/* - * This routine handles present pages, when - * * users try to write to a shared page (FAULT_FLAG_WRITE) - * * GUP wants to take a R/O pin on a possibly shared anonymous page - * (FAULT_FLAG_UNSHARE) - * - * It is done by copying the page to a new address and decrementing the - * shared-page counter for the old page. - * - * Note that this routine assumes that the protection checks have been - * done by the caller (the low-level page fault routine in most cases). - * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've - * done any necessary COW. - * - * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even - * though the page will change only once the write actually happens. This - * avoids a few races, and potentially makes it more efficient. - * - * We enter with non-exclusive mmap_lock (to exclude vma changes, - * but allow concurrent faults), with pte both mapped and locked. - * We return with mmap_lock still held, but pte unmapped and unlocked. - */ -static vm_fault_t do_wp_page(struct vm_fault *vmf) +#ifdef CONFIG_MM_SUBPAGE +/* Enter with the original PTE locked; a slot, not its native folio, is copied. */ +static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) __releases(vmf->ptl) { - const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; struct vm_area_struct *vma = vmf->vma; - struct folio *folio = NULL; - pte_t pte; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *source = NULL, *dest = NULL; + struct mm_subpage_pool *pool = NULL; + struct folio *old = vmf->page ? page_folio(vmf->page) : NULL; + struct folio *new = NULL; + struct mmu_notifier_range range; + bool copied = false, unshare = vmf->flags & FAULT_FLAG_UNSHARE; + unsigned int offset; + vm_fault_t ret = VM_FAULT_SIGBUS; + pte_t entry; + int err; - if (likely(!unshare)) { - if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) { - if (!userfaultfd_wp_async(vma)) { - pte_unmap_unlock(vmf->pte, vmf->ptl); - return handle_userfault(vmf, VM_UFFD_WP); - } + if (old && folio_test_anon(old)) { + source = mm_subpage_get_from_phys(pte_phys_mm(mm, vmf->orig_pte)); + if (WARN_ON_ONCE(!source)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + return VM_FAULT_SIGBUS; + } + if (!mm_subpage_is_exclusive(source) && mm_subpage_mapcount(source) == 1 && + folio_trylock(old)) { + bool reusable = true; - /* - * Nothing needed (cache flush, TLB invalidations, - * etc.) because we're only removing the uffd-wp bit, - * which is completely invisible to the user. - */ - pte = pte_clear_uffd_wp(ptep_get(vmf->pte)); + if (folio_test_swapcache(old)) { + swp_entry_t entry = page_swap_entry(&old->page); - set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); - /* - * Update this to be prepared for following up CoW - * handling - */ - vmf->orig_pte = pte; + reusable = !swap_subpage_count(entry, mm_subpage_offset(source)) && + !(folio_test_writeback(old) && + (folio_swap_flags(old) & SWP_STABLE_WRITES)); + } + if (reusable) + mm_subpage_try_reuse_anon(source); + folio_unlock(old); } - - /* - * Userfaultfd write-protect can defer flushes. Ensure the TLB - * is flushed in this case before copying. - */ - if (unlikely(userfaultfd_wp(vmf->vma) && - mm_tlb_flush_pending(vmf->vma->vm_mm))) - flush_tlb_page(vmf->vma, vmf->address); + if (mm_subpage_is_exclusive(source)) { + if (unshare) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + } else { + folio_xchg_last_cpupid(old, (1 << LAST_CPUPID_SHIFT) - 1); + /* Exclusivity belongs to the slot, not PageAnonExclusive. */ + wp_page_reuse(vmf, NULL); + } + mm_subpage_put(source); + return 0; + } + } else if (old) { + folio_get(old); + } else if (!is_zero_pfn(pte_pfn(vmf->orig_pte))) { + /* Private device/PFN mappings need their own subpage contract. */ + pte_unmap_unlock(vmf->pte, vmf->ptl); + return VM_FAULT_SIGBUS; } - + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + delayacct_wpcopy_start(); + ret = vmf_anon_prepare(vmf); + if (ret) + goto out; + offset = vma_page_offset_at(vma, vmf->address).offset; + ret = VM_FAULT_OOM; + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + goto out; + new = folio_prealloc(mm, vma, vmf->address, false); + if (!new) + goto out; + err = mm_subpage_pool_add_folio(pool, new, GFP_KERNEL); + if (err) { + folio_put(new); + goto out; + } + dest = source ? mm_subpage_copy_at(pool, source, offset) : + mm_subpage_alloc_at(pool, offset); + if (IS_ERR(dest)) { + if (PTR_ERR(dest) == -EHWPOISON) + ret = VM_FAULT_HWPOISON; + dest = NULL; + goto out; + } + if (old && !source) { + err = copy_user_subpage_range(&new->page, offset, vmf->page, + pte_phys_mm(mm, vmf->orig_pte) & ~PAGE_MASK, + mm_page_size(mm)); + if (err) { + ret = VM_FAULT_HWPOISON; + goto out; + } + } + mm_subpage_pool_close(pool); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, + vmf->address, vmf->address + mm_page_size(mm)); + mmu_notifier_invalidate_range_start(&range); + /* Keep the folio alive after a concurrent zap can consume its PTE ref. */ + folio_get(new); + folio_lock(new); + ret = 0; + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock_pte; + } + mm_subpage_set_exclusive(dest); + err = mm_subpage_add_new_anon_rmap(dest, vma, vmf->address); + if (WARN_ON_ONCE(err)) { + ret = VM_FAULT_SIGBUS; + goto unlock_pte; + } + entry = pte_sw_mkyoung(phys_pte_mm(mm, mm_subpage_phys(dest), vma->vm_page_prot)); + if (unshare) { + if (pte_soft_dirty(vmf->orig_pte)) + entry = pte_mksoft_dirty(entry); + if (pte_uffd_wp(vmf->orig_pte)) + entry = pte_mkuffd_wp(entry); + } else { + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + } + BUG_ON(unshare && pte_write(entry)); + flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); + /* Break-before-make also orders removal of the old slot's rmap. */ + ptep_clear_flush(vma, vmf->address, vmf->pte); + subpage_add_new_lru(new, vma, 1); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + if (source) { + mm_subpage_remove_anon_rmap(source, vma); + } else if (old) { + mm_subpage_file_map_del(vma, pte_phys_mm(mm, vmf->orig_pte)); + folio_remove_rmap_pte(old, vmf->page, vma); + dec_mm_counter(mm, mm_counter_file(old)); + inc_mm_counter(mm, MM_ANONPAGES); + } else { + ksm_might_unmap_zero_page(mm, vmf->orig_pte); + inc_mm_counter(mm, MM_ANONPAGES); + } + dest = NULL; /* The new PTE owns the reference. */ + copied = true; +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + folio_unlock(new); + folio_put(new); + mmu_notifier_invalidate_range_end(&range); +out: + if (pool) { + mm_subpage_pool_close(pool); + if (dest) + mm_subpage_put(dest); + mm_subpage_pool_put(pool); + } + if (source) { + if (copied) + mm_subpage_put(source); /* Retired PTE reference, after TLBI. */ + mm_subpage_put(source); /* Source stabilisation reference. */ + } else if (old) { + if (copied) + folio_put(old); + folio_put(old); + } + delayacct_wpcopy_end(); + return ret; +} +#endif + +/* + * This routine handles present pages, when + * * users try to write to a shared page (FAULT_FLAG_WRITE) + * * GUP wants to take a R/O pin on a possibly shared anonymous page + * (FAULT_FLAG_UNSHARE) + * + * It is done by copying the page to a new address and decrementing the + * shared-page counter for the old page. + * + * Note that this routine assumes that the protection checks have been + * done by the caller (the low-level page fault routine in most cases). + * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've + * done any necessary COW. + * + * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even + * though the page will change only once the write actually happens. This + * avoids a few races, and potentially makes it more efficient. + * + * We enter with non-exclusive mmap_lock (to exclude vma changes, + * but allow concurrent faults), with pte both mapped and locked. + * We return with mmap_lock still held, but pte unmapped and unlocked. + */ +static vm_fault_t do_wp_page(struct vm_fault *vmf) + __releases(vmf->ptl) +{ + const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; + struct vm_area_struct *vma = vmf->vma; + struct folio *folio = NULL; + pte_t pte; + + if (likely(!unshare)) { + if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) { + if (!userfaultfd_wp_async(vma)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + return handle_userfault(vmf, VM_UFFD_WP); + } + + /* + * Nothing needed (cache flush, TLB invalidations, + * etc.) because we're only removing the uffd-wp bit, + * which is completely invisible to the user. + */ + pte = pte_clear_uffd_wp(ptep_get(vmf->pte)); + + set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); + /* + * Update this to be prepared for following up CoW + * handling + */ + vmf->orig_pte = pte; + } + + /* + * Userfaultfd write-protect can defer flushes. Ensure the TLB + * is flushed in this case before copying. + */ + if (unlikely(userfaultfd_wp(vmf->vma) && + mm_tlb_flush_pending(vmf->vma->vm_mm))) + flush_tlb_page(vmf->vma, vmf->address); + } + vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte); if (vmf->page) @@ -4292,6 +4789,11 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) return wp_page_shared(vmf, folio); } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return wp_subpage_copy(vmf); +#endif + /* * Private mapping: create an exclusive anonymous page copy if reuse * is impossible. We might miss VM_WRITE for FOLL_FORCE handling. @@ -4301,8 +4803,12 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) */ if (folio && folio_test_anon(folio) && (PageAnonExclusive(vmf->page) || wp_can_reuse_anon_folio(folio, vma))) { - if (!PageAnonExclusive(vmf->page)) - SetPageAnonExclusive(vmf->page); + if (!PageAnonExclusive(vmf->page)) { + unsigned int i, nr = mm_pte_native_pages(vma->vm_mm); + + for (i = 0; i < nr; i++) + SetPageAnonExclusive(vmf->page + i); + } if (unlikely(unshare)) { pte_unmap_unlock(vmf->pte, vmf->ptl); return 0; @@ -4340,12 +4846,116 @@ static inline void unmap_mapping_range_tree(struct rb_root_cached *root, start = vma->vm_start + ((start_idx - vma->vm_pgoff) << PAGE_SHIFT); size = (end_idx - start_idx) << PAGE_SHIFT; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + unsigned long end = vma->vm_end; + + start = vma_address(vma, first_index, last_index - first_index + 1); + /* The interval tree guarantees a nonempty intersection. */ + VM_BUG_ON_VMA(IS_ERR_VALUE(start), vma); + if (last_index < vma_last_pgoff(vma)) + end = vma_address_at_offset(vma, (struct vm_page_offset) { + .index = last_index + 1, + }); + size = end - start; + } else if (mm_page_size(vma->vm_mm) > PAGE_SIZE) { + unsigned long granule = mm_page_size(vma->vm_mm); + unsigned long end = min(start + size, vma->vm_end); + + /* A native-page invalidation removes every overlapping leaf. */ + start = max(round_down(start, granule), vma->vm_start); + end = min(round_up(end, granule), vma->vm_end); + size = end - start; + } +#endif + tlb_gather_mmu(&tlb, vma->vm_mm); zap_vma_range_batched(&tlb, vma, start, size, details); tlb_finish_mmu(&tlb); } } +#ifdef CONFIG_MM_SUBPAGE +/* Unmap affected user leaves, including coarse aliases; preserve private COW. */ +void unmap_mapping_subpage_hole(struct address_space *mapping, loff_t first, u64 last) +{ + struct vm_area_struct *vma; + struct zap_details details = { .skip_cows = true }; + + i_mmap_lock_read(mapping); + vma_interval_tree_foreach(vma, &mapping->i_mmap, + first >> PAGE_SHIFT, (last - 1) >> PAGE_SHIFT) { + unsigned long start, end, granule = mm_page_size(vma->vm_mm); + u64 low = round_down((u64)first, granule), high = round_up(last, granule); + struct vm_page_offset begin = { low >> PAGE_SHIFT, low & ~PAGE_MASK }; + struct vm_page_offset finish = { high >> PAGE_SHIFT, high & ~PAGE_MASK }; + struct vm_page_offset vbegin = vma_page_offset_at(vma, vma->vm_start); + struct vm_page_offset vend = vma_page_offset_at(vma, vma->vm_end); + struct mmu_gather tlb; + + if (begin.index < vbegin.index || + (begin.index == vbegin.index && begin.offset <= vbegin.offset)) + start = vma->vm_start; + else + start = vma_address_at_offset(vma, begin); + if (finish.index > vend.index || + (finish.index == vend.index && finish.offset >= vend.offset)) + end = vma->vm_end; + else + end = vma_address_at_offset(vma, finish); + if (IS_ERR_VALUE(start) || IS_ERR_VALUE(end) || start >= end) + continue; + tlb_gather_mmu(&tlb, vma->vm_mm); + zap_vma_range_batched(&tlb, vma, start, end - start, &details); + tlb_finish_mmu(&tlb); + } + i_mmap_unlock_read(mapping); +} + +/* + * Native truncation keeps its final partial page. Smaller-granule mms must + * additionally discard complete granules in that native page, including + * private COW copies. Call on both sides of cache truncation, as for the + * ordinary unmap_mapping_range() calls. + */ +void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize) +{ + struct vm_area_struct *vma; + struct zap_details details = { }; + pgoff_t index = newsize >> PAGE_SHIFT; + unsigned int offset = newsize & ~PAGE_MASK; + + if (!offset) + return; + i_mmap_lock_read(mapping); + vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) { + unsigned long start, end, granule = mm_page_size(vma->vm_mm); + struct vm_page_offset pos = { .index = index, .offset = round_up(offset, granule) }; + struct vm_page_offset vpos = vma_page_offset_at(vma, vma->vm_start); + struct mmu_gather tlb; + + if (pos.offset >= PAGE_SIZE) + continue; + if (pos.index < vpos.index || + (pos.index == vpos.index && pos.offset < vpos.offset)) + start = vma->vm_start; + else + start = vma_address_at_offset(vma, pos); + if (IS_ERR_VALUE(start)) + continue; + end = vma->vm_end; + if (index < vma_last_pgoff(vma)) + end = vma_address_at_offset(vma, (struct vm_page_offset) { + .index = index + 1, + }); + tlb_gather_mmu(&tlb, vma->vm_mm); + zap_vma_range_batched(&tlb, vma, start, end - start, &details); + tlb_finish_mmu(&tlb); + } + i_mmap_unlock_read(mapping); +} +#endif + /** * unmap_mapping_folio() - Unmap single folio from processes. * @folio: The locked folio to be unmapped. @@ -4772,6 +5382,134 @@ static void check_swap_exclusive(struct folio *folio, swp_entry_t entry, } while (--nr_pages); } +#ifdef CONFIG_MM_SUBPAGE +/* The swap-cache index remains native-sized; the PTE retains its slot offset. */ +static vm_fault_t do_swap_subpage(struct vm_fault *vmf, swp_entry_t entry) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct swap_info_struct *si; + struct mm_subpage *slot = NULL; + struct folio *folio; + unsigned int offset = pte_swp_subpage_offset(vmf->orig_pte); + vm_fault_t ret; + pte_t pte; + int err; + + ret = vmf_anon_prepare(vmf); + if (ret) + return ret; + /* Swap encodes the physical slot, independent of its virtual position. */ + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(mm))) + return VM_FAULT_SIGBUS; + si = get_swap_device(entry); + if (!si) + return 0; + folio = swap_cache_get_folio(entry); + if (folio) { + swap_update_readahead(folio, vma, vmf->address); + } else { + folio = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, vmf); + if (!folio) { + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (vmf->pte) { + if (pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + ret = VM_FAULT_OOM; + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + } + goto out_device; + } + ret = VM_FAULT_MAJOR; + count_vm_event(PGMAJFAULT); + count_memcg_event_mm(mm, PGMAJFAULT); + } + ret |= folio_lock_or_retry(folio, vmf); + if (ret & VM_FAULT_RETRY) + goto out_folio; + if (!folio_matches_swap_entry(folio, entry)) + goto unlock_folio; + if (folio_order(folio) || !folio_test_uptodate(folio) || folio_test_ksm(folio)) { + ret = VM_FAULT_SIGBUS; + goto unlock_folio; + } + if (PageHWPoison(&folio->page)) { + ret = VM_FAULT_HWPOISON; + goto unlock_folio; + } + arch_swap_restore(folio_swap(entry, folio), folio); + slot = mm_subpage_restore_granule(folio, offset, mm_page_shift(mm), GFP_KERNEL); + if (IS_ERR(slot)) { + err = PTR_ERR(slot); + slot = NULL; + if (err != -EAGAIN) + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_folio; + } + folio_throttle_swaprate(folio, GFP_KERNEL); + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + goto unlock_pte; + err = mm_subpage_add_anon_rmap(slot, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_pte; + } + /* + * A carried marker proves exclusivity directly. After the last fork + * sharer exits, recover it only with one swap reference to this exact + * quarter and no other mapping or escaped slot reference. Dropping a + * nonpresent PTE in zap flushes pending reclaim TLB invalidations first. + * Stable-write backends must still finish writeback before reuse. + */ + if (!(folio_test_writeback(folio) && data_race(si->flags & SWP_STABLE_WRITES))) { + if (pte_swp_exclusive(vmf->orig_pte)) + mm_subpage_set_exclusive(slot); + else if (swap_subpage_count(entry, offset) == 1) + mm_subpage_try_reuse_swap(slot); + } + pte = pte_sw_mkyoung(phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot)); + if (pte_swp_soft_dirty(vmf->orig_pte)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(vmf->orig_pte)) + pte = pte_mkuffd_wp(pte); + inc_mm_counter(mm, MM_ANONPAGES); + dec_mm_counter(mm, MM_SWAPENTS); + swap_subpage_put(entry, offset, false); + set_pte_at(mm, vmf->address, vmf->pte, pte); + arch_do_swap_page_nr(mm, vma, vmf->address, pte, pte, 1); + slot = NULL; /* The restored PTE owns the slot reference. */ + vmf->orig_pte = pte; + if (should_try_to_free_swap(si, folio, vma, 2, vmf->flags)) + folio_free_swap(folio); + folio_unlock(folio); + folio_put(folio); /* Drop the cache lookup, not the PTE reference. */ + put_swap_device(si); + if (vmf->flags & FAULT_FLAG_WRITE) { + ret |= do_wp_page(vmf); + return ret & VM_FAULT_ERROR ? ret & VM_FAULT_ERROR : ret; + } + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + return ret; +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + if (slot) + mm_subpage_put(slot); + folio_unlock(folio); +out_folio: + folio_put(folio); +out_device: + put_swap_device(si); + return ret; +} +#endif + /* * We enter with non-exclusive mmap_lock (to exclude vma changes, * but allow concurrent faults), and pte mapped but not yet locked. @@ -4780,6 +5518,209 @@ static void check_swap_exclusive(struct folio *folio, swp_entry_t entry, * We return with the mmap_lock locked or unlocked in the same cases * as does filemap_fault(). */ +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SWAP) +/* One coarse PTE owns a naturally aligned group of native swap entries. */ +vm_fault_t do_swap_coarse_page(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + /* Arm64's largest base user page is 64K, its smallest native page 4K. */ + struct folio *cache[SZ_64K / PAGE_SIZE]; + unsigned int nr = mm_pte_native_pages(mm), covered = 0, count = 0, i; + swp_entry_t entry = softleaf_from_pte(vmf->orig_pte), next; + struct swap_info_struct *si; + struct folio *backing = NULL; + struct page *page; + unsigned int start; + bool copied = false, exclusive, mapped = false; + vm_fault_t ret = 0; + pte_t pte; + + vmf->pte = NULL; + if (WARN_ON_ONCE(nr <= 1 || nr > ARRAY_SIZE(cache) || + !IS_ALIGNED(swp_offset(entry), nr))) + return VM_FAULT_SIGBUS; + si = get_swap_device(entry); + if (!si) + return 0; + + while (covered < nr) { + struct folio *folio; + unsigned int take; + vm_fault_t wait; + + next = entry; + next.val += covered; + folio = swap_cache_get_folio(next); + if (!folio) { + folio = read_swap_cache_async(next, GFP_HIGHUSER_MOVABLE, vma, + vmf->address + covered * PAGE_SIZE, NULL); + if (!(ret & VM_FAULT_MAJOR)) { + ret |= VM_FAULT_MAJOR; + count_vm_event(PGMAJFAULT); + count_memcg_event_mm(mm, PGMAJFAULT); + } + } + if (!folio) + goto oom; + if (!count) { + wait = folio_lock_or_retry(folio, vmf); + if (wait & VM_FAULT_RETRY) { + folio_put(folio); + ret |= wait; + goto out; + } + } else if (!folio_trylock(folio)) { + /* Drop earlier cache locks before waiting for this I/O or owner. */ + for (i = 0; i < count; i++) { + folio_unlock(cache[i]); + folio_put(cache[i]); + } + count = 0; + covered = 0; + wait = folio_lock_or_retry(folio, vmf); + if (wait & VM_FAULT_RETRY) { + folio_put(folio); + ret |= wait; + goto out; + } + folio_unlock(folio); + folio_put(folio); + cond_resched(); + continue; + } + cache[count++] = folio; + if (!folio_matches_swap_entry(folio, next)) + goto out; + if (!folio_test_uptodate(folio)) { + ret = VM_FAULT_SIGBUS; + goto out; + } + start = swp_offset(next) - swp_offset(folio->swap); + take = min(nr - covered, folio_nr_pages(folio) - start); + for (i = 0; i < take; i++) { + if (PageHWPoison(folio_page(folio, start + i))) { + ret = VM_FAULT_HWPOISON; + goto out; + } + } + arch_swap_restore(folio_swap(next, folio), folio); + covered += take; + } + + start = swp_offset(entry) - swp_offset(cache[0]->swap); + if (count == 1 && IS_ALIGNED(start, nr) && + (folio_test_anon(cache[0]) || folio_nr_pages(cache[0]) == nr)) { + backing = cache[0]; + page = folio_page(backing, start); + } else { + backing = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, ilog2(nr), vma, vmf->address); + if (!backing) + goto oom; + copied = true; + if (mem_cgroup_swapin_charge_folio(backing, mm, GFP_KERNEL, entry)) + goto oom; + covered = 0; + for (i = 0; i < count; i++) { + unsigned int j, take; + + start = swp_offset(entry) + covered - swp_offset(cache[i]->swap); + take = min(nr - covered, folio_nr_pages(cache[i]) - start); + for (j = 0; j < take; j++) { + struct page *dst = folio_page(backing, covered + j); + struct page *src = folio_page(cache[i], start + j); + + if (copy_mc_user_highpage(dst, src, + vmf->address + (covered + j) * PAGE_SIZE, vma)) { + ret = VM_FAULT_HWPOISON; + goto out; + } + kmsan_copy_page_meta(dst, src); + } + covered += take; + } + folio_set_dirty(backing); + __folio_mark_uptodate(backing); + page = &backing->page; + } + + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + goto out; + exclusive = copied || pte_swp_exclusive(vmf->orig_pte); + if (!copied && exclusive) { + check_swap_exclusive(backing, entry, nr); + if (folio_test_writeback(backing) && data_race(si->flags & SWP_STABLE_WRITES)) + exclusive = false; + } + pte = mk_pte(page, vma->vm_page_prot); + if (pte_swp_soft_dirty(vmf->orig_pte)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(vmf->orig_pte)) + pte = pte_mkuffd_wp(pte); + if (exclusive && (vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && + !pte_needs_soft_dirty_wp(vma, pte)) { + pte = pte_mkwrite(pte, vma); + if (vmf->flags & FAULT_FLAG_WRITE) { + pte = pte_mkdirty(pte); + vmf->flags &= ~FAULT_FLAG_WRITE; + } + } + folio_ref_add(backing, nr - copied); + if (!folio_test_anon(backing)) + folio_add_new_anon_rmap(backing, vma, vmf->address, + exclusive ? RMAP_EXCLUSIVE : RMAP_NONE); + else + folio_add_anon_rmap_pte(backing, page, vma, vmf->address, + exclusive ? RMAP_EXCLUSIVE : RMAP_NONE); + if (copied) + folio_add_lru_vma(backing, vma); + inc_mm_counter(mm, MM_ANONPAGES); + dec_mm_counter(mm, MM_SWAPENTS); + flush_icache_pages(vma, page, nr); + vmf->orig_pte = pte; + set_pte_at(mm, vmf->address, vmf->pte, pte); + arch_do_swap_page_nr(mm, vma, vmf->address, pte, pte, 1); + mapped = true; + covered = 0; + for (i = 0; i < count; i++) { + unsigned int take; + + start = swp_offset(entry) + covered - swp_offset(cache[i]->swap); + take = min(nr - covered, folio_nr_pages(cache[i]) - start); + folio_put_swap_range(cache[i], folio_page(cache[i], start), take); + covered += take; + } + goto out; + +oom: + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + ret |= VM_FAULT_OOM; +out: + if (copied && !mapped) + folio_put(backing); + for (i = 0; i < count; i++) { + if (mapped) + folio_free_swap(cache[i]); + folio_unlock(cache[i]); + folio_put(cache[i]); + } + if (mapped && (vmf->flags & FAULT_FLAG_WRITE)) { + ret |= do_wp_page(vmf); + vmf->pte = NULL; /* do_wp_page drops the PT lock. */ + } else if (mapped) { + update_mmu_cache(vma, vmf->address, vmf->pte); + } + if (vmf->pte) + pte_unmap_unlock(vmf->pte, vmf->ptl); + put_swap_device(si); + if (!mapped && !(ret & (VM_FAULT_ERROR | VM_FAULT_RETRY))) + cond_resched(); + return ret; +} +#endif + vm_fault_t do_swap_page(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -4854,6 +5795,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) goto out; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return do_swap_subpage(vmf, entry); +#ifdef CONFIG_SWAP + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + return do_swap_coarse_page(vmf); +#endif +#endif + /* Prevent swapoff from happening to us. */ si = get_swap_device(entry); if (unlikely(!si)) @@ -5291,6 +6241,7 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, bool uffd_wp) { const unsigned int nr_pages = folio_nr_pages(folio); + const unsigned int nr_ptes = folio_size(folio) >> mm_page_shift(vma->vm_mm); pte_t entry = folio_mk_pte(folio, vma->vm_page_prot); entry = pte_sw_mkyoung(entry); @@ -5303,8 +6254,8 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, folio_ref_add(folio, nr_pages - 1); folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); - set_ptes(vma->vm_mm, addr, pte, entry, nr_pages); - update_mmu_cache_range(NULL, vma, addr, pte, nr_pages); + set_ptes(vma->vm_mm, addr, pte, entry, nr_ptes); + update_mmu_cache_range(NULL, vma, addr, pte, nr_ptes); } static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, @@ -5313,10 +6264,123 @@ static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, const unsigned int order = folio_order(folio); map_anon_folio_pte_nopf(folio, pte, vma, addr, uffd_wp); - add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1L << order); + add_mm_counter(vma->vm_mm, MM_ANONPAGES, folio_size(folio) >> mm_page_shift(vma->vm_mm)); count_mthp_stat(order, MTHP_STAT_ANON_FAULT_ALLOC); } +#ifdef CONFIG_MM_SUBPAGE +static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slots[PAGE_SIZE / MM_SUBPAGE_SIZE] = {}; + struct vm_page_offset pos = vma_page_offset_at(vma, vmf->address); + struct mm_subpage_pool *pool; + struct folio *folio; + unsigned long start, end, lower; + unsigned int first_offset, nr, target, i, mapped = 0; + vm_fault_t ret = VM_FAULT_OOM; + pte_t *first; + int err; + + /* One native linear index, clipped to the VMA and this PTE table. */ + lower = max(vma->vm_start, vmf->address & pmd_mask_mm(mm)); + start = vmf->address - min_t(unsigned long, pos.offset, vmf->address - lower); + end = vmf->address + min(PAGE_SIZE - pos.offset, + pmd_addr_end_mm(mm, vmf->address, vma->vm_end) - vmf->address); + first_offset = pos.offset - (vmf->address - start); + nr = (end - start) >> mm_page_shift(mm); + target = (vmf->address - start) >> mm_page_shift(mm); + + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return VM_FAULT_OOM; + /* Supply initialises every byte once, including the unissued slots. */ + folio = folio_prealloc(mm, vma, vmf->address, false); + if (!folio) + goto release; + err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (err) { + folio_put(folio); + goto release; + } + for (i = 0; i < nr; i++) { + slots[i] = mm_subpage_alloc_at(pool, first_offset + i * mm_page_size(mm)); + if (IS_ERR(slots[i])) { + slots[i] = NULL; + goto release; + } + } + + /* PTE references may disappear as soon as we drop the PTL. */ + folio_get(folio); + folio_lock(folio); + ret = 0; + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (vmf_pte_changed(vmf)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock_pte; + } + ret = check_stable_address_space(mm); + if (ret) + goto unlock_pte; + if (userfaultfd_missing(vma)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + folio_unlock(folio); + folio_put(folio); + ret = handle_userfault(vmf, VM_UFFD_MISSING); + goto release; + } + + first = vmf->pte - target; + for (i = 0; i < nr; i++) { + unsigned long address = start + i * mm_page_size(mm); + pte_t entry; + + /* Do not replace existing mappings or userfaultfd markers nearby. */ + if (i != target && !pte_none(ptep_get(first + i))) + continue; + mm_subpage_set_exclusive(slots[i]); + err = mm_subpage_add_new_anon_rmap(slots[i], vma, address); + if (WARN_ON_ONCE(err)) { + ret = VM_FAULT_SIGBUS; + break; + } + entry = phys_pte_mm(mm, mm_subpage_phys(slots[i]), vma->vm_page_prot); + entry = pte_sw_mkyoung(entry); + if (vma->vm_flags & VM_WRITE) + entry = pte_mkwrite(pte_mkdirty(entry), vma); + if (i == target && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + set_pte_at(mm, address, first + i, entry); + update_mmu_cache(vma, address, first + i); + slots[i] = NULL; /* The PTE now owns the reference. */ + mapped++; + } + if (mapped) { + subpage_add_new_lru(folio, vma, mapped); + add_mm_counter(mm, MM_ANONPAGES, mapped); + count_mthp_stat(0, MTHP_STAT_ANON_FAULT_ALLOC); + } +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + folio_unlock(folio); + folio_put(folio); +release: + mm_subpage_pool_close(pool); + for (i = 0; i < ARRAY_SIZE(slots); i++) + if (slots[i]) + mm_subpage_put(slots[i]); + mm_subpage_pool_put(pool); + return ret; +} +#endif + /* * We enter with non-exclusive mmap_lock (to exclude vma changes, * but allow concurrent faults), and pte mapped but not yet locked. @@ -5376,6 +6440,10 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) ret = vmf_anon_prepare(vmf); if (ret) return ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return do_anonymous_subpage(vmf); +#endif /* Returns NULL on OOM or ERR_PTR(-EAGAIN) if we must retry the fault */ folio = alloc_anon_folio(vmf); if (IS_ERR(folio)) @@ -5383,8 +6451,8 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) if (!folio) goto oom; - nr_pages = folio_nr_pages(folio); - addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE); + nr_pages = folio_size(folio) >> mm_page_shift(vma->vm_mm); + addr = ALIGN_DOWN(vmf->address, folio_size(folio)); /* * The memory barrier inside __folio_mark_uptodate makes sure that @@ -5517,6 +6585,10 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa * PMD mappings if THPs are disabled. As we already have a THP, * behave as if we are forcing a collapse. */ + /* Match the native PTE fault policy: partial data exposes its zero subunits. */ + if (userfaultfd_missing(vma) && shmem_uffd_has_missing_native_page(folio)) + return ret; + if (thp_disabled_by_hw() || vma_thp_disabled(vma, vma->vm_flags, /* forced_collapse=*/ true)) return ret; @@ -5558,6 +6630,8 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); add_mm_counter(vma->vm_mm, mm_counter_file(folio), HPAGE_PMD_NR); + shmem_uffd_mark_mapped(folio, folio_page_idx(folio, page) << PAGE_SHIFT, + HPAGE_PMD_SIZE); folio_add_file_rmap_pmd(folio, page, vma); /* @@ -5620,6 +6694,8 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); } else { + shmem_uffd_mark_mapped(folio, folio_page_idx(folio, page) << PAGE_SHIFT, + (unsigned long)nr << PAGE_SHIFT); folio_add_file_rmap_ptes(folio, page, nr, vma); } set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr); @@ -5636,6 +6712,252 @@ static bool vmf_pte_changed(struct vm_fault *vmf) return !pte_none(ptep_get(vmf->pte)); } +#ifdef CONFIG_MM_SUBPAGE +static bool folio_user_range_valid(struct folio *folio, unsigned long offset, + unsigned long size) +{ + return offset <= folio_size(folio) && size <= folio_size(folio) - offset && + IS_ALIGNED(page_to_phys(&folio->page) + offset, size); +} + +/* + * Map the intersection of a locked cache folio, file-index range, VMA and + * the fault's PTE table. vmf->pte points at vmf->address and its lock is held. + * Each PTE owns a reference per covered native page (one for a subpage). + * The caller's reference is untouched, including when no PTE is installed. + */ +vm_fault_t map_file_user_pages(struct vm_fault *vmf, struct folio *folio, + pgoff_t first, pgoff_t last, bool fault) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + unsigned long addr, end, table_start = vmf->address & pmd_mask_mm(mm); + unsigned long size = mm_page_size(mm); + loff_t file_size = i_size_read(vma->vm_file->f_mapping->host); + vm_fault_t ret = 0; + unsigned int mapped = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + first = max(first, folio->index); + last = min(last, folio_next_index(folio) - 1); + if (first > last) + return 0; + addr = vma_address(vma, first, last - first + 1); + if (IS_ERR_VALUE(addr)) + return 0; + addr = ALIGN(max(addr, table_start), size); + end = pmd_addr_end_mm(mm, vmf->address, vma->vm_end); + if (userfaultfd_armed(vma)) { + addr = max(addr, vmf->address); + end = min(end, vmf->address + size); + } + for (; addr < end; addr += size) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + struct page *page; + unsigned long offset; + unsigned int i, nr = mm_pte_native_pages(mm); + bool poisoned = false; + pte_t entry, *ptep; + + if (pos.index > last || !vma_file_offset_valid(vma, addr, file_size)) + break; + offset = ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset; + /* + * A coarse leaf cannot span separate cache folios or a partial + * requested index range, even if neighboring PFNs happen to match. + */ + if (nr - 1 > last - pos.index || + !folio_user_range_valid(folio, offset, size)) + continue; + /* A native cache folio can contain independently missing leaves. */ + if (userfaultfd_missing(vma) && + shmem_uffd_range_missing(folio, pos.index, pos.offset, size)) + continue; + page = folio_file_page(folio, pos.index); + ptep = vmf->pte + ((long)(addr - vmf->address) >> mm_page_shift(mm)); + for (i = 0; i < nr; i++) + poisoned |= PageHWPoison(page + i); + if (poisoned) + continue; + if (!pte_none(ptep_get(ptep)) && + !(fault && addr == vmf->address && !vmf_pte_changed(vmf))) + continue; + flush_icache_pages(vma, page, nr); + entry = phys_pte_mm(mm, page_to_phys(page) + pos.offset, vma->vm_page_prot); + if (addr != vmf->address && arch_wants_old_prefaulted_pte()) + entry = pte_mkold(entry); + else + entry = pte_sw_mkyoung(entry); + if (vmf->flags & FAULT_FLAG_WRITE) + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + else if (pte_write(entry) && folio_test_dirty(folio)) + entry = pte_mkdirty(entry); + if (fault && addr == vmf->address && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_ref_add(folio, nr); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, entry)); + shmem_uffd_mark_mapped(folio, + ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset, size); + folio_add_file_rmap_pte(folio, page, vma); + set_pte_at(mm, addr, ptep, entry); + update_mmu_cache_range(vmf, vma, addr, ptep, 1); + mapped++; + if (addr == vmf->address) + ret = VM_FAULT_NOPAGE; + } + add_mm_counter(mm, mm_counter_file(folio), mapped); + return ret; +} + +/* + * A non-file fault callback supplies one native page, not a cache-indexed + * folio. In particular, VVAR callbacks may reorder native pages. Only map + * quarters of this supplied page; never infer adjacent pages from its folio. + */ +static vm_fault_t map_special_subpages(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct page *page = vmf->page; + struct folio *folio = page_folio(page); + unsigned int offset = vma_page_offset_at(vma, vmf->address).offset; + unsigned long size = mm_page_size(mm), addr, end; + vm_fault_t ret = 0; + unsigned int mapped = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (PageHWPoison(page)) + return 0; + addr = offset > vmf->address ? vma->vm_start : + max(vma->vm_start, vmf->address - offset); + addr = max(addr, vmf->address & pmd_mask_mm(mm)); + end = min(vmf->address + PAGE_SIZE - offset, + pmd_addr_end_mm(mm, vmf->address, vma->vm_end)); + if (userfaultfd_armed(vma)) { + addr = vmf->address; + end = vmf->address + size; + } + flush_icache_pages(vma, page, 1); + for (; addr < end; addr += size) { + unsigned int physical_offset = addr < vmf->address ? + offset - (vmf->address - addr) : offset + (addr - vmf->address); + pte_t *ptep = vmf->pte + ((long)(addr - vmf->address) >> mm_page_shift(mm)); + pte_t entry; + + if (!pte_none(ptep_get(ptep)) && + !(addr == vmf->address && !vmf_pte_changed(vmf))) + continue; + entry = phys_pte_mm(mm, page_to_phys(page) + physical_offset, vma->vm_page_prot); + entry = pte_sw_mkyoung(entry); + if (vmf->flags & FAULT_FLAG_WRITE) + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + if (addr == vmf->address && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_get(folio); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, entry)); + folio_add_file_rmap_pte(folio, page, vma); + set_pte_at(mm, addr, ptep, entry); + update_mmu_cache_range(vmf, vma, addr, ptep, 1); + mapped++; + if (addr == vmf->address) + ret = VM_FAULT_NOPAGE; + } + add_mm_counter(mm, mm_counter_file(folio), mapped); + return ret; +} + +static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slot = vmf->cow_subpage; + bool is_cow = (vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED); + struct folio *folio = page_folio(is_cow ? vmf->cow_page : vmf->page); + vm_fault_t ret; + pte_t entry; + + if (!(vma->vm_flags & VM_SHARED)) { + ret = check_stable_address_space(mm); + if (ret) + return ret; + } + if (pmd_none(*vmf->pmd)) { + if (vmf->prealloc_pte) + pmd_install(mm, vmf->pmd, &vmf->prealloc_pte); + else if (pte_alloc(mm, vmf->pmd)) + return VM_FAULT_OOM; + } + /* The private destination is not yet visible through a PTE. */ + if (is_cow) { + folio_get(folio); + folio_lock(folio); + } + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + ret = VM_FAULT_NOPAGE; + if (!vmf->pte) + goto unlock_folio; + if (vmf_pte_changed(vmf)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock; + } + if (slot) { + mm_subpage_set_exclusive(slot); + if (WARN_ON_ONCE(mm_subpage_add_new_anon_rmap(slot, vma, vmf->address))) { + ret = VM_FAULT_SIGBUS; + goto unlock; + } + entry = phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot); + entry = maybe_mkwrite(pte_mkdirty(pte_sw_mkyoung(entry)), vma); + if (vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + subpage_add_new_lru(folio, vma, 1); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + inc_mm_counter(mm, MM_ANONPAGES); + vmf->cow_subpage = NULL; + ret = 0; + } else if (is_cow) { + if (!folio_user_range_valid(folio, 0, mm_page_size(mm))) { + ret = VM_FAULT_SIGBUS; + goto unlock; + } + entry = phys_pte_mm(mm, page_to_phys(&folio->page), vma->vm_page_prot); + entry = maybe_mkwrite(pte_mkdirty(pte_sw_mkyoung(entry)), vma); + if (vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_ref_add(folio, mm_pte_native_pages(mm) - 1); + folio_add_new_anon_rmap(folio, vma, vmf->address, RMAP_EXCLUSIVE); + folio_add_lru_vma(folio, vma); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + inc_mm_counter(mm, MM_ANONPAGES); + ret = 0; + } else { + if (vma->vm_file) + ret = map_file_user_pages(vmf, folio, folio->index, + folio_next_index(folio) - 1, true); + else if (mm_page_size(mm) < PAGE_SIZE) + ret = map_special_subpages(vmf); + else + ret = VM_FAULT_SIGBUS; + if (ret & VM_FAULT_NOPAGE) { + folio_put(folio); /* Consume the fault callback's reference. */ + ret = 0; + } else { + ret = VM_FAULT_SIGBUS; + } + } +unlock: + pte_unmap_unlock(vmf->pte, vmf->ptl); +unlock_folio: + if (is_cow) { + folio_unlock(folio); + folio_put(folio); + } + return ret; +} +#endif + /** * finish_fault - finish page fault once we have prepared the page to fault * @@ -5663,6 +6985,11 @@ vm_fault_t finish_fault(struct vm_fault *vmf) unsigned long addr; bool needs_fallback = false; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return finish_user_page_fault(vmf); +#endif + fallback: addr = vmf->address; @@ -5768,13 +7095,12 @@ vm_fault_t finish_fault(struct vm_fault *vmf) return ret; } -static unsigned long fault_around_pages __read_mostly = - 65536 >> PAGE_SHIFT; +static unsigned long fault_around_bytes __read_mostly = SZ_64K; #ifdef CONFIG_DEBUG_FS static int fault_around_bytes_get(void *data, u64 *val) { - *val = fault_around_pages << PAGE_SHIFT; + *val = READ_ONCE(fault_around_bytes); return 0; } @@ -5788,11 +7114,13 @@ static int fault_around_bytes_set(void *data, u64 val) return -EINVAL; /* - * The minimum value is 1 page, however this results in no fault-around - * at all. See should_fault_around(). + * Keep the policy in bytes: one native page can still contain several + * user leaves. The smallest supported leaf disables fault-around for + * every mm; larger values are compared with each mm's own granule. */ - val = max(val, PAGE_SIZE); - fault_around_pages = rounddown_pow_of_two(val) >> PAGE_SHIFT; + val = max_t(u64, val, IS_ENABLED(CONFIG_MM_SUBPAGE) ? + MM_SUBPAGE_SIZE : PAGE_SIZE); + WRITE_ONCE(fault_around_bytes, rounddown_pow_of_two(val)); return 0; } @@ -5819,24 +7147,42 @@ late_initcall(fault_around_debugfs); * This function doesn't cross VMA or page table boundaries, in order to call * map_pages() and acquire a PTE lock only once. * - * fault_around_pages defines how many pages we'll try to map. - * do_fault_around() expects it to be set to a power of two less than or equal - * to PTRS_PER_PTE. - * - * The virtual address of the area that we map is naturally aligned to - * fault_around_pages * PAGE_SIZE rounded down to the machine page size - * (and therefore to page order). This way it's easier to guarantee - * that we don't cross page table boundaries. + * fault_around_bytes is a power-of-two byte policy, independent of either + * physical or user granules. The native path converts it to native pages; + * the alternative path clamps it to the target mm's PTE-table coverage. + * Natural alignment keeps the window inside one page table. */ static vm_fault_t do_fault_around(struct vm_fault *vmf) { - pgoff_t nr_pages = READ_ONCE(fault_around_pages); + unsigned long configured_bytes = READ_ONCE(fault_around_bytes); + pgoff_t nr_pages = max(1UL, configured_bytes >> PAGE_SHIFT); pgoff_t pte_off = pte_index(vmf->address); /* The page offset of vmf->address within the VMA. */ pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff; pgoff_t from_pte, to_pte; vm_fault_t ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) != PAGE_SIZE) { + struct vm_area_struct *vma = vmf->vma; + unsigned long bytes = min(configured_bytes, + pmd_size_mm(vma->vm_mm)); + unsigned long start = max(ALIGN_DOWN(vmf->address, bytes), vma->vm_start); + unsigned long end = min(ALIGN_DOWN(vmf->address, bytes) + bytes, vma->vm_end); + + if (pmd_none(*vmf->pmd)) { + vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); + if (!vmf->prealloc_pte) + return VM_FAULT_OOM; + } + rcu_read_lock(); + ret = vma->vm_ops->map_pages(vmf, vma_page_offset_at(vma, start).index, + vma_page_offset_at(vma, end - 1).index); + rcu_read_unlock(); + return ret; + } +#endif + /* The PTE offset of the start address, clamped to the VMA. */ from_pte = max(ALIGN_DOWN(pte_off, nr_pages), pte_off - min(pte_off, vma_off)); @@ -5871,7 +7217,7 @@ static inline bool should_fault_around(struct vm_fault *vmf) return false; /* A single page implies no faulting 'around' at all. */ - return fault_around_pages > 1; + return READ_ONCE(fault_around_bytes) > mm_page_size(vmf->vma->vm_mm); } static vm_fault_t do_read_fault(struct vm_fault *vmf) @@ -5918,10 +7264,26 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) if (ret) return ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + vmf->cow_subpage = anon_subpage_prealloc(vma->vm_mm, vma, vmf->address); + if (IS_ERR(vmf->cow_subpage)) { + vmf->cow_subpage = NULL; + return VM_FAULT_OOM; + } + folio = mm_subpage_folio(vmf->cow_subpage); + goto allocated; + } +#endif + folio = folio_prealloc(vma->vm_mm, vma, vmf->address, false); if (!folio) return VM_FAULT_OOM; +#ifdef CONFIG_MM_SUBPAGE +allocated: +#endif + vmf->cow_page = &folio->page; ret = __do_fault(vmf); @@ -5930,10 +7292,38 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) if (ret & VM_FAULT_DONE_COW) return ret; - if (copy_mc_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma)) { - ret = VM_FAULT_HWPOISON; +#ifdef CONFIG_MM_SUBPAGE + if (vmf->cow_subpage) { + if (copy_user_subpage_range(vmf->cow_page, mm_subpage_offset(vmf->cow_subpage), + vmf->page, vma_page_offset_at(vma, vmf->address).offset, + mm_page_size(vma->vm_mm))) { + ret = VM_FAULT_HWPOISON; + goto unlock; + } + goto copied; + } +#endif + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) > PAGE_SIZE && + !folio_user_range_valid(page_folio(vmf->page), + folio_page_idx(page_folio(vmf->page), vmf->page) << PAGE_SHIFT, + mm_page_size(vma->vm_mm))) { + ret = VM_FAULT_SIGBUS; goto unlock; } +#endif + for (unsigned int i = 0; i < mm_pte_native_pages(vma->vm_mm); i++) { + if (copy_mc_user_highpage(vmf->cow_page + i, vmf->page + i, + vmf->address + i * PAGE_SIZE, vma)) { + ret = VM_FAULT_HWPOISON; + goto unlock; + } + } +#ifdef CONFIG_MM_SUBPAGE +copied: +#endif + __folio_mark_uptodate(folio); ret |= finish_fault(vmf); @@ -5944,6 +7334,13 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) goto uncharge_out; return ret; uncharge_out: +#ifdef CONFIG_MM_SUBPAGE + if (vmf->cow_subpage) { + mm_subpage_put(vmf->cow_subpage); + vmf->cow_subpage = NULL; + return ret; + } +#endif folio_put(folio); return ret; } @@ -6104,31 +7501,33 @@ static void numa_rebuild_single_mapping(struct vm_fault *vmf, struct vm_area_str update_mmu_cache_range(vmf, vma, fault_addr, fault_pte, 1); } -static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, +static void numa_rebuild_folio_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, struct folio *folio, pte_t fault_pte, bool ignore_writable, bool pte_write_upgrade) { - int nr = pte_pfn(fault_pte) - folio_pfn(folio); + struct mm_struct *mm = vma->vm_mm; + unsigned int page_shift = mm_page_shift(mm); + unsigned long page_size = 1UL << page_shift; + unsigned long offset = pte_phys_mm(mm, fault_pte) - PFN_PHYS(folio_pfn(folio)); unsigned long start, end, addr = vmf->address; - unsigned long addr_start = addr - (nr << PAGE_SHIFT); - unsigned long pt_start = ALIGN_DOWN(addr, PMD_SIZE); + unsigned long lower = max(addr & pmd_mask_mm(mm), vma->vm_start); + unsigned long upper = pmd_addr_end_mm(mm, addr, vma->vm_end); pte_t *start_ptep; - /* Stay within the VMA and within the page table. */ - start = max3(addr_start, pt_start, vma->vm_start); - end = min3(addr_start + folio_size(folio), pt_start + PMD_SIZE, - vma->vm_end); - start_ptep = vmf->pte - ((addr - start) >> PAGE_SHIFT); + /* Stay within the VMA, page table and physical owner without wrapping. */ + start = addr - min(offset, addr - lower); + end = addr + min(folio_size(folio) - offset, upper - addr); + start_ptep = vmf->pte - ((addr - start) >> page_shift); - /* Restore all PTEs' mapping of the large folio */ - for (addr = start; addr != end; start_ptep++, addr += PAGE_SIZE) { + /* Restore the owner's PTEs, including smaller leaves in a native folio. */ + for (addr = start; addr != end; start_ptep++, addr += page_size) { pte_t ptent = ptep_get(start_ptep); bool writable = false; if (!pte_present(ptent) || !pte_protnone(ptent)) continue; - if (pfn_folio(pte_pfn(ptent)) != folio) + if (pfn_folio(PHYS_PFN(pte_phys_mm(mm, ptent))) != folio) continue; if (!ignore_writable) { @@ -6221,8 +7620,8 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) * Make it present again, depending on how arch implements * non-accessible ptes, some can allow access by kernel mode. */ - if (folio && folio_test_large(folio)) - numa_rebuild_large_mapping(vmf, vma, folio, pte, ignore_writable, + if (folio && (folio_test_large(folio) || mm_page_shift(vma->vm_mm) < PAGE_SHIFT)) + numa_rebuild_folio_mapping(vmf, vma, folio, pte, ignore_writable, pte_write_upgrade); else numa_rebuild_single_mapping(vmf, vma, vmf->address, vmf->pte, @@ -6455,7 +7854,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, { struct vm_fault vmf = { .vma = vma, - .address = address & PAGE_MASK, + .address = address & mm_page_mask(vma->vm_mm), .real_address = address, .flags = flags, .pgoff = linear_page_index(vma, address), @@ -6786,7 +8185,7 @@ int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) return -ENOMEM; spin_lock(&mm->page_table_lock); - if (!p4d_present(*p4d)) { + if (!p4d_present_mm(mm, *p4d)) { mm_inc_nr_puds(mm); smp_wmb(); /* See comment in pmd_install() */ p4d_populate(mm, p4d, new); @@ -6824,16 +8223,19 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) static inline int pfnmap_args_setup(struct follow_pfnmap_args *args, spinlock_t *lock, pte_t *ptep, - pgprot_t pgprot, unsigned long pfn_base, + pgprot_t pgprot, phys_addr_t phys_base, unsigned long addr_mask, bool writable, bool special) { + phys_addr_t phys = phys_base + (args->address & ~addr_mask); + if (!writable && args->write) return -EFAULT; args->lock = lock; args->ptep = ptep; - args->pfn = pfn_base + ((args->address & ~addr_mask) >> PAGE_SHIFT); + args->pfn = PHYS_PFN(phys); + args->offset = offset_in_page(phys); args->addr_mask = addr_mask; args->pgprot = pgprot; args->writable = writable; @@ -6885,8 +8287,14 @@ static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma) * * This function must not be used to modify PTE content. * + * Legacy callers retain the native-page contract. A caller setting + * allow_subpage must use pfn plus offset to identify the requested physical + * byte and must limit access to the leaf described by addr_mask. This does + * not pin the native page or grant access to neighbouring user mappings. + * * Return: zero on success, -EFAULT if @args->write was set but the - * mapping is not writable, -EINVAL if there is no mapping at all. + * mapping is not writable, -EINVAL if there is no mapping at all, or + * -EOPNOTSUPP for a non-native granule without allow_subpage. */ int follow_pfnmap_start(struct follow_pfnmap_args *args) { @@ -6908,17 +8316,20 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) goto out; + /* A legacy PFN consumer cannot represent partial native-page ownership. */ + if (mm_page_size(mm) != PAGE_SIZE && !args->allow_subpage) + return -EOPNOTSUPP; retry: pgdp = pgd_offset(mm, address); if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) goto out; - p4dp = p4d_offset(pgdp, address); + p4dp = p4d_offset_mm(mm, pgdp, address); p4d = p4dp_get(p4dp); - if (p4d_none(p4d) || unlikely(p4d_bad(p4d))) + if (p4d_none_mm(mm, p4d) || unlikely(p4d_bad_mm(mm, p4d))) goto out; - pudp = pud_offset(p4dp, address); + pudp = pud_offset_mm(mm, p4dp, address); pud = pudp_get(pudp); if (!pud_present(pud)) goto out; @@ -6934,14 +8345,14 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) goto retry; } r = pfnmap_args_setup(args, lock, NULL, pud_pgprot(pud), - pud_pfn(pud), PUD_MASK, pud_write(pud), + PFN_PHYS(pud_pfn(pud)), pud_mask_mm(mm), pud_write(pud), pud_special(pud)); if (r) spin_unlock(lock); return r; } - pmdp = pmd_offset(pudp, address); + pmdp = pmd_offset_mm(mm, pudp, address); pmd = pmdp_get_lockless(pmdp); if (!pmd_present(pmd)) goto out; @@ -6957,7 +8368,7 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) goto retry; } r = pfnmap_args_setup(args, lock, NULL, pmd_pgprot(pmd), - pmd_pfn(pmd), PMD_MASK, pmd_write(pmd), + PFN_PHYS(pmd_pfn(pmd)), pmd_mask_mm(mm), pmd_write(pmd), pmd_special(pmd)); if (r) spin_unlock(lock); @@ -6970,8 +8381,8 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) pte = ptep_get(ptep); if (!pte_present(pte)) goto unlock; - r = pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte), - pte_pfn(pte), PAGE_MASK, pte_write(pte), + r = pfnmap_args_setup(args, lock, ptep, pte_pgprot_mm(mm, pte), + pte_phys_mm(mm, pte), mm_page_mask(mm), pte_write(pte), pte_special(pte)); if (r) goto unlock; @@ -7018,20 +8429,30 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, resource_size_t phys_addr; pgprot_t prot = __pgprot(0); void __iomem *maddr; - int offset = offset_in_page(addr); + int offset; int ret = -EINVAL; bool writable; struct follow_pfnmap_args args = { .vma = vma, .address = addr, - .write = !!(write & FOLL_WRITE) + .write = !!(write & FOLL_WRITE), + .allow_subpage = true }; + if (len <= 0) + return 0; + if (addr < vma->vm_start || addr >= vma->vm_end) + return -EINVAL; + /* Each following leaf needs its own translation and permission check. */ + len = min_t(unsigned long, len, vma->vm_end - addr); + len = min_t(unsigned long, len, mm_page_size(vma->vm_mm) - + (addr & ~mm_page_mask(vma->vm_mm))); retry: if (follow_pfnmap_start(&args)) return -EINVAL; prot = args.pgprot; phys_addr = (resource_size_t)args.pfn << PAGE_SHIFT; + offset = args.offset; writable = args.writable; follow_pfnmap_end(&args); @@ -7043,7 +8464,8 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, goto out_unmap; if ((pgprot_val(prot) != pgprot_val(args.pgprot)) || - (phys_addr != (args.pfn << PAGE_SHIFT)) || + (phys_addr != ((resource_size_t)args.pfn << PAGE_SHIFT)) || + (offset != args.offset) || (writable != args.writable)) { follow_pfnmap_end(&args); iounmap(maddr); @@ -7089,8 +8511,19 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, void *maddr; struct folio *folio; struct vm_area_struct *vma = NULL; - struct page *page = get_user_page_vma_remote(mm, addr, - gup_flags, &vma); + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment = {}; + + if (mm_page_size(mm) != PAGE_SIZE) { + int got = get_user_fragment_vma_remote(mm, addr, len, + gup_flags, &fragment, &vma); + + page = got > 0 ? folio_page(fragment.folio, + fragment.offset >> PAGE_SHIFT) : ERR_PTR(got ?: -EFAULT); + } else +#endif + page = get_user_page_vma_remote(mm, addr, gup_flags, &vma); if (IS_ERR(page)) { /* We might need to expand the stack to access it */ @@ -7124,6 +8557,12 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, offset = addr & (PAGE_SIZE-1); if (bytes > PAGE_SIZE-offset) bytes = PAGE_SIZE-offset; +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) { + offset = offset_in_page(fragment.offset); + bytes = fragment.length; + } +#endif maddr = kmap_local_folio(folio, folio_page_idx(folio, page) * PAGE_SIZE); if (write) { @@ -7134,7 +8573,13 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, copy_from_user_page(vma, page, addr, buf, maddr + offset, bytes); } - folio_release_kmap(folio, maddr); +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) { + kunmap_local(maddr); + release_user_fragments(&fragment, 1, false); + } else +#endif + folio_release_kmap(folio, maddr); } len -= bytes; buf += bytes; diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 4e4421b22b59f..3b97c71c9d3cf 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -78,6 +78,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -688,6 +689,8 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, struct folio *folio; struct queue_pages *qp = walk->private; unsigned long flags = qp->flags; + unsigned int page_shift = mm_page_shift(walk->mm); + unsigned long page_size = 1UL << page_shift; pte_t *pte, *mapped_pte; pte_t ptent; spinlock_t *ptl; @@ -705,8 +708,8 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte += nr, addr += nr * PAGE_SIZE) { - max_nr = (end - addr) >> PAGE_SHIFT; + for (; addr != end; pte += nr, addr += nr * page_size) { + max_nr = (end - addr) >> page_shift; nr = 1; ptent = ptep_get(pte); if (pte_none(ptent)) @@ -721,7 +724,11 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, folio = vm_normal_folio(vma, addr, ptent); if (!folio || folio_is_zone_device(folio)) continue; - if (folio_test_large(folio) && max_nr != 1) +#ifdef CONFIG_MM_SUBPAGE + if (page_shift < PAGE_SHIFT) + nr = folio_subpage_pte_batch(vma, folio, pte, ptent, max_nr, 0); +#endif + if (page_shift >= PAGE_SHIFT && folio_test_large(folio) && max_nr != 1) nr = folio_pte_batch(folio, pte, ptent, max_nr); /* * vm_normal_folio() filters out zero pages, but there might @@ -731,9 +738,10 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, continue; if (!queue_folio_required(folio, qp)) continue; - if (folio_test_large(folio)) { + if (folio_test_large(folio) || page_shift < PAGE_SHIFT) { /* - * A large folio can only be isolated from LRU once, + * A native folio backing smaller leaves, like a large + * folio, can only be isolated from LRU once, * but may be mapped by many PTEs (and Copy-On-Write may * intersperse PTEs of other, order 0, folios). This is * a common case, so don't mistake it for failure (but @@ -755,7 +763,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) || !vma_migratable(vma) || !migrate_folio_add(folio, qp->pagelist, flags)) { - qp->nr_failed += nr; + qp->nr_failed += page_shift < PAGE_SHIFT ? 1 : nr; if (strictly_unmovable(flags)) break; } @@ -824,6 +832,7 @@ static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask, * @folio: The folio whose mapping considered for being made NUMA hintable * @vma: The VMA that the folio belongs to. * @is_private_single_threaded: Is this a single-threaded private VMA or not + * @shared: Is the mapped memory potentially shared between address spaces * * This function checks to see if the folio actually indicates that * we need to make the mapping one which causes a NUMA hinting fault, @@ -833,7 +842,7 @@ static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask, * Return: True if the mapping of the folio needs to be changed, false otherwise. */ bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, - bool is_private_single_threaded) + bool is_private_single_threaded, bool shared) { int nid; @@ -841,7 +850,7 @@ bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, return false; /* Also skip shared copy-on-write folios */ - if (is_cow_mapping(vma->vm_flags) && folio_maybe_mapped_shared(folio)) + if (is_cow_mapping(vma->vm_flags) && shared) return false; /* Folios are pinned and can't be migrated */ @@ -1050,6 +1059,12 @@ static int mbind_range(struct vma_iterator *vmi, struct vm_area_struct *vma, vmstart = vma->vm_start; } + /* Shared policy is indexed by native page-cache allocation units. */ + if (vma_is_shmem(vma) && + (vma_page_offset_at(vma, vmstart).offset || + ((vmend - vmstart) & ~PAGE_MASK))) + return -EINVAL; + if (mpol_equal(vma->vm_policy, new_pol)) { *prev = vma; return 0; @@ -1135,6 +1150,18 @@ static int lookup_node(struct mm_struct *mm, unsigned long addr) struct page *p = NULL; int ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_shift(mm) != PAGE_SHIFT) { + struct user_page_fragment fragment; + + ret = get_user_fragments_remote(mm, addr, 1, 0, &fragment, 1); + if (ret > 0) { + ret = folio_nid(fragment.folio); + release_user_fragments(&fragment, 1, false); + } + return ret; + } +#endif ret = get_user_pages_fast(addr & PAGE_MASK, 1, 0, &p); if (ret > 0) { ret = page_to_nid(p); @@ -1502,13 +1529,13 @@ static long do_mbind(unsigned long start, unsigned long len, if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) return -EPERM; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; if (mode == MPOL_DEFAULT) flags &= ~MPOL_MF_STRICT; - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); end = start + len; if (end < start) @@ -1768,7 +1795,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le VMA_ITERATOR(vmi, mm, start); start = untagged_addr(start); - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; /* * flags is used for future extension if any. @@ -1783,7 +1810,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le if (home_node >= MAX_NUMNODES || !node_online(home_node)) return -EINVAL; - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); end = start + len; if (end < start) @@ -2049,7 +2076,7 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, if (pol->mode == MPOL_INTERLEAVE || pol->mode == MPOL_WEIGHTED_INTERLEAVE) { *ilx += vma->vm_pgoff >> order; - *ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order); + *ilx += (vma_page_offset_at(vma, addr).index - vma->vm_pgoff) >> order; } return pol; } @@ -3254,6 +3281,10 @@ int mpol_set_shared_policy(struct shared_policy *sp, struct sp_node *new = NULL; unsigned long sz = vma_pages(vma); + /* Never truncate a partial cache-page policy update to zero or a neighbour. */ + if (vma_subpage_offset(vma) || ((vma->vm_end - vma->vm_start) & ~PAGE_MASK)) + return -EINVAL; + if (pol) { new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, pol); if (!new) diff --git a/mm/migrate.c b/mm/migrate.c index 8a64291ab5b44..795143e1b4cc4 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -14,6 +14,8 @@ */ #include +#include +#include #include #include #include @@ -340,14 +342,50 @@ struct rmap_walk_arg { bool map_unused_to_zeropage; }; +#ifdef CONFIG_MM_SUBPAGE +static void remove_migration_subpage(struct folio *folio, + struct page_vma_mapped_walk *pvmw, pte_t old) +{ + struct vm_area_struct *vma = pvmw->vma; + softleaf_t entry = softleaf_from_pte(old); + struct mm_subpage *slot; + pte_t pte; + int ret; + + slot = mm_subpage_migrate_restore(folio, pte_swp_subpage_offset(old)); + pte = phys_pte_mm(vma->vm_mm, mm_subpage_phys(slot), READ_ONCE(vma->vm_page_prot)); + if (!softleaf_is_migration_young(entry)) + pte = pte_mkold(pte); + if (folio_test_dirty(folio) && softleaf_is_migration_dirty(entry)) + pte = pte_mkdirty(pte); + if (pte_swp_soft_dirty(old)) + pte = pte_mksoft_dirty(pte); + else + pte = pte_clear_soft_dirty(pte); + if (softleaf_is_migration_write(entry)) + pte = pte_mkwrite(pte, vma); + else if (pte_swp_uffd_wp(old)) + pte = pte_mkuffd_wp(pte); + if (!softleaf_is_migration_read(entry)) + mm_subpage_set_exclusive(slot); + ret = mm_subpage_add_anon_rmap(slot, vma, pvmw->address); + VM_BUG_ON_FOLIO(ret, folio); + set_pte_at(vma->vm_mm, pvmw->address, pvmw->pte, pte); + if (READ_ONCE(vma->vm_flags) & VM_LOCKED) + mlock_drain_local(); + trace_remove_migration_pte(pvmw->address, pte_val(pte), 0); + update_mmu_cache(vma, pvmw->address, pvmw->pte); +} +#endif + /* * Restore a potential migration pte to a working pte entry */ static bool remove_migration_pte(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { struct rmap_walk_arg *rmap_walk_arg = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION); + DEFINE_FOLIO_RMAP_WALK(pvmw, rmap_walk_arg->folio, vma, rmap, PVMW_SYNC | PVMW_MIGRATION); while (page_vma_mapped_walk(&pvmw)) { rmap_t rmap_flags = RMAP_NONE; @@ -372,13 +410,29 @@ static bool remove_migration_pte(struct folio *folio, } #endif old_pte = ptep_get(pvmw.pte); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE && folio_test_anon(folio)) { + remove_migration_subpage(folio, &pvmw, old_pte); + continue; + } +#endif if (rmap_walk_arg->map_unused_to_zeropage && try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx)) continue; - folio_get(folio); + /* Match the native-page references dropped by try_to_migrate_one(). */ + if (!folio_test_hugetlb(folio)) + folio_ref_add(folio, mm_pte_native_pages(vma->vm_mm)); + else + folio_get(folio); pte = mk_pte(new, READ_ONCE(vma->vm_page_prot)); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + pte = phys_pte_mm(vma->vm_mm, page_to_phys(new) + + pte_swp_subpage_offset(old_pte), READ_ONCE(vma->vm_page_prot)); +#endif + entry = softleaf_from_pte(old_pte); if (!softleaf_is_migration_young(entry)) pte = pte_mkold(pte); @@ -431,8 +485,10 @@ static bool remove_migration_pte(struct folio *folio, if (folio_test_anon(folio)) folio_add_anon_rmap_pte(folio, new, vma, pvmw.address, rmap_flags); - else + else { + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, pte)); folio_add_file_rmap_pte(folio, new, vma); + } set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); } if (READ_ONCE(vma->vm_flags) & VM_LOCKED) @@ -640,10 +696,12 @@ static int __folio_migrate_mapping(struct address_space *mapping, folio_set_dirty(newfolio); } - if (folio_test_swapcache(folio)) + if (folio_test_swapcache(folio)) { __swap_cache_replace_folio(ci, folio, newfolio); - else + } else { + shmem_uffd_migrate(newfolio, folio); xas_store(&xas, newfolio); + } /* * Drop cache reference from old folio by unfreezing @@ -1170,8 +1228,10 @@ static void migrate_folio_undo_src(struct folio *src, /* Drop an anon_vma reference if we took one */ if (anon_vma) put_anon_vma(anon_vma); - if (locked) + if (locked) { + mm_subpage_migrate_finish(src); folio_unlock(src); + } if (ret) list_move_tail(&src->lru, ret); } @@ -1180,8 +1240,10 @@ static void migrate_folio_undo_src(struct folio *src, static void migrate_folio_undo_dst(struct folio *dst, bool locked, free_folio_t put_new_folio, unsigned long private) { - if (locked) + if (locked) { + mm_subpage_migrate_finish(dst); folio_unlock(dst); + } if (put_new_folio) put_new_folio(dst, private); else @@ -1306,6 +1368,11 @@ static int migrate_folio_unmap(new_folio_t get_new_folio, return 0; } + rc = mm_subpage_migrate_prepare(src, dst); + if (rc) + goto out; + rc = -EAGAIN; + /* * Corner case handling: * 1. When a new swap-cache page is read into, it is added to the LRU @@ -1411,6 +1478,8 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private, remove_migration_ptes(src, dst, 0); out_unlock_both: + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); folio_unlock(dst); folio_set_owner_migrate_reason(dst, reason); /* diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 17074c2573b94..bae35a327cf1b 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -408,7 +408,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, if (anon_exclusive) { pte = ptep_clear_flush(vma, addr, ptep); - if (folio_try_share_anon_rmap_pte(folio, page)) { + if (folio_try_share_anon_rmap_pte(folio, page, vma)) { set_pte_at(mm, addr, ptep, pte); if (fault_folio != folio) folio_unlock(folio); diff --git a/mm/mincore.c b/mm/mincore.c index c8757c5085bf9..66458b6bb2091 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -48,7 +48,7 @@ static int mincore_hugetlb(pte_t *pte, unsigned long hmask, unsigned long addr, present = 1; } - for (; addr != end; vec++, addr += PAGE_SIZE) + for (; addr != end; vec++, addr += mm_page_size(walk->mm)) *vec = present; walk->private = vec; spin_unlock(ptl); @@ -135,15 +135,22 @@ static unsigned char mincore_page(struct address_space *mapping, pgoff_t index) static int __mincore_unmapped_range(unsigned long addr, unsigned long end, struct vm_area_struct *vma, unsigned char *vec) { - unsigned long nr = (end - addr) >> PAGE_SHIFT; + unsigned long nr = (end - addr) >> mm_page_shift(vma->vm_mm); int i; if (vma->vm_file) { - pgoff_t pgoff; + unsigned int shift = mm_page_shift(vma->vm_mm); - pgoff = linear_page_index(vma, addr); - for (i = 0; i < nr; i++, pgoff++) - vec[i] = mincore_page(vma->vm_file->f_mapping, pgoff); + /* Cache indices are native; several user leaves can share one. */ + for (i = 0; i < nr; ) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + unsigned long batch = min(nr - i, (PAGE_SIZE - pos.offset) >> shift); + unsigned char present = mincore_page(vma->vm_file->f_mapping, pos.index); + + memset(vec + i, present, batch); + i += batch; + addr += batch << shift; + } } else { for (i = 0; i < nr; i++) vec[i] = 0; @@ -167,7 +174,7 @@ static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct vm_area_struct *vma = walk->vma; pte_t *ptep; unsigned char *vec = walk->private; - int nr = (end - addr) >> PAGE_SHIFT; + int nr = (end - addr) >> mm_page_shift(walk->mm); int step, i; ptl = pmd_trans_huge_lock(pmd, vma); @@ -182,19 +189,19 @@ static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; ptep += step, addr += step * PAGE_SIZE) { + for (; addr != end; ptep += step, addr += step * mm_page_size(walk->mm)) { pte_t pte = ptep_get(ptep); step = 1; /* We need to do cache lookup too for markers */ if (pte_none(pte) || pte_is_marker(pte)) - __mincore_unmapped_range(addr, addr + PAGE_SIZE, + __mincore_unmapped_range(addr, addr + mm_page_size(walk->mm), vma, vec); else if (pte_present(pte)) { unsigned int batch = pte_batch_hint(ptep, pte); if (batch > 1) { - unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + unsigned int max_nr = (end - addr) >> mm_page_shift(walk->mm); step = min_t(unsigned int, batch, max_nr); } @@ -252,16 +259,16 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v vma = vma_lookup(current->mm, addr); if (!vma) return -ENOMEM; - end = min(vma->vm_end, addr + (pages << PAGE_SHIFT)); + end = min(vma->vm_end, addr + (pages << mm_page_shift(vma->vm_mm))); if (!can_do_mincore(vma)) { - unsigned long pages = DIV_ROUND_UP(end - addr, PAGE_SIZE); + unsigned long pages = DIV_ROUND_UP(end - addr, mm_page_size(vma->vm_mm)); memset(vec, 1, pages); return pages; } err = walk_page_range(vma->vm_mm, addr, end, &mincore_walk_ops, vec); if (err < 0) return err; - return (end - addr) >> PAGE_SHIFT; + return (end - addr) >> mm_page_shift(vma->vm_mm); } /* @@ -281,7 +288,7 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v * return values: * zero - success * -EFAULT - vec points to an illegal address - * -EINVAL - addr is not a multiple of PAGE_SIZE + * -EINVAL - addr is not aligned to the process page size * -ENOMEM - Addresses in the range [addr, addr + len] are * invalid for the address space of this process, or * specify one or more pages which are not currently @@ -298,7 +305,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, start = untagged_addr(start); /* Check the start address: needs to be page-aligned.. */ - if (unlikely(start & ~PAGE_MASK)) + if (unlikely(start & ~mm_page_mask(current->mm))) return -EINVAL; /* ..and we need to be passed a valid user-space range */ @@ -306,8 +313,8 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, return -ENOMEM; /* This also avoids any overflows on PAGE_ALIGN */ - pages = len >> PAGE_SHIFT; - pages += (offset_in_page(len)) != 0; + pages = len >> mm_page_shift(current->mm); + pages += (len & ~mm_page_mask(current->mm)) != 0; if (!access_ok(vec, pages)) return -EFAULT; @@ -334,7 +341,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, } pages -= retval; vec += retval; - start += retval << PAGE_SHIFT; + start += retval << mm_page_shift(current->mm); retval = 0; } free_page((unsigned long) tmp); diff --git a/mm/mlock.c b/mm/mlock.c index 8c227fefa2df0..a6b67d6b20cac 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -304,15 +304,19 @@ void munlock_folio(struct folio *folio) local_unlock(&mlock_fbatch.lock); } -static inline unsigned int folio_mlock_step(struct folio *folio, +static inline unsigned int folio_mlock_step(struct vm_area_struct *vma, struct folio *folio, pte_t *pte, unsigned long addr, unsigned long end) { - unsigned int count = (end - addr) >> PAGE_SHIFT; + unsigned int count = (end - addr) >> mm_page_shift(vma->vm_mm); pte_t ptent = ptep_get(pte); if (!folio_test_large(folio)) return 1; + #ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch(vma, folio, pte, ptent, count, 0); +#endif return folio_pte_batch(folio, pte, ptent, count); } @@ -341,7 +345,7 @@ static inline bool allow_mlock_munlock(struct folio *folio, return false; /* folio is not fully mapped, skip mlock */ - if (step != folio_nr_pages(folio)) + if (step != (folio_size(folio) >> mm_page_shift(vma->vm_mm))) return false; return true; @@ -381,7 +385,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, return 0; } - for (pte = start_pte; addr != end; pte++, addr += PAGE_SIZE) { + for (pte = start_pte; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) { ptent = ptep_get(pte); if (!pte_present(ptent)) continue; @@ -389,7 +393,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, if (!folio || folio_is_zone_device(folio)) continue; - step = folio_mlock_step(folio, pte, addr, end); + step = folio_mlock_step(vma, folio, pte, addr, end); if (!allow_mlock_munlock(folio, vma, start, end, step)) goto next_entry; @@ -400,7 +404,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, next_entry: pte += step - 1; - addr += (step - 1) << PAGE_SHIFT; + addr += (step - 1) << mm_page_shift(vma->vm_mm); } pte_unmap(start_pte); out: @@ -471,7 +475,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags); const vma_flags_t old_vma_flags = vma->flags; struct mm_struct *mm = vma->vm_mm; - int nr_pages; + long nr_pages; int ret = 0; if (vma_flags_same_pair(&old_vma_flags, &new_vma_flags) || @@ -492,7 +496,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, /* * Keep track of amount of locked VM. */ - nr_pages = (end - start) >> PAGE_SHIFT; + nr_pages = (end - start) >> MM_ACCOUNT_SHIFT; if (!vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT)) nr_pages = -nr_pages; else if (vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) @@ -524,8 +528,8 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, struct vm_area_struct *vma, *prev; VMA_ITERATOR(vmi, current->mm, start); - VM_BUG_ON(offset_in_page(start)); - VM_BUG_ON(len != PAGE_ALIGN(len)); + VM_BUG_ON((start & ~mm_page_mask(current->mm))); + VM_BUG_ON(len != mm_page_align(current->mm, len)); end = start + len; if (end < start) return -EINVAL; @@ -572,7 +576,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, * vma pages, as return value. * Note deferred memory locking case(mlock2(,,MLOCK_ONFAULT) * is also counted. - * Return value: previously mlocked page counts + * Return value: previously mlocked MM_ACCOUNT_SHIFT units */ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, unsigned long start, size_t len) @@ -600,7 +604,7 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, } } - return count >> PAGE_SHIFT; + return count >> MM_ACCOUNT_SHIFT; } /* @@ -626,12 +630,12 @@ static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t fla if (!can_do_mlock()) return -EPERM; - len = PAGE_ALIGN(len + (offset_in_page(start))); - start &= PAGE_MASK; + len = mm_page_align(current->mm, len + (start & ~mm_page_mask(current->mm))); + start &= mm_page_mask(current->mm); lock_limit = rlimit(RLIMIT_MEMLOCK); - lock_limit >>= PAGE_SHIFT; - locked = len >> PAGE_SHIFT; + lock_limit >>= MM_ACCOUNT_SHIFT; + locked = len >> MM_ACCOUNT_SHIFT; if (mmap_write_lock_killable(current->mm)) return -EINTR; @@ -686,8 +690,8 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) start = untagged_addr(start); - len = PAGE_ALIGN(len + (offset_in_page(start))); - start &= PAGE_MASK; + len = mm_page_align(current->mm, len + (start & ~mm_page_mask(current->mm))); + start &= mm_page_mask(current->mm); if (mmap_write_lock_killable(current->mm)) return -EINTR; @@ -761,7 +765,7 @@ SYSCALL_DEFINE1(mlockall, int, flags) return -EPERM; lock_limit = rlimit(RLIMIT_MEMLOCK); - lock_limit >>= PAGE_SHIFT; + lock_limit >>= mm_page_shift(current->mm); if (mmap_write_lock_killable(current->mm)) return -EINTR; diff --git a/mm/mm_init.c b/mm/mm_init.c index c7e498fc1c4c0..8d14ce83be36b 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -57,7 +57,8 @@ unsigned long zero_page_pfn __ro_after_init; EXPORT_SYMBOL(zero_page_pfn); #ifndef __HAVE_COLOR_ZERO_PAGE -uint8_t empty_zero_page[PAGE_SIZE] __page_aligned_bss; +uint8_t empty_zero_page[ARCH_ZERO_PAGE_SIZE] __page_aligned_bss + __aligned(ARCH_ZERO_PAGE_SIZE); EXPORT_SYMBOL(empty_zero_page); struct page *__zero_page __ro_after_init; diff --git a/mm/mmap.c b/mm/mmap.c index 2311ae7c2ff45..097161e3e61ef 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -151,8 +151,8 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) mm->end_data, mm->start_data)) goto out; - newbrk = PAGE_ALIGN(brk); - oldbrk = PAGE_ALIGN(mm->brk); + newbrk = mm_page_align(mm, brk); + oldbrk = mm_page_align(mm, mm->brk); if (oldbrk == newbrk) { mm->brk = brk; goto success; @@ -186,8 +186,8 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) * expansion area */ vma_iter_init(&vmi, mm, oldbrk); - next = vma_find(&vmi, newbrk + PAGE_SIZE + stack_guard_gap); - if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) + next = vma_find(&vmi, newbrk + mm_page_size(mm) + stack_guard_gap); + if (next && newbrk + mm_page_size(mm) > vm_start_gap(next)) goto out; brkvma = vma_prev_limit(&vmi, mm->start_brk); @@ -218,12 +218,12 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) * If a hint addr is less than mmap_min_addr change hint to be as * low as possible but still greater than mmap_min_addr */ -static inline unsigned long round_hint_to_min(unsigned long hint) +static inline unsigned long round_hint_to_min(struct mm_struct *mm, unsigned long hint) { - hint &= PAGE_MASK; + hint &= mm_page_mask(mm); if (((void *)hint != NULL) && (hint < mmap_min_addr)) - return PAGE_ALIGN(mmap_min_addr); + return mm_page_align(mm, mmap_min_addr); return hint; } @@ -235,11 +235,11 @@ bool mlock_future_ok(const struct mm_struct *mm, bool is_vma_locked, if (!is_vma_locked || capable(CAP_IPC_LOCK)) return true; - locked_pages = bytes >> PAGE_SHIFT; + locked_pages = bytes >> MM_ACCOUNT_SHIFT; locked_pages += mm->locked_vm; limit_pages = rlimit(RLIMIT_MEMLOCK); - limit_pages >>= PAGE_SHIFT; + limit_pages >>= MM_ACCOUNT_SHIFT; return locked_pages <= limit_pages; } @@ -264,24 +264,25 @@ static inline u64 file_mmap_size_max(struct file *file, struct inode *inode) } static inline bool file_mmap_ok(struct file *file, struct inode *inode, - unsigned long pgoff, unsigned long len) + struct vm_page_offset pos, unsigned long len) { u64 maxsize = file_mmap_size_max(file, inode); if (maxsize && len > maxsize) return false; maxsize -= len; - if (pgoff > maxsize >> PAGE_SHIFT) + if (pos.index > maxsize >> PAGE_SHIFT || + (pos.index == maxsize >> PAGE_SHIFT && pos.offset > (maxsize & ~PAGE_MASK))) return false; return true; } /** - * do_mmap() - Perform a userland memory mapping into the current process + * do_mmap_internal() - Perform a userland memory mapping into the current process * address space of length @len with protection bits @prot, mmap flags @flags * (from which VMA flags will be inferred), and any additional VMA flags to * apply @vm_flags. If this is a file-backed mapping then the file is specified - * in @file and page offset into the file via @pgoff. + * in @file and its byte position via @pos. * * This function does not perform security checks on the file and assumes, if * @uf is non-NULL, the caller has provided a list head to track unmap events @@ -321,7 +322,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * @flags: Flags specifying how the mapping should be performed, see mmap (2) * for details. * @vm_flags: VMA flags which should be set by default, or 0 otherwise. - * @pgoff: Page offset into the @file if file-backed, should be 0 otherwise. + * @pos: Native cache index and byte offset; ignored for anonymous mappings. * @populate: A pointer to a value which will be set to 0 if no population of * the range is required, or the number of bytes to populate if it is. Must be * non-NULL. See mmap (2) for details as to under what circumstances population @@ -333,20 +334,22 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * Returns: Either an error, or the address at which the requested mapping has * been performed. */ -unsigned long do_mmap(struct file *file, unsigned long addr, +static unsigned long do_mmap_internal(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, vm_flags_t vm_flags, - unsigned long pgoff, unsigned long *populate, + struct vm_page_offset pos, unsigned long *populate, struct list_head *uf) { struct mm_struct *mm = current->mm; + unsigned long pgoff = pos.index; int pkey = 0; *populate = 0; mmap_assert_write_locked(mm); - if (!len) + if (!len || pos.offset >= PAGE_SIZE || + !IS_ALIGNED(pos.offset, mm_page_size(mm))) return -EINVAL; /* @@ -364,15 +367,15 @@ unsigned long do_mmap(struct file *file, unsigned long addr, flags |= MAP_FIXED; if (!(flags & MAP_FIXED)) - addr = round_hint_to_min(addr); + addr = round_hint_to_min(mm, addr); /* Careful about overflows.. */ - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); if (!len) return -ENOMEM; /* offset overflow? */ - if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) + if (vm_page_offset_add(pos, len).index < pgoff) return -EOVERFLOW; /* Too many mappings? */ @@ -426,7 +429,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long flags_mask; int err; - if (!file_mmap_ok(file, inode, pgoff, len)) + if (!file_mmap_ok(file, inode, pos, len)) return -EOVERFLOW; flags_mask = LEGACY_MAP_MASK; @@ -500,6 +503,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * Ignore pgoff. */ pgoff = 0; + pos.offset = 0; vm_flags |= VM_SHARED | VM_MAYSHARE; break; case MAP_DROPPABLE: @@ -537,6 +541,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * Set pgoff according to addr for anon_vma. */ pgoff = addr >> PAGE_SHIFT; + pos.offset = addr & ~PAGE_MASK; break; default: return -EINVAL; @@ -557,7 +562,11 @@ unsigned long do_mmap(struct file *file, unsigned long addr, vm_flags |= VM_NORESERVE; } - addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); + addr = mmap_region(file, addr, len, vm_flags, + (struct vm_page_offset) { + .index = pgoff, + .offset = pos.offset, + }, uf); if (!IS_ERR_VALUE(addr) && ((vm_flags & VM_LOCKED) || (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) @@ -565,9 +574,28 @@ unsigned long do_mmap(struct file *file, unsigned long addr, return addr; } -unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, +unsigned long do_mmap(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, + struct list_head *uf) +{ + return do_mmap_internal(file, addr, len, prot, flags, vm_flags, + (struct vm_page_offset) { pgoff, 0 }, populate, uf); +} + +#ifdef CONFIG_MM_SUBPAGE +unsigned long do_mmap_offset(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, struct vm_page_offset pos, unsigned long *populate, + struct list_head *uf) +{ + return do_mmap_internal(file, addr, len, prot, flags, vm_flags, pos, populate, uf); +} +#endif + +static unsigned long ksys_mmap_offset(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, - unsigned long fd, unsigned long pgoff) + unsigned long fd, unsigned long pgoff, unsigned int suboffset) { struct file *file = NULL; unsigned long retval; @@ -603,13 +631,34 @@ unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, return PTR_ERR(file); } - retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); + if (suboffset) + retval = vm_mmap(file, addr, len, prot, flags, + (pgoff << PAGE_SHIFT) | suboffset); + else + retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); out_fput: if (file) fput(file); return retval; } +unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long pgoff) +{ + return ksys_mmap_offset(addr, len, prot, flags, fd, pgoff, 0); +} + +#ifdef CONFIG_MM_SUBPAGE +unsigned long ksys_mmap_user(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long offset) +{ + if (!IS_ALIGNED(offset, mm_page_size(current->mm))) + return -EINVAL; + return ksys_mmap_offset(addr, len, prot, flags, fd, + offset >> PAGE_SHIFT, offset & ~PAGE_MASK); +} +#endif + SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, pgoff) @@ -703,7 +752,7 @@ generic_get_unmapped_area(struct file *filp, unsigned long addr, return addr; if (addr) { - addr = PAGE_ALIGN(addr); + addr = mm_page_align(mm, addr); vma = find_vma_prev(mm, addr, &prev); if (mmap_end - len >= addr && addr >= mmap_min_addr && (!vma || addr + len <= vm_start_gap(vma)) && @@ -754,7 +803,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, /* requesting a specific address */ if (addr) { - addr = PAGE_ALIGN(addr); + addr = mm_page_align(mm, addr); vma = find_vma_prev(mm, addr, &prev); if (mmap_end - len >= addr && addr >= mmap_min_addr && (!vma || addr + len <= vm_start_gap(vma)) && @@ -764,7 +813,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.length = len; - info.low_limit = PAGE_SIZE; + info.low_limit = mm_page_size(mm); info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); info.start_gap = stack_guard_placement(vm_flags); if (filp && is_file_hugepages(filp)) @@ -777,7 +826,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, * can happen with large stack limits and large mmap() * allocations. */ - if (offset_in_page(addr)) { + if (addr & ~mm_page_mask(mm)) { VM_BUG_ON(addr != -ENOMEM); info.flags = 0; info.low_limit = TASK_UNMAPPED_BASE; @@ -857,7 +906,7 @@ __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, if (addr > TASK_SIZE - len) return -ENOMEM; - if (offset_in_page(addr)) + if (addr & ~mm_page_mask(current->mm)) return -EINVAL; error = security_mmap_addr(addr); @@ -962,7 +1011,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon { struct vm_area_struct *vma, *prev; - addr &= PAGE_MASK; + addr &= mm_page_mask(mm); vma = find_vma_prev(mm, addr, &prev); if (vma && (vma->vm_start <= addr)) return vma; @@ -985,7 +1034,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon struct vm_area_struct *vma; unsigned long start; - addr &= PAGE_MASK; + addr &= mm_page_mask(mm); vma = find_vma(mm, addr); if (!vma) return NULL; @@ -1214,7 +1263,7 @@ int vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec) LIST_HEAD(uf); VMA_ITERATOR(vmi, mm, addr); - len = PAGE_ALIGN(request); + len = mm_page_align(mm, request); if (len < request) return -ENOMEM; if (!len) @@ -1257,7 +1306,7 @@ unsigned long tear_down_vmas(struct mm_struct *mm, struct vma_iterator *vmi, vma_iter_set(vmi, vma->vm_end); do { if (vma->vm_flags & VM_ACCOUNT) - nr_accounted += vma_pages(vma); + nr_accounted += vma_user_pages(vma); vma_mark_detached(vma); remove_vma(vma); count++; @@ -1322,8 +1371,8 @@ void exit_mmap(struct mm_struct *mm) destroy: __mt_destroy(&mm->mm_mt); trace_exit_mmap(mm); + mm_unacct_memory(mm, nr_accounted); mmap_write_unlock(mm); - vm_unacct_memory(nr_accounted); } /* @@ -1333,19 +1382,19 @@ void exit_mmap(struct mm_struct *mm) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages) { - if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT) + if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> mm_page_shift(mm)) return false; if (is_data_mapping_vma_flags(vma_flags) && - mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) { + mm->data_vm + npages > rlimit(RLIMIT_DATA) >> mm_page_shift(mm)) { /* Workaround for Valgrind */ if (rlimit(RLIMIT_DATA) == 0 && - mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT) + mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> mm_page_shift(mm)) return true; pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n", current->comm, current->pid, - (mm->data_vm + npages) << PAGE_SHIFT, + (mm->data_vm + npages) << mm_page_shift(mm), rlimit(RLIMIT_DATA), ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data"); @@ -1474,7 +1523,7 @@ static struct vm_area_struct *__install_special_mapping( if (ret) goto out; - vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); + vm_stat_account(mm, vma->vm_flags, len >> mm_page_shift(mm)); perf_event_mmap(vma); @@ -1771,14 +1820,14 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) if (retval) goto loop_out; - vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt)); + vm_stat_account(mm, mpnt->vm_flags, -vma_user_pages(mpnt)); continue; } charge = 0; if (mpnt->vm_flags & VM_ACCOUNT) { - unsigned long len = vma_pages(mpnt); + unsigned long len = vma_user_pages(mpnt); - if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ + if (mm_account_memory(mm, oldmm, len)) /* sic */ goto fail_nomem; charge = len; } @@ -1890,7 +1939,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) flush_cache_mm(mm); unmap_region(&unmap); charge = tear_down_vmas(mm, &vmi, tmp, end); - vm_unacct_memory(charge); + mm_unacct_memory(mm, charge); } __mt_destroy(&mm->mm_mt); /* @@ -1916,6 +1965,6 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) vm_area_free(tmp); fail_nomem: retval = -ENOMEM; - vm_unacct_memory(charge); + mm_unacct_memory(mm, charge); goto loop_out; } diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 3985d856de7f9..42419a47a1880 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -12,6 +12,9 @@ #include #include #include +#include + +#include "user-subpage-internal.h" #include @@ -100,6 +103,49 @@ void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma) */ #define MAX_NR_FOLIOS_PER_FREE 512 +#ifdef CONFIG_MM_SUBPAGE +/* Private to gather batches: never pass this token to encoded_page_ptr(). */ +#define TLB_SUBPAGE_TAG BIT(2) + +static void tlb_free_pages_and_subpages(struct encoded_page **pages, unsigned int nr) +{ + unsigned int start = 0, i = 0; + + while (i < nr) { + unsigned long token = (unsigned long)pages[i]; + + if (token & TLB_SUBPAGE_TAG) { + if (i != start) + free_pages_and_swap_cache(pages + start, i - start); + mm_subpage_put((struct mm_subpage *)(token & ~TLB_SUBPAGE_TAG)); + start = ++i; + } else { + /* The count payload is not a pointer and may have bit 2 set. */ + i += encoded_page_flags(pages[i]) & ENCODED_PAGE_BIT_NR_PAGES_NEXT ? 2 : 1; + } + } + if (i != start) + free_pages_and_swap_cache(pages + start, i - start); +} + +bool __tlb_remove_subpage(struct mmu_gather *tlb, struct mm_subpage *subpage) +{ + struct mmu_gather_batch *batch = tlb->active; + + BUILD_BUG_ON(__alignof__(struct page) < 8); + BUILD_BUG_ON(__alignof__(struct mm_subpage) < 8); + BUILD_BUG_ON(TLB_SUBPAGE_TAG & ENCODED_PAGE_BITS); + VM_BUG_ON(!tlb->end); + VM_BUG_ON(mm_page_shift(tlb->mm) != mm_subpage_shift(subpage)); + batch->encoded_pages[batch->nr++] = + (struct encoded_page *)((unsigned long)subpage | TLB_SUBPAGE_TAG); + /* Keep room for a following native page/count pair. */ + return batch->nr >= batch->max - 1 && !tlb_next_batch(tlb); +} +#else +#define tlb_free_pages_and_subpages free_pages_and_swap_cache +#endif + static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) { struct encoded_page **pages = batch->encoded_pages; @@ -135,7 +181,7 @@ static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) } } - free_pages_and_swap_cache(pages, nr); + tlb_free_pages_and_subpages(pages, nr); pages += nr; batch->nr -= nr; @@ -174,7 +220,15 @@ static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb, #ifdef CONFIG_MMU_GATHER_PAGE_SIZE VM_WARN_ON(tlb->page_size != page_size); - VM_WARN_ON_ONCE(nr_pages != 1 && page_size != PAGE_SIZE); + if (page_size > PAGE_SIZE && page_size == mm_page_size(tlb->mm)) { + /* A coarse base PTE owns one reference per covered native page. */ + unsigned long span = mm_pte_native_pages(tlb->mm); + + VM_WARN_ON_ONCE(!nr_pages || nr_pages % span || + !IS_ALIGNED(page_to_pfn(page), span)); + } else { + VM_WARN_ON_ONCE(nr_pages != 1 && page_size != PAGE_SIZE); + } VM_WARN_ON_ONCE(page_folio(page) != page_folio(page + nr_pages - 1)); #endif @@ -208,7 +262,7 @@ bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page, unsigned int nr_pages, bool delay_rmap) { return __tlb_remove_folio_pages_size(tlb, page, nr_pages, delay_rmap, - PAGE_SIZE); + mm_page_size(tlb->mm)); } bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size) @@ -352,6 +406,13 @@ static inline void __tlb_remove_table_one(void *table) { struct ptdesc *ptdesc; + /* Preserve opaque architecture tokens through the RCU grace period. */ + if (!tlb_table_is_ptdesc(table)) { + tlb_remove_table_sync_rcu(); + __tlb_remove_table(table); + return; + } + ptdesc = table; call_rcu(&ptdesc->pt_rcu_head, __tlb_remove_table_one_rcu); } diff --git a/mm/mprotect.c b/mm/mprotect.c index 9cbf932b028cf..0f5ea22bf4c90 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -58,6 +59,31 @@ static bool maybe_change_pte_writable(struct vm_area_struct *vma, pte_t pte) return true; } +#ifdef CONFIG_MM_SUBPAGE +static bool subpage_pte_exclusive(struct vm_area_struct *vma, pte_t pte) +{ + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + bool exclusive; + + if (!slot) + return false; + exclusive = mm_subpage_is_exclusive(slot); + mm_subpage_put(slot); + return exclusive; +} + +static int subpage_exclusive_pte_batch(struct vm_area_struct *vma, pte_t *ptep, + int nr_ptes, bool exclusive) +{ + int nr; + + for (nr = 1; nr < nr_ptes; nr++) + if (subpage_pte_exclusive(vma, ptep_get(ptep + nr)) != exclusive) + break; + return nr; +} +#endif + static bool can_change_private_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte) { @@ -73,6 +99,10 @@ static bool can_change_private_pte_writable(struct vm_area_struct *vma, * any additional checks while holding the PT lock. */ page = vm_normal_page(vma, addr, pte); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return page && PageAnon(page) && subpage_pte_exclusive(vma, pte); +#endif return page && PageAnon(page) && PageAnonExclusive(page); } @@ -103,13 +133,20 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, return can_change_shared_pte_writable(vma, pte); } -static int mprotect_folio_pte_batch(struct folio *folio, pte_t *ptep, - pte_t pte, int max_nr_ptes, fpb_t flags) + +static __always_inline int mprotect_folio_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t pte, int max_nr_ptes, + fpb_t flags, unsigned int page_shift) { /* No underlying folio, so cannot batch */ if (!folio) return 1; +#ifdef CONFIG_MM_SUBPAGE + if (page_shift != PAGE_SHIFT) + return folio_subpage_pte_batch(vma, folio, ptep, pte, max_nr_ptes, flags); +#endif + if (!folio_test_large(folio)) return 1; @@ -125,17 +162,17 @@ static __always_inline void prot_commit_flush_ptes(struct vm_area_struct *vma, * Advance the position in the batch by idx; note that if idx > 0, * then the nr_ptes passed here is <= batch size - idx. */ - addr += idx * PAGE_SIZE; + addr += idx * mm_page_size(vma->vm_mm); ptep += idx; - oldpte = pte_advance_pfn(oldpte, idx); - ptent = pte_advance_pfn(ptent, idx); + oldpte = pte_advance_pfn_mm(vma->vm_mm, oldpte, idx); + ptent = pte_advance_pfn_mm(vma->vm_mm, ptent, idx); if (set_write) ptent = pte_mkwrite(ptent, vma); modify_prot_commit_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes); if (pte_needs_flush(oldpte, ptent)) - tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE); + tlb_flush_pte_range(tlb, addr, nr_ptes * mm_page_size(vma->vm_mm)); } /* @@ -177,6 +214,25 @@ static __always_inline void commit_anon_folio_batch(struct vm_area_struct *vma, int sub_batch_idx = 0; int len; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + while (nr_ptes) { + expected_anon_exclusive = subpage_pte_exclusive(vma, + pte_advance_pfn_mm(vma->vm_mm, oldpte, sub_batch_idx)); + for (len = 1; len < nr_ptes; len++) + if (subpage_pte_exclusive(vma, + pte_advance_pfn_mm(vma->vm_mm, oldpte, sub_batch_idx + len)) != + expected_anon_exclusive) + break; + prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, len, + sub_batch_idx, expected_anon_exclusive, tlb); + sub_batch_idx += len; + nr_ptes -= len; + } + return; + } +#endif + while (nr_ptes) { expected_anon_exclusive = PageAnonExclusive(first_page + sub_batch_idx); len = page_anon_exclusive_sub_batch(sub_batch_idx, nr_ptes, @@ -265,6 +321,11 @@ static long change_softleaf_pte(struct vm_area_struct *vma, newpte = oldpte; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + newpte = pte_swp_set_subpage_offset(newpte, pte_swp_subpage_offset(oldpte)); +#endif + if (uffd_wp) newpte = pte_swp_mkuffd_wp(newpte); else if (uffd_wp_resolve) @@ -320,6 +381,9 @@ static long change_pte_range(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, pgprot_t newprot, unsigned long cp_flags) { + /* The mm's mapping granule cannot change during the walk. */ + const unsigned int page_shift = mm_page_shift(vma->vm_mm); + const unsigned long page_size = 1UL << page_shift; pte_t *pte, oldpte; spinlock_t *ptl; long pages = 0; @@ -328,7 +392,7 @@ static long change_pte_range(struct mmu_gather *tlb, bool uffd_wp = cp_flags & MM_CP_UFFD_WP; int nr_ptes; - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, page_size); pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); if (!pte) return -EAGAIN; @@ -343,7 +407,7 @@ static long change_pte_range(struct mmu_gather *tlb, oldpte = ptep_get(pte); if (pte_present(oldpte)) { const fpb_t flags = FPB_RESPECT_SOFT_DIRTY | FPB_RESPECT_WRITE; - int max_nr_ptes = (end - addr) >> PAGE_SHIFT; + int max_nr_ptes = (end - addr) >> page_shift; struct folio *folio = NULL; struct page *page; @@ -359,17 +423,32 @@ static long change_pte_range(struct mmu_gather *tlb, * Avoid trapping faults against the zero or KSM * pages. See similar comment in change_huge_pmd. */ - if (prot_numa && - !folio_can_map_prot_numa(folio, vma, - is_private_single_threaded)) { - - /* determine batch to skip */ - nr_ptes = mprotect_folio_pte_batch(folio, - pte, oldpte, max_nr_ptes, /* flags = */ 0); - continue; + if (prot_numa) { + bool shared = folio && folio_maybe_mapped_shared(folio); + +#ifdef CONFIG_MM_SUBPAGE + /* Multiple private leaves do not imply shared COW data. */ + if (folio && folio_test_anon(folio) && page_shift < PAGE_SHIFT) { + shared = !subpage_pte_exclusive(vma, oldpte); + max_nr_ptes = folio_subpage_pte_batch(vma, folio, + pte, oldpte, max_nr_ptes, flags); + /* COW sharing can differ within one physical owner. */ + max_nr_ptes = subpage_exclusive_pte_batch(vma, pte, + max_nr_ptes, !shared); + } +#endif + if (!folio_can_map_prot_numa(folio, vma, + is_private_single_threaded, shared)) { + /* determine batch to skip */ + nr_ptes = mprotect_folio_pte_batch(vma, folio, + pte, oldpte, max_nr_ptes, /* flags = */ 0, + page_shift); + continue; + } } - nr_ptes = mprotect_folio_pte_batch(folio, pte, oldpte, max_nr_ptes, flags); + nr_ptes = mprotect_folio_pte_batch(vma, folio, pte, oldpte, + max_nr_ptes, flags, page_shift); /* * Optimize for the small-folio common case by @@ -408,7 +487,7 @@ static long change_pte_range(struct mmu_gather *tlb, } else { pages += change_softleaf_pte(vma, addr, pte, oldpte, cp_flags); } - } while (pte += nr_ptes, addr += nr_ptes * PAGE_SIZE, addr != end); + } while (pte += nr_ptes, addr += nr_ptes * page_size, addr != end); lazy_mmu_mode_disable(); pte_unmap_unlock(pte - 1, ptl); @@ -487,12 +566,12 @@ static inline long change_pmd_range(struct mmu_gather *tlb, long pages = 0; unsigned long nr_huge_updates = 0; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(vma->vm_mm, pud, addr); do { long ret; pmd_t _pmd; again: - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(vma->vm_mm, addr, end); ret = change_pmd_prepare(vma, pmd, cp_flags); if (ret) { @@ -559,10 +638,10 @@ static inline long change_pud_range(struct mmu_gather *tlb, range.start = 0; - pudp = pud_offset(p4d, addr); + pudp = pud_offset_mm(vma->vm_mm, p4d, addr); do { again: - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, pudp, pmd, addr, cp_flags); if (ret) { pages = ret; @@ -615,13 +694,13 @@ static inline long change_p4d_range(struct mmu_gather *tlb, unsigned long next; long pages = 0, ret; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(vma->vm_mm, pgd, addr); do { - next = p4d_addr_end(addr, end); + next = p4d_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, p4d, pud, addr, cp_flags); if (ret) return ret; - if (p4d_none_or_clear_bad(p4d)) + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4d)) continue; pages += change_pud_range(tlb, vma, p4d, addr, next, newprot, cp_flags); @@ -643,7 +722,7 @@ static long change_protection_range(struct mmu_gather *tlb, pgd = pgd_offset(mm, addr); tlb_start_vma(tlb, vma); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, pgd, p4d, addr, cp_flags); if (ret) { pages = ret; @@ -729,7 +808,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, struct mm_struct *mm = vma->vm_mm; const vma_flags_t old_vma_flags = READ_ONCE(vma->flags); vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags); - long nrpages = (end - start) >> PAGE_SHIFT; + long nrpages = (end - start) >> mm_page_shift(mm); unsigned int mm_cp_flags = 0; unsigned long charged = 0; int error; @@ -777,7 +856,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, VMA_ACCOUNT_BIT, VMA_WRITE_BIT, VMA_HUGETLB_BIT, VMA_SHARED_BIT, VMA_NORESERVE_BIT)) { charged = nrpages; - if (security_vm_enough_memory_mm(mm, charged)) + if (mm_account_memory(mm, mm, charged)) return -ENOMEM; vma_flags_set(&new_vma_flags, VMA_ACCOUNT_BIT); } @@ -808,7 +887,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, if (vma_flags_test(&old_vma_flags, VMA_ACCOUNT_BIT) && !vma_flags_test(&new_vma_flags, VMA_ACCOUNT_BIT)) - vm_unacct_memory(nrpages); + mm_unacct_memory(mm, nrpages); /* * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major @@ -826,7 +905,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, return 0; fail: - vm_unacct_memory(charged); + mm_unacct_memory(mm, charged); return error; } @@ -851,11 +930,11 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ return -EINVAL; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(current->mm)) return -EINVAL; if (!len) return 0; - len = PAGE_ALIGN(len); + len = mm_page_align(current->mm, len); end = start + len; if (end <= start) return -ENOMEM; diff --git a/mm/mremap.c b/mm/mremap.c index e9c8b1d05832b..97d5ebf945285 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -82,11 +82,11 @@ static pud_t *get_old_pud(struct mm_struct *mm, unsigned long addr) if (pgd_none_or_clear_bad(pgd)) return NULL; - p4d = p4d_offset(pgd, addr); - if (p4d_none_or_clear_bad(p4d)) + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_or_clear_bad_mm(mm, p4d)) return NULL; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(mm, p4d, addr); if (pud_none_or_clear_bad(pud)) return NULL; @@ -102,7 +102,7 @@ static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr) if (!pud) return NULL; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(mm, pud, addr); if (pmd_none(*pmd)) return NULL; @@ -183,6 +183,14 @@ static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr if (max_nr == 1) return 1; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + folio = vm_normal_folio(vma, addr, pte); + return folio ? folio_subpage_pte_batch(vma, folio, ptep, pte, + max_nr, FPB_RESPECT_WRITE) : 1; + } +#endif + /* Avoid expensive folio lookup if we stand no chance of benefit. */ if (pte_batch_hint(ptep, pte) == 1) return 1; @@ -262,12 +270,12 @@ static int move_ptes(struct pagetable_move_control *pmc, flush_tlb_batched_pending(vma->vm_mm); lazy_mmu_mode_enable(); - for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE, - new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) { + for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * mm_page_size(mm), + new_ptep += nr_ptes, new_addr += nr_ptes * mm_page_size(mm)) { VM_WARN_ON_ONCE(!pte_none(*new_ptep)); nr_ptes = 1; - max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT; + max_nr_ptes = (old_end - old_addr) >> mm_page_shift(mm); old_pte = ptep_get(old_ptep); if (pte_none(old_pte)) continue; @@ -407,7 +415,7 @@ static bool move_normal_pmd(struct pagetable_move_control *pmc, VM_BUG_ON(!pmd_none(*new_pmd)); pmd_populate(mm, new_pmd, pmd_pgtable(pmd)); - flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PMD_SIZE); + flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + pmd_size_mm(mm)); out_unlock: if (new_ptl != old_ptl) spin_unlock(new_ptl); @@ -458,8 +466,8 @@ static bool move_normal_pud(struct pagetable_move_control *pmc, VM_BUG_ON(!pud_none(*new_pud)); - pud_populate(mm, new_pud, pud_pgtable(pud)); - flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PUD_SIZE); + pud_populate(mm, new_pud, pud_pgtable_mm(mm, pud)); + flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + pud_size_mm(mm)); if (new_ptl != old_ptl) spin_unlock(new_ptl); spin_unlock(old_ptl); @@ -549,13 +557,13 @@ static __always_inline unsigned long get_extent(enum pgt_entry entry, switch (entry) { case HPAGE_PMD: case NORMAL_PMD: - mask = PMD_MASK; - size = PMD_SIZE; + mask = pmd_mask_mm(pmc->old->vm_mm); + size = pmd_size_mm(pmc->old->vm_mm); break; case HPAGE_PUD: case NORMAL_PUD: - mask = PUD_MASK; - size = PUD_SIZE; + mask = pud_mask_mm(pmc->old->vm_mm); + size = pud_size_mm(pmc->old->vm_mm); break; default: BUILD_BUG(); @@ -811,7 +819,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) * If possible, realign addresses to PMD boundary for faster copy. * Only realign if the mremap copying hits a PMD boundary. */ - try_realign_addr(pmc, PMD_MASK); + try_realign_addr(pmc, pmd_mask_mm(mm)); flush_cache_range(pmc->old, pmc->old_addr, pmc->old_end); mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, mm, @@ -838,7 +846,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) /* We ignore and continue on error? */ continue; } - } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == PUD_SIZE) { + } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == pud_size_mm(mm)) { if (move_pgt_entry(pmc, NORMAL_PUD, old_pud, new_pud)) continue; } @@ -857,7 +865,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) continue; split_huge_pmd(pmc->old, old_pmd, pmc->old_addr); } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PMD) && - extent == PMD_SIZE) { + extent == pmd_size_mm(mm)) { /* * If the extent is PMD-sized, try to speed the move by * moving at the PMD level if possible. @@ -947,9 +955,7 @@ static unsigned long vrm_set_new_addr(struct vma_remap_struct *vrm) { struct vm_area_struct *vma = vrm->vma; unsigned long map_flags = 0; - /* Page Offset _into_ the VMA. */ - pgoff_t internal_pgoff = (vrm->addr - vma->vm_start) >> PAGE_SHIFT; - pgoff_t pgoff = vma->vm_pgoff + internal_pgoff; + pgoff_t pgoff = vma_page_offset_at(vma, vrm->addr).index; unsigned long new_addr = vrm_implies_new_addr(vrm) ? vrm->new_addr : 0; unsigned long res; @@ -985,13 +991,13 @@ static bool vrm_calc_charge(struct vma_remap_struct *vrm) * the length of the new one. Otherwise it's just the delta in size. */ if (vrm->flags & MREMAP_DONTUNMAP) - charged = vrm->new_len >> PAGE_SHIFT; + charged = vrm->new_len >> mm_page_shift(current->mm); else - charged = vrm->delta >> PAGE_SHIFT; + charged = vrm->delta >> mm_page_shift(current->mm); /* This accounts 'charged' pages of memory. */ - if (security_vm_enough_memory_mm(current->mm, charged)) + if (mm_account_memory(current->mm, current->mm, charged)) return false; vrm->charged = charged; @@ -1007,7 +1013,7 @@ static void vrm_uncharge(struct vma_remap_struct *vrm) if (!(vrm->vma->vm_flags & VM_ACCOUNT)) return; - vm_unacct_memory(vrm->charged); + mm_unacct_memory(current->mm, vrm->charged); vrm->charged = 0; } @@ -1019,13 +1025,13 @@ static void vrm_uncharge(struct vma_remap_struct *vrm) static void vrm_stat_account(struct vma_remap_struct *vrm, unsigned long bytes) { - unsigned long pages = bytes >> PAGE_SHIFT; + unsigned long pages = bytes >> mm_page_shift(current->mm); struct mm_struct *mm = current->mm; struct vm_area_struct *vma = vrm->vma; vm_stat_account(mm, vma->vm_flags, pages); if (vma->vm_flags & VM_LOCKED) - mm->locked_vm += pages; + mm->locked_vm += bytes >> MM_ACCOUNT_SHIFT; } static bool __check_map_count_against_split(struct mm_struct *mm, @@ -1198,7 +1204,7 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) vrm->vmi_needs_invalidate = true; if (err) { /* OOM: unable to split vma, just get accounts right */ - vm_acct_memory(len >> PAGE_SHIFT); + mm_acct_memory(current->mm, len >> mm_page_shift(current->mm)); return; } @@ -1255,16 +1261,14 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) static int copy_vma_and_data(struct vma_remap_struct *vrm, struct vm_area_struct **new_vma_ptr) { - unsigned long internal_offset = vrm->addr - vrm->vma->vm_start; - unsigned long internal_pgoff = internal_offset >> PAGE_SHIFT; - unsigned long new_pgoff = vrm->vma->vm_pgoff + internal_pgoff; + struct vm_page_offset pos = vma_page_offset_at(vrm->vma, vrm->addr); unsigned long moved_len; struct vm_area_struct *vma = vrm->vma; struct vm_area_struct *new_vma; int err = 0; PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len); - new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, new_pgoff, + new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, pos, &pmc.need_rmap_locks); if (!new_vma) { vrm_uncharge(vrm); @@ -1473,7 +1477,7 @@ static unsigned long mremap_to(struct vma_remap_struct *vrm) /* MREMAP_DONTUNMAP expands by old_len since old_len == new_len */ if (vrm->flags & MREMAP_DONTUNMAP) { vma_flags_t vma_flags = vrm->vma->flags; - unsigned long pages = vrm->old_len >> PAGE_SHIFT; + unsigned long pages = vrm->old_len >> mm_page_shift(current->mm); if (!may_expand_vm(mm, &vma_flags, pages)) return -ENOMEM; @@ -1495,7 +1499,7 @@ static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) if (find_vma_intersection(vma->vm_mm, vma->vm_end, end)) return 0; if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, - 0, MAP_FIXED) & ~PAGE_MASK) + 0, MAP_FIXED) & ~mm_page_mask(vma->vm_mm)) return 0; return 1; } @@ -1727,7 +1731,8 @@ static int check_prep_vma(struct vma_remap_struct *vrm) struct vm_area_struct *vma = vrm->vma; struct mm_struct *mm = current->mm; unsigned long addr = vrm->addr; - unsigned long old_len, new_len, pgoff; + unsigned long old_len, new_len; + struct vm_page_offset pos, end; if (!vma) return -EFAULT; @@ -1802,9 +1807,9 @@ static int check_prep_vma(struct vma_remap_struct *vrm) vrm->populate_expand = true; /* Need to be careful about a growing mapping */ - pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; - if (pgoff + (new_len >> PAGE_SHIFT) < pgoff) + pos = vma_page_offset_at(vma, addr); + end = vm_page_offset_add(pos, new_len); + if (end.index < pos.index) return -EINVAL; if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) @@ -1813,7 +1818,7 @@ static int check_prep_vma(struct vma_remap_struct *vrm) if (!mlock_future_ok(mm, vma->vm_flags & VM_LOCKED, vrm->delta)) return -EAGAIN; - if (!may_expand_vm(mm, &vma->flags, vrm->delta >> PAGE_SHIFT)) + if (!may_expand_vm(mm, &vma->flags, vrm->delta >> mm_page_shift(current->mm))) return -ENOMEM; return 0; @@ -1834,7 +1839,7 @@ static unsigned long check_mremap_params(struct vma_remap_struct *vrm) return -EINVAL; /* Start address must be page-aligned. */ - if (offset_in_page(addr)) + if (addr & ~mm_page_mask(current->mm)) return -EINVAL; /* @@ -1858,7 +1863,7 @@ static unsigned long check_mremap_params(struct vma_remap_struct *vrm) return -EINVAL; /* The new address must be page-aligned. */ - if (offset_in_page(vrm->new_addr)) + if (vrm->new_addr & ~mm_page_mask(current->mm)) return -EINVAL; /* A fixed address implies a move. */ @@ -1971,8 +1976,8 @@ static unsigned long do_mremap(struct vma_remap_struct *vrm) unsigned long res; bool failed; - vrm->old_len = PAGE_ALIGN(vrm->old_len); - vrm->new_len = PAGE_ALIGN(vrm->new_len); + vrm->old_len = mm_page_align(mm, vrm->old_len); + vrm->new_len = mm_page_align(mm, vrm->new_len); res = check_mremap_params(vrm); if (res) diff --git a/mm/mseal.c b/mm/mseal.c index e2093ae3d25cd..62df360b43545 100644 --- a/mm/mseal.c +++ b/mm/mseal.c @@ -151,10 +151,10 @@ int do_mseal(unsigned long start, size_t len_in, unsigned long flags) return -EINVAL; start = untagged_addr(start); - if (!PAGE_ALIGNED(start)) + if (start & ~mm_page_mask(mm)) return -EINVAL; - len = PAGE_ALIGN(len_in); + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero. */ if (len_in && !len) return -EINVAL; diff --git a/mm/msync.c b/mm/msync.c index ac4c9bfea2e7f..6d9027ee47246 100644 --- a/mm/msync.c +++ b/mm/msync.c @@ -41,12 +41,12 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) if (flags & ~(MS_ASYNC | MS_INVALIDATE | MS_SYNC)) goto out; - if (offset_in_page(start)) + if (start & ~mm_page_mask(mm)) goto out; if ((flags & MS_ASYNC) && (flags & MS_SYNC)) goto out; error = -ENOMEM; - len = (len + ~PAGE_MASK) & PAGE_MASK; + len = mm_page_align(mm, len); end = start + len; if (end < start) goto out; @@ -86,7 +86,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) } file = vma->vm_file; fstart = (start - vma->vm_start) + - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); fend = fstart + (min(end, vma->vm_end) - start) - 1; start = vma->vm_end; if ((flags & MS_SYNC) && file && diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 5f372f6e26fa3..5345f43c10b6f 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -225,7 +225,8 @@ long oom_badness(struct task_struct *p, unsigned long totalpages) * The baseline for the badness score is the proportion of RAM that each * task's rss, pagetable and swap space use. */ - points = get_mm_rss_sum(p->mm) + get_mm_counter_sum(p->mm, MM_SWAPENTS) + + points = mm_pages_to_native(p->mm, get_mm_rss_sum(p->mm) + + get_mm_counter_sum(p->mm, MM_SWAPENTS)) + mm_pgtables_bytes(p->mm) / PAGE_SIZE; task_unlock(p); @@ -399,10 +400,10 @@ static int dump_task(struct task_struct *p, void *arg) pr_info("[%7d] %5d %5d %8lu %8lu %8lu %8lu %9lu %8ld %8lu %5hd %s\n", task->pid, from_kuid(&init_user_ns, task_uid(task)), - task->tgid, task->mm->total_vm, get_mm_rss_sum(task->mm), - get_mm_counter_sum(task->mm, MM_ANONPAGES), get_mm_counter_sum(task->mm, MM_FILEPAGES), - get_mm_counter_sum(task->mm, MM_SHMEMPAGES), mm_pgtables_bytes(task->mm), - get_mm_counter_sum(task->mm, MM_SWAPENTS), + task->tgid, mm_pages_to_native(task->mm, task->mm->total_vm), mm_pages_to_native(task->mm, get_mm_rss_sum(task->mm)), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_ANONPAGES)), mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_FILEPAGES)), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_SHMEMPAGES)), mm_pgtables_bytes(task->mm), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_SWAPENTS)), task->signal->oom_score_adj, task->comm); task_unlock(task); @@ -588,9 +589,9 @@ static bool oom_reap_task_mm(struct task_struct *tsk, struct mm_struct *mm) pr_info("oom_reaper: reaped process %d (%s), now anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n", task_pid_nr(tsk), tsk->comm, - K(get_mm_counter_sum(mm, MM_ANONPAGES)), - K(get_mm_counter_sum(mm, MM_FILEPAGES)), - K(get_mm_counter_sum(mm, MM_SHMEMPAGES))); + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_ANONPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_FILEPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_SHMEMPAGES))); out_finish: trace_finish_task_reaping(tsk->pid); out_unlock: @@ -943,10 +944,10 @@ static void __oom_kill_process(struct task_struct *victim, const char *message) do_send_sig_info(SIGKILL, SEND_SIG_PRIV, victim, PIDTYPE_TGID); mark_oom_victim(victim); pr_err("%s: Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB, UID:%u pgtables:%lukB oom_score_adj:%d\n", - message, task_pid_nr(victim), victim->comm, K(mm->total_vm), - K(get_mm_counter_sum(mm, MM_ANONPAGES)), - K(get_mm_counter_sum(mm, MM_FILEPAGES)), - K(get_mm_counter_sum(mm, MM_SHMEMPAGES)), + message, task_pid_nr(victim), victim->comm, mm_pages_to_kb(mm, mm->total_vm), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_ANONPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_FILEPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_SHMEMPAGES)), from_kuid(&init_user_ns, task_uid(victim)), mm_pgtables_bytes(mm) >> 10, victim->signal->oom_score_adj); task_unlock(victim); diff --git a/mm/page_idle.c b/mm/page_idle.c index 9c67cbac29659..8af46b0d8f746 100644 --- a/mm/page_idle.c +++ b/mm/page_idle.c @@ -51,9 +51,10 @@ static struct folio *page_idle_get_folio(unsigned long pfn) static bool page_idle_clear_pte_refs_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long addr, void *arg) + struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + unsigned long addr = rmap.address; + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool referenced = false; while (page_vma_mapped_walk(&pvmw)) { @@ -69,7 +70,7 @@ static bool page_idle_clear_pte_refs_one(struct folio *folio, */ if (likely(pte_present(ptep_get(pvmw.pte)))) referenced |= ptep_test_and_clear_young(vma, addr, pvmw.pte); - referenced |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE); + referenced |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + mm_page_size(vma->vm_mm)); } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { pmd_t pmdval = pmdp_get(pvmw.pmd); diff --git a/mm/page_io.c b/mm/page_io.c index 70cea9e24d2fd..290706f931bdc 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "swap.h" static void __end_swap_bio_write(struct bio *bio) @@ -411,6 +412,58 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug) *swap_plug = sio; } +static DEFINE_MUTEX(swap_extent_io_mutex); +static struct bio_set swap_extent_bioset; + +int swap_extent_io_init(void) +{ + int ret = 0; + + /* Swapon prepares the reserve before any reclaim can use this file. */ + mutex_lock(&swap_extent_io_mutex); + if (!bioset_initialized(&swap_extent_bioset)) + ret = bioset_init(&swap_extent_bioset, SWAP_CLUSTER_MAX, 0, + BIOSET_NEED_RESCUER); + mutex_unlock(&swap_extent_io_mutex); + return ret; +} + +/* + * A mandatory large base page cannot split to fit a swap-file extent. Chain + * each physical run to the original bio: its folio completion must wait for + * every run, including on partial I/O or allocation failure. + */ +static void swap_submit_bio(struct bio *bio, void *private) +{ + struct folio *folio = private; + struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); + swp_entry_t entry = folio->swap; + + if (!(READ_ONCE(sis->flags) & SWP_BLKDEV) && folio_test_large(folio)) { + while (true) { + unsigned long nr = bio->bi_iter.bi_size >> PAGE_SHIFT; + unsigned int bytes; + struct bio *split; + + bio->bi_iter.bi_sector = swap_extent_sector(entry, &nr); + bytes = nr << PAGE_SHIFT; + if (bytes == bio->bi_iter.bi_size) + break; + /* The unsubmitted parent may own a bio from fs_bio_set. */ + split = bio_split(bio, bytes >> 9, GFP_NOIO, &swap_extent_bioset); + if (IS_ERR(split)) { + bio->bi_status = BLK_STS_RESOURCE; + bio_endio(bio); + return; + } + bio_chain(split, bio); + submit_bio(split); + entry.val += nr; + } + } + submit_bio(bio); +} + static void swap_writepage_bdev_sync(struct folio *folio, struct swap_info_struct *sis) { @@ -427,7 +480,7 @@ static void swap_writepage_bdev_sync(struct folio *folio, folio_start_writeback(folio); folio_unlock(folio); - submit_bio_wait(&bio); + bio_await(&bio, folio, swap_submit_bio); __end_swap_bio_write(&bio); } @@ -445,7 +498,7 @@ static void swap_writepage_bdev_async(struct folio *folio, count_swpout_vm_event(folio); folio_start_writeback(folio); folio_unlock(folio); - submit_bio(bio); + swap_submit_bio(bio, folio); } void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug) @@ -590,7 +643,7 @@ static void swap_read_folio_bdev_sync(struct folio *folio, count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio_wait(&bio); + bio_await(&bio, folio, swap_submit_bio); __end_swap_bio_read(&bio); put_task_struct(current); } @@ -607,7 +660,7 @@ static void swap_read_folio_bdev_async(struct folio *folio, count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio(bio); + swap_submit_bio(bio, folio); } void swap_read_folio(struct folio *folio, struct swap_iocb **plug) diff --git a/mm/page_table_check.c b/mm/page_table_check.c index 2403f5a114105..dc6797a41daaa 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -4,6 +4,7 @@ * Copyright (c) 2021, Google LLC. * Pasha Tatashin */ +#include #include #include #include @@ -13,11 +14,21 @@ #undef pr_fmt #define pr_fmt(fmt) "page_table_check: " fmt -struct page_table_check { +#ifdef CONFIG_MM_SUBPAGE +#define PAGE_TABLE_CHECK_SLOTS (PAGE_SIZE / SZ_4K) +#else +#define PAGE_TABLE_CHECK_SLOTS 1 +#endif + +struct page_table_check_slot { atomic_t anon_map_count; atomic_t file_map_count; }; +struct page_table_check { + struct page_table_check_slot slots[PAGE_TABLE_CHECK_SLOTS]; +}; + static bool __page_table_check_enabled __initdata = IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED); @@ -56,6 +67,74 @@ static struct page_table_check *get_page_table_check(struct page_ext *page_ext) return page_ext_data(page_ext, &page_table_check_ops); } +#if IS_ENABLED(CONFIG_KUNIT) +/* Diagnostic snapshot for private test allocations, not a synchronization API. */ +int page_table_check_get_counts(struct page *page, unsigned int offset, + int *anon, int *file) +{ + struct page_ext *ext; + struct page_table_check *ptc; + unsigned int slot = 0; + + if (static_branch_likely(&page_table_check_disabled)) + return -EOPNOTSUPP; + if (offset >= PAGE_SIZE) + return -EINVAL; +#ifdef CONFIG_MM_SUBPAGE + slot = offset / SZ_4K; +#endif + ext = page_ext_get(page); + if (!ext) + return -ENOENT; + ptc = get_page_table_check(ext); + *anon = atomic_read(&ptc->slots[slot].anon_map_count); + *file = atomic_read(&ptc->slots[slot].file_map_count); + page_ext_put(ext); + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(page_table_check_get_counts); +#endif + +static void page_table_check_update(struct page_table_check_slot *slot, + bool anon, bool clear, bool rw) +{ + if (anon) { + BUG_ON(atomic_read(&slot->file_map_count)); + if (clear) + BUG_ON(atomic_dec_return(&slot->anon_map_count) < 0); + else + BUG_ON(atomic_inc_return(&slot->anon_map_count) > 1 && rw); + } else { + BUG_ON(atomic_read(&slot->anon_map_count)); + if (clear) + BUG_ON(atomic_dec_return(&slot->file_map_count) < 0); + else + BUG_ON(atomic_inc_return(&slot->file_map_count) < 0); + } +} + +#ifdef CONFIG_MM_SUBPAGE +static void page_table_check_subpage(phys_addr_t phys, unsigned long size, + bool clear, bool rw) +{ + unsigned long pfn = PHYS_PFN(phys); + struct page_table_check *ptc; + struct page_ext *ext; + struct page *page; + unsigned int first = offset_in_page(phys) / SZ_4K, i; + + if (!pfn_valid(pfn)) + return; + page = pfn_to_page(pfn); + BUG_ON(PageSlab(page)); + ext = page_ext_get(page); + ptc = get_page_table_check(ext); + for (i = first; i < first + size / SZ_4K; i++) + page_table_check_update(&ptc->slots[i], PageAnon(page), clear, rw); + page_ext_put(ext); +} +#endif + /* * An entry is removed from the page table, decrement the counters for that page * verify that it is of correct type and counters do not become negative. @@ -78,13 +157,8 @@ static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt) for_each_page_ext(page, pgcnt, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - if (anon) { - BUG_ON(atomic_read(&ptc->file_map_count)); - BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0); - } else { - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0); - } + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) + page_table_check_update(&ptc->slots[i], anon, true, false); } rcu_read_unlock(); } @@ -113,13 +187,8 @@ static void page_table_check_set(unsigned long pfn, unsigned long pgcnt, for_each_page_ext(page, pgcnt, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - if (anon) { - BUG_ON(atomic_read(&ptc->file_map_count)); - BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw); - } else { - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0); - } + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) + page_table_check_update(&ptc->slots[i], anon, false, rw); } rcu_read_unlock(); } @@ -139,8 +208,10 @@ void __page_table_check_zero(struct page *page, unsigned int order) for_each_page_ext(page, 1 << order, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_read(&ptc->file_map_count)); + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) { + BUG_ON(atomic_read(&ptc->slots[i].anon_map_count)); + BUG_ON(atomic_read(&ptc->slots[i].file_map_count)); + } } rcu_read_unlock(); } @@ -151,8 +222,15 @@ void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, if (&init_mm == mm) return; - if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte)) - page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT); + if (!pte_user_accessible_page(mm, addr, pte) || pte_special(pte)) + return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + page_table_check_subpage(pte_phys_mm(mm, pte), mm_page_size(mm), true, false); + return; + } +#endif + page_table_check_clear(pte_pfn(pte), mm_pte_native_pages(mm)); } EXPORT_SYMBOL(__page_table_check_pte_clear); @@ -218,9 +296,20 @@ void __page_table_check_ptes_set(struct mm_struct *mm, unsigned long addr, page_table_check_pte_flags(pte); for (i = 0; i < nr; i++) - __page_table_check_pte_clear(mm, addr + PAGE_SIZE * i, ptep_get(ptep + i)); - if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte)) - page_table_check_set(pte_pfn(pte), nr, pte_write(pte)); + __page_table_check_pte_clear(mm, addr + mm_page_size(mm) * i, ptep_get(ptep + i)); + if (!pte_user_accessible_page(mm, addr, pte) || pte_special(pte)) + return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + phys_addr_t phys = pte_phys_mm(mm, pte); + + for (i = 0; i < nr; i++) + page_table_check_subpage(phys + i * mm_page_size(mm), mm_page_size(mm), false, + pte_write(pte)); + return; + } +#endif + page_table_check_set(pte_pfn(pte), nr * mm_pte_native_pages(mm), pte_write(pte)); } EXPORT_SYMBOL(__page_table_check_ptes_set); @@ -279,16 +368,17 @@ void __page_table_check_pte_clear_range(struct mm_struct *mm, return; if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { - pte_t *ptep = pte_offset_map(&pmd, addr); + pte_t *ptep = pte_offset_map_mm(mm, &pmd, addr); + unsigned long nr = pte_table_bytes_mm(mm) / sizeof(pte_t); unsigned long i; if (WARN_ON(!ptep)) return; - for (i = 0; i < PTRS_PER_PTE; i++) { + for (i = 0; i < nr; i++) { __page_table_check_pte_clear(mm, addr, ptep_get(ptep)); - addr += PAGE_SIZE; + addr += mm_page_size(mm); ptep++; } - pte_unmap(ptep - PTRS_PER_PTE); + pte_unmap(ptep - nr); } } diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index ccd1c622c8b24..e3b0b64fbfba2 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -87,7 +87,7 @@ static bool map_pte(struct page_vma_mapped_walk *pvmw, pmd_t *pmdvalp, * mapped at the @pvmw->pte * @pvmw: page_vma_mapped_walk struct, includes a pair pte and pfn range * for checking - * @pte_nr: the number of small pages described by @pvmw->pte. + * @pte_nr: the number of native physical pages covered by @pvmw->pte. * * page_vma_mapped_walk() found a place where pfn range is *potentially* * mapped. check_pte() has to validate this. @@ -109,6 +109,18 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr) unsigned long pfn; pte_t ptent = ptep_get(pvmw->pte); +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) { + if (!(pvmw->flags & PVMW_MIGRATION)) + return pte_present(ptent) && + pte_phys_mm(pvmw->vma->vm_mm, ptent) == + PFN_PHYS(pvmw->pfn) + pvmw->subpage_offset; + if (pte_swp_subpage_offset(ptent) != pvmw->subpage_offset) + return false; + /* The common migration check below validates the native PFN. */ + } +#endif + if (pvmw->flags & PVMW_MIGRATION) { const softleaf_t entry = softleaf_from_pte(ptent); @@ -188,6 +200,18 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) pud_t *pud; pmd_t pmde; +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) { + if (WARN_ON_ONCE(pvmw->nr_pages != 1 || + mm_page_size(mm) >= PAGE_SIZE || + !IS_ALIGNED(pvmw->address, mm_page_size(mm)) || + pvmw->subpage_offset >= PAGE_SIZE || + !IS_ALIGNED(pvmw->subpage_offset, mm_page_size(mm)) || + is_vm_hugetlb_page(vma))) + return not_found(pvmw); + } +#endif + /* The only possible pmd mapping has been handled on last iteration */ if (pvmw->pmd && !pvmw->pte) return not_found(pvmw); @@ -213,6 +237,9 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) return true; } + /* A query for an interior native page still visits one whole user leaf. */ + if (mm_page_size(mm) > PAGE_SIZE) + pvmw->address &= mm_page_mask(mm); end = vma_address_end(pvmw); if (pvmw->pte) goto next_pte; @@ -220,21 +247,21 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) do { pgd = pgd_offset(mm, pvmw->address); if (!pgd_present(*pgd)) { - step_forward(pvmw, PGDIR_SIZE); + step_forward(pvmw, pgd_size_mm(mm)); continue; } - p4d = p4d_offset(pgd, pvmw->address); - if (!p4d_present(*p4d)) { - step_forward(pvmw, P4D_SIZE); + p4d = p4d_offset_mm(mm, pgd, pvmw->address); + if (!p4d_present_mm(mm, *p4d)) { + step_forward(pvmw, p4d_size_mm(mm)); continue; } - pud = pud_offset(p4d, pvmw->address); + pud = pud_offset_mm(mm, p4d, pvmw->address); if (!pud_present(*pud)) { - step_forward(pvmw, PUD_SIZE); + step_forward(pvmw, pud_size_mm(mm)); continue; } - pvmw->pmd = pmd_offset(pud, pvmw->address); + pvmw->pmd = pmd_offset_mm(mm, pud, pvmw->address); /* * Make sure the pmd value isn't cached in a register by the * compiler and used as a stale value after we've observed a @@ -245,6 +272,9 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && (pmd_trans_huge(pmde) || pmd_is_migration_entry(pmde) || pmd_is_device_private_entry(pmde))) { + /* Subpage PMD mappings need a distinct huge-page implementation. */ + if (mm_page_size(mm) != PAGE_SIZE) + return not_found(pvmw); pvmw->ptl = pmd_lock(mm, pvmw->pmd); pmde = *pvmw->pmd; if (pmd_is_migration_entry(pmde)) { @@ -285,7 +315,7 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) (pvmw->nr_pages >= HPAGE_PMD_NR)) sync_with_folio_pmd_zap(mm, pvmw->pmd); - step_forward(pvmw, PMD_SIZE); + step_forward(pvmw, pmd_size_mm(mm)); continue; } if (!map_pte(pvmw, &pmde, &ptl)) { @@ -294,15 +324,15 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) goto next_pte; } this_pte: - if (check_pte(pvmw, 1)) + if (check_pte(pvmw, mm_pte_native_pages(mm))) return true; next_pte: do { - pvmw->address += PAGE_SIZE; + pvmw->address += mm_page_size(mm); if (pvmw->address >= end) return not_found(pvmw); /* Did we cross page table boundary? */ - if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) { + if ((pvmw->address & (pmd_size_mm(mm) - mm_page_size(mm))) == 0) { if (pvmw->ptl) { spin_unlock(pvmw->ptl); pvmw->ptl = NULL; @@ -345,6 +375,7 @@ unsigned long page_mapped_in_vma(const struct page *page, struct vm_area_struct *vma) { const struct folio *folio = page_folio(page); + unsigned long address; struct page_vma_mapped_walk pvmw = { .pfn = page_to_pfn(page), .nr_pages = 1, @@ -352,13 +383,15 @@ unsigned long page_mapped_in_vma(const struct page *page, .flags = PVMW_SYNC, }; - pvmw.address = vma_address(vma, page_pgoff(folio, page), 1); + address = vma_address(vma, page_pgoff(folio, page), 1); + pvmw.address = address; if (pvmw.address == -EFAULT) goto out; if (!page_vma_mapped_walk(&pvmw)) return -EFAULT; page_vma_mapped_walk_done(&pvmw); out: - return pvmw.address; + /* Report this native page's byte address, not the covering leaf start. */ + return address; } #endif diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 5d87c632a2550..c3216b4852399 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -30,13 +30,14 @@ static int walk_pte_range_inner(pte_t *pte, unsigned long addr, unsigned long end, struct mm_walk *walk) { const struct mm_walk_ops *ops = walk->ops; + const unsigned long page_size = mm_page_size(walk->mm); int err = 0; for (;;) { if (ops->install_pte && pte_none(ptep_get(pte))) { pte_t new_pte; - err = ops->install_pte(addr, addr + PAGE_SIZE, &new_pte, + err = ops->install_pte(addr, addr + page_size, &new_pte, walk); if (err) break; @@ -46,13 +47,13 @@ static int walk_pte_range_inner(pte_t *pte, unsigned long addr, if (!WARN_ON_ONCE(walk->no_vma)) update_mmu_cache(walk->vma, addr, pte); } else { - err = ops->pte_entry(pte, addr, addr + PAGE_SIZE, walk); + err = ops->pte_entry(pte, addr, addr + page_size, walk); if (err) break; } - if (addr >= end - PAGE_SIZE) + if (addr >= end - page_size) break; - addr += PAGE_SIZE; + addr += page_size; pte++; } return err; @@ -67,15 +68,15 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, if (walk->no_vma) { /* - * pte_offset_map() might apply user-specific validation. + * pte_offset_map_mm() might apply user-specific validation. * Indeed, on x86_64 the pmd entries set up by init_espfix_ap() * fit its pmd_bad() check (_PAGE_NX set and _PAGE_RW clear), * and CONFIG_EFI_PGT_DUMP efi_mm goes so far as to walk them. */ if (walk->mm == &init_mm || addr >= TASK_SIZE) - pte = pte_offset_kernel(pmd, addr); + pte = pte_offset_kernel_mm(walk->mm, pmd, addr); else - pte = pte_offset_map(pmd, addr); + pte = pte_offset_map_mm(walk->mm, pmd, addr); if (pte) { err = walk_pte_range_inner(pte, addr, end, walk); if (walk->mm != &init_mm && addr < TASK_SIZE) @@ -123,10 +124,10 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, return 0; } - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(walk->mm, pud, addr); do { again: - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(walk->mm, addr, end); if (pmd_none(*pmd)) { if (has_install) err = __pte_alloc(walk->mm, pmd); @@ -193,10 +194,10 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, int err = 0; int depth = real_depth(2); - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(walk->mm, p4d, addr); do { again: - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(walk->mm, addr, end); if (pud_none(*pud)) { if (has_install) err = __pmd_alloc(walk->mm, pud, addr); @@ -258,10 +259,10 @@ static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, int err = 0; int depth = real_depth(1); - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(walk->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) { + next = p4d_addr_end_mm(walk->mm, addr, end); + if (p4d_none_or_clear_bad_mm(walk->mm, p4d)) { if (has_install) err = __pud_alloc(walk->mm, p4d, addr); else if (ops->pte_hole) @@ -297,11 +298,11 @@ static int walk_pgd_range(unsigned long addr, unsigned long end, int err = 0; if (walk->pgd) - pgd = walk->pgd + pgd_index(addr); + pgd = pgd_offset_pgd_mm(walk->mm, walk->pgd, addr); else pgd = pgd_offset(walk->mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(walk->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) { if (has_install) err = __p4d_alloc(walk->mm, pgd, addr); @@ -924,11 +925,11 @@ struct folio *folio_walk_start(struct folio_walk *fw, if (pgd_none_or_clear_bad(pgdp)) goto not_found; - p4dp = p4d_offset(pgdp, addr); - if (p4d_none_or_clear_bad(p4dp)) + p4dp = p4d_offset_mm(vma->vm_mm, pgdp, addr); + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4dp)) goto not_found; - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_mm(vma->vm_mm, p4dp, addr); pud = pudp_get(pudp); if (pud_none(pud)) goto not_found; @@ -937,7 +938,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, ptl = pud_lock(vma->vm_mm, pudp); pud = pudp_get(pudp); - entry_size = PUD_SIZE; + entry_size = pud_size_mm(vma->vm_mm); fw->level = FW_LEVEL_PUD; fw->pudp = pudp; fw->pud = pud; @@ -959,7 +960,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, pmd_table: VM_WARN_ON_ONCE(!pud_present(pud) || pud_leaf(pud)); - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_mm(vma->vm_mm, pudp, addr); pmd = pmdp_get_lockless(pmdp); if (pmd_none(pmd)) goto not_found; @@ -968,7 +969,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, ptl = pmd_lock(vma->vm_mm, pmdp); pmd = pmdp_get(pmdp); - entry_size = PMD_SIZE; + entry_size = pmd_size_mm(vma->vm_mm); fw->level = FW_LEVEL_PMD; fw->pmdp = pmdp; fw->pmd = pmd; @@ -1001,7 +1002,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, goto not_found; pte = ptep_get(ptep); - entry_size = PAGE_SIZE; + entry_size = mm_page_size(vma->vm_mm); fw->level = FW_LEVEL_PTE; fw->ptep = ptep; fw->pte = pte; @@ -1028,5 +1029,5 @@ struct folio *folio_walk_start(struct folio_walk *fw, else fw->page = NULL; fw->ptl = ptl; - return page_folio(page); + return page_folio(zeropage ? page : fw->page); } diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index b91b1a98029c7..7a9eef9b4f5b7 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -168,13 +168,15 @@ pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable) { + struct page *page = pgtable_to_page(pgtable); + assert_spin_locked(pmd_lockptr(mm, pmdp)); /* FIFO */ if (!pmd_huge_pte(mm, pmdp)) - INIT_LIST_HEAD(&pgtable->lru); + INIT_LIST_HEAD(&page->lru); else - list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru); + list_add(&page->lru, &pgtable_to_page(pmd_huge_pte(mm, pmdp))->lru); pmd_huge_pte(mm, pmdp) = pgtable; } #endif @@ -184,15 +186,17 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) { pgtable_t pgtable; + struct page *page, *next; assert_spin_locked(pmd_lockptr(mm, pmdp)); /* FIFO */ pgtable = pmd_huge_pte(mm, pmdp); - pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru, - struct page, lru); + page = pgtable_to_page(pgtable); + next = list_first_entry_or_null(&page->lru, struct page, lru); + pmd_huge_pte(mm, pmdp) = pgtable_from_page(next); if (pmd_huge_pte(mm, pmdp)) - list_del(&pgtable->lru); + list_del(&page->lru); return pgtable; } #endif @@ -282,7 +286,12 @@ static unsigned long pmdp_get_lockless_start(void) { return 0; } static void pmdp_get_lockless_end(unsigned long irqflags) { } #endif +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, pmd_t *pmdvalp) +#else pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) +#endif { unsigned long irqflags; pmd_t pmdval; @@ -302,19 +311,27 @@ pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) pmd_clear_bad(pmd); goto nomap; } - return __pte_map(&pmdval, addr); + return __pte_map_mm(mm, &pmdval, addr); nomap: rcu_read_unlock(); return NULL; } +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) +{ + /* NULL retains native geometry for callers not yet converted. */ + return __pte_offset_map_mm(NULL, pmd, addr, pmdvalp); +} +#endif + pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp) { pmd_t pmdval; pte_t *pte; - pte = __pte_offset_map(pmd, addr, &pmdval); + pte = __pte_offset_map_mm(mm, pmd, addr, &pmdval); if (likely(pte)) *ptlp = pte_lockptr(mm, &pmdval); return pte; @@ -327,7 +344,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, pte_t *pte; VM_WARN_ON_ONCE(!pmdvalp); - pte = __pte_offset_map(pmd, addr, pmdvalp); + pte = __pte_offset_map_mm(mm, pmd, addr, pmdvalp); if (likely(pte)) *ptlp = pte_lockptr(mm, pmdvalp); return pte; @@ -397,7 +414,7 @@ pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, pmd_t pmdval; pte_t *pte; again: - pte = __pte_offset_map(pmd, addr, &pmdval); + pte = __pte_offset_map_mm(mm, pmd, addr, &pmdval); if (unlikely(!pte)) return pte; ptl = pte_lockptr(mm, &pmdval); diff --git a/mm/process_vm_access.c b/mm/process_vm_access.c index 656d3e88755b6..f178b5fdb6589 100644 --- a/mm/process_vm_access.c +++ b/mm/process_vm_access.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,8 @@ #include #include +#include "internal.h" + /** * process_vm_rw_pages - read/write pages from task specified * @pages: array of pointers to pages we want to copy @@ -135,6 +138,63 @@ static int process_vm_rw_single_vec(unsigned long addr, which lives on stack */ #define PVM_MAX_PP_ARRAY_COUNT 16 +#ifdef CONFIG_MM_SUBPAGE +/* Same pin/copy/dirty lifetime as the page-array path, with explicit offsets. */ +int process_vm_rw_fragments(struct mm_struct *mm, unsigned long addr, + unsigned long len, struct iov_iter *iter, bool write) +{ + struct user_page_fragment stack[PVM_MAX_PP_ARRAY_COUNT]; + struct user_page_fragment *fragments = stack; + unsigned long capacity, nr; + int ret = 0; + + if (!len || !iov_iter_count(iter)) + return 0; + capacity = 1 + ((len - 1) >> MM_SUBPAGE_SHIFT) + + ((((len - 1) & (MM_SUBPAGE_SIZE - 1)) + + (addr & (MM_SUBPAGE_SIZE - 1))) >> MM_SUBPAGE_SHIFT); + capacity = min_t(unsigned long, capacity, + PVM_MAX_KMALLOC_PAGES * PAGE_SIZE / sizeof(*fragments)); + if (capacity > ARRAY_SIZE(stack)) { + fragments = kmalloc_array(capacity, sizeof(*fragments), GFP_KERNEL); + if (!fragments) + return -ENOMEM; + } + while (len && iov_iter_count(iter)) { + long got = pin_user_fragments_remote(mm, addr, len, + write ? FOLL_WRITE : 0, fragments, capacity); + + if (got <= 0) { + ret = -EFAULT; + break; + } + for (nr = 0; nr < got && iov_iter_count(iter); nr++) { + struct user_page_fragment *f = &fragments[nr]; + struct page *page = folio_page(f->folio, f->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(f->offset); + size_t copied; + + if (write) + copied = copy_page_from_iter(page, offset, f->length, iter); + else + copied = copy_page_to_iter(page, offset, f->length, iter); + addr += copied; + len -= copied; + if (copied < f->length && iov_iter_count(iter)) { + ret = -EFAULT; + break; + } + } + release_user_fragments(fragments, got, write); + if (ret) + break; + } + if (fragments != stack) + kfree(fragments); + return ret; +} +#endif + /** * process_vm_rw_core - core of reading/writing pages from task specified * @pid: PID of process to read/write from/to @@ -182,17 +242,6 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, if (nr_pages == 0) return 0; - if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) { - /* For reliability don't try to kmalloc more than - 2 pages worth */ - process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES * PAGE_SIZE, - sizeof(struct page *)*nr_pages), - GFP_KERNEL); - - if (!process_pages) - return -ENOMEM; - } - /* Get process information */ task = find_get_task_by_vpid(pid); if (!task) { @@ -212,11 +261,35 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, goto put_task_struct; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + for (i = 0; i < riovcnt && iov_iter_count(iter) && !rc; i++) + rc = process_vm_rw_fragments(mm, (unsigned long)rvec[i].iov_base, + rvec[i].iov_len, iter, vm_write); + goto copied; + } +#endif + if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) { + /* For reliability don't try to kmalloc more than + 2 pages worth */ + process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES * PAGE_SIZE, + sizeof(struct page *)*nr_pages), + GFP_KERNEL); + + if (!process_pages) { + rc = -ENOMEM; + goto put_mm; + } + } + for (i = 0; i < riovcnt && iov_iter_count(iter) && !rc; i++) rc = process_vm_rw_single_vec( (unsigned long)rvec[i].iov_base, rvec[i].iov_len, iter, process_pages, mm, task, vm_write); +#ifdef CONFIG_MM_SUBPAGE +copied: +#endif /* copied = space before - space after */ total_len -= iov_iter_count(iter); @@ -226,6 +299,7 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, if (total_len) rc = total_len; +put_mm: mmput(mm); put_task_struct: diff --git a/mm/rmap.c b/mm/rmap.c index 99e1b3dc390b7..795b568b9f7c5 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -75,6 +75,7 @@ #include #include #include +#include #include @@ -83,17 +84,22 @@ #include "internal.h" #include "swap.h" +#include "user-subpage-internal.h" static struct kmem_cache *anon_vma_cachep; static struct kmem_cache *anon_vma_chain_cachep; -static inline struct anon_vma *anon_vma_alloc(void) +static inline struct anon_vma *anon_vma_alloc(struct vm_area_struct *vma) { struct anon_vma *anon_vma; anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); if (anon_vma) { atomic_set(&anon_vma->refcount, 1); +#ifdef CONFIG_MM_SUBPAGE + anon_vma->min_folio_order = + max_t(unsigned int, mm_page_shift(vma->vm_mm), PAGE_SHIFT) - PAGE_SHIFT; +#endif anon_vma->num_children = 0; anon_vma->num_active_vmas = 0; anon_vma->parent = anon_vma; @@ -198,7 +204,7 @@ int __anon_vma_prepare(struct vm_area_struct *vma) anon_vma = find_mergeable_anon_vma(vma); allocated = NULL; if (!anon_vma) { - anon_vma = anon_vma_alloc(); + anon_vma = anon_vma_alloc(vma); if (unlikely(!anon_vma)) goto out_enomem_free_avc; anon_vma->num_children++; /* self-parent link for new root */ @@ -388,7 +394,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) /* Drop inherited anon_vma, we'll reuse existing or allocate new. */ vma->anon_vma = NULL; - anon_vma = anon_vma_alloc(); + anon_vma = anon_vma_alloc(vma); if (!anon_vma) return -ENOMEM; avc = anon_vma_chain_alloc(GFP_KERNEL); @@ -622,6 +628,35 @@ struct anon_vma *folio_get_anon_vma(const struct folio *folio) return anon_vma; } +#ifdef CONFIG_MM_SUBPAGE +/* The caller retains the folio; splitting rechecks the order under its lock. */ +unsigned int folio_anon_min_order(struct folio *folio) +{ + struct anon_vma *anon_vma = NULL; + unsigned long mapping; + unsigned int order = 0; + + /* Also used by memory-failure callers before they lock the folio. */ + rcu_read_lock(); + mapping = (unsigned long)READ_ONCE(folio->mapping); + if ((mapping & FOLIO_MAPPING_FLAGS) != FOLIO_MAPPING_ANON || !folio_mapped(folio)) + goto out; + anon_vma = (struct anon_vma *)(mapping - FOLIO_MAPPING_ANON); + if (!atomic_inc_not_zero(&anon_vma->refcount)) { + anon_vma = NULL; + goto out; + } + /* Revalidate against unmap/reuse of the RCU-typesafe anon_vma slab. */ + if (folio_mapped(folio) && mapping == (unsigned long)READ_ONCE(folio->mapping)) + order = anon_vma->min_folio_order; +out: + rcu_read_unlock(); + if (anon_vma) + put_anon_vma(anon_vma); + return order; +} +#endif + /* * Similar to folio_get_anon_vma() except it locks the anon_vma. * @@ -856,6 +891,13 @@ static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) unsigned long page_address_in_vma(const struct folio *folio, const struct page *page, const struct vm_area_struct *vma) { +#ifdef CONFIG_MM_SUBPAGE + unsigned long address; + + if (folio_test_anon(folio) && !folio_test_large(folio) && + mm_subpage_address_in_vma(folio, vma, &address)) + return address; +#endif if (folio_test_anon(folio)) { struct anon_vma *anon_vma = folio_anon_vma(folio); /* @@ -891,15 +933,15 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) if (!pgd_present(*pgd)) goto out; - p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + p4d = p4d_offset_mm(mm, pgd, address); + if (!p4d_present_mm(mm, *p4d)) goto out; - pud = pud_offset(p4d, address); + pud = pud_offset_mm(mm, p4d, address); if (!pud_present(*pud)) goto out; - pmd = pmd_offset(pud, address); + pmd = pmd_offset_mm(mm, pud, address); out: return pmd; } @@ -915,10 +957,11 @@ struct folio_referenced_arg { * arg: folio_referenced_arg will be passed */ static bool folio_referenced_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long address, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct folio_referenced_arg *pra = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); int ptes = 0, referenced = 0; unsigned int nr; @@ -928,10 +971,11 @@ static bool folio_referenced_one(struct folio *folio, if (vma->vm_flags & VM_LOCKED) { ptes++; - pra->mapcount--; + pra->mapcount -= pvmw.pte ? mm_pte_native_pages(vma->vm_mm) : 1; /* Only mlock fully mapped pages */ - if (pvmw.pte && ptes != pvmw.nr_pages) + if (pvmw.pte && ptes != (folio_test_large(folio) ? + ((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm) : 1)) continue; /* @@ -966,11 +1010,18 @@ static bool folio_referenced_one(struct folio *folio, } if (pvmw.pte && folio_test_large(folio)) { - const unsigned long end_addr = pmd_addr_end(address, vma->vm_end); - const unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT; + const unsigned long end_addr = pmd_addr_end_mm(vma->vm_mm, + address, vma->vm_end); + const unsigned int max_nr = (end_addr - address) >> mm_page_shift(vma->vm_mm); pte_t pteval = ptep_get(pvmw.pte); - nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + nr = folio_subpage_pte_batch(vma, folio, pvmw.pte, + pteval, max_nr, 0); + else +#endif + nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr); } /* @@ -994,19 +1045,19 @@ static bool folio_referenced_one(struct folio *folio, } ptes += nr; - pra->mapcount -= nr; + pra->mapcount -= nr * (pvmw.pte ? mm_pte_native_pages(vma->vm_mm) : 1); /* * If we are sure that we batched the entire folio, * we can just optimize and stop right here. */ - if (ptes == pvmw.nr_pages) { + if (ptes == (((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm))) { page_vma_mapped_walk_done(&pvmw); break; } /* Skip the batched PTEs */ pvmw.pte += nr - 1; - pvmw.address += (nr - 1) * PAGE_SIZE; + pvmw.address += (nr - 1) * mm_page_size(vma->vm_mm); } if (referenced) @@ -1178,9 +1229,9 @@ static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw) } static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, PVMW_SYNC); int *cleaned = arg; *cleaned += page_vma_mkclean_one(&pvmw); @@ -1229,8 +1280,9 @@ struct wrprotect_file_state { }; static bool mapping_wrprotect_range_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long address, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct wrprotect_file_state *state = (struct wrprotect_file_state *)arg; struct page_vma_mapped_walk pvmw = { .pfn = state->pfn, @@ -1515,7 +1567,7 @@ static void __page_check_anon_rmap(const struct folio *folio, static __always_inline void __folio_add_anon_rmap(struct folio *folio, struct page *page, int nr_pages, struct vm_area_struct *vma, - unsigned long address, rmap_t flags, enum pgtable_level level) + unsigned long address, rmap_t flags, enum pgtable_level level, bool new) { int i; @@ -1572,7 +1624,7 @@ static __always_inline void __folio_add_anon_rmap(struct folio *folio, * Partially mapped folios can be split on reclaim and part outside * of mlocked VMA can be evicted or freed. */ - if (folio_nr_pages(folio) == nr_pages) + if (!new && folio_nr_pages(folio) == nr_pages) mlock_vma_folio(folio, vma); } @@ -1597,7 +1649,7 @@ void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page, rmap_t flags) { __folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags, - PGTABLE_LEVEL_PTE); + PGTABLE_LEVEL_PTE, false); } /** @@ -1618,7 +1670,7 @@ void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page, { #ifdef CONFIG_TRANSPARENT_HUGEPAGE __folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags, - PGTABLE_LEVEL_PMD); + PGTABLE_LEVEL_PMD, false); #else WARN_ON_ONCE(true); #endif @@ -1697,6 +1749,112 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1); } +#ifdef CONFIG_MM_SUBPAGE +static int __mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address, bool new) +{ + struct folio *folio = mm_subpage_folio(subpage); + int ret; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (mm_page_shift(vma->vm_mm) != mm_subpage_shift(subpage) || !vma->anon_vma || + address < vma->vm_start || address >= vma->vm_end || + !IS_ALIGNED(address, mm_subpage_size(subpage))) + return -EINVAL; + if (mm_subpage_anon_root(subpage)) { + if (mm_subpage_is_droppable(subpage) != !!(vma->vm_flags & VM_DROPPABLE)) + return -EINVAL; + if (!folio_test_anon(folio) || folio_test_ksm(folio) || + PageAnonExclusive(&folio->page)) + return -EINVAL; + } else { + bool was_anon = folio_test_anon(folio); + + /* Swap-cache folios can retain a stale, unmapped anon pointer. */ + if (folio_mapped(folio) || folio_test_ksm(folio) || + (folio->mapping && !was_anon) || + (was_anon && PageAnonExclusive(&folio->page))) + return -EINVAL; + /* Slot references can survive the last VMA in this root. */ + mm_subpage_hold_anon_root(subpage, vma->anon_vma->root, + vma->vm_flags & VM_DROPPABLE); + /* + * A restored folio is already visible in the swap cache. Other + * faults may set PG_waiters while we hold its lock; preserve those + * concurrent flag updates even when PG_swapbacked is already set. + */ + if (!(vma->vm_flags & VM_DROPPABLE)) + folio_set_swapbacked(folio); + /* Neighboring slots may acquire different fork/COW relationships. */ + __folio_set_anon(folio, vma, address, false); + if (!was_anon) + mod_mthp_stat(0, MTHP_STAT_NR_ANON, 1); + } + ret = mm_subpage_bind_rmap(subpage, vma, address); + if (ret) + return ret; + atomic_inc(&subpage->mapcount); + /* Identity was checked per slot, not against the folio's linear anchor. */ + __folio_add_rmap(folio, &folio->page, 1, vma, PGTABLE_LEVEL_PTE); + if (!new) + mlock_vma_folio(folio, vma); + return 0; +} + +int mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address) +{ + return __mm_subpage_add_anon_rmap(subpage, vma, address, false); +} + +/* Caller adds this fresh folio to LRU and accounts its locked leaves together. */ +int mm_subpage_add_new_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address) +{ + return __mm_subpage_add_anon_rmap(subpage, vma, address, true); +} + +void mm_subpage_remove_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma) +{ + struct folio *folio = mm_subpage_folio(subpage); + + VM_BUG_ON_FOLIO(atomic_read(&subpage->mapcount) <= 0, folio); + atomic_dec(&subpage->mapcount); + folio_remove_rmap_pte(folio, &folio->page, vma); +} + +int mm_subpage_dup_anon_rmap(struct mm_subpage *subpage, + struct vm_area_struct *dst, struct vm_area_struct *src, unsigned long address) +{ + struct folio *folio = mm_subpage_folio(subpage); + struct vm_page_offset pos; + int ret; + + mmap_assert_write_locked(src->vm_mm); + mmap_assert_write_locked(dst->vm_mm); + if (mm_page_shift(src->vm_mm) != mm_subpage_shift(subpage) || + mm_page_shift(dst->vm_mm) != mm_subpage_shift(subpage) || + !src->anon_vma || !dst->anon_vma || !folio_test_anon(folio) || + folio_test_ksm(folio) || PageAnonExclusive(&folio->page) || + address < dst->vm_start || address >= dst->vm_end || + address < src->vm_start || address >= src->vm_end) + return -EINVAL; + pos = vma_page_offset_at(dst, address); + /* Both mmap locks and the source PTE stabilize this slot's identity. */ + if (!mm_subpage_match_rmap(subpage, dst, address) || + src->anon_vma->root != dst->anon_vma->root || + !vm_page_offset_equal(pos, vma_page_offset_at(src, address))) + return -EINVAL; + /* A pin in a neighboring physical slot does not share this slot. */ + if (!mm_subpage_try_share(subpage)) + return -EBUSY; + ret = folio_try_dup_anon_rmap_pte(folio, &folio->page, dst, src); + if (!ret) + atomic_inc(&subpage->mapcount); + return ret; +} +#endif + static __always_inline void __folio_add_file_rmap(struct folio *folio, struct page *page, int nr_pages, struct vm_area_struct *vma, enum pgtable_level level) @@ -1949,14 +2107,20 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, struct vm_area_struct *vma = pvmw->vma; unsigned int max_nr; +#ifdef CONFIG_MM_SUBPAGE + /* Native PFN batching advances physical pages, not 4K file quarters. */ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return 1; +#endif + if (flags & TTU_HWPOISON) return 1; if (!folio_test_large(folio)) return 1; /* We may only batch within a single VMA and a single page table. */ - end_addr = pmd_addr_end(addr, vma->vm_end); - max_nr = (end_addr - addr) >> PAGE_SHIFT; + end_addr = pmd_addr_end_mm(vma->vm_mm, addr, vma->vm_end); + max_nr = (end_addr - addr) >> mm_page_shift(vma->vm_mm); /* We only support lazyfree or file folios batching for now ... */ if (folio_test_anon(folio) && folio_test_swapbacked(folio)) @@ -1978,20 +2142,116 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY); } +#ifdef CONFIG_MM_SUBPAGE +/* Folio and PTE locks held. Each PTE takes one reference to a native swap entry. */ +static bool try_to_unmap_subpage(struct folio *folio, struct vm_area_struct *vma, + struct page_vma_mapped_walk *pvmw, enum ttu_flags flags) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long address = pvmw->address, end = address + mm_page_size(mm); + struct mm_subpage *slot; + pte_t pte = ptep_get(pvmw->pte), swapped; + swp_entry_t entry; + bool exclusive, ret = false; + + if (!pte_present(pte) || (flags & TTU_HWPOISON)) + return false; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (WARN_ON_ONCE(!slot)) + return false; + if ((vma->vm_flags & VM_DROPPABLE) || mm_subpage_is_lazyfree(slot)) { + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + /* A released quarter can be reused before its native folio is freed. */ + flush_tlb_range(vma, address, end); + smp_mb(); + if (((vma->vm_flags & VM_DROPPABLE) || !pte_dirty(pte)) && + mm_subpage_discardable(slot) && + arch_unmap_one(mm, vma, address, pte) >= 0) { + update_hiwater_rss(mm); + dec_mm_counter(mm, MM_ANONPAGES); + mm_subpage_remove_anon_rmap(slot, vma); + mm_subpage_put(slot); /* Removed PTE reference. */ + ret = true; + goto out; + } + set_pte_at(mm, address, pvmw->pte, pte); + /* Retained references must not turn a droppable folio into swap data. */ + if (vma->vm_flags & VM_DROPPABLE) + goto out; + if (pte_dirty(pte)) { + mm_subpage_cancel_lazyfree(slot); + folio_mark_dirty(folio); + } + } + if (!folio_test_swapbacked(folio)) { + /* A write or retained reference cancelled whole-folio lazy reclaim. */ + folio_set_swapbacked(folio); + goto out; + } + if (!folio_test_swapcache(folio)) + goto out; + entry = page_swap_entry(&folio->page); + if (swap_subpage_dup(entry, mm_subpage_offset(slot)) < 0) + goto out; + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + if (should_defer_flush(mm, flags)) + set_tlb_ubc_flush_pending(mm, pte, address, end); + else + flush_tlb_range(vma, address, end); + if (pte_dirty(pte)) + folio_mark_dirty(folio); + /* The PTE lock serializes fork and this slot's ownership transfer. */ + exclusive = mm_subpage_is_exclusive(slot); + if (arch_unmap_one(mm, vma, address, pte) < 0 || !mm_subpage_try_share(slot)) { + swap_subpage_put(entry, mm_subpage_offset(slot), false); + set_pte_at(mm, address, pvmw->pte, pte); + goto out; + } + if (list_empty(&mm->mmlist)) { + spin_lock(&mmlist_lock); + if (list_empty(&mm->mmlist)) + list_add(&mm->mmlist, &init_mm.mmlist); + spin_unlock(&mmlist_lock); + } + swapped = pte_swp_set_subpage_offset(swp_entry_to_pte(entry), mm_subpage_offset(slot)); + if (exclusive) + swapped = pte_swp_mkexclusive(swapped); + if (pte_soft_dirty(pte)) + swapped = pte_swp_mksoft_dirty(swapped); + if (pte_uffd_wp(pte)) + swapped = pte_swp_mkuffd_wp(swapped); + update_hiwater_rss(mm); + dec_mm_counter(mm, MM_ANONPAGES); + inc_mm_counter(mm, MM_SWAPENTS); + set_pte_at(mm, address, pvmw->pte, swapped); + mm_subpage_remove_anon_rmap(slot, vma); + /* Swap-cache backing cannot be recycled while a deferred TLBI is pending. */ + mm_subpage_put(slot); + ret = true; +out: + mm_subpage_put(slot); + return ret; +} +#endif + /* * @arg: enum ttu_flags will be passed to this argument */ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct mm_struct *mm = vma->vm_mm; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool anon_exclusive, ret = true; pte_t pteval; + phys_addr_t file_phys = 0; struct page *subpage; struct mmu_notifier_range range; enum ttu_flags flags = (enum ttu_flags)(long)arg; - unsigned long nr_pages = 1, end_addr; + unsigned long nr_pages = 1, end_addr, nr_refs; unsigned long pfn; unsigned long hsz = 0; int ptes = 0; @@ -2003,7 +2263,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * if page table locking is skipped: use TTU_SYNC to wait for that. */ if (flags & TTU_SYNC) - pvmw.flags = PVMW_SYNC; + pvmw.flags |= PVMW_SYNC; /* * For THP, we have to assume the worse case ie pmd for invalidation. @@ -2049,7 +2309,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, ret = false; /* Only mlock fully mapped pages */ - if (pvmw.pte && ptes != pvmw.nr_pages) + if (pvmw.pte && ptes != (folio_test_large(folio) ? + ((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm) : 1)) continue; /* @@ -2100,6 +2361,15 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * actually map pages. */ pteval = ptep_get(pvmw.pte); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE && folio_test_anon(folio)) { + if (!try_to_unmap_subpage(folio, vma, &pvmw, flags)) + goto walk_abort; + /* One native folio may cover four independently mapped slots. */ + continue; + } +#endif + if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { @@ -2110,6 +2380,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, } subpage = folio_page(folio, pfn - folio_pfn(folio)); + if (!folio_test_anon(folio)) + file_phys = pte_phys_mm(mm, ptep_get(pvmw.pte)); address = pvmw.address; anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(subpage); @@ -2167,7 +2439,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, folio_mark_dirty(folio); } else if (likely(pte_present(pteval))) { nr_pages = folio_unmap_pte_batch(folio, &pvmw, flags, pteval); - end_addr = address + nr_pages * PAGE_SIZE; + end_addr = address + nr_pages * mm_page_size(mm); flush_cache_range(vma, address, end_addr); /* Nuke the page table entry. */ @@ -2281,7 +2553,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, goto discard; } - if (folio_dup_swap(folio, subpage) < 0) { + if (folio_dup_swap_range(folio, subpage, mm_pte_native_pages(mm)) < 0) { set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } @@ -2292,15 +2564,15 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * so we'll not check/care. */ if (arch_unmap_one(mm, vma, address, pteval) < 0) { - folio_put_swap(folio, subpage); + folio_put_swap_range(folio, subpage, mm_pte_native_pages(mm)); set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } /* See folio_try_share_anon_rmap(): clear PTE first. */ if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, subpage)) { - folio_put_swap(folio, subpage); + folio_try_share_anon_rmap_pte(folio, subpage, vma)) { + folio_put_swap_range(folio, subpage, mm_pte_native_pages(mm)); set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } @@ -2342,20 +2614,26 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, add_mm_counter(mm, mm_counter_file(folio), -nr_pages); } discard: + nr_refs = nr_pages; + /* A base user leaf can own references to several native pages. */ + if (!folio_test_hugetlb(folio)) + nr_refs *= mm_pte_native_pages(mm); + if (!folio_test_anon(folio)) + mm_subpage_file_map_del(vma, file_phys); if (unlikely(folio_test_hugetlb(folio))) { hugetlb_remove_rmap(folio); } else { - folio_remove_rmap_ptes(folio, subpage, nr_pages, vma); + folio_remove_rmap_ptes(folio, subpage, nr_refs, vma); } if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); - folio_put_refs(folio, nr_pages); + folio_put_refs(folio, nr_refs); /* * If we are sure that we batched the entire folio and cleared * all PTEs, we can just optimize and stop right here. */ - if (nr_pages == folio_nr_pages(folio)) + if (nr_pages == folio_nr_pages(folio) && mm_page_size(mm) == PAGE_SIZE) goto walk_done; continue; walk_abort: @@ -2412,11 +2690,72 @@ void try_to_unmap(struct folio *folio, enum ttu_flags flags) * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs * containing migration entries. */ +#ifdef CONFIG_MM_SUBPAGE +static bool try_to_migrate_subpage(struct folio *folio, struct vm_area_struct *vma, + struct page_vma_mapped_walk *pvmw, enum ttu_flags flags) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long address = pvmw->address, end = address + mm_page_size(mm); + struct mm_subpage *slot; + pte_t pte = ptep_get(pvmw->pte), migration; + swp_entry_t entry; + bool ret = false; + + /* Other migration drivers must prepare ownership before removing PTEs. */ + if (!pte_present(pte) || !mm_subpage_migrating(folio) || + folio_test_hwpoison(folio) || folio_is_zone_device(folio)) + return false; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (WARN_ON_ONCE(!slot)) + return false; + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + if (should_defer_flush(mm, flags)) + set_tlb_ubc_flush_pending(mm, pte, address, end); + else + flush_tlb_range(vma, address, end); + if (pte_dirty(pte)) + folio_mark_dirty(folio); + if (arch_unmap_one(mm, vma, address, pte) < 0) { + set_pte_at(mm, address, pvmw->pte, pte); + goto out; + } + if (pte_write(pte)) + entry = make_writable_migration_entry(folio_pfn(folio)); + else if (mm_subpage_is_exclusive(slot)) + entry = make_readable_exclusive_migration_entry(folio_pfn(folio)); + else + entry = make_readable_migration_entry(folio_pfn(folio)); + if (pte_young(pte)) + entry = make_migration_entry_young(entry); + if (pte_dirty(pte)) + entry = make_migration_entry_dirty(entry); + migration = pte_swp_set_subpage_offset(swp_entry_to_pte(entry), mm_subpage_offset(slot)); + if (pte_soft_dirty(pte)) + migration = pte_swp_mksoft_dirty(migration); + if (pte_uffd_wp(pte)) + migration = pte_swp_mkuffd_wp(migration); + set_pte_at(mm, address, pvmw->pte, migration); + trace_set_migration_pte(address, pte_val(migration), 0); + mm_subpage_migrate_save_lazyfree(slot); + mm_subpage_remove_anon_rmap(slot, vma); + if (vma->vm_flags & VM_LOCKED) + mlock_drain_local(); + /* The driver holds backing until its batched invalidation completes. */ + mm_subpage_put(slot); + ret = true; +out: + mm_subpage_put(slot); + return ret; +} +#endif + static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct mm_struct *mm = vma->vm_mm; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool anon_exclusive, writable, ret = true; pte_t pteval; struct page *subpage; @@ -2424,6 +2763,11 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, enum ttu_flags flags = (enum ttu_flags)(long)arg; unsigned long pfn; unsigned long hsz = 0; + unsigned int nr_refs = 1; + + /* A base PTE owns one reference for each covered native page. */ + if (!folio_test_hugetlb(folio)) + nr_refs = mm_pte_native_pages(mm); /* * When racing against e.g. zap_pte_range() on another cpu, @@ -2432,7 +2776,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, * if page table locking is skipped: use TTU_SYNC to wait for that. */ if (flags & TTU_SYNC) - pvmw.flags = PVMW_SYNC; + pvmw.flags |= PVMW_SYNC; /* * For THP, we have to assume the worse case ie pmd for invalidation. @@ -2501,6 +2845,17 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, /* Unexpected PMD-mapped THP? */ VM_BUG_ON_FOLIO(!pvmw.pte, folio); + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE && folio_test_anon(folio)) { + if (!try_to_migrate_subpage(folio, vma, &pvmw, flags)) { + ret = false; + page_vma_mapped_walk_done(&pvmw); + break; + } + continue; + } +#endif /* * Handle PFN swap PTEs, such as device-exclusive ones, that * actually map pages. @@ -2586,7 +2941,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ pteval = ptep_get_and_clear(mm, address, pvmw.pte); - set_tlb_ubc_flush_pending(mm, pteval, address, address + PAGE_SIZE); + set_tlb_ubc_flush_pending(mm, pteval, address, address + mm_page_size(mm)); } else { pteval = ptep_clear_flush(vma, address, pvmw.pte); } @@ -2663,7 +3018,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, break; } } else if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, subpage)) { + folio_try_share_anon_rmap_pte(folio, subpage, vma)) { set_pte_at(mm, address, pvmw.pte, pteval); ret = false; page_vma_mapped_walk_done(&pvmw); @@ -2701,6 +3056,13 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, if (pte_swp_uffd_wp(pteval)) swp_pte = pte_swp_mkuffd_wp(swp_pte); } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) + swp_pte = pte_swp_set_subpage_offset(swp_pte, + pte_present(pteval) ? pte_phys_mm(mm, pteval) & ~PAGE_MASK : + pte_swp_subpage_offset(pteval)); +#endif + if (folio_test_hugetlb(folio)) set_huge_pte_at(mm, address, pvmw.pte, swp_pte, hsz); @@ -2714,13 +3076,15 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ } + if (!folio_test_anon(folio)) + mm_subpage_file_map_del(vma, pte_phys_mm(mm, pteval)); if (unlikely(folio_test_hugetlb(folio))) hugetlb_remove_rmap(folio); else folio_remove_rmap_pte(folio, subpage, vma); if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); - folio_put(folio); + folio_put_refs(folio, nr_refs); } mmu_notifier_invalidate_range_end(&range); @@ -2951,6 +3315,61 @@ static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio, return anon_vma; } +#ifdef CONFIG_MM_SUBPAGE +noinline bool rmap_walk_subpages(struct folio *folio, + struct rmap_walk_control *rwc, bool locked) +{ + struct mm_subpage_rmap maps[MM_SUBPAGES_PER_PAGE]; + int i, nr = mm_subpage_snapshot_rmap(folio, maps); + + if (nr < 0) + return false; + /* Existing locked anonymous walks operate on large folios. */ + VM_BUG_ON_FOLIO(locked, folio); + for (i = 0; i < nr; i++) { + struct mm_subpage_rmap *map = &maps[i]; + struct anon_vma_chain *avc; + bool stop = false; + + if (!anon_vma_trylock_read(map->root)) { + if (rwc->try_lock) { + rwc->contended = true; + break; + } + anon_vma_lock_read(map->root); + } + anon_vma_interval_tree_foreach(avc, &map->root->rb_root, + map->pos.index, map->pos.index) { + struct vm_area_struct *vma = avc->vma; + unsigned long address = vma_address_at_offset(vma, map->pos); + struct rmap_walk_range range = { + .address = address, + .subpage = true, + .subpage_offset = map->offset, + }; + + /* Native-index overlap need not include this exact leaf. */ + if (address == -EFAULT) + continue; + cond_resched(); + if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) + continue; + if (!rwc->rmap_one(folio, vma, range, rwc->arg) || + (rwc->done && rwc->done(folio))) { + stop = true; + break; + } + } + anon_vma_unlock_read(map->root); + if (stop) + break; + } + for (i = 0; i < nr; i++) + put_anon_vma(maps[i].root); + return true; +} +#endif + /* * rmap_walk_anon - do something to anonymous page using the object-based * rmap method @@ -2973,6 +3392,10 @@ static void rmap_walk_anon(struct folio *folio, * to an anon_vma with different root. */ VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); +#ifdef CONFIG_MM_SUBPAGE + if (!folio_test_large(folio) && rmap_walk_subpages(folio, rwc, locked)) + return; +#endif if (locked) { anon_vma = folio_anon_vma(folio); @@ -2998,7 +3421,8 @@ static void rmap_walk_anon(struct folio *folio, if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, address, rwc->arg)) + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = address }, rwc->arg)) break; if (rwc->done && rwc->done(folio)) break; @@ -3061,7 +3485,8 @@ static void __rmap_walk_file(struct folio *folio, struct address_space *mapping, if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, address, rwc->arg)) + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = address }, rwc->arg)) goto done; if (rwc->done && rwc->done(folio)) goto done; diff --git a/mm/shmem.c b/mm/shmem.c index 7b1ea9fb598f4..705765dddf1d2 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -30,6 +30,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -875,6 +878,266 @@ static void shmem_update_stats(struct folio *folio, int nr_pages) lruvec_stat_mod_folio(folio, NR_SHMEM, nr_pages); } +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +#define SHMEM_UFFD_ALL_UNITS GENMASK(PAGE_SIZE / SZ_4K - 1, 0) + +/* + * Resident masks live in page_ext, allowing partial truncate to update them + * without allocation. All cache insertion/replacement paths initialize or + * transfer them under the folio lock. The inode XArray persists masks only + * for swapped indices. Its publication/removal additionally holds i_pages, + * before taking uffd_missing's XArray lock. + */ +static void shmem_uffd_enable(struct inode *inode) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + + spin_lock(&info->lock); + info->flags |= SHMEM_F_UFFD_SUBPAGE; + spin_unlock(&info->lock); +} + +static unsigned long shmem_uffd_mask(unsigned int offset, unsigned long length) +{ + return GENMASK((offset + length - 1) / SZ_4K, offset / SZ_4K); +} + +unsigned long shmem_uffd_missing_mask(struct folio *folio, pgoff_t index) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); + VM_BUG_ON_FOLIO(!folio_mapping(folio) || !shmem_mapping(folio_mapping(folio)), folio); + return mm_subpage_shmem_missing(folio_page(folio, index - folio->index)); +} + +int shmem_uffd_set_missing(struct folio *folio, pgoff_t index, + unsigned long missing) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!folio_mapping(folio) || !shmem_mapping(folio_mapping(folio)) || + !folio_contains(folio, index) || + !folio_test_uptodate(folio) || (missing & ~SHMEM_UFFD_ALL_UNITS)) + return -EINVAL; + if (missing) + shmem_uffd_enable(folio->mapping->host); + mm_subpage_shmem_set_missing(folio_page(folio, index - folio->index), missing); + return 0; +} + +bool shmem_uffd_has_missing_native_page(struct folio *folio) +{ + struct address_space *mapping = folio_mapping(folio); + unsigned long i; + + if (!mapping || !shmem_mapping(mapping) || + !(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return false; + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + /* Native PMD exposure is allowed when every native leaf has some data. */ + for (i = 0; i < folio_nr_pages(folio); i++) + if (mm_subpage_shmem_missing(folio_page(folio, i)) == SHMEM_UFFD_ALL_UNITS) + return true; + return false; +} + +bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length) +{ + struct address_space *mapping = folio_mapping(folio); + unsigned long mask; + + if (!mapping || !shmem_mapping(mapping)) + return false; + mask = shmem_uffd_mask(offset, length); + return (shmem_uffd_missing_mask(folio, index) & mask) == mask; +} + +/* A coarse alias with any populated units preserves them and exposes zero holes. */ +static unsigned long shmem_uffd_reason(struct vm_fault *vmf, unsigned long missing) +{ + struct vm_area_struct *vma = vmf->vma; + struct vm_page_offset pos = vma_page_offset_at(vma, vmf->address); + unsigned long mask = shmem_uffd_mask(pos.offset, mm_page_size(vma->vm_mm)); + + if ((missing & mask) == mask) + return userfaultfd_missing(vma) ? VM_UFFD_MISSING : 0; + return userfaultfd_minor(vma) ? VM_UFFD_MINOR : 0; +} + +void shmem_uffd_mark_present(struct folio *folio, pgoff_t index, + unsigned long present) +{ + unsigned long missing = shmem_uffd_missing_mask(folio, index); + + if (missing & present) + mm_subpage_shmem_set_missing(folio_page(folio, index - folio->index), + missing & ~present); +} + +void shmem_uffd_mark_mapped(struct folio *folio, unsigned long offset, + unsigned long length) +{ + struct address_space *mapping = folio_mapping(folio); + + if (!mapping || !shmem_mapping(mapping) || + !(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return; + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(offset + length > folio_size(folio), folio); + while (length) { + unsigned int low = offset & ~PAGE_MASK; + unsigned long size = min(length, PAGE_SIZE - low); + unsigned long mask = GENMASK((low + size - 1) / SZ_4K, low / SZ_4K); + + shmem_uffd_mark_present(folio, folio->index + (offset >> PAGE_SHIFT), mask); + offset += size; + length -= size; + } +} + +void shmem_uffd_forget(struct address_space *mapping, pgoff_t start, + unsigned long nr) +{ + struct xarray *xa; + unsigned long index; + void *entry; + + if (!shmem_mapping(mapping)) + return; + lockdep_assert_held(&mapping->i_pages.xa_lock); + xa = &SHMEM_I(mapping->host)->uffd_missing; + if (xa_empty(xa)) + return; + xa_lock(xa); + xa_for_each_range(xa, index, entry, start, start + nr - 1) + __xa_erase(xa, index); + xa_unlock(xa); +} + +/* New cache identity or swap-in: initialize every native page before unlock. */ +static void shmem_uffd_cache_init(struct folio *folio, bool from_swap) +{ + struct address_space *mapping = folio->mapping; + struct xarray *xa = &SHMEM_I(mapping->host)->uffd_missing; + unsigned long i; + + lockdep_assert_held(&mapping->i_pages.xa_lock); + for (i = 0; i < folio_nr_pages(folio); i++) { + void *entry = from_swap ? xa_load(xa, folio->index + i) : NULL; + + mm_subpage_shmem_set_missing(folio_page(folio, i), + entry ? xa_to_value(entry) : 0); + } + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); +} + +/* + * Reserve persistence before committing swap. On failure the cache folio and + * its resident mask stay intact; reclaim may retry once memory is available. + */ +static int shmem_uffd_swap_prepare(struct folio *folio) +{ + struct address_space *mapping = folio->mapping; + struct xarray *xa = &SHMEM_I(mapping->host)->uffd_missing; + struct mem_cgroup *memcg, *saved; + unsigned long i; + int err = 0; + + if (!(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return 0; + /* Reclaim may run outside the backing folio's charge domain. */ + rcu_read_lock(); + do { + memcg = folio_memcg(folio) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + for (i = 0; i < folio_nr_pages(folio); i++) { + unsigned long mask = mm_subpage_shmem_missing(folio_page(folio, i)); + + if (!mask) + continue; + err = xa_err(xa_store(xa, folio->index + i, xa_mk_value(mask), + GFP_NOWAIT | __GFP_ACCOUNT | __GFP_NOWARN)); + if (err) + break; + } + set_active_memcg(saved); + mem_cgroup_put(memcg); + if (err) { + xa_lock_irq(&mapping->i_pages); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); + xa_unlock_irq(&mapping->i_pages); + } + return err; +} + +void shmem_uffd_copy_page(struct page *dst, struct page *src) +{ + mm_subpage_shmem_set_missing(dst, src ? mm_subpage_shmem_missing(src) : 0); +} + +void shmem_uffd_migrate(struct folio *dst, struct folio *src) +{ + unsigned long i; + + if (!shmem_mapping(src->mapping)) + return; + VM_BUG_ON_FOLIO(!folio_test_locked(src), src); + VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst); + for (i = 0; i < folio_nr_pages(src); i++) + shmem_uffd_copy_page(folio_page(dst, i), folio_page(src, i)); +} + +/* Only fully removed 4K units become holes; sub-unit edge bytes stay valid. */ +static void shmem_uffd_partial_hole(struct folio *folio, loff_t start, uoff_t end) +{ + loff_t first = max(start, folio_pos(folio)); + u64 last = min_t(u64, end, folio_next_pos(folio) - 1); + unsigned long offset, length; + bool pinned; + + if (first >= last || + (first == folio_pos(folio) && last == folio_next_pos(folio) - 1)) + return; + first = round_up(first, SZ_4K); + last = round_down(last + 1, SZ_4K); + if (first >= last) + return; + shmem_uffd_enable(folio->mapping->host); + /* Revoke covering coarse aliases before their holes can be refilled. */ + unmap_mapping_subpage_hole(folio->mapping, first, last); + /* Pair with fast GUP's pin-before-PTE-recheck barrier. */ + smp_mb(); + pinned = folio_maybe_dma_pinned(folio); + offset = first - folio_pos(folio); + length = last - first; + while (length) { + unsigned int low = offset & ~PAGE_MASK; + unsigned long nr = min(length, PAGE_SIZE - low); + struct page *page = folio_page(folio, offset >> PAGE_SHIFT); + unsigned long mask = shmem_uffd_mask(low, nr); + unsigned long missing = mm_subpage_shmem_missing(page); + + /* + * A pin survives PTE removal and can modify retained backing. + * Preserve native partial-hole semantics: zero it, but do not + * let a missing-page COPY overwrite a subsequent device write. + * Pins have folio granularity, so exposure is conservative when + * only a neighbouring range is pinned. Full-folio removal still + * detaches old pins from the file's new cache identity. + */ + mm_subpage_shmem_set_missing(page, pinned ? missing & ~mask : missing | mask); + offset += nr; + length -= nr; + } +} +#else +static void shmem_uffd_cache_init(struct folio *folio, bool from_swap) { } +static int shmem_uffd_swap_prepare(struct folio *folio) { return 0; } +static void shmem_uffd_partial_hole(struct folio *folio, loff_t start, uoff_t end) { } +#endif + /* * Somewhat like filemap_add_folio, but error if expected item has gone. */ @@ -922,6 +1185,7 @@ int shmem_add_to_page_cache(struct folio *folio, xas_store(&xas, folio); if (xas_error(&xas)) goto unlock; + shmem_uffd_cache_init(folio, expected != NULL); shmem_update_stats(folio, nr); mapping->nrpages += nr; unlock: @@ -976,8 +1240,10 @@ static long shmem_free_swap(struct address_space *mapping, base = round_down(xas.xa_index, nr_pages); if (base < index || base + nr_pages - 1 > end) nr_pages = 0; - else + else { + shmem_uffd_forget(mapping, base, nr_pages); xas_store(&xas, NULL); + } } xas_unlock_irq(&xas); @@ -1020,6 +1286,39 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping, return swapped << PAGE_SHIFT; } +/* Byte-clipped variant for mappings smaller than a native cache page. */ +unsigned long shmem_partial_swap_usage_bytes(struct address_space *mapping, + u64 start, u64 end) +{ + XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT); + void *entry; + unsigned long swapped = 0, last; + + if (start >= end) + return 0; + last = (end - 1) >> PAGE_SHIFT; + rcu_read_lock(); + xas_for_each(&xas, entry, last) { + if (xas_retry(&xas, entry)) + continue; + if (xa_is_value(entry)) { + unsigned int order = xas_get_order(&xas); + u64 first = ((u64)xas.xa_index & ~((1UL << order) - 1)) << PAGE_SHIFT; + u64 stop = first + ((u64)PAGE_SIZE << order); + + swapped += min(stop, end) - max(first, start); + } + if (xas.xa_index == last) + break; + if (need_resched()) { + xas_pause(&xas); + cond_resched_rcu(); + } + } + rcu_read_unlock(); + return swapped; +} + /* * Determine (in bytes) how many of the shmem object's pages mapped by the * given vma is swapped out. @@ -1044,6 +1343,12 @@ unsigned long shmem_swap_usage(struct vm_area_struct *vma) */ if (!swapped) return 0; + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + u64 start = ((u64)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); + + return shmem_partial_swap_usage_bytes(mapping, start, + start + vma->vm_end - vma->vm_start); + } if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) return swapped << PAGE_SHIFT; @@ -1101,6 +1406,12 @@ static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index) return folio; } +static bool shmem_truncate_partial_folio(struct folio *folio, loff_t start, uoff_t end) +{ + shmem_uffd_partial_hole(folio, start, end); + return truncate_inode_partial_folio(folio, start, end); +} + /* * Remove range of pages and swap entries from page cache, and free them. * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. @@ -1164,7 +1475,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, if (folio) { same_folio = lend < folio_next_pos(folio); folio_mark_dirty(folio); - if (!truncate_inode_partial_folio(folio, lstart, lend)) { + if (!shmem_truncate_partial_folio(folio, lstart, lend)) { start = folio_next_index(folio); if (same_folio) end = folio->index; @@ -1178,7 +1489,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT); if (folio) { folio_mark_dirty(folio); - if (!truncate_inode_partial_folio(folio, lstart, lend)) + if (!shmem_truncate_partial_folio(folio, lstart, lend)) end = folio->index; folio_unlock(folio); folio_put(folio); @@ -1247,7 +1558,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, if (!folio_test_large(folio)) { truncate_inode_folio(mapping, folio); - } else if (truncate_inode_partial_folio(folio, lstart, lend)) { + } else if (shmem_truncate_partial_folio(folio, lstart, lend)) { /* * If we split a page, reset the loop so * that we pick up the new sub pages. @@ -1355,6 +1666,7 @@ static int shmem_setattr(struct mnt_idmap *idmap, } if (newsize <= oldsize) { loff_t holebegin = round_up(newsize, PAGE_SIZE); + unmap_mapping_subpage_tail(inode->i_mapping, newsize); if (oldsize > holebegin) unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); @@ -1362,6 +1674,7 @@ static int shmem_setattr(struct mnt_idmap *idmap, shmem_truncate_range(inode, newsize, (loff_t)-1); /* unmap again to remove racily COWed private pages */ + unmap_mapping_subpage_tail(inode->i_mapping, newsize); if (oldsize > holebegin) unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); @@ -1677,6 +1990,8 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, folio_mark_uptodate(folio); } + if (shmem_uffd_swap_prepare(folio)) + goto redirty; if (!folio_alloc_swap(folio)) { bool first_swapped = shmem_recalc_inode(inode, 0, nr_pages); int error; @@ -1730,6 +2045,10 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, swap_cache_del_folio(folio); goto redirty; } + /* Allocation failed: no swap entry owns the reserved inode metadata. */ + xa_lock_irq(&mapping->i_pages); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); + xa_unlock_irq(&mapping->i_pages); if (nr_pages > 1) goto try_split; redirty: @@ -1934,6 +2253,23 @@ static struct folio *shmem_alloc_folio(gfp_t gfp, int order, return folio; } +/* Allocate unexposed promotion backing with the inode's zone/NUMA policy. */ +struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, struct inode *inode, + pgoff_t index, unsigned int order) +{ + gfp_t gfp = limit_gfp_mask(GFP_TRANSHUGE, mapping_gfp_mask(inode->i_mapping)); + struct folio *folio; + + folio = shmem_alloc_folio(gfp, order, SHMEM_I(inode), index); + if (!folio) + return ERR_PTR(-ENOMEM); + if (mem_cgroup_charge(folio, mm, gfp)) { + folio_put(folio); + return ERR_PTR(-ENOMEM); + } + return folio; +} + static struct folio *shmem_alloc_and_add_folio(struct vm_fault *vmf, gfp_t gfp, struct inode *inode, pgoff_t index, struct mm_struct *fault_mm, unsigned long orders) @@ -2484,13 +2820,36 @@ static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index, fault_mm = vma ? vma->vm_mm : NULL; folio = filemap_get_entry(inode->i_mapping, index); - if (folio && vma && userfaultfd_minor(vma)) { + if (folio && vma && userfaultfd_minor(vma) && + !(READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { if (!xa_is_value(folio)) folio_put(folio); *fault_type = handle_userfault(vmf, VM_UFFD_MINOR); return 0; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + if (xa_is_value(folio) && vma && userfaultfd_armed(vma) && + (READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { + unsigned long reason; + void *partial; + + /* Cache identity and its persisted mask must describe the same entry. */ + xa_lock_irq(&inode->i_mapping->i_pages); + if (xa_load(&inode->i_mapping->i_pages, index) != folio) { + xa_unlock_irq(&inode->i_mapping->i_pages); + goto repeat; + } + partial = xa_load(&SHMEM_I(inode)->uffd_missing, index); + reason = shmem_uffd_reason(vmf, partial ? xa_to_value(partial) : 0); + xa_unlock_irq(&inode->i_mapping->i_pages); + if (reason) { + *fault_type = handle_userfault(vmf, reason); + return 0; + } + } +#endif + if (xa_is_value(folio)) { error = shmem_swapin_folio(inode, index, &folio, sgp, gfp, vma, fault_type); @@ -2510,6 +2869,20 @@ static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index, folio_put(folio); goto repeat; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + if (vma && userfaultfd_armed(vma) && + (READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { + unsigned long reason = shmem_uffd_reason(vmf, + shmem_uffd_missing_mask(folio, index)); + + if (reason) { + folio_unlock(folio); + folio_put(folio); + *fault_type = handle_userfault(vmf, reason); + return 0; + } + } +#endif if (sgp == SGP_WRITE) folio_mark_accessed(folio); if (folio_test_uptodate(folio)) @@ -2748,15 +3121,33 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) struct folio *folio = NULL; vm_fault_t ret = 0; int err; +#ifdef CONFIG_MM_SUBPAGE + bool promoted = false; + unsigned long user_size = mm_page_size(vmf->vma->vm_mm); + + /* Missing coarse PTEs require a blocking caller to prepare backing. */ + if (user_size > PAGE_SIZE && (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) + return VM_FAULT_RETRY; +#endif +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) + return VM_FAULT_SIGBUS; +#endif + +#ifdef CONFIG_MM_SUBPAGE +retry: +#endif /* * Trinity finds that probing a hole which tmpfs is punching can * prevent the hole-punch from ever completing: noted in i_private. */ if (unlikely(inode->i_private)) { - ret = shmem_falloc_wait(vmf, inode); - if (ret) - return ret; + vm_fault_t wait = shmem_falloc_wait(vmf, inode); + + if (wait) + return ret | wait; } WARN_ON_ONCE(vmf->page != NULL); @@ -2765,6 +3156,45 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) if (err) return vmf_error(err); if (folio) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) { + folio_unlock(folio); + folio_put(folio); + return VM_FAULT_SIGBUS; + } + if (user_size > PAGE_SIZE) { + struct vm_page_offset pos = vma_page_offset_at(vmf->vma, vmf->address); + unsigned long nr = user_size >> PAGE_SHIFT; + + if (pos.offset || !IS_ALIGNED(pos.index, nr)) { + folio_unlock(folio); + folio_put(folio); + return VM_FAULT_SIGBUS; + } + if (folio_size(folio) < user_size) { + folio_unlock(folio); + folio_put(folio); + folio = NULL; + /* Bound work if splitting races a successful promotion. */ + if (promoted) { + cond_resched(); + return ret | VM_FAULT_NOPAGE; + } + err = collapse_file_user_page(vmf->vma->vm_mm, + vmf->vma->vm_file, pos.index); + if (err == -EAGAIN) { + cond_resched(); + return ret | VM_FAULT_NOPAGE; + } + if (err) + return vmf_error(err); + promoted = true; + goto retry; + } + } +#endif + vmf->page = folio_file_page(folio, vmf->pgoff); ret |= VM_FAULT_LOCKED; } @@ -2899,7 +3329,7 @@ static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, * by page order, as in shmem_get_pgoff_policy() and get_vma_policy()). */ *ilx = inode->i_ino; - index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + index = vma_page_offset_at(vma, addr).index; return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); } @@ -3086,6 +3516,9 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap, memset(info, 0, (char *)inode - (char *)info); INIT_LIST_HEAD_RCU(&info->xattrs); spin_lock_init(&info->lock); +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + xa_init_flags(&info->uffd_missing, XA_FLAGS_ACCOUNT); +#endif atomic_set(&info->stop_eviction, 0); info->seals = F_SEAL_SEAL; info->flags = vma_flags_test(&flags, VMA_NORESERVE_BIT) @@ -3245,6 +3678,57 @@ static void shmem_mfill_filemap_remove(struct folio *folio, folio_unlock(folio); } +#ifdef CONFIG_MM_SUBPAGE +struct folio *shmem_uffd_prepare_folio(struct vm_area_struct *vma, + unsigned long address, bool *new) +{ + struct inode *inode = file_inode(vma->vm_file); + pgoff_t index = vma_page_offset_at(vma, address).index; + struct folio *folio; + int err; + + *new = false; + /* Must precede cache insertion: minor faults must wait for initialization. */ + shmem_uffd_enable(inode); +retry: + err = shmem_get_folio(inode, index, 0, &folio, SGP_NOALLOC); + if (!err) + return folio; + if (err != -ENOENT) + return ERR_PTR(err); + folio = shmem_mfill_folio_alloc(vma, address); + if (!folio) + return ERR_PTR(-ENOMEM); + clear_highpage(&folio->page); + flush_dcache_folio(folio); + __folio_mark_uptodate(folio); + err = shmem_mfill_filemap_add(folio, vma, address); + if (err) { + folio_put(folio); + if (err == -EEXIST) + goto retry; + return ERR_PTR(err); + } + err = shmem_uffd_set_missing(folio, index, SHMEM_UFFD_ALL_UNITS); + if (err) { + shmem_mfill_filemap_remove(folio, vma); + folio_put(folio); + return ERR_PTR(err); + } + *new = true; + return folio; +} + +void shmem_uffd_abort_folio(struct folio *folio, struct vm_area_struct *vma, bool new) +{ + if (new) + shmem_mfill_filemap_remove(folio, vma); + else + folio_unlock(folio); + folio_put(folio); +} +#endif + static struct folio *shmem_get_folio_noalloc(struct inode *inode, pgoff_t pgoff) { struct folio *folio; @@ -3331,6 +3815,7 @@ shmem_write_end(const struct kiocb *iocb, struct address_space *mapping, } folio_mark_uptodate(folio); } + shmem_uffd_mark_mapped(folio, offset_in_folio(folio, pos), copied); folio_mark_dirty(folio); folio_unlock(folio); folio_put(folio); @@ -5143,6 +5628,10 @@ static struct inode *shmem_alloc_inode(struct super_block *sb) info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL); if (!info) return NULL; +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + /* Also cover inode-init failure before __shmem_get_inode() runs. */ + xa_init_flags(&info->uffd_missing, XA_FLAGS_ACCOUNT); +#endif return &info->vfs_inode; } @@ -5155,6 +5644,9 @@ static void shmem_free_in_core_inode(struct inode *inode) static void shmem_destroy_inode(struct inode *inode) { +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + xa_destroy(&SHMEM_I(inode)->uffd_missing); +#endif if (S_ISREG(inode->i_mode)) mpol_free_shared_policy(&SHMEM_I(inode)->policy); if (S_ISDIR(inode->i_mode)) @@ -5972,6 +6464,8 @@ struct folio *shmem_read_folio_gfp(struct address_space *mapping, if (error) return ERR_PTR(error); + /* A kernel caller receives and may modify the complete native folio. */ + shmem_uffd_mark_mapped(folio, 0, folio_size(folio)); folio_unlock(folio); return folio; #else diff --git a/mm/swap.h b/mm/swap.h index a77016f2423b9..5d064cab36f4c 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -10,9 +10,12 @@ extern int page_cluster; #ifdef CONFIG_THP_SWAP #define SWAPFILE_CLUSTER HPAGE_PMD_NR -#define swap_entry_order(order) (order) #else #define SWAPFILE_CLUSTER 256 +#endif +#if defined(CONFIG_THP_SWAP) || defined(CONFIG_MM_SUBPAGE) +#define swap_entry_order(order) (order) +#else #define swap_entry_order(order) 0 #endif @@ -38,6 +41,9 @@ struct swap_cluster_info { u8 order; atomic_long_t __rcu *table; /* Swap table entries, see mm/swap_table.h */ unsigned int *extend_table; /* For large swap count, protected by ci->lock */ +#ifdef CONFIG_MM_SUBPAGE + unsigned int *subpage_counts; /* Per-quarter PTE refs, protected by ci->lock */ +#endif struct list_head list; }; @@ -207,6 +213,8 @@ extern int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp); */ int folio_alloc_swap(struct folio *folio); int folio_dup_swap(struct folio *folio, struct page *subpage); +int folio_dup_swap_range(struct folio *folio, struct page *page, unsigned int nr); +void folio_put_swap_range(struct folio *folio, struct page *page, unsigned int nr); void folio_put_swap(struct folio *folio, struct page *subpage); /* For internal use */ @@ -216,6 +224,7 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si, /* linux/mm/page_io.c */ int sio_pool_init(void); +int swap_extent_io_init(void); struct swap_iocb; void swap_read_folio(struct folio *folio, struct swap_iocb **plug); void __swap_read_unplug(struct swap_iocb *plug); @@ -399,6 +408,17 @@ static inline void folio_put_swap(struct folio *folio, struct page *page) { } +static inline int folio_dup_swap_range(struct folio *folio, struct page *page, + unsigned int nr) +{ + return -EINVAL; +} + +static inline void folio_put_swap_range(struct folio *folio, struct page *page, + unsigned int nr) +{ +} + static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug) { } diff --git a/mm/swap_state.c b/mm/swap_state.c index 1415a5c54a43b..db1285bc1880d 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -427,6 +427,10 @@ void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, if (unlikely(folio_test_large(folio))) return; + /* Small-granule PTEs currently use native swap-cluster readahead. */ + if (vma && mm_page_size(vma->vm_mm) != PAGE_SIZE) + vma = NULL; + readahead = folio_test_clear_readahead(folio); if (vma && vma_ra) { unsigned long ra_val; @@ -896,7 +900,7 @@ struct folio *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, struct folio *folio; mpol = get_vma_policy(vmf->vma, vmf->address, 0, &ilx); - folio = swap_use_vma_readahead() ? + folio = swap_use_vma_readahead() && mm_page_size(vmf->vma->vm_mm) == PAGE_SIZE ? swap_vma_readahead(entry, gfp_mask, mpol, ilx, vmf) : swap_cluster_readahead(entry, gfp_mask, mpol, ilx); mpol_cond_put(mpol); diff --git a/mm/swapfile.c b/mm/swapfile.c index 74a1e324449dc..e5faba70227c0 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -329,17 +330,22 @@ offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset) BUG(); } +/* The extent tree is immutable while the swap entry is owned. */ +sector_t swap_extent_sector(swp_entry_t entry, unsigned long *nr_pages) +{ + struct swap_info_struct *sis = __swap_entry_to_info(entry); + pgoff_t offset = swp_offset(entry); + struct swap_extent *se = offset_to_swap_extent(sis, offset); + + *nr_pages = min(*nr_pages, se->start_page + se->nr_pages - offset); + return (se->start_block + offset - se->start_page) << (PAGE_SHIFT - 9); +} + sector_t swap_folio_sector(struct folio *folio) { - struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); - struct swap_extent *se; - sector_t sector; - pgoff_t offset; + unsigned long nr = folio_nr_pages(folio); - offset = swp_offset(folio->swap); - se = offset_to_swap_extent(sis, offset); - sector = se->start_block + (offset - se->start_page); - return sector << (PAGE_SHIFT - 9); + return swap_extent_sector(folio->swap, &nr); } /* @@ -479,6 +485,16 @@ static void swap_cluster_free_table(struct swap_cluster_info *ci) lockdep_assert_held(&ci->lock); table = (void *)rcu_dereference_protected(ci->table, true); rcu_assign_pointer(ci->table, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (ci->subpage_counts) { + unsigned int i; + + for (i = 0; i < SWAPFILE_CLUSTER * (PAGE_SIZE / MM_SUBPAGE_SIZE); i++) + VM_WARN_ON_ONCE(ci->subpage_counts[i]); + kfree(ci->subpage_counts); + ci->subpage_counts = NULL; + } +#endif swap_table_free(table); } @@ -1067,6 +1083,19 @@ static void swap_reclaim_work(struct work_struct *work) swap_reclaim_full_clusters(si, true); } +static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio *folio) +{ + unsigned int flags; + + if (!folio || !folio_order(folio)) + return true; + flags = READ_ONCE(si->flags); + if (flags & SWP_BLKDEV) + return true; + /* Filesystem swap_rw implementations retain their existing contract. */ + return si->bdev && !(flags & SWP_FS_OPS) && folio_anon_min_order(folio); +} + /* * Try to allocate swap entries with specified order and try set a new * cluster for current CPU too. @@ -1078,11 +1107,7 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, unsigned int order = likely(folio) ? folio_order(folio) : 0; unsigned int offset = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID; - /* - * Swapfile is not block device so unable - * to allocate large entries. - */ - if (order && !(si->flags & SWP_BLKDEV)) + if (!swap_folio_alloc_allowed(si, folio)) return 0; if (!(si->flags & SWP_SOLIDSTATE)) { @@ -1361,6 +1386,10 @@ static bool swap_alloc_fast(struct folio *folio) offset = this_cpu_read(percpu_swap_cluster.offset[order]); if (!si || !offset || !get_swap_device_info(si)) return false; + if (!swap_folio_alloc_allowed(si, folio)) { + put_swap_device(si); + return false; + } ci = swap_cluster_lock(si, offset); if (cluster_is_usable(ci, order)) { @@ -1543,7 +1572,7 @@ static void __swap_cluster_put_entry(struct swap_cluster_info *ci, */ static void swap_put_entries_cluster(struct swap_info_struct *si, pgoff_t offset, int nr, - bool reclaim_cache) + bool reclaim_cache, int subpage) { struct swap_cluster_info *ci; unsigned int ci_off, ci_end; @@ -1556,6 +1585,17 @@ static void swap_put_entries_cluster(struct swap_info_struct *si, ci = swap_cluster_lock(si, offset); ci_off = offset % SWAPFILE_CLUSTER; ci_end = ci_off + nr; +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) { + unsigned int index = ci_off * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage; + + if (WARN_ON_ONCE(nr != 1 || !ci->subpage_counts || !ci->subpage_counts[index])) { + swap_cluster_unlock(ci); + return; + } + ci->subpage_counts[index]--; + } +#endif do { swp_tb = __swap_table_get(ci, ci_off); if (swp_tb_get_count(swp_tb) == 1) { @@ -1649,7 +1689,7 @@ static int __swap_cluster_dup_entry(struct swap_cluster_info *ci, * and failed to allocate an extended table, -EINVAL if any entry is bad entry. */ static int swap_dup_entries_cluster(struct swap_info_struct *si, - pgoff_t offset, int nr) + pgoff_t offset, int nr, int subpage) { int err; struct swap_cluster_info *ci; @@ -1659,6 +1699,25 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, ci_end = ci_start + nr; ci_off = ci_start; ci = swap_cluster_lock(si, offset); +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) { + unsigned int index = ci_start * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage; + + VM_BUG_ON(nr != 1); + if (!ci->subpage_counts) + ci->subpage_counts = kcalloc(SWAPFILE_CLUSTER, + (PAGE_SIZE / MM_SUBPAGE_SIZE) * sizeof(*ci->subpage_counts), + GFP_ATOMIC | __GFP_NOWARN); + if (!ci->subpage_counts) { + err = -ENOMEM; + goto failed; + } + if (ci->subpage_counts[index] == UINT_MAX) { + err = -EOVERFLOW; + goto failed; + } + } +#endif restart: do { err = __swap_cluster_dup_entry(ci, ci_off); @@ -1673,6 +1732,10 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, goto failed; } } while (++ci_off < ci_end); +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) + ci->subpage_counts[ci_start * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage]++; +#endif swap_cluster_unlock(ci); return 0; failed: @@ -1683,6 +1746,132 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, return err; } +#ifdef CONFIG_MM_SUBPAGE +static bool swap_subpage_offset_valid(unsigned int offset) +{ + return offset < PAGE_SIZE && IS_ALIGNED(offset, MM_SUBPAGE_SIZE); +} + +int swap_subpage_dup(swp_entry_t entry, unsigned int offset) +{ + struct swap_info_struct *si = swap_entry_to_info(entry); + + if (WARN_ON_ONCE(!si || !swap_subpage_offset_valid(offset))) + return -EINVAL; + /* A locked swap-cache folio also permits the initial zero-to-one ref. */ + return swap_dup_entries_cluster(si, swp_offset(entry), 1, offset >> MM_SUBPAGE_SHIFT); +} + +void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache) +{ + struct swap_info_struct *si; + + if (WARN_ON_ONCE(!swap_subpage_offset_valid(offset))) + return; + si = get_swap_device(entry); + if (WARN_ON_ONCE(!si)) + return; + swap_put_entries_cluster(si, swp_offset(entry), 1, reclaim_cache, + offset >> MM_SUBPAGE_SHIFT); + put_swap_device(si); +} + +unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset) +{ + struct swap_cluster_info *ci; + struct swap_info_struct *si; + unsigned int count = 0; + + if (!swap_subpage_offset_valid(offset)) + return 0; + si = get_swap_device(entry); + if (!si) + return 0; + ci = swap_cluster_lock(si, swp_offset(entry)); + if (ci->subpage_counts) + count = ci->subpage_counts[swp_cluster_offset(entry) * + (PAGE_SIZE / MM_SUBPAGE_SIZE) + (offset >> MM_SUBPAGE_SHIFT)]; + swap_cluster_unlock(ci); + put_swap_device(si); + return count; +} +#endif + +/* + * Optional THPs can split if their order-specific cluster lists have no room. + * A mandatory base user page cannot: find a free aligned group without keeping + * the per-CPU or device allocation lock held over a device-wide scan. + */ +static void swap_alloc_required_from_device(struct swap_info_struct *si, + struct folio *folio) +{ + unsigned int order = folio_order(folio), nr = folio_nr_pages(folio); + unsigned long start; + + if (!swap_folio_alloc_allowed(si, folio)) + return; + for (start = 0; start < si->max; start += SWAPFILE_CLUSTER) { + struct swap_cluster_info *ci = __swap_offset_to_cluster(si, start); + unsigned long offset, end = min(start + SWAPFILE_CLUSTER, si->max); + bool found = false; + + if (!(READ_ONCE(si->flags) & SWP_WRITEOK)) + break; + if (!spin_trylock(&ci->lock)) + goto next; + /* Order zero's usability test permits occupied clusters of any order. */ + if (!cluster_is_usable(ci, 0) || ci->count + nr > SWAPFILE_CLUSTER) + goto unlock; + for (offset = start; offset + nr <= end; offset += nr) { + bool need_reclaim = false; + + /* Only genuinely free slots qualify; retain every existing owner. */ + if (!cluster_scan_range(si, ci, offset, nr, &need_reclaim) || need_reclaim) + continue; + if (!__swap_cluster_alloc_entries(si, ci, folio, offset - start)) + break; + ci->order = order; + if (ci->count < SWAPFILE_CLUSTER && ci->flags == CLUSTER_FLAG_FRAG) { + /* Same flag, different order: relocate the list explicitly. */ + spin_lock(&si->lock); + list_move_tail(&ci->list, &si->frag_clusters[order]); + spin_unlock(&si->lock); + } else { + relocate_cluster(si, ci); + } + found = true; + break; + } +unlock: + swap_cluster_unlock(ci); + if (found) + return; +next: + cond_resched(); + } +} + +static void swap_alloc_required(struct folio *folio) +{ + struct swap_info_struct *devices[MAX_SWAPFILES], *si; + unsigned int count = 0, i; + + if (get_nr_swap_pages() < folio_nr_pages(folio)) + return; + /* Pin a bounded snapshot in the allocator's existing priority order. */ + spin_lock(&swap_avail_lock); + plist_for_each_entry(si, &swap_avail_head, avail_list) { + if (get_swap_device_info(si)) + devices[count++] = si; + } + spin_unlock(&swap_avail_lock); + for (i = 0; i < count; i++) { + if (!folio_test_swapcache(folio)) + swap_alloc_required_from_device(devices[i], folio); + put_swap_device(devices[i]); + } +} + /** * folio_alloc_swap - allocate swap space for a folio * @folio: folio we want to move to swap @@ -1697,16 +1886,17 @@ int folio_alloc_swap(struct folio *folio) { unsigned int order = folio_order(folio); unsigned int size = 1 << order; + bool mandatory = order && folio_anon_min_order(folio); VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); VM_BUG_ON_FOLIO(!folio_test_uptodate(folio), folio); if (order) { /* - * Reject large allocation when THP_SWAP is disabled, - * the caller should split the folio and try again. + * Optional THP backing may split when THP_SWAP is disabled. + * A larger base user page must retain contiguous swap ownership. */ - if (!IS_ENABLED(CONFIG_THP_SWAP)) + if (!IS_ENABLED(CONFIG_THP_SWAP) && !mandatory) return -EAGAIN; /* @@ -1725,6 +1915,9 @@ int folio_alloc_swap(struct folio *folio) swap_alloc_slow(folio); local_unlock(&percpu_swap_cluster.lock); + if (mandatory && unlikely(!folio_test_swapcache(folio))) + swap_alloc_required(folio); + if (!order && unlikely(!folio_test_swapcache(folio))) { if (swap_sync_discard()) goto again; @@ -1757,6 +1950,24 @@ int folio_alloc_swap(struct folio *folio) * swap_put_entries_direct on its swap entry before this helper returns, or * the swap count may underflow. */ +int folio_dup_swap_range(struct folio *folio, struct page *page, unsigned int nr) +{ + swp_entry_t entry = page_swap_entry(page); + + VM_BUG_ON_FOLIO(!folio_test_locked(folio) || !folio_test_swapcache(folio), folio); + VM_BUG_ON_FOLIO(!nr || folio_page_idx(folio, page) + nr > folio_nr_pages(folio), folio); + return swap_dup_entries_cluster(swap_entry_to_info(entry), swp_offset(entry), nr, -1); +} + +void folio_put_swap_range(struct folio *folio, struct page *page, unsigned int nr) +{ + swp_entry_t entry = page_swap_entry(page); + + VM_BUG_ON_FOLIO(!folio_test_locked(folio) || !folio_test_swapcache(folio), folio); + VM_BUG_ON_FOLIO(!nr || folio_page_idx(folio, page) + nr > folio_nr_pages(folio), folio); + swap_put_entries_cluster(__swap_entry_to_info(entry), swp_offset(entry), nr, false, -1); +} + int folio_dup_swap(struct folio *folio, struct page *subpage) { swp_entry_t entry = folio->swap; @@ -1771,7 +1982,7 @@ int folio_dup_swap(struct folio *folio, struct page *subpage) } return swap_dup_entries_cluster(swap_entry_to_info(entry), - swp_offset(entry), nr_pages); + swp_offset(entry), nr_pages, -1); } /** @@ -1797,7 +2008,7 @@ void folio_put_swap(struct folio *folio, struct page *subpage) nr_pages = 1; } - swap_put_entries_cluster(si, swp_offset(entry), nr_pages, false); + swap_put_entries_cluster(si, swp_offset(entry), nr_pages, false, -1); } /* @@ -2070,7 +2281,7 @@ void swap_put_entries_direct(swp_entry_t entry, int nr) offset = start_offset; do { cluster_end = min(round_up(offset + 1, SWAPFILE_CLUSTER), end_offset); - swap_put_entries_cluster(si, offset, cluster_end - offset, true); + swap_put_entries_cluster(si, offset, cluster_end - offset, true, -1); offset = cluster_end; } while (offset < end_offset); out: @@ -2239,14 +2450,80 @@ static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte) return pte_same(pte_swp_clear_flags(pte), swp_pte); } +#ifdef CONFIG_MM_SUBPAGE +/* Caller holds the swap-cache folio lock and the VMA's mmap read lock. */ +static int unuse_subpage(struct vm_area_struct *vma, pmd_t *pmd, + unsigned long addr, swp_entry_t entry, struct folio *folio, + unsigned int offset) +{ + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slot = NULL; + spinlock_t *ptl; + pte_t *ptep, old, pte; + bool poisoned = PageHWPoison(&folio->page) || !folio_test_uptodate(folio); + int ret = 0; + + if (folio_order(folio) || folio_test_ksm(folio)) + return -EOPNOTSUPP; + if (!poisoned) { + arch_swap_restore(folio_swap(entry, folio), folio); + /* Allocate metadata before taking the PTE spinlock. */ + slot = mm_subpage_restore_granule(folio, offset, mm_page_shift(mm), GFP_KERNEL); + if (IS_ERR(slot)) + return PTR_ERR(slot) == -EAGAIN ? 0 : PTR_ERR(slot); + } + ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!ptep) + goto out; + old = ptep_get(ptep); + if (softleaf_from_pte(old).val != entry.val) + goto unlock; + /* A moved swap PTE retains its backing offset, not its VMA offset. */ + if (pte_swp_subpage_offset(old) != offset) + goto unlock; + if (poisoned) { + pte = swp_entry_to_pte(PageHWPoison(&folio->page) ? + make_hwpoison_entry(&folio->page) : make_poisoned_swp_entry()); + } else { + ret = mm_subpage_add_anon_rmap(slot, vma, addr); + if (ret) + goto unlock; + /* unuse_pte_range() waited for writeback and holds the folio lock. */ + VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio); + if (pte_swp_exclusive(old)) + mm_subpage_set_exclusive(slot); + else if (swap_subpage_count(entry, offset) == 1) + mm_subpage_try_reuse_swap(slot); + pte = pte_mkold(phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot)); + if (pte_swp_soft_dirty(old)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(old)) + pte = pte_mkuffd_wp(pte); + inc_mm_counter(mm, MM_ANONPAGES); + slot = NULL; + ret = 1; + } + dec_mm_counter(mm, MM_SWAPENTS); + set_pte_at(mm, addr, ptep, pte); + swap_subpage_put(entry, offset, false); +unlock: + pte_unmap_unlock(ptep, ptl); +out: + if (slot) + mm_subpage_put(slot); + return ret; +} +#endif + /* * No need to decide whether this PTE shares the swap entry with others, * just let do_wp_page work it out if a write is requested later - to * force COW, vm_page_prot omits write permission from any private vma. */ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, - unsigned long addr, swp_entry_t entry, struct folio *folio) + unsigned long addr, pte_t orig_pte, struct folio *folio) { + swp_entry_t entry = softleaf_from_pte(orig_pte); struct page *page; struct folio *swapcache; spinlock_t *ptl; @@ -2261,6 +2538,12 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, if (!folio_matches_swap_entry(folio, entry)) return 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return unuse_subpage(vma, pmd, addr, entry, folio, + pte_swp_subpage_offset(orig_pte)); +#endif + swapcache = folio; folio = ksm_might_need_to_copy(folio, vma, addr); if (unlikely(!folio)) @@ -2366,7 +2649,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, pte_t ptent; if (!pte++) { - pte = pte_offset_map(pmd, addr); + pte = pte_offset_map_mm(vma->vm_mm, pmd, addr); if (!pte) break; } @@ -2382,6 +2665,22 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, pte_unmap(pte); pte = NULL; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) { + struct vm_fault vmf = { + .vma = vma, .address = addr, .real_address = addr, + .pmd = pmd, .orig_pte = ptent, + }; + vm_fault_t fault = do_swap_coarse_page(&vmf); + + if (fault & VM_FAULT_OOM) + return -ENOMEM; + if (fault & VM_FAULT_ERROR) + return -EIO; + continue; + } +#endif + folio = swap_cache_get_folio(entry); if (!folio) { struct vm_fault vmf = { @@ -2404,7 +2703,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, folio_lock(folio); folio_wait_writeback(folio); - ret = unuse_pte(vma, pmd, addr, entry, folio); + ret = unuse_pte(vma, pmd, addr, ptent, folio); if (ret < 0) { folio_unlock(folio); folio_put(folio); @@ -2414,7 +2713,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, folio_free_swap(folio); folio_unlock(folio); folio_put(folio); - } while (addr += PAGE_SIZE, addr != end); + } while (addr += mm_page_size(vma->vm_mm), addr != end); if (pte) pte_unmap(pte); @@ -2429,10 +2728,10 @@ static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, unsigned long next; int ret; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(vma->vm_mm, pud, addr); do { cond_resched(); - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(vma->vm_mm, addr, end); ret = unuse_pte_range(vma, pmd, addr, next, type); if (ret) return ret; @@ -2448,9 +2747,9 @@ static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, unsigned long next; int ret; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(vma->vm_mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(vma->vm_mm, addr, end); if (pud_none_or_clear_bad(pud)) continue; ret = unuse_pmd_range(vma, pud, addr, next, type); @@ -2468,10 +2767,10 @@ static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, unsigned long next; int ret; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(vma->vm_mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(vma->vm_mm, addr, end); + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4d)) continue; ret = unuse_pud_range(vma, p4d, addr, next, type); if (ret) @@ -2491,7 +2790,7 @@ static int unuse_vma(struct vm_area_struct *vma, unsigned int type) pgd = pgd_offset(vma->vm_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(vma->vm_mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; ret = unuse_p4d_range(vma, pgd, addr, next, type); @@ -2798,10 +3097,15 @@ static int setup_swap_extents(struct swap_info_struct *sis, destroy_swap_extents(sis, swap_file); return -ENOMEM; } - return ret; + } else { + ret = generic_swapfile_activate(sis, swap_file, span); } - - return generic_swapfile_activate(sis, swap_file, span); + if (ret >= 0 && sis->bdev && + !(sis->flags & SWP_FS_OPS) && swap_extent_io_init()) { + destroy_swap_extents(sis, swap_file); + return -ENOMEM; + } + return ret; } static void _enable_swap_info(struct swap_info_struct *si) @@ -3708,7 +4012,7 @@ void si_swapinfo(struct sysinfo *val) * Also the swap entry must have a count >= 1. Otherwise folio_dup_swap should * be used. */ -int swap_dup_entry_direct(swp_entry_t entry) +int swap_dup_entries_direct(swp_entry_t entry, unsigned int nr) { struct swap_info_struct *si; @@ -3725,7 +4029,14 @@ int swap_dup_entry_direct(swp_entry_t entry) */ VM_WARN_ON_ONCE(!swap_entry_swapped(si, entry)); - return swap_dup_entries_cluster(si, swp_offset(entry), 1); + if (WARN_ON_ONCE(!nr || swp_cluster_offset(entry) + nr > SWAPFILE_CLUSTER)) + return -EINVAL; + return swap_dup_entries_cluster(si, swp_offset(entry), nr, -1); +} + +int swap_dup_entry_direct(swp_entry_t entry) +{ + return swap_dup_entries_direct(entry, 1); } #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) diff --git a/mm/truncate.c b/mm/truncate.c index 12cc89f89afcf..ca1830b647a97 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -794,8 +794,10 @@ void truncate_pagecache(struct inode *inode, loff_t newsize) * truncate_inode_pages finishes, hence the second * unmap_mapping_range call must be made for correctness. */ + unmap_mapping_subpage_tail(mapping, newsize); unmap_mapping_range(mapping, holebegin, 0, 1); truncate_inode_pages(mapping, newsize); + unmap_mapping_subpage_tail(mapping, newsize); unmap_mapping_range(mapping, holebegin, 0, 1); } EXPORT_SYMBOL(truncate_pagecache); diff --git a/mm/user-subpage-internal.h b/mm/user-subpage-internal.h new file mode 100644 index 0000000000000..bddb5eeb2ca7b --- /dev/null +++ b/mm/user-subpage-internal.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _MM_USER_SUBPAGE_INTERNAL_H +#define _MM_USER_SUBPAGE_INTERNAL_H + +#include +#include +#include +#include + +struct mm_subpage_owner; +struct rmap_walk_control; +struct anon_vma; + +struct mm_subpage { + struct mm_subpage_owner *owner; + refcount_t refs; + atomic_t mapcount; + unsigned int index; + u8 shift; + unsigned int pins; + bool exclusive; + bool lazyfree; + bool rmap_bound; +}; + +/* Logical identity is independent of the physical slot after relocation. */ +struct mm_subpage_rmap { + struct anon_vma *root; + struct vm_page_offset pos; + unsigned int offset; +}; + +struct subpage_retired_root; +struct mm_subpage_move { + struct mm_subpage *slot; + struct mm_subpage_rmap map; + struct subpage_retired_root *retired; + bool swap; + bool root_changed; + bool committed; +}; + +/* Folio and both PTE locks stay held across prepare, commit and finish. */ +int mm_subpage_move_prepare(struct mm_subpage_move *move, struct mm_subpage *slot, + struct vm_area_struct *src, unsigned long src_address, + struct vm_area_struct *dst, unsigned long dst_address, bool swap); +/* Present source PTE must be cleared and its TLB invalidated before commit. */ +int mm_subpage_move_commit(struct mm_subpage_move *move); +void mm_subpage_move_finish(struct mm_subpage_move *move); + +int mm_subpage_bind_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address); +bool mm_subpage_match_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address); +/* Return -1 for ordinary linear rmap; otherwise return referenced identities. */ +int mm_subpage_snapshot_rmap(struct folio *folio, struct mm_subpage_rmap *maps); +bool mm_subpage_address_in_vma(const struct folio *folio, + const struct vm_area_struct *vma, unsigned long *address); +bool rmap_walk_subpages(struct folio *folio, struct rmap_walk_control *rwc, + bool locked); + +struct anon_vma *mm_subpage_anon_root(const struct mm_subpage *subpage); + +/* Folio lock held; called once before publishing anonymous folio mapping. */ +void mm_subpage_hold_anon_root(struct mm_subpage *subpage, struct anon_vma *root, + bool droppable); +bool mm_subpage_is_droppable(const struct mm_subpage *subpage); + +#endif diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 01b151dca3085..8d72d991b05e5 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -16,10 +16,19 @@ #include #include #include +#include +#include #include #include #include "internal.h" #include "swap.h" +#ifdef CONFIG_MM_SUBPAGE +#include "user-subpage-internal.h" +#endif + +struct mfill_shared_data { + struct folio *source; /* NULL requests a zero-fill. */ +}; struct mfill_state { struct userfaultfd_ctx *ctx; @@ -32,6 +41,10 @@ struct mfill_state { unsigned long src_addr; unsigned long dst_addr; pmd_t *pmd; + struct folio *copy_buffer; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *slots[MM_SUBPAGES_PER_PAGE]; +#endif }; static bool anon_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags) @@ -194,8 +207,24 @@ static void uffd_mfill_unlock(struct vm_area_struct *vma) } #endif +/* Reservations keep the uninstalled leaves private to this operation. */ +static void mfill_free_subpages(struct mfill_state *state) +{ +#ifdef CONFIG_MM_SUBPAGE + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->slots); i++) { + if (state->slots[i]) { + mm_subpage_put(state->slots[i]); + state->slots[i] = NULL; + } + } +#endif +} + static void mfill_put_vma(struct mfill_state *state) { + mfill_free_subpages(state); if (!state->vma) return; @@ -248,6 +277,7 @@ static int mfill_get_vma(struct mfill_state *state) if ((flags & MFILL_ATOMIC_WP) && !(dst_vma->vm_flags & VM_UFFD_WP)) goto out_unlock; + if (is_vm_hugetlb_page(dst_vma)) return 0; @@ -320,16 +350,11 @@ static int mfill_establish_pmd(struct mfill_state *state) static bool mfill_file_over_size(struct vm_area_struct *dst_vma, unsigned long dst_addr) { - struct inode *inode; - pgoff_t offset, max_off; - if (!dst_vma->vm_file) return false; - inode = dst_vma->vm_file->f_inode; - offset = linear_page_index(dst_vma, dst_addr); - max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - return offset >= max_off; + return !vma_file_offset_valid(dst_vma, dst_addr, + i_size_read(file_inode(dst_vma->vm_file))); } /* @@ -341,7 +366,8 @@ static bool mfill_file_over_size(struct vm_area_struct *dst_vma, static int mfill_atomic_install_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, struct page *page, - uffd_flags_t flags) + uffd_flags_t flags, + const struct mfill_shared_data *shared_data) { int ret; struct mm_struct *dst_mm = dst_vma->vm_mm; @@ -354,6 +380,11 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, pte_t dst_ptep; _dst_pte = mk_pte(page, dst_vma->vm_page_prot); +#ifdef CONFIG_MM_SUBPAGE + if (page_in_cache && mm_page_size(dst_mm) < PAGE_SIZE) + _dst_pte = phys_pte_mm(dst_mm, page_to_phys(page) + + vma_page_offset_at(dst_vma, dst_addr).offset, dst_vma->vm_page_prot); +#endif _dst_pte = pte_mkdirty(_dst_pte); if (page_in_cache && !vm_shared) writable = false; @@ -384,7 +415,39 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, if (!pte_none(dst_ptep) && !pte_is_uffd_marker(dst_ptep)) goto out_unlock; +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + if (shared_data) { + struct vm_page_offset pos = vma_page_offset_at(dst_vma, dst_addr); + unsigned long offset = ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset; + unsigned long size = mm_page_size(dst_mm); + void *dst, *src; + + if (PageHWPoison(page)) { + ret = -EIO; + goto out_unlock; + } + if (!shmem_uffd_range_missing(folio, pos.index, pos.offset, size)) + goto out_unlock; + dst = kmap_local_folio(folio, offset); + if (shared_data->source) { + src = kmap_local_folio(shared_data->source, 0); + memcpy(dst, src, size); + kunmap_local(src); + } else { + memset(dst, 0, size); + } + kunmap_local(dst); + flush_dcache_folio(folio); + __folio_mark_uptodate(folio); + folio_mark_dirty(folio); + } +#endif + if (page_in_cache) { + mm_subpage_file_map_add(dst_vma, pte_phys_mm(dst_mm, _dst_pte)); + shmem_uffd_mark_mapped(folio, + (folio_page_idx(folio, page) << PAGE_SHIFT) + + vma_page_offset_at(dst_vma, dst_addr).offset, mm_page_size(dst_mm)); folio_add_file_rmap_pte(folio, page, dst_vma); } else { folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE); @@ -411,12 +474,13 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, return ret; } -static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) +static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr, + unsigned int offset, unsigned long len) { void *kaddr; int ret; - kaddr = kmap_local_folio(folio, 0); + kaddr = kmap_local_folio(folio, offset); /* * The read mmap_lock is held here. Despite the * mmap_lock being read recursive a deadlock is still @@ -433,8 +497,7 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) * and retry the copy outside the mmap_lock. */ pagefault_disable(); - ret = copy_from_user(kaddr, (const void __user *) src_addr, - PAGE_SIZE); + ret = copy_from_user(kaddr, (const void __user *)src_addr, len); pagefault_enable(); kunmap_local(kaddr); @@ -507,8 +570,11 @@ DEFINE_FREE(retry_put, struct mfill_retry_state *, if (_T) mfill_retry_state_put(_T)); static int mfill_copy_folio_retry(struct mfill_state *mfill_state, - struct folio *folio) + struct folio *folio, unsigned int offset, + unsigned long len) { + struct vm_page_offset pos = vma_page_offset_at(mfill_state->vma, + mfill_state->dst_addr); struct mfill_retry_state retry_state = { 0 }; struct mfill_retry_state *for_free __free(retry_put) = &retry_state; unsigned long src_addr = mfill_state->src_addr; @@ -520,8 +586,8 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, /* retry copying with mm_lock dropped */ mfill_put_vma(mfill_state); - kaddr = kmap_local_folio(folio, 0); - err = copy_from_user(kaddr, (const void __user *) src_addr, PAGE_SIZE); + kaddr = kmap_local_folio(folio, offset); + err = copy_from_user(kaddr, (const void __user *) src_addr, len); kunmap_local(kaddr); if (unlikely(err)) return -EFAULT; @@ -533,7 +599,10 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, if (err) return err; - if (mfill_retry_state_changed(&retry_state, mfill_state->vma)) + if (mfill_retry_state_changed(&retry_state, mfill_state->vma) || + (mm_page_size(mfill_state->ctx->mm) < PAGE_SIZE && + !vm_page_offset_equal(pos, vma_page_offset_at(mfill_state->vma, + mfill_state->dst_addr)))) return -EAGAIN; err = mfill_establish_pmd(mfill_state); @@ -543,6 +612,28 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, return 0; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) +/* Fill one missing leaf in new or retained shmem backing using staged data. */ +static int mfill_atomic_shmem_hole(struct mfill_state *state, + const struct mfill_shared_data *data) +{ + struct vm_page_offset pos; + struct folio *folio; + bool new; + int err; + + folio = shmem_uffd_prepare_folio(state->vma, state->dst_addr, &new); + if (IS_ERR(folio)) + return PTR_ERR(folio); + pos = vma_page_offset_at(state->vma, state->dst_addr); + err = mfill_atomic_install_pte(state->pmd, state->vma, state->dst_addr, + folio_file_page(folio, pos.index), state->flags, data); + if (err) + shmem_uffd_abort_folio(folio, state->vma, new); + return err; +} +#endif + static int __mfill_atomic_pte(struct mfill_state *state, const struct vm_uffd_ops *ops) { @@ -562,7 +653,7 @@ static int __mfill_atomic_pte(struct mfill_state *state, return -ENOMEM; if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) { - ret = mfill_copy_folio_locked(folio, src_addr); + ret = mfill_copy_folio_locked(folio, src_addr, 0, PAGE_SIZE); /* * Fallback to copy_from_user outside mmap_lock. * If retry is successful, mfill_copy_folio_locked() returns @@ -571,7 +662,7 @@ static int __mfill_atomic_pte(struct mfill_state *state, * will take care of unlocking if needed. */ if (unlikely(ret)) { - ret = mfill_copy_folio_retry(state, folio); + ret = mfill_copy_folio_retry(state, folio, 0, PAGE_SIZE); if (ret) goto err_folio_put; } @@ -590,12 +681,29 @@ static int __mfill_atomic_pte(struct mfill_state *state, if (ops->filemap_add) { ret = ops->filemap_add(folio, state->vma, state->dst_addr); +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + /* + * A pinned large shmem folio can survive a partial hole punch. + * Its cache entry exists while this native-sized range is missing. + * Reuse the prepared source and preserve all neighbouring bytes. + * Ordinary native insertion and non-shmem paths stay unchanged. + */ + if (ret == -EEXIST && (state->vma->vm_flags & VM_SHARED) && + shmem_file(state->vma->vm_file) && + (READ_ONCE(SHMEM_I(file_inode(state->vma->vm_file))->flags) & + SHMEM_F_UFFD_SUBPAGE)) { + struct mfill_shared_data data = { .source = folio }; + + ret = mfill_atomic_shmem_hole(state, &data); + goto err_folio_put; + } +#endif if (ret) goto err_folio_put; } ret = mfill_atomic_install_pte(state->pmd, state->vma, dst_addr, - &folio->page, flags); + &folio->page, flags, NULL); if (ret) goto err_filemap_remove; @@ -609,10 +717,184 @@ static int __mfill_atomic_pte(struct mfill_state *state, return ret; } +#ifdef CONFIG_MM_SUBPAGE +/* Pack only this request's leaves in one native linear index. */ +static struct mm_subpage *mfill_alloc_subpage(struct mfill_state *state) +{ + struct vm_area_struct *vma = state->vma; + struct mm_struct *mm = vma->vm_mm; + struct vm_page_offset pos = vma_page_offset_at(vma, state->dst_addr); + unsigned long size = mm_page_size(mm); + unsigned int first = pos.offset >> mm_page_shift(mm), i, nr; + struct mm_subpage_pool *pool; + struct mm_subpage *slot; + struct folio *folio; + int err = -ENOMEM; + + if (state->slots[first]) + goto take; + + mfill_free_subpages(state); + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return ERR_PTR(-ENOMEM); + folio = anon_alloc_folio(vma, state->dst_addr); + if (!folio) + goto out; + folio_throttle_swaprate(folio, GFP_KERNEL); + err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (err) { + folio_put(folio); + goto out; + } + nr = min_t(unsigned long, PAGE_SIZE - pos.offset, + state->dst_start + state->len - state->dst_addr) / size; + for (i = first; i < first + nr; i++) { + slot = mm_subpage_alloc_at(pool, i * size); + if (IS_ERR(slot)) { + err = PTR_ERR(slot); + mfill_free_subpages(state); + goto out; + } + state->slots[i] = slot; + } +out: + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + if (err) + return ERR_PTR(err); +take: + slot = state->slots[first]; + state->slots[first] = NULL; + return slot; +} + +static int mfill_atomic_subpage(struct mfill_state *state) +{ + struct mm_struct *mm = state->ctx->mm; + struct mm_subpage *slot = mfill_alloc_subpage(state); + struct vm_area_struct *vma; + struct folio *folio; + pte_t entry, *pte; + spinlock_t *ptl; + bool fresh; + int err; + + if (IS_ERR(slot)) + return PTR_ERR(slot); + folio = mm_subpage_folio(slot); + folio_lock(folio); + /* A previous leaf may have entered reclaim between iterations. */ + err = -EAGAIN; + if (folio_test_swapcache(folio)) + goto unlock_folio; + if (uffd_flags_mode_is(state->flags, MFILL_ATOMIC_COPY)) { + err = mfill_copy_folio_locked(folio, state->src_addr, + mm_subpage_offset(slot), mm_page_size(mm)); + if (err) { + /* + * The faulting retry must not hold the folio lock, or write + * a folio whose already-visible neighbours can be reclaimed. + * Use fresh backing and discard the remaining reservations + * before dropping the VMA lock. + */ + folio_unlock(folio); + mm_subpage_put(slot); + mfill_free_subpages(state); + slot = mfill_alloc_subpage(state); + if (IS_ERR(slot)) + return PTR_ERR(slot); + folio = mm_subpage_folio(slot); + err = mfill_copy_folio_retry(state, folio, + mm_subpage_offset(slot), mm_page_size(mm)); + if (err) + goto put_slot; + folio_lock(folio); + } + } + /* The allocator zeroed every byte, including unissued neighbours. */ + __folio_mark_uptodate(folio); + vma = state->vma; + pte = pte_offset_map_lock(mm, state->pmd, state->dst_addr, &ptl); + err = -EAGAIN; + if (!pte) + goto unlock_folio; + err = -EFAULT; + if (mfill_file_over_size(vma, state->dst_addr)) + goto unlock_pte; + err = -EEXIST; + if (!pte_none(ptep_get(pte)) && !pte_is_uffd_marker(ptep_get(pte))) + goto unlock_pte; + fresh = !folio_test_anon(folio); + mm_subpage_set_exclusive(slot); + err = mm_subpage_add_new_anon_rmap(slot, vma, state->dst_addr); + if (err) + goto unlock_pte; + if (fresh) + folio_add_lru_vma(folio, vma); + else if ((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED) + mlock_folio(folio); + inc_mm_counter(mm, MM_ANONPAGES); + entry = phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot); + entry = pte_mkdirty(entry); + if (vma->vm_flags & VM_WRITE) + entry = pte_mkwrite(entry, vma); + if (state->flags & MFILL_ATOMIC_WP) + entry = pte_mkuffd_wp(entry); + set_pte_at(mm, state->dst_addr, pte, entry); + update_mmu_cache(vma, state->dst_addr, pte); + /* The PTE owns slot now; release the folio lock before dropping PTL. */ + folio_unlock(folio); + pte_unmap_unlock(pte, ptl); + return 0; +unlock_pte: + pte_unmap_unlock(pte, ptl); +unlock_folio: + folio_unlock(folio); +put_slot: + mm_subpage_put(slot); + return err; +} +#endif + +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) +static int mfill_atomic_shared_subpage(struct mfill_state *state) +{ + struct mfill_shared_data data = {}; + int err; + + if (uffd_flags_mode_is(state->flags, MFILL_ATOMIC_COPY)) { + if (!state->copy_buffer) + state->copy_buffer = folio_alloc(GFP_KERNEL_ACCOUNT, 0); + data.source = state->copy_buffer; + if (!data.source) + return -ENOMEM; + err = mfill_copy_folio_locked(data.source, state->src_addr, 0, + mm_page_size(state->ctx->mm)); + if (err) + err = mfill_copy_folio_retry(state, data.source, 0, + mm_page_size(state->ctx->mm)); + if (err) + return err; + } + return mfill_atomic_shmem_hole(state, &data); +} +#endif + static int mfill_atomic_pte_copy(struct mfill_state *state) { const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(state->ctx->mm) < PAGE_SIZE) { +#ifdef CONFIG_SHMEM + if (state->vma->vm_flags & VM_SHARED) + return mfill_atomic_shared_subpage(state); +#endif + return mfill_atomic_subpage(state); + } +#endif + /* * The normal page fault path for a MAP_PRIVATE mapping in a * file-backed VMA will invoke the fault, fill the hole in the file and @@ -632,6 +914,16 @@ static int mfill_atomic_pte_zeroed_folio(struct mfill_state *state) { const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(state->ctx->mm) < PAGE_SIZE) { +#ifdef CONFIG_SHMEM + if (state->vma->vm_flags & VM_SHARED) + return mfill_atomic_shared_subpage(state); +#endif + return mfill_atomic_subpage(state); + } +#endif + return __mfill_atomic_pte(state, ops); } @@ -695,6 +987,13 @@ static int mfill_atomic_pte_continue(struct mfill_state *state) if (IS_ERR_OR_NULL(folio)) return -EFAULT; +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + if (vma_is_shmem(dst_vma) && shmem_uffd_range_missing(folio, pgoff, + vma_page_offset_at(dst_vma, dst_addr).offset, mm_page_size(dst_vma->vm_mm))) { + ret = -EFAULT; + goto out_release; + } +#endif page = folio_file_page(folio, pgoff); if (PageHWPoison(page)) { ret = -EIO; @@ -702,7 +1001,7 @@ static int mfill_atomic_pte_continue(struct mfill_state *state) } ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, - page, flags); + page, flags, NULL); if (ret) goto out_release; @@ -964,8 +1263,8 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, /* * Sanitize the command parameters: */ - VM_WARN_ON_ONCE(dst_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(dst_start & ~mm_page_mask(ctx->mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(ctx->mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(src_start + len <= src_start); @@ -998,9 +1297,9 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, cond_resched(); if (!err) { - state.dst_addr += PAGE_SIZE; - state.src_addr += PAGE_SIZE; - copied += PAGE_SIZE; + state.dst_addr += mm_page_size(ctx->mm); + state.src_addr += mm_page_size(ctx->mm); + copied += mm_page_size(ctx->mm); if (fatal_signal_pending(current)) err = -EINTR; @@ -1011,6 +1310,8 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, mfill_put_vma(&state); out: + if (state.copy_buffer) + folio_put(state.copy_buffer); VM_WARN_ON_ONCE(copied < 0); VM_WARN_ON_ONCE(err > 0); VM_WARN_ON_ONCE(!copied && !err); @@ -1099,8 +1400,8 @@ int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, /* * Sanitize the command parameters: */ - VM_WARN_ON_ONCE(start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(start & ~mm_page_mask(dst_mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(dst_mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(start + len <= start); @@ -1303,7 +1604,102 @@ static long move_present_ptes(struct mm_struct *mm, return src_addr > src_start ? src_addr - src_start : err; } +#ifdef CONFIG_MM_SUBPAGE +static long move_present_subpage(struct mm_struct *mm, + struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, + unsigned long dst_addr, unsigned long src_addr, + pte_t *dst_pte, pte_t *src_pte, pte_t orig_dst, pte_t orig_src, + pmd_t *dst_pmd, pmd_t dst_pmdval, + spinlock_t *dst_ptl, spinlock_t *src_ptl, struct folio *folio) +{ + struct mm_subpage_move move; + struct mm_subpage *slot = NULL; + pte_t pte; + long ret = -EAGAIN; + + flush_cache_range(src_vma, src_addr, src_addr + mm_page_size(mm)); + double_pt_lock(dst_ptl, src_ptl); + if (!is_pte_pages_stable(dst_pte, src_pte, orig_dst, orig_src, + dst_pmd, dst_pmdval)) + goto out; + ret = -EBUSY; + if (!folio_test_anon(folio) || folio_test_large(folio)) + goto out; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, orig_src)); + if (!slot || mm_subpage_folio(slot) != folio) + goto out; + ret = mm_subpage_move_prepare(&move, slot, src_vma, src_addr, + dst_vma, dst_addr, false); + if (ret) + goto out; + orig_src = ptep_clear_flush(src_vma, src_addr, src_pte); + ret = mm_subpage_move_commit(&move); + if (ret) { + set_pte_at(mm, src_addr, src_pte, orig_src); + } else { + pte = phys_pte_mm(mm, mm_subpage_phys(slot), dst_vma->vm_page_prot); + if (pgtable_supports_soft_dirty()) + pte = pte_mksoft_dirty(pte); + if (pte_dirty(orig_src)) + pte = pte_mkdirty(pte); + pte = pte_mkwrite(pte, dst_vma); + set_pte_at(mm, dst_addr, dst_pte, pte); + ret = mm_page_size(mm); + } + mm_subpage_move_finish(&move); +out: + if (slot) + mm_subpage_put(slot); + double_pt_unlock(dst_ptl, src_ptl); + return ret; +} + +/* Both PTLs and, if present, the swap-cache folio lock are held. */ +static int move_swap_subpage(struct mm_struct *mm, struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, unsigned long dst_addr, + unsigned long src_addr, pte_t *dst_pte, pte_t *src_pte, + pte_t orig_src, struct folio *folio, swp_entry_t entry) +{ + unsigned int offset = pte_swp_subpage_offset(orig_src); + struct mm_subpage_move move; + struct mm_subpage *slot = NULL; + int ret = -EBUSY; + + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(mm))) + return -EFAULT; + if (swap_subpage_count(entry, offset) != 1) + return -EBUSY; + if (folio) { + slot = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(folio)) + offset); + if (slot) { + ret = mm_subpage_move_prepare(&move, slot, src_vma, src_addr, + dst_vma, dst_addr, true); + if (ret) + goto out; + ret = mm_subpage_move_commit(&move); + if (ret) { + mm_subpage_move_finish(&move); + goto out; + } + } + } + /* No mapping count changes: the same-mm exclusive swap PTE is transferred. */ + orig_src = ptep_get_and_clear(mm, src_addr, src_pte); + if (pgtable_supports_soft_dirty()) + orig_src = pte_swp_mksoft_dirty(orig_src); + set_pte_at(mm, dst_addr, dst_pte, orig_src); + if (slot) + mm_subpage_move_finish(&move); + ret = mm_page_size(mm); +out: + if (slot) + mm_subpage_put(slot); + return ret; +} +#endif + static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, unsigned long dst_addr, unsigned long src_addr, pte_t *dst_pte, pte_t *src_pte, pte_t orig_dst_pte, pte_t orig_src_pte, @@ -1329,6 +1725,21 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, return -EAGAIN; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + int ret; + + if (!src_folio && swap_cache_has_folio(entry)) { + double_pt_unlock(dst_ptl, src_ptl); + return -EAGAIN; + } + ret = move_swap_subpage(mm, dst_vma, src_vma, dst_addr, src_addr, + dst_pte, src_pte, orig_src_pte, src_folio, entry); + double_pt_unlock(dst_ptl, src_ptl); + return ret; + } +#endif + /* * The src_folio resides in the swapcache, requiring an update to its * index and mapping to align with the dst_vma, where a swap-in may @@ -1381,13 +1792,13 @@ static int move_zeropage_pte(struct mm_struct *mm, return -EAGAIN; } - zero_pte = pte_mkspecial(pfn_pte(zero_pfn(dst_addr), - dst_vma->vm_page_prot)); + zero_pte = pte_mkspecial(phys_pte_mm(mm, PFN_PHYS(zero_pfn(dst_addr)), + dst_vma->vm_page_prot)); ptep_clear_flush(src_vma, src_addr, src_pte); set_pte_at(mm, dst_addr, dst_pte, zero_pte); double_pt_unlock(dst_ptl, src_ptl); - return PAGE_SIZE; + return mm_page_size(mm); } @@ -1474,7 +1885,7 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES)) ret = -ENOENT; else /* nothing to do to move a hole */ - ret = PAGE_SIZE; + ret = mm_page_size(mm); goto out; } @@ -1514,7 +1925,8 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd } folio = vm_normal_folio(src_vma, src_addr, orig_src_pte); - if (!folio || !PageAnonExclusive(&folio->page)) { + if (!folio || (mm_page_size(mm) >= PAGE_SIZE && + !PageAnonExclusive(&folio->page))) { spin_unlock(src_ptl); ret = -EBUSY; goto out; @@ -1554,6 +1966,15 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd } } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + ret = move_present_subpage(mm, dst_vma, src_vma, dst_addr, src_addr, + dst_pte, src_pte, orig_dst_pte, orig_src_pte, + dst_pmd, dst_pmdval, dst_ptl, src_ptl, src_folio); + goto out; + } +#endif + /* at this point we have src_folio locked */ if (folio_test_large(src_folio)) { /* split_folio() can block */ @@ -1592,7 +2013,8 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd goto out; } - if (!pte_swp_exclusive(orig_src_pte)) { + /* Smaller leaves validate per-slot swap and mapping counts under PTL. */ + if (mm_page_size(mm) >= PAGE_SIZE && !pte_swp_exclusive(orig_src_pte)) { ret = -EBUSY; goto out; } @@ -1635,7 +2057,7 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd goto retry; } } - ret = move_swap_pte(mm, dst_vma, dst_addr, src_addr, dst_pte, src_pte, + ret = move_swap_pte(mm, dst_vma, src_vma, dst_addr, src_addr, dst_pte, src_pte, orig_dst_pte, orig_src_pte, dst_pmd, dst_pmdval, dst_ptl, src_ptl, src_folio, si, entry); } @@ -1694,6 +2116,11 @@ static int validate_move_areas(struct userfaultfd_ctx *ctx, pgprot_val(src_vma->vm_page_prot) != pgprot_val(dst_vma->vm_page_prot)) return -EINVAL; + /* Native backing cannot mix droppable and swap-backed slot policies. */ + if (mm_page_size(src_vma->vm_mm) < PAGE_SIZE && + ((src_vma->vm_flags ^ dst_vma->vm_flags) & VM_DROPPABLE)) + return -EINVAL; + /* Only allow moving if both are mlocked or both aren't */ if ((src_vma->vm_flags & VM_LOCKED) != (dst_vma->vm_flags & VM_LOCKED)) return -EINVAL; @@ -1926,9 +2353,9 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, ssize_t moved = 0; /* Sanitize the command parameters. */ - VM_WARN_ON_ONCE(src_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(dst_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(src_start & ~mm_page_mask(mm)); + VM_WARN_ON_ONCE(dst_start & ~mm_page_mask(mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(src_start + len < src_start); @@ -2053,10 +2480,12 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, ret = move_pages_ptes(mm, dst_pmd, src_pmd, dst_vma, src_vma, dst_addr, src_addr, src_end - src_addr, mode); - if (ret < 0) + if (ret < 0) { err = ret; - else + } else { + err = 0; step_size = ret; + } } cond_resched(); @@ -2095,6 +2524,17 @@ bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags, { const struct vm_uffd_ops *ops = vma_uffd_ops(vma); + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + return false; + if (vma_is_anonymous(vma)) { + if (vm_flags & VM_UFFD_MINOR) + return false; + } else if (!vma_is_shmem(vma)) { + return false; + } + } + if (vma->vm_flags & (VM_DROPPABLE | VM_SHADOW_STACK)) return false; diff --git a/mm/util.c b/mm/util.c index 3b26f5bf92012..1f41dfdd3ad13 100644 --- a/mm/util.c +++ b/mm/util.c @@ -344,21 +344,22 @@ unsigned long randomize_stack_top(unsigned long stack_top) if (current->flags & PF_RANDOMIZE) { random_variable = get_random_long(); - random_variable &= STACK_RND_MASK; - random_variable <<= PAGE_SHIFT; + random_variable &= mm_aslr_mask(current->mm, STACK_RND_MASK); + random_variable <<= mm_page_shift(current->mm); } #ifdef CONFIG_STACK_GROWSUP - return PAGE_ALIGN(stack_top) + random_variable; + return mm_page_align(current->mm, stack_top) + random_variable; #else - return PAGE_ALIGN(stack_top) - random_variable; + return mm_page_align(current->mm, stack_top) - random_variable; #endif } /** - * randomize_page - Generate a random, page aligned address + * randomize_page_shift - Generate a random, granule aligned address * @start: The smallest acceptable address the caller will take. * @range: The size of the area, starting at @start, within which the * random address must fall. + * @shift: Log2 of the required address alignment. * * If @start + @range would overflow, @range is capped. * @@ -368,22 +369,31 @@ unsigned long randomize_stack_top(unsigned long stack_top) * Return: A page aligned address within [start, start + range). On error, * @start is returned. */ -unsigned long randomize_page(unsigned long start, unsigned long range) +static unsigned long randomize_page_shift(unsigned long start, + unsigned long range, unsigned int shift) { - if (!PAGE_ALIGNED(start)) { - range -= PAGE_ALIGN(start) - start; - start = PAGE_ALIGN(start); + unsigned long page_size = 1UL << shift; + + if (!IS_ALIGNED(start, page_size)) { + range -= ALIGN(start, page_size) - start; + start = ALIGN(start, page_size); } if (start > ULONG_MAX - range) range = ULONG_MAX - start; - range >>= PAGE_SHIFT; + range >>= shift; if (range == 0) return start; - return start + (get_random_long() % range << PAGE_SHIFT); + return start + (get_random_long() % range << shift); +} + +/* Existing callers retain native-page alignment. */ +unsigned long randomize_page(unsigned long start, unsigned long range) +{ + return randomize_page_shift(start, range, PAGE_SHIFT); } #ifdef CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT @@ -391,23 +401,25 @@ unsigned long __weak arch_randomize_brk(struct mm_struct *mm) { /* Is the current task 32bit ? */ if (!IS_ENABLED(CONFIG_64BIT) || is_compat_task()) - return randomize_page(mm->brk, SZ_32M); + return randomize_page_shift(mm->brk, SZ_32M, mm_page_shift(mm)); - return randomize_page(mm->brk, SZ_1G); + return randomize_page_shift(mm->brk, SZ_1G, mm_page_shift(mm)); } unsigned long arch_mmap_rnd(void) { - unsigned long rnd; + unsigned int bits; + unsigned long mask; #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS if (is_compat_task()) - rnd = get_random_long() & ((1UL << mmap_rnd_compat_bits) - 1); + bits = mmap_rnd_compat_bits; else #endif /* CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS */ - rnd = get_random_long() & ((1UL << mmap_rnd_bits) - 1); + bits = mmap_rnd_bits; - return rnd << PAGE_SHIFT; + mask = mm_aslr_mask(current->mm, (1UL << bits) - 1); + return (get_random_long() & mask) << mm_page_shift(current->mm); } static int mmap_is_legacy(const struct rlimit *rlim_stack) @@ -440,14 +452,15 @@ static unsigned long mmap_base(const unsigned long rnd, const struct rlimit *rli * task. mmap_base starts directly below the stack and grows * downwards. */ - return PAGE_ALIGN_DOWN(mmap_upper_limit(rlim_stack) - rnd); + return (mmap_upper_limit(rlim_stack) - rnd) & mm_page_mask(current->mm); #else unsigned long gap = rlim_stack->rlim_cur; unsigned long pad = stack_guard_gap; /* Account for stack randomization if necessary */ if (current->flags & PF_RANDOMIZE) - pad += (STACK_RND_MASK << PAGE_SHIFT); + pad += mm_aslr_mask(current->mm, STACK_RND_MASK) << + mm_page_shift(current->mm); /* Values close to RLIM_INFINITY can overflow. */ if (gap + pad > gap) @@ -458,7 +471,7 @@ static unsigned long mmap_base(const unsigned long rnd, const struct rlimit *rli else if (gap > MAX_GAP) gap = MAX_GAP; - return PAGE_ALIGN(STACK_TOP - gap - rnd); + return mm_page_align(current->mm, STACK_TOP - gap - rnd); #endif } @@ -491,7 +504,7 @@ EXPORT_SYMBOL_IF_KUNIT(arch_pick_mmap_layout); /** * __account_locked_vm - account locked pages to an mm's locked_vm * @mm: mm to account against - * @pages: number of pages to account + * @pages: number of native physical pages to account * @inc: %true if @pages should be considered positive, %false if not * @task: task used to check RLIMIT_MEMLOCK * @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped @@ -507,27 +520,35 @@ int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, const struct task_struct *task, bool bypass_rlim) { unsigned long locked_vm, limit; + unsigned int shift = PAGE_SHIFT - MM_ACCOUNT_SHIFT; int ret = 0; mmap_assert_write_locked(mm); + /* This exported API receives native physical-page counts. */ + if (pages > ULONG_MAX >> shift) + return -ENOMEM; + pages <<= shift; locked_vm = mm->locked_vm; if (inc) { if (!bypass_rlim) { - limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT; - if (locked_vm + pages > limit) + limit = task_rlimit(task, RLIMIT_MEMLOCK) >> MM_ACCOUNT_SHIFT; + if (pages > limit || locked_vm > limit - pages) ret = -ENOMEM; } + if (!ret && pages > ULONG_MAX - locked_vm) + ret = -ENOMEM; if (!ret) mm->locked_vm = locked_vm + pages; } else { - WARN_ON_ONCE(pages > locked_vm); + if (WARN_ON_ONCE(pages > locked_vm)) + return -EINVAL; mm->locked_vm = locked_vm - pages; } pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid, - (void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT, - locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK), + (void *)_RET_IP_, (inc) ? '+' : '-', pages << MM_ACCOUNT_SHIFT, + locked_vm << MM_ACCOUNT_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK), ret ? " - exceeded" : ""); return ret; @@ -562,11 +583,11 @@ int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc) } EXPORT_SYMBOL_GPL(account_locked_vm); -unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, +static unsigned long vm_mmap_offset_internal(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, - unsigned long flag, unsigned long pgoff) + unsigned long flag, struct vm_page_offset pos) { - loff_t off = (loff_t)pgoff << PAGE_SHIFT; + loff_t off = ((loff_t)pos.index << PAGE_SHIFT) | pos.offset; unsigned long ret; struct mm_struct *mm = current->mm; unsigned long populate; @@ -578,8 +599,11 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, if (!ret) { if (mmap_write_lock_killable(mm)) return -EINTR; - ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate, - &uf); +#ifdef CONFIG_MM_SUBPAGE + ret = do_mmap_offset(file, addr, len, prot, flag, 0, pos, &populate, &uf); +#else + ret = do_mmap(file, addr, len, prot, flag, 0, pos.index, &populate, &uf); +#endif mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate) @@ -588,6 +612,13 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, return ret; } +unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flag, unsigned long pgoff) +{ + return vm_mmap_offset_internal(file, addr, len, prot, flag, + (struct vm_page_offset) { pgoff, 0 }); +} + /* * Perform a userland memory mapping into the current process address space. See * the comment for do_mmap() for more details on this operation in general. @@ -609,12 +640,13 @@ unsigned long vm_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flag, unsigned long offset) { - if (unlikely(offset + PAGE_ALIGN(len) < offset)) + if (unlikely(offset + mm_page_align(current->mm, len) < offset)) return -EINVAL; - if (unlikely(offset_in_page(offset))) + if (unlikely(offset & ~mm_page_mask(current->mm))) return -EINVAL; - return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT); + return vm_mmap_offset_internal(file, addr, len, prot, flag, + (struct vm_page_offset) { offset >> PAGE_SHIFT, offset & ~PAGE_MASK }); } EXPORT_SYMBOL(vm_mmap); @@ -988,7 +1020,8 @@ int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin if (mm) { long reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10); - allowed -= min_t(long, mm->total_vm / 32, reserve); + allowed -= min_t(long, + mm_pages_to_bytes(mm, mm->total_vm) >> (PAGE_SHIFT + 5), reserve); } if (percpu_counter_read_positive(&vm_committed_as) < allowed) @@ -1189,6 +1222,9 @@ void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->end = vma->vm_end; desc->pgoff = vma->vm_pgoff; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = vma->vm_subpage_offset; +#endif desc->vm_file = vma->vm_file; desc->vma_flags = vma->flags; desc->page_prot = vma->vm_page_prot; diff --git a/mm/vma.c b/mm/vma.c index d90791b00a7b8..4161f7739a10e 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -13,7 +13,7 @@ struct mmap_state { unsigned long addr; unsigned long end; - pgoff_t pgoff; + struct vm_page_offset offset; unsigned long pglen; union { vm_flags_t vm_flags; @@ -42,19 +42,25 @@ struct mmap_state { bool file_doesnt_need_get :1; }; -#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, vma_flags_, file_) \ +#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pos_, vma_flags_, file_) \ struct mmap_state name = { \ .mm = mm_, \ .vmi = vmi_, \ .addr = addr_, \ .end = (addr_) + (len_), \ - .pgoff = pgoff_, \ - .pglen = PHYS_PFN(len_), \ + .offset = pos_, \ + .pglen = (len_) >> mm_page_shift(mm_), \ .vma_flags = vma_flags_, \ .file = file_, \ .page_prot = vma_get_page_prot(vma_flags_), \ } +#ifdef CONFIG_MM_SUBPAGE +#define VMG_MMAP_SUBPAGE_STATE(map_) .subpage_offset = (map_)->offset.offset, +#else +#define VMG_MMAP_SUBPAGE_STATE(map_) +#endif + #define VMG_MMAP_STATE(name, map_, vma_) \ struct vma_merge_struct name = { \ .mm = (map_)->mm, \ @@ -62,7 +68,8 @@ struct mmap_state { .start = (map_)->addr, \ .end = (map_)->end, \ .vma_flags = (map_)->vma_flags, \ - .pgoff = (map_)->pgoff, \ + .pgoff = (map_)->offset.index, \ + VMG_MMAP_SUBPAGE_STATE(map_) \ .file = (map_)->file, \ .prev = (map_)->prev, \ .middle = vma_, \ @@ -197,11 +204,12 @@ static void init_multi_vma_prep(struct vma_prepare *vp, */ static bool can_vma_merge_before(struct vma_merge_struct *vmg) { - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); + struct vm_page_offset end = vm_page_offset_add(vmg_page_offset(vmg), + vmg->end - vmg->start); if (is_mergeable_vma(vmg, /* merge_next = */ true) && is_mergeable_anon_vma(vmg, /* merge_next = */ true)) { - if (vmg->next->vm_pgoff == vmg->pgoff + pglen) + if (vm_page_offset_equal(vma_page_offset_at(vmg->next, vmg->next->vm_start), end)) return true; } @@ -221,7 +229,8 @@ static bool can_vma_merge_after(struct vma_merge_struct *vmg) { if (is_mergeable_vma(vmg, /* merge_next = */ false) && is_mergeable_anon_vma(vmg, /* merge_next = */ false)) { - if (vmg->prev->vm_pgoff + vma_pages(vmg->prev) == vmg->pgoff) + if (vm_page_offset_equal(vma_page_offset_at(vmg->prev, vmg->prev->vm_end), + vmg_page_offset(vmg))) return true; } return false; @@ -521,7 +530,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, new->vm_end = addr; } else { new->vm_start = addr; - new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); + vma_set_page_offset(new, vma_page_offset_at(vma, addr)); } err = -ENOMEM; @@ -560,7 +569,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, if (new_below) { vma->vm_start = addr; - vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT; + vma_set_page_offset(vma, vma_page_offset_at(new, addr)); } else { vma->vm_end = addr; } @@ -705,19 +714,21 @@ void validate_mm(struct mm_struct *mm) static void vmg_adjust_set_range(struct vma_merge_struct *vmg) { struct vm_area_struct *adjust; - pgoff_t pgoff; + struct vm_page_offset pos; if (vmg->__adjust_middle_start) { adjust = vmg->middle; - pgoff = adjust->vm_pgoff + PHYS_PFN(vmg->end - adjust->vm_start); + pos = vma_page_offset_at(adjust, vmg->end); } else if (vmg->__adjust_next_start) { adjust = vmg->next; - pgoff = adjust->vm_pgoff - PHYS_PFN(adjust->vm_start - vmg->end); + pos = vm_page_offset_sub(vma_page_offset_at(adjust, adjust->vm_start), + adjust->vm_start - vmg->end); } else { return; } - vma_set_range(adjust, vmg->end, adjust->vm_end, pgoff); + vma_set_range(adjust, vmg->end, adjust->vm_end, pos.index); + vma_set_page_offset(adjust, pos); } /* @@ -762,6 +773,7 @@ static int commit_merge(struct vma_merge_struct *vmg) vma_adjust_trans_huge(vma, vmg->start, vmg->end, vmg->__adjust_middle_start ? vmg->middle : NULL); vma_set_range(vma, vmg->start, vmg->end, vmg->pgoff); + vma_set_page_offset(vma, vmg_page_offset(vmg)); vmg_adjust_set_range(vmg); vma_iter_store_overwrite(vmg->vmi, vmg->target); @@ -931,7 +943,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( vmg->start = prev->vm_start; vmg->end = next->vm_end; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); /* * We already ensured anon_vma compatibility above, so now it's @@ -950,7 +962,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( */ vmg->start = prev->vm_start; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); if (!vmg->__remove_middle) vmg->__adjust_middle_start = true; @@ -965,7 +977,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( * shrink/delete extend */ - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); + unsigned long len = vmg->end - vmg->start; VM_WARN_ON_VMG(!merge_right, vmg); /* If we are offset into a VMA, then prev must be middle. */ @@ -973,13 +985,14 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( if (vmg->__remove_middle) { vmg->end = next->vm_end; - vmg->pgoff = next->vm_pgoff - pglen; + vmg_set_page_offset(vmg, + vm_page_offset_sub(vma_page_offset_at(next, next->vm_start), len)); } else { /* We shrink middle and expand next. */ vmg->__adjust_next_start = true; vmg->start = middle->vm_start; vmg->end = start; - vmg->pgoff = middle->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(middle, middle->vm_start)); } err = dup_anon_vma(next, middle, &anon_dup); @@ -1088,7 +1101,7 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg) if (can_merge_left) { vmg->start = prev->vm_start; vmg->target = prev; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); /* * If this merge would result in removal of the next VMA but we @@ -1245,7 +1258,7 @@ int vma_expand(struct vma_merge_struct *vmg) * Returns: 0 on success, -ENOMEM otherwise */ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff) + unsigned long start, unsigned long end, struct vm_page_offset pos) { struct vma_prepare vp; @@ -1266,7 +1279,8 @@ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_adjust_trans_huge(vma, start, end, NULL); vma_iter_clear(vmi); - vma_set_range(vma, start, end, pgoff); + vma_set_range(vma, start, end, pos.index); + vma_set_page_offset(vma, pos); vma_complete(&vp, vmi, vma->vm_mm); validate_mm(vma->vm_mm); return 0; @@ -1335,7 +1349,7 @@ static void vms_complete_munmap_vmas(struct vma_munmap_struct *vms, mm = current->mm; mm->map_count -= vms->vma_count; - mm->locked_vm -= vms->locked_vm; + mm->locked_vm -= mm_pages_to_account(mm, vms->locked_vm); if (vms->unlock) mmap_write_downgrade(mm); @@ -1360,7 +1374,7 @@ static void vms_complete_munmap_vmas(struct vma_munmap_struct *vms, mas_for_each(mas_detach, vma, ULONG_MAX) remove_vma(vma); - vm_unacct_memory(vms->nr_accounted); + mm_unacct_memory(mm, vms->nr_accounted); validate_mm(mm); if (vms->unlock) mmap_read_unlock(mm); @@ -1456,7 +1470,7 @@ static int vms_gather_munmap_vmas(struct vma_munmap_struct *vms, goto munmap_gather_failed; vma_mark_detached(next); - nrpages = vma_pages(next); + nrpages = vma_user_pages(next); vms->nr_pages += nrpages; if (vma_test(next, VMA_LOCKED_BIT)) @@ -1634,10 +1648,10 @@ int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long end; struct vm_area_struct *vma; - if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) + if ((start & ~mm_page_mask(mm)) || start > TASK_SIZE || len > TASK_SIZE-start) return -EINVAL; - end = start + PAGE_ALIGN(len); + end = start + mm_page_align(mm, len); if (end == start) return -EINVAL; @@ -1862,7 +1876,7 @@ static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) * prior to moving page table entries, to effect an mremap move. */ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - unsigned long addr, unsigned long len, pgoff_t pgoff, + unsigned long addr, unsigned long len, struct vm_page_offset pos, bool *need_rmap_locks) { struct vm_area_struct *vma = *vmap; @@ -1878,7 +1892,10 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, * to match new location, to increase its chance of merging. */ if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { - pgoff = addr >> PAGE_SHIFT; + pos = (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, + .offset = addr & ~PAGE_MASK, + }; faulted_in_anon_vma = false; } @@ -1894,7 +1911,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, if (new_vma && new_vma->vm_start < addr + len) return NULL; /* should never get here */ - vmg.pgoff = pgoff; + vmg_set_page_offset(&vmg, pos); vmg.next = vma_iter_next_rewind(&vmi, NULL); new_vma = vma_merge_copied_range(&vmg); @@ -1924,7 +1941,8 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, new_vma = vm_area_dup(vma); if (!new_vma) goto out; - vma_set_range(new_vma, addr, addr + len, pgoff); + vma_set_range(new_vma, addr, addr + len, pos.index); + vma_set_page_offset(new_vma, pos); if (vma_dup_policy(vma, new_vma)) goto out_free_vma; if (anon_vma_clone(new_vma, vma, VMA_OP_REMAP)) @@ -1979,7 +1997,8 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct * mpol_equal(vma_policy(a), vma_policy(b)) && a->vm_file == b->vm_file && vma_flags_empty(&diff) && - b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); + vm_page_offset_equal(vma_page_offset_at(a, b->vm_start), + vma_page_offset_at(b, b->vm_start)); } /* @@ -2398,7 +2417,10 @@ static void set_desc_from_map(struct vm_area_desc *desc, desc->start = map->addr; desc->end = map->end; - desc->pgoff = map->pgoff; + desc->pgoff = map->offset.index; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = map->offset.offset; +#endif desc->vm_file = map->file; desc->vma_flags = map->vma_flags; desc->page_prot = map->page_prot; @@ -2458,7 +2480,7 @@ static int __mmap_setup(struct mmap_state *map, struct vm_area_desc *desc, map->charged = map->pglen; map->charged -= vms->nr_accounted; if (map->charged) { - error = security_vm_enough_memory_mm(map->mm, map->charged); + error = mm_account_memory(map->mm, map->mm, map->charged); if (error) return error; } @@ -2549,7 +2571,8 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, return -ENOMEM; vma_iter_config(vmi, map->addr, map->end); - vma_set_range(vma, map->addr, map->end, map->pgoff); + vma_set_range(vma, map->addr, map->end, map->offset.index); + vma_set_page_offset(vma, map->offset); vma->flags = map->vma_flags; vma->vm_page_prot = map->page_prot; @@ -2621,7 +2644,7 @@ static void __mmap_complete(struct mmap_state *map, struct vm_area_struct *vma) if (!vma_supports_mlock(vma)) vma_clear_flags_mask(vma, VMA_LOCKED_MASK); else - mm->locked_vm += map->pglen; + mm->locked_vm += mm_pages_to_account(mm, map->pglen); } if (vma->vm_file) @@ -2678,7 +2701,10 @@ static int call_mmap_prepare(struct mmap_state *map, return err; /* Update fields permitted to be changed. */ - map->pgoff = desc->pgoff; + map->offset.index = desc->pgoff; +#ifdef CONFIG_MM_SUBPAGE + map->offset.offset = desc->subpage_offset; +#endif if (desc->vm_file != map->file) { map->file_doesnt_need_get = true; map->file = desc->vm_file; @@ -2689,6 +2715,12 @@ static int call_mmap_prepare(struct mmap_state *map, map->vm_ops = desc->vm_ops; map->vm_private_data = desc->private_data; +#ifdef CONFIG_MM_SUBPAGE + if (map->offset.offset >= PAGE_SIZE || + !IS_ALIGNED(map->offset.offset, mm_page_size(map->mm))) + return -EINVAL; +#endif + return 0; } @@ -2731,13 +2763,13 @@ static bool can_set_ksm_flags_early(struct mmap_state *map) static unsigned long __mmap_region(struct file *file, unsigned long addr, unsigned long len, vma_flags_t vma_flags, - unsigned long pgoff, struct list_head *uf) + struct vm_page_offset pos, struct list_head *uf) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma = NULL; bool have_mmap_prepare = file && file->f_op->mmap_prepare; VMA_ITERATOR(vmi, mm, addr); - MMAP_STATE(map, mm, &vmi, addr, len, pgoff, vma_flags, file); + MMAP_STATE(map, mm, &vmi, addr, len, pos, vma_flags, file); struct vm_area_desc desc = { .mm = mm, .file = file, @@ -2791,7 +2823,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, /* Accounting was done by __mmap_setup(). */ unacct_error: if (map.charged) - vm_unacct_memory(map.charged); + mm_unacct_memory(mm, map.charged); abort_munmap: /* * This indicates that .mmap_prepare has set a new file, differing from @@ -2807,7 +2839,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, /** * mmap_region() - Actually perform the userland mapping of a VMA into * current->mm with known, aligned and overflow-checked @addr and @len, and - * correctly determined VMA flags @vm_flags and page offset @pgoff. + * correctly determined VMA flags @vm_flags and file byte position @pos. * * This is an internal memory management function, and should not be used * directly. @@ -2819,8 +2851,8 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, * @addr: The page-aligned address at which to perform the mapping. * @len: The page-aligned, non-zero, length of the mapping. * @vm_flags: The VMA flags which should be applied to the mapping. - * @pgoff: If @file is specified, the page offset into the file, if not then - * the virtual page offset in memory of the anonymous mapping. + * @pos: Native page-cache index and byte offset within it. For anonymous + * mappings, the position is derived from the virtual address. * @uf: Optionally, a pointer to a list head used for tracking userfaultfd unmap * events. * @@ -2829,13 +2861,15 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, */ unsigned long mmap_region(struct file *file, unsigned long addr, unsigned long len, vm_flags_t vm_flags, - unsigned long pgoff, struct list_head *uf) + struct vm_page_offset pos, struct list_head *uf) { unsigned long ret; bool writable_file_mapping = false; const vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags); mmap_assert_write_locked(current->mm); + if (pos.offset >= PAGE_SIZE || !IS_ALIGNED(pos.offset, mm_page_size(current->mm))) + return -EINVAL; /* Check to see if MDWE is applicable. */ if (map_deny_write_exec(&vma_flags, &vma_flags)) @@ -2854,7 +2888,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, writable_file_mapping = true; } - ret = __mmap_region(file, addr, len, vma_flags, pgoff, uf); + ret = __mmap_region(file, addr, len, vma_flags, pos, uf); /* Clear our write mapping regardless of error. */ if (writable_file_mapping) @@ -2892,13 +2926,13 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_flags_set_mask(&vma_flags, mm->def_vma_flags); vma_flags = ksm_vma_flags(mm, NULL, vma_flags); - if (!may_expand_vm(mm, &vma_flags, len >> PAGE_SHIFT)) + if (!may_expand_vm(mm, &vma_flags, len >> mm_page_shift(mm))) return -ENOMEM; if (mm->map_count > get_sysctl_max_map_count()) return -ENOMEM; - if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) + if (mm_account_memory(mm, mm, len >> mm_page_shift(mm))) return -ENOMEM; /* @@ -2908,6 +2942,10 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, if (vma && vma->vm_end == addr) { VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, PHYS_PFN(addr)); + vmg_set_page_offset(&vmg, (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, .offset = addr & ~PAGE_MASK, + }); + vmg.prev = vma; /* vmi is positioned at prev, which this mode expects. */ vmg.just_expand = true; @@ -2927,6 +2965,9 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_set_anonymous(vma); vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT); + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, .offset = addr & ~PAGE_MASK, + }); vma->flags = vma_flags; vma->vm_page_prot = vm_get_page_prot(vma_flags_to_legacy(vma_flags)); vma_start_write(vma); @@ -2937,10 +2978,10 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, validate_mm(mm); out: perf_event_mmap(vma); - mm->total_vm += len >> PAGE_SHIFT; - mm->data_vm += len >> PAGE_SHIFT; + mm->total_vm += len >> mm_page_shift(mm); + mm->data_vm += len >> mm_page_shift(mm); if (vma_flags_test(&vma_flags, VMA_LOCKED_BIT)) - mm->locked_vm += (len >> PAGE_SHIFT); + mm->locked_vm += len >> MM_ACCOUNT_SHIFT; if (pgtable_supports_soft_dirty()) vma_set_flags(vma, VMA_SOFTDIRTY_BIT); return 0; @@ -2948,7 +2989,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, mas_store_fail: vm_area_free(vma); unacct_fail: - vm_unacct_memory(len >> PAGE_SHIFT); + mm_unacct_memory(mm, len >> mm_page_shift(mm)); return -ENOMEM; } @@ -3084,7 +3125,7 @@ static int acct_stack_growth(struct vm_area_struct *vma, /* mlock limit tests */ if (!mlock_future_ok(mm, vma_test(vma, VMA_LOCKED_BIT), - grow << PAGE_SHIFT)) + grow << mm_page_shift(mm))) return -ENOMEM; /* Check to ensure the stack will not grow into a hugetlb-only region */ @@ -3100,7 +3141,7 @@ static int acct_stack_growth(struct vm_area_struct *vma, * Overcommit.. This must be the final test, as it will * update security statistics. */ - if (security_vm_enough_memory_mm(mm, grow)) + if (mm_account_memory(mm, mm, grow)) return -ENOMEM; return 0; @@ -3125,10 +3166,10 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) mmap_assert_write_locked(mm); /* Guard against exceeding limits of the address space. */ - address &= PAGE_MASK; - if (address >= (TASK_SIZE & PAGE_MASK)) + address &= mm_page_mask(mm); + if (address >= (TASK_SIZE & mm_page_mask(mm))) return -ENOMEM; - address += PAGE_SIZE; + address += mm_page_size(mm); /* Enforce stack_guard_gap */ gap_addr = address + stack_guard_gap; @@ -3167,14 +3208,14 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) unsigned long size, grow; size = address - vma->vm_start; - grow = (address - vma->vm_end) >> PAGE_SHIFT; + grow = (address - vma->vm_end) >> mm_page_shift(mm); error = -ENOMEM; if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) - mm->locked_vm += grow; + mm->locked_vm += mm_pages_to_account(mm, grow); vm_stat_account(mm, vma->vm_flags, grow); anon_vma_interval_tree_pre_update_vma(vma); vma->vm_end = address; @@ -3209,7 +3250,7 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) mmap_assert_write_locked(mm); - address &= PAGE_MASK; + address &= mm_page_mask(mm); if (address < mmap_min_addr || address < FIRST_USER_ADDRESS) return -EPERM; @@ -3244,20 +3285,22 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) /* Somebody else might have raced and expanded it already */ if (address < vma->vm_start) { unsigned long size, grow; + struct vm_page_offset pos = vm_page_offset_sub( + vma_page_offset_at(vma, vma->vm_start), vma->vm_start - address); size = vma->vm_end - address; - grow = (vma->vm_start - address) >> PAGE_SHIFT; + grow = (vma->vm_start - address) >> mm_page_shift(mm); error = -ENOMEM; - if (grow <= vma->vm_pgoff) { + if (pos.index <= vma->vm_pgoff) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) - mm->locked_vm += grow; + mm->locked_vm += mm_pages_to_account(mm, grow); vm_stat_account(mm, vma->vm_flags, grow); anon_vma_interval_tree_pre_update_vma(vma); vma->vm_start = address; - vma->vm_pgoff -= grow; + vma_set_page_offset(vma, pos); /* Overwrite old entry in mtree. */ vma_iter_store_overwrite(&vmi, vma); anon_vma_interval_tree_post_update_vma(vma); @@ -3296,13 +3339,13 @@ int __vm_munmap(unsigned long start, size_t len, bool unlock) */ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) { - unsigned long charged = vma_pages(vma); + unsigned long charged = vma_user_pages(vma); if (find_vma_intersection(mm, vma->vm_start, vma->vm_end)) return -ENOMEM; if (vma_test(vma, VMA_ACCOUNT_BIT) && - security_vm_enough_memory_mm(mm, charged)) + mm_account_memory(mm, mm, charged)) return -ENOMEM; /* @@ -3319,12 +3362,15 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) */ if (vma_is_anonymous(vma)) { BUG_ON(vma->anon_vma); - vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = vma->vm_start >> PAGE_SHIFT, + .offset = vma->vm_start & ~PAGE_MASK, + }); } if (vma_link(mm, vma)) { if (vma_test(vma, VMA_ACCOUNT_BIT)) - vm_unacct_memory(charged); + mm_unacct_memory(mm, charged); return -ENOMEM; } diff --git a/mm/vma.h b/mm/vma.h index 8e4b61a7304c6..3d9c8ee4be05d 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -97,6 +97,9 @@ struct vma_merge_struct { unsigned long start; unsigned long end; pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif union { /* Temporary while VMA flags are being converted. */ @@ -234,9 +237,34 @@ static inline bool vmg_nomem(struct vma_merge_struct *vmg) static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, unsigned long addr) { - return vma->vm_pgoff + PHYS_PFN(addr - vma->vm_start); + return vma_page_offset_at(vma, addr).index; +} + +static inline struct vm_page_offset vmg_page_offset(const struct vma_merge_struct *vmg) +{ + return (struct vm_page_offset) { + .index = vmg->pgoff, +#ifdef CONFIG_MM_SUBPAGE + .offset = vmg->subpage_offset, +#endif + }; +} + +static inline void vmg_set_page_offset(struct vma_merge_struct *vmg, struct vm_page_offset pos) +{ + vmg->pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vmg->subpage_offset = pos.offset; +#endif } +#ifdef CONFIG_MM_SUBPAGE +#define VMG_SUBPAGE_STATE(vma, start) \ + .subpage_offset = vma_page_offset_at(vma, start).offset, +#else +#define VMG_SUBPAGE_STATE(vma, start) +#endif + #define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_) \ struct vma_merge_struct name = { \ .mm = mm_, \ @@ -259,6 +287,7 @@ static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, .end = end_, \ .vm_flags = vma_->vm_flags, \ .pgoff = vma_pgoff_offset(vma_, start_), \ + VMG_SUBPAGE_STATE(vma_, start_) \ .file = vma_->vm_file, \ .anon_vma = vma_->anon_vma, \ .policy = vma_policy(vma_), \ @@ -276,7 +305,7 @@ void validate_mm(struct mm_struct *mm); __must_check int vma_expand(struct vma_merge_struct *vmg); __must_check int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff); + unsigned long start, unsigned long end, struct vm_page_offset pos); static inline int vma_iter_store_gfp(struct vma_iterator *vmi, struct vm_area_struct *vma, gfp_t gfp) @@ -311,6 +340,9 @@ static inline void compat_set_vma_from_desc(struct vm_area_struct *vma, /* Mutable fields. Populated with initial state. */ vma->vm_pgoff = desc->pgoff; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = desc->subpage_offset; +#endif if (desc->vm_file != vma->vm_file) vma_set_file(vma, desc->vm_file); vma->flags = desc->vma_flags; @@ -448,7 +480,7 @@ void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, struct vm_area_struct *vma); struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - unsigned long addr, unsigned long len, pgoff_t pgoff, + unsigned long addr, unsigned long len, struct vm_page_offset pos, bool *need_rmap_locks); struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma); @@ -460,7 +492,7 @@ int mm_take_all_locks(struct mm_struct *mm); void mm_drop_all_locks(struct mm_struct *mm); unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, + unsigned long len, vm_flags_t vm_flags, struct vm_page_offset pos, struct list_head *uf); int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, diff --git a/mm/vma_exec.c b/mm/vma_exec.c index 5cee8b7efa0f2..8394f45a91113 100644 --- a/mm/vma_exec.c +++ b/mm/vma_exec.c @@ -42,6 +42,8 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) struct mmu_gather tlb; PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length); + vmg_set_page_offset(&vmg, vma_page_offset_at(vma, old_start)); + BUG_ON(new_start > new_end); /* @@ -89,7 +91,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) vma_prev(&vmi); /* Shrink the vma to just the new range */ - return vma_shrink(&vmi, vma, new_start, new_end, vma->vm_pgoff); + return vma_shrink(&vmi, vma, new_start, new_end, vma_page_offset_at(vma, vma->vm_start)); } /* @@ -138,7 +140,7 @@ int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, */ VM_WARN_ON_ONCE(VM_STACK_FLAGS & VM_STACK_INCOMPLETE_SETUP); vma->vm_end = STACK_TOP_MAX; - vma->vm_start = vma->vm_end - PAGE_SIZE; + vma->vm_start = vma->vm_end - mm_page_size(mm); if (pgtable_supports_soft_dirty()) flags |= VM_SOFTDIRTY; vm_flags_init(vma, flags); diff --git a/mm/vma_init.c b/mm/vma_init.c index 3c0b65950510e..a41ba35de7320 100644 --- a/mm/vma_init.c +++ b/mm/vma_init.c @@ -46,7 +46,10 @@ static void vm_area_init_from(const struct vm_area_struct *src, dest->vm_start = src->vm_start; dest->vm_end = src->vm_end; dest->anon_vma = src->anon_vma; - dest->vm_pgoff = src->vm_pgoff; + vma_set_page_offset(dest, (struct vm_page_offset) { + .index = src->vm_pgoff, + .offset = vma_subpage_offset(src), + }); dest->vm_file = src->vm_file; dest->vm_private_data = src->vm_private_data; vm_flags_init(dest, src->vm_flags); diff --git a/mm/vmscan.c b/mm/vmscan.c index 1af6c96d1bc14..fe28f70fd51b1 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3462,6 +3462,46 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, } } +static unsigned long lru_pte_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_page_size(mm) == PAGE_SIZE) + return pte_index(addr); + return (addr >> mm_page_shift(mm)) & (pte_table_bytes_mm(mm) / sizeof(pte_t) - 1); +} + +static unsigned long lru_pmd_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_page_size(mm) == PAGE_SIZE) + return pmd_index(addr); + return (addr >> pmd_shift_mm(mm)) & (pmd_table_bytes_mm(mm) / sizeof(pmd_t) - 1); +} + +static unsigned long lru_pud_entries(struct mm_struct *mm) +{ + return mm_pud_folded(mm) ? 1 : pud_table_bytes_mm(mm) / sizeof(pud_t); +} + +static unsigned long lru_pud_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_pud_folded(mm)) + return 0; + if (mm_page_size(mm) == PAGE_SIZE) + return pud_index(addr); + return (addr >> pud_shift_mm(mm)) & (lru_pud_entries(mm) - 1); +} + +static unsigned int lru_folio_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t *entry, unsigned int max_nr) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch_flags(vma, folio, ptep, entry, + max_nr, FPB_MERGE_YOUNG_DIRTY); +#endif + return folio_pte_batch_flags(folio, NULL, ptep, entry, max_nr, + FPB_MERGE_YOUNG_DIRTY); +} + static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, struct mm_walk *args) { @@ -3481,7 +3521,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, unsigned int nr; pmd_t pmdval; - pte = pte_offset_map_rw_nolock(args->mm, pmd, start & PMD_MASK, &pmdval, &ptl); + pte = pte_offset_map_rw_nolock(args->mm, pmd, start & pmd_mask_mm(args->mm), &pmdval, &ptl); if (!pte) return false; @@ -3497,7 +3537,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, lazy_mmu_mode_enable(); restart: - for (i = pte_index(start), addr = start; addr != end; i += nr, addr += nr * PAGE_SIZE) { + for (i = lru_pte_index(args->mm, start), addr = start; addr != end; i += nr, addr += nr * mm_page_size(args->mm)) { unsigned long pfn; struct folio *folio; pte_t *cur_pte = pte + i; @@ -3516,10 +3556,9 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, continue; if (folio_test_large(folio)) { - const unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + const unsigned int max_nr = (end - addr) >> mm_page_shift(args->mm); - nr = folio_pte_batch_flags(folio, NULL, cur_pte, &ptent, - max_nr, FPB_MERGE_YOUNG_DIRTY); + nr = lru_folio_pte_batch(args->vma, folio, cur_pte, &ptent, max_nr); total += nr - 1; walk->mm_stats[MM_LEAF_TOTAL] += nr - 1; } @@ -3544,7 +3583,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, walk_update_folio(walk, last, gen, dirty); last = NULL; - if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) + if (i < pte_table_bytes_mm(args->mm) / sizeof(pte_t) && + get_next_vma(pmd_mask_mm(args->mm), mm_page_size(args->mm), args, &start, &end)) goto restart; lazy_mmu_mode_disable(); @@ -3576,13 +3616,13 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area return; } - i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first); + i = addr == -1 ? 0 : lru_pmd_index(args->mm, addr) - lru_pmd_index(args->mm, *first); if (i && i <= MIN_LRU_BATCH) { __set_bit(i - 1, bitmap); return; } - pmd = pmd_offset(pud, *first); + pmd = pmd_offset_mm(args->mm, pud, *first); ptl = pmd_lockptr(args->mm, pmd); if (!spin_trylock(ptl)) @@ -3595,7 +3635,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area struct folio *folio; /* don't round down the first address */ - addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first; + addr = i ? (*first & pmd_mask_mm(args->mm)) + i * pmd_size_mm(args->mm) : *first; if (!pmd_present(pmd[i])) goto next; @@ -3661,14 +3701,14 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, * tables to avoid taking the PMD lock; the second, if necessary, takes * the PMD lock to clear the accessed bit in PMD entries. */ - pmd = pmd_offset(pud, start & PUD_MASK); + pmd = pmd_offset_mm(args->mm, pud, start & pud_mask_mm(args->mm)); restart: /* walk_pte_range() may call get_next_vma() */ vma = args->vma; - for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { + for (i = lru_pmd_index(args->mm, start), addr = start; addr != end; i++, addr = next) { pmd_t val = pmdp_get_lockless(pmd + i); - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(args->mm, addr, end); if (!pmd_present(val) || is_huge_zero_pmd(val)) { walk->mm_stats[MM_LEAF_TOTAL]++; @@ -3710,7 +3750,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first); - if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) + if (i < pmd_table_bytes_mm(args->mm) / sizeof(pmd_t) && + get_next_vma(pud_mask_mm(args->mm), pmd_size_mm(args->mm), args, &start, &end)) goto restart; } @@ -3725,12 +3766,12 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, VM_WARN_ON_ONCE(p4d_leaf(*p4d)); - pud = pud_offset(p4d, start & P4D_MASK); + pud = pud_offset_mm(args->mm, p4d, start & p4d_mask_mm(args->mm)); restart: - for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { + for (i = lru_pud_index(args->mm, start), addr = start; addr != end; i++, addr = next) { pud_t val = pudp_get(pud + i); - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(args->mm, addr, end); if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) continue; @@ -3738,15 +3779,16 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, walk_pmd_range(&val, addr, next, args); if (need_resched() || walk->batched >= MAX_LRU_BATCH) { - end = (addr | ~PUD_MASK) + 1; + end = (addr | ~pud_mask_mm(args->mm)) + 1; goto done; } } - if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) + if (i < lru_pud_entries(args->mm) && + get_next_vma(p4d_mask_mm(args->mm), pud_size_mm(args->mm), args, &start, &end)) goto restart; - end = round_up(end, P4D_SIZE); + end = round_up(end, p4d_size_mm(args->mm)); done: if (!end || !args->vma) return 1; @@ -4215,20 +4257,20 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) /* avoid taking the LRU lock under the PTL when possible */ walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; - start = max(addr & PMD_MASK, vma->vm_start); - end = min(addr | ~PMD_MASK, vma->vm_end - 1) + 1; + start = max(addr & pmd_mask_mm(vma->vm_mm), vma->vm_start); + end = min(addr | ~pmd_mask_mm(vma->vm_mm), vma->vm_end - 1) + 1; - if (end - start == PAGE_SIZE) + if (end - start == mm_page_size(vma->vm_mm)) return true; - if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { - if (addr - start < MIN_LRU_BATCH * PAGE_SIZE / 2) - end = start + MIN_LRU_BATCH * PAGE_SIZE; - else if (end - addr < MIN_LRU_BATCH * PAGE_SIZE / 2) - start = end - MIN_LRU_BATCH * PAGE_SIZE; + if (end - start > MIN_LRU_BATCH * mm_page_size(vma->vm_mm)) { + if (addr - start < MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2) + end = start + MIN_LRU_BATCH * mm_page_size(vma->vm_mm); + else if (end - addr < MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2) + start = end - MIN_LRU_BATCH * mm_page_size(vma->vm_mm); else { - start = addr - MIN_LRU_BATCH * PAGE_SIZE / 2; - end = addr + MIN_LRU_BATCH * PAGE_SIZE / 2; + start = addr - MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2; + end = addr + MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2; } } @@ -4240,10 +4282,10 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) lazy_mmu_mode_enable(); - pte -= (addr - start) / PAGE_SIZE; + pte -= (addr - start) / mm_page_size(vma->vm_mm); for (i = 0, addr = start; addr != end; - i += nr, pte += nr, addr += nr * PAGE_SIZE) { + i += nr, pte += nr, addr += nr * mm_page_size(vma->vm_mm)) { unsigned long pfn; pte_t ptent = ptep_get(pte); @@ -4257,10 +4299,9 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) continue; if (folio_test_large(folio)) { - const unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + const unsigned int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); - nr = folio_pte_batch_flags(folio, NULL, pte, &ptent, - max_nr, FPB_MERGE_YOUNG_DIRTY); + nr = lru_folio_pte_batch(vma, folio, pte, &ptent, max_nr); } if (!test_and_clear_young_ptes_notify(vma, addr, pte, nr)) @@ -4634,6 +4675,16 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c sc->nr.unqueued_dirty += delta; } + /* + * Legacy memcg reclaim relies on shrink_folio_list() waiting for + * writeback when reclaim-marked folios cycle through the LRU. Moving + * all of them to a younger generation can exhaust the charge limit + * before I/O completes. Let the common path apply its GFP and mapping + * deadlock checks instead of hiding those folios from it. + */ + if (writeback && !writeback_throttling_sane(sc)) + return false; + /* waiting for writeback */ if (writeback || (type == LRU_GEN_FILE && dirty)) { gen = folio_inc_gen(lruvec, folio, true); @@ -4668,8 +4719,9 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca if (!folio_test_referenced(folio)) set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0); - /* for shrink_folio_list() */ - folio_clear_reclaim(folio); + /* Preserve legacy memcg's repeated-writeback marker for shrink_folio_list(). */ + if (writeback_throttling_sane(sc) || !folio_test_writeback(folio)) + folio_clear_reclaim(folio); success = lru_gen_del_folio(lruvec, folio, true); VM_WARN_ON_ONCE_FOLIO(!success, folio); From c1a5f4938d74e70de292adee44236d69f7b14a2b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0422/1579] mm: add retained subpage backing ownership Implement reference-counted user slots within native folios, including zeroing, mapping and pin ownership, migration and deferred lifetime. The backend remains disabled until its architecture integration. Signed-off-by: Diljit Singh --- mm/Makefile | 1 + mm/page_ext.c | 4 + mm/user-subpage.c | 1673 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1678 insertions(+) create mode 100644 mm/user-subpage.c diff --git a/mm/Makefile b/mm/Makefile index 8ad2ab08244eb..f531742956b8e 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -150,3 +150,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o obj-$(CONFIG_EXECMEM) += execmem.o obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o +obj-$(CONFIG_MM_SUBPAGE) += user-subpage.o diff --git a/mm/page_ext.c b/mm/page_ext.c index e2e92bd27ebdd..a73aab719c040 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include @@ -78,6 +79,9 @@ static struct page_ext_operations page_idle_ops __initdata = { #endif static struct page_ext_operations *page_ext_ops[] __initdata = { +#ifdef CONFIG_MM_SUBPAGE + &mm_subpage_ext_ops, +#endif #ifdef CONFIG_PAGE_OWNER &page_owner_ops, #endif diff --git a/mm/user-subpage.c b/mm/user-subpage.c new file mode 100644 index 0000000000000..133d75241485a --- /dev/null +++ b/mm/user-subpage.c @@ -0,0 +1,1673 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Reference-counted user data slots in native folios. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "user-subpage-internal.h" +#include "internal.h" + +#define SUBPAGES_PER_FOLIO (PAGE_SIZE / MM_SUBPAGE_SIZE) +static bool subpage_shift_valid(unsigned int shift) +{ + return shift >= MM_SUBPAGE_SHIFT && shift < PAGE_SHIFT; +} + +static unsigned long subpage_mask(unsigned int shift) +{ + return GENMASK((PAGE_SIZE >> shift) - 1, 0); +} + +static unsigned long offset_mask(unsigned int shift, unsigned int offset) +{ + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, 1U << shift)) + return 0; + return BIT(offset >> shift); +} + +struct mm_subpage_owner { + struct rcu_head rcu; + struct work_struct free_work; + struct mm_subpage_pool *pool; + struct folio *folio; + struct anon_vma *anon_root; + struct mm_subpage_owner *migration_target; + /* Allocated only when slots need independent logical identities. */ + struct mm_subpage_rmap *rmap; + struct list_head all; + struct list_head available; + unsigned long used; + unsigned long zeroed; + u8 shift; + bool detached; + bool migrating; + bool droppable; + struct mm_subpage slots[SUBPAGES_PER_FOLIO]; +}; + +struct mm_subpage_ext { + struct mm_subpage_owner __rcu *owner; + atomic_t file_maps[SUBPAGES_PER_FOLIO]; + atomic_t file_leaves; + /* Folio lock protects validity; fits the 16K/64K layout padding. */ + u16 shmem_missing; +}; +static_assert(SUBPAGES_PER_FOLIO == 1 || sizeof(struct mm_subpage_ext) == + ALIGN(sizeof(void *) + sizeof(atomic_t) * (SUBPAGES_PER_FOLIO + 1), + sizeof(void *))); +static_assert(SUBPAGES_PER_FOLIO <= sizeof(u16) * BITS_PER_BYTE); + +unsigned long mm_subpage_shmem_missing(struct page *page) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + unsigned long missing; + + /* A referenced native page keeps its initialized section online. */ + VM_BUG_ON(!ext); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + missing = subext->shmem_missing; + page_ext_put(ext); + return missing; +} + +void mm_subpage_shmem_set_missing(struct page *page, unsigned long missing) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + + VM_BUG_ON(!ext); + VM_BUG_ON(missing & ~GENMASK(SUBPAGES_PER_FOLIO - 1, 0)); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + subext->shmem_missing = missing; + page_ext_put(ext); +} + +static bool __init subpage_ext_needed(void) +{ + return true; +} + +struct page_ext_operations mm_subpage_ext_ops = { + .size = sizeof(struct mm_subpage_ext), + .need = subpage_ext_needed, +}; + +/* Only alternative PTE mappings contribute; native rmap counts stay unchanged. */ +static void subpage_file_map_update(phys_addr_t phys, unsigned long size, int delta) +{ + struct page_ext *ext; + struct mm_subpage_ext *subext; + unsigned int index, nr, i; + int count; + + VM_BUG_ON(!is_power_of_2(size) || size < MM_SUBPAGE_SIZE || + size >= PAGE_SIZE || !IS_ALIGNED(phys, size)); + ext = page_ext_from_phys(phys); + if (WARN_ON_ONCE(!ext)) + return; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + index = offset_in_page(phys) >> MM_SUBPAGE_SHIFT; + nr = size >> MM_SUBPAGE_SHIFT; + for (i = index; i < index + nr; i++) { + count = atomic_add_return(delta, &subext->file_maps[i]); + VM_WARN_ON(count < 0); + } + /* Rmap counts one mapping, regardless of its coverage in 4K units. */ + count = atomic_add_return(delta, &subext->file_leaves); + VM_WARN_ON(count < 0); + page_ext_put(ext); +} + +void __mm_subpage_file_map_add(phys_addr_t phys, unsigned long size) +{ + subpage_file_map_update(phys, size, 1); +} + +void __mm_subpage_file_map_del(phys_addr_t phys, unsigned long size) +{ + subpage_file_map_update(phys, size, -1); +} + +/* Snapshot for statistics only, never an ownership or pinning decision. */ +unsigned int mm_subpage_file_mapcounts(struct page *page, struct mm_subpage_mapcounts *counts) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + unsigned int i, leaves; + + if (WARN_ON_ONCE(!ext)) { + memset(counts, 0, sizeof(*counts)); + return 0; + } + subext = page_ext_data(ext, &mm_subpage_ext_ops); + for (i = 0; i < SUBPAGES_PER_FOLIO; i++) + counts->slots[i] = max(atomic_read(&subext->file_maps[i]), 0); + leaves = max(atomic_read(&subext->file_leaves), 0); + page_ext_put(ext); + return leaves; +} + +static int owner_bind(struct mm_subpage_owner *owner) +{ + struct page_ext *ext = page_ext_get(&owner->folio->page); + struct mm_subpage_ext *subext; + int ret; + + if (!ext) + return -EOPNOTSUPP; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + ret = cmpxchg(&subext->owner, NULL, owner) ? -EEXIST : 0; + page_ext_put(ext); + return ret; +} + +static void owner_unbind(struct mm_subpage_owner *owner) +{ + struct page_ext *ext = page_ext_get(&owner->folio->page); + struct mm_subpage_ext *subext; + + /* The allocated, referenced folio prevents its section going offline. */ + VM_BUG_ON(!ext); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + VM_BUG_ON(rcu_access_pointer(subext->owner) != owner); + rcu_assign_pointer(subext->owner, NULL); + page_ext_put(ext); +} + +struct mm_subpage *mm_subpage_get_from_phys(phys_addr_t phys) +{ + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + struct mm_subpage *slot = NULL; + struct page_ext *ext; + unsigned int index; + + if (!IS_ALIGNED(phys, MM_SUBPAGE_SIZE)) + return NULL; + ext = page_ext_from_phys(phys); + if (!ext) + return NULL; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + index = owner ? offset_in_page(phys) >> owner->shift : 0; + /* Keep the native allocation alive before publishing another slot ref. */ + if (owner && folio_try_get(owner->folio)) { + if (refcount_inc_not_zero_acquire(&owner->slots[index].refs)) + slot = &owner->slots[index]; + else + folio_put(owner->folio); + } + page_ext_put(ext); + return slot; +} + +struct mm_subpage_pool { + struct rcu_head rcu; + spinlock_t lock; + refcount_t refs; + struct list_head owners; + struct list_head available; + struct mm_subpage_owner *unissued; + unsigned long backing_pages; + unsigned long all_slots; + u8 shift; + bool closed; +}; + +static void owner_init(struct mm_subpage_owner *owner, struct mm_subpage_pool *pool, + struct folio *folio) +{ + unsigned int i; + + owner->pool = pool; + owner->shift = pool->shift; + owner->folio = folio; + INIT_LIST_HEAD(&owner->available); + for (i = 0; i < (PAGE_SIZE >> pool->shift); i++) { + owner->slots[i].owner = owner; + owner->slots[i].index = i; + owner->slots[i].shift = pool->shift; + refcount_set(&owner->slots[i].refs, 0); + atomic_set(&owner->slots[i].mapcount, 0); + } +} + +struct mm_subpage_pool *mm_subpage_pool_create_granule(gfp_t gfp, unsigned int shift) +{ + struct mm_subpage_pool *pool; + + BUILD_BUG_ON(PAGE_SHIFT <= MM_SUBPAGE_SHIFT); + BUILD_BUG_ON(SUBPAGES_PER_FOLIO > BITS_PER_LONG); + if (!subpage_shift_valid(shift)) + return NULL; + pool = kzalloc(sizeof(*pool), gfp | __GFP_ACCOUNT); + if (!pool) + return NULL; + pool->shift = shift; + pool->all_slots = subpage_mask(shift); + spin_lock_init(&pool->lock); + refcount_set(&pool->refs, 1); + INIT_LIST_HEAD(&pool->owners); + INIT_LIST_HEAD(&pool->available); + return pool; +} + +struct mm_subpage_pool *mm_subpage_pool_create(gfp_t gfp) +{ + return mm_subpage_pool_create_granule(gfp, MM_SUBPAGE_SHIFT); +} + +void mm_subpage_pool_get(struct mm_subpage_pool *pool) +{ + refcount_inc(&pool->refs); +} + +void mm_subpage_pool_put(struct mm_subpage_pool *pool) +{ + if (!refcount_dec_and_test(&pool->refs)) + return; + WARN_ON_ONCE(!pool->closed || !list_empty(&pool->owners)); + kfree_rcu(pool, rcu); +} + +static void owner_remove(struct mm_subpage_owner *owner) +{ + lockdep_assert_held(&owner->pool->lock); + owner->detached = true; + list_del(&owner->all); + list_del_init(&owner->available); + if (owner->pool->unissued == owner) + owner->pool->unissued = NULL; + owner->pool->backing_pages--; +} + +static void owner_free_rmap(struct mm_subpage_owner *owner) +{ + unsigned int i; + + if (!owner->rmap) + return; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (owner->rmap[i].root) + put_anon_vma(owner->rmap[i].root); + kfree(owner->rmap); +} + +static void owner_free_metadata(struct mm_subpage_owner *owner) +{ + struct mm_subpage_pool *pool = owner->pool; + + if (owner->anon_root) + put_anon_vma(owner->anon_root); + owner_free_rmap(owner); + kfree_rcu(owner, rcu); + mm_subpage_pool_put(pool); +} + +static void owner_free_work(struct work_struct *work) +{ + owner_free_metadata(container_of(work, struct mm_subpage_owner, free_work)); +} + +static void owner_free(struct mm_subpage_owner *owner, bool put_folio) +{ + owner_unbind(owner); + if (put_folio) { + if (owner->anon_root) + free_swap_cache(owner->folio); + /* Last slot reference, or wholly unissued supply. */ + folio_put(owner->folio); + } + if (owner->rmap) { + /* Historical roots can have no live VMA; their final put may sleep. */ + INIT_WORK(&owner->free_work, owner_free_work); + queue_work(system_unbound_wq, &owner->free_work); + } else { + owner_free_metadata(owner); + } +} + +struct anon_vma *mm_subpage_anon_root(const struct mm_subpage *subpage) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + return subpage->owner->anon_root; +} + +void mm_subpage_hold_anon_root(struct mm_subpage *subpage, struct anon_vma *root, + bool droppable) +{ + struct mm_subpage_owner *owner = subpage->owner; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + VM_BUG_ON(owner->anon_root || root != root->root); + get_anon_vma(root); + owner->droppable = droppable; + owner->anon_root = root; +} + +static struct mm_subpage_rmap slot_rmap(struct mm_subpage *slot) +{ + struct mm_subpage_owner *owner = slot->owner; + + lockdep_assert_held(&owner->pool->lock); + if (owner->rmap) + return owner->rmap[slot->index]; + return (struct mm_subpage_rmap) { + .root = owner->anon_root, + .pos = { .index = owner->folio->index, + .offset = mm_subpage_offset(slot) }, + .offset = mm_subpage_offset(slot), + }; +} + +static bool rmap_matches(struct mm_subpage_rmap map, struct vm_area_struct *vma, + unsigned long address) +{ + return map.root == vma->anon_vma->root && + vm_page_offset_equal(map.pos, vma_page_offset_at(vma, address)); +} + +bool mm_subpage_match_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address) +{ + struct mm_subpage_pool *pool = slot->owner->pool; + unsigned long flags; + bool matches; + + spin_lock_irqsave(&pool->lock, flags); + matches = slot->rmap_bound && rmap_matches(slot_rmap(slot), vma, address); + spin_unlock_irqrestore(&pool->lock, flags); + return matches; +} + +static void *owner_kcalloc(struct mm_subpage_owner *owner, size_t nr, size_t size) +{ + struct mem_cgroup *memcg, *saved; + void *maps; + + rcu_read_lock(); + do { + memcg = mem_cgroup_from_virt(owner) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + maps = kcalloc(nr, size, GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + set_active_memcg(saved); + mem_cgroup_put(memcg); + return maps; +} + +struct subpage_retired_root { + struct work_struct work; + struct anon_vma *root; +}; + +static void subpage_put_retired_root(struct work_struct *work) +{ + struct subpage_retired_root *retired = + container_of(work, struct subpage_retired_root, work); + + put_anon_vma(retired->root); + kfree(retired); +} + +/* The folio lock serializes binding and sidecar creation against migration. */ +int mm_subpage_bind_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address) +{ + struct mm_subpage_owner *owner = slot->owner; + struct mm_subpage_pool *pool = owner->pool; + struct mm_subpage_rmap *maps = NULL; + struct subpage_retired_root *retired = NULL; + unsigned long flags; + unsigned int i; + int ret = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + spin_lock_irqsave(&pool->lock, flags); + if (slot->rmap_bound) { + ret = rmap_matches(slot_rmap(slot), vma, address) ? 0 : -EINVAL; + goto unlock; + } + if (!owner->rmap && !rmap_matches(slot_rmap(slot), vma, address)) { + spin_unlock_irqrestore(&pool->lock, flags); + /* The caller may also hold a PTE lock. Never enter reclaim here. */ + maps = owner_kcalloc(owner, PAGE_SIZE >> owner->shift, sizeof(*maps)); + if (!maps) + return -ENOMEM; + spin_lock_irqsave(&pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + maps[i].offset = i << owner->shift; + if (!owner->slots[i].rmap_bound) + continue; + maps[i] = slot_rmap(&owner->slots[i]); + get_anon_vma(maps[i].root); + } + owner->rmap = maps; + } + if (owner->rmap) { + struct mm_subpage_rmap *map = &owner->rmap[slot->index]; + + if (map->root && map->root != vma->anon_vma->root) { + /* A final anon_vma put may sleep; the caller can hold a PTL. */ + spin_unlock_irqrestore(&pool->lock, flags); + retired = owner_kcalloc(owner, 1, sizeof(*retired)); + if (!retired) + return -ENOMEM; + INIT_WORK(&retired->work, subpage_put_retired_root); + spin_lock_irqsave(&pool->lock, flags); + retired->root = map->root; + } + if (map->root != vma->anon_vma->root) { + get_anon_vma(vma->anon_vma->root); + map->root = vma->anon_vma->root; + } + map->pos = vma_page_offset_at(vma, address); + map->offset = mm_subpage_offset(slot); + } + slot->rmap_bound = true; +unlock: + spin_unlock_irqrestore(&pool->lock, flags); + if (retired) + queue_work(system_unbound_wq, &retired->work); + return ret; +} + +static bool subpage_move_allowed(struct mm_subpage *slot, bool swap) +{ + lockdep_assert_held(&slot->owner->pool->lock); + return !slot->owner->migrating && !slot->pins && + atomic_read(&slot->mapcount) == (swap ? 0 : 1) && + (swap || (slot->rmap_bound && slot->exclusive)); +} + +int mm_subpage_move_prepare(struct mm_subpage_move *move, struct mm_subpage *slot, + struct vm_area_struct *src, unsigned long src_address, + struct vm_area_struct *dst, unsigned long dst_address, bool swap) +{ + struct mm_subpage_owner *owner = slot->owner; + struct mm_subpage_rmap *maps, old; + unsigned long flags; + unsigned int i; + int ret = -EBUSY; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + memset(move, 0, sizeof(*move)); + if (!src->anon_vma || !dst->anon_vma || !owner->anon_root || + mm_page_shift(src->vm_mm) != slot->shift || + mm_page_shift(dst->vm_mm) != slot->shift || + owner->droppable != !!(dst->vm_flags & VM_DROPPABLE)) + return -EINVAL; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!subpage_move_allowed(slot, swap) || + (slot->rmap_bound && !rmap_matches(slot_rmap(slot), src, src_address))) + goto unlock; + spin_unlock_irqrestore(&owner->pool->lock, flags); + + /* Allocate before touching either PTE. The folio lock serializes writers. */ + if (!owner->rmap) { + maps = owner_kcalloc(owner, PAGE_SIZE >> owner->shift, sizeof(*maps)); + if (!maps) + return -ENOMEM; + spin_lock_irqsave(&owner->pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + maps[i].offset = i << owner->shift; + if (!owner->slots[i].rmap_bound) + continue; + maps[i] = slot_rmap(&owner->slots[i]); + get_anon_vma(maps[i].root); + } + owner->rmap = maps; + spin_unlock_irqrestore(&owner->pool->lock, flags); + } + old = owner->rmap[slot->index]; + move->root_changed = old.root != dst->anon_vma->root; + if (move->root_changed && old.root) { + move->retired = owner_kcalloc(owner, 1, sizeof(*move->retired)); + if (!move->retired) + return -ENOMEM; + INIT_WORK(&move->retired->work, subpage_put_retired_root); + move->retired->root = old.root; + } + if (move->root_changed) + get_anon_vma(dst->anon_vma->root); + move->slot = slot; + move->swap = swap; + move->map = (struct mm_subpage_rmap) { + .root = dst->anon_vma->root, + .pos = vma_page_offset_at(dst, dst_address), + .offset = mm_subpage_offset(slot), + }; + return 0; +unlock: + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +int mm_subpage_move_commit(struct mm_subpage_move *move) +{ + struct mm_subpage *slot = move->slot; + struct mm_subpage_owner *owner = slot->owner; + unsigned long flags; + int ret = -EBUSY; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + spin_lock_irqsave(&owner->pool->lock, flags); + /* Recheck pins after source-PTE clearing and the matching TLB barrier. */ + if (subpage_move_allowed(slot, move->swap)) { + owner->rmap[slot->index] = move->map; + slot->rmap_bound = true; + move->committed = true; + ret = 0; + } + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +void mm_subpage_move_finish(struct mm_subpage_move *move) +{ + if (move->committed) { + if (move->retired) + queue_work(system_unbound_wq, &move->retired->work); + } else { + /* The locked destination VMA still holds this root. */ + if (move->root_changed) + put_anon_vma(move->map.root); + kfree(move->retired); + } +} + +int mm_subpage_snapshot_rmap(struct folio *folio, struct mm_subpage_rmap *maps) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner; + unsigned long flags; + unsigned int i; + int nr = -1; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return nr; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->detached && owner->rmap) { + nr = 0; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + if (!owner->slots[i].rmap_bound) + continue; + maps[nr] = owner->rmap[i]; + get_anon_vma(maps[nr].root); + nr++; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); +out: + page_ext_put(ext); + return nr; +} + +/* A referenced folio plus a stable VMA suffices; no folio lock required. */ +bool mm_subpage_address_in_vma(const struct folio *folio, + const struct vm_area_struct *vma, unsigned long *address) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner; + unsigned long flags; + unsigned int i; + bool nonlinear = false; + + if (!ext) + return false; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->detached && owner->rmap) { + nonlinear = true; + *address = -EFAULT; + for (i = 0; vma->anon_vma && i < (PAGE_SIZE >> owner->shift); i++) { + if (!owner->slots[i].rmap_bound || + owner->rmap[i].root != vma->anon_vma->root) + continue; + *address = vma_address_at_offset(vma, owner->rmap[i].pos); + if (*address != -EFAULT) + break; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); +out: + page_ext_put(ext); + return nonlinear; +} + +bool mm_subpage_is_droppable(const struct mm_subpage *subpage) +{ + return subpage->owner->droppable; +} + +void mm_subpage_pool_close(struct mm_subpage_pool *pool) +{ + struct mm_subpage_owner *owner; + unsigned long flags; + + spin_lock_irqsave(&pool->lock, flags); + pool->closed = true; + /* At most one newly supplied owner has no issued slots. Close is O(1). */ + owner = pool->unissued; + if (owner) + owner_remove(owner); + spin_unlock_irqrestore(&pool->lock, flags); + if (owner) + owner_free(owner, true); +} + +static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, + unsigned long eligible) +{ + struct mm_subpage_owner *owner; + + lockdep_assert_held(&pool->lock); + list_for_each_entry(owner, &pool->available, available) + /* A nonpresent PTE may still need any of this folio's slot bytes. */ + if (!owner->migrating && (~owner->used & eligible) && + !folio_test_swapcache(owner->folio)) + return owner; + return NULL; +} + +static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + unsigned long eligible, gfp_t gfp) +{ + struct mm_subpage_owner *owner; + unsigned long flags; + int ret = 0; + + /* Only an unexposed order-0 allocation can become a slot owner. */ + if (folio_order(folio) || folio_ref_count(folio) != 1 || + folio->mapping || folio_mapped(folio) || folio_test_lru(folio) || + folio_test_private(folio) || folio_test_swapcache(folio)) + return -EINVAL; + owner = kzalloc(sizeof(*owner), gfp | __GFP_ACCOUNT); + if (!owner) + return -ENOMEM; + owner_init(owner, pool, folio); + /* Reject duplicate ownership before touching any existing data. */ + ret = owner_bind(owner); + if (ret) { + kfree(owner); + return ret; + } + /* Initialise even unissued bytes before any future whole-folio operation. */ + clear_user_subpage_range(&folio->page, 0, PAGE_SIZE); + __folio_mark_uptodate(folio); + owner->zeroed = pool->all_slots; + spin_lock_irqsave(&pool->lock, flags); + if (pool->closed) + ret = -ESHUTDOWN; + else if (find_available(pool, eligible)) + /* Another allocator supplied backing while we allocated metadata. */ + ret = -EEXIST; + else { + mm_subpage_pool_get(pool); + list_add_tail(&owner->all, &pool->owners); + list_add_tail(&owner->available, &pool->available); + pool->backing_pages++; + pool->unissued = owner; + } + spin_unlock_irqrestore(&pool->lock, flags); + if (ret) { + owner_unbind(owner); + kfree_rcu(owner, rcu); + } + return ret; +} + +int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + gfp_t gfp) +{ + return pool_add_folio(pool, folio, pool->all_slots, gfp); +} + +int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int offset, gfp_t gfp) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return -EINVAL; + return pool_add_folio(pool, folio, eligible, gfp); +} + +unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) +{ + unsigned long pages, flags; + + spin_lock_irqsave(&pool->lock, flags); + pages = pool->backing_pages; + spin_unlock_irqrestore(&pool->lock, flags); + return pages; +} + +static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, + unsigned long eligible, bool *needs_zero) +{ + struct mm_subpage_owner *owner; + struct mm_subpage *subpage; + unsigned long flags; + unsigned int index; + + spin_lock_irqsave(&pool->lock, flags); + if (pool->closed) { + subpage = ERR_PTR(-ESHUTDOWN); + goto out; + } + owner = find_available(pool, eligible); + if (!owner) { + subpage = ERR_PTR(-EAGAIN); + goto out; + } + if (pool->unissued == owner) + /* Transfer the supplied reference to the first reservation. */ + pool->unissued = NULL; + else + folio_get(owner->folio); + index = __ffs(~owner->used & eligible); + __set_bit(index, &owner->used); + if (needs_zero) + *needs_zero = !test_bit(index, &owner->zeroed); + __clear_bit(index, &owner->zeroed); + if (owner->used == pool->all_slots) + list_del_init(&owner->available); + subpage = &owner->slots[index]; + WRITE_ONCE(subpage->lazyfree, false); + /* The used bit keeps backing alive while data is initialised. */ + VM_BUG_ON(refcount_read(&subpage->refs)); + VM_BUG_ON(atomic_read(&subpage->mapcount)); +out: + spin_unlock_irqrestore(&pool->lock, flags); + return subpage; +} + +static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible) +{ + bool needs_zero; + struct mm_subpage *subpage = subpage_reserve(pool, eligible, &needs_zero); + + if (IS_ERR(subpage)) + return subpage; + /* Fresh supply was already zeroed as a whole; reused slots must be cleared. */ + if (needs_zero) + clear_user_subpage_range(&subpage->owner->folio->page, + mm_subpage_offset(subpage), mm_subpage_size(subpage)); + /* Data stores must precede publication in a PTE or another CPU's reference. */ + refcount_set_release(&subpage->refs, 1); + return subpage; +} + +struct mm_subpage *mm_subpage_alloc(struct mm_subpage_pool *pool) +{ + return subpage_alloc(pool, pool->all_slots); +} + +struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return ERR_PTR(-EINVAL); + return subpage_alloc(pool, eligible); +} + +static void subpage_release_locked(struct mm_subpage *subpage, unsigned long flags) +{ + struct mm_subpage_owner *owner = subpage->owner; + struct mm_subpage_pool *pool = owner->pool; + struct folio *folio = owner->folio; + bool release; + + lockdep_assert_held(&pool->lock); + VM_BUG_ON(refcount_read(&subpage->refs)); + VM_BUG_ON(atomic_read(&subpage->mapcount)); + VM_BUG_ON(subpage->pins); + WRITE_ONCE(subpage->exclusive, false); + if (!owner->migrating) { + WRITE_ONCE(subpage->lazyfree, false); + subpage->rmap_bound = false; + } + __clear_bit(subpage->index, &owner->used); + release = !owner->used && !owner->migrating; + if (release) + owner_remove(owner); + else if (!pool->closed && !owner->migrating && list_empty(&owner->available)) + list_add(&owner->available, &pool->available); + spin_unlock_irqrestore(&pool->lock, flags); + if (release) + owner_free(owner, true); + else + folio_put(folio); +} + +static void subpage_release(struct mm_subpage *subpage) +{ + unsigned long flags; + + spin_lock_irqsave(&subpage->owner->pool->lock, flags); + subpage_release_locked(subpage, flags); +} + +static struct mm_subpage *subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned long eligible) +{ + struct mm_subpage *dest; + int err; + + if (pool->shift != mm_subpage_shift(source)) + return ERR_PTR(-EINVAL); + dest = subpage_reserve(pool, eligible, NULL); + + if (IS_ERR(dest)) + return dest; + err = copy_user_subpage_range(&dest->owner->folio->page, mm_subpage_offset(dest), + &source->owner->folio->page, mm_subpage_offset(source), + mm_subpage_size(source)); + if (err) { + /* Never publish a reservation whose copy did not finish. */ + subpage_release(dest); + return ERR_PTR(err); + } + refcount_set_release(&dest->refs, 1); + return dest; +} + +struct mm_subpage *mm_subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source) +{ + return subpage_copy(pool, source, pool->all_slots); +} + +struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return ERR_PTR(-EINVAL); + return subpage_copy(pool, source, eligible); +} + +/* The page-extension RCU section also protects the pool until its lock is held. */ +static struct mm_subpage *restore_bound_slot(struct folio *folio, unsigned int offset, + unsigned int shift) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_pool *pool; + struct mm_subpage_ext *subext; + struct mm_subpage *slot = NULL; + unsigned int index; + unsigned long flags; + + if (!ext) + return ERR_PTR(-EOPNOTSUPP); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + if ((shift && shift != owner->shift) || !offset_mask(owner->shift, offset)) { + slot = ERR_PTR(-EINVAL); + goto out; + } + index = offset >> owner->shift; + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->detached) { + slot = ERR_PTR(-EAGAIN); + goto unlock; + } + slot = &owner->slots[index]; + if (test_bit(index, &owner->used)) { + folio_get(folio); + if (!refcount_inc_not_zero_acquire(&slot->refs)) { + /* Its final put or initial publication has not finished. */ + folio_put(folio); + slot = ERR_PTR(-EAGAIN); + } + goto unlock; + } + VM_BUG_ON(refcount_read(&slot->refs) || atomic_read(&slot->mapcount)); + if (pool->unissued == owner) + pool->unissued = NULL; + else + folio_get(folio); + __set_bit(index, &owner->used); + __clear_bit(index, &owner->zeroed); + if (owner->used == pool->all_slots) + list_del_init(&owner->available); + /* The folio lock and uptodate state stabilise the restored bytes. */ + refcount_set_release(&slot->refs, 1); +unlock: + spin_unlock_irqrestore(&pool->lock, flags); +out: + page_ext_put(ext); + return slot; +} + +struct mm_subpage *mm_subpage_restore_granule(struct folio *folio, unsigned int offset, + unsigned int shift, gfp_t gfp) +{ + struct mm_subpage_pool *pool; + struct mm_subpage_owner *owner; + struct mm_subpage *slot; + unsigned int index; + int err; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!subpage_shift_valid(shift) || !offset_mask(shift, offset) || folio_order(folio) || !folio_test_swapcache(folio) || + !folio_test_uptodate(folio) || folio_test_ksm(folio) || + (folio->mapping && !folio_test_anon(folio)) || + (folio_test_anon(folio) && PageAnonExclusive(&folio->page))) + return ERR_PTR(-EINVAL); + index = offset >> shift; + slot = restore_bound_slot(folio, offset, shift); + if (slot) + return slot; + + /* A reclaimed folio has no surviving owner; do not initialise its data. */ + pool = mm_subpage_pool_create_granule(gfp, shift); + if (!pool) + return ERR_PTR(-ENOMEM); + pool->closed = true; + owner = kzalloc(sizeof(*owner), gfp | __GFP_ACCOUNT); + if (!owner) { + mm_subpage_pool_put(pool); + return ERR_PTR(-ENOMEM); + } + owner_init(owner, pool, folio); + owner->used = BIT(index); + refcount_set(&owner->slots[index].refs, 1); + list_add(&owner->all, &pool->owners); + pool->backing_pages = 1; + folio_get(folio); + err = owner_bind(owner); + if (err) { + folio_put(folio); + list_del(&owner->all); + pool->backing_pages = 0; + kfree(owner); + mm_subpage_pool_put(pool); + return ERR_PTR(err == -EEXIST ? -EAGAIN : err); + } + /* The private closed pool's initial reference belongs to this owner. */ + return &owner->slots[index]; +} + +struct mm_subpage *mm_subpage_restore(struct folio *folio, unsigned int offset, gfp_t gfp) +{ + return mm_subpage_restore_granule(folio, offset, MM_SUBPAGE_SHIFT, gfp); +} + +/* Preserve the metadata charge when migration runs outside its owning task. */ +static struct mm_subpage_owner *owner_alloc_migration(struct mm_subpage_owner *owner) +{ + struct mem_cgroup *memcg, *saved; + struct mm_subpage_owner *new; + + rcu_read_lock(); + do { + memcg = mem_cgroup_from_virt(owner) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + /* Folio locks are held: do not recurse into reclaim or trigger memcg OOM. */ + new = kzalloc(sizeof(*new), GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + if (new && owner->rmap) { + new->rmap = kcalloc(PAGE_SIZE >> owner->shift, sizeof(*new->rmap), + GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + if (!new->rmap) { + kfree(new); + new = NULL; + } + } + set_active_memcg(saved); + mem_cgroup_put(memcg); + return new; +} + +/* A migration hold pins metadata, not the native folio: the driver pins it. */ +int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) +{ + struct page_ext *ext; + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner, *new; + struct mm_subpage_pool *pool; + unsigned long flags; + int ret = 0; + unsigned int i; + + VM_BUG_ON_FOLIO(!folio_test_locked(src), src); + VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst); + ext = page_ext_get(&src->page); + if (!ext) + return 0; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) { + page_ext_put(ext); + return 0; + } + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->detached) { + owner = NULL; + } else if (src == dst || folio_order(src) || folio_order(dst) || + owner->migrating || pool->unissued == owner) { + ret = -EBUSY; + } else { + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (owner->slots[i].pins) + ret = -EBUSY; + if (!ret) + owner->migrating = true; + } + spin_unlock_irqrestore(&pool->lock, flags); + page_ext_put(ext); + if (!owner || ret) + return ret; + + /* Async migration must not recurse into reclaim while holding folio locks. */ + new = owner_alloc_migration(owner); + if (!new) { + ret = -ENOMEM; + goto undo; + } + owner_init(new, pool, dst); + new->migrating = true; + new->zeroed = owner->zeroed; + spin_lock_irqsave(&pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + new->slots[i].rmap_bound = owner->slots[i].rmap_bound; + if (new->rmap) { + new->rmap[i] = owner->rmap[i]; + if (new->rmap[i].root) + get_anon_vma(new->rmap[i].root); + } + } + spin_unlock_irqrestore(&pool->lock, flags); + + new->droppable = owner->droppable; + new->anon_root = owner->anon_root; + if (new->anon_root) + get_anon_vma(new->anon_root); + mm_subpage_pool_get(pool); + spin_lock_irqsave(&pool->lock, flags); + list_add_tail(&new->all, &pool->owners); + pool->backing_pages++; + ret = owner_bind(new); + if (ret) + owner_remove(new); + else + owner->migration_target = new; + spin_unlock_irqrestore(&pool->lock, flags); + if (!ret) + return 0; + if (new->anon_root) + put_anon_vma(new->anon_root); + owner_free_rmap(new); + kfree(new); + mm_subpage_pool_put(pool); +undo: + mm_subpage_migrate_finish(src); + return ret; +} + +bool mm_subpage_migrating(struct folio *folio) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + bool active = false; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return false; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (owner) + active = READ_ONCE(owner->migrating); + page_ext_put(ext); + return active; +} + +/* Source folio/PTE locked: writable GETs cannot race this final snapshot. */ +void mm_subpage_migrate_save_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + + VM_BUG_ON_FOLIO(!owner->migrating || !owner->migration_target, owner->folio); + WRITE_ONCE(owner->migration_target->slots[subpage->index].lazyfree, + READ_ONCE(subpage->lazyfree)); +} + +struct mm_subpage *mm_subpage_migrate_restore(struct folio *folio, unsigned int offset) +{ + struct mm_subpage *slot; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(!offset_mask(MM_SUBPAGE_SHIFT, offset) || !mm_subpage_migrating(folio), folio); + /* No allocation or data initialisation can fail during migration recovery. */ + slot = restore_bound_slot(folio, offset, 0); + VM_BUG_ON_FOLIO(IS_ERR_OR_NULL(slot), folio); + return slot; +} + +void mm_subpage_migrate_finish(struct folio *folio) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + struct mm_subpage_pool *pool; + unsigned long flags; + unsigned int i; + bool release = false; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->migrating) { + VM_BUG_ON(owner->detached); + owner->migrating = false; + owner->migration_target = NULL; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (!(owner->used & BIT(i))) + owner->slots[i].rmap_bound = false; + release = !owner->used; + if (release) + owner_remove(owner); + else if (!pool->closed && owner->used != pool->all_slots && + list_empty(&owner->available)) + list_add(&owner->available, &pool->available); + } + spin_unlock_irqrestore(&pool->lock, flags); +out: + page_ext_put(ext); + if (release) + /* No slot references remain; only the migration driver pins backing. */ + owner_free(owner, false); +} + +void mm_subpage_get(struct mm_subpage *subpage) +{ + folio_get(subpage->owner->folio); + refcount_inc(&subpage->refs); +} + +void mm_subpage_put(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + struct mm_subpage_pool *pool = owner->pool; + struct folio *folio = owner->folio; + unsigned long flags; + + /* + * Snapshot the backing while our slot reference still retains owner. + * A non-final decrement can race the final put and metadata reclamation; + * only our physical folio reference survives that decrement. + */ + if (refcount_dec_and_lock_irqsave(&subpage->refs, &pool->lock, &flags)) + subpage_release_locked(subpage, flags); + else + folio_put(folio); +} + +void mm_subpage_set_exclusive(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + VM_BUG_ON(atomic_read(&subpage->mapcount) > 1 || !refcount_read(&subpage->refs)); + WRITE_ONCE(subpage->exclusive, true); + spin_unlock_irqrestore(&pool->lock, flags); +} + +bool mm_subpage_is_exclusive(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->exclusive); +} + +static bool subpage_try_reuse(struct mm_subpage *subpage, unsigned int references) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool reusable; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + reusable = atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == references && !subpage->pins; + if (reusable) + WRITE_ONCE(subpage->exclusive, true); + spin_unlock_irqrestore(&pool->lock, flags); + return reusable; +} + +bool mm_subpage_try_reuse_swap(struct mm_subpage *subpage) +{ + return subpage_try_reuse(subpage, 1); +} + +bool mm_subpage_try_reuse_anon(struct mm_subpage *subpage) +{ + return subpage_try_reuse(subpage, 2); +} + +bool mm_subpage_can_zap(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + spin_lock_irqsave(&pool->lock, flags); + eligible = !subpage->owner->migrating && !subpage->pins && + atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == 2; + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +bool mm_subpage_try_share(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool shared; + + spin_lock_irqsave(&pool->lock, flags); + shared = !subpage->pins; + if (shared) + WRITE_ONCE(subpage->exclusive, false); + spin_unlock_irqrestore(&pool->lock, flags); + return shared; +} + +bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->lazyfree); +} + +void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage) +{ + WRITE_ONCE(subpage->lazyfree, false); +} + +bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + eligible = atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == 2 && !subpage->pins; + if (eligible) + WRITE_ONCE(subpage->lazyfree, true); + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +bool mm_subpage_all_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + unsigned int i; + bool all = true; + + spin_lock_irqsave(&owner->pool->lock, flags); + for (i = 0; i < SUBPAGES_PER_FOLIO; i++) { + if (!(owner->used & BIT(i))) + continue; + if (!READ_ONCE(owner->slots[i].lazyfree)) { + all = false; + break; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); + return all; +} + +bool mm_subpage_discardable(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + spin_lock_irqsave(&pool->lock, flags); + eligible = (subpage->owner->droppable || READ_ONCE(subpage->lazyfree)) && + !subpage->pins && + refcount_read(&subpage->refs) == atomic_read(&subpage->mapcount) + 1; + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +int mm_subpage_pin(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + int ret = -EAGAIN; + + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->anon_root || owner->migrating || !subpage->exclusive || + !atomic_read(&subpage->mapcount)) + goto out; + ret = try_grab_folio(owner->folio, 1, FOLL_PIN); + if (ret) + goto out; + mm_subpage_get(subpage); + subpage->pins++; +out: + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +void mm_subpage_unpin(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + + spin_lock_irqsave(&owner->pool->lock, flags); + VM_BUG_ON(!subpage->pins || !subpage->exclusive); + subpage->pins--; + spin_unlock_irqrestore(&owner->pool->lock, flags); + unpin_folio(owner->folio); + mm_subpage_put(subpage); +} + +unsigned int mm_subpage_pincount(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->pins); +} + +int mm_subpage_mapcount(const struct mm_subpage *subpage) +{ + return atomic_read(&subpage->mapcount); +} + +struct folio *mm_subpage_folio(const struct mm_subpage *subpage) +{ + return subpage->owner->folio; +} + +unsigned int mm_subpage_shift(const struct mm_subpage *subpage) +{ + return subpage->shift; +} + +unsigned int mm_subpage_size(const struct mm_subpage *subpage) +{ + return 1U << subpage->shift; +} + +unsigned int mm_subpage_offset(const struct mm_subpage *subpage) +{ + return subpage->index << subpage->shift; +} + +phys_addr_t mm_subpage_phys(const struct mm_subpage *subpage) +{ + return PFN_PHYS(folio_pfn(subpage->owner->folio)) + mm_subpage_offset(subpage); +} + +/* + * Round the mm's aggregate reservation, never each VMA: splitting a VMA + * must not create extra charges or refunds. mmap writers serialize updates; + * munmap may retain the lock downgraded while finishing its refund. + */ +static int mm_commit_delta(struct mm_struct *mm, long pages, + unsigned long *next, long *delta) +{ + unsigned long old = mm->committed_user_pages; + unsigned int shift = mm_page_shift(mm); + + mmap_assert_locked(mm); + if (pages < 0 && WARN_ON_ONCE(pages < -(long)old)) + return -EINVAL; + if (pages > 0 && (unsigned long)pages > LONG_MAX - old) + return -ENOMEM; + *next = old + pages; + if (shift > PAGE_SHIFT) { + unsigned int extra = shift - PAGE_SHIFT; + + /* A logical reservation may span several native commit units. */ + if (*next > (LONG_MAX >> extra) || old > (LONG_MAX >> extra)) + return -ENOMEM; + *delta = (long)(*next << extra) - (long)(old << extra); + } else { + unsigned long per_native = 1UL << (PAGE_SHIFT - shift); + + *delta = DIV_ROUND_UP(*next, per_native) - DIV_ROUND_UP(old, per_native); + } + return 0; +} + +int mm_account_memory(struct mm_struct *mm, struct mm_struct *policy_mm, long pages) +{ + unsigned long next; + long delta; + int ret; + + if (mm_page_size(mm) == PAGE_SIZE) + return security_vm_enough_memory_mm(policy_mm, pages); + ret = mm_commit_delta(mm, pages, &next, &delta); + if (ret) + return ret; + ret = security_vm_enough_memory_mm(policy_mm, delta); + if (!ret) + mm->committed_user_pages = next; + return ret; +} + +void mm_acct_memory(struct mm_struct *mm, long pages) +{ + unsigned long next; + long delta; + + if (mm_page_size(mm) == PAGE_SIZE) { + vm_acct_memory(pages); + return; + } + if (WARN_ON_ONCE(mm_commit_delta(mm, pages, &next, &delta))) + return; + vm_acct_memory(delta); + mm->committed_user_pages = next; +} + +void mm_unacct_memory(struct mm_struct *mm, unsigned long pages) +{ + mm_acct_memory(mm, -(long)pages); +} + +/* Shared ownership for consumers whose native page spans contain user slots. */ +struct mm_subpage_refs { + refcount_t refs; + unsigned int count; + unsigned int capacity; + struct mm_subpage *slots[]; +}; + +struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs) +{ + if (refs) + refcount_inc(&refs->refs); + return refs; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_get); + +void mm_subpage_refs_put(struct mm_subpage_refs *refs) +{ + unsigned int i; + + if (!refs || !refcount_dec_and_test(&refs->refs)) + return; + for (i = 0; i < refs->count; i++) + mm_subpage_put(refs->slots[i]); + kfree(refs); +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_put); + +static struct mm_subpage_refs *subpage_refs_new(unsigned int capacity, gfp_t gfp) +{ + struct mm_subpage_refs *refs = kmalloc(struct_size(refs, slots, capacity), gfp); + + if (!refs) + return NULL; + refcount_set(&refs->refs, 1); + refs->count = 0; + refs->capacity = capacity; + return refs; +} + +static int subpage_refs_prepare(struct mm_subpage_refs **refsp, + unsigned int needed, gfp_t gfp) +{ + struct mm_subpage_refs *old = *refsp, *refs; + unsigned int i; + + if (old && refcount_read(&old->refs) == 1 && old->capacity >= needed) + return 0; + refs = subpage_refs_new(max(needed, 4U), gfp); + if (!refs) + return -ENOMEM; + if (old) { + for (i = 0; i < old->count; i++) { + mm_subpage_get(old->slots[i]); + refs->slots[refs->count++] = old->slots[i]; + } + } + *refsp = refs; + mm_subpage_refs_put(old); + return 0; +} + +int mm_subpage_refs_add(struct mm_subpage_refs **refsp, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp) +{ + struct mm_subpage *slots[PAGE_SIZE / MM_SUBPAGE_SIZE]; + struct mm_subpage_refs *refs = *refsp; + unsigned int pos, count = 0, i, j; + int ret = 0; + bool found = false; + + if (!len || folio_order(page_folio(page)) || !folio_test_anon(page_folio(page))) + return 0; + if (offset >= PAGE_SIZE || len > PAGE_SIZE - offset) + return -EINVAL; + for (pos = round_down(offset, MM_SUBPAGE_SIZE); pos < offset + len; + pos += MM_SUBPAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(page_to_phys(page) + pos); + + if (!slot) { + /* A native anonymous allocation has no quarter owner. */ + ret = found ? -EFAULT : 0; + goto out; + } + found = true; + for (j = 0; refs && j < refs->count; j++) + if (refs->slots[j] == slot) + break; + if (refs && j < refs->count) { + mm_subpage_put(slot); + continue; + } + for (j = 0; j < count; j++) + if (slots[j] == slot) + break; + if (j < count) + mm_subpage_put(slot); + else + slots[count++] = slot; + } + if (!count) + return 0; + if (refs && refs->count > UINT_MAX - count) { + ret = -ENOMEM; + goto out; + } + ret = subpage_refs_prepare(refsp, (refs ? refs->count : 0) + count, gfp); + if (ret) + goto out; + refs = *refsp; + for (i = 0; i < count; i++) + refs->slots[refs->count++] = slots[i]; + return 0; +out: + for (i = 0; i < count; i++) + mm_subpage_put(slots[i]); + return ret; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_add); + +int mm_subpage_refs_prune(struct mm_subpage_refs **refsp, + bool (*keep)(const struct mm_subpage *, void *), + void *context, gfp_t gfp) +{ + struct mm_subpage_refs *refs = *refsp, *new; + unsigned int i, count = 0; + + if (!refs) + return 0; + for (i = 0; i < refs->count; i++) + count += keep(refs->slots[i], context); + if (count == refs->count) + return 0; + if (!count) { + *refsp = NULL; + mm_subpage_refs_put(refs); + return 0; + } + if (refcount_read(&refs->refs) == 1) { + for (i = 0, count = 0; i < refs->count; i++) { + if (keep(refs->slots[i], context)) + refs->slots[count++] = refs->slots[i]; + else + mm_subpage_put(refs->slots[i]); + } + refs->count = count; + return 0; + } + new = subpage_refs_new(count, gfp); + if (!new) + return -ENOMEM; + for (i = 0; i < refs->count; i++) { + if (!keep(refs->slots[i], context)) + continue; + mm_subpage_get(refs->slots[i]); + new->slots[new->count++] = refs->slots[i]; + } + *refsp = new; + mm_subpage_refs_put(refs); + return 0; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_prune); From c16faad6c36dd65a3cf47cbde2e89b7ac84e46a5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0423/1579] exec: stage arguments and ELF mappings at the selected user page size Resolve the per-thread exec preference before exposing the new address space. Transfer staged arguments across granules, bound temporary copies by their native mappings, and use the selected ABI for ELF and core dumps. Signed-off-by: Diljit Singh --- fs/binfmt_elf.c | 42 ++++++-- fs/coredump.c | 81 ++++++++++++--- fs/exec.c | 265 +++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 313 insertions(+), 75 deletions(-) diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 16a56b6b3f6ca..826235fd977c2 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -79,11 +79,11 @@ static int elf_core_dump(struct coredump_params *cprm); #define elf_core_dump NULL #endif -#if ELF_EXEC_PAGESIZE > PAGE_SIZE -#define ELF_MIN_ALIGN ELF_EXEC_PAGESIZE -#else -#define ELF_MIN_ALIGN PAGE_SIZE +#ifndef ELF_EXEC_PAGESIZE_MM +#define ELF_EXEC_PAGESIZE_MM(mm) ELF_EXEC_PAGESIZE #endif +#define ELF_MIN_ALIGN max_t(unsigned long, mm_page_size(current->mm), \ + ELF_EXEC_PAGESIZE_MM(current->mm)) #ifndef ELF_CORE_EFLAGS #define ELF_CORE_EFLAGS 0 @@ -93,12 +93,20 @@ static int elf_core_dump(struct coredump_params *cprm); #define ELF_PAGEOFFSET(_v) ((_v) & (ELF_MIN_ALIGN-1)) #define ELF_PAGEALIGN(_v) (((_v) + ELF_MIN_ALIGN - 1) & ~(ELF_MIN_ALIGN - 1)) +#ifdef CONFIG_COREDUMP +static unsigned long elf_min_coredump_size(void) +{ + return ELF_EXEC_PAGESIZE_MM(current->mm); +} +#endif + static struct linux_binfmt elf_format = { .module = THIS_MODULE, .load_binary = load_elf_binary, #ifdef CONFIG_COREDUMP .core_dump = elf_core_dump, .min_coredump = ELF_EXEC_PAGESIZE, + .min_coredump_size = elf_min_coredump_size, #endif }; @@ -248,7 +256,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec, ARCH_DLINFO; #endif NEW_AUX_ENT(AT_HWCAP, ELF_HWCAP); - NEW_AUX_ENT(AT_PAGESZ, ELF_EXEC_PAGESIZE); + NEW_AUX_ENT(AT_PAGESZ, ELF_EXEC_PAGESIZE_MM(mm)); NEW_AUX_ENT(AT_CLKTCK, CLOCKS_PER_SEC); NEW_AUX_ENT(AT_PHDR, phdr_addr); NEW_AUX_ENT(AT_PHENT, sizeof(struct elf_phdr)); @@ -622,6 +630,13 @@ static inline int arch_check_elf(struct elfhdr *ehdr, bool has_interp, #endif /* !CONFIG_ARCH_BINFMT_ELF_STATE */ +#ifndef arch_bprm_prepare_mm +static inline int arch_bprm_prepare_mm(struct linux_binprm *bprm, bool compat) +{ + return 0; +} +#endif + static inline int make_prot(u32 p_flags, struct arch_elf_state *arch_state, bool has_interp, bool is_interp) { @@ -1006,6 +1021,11 @@ static int load_elf_binary(struct linux_binprm *bprm) if (retval) goto out_free_dentry; + retval = arch_bprm_prepare_mm(bprm, + elf_ex->e_ident[EI_CLASS] == ELFCLASS32); + if (retval) + goto out_free_dentry; + /* Flush all traces of the currently running executable */ retval = begin_new_exec(bprm); if (retval) @@ -1335,7 +1355,7 @@ static int load_elf_binary(struct linux_binprm *bprm) * leave a gap between .bss and brk. */ if (!brk_moved) - mm->brk = mm->start_brk = mm->brk + PAGE_SIZE; + mm->brk = mm->start_brk = mm->brk + mm_page_size(mm); mm->brk = mm->start_brk = arch_randomize_brk(mm); brk_moved = true; @@ -1351,10 +1371,10 @@ static int load_elf_binary(struct linux_binprm *bprm) and some applications "depend" upon this behavior. Since we do not have the power to recompile these, we emulate the SVr4 behavior. Sigh. */ - error = vm_mmap(NULL, 0, PAGE_SIZE, PROT_READ | PROT_EXEC, + error = vm_mmap(NULL, 0, mm_page_size(mm), PROT_READ | PROT_EXEC, MAP_FIXED | MAP_PRIVATE, 0); - retval = do_mseal(0, PAGE_SIZE, 0); + retval = do_mseal(0, mm_page_size(mm), 0); if (retval) pr_warn_ratelimited("pid=%d, couldn't seal address 0, ret=%d.\n", task_pid_nr(current), retval); @@ -1658,7 +1678,7 @@ static int fill_files_note(struct memelfnote *note, struct coredump_params *cprm /* Now we know exact count of files, can store it */ data[0] = count; - data[1] = PAGE_SIZE; + data[1] = mm_page_size(current->mm); /* * Count usually is less than mm->map_count, * we need to move filenames down. @@ -2043,7 +2063,7 @@ static int elf_core_dump(struct coredump_params *cprm) offset += sz; } - dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE); + dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE_MM(current->mm)); offset += cprm->vma_data_size; offset += elf_core_extra_data_size(cprm); @@ -2083,7 +2103,7 @@ static int elf_core_dump(struct coredump_params *cprm) phdr.p_flags |= PF_W; if (meta->flags & VM_EXEC) phdr.p_flags |= PF_X; - phdr.p_align = ELF_EXEC_PAGESIZE; + phdr.p_align = ELF_EXEC_PAGESIZE_MM(current->mm); if (!dump_emit(cprm, &phdr, sizeof(phdr))) goto end_coredump; diff --git a/fs/coredump.c b/fs/coredump.c index bb6fdb1f458e9..6ab5f2e076698 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -884,7 +885,8 @@ static bool coredump_file(struct core_name *cn, struct coredump_params *cprm, struct file *file __free(fput) = NULL; int open_flags = O_CREAT | O_WRONLY | O_NOFOLLOW | O_LARGEFILE | O_EXCL; - if (cprm->limit < binfmt->min_coredump) + if (cprm->limit < (binfmt->min_coredump_size ? + binfmt->min_coredump_size() : binfmt->min_coredump)) return false; if (coredump_force_suid_safe(cprm) && cn->corename[0] != '/') { @@ -1278,7 +1280,8 @@ void dump_skip(struct coredump_params *cprm, size_t nr) EXPORT_SYMBOL(dump_skip); #ifdef CONFIG_ELF_CORE -static int dump_emit_page(struct coredump_params *cprm, struct page *page) +static int dump_emit_page(struct coredump_params *cprm, struct page *page, + unsigned int offset, unsigned int size) { struct bio_vec bvec; struct iov_iter iter; @@ -1294,19 +1297,19 @@ static int dump_emit_page(struct coredump_params *cprm, struct page *page) return 0; cprm->to_skip = 0; } - if (cprm->written + PAGE_SIZE > cprm->limit) + if (cprm->written + size > cprm->limit) return 0; if (dump_interrupted()) return 0; pos = file->f_pos; - bvec_set_page(&bvec, page, PAGE_SIZE, 0); - iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, PAGE_SIZE); + bvec_set_page(&bvec, page, size, offset); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size); n = __kernel_write_iter(cprm->file, &iter, &pos); - if (n != PAGE_SIZE) + if (n != size) return 0; file->f_pos = pos; - cprm->written += PAGE_SIZE; - cprm->pos += PAGE_SIZE; + cprm->written += size; + cprm->pos += size; return 1; } @@ -1321,10 +1324,11 @@ static int dump_emit_page(struct coredump_params *cprm, struct page *page) #define dump_page_alloc() alloc_page(GFP_KERNEL) #define dump_page_free(x) __free_page(x) -static struct page *dump_page_copy(struct page *src, struct page *dst) +static struct page *dump_page_copy(struct page *src, struct page *dst, + unsigned int offset, unsigned int size) { void *buf = kmap_local_page(src); - size_t left = copy_mc_to_kernel(page_address(dst), buf, PAGE_SIZE); + size_t left = copy_mc_to_kernel(page_address(dst) + offset, buf + offset, size); kunmap_local(buf); return left ? NULL : dst; } @@ -1334,12 +1338,51 @@ static struct page *dump_page_copy(struct page *src, struct page *dst) /* We just want to return non-NULL; it's never used. */ #define dump_page_alloc() ERR_PTR(-EINVAL) #define dump_page_free(x) ((void)(x)) -static inline struct page *dump_page_copy(struct page *src, struct page *dst) +static inline struct page *dump_page_copy(struct page *src, struct page *dst, + unsigned int offset, unsigned int size) { return src; } #endif +#ifdef CONFIG_MM_SUBPAGE +static int dump_subpage_range(struct coredump_params *cprm, unsigned long start, + unsigned long len) +{ + unsigned long size = mm_user_fragment_size(current->mm), addr; + struct page *scratch = dump_page_alloc(); + int ret = 0; + + if (!scratch) + return 0; + for (addr = start; addr < start + len; addr += size) { + struct user_page_fragment fragment; + + if (get_dump_fragment(addr, &fragment) == 1) { + struct page *page = folio_page(fragment.folio, + fragment.offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(fragment.offset); + int emitted; + + emitted = dump_emit_page(cprm, + dump_page_copy(page, scratch, offset, size), offset, size); + release_user_fragments(&fragment, 1, false); + if (!emitted) + goto out; + } else { + dump_skip(cprm, size); + } + if (dump_interrupted()) + goto out; + cond_resched(); + } + ret = 1; +out: + dump_page_free(scratch); + return ret; +} +#endif + int dump_user_range(struct coredump_params *cprm, unsigned long start, unsigned long len) { @@ -1347,6 +1390,10 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start, struct page *dump_page; int locked, ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return dump_subpage_range(cprm, start, len); +#endif dump_page = dump_page_alloc(); if (!dump_page) return 0; @@ -1375,7 +1422,8 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start, mmap_read_unlock(current->mm); locked = 0; } - int stop = !dump_emit_page(cprm, dump_page_copy(page, dump_page)); + int stop = !dump_emit_page(cprm, + dump_page_copy(page, dump_page, 0, PAGE_SIZE), 0, PAGE_SIZE); put_page(page); if (stop) goto out; @@ -1652,9 +1700,10 @@ static unsigned long vma_dump_size(struct vm_area_struct *vma, * dump the first page to aid in determining what was mapped here. */ if (FILTER(ELF_HEADERS) && - vma->vm_pgoff == 0 && (vma->vm_flags & VM_READ)) { + vma->vm_pgoff == 0 && !vma_subpage_offset(vma) && + (vma->vm_flags & VM_READ)) { if ((READ_ONCE(file_inode(vma->vm_file)->i_mode) & 0111) != 0) - return PAGE_SIZE; + return min(mm_page_size(vma->vm_mm), vma->vm_end - vma->vm_start); /* * ELF libraries aren't always executable. @@ -1754,7 +1803,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) m->end = vma->vm_end; m->flags = vma->vm_flags; m->dump_size = vma_dump_size(vma, cprm->mm_flags); - m->pgoff = vma->vm_pgoff; + m->pgoff = core_vma_pgoff(vma); m->file = vma->vm_file; if (m->file) get_file(m->file); @@ -1773,7 +1822,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) memcmp(elfmag, ELFMAG, SELFMAG) != 0) { m->dump_size = 0; } else { - m->dump_size = PAGE_SIZE; + m->dump_size = min(mm_page_size(mm), m->end - m->start); } } diff --git a/fs/exec.c b/fs/exec.c index 2eacc1441c59f..cb5cda32f8e30 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -122,6 +123,54 @@ bool path_noexec(const struct path *path) (path->mnt->mnt_sb->s_iflags & SB_I_NOEXEC); } +/* A temporary argument mapping owns one native page or an exact subpage slot. */ +struct arg_page { + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *subpage; + unsigned int offset; +#endif +}; + +static unsigned long bprm_page_size(const struct linux_binprm *bprm) +{ +#ifdef CONFIG_MMU + return mm_page_size(bprm->mm); +#else + return PAGE_SIZE; +#endif +} + +/* kmap_local_page() maps one native page, even for a larger user leaf. */ +static unsigned long bprm_copy_size(const struct linux_binprm *bprm) +{ + return min(bprm_page_size(bprm), PAGE_SIZE); +} + +static unsigned long bprm_copy_mask(const struct linux_binprm *bprm) +{ + return ~(bprm_copy_size(bprm) - 1); +} + +static unsigned long bprm_max_arg_strlen(const struct linux_binprm *bprm) +{ + return (MAX_ARG_STRLEN / PAGE_SIZE) * bprm_page_size(bprm); +} + +static unsigned int arg_page_offset(struct arg_page page) +{ +#ifdef CONFIG_MM_SUBPAGE + return page.offset; +#else + return 0; +#endif +} + +static void *map_arg_page(struct arg_page page) +{ + return kmap_local_page(page.page) + arg_page_offset(page); +} + #ifdef CONFIG_MMU /* * The nascent bprm->mm is not visible until exec_mmap() but it can @@ -141,10 +190,10 @@ static void acct_arg_size(struct linux_binprm *bprm, unsigned long pages) add_mm_counter(mm, MM_ANONPAGES, diff); } -static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, +static struct arg_page get_arg_page(struct linux_binprm *bprm, unsigned long pos, int write) { - struct page *page; + struct arg_page page = {}; struct vm_area_struct *vma = bprm->vma; struct mm_struct *mm = bprm->mm; int ret; @@ -155,28 +204,47 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, * ahead of time. */ if (!mmap_read_lock_maybe_expand(mm, vma, pos, write)) - return NULL; + return page; /* * We are doing an exec(). 'current' is the process * doing the exec and 'mm' is the new process's mm. */ - ret = get_user_pages_remote(mm, pos, 1, - write ? FOLL_WRITE : 0, - &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + struct vm_area_struct *area; + + ret = get_user_fragment_vma_remote(mm, pos, 1, write ? FOLL_WRITE : 0, + &fragment, &area); + if (ret > 0) { + page.page = folio_page(fragment.folio, fragment.offset >> PAGE_SHIFT); + page.subpage = fragment.subpage; + page.offset = offset_in_page(fragment.offset) & mm_page_mask(mm); + } + } else +#endif + ret = get_user_pages_remote(mm, pos, 1, + write ? FOLL_WRITE : 0, &page.page, NULL); mmap_read_unlock(mm); if (ret <= 0) - return NULL; + return page; if (write) - acct_arg_size(bprm, vma_pages(vma)); + acct_arg_size(bprm, mm_pages_for_reader(mm, current->mm, vma_user_pages(vma))); return page; } -static void put_arg_page(struct page *page) +static void put_arg_page(struct arg_page page) { - put_page(page); +#ifdef CONFIG_MM_SUBPAGE + if (page.subpage) { + mm_subpage_put(page.subpage); + return; + } +#endif + put_page(page.page); } static void free_arg_pages(struct linux_binprm *bprm) @@ -184,14 +252,14 @@ static void free_arg_pages(struct linux_binprm *bprm) } static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos, - struct page *page) + struct arg_page page) { - flush_cache_page(bprm->vma, pos, page_to_pfn(page)); + flush_cache_page(bprm->vma, pos, page_to_pfn(page.page)); } static bool valid_arg_len(struct linux_binprm *bprm, long len) { - return len <= MAX_ARG_STRLEN; + return len <= bprm_max_arg_strlen(bprm); } #else @@ -200,7 +268,7 @@ static inline void acct_arg_size(struct linux_binprm *bprm, unsigned long pages) { } -static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, +static struct arg_page get_arg_page(struct linux_binprm *bprm, unsigned long pos, int write) { struct page *page; @@ -209,14 +277,14 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, if (!page && write) { page = alloc_page(GFP_HIGHUSER|__GFP_ZERO); if (!page) - return NULL; + return (struct arg_page) {}; bprm->page[pos / PAGE_SIZE] = page; } - return page; + return (struct arg_page) { .page = page }; } -static void put_arg_page(struct page *page) +static void put_arg_page(struct arg_page page) { } @@ -237,7 +305,7 @@ static void free_arg_pages(struct linux_binprm *bprm) } static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos, - struct page *page) + struct arg_page page) { } @@ -259,7 +327,7 @@ static int bprm_mm_init(struct linux_binprm *bprm) int err; struct mm_struct *mm = NULL; - bprm->mm = mm = mm_alloc(); + bprm->mm = mm = mm_alloc_exec(); err = -ENOMEM; if (!mm) goto err; @@ -288,6 +356,102 @@ static int bprm_mm_init(struct linux_binprm *bprm) return err; } +#ifdef CONFIG_MM_SUBPAGE +/* + * Final ELF selection may change the granule after argv, envp and script + * arguments have been staged. Table geometry is immutable: build a second mm + * and transfer bytes at the same temporary stack addresses before committing + * exec. Until success, the original bprm and its argument charge remain intact. + */ +int bprm_set_page_shift(struct linux_binprm *bprm, unsigned int page_shift) +{ + struct linux_binprm replacement = {}; + struct mm_struct *old_mm; + unsigned long pos, end, new_top, string_len = 0; + unsigned long old_size = mm_user_fragment_size(bprm->mm), new_size; + int ret; + + if (mm_page_shift(bprm->mm) == page_shift) + return 0; + if (WARN_ON_ONCE(bprm->point_of_no_return)) + return -EINVAL; + + replacement.mm = mm_alloc_exec_page_shift(page_shift); + if (!replacement.mm) + return -ENOMEM; + ret = -EINVAL; + if (mm_page_shift(replacement.mm) != page_shift) + goto out; + new_size = mm_user_fragment_size(replacement.mm); + ret = create_init_stack_vma(replacement.mm, &replacement.vma, &new_top); + if (ret) + goto out; + end = bprm->vma->vm_end - sizeof(void *); + ret = -EINVAL; + if (new_top != end || bprm->p > end) + goto out; + + for (pos = bprm->p; pos < end;) { + struct arg_page src, dst; + unsigned long src_off = pos & (old_size - 1); + unsigned long dst_off = pos & (new_size - 1); + unsigned long len = min3(end - pos, old_size - src_off, + new_size - dst_off); + const char *from; + char *to; + + ret = -ERESTARTNOHAND; + if (fatal_signal_pending(current)) + goto out; + src = get_arg_page(bprm, pos, 0); + ret = -EFAULT; + if (!src.page) + goto out; + dst = get_arg_page(&replacement, pos, 1); + if (!dst.page) { + put_arg_page(src); + ret = -ENOMEM; + goto out; + } + from = map_arg_page(src); + to = map_arg_page(dst); + /* Recheck per-string limits if the selected granule shrank. */ + ret = 0; + for (unsigned long i = 0; i < len; i++) { + if (++string_len > bprm_max_arg_strlen(&replacement)) { + ret = -E2BIG; + break; + } + if (!from[src_off + i]) + string_len = 0; + } + if (!ret) + memcpy(to + dst_off, from + src_off, len); + kunmap_local(to); + kunmap_local(from); + flush_arg_page(&replacement, pos, dst); + put_arg_page(dst); + put_arg_page(src); + if (ret) + goto out; + pos += len; + cond_resched(); + } + + old_mm = bprm->mm; + acct_arg_size(bprm, 0); + bprm->mm = replacement.mm; + bprm->vma = replacement.vma; + bprm->vma_pages = replacement.vma_pages; + mmput(old_mm); + return 0; +out: + acct_arg_size(&replacement, 0); + mmput(replacement.mm); + return ret; +} +#endif + struct user_arg_ptr { #ifdef CONFIG_COMPAT bool is_compat; @@ -449,7 +613,7 @@ static int bprm_stack_limits(struct linux_binprm *bprm) static int copy_strings(int argc, struct user_arg_ptr argv, struct linux_binprm *bprm) { - struct page *kmapped_page = NULL; + struct arg_page kmapped_page = {}; char *kaddr = NULL; unsigned long kpos = 0; int ret; @@ -464,7 +628,7 @@ static int copy_strings(int argc, struct user_arg_ptr argv, if (IS_ERR(str)) goto out; - len = strnlen_user(str, MAX_ARG_STRLEN); + len = strnlen_user(str, bprm_max_arg_strlen(bprm)); if (!len) goto out; @@ -488,9 +652,9 @@ static int copy_strings(int argc, struct user_arg_ptr argv, } cond_resched(); - offset = pos % PAGE_SIZE; + offset = pos & ~bprm_copy_mask(bprm); if (offset == 0) - offset = PAGE_SIZE; + offset = bprm_copy_size(bprm); bytes_to_copy = offset; if (bytes_to_copy > len) @@ -501,23 +665,23 @@ static int copy_strings(int argc, struct user_arg_ptr argv, str -= bytes_to_copy; len -= bytes_to_copy; - if (!kmapped_page || kpos != (pos & PAGE_MASK)) { - struct page *page; + if (!kmapped_page.page || kpos != (pos & bprm_copy_mask(bprm))) { + struct arg_page page; page = get_arg_page(bprm, pos, 1); - if (!page) { + if (!page.page) { ret = -E2BIG; goto out; } - if (kmapped_page) { - flush_dcache_page(kmapped_page); + if (kmapped_page.page) { + flush_dcache_page(kmapped_page.page); kunmap_local(kaddr); put_arg_page(kmapped_page); } kmapped_page = page; - kaddr = kmap_local_page(kmapped_page); - kpos = pos & PAGE_MASK; + kaddr = map_arg_page(kmapped_page); + kpos = pos & bprm_copy_mask(bprm); flush_arg_page(bprm, kpos, kmapped_page); } if (copy_from_user(kaddr+offset, str, bytes_to_copy)) { @@ -528,8 +692,8 @@ static int copy_strings(int argc, struct user_arg_ptr argv, } ret = 0; out: - if (kmapped_page) { - flush_dcache_page(kmapped_page); + if (kmapped_page.page) { + flush_dcache_page(kmapped_page.page); kunmap_local(kaddr); put_arg_page(kmapped_page); } @@ -541,7 +705,7 @@ static int copy_strings(int argc, struct user_arg_ptr argv, */ int copy_string_kernel(const char *arg, struct linux_binprm *bprm) { - int len = strnlen(arg, MAX_ARG_STRLEN) + 1 /* terminating NUL */; + int len = strnlen(arg, bprm_max_arg_strlen(bprm)) + 1 /* terminating NUL */; unsigned long pos = bprm->p; if (len == 0) @@ -557,18 +721,19 @@ int copy_string_kernel(const char *arg, struct linux_binprm *bprm) while (len > 0) { unsigned int bytes_to_copy = min(len, - min_not_zero(offset_in_page(pos), PAGE_SIZE)); - struct page *page; + min_not_zero(pos & ~bprm_copy_mask(bprm), bprm_copy_size(bprm))); + struct arg_page page; pos -= bytes_to_copy; arg -= bytes_to_copy; len -= bytes_to_copy; page = get_arg_page(bprm, pos, 1); - if (!page) + if (!page.page) return -E2BIG; - flush_arg_page(bprm, pos & PAGE_MASK, page); - memcpy_to_page(page, offset_in_page(pos), arg, bytes_to_copy); + flush_arg_page(bprm, pos & bprm_copy_mask(bprm), page); + memcpy_to_page(page.page, arg_page_offset(page) + + (pos & ~bprm_copy_mask(bprm)), arg, bytes_to_copy); put_arg_page(page); } @@ -621,20 +786,20 @@ int setup_arg_pages(struct linux_binprm *bprm, /* Add space for stack randomization. */ if (current->flags & PF_RANDOMIZE) - stack_base += (STACK_RND_MASK << PAGE_SHIFT); + stack_base += (STACK_RND_MASK << mm_page_shift(mm)); /* Make sure we didn't let the argument array grow too large. */ if (vma->vm_end - vma->vm_start > stack_base) return -ENOMEM; - stack_base = PAGE_ALIGN(stack_top - stack_base); + stack_base = mm_page_align(mm, stack_top - stack_base); stack_shift = vma->vm_start - stack_base; mm->arg_start = bprm->p - stack_shift; bprm->p = vma->vm_end - stack_shift; #else stack_top = arch_align_stack(stack_top); - stack_top = PAGE_ALIGN(stack_top); + stack_top = mm_page_align(mm, stack_top); if (unlikely(stack_top < mmap_min_addr) || unlikely(vma->vm_end - vma->vm_start >= stack_top - mmap_min_addr)) @@ -702,7 +867,7 @@ int setup_arg_pages(struct linux_binprm *bprm, * Align this down to a page boundary as expand_stack * will align it up. */ - rlim_stack = bprm->rlim_stack.rlim_cur & PAGE_MASK; + rlim_stack = bprm->rlim_stack.rlim_cur & mm_page_mask(mm); stack_expand = min(rlim_stack, stack_size + stack_expand); @@ -1323,6 +1488,10 @@ void setup_new_exec(struct linux_binprm * bprm) arch_pick_mmap_layout(me->mm, &bprm->rlim_stack); + /* Consume the one-shot request only after committing to the new image. */ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + me->exec_page_shift = 0; +#endif arch_setup_new_exec(); /* Set the new mm task size. We have to do that late because it may @@ -1616,25 +1785,25 @@ int remove_arg_zero(struct linux_binprm *bprm) { unsigned long offset; char *kaddr; - struct page *page; + struct arg_page page; if (!bprm->argc) return 0; do { - offset = bprm->p & ~PAGE_MASK; + offset = bprm->p & ~bprm_copy_mask(bprm); page = get_arg_page(bprm, bprm->p, 0); - if (!page) + if (!page.page) return -EFAULT; - kaddr = kmap_local_page(page); + kaddr = map_arg_page(page); - for (; offset < PAGE_SIZE && kaddr[offset]; + for (; offset < bprm_copy_size(bprm) && kaddr[offset]; offset++, bprm->p++) ; kunmap_local(kaddr); put_arg_page(page); - } while (offset == PAGE_SIZE); + } while (offset == bprm_copy_size(bprm)); bprm->p++; bprm->argc--; From c5f12989feb7fadc77031ea60829bdc879112864 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0424/1579] block: preserve user fragment ownership across I/O Carry the backing offset and retained fragment reference through bio, splice and io_uring paths so a smaller user mapping is not treated as an entire native physical page. Signed-off-by: Diljit Singh --- block/bio.c | 150 +++++++++++++++++++++++++++++++++ block/blk-map.c | 2 +- fs/splice.c | 172 ++++++++++++++++++++++++++++++++++++++ include/linux/bio.h | 8 ++ include/linux/blk_types.h | 4 + io_uring/memmap.c | 8 +- io_uring/rsrc.c | 124 +++++++++++++++++++++++++++ 7 files changed, 466 insertions(+), 2 deletions(-) diff --git a/block/bio.c b/block/bio.c index 5f10900b3f42a..846417dec7d16 100644 --- a/block/bio.c +++ b/block/bio.c @@ -3,6 +3,7 @@ * Copyright (C) 2001 Jens Axboe */ #include +#include #include #include #include @@ -246,6 +247,10 @@ void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, #endif #ifdef CONFIG_BLK_DEV_INTEGRITY bio->bi_integrity = NULL; +#endif +#ifdef CONFIG_MM_SUBPAGE + bio->bi_user_fragments = NULL; + bio->bi_nr_user_fragments = 0; #endif bio->bi_vcnt = 0; @@ -1159,6 +1164,18 @@ void __bio_release_pages(struct bio *bio, bool mark_dirty) { struct folio_iter fi; +#ifdef CONFIG_MM_SUBPAGE + if (bio->bi_user_fragments) { + release_user_fragments(bio->bi_user_fragments, + bio->bi_nr_user_fragments, mark_dirty); + kfree(bio->bi_user_fragments); + bio->bi_user_fragments = NULL; + bio->bi_nr_user_fragments = 0; + bio_clear_flag(bio, BIO_PAGE_PINNED); + return; + } +#endif + bio_for_each_folio_all(fi, bio) { size_t nr_pages; @@ -1220,6 +1237,134 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, return 0; } +#ifdef CONFIG_MM_SUBPAGE +static bool bio_subpage_iter(const struct iov_iter *iter) +{ + return user_backed_iter(iter) && current->mm && + mm_page_size(current->mm) < PAGE_SIZE; +} + +int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs) +{ + struct iov_iter walk = *iter; + unsigned int count = 0; + + if (!bio_subpage_iter(iter)) + return iov_iter_npages(iter, max_vecs); + while (iov_iter_count(&walk) && count < max_vecs) { + size_t len, offset, nr; + + iov_iter_advance(&walk, 0); + len = min(iter_iov_len(&walk), iov_iter_count(&walk)); + if (WARN_ON_ONCE(!len)) + return max_vecs; + offset = (unsigned long)iter_iov_addr(&walk) & (MM_SUBPAGE_SIZE - 1); + nr = (len >> MM_SUBPAGE_SHIFT) + + DIV_ROUND_UP(offset + (len & (MM_SUBPAGE_SIZE - 1)), MM_SUBPAGE_SIZE); + if (nr >= max_vecs - count) + return max_vecs; + count += nr; + iov_iter_advance(&walk, len); + } + return count; +} +EXPORT_SYMBOL_GPL(bio_iov_user_vecs); + +/* Pins belong to the submitting bio, just as with ordinary BIO_PAGE_PINNED. + * Split/cloned bios borrow its vectors and complete before their owner does. + */ +static int bio_iov_iter_get_fragments(struct bio *bio, struct iov_iter *iter, + unsigned int align_mask) +{ + unsigned int capacity = bio->bi_max_vecs, total = 0; + unsigned int flags = iov_iter_rw(iter) == READ ? FOLL_WRITE : 0; + int ret = 0; + + if (WARN_ON_ONCE(bio->bi_vcnt || bio->bi_user_fragments || !capacity)) + return -EINVAL; + bio->bi_user_fragments = kmalloc_array(capacity, + sizeof(*bio->bi_user_fragments), GFP_KERNEL); + if (!bio->bi_user_fragments) + return -ENOMEM; + while (iov_iter_count(iter) && bio->bi_nr_user_fragments < capacity && + total < BIO_MAX_SIZE) { + unsigned int i, bytes = 0; + long nr; + + iov_iter_advance(iter, 0); + nr = pin_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(iter), + min3(iter_iov_len(iter), iov_iter_count(iter), + (size_t)(BIO_MAX_SIZE - total)), flags, + bio->bi_user_fragments + bio->bi_nr_user_fragments, + capacity - bio->bi_nr_user_fragments); + if (nr <= 0) { + ret = nr ?: -EFAULT; + break; + } + for (i = 0; i < nr; i++) { + struct user_page_fragment *fragment = + &bio->bi_user_fragments[bio->bi_nr_user_fragments++]; + struct page *page = folio_page(fragment->folio, + fragment->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(fragment->offset); + struct bio_vec *prev = bio->bi_vcnt ? + &bio->bi_io_vec[bio->bi_vcnt - 1] : NULL; + + /* Coalesce physically adjacent quarters within one folio. */ + if (prev && page_folio(prev->bv_page) == fragment->folio && + page_to_phys(prev->bv_page) + prev->bv_offset + prev->bv_len == + page_to_phys(page) + offset) + prev->bv_len += fragment->length; + else + bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt++], page, + fragment->length, offset); + bytes += fragment->length; + } + iov_iter_advance(iter, bytes); + total += bytes; + } + /* Trim pins separately from vectors: several pins may share one bvec. */ + if (total & align_mask) { + unsigned int trim = total & align_mask, left = trim; + + while (left) { + struct user_page_fragment *fragment = + &bio->bi_user_fragments[bio->bi_nr_user_fragments - 1]; + unsigned int bytes = min(left, fragment->length); + + if (bytes == fragment->length) { + release_user_fragments(fragment, 1, false); + bio->bi_nr_user_fragments--; + } else { + fragment->length -= bytes; + } + left -= bytes; + } + left = trim; + while (left) { + struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; + unsigned int bytes = min(left, bv->bv_len); + + bv->bv_len -= bytes; + if (!bv->bv_len) + bio->bi_vcnt--; + left -= bytes; + } + iov_iter_revert(iter, trim); + total -= trim; + } + bio->bi_iter.bi_size = total; + if (!total) { + kfree(bio->bi_user_fragments); + bio->bi_user_fragments = NULL; + return ret ?: -EFAULT; + } + bio_set_flag(bio, BIO_PAGE_PINNED); + return 0; +} +#endif + /** * bio_iov_iter_get_pages - add user or kernel pages to a bio * @bio: bio to add pages to @@ -1255,6 +1400,11 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, return 0; } +#ifdef CONFIG_MM_SUBPAGE + if (bio_subpage_iter(iter)) + return bio_iov_iter_get_fragments(bio, iter, len_align_mask); +#endif + if (iov_iter_extract_will_pin(iter)) bio_set_flag(bio, BIO_PAGE_PINNED); if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue)) diff --git a/block/blk-map.c b/block/blk-map.c index 768549f19f97e..b98eed276aea2 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -260,7 +260,7 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, gfp_t gfp_mask) { - unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS); + unsigned int nr_vecs = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS); struct bio *bio; int ret; diff --git a/fs/splice.c b/fs/splice.c index 9d8f63e2fd1ab..4b707ce00e86d 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -183,6 +185,79 @@ static void wakeup_pipe_readers(struct pipe_inode_info *pipe) kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); } +#ifdef CONFIG_MM_SUBPAGE +/* A shared lifetime for all user leaves coalesced into one native pipe page. */ +struct subpage_pipe_refs { + refcount_t refs; + unsigned int count; + struct user_page_fragment fragments[]; +}; + +static void subpage_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs = (void *)buf->private; + + if (refcount_dec_and_test(&refs->refs)) { + release_user_fragments(refs->fragments, refs->count, false); + kfree(refs); + } +} + +static bool subpage_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs = (void *)buf->private; + + return refcount_inc_not_zero(&refs->refs); +} + +static const struct pipe_buf_operations subpage_pipe_buf_ops = { + .release = subpage_pipe_buf_release, + .get = subpage_pipe_buf_get, + /* A gift of user leaves is not ownership of the containing native page. */ +}; + +static int subpage_pipe_buf_retain(struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs; + unsigned int pos, count = 0; + + if (!buf->len || buf->ops != &nosteal_pipe_buf_ops || + !folio_test_anon(page_folio(buf->page))) + return 0; + refs = kmalloc(struct_size(refs, fragments, PAGE_SIZE / MM_SUBPAGE_SIZE), GFP_KERNEL); + if (!refs) + return -ENOMEM; + for (pos = round_down(buf->offset, MM_SUBPAGE_SIZE); + pos < buf->offset + buf->len; pos += MM_SUBPAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(page_to_phys(buf->page) + pos); + + if (!slot) { + release_user_fragments(refs->fragments, count, false); + kfree(refs); + return count ? -EFAULT : 0; + } + refs->fragments[count++] = (struct user_page_fragment) { + .folio = page_folio(buf->page), + .subpage = slot, + .offset = pos, + .length = MM_SUBPAGE_SIZE, + }; + } + refcount_set(&refs->refs, 1); + refs->count = count; + /* Replace the incoming native reference with the typed references. */ + put_page(buf->page); + buf->ops = &subpage_pipe_buf_ops; + buf->private = (unsigned long)refs; + return 0; +} +#else +static int subpage_pipe_buf_retain(struct pipe_buffer *buf) +{ + return 0; +} +#endif + /** * splice_to_pipe - fill passed data into a pipe * @pipe: pipe to fill @@ -221,6 +296,16 @@ ssize_t splice_to_pipe(struct pipe_inode_info *pipe, buf->private = spd->partial[page_nr].private; buf->ops = spd->ops; buf->flags = 0; + { + int err = subpage_pipe_buf_retain(buf); + + if (err) { + buf->ops = NULL; + if (!ret) + ret = err; + break; + } + } head++; pipe->head = head; @@ -852,6 +937,16 @@ ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out, continue; } +#ifdef CONFIG_MM_SUBPAGE + /* Other transports still need their downstream lifetime audit. */ + if (buf->ops == &subpage_pipe_buf_ops && + sock->ops->family != AF_UNIX && + sock->ops->family != AF_INET && + sock->ops->family != AF_INET6) { + ret = -EOPNOTSUPP; + break; + } +#endif seg = min_t(size_t, remain, buf->len); ret = pipe_buf_confirm(pipe, buf); @@ -1440,6 +1535,79 @@ static ssize_t __do_splice(struct file *in, loff_t __user *off_in, return ret; } +#ifdef CONFIG_MM_SUBPAGE +static ssize_t subpage_iter_to_pipe(struct iov_iter *from, + struct pipe_inode_info *pipe, unsigned int flags) +{ + struct user_page_fragment fragments[16]; + size_t total = 0; + long ret = 0; + + if (WARN_ON_ONCE(!user_backed_iter(from))) + return -EINVAL; + while (iov_iter_count(from)) { + unsigned int i = 0; + long count; + + /* Normalize empty iovecs before taking the next address. */ + iov_iter_advance(from, 0); + count = get_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(from), + min(iov_iter_count(from), iter_iov_len(from)), 0, + fragments, ARRAY_SIZE(fragments)); + if (count <= 0) { + ret = count; + break; + } + while (i < count) { + struct user_page_fragment *first = &fragments[i]; + struct page *page = folio_page(first->folio, first->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(first->offset); + unsigned int length = first->length, nr = 1; + struct subpage_pipe_refs *refs; + struct pipe_buffer buf; + + /* Preserve native pipe capacity for contiguous packed quarters. */ + while (i + nr < count && length < PAGE_SIZE - offset) { + struct user_page_fragment *next = &fragments[i + nr]; + + if (folio_page(next->folio, next->offset >> PAGE_SHIFT) != page || + offset_in_page(next->offset) != offset + length) + break; + length += next->length; + nr++; + } + refs = kmalloc(struct_size(refs, fragments, nr), GFP_KERNEL); + if (!refs) { + ret = -ENOMEM; + break; + } + refcount_set(&refs->refs, 1); + refs->count = nr; + memcpy(refs->fragments, first, sizeof(*first) * nr); + buf = (struct pipe_buffer) { + .page = page, + .offset = offset, + .len = length, + .ops = &subpage_pipe_buf_ops, + .flags = flags, + .private = (unsigned long)refs, + }; + i += nr; /* add_to_pipe consumes these references even on failure. */ + ret = add_to_pipe(pipe, &buf); + if (ret < 0) + break; + iov_iter_advance(from, ret); + total += ret; + } + release_user_fragments(fragments + i, count - i, false); + if (ret < 0) + break; + } + return total ? total : ret; +} +#endif + static ssize_t iter_to_pipe(struct iov_iter *from, struct pipe_inode_info *pipe, unsigned int flags) @@ -1451,6 +1619,10 @@ static ssize_t iter_to_pipe(struct iov_iter *from, size_t total = 0; ssize_t ret = 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return subpage_iter_to_pipe(from, pipe, flags); +#endif while (iov_iter_count(from)) { struct page *pages[16]; ssize_t left; diff --git a/include/linux/bio.h b/include/linux/bio.h index dc17780d6c1e3..714e44044373a 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -389,11 +389,19 @@ static inline void bio_wouldblock_error(struct bio *bio) * pointed by @iter. If @iter is backed by bvec it's going to be reused * instead of allocating a new one. */ +#ifdef CONFIG_MM_SUBPAGE +int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs); +#endif + static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs) { if (iov_iter_is_bvec(iter)) return 0; +#ifdef CONFIG_MM_SUBPAGE + return bio_iov_user_vecs(iter, max_segs); +#else return iov_iter_npages(iter, max_segs); +#endif } /** diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 8808ee76e73c0..cc9b1eeddd415 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -267,6 +267,10 @@ struct bio { struct bio_integrity_payload *bi_integrity; /* data integrity */ #endif +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *bi_user_fragments; + unsigned short bi_nr_user_fragments; +#endif unsigned short bi_vcnt; /* how many bio_vec's */ /* diff --git a/io_uring/memmap.c b/io_uring/memmap.c index 4f9b439319c46..72e123757bea4 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -289,7 +289,10 @@ static int io_region_mmap(struct io_ring_ctx *ctx, unsigned long nr_pages = min(mr->nr_pages, max_pages); vm_flags_set(vma, VM_DONTEXPAND); - return vm_insert_pages(vma, vma->vm_start, mr->pages, &nr_pages); + if (mm_page_size(vma->vm_mm) == PAGE_SIZE) + return vm_insert_pages(vma, vma->vm_start, mr->pages, &nr_pages); + return vm_insert_pages_range(vma, vma->vm_start, mr->pages, nr_pages, 0, + min_t(unsigned long, vma->vm_end - vma->vm_start, nr_pages << PAGE_SHIFT)); } __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma) @@ -301,6 +304,9 @@ __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma) struct io_mapped_region *region; void *ptr; + if (vma_subpage_offset(vma)) + return -EINVAL; + guard(mutex)(&ctx->mmap_lock); ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff); diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 2dfc26233e286..33cc1e2688480 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -762,6 +763,95 @@ bool io_check_coalesce_buffer(struct page **page_array, int nr_pages, return true; } +#ifdef CONFIG_MM_SUBPAGE +struct io_fragment_buffer { + unsigned int count; + struct user_page_fragment fragments[]; +}; + +static void io_release_fragment_buffer(void *priv) +{ + struct io_fragment_buffer *buffer = priv; + + release_user_fragments(buffer->fragments, buffer->count, false); + kvfree(buffer); +} + +static struct io_rsrc_node *io_register_fragment_buffer(struct io_ring_ctx *ctx, + struct iovec *iov) +{ + unsigned long size = mm_page_size(current->mm); + unsigned long first = (unsigned long)iov->iov_base & (size - 1); + unsigned int count = DIV_ROUND_UP(first + iov->iov_len, size); + struct io_fragment_buffer *buffer; + struct io_mapped_ubuf *imu = NULL; + struct io_rsrc_node *node; + unsigned long charged = 0; + unsigned int i; + struct xarray seen; + long nr; + int ret = -ENOMEM; + + node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); + if (!node) + return ERR_PTR(-ENOMEM); + buffer = kvmalloc(struct_size(buffer, fragments, count), GFP_KERNEL_ACCOUNT); + if (!buffer) + goto free_node; + buffer->count = 0; + nr = pin_user_fragments_remote(current->mm, (unsigned long)iov->iov_base, + iov->iov_len, FOLL_WRITE | FOLL_LONGTERM, buffer->fragments, count); + if (nr > 0) + buffer->count = nr; + if (nr != count) { + ret = nr < 0 ? nr : -EFAULT; + goto free_buffer; + } + imu = io_alloc_imu(ctx, count); + if (!imu) + goto free_buffer; + imu->nr_bvecs = count; + imu->acct_pages = 0; + /* Several user leaves can share native backing, including aliases. */ + xa_init(&seen); + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &buffer->fragments[i]; + + ret = xa_insert(&seen, folio_pfn(f->folio), xa_mk_value(1), GFP_KERNEL); + if (!ret) + charged += folio_nr_pages(f->folio); + else if (ret != -EBUSY) + break; + bvec_set_page(&imu->bvec[i], folio_page(f->folio, f->offset >> PAGE_SHIFT), + f->length, offset_in_page(f->offset)); + } + xa_destroy(&seen); + if (ret && ret != -EBUSY) + goto free_imu; + ret = io_account_mem(ctx->user, ctx->mm_account, charged); + if (ret) + goto free_imu; + imu->acct_pages = charged; + imu->ubuf = (unsigned long)iov->iov_base; + imu->len = iov->iov_len; + imu->folio_shift = mm_page_shift(current->mm); + imu->release = io_release_fragment_buffer; + imu->priv = buffer; + imu->flags = 0; + imu->dir = IO_IMU_DEST | IO_IMU_SOURCE; + refcount_set(&imu->refs, 1); + node->buf = imu; + return node; +free_imu: + io_free_imu(ctx, imu); +free_buffer: + io_release_fragment_buffer(buffer); +free_node: + io_cache_free(&ctx->node_cache, node); + return ERR_PTR(ret); +} +#endif + static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, struct page **last_hpage) @@ -787,6 +877,11 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, if (ret) return ERR_PTR(ret); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) < PAGE_SIZE) + return io_register_fragment_buffer(ctx, iov); +#endif + node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); if (!node) return ERR_PTR(-ENOMEM); @@ -1095,6 +1190,14 @@ static int io_import_fixed(int ddir, struct iov_iter *iter, bvec += seg_skip; offset &= folio_mask; } +#ifdef CONFIG_MM_SUBPAGE + if (imu->folio_shift < PAGE_SHIFT) { + /* Physical fragment offsets are independent of the user address. */ + size_t first = min_t(size_t, len, bvec->bv_len - offset); + + nr_segs = 1 + DIV_ROUND_UP(len - first, 1UL << imu->folio_shift); + } else +#endif nr_segs = (offset + len + bvec->bv_offset + folio_mask) >> imu->folio_shift; iov_iter_bvec(iter, ddir, bvec, nr_segs, len); iter->iov_offset = offset; @@ -1356,6 +1459,27 @@ static int io_vec_fill_bvec(int ddir, struct iov_iter *iter, return -EOVERFLOW; offset = buf_addr - imu->ubuf; +#ifdef CONFIG_MM_SUBPAGE + if (imu->folio_shift < PAGE_SHIFT) { + unsigned int index; + unsigned long first = imu->ubuf & folio_mask; + + offset += first; + index = offset >> imu->folio_shift; + offset &= folio_mask; + if (!index) + offset -= first; + src_bvec = imu->bvec + index; + for (; iov_len; offset = 0, bvec_idx++, src_bvec++) { + size_t bytes = min_t(size_t, iov_len, src_bvec->bv_len - offset); + + bvec_set_page(&res_bvec[bvec_idx], src_bvec->bv_page, + bytes, src_bvec->bv_offset + offset); + iov_len -= bytes; + } + continue; + } +#endif /* * Only the first bvec can have non zero bv_offset, account it * here and work with full folios below. From e6bf191ec8db185fcc908e5a5410497aac3a7543 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0425/1579] net: retain user fragments in networking and BPF consumers Preserve fragment ownership and byte ranges in socket, TLS, BPF and vhost consumers of user memory. Signed-off-by: Diljit Singh --- drivers/vhost/vhost.c | 21 +++ include/linux/bpf.h | 22 +++ include/linux/skmsg.h | 40 +++++- include/net/tls.h | 7 + kernel/bpf/bpf_local_storage.c | 23 ++-- kernel/bpf/syscall.c | 128 ++++++++++++++++-- net/core/skbuff.c | 119 ++++++++++++++++- net/core/skmsg.c | 88 +++++++++++- net/ipv4/tcp_bpf.c | 5 + net/tls/tls_device.c | 22 +++ net/tls/tls_sw.c | 238 +++++++++++++++++++++++++++++++-- 11 files changed, 673 insertions(+), 40 deletions(-) diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index e4c44f2ea413a..4c3577ce7b85b 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -2436,6 +2437,26 @@ static int set_bit_to_user(int nr, void __user *addr) int bit = nr + (log % PAGE_SIZE) * 8; int r; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + unsigned long word = log & ~(sizeof(unsigned long) - 1); + + /* Pin the complete atomic word, which stays inside one leaf. + * A relocated leaf's physical offset need not match its VA. + */ + r = pin_user_fragments_remote(current->mm, word, + sizeof(unsigned long), FOLL_WRITE, &fragment, 1); + if (r != 1) + return r < 0 ? r : -EFAULT; + base = kmap_local_folio(fragment.folio, fragment.offset); + set_bit(nr + (log - word) * 8, base); + kunmap_local(base); + release_user_fragments(&fragment, 1, true); + return 0; + } +#endif + r = pin_user_pages_fast(log, 1, FOLL_WRITE, &page); if (r < 0) return r; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index a2cf24d168802..cfec2e3b66979 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -263,6 +263,11 @@ struct btf_record { int wq_off; int refcount_off; int task_work_off; +#ifdef CONFIG_MM_SUBPAGE + /* Private ownership pointer appended to task-storage values with UPTRs. */ + u32 uptr_pins_off; + u32 uptr_extra_size; +#endif struct btf_field fields[]; }; @@ -563,6 +568,15 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src bpf_obj_memcpy(map->record, dst, src, map->value_size, true); } +static inline u32 bpf_obj_uptr_extra_size(const struct btf_record *rec) +{ +#ifdef CONFIG_MM_SUBPAGE + return IS_ERR_OR_NULL(rec) ? 0 : rec->uptr_extra_size; +#else + return 0; +#endif +} + static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) { unsigned long *src_uptr, *dst_uptr; @@ -580,6 +594,14 @@ static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, v dst_uptr = dst + field->offset; swap(*src_uptr, *dst_uptr); } +#ifdef CONFIG_MM_SUBPAGE + if (rec->uptr_extra_size) { + void **src_pins = src + rec->uptr_pins_off; + void **dst_pins = dst + rec->uptr_pins_off; + + swap(*src_pins, *dst_pins); + } +#endif } static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size) diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 937823856de59..30406f8cee116 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -41,6 +41,8 @@ struct sk_msg_sg { }; /* UAPI in filter.c depends on struct sk_msg_sg being first element. */ +struct mm_subpage_refs; + struct sk_msg { struct sk_msg_sg sg; void *data; @@ -49,6 +51,9 @@ struct sk_msg { u32 cork_bytes; u32 flags; struct sk_buff *skb; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage_refs *subpage_refs; +#endif struct sock *sk_redir; struct sock *sk; struct list_head list; @@ -124,6 +129,33 @@ struct sk_psock { struct rcu_work rwork; }; +#ifdef CONFIG_MM_SUBPAGE +int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len); +void sk_msg_subpages_get(struct sk_msg *msg); +void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src); +void sk_msg_subpages_release(struct sk_msg *msg); +static inline void sk_msg_subpages_transfer(struct sk_msg *dst, struct sk_msg *src) +{ + dst->subpage_refs = src->subpage_refs; + src->subpage_refs = NULL; +} +#else +static inline int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len) +{ + return 0; +} +static inline void sk_msg_subpages_get(struct sk_msg *msg) { } +static inline void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src) { } +static inline void sk_msg_subpages_release(struct sk_msg *msg) { } +static inline void sk_msg_subpages_transfer(struct sk_msg *dst, struct sk_msg *src) { } +#endif + +static inline void sk_msg_subpages_drop_empty(struct sk_msg *msg) +{ + if (!msg->sg.size) + sk_msg_subpages_release(msg); +} + int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, int elem_first_coalesce); int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, @@ -259,10 +291,14 @@ static inline void sk_msg_compute_data_pointers(struct sk_msg *msg) } } -static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, +static inline int sk_msg_page_add(struct sk_msg *msg, struct page *page, u32 len, u32 offset) { struct scatterlist *sge; + int ret = sk_msg_subpages_add(msg, page, offset, len); + + if (ret) + return ret; get_page(page); sge = sk_msg_elem(msg, msg->sg.end); @@ -272,6 +308,7 @@ static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, __set_bit(msg->sg.end, msg->sg.copy); msg->sg.size += len; sk_msg_iter_next(msg, end); + return 0; } static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state) @@ -420,6 +457,7 @@ static inline bool sk_psock_queue_empty(const struct sk_psock *psock) static inline void kfree_sk_msg(struct sk_msg *msg) { + sk_msg_subpages_release(msg); if (msg->skb) consume_skb(msg->skb); kfree(msg); diff --git a/include/net/tls.h b/include/net/tls.h index ebd2550280ae2..79d358a5fe887 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -34,6 +34,7 @@ #ifndef _TLS_OFFLOAD_H #define _TLS_OFFLOAD_H +#include #include #include #include @@ -139,6 +140,9 @@ struct tls_sw_context_rx { struct tls_strparser strp; atomic_t decrypt_pending; +#ifdef CONFIG_MM_SUBPAGE + struct llist_head subpage_cleanup; +#endif struct sk_buff_head async_hold; struct wait_queue_head wq; }; @@ -148,6 +152,9 @@ struct tls_record_info { u32 end_seq; int len; int num_frags; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage_refs *subpage_refs; +#endif skb_frag_t frags[MAX_SKB_FRAGS]; }; diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 6fc6a4b672b55..9abd4972966ff 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -24,6 +24,11 @@ select_bucket(struct bpf_local_storage_map *smap, return &smap->buckets[hash_ptr(local_storage, smap->bucket_log)]; } +static u32 bpf_selem_size(const struct bpf_local_storage_map *smap) +{ + return smap->elem_size + bpf_obj_uptr_extra_size(smap->map.record); +} + static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size) { struct bpf_map *map = &smap->map; @@ -72,10 +77,10 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, { struct bpf_local_storage_elem *selem; - if (mem_charge(smap, owner, smap->elem_size)) + if (mem_charge(smap, owner, bpf_selem_size(smap))) return NULL; - selem = bpf_map_kmalloc_nolock(&smap->map, smap->elem_size, + selem = bpf_map_kmalloc_nolock(&smap->map, bpf_selem_size(smap), __GFP_ZERO, NUMA_NO_NODE); if (selem) { @@ -91,7 +96,7 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, return selem; } - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); return NULL; } @@ -178,7 +183,7 @@ static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem * bool free_local_storage, bool pin_owner) { void *owner = local_storage->owner; - u32 uncharge = smap->elem_size; + u32 uncharge = bpf_selem_size(smap); if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) == SDATA(selem)) @@ -204,7 +209,7 @@ static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem * /* local_storage->lock must be held and selem->local_storage == local_storage. * The caller must ensure selem->smap is still valid to be - * dereferenced for its smap->elem_size and smap->cache_idx. + * dereferenced for its element size and smap->cache_idx. */ static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, struct bpf_local_storage_elem *selem, @@ -234,7 +239,7 @@ void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage, struct bpf_local_storage_map *smap; smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); - local_storage->mem_charge += smap->elem_size; + local_storage->mem_charge += bpf_selem_size(smap); RCU_INIT_POINTER(selem->local_storage, local_storage); hlist_add_head_rcu(&selem->snode, &local_storage->list); @@ -577,7 +582,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, err = bpf_local_storage_alloc(owner, smap, selem); if (err) { bpf_selem_free(selem, true); - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); return ERR_PTR(err); } @@ -661,7 +666,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, free_selem: bpf_selem_free_list(&old_selem_free_list, false); if (alloc_selem) { - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); bpf_selem_free(alloc_selem, true); } return err ? ERR_PTR(err) : SDATA(selem); @@ -862,7 +867,7 @@ void bpf_local_storage_map_free(struct bpf_map *map, * exit immediately. * * However, while freeing the storage one still needs to access the - * smap->elem_size to do the uncharging in + * bpf_selem_size(smap) to do the uncharging in * bpf_selem_unlink_storage_nolock(). * * Hence, wait another rcu grace period for the storage to be freed. diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 96d1b797e1523..c084c1cad03d8 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include #include @@ -164,6 +166,43 @@ static void maybe_wait_bpf_programs(struct bpf_map *map) synchronize_rcu_expedited(); } +#ifdef CONFIG_MM_SUBPAGE +struct bpf_uptr_pins { + struct work_struct work; + u32 count; + struct user_page_fragment fragments[]; +}; + +static void bpf_uptr_release_work(struct work_struct *work) +{ + struct bpf_uptr_pins *pins = container_of(work, struct bpf_uptr_pins, work); + + release_user_fragments(pins->fragments, pins->count, true); + kfree(pins); +} +#endif + +/* May run from an RCU callback. The bundle owns its pins independently of + * the map, task and value, so dirty release can wait for folio locks in work. + */ +static bool bpf_obj_release_uptr_pins(const struct btf_record *rec, void *obj) +{ +#ifdef CONFIG_MM_SUBPAGE + struct bpf_uptr_pins **owner, *pins; + + if (!bpf_obj_uptr_extra_size(rec)) + return false; + owner = obj + rec->uptr_pins_off; + pins = *owner; + if (pins) { + *owner = NULL; + queue_work(system_unbound_wq, &pins->work); + return true; + } +#endif + return false; +} + static void unpin_uptr_kaddr(void *kaddr) { if (kaddr) @@ -176,6 +215,9 @@ static void __bpf_obj_unpin_uptrs(struct btf_record *rec, u32 cnt, void *obj) void **uptr_addr; int i; + if (bpf_obj_release_uptr_pins(rec, obj)) + return; + for (i = 0, field = rec->fields; i < cnt; i++, field++) { if (field->type != BPF_UPTR) continue; @@ -193,8 +235,12 @@ static void bpf_obj_unpin_uptrs(struct btf_record *rec, void *obj) __bpf_obj_unpin_uptrs(rec, rec->cnt, obj); } -static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) +static int bpf_obj_pin_uptrs(struct bpf_map *map, void *obj) { + struct btf_record *rec = map->record; +#ifdef CONFIG_MM_SUBPAGE + struct bpf_uptr_pins *pins = NULL; +#endif const struct btf_field *field; const struct btf_type *t; unsigned long start, end; @@ -205,6 +251,18 @@ static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) if (!btf_record_has_field(rec, BPF_UPTR)) return 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + pins = bpf_map_kzalloc(map, struct_size(pins, fragments, rec->cnt), + GFP_KERNEL); + if (!pins) + return -ENOMEM; + INIT_WORK(&pins->work, bpf_uptr_release_work); + pins->count = rec->cnt; + *(struct bpf_uptr_pins **)(obj + rec->uptr_pins_off) = pins; + } +#endif + for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) { if (field->type != BPF_UPTR) continue; @@ -221,12 +279,35 @@ static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) goto unpin_all; } - /* The uptr's struct cannot span across two pages */ - if ((start & PAGE_MASK) != (end & PAGE_MASK)) { + /* The object must fit in one page of the supplying process. */ + if ((start & mm_page_mask(current->mm)) != + (end & mm_page_mask(current->mm))) { err = -EOPNOTSUPP; goto unpin_all; } +#ifdef CONFIG_MM_SUBPAGE + if (pins) { + struct user_page_fragment *fragment = &pins->fragments[i]; + + err = pin_user_fragments_remote(current->mm, start, t->size, + FOLL_LONGTERM | FOLL_WRITE, fragment, 1); + if (err != 1) { + if (!err) + err = -EFAULT; + goto unpin_all; + } + if (fragment->length != t->size || + PageHighMem(folio_page(fragment->folio, + fragment->offset >> PAGE_SHIFT))) { + err = -EOPNOTSUPP; + goto unpin_all; + } + *uptr_addr = folio_address(fragment->folio) + fragment->offset; + continue; + } +#endif + err = pin_user_pages_fast(start, 1, FOLL_LONGTERM | FOLL_WRITE, &page); if (err != 1) goto unpin_all; @@ -291,7 +372,7 @@ static int bpf_map_update_value(struct bpf_map *map, struct file *map_file, map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) { err = map->ops->map_push_elem(map, value, flags); } else { - err = bpf_obj_pin_uptrs(map->record, value); + err = bpf_obj_pin_uptrs(map, value); if (!err) { rcu_read_lock(); err = map->ops->map_update_elem(map, key, value, flags); @@ -815,10 +896,12 @@ void bpf_obj_cancel_fields(struct bpf_map *map, void *obj) void bpf_obj_free_fields(const struct btf_record *rec, void *obj) { const struct btf_field *fields; + bool fragment_pins; int i; if (IS_ERR_OR_NULL(rec)) return; + fragment_pins = bpf_obj_release_uptr_pins(rec, obj); fields = rec->fields; for (i = 0; i < rec->cnt; i++) { struct btf_struct_meta *pointee_struct_meta; @@ -859,8 +942,9 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj) } break; case BPF_UPTR: - /* The caller ensured that no one is using the uptr */ - unpin_uptr_kaddr(*(void **)field_ptr); + /* The caller ensured that no one is using the uptr. */ + if (!fragment_pins) + unpin_uptr_kaddr(*(void **)field_ptr); break; case BPF_LIST_HEAD: if (WARN_ON_ONCE(rec->spin_lock_off < 0)) @@ -1350,6 +1434,14 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, if (ret < 0) goto free_map_tab; +#ifdef CONFIG_MM_SUBPAGE + if (btf_record_has_field(map->record, BPF_UPTR)) { + map->record->uptr_pins_off = ALIGN(map->value_size, sizeof(void *)); + map->record->uptr_extra_size = map->record->uptr_pins_off + + sizeof(void *) - map->value_size; + } +#endif + if (map->ops->map_check_btf) { ret = map->ops->map_check_btf(map, btf, key_type, value_type); if (ret < 0) @@ -1775,6 +1867,24 @@ static int map_lookup_elem(union bpf_attr *attr) #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags +static void *bpf_map_memdup_value(struct bpf_map *map, bpfptr_t src, u32 size) +{ + u32 extra = bpf_obj_uptr_extra_size(map->record); + void *value; + + if (!extra) + return kvmemdup_bpfptr(src, size); + value = kvmalloc(size + extra, GFP_USER); + if (!value) + return ERR_PTR(-ENOMEM); + memset(value + size, 0, extra); + if (copy_from_bpfptr(value, src, size)) { + kvfree(value); + return ERR_PTR(-EFAULT); + } + return value; +} + static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) { bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel); @@ -1808,7 +1918,7 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) } value_size = bpf_map_value_size(map, attr->flags); - value = kvmemdup_bpfptr(uvalue, value_size); + value = bpf_map_memdup_value(map, uvalue, value_size); if (IS_ERR(value)) { err = PTR_ERR(value); goto free_key; @@ -2021,7 +2131,9 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file, if (!key) return -ENOMEM; - value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); + value = kvmalloc(value_size + bpf_obj_uptr_extra_size(map->record), + GFP_USER | __GFP_NOWARN | + (bpf_obj_uptr_extra_size(map->record) ? __GFP_ZERO : 0)); if (!value) { kvfree(key); return -ENOMEM; diff --git a/net/core/skbuff.c b/net/core/skbuff.c index a5cdcc0fc55e1..0bc6467aae5d4 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -7347,8 +7348,113 @@ nodefer: kfree_skb_napi_cache(skb); kick_defer_list_purge(cpu); } +#ifdef CONFIG_MM_SUBPAGE +/* The ordinary frag page ref protects allocation, not an anonymous quarter. */ +struct skb_subpage_refs { + struct ubuf_info ubuf; + struct mm_subpage_refs *owner; +}; + +static void skb_subpage_complete(struct sk_buff *skb, struct ubuf_info *ubuf, + bool success) +{ + struct skb_subpage_refs *refs = container_of(ubuf, struct skb_subpage_refs, ubuf); + + if (!refcount_dec_and_test(&ubuf->refcnt)) + return; + mm_subpage_refs_put(refs->owner); + kfree(refs); +} + +static const struct ubuf_info_ops skb_subpage_ops = { + .complete = skb_subpage_complete, +}; + +static bool skb_contains_subpage(const struct sk_buff *skb, phys_addr_t phys, + unsigned int size, unsigned int depth) +{ + struct sk_buff *child; + unsigned int i; + + /* Match the networking recursion bound, conservatively keeping unknowns. */ + if (depth >= 24) + return true; + if (skb->head_frag && skb_headlen(skb)) { + phys_addr_t start = page_to_phys(virt_to_page(skb->data)) + offset_in_page(skb->data); + + if (start < phys + size && phys < start + skb_headlen(skb)) + return true; + } + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + struct page *page = skb_frag_page(frag); + phys_addr_t start; + + if (!page) + continue; + start = page_to_phys(page) + skb_frag_off(frag); + if (start < phys + size && phys < start + skb_frag_size(frag)) + return true; + } + skb_walk_frags(skb, child) + if (skb_contains_subpage(child, phys, size, depth + 1)) + return true; + return false; +} + +static bool skb_keeps_subpage(const struct mm_subpage *slot, void *context) +{ + return skb_contains_subpage(context, mm_subpage_phys(slot), mm_subpage_size(slot), 0); +} + +/* Caller owns the writable skb and keeps the source slot identity alive. */ +static int skb_splice_hold_subpages(struct sk_buff *skb, struct page *page, + unsigned int off, unsigned int len) +{ + struct skb_subpage_refs *refs, *new; + struct ubuf_info *old = skb_zcopy(skb); + struct mm_subpage *probe; + int ret; + + if (!len || folio_order(page_folio(page)) || !folio_test_anon(page_folio(page))) + return 0; + /* Preserve the allocation-free native anonymous path. */ + probe = mm_subpage_get_from_phys(page_to_phys(page) + round_down(off, MM_SUBPAGE_SIZE)); + if (!probe) + return 0; + mm_subpage_put(probe); + if (old && (skb_zcopy_is_nouarg(skb) || old->ops != &skb_subpage_ops)) + return -EOPNOTSUPP; + refs = old ? container_of(old, struct skb_subpage_refs, ubuf) : NULL; + if (!refs || refcount_read(&old->refcnt) != 1) { + new = kzalloc(sizeof(*new), GFP_ATOMIC); + if (!new) + return -ENOMEM; + new->ubuf.ops = &skb_subpage_ops; + new->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; + refcount_set(&new->ubuf.refcnt, 1); + if (refs) + new->owner = mm_subpage_refs_get(refs->owner); + skb_zcopy_init(skb, &new->ubuf); + net_zcopy_put(old); + refs = new; + } + /* Repeated trim/append must not accumulate references to discarded data. */ + ret = mm_subpage_refs_prune(&refs->owner, skb_keeps_subpage, skb, GFP_ATOMIC); + if (ret) + return ret; + return mm_subpage_refs_add(&refs->owner, page, off, len, GFP_ATOMIC); +} +#else +static int skb_splice_hold_subpages(struct sk_buff *skb, struct page *page, + unsigned int off, unsigned int len) +{ + return 0; +} +#endif + static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, - size_t offset, size_t len) + size_t offset, size_t len, size_t progress) { const char *kaddr; __wsum csum; @@ -7356,7 +7462,8 @@ static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, kaddr = kmap_local_page(page); csum = csum_partial(kaddr + offset, len, 0); kunmap_local(kaddr); - skb->csum = csum_block_add(skb->csum, csum, skb->len); + /* skb->len is updated only after the entire iterator batch. */ + skb->csum = csum_block_add(skb->csum, csum, skb->len + progress); } /** @@ -7408,6 +7515,12 @@ ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, if (WARN_ON_ONCE(!sendpage_ok(page))) goto out; + ret = skb_splice_hold_subpages(skb, page, off, part); + if (ret < 0) { + iov_iter_revert(iter, len); + goto out; + } + ret = skb_append_pagefrags(skb, page, off, part, frag_limit); if (ret < 0) { @@ -7416,7 +7529,7 @@ ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, } if (skb->ip_summed == CHECKSUM_NONE) - skb_splice_csum_page(skb, page, off, part); + skb_splice_csum_page(skb, page, off, part, spliced); off = 0; spliced += part; diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 30c3b9a2681c4..10eb255be0ffa 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */ +#include +#include #include #include #include @@ -24,6 +26,65 @@ static bool sk_msg_try_coalesce_ok(struct sk_msg *msg, int elem_first_coalesce) return false; } +#ifdef CONFIG_MM_SUBPAGE +static bool sk_msg_keeps_subpage(const struct mm_subpage *slot, void *context) +{ + struct sk_msg *msg = context; + phys_addr_t phys = mm_subpage_phys(slot); + phys_addr_t end = phys + mm_subpage_size(slot); + u32 i = msg->sg.start; + + if (!msg->sg.size) + return false; + while (i != msg->sg.end) { + struct scatterlist *sg = &msg->sg.data[i]; + + if (sg->length) { + phys_addr_t start = page_to_phys(sg_page(sg)) + sg->offset; + + if (start < end && phys < start + sg->length) + return true; + } + sk_msg_iter_var_next(i); + } + return false; +} + +int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len) +{ + int ret; + + ret = mm_subpage_refs_prune(&msg->subpage_refs, sk_msg_keeps_subpage, msg, GFP_ATOMIC); + if (ret) + return ret; + return mm_subpage_refs_add(&msg->subpage_refs, page, offset, len, GFP_ATOMIC); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_add); + +/* Called after a borrowed descriptor copy whose source remains live. */ +void sk_msg_subpages_get(struct sk_msg *msg) +{ + mm_subpage_refs_get(msg->subpage_refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_get); + +void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src) +{ + WARN_ON_ONCE(dst->subpage_refs); + dst->subpage_refs = mm_subpage_refs_get(src->subpage_refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_share); + +void sk_msg_subpages_release(struct sk_msg *msg) +{ + struct mm_subpage_refs *refs = msg->subpage_refs; + + msg->subpage_refs = NULL; + mm_subpage_refs_put(refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_release); +#endif + int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, int elem_first_coalesce) { @@ -88,7 +149,7 @@ EXPORT_SYMBOL_GPL(sk_msg_alloc); int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, u32 off, u32 len) { - int i = src->sg.start; + int i = src->sg.start, ret; struct scatterlist *sge = sk_msg_elem(src, i); struct scatterlist *sgd = NULL; u32 sge_len, sge_off; @@ -114,11 +175,16 @@ int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, if (sgd && (sg_page(sge) == sg_page(sgd)) && (sg_virt(sge) + off == sg_virt(sgd) + sgd->length)) { + ret = sk_msg_subpages_add(dst, sg_page(sge), sge->offset + off, sge_len); + if (ret) + return ret; sgd->length += sge_len; dst->sg.size += sge_len; } else if (!sk_msg_full(dst)) { sge_off = sge->offset + off; - sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off); + ret = sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off); + if (ret) + return ret; } else { return -ENOSPC; } @@ -205,6 +271,7 @@ static int __sk_msg_free(struct sock *sk, struct sk_msg *msg, u32 i, sk_msg_check_to_free(msg, i, msg->sg.size); sge = sk_msg_elem(msg, i); } + sk_msg_subpages_release(msg); consume_skb(msg->skb); sk_msg_init(msg); return freed; @@ -248,6 +315,7 @@ static void __sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, sk_msg_check_to_free(msg, i, bytes); } msg->sg.start = i; + sk_msg_subpages_drop_empty(msg); } void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes) @@ -267,6 +335,7 @@ void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len) int trim = msg->sg.size - len; u32 i = msg->sg.end; + sk_msg_subpages_drop_empty(msg); if (trim <= 0) { WARN_ON(trim < 0); return; @@ -299,6 +368,7 @@ void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len) * does not require an update. */ if (!msg->sg.size) { + sk_msg_subpages_release(msg); msg->sg.curr = msg->sg.start; msg->sg.copybreak = 0; } else if (sk_msg_iter_dist(msg->sg.start, msg->sg.curr) >= @@ -334,11 +404,19 @@ int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from, goto out; } - bytes -= copied; - msg->sg.size += copied; - while (copied) { use = min_t(int, copied, PAGE_SIZE - offset); + ret = sk_msg_subpages_add(msg, pages[i], offset, use); + if (ret) { + unsigned int nr = DIV_ROUND_UP(offset + copied, PAGE_SIZE); + + while (nr--) + put_page(pages[i++]); + iov_iter_revert(from, copied); + goto out; + } + bytes -= use; + msg->sg.size += use; sg_set_page(&msg->sg.data[msg->sg.end], pages[i], use, offset); sg_unmark_end(&msg->sg.data[msg->sg.end]); diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index 18b45950d1acc..d082be17d76fc 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -44,6 +44,7 @@ static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, return -ENOMEM; lock_sock(sk); + sk_msg_subpages_share(tmp, msg); tmp->sg.start = msg->sg.start; i = msg->sg.start; do { @@ -76,6 +77,7 @@ static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, if (!ret) { msg->sg.start = i; + sk_msg_subpages_drop_empty(msg); if (!sk_psock_queue_msg(psock, tmp)) atomic_sub(copied, &sk->sk_rmem_alloc); sk_psock_data_ready(sk, psock); @@ -147,6 +149,7 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, break; } + sk_msg_subpages_drop_empty(msg); return 0; } @@ -434,6 +437,8 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, } } memcpy(psock->cork, msg, sizeof(*msg)); + if (psock->cork != msg) + sk_msg_subpages_transfer(psock->cork, msg); return 0; } diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 33c42f65364a2..10da3de523b93 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -32,6 +32,8 @@ #include #include #include +#include +#include #include #include #include @@ -144,6 +146,10 @@ static void destroy_record(struct tls_record_info *record) for (i = 0; i < record->num_frags; i++) __skb_frag_unref(&record->frags[i], false); + +#ifdef CONFIG_MM_SUBPAGE + mm_subpage_refs_put(record->subpage_refs); +#endif kfree(record); } @@ -349,6 +355,9 @@ static int tls_create_new_record(struct tls_offload_context_tx *offload_ctx, record = kmalloc_obj(*record); if (!record) return -ENOMEM; +#ifdef CONFIG_MM_SUBPAGE + record->subpage_refs = NULL; +#endif frag = &record->frags[0]; skb_frag_fill_page_desc(frag, pfrag->page, pfrag->offset, @@ -513,6 +522,15 @@ static int tls_push_data(struct sock *sk, goto handle_error; } +#ifdef CONFIG_MM_SUBPAGE + rc = mm_subpage_refs_add(&record->subpage_refs, zc_pfrag.page, + off, copy, GFP_KERNEL); + if (rc) { + iov_iter_revert(iter, copy); + goto handle_error; + } +#endif + zc_pfrag.offset = off; zc_pfrag.size = copy; tls_append_frag(record, &zc_pfrag, copy); @@ -1118,6 +1136,10 @@ int tls_set_device_offload(struct sock *sk) goto release_netdev; } + +#ifdef CONFIG_MM_SUBPAGE + start_marker_record->subpage_refs = NULL; +#endif offload_ctx = alloc_offload_ctx_tx(ctx); if (!offload_ctx) { rc = -ENOMEM; diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index ca0f657121a3b..f4f708533f79b 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -35,8 +35,11 @@ * SOFTWARE. */ +#include #include #include +#include +#include #include #include #include @@ -65,9 +68,36 @@ struct tls_decrypt_ctx { u8 aad[TLS_MAX_AAD_SIZE]; u8 tail; bool free_sgout; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *fragments; + unsigned int nr_fragments; + bool fragments_dirty; + struct llist_node cleanup; + void *allocation; +#endif struct scatterlist sg[]; }; +#if IS_ENABLED(CONFIG_KUNIT) && IS_ENABLED(CONFIG_MM_SUBPAGE) +/* Explicit test-only async provider selection; normal cipher selection stays intact. */ +static bool user4k_test_async; +module_param_named(user4k_test_async, user4k_test_async, bool, 0400); +MODULE_PARM_DESC(user4k_test_async, "KUnit: select cryptd for TLS receive testing"); +static atomic_long_t user4k_rx_deferred = ATOMIC_LONG_INIT(0); +static atomic_long_t user4k_rx_released = ATOMIC_LONG_INIT(0); + +static int user4k_read_count(char *buffer, const struct kernel_param *param) +{ + return scnprintf(buffer, PAGE_SIZE, "%ld\n", atomic_long_read(param->arg)); +} + +static const struct kernel_param_ops user4k_count_ops = { + .get = user4k_read_count, +}; +module_param_cb(user4k_rx_deferred, &user4k_count_ops, &user4k_rx_deferred, 0444); +module_param_cb(user4k_rx_released, &user4k_count_ops, &user4k_rx_released, 0444); +#endif + noinline void tls_err_abort(struct sock *sk, int err) { WARN_ON_ONCE(err >= 0); @@ -184,6 +214,35 @@ static int tls_padding_length(struct tls_prot_info *prot, struct sk_buff *skb, return sub; } +#ifdef CONFIG_MM_SUBPAGE +static void tls_release_user_fragments(struct tls_decrypt_ctx *dctx) +{ + release_user_fragments(dctx->fragments, dctx->nr_fragments, dctx->fragments_dirty); + kfree(dctx->fragments); + dctx->fragments = NULL; + dctx->nr_fragments = 0; +} +#endif + +/* Dirty pin release can take folio locks, so it belongs to the waiting task. */ +static void tls_finish_subpage_decrypts(struct tls_sw_context_rx *ctx) +{ +#ifdef CONFIG_MM_SUBPAGE + struct tls_decrypt_ctx *dctx, *next; + struct llist_node *head = llist_del_all(&ctx->subpage_cleanup); + + llist_for_each_entry_safe(dctx, next, head, cleanup) { + void *allocation = dctx->allocation; + + tls_release_user_fragments(dctx); +#if IS_ENABLED(CONFIG_KUNIT) + atomic_long_inc(&user4k_rx_released); +#endif + kfree(allocation); + } +#endif +} + static void tls_decrypt_done(void *data, int err) { struct aead_request *aead_req = data; @@ -196,6 +255,7 @@ static void tls_decrypt_done(void *data, int err) unsigned int pages; struct sock *sk; int aead_size; + bool deferred = false; /* If requests get too backlogged crypto API returns -EBUSY and calls * ->complete(-EINPROGRESS) immediately followed by ->complete(0) @@ -224,17 +284,29 @@ static void tls_decrypt_done(void *data, int err) tls_err_abort(sk, err); } - /* Free the destination pages if skb was not decrypted inplace */ - if (dctx->free_sgout) { - /* Skip the first S/G entry as it points to AAD */ - for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) { - if (!sg) - break; - put_page(sg_page(sg)); +#ifdef CONFIG_MM_SUBPAGE + if (dctx->fragments) { + dctx->allocation = aead_req; +#if IS_ENABLED(CONFIG_KUNIT) + atomic_long_inc(&user4k_rx_deferred); +#endif + llist_add(&dctx->cleanup, &ctx->subpage_cleanup); + deferred = true; + } +#endif + if (!deferred) { + /* Free the destination pages if skb was not decrypted inplace */ + if (dctx->free_sgout) { + /* Skip the first S/G entry as it points to AAD */ + for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) { + if (!sg) + break; + put_page(sg_page(sg)); + } } - } - kfree(aead_req); + kfree(aead_req); + } if (atomic_dec_and_test(&ctx->decrypt_pending)) complete(&ctx->async_wait.completion); @@ -246,6 +318,7 @@ static int tls_decrypt_async_wait(struct tls_sw_context_rx *ctx) crypto_wait_req(-EINPROGRESS, &ctx->async_wait); atomic_inc(&ctx->decrypt_pending); + tls_finish_subpage_decrypts(ctx); __skb_queue_purge(&ctx->async_hold); return ctx->async_wait.err; } @@ -670,6 +743,7 @@ static int tls_split_open_record(struct sock *sk, struct tls_rec *from, msg_opl->sg.size = bytes; msg_npl = &new->msg_plaintext; + sk_msg_subpages_share(msg_npl, msg_opl); msg_npl->apply_bytes = apply; msg_npl->sg.size = orig_size - bytes; @@ -733,6 +807,7 @@ static void tls_merge_open_record(struct sock *sk, struct tls_rec *to, sk_msg_free(sk, &to->msg_encrypted); sk_msg_xfer_full(&to->msg_encrypted, &from->msg_encrypted); + sk_msg_subpages_release(msg_npl); kfree(from); } @@ -935,12 +1010,14 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, redir_ingress = psock->redir_ingress; sk_redir = psock->sk_redir; memcpy(&msg_redir, msg, sizeof(*msg)); + sk_msg_subpages_get(&msg_redir); if (msg->apply_bytes < send) msg->apply_bytes = 0; else msg->apply_bytes -= send; sk_msg_return_zero(sk, msg, send); msg->sg.size -= send; + sk_msg_subpages_drop_empty(msg); release_sock(sk); err = tcp_bpf_sendmsg_redir(sk_redir, redir_ingress, &msg_redir, send, flags); @@ -956,6 +1033,7 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, *copied -= sk_msg_free_nocharge(sk, &msg_redir); msg->sg.size = 0; } + sk_msg_subpages_release(&msg_redir); if (msg->sg.size == 0) tls_free_open_rec(sk); break; @@ -1036,7 +1114,14 @@ static int tls_sw_sendmsg_splice(struct sock *sk, struct msghdr *msg, return -EIO; } - sk_msg_page_add(msg_pl, page, part, off); + { + int ret = sk_msg_page_add(msg_pl, page, part, off); + + if (ret) { + iov_iter_revert(&msg->msg_iter, part); + return ret; + } + } msg_pl->sg.copybreak = 0; msg_pl->sg.curr = msg_pl->sg.end; sk_mem_charge(sk, part); @@ -1454,10 +1539,96 @@ tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock, return 1; } +#ifdef CONFIG_MM_SUBPAGE +static bool tls_subpage_iter(const struct iov_iter *iter) +{ + return user_backed_iter(iter) && current->mm && mm_page_size(current->mm) < PAGE_SIZE; +} +#endif + +static int tls_iter_sg_capacity(struct iov_iter *iter, size_t length) +{ +#ifdef CONFIG_MM_SUBPAGE + if (tls_subpage_iter(iter)) { + struct iov_iter walk = *iter; + unsigned int count = 0; + + length = min(length, iov_iter_count(iter)); + while (length) { + size_t bytes, offset; + + iov_iter_advance(&walk, 0); + bytes = min(length, iter_iov_len(&walk)); + if (WARN_ON_ONCE(!bytes)) + return INT_MAX; + offset = (unsigned long)iter_iov_addr(&walk) & (MM_SUBPAGE_SIZE - 1); + count += DIV_ROUND_UP(offset + bytes, MM_SUBPAGE_SIZE); + iov_iter_advance(&walk, bytes); + length -= bytes; + } + return count; + } +#endif + return iov_iter_npages_cap(iter, INT_MAX, length); +} + +#ifdef CONFIG_MM_SUBPAGE +static int tls_setup_subpage_iter(struct iov_iter *from, int length, int *pages_used, + struct scatterlist *to, int capacity, + struct tls_decrypt_ctx *dctx) +{ + unsigned int used = 0, total = 0; + int ret = 0; + + if (!length) + return 0; + if (WARN_ON_ONCE(*pages_used || capacity <= 0)) + return -EINVAL; + dctx->fragments = kmalloc_array(capacity, sizeof(*dctx->fragments), GFP_KERNEL); + if (!dctx->fragments) + return -ENOMEM; + while (length) { + long nr; + unsigned int i, bytes = 0; + + iov_iter_advance(from, 0); + nr = pin_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(from), + min_t(size_t, length, iter_iov_len(from)), FOLL_WRITE, + dctx->fragments + used, capacity - used); + if (nr <= 0) { + ret = nr ?: -EFAULT; + goto out; + } + for (i = 0; i < nr; i++) { + struct user_page_fragment *fragment = &dctx->fragments[used]; + + sg_set_page(&to[used], + folio_page(fragment->folio, fragment->offset >> PAGE_SHIFT), + fragment->length, offset_in_page(fragment->offset)); + sg_unmark_end(&to[used]); + bytes += fragment->length; + used++; + } + dctx->nr_fragments = used; + iov_iter_advance(from, bytes); + length -= bytes; + total += bytes; + } + if (used) + sg_mark_end(&to[used - 1]); +out: + if (ret) + iov_iter_revert(from, total); + *pages_used = used; + return ret; +} +#endif + static int tls_setup_from_iter(struct iov_iter *from, int length, int *pages_used, struct scatterlist *to, - int to_max_pages) + int to_max_pages, struct tls_decrypt_ctx *dctx) { int rc = 0, i = 0, num_elem = *pages_used, maxpages; struct page *pages[MAX_SKB_FRAGS]; @@ -1465,6 +1636,10 @@ static int tls_setup_from_iter(struct iov_iter *from, ssize_t copied, use; size_t offset; +#ifdef CONFIG_MM_SUBPAGE + if (tls_subpage_iter(from)) + return tls_setup_subpage_iter(from, length, pages_used, to, to_max_pages, dctx); +#endif while (length > 0) { i = 0; maxpages = to_max_pages - num_elem; @@ -1582,7 +1757,7 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, if (out_iov) n_sgout = 1 + tail_pages + - iov_iter_npages_cap(out_iov, INT_MAX, data_len); + tls_iter_sg_capacity(out_iov, data_len); else n_sgout = sg_nents(out_sg); } else { @@ -1615,6 +1790,11 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, aead_req = (struct aead_request *)mem; dctx = (struct tls_decrypt_ctx *)(mem + aead_size); dctx->sk = sk; +#ifdef CONFIG_MM_SUBPAGE + dctx->fragments = NULL; + dctx->nr_fragments = 0; + dctx->fragments_dirty = false; +#endif sgin = &dctx->sg[0]; sgout = &dctx->sg[n_sgin]; @@ -1672,7 +1852,7 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, sg_set_buf(&sgout[0], dctx->aad, prot->aad_size); err = tls_setup_from_iter(out_iov, data_len, &pages, &sgout[1], - (n_sgout - 1 - tail_pages)); + (n_sgout - 1 - tail_pages), dctx); if (err < 0) goto exit_free_pages; @@ -1687,6 +1867,10 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, } dctx->free_sgout = !!pages; +#ifdef CONFIG_MM_SUBPAGE + /* An error from crypto may still leave writes in the destination. */ + dctx->fragments_dirty = true; +#endif /* Prepare and submit AEAD request */ err = tls_do_decryption(sk, sgin, sgout, dctx->iv, data_len + prot->tail_size, aead_req, darg); @@ -1715,6 +1899,12 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, darg->tail = dctx->tail; exit_free_pages: +#ifdef CONFIG_MM_SUBPAGE + if (dctx->fragments) { + tls_release_user_fragments(dctx); + pages = 0; + } +#endif /* Release the pages in case iov was mapped to pages */ for (; pages > 0; pages--) put_page(sg_page(&sgout[pages])); @@ -2795,6 +2985,9 @@ static struct tls_sw_context_rx *init_ctx_rx(struct tls_context *ctx) crypto_init_wait(&sw_ctx_rx->async_wait); atomic_set(&sw_ctx_rx->decrypt_pending, 1); +#ifdef CONFIG_MM_SUBPAGE + init_llist_head(&sw_ctx_rx->subpage_cleanup); +#endif init_waitqueue_head(&sw_ctx_rx->wq); skb_queue_head_init(&sw_ctx_rx->rx_list); skb_queue_head_init(&sw_ctx_rx->async_hold); @@ -2903,7 +3096,24 @@ int tls_set_sw_offload(struct sock *sk, int tx, rec_seq = crypto_info_rec_seq(src_crypto_info, cipher_desc); if (!*aead) { - *aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0); +#if IS_ENABLED(CONFIG_KUNIT) && IS_ENABLED(CONFIG_MM_SUBPAGE) + if (!tx && user4k_test_async) { + struct crypto_aead *sync; + char name[CRYPTO_MAX_ALG_NAME]; + + /* cryptd instances are named for the concrete child driver. */ + sync = crypto_alloc_aead(cipher_desc->cipher_name, 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(sync)) { + *aead = sync; + } else { + snprintf(name, sizeof(name), "cryptd(%s)", + crypto_tfm_alg_driver_name(crypto_aead_tfm(sync))); + *aead = crypto_alloc_aead(name, CRYPTO_ALG_ASYNC, CRYPTO_ALG_ASYNC); + crypto_free_aead(sync); + } + } else +#endif + *aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0); if (IS_ERR(*aead)) { rc = PTR_ERR(*aead); *aead = NULL; From 25dc8bcddb0298f46cbe5d335f6b56388e43e198 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0426/1579] iommu: carry exact user fragments through device mappings Convert IOMMUFD, VFIO and GEM/DMA-BUF paths to preserve the backing offset, mapping span and ownership of smaller user pages. Signed-off-by: Diljit Singh --- drivers/dma-buf/dma-buf.c | 35 +- drivers/gpu/drm/drm_gem.c | 7 +- drivers/gpu/drm/drm_gem_shmem_helper.c | 32 +- drivers/iommu/iommufd/Makefile | 2 + drivers/iommu/iommufd/device.c | 102 +++- drivers/iommu/iommufd/fragments.c | 421 +++++++++++++++ drivers/iommu/iommufd/io_pagetable.c | 15 +- drivers/iommu/iommufd/io_pagetable.h | 77 ++- drivers/iommu/iommufd/iommufd_private.h | 4 + drivers/iommu/iommufd/iova_bitmap.c | 131 ++++- drivers/iommu/iommufd/pages.c | 99 +++- drivers/iommu/iommufd/selftest.c | 61 ++- drivers/iommu/iommufd/viommu.c | 56 +- drivers/vfio/vfio_iommu_type1.c | 534 +++++++++++++------ drivers/vfio/vfio_iommu_type1_fragments.c | 609 ++++++++++++++++++++++ 15 files changed, 1918 insertions(+), 267 deletions(-) create mode 100644 drivers/iommu/iommufd/fragments.c create mode 100644 drivers/vfio/vfio_iommu_type1_fragments.c diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index d504c636dc294..0cb790ab2b6a3 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -225,9 +225,23 @@ static struct file_system_type dma_buf_fs_type = { .kill_sb = kill_anon_super, }; +static int dma_buf_mmap_range_check(struct dma_buf *dmabuf, + struct vm_area_struct *vma, unsigned long pgoff, unsigned int suboffset) +{ + unsigned long offset; + + if (check_mul_overflow(pgoff, PAGE_SIZE, &offset) || + check_add_overflow(offset, (unsigned long)suboffset, &offset)) + return -EOVERFLOW; + if (offset > dmabuf->size || vma->vm_end - vma->vm_start > dmabuf->size - offset) + return -EINVAL; + return 0; +} + static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) { struct dma_buf *dmabuf; + int ret; if (!is_dma_buf_file(file)) return -EINVAL; @@ -239,9 +253,9 @@ static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) return -EINVAL; /* check for overflowing the buffer's size */ - if (vma->vm_pgoff + vma_pages(vma) > - dmabuf->size >> PAGE_SHIFT) - return -EINVAL; + ret = dma_buf_mmap_range_check(dmabuf, vma, vma->vm_pgoff, vma_subpage_offset(vma)); + if (ret) + return ret; DMA_BUF_TRACE(trace_dma_buf_mmap_internal, dmabuf); @@ -1583,6 +1597,8 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_end_cpu_access, "DMA_BUF"); int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, unsigned long pgoff) { + int ret; + if (WARN_ON(!dmabuf || !vma)) return -EINVAL; @@ -1590,18 +1606,13 @@ int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, if (!dmabuf->ops->mmap) return -EINVAL; - /* check for offset overflow */ - if (pgoff + vma_pages(vma) < pgoff) - return -EOVERFLOW; - - /* check for overflowing the buffer's size */ - if (pgoff + vma_pages(vma) > - dmabuf->size >> PAGE_SHIFT) - return -EINVAL; + ret = dma_buf_mmap_range_check(dmabuf, vma, pgoff, 0); + if (ret) + return ret; /* readjust the vma */ vma_set_file(vma, dmabuf->file); - vma->vm_pgoff = pgoff; + vma_set_page_offset(vma, (struct vm_page_offset) { .index = pgoff }); DMA_BUF_TRACE(trace_dma_buf_mmap, dmabuf); diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index 3b2448a3a9de8..2176ab547bfb6 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -1343,7 +1343,7 @@ unsigned long drm_gem_get_unmapped_area(struct file *filp, unsigned long uaddr, struct drm_gem_object *obj; unsigned long ret; - obj = drm_gem_object_lookup_at_offset(filp, pgoff, len >> PAGE_SHIFT); + obj = drm_gem_object_lookup_at_offset(filp, pgoff, DIV_ROUND_UP(len, PAGE_SIZE)); if (IS_ERR(obj)) obj = NULL; @@ -1379,8 +1379,11 @@ int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma) struct drm_gem_object *obj; int ret; + /* GEM fake offsets identify an exact object start, not an interior byte. */ + if (vma_subpage_offset(vma)) + return -EINVAL; obj = drm_gem_object_lookup_at_offset(filp, vma->vm_pgoff, - vma_pages(vma)); + DIV_ROUND_UP(vma->vm_end - vma->vm_start, PAGE_SIZE)); if (IS_ERR(obj)) return PTR_ERR(obj); diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 5b8044508cfcf..7d019582c0cc2 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -554,13 +554,23 @@ int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev, } EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create); +/* GEM and PRIME add the object's fake offset; VMA splits retain it. + * Direct helper callers may instead supply an object-relative offset. + */ +static pgoff_t drm_gem_shmem_object_offset(struct drm_gem_object *obj, pgoff_t pgoff) +{ + pgoff_t base = drm_vma_node_start(&obj->vma_node); + + return pgoff >= base ? pgoff - base : pgoff; +} + static void drm_gem_shmem_record_mkwrite(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *obj = vma->vm_private_data; struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); loff_t num_pages = obj->size >> PAGE_SHIFT; - pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */ + pgoff_t page_offset = drm_gem_shmem_object_offset(obj, vmf->pgoff); if (drm_WARN_ON(obj->dev, !shmem->pages || page_offset >= num_pages)) return; @@ -587,7 +597,10 @@ static vm_fault_t try_insert_pfn(struct vm_fault *vmf, unsigned int order, vm_fault_t ret = VM_FAULT_FALLBACK; if (!order) { - ret = vmf_insert_pfn_mkwrite(vmf->vma, vmf->address, pfn, write); + struct vm_page_offset pos = vma_page_offset_at(vmf->vma, vmf->address); + + ret = vmf_insert_pfn_prot_mkwrite_offset(vmf->vma, vmf->address, + pfn, pos.offset, vmf->vma->vm_page_prot, write); #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP } else if (order == PMD_ORDER) { unsigned long paddr = pfn << PAGE_SHIFT; @@ -619,12 +632,12 @@ static vm_fault_t drm_gem_shmem_any_fault(struct vm_fault *vmf, unsigned int ord loff_t num_pages = obj->size >> PAGE_SHIFT; vm_fault_t ret = VM_FAULT_SIGBUS; struct page **pages = shmem->pages; - pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */ + pgoff_t page_offset = drm_gem_shmem_object_offset(obj, vmf->pgoff); struct page *page; struct folio *folio; unsigned long pfn; - if (order && order != PMD_ORDER) + if (order && (order != PMD_ORDER || mm_page_size(vma->vm_mm) != PAGE_SIZE)) return VM_FAULT_FALLBACK; dma_resv_lock(obj->resv, NULL); @@ -742,6 +755,17 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct if (is_cow_mapping(vma->vm_flags)) return -EINVAL; + /* Validate bytes, including a partial native-page offset from PRIME. */ + { + pgoff_t index = drm_gem_shmem_object_offset(obj, vma->vm_pgoff); + unsigned long offset, length = vma->vm_end - vma->vm_start; + + if (check_mul_overflow(index, PAGE_SIZE, &offset) || + check_add_overflow(offset, (unsigned long)vma_subpage_offset(vma), &offset) || + offset > obj->size || length > obj->size - offset) + return -EINVAL; + } + dma_resv_lock(shmem->base.resv, NULL); ret = drm_gem_shmem_get_pages_locked(shmem); dma_resv_unlock(shmem->base.resv); diff --git a/drivers/iommu/iommufd/Makefile b/drivers/iommu/iommufd/Makefile index 71d692c9a8f49..a868155e208d5 100644 --- a/drivers/iommu/iommufd/Makefile +++ b/drivers/iommu/iommufd/Makefile @@ -10,6 +10,8 @@ iommufd-y := \ vfio_compat.o \ viommu.o +iommufd-$(CONFIG_MM_SUBPAGE) += fragments.o + iommufd-$(CONFIG_IOMMUFD_TEST) += selftest.o obj-$(CONFIG_IOMMUFD) += iommufd.o diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c index 2895e5370910f..3ccb352bb3e50 100644 --- a/drivers/iommu/iommufd/device.c +++ b/drivers/iommu/iommufd/device.c @@ -1171,6 +1171,18 @@ struct iommufd_access *iommufd_access_create_internal(struct iommufd_ctx *ictx) return access; } +/* Physical byte-span consumers do not impose native-page IOVA alignment. */ +struct iommufd_access *iommufd_access_create_internal_phys(struct iommufd_ctx *ictx) +{ + struct iommufd_access *access = __iommufd_access_create(ictx); + + if (IS_ERR(access)) + return access; + access->iova_alignment = 1; + iommufd_object_finalize(ictx, &access->obj); + return access; +} + /** * iommufd_access_create - Create an iommufd_access * @ictx: iommufd file descriptor @@ -1404,6 +1416,12 @@ static bool check_area_prot(struct iopt_area *area, unsigned int flags) * ioas alignment is >= PAGE_SIZE and the iova is PAGE_SIZE aligned. However * smaller alignments have corner cases where this API can fail on otherwise * aligned iova. + * + * Alternative user granules are accepted only if each native output page has + * an aligned physical base and consecutive backing bytes within the requested + * range. The driver must access only @length bytes: a partial final page does + * not grant access to its remaining bytes. Fragment ownership stays in the IOAS + * until unpin; incompatible physical layouts return -EINVAL. */ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, unsigned long length, struct page **out_pages, @@ -1456,7 +1474,7 @@ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, flags, internal); if (rc) goto err_remove; - out_pages += last_index - index + 1; + out_pages += (last - iter.cur_iova) / PAGE_SIZE + 1; } if (!iopt_area_contig_done(&iter)) { rc = -ENOENT; @@ -1485,6 +1503,88 @@ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, } EXPORT_SYMBOL_NS_GPL(iommufd_access_pin_pages, "IOMMUFD"); +/** + * iommufd_access_pin_phys - Pin an internal contiguous physical byte span + * @access: Internal access created for byte-span pinning + * @iova: First IOVA byte + * @length: Exact byte length + * @out_phys: Physical address of the first byte, set only on success + * + * Retains native or fragment pin ownership and prevents IOAS unmap until the + * matching iommufd_access_unpin_pages(). No surrounding IOVA bytes are required. + * The caller must stop hardware access before unpinning or destroying access. + */ +int iommufd_access_pin_phys(struct iommufd_access *access, unsigned long iova, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_area_contig_iter iter; + struct io_pagetable *iopt; + struct iopt_area *area; + unsigned long last_iova; + phys_addr_t base = 0; + int rc; + + if (!iommufd_access_is_internal(access) || !out_phys || !length) + return -EINVAL; + if (check_add_overflow(iova, length - 1, &last_iova)) + return -EOVERFLOW; + + mutex_lock(&access->ioas_lock); + if (!access->ioas) { + rc = -ENOENT; + goto out_unlock; + } + iopt = &access->ioas->iopt; + down_read(&iopt->iova_rwsem); + iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) { + unsigned long last = min(last_iova, iopt_area_last_iova(area)); + phys_addr_t phys; + + if (area->prevent_access || !(area->iommu_prot & IOMMU_READ)) { + rc = -EPERM; + goto out_remove; + } + rc = iopt_area_add_phys_access(area, + iopt_area_start_byte(area, iter.cur_iova), + last - iter.cur_iova + 1, &phys); + if (rc) + goto out_remove; + if (iter.cur_iova == iova) { + base = phys; + } else if (phys != base + (iter.cur_iova - iova)) { + iopt_area_remove_access(area, + iopt_area_iova_to_index(area, iter.cur_iova), + iopt_area_iova_to_index(area, last), true); + rc = -EFAULT; + goto out_remove; + } + } + if (!iopt_area_contig_done(&iter)) { + rc = -ENOENT; + goto out_remove; + } + *out_phys = base; + up_read(&iopt->iova_rwsem); + rc = 0; + goto out_unlock; + +out_remove: + /* iter.cur_iova is the first byte whose access was not retained. */ + if (iter.cur_iova > iova) { + unsigned long end = iter.cur_iova - 1; + + iopt_for_each_contig_area(&iter, area, iopt, iova, end) + iopt_area_remove_access(area, + iopt_area_iova_to_index(area, iter.cur_iova), + iopt_area_iova_to_index(area, min(end, iopt_area_last_iova(area))), + true); + } + up_read(&iopt->iova_rwsem); +out_unlock: + mutex_unlock(&access->ioas_lock); + return rc; +} + /** * iommufd_access_rw - Read or write data under the iova * @access: IOAS access to act on diff --git a/drivers/iommu/iommufd/fragments.c b/drivers/iommu/iommufd/fragments.c new file mode 100644 index 0000000000000..5229605852133 --- /dev/null +++ b/drivers/iommu/iommufd/fragments.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Alternative user granules need an ownership record in addition to a physical + * address. A device translation cannot retain a subpage allocation identity. + * Keep those records until the final domain interval has gone away. The native + * PFN reader remains in pages.c; file and dma-buf suppliers retain its geometry. + */ +#include +#include +#include +#include + +#include "iommufd_private.h" +#include "io_pagetable.h" + +static phys_addr_t fragment_phys(const struct user_page_fragment *fragment) +{ + return PFN_PHYS(folio_pfn(fragment->folio)) + fragment->offset; +} + +/* Charge each native physical page once, even when several slots pin it. */ +static int fragment_store(struct iopt_pages *pages, unsigned long index, + struct user_page_fragment *fragment) +{ + unsigned long pfn = PHYS_PFN(fragment_phys(fragment)); + unsigned long count = xa_to_value(xa_load(&pages->fragment_pages, pfn)); + struct user_page_fragment *record; + int rc; + + record = kmemdup(fragment, sizeof(*record), GFP_KERNEL_ACCOUNT); + if (!record) + return -ENOMEM; + rc = xa_reserve(&pages->fragments, index, GFP_KERNEL_ACCOUNT); + if (rc) + goto out_free; + rc = xa_err(xa_store(&pages->fragment_pages, pfn, xa_mk_value(count + 1), + GFP_KERNEL_ACCOUNT)); + if (rc) + goto out_release; + /* xa_reserve() made this infallible. Ownership transfers here. */ + xa_store(&pages->fragments, index, record, GFP_NOWAIT); + if (!count) + pages->npinned++; + *fragment = (struct user_page_fragment) {}; + return 0; + +out_release: + xa_release(&pages->fragments, index); +out_free: + kfree(record); + return rc; +} + +/* No allocation on teardown, and callers have already unmapped all devices. */ +static void fragments_release_unused(struct iopt_pages *pages, + unsigned long first, unsigned long last) +{ + struct user_page_fragment *record; + unsigned long index = first; + + lockdep_assert_held(&pages->mutex); + xa_for_each_range(&pages->fragments, index, record, first, last) { + unsigned long pfn, count; + + if (interval_tree_iter_first(&pages->domains_itree, index, index) || + interval_tree_iter_first(&pages->access_itree, index, index)) + continue; + pfn = PHYS_PFN(fragment_phys(record)); + count = xa_to_value(xa_load(&pages->fragment_pages, pfn)); + if (WARN_ON(!count)) + continue; + if (count == 1) { + xa_erase(&pages->fragment_pages, pfn); + pages->npinned--; + } else { + xa_store(&pages->fragment_pages, pfn, xa_mk_value(count - 1), + GFP_NOWAIT); + } + xa_erase(&pages->fragments, index); + release_user_fragments(record, 1, pages->writable); + kfree(record); + cond_resched(); + } + if (pages->npinned < pages->last_npinned) + iopt_pages_update_pinned(pages, pages->last_npinned - pages->npinned, + false, NULL); +} + +/* Reuse existing records even if the original VA was unmapped or replaced. */ +static int fragments_acquire(struct iopt_pages *pages, unsigned long first, + unsigned long last) +{ + unsigned long size = 1UL << pages->page_shift, index = first; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + struct user_page_fragment *batch = NULL; + bool mm_live = false; + int rc = 0; + + lockdep_assert_held(&pages->mutex); + while (index <= last) { + unsigned long next = index, end, i; + long count; + + if (xa_load(&pages->fragments, index)) { + index++; + continue; + } + if (!mm_live) { + if (!mmget_not_zero(pages->source_mm)) { + rc = -EFAULT; + goto out; + } + mm_live = true; + batch = kmalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!batch) { + rc = -ENOMEM; + goto out; + } + } + /* Stop before any cached pin; never replace its ownership record. */ + end = xa_find(&pages->fragments, &next, last, XA_PRESENT) ? next : last + 1; + end = min(end, index + capacity); + count = pin_user_fragments_remote(pages->source_mm, + (unsigned long)pages->uptr + (index << pages->page_shift), + (end - index) << pages->page_shift, + FOLL_LONGTERM | (pages->writable ? FOLL_WRITE : 0), batch, capacity); + if (count <= 0) { + rc = count ?: -EFAULT; + goto out; + } + for (i = 0; i < count; i++) { + /* Each aligned logical index must have exactly one full pin. */ + if (WARN_ON(batch[i].length != size || + !IS_ALIGNED(fragment_phys(&batch[i]), size))) + rc = -EFAULT; + else + rc = fragment_store(pages, index, &batch[i]); + if (rc) { + release_user_fragments(batch + i, count - i, false); + goto out; + } + index++; + } + cond_resched(); + } + if (pages->npinned != pages->last_npinned) { + if (iommufd_should_fail()) + rc = -ENOMEM; + else + rc = iopt_pages_update_pinned(pages, + pages->npinned - pages->last_npinned, true, NULL); + } +out: + kfree(batch); + if (mm_live) + mmput(pages->source_mm); + if (rc) + fragments_release_unused(pages, first, last); + return rc; +} + +static void fragments_unmap(struct iommu_domain *domain, unsigned long iova, + unsigned long length) +{ + if (length) + WARN_ON(iommu_unmap(domain, iova, length) != length); +} + +/* All pins are present. Merge physically consecutive fragments before mapping. */ +static int fragments_map(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + unsigned long size = 1UL << pages->page_shift; + unsigned long index = iopt_area_index(area), last = iopt_area_last_index(area); + unsigned long remaining = iopt_area_length(area), done = 0; + unsigned long offset = area->page_offset; + unsigned long min_size = 1UL << __ffs(domain->pgsize_bitmap); + int rc; + + while (remaining) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + phys_addr_t phys = fragment_phys(record) + offset; + unsigned long bytes = min(size - offset, remaining); + + index++; + /* Legacy splitting forbids mappings larger than a native page. */ + while (index <= last && bytes < remaining && + (!area->iopt->disable_large_pages || bytes < PAGE_SIZE)) { + record = xa_load(&pages->fragments, index); + if (fragment_phys(record) != phys + bytes) + break; + bytes += min(size, remaining - bytes); + index++; + } + /* Never widen a device mapping to include an unowned neighbour. */ + if (!IS_ALIGNED(phys | (iopt_area_iova(area) + done) | bytes, min_size)) { + rc = -EINVAL; + goto out_unmap; + } + rc = iommu_map(domain, iopt_area_iova(area) + done, phys, bytes, + area->iommu_prot, GFP_KERNEL_ACCOUNT); + if (rc) + goto out_unmap; + done += bytes; + remaining -= bytes; + offset = 0; + cond_resched(); + } + return 0; + +out_unmap: + fragments_unmap(domain, iopt_area_iova(area), done); + return rc; +} + +int iopt_fragments_fill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + int rc; + + lockdep_assert_held(&pages->mutex); + rc = fragments_acquire(pages, iopt_area_index(area), iopt_area_last_index(area)); + if (rc) + return rc; + rc = fragments_map(area, pages, domain); + if (rc) + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); + return rc; +} + +int iopt_fragments_fill_domains(struct iopt_area *area, struct iopt_pages *pages) +{ + struct iommu_domain *domain; + unsigned long index, failed; + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, iopt_area_index(area), iopt_area_last_index(area)); + if (rc) + return rc; + xa_for_each(&area->iopt->domains, index, domain) { + rc = fragments_map(area, pages, domain); + if (rc) + goto out_unmap; + } + area->storage_domain = xa_load(&area->iopt->domains, 0); + interval_tree_insert(&area->pages_node, &pages->domains_itree); + return 0; + +out_unmap: + failed = index; + xa_for_each(&area->iopt->domains, index, domain) { + if (index == failed) + break; + fragments_unmap(domain, iopt_area_iova(area), iopt_area_length(area)); + } + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); + return rc; +} + +void iopt_fragments_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + lockdep_assert_held(&pages->mutex); + fragments_unmap(domain, iopt_area_iova(area), iopt_area_length(area)); + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); +} + +int iopt_fragments_rw(struct iopt_pages *pages, unsigned long start_byte, + void *data, unsigned long length, unsigned int flags) +{ + unsigned long size = 1UL << pages->page_shift; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + unsigned long index, offset = start_byte & (size - 1); + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + for (index = first; index <= last; index++) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + unsigned long bytes = min(size - offset, length); + void *addr = kmap_local_folio(record->folio, record->offset + offset); + + if (flags & IOMMUFD_ACCESS_RW_WRITE) + memcpy(addr, data, bytes); + else + memcpy(data, addr, bytes); + kunmap_local(addr); + data += bytes; + length -= bytes; + offset = 0; + } + fragments_release_unused(pages, first, last); + return 0; +} + +/* The retained typed records, not the output page array, own the pins. */ +static int fragments_register_access(struct iopt_area *area, unsigned long first, + unsigned long last, bool lock_area) +{ + struct iopt_pages *pages = area->pages; + struct iopt_pages_access *access; + struct interval_tree_node *node; + + lockdep_assert_held(&pages->mutex); + for (node = interval_tree_iter_first(&pages->access_itree, first, last); + node; node = interval_tree_iter_next(node, first, last)) { + if (node->start == first && node->last == last) { + access = container_of(node, struct iopt_pages_access, node); + access->users++; + goto out_count; + } + } + access = kzalloc_obj(*access, GFP_KERNEL_ACCOUNT); + if (!access) + return -ENOMEM; + access->node.start = first; + access->node.last = last; + access->users = 1; + interval_tree_insert(&access->node, &pages->access_itree); +out_count: + area->num_accesses++; + if (lock_area) + area->num_locks++; + return 0; +} + +/* Register a physical byte span without discarding fragment ownership. */ +int iopt_fragments_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_pages *pages = area->pages; + unsigned long size = 1UL << pages->page_shift; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + struct user_page_fragment *record; + unsigned long index; + phys_addr_t base; + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + record = xa_load(&pages->fragments, first); + base = fragment_phys(record); + for (index = first + 1; index <= last; index++) { + record = xa_load(&pages->fragments, index); + if (fragment_phys(record) != + base + ((index - first) << pages->page_shift)) { + rc = -EFAULT; + goto out_release; + } + cond_resched(); + } + rc = fragments_register_access(area, first, last, true); + if (rc) + goto out_release; + *out_phys = base + (start_byte & (size - 1)); + return 0; + +out_release: + fragments_release_unused(pages, first, last); + return rc; +} + +/* + * A native page pointer has no fragment offset. Require an aligned physical + * base and consecutive fragments within each output page; different output + * pages may be scattered. Only the requested logical interval is retained. + */ +int iopt_fragments_add_access(struct iopt_area *area, unsigned long first, + unsigned long last, struct page **out_pages, + bool lock_area) +{ + struct iopt_pages *pages = area->pages; + unsigned long per_page = 1UL << (PAGE_SHIFT - pages->page_shift); + unsigned long index; + phys_addr_t base = 0; + int rc; + + if (first & (per_page - 1)) + return -EINVAL; + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + for (index = first; index <= last; index++) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + unsigned long offset = (index - first) & (per_page - 1); + phys_addr_t phys = fragment_phys(record); + + if (!offset) { + base = phys; + if (!IS_ALIGNED(base, PAGE_SIZE)) { + rc = -EINVAL; + goto out_release; + } + *out_pages++ = pfn_to_page(PHYS_PFN(base)); + } else if (phys != base + (offset << pages->page_shift)) { + rc = -EINVAL; + goto out_release; + } + cond_resched(); + } + rc = fragments_register_access(area, first, last, lock_area); + if (!rc) + return 0; +out_release: + fragments_release_unused(pages, first, last); + return rc; +} + +void iopt_fragments_unfill_access(struct iopt_pages *pages, unsigned long first, + unsigned long last) +{ + fragments_release_unused(pages, first, last); +} diff --git a/drivers/iommu/iommufd/io_pagetable.c b/drivers/iommu/iommufd/io_pagetable.c index 24d4917105d94..bf8cabee53fb6 100644 --- a/drivers/iommu/iommufd/io_pagetable.c +++ b/drivers/iommu/iommufd/io_pagetable.c @@ -217,7 +217,8 @@ static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area, return -EPERM; area->iommu_prot = iommu_prot; - area->page_offset = start_byte % PAGE_SIZE; + area->page_shift = pages->page_shift; + area->page_offset = start_byte & ((1UL << area->page_shift) - 1); if (area->page_offset & (iopt->iova_alignment - 1)) return -EINVAL; @@ -225,10 +226,10 @@ static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area, if (check_add_overflow(iova, length - 1, &area->node.last)) return -EOVERFLOW; - area->pages_node.start = start_byte / PAGE_SIZE; + area->pages_node.start = start_byte >> area->page_shift; if (check_add_overflow(start_byte, length - 1, &area->pages_node.last)) return -EOVERFLOW; - area->pages_node.last = area->pages_node.last / PAGE_SIZE; + area->pages_node.last >>= area->page_shift; if (WARN_ON(area->pages_node.last >= pages->npages)) return -EOVERFLOW; @@ -572,7 +573,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, struct iommu_dirty_bitmap dirty; struct iova_bitmap_fn_arg arg; struct iova_bitmap *iter; - int ret = 0; + int ret; if (!ops || !ops->read_and_clear_dirty) return -EOPNOTSUPP; @@ -581,7 +582,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, bitmap->page_size, u64_to_user_ptr(bitmap->data)); if (IS_ERR(iter)) - return -ENOMEM; + return PTR_ERR(iter); iommu_dirty_bitmap_init(&dirty, iter, &gather); @@ -589,7 +590,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, arg.iopt = iopt; arg.domain = domain; arg.dirty = &dirty; - iova_bitmap_for_each(iter, &arg, __iommu_read_and_clear_dirty); + ret = iova_bitmap_for_each(iter, &arg, __iommu_read_and_clear_dirty); if (!(flags & IOMMU_DIRTY_NO_CLEAR)) iommu_iotlb_sync(domain, &gather); @@ -615,7 +616,7 @@ int iommufd_check_iova_range(struct io_pagetable *iopt, ((last_iova + 1) & (iommu_pgsize - 1))) return -EINVAL; - if (!bitmap->page_size) + if (!is_power_of_2(bitmap->page_size)) return -EINVAL; if ((bitmap->iova & (bitmap->page_size - 1)) || diff --git a/drivers/iommu/iommufd/io_pagetable.h b/drivers/iommu/iommufd/io_pagetable.h index 27e3e311d395b..d15e51fee0c10 100644 --- a/drivers/iommu/iommufd/io_pagetable.h +++ b/drivers/iommu/iommufd/io_pagetable.h @@ -5,6 +5,7 @@ #ifndef __IO_PAGETABLE_H #define __IO_PAGETABLE_H +#include #include #include #include @@ -45,6 +46,7 @@ struct iopt_area { struct iommu_domain *storage_domain; /* How many bytes into the first page the area starts */ unsigned int page_offset; + u8 page_shift; /* IOMMU_READ, IOMMU_WRITE, etc */ int iommu_prot; bool prevent_access : 1; @@ -107,8 +109,8 @@ static inline size_t iopt_area_length(struct iopt_area *area) /* * Number of bytes from the start of the iopt_pages that the iova begins. - * iopt_area_start_byte() / PAGE_SIZE encodes the starting page index - * iopt_area_start_byte() % PAGE_SIZE encodes the offset within that page + * iopt_area_start_byte() >> area->page_shift encodes the starting page index + * iopt_area_start_byte() % (1UL << area->page_shift) encodes the offset within that page */ static inline unsigned long iopt_area_start_byte(struct iopt_area *area, unsigned long iova) @@ -117,13 +119,13 @@ static inline unsigned long iopt_area_start_byte(struct iopt_area *area, WARN_ON(iova < iopt_area_iova(area) || iova > iopt_area_last_iova(area)); return (iova - iopt_area_iova(area)) + area->page_offset + - iopt_area_index(area) * PAGE_SIZE; + (iopt_area_index(area) << area->page_shift); } static inline unsigned long iopt_area_iova_to_index(struct iopt_area *area, unsigned long iova) { - return iopt_area_start_byte(area, iova) / PAGE_SIZE; + return iopt_area_start_byte(area, iova) >> area->page_shift; } #define __make_iopt_iter(name) \ @@ -214,7 +216,7 @@ struct iopt_pages_dmabuf { * io_pagetable's, through their iopt_area's, can share a single iopt_pages * which avoids multi-pinning and double accounting of page consumption. * - * indexes in this structure are measured in PAGE_SIZE units, are 0 based from + * Indexes use (1UL << page_shift) units and are 0 based from * the start of the uptr and extend to npages. pages are pinned dynamically * according to the intervals in the access_itree and domains_itree, npinned * records the current number of pages pinned. @@ -222,6 +224,8 @@ struct iopt_pages_dmabuf { struct iopt_pages { struct kref kref; struct mutex mutex; + /* Logical indices use page_shift; pin accounting remains native pages. */ + u8 page_shift; size_t npages; size_t npinned; size_t last_npinned; @@ -242,12 +246,73 @@ struct iopt_pages { u8 account_mode; struct xarray pinned_pfns; +#ifdef CONFIG_MM_SUBPAGE + /* Exact fragment pins, indexed independently of device translations. */ + struct xarray fragments; + /* Native physical PFN -> fragment count, for pin accounting. */ + struct xarray fragment_pages; +#endif /* Of iopt_pages_access::node */ struct rb_root_cached access_itree; /* Of iopt_area::pages_node */ struct rb_root_cached domains_itree; }; +static inline bool iopt_pages_use_fragments(struct iopt_pages *pages) +{ + return IS_ENABLED(CONFIG_MM_SUBPAGE) && pages->type == IOPT_ADDRESS_USER && + mm_page_size(pages->source_mm) != PAGE_SIZE; +} + +#ifdef CONFIG_MM_SUBPAGE +int iopt_fragments_fill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain); +int iopt_fragments_fill_domains(struct iopt_area *area, struct iopt_pages *pages); +void iopt_fragments_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain); +int iopt_fragments_add_access(struct iopt_area *area, unsigned long first, + unsigned long last, struct page **out_pages, + bool lock_area); +int iopt_fragments_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys); +void iopt_fragments_unfill_access(struct iopt_pages *pages, unsigned long first, + unsigned long last); +int iopt_fragments_rw(struct iopt_pages *pages, unsigned long start_byte, + void *data, unsigned long length, unsigned int flags); +#else +static inline int iopt_fragments_fill_domain(struct iopt_area *area, + struct iopt_pages *pages, struct iommu_domain *domain) +{ + return -EOPNOTSUPP; +} +static inline int iopt_fragments_fill_domains(struct iopt_area *area, + struct iopt_pages *pages) +{ + return -EOPNOTSUPP; +} +static inline void iopt_fragments_unfill_domain(struct iopt_area *area, + struct iopt_pages *pages, struct iommu_domain *domain) { } +static inline int iopt_fragments_add_access(struct iopt_area *area, + unsigned long first, unsigned long last, struct page **out_pages, + bool lock_area) +{ + return -EOPNOTSUPP; +} +static inline int iopt_fragments_add_phys_access(struct iopt_area *area, + unsigned long start_byte, unsigned long length, phys_addr_t *out_phys) +{ + return -EOPNOTSUPP; +} +static inline void iopt_fragments_unfill_access(struct iopt_pages *pages, + unsigned long first, unsigned long last) { } +static inline int iopt_fragments_rw(struct iopt_pages *pages, + unsigned long start_byte, void *data, unsigned long length, + unsigned int flags) +{ + return -EOPNOTSUPP; +} +#endif + static inline bool iopt_is_dmabuf(struct iopt_pages *pages) { if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) @@ -290,6 +355,8 @@ void iopt_pages_unfill_xarray(struct iopt_pages *pages, unsigned long start, int iopt_area_add_access(struct iopt_area *area, unsigned long start, unsigned long last, struct page **out_pages, unsigned int flags, bool lock_area); +int iopt_area_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys); void iopt_area_remove_access(struct iopt_area *area, unsigned long start, unsigned long last, bool unlock_area); int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h index 43fbc5bed8de3..d1a14e51df55e 100644 --- a/drivers/iommu/iommufd/iommufd_private.h +++ b/drivers/iommu/iommufd/iommufd_private.h @@ -531,6 +531,10 @@ static inline bool iommufd_access_is_internal(struct iommufd_access *access) } struct iommufd_access *iommufd_access_create_internal(struct iommufd_ctx *ictx); +struct iommufd_access *iommufd_access_create_internal_phys(struct iommufd_ctx *ictx); +int iommufd_access_pin_phys(struct iommufd_access *access, unsigned long iova, + unsigned long length, phys_addr_t *out_phys); + static inline void iommufd_access_destroy_internal(struct iommufd_ctx *ictx, diff --git a/drivers/iommu/iommufd/iova_bitmap.c b/drivers/iommu/iommufd/iova_bitmap.c index dac3e657d498d..12e1dd035871a 100644 --- a/drivers/iommu/iommufd/iova_bitmap.c +++ b/drivers/iommu/iommufd/iova_bitmap.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #define BITS_PER_PAGE (PAGE_SIZE * BITS_PER_BYTE) @@ -44,11 +45,16 @@ struct iova_bitmap_map { /* page offset of the first user page pinned */ unsigned long pgoff; - /* number of pages pinned */ + /* number of native pages or typed fragments pinned */ unsigned long npages; /* pinned pages representing the bitmap data */ struct page **pages; +#ifdef CONFIG_MM_SUBPAGE + /* Exact byte ranges, including offsets unrelated to the user VA. */ + struct user_page_fragment *fragments; + unsigned long pinned_bytes; +#endif }; /* @@ -63,7 +69,10 @@ struct iova_bitmap_map { * The bitmap object uses one base page to store all the pinned pages * pointers related to the bitmap. For sizeof(struct page*) == 8 it stores * 512 struct page pointers which, if the base page size is 4K, it means - * 2M of bitmap data is pinned at a time. If the iova_bitmap page size is + * 2M of bitmap data is pinned at a time. Alternative-granule processes use + * one native page of fragment descriptors instead, retaining each physical + * byte offset and bounding the window by the sum of fragment lengths. + * If the iova_bitmap page size is * also 4K then the range window to iterate is 64G. * * For example iterating on a total IOVA range of 4G..128G, it will walk @@ -116,6 +125,9 @@ struct iova_bitmap { /* length of the IOVA range for the whole bitmap */ size_t length; + + /* First lazy pin/write error during an iteration. */ + int error; }; /* @@ -192,6 +204,26 @@ static int iova_bitmap_get(struct iova_bitmap *bitmap) npages = min(npages + !!offset_in_page(addr), PAGE_SIZE / sizeof(struct page *)); +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + unsigned long bytes = bitmap->mapped_total_index - bitmap->mapped_base_index; + unsigned long i; + + ret = pin_user_fragments_remote(current->mm, (unsigned long)addr, + bytes, FOLL_WRITE, mapped->fragments, + PAGE_SIZE / sizeof(*mapped->fragments)); + if (ret <= 0) + return -EFAULT; + mapped->npages = ret; + mapped->pinned_bytes = 0; + for (i = 0; i < mapped->npages; i++) + mapped->pinned_bytes += mapped->fragments[i].length; + mapped->pgoff = 0; + mapped->iova = iova_bitmap_mapped_iova(bitmap); + mapped->length = iova_bitmap_mapped_length(bitmap); + return 0; + } +#endif ret = pin_user_pages_fast((unsigned long)addr, npages, FOLL_WRITE, mapped->pages); if (ret <= 0) @@ -221,7 +253,13 @@ static void iova_bitmap_put(struct iova_bitmap *bitmap) struct iova_bitmap_map *mapped = &bitmap->mapped; if (mapped->npages) { - unpin_user_pages(mapped->pages, mapped->npages); +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + release_user_fragments(mapped->fragments, mapped->npages, true); + mapped->pinned_bytes = 0; + } else +#endif + unpin_user_pages_dirty_lock(mapped->pages, mapped->npages, true); mapped->npages = 0; } } @@ -259,6 +297,16 @@ struct iova_bitmap *iova_bitmap_alloc(unsigned long iova, size_t length, bitmap->iova = iova; bitmap->length = length; mapped->iova = iova; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + mapped->fragments = (void *)__get_free_page(GFP_KERNEL); + if (!mapped->fragments) { + rc = -ENOMEM; + goto err; + } + return bitmap; + } +#endif mapped->pages = (struct page **)__get_free_page(GFP_KERNEL); if (!mapped->pages) { rc = -ENOMEM; @@ -291,6 +339,10 @@ void iova_bitmap_free(struct iova_bitmap *bitmap) mapped->pages = NULL; } +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) + free_page((unsigned long)mapped->fragments); +#endif kfree(bitmap); } EXPORT_SYMBOL_NS_GPL(iova_bitmap_free, "IOMMUFD"); @@ -303,7 +355,12 @@ static unsigned long iova_bitmap_mapped_remaining(struct iova_bitmap *bitmap) { unsigned long remaining, bytes; - bytes = (bitmap->mapped.npages << PAGE_SHIFT) - bitmap->mapped.pgoff; +#ifdef CONFIG_MM_SUBPAGE + if (bitmap->mapped.fragments) + bytes = bitmap->mapped.pinned_bytes; + else +#endif + bytes = (bitmap->mapped.npages << PAGE_SHIFT) - bitmap->mapped.pgoff; remaining = bitmap->mapped_total_index - bitmap->mapped_base_index; remaining = min_t(unsigned long, remaining, @@ -356,16 +413,21 @@ static int iova_bitmap_advance_to(struct iova_bitmap *bitmap, unsigned long iova) { unsigned long index; + int ret = -EINVAL; index = iova_bitmap_offset_to_index(bitmap, iova - bitmap->iova); if (index >= bitmap->mapped_total_index) - return -EINVAL; + goto out; bitmap->mapped_base_index = index; iova_bitmap_put(bitmap); /* Pin the next set of bitmap pages */ - return iova_bitmap_get(bitmap); + ret = iova_bitmap_get(bitmap); +out: + if (ret && !bitmap->error) + bitmap->error = ret; + return ret; } /** @@ -379,15 +441,58 @@ static int iova_bitmap_advance_to(struct iova_bitmap *bitmap, * mapped bitmap user pages into IOVA ranges to process. * * Return: 0 on success, and an error on failure either upon - * iteration or when the callback returns an error. + * iteration, a lazy bitmap write, or when the callback returns an error. */ int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque, iova_bitmap_fn_t fn) { - return fn(bitmap, bitmap->iova, bitmap->length, opaque); + int ret; + + bitmap->error = 0; + ret = fn(bitmap, bitmap->iova, bitmap->length, opaque); + return ret ?: bitmap->error; } EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, "IOMMUFD"); +#ifdef CONFIG_MM_SUBPAGE +static void iova_bitmap_set_fragments(struct iova_bitmap *bitmap, + unsigned long iova, size_t length) +{ + struct iova_bitmap_map *mapped = &bitmap->mapped; + unsigned long bit = (iova - bitmap->iova) >> mapped->pgshift; + unsigned long end = ((iova + length - 1 - bitmap->iova) >> mapped->pgshift) + 1; + + while (bit < end) { + unsigned long base, relative, i; + + iova = bitmap->iova + (bit << mapped->pgshift); + if (!iova_bitmap_mapped_range(mapped, iova, 1) && + iova_bitmap_advance_to(bitmap, iova)) + return; + base = bitmap->mapped_base_index * BITS_PER_BYTE; + relative = bit - base; + for (i = 0; i < mapped->npages && bit < end; i++) { + struct user_page_fragment *fragment = &mapped->fragments[i]; + unsigned long bits = fragment->length * BITS_PER_BYTE; + unsigned long count; + void *addr; + + if (relative >= bits) { + relative -= bits; + continue; + } + count = min(end - bit, bits - relative); + addr = kmap_local_folio(fragment->folio, fragment->offset & PAGE_MASK); + bitmap_set(addr, offset_in_page(fragment->offset) * BITS_PER_BYTE + + relative, count); + kunmap_local(addr); + bit += count; + relative = 0; + } + } +} +#endif + /** * iova_bitmap_set() - Records an IOVA range in bitmap * @bitmap: IOVA bitmap @@ -395,7 +500,8 @@ EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, "IOMMUFD"); * @length: IOVA range length * * Set the bits corresponding to the range [iova .. iova+length-1] in - * the user bitmap. + * the user bitmap. A lazy pin failure is retained for the enclosing + * iova_bitmap_for_each() to report; successful prefix writes are not undone. * */ void iova_bitmap_set(struct iova_bitmap *bitmap, @@ -404,6 +510,13 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, struct iova_bitmap_map *mapped = &bitmap->mapped; unsigned long cur_bit, last_bit, last_page_idx; +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + iova_bitmap_set_fragments(bitmap, iova, length); + return; + } +#endif + update_indexes: if (unlikely(!iova_bitmap_mapped_range(mapped, iova, length))) { /* diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c index 404f31d8f7291..0839878bfb67c 100644 --- a/drivers/iommu/iommufd/pages.c +++ b/drivers/iommu/iommufd/pages.c @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -207,7 +208,7 @@ static unsigned long iopt_area_index_to_iova(struct iopt_area *area, index -= iopt_area_index(area); if (index == 0) return iopt_area_iova(area); - return iopt_area_iova(area) - area->page_offset + index * PAGE_SIZE; + return iopt_area_iova(area) - area->page_offset + (index << area->page_shift); } static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area, @@ -219,7 +220,7 @@ static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area, if (index == iopt_area_last_index(area)) return iopt_area_last_iova(area); return iopt_area_iova(area) - area->page_offset + - (index - iopt_area_index(area) + 1) * PAGE_SIZE - 1; + ((index - iopt_area_index(area) + 1) << area->page_shift) - 1; } static void iommu_unmap_nofail(struct iommu_domain *domain, unsigned long iova, @@ -993,9 +994,11 @@ static int update_mm_locked_vm(struct iopt_pages *pages, unsigned long npages, } else if ((!user || (!user->upages && !user->ufolios)) && pages->source_mm != current->mm) { - if (!mmget_not_zero(pages->source_mm)) + if (mmget_not_zero(pages->source_mm)) + do_put = true; + else if (inc) return -EINVAL; - do_put = true; + /* mmgrab() keeps the lock and counters alive for final uncharge. */ } mmap_write_lock(pages->source_mm); @@ -1378,10 +1381,15 @@ static struct iopt_pages *iopt_alloc_pages(unsigned long start_byte, kref_init(&pages->kref); xa_init_flags(&pages->pinned_pfns, XA_FLAGS_ACCOUNT); +#ifdef CONFIG_MM_SUBPAGE + xa_init_flags(&pages->fragments, XA_FLAGS_ACCOUNT); + xa_init_flags(&pages->fragment_pages, XA_FLAGS_ACCOUNT); +#endif + pages->page_shift = PAGE_SHIFT; mutex_init(&pages->mutex); pages->source_mm = current->mm; mmgrab(pages->source_mm); - pages->npages = DIV_ROUND_UP(length + start_byte, PAGE_SIZE); + pages->npages = ((length - 1 + start_byte) >> PAGE_SHIFT) + 1; pages->access_itree = RB_ROOT_CACHED; pages->domains_itree = RB_ROOT_CACHED; pages->writable = writable; @@ -1400,8 +1408,10 @@ struct iopt_pages *iopt_alloc_user_pages(void __user *uptr, { struct iopt_pages *pages; unsigned long end; + unsigned int shift = min_t(unsigned int, mm_page_shift(current->mm), PAGE_SHIFT); + unsigned long granule = 1UL << shift; void __user *uptr_down = - (void __user *)ALIGN_DOWN((uintptr_t)uptr, PAGE_SIZE); + (void __user *)ALIGN_DOWN((uintptr_t)uptr, granule); if (check_add_overflow((unsigned long)uptr, length, &end)) return ERR_PTR(-EOVERFLOW); @@ -1409,6 +1419,8 @@ struct iopt_pages *iopt_alloc_user_pages(void __user *uptr, pages = iopt_alloc_pages(uptr - uptr_down, length, writable); if (IS_ERR(pages)) return pages; + pages->page_shift = shift; + pages->npages = ((length - 1 + (uptr - uptr_down)) >> shift) + 1; pages->uptr = uptr_down; pages->type = IOPT_ADDRESS_USER; return pages; @@ -1656,6 +1668,12 @@ void iopt_release_pages(struct kref *kref) WARN_ON(!RB_EMPTY_ROOT(&pages->domains_itree.rb_root)); WARN_ON(pages->npinned); WARN_ON(!xa_empty(&pages->pinned_pfns)); +#ifdef CONFIG_MM_SUBPAGE + WARN_ON(!xa_empty(&pages->fragments)); + WARN_ON(!xa_empty(&pages->fragment_pages)); + xa_destroy(&pages->fragments); + xa_destroy(&pages->fragment_pages); +#endif if (iopt_is_dmabuf(pages) && pages->dmabuf.attach) { struct dma_buf *dmabuf = pages->dmabuf.attach->dmabuf; @@ -1832,6 +1850,9 @@ void iopt_area_unmap_domain(struct iopt_area *area, struct iommu_domain *domain) void iopt_area_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, struct iommu_domain *domain) { + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_unfill_domain(area, pages, domain); + if (iopt_dmabuf_revoked(pages)) return; @@ -1855,6 +1876,9 @@ int iopt_area_fill_domain(struct iopt_area *area, struct iommu_domain *domain) lockdep_assert_held(&area->pages->mutex); + if (iopt_pages_use_fragments(area->pages)) + return iopt_fragments_fill_domain(area, area->pages, domain); + if (iopt_dmabuf_revoked(area->pages)) return 0; @@ -1916,6 +1940,9 @@ int iopt_area_fill_domains(struct iopt_area *area, struct iopt_pages *pages) if (xa_empty(&area->iopt->domains)) return 0; + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_fill_domains(area, pages); + mutex_lock(&pages->mutex); if (iopt_is_dmabuf(pages)) { rc = iopt_dmabuf_track_all_domains(area, pages); @@ -2365,6 +2392,10 @@ int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, if (iopt_is_dmabuf(pages)) return -EINVAL; + if (iopt_pages_use_fragments(pages) && change_mm && + !(flags & IOMMUFD_ACCESS_RW_KTHREAD)) + return iopt_fragments_rw(pages, start_byte, data, length, flags); + if (pages->type != IOPT_ADDRESS_USER) return iopt_pages_rw_slow(pages, start_index, last_index, start_byte % PAGE_SIZE, data, length, @@ -2385,11 +2416,14 @@ int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, * ignore any pinning inconsistencies, unlike a real DMA path. */ if (change_mm) { - if (!mmget_not_zero(pages->source_mm)) + if (!mmget_not_zero(pages->source_mm)) { + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_rw(pages, start_byte, data, length, flags); return iopt_pages_rw_slow(pages, start_index, last_index, start_byte % PAGE_SIZE, data, length, flags); + } kthread_use_mm(pages->source_mm); } @@ -2454,6 +2488,11 @@ int iopt_area_add_access(struct iopt_area *area, unsigned long start_index, if (iopt_is_dmabuf(pages)) return -EINVAL; + /* Preserve typed ownership and validate the native output geometry. */ + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_add_access(area, start_index, last_index, + out_pages, lock_area); + mutex_lock(&pages->mutex); access = iopt_pages_get_exact_access(pages, start_index, last_index); if (access) { @@ -2494,6 +2533,47 @@ int iopt_area_add_access(struct iopt_area *area, unsigned long start_index, return rc; } +/* Internal hardware queues consume a contiguous physical byte range. */ +int iopt_area_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_pages *pages = area->pages; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + unsigned long count = last - first + 1, i; + struct page **list; + phys_addr_t base; + int rc; + + if (iopt_is_dmabuf(pages)) + return -EINVAL; + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_add_phys_access(area, start_byte, length, out_phys); + + /* Retain the native page supplier and its accounting/batching rules. */ + list = kvcalloc(count, sizeof(*list), GFP_KERNEL_ACCOUNT); + if (!list) + return -ENOMEM; + rc = iopt_area_add_access(area, first, last, list, 0, true); + if (rc) + goto out_free; + base = PFN_PHYS(page_to_pfn(list[0])); + for (i = 1; i < count; i++) { + if (page_to_pfn(list[i]) != page_to_pfn(list[0]) + i) { + rc = -EFAULT; + goto out_unpin; + } + } + *out_phys = base + offset_in_page(start_byte); + goto out_free; + +out_unpin: + iopt_area_remove_access(area, first, last, true); +out_free: + kvfree(list); + return rc; +} + /** * iopt_area_remove_access() - Release an in-kernel access for PFNs * @area: The source of PFNs @@ -2526,7 +2606,10 @@ void iopt_area_remove_access(struct iopt_area *area, unsigned long start_index, goto out_unlock; interval_tree_remove(&access->node, &pages->access_itree); - iopt_pages_unfill_xarray(pages, start_index, last_index); + if (iopt_pages_use_fragments(pages)) + iopt_fragments_unfill_access(pages, start_index, last_index); + else + iopt_pages_unfill_xarray(pages, start_index, last_index); kfree(access); out_unlock: mutex_unlock(&pages->mutex); diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c index af07c642a5260..146b1fefa0475 100644 --- a/drivers/iommu/iommufd/selftest.c +++ b/drivers/iommu/iommufd/selftest.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -1201,6 +1202,28 @@ static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd, unsigned long pfn; long npages; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + phys_addr_t user_phys; + + npages = get_user_fragments_remote(current->mm, + (uintptr_t)uptr, page_size, 0, &fragment, 1); + if (npages != 1) { + rc = npages < 0 ? npages : -EFAULT; + goto out_put; + } + user_phys = PFN_PHYS(folio_pfn(fragment.folio)) + fragment.offset; + io_phys = iommu_iova_to_phys(&mock->domain, iova); + rc = fragment.length == page_size && io_phys == user_phys ? 0 : -EINVAL; + release_user_fragments(&fragment, 1, false); + if (rc) + goto out_put; + iova += page_size; + uptr += page_size; + continue; + } +#endif npages = get_user_pages_fast((uintptr_t)uptr & PAGE_MASK, 1, 0, pages); if (npages < 0) { @@ -1506,9 +1529,39 @@ static int iommufd_test_access_replace_ioas(struct iommufd_ucmd *ucmd, /* Check that the pages in a page array match the pages in the user VA */ static int iommufd_test_check_pages(void __user *uptr, struct page **pages, - size_t npages) -{ - for (; npages; npages--) { + size_t length) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + unsigned long offset = 0; + + while (offset < length) { + struct user_page_fragment fragment; + unsigned long bytes = min_t(unsigned long, length - offset, + mm_user_fragment_size(current->mm) - + (((unsigned long)uptr + offset) & + (mm_user_fragment_size(current->mm) - 1))); + phys_addr_t actual, expected; + long rc; + + rc = get_user_fragments_remote(current->mm, + (unsigned long)uptr + offset, bytes, 0, &fragment, 1); + if (rc <= 0) + return rc ?: -EFAULT; + actual = PFN_PHYS(folio_pfn(fragment.folio)) + fragment.offset; + expected = PFN_PHYS(page_to_pfn(pages[offset / PAGE_SIZE])) + + offset_in_page(offset); + bytes = fragment.length; + release_user_fragments(&fragment, 1, false); + if (actual != expected || !bytes || + bytes > PAGE_SIZE - offset_in_page(offset)) + return -EBADE; + offset += bytes; + } + return 0; + } +#endif + for (size_t npages = DIV_ROUND_UP(length, PAGE_SIZE); npages; npages--) { struct page *tmp_pages[1]; long rc; @@ -1586,7 +1639,7 @@ static int iommufd_test_access_pages(struct iommufd_ucmd *ucmd, if (uptr) { rc = iommufd_test_check_pages( uptr - (iova - ALIGN_DOWN(iova, PAGE_SIZE)), pages, - npages); + length + offset_in_page(iova)); if (rc) goto out_unaccess; } diff --git a/drivers/iommu/iommufd/viommu.c b/drivers/iommu/iommufd/viommu.c index fc13cf4737ea3..fa8296b15ef59 100644 --- a/drivers/iommu/iommufd/viommu.c +++ b/drivers/iommu/iommufd/viommu.c @@ -263,11 +263,7 @@ static void iommufd_hw_queue_destroy_access(struct iommufd_ctx *ictx, struct iommufd_access *access, u64 base_iova, size_t length) { - u64 aligned_iova = PAGE_ALIGN_DOWN(base_iova); - u64 offset = base_iova - aligned_iova; - - iommufd_access_unpin_pages(access, aligned_iova, - PAGE_ALIGN(length + offset)); + iommufd_access_unpin_pages(access, base_iova, length); iommufd_access_detach_internal(access); iommufd_access_destroy_internal(ictx, access); } @@ -300,66 +296,28 @@ static struct iommufd_access * iommufd_hw_queue_alloc_phys(struct iommu_hw_queue_alloc *cmd, struct iommufd_viommu *viommu, phys_addr_t *base_pa) { - u64 aligned_iova = PAGE_ALIGN_DOWN(cmd->nesting_parent_iova); - u64 offset = cmd->nesting_parent_iova - aligned_iova; struct iommufd_access *access; - struct page **pages; - size_t max_npages; - size_t length; - size_t i; int rc; - /* max_npages = DIV_ROUND_UP(offset + cmd->length, PAGE_SIZE) */ - if (check_add_overflow(offset, cmd->length, &length)) - return ERR_PTR(-ERANGE); - if (check_add_overflow(length, PAGE_SIZE - 1, &length)) + if (cmd->nesting_parent_iova > ULONG_MAX || cmd->length > ULONG_MAX) return ERR_PTR(-ERANGE); - max_npages = length / PAGE_SIZE; - /* length needs to be page aligned too */ - length = max_npages * PAGE_SIZE; - - /* - * Use kvcalloc() to avoid memory fragmentation for a large page array. - * Set __GFP_NOWARN to avoid syzkaller blowups - */ - pages = kvzalloc_objs(*pages, max_npages, GFP_KERNEL | __GFP_NOWARN); - if (!pages) - return ERR_PTR(-ENOMEM); - - access = iommufd_access_create_internal(viommu->ictx); - if (IS_ERR(access)) { - rc = PTR_ERR(access); - goto out_free; - } + access = iommufd_access_create_internal_phys(viommu->ictx); + if (IS_ERR(access)) + return access; rc = iommufd_access_attach_internal(access, viommu->hwpt->ioas); if (rc) goto out_destroy; - - rc = iommufd_access_pin_pages(access, aligned_iova, length, pages, 0); + rc = iommufd_access_pin_phys(access, cmd->nesting_parent_iova, + cmd->length, base_pa); if (rc) goto out_detach; - - /* Validate if the underlying physical pages are contiguous */ - for (i = 1; i < max_npages; i++) { - if (page_to_pfn(pages[i]) == page_to_pfn(pages[i - 1]) + 1) - continue; - rc = -EFAULT; - goto out_unpin; - } - - *base_pa = (page_to_pfn(pages[0]) << PAGE_SHIFT) + offset; - kvfree(pages); return access; -out_unpin: - iommufd_access_unpin_pages(access, aligned_iova, length); out_detach: iommufd_access_detach_internal(access); out_destroy: iommufd_access_destroy_internal(viommu->ictx, access); -out_free: - kvfree(pages); return ERR_PTR(rc); } diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index e6d3a2311a99b..df079de9ee5de 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -28,11 +28,14 @@ #include #include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -84,6 +87,8 @@ struct vfio_domain { bool enforce_cache_coherency : 1; }; +struct vfio_fragment_provider; + struct vfio_dma { struct rb_node node; dma_addr_t iova; /* Device address */ @@ -99,6 +104,7 @@ struct vfio_dma { unsigned long *bitmap; struct mm_struct *mm; size_t locked_vm; + struct vfio_fragment_provider *fragments; }; struct vfio_batch { @@ -128,7 +134,9 @@ struct vfio_pfn { struct rb_node node; dma_addr_t iova; /* Device address */ unsigned long pfn; /* Host pfn */ + unsigned long length; /* Bytes retained in this DMA */ unsigned int ref_count; + bool native_pin; /* separately adopted legacy external PIN */ }; struct vfio_regions { @@ -152,6 +160,9 @@ struct vfio_regions { #define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX) static int put_pfn(unsigned long pfn, int prot); +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn); +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length); static struct vfio_iommu_group* vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, @@ -238,6 +249,12 @@ static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old) } +/* Software dirty granularity must survive domain attach/detach unchanged. */ +static size_t vfio_dirty_pgsize(struct vfio_iommu *iommu) +{ + return IS_ENABLED(CONFIG_MM_SUBPAGE) ? SZ_4K : 1UL << __ffs(iommu->pgsize_bitmap); +} + static int vfio_dma_bitmap_alloc(struct vfio_dma *dma, size_t pgsize) { uint64_t npages = dma->size / pgsize; @@ -272,14 +289,15 @@ static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize) for (p = rb_first(&dma->pfn_list); p; p = rb_next(p)) { struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, node); - bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, 1); + bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, + vpfn->length >> pgshift); } } static void vfio_iommu_populate_bitmap_full(struct vfio_iommu *iommu) { struct rb_node *n; - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); @@ -383,7 +401,7 @@ static void vfio_unlink_pfn(struct vfio_dma *dma, struct vfio_pfn *old) } static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova, - unsigned long pfn) + unsigned long pfn, unsigned long length) { struct vfio_pfn *vpfn; @@ -393,6 +411,7 @@ static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova, vpfn->iova = iova; vpfn->pfn = pfn; + vpfn->length = length; vpfn->ref_count = 1; vfio_link_pfn(dma, vpfn); return 0; @@ -421,8 +440,21 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) vpfn->ref_count--; if (!vpfn->ref_count) { - ret = put_pfn(vpfn->pfn, dma->prot); - vfio_remove_from_pfn_list(dma, vpfn); + if (dma->fragments) { + unsigned long offset = vpfn->iova - dma->iova; + unsigned long length = vpfn->length; + + unsigned long pfn = vpfn->pfn; + bool native_pin = vpfn->native_pin; + + vfio_remove_from_pfn_list(dma, vpfn); + if (native_pin) + vfio_fragments_put_native_pin(dma, pfn); + vfio_fragments_release_range(dma, offset, length); + } else { + ret = put_pfn(vpfn->pfn, dma->prot); + vfio_remove_from_pfn_list(dma, vpfn); + } } return ret; } @@ -430,10 +462,19 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) static int mm_lock_acct(struct task_struct *task, struct mm_struct *mm, bool lock_cap, long npage) { - int ret = mmap_write_lock_killable(mm); + int ret; - if (ret) - return ret; + if (!npage) + return 0; + + /* A released pin must be refunded even if the caller is interrupted. */ + if (npage < 0) { + mmap_write_lock(mm); + } else { + ret = mmap_write_lock_killable(mm); + if (ret) + return ret; + } ret = __account_locked_vm(mm, abs(npage), npage > 0, task, lock_cap); mmap_write_unlock(mm); @@ -443,25 +484,28 @@ static int mm_lock_acct(struct task_struct *task, struct mm_struct *mm, static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) { struct mm_struct *mm; + bool get_mm = async && npage > 0; int ret; if (!npage) return 0; mm = dma->mm; - if (async && !mmget_not_zero(mm)) + if (get_mm && !mmget_not_zero(mm)) return -ESRCH; /* process exited */ ret = mm_lock_acct(dma->task, mm, dma->lock_cap, npage); if (!ret) dma->locked_vm += npage; - if (async) + if (get_mm) mmput(mm); return ret; } +#include "vfio_iommu_type1_fragments.c" + /* * Some mappings aren't backed by a struct page, for example an mmap'd * MMIO range for our own or another device. These use a different @@ -765,7 +809,8 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, */ if (acct_pages) { if (!dma->lock_cap && - mm->locked_vm + lock_acct + acct_pages > limit) { + mm_locked_vm_native_pages(mm) + + lock_acct + acct_pages > limit) { pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__, limit << PAGE_SHIFT); ret = -ENOMEM; @@ -889,6 +934,78 @@ static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, return unlocked; } +/* A native page may contain independently owned fine-grained DMA areas. */ +static unsigned long vfio_unpin_external_range(struct vfio_iommu *iommu, + dma_addr_t start, unsigned long length, + bool do_accounting) +{ + unsigned long done = 0; + + while (done < length) { + dma_addr_t iova = start + done; + struct vfio_dma *dma = vfio_find_dma(iommu, iova, 1); + unsigned long bytes; + + if (!dma || iova < dma->iova) + break; + bytes = min_t(unsigned long, length - done, + PAGE_SIZE - offset_in_page(iova)); + bytes = min(bytes, dma->size - (iova - dma->iova)); + vfio_unpin_page_external(dma, iova, do_accounting); + done += bytes; + } + return done; +} + +static int vfio_pin_external_segment(struct vfio_iommu *iommu, + struct vfio_dma *dma, dma_addr_t iova, unsigned long length, + int prot, bool do_accounting, unsigned long *pfn) +{ + struct vfio_pfn *vpfn; + int ret; + + if ((dma->prot & prot) != prot) + return -EPERM; + vpfn = vfio_iova_get_vfio_pfn(dma, iova); + if (vpfn) { + if (vpfn->length != length) { + vfio_unpin_page_external(dma, iova, do_accounting); + return -EINVAL; + } + *pfn = vpfn->pfn; + } else if (dma->fragments) { + ret = vfio_fragments_external(dma, iova, length, pfn); + if (ret) + return ret; + } else { + /* Ordinary native mappings still use one native GUP per page. */ + if (length != PAGE_SIZE) + return -EINVAL; + ret = vfio_pin_page_external(dma, dma->vaddr + iova - dma->iova, + pfn, do_accounting); + if (ret) + return ret; + if (!pfn_valid(*pfn)) { + ret = -EINVAL; + goto put_native; + } + ret = vfio_add_to_pfn_list(dma, iova, *pfn, length); + if (ret) + goto put_native; + } + if (iommu->dirty_page_tracking) { + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); + + bitmap_set(dma->bitmap, (iova - dma->iova) >> pgshift, + length >> pgshift); + } + return 0; +put_native: + if (put_pfn(*pfn, dma->prot) && do_accounting) + vfio_lock_acct(dma, -1, true); + return ret; +} + static int vfio_iommu_type1_pin_pages(void *iommu_data, struct iommu_group *iommu_group, dma_addr_t user_iova, @@ -897,119 +1014,72 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, { struct vfio_iommu *iommu = iommu_data; struct vfio_iommu_group *group; - int i, j, ret; - unsigned long remote_vaddr; - struct vfio_dma *dma; + unsigned long pinned = 0; bool do_accounting; dma_addr_t iova_end; size_t iova_size; + int i, ret; if (!iommu || !pages || npage <= 0) return -EINVAL; - - /* Supported for v2 version only */ + /* Callers recover the original byte offset from the returned page. */ + user_iova = ALIGN_DOWN(user_iova, PAGE_SIZE); if (!iommu->v2) return -EACCES; - if (check_mul_overflow(npage, PAGE_SIZE, &iova_size) || check_add_overflow(user_iova, iova_size - 1, &iova_end)) return -EOVERFLOW; mutex_lock(&iommu->lock); - if (WARN_ONCE(iommu->vaddr_invalid_count, "vfio_pin_pages not allowed with VFIO_UPDATE_VADDR\n")) { ret = -EBUSY; goto pin_done; } - - /* Fail if no dma_umap notifier is registered */ if (list_empty(&iommu->device_list)) { ret = -EINVAL; goto pin_done; } - - /* - * If iommu capable domain exist in the container then all pages are - * already pinned and accounted. Accounting should be done if there is no - * iommu capable domain in the container. - */ do_accounting = list_empty(&iommu->domain_list); - for (i = 0; i < npage; i++) { - unsigned long phys_pfn; - dma_addr_t iova; - struct vfio_pfn *vpfn; - - iova = user_iova + PAGE_SIZE * i; - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - if (!dma) { - ret = -EINVAL; - goto pin_unwind; - } - - if ((dma->prot & prot) != prot) { - ret = -EPERM; - goto pin_unwind; - } - - vpfn = vfio_iova_get_vfio_pfn(dma, iova); - if (vpfn) { - pages[i] = pfn_to_page(vpfn->pfn); - continue; - } - - remote_vaddr = dma->vaddr + (iova - dma->iova); - ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn, - do_accounting); - if (ret) - goto pin_unwind; - - if (!pfn_valid(phys_pfn)) { - ret = -EINVAL; - goto pin_unwind; - } - - ret = vfio_add_to_pfn_list(dma, iova, phys_pfn); - if (ret) { - if (put_pfn(phys_pfn, dma->prot) && do_accounting) - vfio_lock_acct(dma, -1, true); - goto pin_unwind; - } + unsigned long done = 0; - pages[i] = pfn_to_page(phys_pfn); + pages[i] = NULL; + while (done < PAGE_SIZE) { + dma_addr_t iova = user_iova + PAGE_SIZE * i + done; + struct vfio_dma *dma = vfio_find_dma(iommu, iova, 1); + unsigned long length, pfn; - if (iommu->dirty_page_tracking) { - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); - - /* - * Bitmap populated with the smallest supported page - * size - */ - bitmap_set(dma->bitmap, - (iova - dma->iova) >> pgshift, 1); + if (!dma || iova < dma->iova) { + ret = -EINVAL; + goto pin_unwind; + } + length = min_t(unsigned long, PAGE_SIZE - done, + dma->size - (iova - dma->iova)); + ret = vfio_pin_external_segment(iommu, dma, iova, length, + prot, do_accounting, &pfn); + if (ret) + goto pin_unwind; + pinned += length; + if (pages[i] && page_to_pfn(pages[i]) != pfn) { + ret = -EINVAL; + goto pin_unwind; + } + pages[i] = pfn_to_page(pfn); + done += length; } } - ret = i; - + ret = npage; group = vfio_iommu_find_iommu_group(iommu, iommu_group); if (!group->pinned_page_dirty_scope) { group->pinned_page_dirty_scope = true; iommu->num_non_pinned_groups--; } - goto pin_done; - pin_unwind: - pages[i] = NULL; - for (j = 0; j < i; j++) { - dma_addr_t iova; - - iova = user_iova + PAGE_SIZE * j; - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - vfio_unpin_page_external(dma, iova, do_accounting); - pages[j] = NULL; - } + vfio_unpin_external_range(iommu, user_iova, pinned, do_accounting); + while (i >= 0) + pages[i--] = NULL; pin_done: mutex_unlock(&iommu->lock); return ret; @@ -1019,39 +1089,21 @@ static void vfio_iommu_type1_unpin_pages(void *iommu_data, dma_addr_t user_iova, int npage) { struct vfio_iommu *iommu = iommu_data; - bool do_accounting; dma_addr_t iova_end; size_t iova_size; - int i; - - /* Supported for v2 version only */ - if (WARN_ON(!iommu->v2)) - return; + unsigned long unpinned; - if (WARN_ON(npage <= 0)) + if (WARN_ON(!iommu->v2) || WARN_ON(npage <= 0)) return; - + user_iova = ALIGN_DOWN(user_iova, PAGE_SIZE); if (WARN_ON(check_mul_overflow(npage, PAGE_SIZE, &iova_size) || check_add_overflow(user_iova, iova_size - 1, &iova_end))) return; - mutex_lock(&iommu->lock); - - do_accounting = list_empty(&iommu->domain_list); - for (i = 0; i < npage; i++) { - dma_addr_t iova = user_iova + PAGE_SIZE * i; - struct vfio_dma *dma; - - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - if (!dma) - break; - - vfio_unpin_page_external(dma, iova, do_accounting); - } - + unpinned = vfio_unpin_external_range(iommu, user_iova, iova_size, + list_empty(&iommu->domain_list)); mutex_unlock(&iommu->lock); - - WARN_ON(i != npage); + WARN_ON(unpinned != iova_size); } static long vfio_sync_unpin(struct vfio_dma *dma, struct vfio_domain *domain, @@ -1155,6 +1207,10 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, if (!dma->size) return 0; + if (dma->fragments) { + vfio_fragments_unmap(iommu, dma); + return 0; + } if (list_empty(&iommu->domain_list)) return 0; @@ -1233,6 +1289,7 @@ static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) { WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list)); vfio_unmap_unpin(iommu, dma, true); + vfio_fragments_destroy(dma); vfio_unlink_dma(iommu, dma); put_task_struct(dma->task); mmdrop(dma->mm); @@ -1246,21 +1303,18 @@ static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu) { struct vfio_domain *domain; + unsigned long minimum = SZ_4K; iommu->pgsize_bitmap = ULONG_MAX; - - list_for_each_entry(domain, &iommu->domain_list, next) + list_for_each_entry(domain, &iommu->domain_list, next) { iommu->pgsize_bitmap &= domain->domain->pgsize_bitmap; - - /* - * In case the IOMMU supports page sizes smaller than PAGE_SIZE - * we pretend PAGE_SIZE is supported and hide sub-PAGE_SIZE sizes. - * That way the user will be able to map/unmap buffers whose size/ - * start address is aligned with PAGE_SIZE. Pinning code uses that - * granularity while iommu driver can use the sub-PAGE_SIZE size - * to map the buffer. - */ - if (iommu->pgsize_bitmap & ~PAGE_MASK) { + minimum = max(minimum, 1UL << __ffs(domain->domain->pgsize_bitmap)); + } + if (IS_ENABLED(CONFIG_MM_SUBPAGE)) { + /* Every domain can compose its minimum pages into this common unit. */ + iommu->pgsize_bitmap &= ~(minimum - 1); + iommu->pgsize_bitmap |= minimum; + } else if (iommu->pgsize_bitmap & ~PAGE_MASK) { iommu->pgsize_bitmap &= PAGE_MASK; iommu->pgsize_bitmap |= PAGE_SIZE; } @@ -1367,6 +1421,8 @@ static void vfio_notify_dma_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) { struct vfio_device *device; + dma_addr_t iova = dma->iova; + size_t size = dma->size; if (list_empty(&iommu->device_list)) return; @@ -1377,11 +1433,12 @@ static void vfio_notify_dma_unmap(struct vfio_iommu *iommu, * call back down to this code and try to obtain the iommu->lock we must * drop it. */ - mutex_lock(&iommu->device_list_lock); + /* The range is a snapshot: another unmap may free dma while unlocked. */ mutex_unlock(&iommu->lock); + mutex_lock(&iommu->device_list_lock); list_for_each_entry(device, &iommu->device_list, iommu_entry) - device->ops->dma_unmap(device, dma->iova, dma->size); + device->ops->dma_unmap(device, iova, size); mutex_unlock(&iommu->device_list_lock); mutex_lock(&iommu->lock); @@ -1437,11 +1494,10 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, /* When dirty tracking is enabled, allow only min supported pgsize */ if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && - (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) { + (!iommu->dirty_page_tracking || (bitmap->pgsize != vfio_dirty_pgsize(iommu)))) { goto unlock; } - WARN_ON((pgsize - 1) & PAGE_MASK); again: /* * vfio-iommu-type1 (v1) - User mappings were coalesced together to @@ -1530,7 +1586,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { ret = update_user_bitmap(bitmap->data, iommu, dma, - iova, pgsize); + iova, vfio_dirty_pgsize(iommu)); if (ret) break; } @@ -1587,6 +1643,13 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret = 0; + if (dma->fragments) { + dma->size = map_size; + ret = vfio_fragments_map_all(iommu, dma); + if (ret) + vfio_remove_dma(iommu, dma); + return ret; + } vfio_batch_init(&batch); while (size) { @@ -1644,27 +1707,30 @@ static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu, return list_empty(iova); } -static int vfio_change_dma_owner(struct vfio_dma *dma) +static int vfio_change_dma_mm_pages(struct vfio_dma *dma, struct task_struct *task, + struct mm_struct *mm, bool lock_cap, unsigned long pages) { - struct task_struct *task = current->group_leader; - struct mm_struct *mm = current->mm; - long npage = dma->locked_vm; - bool lock_cap; + long old_pages = dma->locked_vm; int ret; - if (mm == dma->mm) - return 0; - - lock_cap = capable(CAP_IPC_LOCK); - ret = mm_lock_acct(task, mm, lock_cap, npage); + if (mm == dma->mm) { + if (pages == old_pages) + return 0; + ret = mm_lock_acct(task, mm, lock_cap, (long)pages - old_pages); + if (!ret) + dma->locked_vm = pages; + return ret; + } + ret = mm_lock_acct(task, mm, lock_cap, pages); if (ret) return ret; - if (mmget_not_zero(dma->mm)) { - mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -npage); - mmput(dma->mm); + /* Metadata adoption can deduplicate charges, but not original PIN refs. */ + ret = mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -old_pages); + if (ret) { + mm_lock_acct(task, mm, lock_cap, -(long)pages); + return ret; } - if (dma->task != task) { put_task_struct(dma->task); dma->task = get_task_struct(task); @@ -1673,9 +1739,40 @@ static int vfio_change_dma_owner(struct vfio_dma *dma) dma->mm = mm; mmgrab(dma->mm); dma->lock_cap = lock_cap; + dma->locked_vm = pages; return 0; } +static int vfio_change_dma_mm(struct vfio_dma *dma, struct task_struct *task, + struct mm_struct *mm, bool lock_cap) +{ + return vfio_change_dma_mm_pages(dma, task, mm, lock_cap, dma->locked_vm); +} + +static int vfio_change_dma_owner(struct vfio_iommu *iommu, struct vfio_dma *dma, + unsigned long vaddr) +{ + struct vfio_fragment_provider *provider; + int ret; + + if (dma->fragments || (mm_page_size(current->mm) == PAGE_SIZE && + IS_ALIGNED(vaddr, PAGE_SIZE))) + return vfio_change_dma_mm(dma, current->group_leader, current->mm, + capable(CAP_IPC_LOCK)); + + /* Prepare only metadata: failure must preserve every legacy pin/charge. */ + provider = vfio_fragments_prepare_native(iommu, dma); + if (IS_ERR(provider)) + return PTR_ERR(provider); + ret = vfio_change_dma_mm_pages(dma, current->group_leader, current->mm, + capable(CAP_IPC_LOCK), vfio_fragments_native_pages(provider)); + if (ret) + vfio_fragments_abort_native(provider); + else + vfio_fragments_commit_native(dma, provider); + return ret; +} + static int vfio_dma_do_map(struct vfio_iommu *iommu, struct vfio_iommu_type1_dma_map *map) { @@ -1713,8 +1810,6 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); - WARN_ON((pgsize - 1) & PAGE_MASK); - if ((size | iova | vaddr) & (pgsize - 1)) { ret = -EINVAL; goto out_unlock; @@ -1728,7 +1823,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, dma->size != size) { ret = -EINVAL; } else { - ret = vfio_change_dma_owner(dma); + ret = vfio_change_dma_owner(iommu, dma, vaddr); if (ret) goto out_unlock; dma->vaddr = vaddr; @@ -1781,6 +1876,11 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, /* Insert zero-sized and grow as we map chunks of it */ vfio_link_dma(iommu, dma); + ret = vfio_fragments_init(dma, size); + if (ret) { + vfio_remove_dma(iommu, dma); + goto out_unlock; + } /* Don't pin and map if container doesn't contain IOMMU capable domain*/ if (list_empty(&iommu->domain_list)) @@ -1789,7 +1889,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, ret = vfio_pin_map_dma(iommu, dma, size); if (!ret && iommu->dirty_page_tracking) { - ret = vfio_dma_bitmap_alloc(dma, pgsize); + ret = vfio_dma_bitmap_alloc(dma, vfio_dirty_pgsize(iommu)); if (ret) vfio_remove_dma(iommu, dma); } @@ -1822,6 +1922,14 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, size_t pos = 0; dma = rb_entry(n, struct vfio_dma, node); + if (dma->fragments) { + ret = vfio_fragments_acquire(dma, 0, dma->size); + if (!ret) + ret = vfio_fragments_map_domain(dma, domain->domain); + if (ret) + goto unwind; + continue; + } while (pos < dma->size) { dma_addr_t iova = dma->iova + pos; @@ -1906,6 +2014,11 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); size_t pos = 0; + if (dma->fragments) { + iommu_unmap(domain->domain, dma->iova, dma->size); + vfio_fragments_release_unused(dma); + continue; + } if (dma->iommu_mapped) { iommu_unmap(domain->domain, dma->iova, dma->size); continue; @@ -2442,6 +2555,10 @@ static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu) long locked = 0, unlocked = 0; dma = rb_entry(n, struct vfio_dma, node); + if (dma->fragments) { + vfio_unmap_unpin(iommu, dma, true); + continue; + } unlocked += vfio_unmap_unpin(iommu, dma, false); p = rb_first(&dma->pfn_list); for (; p; p = rb_next(p)) { @@ -2646,7 +2763,8 @@ static void *vfio_iommu_type1_open(unsigned long arg) mutex_init(&iommu->lock); mutex_init(&iommu->device_list_lock); INIT_LIST_HEAD(&iommu->device_list); - iommu->pgsize_bitmap = PAGE_MASK; + iommu->pgsize_bitmap = IS_ENABLED(CONFIG_MM_SUBPAGE) ? + GENMASK(BITS_PER_LONG - 1, 12) : PAGE_MASK; INIT_LIST_HEAD(&iommu->emulated_iommu_groups); return iommu; @@ -2812,7 +2930,7 @@ static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu, cap_mig.flags = 0; /* support minimum pgsize */ - cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap); + cap_mig.pgsize_bitmap = vfio_dirty_pgsize(iommu); cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX; return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig)); @@ -2910,6 +3028,63 @@ static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu, return vfio_dma_do_map(iommu, &map); } +#ifdef CONFIG_COMPAT +struct compat_vfio_bitmap { + compat_u64 pgsize; + compat_u64 size; + compat_uptr_t data; +}; +#endif + +static size_t vfio_bitmap_size(void) +{ +#ifdef CONFIG_COMPAT + /* IA32 differs from AArch32/x32 in its u64 alignment and tail padding. */ + if (in_compat_syscall() && compat_need_64bit_alignment_fixup()) + return sizeof(struct compat_vfio_bitmap); +#endif + return sizeof(struct vfio_bitmap); +} + +static size_t vfio_dirty_bitmap_get_size(void) +{ + return offsetof(struct vfio_iommu_type1_dirty_bitmap_get, bitmap) + + vfio_bitmap_size(); +} + +static int vfio_bitmap_from_user(struct vfio_bitmap *bitmap, void __user *user) +{ +#ifdef CONFIG_COMPAT + if (in_compat_syscall()) { + compat_uptr_t data; + unsigned long offset = offsetof(struct vfio_bitmap, data); + + /* Read only the pointer field, never its trailing compat padding. */ + if (copy_from_user(bitmap, user, offset) || + get_user(data, (compat_uptr_t __user *)(user + offset))) + return -EFAULT; + bitmap->data = compat_ptr(data); + return 0; + } +#endif + return copy_from_user(bitmap, user, sizeof(*bitmap)) ? -EFAULT : 0; +} + +static int vfio_dirty_bitmap_get_from_user( + struct vfio_iommu_type1_dirty_bitmap_get *range, void __user *user) +{ +#ifdef CONFIG_COMPAT + if (in_compat_syscall()) { + unsigned long offset = offsetof(typeof(*range), bitmap); + + if (copy_from_user(range, user, offset)) + return -EFAULT; + return vfio_bitmap_from_user(&range->bitmap, user + offset); + } +#endif + return copy_from_user(range, user, sizeof(*range)) ? -EFAULT : 0; +} + static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, unsigned long arg) { @@ -2937,13 +3112,12 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { unsigned long pgshift; - if (unmap.argsz < (minsz + sizeof(bitmap))) + if (unmap.argsz < minsz + vfio_bitmap_size()) return -EINVAL; - if (copy_from_user(&bitmap, - (void __user *)(arg + minsz), - sizeof(bitmap))) - return -EFAULT; + ret = vfio_bitmap_from_user(&bitmap, (void __user *)(arg + minsz)); + if (ret) + return ret; if (!access_ok((void __user *)bitmap.data, bitmap.size)) return -EINVAL; @@ -2992,7 +3166,7 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, size_t pgsize; mutex_lock(&iommu->lock); - pgsize = 1 << __ffs(iommu->pgsize_bitmap); + pgsize = vfio_dirty_pgsize(iommu); if (!iommu->dirty_page_tracking) { ret = vfio_dma_bitmap_alloc_all(iommu, pgsize); if (!ret) @@ -3015,12 +3189,12 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, size_t size, iommu_pgsize; dma_addr_t iova, iova_end; - if (!data_size || data_size < sizeof(range)) + if (data_size < vfio_dirty_bitmap_get_size()) return -EINVAL; - if (copy_from_user(&range, (void __user *)(arg + minsz), - sizeof(range))) - return -EFAULT; + ret = vfio_dirty_bitmap_get_from_user(&range, (void __user *)(arg + minsz)); + if (ret) + return ret; iova = range.iova; size = range.size; @@ -3046,7 +3220,7 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, mutex_lock(&iommu->lock); - iommu_pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); + iommu_pgsize = vfio_dirty_pgsize(iommu); /* allow only smallest supported pgsize */ if (range.bitmap.pgsize != iommu_pgsize) { @@ -3112,11 +3286,12 @@ static void vfio_iommu_type1_register_device(void *iommu_data, * Holding both locks here allows avoiding the device_list_lock in * several fast paths. See vfio_notify_dma_unmap() */ - mutex_lock(&iommu->lock); + /* Unmap callbacks hold device_list_lock while reentering iommu->lock. */ mutex_lock(&iommu->device_list_lock); + mutex_lock(&iommu->lock); list_add(&vdev->iommu_entry, &iommu->device_list); - mutex_unlock(&iommu->device_list_lock); mutex_unlock(&iommu->lock); + mutex_unlock(&iommu->device_list_lock); } static void vfio_iommu_type1_unregister_device(void *iommu_data, @@ -3127,11 +3302,12 @@ static void vfio_iommu_type1_unregister_device(void *iommu_data, if (!vdev->ops->dma_unmap) return; - mutex_lock(&iommu->lock); + /* Unmap callbacks hold device_list_lock while reentering iommu->lock. */ mutex_lock(&iommu->device_list_lock); + mutex_lock(&iommu->lock); list_del(&vdev->iommu_entry); - mutex_unlock(&iommu->device_list_lock); mutex_unlock(&iommu->lock); + mutex_unlock(&iommu->device_list_lock); } static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, @@ -3156,6 +3332,22 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, return -EPERM; mm = dma->mm; + if (dma->fragments) { + int ret; + + if (!kthread && current->mm != mm) + return -EFAULT; + offset = user_iova - dma->iova; + count = min(count, dma->size - offset); + ret = vfio_fragments_rw(dma, offset, data, count, write, copied); + if (*copied && write && iommu->dirty_page_tracking) { + unsigned long shift = __ffs(vfio_dirty_pgsize(iommu)); + + bitmap_set(dma->bitmap, offset >> shift, + ((offset + *copied - 1) >> shift) - (offset >> shift) + 1); + } + return ret; + } if (!mmget_not_zero(mm)) return -EPERM; @@ -3175,7 +3367,7 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, *copied = copy_to_user((void __user *)vaddr, data, count) ? 0 : count; if (*copied && iommu->dirty_page_tracking) { - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); /* * Bitmap populated with the smallest supported page * size @@ -3281,3 +3473,13 @@ MODULE_VERSION(DRIVER_VERSION); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR(DRIVER_AUTHOR); MODULE_DESCRIPTION(DRIVER_DESC); + +#ifdef CONFIG_VFIO_TYPE1_ACCOUNT_KUNIT_TEST +#include "vfio_iommu_type1_account_test.c" +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_ARM64_USER4K_EXPERIMENTAL) +#include "vfio_iommu_type1_fragments_test.c" +#ifdef CONFIG_ARM64_USER4K_VFIO_IOCTL_TEST +#include "vfio_iommu_type1_ioctl_test.c" +#endif +#endif +#endif diff --git a/drivers/vfio/vfio_iommu_type1_fragments.c b/drivers/vfio/vfio_iommu_type1_fragments.c new file mode 100644 index 0000000000000..22d46a1b805a7 --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_fragments.c @@ -0,0 +1,609 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by type1: retained byte ranges, rather than IOMMU PFNs, own pins. */ +#ifdef CONFIG_MM_SUBPAGE +struct vfio_fragment { + struct rb_node node; + unsigned long start; + struct user_page_fragment pin; +}; + +struct vfio_fragment_provider { + struct rb_root records; + struct xarray physical_pages; + unsigned long npinned; +}; + +static phys_addr_t vfio_fragment_phys(const struct user_page_fragment *pin) +{ + return PFN_PHYS(folio_pfn(pin->folio)) + pin->offset; +} + +/* Returns a containing range, or the first range after the requested byte. */ +static struct vfio_fragment *vfio_fragment_find(struct vfio_dma *dma, + unsigned long offset) +{ + struct rb_node *node = dma->fragments->records.rb_node; + struct vfio_fragment *next = NULL; + + while (node) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + + if (offset < record->start) { + next = record; + node = node->rb_left; + } else if (offset - record->start >= record->pin.length) { + node = node->rb_right; + } else { + return record; + } + } + return next; +} + +static int vfio_fragments_init(struct vfio_dma *dma, size_t length) +{ + struct vfio_fragment_provider *provider; + + if (mm_page_size(dma->mm) == PAGE_SIZE && + IS_ALIGNED(dma->vaddr | dma->iova | length, PAGE_SIZE)) + return 0; + provider = kzalloc_obj(*provider, GFP_KERNEL_ACCOUNT); + if (!provider) + return -ENOMEM; + provider->records = RB_ROOT; + xa_init(&provider->physical_pages); + dma->fragments = provider; + return 0; +} + +static int vfio_fragment_store(struct vfio_dma *dma, unsigned long start, + struct user_page_fragment *pin) +{ + struct vfio_fragment_provider *provider = dma->fragments; + struct rb_node **link = &provider->records.rb_node, *parent = NULL; + unsigned long pfn = PHYS_PFN(vfio_fragment_phys(pin)); + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + struct vfio_fragment *record; + int ret; + + record = kmalloc_obj(*record, GFP_KERNEL_ACCOUNT); + if (!record) + return -ENOMEM; + ret = xa_err(xa_store(&provider->physical_pages, pfn, xa_mk_value(count + 1), + GFP_KERNEL_ACCOUNT)); + if (ret) { + kfree(record); + return ret; + } + while (*link) { + struct vfio_fragment *cur = rb_entry(*link, struct vfio_fragment, node); + + parent = *link; + link = start < cur->start ? &(*link)->rb_left : &(*link)->rb_right; + } + record->start = start; + record->pin = *pin; + rb_link_node(&record->node, parent, link); + rb_insert_color(&record->node, &provider->records); + if (!count) + provider->npinned++; + *pin = (struct user_page_fragment) {}; + return 0; +} + +/* These descriptors borrow existing native GUP pins until owner commit. */ +static int vfio_fragment_native_pin(unsigned long pfn, struct user_page_fragment *pin) +{ + struct page *page; + + if (!pfn_valid(pfn)) + return -EOPNOTSUPP; + page = pfn_to_page(pfn); + if (PageReserved(page)) + return -EOPNOTSUPP; + *pin = (struct user_page_fragment) { + .folio = page_folio(page), + .offset = page_to_phys(page) - PFN_PHYS(folio_pfn(page_folio(page))), + .length = PAGE_SIZE, + .pinned = true, + }; + return 0; +} + +/* No PIN, GET or account reference was acquired while preparing metadata. */ +static void vfio_fragments_abort_native(struct vfio_fragment_provider *provider) +{ + struct rb_node *node; + + while ((node = rb_first(&provider->records))) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + + rb_erase(node, &provider->records); + kfree(record); + } + xa_destroy(&provider->physical_pages); + kfree(provider); +} + +static struct vfio_fragment_provider * +vfio_fragments_prepare_native(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_fragment_provider *provider; + struct vfio_dma stage = {}; + struct vfio_domain *domain = NULL; + struct rb_node *node; + int ret; + + if (mm_page_size(dma->mm) != PAGE_SIZE || dma->has_rsvd) + return ERR_PTR(-EOPNOTSUPP); + if (dma->iommu_mapped) { + if (list_empty(&iommu->domain_list)) + return ERR_PTR(-EFAULT); + domain = list_first_entry(&iommu->domain_list, struct vfio_domain, next); + } + provider = kzalloc_obj(*provider, GFP_KERNEL_ACCOUNT); + if (!provider) + return ERR_PTR(-ENOMEM); + provider->records = RB_ROOT; + xa_init(&provider->physical_pages); + stage.fragments = provider; + /* One descriptor owns each original native GUP pin, including aliases. */ + if (domain) { + for (unsigned long offset = 0; offset < dma->size; offset += PAGE_SIZE) { + phys_addr_t phys = iommu_iova_to_phys(domain->domain, dma->iova + offset); + struct user_page_fragment pin; + + if (!phys || !IS_ALIGNED(phys, PAGE_SIZE)) { + ret = -EFAULT; + goto out; + } + ret = vfio_fragment_native_pin(PHYS_PFN(phys), &pin); + if (!ret) + ret = vfio_fragment_store(&stage, offset, &pin); + if (ret) + goto out; + cond_resched(); + } + } + for (node = rb_first(&dma->pfn_list); node; node = rb_next(node)) { + struct vfio_pfn *vpfn = rb_entry(node, struct vfio_pfn, node); + struct user_page_fragment pin; + + ret = vfio_fragment_native_pin(vpfn->pfn, &pin); + if (ret) + goto out; + if (!domain) { + /* Without a domain, the external pin is the cached buffer. */ + ret = vfio_fragment_store(&stage, vpfn->iova - dma->iova, &pin); + } else { + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, + vpfn->pfn)); + + /* The external GUP reference may identify a different page. */ + ret = xa_err(xa_store(&provider->physical_pages, vpfn->pfn, + xa_mk_value(count + 1), GFP_KERNEL_ACCOUNT)); + if (!ret && !count) + provider->npinned++; + } + if (ret) + goto out; + } + return provider; +out: + vfio_fragments_abort_native(provider); + return ERR_PTR(ret); +} + +static unsigned long vfio_fragments_native_pages(struct vfio_fragment_provider *provider) +{ + return provider->npinned; +} + +static void vfio_fragments_commit_native(struct vfio_dma *dma, + struct vfio_fragment_provider *provider) +{ + struct rb_node *node; + + /* Mapped legacy external consumers own separate native GUP references. */ + if (dma->iommu_mapped) { + for (node = rb_first(&dma->pfn_list); node; node = rb_next(node)) + rb_entry(node, struct vfio_pfn, node)->native_pin = true; + } + dma->fragments = provider; +} + +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn) +{ + struct vfio_fragment_provider *provider = dma->fragments; + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + struct user_page_fragment pin; + + if (WARN_ON(!count || vfio_fragment_native_pin(pfn, &pin))) + return; + if (count == 1) { + xa_erase(&provider->physical_pages, pfn); + provider->npinned--; + } else { + xa_store(&provider->physical_pages, pfn, xa_mk_value(count - 1), GFP_NOWAIT); + } + release_user_fragments(&pin, 1, dma->prot & IOMMU_WRITE); + if (provider->npinned < dma->locked_vm) + vfio_lock_acct(dma, -(long)(dma->locked_vm - provider->npinned), true); +} + +static bool vfio_fragments_external_overlap(struct vfio_dma *dma, + dma_addr_t start, unsigned long length) +{ + struct rb_node *node = dma->pfn_list.rb_node; + + while (node) { + struct vfio_pfn *vpfn = rb_entry(node, struct vfio_pfn, node); + + if (start + length <= vpfn->iova) + node = node->rb_left; + else if (start >= vpfn->iova + vpfn->length) + node = node->rb_right; + else + return true; + } + return false; +} + +/* Device unmaps and their IOTLB synchronization must precede this call. */ +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + struct vfio_fragment_provider *provider = dma->fragments; + struct vfio_fragment *first = vfio_fragment_find(dma, start); + struct rb_node *node = first ? &first->node : NULL; + unsigned long end = start + length; + + if (dma->iommu_mapped) + return; + while (node) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + dma_addr_t iova = dma->iova + record->start; + unsigned long pfn, count; + + if (record->start >= end) + break; + node = rb_next(node); + /* Native-page consumers retain all records overlapping their page. */ + if (vfio_fragments_external_overlap(dma, iova, record->pin.length)) + continue; + pfn = PHYS_PFN(vfio_fragment_phys(&record->pin)); + count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + if (WARN_ON(!count)) + continue; + if (count == 1) { + xa_erase(&provider->physical_pages, pfn); + provider->npinned--; + } else { + xa_store(&provider->physical_pages, pfn, xa_mk_value(count - 1), + GFP_NOWAIT); + } + rb_erase(&record->node, &provider->records); + release_user_fragments(&record->pin, 1, dma->prot & IOMMU_WRITE); + kfree(record); + cond_resched(); + } + if (provider->npinned < dma->locked_vm) + vfio_lock_acct(dma, -(long)(dma->locked_vm - provider->npinned), true); +} + +static void vfio_fragments_release_unused(struct vfio_dma *dma) +{ + vfio_fragments_release_range(dma, 0, dma->size); +} + +/* Cached records survive source VA replacement and the last source mm user. */ +static int vfio_fragments_acquire(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + unsigned long end = start + length, original = start; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + struct user_page_fragment *batch = NULL; + bool live = false; + int ret = 0; + + while (start < end) { + struct vfio_fragment *record = vfio_fragment_find(dma, start); + unsigned long stop = end; + long count; + + if (record && record->start <= start) { + start = record->start + record->pin.length; + continue; + } + if (record) + stop = min(stop, record->start); + if (!live) { + if (!mmget_not_zero(dma->mm)) { + ret = -EFAULT; + goto out; + } + live = true; + batch = kmalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!batch) { + ret = -ENOMEM; + goto out; + } + } + count = pin_user_fragments_remote(dma->mm, dma->vaddr + start, + stop - start, FOLL_LONGTERM | (dma->prot & IOMMU_WRITE ? FOLL_WRITE : 0), + batch, capacity); + if (count <= 0) { + ret = count ?: -EFAULT; + goto out; + } + for (unsigned long i = 0; i < count; i++) { + unsigned long bytes = batch[i].length; + + if (WARN_ON(!bytes || bytes > stop - start || + offset_in_page(vfio_fragment_phys(&batch[i])) > + PAGE_SIZE - bytes)) + ret = -EFAULT; + else + ret = vfio_fragment_store(dma, start, &batch[i]); + if (ret) { + release_user_fragments(batch + i, count - i, false); + goto out; + } + start += bytes; + } + cond_resched(); + } + if (dma->fragments->npinned > dma->locked_vm) + ret = vfio_lock_acct(dma, dma->fragments->npinned - dma->locked_vm, false); +out: + kfree(batch); + if (live) + mmput(dma->mm); + if (ret) + vfio_fragments_release_range(dma, original, length); + return ret; +} + +/* Merge physical byte runs while retaining each original pin identity. */ +static int vfio_fragments_map_domain(struct vfio_dma *dma, struct iommu_domain *domain) +{ + unsigned long done = 0, min_size = 1UL << __ffs(domain->pgsize_bitmap); + int ret = 0; + + while (done < dma->size) { + struct vfio_fragment *record = vfio_fragment_find(dma, done); + unsigned long bytes, limit = dma->size - done; + phys_addr_t phys; + + if (WARN_ON(!record || record->start > done)) { + ret = -EFAULT; + goto out; + } + if (disable_hugepages) + limit = min(limit, PAGE_SIZE); + phys = vfio_fragment_phys(&record->pin) + done - record->start; + bytes = min(record->pin.length - (done - record->start), limit); + while (bytes < limit) { + record = vfio_fragment_find(dma, done + bytes); + if (!record || record->start != done + bytes || + vfio_fragment_phys(&record->pin) != phys + bytes) + break; + bytes += min_t(unsigned long, record->pin.length, limit - bytes); + } + if (!IS_ALIGNED((dma->iova + done) | phys | bytes, min_size)) { + ret = -EINVAL; + goto out; + } + ret = iommu_map(domain, dma->iova + done, phys, bytes, + dma->prot | IOMMU_CACHE, GFP_KERNEL_ACCOUNT); + if (ret) + goto out; + done += bytes; + cond_resched(); + } +out: + if (ret && done) + iommu_unmap(domain, dma->iova, done); + return ret; +} + +static int vfio_fragments_map_all(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_domain *domain; + int ret; + + ret = vfio_fragments_acquire(dma, 0, dma->size); + if (ret) + return ret; + list_for_each_entry(domain, &iommu->domain_list, next) { + ret = vfio_fragments_map_domain(dma, domain->domain); + if (ret) + goto unwind; + } + dma->iommu_mapped = true; + return 0; +unwind: + list_for_each_entry_continue_reverse(domain, &iommu->domain_list, next) + iommu_unmap(domain->domain, dma->iova, dma->size); + vfio_fragments_release_unused(dma); + return ret; +} + +static void vfio_fragments_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_domain *domain; + + if (dma->iommu_mapped) { + list_for_each_entry(domain, &iommu->domain_list, next) + WARN_ON(iommu_unmap(domain->domain, dma->iova, dma->size) != dma->size); + dma->iommu_mapped = false; + } + vfio_fragments_release_unused(dma); +} + +static void vfio_fragments_destroy(struct vfio_dma *dma) +{ + if (!dma->fragments) + return; + vfio_fragments_release_unused(dma); + if (WARN_ON(!RB_EMPTY_ROOT(&dma->fragments->records))) + return; + xa_destroy(&dma->fragments->physical_pages); + kfree(dma->fragments); + dma->fragments = NULL; +} + +/* The output page has no offset field: validate every byte and its native offset. */ +static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, + unsigned long length, unsigned long *pfn) +{ + unsigned long first = iova - dma->iova, done = 0; + phys_addr_t phys = 0; + int ret; + + ret = vfio_fragments_acquire(dma, first, length); + if (ret) + return ret; + while (done < length) { + struct vfio_fragment *record = vfio_fragment_find(dma, first + done); + phys_addr_t next; + unsigned long offset; + + if (WARN_ON(!record || record->start > first + done)) { + ret = -EFAULT; + goto out; + } + offset = first + done - record->start; + next = vfio_fragment_phys(&record->pin) + offset; + if (!done) + phys = next; + if (offset_in_page(phys) != offset_in_page(iova) || next != phys + done) { + ret = -EINVAL; + goto out; + } + done += min_t(unsigned long, record->pin.length - offset, length - done); + } + *pfn = PHYS_PFN(phys); + ret = vfio_add_to_pfn_list(dma, iova, *pfn, length); +out: + if (ret) + vfio_fragments_release_range(dma, first, length); + return ret; +} + +/* CPU-only missing ranges take GET references, not new long-term DMA pins. */ +static int vfio_fragments_rw(struct vfio_dma *dma, unsigned long offset, void *data, + unsigned long length, bool write, size_t *copied) +{ + bool live = false; + int ret = 0; + + while (length) { + struct vfio_fragment *record = vfio_fragment_find(dma, offset); + struct user_page_fragment temp = {}, *pin; + unsigned long in_record = 0, bytes = length; + void *addr; + + if (record && record->start <= offset) { + pin = &record->pin; + in_record = offset - record->start; + } else { + long count; + + if (record) + bytes = min(bytes, record->start - offset); + if (!live) { + if (!mmget_not_zero(dma->mm)) { + ret = -EFAULT; + break; + } + live = true; + } + count = get_user_fragments_remote(dma->mm, dma->vaddr + offset, bytes, + write ? FOLL_WRITE : 0, &temp, 1); + if (count <= 0) { + ret = count ?: -EFAULT; + break; + } + pin = &temp; + } + bytes = min_t(unsigned long, bytes, pin->length - in_record); + addr = kmap_local_folio(pin->folio, pin->offset + in_record); + if (write) + memcpy(addr, data, bytes); + else + memcpy(data, addr, bytes); + kunmap_local(addr); + if (write && record && record->start <= offset) + mark_user_fragment_dirty(pin); + release_user_fragments(&temp, 1, write); + data += bytes; + offset += bytes; + length -= bytes; + *copied += bytes; + } + if (live) + mmput(dma->mm); + return ret; +} +#else +static struct vfio_fragment_provider * +vfio_fragments_prepare_native(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static void vfio_fragments_abort_native(struct vfio_fragment_provider *provider) +{ +} +static unsigned long vfio_fragments_native_pages(struct vfio_fragment_provider *provider) +{ + return 0; +} +static void vfio_fragments_commit_native(struct vfio_dma *dma, + struct vfio_fragment_provider *provider) +{ +} +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn) +{ +} +static int vfio_fragments_init(struct vfio_dma *dma, size_t length) +{ + return 0; +} +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ +} +static void vfio_fragments_release_unused(struct vfio_dma *dma) +{ +} +static int vfio_fragments_acquire(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_map_domain(struct vfio_dma *dma, struct iommu_domain *domain) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_map_all(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + return -EOPNOTSUPP; +} +static void vfio_fragments_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ +} +static void vfio_fragments_destroy(struct vfio_dma *dma) +{ +} +static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, + unsigned long length, unsigned long *pfn) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_rw(struct vfio_dma *dma, unsigned long offset, void *data, + unsigned long length, bool write, size_t *copied) +{ + return -EOPNOTSUPP; +} +#endif From d2bfa3d271d01d3204306eb4c8cb6875a1310e35 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0427/1579] arm64: support smaller userspace translation granules per mm Keep kernel mappings at their native granule while selecting TTBR0 geometry per address space. Add capability checks, table-fragment lifetime, TLB invalidation and exec selection for 4K on 16K/64K and 16K on 64K. The backend remains opt-in; private launch commands are experimental. Signed-off-by: Diljit Singh --- arch/arm64/Kconfig | 1 + arch/arm64/Kconfig.debug | 21 +++ arch/arm64/include/asm/cpucaps.h | 5 + arch/arm64/include/asm/elf.h | 9 ++ arch/arm64/include/asm/kernel-pgtable.h | 2 +- arch/arm64/include/asm/mmu.h | 40 ++++++ arch/arm64/include/asm/mmu_context.h | 61 +++++++- arch/arm64/include/asm/mte.h | 8 ++ arch/arm64/include/asm/page.h | 33 ++++- arch/arm64/include/asm/pgalloc.h | 35 ++++- arch/arm64/include/asm/pgtable-hwdef.h | 12 +- arch/arm64/include/asm/pgtable.h | 51 +++++-- arch/arm64/include/asm/shmparam.h | 6 + arch/arm64/include/asm/tlb.h | 27 +++- arch/arm64/include/asm/tlbflush.h | 124 +++++++++++------ arch/arm64/include/asm/user-pgalloc.h | 59 ++++++++ arch/arm64/include/asm/user-pgtable.h | 164 ++++++++++++++++++++++ arch/arm64/include/asm/user-pte.h | 116 ++++++++++++++++ arch/arm64/kernel/cpufeature.c | 83 ++++++++++- arch/arm64/kernel/pi/idreg-override.c | 32 +++++ arch/arm64/kernel/pi/map_kernel.c | 2 +- arch/arm64/kernel/process.c | 33 ++++- arch/arm64/kernel/sys.c | 5 + arch/arm64/kernel/sys32.c | 6 + arch/arm64/kernel/vmlinux.lds.S | 5 +- arch/arm64/lib/mte.S | 46 +++++++ arch/arm64/mm/Makefile | 1 + arch/arm64/mm/context.c | 43 ++++++ arch/arm64/mm/contpte.c | 143 +++++++++++-------- arch/arm64/mm/copypage.c | 52 +++++++ arch/arm64/mm/fault.c | 30 ++-- arch/arm64/mm/fixmap.c | 2 +- arch/arm64/mm/init.c | 5 +- arch/arm64/mm/mmu.c | 4 +- arch/arm64/mm/pgd.c | 41 +++++- arch/arm64/mm/proc.S | 16 +-- arch/arm64/mm/user4k-pgalloc.c | 176 ++++++++++++++++++++++++ arch/arm64/tools/cpucaps | 2 + 38 files changed, 1349 insertions(+), 152 deletions(-) create mode 100644 arch/arm64/include/asm/user-pgalloc.h create mode 100644 arch/arm64/include/asm/user-pgtable.h create mode 100644 arch/arm64/include/asm/user-pte.h create mode 100644 arch/arm64/mm/user4k-pgalloc.c diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 10c69474f2761..3c1872f790aac 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -362,6 +362,7 @@ config FIX_EARLYCON_MEM config PGTABLE_LEVELS int + default 4 if ARM64_USER4K_EXPERIMENTAL default 2 if ARM64_16K_PAGES && ARM64_VA_BITS_36 default 2 if ARM64_64K_PAGES && ARM64_VA_BITS_42 default 3 if ARM64_64K_PAGES && (ARM64_VA_BITS_48 || ARM64_VA_BITS_52) diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 265c4461031f4..d15ccdeee2125 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -21,3 +21,24 @@ config ARM64_RELOC_TEST tristate "Relocation testing module" source "drivers/hwtracing/coresight/Kconfig" + +config ARM64_USER4K_EXPERIMENTAL + bool "Experimental mixed-granule userspace" + select ARCH_HAS_MM_PGTABLE + select ARCH_HAS_USER_PAGE_SIZE + select MM_SUBPAGE + select MM_ID + select HAVE_ARCH_TLB_REMOVE_TABLE + select ARCH_SUPPORTS_PT_RECLAIM + depends on (ARM64_16K_PAGES || ARM64_64K_PAGES) && ARM64_VA_BITS_48 && ARM64_PA_BITS_48 + depends on !ARM64_LPA2 && !NO_PAGE_MAPCOUNT + default n + help + Build the in-development MM support for alternative 4K user tables + on 16K/64K kernels and 16K user tables on 64K kernels. + Ordinary address spaces and kernel tables retain the native granule. An experimental + one-shot prctl selects a supported granule for the next exec. + This is a development ABI: several MM interfaces and device mappings + still require integration and validation. The private prctl numbers + are not upstream assignments or a stable ABI promise. + Say N for normal kernels. diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index d0d3cdd5763ca..fff3a9570ad65 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -19,6 +19,11 @@ cpucap_is_possible(const unsigned int cap) "cap must be < ARM64_NCAPS"); switch (cap) { + case ARM64_HAS_USER16K_GRANULE: + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && + IS_ENABLED(CONFIG_ARM64_64K_PAGES); + case ARM64_HAS_USER4K_GRANULE: + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL); case ARM64_HAS_EPAN: return IS_ENABLED(CONFIG_ARM64_EPAN); case ARM64_SVE: diff --git a/arch/arm64/include/asm/elf.h b/arch/arm64/include/asm/elf.h index d2779d604c7be..23320b49c3208 100644 --- a/arch/arm64/include/asm/elf.h +++ b/arch/arm64/include/asm/elf.h @@ -121,6 +121,9 @@ #define CORE_DUMP_USE_REGSET #define ELF_EXEC_PAGESIZE PAGE_SIZE +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define ELF_EXEC_PAGESIZE_MM(mm) mm_page_size(mm) +#endif /* * This is the base location for PIE (ET_DYN with INTERP) loads. On @@ -181,6 +184,12 @@ do { \ NEW_AUX_ENT(AT_IGNORE, 0); \ } while (0) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +struct linux_binprm; +int arm64_bprm_prepare_mm(struct linux_binprm *bprm, bool compat); +#define arch_bprm_prepare_mm arm64_bprm_prepare_mm +#endif + #define ARCH_HAS_SETUP_ADDITIONAL_PAGES struct linux_binprm; extern int arch_setup_additional_pages(struct linux_binprm *bprm, diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h index 229ee7976f693..2bd73e773095c 100644 --- a/arch/arm64/include/asm/kernel-pgtable.h +++ b/arch/arm64/include/asm/kernel-pgtable.h @@ -27,7 +27,7 @@ #endif #define SWAPPER_BLOCK_SIZE (UL(1) << SWAPPER_BLOCK_SHIFT) -#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - SWAPPER_SKIP_LEVEL) +#define SWAPPER_PGTABLE_LEVELS (ARM64_NATIVE_PGTABLE_LEVELS - SWAPPER_SKIP_LEVEL) #define INIT_IDMAP_PGTABLE_LEVELS (IDMAP_LEVELS - SWAPPER_SKIP_LEVEL) #define IDMAP_VA_BITS 48 diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 5e1211c540ab2..15943bec53392 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -8,6 +8,10 @@ #include #define MMCF_AARCH32 0x1 /* mm context flag for AArch32 executables */ +#define MMCF_USER_4K 0x2 /* experimental, immutable mm table geometry */ +#define MMCF_USER_16K 0x4 /* experimental, immutable mm table geometry */ +#define MMCF_USER_64K 0x8 /* internal geometry; larger user ABI not enabled */ +#define MMCF_USER_GRANULE (MMCF_USER_4K | MMCF_USER_16K | MMCF_USER_64K) #define USER_ASID_BIT 48 #define USER_ASID_FLAG (UL(1) << USER_ASID_BIT) @@ -16,6 +20,15 @@ #include #include +/* AArch32 falls back to the native ABI when 4K translation is unavailable. */ +static inline unsigned int arm64_exec_page_shift(bool compat, bool has_4k, + unsigned int native_shift, unsigned int requested_shift) +{ + if (compat) + return has_4k ? 12 : native_shift; + return requested_shift ?: native_shift; +} + typedef struct { atomic64_t id; #ifdef CONFIG_COMPAT @@ -25,8 +38,35 @@ typedef struct { void *vdso; unsigned long flags; u8 pkey_allocation_map; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* Unissued PTE/PMD/PUD fragments; protected by page_table_lock. */ + void *user4k_pt_frag[3]; +#endif } mm_context_t; +static inline bool arm64_context_user4k(const mm_context_t *context) +{ + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && + (context->flags & MMCF_USER_4K); +} + +static inline unsigned int arm64_context_page_shift(const mm_context_t *context) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL)) { + if (context->flags & MMCF_USER_4K) + return 12; + if (context->flags & MMCF_USER_16K) + return 14; + if (context->flags & MMCF_USER_64K) + return 16; + } + return PAGE_SHIFT; +} + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_mm_page_shift(mm) arm64_context_page_shift(&(mm)->context) +#endif + /* * We use atomic64_read() here because the ASID for an 'mm_struct' can * be reallocated when scheduling one of its threads following a diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index 803b687581527..db27ff606555b 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -26,6 +26,17 @@ #include #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_mm_init_exec arch_mm_init_exec +static inline void arch_mm_init_exec(struct mm_struct *mm, unsigned int page_shift) +{ + if (page_shift == 12) + mm->context.flags |= MMCF_USER_4K; + else if (page_shift == 14 && PAGE_SHIFT > 14) + mm->context.flags |= MMCF_USER_16K; +} +#endif + extern bool rodata_full; static inline void contextidr_thread_switch(struct task_struct *next) @@ -53,6 +64,29 @@ static inline void cpu_set_reserved_ttbr0(void) isb(); } +/* + * The caller has installed reserved_pg_dir in TTBR0 and excludes preemption. + * On a granule transition, complete that installation before changing TG0, + * then synchronise the new walk geometry before installing a user root. + * ASIDs remain unique across both granules; switching does not flush them. + */ +static inline void cpu_set_mm_tg0(struct mm_struct *mm) +{ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + unsigned long tcr = read_sysreg(tcr_el1); + unsigned int shift = mm_page_shift(mm); + unsigned long tg0 = shift == 12 ? TCR_TG0_4K : + (shift == 14 ? TCR_TG0_16K : TCR_TG0_64K); + + if ((tcr & TCR_EL1_TG0_MASK) == tg0) + return; + + isb(); + write_sysreg((tcr & ~TCR_EL1_TG0_MASK) | tg0, tcr_el1); + isb(); +#endif +} + void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm) @@ -62,16 +96,21 @@ static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm) } /* - * Ensure TCR.T0SZ is set to the provided value. + * Set TCR.T0SZ and restore native TG0 for the idmap/transition tables. */ static inline void __cpu_set_tcr_t0sz(unsigned long t0sz) { unsigned long tcr = read_sysreg(tcr_el1); + unsigned long mask = TCR_EL1_T0SZ_MASK; - if ((tcr & TCR_EL1_T0SZ_MASK) == t0sz) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + mask |= TCR_EL1_TG0_MASK; + t0sz |= IS_ENABLED(CONFIG_ARM64_64K_PAGES) ? TCR_TG0_64K : TCR_TG0_16K; +#endif + if ((tcr & mask) == t0sz) return; - tcr &= ~TCR_EL1_T0SZ_MASK; + tcr &= ~mask; tcr |= t0sz; write_sysreg(tcr, tcr_el1); isb(); @@ -97,6 +136,8 @@ static inline void cpu_uninstall_idmap(void) local_flush_tlb_all(); __cpu_set_tcr_t0sz(TCR_T0SZ(vabits_actual)); + /* SW PAN will restore TTBR0 later, but already needs the mm's TG0. */ + cpu_set_mm_tg0(mm); if (mm != &init_mm && !system_uses_ttbr0_pan()) cpu_switch_mm(mm->pgd, mm); } @@ -168,6 +209,9 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) { atomic64_set(&mm->context.id, 0); refcount_set(&mm->context.pinned, 0); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + memset(mm->context.user4k_pt_frag, 0, sizeof(mm->context.user4k_pt_frag)); +#endif /* pkey 0 is the default, so always reserve it. */ mm->context.pkey_allocation_map = BIT(0); @@ -175,6 +219,16 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) return 0; } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm); + +static inline void destroy_context(struct mm_struct *mm) +{ + arm64_user4k_pt_cache_destroy(mm); +} +#define destroy_context destroy_context +#endif + static inline void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm) { @@ -241,6 +295,7 @@ static inline void __switch_mm(struct mm_struct *next) */ if (next == &init_mm) { cpu_set_reserved_ttbr0(); + cpu_set_mm_tg0(next); return; } diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 7f7b97e099968..890d49a4398b2 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -98,6 +98,8 @@ static inline bool try_page_mte_tagging(struct page *page) void mte_zero_clear_page_tags(void *addr); void mte_sync_tags(pte_t pte, unsigned int nr_pages); void mte_copy_page_tags(void *kto, const void *kfrom); +void mte_copy_subpage_tags_range(void *kto, const void *kfrom, unsigned long size); +void mte_clear_subpage_tags_range(void *addr, unsigned long size); void mte_thread_init_user(void); void mte_thread_switch(struct task_struct *next); void mte_cpu_setup(void); @@ -134,6 +136,12 @@ static inline void mte_sync_tags(pte_t pte, unsigned int nr_pages) static inline void mte_copy_page_tags(void *kto, const void *kfrom) { } +static inline void mte_copy_subpage_tags_range(void *kto, const void *kfrom, unsigned long size) +{ +} +static inline void mte_clear_subpage_tags_range(void *addr, unsigned long size) +{ +} static inline void mte_thread_init_user(void) { } diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h index 58200de8a221a..96fa9d8ea23c5 100644 --- a/arch/arm64/include/asm/page.h +++ b/arch/arm64/include/asm/page.h @@ -10,6 +10,11 @@ #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +/* A shared zero mapping must cover the largest supported user granule. */ +#define ARCH_ZERO_PAGE_SIZE (1UL << 16) +#endif + #ifndef __ASSEMBLER__ #include /* for READ_IMPLIES_EXEC */ @@ -26,11 +31,28 @@ void copy_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma); #define __HAVE_ARCH_COPY_USER_HIGHPAGE +#ifdef CONFIG_MM_SUBPAGE +void clear_user_subpage_range(struct page *page, unsigned int offset, + unsigned int size); +int copy_user_subpage_range(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset, + unsigned int size); +void clear_user_subpage(struct page *page, unsigned int offset); +int copy_user_subpage(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset); +#endif + void copy_highpage(struct page *to, struct page *from); #define __HAVE_ARCH_COPY_HIGHPAGE -struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, - unsigned long vaddr); +struct folio *vma_alloc_zeroed_movable_folio_order(struct vm_area_struct *vma, + unsigned long vaddr, unsigned int order); +#define vma_alloc_zeroed_movable_folio_order vma_alloc_zeroed_movable_folio_order +static inline struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, + unsigned long vaddr) +{ + return vma_alloc_zeroed_movable_folio_order(vma, vaddr, 0); +} #define vma_alloc_zeroed_movable_folio vma_alloc_zeroed_movable_folio bool tag_clear_highpages(struct page *to, int numpages, bool clear_pages); @@ -38,7 +60,14 @@ bool tag_clear_highpages(struct page *to, int numpages, bool clear_pages); #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +/* A table may occupy a 4K fragment of a native 16K physical page. */ +typedef pte_t *pgtable_t; +#define pgtable_to_page(table) virt_to_page(table) +#define pgtable_from_page(page) ((page) ? (pgtable_t)page_address(page) : NULL) +#else typedef struct page *pgtable_t; +#endif int pfn_is_map_memory(unsigned long pfn); diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h index 1b4509d3382c6..41b81203e575c 100644 --- a/arch/arm64/include/asm/pgalloc.h +++ b/arch/arm64/include/asm/pgalloc.h @@ -15,7 +15,15 @@ #define __HAVE_ARCH_PGD_FREE #define __HAVE_ARCH_PUD_FREE +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define __HAVE_ARCH_PTE_ALLOC_ONE +#define __HAVE_ARCH_PTE_FREE +#define __HAVE_ARCH_PMD_ALLOC_ONE +#define __HAVE_ARCH_PMD_FREE +#define __HAVE_ARCH_PUD_ALLOC_ONE +#endif #include +#include #define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) @@ -31,6 +39,12 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_AF; pudval |= (mm == &init_mm) ? PUD_TABLE_UXN : PUD_TABLE_PXN; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_pud(pudp, __pud(__pa(pmdp) | pudval)); + return; + } +#endif __pud_populate(pudp, __pa(pmdp), pudval); } #else @@ -53,14 +67,24 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_AF; p4dval |= (mm == &init_mm) ? P4D_TABLE_UXN : P4D_TABLE_PXN; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_p4d(p4dp, __p4d(__pa(pudp) | p4dval)); + return; + } +#endif __p4d_populate(p4dp, __pa(pudp), p4dval); } static inline void pud_free(struct mm_struct *mm, pud_t *pud) { - if (!pgtable_l4_enabled()) + if (mm_pud_folded(mm)) return; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + arm64_pgtable_free(pud); +#else __pud_free(mm, pud); +#endif } #else static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) @@ -117,7 +141,16 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) { VM_BUG_ON(mm == &init_mm); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_pmd(pmdp, __pmd(__pa(ptep) | PMD_TYPE_TABLE | + PMD_TABLE_AF | PMD_TABLE_PXN)); + return; + } + __pmd_populate(pmdp, __pa(ptep), +#else __pmd_populate(pmdp, page_to_phys(ptep), +#endif PMD_TYPE_TABLE | PMD_TABLE_AF | PMD_TABLE_PXN); } diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h index 72f31800c7030..cb211cbcdb7fd 100644 --- a/arch/arm64/include/asm/pgtable-hwdef.h +++ b/arch/arm64/include/asm/pgtable-hwdef.h @@ -28,8 +28,10 @@ * * which gets simplified as : */ +#define ARM64_HW_PGTABLE_LEVELS_FOR(va_bits, page_shift) \ + (((va_bits) - PTDESC_ORDER - 1) / ((page_shift) - PTDESC_ORDER)) #define ARM64_HW_PGTABLE_LEVELS(va_bits) \ - (((va_bits) - PTDESC_ORDER - 1) / PTDESC_TABLE_SHIFT) + ARM64_HW_PGTABLE_LEVELS_FOR(va_bits, PAGE_SHIFT) /* * Size mapped by an entry at level n ( -1 <= n <= 3) @@ -44,7 +46,10 @@ * Rearranging it a bit we get : * (4 - n) * PTDESC_TABLE_SHIFT + PTDESC_ORDER */ -#define ARM64_HW_PGTABLE_LEVEL_SHIFT(n) (PTDESC_TABLE_SHIFT * (4 - (n)) + PTDESC_ORDER) +#define ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(n, page_shift) \ + (((page_shift) - PTDESC_ORDER) * (4 - (n)) + PTDESC_ORDER) +#define ARM64_HW_PGTABLE_LEVEL_SHIFT(n) \ + ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(n, PAGE_SHIFT) #define PTRS_PER_PTE (1 << PTDESC_TABLE_SHIFT) @@ -79,7 +84,8 @@ * PGDIR_SHIFT determines the size a top-level page table entry can map * (depending on the configuration, this level can be -1, 0, 1 or 2). */ -#define PGDIR_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - CONFIG_PGTABLE_LEVELS) +#define ARM64_NATIVE_PGTABLE_LEVELS ARM64_HW_PGTABLE_LEVELS(VA_BITS) +#define PGDIR_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - ARM64_NATIVE_PGTABLE_LEVELS) #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) #define PTRS_PER_PGD (1 << (VA_BITS - PGDIR_SHIFT)) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 4dfa42b7d0535..1ce97b836c8ac 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -467,6 +467,8 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte)); } +#include + /* * Hugetlb definitions. */ @@ -711,6 +713,23 @@ static inline void __set_ptes_anysz(struct mm_struct *mm, unsigned long addr, static inline void __set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte, unsigned int nr) { + if (arm64_mm_alt_granule(mm)) { + unsigned int native_pages = DIV_ROUND_UP((pte_phys_mm(mm, pte) & ~PAGE_MASK) + + ((unsigned long)nr << mm_page_shift(mm)), PAGE_SIZE); + + page_table_check_ptes_set(mm, addr, ptep, pte, nr); + __sync_cache_and_tags(pte, native_pages); + for (;;) { + __check_safe_pte_update(mm, ptep, pte); + __set_pte_nosync(ptep, pte); + if (--nr == 0) + break; + ptep++; + pte = pte_advance_pfn_mm(mm, pte, 1); + } + __set_pte_complete(pte); + return; + } __set_ptes_anysz(mm, addr, ptep, pte, nr, PAGE_SIZE); } @@ -946,6 +965,8 @@ static inline pmd_t *pud_pgtable(pud_t pud) static __always_inline bool pgtable_l4_enabled(void) { + if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) + return false; if (CONFIG_PGTABLE_LEVELS > 4 || !IS_ENABLED(CONFIG_ARM64_LPA2)) return true; if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT)) @@ -955,6 +976,8 @@ static __always_inline bool pgtable_l4_enabled(void) static inline bool mm_pud_folded(const struct mm_struct *mm) { + if (arm64_mm_alt_granule(mm)) + return mm_page_shift(mm) == 16; return !pgtable_l4_enabled(); } #define mm_pud_folded mm_pud_folded @@ -994,6 +1017,8 @@ static inline phys_addr_t p4d_page_paddr(p4d_t p4d) static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr) { + if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) + return (pud_t *)p4dp; /* Ensure that 'p4dp' indexes a page table according to 'addr' */ VM_BUG_ON(((addr >> P4D_SHIFT) ^ ((u64)p4dp >> 3)) % PTRS_PER_P4D); @@ -1216,6 +1241,8 @@ p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr) #endif /* CONFIG_PGTABLE_LEVELS > 4 */ +#include + #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) @@ -1261,7 +1288,7 @@ static inline int __ptep_set_access_flags(struct vm_area_struct *vma, pte_t entry, int dirty) { return __ptep_set_access_flags_anysz(vma, address, ptep, entry, dirty, - PAGE_SIZE); + mm_page_size(vma->vm_mm)); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -1389,7 +1416,7 @@ static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr, if (--nr == 0) break; ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(mm); } } @@ -1402,7 +1429,7 @@ static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm, pte = __ptep_get_and_clear(mm, addr, ptep); while (--nr) { ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(mm); tmp_pte = __ptep_get_and_clear(mm, addr, ptep); if (pte_dirty(tmp_pte)) pte = pte_mkdirty(pte); @@ -1450,7 +1477,7 @@ static inline void __wrprotect_ptes(struct mm_struct *mm, unsigned long address, { unsigned int i; - for (i = 0; i < nr; i++, address += PAGE_SIZE, ptep++) + for (i = 0; i < nr; i++, address += mm_page_size(mm), ptep++) __ptep_set_wrprotect(mm, address, ptep); } @@ -1490,7 +1517,7 @@ static inline void __clear_young_dirty_ptes(struct vm_area_struct *vma, if (--nr == 0) break; ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(vma->vm_mm); } } @@ -1516,9 +1543,11 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, * bits 0-1: present (must be zero) * bits 2: remember PG_anon_exclusive * bit 3: remember uffd-wp state + * bits 4-5: 4K slot offset within a native swap page (MM_SUBPAGE) * bits 6-10: swap type * bit 11: PTE_PRESENT_INVALID (must be zero) * bits 12-61: swap offset + * bits 62-63: high 4K slot offset bits (MM_SUBPAGE, up to 64K) */ #define __SWP_TYPE_SHIFT 6 #define __SWP_TYPE_BITS 5 @@ -1688,11 +1717,11 @@ static __always_inline void contpte_try_fold(struct mm_struct *mm, * We can't fold special mappings, because there is no associated folio. */ - const unsigned long contmask = CONT_PTES - 1; - bool valign = ((addr >> PAGE_SHIFT) & contmask) == contmask; + const unsigned long contmask = cont_ptes_mm(mm) - 1; + bool valign = ((addr >> mm_page_shift(mm)) & contmask) == contmask; if (unlikely(valign)) { - bool palign = (pte_pfn(pte) & contmask) == contmask; + bool palign = ((pte_phys_mm(mm, pte) >> mm_page_shift(mm)) & contmask) == contmask; if (unlikely(palign && pte_valid(pte) && !pte_cont(pte) && !pte_special(pte))) @@ -1713,7 +1742,11 @@ static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte) if (!pte_valid_cont(pte)) return 1; - return CONT_PTES - (((unsigned long)ptep >> 3) & (CONT_PTES - 1)); + { + unsigned int nr = arm64_cont_ptes(arm64_ptep_page_shift(ptep)); + + return nr - (((unsigned long)ptep >> 3) & (nr - 1)); + } } /* diff --git a/arch/arm64/include/asm/shmparam.h b/arch/arm64/include/asm/shmparam.h index f920e22ec677c..75120d3a71889 100644 --- a/arch/arm64/include/asm/shmparam.h +++ b/arch/arm64/include/asm/shmparam.h @@ -12,6 +12,12 @@ */ #define COMPAT_SHMLBA (4 * PAGE_SIZE) +/* SHMLBA also aligns kernel allocations; scale only SysV attachments. */ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_shm_attach_align(mm, align) \ + (((align) >> PAGE_SHIFT) << mm_page_shift(mm)) +#endif + #include #endif /* __ASM_SHMPARAM_H */ diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index 751bd57bc3bac..21453f3125e0f 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -10,6 +10,11 @@ #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +void __tlb_remove_table(void *token); +/* Fragment tokens do not carry a private descriptor callback head. */ +#define tlb_table_is_ptdesc(token) (!((unsigned long)(token) & 1UL)) +#endif #define tlb_flush tlb_flush static void tlb_flush(struct mmu_gather *tlb); @@ -76,8 +81,14 @@ static inline void tlb_flush(struct mmu_gather *tlb) static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr) { - struct ptdesc *ptdesc = page_ptdesc(pte); + struct ptdesc *ptdesc = page_ptdesc(pgtable_to_page(pte)); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pte | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } @@ -87,6 +98,12 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, { struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pmdp | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } #endif @@ -97,9 +114,15 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, { struct ptdesc *ptdesc = virt_to_ptdesc(pudp); - if (!pgtable_l4_enabled()) + if (mm_pud_folded(tlb->mm)) return; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pudp | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } #endif diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 57b4eda6a72bb..56062bf906cd4 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -56,8 +57,8 @@ }) /* - * Get translation granule of the system, which is decided by - * PAGE_SIZE. Used by TTL. + * Encode a translation granule for TTL and range operands. Native kernel + * and stage-2 callers use PAGE_SHIFT; userspace callers use their mm. * - 4KB : 1 * - 16KB : 2 * - 64KB : 3 @@ -66,20 +67,25 @@ #define TLBI_TTL_TG_16K 2 #define TLBI_TTL_TG_64K 3 -static inline unsigned long get_trans_granule(void) +static inline unsigned long get_trans_granule_shift(unsigned int page_shift) { - switch (PAGE_SIZE) { - case SZ_4K: + switch (page_shift) { + case 12: return TLBI_TTL_TG_4K; - case SZ_16K: + case 14: return TLBI_TTL_TG_16K; - case SZ_64K: + case 16: return TLBI_TTL_TG_64K; default: return 0; } } +static inline unsigned long get_trans_granule(void) +{ + return get_trans_granule_shift(PAGE_SHIFT); +} + #ifdef CONFIG_ARM64_ERRATUM_4193714 extern cpumask_t sme_active_cpus; @@ -175,13 +181,14 @@ static __always_inline void ipas2e1is(u64 arg) __tlbi(ipas2e1is, arg); } -static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, - u16 asid) +static __always_inline void __tlbi_level_asid_granule(tlbi_op op, u64 addr, + u32 level, u16 asid, + unsigned int page_shift) { u64 arg = __TLBI_VADDR(addr, asid); if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && level <= 3) { - u64 ttl = level | (get_trans_granule() << 2); + u64 ttl = level | (get_trans_granule_shift(page_shift) << 2); FIELD_MODIFY(TLBI_TTL_MASK, &arg, ttl); } @@ -189,6 +196,12 @@ static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, op(arg); } +static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, + u16 asid) +{ + __tlbi_level_asid_granule(op, addr, level, asid, PAGE_SHIFT); +} + static inline void __tlbi_level(tlbi_op op, u64 addr, u32 level) { __tlbi_level_asid(op, addr, level, 0); @@ -471,36 +484,45 @@ static __always_inline void ripas2e1is(u64 arg) __tlbi(ripas2e1is, arg); } -static __always_inline void __tlbi_range(tlbi_op op, u64 addr, - u16 asid, int scale, int num, - u32 level, bool lpa2) +static __always_inline void __tlbi_range_granule(tlbi_op op, u64 addr, + u16 asid, int scale, int num, + u32 level, bool lpa2, + unsigned int page_shift) { u64 arg = 0; - arg |= FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : PAGE_SHIFT)); + arg |= FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : page_shift)); arg |= FIELD_PREP(TLBIR_TTL_MASK, level > 3 ? 0 : level); arg |= FIELD_PREP(TLBIR_NUM_MASK, num); arg |= FIELD_PREP(TLBIR_SCALE_MASK, scale); - arg |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); + arg |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule_shift(page_shift)); arg |= FIELD_PREP(TLBIR_ASID_MASK, asid); op(arg); } -static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, +/* + * Production callers supply the constant system_supports_tlb_range function, + * which inlines to the existing alternative branch inside the loop. Do not + * evaluate it into a bool outside the loop: that adds runtime flag handling + * to the native path. Recording tests supply constant true/false predicates. + */ +static __always_inline void __flush_tlb_range_op_granule(tlbi_op lop, tlbi_op rop, u64 start, size_t pages, u64 stride, u16 asid, - u32 level, bool lpa2) + u32 level, bool lpa2, + unsigned int page_shift, + bool (*range_supported)(void)) { - u64 addr = start, end = start + pages * PAGE_SIZE; + u64 addr = start, end = start + (pages << page_shift); int scale = 3; while (addr != end) { int num; - pages = (end - addr) >> PAGE_SHIFT; + pages = (end - addr) >> page_shift; - if (!system_supports_tlb_range() || pages == 1) + if (!range_supported() || pages == 1) goto invalidate_one; if (lpa2 && !IS_ALIGNED(addr, SZ_64K)) @@ -508,26 +530,41 @@ static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, num = __TLBI_RANGE_NUM(pages, scale); if (num >= 0) { - __tlbi_range(rop, addr, asid, scale, num, level, lpa2); - addr += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; + __tlbi_range_granule(rop, addr, asid, scale, num, level, + lpa2, page_shift); + addr += __TLBI_RANGE_PAGES(num, scale) << page_shift; } scale--; continue; invalidate_one: - __tlbi_level_asid(lop, addr, level, asid); + __tlbi_level_asid_granule(lop, addr, level, asid, page_shift); addr += stride; } } +static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, + u64 start, size_t pages, + u64 stride, u16 asid, + u32 level, bool lpa2) +{ + __flush_tlb_range_op_granule(lop, rop, start, pages, stride, asid, + level, lpa2, PAGE_SHIFT, system_supports_tlb_range); +} + #define __flush_s1_tlb_range_op(op, start, pages, stride, asid, tlb_level) \ __flush_tlb_range_op(op, r##op, start, pages, stride, asid, tlb_level, lpa2_is_enabled()) #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ __flush_tlb_range_op(op, r##op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_enabled()) -static inline bool __flush_tlb_range_limit_excess(unsigned long pages, - unsigned long stride) +#define __flush_user_tlb_range_op(op, start, pages, stride, asid, level, shift) \ + __flush_tlb_range_op_granule(op, r##op, start, pages, stride, asid, \ + level, lpa2_is_enabled(), shift, system_supports_tlb_range) + +static inline bool __flush_tlb_range_limit_excess_granule(unsigned long pages, + unsigned long stride, + unsigned int page_shift) { /* * Assume that the worst case number of DVM ops required to flush a @@ -538,7 +575,13 @@ static inline bool __flush_tlb_range_limit_excess(unsigned long pages, if (system_supports_tlb_range()) return pages > MAX_TLBI_RANGE_PAGES; - return pages >= (MAX_DVM_OPS * stride) >> PAGE_SHIFT; + return pages >= (MAX_DVM_OPS * stride) >> page_shift; +} + +static inline bool __flush_tlb_range_limit_excess(unsigned long pages, + unsigned long stride) +{ + return __flush_tlb_range_limit_excess_granule(pages, stride, PAGE_SHIFT); } typedef unsigned __bitwise tlbf_t; @@ -565,10 +608,11 @@ static __always_inline void __do_flush_tlb_range(struct vm_area_struct *vma, { struct mm_struct *mm = vma->vm_mm; unsigned long asid, pages; + unsigned int page_shift = mm_page_shift(mm); - pages = (end - start) >> PAGE_SHIFT; + pages = (end - start) >> page_shift; - if (__flush_tlb_range_limit_excess(pages, stride)) { + if (__flush_tlb_range_limit_excess_granule(pages, stride, page_shift)) { flush_tlb_mm(mm); return; } @@ -582,20 +626,20 @@ static __always_inline void __do_flush_tlb_range(struct vm_area_struct *vma, switch (flags & (TLBF_NOWALKCACHE | TLBF_NOBROADCAST)) { case TLBF_NONE: - __flush_s1_tlb_range_op(vae1is, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vae1is, start, pages, stride, + asid, tlb_level, page_shift); break; case TLBF_NOWALKCACHE: - __flush_s1_tlb_range_op(vale1is, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vale1is, start, pages, stride, + asid, tlb_level, page_shift); break; case TLBF_NOBROADCAST: /* Combination unused */ BUG(); break; case TLBF_NOWALKCACHE | TLBF_NOBROADCAST: - __flush_s1_tlb_range_op(vale1, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vale1, start, pages, stride, + asid, tlb_level, page_shift); break; } @@ -629,16 +673,18 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, * Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough * information here. */ - __flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE); + __flush_tlb_range(vma, start, end, mm_page_size(vma->vm_mm), + TLBI_TTL_UNKNOWN, TLBF_NONE); } static inline void __flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr, tlbf_t flags) { - unsigned long start = round_down(uaddr, PAGE_SIZE); - unsigned long end = start + PAGE_SIZE; + unsigned long page_size = mm_page_size(vma->vm_mm); + unsigned long start = round_down(uaddr, page_size); + unsigned long end = start + page_size; - __do_flush_tlb_range(vma, start, end, PAGE_SIZE, 3, + __do_flush_tlb_range(vma, start, end, page_size, 3, TLBF_NOWALKCACHE | flags); } @@ -688,7 +734,7 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b { struct vm_area_struct vma = { .vm_mm = mm, .vm_flags = 0 }; - __flush_tlb_range(&vma, start, end, PAGE_SIZE, 3, + __flush_tlb_range(&vma, start, end, mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); } diff --git a/arch/arm64/include/asm/user-pgalloc.h b/arch/arm64/include/asm/user-pgalloc.h new file mode 100644 index 0000000000000..3b4dbde0c7736 --- /dev/null +++ b/arch/arm64/include/asm/user-pgalloc.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PGALLOC_H +#define __ASM_USER_PGALLOC_H + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + +enum arm64_user4k_pt_level { + USER4K_PT_PTE, + USER4K_PT_PMD, + USER4K_PT_PUD, +}; + +/* Explicit geometry entry point; callers must keep each mm's granule fixed. */ +void *arm64_user_pt_alloc_granule(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift); +void *arm64_user4k_pt_alloc(struct mm_struct *mm, enum arm64_user4k_pt_level level); +void arm64_user4k_pt_free(void *table); +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm); +void arm64_pgtable_free(void *table); +void arm64_pte_free_defer(struct mm_struct *mm, pgtable_t table); + +static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PTE, mm_page_shift(mm)); + return __pte_alloc_one_noprof(mm, GFP_PGTABLE_USER); +} +#define pte_alloc_one(...) alloc_hooks(pte_alloc_one_noprof(__VA_ARGS__)) + +static inline void pte_free(struct mm_struct *mm, pgtable_t table) +{ + arm64_pgtable_free(table); +} + +static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PMD, mm_page_shift(mm)); + return __pmd_alloc_one_noprof(mm, addr); +} +#define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__)) + +static inline void pmd_free(struct mm_struct *mm, pmd_t *table) +{ + arm64_pgtable_free(table); +} + +static inline pud_t *pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PUD, mm_page_shift(mm)); + return __pud_alloc_one_noprof(mm, addr); +} +#define pud_alloc_one(...) alloc_hooks(pud_alloc_one_noprof(__VA_ARGS__)) + +#define pte_free_defer arm64_pte_free_defer + +#endif +#endif diff --git a/arch/arm64/include/asm/user-pgtable.h b/arch/arm64/include/asm/user-pgtable.h new file mode 100644 index 0000000000000..b244f2387a7b4 --- /dev/null +++ b/arch/arm64/include/asm/user-pgtable.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PGTABLE_H +#define __ASM_USER_PGTABLE_H + +/* Included by asm/pgtable.h after the native accessors. */ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + +/* Alternative tables use 48-bit VA; 64K folds the PUD level. */ +static inline unsigned int arm64_user_level_shift(const struct mm_struct *mm, + unsigned int level) +{ + unsigned int shift = mm_page_shift(mm); + + if (shift == 16 && !level) + level = 1; + return shift + (3 - level) * (shift - 3); +} + +static inline unsigned long arm64_user_table_index(const struct mm_struct *mm, + unsigned long addr, unsigned int level) +{ + unsigned int shift = arm64_user_level_shift(mm, level); + unsigned int bits = level ? mm_page_shift(mm) - 3 : 48 - shift; + + return (addr >> shift) & ((1UL << bits) - 1); +} + +#define p4d_none_mm(mm, p4d) \ + (arm64_mm_alt_granule(mm) ? (!mm_pud_folded(mm) && !p4d_val(p4d)) : p4d_none(p4d)) +#define p4d_bad_mm(mm, p4d) \ + (arm64_mm_alt_granule(mm) ? (!mm_pud_folded(mm) && ((p4d_val(p4d) & P4D_TYPE_MASK) != P4D_TYPE_TABLE)) : p4d_bad(p4d)) +#define p4d_present_mm(mm, p4d) (!p4d_none_mm(mm, p4d)) +static inline void p4d_clear_mm(struct mm_struct *mm, p4d_t *p4d) +{ + if (arm64_mm_alt_granule(mm)) { + if (!mm_pud_folded(mm)) + set_p4d(p4d, __p4d(0)); + } else { + p4d_clear(p4d); + } +} +#define p4d_clear_mm p4d_clear_mm + + +/* pgtable_t preserves the fragment offset; pmd_page() remains its owner. */ +#define pmd_pgtable(pmd) ((pgtable_t)__va(pmd_val(pmd) & ARM64_USER4K_TABLE_ADDR_MASK)) + +static inline void *arm64_user4k_table(u64 descriptor) +{ + /* Unlike native __pXd_to_phys(), retain sub-native table address bits. */ + return __va(descriptor & ARM64_USER4K_TABLE_ADDR_MASK); +} + +static inline pgd_t *pgd_offset_pgd_mm(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return pgd + arm64_user_table_index(mm, addr, 0); + return pgd + ((addr >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)); +} + +#define pgd_offset_pgd_mm pgd_offset_pgd_mm + +static inline pgd_t *pgd_offset_mm(struct mm_struct *mm, unsigned long addr) +{ + return pgd_offset_pgd_mm(mm, mm->pgd, addr); +} + +#define pgd_offset(mm, addr) pgd_offset_mm(mm, addr) + +static inline p4d_t *p4d_offset_mm(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (p4d_t *)pgd; + return p4d_offset(pgd, addr); +} + +#define p4d_offset_mm p4d_offset_mm + +static inline pud_t *pud_offset_mm(struct mm_struct *mm, p4d_t *p4d, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) { + if (mm_pud_folded(mm)) + return (pud_t *)p4d; + return (pud_t *)arm64_user4k_table(p4d_val(READ_ONCE(*p4d))) + + arm64_user_table_index(mm, addr, 1); + } + return pud_offset(p4d, addr); +} + +#define pud_offset_mm pud_offset_mm + +static inline pmd_t *pud_pgtable_mm(struct mm_struct *mm, pud_t pud) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user4k_table(pud_val(pud)); + return pud_pgtable(pud); +} + +#define pud_pgtable_mm pud_pgtable_mm + +static inline pmd_t *pmd_offset_mm(struct mm_struct *mm, pud_t *pud, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (pmd_t *)arm64_user4k_table(pud_val(READ_ONCE(*pud))) + + arm64_user_table_index(mm, addr, 2); + return (pmd_t *)pud_pgtable(READ_ONCE(*pud)) + + ((addr >> PMD_SHIFT) & (PTRS_PER_PMD - 1)); +} + +#define pmd_offset_mm pmd_offset_mm + +static inline pte_t *pte_offset_kernel_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (pte_t *)arm64_user4k_table(pmd_val(READ_ONCE(*pmd))) + + arm64_user_table_index(mm, addr, 3); + return (pte_t *)pmd_page_vaddr(READ_ONCE(*pmd)) + + ((addr >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); +} + +#define pte_offset_kernel_mm pte_offset_kernel_mm +#define __pte_map_mm(mm, pmd, addr) pte_offset_kernel_mm(mm, pmd, addr) + +#define pmd_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 2) : PMD_SHIFT) +#define pud_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 1) : PUD_SHIFT) +#define p4d_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 0) : P4D_SHIFT) +#define pmd_size_mm(mm) (1UL << pmd_shift_mm(mm)) +#define pud_size_mm(mm) (1UL << pud_shift_mm(mm)) +#define p4d_size_mm(mm) (1UL << p4d_shift_mm(mm)) +#define pgd_size_mm(mm) (arm64_mm_alt_granule(mm) ? (1UL << arm64_user_level_shift(mm, 0)) : PGDIR_SIZE) +#define pte_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PTE * sizeof(pte_t)) +#define pmd_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PMD * sizeof(pmd_t)) +#define pud_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PUD * sizeof(pud_t)) +#define pmd_mask_mm(mm) (~(pmd_size_mm(mm) - 1)) +#define pud_mask_mm(mm) (~(pud_size_mm(mm) - 1)) +#define p4d_mask_mm(mm) (~(p4d_size_mm(mm) - 1)) +#define pgd_mask_mm(mm) (~(pgd_size_mm(mm) - 1)) + +static inline unsigned long arm64_user4k_addr_end(unsigned long addr, + unsigned long end, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned long boundary = (addr + size) & ~(size - 1); + + return boundary - 1 < end - 1 ? boundary : end; +} + +/* Native fallbacks retain their architecture-specific folding semantics. */ +#define pgd_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 0)) : pgd_addr_end(addr, end)) +#define p4d_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? (end) : p4d_addr_end(addr, end)) +#define pud_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 1)) : pud_addr_end(addr, end)) +#define pmd_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 2)) : pmd_addr_end(addr, end)) + +#endif /* CONFIG_ARM64_USER4K_EXPERIMENTAL */ +#endif diff --git a/arch/arm64/include/asm/user-pte.h b/arch/arm64/include/asm/user-pte.h new file mode 100644 index 0000000000000..aa4c1e3a5aab9 --- /dev/null +++ b/arch/arm64/include/asm/user-pte.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PTE_H +#define __ASM_USER_PTE_H + +/* Included after native leaf accessors, before their batched setters. */ +#define ARM64_USER4K_TABLE_ADDR_MASK GENMASK_ULL(47, 12) + +static inline bool arm64_mm_user4k(const struct mm_struct *mm) +{ + return mm && arm64_context_user4k(&mm->context); +} + +static inline bool arm64_mm_alt_granule(const struct mm_struct *mm) +{ + return mm && arm64_context_page_shift(&mm->context) != PAGE_SHIFT; +} + +static inline u64 arm64_user_leaf_addr_mask(const struct mm_struct *mm) +{ + return ARM64_USER4K_TABLE_ADDR_MASK & mm_page_mask(mm); +} + +/* PFNs still identify native pages; leaf physical addresses identify slots. */ +static inline phys_addr_t pte_phys_mm(const struct mm_struct *mm, pte_t pte) +{ + if (arm64_mm_alt_granule(mm)) + return pte_val(pte) & arm64_user_leaf_addr_mask(mm); + return __pte_to_phys(pte); +} +#define pte_phys_mm pte_phys_mm + +static inline pte_t phys_pte_mm(const struct mm_struct *mm, phys_addr_t phys, + pgprot_t prot) +{ + if (arm64_mm_alt_granule(mm)) { + VM_BUG_ON(phys & ~arm64_user_leaf_addr_mask(mm)); + return __pte(phys | pgprot_val(prot)); + } + return pfn_pte(phys >> PAGE_SHIFT, prot); +} +#define phys_pte_mm phys_pte_mm + +static inline pgprot_t pte_pgprot_mm(const struct mm_struct *mm, pte_t pte) +{ + if (arm64_mm_alt_granule(mm)) + return __pgprot(pte_val(pte) & ~arm64_user_leaf_addr_mask(mm)); + return pte_pgprot(pte); +} +#define pte_pgprot_mm pte_pgprot_mm + + +#ifdef CONFIG_MM_SUBPAGE +/* Slot bits belong to the PTE, not the native swap-cache index. */ +#define ARM64_SWP_SUBPAGE_MASK (GENMASK_ULL(5, 4) | GENMASK_ULL(63, 62)) + +static inline unsigned int pte_swp_subpage_offset(pte_t pte) +{ + return ((((pte_val(pte) >> 4) & 3) | + ((pte_val(pte) >> 60) & 12)) << 12); +} + +static inline pte_t pte_swp_set_subpage_offset(pte_t pte, unsigned int offset) +{ + VM_BUG_ON(!IS_ALIGNED(offset, SZ_4K) || offset >= PAGE_SIZE); + return __pte((pte_val(pte) & ~ARM64_SWP_SUBPAGE_MASK) | + (((u64)(offset >> 12) & 3) << 4) | + (((u64)(offset >> 12) & 12) << 60)); +} +#endif + +static inline pte_t pte_advance_pfn_mm(const struct mm_struct *mm, pte_t pte, + unsigned long nr) +{ + if (arm64_mm_alt_granule(mm)) + return phys_pte_mm(mm, pte_phys_mm(mm, pte) + (nr << mm_page_shift(mm)), + pte_pgprot_mm(mm, pte)); + return pte_advance_pfn(pte, nr); +} + +#define pte_advance_pfn_mm pte_advance_pfn_mm + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +unsigned int arm64_ptep_page_shift(pte_t *ptep); +#else +static inline unsigned int arm64_ptep_page_shift(pte_t *ptep) +{ + return PAGE_SHIFT; +} +#endif + +static inline unsigned int arm64_cont_ptes(unsigned int shift) +{ + /* Architectural contiguous group lengths for 4K, 16K and 64K. */ + return shift == 12 ? 16 : shift == 14 ? 128 : 32; +} + +static inline unsigned int cont_ptes_mm(const struct mm_struct *mm) +{ + return arm64_cont_ptes(mm_page_shift(mm)); +} + +static inline unsigned long cont_pte_size_mm(const struct mm_struct *mm) +{ + return (unsigned long)cont_ptes_mm(mm) << mm_page_shift(mm); +} + +static inline unsigned long pte_cont_addr_end_mm(const struct mm_struct *mm, + unsigned long addr, unsigned long end) +{ + unsigned long size = cont_pte_size_mm(mm); + unsigned long boundary = (addr + size) & ~(size - 1); + + return boundary - 1 < end - 1 ? boundary : end; +} + +#endif diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 6d53bb15cf7bb..ea314e3db67ea 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1661,6 +1661,62 @@ has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) return feature_matches(val, entry); } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST +/* Removal-only model for late-CPU verification in a disposable test VM. */ +static int user4k_deny_late_cpu __ro_after_init = -1; + +static int __init user4k_deny_late_cpu_setup(char *str) +{ + int cpu; + + if (kstrtoint(str, 0, &cpu) || cpu <= 0 || cpu >= NR_CPUS) + return -EINVAL; + user4k_deny_late_cpu = cpu; + return 0; +} +early_param("arm64.user4k_deny_late_cpu", user4k_deny_late_cpu_setup); +#ifdef CONFIG_ARM64_64K_PAGES +static int user16k_deny_late_cpu __ro_after_init = -1; + +static int __init user16k_deny_late_cpu_setup(char *str) +{ + int cpu; + + if (kstrtoint(str, 0, &cpu) || cpu <= 0 || cpu >= NR_CPUS) + return -EINVAL; + user16k_deny_late_cpu = cpu; + return 0; +} +early_param("arm64.user16k_deny_late_cpu", user16k_deny_late_cpu_setup); +#endif +#endif + +static bool has_user4k_granule(const struct arm64_cpu_capabilities *entry, + int scope) +{ +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (scope == SCOPE_LOCAL_CPU && system_capabilities_finalized() && + smp_processor_id() == user4k_deny_late_cpu) + return false; +#endif + return has_cpuid_feature(entry, scope); +} + +#ifdef CONFIG_ARM64_64K_PAGES +static bool has_user16k_granule(const struct arm64_cpu_capabilities *entry, + int scope) +{ +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (scope == SCOPE_LOCAL_CPU && system_capabilities_finalized() && + smp_processor_id() == user16k_deny_late_cpu) + return false; +#endif + return has_cpuid_feature(entry, scope); +} +#endif +#endif + const struct cpumask *system_32bit_el0_cpumask(void) { if (!system_supports_32bit_el0()) @@ -2391,8 +2447,11 @@ static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap) * clear the tags or set PG_mte_tagged. */ if (!cleared_zero_page) { + unsigned long offset; + cleared_zero_page = true; - mte_clear_page_tags(lm_alias(empty_zero_page)); + for (offset = 0; offset < sizeof(empty_zero_page); offset += PAGE_SIZE) + mte_clear_page_tags(lm_alias(empty_zero_page + offset)); } kasan_init_hw_tags_cpu(); @@ -2529,6 +2588,28 @@ test_has_gicv5_legacy(const struct arm64_cpu_capabilities *entry, int scope) } static const struct arm64_cpu_capabilities arm64_features[] = { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + { + .desc = "4K userspace translation granule", + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .capability = ARM64_HAS_USER4K_GRANULE, + .matches = has_user4k_granule, + __ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN4, + ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN, + ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MAX) + }, +#ifdef CONFIG_ARM64_64K_PAGES + { + .desc = "16K userspace translation granule", + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .capability = ARM64_HAS_USER16K_GRANULE, + .matches = has_user16k_granule, + __ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN16, + ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MIN, + ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MAX) + }, +#endif +#endif { .capability = ARM64_ALWAYS_BOOT, .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, diff --git a/arch/arm64/kernel/pi/idreg-override.c b/arch/arm64/kernel/pi/idreg-override.c index bc57b290e5e7b..19b98dffa71bc 100644 --- a/arch/arm64/kernel/pi/idreg-override.c +++ b/arch/arm64/kernel/pi/idreg-override.c @@ -38,11 +38,32 @@ struct ftr_set_desc { #define FIELD(n, s, f) { .name = n, .shift = s, .width = 4, .filter = f } +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && !defined(CONFIG_ARM64_4K_PAGES) +/* VM capability-model test: only remove 4K support, never fabricate it. */ +static bool __init user4k_granule_test_filter(u64 val) +{ + return val == ID_AA64MMFR0_EL1_TGRAN4_NI; +} + +#ifdef CONFIG_ARM64_64K_PAGES +static bool __init user16k_granule_test_filter(u64 val) +{ + return val == ID_AA64MMFR0_EL1_TGRAN16_NI; +} +#endif +#endif + static const struct ftr_set_desc mmfr0 __prel64_initconst = { .name = "id_aa64mmfr0", .override = &id_aa64mmfr0_override, .fields = { FIELD("ecv", ID_AA64MMFR0_EL1_ECV_SHIFT, NULL), +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && !defined(CONFIG_ARM64_4K_PAGES) + FIELD("tgran4", ID_AA64MMFR0_EL1_TGRAN4_SHIFT, user4k_granule_test_filter), +#ifdef CONFIG_ARM64_64K_PAGES + FIELD("tgran16", ID_AA64MMFR0_EL1_TGRAN16_SHIFT, user16k_granule_test_filter), +#endif +#endif {} }, }; @@ -59,11 +80,22 @@ static bool __init mmfr1_vh_filter(u64 val) val == 0); } +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && defined(CONFIG_ARM64_SW_TTBR0_PAN) +/* VM-only coverage of deferred TTBR0 restore on CPUs which implement PAN. */ +static bool __init user4k_pan_test_filter(u64 val) +{ + return val == 0; +} +#endif + static const struct ftr_set_desc mmfr1 __prel64_initconst = { .name = "id_aa64mmfr1", .override = &id_aa64mmfr1_override, .fields = { FIELD("vh", ID_AA64MMFR1_EL1_VH_SHIFT, mmfr1_vh_filter), +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && defined(CONFIG_ARM64_SW_TTBR0_PAN) + FIELD("pan", ID_AA64MMFR1_EL1_PAN_SHIFT, user4k_pan_test_filter), +#endif {} }, }; diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c index a852264958c36..c7dd0ba8d800e 100644 --- a/arch/arm64/kernel/pi/map_kernel.c +++ b/arch/arm64/kernel/pi/map_kernel.c @@ -243,7 +243,7 @@ asmlinkage void __init early_map_kernel(u64 boot_status, phys_addr_t fdt) static char const chosen_str[] __initconst = "/chosen"; u64 va_base, pa_base = (u64)&_text; u64 kaslr_offset = pa_base % MIN_KIMG_ALIGN; - int root_level = 4 - CONFIG_PGTABLE_LEVELS; + int root_level = 4 - ARM64_NATIVE_PGTABLE_LEVELS; int va_bits = VA_BITS; int chosen; void *fdt_mapped = map_fdt(fdt); diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 581f80e9b9b73..0aa0638a83072 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -7,11 +7,13 @@ * Copyright (C) 2012 ARM Ltd. */ #include +#include #include #include #include #include #include +#include #include #include #include @@ -817,6 +819,33 @@ unsigned long arch_align_stack(unsigned long sp) return sp & ~0xf; } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +int arm64_bprm_prepare_mm(struct linux_binprm *bprm, bool compat) +{ + unsigned int requested = current->exec_page_shift ?: + current->default_exec_page_shift; + unsigned int shift = arm64_exec_page_shift(compat, + cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE), PAGE_SHIFT, + requested); + + return bprm_set_page_shift(bprm, shift); +} + +/* Return zero for the native default, or a validated alternative shift. */ +int arch_exec_page_size_shift(unsigned long size) +{ + if (!size || size == PAGE_SIZE) + return 0; + if (size == SZ_4K) + return cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE) ? + 12 : -EOPNOTSUPP; + if (PAGE_SIZE > SZ_16K && size == SZ_16K) + return cpus_have_final_cap(ARM64_HAS_USER16K_GRANULE) ? + 14 : -EOPNOTSUPP; + return -EINVAL; +} +#endif + #ifdef CONFIG_COMPAT int compat_elf_check_arch(const struct elf32_hdr *hdr) { @@ -844,10 +873,10 @@ int compat_elf_check_arch(const struct elf32_hdr *hdr) */ void arch_setup_new_exec(void) { - unsigned long mmflags = 0; + unsigned long mmflags = current->mm->context.flags & MMCF_USER_GRANULE; if (is_compat_task()) { - mmflags = MMCF_AARCH32; + mmflags |= MMCF_AARCH32; /* * Restrict the CPU affinity mask for a 32-bit task so that diff --git a/arch/arm64/kernel/sys.c b/arch/arm64/kernel/sys.c index f08408b6e8263..bdd2531554f27 100644 --- a/arch/arm64/kernel/sys.c +++ b/arch/arm64/kernel/sys.c @@ -22,6 +22,11 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, off) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return ksys_mmap_user(addr, len, prot, flags, fd, off); +#endif + if (offset_in_page(off) != 0) return -EINVAL; diff --git a/arch/arm64/kernel/sys32.c b/arch/arm64/kernel/sys32.c index 12a948f3a5043..ba66b65f506f4 100644 --- a/arch/arm64/kernel/sys32.c +++ b/arch/arm64/kernel/sys32.c @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -51,6 +52,11 @@ COMPAT_SYSCALL_DEFINE6(aarch32_mmap2, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, off_4k) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return ksys_mmap_user(addr, len, prot, flags, fd, off_4k << 12); +#endif + if (off_4k & (~PAGE_MASK >> 12)) return -EINVAL; diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index e1ac876200a3d..688b82c0cf30d 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -66,9 +66,10 @@ * We require that __hyp_bss_start and __bss_start are aligned, and enforce it * with an assertion. But the BSS_SECTION macro places an empty .sbss section * between them, which can in some cases cause the linker to misalign them. To - * work around the issue, force a page alignment for __bss_start. + * work around the issue, align __bss_start to the BSS section's alignment, + * which can exceed PAGE_SIZE for the mixed-granule zero page. */ -#define SBSS_ALIGN PAGE_SIZE +#define SBSS_ALIGN MAX(PAGE_SIZE, ALIGNOF(.bss)) #else /* CONFIG_KVM */ #define HYPERVISOR_EXTABLE #define HYPERVISOR_RODATA_SECTIONS diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S index 5018ac03b6bf3..c1ce370068cf4 100644 --- a/arch/arm64/lib/mte.S +++ b/arch/arm64/lib/mte.S @@ -80,6 +80,52 @@ SYM_FUNC_START(mte_copy_page_tags) ret SYM_FUNC_END(mte_copy_page_tags) +#ifdef CONFIG_MM_SUBPAGE +/* Copy one aligned power-of-two extent, preserving neighbouring tags. */ +SYM_FUNC_START(mte_copy_subpage_tags_range) + mov x6, x2 + add x2, x0, x2 + multitag_transfer_size x4, x5 + cmp x4, x6 + b.hi 2f +1: ldgm x3, [x1] + stgm x3, [x0] + add x0, x0, x4 + add x1, x1, x4 + cmp x0, x2 + b.ne 1b + ret +2: mov x3, xzr +3: ldg x3, [x1] + stg x3, [x0] + add x0, x0, #MTE_GRANULE_SIZE + add x1, x1, #MTE_GRANULE_SIZE + cmp x0, x2 + b.ne 3b + ret +SYM_FUNC_END(mte_copy_subpage_tags_range) + +/* Clear an aligned extent without changing data or neighbouring tags. */ +SYM_FUNC_START(mte_clear_subpage_tags_range) + mov x4, x1 + add x1, x0, x1 + multitag_transfer_size x2, x3 + cmp x2, x4 + b.hi 2f +1: stgm xzr, [x0] + add x0, x0, x2 + cmp x0, x1 + b.ne 1b + ret +2: mov x2, xzr +3: stg x2, [x0] + add x0, x0, #MTE_GRANULE_SIZE + cmp x0, x1 + b.ne 3b + ret +SYM_FUNC_END(mte_clear_subpage_tags_range) +#endif + /* * Read tags from a user buffer (one tag per byte) and set the corresponding * tags at the given kernel address. Used by PTRACE_POKEMTETAGS. diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index c26489cf96cd9..32d18488e527c 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_TRANS_TABLE) += trans_pgd-asm.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ARM64_MTE) += mteswap.o obj-$(CONFIG_ARM64_GCS) += gcs.o +obj-$(CONFIG_ARM64_USER4K_EXPERIMENTAL) += user4k-pgalloc.o KASAN_SANITIZE_physaddr.o += n obj-$(CONFIG_KASAN) += kasan_init.o diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index 0f4a28b874692..cd3f56b942708 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -38,6 +39,34 @@ static unsigned long *pinned_asid_map; #define ctxid2asid(asid) ((asid) & ~ASID_MASK) #define asid2ctxid(asid, genid) ((asid) | (genid)) +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST +/* Test-only capacity reduction; hardware ASID width and normal boots are unchanged. */ +static bool user4k_test_small_asids; + +static int __init user4k_test_asid_bits_setup(char *str) +{ + unsigned int bits; + + if (kstrtouint(str, 0, &bits) || bits != 8) + return -EINVAL; + user4k_test_small_asids = true; + return 0; +} +early_param("arm64.user4k_test_asid_bits", user4k_test_asid_bits_setup); + +/* Read-only evidence that an EL0 stress run actually crossed ASID rollover. */ +static int user4k_asid_generation_get(char *buffer, const struct kernel_param *kp) +{ + return sysfs_emit(buffer, "%llu\n", (unsigned long long)atomic64_read(&asid_generation)); +} + +static const struct kernel_param_ops user4k_asid_generation_ops = { + .get = user4k_asid_generation_get, +}; +module_param_cb(user4k_asid_generation, &user4k_asid_generation_ops, NULL, 0444); +MODULE_PARM_DESC(user4k_asid_generation, "Current ASID generation for mixed-granule VM tests"); +#endif + /* Get the ASIDBits supported by the current CPU */ static u32 get_cpu_asid_bits(void) { @@ -268,6 +297,13 @@ void check_and_switch_context(struct mm_struct *mm) */ if (!system_uses_ttbr0_pan()) cpu_switch_mm(mm->pgd, mm); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + else { + /* uaccess/EL0 return restore only TTBRs, so select TG0 here. */ + cpu_set_reserved_ttbr0_nosync(); + cpu_set_mm_tg0(mm); + } +#endif } unsigned long arm64_mm_context_get(struct mm_struct *mm) @@ -365,6 +401,7 @@ void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) ttbr1 |= FIELD_PREP(TTBRx_EL1_ASID_MASK, asid); cpu_set_reserved_ttbr0_nosync(); + cpu_set_mm_tg0(mm); write_sysreg(ttbr1, ttbr1_el1); write_sysreg(ttbr0, ttbr0_el1); isb(); @@ -401,6 +438,12 @@ arch_initcall(asids_update_limit); static int asids_init(void) { asid_bits = get_cpu_asid_bits(); +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (user4k_test_small_asids) { + pr_info("user4k test: limiting ASID allocator to 8 bits\n"); + asid_bits = 8; + } +#endif atomic64_set(&asid_generation, ASID_FIRST_VERSION); asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); if (!asid_map) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index 2de12656b4d8b..f8faf1a99eeca 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -8,6 +8,25 @@ #include #include +static inline unsigned int contpte_nr(pte_t *ptep) +{ + return arm64_cont_ptes(arm64_ptep_page_shift(ptep)); +} + +static inline phys_addr_t contpte_phys(pte_t pte, unsigned int shift) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && shift != PAGE_SHIFT) + return pte_val(pte) & (ARM64_USER4K_TABLE_ADDR_MASK & ~((1ULL << shift) - 1)); + return __pte_to_phys(pte); +} + +static inline pgprot_t contpte_prot(pte_t pte, unsigned int shift) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && shift != PAGE_SHIFT) + return __pgprot(pte_val(pte) & ~(ARM64_USER4K_TABLE_ADDR_MASK & ~((1ULL << shift) - 1))); + return pte_pgprot(pte); +} + static inline bool mm_is_user(struct mm_struct *mm) { /* @@ -23,10 +42,10 @@ static inline bool mm_is_user(struct mm_struct *mm) static inline pte_t *contpte_align_down(pte_t *ptep) { - return PTR_ALIGN_DOWN(ptep, sizeof(*ptep) * CONT_PTES); + return PTR_ALIGN_DOWN(ptep, sizeof(*ptep) * contpte_nr(ptep)); } -static inline pte_t *contpte_align_addr_ptep(unsigned long *start, +static inline pte_t *contpte_align_addr_ptep(struct mm_struct *mm, unsigned long *start, unsigned long *end, pte_t *ptep, unsigned int nr) { @@ -36,10 +55,10 @@ static inline pte_t *contpte_align_addr_ptep(unsigned long *start, * single VMA and a single page table. */ if (pte_cont(__ptep_get(ptep + nr - 1))) - *end = ALIGN(*end, CONT_PTE_SIZE); + *end = ALIGN(*end, cont_pte_size_mm(mm)); if (pte_cont(__ptep_get(ptep))) { - *start = ALIGN_DOWN(*start, CONT_PTE_SIZE); + *start = ALIGN_DOWN(*start, cont_pte_size_mm(mm)); ptep = contpte_align_down(ptep); } @@ -54,11 +73,11 @@ static void contpte_try_unfold_partial(struct mm_struct *mm, unsigned long addr, * of the range. */ - if (ptep != contpte_align_down(ptep) || nr < CONT_PTES) + if (ptep != contpte_align_down(ptep) || nr < cont_ptes_mm(mm)) contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); if (ptep + nr != contpte_align_down(ptep + nr)) { - unsigned long last_addr = addr + PAGE_SIZE * (nr - 1); + unsigned long last_addr = addr + mm_page_size(mm) * (nr - 1); pte_t *last_ptep = ptep + nr - 1; contpte_try_unfold(mm, last_addr, last_ptep, @@ -75,10 +94,11 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, int i; start_ptep = ptep = contpte_align_down(ptep); - start_addr = addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); - pte = pfn_pte(ALIGN_DOWN(pte_pfn(pte), CONT_PTES), pte_pgprot(pte)); + start_addr = addr = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); + pte = phys_pte_mm(mm, ALIGN_DOWN(pte_phys_mm(mm, pte), cont_pte_size_mm(mm)), + pte_pgprot_mm(mm, pte)); - for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) { + for (i = 0; i < cont_ptes_mm(mm); i++, ptep++, addr += mm_page_size(mm)) { pte_t ptent = __ptep_get_and_clear(mm, addr, ptep); if (pte_dirty(ptent)) @@ -225,10 +245,10 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, */ if (!system_supports_bbml2_noabort()) - __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, 3, + __flush_tlb_range(&vma, start_addr, addr, mm_page_size(mm), 3, TLBF_NOWALKCACHE); - __set_ptes(mm, start_addr, start_ptep, pte, CONT_PTES); + __set_ptes(mm, start_addr, start_ptep, pte, cont_ptes_mm(mm)); } void __contpte_try_fold(struct mm_struct *mm, unsigned long addr, @@ -269,24 +289,26 @@ void __contpte_try_fold(struct mm_struct *mm, unsigned long addr, page = pte_page(pte); folio = page_folio(page); folio_start = addr - (page - &folio->page) * PAGE_SIZE; + if (arm64_mm_alt_granule(mm)) + folio_start -= pte_phys_mm(mm, pte) & ~PAGE_MASK; folio_end = folio_start + folio_nr_pages(folio) * PAGE_SIZE; - cont_start = ALIGN_DOWN(addr, CONT_PTE_SIZE); - cont_end = cont_start + CONT_PTE_SIZE; + cont_start = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); + cont_end = cont_start + cont_pte_size_mm(mm); if (folio_start > cont_start || folio_end < cont_end) return; - pfn = ALIGN_DOWN(pte_pfn(pte), CONT_PTES); - prot = pte_pgprot(pte_mkold(pte_mkclean(pte))); - expected_pte = pfn_pte(pfn, prot); + pfn = ALIGN_DOWN(pte_phys_mm(mm, pte) >> mm_page_shift(mm), cont_ptes_mm(mm)); + prot = pte_pgprot_mm(mm, pte_mkold(pte_mkclean(pte))); + expected_pte = phys_pte_mm(mm, pfn << mm_page_shift(mm), prot); orig_ptep = ptep; ptep = contpte_align_down(ptep); - for (i = 0; i < CONT_PTES; i++) { + for (i = 0; i < cont_ptes_mm(mm); i++) { subpte = pte_mkold(pte_mkclean(__ptep_get(ptep))); if (!pte_same(subpte, expected_pte)) return; - expected_pte = pte_advance_pfn(expected_pte, 1); + expected_pte = pte_advance_pfn_mm(mm, expected_pte, 1); ptep++; } @@ -319,17 +341,18 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) * feet. */ + unsigned int nr = contpte_nr(ptep); pte_t pte; int i; ptep = contpte_align_down(ptep); - for (i = 0; i < CONT_PTES; i++, ptep++) { + for (i = 0; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); - for (; i < CONT_PTES; i++, ptep++) { + for (; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_young(pte)) { orig_pte = pte_mkyoung(orig_pte); @@ -343,7 +366,7 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) orig_pte = pte_mkyoung(orig_pte); i++; ptep++; - for (; i < CONT_PTES; i++, ptep++) { + for (; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); @@ -359,11 +382,12 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) EXPORT_SYMBOL_GPL(contpte_ptep_get); static inline bool contpte_is_consistent(pte_t pte, unsigned long pfn, + unsigned int shift, pgprot_t orig_prot) { - pgprot_t prot = pte_pgprot(pte_mkold(pte_mkclean(pte))); + pgprot_t prot = contpte_prot(pte_mkold(pte_mkclean(pte)), shift); - return pte_valid_cont(pte) && pte_pfn(pte) == pfn && + return pte_valid_cont(pte) && (contpte_phys(pte, shift) >> shift) == pfn && pgprot_val(prot) == pgprot_val(orig_prot); } @@ -386,6 +410,8 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) * because it is not part of a contpte range. */ + unsigned int shift = arm64_ptep_page_shift(orig_ptep); + unsigned int nr = arm64_cont_ptes(shift); pgprot_t orig_prot; unsigned long pfn; pte_t orig_pte; @@ -399,22 +425,22 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) if (!pte_valid_cont(orig_pte)) return orig_pte; - orig_prot = pte_pgprot(pte_mkold(pte_mkclean(orig_pte))); + orig_prot = contpte_prot(pte_mkold(pte_mkclean(orig_pte)), shift); ptep = contpte_align_down(orig_ptep); - pfn = pte_pfn(orig_pte) - (orig_ptep - ptep); + pfn = (contpte_phys(orig_pte, shift) >> shift) - (orig_ptep - ptep); - for (i = 0; i < CONT_PTES; i++, ptep++, pfn++) { + for (i = 0; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); - for (; i < CONT_PTES; i++, ptep++, pfn++) { + for (; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_young(pte)) { @@ -430,10 +456,10 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) i++; ptep++; pfn++; - for (; i < CONT_PTES; i++, ptep++, pfn++) { + for (; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_dirty(pte)) { @@ -468,16 +494,16 @@ void contpte_set_ptes(struct mm_struct *mm, unsigned long addr, if (!mm_is_user(mm)) return __set_ptes(mm, addr, ptep, pte, nr); - end = addr + (nr << PAGE_SHIFT); - pfn = pte_pfn(pte); - prot = pte_pgprot(pte); + end = addr + (nr << mm_page_shift(mm)); + pfn = pte_phys_mm(mm, pte) >> mm_page_shift(mm); + prot = pte_pgprot_mm(mm, pte); do { - next = pte_cont_addr_end(addr, end); - nr = (next - addr) >> PAGE_SHIFT; - pte = pfn_pte(pfn, prot); + next = pte_cont_addr_end_mm(mm, addr, end); + nr = (next - addr) >> mm_page_shift(mm); + pte = phys_pte_mm(mm, pfn << mm_page_shift(mm), prot); - if (((addr | next | (pfn << PAGE_SHIFT)) & ~CONT_PTE_MASK) == 0) + if (((addr | next | (pfn << mm_page_shift(mm))) & (cont_pte_size_mm(mm) - 1)) == 0) pte = pte_mkcont(pte); else pte = pte_mknoncont(pte); @@ -512,6 +538,8 @@ EXPORT_SYMBOL_GPL(contpte_get_and_clear_full_ptes); bool contpte_test_and_clear_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr) { + struct mm_struct *mm = vma->vm_mm; + /* * ptep_clear_flush_young() technically requires us to clear the access * flag for a _single_ pte. However, the core-mm code actually tracks @@ -524,11 +552,11 @@ bool contpte_test_and_clear_young_ptes(struct vm_area_struct *vma, * of the same large folio in a single VMA and a single page table. */ - unsigned long end = addr + nr * PAGE_SIZE; + unsigned long end = addr + nr * mm_page_size(mm); bool young = false; - ptep = contpte_align_addr_ptep(&addr, &end, ptep, nr); - for (; addr != end; ptep++, addr += PAGE_SIZE) + ptep = contpte_align_addr_ptep(vma->vm_mm, &addr, &end, ptep, nr); + for (; addr != end; ptep++, addr += mm_page_size(mm)) young |= __ptep_test_and_clear_young(vma, addr, ptep); return young; @@ -538,19 +566,21 @@ EXPORT_SYMBOL_GPL(contpte_test_and_clear_young_ptes); bool contpte_clear_flush_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr) { + struct mm_struct *mm = vma->vm_mm; + bool young; young = contpte_test_and_clear_young_ptes(vma, addr, ptep, nr); if (young) { - unsigned long end = addr + nr * PAGE_SIZE; + unsigned long end = addr + nr * mm_page_size(mm); - contpte_align_addr_ptep(&addr, &end, ptep, nr); + contpte_align_addr_ptep(vma->vm_mm, &addr, &end, ptep, nr); /* * See comment in __ptep_clear_flush_young(); same rationale for * eliding the trailing DSB applies here. */ - __flush_tlb_range(vma, addr, end, PAGE_SIZE, 3, + __flush_tlb_range(vma, addr, end, mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); } @@ -580,6 +610,8 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr, cydp_t flags) { + struct mm_struct *mm = vma->vm_mm; + /* * We can safely clear access/dirty without needing to unfold from * the architectures perspective, even when contpte is set. If the @@ -591,14 +623,15 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, * clearing access/dirty for the whole block. */ unsigned long start = addr; - unsigned long end = start + nr * PAGE_SIZE; + unsigned long end = start + nr * mm_page_size(mm); - ptep = contpte_align_addr_ptep(&start, &end, ptep, nr); - __clear_young_dirty_ptes(vma, start, ptep, (end - start) / PAGE_SIZE, flags); + ptep = contpte_align_addr_ptep(vma->vm_mm, &start, &end, ptep, nr); + __clear_young_dirty_ptes(vma, start, ptep, (end - start) >> mm_page_shift(mm), flags); } EXPORT_SYMBOL_GPL(contpte_clear_young_dirty_ptes); -static bool contpte_all_subptes_match_access_flags(pte_t *ptep, pte_t entry) +static bool contpte_all_subptes_match_access_flags(struct mm_struct *mm, + pte_t *ptep, pte_t entry) { pte_t *cont_ptep = contpte_align_down(ptep); /* @@ -609,7 +642,7 @@ static bool contpte_all_subptes_match_access_flags(pte_t *ptep, pte_t entry) pteval_t entry_cmp = pte_val(entry) & cmp_mask; int i; - for (i = 0; i < CONT_PTES; i++) { + for (i = 0; i < cont_ptes_mm(mm); i++) { pteval_t pte_cmp = pte_val(__ptep_get(cont_ptep + i)) & cmp_mask; if (pte_cmp != entry_cmp) @@ -623,6 +656,8 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t entry, int dirty) { + struct mm_struct *mm = vma->vm_mm; + unsigned long start_addr; pte_t orig_pte; int i; @@ -651,7 +686,7 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * no-op, and when any sub-PTE mismatches, proceed to update the whole * range. */ - if (contpte_all_subptes_match_access_flags(ptep, entry)) + if (contpte_all_subptes_match_access_flags(mm, ptep, entry)) return 0; /* @@ -672,7 +707,7 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * and instead flush the whole range at the end. */ ptep = contpte_align_down(ptep); - start_addr = addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); + start_addr = addr = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); /* * We are not advancing entry because __ptep_set_access_flags() @@ -680,13 +715,13 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * for the whole contpte block and returned early, pte_same() * within __ptep_set_access_flags() is likely false. */ - for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) + for (i = 0; i < cont_ptes_mm(mm); i++, ptep++, addr += mm_page_size(mm)) __ptep_set_access_flags(vma, addr, ptep, entry, 0); if (dirty) __flush_tlb_range(vma, start_addr, - start_addr + CONT_PTE_SIZE, - PAGE_SIZE, 3, + start_addr + cont_pte_size_mm(mm), + mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOBROADCAST); } else { __contpte_try_unfold(vma->vm_mm, addr, ptep, orig_pte); diff --git a/arch/arm64/mm/copypage.c b/arch/arm64/mm/copypage.c index cd5912ba617b7..b81ccf42b22d1 100644 --- a/arch/arm64/mm/copypage.c +++ b/arch/arm64/mm/copypage.c @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -72,3 +73,54 @@ void copy_user_highpage(struct page *to, struct page *from, flush_dcache_page(to); } EXPORT_SYMBOL_GPL(copy_user_highpage); + +#ifdef CONFIG_MM_SUBPAGE +void clear_user_subpage_range(struct page *page, unsigned int offset, unsigned int size) +{ + void *addr = page_address(page) + offset; + + VM_BUG_ON(!is_power_of_2(size) || size < SZ_4K || size > PAGE_SIZE); + VM_BUG_ON(!IS_ALIGNED(offset, size) || offset > PAGE_SIZE - size); + memset(addr, 0, size); + if (system_supports_mte() && page_mte_tagged(page)) + mte_clear_subpage_tags_range(addr, size); + flush_dcache_page(page); +} + +int copy_user_subpage_range(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset, unsigned int size) +{ + void *kto = page_address(to), *kfrom = page_address(from); + + VM_BUG_ON(!is_power_of_2(size) || size < SZ_4K || size > PAGE_SIZE); + VM_BUG_ON(!IS_ALIGNED(to_offset, size) || to_offset > PAGE_SIZE - size); + VM_BUG_ON(!IS_ALIGNED(from_offset, size) || from_offset > PAGE_SIZE - size); + if (PageHWPoison(from) || copy_mc_to_kernel(kto + to_offset, kfrom + from_offset, size)) + return -EHWPOISON; + if (kasan_hw_tags_enabled()) + page_kasan_tag_reset(to); + if (system_supports_mte()) { + if (page_mte_tagged(from)) { + if (try_page_mte_tagging(to)) + mte_clear_page_tags(kto); + mte_copy_subpage_tags_range(kto + to_offset, kfrom + from_offset, size); + set_page_mte_tagged(to); + } else if (page_mte_tagged(to)) { + mte_clear_subpage_tags_range(kto + to_offset, size); + } + } + flush_dcache_page(to); + return 0; +} +/* Existing 4K fault and ownership callers keep their fixed-size contract. */ +void clear_user_subpage(struct page *page, unsigned int offset) +{ + clear_user_subpage_range(page, offset, SZ_4K); +} + +int copy_user_subpage(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset) +{ + return copy_user_subpage_range(to, to_offset, from, from_offset, SZ_4K); +} +#endif diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 739800835920c..6a37063417c4c 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -150,7 +150,7 @@ static void show_pte(unsigned long addr) } pr_alert("%s pgtable: %luk pages, %llu-bit VAs, pgdp=%016lx\n", - mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K, + mm == &init_mm ? "swapper" : "user", mm_page_size(mm) / SZ_1K, vabits_actual, mm_to_pgd_phys(mm)); pgdp = pgd_offset(mm, addr); pgd = READ_ONCE(*pgdp); @@ -165,25 +165,25 @@ static void show_pte(unsigned long addr) if (pgd_none(pgd) || pgd_bad(pgd)) break; - p4dp = p4d_offset(pgdp, addr); + p4dp = p4d_offset_mm(mm, pgdp, addr); p4d = READ_ONCE(*p4dp); pr_cont(", p4d=%016llx", p4d_val(p4d)); - if (p4d_none(p4d) || p4d_bad(p4d)) + if (p4d_none_mm(mm, p4d) || p4d_bad_mm(mm, p4d)) break; - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_mm(mm, p4dp, addr); pud = READ_ONCE(*pudp); pr_cont(", pud=%016llx", pud_val(pud)); if (pud_none(pud) || pud_bad(pud)) break; - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_mm(mm, pudp, addr); pmd = READ_ONCE(*pmdp); pr_cont(", pmd=%016llx", pmd_val(pmd)); if (pmd_none(pmd) || pmd_bad(pmd)) break; - ptep = pte_offset_map(pmdp, addr); + ptep = pte_offset_map_mm(mm, pmdp, addr); if (!ptep) break; @@ -241,19 +241,15 @@ int __ptep_set_access_flags_anysz(struct vm_area_struct *vma, * flush_tlb_fix_spurious_fault(). */ if (dirty) { - switch (pgsize) { - case PAGE_SIZE: + if (pgsize == mm_page_size(vma->vm_mm)) { level = 3; - break; - case PMD_SIZE: + } else if (pgsize == pmd_size_mm(vma->vm_mm)) { level = 2; - break; #ifndef __PAGETABLE_PMD_FOLDED - case PUD_SIZE: + } else if (pgsize == pud_size_mm(vma->vm_mm)) { level = 1; - break; #endif - default: + } else { level = TLBI_TTL_UNKNOWN; WARN_ON(1); } @@ -1002,8 +998,8 @@ NOKPROBE_SYMBOL(do_sp_pc_abort); /* * Used during anonymous page fault handling. */ -struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, - unsigned long vaddr) +struct folio *vma_alloc_zeroed_movable_folio_order(struct vm_area_struct *vma, + unsigned long vaddr, unsigned int order) { gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO; @@ -1015,7 +1011,7 @@ struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, if (vma->vm_flags & VM_MTE) flags |= __GFP_ZEROTAGS; - return vma_alloc_folio(flags, 0, vma, vaddr); + return vma_alloc_folio(flags, order, vma, vaddr); } bool tag_clear_highpages(struct page *page, int numpages, bool clear_pages) diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c index c5c5425791da7..b670d27a898cc 100644 --- a/arch/arm64/mm/fixmap.c +++ b/arch/arm64/mm/fixmap.c @@ -77,7 +77,7 @@ static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr, p4d_t p4d = READ_ONCE(*p4dp); pud_t *pudp; - if (CONFIG_PGTABLE_LEVELS > 3 && !p4d_none(p4d) && + if (pgtable_l4_enabled() && !p4d_none(p4d) && p4d_page_paddr(p4d) != __pa_symbol(bm_pud)) { /* * We only end up here if the kernel mapping and the fixmap diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 97987f850a33c..1e5f314b542d5 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -368,8 +368,9 @@ void __init arch_mm_preinit(void) * Selected page table levels should match when derived from * scratch using the virtual address range and page size. */ - BUILD_BUG_ON(ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS) != - CONFIG_PGTABLE_LEVELS); + BUILD_BUG_ON(CONFIG_PGTABLE_LEVELS != + (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) ? 4 : + ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS))); if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) { extern int sysctl_overcommit_memory; diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 09fd9da4eeed4..976d115d72f3f 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -2253,8 +2253,8 @@ pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr, * in cases where cpu is affected with errata #2645198. */ if (pte_accessible(vma->vm_mm, pte) && pte_user_exec(pte)) - __flush_tlb_range(vma, addr, nr * PAGE_SIZE, - PAGE_SIZE, 3, TLBF_NOWALKCACHE); + __flush_tlb_range(vma, addr, addr + nr * mm_page_size(vma->vm_mm), + mm_page_size(vma->vm_mm), 3, TLBF_NOWALKCACHE); } return pte; diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c index bf5110b91e2fa..3b361b95322e1 100644 --- a/arch/arm64/mm/pgd.c +++ b/arch/arm64/mm/pgd.c @@ -16,14 +16,19 @@ #include static struct kmem_cache *pgd_cache __ro_after_init; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +static struct kmem_cache *user4k_pgd_cache __ro_after_init; +static struct kmem_cache *user16k_pgd_cache __ro_after_init; +static struct kmem_cache *user64k_pgd_cache __ro_after_init; +#endif static bool pgdir_is_page_size(void) { if (PGD_SIZE == PAGE_SIZE) return true; - if (CONFIG_PGTABLE_LEVELS == 4) + if (ARM64_NATIVE_PGTABLE_LEVELS == 4) return !pgtable_l4_enabled(); - if (CONFIG_PGTABLE_LEVELS == 5) + if (ARM64_NATIVE_PGTABLE_LEVELS == 5) return !pgtable_l5_enabled(); return false; } @@ -32,6 +37,14 @@ pgd_t *pgd_alloc(struct mm_struct *mm) { gfp_t gfp = GFP_PGTABLE_USER; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_context_user4k(&mm->context)) + return kmem_cache_alloc(user4k_pgd_cache, gfp); + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 14) + return kmem_cache_alloc(user16k_pgd_cache, gfp); + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 16) + return kmem_cache_alloc(user64k_pgd_cache, gfp); +#endif if (pgdir_is_page_size()) return __pgd_alloc(mm, 0); else @@ -40,6 +53,20 @@ pgd_t *pgd_alloc(struct mm_struct *mm) void pgd_free(struct mm_struct *mm, pgd_t *pgd) { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_context_user4k(&mm->context)) { + kmem_cache_free(user4k_pgd_cache, pgd); + return; + } + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 14) { + kmem_cache_free(user16k_pgd_cache, pgd); + return; + } + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 16) { + kmem_cache_free(user64k_pgd_cache, pgd); + return; + } +#endif if (pgdir_is_page_size()) __pgd_free(mm, pgd); else @@ -48,6 +75,16 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd) void __init pgtable_cache_init(void) { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (PAGE_SHIFT < 16) + user64k_pgd_cache = kmem_cache_create("user64k_pgd_cache", SZ_64K, SZ_64K, + SLAB_PANIC, NULL); + if (PAGE_SHIFT != 14) + user16k_pgd_cache = kmem_cache_create("user16k_pgd_cache", SZ_16K, SZ_16K, + SLAB_PANIC, NULL); + user4k_pgd_cache = kmem_cache_create("user4k_pgd_cache", SZ_4K, SZ_4K, + SLAB_PANIC, NULL); +#endif if (pgdir_is_page_size()) return; diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 22866b49be372..7f947589a3b4d 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -291,7 +291,7 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) cbnz cpu, __idmap_kpti_secondary -#if CONFIG_PGTABLE_LEVELS > 4 +#if ARM64_NATIVE_PGTABLE_LEVELS > 4 stp x29, x30, [sp, #-32]! mov x29, sp stp x19, x20, [sp, #16] @@ -352,7 +352,7 @@ alternative_else_nop_endif /* Set the flag to zero to indicate that we're all done */ str wzr, [flag_ptr] -#if CONFIG_PGTABLE_LEVELS > 4 +#if ARM64_NATIVE_PGTABLE_LEVELS > 4 ldp x19, x20, [sp, #16] ldp x29, x30, [sp], #32 #endif @@ -360,39 +360,39 @@ alternative_else_nop_endif .Lderef_pgd: /* P4D */ - .if CONFIG_PGTABLE_LEVELS > 4 + .if ARM64_NATIVE_PGTABLE_LEVELS > 4 p4d .req x30 pte_to_phys cur_p4dp, pgd kpti_map_pgtbl p4d, 0 kpti_mk_tbl_ng p4d, PTRS_PER_P4D b .Lnext_pgd - .else /* CONFIG_PGTABLE_LEVELS <= 4 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 4 */ p4d .req pgd .set .Lnext_p4d, .Lnext_pgd .endif .Lderef_p4d: /* PUD */ - .if CONFIG_PGTABLE_LEVELS > 3 + .if ARM64_NATIVE_PGTABLE_LEVELS > 3 pud .req x10 pte_to_phys cur_pudp, p4d kpti_map_pgtbl pud, 1 kpti_mk_tbl_ng pud, PTRS_PER_PUD b .Lnext_p4d - .else /* CONFIG_PGTABLE_LEVELS <= 3 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 3 */ pud .req pgd .set .Lnext_pud, .Lnext_pgd .endif .Lderef_pud: /* PMD */ - .if CONFIG_PGTABLE_LEVELS > 2 + .if ARM64_NATIVE_PGTABLE_LEVELS > 2 pmd .req x13 pte_to_phys cur_pmdp, pud kpti_map_pgtbl pmd, 2 kpti_mk_tbl_ng pmd, PTRS_PER_PMD b .Lnext_pud - .else /* CONFIG_PGTABLE_LEVELS <= 2 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 2 */ pmd .req pgd .set .Lnext_pmd, .Lnext_pgd .endif diff --git a/arch/arm64/mm/user4k-pgalloc.c b/arch/arm64/mm/user4k-pgalloc.c new file mode 100644 index 0000000000000..0bcaab5c3a8d2 --- /dev/null +++ b/arch/arm64/mm/user4k-pgalloc.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Hardware tables have immutable granule/owner metadata. Smaller tables share + * a native page; larger tables own a compound allocation. Only unissued slots + * in a shared native page are cached by the mm. + */ +#include +#include +#include +#include +#include + +unsigned int arm64_ptep_page_shift(pte_t *ptep) +{ + struct ptdesc *pt = virt_to_ptdesc(ptep); + + /* Table owners retain geometry through their last RCU reader. */ + return test_bit(PT_fragmented, &pt->pt_flags.f) ? pt->pt_frag_shift : PAGE_SHIFT; +} +EXPORT_SYMBOL_GPL(arm64_ptep_page_shift); + +static void fragment_owner_free(struct ptdesc *pt) +{ + clear_bit(PT_fragmented, &pt->pt_flags.f); + pagetable_dtor_free(pt); +} + +static void fragment_owner_free_rcu(struct rcu_head *rcu) +{ + fragment_owner_free(container_of(rcu, struct ptdesc, pt_rcu_head)); +} + +static void fragment_put(struct ptdesc *pt, unsigned int nr) +{ + if (!atomic_sub_and_test(nr, &pt->pt_frag_refcount)) + return; + if (test_and_clear_bit(PT_frag_defer, &pt->pt_flags.f)) + call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); + else + fragment_owner_free(pt); +} + +void *arm64_user_pt_alloc_granule(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift) +{ + struct ptdesc *pt; + void *table, *next; + bool constructed; + unsigned int frag_size, nr_fragments, order; + + if (shift != 12 && shift != 14 && shift != 16) + return NULL; + frag_size = 1U << shift; + order = shift > PAGE_SHIFT ? shift - PAGE_SHIFT : 0; + nr_fragments = (PAGE_SIZE << order) >> shift; + if (WARN_ON_ONCE((unsigned int)level >= ARRAY_SIZE(mm->context.user4k_pt_frag))) + return NULL; + + spin_lock(&mm->page_table_lock); + table = mm->context.user4k_pt_frag[level]; + if (table) { + /* An mm's geometry is immutable; never consume another granule. */ + if (virt_to_ptdesc(table)->pt_frag_shift != shift) { + spin_unlock(&mm->page_table_lock); + return NULL; + } + next = table + frag_size; + mm->context.user4k_pt_frag[level] = + ((unsigned long)next & ~PAGE_MASK) ? next : NULL; + } + spin_unlock(&mm->page_table_lock); + if (table) + return table; + + pt = pagetable_alloc(GFP_PGTABLE_USER, order); + if (!pt) + return NULL; + switch (level) { + case USER4K_PT_PTE: + constructed = pagetable_pte_ctor(mm, pt); + break; + case USER4K_PT_PMD: + constructed = pagetable_pmd_ctor(mm, pt); + break; + case USER4K_PT_PUD: + pagetable_pud_ctor(pt); + constructed = true; + break; + default: + constructed = false; + } + if (!constructed) { + pagetable_free(pt); + return NULL; + } + pt->pt_frag_shift = shift; + set_bit(PT_fragmented, &pt->pt_flags.f); + atomic_set(&pt->pt_frag_refcount, 1); + table = ptdesc_address(pt); + spin_lock(&mm->page_table_lock); + if (nr_fragments > 1 && !mm->context.user4k_pt_frag[level]) { + atomic_set(&pt->pt_frag_refcount, nr_fragments); + mm->context.user4k_pt_frag[level] = table + frag_size; + } + /* A racing allocation may own the cache; this page then has one user. */ + spin_unlock(&mm->page_table_lock); + return table; +} + +void *arm64_user4k_pt_alloc(struct mm_struct *mm, enum arm64_user4k_pt_level level) +{ + return arm64_user_pt_alloc_granule(mm, level, 12); +} + +void arm64_user4k_pt_free(void *table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + VM_BUG_ON(!test_bit(PT_fragmented, &pt->pt_flags.f)); + VM_BUG_ON(!IS_ALIGNED((unsigned long)table, 1UL << pt->pt_frag_shift)); + VM_BUG_ON(atomic_read(&pt->pt_frag_refcount) <= 0); + fragment_put(pt, 1); +} + +void arm64_pgtable_free(void *table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + if (test_bit(PT_fragmented, &pt->pt_flags.f)) + arm64_user4k_pt_free(table); + else { + VM_BUG_ON(!PAGE_ALIGNED(table)); + pagetable_dtor_free(pt); + } +} + +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm) +{ + unsigned int level; + + /* The mm is private or dying; no allocator may still access its cache. */ + for (level = 0; level < ARRAY_SIZE(mm->context.user4k_pt_frag); level++) { + void *next = mm->context.user4k_pt_frag[level]; + struct ptdesc *pt; + unsigned int issued, shift; + + if (!next) + continue; + mm->context.user4k_pt_frag[level] = NULL; + pt = virt_to_ptdesc(next); + shift = pt->pt_frag_shift; + issued = ((unsigned long)next & ~PAGE_MASK) >> shift; + fragment_put(pt, (PAGE_SIZE >> shift) - issued); + } +} + +void arm64_pte_free_defer(struct mm_struct *mm, pgtable_t table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + if (test_bit(PT_fragmented, &pt->pt_flags.f)) { + /* No freed slot is reused. Delay the final owner release for RCU. */ + set_bit(PT_frag_defer, &pt->pt_flags.f); + arm64_user4k_pt_free(table); + } else { + call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); + } +} + +void __tlb_remove_table(void *token) +{ + if ((unsigned long)token & 1) + arm64_user4k_pt_free((void *)((unsigned long)token & ~1UL)); + else + pagetable_dtor_free(token); +} diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 811c2479e82d6..ecb884de60b0c 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -65,6 +65,8 @@ HAS_STAGE2_FWB HAS_TCR2 HAS_TIDCP1 HAS_TLB_RANGE +HAS_USER4K_GRANULE +HAS_USER16K_GRANULE HAS_VA52 HAS_VIRT_HOST_EXTN HAS_WFXT From e86efe2f2aaec4012ba91165f5211ca8c9c7716c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0428/1579] kunit: cover mixed-page ownership and translation lifetimes Add internal MM, table, subpage and recorded VFIO tests with separate opt-in controls. Translation fixtures require a disposable guest and recorded device mappings do not establish physical DMA qualification. Signed-off-by: Diljit Singh --- arch/arm64/Kconfig.debug | 70 + arch/arm64/mm/Makefile | 2 + arch/arm64/mm/user4k-data-test.c | 1835 ++ arch/arm64/mm/user4k-test.c | 14237 ++++++++++++++++ arch/arm64/mm/user4k-tlb-test.c | 184 + drivers/vfio/Kconfig | 12 + drivers/vfio/vfio_iommu_type1_account_test.c | 192 + .../vfio/vfio_iommu_type1_fragments_test.c | 1177 ++ drivers/vfio/vfio_iommu_type1_ioctl_test.c | 305 + drivers/vfio/vfio_iommu_type1_test.h | 36 + 10 files changed, 18050 insertions(+) create mode 100644 arch/arm64/mm/user4k-data-test.c create mode 100644 arch/arm64/mm/user4k-test.c create mode 100644 arch/arm64/mm/user4k-tlb-test.c create mode 100644 drivers/vfio/vfio_iommu_type1_account_test.c create mode 100644 drivers/vfio/vfio_iommu_type1_fragments_test.c create mode 100644 drivers/vfio/vfio_iommu_type1_ioctl_test.c create mode 100644 drivers/vfio/vfio_iommu_type1_test.h diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index d15ccdeee2125..58d7a1dc98405 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -42,3 +42,73 @@ config ARM64_USER4K_EXPERIMENTAL still require integration and validation. The private prctl numbers are not upstream assignments or a stable ABI promise. Say N for normal kernels. + +config ARM64_USER4K_KUNIT_TEST + bool "Test mixed userspace granules with larger kernel pages" + depends on KUNIT=y && ARM64_USER4K_EXPERIMENTAL + depends on ARM64_PA_BITS_48 + select ARM64_USER4K_DATA_KUNIT_TEST + select IOMMUFD + select IOMMUFD_DRIVER + default n + help + Exercise independent TTBR0 translation granules, subpage permissions + and invalidation while TTBR1 retains the normal native kernel tables. + Hardware translation/uaccess fixtures require arm64.user4k_test=1 + as a second opt-in and are disabled with KASAN. Software MM fixtures + can run with KASAN without borrowing or replacing TTBR0. + Run only in a disposable test VM. This regression harness does not + establish hardware qualification or production ABI readiness. + Say N for normal kernels. + +config ARM64_USER4K_IOMMUFD_TEST + bool "IOMMUFD ioctl harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && DEBUG_KERNEL + depends on FAULT_INJECTION && RUNTIME_TESTING_MENU + select GENERIC_PT + select IOMMU_PT + select IOMMU_PT_AMDV1 + select IOMMUFD + select IOMMUFD_TEST + default n + help + Enable the IOMMUFD mock-device ioctls and generic AMDv1 page tables + for the mixed-granule userspace ioctl fixture. This exposes privileged + test interfaces; use only in a disposable test VM. + Say N for normal kernels. + +config ARM64_USER4K_VFIO_ACCOUNT_TEST + bool "VFIO type1 pinning harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y + select VFIO + select VFIO_GROUP + select VFIO_CONTAINER + select VFIO_IOMMU_TYPE1 + select VFIO_TYPE1_ACCOUNT_KUNIT_TEST + default n + help + Enable built-in VFIO type1 accounting and retained-fragment tests + in a disposable mixed-granule guest. Recorded IOMMU translations + do not establish physical-device DMA support. + Say N for normal kernels. + +config ARM64_USER4K_VFIO_IOCTL_TEST + bool "Private VFIO type1 ioctl harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y && DEBUG_KERNEL + depends on RUNTIME_TESTING_MENU + select ARM64_USER4K_VFIO_ACCOUNT_TEST + default n + help + Expose a root-only disposable-guest misc device forwarding real EL0 + arguments to type1 ioctls and supplying recording IOMMU domains. + These privileged test controls perform no physical device DMA. + Say N for normal kernels. + +config ARM64_USER4K_DATA_KUNIT_TEST + bool "Test packed userspace allocation ownership" + depends on KUNIT=y && ARM64_USER4K_EXPERIMENTAL + default n + help + Exercise packed allocation, slot reuse, copying and deferred lifetime. + These tests use native kernel addresses without changing TTBR0, and + can run with KASAN even when the architectural tests are disabled. diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index 32d18488e527c..757f7eb07ccac 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -12,6 +12,8 @@ obj-$(CONFIG_TRANS_TABLE) += trans_pgd-asm.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ARM64_MTE) += mteswap.o obj-$(CONFIG_ARM64_GCS) += gcs.o +obj-$(CONFIG_ARM64_USER4K_KUNIT_TEST) += user4k-test.o user4k-tlb-test.o +obj-$(CONFIG_ARM64_USER4K_DATA_KUNIT_TEST) += user4k-data-test.o obj-$(CONFIG_ARM64_USER4K_EXPERIMENTAL) += user4k-pgalloc.o KASAN_SANITIZE_physaddr.o += n diff --git a/arch/arm64/mm/user4k-data-test.c b/arch/arm64/mm/user4k-data-test.c new file mode 100644 index 0000000000000..d7d550c9e1420 --- /dev/null +++ b/arch/arm64/mm/user4k-data-test.c @@ -0,0 +1,1835 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Backing ownership tests; these slots are not fault/rmap-owned mappings. */ +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_NET +#include +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) +#include +#endif +#ifdef CONFIG_INET +#include +#include +#endif +#include +#endif +#include +#include + +#define DATA_SLOTS MM_SUBPAGES_PER_PAGE + +struct data_fixture { + struct mm_subpage_pool *pool; + struct mm_subpage *held[2 * DATA_SLOTS + 2]; +}; + +static void data_fixture_free(void *arg) +{ + struct data_fixture *fixture = arg; + unsigned int i; + + if (fixture->pool) + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < ARRAY_SIZE(fixture->held); i++) + if (fixture->held[i]) + mm_subpage_put(fixture->held[i]); + if (fixture->pool) + mm_subpage_pool_put(fixture->pool); +} + +static struct data_fixture *data_fixture_create_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + + if (!fixture) + return NULL; + fixture->pool = mm_subpage_pool_create_granule(GFP_KERNEL, shift); + if (!fixture->pool) + return NULL; + if (kunit_add_action_or_reset(test, data_fixture_free, fixture)) + return NULL; + return fixture; +} + +static struct data_fixture *data_fixture_create(struct kunit *test) +{ + return data_fixture_create_granule(test, MM_SUBPAGE_SHIFT); +} + +static void put_held(struct data_fixture *fixture, unsigned int index) +{ + mm_subpage_put(fixture->held[index]); + fixture->held[index] = NULL; +} + +static struct mm_subpage *hold_alloc(struct data_fixture *fixture, unsigned int index) +{ + struct mm_subpage *slot = mm_subpage_alloc(fixture->pool); + + if (!IS_ERR(slot)) + fixture->held[index] = slot; + return slot; +} + +static int supply_backing(struct mm_subpage_pool *pool) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + int ret; + + if (!folio) + return -ENOMEM; + /* Check that stale allocation contents never become exposed. */ + memset(folio_address(folio), 0xcc, PAGE_SIZE); + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) + folio_put(folio); + return ret; +} + +static int supply_backing_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + int ret; + + if (!folio) + return -ENOMEM; + ret = mm_subpage_pool_add_folio_at(pool, folio, offset, GFP_KERNEL); + if (ret) + folio_put(folio); + return ret; +} + +static struct mm_subpage *hold_alloc_at(struct data_fixture *fixture, unsigned int index, + unsigned int offset) +{ + struct mm_subpage *slot = mm_subpage_alloc_at(fixture->pool, offset); + + if (!IS_ERR(slot)) + fixture->held[index] = slot; + return slot; +} + +static void *slot_address(struct mm_subpage *slot) +{ + return folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot); +} + +static void user4k_data_reuse_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct folio *folio; + struct mm_subpage *slot; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -EEXIST); + for (i = 0; i < DATA_SLOTS; i++) { + slot = hold_alloc(fixture, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)i + 1); + KUNIT_EXPECT_PTR_EQ(test, folio, mm_subpage_folio(fixture->held[0])); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), PFN_PHYS(folio_pfn(folio)) + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_4K), NULL); + memset(slot_address(slot), 0x10 + i, SZ_4K); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + /* A pin, alias or deferred-TLB holder keeps this slot unavailable. */ + fixture->held[DATA_SLOTS] = fixture->held[1]; + mm_subpage_get(fixture->held[DATA_SLOTS]); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS + 1); + put_held(fixture, 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + put_held(fixture, DATA_SLOTS); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS - 1); + slot = hold_alloc(fixture, DATA_SLOTS + 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_4K), NULL); + for (i = 0; i < DATA_SLOTS; i++) { + if (i != 1) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i]), + 0x10 + i, SZ_4K), NULL); + } + mm_subpage_pool_close(fixture->pool); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -ESHUTDOWN); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -ESHUTDOWN); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 1) + put_held(fixture, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + put_held(fixture, DATA_SLOTS + 1); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_copy_test(struct kunit *test) +{ + struct data_fixture *source = data_fixture_create(test); + struct data_fixture *dest = data_fixture_create(test); + struct mm_subpage *copy; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, source); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_EQ(test, supply_backing(source->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(dest->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(source, i)); + memset(slot_address(source->held[i]), 0x50 + i, SZ_4K); + } + source->held[DATA_SLOTS] = source->held[1]; + mm_subpage_get(source->held[DATA_SLOTS]); + copy = mm_subpage_copy(dest->pool, source->held[1]); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + dest->held[0] = copy; + KUNIT_EXPECT_NE(test, mm_subpage_phys(copy), mm_subpage_phys(source->held[1])); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x51, SZ_4K), NULL); + memset(slot_address(copy), 0xee, SZ_4K); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[i]), 0x50 + i, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(source->pool, source->held[1])), -EAGAIN); + /* Copy into a free neighboring slot in the same physical owner. */ + put_held(source, 2); + copy = mm_subpage_copy(source->pool, source->held[1]); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + source->held[2] = copy; + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(copy), mm_subpage_folio(source->held[1])); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(copy), 2U * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x51, SZ_4K), NULL); + memset(slot_address(copy), 0xdd, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[1]), 0x51, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[3]), 0x53, SZ_4K), NULL); +} + +static void user4k_data_lookup_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *slot, *found; + struct folio *folio; + phys_addr_t phys; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + phys = PFN_PHYS(folio_pfn(folio)); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys), NULL); + if (mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL)) { + folio_put(folio); + KUNIT_FAIL(test, "failed to supply lookup backing"); + return; + } + /* A published owner does not expose uninitialised or unissued slots. */ + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K), NULL); + for (i = 0; i < DATA_SLOTS; i++) { + slot = hold_alloc(fixture, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memset(slot_address(slot), 0x90 + i, SZ_4K); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2 * (int)i + 1); + if (!i) + KUNIT_EXPECT_EQ(test, mm_subpage_pool_add_folio(fixture->pool, folio, + GFP_KERNEL), -EEXIST); + found = mm_subpage_get_from_phys(phys + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, found); + fixture->held[i + DATA_SLOTS] = found; + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2 * (int)i + 2); + KUNIT_EXPECT_PTR_EQ(test, found, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K + 1), NULL); + } + /* A second pool must not zero or acquire already owned backing. */ + KUNIT_EXPECT_EQ(test, mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL), + -EINVAL); + for (i = 0; i < DATA_SLOTS; i++) { + put_held(fixture, i); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i + DATA_SLOTS]), + 0x90 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + put_held(fixture, DATA_SLOTS + 1); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + SZ_4K), NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[1]), 0, SZ_4K), NULL); + mm_subpage_pool_close(fixture->pool); + /* Keep the physical allocation stable while checking final unbinding. */ + folio_get(folio); + put_held(fixture, 1); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 1) + put_held(fixture, i + DATA_SLOTS); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); +} + +static void user4k_data_exact_slot_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *source, *copy, *reused; + struct folio *first; + static const unsigned int neighbors[] = { 0, SZ_4K, 3 * SZ_4K }; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_EXPECT_EQ(test, supply_backing_at(fixture->pool, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(fixture->pool, PAGE_SIZE)), -EINVAL); + KUNIT_ASSERT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), 0); + source = hold_alloc_at(fixture, 0, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, source); + first = mm_subpage_folio(source); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(source), (unsigned int)SZ_8K); + memset(slot_address(source), 0xa7, SZ_4K); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy_at(fixture->pool, source, 1)), -EINVAL); + /* Three free neighbors cannot satisfy a request for this occupied slot. */ + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy_at(fixture->pool, source, SZ_8K)), -EAGAIN); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -EEXIST); + KUNIT_ASSERT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), 0); + KUNIT_EXPECT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), -EEXIST); + copy = mm_subpage_copy_at(fixture->pool, source, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + fixture->held[1] = copy; + KUNIT_EXPECT_PTR_NE(test, mm_subpage_folio(copy), first); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(copy), (unsigned int)SZ_8K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0xa7, SZ_4K), NULL); + memset(slot_address(copy), 0x4b, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source), 0xa7, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 2UL); + for (i = 0; i < ARRAY_SIZE(neighbors); i++) { + struct mm_subpage *slot = hold_alloc_at(fixture, i + 2, neighbors[i]); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), first); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), neighbors[i]); + memset(slot_address(slot), 0x30 + i, SZ_4K); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(first), 4); + fixture->held[5] = source; + mm_subpage_get(source); + put_held(fixture, 0); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(fixture->pool, SZ_8K)), -EAGAIN); + put_held(fixture, 5); + reused = hold_alloc_at(fixture, 6, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, reused); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(reused), first); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(reused), 0, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(first), 4); + for (i = 0; i < ARRAY_SIZE(neighbors); i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i + 2]), + 0x30 + i, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x4b, SZ_4K), NULL); + mm_subpage_pool_close(fixture->pool); + for (i = 2; i <= 4; i++) + put_held(fixture, i); + put_held(fixture, 6); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + put_held(fixture, 1); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_close_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *slot; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + /* Closing with no issued slots must release that entire unused owner. */ + mm_subpage_pool_close(fixture->pool); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); + fixture = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + slot = hold_alloc(fixture, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memset(slot_address(slot), 0xa7, SZ_4K); + mm_subpage_pool_close(fixture->pool); + mm_subpage_pool_put(fixture->pool); + fixture->pool = NULL; + /* The originating pool/mm reference is gone; a child's slot remains live. */ + fixture->held[1] = slot; + mm_subpage_get(slot); + put_held(fixture, 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0xa7, SZ_4K), NULL); + put_held(fixture, 1); +} + +struct deferred_slot { + struct rcu_head rcu; + struct mm_subpage *slot; + struct completion done; +}; + +static void deferred_slot_put(struct rcu_head *rcu) +{ + struct deferred_slot *pending = container_of(rcu, struct deferred_slot, rcu); + + mm_subpage_put(pending->slot); + complete(&pending->done); +} + +static void data_rcu_barrier(void *unused) +{ + rcu_barrier(); +} + +static void user4k_data_deferred_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct deferred_slot *pending = kunit_kzalloc(test, sizeof(*pending), GFP_KERNEL); + struct mm_subpage *probe; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, pending); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, i)); + init_completion(&pending->done); + pending->slot = fixture->held[0]; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, data_rcu_barrier, NULL), 0); + mm_subpage_get(pending->slot); + rcu_read_lock(); + call_rcu(&pending->rcu, deferred_slot_put); + put_held(fixture, 0); + /* The delayed reference must prevent recycling before the grace period. */ + probe = mm_subpage_alloc(fixture->pool); + for (i = 1; i < DATA_SLOTS; i++) + put_held(fixture, i); + rcu_read_unlock(); + if (!IS_ERR(probe)) + fixture->held[DATA_SLOTS] = probe; + KUNIT_EXPECT_EQ(test, PTR_ERR(probe), -EAGAIN); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&pending->done, 5 * HZ), 0UL); + /* The callback retired the final live slot and the whole native owner. */ + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_backing_reject_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + folio = folio_alloc(GFP_KERNEL, 1); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + if (ret) + folio_put(folio); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_get(folio); + ret = mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2); + folio_put(folio); + if (ret) + folio_put(folio); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +struct data_worker { + struct mm_subpage_pool *pool; + struct task_struct *task; + struct completion ready, go, done; + unsigned int id, checks; + int error; +}; + +static struct mm_subpage *alloc_with_backing(struct mm_subpage_pool *pool) +{ + unsigned int tries; + + for (tries = 0; tries < 128; tries++) { + struct mm_subpage *slot = mm_subpage_alloc(pool); + int ret; + + if (!IS_ERR(slot) || PTR_ERR(slot) != -EAGAIN) + return slot; + ret = supply_backing(pool); + if (ret && ret != -EEXIST) + return ERR_PTR(ret); + cond_resched(); + } + return ERR_PTR(-EAGAIN); +} + +static int data_worker_run(void *arg) +{ + struct data_worker *worker = arg; + struct mm_subpage *slot = NULL; + unsigned int round; + u8 pattern = 0; + + for (round = 0; round < 64; round++) { + slot = alloc_with_backing(worker->pool); + if (IS_ERR(slot)) { + worker->error = PTR_ERR(slot); + slot = NULL; + break; + } + pattern = 1 + round + worker->id * 64; + memset(slot_address(slot), pattern, SZ_4K); + mm_subpage_get(slot); + mm_subpage_put(slot); + schedule_timeout_uninterruptible(1); + if (memchr_inv(slot_address(slot), pattern, SZ_4K)) { + worker->error = -EIO; + break; + } + worker->checks++; + if (round != 63) { + mm_subpage_put(slot); + slot = NULL; + } + } + complete(&worker->ready); + wait_for_completion(&worker->go); + if (slot) { + if (memchr_inv(slot_address(slot), pattern, SZ_4K)) + worker->error = -EIO; + mm_subpage_put(slot); + } + complete(&worker->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void data_worker_stop(void *arg) +{ + struct data_worker *worker = arg; + + complete_all(&worker->go); + kthread_stop(worker->task); +} + +static void user4k_data_concurrent_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct data_worker *workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].pool = fixture->pool; + workers[i].id = i; + init_completion(&workers[i].ready); + init_completion(&workers[i].go); + init_completion(&workers[i].done); + workers[i].task = kthread_run(data_worker_run, &workers[i], "user4k-data-%u", i); + KUNIT_ASSERT_FALSE(test, IS_ERR(workers[i].task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, data_worker_stop, &workers[i]), 0); + } + for (i = 0; i < 2; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < 2; i++) + complete(&workers[i].go); + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_EQ(test, workers[i].checks, 64U); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +struct lookup_worker { + struct task_struct *task; + struct completion observed; + phys_addr_t phys; + unsigned long checks; + int error; +}; + +static int lookup_worker_run(void *arg) +{ + struct lookup_worker *worker = arg; + bool observed = false; + + while (!kthread_should_stop()) { + struct mm_subpage *slot = mm_subpage_get_from_phys(worker->phys); + + if (slot) { + unsigned char *data = slot_address(slot); + unsigned char value = READ_ONCE(*data); + + if (mm_subpage_phys(slot) != worker->phys || + (value != 0x5a && value != 0xab) || + memchr_inv(data, value, SZ_4K)) + worker->error = -EINVAL; + worker->checks++; + mm_subpage_put(slot); + if (!observed) { + complete(&worker->observed); + observed = true; + } + if (worker->error) + break; + } + cond_resched(); + } + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void lookup_worker_stop(void *arg) +{ + struct lookup_worker *worker = arg; + + if (worker->task) { + kthread_stop(worker->task); + worker->task = NULL; + } +} + +static void user4k_data_lookup_race_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct lookup_worker *worker = kunit_kzalloc(test, sizeof(*worker), GFP_KERNEL); + unsigned int copied = 0, i; + unsigned long deadline; + struct folio *folio; + struct mm_subpage *found; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, worker); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, i)); + memset(slot_address(fixture->held[0]), 0x5a, SZ_4K); + memset(slot_address(fixture->held[1]), 0xab, SZ_4K); + worker->phys = mm_subpage_phys(fixture->held[2]); + put_held(fixture, 2); + init_completion(&worker->observed); + worker->task = kthread_run(lookup_worker_run, worker, "user4k-lookup"); + KUNIT_ASSERT_FALSE(test, IS_ERR(worker->task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, lookup_worker_stop, worker), 0); + if (num_online_cpus() > 1) { + unsigned int cpu = cpumask_next(cpumask_first(cpu_online_mask), cpu_online_mask); + + KUNIT_ASSERT_EQ(test, set_cpus_allowed_ptr(worker->task, cpumask_of(cpu)), 0); + } + /* Alternate complete contents; a reader must never see a partial copy. */ + deadline = jiffies + 30 * HZ; + while (copied < 2048 && time_before(jiffies, deadline)) { + struct mm_subpage *dest = mm_subpage_copy(fixture->pool, + fixture->held[copied & 1]); + bool observed = true; + phys_addr_t phys; + + if (PTR_ERR_OR_ZERO(dest) == -EAGAIN) { + /* Let a scheduled-out lookup holder retire its reference. */ + schedule_timeout_uninterruptible(1); + continue; + } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dest); + phys = mm_subpage_phys(dest); + if (!copied) + observed = wait_for_completion_timeout(&worker->observed, 5 * HZ) != 0; + mm_subpage_put(dest); + KUNIT_ASSERT_TRUE(test, observed); + KUNIT_ASSERT_EQ(test, phys, worker->phys); + copied++; + cond_resched(); + } + /* Retire the owner while lookup is still running, then verify unbinding. */ + folio = mm_subpage_folio(fixture->held[0]); + folio_get(folio); + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 2) + put_held(fixture, i); + lookup_worker_stop(worker); + found = mm_subpage_get_from_phys(worker->phys); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + if (found) + mm_subpage_put(found); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); + KUNIT_EXPECT_EQ(test, worker->error, 0); + KUNIT_EXPECT_GT(test, worker->checks, 0UL); + KUNIT_EXPECT_EQ(test, copied, 2048U); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_swapcache_reuse_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct mm_subpage *attempt; + struct folio *folio; + unsigned int i; + bool intact; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * MM_SUBPAGE_SIZE)); + memset(slot_address(f->held[i]), 0x60 + i, MM_SUBPAGE_SIZE); + } + folio = mm_subpage_folio(f->held[0]); + put_held(f, 1); + /* Synthetic cache membership only; no swap entry or I/O is fabricated. */ + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + attempt = mm_subpage_alloc_at(f->pool, MM_SUBPAGE_SIZE); + intact = !memchr_inv(folio_address(folio) + MM_SUBPAGE_SIZE, 0x61, MM_SUBPAGE_SIZE); + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + /* Restore the real state before assertions can unwind the fixture. */ + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_EXPECT_TRUE(test, intact); + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, MM_SUBPAGE_SIZE)); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x60 + i, MM_SUBPAGE_SIZE), NULL); +} + +static void restore_folio_free(void *arg) +{ + struct folio *folio = arg; + + /* Data-only fixtures never create a real swap entry or mapping. */ + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + folio_put(folio); +} + +static struct mm_subpage *restore_test_slot_granule(struct folio *folio, unsigned int offset, unsigned int shift) +{ + struct mm_subpage *slot; + + folio_lock(folio); + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + slot = mm_subpage_restore_granule(folio, offset, shift, GFP_KERNEL); + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + folio_unlock(folio); + return slot; +} + +static struct mm_subpage *restore_test_slot(struct folio *folio, unsigned int offset) +{ + return restore_test_slot_granule(folio, offset, MM_SUBPAGE_SHIFT); +} + +static void test_data_restore_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct folio *folio; + struct mm_subpage *slot; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + memset(slot_address(f->held[i]), 0x70 + i, size); + } + folio = mm_subpage_folio(f->held[0]); + folio_get(folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, folio), 0); + put_held(f, 1); + slot = restore_test_slot_granule(folio, size, shift); + if (!IS_ERR(slot)) + f->held[1] = slot; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)size); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 5); + slot = restore_test_slot_granule(folio, 0, shift); + if (!IS_ERR(slot)) + f->held[4] = slot; + KUNIT_ASSERT_PTR_EQ(test, slot, f->held[0]); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 6); + put_held(f, 4); + /* Retire the entire old owner, retaining only a native backing reference. */ + for (i = 0; i < 4; i++) + put_held(f, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), 0UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + for (i = 0; i < 4; i++) { + slot = restore_test_slot_granule(folio, i * size, shift); + if (!IS_ERR(slot)) + f->held[i] = slot; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0x70 + i, + size), NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 5); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, 1, shift)), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, PAGE_SIZE, shift)), -EINVAL); + if (shift != MM_SUBPAGE_SHIFT) { + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot(folio, 0)), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, SZ_4K, shift)), -EINVAL); + } + +} + +static void user4k_data_restore_test(struct kunit *test) +{ + test_data_restore_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_restore_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_restore_granule(test, 14); +} + +struct restore_worker { + struct folio *folio; + struct task_struct *task; + struct completion done; + unsigned int checks; + int error; +}; + +static int restore_worker_run(void *arg) +{ + struct restore_worker *w = arg; + unsigned long deadline = jiffies + 20 * HZ; + + while (!kthread_should_stop() && w->checks < 1024 && time_before(jiffies, deadline)) { + struct mm_subpage *slot; + + folio_lock(w->folio); + slot = mm_subpage_restore(w->folio, MM_SUBPAGE_SIZE, GFP_KERNEL); + folio_unlock(w->folio); + if (PTR_ERR_OR_ZERO(slot) == -EAGAIN) { + cond_resched(); + continue; + } + if (IS_ERR(slot)) { + w->error = PTR_ERR(slot); + break; + } + if (memchr_inv(slot_address(slot), 0x83, MM_SUBPAGE_SIZE)) + w->error = -EIO; + /* Retirement deliberately occurs without the folio lock. */ + mm_subpage_put(slot); + if (w->error) + break; + w->checks++; + cond_resched(); + } + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void restore_worker_stop(void *arg) +{ + struct restore_worker *w = arg; + + if (w->task) { + kthread_stop(w->task); + w->task = NULL; + } +} + +static void user4k_data_restore_race_test(struct kunit *test) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + struct restore_worker *workers; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, folio), 0); + memset(folio_address(folio), 0x83, PAGE_SIZE); + __folio_mark_uptodate(folio); + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].folio = folio; + init_completion(&workers[i].done); + workers[i].task = kthread_run(restore_worker_run, &workers[i], "user4k-restore-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "restore worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_worker_stop, &workers[i]), 0); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 25 * HZ), 0UL); + restore_worker_stop(&workers[i]); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_EQ(test, workers[i].checks, 1024U); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); +} + +static void test_data_migration_granule(struct kunit *test, unsigned int shift) +{ + unsigned int mode, size = 1U << shift; + + /* Success, rollback, all PTEs discarded, and partial-folio success. */ + for (mode = 0; mode < 4; mode++) { + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct folio *src, *dst, *target; + unsigned int i, nr = mode == 3 ? 3 : 4; + int ret, src_refs = 0, dst_refs = 0, blocked = 0; + bool unpublished = true, active = false; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, index * size)); + memset(slot_address(f->held[i]), 0x80 + index, size); + } + src = mm_subpage_folio(f->held[0]); + folio_get(src); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, src), 0); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, dst), 0); + folio_lock(src); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(src, dst); + if (!ret) { + struct mm_subpage *probe; + + src_refs = folio_ref_count(src); + dst_refs = folio_ref_count(dst); + active = mm_subpage_migrating(src) && mm_subpage_migrating(dst); + probe = mm_subpage_alloc_at(f->pool, size); + blocked = PTR_ERR_OR_ZERO(probe); + if (!IS_ERR(probe)) + mm_subpage_put(probe); + /* Emulate migration entries: their old mapping refs are gone. */ + for (i = 0; i < nr; i++) + put_held(f, i); + probe = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(src))); + unpublished &= !probe; + if (probe) + mm_subpage_put(probe); + probe = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(dst))); + unpublished &= !probe; + if (probe) + mm_subpage_put(probe); + if (mode != 1 && mode != 2) { + copy_user_highpage(&dst->page, &src->page, 0, NULL); + __folio_mark_uptodate(dst); + } + target = mode == 1 ? src : dst; + if (mode != 2) + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + f->held[i] = mm_subpage_migrate_restore(target, index * size); + } + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, active); + KUNIT_EXPECT_TRUE(test, unpublished); + KUNIT_EXPECT_EQ(test, src_refs, (int)nr + 1); + KUNIT_EXPECT_EQ(test, dst_refs, 1); + KUNIT_EXPECT_EQ(test, blocked, -EAGAIN); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), mode == 2 ? 0UL : 1UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(src), mode == 1 ? (int)nr + 1 : 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(dst), mode == 0 || mode == 3 ? (int)nr + 1 : 1); + if (mode != 2) + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->held[i]), mode == 1 ? src : dst); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + 0x80 + index, size), NULL); + } + if (mode == 3) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 3, size)); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->held[3]), dst); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[3]), 0, + size), NULL); + } + } +} + +static void user4k_data_migration_owner_test(struct kunit *test) +{ + test_data_migration_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_migration_owner_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_migration_granule(test, 14); +} + +#ifdef CONFIG_ARM64_MTE +static void test_slot_tags_set(void *addr, unsigned int tag) +{ + unsigned long value = (unsigned long)tag << MTE_TAG_SHIFT; + unsigned int offset; + + for (offset = 0; offset < MM_SUBPAGE_SIZE; offset += MTE_GRANULE_SIZE) + asm volatile(".arch_extension memtag\n\tstg %0, [%1]" + : : "r" (value), "r" (addr + offset) : "memory"); +} + +static bool test_slot_tags_equal(void *addr, unsigned int expected) +{ + unsigned int offset; + + for (offset = 0; offset < MM_SUBPAGE_SIZE; offset += MTE_GRANULE_SIZE) { + unsigned long value = 0; + + asm volatile(".arch_extension memtag\n\tldg %0, [%1]" + : "+r" (value) : "r" (addr + offset) : "memory"); + if (((value >> MTE_TAG_SHIFT) & 0xf) != expected) + return false; + } + return true; +} +#endif + +static void user4k_data_mte_test(struct kunit *test) +{ +#ifdef CONFIG_ARM64_MTE + struct data_fixture *f; + struct folio *folio; + unsigned int i; + + if (!system_supports_mte()) { + kunit_skip(test, "MTE is not available on this CPU"); + return; + } + f = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * MM_SUBPAGE_SIZE)); + folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_TRUE(test, try_page_mte_tagging(&folio->page)); + mte_clear_page_tags(folio_address(folio)); + for (i = 0; i < 4; i++) { + memset(slot_address(f->held[i]), 0x40 + i, MM_SUBPAGE_SIZE); + test_slot_tags_set(slot_address(f->held[i]), i + 1); + } + set_page_mte_tagged(&folio->page); + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, MM_SUBPAGE_SIZE)); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(slot_address(f->held[i]), i == 1 ? 0 : i + 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x40 + i, MM_SUBPAGE_SIZE), NULL); + } + put_held(f, 1); + f->held[1] = mm_subpage_copy_at(f->pool, f->held[0], MM_SUBPAGE_SIZE); + if (IS_ERR(f->held[1])) { + f->held[1] = NULL; + KUNIT_FAIL(test, "tagged slot copy failed"); + return; + } + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(slot_address(f->held[i]), i == 1 ? 1 : i + 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0x40 : 0x40 + i, MM_SUBPAGE_SIZE), NULL); + } + /* A fresh destination must initialise tags outside the copied slot too. */ + KUNIT_ASSERT_EQ(test, supply_backing_at(f->pool, 3 * MM_SUBPAGE_SIZE), 0); + f->held[4] = mm_subpage_copy_at(f->pool, f->held[2], 3 * MM_SUBPAGE_SIZE); + if (IS_ERR(f->held[4])) { + f->held[4] = NULL; + KUNIT_FAIL(test, "fresh tagged slot copy failed"); + return; + } + folio = mm_subpage_folio(f->held[4]); + KUNIT_EXPECT_TRUE(test, page_mte_tagged(&folio->page)); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(folio_address(folio) + + i * MM_SUBPAGE_SIZE, i == 3 ? 3 : 0)); +#else + kunit_skip(test, "MTE is not configured"); +#endif +} + +static void extent_folio_put(void *arg) +{ + folio_put(arg); +} + +static void user4k_data_copy_extent_test(struct kunit *test) +{ + const unsigned int sizes[] = { SZ_4K, SZ_16K, SZ_64K }; + + for (unsigned int n = 0; n < ARRAY_SIZE(sizes); n++) { + struct folio *source, *dest; + unsigned int size = sizes[n], to_offset; + void *from, *to; + + if (size > PAGE_SIZE) + continue; + source = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, source); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, source), 0); + dest = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, dest), 0); + from = folio_address(source); + to = folio_address(dest); + to_offset = PAGE_SIZE - size; + for (unsigned int i = 0; i < DATA_SLOTS; i++) + memset(from + i * SZ_4K, 0x40 + i, SZ_4K); + for (unsigned int tagged = 0; tagged < 2; tagged++) { + memset(to, 0xac, PAGE_SIZE); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) { + if (try_page_mte_tagging(&dest->page)) + mte_clear_page_tags(to); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(to + i * SZ_4K, 9); + set_page_mte_tagged(&dest->page); + if (tagged) { + if (try_page_mte_tagging(&source->page)) + mte_clear_page_tags(from); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(from + i * SZ_4K, 1 + i % 15); + set_page_mte_tagged(&source->page); + } + } +#endif + KUNIT_ASSERT_EQ(test, copy_user_subpage_range(&dest->page, to_offset, + &source->page, 0, size), 0); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int offset = i * SZ_4K; + bool copied = offset >= to_offset; + unsigned int source_index = copied ? (offset - to_offset) / SZ_4K : 0; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(to + offset, + copied ? 0x40 + source_index : 0xac, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(from + offset, 0x40 + i, SZ_4K), NULL); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(to + offset, + copied ? (tagged ? 1 + source_index % 15 : 0) : 9)); +#endif + } + clear_user_subpage_range(&dest->page, to_offset, size); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int offset = i * SZ_4K; + bool cleared = offset >= to_offset; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(to + offset, cleared ? 0 : 0xac, + SZ_4K), NULL); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(to + offset, cleared ? 0 : 9)); +#endif + } + } + } +} + +#ifdef CONFIG_NET +struct skb_test_state { + struct folio *folio; +#ifdef CONFIG_INET + struct socket *socket; +#endif + struct sk_buff *skbs[5]; +}; + +static void skb_test_cleanup(void *arg) +{ + struct skb_test_state *state = arg; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->skbs); i++) { + if (state->skbs[i]) + state->skbs[i]->sk = NULL; + kfree_skb(state->skbs[i]); + } +#ifdef CONFIG_INET + if (state->socket) + sock_release(state->socket); +#endif + /* This fixture marks ownership only; it installs no anonymous rmap. */ + if (state->folio) + state->folio->mapping = NULL; +} + +static void user4k_data_skb_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct iov_iter iter; + struct bio_vec bvec; + unsigned char *bytes = kunit_kmalloc(test, SZ_4K, GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_NOT_NULL(test, bytes); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + memset(slot_address(f->held[i]), 0x51 + i, SZ_4K); + } + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + state->skbs[0]->ip_summed = CHECKSUM_UNNECESSARY; + bvec_set_page(&bvec, &state->folio->page, SZ_4K, SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(state->skbs[0], &iter, SZ_4K), (ssize_t)SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, skb_zcopy(state->skbs[0])); + state->skbs[1] = pskb_copy(state->skbs[0], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[1]); + /* Append a distinct quarter while the callback object has two owners. */ + bvec_set_page(&bvec, &state->folio->page, SZ_4K, 2 * SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(state->skbs[0], &iter, SZ_4K), (ssize_t)SZ_4K); + KUNIT_EXPECT_PTR_NE(test, skb_zcopy(state->skbs[0]), skb_zcopy(state->skbs[1])); + state->skbs[2] = skb_clone(state->skbs[1], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[2]); + state->skbs[3] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[3]); + skb_split(state->skbs[0], state->skbs[3], SZ_2K); + state->skbs[4] = skb_copy(state->skbs[3], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[4]); + put_held(f, 1); + put_held(f, 2); + attempt = mm_subpage_alloc_at(f->pool, SZ_4K); + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_EXPECT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + for (i = 0; i < 2; i++) { + kfree_skb(state->skbs[i]); + state->skbs[i] = NULL; + } + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[2], 0, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x52, SZ_4K), NULL); + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[3], SZ_2K, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x53, SZ_4K), NULL); + for (i = 2; i < 4; i++) { + kfree_skb(state->skbs[i]); + state->skbs[i] = NULL; + } + /* A full data copy must not keep ownership of the original quarters. */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_4K)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 2, 2 * SZ_4K)); + memset(slot_address(f->held[1]), 0xe1, SZ_4K); + memset(slot_address(f->held[2]), 0xe2, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[4], SZ_2K, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x53, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} +#ifdef CONFIG_INET +static void test_data_skb_prune_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct sk_buff *skb; + struct bio_vec bvec; + struct iov_iter iter; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + KUNIT_ASSERT_EQ(test, sock_create_kern(&init_net, AF_INET, SOCK_STREAM, 0, &state->socket), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + skb = state->skbs[0]; + /* A live socket association prevents trim from condensing data into the head. */ + skb->sk = state->socket->sk; + skb->ip_summed = CHECKSUM_UNNECESSARY; + bvec_set_page(&bvec, &state->folio->page, 16, size - SZ_4K + 16); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, 16); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, 16), (ssize_t)16); + for (i = 0; i < 1024; i++) { + bvec_set_page(&bvec, &state->folio->page, size, (1 + i % 3) * size); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, size), (ssize_t)size); + /* Four fixture refs, two frag refs, two current quarter refs. */ + KUNIT_ASSERT_EQ(test, folio_ref_count(state->folio), 8); + KUNIT_ASSERT_EQ(test, pskb_trim(skb, 16), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 7); + } + skb->sk = NULL; + kfree_skb(skb); + state->skbs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} + +static void user4k_data_skb_prune_test(struct kunit *test) +{ + test_data_skb_prune_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_skb_prune_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_skb_prune_granule(test, 14); +} + +#endif + +static void user4k_data_skb_checksum_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct bio_vec bvec[2]; + struct iov_iter iter; + unsigned char *bytes; + unsigned int i, owned; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + bytes = folio_address(state->folio); + for (i = 0; i < PAGE_SIZE; i++) + bytes[i] = i * 13 + (i >> 12) * 31; + for (owned = 0; owned < 2; owned++) { + struct sk_buff *skb; + + state->folio->mapping = owned ? (void *)FOLIO_MAPPING_ANON : NULL; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + skb = state->skbs[0]; + skb->ip_summed = CHECKSUM_NONE; + bvec_set_page(&bvec[0], &state->folio->page, SZ_4K - 13, 13); + bvec_set_page(&bvec[1], &state->folio->page, SZ_4K, SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, bvec, 2, SZ_8K - 13); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, SZ_8K - 13), + (ssize_t)(SZ_8K - 13)); + KUNIT_EXPECT_EQ(test, (__force u16)csum_fold(skb->csum), + (__force u16)csum_fold(skb_checksum(skb, 0, skb->len, 0))); + kfree_skb(skb); + state->skbs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + } +} + +#endif + +struct refs_test_state { + struct folio *folio; + struct mm_subpage_refs *refs[3]; +}; + +static void refs_test_cleanup(void *arg) +{ + struct refs_test_state *state = arg; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->refs); i++) + mm_subpage_refs_put(state->refs[i]); + if (state->folio) + state->folio->mapping = NULL; +} + +static bool refs_test_keep(const struct mm_subpage *slot, void *context) +{ + unsigned long mask = *(unsigned long *)context; + + return mask & BIT(mm_subpage_offset(slot) >> MM_SUBPAGE_SHIFT); +} + +static void user4k_data_refset_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct refs_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct page *page; + unsigned long mask; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + state->folio = mm_subpage_folio(f->held[0]); + page = &state->folio->page; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, refs_test_cleanup, state), 0); + /* Ownership fixture only; no anonymous rmap is installed. */ + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, + SZ_4K + 17, SZ_8K - 34, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, SZ_4K, SZ_4K, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + state->refs[1] = mm_subpage_refs_get(state->refs[0]); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[1], page, 3 * SZ_4K, SZ_4K, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_NE(test, state->refs[0], state->refs[1]); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 5); + mask = BIT(2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 3); + state->refs[2] = mm_subpage_refs_get(state->refs[0]); + mask = BIT(1); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[2], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_NE(test, state->refs[0], state->refs[2]); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 4); + KUNIT_EXPECT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, PAGE_SIZE, 1, GFP_KERNEL), -EINVAL); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 4); + mask = 0; + for (i = 1; i < 3; i++) { + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[i], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_EQ(test, state->refs[i], NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + put_held(f, 1); + attempt = mm_subpage_alloc_at(f->pool, SZ_4K); + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + mm_subpage_refs_put(state->refs[0]); + state->refs[0] = NULL; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_4K)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS); + for (i = 0; i < 1024; i++) { + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 2 * (int)DATA_SLOTS); + mask = BIT(i % DATA_SLOTS); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[0], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 1); + } + mm_subpage_refs_put(state->refs[0]); + state->refs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS); +} + +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) +struct skmsg_test_state { + struct folio *folio; + struct socket *socket; + struct sk_msg messages[2]; + struct sk_msg borrowed; +}; + +static void skmsg_test_cleanup(void *arg) +{ + struct skmsg_test_state *state = arg; + unsigned int i; + + if (state->socket) { + for (i = 0; i < ARRAY_SIZE(state->messages); i++) + sk_msg_free(state->socket->sk, &state->messages[i]); + sk_msg_subpages_release(&state->borrowed); + sock_release(state->socket); + } + if (state->folio) + state->folio->mapping = NULL; +} + +static void test_data_skmsg_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct skmsg_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct sk_msg *src, *dst; + struct page *page; + struct sock *sk; + struct bio_vec bvec; + struct iov_iter iter; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + memset(slot_address(f->held[i]), 0x60 + i, size); + } + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skmsg_test_cleanup, state), 0); + KUNIT_ASSERT_EQ(test, sock_create_kern(&init_net, AF_INET, SOCK_STREAM, 0, &state->socket), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + page = &state->folio->page; + sk = state->socket->sk; + src = &state->messages[0]; + dst = &state->messages[1]; + sk_msg_init(src); + sk_msg_init(dst); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, (size / 2) - 13, size + (size / 2) + 13), 0); + sk_mem_charge(sk, (size / 2) - 13); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 2 * size), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + KUNIT_ASSERT_EQ(test, sk_msg_clone(sk, dst, src, 0, size + (size / 2) - 13), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 11); + sk_msg_free_partial(sk, src, (size / 2) - 13); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 0), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 11); + sk_msg_free(sk, dst); + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, size)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + sk_msg_free_partial(sk, src, size); + for (i = 0; i < 1024; i++) { + unsigned int index = (i + 1) & 3; + + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, index * size), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + sk_msg_free_partial(sk, src, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 7); + } + sk_msg_free(sk, src); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + /* A borrowed descriptor copy has an independent ownership lifetime. */ + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 0), 0); + sk_mem_charge(sk, size); + memcpy(&state->borrowed, src, sizeof(*src)); + sk_msg_subpages_get(&state->borrowed); + sk_msg_free(sk, src); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 5); + put_held(f, 0); + attempt = mm_subpage_alloc_at(f->pool, 0); + if (!IS_ERR(attempt)) + f->held[0] = attempt; + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(sg_virt(&state->borrowed.sg.data[0]), 0x60, size), NULL); + sk_msg_subpages_release(&state->borrowed); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 0, 0)); + /* Full transfer moves the set, and zero-copy iterator trim releases its references. */ + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 3 * size), 0); + sk_mem_charge(sk, size); + sk_msg_xfer_full(dst, src); + KUNIT_EXPECT_EQ(test, src->sg.size, 0U); + sk_msg_free(sk, dst); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + bvec_set_page(&bvec, page, size - 17, 3 * size + 17); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size - 17); + KUNIT_ASSERT_EQ(test, sk_msg_zerocopy_from_iter(sk, &iter, src, size - 17), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 6); + sk_msg_trim(sk, src, 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} + +static void user4k_data_skmsg_test(struct kunit *test) +{ + test_data_skmsg_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_skmsg_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_skmsg_granule(test, 14); +} +#endif + +static void user16k_data_owner_test(struct kunit *test) +{ + struct data_fixture *f, *dest, *small; + struct mm_subpage *slot, *found; + unsigned int count = PAGE_SIZE / SZ_16K; + + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + dest = data_fixture_create_granule(test, 14); + small = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(dest->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(small->pool), 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(small, 0)); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_4K)), -EINVAL); + for (unsigned int i = 0; i < count; i++) { + slot = hold_alloc_at(f, i, i * SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_shift(slot), 14U); + KUNIT_EXPECT_EQ(test, mm_subpage_size(slot), (unsigned int)SZ_16K); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)SZ_16K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_16K), NULL); + memset(slot_address(slot), 0x30 + i, SZ_16K); + for (unsigned int quarter = 0; quarter < 4; quarter++) { + found = mm_subpage_get_from_phys(mm_subpage_phys(slot) + quarter * SZ_4K); + KUNIT_ASSERT_PTR_EQ(test, found, slot); + mm_subpage_put(found); + } + } + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(f->pool)), -EAGAIN); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(dest->pool, small->held[0])), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(small->pool, f->held[0])), -EINVAL); + slot = mm_subpage_copy_at(dest->pool, f->held[count - 1], SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + dest->held[0] = slot; + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0x30 + count - 1, SZ_16K), NULL); + /* An interior-byte lookup holds the whole identity across release/reuse. */ + f->held[count] = mm_subpage_get_from_phys(mm_subpage_phys(f->held[1]) + 3 * SZ_4K); + KUNIT_ASSERT_PTR_EQ(test, f->held[count], f->held[1]); + put_held(f, 1); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_16K)), -EAGAIN); + put_held(f, count); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + for (unsigned int i = 0; i < count; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x30 + i, SZ_16K), NULL); + mm_subpage_pool_close(f->pool); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(f->pool)), -ESHUTDOWN); + for (unsigned int i = 0; i < count; i++) + put_held(f, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), 0UL); +} + +static void user16k_data_refset_test(struct kunit *test) +{ + struct data_fixture *f; + struct refs_test_state *state; + unsigned int count = PAGE_SIZE / SZ_16K; + unsigned long mask; + struct page *page; + + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (unsigned int i = 0; i < count; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_16K)); + state->folio = mm_subpage_folio(f->held[0]); + page = &state->folio->page; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, refs_test_cleanup, state), 0); + /* Metadata-only ownership fixture: no VMA or rmap is published. */ + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, + SZ_16K - 1024, 2048, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count + 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2); + state->refs[1] = mm_subpage_refs_get(state->refs[0]); + mask = BIT(SZ_16K >> MM_SUBPAGE_SHIFT); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2 + 1); + mask = 0; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[0], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count + 1); + put_held(f, 1); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_16K)), -EAGAIN); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count); +} + +static void user16k_data_mte_test(struct kunit *test) +{ +#ifdef CONFIG_ARM64_MTE + struct data_fixture *f; + struct folio *folio; + struct mm_subpage *copy; + unsigned int count = PAGE_SIZE / SZ_16K; + int ret; + + if (PAGE_SIZE <= SZ_16K || !system_supports_mte()) { + kunit_skip(test, "requires MTE and native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + KUNIT_ASSERT_NOT_NULL(test, f); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + if (try_page_mte_tagging(&folio->page)) + mte_clear_page_tags(folio_address(folio)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(folio_address(folio) + i * SZ_4K, 9); + set_page_mte_tagged(&folio->page); + ret = mm_subpage_pool_add_folio(f->pool, folio, GFP_KERNEL); + if (ret) + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + for (unsigned int i = 0; i < count; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_16K)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, 0)); + memset(address, 0x50 + i, SZ_4K); + test_slot_tags_set(address, 1 + i % 15); + } + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + bool cleared = i >= 4 && i < 8; + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, cleared ? 0 : 1 + i % 15)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(address, cleared ? 0 : 0x50 + i, SZ_4K), NULL); + } + put_held(f, 1); + copy = mm_subpage_copy_at(f->pool, f->held[count - 1], SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + f->held[1] = copy; + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int source = i >= 4 && i < 8 ? i + DATA_SLOTS - 8 : i; + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, 1 + source % 15)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(address, 0x50 + source, SZ_4K), NULL); + } +#else + kunit_skip(test, "MTE is not configured"); +#endif +} + +static void expect_file_coverage(struct kunit *test, struct folio *folio, + unsigned int leaves, unsigned int first, unsigned int tail, + unsigned int final) +{ + struct mm_subpage_mapcounts counts; + unsigned int i; + + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(&folio->page, &counts), leaves); + for (i = 0; i < DATA_SLOTS; i++) { + unsigned int expected = i == 0 ? first : + i == DATA_SLOTS - 1 ? final : i >= DATA_SLOTS - 4 ? tail : 0; + + KUNIT_EXPECT_EQ_MSG(test, counts.slots[i], expected, "coverage slot %u", i); + } +} + +static void user16k_file_coverage_test(struct kunit *test) +{ + struct folio *folio; + phys_addr_t base, tail, last; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative file coverage requires larger native backing"); + return; + } + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, folio), 0); + base = PFN_PHYS(folio_pfn(folio)); + tail = base + PAGE_SIZE - SZ_16K; + last = base + PAGE_SIZE - SZ_4K; + expect_file_coverage(test, folio, 0, 0, 0, 0); + __mm_subpage_file_map_add(tail, SZ_16K); + expect_file_coverage(test, folio, 1, 0, 1, 1); + __mm_subpage_file_map_add(last, SZ_4K); + expect_file_coverage(test, folio, 2, 0, 1, 2); + __mm_subpage_file_map_add(tail, SZ_16K); + expect_file_coverage(test, folio, 3, 0, 2, 3); + __mm_subpage_file_map_add(base, SZ_4K); + expect_file_coverage(test, folio, 4, 1, 2, 3); + __mm_subpage_file_map_del(tail, SZ_16K); + expect_file_coverage(test, folio, 3, 1, 1, 2); + __mm_subpage_file_map_del(last, SZ_4K); + expect_file_coverage(test, folio, 2, 1, 1, 1); + __mm_subpage_file_map_del(tail, SZ_16K); + expect_file_coverage(test, folio, 1, 1, 0, 0); + __mm_subpage_file_map_del(base, SZ_4K); + expect_file_coverage(test, folio, 0, 0, 0, 0); +} + +static struct kunit_case user4k_data_cases[] = { + KUNIT_CASE(user4k_data_reuse_test), + KUNIT_CASE(user4k_data_refset_test), +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) + KUNIT_CASE(user4k_data_skmsg_test), + KUNIT_CASE(user16k_data_skmsg_test), +#endif +#ifdef CONFIG_NET + KUNIT_CASE(user4k_data_skb_test), + KUNIT_CASE(user4k_data_skb_checksum_test), + +#ifdef CONFIG_INET + KUNIT_CASE(user4k_data_skb_prune_test), + KUNIT_CASE(user16k_data_skb_prune_test), +#endif +#endif + KUNIT_CASE(user4k_data_mte_test), + KUNIT_CASE(user4k_data_copy_extent_test), + KUNIT_CASE(user16k_data_owner_test), + KUNIT_CASE(user16k_file_coverage_test), + KUNIT_CASE(user16k_data_refset_test), + KUNIT_CASE(user16k_data_mte_test), + KUNIT_CASE(user4k_data_swapcache_reuse_test), + KUNIT_CASE(user4k_data_restore_test), + KUNIT_CASE(user16k_data_restore_test), + KUNIT_CASE(user4k_data_restore_race_test), + KUNIT_CASE(user4k_data_migration_owner_test), + KUNIT_CASE(user16k_data_migration_owner_test), + KUNIT_CASE(user4k_data_copy_test), + KUNIT_CASE(user4k_data_lookup_test), + KUNIT_CASE(user4k_data_exact_slot_test), + KUNIT_CASE(user4k_data_close_test), + KUNIT_CASE(user4k_data_deferred_test), + KUNIT_CASE(user4k_data_backing_reject_test), + KUNIT_CASE(user4k_data_concurrent_test), + KUNIT_CASE(user4k_data_lookup_race_test), + {} +}; + +static void user4k_data_suite_exit(struct kunit_suite *suite) +{ + /* Drain batched metadata frees before the disposable guest powers off. */ + kvfree_rcu_barrier(); +} + +static struct kunit_suite user4k_data_suite = { + .name = "arm64-user4k-data", + .test_cases = user4k_data_cases, + .suite_exit = user4k_data_suite_exit, +}; + +kunit_test_suite(user4k_data_suite); +MODULE_LICENSE("GPL"); diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c new file mode 100644 index 0000000000000..e406b97ea6235 --- /dev/null +++ b/arch/arm64/mm/user4k-test.c @@ -0,0 +1,14237 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Architectural prerequisite for mixed-granule userspace. + * + * Packed architectural tables and isolated Linux mms are separate fixtures. + * PAGE_SIZE, struct page, all ordinary mms and TTBR1 retain native geometry. Only bounded + * kernel workers may borrow the isolated Linux mms. Never pass + * packed architectural tables to generic MM walkers or expose them to a task. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../../drivers/iommu/iommufd/iommufd_private.h" +#include "../../../drivers/iommu/iommufd/io_pagetable.h" +#include "../../../mm/internal.h" +#include "../../../mm/vma.h" +#include "../../../mm/swap.h" +#ifdef CONFIG_FUTEX +#include "../../../kernel/futex/futex.h" +#endif + +#define USER4K_SHIFT 12 +#define TEST_SLOTS MM_SUBPAGES_PER_PAGE +#define USER4K_SIZE BIT(USER4K_SHIFT) +#define TEST_VA (4UL * SZ_1M) +#define TEST_LEVELS 4 +#define PROBE_SLOTS 4 +#define TEST_ROUNDS 8 +#define TEST_PROT (PTE_TYPE_PAGE | PTE_USER | PTE_AF | PTE_NG | \ + PTE_SHARED | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL)) + +static bool user4k_test_enabled; + +static int __init user4k_test_setup(char *str) +{ + bool enabled; + int ret = kstrtobool(str, &enabled); + + if (ret) + return ret; + /* Software MM tests are sanitizable; raw/borrowed TTBR0 tests stay gated. */ + if (enabled && IS_ENABLED(CONFIG_KASAN)) + return -EINVAL; + user4k_test_enabled = enabled; + return 0; +} +early_param("arm64.user4k_test", user4k_test_setup); + +struct test_tables { + u64 *level[TEST_LEVELS]; + unsigned int shift; +}; + +struct test_result { + u64 native[PROBE_SLOTS]; + u64 read[PROBE_SLOTS]; + u64 write[PROBE_SLOTS]; + u64 replaced[PROBE_SLOTS]; + u64 replaced_read; + u64 kernel; + u64 tcr; + u64 ttbr1; + u64 restored_tcr; + u64 restored_ttbr0; +}; + +static void test_free_page(void *ptr) +{ + free_page((unsigned long)ptr); +} + +static void *test_alloc_page(struct kunit *test) +{ + void *ptr = (void *)get_zeroed_page(GFP_KERNEL); + + if (!ptr) + return NULL; + if (kunit_add_action_or_reset(test, test_free_page, ptr)) + return NULL; + return ptr; +} + +static void test_put_compound(void *ptr) +{ + put_page(ptr); +} + +static void *test_alloc_granule(struct kunit *test, unsigned int shift) +{ + struct page *page; + + if (shift <= PAGE_SHIFT) + return test_alloc_page(test); + page = alloc_pages(GFP_KERNEL | __GFP_ZERO | __GFP_COMP, shift - PAGE_SHIFT); + if (!page) + return NULL; + if (kunit_add_action_or_reset(test, test_put_compound, page)) + return NULL; + return page_address(page); +} + +static unsigned int test_index(unsigned long va, unsigned int shift, + unsigned int level) +{ + unsigned int bits = shift - PTDESC_ORDER; + + return (va >> ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, shift)) & (BIT(bits) - 1); +} + +static int test_alloc_tables(struct kunit *test, struct test_tables *tables, + unsigned int shift) +{ + int level; + void *packed = NULL; + + tables->shift = shift; + if (shift == USER4K_SHIFT) { + /* Also test table address bits 13:12, not just leaf addresses. */ + packed = test_alloc_page(test); + if (!packed) + return -ENOMEM; + } + for (level = 0; level < TEST_LEVELS; level++) { + tables->level[level] = packed ? packed + level * USER4K_SIZE : + test_alloc_page(test); + if (!tables->level[level]) + return -ENOMEM; + } + for (level = 0; level < TEST_LEVELS - 1; level++) { + unsigned int index = test_index(TEST_VA, shift, level); + + tables->level[level][index] = + virt_to_phys(tables->level[level + 1]) | PMD_TYPE_TABLE; + } + return 0; +} + +static void user4k_geometry_test(struct kunit *test) +{ + static const unsigned int shifts4k[] = { 39, 30, 21, 12 }; + static const unsigned int shifts16k[] = { 47, 36, 25, 14 }; + static const unsigned int shifts64k[] = { 55, 42, 29, 16 }; + unsigned int level; + + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 12), 4); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 14), 4); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 16), 3); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(42, 16), 2); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(39, 12), 3); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(47, 14), 3); + for (level = 0; level < TEST_LEVELS; level++) { + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, 12), + shifts4k[level]); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, 14), + shifts16k[level]); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT(level), + PAGE_SHIFT == 16 ? shifts64k[level] : shifts16k[level]); + KUNIT_EXPECT_EQ(test, test_index(BIT(shifts4k[level]), 12, level), 1U); + KUNIT_EXPECT_EQ(test, test_index(BIT(shifts4k[level]) - 1, 12, level), 0U); + } +} + +static void test_leaf_encoding(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + const phys_addr_t base = GENMASK_ULL(47, PAGE_SHIFT); + pgprot_t prot = __pgprot(TEST_PROT | PTE_RDONLY); + unsigned int i; + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, mm); + pte = phys_pte_mm(mm, base, prot); + KUNIT_EXPECT_EQ(test, pte_val(pte), pte_val(pfn_pte(base >> PAGE_SHIFT, prot))); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), base); + KUNIT_EXPECT_EQ(test, pgprot_val(pte_pgprot_mm(mm, pte)), pgprot_val(prot)); + mm->context.flags |= shift == 12 ? MMCF_USER_4K : MMCF_USER_16K; + for (i = 0; i < (PAGE_SIZE >> shift); i++) { + phys_addr_t phys = base + i * (1UL << shift); + + pte = phys_pte_mm(mm, phys, prot); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys); + KUNIT_EXPECT_EQ(test, pte_pfn(pte), (unsigned long)(base >> PAGE_SHIFT)); + KUNIT_EXPECT_EQ(test, pgprot_val(pte_pgprot_mm(mm, pte)), pgprot_val(prot)); + if (i + 1 < (PAGE_SIZE >> shift)) + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte_advance_pfn_mm(mm, pte, 1)), + phys + (1UL << shift)); + pte = pte_modify(pte, __pgprot(TEST_PROT)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys); + KUNIT_EXPECT_FALSE(test, pte_val(pte) & PTE_RDONLY); + } +} + +static void user4k_leaf_encoding_test(struct kunit *test) +{ + test_leaf_encoding(test, 12); +} + +static void user16k_leaf_encoding_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative leaves require larger native backing"); + return; + } + test_leaf_encoding(test, 14); +} + + +/* Caller masks all asynchronous exceptions; this function cannot sleep. */ +static void test_switch_ttbr0(u64 ttbr, u64 tcr) +{ + /* The zero root is valid for both tested granules. */ + cpu_set_reserved_ttbr0(); + local_flush_tlb_all(); + write_sysreg(tcr, tcr_el1); + isb(); + write_sysreg(ttbr, ttbr0_el1); + isb(); +} + +static u64 test_translate(unsigned long va, bool write) +{ + if (write) + asm volatile("at s1e0w, %0" : : "r" (va) : "memory"); + else + asm volatile("at s1e0r, %0" : : "r" (va) : "memory"); + isb(); + return read_sysreg_par(); +} + +#define MM_TEST_MAPPINGS 11 + +struct test_mm { + struct kunit *test; + struct mm_struct *mm; + unsigned long address[MM_TEST_MAPPINGS]; + phys_addr_t phys[MM_TEST_MAPPINGS]; +}; + +static pte_t *test_mm_lookup(struct mm_struct *mm, unsigned long addr, + spinlock_t **ptl); + +static void test_mm_hardware(struct test_mm *ctx) +{ + struct kunit *test = ctx->test; + struct vm_area_struct vma = TLB_FLUSH_VMA(ctx->mm, 0); + struct arch_tlbflush_unmap_batch batch = {}; + pte_t *leaves[4]; + u64 invalidated[3][4]; + u64 result[MM_TEST_MAPPINGS], writes[MM_TEST_MAPPINGS]; + u64 old_tcr, old_ttbr0, old_ttbr1, old_par, tcr, kernel; + u64 old_id = atomic64_read(&ctx->mm->context.id); + u64 restored_tcr, restored_ttbr0, retained_ttbr1; + unsigned long flags; + unsigned int i, step; + + for (i = 0; i < ARRAY_SIZE(leaves); i++) { + spinlock_t *ptl; + + leaves[i] = test_mm_lookup(ctx->mm, ctx->address[i], &ptl); + if (leaves[i]) + pte_unmap_unlock(leaves[i], ptl); + KUNIT_ASSERT_NOT_NULL(test, leaves[i]); + } + + preempt_disable(); + flags = local_daif_save(); + old_tcr = read_sysreg(tcr_el1); + old_ttbr0 = read_sysreg(ttbr0_el1); + old_ttbr1 = read_sysreg(ttbr1_el1); + old_par = read_sysreg_par(); + if ((old_tcr & (TCR_EL1_DS | TCR_EL1_TG1_MASK | TCR_EL1_A1)) != + (TCR_TG1_16K | TCR_EL1_A1)) { + local_daif_restore(flags); + preempt_enable(); + kunit_skip(test, "requires 16K TTBR1, DS=0, and A1=1"); + return; + } + tcr = (old_tcr & ~(TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK | TCR_EL1_EPD0)) | + TCR_TG0_4K | TCR_T0SZ(48); + test_switch_ttbr0(virt_to_phys(ctx->mm->pgd), tcr); + /* A1 selects the saved TTBR1 ASID for these private test translations. */ + atomic64_set(&ctx->mm->context.id, old_ttbr1 >> 48); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + result[i] = test_translate(ctx->address[i], false); + writes[i] = test_translate(ctx->address[i], true); + } + for (step = 0; step < ARRAY_SIZE(invalidated); step++) { + unsigned int first = step == 2 ? 3 : 1; + unsigned int last = step == 1 ? 3 : first + 1; + + for (i = first; i < last; i++) + WRITE_ONCE(*leaves[i], __pte(0)); + switch (step) { + case 0: + __flush_tlb_page(&vma, ctx->address[first] + 1, + TLBF_NONOTIFY | TLBF_NOBROADCAST); + break; + case 1: + __flush_tlb_range(&vma, ctx->address[first], ctx->address[last], + SZ_4K, 3, TLBF_NONOTIFY); + break; + case 2: + arch_tlbbatch_add_pending(&batch, ctx->mm, ctx->address[first], + ctx->address[first] + SZ_4K); + arch_tlbbatch_flush(&batch); + break; + } + isb(); + for (i = 0; i < ARRAY_SIZE(leaves); i++) + invalidated[step][i] = test_translate(ctx->address[i], false); + for (i = first; i < last; i++) + WRITE_ONCE(*leaves[i], __pte(ctx->phys[i] | TEST_PROT)); + dsb(ishst); + isb(); + /* Repopulate before the next independent invalidation. */ + for (i = 0; i < ARRAY_SIZE(leaves); i++) + test_translate(ctx->address[i], false); + } + asm volatile("at s1e1r, %0" : : "r" (ctx->mm->pgd) : "memory"); + isb(); + kernel = read_sysreg_par(); + retained_ttbr1 = read_sysreg(ttbr1_el1); + test_switch_ttbr0(old_ttbr0, old_tcr); + restored_tcr = read_sysreg(tcr_el1); + restored_ttbr0 = read_sysreg(ttbr0_el1); + write_sysreg(old_par, par_el1); + isb(); + atomic64_set(&ctx->mm->context.id, old_id); + local_daif_restore(flags); + preempt_enable(); + + KUNIT_EXPECT_EQ(test, restored_tcr, old_tcr); + KUNIT_EXPECT_EQ(test, restored_ttbr0, old_ttbr0); + KUNIT_EXPECT_EQ(test, retained_ttbr1, old_ttbr1); + KUNIT_EXPECT_EQ(test, kernel & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, kernel & SYS_PAR_EL1_PA, + virt_to_phys(ctx->mm->pgd) & SYS_PAR_EL1_PA); + for (step = 0; step < ARRAY_SIZE(invalidated); step++) { + for (i = 0; i < ARRAY_SIZE(leaves); i++) { + bool hole = step == 0 ? i == 1 : step == 1 ? i == 1 || i == 2 : i == 3; + + KUNIT_EXPECT_EQ(test, invalidated[step][i] & SYS_PAR_EL1_F, (u64)hole); + if (!hole) + KUNIT_EXPECT_EQ(test, invalidated[step][i] & SYS_PAR_EL1_PA, + ctx->phys[i]); + } + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + KUNIT_EXPECT_EQ(test, result[i] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, result[i] & SYS_PAR_EL1_PA, ctx->phys[i]); + KUNIT_EXPECT_EQ(test, writes[i] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, writes[i] & SYS_PAR_EL1_PA, ctx->phys[i]); + } +} + +struct test_walk { + struct test_mm *ctx; + unsigned long size; + unsigned long holes; + unsigned int present; +}; + +/* Callbacks run under the PTE lock; record errors without invoking KUnit. */ +static int test_walk_pte(pte_t *pte, unsigned long addr, unsigned long next, + struct mm_walk *walk) +{ + struct test_walk *state = walk->private; + u64 value = pte_val(ptep_get(pte)); + unsigned int i; + + if (next - addr != state->size) + return -EINVAL; + if (!value) { + state->holes += next - addr; + return 0; + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + if (state->ctx->address[i] != addr) + continue; + if (value != (state->ctx->phys[i] | TEST_PROT)) + return -EINVAL; + state->present++; + return 0; + } + return -EINVAL; +} + +static int test_walk_hole(unsigned long addr, unsigned long next, int depth, + struct mm_walk *walk) +{ + struct test_walk *state = walk->private; + + state->holes += next - addr; + return 0; +} + +static const struct mm_walk_ops test_walk_ops = { + .pte_entry = test_walk_pte, + .pte_hole = test_walk_hole, +}; + +static pte_t *test_mm_lookup(struct mm_struct *mm, unsigned long addr, + spinlock_t **ptl) +{ + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + if (pgd_none(*pgd) || pgd_bad(*pgd)) + return NULL; + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_mm(mm, *p4d) || p4d_bad_mm(mm, *p4d)) + return NULL; + pud = pud_offset_mm(mm, p4d, addr); + if (pud_none(*pud) || pud_bad(*pud)) + return NULL; + pmd = pmd_offset_mm(mm, pud, addr); + if (pmd_none(*pmd) || pmd_bad(*pmd)) + return NULL; + return pte_offset_map_lock(mm, pmd, addr, ptl); +} + +static void test_mm_destroy(void *ptr) +{ + struct test_mm *ctx = ptr; + struct mm_struct *mm = ctx->mm; + struct mmu_gather tlb; + unsigned int i; + + if (!mm) + return; + mmap_write_lock(mm); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + + if (pte) { + /* Test-owned physical pages, not faulted/rmap-owned pages. */ + pte_clear(mm, ctx->address[i], pte); + pte_unmap_unlock(pte, ptl); + } + } + /* Exercise the real generic gather/teardown path, not a test walker. */ + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(ctx->test, mm_pgtables_bytes(mm), 0UL); + mmput(mm); + ctx->mm = NULL; +} + +static unsigned int test_mm_backing_pages(struct test_mm *ctx) +{ + struct page *owners[3 * MM_TEST_MAPPINGS]; + unsigned int count = 0, total = 0, i, j, level; + struct mm_struct *mm = ctx->mm; + + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + unsigned long addr = ctx->address[i]; + p4d_t *p4d = p4d_offset_mm(mm, pgd_offset(mm, addr), addr); + pud_t *pud = pud_offset_mm(mm, p4d, addr); + pmd_t *pmd = pmd_offset_mm(mm, pud, addr); + pte_t *pte = pte_offset_kernel_mm(mm, pmd, addr); + struct page *pages[] = { virt_to_page(pud), virt_to_page(pmd), virt_to_page(pte) }; + + for (level = mm_pud_folded(mm) ? 1 : 0; level < ARRAY_SIZE(pages); level++) { + pages[level] = compound_head(pages[level]); + for (j = 0; j < count; j++) + if (owners[j] == pages[level]) + break; + if (j == count) { + owners[count++] = pages[level]; + total += 1U << compound_order(pages[level]); + } + } + } + return total; +} + +/* Only valid for a private, empty mm with no ASID or hardware references. */ +static int test_mm_select_granule(struct mm_struct *mm, unsigned int shift) +{ + pgd_t *native_pgd = mm->pgd, *new_pgd; + unsigned long flag = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + + mm->context.flags |= flag; + new_pgd = pgd_alloc(mm); + mm->context.flags &= ~flag; + if (!new_pgd) + return -ENOMEM; + pgd_free(mm, native_pgd); + mm->pgd = new_pgd; + mm->context.flags |= flag; + return 0; +} + +static int test_mm_select4k(struct mm_struct *mm) +{ + return test_mm_select_granule(mm, 12); +} + +static void test_mm_tables(struct kunit *test, unsigned int shift, bool hardware) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + bool small = shift != PAGE_SHIFT; + unsigned int pmd_shift = small ? shift + (shift - 3) : PMD_SHIFT; + unsigned int pud_shift = small ? shift + 2 * (shift - 3) : PUD_SHIFT; + unsigned int pgd_shift = small ? (shift == 16 ? pud_shift : + shift + 3 * (shift - 3)) : PGDIR_SHIFT; + unsigned long size = 1UL << shift; + struct mm_struct *mm; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + for (i = 0; i < 4; i++) + ctx->address[i] = TEST_VA + i * size; + ctx->address[4] = (1UL << pmd_shift) - size; + ctx->address[5] = 1UL << pmd_shift; + ctx->address[6] = (1UL << pud_shift) - size; + ctx->address[7] = 1UL << pud_shift; + ctx->address[8] = (1UL << pgd_shift) - size; + ctx->address[9] = 1UL << pgd_shift; + ctx->address[10] = (1UL << 48) - size; + if (pud_shift == pgd_shift) { + ctx->address[6] -= size; + ctx->address[7] += size; + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + void *data = test_alloc_granule(test, shift); + + KUNIT_ASSERT_NOT_NULL(test, data); + ctx->phys[i] = virt_to_phys(data) + (shift < PAGE_SHIFT ? (i % (PAGE_SIZE >> shift)) * size : 0); + } + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + ctx->mm = mm; + /* Register after the data allocations: detach tables before freeing data. */ + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_EXPECT_EQ(test, mm_page_shift(mm), (unsigned int)PAGE_SHIFT); + if (small) { + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shift), 0); + KUNIT_EXPECT_EQ(test, (unsigned long)mm->pgd & (size - 1), 0UL); + } + KUNIT_EXPECT_EQ(test, mm_page_shift(mm), shift); + if (small) + KUNIT_EXPECT_EQ(test, mm_pud_folded(mm), shift == 16); + KUNIT_EXPECT_EQ(test, mm_page_size(mm), size); + KUNIT_EXPECT_EQ(test, mm_page_mask(mm), ~(size - 1)); + KUNIT_EXPECT_EQ(test, mm_page_shift(&init_mm), (unsigned int)PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, pmd_addr_end_mm(mm, (1UL << pmd_shift) - 1, + (1UL << pmd_shift) + size), + 1UL << pmd_shift); + KUNIT_EXPECT_EQ(test, pgd_addr_end_mm(mm, ULONG_MAX - size, 0), 0UL); + + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + unsigned long addr = ctx->address[i]; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d = p4d_alloc(mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + KUNIT_ASSERT_NOT_NULL(test, p4d); + pud = pud_alloc(mm, p4d, addr); + KUNIT_ASSERT_NOT_NULL(test, pud); + pmd = pmd_alloc(mm, pud, addr); + KUNIT_ASSERT_NOT_NULL(test, pmd); + pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + /* Do not assert while holding the spinlock/RCU section. */ + set_pte(pte, __pte(ctx->phys[i] | TEST_PROT)); + pte_unmap_unlock(pte, ptl); + } + /* Installed table bytes and physical backing (including cached slots) differ. */ + { + unsigned int shifts[] = { pmd_shift, pud_shift, pgd_shift }; + unsigned int tables = 0, owners = 0; + + for (unsigned int level = 0; level < (mm_pud_folded(mm) ? 2 : 3); level++) { + unsigned int unique = 0; + + for (unsigned int a = 0; a < MM_TEST_MAPPINGS; a++) { + unsigned int b; + + for (b = 0; b < a; b++) + if ((ctx->address[a] >> shifts[level]) == + (ctx->address[b] >> shifts[level])) + break; + unique += b == a; + } + tables += unique; + if (shift < PAGE_SHIFT) + owners += DIV_ROUND_UP(unique, PAGE_SIZE >> shift); + else + owners += unique << (shift - PAGE_SHIFT); + } + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), tables * size); + KUNIT_EXPECT_EQ(test, test_mm_backing_pages(ctx), owners); + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + u64 value; + + KUNIT_ASSERT_NOT_NULL(test, pte); + value = pte_val(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, value, ctx->phys[i] | TEST_PROT); + } + { + struct vm_area_struct *vma = vm_area_alloc(mm); + struct test_walk state = { .ctx = ctx, .size = size }; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_start = 0; + vma->vm_end = 1UL << 48; + mmap_write_lock(mm); + ret = walk_page_range_vma(vma, 0, 1UL << 48, &test_walk_ops, &state); + mmap_write_unlock(mm); + vm_area_free(vma); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, state.present, MM_TEST_MAPPINGS); + KUNIT_EXPECT_EQ(test, state.holes, (1UL << 48) - MM_TEST_MAPPINGS * size); + } + if (hardware) + test_mm_hardware(ctx); + { + struct mmu_gather tlb; + unsigned long before = mm_pgtables_bytes(mm); + unsigned long boundary = 1UL << pmd_shift; + + mmap_write_lock(mm); + /* An adjacent VMA's unaligned ceiling must keep the shared table. */ + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, boundary - size, 0, boundary - size); + tlb_finish_mmu(&tlb); + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), before); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte; + + if (ctx->address[i] >= boundary) + continue; + pte = test_mm_lookup(mm, ctx->address[i], &ptl); + if (pte) { + pte_clear(mm, ctx->address[i], pte); + pte_unmap_unlock(pte, ptl); + } + } + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, boundary, 0, boundary); + tlb_finish_mmu(&tlb); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), before - size); + /* The next table and the higher levels must still be walkable. */ + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + u64 value; + + if (ctx->address[i] < boundary) { + if (pte) + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_PTR_EQ(test, pte, NULL); + continue; + } + KUNIT_ASSERT_NOT_NULL(test, pte); + value = pte_val(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, value, ctx->phys[i] | TEST_PROT); + } + } + test_mm_destroy(ctx); +} + +static void user4k_mm_tables_test(struct kunit *test) +{ + test_mm_tables(test, 12, false); +} + +static void native_mm_tables_test(struct kunit *test) +{ + test_mm_tables(test, PAGE_SHIFT, false); +} + +static void user16k_mm_tables_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative tables require larger native backing"); + return; + } + test_mm_tables(test, 14, false); +} + +static void user64k_mm_tables_test(struct kunit *test) +{ + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "64K is already the native geometry"); + return; + } + test_mm_tables(test, 16, false); +} + +static void test_free_vma(void *ptr) +{ + vm_area_free(ptr); +} + +static void user4k_anon_read_fault_test(struct kunit *test) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + struct vm_area_struct *vma; + vm_flags_t vm_flags = VM_READ | VM_MAYREAD; + unsigned int i, j; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + ctx->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, ctx->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_ASSERT_EQ(test, test_mm_select4k(ctx->mm), 0); + for (i = 0; i < 4; i++) + ctx->address[i] = TEST_VA + i * SZ_4K; + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + PAGE_SIZE; + vma_set_anonymous(vma); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + vm_flags |= VM_NOHUGEPAGE; +#endif + vm_flags_init(vma, vm_flags); + vma->vm_page_prot = vm_get_page_prot(vm_flags); + /* This VMA is a private fault fixture, not inserted into the VMA tree. */ + for (i = 0; i < 4; i++) { + vm_fault_t ret; + + mmap_read_lock(ctx->mm); + ret = handle_mm_fault(vma, ctx->address[i] + 1, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(ctx->mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + for (j = 0; j < 4; j++) { + spinlock_t *ptl; + pte_t *ptep = test_mm_lookup(ctx->mm, ctx->address[j], &ptl); + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, ptep); + pte = ptep_get(ptep); + pte_unmap_unlock(ptep, ptl); + if (j > i) { + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + continue; + } + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_special(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_TRUE(test, is_zero_pfn(pte_pfn(pte))); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(ctx->mm, MM_ANONPAGES), 0L); +} + +static void user4k_mm_hardware_test(struct kunit *test) +{ + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE)) { + kunit_skip(test, "requires 4K, no CnP, and direct PTE permissions"); + return; + } + test_mm_tables(test, 12, true); +} + +struct fragment_fixture { + struct mm_struct *mm; + void *table[TEST_SLOTS + 2]; + struct page *pinned[3]; +}; + +static void fragment_fixture_destroy(void *data) +{ + struct fragment_fixture *f = data; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(f->table); i++) + if (f->table[i]) + arm64_pgtable_free(f->table[i]); + if (f->mm) + mmput(f->mm); + rcu_barrier(); + for (i = 0; i < ARRAY_SIZE(f->pinned); i++) + if (f->pinned[i]) + put_page(f->pinned[i]); +} + +static void test_fragment_lifetime(struct kunit *test, unsigned int shift) +{ + struct fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct ptdesc *pte_owner; + unsigned int i, slots = PAGE_SIZE >> shift; + unsigned long size = 1UL << shift; + unsigned int read_shift = 0; + bool live; + + KUNIT_ASSERT_NOT_NULL(test, f); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, fragment_fixture_destroy, f), 0); + /* Direct allocator fixture: the mm's native root is never changed or used. */ + for (i = 0; i < slots + 2; i++) { + enum arm64_user4k_pt_level level = i < slots ? USER4K_PT_PTE : i - slots + 1; + + f->table[i] = arm64_user_pt_alloc_granule(f->mm, level, shift); + KUNIT_ASSERT_NOT_NULL(test, f->table[i]); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)f->table[i], size)); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(f->table[i]), shift); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(f->table[i] + size - sizeof(pte_t)), shift); + KUNIT_EXPECT_EQ(test, pte_batch_hint(f->table[i], __pte(TEST_PROT | PTE_CONT)), + IS_ENABLED(CONFIG_ARM64_CONTPTE) ? arm64_cont_ptes(shift) : 1U); + if (!i && PAGE_SHIFT == 16) { + void *cached = f->mm->context.user4k_pt_frag[level]; + + KUNIT_EXPECT_PTR_EQ(test, arm64_user_pt_alloc_granule(f->mm, level, + shift == 12 ? 14 : 12), NULL); + KUNIT_EXPECT_PTR_EQ(test, f->mm->context.user4k_pt_frag[level], cached); + } + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0, size), NULL); + if (i == 0 || i >= slots) { + unsigned int slot = i == 0 ? 0 : i - slots + 1; + + f->pinned[slot] = virt_to_page(f->table[i]); + get_page(f->pinned[slot]); + } else { + KUNIT_EXPECT_PTR_EQ(test, virt_to_page(f->table[i]), f->pinned[0]); + KUNIT_EXPECT_EQ(test, (unsigned long)f->table[i] - + (unsigned long)f->table[0], i * (unsigned long)size); + } + memset(f->table[i], 0x50 + i, size); + } + for (i = 0; i < slots + 2; i++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0x50 + i, size), NULL); + memset(f->table[i], 0, size); + } + /* Pointer-only walkers must inspect all 128 entries of a 16K group. */ + if (IS_ENABLED(CONFIG_ARM64_CONTPTE)) { + pte_t *ptes = f->table[0]; + unsigned int nr = arm64_cont_ptes(shift), j; + pte_t observed; + + for (j = 0; j < nr; j++) + ptes[j] = pte_mkold(pte_mkclean(__pte(TEST_PROT | PTE_CONT | + (SZ_4M + (j << shift))))); + ptes[nr - 1] = pte_mkyoung(pte_mkdirty(ptes[nr - 1])); + observed = ptep_get(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + observed = ptep_get_lockless(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + KUNIT_EXPECT_EQ(test, pte_val(observed) & + (ARM64_USER4K_TABLE_ADDR_MASK & ~(size - 1)), + (u64)(SZ_4M + size)); + memset(ptes, 0, size); + } + KUNIT_EXPECT_PTR_NE(test, f->pinned[0], f->pinned[1]); + KUNIT_EXPECT_PTR_NE(test, f->pinned[1], f->pinned[2]); + pte_owner = page_ptdesc(f->pinned[0]); + KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots); + arm64_pte_free_defer(f->mm, f->table[0]); + f->table[0] = NULL; + KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots - 1); + for (i = 1; i < slots - 1; i++) { + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + } + /* A pre-existing software walker must survive the last fragment release. */ + rcu_read_lock(); + arm64_pgtable_free(f->table[slots - 1]); + f->table[slots - 1] = NULL; + live = folio_test_pgtable(page_folio(f->pinned[0])); + read_shift = arm64_ptep_page_shift(page_address(f->pinned[0])); + rcu_read_unlock(); + KUNIT_EXPECT_TRUE(test, live); + KUNIT_EXPECT_EQ(test, read_shift, shift); + rcu_barrier(); + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[0]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[0]), 1); + for (i = slots; i < slots + 2; i++) { + struct ptdesc *pt = virt_to_ptdesc(f->table[i]); + + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), (int)slots - 1); + KUNIT_EXPECT_TRUE(test, folio_test_pgtable(ptdesc_folio(pt))); + } + mmput(f->mm); + f->mm = NULL; + for (i = 1; i < 3; i++) { + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[i]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[i]), 1); + } +} + +/* Direct table-owner test; no larger-than-native user mm is enabled here. */ +static void user64k_table_owner_test(struct kunit *test) +{ + struct fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + unsigned int i, offset, order = 16 - PAGE_SHIFT; + unsigned int shift; + bool live; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, fragment_fixture_destroy, f), 0); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + for (i = 0; i < 3; i++) { + struct ptdesc *pt; + + f->table[i] = arm64_user_pt_alloc_granule(f->mm, i, 16); + KUNIT_ASSERT_NOT_NULL(test, f->table[i]); + pt = virt_to_ptdesc(f->table[i]); + f->pinned[i] = ptdesc_page(pt); + get_page(f->pinned[i]); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)f->table[i], SZ_64K)); + KUNIT_EXPECT_EQ(test, compound_order(f->pinned[i]), order); + KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), 1); + KUNIT_EXPECT_PTR_EQ(test, f->mm->context.user4k_pt_frag[i], NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0, SZ_64K), NULL); + /* Include the last entry of every constituent native page. */ + for (offset = PAGE_SIZE - sizeof(pte_t); offset < SZ_64K; offset += PAGE_SIZE) { + void *entry = f->table[i] + offset; + + KUNIT_EXPECT_PTR_EQ(test, virt_to_ptdesc(entry), pt); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(entry), 16U); + KUNIT_EXPECT_EQ(test, pte_batch_hint(entry, __pte(TEST_PROT | PTE_CONT)), + 32U - ((offset / sizeof(pte_t)) % 32)); + if (i == USER4K_PT_PTE) + KUNIT_EXPECT_PTR_EQ(test, ptep_lockptr(f->mm, entry), + ptep_lockptr(f->mm, f->table[i])); + if (i == USER4K_PT_PMD) + KUNIT_EXPECT_PTR_EQ(test, pmd_lockptr(f->mm, entry), + pmd_lockptr(f->mm, f->table[i])); + } + memset(f->table[i], 0x51 + i, SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0x51 + i, SZ_64K), NULL); + memset(f->table[i], 0, SZ_64K); + } + /* Contiguous readers on the last native page must still use 64K geometry. */ + if (IS_ENABLED(CONFIG_ARM64_CONTPTE)) { + pte_t *ptes = f->table[0] + SZ_64K - 32 * sizeof(pte_t); + pte_t observed; + unsigned int j; + + for (j = 0; j < 32; j++) + ptes[j] = pte_mkold(pte_mkclean(__pte(TEST_PROT | PTE_CONT | + (SZ_4M + (j << 16))))); + ptes[31] = pte_mkyoung(pte_mkdirty(ptes[31])); + observed = ptep_get(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + observed = ptep_get_lockless(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + KUNIT_EXPECT_EQ(test, pte_val(observed) & + (ARM64_USER4K_TABLE_ADDR_MASK & ~(SZ_64K - 1UL)), + (u64)(SZ_4M + SZ_64K)); + memset(f->table[0], 0, SZ_64K); + } + rcu_read_lock(); + arm64_pte_free_defer(f->mm, f->table[0]); + live = folio_test_pgtable(page_folio(f->pinned[0])); + shift = arm64_ptep_page_shift(f->table[0] + SZ_64K - sizeof(pte_t)); + f->table[0] = NULL; + rcu_read_unlock(); + KUNIT_EXPECT_TRUE(test, live); + KUNIT_EXPECT_EQ(test, shift, 16U); + rcu_barrier(); + for (i = 0; i < 3; i++) { + if (f->table[i]) { + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + } + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[i]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[i]), 1); + KUNIT_EXPECT_EQ(test, compound_order(f->pinned[i]), order); + } +} + +static void user4k_fragment_lifetime_test(struct kunit *test) +{ + test_fragment_lifetime(test, 12); +} + +static void user16k_fragment_lifetime_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K fragments require larger native backing"); + return; + } + test_fragment_lifetime(test, 14); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static void test_put_mm(void *ptr) +{ + mmput(ptr); +} + +static void native_pgtable_deposit_test(struct kunit *test) +{ + struct mm_struct *mm = mm_alloc(); + pgtable_t first, second, out1, out2; + pmd_t *pmd; + spinlock_t *ptl; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_put_mm, mm), 0); + pmd = pmd_alloc_one(mm, 0); + KUNIT_ASSERT_NOT_NULL(test, pmd); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, pmd), 0); + first = pte_alloc_one(mm); + KUNIT_ASSERT_NOT_NULL(test, first); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, first), 0); + second = pte_alloc_one(mm); + KUNIT_ASSERT_NOT_NULL(test, second); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, second), 0); + ptl = pmd_lock(mm, pmd); + pgtable_trans_huge_deposit(mm, pmd, first); + pgtable_trans_huge_deposit(mm, pmd, second); + out1 = pgtable_trans_huge_withdraw(mm, pmd); + out2 = pgtable_trans_huge_withdraw(mm, pmd); + spin_unlock(ptl); + KUNIT_EXPECT_PTR_EQ(test, out1, second); + KUNIT_EXPECT_PTR_EQ(test, out2, first); +} +#endif + +static void test_data_slot_put(void *ptr) +{ + mm_subpage_put(ptr); +} + +static void test_data_pool_free(void *ptr) +{ + mm_subpage_pool_close(ptr); + mm_subpage_pool_put(ptr); +} + +static bool test_alloc_data_slots(struct kunit *test, struct test_mm *ctx) +{ + struct mm_subpage_pool *pool = mm_subpage_pool_create(GFP_KERNEL); + struct folio *folio; + unsigned int i; + int ret; + + if (!pool) + return false; + if (kunit_add_action_or_reset(test, test_data_pool_free, pool)) + return false; + folio = folio_alloc(GFP_KERNEL, 0); + if (!folio) + return false; + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + return false; + } + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc(pool); + + if (IS_ERR(slot)) + return false; + if (kunit_add_action_or_reset(test, test_data_slot_put, slot)) + return false; + ctx->phys[i] = mm_subpage_phys(slot); + } + return true; +} + +/* These private mappings are borrowed only for bounded, fault-disabled uaccess. */ +static struct test_mm *test_usercopy_mm(struct kunit *test, bool small) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + unsigned long size = small ? SZ_4K : PAGE_SIZE; + unsigned int i; + + if (!ctx) + return NULL; + ctx->test = test; + if (small && !test_alloc_data_slots(test, ctx)) + return NULL; + for (i = 0; i < 4; i++) { + if (!small) { + void *data = test_alloc_page(test); + + if (!data) + return NULL; + ctx->phys[i] = virt_to_phys(data); + } + ctx->address[i] = TEST_VA + i * size; + memset(phys_to_virt(ctx->phys[i]), (small ? 0x40 : 0x80) + i, 64); + } + ctx->mm = mm_alloc(); + if (!ctx->mm) + return NULL; + if (kunit_add_action_or_reset(test, test_mm_destroy, ctx)) + return NULL; + if (small && test_mm_select4k(ctx->mm)) + return NULL; + for (i = 0; i < 4; i++) { + unsigned long addr = ctx->address[i]; + pgd_t *pgd = pgd_offset(ctx->mm, addr); + p4d_t *p4d = p4d_alloc(ctx->mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + if (!p4d) + return NULL; + pud = pud_alloc(ctx->mm, p4d, addr); + if (!pud) + return NULL; + pmd = pmd_alloc(ctx->mm, pud, addr); + if (!pmd) + return NULL; + pte = pte_alloc_map_lock(ctx->mm, pmd, addr, &ptl); + if (!pte) + return NULL; + if (i != 2) + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[i], + __pgprot(TEST_PROT | (i ? 0 : PTE_RDONLY)))); + pte_unmap_unlock(pte, ptl); + } + return ctx; +} + +static void user4k_rmap_walk_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, true); + struct vm_area_struct *vma; + unsigned int i, j, sync; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * PAGE_SIZE; + /* A later alias must not extend a native-owner range walk's bound. */ + ctx->address[4] = TEST_VA + PAGE_SIZE; + ctx->phys[4] = ctx->phys[0]; + pte = test_mm_lookup(ctx->mm, ctx->address[4], &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[4], __pgprot(TEST_PROT))); + pte_unmap_unlock(pte, ptl); + /* Same physical slot at a different virtual byte offset and native index. */ + ctx->address[5] = TEST_VA + PAGE_SIZE + SZ_4K; + ctx->phys[5] = ctx->phys[3]; + pte = test_mm_lookup(ctx->mm, ctx->address[5], &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[5], __pgprot(TEST_PROT))); + pte_unmap_unlock(pte, ptl); + for (sync = 0; sync < 2; sync++) { + struct page_vma_mapped_walk whole = { + .pfn = ctx->phys[0] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = TEST_VA, + .flags = sync ? PVMW_SYNC : 0, + }; + unsigned int count = 0; + bool valid = true; + + /* A native-owner walk must visit every mapped 4K occupant. */ + while (page_vma_mapped_walk(&whole)) { + unsigned int index = (whole.address - TEST_VA) / SZ_4K; + + if (index >= 4 || index == 2 || !whole.pte || + pte_phys_mm(ctx->mm, ptep_get(whole.pte)) != ctx->phys[index]) + valid = false; + if (++count > 4) { + page_vma_mapped_walk_done(&whole); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 3U); + for (i = 0; i < 4; i++) { + for (j = 0; j < 6; j++) { + struct rmap_walk_range range = { + .address = ctx->address[j], + .subpage = true, + .subpage_offset = offset_in_page(ctx->phys[i]), + }; + DEFINE_FOLIO_RMAP_WALK(one, page_folio(pfn_to_page(PHYS_PFN(ctx->phys[i]))), + vma, range, sync ? PVMW_SYNC : 0); + struct mm_subpage *slot = NULL; + bool found = page_vma_mapped_walk(&one); + phys_addr_t phys = 0; + + if (found) { + phys = pte_phys_mm(ctx->mm, ptep_get(one.pte)); + slot = mm_subpage_get_from_phys(phys); + page_vma_mapped_walk_done(&one); + } + KUNIT_EXPECT_EQ(test, found, (i == j && j != 2) || + (j == 4 && i == 0) || (j == 5 && i == 3)); + if (found) { + KUNIT_EXPECT_EQ(test, phys, ctx->phys[i]); + KUNIT_EXPECT_NOT_NULL(test, slot); + if (slot) { + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), phys); + mm_subpage_put(slot); + } + } + } + } + } +} + +static int test_map_alias(struct test_mm *ctx, unsigned int index, + unsigned long addr, phys_addr_t phys) +{ + struct mm_struct *mm = ctx->mm; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d = p4d_alloc(mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + ctx->address[index] = addr; + ctx->phys[index] = phys; + if (!p4d) + return -ENOMEM; + pud = pud_alloc(mm, p4d, addr); + if (!pud) + return -ENOMEM; + pmd = pmd_alloc(mm, pud, addr); + if (!pmd) + return -ENOMEM; + pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); + if (!pte) + return -ENOMEM; + set_pte(pte, pte_mkspecial(phys_pte_mm(mm, phys, __pgprot(TEST_PROT)))); + pte_unmap_unlock(pte, ptl); + return 0; +} + +/* Physical-range lookup only; coarse anonymous ownership is a separate step. */ +static void user64k_rmap_lookup_test(struct kunit *test) +{ + struct test_mm *ctx; + struct vm_area_struct *vma; + void *data; + unsigned long pfn, per_leaf = SZ_64K / PAGE_SIZE; + unsigned int mode, sync, i; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace leaf larger than native page"); + return; + } + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + data = test_alloc_granule(test, 17); + KUNIT_ASSERT_NOT_NULL(test, data); + pfn = virt_to_phys(data) >> PAGE_SHIFT; + ctx->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, ctx->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(ctx->mm, 16), 0); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = (1UL << 29) - SZ_64K; + vma->vm_end = vma->vm_start + 4 * SZ_64K; + vma->vm_pgoff = 4096; + page_folio(virt_to_page(data))->index = vma->vm_pgoff; + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, test_map_alias(ctx, i, vma->vm_start + i * SZ_64K, + virt_to_phys(data) + (i == 1 ? SZ_64K : 0)), 0); + + for (mode = 0; mode < 2; mode++) { + if (mode) { + for (i = 0; i < 3; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(ctx->mm, ctx->address[i], &ptl); + + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, swp_entry_to_pte(make_readable_migration_entry( + ctx->phys[i] >> PAGE_SHIFT))); + pte_unmap_unlock(pte, ptl); + } + } + for (sync = 0; sync < 2; sync++) { + unsigned int flags = (sync ? PVMW_SYNC : 0) | + (mode ? PVMW_MIGRATION : 0); + + for (i = 0; i < 2 * per_leaf; i++) { + struct page_vma_mapped_walk walk = { + .pfn = pfn + i, .nr_pages = 1, + /* Deliberately invalid pgoff: single-page callers may omit it. */ + .pgoff = 0, .vma = vma, + .address = vma->vm_start + i * PAGE_SIZE, + .flags = flags, + }; + unsigned int count = 0; + bool valid = true; + + while (page_vma_mapped_walk(&walk)) { + valid &= walk.pte && walk.address == + vma->vm_start + (i / per_leaf) * SZ_64K; + if (++count > 1) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 1U); +#ifdef CONFIG_MEMORY_FAILURE + if (!mode) + KUNIT_EXPECT_EQ(test, page_mapped_in_vma(pfn_to_page(pfn + i), vma), + vma->vm_start + i * PAGE_SIZE); +#endif + } + /* Two native pages straddle a coarse-leaf and PTE-table boundary. */ + { + struct page_vma_mapped_walk walk = { + .pfn = pfn + per_leaf - 1, .nr_pages = 2, + .pgoff = vma->vm_pgoff + per_leaf - 1, .vma = vma, + .address = vma->vm_start + SZ_64K - PAGE_SIZE, + .flags = flags, + }; + unsigned int count = 0; + bool valid = true; + + KUNIT_EXPECT_EQ(test, vma_address_end(&walk), vma->vm_start + 2 * SZ_64K); + while (page_vma_mapped_walk(&walk)) { + valid &= walk.address == vma->vm_start + count * SZ_64K; + if (++count > 2) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 2U); + KUNIT_EXPECT_TRUE(test, walk.flags & PVMW_PGTABLE_CROSSED); + } + for (i = 0; i < 2; i++) { + struct page_vma_mapped_walk walk = { + .pfn = i ? pfn + 2 * per_leaf : pfn - 1, + .nr_pages = 1, .vma = vma, + .address = vma->vm_start, .flags = flags, + }; + bool found = page_vma_mapped_walk(&walk); + + if (found) + page_vma_mapped_walk_done(&walk); + KUNIT_EXPECT_FALSE(test, found); + } + } + } +} + +static void user4k_rmap_boundary_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, true); + struct vm_area_struct *vma; + unsigned int i, count = 0; + struct page_vma_mapped_walk walk; + bool valid = true; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = SZ_2M - SZ_4K; + vma->vm_end = vma->vm_start + PAGE_SIZE; + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, test_map_alias(ctx, 4 + i, vma->vm_start + i * SZ_4K, + ctx->phys[i]), 0); + walk = (struct page_vma_mapped_walk) { + .pfn = ctx->phys[0] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = vma->vm_start, + .flags = PVMW_SYNC, + }; + while (page_vma_mapped_walk(&walk)) { + if (count >= 3 || walk.address != vma->vm_start + count * SZ_4K || + pte_phys_mm(ctx->mm, ptep_get(walk.pte)) != ctx->phys[count]) + valid = false; + if (++count > 3) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 3U); + KUNIT_EXPECT_TRUE(test, walk.flags & PVMW_PGTABLE_CROSSED); +} + +static void native_rmap_walk_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, false); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 4 * PAGE_SIZE; + for (i = 0; i < 4; i++) { + struct page_vma_mapped_walk walk = { + .pfn = ctx->phys[i] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = ctx->address[i], + .flags = PVMW_SYNC, + }; + unsigned int count = 0; + bool valid = true; + + while (page_vma_mapped_walk(&walk)) { + if (walk.address != ctx->address[i] || + pte_phys_mm(ctx->mm, ptep_get(walk.pte)) != ctx->phys[i]) + valid = false; + if (++count > 1) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, i != 2 ? 1U : 0U); +#ifdef CONFIG_MEMORY_FAILURE + page_folio(pfn_to_page(ctx->phys[i] >> PAGE_SHIFT))->index = i; + KUNIT_EXPECT_EQ(test, page_mapped_in_vma(pfn_to_page(ctx->phys[i] >> PAGE_SHIFT), vma), + i != 2 ? ctx->address[i] : (unsigned long)-EFAULT); +#endif + } +} + +static void user4k_vma_offsets_test(struct kunit *test) +{ + struct vm_area_struct *vma = kunit_kzalloc(test, sizeof(*vma), GFP_KERNEL); + unsigned int slot, length, step; + + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_start = (1UL << 48) - 8 * PAGE_SIZE; + for (slot = 0; slot < TEST_SLOTS; slot++) { + for (length = 1; length <= 8; length++) { + struct vm_page_offset initial = { ULONG_MAX - 3, slot * SZ_4K }; + struct vm_page_offset pos; + + vma->vm_end = vma->vm_start + length * SZ_4K; + vma_set_page_offset(vma, initial); + KUNIT_EXPECT_EQ(test, vma_last_pgoff(vma), + initial.index + (slot + length - 1) / TEST_SLOTS); + for (step = 0; step < length; step++) { + unsigned long addr = vma->vm_start + step * SZ_4K; + + pos = vma_page_offset_at(vma, addr); + KUNIT_EXPECT_EQ(test, pos.index, initial.index + (slot + step) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, pos.offset, ((slot + step) % TEST_SLOTS) * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, linear_page_index(vma, addr), pos.index); + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), addr); + } + pos = vma_page_offset_at(vma, vma->vm_end); + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + pos = initial; + pos.index--; + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + if (slot) { + pos = initial; + pos.offset--; + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + } + } + } +} + +static void user4k_vma_intervals_test(struct kunit *test) +{ + struct vm_area_struct *vmas = kunit_kcalloc(test, 3, sizeof(*vmas), GFP_KERNEL); + struct rb_root_cached root = RB_ROOT_CACHED; + struct vm_area_struct *vma; + unsigned int i, count = 0; + + KUNIT_ASSERT_NOT_NULL(test, vmas); + for (i = 0; i < 3; i++) { + vmas[i].vm_start = TEST_VA + i * SZ_8K; + vmas[i].vm_end = vmas[i].vm_start + SZ_4K; + vma_set_page_offset(&vmas[i], (struct vm_page_offset) { 77, i * SZ_4K }); + vma_interval_tree_insert(&vmas[i], &root); + } + vma_interval_tree_foreach(vma, &root, 77, 77) { + unsigned long address = vma_address_at_offset(vma, + (struct vm_page_offset) { 77, SZ_4K }); + + KUNIT_EXPECT_EQ(test, address, + vma == &vmas[1] ? vma->vm_start : (unsigned long)-EFAULT); + count++; + } + KUNIT_EXPECT_EQ(test, count, 3U); + KUNIT_EXPECT_PTR_EQ(test, vma_interval_tree_iter_first(&root, 76, 76), NULL); + KUNIT_EXPECT_PTR_EQ(test, vma_interval_tree_iter_first(&root, 78, 78), NULL); + for (i = 0; i < 3; i++) + vma_interval_tree_remove(&vmas[i], &root); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&root.rb_root)); +} + +static void user4k_vma_mm_free(void *arg) +{ + mmput(arg); +} + +/* These private mms have no concurrent VMA writers after dropping the lock. */ +static struct vm_area_struct *test_vma_lookup(struct mm_struct *mm, unsigned long addr) +{ + struct vm_area_struct *vma; + + mmap_read_lock(mm); + vma = vma_lookup(mm, addr); + mmap_read_unlock(mm); + return vma; +} + +static void user4k_vma_split_test(struct kunit *test) +{ + struct mm_struct *mm = mm_alloc(); + struct vm_area_struct *vma, *middle, *part; + unsigned long start = TEST_VA + SZ_4K; + vma_flags_t flags; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select4k(mm), 0); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Dummy vm_ops keep insert_vm_struct from replacing this file-like offset. */ + vma->vm_start = start; + vma->vm_end = start + 3 * SZ_4K; + vm_flags_init(vma, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_offset(vma, (struct vm_page_offset) { 77, 3 * SZ_4K }); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + if (ret) { + mmap_write_unlock(mm); + vm_area_free(vma); + KUNIT_FAIL(test, "insert_vm_struct: %d", ret); + return; + } + { + VMA_ITERATOR(vmi, mm, start); + + vma_iter_load(&vmi); + flags = vma->flags; + vma_flags_set(&flags, VMA_WRITE_BIT); + middle = vma_modify_flags(&vmi, vma, vma, start + SZ_4K, + start + 2 * SZ_4K, &flags); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, middle); + KUNIT_EXPECT_EQ(test, mm->map_count, 3); + for (i = 0; i < 3; i++) { + part = test_vma_lookup(mm, start + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, part); + KUNIT_EXPECT_EQ(test, part->vm_start, start + i * SZ_4K); + KUNIT_EXPECT_EQ(test, part->vm_end, start + (i + 1) * SZ_4K); + KUNIT_EXPECT_EQ(test, part->vm_pgoff, 77UL + (i + 3) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(part), ((i + 3) % TEST_SLOTS) * (unsigned int)SZ_4K); + } +} + +static struct mm_struct *test_vma_mm_granule(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm = mm_alloc(); + + if (!mm) + return NULL; + if (kunit_add_action_or_reset(test, user4k_vma_mm_free, mm)) + return NULL; + if (test_mm_select_granule(mm, shift)) + return NULL; + return mm; +} + +static struct mm_struct *test_vma_mm(struct kunit *test) +{ + return test_vma_mm_granule(test, 12); +} + +static struct vm_area_struct *test_vma_add(struct mm_struct *mm, unsigned long start, + unsigned long len, struct vm_page_offset pos, + vm_flags_t flags) +{ + struct vm_area_struct *vma = vm_area_alloc(mm); + int ret; + + if (!vma) + return ERR_PTR(-ENOMEM); + vma->vm_start = start; + vma->vm_end = start + len; + vm_flags_init(vma, flags | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_offset(vma, pos); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + mmap_write_unlock(mm); + if (ret) { + vm_area_free(vma); + return ERR_PTR(ret); + } + return vma; +} + +static void user4k_vma_merge_test(struct kunit *test) +{ + unsigned long start = TEST_VA + SZ_4K; + unsigned int scenario, i; + + for (scenario = 0; scenario < 4; scenario++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vmas[3], *merged; + struct vm_page_offset initial = { 77, 3 * SZ_4K }; + struct vm_page_offset expected; + unsigned long left = scenario >= 2 ? SZ_4K : 0; + unsigned long right = scenario == 1 || scenario == 3 ? 2 * SZ_4K : 3 * SZ_4K; + vma_flags_t flags; + + KUNIT_ASSERT_NOT_NULL(test, mm); + for (i = 0; i < 3; i++) { + struct vm_page_offset pos = vm_page_offset_add(initial, i * SZ_4K); + vm_flags_t bits = VM_READ; + + if (i == 1) + bits |= VM_WRITE; + if ((!i && scenario >= 2) || (i == 2 && scenario == 1)) + bits |= VM_EXEC; + /* Same native index, wrong byte offset: this must not merge. */ + if (i == 2 && scenario == 3) + pos.offset = 0; + vmas[i] = test_vma_add(mm, start + i * SZ_4K, SZ_4K, pos, bits); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vmas[i]); + } + flags = vmas[1]->flags; + vma_flags_clear(&flags, VMA_WRITE_BIT); + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, start + SZ_4K); + + vma_iter_load(&vmi); + merged = vma_modify_flags(&vmi, vmas[0], vmas[1], start + SZ_4K, + start + 2 * SZ_4K, &flags); + if (!IS_ERR(merged)) { + vma_start_write(merged); + merged->flags = flags; + } + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, merged); + expected = vm_page_offset_add(initial, left); + KUNIT_EXPECT_EQ(test, mm->map_count, scenario == 0 ? 1 : scenario == 3 ? 3 : 2); + KUNIT_EXPECT_EQ(test, merged->vm_start, start + left); + KUNIT_EXPECT_EQ(test, merged->vm_end, start + right); + KUNIT_EXPECT_EQ(test, merged->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(merged), expected.offset); + } +} + +static void user4k_vma_partial_merge_test(struct kunit *test) +{ + unsigned long start = TEST_VA; + unsigned int side; + + for (side = 0; side < 2; side++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_page_offset initial = { 77, 2 * SZ_4K }; + struct vm_area_struct *first, *second, *merged, *remaining; + unsigned long first_len = side ? SZ_4K : 2 * SZ_4K; + vma_flags_t flags; + struct vm_page_offset expected; + + KUNIT_ASSERT_NOT_NULL(test, mm); + first = test_vma_add(mm, start, first_len, initial, VM_READ | (side ? 0 : VM_WRITE)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, first); + second = test_vma_add(mm, start + first_len, 3 * SZ_4K - first_len, + vm_page_offset_add(initial, first_len), VM_READ | (side ? VM_WRITE : 0)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, second); + flags = side ? first->flags : second->flags; + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, start + SZ_4K); + + vma_iter_load(&vmi); + merged = vma_modify_flags(&vmi, first, side ? second : first, + start + SZ_4K, start + 2 * SZ_4K, &flags); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, merged); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + KUNIT_EXPECT_EQ(test, merged->vm_start, start + (side ? 0 : SZ_4K)); + KUNIT_EXPECT_EQ(test, merged->vm_end, start + (side ? 2 : 3) * SZ_4K); + expected = vm_page_offset_add(initial, side ? 0 : SZ_4K); + KUNIT_EXPECT_EQ(test, merged->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(merged), expected.offset); + remaining = test_vma_lookup(mm, start + (side ? 2 * SZ_4K : 0)); + KUNIT_ASSERT_NOT_NULL(test, remaining); + expected = vm_page_offset_add(initial, side ? 2 * SZ_4K : 0); + KUNIT_EXPECT_EQ(test, remaining->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(remaining), expected.offset); + } +} + +static void user4k_vma_copy_shrink_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *source, *dest; + struct vm_page_offset initial = { 77, 3 * SZ_4K }; + unsigned long start = TEST_VA + SZ_4K; + unsigned long target = TEST_VA + 16 * PAGE_SIZE + SZ_4K; + bool need_locks = false; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + source = test_vma_add(mm, start, PAGE_SIZE, initial, VM_READ); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, source); + for (i = 0; i < 3; i++) { + unsigned long addr = i == 2 ? target - SZ_4K : target + i * SZ_4K; + struct vm_page_offset pos = vm_page_offset_add(initial, i == 2 ? 0 : (i + 1) * SZ_4K); + + mmap_write_lock(mm); + dest = copy_vma(&source, addr, SZ_4K, pos, &need_locks); + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + KUNIT_EXPECT_EQ(test, dest->vm_start, i == 2 ? target - SZ_4K : target); + KUNIT_EXPECT_EQ(test, dest->vm_end, i ? target + 2 * SZ_4K : target + SZ_4K); + KUNIT_EXPECT_EQ(test, dest->vm_pgoff, 77UL + (i == 2 ? 3UL : 4UL) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(dest), ((i == 2 ? 3U : 4U) % TEST_SLOTS) * SZ_4K); + KUNIT_EXPECT_EQ(test, source->vm_pgoff, initial.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(source), initial.offset); + } + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, dest->vm_start); + struct vm_page_offset pos = vma_page_offset_at(dest, target); + + vma_iter_load(&vmi); + ret = vma_shrink(&vmi, dest, target, dest->vm_end, pos); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, dest->vm_start, target); + KUNIT_EXPECT_EQ(test, dest->vm_end, target + 2 * SZ_4K); + KUNIT_EXPECT_EQ(test, dest->vm_pgoff, 77UL + 4 / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(dest), (4 % TEST_SLOTS) * SZ_4K); +} + +struct anon_slot_fixture { + struct mm_struct *mm; + struct vm_area_struct *vma; + struct mm_subpage_pool *pool; + struct mm_subpage *slots[4]; + bool mapped[4]; +}; + +static void anon_slot_fixture_free(void *arg) +{ + struct anon_slot_fixture *f = arg; + struct mmu_gather tlb; + unsigned int i; + + if (f->pool) + mm_subpage_pool_close(f->pool); + if (f->mm) { + mmap_write_lock(f->mm); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, addr, &ptl); + + if (!pte) + continue; + ptep_get_and_clear_full(f->mm, addr, pte, 0); + if (f->mapped[i]) { + struct vm_area_struct *vma = vma_lookup(f->mm, addr); + + mm_subpage_remove_anon_rmap(f->slots[i], vma); + dec_mm_counter(f->mm, MM_ANONPAGES); + f->mapped[i] = false; + } + pte_unmap_unlock(pte, ptl); + } + tlb_gather_mmu_fullmm(&tlb, f->mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(f->mm); + } + /* No slot reference is released before the PTEs and TLBs are retired. */ + for (i = 0; i < 4; i++) + if (f->slots[i]) + mm_subpage_put(f->slots[i]); + if (f->pool) + mm_subpage_pool_put(f->pool); + if (f->mm) + mmput(f->mm); + memset(f, 0, sizeof(*f)); +} + +static struct anon_slot_fixture *anon_slot_fixture_create_from(struct kunit *test, + struct anon_slot_fixture *parent) +{ + struct anon_slot_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct vm_area_struct *vma; + struct folio *folio; + int ret; + + if (!f || kunit_add_action_or_reset(test, anon_slot_fixture_free, f)) + return NULL; + f->mm = mm_alloc(); + if (!f->mm || test_mm_select4k(f->mm)) + return NULL; + f->pool = mm_subpage_pool_create(GFP_KERNEL); + if (!f->pool) + return NULL; + folio = folio_alloc(GFP_KERNEL, 0); + if (!folio) + return NULL; + ret = mm_subpage_pool_add_folio(f->pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + return NULL; + } + vma = vm_area_alloc(f->mm); + if (!vma) + return NULL; + vma_set_anonymous(vma); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * PAGE_SIZE; + vm_flags_init(vma, VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_prot(vma); + if (parent) { + mmap_write_lock(parent->mm); + mmap_write_lock_nested(f->mm, SINGLE_DEPTH_NESTING); + vma_set_page_offset(vma, vma_page_offset_at(parent->vma, TEST_VA)); + ret = anon_vma_fork(vma, parent->vma); + } else { + mmap_write_lock(f->mm); + ret = 0; + } + if (!ret && parent) { + VMA_ITERATOR(vmi, f->mm, TEST_VA); + + vma_start_write(vma); + ret = vma_iter_store_gfp(&vmi, vma, GFP_KERNEL); + if (!ret) + f->mm->map_count++; + } else if (!ret) { + ret = insert_vm_struct(f->mm, vma); + } + if (!ret) { + f->vma = vma; + ret = anon_vma_prepare(vma); + } + if (!f->vma) + unlink_anon_vmas(vma); + mmap_write_unlock(f->mm); + if (parent) + mmap_write_unlock(parent->mm); + if (!f->vma) + vm_area_free(vma); + return ret ? NULL : f; +} + +static struct anon_slot_fixture *anon_slot_fixture_create(struct kunit *test) +{ + return anon_slot_fixture_create_from(test, NULL); +} + +static void user4k_anon_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct folio *folio = NULL; + unsigned int i; + vm_flags_t flags; + int ret, bad_offset = 0, bad_index = 0; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + struct test_mm tables = { .mm = f->mm }; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + f->slots[i] = slot; + folio = mm_subpage_folio(slot); + /* Allocate the table first; clear the test leaf before rmap attachment. */ + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, addr, mm_subpage_phys(slot)), 0); + mmap_read_lock(f->mm); + folio_lock(folio); + pte = test_mm_lookup(f->mm, addr, &ptl); + if (!pte) { + folio_unlock(folio); + mmap_read_unlock(f->mm); + KUNIT_FAIL(test, "missing allocated PTE"); + return; + } + pte_clear(f->mm, addr, pte); + if (!i) + bad_offset = mm_subpage_add_anon_rmap(slot, f->vma, addr + 1); + ret = mm_subpage_add_anon_rmap(slot, f->vma, addr); + if (!ret) { + f->mapped[i] = true; + inc_mm_counter(f->mm, MM_ANONPAGES); + set_pte_at(f->mm, addr, pte, phys_pte_mm(f->mm, mm_subpage_phys(slot), + __pgprot(TEST_PROT | PTE_RDONLY))); + if (!i) + bad_index = mm_subpage_add_anon_rmap(slot, f->vma, addr + PAGE_SIZE); + } + pte_unmap_unlock(pte, ptl); + folio_unlock(folio); + mmap_read_unlock(f->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)i + 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)i + 1); + } + KUNIT_EXPECT_EQ(test, bad_offset, -EINVAL); + KUNIT_EXPECT_EQ(test, bad_index, -EINVAL); + KUNIT_EXPECT_EQ(test, folio->index, TEST_VA >> PAGE_SHIFT); + KUNIT_EXPECT_FALSE(test, PageAnonExclusive(&folio->page)); + KUNIT_EXPECT_EQ(test, folio_expected_ref_count(folio), folio_ref_count(folio)); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 1); + for (i = 0; i < 4; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, TEST_VA + i * SZ_4K, &ptl); + bool young; + + KUNIT_ASSERT_NOT_NULL(test, pte); + young = pte_young(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_FALSE(test, young); + } + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); +} + +/* The fixture takes one existing slot reference, including on failure. */ +static int anon_slot_fixture_map(struct anon_slot_fixture *f, + unsigned int index, struct mm_subpage *slot) +{ + unsigned long addr = TEST_VA + index * SZ_4K; + struct test_mm tables = { .mm = f->mm }; + struct folio *folio = mm_subpage_folio(slot); + spinlock_t *ptl; + pte_t *pte; + int ret; + + f->slots[index] = slot; + ret = test_map_alias(&tables, 0, addr, mm_subpage_phys(slot)); + if (ret) + return ret; + mmap_read_lock(f->mm); + folio_lock(folio); + pte = test_mm_lookup(f->mm, addr, &ptl); + if (!pte) { + ret = -ENOMEM; + goto unlock; + } + pte_clear(f->mm, addr, pte); + ret = mm_subpage_add_anon_rmap(slot, f->vma, addr); + if (!ret) { + f->mapped[index] = true; + inc_mm_counter(f->mm, MM_ANONPAGES); + set_pte_at(f->mm, addr, pte, phys_pte_mm(f->mm, mm_subpage_phys(slot), + __pgprot(TEST_PROT | PTE_RDONLY))); + } + pte_unmap_unlock(pte, ptl); +unlock: + folio_unlock(folio); + mmap_read_unlock(f->mm); + return ret; +} + +static void user4k_anon_fork_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *parent = anon_slot_fixture_create(test); + struct anon_slot_fixture *child; + struct mm_subpage *slot; + struct folio *folio; + vm_flags_t flags; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, parent); + child = anon_slot_fixture_create_from(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + for (i = 0; i < 4; i++) { + slot = mm_subpage_alloc_at(parent->pool, i * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(parent, i, slot), 0); + mm_subpage_get(slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(child, i, slot), 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 2); + } + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 8); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 8); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + anon_slot_fixture_free(parent); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 4); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(child->slots[i]), 1); + /* The child mappings and root remain valid after the parent disappears. */ + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); +} + +static void user4k_nonlinear_rmap_reuse_test(struct kunit *test) +{ + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct anon_slot_fixture *c = anon_slot_fixture_create(test); + struct mm_subpage *slot; + struct folio *folio; + struct anon_vma_chain *avc; + phys_addr_t phys; + vm_flags_t flags; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, c); + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + folio = mm_subpage_folio(slot); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + phys = mm_subpage_phys(slot); +#ifdef CONFIG_FAILSLAB + { + unsigned int pending; + + mmap_read_lock(b->mm); + folio_lock(folio); + WRITE_ONCE(current->fail_nth, 1); + ret = mm_subpage_add_anon_rmap(slot, b->vma, TEST_VA + SZ_4K); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + /* Keep cleanup valid even if allocation unexpectedly succeeds. */ + if (!ret) + mm_subpage_remove_anon_rmap(slot, b->vma); + folio_unlock(folio); + mmap_read_unlock(b->mm); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); + } +#endif + /* Allocation failure left the slot available for the same operation. */ + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + anon_slot_fixture_free(b); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), phys); + + /* Give this empty VMA a different logical native index as well. */ + mmap_write_lock(c->mm); + anon_vma_lock_write(c->vma->anon_vma); + list_for_each_entry(avc, &c->vma->anon_vma_chain, same_vma) + anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); + c->vma->vm_pgoff += 9; + list_for_each_entry(avc, &c->vma->anon_vma_chain, same_vma) + anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); + anon_vma_unlock_write(c->vma->anon_vma); + mmap_write_unlock(c->mm); +#ifdef CONFIG_FAILSLAB + /* Retiring a previous root also needs failure-atomic bookkeeping. */ + mmap_read_lock(c->mm); + folio_lock(folio); + WRITE_ONCE(current->fail_nth, 1); + ret = mm_subpage_add_anon_rmap(slot, c->vma, TEST_VA + 2 * SZ_4K); + WRITE_ONCE(current->fail_nth, 0); + if (!ret) + mm_subpage_remove_anon_rmap(slot, c->vma); + folio_unlock(folio); + mmap_read_unlock(c->mm); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); +#endif + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(c, 2, slot), 0); + mmap_read_lock(c->mm); + KUNIT_EXPECT_EQ(test, page_address_in_vma(folio, &folio->page, c->vma), + TEST_VA + 2 * SZ_4K); + mmap_read_unlock(c->mm); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + /* The original root may disappear before the other slot's last mapping. */ + anon_slot_fixture_free(a); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(c->slots[2]), 1); +} + +static void user4k_dup_mmap_test(struct kunit *test) +{ + struct anon_slot_fixture *parent = anon_slot_fixture_create(test); + struct mm_struct *child; + struct folio *folio; + unsigned int i; + int ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, parent); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(parent->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(parent, i, slot), 0); + pte = test_mm_lookup(parent->mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte_at(parent->mm, addr, pte, + pte_mkwrite_novma(pte_mkdirty(ptep_get(pte)))); + pte_unmap_unlock(pte, ptl); + } + /* Split into four 4K VMAs plus a trailing hole, preserving COW flags. */ + mmap_write_lock(parent->mm); + for (i = 1; i < 4; i += 2) { + unsigned long addr = TEST_VA + i * SZ_4K; + VMA_ITERATOR(vmi, parent->mm, addr); + struct vm_area_struct *vma = vma_iter_load(&vmi); + vma_flags_t flags = vma->flags; + + vma_flags_set(&flags, VMA_DONTDUMP_BIT); + vma = vma_modify_flags(&vmi, vma, vma, addr, addr + SZ_4K, &flags); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + break; + } + } + mmap_write_unlock(parent->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, parent->mm->map_count, 5); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, child->map_count, 5); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 4UL); + folio = mm_subpage_folio(parent->slots[0]); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 8); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 8); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + struct vm_area_struct *vma = test_vma_lookup(child, addr); + unsigned int side; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(parent->slots[i]), 2); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->mm; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, addr, &ptl); + pte_t entry; + + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_FALSE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), + mm_subpage_phys(parent->slots[i])); + } + } + anon_slot_fixture_free(parent); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 4); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + /* KUnit's child-mm cleanup now exercises ordinary exit_mmap(). */ +} + +static void test_subpage_put(void *arg) +{ + mm_subpage_put(arg); +} + +static void user4k_anon_root_lifetime_test(struct kunit *test) +{ + struct anon_slot_fixture *old = anon_slot_fixture_create(test); + struct anon_slot_fixture *other = anon_slot_fixture_create(test); + struct mm_subpage *slot; + struct anon_vma *root; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, other); + slot = mm_subpage_alloc_at(old->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(old, 0, slot), 0); + folio = mm_subpage_folio(slot); + root = old->vma->anon_vma->root; + mm_subpage_get(slot); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slot), 0); + anon_slot_fixture_free(old); + /* Only the owner now pins the root; no VMA or PTE survives. */ + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + mm_subpage_get(slot); + ret = anon_slot_fixture_map(other, 0, slot); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); +} + +static void test_folio_put(void *arg) +{ + folio_put(arg); +} + +static void user4k_data_tlb_batch_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct folio *native = folio_alloc(GFP_KERNEL, 2); + struct mmu_gather tlb; + struct folio *backing; + unsigned int i; + bool full = false; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, native), 0); + for (i = 0; i < 4; i++) { + f->slots[i] = mm_subpage_alloc_at(f->pool, i * SZ_4K); + if (IS_ERR(f->slots[i])) { + f->slots[i] = NULL; + KUNIT_FAIL(test, "slot allocation failed"); + return; + } + } + backing = mm_subpage_folio(f->slots[0]); + memset(folio_address(backing), 0x5a, SZ_4K); + tlb_gather_mmu(&tlb, f->mm); + tlb_change_page_size(&tlb, MM_SUBPAGE_SIZE); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + for (i = 0; i < 64; i++) { + mm_subpage_get(f->slots[i % 4]); + if (__tlb_remove_subpage(&tlb, f->slots[i % 4])) { + full = true; + tlb_flush_mmu(&tlb); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + } + if (i == 12) { + /* Exercise the native count payload (3 << 2 has tag bit 2). */ + folio_ref_add(native, 3); + if (__tlb_remove_folio_pages(&tlb, &native->page, 3, false)) { + full = true; + tlb_flush_mmu(&tlb); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + } + } + } + mm_subpage_put(f->slots[0]); + f->slots[0] = NULL; + /* Without memory pressure these fit in the local plus one dynamic batch. */ + if (!full) { + struct mm_subpage *extra = mm_subpage_alloc_at(f->pool, 0); + + KUNIT_EXPECT_EQ(test, PTR_ERR(extra), -EAGAIN); + if (!IS_ERR(extra)) + mm_subpage_put(extra); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), 67); + KUNIT_EXPECT_EQ(test, folio_ref_count(native), 4); + KUNIT_EXPECT_GT(test, tlb.batch_count, 0U); + } + tlb_finish_mmu(&tlb); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), 3); + KUNIT_EXPECT_EQ(test, folio_ref_count(native), 1); + f->slots[0] = mm_subpage_alloc_at(f->pool, 0); + if (IS_ERR(f->slots[0])) { + f->slots[0] = NULL; + KUNIT_FAIL(test, "slot was not reusable after TLB completion"); + return; + } + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->slots[0]), backing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(backing), 0, SZ_4K), NULL); + for (i = 0; i < 4; i++) { + struct mm_subpage *extra = mm_subpage_alloc_at(f->pool, i * SZ_4K); + + KUNIT_EXPECT_EQ(test, PTR_ERR(extra), -EAGAIN); + if (!IS_ERR(extra)) + mm_subpage_put(extra); + } +} + +static void user4k_anon_zap_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct mm_subpage *slots[4]; + struct folio *folio; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + slots[i] = mm_subpage_alloc_at(f->pool, i * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slots[i]); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slots[i]), 0); + mm_subpage_get(slots[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slots[i]), 0); + } + folio = mm_subpage_folio(slots[0]); + mmap_read_lock(f->mm); + zap_vma_range(f->vma, TEST_VA + SZ_4K, SZ_4K); + f->mapped[1] = false; + f->slots[1] = NULL; + mmap_read_unlock(f->mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 3); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 7); + for (i = 0; i < 4; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, TEST_VA + i * SZ_4K, &ptl); + bool present; + + KUNIT_ASSERT_NOT_NULL(test, pte); + present = pte_present(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, present, i != 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), i == 1 ? 0 : 1); + } + /* Include both the existing hole and a trailing run of empty entries. */ + mmap_read_lock(f->mm); + zap_vma(f->vma); + for (i = 0; i < 4; i++) { + f->mapped[i] = false; + f->slots[i] = NULL; + } + mmap_read_unlock(f->mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm, MM_ANONPAGES), 0UL); +} + +static void user4k_anon_writable_slots_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte; + bool writable; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + pte = test_mm_lookup(f->mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte_at(f->mm, addr, pte, + pte_mkwrite_novma(pte_mkdirty(ptep_get(pte)))); + writable = pte_write(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, writable); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + } + KUNIT_EXPECT_EQ(test, folio_mapcount(mm_subpage_folio(f->slots[0])), 4); +} + +static struct vm_area_struct *test_fault_vma_in_mm(struct mm_struct *mm, + unsigned long start, unsigned long len) +{ + struct vm_area_struct *vma; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + int ret; + + if (!mm) + return NULL; + vma = vm_area_alloc(mm); + if (!vma) + return NULL; + vma_set_anonymous(vma); + vma->vm_start = start; + vma->vm_end = start + len; + vm_flags_init(vma, flags); + vma->vm_page_prot = vm_get_page_prot(flags); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + mmap_write_unlock(mm); + if (ret) { + vm_area_free(vma); + return NULL; + } + return vma; +} + +struct coarse_anon_fixture { + struct mm_struct *mm[2]; + bool attached[2]; + struct vm_area_struct *vma[2]; + struct folio *folio; + unsigned long mapped[2]; + unsigned int leaves; + bool pinned; +}; + +static void coarse_anon_fixture_free(void *ptr) +{ + struct coarse_anon_fixture *f = ptr; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + + if (f->pinned) { + unpin_user_page(folio_page(f->folio, nr - 1)); + f->pinned = false; + } + for (i = 0; i < 2; i++) { + unsigned int leaf; + + if (!f->mm[i]) + continue; + mmap_write_lock(f->mm[i]); + for (leaf = 0; leaf < (f->leaves ?: 1); leaf++) { + unsigned long addr = TEST_VA + leaf * SZ_64K; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm[i], addr, &ptl); + + if (pte) { + ptep_get_and_clear_full(f->mm[i], addr, pte, 0); + if (f->mapped[i] & BIT(leaf)) { + folio_remove_rmap_pte(f->folio, + folio_page(f->folio, leaf * nr), f->vma[i]); + dec_mm_counter(f->mm[i], MM_ANONPAGES); + } + pte_unmap_unlock(pte, ptl); + } + if (f->mapped[i] & BIT(leaf)) { + folio_put_refs(f->folio, nr); + f->mapped[i] &= ~BIT(leaf); + } + } + mmap_write_unlock(f->mm[i]); + if (f->vma[i] && !f->attached[i]) { + unlink_anon_vmas(f->vma[i]); + vm_area_free(f->vma[i]); + } + mmput(f->mm[i]); + f->vma[i] = NULL; + f->mm[i] = NULL; + } + if (f->folio) { + folio_put(f->folio); + f->folio = NULL; + } +} + +static void expect_coarse_anon_state(struct kunit *test, struct folio *folio, + unsigned int maps, bool exclusive) +{ + unsigned int i, nr = folio_nr_pages(folio); + + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(maps * nr)); + for (i = 0; i < nr; i++) { + struct page *page = folio_page(folio, i); + + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, (atomic_read(&page->_mapcount) + 1), (int)maps); + KUNIT_EXPECT_EQ(test, PageAnonExclusive(page), exclusive); + } +} + +static void user64k_anon_rmap_span_test(struct kunit *test) +{ + struct coarse_anon_fixture *f; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + pte_t entry, *pte; + spinlock_t *ptl; + int ret, dup_ret, share_ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous leaf"); + return; + } + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_anon_fixture_free, f), 0); + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 16 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + for (i = 0; i < 2; i++) { + struct test_mm tables = {}; + + f->mm[i] = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm[i]); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(f->mm[i], 16), 0); + if (!i) { + f->vma[i] = test_fault_vma_in_mm(f->mm[i], TEST_VA, SZ_64K); + f->attached[i] = !!f->vma[i]; + } else { + f->vma[i] = vm_area_alloc(f->mm[i]); + if (f->vma[i]) { + vma_set_anonymous(f->vma[i]); + f->vma[i]->vm_start = TEST_VA; + f->vma[i]->vm_end = TEST_VA + SZ_64K; + vm_flags_init(f->vma[i], f->vma[0]->vm_flags); + f->vma[i]->vm_page_prot = f->vma[0]->vm_page_prot; + vma_set_page_offset(f->vma[i], vma_page_offset_at(f->vma[0], TEST_VA)); + } + } + KUNIT_ASSERT_NOT_NULL(test, f->vma[i]); + tables.mm = f->mm[i]; + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, TEST_VA, PFN_PHYS(folio_pfn(f->folio))), 0); + pte = test_mm_lookup(f->mm[i], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + pte_clear(f->mm[i], TEST_VA, pte); + pte_unmap_unlock(pte, ptl); + } + mmap_write_lock(f->mm[0]); + ret = anon_vma_prepare(f->vma[0]); + mmap_write_unlock(f->mm[0]); + KUNIT_ASSERT_EQ(test, ret, 0); + mmap_write_lock(f->mm[0]); + mmap_write_lock_nested(f->mm[1], SINGLE_DEPTH_NESTING); + ret = anon_vma_fork(f->vma[1], f->vma[0]); + if (!ret) { + VMA_ITERATOR(vmi, f->mm[1], TEST_VA); + + vma_start_write(f->vma[1]); + ret = vma_iter_store_gfp(&vmi, f->vma[1], GFP_KERNEL); + if (!ret) { + f->mm[1]->map_count++; + f->attached[1] = true; + } + } + mmap_write_unlock(f->mm[1]); + mmap_write_unlock(f->mm[0]); + KUNIT_ASSERT_EQ(test, ret, 0); + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_new_anon_rmap(f->folio, f->vma[0], TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, nr); /* Fixture retains its original allocation ref. */ + inc_mm_counter(f->mm[0], MM_ANONPAGES); + f->mapped[0] = true; + entry = pte_mkwrite(pte_mkdirty(phys_pte_mm(f->mm[0], PFN_PHYS(folio_pfn(f->folio)), + f->vma[0]->vm_page_prot)), f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + expect_coarse_anon_state(test, f->folio, 1, true); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)nr + 1); + + mm_flags_set(MMF_HAS_PINNED, f->mm[0]); + ret = try_grab_folio(page_folio(folio_page(f->folio, nr - 1)), 1, FOLL_PIN); + f->pinned = !ret; + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + ptep_set_wrprotect(f->mm[0], TEST_VA, pte); + raw_write_seqcount_begin(&f->mm[0]->write_protect_seq); + dup_ret = folio_try_dup_anon_rmap_pte(f->folio, &f->folio->page, f->vma[1], f->vma[0]); + raw_write_seqcount_end(&f->mm[0]->write_protect_seq); + entry = ptep_get_and_clear(f->mm[0], TEST_VA, pte); + share_ret = folio_try_share_anon_rmap_pte(f->folio, &f->folio->page, f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, dup_ret, -EBUSY); + KUNIT_EXPECT_EQ(test, share_ret, -EBUSY); + expect_coarse_anon_state(test, f->folio, 1, true); + unpin_user_page(folio_page(f->folio, nr - 1)); + f->pinned = false; + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get_and_clear(f->mm[0], TEST_VA, pte); + share_ret = folio_try_share_anon_rmap_pte(f->folio, &f->folio->page, f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, pte_wrprotect(entry), 1); + pte_unmap_unlock(pte, ptl); + KUNIT_ASSERT_EQ(test, share_ret, 0); + expect_coarse_anon_state(test, f->folio, 1, false); + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + raw_write_seqcount_begin(&f->mm[0]->write_protect_seq); + dup_ret = folio_try_dup_anon_rmap_pte(f->folio, &f->folio->page, f->vma[1], f->vma[0]); + raw_write_seqcount_end(&f->mm[0]->write_protect_seq); + if (!dup_ret) { + folio_ref_add(f->folio, nr); + inc_mm_counter(f->mm[1], MM_ANONPAGES); + f->mapped[1] = true; + } + pte_unmap_unlock(pte, ptl); + KUNIT_ASSERT_EQ(test, dup_ret, 0); + pte = test_mm_lookup(f->mm[1], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_ptes(f->mm[1], TEST_VA, pte, pte_wrprotect(entry), 1); + pte_unmap_unlock(pte, ptl); + expect_coarse_anon_state(test, f->folio, 2, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)(2 * nr + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[0], MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[1], MM_ANONPAGES), 1L); + + /* Exercise ordinary unmapping of each shared coarse leaf. */ + for (i = 0; i < 2; i++) { + mmap_read_lock(f->mm[i]); + zap_vma_range(f->vma[i], TEST_VA, SZ_64K); + f->mapped[i] = 0; + mmap_read_unlock(f->mm[i]); + expect_coarse_anon_state(test, f->folio, 1 - i, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)((1 - i) * nr + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[i], MM_ANONPAGES), 0L); + } + + /* Keep an observation ref while cleanup releases the empty address spaces. */ + { + struct folio *folio = f->folio; + + folio_get(folio); + coarse_anon_fixture_free(f); + expect_coarse_anon_state(test, folio, 0, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); + } +} + +static void user64k_anon_zap_batch_test(struct kunit *test) +{ + struct coarse_anon_fixture *f; + struct test_mm tables = {}; + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned int i, nr = SZ_64K / PAGE_SIZE, batch; + pte_t entry, *pte; + spinlock_t *ptl; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous leaves"); + return; + } + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_anon_fixture_free, f), 0); + f->leaves = 2; + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 17 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + mm = f->mm[0] = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + /* Partial unmap can enqueue this folio on the memcg split queue. */ + KUNIT_ASSERT_EQ(test, mem_cgroup_charge(f->folio, mm, GFP_KERNEL), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, 16), 0); + vma = f->vma[0] = test_fault_vma_in_mm(mm, TEST_VA, 2 * SZ_64K); + f->attached[0] = !!vma; + KUNIT_ASSERT_NOT_NULL(test, vma); + tables.mm = mm; + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, i, addr, + PFN_PHYS(folio_pfn(f->folio)) + i * SZ_64K), 0); + pte = test_mm_lookup(mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + pte_clear(mm, addr, pte); + pte_unmap_unlock(pte, ptl); + } + mmap_write_lock(mm); + ret = anon_vma_prepare(vma); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_new_anon_rmap(f->folio, vma, TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, 2 * nr); + add_mm_counter(mm, MM_ANONPAGES, 2); + f->mapped[0] = BIT(0) | BIT(1); + entry = pte_mkwrite(pte_mkdirty(phys_pte_mm(mm, + PFN_PHYS(folio_pfn(f->folio)), vma->vm_page_prot)), vma); + set_ptes(mm, TEST_VA, pte, entry, 2); + batch = folio_subpage_pte_batch(vma, f->folio, pte, entry, 2, 0); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, batch, 2U); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)(2 * nr)); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)(2 * nr + 1)); + + /* The reference walker must consume logical PTEs and native mapcounts. */ + { + vm_flags_t flags; + bool young; + + folio_lock(f->folio); + ret = folio_referenced(f->folio, 1, NULL, &flags); + folio_unlock(f->folio); + KUNIT_EXPECT_EQ(test, ret, 1); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + young = pte_young(ptep_get(pte)) || pte_young(ptep_get(pte + 1)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_FALSE(test, young); + folio_lock(f->folio); + ret = folio_referenced(f->folio, 1, NULL, &flags); + folio_unlock(f->folio); + KUNIT_EXPECT_EQ(test, ret, 0); + } + + /* Removing one logical leaf must leave the adjacent leaf's native span. */ + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, SZ_64K); + f->mapped[0] &= ~BIT(0); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)nr); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)nr + 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1L); + for (i = 0; i < 2 * nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(f->folio, i)->_mapcount), + i < nr ? -1 : 0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + ret = pte_none(ptep_get(pte)) && pte_present(ptep_get(pte + 1)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, ret); + + /* Restore the first leaf, then remove both through one batched walk. */ + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_anon_rmap_pte(f->folio, &f->folio->page, vma, + TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, nr); + inc_mm_counter(mm, MM_ANONPAGES); + f->mapped[0] |= BIT(0); + set_ptes(mm, TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)(2 * nr)); + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, 2 * SZ_64K); + f->mapped[0] = 0; + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + for (i = 0; i < 2 * nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(f->folio, i)->_mapcount), -1); +} + +static void user64k_file_rmap_span_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse nonanonymous leaf"); + return; + } + mm = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * SZ_64K; + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 16 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, folio); + /* Private metadata-only fixture: no file-cache promotion or fault is implied. */ + folio_lock(folio); + folio_add_file_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount) + 1, 1); + folio_dup_file_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(2 * nr)); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount) + 1, 2); + folio_remove_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + folio_remove_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount), -1); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); +} + +static struct vm_area_struct *test_fault_vma(struct kunit *test, + unsigned long start, unsigned long len) +{ + return test_fault_vma_in_mm(test_vma_mm(test), start, len); +} + +static struct vm_area_struct *test_fault_vma_granule(struct kunit *test, + unsigned int shift, unsigned long start, unsigned long len) +{ + return test_fault_vma_in_mm(test_vma_mm_granule(test, shift), start, len); +} + +static void test_anon_write_fault_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned int shape; + + /* Swap/reclaim integration is a later gate, before any ABI activation. */ + KUNIT_ASSERT_EQ(test, total_swap_pages, 0UL); + for (shape = 0; shape < 2; shape++) { + unsigned long start = TEST_VA + (shape ? size : 0); + unsigned long len = shape ? size : 2 * PAGE_SIZE; + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, start, len); + struct folio *backing = NULL; + unsigned int i, mapped = 0; + vm_fault_t ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + ret = handle_mm_fault(vma, TEST_VA + size + 3, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + /* Remove transient LRU-add references before checking ownership. */ + lru_add_drain_all(); + for (i = 0; i < (PAGE_SIZE >> shift); i++) { + unsigned long addr = TEST_VA + i * size; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(vma->vm_mm, addr, &ptl); + struct mm_subpage *slot = NULL; + bool expected = !shape || i == 1; + bool present, writable = false, zero = false, identity = false; + + KUNIT_ASSERT_NOT_NULL(test, pte); + present = pte_present(ptep_get(pte)); + if (present) { + slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, ptep_get(pte))); + writable = pte_write(ptep_get(pte)); + if (slot) { + struct folio *folio = mm_subpage_folio(slot); + + if (!backing) + backing = folio; + identity = folio == backing && + mm_subpage_offset(slot) == i * size; + zero = !memchr_inv(folio_address(folio) + mm_subpage_offset(slot), + 0, size); + mm_subpage_put(slot); + } + mapped++; + } + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, present, expected); + if (expected) { + KUNIT_EXPECT_TRUE(test, writable); + KUNIT_EXPECT_TRUE(test, identity); + KUNIT_EXPECT_TRUE(test, zero); + } + } + KUNIT_ASSERT_NOT_NULL(test, backing); + KUNIT_EXPECT_EQ(test, mapped, shape ? 1U : (PAGE_SIZE >> shift)); + KUNIT_EXPECT_EQ(test, folio_mapcount(backing), (int)mapped); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), (int)mapped); + KUNIT_EXPECT_TRUE(test, folio_test_lru(backing)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), + (unsigned long)mapped); + } +} + +static void user4k_anon_write_fault_test(struct kunit *test) +{ + test_anon_write_fault_granule(test, 12); +} + +static void user16k_anon_write_fault_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_anon_write_fault_granule(test, 14); +} + +static void user4k_swap_encoding_test(struct kunit *test) +{ + const unsigned long offsets[] = { 1, 2, (1UL << __SWP_OFFSET_BITS) - 2 }; + const unsigned int types[] = { 0, 1, __SWP_TYPE_MASK }; + unsigned int i, j, slot, flags; + + for (i = 0; i < ARRAY_SIZE(offsets); i++) { + for (j = 0; j < ARRAY_SIZE(types); j++) { + for (slot = 0; slot < PAGE_SIZE / SZ_4K; slot++) { + for (flags = 0; flags < 4; flags++) { + softleaf_t entry = swp_entry(types[j], offsets[i]); + pte_t pte = softleaf_to_pte(entry), moved; + + pte = pte_swp_set_subpage_offset(pte, slot * SZ_4K); + if (flags & 1) + pte = pte_swp_mkexclusive(pte); + if (flags & 2) + pte = pte_swp_mkuffd_wp(pte); + KUNIT_EXPECT_FALSE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), slot * (unsigned int)SZ_4K); + moved = pte_move_swp_offset(pte, 1); + KUNIT_EXPECT_EQ(test, swp_offset(softleaf_from_pte(moved)), offsets[i] + 1); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(moved), slot * (unsigned int)SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_same(pte_move_swp_offset(moved, -1), pte)); + moved = pte_swp_set_subpage_offset(pte, ((slot + 1) % (PAGE_SIZE / SZ_4K)) * SZ_4K); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(moved).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(moved), ((slot + 1) % (PAGE_SIZE / SZ_4K)) * (unsigned int)SZ_4K); + } + } + } + } +} + +static pte_t test_fault_entry(struct kunit *test, struct mm_struct *mm, unsigned long addr) +{ + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, addr, &ptl); + pte_t entry = __pte(0); + + KUNIT_EXPECT_NOT_NULL(test, pte); + if (pte) { + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + } + return entry; +} + + +static void test_file_put(void *file) +{ + fput(file); +} + +static struct file *test_cache_file(struct kunit *test, unsigned long size, bool fill) +{ + struct file *file = shmem_file_setup("user4k-file-test", size, EMPTY_VMA_FLAGS); + unsigned char *buf; + loff_t pos = 0; + unsigned int i; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, test_file_put, file)) + return ERR_PTR(-ENOMEM); + if (!fill) + return file; + buf = kunit_kmalloc(test, size, GFP_KERNEL); + if (!buf) + return ERR_PTR(-ENOMEM); + for (i = 0; i < size; i++) + buf[i] = 0x31 + (i / SZ_4K); + if (kernel_write(file, buf, size, &pos) != size) + return ERR_PTR(-EIO); + return file; +} + +static struct vm_area_struct *test_file_vma_granule(struct kunit *test, struct file *file, + unsigned int shift, bool shared, unsigned long start, unsigned long len, + unsigned long offset) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + int ret; + + mm = test_vma_mm_granule(test, shift); + if (!mm) + return NULL; + vma = vm_area_alloc(mm); + if (!vma) + return NULL; + vma->vm_start = start; + vma->vm_end = start + len; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = offset >> PAGE_SHIFT, .offset = offset & ~PAGE_MASK, + }); + if (shared) + flags |= VM_SHARED | VM_MAYSHARE; + vm_flags_init(vma, flags); + vma->vm_file = get_file(file); + vma->vm_page_prot = vm_get_page_prot(flags); + mmap_write_lock(mm); + ret = vfs_mmap(file, vma); + if (!ret) { + vma_set_page_prot(vma); + ret = insert_vm_struct(mm, vma); + } + mmap_write_unlock(mm); + if (ret) { + fput(vma->vm_file); + vm_area_free(vma); + return NULL; + } + return vma; +} + +static struct vm_area_struct *test_file_vma(struct kunit *test, struct file *file, + bool native, bool shared, unsigned long start, unsigned long len, + unsigned long offset) +{ + return test_file_vma_granule(test, file, native ? PAGE_SHIFT : 12, + shared, start, len, offset); +} + +static void user4k_mmap_descriptor_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct vm_area_desc desc; + + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + PAGE_SIZE; + vma_set_page_offset(vma, (struct vm_page_offset) { 7, SZ_4K }); + compat_set_desc_from_vma(&desc, NULL, vma); + KUNIT_EXPECT_EQ(test, desc.pgoff, 7UL); + KUNIT_EXPECT_EQ(test, desc.subpage_offset, (unsigned int)SZ_4K); + desc.pgoff = 19; + desc.subpage_offset = 3 * SZ_4K; + KUNIT_ASSERT_EQ(test, __compat_vma_mmap(&desc, vma), 0); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, TEST_VA), + (struct vm_page_offset) { 19, 3 * SZ_4K })); +} + +static void user4k_mmap_region_offsets_test(struct kunit *test) +{ + struct file *file; + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_SHARED | VM_MAYSHARE | VM_NORESERVE; + unsigned long result[4]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 8 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + /* Only the full native-sized lengths are used before accounting conversion. */ + kthread_use_mm(mm); + mmap_write_lock(mm); + result[0] = mmap_region(file, start + PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 1, SZ_4K }, NULL); + result[1] = mmap_region(file, start + 2 * PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 2, SZ_4K }, NULL); + result[2] = mmap_region(file, start, PAGE_SIZE, flags, + (struct vm_page_offset) { 0, SZ_4K }, NULL); + /* Adjacent virtual bytes with a different file offset must not merge. */ + result[3] = mmap_region(file, start + 3 * PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 3, 2 * SZ_4K }, NULL); + mmap_write_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, result[0], start + PAGE_SIZE); + KUNIT_ASSERT_EQ(test, result[1], start + 2 * PAGE_SIZE); + KUNIT_ASSERT_EQ(test, result[2], start); + KUNIT_ASSERT_EQ(test, result[3], start + 3 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_start, start); + KUNIT_EXPECT_EQ(test, vma->vm_end, start + 3 * PAGE_SIZE); + for (i = 0; i < 12; i++) + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, start + i * SZ_4K), + (struct vm_page_offset) { (i + 1) / TEST_SLOTS, ((i + 1) % TEST_SLOTS) * SZ_4K })); + vma = test_vma_lookup(mm, start + 3 * PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 2U * SZ_4K); +} + +#ifdef CONFIG_PROC_FS +/* Exercise the actual status/statm producers without publishing a fake task. */ +extern void task_mem(struct seq_file *m, struct mm_struct *mm); +extern unsigned long task_vsize(struct mm_struct *mm); +extern unsigned long task_statm(struct mm_struct *mm, unsigned long *shared, + unsigned long *text, unsigned long *data, unsigned long *resident); + +static void expect_status_kb(struct kunit *test, struct seq_file *seq, + const char *name, unsigned long expected) +{ + const char *field = strstr(seq->buf, name); + unsigned long value = ULONG_MAX; + + KUNIT_ASSERT_NOT_NULL(test, field); + KUNIT_EXPECT_EQ(test, sscanf(field + strlen(name), "%lu", &value), 1); + KUNIT_EXPECT_EQ_MSG(test, value, expected, "%s", name); +} + +static void user4k_accounting_readers_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test); + struct mm_struct *native = mm_alloc(); + struct seq_file seq = {}; + unsigned long shared, text, data, resident, size, maxrss = 0; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + seq.size = 2048; + seq.buf = kunit_kzalloc(test, seq.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, seq.buf); + /* Same logical counts, deliberately not multiples of a native page. */ + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? native : small; + unsigned long kb = i ? PAGE_SIZE / 1024 : 4; + + mm->total_vm = 11; + mm->hiwater_vm = 13; + mm->data_vm = 5; + mm->stack_vm = 2; + mm->exec_vm = 3; + mm->locked_vm = mm_pages_to_account(mm, 1); + atomic64_set(&mm->pinned_vm, 2); + mm->hiwater_rss = 10; + mm->start_code = TEST_VA + mm_page_size(mm); + mm->end_code = mm->start_code + 2 * mm_page_size(mm); + percpu_counter_set(&mm->rss_stat[MM_ANONPAGES], 5); + percpu_counter_set(&mm->rss_stat[MM_FILEPAGES], 2); + percpu_counter_set(&mm->rss_stat[MM_SHMEMPAGES], 1); + percpu_counter_set(&mm->rss_stat[MM_SWAPENTS], 3); + seq.count = 0; + task_mem(&seq, mm); + KUNIT_EXPECT_FALSE(test, seq_has_overflowed(&seq)); + seq.buf[min(seq.count, seq.size - 1)] = '\0'; + expect_status_kb(test, &seq, "VmPeak:", 13 * kb); + expect_status_kb(test, &seq, "VmSize:", 11 * kb); + expect_status_kb(test, &seq, "VmLck:", kb); + expect_status_kb(test, &seq, "VmPin:", 2 * PAGE_SIZE / 1024); + expect_status_kb(test, &seq, "VmHWM:", 10 * kb); + expect_status_kb(test, &seq, "VmRSS:", 8 * kb); + expect_status_kb(test, &seq, "RssAnon:", 5 * kb); + expect_status_kb(test, &seq, "RssFile:", 2 * kb); + expect_status_kb(test, &seq, "RssShmem:", kb); + expect_status_kb(test, &seq, "VmData:", 5 * kb); + expect_status_kb(test, &seq, "VmStk:", 2 * kb); + expect_status_kb(test, &seq, "VmExe:", 2 * kb); + expect_status_kb(test, &seq, "VmLib:", kb); + expect_status_kb(test, &seq, "VmSwap:", 3 * kb); + KUNIT_EXPECT_EQ(test, task_vsize(mm), 11 * kb * 1024); + { + struct vm_area_struct vma = { .vm_mm = mm }; + + KUNIT_EXPECT_EQ(test, vma_kernel_pagesize(&vma), kb * 1024); + KUNIT_EXPECT_EQ(test, vma_mmu_pagesize(&vma), kb * 1024); + } + } + /* Maxima must compare bytes, not incompatible 4K/16K counts. */ + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, 40UL); + setmax_mm_hiwater_rss(&maxrss, native); + KUNIT_EXPECT_EQ(test, maxrss, 10UL * PAGE_SIZE / 1024); + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, 10UL * PAGE_SIZE / 1024); + small->hiwater_rss = 10 * TEST_SLOTS + 1; + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, (10UL * TEST_SLOTS + 1) * 4); + + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 0), 0UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 1), 1UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 5), DIV_ROUND_UP(5UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(native, small, 5), 5UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(small, native, 5), DIV_ROUND_UP(5UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(small, small, 5), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(native, native, 5), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_shift(small, 5, MM_ACCOUNT_SHIFT), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_shift(native, 5, MM_ACCOUNT_SHIFT), 5UL * TEST_SLOTS); + + /* A kernel reader defaults to native units; partial pages round up. */ + size = task_statm(small, &shared, &text, &data, &resident); + KUNIT_EXPECT_EQ(test, size, DIV_ROUND_UP(11UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, shared, 1UL); + KUNIT_EXPECT_EQ(test, text, 1UL); + KUNIT_EXPECT_EQ(test, data, DIV_ROUND_UP(7UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, resident, DIV_ROUND_UP(8UL, TEST_SLOTS)); + if (user4k_test_enabled || IS_ENABLED(CONFIG_KASAN)) { + kthread_use_mm(small); + size = task_statm(native, &shared, &text, &data, &resident); + kthread_unuse_mm(small); + KUNIT_EXPECT_EQ(test, size, 11UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, shared, 3UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, text, 2UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, data, 7UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, resident, 8UL * TEST_SLOTS); + } + /* Synthetic counters have no page references; restore before mmput. */ + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? native : small; + unsigned int counter; + + for (counter = 0; counter < NR_MM_COUNTERS; counter++) + percpu_counter_set(&mm->rss_stat[counter], 0); + mm->total_vm = mm->data_vm = mm->stack_vm = mm->exec_vm = 0; + mm->locked_vm = 0; + atomic64_set(&mm->pinned_vm, 0); + } +} +#endif + +static void mixed_locked_account_units_test(struct kunit *test) +{ + unsigned int shift; + unsigned long native_units = PAGE_SIZE >> MM_ACCOUNT_SHIFT; + + for (shift = 12; shift <= 16; shift += 2) { + struct mm_struct *mm = mm_alloc(); + unsigned long size = 1UL << shift; + unsigned long base, limit; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + if (shift != PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shift), 0); + base = mm_pages_to_account(mm, 1); + mmap_write_lock(mm); + /* Model one locked VMA leaf and two independently charged native pins. */ + mm->locked_vm = base; + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base + native_units); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size + PAGE_SIZE); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(mm), + DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE)); + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base + 2 * native_units); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size + PAGE_SIZE); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size); + + /* A refused charge must leave the exact pre-existing amount intact. */ + limit = task_rlimit(current, RLIMIT_MEMLOCK) >> MM_ACCOUNT_SHIFT; + mm->locked_vm = limit; + ret = __account_locked_vm(mm, 1, true, current, false); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, limit); + mm->locked_vm = ULONG_MAX - native_units + 1; + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, ULONG_MAX - native_units + 1); + mm->locked_vm = 0; + if (native_units > 1) { + ret = __account_locked_vm(mm, + (ULONG_MAX >> (PAGE_SHIFT - MM_ACCOUNT_SHIFT)) + 1, + true, current, true); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + } + mmap_write_unlock(mm); + } +} + +/* Prior shmem fixtures can refund global commit from deferred fput work. */ +static s64 test_commit_baseline(struct kunit *test) +{ + s64 before = percpu_counter_sum(&vm_committed_as), after; + + /* No mmap, file, or filesystem locks are held at these call sites. */ + flush_delayed_fput(); + after = percpu_counter_sum(&vm_committed_as); + if (after != before) + kunit_info(test, "drained deferred file refunds: commit delta %lld", after - before); + return after; +} + +static void user64k_commit_accounting_test(struct kunit *test) +{ + struct mm_struct *mm; + s64 before; + unsigned int i; + unsigned long native_per_leaf; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace granule larger than native"); + return; + } + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, 16), 0); + native_per_leaf = SZ_64K / PAGE_SIZE; + before = test_commit_baseline(test); + mmap_write_lock(mm); + for (i = 0; i < 9; i++) { + ret = mm_account_memory(mm, mm, 1); + KUNIT_EXPECT_EQ(test, ret, 0); + if (ret) + break; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i + 1); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)((i + 1) * native_per_leaf)); + } + /* Multiplication overflow must not mutate either reservation counter. */ + ret = mm_account_memory(mm, mm, LONG_MAX / native_per_leaf + 1); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + mm_acct_memory(mm, 3); + i += 3; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + while (i) { + mm_unacct_memory(mm, 1); + i--; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + } + mmap_write_unlock(mm); +} + +#ifdef CONFIG_PAGE_TABLE_CHECK +static void expect_coarse_file_counts(struct kunit *test, void *data, + unsigned int phase) +{ + unsigned int offset; + + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_4K) { + int anon = -1, file = -1, expected; + int ret = page_table_check_get_counts(virt_to_page(data + offset), + offset & ~PAGE_MASK, &anon, &file); + + /* Phase0: both coarse leaves;1:plusalias;2:firstclear;3:bothclear;4:allclear. */ + expected = phase < 2 || (phase == 2 && offset >= SZ_64K); + if (phase >= 1 && phase <= 3 && offset == 2 * SZ_64K - SZ_4K) + expected++; + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, anon, 0); + KUNIT_EXPECT_EQ_MSG(test, file, expected, "phase=%u offset=%u", phase, offset); + } +} +#endif + +static void user64k_page_table_check_test(struct kunit *test) +{ +#ifdef CONFIG_PAGE_TABLE_CHECK + struct mm_struct *coarse, *small; + void *data; + pte_t ptes[2] = {}, alias = __pte(0), first, last; + int anon, file; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace leaf larger than native page"); + return; + } + data = test_alloc_granule(test, 17); + KUNIT_ASSERT_NOT_NULL(test, data); + if (page_table_check_get_counts(virt_to_page(data), 0, &anon, &file)) { + kunit_skip(test, "requires page-table-check enabled at boot"); + return; + } + coarse = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, coarse), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(coarse, 16), 0); + small = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, small), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(small, 12), 0); + first = phys_pte_mm(coarse, virt_to_phys(data), __pgprot(TEST_PROT)); + last = phys_pte_mm(small, virt_to_phys(data) + 2 * SZ_64K - SZ_4K, + __pgprot(TEST_PROT)); + + /* Exercise checker hooks directly; no actual coarse file fault is claimed. */ + page_table_check_ptes_set(coarse, TEST_VA, ptes, first, 2); + ptes[0] = first; + ptes[1] = pte_advance_pfn_mm(coarse, first, 1); + expect_coarse_file_counts(test, data, 0); + page_table_check_ptes_set(small, TEST_VA, &alias, last, 1); + alias = last; + expect_coarse_file_counts(test, data, 1); + page_table_check_pte_clear(coarse, TEST_VA, ptes[0]); + ptes[0] = __pte(0); + expect_coarse_file_counts(test, data, 2); + page_table_check_pte_clear(coarse, TEST_VA + SZ_64K, ptes[1]); + ptes[1] = __pte(0); + expect_coarse_file_counts(test, data, 3); + page_table_check_pte_clear(small, TEST_VA, alias); + expect_coarse_file_counts(test, data, 4); +#else + kunit_skip(test, "requires CONFIG_PAGE_TABLE_CHECK"); +#endif +} + +static void user4k_commit_accounting_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + s64 before; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + mmap_write_lock(mm); + for (i = 0; i < 9; i++) { + ret = mm_account_memory(mm, mm, 1); + KUNIT_EXPECT_EQ(test, ret, 0); + if (ret) + break; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i + 1); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)DIV_ROUND_UP(i + 1, TEST_SLOTS)); + } + if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) { + unsigned long old = mm->committed_user_pages; + s64 committed = percpu_counter_sum(&vm_committed_as); + + ret = mm_account_memory(mm, mm, (totalram_pages() + total_swap_pages + 1) * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, old); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as), committed); + } + /* Native physical-page callers must charge four user granules per page. */ + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, (unsigned long)TEST_SLOTS); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + + /* Refund one granule at a time: splitting must not multiply refunds. */ + while (i) { + mm_unacct_memory(mm, 1); + i--; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)DIV_ROUND_UP(i, TEST_SLOTS)); + } + mmap_write_unlock(mm); +} + +/* Prefaulting must stop at each process-page boundary, not native pages. */ +static void user_granule_prefault_test(struct kunit *test) +{ + unsigned int shift; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + for (shift = 12; shift <= PAGE_SHIFT; shift += 2) { + struct mm_struct *mm = test_vma_mm_granule(test, shift); + unsigned long size = 1UL << shift, start = TEST_VA; + unsigned long flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long mapped[3]; + size_t left[3][4] = {}; + int phase, ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped[0] = vm_mmap(NULL, start, size, PROT_READ | PROT_WRITE, flags, 0); + mapped[2] = vm_mmap(NULL, start + 2 * size, size, + PROT_READ | PROT_WRITE, flags, 0); + mapped[1] = start + size; + if (mapped[0] == start && mapped[2] == start + 2 * size) { + for (phase = 0; phase < 3; phase++) { + char __user *addr = (char __user *)(start + 17); + size_t len = 3 * size - 17; + + /* Missing middle, then read-only, then fully writable. */ + if (phase) { + mapped[1] = vm_mmap(NULL, start + size, size, + PROT_READ | (phase == 2 ? PROT_WRITE : 0), + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + if (mapped[1] != start + size) + break; + } + left[phase][0] = fault_in_safe_writeable(addr, len); + left[phase][1] = fault_in_readable(addr, len); + left[phase][2] = fault_in_writeable(addr, len); + left[phase][3] = fault_in_subpage_writeable(addr, len); + } + ret = vm_munmap(start, 3 * size); + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped[0], start); + KUNIT_ASSERT_EQ(test, mapped[1], start + size); + KUNIT_ASSERT_EQ(test, mapped[2], start + 2 * size); + KUNIT_EXPECT_EQ(test, ret, 0); + for (phase = 0; phase < 3; phase++) { + KUNIT_EXPECT_EQ(test, left[phase][0], phase < 2 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][1], phase == 0 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][2], phase < 2 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][3], phase < 2 ? 2 * size : 0UL); + } + } +} + +static void user4k_mmap_accounting_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K, mapped; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + s64 before; + unsigned int i; + int ret = 0; + vm_fault_t fault = 0; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + kthread_use_mm(mm); + mmap_write_lock(mm); + mapped = mmap_region(NULL, start, 3 * SZ_4K, flags, + (struct vm_page_offset) { start >> PAGE_SHIFT, SZ_4K }, NULL); + mmap_write_unlock(mm); + if (!IS_ERR_VALUE(mapped)) { + mmap_read_lock(mm); + vma = vma_lookup(mm, start); + fault = handle_mm_fault(vma, start, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->data_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, 1LL); + /* Remove the middle first, then the two neighbors. */ + for (i = 0; i < 3; i++) { + unsigned int slot = i == 0 ? 1 : i == 1 ? 0 : 2; + VMA_ITERATOR(vmi, mm, start + slot * SZ_4K); + + kthread_use_mm(mm); + mmap_write_lock(mm); + ret = do_vmi_munmap(&vmi, mm, start + slot * SZ_4K, SZ_4K, NULL, false); + mmap_write_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 2UL - i); + KUNIT_EXPECT_EQ(test, mm->data_vm, 2UL - i); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 2UL - i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, i == 2 ? 0LL : 1LL); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); +} + +static vm_fault_t test_file_fault(struct vm_area_struct *vma, unsigned long addr, bool write) +{ + vm_fault_t ret; + + mmap_read_lock(vma->vm_mm); + ret = handle_mm_fault(vma, addr, FAULT_FLAG_REMOTE | + (write ? FAULT_FLAG_WRITE : 0), NULL); + mmap_read_unlock(vma->vm_mm); + return ret; +} + +static void user4k_vm_mmap_test(struct kunit *test) +{ + struct mm_struct *mm; + struct file *file; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + unsigned long anon_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long file_flags = MAP_SHARED | MAP_FIXED_NOREPLACE; + unsigned long result[7]; + int unmap[4]; + pte_t pte; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + mm->mmap_base = TEST_VA + 2 * PAGE_SIZE + SZ_4K; + mm_flags_clear(MMF_TOPDOWN, mm); + kthread_use_mm(mm); + result[0] = vm_mmap(NULL, start, SZ_4K, PROT_READ | PROT_WRITE, anon_flags, 0); + result[1] = vm_mmap(file, start + 2 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE, + file_flags, 3 * SZ_4K); + result[2] = vm_mmap(file, start + 3 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE, + file_flags, 4 * SZ_4K); + result[3] = vm_mmap(NULL, start, SZ_4K, PROT_READ, anon_flags, 0); + result[4] = vm_mmap(NULL, start + 17, SZ_4K, PROT_READ, anon_flags, 0); + result[5] = vm_mmap(file, start + SZ_4K, SZ_4K, PROT_READ, file_flags, SZ_4K + 1); + result[6] = vm_mmap(NULL, 0, SZ_4K + 1, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, result[0], start); + KUNIT_ASSERT_EQ(test, result[1], start + 2 * SZ_4K); + KUNIT_ASSERT_EQ(test, result[2], start + 3 * SZ_4K); + KUNIT_EXPECT_EQ(test, result[3], (unsigned long)-EEXIST); + KUNIT_EXPECT_EQ(test, result[4], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, result[5], (unsigned long)-EINVAL); + KUNIT_ASSERT_EQ(test, result[6], mm->mmap_base); + KUNIT_EXPECT_EQ(test, mm->total_vm, 5UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start, true) & VM_FAULT_ERROR, 0U); + vma = test_vma_lookup(mm, start + 2 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_end - vma->vm_start, 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 3U * SZ_4K); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + 2 * SZ_4K, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, mm, start + 2 * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(mm, pte)), (u8)0x34); + pte = test_fault_entry(test, mm, start + 3 * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(mm, pte)), (u8)0x35); + kthread_use_mm(mm); + unmap[0] = vm_munmap(start + 2 * SZ_4K, 1); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, unmap[0], 0); + vma = test_vma_lookup(mm, start + 3 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_pgoff, 4UL / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), (4U % TEST_SLOTS) * SZ_4K); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + kthread_use_mm(mm); + unmap[1] = vm_munmap(start, SZ_4K); + unmap[2] = vm_munmap(start + 3 * SZ_4K, SZ_4K); + unmap[3] = vm_munmap(result[6], 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, unmap[1], 0); + KUNIT_EXPECT_EQ(test, unmap[2], 0); + KUNIT_EXPECT_EQ(test, unmap[3], 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 0UL); + KUNIT_EXPECT_EQ(test, mm->map_count, 0); +} + +static void test_mlock_drain_all(struct kunit *test, struct folio *folio) +{ + unsigned int before, after; + bool counted; + + lru_add_drain(); + counted = folio_test_unevictable(folio); + before = counted ? folio->mlock_count : 0; + /* A sleeping fixture can enqueue mlock work on more than one CPU. */ + lru_add_drain_all(); + after = folio_test_unevictable(folio) ? folio->mlock_count : 0; + if (counted && before != after) + kunit_info(test, "remote mlock drain changed count %u -> %u", before, after); +} + +static void user4k_mlock_counts_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct mm_subpage *slot; + struct folio *folio; + unsigned long addr; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE | MAP_LOCKED, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, test_fault_entry(test, mm, TEST_VA))); + KUNIT_ASSERT_NOT_NULL(test, slot); + folio = mm_subpage_folio(slot); + folio_get(folio); + mm_subpage_put(slot); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm->locked_vm, (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + int ret; + + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA + i * SZ_4K, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + /* mlock_count shares storage with the evictable LRU links. */ + if (i != TEST_SLOTS - 1) + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS - 1 - i); + KUNIT_EXPECT_EQ(test, mm->locked_vm, TEST_SLOTS - 1UL - i); + KUNIT_EXPECT_EQ(test, folio_test_mlocked(folio), i != TEST_SLOTS - 1); + } +} + +static void test_mlock_enable_lru_cache(void *unused) +{ + lru_cache_enable(); +} + +static void user4k_mlock_shared_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *folio; + unsigned long addr; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + if (native) + mmput(native); + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + /* Exact counts require synchronous LRU/lock accounting. Normal deferred + * batches may legitimately miss locks while the folio is off LRU; + * user4k_mlock_repair_test checks that reclaim repairs that case. + */ + lru_cache_disable(); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_mlock_enable_lru_cache, NULL), 0); + kthread_use_mm(small); + addr = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + kthread_use_mm(native); + addr = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(native); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS + 1); + kthread_use_mm(small); + ret = vm_munmap(TEST_VA + SZ_4K, SZ_4K); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS); + kthread_use_mm(native); + ret = vm_munmap(TEST_VA, PAGE_SIZE); + kthread_unuse_mm(native); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS - 1); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + kthread_use_mm(small); + ret = vm_munmap(TEST_VA, PAGE_SIZE); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); +} + +static void test_mlock_putback(void *folio) +{ + folio_putback_lru(folio); +} + +static void user4k_mlock_repair_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode, walker; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + unsigned long size = 1UL << shifts[mode]; + + for (walker = 0; walker < 2; walker++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *folio; + unsigned long addr, offset; + vm_flags_t flags = 0; + int maps, ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_folio_put, folio), 0); + lru_add_drain_all(); + KUNIT_ASSERT_TRUE(test, folio_isolate_lru(folio)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_mlock_putback, folio), 0); + /* Two aliases leave a locked mapping even after native unmap. + * Force deferred mlocks to run while the folio is isolated. + */ + for (offset = 0; offset < 2 * PAGE_SIZE; offset += PAGE_SIZE) { + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA + offset, PAGE_SIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + offset); + } + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_lru(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + kunit_release_action(test, test_mlock_putback, folio); + lru_add_drain_all(); + KUNIT_ASSERT_TRUE(test, folio_test_unevictable(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, 0U); + + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); + maps = folio_mapcount(folio); + KUNIT_EXPECT_EQ(test, maps, (int)(2 * PAGE_SIZE / size) - 1); + folio_lock(folio); + if (walker) + ret = folio_referenced(folio, 1, NULL, &flags); + else + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + lru_add_drain_all(); + if (walker) { + KUNIT_EXPECT_GE(test, ret, 0); + KUNIT_EXPECT_TRUE(test, flags & VM_LOCKED); + } + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), maps); + for (offset = size; offset < 2 * PAGE_SIZE; offset += size) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + offset); + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_PTR_EQ(test, + page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(mm, pte)))), + folio); + } + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, 2 * PAGE_SIZE); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + kunit_info(test, "%uK %s repairs deferred mlock without unmapping", + 1U << (shifts[mode] - 10), walker ? "referenced" : "unmap"); + } + } +} + +asmlinkage long __arm64_sys_mremap(const struct pt_regs *regs); + +static unsigned long test_mremap(unsigned long old, unsigned long old_len, + unsigned long new_len, unsigned long flags, unsigned long new) +{ + struct pt_regs regs = { .regs = { old, old_len, new_len, flags, new } }; + + return __arm64_sys_mremap(®s); +} + +static int test_iova_bitmap_fill(struct iova_bitmap *bitmap, unsigned long iova, + size_t length, void *opaque) +{ + unsigned long unit = *(unsigned long *)opaque; + + iova_bitmap_set(bitmap, iova + 3 * unit, length - 10 * unit); + /* A second write advances backwards into an already populated range. */ + iova_bitmap_set(bitmap, iova, 5 * unit); + return 0; +} + +/* Real IOMMUFD lifecycle with a translation-recording IOMMU, no physical DMA. */ +struct test_fragment_domain { + struct iommu_domain domain; + struct xarray translations; + unsigned long fail_after; + unsigned long largest_map; + unsigned long mapped; + unsigned long map_calls; +}; + +struct test_fragment_iopt { + struct io_pagetable table; + struct test_fragment_domain domains[2]; + struct kunit *test; +}; + +static int test_fragment_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t phys, size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + unsigned long length = pgsize * pgcount, done; + int rc; + + mock->map_calls++; + mock->largest_map = max(mock->largest_map, length); + for (done = 0; done < length; done += SZ_4K) { + if (mock->fail_after && mock->mapped >= mock->fail_after) + return -ENOMEM; + rc = xa_insert(&mock->translations, (iova + done) >> 12, + xa_mk_value((phys + done) >> 12), gfp); + if (rc) + return rc; + *mapped += SZ_4K; + mock->mapped += SZ_4K; + } + return 0; +} + +static size_t test_fragment_unmap(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + unsigned long length = pgsize * pgcount, done; + + for (done = 0; done < length; done += SZ_4K) { + if (!xa_erase(&mock->translations, (iova + done) >> 12)) + break; + mock->mapped -= SZ_4K; + } + return done; +} + +static phys_addr_t test_fragment_translate(struct iommu_domain *domain, dma_addr_t iova) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + void *entry = xa_load(&mock->translations, iova >> 12); + + return entry ? ((phys_addr_t)xa_to_value(entry) << 12) + (iova & (SZ_4K - 1)) : 0; +} + +static void test_fragment_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) +{ +} + +static const struct iommu_domain_ops test_fragment_domain_ops = { + .map_pages = test_fragment_map, + .unmap_pages = test_fragment_unmap, + .iova_to_phys = test_fragment_translate, + .iotlb_sync = test_fragment_sync, +}; + +static void test_fragment_iopt_free(void *data) +{ + struct test_fragment_iopt *fixture = data; + struct iommu_domain *domain; + unsigned long unmapped, index; + unsigned int i; + + iopt_unmap_all(&fixture->table, &unmapped); + while ((domain = xa_find(&fixture->table.domains, &(unsigned long){0}, + ULONG_MAX, XA_PRESENT))) + iopt_table_remove_domain(&fixture->table, domain); + iopt_destroy_table(&fixture->table); + for (i = 0; i < ARRAY_SIZE(fixture->domains); i++) { + struct test_fragment_domain *mock = &fixture->domains[i]; + void *entry; + + KUNIT_EXPECT_TRUE(fixture->test, xa_empty(&mock->translations)); + xa_for_each(&mock->translations, index, entry) + xa_erase(&mock->translations, index); + xa_destroy(&mock->translations); + } +} + +static struct test_fragment_iopt *test_fragment_iopt_alloc(struct kunit *test) +{ + struct test_fragment_iopt *fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + unsigned int i; + + if (!fixture) + return NULL; + fixture->test = test; + iopt_init_table(&fixture->table); + for (i = 0; i < ARRAY_SIZE(fixture->domains); i++) { + struct test_fragment_domain *mock = &fixture->domains[i]; + + xa_init(&mock->translations); + mock->domain.ops = &test_fragment_domain_ops; + mock->domain.type = IOMMU_DOMAIN_UNMANAGED; + mock->domain.pgsize_bitmap = SZ_4K; + mock->domain.geometry.aperture_end = ULONG_MAX; + } + if (kunit_add_action_or_reset(test, test_fragment_iopt_free, fixture)) + return NULL; + return fixture; +} + +static int test_fragment_map_user(struct test_fragment_iopt *fixture, + struct mm_struct *mm, unsigned long va, unsigned long iova, unsigned long length) +{ + struct iommufd_ctx ctx = {}; + int rc; + + kthread_use_mm(mm); + rc = iopt_map_user_pages(&ctx, &fixture->table, &iova, (void __user *)va, + length, IOMMU_READ | IOMMU_WRITE, 0); + kthread_unuse_mm(mm); + return rc; +} + +static void user4k_iommufd_lifetime_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14, 16 }, mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + for (mode = 0; mode < ARRAY_SIZE(shifts); mode++) { + unsigned long size = 1UL << shifts[mode]; + unsigned long len = 2 * max(size, PAGE_SIZE); + unsigned long dest = TEST_VA + 32 * PAGE_SIZE + size; + unsigned long iova = SZ_1M, alias = SZ_16M, addr, moved, missed, unmapped; + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct iopt_pages *pages; + struct iopt_area *area; + unsigned char *buffer; + unsigned long off, pinned; + LIST_HEAD(copy); + int rc; + + if (mode && shifts[mode] == PAGE_SHIFT) + continue; + mm = test_vma_mm_granule(test, shifts[mode]); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + buffer = kunit_kmalloc(test, len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + memset(buffer, 0x31, len); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, buffer, len); + moved = missed ? 0 : test_mremap(addr, len, len, MREMAP_FIXED | MREMAP_MAYMOVE, dest); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, dest); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, dest, iova, len), 0); + pinned = atomic64_read(&mm->pinned_vm); + KUNIT_EXPECT_EQ(test, pinned, len / PAGE_SIZE); + KUNIT_EXPECT_GE(test, fixture->domains[0].largest_map, PAGE_SIZE); + for (off = 0; off < len; off += size) { + pte_t pte = test_fault_entry(test, mm, dest + off); + phys_addr_t actual = iommu_iova_to_phys(&fixture->domains[0].domain, iova + off); + + KUNIT_EXPECT_EQ(test, actual, pte_phys_mm(mm, pte)); + } + KUNIT_ASSERT_EQ(test, iopt_get_pages(&fixture->table, iova + size, len - size, ©), 0); + rc = iopt_map_pages(&fixture->table, ©, len - size, &alias, IOMMU_READ | IOMMU_WRITE, 0); + iopt_free_pages_list(©); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), pinned); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), pinned); + + /* Replace the original VA: mappings and copied IOAS must retain old pins. */ + kthread_use_mm(mm); + addr = vm_mmap(NULL, dest, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + memset(buffer, 0x5a, len); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)dest, buffer, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, missed, 0UL); + down_read(&fixture->table.iova_rwsem); + area = iopt_area_iter_first(&fixture->table, iova, iova); + pages = area->pages; + /* Force the domain-backed reader for the native control too. */ + memset(buffer, 0x77, len); + rc = iopt_pages_rw_access(pages, 0, buffer, len, IOMMUFD_ACCESS_RW_WRITE); + up_read(&fixture->table.iova_rwsem); + KUNIT_ASSERT_EQ(test, rc, 0); + for (off = 0; off < len; off += SZ_4K) { + phys_addr_t phys = iommu_iova_to_phys(&fixture->domains[1].domain, iova + off); + void *ptr = kmap_local_page(pfn_to_page(PHYS_PFN(phys))); + bool correct = !memchr_inv(ptr + offset_in_page(phys), 0x77, SZ_4K); + + kunmap_local(ptr); + KUNIT_EXPECT_TRUE(test, correct); + } + kthread_use_mm(mm); + missed = copy_from_user(buffer, (void __user *)dest, len); + /* Local CPU access retains the existing copy_from_user fast path. */ + rc = iopt_pages_rw_access(pages, 0, buffer, len, 0); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0x5a, len), NULL); + KUNIT_ASSERT_EQ(test, iopt_unmap_iova(&fixture->table, iova, len, &unmapped), 0); + KUNIT_EXPECT_EQ(test, unmapped, len); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), (len - (size < PAGE_SIZE ? 0 : size)) / PAGE_SIZE); + iopt_table_remove_domain(&fixture->table, &fixture->domains[0].domain); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, len - size); + KUNIT_ASSERT_EQ(test, iopt_unmap_iova(&fixture->table, alias, len - size, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOMMUFD %luK: relocated PA, batching, copy, two domains, VA replacement and unpin passed", size / SZ_1K); + } +} + +static void user4k_iommufd_rollback_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + unsigned long addr, iova = SZ_1M, size = SZ_4K; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 3 * size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + rc = vm_munmap(TEST_VA + size, size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, addr, iova, 3 * size), -EFAULT); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA + size, size, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + size); + KUNIT_EXPECT_LT(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA + size, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + size); + fixture->domains[1].fail_after = size; + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), -ENOMEM); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + fixture->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), 0); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 3 * size); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 3 * size); + kunit_release_action(test, test_fragment_iopt_free, fixture); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void user4k_iommufd_granule_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + unsigned long addr, unmapped, cut = SZ_1M + PAGE_SIZE - 1; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + fixture->domains[0].domain.pgsize_bitmap = PAGE_SIZE; + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 3 * PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + /* The device cannot cover a misaligned fragment by widening the mapping. */ + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA + SZ_4K, SZ_1M, PAGE_SIZE), -EINVAL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, SZ_4K), -EINVAL); + /* Full contiguous native spans still work, including legacy area splitting. */ + KUNIT_ASSERT_EQ(test, iopt_disable_large_pages(&fixture->table), 0); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, 2 * PAGE_SIZE), 0); + KUNIT_EXPECT_EQ(test, fixture->domains[0].largest_map, PAGE_SIZE); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 2LL); + KUNIT_ASSERT_EQ(test, iopt_cut_iova(&fixture->table, &cut, 1), 0); + rc = iopt_unmap_iova(&fixture->table, SZ_1M, PAGE_SIZE, &unmapped); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + /* A closed IOVA interval ending at ULONG_MAX must not wrap at map/unmap. */ + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, + ULONG_MAX - 2 * PAGE_SIZE + 1, 2 * PAGE_SIZE), 0); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, unmapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void user4k_iommufd_shared_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct file *file; + struct folio *folio; + unsigned long addr, unmapped, iova = SZ_1M; + unsigned char *buffer; + loff_t pos = 0; + phys_addr_t phys; + void *ptr; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + file = test_cache_file(test, 2 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + buffer = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA, 2 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 2 * SZ_4K), 0); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + folio_lock(folio); + folio_clear_dirty_for_io(folio); + folio_unlock(folio); + phys = iommu_iova_to_phys(&fixture->domains[0].domain, iova); + KUNIT_EXPECT_EQ(test, phys, PFN_PHYS(folio_pfn(folio)) + SZ_4K); + ptr = kmap_local_folio(folio, SZ_4K); + memset(ptr, 0x93, 2 * SZ_4K); + kunmap_local(ptr); + kthread_use_mm(mm); + rc = vm_munmap(TEST_VA, 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, rc, 0); + /* A failed second-domain attachment must retain the first domain's pins. */ + fixture->domains[1].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), -ENOMEM); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 2 * SZ_4K); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(folio)); + fixture->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_ASSERT_EQ(test, kernel_read(file, buffer, PAGE_SIZE, &pos), (ssize_t)PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + SZ_4K, 0x93, 2 * SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 3 * SZ_4K, 0, PAGE_SIZE - 3 * SZ_4K), NULL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void test_iommufd_exit_account(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct iopt_pages *pages; + struct iopt_area *area; + unsigned long addr, unmapped; + LIST_HEAD(keep); + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 2 * PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, 2 * PAGE_SIZE), 0); + down_read(&fixture->table.iova_rwsem); + area = iopt_area_iter_first(&fixture->table, SZ_1M, SZ_1M); + pages = area->pages; + mutex_lock(&pages->mutex); + /* Select VFIO-compatible accounting before the lazy range acquires pins. */ + pages->account_mode = IOPT_PAGES_ACCOUNT_MM; + mutex_unlock(&pages->mutex); + up_read(&fixture->table.iova_rwsem); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 2LL); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 2 * (PAGE_SIZE >> MM_ACCOUNT_SHIFT)); + /* Keep the provider, and thus mm_count, after mm_users reaches zero. */ + KUNIT_ASSERT_EQ(test, iopt_get_pages(&fixture->table, SZ_1M, 2 * PAGE_SIZE, &keep), 0); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, atomic_read(&mm->mm_users), 0); + KUNIT_EXPECT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_EXPECT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, pages->npinned, 0UL); + KUNIT_EXPECT_EQ(test, pages->last_npinned, 0UL); + iopt_free_pages_list(&keep); + kunit_release_action(test, test_fragment_iopt_free, fixture); +} + +static void user4k_iommufd_exit_account_test(struct kunit *test) +{ + test_iommufd_exit_account(test, PAGE_SHIFT); + test_iommufd_exit_account(test, 12); +} + +static void user4k_iova_bitmap_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 4 * size; + unsigned long old = TEST_VA, dest = TEST_VA + 32 * PAGE_SIZE + size; + unsigned long start = dest + size - sizeof(u64), bytes = 2 * size; + unsigned long unit = SZ_4K, iova = 1UL << 32, addr, moved; + unsigned char *actual, *expected; + struct iova_bitmap *bitmap; + unsigned long missed; + int ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + actual = kunit_kzalloc(test, len, GFP_KERNEL); + expected = kunit_kzalloc(test, len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, actual); + KUNIT_ASSERT_NOT_NULL(test, expected); + kthread_use_mm(mm); + addr = vm_mmap(NULL, old, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, actual, len); + moved = missed ? 0 : test_mremap(old, len, len, + MREMAP_FIXED | MREMAP_MAYMOVE, dest); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, old); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_EQ(test, moved, dest); + /* VA and PA offsets differ after moving by one alternative leaf. */ + if (size < PAGE_SIZE) { + pte_t pte = test_fault_entry(test, mm, dest); + + KUNIT_EXPECT_NE(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + dest & ~PAGE_MASK); + } + kthread_use_mm(mm); + bitmap = iova_bitmap_alloc(iova, bytes * 8 * unit, unit, + (u64 __user *)start); + if (!IS_ERR(bitmap)) { + ret = iova_bitmap_for_each(bitmap, &unit, test_iova_bitmap_fill); + iova_bitmap_free(bitmap); + } else { + ret = PTR_ERR(bitmap); + } + missed = copy_from_user(actual, (void __user *)dest, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + memset(expected + size - sizeof(u64), 0xff, bytes); + expected[size - sizeof(u64) + bytes - 1] = 1; + KUNIT_EXPECT_EQ(test, memcmp(actual, expected, len), 0); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap relocated %luK user leaves checked", size / SZ_1K); + } +} + +static void test_iova_free(void *bitmap) +{ + iova_bitmap_free(bitmap); +} + +static void test_iova_kvfree(void *buffer) +{ + kvfree(buffer); +} + +static void user4k_iova_bitmap_window_test(struct kunit *test) +{ + struct mm_struct *mm; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + unsigned long bytes = (capacity + 3) * SZ_4K, len = bytes + 2 * SZ_4K; + unsigned long start = TEST_VA + SZ_4K - sizeof(u64), unit = SZ_4K; + unsigned long addr, missed; + unsigned char *actual; + struct iova_bitmap *bitmap; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + actual = kvzalloc(len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, actual); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_kvfree, actual), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + bitmap = iova_bitmap_alloc(0, bytes * 8 * unit, unit, (u64 __user *)start); + ret = IS_ERR(bitmap) ? PTR_ERR(bitmap) : + iova_bitmap_for_each(bitmap, &unit, test_iova_bitmap_fill); + if (!IS_ERR(bitmap)) + iova_bitmap_free(bitmap); + missed = IS_ERR_VALUE(addr) ? len : copy_from_user(actual, (void __user *)addr, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual, 0, SZ_4K - sizeof(u64)), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual + SZ_4K - sizeof(u64), 0xff, bytes - 1), NULL); + KUNIT_EXPECT_EQ(test, actual[SZ_4K - sizeof(u64) + bytes - 1], (unsigned char)1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual + SZ_4K - sizeof(u64) + bytes, + 0, SZ_4K + sizeof(u64)), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap crossed %lu-record window and revisited first byte", capacity); +} + +static void user4k_iova_bitmap_retained_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 2 * size; + struct file *file = test_cache_file(test, 4 * PAGE_SIZE, false); + unsigned char *buffer = kunit_kzalloc(test, len, GFP_KERNEL); + unsigned long addr, replacement = 0, missed = 0; + struct iova_bitmap *bitmap; + struct folio *folio; + loff_t pos = size; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_NOT_NULL(test, buffer); + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, size); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, + (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) + iova_bitmap_set(bitmap, 3 * SZ_4K, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_free, bitmap), 0); + folio = filemap_get_folio(file->f_mapping, size >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, len); + if (!ret) { + replacement = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + if (!IS_ERR_VALUE(replacement)) { + memset(buffer, 0x5a, len); + missed = copy_to_user((void __user *)replacement, buffer, len); + } + } + /* The cached pins still refer to the old file, not the new VMA. */ + iova_bitmap_set(bitmap, 9 * SZ_4K, 2 * SZ_4K); + if (!ret && !IS_ERR_VALUE(replacement)) + missed |= copy_from_user(buffer, (void __user *)replacement, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, replacement, TEST_VA); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0x5a, len), NULL); + kunit_release_action(test, test_iova_free, bitmap); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_ASSERT_EQ(test, kernel_read(file, buffer, len, &pos), (ssize_t)len); + KUNIT_EXPECT_EQ(test, buffer[0], (unsigned char)8); + KUNIT_EXPECT_EQ(test, buffer[1], (unsigned char)6); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 2, 0, len - 2), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap %luK shared backing retained across unmap/replacement", size / SZ_1K); + } +} + +static long test_mprotect(unsigned long start, unsigned long len, unsigned long prot); + +static void user4k_iova_bitmap_partial_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 3 * size; + unsigned char *buffer = kunit_kzalloc(test, len, GFP_KERNEL); + unsigned long addr, refill, missed; + struct iova_bitmap *bitmap; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, buffer); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, buffer, len); + ret = vm_munmap(TEST_VA + size, size); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + missed |= copy_from_user(buffer, (void __user *)TEST_VA, size); + missed |= copy_from_user(buffer + 2 * size, (void __user *)(TEST_VA + 2 * size), size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_free, bitmap), 0); + /* Preserve the existing void API: pin failure stops at the valid prefix. */ + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0xff, size), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 2 * size, 0, size), NULL); + kthread_use_mm(mm); + refill = vm_mmap(NULL, TEST_VA + size, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + missed = copy_from_user(buffer, (void __user *)TEST_VA, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, refill, TEST_VA + size); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0xff, len), NULL); + kunit_release_action(test, test_iova_free, bitmap); + memset(buffer, 0, len); + kthread_use_mm(mm); + missed = copy_to_user((void __user *)TEST_VA, buffer, len); + ret = test_mprotect(TEST_VA, len, PROT_READ); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) { + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + iova_bitmap_free(bitmap); + } + missed |= copy_from_user(buffer, (void __user *)TEST_VA, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, len), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap %luK partial pin, retry and read-only checks passed", size / SZ_1K); + } +} + +static u64 test_table_descriptor(struct mm_struct *mm, unsigned long addr, bool pud) +{ + pgd_t *pgdp = pgd_offset(mm, addr); + p4d_t *p4dp = p4d_offset_mm(mm, pgdp, addr); + pud_t *pudp = pud_offset_mm(mm, p4dp, addr); + + return pud ? pud_val(*pudp) : pmd_val(*pmd_offset_mm(mm, pudp, addr)); +} + +static void user4k_mremap_move_test(struct kunit *test) +{ + static const struct { unsigned long old, new, len; unsigned int level; } cases[] = { + { TEST_VA + SZ_4K, 3 * SZ_2M + 2 * SZ_4K, 3 * SZ_4K, 0 }, + { SZ_2M - SZ_4K, SZ_1G - 2 * SZ_4K, 3 * SZ_4K, 0 }, + { SZ_1G - SZ_4K, BIT_ULL(39) - 2 * SZ_4K, 3 * SZ_4K, 0 }, + { 4 * SZ_2M, 8 * SZ_2M, SZ_2M, 1 }, + { 2UL * SZ_1G, 4UL * SZ_1G, SZ_1G, 2 }, + }; + unsigned int shape; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + for (shape = 0; shape < ARRAY_SIZE(cases); shape++) { + struct mm_struct *mm = test_vma_mm(test); + unsigned long old = cases[shape].old, new = cases[shape].new; + unsigned long len = cases[shape].len; + unsigned long offsets[] = { 0, SZ_4K, len - SZ_4K }; + unsigned long result, warm, moved, vm_before, commit_before, rss_before; + unsigned long flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long old_left[3], new_left[3]; + phys_addr_t phys[3]; + u8 readback[3], value = 0x79, byte; + u64 table = 0; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (cases[shape].level) + flags |= MAP_NORESERVE; + kthread_use_mm(mm); + warm = vm_mmap(NULL, SZ_1M, SZ_4K, PROT_READ | PROT_WRITE, flags, 0); + result = vm_mmap(NULL, old, len, PROT_READ | PROT_WRITE, flags, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, warm, SZ_1M); + KUNIT_ASSERT_EQ(test, result, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, warm, &value, 1, FOLL_WRITE), 1); + for (i = 0; i < 3; i++) { + value = 0x80 + i; + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + offsets[i], &value, 1, FOLL_WRITE), 1); + phys[i] = pte_phys_mm(mm, test_fault_entry(test, mm, old + offsets[i])); + } + if (cases[shape].level) { + table = test_table_descriptor(mm, old, cases[shape].level == 2); + /* Warm-up consumed the first fragment; the moved table is packed. */ + KUNIT_EXPECT_NE(test, table & (3UL * SZ_4K), 0ULL); + } + vm_before = mm->total_vm; + commit_before = mm->committed_user_pages; + rss_before = get_mm_rss_sum(mm); + kthread_use_mm(mm); + pagefault_disable(); + for (i = 0; i < 3; i++) + old_left[i] = copy_from_user(&byte, (void __user *)(old + offsets[i]), 1); + pagefault_enable(); + moved = test_mremap(old, len, len, MREMAP_FIXED | MREMAP_MAYMOVE, new); + pagefault_disable(); + for (i = 0; i < 3; i++) { + new_left[i] = copy_from_user(&readback[i], (void __user *)(new + offsets[i]), 1); + old_left[i] += !copy_from_user(&byte, (void __user *)(old + offsets[i]), 1); + } + pagefault_enable(); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + KUNIT_EXPECT_EQ(test, mm->total_vm, vm_before); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, commit_before); + KUNIT_EXPECT_EQ(test, get_mm_rss_sum(mm), rss_before); + for (i = 0; i < 3; i++) { + KUNIT_EXPECT_EQ(test, old_left[i], 0UL); + KUNIT_EXPECT_EQ(test, new_left[i], 0UL); + KUNIT_EXPECT_EQ(test, readback[i], (u8)(0x80 + i)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, new + offsets[i])), phys[i]); + } + if (cases[shape].level) + KUNIT_EXPECT_EQ(test, test_table_descriptor(mm, new, cases[shape].level == 2), table); + } +} + +static void user4k_stack_relocate_test(struct kunit *test) +{ + const unsigned long shifts[] = { SZ_4K, SZ_2M + SZ_4K, SZ_1G + 3 * SZ_4K }; + unsigned int shape, i; + + for (shape = 0; shape < ARRAY_SIZE(shifts); shape++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct vm_page_offset pos; + unsigned long top, old, new; + phys_addr_t phys[3]; + u8 bytes[3] = { 0x42, 0x73, 0xb5 }; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, create_init_stack_vma(mm, &vma, &top), 0); + old = vma->vm_end - 3 * SZ_4K; + new = old - shifts[shape]; + mmap_write_lock(mm); + ret = expand_stack_locked(vma, old); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, + &bytes[i], 1, FOLL_WRITE), 1); + phys[i] = pte_phys_mm(mm, test_fault_entry(test, mm, old + i * SZ_4K)); + } + pos = vma_page_offset_at(vma, old); + mmap_write_lock(mm); + ret = relocate_vma_down(vma, shifts[shape]); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, vma->vm_start, new); + KUNIT_EXPECT_EQ(test, vma->vm_end, new + 3 * SZ_4K); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, new), pos)); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, new + i * SZ_4K)), phys[i]); + } + /* Grow after relocation: keep the file-like anon offset linear. */ + mmap_write_lock(mm); + vm_flags_clear(vma, VM_STACK_INCOMPLETE_SETUP); + ret = expand_stack_locked(vma, new - SZ_4K); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 4UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new - SZ_4K, &bytes[0], 1, FOLL_WRITE), 1); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, new), pos)); + } +} + +static void user4k_mremap_cow_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + unsigned long old = TEST_VA + SZ_4K, new = SZ_2M - 2 * SZ_4K, mapped; + unsigned long moved; + u8 value = 0x29, byte; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + parent = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + kthread_use_mm(parent); + mapped = vm_mmap(NULL, old, 3 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(parent); + KUNIT_ASSERT_EQ(test, mapped, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(parent, old + SZ_4K, &value, 1, FOLL_WRITE), 1); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(child); + /* Move only the middle quarter, leaving both neighbors mapped. */ + moved = test_mremap(old + SZ_4K, SZ_4K, SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(child); + KUNIT_ASSERT_EQ(test, moved, new); + value = 0xa4; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, new, &value, 1, FOLL_WRITE), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(parent, old + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x29); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, new, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, value); + KUNIT_EXPECT_EQ(test, mm_pages_to_bytes(child, child->total_vm), 3UL * SZ_4K); + KUNIT_EXPECT_EQ(test, child->committed_user_pages, 3UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, old, &value, 1, 0), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, old + 2 * SZ_4K, &value, 1, 0), 1); +} + +static void user4k_mremap_resize_test(struct kunit *test) +{ + struct mm_struct *mm; + unsigned long old = TEST_VA + SZ_4K, new = TEST_VA + 8 * PAGE_SIZE + 2 * SZ_4K; + unsigned long res[7]; + u8 value = 0x98, byte = 0; + int unmap[2]; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + res[0] = vm_mmap(NULL, old, SZ_4K, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + res[1] = test_mremap(old, SZ_4K, 3 * SZ_4K - 1, 0, 0); + res[2] = test_mremap(old + 1, SZ_4K, SZ_4K, 0, 0); + res[3] = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new + 1); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, res[0], old); + KUNIT_ASSERT_EQ(test, res[1], old); + KUNIT_EXPECT_EQ(test, res[2], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, res[3], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + SZ_4K, &value, 1, FOLL_WRITE), 1); + kthread_use_mm(mm); + res[4] = test_mremap(old, 3 * SZ_4K, 2 * SZ_4K, 0, 0); + res[5] = test_mremap(old, 2 * SZ_4K, 2 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, res[4], old); + KUNIT_ASSERT_EQ(test, res[5], new); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 4UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, value); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, old + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0); + kthread_use_mm(mm); + unmap[0] = vm_munmap(old, 2 * SZ_4K); + unmap[1] = vm_munmap(new, 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, unmap[0], 0); + KUNIT_EXPECT_EQ(test, unmap[1], 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 0UL); +} + +static int test_reject_mremap(struct vm_area_struct *vma) +{ + return -EIO; +} + +static void user4k_cold_batch_test(struct kunit *test) +{ + struct mm_struct *mm, *child; + struct vm_area_struct *vma; + unsigned long start = TEST_VA, mapped; + spinlock_t *ptl; + pte_t *ptep, entry; + u8 byte = 0x68; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm cold fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start, &byte, 1, FOLL_WRITE), 1); + /* Only the last quarter is young: batching must aggregate that bit. */ + mmap_read_lock(mm); + vma = vma_lookup(mm, start); + for (i = 0; i < 3; i++) { + unsigned long addr = start + i * SZ_4K; + + ptep = test_mm_lookup(mm, addr, &ptl); + if (!ptep) { + mmap_read_unlock(mm); + KUNIT_FAIL(test, "missing cold test PTE"); + return; + } + entry = ptep_clear_flush(vma, addr, ptep); + set_pte_at(mm, addr, ptep, pte_mkold(entry)); + pte_unmap_unlock(ptep, ptl); + } + mmap_read_unlock(mm); + kthread_use_mm(mm); + ret = do_madvise(mm, start + 3 * SZ_4K, SZ_4K, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, pte_young(test_fault_entry(test, mm, start + 3 * SZ_4K))); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(mm); + ret = do_madvise(mm, start, PAGE_SIZE, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, pte_young(test_fault_entry(test, mm, start + 3 * SZ_4K))); + kunit_release_action(test, user4k_vma_mm_free, child); + kthread_use_mm(mm); + ret = do_madvise(mm, start, PAGE_SIZE, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_FALSE(test, pte_young(test_fault_entry(test, mm, start + i * SZ_4K))); + byte = 0; + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, start, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x68); +} + +static void user4k_prefault_dirty_test(struct kunit *test) +{ + struct mm_struct *mm; + struct folio *folio; + unsigned long start = TEST_VA + SZ_4K, mapped; + spinlock_t *ptl; + pte_t *ptep, entry; + u8 byte = 0x7a; + int locked = 1; + long ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm prefault fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, SZ_4K, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start, &byte, 1, FOLL_WRITE), 1); + mmap_read_lock(mm); + ptep = test_mm_lookup(mm, start, &ptl); + if (!ptep) { + mmap_read_unlock(mm); + KUNIT_FAIL(test, "missing prefault test PTE"); + return; + } + entry = ptep_clear_flush(vma_lookup(mm, start), start, ptep); + folio = page_folio(pte_page(entry)); + folio_get(folio); + set_pte_at(mm, start, ptep, pte_mkclean(entry)); + folio_clear_dirty(folio); + pte_unmap_unlock(ptep, ptl); + mmap_read_unlock(mm); + kthread_use_mm(mm); + mmap_read_lock(mm); + ret = faultin_page_range(mm, start, start + SZ_4K, true, &locked); + if (locked) + mmap_read_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 1L); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + folio_put(folio); +} + +struct test_fsync_record { + loff_t start, end; + unsigned int calls; + int datasync; +}; + +static int test_fsync_range(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct test_fsync_record *record = file->private_data; + + *record = (struct test_fsync_record) { start, end, record->calls + 1, datasync }; + return 0; +} + +asmlinkage long __arm64_sys_msync(const struct pt_regs *regs); + +static void user4k_msync_offsets_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vma; + struct file_operations ops; + const struct file_operations *original_ops; + void *original_private; + struct test_fsync_record records[3] = {}, record = {}; + struct pt_regs regs = { .regs = { TEST_VA + SZ_4K, 1, MS_SYNC } }; + long result[3]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm sync fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 3 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 5 * SZ_4K, 3 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + original_ops = file->f_op; + original_private = file->private_data; + ops = *original_ops; + ops.fsync = test_fsync_range; + file->f_op = &ops; + file->private_data = &record; + kthread_use_mm(vma->vm_mm); + for (i = 0; i < ARRAY_SIZE(result); i++) { + regs.regs[0] = vma->vm_start + i * SZ_4K; + result[i] = __arm64_sys_msync(®s); + records[i] = record; + } + kthread_unuse_mm(vma->vm_mm); + file->f_op = original_ops; + file->private_data = original_private; + for (i = 0; i < ARRAY_SIZE(result); i++) { + KUNIT_EXPECT_EQ(test, result[i], 0L); + KUNIT_EXPECT_EQ(test, records[i].calls, i + 1); + KUNIT_EXPECT_EQ(test, records[i].start, (loff_t)(3 + i) * SZ_4K); + KUNIT_EXPECT_EQ(test, records[i].end, (loff_t)(4 + i) * SZ_4K - 1); + KUNIT_EXPECT_EQ(test, records[i].datasync, 1); + } +} + +static void user4k_mremap_file_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + struct file *file; + struct vm_operations_struct rejecting_ops; + const struct vm_operations_struct *original_ops; + unsigned long old = TEST_VA + SZ_4K, new = SZ_1G - SZ_4K; + unsigned long mapped, moved; + u8 bytes[3]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(file, old, 3 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, 3 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, old); + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, &bytes[i], 1, 0), 1); + vma = test_vma_lookup(mm, old); + KUNIT_ASSERT_NOT_NULL(test, vma); + original_ops = vma->vm_ops; + rejecting_ops = *original_ops; + rejecting_ops.mremap = test_reject_mremap; + mmap_write_lock(mm); + vma->vm_ops = &rejecting_ops; + mmap_write_unlock(mm); + kthread_use_mm(mm); + moved = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + /* Restore stack-owned callbacks before any assertion can leave this scope. */ + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *part; + + for_each_vma(vmi, part) + if (part->vm_ops == &rejecting_ops) + part->vm_ops = original_ops; + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, moved, (unsigned long)-EIO); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 1); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + kthread_use_mm(mm); + moved = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + vma = test_vma_lookup(mm, new); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 3U * SZ_4K); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + } +} + +static void user4k_exec_arguments_test(struct kunit *test) +{ + unsigned int native; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm stack setup requires hardware opt-in"); + return; + } + for (native = 0; native < 2; native++) { + struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); + struct mm_struct *mm; + unsigned long size, mask, initial, copied, new_top; + unsigned long a_len = 2 * PAGE_SIZE + SZ_4K + 31, b_len = SZ_4K + 43; + char *a, *b, *buf, *too_long; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, bprm); + mm = native ? mm_alloc() : test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + size = mm_page_size(mm); + mask = mm_page_mask(mm); + a_len = min(a_len, 16 * size + 31); + bprm->mm = mm; + KUNIT_ASSERT_EQ(test, create_init_stack_vma(mm, &bprm->vma, &bprm->p), 0); + initial = bprm->p; + bprm->argmin = initial - SZ_2M; + bprm->rlim_stack.rlim_cur = SZ_8M; + bprm->rlim_stack.rlim_max = RLIM_INFINITY; + a = kunit_kmalloc(test, a_len + 1, GFP_KERNEL); + b = kunit_kmalloc(test, b_len + 1, GFP_KERNEL); + buf = kunit_kmalloc(test, a_len + b_len + 7, GFP_KERNEL); + too_long = kunit_kmalloc(test, 32 * size + 1, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, buf); + KUNIT_ASSERT_NOT_NULL(test, too_long); + memset(a, 'A', a_len); + a[a_len] = 0; + memset(b, 'b', b_len); + b[b_len] = 0; + memset(too_long, 'x', 32 * size); + too_long[32 * size] = 0; + KUNIT_EXPECT_EQ(test, copy_string_kernel(too_long, bprm), -E2BIG); + KUNIT_EXPECT_EQ(test, bprm->p, initial); + KUNIT_ASSERT_EQ(test, copy_string_kernel("tail", bprm), 0); + bprm->exec = bprm->p; + KUNIT_ASSERT_EQ(test, copy_string_kernel(a, bprm), 0); + KUNIT_ASSERT_EQ(test, copy_string_kernel(b, bprm), 0); + bprm->argc = 3; + copied = initial - bprm->p; + KUNIT_EXPECT_EQ(test, copied, a_len + b_len + 7); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, 0), (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_MEMEQ(test, buf + b_len + 1, a, a_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + new_top = (bprm->vma->vm_end - SZ_2M - size) & mask; + kthread_use_mm(mm); + ret = setup_arg_pages(bprm, new_top, EXSTACK_DISABLE_X); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Subtract < PAGE_SIZE, then round down to 16 bytes and up to a leaf. */ + KUNIT_EXPECT_LE(test, bprm->vma->vm_end, new_top); + KUNIT_EXPECT_LE(test, new_top - bprm->vma->vm_end, PAGE_SIZE); + KUNIT_EXPECT_EQ(test, bprm->vma->vm_end & ~mask, 0UL); + KUNIT_EXPECT_EQ(test, mm->start_stack, bprm->p); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, 0), (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_MEMEQ(test, buf + b_len + 1, a, a_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + /* /proc/PID/cmdline and environ read the stack with FOLL_ANON. */ + memset(buf, 0, copied); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, FOLL_ANON), + (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + /* Interpreter argument removal scans several user granules. */ + initial = bprm->p; + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + 1); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + a_len + 2); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + copied); + KUNIT_EXPECT_EQ(test, bprm->argc, 0); + KUNIT_EXPECT_EQ(test, bprm->vma_pages, 0UL); + } +} + +/* The transfer can replace bprm->mm, so release its final owner. */ +static void test_exec_mm_free(void *data) +{ + struct linux_binprm *bprm = data; + + mmput(bprm->mm); +} + +static void user64k_exec_arguments_test(struct kunit *test) +{ + unsigned int shifts[] = { 16, 12, PAGE_SHIFT }; + unsigned int mode; + + if (PAGE_SHIFT >= 16 || + (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { + kunit_skip(test, "requires internal larger user granule and MM opt-in"); + return; + } + for (mode = 0; mode < ARRAY_SIZE(shifts); mode++) { + struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); + unsigned long a_len = mode ? SZ_64K + SZ_16K + 31 : 32 * PAGE_SIZE + 31; + unsigned long b_len = PAGE_SIZE + 17, initial, copied; + char *a, *b, *buf; + unsigned long i; + + KUNIT_ASSERT_NOT_NULL(test, bprm); + bprm->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, bprm->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_exec_mm_free, bprm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(bprm->mm, 16), 0); + KUNIT_ASSERT_EQ(test, create_init_stack_vma(bprm->mm, &bprm->vma, &bprm->p), 0); + initial = bprm->p; + bprm->argmin = initial - SZ_2M; + bprm->rlim_stack.rlim_cur = SZ_8M; + bprm->rlim_stack.rlim_max = RLIM_INFINITY; + a = kunit_kmalloc(test, a_len + 1, GFP_KERNEL); + b = kunit_kmalloc(test, b_len + 1, GFP_KERNEL); + buf = kunit_kmalloc(test, a_len + b_len + 7, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, buf); + for (i = 0; i < a_len; i++) + a[i] = 'A' + i % 23; + a[a_len] = 0; + memset(b, 'b', b_len); + b[b_len] = 0; + KUNIT_ASSERT_EQ(test, copy_string_kernel("tail", bprm), 0); + KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, 5, 0), 5); + KUNIT_ASSERT_STREQ(test, buf, "tail"); + KUNIT_ASSERT_EQ(test, copy_string_kernel(a, bprm), 0); + KUNIT_ASSERT_EQ(test, copy_string_kernel(b, bprm), 0); + bprm->argc = 3; + copied = initial - bprm->p; + KUNIT_ASSERT_EQ(test, copied, a_len + b_len + 7); + KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), + (int)copied); + KUNIT_ASSERT_EQ(test, memcmp(buf, b, b_len + 1), 0); + KUNIT_ASSERT_EQ(test, memcmp(buf + b_len + 1, a, a_len + 1), 0); + KUNIT_ASSERT_STREQ(test, buf + b_len + a_len + 2, "tail"); + /* Exercise copying out of every native page within a coarse leaf. */ + if (shifts[mode] != 16) { + KUNIT_ASSERT_EQ(test, bprm_set_page_shift(bprm, shifts[mode]), 0); + KUNIT_ASSERT_EQ(test, mm_page_shift(bprm->mm), shifts[mode]); + memset(buf, 0, copied); + KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), + (int)copied); + KUNIT_ASSERT_EQ(test, memcmp(buf, b, b_len + 1), 0); + KUNIT_ASSERT_EQ(test, memcmp(buf + b_len + 1, a, a_len + 1), 0); + KUNIT_ASSERT_STREQ(test, buf + b_len + a_len + 2, "tail"); + } + initial = bprm->p; + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + 1); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + a_len + 2); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + copied); + KUNIT_EXPECT_EQ(test, bprm->argc, 0); + KUNIT_EXPECT_EQ(test, bprm->vma_pages, 0UL); + } +} + +static struct vm_area_struct *test_special_vma(struct mm_struct *mm, + const struct vm_special_mapping *sm, unsigned long start, unsigned long len, + vm_flags_t flags) +{ + struct vm_area_struct *vma; + + mmap_write_lock(mm); + vma = _install_special_mapping(mm, start, len, flags, sm); + mmap_write_unlock(mm); + return vma; +} + +static void user4k_special_mapping_test(struct kunit *test) +{ + struct vm_special_mapping *sm = kunit_kzalloc(test, sizeof(*sm), GFP_KERNEL); + struct page **pages = kunit_kcalloc(test, 3, sizeof(*pages), GFP_KERNEL); + void *data[2] = { test_alloc_page(test), test_alloc_page(test) }; + struct mm_struct *small = test_vma_mm(test), *cold = test_vma_mm(test); + struct mm_struct *native = mm_alloc(); + struct vm_area_struct *vma[3]; + unsigned long start = SZ_2M - SZ_4K; + unsigned int i, side; + u8 byte; + + KUNIT_ASSERT_NOT_NULL(test, sm); + KUNIT_ASSERT_NOT_NULL(test, pages); + KUNIT_ASSERT_NOT_NULL(test, data[0]); + KUNIT_ASSERT_NOT_NULL(test, data[1]); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, cold); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + /* Reverse the supplied native pages: folio indices cannot identify them. */ + pages[0] = virt_to_page(data[1]); + pages[1] = virt_to_page(data[0]); + sm->pages = pages; + sm->name = "[user4k-special-test]"; + for (i = 0; i < 2 * TEST_SLOTS; i++) + memset(page_address(pages[i / TEST_SLOTS]) + (i % TEST_SLOTS) * SZ_4K, 0x41 + i, SZ_4K); + vma[0] = test_special_vma(small, sm, start, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma[1] = test_special_vma(cold, sm, start, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma[2] = test_special_vma(native, sm, TEST_VA, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + for (i = 0; i < 3; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma[i]); + for (i = 0; i < 2 * TEST_SLOTS; i++) { + unsigned long addr = start + i * SZ_4K; + + KUNIT_ASSERT_EQ(test, access_remote_vm(small, addr, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0x41 + i)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small, test_fault_entry(test, small, addr)), + page_to_phys(pages[i / TEST_SLOTS]) + (i % TEST_SLOTS) * SZ_4K); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(small, MM_FILEPAGES), 2UL * TEST_SLOTS); + /* Both present-PTE and cold-PTE forced writes copy only their quarter. */ + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? cold : small; + + byte = 0xe1 + side; + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start + 2 * SZ_4K + 7, + &byte, 1, FOLL_WRITE | FOLL_FORCE), 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, start + 2 * SZ_4K + 7, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0xe1 + side)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native, TEST_VA + 2 * SZ_4K + 7, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x43); + } + { + struct user_page_fragment fragment; + long ret = pin_user_fragments_remote(small, start + 5 * SZ_4K + 13, + 7, 0, &fragment, 1); + + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_NOT_NULL(test, fragment.subpage); + KUNIT_EXPECT_EQ(test, fragment.offset, (5 * SZ_4K + 13UL) & ~PAGE_MASK); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment.folio) + fragment.offset, + 0x46, fragment.length), NULL); + release_user_fragments(&fragment, 1, false); + } +} + +static void user4k_insert_range_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + void *data[2] = { test_alloc_page(test), test_alloc_page(test) }; + struct page *pages[2]; + struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long size, start, offset, length, num; + unsigned int i; + int ret, invalid[4]; + + KUNIT_ASSERT_NOT_NULL(test, data[0]); + KUNIT_ASSERT_NOT_NULL(test, data[1]); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + pages[0] = virt_to_page(data[1]); + pages[1] = virt_to_page(data[0]); + size = mm_page_size(mm); + start = pmd_size_mm(mm) - size; + offset = native ? 0 : 3 * SZ_4K; + length = 2 * PAGE_SIZE - offset; + vma = test_vma_add(mm, start, length, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + invalid[0] = vm_insert_pages_range(vma, start + 1, pages, 2, offset, length); + invalid[1] = vm_insert_pages_range(vma, start, pages, 2, offset + 1, length); + invalid[2] = vm_insert_pages_range(vma, start, pages, 2, offset, length + size); + invalid[3] = vm_insert_pages_range(vma, start, pages, ULONG_MAX, 0, length); + ret = vm_insert_pages_range(vma, start, pages, 2, offset, length); + mmap_write_unlock(mm); + for (i = 0; i < ARRAY_SIZE(invalid); i++) + KUNIT_EXPECT_EQ(test, invalid[i], -EINVAL); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < length / size; i++) { + unsigned long source = offset + i * size; + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[source >> PAGE_SHIFT]) + offset_in_page(source)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), length / size); + /* The original native-page-count API also expands all user leaves. */ + start += 4 * PAGE_SIZE; + vma = test_vma_add(mm, start, 2 * PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + num = 2; + mmap_write_lock(mm); + ret = vm_insert_pages(vma, start, pages, &num); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, num, 0UL); + for (i = 0; i < 2 * PAGE_SIZE / size; i++) + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[(i * size) >> PAGE_SHIFT]) + offset_in_page(i * size)); + num = 2; + mmap_write_lock(mm); + ret = vm_insert_pages(vma, start, pages, &num); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EBUSY); + KUNIT_EXPECT_EQ(test, num, 2UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), + (length + 2 * PAGE_SIZE) / size); + start += 4 * PAGE_SIZE; + offset = native ? PAGE_SIZE : 3 * SZ_4K; + length = 2 * PAGE_SIZE - offset; + vma = test_vma_add(mm, start, length, + (struct vm_page_offset) { .index = offset >> PAGE_SHIFT, + .offset = offset_in_page(offset) }, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + ret = vm_map_pages(vma, pages, 2); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < length / size; i++) { + unsigned long source = offset + i * size; + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[source >> PAGE_SHIFT]) + offset_in_page(source)); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(pages[0]), 1); + KUNIT_EXPECT_EQ(test, page_ref_count(pages[1]), 1); + } +} + +static void user4k_remap_pfn_range_test(struct kunit *test) +{ + struct folio *folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 1); + + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (unsigned int native = 0; native < 2; native++) { + struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); + unsigned long size, boundaries[3]; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, mm), 0); + size = mm_page_size(mm); + boundaries[0] = pmd_size_mm(mm); + boundaries[1] = pud_size_mm(mm); + boundaries[2] = pgd_size_mm(mm); + for (unsigned int shape = 0; shape < ARRAY_SIZE(boundaries); shape++) { + unsigned long start = boundaries[shape] - size; + unsigned long length = PAGE_SIZE + size; + struct vm_area_struct *vma; + int ret; + + if (shape && boundaries[shape] == boundaries[shape - 1]) + continue; + vma = test_vma_add(mm, start, length, (struct vm_page_offset) {}, + VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_SHARED | VM_MAYSHARE); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + ret = remap_pfn_range(vma, start, folio_pfn(folio), length, + vma->vm_page_prot); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + for (unsigned long offset = 0; offset < length; offset += size) { + pte_t pte = test_fault_entry(test, mm, start + offset); + + struct follow_pfnmap_args args = { + .vma = vma, .address = start + offset + 31, + }; + phys_addr_t expected = PFN_PHYS(folio_pfn(folio)) + offset + 31; + int ret; + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_special(pte)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), + PFN_PHYS(folio_pfn(folio)) + offset); + + mmap_read_lock(mm); + ret = follow_pfnmap_start(&args); + if (!ret) + follow_pfnmap_end(&args); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, native ? 0 : -EOPNOTSUPP); + args.allow_subpage = true; + mmap_read_lock(mm); + ret = follow_pfnmap_start(&args); + if (!ret) { + KUNIT_EXPECT_EQ(test, args.pfn, (unsigned long)PHYS_PFN(expected)); + KUNIT_EXPECT_EQ(test, args.offset, (unsigned int)offset_in_page(expected)); + KUNIT_EXPECT_EQ(test, args.addr_mask, mm_page_mask(mm)); + KUNIT_EXPECT_TRUE(test, args.special); + follow_pfnmap_end(&args); + } + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + } + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, start - size))); + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, start + length))); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0UL); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + } +} + +#ifdef CONFIG_HAVE_IOREMAP_PROT +static void test_iounmap(void *arg) +{ + iounmap((void __iomem *)arg); +} + +static const struct vm_operations_struct test_iomem_ops = { + .access = generic_access_phys, +}; +#endif + +static void user4k_iomem_access_test(struct kunit *test) +{ +#if defined(CONFIG_HAVE_IOREMAP_PROT) && defined(CONFIG_OF) + struct device_node *node; + struct resource resource; + struct mm_struct *mm; + struct vm_area_struct *vma; + void __iomem *uart; + unsigned long start = SZ_2M + PAGE_SIZE - SZ_4K; + u8 reference[16], buffer[32]; + vm_fault_t fault; + int ret; + + if (!of_machine_is_compatible("linux,dummy-virt")) { + kunit_skip(test, "requires disposable QEMU virt PL011 identification registers"); + return; + } + node = of_find_compatible_node(NULL, NULL, "arm,pl011"); + KUNIT_ASSERT_NOT_NULL(test, node); + ret = of_address_to_resource(node, 0, &resource); + uart = ret ? NULL : of_iomap(node, 0); + of_node_put(node); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_NOT_NULL(test, uart); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iounmap, + (void *)uart), 0); + KUNIT_ASSERT_GE(test, resource_size(&resource), (resource_size_t)SZ_4K); + memcpy_fromio(reference, uart + 0xff0, sizeof(reference)); + KUNIT_ASSERT_NOT_NULL(test, memchr_inv(reference, 0, sizeof(reference))); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = test_vma_add(mm, start, 2 * SZ_4K, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_SHARED | VM_MAYSHARE | VM_IO | VM_PFNMAP); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_ops = &test_iomem_ops; + vma->vm_page_prot = pgprot_device(vm_get_page_prot(vma->vm_flags)); + mmap_read_lock(mm); + fault = vmf_insert_pfn_prot_mkwrite_offset(vma, start, + PHYS_PFN(resource.start), offset_in_page(resource.start), + vma->vm_page_prot, false); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)VM_FAULT_NOPAGE); + memset(buffer, 0x6d, sizeof(buffer)); + mmap_read_lock(mm); + ret = generic_access_phys(vma, start + 0xff0, buffer, sizeof(buffer), 0); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 16); + KUNIT_EXPECT_MEMEQ(test, buffer, reference, sizeof(reference)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 16, 0x6d, 16), NULL); + memset(buffer, 0x5a, sizeof(buffer)); + ret = access_remote_vm(mm, start + 0xff0, buffer, sizeof(buffer), 0); + KUNIT_EXPECT_EQ(test, ret, 16); + KUNIT_EXPECT_MEMEQ(test, buffer, reference, sizeof(reference)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 16, 0x5a, 16), NULL); + /* Writes must fail before touching the read-only device mapping. */ + mmap_read_lock(mm); + ret = generic_access_phys(vma, start + 0xff0, buffer, 1, FOLL_WRITE); + mmap_read_unlock(mm); + KUNIT_EXPECT_LT(test, ret, 0); +#else + kunit_skip(test, "requires OF and iomem access support"); +#endif +} + +static void user4k_pfn_offset_test(struct kunit *test) +{ + void *data = test_alloc_page(test); + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long start = SZ_2M - SZ_4K; + unsigned long pfn; + unsigned int i; + vm_fault_t result[4], invalid[3], write; + + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, mm); + pfn = page_to_pfn(virt_to_page(data)); + vma = test_vma_add(mm, start, PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | VM_SHARED | VM_MAYSHARE | + VM_PFNMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + /* Model a write-notify VMA: initial PTEs are read-only. */ + vma->vm_page_prot = vm_get_page_prot(VM_READ | VM_SHARED); + mmap_read_lock(mm); + for (i = 0; i < 4; i++) + result[i] = vmf_insert_pfn_prot_mkwrite_offset(vma, start + i * SZ_4K, + pfn, (3 - i) * SZ_4K, vma->vm_page_prot, false); + invalid[0] = vmf_insert_pfn_prot_mkwrite_offset(vma, start + 1, pfn, + 0, vma->vm_page_prot, false); + invalid[1] = vmf_insert_pfn_prot_mkwrite_offset(vma, start, pfn, + 1, vma->vm_page_prot, false); + invalid[2] = vmf_insert_pfn_prot_mkwrite_offset(vma, start, pfn, + PAGE_SIZE, vma->vm_page_prot, false); + write = vmf_insert_pfn_prot_mkwrite_offset(vma, start + SZ_4K, pfn, + 2 * SZ_4K, vma->vm_page_prot, true); + mmap_read_unlock(mm); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, mm, start + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, result[i], (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), PFN_PHYS(pfn) + (3 - i) * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_special(entry)); + KUNIT_EXPECT_EQ(test, !!pte_write(entry), i == 1); + } + for (i = 0; i < ARRAY_SIZE(invalid); i++) + KUNIT_EXPECT_EQ(test, invalid[i], (vm_fault_t)VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, write, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 0UL); + KUNIT_EXPECT_EQ(test, page_ref_count(virt_to_page(data)), 1); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(virt_to_page(data)), 1); +} + +static void user16k_insert_shared_test(struct kunit *test) +{ + struct mm_struct *middle, *small, *native; + struct vm_area_struct *vmas[3]; + struct mm_subpage_mapcounts counts; + struct page *page; + void *data; + vm_fault_t result; + unsigned int i; + vm_flags_t flags = VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "mixed16K file insertion requires larger native backing"); + return; + } + data = test_alloc_page(test); + KUNIT_ASSERT_NOT_NULL(test, data); + page = virt_to_page(data); + middle = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, middle); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, middle), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(middle, 14), 0); + small = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, small); + native = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + vmas[0] = test_vma_add(middle, TEST_VA, PAGE_SIZE, (struct vm_page_offset) {}, flags); + vmas[1] = test_vma_add(small, TEST_VA + SZ_4K, SZ_4K, (struct vm_page_offset) {}, flags); + vmas[2] = test_vma_add(native, TEST_VA, PAGE_SIZE, (struct vm_page_offset) {}, flags); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vmas[i]); + vmas[i]->vm_page_prot = vm_get_page_prot(vmas[i]->vm_flags); + } + for (i = 0; i < PAGE_SIZE / SZ_16K; i++) { + unsigned long address = TEST_VA + i * SZ_16K; + + mmap_read_lock(middle); + result = vmf_insert_page_offset(vmas[0], address, page, i * SZ_16K); + mmap_read_unlock(middle); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, pte_phys_mm(middle, test_fault_entry(test, middle, address)), + page_to_phys(page) + i * SZ_16K); + } + mmap_read_lock(small); + result = vmf_insert_page_offset(vmas[1], vmas[1]->vm_start, page, PAGE_SIZE - SZ_4K); + mmap_read_unlock(small); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + mmap_read_lock(native); + result = vmf_insert_page_offset(vmas[2], vmas[2]->vm_start, page, 0); + mmap_read_unlock(native); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 5U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], i == TEST_SLOTS - 1 ? 2U : 1U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 6); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 7); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(middle, MM_FILEPAGES), 4UL); + /* Real mm teardown removes whole16K leaves and preserves both aliases. */ + kunit_release_action(test, user4k_vma_mm_free, middle); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 1U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], i == TEST_SLOTS - 1 ? 1U : 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 2); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 3); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small, test_fault_entry(test, small, vmas[1]->vm_start)), + page_to_phys(page) + PAGE_SIZE - SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(native, test_fault_entry(test, native, vmas[2]->vm_start)), + page_to_phys(page)); + kunit_release_action(test, user4k_vma_mm_free, small); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 1); + kunit_release_action(test, user4k_vma_mm_free, native); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 0); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 1); +} + +static void user4k_insert_fragment_test(struct kunit *test) +{ + void *data = test_alloc_page(test); + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + struct page *page; + vm_fault_t result[4], invalid[4]; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, mm); + page = virt_to_page(data); + vma = test_vma_add(mm, start, PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_read_lock(mm); + for (i = 0; i < 4; i++) + result[i] = vmf_insert_page_offset(vma, start + i * SZ_4K, page, (3 - i) * SZ_4K); + invalid[0] = vmf_insert_page_offset(vma, start + 1, page, 0); + invalid[1] = vmf_insert_page_offset(vma, start, page, 1); + invalid[2] = vmf_insert_page_offset(vma, start, page, PAGE_SIZE); + invalid[3] = vmf_insert_page_offset(vma, vma->vm_end, page, 0); + mmap_read_unlock(mm); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, result[i], (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, invalid[i], (vm_fault_t)VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, start + i * SZ_4K)), + page_to_phys(page) + (3 - i) * SZ_4K); + } + KUNIT_EXPECT_EQ(test, page_ref_count(page), 5); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 4UL); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 1); +} + +#if defined(CONFIG_GENERIC_GETTIMEOFDAY) && defined(CONFIG_VDSO_GETRANDOM) +static void user4k_vdso_mapping_test(struct kunit *test) +{ + struct mm_struct *mm; + struct linux_binprm bprm = {}; + unsigned long code, base, left[8]; + u8 bytes[8], magic[4]; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm vDSO fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + mm->mmap_base = TEST_VA + SZ_4K; + mm_flags_clear(MMF_TOPDOWN, mm); + kthread_use_mm(mm); + ret = arch_setup_additional_pages(&bprm, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + code = (unsigned long)mm->context.vdso; + KUNIT_ASSERT_NE(test, code, 0UL); + base = code - VDSO_NR_PAGES * PAGE_SIZE; + KUNIT_EXPECT_EQ(test, base & ~PAGE_MASK, SZ_4K); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, code, magic, sizeof(magic), 0), (int)sizeof(magic)); + KUNIT_EXPECT_MEMEQ(test, magic, "\177ELF", 4); + /* VVAR forbids remote GUP; ordinary uaccess must still fault it in. */ + kthread_use_mm(mm); + for (i = 0; i < 4; i++) { + left[i] = copy_from_user(&bytes[i], (void __user *)(base + i * SZ_4K), 1); + left[4 + i] = copy_from_user(&bytes[4 + i], + (void __user *)(base + VDSO_RNG_PAGE_OFFSET * PAGE_SIZE + i * SZ_4K), 1); + } + kthread_unuse_mm(mm); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, left[i], 0UL); + KUNIT_EXPECT_EQ(test, left[4 + i], 0UL); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, base + i * SZ_4K)), + page_to_phys(virt_to_page(vdso_k_time_data)) + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, + base + VDSO_RNG_PAGE_OFFSET * PAGE_SIZE + i * SZ_4K)), + page_to_phys(virt_to_page(vdso_k_rng_data)) + i * SZ_4K); + } + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, base, magic, sizeof(magic), 0), 0); +} + +#endif + +static void user4k_remote_access_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA + SZ_4K, 3 * SZ_4K); + struct mm_struct *child = test_vma_mm(test); + struct vm_area_struct *child_vma; + unsigned long start = TEST_VA + SZ_4K; + u8 *buf = kunit_kmalloc(test, 3 * SZ_4K, GFP_KERNEL); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + KUNIT_ASSERT_NOT_NULL(test, buf); + memset(buf, 0x5a, 3 * SZ_4K); + KUNIT_ASSERT_EQ(test, access_remote_vm(vma->vm_mm, start, buf, 3 * SZ_4K, FOLL_WRITE), + 3 * (int)SZ_4K); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Deliberately straddle quarters whose physical/virtual offsets differ. */ + memset(buf, 0x96, SZ_4K + 34); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + SZ_4K - 17, + buf, SZ_4K + 34, FOLL_WRITE), (int)SZ_4K + 34); + memset(buf, 0, 3 * SZ_4K); + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, start, buf, 3 * SZ_4K, 0), + 3 * (int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x5a, 3 * SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start, buf, 3 * SZ_4K, 0), 3 * (int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x5a, SZ_4K - 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + SZ_4K - 17, 0x96, SZ_4K + 34), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 2 * SZ_4K + 17, 0x5a, SZ_4K - 17), NULL); + child_vma = test_vma_lookup(child, start); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + mmap_write_lock(child); + vm_flags_clear(child_vma, VM_WRITE); + mmap_write_unlock(child); + buf[0] = 0x27; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, FOLL_WRITE), 0); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, + FOLL_WRITE | FOLL_FORCE), 1); + buf[0] = 0; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, 0), 1); + KUNIT_EXPECT_EQ(test, buf[0], (u8)0x27); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 3 * SZ_4K - 5, buf, 20, 0), 5); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 3 * SZ_4K, buf, 20, 0), 0); +} + +static void user4k_remote_file_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *small, *native; + u8 buf[34]; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, 2 * SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 3 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x34, 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 17, 0x35, 17), NULL); + memset(buf, 0xa6, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 3 * SZ_4K - 17, + buf, sizeof(buf), FOLL_WRITE), (int)sizeof(buf)); + memset(buf, 0, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 4 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0xa6, sizeof(buf)), NULL); +#ifdef CONFIG_CROSS_MEMORY_ATTACH + { + struct kvec vec = { .iov_base = buf, .iov_len = sizeof(buf) }; + struct iov_iter iter; + + memset(buf, 0xb7, sizeof(buf)); + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, sizeof(buf)); + KUNIT_EXPECT_EQ(test, process_vm_rw_fragments(small->vm_mm, + TEST_VA + 3 * SZ_4K - 17, sizeof(buf), &iter, true), 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + memset(buf, 0, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 4 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0xb7, sizeof(buf)), NULL); + } +#endif + mmap_write_lock(small->vm_mm); + vm_flags_clear(small, VM_WRITE); + mmap_write_unlock(small->vm_mm); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, small->vm_start, buf, 1, + FOLL_WRITE | FOLL_FORCE), 0); +#ifdef CONFIG_CROSS_MEMORY_ATTACH + { + struct kvec vec = { .iov_base = buf, .iov_len = sizeof(buf) }; + struct iov_iter iter; + + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, sizeof(buf)); + KUNIT_EXPECT_EQ(test, process_vm_rw_fragments(small->vm_mm, + small->vm_start, sizeof(buf), &iter, true), -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), sizeof(buf)); + } +#endif +} + +#ifdef CONFIG_CROSS_MEMORY_ATTACH +static void user4k_process_vm_test(struct kunit *test) +{ + /* Exceed the stack batch, cross table boundaries and include a partial tail. */ + unsigned long start = SZ_2M - SZ_4K, len = 21 * SZ_4K; + struct vm_area_struct *vma = test_fault_vma(test, start, len); + struct mm_struct *child = test_vma_mm(test); + struct iov_iter iter; + struct kvec vec; + u8 *buf = kunit_kmalloc(test, len + 17, GFP_KERNEL); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + KUNIT_ASSERT_NOT_NULL(test, buf); + memset(buf, 0x61, len); + vec = (struct kvec) { .iov_base = buf, .iov_len = len }; + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, len); + ret = process_vm_rw_fragments(vma->vm_mm, start, len, &iter, true); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, iov_iter_count(&iter), 0UL); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(buf, 0xc7, len); + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, len - 34); + ret = process_vm_rw_fragments(child, start + 17, len - 34, &iter, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + memset(buf, 0, len); + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, len); + ret = process_vm_rw_fragments(vma->vm_mm, start, len, &iter, false); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x61, len), NULL); + vec.iov_len = len + 17; + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, len + 17); + ret = process_vm_rw_fragments(child, start, len + 17, &iter, false); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 17UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x61, 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 17, 0xc7, len - 34), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + len - 17, 0x61, 17), NULL); + /* A shorter local iterator ends successfully before the remote range. */ + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, 23); + ret = process_vm_rw_fragments(child, start + 13, len, &iter, false); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); +} +#endif + +#ifdef CONFIG_FUTEX +static void test_stop_kthread(void *arg); + +struct test_futex_waiter { + struct mm_struct *mm; + unsigned long address; + struct completion ready; + struct completion done; + int result; +}; + +static int test_futex_wait_run(void *arg) +{ + struct test_futex_waiter *w = arg; + ktime_t deadline = ktime_add_ms(ktime_get(), 10000); + + kthread_use_mm(w->mm); + complete(&w->ready); + w->result = futex_wait((u32 __user *)w->address, FLAGS_SIZE_32 | FLAGS_SHARED, + 0x33333333, &deadline, FUTEX_BITSET_MATCH_ANY); + kthread_unuse_mm(w->mm); + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void user4k_futex_wake_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vmas[2]; + unsigned long addresses[2] = { TEST_VA + SZ_4K + 64, TEST_VA + 2 * SZ_4K + 64 }; + unsigned int round; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm futex fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vmas[0] = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, SZ_4K, 2 * SZ_4K); + vmas[1] = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, vmas[0]); + KUNIT_ASSERT_NOT_NULL(test, vmas[1]); + /* Both directions use the real wait queue and atomic user-value read. */ + for (round = 0; round < 2; round++) { + struct test_futex_waiter *w = kunit_kzalloc(test, sizeof(*w), GFP_KERNEL); + struct task_struct *task; + unsigned long until; + int woke = 0, wrong; + + KUNIT_ASSERT_NOT_NULL(test, w); + w->mm = vmas[round]->vm_mm; + w->address = addresses[round]; + init_completion(&w->ready); + init_completion(&w->done); + task = kthread_run(test_futex_wait_run, w, "user4k-futex-%u", round); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->ready, 5 * HZ), 0UL); + until = jiffies + 5 * HZ; + kthread_use_mm(vmas[round ^ 1]->vm_mm); + wrong = futex_wake((u32 __user *)(addresses[round ^ 1] + 4), + FLAGS_SIZE_32 | FLAGS_SHARED, 1, FUTEX_BITSET_MATCH_ANY); + do { + woke = futex_wake((u32 __user *)addresses[round ^ 1], + FLAGS_SIZE_32 | FLAGS_SHARED, 1, FUTEX_BITSET_MATCH_ANY); + if (woke || completion_done(&w->done)) + break; + schedule_timeout_uninterruptible(1); + } while (time_before(jiffies, until)); + kthread_unuse_mm(vmas[round ^ 1]->vm_mm); + KUNIT_EXPECT_EQ(test, wrong, 0); + KUNIT_EXPECT_EQ(test, woke, 1); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->done, 12 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, w->result, 0); + kunit_release_action(test, test_stop_kthread, task); + } +} + +static void user4k_futex_keys_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *small, *native; + union futex_key keys[2], other; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm futex fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, 2 * SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + for (i = 0; i < 3; i++) { + unsigned long va = small->vm_start + i * SZ_4K + 64; + unsigned long fileoff = (i + 2) * SZ_4K + 64; + + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)va, FLAGS_SIZE_32 | FLAGS_SHARED, + &keys[0], FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(native->vm_mm); + ret = get_futex_key((u32 __user *)(TEST_VA + fileoff), + FLAGS_SIZE_32 | FLAGS_SHARED, &keys[1], FUTEX_READ); + kthread_unuse_mm(native->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, futex_match(&keys[0], &keys[1])); + KUNIT_EXPECT_EQ(test, keys[0].shared.pgoff, fileoff / PAGE_SIZE); + KUNIT_EXPECT_EQ(test, keys[0].both.offset, (unsigned int)(fileoff % PAGE_SIZE) | FUT_OFF_INODE); + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)(va + 4), FLAGS_SIZE_32 | FLAGS_SHARED, + &other, FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_FALSE(test, futex_match(&keys[0], &other)); + } + mmap_write_lock(small->vm_mm); + vm_flags_clear(small, VM_WRITE); + mmap_write_unlock(small->vm_mm); + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)(small->vm_start + 2 * SZ_4K + 64), + FLAGS_SIZE_32 | FLAGS_SHARED, &other, FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, futex_match(&keys[1], &other)); +} +#endif + +static void test_swap_usage_mapping_free(void *arg) +{ + struct address_space *mapping = arg; + + xa_destroy(&mapping->i_pages); +} + +static void user4k_shmem_swap_usage_test(struct kunit *test) +{ + struct address_space *mapping = kunit_kzalloc(test, sizeof(*mapping), GFP_KERNEL); + void *ret; + + KUNIT_ASSERT_NOT_NULL(test, mapping); + xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_swap_usage_mapping_free, mapping), 0); + /* Synthetic swap values exercise accounting without allocating swap slots. */ + ret = xa_store(&mapping->i_pages, 1, xa_mk_value(1), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + if (IS_ENABLED(CONFIG_XARRAY_MULTI)) { + ret = xa_store_range(&mapping->i_pages, 4, 7, + xa_mk_value(2), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + } else { + unsigned long index; + + /* Without multi-index entries, shmem stores one value per page. */ + for (index = 4; index <= 7; index++) { + ret = xa_store(&mapping->i_pages, index, + xa_mk_value(2), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + } + } + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, PAGE_SIZE + SZ_4K, + PAGE_SIZE + 3 * SZ_4K), 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 3 * SZ_4K, + 2 * PAGE_SIZE + SZ_4K), PAGE_SIZE); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 4 * PAGE_SIZE + SZ_4K, + 7 * PAGE_SIZE + 3 * SZ_4K), 3 * PAGE_SIZE + 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 5 * PAGE_SIZE + SZ_4K, + 5 * PAGE_SIZE + 2 * SZ_4K), (unsigned long)SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 0, SZ_4K), 0UL); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, PAGE_SIZE, PAGE_SIZE), 0UL); +} + +static void test_file_mapcounts(struct kunit *test, struct folio *folio, + const unsigned int expected[TEST_SLOTS], unsigned int native) +{ + struct mm_subpage_mapcounts counts; + unsigned int i, total = 0; + + for (i = 0; i < TEST_SLOTS; i++) + total += expected[i]; + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(&folio->page, &counts), total); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], expected[i]); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(total + native)); +} + +static void user4k_file_mapcounts_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *shared, *private, *native, *cvma; + struct mm_struct *child; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA, 3 * SZ_4K, SZ_4K); + private = test_file_vma(test, file, false, false, TEST_VA, 3 * SZ_4K, SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 2 }, 1); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA + 2 * SZ_4K, true) & + VM_FAULT_ERROR, 0U); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 1 }, 1); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, private->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 3, 3, 1 }, 1); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 3, 2, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, child); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, private->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 1, 1, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 1, 1, 1 }, 0); + kunit_release_action(test, user4k_vma_mm_free, shared->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 0, 0, 0 }, 0); +} + +/* A unique RAM-backed test inode: exercise generic file faults without + * making unsupported filesystem promotion part of this mapping fixture. + */ +static int test_coarse_read_folio(struct file *file, struct folio *folio) +{ + unsigned long i; + loff_t base = (loff_t)folio->index << PAGE_SHIFT; + loff_t size = i_size_read(file_inode(file)); + u8 *data = folio_address(folio); + + for (i = 0; i < folio_size(folio); i++) + data[i] = base + i < size ? 0x31 + ((base + i) / SZ_4K) : 0; + __folio_mark_uptodate(folio); + folio_unlock(folio); + return 0; +} + +static struct file *test_coarse_cache_file(struct kunit *test, unsigned int order, + unsigned long size) +{ + static const struct address_space_operations aops = { + .read_folio = test_coarse_read_folio, + .dirty_folio = filemap_dirty_folio, + }; + static const struct file_operations fops = { + .mmap = generic_file_mmap, + .read_iter = generic_file_read_iter, + .llseek = generic_file_llseek, + }; + struct file *file = anon_inode_create_getfile("user-coarse-file", &fops, + NULL, O_RDWR, NULL); + struct folio *folio; + int ret; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, test_file_put, file)) + return ERR_PTR(-ENOMEM); + file->f_mapping->a_ops = &aops; + mapping_set_large_folios(file->f_mapping); + i_size_write(file_inode(file), size); + folio = folio_alloc(GFP_KERNEL, order); + if (!folio) + return ERR_PTR(-ENOMEM); + ret = filemap_add_folio(file->f_mapping, folio, 0, GFP_KERNEL); + if (!ret) + test_coarse_read_folio(file, folio); /* Unlocks the installed folio. */ + folio_put(folio); + return ret ? ERR_PTR(ret) : file; +} + +static bool test_coarse_file_enabled(struct kunit *test) +{ + if (PAGE_SHIFT >= 16 || + (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { + kunit_skip(test, "requires internal larger user granule and borrowed-mm opt-in"); + return false; + } + return true; +} + +static void test_promote_unpin(void *data) +{ + struct user_page_fragment *fragment = data; + + if (fragment->folio) { + release_user_fragments(fragment, 1, true); + fragment->folio = NULL; + } +} + +static void user64k_file_promote_test(struct kunit *test) +{ + unsigned int pinned; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "cache promotion currently uses the collapse implementation"); + return; + } + for (pinned = 0; pinned < 2; pinned++) { + struct file *file = test_cache_file(test, 2 * SZ_64K, true); + struct vm_area_struct *coarse, *small; + struct user_page_fragment *fragment; + struct folio *folio; + phys_addr_t old[SZ_64K / PAGE_SIZE], untouched; + unsigned int nr = SZ_64K / PAGE_SIZE, i; + unsigned long offset, modified = SZ_64K / 2 + 19; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, SZ_64K); + small = test_file_vma(test, file, false, true, TEST_VA, SZ_64K, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, small); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + untouched = page_to_phys(&folio->page); + folio_put(folio); + for (i = 0; i < nr; i++) { + folio = filemap_get_folio(file->f_mapping, nr + i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + old[i] = page_to_phys(&folio->page); + folio_put(folio); + } + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + if (pinned) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_promote_unpin, + fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + TEST_VA + modified, 1, FOLL_WRITE, fragment, 1), 1L); + *(u8 *)(folio_address(fragment->folio) + fragment->offset) = 0xa7; + KUNIT_EXPECT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), -EBUSY); + /* A pin in the middle forces rollback of earlier isolated folios. */ + for (i = 0; i < nr; i++) { + folio = filemap_get_folio(file->f_mapping, nr + i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, page_to_phys(&folio->page), old[i]); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), (u8)0xa7); + kunit_release_action(test, test_promote_unpin, fragment); + } + KUNIT_ASSERT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), 0); + folio = filemap_get_folio(file->f_mapping, nr); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_order(folio), 16U - PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, folio->index, (pgoff_t)nr); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, (unsigned long)2 * nr); + for (offset = 0; offset < SZ_64K; offset++) { + u8 expected = pinned && offset == modified ? 0xa7 : 0x41 + offset / SZ_4K; + + if (((u8 *)folio_address(folio))[offset] != expected) { + KUNIT_FAIL(test, "promoted data mismatch at %lu", offset); + break; + } + } + /* An already suitable folio is an idempotent success, including with a GET. */ + KUNIT_EXPECT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), 0); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, coarse->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse->vm_mm, entry), page_to_phys(&folio->page)); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) { + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(&folio->page) + offset); + } + { + struct folio *first = filemap_get_folio(file->f_mapping, 0); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, first); + KUNIT_EXPECT_EQ(test, page_to_phys(&first->page), untouched); + folio_put(first); + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); + } +} + +static void user64k_shmem_fault_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 4; mode++) { + bool seeded = mode & 1, shared = mode < 2; + struct file *file = test_cache_file(test, SZ_64K, seeded); + struct vm_area_struct *vma; + struct folio *cache, *mapped; + unsigned long offset; + pte_t entry; + vm_fault_t fault; + unsigned int attempt; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, shared, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (attempt = 0; attempt < 8; attempt++) { + fault = test_file_fault(vma, TEST_VA, true); + if (!(fault & VM_FAULT_NOPAGE)) + break; + } + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + mapped = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, entry)))); + KUNIT_EXPECT_EQ(test, folio_size(mapped), SZ_64K); + KUNIT_EXPECT_EQ(test, !!folio_test_anon(mapped), !shared); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mapped) + offset, + seeded ? 0x31 + offset / SZ_4K : 0, SZ_4K), NULL); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, cache), 0); + KUNIT_EXPECT_EQ(test, folio_size(cache), SZ_64K); + if (shared) + KUNIT_EXPECT_PTR_EQ(test, mapped, cache); + else { + KUNIT_EXPECT_PTR_NE(test, mapped, cache); + memset(folio_address(mapped), 0xc9, SZ_64K); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + offset, + seeded ? 0x31 + offset / SZ_4K : 0, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, + shared ? MM_SHMEMPAGES : MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, SZ_64K / PAGE_SIZE); + { + u8 byte = 0x83, observed = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, TEST_VA + SZ_64K - 1, + &byte, 1, FOLL_WRITE), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, TEST_VA + SZ_64K - 1, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + KUNIT_EXPECT_EQ(test, ((u8 *)folio_address(cache))[SZ_64K - 1], + shared ? byte : (u8)(seeded ? 0x40 : 0)); + } + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(cache), (int)(SZ_64K / PAGE_SIZE) + 1); + } +} + +static void user64k_shmem_fault_busy_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 3; mode++) { + struct file *file = test_cache_file(test, SZ_64K, mode != 0); + struct vm_area_struct *vma; + struct folio *held; + vm_fault_t fault; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!mode) { + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_REMOTE | + FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT, NULL); + mmap_assert_locked(vma->vm_mm); + mmap_read_unlock(vma->vm_mm); + KUNIT_EXPECT_EQ(test, fault, (vm_fault_t)VM_FAULT_RETRY); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, 0UL); + } else if (mode == 1) { + held = filemap_get_folio(file->f_mapping, SZ_64K / (2 * PAGE_SIZE)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, held); + folio_lock(held); + fault = test_file_fault(vma, TEST_VA, false); + folio_unlock(held); + folio_put(held); + KUNIT_EXPECT_EQ(test, fault, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SHMEMPAGES), 0L); + } else { + struct vm_area_struct *small = test_file_vma(test, file, false, true, + TEST_VA, SZ_64K, 0); + struct user_page_fragment *fragment; + + KUNIT_ASSERT_NOT_NULL(test, small); + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_promote_unpin, + fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + TEST_VA + SZ_64K / 2, 1, FOLL_WRITE, fragment, 1), 1L); + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), (u8)0x39); + kunit_release_action(test, test_promote_unpin, fragment); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + } + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } +} + +static void user64k_shmem_fault_hole_test(struct kunit *test) +{ +#ifdef CONFIG_USERFAULTFD + unsigned int adjacent; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (adjacent = 0; adjacent < 2; adjacent++) { + struct file *file = test_cache_file(test, 2 * SZ_64K, false); + struct vm_area_struct *coarse, *missing, *seed; + struct folio *folio; + vm_fault_t fault; + bool new; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + missing = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, + adjacent ? SZ_64K : PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, missing); + seed = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, 0); + KUNIT_ASSERT_NOT_NULL(test, seed); + mmap_read_lock(seed->vm_mm); + folio = shmem_uffd_prepare_folio(seed, TEST_VA, &new); + mmap_read_unlock(seed->vm_mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, new); + memset(folio_address(folio), 0x6f, SZ_4K); + shmem_uffd_mark_present(folio, 0, BIT(0)); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), + GENMASK(PAGE_SIZE / SZ_4K - 1, 1)); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + /* Model the registration flag used by the collapse interval check. */ + mmap_write_lock(missing->vm_mm); + vma_start_write(missing); + vm_flags_set(missing, VM_UFFD_MISSING); + mmap_write_unlock(missing->vm_mm); + fault = test_file_fault(coarse, TEST_VA, false); + mmap_write_lock(missing->vm_mm); + vma_start_write(missing); + vm_flags_clear(missing, VM_UFFD_MISSING); + mmap_write_unlock(missing->vm_mm); + /* A real coarse fault may populate holes covered by its entire leaf. */ + KUNIT_EXPECT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 1, 0, SZ_4K)); + folio_unlock(folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x6f, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, + SZ_64K - SZ_4K), NULL); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, missing->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, seed->vm_mm); + } +#else + kunit_skip(test, "requires userfaultfd registration flags"); +#endif +} + +static void user64k_file_leaf_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *coarse, *native, *small, *child_vma; + struct mm_struct *coarse_mm, *child; + struct folio *folio; + unsigned long offset; + unsigned int nr = (2 * SZ_64K) / PAGE_SIZE; + unsigned int small_nr = 2 * SZ_64K / SZ_4K - 1; + unsigned int maps = 2 * nr + small_nr; + u8 byte = 0xc7, observed; + int ret; + + if (!test_coarse_file_enabled(test)) + return; + file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, 2 * SZ_64K, 0); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * SZ_64K, 0); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 2 * SZ_64K - SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + coarse_mm = coarse->vm_mm; + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + /* One cache folio supplies two user leaves, with native references per leaf. */ + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_64K) { + pte_t entry = test_fault_entry(test, coarse_mm, TEST_VA + offset); + + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse_mm, entry), + page_to_phys(&folio->page) + offset); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 2L); + for (offset = 0; offset < 2 * SZ_64K; offset += PAGE_SIZE) + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + for (offset = SZ_4K; offset < 2 * SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + KUNIT_ASSERT_EQ(test, access_remote_vm(coarse_mm, TEST_VA + SZ_64K + 3 * SZ_4K, + &byte, 1, FOLL_WRITE), 1); + KUNIT_ASSERT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + SZ_64K + 3 * SZ_4K, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + byte = 0x98; + KUNIT_ASSERT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 7 * SZ_4K, + &byte, 1, FOLL_WRITE), 1); + KUNIT_ASSERT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 7 * SZ_4K, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + KUNIT_EXPECT_EQ(test, ((u8 *)folio_address(folio))[7 * SZ_4K], byte); + + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, coarse_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Ordinary file VMAs are lazily populated in the child after fork. */ + mmap_read_lock(child); + child_vma = find_vma(child, TEST_VA); + mmap_read_unlock(child); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + maps += nr; + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + kthread_use_mm(coarse_mm); + ret = vm_munmap(TEST_VA, SZ_64K); + kthread_unuse_mm(coarse_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + maps -= SZ_64K / PAGE_SIZE; + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 1L); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, child, TEST_VA))); + + /* The native-rounded range must remove every overlapping coarse alias. */ + unmap_mapping_range(file->f_mapping, SZ_64K + SZ_4K, SZ_4K, 0); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, coarse_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, TEST_VA + SZ_64K + SZ_4K))); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + SZ_64K + PAGE_SIZE))); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 0L); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), + (int)(maps - 2 * SZ_64K / PAGE_SIZE - 1 - PAGE_SIZE / SZ_4K)); + + /* An exact subpage hole keeps neighboring small leaves in that native page. */ + mmap_read_lock(coarse_mm); + coarse = find_vma(coarse_mm, TEST_VA + SZ_64K); + mmap_read_unlock(coarse_mm); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < PAGE_SIZE; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + SZ_64K + offset, + false) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + unmap_mapping_subpage_hole(file->f_mapping, SZ_64K + SZ_4K, SZ_64K + 2 * SZ_4K); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, coarse_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA + SZ_64K))); + for (offset = 0; offset < PAGE_SIZE; offset += SZ_4K) + KUNIT_EXPECT_EQ(test, !!pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + SZ_64K + offset)), offset != SZ_4K); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(maps - 2 * SZ_64K / PAGE_SIZE - 2)); + kunit_release_action(test, user4k_vma_mm_free, coarse_mm); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); +} + +static void user64k_file_cow_test(struct kunit *test) +{ + unsigned int cold; + + if (!test_coarse_file_enabled(test)) + return; + for (cold = 0; cold < 2; cold++) { + struct file *file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + struct vm_area_struct *vma; + struct folio *cache, *anon; + unsigned long offset; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, false, TEST_VA, SZ_64K, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!cold) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + anon = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, entry)))); + KUNIT_EXPECT_TRUE(test, folio_test_anon(anon)); + KUNIT_EXPECT_EQ(test, folio_size(anon), SZ_64K); + folio_get(anon); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, anon), 0); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(anon) + offset, + 0x41 + offset / SZ_4K, SZ_4K), NULL); + memset(folio_address(anon), 0x93, SZ_64K); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + offset, + 0x31 + offset / SZ_4K, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + folio_put(cache); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_FILEPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(anon), (int)(SZ_64K / PAGE_SIZE)); + KUNIT_EXPECT_EQ(test, folio_ref_count(anon), (int)(SZ_64K / PAGE_SIZE) + 1); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(anon), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(anon), 1); + } +} + +static void user64k_file_bounds_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int mode; + unsigned long size = SZ_64K + PAGE_SIZE + 19; + pte_t entry; + + if (!test_coarse_file_enabled(test)) + return; + /* Too-small backing and a physically unaligned source cannot form a leaf. */ + for (mode = 0; mode < 2; mode++) { + file = test_coarse_cache_file(test, mode ? 17 - PAGE_SHIFT : 0, 2 * SZ_64K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, false, TEST_VA, SZ_64K, + mode ? PAGE_SIZE : 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } + file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, size); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, 3 * SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K, false) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_64K); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)) + + PAGE_SIZE + 19, 0, SZ_64K - PAGE_SIZE - 19), NULL); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA + 2 * SZ_64K, false) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_FILEPAGES), 2L); +} + +static void user4k_file_shared_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 4 * PAGE_SIZE, true); + struct vm_area_struct *small, *native; + struct folio *folio; + unsigned int i; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + native = test_file_vma(test, file, true, true, TEST_VA, 4 * PAGE_SIZE, 0); + /* Virtual and file offsets deliberately have different native alignment. */ + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 3 * PAGE_SIZE, 2 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + SZ_4K, false) & + VM_FAULT_ERROR, 0U); + /* Fault-around keeps its 64K window and populates all ready cache folios. */ + for (i = 0; i < 12; i++) { + unsigned long addr = small->vm_start + i * SZ_4K; + struct vm_page_offset pos = vma_page_offset_at(small, addr); + + entry = test_fault_entry(test, small->vm_mm, addr); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + folio = filemap_get_folio(file->f_mapping, pos.index); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(folio_file_page(folio, pos.index)) + pos.offset); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(small->vm_mm, entry)), + (u8)(0x33 + i)); + folio_put(folio); + } + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, native->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + memset(phys_to_virt(pte_phys_mm(native->vm_mm, entry)) + 2 * SZ_4K, 0x72, SZ_4K); + entry = test_fault_entry(test, small->vm_mm, small->vm_start); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(small->vm_mm, entry)), + 0x72, SZ_4K), NULL); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + SZ_4K, true) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, small->vm_start + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + memset(phys_to_virt(pte_phys_mm(small->vm_mm, entry)), 0x84, SZ_4K); + entry = test_fault_entry(test, native->vm_mm, TEST_VA); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(native->vm_mm, entry)) + + 3 * SZ_4K, 0x84, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + folio_put(folio); +} + +static void user4k_file_cow_test(struct kunit *test) +{ + unsigned int cold; + + for (cold = 0; cold < 2; cold++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *vma; + struct folio *cache; + pte_t entry; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, false, TEST_VA + SZ_4K, + 3 * SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!cold) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start + SZ_4K, true) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, vma->vm_start + SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + KUNIT_EXPECT_TRUE(test, folio_test_anon(page_folio(pte_page(entry)))); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, entry) & ~PAGE_MASK, 2UL * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), + 0x33, SZ_4K), NULL); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), 0x91, SZ_4K); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + folio_put(cache); + } +} + +static void user4k_file_eof_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, SZ_4K + 37, false); + struct vm_area_struct *small, *native; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, 0U); + for (i = 0; i < 4; i++) { + vm_fault_t ret = test_file_fault(small, TEST_VA + i * SZ_4K, false); + + KUNIT_EXPECT_EQ(test, ret & VM_FAULT_ERROR, i < 2 ? 0U : VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + i * SZ_4K)), i < 2); + } +} + + +static const struct vm_operations_struct test_filemap_fault_ops = { + .fault = filemap_fault, + .page_mkwrite = filemap_page_mkwrite, +}; + +static void user4k_filemap_fault_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * SZ_4K + 17, true); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + 3 * SZ_4K, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Exercise generic filemap_fault against an already populated cache. */ + vma->vm_ops = &test_filemap_fault_ops; + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start + SZ_4K, true) & VM_FAULT_ERROR, 0U); + for (i = 0; i < 3; i++) { + pte_t pte = test_fault_entry(test, vma->vm_mm, vma->vm_start + i * SZ_4K); + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), (u8)(0x31 + i)); + } + KUNIT_EXPECT_EQ(test, test_file_fault(vma, vma->vm_start + 3 * SZ_4K, false) & + VM_FAULT_ERROR, VM_FAULT_SIGBUS); +} + +static void user4k_file_truncate_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *small, *cow, *native; + struct iattr attr = { .ia_valid = ATTR_SIZE, .ia_size = SZ_4K + 37 }; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + PAGE_SIZE, SZ_4K); + cow = test_file_vma(test, file, false, false, TEST_VA, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, false, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, cow); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(cow, TEST_VA + 3 * SZ_4K, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + inode_lock(file_inode(file)); + ret = notify_change(&nop_mnt_idmap, file->f_path.dentry, &attr, NULL); + inode_unlock(file_inode(file)); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) { + unsigned long addr = small->vm_start + i * SZ_4K; + + if (i) + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, addr))); + KUNIT_EXPECT_EQ(test, test_file_fault(small, addr, false) & VM_FAULT_ERROR, + i ? VM_FAULT_SIGBUS : 0U); + } + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, cow->vm_mm, TEST_VA + 3 * SZ_4K))); + KUNIT_EXPECT_EQ(test, test_file_fault(cow, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, + VM_FAULT_SIGBUS); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA))); + KUNIT_EXPECT_EQ(test, test_file_fault(native, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, 0U); +} + +static void user4k_file_invalidate_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 4 * SZ_4K, PAGE_SIZE - SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & VM_FAULT_ERROR, 0U); + unmap_mapping_pages(file->f_mapping, 0, 1, false); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, vma->vm_mm, + vma->vm_start + i * SZ_4K)), i != 0); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & VM_FAULT_ERROR, 0U); + unmap_mapping_pages(file->f_mapping, 1, 1, false); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, vma->vm_mm, + vma->vm_start + i * SZ_4K)), i == 0); +} + +static void user4k_file_fork_hole_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *shared, *private, *cvma; + struct mm_struct *child; + struct inode *inode; + unsigned int i; + pte_t copied; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA, 2 * PAGE_SIZE, 0); + private = test_file_vma(test, file, false, false, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA, false) & VM_FAULT_ERROR, 0U); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, private->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + mmap_read_lock(child); + cvma = find_vma(child, TEST_VA); + mmap_read_unlock(child); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + copied = test_fault_entry(test, child, TEST_VA + SZ_4K); + memset(phys_to_virt(pte_phys_mm(child, copied)), 0x82, SZ_4K); + inode = file_inode(file); + /* A sub-native-page hole zeros shared bytes but retains private COW. */ + ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, SZ_4K, SZ_4K); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, i_size_read(inode), (loff_t)(2 * PAGE_SIZE)); + for (i = 0; i < 2 * TEST_SLOTS; i++) { + pte_t pte; + + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA + i * SZ_4K, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, shared->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(shared->vm_mm, pte)), + i == 1 ? 0 : 0x31 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, test_fault_entry(test, child, TEST_VA + SZ_4K)), + pte_phys_mm(child, copied)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(child, copied)), 0x82, SZ_4K), NULL); + /* A full cache-page hole exercises the file interval-tree invalidation. */ + ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, PAGE_SIZE, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = TEST_SLOTS; i < 2 * TEST_SLOTS; i++) { + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, shared->vm_mm, TEST_VA + i * SZ_4K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + i * SZ_4K))); + } +} + +static void test_zero_cow_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, TEST_VA, PAGE_SIZE); + struct mm_struct *mm; + struct mm_subpage *slot; + pte_t before[4], after; + unsigned int i; + vm_fault_t ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + for (i = 0; i < 4; i++) { + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + i * size, + FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + before[i] = test_fault_entry(test, mm, TEST_VA + i * size); + KUNIT_ASSERT_TRUE(test, pte_present(before[i])); + KUNIT_ASSERT_TRUE(test, is_zero_pfn(pte_pfn(before[i]))); + KUNIT_ASSERT_FALSE(test, pte_write(before[i])); + } + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + size + 7, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + lru_add_drain_all(); + for (i = 0; i < 4; i++) { + after = test_fault_entry(test, mm, TEST_VA + i * size); + if (i != 1) { + KUNIT_EXPECT_TRUE(test, pte_same(before[i], after)); + continue; + } + KUNIT_ASSERT_TRUE(test, pte_write(after)); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)size); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + size, 0, size), NULL); + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1UL); +} + +static void user4k_zero_cow_test(struct kunit *test) +{ + test_zero_cow_granule(test, 12); +} + +static void user16k_zero_cow_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_zero_cow_granule(test, 14); +} + +static void test_cow_reuse_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned int mode; + + for (mode = 0; mode < 2; mode++) { + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, TEST_VA, PAGE_SIZE); + struct mm_subpage *held = NULL; + struct mm_struct *child; + phys_addr_t original[4]; + unsigned int i; + vm_fault_t fault; + int ret; + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * size); + original[i] = pte_phys_mm(vma->vm_mm, pte); + memset(phys_to_virt(original[i]), 0xb0 + i, size); + } + if (mode) { + held = mm_subpage_get_from_phys(original[2]); + KUNIT_ASSERT_NOT_NULL(test, held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + } + child = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + for (i = 0; i < 4; i++) { + unsigned long address = TEST_VA + i * size; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte) == original[i], + !(mode && i == 2)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), + 0xb0 + i, size), NULL); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), 0xd0 + i, size); + } + if (held) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(mm_subpage_phys(held)), + 0xb2, size), NULL); + } +} + +static void user4k_cow_reuse_test(struct kunit *test) +{ + test_cow_reuse_granule(test, 12); +} + +static void user16k_cow_reuse_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_cow_reuse_granule(test, 14); +} + +static void test_fork_cow_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + struct vm_area_struct *parent = test_fault_vma_granule(test, shift, TEST_VA, 4 * size); + struct mm_struct *child = test_vma_mm_granule(test, shift); + struct vm_area_struct *cvma; + struct mm_subpage *held[4]; + pte_t before[4], after; + unsigned int i, side; + vm_fault_t fault; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + before[i] = test_fault_entry(test, parent->vm_mm, TEST_VA + i * size); + held[i] = mm_subpage_get_from_phys(pte_phys_mm(parent->vm_mm, before[i])); + KUNIT_ASSERT_NOT_NULL(test, held[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held[i]), 0); + memset(folio_address(mm_subpage_folio(held[i])) + mm_subpage_offset(held[i]), + 0x31 + i, size); + } + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + size + 15, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + for (i = 0; i < 4; i++) { + struct mm_subpage *slot; + bool changed = side && i == 1; + + after = test_fault_entry(test, mm, TEST_VA + i * size); + KUNIT_EXPECT_EQ(test, !!pte_write(after), changed); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after) != mm_subpage_phys(held[i]), changed); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)size); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0x31 + i, size), NULL); + if (changed) { + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); + memset(folio_address(mm_subpage_folio(slot)) + size, 0x77, size); + } + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 4UL); + } + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(held[i]), i == 1 ? 1 : 2); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(held[i])) + + mm_subpage_offset(held[i]), 0x31 + i, size), NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(held[0])), 11); +} + +static void user4k_fork_cow_test(struct kunit *test) +{ + test_fork_cow_granule(test, 12); +} + +static void user16k_fork_cow_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_fork_cow_granule(test, 14); +} + +static void user4k_folio_walk_geometry_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, PAGE_SHIFT, 14 }; + unsigned int which; + + for (which = 0; which < ARRAY_SIZE(shifts); which++) { + unsigned int shift = shifts[which], leaf; + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned long start, size = 1UL << shift; + vm_fault_t fault; + + if (which == 2 && PAGE_SHIFT <= 14) + continue; + mm = shift == PAGE_SHIFT ? mm_alloc() : test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shift == PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + start = pmd_size_mm(mm) - size; + vma = test_fault_vma_in_mm(mm, start, 2 * size); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (leaf = 0; leaf < 2; leaf++) { + unsigned long address = start + leaf * size; + struct folio_walk fw; + struct folio *folio; + struct page *page = NULL; + pte_t expected, walked = __pte(0); + bool zero = false; + + mmap_read_lock(mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address, 0); + if (folio) + folio_walk_end(&fw, vma); + mmap_read_unlock(mm); + KUNIT_EXPECT_PTR_EQ(test, folio, NULL); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address, FW_ZEROPAGE); + if (folio) { + zero = is_zero_folio(folio) && !fw.page && fw.level == FW_LEVEL_PTE; + folio_walk_end(&fw, vma); + } + mmap_read_unlock(mm); + KUNIT_EXPECT_TRUE(test, zero); + + mmap_read_lock(mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + expected = test_fault_entry(test, mm, address); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address + size - 1, 0); + if (folio) { + page = fw.page; + walked = fw.pte; + folio_walk_end(&fw, vma); + } + mmap_read_unlock(mm); + KUNIT_EXPECT_PTR_EQ(test, page, pfn_to_page(pte_pfn(expected))); + KUNIT_EXPECT_TRUE(test, pte_same(walked, expected)); + KUNIT_EXPECT_PTR_EQ(test, folio, page_folio(pfn_to_page(pte_pfn(expected)))); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_fragment_locked_pin_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, PAGE_SHIFT, 14 }; + unsigned int which; + + for (which = 0; which < ARRAY_SIZE(shifts); which++) { + unsigned int shift = shifts[which]; + struct mm_struct *mm; + struct vm_area_struct *vma, *found; + struct user_page_fragment fragment = { }; + unsigned long address; + int ret; + + if (which == 2 && PAGE_SHIFT <= 14) + continue; + mm = shift == PAGE_SHIFT ? mm_alloc() : test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shift == PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + vma = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + address = TEST_VA + (1UL << shift) - sizeof(long); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_LONGTERM, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOWAIT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + /* Only the first fragment is returned, while retaining the VMA lock. */ + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, 2 * sizeof(long), + FOLL_WRITE, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 1); + if (ret != 1) + continue; + KUNIT_EXPECT_PTR_EQ(test, found, vma); + KUNIT_EXPECT_TRUE(test, fragment.pinned); + KUNIT_EXPECT_EQ(test, fragment.length, (unsigned int)sizeof(long)); + KUNIT_EXPECT_EQ(test, !!fragment.subpage, shift != PAGE_SHIFT); + release_user_fragments(&fragment, 1, false); + /* With backing present, NOFAULT pins the exact same leaf successfully. */ + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 1); + if (ret == 1) { + KUNIT_EXPECT_PTR_EQ(test, found, vma); + release_user_fragments(&fragment, 1, true); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user16k_fragment_offset_test(struct kunit *test) +{ + struct vm_area_struct *vma; + struct user_page_fragment fragment = {}; + unsigned long address = TEST_VA + 3 * SZ_16K + SZ_4K + 13; + long ret; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative fragments require larger native backing"); + return; + } + vma = test_fault_vma_granule(test, 14, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + ret = pin_user_fragments_remote(vma->vm_mm, address, 31, FOLL_WRITE, &fragment, 1); + KUNIT_EXPECT_EQ(test, ret, 1L); + if (ret != 1) + return; + KUNIT_EXPECT_NOT_NULL(test, fragment.subpage); + KUNIT_EXPECT_EQ(test, fragment.offset, 3UL * SZ_16K + SZ_4K + 13); + KUNIT_EXPECT_EQ(test, fragment.length, 31U); + if (fragment.subpage) + KUNIT_EXPECT_EQ(test, mm_subpage_shift(fragment.subpage), 14U); + memset(folio_address(fragment.folio) + fragment.offset, 0x9c, fragment.length); + release_user_fragments(&fragment, 1, true); + { + u8 buf[33]; + + ret = access_remote_vm(vma->vm_mm, address - 1, buf, sizeof(buf), 0); + KUNIT_ASSERT_EQ(test, ret, (long)sizeof(buf)); + KUNIT_EXPECT_EQ(test, buf[0], (u8)0); + KUNIT_EXPECT_EQ(test, buf[32], (u8)0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 1, 0x9c, 31), NULL); + } +} + +static void user4k_anon_exit_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct mm_subpage *held; + struct anon_vma *root; + struct folio *folio; + struct mm_struct *mm; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + } + held = f->slots[0]; + folio = mm_subpage_folio(held); + root = f->vma->anon_vma->root; + mm_subpage_get(held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + mm_subpage_pool_close(f->pool); + mm_subpage_pool_put(f->pool); + /* Leave all four mapping refs and the tables to the real exit path. */ + mm = f->mm; + memset(f, 0, sizeof(*f)); + mmput(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(held), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); + for (i = 1; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_get_from_phys(mm_subpage_phys(held) + i * SZ_4K); + + KUNIT_EXPECT_PTR_EQ(test, slot, NULL); + if (slot) + mm_subpage_put(slot); + } +} + +#ifdef CONFIG_ARM64_CONTPTE +struct contpte_fixture { + struct mm_struct *mm; + struct folio *folio; + unsigned int nr; +}; + +static void contpte_fixture_free(void *arg) +{ + struct contpte_fixture *f = arg; + struct mmu_gather tlb; + unsigned int i; + + if (f->mm) { + mmap_write_lock(f->mm); + for (i = 0; i <= f->nr; i++) { + unsigned long addr = TEST_VA + i * mm_page_size(f->mm); + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, addr, &ptl); + + if (pte) { + pte_clear(f->mm, addr, pte); + pte_unmap_unlock(pte, ptl); + } + } + tlb_gather_mmu_fullmm(&tlb, f->mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(f->mm); + mmput(f->mm); + } + if (f->folio) + folio_put(f->folio); +} + +static void test_contpte_granule(struct kunit *test, bool small) +{ + struct contpte_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct vm_area_struct vma = {}; + struct test_mm tables = {}; + struct { + bool initial, addresses, hints, gathered, aged, access_all, unfolded; + bool permissions, refolded, holes, full_clear, sentinel; + } ok; + unsigned long size; + phys_addr_t phys; + pte_t base, saved, observed; + pte_t *pte; + spinlock_t *ptl; + unsigned int i; + int access_ret, access_same_ret, access_noop_ret; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, contpte_fixture_free, f), 0); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + if (small) + KUNIT_ASSERT_EQ(test, test_mm_select4k(f->mm), 0); + f->nr = cont_ptes_mm(f->mm); + size = mm_page_size(f->mm); + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, + get_order(cont_pte_size_mm(f->mm))); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + phys = PFN_PHYS(folio_pfn(f->folio)); + tables.mm = f->mm; + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, TEST_VA, phys), 0); + vma.vm_mm = f->mm; + base = phys_pte_mm(f->mm, phys, __pgprot(TEST_PROT | PTE_RDONLY)); + memset(&ok, 1, sizeof(ok)); + mmap_read_lock(f->mm); + pte = test_mm_lookup(f->mm, TEST_VA, &ptl); + if (!pte) { + mmap_read_unlock(f->mm); + KUNIT_FAIL(test, "missing PTE table"); + return; + } + pte_clear(f->mm, TEST_VA, pte); + /* The adjacent entry must never be included in this contiguous group. */ + saved = pte_mkspecial(base); + set_pte(pte + f->nr, saved); + set_ptes(f->mm, TEST_VA, pte, base, f->nr); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.initial &= pte_cont(observed); + ok.addresses &= pte_phys_mm(f->mm, observed) == phys + i * size; + ok.hints &= pte_batch_hint(pte + i, observed) == f->nr - i; + } + __set_pte(pte + 3, pte_mkdirty(__ptep_get(pte + 3))); + observed = ptep_get(pte + 1); + ok.gathered &= pte_dirty(observed) && pte_young(observed); + observed = ptep_get_lockless(pte + 2); + ok.gathered &= pte_dirty(observed) && pte_young(observed) && + pte_phys_mm(f->mm, observed) == phys + 2 * size; + clear_young_dirty_ptes(&vma, TEST_VA + 3 * size, pte + 3, 1, + CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.aged &= !pte_young(observed) && !pte_dirty(observed) && pte_cont(observed); + } + observed = pte_mkdirty(pte_mkyoung(__ptep_get(pte + 3))); + access_same_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, + pte + 3, observed, 1); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.access_all &= pte_cont(observed) && pte_young(observed) && + pte_dirty(observed) && !pte_write(observed); + } + access_noop_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, + pte + 3, ptep_get(pte + 3), 1); + observed = pte_mkwrite_novma(pte_mkdirty(pte_mkyoung(__ptep_get(pte + 3)))); + access_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, pte + 3, + observed, 1); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.unfolded &= !pte_cont(observed); + ok.permissions &= pte_write(observed) == (i == 3); + } + set_pte_at(f->mm, TEST_VA + 3 * size, pte + 3, + pte_advance_pfn_mm(f->mm, base, 3)); + /* Last-entry installation must fold using the physical byte offset too. */ + set_pte_at(f->mm, TEST_VA + (f->nr - 1) * size, pte + f->nr - 1, + pte_advance_pfn_mm(f->mm, base, f->nr - 1)); + for (i = 0; i < f->nr; i++) + ok.refolded &= pte_cont(__ptep_get(pte + i)); + get_and_clear_full_ptes(f->mm, TEST_VA + 3 * size, pte + 3, 2, 0); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.holes &= pte_none(observed) == (i == 3 || i == 4); + if (i != 3 && i != 4) + ok.holes &= !pte_cont(observed) && + pte_phys_mm(f->mm, observed) == phys + i * size; + } + set_ptes(f->mm, TEST_VA + 3 * size, pte + 3, + pte_advance_pfn_mm(f->mm, base, 3), 2); + set_pte_at(f->mm, TEST_VA + (f->nr - 1) * size, pte + f->nr - 1, + pte_advance_pfn_mm(f->mm, base, f->nr - 1)); + for (i = 0; i < f->nr; i++) + ok.refolded &= pte_cont(__ptep_get(pte + i)); + clear_full_ptes(f->mm, TEST_VA, pte, f->nr, 0); + for (i = 0; i < f->nr; i++) + ok.full_clear &= pte_none(__ptep_get(pte + i)); + ok.sentinel = pte_same(__ptep_get(pte + f->nr), saved); + pte_unmap_unlock(pte, ptl); + mmap_read_unlock(f->mm); + KUNIT_EXPECT_TRUE(test, ok.initial); + KUNIT_EXPECT_TRUE(test, ok.addresses); + KUNIT_EXPECT_TRUE(test, ok.hints); + KUNIT_EXPECT_TRUE(test, ok.gathered); + KUNIT_EXPECT_TRUE(test, ok.aged); + KUNIT_EXPECT_TRUE(test, ok.access_all); + KUNIT_EXPECT_EQ(test, access_same_ret, 1); + KUNIT_EXPECT_EQ(test, access_noop_ret, 0); + KUNIT_EXPECT_EQ(test, access_ret, 1); + KUNIT_EXPECT_TRUE(test, ok.unfolded); + KUNIT_EXPECT_TRUE(test, ok.permissions); + KUNIT_EXPECT_TRUE(test, ok.refolded); + KUNIT_EXPECT_TRUE(test, ok.holes); + KUNIT_EXPECT_TRUE(test, ok.full_clear); + KUNIT_EXPECT_TRUE(test, ok.sentinel); +} + +static void user4k_contpte_test(struct kunit *test) +{ + test_contpte_granule(test, true); +} + +static void native_contpte_test(struct kunit *test) +{ + test_contpte_granule(test, false); +} +#endif + +struct usercopy_worker { + struct test_mm *ctx[2]; + struct completion done; + unsigned int id; + unsigned int checks; + unsigned int idmaps; + int error; + u64 tcr; +}; + +static int test_usercopy_worker(void *arg) +{ + struct usercopy_worker *worker = arg; + u64 kernel_ttbr = read_sysreg(ttbr1_el1) & ~TTBRx_EL1_ASID_MASK; + unsigned int round, i, byte; + bool borrowed = false; + struct test_mm *ctx = NULL; + + for (round = 0; round < 16; round++) { + unsigned int small = (round + worker->id) & 1; + unsigned int cpu = cpumask_first(cpu_online_mask); + + ctx = worker->ctx[small]; + kthread_use_mm(ctx->mm); + borrowed = true; + if ((round + worker->id) & 2) { + unsigned int next = cpumask_next(cpu, cpu_online_mask); + + if (next < nr_cpu_ids) + cpu = next; + } + if (set_cpus_allowed_ptr(current, cpumask_of(cpu))) { + worker->error = 1; + break; + } + /* Exercise a real scheduler switch while borrowing this mm. */ + schedule_timeout_uninterruptible(1); + if (small && round < 4) { + unsigned long flags; + bool native; + + preempt_disable(); + flags = local_daif_save(); + if (round & 2) + cpu_install_ttbr0(phys_to_ttbr(__pa_symbol(idmap_pg_dir)), + TCR_T0SZ(IDMAP_VA_BITS)); + else + cpu_install_idmap(); + native = (read_sysreg(tcr_el1) & (TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK)) == + (TCR_TG0_16K | TCR_T0SZ(IDMAP_VA_BITS)); + cpu_uninstall_idmap(); + local_daif_restore(flags); + preempt_enable(); + if (!native) { + worker->error = 6; + break; + } + worker->idmaps++; + } + worker->tcr = read_sysreg(tcr_el1); + if ((worker->tcr & TCR_EL1_TG0_MASK) != (small ? TCR_TG0_4K : TCR_TG0_16K) || + (worker->tcr & TCR_EL1_T0SZ_MASK) != TCR_T0SZ(48) || + (worker->tcr & TCR_EL1_TG1_MASK) != TCR_TG1_16K || + (read_sysreg(ttbr1_el1) & ~TTBRx_EL1_ASID_MASK) != kernel_ttbr) { + worker->error = 2; + break; + } + for (i = 0; i < 4; i++) { + unsigned long offset = 64 + worker->id * sizeof(u64); + void __user *address = (void __user *)ctx->address[i]; + u64 value = 0xa5000000 | (worker->id << 16) | (round << 8) | i; + u64 *backing = phys_to_virt(ctx->phys[i]) + offset; + unsigned long read_left, write_left; + u8 data[8]; + + /* No VMA/fault path may be entered for these private mappings. */ + pagefault_disable(); + read_left = copy_from_user(data, address, sizeof(data)); + write_left = copy_to_user(address + offset, &value, sizeof(value)); + pagefault_enable(); + if (read_left != (i == 2 ? sizeof(data) : 0) || + write_left != (i == 0 || i == 2 ? sizeof(value) : 0)) { + worker->error = 3; + goto out; + } + if (i != 2) { + for (byte = 0; byte < sizeof(data); byte++) { + if (data[byte] != (small ? 0x40 : 0x80) + i) { + worker->error = 4; + goto out; + } + } + } + if (READ_ONCE(*backing) != (i == 0 || i == 2 ? 0 : value)) { + worker->error = 5; + goto out; + } + worker->checks++; + } + kthread_unuse_mm(ctx->mm); + borrowed = false; + } +out: + if (borrowed) + kthread_unuse_mm(ctx->mm); + complete(&worker->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void test_stop_kthread(void *arg) +{ + kthread_stop(arg); +} + +struct cow_reader { + struct mm_struct *parent; + struct mm_struct *child; + struct completion ready; + struct completion copied; + struct completion done; + u8 observed[8]; + unsigned long read_left[8], write_left[8]; + int error; + unsigned int checks; +}; + +static int test_cow_reader(void *arg) +{ + struct cow_reader *r = arg; + unsigned int i, side; + + kthread_use_mm(r->child); + /* Resolve fork's old PTEs normally, then cache pre-COW translations. */ + for (i = 0; i < 4; i++) { + u8 value = 0; + unsigned long left; + + left = copy_from_user(&value, (void __user *)(TEST_VA + i * SZ_4K + 64), 1); + if (left || value != 0x41 + i) + r->error = 1; + } + complete(&r->ready); + if (!wait_for_completion_timeout(&r->copied, 10 * HZ)) { + r->error = 2; + goto out; + } + for (side = 0; side < 2; side++) { + if (side) { + kthread_unuse_mm(r->child); + kthread_use_mm(r->parent); + } + for (i = 0; i < 4; i++) { + void __user *addr = (void __user *)(TEST_VA + i * SZ_4K + 64); + u8 value = 0, changed = 0x99; + unsigned long read_left, write_left; + + pagefault_disable(); + read_left = copy_from_user(&value, addr, 1); + write_left = copy_to_user(addr, &changed, 1); + pagefault_enable(); + r->observed[side * 4 + i] = value; + r->read_left[side * 4 + i] = read_left; + r->write_left[side * 4 + i] = write_left; + if (read_left || value != 0x41 + i || + write_left != (side == 0 && i == 1 ? 0 : 1)) + r->error = 3; + r->checks++; + } + } + kthread_unuse_mm(r->parent); + goto done; +out: + kthread_unuse_mm(r->child); +done: + complete(&r->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void user4k_fault_uaccess_test(struct kunit *test) +{ + struct vm_area_struct *parent, *child_vma; + struct mm_struct *child; + struct cow_reader *reader; + struct task_struct *task; + unsigned int i, cpu; + vm_fault_t fault; + int ret; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled || !system_supports_4kb_granule() || + system_supports_cnp() || cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires opt-in and compatible 4K TTBR0 controls"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + + memset(phys_to_virt(pte_phys_mm(parent->vm_mm, entry)), 0x41 + i, SZ_4K); + } + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + child_vma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + reader = kunit_kzalloc(test, sizeof(*reader), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, reader); + reader->parent = parent->vm_mm; + reader->child = child; + init_completion(&reader->ready); + init_completion(&reader->copied); + init_completion(&reader->done); + task = kthread_create(test_cow_reader, reader, "user4k-cow-reader"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + cpu = cpumask_last(cpu_online_mask); + kthread_bind(task, cpu); + wake_up_process(task); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&reader->ready, 10 * HZ), 0UL); + mmap_read_lock(child); + fault = handle_mm_fault(child_vma, TEST_VA + SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + complete(&reader->copied); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&reader->done, 10 * HZ), 0UL); + if (reader->error) + for (i = 0; i < 8; i++) + kunit_info(test, "access %u value=%#x read_left=%lu write_left=%lu\n", + i, reader->observed[i], reader->read_left[i], reader->write_left[i]); + KUNIT_EXPECT_EQ(test, reader->error, 0); + KUNIT_EXPECT_EQ(test, reader->checks, 8U); + { + pte_t entry = test_fault_entry(test, child, TEST_VA + SZ_4K); + + KUNIT_EXPECT_EQ(test, *(u8 *)(phys_to_virt(pte_phys_mm(child, entry)) + 64), (u8)0x99); + entry = test_fault_entry(test, parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, *(u8 *)(phys_to_virt(pte_phys_mm(parent->vm_mm, entry)) + 64), (u8)0x42); + } +} + +static void user4k_context_uaccess_test(struct kunit *test) +{ + struct test_mm *ctx[2]; + struct usercopy_worker *workers; + unsigned int i; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires 4K, no CnP/PIE/POE, DS=0, A1=1 and 48-bit TTBR0"); + return; + } + ctx[0] = test_usercopy_mm(test, false); + KUNIT_ASSERT_NOT_NULL(test, ctx[0]); + ctx[1] = test_usercopy_mm(test, true); + KUNIT_ASSERT_NOT_NULL(test, ctx[1]); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + struct task_struct *task; + + workers[i].ctx[0] = ctx[0]; + workers[i].ctx[1] = ctx[1]; + workers[i].id = i; + init_completion(&workers[i].done); + task = kthread_run(test_usercopy_worker, &workers[i], "user4k-copy-%u", i); + KUNIT_ASSERT_FALSE(test, IS_ERR(task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ_MSG(test, workers[i].error, 0, "worker %u TCR=%#llx", i, workers[i].tcr); + KUNIT_EXPECT_EQ(test, workers[i].checks, 64U); + KUNIT_EXPECT_EQ(test, workers[i].idmaps, 2U); + } + kunit_info(test, "two migrating kthreads, 32 borrowed-mm switches, %s PAN\n", + system_uses_ttbr0_pan() ? "software" : "hardware"); +} + +struct zap_reader { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready; + struct completion proceed; + struct completion done; + unsigned int cpu; + unsigned int checks; + bool zapped; + int error; +}; + +static int test_zap_reader(void *arg) +{ + struct zap_reader *reader = arg; + unsigned int phase, i; + + reader->error = set_cpus_allowed_ptr(current, cpumask_of(reader->cpu)); + if (reader->error) { + complete(&reader->ready); + goto out; + } + kthread_use_mm(reader->mm); + for (phase = 0; phase < 2; phase++) { + if (phase) { + complete(&reader->ready); + wait_for_completion(&reader->proceed); + if (!reader->zapped) + break; + } + for (i = 0; i < 4; i++) { + u64 value = 0, expected = 0xa5a50000UL + i; + unsigned long left; + + pagefault_disable(); + left = copy_from_user(&value, + (void __user *)((unsigned long)TEST_VA + i * SZ_4K), sizeof(value)); + pagefault_enable(); + if (phase && i == 1) { + if (!left) + reader->error = -EUCLEAN; + } else if (left || value != expected) { + reader->error = -EFAULT; + } + reader->checks++; + } + } + kthread_unuse_mm(reader->mm); +out: + complete(&reader->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void test_stop_zap_reader(void *arg) +{ + struct zap_reader *reader = arg; + + complete(&reader->proceed); + kthread_stop(reader->task); +} + +static void user4k_anon_remote_zap_test(struct kunit *test) +{ + struct anon_slot_fixture *f; + struct zap_reader *readers; + unsigned int i, cpu; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled || num_online_cpus() < 2) { + kunit_skip(test, "requires arm64.user4k_test=1 and two CPUs in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires 4K, no CnP/PIE/POE, DS=0, A1=1 and 48-bit TTBR0"); + return; + } + f = anon_slot_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + u64 value = 0xa5a50000UL + i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memcpy(folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot), + &value, sizeof(value)); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + } + readers = kunit_kcalloc(test, 2, sizeof(*readers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, readers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 2; i++) { + readers[i].mm = f->mm; + readers[i].cpu = cpu; + init_completion(&readers[i].ready); + init_completion(&readers[i].proceed); + init_completion(&readers[i].done); + readers[i].task = kthread_run(test_zap_reader, &readers[i], "user4k-zap-%u", i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, readers[i].task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_zap_reader, + &readers[i]), 0); + cpu = cpumask_next(cpu, cpu_online_mask); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].ready, 10 * HZ), 0UL); + KUNIT_ASSERT_EQ(test, readers[i].error, 0); + } + /* Keep an explicit non-mapping reference for fixture cleanup. */ + mm_subpage_get(f->slots[1]); + mmap_read_lock(f->mm); + zap_vma_range(f->vma, TEST_VA + SZ_4K, SZ_4K); + f->mapped[1] = false; + mmap_read_unlock(f->mm); + for (i = 0; i < 2; i++) { + readers[i].zapped = true; + complete(&readers[i].proceed); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, readers[i].error, 0); + KUNIT_EXPECT_EQ(test, readers[i].checks, 8U); + } +} + +static u64 *test_leaf(struct test_tables *tables, unsigned long va) +{ + return &tables->level[3][test_index(va, tables->shift, 3)]; +} + +/* Explicitly 4K, without the native PAGE_SHIFT or a granule/level hint. */ +static void test_invalidate_4k(unsigned long va, u64 ttbr1) +{ + u64 operand = (va >> USER4K_SHIFT) | (ttbr1 & GENMASK_ULL(63, 48)); + + dsb(ishst); + __tlbi(vae1, operand); + dsb(ish); + isb(); +} + +static void user4k_translation_test(struct kunit *test) +{ + struct test_tables small, native; + struct test_result *result; + unsigned long flags; + void *data, *other; + phys_addr_t data_pa, other_pa; + u64 old_tcr, old_ttbr0, old_ttbr1, old_par, tcr4k, tcr16k; + unsigned int round, slot; + + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE)) { + kunit_skip(test, "requires 4K, no CnP, and direct PTE permissions"); + return; + } + + KUNIT_ASSERT_EQ(test, PAGE_SHIFT, 14); + KUNIT_ASSERT_EQ(test, test_alloc_tables(test, &small, USER4K_SHIFT), 0); + KUNIT_ASSERT_EQ(test, test_alloc_tables(test, &native, PAGE_SHIFT), 0); + data = test_alloc_page(test); + other = test_alloc_page(test); + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, other); + data_pa = virt_to_phys(data); + other_pa = virt_to_phys(other); + result = kunit_kcalloc(test, TEST_ROUNDS, sizeof(*result), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, result); + + /* One native mapping and four independently described subpages. */ + *test_leaf(&native, TEST_VA) = other_pa | TEST_PROT; + *test_leaf(&small, TEST_VA) = data_pa | TEST_PROT | PTE_RDONLY; + *test_leaf(&small, TEST_VA + USER4K_SIZE) = + (data_pa + USER4K_SIZE) | TEST_PROT; + /* Slot 2 deliberately unmapped. Slot 3 aliases slot 0, writable. */ + *test_leaf(&small, TEST_VA + 3 * USER4K_SIZE) = data_pa | TEST_PROT; + + preempt_disable(); + flags = local_daif_save(); + old_tcr = read_sysreg(tcr_el1); + old_ttbr0 = read_sysreg(ttbr0_el1); + old_ttbr1 = read_sysreg(ttbr1_el1); + old_par = read_sysreg_par(); + /* DS affects both address ranges: never toggle it under a live TTBR1. */ + if ((old_tcr & (TCR_EL1_DS | TCR_EL1_TG1_MASK | TCR_EL1_A1)) != + (TCR_TG1_16K | TCR_EL1_A1)) { + local_daif_restore(flags); + preempt_enable(); + kunit_skip(test, "requires 16K TTBR1, DS=0, and A1=1"); + return; + } + tcr4k = (old_tcr & ~(TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK | + TCR_EL1_EPD0)) | TCR_TG0_4K | TCR_T0SZ(48); + tcr16k = (tcr4k & ~TCR_EL1_TG0_MASK) | TCR_TG0_16K; + + /* No allocations, logging, KUnit assertions or user access in this window. */ + for (round = 0; round < TEST_ROUNDS; round++) { + struct test_result *r = &result[round]; + + test_switch_ttbr0(virt_to_phys(native.level[0]), tcr16k); + for (slot = 0; slot < PROBE_SLOTS; slot++) + r->native[slot] = test_translate(TEST_VA + slot * USER4K_SIZE, false); + test_switch_ttbr0(virt_to_phys(small.level[0]), tcr4k); + r->tcr = read_sysreg(tcr_el1); + r->ttbr1 = read_sysreg(ttbr1_el1); + for (slot = 0; slot < PROBE_SLOTS; slot++) { + unsigned long va = TEST_VA + slot * USER4K_SIZE; + + r->read[slot] = test_translate(va, false); + r->write[slot] = test_translate(va, true); + } + + /* Break-before-make one 4K PTE and invalidate exactly that VA. */ + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), 0); + test_invalidate_4k(TEST_VA + USER4K_SIZE, old_ttbr1); + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), + (other_pa + USER4K_SIZE) | TEST_PROT | PTE_RDONLY); + dsb(ishst); + isb(); + for (slot = 0; slot < PROBE_SLOTS; slot++) + r->replaced[slot] = test_translate(TEST_VA + slot * USER4K_SIZE, true); + r->replaced_read = test_translate(TEST_VA + USER4K_SIZE, false); + asm volatile("at s1e1r, %0" : : "r" (data) : "memory"); + isb(); + r->kernel = read_sysreg_par(); + + test_switch_ttbr0(old_ttbr0, old_tcr); + r->restored_tcr = read_sysreg(tcr_el1); + r->restored_ttbr0 = read_sysreg(ttbr0_el1); + /* Private tables are inactive and their translations have been flushed. */ + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), + (data_pa + USER4K_SIZE) | TEST_PROT); + } + write_sysreg(old_par, par_el1); + isb(); + local_daif_restore(flags); + preempt_enable(); + + for (round = 0; round < TEST_ROUNDS; round++) { + struct test_result *r = &result[round]; + + KUNIT_EXPECT_EQ(test, r->tcr, tcr4k); + KUNIT_EXPECT_EQ(test, r->ttbr1, old_ttbr1); + KUNIT_EXPECT_EQ(test, r->restored_tcr, old_tcr); + KUNIT_EXPECT_EQ(test, r->restored_ttbr0, old_ttbr0); + KUNIT_EXPECT_EQ(test, r->kernel & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->kernel & SYS_PAR_EL1_PA, data_pa); + KUNIT_EXPECT_EQ(test, r->replaced_read & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->replaced_read & SYS_PAR_EL1_PA, + other_pa + USER4K_SIZE); + for (slot = 0; slot < PROBE_SLOTS; slot++) { + KUNIT_EXPECT_EQ(test, r->native[slot] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->native[slot] & SYS_PAR_EL1_PA, + other_pa + slot * USER4K_SIZE); + if (slot == 2) { + KUNIT_EXPECT_EQ(test, + FIELD_GET(SYS_PAR_EL1_FST, r->read[slot]), 7ULL); + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_F, 1ULL); + KUNIT_EXPECT_EQ(test, r->write[slot] & SYS_PAR_EL1_F, 1ULL); + KUNIT_EXPECT_EQ(test, r->replaced[slot] & SYS_PAR_EL1_F, 1ULL); + continue; + } + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_PA, + data_pa + (slot == 1 ? USER4K_SIZE : 0)); + KUNIT_EXPECT_EQ(test, r->write[slot] & SYS_PAR_EL1_F, + slot == 0 ? 1ULL : 0ULL); + KUNIT_EXPECT_EQ(test, r->replaced[slot] & SYS_PAR_EL1_F, + slot == 3 ? 0ULL : 1ULL); + } + } + kunit_info(test, "16K kernel, 4K EL0 translation/permissions; %u switch cycles\n", + TEST_ROUNDS); +} + +static struct folio *test_fault_folio(struct mm_struct *mm, unsigned long address) +{ + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, address, &ptl); + struct folio *folio = NULL; + + if (pte) { + pte_t entry = ptep_get(pte); + + if (pte_present(entry) && !pte_special(entry)) { + folio = page_folio(pfn_to_page(pte_pfn(entry))); + folio_get(folio); + } + pte_unmap_unlock(pte, ptl); + } + return folio; +} + +static bool test_reclaim_swapcache(swp_entry_t entry) +{ + unsigned int tries; + + for (tries = 0; tries < 100; tries++) { + struct folio *folio = swap_cache_get_folio(entry); + LIST_HEAD(folios); + + if (!folio) + return true; + folio_wait_writeback(folio); + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Only the swap cache and reclaim isolation may retain this folio. */ + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + return false; +} + +static void test_swap_tags_set(struct folio *folio) +{ +#ifdef CONFIG_ARM64_MTE + unsigned int offset; + + if (!system_supports_mte()) + return; + if (try_page_mte_tagging(&folio->page)) + mte_clear_page_tags(folio_address(folio)); + for (offset = 0; offset < PAGE_SIZE; offset += MTE_GRANULE_SIZE) { + unsigned long tag = (1UL + offset / SZ_4K) << MTE_TAG_SHIFT; + + asm volatile(".arch_extension memtag\n\tstg %0, [%1]" + : : "r" (tag), "r" (folio_address(folio) + offset) : "memory"); + } + set_page_mte_tagged(&folio->page); +#endif +} + +static bool test_swap_tags_match(struct folio *folio, unsigned int slot) +{ +#ifdef CONFIG_ARM64_MTE + unsigned int offset; + + if (!system_supports_mte()) + return true; + if (!page_mte_tagged(&folio->page)) + return false; + for (offset = slot * SZ_4K; offset < (slot + 1) * SZ_4K; offset += MTE_GRANULE_SIZE) { + unsigned long tag = 0; + + asm volatile(".arch_extension memtag\n\tldg %0, [%1]" + : "+r" (tag) : "r" (folio_address(folio) + offset) : "memory"); + if (((tag >> MTE_TAG_SHIFT) & 0xf) != ((slot + 1) & 0xf)) + return false; + } +#endif + return true; +} + +struct fragment_test_pins { + struct user_page_fragment fragments[2 * TEST_SLOTS]; + unsigned long count; +}; + +static void fragment_test_unpin(void *arg) +{ + struct fragment_test_pins *pins = arg; + + release_user_fragments(pins->fragments, pins->count, true); + pins->count = 0; +} + +static long fragment_test_pin(struct fragment_test_pins *pins, struct mm_struct *mm, + unsigned long start, unsigned long length, unsigned int flags, + unsigned long capacity) +{ + long ret; + + fragment_test_unpin(pins); + ret = pin_user_fragments_remote(mm, start, length, flags, pins->fragments, capacity); + pins->count = ret > 0 ? ret : 0; + return ret; +} + +static void user4k_fragment_pin_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + struct fragment_test_pins *pins; + struct vm_area_struct *vma; + struct mm_struct *mm, *child; + unsigned long size, total = 0; + unsigned int i, nr; + long ret; + + if (native) { + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + vma = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + } else { + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + } + size = mm_page_size(mm); + nr = PAGE_SIZE / size; + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, mm, TEST_VA, 1, FOLL_WRITE | FOLL_NOFAULT, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, TEST_VA + 123, PAGE_SIZE - 200, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + unsigned int offset = i ? i * size : 123; + unsigned int length = size - (!i ? 123 : 0) - (i == nr - 1 ? 77 : 0); + + KUNIT_EXPECT_EQ(test, f->offset, (unsigned long)offset); + KUNIT_EXPECT_EQ(test, f->length, length); + KUNIT_EXPECT_EQ(test, !!f->subpage, !native); + memset(folio_address(f->folio) + f->offset, 0x71 + i, f->length); + total += f->length; + } + KUNIT_EXPECT_EQ(test, total, PAGE_SIZE - 200); + fragment_test_unpin(pins); + child = native ? mm_alloc() : test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, child), 0); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + /* Ordinary references retain shared COW backing, as native FOLL_GET does. */ + ret = get_user_fragments_remote(child, TEST_VA + 123, PAGE_SIZE - 200, + 0, pins->fragments, ARRAY_SIZE(pins->fragments)); + pins->count = ret > 0 ? ret : 0; + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + pte_t parent_pte = test_fault_entry(test, mm, TEST_VA + i * size); + pte_t child_pte = test_fault_entry(test, child, TEST_VA + i * size); + + KUNIT_EXPECT_FALSE(test, f->pinned); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, parent_pte), pte_phys_mm(child, child_pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x71 + i, f->length), NULL); + } + fragment_test_unpin(pins); + /* Read pins must unshare COW mappings without enabling PTE writes. */ + ret = fragment_test_pin(pins, child, TEST_VA + 123, PAGE_SIZE - 200, FOLL_NOFAULT, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, child, TEST_VA + 123, PAGE_SIZE - 200, 0, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + pte_t pte = test_fault_entry(test, child, TEST_VA + i * size); + + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x71 + i, f->length), NULL); + if (!native) { + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(f->subpage)); + if (i) + KUNIT_EXPECT_PTR_NE(test, f->folio, pins->fragments[0].folio); + } + } + fragment_test_unpin(pins); + /* Return a valid partial result when the byte range reaches an unmapped VMA. */ + ret = fragment_test_pin(pins, mm, TEST_VA, PAGE_SIZE + size, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, (long)nr); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 1); + KUNIT_EXPECT_EQ(test, ret, 1L); + KUNIT_EXPECT_EQ(test, pins->fragments[0].length, (unsigned int)size); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, TEST_VA, size, FOLL_LONGTERM, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, 1L); + ret = fragment_test_pin(pins, mm, ULONG_MAX - 15, ULONG_MAX, 0, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +#ifdef CONFIG_BLOCK +struct bio_fragment_test { + struct bio *bio; + struct bio *clone; + struct user_page_fragment held[TEST_SLOTS]; + unsigned int count; +}; + +static void bio_fragment_test_free(void *arg) +{ + struct bio_fragment_test *state = arg; + + if (state->clone) + bio_put(state->clone); + if (state->bio) { + bio_release_pages(state->bio, false); + bio_put(state->bio); + } + release_user_fragments(state->held, state->count, false); +} + +static void user4k_bio_fragments_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct bio_fragment_test *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct iov_iter iter; + unsigned int i, bytes = round_down(PAGE_SIZE - 200, 512); + long nr; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bio fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, bio_fragment_test_free, state), 0); + /* Populate real writable anonymous mappings and retain independent GETs. */ + nr = get_user_fragments_remote(vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, + state->held, ARRAY_SIZE(state->held)); + state->count = nr > 0 ? nr : 0; + KUNIT_ASSERT_EQ(test, nr, (long)TEST_SLOTS); + state->bio = bio_alloc(NULL, 8, REQ_OP_READ, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->bio); + iov_iter_ubuf(&iter, ITER_DEST, (void __user *)(TEST_VA + 123), PAGE_SIZE - 200); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_vecs_to_alloc(&iter, BIO_MAX_VECS); + kthread_unuse_mm(vma->vm_mm); + KUNIT_EXPECT_EQ(test, ret, (int)TEST_SLOTS); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_iter_get_pages(state->bio, &iter, 511); + kthread_unuse_mm(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, state->bio->bi_iter.bi_size, bytes); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), PAGE_SIZE - 200 - bytes); + KUNIT_EXPECT_EQ(test, state->bio->bi_nr_user_fragments, (int)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, state->bio->bi_vcnt, 1); + KUNIT_EXPECT_EQ(test, state->bio->bi_io_vec[0].bv_offset, 123U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 1U); + state->clone = bio_alloc_clone(NULL, state->bio, GFP_KERNEL, &fs_bio_set); + KUNIT_ASSERT_NOT_NULL(test, state->clone); + KUNIT_EXPECT_PTR_EQ(test, state->clone->bi_user_fragments, NULL); + KUNIT_EXPECT_PTR_EQ(test, state->clone->bi_io_vec, state->bio->bi_io_vec); + /* Completion ownership must outlive all originating VMAs. */ + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 1U); + bio_put(state->clone); + state->clone = NULL; + bio_release_pages(state->bio, false); + KUNIT_EXPECT_PTR_EQ(test, state->bio->bi_user_fragments, NULL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 0U); +} + +static void user4k_bio_trim_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, SZ_4K); + struct bio_fragment_test *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct iov_iter iter; + unsigned int i; + long nr; + int ret, refs; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bio fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, bio_fragment_test_free, state), 0); + nr = get_user_fragments_remote(vma->vm_mm, TEST_VA, SZ_4K, FOLL_WRITE, state->held, 1); + state->count = nr > 0 ? nr : 0; + KUNIT_ASSERT_EQ(test, nr, 1L); + refs = folio_ref_count(state->held[0].folio); + state->bio = bio_alloc(NULL, 4, REQ_OP_READ, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->bio); + for (i = 0; i < 1024; i++) { + iov_iter_ubuf(&iter, ITER_DEST, (void __user *)(TEST_VA + SZ_4K - 256), 512); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_iter_get_pages(state->bio, &iter, 511); + kthread_unuse_mm(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 512UL); + KUNIT_EXPECT_EQ(test, state->bio->bi_vcnt, 0); + KUNIT_EXPECT_PTR_EQ(test, state->bio->bi_user_fragments, NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[0].subpage), 0U); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->held[0].folio), refs); + bio_reset(state->bio, NULL, REQ_OP_READ); + } +} +#endif + +static void user4k_fragment_longterm_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct fragment_test_pins *pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + phys_addr_t before; + bool movable; + unsigned int i; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)TEST_SLOTS); + before = PFN_PHYS(folio_pfn(pins->fragments[0].folio)); + movable = !folio_is_longterm_pinnable(pins->fragments[0].folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0x91 + i, pins->fragments[i].length); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, + FOLL_WRITE | FOLL_LONGTERM, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_TRUE(test, folio_is_longterm_pinnable(f->folio)); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(f->subpage), 1U); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x91 + i, f->length), NULL); + } + if (movable) + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(pins->fragments[0].folio)), before); + kunit_info(test, "long-term source movable=%u migrated=%u\n", movable, + PFN_PHYS(folio_pfn(pins->fragments[0].folio)) != before); + fragment_test_unpin(pins); + /* A long-term flag must never silently create an ordinary GET. */ + ret = get_user_fragments_remote(vma->vm_mm, TEST_VA, SZ_4K, FOLL_LONGTERM, + pins->fragments, 1); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void user4k_file_fragments_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *shared, *private; + struct fragment_test_pins *pins; + phys_addr_t shared_phys; + long ret; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, SZ_4K); + private = test_file_vma(test, file, false, false, TEST_VA + SZ_4K, 3 * SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, shared->vm_mm, shared->vm_start + 17, + 3 * SZ_4K - 33, FOLL_WRITE, 8); + KUNIT_ASSERT_EQ(test, ret, 3L); + shared_phys = PFN_PHYS(folio_pfn(pins->fragments[0].folio)); + for (i = 0; i < 3; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_PTR_EQ(test, f->subpage, NULL); + KUNIT_EXPECT_EQ(test, f->offset, (unsigned long)((i + 1) * SZ_4K + (!i ? 17 : 0))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x32 + i, f->length), NULL); + } + /* File pins stay live across PTE removal and mm exit. */ + kunit_release_action(test, user4k_vma_mm_free, shared->vm_mm); + for (i = 0; i < 3; i++) + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(pins->fragments[i].folio)); + fragment_test_unpin(pins); + ret = get_user_fragments_remote(private->vm_mm, private->vm_start, 3 * SZ_4K, + 0, pins->fragments, 8); + pins->count = ret > 0 ? ret : 0; + KUNIT_ASSERT_EQ(test, ret, 3L); + for (i = 0; i < 3; i++) { + KUNIT_EXPECT_PTR_EQ(test, pins->fragments[i].subpage, NULL); + KUNIT_EXPECT_EQ(test, PFN_PHYS(folio_pfn(pins->fragments[i].folio)), shared_phys); + } + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, private->vm_mm, private->vm_start, 3 * SZ_4K, 0, 8); + KUNIT_ASSERT_EQ(test, ret, 3L); + for (i = 0; i < 3; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_NOT_NULL(test, f->subpage); + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(f->folio)), shared_phys); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x32 + i, f->length), NULL); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, private->vm_mm, + private->vm_start + i * SZ_4K))); + } + fragment_test_unpin(pins); +} + +/* Exercise the same implementation used by the arm64 syscall entry. */ +asmlinkage long __arm64_sys_mprotect(const struct pt_regs *regs); + +static long test_mprotect(unsigned long start, unsigned long len, unsigned long prot) +{ + struct pt_regs regs = { .regs = { start, len, prot } }; + + return __arm64_sys_mprotect(®s); +} + +static unsigned long test_copy_word(unsigned long addr, u64 *value, bool write) +{ + unsigned long left; + + pagefault_disable(); + if (write) + left = copy_to_user((void __user *)addr, value, sizeof(*value)); + else + left = copy_from_user(value, (void __user *)addr, sizeof(*value)); + pagefault_enable(); + return left; +} + +static void user4k_mprotect_test(struct kunit *test) +{ + const unsigned long starts[] = { + TEST_VA, SZ_2M - SZ_4K, SZ_1G - SZ_4K, BIT(39) - SZ_4K, TEST_VA, + }; + unsigned int shape; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + for (shape = 0; shape < ARRAY_SIZE(starts); shape++) { + struct file *file = shape == 4 ? test_cache_file(test, PAGE_SIZE, true) : NULL; + struct mm_struct *mm = test_vma_mm(test); + unsigned long start = starts[shape], mapped; + unsigned long flags = MAP_FIXED_NOREPLACE | (file ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS); + phys_addr_t before[4] = { }; + long protect[4] = { }; + unsigned long initial = 0, final = 0, read_ro = 0, write_ro = 0, read_none = 0; + u64 value, ro_value = 0; + unsigned int i; + vm_fault_t fault = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shape == 4) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + kthread_use_mm(mm); + mapped = vm_mmap(file, start, PAGE_SIZE, PROT_READ | PROT_WRITE, flags, 0); + if (IS_ERR_VALUE(mapped)) + goto detach; + for (i = 0; i < 4; i++) { + struct vm_area_struct *vma = test_vma_lookup(mm, start + i * SZ_4K); + + fault |= test_file_fault(vma, start + i * SZ_4K, true); + if (fault & VM_FAULT_ERROR) + goto detach; + before[i] = pte_phys_mm(mm, test_fault_entry(test, mm, start + i * SZ_4K)); + value = 0x88110000 + i; + initial |= test_copy_word(start + i * SZ_4K, &value, true); + } + /* Keep the mm attached across permission changes and cached accesses. */ + protect[0] = test_mprotect(start + SZ_4K, SZ_4K, PROT_READ); + value = 0xaa; + write_ro = test_copy_word(start + SZ_4K, &value, true); + final |= test_copy_word(start, &value, true); + final |= test_copy_word(start + 3 * SZ_4K, &value, true); + protect[1] = test_mprotect(start + 2 * SZ_4K, SZ_4K, PROT_NONE); + read_none = test_copy_word(start + 2 * SZ_4K, &value, false); + read_ro = test_copy_word(start + SZ_4K, &ro_value, false); + protect[2] = test_mprotect(start, PAGE_SIZE, PROT_READ); + protect[3] = test_mprotect(start, PAGE_SIZE, PROT_READ | PROT_WRITE); + for (i = 0; i < 4; i++) { + value = 0x99220000 + i; + final |= test_copy_word(start + i * SZ_4K, &value, true); + } +detach: + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, initial, 0UL); + KUNIT_EXPECT_EQ(test, final, 0UL); + KUNIT_EXPECT_EQ(test, write_ro, (unsigned long)sizeof(u64)); + KUNIT_EXPECT_EQ(test, read_none, (unsigned long)sizeof(u64)); + KUNIT_EXPECT_EQ(test, read_ro, 0UL); + KUNIT_EXPECT_EQ(test, ro_value, 0x88110001ULL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, start + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, protect[i], 0L); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), before[i]); + } + KUNIT_EXPECT_EQ(test, mm->total_vm, (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm->map_count, 1); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +#ifdef CONFIG_NUMA_BALANCING +static void user_subpage_numa_cow_batch_test(struct kunit *test) +{ + unsigned int shift; + + if (!(READ_ONCE(sysctl_numa_balancing_mode) & NUMA_BALANCING_NORMAL)) { + kunit_skip(test, "requires normal NUMA balancing; use numa_balancing=enable"); + return; + } + for (shift = 12; shift < PAGE_SHIFT; shift += 2) { + unsigned long size = 1UL << shift, start = TEST_VA; + struct vm_area_struct *vma; + struct mm_struct *child = test_vma_mm_granule(test, shift); + struct mm_struct *mm = test_vma_mm_granule(test, shift); + phys_addr_t before; + unsigned long marked, mapped; + long ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + /* Use mmap accounting because the child will partially unmap. */ + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, 4 * size, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + i * size, true), + (vm_fault_t)0); + before = pte_phys_mm(mm, test_fault_entry(test, mm, start + size)); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + kthread_use_mm(child); + ret = vm_munmap(start + size, size); + kthread_unuse_mm(child); + KUNIT_ASSERT_EQ(test, ret, 0L); + /* Reuse the now-private slot without replacing its native owner. */ + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + size, true), (vm_fault_t)0); + KUNIT_ASSERT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + size)), before); + kthread_use_mm(mm); + ret = test_mprotect(start, 4 * size, PROT_READ); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Permit local-node hinting, which normally skips a single user. */ + mmget(mm); + mmap_read_lock(mm); + marked = change_prot_numa(vma, start, start + 4 * size); + mmap_read_unlock(mm); + mmput(mm); + KUNIT_EXPECT_EQ(test, marked, 1UL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, start + i * size); + + KUNIT_EXPECT_EQ(test, !!pte_protnone(pte), i == 1); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + } + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} +#endif + +static void user4k_mprotect_cow_batch_test(struct kunit *test) +{ + struct mm_struct *mm, *child; + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + phys_addr_t before[4]; + unsigned int i; + unsigned long mapped; + long ret[2]; + int copied; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, fragment_test_pin(pins, mm, TEST_VA + SZ_4K, SZ_4K, FOLL_WRITE, 8), 1L); + for (i = 0; i < 4; i++) + before[i] = pte_phys_mm(mm, test_fault_entry(test, mm, TEST_VA + i * SZ_4K)); + uprobe_start_dup_mmap(); + copied = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, copied, 0); + kthread_use_mm(mm); + ret[0] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + ret[1] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret[0], 0L); + KUNIT_EXPECT_EQ(test, ret[1], 0L); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), before[i]); + /* One native folio contains both exclusive and shared COW slots. */ + KUNIT_EXPECT_EQ(test, pte_write(pte), i == 1); + } + fragment_test_unpin(pins); +} + +static void user4k_mprotect_accounting_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + unsigned long mapped, committed[4], data[4]; + s64 global[4], before; + long ret[4]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + if (!IS_ERR_VALUE(mapped)) { + ret[0] = test_mprotect(TEST_VA + SZ_4K, SZ_4K, PROT_READ); + committed[0] = mm->committed_user_pages; + data[0] = mm->data_vm; + global[0] = percpu_counter_sum(&vm_committed_as) - before; + ret[1] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + committed[1] = mm->committed_user_pages; + data[1] = mm->data_vm; + global[1] = percpu_counter_sum(&vm_committed_as) - before; + ret[2] = test_mprotect(TEST_VA + 2 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE); + committed[2] = mm->committed_user_pages; + data[2] = mm->data_vm; + global[2] = percpu_counter_sum(&vm_committed_as) - before; + ret[3] = vm_munmap(TEST_VA, PAGE_SIZE); + committed[3] = mm->committed_user_pages; + data[3] = mm->data_vm; + global[3] = percpu_counter_sum(&vm_committed_as) - before; + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + for (i = 0; i < 4; i++) { + unsigned long expected = i == 0 ? TEST_SLOTS - 1 : i == 2 ? 1 : 0; + + KUNIT_EXPECT_EQ(test, ret[i], 0L); + KUNIT_EXPECT_EQ(test, committed[i], expected); + KUNIT_EXPECT_EQ(test, data[i], expected); + KUNIT_EXPECT_EQ(test, global[i], expected ? 1LL : 0LL); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 0); +} + +static void user4k_fragment_permissions_test(struct kunit *test) +{ + unsigned long start = TEST_VA + SZ_4K; + struct vm_area_struct *vma = test_fault_vma(test, start, SZ_4K); + struct fragment_test_pins *pins; + struct user_page_fragment *fragment; + struct mm_struct *mm; + spinlock_t *ptl; + pte_t *ptep; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + fragment = &pins->fragments[0]; + { + struct page *page = NULL; + int locked = 1; + + mmap_read_lock(mm); + ret = pin_user_pages_remote(mm, start, 1, 0, &page, &locked); + if (locked) + mmap_read_unlock(mm); + if (ret == 1) + unpin_user_page(page); + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + } + ret = fragment_test_pin(pins, mm, start + 17, 31, 0, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_TRUE(test, is_zero_folio(fragment->folio)); + KUNIT_EXPECT_PTR_EQ(test, fragment->subpage, NULL); + /* Zero-page PTEs alias the first physical 4K despite this VA's slot 1. */ + KUNIT_EXPECT_EQ(test, fragment->offset, 17UL); + KUNIT_EXPECT_EQ(test, fragment->length, 31U); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment->folio) + fragment->offset, + 0, fragment->length), NULL); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_NOT_NULL(test, fragment->subpage); + KUNIT_EXPECT_EQ(test, fragment->offset, SZ_4K + 17UL); + memset(folio_address(fragment->folio) + fragment->offset, 0xa4, fragment->length); + fragment_test_unpin(pins); + mmap_write_lock(mm); + vm_flags_clear(vma, VM_WRITE); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + ptep = test_mm_lookup(mm, start, &ptl); + if (ptep) { + ptep_set_wrprotect(mm, start, ptep); + pte_unmap_unlock(ptep, ptl); + } + flush_tlb_range(vma, start, start + SZ_4K); + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE, 8); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE | FOLL_FORCE, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, mm, start))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment->folio) + fragment->offset, + 0xa4, fragment->length), NULL); + fragment_test_unpin(pins); + mmap_write_lock(mm); + vm_flags_clear(vma, VM_READ); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_unlock(mm); + ret = fragment_test_pin(pins, mm, start + 17, 31, 0, 8); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_FORCE, 8); + KUNIT_EXPECT_EQ(test, ret, 1L); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, start, 1, FOLL_GET, 8); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void test_subpage_unpin(void *arg) +{ + mm_subpage_unpin(arg); +} + +static void user4k_slot_pin_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *child; + struct mm_subpage *slots[4]; + struct folio *folio, *dst; + unsigned int i; + vm_fault_t fault; + spinlock_t *ptl; + pte_t *ptep, pte; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + slots[i] = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slots[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slots[i]), 0); + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(slots[i])); + memset(folio_address(mm_subpage_folio(slots[i])) + i * SZ_4K, 0x61 + i, SZ_4K); + } + folio = mm_subpage_folio(slots[1]); + test_swap_tags_set(folio); + for (i = 0; i < 2; i++) { + ptep = test_mm_lookup(vma->vm_mm, TEST_VA + SZ_4K, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = mm_subpage_pin(slots[1]); + pte_unmap_unlock(ptep, ptl); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_unpin, slots[1]), 0); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 2U); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_FALSE(test, mm_subpage_try_share(slots[1])); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + /* Finish even unexpected success so a failing assertion cannot leak metadata. */ + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + KUNIT_EXPECT_EQ(test, ret, -EBUSY); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) { + struct mm_subpage *copy; + + pte = test_fault_entry(test, child, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, pte) == mm_subpage_phys(slots[i]), i != 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), i == 1 ? 1 : 2); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(slots[i]), i == 1); + copy = mm_subpage_get_from_phys(pte_phys_mm(child, pte)); + KUNIT_ASSERT_NOT_NULL(test, copy); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(copy), i == 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(copy)) + i * SZ_4K, + 0x61 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(copy), i)); + mm_subpage_put(copy); + } + /* A read pin on a shared slot must request unsharing, not pin shared COW data. */ + ptep = test_mm_lookup(vma->vm_mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = mm_subpage_pin(slots[0]); + pte_unmap_unlock(ptep, ptl); + if (!ret) + mm_subpage_unpin(slots[0]); + KUNIT_EXPECT_EQ(test, ret, -EAGAIN); + /* Re-enabling writes must retain the exact backing of an exclusive pinned slot. */ + ptep = test_mm_lookup(vma->vm_mm, TEST_VA + SZ_4K, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ptep_set_wrprotect(vma->vm_mm, TEST_VA + SZ_4K, ptep); + pte_unmap_unlock(ptep, ptl); + flush_tlb_range(vma, TEST_VA + SZ_4K, TEST_VA + 2 * SZ_4K); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA + SZ_4K, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), mm_subpage_phys(slots[1])); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + *((u8 *)folio_address(folio) + SZ_4K) = 0xab; + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(child, pte)), (u8)0x62); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[1]), 0); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 2U); + KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + SZ_4K), (u8)0xab); + kunit_release_action(test, test_subpage_unpin, slots[1]); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 1U); + kunit_release_action(test, test_subpage_unpin, slots[1]); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 0U); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); +} + +#ifdef CONFIG_MIGRATION +static struct folio *test_migration_alloc(struct folio *src, unsigned long private) +{ + return folio_alloc(GFP_HIGHUSER_MOVABLE, 0); +} + +asmlinkage long __arm64_sys_madvise(const struct pt_regs *regs); + +static long test_madvise(unsigned long addr, unsigned long size, unsigned long behavior) +{ + struct pt_regs regs = { .regs = { addr, size, behavior } }; + + return __arm64_sys_madvise(®s); +} + +static void user_granule_large_vma_split_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14 }, i; + + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) || + (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { + kunit_skip(test, "requires THP boundary splitting and borrowed-mm execution"); + return; + } + for (i = 0; i < ARRAY_SIZE(shifts); i++) { + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long size = 1UL << shifts[i]; + unsigned long base = 4UL * HPAGE_PMD_SIZE, mapped; + pte_t pte; + long ret; + + if (shifts[i] >= PAGE_SHIFT) + continue; + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shifts[i]), 0); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, base, 2UL * HPAGE_PMD_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE | MAP_NORESERVE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, base); + vma = test_vma_lookup(mm, base); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, base, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, base + size, true) & VM_FAULT_ERROR, 0U); + /* The generic VMA split probes the native THP boundary for this range. */ + kthread_use_mm(mm); + ret = test_mprotect(base + size, size, PROT_READ); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, mm, base + size); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + pte = test_fault_entry(test, mm, base); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + kthread_use_mm(mm); + ret = test_mprotect(base + size, size, PROT_READ | PROT_WRITE); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0L); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user_granule_brk_extent_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int count = PAGE_SHIFT > 14 ? 3 : 2, i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm brk fixture requires hardware opt-in"); + return; + } + for (i = 0; i < count; i++) { + struct mm_struct *mm = mm_alloc(); + struct vm_area_struct *vma; + unsigned long size = 1UL << shifts[i], neighbor; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shifts[i]), 0); + kthread_use_mm(mm); + ret = vm_brk_flags(TEST_VA, size + 1, false); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_start, TEST_VA); + KUNIT_EXPECT_EQ(test, vma->vm_end, TEST_VA + 2 * size); + KUNIT_EXPECT_EQ(test, mm->total_vm, 2UL); + /* FEX reserves exactly this page after its initial ELF/BSS break. */ + kthread_use_mm(mm); + neighbor = vm_mmap(NULL, TEST_VA + 2 * size, size, PROT_NONE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, neighbor, TEST_VA + 2 * size); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_droppable_reclaim_test(struct kunit *test) +{ + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm droppable fixture requires hardware opt-in"); + return; + } + /* Dirty discard, retained GET, retained PIN, migration, zero-page COW. */ + for (mode = 0; mode < 5; mode++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct user_page_fragment held = {}; + struct folio *folio; + unsigned int i, succeeded = 0; + unsigned long mapped; + long ret; + pte_t pte; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_DROPPABLE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_TRUE(test, vma->vm_flags & VM_WIPEONFORK); + KUNIT_EXPECT_TRUE(test, vma->vm_flags & VM_DONTDUMP); + if (mode == 4) { + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_TRUE(test, is_zero_pfn(pte_pfn(pte))); + } + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + if (mode != 4) + for (i = 1; i < PAGE_SIZE / SZ_4K; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + i * SZ_4K, + true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + memset(folio_address(folio), 0xa7, PAGE_SIZE); + folio_put(folio); + if (mode == 1 || mode == 2) { + ret = mode == 1 ? get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1) : + pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + if (mode == 3) { + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 1L); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_EQ(test, ret, 0L); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + } + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + if (mode == 1 || mode == 2) { + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(unsigned char *)folio_address(folio), 0xa7); + release_user_fragments(&held, 1, false); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + } + folio_put(folio); + for (i = 0; i < (mode == 4 ? 1 : PAGE_SIZE / SZ_4K); i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0L); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, PAGE_SIZE), NULL); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_lazyfree_test(struct kunit *test) +{ + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm advice requires hardware opt-in"); + return; + } + /* Partial, redirty, retained GET, typed write, all leaves, migration. */ + for (mode = 0; mode < 10; mode++) { + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *mm; + struct user_page_fragment held = {}; + struct folio *folio; + struct mm_subpage *slot; + unsigned int i, succeeded = 0; + long ret; + pte_t pte; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + if (mode == 6) { + ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + kthread_use_mm(mm); + ret = test_madvise(TEST_VA, + (mode == 4 || mode == 9) ? PAGE_SIZE : SZ_4K, MADV_FREE); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_EQ(test, pte_dirty(pte), mode == 6); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_is_lazyfree(slot), mode != 6); + mm_subpage_put(slot); + if (mode == 1) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + + if (mode == 2 || mode == 3 || mode == 8) { + ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, + mode == 3 ? FOLL_WRITE : 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + if (mode == 3 || mode == 8) { + memset(folio_address(held.folio) + held.offset, 0xd3, SZ_4K); + release_user_fragments(&held, 1, true); + } + } + if (mode == 7) { + ret = pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + if (mode == 5 || mode == 9) { + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 1L); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_EQ(test, ret, 0L); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + } + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_none(pte), (mode == 4 || mode == 9) || + (i == 0 && (mode == 0 || mode == 5))); + } + if (mode == 2 || mode == 6 || mode == 7) + release_user_fragments(&held, 1, false); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_mprotect_migration_test(struct kunit *test) +{ + unsigned int file_case; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + for (file_case = 0; file_case < 2; file_case++) { + struct mm_struct *mm = test_vma_mm(test); + struct file *file = file_case ? test_cache_file(test, PAGE_SIZE, true) : NULL; + struct vm_area_struct *vma; + struct folio *src, *dst; + unsigned long mapped; + unsigned int i; + int prepared, unmapped = -1; + long protected = -EINVAL; + bool offsets = true; + phys_addr_t phys; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (file_case) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + kthread_use_mm(mm); + mapped = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | (file ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + src = test_fault_folio(mm, TEST_VA); + if (!src) { + folio_put(dst); + KUNIT_FAIL(test, "missing migration source"); + return; + } + phys = PFN_PHYS(folio_pfn(src)); + folio_lock(src); + folio_lock(dst); + prepared = mm_subpage_migrate_prepare(src, dst); + if (!prepared) { + try_to_migrate(src, TTU_SYNC); + unmapped = folio_mapcount(src); + kthread_use_mm(mm); + protected = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + kthread_unuse_mm(mm); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + offsets &= softleaf_is_migration(softleaf_from_pte(pte)) && + !softleaf_is_migration_write(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + } + remove_migration_ptes(src, src, 0); + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + folio_put(dst); + folio_put(src); + KUNIT_ASSERT_EQ(test, prepared, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_EQ(test, protected, 0L); + KUNIT_EXPECT_TRUE(test, offsets); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys + i * SZ_4K); + } + } +} + +/* Independent roots and VA offsets must not change physical slot identity. */ +static void user4k_nonlinear_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct mm_struct *child = test_vma_mm(test); + struct mm_subpage *slot; + struct folio *folio, *dst; + phys_addr_t phys; + vm_flags_t flags; + unsigned int succeeded = 0, side; + int ret, unmapped = -1; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, child); + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + /* Unlike the lightweight rmap fixture, real LRU migration needs a charge. */ + ret = mem_cgroup_charge(folio, a->mm, GFP_KERNEL); + if (ret) { + mm_subpage_put(slot); + KUNIT_FAIL(test, "cannot charge nonlinear fixture folio"); + return; + } + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + phys = mm_subpage_phys(slot); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + /* Same native folio, different root and different VA/physical offset. */ + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), folio); + memset(folio_address(folio), 0x53, SZ_4K); + memset(folio_address(folio) + 3 * SZ_4K, 0xa7, SZ_4K); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, b->mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 2); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, + test_fault_entry(test, child, TEST_VA + SZ_4K)), phys + 3 * SZ_4K); + kunit_release_action(test, user4k_vma_mm_free, child); + + /* Rollback must find migration entries through both roots. */ + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + folio_get(folio); + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + if (!ret) { + try_to_migrate(folio, TTU_SYNC); + unmapped = folio_mapcount(folio); + remove_migration_ptes(folio, folio, 0); + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 2); + + /* Successful migration exercises copied identities in the new owner. */ + folio_add_lru(folio); + lru_add_drain_all(); + folio_get(folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (ret) { + KUNIT_FAIL(test, "nonlinear migration failed: %d", ret); + return; + } + /* Refresh fixture references before assertions/cleanup after migration. */ + for (side = 0; side < 2; side++) { + struct anon_slot_fixture *f = side ? b : a; + pte_t pte = test_fault_entry(test, f->mm, TEST_VA + side * SZ_4K); + + f->slots[side] = NULL; + f->mapped[side] = false; + if (!pte_present(pte)) + continue; + slot = mm_subpage_get_from_phys(pte_phys_mm(f->mm, pte)); + if (!slot) + continue; + f->slots[side] = slot; + f->mapped[side] = true; + mm_subpage_put(slot); + } + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + for (side = 0; side < 2; side++) { + struct anon_slot_fixture *f = side ? b : a; + + slot = f->slots[side]; + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), side ? 3U * SZ_4K : 0U); + KUNIT_EXPECT_NE(test, mm_subpage_phys(slot), phys + (side ? 3 * SZ_4K : 0)); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), side ? 0xa7 : 0x53, SZ_4K), NULL); + } +} + +static void user4k_migration_entries_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *child = test_vma_mm(test); + struct folio *src, *dst; + unsigned int i, side; + vm_fault_t fault; + bool encoded = true, copied = true, exact = true; + int ret, fork_ret = -EINVAL, unmapped = -1; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + src = test_fault_folio(vma->vm_mm, TEST_VA); + if (!src) { + folio_put(dst); + KUNIT_FAIL(test, "missing source folio"); + return; + } + folio_lock(src); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(src, dst); + if (!ret) { + try_to_migrate(src, TTU_SYNC); + unmapped = folio_mapcount(src); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + struct rmap_walk_range range = { + .address = TEST_VA + i * SZ_4K, + .subpage = true, + .subpage_offset = i * SZ_4K, + }; + DEFINE_FOLIO_RMAP_WALK(walk, src, vma, range, + PVMW_SYNC | PVMW_MIGRATION); + unsigned int count = 0; + + encoded &= softleaf_is_migration_write(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + while (page_vma_mapped_walk(&walk)) { + exact &= walk.address == TEST_VA + i * SZ_4K; + count++; + } + exact &= count == 1; + } + uprobe_start_dup_mmap(); + fork_ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + if (!fork_ret) { + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + copied &= softleaf_is_migration_read(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + } + } + } + /* Drop one migration entry while the other mm still retains it. */ + mmap_read_lock(vma->vm_mm); + zap_vma_range(vma, TEST_VA + SZ_4K, SZ_4K); + mmap_read_unlock(vma->vm_mm); + remove_migration_ptes(src, src, 0); + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + folio_put(dst); + folio_put(src); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, fork_ret, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_TRUE(test, encoded); + KUNIT_EXPECT_TRUE(test, copied); + KUNIT_EXPECT_TRUE(test, exact); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), side ? (unsigned long)TEST_SLOTS : TEST_SLOTS - 1UL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + struct mm_subpage *slot; + + if (!side && i == 1) { + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + continue; + } + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), i == 1 ? 1 : 2); + mm_subpage_put(slot); + } + } +} + +static struct folio *test_file_migration_alloc(struct folio *src, unsigned long private) +{ + return folio_alloc(GFP_HIGHUSER_MOVABLE, folio_order(src)); +} + +static void user64k_file_migration_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 6; mode++) { + bool shmem = mode & 1, fail = mode >= 2, pin = mode >= 4; + bool isolated; + unsigned long len = shmem ? SZ_64K : 2 * SZ_64K; + unsigned long base = shmem ? SZ_64K : 0, offset; + unsigned int nr = len / PAGE_SIZE, succeeded = 0; + unsigned int maps = 2 * nr + len / SZ_4K - 1; + struct file *file = shmem ? test_cache_file(test, 2 * SZ_64K, true) : + test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + struct vm_area_struct *coarse, *native, *small; + struct folio *folio; + struct user_page_fragment *fragment = NULL; + phys_addr_t old_phys, phys; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, len, base); + native = test_file_vma(test, file, true, true, TEST_VA, len, base); + /* Shifted file offset and a small-page table boundary in the alias. */ + small = test_file_vma(test, file, false, true, SZ_2M - SZ_4K, + len - SZ_4K, base + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + for (offset = 0; offset < len; offset += SZ_64K) + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + offset, shmem) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < len; offset += PAGE_SIZE) + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + offset, shmem) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < len - SZ_4K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + offset, + shmem) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, base >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), len); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + old_phys = page_to_phys(&folio->page); + if (pin) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_promote_unpin, fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + small->vm_end - SZ_4K + 37, 1, 0, fragment, 1), 1L); + KUNIT_EXPECT_PTR_EQ(test, fragment->folio, folio); + KUNIT_EXPECT_EQ(test, fragment->offset, len - SZ_4K + 37); + } + isolated = isolate_folio_to_list(folio, &folios); + /* Retain either a GET or a typed PIN to force migration rollback. */ + if (!fail || pin || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_file_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (pin) { + KUNIT_EXPECT_EQ(test, page_to_phys(&fragment->folio->page), old_phys); + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), + (u8)(0x31 + (base + len - SZ_4K) / SZ_4K)); + kunit_release_action(test, test_promote_unpin, fragment); + } else if (fail) { + folio_put(folio); + } + KUNIT_EXPECT_EQ_MSG(test, ret, fail ? 1 : 0, "mode=%u", mode); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : nr); + folio = filemap_get_folio(file->f_mapping, base >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + phys = page_to_phys(&folio->page); + KUNIT_EXPECT_EQ(test, phys == old_phys, fail); + KUNIT_EXPECT_EQ(test, folio_size(folio), len); + KUNIT_EXPECT_EQ(test, folio->index, base >> PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse->vm_mm, + shmem ? MM_SHMEMPAGES : MM_FILEPAGES), (long)(len / SZ_64K)); + for (offset = 0; offset < len; offset += SZ_4K) { + pte_t pte; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0x31 + (base + offset) / SZ_4K, SZ_4K), NULL); + if (offset) { + pte = test_fault_entry(test, small->vm_mm, + small->vm_start + offset - SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, pte), phys + offset); + } + if (!(offset % PAGE_SIZE)) { + pte = test_fault_entry(test, native->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(native->vm_mm, pte), phys + offset); + } + if (!(offset % SZ_64K)) { + pte = test_fault_entry(test, coarse->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse->vm_mm, pte), phys + offset); + } + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); + } +} + +static void user64k_anon_migration_test(struct kunit *test) +{ + unsigned int mode, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_file_enabled(test)) + return; + for (mode = 0; mode < 5; mode++) { + bool forked = mode & 1, fail = mode >= 2, pin = mode == 4; + struct vm_area_struct *vma, *cvma = NULL; + struct mm_struct *child = NULL; + struct user_page_fragment *fragment = NULL; + struct folio *folio; + phys_addr_t old_phys, phys; + unsigned int i, succeeded = 0; + pte_t pte; + bool isolated; + int ret; + LIST_HEAD(folios); + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, true) & + VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + old_phys = pte_phys_mm(vma->vm_mm, pte); + folio = page_folio(pfn_to_page(PHYS_PFN(old_phys))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(folio, i)), 0x51 + i, PAGE_SIZE); + if (forked) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + } + if (pin) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_promote_unpin, fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(vma->vm_mm, + TEST_VA + SZ_64K - 1, 1, FOLL_WRITE, fragment, 1), 1L); + KUNIT_EXPECT_PTR_EQ(test, fragment->folio, folio); + KUNIT_EXPECT_EQ(test, fragment->offset, (unsigned int)SZ_64K - 1); + } + lru_add_drain_all(); + folio_get(folio); + isolated = isolate_folio_to_list(folio, &folios); + if (!fail || pin || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_file_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (pin) + kunit_release_action(test, test_promote_unpin, fragment); + else if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ_MSG(test, ret, fail ? 1 : 0, "mode=%u", mode); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : nr); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), !forked); + phys = pte_phys_mm(vma->vm_mm, pte); + KUNIT_EXPECT_EQ(test, phys == old_phys, fail); + folio = page_folio(pfn_to_page(PHYS_PFN(phys))); + folio_get(folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + expect_coarse_anon_state(test, folio, forked ? 2 : 1, !forked); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr * (forked ? 2 : 1) + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + if (forked) { + struct folio *copied; + + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, pte), phys); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1L); + /* A restored shared migration entry must still perform full-leaf COW. */ + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_64K - 1, + true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), phys); + copied = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(child, pte)))); + expect_coarse_anon_state(test, copied, 1, true); + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(copied, i)), + 0x51 + i, PAGE_SIZE), NULL); + memset(folio_address(copied), 0xa7, SZ_64K); + kunit_release_action(test, user4k_vma_mm_free, child); + } + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(folio, i)), + 0x51 + i, PAGE_SIZE), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + } +} + +static void user4k_file_migration_test(struct kunit *test) +{ + unsigned int fail; + + for (fail = 0; fail < 2; fail++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *small, *native; + struct folio *folio; + phys_addr_t old_phys; + unsigned int i, succeeded = 0; + int ret; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, SZ_2M - SZ_4K, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + i * SZ_4K, + true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { [0 ... TEST_SLOTS - 1] = 1 }, 1); + old_phys = PFN_PHYS(folio_pfn(folio)); + isolated = isolate_folio_to_list(folio, &folios); + if (!fail || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ(test, ret, fail ? 1 : 0); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : 1U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { [0 ... TEST_SLOTS - 1] = 1 }, 1); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, small->vm_mm, small->vm_start + i * SZ_4K); + + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + PFN_PHYS(folio_pfn(folio)) + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry) == old_phys + i * SZ_4K, !!fail); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, pte_phys_mm(native->vm_mm, + test_fault_entry(test, native->vm_mm, TEST_VA)), PFN_PHYS(folio_pfn(folio))); + /* File reclaim must visit every 4K alias, including across PTE tables. */ + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + KUNIT_EXPECT_FALSE(test, folio_mapped(folio)); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 0, 0, 0 }, 0); + folio_unlock(folio); + folio_put(folio); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, + small->vm_start + i * SZ_4K))); + } + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + i * SZ_4K, + false) & VM_FAULT_ERROR, 0U); + } +} + +static void user4k_migration_test(struct kunit *test) +{ + const unsigned long boundaries[] = { + SZ_2M - SZ_4K, SZ_2M, SZ_1G - SZ_4K, SZ_1G, + BIT(39) - SZ_4K, BIT(39), + }; + unsigned int mode; + + /* Dense/sparse, forks, refcount rollback, batched TLBI and table edges. */ + for (mode = 0; mode < 16; mode++) { + bool edge = mode >= 6 && mode < 12; + bool locked = mode >= 12; + bool sparse = mode == 1 || edge || mode == 13; + bool forked = mode == 2 || mode == 4 || edge || mode == 14; + bool fail = mode == 3 || mode == 4 || mode == 15; + unsigned long start = edge ? boundaries[mode - 6] : + TEST_VA + (sparse ? SZ_4K : 0); + unsigned int nr = sparse ? 1 : 4, i, side, succeeded = 0; + struct vm_area_struct *vma = test_fault_vma(test, start, nr * SZ_4K); + struct mm_struct *child = NULL; + struct folio *folio; + phys_addr_t old_phys; + vm_fault_t fault; + int ret; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, vma); + if (locked) { + mmap_write_lock(vma->vm_mm); + vm_flags_set(vma, VM_LOCKED); + vma->vm_mm->locked_vm = nr; + mmap_write_unlock(vma->vm_mm); + } + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, start, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + old_phys = PFN_PHYS(folio_pfn(folio)); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x41 + i + mode, SZ_4K); + test_swap_tags_set(folio); + folio_put(folio); + if (forked) { + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + } + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + /* An extra native reference forces the normal migration rollback. */ + if (!fail || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + mode == 5 ? MIGRATE_ASYNC : MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ(test, ret, fail ? 1 : 0); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : 1U); + for (side = 0; side <= (unsigned int)forked; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + for (i = 0; i < nr; i++) { + unsigned int index = sparse ? offset_in_page(start) / SZ_4K : i; + pte_t pte = test_fault_entry(test, mm, start + i * SZ_4K); + struct mm_subpage *slot; + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) == old_phys + index * SZ_4K, fail); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), !forked); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), index * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), forked ? 2 : 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + index * SZ_4K, + 0x41 + index + mode, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, index)); + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)nr); + } + if (locked) { + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + if (!fail) + KUNIT_EXPECT_EQ(test, folio->mlock_count, nr); + folio_put(folio); + } + if (child) + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } +} +struct fragment_pin_worker { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready; + unsigned long checks; + int error; +}; + +static int fragment_pin_worker_run(void *arg) +{ + struct fragment_pin_worker *worker = arg; + struct user_page_fragment fragments[4]; + + while (!kthread_should_stop()) { + long nr = pin_user_fragments_remote(worker->mm, TEST_VA, PAGE_SIZE, + 0, fragments, ARRAY_SIZE(fragments)); + unsigned int i; + + if (nr != 4) { + worker->error = nr < 0 ? nr : -EIO; + } else { + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &fragments[i]; + + if (memchr_inv(folio_address(f->folio) + f->offset, 0x51 + i, f->length)) + worker->error = -EILSEQ; + worker->checks++; + } + } + if (nr > 0) + release_user_fragments(fragments, nr, false); + complete_all(&worker->ready); + if (worker->error) + break; + cond_resched(); + } + return 0; +} + +static void fragment_pin_worker_stop(void *arg) +{ + struct fragment_pin_worker *worker = arg; + + if (worker->task) { + kthread_stop(worker->task); + worker->task = NULL; + } +} + +static void user4k_fragment_pin_race_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct fragment_test_pins *pins; + struct fragment_pin_worker *workers; + unsigned int i, round, migrated = 0, busy = 0; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 4); + KUNIT_ASSERT_EQ(test, ret, 4L); + for (i = 0; i < 4; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0x51 + i, pins->fragments[i].length); + fragment_test_unpin(pins); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].mm = vma->vm_mm; + init_completion(&workers[i].ready); + workers[i].task = kthread_run(fragment_pin_worker_run, &workers[i], "user4k-pin-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "pin worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_pin_worker_stop, &workers[i]), 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + } + for (round = 0; round < 32; round++) { + struct mm_struct *child = test_vma_mm(test); + unsigned long addr = TEST_VA + (round % 4) * SZ_4K; + struct vm_area_struct *cvma; + struct folio *folio; + unsigned int succeeded = 0; + vm_fault_t fault; + struct user_page_fragment write_fragment; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + cvma = test_vma_lookup(child, addr); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + ret = pin_user_fragments_remote(child, addr, 1, FOLL_WRITE, &write_fragment, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + *((u8 *)folio_address(write_fragment.folio) + write_fragment.offset) = 0xa0 + round; + release_user_fragments(&write_fragment, 1, true); + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + if (ret) { + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_ASYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_TRUE(test, ret == 0 || ret == 1); + migrated += succeeded; + busy += ret == 1; + } + kunit_release_action(test, user4k_vma_mm_free, child); + } + for (i = 0; i < 2; i++) { + fragment_pin_worker_stop(&workers[i]); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_GE(test, workers[i].checks, 4UL); + } + kunit_info(test, "32 fork/COW/exit cycles against two pin workers; migrations=%u not_migrated=%u\n", + migrated, busy); +} + +#endif + +static int test_swap_control(const char *operation) +{ + char *argv[] = { "/init", (char *)operation, NULL }; + char *envp[] = { NULL }; + + return call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC); +} + +static void test_reenable_swap(void *arg) +{ + bool *disabled = arg; + + if (*disabled) + test_swap_control("--swapon-helper"); +} + +/* Exclusive slots must remain packed after read swap-in and later writes. */ +static void test_user4k_swap_exclusive(struct kunit *test, unsigned int mode) +{ + struct vm_area_struct *vma; + struct folio *folio; + struct mm_subpage *held = NULL; + bool *disabled; + phys_addr_t restored[TEST_SLOTS]; + swp_entry_t entry; + vm_fault_t fault; + unsigned int i; + int ret, mapped, hold_err = 0; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0xc0 + i, SZ_4K); + test_swap_tags_set(folio); + if (mode == 5) { + held = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(folio)) + 2 * SZ_4K); + if (held) { + hold_err = kunit_add_action_or_reset(test, test_subpage_put, held); + if (hold_err) + held = NULL; + } + } + ret = folio_alloc_swap(folio); + entry = folio->swap; + if (!ret) + try_to_unmap(folio, TTU_SYNC); + mapped = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + if (mode == 5) { + KUNIT_ASSERT_EQ(test, hold_err, 0); + KUNIT_ASSERT_NOT_NULL(test, held); + } + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(pte))); + KUNIT_EXPECT_TRUE(test, pte_swp_exclusive(pte)); + } + if (mode >= 3) { + struct mm_struct *child = test_vma_mm(test); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + } + } + if (mode == 1 || mode == 2 || mode == 4 || mode == 6) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + if (mode == 2 || mode == 6) { + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, *disabled); + } + for (i = 0; i < TEST_SLOTS; i++) { + struct mm_subpage *slot; + unsigned long address = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + restored[i] = pte_phys_mm(vma->vm_mm, pte); + slot = mm_subpage_get_from_phys(restored[i]); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(slot), !(held && i == 2)); + KUNIT_EXPECT_EQ(test, restored[i], restored[0] + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xc0 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(slot), i)); + mm_subpage_put(slot); + } + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long address = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + if (held && i == 2) { + KUNIT_EXPECT_NE(test, pte_phys_mm(vma->vm_mm, pte), restored[i]); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), 0xdd, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(held)) + + mm_subpage_offset(held), 0xc2, SZ_4K), NULL); + } else { + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), restored[i]); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0UL); + if (*disabled) { + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_EXPECT_EQ(test, ret, 0); + } +} + +static void user4k_swap_exclusive_cached_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 0); +} + +static void user4k_swap_exclusive_disk_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 1); +} + +static void user4k_swap_exclusive_swapoff_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 2); +} + +static void user4k_swap_reuse_cached_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 3); +} + +static void user4k_swap_reuse_disk_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 4); +} + +static void user4k_swap_reuse_get_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 5); +} + +static void user4k_swap_reuse_swapoff_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 6); +} + +static void test_user4k_swap_roundtrip(struct kunit *test, unsigned int mode, u8 base) +{ + const bool evict = mode != 0; + struct vm_area_struct *parent, *cvma; + struct mm_struct *child; + struct folio *folio; + swp_entry_t entry = { 0 }; + unsigned int i, count; + int err, mapped; + vm_fault_t fault; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "rerun this suite after enabling the disposable guest swap disk"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, base + i, SZ_4K); + test_swap_tags_set(folio); + if (mode == 2) { + LIST_HEAD(folios); + + folio_unlock(folio); + err = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!err) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Let the normal reclaim path allocate swap and unmap all slots. */ + reclaim_pages(&folios); + mapped = 0; + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + mapped += pte_present(pte); + } + entry = softleaf_from_pte(pte); + } else { + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC | (evict ? TTU_BATCH_FLUSH : 0)); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + try_to_unmap_flush(); + folio_put(folio); + } + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_ANONPAGES), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_swp_exclusive(pte)); + } + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, (unsigned int)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + if (evict) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + + /* Fork while all slots are nonpresent, preserving each byte offset. */ + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, 2U * TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 2U); + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long address = TEST_VA + ((i * 5 + 2) % TEST_SLOTS) * SZ_4K; + struct mm_subpage *slot; + spinlock_t *ptl; + pte_t *ptep; + bool intact = false; + + pte = test_fault_entry(test, parent->vm_mm, address); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + pte = test_fault_entry(test, child, address); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), ((i * 5 + 2) % TEST_SLOTS) * (unsigned int)SZ_4K); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + KUNIT_EXPECT_EQ(test, !!(fault & VM_FAULT_MAJOR), evict && !i); + ptep = test_mm_lookup(parent->vm_mm, address, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + pte = ptep_get(ptep); + slot = pte_present(pte) ? mm_subpage_get_from_phys(pte_phys_mm(parent->vm_mm, pte)) : NULL; + if (slot) { + intact = !memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), base + ((i * 5 + 2) % TEST_SLOTS), SZ_4K) && + test_swap_tags_match(mm_subpage_folio(slot), ((i * 5 + 2) % TEST_SLOTS)); + mm_subpage_put(slot); + } + pte_unmap_unlock(ptep, ptl); + KUNIT_EXPECT_TRUE(test, intact); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), 0UL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + /* A write fault must swap in and then COW just the child's selected slot. */ + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + SZ_4K, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, 1)); + memset(folio_address(folio) + SZ_4K, 0xa7, SZ_4K); + folio_put(folio); + folio = test_fault_folio(parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), TEST_SLOTS - 1UL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), i == 1 ? 0U : 1U); + /* Exit with three swapped slots, then tear down the restored parent. */ + kunit_release_action(test, user4k_vma_mm_free, child); + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, 0U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 0U); + kunit_release_action(test, user4k_vma_mm_free, parent->vm_mm); +} + +static void user4k_swap_mremap_test(struct kunit *test) +{ + struct mm_struct *mm; + struct folio *folio; + unsigned long old = TEST_VA, new = SZ_1G - SZ_4K, mapped, moved; + pte_t saved[TEST_SLOTS]; + u8 byte = 0x55; + int ret; + unsigned int i; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, old, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old, &byte, 1, FOLL_WRITE), 1); + folio = test_fault_folio(mm, old); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x81 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < TEST_SLOTS; i++) { + saved[i] = test_fault_entry(test, mm, old + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(saved[i]))); + } + kthread_use_mm(mm); + moved = test_mremap(old, PAGE_SIZE, PAGE_SIZE, MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte_t entry = test_fault_entry(test, mm, new + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, softleaf_from_pte(entry).val, softleaf_from_pte(saved[i]).val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(entry), i * (unsigned int)SZ_4K); + } + for (i = 0; i < TEST_SLOTS; i++) { + unsigned int slot = (i + 2) % TEST_SLOTS; + unsigned long addr = new + slot * SZ_4K; + + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, addr, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0x81 + slot)); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, slot)); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); +} + +static void user4k_swap_pin_test(struct kunit *test) +{ + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + struct folio *folio; + phys_addr_t pinned_phys; + unsigned int i; + vm_fault_t fault; + long ret; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 4); + KUNIT_ASSERT_EQ(test, ret, 4L); + for (i = 0; i < 4; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0xb1 + i, pins->fragments[i].length); + test_swap_tags_set(pins->fragments[0].folio); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA + SZ_4K, SZ_4K, FOLL_WRITE, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + pinned_phys = mm_subpage_phys(pins->fragments[0].subpage); + folio = pins->fragments[0].folio; + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), pinned_phys); + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(pins->fragments[0].subpage)); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(pins->fragments[0].subpage), 1U); + memset(folio_address(folio) + SZ_4K, 0xd2, SZ_4K); + fragment_test_unpin(pins); + /* Once released, the formerly pinned PTE can be swapped normally. */ + folio = test_fault_folio(vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + ret = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0L); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + i == 1 ? 0xd2 : 0xb1 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } +} + +#ifdef CONFIG_MIGRATION +static void user4k_swap_migration_test(struct kunit *test) +{ + struct vm_area_struct *vma, *cvma; + struct mm_struct *child; + struct folio *folio; + phys_addr_t old_phys; + unsigned int i, side, succeeded = 0; + vm_fault_t fault; + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + old_phys = PFN_PHYS(folio_pfn(folio)); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0xa1 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + SZ_4K, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_swapcache(folio)); + ret = isolate_folio_to_list(folio, &folios) ? 0 : -EBUSY; + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + /* One present slot and seven swap entries must all follow the new cache folio. */ + for (side = 0; side < 2; side++) { + struct vm_area_struct *area = side ? cvma : vma; + struct mm_struct *mm = area->vm_mm; + + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + + mmap_read_lock(mm); + fault = handle_mm_fault(area, addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(folio)), old_phys); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0xa1 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + } +} +#endif + +static void user4k_swap_cached_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 0, 0x51); +} + +static void user4k_swap_disk_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 1, 0x61); +} + +static void user4k_swap_reclaim_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 2, 0x71); +} + +static void user4k_swap_stress_test(struct kunit *test) +{ + unsigned int i; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (i = 0; i < 32; i++) + test_user4k_swap_roundtrip(test, i % 3, 0x21 + i * 5); + kunit_info(test, "32 swap/fork/COW/exit cycles with distinct data patterns\n"); +} + +static void user4k_swapoff_test(struct kunit *test) +{ + struct vm_area_struct *parent, *cvma; + struct mm_struct *child; + struct folio *folio; + bool *disabled; + vm_fault_t fault; + unsigned int i, side; + swp_entry_t entry; + int ret; + pte_t pte; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x91 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_fault_entry(test, parent->vm_mm, TEST_VA); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, *disabled); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + i * (unsigned long)SZ_4K); + folio = test_fault_folio(mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x91 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } + } + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + 3 * SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + 3 * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), pte_phys_mm(parent->vm_mm, + test_fault_entry(test, parent->vm_mm, TEST_VA + 3 * SZ_4K))); + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_EXPECT_EQ(test, ret, 0); +} + +static void user4k_swapoff_boundaries_test(struct kunit *test) +{ + const unsigned long boundaries[] = { SZ_2M, SZ_1G, 1UL << 39 }; + bool *disabled; + unsigned int boundary; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + for (boundary = 0; boundary < ARRAY_SIZE(boundaries); boundary++) { + unsigned long start = boundaries[boundary] - SZ_4K; + struct vm_area_struct *parent = test_fault_vma(test, start, 2 * SZ_4K); + struct mm_struct *child; + unsigned int i, side; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, parent); + for (i = 0; i < 2; i++) { + unsigned long addr = start + i * SZ_4K; + unsigned int offset = vma_page_offset_at(parent, addr).offset; + struct folio *folio; + vm_fault_t fault; + swp_entry_t entry; + LIST_HEAD(folios); + + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + memset(folio_address(folio) + offset, 0xb0 + boundary * 2 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = softleaf_from_pte(test_fault_entry(test, parent->vm_mm, addr)); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + } + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 2UL); + for (i = 0; i < 2; i++) { + unsigned long addr = start + i * SZ_4K; + unsigned int offset = vma_page_offset_at(parent, addr).offset; + pte_t pte = test_fault_entry(test, mm, addr); + struct folio *folio; + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + (unsigned long)offset); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0xb0 + boundary * 2 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, offset / SZ_4K)); + folio_put(folio); + } + } + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, parent->vm_mm); + } +} + +struct live_swap_reader { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready, resume, entered, done; + unsigned int id, checks; + int error; +}; + +static int live_swap_reader_run(void *arg) +{ + struct live_swap_reader *r = arg; + unsigned int phase, i; + + kthread_use_mm(r->mm); + for (phase = 0; phase < 2; phase++) { + if (phase) + complete(&r->entered); + for (i = 0; i < 4; i++) { + u8 value = 0; + void __user *addr = (void __user *)(TEST_VA + i * SZ_4K + 64 + r->id); + + /* Resolve AF, swap and COW faults through the real uaccess path. */ + if (copy_from_user(&value, addr, 1) || value != 0xd1 + i) + r->error = phase ? 2 : 1; + r->checks++; + } + if (!phase) { + complete(&r->ready); + if (!wait_for_completion_timeout(&r->resume, 15 * HZ)) { + r->error = 3; + goto out; + } + } + } + { + u8 value = 0xe0 + r->id; + void __user *addr = (void __user *)(TEST_VA + SZ_4K + 64 + r->id); + + if (copy_to_user(addr, &value, 1)) + r->error = 4; + r->checks++; + } +out: + kthread_unuse_mm(r->mm); + complete(&r->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void live_swap_reader_stop(void *arg) +{ + struct live_swap_reader *r = arg; + + if (r->task) { + complete_all(&r->resume); + kthread_stop(r->task); + r->task = NULL; + } +} + +static void test_live_fault_roundtrip(struct kunit *test, unsigned int mode) +{ + bool migration = mode != 0; + struct vm_area_struct *parent; + struct mm_struct *child; + struct live_swap_reader *readers; + struct folio *folio; + vm_fault_t fault; + unsigned int i, cpu; + swp_entry_t entry; + u64 tcr = read_sysreg(tcr_el1); + int ret; + LIST_HEAD(folios); + + if ((!migration && !total_swap_pages) || !user4k_test_enabled || num_online_cpus() < 2 || + !system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires two CPUs, compatible opt-in 4K TTBR0 controls, and swap for reclaim"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < 4; i++) + memset(folio_address(folio) + i * SZ_4K, 0xd1 + i, SZ_4K); + folio_put(folio); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + readers = kunit_kcalloc(test, 2, sizeof(*readers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, readers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 2; i++) { + readers[i].mm = child; + readers[i].id = i; + init_completion(&readers[i].ready); + init_completion(&readers[i].resume); + init_completion(&readers[i].entered); + init_completion(&readers[i].done); + readers[i].task = kthread_create(live_swap_reader_run, &readers[i], + "user4k-swap-read-%u", i); + if (IS_ERR(readers[i].task)) { + readers[i].task = NULL; + KUNIT_FAIL(test, "live swap worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, live_swap_reader_stop, + &readers[i]), 0); + kthread_bind(readers[i].task, cpu); + cpu = cpumask_next(cpu, cpu_online_mask); + wake_up_process(readers[i].task); + } + for (i = 0; i < 2; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].ready, 10 * HZ), 0UL); + lru_add_drain_all(); + folio = test_fault_folio(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + if (mode == 2) { +#ifdef CONFIG_MIGRATION + struct folio *dst = folio_alloc(GFP_KERNEL, 0); + bool entered = true, blocked = true, unmapped = false; + + if (!dst) { + folio_put(folio); + KUNIT_FAIL(test, "migration waiter destination allocation failed"); + return; + } + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + if (!ret) { + try_to_migrate(folio, TTU_SYNC); + unmapped = !folio_mapped(folio); + for (i = 0; i < 2; i++) + complete(&readers[i].resume); + for (i = 0; i < 2; i++) { + entered &= !!wait_for_completion_timeout(&readers[i].entered, HZ); + if (wait_for_completion_timeout(&readers[i].done, max(1UL, HZ / 20UL))) { + blocked = false; + complete(&readers[i].done); + } + } + remove_migration_ptes(folio, folio, 0); + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, entered); + KUNIT_EXPECT_TRUE(test, blocked); + KUNIT_EXPECT_TRUE(test, unmapped); +#endif + } else if (migration) { +#ifdef CONFIG_MIGRATION + unsigned int succeeded = 0; + phys_addr_t old_phys = PFN_PHYS(folio_pfn(folio)); + + ret = isolate_folio_to_list(folio, &folios) ? 0 : -EBUSY; + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_ASYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, test_fault_entry(test, child, TEST_VA)), old_phys); +#endif + } else { + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = softleaf_from_pte(test_fault_entry(test, child, TEST_VA)); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + } + for (i = 0; i < 2; i++) + complete(&readers[i].resume); + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].done, 15 * HZ), 0UL); + live_swap_reader_stop(&readers[i]); + KUNIT_EXPECT_EQ(test, readers[i].error, 0); + KUNIT_EXPECT_EQ(test, readers[i].checks, 9U); + } + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + SZ_4K + 64 + i), (u8)(0xe0 + i)); + folio_put(folio); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA + SZ_4K, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xd2, SZ_4K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 4UL); + kunit_info(test, "two CPUs access through uaccess after %s, then write one child slot\n", + mode == 2 ? "migration-entry rollback" : migration ? "migration" : "reclaim"); +} + +static void user4k_live_swap_fault_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 0); +} + +#ifdef CONFIG_MIGRATION +static void user4k_live_migration_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 1); +} + +static void user4k_migration_wait_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 2); +} +#endif + +/* Transfer PTE ownership to normal mm teardown, which also handles swap. */ +static int anon_slot_fixture_handoff(struct kunit *test, struct anon_slot_fixture *f) +{ + struct mm_struct *mm = f->mm; + + f->mm = NULL; + f->vma = NULL; + memset(f->slots, 0, sizeof(f->slots)); + memset(f->mapped, 0, sizeof(f->mapped)); + return kunit_add_action_or_reset(test, user4k_vma_mm_free, mm); +} + +static void test_nonlinear_swap(struct kunit *test, bool evict) +{ + unsigned int first; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (first = 0; first < 2; first++) { + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct vm_area_struct *vmas[2]; + struct mm_struct *child; + struct mm_subpage *slot; + struct folio *folio; + swp_entry_t entry = { }; + unsigned int side; + vm_fault_t fault; + int err, mapped; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + vmas[0] = a->vma; + vmas[1] = b->vma; + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + err = mem_cgroup_charge(folio, a->mm, GFP_KERNEL); + if (err) { + mm_subpage_put(slot); + KUNIT_FAIL(test, "cannot charge swap fixture: %d", err); + return; + } + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + folio_lock(folio); + mm_subpage_set_exclusive(slot); + folio_unlock(folio); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + folio_lock(folio); + mm_subpage_set_exclusive(slot); + folio_unlock(folio); + memset(folio_address(folio), 0x5b, SZ_4K); + memset(folio_address(folio) + 3 * SZ_4K, 0xd3, SZ_4K); + test_swap_tags_set(folio); + folio_mark_dirty(folio); + folio_add_lru(folio); + lru_add_drain_all(); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_handoff(test, a), 0); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_handoff(test, b), 0); + folio_get(folio); + folio_lock(folio); + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + for (side = 0; side < 2; side++) { + pte_t pte = test_fault_entry(test, vmas[side]->vm_mm, + TEST_VA + side * SZ_4K); + + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), side ? 3U * SZ_4K : 0U); + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, side ? 3 * SZ_4K : 0), 1U); + } + if (evict) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + /* Preserve physical offsets when a nonpresent nonlinear slot is forked. */ + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vmas[1]->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + for (side = 0; side < 2; side++) { + unsigned int which = side ^ first, physical = which ? 3 : 0; + struct vm_area_struct *vma = vmas[which]; + unsigned long address = TEST_VA + which * SZ_4K; + pte_t pte; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ_MSG(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), + (vm_fault_t)0, "evict=%d first=%u side=%u", evict, first, which); + KUNIT_EXPECT_EQ(test, !!(fault & VM_FAULT_MAJOR), evict && !side); + pte = test_fault_entry(test, vma->vm_mm, address); + slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), physical * (unsigned int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), which ? 0xd3 : 0x5b, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(slot), physical)); + mm_subpage_put(slot); + } + /* A child's write must copy only its slot at the new virtual offset. */ + { + struct vm_area_struct *vma = test_vma_lookup(child, TEST_VA + SZ_4K); + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(child); + fault = handle_mm_fault(vma, TEST_VA + SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + slot = mm_subpage_get_from_phys(pte_phys_mm(child, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xd3, SZ_4K), NULL); + memset(folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot), 0x91, SZ_4K); + mm_subpage_put(slot); + pte = test_fault_entry(test, vmas[1]->vm_mm, TEST_VA + SZ_4K); + slot = mm_subpage_get_from_phys(pte_phys_mm(vmas[1]->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xd3, SZ_4K), NULL); + mm_subpage_put(slot); + } + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vmas[1]->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, vmas[0]->vm_mm); + } +} + +static void user4k_nonlinear_swap_cached_test(struct kunit *test) +{ + test_nonlinear_swap(test, false); +} + +static void user4k_nonlinear_swap_disk_test(struct kunit *test) +{ + test_nonlinear_swap(test, true); +} + +#ifdef CONFIG_USERFAULTFD +/* The context is attached only during the call; ordinary mm cleanup owns the VMAs. */ +static ssize_t test_uffd_move(struct vm_area_struct *dst, unsigned long dst_addr, + unsigned long src_addr, unsigned long size) +{ + struct userfaultfd_ctx ctx = { .mm = dst->vm_mm }; + ssize_t ret; + + init_rwsem(&ctx.map_changing_lock); + atomic_set(&ctx.mmap_changing, 0); + mmap_write_lock(ctx.mm); + vma_start_write(dst); + dst->vm_userfaultfd_ctx.ctx = &ctx; + mmap_write_unlock(ctx.mm); + ret = move_pages(&ctx, dst_addr, src_addr, size, 0); + mmap_write_lock(ctx.mm); + vma_start_write(dst); + dst->vm_userfaultfd_ctx.ctx = NULL; + mmap_write_unlock(ctx.mm); + return ret; +} + +struct move_migration_worker { + struct userfaultfd_ctx ctx; + struct vm_area_struct *vma[2]; + struct task_struct *task; + struct completion ready; + unsigned long address[2], size; + unsigned long moves, busy; + int error; +}; + +static int move_migration_worker_run(void *arg) +{ + struct move_migration_worker *w = arg; + + while (!kthread_should_stop()) { + unsigned int side = w->moves & 1; + ssize_t ret = move_pages(&w->ctx, w->address[side ^ 1], + w->address[side], w->size, 0); + + if (ret == w->size) + WRITE_ONCE(w->moves, w->moves + 1); + else if (ret == -EAGAIN || ret == -EBUSY) + w->busy++; + else + w->error = ret < 0 ? ret : -EIO; + if (w->moves || w->error) + complete_all(&w->ready); + if (w->error) { + /* Keep the task valid until the owner joins it. */ + set_current_state(TASK_INTERRUPTIBLE); + if (!kthread_should_stop()) + schedule(); + __set_current_state(TASK_RUNNING); + } + /* Leave bounded gaps so migration can complete between contended moves. */ + if (ret == w->size) + usleep_range(1000, 2000); + else + cond_resched(); + } + return 0; +} + +static void move_migration_worker_stop(void *arg) +{ + struct move_migration_worker *w = arg; + unsigned int i; + + if (w->task) { + kthread_stop(w->task); + w->task = NULL; + } + mmap_write_lock(w->ctx.mm); + for (i = 0; i < 2; i++) { + vma_start_write(w->vma[i]); + w->vma[i]->vm_userfaultfd_ctx.ctx = NULL; + } + mmap_write_unlock(w->ctx.mm); +} + +static void user4k_move_migration_race_test(struct kunit *test) +{ + unsigned int shift; + + for (shift = 12; shift < PAGE_SHIFT; shift += 2) { + struct mm_struct *mm = test_vma_mm_granule(test, shift); + struct move_migration_worker *w; + unsigned long size = 1UL << shift, i, initial_moves; + unsigned int round, migrated = 0, failed = 0; + pte_t pte; + vm_fault_t fault; + + KUNIT_ASSERT_NOT_NULL(test, mm); + w = kunit_kzalloc(test, sizeof(*w), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, w); + w->ctx.mm = mm; + init_rwsem(&w->ctx.map_changing_lock); + atomic_set(&w->ctx.mmap_changing, 0); + init_completion(&w->ready); + w->size = size; + w->address[0] = TEST_VA + 3 * size; + w->address[1] = TEST_VA + SZ_128M + size; + w->vma[0] = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + w->vma[1] = test_fault_vma_in_mm(mm, TEST_VA + SZ_128M, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, w->vma[0]); + KUNIT_ASSERT_NOT_NULL(test, w->vma[1]); + for (i = 0; i < PAGE_SIZE; i += size) { + mmap_read_lock(mm); + fault = handle_mm_fault(w->vma[0], TEST_VA + i, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, mm, TEST_VA + i); + memset(phys_to_virt(pte_phys_mm(mm, pte)), 0x60 + i / size, size); + } + /* Both directions are registered before starting the concurrent mover. */ + mmap_write_lock(mm); + for (i = 0; i < 2; i++) { + vma_start_write(w->vma[i]); + w->vma[i]->vm_userfaultfd_ctx.ctx = &w->ctx; + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, move_migration_worker_stop, w), 0); + w->task = kthread_create(move_migration_worker_run, w, "user4k-move"); + if (IS_ERR(w->task)) { + w->task = NULL; + KUNIT_FAIL(test, "MOVE worker creation failed"); + return; + } + if (num_online_cpus() > 1) + kthread_bind(w->task, cpumask_last(cpu_online_mask)); + wake_up_process(w->task); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->ready, 10 * HZ), 0UL); + initial_moves = READ_ONCE(w->moves); + lru_add_drain_all(); + for (round = 0; round < 64; round++) { + struct folio *folio; + unsigned int succeeded = 0; + bool isolated; + int ret; + LIST_HEAD(folios); + + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE_MSG(test, isolated, "round=%u", round); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_GE(test, ret, 0); + migrated += succeeded; + failed += ret > 0; + cond_resched(); + } + kunit_release_action(test, move_migration_worker_stop, w); + kunit_info(test, "%uK MOVE/migration: moves=%lu retries=%lu migrated=%u busy=%u\n", + 1U << (shift - 10), w->moves, w->busy, migrated, failed); + KUNIT_EXPECT_EQ(test, w->error, 0); + KUNIT_EXPECT_GT(test, w->moves, initial_moves); + KUNIT_EXPECT_GT(test, migrated, 0U); + for (i = 0; i < PAGE_SIZE; i += size) { + unsigned long addr = i == 3 * size ? w->address[w->moves & 1] : TEST_VA + i; + + pte = test_fault_entry(test, mm, addr); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(mm, pte)), + 0x60 + i / size, size), NULL); + } + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, w->address[(w->moves & 1) ^ 1]))); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_move_swap_get_test(struct kunit *test) +{ + unsigned int variant; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (variant = 0; variant < PAGE_SHIFT - 12; variant++) { + unsigned int shift = 12 + (variant / 2) * 2; + bool forked = variant & 1; + unsigned long size = 1UL << shift; + unsigned long src_addr = TEST_VA + 3 * size; + unsigned long dst_addr = TEST_VA + SZ_128M + size; + struct mm_struct *mm = test_vma_mm_granule(test, shift); + struct vm_area_struct *src, *dst; + struct mm_subpage *held; + struct folio *folio; + phys_addr_t physical; + swp_entry_t entry = { }; + pte_t before, after; + vm_fault_t fault; + unsigned long i; + int err, mapped; + + KUNIT_ASSERT_NOT_NULL(test, mm); + src = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + dst = test_fault_vma_in_mm(mm, TEST_VA + SZ_128M, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, src); + KUNIT_ASSERT_NOT_NULL(test, dst); + for (i = 0; i < PAGE_SIZE; i += size) { + mmap_read_lock(mm); + fault = handle_mm_fault(src, TEST_VA + i, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + before = test_fault_entry(test, mm, TEST_VA + i); + memset(phys_to_virt(pte_phys_mm(mm, before)), 0x50 + i / size, size); + } + before = test_fault_entry(test, mm, src_addr); + physical = pte_phys_mm(mm, before); + held = mm_subpage_get_from_phys(physical); + KUNIT_ASSERT_NOT_NULL(test, held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + KUNIT_ASSERT_EQ(test, mm_subpage_offset(held), (unsigned int)(3 * size)); + folio = mm_subpage_folio(held); + lru_add_drain_all(); + folio_lock(folio); + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + KUNIT_ASSERT_EQ(test, mm_subpage_mapcount(held), 0); + KUNIT_ASSERT_EQ(test, swap_subpage_count(entry, 3 * size), 1U); + before = test_fault_entry(test, mm, src_addr); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(before))); + if (forked) { + struct mm_struct *child = test_vma_mm_granule(test, shift); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, test_uffd_move(dst, dst_addr, src_addr, size), + (ssize_t)-EBUSY); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(test_fault_entry(test, mm, + src_addr)).val, entry.val); + kunit_release_action(test, user4k_vma_mm_free, child); + before = test_fault_entry(test, mm, src_addr); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(before)); + KUNIT_ASSERT_EQ(test, swap_subpage_count(entry, 3 * size), 1U); + } + KUNIT_ASSERT_EQ(test, test_uffd_move(dst, dst_addr, src_addr, size), (ssize_t)size); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(after).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(after), (unsigned int)(3 * size)); + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, src_addr))); + /* Restore with the retained metadata already rebound to the new root. */ + mmap_read_lock(mm); + fault = handle_mm_fault(dst, dst_addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_ASSERT_TRUE(test, pte_present(after)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after), physical); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(physical), 0x53, size), NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(held), !forked); + /* Fork-cleared exclusivity requires COW while a GET is retained. */ + mmap_read_lock(mm); + fault = handle_mm_fault(dst, dst_addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_ASSERT_TRUE(test, pte_write(after)); + if (forked) + KUNIT_EXPECT_NE(test, pte_phys_mm(mm, after), physical); + else + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after), physical); + memset(phys_to_virt(pte_phys_mm(mm, after)), 0xa9, size); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(physical), + forked ? 0x53 : 0xa9, size), NULL); + for (i = 0; i < PAGE_SIZE; i += size) { + if (i == 3 * size) + continue; + mmap_read_lock(mm); + fault = handle_mm_fault(src, TEST_VA + i, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, TEST_VA + i); + KUNIT_ASSERT_TRUE(test, pte_present(after)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(mm, after)), + 0x50 + i / size, size), NULL); + } + kunit_release_action(test, test_subpage_put, held); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} +#endif + +static struct kunit_case user4k_swap_cases[] = { + KUNIT_CASE(user4k_swap_reuse_get_test), + KUNIT_CASE(user4k_swap_reuse_swapoff_test), + KUNIT_CASE(user4k_swap_reuse_cached_test), + KUNIT_CASE(user4k_swap_reuse_disk_test), + KUNIT_CASE(user4k_swap_exclusive_cached_test), + KUNIT_CASE(user4k_swap_exclusive_disk_test), + KUNIT_CASE(user4k_swap_exclusive_swapoff_test), + KUNIT_CASE(user4k_swap_cached_test), + KUNIT_CASE(user4k_swap_pin_test), + KUNIT_CASE(user4k_swap_mremap_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_swap_migration_test), +#endif + KUNIT_CASE(user4k_swap_disk_test), + KUNIT_CASE(user4k_swap_reclaim_test), + KUNIT_CASE(user4k_swapoff_test), + KUNIT_CASE(user4k_swap_stress_test), + KUNIT_CASE(user4k_swapoff_boundaries_test), + KUNIT_CASE(user4k_live_swap_fault_test), + {} +}; + +static void user4k_swap_suite_exit(struct kunit_suite *suite) +{ + kvfree_rcu_barrier(); +} + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +static void test_shmem_uffd_swap(struct kunit *test, unsigned int mode) +{ + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, mode == 3 ? 2 : 1); + struct swap_iocb *plug = NULL; + struct file *file; + struct folio *folio; + void *entry; + bool *disabled; + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires disposable guest swap"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + file = test_cache_file(test, PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xb6, mode == 3 ? 2 * SZ_4K : SZ_4K); + ret = shmem_uffd_set_missing(folio, 0, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + /* Match pageout(): submitted data is clean unless writeout redirties it. */ + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + ret = shmem_writeout(folio, &plug, &folios); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_TRUE(test, list_empty(&folios)); + entry = filemap_get_entry(file->f_mapping, 0); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + KUNIT_EXPECT_EQ(test, xa_to_value(xa_load(&SHMEM_I(file_inode(file))->uffd_missing, 0)), missing); + if (mode == 1) { + shmem_truncate_range(file_inode(file), 0, PAGE_SIZE - 1); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + } else if (mode == 3) { + shmem_truncate_range(file_inode(file), 0, SZ_4K - 1); + } else if (mode == 2) { + ret = test_swap_control("--swapoff-helper"); + *disabled = ret == 0; + KUNIT_ASSERT_EQ(test, ret, 0); + } + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), + mode == 1 ? 0UL : mode == 3 ? missing | BIT(0) : missing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), + mode == 1 || mode == 3 ? 0 : 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, + mode == 3 ? 0xb6 : 0, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + 2 * SZ_4K, + 0, PAGE_SIZE - 2 * SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); +} + +static void user4k_shmem_uffd_swapin_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 0); +} + +static void user4k_shmem_uffd_swapped_truncate_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 1); +} + +static void user4k_shmem_uffd_swapped_partial_hole_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 3); +} + +static void user4k_shmem_uffd_swapoff_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 2); +} + +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAULT_INJECTION_DEBUG_FS) +static void test_disable_failslab(void *unused) +{ + test_swap_control("--failslab-off-helper"); +} +#endif + +static void user4k_shmem_uffd_swap_alloc_fail_test(struct kunit *test) +{ +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAULT_INJECTION_DEBUG_FS) + unsigned int attempt; + + if (!total_swap_pages) { + kunit_skip(test, "requires disposable guest swap"); + return; + } + KUNIT_ASSERT_EQ(test, test_swap_control("--failslab-on-helper"), 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_disable_failslab, NULL), 0); + for (attempt = 1; attempt <= 2; attempt++) { + pgoff_t index = attempt == 1 ? 1 : 64; + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + struct file *file = test_cache_file(test, (index + 1) * PAGE_SIZE, false); + struct swap_iocb *plug = NULL; + struct folio *folio; + unsigned int pending; + unsigned long pfn; + void *entry; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), index, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xb6, SZ_4K); + ret = shmem_uffd_set_missing(folio, index, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, index); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + pfn = folio_pfn(folio); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + /* Fail this task's selected allocation and its allocator retries. */ + WRITE_ONCE(current->make_it_fail, 1); + WRITE_ONCE(current->fail_nth, attempt); + ret = shmem_writeout(folio, &plug, &folios); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + WRITE_ONCE(current->make_it_fail, 0); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_EXPECT_EQ(test, ret, AOP_WRITEPAGE_ACTIVATE); + if (ret != AOP_WRITEPAGE_ACTIVATE) { + if (plug) + swap_write_unplug(plug); + folio_wait_writeback(folio); + folio_put(folio); + return; + } + KUNIT_EXPECT_TRUE(test, folio_test_locked(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + KUNIT_EXPECT_PTR_EQ(test, folio->mapping, file->f_mapping); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, index), missing); + /* A failed multi-level insertion may retain an empty internal node. */ + { + unsigned long cursor = 0; + + KUNIT_EXPECT_PTR_EQ(test, xa_find(&SHMEM_I(file_inode(file))->uffd_missing, + &cursor, ULONG_MAX, XA_PRESENT), NULL); + } + KUNIT_EXPECT_PTR_EQ(test, plug, NULL); + KUNIT_EXPECT_TRUE(test, list_empty(&folios)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, PAGE_SIZE - SZ_4K), NULL); + /* No restart or cache repair: retry exactly the retained folio. */ + KUNIT_EXPECT_EQ(test, folio_pfn(folio), pfn); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + ret = shmem_writeout(folio, &plug, &folios); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = filemap_get_entry(file->f_mapping, index); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + KUNIT_EXPECT_EQ(test, xa_to_value(xa_load(&SHMEM_I(file_inode(file))->uffd_missing, index)), missing); + ret = shmem_get_folio(file_inode(file), index, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, index), missing); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, PAGE_SIZE - SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); + kunit_info(test, "failed metadata allocation %u at index %lu; same-folio retry and disk swap-in preserved bytes/mask", attempt, index); + } +#else + kunit_skip(test, "requires FAILSLAB and fault-injection debugfs"); +#endif +} + +#ifdef CONFIG_MEMCG +static void test_memcg_put(void *memcg) +{ + mem_cgroup_put(memcg); +} +#endif + +static void user4k_shmem_uffd_swap_charge_test(struct kunit *test) +{ +#ifdef CONFIG_MEMCG + struct cgroup *cgroup; + struct cgroup_subsys_state *css; + struct mem_cgroup *memcg, *saved; + struct file *file; + struct folio *folio; + struct swap_iocb *plug = NULL; + void *entry; + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages || mem_cgroup_disabled()) { + kunit_skip(test, "requires disposable guest swap and memcg"); + return; + } + KUNIT_ASSERT_EQ(test, test_swap_control("--memcg-setup-helper"), 0); + cgroup = cgroup_get_from_path("/uffd-metadata"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cgroup); + css = cgroup_get_e_css(cgroup, &memory_cgrp_subsys); + cgroup_put(cgroup); + KUNIT_ASSERT_NOT_NULL(test, css); + memcg = mem_cgroup_from_css(css); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_memcg_put, memcg), 0); + KUNIT_ASSERT_PTR_NE(test, memcg, root_mem_cgroup); + file = test_cache_file(test, 65 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + saved = set_active_memcg(memcg); + ret = shmem_get_folio(file_inode(file), 64, 0, &folio, SGP_CACHE); + set_active_memcg(saved); + KUNIT_ASSERT_EQ(test, ret, 0); + rcu_read_lock(); + KUNIT_EXPECT_PTR_EQ(test, folio_memcg(folio), memcg); + rcu_read_unlock(); + ret = shmem_uffd_set_missing(folio, 64, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, 64); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + /* Reclaim from the root charge domain, not the folio's cgroup. */ + saved = set_active_memcg(root_mem_cgroup); + ret = shmem_writeout(folio, &plug, &folios); + set_active_memcg(saved); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + rcu_read_lock(); + entry = rcu_dereference(SHMEM_I(file_inode(file))->uffd_missing.xa_head); + if (xa_is_node(entry)) + KUNIT_EXPECT_PTR_EQ(test, mem_cgroup_from_virt(xa_to_node(entry)), memcg); + rcu_read_unlock(); + KUNIT_ASSERT_TRUE(test, xa_is_node(entry)); + entry = filemap_get_entry(file->f_mapping, 64); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + ret = shmem_get_folio(file_inode(file), 64, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 64), missing); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); +#else + kunit_skip(test, "requires memcg"); +#endif +} + +static bool test_coarse_swap_enabled(struct kunit *test) +{ + if (!test_coarse_file_enabled(test)) + return false; + if (!total_swap_pages || !IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "requires disposable coarse-shmem swap runner"); + return false; + } + return true; +} + +static bool test_coarse_swap_out(struct kunit *test, struct file *file, pgoff_t index) +{ + unsigned int tries; + + for (tries = 0; tries < 100; tries++) { + struct folio *folio; + void *entry = filemap_get_entry(file->f_mapping, index); + LIST_HEAD(folios); + + if (xa_is_value(entry)) + return test_reclaim_swapcache(radix_to_swp_entry(entry)); + if (!entry) { + KUNIT_FAIL(test, "reclaim removed data without a swap entry"); + return false; + } + folio = entry; + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Real reclaim owns any folios produced by a large-folio split. */ + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + KUNIT_FAIL(test, "file index %lu did not reach swap", index); + return false; +} + +static void test_coarse_cold_shmem(struct kunit *test, bool promoted) +{ + struct file *file; + struct vm_area_struct *coarse, *small; + struct folio *folio; + unsigned int nr = SZ_64K / PAGE_SIZE, i; + unsigned long offset; + vm_fault_t fault; + + if (!test_coarse_swap_enabled(test)) + return; + file = test_cache_file(test, SZ_64K, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + SZ_64K - SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, small); + if (promoted) { + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + folio_put(folio); + } + for (offset = SZ_4K; offset < SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + for (i = 0; i < nr; i++) + KUNIT_ASSERT_TRUE(test, test_coarse_swap_out(test, file, i)); + /* XArray swap entries and absent swapcache prove this is a cold refault. */ + for (i = 0; i < nr; i++) { + void *entry = filemap_get_entry(file->f_mapping, i); + struct folio *cached = NULL; + + KUNIT_EXPECT_TRUE(test, xa_is_value(entry)); + if (entry && !xa_is_value(entry)) { + folio_put(entry); + return; + } + KUNIT_ASSERT_NOT_NULL(test, entry); + cached = swap_cache_get_folio(radix_to_swp_entry(entry)); + KUNIT_EXPECT_PTR_EQ(test, cached, NULL); + if (cached) + folio_put(cached); + } + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, 0UL); + KUNIT_EXPECT_EQ(test, SHMEM_I(file_inode(file))->swapped, (unsigned long)nr); + fault = test_file_fault(coarse, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0x31 + offset / SZ_4K, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, (unsigned long)nr); + KUNIT_EXPECT_EQ(test, SHMEM_I(file_inode(file))->swapped, 0UL); + for (offset = SZ_4K; offset < SZ_64K; offset += SZ_4K) { + pte_t entry; + + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(&folio->page) + offset); + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); +} + +static void user64k_shmem_cold_constituents_test(struct kunit *test) +{ + test_coarse_cold_shmem(test, false); +} + +static void user64k_shmem_cold_promoted_test(struct kunit *test) +{ + test_coarse_cold_shmem(test, true); +} + +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_MEMCG) +static void test_coarse_injection_off(void *unused) +{ + test_swap_control("--coarse-injection-off"); +} + +#endif + +static void user64k_shmem_fault_alloc_fail_test(struct kunit *test) +{ +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_MEMCG) + unsigned int mode; + + if (!test_coarse_swap_enabled(test)) + return; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_injection_off, NULL), 0); + for (mode = 0; mode < 3; mode++) { + struct file *file = test_cache_file(test, SZ_64K, false); + struct vm_area_struct *vma; + struct folio *folio; + phys_addr_t original[SZ_64K / PAGE_SIZE]; + unsigned char *buffer; + loff_t pos = 0; + unsigned int i, pending = 0; + vm_fault_t result; + bool prior_fail; + char operation[80]; + int enter = 0, leave = 0; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + /* Make the zero-limit charge case nonblocking and avoid an OOM kill. */ + if (mode == 2) + mapping_set_gfp_mask(file->f_mapping, + mapping_gfp_mask(file->f_mapping) & ~__GFP_DIRECT_RECLAIM); + buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + memset(buffer, 0x95, SZ_64K); + KUNIT_ASSERT_EQ(test, kernel_write(file, buffer, SZ_64K, &pos), (ssize_t)SZ_64K); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + folio = filemap_get_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + original[i] = page_to_phys(&folio->page); + folio_put(folio); + } + if (mode < 2) { + KUNIT_ASSERT_EQ(test, test_swap_control(mode ? "--coarse-page-on" : + "--coarse-slab-on"), 0); + } else { + snprintf(operation, sizeof(operation), "--coarse-memcg-enter=%d", + task_pid_nr(current)); + enter = test_swap_control(operation); + } + /* Direct callback isolates backing allocation from page-table allocation. */ + if (!enter) { + struct vm_fault vmf = { .vma = vma, .address = TEST_VA, + .pgoff = 0, .flags = FAULT_FLAG_REMOTE }; + + mmap_read_lock(vma->vm_mm); + prior_fail = current->make_it_fail; + if (!mode) + WRITE_ONCE(current->fail_nth, 1); + else if (mode == 1) + current->make_it_fail = true; + result = vma->vm_ops->fault(&vmf); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + current->make_it_fail = prior_fail; + mmap_read_unlock(vma->vm_mm); + /* Keep unexpected success safe for test cleanup. */ + if (vmf.page) { + if (result & VM_FAULT_LOCKED) + unlock_page(vmf.page); + put_page(vmf.page); + } + } else { + result = 0; + } + if (mode == 2) { + snprintf(operation, sizeof(operation), "--coarse-memcg-leave=%d", + task_pid_nr(current)); + leave = test_swap_control(operation); + } else { + leave = test_swap_control("--coarse-injection-off"); + } + KUNIT_ASSERT_EQ(test, enter, 0); + KUNIT_ASSERT_EQ(test, leave, 0); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_ASSERT_TRUE(test, result & VM_FAULT_OOM); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + folio = filemap_get_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + KUNIT_EXPECT_EQ(test, page_to_phys(&folio->page), original[i]); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x95, PAGE_SIZE), NULL); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, SZ_64K / PAGE_SIZE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SHMEMPAGES), 0L); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x95, SZ_64K), NULL); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + kunit_info(test, "coarse allocation failure mode %u preserved data and recovered", mode); + } +#else + kunit_skip(test, "requires slab/page fault injection and memcg"); +#endif +} + +static void user64k_anon_swap_test(struct kunit *test) +{ + unsigned int mode, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_swap_enabled(test)) + return; + for (mode = 0; mode < 7; mode++) { + struct vm_area_struct *vma, *cvma = NULL; + struct mm_struct *child = NULL; + struct folio *folio, *held = NULL; + swp_entry_t entry, part; + unsigned int i, tries; + pte_t pte; + bool *disabled; + bool warm = !mode || mode == 6; + vm_fault_t fault; + int err; + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x31 + i, SZ_4K); + if (mode == 2 || mode == 6) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + } + if (warm) { + folio_get(folio); + held = folio; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, held), 0); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_EXPECT_EQ(test, min_order_for_split(folio), 16U - PAGE_SHIFT); +#endif + folio_lock(folio); + KUNIT_EXPECT_EQ(test, folio_anon_min_order(folio), 16U - PAGE_SHIFT); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_EXPECT_EQ(test, split_huge_page_to_list_to_order(&folio->page, NULL, 0), + -EBUSY); + KUNIT_EXPECT_EQ(test, folio_order(folio), 16U - PAGE_SHIFT); + expect_coarse_anon_state(test, folio, child ? 2 : 1, !child); +#endif + err = folio_alloc_swap(folio); + if (!err) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, err, 0); + } else { + for (tries = 0; tries < 100; tries++) { + LIST_HEAD(folios); + + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + if (!pte_present(pte)) + break; + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + folio_get(folio); + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + } + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE_MSG(test, pte_present(pte), "mode=%u", mode); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(swp_offset(entry), nr)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 1L); + if (mode == 3) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + } + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), child ? 2 : 1); + if (!warm) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + if (mode == 3) { + /* Exit a swapped child: every constituent count must be released. */ + kunit_release_action(test, user4k_vma_mm_free, child); + child = NULL; + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + } + } + if (mode == 5) { +#if defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_FAULT_INJECTION) + bool old = current->make_it_fail; + + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-page-on"), 0); + current->make_it_fail = true; + fault = test_file_fault(vma, TEST_VA, false); + current->make_it_fail = old; + err = test_swap_control("--coarse-injection-off"); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_OOM); + KUNIT_EXPECT_TRUE(test, pte_same(test_fault_entry(test, vma->vm_mm, TEST_VA), pte)); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + } +#endif + } + if (mode == 4) { + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + err = test_swap_control("--swapoff-helper"); + if (!err) + *disabled = true; + KUNIT_ASSERT_EQ(test, err, 0); + } else { + fault = test_file_fault(vma, TEST_VA + SZ_64K - 1, false); + KUNIT_ASSERT_EQ_MSG(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U, + "mode=%u fault=%x", mode, fault); + if (!warm && mode != 5) + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + } + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE_MSG(test, pte_present(pte), "mode=%u", mode); + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + if (held) { + KUNIT_EXPECT_PTR_EQ(test, folio, held); /* Warm aligned cache: no copy. */ + } else { + folio_get(folio); + held = folio; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, held), 0); + } + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + if (child) { + struct folio *copied; + + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), page_to_phys(&folio->page)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + copied = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(child, pte)))); + expect_coarse_anon_state(test, copied, 1, true); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(copied) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + memset(folio_address(copied), 0xa7, SZ_64K); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, child); + } + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + if (held) { + KUNIT_EXPECT_EQ(test, folio_mapcount(held), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(held), 1); + } + for (i = 0; i < nr; i++) { + struct swap_info_struct *si; + + part = entry; + part.val += i; + si = get_swap_device(part); + if (si) { + KUNIT_EXPECT_FALSE(test, swap_entry_swapped(si, part)); + put_swap_device(si); + } else { + KUNIT_EXPECT_EQ(test, mode, 4U); + } + } + if (mode == 4) + kunit_release_action(test, test_reenable_swap, disabled); + kunit_info(test, "completed coarse anonymous swap mode %u", mode); + } +} + +/* The caller's private fixture has no concurrent PTE writers during reclaim. */ +static void test_coarse_anon_reclaim_once(struct vm_area_struct *vma) +{ + struct folio *folio = test_fault_folio(vma->vm_mm, vma->vm_start); + LIST_HEAD(folios); + + if (!folio) + return; + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); +} + +static void user64k_anon_swap_limit_test(struct kunit *test) +{ +#ifdef CONFIG_MEMCG + struct vm_area_struct *vma; + struct cgroup *cgroup; + struct cgroup_subsys_state *css; + struct mem_cgroup *memcg; + struct folio *folio; + swp_entry_t entry, part; + phys_addr_t original; + unsigned int i, tries, nr = SZ_64K / PAGE_SIZE; + long before, denied; + char operation[80]; + vm_fault_t fault; + pte_t pte; + int ret, leave; + + if (!test_coarse_swap_enabled(test)) + return; + if (mem_cgroup_disabled()) { + kunit_skip(test, "requires memory cgroups"); + return; + } + snprintf(operation, sizeof(operation), "--coarse-swap-enter=%d", task_pid_nr(current)); + KUNIT_ASSERT_EQ(test, test_swap_control(operation), 0); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + fault = vma ? test_file_fault(vma, TEST_VA, true) : VM_FAULT_OOM; + snprintf(operation, sizeof(operation), "--coarse-swap-leave=%d", task_pid_nr(current)); + leave = test_swap_control(operation); + KUNIT_ASSERT_EQ(test, leave, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + cgroup = cgroup_get_from_path("/coarse-swap-limit"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cgroup); + css = cgroup_get_e_css(cgroup, &memory_cgrp_subsys); + cgroup_put(cgroup); + KUNIT_ASSERT_NOT_NULL(test, css); + memcg = mem_cgroup_from_css(css); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_memcg_put, memcg), 0); + KUNIT_ASSERT_PTR_NE(test, memcg, root_mem_cgroup); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + rcu_read_lock(); + KUNIT_EXPECT_PTR_EQ(test, folio_memcg(folio), memcg); + rcu_read_unlock(); + original = page_to_phys(&folio->page); + memset(folio_address(folio), 0xc9, SZ_64K); + before = atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]); + folio_lock(folio); + ret = folio_alloc_swap(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + denied = atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]); + KUNIT_EXPECT_GT(test, denied, before); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); + /* Exercise vmscan's failed-allocation/split fallback without a retained GET. */ + for (tries = 0; tries < 8; tries++) + test_coarse_anon_reclaim_once(vma); + KUNIT_EXPECT_GT(test, atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]), denied); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), original); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xc9, SZ_64K), NULL); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-swap-unlimit"), 0); + for (tries = 0; tries < 100; tries++) { + test_coarse_anon_reclaim_once(vma); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + if (!pte_present(pte)) + break; + } + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), (unsigned long)nr); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xc9, SZ_64K), NULL); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); +#else + kunit_skip(test, "requires memory cgroups"); +#endif +} + +struct coarse_swap_worker { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready, go, entered, done; + u8 *buffer; + unsigned int id; + bool *disabled; + int error; +}; + +static int coarse_swap_worker_run(void *arg) +{ + struct coarse_swap_worker *w = arg; + unsigned int i; + u8 value = 0x91 + w->id; + + complete(&w->ready); + if (!wait_for_completion_timeout(&w->go, 15 * HZ) || kthread_should_stop()) { + w->error = -ETIMEDOUT; + goto out; + } + if (!w->mm) { + complete(&w->entered); + w->error = test_swap_control("--swapoff-helper"); + if (!w->error) + WRITE_ONCE(*w->disabled, true); + goto out; + } + kthread_use_mm(w->mm); + complete(&w->entered); + if (copy_from_user(w->buffer, (void __user *)TEST_VA, SZ_64K)) { + w->error = 1; + goto unuse; + } + for (i = 0; i < SZ_64K; i++) { + if (w->buffer[i] != (u8)(0x31 + i / SZ_4K)) { + w->error = 2; + goto unuse; + } + } + if (copy_to_user((void __user *)(TEST_VA + SZ_64K - 1), &value, 1) || + copy_from_user(w->buffer, (void __user *)TEST_VA, SZ_64K)) { + w->error = 3; + goto unuse; + } + for (i = 0; i < SZ_64K; i++) { + u8 expected = i == SZ_64K - 1 ? value : 0x31 + i / SZ_4K; + + if (w->buffer[i] != expected) { + w->error = 4; + break; + } + } +unuse: + kthread_unuse_mm(w->mm); +out: + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void coarse_swap_worker_stop(void *arg) +{ + struct coarse_swap_worker *w = arg; + + if (w->task) { + complete_all(&w->go); + kthread_stop(w->task); + w->task = NULL; + } +} + +static void user64k_anon_swap_race_test(struct kunit *test) +{ + unsigned int round, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_swap_enabled(test)) + return; + if (num_online_cpus() < 2 || !system_supports_64kb_granule()) { + kunit_skip(test, "requires two CPUs and 64K TTBR0 support"); + return; + } + for (round = 0; round < 8; round++) { + struct vm_area_struct *vma; + struct mm_struct *child; + struct coarse_swap_worker *workers; + struct folio *folio, *blocker; + struct swap_info_struct *si; + swp_entry_t entry, part; + bool *disabled, started = true, blocked; + bool off_started = false, cold = round & 1; + unsigned int i, j, tries; + pte_t pte; + int ret, cpu; + + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x31 + i, SZ_4K); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + if (!ret) { + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + } + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 2); + if (cold) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + part = entry; + part.val += nr - 1; + blocker = read_swap_cache_async(part, GFP_KERNEL, vma, + TEST_VA + SZ_64K - PAGE_SIZE, NULL); + KUNIT_ASSERT_NOT_NULL(test, blocker); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, blocker), 0); + KUNIT_EXPECT_EQ(test, folio_size(blocker), cold ? PAGE_SIZE : (unsigned long)SZ_64K); + workers = kunit_kcalloc(test, 3, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 3; i++) { + workers[i].mm = i < 2 ? (i ? child : vma->vm_mm) : NULL; + workers[i].id = i; + workers[i].disabled = disabled; + workers[i].buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers[i].buffer); + init_completion(&workers[i].ready); + init_completion(&workers[i].go); + init_completion(&workers[i].entered); + init_completion(&workers[i].done); + workers[i].task = kthread_create(coarse_swap_worker_run, &workers[i], + "user64k-swap-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "coarse swap worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + coarse_swap_worker_stop, &workers[i]), 0); + if (i < 2) { + kthread_bind(workers[i].task, cpu); + cpu = cpumask_next(cpu, cpu_online_mask); + } + wake_up_process(workers[i].task); + } + for (i = 0; i < 3; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + folio_lock(blocker); + for (i = 0; i < 3; i++) + complete(&workers[i].go); + for (i = 0; i < 3; i++) + started &= !!wait_for_completion_timeout(&workers[i].entered, HZ); + for (tries = 0; tries < 3 * HZ; tries++) { + si = get_swap_device(entry); + if (si) { + off_started = !(READ_ONCE(si->flags) & SWP_WRITEOK); + put_swap_device(si); + } + if (off_started) + break; + schedule_timeout_uninterruptible(1); + } + blocked = !completion_done(&workers[0].done) && + !completion_done(&workers[1].done) && !completion_done(&workers[2].done); + /* Never execute a fatal assertion while retaining the blocking lock. */ + folio_unlock(blocker); + KUNIT_EXPECT_TRUE(test, started); + KUNIT_EXPECT_TRUE(test, off_started); + KUNIT_EXPECT_TRUE(test, blocked); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 20 * HZ), 0UL); + KUNIT_EXPECT_EQ_MSG(test, workers[i].error, 0, "round=%u worker=%u", round, i); + kunit_release_action(test, coarse_swap_worker_stop, &workers[i]); + } + KUNIT_EXPECT_TRUE(test, *disabled); + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? child : vma->vm_mm; + + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (j = 0; j < SZ_64K; j++) { + u8 expected = j == SZ_64K - 1 ? 0x91 + i : 0x31 + j / SZ_4K; + + if (((u8 *)folio_address(folio))[j] != expected) { + KUNIT_FAIL(test, "round=%u side=%u byte=%u", round, i, j); + break; + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1L); + kunit_release_action(test, user4k_vma_mm_free, mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), folio == blocker ? 2 : 1); + kunit_release_action(test, test_folio_put, folio); + } + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(blocker)); + KUNIT_EXPECT_EQ(test, folio_ref_count(blocker), 1); + kunit_release_action(test, test_folio_put, blocker); + kunit_release_action(test, test_reenable_swap, disabled); + KUNIT_ASSERT_GT(test, total_swap_pages, 0UL); + } + kunit_info(test, "8 controlled warm/cold fault+COW+swapoff rounds completed"); +} + +struct coarse_swap_reservation { + struct folio **folios; + unsigned int count; +}; + +static void test_coarse_release_reservation(struct folio *folio) +{ + folio_lock(folio); + folio_put_swap(folio, NULL); + folio_free_swap(folio); + folio_unlock(folio); + folio_put(folio); +} + +static void test_coarse_release_reservations(void *arg) +{ + struct coarse_swap_reservation *r = arg; + unsigned int i; + + for (i = 0; i < r->count; i++) { + if (!r->folios[i]) + continue; + test_coarse_release_reservation(r->folios[i]); + r->folios[i] = NULL; + cond_resched(); + } +} + +static void test_coarse_swap_refusal(struct kunit *test, struct vm_area_struct *vma, + phys_addr_t original) +{ + struct folio *folio = test_fault_folio(vma->vm_mm, TEST_VA); + pte_t pte; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + for (i = 0; i < 4; i++) + test_coarse_anon_reclaim_once(vma); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), original); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xd6, SZ_64K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); +} + +static void user64k_anon_swap_fragment_test(struct kunit *test) +{ + struct coarse_swap_reservation *r; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int i, nr = SZ_64K / PAGE_SIZE, released = 0; + unsigned long hole, free_before; + phys_addr_t original; + swp_entry_t entry, part; + vm_fault_t fault; + pte_t pte; + int ret = 0; + + if (!test_coarse_swap_enabled(test)) + return; + /* Exhaust only the bounded disposable disk owned by this test's PID1. */ + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-fixture-check"), 0); + KUNIT_ASSERT_LT(test, total_swap_pages, (unsigned long)(SZ_256M / PAGE_SIZE)); + KUNIT_ASSERT_GT(test, total_swap_pages, 2UL * SWAPFILE_CLUSTER); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + original = page_to_phys(&folio->page); + memset(folio_address(folio), 0xd6, SZ_64K); + folio_put(folio); + r = kunit_kzalloc(test, sizeof(*r), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, r); + r->folios = kunit_kcalloc(test, total_swap_pages + 1, sizeof(*r->folios), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, r->folios); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_release_reservations, r), 0); + /* Each cache-only folio has one explicit slot owner, so allocation cannot reclaim it. */ + for (i = 0; i <= total_swap_pages; i++) { + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL); + if (ret) { + folio_put(folio); + KUNIT_FAIL(test, "reservation memory charge failed"); + return; + } + __folio_set_swapbacked(folio); + __folio_mark_uptodate(folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) { + ret = folio_dup_swap(folio, NULL); + if (ret) + folio_free_swap(folio); + } + folio_unlock(folio); + if (ret) { + folio_put(folio); + break; + } + r->folios[r->count++] = folio; + cond_resched(); + } + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + KUNIT_ASSERT_EQ(test, atomic_long_read(&nr_swap_pages), 0L); + test_coarse_swap_refusal(test, vma, original); + /* Retain exactly one native entry in each naturally aligned coarse group. */ + for (i = 0; i < r->count; i++) { + folio = r->folios[i]; + if (swp_offset(folio->swap) % nr == 1) + continue; + test_coarse_release_reservation(folio); + r->folios[i] = NULL; + } + free_before = atomic_long_read(&nr_swap_pages); + KUNIT_EXPECT_GT(test, free_before, total_swap_pages / 2); + KUNIT_EXPECT_LT(test, free_before, total_swap_pages); + test_coarse_swap_refusal(test, vma, original); + /* Free one aligned group near the end, without freeing a whole native-order cluster. */ + hole = round_down(total_swap_pages + 1, (unsigned long)SWAPFILE_CLUSTER) - nr; + for (i = 0; i < r->count; i++) { + folio = r->folios[i]; + if (!folio || swp_offset(folio->swap) < hole || swp_offset(folio->swap) >= hole + nr) + continue; + test_coarse_release_reservation(folio); + r->folios[i] = NULL; + released++; + } + KUNIT_ASSERT_EQ(test, released, 1U); + KUNIT_EXPECT_EQ(test, atomic_long_read(&nr_swap_pages), (long)free_before + 1); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ_MSG(test, ret, 0, "a complete aligned group is free in an order-0 cluster"); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, swp_offset(entry), hole); + for (i = 0; i < r->count; i++) { + if (r->folios[i]) + KUNIT_EXPECT_EQ(test, swp_swapcount(r->folios[i]->swap), 1); + } + kunit_release_action(test, test_coarse_release_reservations, r); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xd6, SZ_64K), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + kunit_info(test, "exhaustion, scattered free space and one-group recovery checked"); +} + +struct coarse_swap_io_mode { + struct swap_info_struct *si; + bool synchronous; +}; + +static void test_coarse_restore_io_mode(void *arg) +{ + struct coarse_swap_io_mode *mode = arg; + + spin_lock(&mode->si->lock); + if (mode->synchronous) + mode->si->flags |= SWP_SYNCHRONOUS_IO; + else + mode->si->flags &= ~SWP_SYNCHRONOUS_IO; + spin_unlock(&mode->si->lock); + put_swap_device(mode->si); +} + +static void user64k_anon_swap_extent_test(struct kunit *test) +{ + unsigned int mode, i; + + if (!test_coarse_swap_enabled(test)) + return; + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-fixture-check"), 0); + for (mode = 0; mode < 2; mode++) { + struct vm_area_struct *vma; + struct coarse_swap_io_mode *saved; + struct folio *folio; + unsigned long nr = SZ_64K / PAGE_SIZE, run = nr; + swp_entry_t entry; + sector_t first, next; + bool file; + int ret; + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x63 + i, SZ_4K); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, folio_mapcount(folio), 0); + entry = folio->swap; + saved = kunit_kzalloc(test, sizeof(*saved), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, saved); + saved->si = get_swap_device(entry); + KUNIT_ASSERT_NOT_NULL(test, saved->si); + saved->synchronous = saved->si->flags & SWP_SYNCHRONOUS_IO; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_restore_io_mode, saved), 0); + file = !(saved->si->flags & SWP_BLKDEV); + KUNIT_ASSERT_FALSE(test, saved->si->flags & SWP_FS_OPS); + first = swap_extent_sector(entry, &run); + if (file) { + unsigned long remaining; + swp_entry_t part = entry; + + KUNIT_ASSERT_LT_MSG(test, run, nr, "fragmented fixture must cross an extent"); + part.val += run; + remaining = nr - run; + next = swap_extent_sector(part, &remaining); + KUNIT_EXPECT_NE(test, next, first + (run << (PAGE_SHIFT - 9))); + } else { + KUNIT_EXPECT_EQ(test, run, nr); + } + /* The verified disposable fixture owns this otherwise idle swap area. */ + spin_lock(&saved->si->lock); + if (mode) + saved->si->flags |= SWP_SYNCHRONOUS_IO; + else + saved->si->flags &= ~SWP_SYNCHRONOUS_IO; + spin_unlock(&saved->si->lock); + folio_lock(folio); + folio_clear_dirty_for_io(folio); + __swap_writepage(folio, NULL); + folio_wait_writeback(folio); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, folio_test_dirty(folio)); + /* A full-folio read must reconstruct all physical runs, not warm cache. */ + memset(folio_address(folio), 0, SZ_64K); + folio_clear_uptodate(folio); + swap_read_folio(folio, NULL); + folio_lock(folio); + KUNIT_EXPECT_TRUE(test, folio_test_uptodate(folio)); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x63 + i, SZ_4K), NULL); + folio_unlock(folio); + kunit_release_action(test, test_coarse_restore_io_mode, saved); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + kunit_info(test, "64K full-folio %s I/O: file=%u first-run=%lu/%lu native pages", + mode ? "synchronous" : "asynchronous", file, run, nr); + } +} + +static struct kunit_case user4k_coarse_shmem_cases[] = { + KUNIT_CASE(user64k_anon_swap_extent_test), + KUNIT_CASE(user64k_anon_swap_test), + KUNIT_CASE(user64k_anon_swap_limit_test), + KUNIT_CASE(user64k_anon_swap_fragment_test), + KUNIT_CASE(user64k_anon_swap_race_test), + KUNIT_CASE(user64k_shmem_cold_constituents_test), + KUNIT_CASE(user64k_shmem_cold_promoted_test), + KUNIT_CASE(user64k_shmem_fault_alloc_fail_test), + {} +}; + +static struct kunit_suite user4k_coarse_shmem_suite = { + .name = "arm64-user4k-coarse-shmem", + .test_cases = user4k_coarse_shmem_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static struct kunit_case user4k_shmem_swap_cases[] = { + KUNIT_CASE(user4k_shmem_uffd_swap_alloc_fail_test), + KUNIT_CASE(user4k_shmem_uffd_swap_charge_test), + KUNIT_CASE(user4k_shmem_uffd_swapin_test), + KUNIT_CASE(user4k_shmem_uffd_swapped_truncate_test), + KUNIT_CASE(user4k_shmem_uffd_swapoff_test), + KUNIT_CASE(user4k_shmem_uffd_swapped_partial_hole_test), + {} +}; + +static struct kunit_suite user4k_shmem_swap_suite = { + .name = "arm64-user4k-shmem-swap", + .test_cases = user4k_shmem_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; +#endif + +static struct kunit_case user4k_nonlinear_swap_cases[] = { +#ifdef CONFIG_USERFAULTFD + KUNIT_CASE(user4k_move_swap_get_test), + KUNIT_CASE(user4k_move_migration_race_test), +#endif + KUNIT_CASE(user4k_nonlinear_swap_cached_test), + KUNIT_CASE(user4k_nonlinear_swap_disk_test), + {} +}; + +static struct kunit_suite user4k_nonlinear_swap_suite = { + .name = "arm64-user4k-nonlinear-swap", + .test_cases = user4k_nonlinear_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static struct kunit_suite user4k_swap_suite = { + .name = "arm64-user4k-swap", + .test_cases = user4k_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static void user4k_thp_eligibility_test(struct kunit *test) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + const enum tva_type types[] = { + TVA_PAGEFAULT, TVA_SMAPS, TVA_KHUGEPAGED, TVA_FORCED_COLLAPSE, + }; + struct vm_area_struct *vma, *normal; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_HUGEPAGE; + + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + vma = test_vma_add(small, SZ_1G, SZ_1G, (struct vm_page_offset) {}, flags); + normal = test_vma_add(native, SZ_1G, SZ_1G, (struct vm_page_offset) {}, flags); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, normal); + /* Neither user advice nor forced collapse can bypass geometry limits. */ + KUNIT_EXPECT_TRUE(test, vma_thp_disabled(vma, flags, false)); + KUNIT_EXPECT_TRUE(test, vma_thp_disabled(vma, flags, true)); + KUNIT_EXPECT_FALSE(test, vma_thp_disabled(normal, flags, false)); + KUNIT_EXPECT_FALSE(test, vma_thp_disabled(normal, flags, true)); + for (unsigned int i = 0; i < ARRAY_SIZE(types); i++) + KUNIT_EXPECT_EQ(test, __thp_vma_allowable_orders(vma, flags, + types[i], THP_ORDERS_ALL_ANON), 0UL); +#else + kunit_skip(test, "requires transparent hugepage support"); +#endif +} + +static void user4k_aslr_window_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + const unsigned long windows[] = { SZ_8M, SZ_32M, SZ_1G }; + + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + for (unsigned int i = 0; i < ARRAY_SIZE(windows); i++) { + unsigned long mask = (windows[i] >> PAGE_SHIFT) - 1; + unsigned long converted = mm_aslr_mask(small, mask); + + KUNIT_EXPECT_EQ(test, mm_aslr_mask(native, mask), mask); + KUNIT_EXPECT_EQ(test, (converted + 1) << mm_page_shift(small), + windows[i]); + KUNIT_EXPECT_EQ(test, converted & (TEST_SLOTS - 1), + (unsigned long)TEST_SLOTS - 1); + } +} + +static void user4k_exec_granule_policy_test(struct kunit *test) +{ + /* Model all native granules, including AArch32 with NO 4K support. */ + const unsigned int native[] = { 12, 14, 16 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(native); i++) { + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, false, native[i], 0), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, false, native[i], 12), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, true, native[i], 0), + 12U); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(false, true, native[i], 0), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(false, true, native[i], 12), + 12U); + } +} + +static void user_granule_zero_region_test(struct kunit *test) +{ + unsigned long base_pfn = zero_pfn(0); + unsigned int shift, i; + u8 *buffer; + + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(PFN_PHYS(base_pfn), SZ_64K)); + KUNIT_EXPECT_FALSE(test, is_zero_pfn(base_pfn - 1)); + KUNIT_EXPECT_FALSE(test, is_zero_pfn(base_pfn + SZ_64K / PAGE_SIZE)); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + KUNIT_EXPECT_TRUE(test, is_zero_pfn(base_pfn + i)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(pfn_to_page(base_pfn + i)), + 0, PAGE_SIZE), NULL); + } + buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + for (shift = 12; shift <= 16; shift += 2) { + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned long size = 1UL << shift, left; + vm_fault_t ret; + spinlock_t *ptl; + pte_t *pte, entry; + + if ((shift == 12 && !system_supports_4kb_granule()) || + (shift == 14 && !system_supports_16kb_granule()) || + (shift == 16 && !system_supports_64kb_granule())) { + kunit_info(test, "hardware zero mapping skipped for shift %u", shift); + continue; + } + /* Internal coarse mm: the public opt-in remains disabled. */ + mm = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = test_fault_vma_in_mm(mm, TEST_VA, size); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + size - 1, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_special(entry)); + KUNIT_EXPECT_FALSE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), PFN_PHYS(base_pfn)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + + memset(buffer, 0xa5, size); + kthread_use_mm(mm); + left = copy_from_user(buffer, (void __user *)TEST_VA, size); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, left, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, size), NULL); + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, size); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + } +} + +#ifdef CONFIG_COREDUMP +static void user_granule_core_offsets_test(struct kunit *test) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + struct vm_area_struct *vma = kunit_kzalloc(test, sizeof(*vma), GFP_KERNEL); + unsigned int shift; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_mm = mm; + for (shift = 12; shift <= 16; shift += 2) { + unsigned long bytes = (1UL << 55) + 3 * SZ_64K + (1UL << shift); + + mm->context.flags = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = bytes >> PAGE_SHIFT, .offset = bytes & ~PAGE_MASK, + }); + KUNIT_EXPECT_EQ(test, core_vma_pgoff(vma), bytes >> shift); + KUNIT_EXPECT_EQ(test, mm_user_fragment_size(mm), min(1UL << shift, PAGE_SIZE)); + vma_set_page_offset(vma, (struct vm_page_offset) {}); + KUNIT_EXPECT_EQ(test, core_vma_pgoff(vma), 0UL); + } +} +#endif + +static void user64k_fragment_ownership_test(struct kunit *test) +{ + struct user_page_fragment fragments[SZ_64K / PAGE_SIZE] = {}; + struct mm_struct *parent, *child; + struct vm_area_struct *vma, *cvma; + struct folio *original, *copied; + unsigned int i, nr = ARRAY_SIZE(fragments); + long count; + int ret, refs; + pte_t entry; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires a user leaf larger than native pages"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + count = get_user_fragments_remote(parent, TEST_VA + 17, SZ_64K - 34, + 0, fragments, nr); + if (count > 0) { + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &fragments[i]; + unsigned int bytes = PAGE_SIZE - (i == 0 || i == nr - 1 ? 17 : 0); + + KUNIT_EXPECT_TRUE(test, is_zero_folio(f->folio)); + KUNIT_EXPECT_EQ(test, folio_pfn(f->folio), zero_pfn(TEST_VA) + i); + KUNIT_EXPECT_PTR_EQ(test, f->subpage, NULL); + KUNIT_EXPECT_EQ(test, f->offset, i == 0 ? 17UL : 0UL); + KUNIT_EXPECT_EQ(test, f->length, bytes); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0, f->length), NULL); + } + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, parent, TEST_VA); + original = page_folio(pfn_to_page(pte_pfn(entry))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(original, i)), 0x51 + i, PAGE_SIZE); + lru_add_drain_all(); + refs = folio_ref_count(original); + count = get_user_fragments_remote(parent, TEST_VA + 17, SZ_64K - 34, + 0, fragments, nr); + if (count > 0) { + KUNIT_EXPECT_EQ(test, folio_ref_count(original), refs + (int)count); + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &fragments[i]; + + KUNIT_EXPECT_PTR_EQ(test, f->folio, original); + KUNIT_EXPECT_EQ(test, f->offset, i * PAGE_SIZE + (i == 0 ? 17UL : 0UL)); + KUNIT_EXPECT_LE(test, offset_in_page(f->offset) + f->length, PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x51 + i, f->length), NULL); + } + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + KUNIT_EXPECT_EQ(test, folio_ref_count(original), refs); + + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + /* A read pin after fork must unshare the complete coarse leaf. */ + count = pin_user_fragments_remote(child, TEST_VA, SZ_64K, 0, fragments, nr); + if (count > 0) { + copied = fragments[0].folio; + KUNIT_EXPECT_PTR_NE(test, copied, original); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(copied)); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, child, TEST_VA))); + for (i = 0; i < count; i++) { + KUNIT_EXPECT_PTR_EQ(test, fragments[i].folio, copied); + KUNIT_EXPECT_PTR_EQ(test, fragments[i].subpage, NULL); + KUNIT_EXPECT_TRUE(test, fragments[i].pinned); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(copied, i))); + KUNIT_EXPECT_EQ(test, fragments[i].offset, i * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, fragments[i].length, (unsigned int)PAGE_SIZE); + } + mmap_read_lock(child); + zap_vma_range(cvma, TEST_VA, SZ_64K); + mmap_read_unlock(child); + KUNIT_EXPECT_EQ(test, folio_mapcount(copied), 0); + for (i = 0; i < count; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(copied) + i * PAGE_SIZE, + 0x51 + i, PAGE_SIZE), NULL); + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(original) + i * PAGE_SIZE, + 0x51 + i, PAGE_SIZE), NULL); +} + +static void user64k_anon_fault_cow_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + struct vm_area_struct *vma, *cvma; + pte_t orig[2], copied; + struct folio *folio; + unsigned int i, j, nr = SZ_64K / PAGE_SIZE; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous faults"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, 2 * SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Leaf0 exercises zero-page COW; leaf1 exercises a direct write fault. */ + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, false) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_TRUE(test, pte_special(test_fault_entry(test, parent, TEST_VA))); + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + + KUNIT_ASSERT_EQ(test, test_file_fault(vma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + orig[i] = test_fault_entry(test, parent, addr); + KUNIT_ASSERT_TRUE(test, pte_present(orig[i])); + KUNIT_EXPECT_TRUE(test, pte_write(orig[i])); + KUNIT_EXPECT_FALSE(test, pte_special(orig[i])); + folio = page_folio(pfn_to_page(pte_pfn(orig[i]))); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, SZ_64K), NULL); + for (j = 0; j < nr; j++) { + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(folio, j))); + memset(page_address(folio_page(folio, j)), 0x31 + i * nr + j, PAGE_SIZE); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 2L); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 2L); + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + struct folio *old = page_folio(pfn_to_page(pte_pfn(orig[i]))); + + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, test_fault_entry(test, child, addr)), + pte_phys_mm(parent, orig[i])); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, parent, addr))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)(2 * nr)); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + copied = test_fault_entry(test, child, addr); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, copied), pte_phys_mm(parent, orig[i])); + folio = page_folio(pfn_to_page(pte_pfn(copied))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)nr); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + for (j = 0; j < nr; j++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(folio, j)), + 0x31 + i * nr + j, PAGE_SIZE), NULL); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(folio, j))); + } + memset(folio_address(folio), 0xa5, SZ_64K); + for (j = 0; j < nr; j++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(old, j)), + 0x31 + i * nr + j, PAGE_SIZE), NULL); + /* Parent reuse must restore exclusivity across the entire leaf. */ + lru_add_drain_all(); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, pte_phys_mm(parent, test_fault_entry(test, parent, addr)), + pte_phys_mm(parent, orig[i])); + for (j = 0; j < nr; j++) + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(old, j))); + } + if (system_supports_64kb_granule()) { + long protect[2]; + bool writable = false; + + kthread_use_mm(parent); + protect[0] = test_mprotect(TEST_VA, 2 * SZ_64K, PROT_READ); + kthread_unuse_mm(parent); + for (i = 0; i < 2; i++) + writable |= pte_write(test_fault_entry(test, parent, TEST_VA + i * SZ_64K)); + KUNIT_EXPECT_EQ(test, protect[0], 0L); + KUNIT_EXPECT_FALSE(test, writable); + kthread_use_mm(parent); + protect[1] = test_mprotect(TEST_VA, 2 * SZ_64K, PROT_READ | PROT_WRITE); + kthread_unuse_mm(parent); + KUNIT_EXPECT_EQ(test, protect[1], 0L); + for (i = 0; i < 2; i++) + KUNIT_EXPECT_TRUE(test, pte_write(test_fault_entry(test, parent, TEST_VA + i * SZ_64K))); + vma = test_vma_lookup(parent, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + } + mmap_read_lock(child); + zap_vma_range(cvma, TEST_VA, 2 * SZ_64K); + mmap_read_unlock(child); + mmap_read_lock(parent); + zap_vma_range(vma, TEST_VA, 2 * SZ_64K); + mmap_read_unlock(parent); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 0L); +} + +struct coarse_test_pin { + struct page *page; +}; + +static void coarse_test_unpin(void *arg) +{ + struct coarse_test_pin *pin = arg; + + if (pin->page) { + unpin_user_page(pin->page); + pin->page = NULL; + } +} + +static void user64k_pinned_fork_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + struct vm_area_struct *vma; + struct coarse_test_pin *pin; + struct folio *old, *new; + pte_t original, copied; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires pinned coarse leaf"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + original = test_fault_entry(test, parent, TEST_VA); + old = page_folio(pfn_to_page(pte_pfn(original))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(old, i)), 0x70 + i, PAGE_SIZE); + pin = kunit_kzalloc(test, sizeof(*pin), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pin); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_test_unpin, pin), 0); + mm_flags_set(MMF_HAS_PINNED, parent); + ret = try_grab_folio(old, 1, FOLL_PIN); + if (!ret) + pin->page = folio_page(old, nr - 1); + KUNIT_ASSERT_EQ(test, ret, 0); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + copied = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, copied), pte_phys_mm(parent, original)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(parent, test_fault_entry(test, parent, TEST_VA)), + pte_phys_mm(parent, original)); + new = page_folio(pfn_to_page(pte_pfn(copied))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)nr); + KUNIT_EXPECT_EQ(test, folio_mapcount(new), (int)nr); + for (i = 0; i < nr; i++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(new, i)), + 0x70 + i, PAGE_SIZE), NULL); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(new, i))); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(old, i))); + } + memset(page_address(pin->page), 0xe3, PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(new, nr - 1)), + 0x70 + nr - 1, PAGE_SIZE), NULL); + coarse_test_unpin(pin); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1L); +} + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +static void user4k_shmem_partial_hole_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 3 * PAGE_SIZE, true); + struct folio *folio; + loff_t start = PAGE_SIZE + SZ_4K + 17, end = PAGE_SIZE + 3 * SZ_4K + 16; + unsigned int i; + unsigned char byte = 0xa5; + loff_t write_pos = PAGE_SIZE + 2 * SZ_4K + 23; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), start, end); + folio = filemap_lock_folio(file->f_mapping, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), BIT(2)); + for (i = 0; i < PAGE_SIZE; i++) { + unsigned char expected = (i + PAGE_SIZE >= start && i + PAGE_SIZE <= end) ? + 0 : 0x31 + (PAGE_SIZE + i) / SZ_4K; + + if (((unsigned char *)folio_address(folio))[i] != expected) { + KUNIT_FAIL(test, "partial hole changed byte %u outside its range", i); + break; + } + } + /* Resident hole metadata has no dynamically allocated inode record. */ + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); + shmem_truncate_range(file_inode(file), PAGE_SIZE + SZ_4K, PAGE_SIZE + 2 * SZ_4K - 1); + KUNIT_ASSERT_EQ(test, kernel_write(file, &byte, 1, &write_pos), 1L); + folio = filemap_lock_folio(file->f_mapping, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), BIT(1)); + KUNIT_EXPECT_EQ(test, ((unsigned char *)folio_address(folio))[2 * SZ_4K + 23], byte); + folio_unlock(folio); + folio_put(folio); +} + +/* A surviving writable pin exposes bytes even after its PTE is revoked. */ +static void user4k_shmem_pinned_hole_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + struct folio *folio; + unsigned long offset = native ? 0 : SZ_4K; + unsigned long length = native ? PAGE_SIZE : SZ_4K; + unsigned long pfn; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, native, true, TEST_VA, length, offset); + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + KUNIT_ASSERT_EQ(test, fragment_test_pin(pins, vma->vm_mm, TEST_VA, + length, FOLL_WRITE | FOLL_LONGTERM, 1), 1L); + folio = pins->fragments[0].folio; + pfn = folio_pfn(folio); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, SZ_4K), NULL); + /* Model a completed device write through the original pin. */ + memset(folio_address(folio) + SZ_4K, 0xb7, SZ_4K); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + folio_unlock(folio); + fragment_test_unpin(pins); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_pfn(folio), pfn); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xb7, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x31, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + 2 * SZ_4K, 0x33, SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); + /* With the external writer gone, another hole can be missing. */ + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + folio_unlock(folio); + folio_put(folio); + } +} + +static void test_unpin_folio(void *folio) +{ + unpin_folio(folio); +} + +/* The direct memfd API must publish cache exposure without a user PTE. */ +static void user4k_shmem_memfd_pin_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *pin, *folio; + pgoff_t offset; + long ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + ret = memfd_pin_folios(file, SZ_4K, 2 * SZ_4K - 1, &pin, 1, &offset); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_unpin_folio, pin), 0); + KUNIT_EXPECT_EQ(test, offset, (pgoff_t)SZ_4K); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(pin)); + memset(folio_address(pin) + offset, 0xc8, SZ_4K); + folio_lock(pin); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(pin, 0, SZ_4K, SZ_4K)); + folio_unlock(pin); + kunit_release_action(test, test_unpin_folio, pin); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xc8, SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); +} + +static void user4k_shmem_replace_validity_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *old, *new; + unsigned long old_pfn; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + new = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, new); + folio_lock(new); + old = filemap_lock_folio(file->f_mapping, 0); + if (IS_ERR(old)) { + folio_unlock(new); + folio_put(new); + KUNIT_FAIL(test, "could not lock source cache folio"); + return; + } + old_pfn = folio_pfn(old); + memcpy(folio_address(new), folio_address(old), PAGE_SIZE); + __folio_set_swapbacked(new); + __folio_mark_uptodate(new); + folio_mark_dirty(new); + replace_page_cache_folio(old, new); + folio_add_lru(new); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(new, 0), BIT(1)); + KUNIT_EXPECT_NE(test, folio_pfn(new), old_pfn); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(new), 0x31, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(new) + SZ_4K, 0, SZ_4K), NULL); + folio_unlock(old); + folio_put(old); + folio_unlock(new); + folio_put(new); +} + +static void user4k_shmem_uffd_alias_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, false); + const unsigned long all = GENMASK(PAGE_SIZE / SZ_4K - 1, 0); + struct vm_area_struct *small, *native; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = shmem_uffd_set_missing(folio, 0, all); + KUNIT_EXPECT_TRUE(test, shmem_uffd_has_missing_native_page(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + small = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), all & ~BIT(1)); + KUNIT_EXPECT_FALSE(test, shmem_uffd_has_missing_native_page(folio)); + folio_unlock(folio); + folio_put(folio); + + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), 0UL); + folio_unlock(folio); + folio_put(folio); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); +} + +static void user4k_shmem_uffd_state_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 3 * PAGE_SIZE, false); + const unsigned long all = GENMASK(PAGE_SIZE / SZ_4K - 1, 0); + struct folio *folio; + unsigned long mask; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + for (i = 0; i < 3; i++) { + ret = shmem_get_folio(file_inode(file), i, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), 0UL); + ret = shmem_uffd_set_missing(folio, i, all & ~BIT(0)); + KUNIT_EXPECT_EQ(test, ret, 0); + shmem_uffd_mark_present(folio, i, BIT(1)); + mask = shmem_uffd_missing_mask(folio, i); + KUNIT_EXPECT_EQ(test, mask, all & ~GENMASK(1, 0)); + ret = shmem_uffd_set_missing(folio, i, BIT(PAGE_SIZE / SZ_4K)); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), mask); + folio_unlock(folio); + folio_put(folio); + } + /* Real truncate removes one index without discarding neighbouring state. */ + shmem_truncate_range(file_inode(file), PAGE_SIZE, 2 * PAGE_SIZE - 1); + KUNIT_EXPECT_PTR_EQ(test, xa_load(&SHMEM_I(file_inode(file))->uffd_missing, 1), NULL); + for (i = 0; i < 3; i += 2) { + folio = filemap_lock_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), all & ~GENMASK(1, 0)); + folio_unlock(folio); + folio_put(folio); + } + /* Generic batch deletion must not strand inode metadata either. */ + truncate_inode_pages(file->f_mapping, 0); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + ret = shmem_get_folio(file_inode(file), 1, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, PAGE_SIZE), NULL); + folio_unlock(folio); + folio_put(folio); +} + +#ifdef CONFIG_MIGRATION +static void user4k_shmem_uffd_migration_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, false); + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + struct folio *folio; + unsigned long old_pfn; + unsigned int succeeded = 0; + bool isolated; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xa5, SZ_4K); + folio_mark_dirty(folio); + ret = shmem_uffd_set_missing(folio, 0, missing); + old_pfn = folio_pfn(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, succeeded, 1U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_NE(test, folio_pfn(folio), old_pfn); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), missing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xa5, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, + PAGE_SIZE - SZ_4K), NULL); + shmem_uffd_mark_present(folio, 0, missing); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), 0UL); + folio_unlock(folio); + folio_put(folio); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); +} +#endif +#endif + +static struct kunit_case user4k_test_cases[] = { + KUNIT_CASE(user4k_exec_granule_policy_test), + KUNIT_CASE(user4k_aslr_window_test), + KUNIT_CASE(user4k_thp_eligibility_test), + KUNIT_CASE(user4k_geometry_test), + KUNIT_CASE(user4k_leaf_encoding_test), + KUNIT_CASE(user16k_leaf_encoding_test), + KUNIT_CASE(user4k_rmap_walk_test), + KUNIT_CASE(user4k_rmap_boundary_test), + KUNIT_CASE(native_rmap_walk_test), + KUNIT_CASE(user64k_rmap_lookup_test), + KUNIT_CASE(user64k_anon_rmap_span_test), + KUNIT_CASE(user64k_anon_zap_batch_test), + KUNIT_CASE(user_granule_zero_region_test), +#ifdef CONFIG_COREDUMP + KUNIT_CASE(user_granule_core_offsets_test), +#endif + KUNIT_CASE(user64k_fragment_ownership_test), + KUNIT_CASE(user64k_anon_fault_cow_test), + KUNIT_CASE(user64k_pinned_fork_test), + KUNIT_CASE(user64k_file_rmap_span_test), + KUNIT_CASE(user4k_vma_offsets_test), + KUNIT_CASE(user4k_vma_intervals_test), + KUNIT_CASE(user4k_vma_split_test), + KUNIT_CASE(user4k_vma_merge_test), + KUNIT_CASE(user4k_vma_partial_merge_test), + KUNIT_CASE(user4k_vma_copy_shrink_test), + KUNIT_CASE(user4k_anon_rmap_test), + KUNIT_CASE(user4k_anon_fork_rmap_test), + KUNIT_CASE(user4k_nonlinear_rmap_reuse_test), + KUNIT_CASE(user4k_dup_mmap_test), + KUNIT_CASE(user4k_anon_root_lifetime_test), + KUNIT_CASE(user4k_data_tlb_batch_test), + KUNIT_CASE(user4k_anon_zap_test), + KUNIT_CASE(user4k_anon_writable_slots_test), + KUNIT_CASE(user4k_anon_write_fault_test), + KUNIT_CASE(user16k_anon_write_fault_test), + KUNIT_CASE(user4k_mlock_counts_test), + KUNIT_CASE(user4k_mlock_shared_test), + KUNIT_CASE(user4k_mlock_repair_test), + KUNIT_CASE(user_granule_large_vma_split_test), + KUNIT_CASE(user_granule_brk_extent_test), + KUNIT_CASE(user4k_droppable_reclaim_test), + KUNIT_CASE(user4k_lazyfree_test), + KUNIT_CASE(user4k_zero_cow_test), + KUNIT_CASE(user16k_zero_cow_test), + KUNIT_CASE(user4k_shmem_swap_usage_test), +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + KUNIT_CASE(user4k_shmem_partial_hole_test), + KUNIT_CASE(user4k_shmem_pinned_hole_test), + KUNIT_CASE(user4k_shmem_memfd_pin_test), + KUNIT_CASE(user4k_shmem_replace_validity_test), + KUNIT_CASE(user4k_shmem_uffd_alias_test), + KUNIT_CASE(user4k_shmem_uffd_state_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_shmem_uffd_migration_test), +#endif +#endif + KUNIT_CASE(user4k_file_mapcounts_test), + KUNIT_CASE(user64k_file_promote_test), + KUNIT_CASE(user64k_shmem_fault_test), + KUNIT_CASE(user64k_shmem_fault_busy_test), + KUNIT_CASE(user64k_shmem_fault_hole_test), + KUNIT_CASE(user64k_file_leaf_test), + KUNIT_CASE(user64k_file_cow_test), + KUNIT_CASE(user64k_file_bounds_test), + KUNIT_CASE(user4k_file_shared_test), + KUNIT_CASE(user4k_iommufd_lifetime_test), + KUNIT_CASE(user4k_iommufd_rollback_test), + KUNIT_CASE(user4k_iommufd_granule_test), + KUNIT_CASE(user4k_iommufd_shared_test), + KUNIT_CASE(user4k_iommufd_exit_account_test), + KUNIT_CASE(user4k_iova_bitmap_test), + KUNIT_CASE(user4k_iova_bitmap_window_test), + KUNIT_CASE(user4k_iova_bitmap_retained_test), + KUNIT_CASE(user4k_iova_bitmap_partial_test), + KUNIT_CASE(user4k_mremap_move_test), + KUNIT_CASE(user4k_mremap_resize_test), + KUNIT_CASE(user4k_stack_relocate_test), + KUNIT_CASE(user4k_exec_arguments_test), + KUNIT_CASE(user64k_exec_arguments_test), + KUNIT_CASE(user4k_special_mapping_test), + KUNIT_CASE(user4k_insert_fragment_test), + KUNIT_CASE(user16k_insert_shared_test), + KUNIT_CASE(user4k_insert_range_test), + KUNIT_CASE(user4k_pfn_offset_test), + KUNIT_CASE(user4k_remap_pfn_range_test), + KUNIT_CASE(user4k_iomem_access_test), +#if defined(CONFIG_GENERIC_GETTIMEOFDAY) && defined(CONFIG_VDSO_GETRANDOM) + KUNIT_CASE(user4k_vdso_mapping_test), +#endif + KUNIT_CASE(user4k_mremap_cow_test), + KUNIT_CASE(user4k_mremap_file_test), + KUNIT_CASE(user4k_msync_offsets_test), + KUNIT_CASE(user4k_prefault_dirty_test), + KUNIT_CASE(user4k_cold_batch_test), + KUNIT_CASE(user4k_remote_access_test), + KUNIT_CASE(user4k_remote_file_test), +#ifdef CONFIG_CROSS_MEMORY_ATTACH + KUNIT_CASE(user4k_process_vm_test), +#endif +#ifdef CONFIG_FUTEX + KUNIT_CASE(user4k_futex_keys_test), + KUNIT_CASE(user4k_futex_wake_test), +#endif + KUNIT_CASE(user4k_mmap_descriptor_test), + KUNIT_CASE(user4k_mmap_region_offsets_test), + KUNIT_CASE(user4k_commit_accounting_test), + KUNIT_CASE(mixed_locked_account_units_test), + KUNIT_CASE(user64k_commit_accounting_test), + KUNIT_CASE(user64k_page_table_check_test), +#ifdef CONFIG_PROC_FS + KUNIT_CASE(user4k_accounting_readers_test), +#endif + KUNIT_CASE(user_granule_prefault_test), + KUNIT_CASE(user4k_mmap_accounting_test), + KUNIT_CASE(user4k_vm_mmap_test), + KUNIT_CASE(user4k_mprotect_test), + KUNIT_CASE(user4k_mprotect_cow_batch_test), + KUNIT_CASE(user4k_mprotect_accounting_test), + KUNIT_CASE(user4k_file_cow_test), + KUNIT_CASE(user4k_file_eof_test), + KUNIT_CASE(user4k_file_truncate_test), + KUNIT_CASE(user4k_filemap_fault_test), + KUNIT_CASE(user4k_file_invalidate_test), + KUNIT_CASE(user4k_file_fragments_test), + KUNIT_CASE(user4k_file_fork_hole_test), + KUNIT_CASE(user4k_swap_encoding_test), + KUNIT_CASE(user4k_cow_reuse_test), + KUNIT_CASE(user16k_cow_reuse_test), + KUNIT_CASE(user4k_fork_cow_test), + KUNIT_CASE(user16k_fork_cow_test), + KUNIT_CASE(user4k_slot_pin_test), + KUNIT_CASE(user4k_fragment_pin_test), + KUNIT_CASE(user4k_fragment_locked_pin_test), + KUNIT_CASE(user4k_folio_walk_geometry_test), + KUNIT_CASE(user4k_fragment_longterm_test), +#ifdef CONFIG_BLOCK + KUNIT_CASE(user4k_bio_fragments_test), + KUNIT_CASE(user4k_bio_trim_test), +#endif + KUNIT_CASE(user4k_fragment_permissions_test), + KUNIT_CASE(user16k_fragment_offset_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_migration_test), + KUNIT_CASE(user4k_file_migration_test), + KUNIT_CASE(user64k_file_migration_test), + KUNIT_CASE(user64k_anon_migration_test), + KUNIT_CASE(user4k_fragment_pin_race_test), + KUNIT_CASE(user4k_migration_entries_test), + KUNIT_CASE(user4k_nonlinear_rmap_test), + KUNIT_CASE(user4k_mprotect_migration_test), + KUNIT_CASE(user4k_live_migration_test), + KUNIT_CASE(user4k_migration_wait_test), +#endif + KUNIT_CASE(user4k_anon_exit_test), + KUNIT_CASE(user4k_anon_remote_zap_test), +#ifdef CONFIG_ARM64_CONTPTE + KUNIT_CASE(user4k_contpte_test), + KUNIT_CASE(native_contpte_test), +#endif + KUNIT_CASE(user4k_mm_tables_test), + KUNIT_CASE(user16k_mm_tables_test), + KUNIT_CASE(user64k_mm_tables_test), + KUNIT_CASE(native_mm_tables_test), + KUNIT_CASE(user4k_anon_read_fault_test), + KUNIT_CASE(user4k_mm_hardware_test), + KUNIT_CASE(user4k_fragment_lifetime_test), + KUNIT_CASE(user16k_fragment_lifetime_test), + KUNIT_CASE(user64k_table_owner_test), +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_CASE(native_pgtable_deposit_test), +#endif + KUNIT_CASE(user4k_translation_test), + KUNIT_CASE(user4k_context_uaccess_test), + KUNIT_CASE(user4k_fault_uaccess_test), + {} +}; + +#ifdef CONFIG_NUMA_BALANCING +/* + * The COW hinting expectations require a stable normal-balancing policy. + * Runtime sysctl writers do not take the fixture's mm or PTE locks, so run + * this case at boot and do not expose it through the debugfs rerun control. + */ +static struct kunit_case user4k_numa_init_cases[] = { + KUNIT_CASE(user_subpage_numa_cow_batch_test), + {} +}; + +static struct kunit_suite user4k_numa_init_suite = { + .name = "arm64-user4k-numa-init", + .test_cases = user4k_numa_init_cases, +}; + +kunit_test_init_section_suite(user4k_numa_init_suite); +#endif + +static struct kunit_suite user4k_test_suite = { + .name = "arm64-user4k", + .test_cases = user4k_test_cases, +}; + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_shmem_swap_suite, + &user4k_coarse_shmem_suite, + &user4k_nonlinear_swap_suite); +#else +kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_nonlinear_swap_suite); +#endif + +MODULE_LICENSE("GPL"); diff --git a/arch/arm64/mm/user4k-tlb-test.c b/arch/arm64/mm/user4k-tlb-test.c new file mode 100644 index 0000000000000..e5671256b7c76 --- /dev/null +++ b/arch/arm64/mm/user4k-tlb-test.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Decode emitted operands without executing TLBI instructions. */ +#include +#include +#include +#include + +struct tlbi_test_state { + u64 next; + u64 end; + u64 stride; + u16 asid; + u32 level; + unsigned int shift; + unsigned int single_ops; + unsigned int range_ops; + bool lpa2; +}; + +static unsigned int test_tg(unsigned int shift) +{ + return shift == 12 ? 1 : shift == 14 ? 2 : 3; +} + +static void record_single(u64 arg) +{ + struct kunit *test = kunit_get_current_test(); + struct tlbi_test_state *state = test->priv; + u64 address = (arg & GENMASK_ULL(43, 0)) << 12; + u64 hint = 0; + + if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && state->level <= 3) + hint = (test_tg(state->shift) << 2) | state->level; + KUNIT_EXPECT_EQ(test, address, state->next); + KUNIT_EXPECT_EQ(test, arg >> 48, (u64)state->asid); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBI_TTL_MASK, arg), hint); + state->next += state->stride; + state->single_ops++; + KUNIT_EXPECT_LE(test, state->next, state->end); +} + +static void record_range(u64 arg) +{ + struct kunit *test = kunit_get_current_test(); + struct tlbi_test_state *state = test->priv; + u64 address = FIELD_GET(TLBIR_BADDR_MASK, arg) << (state->lpa2 ? 16 : state->shift); + u64 count = (FIELD_GET(TLBIR_NUM_MASK, arg) + 1) << + (5 * FIELD_GET(TLBIR_SCALE_MASK, arg) + 1); + + KUNIT_EXPECT_EQ(test, address, state->next); + KUNIT_EXPECT_EQ(test, arg >> 48, (u64)state->asid); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBIR_TG_MASK, arg), (u64)test_tg(state->shift)); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBIR_TTL_MASK, arg), + state->level > 3 ? 0ULL : (u64)state->level); + if (state->lpa2) + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(address, SZ_64K)); + state->next += count << state->shift; + state->range_ops++; + KUNIT_EXPECT_LE(test, state->next, state->end); +} + +static bool range_supported(void) +{ + return true; +} + +static bool range_unsupported(void) +{ + return false; +} + +static void test_flush(struct kunit *test, unsigned int shift, u64 start, + unsigned long pages, u64 stride, u32 level, bool lpa2, + bool has_range) +{ + struct tlbi_test_state state = { + .next = start, + .end = start + (pages << shift), + .stride = stride, + .asid = 0xa55a, + .level = level, + .shift = shift, + .lpa2 = lpa2, + }; + + test->priv = &state; + __flush_tlb_range_op_granule(record_single, record_range, start, pages, + stride, state.asid, level, lpa2, shift, + has_range ? range_supported : range_unsupported); + KUNIT_EXPECT_EQ(test, state.next, state.end); + if (has_range) { + KUNIT_EXPECT_LE(test, state.single_ops + state.range_ops, 20U); + if (pages >= 32) + KUNIT_EXPECT_GT(test, state.range_ops, 0U); + } else { + KUNIT_EXPECT_EQ(test, state.range_ops, 0U); + KUNIT_EXPECT_EQ(test, state.single_ops, (unsigned int)((pages << shift) / stride)); + } + test->priv = NULL; +} + +static void user4k_tlbi_operands_test(struct kunit *test) +{ + static const unsigned long counts[] = { + 0, 1, 2, 3, 31, 32, 33, 63, 64, 65, 2047, 2048, 2049, + 65535, 65536, 65537, MAX_TLBI_RANGE_PAGES - 1, MAX_TLBI_RANGE_PAGES, + }; + static const u32 levels[] = { 2, 3, TLBI_TTL_UNKNOWN }; + unsigned int shift, i, j; + + /* Force both algorithms using recording callbacks, independent of CPU caps. */ + for (shift = 12; shift <= 16; shift += 2) { + u64 size = 1UL << shift; + + for (j = 0; j < ARRAY_SIZE(levels); j++) { + for (i = 0; i < ARRAY_SIZE(counts); i++) { + unsigned long pages = counts[i]; + + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], false, true); + if (shift != 16) + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], true, true); + if (pages <= 65) + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], false, false); + } + } + /* Preserve a larger leaf stride when range instructions are absent. */ + test_flush(test, shift, SZ_1G, 3 * 512, 512 * size, 2, false, false); + /* Last virtual page must not lose high address bits. */ + test_flush(test, shift, (1ULL << 48) - size, 1, size, 3, false, false); + } +} + +static void user4k_tlb_gather_test(struct kunit *test) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + pte_t entries[3] = {}; + unsigned int small; + + KUNIT_ASSERT_NOT_NULL(test, mm); + for (small = 0; small < 2; small++) { + struct mmu_gather tlb = { .mm = mm, .start = ULONG_MAX }; + unsigned long size = small ? SZ_4K : PAGE_SIZE; + unsigned long address = SZ_4M + size; + + mm->context.flags = small ? MMCF_USER_4K : 0; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + tlb_remove_tlb_entries(&tlb, entries, 3, address); + KUNIT_EXPECT_EQ(test, tlb.start, address); + KUNIT_EXPECT_EQ(test, tlb.end, address + 3 * size); + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), 3); + tlb.cleared_pmds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), TLBI_TTL_UNKNOWN); + tlb.cleared_ptes = 0; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? SZ_2M : PMD_SIZE); + tlb.cleared_pmds = 0; + tlb.cleared_puds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? SZ_1G : PUD_SIZE); + tlb.cleared_puds = 0; + tlb.cleared_p4ds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? 1UL << 39 : P4D_SIZE); + tlb.freed_tables = 1; + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), TLBI_TTL_UNKNOWN); + } +} + +static struct kunit_case user4k_tlb_test_cases[] = { + KUNIT_CASE(user4k_tlbi_operands_test), + KUNIT_CASE(user4k_tlb_gather_test), + {} +}; + +static struct kunit_suite user4k_tlb_test_suite = { + .name = "arm64-user4k-tlb", + .test_cases = user4k_tlb_test_cases, +}; + +kunit_test_suite(user4k_tlb_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index ceae52fd7586d..33f6c7e9608d5 100644 --- a/drivers/vfio/Kconfig +++ b/drivers/vfio/Kconfig @@ -54,6 +54,18 @@ config VFIO_IOMMU_TYPE1 tristate default n +config VFIO_TYPE1_ACCOUNT_KUNIT_TEST + bool "Test VFIO type1 pin accounting and retained fragments" if !KUNIT_ALL_TESTS + depends on KUNIT=y && VFIO_IOMMU_TYPE1=y + default KUNIT_ALL_TESTS + help + Exercise locked-memory refunds after source-mm exit and DMA owner + changes. Arm64 mixed-granule configurations also test accounting + conversions between user and native page sizes. + Mixed-granule tests exercise retained pins, mappings, replay and + external page arrays with recorded IOMMU translations. No physical + device DMA is performed. + config VFIO_IOMMU_SPAPR_TCE tristate depends on SPAPR_TCE_IOMMU diff --git a/drivers/vfio/vfio_iommu_type1_account_test.c b/drivers/vfio/vfio_iommu_type1_account_test.c new file mode 100644 index 0000000000000..5447258ce49b8 --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_account_test.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by type1 so the tests exercise the actual accounting helpers. */ +#include +#include + +struct vfio_account_mm { + struct mm_struct *mm; + bool live; +}; + +static void vfio_account_mm_free(void *data) +{ + struct vfio_account_mm *ctx = data; + + if (ctx->live) + mmput(ctx->mm); + mmdrop(ctx->mm); +} + +static struct vfio_account_mm *vfio_account_mm_new(struct kunit *test, + unsigned int shift) +{ + struct vfio_account_mm *ctx; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return NULL; + ctx->mm = mm_alloc(); + if (!ctx->mm) + return NULL; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (shift != PAGE_SHIFT) { + pgd_t *old = ctx->mm->pgd, *pgd; + unsigned long flag = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + + ctx->mm->context.flags |= flag; + pgd = pgd_alloc(ctx->mm); + ctx->mm->context.flags &= ~flag; + if (!pgd) { + mmput(ctx->mm); + return NULL; + } + pgd_free(ctx->mm, old); + ctx->mm->pgd = pgd; + ctx->mm->context.flags |= flag; + } +#endif + ctx->live = true; + mmgrab(ctx->mm); + if (kunit_add_action_or_reset(test, vfio_account_mm_free, ctx)) + return NULL; + return ctx; +} + +static void vfio_account_dma_free(void *data) +{ + struct vfio_dma *dma = data; + + /* Cleanup must also work with the negative-control implementation. */ + if (dma->locked_vm) + mm_lock_acct(dma->task, dma->mm, true, -dma->locked_vm); + mmdrop(dma->mm); + put_task_struct(dma->task); +} + +static struct vfio_dma *vfio_account_dma_new(struct kunit *test, + struct vfio_account_mm *ctx) +{ + struct vfio_dma *dma = kunit_kzalloc(test, sizeof(*dma), GFP_KERNEL); + + if (!dma) + return NULL; + dma->mm = ctx->mm; + mmgrab(dma->mm); + dma->task = get_task_struct(current->group_leader); + dma->lock_cap = true; + if (kunit_add_action_or_reset(test, vfio_account_dma_free, dma)) + return NULL; + return dma; +} + +static void vfio_account_exit_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + 12, 14, 16, +#endif + }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_dma *dma; + + if (i && shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_EQ(test, mm_page_shift(ctx->mm), shifts[i]); + dma = vfio_account_dma_new(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, 2, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(ctx->mm), 2UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 2UL * PAGE_SIZE); + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, 1, true), -ESRCH); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, -1, true), 0); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(ctx->mm), 1UL); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, -1, true), 0); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + kunit_info(test, "VFIO %uK user/%luK native dead-mm refunds checked", + 1U << (shifts[i] - 10), PAGE_SIZE >> 10); + } +} + +static void vfio_account_owner_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + 12, 14, 16, +#endif + }; + + for (unsigned int reverse = 0; reverse < 2; reverse++) { + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *old, *next; + struct vfio_dma *dma; + + if ((i || reverse) && shifts[i] == PAGE_SHIFT) + continue; + old = vfio_account_mm_new(test, reverse ? shifts[i] : PAGE_SHIFT); + next = vfio_account_mm_new(test, reverse ? PAGE_SHIFT : shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + dma = vfio_account_dma_new(test, old); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, 3, true), 0); + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&old->mm->mm_users), 0); + if (task_rlimit(dma->task, RLIMIT_MEMLOCK) != RLIM_INFINITY) { + unsigned long budget = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> + MM_ACCOUNT_SHIFT; + + /* Model a recipient whose existing pins consume its budget. */ + next->mm->locked_vm = budget; + KUNIT_EXPECT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, false), + -ENOMEM); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, old->mm); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 3UL); + KUNIT_EXPECT_EQ(test, next->mm->locked_vm, budget); + next->mm->locked_vm = 0; + } + KUNIT_ASSERT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, next->mm); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 3UL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 3UL * PAGE_SIZE); + /* Same-mm ownership is idempotent; it must not double the charge. */ + KUNIT_ASSERT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 3UL); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, -3, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 0UL); + kunit_info(test, "VFIO dead %uK mm -> %uK mm ownership refund checked", + 1U << (mm_page_shift(old->mm) - 10), + 1U << (mm_page_shift(next->mm) - 10)); + } + } +} + +static struct kunit_case vfio_account_cases[] = { + KUNIT_CASE(vfio_account_exit_test), + KUNIT_CASE(vfio_account_owner_test), + {} +}; + +static struct kunit_suite vfio_account_suite = { + .name = "vfio-type1-account", + .test_cases = vfio_account_cases, +}; + +kunit_test_suite(vfio_account_suite); diff --git a/drivers/vfio/vfio_iommu_type1_fragments_test.c b/drivers/vfio/vfio_iommu_type1_fragments_test.c new file mode 100644 index 0000000000000..60fffff10c6be --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_fragments_test.c @@ -0,0 +1,1177 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Actual type1 lifecycle with recorded translations; no physical device DMA. */ +#include +#include +#include +#include + +#define VFIO_TEST_VA SZ_64M +#define VFIO_TEST_IOVA SZ_256M + +struct vfio_test_domain { + struct iommu_domain core; + struct vfio_domain type1; + struct xarray translations; + unsigned long mapped; + unsigned long fail_after; + unsigned long largest_map; + unsigned int syncs; + struct folio *observed; + struct kunit *test; +}; + +struct vfio_fragment_fixture { + struct vfio_iommu *iommu; + struct vfio_test_domain domains[2]; + struct vfio_iommu_group group; + struct vfio_device device; + struct kunit *test; + unsigned int external_refs; + unsigned int notifications; +}; + +static int vfio_test_domain_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t phys, size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + unsigned long length = pgsize * pgcount; + + mock->largest_map = max(mock->largest_map, length); + for (unsigned long done = 0; done < length; done += SZ_4K) { + int ret; + + if (mock->fail_after && mock->mapped >= mock->fail_after) + return -ENOMEM; + ret = xa_insert(&mock->translations, (iova + done) >> 12, + xa_mk_value((phys + done) >> 12), gfp); + if (ret) + return ret; + mock->mapped += SZ_4K; + *mapped += SZ_4K; + } + return 0; +} + +static size_t vfio_test_domain_unmap(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + unsigned long length = pgsize * pgcount, done; + + for (done = 0; done < length; done += SZ_4K) { + if (!xa_erase(&mock->translations, (iova + done) >> 12)) + break; + mock->mapped -= SZ_4K; + } + return done; +} + +static phys_addr_t vfio_test_domain_translate(struct iommu_domain *domain, dma_addr_t iova) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + void *entry = xa_load(&mock->translations, iova >> 12); + + return entry ? ((phys_addr_t)xa_to_value(entry) << 12) + (iova & (SZ_4K - 1)) : 0; +} + +static void vfio_test_domain_sync(struct iommu_domain *domain, + struct iommu_iotlb_gather *gather) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + + mock->syncs++; + if (mock->observed) + KUNIT_EXPECT_TRUE(mock->test, folio_maybe_dma_pinned(mock->observed)); +} + +static const struct iommu_domain_ops vfio_test_domain_ops = { + .map_pages = vfio_test_domain_map, + .unmap_pages = vfio_test_domain_unmap, + .iova_to_phys = vfio_test_domain_translate, + .iotlb_sync = vfio_test_domain_sync, +}; + +static void vfio_test_notify(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_fragment_fixture *f = container_of(device, typeof(*f), device); + + f->notifications++; + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } +} + +static const struct vfio_device_ops vfio_test_device_ops = { + .dma_unmap = vfio_test_notify, +}; + +static void vfio_fragment_fixture_free(void *ptr) +{ + struct vfio_fragment_fixture *f = ptr; + struct rb_node *node; + + while ((node = rb_first(&f->iommu->dma_list))) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + struct rb_node *pin; + + while ((pin = rb_first(&dma->pfn_list))) { + struct vfio_pfn *vpfn = rb_entry(pin, struct vfio_pfn, node); + + vfio_unpin_page_external(dma, vpfn->iova, + list_empty(&f->iommu->domain_list)); + } + vfio_remove_dma(f->iommu, dma); + } + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + KUNIT_EXPECT_EQ(f->test, f->domains[i].mapped, 0UL); + KUNIT_EXPECT_TRUE(f->test, xa_empty(&f->domains[i].translations)); + xa_destroy(&f->domains[i].translations); + } + kfree(f->iommu); +} + +static struct vfio_fragment_fixture *vfio_fragment_fixture_new(struct kunit *test) +{ + struct vfio_fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + + if (!f) + return NULL; + f->test = test; + f->iommu = vfio_iommu_type1_open(VFIO_TYPE1v2_IOMMU); + if (IS_ERR(f->iommu)) + return NULL; + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + struct vfio_test_domain *mock = &f->domains[i]; + + xa_init(&mock->translations); + mock->test = test; + mock->core.ops = &vfio_test_domain_ops; + mock->core.type = IOMMU_DOMAIN_UNMANAGED; + mock->core.pgsize_bitmap = SZ_4K | PAGE_SIZE; + mock->type1.domain = &mock->core; + INIT_LIST_HEAD(&mock->type1.next); + INIT_LIST_HEAD(&mock->type1.group_list); + } + f->group.iommu_group = (struct iommu_group *)f; + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + f->iommu->num_non_pinned_groups = 1; + f->device.ops = &vfio_test_device_ops; + vfio_iommu_type1_register_device(f->iommu, &f->device); + if (kunit_add_action_or_reset(test, vfio_fragment_fixture_free, f)) + return NULL; + return f; +} + +static void vfio_test_file_put(void *ptr) +{ + fput(ptr); +} + +static struct file *vfio_fragment_file(struct kunit *test, unsigned long length, + unsigned char value) +{ + struct file *file = shmem_file_setup("vfio-fragments", length, EMPTY_VMA_FLAGS); + unsigned char *data; + loff_t pos = 0; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, vfio_test_file_put, file)) + return ERR_PTR(-ENOMEM); + data = kunit_kmalloc(test, length, GFP_KERNEL); + if (!data) + return ERR_PTR(-ENOMEM); + memset(data, value, length); + if (kernel_write(file, data, length, &pos) != length) + return ERR_PTR(-EIO); + return file; +} + +static unsigned long vfio_fragment_source(struct vfio_account_mm *ctx, + struct file *file, unsigned long length) +{ + unsigned long addr; + + kthread_use_mm(ctx->mm); + addr = vm_mmap(file, VFIO_TEST_VA, length, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(ctx->mm); + return addr; +} + +static int vfio_fragment_test_map(struct vfio_fragment_fixture *f, + struct vfio_account_mm *ctx, + unsigned long offset, unsigned long length) +{ + struct vfio_iommu_type1_dma_map map = { + .argsz = sizeof(map), + .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, + .vaddr = VFIO_TEST_VA + offset, + .iova = VFIO_TEST_IOVA, + .size = length, + }; + int ret; + + kthread_use_mm(ctx->mm); + ret = vfio_dma_do_map(f->iommu, &map); + kthread_unuse_mm(ctx->mm); + return ret; +} + +static void vfio_fragments_lifetime_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + unsigned char data[64]; + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, .size = 2 * PAGE_SIZE, + }; + struct vfio_bitmap bitmap = {}; + + if (shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x71); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + for (unsigned int d = 0; d < 2; d++) + f->domains[d].observed = vfio_fragment_find(dma, 0)->pin.folio; + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_GE(test, f->domains[0].largest_map, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 17, 1, IOMMU_READ | IOMMU_WRITE, &page), 1); + f->external_refs++; + KUNIT_EXPECT_EQ(test, page_to_phys(page), + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* The user VA goes away; retained pins are still the DMA buffer. */ + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x71, sizeof(data)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + KUNIT_EXPECT_EQ(test, f->domains[1].mapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA), + iommu_iova_to_phys(&f->domains[1].core, VFIO_TEST_IOVA)); + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, f->notifications, 1U); + KUNIT_EXPECT_GT(test, f->domains[0].syncs, 0U); + KUNIT_EXPECT_GT(test, f->domains[1].syncs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + kunit_info(test, "type1 retained %uK user/%luK native pins after mm exit and replay", + 1U << (shifts[i] - 10), PAGE_SIZE >> 10); + } +} + +static void vfio_fragments_lazy_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct file *file; + struct vfio_dma *dma; + struct page *page; + unsigned char data[64]; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x52); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x52, sizeof(data)), NULL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 37, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + PAGE_SIZE, 1, IOMMU_READ, &page), -EFAULT); + for (unsigned int i = 0; i < 3; i++) + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 37, 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->fragments->records)); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); +} + +static void vfio_fragments_map_failure_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct file *file; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x25); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + f->domains[1].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), -ENOMEM); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&f->iommu->dma_list)); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, f->domains[1].mapped, 0UL); + f->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); +} + +static void vfio_fragments_fine_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page = NULL; + unsigned char data[64]; + unsigned long length = PAGE_SIZE + SZ_4K; + size_t copied = 0; + + if (i && shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x64); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, SZ_4K, length), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, length); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* The physical base is offset inside a native page, not representable. */ + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), -EINVAL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + PAGE_SIZE, 1, IOMMU_READ, &page), -EINVAL); + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, + vfio_dirty_pgsize(f->iommu)), 0); + f->iommu->dirty_page_tracking = true; + memset(data, 0x97, sizeof(data)); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw_chunk(f->iommu, + VFIO_TEST_IOVA + SZ_4K - 17, data, sizeof(data), true, &copied), 0); + KUNIT_EXPECT_EQ(test, copied, sizeof(data)); + KUNIT_EXPECT_EQ(test, bitmap_weight(dma->bitmap, length / SZ_4K), 2U); + KUNIT_EXPECT_TRUE(test, test_bit(0, dma->bitmap)); + KUNIT_EXPECT_TRUE(test, test_bit(1, dma->bitmap)); + memset(data, 0, sizeof(data)); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, + VFIO_TEST_IOVA + SZ_4K - 17, data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x97, sizeof(data)), NULL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, + VFIO_TEST_IOVA + length, data, sizeof(data), false), -EINVAL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_info(test, "%uK userspace fine IOVA/physical offsets and dirty bytes checked", + 1U << (shifts[i] - 10)); + } +} + +static void vfio_fragments_replay_failure_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x58); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + f->domains[0].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), -ENOMEM); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_FALSE(test, dma->iommu_mapped); + f->domains[0].fail_after = 0; + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), 0); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* Detaching the last hardware domain keeps only the external page. */ + vfio_iommu_unmap_unpin_reaccount(f->iommu); + list_del_init(&f->domains[0].type1.next); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); +} + +static void vfio_fragments_anon_owner_test(struct kunit *test) +{ + struct vfio_account_mm *old = vfio_account_mm_new(test, 12); + struct vfio_account_mm *next = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + unsigned char *data = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL); + unsigned long addr, copied, charge; + unsigned char check[64]; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + KUNIT_ASSERT_NOT_NULL(test, data); + memset(data, 0x49, PAGE_SIZE); + kthread_use_mm(old->mm); + addr = vm_mmap(NULL, VFIO_TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + copied = addr == VFIO_TEST_VA ? + copy_to_user((void __user *)addr, data, PAGE_SIZE) : PAGE_SIZE; + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, copied, 0UL); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, old, 0, PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, vfio_fragment_find(dma, 0)->pin.subpage); + charge = dma->locked_vm; + KUNIT_ASSERT_GT(test, charge, 0UL); + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(old->mm), 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), charge); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + check, sizeof(check), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(check, 0x49, sizeof(check)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 0UL); + kunit_info(test, "anonymous 4K slot pins survive source exit and native-owner transfer"); +} + +static void vfio_fragments_domain_granule_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, PAGE_SIZE, 0x73); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, + vfio_dirty_pgsize(f->iommu)), 0); + f->iommu->dirty_page_tracking = true; + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 31, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, bitmap_weight(dma->bitmap, PAGE_SIZE / SZ_4K), + (unsigned int)(PAGE_SIZE / SZ_4K)); + f->domains[0].core.pgsize_bitmap = SZ_16K; + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), 0); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, 1UL << __ffs(f->iommu->pgsize_bitmap), SZ_16K); + KUNIT_EXPECT_EQ(test, vfio_dirty_pgsize(f->iommu), SZ_4K); + vfio_iommu_unmap_unpin_reaccount(f->iommu); + list_del_init(&f->domains[0].type1.next); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, 1UL << __ffs(f->iommu->pgsize_bitmap), SZ_4K); + KUNIT_EXPECT_EQ(test, vfio_dirty_pgsize(f->iommu), SZ_4K); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 31, 1); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); +} + +static void vfio_fragments_native_owner_test(struct kunit *test) +{ + for (unsigned int variant = 0; variant < 9; variant++) { + bool same_mm = variant == 8; + bool mapped = same_mm || (variant & 1); + bool alias = same_mm || (variant & 2), different = variant & 4; + struct vfio_account_mm *old = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_account_mm *next = same_mm ? old : vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_fragment_provider *prepared; + struct file *file, *replacement; + struct vfio_dma *dma; + struct page *pages[2]; + phys_addr_t external[2]; + unsigned long unique, addr; + unsigned char data[32]; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x63); + replacement = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x44); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, replacement); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(old, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + if (alias) { + kthread_use_mm(old->mm); + addr = vm_mmap(file, VFIO_TEST_VA + PAGE_SIZE, PAGE_SIZE, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, 0); + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA + PAGE_SIZE); + } + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, old, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NULL(test, dma->fragments); + if (different) { + kthread_use_mm(old->mm); + addr = vm_mmap(replacement, VFIO_TEST_VA, 2 * PAGE_SIZE, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, 0); + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA); + } + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 17, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + for (unsigned int i = 0; i < 2; i++) + external[i] = page_to_phys(pages[i]); + if (mapped && different) + KUNIT_EXPECT_NE(test, external[0], + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + KUNIT_ASSERT_EQ(test, dma->locked_vm, 2UL); + if (!same_mm) { + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&old->mm->mm_users), 0); + } + + prepared = vfio_fragments_prepare_native(f->iommu, dma); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, prepared); + unique = mapped ? (alias ? 1UL : 2UL) : 0; + if (!mapped || different) + unique += different || !alias ? 2UL : 1UL; + KUNIT_EXPECT_EQ(test, vfio_fragments_native_pages(prepared), unique); + /* Failed new-owner credit must not publish metadata or release pins. */ + if (!same_mm && task_rlimit(dma->task, RLIMIT_MEMLOCK) != RLIM_INFINITY) { + unsigned long budget = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> + mm_page_shift(next->mm); + + next->mm->locked_vm = budget; + KUNIT_EXPECT_EQ(test, vfio_change_dma_mm_pages(dma, dma->task, + next->mm, false, unique), -ENOMEM); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, old->mm); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 2UL); + KUNIT_EXPECT_EQ(test, next->mm->locked_vm, budget); + next->mm->locked_vm = 0; + } + vfio_fragments_abort_native(prepared); + KUNIT_EXPECT_PTR_EQ(test, dma->fragments, NULL); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(page_folio(pages[0]))); + kthread_use_mm(next->mm); + ret = vfio_change_dma_owner(f->iommu, dma, + VFIO_TEST_VA + (same_mm ? SZ_4K : 0)); + kthread_unuse_mm(next->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, unique); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), same_mm ? unique : 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), unique); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, + memchr_inv(data, !mapped && different ? 0x44 : 0x63, sizeof(data)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 31, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + for (unsigned int i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, page_to_phys(pages[i]), external[i]); + KUNIT_EXPECT_EQ(test, dma->locked_vm, unique); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 31, 2); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 17, 2); + KUNIT_EXPECT_EQ(test, dma->locked_vm, + mapped ? (alias ? 1UL : 2UL) : unique); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 0UL); + kunit_info(test, "native ownership variant=%u preserves PIN identity and unique charge", + variant); + } +} + +static int vfio_fragment_map_area(struct vfio_fragment_fixture *f, + struct vfio_account_mm *ctx, unsigned long iova_offset, + unsigned long va_offset, unsigned long length, unsigned int flags) +{ + struct vfio_iommu_type1_dma_map map = { + .argsz = sizeof(map), + .flags = flags, + .vaddr = VFIO_TEST_VA + va_offset, + .iova = VFIO_TEST_IOVA + iova_offset, + .size = length, + }; + int ret; + + kthread_use_mm(ctx->mm); + ret = vfio_dma_do_map(f->iommu, &map); + kthread_unuse_mm(ctx->mm); + return ret; +} + +static void vfio_fragments_external_span_test(struct kunit *test) +{ + unsigned int rw = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_account_mm *a = vfio_account_mm_new(test, 12); + struct vfio_account_mm *b = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x6b); + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + struct vfio_dma *first, *rest; + struct page *pages[3]; + unsigned long bits = 2 * PAGE_SIZE / SZ_4K - 1; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(a, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(b, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, a, 0, 0, SZ_4K, rw), 0); + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, b, SZ_4K, SZ_4K, + 2 * PAGE_SIZE - SZ_4K, rw), 0); + first = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + rest = vfio_find_dma(f->iommu, VFIO_TEST_IOVA + SZ_4K, 1); + KUNIT_ASSERT_NOT_NULL(test, first); + KUNIT_ASSERT_NOT_NULL(test, rest); + f->iommu->dirty_page_tracking = true; + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, SZ_4K), 0); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 37, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + KUNIT_EXPECT_EQ(test, first->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, rest->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, bitmap_weight(first->bitmap, 1), 1); + KUNIT_EXPECT_EQ(test, bitmap_weight(rest->bitmap, bits), (int)bits); + bitmap_clear(rest->bitmap, 0, bits); + vfio_dma_populate_bitmap(rest, SZ_4K); + KUNIT_EXPECT_EQ(test, bitmap_weight(rest->bitmap, bits), (int)bits); + if (mapped) + KUNIT_EXPECT_EQ(test, page_to_phys(pages[0]), + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + /* A failed larger request must leave prior caller references intact. */ + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 3, IOMMU_READ, pages), -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, pages[0], NULL); + KUNIT_EXPECT_PTR_EQ(test, pages[1], NULL); + KUNIT_EXPECT_PTR_EQ(test, pages[2], NULL); + a->live = b->live = false; + mmput(a->mm); + mmput(b->mm); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 2, IOMMU_READ, pages), 2); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 2); + /* Release the second page first: it shares the second DMA owner. */ + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + PAGE_SIZE + 7, 1); + KUNIT_EXPECT_EQ(test, rest->locked_vm, mapped ? 2UL : 1UL); + f->external_refs = 1; + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, f->notifications, 1U); + KUNIT_EXPECT_EQ(test, f->external_refs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(a->mm), 0UL); + KUNIT_EXPECT_EQ(test, rest->locked_vm, mapped ? 2UL : 0UL); + unmap.iova += SZ_4K; + unmap.size = 2 * PAGE_SIZE - SZ_4K; + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(b->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + } + for (unsigned int failure = 0; failure < 4; failure++) { + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x3a); + struct page *page = NULL; + struct rb_node *node; + unsigned long va = failure == 2 ? PAGE_SIZE + SZ_4K : SZ_4K; + int expected = failure == 1 ? -EPERM : -EINVAL; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, 0, 0, SZ_4K, rw), 0); + if (failure == 3) + va += SZ_4K; + if (failure != 0) + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, SZ_4K, va, + SZ_4K, failure == 1 ? VFIO_DMA_MAP_FLAG_READ : rw), 0); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 1, + IOMMU_READ | IOMMU_WRITE, &page), expected); + KUNIT_EXPECT_PTR_EQ(test, page, NULL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + for (node = rb_first(&f->iommu->dma_list); node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->pfn_list)); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->fragments->records)); + } + kunit_release_action(test, vfio_fragment_fixture_free, f); + } +} + +#define VFIO_STRESS_CYCLES 1024 + +struct vfio_pin_stress { + struct vfio_fragment_fixture *fixture; + struct vfio_device consumer; + struct vfio_device registrar; + struct vfio_device observer; + struct task_struct *competing_thread; + struct task_struct *pin_thread; + struct task_struct *register_thread; + struct mutex consumer_lock; + struct completion pin_go; + struct completion pin_ready; + struct completion notify_entered; + struct completion pin_done; + struct completion register_go; + struct completion register_entered; + struct completion register_done; + struct completion competing_go; + struct completion competing_done; + unsigned long competing_size; + int competing_ret; + bool competing; + unsigned int competing_unmaps; + unsigned int observations; + atomic_t error; + unsigned int pins; + unsigned int registrations; + bool stopping; +}; + +static void vfio_stress_error(struct vfio_pin_stress *s, int ret) +{ + atomic_cmpxchg(&s->error, 0, ret < 0 ? ret : -EIO); +} + +static void vfio_stress_notify(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_pin_stress *s = container_of(device, typeof(*s), consumer); + struct vfio_fragment_fixture *f = s->fixture; + + /* The callback holds the production device-list lock, not the DMA lock. */ + complete(&s->register_go); + if (!wait_for_completion_timeout(&s->register_entered, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + complete(&s->notify_entered); + mutex_lock(&s->consumer_lock); + f->notifications++; + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + mutex_unlock(&s->consumer_lock); + if (s->competing) { + /* A second unmap may free the DMA while this callback still runs. */ + complete(&s->competing_go); + if (!wait_for_completion_timeout(&s->competing_done, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + } +} + +static void vfio_stress_observe(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_pin_stress *s = container_of(device, typeof(*s), observer); + + s->observations++; + if (iova != VFIO_TEST_IOVA || length != SZ_4K) + vfio_stress_error(s, -ESTALE); +} + +static void vfio_stress_empty_notify(struct vfio_device *device, u64 iova, u64 length) +{ +} + +static const struct vfio_device_ops vfio_stress_consumer_ops = { + .dma_unmap = vfio_stress_notify, +}; + +static const struct vfio_device_ops vfio_stress_registrar_ops = { + .dma_unmap = vfio_stress_empty_notify, +}; + +static const struct vfio_device_ops vfio_stress_observer_ops = { + .dma_unmap = vfio_stress_observe, +}; + +static int vfio_stress_competing_unmaps(void *data) +{ + struct vfio_pin_stress *s = data; + + for (;;) { + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, + .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + + wait_for_completion(&s->competing_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + s->competing_ret = vfio_dma_do_unmap(s->fixture->iommu, &unmap, &bitmap); + s->competing_size = unmap.size; + s->competing_unmaps++; + complete(&s->competing_done); + } + return 0; +} + +static int vfio_stress_pinner(void *data) +{ + struct vfio_pin_stress *s = data; + struct vfio_fragment_fixture *f = s->fixture; + + for (;;) { + struct page *page = NULL, *original; + void *bytes; + int ret; + + wait_for_completion(&s->pin_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + mutex_lock(&s->consumer_lock); + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 37, 1, IOMMU_READ | IOMMU_WRITE, &page); + if (ret != 1) { + vfio_stress_error(s, ret); + mutex_unlock(&s->consumer_lock); + complete(&s->pin_ready); + complete(&s->pin_done); + continue; + } + f->external_refs++; + s->pins++; + original = page; + complete(&s->pin_ready); + /* Unmap cannot revoke a reference before the consumer publishes it. */ + if (!wait_for_completion_timeout(&s->notify_entered, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + if (!READ_ONCE(s->stopping)) { + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 17, 1, IOMMU_READ, &page); + if (ret == 1) { + f->external_refs++; + s->pins++; + if (page != original) + vfio_stress_error(s, -ESTALE); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 17, 1); + f->external_refs--; + } else { + vfio_stress_error(s, ret); + } + bytes = kmap_local_page(original); + if (memchr_inv(bytes, 0x6b, PAGE_SIZE)) + vfio_stress_error(s, -EIO); + kunmap_local(bytes); + } + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 37, 1); + f->external_refs--; + mutex_unlock(&s->consumer_lock); + complete(&s->pin_done); + } + return 0; +} + +static int vfio_stress_registrations(void *data) +{ + struct vfio_pin_stress *s = data; + + for (;;) { + wait_for_completion(&s->register_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + complete(&s->register_entered); + vfio_iommu_type1_register_device(s->fixture->iommu, &s->registrar); + vfio_iommu_type1_unregister_device(s->fixture->iommu, &s->registrar); + s->registrations++; + complete(&s->register_done); + } + return 0; +} + +static void vfio_stress_free(void *data) +{ + struct vfio_pin_stress *s = data; + struct vfio_fragment_fixture *f = s->fixture; + + WRITE_ONCE(s->stopping, true); + complete_all(&s->pin_go); + complete_all(&s->notify_entered); + complete_all(&s->register_go); + complete_all(&s->competing_go); + if (s->competing_thread) { + kthread_stop(s->competing_thread); + put_task_struct(s->competing_thread); + } + if (s->pin_thread) { + kthread_stop(s->pin_thread); + put_task_struct(s->pin_thread); + } + if (s->register_thread) { + kthread_stop(s->register_thread); + put_task_struct(s->register_thread); + } + mutex_lock(&s->consumer_lock); + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + mutex_unlock(&s->consumer_lock); + vfio_iommu_type1_unregister_device(f->iommu, &s->consumer); + vfio_iommu_type1_unregister_device(f->iommu, &s->observer); +} + +static struct vfio_pin_stress *vfio_stress_new(struct kunit *test, + struct vfio_fragment_fixture *f) +{ + struct vfio_pin_stress *s = kunit_kzalloc(test, sizeof(*s), GFP_KERNEL); + + if (!s) + return NULL; + s->fixture = f; + mutex_init(&s->consumer_lock); + init_completion(&s->pin_go); + init_completion(&s->pin_ready); + init_completion(&s->notify_entered); + init_completion(&s->pin_done); + init_completion(&s->register_go); + init_completion(&s->register_entered); + init_completion(&s->register_done); + init_completion(&s->competing_go); + init_completion(&s->competing_done); + atomic_set(&s->error, 0); + s->consumer.ops = &vfio_stress_consumer_ops; + s->registrar.ops = &vfio_stress_registrar_ops; + s->observer.ops = &vfio_stress_observer_ops; + vfio_iommu_type1_unregister_device(f->iommu, &f->device); + /* list_add puts the consumer before the observer in callback order. */ + vfio_iommu_type1_register_device(f->iommu, &s->observer); + vfio_iommu_type1_register_device(f->iommu, &s->consumer); + if (kunit_add_action_or_reset(test, vfio_stress_free, s)) + return NULL; + s->pin_thread = kthread_run(vfio_stress_pinner, s, "vfio-pin-stress"); + if (IS_ERR(s->pin_thread)) { + s->pin_thread = NULL; + return NULL; + } + get_task_struct(s->pin_thread); + s->register_thread = kthread_run(vfio_stress_registrations, s, "vfio-reg-stress"); + if (IS_ERR(s->register_thread)) { + s->register_thread = NULL; + return NULL; + } + get_task_struct(s->register_thread); + s->competing_thread = kthread_run(vfio_stress_competing_unmaps, s, "vfio-unmap-stress"); + if (IS_ERR(s->competing_thread)) { + s->competing_thread = NULL; + return NULL; + } + get_task_struct(s->competing_thread); + return s; +} + +static void vfio_fragments_concurrency_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14, 16 }; + unsigned int rw = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + + for (unsigned int shift = 0; shift < ARRAY_SIZE(shifts); shift++) { + if (shifts[shift] > PAGE_SHIFT) + continue; + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_account_mm *ctx = vfio_account_mm_new(test, shifts[shift]); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, PAGE_SIZE, 0x6b); + struct vfio_pin_stress *s; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, PAGE_SIZE), + VFIO_TEST_VA); + s = vfio_stress_new(test, f); + KUNIT_ASSERT_NOT_NULL(test, s); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + for (unsigned int cycle = 0; cycle < VFIO_STRESS_CYCLES; cycle++) { + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, + .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + bool fine = cycle & 1; + unsigned long remaining = PAGE_SIZE - SZ_4K; + + KUNIT_ASSERT_EQ(test, + vfio_fragment_map_area(f, ctx, 0, 0, SZ_4K, rw), 0); + for (unsigned long offset = SZ_4K; offset < PAGE_SIZE;) { + unsigned long length = fine ? SZ_4K : remaining; + + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, offset, + offset, length, rw), 0); + offset += length; + } + if (cycle == VFIO_STRESS_CYCLES - 1) { + struct page *page; + + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 1, + IOMMU_READ, &page), 1); + f->external_refs = 1; + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + } + s->competing = cycle & 2; + complete(&s->pin_go); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->pin_ready, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_EQ(test, atomic_read(&s->error), 0); + KUNIT_ASSERT_EQ(test, + vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, s->competing ? 0UL : SZ_4K); + if (s->competing) { + KUNIT_EXPECT_EQ(test, s->competing_ret, 0); + KUNIT_EXPECT_EQ(test, s->competing_size, SZ_4K); + } + unmap.iova += SZ_4K; + unmap.size = remaining; + KUNIT_ASSERT_EQ(test, + vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, remaining); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->pin_done, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->register_done, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_EQ(test, atomic_read(&s->error), 0); + KUNIT_EXPECT_EQ(test, f->external_refs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&f->iommu->dma_list)); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + reinit_completion(&s->pin_ready); + reinit_completion(&s->notify_entered); + reinit_completion(&s->pin_done); + reinit_completion(&s->register_entered); + reinit_completion(&s->register_done); + reinit_completion(&s->competing_done); + } + KUNIT_EXPECT_EQ(test, s->pins, 2U * VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->registrations, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, f->notifications, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->observations, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->competing_unmaps, VFIO_STRESS_CYCLES / 2U); + kunit_info(test, "%uK user/%luK native mapped=%u: %u forced pin/unmap/register cycles, half with competing unmap", + 1U << (shifts[shift] - 10), PAGE_SIZE >> 10, + mapped, VFIO_STRESS_CYCLES); + kunit_release_action(test, vfio_stress_free, s); + kunit_release_action(test, vfio_fragment_fixture_free, f); + } + } +} + +static struct kunit_case vfio_fragments_cases[] = { + KUNIT_CASE(vfio_fragments_lifetime_test), + KUNIT_CASE(vfio_fragments_lazy_test), + KUNIT_CASE(vfio_fragments_map_failure_test), + KUNIT_CASE(vfio_fragments_fine_test), + KUNIT_CASE(vfio_fragments_replay_failure_test), + KUNIT_CASE(vfio_fragments_anon_owner_test), + KUNIT_CASE(vfio_fragments_domain_granule_test), + KUNIT_CASE(vfio_fragments_native_owner_test), + KUNIT_CASE(vfio_fragments_external_span_test), + KUNIT_CASE(vfio_fragments_concurrency_test), + {} +}; + +static struct kunit_suite vfio_fragments_suite = { + .name = "vfio-type1-fragments", + .test_cases = vfio_fragments_cases, +}; + +kunit_test_suite(vfio_fragments_suite); diff --git a/drivers/vfio/vfio_iommu_type1_ioctl_test.c b/drivers/vfio/vfio_iommu_type1_ioctl_test.c new file mode 100644 index 0000000000000..33e981bcdf74d --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_ioctl_test.c @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included after the KUnit harness; real EL0 arguments, recorded domains only. */ +#include +#include +#include +#include "vfio_iommu_type1_test.h" + +struct vfio_ioctl_fixture { + struct vfio_fragment_fixture pins; + struct mutex request_lock; +}; + +struct vfio_ioctl_rw { + struct work_struct work; + struct vfio_iommu *iommu; + dma_addr_t iova; + void *data; + size_t size; + bool write; + int ret; +}; + +static void vfio_ioctl_rw_work(struct work_struct *work) +{ + struct vfio_ioctl_rw *rw = container_of(work, typeof(*rw), work); + + rw->ret = vfio_iommu_type1_dma_rw(rw->iommu, rw->iova, rw->data, + rw->size, rw->write); +} + +static int vfio_ioctl_test_open(struct inode *inode, struct file *file) +{ + struct vfio_ioctl_fixture *ctx; + struct vfio_fragment_fixture *f; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + f = &ctx->pins; + f->iommu = vfio_iommu_type1_open(VFIO_TYPE1v2_IOMMU); + if (IS_ERR(f->iommu)) { + int ret = PTR_ERR(f->iommu); + + kfree(ctx); + return ret; + } + mutex_init(&ctx->request_lock); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + struct vfio_test_domain *mock = &f->domains[i]; + + xa_init(&mock->translations); + mock->core.ops = &vfio_test_domain_ops; + mock->core.type = IOMMU_DOMAIN_UNMANAGED; + mock->core.pgsize_bitmap = SZ_4K | PAGE_SIZE; + mock->type1.domain = &mock->core; + INIT_LIST_HEAD(&mock->type1.next); + INIT_LIST_HEAD(&mock->type1.group_list); + } + f->group.iommu_group = (struct iommu_group *)ctx; + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + f->iommu->num_non_pinned_groups = 1; + f->device.ops = &vfio_test_device_ops; + vfio_iommu_type1_register_device(f->iommu, &f->device); + file->private_data = ctx; + return 0; +} + +static int vfio_ioctl_test_release(struct inode *inode, struct file *file) +{ + struct vfio_ioctl_fixture *ctx = file->private_data; + struct vfio_fragment_fixture *f = &ctx->pins; + + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + vfio_iommu_type1_unregister_device(f->iommu, &f->device); + vfio_iommu_unmap_unpin_all(f->iommu); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + list_del_init(&f->domains[i].type1.next); + xa_destroy(&f->domains[i].translations); + } + list_del(&f->group.next); + vfio_iommu_type1_release(f->iommu); + kfree(ctx); + return 0; +} + +static int vfio_ioctl_test_domain(struct vfio_fragment_fixture *f, + struct vfio_type1_test_cmd *cmd, bool add) +{ + struct vfio_test_domain *mock; + struct rb_node *node; + int ret = 0; + + if (cmd->domain >= ARRAY_SIZE(f->domains) || cmd->flags) + return -EINVAL; + mock = &f->domains[cmd->domain]; + mutex_lock(&f->iommu->lock); + if (add) { + if (!list_empty(&mock->type1.next)) { + ret = -EEXIST; + goto out; + } + if (cmd->size != SZ_4K && cmd->size != SZ_16K && cmd->size != SZ_64K) { + ret = -EINVAL; + goto out; + } + mock->core.pgsize_bitmap = cmd->size; + mock->fail_after = cmd->fail_after; + ret = vfio_iommu_replay(f->iommu, &mock->type1); + if (ret) + goto out; + list_add_tail(&mock->type1.next, &f->iommu->domain_list); + } else { + if (list_empty(&mock->type1.next)) { + ret = -ENOENT; + goto out; + } + if (list_is_singular(&f->iommu->domain_list)) { + vfio_iommu_unmap_unpin_reaccount(f->iommu); + } else { + for (node = rb_first(&f->iommu->dma_list); node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + iommu_unmap(&mock->core, dma->iova, dma->size); + } + } + list_del_init(&mock->type1.next); + } + vfio_update_pgsize_bitmap(f->iommu); +out: + mutex_unlock(&f->iommu->lock); + return ret; +} + +static int vfio_ioctl_test_rw(struct vfio_fragment_fixture *f, + struct vfio_type1_test_cmd *cmd) +{ + struct vfio_ioctl_rw rw = { + .iommu = f->iommu, .iova = cmd->iova, .size = cmd->size, + .write = cmd->flags & VFIO_TYPE1_TEST_WRITE, + }; + void __user *buffer = u64_to_user_ptr(cmd->buffer); + int ret; + + if (!cmd->size || cmd->size > 256 || + cmd->flags & ~(VFIO_TYPE1_TEST_WRITE | VFIO_TYPE1_TEST_WORKER)) + return -EINVAL; + rw.data = kzalloc(cmd->size, GFP_KERNEL); + if (!rw.data) + return -ENOMEM; + if (rw.write && copy_from_user(rw.data, buffer, cmd->size)) { + ret = -EFAULT; + goto out; + } + if (cmd->flags & VFIO_TYPE1_TEST_WORKER) { + INIT_WORK(&rw.work, vfio_ioctl_rw_work); + queue_work(system_unbound_wq, &rw.work); + flush_work(&rw.work); + ret = rw.ret; + } else { + ret = vfio_iommu_type1_dma_rw(f->iommu, rw.iova, rw.data, rw.size, rw.write); + } + if (!ret && !rw.write && copy_to_user(buffer, rw.data, cmd->size)) + ret = -EFAULT; +out: + kfree(rw.data); + return ret; +} + +static long vfio_ioctl_test_ioctl(struct file *file, unsigned int request, + unsigned long arg) +{ + struct vfio_ioctl_fixture *ctx = file->private_data; + struct vfio_fragment_fixture *f = &ctx->pins; + struct vfio_type1_test_cmd cmd; + void __user *user = (void __user *)arg; + struct page *page; + int ret = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + /* Request serialization belongs to this fixture, not production type1. */ + mutex_lock(&ctx->request_lock); + switch (request) { + case VFIO_CHECK_EXTENSION: + case VFIO_IOMMU_GET_INFO: + case VFIO_IOMMU_MAP_DMA: + case VFIO_IOMMU_UNMAP_DMA: + case VFIO_IOMMU_DIRTY_PAGES: + ret = vfio_iommu_type1_ioctl(f->iommu, request, arg); + goto out; + default: + break; + } + if (copy_from_user(&cmd, user, sizeof(cmd))) { + ret = -EFAULT; + goto out; + } + if (cmd.reserved) { + ret = -EINVAL; + goto out; + } + switch (request) { + case VFIO_TYPE1_TEST_DOMAIN_ADD: + case VFIO_TYPE1_TEST_DOMAIN_DEL: + ret = vfio_ioctl_test_domain(f, &cmd, request == VFIO_TYPE1_TEST_DOMAIN_ADD); + break; + case VFIO_TYPE1_TEST_RW: + ret = vfio_ioctl_test_rw(f, &cmd); + break; + case VFIO_TYPE1_TEST_PIN: + case VFIO_TYPE1_TEST_UNPIN: + if (ALIGN_DOWN(cmd.iova, PAGE_SIZE) != VFIO_TEST_IOVA || cmd.flags) { + ret = -EINVAL; + break; + } + if (request == VFIO_TYPE1_TEST_PIN) { + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + cmd.iova, 1, IOMMU_READ | IOMMU_WRITE, &page); + if (ret == 1) { + f->external_refs++; + ret = 0; + } + } else if (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, cmd.iova, 1); + f->external_refs--; + } else { + ret = -ENOENT; + } + break; + case VFIO_TYPE1_TEST_EMULATED: + if (cmd.flags > 1 || f->external_refs) { + ret = -EBUSY; + break; + } + mutex_lock(&f->iommu->lock); + if (cmd.flags && list_empty(&f->group.next)) { + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + if (!f->group.pinned_page_dirty_scope) + f->iommu->num_non_pinned_groups++; + } else if (!cmd.flags && !list_empty(&f->group.next)) { + list_del_init(&f->group.next); + if (!f->group.pinned_page_dirty_scope) + f->iommu->num_non_pinned_groups--; + } + mutex_unlock(&f->iommu->lock); + break; + case VFIO_TYPE1_TEST_STATE: + cmd.locked_bytes = 0; + cmd.owner_locked_bytes = mm_locked_vm_bytes(current->mm); + cmd.flags = 0; + cmd.owner_users = 0; + mutex_lock(&f->iommu->lock); + for (struct rb_node *node = rb_first(&f->iommu->dma_list); + node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + cmd.locked_bytes += dma->locked_vm * PAGE_SIZE; + cmd.owner_locked_bytes = mm_locked_vm_bytes(dma->mm); + cmd.owner_users = atomic_read(&dma->mm->mm_users); + if (dma->fragments) + cmd.flags |= VFIO_TYPE1_TEST_STATE_FRAGMENTS; + } + cmd.native_page_size = PAGE_SIZE; + cmd.minimum_page_size = 1UL << __ffs(f->iommu->pgsize_bitmap); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) + cmd.mapped_bytes[i] = f->domains[i].mapped; + mutex_unlock(&f->iommu->lock); + cmd.notifications = f->notifications; + cmd.external_refs = f->external_refs; + break; + default: + ret = -ENOTTY; + } + if (!ret && copy_to_user(user, &cmd, sizeof(cmd))) + ret = -EFAULT; +out: + mutex_unlock(&ctx->request_lock); + return ret; +} + +static const struct file_operations vfio_ioctl_test_fops = { + .owner = THIS_MODULE, + .open = vfio_ioctl_test_open, + .release = vfio_ioctl_test_release, + .unlocked_ioctl = vfio_ioctl_test_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static struct miscdevice vfio_ioctl_test_device = { + .minor = MISC_DYNAMIC_MINOR, + .name = "aurora-vfio-type1-test", + .fops = &vfio_ioctl_test_fops, + .mode = 0600, +}; + +static int __init vfio_ioctl_test_init(void) +{ + return misc_register(&vfio_ioctl_test_device); +} +late_initcall(vfio_ioctl_test_init); diff --git a/drivers/vfio/vfio_iommu_type1_test.h b/drivers/vfio/vfio_iommu_type1_test.h new file mode 100644 index 0000000000000..36c09030bcf0e --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_test.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* Private disposable-guest test protocol, not a supported VFIO ABI. */ +#ifndef VFIO_IOMMU_TYPE1_TEST_H +#define VFIO_IOMMU_TYPE1_TEST_H +#include +#include + +struct vfio_type1_test_cmd { + __u64 iova; + __u64 size; + __u64 buffer; + __u64 fail_after; + __u64 locked_bytes; + __u64 owner_locked_bytes; + __u64 mapped_bytes[2]; + __u64 native_page_size; + __u64 minimum_page_size; + __u32 domain; + __u32 flags; + __u32 owner_users; + __u32 notifications; + __u32 external_refs; + __u32 reserved; +}; + +#define VFIO_TYPE1_TEST_DOMAIN_ADD _IOWR('V', 1, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_DOMAIN_DEL _IOWR('V', 2, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_RW _IOWR('V', 3, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_PIN _IOWR('V', 4, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_UNPIN _IOWR('V', 5, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_STATE _IOWR('V', 6, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_EMULATED _IOWR('V', 7, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_STATE_FRAGMENTS 1U +#define VFIO_TYPE1_TEST_WRITE 1U +#define VFIO_TYPE1_TEST_WORKER 2U +#endif From ad36aab5acc6e025db2a66192d259a1058d77d02 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:23 -0400 Subject: [PATCH 0429/1579] selftests: exercise the mixed userspace page-size ABI Cover exec policy, AArch32 fallback, mappings, sharing, reclaim, accounting and converted I/O interfaces. Keep extended PID1 fixtures separate from the ordinary page-contract executable. Signed-off-by: Diljit Singh --- .../testing/kunit/configs/arm64_user4k.config | 33 + .../configs/arm64_user4k_data_kasan.config | 18 + .../configs/arm64_user4k_mm_kasan.config | 38 + tools/testing/selftests/arm64/Makefile | 2 +- .../testing/selftests/arm64/user4k/.gitignore | 3 + tools/testing/selftests/arm64/user4k/Makefile | 134 ++ tools/testing/selftests/arm64/user4k/advice.c | 199 +++ .../selftests/arm64/user4k/aio-ring-init.c | 178 +++ .../arm64/user4k/aslr-fixture-init.c | 175 +++ .../selftests/arm64/user4k/bpf-uptr-init.c | 583 ++++++++ .../arm64/user4k/coarse-shmem-init.c | 223 +++ .../arm64/user4k/compat-aslr-probe.c | 134 ++ .../arm64/user4k/compat-fixture-init.c | 149 ++ .../arm64/user4k/compat-mlock-init.c | 62 + .../arm64/user4k/compat-mlock-probe.c | 142 ++ .../user4k/compat-numa-balancing-probe.c | 223 +++ .../arm64/user4k/compat-numa-probe.c | 117 ++ .../arm64/user4k/compat-perf-probe.c | 107 ++ .../arm64/user4k/compat-policy-probe.c | 66 + .../selftests/arm64/user4k/compat-probe.c | 248 ++++ .../selftests/arm64/user4k/compat-script | 1 + .../selftests/arm64/user4k/compat-shm-probe.c | 111 ++ .../selftests/arm64/user4k/compat-uffd-init.c | 154 ++ .../arm64/user4k/compat-uffd-probe.c | 149 ++ .../arm64/user4k/compat-vfio-probe.c | 250 ++++ .../arm64/user4k/context-stress-init.c | 275 ++++ .../arm64/user4k/core-fixture-init.c | 447 ++++++ .../selftests/arm64/user4k/cpu-granule-init.c | 187 +++ .../arm64/user4k/cpu-granule16-init.c | 192 +++ .../selftests/arm64/user4k/dio-fixture-init.c | 266 ++++ .../arm64/user4k/exec-fixture-init.c | 167 +++ .../selftests/arm64/user4k/exec-policy-init.c | 214 +++ .../arm64/user4k/fex-contract-init.c | 52 + .../selftests/arm64/user4k/fex-native-abi.c | 20 + .../selftests/arm64/user4k/fex-x86-contract.c | 132 ++ .../selftests/arm64/user4k/gem-fixture-init.c | 325 ++++ .../arm64/user4k/granule-proc-init.c | 141 ++ .../arm64/user4k/granule-swap-init.c | 248 ++++ .../arm64/user4k/granule-threeway-init.c | 248 ++++ .../selftests/arm64/user4k/iommufd-init.c | 1314 +++++++++++++++++ .../arm64/user4k/lazyfree-fixture-init.c | 274 ++++ .../arm64/user4k/memcg-accounting-init.c | 153 ++ .../arm64/user4k/memcg-pressure-init.c | 417 ++++++ .../arm64/user4k/memory-interfaces.c | 165 +++ .../arm64/user4k/mglru-granule-init.c | 365 +++++ .../selftests/arm64/user4k/mixed-process.c | 209 +++ .../arm64/user4k/mlock-fixture-init.c | 249 ++++ .../arm64/user4k/numa-balancing-init.c | 307 ++++ .../selftests/arm64/user4k/numa-policy-init.c | 236 +++ .../selftests/arm64/user4k/page-contract.c | 232 +++ .../selftests/arm64/user4k/perf-ring-init.c | 323 ++++ .../selftests/arm64/user4k/proc-args.c | 164 ++ .../selftests/arm64/user4k/proc-memory.c | 340 +++++ .../selftests/arm64/user4k/proc-pages.c | 318 ++++ .../selftests/arm64/user4k/reclaim-advice.c | 366 +++++ .../selftests/arm64/user4k/runtime-app-init.c | 76 + .../selftests/arm64/user4k/runtime-app.py | 91 ++ .../arm64/user4k/shmem-collapse-init.c | 396 +++++ .../selftests/arm64/user4k/shmem-hole-init.c | 358 +++++ .../selftests/arm64/user4k/shmem-pin-init.c | 358 +++++ .../arm64/user4k/shmem-pressure-init.c | 225 +++ .../selftests/arm64/user4k/shmem-race-init.c | 308 ++++ .../selftests/arm64/user4k/shmem-thp-init.c | 271 ++++ .../arm64/user4k/shmem-uffd-swap-init.c | 160 ++ .../arm64/user4k/splice-fixture-init.c | 610 ++++++++ .../arm64/user4k/swap-fixture-init.c | 216 +++ .../selftests/arm64/user4k/swapfile-fixture.h | 50 + .../selftests/arm64/user4k/sysv-shm-init.c | 229 +++ .../arm64/user4k/thread-fixture-init.c | 424 ++++++ .../selftests/arm64/user4k/uffd-move-init.c | 451 ++++++ .../selftests/arm64/user4k/uffd-wp-init.c | 913 ++++++++++++ .../selftests/arm64/user4k/uprobes-init.c | 318 ++++ .../arm64/user4k/uring-fixture-init.c | 467 ++++++ .../selftests/arm64/user4k/user-events-init.c | 268 ++++ .../arm64/user4k/user16k-smoke-init.c | 229 +++ .../arm64/user4k/vfio-fixture-init.c | 542 +++++++ .../selftests/arm64/user4k/vhost-log-init.c | 392 +++++ .../selftests/arm64/user4k/wine-kernel32.def | 20 + .../arm64/user4k/wine-page-contract.c | 181 +++ .../selftests/arm64/user4k/x18-probe.S | 25 + tools/testing/vma/include/dup.h | 117 ++ tools/testing/vma/tests/merge.c | 2 +- tools/testing/vma/tests/mmap.c | 8 +- tools/testing/vma/tests/vma.c | 4 +- 84 files changed, 19546 insertions(+), 8 deletions(-) create mode 100644 tools/testing/kunit/configs/arm64_user4k.config create mode 100644 tools/testing/kunit/configs/arm64_user4k_data_kasan.config create mode 100644 tools/testing/kunit/configs/arm64_user4k_mm_kasan.config create mode 100644 tools/testing/selftests/arm64/user4k/.gitignore create mode 100644 tools/testing/selftests/arm64/user4k/Makefile create mode 100644 tools/testing/selftests/arm64/user4k/advice.c create mode 100644 tools/testing/selftests/arm64/user4k/aio-ring-init.c create mode 100644 tools/testing/selftests/arm64/user4k/aslr-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/bpf-uptr-init.c create mode 100644 tools/testing/selftests/arm64/user4k/coarse-shmem-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-aslr-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-mlock-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-mlock-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-numa-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-perf-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-policy-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-script create mode 100644 tools/testing/selftests/arm64/user4k/compat-shm-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-uffd-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-uffd-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-vfio-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/context-stress-init.c create mode 100644 tools/testing/selftests/arm64/user4k/core-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/cpu-granule-init.c create mode 100644 tools/testing/selftests/arm64/user4k/cpu-granule16-init.c create mode 100644 tools/testing/selftests/arm64/user4k/dio-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/exec-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/exec-policy-init.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-contract-init.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-native-abi.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-x86-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/gem-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-proc-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-swap-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-threeway-init.c create mode 100644 tools/testing/selftests/arm64/user4k/iommufd-init.c create mode 100644 tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memcg-accounting-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memcg-pressure-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memory-interfaces.c create mode 100644 tools/testing/selftests/arm64/user4k/mglru-granule-init.c create mode 100644 tools/testing/selftests/arm64/user4k/mixed-process.c create mode 100644 tools/testing/selftests/arm64/user4k/mlock-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/numa-balancing-init.c create mode 100644 tools/testing/selftests/arm64/user4k/numa-policy-init.c create mode 100644 tools/testing/selftests/arm64/user4k/page-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/perf-ring-init.c create mode 100644 tools/testing/selftests/arm64/user4k/proc-args.c create mode 100644 tools/testing/selftests/arm64/user4k/proc-memory.c create mode 100644 tools/testing/selftests/arm64/user4k/proc-pages.c create mode 100644 tools/testing/selftests/arm64/user4k/reclaim-advice.c create mode 100644 tools/testing/selftests/arm64/user4k/runtime-app-init.c create mode 100755 tools/testing/selftests/arm64/user4k/runtime-app.py create mode 100644 tools/testing/selftests/arm64/user4k/shmem-collapse-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-hole-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-pin-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-pressure-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-race-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-thp-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c create mode 100644 tools/testing/selftests/arm64/user4k/splice-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/swap-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/swapfile-fixture.h create mode 100644 tools/testing/selftests/arm64/user4k/sysv-shm-init.c create mode 100644 tools/testing/selftests/arm64/user4k/thread-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uffd-move-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uffd-wp-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uprobes-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uring-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/user-events-init.c create mode 100644 tools/testing/selftests/arm64/user4k/user16k-smoke-init.c create mode 100644 tools/testing/selftests/arm64/user4k/vfio-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/vhost-log-init.c create mode 100644 tools/testing/selftests/arm64/user4k/wine-kernel32.def create mode 100644 tools/testing/selftests/arm64/user4k/wine-page-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/x18-probe.S diff --git a/tools/testing/kunit/configs/arm64_user4k.config b/tools/testing/kunit/configs/arm64_user4k.config new file mode 100644 index 0000000000000..de6c8ec2fdff8 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k.config @@ -0,0 +1,33 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_KUNIT_TEST=y +CONFIG_ARM64_SW_TTBR0_PAN=y +# CONFIG_ARM64_CNP is not set +# CONFIG_ARM64_POE is not set +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_VM=y +CONFIG_DEBUG_VM_MAPLE_TREE=y +CONFIG_DEBUG_VM_RB=y +CONFIG_PROVE_LOCKING=y +CONFIG_PAGE_TABLE_CHECK=y +CONFIG_PAGE_TABLE_CHECK_ENFORCED=y +# Guest-only infrastructure for late swap/reclaim fixtures. +CONFIG_BLOCK=y +CONFIG_VIRTIO=y +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_BLK=y +CONFIG_DEVTMPFS=y +CONFIG_DEBUG_FS=y +CONFIG_KUNIT_DEBUGFS=y diff --git a/tools/testing/kunit/configs/arm64_user4k_data_kasan.config b/tools/testing/kunit/configs/arm64_user4k_data_kasan.config new file mode 100644 index 0000000000000..2e3e50bd67768 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k_data_kasan.config @@ -0,0 +1,18 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_DATA_KUNIT_TEST=y +# CONFIG_ARM64_USER4K_KUNIT_TEST is not set +CONFIG_KASAN=y +CONFIG_KASAN_GENERIC=y +CONFIG_KASAN_INLINE=y +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y diff --git a/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config b/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config new file mode 100644 index 0000000000000..11b9935770f53 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config @@ -0,0 +1,38 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_KUNIT_TEST=y +CONFIG_ARM64_SW_TTBR0_PAN=y +# CONFIG_ARM64_CNP is not set +# CONFIG_ARM64_POE is not set +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_VM=y +CONFIG_DEBUG_VM_MAPLE_TREE=y +CONFIG_DEBUG_VM_RB=y +CONFIG_PROVE_LOCKING=y +CONFIG_PAGE_TABLE_CHECK=y +CONFIG_PAGE_TABLE_CHECK_ENFORCED=y +# Guest-only infrastructure for late swap/reclaim fixtures. +CONFIG_BLOCK=y +CONFIG_VIRTIO=y +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_BLK=y +CONFIG_DEVTMPFS=y +CONFIG_DEBUG_FS=y +CONFIG_KUNIT_DEBUGFS=y + +# Software MM fixtures only; the hardware boot opt-in is rejected under KASAN. +CONFIG_KASAN=y +CONFIG_KASAN_GENERIC=y +CONFIG_KASAN_INLINE=y diff --git a/tools/testing/selftests/arm64/Makefile b/tools/testing/selftests/arm64/Makefile index e456f3b62fa13..d8948f5fb1ef8 100644 --- a/tools/testing/selftests/arm64/Makefile +++ b/tools/testing/selftests/arm64/Makefile @@ -4,7 +4,7 @@ ARCH ?= $(shell uname -m 2>/dev/null || echo not) ifneq (,$(filter $(ARCH),aarch64 arm64)) -ARM64_SUBTARGETS ?= tags signal pauth fp mte bti abi gcs +ARM64_SUBTARGETS ?= tags signal pauth fp mte bti abi gcs user4k else ARM64_SUBTARGETS := endif diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore new file mode 100644 index 0000000000000..2e6224b832f98 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -0,0 +1,3 @@ +page-contract + +swap-fixture-init diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile new file mode 100644 index 0000000000000..cc89cc93293fe --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: GPL-2.0 +TEST_GEN_PROGS := page-contract proc-args +TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init +TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init +TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init +CFLAGS += -Wall -Wextra -O2 -ffixed-x18 +TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe +TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe +TEST_GEN_FILES += compat-shm-probe compat-perf-probe compat-numa-probe compat-numa-balancing-probe +TEST_FILES += compat-script runtime-app.py +include ../../lib.mk + +$(OUTPUT)/page-contract: page-contract.c x18-probe.S + +$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init: LDLIBS += -pthread + +# Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. +AARCH32_CC ?= clang --target=arm-linux-gnueabihf +AARCH32_FLAGS := -march=armv8-a -marm -mfloat-abi=soft -ffreestanding \ + -fno-builtin -fno-stack-protector -O2 -nostdlib -static -fuse-ld=lld +$(OUTPUT)/compat-probe-4k: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=4096 $< -o $@ +$(OUTPUT)/compat-probe-64k: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ +$(OUTPUT)/compat-interp32: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536,--image-base=0x10000000 $< -o $@ +$(OUTPUT)/compat-probe-dynamic: compat-probe.c + $(AARCH32_CC) $(filter-out -static,$(AARCH32_FLAGS)) -no-pie \ + -Wl,-z,max-page-size=65536,--dynamic-linker=/interp32 $< -o $@ + +$(OUTPUT)/compat-aslr-probe: compat-aslr-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/compat-mlock-probe: compat-mlock-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# The disposable PID1 fixture also runs with the native 64K process ABI. +$(OUTPUT)/mglru-granule-init $(OUTPUT)/memcg-pressure-init $(OUTPUT)/memcg-accounting-init $(OUTPUT)/runtime-app-init $(OUTPUT)/fex-contract-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +# FEX fixture: the AArch64 emulator runs the freestanding x86-64 contract. +X86_64_CC ?= clang --target=x86_64-linux-gnu +$(OUTPUT)/fex-x86-contract: fex-x86-contract.c + $(X86_64_CC) -O2 -fno-builtin -fno-stack-protector -nostdlib -static \ + -fuse-ld=lld -Wl,-z,max-page-size=4096,-e,_start $< -o $@ + +$(OUTPUT)/libfex-abi.so: fex-native-abi.c + $(CC) $(CFLAGS) -shared -fPIC -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/exec-policy-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/exec-policy-init: LDLIBS += -pthread +$(OUTPUT)/compat-policy-probe: compat-policy-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# Optional Windows payload; requires Wine/FEX in a disposable guest to execute. +# Keep its toolchain out of the ordinary Linux selftest build. +WINDOWS_CC ?= clang --target=x86_64-pc-windows-msvc +WINDOWS_DLLTOOL ?= llvm-dlltool +WINDOWS_LD ?= lld-link +.PHONY: wine-contract +wine-contract: $(OUTPUT)/wine-page-contract.exe +$(OUTPUT)/wine-kernel32.lib: wine-kernel32.def + $(WINDOWS_DLLTOOL) -m i386:x86-64 -d $< -l $@ +$(OUTPUT)/wine-page-contract.obj: wine-page-contract.c + $(WINDOWS_CC) -O2 -Wall -Wextra -Werror -ffreestanding -fno-stack-protector -c $< -o $@ +$(OUTPUT)/wine-page-contract.exe: $(OUTPUT)/wine-page-contract.obj $(OUTPUT)/wine-kernel32.lib + $(WINDOWS_LD) /entry:mainCRTStartup /subsystem:console /nodefaultlib /out:$@ $^ +EXTRA_CLEAN += $(OUTPUT)/wine-page-contract.exe $(OUTPUT)/wine-page-contract.obj $(OUTPUT)/wine-kernel32.lib + +$(OUTPUT)/uffd-wp-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/uffd-wp-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-uffd-swap-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/compat-uffd-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-uffd-probe: compat-uffd-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/shmem-hole-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-hole-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-thp-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-thp-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-collapse-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/shmem-pin-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-pin-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-race-init $(OUTPUT)/shmem-pressure-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-race-init: LDLIBS += -pthread + +$(OUTPUT)/uffd-move-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/uffd-move-init: LDLIBS += -pthread + +$(OUTPUT)/user-events-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/uprobes-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/vhost-log-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/bpf-uptr-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/coarse-shmem-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/iommufd-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/iommufd-init: LDLIBS += -pthread +$(OUTPUT)/iommufd-init: CFLAGS += $(KHDR_INCLUDES) + +# Privileged recording-domain fixture; run only in a disposable test VM. +$(OUTPUT)/vfio-fixture-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-vfio-probe: compat-vfio-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/sysv-shm-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-shm-probe: compat-shm-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# Disposable guests exercise perf layout and software sampling across mm granules. +$(OUTPUT)/perf-ring-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-perf-probe: compat-perf-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/aio-ring-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +# Requires a disposable guest with two memory nodes and CONFIG_COMPAT. +$(OUTPUT)/numa-policy-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-numa-probe: compat-numa-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/numa-balancing-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/numa-balancing-init: LDLIBS += -pthread + +$(OUTPUT)/compat-numa-balancing-probe: compat-numa-balancing-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ diff --git a/tools/testing/selftests/arm64/user4k/advice.c b/tools/testing/selftests/arm64/user4k/advice.c new file mode 100644 index 0000000000000..4d4cb89350985 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/advice.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static size_t page; + +static bool wait_child(pid_t pid, int sig) +{ + int status = 0; + + if (pid <= 0 || waitpid(pid, &status, 0) != pid) + return false; + if (sig) + return WIFSIGNALED(status) && WTERMSIG(status) == sig; + return WIFEXITED(status) && !WEXITSTATUS(status); +} + +static void *anon(size_t length, int prot, int extra) +{ + void *p = mmap(NULL, length, prot, MAP_PRIVATE | MAP_ANONYMOUS | extra, + -1, 0); + + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap\n"); + return p; +} + +static void discard_and_guards(void) +{ + unsigned char *p = anon(3 * page, PROT_READ | PROT_WRITE, 0); + bool ok; + pid_t child; + + memset(p, 0x61, 3 * page); + ok = madvise(p + page, 1, MADV_DONTNEED) == 0; + ok &= p[0] == 0x61 && p[page] == 0 && p[2 * page] == 0x61; + ksft_test_result(ok, + "DONTNEED discards exactly one rounded user page\n"); + memset(p + page, 0x71, page); + child = fork(); + if (!child) { + ok = madvise(p + page, page, MADV_DONTNEED) == 0; + _exit(!(ok && p[page] == 0 && p[0] == 0x61)); + } + ksft_test_result( + wait_child(child, 0) && p[page] == 0x71, + "discard in a fork child leaves the parent quarter intact\n"); + ok = madvise(p + page, 1, MADV_GUARD_INSTALL) == 0; + ok &= madvise(p + page, page, MADV_DONTNEED) == 0; + child = fork(); + if (!child) { + volatile unsigned char value = p[page]; + + (void)value; + _exit(1); + } + ksft_test_result(ok && wait_child(child, SIGSEGV), + "a user-page guard survives discard and fork\n"); + ok = madvise(p + page, 1, MADV_GUARD_REMOVE) == 0; + ok &= p[page] == 0 && p[0] == 0x61 && p[2 * page] == 0x61; + ksft_test_result(ok, + "guard removal restores only the requested page\n"); + munmap(p, 3 * page); +} + +static void population(void) +{ + unsigned char *p = anon(5 * page, PROT_READ, 0), vec[5]; + bool ok; + unsigned int i; + + ok = mprotect(p + 2 * page, page, PROT_READ | PROT_WRITE) == 0; + ok &= madvise(p + 2 * page, 1, MADV_POPULATE_WRITE) == 0; + ok &= mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == (i == 2); + ksft_test_result( + ok, "POPULATE_WRITE faults only the selected user-page VMA\n"); + ok = madvise(p + page, 1, MADV_POPULATE_READ) == 0; + ok &= mincore(p + page, page, vec) == 0 && vec[0] == 1; + errno = 0; + ok &= madvise(p, page, MADV_POPULATE_WRITE) == -1 && errno == EINVAL; + ok &= mprotect(p + 3 * page, page, PROT_NONE) == 0; + errno = 0; + ok &= madvise(p + 3 * page, page, MADV_POPULATE_READ) == -1 && + errno == EINVAL; + ok &= munmap(p + 3 * page, page) == 0; + errno = 0; + ok &= madvise(p + 3 * page, page, MADV_POPULATE_READ) == -1 && + errno == ENOMEM; + ksft_test_result( + ok, "POPULATE_READ honors permissions and reports holes\n"); + munmap(p, 5 * page); + p = anon(5 * page, PROT_READ | PROT_WRITE, MAP_POPULATE); + ok = mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == 1; + ksft_test_result(ok, "MAP_POPULATE materializes every user page\n"); + munmap(p, 5 * page); +} + +static void fork_advice(void) +{ + unsigned char *p = anon(3 * page, PROT_READ | PROT_WRITE, 0); + bool ok; + pid_t child; + + memset(p, 0x58, 3 * page); + ok = madvise(p + page, 1, MADV_WIPEONFORK) == 0; + child = fork(); + if (!child) + _exit(!(p[page] == 0 && p[0] == 0x58 && p[2 * page] == 0x58)); + ok &= wait_child(child, 0) && p[page] == 0x58; + ksft_test_result( + ok, + "WIPEONFORK clears a single child page and preserves neighbors\n"); + ok = madvise(p + page, 1, MADV_KEEPONFORK) == 0; + ok &= madvise(p + page, 1, MADV_DONTFORK) == 0; + child = fork(); + if (!child) { + unsigned char vec; + + errno = 0; + _exit(!(mincore(p + page, page, &vec) == -1 && + errno == ENOMEM && p[0] == 0x58 && + p[2 * page] == 0x58)); + } + ok &= wait_child(child, 0); + ksft_test_result(ok, "DONTFORK omits only its selected user page\n"); + munmap(p, 3 * page); +} + +static void remove_file(void) +{ + int fd = memfd_create("advice-file", 0); + unsigned char *shared, *alias, *private; + bool ok; + size_t i; + + if (fd < 0 || ftruncate(fd, 5 * page)) + ksft_exit_fail_msg("memfd setup\n"); + shared = + mmap(NULL, 5 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + alias = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page); + private = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, + page); + if (shared == MAP_FAILED || alias == MAP_FAILED || + private == MAP_FAILED) + ksft_exit_fail_msg("file mmap\n"); + for (i = 0; i < 5 * page; i++) + shared[i] = 0x40 + i / page; + private[page] = 0x79; + ok = madvise(alias + page, 1, MADV_REMOVE) == 0; + for (i = 0; i < 5 * page; i++) + ok &= shared[i] == (i / page == 2 ? 0 : 0x40 + i / page); + ok &= private[page] == 0x79; + ksft_test_result( + ok, + "REMOVE uses the file byte offset and preserves private COW\n"); + ok = madvise(alias, 3 * page, MADV_WILLNEED) == 0; + ksft_test_result( + ok, "WILLNEED accepts a file range with the process granule\n"); + munmap(shared, 5 * page); + munmap(alias, 3 * page); + munmap(private, 3 * page); + close(fd); +} + +int main(int argc, char **argv) +{ + const struct rlimit no_core = { 0, 0 }; + + page = getauxval(AT_PAGESZ); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + ksft_exit_fail_msg("unexpected process page ABI: %zu\n", page); + ksft_print_msg("process page size: %zu\n", page); + if (setrlimit(RLIMIT_CORE, &no_core)) + return 1; + ksft_print_header(); + ksft_set_plan(11); + discard_and_guards(); + population(); + fork_advice(); + remove_file(); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/aio-ring-init.c b/tools/testing/selftests/arm64/user4k/aio-ring-init.c new file mode 100644 index 0000000000000..74f34d16cac29 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/aio-ring-init.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: legacy AIO ring relocation with per-process page sizes. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct aio_ring_header { + unsigned int id, nr, head, tail; + unsigned int magic, compat_features, incompat_features, header_length; +}; + +static void check(int good, const char *expression, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, + errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int wait_child(pid_t child) +{ + int status; + pid_t got; + + if (child < 0) + return 1; + do { + got = waitpid(child, &status, 0); + } while (got < 0 && errno == EINTR); + return got != child || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static void complete(aio_context_t context, struct iocb *request, + unsigned long bytes, int event_fd) +{ + struct io_event event; + struct timespec timeout = { .tv_sec = 5 }; + uint64_t notifications; + + CHECK(syscall(SYS_io_getevents, context, 1, 1, &event, &timeout) == 1); + CHECK(event.data == request->aio_data && + event.obj == (uintptr_t)request); + CHECK(event.res == (long long)bytes && event.res2 == 0); + CHECK(read(event_fd, ¬ifications, sizeof(notifications)) == + sizeof(notifications)); + CHECK(notifications == 1); +} + +static void exercise(unsigned long native, unsigned int requested) +{ + unsigned long page = getauxval(AT_PAGESZ), bytes = 2 * page + 37; + aio_context_t context = 0; + int file = memfd_create("aio-granule", 0); + int event_fd = eventfd(0, EFD_NONBLOCK); + unsigned char *source, *destination, *reserve, *target; + struct aio_ring_header *header; + unsigned long ring_size; + struct iocb request = { 0 }, *requests[] = { &request }; + unsigned char residency; + + CHECK(file >= 0 && event_fd >= 0); + CHECK(!ftruncate(file, 4 * page)); + source = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + destination = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(source != MAP_FAILED && destination != MAP_FAILED); + for (unsigned long i = 0; i < bytes; i++) + source[page - 19 + i] = i * 17 + i / page; + CHECK(!syscall(SYS_io_setup, requested, &context)); + header = (void *)(uintptr_t)context; + CHECK(header->magic == 0xa10a10a1 && + header->header_length == sizeof(*header)); + CHECK(header->nr >= requested); + ring_size = + header->header_length + header->nr * sizeof(struct io_event); + CHECK(ring_size && !(ring_size % native)); + reserve = mmap(NULL, ring_size + 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + target = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + target += native + (page < native ? page : 0); + target[-1] = 0x51; + target[ring_size] = 0x72; + CHECK(!munmap(target, ring_size)); + request.aio_data = 0x12345678; + request.aio_lio_opcode = IOCB_CMD_PWRITE; + request.aio_fildes = file; + request.aio_buf = (uintptr_t)(source + page - 19); + request.aio_nbytes = bytes; + request.aio_offset = 7; + request.aio_flags = IOCB_FLAG_RESFD; + request.aio_resfd = event_fd; + CHECK(syscall(SYS_io_submit, context, 1, requests) == 1); + /* Retain a queued completion while relocating the ring. */ + CHECK(mremap(header, ring_size, ring_size, + MREMAP_MAYMOVE | MREMAP_FIXED, target) == target); + context = (uintptr_t)target; + header = (void *)target; + CHECK(header->magic == 0xa10a10a1); + CHECK(target[-1] == 0x51 && target[ring_size] == 0x72); + complete(context, &request, bytes, event_fd); + request.aio_data = 0x87654321; + request.aio_lio_opcode = IOCB_CMD_PREAD; + request.aio_buf = (uintptr_t)(destination + page - 11); + CHECK(syscall(SYS_io_submit, context, 1, requests) == 1); + complete(context, &request, bytes, event_fd); + CHECK(!memcmp(source + page - 19, destination + page - 11, bytes)); + CHECK(!syscall(SYS_io_destroy, context)); + errno = 0; + CHECK(mincore(target, page, &residency) == -1 && errno == ENOMEM); + CHECK(target[-1] == 0x51 && target[ring_size] == 0x72); + CHECK(!munmap(reserve, ring_size + 4 * native)); + CHECK(!munmap(source, 4 * page) && !munmap(destination, 4 * page)); + CHECK(!close(file) && !close(event_fd)); + printf("ok - AIO page=%lu requested=%u ring=%lu relocation, I/O, eventfd and teardown\n", + page, requested, ring_size); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--case")) { + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + exercise(native, 16); + exercise(native, 4096); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + if (sizes[i] > native) + continue; + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], + 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", n, s, NULL); + _exit(2); + } + int bad = wait_child(child); + + printf("%s - AIO native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, sizes[i]); + failed |= bad; + } + printf("AIO GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c b/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c new file mode 100644 index 0000000000000..4ac03136f2ec0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c @@ -0,0 +1,175 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define SAMPLES 64 +struct sample { + unsigned long page_size, stack, mapping, brk; +}; + +static int get_sample(struct sample *s) +{ + char buf[4096], *pos, *end; + unsigned long low, high; + FILE *f; + int fd; + ssize_t size; + void *p; + + s->page_size = getauxval(AT_PAGESZ); + p = mmap(NULL, s->page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + s->mapping = (unsigned long)p; + f = fopen("/proc/self/maps", "r"); + if (!f) + return 1; + while (fgets(buf, sizeof(buf), f)) + if (strstr(buf, "[stack]") && + sscanf(buf, "%lx-%lx", &low, &high) == 2) + s->stack = high; + fclose(f); + fd = open("/proc/self/stat", O_RDONLY); + if (fd < 0) + return 1; + size = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (size <= 0) + return 1; + buf[size] = 0; + pos = strrchr(buf, ')'); + if (!pos) + return 1; + pos += 2; + /* Fields 1 and 2 precede the closing parenthesis; start_brk is 47. */ + for (unsigned int field = 3; field < 47; field++) { + pos = strchr(pos, ' '); + if (!pos) + return 1; + pos++; + } + errno = 0; + s->brk = strtoul(pos, &end, 10); + return errno || end == pos || !s->brk || !s->stack; +} + +static int collect(unsigned int mode, bool disabled, struct sample *s) +{ + int pipefd[2], status; + pid_t pid; + ssize_t size; + + if (pipe(pipefd)) + return 1; + pid = fork(); + if (!pid) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + if (prctl(SET_EXEC_PAGE_SIZE, mode == 1 ? 4096UL : 0UL, 0UL, + 0UL, 0UL)) + _exit(2); + if (disabled && personality(ADDR_NO_RANDOMIZE) < 0) + _exit(3); + execl(mode == 2 ? "/compat-aslr" : "/init", "/init", "--sample", + fd, NULL); + _exit(4); + } + close(pipefd[1]); + size = read(pipefd[0], s, sizeof(*s)); + close(pipefd[0]); + return pid < 0 || waitpid(pid, &status, 0) != pid || + !WIFEXITED(status) || WEXITSTATUS(status) || size != sizeof(*s); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + if (argc == 3 && !strcmp(argv[1], "--sample")) { + struct sample s = { 0 }; + if (getppid() != 1 || get_sample(&s)) + return 1; + return write(atoi(argv[2]), &s, sizeof(s)) != sizeof(s); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + if (mount("proc", "/proc", "proc", 0, NULL)) + failed++; + int control = open("/proc/sys/kernel/randomize_va_space", O_WRONLY); + if (control < 0 || write(control, "2", 1) != 1) + failed++; + if (control >= 0) + close(control); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + if (prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL)) + failed++; + for (unsigned int mode = 0; mode < 3; mode++) { + if (mode == 1 && !has4k) + continue; + unsigned long ps = mode && has4k ? 4096 : native; + struct sample first = { 0 }; + bool varied[3] = { 0 }, subnative[3] = { 0 }; + int ok = 1; + + for (unsigned int i = 0; i < SAMPLES; i++) { + struct sample s = { 0 }; + if (collect(mode, false, &s)) { + ok = 0; + break; + } + unsigned long address[] = { s.stack, s.mapping, s.brk }; + unsigned long original[] = { first.stack, first.mapping, + first.brk }; + ok &= s.page_size == ps; + for (unsigned int n = 0; n < 3; n++) { + ok &= !(address[n] & (ps - 1)); + varied[n] |= i && address[n] != original[n]; + subnative[n] |= i && + ((address[n] ^ original[n]) & + (native - 1)); + } + if (!i) + first = s; + } + for (unsigned int n = 0; n < 3; n++) + ok &= varied[n] && (ps == native || subnative[n]); + printf("%s - %s %luK ASLR: 64 execs, stack/mmap/brk varied=%d%d%d low-bits-varied=%d%d%d\n", + ok ? "ok" : "not ok", mode == 2 ? "AArch32" : "AArch64", + ps / 1024, varied[0], varied[1], varied[2], subnative[0], + subnative[1], subnative[2]); + failed += !ok; + struct sample a = { 0 }, b = { 0 }; + ok = !collect(mode, true, &a) && !collect(mode, true, &b) && + !memcmp(&a, &b, sizeof(a)) && a.page_size == ps; + printf("%s - %s %luK ADDR_NO_RANDOMIZE retains stable layout\n", + ok ? "ok" : "not ok", mode == 2 ? "AArch32" : "AArch64", + ps / 1024); + failed += !ok; + } + printf("GRANULE ASLR %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c b/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c new file mode 100644 index 0000000000000..33793cebc2b8c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c @@ -0,0 +1,583 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +struct user_data { + uint64_t a, b, result, marker; +}; +struct value { + struct user_data *p, *q; + uint64_t cookie; + uint8_t trailing; +} __attribute__((packed)); +static char verifier_log[65536]; +static int bpf(enum bpf_cmd cmd, union bpf_attr *a) +{ + size_t size; +#define END(field) (offsetof(union bpf_attr, field) + sizeof(a->field)) + switch (cmd) { + case BPF_BTF_LOAD: + size = END(btf_log_level); + break; + case BPF_MAP_CREATE: + size = END(btf_value_type_id); + break; + case BPF_PROG_LOAD: + size = END(log_buf); + break; + case BPF_RAW_TRACEPOINT_OPEN: + size = END(raw_tracepoint.prog_fd); + break; + case BPF_MAP_UPDATE_ELEM: + size = END(flags); + break; + case BPF_MAP_LOOKUP_ELEM: + size = END(flags); + break; + case BPF_MAP_DELETE_ELEM: + size = END(key); + break; + default: + errno = EINVAL; + return -1; + } +#undef END + return syscall(__NR_bpf, cmd, a, size); +} +struct btf_blob { + unsigned char types[1024]; + char strings[512]; + unsigned nt, ns; +}; +static uint32_t str(struct btf_blob *b, const char *s) +{ + unsigned at = b->ns; + size_t n = strlen(s) + 1; + memcpy(b->strings + b->ns, s, n); + b->ns += n; + return at; +} +static void push(struct btf_blob *b, const void *p, size_t n) +{ + memcpy(b->types + b->nt, p, n); + b->nt += n; +} +static void type(struct btf_blob *b, const char *name, unsigned kind, + unsigned n, unsigned size_type) +{ + struct btf_type t = { .name_off = name ? str(b, name) : 0, + .info = kind << 24 | n, + .size = size_type }; + push(b, &t, sizeof(t)); +} +static void member(struct btf_blob *b, const char *name, unsigned id, + unsigned offset) +{ + struct btf_member m = { .name_off = str(b, name), + .type = id, + .offset = offset * 8 }; + push(b, &m, sizeof(m)); +} +static int create_map(void) +{ + struct btf_blob b = { .ns = 1 }; + uint32_t encoding; + type(&b, "int", BTF_KIND_INT, 0, 4); + encoding = BTF_INT_SIGNED << 24 | 32; + push(&b, &encoding, 4); + type(&b, "u64", BTF_KIND_INT, 0, 8); + encoding = 64; + push(&b, &encoding, 4); + type(&b, "user_data", BTF_KIND_STRUCT, 4, sizeof(struct user_data)); + member(&b, "a", 2, 0); + member(&b, "b", 2, 8); + member(&b, "result", 2, 16); + member(&b, "marker", 2, 24); + type(&b, "uptr", BTF_KIND_TYPE_TAG, 0, 3); + type(&b, NULL, BTF_KIND_PTR, 0, 4); + type(&b, "value_type", BTF_KIND_STRUCT, 4, sizeof(struct value)); + member(&b, "p", 5, 0); + member(&b, "q", 5, 8); + member(&b, "cookie", 2, 16); + member(&b, "trailing", 7, 24); + type(&b, "u8", BTF_KIND_INT, 0, 1); + encoding = 8; + push(&b, &encoding, 4); + unsigned char data[2048]; + struct btf_header h = { .magic = BTF_MAGIC, + .version = BTF_VERSION, + .hdr_len = sizeof(h), + .type_len = b.nt, + .str_off = b.nt, + .str_len = b.ns }; + memcpy(data, &h, sizeof(h)); + memcpy(data + sizeof(h), b.types, b.nt); + memcpy(data + sizeof(h) + b.nt, b.strings, b.ns); + union bpf_attr a = { .btf = (uintptr_t)data, + .btf_size = sizeof(h) + b.nt + b.ns, + .btf_log_buf = (uintptr_t)verifier_log, + .btf_log_size = sizeof(verifier_log), + .btf_log_level = 1 }; + int btf = bpf(BPF_BTF_LOAD, &a); + if (btf < 0) { + printf("BTF load errno=%d: %s\n", errno, verifier_log); + return -1; + } + a = (union bpf_attr){ .map_type = BPF_MAP_TYPE_TASK_STORAGE, + .key_size = sizeof(int), + .value_size = sizeof(struct value), + .map_flags = BPF_F_NO_PREALLOC, + .btf_fd = btf, + .btf_key_type_id = 1, + .btf_value_type_id = 6 }; + int fd = bpf(BPF_MAP_CREATE, &a); + close(btf); + return fd; +} +#define I(c, d, s, o, i) \ + ((struct bpf_insn){ .code = (c), \ + .dst_reg = (d), \ + .src_reg = (s), \ + .off = (o), \ + .imm = (i) }) +static int create_prog(int map, int *link) +{ + struct bpf_insn insns[40]; + unsigned n = 0, jumps[4]; +#define EMIT(c, d, s, o, i) (insns[n++] = I(c, d, s, o, i)) + EMIT(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_current_task_btf); + EMIT(BPF_ALU64 | BPF_MOV | BPF_X, 2, 0, 0, 0); + EMIT(BPF_LD | BPF_DW | BPF_IMM, 1, BPF_PSEUDO_MAP_FD, 0, map); + EMIT(0, 0, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 3, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 4, 0, 0, 0); + EMIT(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_task_storage_get); + jumps[0] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 0, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_X, 6, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 7, 6, 16, 0); + jumps[1] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 7, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 1, 6, 0, 0); + jumps[2] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 1, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 2, 1, 0, 0); + EMIT(BPF_ALU64 | BPF_XOR | BPF_X, 2, 7, 0, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 2, 16, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 7, 24, 0); + insns[jumps[2]].off = n - jumps[2] - 1; + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 1, 6, 8, 0); + jumps[3] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 1, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 2, 1, 8, 0); + EMIT(BPF_ALU64 | BPF_ADD | BPF_X, 2, 7, 0, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 2, 16, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 7, 24, 0); + for (unsigned i = 0; i < 4; i++) + if (i != 2) + insns[jumps[i]].off = n - jumps[i] - 1; + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 0, 0, 0, 0); + EMIT(BPF_JMP | BPF_EXIT, 0, 0, 0, 0); +#undef EMIT + union bpf_attr a = { .prog_type = BPF_PROG_TYPE_RAW_TRACEPOINT, + .insn_cnt = n, + .insns = (uintptr_t)insns, + .license = (uintptr_t)"GPL", + .log_buf = (uintptr_t)verifier_log, + .log_size = sizeof(verifier_log), + .log_level = 1 }; + verifier_log[0] = 0; + int fd = bpf(BPF_PROG_LOAD, &a); + if (fd < 0) { + printf("prog load errno=%d: %s\n", errno, verifier_log); + return -1; + } + a = (union bpf_attr){}; + a.raw_tracepoint.name = (uintptr_t)"sys_enter"; + a.raw_tracepoint.prog_fd = fd; + *link = bpf(BPF_RAW_TRACEPOINT_OPEN, &a); + if (*link < 0) { + close(fd); + return -1; + } + return fd; +} +static int update(int map, int key, struct value *v, unsigned flags) +{ + union bpf_attr a = { .map_fd = map, + .key = (uintptr_t)&key, + .value = (uintptr_t)v, + .flags = flags }; + return bpf(BPF_MAP_UPDATE_ELEM, &a); +} +static int delete(int map, int key) +{ + union bpf_attr a = { .map_fd = map, .key = (uintptr_t)&key }; + return bpf(BPF_MAP_DELETE_ELEM, &a); +} +struct pins { + unsigned long long acquired, released; +}; +static int pin_stats(struct pins *p) +{ + int fd = open("/proc/sys/vm/stat_refresh", O_RDONLY); + char c; + if (fd < 0) + return -1; + int ret = read(fd, &c, 1); + close(fd); + if (ret < 0) + return -1; + FILE *f = fopen("/proc/vmstat", "r"); + if (!f) + return -1; + char name[128]; + unsigned long long value; + unsigned found = 0; + while (fscanf(f, "%127s %llu", name, &value) == 2) { + if (!strcmp(name, "nr_foll_pin_acquired")) { + p->acquired = value; + found |= 1; + } + if (!strcmp(name, "nr_foll_pin_released")) { + p->released = value; + found |= 2; + } + } + fclose(f); + return found == 3 ? 0 : -1; +} +static int wait_pins(struct pins base, unsigned active) +{ + struct pins p = {}; + for (unsigned i = 0; i < 500; i++) { + if (pin_stats(&p)) + return 0; + if (p.acquired - base.acquired >= active && + p.acquired - base.acquired - (p.released - base.released) == + active) + return 1; + usleep(10000); + } + printf("pin delta acquired=%llu released=%llu expected-active=%u\n", + p.acquired - base.acquired, p.released - base.released, active); + return 0; +} +static int filled(unsigned char *p, size_t len, unsigned char value) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value) + return 0; + return 1; +} +static int values(struct user_data *p, struct user_data *q, uint64_t cookie) +{ + return p->result == (p->a ^ cookie) && q->result == q->b + cookie && + p->marker == cookie && q->marker == cookie; +} +static int exercise_case(size_t ps, unsigned shared) +{ + struct pins base; + CHECK(!pin_stats(&base)); + int backing = -1; + if (shared) { + backing = memfd_create("bpf-uptr", 0); + CHECK(backing >= 0 && !ftruncate(backing, 4 * ps)); + } + unsigned char *src = mmap( + (void *)0x20000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), + backing, 0); + unsigned char *dst = + mmap((void *)0x30000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, 4 * ps); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps; + struct user_data *p = (void *)(leaf + ps - 64), *q = p + 1; + *p = (struct user_data){ .a = 0x12345678, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x87654321 }; + int map = create_map(); + CHECK(map >= 0); + int link = -1, prog = create_prog(map, &link); + CHECK(prog >= 0 && link >= 0); + int key = syscall(SYS_pidfd_open, getpid(), 0); + CHECK(key >= 0); + struct value probe = {}; + int probe_ret = update(map, key, &probe, BPF_NOEXIST); + CHECK(!probe_ret); + CHECK(!delete(map, key)); + struct value v = { + .p = p, .q = q, .cookie = 0xabc001, .trailing = 0x7d + }; + CHECK(!update(map, key, &v, BPF_NOEXIST)); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + for (unsigned i = 0; i < 64; i++) { + v.cookie++; + CHECK(!update(map, key, &v, BPF_EXIST)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + } + CHECK(wait_pins(base, 2)); + struct { + struct value v; + unsigned char tail[16]; + } lookup; + memset(&lookup, 0xa5, sizeof(lookup)); + union bpf_attr a = { .map_fd = map, + .key = (uintptr_t)&key, + .value = (uintptr_t)&lookup }; + CHECK(!bpf(BPF_MAP_LOOKUP_ELEM, &a)); + CHECK(!lookup.v.p && !lookup.v.q && lookup.v.cookie == v.cookie && + lookup.v.trailing == v.trailing && + filled(lookup.tail, sizeof(lookup.tail), 0xa5)); + errno = 0; + CHECK(update(map, key, &v, BPF_NOEXIST) == -1 && errno == EEXIST); + CHECK(wait_pins(base, 2)); + struct value half = v; + half.p = NULL; + CHECK(!update(map, key, &half, BPF_EXIST)); + CHECK(wait_pins(base, 1)); + p->result = p->marker = 0; + syscall(SYS_getpid); + CHECK(!p->result && !p->marker && q->result == q->b + v.cookie); + CHECK(!update(map, key, &v, BPF_EXIST)); + CHECK(wait_pins(base, 2)); + struct value bad = v; + bad.q = (void *)5; + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + bad.q = (void *)(leaf + ps - 16); + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EOPNOTSUPP); + CHECK(wait_pins(base, 2)); + bad.q = (void *)(UINTPTR_MAX - 15); + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(wait_pins(base, 2)); + CHECK(!mprotect(leaf, ps, PROT_READ)); + errno = 0; + CHECK(update(map, key, &v, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(!mprotect(leaf, ps, PROT_READ | PROT_WRITE)); + CHECK(wait_pins(base, 2)); + struct value empty = {}; + CHECK(!update(map, key, &empty, BPF_EXIST)); + CHECK(wait_pins(base, 0)); + p->result = q->result = p->marker = q->marker = 0; + syscall(SYS_getpid); + CHECK(!p->result && !q->result); + CHECK(!update(map, key, &v, BPF_EXIST)); + CHECK(wait_pins(base, 2)); + /* The child owns storage pointing into the parent's pinned memory. */ + int barrier[2]; + CHECK(!pipe(barrier)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + close(barrier[1]); + char token; + if (read(barrier[0], &token, 1) != 1) + _exit(3); + if (!shared) { + p->a = 1; + q->b = 2; + } + syscall(SYS_getpid); + _exit(0); + } + close(barrier[0]); + int childkey = syscall(SYS_pidfd_open, child, 0); + CHECK(childkey >= 0); + CHECK(!update(map, childkey, &v, BPF_NOEXIST)); + CHECK(!delete(map, key)); + CHECK(wait_pins(base, 2)); + p->result = q->result = p->marker = q->marker = 0; + CHECK(write(barrier[1], "x", 1) == 1); + close(barrier[1]); + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + close(childkey); + CHECK(values(p, q, v.cookie) && p->a == 0x12345678 && + q->b == 0x87654321); + CHECK(wait_pins(base, 0)); + /* The supplying mm exits while a different task retains its UPTRs. */ + int foreign_pipe[2]; + CHECK(!pipe(foreign_pipe)); + pid_t supplier = fork(); + CHECK(supplier >= 0); + if (!supplier) { + close(foreign_pipe[0]); + *p = (struct user_data){ .a = 0x1111, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x2222 }; + struct value foreign = { .p = p, .q = q, .cookie = 0x765432 }; + if (update(map, key, &foreign, BPF_NOEXIST)) + _exit(4); + struct iovec data = { .iov_base = p, .iov_len = 64 }; + if (vmsplice(foreign_pipe[1], &data, 1, 0) != 64) + _exit(5); + _exit(0); + } + close(foreign_pipe[1]); + CHECK(waitpid(supplier, &status, 0) == supplier && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + struct user_data retired[2]; + size_t done = 0; + while (done < sizeof(retired)) { + ssize_t got = read(foreign_pipe[0], (char *)retired + done, + sizeof(retired) - done); + CHECK(got > 0); + done += got; + } + close(foreign_pipe[0]); + CHECK(retired[0].a == 0x1111 && retired[1].b == 0x2222 && + values(retired, retired + 1, 0x765432)); + CHECK(!delete(map, key) && wait_pins(base, 0)); + *p = (struct user_data){ .a = 0x12345678, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x87654321 }; + CHECK(!update(map, key, &v, BPF_NOEXIST)); + CHECK(wait_pins(base, 2)); + CHECK(filled(leaf, ps - 64, 0x64)); + int pipefd[2]; + CHECK(!pipe(pipefd)); + struct iovec iov = { .iov_base = leaf, .iov_len = ps }; + CHECK(vmsplice(pipefd[1], &iov, 1, 0) == (ssize_t)ps); + CHECK(!munmap(leaf, ps)); + CHECK(mmap(leaf, ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, + 0) == leaf); + memset(leaf, 0xe7, ps); + unsigned char *old = malloc(ps); + CHECK(old); + size_t n = 0; + while (n < ps) { + ssize_t got = read(pipefd[0], old + n, ps - n); + CHECK(got > 0); + n += got; + } + close(pipefd[0]); + close(pipefd[1]); + CHECK(filled(old, ps - 64, 0x64) && + values((void *)(old + ps - 64), (void *)(old + ps - 32), + v.cookie)); + CHECK(filled(leaf, ps, 0xe7)); + free(old); + /* Closing the map alone is insufficient: the program retains its map ref. */ + close(map); + close(link); + close(prog); + close(key); + CHECK(wait_pins(base, 0)); + CHECK(filled(dst, ps, 0x91) && filled(dst + 2 * ps, 2 * ps, 0x91)); + for (unsigned i = 0; i < 3; i++) + CHECK(filled(src + i * ps, ps, 0x61 + i)); + CHECK(!munmap(src, 3 * ps) && !munmap(dst, 4 * ps)); + if (backing >= 0) + close(backing); + printf("ok - %zuK BPF UPTR %s: relocated writes, updates, rollback, storage/supplier exit, unmap and pin balance\n", + ps / 1024, shared ? "shared" : "private"); + return 0; +} +static int exercise(size_t ps) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(90); + for (unsigned shared = 0; shared < 2; shared++) + CHECK(!exercise_case(ps, shared)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char size[24]; + snprintf(size, sizeof(size), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", size, NULL); + _exit(5); + } + int ok = 0; + for (unsigned ticks = 0; c > 0 && ticks < 1000; ticks++) { + pid_t got = waitpid(c, &status, WNOHANG); + if (got == c) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 999) { + kill(c, SIGKILL); + waitpid(c, &status, 0); + break; + } + usleep(100000); + } + printf("%s - ABI %zuK BPF UPTR status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("BPF UPTR %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c b/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c new file mode 100644 index 0000000000000..2a00ca546751e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1 runner; fault injection is selected by the KUnit worker. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "swapfile-fixture.h" + +static int setting(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY | O_CLOEXEC); + ssize_t n; + + if (fd < 0) + return -1; + n = write(fd, value, strlen(value)); + close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} + +static int fault_setting(const char *kind, const char *name, const char *value) +{ + char path[192]; + + snprintf(path, sizeof(path), "/sys/kernel/debug/%s/%s", kind, name); + return setting(path, value); +} + +static int injection_off(void) +{ + int ret = fault_setting("failslab", "probability", "0"); + + ret |= fault_setting("fail_page_alloc", "probability", "0"); + ret |= fault_setting("failslab", "ignore-gfp-wait", "Y"); + ret |= fault_setting("fail_page_alloc", "ignore-gfp-wait", "Y"); + ret |= fault_setting("fail_page_alloc", "ignore-gfp-highmem", "Y"); + return ret; +} + +static int injection_on(bool page) +{ + const char *kind = page ? "fail_page_alloc" : "failslab"; + + if (injection_off() || fault_setting(kind, "task-filter", "Y") || + fault_setting(kind, "verbose", "0") || + fault_setting(kind, "ignore-gfp-wait", "N") || + fault_setting(kind, "times", "-1") || + fault_setting(kind, "interval", "1") || + fault_setting(kind, "space", "0")) + return -1; + if (!page) + return 0; /* Only current->fail_nth selects the slab allocation. */ + return fault_setting(kind, "ignore-gfp-highmem", "N") || + fault_setting(kind, "min-order", "2") || + fault_setting(kind, "probability", "100"); +} + +static int move_worker(const char *text, bool enter, bool swap_limit) +{ + char *end, path[192], pid[32], comm[64]; + long nr = strtol(text, &end, 10); + const char *group = swap_limit ? "coarse-swap-limit" : "coarse-zero"; + int fd; + ssize_t n; + + if (*end || nr <= 1 || nr == getpid()) + return -1; + snprintf(path, sizeof(path), "/proc/%ld/comm", nr); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = read(fd, comm, sizeof(comm) - 1); + close(fd); + if (n < 1) + return -1; + comm[n] = 0; + if (strncmp(comm, "kunit_", 6)) + return -1; + if (enter) { + if (mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) && + errno != EBUSY) + return -1; + if (setting("/sys/fs/cgroup/cgroup.subtree_control", "+memory")) + return -1; + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s", group); + if (mkdir(path, 0755) && errno != EEXIST) + return -1; + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s/memory.max", + group); + if (setting(path, swap_limit ? "max" : "0")) + return -1; + if (swap_limit && + setting("/sys/fs/cgroup/coarse-swap-limit/memory.swap.max", + "0")) + return -1; + } + snprintf(pid, sizeof(pid), "%ld", nr); + if (enter) + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s/cgroup.procs", + group); + else + snprintf(path, sizeof(path), "/sys/fs/cgroup/cgroup.procs"); + return setting(path, pid); +} + +static int helper(const char *op) +{ + struct stat init, self; + + if (getpid() == 1 || stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || init.st_dev != self.st_dev || + init.st_ino != self.st_ino) + return 1; + if (!strcmp(op, "--coarse-fixture-check")) + return getauxval(AT_PAGESZ) != 16384; + if (!strcmp(op, "--swapoff-helper")) + return swapoff(swap_path()) != 0; + if (!strcmp(op, "--swapon-helper")) + return swapon(swap_path(), 0) != 0; + if (!strncmp(op, "--coarse-swap-enter=", 20)) + return move_worker(op + 20, true, true) != 0; + if (!strncmp(op, "--coarse-swap-leave=", 20)) + return move_worker(op + 20, false, true) != 0; + if (!strcmp(op, "--coarse-swap-unlimit")) + return setting("/sys/fs/cgroup/coarse-swap-limit/memory.swap.max", + "max") != 0; + if (!strcmp(op, "--coarse-injection-off")) + return injection_off() != 0; + if (!strcmp(op, "--coarse-slab-on")) + return injection_on(false) != 0; + if (!strcmp(op, "--coarse-page-on")) + return injection_on(true) != 0; + if (!strncmp(op, "--coarse-memcg-enter=", 21)) + return move_worker(op + 21, true, false) != 0; + if (!strncmp(op, "--coarse-memcg-leave=", 21)) + return move_worker(op + 21, false, false) != 0; + return 1; +} + +static int run_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-coarse-shmem/"; + char path[192], *log = calloc(1, 1024 * 1024); + size_t used = 0, capacity = 1024 * 1024; + ssize_t n = 0; + int fd, ret = -1; + + if (!log) + return -1; + snprintf(path, sizeof(path), "%srun", base); + if (setting(path, "1")) + goto out; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + goto out; + while (used < capacity - 1 && + (n = read(fd, log + used, capacity - used - 1)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (!n && used < capacity - 1 && + strstr(log, "\nok 1 arm64-user4k-coarse-shmem\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; +out: + free(log); + return ret; +} + +int main(int argc, char **argv) +{ + struct stat st; + bool active = false; + int ret = 1; + unsigned int i; + + if (getpid() != 1 && argc == 2) + return helper(argv[1]); + if (getpid() != 1 || + (argc != 1 && (argc != 2 || strcmp(argv[1], "--swapfile")))) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + if (getauxval(AT_PAGESZ) != 16384) + goto out; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) + goto out; + for (i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode)) + goto out; + if (argc == 2 && prepare_swapfile()) + goto out; + printf("COARSE SWAP BACKING=%s\n", swap_path()); + if (swapon(swap_path(), 0)) + goto out; + active = true; + ret = run_suite() ? 1 : 0; + if (injection_off()) + ret = 1; +out: + if (active && swapoff(swap_path())) + ret = 1; + if (active && argc == 2 && umount("/swapdir")) + ret = 1; + printf("COARSE SHMEM SWAP PASS=%d errno=%d\n", !ret, errno); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c b/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c new file mode 100644 index 0000000000000..3a88db7511215 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding ELF32 address sampler for the disposable ASLR fixture. */ +typedef unsigned int u32; +typedef unsigned long long u64; +struct sample { + u64 page_size, stack, mapping, brk; +}; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static int error(long value) +{ + return (u32)value >= 0xfffff001U; +} +static unsigned number(const char *p) +{ + unsigned n = 0; + while (*p >= '0' && *p <= '9') + n = n * 10 + *p++ - '0'; + return n; +} +static char *find(char *p, char c) +{ + while (*p && *p != c) + p++; + return *p ? p : (char *)0; +} +static char *find_stack(char *p) +{ + const char name[] = "[stack]"; + while (*p) { + unsigned i = 0; + while (name[i] && p[i] == name[i]) + i++; + if (!name[i]) + return p; + p++; + } + return (char *)0; +} +static unsigned hex(char *p) +{ + unsigned n = 0, v; + for (;;) { + if (*p >= '0' && *p <= '9') + v = *p - '0'; + else if (*p >= 'a' && *p <= 'f') + v = *p - 'a' + 10; + else + return n; + n = (n << 4) | v; + p++; + } +} +static int load(const char *path, char *buf, unsigned size) +{ + long fd = call(5, (long)path, 0, 0, 0, 0, 0), n; + if (error(fd)) + return 1; + n = call(3, fd, (long)buf, size - 1, 0, 0, 0); + call(6, fd, 0, 0, 0, 0, 0); + if (n <= 0) + return 1; + buf[n] = 0; + return 0; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + unsigned argc = *sp++; + char **argv = (void *)sp, buf[4096], *p, *q; + struct sample sample = { 0 }; + long mapping; + int bad = 1; + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + sample.page_size = sp[1]; + sp += 2; + } + if (argc != 3 || call(64, 0, 0, 0, 0, 0, 0) != 1) + goto done; + mapping = call(192, 0, sample.page_size, 3, 0x22, -1, 0); + if (error(mapping)) + goto done; + sample.mapping = (u32)mapping; + if (load("/proc/self/maps", buf, sizeof(buf))) + goto done; + p = find_stack(buf); + if (!p) + goto done; + while (p > buf && p[-1] != '\n') + p--; + p = find(p, '-'); + if (!p) + goto done; + sample.stack = hex(p + 1); + if (load("/proc/self/stat", buf, sizeof(buf))) + goto done; + p = find(buf, ')'); + if (!p) + goto done; + while ((q = find(p + 1, ')'))) + p = q; + p += 2; + for (unsigned field = 3; field < 47; field++) { + p = find(p, ' '); + if (!p) + goto done; + p++; + } + sample.brk = number(p); + if (!sample.page_size || !sample.stack || !sample.brk) + goto done; + bad = call(4, number(argv[2]), (long)&sample, sizeof(sample), 0, 0, + 0) != sizeof(sample); +done: + call(1, bad, 0, 0, 0, 0, 0); + for (;;) { + } +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-fixture-init.c b/tools/testing/selftests/arm64/user4k/compat-fixture-init.c new file mode 100644 index 0000000000000..f576ed93623ed --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-fixture-init.c @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static unsigned long native_size; +static void result(int ok, const char *s) +{ + printf("%s - launcher %s\n", ok ? "ok" : "not ok", s); + if (!ok) + failed++; +} +static int launch(const char *path, char **args, int request, int expect_errno) +{ + pid_t child = fork(); + int status = 0; + if (!child) { + if (request && prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(95); + unsigned char *canary = + mmap(NULL, native_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (canary == MAP_FAILED) + _exit(94); + memset(canary, 0x69, native_size); + int before = prctl(0x41555002, 0UL, 0UL, 0UL, 0UL); + for (int i = 0; i < (expect_errno ? 64 : 1); i++) { + execv(path, args); + if (!expect_errno || errno != expect_errno) + _exit(96); + if (getauxval(AT_PAGESZ) != native_size || + prctl(0x41555002, 0UL, 0UL, 0UL, 0UL) != before) + _exit(93); + for (int j = 0; j < native_size; j++) + if (canary[j] != 0x69) + _exit(92); + } + _exit(0); + } + if (child < 0 || waitpid(child, &status, 0) != child) + return 0; + if (status) + printf("# child %s status=%#x\n", path, status); + return WIFEXITED(status) && !WEXITSTATUS(status); +} +int main(int argc, char **argv) +{ + if (argc > 1 && !strcmp(argv[1], "--returned")) + return getauxval(AT_PAGESZ) == + strtoul(getenv("AURORA_NATIVE_PAGE_SIZE"), + NULL, 10) ? + 0 : + 1; + if (argc > 1 && !strcmp(argv[1], "--returned4k")) + return getauxval(AT_PAGESZ) == 4096 ? 0 : 1; + if (getpid() != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + native_size = getauxval(AT_PAGESZ); + char native_text[24]; + snprintf(native_text, sizeof(native_text), "%lu", native_size); + setenv("AURORA_NATIVE_PAGE_SIZE", native_text, 1); + setvbuf(stdout, NULL, _IONBF, 0); + mount("proc", "/proc", "proc", 0, NULL); + int no4k = argc > 1 && !strcmp(argv[1], "--no4k"), + fd = memfd_create("compat-offsets", 0); + if (no4k) { + errno = 0; + result(prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL) == -1 && + errno == EOPNOTSUPP, + "4KB capability is actually masked"); + } + result(fd >= 0 && !ftruncate(fd, 0x100000000LL + 4 * native_size), + "sparse file fixture above 4GB"); + unsigned char page[4096]; + for (unsigned q = 0; q < 4 * native_size / 4096; q++) { + memset(page, 0x40 + q, sizeof(page)); + if (pwrite(fd, page, sizeof(page), q * 4096) != sizeof(page)) + failed++; + memset(page, 0x80 + q, sizeof(page)); + if (pwrite(fd, page, sizeof(page), 0x100000000LL + q * 4096) != + sizeof(page)) + failed++; + } + char fdstr[24]; + snprintf(fdstr, sizeof(fdstr), "%d", fd); + char *arg = malloc(90001); + for (unsigned i = 0; i < 90000; i++) + arg[i] = 'a' + i % 23; + arg[90000] = 0; + char *args[] = { "/probe32-64k", no4k ? native_text : "4096", fdstr, + arg, NULL }; + result(launch(args[0], args, 0, 0), + "64KB-aligned ELF32 automatically selects available ABI"); + if (!no4k) { + args[0] = "/probe32-4k"; + result(launch(args[0], args, 0, 0), + "4KB-aligned ELF32 executes with 4KB translation"); + args[0] = "/probe32-64k"; + result(launch(args[0], args, 1, 0), + "explicit pending 4KB request allows ELF32"); + } + args[0] = "/script32"; + result(launch(args[0], args, 0, 0), + "script final ELF class determines page granule"); + args[0] = "/probe32-dynamic"; + result(launch(args[0], args, 0, 0), + "ELF32 interpreter mapping uses selected page granule"); + char *chain[] = { + "/probe32-64k", "--exec32", no4k ? native_text : "4096", + fdstr, arg, NULL + }; + result(launch(chain[0], chain, 0, 0), + "ELF32 to ELF32 restages arguments and selects ABI"); + char *back[] = { "/probe32-64k", "--exec64", NULL }; + result(launch(back[0], back, 0, 0), + "ELF32 to ELF64 exec restores native default"); + if (!no4k) { + back[1] = "--exec64-4k"; + result(launch(back[0], back, 0, 0), + "ELF32 can request alternative ELF64 exec granule"); + } + char *big = malloc(200001); + memset(big, 'x', 200000); + big[200000] = 0; + args[0] = "/probe32-64k"; + args[3] = big; + if (!no4k) + result(launch(args[0], args, 0, E2BIG), + "64 oversized transfers return E2BIG and preserve old mm and request"); + printf("COMPAT FIXTURE %s mode=%s\n", failed ? "FAIL" : "PASS", + no4k ? "no4k-native-fallback" : "auto4k"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/compat-mlock-init.c b/tools/testing/selftests/arm64/user4k/compat-mlock-init.c new file mode 100644 index 0000000000000..679c39370cd6d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-mlock-init.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +struct result { + uint32_t pagesize, passed, failed, error; +}; +int main(int argc, char **argv) +{ + struct result out = { 0 }; + unsigned long expected; + int pipes[2], status = -1; + pid_t pid; + size_t done = 0; + int ok; + + if (getpid() != 1 || + (argc != 1 && (argc != 2 || strcmp(argv[1], "--no4k")))) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + expected = argc == 2 ? getauxval(AT_PAGESZ) : 4096; + if (mount("proc", "/proc", "proc", 0, NULL) || pipe(pipes)) + return 3; + pid = fork(); + if (!pid) { + char value[32]; + close(pipes[0]); + if (dup2(pipes[1], 1) < 0) + _exit(4); + close(pipes[1]); + snprintf(value, sizeof(value), "%lu", expected); + execl("/compat-mlock", "/compat-mlock", value, NULL); + _exit(5); + } + close(pipes[1]); + while (done < sizeof(out)) { + ssize_t n = + read(pipes[0], (char *)&out + done, sizeof(out) - done); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + break; + done += n; + } + close(pipes[0]); + ok = pid > 0 && waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status) && done == sizeof(out) && + out.pagesize == expected && out.passed == 6 && !out.failed; + printf("%s - AArch32 memlock expected=%lu actual=%u passed=%u stage=%u error=%d status=%d\n", + ok ? "ok" : "not ok", expected, out.pagesize, out.passed, + out.failed, (int32_t)out.error, status); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c b/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c new file mode 100644 index 0000000000000..97e76dd94bae9 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding ELF32 memlock probe, only for the disposable PID1 fixture. */ +typedef unsigned int u32; +struct result { + u32 pagesize, passed, failed, error; +}; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static int error(long n) +{ + return (u32)n >= 0xfffff001U; +} +static unsigned number(const char *p) +{ + unsigned n = 0; + while (*p >= '0' && *p <= '9') + n = n * 10 + *p++ - '0'; + return n; +} +static long locked_kb(void) +{ + char buf[8192]; + long fd = call(5, (long)"/proc/self/status", 0, 0, 0, 0, 0), n; + const char name[] = "VmLck:"; + + if (error(fd)) + return -1; + n = call(3, fd, (long)buf, sizeof(buf) - 1, 0, 0, 0); + call(6, fd, 0, 0, 0, 0, 0); + if (n <= 0) + return -1; + buf[n] = 0; + for (long i = 0; i < n; i++) { + unsigned j = 0; + while (name[j] && i + j < (unsigned)n && buf[i + j] == name[j]) + j++; + if (!name[j]) { + const char *p = buf + i + j; + while (*p == ' ' || *p == '\t') + p++; + return number(p); + } + } + return -1; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + unsigned argc = *sp++, ps = 0, kb, stage = 1; + char **argv = (void *)sp; + struct result out = { 0 }; + long p, q, pid, ret = -1; + int status = -1; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + ps = sp[1]; + sp += 2; + } + out.pagesize = ps; + if (argc != 2 || call(64, 0, 0, 0, 0, 0, 0) != 1 || + ps != number(argv[1])) + goto done; + kb = ps / 1024; + p = call(192, 0, 8 * ps, 3, 0x22, -1, 0); + if (error(p) || ((u32)p & (ps - 1)) || locked_kb() != 0) + goto done; + ret = call(150, p + ps + 17, ps - 32, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) + goto done; + out.passed++; + stage++; + ret = call(150, p + ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) + goto done; + ret = call(150, p + 2 * ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)(2 * kb)) + goto done; + ret = call(151, p + ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) + goto done; + out.passed++; + stage++; + *(volatile unsigned char *)(p + 2 * ps) = 0x63; + q = call(163, p + 2 * ps, ps, 2 * ps, 1, 0, 0); + ret = q; + if (error(q) || locked_kb() != (long)(2 * kb) || + *(volatile unsigned char *)q != 0x63) + goto done; + for (unsigned i = ps; i < 2 * ps; i++) + if (((volatile unsigned char *)q)[i]) + goto done; + out.passed++; + stage++; + pid = call(2, 0, 0, 0, 0, 0, 0); + if (!pid) { + call(1, locked_kb() != 0, 0, 0, 0, 0, 0); + for (;;) { + } + } + ret = call(114, pid, (long)&status, 0, 0, 0, 0); + if (error(pid) || ret != pid || status || locked_kb() != (long)(2 * kb)) + goto done; + out.passed++; + stage++; + ret = call(91, q, 2 * ps, 0, 0, 0, 0); + if (ret || locked_kb() != 0) + goto done; + out.passed++; + stage++; + ret = call(390, p + 4 * ps, 2 * ps, 1, 0, 0, 0); + if (ret || locked_kb() != (long)(2 * kb)) + goto done; + *(volatile unsigned char *)(p + 4 * ps) = 0x75; + ret = call(151, p + 4 * ps, 2 * ps, 0, 0, 0, 0); + if (ret || locked_kb() != 0) + goto done; + out.passed++; + stage = 0; +done: + out.failed = stage; + out.error = stage ? (u32)ret : 0; + call(4, 1, (long)&out, sizeof(out), 0, 0, 0); + call(1, stage != 0, 0, 0, 0, 0, 0); + for (;;) { + } +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c b/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c new file mode 100644 index 0000000000000..f2c155a888c15 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 automatic NUMA migration and fallback checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 NUMAB FAIL stage=00\n"; + + if (code) { + text[sizeof(text) - 4] = '0' + code / 10; + text[sizeof(text) - 3] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 NUMAB PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void value(const char *label, unsigned int length, u32 n) +{ + char text[16]; + unsigned int pos = sizeof(text); + + text[--pos] = '\n'; + do { + text[--pos] = '0' + n % 10; + n /= 10; + } while (n); + call(4, 1, (long)label, length, 0, 0, 0); + call(4, 1, (long)(text + pos), sizeof(text) - pos, 0, 0, 0); +} + +static u32 seconds(void) +{ + struct { + long long sec, nsec; + } ts; + + /* Use time64 even when CONFIG_COMPAT_32BIT_TIME is disabled. */ + if (call(403, 1, (long)&ts, 0, 0, 0, 0)) + finish(11); + return ts.sec; +} + +static void cpu(unsigned int id) +{ + u32 mask = 1U << id; + + if (call(241, 0, sizeof(mask), (long)&mask, 0, 0, 0)) + finish(12); +} + +static int node_at(unsigned char *address) +{ + int node = -1; + + if (call(320, (long)&node, 0, 0, (long)address, 3, 0)) + finish(13); + return node; +} + +static u32 sample(unsigned char *base, u32 length, u32 page) +{ + u32 moved = 0; + + for (u32 off = 0; off < length; off += page) + if (off != 2 * page) + moved += node_at(base + off + 16) == 1; + return moved; +} + +static void forbidden(unsigned char *address, int write_access) +{ + long child = call(2, 0, 0, 0, 0, 0, 0); + int status = 0; + + if (child < 0) + finish(40); + if (!child) { + /* Force the intended access; this is not thread synchronization. */ + if (write_access) + *(volatile unsigned char *)address = 0x27; + else + (void)*(volatile unsigned char *)address; + call(1, 2, 0, 0, 0, 0, 0); + for (;;) + ; + } + if (call(114, child, (long)&status, 0, 0, 0, 0) != child || + (status & 0x7f) != 11) + finish(41); +} + +static void mapping(u32 page) +{ + const u32 length = 8U << 20; + u32 mask[2] = { 1, 0 }, got[2] = { 0, 0 }; + u32 loops = 0, moved = 0, started, total = length / page - 1; + int mode = -1; + long raw; + unsigned char *base; + + if (call(320, (long)&mode, (long)got, 64, 0, 4, 0) || got[0] != 3 || + got[1]) + finish(20); + cpu(0); + raw = call(192, 0, length, 3, 0x22, -1, 0); + if ((u32)raw >= 0xfffff001U) + finish(21); + base = (void *)raw; + /* A missing 4K capability must retain native alignment restrictions. */ + if (call(125, raw + 4096, 4096, 1, 0, 0, 0) != (page == 4096 ? 0 : -22)) + finish(22); + if (call(125, raw, length, 3, 0, 0, 0)) + finish(23); + if (call(220, raw, length, 15, 0, 0, 0)) /* MADV_NOHUGEPAGE */ + finish(24); + if (call(319, raw, length, 2, (long)mask, 64, 0)) + finish(25); + for (u32 off = 0; off < length; off += page) { + u32 *p = (void *)(base + off); + + p[0] = 0x91720000 + off / page; + p[1] = 0; + p[page / 4 - 1] = 0x51320000 + off / page; + if (node_at(base + off + 16) != 0) + finish(26); + } + if (call(125, raw + page, page, 1, 0, 0, 0) || + call(125, raw + 2 * page, page, 0, 0, 0, 0)) + finish(27); + if (call(319, raw, length, 0, 0, 0, + 0)) /* MPOL_DEFAULT, no MOVE flags */ + finish(28); + cpu(1); + started = seconds(); + do { + for (u32 off = 0; off < length; off += page) { + volatile u32 *p = (void *)(base + off); + + if (off == 2 * page) + continue; + if (p[0] != 0x91720000 + off / page || + p[page / 4 - 1] != 0x51320000 + off / page) + finish(29); + if (off != page) + p[1]++; + } + loops++; + if (!(loops % 128)) { + moved = sample(base, length, page); + if (moved == total) + break; + } + } while (seconds() - started < 30); + moved = sample(base, length, page); + value("A32 NUMAB page=", sizeof("A32 NUMAB page=") - 1, page); + value("A32 NUMAB moved=", sizeof("A32 NUMAB moved=") - 1, moved); + value("A32 NUMAB expected=", sizeof("A32 NUMAB expected=") - 1, total); + value("A32 NUMAB loops=", sizeof("A32 NUMAB loops=") - 1, loops); + if (moved != total) + finish(30); + for (u32 off = 0; off < length; off += page) + if (off != page && off != 2 * page && + *(u32 *)(base + off + 4) != loops) + finish(31); + forbidden(base + page, 1); + forbidden(base + 2 * page, 0); + if (call(125, raw + 2 * page, page, 1, 0, 0, 0) || + *(u32 *)(base + 2 * page) != 0x91720002) + finish(32); + if (call(91, raw, length, 0, 0, 0, 0)) + finish(33); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, page = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + page = sp[1]; + sp += 2; + } + if (argc != 2 || page != number(argv[1])) + finish(10); + mapping(page); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-numa-probe.c b/tools/testing/selftests/arm64/user4k/compat-numa-probe.c new file mode 100644 index 0000000000000..b2652fe348b6e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-numa-probe.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 NUMA policy and native-granule fallback checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 NUMA FAIL stage=00\n"; + + if (code) { + text[sizeof(text) - 4] = '0' + code / 10; + text[sizeof(text) - 3] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 NUMA PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long native, unsigned long page) +{ + unsigned long length = 4 * native; + long raw = call(192, 0, length, 3, 0x22, -1, 0); + unsigned char *base; + u32 mask[2] = { 1, 0 }, got[2] = { 0, 0 }; + int mode = -1, node; + + if ((unsigned long)raw >= 0xfffff001UL) + finish(20); + base = (void *)(((unsigned long)raw + native - 1) & ~(native - 1)); + if (call(320, (long)&mode, (long)got, 64, 0, 4, 0) || got[0] != 3 || + got[1]) + finish(21); + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 0)) + finish(22); + mask[0] = 2; + if (call(319, (long)(base + page), 1, 2, (long)mask, 64, 0)) + finish(23); + if (call(450, (long)(base + page), 1, 1, 0, 0, 0)) + finish(24); + if (call(320, (long)&mode, (long)got, 64, (long)(base + page), 2, 0) || + mode != 2 || got[0] != 2 || got[1]) + finish(25); + for (unsigned long off = 0; off < 2 * native; off += page) { + base[off + 17] = 0x51 + off / page; + node = -1; + if (call(320, (long)&node, 0, 0, (long)(base + off + 17), 3, + 0) || + node != (off == page ? 1 : 0)) + finish(26); + } + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 1) != -5) + finish(27); + if (call(319, (long)(base + 4096), 4096, 2, (long)mask, 64, 0) != + (page == 4096 ? 0 : -22)) + finish(28); + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 5)) + finish(29); + for (unsigned long off = 0; off < 2 * native; off += page) { + node = -1; + if (base[off + 17] != (unsigned char)(0x51 + off / page) || + call(320, (long)&node, 0, 0, (long)(base + off + 17), 3, + 0) || + node != 1) + finish(30); + } + if (call(91, raw, length, 0, 0, 0, 0)) + finish(31); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + mapping(number(argv[1]), pages); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-perf-probe.c b/tools/testing/selftests/arm64/user4k/compat-perf-probe.c new file mode 100644 index 0000000000000..ee02fc34051ad --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-perf-probe.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 perf mmap layout, including native-page fallback. */ +typedef unsigned int u32; +typedef unsigned long long u64; + +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 PERF FAIL stage=00\n"; + + if (code) { + text[20] = '0' + code / 10; + text[21] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 PERF PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long pages, unsigned long native, + unsigned int count) +{ + static const struct { + u32 type, size; + u64 config, sample_period, sample_type, read_format, flags; + u32 wakeup, bp_type; + u64 config1; + } attr = { .type = 1, .size = 64, .flags = 1 }; + long fd = call(364, (long)&attr, 0, -1, -1, 0, 0); + unsigned long length = (1 + count) * pages; + long raw; + volatile u64 *metadata; + + if (fd < 0) + finish(20); + raw = call(192, 0, length, 3, 1, fd, 0); + if ((unsigned long)raw >= 0xfffff001UL) + finish(21); + metadata = (void *)(raw + 1024); + if (metadata[2] != pages || metadata[3] != count * pages) + finish(22); + metadata[1] = 0; + metadata[6] = length; + metadata[7] = native; + /* mmap2 always takes 4K units, including the no4K compat fallback. */ + if (call(192, 0, native, 1, 1, fd, length / 4096) != -95) + finish(23); + if (call(91, raw, length, 0, 0, 0, 0)) + finish(24); + if (call(6, fd, 0, 0, 0, 0, 0)) + finish(25); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + unsigned long native = number(argv[1]); + + mapping(pages, native, 0); + mapping(pages, native, 1); + mapping(pages, native, 2); + mapping(pages, native, 2 * native / pages); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-policy-probe.c b/tools/testing/selftests/arm64/user4k/compat-policy-probe.c new file mode 100644 index 0000000000000..dab5a79abe130 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-policy-probe.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 -> AArch64 exec with an inherited granule preference. */ +#include "../../../../../include/uapi/linux/prctl.h" +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static unsigned int number(const char *s) +{ + unsigned int n = 0; + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} +static void fail(int code) +{ + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + sp += argc + 1; + char **env = (void *)sp; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 4 || pages != number(argv[3])) + fail(10); + if (call(172, PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0, 0, 0, 0, 0) != + number(argv[2])) + fail(11); + if (call(172, PR_AURORA_GET_EXEC_PAGE_SIZE, 0, 0, 0, 0, 0)) + fail(12); + long mem = call(192, 0, 4 * pages, 3, 0x22, -1, 0); + if ((unsigned long)mem >= 0xfffff001UL) + fail(13); + long ret = call(125, mem + 4096, 4096, 1, 0, 0, 0); + if (ret != (pages == 4096 ? 0 : -22)) + fail(14); + if (call(91, mem, 4 * pages, 0, 0, 0, 0)) + fail(15); + char *args[] = { "/init", "--compat-return", argv[1], argv[2], 0 }; + call(11, (long)args[0], (long)args, (long)env, 0, 0, 0); + fail(16); + for (;;) + ; +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-probe.c b/tools/testing/selftests/arm64/user4k/compat-probe.c new file mode 100644 index 0000000000000..14a4deb716412 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-probe.c @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding AArch32 process; run only by compat-fixture-init in a guest. */ +typedef unsigned int u32; +typedef unsigned short u16; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f, r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static unsigned length(const char *s) +{ + unsigned n = 0; + while (s[n]) + n++; + return n; +} +static int equal(const char *a, const char *b) +{ + while (*a && *a == *b) { + a++; + b++; + } + return *a == *b; +} +static unsigned number(const char *s) +{ + unsigned n = 0; + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} +static void out(const char *s) +{ + call(4, 1, (long)s, length(s), 0, 0, 0); +} +static int failures, checks; +static void result(int pass, const char *s) +{ + checks++; + if (!pass) + failures++; + out(pass ? "ok - compat " : "not ok - compat "); + out(s); + out("\n"); +} +static int error(long v) +{ + return (unsigned long)v >= 0xfffff001UL; +} +static long map(unsigned size, int prot, int flags, int fd, unsigned off) +{ + return call(192, 0, size, prot, flags, fd, off); +} +static void unmap(long p, unsigned size) +{ + if (!error(p)) + call(91, p, size, 0, 0, 0, 0); +} +static volatile unsigned signals; +static void handler(int n) +{ + if (n == 10) + signals++; +} +struct action { + u32 handler, flags, restorer, mask[2]; +}; +struct ehdr { + unsigned char ident[16]; + u16 type, machine; + u32 version, entry, phoff, shoff, flags; + u16 ehsize, phentsize, phnum, shentsize, shnum, shstrndx; +}; +struct phdr { + u32 type, offset, vaddr, paddr, filesz, memsz, flags, align; +}; +struct dyn { + u32 tag, val; +}; +struct sym { + u32 name, value, size; + unsigned char info, other; + u16 shndx; +}; +static void *vdso_symbol(unsigned base, const char *name) +{ + struct ehdr *h = (void *)base; + struct phdr *p = (void *)(base + h->phoff); + struct dyn *d = 0; + char *strings = 0; + struct sym *symbols = 0; + u32 *hash = 0; + for (unsigned i = 0; i < h->phnum; i++) + if (p[i].type == 2) + d = (void *)(base + p[i].vaddr); + if (!d) + return 0; + for (; d->tag; d++) { + if (d->tag == 5) + strings = (void *)(base + d->val); + if (d->tag == 6) + symbols = (void *)(base + d->val); + if (d->tag == 4) + hash = (void *)(base + d->val); + } + if (!strings || !symbols || !hash) + return 0; + for (unsigned i = 0; i < hash[1]; i++) + if (equal(strings + symbols[i].name, name)) + return (void *)(base + symbols[i].value); + return 0; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + unsigned argc = *sp++, pagesz = 0, vdso = 0; + char **argv = (void *)sp; + sp += argc + 1; + char **env = (void *)sp; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pagesz = sp[1]; + if (sp[0] == 33) + vdso = sp[1]; + sp += 2; + } + if (argc > 1 && equal(argv[1], "--exec32")) { + char *a[] = { argv[0], argv[2], argv[3], argv[4], 0 }; + call(11, (long)a[0], (long)a, (long)env, 0, 0, 0); + call(1, 97, 0, 0, 0, 0, 0); + for (;;) + ; + } + if (argc > 1 && + (equal(argv[1], "--exec64") || equal(argv[1], "--exec64-4k"))) { + int alternative = equal(argv[1], "--exec64-4k"); + if (alternative && call(172, 0x41555001, 4096, 0, 0, 0, 0)) { + call(1, 99, 0, 0, 0, 0, 0); + for (;;) + ; + } + char *a[] = { "/init", + alternative ? "--returned4k" : "--returned", 0 }; + call(11, (long)a[0], (long)a, (long)env, 0, 0, 0); + call(1, 98, 0, 0, 0, 0, 0); + for (;;) + ; + } + /* Script interpreter contributes its pathname before the ordinary arguments. */ + unsigned start = argc > 1 && equal(argv[1], "/script32") ? 2 : 1; + unsigned expected = argc > start ? number(argv[start]) : 0; + int fd = argc > start + 1 ? (int)number(argv[start + 1]) : -1; + result(pagesz == expected, "AT_PAGESZ reflects final ELF32 selection"); + int args = argc > start + 2 && length(argv[start + 2]) == 90000; + if (args) + for (unsigned i = 0; i < 90000; i++) + if (argv[start + 2][i] != (char)('a' + i % 23)) { + args = 0; + break; + } + result(args, "90KB staged argument survives granule transfer"); + long mem = map(4 * pagesz, 3, 0x22, -1, 0); + int ok = !error(mem); + if (ok) { + unsigned char *p = (void *)mem; + for (unsigned i = 0; i < 4 * pagesz; i++) + p[i] = 0x30 + i / pagesz; + for (unsigned i = 0; i < 4 * pagesz; i++) + if (p[i] != (unsigned char)(0x30 + i / pagesz)) + ok = 0; + } + result(ok, "anonymous user leaves have independent bytes"); + result(!error(mem) && !call(125, mem + pagesz, pagesz, 1, 0, 0, 0) && + !call(125, mem + pagesz, pagesz, 3, 0, 0, 0), + "mprotect at selected page boundaries"); + result(!error(mem) && call(125, mem + 1, pagesz, 1, 0, 0, 0) == -22 && + call(91, mem + 1, pagesz, 0, 0, 0, 0) == -22, + "unaligned memory operations retain restrictions"); + long q = map(pagesz, 1, 1, fd, 3 * (pagesz >> 12)); + ok = !error(q); + if (ok) + for (unsigned i = 0; i < pagesz; i++) + if (((unsigned char *)q)[i] != + (unsigned char)(0x40 + (3 * pagesz + i) / 4096)) + ok = 0; + result(ok, "mmap2 native-aligned file offset"); + unmap(q, pagesz); + q = map(4096, 1, 1, fd, 1); + ok = pagesz == 4096 ? !error(q) && *(unsigned char *)q == 0x41 : + q == -22; + result(ok, "mmap2 4KB offset accepted only by matching ABI"); + unmap(q, pagesz); + q = map(pagesz, 1, 1, fd, 0x100000 + (pagesz >> 12)); + ok = !error(q); + if (ok) + for (unsigned i = 0; i < pagesz; i++) + if (((unsigned char *)q)[i] != + (unsigned char)(0x80 + (pagesz + i) / 4096)) + ok = 0; + result(ok, "mmap2 offset above 4GB remains full width"); + unmap(q, pagesz); + long child = call(2, 0, 0, 0, 0, 0, 0); + int status = 0; + if (!child) { + if (!error(mem)) + *(unsigned char *)mem = 0xa7; + call(1, 0, 0, 0, 0, 0, 0); + for (;;) + ; + } + ok = child > 0 && + call(114, child, (long)&status, 0, 0, 0, 0) == child && !status && + !error(mem) && *(unsigned char *)mem == 0x30; + result(ok, "fork isolates private COW data"); + unmap(mem, 4 * pagesz); + struct action act = { (u32)handler, 0, 0, { 0, 0 } }; + ok = !call(174, 10, (long)&act, 0, 8, 0, 0); + if (ok) + ok = !call(37, call(20, 0, 0, 0, 0, 0, 0), 10, 0, 0, 0, 0) && + signals == 1; + result(ok, "AArch32 signal frame and kernel sigreturn trampoline"); + volatile unsigned word = 17; + int (*cmpxchg)(unsigned, unsigned, volatile unsigned *) = + (void *)0xffff0fc0; + result(*(volatile unsigned *)0xffff0ffc >= 5 && + !cmpxchg(17, 29, &word) && word == 29, + "kuser helper page and atomic compare exchange"); + int (*gettime)(int, void *) = + vdso ? (void *)vdso_symbol(vdso, "__vdso_clock_gettime") : 0; + long time[2] = { 0, 0 }; + result(gettime && !gettime(1, time) && time[0] >= 0 && time[1] >= 0 && + time[1] < 1000000000, + "compat vDSO clock_gettime and vvar mapping"); + out(failures ? "COMPAT PROBE FAIL\n" : "COMPAT PROBE PASS\n"); + call(1, failures ? 1 : 0, 0, 0, 0, 0, 0); + for (;;) + ; +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-script b/tools/testing/selftests/arm64/user4k/compat-script new file mode 100644 index 0000000000000..9b56d0c7567f5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-script @@ -0,0 +1 @@ +#!/probe32-64k diff --git a/tools/testing/selftests/arm64/user4k/compat-shm-probe.c b/tools/testing/selftests/arm64/user4k/compat-shm-probe.c new file mode 100644 index 0000000000000..c4ee1da2c5a12 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-shm-probe.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 SysV attach/detach and fallback-alignment checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 SHM FAIL stage=00\n"; + + if (code) { + text[19] = '0' + code / 10; + text[20] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 SHM PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long pages, int rounded) +{ + unsigned long align = 4 * pages, length = 4 * pages; + long raw = call(192, 0, 12 * align, 3, 0x22, -1, 0); + unsigned char *target; + long id, result; + + if ((unsigned long)raw >= 0xfffff001UL) + finish(20); + result = call(125, raw + 4096, 4096, 1, 0, 0, 0); + if (result != (pages == 4096 ? 0 : -22)) + finish(21); + if (call(125, raw, 12 * align, 3, 0, 0, 0)) + finish(22); + target = (void *)(((unsigned long)raw + align - 1) & ~(align - 1)); + target += align + (rounded ? 0 : pages); + target[-1] = 0x51; + target[length] = 0x72; + if (call(91, (long)target, length, 0, 0, 0, 0)) + finish(23); + id = call(307, 0, length - 1, 0600, 0, 0, 0); + if (id < 0) + finish(24); + result = call(305, id, (long)(target + (rounded ? pages + 17 : 0)), + rounded ? 020000 : 0, 0, 0, 0); + if (result != (long)target) { + call(308, id, 0, 0, 0, 0, 0); + finish(rounded ? 26 : 25); + } + if (call(308, id, 0, 0, 0, 0, 0)) + finish(27); + target[pages + 13] = 0x67; + if (call(125, (long)(target + pages), pages, 1, 0, 0, 0)) + finish(28); + if (call(91, (long)target, pages, 0, 0, 0, 0)) + finish(29); + if (call(306, (long)target, 0, 0, 0, 0, 0)) + finish(30); + if (target[-1] != 0x51 || target[length] != 0x72) + finish(31); + if (call(91, raw, 12 * align, 0, 0, 0, 0)) + finish(32); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + mapping(pages, 0); + mapping(pages, 1); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-uffd-init.c b/tools/testing/selftests/arm64/user4k/compat-uffd-init.c new file mode 100644 index 0000000000000..5f1e90f3e484f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-uffd-init.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - compat uffd line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct wire { + uint32_t address, pagesize; +}; +static void expect_invalid(int fd, uint64_t dst, uint64_t src, uint64_t len) +{ + struct uffdio_copy c = { .dst = dst, .src = src, .len = len }; + errno = 0; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && errno == EINVAL); +} +static void run(unsigned long native, unsigned long expected, int shared) +{ + int sock[2], file = -1, status; + CHECK(!socketpair(AF_UNIX, SOCK_SEQPACKET, 0, sock)); + if (shared) { + file = memfd_create("compat-uffd", 0); + CHECK(file >= 0 && !ftruncate(file, 3 * expected)); + } + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char page_arg[24], sock_arg[24], file_arg[24]; + close(sock[0]); + snprintf(page_arg, sizeof(page_arg), "%lu", expected); + snprintf(sock_arg, sizeof(sock_arg), "%d", sock[1]); + snprintf(file_arg, sizeof(file_arg), "%d", file); + execl("/compat-uffd", "/compat-uffd", page_arg, sock_arg, + file_arg, NULL); + _exit(99); + } + close(sock[1]); + struct wire data = { 0 }; + struct iovec iov = { &data, sizeof(data) }; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct msghdr msg = { .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control.bytes) }; + CHECK(recvmsg(sock[0], &msg, 0) == sizeof(data)); + CHECK(!(msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) && + data.pagesize == expected); + struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg); + CHECK(cmsg && cmsg->cmsg_level == SOL_SOCKET && + cmsg->cmsg_type == SCM_RIGHTS && + cmsg->cmsg_len == CMSG_LEN(sizeof(int))); + int uffd; + memcpy(&uffd, CMSG_DATA(cmsg), sizeof(uffd)); + /* Force the source above every possible AArch32 task limit. */ + unsigned char *source = + mmap((void *)0x1000000000UL, 2 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(source != MAP_FAILED && (uintptr_t)source > UINT32_MAX); + /* Source alignment is independent of both processes' granules. */ + unsigned char *bytes = source + 17; + for (unsigned leaf = 0; leaf < 3; leaf++) { + struct pollfd pfd = { .fd = uffd, .events = POLLIN }; + struct uffd_msg event; + CHECK(poll(&pfd, 1, 10000) == 1 && (pfd.revents & POLLIN)); + CHECK(read(uffd, &event, sizeof(event)) == sizeof(event)); + uint64_t dst = (uint64_t)data.address + leaf * expected; + CHECK(event.event == UFFD_EVENT_PAGEFAULT && + !event.arg.pagefault.flags && + event.arg.pagefault.address == dst); + if (!leaf) { + expect_invalid(uffd, dst, (uintptr_t)bytes, 0); + expect_invalid(uffd, dst, (uintptr_t)bytes, + expected - 1); + expect_invalid(uffd, dst + 1, (uintptr_t)bytes, + expected); + expect_invalid(uffd, 0x100000000ULL, (uintptr_t)bytes, + expected); + expect_invalid(uffd, dst, UINT64_MAX - expected + 1, + expected); + if (expected > 4096) + expect_invalid(uffd, dst, (uintptr_t)bytes, + 4096); + } + if (leaf == 1) { + /* A valid but inaccessible pager source must leave the target missing. */ + CHECK(!mprotect(source, 2 * native, PROT_NONE)); + struct uffdio_copy bad = { .dst = dst, + .src = (uintptr_t)bytes, + .len = expected }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &bad) == -1 && + errno == EFAULT && bad.copy == -EFAULT); + CHECK(!mprotect(source, 2 * native, + PROT_READ | PROT_WRITE)); + } + memset(bytes, 0x51 + leaf, expected); + struct uffdio_copy c = { .dst = dst, + .src = (uintptr_t)bytes, + .len = expected }; + CHECK(!ioctl(uffd, UFFDIO_COPY, &c) && + c.copy == (long long)expected); + } + CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!munmap(source, 2 * native)); + CHECK(!close(uffd) && !close(sock[0])); + if (shared) + CHECK(!close(file)); + printf("ok - native %lu pager high unaligned COPY to AArch32 %lu %s, source failure and target alignment\n", + native, expected, shared ? "shared" : "anonymous"); +} +int main(void) +{ + if (getpid() != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + alarm(90); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + char cmdline[2048] = { 0 }; + int fd = open("/proc/cmdline", O_RDONLY); + CHECK(fd >= 0 && read(fd, cmdline, sizeof(cmdline) - 1) > 0 && + !close(fd)); + unsigned long native = getauxval(AT_PAGESZ); + unsigned long expected = + strstr(cmdline, "id_aa64mmfr0.tgran4=f") ? native : 4096; + run(native, expected, 0); + run(native, expected, 1); + puts("COMPAT UFFD PASS"); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c b/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c new file mode 100644 index 0000000000000..6abcdb86bb4d8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding ELF32 half of compat-uffd-init; no armhf sysroot required. */ +typedef unsigned int u32; +typedef unsigned long long u64; +struct api { + u64 api, features, ioctls; +}; +struct registration { + u64 start, len, mode, ioctls; +}; +struct movement { + u64 dst, src, len, mode; + long long moved; +}; +struct wire { + u32 address, pagesize; +}; +struct iovec32 { + void *base; + u32 len; +}; +struct message32 { + void *name; + u32 namelen; + struct iovec32 *iov; + u32 iovlen; + void *control; + u32 controllen, flags; +}; +struct rights32 { + u32 len, level, type; + int fd; +}; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static u32 number(const char *s) +{ + u32 n = 0; + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} +__attribute__((noreturn)) static void finish(int code) +{ + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 4 || pages != number(argv[1])) + finish(10); + int sock = number(argv[2]), shared = argv[3][0] != '-'; + int file = shared ? (int)number(argv[3]) : -1; + long map = call(192, 0, 3 * pages, 3, shared ? 1 : 0x22, file, 0); + if ((unsigned long)map >= 0xfffff001UL) + finish(11); + /* Fine operations must retain native alignment restrictions on fallback. */ + if (call(125, map + 4096, 4096, 1, 0, 0, 0) != + (pages == 4096 ? 0 : -22)) + finish(12); + if (call(125, map, 3 * pages, 3, 0, 0, 0)) + finish(13); + long fd = call(388, 0x80801, 0, 0, 0, 0, + 0); /* CLOEXEC|NONBLOCK|USER_MODE_ONLY */ + if (fd < 0) + finish(14); + /* Negotiate shared missing support under both fine and fallback ABIs. */ + struct api api = { 0xaa, (1ULL << 5) | (1ULL << 16), 0 }; + if (call(54, fd, 0xc018aa3fU, (long)&api, 0, 0, 0) || + !(api.features & (1ULL << 5))) + finish(15); + struct registration reg = { (u32)map, 3 * pages, 1, 0 }; + if (call(54, fd, 0xc020aa00U, (long)®, 0, 0, 0)) + finish(16); + if (!(reg.ioctls & (1ULL << 3)) || !(api.features & (1ULL << 16))) + finish(17); /* COPY and MOVE */ + struct wire data = { (u32)map, pages }; + struct iovec32 iov = { &data, sizeof(data) }; + struct rights32 rights = { sizeof(rights), 1, 1, fd }; + struct message32 msg = { 0, 0, &iov, 1, &rights, sizeof(rights), 0 }; + if (call(296, sock, (long)&msg, 0, 0, 0, 0) != sizeof(data)) + finish(18); + volatile unsigned char *p = (void *)map; + /* Each first access waits for the native pager's COPY. */ + for (u32 leaf = 0; leaf < 3; leaf++) + for (u32 i = 0; i < pages; i++) + if (p[leaf * pages + i] != (unsigned char)(0x51 + leaf)) + finish(19); + if (!shared) { + long target = call(192, 0, 3 * pages, 3, 0x22, -1, 0); + if ((unsigned long)target >= 0xfffff001UL) + finish(21); + struct registration dst = { (u32)target, 3 * pages, 1, 0 }; + if (call(54, fd, 0xc020aa00U, (long)&dst, 0, 0, 0) || + !(dst.ioctls & (1ULL << 5))) + finish(22); + struct movement m = { 1ULL << 32, (u32)map + 2 * pages, pages, + 0, 0 }; + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) != -22) + finish(23); + m.dst = (u32)target + pages; + if (pages > 4096) { + m.len = 4096; + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) != -22) + finish(24); + m.len = pages; + } + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) || + m.moved != pages) + finish(25); + volatile unsigned char *q = (void *)target; + for (u32 i = 0; i < pages; i++) + if (q[pages + i] != 0x53 || p[i] != 0x51 || + p[pages + i] != 0x52) + finish(26); + if (call(91, target, 3 * pages, 0, 0, 0, 0)) + finish(27); + static const char ok[] = + "ok - AArch32 MOVE, neighbours and native-fallback alignment\n"; + call(4, 1, (long)ok, sizeof(ok) - 1, 0, 0, 0); + } + if (call(91, map, 3 * pages, 0, 0, 0, 0)) + finish(20); + finish(0); +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c b/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c new file mode 100644 index 0000000000000..231e4b7996613 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 type1 probe; no armhf sysroot or physical DMA. */ +#define __user +#include "../../../../../include/uapi/linux/vfio.h" +#include "../../../../../drivers/vfio/vfio_iommu_type1_test.h" + +typedef unsigned int u32; +typedef unsigned long long u64; +#define IOVA (256UL * 1024 * 1024) +static unsigned int stage; + +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static u32 number(const char *s) +{ + u32 n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static void clear(void *data, u32 size) +{ + volatile unsigned char *p = data; + + for (u32 i = 0; i < size; i++) + p[i] = 0; +} + +/* Aggregate initialization may use these compiler-runtime entry points. */ +void __aeabi_memclr8(void *data, u32 size) +{ + clear(data, size); +} +void __aeabi_memclr4(void *data, u32 size) +{ + clear(data, size); +} +void __aeabi_memclr(void *data, u32 size) +{ + clear(data, size); +} + +_Noreturn static void finish(int code) +{ + if (code) { + char msg[] = "not ok - AArch32 type1 stage=00\n"; + + msg[29] = '0' + stage / 10; + msg[30] = '0' + stage % 10; + call(4, 1, (long)msg, sizeof(msg) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} +#define CHECK(x) \ + do { \ + stage++; \ + if (!(x)) \ + finish(stage); \ + } while (0) + +static long ioctl_test(int fd, u32 request, void *arg) +{ + return call(54, fd, request, (long)arg, 0, 0, 0); +} + +static int state(int fd, struct vfio_type1_test_cmd *cmd) +{ + clear(cmd, sizeof(*cmd)); + return ioctl_test(fd, VFIO_TYPE1_TEST_STATE, cmd); +} + +static long map(int fd, u32 address, u32 size) +{ + struct vfio_iommu_type1_dma_map cmd; + + clear(&cmd, sizeof(cmd)); + cmd.argsz = sizeof(cmd); + cmd.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + cmd.iova = IOVA; + cmd.vaddr = address; + cmd.size = size; + return ioctl_test(fd, VFIO_IOMMU_MAP_DMA, &cmd); +} + +static long unmap(int fd, u32 size) +{ + struct vfio_iommu_type1_dma_unmap cmd; + long ret; + + clear(&cmd, sizeof(cmd)); + cmd.argsz = sizeof(cmd); + cmd.iova = IOVA; + cmd.size = size; + ret = ioctl_test(fd, VFIO_IOMMU_UNMAP_DMA, &cmd); + return ret ? ret : cmd.size == size ? 0 : -1; +} + +static long rw(int fd, unsigned char *bytes, unsigned int flags) +{ + struct vfio_type1_test_cmd cmd; + + clear(&cmd, sizeof(cmd)); + cmd.iova = IOVA; + cmd.buffer = (u32)bytes; + cmd.size = 16; + cmd.flags = flags; + return ioctl_test(fd, VFIO_TYPE1_TEST_RW, &cmd); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_dirty_bitmap dirty; + struct vfio_iommu_type1_info info; + unsigned char bytes[16]; + long fd, file, data, pid; + int status = -1; + u32 native, length; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + CHECK(argc == 3 && pages == number(argv[2])); + native = number(argv[1]); + length = native + 4096; + fd = call(5, (long)"/dev/vfio-test", 2, 0, 0, 0, 0); + file = call(385, (long)"compat-vfio", 0, 0, 0, 0, 0); + CHECK(fd >= 0 && file >= 0 && !call(93, file, 2 * native, 0, 0, 0, 0)); + data = call(192, 0, 2 * native, 3, 1, file, 0); + CHECK((u32)data < 0xfffff001U); + CHECK(call(125, data + 4096, 4096, 1, 0, 0, 0) == + (pages == 4096 ? 0 : -22)); + CHECK(!call(125, data, 2 * native, 3, 0, 0, 0)); + for (u32 i = 0; i < 2 * native; i++) + ((volatile unsigned char *)data)[i] = 0x53; + clear(&info, sizeof(info)); + info.argsz = sizeof(info); + CHECK(!ioctl_test(fd, VFIO_IOMMU_GET_INFO, &info) && + (info.iova_pgsizes & 4096)); + CHECK(!map(fd, data, 2 * native)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes); + CHECK(!rw(fd, bytes, 0) && bytes[0] == 0x53 && bytes[15] == 0x53); + clear(&dirty, sizeof(dirty)); + dirty.argsz = sizeof(dirty); + dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_START; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, &dirty)); + clear(&cmd, sizeof(cmd)); + cmd.iova = IOVA + 19; + CHECK(!ioctl_test(fd, VFIO_TYPE1_TEST_PIN, &cmd)); + CHECK(!state(fd, &cmd) && cmd.locked_bytes == native && + cmd.external_refs == 1); + /* Compat pointer padding must not become part of the native pointer. */ + _Alignas(struct vfio_iommu_type1_dirty_bitmap_get) unsigned char + buffer[sizeof(dirty) + + sizeof(struct vfio_iommu_type1_dirty_bitmap_get)]; + struct vfio_iommu_type1_dirty_bitmap *header = (void *)buffer; + struct vfio_iommu_type1_dirty_bitmap_get *range = (void *)header->data; + u64 bits = 0; + + clear(buffer, sizeof(buffer)); + header->argsz = sizeof(buffer); + header->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP; + range->iova = IOVA; + range->size = 2 * native; + range->bitmap.pgsize = 4096; + range->bitmap.size = sizeof(bits); + range->bitmap.data = &bits; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, buffer) && + bits == ((1ULL << (native >> 12)) - 1)); + _Static_assert( + sizeof(range->bitmap.data) == 4 && sizeof(range->bitmap) == 24, + "AArch32 bitmap has a 32-bit pointer and trailing padding"); + ((unsigned char *)&range + ->bitmap)[__builtin_offsetof(struct vfio_bitmap, data) + + sizeof(range->bitmap.data)] = 0xa5; + bits = 0; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, buffer) && + bits == ((1ULL << (native >> 12)) - 1)); + _Alignas(struct vfio_bitmap) unsigned char + unmap_buffer[sizeof(struct vfio_iommu_type1_dma_unmap) + + sizeof(struct vfio_bitmap)]; + struct vfio_iommu_type1_dma_unmap *remove = (void *)unmap_buffer; + struct vfio_bitmap *remove_bitmap = (void *)remove->data; + + clear(unmap_buffer, sizeof(unmap_buffer)); + remove->argsz = sizeof(unmap_buffer); + remove->flags = VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP; + remove->iova = IOVA; + remove->size = 2 * native; + remove_bitmap->pgsize = 4096; + remove_bitmap->size = sizeof(bits); + remove_bitmap->data = &bits; + ((unsigned char *) + remove_bitmap)[__builtin_offsetof(struct vfio_bitmap, data) + + sizeof(remove_bitmap->data)] = 0x5a; + bits = 0; + CHECK(!ioctl_test(fd, VFIO_IOMMU_UNMAP_DMA, unmap_buffer) && + remove->size == 2 * native && + bits == ((1ULL << (native >> 12)) - 1)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.external_refs && + cmd.notifications == 1); + clear(&cmd, sizeof(cmd)); + cmd.size = 4096; + CHECK(!ioctl_test(fd, VFIO_TYPE1_TEST_DOMAIN_ADD, &cmd)); + pid = call(2, 0, 0, 0, 0, 0, 0); + if (!pid) + finish(map(fd, data + 4096, length) ? 99 : 0); + CHECK(pid > 0 && call(114, pid, (long)&status, 0, 0, 0, 0) == pid && + !status); + CHECK(!state(fd, &cmd) && !cmd.owner_users && + cmd.locked_bytes == 2 * native && cmd.mapped_bytes[0] == length); + CHECK(rw(fd, bytes, 0) == -14); + CHECK(!rw(fd, bytes, VFIO_TYPE1_TEST_WORKER) && bytes[0] == 0x53 && + bytes[15] == 0x53); + CHECK(!unmap(fd, length)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0]); + CHECK(!call(91, data, 2 * native, 0, 0, 0, 0)); + CHECK(!call(6, file, 0, 0, 0, 0, 0) && !call(6, fd, 0, 0, 0, 0, 0)); + static const char ok[] = + "ok - AArch32 type1 real ioctls, source exit and fallback alignment\n"; + + call(4, 1, (long)ok, sizeof(ok) - 1, 0, 0, 0); + finish(0); +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/context-stress-init.c b/tools/testing/selftests/arm64/user4k/context-stress-init.c new file mode 100644 index 0000000000000..869662502c15f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/context-stress-init.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* PID-1 mixed-granule EL0 stress runner for disposable VMs. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define DATA_ADDRESS 0x200000000UL +#define CODE_ADDRESS 0x202000000UL +#define CONTEXTS 70000 +#define SHORT_CONTEXTS 2048 + +static sigjmp_buf fault_return; +static volatile uintptr_t expected_fault; +static char generation_path[256]; + +static void fault(int sig, siginfo_t *info, void *context) +{ + (void)context; + if (sig != SIGSEGV || !expected_fault || + (uintptr_t)info->si_addr != expected_fault || + info->si_code != SEGV_ACCERR) + _exit(90); + expected_fault = 0; + siglongjmp(fault_return, 1); +} + +static bool rejects_write(uintptr_t address) +{ + expected_fault = address; + if (!sigsetjmp(fault_return, 1)) { + *(volatile unsigned char *)expected_fault = 1; + expected_fault = 0; + return false; + } + return true; +} + +static int migrate(unsigned int cpu) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(0, sizeof(set), &set); +} + +static unsigned char pattern(unsigned long serial, unsigned long offset) +{ + serial ^= (serial >> 8) ^ (serial >> 16); + return (serial * 37 + offset * 17 + (offset >> 8)) & 255; +} + +static bool validate(unsigned char *p, unsigned long serial, unsigned long len) +{ + for (unsigned long i = 0; i < len; i++) + if (p[i] != pattern(serial, i)) + return false; + return true; +} + +static int worker(unsigned long serial, unsigned long ps, int control) +{ + struct sigaction action = { .sa_sigaction = fault, + .sa_flags = SA_SIGINFO }; + unsigned char *data; + uint32_t *code; + unsigned int cpus = sysconf(_SC_NPROCESSORS_ONLN); + struct stat self, init; + unsigned long iterations = 0; + + if (getauxval(AT_PAGESZ) != ps || getppid() != 1 || + stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) != 0) + return 91; + if (cpus < 2 || migrate(serial % cpus) || + sigaction(SIGSEGV, &action, NULL)) + return 92; + data = mmap((void *)DATA_ADDRESS, 4 * ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + code = mmap((void *)CODE_ADDRESS, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (data == MAP_FAILED || code == MAP_FAILED || + mprotect(data, 3 * ps, PROT_READ | PROT_WRITE)) + return 93; + for (unsigned long i = 0; i < 3 * ps; i++) + data[i] = pattern(serial, i); + if (!rejects_write(DATA_ADDRESS + 3 * ps)) + return 94; + if (mprotect(data + ps, ps, PROT_READ)) + return 95; + if (!rejects_write(DATA_ADDRESS + ps)) + return 96; + do { + unsigned int expected = (serial + iterations * 13) & 65535; + int (*fn)(void) = (void *)code; + char stop; + ssize_t got; + + if (mprotect(code, ps, PROT_READ | PROT_WRITE)) + return 97; + code[0] = 0x52800000 | (expected << 5); /* movz w0, #expected */ + code[1] = 0xd65f03c0; /* ret */ + __builtin___clear_cache((char *)code, (char *)(code + 2)); + if (mprotect(code, ps, PROT_READ | PROT_EXEC) || + migrate((serial + iterations + 1) % cpus)) + return 98; + sched_yield(); + if (fn() != (int)expected || !validate(data, serial, 3 * ps)) + return 99; + if (!(iterations % 64) && + (!rejects_write(DATA_ADDRESS + 3 * ps) || + !rejects_write(DATA_ADDRESS + ps))) + return 101; + iterations++; + if (control < 0) + break; + got = read(control, &stop, 1); + if (got == 1) + break; + if (got != -1 || (errno != EAGAIN && errno != EINTR)) + return 100; + } while (true); + if (control >= 0) + printf("# persistent %luK worker: %lu migration/JIT/data cycles\n", + ps / 1024, iterations); + return 0; +} + +static pid_t spawn(unsigned long serial, unsigned long ps, int control, + int close_fd) +{ + pid_t pid = fork(); + + if (!pid) { + char seq[32], size[32], fd[32]; + + if (close_fd >= 0) + close(close_fd); + if (control >= 0 && fcntl(control, F_SETFD, 0)) + _exit(110); + snprintf(seq, sizeof(seq), "%lu", serial); + snprintf(size, sizeof(size), "%lu", ps); + snprintf(fd, sizeof(fd), "%d", control); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, ps, 0UL, 0UL, 0UL)) + _exit(111); + execl("/init", "/init", "--worker", seq, size, fd, NULL); + _exit(112); + } + return pid; +} + +static bool reap(pid_t pid) +{ + int status = 0; + pid_t got; + + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + if (got == pid && WIFEXITED(status) && !WEXITSTATUS(status)) + return true; + printf("not ok - worker pid=%d wait=%d status=%#x\n", pid, got, status); + return false; +} + +static unsigned long long generation(void) +{ + unsigned long long value = 0; + FILE *f = fopen(generation_path, "re"); + + if (!f || fscanf(f, "%llu", &value) != 1) { + perror("ASID generation"); + return 0; + } + fclose(f); + return value; +} + +int main(int argc, char **argv) +{ + unsigned long long before, after; + unsigned char *data; + pid_t persistent[3]; + int commands[3], pipes[2]; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + glob_t paths; + bool ok = true; + unsigned int contexts = CONTEXTS; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (argc == 2 && !strcmp(argv[1], "--short")) + contexts = SHORT_CONTEXTS; + if (getpid() != 1 || (argc != 1 && contexts != SHORT_CONTEXTS)) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + glob("/sys/module/*/parameters/user4k_asid_generation", 0, NULL, + &paths) || + paths.gl_pathc != 1) { + perror("fixture setup"); + return 2; + } + snprintf(generation_path, sizeof(generation_path), "%s", + paths.gl_pathv[0]); + globfree(&paths); + data = mmap((void *)DATA_ADDRESS, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (data == MAP_FAILED) + return 3; + for (unsigned long i = 0; i < 4 * native; i++) + data[i] = pattern(0, i); + before = generation(); + printf("# mixed context stress: %u fork/exec workers, initial ASID generation=%llu\n", + contexts, before); + for (unsigned int i = 0; i < nr; i++) { + if (pipe2(pipes, O_NONBLOCK | O_CLOEXEC)) + return 4; + persistent[i] = spawn(CONTEXTS + i + 1, granules[i], pipes[0], + pipes[1]); + if (persistent[i] < 0) + return 5; + close(pipes[0]); + commands[i] = pipes[1]; + } + for (unsigned long i = 1; i <= contexts; i++) { + pid_t pid = spawn(i, granules[i % nr], -1, -1); + + if (pid < 0 || !reap(pid) || !validate(data, 0, 4 * native)) { + printf("not ok - churn/parent isolation at worker %lu\n", + i); + ok = false; + break; + } + if (!(i % 128)) + printf("# workers=%lu ASID generation=%llu\n", i, + generation()); + } + after = generation(); + for (unsigned int i = 0; i < nr; i++) { + ok &= write(commands[i], "x", 1) == 1; + close(commands[i]); + ok &= reap(persistent[i]); + } + ok &= before && after >= before + 2 * (1ULL << 16); + printf("%s - mixed context stress: generation=%llu->%llu, measured rollovers=%llu\n", + ok ? "ok" : "not ok", before, after, (after - before) >> 16); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/core-fixture-init.c b/tools/testing/selftests/arm64/user4k/core-fixture-init.c new file mode 100644 index 0000000000000..4caadf98296a6 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/core-fixture-init.c @@ -0,0 +1,447 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Core-dump regression runner: PID 1 in a disposable VM only. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define ANON 0x100000000UL +#define FILEMAP 0x102000000UL +#define SPARSE 0x104000000UL +#define EXCLUDED 0x106000000UL +#define SWAPPED 0x108000000UL +#define SWAPPED_SIZE (1UL << 20) +#define SPARSE_SIZE (8UL << 20) + +static void die(const char *what) +{ + perror(what); + _exit(120); +} + +static void write_text(const char *path, const char *text) +{ + int fd = open(path, O_WRONLY); + size_t len = strlen(text); + + if (fd < 0 || write(fd, text, len) != (ssize_t)len) + die(path); + close(fd); +} + +static void *map_at(unsigned long address, size_t len, int prot, int fd, + off_t off) +{ + void *p = mmap((void *)address, len, prot, + MAP_PRIVATE | MAP_FIXED_NOREPLACE | + (fd < 0 ? MAP_ANONYMOUS : 0), + fd, off); + + if (p == MAP_FAILED) + die("mmap"); + return p; +} + +static void crash(bool headers, int bounded) +{ + unsigned long ps = getauxval(AT_PAGESZ); + struct rlimit limit = { 64UL << 20, 64UL << 20 }; + struct stat self, init; + unsigned char *p, *data; + int fd; + + if (getppid() != 1 || stat("/proc/self/exe", &self) || + stat("/proc/1/exe", &init) || self.st_dev != init.st_dev || + self.st_ino != init.st_ino) + _exit(121); + if (bounded) + limit.rlim_cur = limit.rlim_max = ps - (bounded < 0); + if (setrlimit(RLIMIT_CORE, &limit)) + die("rlimit"); + write_text("/proc/self/coredump_filter", headers ? "0x10" : "0x1f"); + p = map_at(ANON, 4 * ps, PROT_READ | PROT_WRITE, -1, 0); + for (unsigned int i = 0; i < 4; i++) + memset(p + i * ps, 0x31 + i, ps); + if (mprotect(p + ps, ps, PROT_NONE) || + mprotect(p + 2 * ps, ps, PROT_READ)) + die("mprotect"); + p = map_at(SPARSE, SPARSE_SIZE, PROT_READ | PROT_WRITE, -1, 0); + memset(p, 0x65, ps); + /* Also exercise an existing read-only zero PTE. */ + if (*(volatile unsigned char *)(p + ps) != 0) + _exit(122); + p = map_at(EXCLUDED, ps, PROT_READ | PROT_WRITE, -1, 0); + memset(p, 0xa7, ps); + if (madvise(p, ps, MADV_DONTDUMP)) + die("dontdump"); + fd = open("/core-input", O_CREAT | O_TRUNC | O_RDWR, 0700); + data = malloc(19 * ps); + if (fd < 0 || !data) + die("file setup"); + for (unsigned int i = 0; i < 19; i++) + memset(data + i * ps, 0x81 + i, ps); + memcpy(data, ELFMAG, SELFMAG); + if (write(fd, data, 19 * ps) != (ssize_t)(19 * ps)) + die("file data"); + /* Offset one user page: three mappings select non-native quarters. */ + map_at(FILEMAP, ps, PROT_READ, fd, headers ? 0 : ps); + map_at(FILEMAP + 2 * ps, 2 * ps, PROT_READ, fd, 2 * ps); + map_at(FILEMAP + 8 * ps, ps, PROT_READ, fd, 17 * ps); + close(fd); + fd = open("/core-library", O_CREAT | O_TRUNC | O_RDWR, 0600); + if (fd < 0 || write(fd, data, ps) != (ssize_t)ps) + die("non-executable ELF file"); + map_at(FILEMAP + 12 * ps, ps, PROT_READ, fd, 0); + close(fd); + free(data); + if (!headers && !bounded) { + unsigned int missing = 0; + uint64_t entry; + int pm = open("/proc/self/pagemap", O_RDONLY); + + p = map_at(SWAPPED, SWAPPED_SIZE, PROT_READ | PROT_WRITE, -1, + 0); + memset(p, 0xc4, SWAPPED_SIZE); + for (unsigned int tries = 0; tries < 200; tries++) { + if (madvise(p, SWAPPED_SIZE, MADV_COLD) || + madvise(p, SWAPPED_SIZE, MADV_PAGEOUT)) + die("pageout"); + missing = 0; + for (unsigned long i = 0; i < SWAPPED_SIZE / ps; i++) { + if (pread(pm, &entry, sizeof(entry), + (SWAPPED / ps + i) * 8) != 8) + die("pagemap"); + missing += !(entry & (1ULL << 62)); + } + if (!missing) + break; + usleep(10000); + } + close(pm); + if (missing) + _exit(124); + printf("# %luK dump precondition: all %lu leaves swapped\n", + ps / 1024, SWAPPED_SIZE / ps); + } + /* The file VMAs remain unfaulted, so dumping must retrieve file data. */ + raise(SIGABRT); + _exit(123); +} + +static Elf64_Phdr *find_load(unsigned char *data, Elf64_Ehdr *elf, + unsigned long addr) +{ + Elf64_Phdr *ph = (void *)(data + elf->e_phoff); + + for (unsigned int i = 0; i < elf->e_phnum; i++) + if (ph[i].p_type == PT_LOAD && addr >= ph[i].p_vaddr && + addr - ph[i].p_vaddr < ph[i].p_memsz) + return ph + i; + return NULL; +} + +static bool bytes_at(unsigned char *data, Elf64_Ehdr *elf, size_t len, + unsigned long addr, size_t count, unsigned char expected) +{ + Elf64_Phdr *p = find_load(data, elf, addr); + unsigned long off; + + if (!p || addr - p->p_vaddr + count > p->p_filesz) + return false; + off = p->p_offset + addr - p->p_vaddr; + if (off > len || count > len - off) + return false; + for (size_t i = 0; i < count; i++) + if (data[off + i] != expected) + return false; + return true; +} + +static void inspect(pid_t pid, unsigned long ps, bool headers) +{ + char path[64]; + struct stat st; + unsigned char *data; + Elf64_Ehdr *elf; + Elf64_Phdr *ph, *p; + bool geometry = true, aux = false; + unsigned int file = 0; + unsigned long vdso = 0; + int fd; + + snprintf(path, sizeof(path), "/core.%d", pid); + fd = open(path, O_RDONLY); + if (fd < 0 || fstat(fd, &st) || st.st_size < (off_t)sizeof(*elf)) + ksft_exit_fail_msg("missing/short core %s\n", path); + data = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (data == MAP_FAILED) + die("core read"); + elf = (void *)data; + if (memcmp(elf->e_ident, ELFMAG, SELFMAG) || elf->e_type != ET_CORE || + elf->e_phentsize != sizeof(*ph) || + elf->e_phoff > (uint64_t)st.st_size || + elf->e_phnum * sizeof(*ph) > (uint64_t)st.st_size - elf->e_phoff) + ksft_exit_fail_msg("invalid core ELF\n"); + ph = (void *)(data + elf->e_phoff); + for (unsigned int i = 0; i < elf->e_phnum; i++) { + if (ph[i].p_type == PT_LOAD) { + geometry &= ph[i].p_align == ps && + ph[i].p_filesz <= ph[i].p_memsz && + ph[i].p_offset % ps == ph[i].p_vaddr % ps; + } else if (ph[i].p_type == PT_NOTE) { + size_t at = ph[i].p_offset, end = at + ph[i].p_filesz; + + if (end > (size_t)st.st_size || end < at) + ksft_exit_fail_msg("invalid notes\n"); + while (at + sizeof(Elf64_Nhdr) <= end) { + Elf64_Nhdr *n = (void *)(data + at); + size_t desc = at + sizeof(*n) + + ((n->n_namesz + 3UL) & ~3UL); + uint64_t *words = (void *)(data + desc); + + if (desc > end || n->n_descsz > end - desc) + ksft_exit_fail_msg( + "invalid note length\n"); + if (n->n_type == NT_AUXV) { + for (size_t j = 0; + j + 1 < n->n_descsz / 8; j += 2) { + if (words[j] == AT_PAGESZ) + aux = words[j + 1] == + ps; + else if (words[j] == + AT_SYSINFO_EHDR) + vdso = words[j + 1]; + } + } + if (n->n_type == NT_FILE && n->n_descsz >= 16 && + words[0] <= (n->n_descsz / 8 - 2) / 3 && + words[1] == ps) { + for (size_t j = 0; j < words[0]; j++) { + unsigned long addr = + words[2 + 3 * j]; + unsigned long offset = + words[4 + 3 * j]; + + if (addr == FILEMAP && + offset == (headers ? 0UL : + 1UL)) + file |= 1; + if (addr == FILEMAP + 2 * ps && + offset == 2) + file |= 2; + if (addr == FILEMAP + 8 * ps && + offset == 17) + file |= 4; + } + } + at = desc + ((n->n_descsz + 3UL) & ~3UL); + } + } + } + ksft_test_result(geometry, "%luK core PT_LOAD granule/alignment%s\n", + ps / 1024, headers ? " headers-only" : ""); + ksft_test_result(aux && file == 7, + "%luK core AUXV and NT_FILE offsets%s\n", ps / 1024, + headers ? " headers-only" : ""); + if (headers) { + bool skipped; + + p = find_load(data, elf, FILEMAP + 2 * ps); + skipped = p && !p->p_filesz; + p = find_load(data, elf, FILEMAP + 8 * ps); + ksft_test_result( + skipped && p && !p->p_filesz, + "%luK header filter excludes nonzero quarter/native file offsets\n", + ps / 1024); + p = find_load(data, elf, FILEMAP + 12 * ps); + ksft_test_result( + p && p->p_filesz == ps && + p->p_offset + ps <= (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, SELFMAG), + "%luK header filter recognizes non-executable ELF file\n", + ps / 1024); + p = find_load(data, elf, FILEMAP); + ksft_test_result( + p && p->p_filesz == ps && + p->p_offset + ps <= (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, SELFMAG), + "%luK executable file header limited to one user page\n", + ps / 1024); + } else { + bool anon = true, files = true; + + p = find_load(data, elf, vdso); + ksft_test_result(p && p->p_filesz >= SELFMAG && + p->p_offset + p->p_filesz <= + (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, + SELFMAG), + "%luK core vDSO ELF bytes\n", ps / 1024); + ksft_test_result(bytes_at(data, elf, st.st_size, SWAPPED, + SWAPPED_SIZE, 0xc4), + "%luK core swapped anonymous bytes\n", + ps / 1024); + + for (unsigned int i = 0; i < 4; i++) + anon &= bytes_at(data, elf, st.st_size, ANON + i * ps, + ps, 0x31 + i); + for (unsigned int i = 1; i < 4; i++) + files &= bytes_at(data, elf, st.st_size, + FILEMAP + (i == 1 ? 0 : i * ps), ps, + 0x81 + i); + ksft_test_result( + anon, "%luK core anonymous and PROT_NONE/READ bytes\n", + ps / 1024); + files &= bytes_at(data, elf, st.st_size, FILEMAP + 8 * ps, ps, + 0x81 + 17); + ksft_test_result(files, "%luK core unfaulted file quarters\n", + ps / 1024); + ksft_test_result( + bytes_at(data, elf, st.st_size, SPARSE, ps, 0x65) && + bytes_at(data, elf, st.st_size, SPARSE + ps, + SPARSE_SIZE - ps, 0), + "%luK core sparse and zero-page bytes\n", ps / 1024); + ksft_test_result( + st.st_blocks * 512 < st.st_size / 2, + "%luK core keeps untouched anonymous range sparse\n", + ps / 1024); + p = find_load(data, elf, EXCLUDED); + ksft_test_result(p && !p->p_filesz, + "%luK core DONTDUMP excludes data\n", + ps / 1024); + } + munmap(data, st.st_size); + close(fd); + unlink(path); +} + +int main(int argc, char **argv) +{ + unsigned long native_size = getauxval(AT_PAGESZ); + unsigned long sizes[] = { native_size, 4096, 16384 }; + unsigned int modes = native_size == 65536 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && !strcmp(argv[1], "--crash")) + crash(false, 0); + if (argc == 2 && !strcmp(argv[1], "--headers")) + crash(true, 0); + if (argc == 2 && !strcmp(argv[1], "--limit")) + crash(false, 1); + if (argc == 2 && !strcmp(argv[1], "--below-limit")) + crash(false, -1); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Use only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) && errno != EBUSY) + die("proc"); + if (mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + swapon("/dev/vda", 0)) + die("VM swap setup"); + write_text("/proc/sys/kernel/core_pattern", "/core.%p"); + ksft_print_header(); + ksft_set_plan(16 * modes); + for (unsigned int mode = 0; mode < modes; mode++) { + unsigned long ps = sizes[mode]; + for (unsigned int headers = 0; headers < 4; headers++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + ps == native_size ? 0UL : ps, 0UL, + 0UL, 0UL)) + die("exec size"); + execl("/init", "/init", + headers == 3 ? "--below-limit" : + headers == 2 ? "--limit" : + headers ? "--headers" : + "--crash", + NULL); + die("exec"); + } + if (pid < 0 || waitpid(pid, &status, 0) != pid || + !WIFSIGNALED(status) || + WTERMSIG(status) != SIGABRT || + (headers < 3 && !WCOREDUMP(status))) + ksft_exit_fail_msg( + "child/core failure status=%#x\n", + status); + if (headers >= 2) { + char path[64]; + struct stat st; + bool exists, no_data = false; + + snprintf(path, sizeof(path), "/core.%d", pid); + exists = !stat(path, &st); + if (exists && headers == 2) { + Elf64_Ehdr elf; + Elf64_Phdr ph; + uint64_t data_start = UINT64_MAX; + int fd = open(path, O_RDONLY); + + if (fd < 0 || + pread(fd, &elf, sizeof(elf), 0) != + sizeof(elf)) + die("limited core header"); + for (unsigned int i = 0; + i < elf.e_phnum; i++) { + if (pread(fd, &ph, sizeof(ph), + elf.e_phoff + + i * sizeof(ph)) != + sizeof(ph)) + die("limited core phdr"); + if (ph.p_type == PT_LOAD && + ph.p_offset < data_start) + data_start = + ph.p_offset; + } + close(fd); + /* Sparse note padding is not charged to RLIMIT_CORE. */ + no_data = data_start != UINT64_MAX && + st.st_size <= + (off_t)data_start; + printf("# limited core: bytes=%lld first-data=%llu limit=%lu\n", + (long long)st.st_size, + (unsigned long long)data_start, + ps); + } + ksft_test_result( + headers == 2 ? + exists && st.st_size > 0 && + no_data : + !exists && errno == ENOENT, + "%luK core RLIMIT_CORE %s one user page\n", + ps / 1024, + headers == 2 ? "at" : "below"); + unlink(path); + } else { + inspect(pid, ps, headers); + } + } + } + ksft_print_cnts(); + printf("%s - core fixture complete\n", + ksft_get_fail_cnt() ? "not ok" : "ok"); + reboot(RB_POWER_OFF); + return ksft_get_fail_cnt() ? 1 : 0; +} diff --git a/tools/testing/selftests/arm64/user4k/cpu-granule-init.c b/tools/testing/selftests/arm64/user4k/cpu-granule-init.c new file mode 100644 index 0000000000000..1e983f91f74c7 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/cpu-granule-init.c @@ -0,0 +1,187 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static int pin_cpu(pid_t pid, unsigned int cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(pid, sizeof(set), &set); +} +static int online_cpu1(void) +{ + char c = '?'; + int fd = open("/sys/devices/system/cpu/cpu1/online", O_RDONLY); + if (fd >= 0) { + if (read(fd, &c, 1) != 1) + c = '?'; + close(fd); + } + return c == '0' ? 0 : c == '1' ? 1 : -1; +} +static int worker(int ready, int command, unsigned long expected) +{ + unsigned char *p; + unsigned long ps = getauxval(AT_PAGESZ); + char cpu; + int ok = getppid() == 1 && ps == expected; + + p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 2; + for (unsigned long i = 0; i < 4 * ps; i++) + p[i] = i ^ (i >> 12); + if (write(ready, "r", 1) != 1 || read(command, &cpu, 1) != 1) + return 3; + ok &= sched_getcpu() == cpu - '0'; + for (unsigned int round = 0; round < 128; round++) { + for (unsigned long i = 0; i < 4 * ps; i++) + if (p[i] != (unsigned char)(i ^ (i >> 12))) + ok = 0; + sched_yield(); + } + munmap(p, 4 * ps); + return ok ? 0 : 4; +} +static int compat_sample(unsigned long expected) +{ + uint64_t data[4] = { 0 }; + int pipefd[2], status; + pid_t child; + ssize_t size; + + if (pipe(pipefd)) + return 0; + child = fork(); + if (!child) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + execl("/compat-aslr", "/compat-aslr", "--sample", fd, NULL); + _exit(2); + } + close(pipefd[1]); + size = read(pipefd[0], data, sizeof(data)); + close(pipefd[0]); + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && + size == sizeof(data) && data[0] == expected; +} +int main(int argc, char **argv) +{ + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10)); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, + "Dedicated two-CPU disposable VM with maxcpus=1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) { + perror("mount"); + failed++; + goto done; + } + unsigned long native = getauxval(AT_PAGESZ); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + result(!prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL), + "native exec default restored"); + char cmdline[4096] = { 0 }; + int fd = open("/proc/cmdline", O_RDONLY); + if (fd < 0 || read(fd, cmdline, sizeof(cmdline) - 1) < 0) { + failed++; + goto done; + } + close(fd); + int reject = has4k && strstr(cmdline, "arm64.user4k_deny_late_cpu=1"); + result(online_cpu1() == 0, "CPU1 starts offline"); + if (online_cpu1() != 0) + goto done; + int ready[2], command[2]; + if (pipe(ready) || pipe(command)) { + failed++; + goto done; + } + pid_t child = fork(); + if (!child) { + char ready_fd[32], command_fd[32], ps[32]; + close(ready[0]); + close(command[1]); + snprintf(ready_fd, sizeof(ready_fd), "%d", ready[1]); + snprintf(command_fd, sizeof(command_fd), "%d", command[0]); + snprintf(ps, sizeof(ps), "%lu", has4k ? 4096UL : native); + if (prctl(SET_EXEC_PAGE_SIZE, has4k ? 4096UL : 0UL, 0UL, 0UL, + 0UL)) + _exit(5); + execl("/init", "/init", "--worker", ready_fd, command_fd, ps, + NULL); + _exit(6); + } + close(ready[1]); + close(command[0]); + if (child < 0) { + failed++; + goto done; + } + char token; + result(read(ready[0], &token, 1) == 1 && token == 'r', + "worker has live mappings before hotplug"); + close(ready[0]); + fd = open("/sys/devices/system/cpu/cpu1/online", O_WRONLY); + errno = 0; + ssize_t written = fd < 0 ? -1 : write(fd, "1", 1); + int saved_errno = errno; + if (fd >= 0) + close(fd); + int online = online_cpu1(); + printf("# has4k=%d expected-reject=%d write=%zd errno=%d online=%d\n", + has4k, reject, written, saved_errno, online); + result(reject ? written < 0 && online == 0 : + written == 1 && online == 1, + "late CPU admission matches advertised user granule"); + unsigned int cpu = online == 1 ? 1 : 0; + result(!pin_cpu(child, cpu), "worker affinity targets admitted CPU"); + token = '0' + cpu; + result(write(command[1], &token, 1) == 1, + "worker resumes after CPU admission"); + close(command[1]); + int status; + result(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status), + "live mapping bytes survive and worker runs on expected CPU"); + result(!pin_cpu(0, cpu), "launcher affinity targets admitted CPU"); + result(compat_sample(has4k ? 4096 : native), + "AArch32 exec retains selected or native-fallback granule"); +done: + printf("CPU GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c b/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c new file mode 100644 index 0000000000000..ecf682a22f732 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static int pin_cpu(pid_t pid, unsigned int cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(pid, sizeof(set), &set); +} +static int online_cpu1(void) +{ + char c = '?'; + int fd = open("/sys/devices/system/cpu/cpu1/online", O_RDONLY); + if (fd >= 0) { + if (read(fd, &c, 1) != 1) + c = '?'; + close(fd); + } + return c == '0' ? 0 : c == '1' ? 1 : -1; +} +static int worker(int ready, int command, unsigned long expected) +{ + unsigned char *p; + unsigned long ps = getauxval(AT_PAGESZ); + char cpu; + int ok = getppid() == 1 && ps == expected; + + p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 2; + for (unsigned long i = 0; i < 4 * ps; i++) + p[i] = i ^ (i >> 12); + if (write(ready, "r", 1) != 1 || read(command, &cpu, 1) != 1) + return 3; + ok &= sched_getcpu() == cpu - '0'; + for (unsigned int round = 0; round < 128; round++) { + for (unsigned long i = 0; i < 4 * ps; i++) + if (p[i] != (unsigned char)(i ^ (i >> 12))) + ok = 0; + sched_yield(); + } + munmap(p, 4 * ps); + return ok ? 0 : 4; +} +static int compat_sample(unsigned long expected) +{ + uint64_t data[4] = { 0 }; + int pipefd[2], status; + pid_t child; + ssize_t size; + + if (pipe(pipefd)) + return 0; + child = fork(); + if (!child) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + execl("/compat-aslr", "/compat-aslr", "--sample", fd, NULL); + _exit(2); + } + close(pipefd[1]); + size = read(pipefd[0], data, sizeof(data)); + close(pipefd[0]); + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && + size == sizeof(data) && data[0] == expected; +} +int main(int argc, char **argv) +{ + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10)); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, + "Dedicated two-CPU disposable VM with maxcpus=1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) { + perror("mount"); + failed++; + goto done; + } + unsigned long native = getauxval(AT_PAGESZ); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + int has16k = native > 16384 && + !prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL); + result(!prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL), + "native exec default restored"); + char cmdline[4096] = { 0 }; + int fd = open("/proc/cmdline", O_RDONLY); + if (fd < 0 || read(fd, cmdline, sizeof(cmdline) - 1) < 0) { + failed++; + goto done; + } + close(fd); + int reject = has16k && strstr(cmdline, "arm64.user16k_deny_late_cpu=1"); + result(online_cpu1() == 0, "CPU1 starts offline"); + if (online_cpu1() != 0) + goto done; + int ready[2], command[2]; + if (pipe(ready) || pipe(command)) { + failed++; + goto done; + } + pid_t child = fork(); + if (!child) { + char ready_fd[32], command_fd[32], ps[32]; + close(ready[0]); + close(command[1]); + snprintf(ready_fd, sizeof(ready_fd), "%d", ready[1]); + snprintf(command_fd, sizeof(command_fd), "%d", command[0]); + snprintf(ps, sizeof(ps), "%lu", has16k ? 16384UL : native); + if (prctl(SET_EXEC_PAGE_SIZE, has16k ? 16384UL : 0UL, 0UL, 0UL, + 0UL)) + _exit(5); + execl("/init", "/init", "--worker", ready_fd, command_fd, ps, + NULL); + _exit(6); + } + close(ready[1]); + close(command[0]); + if (child < 0) { + failed++; + goto done; + } + char token; + result(read(ready[0], &token, 1) == 1 && token == 'r', + "worker has live mappings before hotplug"); + close(ready[0]); + fd = open("/sys/devices/system/cpu/cpu1/online", O_WRONLY); + errno = 0; + ssize_t written = fd < 0 ? -1 : write(fd, "1", 1); + int saved_errno = errno; + if (fd >= 0) + close(fd); + int online = online_cpu1(); + printf("# has16k=%d expected-reject=%d write=%zd errno=%d online=%d\n", + has16k, reject, written, saved_errno, online); + result(reject ? written < 0 && online == 0 : + written == 1 && online == 1, + "late CPU admission matches advertised user granule"); + unsigned int cpu = online == 1 ? 1 : 0; + result(!pin_cpu(child, cpu), "worker affinity targets admitted CPU"); + token = '0' + cpu; + result(write(command[1], &token, 1) == 1, + "worker resumes after CPU admission"); + close(command[1]); + int status; + result(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status), + "live mapping bytes survive and worker runs on expected CPU"); + result(!pin_cpu(0, cpu), "launcher affinity targets admitted CPU"); + if (has16k) + result(!prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL), + "16K request staged before compat exec"); + result(compat_sample(has4k ? 4096 : native), + "AArch32 exec selects4K or native despite pending16K"); +done: + printf("CPU GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/dio-fixture-init.c b/tools/testing/selftests/arm64/user4k/dio-fixture-init.c new file mode 100644 index 0000000000000..36ebec4350c76 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/dio-fixture-init.c @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static bool use_file; +static unsigned char pattern(size_t i) +{ + return (i * 17 + (i >> 12) * 29 + 71) & 255; +} +static void fill(unsigned char *p, size_t len) +{ + for (size_t i = 0; i < len; i++) + p[i] = pattern(i); +} +static bool check(const unsigned char *p, size_t len, size_t off) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != pattern(i + off)) { + ksft_print_msg("mismatch at %zu: %u vs %u\n", i + off, + p[i], pattern(i + off)); + return false; + } + return true; +} +static void report(bool ok, size_t ps, const char *name) +{ + ksft_test_result(ok, "%zuK %s (errno=%d)\n", ps / 1024, name, errno); +} +static bool aio_transfer(int fd, void *p, size_t len, off_t off, bool write) +{ + aio_context_t ctx = 0; + struct iocb cb = { .aio_lio_opcode = write ? IOCB_CMD_PWRITE : + IOCB_CMD_PREAD, + .aio_fildes = fd, + .aio_buf = (uintptr_t)p, + .aio_nbytes = len, + .aio_offset = off }; + struct iocb *ptr = &cb; + struct io_event ev; + struct timespec timeout = { .tv_sec = 10 }; + bool ok; + + if (syscall(__NR_io_setup, 8, &ctx)) + return false; + ok = syscall(__NR_io_submit, ctx, 1, &ptr) == 1; + if (ok) + ok = syscall(__NR_io_getevents, ctx, 1, 1, &ev, &timeout) == + 1 && + ev.res == (long)len && !ev.res2; + ok &= !syscall(__NR_io_destroy, ctx); + return ok; +} +static int exercise(void) +{ + size_t ps = getauxval(AT_PAGESZ), len = 2 * 1024 * 1024; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *q = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + struct iovec vec[4]; + int fd = open(use_file ? "/fs/test" : "/dev/user4k-test", + O_RDWR | O_DIRECT | (use_file ? O_CREAT | O_TRUNC : 0), + 0600), + memfd; + bool ok; + ssize_t ret; + + ksft_print_header(); + ksft_set_plan(10); + ksft_print_msg("backing=%s\n", use_file ? "ext4" : "raw block"); + if ((ps != 4096 && ps != 16384) || p == MAP_FAILED || q == MAP_FAILED || + fd < 0) + ksft_exit_fail_msg("setup failed errno=%d\n", errno); + if (use_file && ftruncate(fd, 32 * 1024 * 1024)) + ksft_exit_fail_msg("ftruncate failed\n"); + fill(p, len); + memset(q, 0, len); + ok = pwrite(fd, p + ps, 3 * ps, 0) == (ssize_t)(3 * ps) && + pread(fd, q + ps, 3 * ps, 0) == (ssize_t)(3 * ps) && + check(q + ps, 3 * ps, ps); + report(ok, ps, "O_DIRECT exact user offsets and bytes"); + + vec[0] = (struct iovec){ p + 512, ps - 512 }; + vec[1] = (struct iovec){ NULL, 0 }; + vec[2] = (struct iovec){ p + 5 * ps, ps + 512 }; + vec[3] = (struct iovec){ p + 11 * ps + 512, 512 }; + ok = pwritev(fd, vec, 4, 65536) == (ssize_t)(2 * ps + 512); + memset(q, 0, 3 * ps); + ok &= pread(fd, q, 2 * ps + 512, 65536) == (ssize_t)(2 * ps + 512); + ok &= check(q, ps - 512, 512) && + check(q + ps - 512, ps + 512, 5 * ps) && + check(q + 2 * ps, 512, 11 * ps + 512); + report(ok, ps, "fragmented vectors and empty segment"); + + ok = pwrite(fd, p, len, 4 * 1024 * 1024) == (ssize_t)len; + memset(q, 0, len); + ok &= pread(fd, q, len, 4 * 1024 * 1024) == (ssize_t)len && + check(q, len, 0); + report(ok, ps, "multi-bio two-megabyte transfer"); + + ok = !mprotect(p + ps, ps, PROT_NONE); + ret = pwrite(fd, p, 2 * ps, 8 * 1024 * 1024); + ok &= use_file ? ret == -1 && errno == EFAULT : ret == (ssize_t)ps; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE); + if (use_file) + ok &= pwrite(fd, p, ps, 8 * 1024 * 1024) == (ssize_t)ps; + memset(q, 0, ps); + ok &= pread(fd, q, ps, 8 * 1024 * 1024) == (ssize_t)ps && + check(q, ps, 0); + report(ok, ps, "source fault and subsequent I/O recovery"); + + ok = !mprotect(q + ps, ps, PROT_NONE); + ret = pread(fd, q, 2 * ps, 4 * 1024 * 1024); + ok &= (use_file ? ret == -1 && errno == EFAULT : ret == (ssize_t)ps) && + check(q, ps, 0); + ok &= !mprotect(q + ps, ps, PROT_READ | PROT_WRITE); + report(ok, ps, "partial destination fault"); + + ok = !mprotect(p + ps, ps, PROT_NONE); + /* Only 256 accessible bytes: none can form a 512-byte device block. */ + ret = pwrite(fd, p + ps - 256, 512, 10 * 1024 * 1024); + ok &= ret == -1 && errno == EFAULT; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE); + report(ok, ps, "alignment trimming releases all short-fragment pins"); + + memfd = memfd_create("dio-shared", 0); + ok = memfd >= 0 && !ftruncate(memfd, 4 * ps); + unsigned char *shared = ok ? mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, 0) : + MAP_FAILED; + ok &= shared != MAP_FAILED; + if (ok) { + memset(shared, 0xa7, 4 * ps); + ok = pread(fd, shared + ps, 2 * ps, 4 * 1024 * 1024) == + (ssize_t)(2 * ps) && + !msync(shared, 4 * ps, MS_SYNC) && + pread(memfd, q, 4 * ps, 0) == (ssize_t)(4 * ps); + ok &= check(q + ps, 2 * ps, 0); + for (size_t i = 0; i < ps; i++) + ok &= q[i] == 0xa7 && q[3 * ps + i] == 0xa7; + munmap(shared, 4 * ps); + } + if (memfd >= 0) + close(memfd); + report(ok, ps, "shared-file destination dirtying and adjacent guards"); + + ok = aio_transfer(fd, p + ps, 7 * ps, 12 * 1024 * 1024, true); + memset(q, 0, 8 * ps); + ok &= aio_transfer(fd, q + ps, 7 * ps, 12 * 1024 * 1024, false) && + check(q + ps, 7 * ps, ps); + report(ok, ps, "asynchronous AIO read/write completion"); + + memset(q, 0x96, 8 * ps); + pid_t child = fork(); + int status = 0; + if (!child) { + _exit(aio_transfer(fd, q + ps, 7 * ps, 12 * 1024 * 1024, + false) && + check(q + ps, 7 * ps, ps) ? + 0 : + 1); + } + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + for (size_t i = 0; i < 8 * ps; i++) + ok &= q[i] == 0x96; + report(ok, ps, "writable-pin COW after fork"); + + ok = true; + for (unsigned i = 0; i < 128 && ok; i++) { + ok = !madvise(q + ps, ps, MADV_DONTNEED) && + pread(fd, q + ps, ps, 4 * 1024 * 1024) == (ssize_t)ps && + check(q + ps, ps, 0); + ok &= !mprotect(q + 2 * ps, ps, PROT_NONE); + ok &= pread(fd, q + 2 * ps, ps, 4 * 1024 * 1024) == -1 && + errno == EFAULT; + ok &= !mprotect(q + 2 * ps, ps, PROT_READ | PROT_WRITE); + } + report(ok, ps, "128 reuse and failed-pin cleanup cycles"); + close(fd); + munmap(p, len); + munmap(q, len); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +int main(int argc, char **argv) +{ + bool ok = true; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && + (!strcmp(argv[1], "--test") || !strcmp(argv[1], "--file")) && + getppid() == 1) { + use_file = !strcmp(argv[1], "--file"); + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + for (unsigned disk = 0; disk < 2; disk++) { + char path[80], serial[64]; + unsigned maj, min; + snprintf(path, sizeof(path), "/sys/class/block/vd%c/serial", + 'a' + disk); + FILE *dev = fopen(path, "r"); + if (!dev || fscanf(dev, "%63s", serial) != 1) + return 4; + fclose(dev); + snprintf(path, sizeof(path), "/sys/class/block/vd%c/dev", + 'a' + disk); + dev = fopen(path, "r"); + if (!dev || fscanf(dev, "%u:%u", &maj, &min) != 2) + return 4; + fclose(dev); + const char *node = + !strcmp(serial, "au4k-raw") ? "/dev/user4k-test" : + !strcmp(serial, "au4k-ext4") ? "/dev/user4k-fs" : + NULL; + if (!node || mknod(node, S_IFBLK | 0600, makedev(maj, min))) + return 5; + } + if (mount("/dev/user4k-fs", "/fs", "ext4", 0, NULL)) + return 7; + for (unsigned mode = 0; mode < 4; mode++) { + unsigned small = mode & 1; + pid_t pid = fork(); + int status = 0; + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + small ? 4096UL : 0UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", mode >= 2 ? "--file" : "--test", + NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + ok &= !umount("/fs"); + printf("%s - direct I/O fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/exec-fixture-init.c b/tools/testing/selftests/arm64/user4k/exec-fixture-init.c new file mode 100644 index 0000000000000..8a390d5df3ddc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/exec-fixture-init.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* PID-1 runner for the experimental one-shot exec ABI in a disposable VM. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include + +static int exec_size(unsigned long size) +{ + return prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL); +} + +static int pending_size(void) +{ + return prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); +} + +static int run(const char *path, const char *arg, unsigned long size) +{ + pid_t pid = fork(), got; + int status = 0; + + if (!pid) { + if (size && exec_size(size)) { + perror("set exec page size"); + _exit(125); + } + execl(path, path, arg, NULL); + perror(path); + _exit(126); + } + if (pid < 0) + return -1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + printf("# child %s %s: wait=%d status=%#x\n", path, arg ? arg : "", got, + status); + return got == pid && WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +static int argument_exec(unsigned long size) +{ + pid_t pid = fork(), got; + int status = 0; + + if (!pid) { + char *payload = malloc(140000), expected[16]; + unsigned int i; + + if (!payload || exec_size(size)) + _exit(120); + for (i = 0; i < 139999; i++) + payload[i] = 'a' + i % 23; + payload[139999] = 0; + if (size == 4096) { + execl("/init", "/init", "--too-long", payload, NULL); + if (errno != E2BIG || pending_size() != 4096) + _exit(121); + } + payload[120000] = 0; + if (setenv("ARG_PAYLOAD", payload, 1)) + _exit(122); + snprintf(expected, sizeof(expected), "%lu", + size ? size : 16384UL); + execl("/init", "/init", "--args-check", expected, payload, + NULL); + perror("argument exec"); + _exit(123); + } + if (pid < 0) + return -1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + printf("# large arguments from %lu to %lu: status=%#x\n", + getauxval(AT_PAGESZ), size ? size : 16384UL, status); + return got == pid && WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +int main(int argc, char **argv) +{ + bool ok; + int failures = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--args-check")) { + const char *env = getenv("ARG_PAYLOAD"); + unsigned int i; + + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10) || + pending_size() != 0 || strlen(argv[3]) != 120000 || !env || + strlen(env) != 120000) + return 1; + for (i = 0; i < 120000; i++) + if (argv[3][i] != 'a' + i % 23 || env[i] != argv[3][i]) + return 2; + return 0; + } + if (argc == 2 && !strcmp(argv[1], "--args-from-4k")) { + if (getauxval(AT_PAGESZ) != 4096) + return 1; + return argument_exec(4096) || argument_exec(0); + } + if (argc == 2 && !strcmp(argv[1], "--exec-cycle")) { + if (getauxval(AT_PAGESZ) != 4096 || pending_size() != 0) + return 1; + /* A successful exec consumes the request; ordinary exec is native. */ + execl("/init", "/init", "--native-after-4k", NULL); + return 2; + } + if (argc == 2 && !strcmp(argv[1], "--native-after-4k")) + return getauxval(AT_PAGESZ) != 16384 || pending_size() != 0; + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as /init in a disposable VM.\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) && errno != EBUSY) + return 3; + printf("# exec fixture: native AT_PAGESZ=%lu\n", getauxval(AT_PAGESZ)); + failures += run("/page-contract", NULL, 0) != 0; + ok = pending_size() == 0; + errno = 0; + ok &= exec_size(8192) == -1 && errno == EINVAL; + ok &= prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096UL, 1UL, 0UL, 0UL) == -1; + ok &= exec_size(4096) == 0 && pending_size() == 4096; + execl("/does-not-exist", "/does-not-exist", NULL); + ok &= errno == ENOENT && pending_size() == 4096; + ok &= getauxval(AT_PAGESZ) == 16384; + printf("%s - request validation and failed exec preserve live geometry\n", + ok ? "ok" : "not ok"); + failures += !ok; + /* fork inherits the pending request; successful child exec consumes it. */ + failures += run("/page-contract-4k", "--expect-4k", 0) != 0; + ok = pending_size() == 4096 && exec_size(0) == 0 && pending_size() == 0; + printf("%s - child exec leaves parent request intact; reset works\n", + ok ? "ok" : "not ok"); + failures += !ok; + failures += run("/page-contract-dynamic-4k", "--expect-4k", 4096) != 0; + failures += run("/init", "--exec-cycle", 4096) != 0; + failures += run("/proc-memory", NULL, 0) != 0; + failures += run("/proc-memory", "--expect-4k", 4096) != 0; + failures += run("/proc-pages", NULL, 0) != 0; + failures += run("/proc-pages", "--expect-4k", 4096) != 0; + failures += run("/advice", NULL, 0) != 0; + failures += run("/advice", "--expect-4k", 4096) != 0; + failures += run("/memory-interfaces", NULL, 0) != 0; + failures += run("/memory-interfaces", "--expect-4k", 4096) != 0; + failures += run("/mixed-process", NULL, 0) != 0; + failures += run("/mixed-process", NULL, 4096) != 0; + failures += argument_exec(4096) != 0; + failures += run("/init", "--args-from-4k", 4096) != 0; + failures += run("/page-contract", NULL, 0) != 0; + printf("%s - exec fixture complete: failures=%d\n", + failures ? "not ok" : "ok", failures); + reboot(RB_POWER_OFF); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/arm64/user4k/exec-policy-init.c b/tools/testing/selftests/arm64/user4k/exec-policy-init.c new file mode 100644 index 0000000000000..02fd3f9552237 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/exec-policy-init.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable guest PID1: per-thread one-shot and inherited exec preferences. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define REQUIRE(x) \ + do { \ + if (!(x)) { \ + fprintf(stderr, "FAIL line %d: %s errno=%d\n", \ + __LINE__, #x, errno); \ + exit(1); \ + } \ + } while (0) + +static int policy(int op, unsigned long value) +{ + return prctl(op, value, 0UL, 0UL, 0UL); +} + +static void state(unsigned long pages, unsigned long preferred, int pending) +{ + REQUIRE(getauxval(AT_PAGESZ) == pages); + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 0) == pending); +} + +static void chain(int stage, unsigned long native, unsigned long preferred) +{ + char s[32], n[32], p[32]; + + snprintf(s, sizeof(s), "%d", stage); + snprintf(n, sizeof(n), "%lu", native); + snprintf(p, sizeof(p), "%lu", preferred); + execl("/init", "/init", "--chain", s, n, p, NULL); + REQUIRE(0); +} + +static void *thread_policy(void *arg) +{ + unsigned long preferred = *(unsigned long *)arg; + + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 0)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 0)); + state(getauxval(AT_PAGESZ), 0, 0); + return NULL; +} + +static void *thread_exec(void *arg) +{ + unsigned long *sizes = arg; + + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, sizes[1])); + chain(0, sizes[0], sizes[1]); + return NULL; +} + +static int wait_child(pid_t pid) +{ + int status; + + REQUIRE(pid > 0); + REQUIRE(waitpid(pid, &status, 0) == pid); + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +static int run_preference(unsigned long native, unsigned long preferred) +{ + pid_t pid = fork(); + + if (!pid) { + pthread_t thread; + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, preferred)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, preferred)); + REQUIRE(!pthread_create(&thread, NULL, thread_policy, + &preferred)); + REQUIRE(!pthread_join(thread, NULL)); + state(native, preferred, preferred); + /* Rejected values and reserved arguments must preserve both policies. */ + REQUIRE(policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 8192) == -1 && + errno == EINVAL); + REQUIRE(policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 8192) == -1 && + errno == EINVAL); + int ops[] = { PR_AURORA_SET_DEFAULT_PAGE_SIZE, + PR_AURORA_GET_DEFAULT_PAGE_SIZE, + PR_AURORA_SET_EXEC_PAGE_SIZE, + PR_AURORA_GET_EXEC_PAGE_SIZE }; + for (unsigned int i = 0; i < sizeof(ops) / sizeof(ops[0]); + i++) { + REQUIRE(prctl(ops[i], 0UL, 1UL, 0UL, 0UL) == -1 && + errno == EINVAL); + REQUIRE(prctl(ops[i], 0UL, 0UL, 1UL, 0UL) == -1 && + errno == EINVAL); + REQUIRE(prctl(ops[i], 0UL, 0UL, 0UL, 1UL) == -1 && + errno == EINVAL); + } + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 1) == -1 && + errno == EINVAL); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 1) == -1 && + errno == EINVAL); + execl("/does-not-exist", "/does-not-exist", NULL); + REQUIRE(errno == ENOENT); + state(native, preferred, preferred); + chain(0, native, preferred); + } + return wait_child(pid); +} + +static int compat(unsigned long native, unsigned long preferred, int no4k) +{ + pid_t pid = fork(); + + if (!pid) { + char n[32], p[32], e[32]; + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, preferred)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, native)); + snprintf(n, sizeof(n), "%lu", native); + snprintf(p, sizeof(p), "%lu", preferred); + snprintf(e, sizeof(e), "%lu", no4k ? native : 4096UL); + execl("/compat-policy", "/compat-policy", n, p, e, NULL); + REQUIRE(0); + } + return wait_child(pid); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--chain")) { + int stage = atoi(argv[2]); + unsigned long n = strtoul(argv[3], NULL, 10); + unsigned long p = strtoul(argv[4], NULL, 10); + + state(stage == 2 || stage == 5 ? n : p, stage < 4 ? p : 0, 0); + if (stage == 5) + return 0; + if (stage == 1) { + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 0)); + state(p, p, n); + } + if (stage == 3) { + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 0)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, p)); + } + chain(stage + 1, n, p); + } + if (argc == 4 && !strcmp(argv[1], "--compat-return")) { + unsigned long n = strtoul(argv[2], NULL, 10); + unsigned long p = strtoul(argv[3], NULL, 10); + state(p ? p : n, p, 0); + return 0; + } + REQUIRE(getpid() == 1); + REQUIRE(!mount("proc", "/proc", "proc", 0, NULL)); + state(native, 0, 0); + int no4k = argc == 2 && !strcmp(argv[1], "--no4k"); + unsigned long preferences[] = { 4096, 16384 }; + for (unsigned int i = 0; i < 2; i++) { + unsigned long p = preferences[i]; + if (p >= native || (no4k && p == 4096)) + continue; + int ok = run_preference(native, p); + printf("%s - exec policy native=%lu alternative=%lu\n", + ok ? "ok" : "not ok", native, p); + failed |= !ok; + pid_t pid = fork(); + if (!pid) { + pthread_t thread; + unsigned long sizes[] = { native, p }; + REQUIRE(!pthread_create(&thread, NULL, thread_exec, + sizes)); + for (;;) + pause(); + } + ok = wait_child(pid); + printf("%s - non-leader exec alternative=%lu\n", + ok ? "ok" : "not ok", p); + failed |= !ok; + ok = compat(native, p, no4k); + printf("%s - compat policy alternative=%lu no4k=%d\n", + ok ? "ok" : "not ok", p, no4k); + failed |= !ok; + } + if (no4k) { + int ok = policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 4096) == -1 && + errno == EOPNOTSUPP; + ok &= policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096) == -1 && + errno == EOPNOTSUPP; + ok &= compat(native, 0, 1); + printf("%s - no4k preserves native compat restrictions\n", + ok ? "ok" : "not ok"); + failed |= !ok; + } + state(native, 0, 0); + printf("EXEC POLICY %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + return failed; +} diff --git a/tools/testing/selftests/arm64/user4k/fex-contract-init.c b/tools/testing/selftests/arm64/user4k/fex-contract-init.c new file mode 100644 index 0000000000000..3273af30eea7f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-contract-init.c @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +int main(void) +{ + if (getpid() != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + int failed = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + failed = 1; + goto done; + } + pid_t child = fork(); + if (!child) { + setenv("HOME", "/root", 1); + setenv("PATH", "/usr/bin:/bin", 1); + setenv("XDG_RUNTIME_DIR", "/run", 1); + setenv("FEX_ROOTFS", "/", 1); + setenv("LD_PRELOAD", "/usr/lib/libfex-abi.so", 1); + if (prctl(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 4096UL, 0UL, 0UL, + 0UL) || + prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/usr/bin/FEXInterpreter", "/usr/bin/FEXInterpreter", + "/x86-contract", NULL); + perror("FEX exec"); + _exit(4); + } + int status = 0; + int ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - FEX 4K native-process / x86 workload status=%#x\n", + ok ? "ok" : "not ok", status); + failed |= !ok; +done: + printf("GRANULE FEX RUNTIME %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/fex-native-abi.c b/tools/testing/selftests/arm64/user4k/fex-native-abi.c new file mode 100644 index 0000000000000..86ccb6e389d9d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-native-abi.c @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +__attribute__((constructor)) static void check_abi(void) +{ + char path[1024]; + ssize_t n = readlink("/proc/self/exe", path, sizeof(path) - 1); + if (n < 0) + return; + path[n] = 0; + if (!strstr(path, "FEXInterpreter")) + return; + unsigned long ps = getauxval(AT_PAGESZ); + dprintf(2, "FEX NATIVE HOST PROCESS PAGE SIZE %lu\n", ps); + if (ps != 4096) + _exit(90); +} diff --git a/tools/testing/selftests/arm64/user4k/fex-x86-contract.c b/tools/testing/selftests/arm64/user4k/fex-x86-contract.c new file mode 100644 index 0000000000000..047a3c9d28ddb --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-x86-contract.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0-only +typedef unsigned long size_t; +static long sc(long n, long a, long b, long c, long d, long e, long f) +{ + register long r10 __asm__("r10") = d, r8 __asm__("r8") = e, + r9 __asm__("r9") = f; + long r; + __asm__ volatile("syscall" + : "=a"(r) + : "a"(n), "D"(a), "S"(b), "d"(c), "r"(r10), "r"(r8), + "r"(r9) + : "rcx", "r11", "memory"); + return r; +} +static void say(const char *s, size_t n) +{ + sc(1, 1, (long)s, n, 0, 0, 0); +} +#define SAY(s) say(s, sizeof(s) - 1) +static void quit(int n) +{ + sc(231, n, 0, 0, 0, 0, 0); + for (;;) + ; +} +static int check(unsigned char *p, size_t size) +{ + for (size_t i = 0; i < size; i++) + if (p[i] != (unsigned char)(i ^ (i >> 12))) + return 0; + return 1; +} +long run(unsigned long *sp) +{ + unsigned long argc = *sp++, pagesz = 0; + sp += argc + 1; + unsigned long *env = sp; + while (*sp) + sp++; + sp++; + while (*sp) { + if (sp[0] == 6) + pagesz = sp[1]; + sp += 2; + } + if (pagesz != 4096) + return 10; + SAY("ok - FEX x86 guest 4K auxv\n"); + unsigned char *p = (void *)sc(9, 0, 12288, 3, 0x22, -1, 0); + if ((long)p < 0) + return 11; + for (size_t i = 0; i < 12288; i++) + p[i] = (unsigned char)(i ^ (i >> 12)); + if (sc(10, (long)p + 4096, 4096, 0, 0, 0, 0)) + return 12; + long child = sc(57, 0, 0, 0, 0, 0, 0); + int status = 0; + if (!child) { + *(volatile unsigned char *)(p + 4096) = 1; + quit(13); + } + if (child < 0 || sc(61, child, (long)&status, 0, 0, 0, 0) != child || + (status & 127) != 11) + return 14; + if (sc(10, (long)p + 4096, 4096, 3, 0, 0, 0) || !check(p, 12288)) + return 15; + SAY("ok - FEX x86 independent 4K protection fault\n"); + child = sc(57, 0, 0, 0, 0, 0, 0); + if (!child) { + p[4096] = 0x71; + quit(0); + } + if (child < 0 || sc(61, child, (long)&status, 0, 0, 0, 0) != child || + status || !check(p, 12288)) + return 16; + SAY("ok - FEX x86 fork COW\n"); + p = (void *)sc(25, (long)p, 12288, 16384, 1, 0, 0); + if ((long)p < 0 || !check(p, 12288)) + return 17; + for (size_t i = 12288; i < 16384; i++) + if (p[i]) + return 18; + SAY("ok - FEX x86 mremap preserve and zero extension\n"); + long fd = sc(319, (long)"fex-shared", 0, 0, 0, 0, 0); + if (fd < 0 || sc(77, fd, 12288, 0, 0, 0, 0)) + return 19; + unsigned char *a = (void *)sc(9, 0, 4096, 3, 1, fd, 4096), + *b = (void *)sc(9, 0, 4096, 3, 1, fd, 4096); + if ((long)a < 0 || (long)b < 0) + return 20; + a[37] = 0x65; + if (b[37] != 0x65) + return 21; + SAY("ok - FEX x86 memfd mapping at 4K file offset\n"); + unsigned char *code = (void *)sc(9, 0, 4096, 3, 0x22, -1, 0); + if ((long)code < 0) + return 22; + code[0] = 0xb8; + code[1] = 42; + code[2] = 0; + code[3] = 0; + code[4] = 0; + code[5] = 0xc3; + if (sc(10, (long)code, 4096, 5, 0, 0, 0) || + ((int (*)(void))code)() != 42) + return 23; + if (sc(10, (long)code, 4096, 3, 0, 0, 0)) + return 24; + code[1] = 99; + if (sc(10, (long)code, 4096, 5, 0, 0, 0) || + ((int (*)(void))code)() != 99) + return 25; + SAY("ok - FEX x86 JIT permission transition and code replacement\n"); + unsigned char random[32]; + if (sc(318, (long)random, 32, 0, 0, 0, 0) != 32) + return 26; + if (argc == 1) { + child = sc(57, 0, 0, 0, 0, 0, 0); + if (!child) { + char *args[] = { "/x86-contract", "--exec-child", 0 }; + sc(59, (long)args[0], (long)args, (long)env, 0, 0, 0); + quit(28); + } + if (child < 0 || + sc(61, child, (long)&status, 0, 0, 0, 0) != child || status) + return 27; + SAY("ok - FEX x86 fork and re-exec with 4K mappings\n"); + } + SAY("FEX X86 4K CONTRACT PASS\n"); + return 0; +} +__asm__(".global _start\n_start:\nmov %rsp,%rdi\nand $-16,%rsp\ncall run\nmov %rax,%rdi\nmov $231,%rax\nsyscall\n"); diff --git a/tools/testing/selftests/arm64/user4k/gem-fixture-init.c b/tools/testing/selftests/arm64/user4k/gem-fixture-init.c new file mode 100644 index 0000000000000..aadf343dc7e8d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/gem-fixture-init.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static unsigned char value(size_t off) +{ + return 0x31 + (off / 4096) % 128; +} +static bool check(unsigned char *p, size_t len, size_t off) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value(i + off)) { + ksft_print_msg("offset=%zu got=%u expected=%u\n", + i + off, p[i], value(i + off)); + return false; + } + return true; +} +static int exercise(size_t native) +{ + size_t ps = getauxval(AT_PAGESZ), + len = native < 8192 ? 65536 : 8 * native; + int fd = open("/dev/vgem", O_RDWR); + struct drm_mode_create_dumb create = { .width = 4096, + .height = len / (4096 * 4), + .bpp = 32 }; + struct drm_mode_map_dumb map = {}; + struct drm_prime_handle prime = { .flags = DRM_RDWR }; + unsigned char *p = MAP_FAILED, *q = MAP_FAILED, *alias = MAP_FAILED; + bool ok = fd >= 0 && !ioctl(fd, DRM_IOCTL_MODE_CREATE_DUMB, &create); + ksft_print_header(); + ksft_set_plan(12); + ksft_print_msg("ABI=%zuK\n", ps / 1024); + map.handle = create.handle; + ok &= !ioctl(fd, DRM_IOCTL_MODE_MAP_DUMB, &map) && create.size >= len; + if (ok) + p = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + ok &= p != MAP_FAILED; + ksft_test_result(ok, "full GEM shmem object mapping\n"); + q = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + bool separate = q != MAP_FAILED; + if (separate) { + memset(q, 0x39, ps); + memset(q + ps, 0xa2, ps); + for (size_t i = 0; i < ps; i++) + separate &= q[i] == 0x39 && q[ps + i] == 0xa2; + munmap(q, 2 * ps); + q = MAP_FAILED; + } + ksft_test_result(separate, + "adjacent user leaves map distinct backing bytes\n"); + if (ok) { + for (size_t i = 0; i < len; i++) + p[i] = value(i); + ok = check(p, len, 0); + } + ksft_test_result(ok, "full object has distinct quarter data\n"); + if (ok) { + q = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + ok = q != MAP_FAILED && !mprotect(q + ps, ps, PROT_READ); + if (ok) + ok = check(q, len, 0); + if (q != MAP_FAILED) + munmap(q, len); + q = MAP_FAILED; + } + ksft_test_result( + ok, + "cold faults after mprotect VMA splitting preserve object offsets\n"); + if (p != MAP_FAILED) { + q = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + ok = q != MAP_FAILED && !munmap(q + ps, ps); + if (ok) + ok = check(q + 2 * ps, len - 2 * ps, 2 * ps) && + check(q, ps, 0); + if (q != MAP_FAILED) + munmap(q, len); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result( + ok, + "cold faults after partial unmap preserve object offsets\n"); + prime.handle = create.handle; + ok = !ioctl(fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &prime); + if (ok) { + alias = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, + prime.fd, 0); + ok = alias != MAP_FAILED && check(alias, len, 0); + } + ksft_test_result(ok, "PRIME exported full mapping shares GEM data\n"); + if (alias != MAP_FAILED) { + q = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_SHARED, prime.fd, + 3 * ps); + ok = q != MAP_FAILED; + if (ok) + ok = check(q, ps, 3 * ps); + if (q != MAP_FAILED) + munmap(q, ps); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result( + ok, "PRIME partial mapping uses requested buffer offset\n"); + if (alias != MAP_FAILED) { + q = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, + prime.fd, 0); + void *target = mmap(NULL, len, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + ok = q != MAP_FAILED && target != MAP_FAILED; + if (ok) { + void *moved = mremap(q, len, len, + MREMAP_MAYMOVE | MREMAP_FIXED, + target); + ok = moved == target; + if (ok) { + q = target; + ok = check(q, len, 0); + } + } + if (q != MAP_FAILED) + munmap(q, len); + if (target != MAP_FAILED && target != q) + munmap(target, len); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result( + ok, + "cold GEM faults after fixed mremap retain object offsets\n"); + if (alias != MAP_FAILED) { + pid_t child = fork(); + int status = 0; + if (!child) { + unsigned char *cold = mmap(NULL, len, + PROT_READ | PROT_WRITE, + MAP_SHARED, prime.fd, 0); + bool pass = cold != MAP_FAILED && check(cold, len, 0); + if (pass) + cold[0] = 0xe1; + _exit(pass ? 0 : 1); + } + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && p[0] == 0xe1; + p[0] = value(0); + } else + ok = false; + ksft_test_result( + ok, "forked GEM mapping shares bytes and retains backing\n"); + if (alias != MAP_FAILED) { + q = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, + prime.fd, len - ps); + ok = q == MAP_FAILED && errno == EINVAL; + if (q != MAP_FAILED) + munmap(q, 2 * ps); + q = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset + ps); + ok &= q == MAP_FAILED && errno == EINVAL; + if (q != MAP_FAILED) + munmap(q, ps); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result(ok, "PRIME bounds and exact GEM offset rejection\n"); + if (alias != MAP_FAILED) { + size_t targets[] = { native, 4096, 16384 }; + + ok = true; + for (unsigned int t = 0; t < 3; t++) { + size_t target = targets[t]; + pid_t child; + int status = 0; + bool pair; + + if (target > native || target == ps || + (t && target == native)) + continue; + child = fork(); + if (!child) { + char number[24], length[24], pages[24]; + + snprintf(number, sizeof(number), "%d", + prime.fd); + snprintf(length, sizeof(length), "%zu", len); + snprintf(pages, sizeof(pages), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + target == native ? 0UL : target, 0UL, + 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--shared", number, + length, pages, NULL); + _exit(4); + } + pair = child > 0 && + waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + for (size_t i = 0; i < len; i += 4096) { + pair &= p[i] == + (unsigned char)(0x80 + i / 4096); + p[i] = value(i); + } + ksft_print_msg( + "PRIME sharing %zuK -> %zuK status=%d bytes=%zu\n", + ps / 1024, target / 1024, status, len); + ok &= pair; + } + } else { + ok = false; + } + ksft_test_result( + ok, + "PRIME backing shared between native and alternative processes\n"); + struct drm_gem_close close_arg = { .handle = create.handle }; + ok = !ioctl(fd, DRM_IOCTL_GEM_CLOSE, &close_arg); + close(fd); + if (alias != MAP_FAILED) { + close(prime.fd); + ok &= check(alias, len, 0); + munmap(alias, len); + } else + ok = false; + if (p != MAP_FAILED) { + ok &= check(p, len, 0); + munmap(p, len); + } else + ok = false; + ksft_test_result( + ok, + "mappings retain object after handles and descriptors close\n"); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +int main(int argc, char **argv) +{ + bool ok = true; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--shared")) { + struct stat init, self; + int fd = atoi(argv[2]); + size_t len = strtoul(argv[3], NULL, 10); + if (stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || + init.st_ino != self.st_ino || init.st_dev != self.st_dev || + len < 65536 || len > 8 * 65536 || + getauxval(AT_PAGESZ) != strtoul(argv[4], NULL, 10)) + return 2; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + bool pass = p != MAP_FAILED && check(p, len, 0); + if (pass) + for (size_t i = 0; i < len; i += 4096) + p[i] = 0x80 + i / 4096; + if (p != MAP_FAILED) + munmap(p, len); + return pass ? 0 : 1; + } + if (argc == 3 && !strcmp(argv[1], "--test") && getppid() == 1) + return exercise(strtoul(argv[2], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + FILE *dev = fopen("/sys/class/drm/card0/dev", "r"); + unsigned maj, min; + if (!dev || fscanf(dev, "%u:%u", &maj, &min) != 2) + return 4; + fclose(dev); + if (mknod("/dev/vgem", S_IFCHR | 0600, makedev(maj, min))) + return 5; + size_t native = getauxval(AT_PAGESZ), + targets[] = { native, 4096, 16384 }; + + for (unsigned int t = 0; t < 3; t++) { + size_t target = targets[t]; + pid_t pid; + int status = 0; + + if (target > native || (t && target == native)) + continue; + pid = fork(); + if (!pid) { + char number[24]; + + snprintf(number, sizeof(number), "%zu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + target == native ? 0UL : target, 0UL, 0UL, + 0UL)) + _exit(3); + execl("/init", "/init", "--test", number, NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("# GEM child ABI=%zuK native=%zuK status=%d\n", + target / 1024, native / 1024, status); + } + printf("%s - GEM fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/granule-proc-init.c b/tools/testing/selftests/arm64/user4k/granule-proc-init.c new file mode 100644 index 0000000000000..e70f8df18e57a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-proc-init.c @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok, const char *s) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", s); + failed += !ok; +} +static long pss(void *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end; + int active = 0; + long value = -1; + if (!f) + return -1; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + active = (unsigned long)p >= start && + (unsigned long)p < end; + else if (active && sscanf(line, "Pss: %ld kB", &value) == 1) + break; + } + free(line); + fclose(f); + return value; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--peer")) { + int fd = atoi(argv[2]), ready = atoi(argv[3]), + release = atoi(argv[4]); + size_t native = strtoul(argv[5], NULL, 10); + unsigned char *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, native - 4096); + char token = 'a'; + int ok = getauxval(AT_PAGESZ) == 4096 && p != MAP_FAILED; + if (ok) { + for (unsigned i = 0; i < 4096; i++) + ok &= p[i] == 0x7b; + p[0] = 0xc9; + ok &= pss(p) == 2; + } + token = ok ? 'a' : 'x'; + if (write(ready, &token, 1) != 1) + return 3; + if (read(release, &token, 1) != 1) + return 4; + return ok ? 0 : 2; + } + if (getpid() != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + mount("proc", "/proc", "proc", 0, NULL); + size_t native = getauxval(AT_PAGESZ); + int fd = memfd_create("high-slot-pss", 0); + result(fd >= 0 && !ftruncate(fd, native), + "native file backing created"); + unsigned char *p = + mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + result(0, "parent mapping"); + goto done; + } + memset(p, 0x7b, native); + long before = pss(p); + printf("# native=%zu initial PSS=%ldKB\n", native, before); + result(before == (long)native / 1024, + "native PSS accounts every physical slot"); + int ready[2], release[2]; + if (pipe(ready) || pipe(release)) { + result(0, "pipes"); + goto done; + } + pid_t child = fork(); + int status = 0; + if (!child) { + char f[24], r[24], w[24], n[24]; + snprintf(f, sizeof(f), "%d", fd); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(w, sizeof(w), "%d", release[0]); + snprintf(n, sizeof(n), "%zu", native); + close(ready[0]); + close(release[1]); + if (prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--peer", f, r, w, n, NULL); + _exit(5); + } + close(ready[1]); + close(release[0]); + char token = 0; + int ok = child > 0 && read(ready[0], &token, 1) == 1 && token == 'a'; + result(ok && p[native - 4096] == 0xc9, + "4K peer shares the final physical slot and observes 2KB PSS"); + long during = pss(p); + printf("# parent mixed PSS=%ldKB expected=%zuKB\n", during, + native / 1024 - 2); + result(during == (long)native / 1024 - 2, + "native PSS weights the high shared slot independently"); + token = 'b'; + if (write(release[1], &token, 1) != 1) + failed++; + ok = waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status); + result(ok && pss(p) == (long)native / 1024, + "PSS returns to native size after peer releases its mapping"); + FILE *f = fopen("/proc/self/smaps_rollup", "r"); + char buf[512]; + ok = f != NULL; + if (f) { + while (fgets(buf, sizeof(buf), f)) { + } + ok &= !ferror(f); + fclose(f); + } + result(ok, "smaps_rollup reads complete native slot accounting"); + munmap(p, native); +done: + printf("GRANULE PROC %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/granule-swap-init.c b/tools/testing/selftests/arm64/user4k/granule-swap-init.c new file mode 100644 index 0000000000000..5791f7e2c0ee9 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-swap-init.c @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "swapfile-fixture.h" +#define SET_EXEC_PAGE_SIZE 0x41555001 +static unsigned char pattern(size_t i, unsigned round) +{ + return (i ^ (i >> 12) ^ (i >> 20) ^ round) & 255; +} +static int check(unsigned char *p, size_t size, unsigned round) +{ + for (size_t i = 0; i < size; i++) + if (p[i] != pattern(i, round)) { + printf("# byte=%zu got=%u expected=%u\n", i, p[i], + pattern(i, round)); + return 0; + } + return 1; +} +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "r"); + char k[80]; + unsigned long n = 0, result = 0; + if (!f) + return 0; + while (fscanf(f, "%79s %lu", k, &n) == 2) + if (!strcmp(k, "pswpout")) + result = n; + fclose(f); + return result; +} +static unsigned swapped(void *p, size_t size, size_t ps) +{ + int fd = open("/proc/self/pagemap", O_RDONLY); + uint64_t bits; + unsigned n = 0; + if (fd < 0) + return 0; + for (size_t i = 0; i < size; i += ps) + if (pread(fd, &bits, 8, ((uintptr_t)p + i) / ps * 8) == 8) + n += (bits >> 62) & 1; + close(fd); + return n; +} +static int exercise(unsigned long expected) +{ + size_t ps = getauxval(AT_PAGESZ), size = 8 * 1024 * 1024; + unsigned char *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int failed = 0; + if (ps != expected || p == MAP_FAILED) { + printf("not ok - expected ABI=%lu actual=%zu\n", expected, ps); + return 1; + } + for (unsigned round = 0; round < 4; round++) { + for (size_t i = 0; i < size; i++) + p[i] = pattern(i, round); + unsigned long before = swapouts(); + int advice = 0; + unsigned n = 0, attempts = 0; + /* PAGEOUT is advisory. Require evidence, with a bounded retry window. */ + do { + advice = madvise(p, size, MADV_PAGEOUT); + attempts++; + if (advice) + break; + usleep(20000); + n = swapped(p, size, ps); + } while (!n && attempts < 100); + unsigned long written = swapouts() - before; + int ok = !advice && written > 0 && n > 0; + printf("%s - %zuK round%u real swapped PTEs=%u/%zu writes=%lu attempts=%u\n", + ok ? "ok" : "not ok", ps / 1024, round, n, size / ps, + written, attempts); + failed += !ok; + int release[2], ready[2]; + if (pipe(release) || pipe(ready)) + return 2; + pid_t child = fork(); + int status = 0; + if (!child) { + char token; + close(release[1]); + close(ready[0]); + int ok = read(release[0], &token, 1) == 1 && + check(p, size, round); + if (ok) { + for (size_t i = 0; i < size; i += ps) + p[i] ^= 0x6d; + } + token = ok ? 'a' : 'x'; + write(ready[1], &token, 1); + _exit(ok ? 0 : 3); + } + close(release[0]); + close(ready[1]); + ok = check(p, size, round); + printf("%s - %zuK round%u disk roundtrip bytes\n", + ok ? "ok" : "not ok", ps / 1024, round); + failed += !ok; + char token = 'b'; + write(release[1], &token, 1); + ok = read(ready[0], &token, 1) == 1 && token == 'a' && + waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status) && check(p, size, round); + printf("%s - %zuK round%u forked swap-in and COW retain parent bytes\n", + ok ? "ok" : "not ok", ps / 1024, round); + failed += !ok; + close(release[1]); + close(ready[0]); + } + munmap(p, size); + return failed ? 1 : 0; +} +static int run_suite(bool nonlinear) +{ + const char *base = + nonlinear ? + "/sys/kernel/debug/kunit/arm64-user4k-nonlinear-swap/" : + "/sys/kernel/debug/kunit/arm64-user4k-swap/"; + char path[160]; + snprintf(path, sizeof(path), "%srun", base); + int fd = open(path, O_WRONLY); + if (fd < 0) + return 1; + int ok = write(fd, "1", 1) == 1; + close(fd); + snprintf(path, sizeof(path), "%sresults", base); + FILE *f = fopen(path, "r"); + char *line = NULL; + size_t cap = 0; + int completed = 0; + if (!f) + return 1; + while (getline(&line, &cap, f) > 0) { + fputs(line, stdout); + if (strstr(line, "not ok") || strstr(line, "# SKIP")) + ok = 0; + if (strstr(line, nonlinear ? + "ok 1 arm64-user4k-nonlinear-swap" : + "ok 1 arm64-user4k-swap")) + completed = 1; + } + free(line); + fclose(f); + return !(ok && completed); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && (!strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--swapon-helper"))) { + struct stat a, b; + if (getpid() == 1 || stat("/proc/1/exe", &a) || + stat("/proc/self/exe", &b) || a.st_dev != b.st_dev || + a.st_ino != b.st_ino) + return 1; + return (!strcmp(argv[1], "--swapoff-helper") ? + swapoff(swap_path()) : + swapon(swap_path(), 0)) ? + 1 : + 0; + } + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 1; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || + (argc != 1 && (argc != 2 || (strcmp(argv[1], "--el0-only") && + strcmp(argv[1], "--nonlinear") && + strcmp(argv[1], "--swapfile"))))) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + int failed = 0, active = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) { + failed++; + goto done; + } + struct stat st; + for (int i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || + (argc == 2 && !strcmp(argv[1], "--swapfile") && + prepare_swapfile()) || + swapon(swap_path(), 0)) { + perror("swapon"); + failed++; + goto done; + } + active = 1; + printf("GRANULE SWAP BACKING=%s\n", swap_path()); + if (argc == 1 || (argc == 2 && !strcmp(argv[1], "--nonlinear"))) + failed += run_suite(argc == 2); + unsigned long sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned abi = 0; abi < (sizes[0] > 16384 ? 3U : 2U); abi++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[32]; + snprintf(expected, sizeof(expected), "%lu", sizes[abi]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[abi], 0UL, 0UL, + 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = c > 0 && waitpid(c, &status, 0) == c && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - swap ABI %luK child status=%#x\n", + ok ? "ok" : "not ok", sizes[abi] / 1024, status); + failed += !ok; + } +done: + if (active && swapoff(swap_path())) { + perror("swapoff"); + failed++; + } + if (active && argc == 2 && !strcmp(argv[1], "--swapfile") && + umount("/swapdir")) { + perror("umount"); + failed++; + } + printf("GRANULE SWAP %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/granule-threeway-init.c b/tools/testing/selftests/arm64/user4k/granule-threeway-init.c new file mode 100644 index 0000000000000..71a459c9b5221 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-threeway-init.c @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +struct snapshot { + long pss; + int first, last, status; +}; +struct peer { + pid_t pid; + int command, reply; + size_t size; +}; +static int failed; +static void result(int ok, const char *s) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", s); + failed += !ok; +} +static long pss(void *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end; + int active = 0; + long value = -1; + if (!f) + return -1; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + active = (unsigned long)p >= start && + (unsigned long)p < end; + else if (active && sscanf(line, "Pss: %ld kB", &value) == 1) + break; + } + free(line); + fclose(f); + return value; +} +static int transfer(int fd, void *data, size_t size, int writing) +{ + size_t done = 0; + while (done < size) { + ssize_t n = + writing ? write(fd, (char *)data + done, size - done) : + read(fd, (char *)data + done, size - done); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + return 0; + done += n; + } + return 1; +} +static int worker(int fd, int command, int reply, size_t size) +{ + const struct rlimit no_core = { 0, 0 }; + setrlimit(RLIMIT_CORE, &no_core); + if (getppid() != 1 || getauxval(AT_PAGESZ) != size) + return 2; + unsigned char *p = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, + fd, 65536 - size); + if (p == MAP_FAILED) + return 3; + for (;;) { + char op; + struct snapshot s = { .pss = -1, .first = -1, .last = -1 }; + if (!transfer(command, &op, 1, 0)) + return 4; + if (op == 'x') { + s.status = munmap(p, size); + transfer(reply, &s, sizeof(s), 1); + return s.status ? 5 : 0; + } + if (op == 'r') + s.status = mprotect(p, size, PROT_READ); + else if (op == 'w') + p[size - 1] = 0x99; + else if (op == 'd') + s.status = madvise(p, size, MADV_DONTNEED); + else if (op == 't') { + pid_t c = fork(); + int status = 0; + if (!c) { + unsigned char value = *( + volatile unsigned char *)(p + size - 1); + _exit(value); + } + if (c < 0 || waitpid(c, &status, 0) != c) + s.status = -1; + else + s.status = WIFSIGNALED(status) ? + 256 + WTERMSIG(status) : + WIFEXITED(status) ? + WEXITSTATUS(status) : + -2; + } + if (op != 'd' && op != 't') { + s.first = p[0]; + s.last = p[size - 1]; + } + s.pss = pss(p); + if (!transfer(reply, &s, sizeof(s), 1)) + return 6; + } +} +static struct peer spawn(int fd, size_t size) +{ + int command[2], reply[2]; + struct peer p = { .pid = -1, .size = size }; + if (pipe(command) || pipe(reply)) + return p; + p.pid = fork(); + if (!p.pid) { + char f[24], c[24], r[24], s[24]; + close(command[1]); + close(reply[0]); + snprintf(f, sizeof(f), "%d", fd); + snprintf(c, sizeof(c), "%d", command[0]); + snprintf(r, sizeof(r), "%d", reply[1]); + snprintf(s, sizeof(s), "%zu", size); + if (prctl(SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL)) + _exit(7); + execl("/init", "/init", "--worker", f, c, r, s, NULL); + _exit(8); + } + close(command[0]); + close(reply[1]); + p.command = command[1]; + p.reply = reply[0]; + return p; +} +static struct snapshot command(struct peer *p, char op) +{ + struct snapshot s = { .pss = -1, .first = -1, .last = -1, .status = -1 }; + if (p->pid <= 0 || !transfer(p->command, &op, 1, 1) || + !transfer(p->reply, &s, sizeof(s), 0)) + failed++; + return s; +} +static void finish(struct peer *p) +{ + struct snapshot s = command(p, 'x'); + int status = 0; + result(!s.status && waitpid(p->pid, &status, 0) == p->pid && + WIFEXITED(status) && !WEXITSTATUS(status), + "peer unmap and exit"); + close(p->command); + close(p->reply); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), + strtoul(argv[5], NULL, 10)); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable64K VM PID1 only\n"); + return 2; + } + mount("proc", "/proc", "proc", 0, NULL); + if (getauxval(AT_PAGESZ) != 65536) { + result(0, "native64K required"); + goto done; + } + int fd = memfd_create("three-granules", 0); + if (fd < 0 || ftruncate(fd, 65536)) { + result(0, "file setup"); + goto done; + } + unsigned char *p = + mmap(NULL, 65536, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + result(0, "native mapping"); + goto done; + } + memset(p, 0x42, 65536); + struct peer medium = spawn(fd, 16384); + struct snapshot s = command(&medium, 's'); + result(s.pss == 8 && s.first == 0x42 && s.last == 0x42 && pss(p) == 56, + "64K/16K initial sharing and PSS"); + struct peer small = spawn(fd, 4096); + s = command(&small, 's'); + result(s.pss == 1 && s.first == 0x42 && s.last == 0x42, + "4K peer shares the final slice with both larger ABIs"); + s = command(&medium, 's'); + result(s.pss == 7 && pss(p) == 55, + "three-way PSS weights partial overlaps independently"); + s = command(&small, 'r'); + result(!s.status, "4K alias becomes read-only"); + s = command(&medium, 'w'); + result(!s.status && p[65535] == 0x99, + "16K alias remains writable across local4K protection"); + s = command(&small, 's'); + result(s.last == 0x99, "read-only4K alias observes16K writes"); + s = command(&small, 'd'); + result(!s.status && s.pss == 0 && pss(p) == 56, + "4K discard removes only its mapping"); + s = command(&medium, 's'); + result(s.pss == 8 && s.last == 0x99, "16K mapping survives4K discard"); + s = command(&small, 's'); + result(s.pss == 1 && s.last == 0x99 && pss(p) == 55, + "4K refault restores coherent three-way sharing"); + result(!ftruncate(fd, 65536 - 4096), + "truncate at4K boundary inside16K leaf"); + s = command(&small, 't'); + result(s.status == 256 + SIGBUS, + "4K mapping wholly beyond EOF faults SIGBUS"); + s = command(&medium, 't'); + result(s.status == 0, "partial16K EOF leaf exposes zero tail"); + result(p[65535] == 0 && p[0] == 0x42, + "native64K sees zero tail and preserved file prefix"); + s = command(&medium, 's'); + result(s.first == 0x42 && s.last == 0 && s.pss == 8 && pss(p) == 56, + "16K EOF refault keeps correct mixed PSS"); + result(!ftruncate(fd, 65536), "restore file length"); + s = command(&small, 's'); + result(s.first == 0 && s.last == 0 && s.pss == 1 && pss(p) == 55, + "4K alias refaults zeroed extension"); + finish(&small); + s = command(&medium, 's'); + result(s.pss == 8 && pss(p) == 56, "16K alias survives4K exit"); + finish(&medium); + result(pss(p) == 64, + "native PSS restored after all alternative aliases exit"); + munmap(p, 65536); + close(fd); +done: + printf("THREE GRANULES %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/iommufd-init.c b/tools/testing/selftests/arm64/user4k/iommufd-init.c new file mode 100644 index 0000000000000..c678f35cf72a4 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/iommufd-init.c @@ -0,0 +1,1314 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable VM PID1: real IOMMUFD ioctls against mock AMDv1 page tables. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../drivers/iommu/iommufd/iommufd_test.h" + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define VA 0x200000000UL +#define IOVA 0x10000000UL +#define ALIAS 0x12000000UL +#define MAP_FLAGS \ + (IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_READABLE | \ + IOMMU_IOAS_MAP_WRITEABLE) +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) + +static size_t native_ps, user_ps; +struct context { + int fd, access; + uint32_t ioas, hwpt[2], dev[2]; +}; + +static int drop_pin_cap(void) +{ + struct __user_cap_header_struct h = { + .version = _LINUX_CAPABILITY_VERSION_3 + }; + struct __user_cap_data_struct d[2]; + unsigned int mask = 1U << (CAP_IPC_LOCK % 32); + + if (syscall(SYS_capget, &h, d)) + return -1; + d[CAP_IPC_LOCK / 32].effective &= ~mask; + d[CAP_IPC_LOCK / 32].permitted &= ~mask; + return syscall(SYS_capset, &h, d); +} + +static int limit_bytes(size_t bytes) +{ + struct rlimit r; + + if (getrlimit(RLIMIT_MEMLOCK, &r)) + return -1; + r.rlim_cur = bytes; + return setrlimit(RLIMIT_MEMLOCK, &r); +} + +static long stat_kb(pid_t pid, const char *key) +{ + char path[80], line[256]; + long result = -1; + FILE *f; + + snprintf(path, sizeof(path), "/proc/%d/status", pid); + f = fopen(path, "r"); + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) + if (!strncmp(line, key, strlen(key))) { + result = strtol(line + strlen(key), NULL, 10); + break; + } + fclose(f); + return result; +} + +static int new_domain(struct context *c, unsigned int index) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_MOCK_DOMAIN, + .id = c->ioas }; + int rc = ioctl(c->fd, IOMMU_TEST_CMD, &cmd); + + if (!rc) { + c->hwpt[index] = cmd.mock_domain.out_hwpt_id; + c->dev[index] = cmd.mock_domain.out_idev_id; + } + return rc; +} + +static int context_open_common(struct context *c, unsigned int mm_account, + bool domain, bool pins) +{ + struct iommu_ioas_alloc a = { .size = sizeof(a) }; + struct iommu_option o = { .size = sizeof(o), + .option_id = IOMMU_OPTION_RLIMIT_MODE, + .op = IOMMU_OPTION_OP_SET, + .val64 = mm_account }; + struct iommu_test_cmd access = { .size = sizeof(access), + .op = IOMMU_TEST_OP_CREATE_ACCESS }; + + memset(c, 0, sizeof(*c)); + c->fd = open("/dev/iommu", O_RDWR); + if (c->fd < 0 || ioctl(c->fd, IOMMU_OPTION, &o) || + ioctl(c->fd, IOMMU_IOAS_ALLOC, &a)) + return -1; + c->ioas = a.out_ioas_id; + if (domain && new_domain(c, 0)) + return -1; + access.id = c->ioas; + access.create_access.flags = + pins ? MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES : 0; + if (ioctl(c->fd, IOMMU_TEST_CMD, &access)) + return -1; + c->access = access.create_access.out_access_fd; + return 0; +} + +static int context_open(struct context *c, unsigned int mm_account) +{ + return context_open_common(c, mm_account, true, false); +} + +static void context_close(struct context *c) +{ + close(c->access); + close(c->fd); +} + +static int map_range(struct context *c, void *p, size_t len, unsigned long iova) +{ + struct iommu_ioas_map m = { .size = sizeof(m), + .flags = MAP_FLAGS, + .ioas_id = c->ioas, + .user_va = (uintptr_t)p, + .length = len, + .iova = iova }; + + return ioctl(c->fd, IOMMU_IOAS_MAP, &m); +} + +static int unmap_range(struct context *c, size_t len, unsigned long iova) +{ + struct iommu_ioas_unmap u = { .size = sizeof(u), + .ioas_id = c->ioas, + .iova = iova, + .length = len }; + int rc = ioctl(c->fd, IOMMU_IOAS_UNMAP, &u); + + return rc ?: (u.length == len ? 0 : -1); +} + +static int check_pa(struct context *c, unsigned int dom, void *p, size_t len, + unsigned long iova) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_MD_CHECK_MAP, + .id = c->hwpt[dom], + .check_map = { .iova = iova, + .length = len, + .uptr = (uintptr_t)p } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static int access_rw(struct context *c, void *p, size_t len, unsigned long iova, + unsigned int flags) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_ACCESS_RW, + .id = c->access, + .access_rw = { .iova = iova, + .length = len, + .uptr = (uintptr_t)p, + .flags = flags } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static bool filled(const void *ptr, unsigned char byte, size_t n) +{ + const unsigned char *p = ptr; + + for (size_t i = 0; i < n; i++) + if (p[i] != byte) + return false; + return true; +} + +static void *relocated(size_t len, unsigned char byte) +{ + void *old = mmap((void *)VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, + 0); + + if (old == MAP_FAILED) + return old; + memset(old, byte, len); + size_t distance = len > 32 * native_ps ? len + native_ps : + 32 * native_ps; + + return mremap(old, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, + (void *)(VA + distance + user_ps)); +} + +static int limits_and_lifetime(unsigned int mode) +{ + struct context c; + size_t len = 2 * native_ps; + unsigned char *p = relocated(3 * native_ps, 0x31), *buf = malloc(len); + struct iommu_ioas_copy copy; + + CHECK(p != MAP_FAILED && buf); + CHECK(!context_open(&c, mode)); + CHECK(!limit_bytes(len)); + CHECK(map_range(&c, p, 3 * native_ps, IOVA) == -1 && errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!map_range(&c, p, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(stat_kb(getpid(), "VmLck:") == (mode ? (long)len / 1024 : 0)); + CHECK(!check_pa(&c, 0, p, len, IOVA)); + CHECK(map_range(&c, p + len, native_ps, ALIAS) == -1 && + errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + copy = (struct iommu_ioas_copy){ .size = sizeof(copy), + .flags = MAP_FLAGS, + .src_ioas_id = c.ioas, + .dst_ioas_id = c.ioas, + .src_iova = IOVA + user_ps, + .dst_iova = ALIAS, + .length = len - user_ps }; + CHECK(!ioctl(c.fd, IOMMU_IOAS_COPY, ©)); + CHECK(!new_domain(&c, 1)); + CHECK(!check_pa(&c, 1, p + user_ps, len - user_ps, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(mmap(p, len, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) == p); + memset(p, 0x5a, len); + CHECK(check_pa(&c, 0, p, len, IOVA) == -1 && errno == EINVAL); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x31, len)); + CHECK(!access_rw(&c, buf, len, IOVA, 0)); + CHECK(filled(buf, 0x5a, len)); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == + (long)(user_ps < native_ps ? len : len - user_ps) / 1024); + CHECK(!unmap_range(&c, len - user_ps, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + /* Fresh anonymous backing follows the shifted VA's native linear index. + * A non-native-aligned replacement spans three physical backing pages. + */ + size_t replacement_pages = user_ps < native_ps ? 3 : 2; + + if (replacement_pages == 3) { + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + } + CHECK(!limit_bytes(replacement_pages * native_ps)); + CHECK(!map_range(&c, p, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == + (long)(replacement_pages * native_ps / 1024)); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!munmap(p, 3 * native_ps)); + context_close(&c); + free(buf); + return 0; +} + +static int holes_and_shared(unsigned int mode) +{ + struct context c; + size_t len = 3 * user_ps, file_len = 4 * native_ps; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0), + *buf = malloc(file_len); + int file = memfd_create("iommu-shared", 0); + + CHECK(p != MAP_FAILED && buf && file >= 0); + CHECK(!context_open(&c, mode) && !limit_bytes(8 * native_ps)); + memset(p, 0x31, len); + CHECK(!munmap(p + user_ps, user_ps)); + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == EFAULT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(mmap(p + user_ps, user_ps, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, + 0) == p + user_ps); + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == EFAULT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!mprotect(p + user_ps, user_ps, PROT_READ | PROT_WRITE)); + CHECK(!map_range(&c, p, len, IOVA) && !check_pa(&c, 0, p, len, IOVA)); + CHECK(!unmap_range(&c, len, IOVA) && !munmap(p, len)); + CHECK(!ftruncate(file, file_len)); + p = mmap(NULL, 2 * user_ps, PROT_READ | PROT_WRITE, MAP_SHARED, file, + user_ps); + CHECK(p != MAP_FAILED); + CHECK(!map_range(&c, p, 2 * user_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, 2 * user_ps, IOVA)); + CHECK(!munmap(p, 2 * user_ps)); + memset(buf, 0x93, 2 * user_ps); + CHECK(!access_rw(&c, buf, 2 * user_ps, IOVA, + MOCK_ACCESS_RW_WRITE | MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(!unmap_range(&c, 2 * user_ps, IOVA)); + CHECK(pread(file, buf, file_len, 0) == (ssize_t)file_len); + CHECK(filled(buf, 0, user_ps)); + CHECK(filled(buf + user_ps, 0x93, 2 * user_ps)); + CHECK(filled(buf + 3 * user_ps, 0, file_len - 3 * user_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + close(file); + context_close(&c); + free(buf); + return 0; +} + +static int producer(int argc, char **argv) +{ + struct context c = { .fd = atoi(argv[4]), + .ioas = strtoul(argv[5], NULL, 0) }; + int ready = atoi(argv[6]), go = atoi(argv[7]); + char byte = 'R'; + void *p; + + (void)argc; + CHECK(!prctl(PR_SET_PDEATHSIG, SIGKILL)); + CHECK(!drop_pin_cap()); + p = relocated(2 * native_ps, 0x6d); + CHECK(p != MAP_FAILED); + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(write(ready, &byte, 1) == 1); + CHECK(read(go, &byte, 1) == 1 && byte == 'G'); + return 0; +} + +static int destroy_id(struct context *c, uint32_t id) +{ + struct iommu_destroy cmd = { .size = sizeof(cmd), .id = id }; + + return ioctl(c->fd, IOMMU_DESTROY, &cmd); +} + +static int create_viommu(struct context *c, uint32_t *id, uint32_t *parent) +{ + struct iommu_hwpt_alloc hwpt = { .size = sizeof(hwpt), + .flags = IOMMU_HWPT_ALLOC_NEST_PARENT, + .dev_id = c->dev[0], + .pt_id = c->ioas }; + struct iommu_viommu_alloc cmd = { .size = sizeof(cmd), + .type = IOMMU_VIOMMU_TYPE_SELFTEST, + .dev_id = c->dev[0] }; + + if (ioctl(c->fd, IOMMU_HWPT_ALLOC, &hwpt)) + return -1; + *parent = hwpt.out_hwpt_id; + cmd.hwpt_id = *parent; + if (ioctl(c->fd, IOMMU_VIOMMU_ALLOC, &cmd)) + return -1; + *id = cmd.out_viommu_id; + return 0; +} + +static int queue_alloc(struct context *c, uint32_t viommu, unsigned int index, + unsigned long iova, size_t length, uint32_t *id) +{ + struct iommu_hw_queue_alloc cmd = { .size = sizeof(cmd), + .viommu_id = viommu, + .type = IOMMU_HW_QUEUE_TYPE_SELFTEST, + .index = index, + .nesting_parent_iova = iova, + .length = length }; + int rc = ioctl(c->fd, IOMMU_HW_QUEUE_ALLOC, &cmd); + + if (!rc) + *id = cmd.out_hw_queue_id; + return rc; +} + +static int queue_after_exit(struct context *c) +{ + uint32_t viommu, parent, id; + + CHECK(!create_viommu(c, &viommu, &parent)); + CHECK(!queue_alloc(c, viommu, 0, IOVA + native_ps / 4 + 3, + native_ps / 4, &id)); + CHECK(unmap_range(c, 2 * native_ps, IOVA) == -1 && errno == EBUSY); + CHECK(!destroy_id(c, id)); + CHECK(!destroy_id(c, viommu)); + CHECK(!destroy_id(c, parent)); + return 0; +} + +static int queue_ranges(unsigned int mode) +{ + struct context c; + unsigned char *p = relocated(2 * native_ps, 0x71); + uint32_t viommu, parent, q0, q1, unused; + size_t offset = user_ps / 4 + 3, length = user_ps / 2 + 17; + int file; + + CHECK(p != MAP_FAILED && !context_open(&c, mode)); + CHECK(!limit_bytes(8 * native_ps) && + !create_viommu(&c, &viommu, &parent)); + /* Map only one user leaf: queue pinning must not require surrounding IOVA. */ + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 1, IOVA + offset, length, &unused) == + -1 && + errno == EIO); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + offset, length, &q0)); + CHECK(queue_alloc(&c, viommu, 0, IOVA + offset, length, &unused) == + -1 && + errno == EEXIST); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(stat_kb(getpid(), "VmPin:") == (long)native_ps / 1024); + /* Pins keep the old physical bytes after replacing the source VMA. */ + CHECK(mmap(p, user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == p); + memset(p, 0x5a, user_ps); + CHECK(!queue_alloc(&c, viommu, 1, IOVA + offset + 7, length - 7, &q1)); + CHECK(destroy_id(&c, q0) == -1 && errno == EBUSY); + CHECK(!destroy_id(&c, q1)); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(!destroy_id(&c, q0)); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(queue_alloc(&c, viommu, 0, IOVA + offset, length, &unused) == + -1 && + errno == ENOENT); + CHECK(!munmap(p, 2 * native_ps)); + + /* Separate mappings of the same file byte range cannot form a physical run. + * The second area must fail after the first area's access was registered. + */ + file = memfd_create("queue-alias", 0); + CHECK(file >= 0 && !ftruncate(file, native_ps)); + p = mmap((void *)VA, 2 * user_ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(p != MAP_FAILED); + CHECK(mmap(p, user_ps, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, + file, 0) == p); + CHECK(mmap(p + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, 0) == p + user_ps); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && + errno == ENOENT); + CHECK(!map_range(&c, p + user_ps, user_ps, IOVA + user_ps)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && + errno == EFAULT); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(!unmap_range(&c, user_ps, IOVA + user_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + /* The same discontinuity inside one provider must also release its access. */ + CHECK(!map_range(&c, p, 2 * user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && + errno == EFAULT); + CHECK(!unmap_range(&c, 2 * user_ps, IOVA)); + if (user_ps < native_ps) { + /* Adjacent file quarters form a real physical run across two areas. */ + CHECK(mmap(p + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, + user_ps) == p + user_ps); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!map_range(&c, p + user_ps, user_ps, IOVA + user_ps)); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + 11, 2 * user_ps - 22, + &q0)); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(unmap_range(&c, user_ps, IOVA + user_ps) == -1 && + errno == EBUSY); + CHECK(!destroy_id(&c, q0)); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(!unmap_range(&c, user_ps, IOVA + user_ps)); + } + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, 2 * user_ps)); + close(file); + CHECK(!destroy_id(&c, viommu) && !destroy_id(&c, parent)); + context_close(&c); + printf("ok - hardware queue byte spans %zuK/%zuK %s, " + "relocated/overlap/rollback/unmap locks\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int cross_process(unsigned int mode) +{ + struct context c; + int ready[2], go[2], status; + pid_t child; + size_t child_ps = user_ps == native_ps ? 4096 : native_ps; + size_t len = 2 * native_ps; + unsigned char *p = mmap((void *)(VA + 128 * native_ps), len, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | + MAP_FIXED_NOREPLACE, + -1, 0), + *buf = malloc(len); + char byte; + + CHECK(p != MAP_FAILED && buf && !pipe(ready) && !pipe(go)); + CHECK(!context_open(&c, mode) && !limit_bytes(len)); + child = fork(); + CHECK(child >= 0); + if (!child) { + char ps[24], native[24], fd[24], ioas[24], r[24], g[24]; + + close(ready[0]); + close(go[1]); + snprintf(ps, sizeof(ps), "%zu", child_ps); + snprintf(native, sizeof(native), "%zu", native_ps); + snprintf(fd, sizeof(fd), "%d", c.fd); + snprintf(ioas, sizeof(ioas), "%u", c.ioas); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(g, sizeof(g), "%d", go[0]); + if (prctl(SET_EXEC_PAGE_SIZE, child_ps, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--producer", ps, native, fd, ioas, r, + g, NULL); + _exit(5); + } + close(ready[1]); + close(go[0]); + CHECK(read(ready[0], &byte, 1) == 1 && byte == 'R'); + CHECK(stat_kb(child, "VmPin:") == (long)len / 1024); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + if (!mode) + CHECK(map_range(&c, p, native_ps, ALIAS) == -1 && + errno == ENOMEM); + else { + CHECK(!map_range(&c, p, native_ps, ALIAS)); + CHECK(!unmap_range(&c, native_ps, ALIAS)); + } + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + byte = 'G'; + CHECK(write(go[1], &byte, 1) == 1); + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!new_domain(&c, 1)); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + CHECK(!queue_after_exit(&c)); + if (!mode) + CHECK(map_range(&c, p, native_ps, ALIAS) == -1 && + errno == ENOMEM); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!map_range(&c, p, len, ALIAS) && !unmap_range(&c, len, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + close(ready[0]); + close(go[1]); + CHECK(!munmap(p, len)); + context_close(&c); + free(buf); + return 0; +} + +struct stress_arg { + struct context *c; + void *p; + unsigned int index; + atomic_int *failed; +}; +static void *stress_thread(void *opaque) +{ + struct stress_arg *a = opaque; + unsigned long iova = IOVA + a->index * 0x100000UL; + size_t len = 2 * native_ps; + + for (unsigned int round = 0; round < 64; round++) { + if (map_range(a->c, a->p, len, iova) || + (!(round % 16) && check_pa(a->c, 1, a->p, len, iova)) || + unmap_range(a->c, len, iova)) { + printf("FAIL concurrent map thread=%u round=%u errno=%d\n", + a->index, round, errno); + atomic_store(a->failed, 1); + break; + } + } + return NULL; +} + +static int concurrent_maps(unsigned int mode) +{ + struct context c; + pthread_t threads[4]; + struct stress_arg args[4]; + atomic_int failed = 0; + void *p = relocated(2 * native_ps, 0x47); + + CHECK(p != MAP_FAILED && !context_open(&c, mode)); + CHECK(!limit_bytes(32 * native_ps) && !new_domain(&c, 1)); + for (unsigned int i = 0; i < 4; i++) { + args[i] = (struct stress_arg){ &c, p, i, &failed }; + CHECK(!pthread_create(&threads[i], NULL, stress_thread, + &args[i])); + } + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_join(threads[i], NULL)); + CHECK(!atomic_load(&failed)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, 2 * native_ps)); + context_close(&c); + return 0; +} + +static int write_attribute(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + ssize_t n; + + if (fd < 0) + return -1; + n = write(fd, value, strlen(value)); + close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} + +static int allocation_failures(unsigned int mode) +{ + struct context c; + void *p = relocated(2 * native_ps, 0x62); + int nth = open("/proc/self/fail-nth", O_RDWR); + unsigned int hit = 0, n; + + CHECK(nth >= 0 && p != MAP_FAILED); + CHECK(!context_open(&c, mode) && !limit_bytes(8 * native_ps)); + CHECK(!new_domain(&c, 1)); + CHECK(!write_attribute("/sys/kernel/debug/failslab/ignore-gfp-wait", + "N")); + CHECK(!write_attribute( + "/sys/kernel/debug/fail_page_alloc/ignore-gfp-wait", "N")); + CHECK(!write_attribute( + "/sys/kernel/debug/fail_page_alloc/ignore-gfp-highmem", "N")); + CHECK(!write_attribute("/sys/kernel/debug/failslab/verbose", "0")); + CHECK(!write_attribute("/sys/kernel/debug/fail_page_alloc/verbose", + "0")); + CHECK(!write_attribute("/sys/kernel/debug/fail_iommufd/verbose", "0")); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n); + ssize_t got, disabled; + int rc; + + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = map_range(&c, p, 2 * native_ps, IOVA); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(disabled == 1 && got > 0); + if (!rc) + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + /* An injected failure must not poison a successful retry or translation. */ + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, 2 * native_ps, IOVA)); + CHECK(!check_pa(&c, 1, p, 2 * native_ps, IOVA)); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + printf("ok - IOMMUFD %zuK %s accounting allocation-failure sweep: %u sites\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + CHECK(!munmap(p, 2 * native_ps)); + close(nth); + context_close(&c); + return 0; +} + +static int queue_failure_sweep(unsigned int mode) +{ + struct context c; + void *p = relocated(2 * native_ps, 0x4c); + int nth = open("/proc/self/fail-nth", O_RDWR); + uint32_t viommu, parent, id; + unsigned int n, hit = 0; + + CHECK(p != MAP_FAILED && nth >= 0 && !context_open(&c, mode)); + CHECK(!limit_bytes(8 * native_ps) && + !create_viommu(&c, &viommu, &parent)); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n), rc; + ssize_t got, disabled; + + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = queue_alloc(&c, viommu, 0, IOVA + user_ps / 4 + 3, + user_ps / 2, &id); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(got > 0 && disabled == 1); + if (!rc) + CHECK(!destroy_id(&c, id)); + /* A failed queue must not leave an internal access blocking unmap. */ + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + user_ps / 4 + 3, + user_ps / 2, &id)); + CHECK(!destroy_id(&c, id) && !unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + CHECK(!destroy_id(&c, viommu) && !destroy_id(&c, parent)); + context_close(&c); + close(nth); + CHECK(!munmap(p, 2 * native_ps)); + printf("ok - hardware queue %zuK %s failure sweep: %u sites, no stale locks/pins\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + return 0; +} + +static int page_access(struct context *c, void *p, size_t len, + unsigned long iova, uint32_t *id) +{ + struct iommu_test_cmd cmd = { + .size = sizeof(cmd), + .op = IOMMU_TEST_OP_ACCESS_PAGES, + .id = c->access, + .access_pages = { .iova = iova, + .length = len, + .uptr = (uintptr_t)p, + .flags = MOCK_FLAGS_ACCESS_WRITE } + }; + int rc = ioctl(c->fd, IOMMU_TEST_CMD, &cmd); + + if (!rc) + *id = cmd.access_pages.out_access_pages_id; + return rc; +} + +static int page_release(struct context *c, uint32_t id) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_DESTROY_ACCESS_PAGES, + .id = c->access, + .destroy_access_pages = { + .access_pages_id = id } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static int page_ranges(unsigned int mode) +{ + struct context c; + int file = memfd_create("native-page-array", 0); + unsigned char *p; + uint32_t a, b, d, e; + size_t len = 3 * native_ps, tail = native_ps + user_ps / 2 + 3; + + CHECK(file >= 0 && !ftruncate(file, 5 * native_ps)); + p = mmap((void *)VA, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED && !limit_bytes(4 * native_ps)); + memset(p, 0x67, len); + CHECK(!context_open_common(&c, mode, false, true)); + /* Separate areas exercise native output stride with alternative indices. */ + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, 2 * native_ps, IOVA + native_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, len, IOVA, &a)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(!page_access(&c, p, len, IOVA, &b)); + CHECK(!page_access(&c, p + native_ps, tail, IOVA + native_ps, &d)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + /* Source VA can move without changing its retained physical offset. */ + unsigned char *moved = mremap(p, len, len, + MREMAP_MAYMOVE | MREMAP_FIXED, + (void *)(VA + 32 * native_ps + user_ps)); + CHECK(moved != MAP_FAILED); + CHECK(!page_access(&c, moved, len, IOVA, &e) && !page_release(&c, e)); + CHECK(!page_release(&c, a) && !page_release(&c, b)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)(2 * native_ps) / 1024); + CHECK(!page_release(&c, d)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(mremap(moved, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, p) == p); + CHECK(!page_access(&c, p, len, IOVA, &a)); + /* Unmap callback releases an access covering both independent areas. */ + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(page_release(&c, a) == -1 && errno == ENOENT); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, len)); + + /* The second area is absent: first area's pin/accounting must roll back. */ + p = mmap((void *)VA, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED && !map_range(&c, p, native_ps, IOVA)); + CHECK(page_access(&c, NULL, 2 * native_ps, IOVA, &a) == -1 && + errno == ENOENT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, native_ps, IOVA, &a) && !page_release(&c, a)); + CHECK(!unmap_range(&c, native_ps, IOVA) && !munmap(p, native_ps)); + + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED); + CHECK(mmap(p + native_ps, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, native_ps) == p + native_ps); + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(!page_access(&c, p, 2 * native_ps, IOVA, &a)); + CHECK(stat_kb(getpid(), "VmPin:") == + (long)((user_ps == native_ps ? 2 : 1) * native_ps) / 1024); + CHECK(!page_release(&c, a) && !unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(!munmap(p, 2 * native_ps)); + + if (user_ps < native_ps) { + /* Native-sized logical range whose physical base has a fragment offset. */ + p = mmap((void *)VA, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, user_ps); + CHECK(p != MAP_FAILED && !map_range(&c, p, native_ps, IOVA)); + CHECK(page_access(&c, p, native_ps, IOVA, &a) == -1 && + errno == EINVAL); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!unmap_range(&c, native_ps, IOVA) && + !munmap(p, native_ps)); + + /* Aligned first fragment followed by an alias, not consecutive bytes. */ + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(p != MAP_FAILED); + CHECK(mmap(p + native_ps + user_ps, user_ps, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, file, + native_ps) == p + native_ps + user_ps); + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, native_ps, + IOVA + native_ps)); + CHECK(page_access(&c, p, 2 * native_ps, IOVA, &a) == -1 && + errno == EINVAL); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, native_ps, IOVA, &a)); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(page_release(&c, a) == -1 && errno == ENOENT); + CHECK(!munmap(p, 2 * native_ps)); + } + context_close(&c); + close(file); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + printf("ok - native page arrays %zuK/%zuK %s, lazy/partial/moved/rollback/callback\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int page_lifetime(unsigned int mode) +{ + struct context c; + int ready[2], go[2], status; + pid_t child; + size_t child_ps = user_ps == native_ps ? 4096 : native_ps; + size_t len = 2 * native_ps; + unsigned char *p = mmap((void *)(VA + 128 * native_ps), len, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | + MAP_FIXED_NOREPLACE, + -1, 0), + *buf = malloc(len); + uint32_t a, b; + char byte; + + CHECK(p != MAP_FAILED && buf && !pipe(ready) && !pipe(go)); + memset(p, 0x42, len); + CHECK(!context_open_common(&c, mode, false, true) && !limit_bytes(len)); + child = fork(); + CHECK(child >= 0); + if (!child) { + char ps[24], native[24], fd[24], ioas[24], r[24], g[24]; + + close(ready[0]); + close(go[1]); + snprintf(ps, sizeof(ps), "%zu", child_ps); + snprintf(native, sizeof(native), "%zu", native_ps); + snprintf(fd, sizeof(fd), "%d", c.fd); + snprintf(ioas, sizeof(ioas), "%u", c.ioas); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(g, sizeof(g), "%d", go[0]); + if (prctl(SET_EXEC_PAGE_SIZE, child_ps, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--producer", ps, native, fd, ioas, r, + g, NULL); + _exit(5); + } + close(ready[1]); + close(go[0]); + CHECK(read(ready[0], &byte, 1) == 1 && byte == 'R'); + CHECK(stat_kb(child, "VmPin:") == 0); + CHECK(!page_access(&c, NULL, len, IOVA, &a)); + CHECK(stat_kb(child, "VmPin:") == (long)len / 1024); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + byte = 'G'; + CHECK(write(go[1], &byte, 1) == 1); + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + /* Reuse retained anonymous fragments after source mm_users reaches zero. */ + CHECK(!page_access(&c, NULL, native_ps, IOVA, &b)); + CHECK(!page_release(&c, a)); + CHECK(!access_rw(&c, buf, native_ps, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, native_ps)); + CHECK(!map_range(&c, p, len, ALIAS)); + if (!mode) + CHECK(page_access(&c, p, len, ALIAS, &a) == -1 && + errno == ENOMEM); + else + CHECK(!page_access(&c, p, len, ALIAS, &a) && + !page_release(&c, a)); + /* Final access release refunds even a dead provider's account. */ + CHECK(!page_release(&c, b) && !unmap_range(&c, len, IOVA)); + CHECK(!page_access(&c, p, len, ALIAS, &a) && !page_release(&c, a)); + CHECK(!unmap_range(&c, len, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, len)); + context_close(&c); + close(ready[0]); + close(go[1]); + free(buf); + printf("ok - native page array anonymous source %zuK -> %zuK consumer, %s exit/refund\n", + child_ps / 1024, user_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int page_failure_sweep(unsigned int mode) +{ + struct context c; + int file = memfd_create("page-array-failures", 0); + int nth = open("/proc/self/fail-nth", O_RDWR); + unsigned char *p; + uint32_t id; + unsigned int n, hit = 0; + + CHECK(file >= 0 && nth >= 0 && !ftruncate(file, 2 * native_ps)); + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(p != MAP_FAILED); + memset(p, 0x53, 2 * native_ps); + CHECK(!context_open_common(&c, mode, false, true) && + !limit_bytes(2 * native_ps)); + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, native_ps, IOVA + native_ps)); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n), rc; + ssize_t got, disabled; + + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = page_access(&c, p, 2 * native_ps, IOVA, &id); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(got > 0 && disabled == 1); + if (!rc) + CHECK(!page_release(&c, id)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!page_access(&c, p, 2 * native_ps, IOVA, &id) && + !page_release(&c, id)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA) && + !munmap(p, 2 * native_ps)); + context_close(&c); + close(file); + close(nth); + printf("ok - native page arrays %zuK %s failure sweep: %u sites, refunds/retries\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + return 0; +} + +struct page_race { + struct context *c; + void *p; + pthread_barrier_t barrier; + atomic_int failed, pinned, absent; +}; + +static void *page_pin_thread(void *opaque) +{ + struct page_race *r = opaque; + + pthread_barrier_wait(&r->barrier); + for (unsigned int i = 0; i < 256; i++) { + uint32_t id; + + if (page_access(r->c, r->p, 2 * native_ps, IOVA, &id)) { + /* Unmap can mark the area inaccessible before notifying drivers. */ + if (errno != ENOENT && errno != EINVAL) + goto fail; + atomic_fetch_add(&r->absent, 1); + } else { + atomic_fetch_add(&r->pinned, 1); + sched_yield(); + if (page_release(r->c, id) && errno != ENOENT) + goto fail; + } + } + return NULL; +fail: + printf("FAIL page pin/unmap race errno=%d\n", errno); + atomic_store(&r->failed, 1); + return NULL; +} + +static void *page_unmap_thread(void *opaque) +{ + struct page_race *r = opaque; + + pthread_barrier_wait(&r->barrier); + for (unsigned int i = 0; i < 256; i++) { + if (unmap_range(r->c, 2 * native_ps, IOVA)) + goto fail; + sched_yield(); + if (map_range(r->c, r->p, 2 * native_ps, IOVA)) + goto fail; + } + return NULL; +fail: + printf("FAIL page unmap/remap race errno=%d\n", errno); + atomic_store(&r->failed, 1); + return NULL; +} + +static int page_concurrency(unsigned int mode) +{ + struct context c; + struct page_race r = { .c = &c }; + pthread_t threads[4]; + int file = memfd_create("page-array-race", 0); + + CHECK(file >= 0 && !ftruncate(file, 2 * native_ps)); + r.p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(r.p != MAP_FAILED); + memset(r.p, 0x49, 2 * native_ps); + CHECK(!context_open_common(&c, mode, false, true) && + !limit_bytes(8 * native_ps)); + CHECK(!map_range(&c, r.p, 2 * native_ps, IOVA)); + CHECK(!pthread_barrier_init(&r.barrier, NULL, 4)); + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_create( + &threads[i], NULL, + i == 3 ? page_unmap_thread : page_pin_thread, &r)); + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_join(threads[i], NULL)); + CHECK(!atomic_load(&r.failed) && atomic_load(&r.pinned) > 0); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(r.p, 2 * native_ps)); + context_close(&c); + close(file); + CHECK(!pthread_barrier_destroy(&r.barrier)); + printf("ok - native page arrays %zuK %s race: %d pins, %d unavailable, 256 remaps\n", + user_ps / 1024, mode ? "MM" : "USER", atomic_load(&r.pinned), + atomic_load(&r.absent)); + return 0; +} + +static int dirty_ioctl(struct context *c, uint32_t hwpt, void *bitmap, + unsigned long iova, size_t length, size_t granule, + unsigned int flags) +{ + struct iommu_hwpt_get_dirty_bitmap cmd = { .size = sizeof(cmd), + .hwpt_id = hwpt, + .flags = flags, + .iova = iova, + .length = length, + .page_size = granule, + .data = (uintptr_t)bitmap }; + + return ioctl(c->fd, IOMMU_HWPT_GET_DIRTY_BITMAP, &cmd); +} + +static int dirty_errors(unsigned int mode) +{ + struct context c; + size_t len = 66 * native_ps; + unsigned char *data = relocated(len, 0x34); + unsigned char *out = mmap(NULL, 2 * user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t *bitmap = (void *)(out + user_ps - 8), seed[2] = { 1, 2 }; + struct iommu_option option = { .size = sizeof(option), + .option_id = IOMMU_OPTION_HUGE_PAGES, + .op = IOMMU_OPTION_OP_SET, + .val64 = 0 }; + struct iommu_hwpt_alloc alloc = { + .size = sizeof(alloc), .flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING + }; + struct iommu_hwpt_set_dirty_tracking enable = { + .size = sizeof(enable), + .flags = IOMMU_HWPT_DIRTY_TRACKING_ENABLE + }; + struct iommu_test_cmd mark = { .size = sizeof(mark), + .op = IOMMU_TEST_OP_DIRTY, + .dirty = { .iova = IOVA, + .length = len, + .page_size = native_ps, + .uptr = (uintptr_t)seed } }; + unsigned int no_clear = IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR; + int failures = 0, rc, error; + + CHECK(data != MAP_FAILED && out != MAP_FAILED); + CHECK(!context_open(&c, mode) && !limit_bytes(128 * native_ps)); + option.object_id = c.ioas; + CHECK(!ioctl(c.fd, IOMMU_OPTION, &option)); + CHECK(!map_range(&c, data, len, IOVA)); + alloc.dev_id = c.dev[0]; + alloc.pt_id = c.ioas; + CHECK(!ioctl(c.fd, IOMMU_HWPT_ALLOC, &alloc)); + enable.hwpt_id = alloc.out_hwpt_id; + CHECK(!ioctl(c.fd, IOMMU_HWPT_SET_DIRTY_TRACKING, &enable)); + mark.id = alloc.out_hwpt_id; + CHECK(!ioctl(c.fd, IOMMU_TEST_CMD, &mark) && + mark.dirty.out_nr_dirty == 2); + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, + no_clear)); + CHECK(bitmap[0] == 1 && bitmap[1] == 2); + CHECK(filled(out, 0, user_ps - 8) && + filled(out + user_ps + 8, 0, user_ps - 8)); + +#define EXPECT_DIRTY_ERROR(call, expected, label) \ + do { \ + errno = 0; \ + rc = (call); \ + error = errno; \ + if (rc != -1 || error != (expected)) { \ + printf("FAIL dirty %s: rc=%d errno=%d expected=%d\n", \ + label, rc, error, expected); \ + failures++; \ + } \ + } while (0) + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, + IOVA + len, native_ps, native_ps, + no_clear), + EINVAL, "unmapped IOVA"); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, (void *)1, IOVA, + len, native_ps, no_clear), + EFAULT, "invalid output pointer"); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, + 4 * native_ps, 3 * native_ps, no_clear), + EINVAL, "non-power-of-two granule"); + CHECK(!mprotect(out, 2 * user_ps, PROT_READ)); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, + native_ps, no_clear), + EFAULT, "read-only output"); + CHECK(!mprotect(out, 2 * user_ps, PROT_READ | PROT_WRITE)); + memset(out, 0, 2 * user_ps); + CHECK(!munmap(out + user_ps, user_ps)); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, + native_ps, no_clear), + EFAULT, "output hole after valid prefix"); + CHECK(bitmap[0] == 1); + CHECK(mmap(out + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, + 0) == out + user_ps); + /* NO_CLEAR errors retain dirty state, so fixing the buffer permits retry. */ + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, + 0)); + CHECK(bitmap[0] == 1 && bitmap[1] == 2); + memset(out, 0, 2 * user_ps); + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, + no_clear)); + CHECK(bitmap[0] == 0 && bitmap[1] == 0); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!munmap(data, len) && !munmap(out, 2 * user_ps)); + context_close(&c); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!failures); +#undef EXPECT_DIRTY_ERROR + return 0; +} + +static int exercise(void) +{ + CHECK(!drop_pin_cap()); + for (unsigned int mode = 0; mode < 2; mode++) { + CHECK(!limits_and_lifetime(mode)); + CHECK(!holes_and_shared(mode)); + CHECK(!queue_ranges(mode)); + CHECK(!page_ranges(mode)); + CHECK(!page_lifetime(mode)); + CHECK(!cross_process(mode)); + CHECK(!concurrent_maps(mode)); + CHECK(!allocation_failures(mode)); + CHECK(!queue_failure_sweep(mode)); + CHECK(!page_failure_sweep(mode)); + CHECK(!page_concurrency(mode)); + CHECK(!dirty_errors(mode)); + printf("ok - IOMMUFD %zuK userspace/%zuK kernel %s accounting, limits/lifetime/shared/cross-process/256 concurrent cycles\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + } + return 0; +} + +int main(int argc, char **argv) +{ + int fail = 0; + + setbuf(stdout, NULL); + if (argc >= 4) { + user_ps = strtoul(argv[2], NULL, 0); + native_ps = strtoul(argv[3], NULL, 0); + CHECK(getauxval(AT_PAGESZ) == user_ps); + if (!strcmp(argv[1], "--exercise")) + return exercise(); + if (!strcmp(argv[1], "--producer") && argc == 8) + return producer(argc, argv); + return 2; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)); + CHECK(!mount("sysfs", "/sys", "sysfs", 0, NULL)); + CHECK(!mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)); + native_ps = getauxval(AT_PAGESZ); + size_t sizes[] = { native_ps, 4096, 16384 }; + + for (unsigned int i = 0; i < (native_ps > 16384 ? 3U : 2U); i++) { + pid_t child = fork(); + int status = 0, ok = 0; + + if (!child) { + char size[24], native[24]; + + snprintf(size, sizeof(size), "%zu", sizes[i]); + snprintf(native, sizeof(native), "%zu", native_ps); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", size, native, + NULL); + _exit(5); + } + for (unsigned int ticks = 0; child > 0 && ticks < 1800; + ticks++) { + if (waitpid(child, &status, WNOHANG) == child) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 1799) { + kill(child, SIGKILL); + waitpid(child, &status, 0); + } + usleep(100000); + } + printf("%s - IOMMUFD ABI %zuK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } + printf("IOMMUFD IOCTL CONTRACT %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c b/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c new file mode 100644 index 0000000000000..b652200c38b42 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static unsigned long stat_value(const char *path, const char *key) +{ + char line[256], name[100]; + unsigned long value = 0; + FILE *f = fopen(path, "re"); + + if (!f) + return 0; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "%99s %lu", name, &value) == 2 && + !strcmp(name, key)) + break; + fclose(f); + return value; +} + +static bool all_bytes(const unsigned char *p, size_t len, unsigned char value) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value) + return false; + return true; +} + +static unsigned long lazy_kb(void *address) +{ + FILE *f = fopen("/proc/self/smaps", "re"); + char line[256]; + unsigned long first, last, value = ~0UL; + bool found = false; + + if (!f) + return value; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &first, &last) == 2) { + if (found) + break; + found = (uintptr_t)address >= first && + (uintptr_t)address < last; + } else if (found && + sscanf(line, "LazyFree: %lu", &value) == 1) { + break; + } + } + fclose(f); + return value; +} + +static uint64_t pagemap(void *p, size_t ps) +{ + uint64_t entry = 0; + int fd = open("/proc/self/pagemap", O_RDONLY); + + if (fd < 0 || + pread(fd, &entry, sizeof(entry), (uintptr_t)p / ps * 8) != 8) + ksft_exit_fail_msg("pagemap: %s\n", strerror(errno)); + close(fd); + return entry; +} + +static bool pageout(void *p, size_t len) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(i, &one); + ok &= !sched_setaffinity(0, sizeof(one), &one); + ok &= !madvise(p, len, MADV_COLD); + ok &= !madvise(p, len, MADV_PAGEOUT); + } + return !sched_setaffinity(0, sizeof(allowed), &allowed) && ok; +} + +static bool mark_free(void *p, size_t len) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(i, &one); + ok &= !sched_setaffinity(0, sizeof(one), &one); + ok &= !madvise(p, len, MADV_FREE); + ok &= !madvise(p, len, MADV_COLD); + } + return !sched_setaffinity(0, sizeof(allowed), &allowed) && ok; +} + +static int exercise(unsigned long native) +{ + const size_t ps = getauxval(AT_PAGESZ), len = 4 * ps; + unsigned char *reservation = mmap(NULL, len + 2 * native, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *p = + (void *)(((uintptr_t)reservation + native - 1) & ~(native - 1)); + bool ok; + unsigned long writes; + uint64_t entry; + + if (reservation == MAP_FAILED || + mprotect(p, len, PROT_READ | PROT_WRITE)) + ksft_exit_fail_msg("mapping\n"); + ksft_print_header(); + ksft_set_plan(8); + for (unsigned int i = 0; i < 4; i++) + memset(p + i * ps, 0x40 + i, ps); + ok = mark_free(p + ps, ps); + if (!ok) + ksft_exit_fail_msg("%zuK MADV_FREE: %s\n", ps / 1024, + strerror(errno)); + ksft_test_result(all_bytes(p + ps, ps, 0x41), + "%zuK FREE retains bytes before reclaim\n", ps / 1024); + ksft_test_result(lazy_kb(p) == ps / 1024, + "%zuK partial LazyFree accounting\n", ps / 1024); + ok = pageout(p, len); + entry = pagemap(p + ps, ps); + ok &= !(entry & ((1ULL << 63) | (1ULL << 62))); + ok &= all_bytes(p + ps, ps, 0) && all_bytes(p, ps, 0x40) && + all_bytes(p + 2 * ps, ps, 0x42) && + all_bytes(p + 3 * ps, ps, 0x43); + ksft_test_result(ok, "%zuK partial FREE reclaims only advised bytes\n", + ps / 1024); + /* Drop old swap/cache state, then test cancellation by a real EL0 write. */ + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0x62, len); + ok &= mark_free(p + ps, ps); + memset(p + ps, 0xa5, ps); + ksft_test_result(lazy_kb(p) == 0, + "%zuK dirty leaf no longer reported LazyFree\n", + ps / 1024); + ok &= pageout(p, len); + ok &= all_bytes(p + ps, ps, 0xa5) && all_bytes(p, ps, 0x62) && + all_bytes(p + 2 * ps, 2 * ps, 0x62); + ksft_test_result(ok, "%zuK write after FREE cancels quarter discard\n", + ps / 1024); + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0x83, len); + writes = stat_value("/proc/vmstat", "pswpout"); + ok &= mark_free(p, len) && pageout(p, len); + for (unsigned int i = 0; i < 4; i++) + ok &= !(pagemap(p + i * ps, ps) & + ((1ULL << 63) | (1ULL << 62))); + printf("# full FREE pswpout=%lu->%lu, entries=%#llx %#llx %#llx %#llx\n", + writes, stat_value("/proc/vmstat", "pswpout"), + (unsigned long long)pagemap(p, ps), + (unsigned long long)pagemap(p + ps, ps), + (unsigned long long)pagemap(p + 2 * ps, ps), + (unsigned long long)pagemap(p + 3 * ps, ps)); + ok &= stat_value("/proc/vmstat", "pswpout") == writes && + all_bytes(p, len, 0); + ksft_test_result( + ok, "%zuK fully free backing reclaimed without swap writes\n", + ps / 1024); + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0xb7, len); + ok &= pageout(p, len); + ok &= !!(pagemap(p + ps, ps) & (1ULL << 62)); + ok &= mark_free(p + ps, ps); + ok &= !(pagemap(p + ps, ps) & ((1ULL << 63) | (1ULL << 62))); + ok &= all_bytes(p + ps, ps, 0) && all_bytes(p, ps, 0xb7) && + all_bytes(p + 2 * ps, 2 * ps, 0xb7); + ksft_test_result( + ok, + "%zuK FREE drops selected swap entry without neighbor loss\n", + ps / 1024); + { + int gate[2], status = 0; + pid_t pid; + char ready; + + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0xc8, len); + if (pipe(gate)) + ksft_exit_fail_msg("pipe\n"); + pid = fork(); + if (!pid) { + close(gate[1]); + if (read(gate[0], &ready, 1) != 1 || + !all_bytes(p, len, 0xc8)) + _exit(2); + _exit(0); + } + close(gate[0]); + ok &= mark_free(p + ps, ps) && pageout(p, len); + ok &= write(gate[1], "x", 1) == 1; + close(gate[1]); + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + ksft_test_result( + ok, + "%zuK FREE after fork preserves other process data\n", + ps / 1024); + } + munmap(reservation, len + 2 * native); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(strtoul(argv[3], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + swapon("/dev/vda", 0)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + char size[32], host_size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + snprintf(host_size, sizeof(host_size), "%lu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", size, host_size, + NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + printf("%s - lazyfree fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c b/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c new file mode 100644 index 0000000000000..f34602bb730e5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c @@ -0,0 +1,153 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/account" +#define LENGTH (32UL * 1024 * 1024) +static int put(const char *p, const char *v) +{ + int fd = open(p, O_WRONLY); + if (fd < 0) + return 1; + int bad = write(fd, v, strlen(v)) != (ssize_t)strlen(v); + close(fd); + return bad; +} +static unsigned long long stat_value(const char *key) +{ + FILE *f = fopen(CG "/memory.stat", "r"); + char name[128]; + unsigned long long value = 0; + if (!f) + exit(20); + while (fscanf(f, "%127s %llu", name, &value) == 2) + if (!strcmp(key, name)) { + fclose(f); + return value; + } + fclose(f); + exit(21); +} +static int worker(unsigned long expected, int ready, int release) +{ + if (getauxval(AT_PAGESZ) != expected) + return 2; + unsigned char *p = mmap(NULL, LENGTH, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || madvise(p, LENGTH, MADV_NOHUGEPAGE)) + return 3; + memset(p, 0x71, LENGTH); + char token = 'R'; + if (write(ready, &token, 1) != 1 || read(release, &token, 1) != 1) + return 4; + /* Keep the writes observable independently of the accounting check. */ + for (size_t i = 0; i < LENGTH; i++) + if (p[i] != 0x71) + return 5; + return munmap(p, LENGTH) ? 6 : 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (getppid() == 1 && argc == 5 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), atoi(argv[3]), + atoi(argv[4])); + if (getpid() != 1 || argc != 1) + return 2; + signal(SIGPIPE, SIG_IGN); + alarm(60); + int failed = 0; + unsigned long long slab[3] = {}; + unsigned long sizes[3] = { getauxval(AT_PAGESZ), 4096, 16384 }; + unsigned int modes = sizes[0] == 65536 ? 3 : 2; + if ((sizes[0] != 16384 && sizes[0] != 65536) || + mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) || + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory")) { + perror("setup"); + failed = 1; + goto done; + } + for (unsigned i = 0; i < modes; i++) { + int ready[2], release[2]; + if (mkdir(CG, 0755) || put(CG "/memory.max", "134217728") || + pipe(ready) || pipe(release)) { + failed = 1; + break; + } + pid_t child = fork(); + if (!child) { + char ps[32], rfd[16], cfd[16]; + close(ready[0]); + close(release[1]); + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(rfd, sizeof(rfd), "%d", ready[1]); + snprintf(cfd, sizeof(cfd), "%d", release[0]); + if (put(CG "/cgroup.procs", "0") || + prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(7); + execl("/init", "/init", "--worker", ps, rfd, cfd, NULL); + _exit(8); + } + close(ready[1]); + close(release[0]); + char token; + int ok = child > 0 && read(ready[0], &token, 1) == 1; + if (ok) { + /* Let per-CPU slab statistics become visible before the comparison. */ + usleep(1200000); + slab[i] = stat_value("slab_unreclaimable"); + printf("# ABI=%lu anon=%llu slab_unreclaimable=%llu pagetables=%llu\n", + sizes[i], stat_value("anon"), slab[i], + stat_value("pagetables")); + ok = stat_value("anon") >= LENGTH && + write(release[1], "G", 1) == 1; + } + int status = 0; + if (child > 0 && !ok) + kill(child, SIGKILL); + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && ok; + close(ready[0]); + close(release[1]); + printf("%s - memcg accounting ABI %luK worker status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + for (unsigned retry = 0; retry < 100 && stat_value("anon"); + retry++) + usleep(20000); + if (stat_value("anon") || rmdir(CG)) + failed = 1; + if (failed) + break; + } + /* Slot metadata must be charged for every alternative granule. The + * loose lower bound tolerates slab/KASAN layout differences. */ + for (unsigned i = 1; !failed && i < modes; i++) { + if (slab[i] < slab[0] + 512 * 1024) { + printf("not ok - slot metadata escaped cgroup accounting ABI=%lu native=%llu alternative=%llu\n", + sizes[i], slab[0], slab[i]); + failed = 1; + } + } + +done: + printf("GRANULE MEMCG ACCOUNTING %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c b/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c new file mode 100644 index 0000000000000..c2f50993ff805 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c @@ -0,0 +1,417 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/pressure" +#define MIB (1024UL * 1024) +#define LENGTH (32 * MIB) +#define ROUNDS 4 + +static int put(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + ssize_t ret = write(fd, value, strlen(value)); + int saved = errno; + close(fd); + errno = saved; + return ret == (ssize_t)strlen(value) ? 0 : -1; +} + +static unsigned long long get(const char *path, const char *key) +{ + FILE *f = fopen(path, "r"); + char name[128]; + unsigned long long value = 0, result = 0; + if (!f) { + perror(path); + exit(10); + } + if (!key) { + if (fscanf(f, "%llu", &result) != 1) + exit(11); + } else + while (fscanf(f, "%127s %llu", name, &value) == 2) + if (!strcmp(name, key)) { + result = value; + break; + } + fclose(f); + return result; +} + +static unsigned char pattern(size_t offset, unsigned seed, unsigned round) +{ + return (offset ^ (offset >> 12) ^ (offset >> 20) ^ (seed * 79) ^ + (round * 43)) & + 255; +} + +static int contents(unsigned char *p, size_t ps, unsigned seed, unsigned round, + int write_data, int child) +{ + size_t pages = LENGTH / ps; + /* Odd multiplication permutes every logical page of this power-of-two range. */ + for (size_t i = 0; i < pages; i++) { + size_t page = (i * 4051 + round * 31) & (pages - 1); + for (size_t j = 0; j < ps; j++) { + size_t offset = page * ps + j; + unsigned char want = pattern(offset, seed, round); + if (child && !(page & 3)) + want ^= 0x69; + if (write_data) + p[offset] = want; + else if (p[offset] != want) { + printf("not ok - ABI=%zu round=%u child=%d byte=%zu got=%u expected=%u\n", + ps, round, child, offset, p[offset], + want); + return 1; + } + } + } + return 0; +} + +static int barrier(int ready, int command) +{ + char token = 'R'; + return write(ready, &token, 1) != 1 || read(command, &token, 1) != 1 || + token != 'G'; +} + +static int worker(unsigned long expected, unsigned seed, int ready, int command) +{ + size_t ps = getauxval(AT_PAGESZ); + unsigned char *p; + if (ps != expected) + return 2; + p = mmap(NULL, LENGTH, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 3; + if (madvise(p, LENGTH, MADV_NOHUGEPAGE)) { + int error = errno; + struct stat st; + + /* Without THP this advice is unavailable and cannot change policy. */ + if (error != EINVAL || + !stat("/sys/kernel/mm/transparent_hugepage/enabled", &st) || + errno != ENOENT) + return 3; + } + if (contents(p, ps, seed, 0, 1, 0) || barrier(ready, command)) + return 4; + for (unsigned round = 0; round < ROUNDS; round++) { + if (contents(p, ps, seed, round, 0, 0)) + return 5; + pid_t child = fork(); + if (child < 0) + return 6; + if (!child) { + /* Dirty a quarter of the logical pages, retaining untouched siblings. */ + for (size_t page = 0; page < LENGTH / ps; page += 4) + for (size_t j = 0; j < ps; j++) + p[page * ps + j] ^= 0x69; + _exit(contents(p, ps, seed, round, 0, 1) ? 7 : 0); + } + int status; + if (waitpid(child, &status, 0) != child || !WIFEXITED(status) || + WEXITSTATUS(status) || contents(p, ps, seed, round, 0, 0)) + return 8; + printf("ok - memcg ABI %zuK round=%u swap-in bytes and fork COW isolation\n", + ps / 1024, round); + if (round + 1 < ROUNDS && + contents(p, ps, seed, round + 1, 1, 0)) + return 9; + if (barrier(ready, command)) + return 10; + } + return munmap(p, LENGTH) ? 11 : 0; +} + +static void report(unsigned phase) +{ + printf("# memcg phase=%u resident=%llu swap=%llu anon=%llu pgscan=%llu pgsteal=%llu max=%llu oom=%llu oom_kill=%llu\n", + phase, get(CG "/memory.current", NULL), + get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "anon"), get(CG "/memory.stat", "pgscan"), + get(CG "/memory.stat", "pgsteal"), + get(CG "/memory.events", "max"), get(CG "/memory.events", "oom"), + get(CG "/memory.events", "oom_kill")); +} + +static void pressure_stacks(void) +{ + FILE *tasks = fopen(CG "/cgroup.procs", "r"); + int pid; + + if (!tasks) + return; + while (fscanf(tasks, "%d", &pid) == 1) { + char path[64], stack[4096]; + int fd; + ssize_t n; + + snprintf(path, sizeof(path), "/proc/%d/stack", pid); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + continue; + n = read(fd, stack, sizeof(stack) - 1); + close(fd); + if (n > 0) { + stack[n] = 0; + printf("# pressure stack pid=%d\n%s", pid, stack); + } + } + fclose(tasks); +} + +static void observe_pressure(void) +{ + unsigned int samples = 0; + + for (;;) { + struct timespec now; + + clock_gettime(CLOCK_MONOTONIC, &now); + printf("# io-pressure ms=%lld resident=%llu swap=%llu writeback=%llu " + "swapcached=%llu scan=%llu steal=%llu max=%llu oom=%llu kill=%llu\n", + (long long)now.tv_sec * 1000 + now.tv_nsec / 1000000, + get(CG "/memory.current", NULL), + get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "file_writeback"), + get(CG "/memory.stat", "swapcached"), + get(CG "/memory.stat", "pgscan"), + get(CG "/memory.stat", "pgsteal"), + get(CG "/memory.events", "max"), + get(CG "/memory.events", "oom"), + get(CG "/memory.events", "oom_kill")); + if (!(++samples % 8) && + get(CG "/memory.stat", "file_writeback") > 8 * MIB) + pressure_stacks(); + usleep(250000); + } +} + +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker") && getppid() == 1) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4]), + atoi(argv[5])); + bool classic = false, native_control = false, user16k = false; + + if (getpid() != 1 || argc > 3) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--classic")) + classic = true; + else if (!strcmp(argv[i], "--native-control")) + native_control = true; + else if (!strcmp(argv[i], "--user16k")) + user16k = true; + else + return 2; + } + if (native_control && user16k) + return 2; + signal(SIGPIPE, SIG_IGN); + alarm(300); + int failed = 0, active = 0; + pid_t children[2] = { -1, -1 }, observer = -1; + int ready[2][2], command[2][2]; + unsigned long sizes[2] = { getauxval(AT_PAGESZ), 4096 }; + if (native_control) + sizes[1] = sizes[0]; + if (user16k) + sizes[1] = 16384; + if ((sizes[0] != 16384 && sizes[0] != 65536) || sizes[1] > sizes[0]) { + failed = 1; + goto done; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) || + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory") || + mkdir(CG, 0755) || put(CG "/memory.max", "50331648") || + put(CG "/memory.swap.max", "268435456")) { + perror("cgroup setup"); + failed = 1; + goto done; + } + /* Explicit diagnostic control, never change the production default. */ + if (classic && put("/sys/kernel/mm/lru_gen/enabled", "0")) { + perror("classic reclaim control"); + failed = 1; + goto done; + } + printf("# reclaim mode=%s\n", + classic ? "classic diagnostic" : "default"); + struct stat st; + for (int i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (swapon("/dev/vda", 0)) { + perror("swapon"); + failed = 1; + goto done; + } + active = 1; + for (unsigned i = 0; i < 2; i++) + if (pipe(ready[i]) || pipe(command[i])) { + failed = 1; + goto done; + } + observer = fork(); + if (!observer) { + for (unsigned int i = 0; i < 2; i++) { + close(ready[i][0]); + close(ready[i][1]); + close(command[i][0]); + close(command[i][1]); + } + observe_pressure(); + _exit(0); + } + if (observer < 0) { + failed = 1; + goto done; + } + for (unsigned i = 0; i < 2; i++) { + children[i] = fork(); + if (!children[i]) { + char ps[32], index[16], rfd[16], cfd[16]; + for (unsigned j = 0; j < 2; j++) { + close(ready[j][0]); + close(command[j][1]); + if (i != j) { + close(ready[j][1]); + close(command[j][0]); + } + } + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(index, sizeof(index), "%u", i); + snprintf(rfd, sizeof(rfd), "%d", ready[i][1]); + snprintf(cfd, sizeof(cfd), "%d", command[i][0]); + if (put(CG "/cgroup.procs", "0") || + prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(12); + execl("/init", "/init", "--worker", ps, index, rfd, cfd, + NULL); + _exit(13); + } + if (children[i] < 0) { + failed = 1; + goto done; + } + } + for (unsigned i = 0; i < 2; i++) { + close(ready[i][1]); + close(command[i][0]); + } + for (unsigned phase = 0; phase <= ROUNDS; phase++) { + char token; + for (unsigned i = 0; i < 2; i++) + if (read(ready[i][0], &token, 1) != 1 || token != 'R') { + failed = 1; + goto done; + } + report(phase); + if (!get(CG "/memory.swap.current", NULL) || + !get(CG "/memory.events", "max") || + !get(CG "/memory.stat", "pgscan") || + !get(CG "/memory.stat", "pgsteal") || + get(CG "/memory.events", "oom") || + get(CG "/memory.events", "oom_kill")) { + printf("not ok - missing pressure evidence or unexpected OOM\n"); + failed = 1; + goto done; + } + for (unsigned i = 0; i < 2; i++) + if (write(command[i][1], "G", 1) != 1) { + failed = 1; + goto done; + } + } + for (unsigned i = 0; i < 2; i++) { + int status = 0; + int ok = waitpid(children[i], &status, 0) == children[i] && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - memcg ABI %luK exit=%#x\n", ok ? "ok" : "not ok", + sizes[i] / 1024, status); + children[i] = -1; + failed |= !ok; + } + /* Pending swap writeback can outlive exit, especially on a slow device. */ + for (unsigned i = 0; i < 3000 && (get(CG "/memory.stat", "anon") || + get(CG "/memory.swap.current", NULL)); + i++) { + if (!(i % 250)) + printf("# exit drain ms=%u swap=%llu writeback=%llu\n", + i * 20, get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "file_writeback")); + /* Exit may leave clean, unmapped swap cache after writeback. Reclaim + * it explicitly; EAGAIN means fewer than the requested bytes existed. */ + if (!get(CG "/memory.stat", "file_writeback") && + put(CG "/memory.reclaim", "67108864 swappiness=200") && + errno != EAGAIN) { + perror("exit cache reclaim"); + failed = 1; + break; + } + usleep(20000); + } + report(ROUNDS + 1); + if (get(CG "/memory.stat", "anon") || + get(CG "/memory.swap.current", NULL)) { + printf("not ok - anonymous or swap charge remains after exit\n"); + failed = 1; + } +done: + if (observer > 0) { + kill(observer, SIGTERM); + waitpid(observer, NULL, 0); + } + if (active && failed) + report(ROUNDS + 2); + for (unsigned i = 0; i < 2; i++) + if (children[i] > 0) { + kill(children[i], SIGKILL); + waitpid(children[i], NULL, 0); + } + if (active && failed) { + put(CG "/cgroup.kill", "1"); + put(CG "/memory.max", "max"); + } + if (active && swapoff("/dev/vda")) { + perror("swapoff"); + failed = 1; + } + printf("GRANULE MEMCG PRESSURE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/memory-interfaces.c b/tools/testing/selftests/arm64/user4k/memory-interfaces.c new file mode 100644 index 0000000000000..330296ed44499 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memory-interfaces.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static size_t page; + +static void residency(void) +{ + unsigned char *p, vec[6]; + bool ok; + unsigned int i; + + p = mmap(NULL, 5 * page, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap\n"); + memset(vec, 0x55, sizeof(vec)); + ok = mincore(p, 5 * page - 1, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == 0; + ok &= vec[5] == 0x55; + ksft_test_result( + ok, "mincore emits exactly one byte per unfaulted user page\n"); + ok = mprotect(p + 2 * page, page, PROT_READ | PROT_WRITE) == 0; + p[2 * page] = 0x37; + ok &= mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == (i == 2); + ksft_test_result( + ok, + "mincore reports a separately faulted VMA among cold neighbors\n"); + ok = munmap(p + 3 * page, page) == 0; + errno = 0; + ok &= mincore(p, 5 * page, vec) == -1 && errno == ENOMEM; + errno = 0; + ok &= mincore(p + 1, page, vec) == -1 && errno == EINVAL; + ksft_test_result(ok, + "mincore rejects an unaligned start and a VMA hole\n"); + munmap(p, 5 * page); +} + +static void file_residency(void) +{ + const size_t native = 16384, count = 3 * native / page; + unsigned char *p, *guard, *vec, byte = 0x7b; + int fd = memfd_create("residency", 0); + bool ok; + size_t i; + + if (fd < 0 || ftruncate(fd, 4 * native) || + pwrite(fd, &byte, 1, native) != 1) + ksft_exit_fail_msg("memfd setup\n"); + p = mmap(NULL, 3 * native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page); + guard = mmap(NULL, 2 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || guard == MAP_FAILED || + mprotect(guard + page, page, PROT_NONE)) + ksft_exit_fail_msg("file/vector mmap\n"); + vec = guard + page - count; + memset(vec, 0x55, count); + ok = mincore(p, 3 * native, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == ((page + i * page) / native == 1); + ksft_test_result( + ok, + "unmapped file leaves share residency at native cache indices\n"); + ok = p[native - page] == byte && mincore(p, 3 * native, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == ((page + i * page) / native == 1); + ksft_test_result( + ok, + "present and cache-only file residency agree at a guarded vector boundary\n"); + ok = msync(p + page, 1, MS_SYNC) == 0; + errno = 0; + ok &= msync(p + page + 1, page, MS_SYNC) == -1 && errno == EINVAL; + ok &= munmap(p + page, page) == 0; + errno = 0; + ok &= msync(p, 3 * page, MS_SYNC) == -1 && errno == ENOMEM; + ksft_test_result(ok, + "msync uses process alignment and reports holes\n"); + munmap(p, 3 * native); + munmap(guard, 2 * page); + close(fd); +} + +static void large_vector(void) +{ + const size_t count = 16384 + 3; + unsigned char *p = mmap(NULL, count * page, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, + 0); + unsigned char *vec = malloc(count + 1); + bool ok; + size_t i; + + if (p == MAP_FAILED || !vec) + ksft_exit_fail_msg("large vector allocation\n"); + memset(vec, 0x55, count + 1); + ok = mincore(p, count * page - 7, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == 0; + ok &= vec[count] == 0x55; + ksft_test_result( + ok, "mincore advances across multiple native output buffers\n"); + free(vec); + munmap(p, count * page); +} + +static void seals(void) +{ + unsigned char *p = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + bool ok; + pid_t pid; + int status = 0; + + if (p == MAP_FAILED) + ksft_exit_fail_msg("seal mmap\n"); + ok = syscall(SYS_mseal, p + page, 1UL, 0UL) == 0; + errno = 0; + ok &= mprotect(p + page, page, PROT_READ) == -1 && errno == EPERM; + errno = 0; + ok &= munmap(p + page, page) == -1 && errno == EPERM; + ok &= mprotect(p, page, PROT_READ) == 0; + ok &= mprotect(p + 2 * page, page, PROT_READ) == 0; + ksft_test_result( + ok, + "a one-byte seal covers only its user page and protects it\n"); + pid = fork(); + if (!pid) + _exit(!(munmap(p + page, page) == -1 && errno == EPERM)); + ok = pid > 0 && waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status); + ksft_test_result(ok, "fork inherits the sealed user page\n"); + munmap(p, page); + munmap(p + 2 * page, page); + /* The sealed VMA is released by process exit. */ +} + +int main(int argc, char **argv) +{ + page = getauxval(AT_PAGESZ); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + ksft_exit_fail_msg("unexpected process page ABI: %zu\n", page); + ksft_print_msg("process page size: %zu\n", page); + ksft_print_header(); + ksft_set_plan(9); + residency(); + file_residency(); + large_vector(); + seals(); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/mglru-granule-init.c b/tools/testing/selftests/arm64/user4k/mglru-granule-init.c new file mode 100644 index 0000000000000..9bb9bb74e38e8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mglru-granule-init.c @@ -0,0 +1,365 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CONTROL "/sys/kernel/debug/lru_gen" +#define FULL "/sys/kernel/debug/lru_gen_full" +static int ready_fd = -1, command_fd = -1; +static unsigned worker_index; +static unsigned char pattern(unsigned region, size_t byte, unsigned round) +{ + return (byte ^ (byte >> 12) ^ (region * 37) ^ (round * 53)) & 255; +} +static int generation(unsigned long long *cg, unsigned *node, + unsigned long *max) +{ + FILE *f = fopen(CONTROL, "r"); + char line[256]; + int have_cg = 0, have_node = 0, have_seq = 0; + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) { + unsigned long long id; + unsigned n; + unsigned long seq, age, anon, file; + if (sscanf(line, "memcg %llu", &id) == 1) { + if (have_cg) + break; + *cg = id; + have_cg = 1; + } else if (sscanf(line, " node %u", &n) == 1) { + if (have_node) + break; + *node = n; + have_node = 1; + } else if (have_node && sscanf(line, "%lu %lu %lu %lu", &seq, + &age, &anon, &file) == 4) { + *max = seq; + have_seq = 1; + } + } + fclose(f); + return have_seq ? 0 : -1; +} +static unsigned long scan_evidence(void) +{ + FILE *f = fopen(FULL, "r"); + char line[512]; + unsigned long total = 0; + if (!f) + return 0; + while (fgets(line, sizeof(line), f)) { + unsigned long t, y, found, added; + if (sscanf(line, " %lut %luy %luf %lua", &t, &y, &found, + &added) == 4) { + printf("# MGLRU scan total=%lu young=%lu tables=%lu added=%lu\n", + t, y, found, added); + total += y; + } + } + fclose(f); + return total; +} +static int age(void) +{ + if (ready_fd >= 0) { + char token = 'A'; + return write(ready_fd, &token, 1) != 1 || + read(command_fd, &token, 1) != 1 || token != 'G'; + } + unsigned long long cg = 0, next_cg = 0; + unsigned node = 0, next_node = 0; + unsigned long seq = 0, next = 0; + char command[128]; + if (generation(&cg, &node, &seq)) + return 1; + /* This tree accepts swappiness here; 200 scans both anonymous and file pages. */ + int len = snprintf(command, sizeof(command), "+ %llu %u %lu 200 1\n", + cg, node, seq); + int fd = open(CONTROL, O_WRONLY); + if (fd < 0) + return 1; + ssize_t written = write(fd, command, len); + close(fd); + if (written != len || generation(&next_cg, &next_node, &next) || + next <= seq) { + printf("not ok - MGLRU generation command=%s result=%zd errno=%d next=%lu\n", + command, written, errno, next); + return 1; + } + printf("ok - MGLRU generation %lu -> %lu\n", seq, next); + return 0; +} +static int check_referenced(unsigned char **regions, unsigned nr, + size_t expected) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char line[512]; + int region = -1, failed = 0; + unsigned seen = 0; + unsigned long start, end, kb; + if (!f) + return 1; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + region = -1; + for (unsigned i = 0; i < nr; i++) + if (start == (uintptr_t)regions[i]) + region = i; + } else if (region >= 0 && + sscanf(line, "Referenced: %lu kB", &kb) == 1) { + printf("%s - region=%d start=%p referenced=%lu expected=%zu kB\n", + kb == expected ? "ok" : "not ok", region, + regions[region], kb, expected); + failed |= kb != expected; + seen++; + } + } + fclose(f); + return failed || seen != nr; +} +static int exercise(unsigned long expected) +{ + static const unsigned shifts[] = { 21, 25, 29, 30, 36, 39, 42, 47 }; + unsigned char *regions[sizeof(shifts) / sizeof(shifts[0])] = {}; + size_t ps = getauxval(AT_PAGESZ), len = 8 * ps; + unsigned nr = sizeof(shifts) / sizeof(shifts[0]); + int fail = 0, fd = memfd_create("granule-aging", 0); + cpu_set_t cpus; + void *scratch = MAP_FAILED; + CPU_ZERO(&cpus); + CPU_SET(ready_fd >= 0 ? worker_index % 2 : 0, &cpus); + if (sched_setaffinity(0, sizeof(cpus), &cpus)) + return 1; + if (ps != expected || fd < 0 || ftruncate(fd, nr * len)) + return 1; + for (unsigned i = 0; i < nr; i++) { + uintptr_t address = (1UL << shifts[i]) - 4 * ps; + regions[i] = mmap((void *)address, len, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | + (i % 2 ? MAP_SHARED : + MAP_PRIVATE | MAP_ANONYMOUS), + i % 2 ? fd : -1, i % 2 ? i * len : 0); + if (regions[i] != (void *)address) { + printf("not ok - %zuK boundary %u mmap errno=%d\n", + ps / 1024, shifts[i], errno); + fail = 1; + goto out; + } + } + for (unsigned i = 0; i < nr; i++) + memset(regions[i], 0x19, len); + /* Publish earlier folios from the bounded per-CPU LRU-add batch. */ + scratch = mmap(NULL, 8UL << 20, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (scratch == MAP_FAILED) { + fail = 1; + goto out; + } + memset(scratch, 0x7b, 8UL << 20); + /* Seed MGLRU's first-access referenced state before testing the next access. */ + if (age()) { + fail = 1; + goto out; + } + for (unsigned round = 0; round < 4; round++) { + if (ready_fd >= 0) { + unsigned cpu = (worker_index + round) % 2; + CPU_ZERO(&cpus); + CPU_SET(cpu, &cpus); + if (sched_setaffinity(0, sizeof(cpus), &cpus) || + sched_getcpu() != (int)cpu) { + fail = 1; + break; + } + printf("# concurrent ABI=%zuK round=%u cpu=%u\n", + ps / 1024, round, cpu); + } + for (unsigned i = 0; i < nr; i++) + for (size_t j = 0; j < len; j++) + regions[i][j] = pattern(i, j, round); + fail |= check_referenced(regions, nr, len / 1024); + if (age() || age()) { + fail = 1; + break; + } + fail |= check_referenced(regions, nr, 0); + int good = 1; + for (unsigned i = 0; i < nr; i++) + for (size_t j = 0; j < len; j++) + if (regions[i][j] != pattern(i, j, round)) { + good = 0; + break; + } + unsigned long young = scan_evidence(); + printf("%s - %zuK round%u eight table boundaries bytes and aging young=%lu\n", + good && young ? "ok" : "not ok", ps / 1024, round, + young); + fail |= !good || !young; + } +out: + if (scratch != MAP_FAILED) + munmap(scratch, 8UL << 20); + for (unsigned i = 0; i < nr; i++) + if (regions[i] && regions[i] != MAP_FAILED) + munmap(regions[i], len); + close(fd); + return fail; +} +static int concurrent(unsigned long *sizes, unsigned nr) +{ + int ready[3][2], commands[3][2], failed = 0; + pid_t workers[3]; + + for (unsigned i = 0; i < nr; i++) + if (pipe(ready[i]) || pipe(commands[i])) + return 1; + for (unsigned i = 0; i < nr; i++) { + workers[i] = fork(); + if (!workers[i]) { + char size[32], index[16], ready_arg[16], + command_arg[16]; + for (unsigned j = 0; j < nr; j++) { + close(ready[j][0]); + close(commands[j][1]); + if (j != i) { + close(ready[j][1]); + close(commands[j][0]); + } + } + snprintf(size, sizeof(size), "%lu", sizes[i]); + snprintf(index, sizeof(index), "%u", i); + snprintf(ready_arg, sizeof(ready_arg), "%d", + ready[i][1]); + snprintf(command_arg, sizeof(command_arg), "%d", + commands[i][0]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--worker", size, index, + ready_arg, command_arg, NULL); + _exit(4); + } + if (workers[i] < 0) + failed = 1; + } + for (unsigned i = 0; i < nr; i++) { + close(ready[i][1]); + close(commands[i][0]); + } + /* One warm-up barrier plus two aging barriers for each of four rounds. */ + for (unsigned phase = 0; phase < 9 && !failed; phase++) { + for (unsigned i = 0; i < nr; i++) { + char token; + if (read(ready[i][0], &token, 1) != 1 || token != 'A') + failed = 1; + } + if (failed || age()) { + failed = 1; + break; + } + for (unsigned i = 0; i < nr; i++) { + char token = 'G'; + if (write(commands[i][1], &token, 1) != 1) + failed = 1; + } + } + for (unsigned i = 0; i < nr; i++) { + close(ready[i][0]); + close(commands[i][1]); + } + for (unsigned i = 0; i < nr; i++) { + int status = 0; + int ok = workers[i] > 0 && + waitpid(workers[i], &status, 0) == workers[i] && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - concurrent MGLRU ABI %luK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + } + return failed; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (argc == 6 && !strcmp(argv[1], "--worker")) { + if (getppid() != 1) + return 2; + worker_index = strtoul(argv[3], NULL, 10); + ready_fd = atoi(argv[4]); + command_fd = atoi(argv[5]); + return exercise(strtoul(argv[2], NULL, 10)); + } + int concurrent_mode = argc == 2 && !strcmp(argv[1], "--concurrent"); + if (getpid() != 1 || (argc != 1 && !concurrent_mode)) + return 2; + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + FILE *f = fopen("/sys/kernel/mm/lru_gen/enabled", "r"); + unsigned enabled = 0; + if (f) { + fscanf(f, "%x", &enabled); + fclose(f); + } + if ((enabled & 3) != 3) { + printf("not ok - MGLRU/MM walk disabled %#x\n", enabled); + fail = 1; + goto done; + } + printf("ok - MGLRU enabled=%#x\n", enabled); + unsigned long sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + if (concurrent_mode) { + fail = concurrent(sizes, sizes[0] > 16384 ? 3 : 2); + goto done; + } + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t child = fork(); + int status = 0; + if (!child) { + char value[32]; + snprintf(value, sizeof(value), "%lu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--exercise", value, NULL); + _exit(4); + } + int good = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - MGLRU ABI %luK status=%#x\n", + good ? "ok" : "not ok", sizes[i] / 1024, status); + fail |= !good; + } +done: + printf("GRANULE MGLRU%s %s\n", concurrent_mode ? " CONCURRENT" : "", + fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/mixed-process.c b/tools/testing/selftests/arm64/user4k/mixed-process.c new file mode 100644 index 0000000000000..6d08ab55991b5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mixed-process.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real EL0 mixed-granule sharing, futex queues, and process_vm copies. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define NATIVE 16384U +#define QUARTER 4096U +#define ROUNDS 2000U +struct channel { + atomic_int turn; + unsigned int value; + unsigned int worker_wakes; +}; +struct ready { + uintptr_t base; + unsigned long page; +}; + +static int wait_turn(struct channel *channel, int wanted) +{ + const struct timespec timeout = { .tv_sec = 5 }; + int value; + + while ((value = atomic_load_explicit(&channel->turn, + memory_order_acquire)) != wanted) { + if (syscall(SYS_futex, &channel->turn, FUTEX_WAIT, value, + &timeout, NULL, 0) < 0 && + errno != EAGAIN && errno != EINTR) + return -1; + } + return 0; +} + +static int send_turn(struct channel *channel, int turn) +{ + atomic_store_explicit(&channel->turn, turn, memory_order_release); + return syscall(SYS_futex, &channel->turn, FUTEX_WAKE, 1, NULL, NULL, 0); +} + +static unsigned char pattern(size_t i) +{ + return (i ^ (i >> 8) ^ (i >> 16)) & 255; +} + +static int worker(int fd, int ready_fd, unsigned long expected) +{ + unsigned long page = getauxval(AT_PAGESZ); + unsigned char *mapping, *remote; + struct channel *channel; + struct ready ready; + size_t i, begin = page - 13, end = 3 * page + 16; + unsigned int round, wakes = 0; + int ret; + + if (page != expected) + return 10; + mapping = mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page == QUARTER ? QUARTER : 0); + remote = mmap(NULL, 5 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED || remote == MAP_FAILED) + return 11; + channel = (void *)(mapping + (page == QUARTER ? 0 : QUARTER)); + for (i = 0; i < 4 * page; i++) + remote[i] = pattern(i); + if (mprotect(remote + 4 * page, page, PROT_NONE)) + return 12; + ready = (struct ready){ (uintptr_t)remote, page }; + if (write(ready_fd, &ready, sizeof(ready)) != sizeof(ready)) + return 13; + close(ready_fd); + for (round = 0; round < ROUNDS; round++) { + if (wait_turn(channel, 1) || + channel->value != (round ^ 0xabcddcbaU)) + return 14; + if (!round) { + for (i = 0; i < 4 * page; i++) + if (remote[i] != + (pattern(i) ^ + (i >= begin && i < end ? 0x5a : 0))) + return 15; + } + channel->value = ~(round ^ 0xabcddcbaU); + ret = send_turn(channel, 0); + if (ret < 0) + return 16; + wakes += ret; + } + channel->worker_wakes = wakes; + return 0; +} + +static bool remote_copies(pid_t pid, const struct ready *ready) +{ + size_t begin = ready->page - 13, len = 2 * ready->page + 29, i; + unsigned char *data = malloc(len), edge[2] = {}; + struct iovec local = { data, len }, + remote = { (void *)(ready->base + begin), len }; + bool ok = data != NULL; + + if (!ok) + return false; + ok = process_vm_readv(pid, &local, 1, &remote, 1, 0) == (ssize_t)len; + for (i = 0; ok && i < len; i++) + ok = data[i] == pattern(i + begin); + for (i = 0; i < len; i++) + data[i] = pattern(i + begin) ^ 0x5a; + ok &= process_vm_writev(pid, &local, 1, &remote, 1, 0) == (ssize_t)len; + local = (struct iovec){ edge, sizeof(edge) }; + remote = (struct iovec){ (void *)(ready->base + 4 * ready->page - 1), + sizeof(edge) }; + ok &= process_vm_readv(pid, &local, 1, &remote, 1, 0) == 1; + ok &= edge[0] == pattern(4 * ready->page - 1); + free(data); + return ok; +} + +int main(int argc, char **argv) +{ + unsigned long page = getauxval(AT_PAGESZ), other; + unsigned char *mapping, *private; + struct channel *channel; + struct ready ready = {}; + int fd, pipefd[2], status = 0, wakes = 0, ret; + unsigned int round = 0; + pid_t pid; + bool ok; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10)); + if (argc != 1 || (page != QUARTER && page != NATIVE)) + return 2; + other = page == NATIVE ? QUARTER : NATIVE; + fd = memfd_create("mixed-process", 0); + if (fd < 0 || ftruncate(fd, 2 * NATIVE) || pipe(pipefd)) + return 3; + mapping = mmap(NULL, 2 * NATIVE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + private = + mmap(NULL, NATIVE, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + if (mapping == MAP_FAILED || private == MAP_FAILED) + return 4; + channel = (void *)(mapping + QUARTER); + private[QUARTER] = 0x7e; + pid = fork(); + if (!pid) { + char fdarg[32], pipearg[32], sizearg[32]; + + close(pipefd[0]); + snprintf(fdarg, sizeof(fdarg), "%d", fd); + snprintf(pipearg, sizeof(pipearg), "%d", pipefd[1]); + snprintf(sizearg, sizeof(sizearg), "%lu", other); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, other, 0UL, 0UL, 0UL)) + _exit(20); + execl("/mixed-process", "/mixed-process", "--worker", fdarg, + pipearg, sizearg, NULL); + _exit(21); + } + close(pipefd[1]); + if (pid < 0) + return 5; + ok = read(pipefd[0], &ready, sizeof(ready)) == sizeof(ready) && + ready.page == other; + close(pipefd[0]); + ok = ok && remote_copies(pid, &ready); + printf("%s - process_vm copies %lu -> %lu, including partial inaccessible boundary\n", + ok ? "ok" : "not ok", page, other); + for (round = 0; ok && round < ROUNDS; round++) { + channel->value = round ^ 0xabcddcbaU; + ret = send_turn(channel, 1); + ok = ret >= 0 && !wait_turn(channel, 0) && + channel->value == ~(round ^ 0xabcddcbaU); + if (ret > 0) + wakes += ret; + } + if (!ok) + kill(pid, SIGKILL); + ret = waitpid(pid, &status, 0); + ok &= ret == pid && WIFEXITED(status) && !WEXITSTATUS(status); + ok &= wakes > 0 && channel->worker_wakes > 0 && + private[QUARTER] == 0x7e; + ok &= mapping[0] == 0 && mapping[2 * QUARTER] == 0 && + mapping[NATIVE - 1] == 0; + printf("%s - mixed %lu/%lu shared futex/COW: rounds=%u wakes=%d/%u child=%#x\n", + ok ? "ok" : "not ok", page, other, round, wakes, + channel->worker_wakes, status); + munmap(private, NATIVE); + munmap(mapping, 2 * NATIVE); + close(fd); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c b/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c new file mode 100644 index 0000000000000..0dc94eabcc4fb --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static long locked_kb(void) +{ + char line[256]; + long value = -1; + FILE *f = fopen("/proc/self/status", "re"); + + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmLck: %ld", &value) == 1) + break; + fclose(f); + return value; +} + +static bool wait_ok(pid_t pid) +{ + int status = 0; + + return pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); +} + +static bool remap_locked(unsigned long ps) +{ + unsigned char *p, *q, *target; + unsigned long kb = ps / 1024; + bool ok; + + p = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || mlock(p, 2 * ps)) + ksft_exit_fail_msg("mlock remap setup\n"); + memset(p, 0x37, 2 * ps); + ok = locked_kb() == (long)(2 * kb); + q = mremap(p, 2 * ps, 4 * ps, MREMAP_MAYMOVE); + if (q == MAP_FAILED) + ksft_exit_fail_msg("locked remap grow: %s\n", strerror(errno)); + ok &= locked_kb() == (long)(4 * kb); + for (unsigned long i = 0; i < 4 * ps; i++) + ok &= q[i] == (i < 2 * ps ? 0x37 : 0); + p = mremap(q, 4 * ps, 3 * ps, 0); + if (p == MAP_FAILED) + ksft_exit_fail_msg("locked remap shrink: %s\n", + strerror(errno)); + ok &= locked_kb() == (long)(3 * kb); + p[2 * ps] = 0xb6; + ok &= !munmap(p + ps, ps) && locked_kb() == (long)(2 * kb); + target = mmap(NULL, ps, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (target == MAP_FAILED) + ksft_exit_fail_msg("locked remap target\n"); + q = mremap(p, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, target); + if (q == MAP_FAILED) + ksft_exit_fail_msg("locked remap move: %s\n", strerror(errno)); + ok &= q == target && locked_kb() == (long)(2 * kb); + for (unsigned long i = 0; i < ps; i++) + ok &= q[i] == 0x37; + ok &= p[2 * ps] == 0xb6; + ok &= !munmap(q, ps) && locked_kb() == (long)kb; + ok &= !munmap(p + 2 * ps, ps) && locked_kb() == 0; + return ok; +} + +static int exercise(void) +{ + unsigned long ps = getauxval(AT_PAGESZ), kb = ps / 1024; + unsigned char *p, *q, residency[4]; + int gate[2]; + pid_t pid; + bool ok; + + ksft_print_header(); + ksft_set_plan(9); + p = mmap(NULL, 32 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || locked_kb() != 0) + ksft_exit_fail_msg("initial mapping/accounting\n"); + ok = !mlock(p + ps + 17, ps - 32) && locked_kb() == (long)kb; + ok &= !mincore(p + ps, ps, residency) && residency[0] & 1; + p[ps] = 0x51; + ksft_test_result( + ok, "%luK unaligned mlock uses exactly one user leaf\n", kb); + ok = !mlock(p + ps, ps) && locked_kb() == (long)kb; + ok &= !mlock(p + 2 * ps, ps) && locked_kb() == (long)(2 * kb); + ok &= !munlock(p + ps, ps) && locked_kb() == (long)kb; + p[2 * ps] = 0x62; + errno = 0; + ok &= madvise(p + 2 * ps, ps, MADV_PAGEOUT) == -1 && errno == EINVAL && + p[2 * ps] == 0x62; + ok &= !munlock(p + 2 * ps, ps) && locked_kb() == 0; + ksft_test_result( + ok, "%luK repeat lock and independent adjacent unlock\n", kb); + ok = !mlock2(p + 4 * ps, 4 * ps, MLOCK_ONFAULT) && + locked_kb() == (long)(4 * kb); + ok &= !mincore(p + 4 * ps, 4 * ps, residency); + for (unsigned int i = 0; i < 4; i++) + ok &= !(residency[i] & 1); + p[4 * ps] = 0x74; + ok &= !mincore(p + 4 * ps, ps, residency) && residency[0] & 1; + ok &= !munlock(p + 4 * ps, 4 * ps) && locked_kb() == 0; + ksft_test_result( + ok, "%luK MLOCK_ONFAULT accounting and deferred faults\n", kb); + q = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_LOCKED, -1, 0); + ok = q != MAP_FAILED && locked_kb() == (long)(4 * kb); + if (q == MAP_FAILED || pipe(gate)) + ksft_exit_fail_msg("locked mmap/fork setup\n"); + memset(q, 0x8a, 4 * ps); + pid = fork(); + if (!pid) { + char token; + + close(gate[1]); + if (read(gate[0], &token, 1) != 1 || locked_kb() != 0) + _exit(2); + for (unsigned long i = 0; i < 4 * ps; i++) + if (q[i] != 0x8a) + _exit(3); + _exit(0); + } + close(gate[0]); + memset(q + ps, 0x9b, ps); + ok &= write(gate[1], "x", 1) == 1; + close(gate[1]); + ok &= wait_ok(pid) && locked_kb() == (long)(4 * kb); + ok &= !munmap(q, 4 * ps) && locked_kb() == 0; + ksft_test_result( + ok, "%luK MAP_LOCKED, fork noninheritance and parent COW\n", + kb); + ok = !mlockall(MCL_FUTURE | MCL_ONFAULT); + q = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + ok &= q != MAP_FAILED && locked_kb() == (long)kb; + if (q != MAP_FAILED) + *q = 0xaf; + ok &= !munlockall() && locked_kb() == 0; + if (q != MAP_FAILED) + munmap(q, ps); + ksft_test_result(ok, "%luK future on-fault locks and munlockall\n", kb); + pid = fork(); + if (!pid) { + struct rlimit limit = { ps, ps }; + + if (setrlimit(RLIMIT_MEMLOCK, &limit) || setuid(1000) || + mlock(p, ps)) + _exit(4); + errno = 0; + if (mlock(p + ps, ps) != -1 || errno != ENOMEM || + locked_kb() != (long)kb) + _exit(5); + _exit(0); + } + ksft_test_result( + wait_ok(pid), + "%luK unprivileged RLIMIT_MEMLOCK in user-page units\n", kb); + pid = fork(); + if (!pid) { + if (mlockall(MCL_CURRENT) || locked_kb() <= (long)kb || + munlockall() || locked_kb() != 0) + _exit(6); + _exit(0); + } + ksft_test_result(wait_ok(pid), "%luK lock/unlock current mappings\n", + kb); + { + unsigned long huge = ((1UL << 31) + 1) * ps; + + q = mmap(NULL, huge, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + ok = q != MAP_FAILED; + if (q != MAP_FAILED) { + ok &= !mlock2(q, huge, MLOCK_ONFAULT) && + locked_kb() == (long)(huge / 1024); + ok &= !munlock(q, huge) && locked_kb() == 0; + munmap(q, huge); + } + ksft_test_result( + ok, + "%luK deferred lock count beyond signed 32-bit leaves\n", + kb); + } + ksft_test_result( + remap_locked(ps), + "%luK locked grow/shrink/move and partial unmap accounting\n", + kb); + munmap(p, 32 * ps); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + + if (!pid) { + char size[32], host_size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + snprintf(host_size, sizeof(host_size), "%lu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(7); + execl("/init", "/init", "--test", size, host_size, + NULL); + _exit(8); + } + ok &= wait_ok(pid); + } + printf("%s - mlock fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/numa-balancing-init.c b/tools/testing/selftests/arm64/user4k/numa-balancing-init.c new file mode 100644 index 0000000000000..2ea31bda12968 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/numa-balancing-init.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Disposable two-node guest: real automatic NUMA hinting and migration. + * Volatile accesses intentionally trigger page faults and repeated accesses; + * thread coordination uses an atomic flag, not volatile storage. + */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int good, const char *expr, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expr, + errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static void setting(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + + CHECK(fd >= 0); + CHECK(write(fd, value, strlen(value)) == (ssize_t)strlen(value)); + CHECK(!close(fd)); +} + +static void cpu(int id) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(id, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} + +static unsigned long counter(const char *key) +{ + FILE *f = fopen("/proc/vmstat", "r"); + char name[128]; + unsigned long value; + + CHECK(f); + while (fscanf(f, "%127s %lu", name, &value) == 2) { + if (!strcmp(name, key)) { + CHECK(!fclose(f)); + return value; + } + } + CHECK(0); + return 0; +} + +static double now(void) +{ + struct timespec ts; + + CHECK(!clock_gettime(CLOCK_MONOTONIC, &ts)); + return ts.tv_sec + ts.tv_nsec / 1e9; +} + +static int node_at(void *address) +{ + int node = -1; + + CHECK(!syscall(SYS_get_mempolicy, &node, NULL, 0UL, address, + MPOL_F_ADDR | MPOL_F_NODE)); + return node; +} + +static unsigned long sample(unsigned char *map, unsigned long length, + unsigned long page) +{ + unsigned long moved = 0; + + for (unsigned long off = 0; off < length; off += page) + if (off != 2 * page) + moved += node_at(map + off + 16) == 1; + return moved; +} + +static void forbidden(unsigned char *address, int write_access) +{ + pid_t child = fork(); + int status; + + CHECK(child >= 0); + if (!child) { + if (write_access) + *(volatile unsigned char *)address = 0x73; + else + (void)*(volatile unsigned char *)address; + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + CHECK(WIFSIGNALED(status) && WTERMSIG(status) == SIGSEGV); +} + +struct local_reader { + unsigned char *map; + unsigned long length, page; + atomic_int stop; +}; + +static void *read_local(void *arg) +{ + struct local_reader *r = arg; + + cpu(1); + while (!atomic_load_explicit(&r->stop, memory_order_relaxed)) { + for (unsigned long off = 0; off < r->length; off += r->page) { + if (off == 2 * r->page) + continue; + CHECK(*(volatile uint64_t *)(r->map + off) == + 0x81720000 + off / r->page); + } + } + return NULL; +} + +static void local_restore(unsigned char *map, unsigned long length, + unsigned long page) +{ + struct local_reader r = { .map = map, .length = length, .page = page }; + unsigned long hints = counter("numa_hint_faults_local"); + pthread_t worker; + double deadline = now() + 2; + + /* Two threads permit hinting of already-local private mappings. */ + CHECK(!pthread_create(&worker, NULL, read_local, &r)); + do { + for (unsigned long off = 0; off < length; off += page) { + if (off == 2 * page) + continue; + CHECK(*(volatile uint64_t *)(map + off) == + 0x81720000 + off / page); + } + } while (now() < deadline); + atomic_store_explicit(&r.stop, 1, memory_order_relaxed); + CHECK(!pthread_join(worker, NULL)); + hints = counter("numa_hint_faults_local") - hints; + printf("NUMAB local page=%lu hints=%lu\n", page, hints); + CHECK(hints > 0); +} + +static void exercise(void) +{ + unsigned long page = getauxval(AT_PAGESZ), length = 8UL << 20; + unsigned long mask = 1, allowed = 0, hints, migrated, iterations = 0; + unsigned long moved = 0, total = length / page - 1; + unsigned char *map; + int mode; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &allowed, 64UL, NULL, + MPOL_F_MEMS_ALLOWED)); + CHECK(allowed == 3); + cpu(0); + map = mmap(NULL, length, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(map != MAP_FAILED); + CHECK(!madvise(map, length, MADV_NOHUGEPAGE)); + CHECK(!syscall(SYS_mbind, map, length, MPOL_BIND, &mask, 64UL, 0)); + for (unsigned long off = 0; off < length; off += page) { + *(uint64_t *)(map + off) = 0x81720000 + off / page; + *(uint64_t *)(map + off + 8) = 0; + *(uint64_t *)(map + off + page - 8) = 0x61320000 + off / page; + CHECK(node_at(map + off + 16) == 0); + } + CHECK(!mprotect(map + page, page, PROT_READ)); + CHECK(!mprotect(map + 2 * page, page, PROT_NONE)); + CHECK(!syscall(SYS_mbind, map, length, MPOL_DEFAULT, NULL, 0UL, 0)); + cpu(1); + hints = counter("numa_hint_faults"); + migrated = counter("numa_pages_migrated"); + double started = now(), deadline = started + 20; + + do { + for (unsigned long off = 0; off < length; off += page) { + volatile uint64_t *p = (void *)(map + off); + + if (off == 2 * page) + continue; + CHECK(p[0] == 0x81720000 + off / page); + CHECK(p[page / 8 - 1] == 0x61320000 + off / page); + if (off != page) + p[1]++; + } + iterations++; + if (!(iterations % 128)) { + moved = sample(map, length, page); + if (moved == total) + break; + } + } while (now() < deadline); + moved = sample(map, length, page); + hints = counter("numa_hint_faults") - hints; + migrated = counter("numa_pages_migrated") - migrated; + printf("NUMAB page=%lu moved=%lu/%lu hints=%lu migrated=%lu loops=%lu seconds=%.3f\n", + page, moved, total, hints, migrated, iterations, + now() - started); + CHECK(moved == total && hints > 0 && migrated > 0); + for (unsigned long off = 0; off < length; off += page) { + if (off != page && off != 2 * page) + CHECK(*(uint64_t *)(map + off + 8) == iterations); + } + local_restore(map, length, page); + forbidden(map + page, 1); + forbidden(map + 2 * page, 0); + CHECK(!mprotect(map + 2 * page, page, PROT_READ)); + CHECK(*(uint64_t *)(map + 2 * page) == 0x81720002); + CHECK(!munmap(map, length)); + printf("ok - NUMAB page=%lu automatic placement, bytes and permissions\n", + page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { 4096, 16384, 65536 }; + int failed = 0, compat_only = 0, no4k = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--case")) { + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[2], NULL, 10)); + exercise(); + return 0; + } + CHECK(getpid() == 1); + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--compat-only")) + compat_only = 1; + else if (!strcmp(argv[i], "--no4k")) + no4k = 1; + else + CHECK(0); + } + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("debugfs", "/debug", "debugfs", 0, NULL)); + setting("/proc/sys/kernel/numa_balancing", "1\n"); + setting("/debug/sched/numa_balancing/scan_delay_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_period_min_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_period_max_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_size_mb", "16\n"); + for (unsigned int i = 0; i < 3; i++) { + pid_t child; + int status; + + if (compat_only || sizes[i] > native || + (no4k && sizes[i] == 4096)) + continue; + child = fork(); + CHECK(child >= 0); + if (!child) { + char s[32]; + + snprintf(s, sizeof(s), "%lu", sizes[i]); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], + 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", s, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + } + CHECK(!access("/compat-numa-balancing-probe", X_OK)); + pid_t child = fork(); + int status; + + CHECK(child >= 0); + if (!child) { + char s[32]; + + snprintf(s, sizeof(s), "%lu", no4k ? native : 4096UL); + execl("/compat-numa-balancing-probe", + "/compat-numa-balancing-probe", s, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + printf("A32 NUMAB expected-page=%lu status=%d\n", + no4k ? native : 4096UL, status); + printf("NUMA BALANCING %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/numa-policy-init.c b/tools/testing/selftests/arm64/user4k/numa-policy-init.c new file mode 100644 index 0000000000000..fedcedb529709 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/numa-policy-init.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable two-node guest: policy ranges and native physical placement. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int good, const char *expr, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expr, + errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static long bind_to(void *addr, unsigned long bytes, int node, + unsigned int flags) +{ + unsigned long mask = 1UL << node; + + return syscall(SYS_mbind, addr, bytes, MPOL_BIND, &mask, 64UL, flags); +} + +static void policy_at(void *addr, int node) +{ + unsigned long mask = 0; + int mode = -1; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &mask, 64UL, addr, + MPOL_F_ADDR)); + CHECK(mode == MPOL_BIND && mask == (1UL << node)); +} + +static void node_at(void *addr, int expected) +{ + int node = -1; + + CHECK(!syscall(SYS_get_mempolicy, &node, NULL, 0UL, addr, + MPOL_F_NODE | MPOL_F_ADDR)); + if (node != expected) + fprintf(stderr, "node=%d expected=%d addr=%p\n", node, expected, + addr); + CHECK(node == expected); +} + +static void exercise(unsigned long native, int baseline) +{ + unsigned long page = getauxval(AT_PAGESZ), allowed = 0; + unsigned char *reserve, *base, *shared, *alias; + int mode, fd; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &allowed, 64UL, NULL, + MPOL_F_MEMS_ALLOWED)); + CHECK(allowed == + 3); /* Require two real memory nodes, not a silent skip. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + CHECK(!bind_to(base, 2 * native, 0, 0)); + if (baseline) { + errno = 0; + long ret = bind_to(base + page, 1, 1, 0); + + CHECK(page < native ? ret == -1 && errno == EINVAL : ret == 0); + printf("ok - baseline native=%lu page=%lu mbind=%ld errno=%d\n", + native, page, ret, errno); + CHECK(!munmap(reserve, 4 * native)); + return; + } + CHECK(!bind_to(base + page, 1, 1, 0)); + policy_at(base, 0); + policy_at(base + page, 1); + if (page < native) + policy_at(base + 2 * page, 0); + CHECK(!syscall(SYS_set_mempolicy_home_node, base + page, 1UL, 1UL, + 0UL)); + errno = 0; + CHECK(bind_to(base + 1, page, 1, 0) == -1 && errno == EINVAL); + for (unsigned long off = 0; off < 2 * native; off += page) { + base[off + 17] = 0x31 + off / page; + node_at(base + off + 17, off == page ? 1 : 0); + } + CHECK(!bind_to(base + page, page, 1, MPOL_MF_STRICT)); + errno = 0; + CHECK(bind_to(base, 2 * native, 1, MPOL_MF_STRICT) == -1 && + errno == EIO); + policy_at(base, 0); + CHECK(!munmap(reserve, 4 * native)); + + /* A split anonymous VMA must carry its native interleave index correctly. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + unsigned long mask = 3, inset = page < native ? page : 0; + + CHECK(!syscall(SYS_mbind, base + inset, 2 * native - 2 * inset, + MPOL_INTERLEAVE, &mask, 64UL, 0)); + for (unsigned long off = inset; off < 2 * native - inset; off += page) { + base[off + 17] = 0x43; + node_at(base + off + 17, ((uintptr_t)base + off) / native % 2); + } + CHECK(!munmap(reserve, 4 * native)); + + /* Moving a fragment moves its native physical owner, retaining bytes. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + CHECK(!bind_to(base, 2 * native, 0, 0)); + for (unsigned long off = 0; off < 2 * native; off += page) + base[off + 17] = 0x51 + off / page; + CHECK(!bind_to(base + native - page, page, 1, + MPOL_MF_MOVE_ALL | MPOL_MF_STRICT)); + for (unsigned long off = 0; off < 2 * native; off += page) { + CHECK(base[off + 17] == (unsigned char)(0x51 + off / page)); + node_at(base + off + 17, off < native ? 1 : 0); + } + CHECK(!bind_to(base, 2 * native, 0, MPOL_MF_MOVE_ALL | MPOL_MF_STRICT)); + for (unsigned long off = 0; off < 2 * native; off += page) + node_at(base + off + 17, 0); + CHECK(!munmap(reserve, 4 * native)); + + fd = memfd_create("numa-shared", 0); + CHECK(fd >= 0 && !ftruncate(fd, 2 * native)); + shared = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + CHECK(shared != MAP_FAILED); + CHECK(!bind_to(shared, native, 0, 0)); + CHECK(!bind_to(shared + native, native, 1, 0)); + if (page < native) { + alias = mmap(NULL, 2 * native - page, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, page); + CHECK(alias != MAP_FAILED); + policy_at(alias, 0); + policy_at(alias + native - page, 1); + errno = 0; + CHECK(bind_to(alias, page, 1, 0) == -1 && errno == EINVAL); + policy_at(alias, 0); + policy_at(shared, 0); + CHECK(!bind_to(alias + native - page, native, 1, 0)); + for (unsigned long off = 0; off < 2 * native - page; + off += page) { + alias[off + 17] = 0x72; + node_at(alias + off + 17, off + page < native ? 0 : 1); + CHECK(shared[off + page + 17] == 0x72); + } + CHECK(!munmap(alias, 2 * native - page)); + } + CHECK(!munmap(shared, 2 * native) && !close(fd)); + printf("ok - NUMA native=%lu page=%lu policy, placement, migration and shared offsets\n", + native, page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { 4096, 16384, 65536 }; + int failed = 0, baseline = 0, no4; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--case")) { + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + exercise(strtoul(argv[2], NULL, 10), atoi(argv[4])); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + FILE *cmdline = fopen("/proc/cmdline", "r"); + char line[4096]; + + CHECK(cmdline && fgets(line, sizeof(line), cmdline)); + baseline = strstr(line, "numa_fixture_baseline=1") != NULL; + no4 = strstr(line, "id_aa64mmfr0.tgran4=f") != NULL; + CHECK(!fclose(cmdline)); + for (unsigned int i = 0; i < 3; i++) { + int status; + pid_t child; + + if (sizes[i] > native || (no4 && sizes[i] == 4096)) + continue; + child = fork(); + CHECK(child >= 0); + if (!child) { + char n[32], s[32], b[4]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + snprintf(b, sizeof(b), "%d", baseline); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], + 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", n, s, b, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + } + if (!baseline) { + int status; + pid_t child = fork(); + + CHECK(child >= 0); + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", no4 ? native : 4096UL); + execl("/compat-numa-probe", "/compat-numa-probe", n, s, + NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + printf("A32 NUMA expected-page=%lu status=%d\n", + no4 ? native : 4096UL, status); + } + printf("NUMA GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/page-contract.c b/tools/testing/selftests/arm64/user4k/page-contract.c new file mode 100644 index 0000000000000..316ffe48d0a47 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/page-contract.c @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Run unchanged under the current process ABI, including native 4K/16K/64K. + * Explicit --expect-{4k,16k,64k} options assert the selected size; they never + * launch a different ABI or emulate it. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define X18_SENTINEL UINT64_C(0xfedcba9876543210) + +extern long x18_syscall(long nr, long arg0, long arg1, long arg2, + uint64_t *seen); + +static volatile sig_atomic_t signal_seen; +static uint64_t signal_x18; + +static void x18_signal(int sig, siginfo_t *info, void *context) +{ + ucontext_t *uc = context; + + (void)sig; + (void)info; + signal_x18 = uc->uc_mcontext.regs[18]; + signal_seen = 1; + /* rt_sigreturn must restore the interrupted value, not this value. */ + /* Built with -ffixed-x18: the compiler never allocates this register. */ + asm volatile("mov x18, xzr"); +} + +static bool wait_for(pid_t child, int expected_signal) +{ + int status; + pid_t ret; + + if (child < 0) + return false; + do { + ret = waitpid(child, &status, 0); + } while (ret < 0 && errno == EINTR); + if (ret != child) + return false; + if (expected_signal) + return WIFSIGNALED(status) && + WTERMSIG(status) == expected_signal; + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +static bool access_faults(volatile unsigned char *ptr, bool write) +{ + pid_t child = fork(); + + if (!child) { + if (write) + *ptr = 0x5a; + else + (void)*ptr; + _exit(1); + } + return wait_for(child, SIGSEGV); +} + +static void private_mappings(size_t page) +{ + unsigned char *p = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + bool ok; + pid_t child; + + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap: %s\n", strerror(errno)); + memset(p, 0x41, 4 * page); + if (page > 4096) { + errno = 0; + ok = mprotect(p + 4096, 4096, PROT_READ) == -1 && + errno == EINVAL; + } else { + ok = mprotect(p + page, page, PROT_READ) == 0; + } + ksft_test_result(ok, + "mapping alignment agrees with the selected ABI\n"); + ok = mprotect(p + page, page, PROT_READ) == 0 && + mprotect(p + 2 * page, page, PROT_NONE) == 0; + ksft_test_result( + ok && access_faults(p + page, true) && + access_faults(p + 2 * page, false), + "read-only and inaccessible neighbors fault independently\n"); + p[0] = 0x10; + p[3 * page] = 0x30; + ksft_test_result( + p[page] == 0x41 && p[0] == 0x10 && p[3 * page] == 0x30, + "neighbor permissions do not remove writable mappings\n"); + child = fork(); + if (!child) { + p[0] = 0x99; + _exit(p[3 * page] != 0x30); + } + ksft_test_result(wait_for(child, 0) && p[0] == 0x10, + "fork preserves private COW isolation\n"); + ok = munmap(p + page, page) == 0; + ksft_test_result(ok && access_faults(p + page, false) && p[0] == 0x10 && + p[3 * page] == 0x30, + "single-page unmap preserves surrounding mappings\n"); + munmap(p, 4 * page); +} + +static void shared_mappings(size_t page) +{ + int fd = memfd_create("page-contract", MFD_CLOEXEC); + unsigned char *a, *b; + bool ok = true; + pid_t child; + unsigned int i; + + if (fd < 0 || ftruncate(fd, 4 * page)) + ksft_exit_fail_msg("memfd: %s\n", strerror(errno)); + a = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + b = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (a == MAP_FAILED || b == MAP_FAILED) + ksft_exit_fail_msg("shared mmap: %s\n", strerror(errno)); + for (i = 0; i < 4; i++) { + a[i * page] = 0x50 + i; + ok &= b[i * page] == 0x50 + i; + } + ksft_test_result(ok, "shared aliases expose the same bytes\n"); + ok = mprotect(a + page, page, PROT_READ) == 0; + child = fork(); + if (!child) { + b[page] = 0x73; + _exit(a[page] != 0x73); + } + ok &= wait_for(child, 0); + ksft_test_result( + ok && a[page] == 0x73 && access_faults(a + page, true), + "shared writes cross fork while alias permissions stay local\n"); + munmap(a, 4 * page); + munmap(b, 4 * page); + close(fd); +} + +static void x18_contract(void) +{ + struct sigaction sa = { .sa_sigaction = x18_signal, + .sa_flags = SA_SIGINFO }; + struct timespec pause = { .tv_nsec = 1000000 }; + struct rusage before, after; + uint64_t seen; + long ret; + bool ok; + + ret = x18_syscall(SYS_getpid, 0, 0, 0, &seen); + ksft_test_result(ret == getpid() && seen == X18_SENTINEL, + "x18 survives a raw syscall\n"); + ok = getrusage(RUSAGE_SELF, &before) == 0; + ret = x18_syscall(SYS_nanosleep, (long)&pause, 0, 0, &seen); + ok &= getrusage(RUSAGE_SELF, &after) == 0; + ksft_test_result(ok && ret == 0 && seen == X18_SENTINEL && + after.ru_nvcsw > before.ru_nvcsw, + "x18 survives a measured voluntary context switch\n"); + sigemptyset(&sa.sa_mask); + ok = sigaction(SIGUSR1, &sa, NULL) == 0; + if (ok) + ret = x18_syscall(SYS_tgkill, getpid(), syscall(SYS_gettid), + SIGUSR1, &seen); + ksft_test_result( + ok && ret == 0 && signal_seen && signal_x18 == X18_SENTINEL && + seen == X18_SENTINEL, + "signal frame and sigreturn preserve x18 after handler clobber\n"); +} + +int main(int argc, char **argv) +{ + const struct rlimit no_core = { 0, 0 }; + size_t expected = getauxval(AT_PAGESZ); + int result; + + if (argc == 2 && !strcmp(argv[1], "--expect-4k")) { + expected = 4096; + } else if (argc == 2 && !strcmp(argv[1], "--expect-16k")) { + expected = 16384; + } else if (argc == 2 && !strcmp(argv[1], "--expect-64k")) { + expected = 65536; + } else if (argc != 1) { + fprintf(stderr, + "usage: %s [--expect-4k|--expect-16k|--expect-64k]\n", + argv[0]); + return KSFT_FAIL; + } + ksft_print_header(); + ksft_set_plan(11); + if (expected != 4096 && expected != 16384 && expected != 65536) + ksft_exit_skip("unsupported process page size: %zu\n", + expected); + if (sysconf(_SC_PAGESIZE) != (long)expected || + getauxval(AT_PAGESZ) != expected) + ksft_exit_fail_msg( + "expected %zu-byte ABI; sysconf=%ld AT_PAGESZ=%lu\n", + expected, sysconf(_SC_PAGESIZE), getauxval(AT_PAGESZ)); + ksft_test_result_pass( + "process reports the expected %zu-byte page ABI\n", expected); + if (setrlimit(RLIMIT_CORE, &no_core)) + ksft_exit_fail_msg("disable child core dumps: %s\n", + strerror(errno)); + private_mappings(expected); + shared_mappings(expected); + x18_contract(); + ksft_print_cnts(); + result = ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS; + /* This static executable can also be /init in a disposable test VM. */ + if (getpid() == 1) { + fflush(NULL); + reboot(RB_POWER_OFF); + } + return result; +} diff --git a/tools/testing/selftests/arm64/user4k/perf-ring-init.c b/tools/testing/selftests/arm64/user4k/perf-ring-init.c new file mode 100644 index 0000000000000..4252f017619ca --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/perf-ring-init.c @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: perf ring layout and sampling with per-mm page sizes. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int ok, const char *expression, int line) +{ + if (!ok) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, + errno); + exit(1); + } +} + +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int no4k; + +static int wait_child(pid_t child) +{ + int status; + pid_t got; + + if (child < 0) + return 1; + do { + got = waitpid(child, &status, 0); + } while (got < 0 && errno == EINTR); + return got != child || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static int event_open(int sampling) +{ + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .size = sizeof(attr), + .config = PERF_COUNT_SW_PAGE_FAULTS, + .disabled = 1, + .sample_period = sampling ? 1 : 0, + .sample_type = PERF_SAMPLE_TID | PERF_SAMPLE_TIME | + PERF_SAMPLE_ADDR | PERF_SAMPLE_PERIOD, + }; + int fd = syscall(SYS_perf_event_open, &attr, 0, -1, -1, 0); + + CHECK(fd >= 0); + return fd; +} + +static sigjmp_buf fault_return; +static volatile sig_atomic_t fault_signal; + +static void fault_handler(int signal) +{ + fault_signal = signal; + siglongjmp(fault_return, 1); +} + +static void readonly_data(void *address) +{ + struct sigaction action = { .sa_handler = fault_handler }, previous; + + CHECK(!sigemptyset(&action.sa_mask)); + CHECK(!sigaction(SIGBUS, &action, &previous)); + fault_signal = 0; + if (!sigsetjmp(fault_return, 1)) + *(volatile unsigned char *)address = 0; + CHECK(fault_signal == SIGBUS); + CHECK(!sigaction(SIGBUS, &previous, NULL)); +} + +static void copy_record(void *destination, const unsigned char *ring, + unsigned long capacity, uint64_t position, size_t size) +{ + unsigned long offset = position & (capacity - 1); + size_t first = capacity - offset; + + if (first > size) + first = size; + memcpy(destination, ring + offset, first); + memcpy((unsigned char *)destination + first, ring, size - first); +} + +static void sampling(unsigned long pages) +{ + unsigned long page = getauxval(AT_PAGESZ), capacity = pages * page; + unsigned long length = page + capacity; + int fd = event_open(1); + struct perf_event_mmap_page *map = + mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + unsigned char *region, *ring; + uint64_t tail = 0, samples = 0, wrapped = 0, region_samples = 0; + + CHECK(map != MAP_FAILED); + CHECK(map->data_offset == page && map->data_size == capacity); + map->data_tail = 0; + ring = (unsigned char *)map + map->data_offset; + for (unsigned long i = 0; i < capacity; i += page) + CHECK(ring[i] == 0); + readonly_data(ring); + region = mmap(NULL, 16 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(region != MAP_FAILED); + for (unsigned int batch = 0; batch < 10000 && tail < 3 * capacity; + batch++) { + CHECK(!madvise(region, 16 * page, MADV_DONTNEED)); + CHECK(!ioctl(fd, PERF_EVENT_IOC_ENABLE, 0)); + for (unsigned int i = 0; i < 16; i++) + *(volatile unsigned char *)(region + i * page) = 1; + CHECK(!ioctl(fd, PERF_EVENT_IOC_DISABLE, 0)); + uint64_t head = + __atomic_load_n(&map->data_head, __ATOMIC_ACQUIRE); + + CHECK(head >= tail && head - tail <= capacity); + while (tail < head) { + struct perf_event_header header; + struct { + struct perf_event_header header; + uint32_t pid, tid; + uint64_t time, address, period; + } record; + + copy_record(&header, ring, capacity, tail, + sizeof(header)); + CHECK(header.size >= sizeof(header) && + header.size <= head - tail); + if (header.type == PERF_RECORD_SAMPLE) { + CHECK(header.size == sizeof(record)); + copy_record(&record, ring, capacity, tail, + sizeof(record)); + CHECK(record.pid == (uint32_t)getpid()); + CHECK(record.tid == (uint32_t)getpid()); + if (record.address >= (uintptr_t)region && + record.address < + (uintptr_t)region + 16 * page) + region_samples++; + CHECK(record.period == 1); + samples++; + if ((tail & (capacity - 1)) + header.size > + capacity) + wrapped++; + } else { + CHECK(header.type == PERF_RECORD_THROTTLE || + header.type == PERF_RECORD_UNTHROTTLE); + } + tail += header.size; + } + __atomic_store_n(&map->data_tail, tail, __ATOMIC_RELEASE); + } + CHECK(tail >= 3 * capacity && samples && wrapped); + /* Library/stack faults may also occur while a batch is enabled. */ + CHECK(region_samples > samples * 9 / 10); + CHECK(!munmap(region, 16 * page)); + CHECK(!close(fd)); + CHECK(map->data_head == tail); + CHECK(!munmap(map, length)); + printf("ok - perf page=%lu ring=%lu samples=%llu region=%llu wrapped-records=%llu\n", + page, capacity, (unsigned long long)samples, + (unsigned long long)region_samples, (unsigned long long)wrapped); +} + +static void layout(unsigned long native) +{ + unsigned long page = getauxval(AT_PAGESZ); + int fd = event_open(0); + struct perf_event_mmap_page *map = + mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + + CHECK(map != MAP_FAILED); + CHECK(map->data_offset == page && map->data_size == 0); + map->data_tail = 0; + CHECK(!munmap(map, page)); + CHECK(!close(fd)); + fd = event_open(0); + errno = 0; + CHECK(mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0) == + MAP_FAILED); + CHECK(errno == EINVAL); + map = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + CHECK(map != MAP_FAILED); + struct perf_event_mmap_page *alias = + mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + CHECK(alias != MAP_FAILED); + map->data_tail = 7; + CHECK(alias->data_tail == 7); + map->data_tail = 0; + CHECK(!munmap(alias, 3 * page)); + /* A software event has no AUX provider. Check byte-exact offset routing. */ + map->aux_offset = 3 * page; + map->aux_size = native; + errno = 0; + CHECK(mmap(NULL, native, PROT_READ, MAP_SHARED, fd, map->aux_offset) == + MAP_FAILED); + CHECK(errno == EOPNOTSUPP); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + if (sizes[i] > native || sizes[i] == page || + (no4k && sizes[i] == 4096)) + continue; + pid_t child = fork(); + + if (!child) { + char f[32], s[32]; + + snprintf(f, sizeof(f), "%d", fd); + snprintf(s, sizeof(s), "%lu", sizes[i]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, + 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--alias", f, s, NULL); + _exit(3); + } + CHECK(!wait_child(child)); + } + CHECK(map->data_size == 2 * page && map->data_offset == page); + CHECK(!munmap(map, 3 * page)); + CHECK(!close(fd)); + printf("ok - perf page=%lu metadata, invalid size, aliases, AUX routing\n", + page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0, compat_only = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + no4k = getenv("PERF_TEST_NO4K") != NULL; + if (argc == 4 && !strcmp(argv[1], "--alias")) { + unsigned long page = strtoul(argv[3], NULL, 10); + + CHECK(getauxval(AT_PAGESZ) == page); + errno = 0; + CHECK(mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, + atoi(argv[2]), 0) == MAP_FAILED); + CHECK(errno == EINVAL); + return 0; + } + if (argc == 4 && !strcmp(argv[1], "--case")) { + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + layout(native); + sampling(1); + sampling(2); + sampling(2 * native / getauxval(AT_PAGESZ)); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--no4k")) { + no4k = 1; + CHECK(!setenv("PERF_TEST_NO4K", "1", 1)); + } + if (!strcmp(argv[i], "--compat-only")) + compat_only = 1; + } + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + if (compat_only || sizes[i] > native || + (no4k && sizes[i] == 4096)) + continue; + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, + 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--case", n, s, NULL); + _exit(3); + } + int bad = wait_child(child); + + printf("%s - perf native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, sizes[i]); + failed |= bad; + } + if (compat_only) + CHECK(access("/compat-perf", X_OK) == 0); + if (access("/compat-perf", X_OK) == 0) { + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", no4k ? native : 4096UL); + execl("/compat-perf", "/compat-perf", n, s, NULL); + _exit(3); + } + int bad = wait_child(child); + + printf("%s - perf AArch32 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, no4k ? native : 4096UL); + failed |= bad; + } + printf("PERF RING GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/proc-args.c b/tools/testing/selftests/arm64/user4k/proc-args.c new file mode 100644 index 0000000000000..300742a258863 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-args.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Compare argv/envp with /proc/PID/cmdline and /proc/PID/environ. + * + * Run it once natively and once under the 4K user granule (for example + * "userpages 4k ./proc-args"). The process reads its own files, and a forked + * child reads the parent's files through /proc/. "--hold SECONDS" + * keeps the process alive so a native process can read it from outside. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char **environ; + +static int failures; + +static ssize_t read_file(const char *path, char *buf, size_t size) +{ + ssize_t got, total = 0; + int fd = open(path, O_RDONLY); + + if (fd < 0) + return -errno; + while ((size_t)total < size) { + got = read(fd, buf + total, size - total); + if (got < 0) { + if (errno == EINTR) + continue; + total = -errno; + break; + } + if (!got) + break; + total += got; + } + close(fd); + return total; +} + +/* Join the complete vector; reserve one extra byte in the read buffer. */ +static char *join(char **vec, size_t *size) +{ + size_t len = 0, offset = 0; + char **entry, *buf; + + for (entry = vec; *entry; entry++) { + size_t n = strlen(*entry) + 1; + + if (n > SSIZE_MAX - 1 - len) { + errno = EOVERFLOW; + return NULL; + } + len += n; + } + buf = malloc(len ? len : 1); + if (!buf) + return NULL; + for (entry = vec; *entry; entry++) { + size_t n = strlen(*entry) + 1; + + memcpy(buf + offset, *entry, n); + offset += n; + } + *size = len; + return buf; +} + +static void check(const char *who, const char *name, pid_t pid, char **vec) +{ + char path[64], *want, *got; + size_t want_len; + ssize_t got_len; + + want = join(vec, &want_len); + got = want ? malloc(want_len + 1) : NULL; + if (!got) { + printf("not ok %s %s allocation: %s\n", who, name, + strerror(errno)); + free(want); + failures++; + return; + } + snprintf(path, sizeof(path), "/proc/%d/%s", pid, name); + /* A trailing byte rejects excess data; exact contents must reach EOF. */ + got_len = read_file(path, got, want_len + 1); + if (got_len == (ssize_t)want_len && !memcmp(want, got, want_len)) { + printf("ok %s %s %zd bytes\n", who, path, got_len); + } else { + printf("not ok %s %s want %zu got %zd", who, path, want_len, + got_len); + if (got_len < 0) + printf(" error %s", strerror(-got_len)); + else if (got_len > 0) + printf(" first [%.*s]", + (int)(got_len < 64 ? got_len : 64), got); + printf("\n"); + failures++; + } + free(got); + free(want); +} + +static void show_layout(pid_t pid, char **argv) +{ + char buf[4096], *p, *field; + ssize_t len; + int i; + + printf("pid %d AT_PAGESZ %lu argv[0] %p environ[0] %p\n", pid, + getauxval(AT_PAGESZ), (void *)argv[0], + environ[0] ? (void *)environ[0] : NULL); + len = read_file("/proc/self/stat", buf, sizeof(buf) - 1); + if (len <= 0) + return; + buf[len] = 0; + /* Fields after the parenthesised comm start at field 3. */ + p = strrchr(buf, ')'); + if (!p || p[1] != ' ') + return; + printf("stat arg_start arg_end env_start env_end:"); + for (i = 3, field = strtok(p + 2, " "); field; + field = strtok(NULL, " "), i++) + if (i >= 48 && i <= 51) + printf(" %#lx", strtoul(field, NULL, 10)); + printf("\n"); +} + +int main(int argc, char **argv) +{ + pid_t self = getpid(), child; + int status; + + setbuf(stdout, NULL); + show_layout(self, argv); + check("self", "cmdline", self, argv); + check("self", "environ", self, environ); + + child = fork(); + if (child < 0) + return 1; + if (!child) { + failures = 0; + check("child", "cmdline", self, argv); + check("child", "environ", self, environ); + _exit(failures ? 1 : 0); + } + if (waitpid(child, &status, 0) != child || !WIFEXITED(status) || + WEXITSTATUS(status)) + failures++; + + if (argc > 2 && !strcmp(argv[1], "--hold")) { + fflush(stdout); + sleep(atoi(argv[2])); + } + printf("%s: %d failure(s)\n", failures ? "FAIL" : "PASS", failures); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/arm64/user4k/proc-memory.c b/tools/testing/selftests/arm64/user4k/proc-memory.c new file mode 100644 index 0000000000000..81ca2d0553dd0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-memory.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +struct stats { + unsigned long size, rss, pss, private_clean, private_dirty, + shared_clean, shared_dirty; + unsigned long anonymous, kernel_page, mmu_page; +}; + +static bool get_stats(pid_t pid, uintptr_t address, struct stats *s) +{ + char path[64], line[512]; + unsigned long start, end, value; + unsigned int fields = 0; + char key[64]; + bool found = false; + FILE *f; + + memset(s, 0, sizeof(*s)); + snprintf(path, sizeof(path), "/proc/%d/smaps", pid); + f = fopen(path, "re"); + if (!f) + return false; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + if (found) + break; + found = start == address; + continue; + } + if (!found) + continue; + if (sscanf(line, "%63[^:]: %lu", key, &value) != 2) + continue; + if (!strcmp(key, "Size")) { + s->size = value; + fields |= 1U << 0; + } else if (!strcmp(key, "Rss")) { + s->rss = value; + fields |= 1U << 1; + } else if (!strcmp(key, "Pss")) { + s->pss = value; + fields |= 1U << 2; + } else if (!strcmp(key, "Private_Clean")) { + s->private_clean = value; + fields |= 1U << 3; + } else if (!strcmp(key, "Private_Dirty")) { + s->private_dirty = value; + fields |= 1U << 4; + } else if (!strcmp(key, "Shared_Clean")) { + s->shared_clean = value; + fields |= 1U << 5; + } else if (!strcmp(key, "Shared_Dirty")) { + s->shared_dirty = value; + fields |= 1U << 6; + } else if (!strcmp(key, "Anonymous")) { + s->anonymous = value; + fields |= 1U << 7; + } else if (!strcmp(key, "KernelPageSize")) { + s->kernel_page = value; + fields |= 1U << 8; + } else if (!strcmp(key, "MMUPageSize")) { + s->mmu_page = value; + fields |= 1U << 9; + } + } + fclose(f); + return found && fields == (1U << 10) - 1; +} + +static bool check(pid_t pid, void *address, size_t page, size_t length, + unsigned long pss, unsigned long private, + unsigned long shared, bool anon) +{ + struct stats s; + bool ok = get_stats(pid, (uintptr_t)address, &s); + + ok &= s.size == length / 1024 && s.rss == length / 1024 && + s.pss == pss && s.private_clean + s.private_dirty == private && + s.shared_clean + s.shared_dirty == shared && + s.anonymous == (anon ? length / 1024 : 0) && + s.kernel_page == page / 1024 && s.mmu_page == page / 1024; + if (!ok) + printf("# smaps size=%lu rss=%lu pss=%lu private=%lu shared=%lu anon=%lu pages=%lu/%lu\n", + s.size, s.rss, s.pss, s.private_clean + s.private_dirty, + s.shared_clean + s.shared_dirty, s.anonymous, + s.kernel_page, s.mmu_page); + return ok; +} + +static int check_numa(void *address, size_t page) +{ + char line[1024]; + unsigned long start, anonymous = 0, pages = 0, granule = 0; + bool found = false; + FILE *f = fopen("/proc/self/numa_maps", "re"); + + if (!f) + return errno == ENOENT ? -1 : 0; + while (fgets(line, sizeof(line), f)) { + char *save, *token; + + if (sscanf(line, "%lx", &start) != 1 || + start != (uintptr_t)address) + continue; + found = true; + for (token = strtok_r(line, "\n ", &save); token; + token = strtok_r(NULL, "\n ", &save)) { + unsigned int node; + unsigned long count; + + if (sscanf(token, "anon=%lu", &count) == 1) + anonymous = count; + else if (sscanf(token, "kernelpagesize_kB=%lu", + &count) == 1) + granule = count; + else if (sscanf(token, "N%u=%lu", &node, &count) == 2) + pages += count; + } + break; + } + fclose(f); + if (!found || anonymous != 4 || pages != 4 || granule != page / 1024) + printf("# numa found=%d anon=%lu pages=%lu granule=%lu\n", + found, anonymous, pages, granule); + return found && anonymous == 4 && pages == 4 && granule == page / 1024; +} + +static bool reap(pid_t child) +{ + int status; + + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); +} + +static bool resident_rewrite_reuses(unsigned char *p, size_t page) +{ + uint64_t before[4], after[4]; + int fd = open("/proc/self/pagemap", O_RDONLY); + unsigned int i; + bool ok = fd >= 0; + + if (ok) + ok = pread(fd, before, sizeof(before), + (uintptr_t)p / page * 8) == sizeof(before); + for (i = 0; i < 4; i++) + p[i * page] ^= 0x21; + if (ok) + ok = pread(fd, after, sizeof(after), (uintptr_t)p / page * 8) == + sizeof(after); + for (i = 0; ok && i < 4; i++) { + const uint64_t frame = (1ULL << 55) - 1; + + ok &= (before[i] & frame) && + (before[i] & frame) == (after[i] & frame) && + (before[i] & (1ULL << 63)) && (after[i] & (1ULL << 63)); + } + if (fd >= 0) + close(fd); + return ok; +} + +static void anonymous_sharing(size_t page) +{ + unsigned char *base, *p; + int ready[2], go[2]; + pid_t child; + char byte; + bool ok; + + base = mmap(NULL, 6 * page, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, + 0); + if (base == MAP_FAILED || + mprotect(base + page, 4 * page, PROT_READ | PROT_WRITE) || + pipe(ready) || pipe(go)) + ksft_exit_fail_msg("anonymous fixture setup\n"); + p = base + page; + memset(p, 0x57, 4 * page); + ksft_test_result( + check(getpid(), p, page, 4 * page, 4 * page / 1024, + 4 * page / 1024, 0, true), + "dense anonymous leaves have exact RSS/PSS/private bytes\n"); + { + int numa = check_numa(p, page); + + if (numa < 0) + ksft_test_result_skip( + "NUMA statistics require CONFIG_NUMA\n"); + else + ksft_test_result( + numa, + "NUMA statistics count target leaves and report their granule\n"); + } + child = fork(); + if (!child) { + close(ready[0]); + close(go[1]); + if (write(ready[1], "a", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(1); + p[page] = 0x96; + if (write(ready[1], "b", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(2); + _exit(0); + } + close(ready[1]); + close(go[0]); + ok = child > 0 && read(ready[0], &byte, 1) == 1 && + check(getpid(), p, page, 4 * page, 2 * page / 1024, 0, + 4 * page / 1024, true); + ksft_test_result( + ok, + "fork divides each anonymous leaf's PSS by its own mapcount\n"); + ok = write(go[1], "x", 1) == 1 && read(ready[0], &byte, 1) == 1 && + check(getpid(), p, page, 4 * page, (5 * page) / 2048, page / 1024, + 3 * page / 1024, true) && + p[page] == 0x57; + ksft_test_result( + ok, + "one child COW changes only that leaf's private/shared accounting\n"); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ksft_test_result(reap(child) && check(getpid(), p, page, 4 * page, + 4 * page / 1024, 4 * page / 1024, + 0, true), + "child exit restores full anonymous PSS\n"); + ksft_test_result( + resident_rewrite_reuses(p, page), + "resident writes reuse physical backing after the child exits\n"); + munmap(base, 6 * page); +} + +static int worker(int fd, int output, int input, size_t expected) +{ + size_t page = getauxval(AT_PAGESZ); + unsigned char *p; + uintptr_t addr; + char byte; + + if (page != expected) + return 1; + p = mmap(NULL, page, PROT_READ, MAP_SHARED, fd, + page == 4096 ? 4096 : 0); + if (p == MAP_FAILED || p[0] != 0x78) + return 2; + addr = (uintptr_t)p; + if (write(output, &addr, sizeof(addr)) != sizeof(addr) || + read(input, &byte, 1) != 1) + return 3; + return 0; +} + +static void mixed_file(size_t page) +{ + size_t target = page == 4096 ? 16384 : 4096; + int ready[2], go[2], fd = memfd_create("mixed-pss", 0); + unsigned char data[16384], *p; + uintptr_t remote; + pid_t child; + bool ok; + + memset(data, 0x78, sizeof(data)); + if (fd < 0 || write(fd, data, sizeof(data)) != sizeof(data) || + pipe(ready) || pipe(go)) + ksft_exit_fail_msg("mixed file fixture setup\n"); + p = mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page == 4096 ? 4096 : 0); + if (p == MAP_FAILED || p[0] != 0x78) + ksft_exit_fail_msg("mixed file mmap\n"); + child = fork(); + if (!child) { + char farg[16], out[16], in[16], expected[16]; + + close(ready[0]); + close(go[1]); + snprintf(farg, sizeof(farg), "%d", fd); + snprintf(out, sizeof(out), "%d", ready[1]); + snprintf(in, sizeof(in), "%d", go[0]); + snprintf(expected, sizeof(expected), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, 0UL, 0UL, 0UL)) + _exit(4); + execl("/proc/self/exe", "/proc/self/exe", "--worker", farg, out, + in, expected, NULL); + _exit(5); + } + close(ready[1]); + close(go[0]); + ok = child > 0 && + read(ready[0], &remote, sizeof(remote)) == sizeof(remote); + ok = ok && + check(getpid(), p, page, page, page == 4096 ? 2 : 14, + page == 4096 ? 0 : 12, 4, false) && + check(child, (void *)remote, target, target, + target == 4096 ? 2 : 14, target == 4096 ? 0 : 12, 4, false); + ksft_test_result( + ok, + "mixed file aliases divide only their overlapping physical quarter\n"); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ksft_test_result(reap(child) && + check(getpid(), p, page, page, page / 1024, + page / 1024, 0, false), + "mixed alias exit restores private file accounting\n"); + munmap(p, page); + close(fd); +} + +int main(int argc, char **argv) +{ + size_t page = getauxval(AT_PAGESZ); + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), + strtoul(argv[5], NULL, 10)); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + ksft_print_header(); + ksft_set_plan(8); + anonymous_sharing(page); + mixed_file(page); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/proc-pages.c b/tools/testing/selftests/arm64/user4k/proc-pages.c new file mode 100644 index 0000000000000..d3aaa11101cf2 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-pages.c @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define PRESENT (1ULL << 63) +#define EXCLUSIVE (1ULL << 56) +#define GUARD (1ULL << 58) +#define FRAME ((1ULL << 55) - 1) +#define NPAGE 5 + +struct ready { + uintptr_t address; + size_t page; + uint64_t entries[NPAGE]; + uint64_t shared_entry; +}; + +static int open_proc(pid_t pid, const char *name, int flags) +{ + char path[80]; + + snprintf(path, sizeof(path), "/proc/%d/%s", pid, name); + return open(path, flags); +} + +static bool entries(int fd, uintptr_t address, size_t page, uint64_t *values, + int nr) +{ + return pread(fd, values, nr * sizeof(*values), + address / page * sizeof(*values)) == + nr * (ssize_t)sizeof(*values); +} + +static unsigned char *mapping(size_t page) +{ + const size_t boundary = 32 * 1024 * 1024; + size_t reserve = boundary + 8 * page, head, tail; + unsigned char *base, *p; + + base = mmap(NULL, reserve, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, + 0); + if (base == MAP_FAILED) + return NULL; + /* Cross both native 32 MiB and alternative 2 MiB PTE-table boundaries. */ + p = (void *)(((uintptr_t)base + 2 * page + boundary - 1) & + ~(boundary - 1)); + p -= 2 * page; + head = p - base; + tail = reserve - head - NPAGE * page; + if ((head && munmap(base, head)) || + (tail && munmap(p + NPAGE * page, tail)) || + mprotect(p, NPAGE * page, PROT_READ | PROT_WRITE)) + return NULL; + memset(p, 0x67, NPAGE * page); + if (munmap(p + 2 * page, page) || + madvise(p + 3 * page, page, MADV_GUARD_INSTALL) || + mprotect(p + 4 * page, page, PROT_NONE)) + return NULL; + return p; +} + +static bool scan(int fd, uintptr_t addr, size_t page) +{ + struct page_region vec[8]; + struct pm_scan_arg arg = { + .size = sizeof(arg), + .start = addr, + .end = addr + NPAGE * page, + .vec = (uintptr_t)vec, + .vec_len = 8, + .return_mask = PAGE_IS_PRESENT | PAGE_IS_GUARD, + }; + int nr = ioctl(fd, PAGEMAP_SCAN, &arg); + bool ok = nr == 3 && arg.walk_end == arg.end; + if (!ok) + printf("# scan initial nr=%d errno=%d end_delta=%lld\n", nr, + errno, (long long)(arg.walk_end - arg.end)); + + /* The unmapped middle page produces no VMA result. */ + if (nr == 3) + ok &= vec[0].start == addr && vec[0].end == addr + 2 * page && + vec[0].categories == PAGE_IS_PRESENT && + vec[1].start == addr + 3 * page && + vec[1].end == addr + 4 * page && + vec[1].categories == PAGE_IS_GUARD && + vec[2].start == addr + 4 * page && + vec[2].end == arg.end && + vec[2].categories == PAGE_IS_PRESENT; + arg.max_pages = 1; + arg.walk_end = 0; + nr = ioctl(fd, PAGEMAP_SCAN, &arg); + ok &= nr == 1 && vec[0].start == addr && vec[0].end == addr + page && + arg.walk_end == addr + page; + arg.start = addr + 1; + errno = 0; + ok &= ioctl(fd, PAGEMAP_SCAN, &arg) == -1 && errno == EINVAL; + return ok; +} + +static int remote_worker(int output, int input, size_t expected, int shared_fd) +{ + struct ready ready = { .page = getauxval(AT_PAGESZ) }; + unsigned char *p; + int fd = open_proc(getpid(), "pagemap", O_RDONLY); + char byte; + unsigned char *shared = + mmap(NULL, 16384, PROT_READ, MAP_SHARED, shared_fd, 0); + size_t offset = ready.page == 4096 ? 4096 : 0; + + if (shared == MAP_FAILED || shared[offset] != 0x83 || + !entries(fd, (uintptr_t)shared + offset, ready.page, + &ready.shared_entry, 1)) + return 2; + /* Allocate aliases first so mmap cannot reuse our deliberate hole. */ + p = mapping(ready.page); + ready.address = (uintptr_t)p; + if (ready.page != expected || !p || fd < 0 || + !entries(fd, ready.address, ready.page, ready.entries, NPAGE) || + write(output, &ready, sizeof(ready)) != sizeof(ready) || + read(input, &byte, 1) != 1) + return 1; + return 0; +} + +static bool remote_read(size_t own_page) +{ + int ready_pipe[2], done_pipe[2], status, fd, shared_fd, self_fd; + unsigned char *shared; + uint64_t self_entry; + size_t offset = own_page == 4096 ? 4096 : 0; + size_t target = own_page == 4096 ? 16384 : 4096; + struct ready ready; + uint64_t actual[NPAGE]; + pid_t child; + bool ok; + + shared_fd = memfd_create("mixed-pagemap", 0); + if (shared_fd < 0 || ftruncate(shared_fd, 16384)) + return false; + shared = mmap(NULL, 16384, PROT_READ | PROT_WRITE, MAP_SHARED, + shared_fd, 0); + self_fd = open_proc(getpid(), "pagemap", O_RDONLY); + if (shared == MAP_FAILED || self_fd < 0) + return false; + memset(shared, 0x83, 16384); + if (!entries(self_fd, (uintptr_t)shared + offset, own_page, &self_entry, + 1)) + return false; + close(self_fd); + if (pipe(ready_pipe) || pipe(done_pipe)) + return false; + child = fork(); + if (!child) { + char output[16], input[16], expected[16], shared_arg[16]; + + close(ready_pipe[0]); + close(done_pipe[1]); + snprintf(output, sizeof(output), "%d", ready_pipe[1]); + snprintf(input, sizeof(input), "%d", done_pipe[0]); + snprintf(expected, sizeof(expected), "%zu", target); + snprintf(shared_arg, sizeof(shared_arg), "%d", shared_fd); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, 0UL, 0UL, 0UL)) + _exit(2); + execl("/proc/self/exe", "/proc/self/exe", "--remote", output, + input, expected, shared_arg, NULL); + _exit(3); + } + close(ready_pipe[1]); + close(done_pipe[0]); + ok = child > 0 && + read(ready_pipe[0], &ready, sizeof(ready)) == sizeof(ready); + fd = open_proc(child, "pagemap", O_RDONLY); + ok = ok && ready.page == target && fd >= 0 && + entries(fd, ready.address, target, actual, NPAGE) && + !memcmp(actual, ready.entries, sizeof(actual)) && + scan(fd, ready.address, target); + if (!ok) + printf("# remote geometry/read failure: target=%zu reported=%zu fd=%d errno=%d\n", + target, ready.page, fd, errno); + if (ok) { + size_t target_offset = target == 4096 ? 4096 : 0; + + /* Compare physical byte addresses, not differently sized PFNs. */ + ok &= (self_entry & FRAME) && (ready.shared_entry & FRAME) && + (self_entry & FRAME) * own_page - offset == + (ready.shared_entry & FRAME) * target - + target_offset; + if (!ok) + printf("# shared PFN mismatch: own_nonzero=%d target_nonzero=%d byte_delta=%lld\n", + !!(self_entry & FRAME), + !!(ready.shared_entry & FRAME), + (long long)((self_entry & FRAME) * own_page - + offset - + (ready.shared_entry & FRAME) * + target + + target_offset)); + } + if (fd >= 0) + close(fd); + write(done_pipe[1], "x", 1); + close(done_pipe[1]); + close(ready_pipe[0]); + ok &= child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + munmap(shared, 16384); + close(shared_fd); + return ok; +} + +int main(int argc, char **argv) +{ + size_t page = getauxval(AT_PAGESZ); + unsigned char *p; + uint64_t values[NPAGE], again[NPAGE]; + int fd, clear, i; + bool ok; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--remote")) + return remote_worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10), atoi(argv[5])); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + ksft_print_header(); + ksft_set_plan(6); + p = mapping(page); + fd = open_proc(getpid(), "pagemap", O_RDONLY); + if (!p || fd < 0) + ksft_exit_fail_msg("proc mapping setup: %s\n", strerror(errno)); + ok = entries(fd, (uintptr_t)p, page, values, NPAGE); + ok &= (values[0] & PRESENT) && (values[1] & PRESENT) && !values[2] && + (values[3] & GUARD) && !(values[3] & PRESENT) && + (values[4] & PRESENT); + ok &= (values[0] & EXCLUSIVE) && (values[1] & EXCLUSIVE) && + (values[4] & EXCLUSIVE); + ksft_test_result( + ok, + "pagemap reports each target leaf, hole, guard and PROT_NONE entry\n"); + ok = true; + for (i = 0; i < NPAGE; i++) { + uint64_t value; + + ok &= entries(fd, (uintptr_t)p + i * page, page, &value, 1) && + value == values[i]; + } + errno = 0; + ok &= pread(fd, again, 7, 0) == -1 && errno == EINVAL; + ksft_test_result( + ok, + "pagemap per-entry offsets agree with batch reads and reject short entries\n"); + ksft_test_result( + scan(fd, (uintptr_t)p, page), + "PAGEMAP_SCAN ranges, max_pages and alignment use target granules\n"); + clear = open_proc(getpid(), "clear_refs", O_WRONLY); + ok = clear >= 0; + for (i = 1; ok && i <= 3; i++) { + char operation = '0' + i; + + ok &= write(clear, &operation, 1) == 1; + } + if (clear >= 0) + close(clear); + ok &= entries(fd, (uintptr_t)p, page, again, NPAGE) && + !memcmp(values, again, sizeof(values)) && p[0] == 0x67 && + p[page] == 0x67; + ksft_test_result( + ok, + "clear_refs traverses short and split VMAs without changing mappings or bytes\n"); + ksft_test_result( + remote_read(page), + "opposite-ABI reader uses target pagemap and scan geometry\n"); + { + pid_t child = fork(); + int status; + + if (!child) { + uint64_t value = 0; + int unprivileged; + + if (setuid(1000) || prctl(PR_SET_DUMPABLE, 1)) + _exit(2); + unprivileged = open_proc(getpid(), "pagemap", O_RDONLY); + ok = unprivileged >= 0 && + entries(unprivileged, (uintptr_t)p, page, &value, + 1) && + (value & PRESENT) && !(value & FRAME); + if (!ok) + printf("# unprivileged fd=%d errno=%d flags=%#llx\n", + unprivileged, errno, + (unsigned long long)(value & ~FRAME)); + _exit(!ok); + } + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + ksft_test_result( + ok, + "unprivileged pagemap reads retain flags and hide physical addresses\n"); + } + close(fd); + munmap(p, NPAGE * page); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/reclaim-advice.c b/tools/testing/selftests/arm64/user4k/reclaim-advice.c new file mode 100644 index 0000000000000..92f28d49b830c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/reclaim-advice.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "re"); + char key[128]; + unsigned long value, result = 0; + + if (!f) + return 0; + while (fscanf(f, "%127s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) + result = value; + fclose(f); + return result; +} + +static unsigned long swapped_kb(void) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256]; + unsigned long result = 0; + + if (!f) + return 0; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmSwap: %lu kB", &result) == 1) + break; + fclose(f); + return result; +} + +static int advise(void *addr, size_t length, int operation, const char *name) +{ + int ret = madvise(addr, length, operation); + + if (ret) + printf("# %s failed: errno=%d (%s)\n", name, errno, + strerror(errno)); + return ret; +} + +static unsigned char pattern(size_t i, unsigned int round) +{ + return (i ^ (i >> 12) ^ (i >> 20) ^ (round * 0x55)) & 255; +} + +static size_t swapped_pages(unsigned char *p, size_t length, size_t page) +{ + uint64_t *values = calloc(length / page, sizeof(*values)); + int fd = open("/proc/self/pagemap", O_RDONLY); + size_t i, count = length / page, swapped = 0; + + if (!values || fd < 0 || + pread(fd, values, count * sizeof(*values), + (uintptr_t)p / page * sizeof(*values)) != + (ssize_t)(count * sizeof(*values))) { + swapped = SIZE_MAX; + } else { + for (i = 0; i < count; i++) + swapped += !!(values[i] & (1ULL << 62)) && + !(values[i] & (1ULL << 63)); + } + if (fd >= 0) + close(fd); + free(values); + return swapped; +} + +static void diagnose_resident(unsigned char *p, size_t length, size_t page) +{ + int pm = open("/proc/self/pagemap", O_RDONLY); + int flags = open("/proc/kpageflags", O_RDONLY); + int count = open("/proc/kpagecount", O_RDONLY); + + for (size_t i = 0; i < length / page; i++) { + uint64_t entry = 0, state = 0, maps = 0; + unsigned long native_pfn; + + if (pread(pm, &entry, 8, ((uintptr_t)p / page + i) * 8) != 8 || + !(entry & (1ULL << 63))) + continue; + native_pfn = (entry & ((1ULL << 55) - 1)) * page / 16384; + if (pread(flags, &state, 8, native_pfn * 8) != 8 || + pread(count, &maps, 8, native_pfn * 8) != 8) + printf("# resident metadata read failed: errno=%d\n", + errno); + printf("# resident leaf=%zu native_pfn=%#lx flags=%#llx maps=%llu\n", + i, native_pfn, (unsigned long long)state, + (unsigned long long)maps); + } + close(pm); + close(flags); + close(count); +} + +/* PAGEOUT drains only the calling CPU's pending LRU additions. */ +static bool drain_pageout_cpus(unsigned char *p, size_t length) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(cpu, &one); + if (sched_setaffinity(0, sizeof(one), &one)) { + ok = false; + break; + } + ok &= advise(p, length, MADV_COLD, "CPU drain COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, "CPU drain PAGEOUT") == 0; + } + ok &= sched_setaffinity(0, sizeof(allowed), &allowed) == 0; + return ok; +} + +static bool swapped_pagemap(unsigned char *p, size_t length, size_t page) +{ + uint64_t *values = calloc(length / page, sizeof(*values)); + int fd = open("/proc/self/pagemap", O_RDONLY); + size_t i, count = length / page; + bool ok = values && fd >= 0; + + if (ok) + ok = pread(fd, values, count * sizeof(*values), + (uintptr_t)p / page * sizeof(*values)) == + (ssize_t)(count * sizeof(*values)); + for (i = 0; ok && i < count; i++) { + uint64_t frame = values[i] & ((1ULL << 55) - 1); + + ok &= !!(values[i] & (1ULL << 62)) && + !(values[i] & (1ULL << 63)); + if (page == 4096) { + ok &= ((frame >> 5) & 3) == + (((uintptr_t)p / page + i) & 3); + if (i && (((uintptr_t)p / page + i) & 3)) + ok &= (values[i] & ((1ULL << 55) - 1)) == + (values[i - 1] & ((1ULL << 55) - 1)) + + (1U << 5); + } + } + if (fd >= 0) + close(fd); + free(values); + printf("%s - %zuK pagemap reports swapped leaves and exact quarter offsets\n", + ok ? "ok" : "not ok", page / 1024); + return ok; +} + +static bool check_swap_pss(pid_t pid, void *p, size_t length, + unsigned long expected_swap, + unsigned long expected_pss) +{ + char path[64], line[512]; + unsigned long start, end, swap = 0, pss = 0; + unsigned int fields = 0; + bool found = false, ok; + FILE *f; + + snprintf(path, sizeof(path), "/proc/%d/smaps", pid); + f = fopen(path, "re"); + if (!f) + return false; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + if (found) + break; + found = start == (uintptr_t)p && + end == (uintptr_t)p + length; + continue; + } + if (!found) + continue; + if (sscanf(line, "Swap: %lu", &swap) == 1) + fields |= 1; + else if (sscanf(line, "SwapPss: %lu", &pss) == 1) + fields |= 2; + } + fclose(f); + ok = found && fields == 3 && swap == expected_swap && + pss == expected_pss; + if (!ok) + printf("# swap stats found=%d fields=%u Swap=%lu/%lu SwapPss=%lu/%lu\n", + found, fields, swap, expected_swap, pss, expected_pss); + return ok; +} + +static bool swapped_accounting(unsigned char *p, size_t length, size_t page, + bool forked) +{ + int ready[2], go[2], status; + pid_t child; + char byte; + bool ok = check_swap_pss(getpid(), p, length, length / 1024, + length / 1024); + + if (!forked || !ok) + goto out; + if (pipe(ready) || pipe(go)) + return false; + child = fork(); + if (child < 0) + return false; + if (!child) { + close(ready[0]); + close(go[1]); + if (read(go[0], &byte, 1) != 1) + _exit(1); + p[page] ^= 0x37; + if (write(ready[1], "x", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(2); + _exit(0); + } + close(ready[1]); + close(go[0]); + ok &= check_swap_pss(getpid(), p, length, length / 1024, length / 2048); + ok &= check_swap_pss(child, p, length, length / 1024, length / 2048); + ok &= write(go[1], "x", 1) == 1 && read(ready[0], &byte, 1) == 1; + ok &= check_swap_pss(getpid(), p, length, length / 1024, + (length + page) / 2048); + ok &= check_swap_pss(child, p, length, (length - page) / 1024, + (length - page) / 2048); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ok &= waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status); + ok &= check_swap_pss(getpid(), p, length, length / 1024, length / 1024); +out: + printf("%s - %zuK SwapPss %s\n", ok ? "ok" : "not ok", page / 1024, + forked ? "fork, one-leaf swap-in/COW and child exit" : + "exclusive quarters"); + return ok; +} + +int main(int argc, char **argv) +{ + const size_t length = 4 * 1024 * 1024; + size_t page = getauxval(AT_PAGESZ), count = length / page, i, j, absent; + unsigned char *p, *vec, *reservation; + unsigned long before, after, swapped_before, swapped_after; + unsigned int round, tries; + size_t bad_bytes, bad_residency; + bool ok = true; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + reservation = mmap(NULL, length + 2 * page, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reservation == MAP_FAILED) + return 3; + p = reservation + page; + vec = malloc(count); + if (!vec || mprotect(p, length, PROT_READ | PROT_WRITE)) + return 3; + for (round = 0; round < 4 && ok; round++) { + for (i = 0; i < length; i++) + p[i] = pattern(i, round); + before = swapouts(); + swapped_before = swapped_kb(); + ok = advise(p, length, MADV_COLD, "COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, "PAGEOUT") == 0; + after = before; + absent = 0; + for (tries = 0; ok && tries < 200; tries++) { + if (mincore(p, length, vec)) { + ok = false; + break; + } + absent = 0; + for (i = 0; i < count; i++) + absent += !(vec[i] & 1); + after = swapouts(); + if (after > before && + swapped_pages(p, length, page) == count) + break; + if (tries == 32) { + printf("# checking pending remote LRU additions before CPU sweep\n"); + diagnose_resident(p, length, page); + ok &= drain_pageout_cpus(p, length); + printf("# after CPU sweep: swapped=%zu/%zu\n", + swapped_pages(p, length, page), count); + } + /* PAGEOUT may skip busy folios; establish the full-swap fixture. */ + usleep(10000); + ok &= advise(p, length, MADV_COLD, "retry COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, + "retry PAGEOUT") == 0; + } + printf("# pageout setup attempts=%u\n", tries + 1); + if (tries == 200) + diagnose_resident(p, length, page); + ok &= tries < 200; + swapped_after = swapped_kb(); + /* mincore also counts warm swap-cache data as resident. */ + ok &= after > before && swapped_after > swapped_before; + ok &= swapped_pagemap(p, length, page); + ok &= swapped_accounting(p, length, page, round == 1); + if (!round) + ok &= advise(p + page, page, MADV_WILLNEED, + "WILLNEED") == 0; + else { + if (round == 3) { + ok &= mlock(p, length) == 0; + ok &= swapped_pages(p, length, page) == 0; + } else { + ok &= advise(p, length, MADV_POPULATE_READ, + "POPULATE_READ") == 0; + } + ok &= mincore(p, length, vec) == 0; + bad_residency = 0; + for (i = 0; i < count; i++) + bad_residency += !(vec[i] & 1); + printf("# after %s: absent=%zu/%zu\n", + round == 3 ? "mlock" : "POPULATE_READ", + bad_residency, count); + ok &= bad_residency == 0; + } + bad_bytes = 0; + /* Nonsequential user-page order, every byte checked after swap. */ + for (i = 0; i < count; i++) { + size_t index = ((i * 61) & (count - 1)) * page; + + for (j = index; j < index + page; j++) + if (p[j] != pattern(j, round)) { + if (!bad_bytes) + printf("# first bad byte at %#zx: got=%#x expected=%#x\n", + j, p[j], + pattern(j, round)); + bad_bytes++; + ok = false; + } + } + if (round == 3) + ok &= munlock(p, length) == 0; + printf("# bad bytes=%zu\n", bad_bytes); + printf("%s - %zuK EL0 pageout round %u: pswpout=%lu->%lu VmSwap=%lu->%lu kB absent=%zu/%zu %s\n", + ok ? "ok" : "not ok", page / 1024, round, before, after, + swapped_before, swapped_after, absent, count, + round == 3 ? "mlock/readback" : + round ? "prefault/readback" : + "willneed/readback"); + } + munmap(reservation, length + 2 * page); + free(vec); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/runtime-app-init.c b/tools/testing/selftests/arm64/user4k/runtime-app-init.c new file mode 100644 index 0000000000000..9d1c7c2c49351 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/runtime-app-init.c @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +int main(int argc, char **argv) +{ + if (argc >= 4 && !strcmp(argv[1], "--exec")) { + if (prctl(SET_EXEC_PAGE_SIZE, strtoul(argv[2], NULL, 10), 0UL, + 0UL, 0UL)) + return 8; + execv(argv[3], argv + 3); + perror("exec"); + return 9; + } + if (getpid() != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + int failed = 0; + pid_t children[3]; + unsigned long sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + unsigned count = sizes[0] > 16384 ? 3 : 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + failed = 1; + goto done; + } + mkdir("/dev/shm", 0777); + mount("tmpfs", "/dev/shm", "tmpfs", 0, NULL); + for (unsigned i = 0; i < count; i++) { + children[i] = fork(); + if (!children[i]) { + char ps[32]; + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(3); + execl("/usr/bin/python3", "/usr/bin/python3", "-I", + "/runtime-app.py", ps, NULL); + perror("exec Python"); + _exit(4); + } + } + for (unsigned i = 0; i < count; i++) { + int status = 0; + int ok = children[i] > 0 && + waitpid(children[i], &status, 0) == children[i] && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - distro Python ABI %luK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + } +done: + if (failed) { + FILE *f = fopen("/trace4k", "r"); + if (f) { + char line[2048]; + while (fgets(line, sizeof(line), f)) + fputs(line, stdout); + fclose(f); + } + } + printf("GRANULE DISTRO RUNTIME %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/runtime-app.py b/tools/testing/selftests/arm64/user4k/runtime-app.py new file mode 100755 index 0000000000000..47fac03300efc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/runtime-app.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: GPL-2.0-only +import os,sys,mmap,ctypes,threading,concurrent.futures,tempfile,subprocess,signal,zlib,gzip,lzma,hashlib,socket,select,json,sqlite3 +ps=int(sys.argv[1]);assert os.sysconf('SC_PAGE_SIZE')==ps==mmap.PAGESIZE,(ps,mmap.PAGESIZE) +libc=ctypes.CDLL(None,use_errno=True);libc.prctl.argtypes=[ctypes.c_int,ctypes.c_ulong,ctypes.c_ulong,ctypes.c_ulong,ctypes.c_ulong] +def passed(name):print(f'ok - distro Python {ps//1024}K {name}',flush=True) +passed('dynamic loader and page-size contract') +with mmap.mmap(-1, 2*ps, flags=0x08|mmap.MAP_ANONYMOUS) as droppable: + assert droppable[:]==bytes(2*ps) + droppable[:]=bytes([0x45])*(2*ps) + child=os.fork() + if child==0:os._exit(0 if droppable[:]==bytes(2*ps) else 9) + assert os.waitpid(child,0)[1]==0 + droppable.madvise(mmap.MADV_DONTNEED) + assert droppable[:]==bytes(2*ps) +libc.arc4random_buf.argtypes=[ctypes.c_void_p,ctypes.c_size_t] +random_buffer=ctypes.create_string_buffer(32) +random_samples=set() +for _ in range(128): + libc.arc4random_buf(random_buffer,len(random_buffer));random_samples.add(random_buffer.raw) +assert len(random_samples)==128 +passed('droppable write, fork wipe, discard and glibc random state') + +blob=bytes(range(256))*4096 +with tempfile.TemporaryDirectory() as tmp: + path=tmp+'/mapped' + with open(path,'w+b') as f: + f.truncate(8*ps) + with mmap.mmap(f.fileno(),2*ps,offset=ps) as m: + m[:]=bytes([0x53])*(2*ps);m.flush() + assert os.pread(f.fileno(),2*ps,ps)==m[:] + m.resize(3*ps);m[2*ps:]=bytes([0x72])*ps;m.flush() + assert os.pread(f.fileno(),ps,3*ps)==bytes([0x72])*ps + passed('file mmap offset, flush and resize') + with mmap.mmap(-1,4*ps,flags=mmap.MAP_PRIVATE|mmap.MAP_ANONYMOUS) as m: + m[:]=bytes([0x39])*(4*ps) + c=os.fork() + if c==0: + m[ps:2*ps]=bytes([0x91])*ps + os._exit(0 if m[:ps]==bytes([0x39])*ps else 5) + assert os.waitpid(c,0)[1]==0 and m[:]==bytes([0x39])*(4*ps) + address=ctypes.addressof(ctypes.c_char.from_buffer(m)) + libc.mprotect.argtypes=[ctypes.c_void_p,ctypes.c_size_t,ctypes.c_int] + assert libc.mprotect(address+ps,ps,0)==0 + c=os.fork() + if c==0: + m[ps]=0 + os._exit(6) + status=os.waitpid(c,0)[1];assert os.WIFSIGNALED(status) and os.WTERMSIG(status)==signal.SIGSEGV + assert m[0]==0x39 and m[2*ps]==0x39 + assert libc.mprotect(address+ps,ps,mmap.PROT_READ|mmap.PROT_WRITE)==0 + passed('fork COW and independent protection fault') + def compressed(i): + d=blob+str(i).encode() + for compress,decompress in [(zlib.compress,zlib.decompress),(gzip.compress,gzip.decompress),(lambda d:lzma.compress(d,preset=1),lzma.decompress)]:assert decompress(compress(d))==d + return hashlib.sha256(d).hexdigest() + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + results=list(pool.map(compressed,range(8))) + assert len(set(results))==8 + passed('threaded native extension allocation, compression and hashing') + db=sqlite3.connect(tmp+'/db') + db.execute('pragma journal_mode=wal');db.execute('pragma mmap_size=1048576') + db.execute('create table entries (n integer primary key, data blob)') + db.executemany('insert into entries values (?,?)',[(i,blob[:4096]) for i in range(128)]);db.commit() + assert db.execute('select count(*) from entries').fetchone()[0]==128 + assert all(row[0]==blob[:4096] for row in db.execute('select data from entries'));db.close() + passed('SQLite WAL and mapped database reads') + left,right=socket.socketpair() + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + task=pool.submit(lambda:(right.sendall(blob),right.shutdown(socket.SHUT_WR))) + out=bytearray() + while True: + assert select.select([left],[],[],5)[0] + part=left.recv(997) + if not part:break + out.extend(part) + task.result();assert bytes(out)==blob + left.close();right.close();passed('socket polling and fragmented transfers') + fd=os.memfd_create('cross-abi',0);os.ftruncate(fd,131072);os.pwrite(fd,b'parent',ps) + # Execute the child in the other ABI, retaining the same shared file bytes. + target=16384 if ps==4096 else 4096 + def select_abi(): + if libc.prctl(0x41555001,target,0,0,0):os._exit(7) + code="import os,mmap,sys;fd=int(sys.argv[1]);off=int(sys.argv[2]);want=int(sys.argv[3]);assert os.sysconf('SC_PAGE_SIZE')==want;m=mmap.mmap(fd,131072);assert m[off:off+6]==b'parent';m[off:off+5]=b'child';m.close()" + subprocess.run([sys.executable,'-I','-c',code,str(fd),str(ps),str(target)],pass_fds=(fd,),preexec_fn=select_abi,check=True) + assert os.pread(fd,5,ps)==b'child';os.close(fd);passed('cross-ABI dynamic exec and memfd sharing') + command="set -e; a=(); for ((i=0;i<2048;i++)); do a+=(\"value-$i\"); done; [[ ${#a[@]} == 2048 ]]; [[ ${a[2047]} == value-2047 ]]; printf 'bash-ok\\n'" + def same_abi(): + if libc.prctl(0x41555001,ps,0,0,0):os._exit(8) + result=subprocess.run(['/bin/bash','--noprofile','--norc','-c',command],preexec_fn=same_abi,stdout=subprocess.PIPE,check=True,text=True) + assert result.stdout=='bash-ok\n';passed('distro Bash dynamic exec, allocation and pipes') +print(f'DISTRO PYTHON {ps} PASS',flush=True) diff --git a/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c b/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c new file mode 100644 index 0000000000000..b040d8fc43a4f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c @@ -0,0 +1,396 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#ifndef MADV_COLLAPSE +#define MADV_COLLAPSE 25 +#endif +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - collapse line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct command { + uint64_t op, offset, len, value; +}; +struct ready { + uint64_t base, head; +}; +static uint64_t entry(const char *path, uint64_t index) +{ + uint64_t value; + int fd = open(path, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &value, 8, index * 8) == 8 && !close(fd)); + return value; +} +static uint64_t mapped_pfn(pid_t pid, uintptr_t address, size_t ps) +{ + char path[80]; + snprintf(path, sizeof(path), "/proc/%d/pagemap", pid); + uint64_t e = entry(path, address / ps); + CHECK(e & (1ULL << 63)); + return e & ((1ULL << 55) - 1); +} +static unsigned char *map(int file, size_t len, size_t align, off_t offset) +{ + void *reserve = mmap(NULL, len + align, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + uintptr_t addr = ((uintptr_t)reserve + align - 1) & ~(align - 1); + CHECK(!munmap(reserve, len + align)); + unsigned char *p = mmap((void *)addr, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, offset); + CHECK(p != MAP_FAILED); + return p; +} +static int register_range(unsigned char *p, size_t len, int minor) +{ + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = minor ? UFFD_FEATURE_MINOR_SHMEM : + 0 }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register r = { + .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + (minor ? UFFDIO_REGISTER_MODE_MINOR : 0) + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, &r)); + return fd; +} +static void copy_leaf(int fd, uintptr_t dst, unsigned char value) +{ + unsigned char src[4096]; + memset(src, value, sizeof(src)); + struct uffdio_copy c = { .dst = dst, + .src = (uintptr_t)src, + .len = sizeof(src) }; + CHECK(!ioctl(fd, UFFDIO_COPY, &c) && c.copy == sizeof(src)); +} +static void cpu(unsigned id) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(id, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +static int worker(int file, int sock, size_t native, size_t len) +{ + CHECK(getauxval(AT_PAGESZ) == 4096); + alarm(90); + cpu(1); + unsigned char *p = map(file, len, len, 0); + unsigned char *q = map(file, len - 4096, len, 4096); + CHECK(p[0] == 0x61 && q[0] == 0x61); + uint64_t old_head = + mapped_pfn(getpid(), (uintptr_t)p, 4096) * 4096 / native; + CHECK(entry("/proc/kpageflags", old_head) & + (1ULL << KPF_COMPOUND_HEAD)); + int fd = register_range(p, len, 1); + for (size_t i = 4 * native; i < len; i += native) + copy_leaf(fd, (uintptr_t)p + i, 0x71); + int pipes[2]; + CHECK(!pipe(pipes)); + struct iovec iov = { p, 4096 }; + CHECK(vmsplice(pipes[1], &iov, 1, 0) == 4096); + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + native, native)); + CHECK(entry("/proc/kpageflags", old_head) & + (1ULL << KPF_COMPOUND_HEAD)); + CHECK(!close(pipes[0]) && !close(pipes[1])); + struct ready r = { (uintptr_t)p, old_head }; + struct iovec v = { &r, sizeof(r) }; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(fd))]; + } control = { 0 }; + struct msghdr msg = { .msg_iov = &v, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control.bytes) }; + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + c->cmsg_level = SOL_SOCKET; + c->cmsg_type = SCM_RIGHTS; + c->cmsg_len = CMSG_LEN(sizeof(fd)); + memcpy(CMSG_DATA(c), &fd, sizeof(fd)); + CHECK(sendmsg(sock, &msg, 0) == sizeof(r)); + struct command cmd; + while (recv(sock, &cmd, sizeof(cmd), 0) == sizeof(cmd)) { + CHECK(cmd.offset < len && cmd.len <= len - cmd.offset); + unsigned char value = 0; + if (!cmd.op) + break; + if (cmd.op == 1) + value = *(volatile unsigned char *)(p + cmd.offset); + else if (cmd.op == 2) + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_KEEP_SIZE, + cmd.offset, cmd.len)); + else if (cmd.op == 3) { + CHECK(cmd.offset >= 4096); + value = *(volatile unsigned char *)(q + cmd.offset - + 4096); + } else if (cmd.op == 4) { + CHECK(cmd.offset >= 4096); + memset(q + cmd.offset - 4096, cmd.value, cmd.len); + } else + CHECK(0); + CHECK(send(sock, &value, 1, 0) == 1); + } + CHECK(!munmap(q, len - 4096) && !munmap(p, len) && !close(fd) && + !close(file) && !close(sock)); + return 0; +} +static void send_command(int sock, uint64_t op, size_t offset, size_t len) +{ + struct command c = { .op = op, .offset = offset, .len = len }; + CHECK(send(sock, &c, sizeof(c), 0) == sizeof(c)); +} +static void reply(int sock, unsigned char expected) +{ + unsigned char value; + CHECK(recv(sock, &value, 1, 0) == 1 && value == expected); +} +static void fault(int sock, int fd, uintptr_t base, size_t offset, int minor, + unsigned char value) +{ + send_command(sock, 1, offset + 17, 0); + struct pollfd p = { .fd = fd, .events = POLLIN }; + struct uffd_msg m; + CHECK(poll(&p, 1, 10000) == 1 && (p.revents & POLLIN)); + CHECK(read(fd, &m, sizeof(m)) == sizeof(m)); + CHECK(m.event == UFFD_EVENT_PAGEFAULT && + m.arg.pagefault.address == base + offset && + m.arg.pagefault.flags == (minor ? UFFD_PAGEFAULT_FLAG_MINOR : 0)); + if (minor) { + struct uffdio_continue c = { .range = { base + offset, 4096 } }; + CHECK(!ioctl(fd, UFFDIO_CONTINUE, &c) && c.mapped == 4096); + } else + copy_leaf(fd, base + offset, value); + reply(sock, value); +} +static unsigned long pmd_mapped(unsigned char *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + CHECK(f); + char *line = NULL; + size_t cap = 0; + unsigned long a, b, kb, result = 0; + int selected = 0; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &a, &b) == 2) + selected = a == (uintptr_t)p; + else if (selected && + sscanf(line, "ShmemPmdMapped: %lu kB", &kb) == 1) + result = kb * 1024; + } + free(line); + fclose(f); + return result; +} +static int collapse(unsigned char *p, size_t len) +{ + int ret; + unsigned tries = 0; + do { + errno = 0; + ret = madvise(p, len, MADV_COLLAPSE); + if (!ret || errno != EAGAIN) + break; + usleep(20000); + } while (++tries < 10); + printf("# collapse ret=%d errno=%d PMD-mapped=%lu\n", ret, + ret ? errno : 0, pmd_mapped(p)); + return ret; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10), + strtoul(argv[5], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + alarm(120); + cpu(0); + CHECK(!mount("proc", "/proc", "proc", 0, NULL) && + !mount("sysfs", "/sys", "sysfs", 0, NULL)); + CHECK(!mount("tmpfs", "/tmp", "tmpfs", 0, + "huge=within_size,size=256m")); + size_t native = getauxval(AT_PAGESZ), len = 0; + FILE *f = fopen("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", + "r"); + CHECK(f && fscanf(f, "%zu", &len) == 1); + fclose(f); + CHECK(native == 16384 && len == 32 * 1024 * 1024); + int file = open("/tmp/collapse", O_CREAT | O_RDWR | O_TRUNC, 0600); + CHECK(file >= 0 && !ftruncate(file, 4 * native)); + unsigned char *initial = map(file, 4 * native, 4 * native, 0); + memset(initial, 0x61, 4 * native); + CHECK(entry("/proc/kpageflags", + mapped_pfn(getpid(), (uintptr_t)initial, native)) & + (1ULL << KPF_COMPOUND_HEAD)); + CHECK(!munmap(initial, 4 * native) && !unlink("/tmp/collapse") && + !ftruncate(file, len)); + int sock[2]; + CHECK(!socketpair(AF_UNIX, SOCK_SEQPACKET, 0, sock)); + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char fd[24], so[24], ns[24], ls[24]; + close(sock[0]); + snprintf(fd, sizeof(fd), "%d", file); + snprintf(so, sizeof(so), "%d", sock[1]); + snprintf(ns, sizeof(ns), "%zu", native); + snprintf(ls, sizeof(ls), "%zu", len); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, 4096, 0UL, 0UL, 0UL)); + execl("/init", "/init", "--worker", fd, so, ns, ls, NULL); + _exit(3); + } + close(sock[1]); + struct ready ready; + struct iovec v = { &ready, sizeof(ready) }; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct msghdr msg = { .msg_iov = &v, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control.bytes) }; + CHECK(recvmsg(sock[0], &msg, 0) == sizeof(ready) && + !(msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))); + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + CHECK(c && c->cmsg_level == SOL_SOCKET && c->cmsg_type == SCM_RIGHTS); + int uffd; + memcpy(&uffd, CMSG_DATA(c), sizeof(uffd)); + unsigned char *p = map(file, len, len, 0); + int native_uffd = register_range(p, len, 0); + CHECK(collapse(p, len) == -1 && errno == EINVAL && pmd_mapped(p) == 0); + fault(sock[0], uffd, ready.base, 0, 1, 0x61); + fault(sock[0], uffd, ready.base, len - native, 1, 0x71); + uint64_t head = mapped_pfn(pid, ready.base, 4096) * 4096 / native; + CHECK(head != ready.head && + (entry("/proc/kpageflags", head) & (1ULL << KPF_COMPOUND_HEAD))); + CHECK(mapped_pfn(pid, ready.base + len - native, 4096) * 4096 / + native == + head + len / native - 1); + for (size_t i = 1; i < len / native; i++) + CHECK(entry("/proc/kpageflags", head + i) & + (1ULL << KPF_COMPOUND_TAIL)); + /* Collapse retained the absent native unit and independent fine holes. */ + fault(sock[0], uffd, ready.base, native, 0, 0x92); + fault(sock[0], uffd, ready.base, 5 * native + 4096, 0, 0xa3); + puts("ok - PMD cache collapse preserves missing/minor state and neighbour data"); + /* Every native leaf now contains data: coarse exposure may populate zero subunits. */ + CHECK(!collapse(p, len) && pmd_mapped(p) == len); + fault(sock[0], uffd, ready.base, 6 * native + 4096, 1, 0); + struct uffdio_copy duplicate = { .dst = ready.base + 6 * native + 8192, + .src = (uintptr_t)p, + .len = 4096 }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &duplicate) == -1 && errno == EEXIST); + puts("ok - native PMD promotion exposes coarse leaf zeros without overwriting fine data"); + /* A new fine hole must invalidate the covering native PMD mapping. */ + send_command(sock[0], 2, 7 * native + 4096, 4096); + reply(sock[0], 0); + CHECK(pmd_mapped(p) == 0); + fault(sock[0], uffd, ready.base, 7 * native + 4096, 0, 0xb4); + CHECK(p[7 * native] == 0x71 && p[7 * native + 4096] == 0xb4 && + p[7 * native + 8192] == 0); + CHECK(p[native] == 0x92 && p[5 * native + 4096] == 0xa3); + puts("ok - fine hole splits PMD alias and preserves populated neighbouring leaves"); + for (unsigned round = 0; round < 8; round++) { + CHECK(!collapse(p, len) && pmd_mapped(p) == len); + size_t off = (16 + round) * native + 4096; + /* Prime CPU0's PMD translation before CPU1 punches the fine hole. */ + CHECK(*(volatile unsigned char *)(p + off) == 0); + send_command(sock[0], 3, off + 17, 0); + reply(sock[0], 0); + send_command(sock[0], 2, off, 4096); + reply(sock[0], 0); + CHECK(pmd_mapped(p) == 0); + if (round % 3) { + /* Stale native or shifted fine translations would miss promotion. */ + if (round % 3 == 1) + memset(p + off, 0xc0 + round, 4096); + else { + struct command write = { 4, off, 4096, + 0xc0 + round }; + CHECK(send(sock[0], &write, sizeof(write), 0) == + sizeof(write)); + reply(sock[0], 0); + } + struct uffdio_copy collision = { .dst = ready.base + + off, + .src = (uintptr_t)p, + .len = 4096 }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &collision) == -1 && + errno == EEXIST); + fault(sock[0], uffd, ready.base, off, 1, 0xc0 + round); + } else + fault(sock[0], uffd, ready.base, off, 0, 0xc0 + round); + CHECK(p[off - 4096] == 0x71 && p[off] == 0xc0 + round && + p[off + 4096] == 0); + send_command(sock[0], 3, off + 17, 0); + reply(sock[0], 0xc0 + round); + } + CHECK(!collapse(p, len) && pmd_mapped(p) == len); + for (size_t i = 0; i < len; i++) { + size_t page = i / native, low = i % native; + unsigned char expected = + page < 4 ? + (page == 1 ? (low < 4096 ? 0x92 : 0) : 0x61) : + (low < 4096 ? 0x71 : 0); + if (low >= 4096 && low < 8192) { + if (page == 5) + expected = 0xa3; + if (page == 7) + expected = 0xb4; + if (page >= 16 && page < 24) + expected = 0xc0 + page - 16; + } + CHECK(p[i] == expected); + } + puts("ok - eight cross-CPU PMD cycles, native/shifted-fine writes and complete 32MiB bytes"); + send_command(sock[0], 0, 0, 0); + int status; + CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!close(uffd) && !close(native_uffd) && !munmap(p, len) && + !close(file) && !close(sock[0])); + puts("SHMEM COLLAPSE CONTRACT PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-hole-init.c b/tools/testing/selftests/arm64/user4k/shmem-hole-init.c new file mode 100644 index 0000000000000..8831cc30a22c3 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-hole-init.c @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem hole line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct access { + volatile unsigned char *p; + unsigned char value; +}; +static void *read_byte(void *opaque) +{ + struct access *a = opaque; + a->value = *a->p; + return NULL; +} +static uint64_t pagemap(pid_t pid, uintptr_t address, size_t granule) +{ + char name[80]; + uint64_t entry; + snprintf(name, sizeof(name), "/proc/%d/pagemap", pid); + int fd = open(name, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &entry, 8, address / granule * 8) == 8); + CHECK(!close(fd)); + return entry; +} +static void fill(int fd, unsigned char *dst, unsigned char *src, size_t ps) +{ + struct uffdio_copy c = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = ps }; + CHECK(!ioctl(fd, UFFDIO_COPY, &c) && c.copy == (long long)ps); +} +static void resolve(int fd, unsigned char *p, size_t ps, int minor, + unsigned char value) +{ + struct access a = { .p = p + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &a)); + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + struct uffd_msg msg; + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == + (minor ? UFFD_PAGEFAULT_FLAG_MINOR : 0)); + if (minor) { + struct uffdio_continue c = { .range = { (uintptr_t)p, ps } }; + CHECK(!ioctl(fd, UFFDIO_CONTINUE, &c) && + c.mapped == (long long)ps); + } else { + unsigned char *source = malloc(ps); + CHECK(source); + memset(source, value, ps); + fill(fd, p, source, ps); + free(source); + } + CHECK(!pthread_join(thread, NULL) && a.value == value); +} +struct command { + uint64_t op, offset, len, value; +}; +static int alias_worker(int file, int input, int output, size_t len, + size_t native) +{ + CHECK(getauxval(AT_PAGESZ) == native); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + CHECK(p != MAP_FAILED); + /* Establish every coarse PTE before the fine process punches its hole. */ + for (size_t i = 0; i < len; i++) + CHECK(p[i] == 0x61); + uint64_t base = (uintptr_t)p; + CHECK(write(output, &base, sizeof(base)) == sizeof(base)); + struct command cmd; + while (read(input, &cmd, sizeof(cmd)) == sizeof(cmd)) { + CHECK(cmd.offset <= len && cmd.len <= len - cmd.offset); + if (!cmd.op) + break; + if (cmd.op == 2) + memset(p + cmd.offset, cmd.value, cmd.len); + for (size_t i = 0; i < cmd.len; i++) + CHECK(p[cmd.offset + i] == cmd.value); + CHECK(write(output, "y", 1) == 1); + } + CHECK(!munmap(p, len)); + return 0; +} +static void tell(int input, int output, uint64_t op, size_t offset, size_t len, + unsigned value) +{ + struct command cmd = { op, offset, len, value }; + char reply; + CHECK(write(input, &cmd, sizeof(cmd)) == sizeof(cmd)); + if (op) + CHECK(read(output, &reply, 1) == 1 && reply == 'y'); +} +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "r"); + char key[80]; + unsigned long value, result = 0; + CHECK(f); + while (fscanf(f, "%79s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) + result = value; + fclose(f); + return result; +} +static unsigned long mapping_swap(void *address) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end, value, result = 0; + int selected = 0; + CHECK(f); + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + selected = start == (uintptr_t)address; + else if (selected && sscanf(line, "Swap: %lu kB", &value) == 1) + result = value; + } + free(line); + fclose(f); + return result; +} +static void swapped_missing(size_t ps, size_t native) +{ + int file = memfd_create("partial-uffd-swap", 0); + CHECK(file >= 0 && !ftruncate(file, native)); + unsigned char *p = + mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *source = malloc(ps); + CHECK(p != MAP_FAILED && source); + memset(source, 0x71, ps); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MINOR_SHMEM }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, native }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_MINOR }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + fill(fd, p, source, ps); + unsigned long before = swapouts(), usage = 0, written; + unsigned tries; + for (tries = 0; tries < 100; tries++) { + CHECK(!madvise(p, native, MADV_PAGEOUT)); + usleep(20000); + usage = mapping_swap(p); + if (usage >= native / 1024 && swapouts() > before) + break; + } + written = swapouts() - before; + CHECK(usage >= native / 1024 && written > 0); + /* The selected cache index is swapped; its unfilled neighbour stays missing. */ + resolve(fd, p + ps, ps, 0, 0x92); + resolve(fd, p, ps, 1, 0x71); + for (size_t i = 0; i < ps; i++) + CHECK(p[i] == 0x71 && p[ps + i] == 0x92); + CHECK(mapping_swap(p) == 0); + printf("ok - shared uffd %zuK on %zuK swapped cache=%lukB writes=%lu attempts=%u missing COPY preserves neighbour\n", + ps / 1024, native / 1024, usage, written, tries + 1); + CHECK(!close(fd) && !munmap(p, native) && !close(file)); + free(source); +} +static int worker(size_t ps, size_t native) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(60); + if (getenv("SHMEM_HOLE_SWAP") && ps < native) + swapped_missing(ps, native); + size_t len = 4 * native; + int file = memfd_create("hole-contract", 0); + CHECK(file >= 0 && !ftruncate(file, len)); + unsigned char *source = malloc(len); + CHECK(source); + memset(source, 0x61, len); + CHECK(pwrite(file, source, len, 0) == (ssize_t)len); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *q = mmap(NULL, len - ps, PROT_READ | PROT_WRITE, + MAP_SHARED, file, ps); + unsigned char *cow = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, file, 0); + CHECK(p != MAP_FAILED && q != MAP_FAILED && cow != MAP_FAILED); + for (size_t i = 0; i < len; i += ps) + CHECK(p[i] == 0x61); + for (size_t i = 0; i < len - ps; i += ps) + CHECK(q[i] == 0x61); + memset(cow + ps, 0xe1, ps); + int commands[2], replies[2]; + CHECK(!pipe(commands) && !pipe(replies)); + pid_t alias = fork(); + CHECK(alias >= 0); + if (!alias) { + char f[24], in[24], out[24], size[24], ns[24]; + close(commands[1]); + close(replies[0]); + snprintf(f, sizeof(f), "%d", file); + snprintf(in, sizeof(in), "%d", commands[0]); + snprintf(out, sizeof(out), "%d", replies[1]); + snprintf(size, sizeof(size), "%zu", len); + snprintf(ns, sizeof(ns), "%zu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, native, 0UL, 0UL, 0UL)); + execl("/init", "/init", "--alias", f, in, out, size, ns, NULL); + _exit(3); + } + close(commands[0]); + close(replies[1]); + uint64_t alias_base; + CHECK(read(replies[0], &alias_base, sizeof(alias_base)) == + sizeof(alias_base)); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MINOR_SHMEM }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_MINOR }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(pagemap(alias, alias_base + ps, native) & (1ULL << 63)); + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, + ps)); + CHECK(!(pagemap(getpid(), (uintptr_t)p + ps, ps) & (1ULL << 63))); + CHECK(!(pagemap(getpid(), (uintptr_t)q, ps) & (1ULL << 63))); + CHECK(!(pagemap(alias, alias_base + ps, native) & (1ULL << 63))); + /* Fine neighbour PTEs survive, while COW retains its independent bytes. */ + CHECK(pagemap(getpid(), (uintptr_t)p, ps) & (1ULL << 63)); + CHECK(pagemap(getpid(), (uintptr_t)p + 2 * ps, ps) & (1ULL << 63)); + CHECK(p[0] == 0x61 && p[2 * ps] == 0x61 && cow[ps] == 0xe1); + resolve(fd, p + ps, ps, 0, 0xa2); + CHECK(q[0] == 0xa2 && cow[ps] == 0xe1); + tell(commands[1], replies[0], 1, ps, ps, 0xa2); + tell(commands[1], replies[0], 1, 0, ps, 0x61); + tell(commands[1], replies[0], 1, 2 * ps, ps, 0x61); + /* Refaulting a coarse writable alias makes its covered units populated. */ + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, + ps)); + tell(commands[1], replies[0], 2, ps, ps, 0xb7); + struct uffdio_copy collision = { .dst = (uintptr_t)p + ps, + .src = (uintptr_t)source, + .len = ps }; + errno = 0; + CHECK(ioctl(fd, UFFDIO_COPY, &collision) == -1 && errno == EEXIST); + resolve(fd, p + ps, ps, 1, 0xb7); + printf("ok - shmem hole %zuK on %zuK precise PTEs, shifted alias, COW, coarse revoke/refault and COPY collision\n", + ps / 1024, native / 1024); + /* Stop the alias before resize, so it cannot intentionally promote holes. */ + tell(commands[1], replies[0], 0, 0, 0, 0); + close(commands[1]); + close(replies[0]); + int status; + CHECK(waitpid(alias, &status, 0) == alias && WIFEXITED(status) && + !WEXITSTATUS(status)); + size_t cut = 2 * ps + ps / 2; + CHECK(!ftruncate(file, cut) && !ftruncate(file, len)); + /* Partial final user leaf remains valid, with only truncated bytes zero. */ + if (!(pagemap(getpid(), (uintptr_t)p + 2 * ps, ps) & (1ULL << 63))) + resolve(fd, p + 2 * ps, ps, 1, 0x61); + CHECK(p[2 * ps] == 0x61 && p[cut - 1] == 0x61 && p[cut] == 0 && + p[3 * ps - 1] == 0); + resolve(fd, p + 3 * ps, ps, 0, 0xc3); + CHECK(q[2 * ps] == 0xc3); + CHECK(cow[ps] == 0xe1); + printf("ok - shmem resize %zuK on %zuK partial tail retained, shrink/regrow missing leaf refilled\n", + ps / 1024, native / 1024); + CHECK(!close(fd) && !munmap(p, len) && !munmap(q, len - ps) && + !munmap(cow, len) && !close(file)); + free(source); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 7 && !strcmp(argv[1], "--alias")) + return alias_worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), + strtoul(argv[5], NULL, 10), + strtoul(argv[6], NULL, 10)); + if (argc == 4 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10)); + int swap_test = argc == 2 && !strcmp(argv[1], "--swap"); + if (getpid() != 1 || (argc != 1 && !swap_test)) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + if (swap_test) { + struct stat st; + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)); + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + CHECK(!stat("/dev/vda", &st) && S_ISBLK(st.st_mode) && + !swapon("/dev/vda", 0)); + CHECK(!setenv("SHMEM_HOLE_SWAP", "1", 1)); + } + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { native, 4096, 16384 }; + int failed = 0; + for (unsigned i = 0; i < (native == 65536 ? 3U : 2U); i++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char ps[24], ns[24]; + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(ns, sizeof(ns), "%lu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, + 0UL)); + execl("/init", "/init", "--worker", ps, ns, NULL); + _exit(3); + } + int status = 0; + int ok = waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status); + printf("%s - shmem hole ABI %luK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + } + if (swap_test) + CHECK(!swapoff("/dev/vda")); + printf("SHMEM HOLE CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-pin-init.c b/tools/testing/selftests/arm64/user4k/shmem-pin-init.c new file mode 100644 index 0000000000000..51deda0fde52c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-pin-init.c @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem pin line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct ring { + int fd; + struct io_uring_params p; + void *sq, *cq, *sqes; + size_t sqlen, cqlen, sqelen; +}; +static unsigned load(unsigned *p) +{ + return __atomic_load_n(p, __ATOMIC_ACQUIRE); +} +static void store(unsigned *p, unsigned v) +{ + __atomic_store_n(p, v, __ATOMIC_RELEASE); +} +static size_t align_size(size_t n) +{ + size_t ps = getauxval(AT_PAGESZ); + return (n + ps - 1) & ~(ps - 1); +} +static void destroy(struct ring *r) +{ + if (r->sq && r->sq != MAP_FAILED) + munmap(r->sq, r->sqlen); + if (r->cq && r->cq != MAP_FAILED && r->cq != r->sq) + munmap(r->cq, r->cqlen); + if (r->sqes && r->sqes != MAP_FAILED) + munmap(r->sqes, r->sqelen); + if (r->fd >= 0) + close(r->fd); + *r = (struct ring){ .fd = -1 }; +} +static bool setup(struct ring *r, unsigned entries, unsigned flags) +{ + *r = (struct ring){ .fd = -1 }; + r->p.flags = flags; + r->fd = syscall(__NR_io_uring_setup, entries, &r->p); + if (r->fd < 0) + return false; + r->sqlen = align_size(r->p.sq_off.array + + r->p.sq_entries * sizeof(unsigned)); + r->cqlen = align_size(r->p.cq_off.cqes + + r->p.cq_entries * sizeof(struct io_uring_cqe) * + (flags & IORING_SETUP_CQE32 ? 2 : 1)); + r->sqelen = align_size(r->p.sq_entries * sizeof(struct io_uring_sqe) * + (flags & IORING_SETUP_SQE128 ? 2 : 1)); + if (r->p.features & IORING_FEAT_SINGLE_MMAP) { + if (r->cqlen > r->sqlen) + r->sqlen = r->cqlen; + r->cqlen = r->sqlen; + } + r->sq = mmap(NULL, r->sqlen, PROT_READ | PROT_WRITE, MAP_SHARED, r->fd, + IORING_OFF_SQ_RING); + r->cq = (r->p.features & IORING_FEAT_SINGLE_MMAP) ? + r->sq : + mmap(NULL, r->cqlen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_CQ_RING); + r->sqes = mmap(NULL, r->sqelen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_SQES); + return r->sq != MAP_FAILED && r->cq != MAP_FAILED && + r->sqes != MAP_FAILED; +} +static bool request(struct ring *r, unsigned op, int fd, void *buf, + unsigned len, uint64_t off, bool async) +{ + unsigned *sqtail = r->sq + r->p.sq_off.tail, + *sqmask = r->sq + r->p.sq_off.ring_mask; + unsigned *cqhead = r->cq + r->p.cq_off.head, + *cqtail = r->cq + r->p.cq_off.tail, + *cqmask = r->cq + r->p.cq_off.ring_mask; + unsigned tail = load(sqtail), head = load(cqhead), + idx = tail & load(sqmask); + unsigned sqstride = sizeof(struct io_uring_sqe) * + (r->p.flags & IORING_SETUP_SQE128 ? 2 : 1); + unsigned cqstride = sizeof(struct io_uring_cqe) * + (r->p.flags & IORING_SETUP_CQE32 ? 2 : 1); + struct io_uring_sqe *sqe = r->sqes + idx * sqstride; + memset(sqe, 0, sqstride); + sqe->opcode = op; + sqe->fd = fd; + sqe->addr = (uintptr_t)buf; + sqe->len = len; + sqe->off = off; + sqe->user_data = (uint64_t)tail + 1234; + if (async) + sqe->flags = IOSQE_ASYNC; + ((unsigned *)(r->sq + r->p.sq_off.array))[idx] = idx; + store(sqtail, tail + 1); + int ret = syscall(__NR_io_uring_enter, r->fd, 1, 1, + IORING_ENTER_GETEVENTS, NULL, 0); + if (ret != 1 || load(cqtail) != head + 1) { + printf("enter=%d errno=%d\n", ret, errno); + return false; + } + struct io_uring_cqe *cqe = + r->cq + r->p.cq_off.cqes + (head & load(cqmask)) * cqstride; + unsigned expected = op == IORING_OP_NOP ? 0 : len; + if (op == IORING_OP_READV_FIXED || op == IORING_OP_WRITEV_FIXED) { + expected = 0; + for (unsigned i = 0; i < len; i++) + expected += ((struct iovec *)buf)[i].iov_len; + } + bool ok = cqe->user_data == (uint64_t)tail + 1234 && + cqe->res == (int)expected; + if (!ok) + printf("cqe res=%d expected=%u\n", cqe->res, expected); + store(cqhead, head + 1); + return ok; +} + +static void fragment_geometry(unsigned char *p, size_t ps, int input) +{ + struct ring r; + CHECK(setup(&r, 8, 0)); + struct iovec reg = { p + 123, 3 * ps - 246 }; + unsigned char *src = malloc(3 * ps), *expected = malloc(3 * ps); + CHECK(src && expected); + memset(expected, 0x61, 3 * ps); + memset(src, 0xb7, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_REGISTER_BUFFERS, + ®, 1)); + CHECK(request(&r, IORING_OP_READ_FIXED, input, p + ps + 71, ps + 31, 17, + true)); + memset(expected + ps + 71, 0xb7, ps + 31); + CHECK(!memcmp(p, expected, 3 * ps)); + memset(src, 0xc8, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_WRITE_FIXED, input, p + ps + 71, ps + 31, 9, + true)); + CHECK(pread(input, src, ps + 31, 9) == (ssize_t)(ps + 31)); + for (size_t i = 0; i < ps + 31; i++) + CHECK(src[i] == 0xb7); + struct iovec vec[2] = { { p + 2 * ps + 17, ps - 400 }, + { p + ps + 91, ps - 600 } }; + memset(src, 0xd5, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_READV_FIXED, input, vec, 2, 13, true)); + for (unsigned i = 0; i < 2; i++) + memset(expected + ((unsigned char *)vec[i].iov_base - p), 0xd5, + vec[i].iov_len); + CHECK(!memcmp(p, expected, 3 * ps)); + memset(src, 0xc8, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_WRITEV_FIXED, input, vec, 2, 23, true)); + size_t total = vec[0].iov_len + vec[1].iov_len; + CHECK(pread(input, src, total, 23) == (ssize_t)total); + for (size_t i = 0; i < total; i++) + CHECK(src[i] == 0xd5); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_UNREGISTER_BUFFERS, + NULL, 0)); + destroy(&r); + free(src); + free(expected); + memset(p, 0x61, 3 * ps); + printf("ok - %zuK fixed/vector IO uses user fragment geometry and preserves guards\n", + ps / 1024); +} + +struct punch_worker { + int file; + size_t off, ps; + pthread_barrier_t start; +}; +static void bind_cpu(unsigned cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +static void *punch_repeatedly(void *arg) +{ + struct punch_worker *w = arg; + bind_cpu(1); + pthread_barrier_wait(&w->start); + for (unsigned i = 0; i < 256; i++) { + CHECK(!fallocate(w->file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + w->off, w->ps)); + sched_yield(); + } + return NULL; +} +static void pin_hole_concurrency(struct ring *r, int file, int input, + unsigned char *p, size_t off, size_t ps) +{ + struct punch_worker w = { .file = file, .off = off, .ps = ps }; + pthread_t thread; + CHECK(!pthread_barrier_init(&w.start, NULL, 2)); + bind_cpu(0); + CHECK(!pthread_create(&thread, NULL, punch_repeatedly, &w)); + pthread_barrier_wait(&w.start); + for (unsigned i = 0; i < 256; i++) + CHECK(request(r, IORING_OP_READ_FIXED, input, p + off, ps, 0, + true)); + CHECK(!pthread_join(thread, NULL) && + !pthread_barrier_destroy(&w.start)); + printf("ok - %zuK concurrent fixed writes and 256 hole punches on separate CPUs\n", + ps / 1024); +} + +static int target(size_t native, size_t ps, int resize) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(30); + size_t len = 4 * native, off = native + ps; + bool retained = ps < native; + int file = memfd_create("pin-cache", 0), + input = memfd_create("pin-input", 0); + CHECK(file >= 0 && input >= 0 && !ftruncate(file, len) && + !ftruncate(input, ps)); + unsigned char *src = malloc(len), *back = malloc(len); + CHECK(src && back); + memset(src, 0x61, len); + CHECK(pwrite(file, src, len, 0) == (ssize_t)len); + memset(src, 0xb7, ps); + CHECK(pwrite(input, src, ps, 0) == (ssize_t)ps); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + CHECK(p != MAP_FAILED && !madvise(p, len, MADV_NOHUGEPAGE)); + CHECK(p[off] == 0x61); + fragment_geometry(p, ps, input); + memset(src, 0xb7, ps); + CHECK(pwrite(input, src, ps, 0) == (ssize_t)ps); + struct ring r; + CHECK(setup(&r, 8, 0)); + struct iovec reg = { p + off, ps }; + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_REGISTER_BUFFERS, + ®, 1)); + int uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register ur = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, &ur)); + if (resize) + CHECK(!ftruncate(file, off) && !ftruncate(file, len)); + else + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + off, ps)); + if (retained && !resize) + pin_hole_concurrency(&r, file, input, p, off, ps); + /* Fixed IO accesses its original FOLL_PIN reference after PTE removal. */ + CHECK(request(&r, IORING_OP_READ_FIXED, input, p + off, ps, 0, true)); + /* Also prove the original pinned object contains the completed write. */ + CHECK(request(&r, IORING_OP_WRITE_FIXED, input, p + off, ps, 0, true)); + CHECK(pread(input, back, ps, 0) == (ssize_t)ps); + for (size_t i = 0; i < ps; i++) + CHECK(back[i] == 0xb7); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_UNREGISTER_BUFFERS, + NULL, 0)); + memset(src, 0xc8, ps); + struct uffdio_copy c = { .dst = (uintptr_t)(p + off), + .src = (uintptr_t)src, + .len = ps }; + errno = 0; + int ret = ioctl(uffd, UFFDIO_COPY, &c); + printf("# pin %zuK/%zuK resize=%d retained=%d COPY=%d errno=%d bytes=%lld\n", + ps / 1024, native / 1024, resize, retained, ret, errno, + (long long)c.copy); + if (retained) + CHECK(ret == -1 && errno == EEXIST && c.copy == -EEXIST); + else + CHECK(!ret && c.copy == (long long)ps); + CHECK(pread(file, back, len, 0) == (ssize_t)len); + for (size_t i = 0; i < len; i++) { + unsigned char expected = i >= off && i < off + ps ? + (retained ? 0xb7 : 0xc8) : + resize && i >= off ? 0 : + 0x61; + CHECK(back[i] == expected); + } + for (size_t i = 0; i < ps; i++) + CHECK(p[off + i] == (retained ? 0xb7 : 0xc8)); + /* A new punch after unpin can once again be populated by the pager. */ + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, off, + ps)); + c.copy = 0; + CHECK(!ioctl(uffd, UFFDIO_COPY, &c) && c.copy == (long long)ps); + for (size_t i = 0; i < ps; i++) + CHECK(p[off + i] == 0xc8); + destroy(&r); + CHECK(!close(uffd) && !munmap(p, len) && !close(file) && !close(input)); + free(src); + free(back); + printf("ok - writable pin %zuK on %zuK resize=%d retention and detach\n", + ps / 1024, native / 1024, resize); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--target")) + return target(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) + for (int resize = 0; resize < 2; resize++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char n[24], g[24], r[12]; + snprintf(n, sizeof(n), "%zu", native); + snprintf(g, sizeof(g), "%zu", ps); + snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); + execl("/init", "init", "--target", n, g, r, + NULL); + _exit(127); + } + int status; + CHECK(waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status)); + } + puts("ok - shmem writable pin fixture complete"); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c b/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c new file mode 100644 index 0000000000000..f321878396f06 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/pressure" +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem pressure line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +static void put(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + CHECK(fd >= 0); + CHECK(write(fd, value, strlen(value)) == (ssize_t)strlen(value) && + !close(fd)); +} +static unsigned long long get(const char *path, const char *key) +{ + FILE *f = fopen(path, "r"); + CHECK(f); + char name[128]; + unsigned long long value, result = 0; + if (!key) + CHECK(fscanf(f, "%llu", &result) == 1); + else + while (fscanf(f, "%127s %llu", name, &value) == 2) + if (!strcmp(key, name)) { + result = value; + break; + } + fclose(f); + return result; +} +static void copy_leaf(int uffd, unsigned char *dst, unsigned char *src, + size_t ps, unsigned char value) +{ + memset(src, value, ps); + struct uffdio_copy c = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = ps }; + int ret = ioctl(uffd, UFFDIO_COPY, &c); + if (ret) + printf("# COPY destination=%p errno=%d transferred=%lld\n", dst, + errno, (long long)c.copy); + CHECK(!ret && c.copy == (long long)ps); +} +static void checkpoint(int out, int in) +{ + char token; + CHECK(write(out, "R", 1) == 1 && read(in, &token, 1) == 1 && + token == 'G'); +} +static int worker(size_t native, size_t ps, int out, int in) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(120); + size_t records = (16UL << 20) / native, + stride = ps < native ? native : 2 * native; + size_t len = records * stride; + int file = memfd_create("pressure-partial-cache", 0); + CHECK(file >= 0 && !ftruncate(file, len)); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *source = malloc(stride), *back = malloc(stride); + CHECK(p != MAP_FAILED && source && back); + CHECK(!madvise(p, len, MADV_NOHUGEPAGE)); + int uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, ®)); + for (size_t i = 0; i < records; i++) + copy_leaf(uffd, p + i * stride, source, ps, 0xa1); + checkpoint(out, in); + /* Revisit in permuted order, filling holes in reclaimed cache backing. */ + for (size_t i = 0; i < records; i++) { + size_t slot = (i * 4051) & (records - 1); + copy_leaf(uffd, p + slot * stride + ps, source, ps, 0xb2); + for (size_t j = 0; j < ps; j++) + CHECK(p[slot * stride + j] == 0xa1 && + p[slot * stride + ps + j] == 0xb2); + } + checkpoint(out, in); + for (size_t i = 0; i < records; i += 2) { + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + i * stride, ps)); + copy_leaf(uffd, p + i * stride, source, ps, 0xc3); + } + checkpoint(out, in); + for (size_t i = 0; i < records; i++) { + size_t slot = (i * 4051 + 31) & (records - 1); + CHECK(pread(file, back, stride, slot * stride) == + (ssize_t)stride); + for (size_t j = 0; j < stride; j++) { + unsigned char expected = + j < ps ? (slot & 1 ? 0xa1 : 0xc3) : + j < 2 * ps ? 0xb2 : + 0; + CHECK(back[j] == expected); + } + struct uffdio_copy duplicate = { + .dst = (uintptr_t)(p + slot * stride), + .src = (uintptr_t)source, + .len = ps + }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &duplicate) == -1 && + errno == EEXIST && duplicate.copy == -EEXIST); + } + checkpoint(out, in); + CHECK(!close(uffd) && !munmap(p, len) && !close(file)); + free(source); + free(back); + printf("ok - shmem pressure %zuK on %zuK partial-cache refill, holes, duplicate rejection and every byte\n", + ps / 1024, native / 1024); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4]), + atoi(argv[5])); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL) && + !mount("sysfs", "/sys", "sysfs", 0, NULL)); + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) && + !mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL)); + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory"); + CHECK(!mkdir(CG, 0755)); + put(CG "/memory.max", "8388608"); + put(CG "/memory.swap.max", "134217728"); + struct stat st; + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + CHECK(!swapon("/dev/vda", 0)); + size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) { + int ready[2], go[2]; + CHECK(!pipe(ready) && !pipe(go)); + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + close(ready[0]); + close(go[1]); + char n[24], p[24], r[24], g[24]; + snprintf(n, sizeof(n), "%zu", native); + snprintf(p, sizeof(p), "%zu", ps); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(g, sizeof(g), "%d", go[0]); + put(CG "/cgroup.procs", "0"); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); + execl("/init", "init", "--worker", n, p, r, g, NULL); + _exit(127); + } + close(ready[1]); + close(go[0]); + for (unsigned phase = 0; phase < 4; phase++) { + char token; + CHECK(read(ready[0], &token, 1) == 1 && token == 'R'); + unsigned long long swap = get(CG "/memory.swap.current", + NULL), + max = get(CG "/memory.events", + "max"); + unsigned long long scan = get(CG "/memory.stat", + "pgscan"), + steal = get(CG "/memory.stat", + "pgsteal"); + printf("# pressure %zuK phase=%u resident=%llu swap=%llu max=%llu scan=%llu steal=%llu\n", + ps / 1024, phase, + get(CG "/memory.current", NULL), swap, max, scan, + steal); + CHECK(swap && max && scan && steal && + !get(CG "/memory.events", "oom") && + !get(CG "/memory.events", "oom_kill")); + CHECK(write(go[1], "G", 1) == 1); + } + int status; + CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status)); + close(ready[0]); + close(go[1]); + /* Closed/unmapped file identity must release all of its cache/swap. */ + for (unsigned i = 0; + i < 3000 && (get(CG "/memory.swap.current", NULL) || + get(CG "/memory.stat", "shmem")); + i++) + usleep(20000); + CHECK(!get(CG "/memory.swap.current", NULL) && + !get(CG "/memory.stat", "shmem")); + } + CHECK(!swapoff("/dev/vda")); + puts("SHMEM PRESSURE CONTRACT PASS"); + sync(); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-race-init.c b/tools/testing/selftests/arm64/user4k/shmem-race-init.c new file mode 100644 index 0000000000000..5109654b6ec21 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-race-init.c @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define ROUNDS 512 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem race line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct control { + unsigned start, stop, ready; + unsigned long aliases; +}; +struct state { + struct control *control; + int file, uffd, resize; + size_t ps, native, len; + unsigned char *p, *q, *source; + unsigned copied, existed, eof, alloc_eof, retry; +}; +static unsigned load(unsigned *p) +{ + return __atomic_load_n(p, __ATOMIC_ACQUIRE); +} +static void store(unsigned *p, unsigned value) +{ + __atomic_store_n(p, value, __ATOMIC_RELEASE); +} +static void bind_cpu(unsigned cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +static void start_wait(struct control *c) +{ + __atomic_add_fetch(&c->ready, 1, __ATOMIC_RELEASE); + while (!load(&c->start)) + sched_yield(); +} +static void *mutate(void *arg) +{ + struct state *s = arg; + bind_cpu(0); + start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + if (s->resize) + CHECK(!ftruncate(s->file, s->ps) && + !ftruncate(s->file, s->len)); + else + CHECK(!fallocate(s->file, + FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_KEEP_SIZE, + s->ps, s->ps)); + sched_yield(); + } + return NULL; +} +static void *populate(void *arg) +{ + struct state *s = arg; + bind_cpu(1); + start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + struct uffdio_copy c = { .dst = (uintptr_t)(s->p + s->ps), + .src = (uintptr_t)s->source, + .len = s->ps }; + errno = 0; + int ret = ioctl(s->uffd, UFFDIO_COPY, &c); + if (!ret) { + CHECK(c.copy == (long long)s->ps); + s->copied++; + } else { + CHECK(ret == -1 && c.copy == -errno); + if (errno == EEXIST) + s->existed++; + else if (errno == EFAULT && s->resize) + s->eof++; + /* Native shmem allocator preserves its NULL/ENOMEM EOF ABI. */ + else if (errno == ENOMEM && s->resize && + s->ps == s->native) + s->alloc_eof++; + else if (errno == EAGAIN) + s->retry++; + else { + printf("# unexpected COPY error=%d bytes=%lld\n", + errno, (long long)c.copy); + CHECK(0); + } + } + sched_yield(); + } + return NULL; +} +static int valid_byte(unsigned char value) +{ + return value == 0 || value == 0x61 || value == 0xa2; +} +static void *fine_alias(void *arg) +{ + struct state *s = arg; + bind_cpu(0); + start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + if (!s->resize) { + CHECK(!madvise(s->q, s->ps, MADV_DONTNEED)); + CHECK(valid_byte(*(volatile unsigned char *)s->q)); + } else { + /* The retained prefix remains inside EOF throughout resize. */ + CHECK(*(volatile unsigned char *)s->p == 0x61); + } + sched_yield(); + } + return NULL; +} +static int native_alias(int file, int control, size_t native, size_t ps, + int resize) +{ + CHECK(getauxval(AT_PAGESZ) == native); + bind_cpu(1); + alarm(60); + struct control *c = mmap(NULL, native, PROT_READ | PROT_WRITE, + MAP_SHARED, control, 0); + unsigned char *p = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_SHARED, file, 0); + CHECK(c != MAP_FAILED && p != MAP_FAILED && + !madvise(p, 4 * native, MADV_NOHUGEPAGE)); + start_wait(c); + unsigned long cycles = 0; + while (!load(&c->stop)) { + CHECK(!madvise(p, 2 * native, MADV_DONTNEED)); + CHECK(*(volatile unsigned char *)p == 0x61); + if (!resize) + CHECK(valid_byte(*(volatile unsigned char *)(p + ps))); + cycles++; + sched_yield(); + } + c->aliases = cycles; + CHECK(!munmap(p, 4 * native) && !munmap(c, native)); + return 0; +} +static int target(size_t native, size_t ps, int resize) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(60); + struct state s = { + .native = native, .ps = ps, .len = 4 * native, .resize = resize + }; + s.file = memfd_create("race-cache", 0); + int control = memfd_create("race-control", 0); + CHECK(s.file >= 0 && control >= 0 && !ftruncate(s.file, s.len) && + !ftruncate(control, native)); + s.control = mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, + control, 0); + s.source = malloc(s.len); + CHECK(s.control != MAP_FAILED && s.source); + memset(s.source, 0x61, s.len); + CHECK(pwrite(s.file, s.source, s.len, 0) == (ssize_t)s.len); + s.p = mmap(NULL, s.len, PROT_READ | PROT_WRITE, MAP_SHARED, s.file, 0); + s.q = mmap(NULL, s.len - ps, PROT_READ | PROT_WRITE, MAP_SHARED, s.file, + ps); + CHECK(s.p != MAP_FAILED && s.q != MAP_FAILED && + !madvise(s.p, s.len, MADV_NOHUGEPAGE)); + CHECK(s.p[0] == + 0x61); /* Avoid blocking the resize prefix reader on UFFD. */ + unsigned char *cow = MAP_FAILED; + if (!resize) { + cow = mmap(NULL, s.len, PROT_READ | PROT_WRITE, MAP_PRIVATE, + s.file, 0); + CHECK(cow != MAP_FAILED); + memset(cow + ps, 0xe1, ps); + } + s.uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(s.uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(s.uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)s.p, s.len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(s.uffd, UFFDIO_REGISTER, ®)); + memset(s.source, 0xa2, ps); + pid_t alias = fork(); + CHECK(alias >= 0); + if (!alias) { + char f[24], c[24], n[24], p[24], r[12]; + snprintf(f, sizeof(f), "%d", s.file); + snprintf(c, sizeof(c), "%d", control); + snprintf(n, sizeof(n), "%zu", native); + snprintf(p, sizeof(p), "%zu", ps); + snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, native, 0, 0, 0)); + execl("/init", "init", "--alias", f, c, n, p, r, NULL); + _exit(127); + } + pthread_t mutation, copy, fine; + CHECK(!pthread_create(&mutation, NULL, mutate, &s) && + !pthread_create(©, NULL, populate, &s) && + !pthread_create(&fine, NULL, fine_alias, &s)); + while (load(&s.control->ready) != 4) + sched_yield(); + store(&s.control->start, 1); + CHECK(!pthread_join(mutation, NULL) && !pthread_join(copy, NULL) && + !pthread_join(fine, NULL)); + store(&s.control->stop, 1); + int status; + CHECK(waitpid(alias, &status, 0) == alias && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(s.control->aliases && + s.copied + s.existed + s.eof + s.alloc_eof + s.retry == ROUNDS); + printf("# race %zuK/%zuK resize=%d COPY=%u EEXIST=%u EOF=%u native-alloc-EOF=%u retry=%u native-refaults=%lu\n", + ps / 1024, native / 1024, resize, s.copied, s.existed, s.eof, + s.alloc_eof, s.retry, s.control->aliases); + /* Once concurrent actors stop, exact cache/PTE state must be recoverable. */ + CHECK(!ftruncate(s.file, s.len)); + CHECK(!fallocate(s.file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, + ps)); + memset(s.source, 0xd4, ps); + struct uffdio_copy fill = { .dst = (uintptr_t)(s.p + ps), + .src = (uintptr_t)s.source, + .len = ps }; + CHECK(!ioctl(s.uffd, UFFDIO_COPY, &fill) && fill.copy == (long long)ps); + unsigned char *back = malloc(s.len); + CHECK(back && pread(s.file, back, s.len, 0) == (ssize_t)s.len); + for (size_t i = 0; i < s.len; i++) { + unsigned char expected = i >= ps && i < 2 * ps ? 0xd4 : + resize && i >= ps ? 0 : + 0x61; + if (back[i] != expected) { + printf("# byte=%zu actual=%#x expected=%#x\n", i, + back[i], expected); + CHECK(0); + } + } + for (size_t i = 0; i < ps; i++) { + CHECK(s.p[ps + i] == 0xd4 && s.q[i] == 0xd4); + if (!resize) + CHECK(cow[ps + i] == 0xe1); + } + CHECK(!close(s.uffd) && !munmap(s.p, s.len) && + !munmap(s.q, s.len - ps)); + if (!resize) + CHECK(!munmap(cow, s.len)); + CHECK(!munmap(s.control, native) && !close(control) && !close(s.file)); + free(back); + free(s.source); + printf("ok - concurrent shmem %zuK on %zuK resize=%d cache bytes, aliases and private COW\n", + ps / 1024, native / 1024, resize); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 7 && !strcmp(argv[1], "--alias")) + return native_alias(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10), + strtoul(argv[5], NULL, 10), atoi(argv[6])); + if (argc == 5 && !strcmp(argv[1], "--target")) + return target(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) + for (int resize = 0; resize < 2; resize++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char n[24], p[24], r[12]; + snprintf(n, sizeof(n), "%zu", native); + snprintf(p, sizeof(p), "%zu", ps); + snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); + execl("/init", "init", "--target", n, p, r, + NULL); + _exit(127); + } + int status; + CHECK(waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status)); + } + puts("SHMEM RACE CONTRACT PASS"); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-thp-init.c b/tools/testing/selftests/arm64/user4k/shmem-thp-init.c new file mode 100644 index 0000000000000..e08ea92103fcc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-thp-init.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem THP line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +static uint64_t read_entry(const char *path, uint64_t index) +{ + uint64_t value; + int fd = open(path, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &value, 8, index * 8) == 8 && !close(fd)); + return value; +} +static uint64_t pfn(void *p, size_t ps) +{ + uint64_t entry = read_entry("/proc/self/pagemap", (uintptr_t)p / ps); + CHECK(entry & (1ULL << 63)); + return entry & ((1ULL << 55) - 1); +} +static unsigned char *map_aligned(int fd, size_t len) +{ + void *reserve = mmap(NULL, 2 * len, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + uintptr_t address = ((uintptr_t)reserve + len - 1) & ~(len - 1); + CHECK(!munmap(reserve, 2 * len)); + unsigned char *p = mmap((void *)address, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, fd, 0); + CHECK(p != MAP_FAILED); + return p; +} +struct reader { + volatile unsigned char *p; + unsigned char value; +}; +static void *read_byte(void *arg) +{ + struct reader *r = arg; + r->value = *r->p; + return NULL; +} +static int target(int file, size_t native, size_t ps, int hold, int zero, + int resize, uint64_t head) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(30); + size_t len = 4 * native, target_off = resize ? 2 * native : native; + size_t cut = native + ps / 2; + unsigned char *p = map_aligned(file, len); + for (size_t i = 0; i < len; i += ps) + CHECK(p[i] == 0x61); + CHECK(pfn(p, ps) * ps / native == head); + int pipes[2]; + CHECK(!pipe(pipes)); + if (hold) { + struct iovec iov = { p, ps }; + CHECK(vmsplice(pipes[1], &iov, 1, 0) == (ssize_t)ps); + } + int uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, ®)); + if (resize) + CHECK(!ftruncate(file, cut) && !ftruncate(file, len)); + else + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + native, ps)); + uint64_t flags = read_entry("/proc/kpageflags", head); + printf("# THP %zuK native=%zuK hold=%d zero=%d resize=%d head_flags=%#llx\n", + ps / 1024, native / 1024, hold, zero, resize, + (unsigned long long)flags); + if (hold) + CHECK((flags & + ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))) == + ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))); + struct reader r = { .p = p + target_off + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct pollfd pollfd = { .fd = uffd, .events = POLLIN }; + struct uffd_msg msg; + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(uffd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && !msg.arg.pagefault.flags && + msg.arg.pagefault.address == (uintptr_t)(p + target_off)); + unsigned char *source = malloc(ps); + CHECK(source); + memset(source, 0x92, ps); + int ret; + struct uffdio_copy invalid = { .dst = (uintptr_t)(p + target_off), + .src = 1, + .len = ps }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &invalid) == -1 && errno == EFAULT && + invalid.copy == -EFAULT); + if (zero) { + struct uffdio_zeropage z = { + .range = { (uintptr_t)(p + target_off), ps } + }; + ret = ioctl(uffd, UFFDIO_ZEROPAGE, &z); + printf("# ZEROPAGE returned=%d errno=%d bytes=%lld\n", ret, + errno, (long long)z.zeropage); + CHECK(!ret && z.zeropage == (long long)ps); + } else { + struct uffdio_copy c = { .dst = (uintptr_t)(p + target_off), + .src = (uintptr_t)source, + .len = ps }; + ret = ioctl(uffd, UFFDIO_COPY, &c); + printf("# COPY returned=%d errno=%d bytes=%lld\n", ret, errno, + (long long)c.copy); + CHECK(!ret && c.copy == (long long)ps); + } + CHECK(!pthread_join(thread, NULL) && r.value == (zero ? 0 : 0x92)); + memset(source, 0xd4, ps); + struct uffdio_copy duplicate = { .dst = (uintptr_t)(p + target_off), + .src = (uintptr_t)source, + .len = ps }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &duplicate) == -1 && errno == EEXIST && + duplicate.copy == -EEXIST); + /* File reads verify zeroed tail holes without faulting unfilled UFFD leaves. */ + unsigned char *back = malloc(len); + CHECK(back); + CHECK(pread(file, back, len, 0) == (ssize_t)len); + for (size_t i = 0; i < len; i++) { + unsigned char expected = i >= target_off && + i < target_off + ps ? + (zero ? 0 : 0x92) : + resize && i >= cut ? 0 : + 0x61; + CHECK(back[i] == expected); + if (!resize || (i >= target_off && i < target_off + ps)) + CHECK(p[i] == expected); + } + free(back); + if (hold) { + CHECK(read(pipes[0], source, ps) == (ssize_t)ps); + for (size_t i = 0; i < ps; i++) + CHECK(source[i] == 0x61); + } + CHECK(!close(pipes[0]) && !close(pipes[1]) && !close(uffd) && + !munmap(p, len) && !close(file)); + free(source); + printf("ok - THP %zuK on %zuK hold=%d zero=%d resize=%d missing refill and all neighbour bytes\n", + ps / 1024, native / 1024, hold, zero, resize); + return 0; +} +static int create_file(size_t native, uint64_t *head) +{ + int fd = open("/tmp/huge", O_CREAT | O_TRUNC | O_RDWR, 0600); + CHECK(fd >= 0); + CHECK(!ftruncate(fd, 4 * native)); + unsigned char *p = map_aligned(fd, 4 * native); + CHECK(!madvise(p, 4 * native, MADV_HUGEPAGE)); + memset(p, 0x61, 4 * native); + *head = pfn(p, native); + uint64_t f = read_entry("/proc/kpageflags", *head); + printf("# initial head=%llu flags=%#llx VA=%p native=%zu\n", + (unsigned long long)*head, (unsigned long long)f, p, native); + CHECK((f & ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))) == + ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))); + for (unsigned i = 1; i < 4; i++) { + CHECK(pfn(p + i * native, native) == *head + i); + CHECK(read_entry("/proc/kpageflags", *head + i) & + (1ULL << KPF_COMPOUND_TAIL)); + } + /* Keep the name until allocation: unlinked mappings use anon-shmem policy. */ + CHECK(!unlink("/tmp/huge") && !munmap(p, 4 * native)); + return fd; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 9 && !strcmp(argv[1], "--target")) + return target(atoi(argv[2]), strtoul(argv[3], NULL, 10), + strtoul(argv[4], NULL, 10), atoi(argv[5]), + atoi(argv[6]), atoi(argv[7]), + strtoull(argv[8], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("tmpfs", "/tmp", "tmpfs", 0, "huge=within_size,size=32m")); + size_t native = getauxval(AT_PAGESZ), sizes[] = { native, 4096, 16384 }; + int failed = 0; + for (unsigned abi = 0; abi < (native == 65536 ? 3U : 2U); abi++) + for (unsigned hold = 0; hold < 2; hold++) + for (unsigned zero = 0; zero < 2; zero++) + for (unsigned resize = 0; resize < 2; + resize++) { + uint64_t head; + int fd = create_file(native, &head); + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char fs[24], ns[24], ps[24], + hs[24], zs[24], rs[24], + pfs[24]; + snprintf(fs, sizeof(fs), "%d", + fd); + snprintf(ns, sizeof(ns), "%zu", + native); + snprintf(ps, sizeof(ps), "%zu", + sizes[abi]); + snprintf(hs, sizeof(hs), "%u", + hold); + snprintf(zs, sizeof(zs), "%u", + zero); + snprintf(rs, sizeof(rs), "%u", + resize); + snprintf(pfs, sizeof(pfs), + "%llu", + (unsigned long long) + head); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, + sizes[abi], 0UL, + 0UL, 0UL)); + execl("/init", "/init", + "--target", fs, ns, ps, + hs, zs, rs, pfs, NULL); + _exit(3); + } + int status = 0; + int ok = waitpid(pid, &status, 0) == + pid && + WIFEXITED(status) && + !WEXITSTATUS(status); + printf("%s - THP ABI %zuK hold=%u zero=%u resize=%u status=%#x\n", + ok ? "ok" : "not ok", + sizes[abi] / 1024, hold, zero, + resize, status); + failed |= !ok; + CHECK(!close(fd)); + } + printf("SHMEM THP CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c b/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c new file mode 100644 index 0000000000000..73ff33d5e0ba1 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PID-1 runner for a disposable VM with a dedicated virtio swap disk. + * Use kunit.autorun=0: tests skipped at boot stay skipped on a debugfs rerun. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int run_swap_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-shmem-swap/"; + char path[128], *log; + size_t used = 0, capacity = 1024 * 1024; + ssize_t n; + int fd, ret = -1; + + snprintf(path, sizeof(path), "%srun", base); + fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = write(fd, "1", 1); + close(fd); + if (n != 1) + return -1; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + log = calloc(1, capacity); + if (!log) { + close(fd); + return -1; + } + while (used < capacity - 1 && + (n = read(fd, log + used, capacity - 1 - used)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (n == 0 && used < capacity - 1 && + strstr(log, "\nok 1 arm64-user4k-shmem-swap\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; + free(log); + return ret; +} + +static int fault_setting(const char *name, const char *value) +{ + char path[160]; + snprintf(path, sizeof(path), "/sys/kernel/debug/failslab/%s", name); + int fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) + return -1; + ssize_t n = write(fd, value, strlen(value)); + close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} +static int configure_failslab(bool enable) +{ + if (!enable) + return fault_setting("probability", "0"); + /* The KUnit worker selects itself only around metadata writeout. */ + return fault_setting("task-filter", "Y") || + fault_setting("verbose", "0") || fault_setting("times", "-1") || + fault_setting("interval", "1") || fault_setting("space", "0") || + fault_setting("probability", "100"); +} + +static int swap_control_helper(const char *operation) +{ + struct stat init, self; + int ret; + + /* A helper is allowed only when this same executable is the VM's PID 1. */ + if (getpid() == 1 || stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || init.st_dev != self.st_dev || + init.st_ino != self.st_ino) { + fprintf(stderr, + "swap helper requires its fixture runner as PID 1\n"); + return 1; + } + if (!strcmp(operation, "--memcg-setup-helper")) { + if (mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) && + errno != EBUSY) + return 1; + int fd = open("/sys/fs/cgroup/cgroup.subtree_control", + O_WRONLY | O_CLOEXEC); + if (fd < 0) + return 1; + ssize_t n = write(fd, "+memory", 7); + close(fd); + if (n != 7) + return 1; + return mkdir("/sys/fs/cgroup/uffd-metadata", 0755) && + errno != EEXIST; + } + if (!strcmp(operation, "--failslab-on-helper")) + return configure_failslab(true); + if (!strcmp(operation, "--failslab-off-helper")) + return configure_failslab(false); + ret = !strcmp(operation, "--swapoff-helper") ? swapoff("/dev/vda") : + swapon("/dev/vda", 0); + if (ret) + perror(operation); + return ret ? 1 : 0; +} + +int main(int argc, char **argv) +{ + struct stat st; + bool active = false; + int ret = 1; + if (argc == 2 && (!strcmp(argv[1], "--swapon-helper") || + !strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--failslab-on-helper") || + !strcmp(argv[1], "--failslab-off-helper") || + !strcmp(argv[1], "--memcg-setup-helper"))) + return swap_control_helper(argv[1]); + if (getpid() != 1 || argc != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + unsigned long ps = getauxval(AT_PAGESZ); + if (ps != 16384 && ps != 65536) + goto out; + if (mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) + goto out; + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || + swapon("/dev/vda", 0)) + goto out; + active = true; + ret = run_swap_suite() ? 1 : 0; +out: + if (active && swapoff("/dev/vda")) + ret = 1; + printf("SHMEM UFFD STATE SWAP %s errno=%d\n", ret ? "FAIL" : "PASS", + errno); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/splice-fixture-init.c b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c new file mode 100644 index 0000000000000..282ba6268b25d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c @@ -0,0 +1,610 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static bool use_tcp, use_ipv6, use_udp; + +static int connected_pair(int sockets[2]) +{ + struct sockaddr_in a4 = { .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK) }; + struct sockaddr_in6 a6 = { .sin6_family = AF_INET6, + .sin6_addr = IN6ADDR_LOOPBACK_INIT }; + struct sockaddr_storage peer; + struct sockaddr *address = use_ipv6 ? (void *)&a6 : (void *)&a4; + socklen_t size = use_ipv6 ? sizeof(a6) : sizeof(a4), + peer_size = sizeof(peer); + int domain = use_ipv6 ? AF_INET6 : AF_INET; + int listener, saved; + + if (!use_tcp && !use_udp) + return socketpair(AF_UNIX, SOCK_STREAM, 0, sockets); + listener = socket(domain, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); + if (listener < 0) + return -1; + sockets[0] = -1; + if (bind(listener, address, size) || + getsockname(listener, address, &size)) + goto error; + if (!use_udp && listen(listener, 1)) + goto error; + sockets[0] = socket(domain, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); + if (sockets[0] < 0 || connect(sockets[0], address, size)) + goto error; + if (use_udp) { + if (getsockname(sockets[0], (void *)&peer, &peer_size) || + connect(listener, (void *)&peer, peer_size)) + goto error; + sockets[1] = listener; + return 0; + } + sockets[1] = accept(listener, NULL, NULL); + if (sockets[1] < 0) + goto error; + close(listener); + return 0; +error: + saved = errno; + if (sockets[0] >= 0) + close(sockets[0]); + close(listener); + errno = saved; + return -1; +} + +static unsigned char value(size_t off) +{ + return (off * 13 + (off >> 12) * 31) & 255; +} +static void fill(unsigned char *p, size_t n) +{ + for (size_t i = 0; i < n; i++) + p[i] = value(i); +} +static bool check(const unsigned char *p, size_t n, size_t off) +{ + for (size_t i = 0; i < n; i++) + if (p[i] != value(i + off)) { + ksft_print_msg( + "data mismatch at %zu: got %02x expected %02x\n", + i + off, p[i], value(i + off)); + return false; + } + return true; +} +static bool read_exact(int fd, unsigned char *p, size_t n) +{ + while (n) { + ssize_t got = read(fd, p, n); + + if (got <= 0) { + ksft_print_msg( + "read fd=%d got=%zd remaining=%zu errno=%d\n", + fd, got, n, errno); + return false; + } + p += got; + n -= got; + } + return true; +} +static bool replace(unsigned char *p, size_t n) +{ + if (munmap(p, n) || + mmap(p, n, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0) != p) + return false; + memset(p, 0xec, n); + return true; +} +static bool cow(unsigned char *p, size_t n) +{ + pid_t child = fork(); + int status = 0; + + if (!child) + _exit(0); + if (child < 0 || waitpid(child, &status, 0) != child || status) + return false; + memset(p, 0xab, n); + return true; +} +static void report(bool ok, size_t ps, const char *name) +{ + ksft_test_result(ok, "%zuK %s\n", ps / 1024, name); +} +static bool tls_pair(int sockets[2], unsigned int version) +{ + struct tls12_crypto_info_aes_gcm_128 crypto = { + .info.version = version, + .info.cipher_type = TLS_CIPHER_AES_GCM_128, + }; + struct timeval timeout = { .tv_sec = 5 }; + int no_padding = 1; + + if (connected_pair(sockets)) + return false; + return !setsockopt(sockets[0], IPPROTO_TCP, TCP_ULP, "tls", 4) && + !setsockopt(sockets[1], IPPROTO_TCP, TCP_ULP, "tls", 4) && + !setsockopt(sockets[0], SOL_TLS, TLS_TX, &crypto, + sizeof(crypto)) && + !setsockopt(sockets[1], SOL_TLS, TLS_RX, &crypto, + sizeof(crypto)) && + (version != TLS_1_3_VERSION || + !setsockopt(sockets[1], SOL_TLS, TLS_RX_EXPECT_NO_PAD, + &no_padding, sizeof(no_padding))) && + !setsockopt(sockets[1], SOL_SOCKET, SO_RCVTIMEO, &timeout, + sizeof(timeout)); +} + +static void tls_checks(size_t ps, unsigned char *p, size_t len, + unsigned char *back) +{ + int sockets[2], fds[2]; + char name[160]; + + if (pipe(fds)) + ksft_exit_fail_msg("TLS pipe errno=%d\n", errno); + for (unsigned int v = 0; v < 2; v++) { + unsigned int version = v ? TLS_1_3_VERSION : TLS_1_2_VERSION; + struct iovec vec; + bool ok; + + if (!tls_pair(sockets, version)) + ksft_exit_fail_msg("TLS %x setup errno=%d\n", version, + errno); + for (unsigned int mode = 0; mode < 2; mode++) { + const size_t count = mode ? 4096 : 4096 - 29; + const size_t offset = ps + (mode ? 0 : 29); + + fill(p, len); + vec = (struct iovec){ p + offset, count }; + if (vmsplice(fds[1], &vec, 1, 0) != (ssize_t)count || + splice(fds[0], NULL, sockets[0], NULL, count, + SPLICE_F_MORE) != (ssize_t)count) + ksft_exit_fail_msg("TLS MORE splice errno=%d\n", + errno); + errno = 0; + if (recv(sockets[1], back, len, MSG_DONTWAIT) != -1 || + errno != EAGAIN) + ksft_exit_fail_msg( + "TLS pending plaintext precondition errno=%d\n", + errno); + ok = mode ? cow(p + ps, 4096) : replace(p + ps, ps); + ok &= send(sockets[0], "!", 1, 0) == 1; + ok &= read_exact(sockets[1], back, count + 1) && + check(back, count, offset) && back[count] == '!'; + snprintf(name, sizeof(name), + "TLS 1.%u pending plaintext survives %s", + v + 2, + mode ? "fork/COW" : "unmap/replacement"); + report(ok, ps, name); + } + { + const size_t bytes = 3 * 16384 + 123, prefix = 137; + const size_t mapping = (bytes + ps - 1) & ~(ps - 1); + unsigned char *source = + mmap(NULL, mapping, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *received = malloc(bytes + prefix); + + if (source == MAP_FAILED || !received) + ksft_exit_fail_msg( + "TLS multi-record allocation\n"); + fill(source, mapping); + ok = send(sockets[0], source, prefix, 0) == + (ssize_t)prefix; + vec = (struct iovec){ source, bytes }; + if (vmsplice(fds[1], &vec, 1, 0) != (ssize_t)bytes || + splice(fds[0], NULL, sockets[0], NULL, bytes, 0) != + (ssize_t)bytes) + ksft_exit_fail_msg( + "TLS multi-record splice errno=%d\n", + errno); + ok &= replace(source, mapping); + ok &= read_exact(sockets[1], received, + bytes + prefix) && + check(received, prefix, 0) && + check(received + prefix, bytes, 0); + snprintf( + name, sizeof(name), + "TLS 1.%u ordinary send and multiple spliced records", + v + 2); + report(ok, ps, name); + munmap(source, mapping); + free(received); + } + { + const size_t bytes = 5003; + int file = memfd_create("tls-receive", 0); + unsigned char *mapped; + + if (file < 0 || ftruncate(file, 3 * ps)) + ksft_exit_fail_msg("TLS receive file setup\n"); + mapped = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, file, 0); + if (mapped == MAP_FAILED) + ksft_exit_fail_msg( + "TLS receive file mapping\n"); + memset(mapped, 0x5c, 3 * ps); + fill(p, len); + ok = send(sockets[0], p, bytes, 0) == (ssize_t)bytes && + read_exact(sockets[1], mapped + 17, bytes); + ok &= !msync(mapped, 3 * ps, MS_SYNC) && + pread(file, back, bytes + 34, 0) == + (ssize_t)(bytes + 34) && + check(back + 17, bytes, 0); + for (unsigned int i = 0; i < 17; i++) + ok &= back[i] == 0x5c && + back[bytes + 17 + i] == 0x5c; + snprintf( + name, sizeof(name), + "TLS 1.%u decrypts into shared file with neighboring bytes intact", + v + 2); + report(ok, ps, name); + munmap(mapped, 3 * ps); + close(file); + } + { + unsigned char *mapped = + mmap(NULL, 6 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + struct iovec vectors[4]; + struct msghdr message = { .msg_iov = vectors, + .msg_iovlen = 4 }; + + if (mapped == MAP_FAILED) + ksft_exit_fail_msg( + "TLS receive iovec mapping\n"); + memset(mapped, 0x81, 6 * ps); + vectors[0] = (struct iovec){ mapped + 13, 5003 }; + vectors[1] = (struct iovec){ NULL, 0 }; + vectors[2] = + (struct iovec){ mapped + 2 * ps + 7, 2222 }; + vectors[3] = + (struct iovec){ mapped + 4 * ps + 5, 1010 }; + fill(p, len); + ok = send(sockets[0], p, 8235, 0) == 8235 && + recvmsg(sockets[1], &message, MSG_WAITALL) == + 8235 && + check(mapped + 13, 5003, 0) && + check(mapped + 2 * ps + 7, 2222, 5003) && + check(mapped + 4 * ps + 5, 1010, 7225); + ok &= mapped[12] == 0x81 && mapped[5016] == 0x81 && + mapped[2 * ps + 6] == 0x81 && + mapped[2 * ps + 2229] == 0x81 && + mapped[4 * ps + 4] == 0x81 && + mapped[4 * ps + 1015] == 0x81; + snprintf( + name, sizeof(name), + "TLS 1.%u scattered receive, empty iovec and subpage boundaries", + v + 2); + report(ok, ps, name); + munmap(mapped, 6 * ps); + } + close(sockets[0]); + close(sockets[1]); + ok = true; + for (unsigned int round = 0; round < 64 && ok; round++) { + if (!tls_pair(sockets, version)) + ksft_exit_fail_msg( + "TLS teardown setup errno=%d\n", errno); + fill(p, len); + vec = (struct iovec){ p + ps, 4096 }; + ok = vmsplice(fds[1], &vec, 1, 0) == 4096 && + splice(fds[0], NULL, sockets[0], NULL, 4096, + SPLICE_F_MORE) == 4096; + close(sockets[1]); + close(sockets[0]); + ok &= !madvise(p + ps, ps, MADV_DONTNEED); + memset(p + ps, 0x75, ps); + } + snprintf(name, sizeof(name), + "TLS 1.%u repeated pending-record teardown and reuse", + v + 2); + report(ok, ps, name); + } + close(fds[0]); + close(fds[1]); +} + +static int exercise(void) +{ + const size_t ps = getauxval(AT_PAGESZ), len = 8 * ps; + unsigned char *p, *back = malloc(len); + int fds[2], dup[2], file, sockets[2]; + struct iovec v[3]; + struct stat self, init; + struct timeval timeout = { .tv_sec = 5 }; + bool ok; + ssize_t result; + + if (stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino) + return 2; + p = mmap((void *)0x300000000UL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (p == MAP_FAILED || !back || pipe(fds) || pipe(dup)) + ksft_exit_fail_msg("setup\n"); + ksft_print_header(); + ksft_set_plan(use_tcp ? 22 : 10); + ksft_print_msg("transport=%s family=%s\n", + use_tcp ? "TCP" : + use_udp ? "UDP" : + "Unix stream", + use_ipv6 ? "IPv6" : "IPv4/Unix"); + fill(p, len); + v[0] = (struct iovec){ NULL, 0 }; + v[1] = (struct iovec){ p + 17, ps - 17 }; + v[2] = (struct iovec){ p + 2 * ps + 31, 2 * ps - 31 }; + result = vmsplice(fds[1], v, 3, 0); + if (result != (ssize_t)(3 * ps - 48)) + ksft_exit_fail_msg("vmsplice returned %zd errno=%d\n", result, + errno); + ok = read_exact(fds[0], back, result) && check(back, ps - 17, 17) && + check(back + ps - 17, 2 * ps - 31, 2 * ps + 31); + report(ok, ps, "unaligned fragmented iovecs and empty segment"); + v[0] = (struct iovec){ p, 16384 }; + ok = fcntl(fds[1], F_SETPIPE_SZ, 16384) == 16384; + result = vmsplice(fds[1], v, 1, SPLICE_F_NONBLOCK); + { + int pending = 0; + + ok &= result == 16384 && !ioctl(fds[0], FIONREAD, &pending) && + pending == 16384; + } + if (result > 0) + ok &= read_exact(fds[0], back, result) && + check(back, result, 0); + report(ok, ps, + "contiguous quarters retain one-native-page pipe capacity"); + fcntl(fds[1], F_SETPIPE_SZ, 16 * 16384); + v[0] = (struct iovec){ p + 11, 3 * ps - 11 }; + result = vmsplice(fds[1], v, 1, 0); + if (result != (ssize_t)(3 * ps - 11) || + tee(fds[0], dup[1], result, 0) != result) + ksft_exit_fail_msg("tee setup\n"); + ok = replace(p, len); + ok &= read_exact(fds[0], back, 19) && check(back, 19, 11); + ok &= read_exact(fds[0], back + 19, result - 19) && + check(back, result, 11); + ok &= read_exact(dup[0], back, result) && check(back, result, 11); + report(ok, ps, + "tee and partial reads retain data across unmap/replacement"); + fill(p, len); + v[0] = (struct iovec){ p, ps }; + ok = vmsplice(fds[1], v, 1, 0) == (ssize_t)ps && cow(p, ps); + ok &= read_exact(fds[0], back, ps) && check(back, ps, 0); + report(ok, ps, "pipe GET retains pre-COW data after fork sharer exit"); + fill(p, len); + file = memfd_create("splice-target", 0); + v[0] = (struct iovec){ p + ps + 29, 2 * ps - 29 }; + result = vmsplice(fds[1], v, 1, SPLICE_F_GIFT); + ok = file >= 0 && result == (ssize_t)(2 * ps - 29) && + splice(fds[0], NULL, file, NULL, result, SPLICE_F_MOVE) == result; + ok &= pread(file, back, result, 0) == result && + check(back, result, ps + 29); + report(ok, ps, "gifted fragments splice to buffered file"); + close(file); + v[0] = (struct iovec){ p, ps }; + v[1] = (struct iovec){ (void *)0x1000, ps }; + result = vmsplice(fds[1], v, 2, 0); + ok = result == (ssize_t)ps && read_exact(fds[0], back, ps) && + check(back, ps, 0); + report(ok, ps, "partial vmsplice before inaccessible iovec"); + if (connected_pair(sockets)) + ksft_exit_fail_msg("socketpair errno=%d\n", errno); + setsockopt(sockets[1], SOL_SOCKET, SO_RCVTIMEO, &timeout, + sizeof(timeout)); + for (unsigned int mode = 0; mode < 3; mode++) { + const size_t bytes = mode == 1 ? ps : + (use_udp && ps == 4096 ? + 6 * ps - 256 : + 2 * ps - 256); + const size_t offset = mode == 1 ? 0 : ps + 29; + int cork = 1; + + if ((use_tcp || use_udp) && mode < 2 && + setsockopt(sockets[0], use_tcp ? IPPROTO_TCP : IPPROTO_UDP, + use_tcp ? TCP_CORK : UDP_CORK, &cork, + sizeof(cork))) + ksft_exit_fail_msg("TCP_CORK setup errno=%d\n", errno); + fill(p, len); + v[0] = (struct iovec){ p + offset, bytes }; + ok = vmsplice(fds[1], v, 1, 0) == (ssize_t)bytes && + splice(fds[0], NULL, sockets[0], NULL, bytes, + SPLICE_F_NONBLOCK) == (ssize_t)bytes; + if (!ok) + ksft_exit_fail_msg("socket splice setup errno=%d\n", + errno); + if (mode == 2) { + ok = splice(sockets[1], NULL, fds[1], NULL, bytes, 0) == + (ssize_t)bytes && + tee(fds[0], dup[1], bytes, 0) == (ssize_t)bytes; + if (!ok) + ksft_exit_fail_msg( + "socket-to-pipe setup errno=%d\n", + errno); + } + if (use_tcp && mode < 2) { + int unsent = 0; + + if (ioctl(sockets[0], SIOCOUTQNSD, &unsent) || + unsent != (int)bytes) + ksft_exit_fail_msg( + "TCP unsent precondition: %d expected %zu errno=%d\n", + unsent, bytes, errno); + } + if (use_udp && mode < 2) { + errno = 0; + if (recv(sockets[1], back, len, MSG_DONTWAIT) != -1 || + errno != EAGAIN) + ksft_exit_fail_msg( + "UDP corked precondition errno=%d\n", + errno); + } + ok &= mode == 1 ? cow(p, ps) : replace(p + ps, 2 * ps); + if ((use_tcp || use_udp) && mode < 2) { + cork = 0; + if (setsockopt(sockets[0], + use_tcp ? IPPROTO_TCP : IPPROTO_UDP, + use_tcp ? TCP_CORK : UDP_CORK, &cork, + sizeof(cork))) { + ksft_print_msg("uncork errno=%d\n", errno); + ok = false; + } + } + ok &= read_exact(mode == 2 ? fds[0] : sockets[1], back, + bytes) && + check(back, bytes, offset); + if (mode == 2) + ok &= read_exact(dup[0], back, bytes) && + check(back, bytes, offset); + report(ok, ps, + mode == 0 ? + "socket retains quarters across partial unmap/replacement" : + mode == 1 ? + "socket GET retains pre-COW data after fork sharer exit" : + "socket-to-pipe and tee retain independent quarter ownership"); + } + close(sockets[0]); + close(sockets[1]); + ok = true; + for (unsigned int round = 0; round < 128 && ok; round++) { + fill(p, len); + ok = !connected_pair(sockets); + v[0] = (struct iovec){ p + ps, ps }; + ok &= vmsplice(fds[1], v, 1, 0) == (ssize_t)ps && + splice(fds[0], NULL, sockets[0], NULL, ps, 0) == + (ssize_t)ps; + close(sockets[1]); + close(sockets[0]); + ok &= !madvise(p + ps, ps, MADV_DONTNEED); + memset(p + ps, 0x43, ps); + } + report(ok, ps, "repeated unread socket teardown and quarter reuse"); + if (use_tcp) + tls_checks(ps, p, len, back); + close(fds[0]); + close(fds[1]); + close(dup[0]); + close(dup[1]); + munmap(p, len); + free(back); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +static bool async_cleanup_balanced(void) +{ + const char *names[] = { "user4k_rx_deferred", "user4k_rx_released" }; + unsigned long counts[2]; + char path[128]; + FILE *file = fopen("/sys/module/tls/parameters/user4k_test_async", "r"); + int enabled; + + if (!file) + return false; + enabled = fgetc(file); + fclose(file); + for (unsigned int i = 0; i < 2; i++) { + snprintf(path, sizeof(path), "/sys/module/tls/parameters/%s", + names[i]); + file = fopen(path, "r"); + if (!file) + return false; + if (fscanf(file, "%lu", &counts[i]) != 1) { + fclose(file); + return false; + } + fclose(file); + } + printf("# TLS async=%c deferred=%lu released=%lu\n", enabled, counts[0], + counts[1]); + return counts[0] == counts[1] && (enabled != 'Y' || counts[0] > 0); +} + +int main(int argc, char **argv) +{ + bool ok = true; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && + (!strcmp(argv[1], "--test") || !strcmp(argv[1], "--tcp") || + !strcmp(argv[1], "--tcp6") || !strcmp(argv[1], "--udp") || + !strcmp(argv[1], "--udp6")) && + getppid() == 1) { + use_tcp = !strncmp(argv[1], "--tcp", 5); + use_udp = !strncmp(argv[1], "--udp", 5); + use_ipv6 = strchr(argv[1], '6') != NULL; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + { + struct ifreq request = { .ifr_name = "lo" }; + int fd = socket(AF_INET, SOCK_DGRAM, 0); + + if (fd < 0 || ioctl(fd, SIOCGIFFLAGS, &request)) + return 4; + request.ifr_flags |= IFF_UP; + if (ioctl(fd, SIOCSIFFLAGS, &request)) + return 4; + close(fd); + } + for (unsigned int mode = 0; mode < 10; mode++) { + unsigned int small = mode & 1; + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + small ? 4096UL : 0UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", + mode >= 8 ? "--udp6" : + mode >= 6 ? "--udp" : + mode >= 4 ? "--tcp6" : + mode >= 2 ? "--tcp" : + "--test", + NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + ok &= async_cleanup_balanced(); + printf("%s - splice fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/swap-fixture-init.c b/tools/testing/selftests/arm64/user4k/swap-fixture-init.c new file mode 100644 index 0000000000000..45f6aefdd094e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/swap-fixture-init.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PID-1 runner for a disposable VM with a dedicated virtio swap disk. + * Use kunit.autorun=0: tests skipped at boot stay skipped on a debugfs rerun. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "re"); + char key[128]; + unsigned long value, found = 0; + + if (!f) + return 0; + while (fscanf(f, "%127s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) + found = value; + fclose(f); + return found; +} + +static int native_swap_smoke(void) +{ + const size_t length = 4 * 1024 * 1024, page = 16384; + unsigned long before = swapouts(), after; + unsigned char *p = mmap(NULL, length, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + size_t i; + unsigned int tries; + int ret = 0; + + if (p == MAP_FAILED) + return -1; + for (i = 0; i < length; i++) + p[i] = (i ^ (i >> 12) ^ (i >> 20)) & 255; + if (madvise(p, length, MADV_PAGEOUT)) { + ret = -1; + goto out; + } + for (tries = 0; tries < 100 && swapouts() == before; tries++) + usleep(20000); + after = swapouts(); + for (i = 0; i < length; i++) + if (p[i] != ((i ^ (i >> 12) ^ (i >> 20)) & 255)) { + ret = -1; + break; + } + if (after <= before) + ret = -1; + printf("%s - native %zuK swap transport: pswpout %lu -> %lu, verified %zu bytes\n", + ret ? "not ok" : "ok", page / 1024, before, after, i); +out: + munmap(p, length); + return ret; +} + +static int run_swap_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-swap/"; + char path[128], *log; + size_t used = 0, capacity = 1024 * 1024; + ssize_t n; + int fd, ret = -1; + + snprintf(path, sizeof(path), "%srun", base); + fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = write(fd, "1", 1); + close(fd); + if (n != 1) + return -1; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + log = calloc(1, capacity); + if (!log) { + close(fd); + return -1; + } + while (used < capacity - 1 && + (n = read(fd, log + used, capacity - 1 - used)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (n == 0 && used < capacity - 1 && + strstr(log, "\nok 1 arm64-user4k-swap\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; + free(log); + return ret; +} + +static int swap_control_helper(const char *operation) +{ + struct stat init, self; + int ret; + + /* A helper is allowed only when this same executable is the VM's PID 1. */ + if (getpid() == 1 || stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || init.st_dev != self.st_dev || + init.st_ino != self.st_ino) { + fprintf(stderr, + "swap helper requires its fixture runner as PID 1\n"); + return 1; + } + ret = !strcmp(operation, "--swapoff-helper") ? swapoff("/dev/vda") : + swapon("/dev/vda", 0); + if (ret) + perror(operation); + return ret ? 1 : 0; +} + +static int pageout_exec(bool small) +{ + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (small && + prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(125); + execl("/reclaim-advice", "/reclaim-advice", + small ? "--expect-4k" : NULL, NULL); + _exit(126); + } + if (pid <= 0 || waitpid(pid, &status, 0) != pid) + return -1; + printf("# EL0 pageout %s child status=%#x\n", small ? "4K" : "native", + status); + return WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +int main(int argc, char **argv) +{ + bool smoke = argc == 2 && !strcmp(argv[1], "--smoke"); + bool pageout = argc == 2 && !strcmp(argv[1], "--pageout"); + bool native_only = argc == 2 && !strcmp(argv[1], "--pageout-native"); + bool active = false; + unsigned int tries; + struct stat st; + pid_t child; + int status, ret = 1; + + if (argc == 2 && (!strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--swapon-helper"))) + return swap_control_helper(argv[1]); + /* This runner must never mount filesystems or enable swap on the host. */ + if (getpid() != 1 || + (argc != 1 && !smoke && !pageout && !native_only)) { + fprintf(stderr, + "Use only as /init in the dedicated disposable test VM.\n"); + return 1; + } + setvbuf(stdout, NULL, _IONBF, 0); + if (getauxval(AT_PAGESZ) != 16384 || sysconf(_SC_PAGESIZE) != 16384) + goto out; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) + goto out; + child = fork(); + if (!child) { + execl("/page-contract", "page-contract", NULL); + _exit(127); + } + if (child < 0 || waitpid(child, &status, 0) != child || + !WIFEXITED(status) || WEXITSTATUS(status)) + goto out; + for (tries = 0; tries < 100 && stat("/dev/vda", &st); tries++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || + swapon("/dev/vda", 0)) + goto out; + active = true; + if (native_swap_smoke()) + goto out; + if (!smoke && !native_only && run_swap_suite()) + goto out; + if ((pageout || native_only) && pageout_exec(false)) + goto out; + if (pageout && pageout_exec(true)) + goto out; + ret = 0; +out: + if (ret) + printf("not ok - swap fixture runner: errno=%d (%s)\n", errno, + strerror(errno)); + if (active && swapoff("/dev/vda")) { + perror("swapoff"); + ret = 1; + } + printf("%s - swap fixture runner (%s)\n", ret ? "not ok" : "ok", + smoke ? "native transport smoke only" : "4K swap suite"); + fflush(NULL); + reboot(RB_POWER_OFF); + return ret; +} diff --git a/tools/testing/selftests/arm64/user4k/swapfile-fixture.h b/tools/testing/selftests/arm64/user4k/swapfile-fixture.h new file mode 100644 index 0000000000000..258b81fa05695 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/swapfile-fixture.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef USER4K_SWAPFILE_FIXTURE_H +#define USER4K_SWAPFILE_FIXTURE_H + +static const char *swap_path(void) +{ + return access("/swapdir/swapfile", F_OK) ? "/dev/vda" : + "/swapdir/swapfile"; +} + +/* Remove every fourth native page: every 64K group crosses file extents. */ +static int prepare_swapfile(void) +{ + const unsigned long page = getauxval(AT_PAGESZ), initial = 32UL << 20; + unsigned char *buf = calloc(1, page); + unsigned long offset; + int fd = -1, ret = -1; + + if (!buf || (mkdir("/swapdir", 0755) && errno != EEXIST) || + mount("/dev/vda", "/swapdir", "ext4", 0, NULL)) + goto out; + fd = open("/swapdir/swapfile", O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, + 0600); + if (fd < 0) + goto out; + memset(buf, 0xa5, page); + for (offset = 0; offset < initial; offset += page) + if (write(fd, buf, page) != (ssize_t)page) + goto out; + if (fsync(fd)) + goto out; + for (offset = initial; offset; offset -= 4 * page) + if (fallocate(fd, FALLOC_FL_COLLAPSE_RANGE, offset - 2 * page, + page)) + goto out; + memset(buf, 0, page); + ((unsigned int *)(buf + 1024))[0] = 1; + ((unsigned int *)(buf + 1024))[1] = (initial * 3 / 4) / page - 1; + memcpy(buf + page - 10, "SWAPSPACE2", 10); + if (pwrite(fd, buf, page, 0) != (ssize_t)page || fsync(fd)) + goto out; + ret = 0; +out: + if (fd >= 0) + close(fd); + free(buf); + return ret; +} + +#endif diff --git a/tools/testing/selftests/arm64/user4k/sysv-shm-init.c b/tools/testing/selftests/arm64/user4k/sysv-shm-init.c new file mode 100644 index 0000000000000..3b66d9c738fde --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/sysv-shm-init.c @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: SysV shared memory with per-process page granules. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int ok, const char *expression, int line) +{ + if (!ok) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, + errno); + exit(1); + } +} + +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int unmapped(void *address, size_t page) +{ + unsigned char resident; + + errno = 0; + return mincore(address, page, &resident) == -1 && errno == ENOMEM; +} + +static int mapping_case(unsigned long native, int rounded, int split) +{ + unsigned long page = getauxval(AT_PAGESZ), length = 4 * page; + unsigned char *reserve, *target, *mapping, *alias; + struct shmid_ds info; + int id; + + reserve = mmap(NULL, 16 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + target = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + target += native + (page < native ? page : 0); + target[-1] = 0x51; + target[length] = 0x72; + CHECK(!munmap(target, length)); + id = shmget(IPC_PRIVATE, length - 1, 0600); + CHECK(id >= 0); + mapping = shmat(id, target + (rounded ? page - 1 : 0), + rounded ? SHM_RND : 0); + if (mapping != target) { + int saved = errno; + + shmctl(id, IPC_RMID, NULL); + errno = saved; + CHECK(mapping == target); + } + alias = shmat(id, NULL, 0); + CHECK(alias != (void *)-1); + CHECK(!shmctl(id, IPC_RMID, NULL)); + mapping[0] = 0x31; + mapping[page + 7] = 0x49; + mapping[3 * page] = 0x63; + CHECK(alias[0] == 0x31 && alias[page + 7] == 0x49 && + alias[3 * page] == 0x63); + if (split) { + CHECK(!mprotect(mapping + page, page, PROT_READ)); + CHECK(!munmap(mapping + 2 * page, page)); + if (split == 2) + CHECK(!munmap(mapping, page)); + } + CHECK(!shmdt(mapping)); + for (unsigned long i = 0; i < length; i += page) + CHECK(unmapped(target + i, page)); + CHECK(target[-1] == 0x51 && target[length] == 0x72); + CHECK(alias[page + 7] == 0x49 && alias[3 * page] == 0x63); + CHECK(!shmdt(alias)); + errno = 0; + CHECK(shmctl(id, IPC_STAT, &info) == -1 && + (errno == EINVAL || errno == EIDRM)); + CHECK(!munmap(reserve, 16 * native)); + printf("ok - SysV page=%lu rounded=%d split=%d detach and alias lifetime\n", + page, rounded, split); + return 0; +} + +static int wait_child(pid_t pid) +{ + int status; + pid_t got; + + if (pid < 0) + return 1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + return got != pid || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static int sharing_child(int id, unsigned long expected) +{ + unsigned char *map; + + CHECK(getauxval(AT_PAGESZ) == expected); + map = shmat(id, NULL, 0); + CHECK(map != (void *)-1); + CHECK(map[37] == 0x35 && map[4096 + 9] == 0x56); + map[4096 + 9] = 0x78; + CHECK(!shmdt(map)); + return 0; +} + +static int sharing(unsigned long other, unsigned long native) +{ + unsigned long page = getauxval(AT_PAGESZ); + unsigned char *map; + pid_t pid; + int id; + + id = shmget(IPC_PRIVATE, 2 * native, 0600); + CHECK(id >= 0); + map = shmat(id, NULL, 0); + CHECK(map != (void *)-1); + CHECK(!shmctl(id, IPC_RMID, NULL)); + map[37] = 0x35; + map[4096 + 9] = 0x56; + pid = fork(); + if (!pid) { + char idarg[32], sizearg[32]; + + snprintf(idarg, sizeof(idarg), "%d", id); + snprintf(sizearg, sizeof(sizearg), "%lu", other); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, other, 0UL, 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--share", idarg, sizearg, NULL); + _exit(3); + } + CHECK(!wait_child(pid)); + CHECK(map[4096 + 9] == 0x78); + CHECK(!shmdt(map)); + printf("ok - SysV shared object %lu -> %lu after IPC_RMID\n", page, + other); + return 0; +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--share")) + return sharing_child(atoi(argv[2]), strtoul(argv[3], NULL, 10)); + if (argc == 5 && !strcmp(argv[1], "--case")) { + unsigned long expected = strtoul(argv[3], NULL, 10); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == expected); + failed |= mapping_case(native, 0, 0); + failed |= mapping_case(native, 1, 0); + failed |= mapping_case(native, 0, 1); + failed |= mapping_case(native, 0, 2); + for (unsigned int i = 0; i < 3; i++) + if (sizes[i] <= native && sizes[i] != expected && + !(atoi(argv[4]) && sizes[i] == 4096)) + failed |= sharing(sizes[i], native); + return failed; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + int no4k = argc == 2 && !strcmp(argv[1], "--no4k"); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + unsigned long page = sizes[i]; + pid_t pid; + + if (page > native || (no4k && page == 4096)) + continue; + pid = fork(); + if (!pid) { + char n[32], e[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(e, sizeof(e), "%lu", page); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, page, 0UL, 0UL, + 0UL)) + _exit(2); + execl("/init", "/init", "--case", n, e, + no4k ? "1" : "0", NULL); + _exit(3); + } + int bad = wait_child(pid); + + printf("%s - SysV AArch64 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, page); + failed |= bad; + } + if (access("/compat-shm", X_OK) == 0) { + pid_t pid = fork(); + + if (!pid) { + char n[32], e[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(e, sizeof(e), "%lu", no4k ? native : 4096UL); + execl("/compat-shm", "/compat-shm", n, e, NULL); + _exit(3); + } + int bad = wait_child(pid); + + printf("%s - SysV AArch32 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, no4k ? native : 4096UL); + failed |= bad; + } + printf("SYSV SHM GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/thread-fixture-init.c b/tools/testing/selftests/arm64/user4k/thread-fixture-init.c new file mode 100644 index 0000000000000..c472ea6697947 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/thread-fixture-init.c @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define THREADS 4 +#define ROUNDS 128 +#define PAGES 32 + +static pthread_barrier_t barrier; +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t condition = PTHREAD_COND_INITIALIZER; +static pthread_mutex_t robust; +static _Thread_local unsigned int tls_id; +static _Thread_local volatile sig_atomic_t signals; +static atomic_uint failures; +static unsigned long total; +static unsigned long ps; +static atomic_int waiter_tid; + +static void handler(int sig) +{ + if (sig == SIGUSR1 && tls_id) + signals++; + else + atomic_fetch_add(&failures, 1); +} + +static bool bytes_equal(unsigned char *p, size_t n, unsigned char value) +{ + for (size_t i = 0; i < n; i++) + if (p[i] != value) + return false; + return true; +} + +static void *worker(void *arg) +{ + unsigned int id = (uintptr_t)arg; + unsigned char *p, *heap; + unsigned long request = id & 1 ? 4096 : 0; + bool ok = true; + + tls_id = id; + p = mmap(NULL, PAGES * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (p == MAP_FAILED) + ok = false; + ok &= !prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, request, 0UL, 0UL, 0UL); + pthread_barrier_wait(&barrier); + for (unsigned int round = 0; round < ROUNDS; round++) { + unsigned char pattern = 1 + (id * 31 + round) % 254; + unsigned char residency[PAGES]; + cpu_set_t cpu; + + if (ok) { + CPU_ZERO(&cpu); + CPU_SET((id + round) % 2, &cpu); + ok &= !sched_setaffinity(0, sizeof(cpu), &cpu); + memset(p, pattern, PAGES * ps); + ok &= !mprotect(p + ps, ps, PROT_READ) && + p[ps] == pattern; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE); + ok &= !madvise(p + 2 * ps, ps, MADV_FREE); + memset(p + 2 * ps, pattern, ps); + ok &= !madvise(p + 3 * ps, ps, MADV_DONTNEED); + ok &= bytes_equal(p + 3 * ps, ps, 0); + memset(p + 3 * ps, pattern, ps); + heap = malloc(5000 + id * 777); + if (!heap) { + ok = false; + } else { + memset(heap, pattern, 5000 + id * 777); + ok &= bytes_equal(heap, 5000 + id * 777, + pattern); + free(heap); + } + ok &= !pthread_kill(pthread_self(), SIGUSR1); + ok &= tls_id == id && + signals == (sig_atomic_t)(round + 1); + ok &= prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, + 0UL) == (int)request; + ok &= !mincore(p, PAGES * ps, residency); + ok &= bytes_equal(p, PAGES * ps, pattern); + } + pthread_mutex_lock(&mutex); + total++; + pthread_cond_signal(&condition); + pthread_mutex_unlock(&mutex); + pthread_barrier_wait(&barrier); + } + if (p != MAP_FAILED) + munmap(p, PAGES * ps); + if (!ok) + atomic_fetch_add(&failures, 1); + return NULL; +} + +static void *owner_dies(void *arg) +{ + (void)arg; + if (pthread_mutex_lock(&robust)) + atomic_fetch_add(&failures, 1); + return NULL; +} + +static void *pi_waiter(void *arg) +{ + int error; + + (void)arg; + atomic_store(&waiter_tid, syscall(SYS_gettid)); + error = pthread_mutex_lock(&robust); + if (!error) + error = pthread_mutex_unlock(&robust); + return (void *)(uintptr_t)error; +} + +static bool wait_for_pi_block(void) +{ + for (unsigned int tries = 0; tries < 1000; tries++) { + int tid = atomic_load(&waiter_tid); + char path[80], line[256]; + unsigned long nr, address, operation; + FILE *f; + + if (tid) { + snprintf(path, sizeof(path), + "/proc/self/task/%d/syscall", tid); + f = fopen(path, "re"); + if (f) { + bool blocked = fgets(line, sizeof(line), f) && + sscanf(line, "%lu %lx %lx", &nr, + &address, + &operation) == 3 && + nr == SYS_futex && + ((operation & FUTEX_CMD_MASK) == + FUTEX_LOCK_PI || + (operation & FUTEX_CMD_MASK) == + FUTEX_LOCK_PI2); + + fclose(f); + if (blocked) + return true; + } + } + usleep(1000); + } + return false; +} + +static int exercise(void) +{ + pthread_t threads[THREADS], owner; + pthread_mutexattr_t attributes; + struct sigaction action = { .sa_handler = handler }; + struct stat self, init; + bool ok = true, forks = true; + int error; + + if (stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino) + return 2; + ps = getauxval(AT_PAGESZ); + ksft_print_header(); + ksft_set_plan(6); + if (sigaction(SIGUSR1, &action, NULL) || + pthread_barrier_init(&barrier, NULL, THREADS + 1)) + ksft_exit_fail_msg("thread setup\n"); + for (unsigned int i = 0; i < THREADS; i++) + if (pthread_create(&threads[i], NULL, worker, + (void *)(uintptr_t)(i + 1))) + ksft_exit_fail_msg("pthread_create\n"); + pthread_barrier_wait(&barrier); + for (unsigned int round = 0; round < ROUNDS; round++) { + pthread_mutex_lock(&mutex); + while (total < (round + 1) * THREADS) + pthread_cond_wait(&condition, &mutex); + pthread_mutex_unlock(&mutex); + ok &= prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) == + 0; + if (!(round % 16)) { + pid_t child = fork(); + int status = 0; + + if (!child) { + unsigned char *p = malloc(32768); + + if (!p || tls_id || + total != (round + 1) * THREADS || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, + 0UL, 0UL, 0UL)) + _exit(10); + memset(p, 0x75, 32768); + if (!bytes_equal(p, 32768, 0x75)) + _exit(11); + free(p); + _exit(0); + } + forks &= child > 0 && + waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + } + pthread_barrier_wait(&barrier); + } + for (unsigned int i = 0; i < THREADS; i++) + ok &= !pthread_join(threads[i], NULL); + ksft_test_result( + ok && !atomic_load(&failures) && total == THREADS * ROUNDS, + "%luK %u concurrent memory/TLS/signal/exec-request rounds across %u threads\n", + ps / 1024, ROUNDS, THREADS); + ksft_test_result( + forks, "%luK fork and allocator from a multithreaded process\n", + ps / 1024); + pthread_barrier_destroy(&barrier); + ok = !pthread_mutexattr_init(&attributes); + ok &= !pthread_mutexattr_setrobust(&attributes, PTHREAD_MUTEX_ROBUST); + ok &= !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + ksft_exit_fail_msg("robust mutex setup\n"); + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + ok &= !pthread_mutex_unlock(&robust); + ksft_test_result(ok, "%luK robust futex owner-death recovery\n", + ps / 1024); + pthread_mutex_destroy(&robust); + /* Exercise repeated TLS/stack allocation and set_tid_address clear/wake. */ + for (unsigned int i = 0; i < 128; i++) { + ok = !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + break; + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + pthread_mutex_unlock(&robust); + pthread_mutex_destroy(&robust); + if (!ok) + break; + } + ksft_test_result(ok && !atomic_load(&failures), + "%luK repeated thread exit/join and robust recovery\n", + ps / 1024); + ok = !pthread_mutexattr_setprotocol(&attributes, PTHREAD_PRIO_INHERIT); + ok &= !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_mutex_lock(&robust); + ok &= !pthread_create(&owner, NULL, pi_waiter, NULL); + if (!ok) + ksft_exit_fail_msg("PI mutex setup\n"); + ok &= wait_for_pi_block(); + ok &= !pthread_mutex_unlock(&robust); + { + void *result = (void *)1; + + ok &= !pthread_join(owner, &result) && !result; + } + ksft_test_result( + ok, + "%luK confirmed contended priority-inheritance futex handoff\n", + ps / 1024); + ok = !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + ksft_exit_fail_msg("PI owner setup\n"); + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + ok &= !pthread_mutex_unlock(&robust); + pthread_mutex_destroy(&robust); + pthread_mutexattr_destroy(&attributes); + ksft_test_result( + ok, "%luK priority-inheritance robust owner-death recovery\n", + ps / 1024); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +static pthread_barrier_t exec_barrier; + +static void *exec_sibling(void *arg) +{ + (void)arg; + pthread_barrier_wait(&exec_barrier); + for (;;) + pause(); + return NULL; +} + +static void *exec_nonleader(void *arg) +{ + unsigned long target = (uintptr_t)arg; + char size[32]; + + snprintf(size, sizeof(size), "%lu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, 0UL, 0UL, 0UL)) + _exit(30); + pthread_barrier_wait(&exec_barrier); + execl("/init", "/init", "--after-thread-exec", size, NULL); + _exit(31); +} + +static int thread_exec(unsigned long target) +{ + pthread_t siblings[2], executor; + + if (pthread_barrier_init(&exec_barrier, NULL, 4)) + return 32; + for (unsigned int i = 0; i < 2; i++) + if (pthread_create(&siblings[i], NULL, exec_sibling, NULL)) + return 33; + if (pthread_create(&executor, NULL, exec_nonleader, + (void *)(uintptr_t)target)) + return 34; + pthread_barrier_wait(&exec_barrier); + for (;;) + pause(); + return 35; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long granules[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + unsigned int nr = granules[0] > 16384 ? 3 : 2; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--thread-exec") && getppid() == 1) + return thread_exec(strtoul(argv[2], NULL, 10)); + if (argc == 3 && !strcmp(argv[1], "--after-thread-exec") && + getppid() == 1) { + unsigned long expected = strtoul(argv[2], NULL, 10); + + return getauxval(AT_PAGESZ) != expected || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, + 0UL) != 0; + } + if (argc == 3 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + char size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", size, NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + for (unsigned int source = 0; source < nr; source++) { + for (unsigned int target = 0; target < nr; target++) { + pid_t pid = fork(); + int status = 0; + bool passed; + + if (!pid) { + char size[32]; + + snprintf(size, sizeof(size), "%lu", + granules[target]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + granules[source], 0UL, 0UL, 0UL)) + _exit(36); + execl("/init", "/init", "--thread-exec", size, + NULL); + _exit(37); + } + passed = pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - non-leader exec %luK -> %luK status=%#x\n", + passed ? "ok" : "not ok", + granules[source] / 1024, granules[target] / 1024, + status); + ok &= passed; + } + } + printf("%s - thread fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/uffd-move-init.c b/tools/testing/selftests/arm64/user4k/uffd-move-init.c new file mode 100644 index 0000000000000..f2649698488b6 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uffd-move-init.c @@ -0,0 +1,451 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - MOVE line=%d errno=%d (%s)\n", \ + __LINE__, errno, strerror(errno)); \ + exit(1); \ + } \ + } while (0) +#define PRESENT (1ULL << 63) +#define SWAPPED (1ULL << 62) +#define VALUE_MASK ((1ULL << 55) - 1) +static uint64_t entry(void *p, size_t ps) +{ + uint64_t e; + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + CHECK(fd >= 0 && pread(fd, &e, 8, (uintptr_t)p / ps * 8) == 8); + CHECK(!close(fd)); + return e; +} +static uint64_t physical(void *p, size_t ps) +{ + uint64_t e = entry(p, ps); + CHECK(e & PRESENT); + return (e & VALUE_MASK) * ps; +} +static unsigned char pattern(size_t i) +{ + return 1 + ((i ^ (i >> 12) ^ (i >> 16)) % 251); +} +static void bytes(unsigned char *p, size_t original, size_t len) +{ + for (size_t i = 0; i < len; i++) + CHECK(p[i] == pattern(original + i)); +} +static long move(int fd, void *dst, void *src, size_t len, unsigned mode, + int expected_errno) +{ + struct uffdio_move m = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = len, + .mode = mode }; + errno = 0; + int r = ioctl(fd, UFFDIO_MOVE, &m); + if (expected_errno) + CHECK(r == -1 && errno == expected_errno); + else + CHECK(r == 0 && m.move == (long long)len); + return m.move; +} +static void on_cpu(unsigned cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +struct tlb_reader { + unsigned char *src, *dst; + atomic_int ready, done; +}; +static void *read_after_move(void *arg) +{ + struct tlb_reader *r = arg; + on_cpu(1); + unsigned char old = *(volatile unsigned char *)r->src; + atomic_store(&r->ready, 1); + while (!atomic_load(&r->done)) + sched_yield(); + CHECK(*(volatile unsigned char *)r->src == 0 && + *(volatile unsigned char *)r->dst == old); + return NULL; +} +struct pin_racer { + void *a, *b; + size_t ps; + atomic_int stop; + atomic_uint accepted, rejected; +}; +static void *race_pins(void *arg) +{ + struct pin_racer *r = arg; + struct io_uring_params p = { 0 }; + on_cpu(1); + int fd = syscall(SYS_io_uring_setup, 2, &p); + CHECK(fd >= 0); + unsigned round = 0; + while (!atomic_load(&r->stop)) { + struct iovec iov = { .iov_base = (round++ & 1) ? r->a : r->b, + .iov_len = r->ps }; + int ret = syscall(SYS_io_uring_register, fd, + IORING_REGISTER_BUFFERS, &iov, 1); + if (!ret) { + atomic_fetch_add(&r->accepted, 1); + usleep(50); + CHECK(!syscall(SYS_io_uring_register, fd, + IORING_UNREGISTER_BUFFERS, NULL, 0)); + } else { + CHECK(errno == EFAULT || errno == EAGAIN); + atomic_fetch_add(&r->rejected, 1); + } + } + CHECK(!close(fd)); + return NULL; +} +static void concurrent_move(size_t ps) +{ + unsigned char *a = + mmap((void *)0x50000000, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + unsigned char *b = + mmap((void *)0x60000000, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(a != MAP_FAILED && b != MAP_FAILED); + memset(a, 0x6d, ps); + uint64_t phys = physical(a, ps); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MOVE }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)a, ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + reg.range.start = (uintptr_t)b; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + struct pin_racer race = { .a = a, .b = b, .ps = ps }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, race_pins, &race)); + while (!atomic_load(&race.accepted)) + sched_yield(); + unsigned char *cur = a, *next = b; + unsigned moved = 0, busy = 0; + for (unsigned tries = 0; moved < 256 && tries < 100000; tries++) { + struct uffdio_move m = { .src = (uintptr_t)cur, + .dst = (uintptr_t)next, + .len = ps }; + int ret = ioctl(fd, UFFDIO_MOVE, &m); + if (ret) { + CHECK(errno == EBUSY && m.move == -EBUSY); + busy++; + sched_yield(); + continue; + } + CHECK(m.move == (long long)ps && physical(next, ps) == phys && + !(entry(cur, ps) & (PRESENT | SWAPPED))); + unsigned char *tmp = cur; + cur = next; + next = tmp; + moved++; + } + atomic_store(&race.stop, 1); + CHECK(!pthread_join(thread, NULL)); + CHECK(moved == 256 && atomic_load(&race.accepted) > 0); + for (size_t i = 0; i < ps; i++) + CHECK(cur[i] == 0x6d); + printf("ok - MOVE %zuK CPU0/CPU1 race moves=%u busy=%u pins=%u holes=%u unchanged PFN/data\n", + ps / 1024, moved, busy, atomic_load(&race.accepted), + atomic_load(&race.rejected)); + CHECK(!close(fd) && !munmap(a, ps) && !munmap(b, ps)); +} + +static void allocation_failure(size_t ps, size_t native) +{ + if (ps == native) + return; + unsigned char *a = + mmap((void *)0x70000000, 2 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + unsigned char *b = + mmap((void *)0x71000000, 2 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(a != MAP_FAILED && b != MAP_FAILED); + memset(a, 0x35, 2 * native); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MOVE }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)a, 2 * native }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + reg.range.start = (uintptr_t)b; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + /* Prepare both roots and target tables using a different native folio. */ + move(fd, b + native, a + native, ps, 0, 0); + uint64_t phys = physical(a, ps); + int inject = open("/proc/self/fail-nth", O_RDWR | O_CLOEXEC); + CHECK(inject >= 0); + unsigned char *src = a, *dst = b; + for (unsigned phase = 0; phase < 2; phase++) { + struct uffdio_move m = { .src = (uintptr_t)src, + .dst = (uintptr_t)dst, + .len = ps }; + char pending[16] = { 0 }; + CHECK(write(inject, "1", 1) == 1); + errno = 0; + int ret = ioctl(fd, UFFDIO_MOVE, &m), saved = errno; + CHECK(pread(inject, pending, sizeof(pending) - 1, 0) > 0); + CHECK(write(inject, "0", 1) == 1); + CHECK(ret == -1 && saved == ENOMEM && m.move == -ENOMEM && + strtoul(pending, NULL, 10) == 0); + CHECK(physical(src, ps) == phys && + !(entry(dst, ps) & (PRESENT | SWAPPED))); + for (size_t i = 0; i < ps; i++) + CHECK(src[i] == 0x35); + move(fd, dst, src, ps, 0, 0); + CHECK(physical(dst, ps) == phys); + unsigned char *tmp = src; + src = dst; + dst = tmp; + } + printf("ok - MOVE %zuK sidecar/retirement allocation failures preserve PTEs and retry\n", + ps / 1024); + CHECK(!close(inject) && !close(fd) && !munmap(a, 2 * native) && + !munmap(b, 2 * native)); +} + +static int exercise(size_t native) +{ + size_t ps = getauxval(AT_PAGESZ), len = 16 * native; + on_cpu(0); + unsigned char *src = + mmap((void *)0x20000000, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + unsigned char *dst = + mmap((void *)0x30000000, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (size_t i = 0; i < len; i++) + src[i] = pattern(i); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MOVE }; + CHECK(!ioctl(fd, UFFDIO_API, &api) && + (api.features & UFFD_FEATURE_MOVE)); + struct uffdio_register reg = { .range = { (uintptr_t)dst, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®) && + (reg.ioctls & (1ULL << _UFFDIO_MOVE))); + uint64_t pfns[256]; + CHECK(native / ps <= sizeof(pfns) / sizeof(pfns[0])); + for (size_t i = 0; i < native / ps; i++) + pfns[i] = physical(src + i * ps, ps); + uint64_t p = physical(src + 3 * ps, ps); + struct tlb_reader reader = { .src = src + 3 * ps, .dst = dst + ps }; + pthread_t tlb_thread; + CHECK(!pthread_create(&tlb_thread, NULL, read_after_move, &reader)); + while (!atomic_load(&reader.ready)) + sched_yield(); + move(fd, dst + ps, src + 3 * ps, ps, 0, 0); + CHECK(physical(dst + ps, ps) == p && + !(entry(src + 3 * ps, ps) & (PRESENT | SWAPPED))); + atomic_store(&reader.done, 1); + CHECK(!pthread_join(tlb_thread, NULL)); + bytes(dst + ps, 3 * ps, ps); + for (size_t i = 0; i < native / ps; i++) + if (i != 3) { + CHECK(physical(src + i * ps, ps) == pfns[i]); + bytes(src + i * ps, i * ps, ps); + } + CHECK(src[3 * ps] == + 0); /* Source has an independent zero-fault after MOVE. */ + printf("ok - MOVE %zuK exact PFN, divergent offsets, neighbour PFNs/data and source TLB\n", + ps / 1024); + + struct io_uring_params ur = { 0 }; + int ring = syscall(SYS_io_uring_setup, 2, &ur); + CHECK(ring >= 0); + struct iovec pin = { .iov_base = src + 2 * ps, .iov_len = ps }; + CHECK(!syscall(SYS_io_uring_register, ring, IORING_REGISTER_BUFFERS, + &pin, 1)); + p = physical(src + 2 * ps, ps); + CHECK(move(fd, dst + 2 * ps, src + 2 * ps, ps, 0, EBUSY) == -EBUSY); + CHECK(physical(src + 2 * ps, ps) == p && + !(entry(dst + 2 * ps, ps) & (PRESENT | SWAPPED))); + bytes(src + 2 * ps, 2 * ps, ps); + uint64_t neighbour = physical(src + ps, ps); + move(fd, dst + 3 * ps, src + ps, ps, 0, 0); + CHECK(physical(dst + 3 * ps, ps) == neighbour); + bytes(dst + 3 * ps, ps, ps); + CHECK(!syscall(SYS_io_uring_register, ring, IORING_UNREGISTER_BUFFERS, + NULL, 0)); + CHECK(!close(ring)); + move(fd, dst + 2 * ps, src + 2 * ps, ps, 0, 0); + CHECK(physical(dst + 2 * ps, ps) == p); + bytes(dst + 2 * ps, 2 * ps, ps); + printf("ok - MOVE %zuK source PIN rejected atomically, neighbour PIN allowed, unpin retry\n", + ps / 1024); + + CHECK(move(fd, dst + ps, src + 4 * ps, ps, 0, EEXIST) == -EEXIST); + bytes(src + 4 * ps, 4 * ps, ps); + bytes(dst + ps, 3 * ps, ps); + CHECK(!madvise(src + 5 * native, native, MADV_DONTNEED)); + CHECK(*(volatile unsigned char *)(src + 5 * native) == 0); + move(fd, dst + 5 * native, src + 5 * native, ps, 0, 0); + CHECK(dst[5 * native] == 0); + dst[5 * native] = 0x7d; + CHECK(src[5 * native] == 0); + CHECK(!madvise(src + 6 * native, native, MADV_DONTNEED)); + CHECK(move(fd, dst + 6 * native, src + 6 * native, ps, 0, ENOENT) == + -ENOENT); + move(fd, dst + 6 * native, src + 6 * native, ps, + UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES, 0); + CHECK(!(entry(dst + 6 * native, ps) & (PRESENT | SWAPPED))); + CHECK(!madvise(src + 10 * native + ps, ps, MADV_DONTNEED)); + p = physical(src + 10 * native, ps); + CHECK(move(fd, dst + 12 * native, src + 10 * native, 2 * ps, 0, + EAGAIN) == (long)ps); + CHECK(physical(dst + 12 * native, ps) == p); + bytes(dst + 12 * native, 10 * native, ps); + CHECK(!(entry(dst + 12 * native + ps, ps) & (PRESENT | SWAPPED))); + printf("ok - MOVE %zuK collision, zero, holes and partial progress\n", + ps / 1024); + + int pipefd[2]; + CHECK(!pipe(pipefd)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + close(pipefd[1]); + char c; + _exit(read(pipefd[0], &c, 1) == 1 ? 0 : 1); + } + close(pipefd[0]); + CHECK(move(fd, dst + 11 * native, src + 12 * native, ps, 0, EBUSY) == + -EBUSY); + CHECK(write(pipefd[1], "x", 1) == 1); + close(pipefd[1]); + int status; + CHECK(waitpid(child, &status, 0) == child && status == 0); + *(volatile unsigned char *)(src + 12 * native) = pattern(12 * native); + p = physical(src + 12 * native, ps); + move(fd, dst + 11 * native, src + 12 * native, ps, 0, 0); + CHECK(physical(dst + 11 * native, ps) == p); + bytes(dst + 11 * native, 12 * native, ps); + printf("ok - MOVE %zuK fork-shared source rejected, exclusive write/retry\n", + ps / 1024); + + /* Keep fault-in coverage as well as restoring an untouched PTE at swapoff. */ + for (unsigned int unuse = 0; unuse < 2; unuse++) { + size_t off = ps < native ? ps : 0; + size_t base = (unuse ? 13 : 8) * native; + unsigned char *s = src + base + off; + unsigned char *d = dst + base + 2 * ps; + + *(volatile unsigned char *)s = pattern(base + off); + for (unsigned int tries = 0; + !(entry(s, ps) & SWAPPED) && tries < 100; tries++) { + CHECK(!madvise(src + base, native, MADV_PAGEOUT)); + usleep(20000); + } + uint64_t old = entry(s, ps); + + CHECK(old & SWAPPED); + move(fd, d, s, ps, 0, 0); + uint64_t now = entry(d, ps); + + CHECK((now & SWAPPED) && ((now & VALUE_MASK) == (old & VALUE_MASK)) && + !(entry(s, ps) & (PRESENT | SWAPPED))); + if (unuse) { + /* Do not fault the moved slot in before swapoff restores it. */ + CHECK(!swapoff("/dev/vda")); + CHECK(entry(d, ps) & PRESENT); + } + bytes(d, base + off, ps); + if (unuse) + CHECK(!swapon("/dev/vda", 0)); + printf("ok - MOVE %zuK %s restores transferred swap PTE at new offset\n", + ps / 1024, unuse ? "swapoff" : "fault"); + } + CHECK(!close(fd)); + CHECK(!munmap(src, len)); + CHECK(!munmap(dst, len)); + concurrent_move(ps); + allocation_failure(ps, native); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)); + struct stat st; + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + CHECK(!stat("/dev/vda", &st) && S_ISBLK(st.st_mode) && + !swapon("/dev/vda", 0)); + size_t native = getauxval(AT_PAGESZ), sizes[] = { native, 4096, 16384 }; + int failed = 0; + for (unsigned i = 0; i < (native > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + CHECK(c >= 0); + if (!c) { + char n[32]; + snprintf(n, sizeof(n), "%zu", native); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", n, NULL); + _exit(5); + } + CHECK(waitpid(c, &status, 0) == c); + printf("%s - MOVE ABI %zuK status=%#x\n", + status ? "not ok" : "ok", sizes[i] / 1024, status); + failed += !!status; + } + if (swapoff("/dev/vda")) + failed++; + printf("UFFD MOVE CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c new file mode 100644 index 0000000000000..19747755c4c6c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c @@ -0,0 +1,913 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - uffd line=%d errno=%d\n", __LINE__, \ + errno); \ + exit(1); \ + } \ + } while (0) +struct writer { + volatile unsigned char *address; + unsigned char value; + atomic_int done, tid; +}; +static void *write_byte(void *opaque) +{ + struct writer *w = opaque; + atomic_store(&w->tid, syscall(SYS_gettid)); + *w->address = w->value; + atomic_store(&w->done, 1); + return NULL; +} +struct forker { + unsigned char *base; + size_t page_size; + pid_t pid; +}; +static void *fork_byte(void *opaque) +{ + struct forker *f = opaque; + f->pid = fork(); + if (!f->pid) { + alarm(10); + f->base[f->page_size + 17] = 0xa5; + _exit(f->base[f->page_size + 17] != 0xa5); + } + return NULL; +} +static struct uffd_msg read_event(int fd) +{ + struct pollfd p = { .fd = fd, .events = POLLIN }; + struct uffd_msg msg; + CHECK(poll(&p, 1, 10000) == 1 && (p.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + return msg; +} +static void protect(int fd, void *start, size_t len, unsigned long mode) +{ + struct uffdio_writeprotect wp = { .range = { (uintptr_t)start, len }, + .mode = mode }; + CHECK(!ioctl(fd, UFFDIO_WRITEPROTECT, &wp)); +} +static void resolve_write(int fd, unsigned char *base, size_t ps, unsigned page, + unsigned char value, int exact) +{ + struct writer w = { .address = base + page * ps + 17, .value = value }; + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + struct uffd_msg msg; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT); + CHECK(msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_WRITE)); + CHECK(msg.arg.pagefault.address == + (uintptr_t)(base + page * ps + (exact ? 17 : 0))); + CHECK(msg.arg.pagefault.feat.ptid == (unsigned int)atomic_load(&w.tid)); + CHECK(!atomic_load(&w.done)); + protect(fd, base + page * ps, ps, UFFDIO_WRITEPROTECT_MODE_DONTWAKE); + struct uffdio_range range = { (uintptr_t)(base + page * ps), ps }; + CHECK(!ioctl(fd, UFFDIO_WAKE, &range)); + CHECK(!pthread_join(thread, NULL) && atomic_load(&w.done)); + CHECK(base[page * ps + 17] == value); + CHECK(poll(&pollfd, 1, 0) == 0); +} +static void fill(int fd, void *dst, void *src, size_t len, unsigned long mode) +{ + struct uffdio_copy copy = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = len, + .mode = mode }; + CHECK(!ioctl(fd, UFFDIO_COPY, ©) && copy.copy == (long long)len); +} +static void zero(int fd, void *dst, size_t len) +{ + struct uffdio_zeropage z = { .range = { (uintptr_t)dst, len } }; + CHECK(!ioctl(fd, UFFDIO_ZEROPAGE, &z) && z.zeropage == (long long)len); +} +static uint64_t physical(void *address, size_t ps) +{ + uint64_t entry; + int fd = open("/proc/self/pagemap", O_RDONLY); + CHECK(fd >= 0 && + pread(fd, &entry, 8, (uintptr_t)address / ps * 8) == 8); + CHECK(!close(fd) && (entry & (1ULL << 63))); + return (entry & ((1ULL << 55) - 1)) * ps; +} +static void missing(size_t ps, size_t native) +{ + size_t len = 4 * native; + unsigned char *map = mmap(NULL, len + native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *src = mmap(NULL, len + native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(map != MAP_FAILED && src != MAP_FAILED); + unsigned char *p = map + ps; + CHECK(!mlock2(p, len, MLOCK_ONFAULT)); + for (size_t i = 0; i < len; i++) + src[i] = (i / ps) ^ (i * 7 + 3); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_WP }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK((reg.ioctls & + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE))) == + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE))); + /* A write blocks only on its missing leaf; COPY_MODE_WP causes a second event. */ + struct writer w = { .address = p + 17, .value = 0xd3 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == UFFD_PAGEFAULT_FLAG_WRITE && + !atomic_load(&w.done)); + fill(fd, p, src, ps, UFFDIO_COPY_MODE_WP); + msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WRITE | UFFD_PAGEFAULT_FLAG_WP)); + protect(fd, p, ps, 0); + CHECK(!pthread_join(thread, NULL) && atomic_load(&w.done)); + src[17] = w.value; + CHECK(!memcmp(p, src, ps)); + zero(fd, p + ps, ps); + for (size_t i = 0; i < ps; i++) + CHECK(p[ps + i] == 0); + p[ps + 17] = 0xe4; + CHECK(!memcmp(p, src, ps)); + /* A batch starts at a user-page boundary, not necessarily a native one. */ + fill(fd, p + 2 * ps, src + 2 * ps, len - 2 * ps, 0); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, len - 2 * ps)); + if (ps < native) { + unsigned char *a = + (void *)(((uintptr_t)(p + 2 * ps) + native - 1) & + ~(native - 1)); + uint64_t start = physical(a, ps); + CHECK(start != 0 && !(start & (native - 1))); + for (size_t i = ps; i < native; i += ps) + CHECK(physical(a + i, ps) == start + i); + } + struct uffdio_copy copy = { .dst = (uintptr_t)p, + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EEXIST && + copy.copy == -EEXIST); + CHECK(!munlock(p, len)); + /* Preserve the successfully copied prefix when the next PTE already exists. */ + CHECK(!madvise(p, ps, MADV_DONTNEED)); + copy = (struct uffdio_copy){ .dst = (uintptr_t)p, + .src = (uintptr_t)src, + .len = 2 * ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EAGAIN && + copy.copy == (long long)ps); + CHECK(!memcmp(p, src, ps) && p[ps + 17] == 0xe4); + /* An invalid source stops at its leaf, without installing it or touching neighbours. */ + CHECK(!madvise(p + 2 * ps, 2 * ps, MADV_DONTNEED)); + CHECK(!mprotect(src + 3 * ps, ps, PROT_NONE)); + copy = (struct uffdio_copy){ .dst = (uintptr_t)(p + 2 * ps), + .src = (uintptr_t)(src + 2 * ps), + .len = 2 * ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EAGAIN && + copy.copy == (long long)ps); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, ps)); + CHECK(!mprotect(src + 3 * ps, ps, PROT_READ | PROT_WRITE)); + /* A fresh source forces the unlocked copy retry. */ + CHECK(!madvise(src + 3 * ps, ps, MADV_DONTNEED)); + fill(fd, p + 3 * ps, src + 3 * ps, ps, 0); + for (size_t i = 0; i < ps; i++) + CHECK(p[3 * ps + i] == 0); + CHECK(!memcmp(p + 4 * ps, src + 4 * ps, len - 4 * ps)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + p[2 * ps + 9] ^= 0xff; + _exit(p[2 * ps + 9] == src[2 * ps + 9]); + } + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, ps)); + CHECK(!close(fd)); + CHECK(!munmap(map, len + native) && !munmap(src, len + native)); + printf("ok - uffd ABI %zuK missing copy/WP, zero, packed batch, partial EEXIST/EFAULT, source retry and COW\n", + ps / 1024); +} + +struct reader { + volatile unsigned char *address; + unsigned char value; + atomic_int done; +}; +static void *read_byte(void *opaque) +{ + struct reader *r = opaque; + r->value = *r->address; + atomic_store(&r->done, 1); + return NULL; +} +static void resume_minor(int fd, void *start, size_t len, unsigned long mode) +{ + struct uffdio_continue c = { .range = { (uintptr_t)start, len }, + .mode = mode }; + CHECK(!ioctl(fd, UFFDIO_CONTINUE, &c) && c.mapped == (long long)len); +} +static void file_minor(size_t ps, size_t native) +{ + for (int shared = 0; shared < 2; shared++) { + int memfd = memfd_create("uffd-minor-granule", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * native)); + unsigned char *alias = mmap(NULL, 4 * native, + PROT_READ | PROT_WRITE, MAP_SHARED, + memfd, 0); + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + shared ? MAP_SHARED : MAP_PRIVATE, + memfd, ps); + CHECK(alias != MAP_FAILED && p != MAP_FAILED); + CHECK(!mlock2(p, 3 * ps, MLOCK_ONFAULT)); + for (size_t i = 0; i < 4 * native; i++) + alias[i] = (i / ps) ^ (i * 3 + 7); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { + .api = UFFD_API, + .features = UFFD_FEATURE_MINOR_SHMEM | + UFFD_FEATURE_PAGEFAULT_FLAG_WP + }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { + .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_MINOR | + UFFDIO_REGISTER_MODE_WP + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(reg.ioctls & (1ULL << _UFFDIO_CONTINUE)); + + for (unsigned i = 0; i < 3; i++) { + struct reader r = { .address = p + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == + (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == + UFFD_PAGEFAULT_FLAG_MINOR && + !atomic_load(&r.done)); + alias[(i + 1) * ps + 17] ^= 0x3c; + resume_minor(fd, p + i * ps, ps, + UFFDIO_CONTINUE_MODE_WP | + UFFDIO_CONTINUE_MODE_DONTWAKE); + struct uffdio_range range = { (uintptr_t)(p + i * ps), + ps }; + CHECK(!ioctl(fd, UFFDIO_WAKE, &range)); + CHECK(!pthread_join(thread, NULL) && + atomic_load(&r.done)); + CHECK(r.value == alias[(i + 1) * ps + 17]); + CHECK(physical(p + i * ps, ps) == + physical(alias + (i + 1) * ps, ps)); + /* Independent write protection survives installing the file leaf. */ + struct writer w = { .address = p + i * ps + 17, + .value = 0xa1 + i }; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == + (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WP | + UFFD_PAGEFAULT_FLAG_WRITE)); + protect(fd, p + i * ps, ps, 0); + CHECK(!pthread_join(thread, NULL) && + atomic_load(&w.done)); + CHECK(p[i * ps + 17] == w.value); + CHECK(shared ? alias[(i + 1) * ps + 17] == w.value : + alias[(i + 1) * ps + 17] == r.value); + } + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + CHECK(poll(&pollfd, 1, 0) == 0); + CHECK(!munlock(p, 3 * ps)); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + resume_minor(fd, p, 3 * ps, 0); + CHECK(!memcmp(p, alias + ps, 3 * ps)); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + CHECK(!ftruncate(memfd, ps + 17)); + resume_minor(fd, p, ps, 0); + struct uffdio_continue c = { .range = { (uintptr_t)(p + ps), + ps } }; + CHECK(ioctl(fd, UFFDIO_CONTINUE, &c) == -1 && errno == EFAULT && + c.mapped == -EFAULT); + CHECK(!close(fd) && !munmap(p, 3 * ps) && + !munmap(alias, 4 * native) && !close(memfd)); + printf("ok - uffd ABI %zuK shmem %s minor/WP, byte offsets, alias coherence, COW, batch continue and EOF\n", + ps / 1024, shared ? "shared" : "private"); + } +} + +static uint64_t pagemap(void *address, size_t ps) +{ + uint64_t entry; + int fd = open("/proc/self/pagemap", O_RDONLY); + CHECK(fd >= 0 && + pread(fd, &entry, 8, (uintptr_t)address / ps * 8) == 8); + CHECK(!close(fd)); + return entry; +} +static sigjmp_buf bus_env; +static void *bus_address; +static void bus_handler(int sig, siginfo_t *info, void *context) +{ + (void)context; + if (sig != SIGBUS) + _exit(91); + bus_address = info->si_addr; + siglongjmp(bus_env, 1); +} +static void markers(size_t ps) +{ + for (int async = 0; async < 2; async++) { + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { + .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_THREAD_ID | + UFFD_FEATURE_WP_UNPOPULATED | + (async ? UFFD_FEATURE_WP_ASYNC : 0) + }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { + .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_WP + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + protect(fd, p, 3 * ps, UFFDIO_WRITEPROTECT_MODE_WP); + for (unsigned i = 0; i < 3; i++) + CHECK(pagemap(p + i * ps, ps) & (1ULL << 57)); + CHECK(*(volatile unsigned char *)(p + ps + 17) == 0); + if (async) + p[ps + 17] = 0x67; + else + resolve_write(fd, p, ps, 1, 0x67, 0); + CHECK(p[ps + 17] == 0x67); + CHECK(pagemap(p, ps) & (1ULL << 57)); + CHECK(!(pagemap(p + ps, ps) & (1ULL << 57))); + CHECK(pagemap(p + 2 * ps, ps) & (1ULL << 57)); + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + CHECK(poll(&pollfd, 1, 0) == 0); + CHECK(!close(fd)); + p[17] = 0x68; + p[2 * ps + 17] = 0x69; + CHECK(!munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK unpopulated WP async=%d per-leaf pagemap state and close\n", + ps / 1024, async); + } + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_POISON }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®) && + (reg.ioctls & (1ULL << _UFFDIO_POISON))); + zero(fd, p, ps); + zero(fd, p + 2 * ps, ps); + struct uffdio_poison poison = { .range = { (uintptr_t)(p + ps), ps } }; + CHECK(!ioctl(fd, UFFDIO_POISON, &poison) && + poison.updated == (long long)ps); + CHECK(ioctl(fd, UFFDIO_POISON, &poison) == -1 && errno == EEXIST && + poison.updated == -EEXIST); + struct sigaction sa = { .sa_sigaction = bus_handler, + .sa_flags = SA_SIGINFO }, + old; + CHECK(!sigemptyset(&sa.sa_mask) && !sigaction(SIGBUS, &sa, &old)); + if (!sigsetjmp(bus_env, 1)) { + (void)*(volatile unsigned char *)(p + ps + 17); + CHECK(0); + } + CHECK(bus_address == p + ps + 17); + CHECK(!sigaction(SIGBUS, &old, NULL)); + p[17] = 0x81; + p[2 * ps + 17] = 0x82; + CHECK(!close(fd) && !munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK poisoned leaf signals SIGBUS and preserves neighbours\n", + ps / 1024); +} +struct pager_request { + uintptr_t address; + size_t size; +}; +static void send_pager(int sock, int fd, struct pager_request *request) +{ + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct iovec iov = { .iov_base = request, .iov_len = sizeof(*request) }; + struct msghdr msg = { .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control) }; + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + c->cmsg_level = SOL_SOCKET; + c->cmsg_type = SCM_RIGHTS; + c->cmsg_len = CMSG_LEN(sizeof(int)); + memcpy(CMSG_DATA(c), &fd, sizeof(fd)); + CHECK(sendmsg(sock, &msg, 0) == sizeof(*request)); +} +static unsigned char pager_pattern(size_t i) +{ + return (i / 4096) ^ (i * 11 + 5); +} +static int pager_worker(int sock, size_t expected) +{ + CHECK(getauxval(AT_PAGESZ) == expected); + alarm(30); + struct pager_request request; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct iovec iov = { .iov_base = &request, .iov_len = sizeof(request) }; + struct msghdr msg = { .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control) }; + CHECK(recvmsg(sock, &msg, 0) == sizeof(request)); + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + CHECK(c && c->cmsg_level == SOL_SOCKET && c->cmsg_type == SCM_RIGHTS && + c->cmsg_len == CMSG_LEN(sizeof(int))); + int fd; + memcpy(&fd, CMSG_DATA(c), sizeof(fd)); + size_t ps = request.size; + unsigned char *source = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(source != MAP_FAILED); + for (size_t i = 0; i < 4 * ps; i++) + source[i] = pager_pattern(i); + for (unsigned i = 0; i < 3; i++) { + struct uffd_msg fault = read_event(fd); + CHECK(fault.event == UFFD_EVENT_PAGEFAULT && + fault.arg.pagefault.address == request.address + i * ps && + !fault.arg.pagefault.flags); + if (i == 1) + zero(fd, (void *)(request.address + ps), ps); + else + fill(fd, (void *)(request.address + i * ps), + source + i * ps, (i == 2 ? 2 : 1) * ps, 0); + } + CHECK(!close(fd) && !close(sock) && !munmap(source, 4 * ps)); + return 0; +} +static void cross_pager(size_t ps, size_t native, int shared) +{ + size_t pager_ps = ps == native ? 4096 : native; + int socks[2]; + CHECK(!socketpair(AF_UNIX, SOCK_SEQPACKET, 0, socks)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + char fd_arg[32], ps_arg[32]; + CHECK(!close(socks[0])); + snprintf(fd_arg, sizeof(fd_arg), "%d", socks[1]); + snprintf(ps_arg, sizeof(ps_arg), "%zu", pager_ps); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, pager_ps, 0UL, 0UL, 0UL)); + execl("/init", "/init", "--pager", fd_arg, ps_arg, NULL); + _exit(3); + } + CHECK(!close(socks[1])); + int memfd = shared ? memfd_create("remote-uffd-shmem", 0) : -1; + if (shared) + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap( + NULL, 4 * ps, PROT_READ | PROT_WRITE, + shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS, memfd, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, 4 * ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + struct pager_request request = { .address = (uintptr_t)p, .size = ps }; + send_pager(socks[0], fd, &request); + for (size_t i = 0; i < 4 * ps; i++) + CHECK(p[i] == (i >= ps && i < 2 * ps ? 0 : pager_pattern(i))); + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!close(fd) && !close(socks[0]) && !munmap(p, 4 * ps)); + if (shared) + CHECK(!close(memfd)); + printf("ok - uffd target %zuK remote pager %zuK shared=%d SCM_RIGHTS copy/zero/batch uses target granule\n", + ps / 1024, pager_ps / 1024, shared); +} + +static void private_missing(size_t ps) +{ + int memfd = memfd_create("uffd-private-hole", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE, memfd, ps); + unsigned char *src = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && src != MAP_FAILED); + for (size_t i = 0; i < 3 * ps; i++) + src[i] = pager_pattern(i); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®) && + (reg.ioctls & (1ULL << _UFFDIO_COPY))); + struct reader r = { .address = p + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + !msg.arg.pagefault.flags); + fill(fd, p, src, 2 * ps, 0); + CHECK(!pthread_join(thread, NULL) && r.value == src[17]); + zero(fd, p + 2 * ps, ps); + CHECK(!memcmp(p, src, 2 * ps)); + unsigned char bytes[256]; + for (size_t off = 0; off < 4 * ps; off += sizeof(bytes)) { + CHECK(pread(memfd, bytes, sizeof(bytes), off) == sizeof(bytes)); + for (unsigned i = 0; i < sizeof(bytes); i++) + CHECK(bytes[i] == 0); + } + CHECK(!madvise(p, ps, MADV_DONTNEED)); + CHECK(!madvise(src, 3 * ps, MADV_DONTNEED)); + fill(fd, p, src, ps, 0); + for (size_t i = 0; i < ps; i++) + CHECK(p[i] == 0); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + CHECK(!ftruncate(memfd, ps + 17)); + struct uffdio_copy copy = { .dst = (uintptr_t)(p + ps), + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EFAULT && + copy.copy == -EFAULT); + fill(fd, p, src, ps, 0); + CHECK(!close(fd) && !close(memfd) && !munmap(p, 3 * ps) && + !munmap(src, 3 * ps)); + printf("ok - uffd ABI %zuK private shmem missing COPY/ZERO, unchanged file bytes, source retry and EOF\n", + ps / 1024); +} + +static int register_shared(void *p, size_t len) +{ + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MISSING_SHMEM | + UFFD_FEATURE_MINOR_SHMEM | + UFFD_FEATURE_PAGEFAULT_FLAG_WP }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_MINOR | + UFFDIO_REGISTER_MODE_WP }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK((reg.ioctls & + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE) | + (1ULL << _UFFDIO_CONTINUE))) == + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE) | + (1ULL << _UFFDIO_CONTINUE))); + return fd; +} +static void shared_missing(size_t ps, size_t native) +{ + int memfd = memfd_create("uffd-shared-holes", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 8 * native)); + unsigned char *p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, ps); + unsigned char *q = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, ps); + unsigned char *src = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && q != MAP_FAILED && src != MAP_FAILED); + for (size_t i = 0; i < 4 * ps; i++) + src[i] = pager_pattern(i); + int fd = register_shared(p, 4 * ps), fd2 = register_shared(q, 4 * ps); + for (unsigned i = 0; i < 2; i++) { + struct reader r = { .address = p + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)(p + i * ps) && + !msg.arg.pagefault.flags); + if (!i) + fill(fd, p, src, ps, UFFDIO_COPY_MODE_WP); + else + zero(fd, p + ps, ps); + CHECK(!pthread_join(thread, NULL) && atomic_load(&r.done)); + CHECK(r.value == (i ? 0 : src[17])); + if (!i) { + struct uffdio_continue c = { + .range = { (uintptr_t)(p + ps), ps } + }; + CHECK(ioctl(fd, UFFDIO_CONTINUE, &c) == -1 && + errno == EFAULT && c.mapped == -EFAULT); + } + } + fill(fd, p + 2 * ps, src + 2 * ps, 2 * ps, 0); + CHECK(!memcmp(p, src, ps) && !memcmp(p + 2 * ps, src + 2 * ps, 2 * ps)); + for (size_t i = 0; i < ps; i++) + CHECK(p[ps + i] == 0); + for (unsigned i = 0; i < 4; i++) { + struct reader r = { .address = q + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd2); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)(q + i * ps) && + msg.arg.pagefault.flags == UFFD_PAGEFAULT_FLAG_MINOR); + resume_minor(fd2, q + i * ps, ps, 0); + CHECK(!pthread_join(thread, NULL) && r.value == p[i * ps + 17]); + CHECK(physical(p + i * ps, ps) == physical(q + i * ps, ps)); + } + CHECK(!memcmp(p, q, 4 * ps)); + if (ps < native) + CHECK(physical(p + ps, ps) == physical(p, ps) + ps); + struct writer w = { .address = p + 17, .value = 0x9a }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WRITE | UFFD_PAGEFAULT_FLAG_WP)); + protect(fd, p, ps, 0); + CHECK(!pthread_join(thread, NULL) && p[17] == w.value && + q[17] == w.value); + struct uffdio_copy c = { .dst = (uintptr_t)(p + ps), + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && errno == EEXIST && + c.copy == -EEXIST); + unsigned char bytes[256]; + CHECK(pread(memfd, bytes, sizeof(bytes), ps) == sizeof(bytes)); + CHECK(!memcmp(bytes, p, sizeof(bytes))); + /* Resolve an untouched source outside VMA locks before taking a cache folio. */ + unsigned char *r = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, 6 * native); + unsigned char *fresh = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(r != MAP_FAILED && fresh != MAP_FAILED); + int fd3 = register_shared(r, 2 * ps); + fill(fd3, r, fresh, ps, 0); + for (size_t i = 0; i < ps; i++) + CHECK(r[i] == 0); + CHECK(!mprotect(fresh, ps, PROT_NONE)); + c = (struct uffdio_copy){ .dst = (uintptr_t)(r + ps), + .src = (uintptr_t)fresh, + .len = ps }; + CHECK(ioctl(fd3, UFFDIO_COPY, &c) == -1 && errno == EFAULT && + c.copy == -EFAULT); + zero(fd3, r + ps, ps); + for (size_t i = 0; i < ps; i++) + CHECK(r[ps + i] == 0); + CHECK(!close(fd3) && !munmap(r, 2 * ps) && !munmap(fresh, ps)); + CHECK(!ftruncate(memfd, 2 * ps + 17)); + c = (struct uffdio_copy){ .dst = (uintptr_t)(p + 2 * ps), + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && + errno == (ps == native ? ENOMEM : EFAULT) && + c.copy == -(ps == native ? ENOMEM : EFAULT)); + CHECK(!close(fd) && !close(fd2) && !close(memfd)); + CHECK(!munmap(p, 4 * ps) && !munmap(q, 4 * ps) && !munmap(src, 4 * ps)); + printf("ok - uffd ABI %zuK shared COPY/ZERO, independent missing/minor pagers, CONTINUE holes, packing, WP/coherence and EOF\n", + ps / 1024); +} + +static void shared_buffered_write(size_t ps) +{ + int memfd = memfd_create("uffd-buffered-validity", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, 0); + unsigned char *src = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && src != MAP_FAILED); + memset(src, 0x57, ps); + int fd = register_shared(p, 4 * ps); + fill(fd, p, src, ps, 0); + CHECK(pwrite(memfd, src, 23, ps + 17) == 23); + for (unsigned i = 1; i < 3; i++) { + struct reader r = { .address = p + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == + (i == 1 ? UFFD_PAGEFAULT_FLAG_MINOR : 0)); + if (i == 1) + resume_minor(fd, p + ps, ps, 0); + else + zero(fd, p + i * ps, ps); + CHECK(!pthread_join(thread, NULL) && + r.value == (i == 1 ? 0x57 : 0)); + } + for (size_t i = 0; i < ps; i++) + CHECK(p[ps + i] == (i >= 17 && i < 40 ? 0x57 : 0)); + CHECK(!memcmp(p, src, ps)); + CHECK(!close(fd) && !close(memfd) && !munmap(p, 4 * ps) && + !munmap(src, ps)); + printf("ok - uffd ABI %zuK buffered short-range write populates its leaf and leaves neighbours missing\n", + ps / 1024); +} + +static int worker(unsigned long expected, unsigned long native) +{ + size_t ps = getauxval(AT_PAGESZ); + CHECK(ps == expected); + alarm(60); + missing(ps, native); + file_minor(ps, native); + markers(ps); + cross_pager(ps, native, 0); + cross_pager(ps, native, 1); + private_missing(ps); + shared_missing(ps, native); + shared_buffered_write(ps); + for (int exact = 0; exact < 2; exact++) { + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + for (unsigned i = 0; i < 3; i++) + memset(p + i * ps, 0x40 + i, ps); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { + .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_THREAD_ID | + (exact ? UFFD_FEATURE_EXACT_ADDRESS | + UFFD_FEATURE_EVENT_FORK : + 0) + }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + CHECK(api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP); + struct uffdio_register reg = { + .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_WP + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(reg.ioctls & (1ULL << _UFFDIO_WRITEPROTECT)); + + for (unsigned i = 0; i < 3; i++) { + protect(fd, p + i * ps, ps, + UFFDIO_WRITEPROTECT_MODE_WP); + /* Neighbours remain directly writable while this page faults. */ + p[((i + 1) % 3) * ps + 31] ^= 1; + resolve_write(fd, p, ps, i, 0x70 + i, exact); + } + protect(fd, p + ps, ps, UFFDIO_WRITEPROTECT_MODE_WP); + pid_t child; + int status; + if (!exact) { + child = fork(); + CHECK(child >= 0); + if (!child) { + alarm(10); + p[ps + 17] = 0xa5; + _exit(p[ps + 17] != 0xa5); + } + } else { + struct forker f = { .base = p, .page_size = ps }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, fork_byte, &f)); + struct uffd_msg event = read_event(fd); + CHECK(event.event == UFFD_EVENT_FORK); + int child_fd = event.arg.fork.ufd; + CHECK(child_fd >= 0); + CHECK(!pthread_join(thread, NULL)); + child = f.pid; + CHECK(child > 0); + event = read_event(child_fd); + CHECK(event.event == UFFD_EVENT_PAGEFAULT && + event.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WP | + UFFD_PAGEFAULT_FLAG_WRITE)); + CHECK(event.arg.pagefault.address == + (uintptr_t)(p + ps + 17)); + CHECK(event.arg.pagefault.feat.ptid == + (unsigned int)child); + protect(child_fd, p + ps, ps, 0); + CHECK(!close(child_fd)); + } + CHECK(waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(p[ps + 17] == 0x71); + resolve_write(fd, p, ps, 1, 0x91, exact); + protect(fd, p, 3 * ps, UFFDIO_WRITEPROTECT_MODE_WP); + struct uffdio_range middle = { (uintptr_t)(p + ps), ps }; + CHECK(!ioctl(fd, UFFDIO_UNREGISTER, &middle)); + p[ps + 17] = 0xb2; + resolve_write(fd, p, ps, 0, 0xb1, exact); + CHECK(!close(fd)); + /* Close removes the remaining registration and write protection. */ + p[2 * ps + 17] = 0xb3; + CHECK(p[17] == 0xb1 && p[ps + 17] == 0xb2 && + p[2 * ps + 17] == 0xb3); + CHECK(!munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK exact=%d events, neighbours, fork delivery/COW, partial unregister and close\n", + ps / 1024, exact); + } + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--pager")) + return pager_worker(atoi(argv[2]), strtoul(argv[3], NULL, 10)); + if (argc == 4 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { native, 4096, 16384 }; + unsigned modes = native == 65536 ? 3 : 2; + int failed = 0; + for (unsigned i = 0; i < modes; i++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char ps[32], ns[32]; + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(ns, sizeof(ns), "%lu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, + 0UL)); + execl("/init", "/init", "--worker", ps, ns, NULL); + _exit(3); + } + int status = 0; + int ok = waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status); + printf("%s - uffd ABI %luK status=%#x\n", ok ? "ok" : "not ok", + sizes[i] / 1024, status); + failed |= !ok; + } + printf("UFFD CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uprobes-init.c b/tools/testing/selftests/arm64/user4k/uprobes-init.c new file mode 100644 index 0000000000000..0c379e4309caf --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uprobes-init.c @@ -0,0 +1,318 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define TRACE "/sys/kernel/tracing/" +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +static int write_file(const char *path, const char *s) +{ + int fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + size_t len = strlen(s); + int ret = write(fd, s, len) == (ssize_t)len ? 0 : -1; + close(fd); + return ret; +} +static int enable(const char *event, int on) +{ + char path[160]; + snprintf(path, sizeof(path), TRACE "events/aurora_up/%s/enable", event); + return write_file(path, on ? "1" : "0"); +} +static uint64_t pagemap(void *p, size_t ps) +{ + uint64_t bits = 0; + int fd = open("/proc/self/pagemap", O_RDONLY); + if (fd >= 0) { + if (pread(fd, &bits, 8, (uintptr_t)p / ps * 8) != 8) + bits = 0; + close(fd); + } + return bits; +} +static unsigned records(const char *event) +{ + FILE *f = fopen(TRACE "trace", "r"); + char *line = NULL; + size_t len = 0; + unsigned count = 0; + char match[96]; + snprintf(match, sizeof(match), "%s:", event); + if (!f) + return 0; + while (getline(&line, &len, f) > 0) + if (strstr(line, match)) + count++; + free(line); + fclose(f); + return count; +} +static int xol_geometry(size_t native) +{ + FILE *f = fopen("/proc/self/maps", "r"); + char *line = NULL; + size_t len = 0; + int ok = 0; + if (!f) + return 0; + while (getline(&line, &len, f) > 0) + if (strstr(line, "[uprobes]")) { + unsigned long a, b; + if (sscanf(line, "%lx-%lx", &a, &b) == 2) { + printf("# XOL %#lx-%#lx native=%zu\n", a, b, + native); + ok = !(a % (unsigned long)native) && + b - a == native; + } + } + free(line); + fclose(f); + return ok; +} +static int exercise(size_t ps, size_t native, unsigned shifted) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(20); + printf("# start ps=%zu shifted=%u\n", ps, shifted); + int fd = memfd_create("aurora-uprobe-code", 0); + CHECK(fd >= 0); + size_t length = 8 * ps; + unsigned char *blob = malloc(length); + CHECK(blob); + memset(blob, 0x71, length); + size_t code_off = 3 * ps + 64, ctr_off = 5 * ps + 32; + size_t map_off = shifted ? 2 * ps : 0, + map_len = shifted ? 2 * ps : 4 * ps; + uint32_t insns[] = { 0x11001c00, 0xd65f03c0 }; + memcpy(blob + code_off, insns, sizeof(insns)); + memset(blob + ctr_off, 0, 2); + CHECK(write(fd, blob, length) == (ssize_t)length); + unsigned char *code = mmap((void *)(0x20000000 + (shifted ? ps : 0)), + map_len, PROT_READ | PROT_EXEC, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, + map_off); + unsigned char *counter = + mmap((void *)(0x30000000 + 2 * ps), ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, 5 * ps); + unsigned char *other = mmap((void *)0x40000000, ps, + PROT_READ | PROT_EXEC, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, 0); + CHECK(code != MAP_FAILED && counter != MAP_FAILED && + other != MAP_FAILED); + unsigned long (*target)(unsigned long) = + (void *)(code + code_off - map_off); + volatile unsigned short *ctr = (void *)(counter + 32); + CHECK(*ctr == 0); + CHECK(target(11) == 18); + /* Occupy the default top-of-address-space XOL hint, forcing a new placement. */ + void *guard = mmap((void *)((1UL << 48) - native), ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, + -1, 0); + CHECK(guard != MAP_FAILED); + char entry[64], retname[64], cmd[256]; + snprintf(entry, sizeof(entry), "entry_%zu_%u", ps, shifted); + snprintf(retname, sizeof(retname), "return_%zu_%u", ps, shifted); + snprintf(cmd, sizeof(cmd), + "p:aurora_up/%s /proc/self/fd/%d:0x%zx(0x%zx)\n", entry, fd, + code_off, ctr_off); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + snprintf(cmd, sizeof(cmd), + "r:aurora_up/%s /proc/self/fd/%d:0x%zx(0x%zx)\n", retname, fd, + code_off, ctr_off); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(*ctr == 1); + printf("# enabling return\n"); + CHECK(!enable(retname, 1)); + printf("# return enabled\n"); + CHECK(*ctr == 1); + uint32_t disk[2]; + CHECK(pread(fd, disk, sizeof(disk), code_off) == sizeof(disk)); + CHECK(!memcmp(disk, insns, sizeof(insns))); + CHECK(!memcmp(other, blob, ps)); + CHECK(!memcmp(counter, blob + 5 * ps, 32)); + CHECK(!memcmp(counter + 34, blob + 5 * ps + 34, ps - 34)); + int tracefd = open(TRACE "trace", O_WRONLY | O_TRUNC); + CHECK(tracefd >= 0); + close(tracefd); + printf("# invoking probes\n"); + for (unsigned i = 0; i < 16; i++) + CHECK(target(i) == i + 7); + printf("# invocations complete\n"); + CHECK(xol_geometry(native)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + int ok = *ctr == 1; + for (unsigned i = 0; i < 8; i++) + ok &= target(100 + i) == 107 + i; + _exit(ok ? 0 : 3); + } + int status; + CHECK(waitpid(child, &status, 0) == child); + CHECK(WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(*ctr == 1); + unsigned entries = records(entry), returns = records(retname); + printf("# ABI=%zu entry=%u return=%u\n", ps, entries, returns); + CHECK(entries == 24 && returns == 24); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 1); + CHECK(!enable(retname, 0)); + CHECK(*ctr == 0); + /* Identical restored text is dropped so it can be shared with the file again. */ + CHECK(!(pagemap((void *)target, ps) >> 63)); + CHECK(target(10) == 17); + CHECK(!memcmp(code, blob + map_off, map_len)); + CHECK(!memcmp(other, blob, ps)); + CHECK(!memcmp(counter, blob + 5 * ps, ps)); + /* A pipe GET must prevent dropping the mapped code slot on removal. */ + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(*ctr == 1); + int pipefd[2]; + CHECK(!pipe(pipefd)); + struct iovec iov = { .iov_base = + (void *)((uintptr_t)target & ~(ps - 1)), + .iov_len = ps }; + CHECK(vmsplice(pipefd[1], &iov, 1, 0) == (ssize_t)ps); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 0); + CHECK(pagemap((void *)target, ps) >> 63); + unsigned char *held = malloc(ps); + CHECK(held); + CHECK(read(pipefd[0], held, ps) == (ssize_t)ps); + CHECK(!memcmp(held, blob + (code_off & ~(ps - 1)), ps)); + free(held); + close(pipefd[0]); + close(pipefd[1]); + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 0); + /* Native folio-wide references may defer this optional optimization. */ + if (ps < native) + CHECK(!(pagemap((void *)target, ps) >> 63)); + CHECK(target(20) == 27); + snprintf(cmd, sizeof(cmd), "-:aurora_up/%s\n", entry); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + snprintf(cmd, sizeof(cmd), "-:aurora_up/%s\n", retname); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + CHECK(!munmap(code, map_len)); + CHECK(!munmap(guard, ps)); + CHECK(!munmap(counter, ps)); + CHECK(!munmap(other, ps)); + close(fd); + free(blob); + printf("ok - %zuK uprobe/uretprobe, byte offsets, counter, fork, restore and file sharing\n", + ps / 1024); + return 0; +} +static int wait_bounded(pid_t child, int *status) +{ + for (unsigned i = 0; i < 250; i++) { + pid_t got = waitpid(child, status, WNOHANG); + if (got == child) + return 1; + if (got < 0) + return 0; + if (i == 50 || i == 200) { + char path[64], buf[4096]; + snprintf(path, sizeof(path), "/proc/%d/stack", child); + int fd = open(path, O_RDONLY); + ssize_t n = fd < 0 ? -1 : + read(fd, buf, sizeof(buf) - 1); + if (fd >= 0) + close(fd); + if (n > 0) { + buf[n] = 0; + printf("# child %d stack:\n%s", child, buf); + } else + printf("# child %d stack unavailable errno=%d\n", + child, errno); + } + if (i == 200) { + printf("# killing stalled child %d\n", child); + kill(child, SIGKILL); + } + usleep(100000); + } + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), + strtoul(argv[4], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("tracefs", TRACE, "tracefs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned run = 0; run < (sizes[0] > 16384 ? 6U : 4U); run++) { + unsigned i = run / 2, shifted = run & 1; + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24], native[24], mode[8]; + snprintf(mode, sizeof(mode), "%u", shifted); + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + snprintf(native, sizeof(native), "%zu", sizes[0]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, native, + mode, NULL); + _exit(5); + } + int ok = c > 0 && wait_bounded(c, &status) && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - ABI %zuK uprobes shifted=%u status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, shifted, status); + fail += !ok; + } +done: + printf("UPROBES %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uring-fixture-init.c b/tools/testing/selftests/arm64/user4k/uring-fixture-init.c new file mode 100644 index 0000000000000..cafc11138d2e8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uring-fixture-init.c @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +struct ring { + int fd; + struct io_uring_params p; + void *sq, *cq, *sqes; + size_t sqlen, cqlen, sqelen; +}; +static unsigned load(unsigned *p) +{ + return __atomic_load_n(p, __ATOMIC_ACQUIRE); +} +static void store(unsigned *p, unsigned v) +{ + __atomic_store_n(p, v, __ATOMIC_RELEASE); +} +static size_t align_size(size_t n) +{ + size_t ps = getauxval(AT_PAGESZ); + return (n + ps - 1) & ~(ps - 1); +} +static void destroy(struct ring *r) +{ + if (r->sq && r->sq != MAP_FAILED) + munmap(r->sq, r->sqlen); + if (r->cq && r->cq != MAP_FAILED && r->cq != r->sq) + munmap(r->cq, r->cqlen); + if (r->sqes && r->sqes != MAP_FAILED) + munmap(r->sqes, r->sqelen); + if (r->fd >= 0) + close(r->fd); + *r = (struct ring){ .fd = -1 }; +} +static bool setup(struct ring *r, unsigned entries, unsigned flags) +{ + *r = (struct ring){ .fd = -1 }; + r->p.flags = flags; + r->fd = syscall(__NR_io_uring_setup, entries, &r->p); + if (r->fd < 0) + return false; + r->sqlen = align_size(r->p.sq_off.array + + r->p.sq_entries * sizeof(unsigned)); + r->cqlen = align_size(r->p.cq_off.cqes + + r->p.cq_entries * sizeof(struct io_uring_cqe) * + (flags & IORING_SETUP_CQE32 ? 2 : 1)); + r->sqelen = align_size(r->p.sq_entries * sizeof(struct io_uring_sqe) * + (flags & IORING_SETUP_SQE128 ? 2 : 1)); + if (r->p.features & IORING_FEAT_SINGLE_MMAP) { + if (r->cqlen > r->sqlen) + r->sqlen = r->cqlen; + r->cqlen = r->sqlen; + } + r->sq = mmap(NULL, r->sqlen, PROT_READ | PROT_WRITE, MAP_SHARED, r->fd, + IORING_OFF_SQ_RING); + r->cq = (r->p.features & IORING_FEAT_SINGLE_MMAP) ? + r->sq : + mmap(NULL, r->cqlen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_CQ_RING); + r->sqes = mmap(NULL, r->sqelen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_SQES); + return r->sq != MAP_FAILED && r->cq != MAP_FAILED && + r->sqes != MAP_FAILED; +} +static bool request(struct ring *r, unsigned op, int fd, void *buf, + unsigned len, uint64_t off, bool async) +{ + unsigned *sqtail = r->sq + r->p.sq_off.tail, + *sqmask = r->sq + r->p.sq_off.ring_mask; + unsigned *cqhead = r->cq + r->p.cq_off.head, + *cqtail = r->cq + r->p.cq_off.tail, + *cqmask = r->cq + r->p.cq_off.ring_mask; + unsigned tail = load(sqtail), head = load(cqhead), + idx = tail & load(sqmask); + unsigned sqstride = sizeof(struct io_uring_sqe) * + (r->p.flags & IORING_SETUP_SQE128 ? 2 : 1); + unsigned cqstride = sizeof(struct io_uring_cqe) * + (r->p.flags & IORING_SETUP_CQE32 ? 2 : 1); + struct io_uring_sqe *sqe = r->sqes + idx * sqstride; + memset(sqe, 0, sqstride); + sqe->opcode = op; + sqe->fd = fd; + sqe->addr = (uintptr_t)buf; + sqe->len = len; + sqe->off = off; + sqe->user_data = (uint64_t)tail + 1234; + if (async) + sqe->flags = IOSQE_ASYNC; + ((unsigned *)(r->sq + r->p.sq_off.array))[idx] = idx; + store(sqtail, tail + 1); + int ret = syscall(__NR_io_uring_enter, r->fd, 1, 1, + IORING_ENTER_GETEVENTS, NULL, 0); + if (ret != 1 || load(cqtail) != head + 1) { + ksft_print_msg("enter=%d errno=%d\n", ret, errno); + return false; + } + struct io_uring_cqe *cqe = + r->cq + r->p.cq_off.cqes + (head & load(cqmask)) * cqstride; + unsigned expected = op == IORING_OP_NOP ? 0 : len; + if (op == IORING_OP_READV_FIXED || op == IORING_OP_WRITEV_FIXED) { + expected = 0; + for (unsigned i = 0; i < len; i++) + expected += ((struct iovec *)buf)[i].iov_len; + } + bool ok = cqe->user_data == (uint64_t)tail + 1234 && + cqe->res == (int)expected; + if (!ok) + ksft_print_msg("cqe res=%d expected=%u\n", cqe->res, expected); + store(cqhead, head + 1); + return ok; +} +static unsigned long pinned_kb(void) +{ + FILE *f = fopen("/proc/self/status", "r"); + char line[160]; + unsigned long n = ~0UL; + if (!f) + return n; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmPin: %lu kB", &n) == 1) + break; + fclose(f); + return n; +} +static bool bytes_are(unsigned char *p, size_t len, unsigned char value) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value) + return false; + return true; +} +static void fixed_checks(int fd, unsigned char *p, size_t len, size_t ps) +{ + struct ring r = { .fd = -1 }; + unsigned char *q = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + struct iovec reg = { p + 512, len - 1024 }; + bool live = false, ok = setup(&r, 8, 0); + unsigned long baseline = pinned_kb(); + memset(p, 0xa6, len); + if (ok) + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + ksft_test_result(ok, "register unaligned long-term buffer\n"); + if (ok) { + ok = request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, 7 * 512, 0, + false); + memset(p + 512, 0, 7 * 512); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + 512, 7 * 512, 0, + false) && + bytes_are(p + 512, 7 * 512, 0xa6); + } + ksft_test_result(ok, "fixed first-quarter slice offsets\n"); + if (ok) { + size_t off = len - 3 * ps; + memset(p + off, 0x49, 2 * ps); + ok = request(&r, IORING_OP_WRITE_FIXED, fd, p + off, 2 * ps, + 65536, true); + memset(p + off, 0, 2 * ps); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + off, 2 * ps, + 65536, true) && + bytes_are(p + off, 2 * ps, 0x49); + } + ksft_test_result(ok, "fixed late-buffer fast indexing and async I/O\n"); + if (ok) { + struct iovec v[3] = { { p + 512, 512 }, + { p + 2 * ps + 512, ps - 512 }, + { p + 17 * ps + 512, 1024 } }; + memset(p, 0x78, len); + ok = request(&r, IORING_OP_WRITEV_FIXED, fd, v, 3, 131072, + true); + memset(p, 0x11, len); + ok &= request(&r, IORING_OP_READV_FIXED, fd, v, 3, 131072, + true); + for (unsigned i = 0; i < 3; i++) + ok &= bytes_are(v[i].iov_base, v[i].iov_len, 0x78); + ok &= p[0] == 0x11 && p[ps + 512] == 0x11 && p[17 * ps] == 0x11; + } + ksft_test_result(ok, "fixed vectored physical offsets and guards\n"); + if (ok) { + memset(p + 512, 0x3e, len - 1024); + ok = request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, len - 1024, + 4 * 1024 * 1024, true); + memset(p + 512, 0, len - 1024); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + 512, len - 1024, + 4 * 1024 * 1024, true) && + bytes_are(p + 512, len - 1024, 0x3e); + } + ksft_test_result(ok, "fixed multi-bio transfer\n"); + if (ok) { + ok = mmap(p, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == p; + memset(p, 0x55, len); + ok &= request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, 3 * ps, + 8 * 1024 * 1024, true); + if (q != MAP_FAILED) { + memset(q, 0, 4 * ps); + ok &= request(&r, IORING_OP_READ, fd, q, 3 * ps, + 8 * 1024 * 1024, false) && + bytes_are(q, 3 * ps, 0x3e) && + bytes_are(p, len, 0x55); + } else + ok = false; + } + ksft_test_result(ok, + "registered source survives unmap and replacement\n"); + if (ok) { + memset(q, 0xc1, 3 * ps); + ok = request(&r, IORING_OP_WRITE, fd, q, 3 * ps, + 10 * 1024 * 1024, false); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + 512, 3 * ps, + 10 * 1024 * 1024, true); + ok &= request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, 3 * ps, + 12 * 1024 * 1024, true); + memset(q, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ, fd, q, 3 * ps, + 12 * 1024 * 1024, false) && + bytes_are(q, 3 * ps, 0xc1) && bytes_are(p, len, 0x55); + } + ksft_test_result( + ok, + "registered destination survives unmap without touching replacement\n"); + if (live) { + ok = syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } else + ok = false; + ok &= pinned_kb() == baseline; + if (ok) { + ok = !mprotect(p + ps, ps, PROT_NONE); + struct iovec bad[2] = { { p, ps }, { p + ps, ps } }; + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, bad, 2) == -1 && + errno == EFAULT; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE) && + pinned_kb() == baseline; + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + } + ksft_test_result( + ok, + "failed multi-buffer registration rolls back pins and accounting\n"); + if (live) { + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } + for (unsigned i = 0; i < 64 && ok; i++) { + reg = (struct iovec){ p + ps + 512, 2 * ps }; + ok = !madvise(p + ps, 3 * ps, MADV_DONTNEED); + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + if (live) { + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } + ok &= pinned_kb() == baseline; + } + ksft_test_result(ok, + "64 register/reuse/unregister cycles balance VmPin\n"); + /* The file offset deliberately differs from the virtual quarter index. */ + int mem = memfd_create("uring-fixed-shared", 0); + ok &= mem >= 0 && !ftruncate(mem, len + 3 * ps) && q != MAP_FAILED; + if (ok) { + ok = mmap(p, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, mem, 3 * ps) == p; + memset(p, 0x6b, len); + reg = (struct iovec){ p + 512, len - 1024 }; + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + struct iovec v[3] = { { p + 512, 512 }, + { p + 3 * ps + 512, ps - 512 }, + { p + 23 * ps + 512, 1024 } }; + if (ok) { + ok = request(&r, IORING_OP_WRITEV_FIXED, fd, v, 3, + 14 * 1024 * 1024, true); + memset(p + 512, 0x18, len - 1024); + ok &= request(&r, IORING_OP_READV_FIXED, fd, v, 3, + 14 * 1024 * 1024, true); + for (unsigned i = 0; i < 3; i++) + ok &= bytes_are(v[i].iov_base, v[i].iov_len, + 0x6b); + ok &= !msync(p, len, MS_SYNC) && + pread(mem, q, ps, 3 * ps) == (ssize_t)ps; + ok &= bytes_are(q, 1024, 0x6b) && q[1024] == 0x18 && + p[len - 1] == 0x6b; + } + } + ksft_test_result( + ok, "registered shared-file quarters use physical offsets\n"); + if (ok) { + struct iovec replacement = { q + 512, 3 * ps }; + struct io_uring_rsrc_update2 update = { + .data = (uintptr_t)&replacement, .nr = 1 + }; + memset(q, 0x9d, 4 * ps); + ok = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS_UPDATE, &update, + sizeof(update)) == 1; + ok &= request(&r, IORING_OP_WRITE_FIXED, fd, q + 512, 3 * ps, + 16 * 1024 * 1024, true); + memset(q + 512, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ_FIXED, fd, q + 512, 3 * ps, + 16 * 1024 * 1024, true) && + bytes_are(q + 512, 3 * ps, 0x9d); + } + if (live) { + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } + ok &= pinned_kb() == baseline; + ksft_test_result( + ok, + "registered-buffer update transfers ownership and balances accounting\n"); + if (mem >= 0) + close(mem); + if (live) + syscall(__NR_io_uring_register, r.fd, IORING_UNREGISTER_BUFFERS, + NULL, 0); + destroy(&r); + if (q != MAP_FAILED) + munmap(q, len); +} + +static int exercise(void) +{ + struct ring r = { .fd = -1 }; + size_t ps = getauxval(AT_PAGESZ), len = 2 * 1024 * 1024; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fd = open("/dev/user4k-test", O_RDWR | O_DIRECT); + bool ok; + ksft_print_header(); + ksft_set_plan(19); + ksft_print_msg("ABI=%zuK\n", ps / 1024); + ok = setup(&r, 4, 0); + ksft_test_result(ok, "minimum rounded ring mappings\n"); + void *bad = ok ? mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_SHARED, + r.fd, 4096) : + MAP_FAILED; + bool rejected = ok && bad == MAP_FAILED && errno == EINVAL; + if (bad != MAP_FAILED) + munmap(bad, ps); + ksft_test_result(rejected, + "reject sub-native offset in ring selector\n"); + if (ok) { + ok = true; + for (unsigned i = 0; i < 64 && ok; i++) + ok = request(&r, IORING_OP_NOP, -1, NULL, 0, 0, false); + } + ksft_test_result(ok, "small submission/completion ring wrap\n"); + if (p == MAP_FAILED || fd < 0) + ksft_exit_fail_msg("buffer/device setup errno=%d\n", errno); + if (ok) { + memset(p + ps, 0x69, 3 * ps); + ok = request(&r, IORING_OP_WRITE, fd, p + ps, 3 * ps, 0, false); + memset(p + ps, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ, fd, p + ps, 3 * ps, 0, false); + for (size_t i = ps; i < 4 * ps; i++) + ok &= p[i] == 0x69; + } + ksft_test_result(ok, + "direct read/write from unregistered user buffers\n"); + if (ok) { + memset(p + ps, 0x37, 3 * ps); + ok = request(&r, IORING_OP_WRITE, fd, p + ps, 3 * ps, 65536, + true); + memset(p + ps, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ, fd, p + ps, 3 * ps, 65536, + true); + for (size_t i = ps; i < 4 * ps; i++) + ok &= p[i] == 0x37; + } + ksft_test_result(ok, "io-worker forced asynchronous read/write\n"); + if (ok) { + memset(p, 0xb6, len); + ok = request(&r, IORING_OP_WRITE, fd, p, len, 4 * 1024 * 1024, + true); + memset(p, 0, len); + ok &= request(&r, IORING_OP_READ, fd, p, len, 4 * 1024 * 1024, + true); + for (size_t i = 0; i < len; i++) + ok &= p[i] == 0xb6; + } + ksft_test_result(ok, "multi-bio io-worker transfer\n"); + destroy(&r); + ok = setup(&r, 4096, 0); + if (ok) + for (unsigned i = 0; i < 8193 && ok; i++) + ok = request(&r, IORING_OP_NOP, -1, NULL, 0, 0, false); + ksft_test_result(ok, "large ring crosses native pages and wraps\n"); + destroy(&r); + ok = setup(&r, 64, IORING_SETUP_SQE128 | IORING_SETUP_CQE32); + if (ok) + for (unsigned i = 0; i < 129 && ok; i++) + ok = request(&r, IORING_OP_NOP, -1, NULL, 0, 0, false); + ksft_test_result(ok, "128-byte SQEs and 32-byte CQEs\n"); + destroy(&r); + fixed_checks(fd, p, len, ps); + close(fd); + munmap(p, len); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +int main(int argc, char **argv) +{ + bool ok = true; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && !strcmp(argv[1], "--test") && getppid() == 1) + return exercise(); + if (getpid() != 1 || argc != 1) + return 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + FILE *dev = fopen("/sys/class/block/vda/dev", "r"); + unsigned maj, min; + if (!dev || fscanf(dev, "%u:%u", &maj, &min) != 2) + return 4; + fclose(dev); + if (mknod("/dev/user4k-test", S_IFBLK | 0600, makedev(maj, min))) + return 5; + for (unsigned small = 0; small < 2; small++) { + pid_t pid = fork(); + int status = 0; + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + small ? 4096UL : 0UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + printf("%s - io_uring fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/user-events-init.c b/tools/testing/selftests/arm64/user4k/user-events-init.c new file mode 100644 index 0000000000000..4b35954e2a210 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/user-events-init.c @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define TRACE "/sys/kernel/tracing/" +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) + +static int toggle(const char *name, int enabled) +{ + char path[192]; + snprintf(path, sizeof(path), TRACE "events/user_events/%s/enable", + name); + int fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + int ret = write(fd, enabled ? "1" : "0", 1) == 1 ? 0 : -1; + close(fd); + return ret; +} +static uint64_t value(void *p, unsigned bytes) +{ + return bytes == 4 ? __atomic_load_n((uint32_t *)p, __ATOMIC_ACQUIRE) : + __atomic_load_n((uint64_t *)p, __ATOMIC_ACQUIRE); +} +static int wait_value(void *p, unsigned bytes, uint64_t expected) +{ + for (unsigned i = 0; i < 3000; i++) { + if (value(p, bytes) == expected) + return 1; + usleep(1000); + } + return 0; +} +static int check_bytes(unsigned char *p, size_t size, unsigned char fill, + void *word, unsigned bytes) +{ + for (size_t i = 0; i < size; i++) + if ((!word || (uintptr_t)(p + i) < (uintptr_t)word || + (uintptr_t)(p + i) >= (uintptr_t)word + bytes) && + p[i] != fill) { + printf("byte mismatch at %zu got=%#x expected=%#x\n", i, + p[i], fill); + return 0; + } + return 1; +} +static int event_recorded(const char *name) +{ + char buf[16384]; + size_t n = 0; + int fd = open(TRACE "trace", O_RDONLY); + if (fd < 0) + return 0; + while (n < sizeof(buf) - 1) { + ssize_t got = read(fd, buf + n, sizeof(buf) - 1 - n); + if (got < 0) { + close(fd); + return 0; + } + if (!got) + break; + n += got; + } + close(fd); + buf[n] = 0; + char *record = strstr(buf, name), + *payload = record ? strstr(record, "payload=") : NULL; + int ok = payload && strtoul(payload + 8, NULL, 0) == 0x12345678; + if (!ok) + printf("trace mismatch:\n%s\n", buf); + return ok; +} +static int exercise_case(size_t ps, unsigned bytes, unsigned shared) +{ + const size_t len = 4 * ps; + unsigned char *src, *dst; + int memfd = -1; + if (shared) { + memfd = memfd_create("user-event-bitmap", 0); + CHECK(memfd >= 0); + CHECK(!ftruncate(memfd, len)); + } + src = mmap((void *)0x20000000, len, PROT_READ | PROT_WRITE, + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS) | + MAP_FIXED_NOREPLACE, + memfd, 0); + dst = mmap((void *)0x30000000, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, len); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps; + void *word = leaf + ps - bytes; + uint64_t mask = bytes == 4 ? 1ULL << 31 : 1ULL << 63; + memset(word, 0, bytes); + char name[80], spec[120]; + snprintf(name, sizeof(name), "aurora_ue_%zu_%u_%u", ps, bytes, shared); + snprintf(spec, sizeof(spec), "%s u32 payload", name); + int tracefd = open(TRACE "trace", O_WRONLY | O_TRUNC); + CHECK(tracefd >= 0); + close(tracefd); + int fd = open(TRACE "user_events_data", O_RDWR); + CHECK(fd >= 0); + struct user_reg reg = { .size = sizeof(reg), + .enable_bit = bytes * 8 - 1, + .enable_size = bytes, + .enable_addr = (uintptr_t)word, + .name_args = (uintptr_t)spec }; + CHECK(!ioctl(fd, DIAG_IOCSREG, ®)); + CHECK(wait_value(word, bytes, 0)); + CHECK(!toggle(name, 1)); + CHECK(wait_value(word, bytes, mask)); + struct { + uint32_t index, payload; + } event = { reg.write_index, 0x12345678 }; + CHECK(write(fd, &event, sizeof(event)) == sizeof(event)); + CHECK(event_recorded(name)); + CHECK(!toggle(name, 0)); + CHECK(wait_value(word, bytes, 0)); + /* Every other byte, including the adjacent half of the kernel word, is stable. */ + CHECK(check_bytes(leaf, ps, 0x64, word, bytes)); + CHECK(check_bytes(dst, ps, 0x91, NULL, 0)); + CHECK(check_bytes(dst + 2 * ps, 2 * ps, 0x91, NULL, 0)); + int commands[2], reply[2]; + CHECK(!pipe(commands) && !pipe(reply)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + close(commands[1]); + close(reply[0]); + char token; + int ok = 1; + for (unsigned round = 0; round < 32; round++) { + if (read(commands[0], &token, 1) != 1) { + ok = 0; + break; + } + ok = wait_value(word, bytes, token == '1' ? mask : 0) && + check_bytes(leaf, ps, 0x64, word, bytes); + token = ok ? 'y' : 'n'; + if (write(reply[1], &token, 1) != 1 || !ok) + break; + } + _exit(ok ? 0 : 3); + } + close(commands[0]); + close(reply[1]); + int ok = 1; + for (unsigned round = 0; round < 32; round++) { + int on = !(round & 1); + char token = on ? '1' : '0'; + if (toggle(name, on) || + !wait_value(word, bytes, on ? mask : 0) || + write(commands[1], &token, 1) != 1 || + read(reply[0], &token, 1) != 1 || token != 'y') { + ok = 0; + break; + } + } + close(commands[1]); + close(reply[0]); + int status; + CHECK(waitpid(child, &status, 0) == child); + CHECK(ok && WIFEXITED(status) && WEXITSTATUS(status) == 0); + CHECK(check_bytes(leaf, ps, 0x64, word, bytes)); + for (unsigned i = 0; i < 3; i++) + CHECK(check_bytes(src + i * ps, ps, 0x61 + i, NULL, 0)); + /* Force the asynchronous missing-PTE fixup without unregistering. */ + CHECK(!madvise(leaf, ps, MADV_DONTNEED)); + CHECK(!toggle(name, 1)); + CHECK(wait_value(word, bytes, mask)); + CHECK(check_bytes(leaf, ps, shared ? 0x64 : 0, word, bytes)); + CHECK(!toggle(name, 0)); + CHECK(wait_value(word, bytes, 0)); + struct user_unreg unreg = { .size = sizeof(unreg), + .disable_bit = reg.enable_bit, + .disable_addr = (uintptr_t)word }; + CHECK(!ioctl(fd, DIAG_IOCSUNREG, &unreg)); + close(fd); + CHECK(!munmap(src, 3 * ps)); + CHECK(!munmap(dst, len)); + if (memfd >= 0) + close(memfd); + printf("ok - %zuK %u-bit %s relocated bitmap, adjacent bytes, event write and fork toggles\n", + ps / 1024, bytes * 8, shared ? "shared" : "private"); + return 0; +} +static int exercise(size_t expected) +{ + size_t ps = getauxval(AT_PAGESZ); + CHECK(ps == expected); + for (unsigned shared = 0; shared < 2; shared++) + for (unsigned bytes = 4; bytes <= 8; bytes += 4) + CHECK(!exercise_case(ps, bytes, shared)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("tracefs", TRACE, "tracefs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24]; + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = c > 0 && waitpid(c, &status, 0) == c && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - ABI %zuK user events status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("USER EVENTS %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c b/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c new file mode 100644 index 0000000000000..b0a2f2bf385f0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c @@ -0,0 +1,229 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define GET_EXEC_PAGE_SIZE 0x41555002 +static int failed; +static _Thread_local int tls = 7; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static long pss(void *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end; + int active = 0; + long value = -1; + if (!f) + return -1; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + active = (unsigned long)p >= start && + (unsigned long)p < end; + else if (active && sscanf(line, "Pss: %ld kB", &value) == 1) + break; + } + free(line); + fclose(f); + return value; +} +static void *thread_worker(void *arg) +{ + unsigned char *p = arg; + if (tls != 7) + return (void *)1; + tls = 91; + for (unsigned i = 0; i < 16384; i++) + if (p[i] != 0x42) + return (void *)2; + return NULL; +} +static void memory_checks(void) +{ + const size_t ps = 16384; + int status = 0; + pid_t c; + unsigned char *p = mmap(NULL, 6 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + result(p != MAP_FAILED, "16K anonymous mmap"); + if (p == MAP_FAILED) + return; + for (unsigned i = 0; i < 6; i++) + memset(p + i * ps, 0x40 + i, ps); + c = fork(); + if (!c) { + memset(p + ps, 0xb7, ps); + _exit(p[0] == 0x40 && p[2 * ps] == 0x42 ? 0 : 1); + } + int ok = c > 0 && waitpid(c, &status, 0) == c && WIFEXITED(status) && + !WEXITSTATUS(status); + for (unsigned i = 0; i < 6 * ps; i++) + ok &= p[i] == 0x40 + i / ps; + result(ok, "16K fork COW preserves every parent byte"); + result(!mprotect(p + ps, ps, PROT_READ), "16K leaf write protection"); + c = fork(); + if (!c) { + *(volatile unsigned char *)(p + ps + 8192) = 3; + _exit(1); + } + result(c > 0 && waitpid(c, &status, 0) == c && WIFSIGNALED(status) && + WTERMSIG(status) == SIGSEGV, + "write protection covers the interior of a16K leaf"); + result(!mprotect(p + ps, ps, PROT_READ | PROT_WRITE), + "16K write permission restored"); + errno = 0; + result(mprotect(p + 4096, ps, PROT_READ) == -1 && errno == EINVAL, + "16K ABI rejects4K-only protection alignment"); + result(!madvise(p + ps, ps, MADV_DONTNEED), + "16K partial unmap via DONTNEED"); + ok = 1; + for (unsigned i = 0; i < ps; i++) + ok &= p[i] == 0x40 && p[ps + i] == 0 && p[2 * ps + i] == 0x42; + result(ok, "16K discard clears one leaf and preserves neighbours"); + pthread_t thread; + void *answer = (void *)3; + int err = pthread_create(&thread, NULL, thread_worker, p + 2 * ps); + if (!err) + err = pthread_join(thread, &answer); + result(!err && !answer && tls == 7, "16K pthread stack and TLS"); + struct timespec ts; + result(!clock_gettime(CLOCK_MONOTONIC, &ts), "16K vDSO clock access"); + munmap(p, 6 * ps); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (argc == 6 && !strcmp(argv[1], "--child")) { + int fd = atoi(argv[2]), ready = atoi(argv[3]), + release = atoi(argv[4]); + size_t native = strtoul(argv[5], NULL, 10); + const size_t ps = 16384; + result(getauxval(AT_PAGESZ) == ps && getpagesize() == ps, + "EL0 selects16K pages"); + result(prctl(GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) == 0, + "exec consumes alternative selection"); + if (!failed) + memory_checks(); + { + int status = 0; + pid_t c = fork(); + if (!c) { + if (prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, + 0UL)) + _exit(4); + execl("/page-contract", "/page-contract", NULL); + _exit(5); + } + result(c > 0 && waitpid(c, &status, 0) == c && + WIFEXITED(status) && + !WEXITSTATUS(status), + "16K-to16K exec and page/x18 contract"); + } + unsigned char *shared = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, native - ps); + unsigned char *private = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE, fd, native - ps); + int ok = shared != MAP_FAILED && private != MAP_FAILED; + if (ok) { + for (unsigned i = 0; i < ps; i++) + ok &= shared[i] == 0x7b && private[i] == 0x7b; + memset(private, 0xd8, ps); + for (unsigned i = 0; i < ps; i++) + ok &= shared[i] == 0x7b && private[i] == 0xd8; + } + result(ok, "16K file COW copies the whole leaf"); + if (private != MAP_FAILED) + munmap(private, ps); + result(shared != MAP_FAILED && pss(shared) == 8, + "16K file PSS shares all four4K slices"); + if (shared != MAP_FAILED) + shared[ps - 1] = 0xc9; + char token = failed ? 'x' : 'a'; + if (write(ready, &token, 1) != 1 || + read(release, &token, 1) != 1) + return 3; + if (shared != MAP_FAILED) + munmap(shared, ps); + return failed ? 1 : 0; + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + mount("proc", "/proc", "proc", 0, NULL); + size_t native = getauxval(AT_PAGESZ); + int fd = memfd_create("mixed16-smoke", 0); + if (native != 65536 || fd < 0 || ftruncate(fd, native)) { + result(0, "64K parent setup"); + goto done; + } + unsigned char *p = + mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + result(0, "parent shared mmap"); + goto done; + } + memset(p, 0x7b, native); + result(pss(p) == 64, "native64K initial PSS"); + int ready[2], release[2]; + if (pipe(ready) || pipe(release)) { + result(0, "pipes"); + goto done; + } + pid_t c = fork(); + int status = 0; + if (!c) { + char f[24], r[24], w[24], n[24]; + snprintf(f, sizeof(f), "%d", fd); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(w, sizeof(w), "%d", release[0]); + snprintf(n, sizeof(n), "%zu", native); + close(ready[0]); + close(release[1]); + if (prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--child", f, r, w, n, NULL); + _exit(5); + } + close(ready[1]); + close(release[0]); + char token = 0; + int ok = c > 0 && read(ready[0], &token, 1) == 1 && token == 'a'; + result(ok && p[native - 1] == 0xc9, + "native64K and16K shared bytes remain coherent"); + result(pss(p) == 56, + "native64K PSS accounts one16K alias as four shared slices"); + token = 'b'; + if (write(release[1], &token, 1) != 1) + failed++; + result(c > 0 && waitpid(c, &status, 0) == c && WIFEXITED(status) && + !WEXITSTATUS(status), + "16K child completes"); + printf("# child status=%#x\n", status); + result(pss(p) == 64, "native PSS returns after16K exit"); + munmap(p, native); +done: + printf("USER16K SMOKE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c b/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c new file mode 100644 index 0000000000000..c43e28ca8b21d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c @@ -0,0 +1,542 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#define __user +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/vfio.h" +#include "../../../../../drivers/vfio/vfio_iommu_type1_test.h" + +#define IOVA (256UL * 1024 * 1024) +static unsigned int number, failures; + +static void result(int ok, const char *name) +{ + printf("%s %u - %s (errno=%d)\n", ok ? "ok" : "not ok", ++number, name, + errno); + failures += !ok; +} + +static int state(int fd, struct vfio_type1_test_cmd *cmd) +{ + memset(cmd, 0, sizeof(*cmd)); + return ioctl(fd, VFIO_TYPE1_TEST_STATE, cmd); +} + +static int domain(int fd, unsigned int which, int add, size_t page, size_t fail) +{ + struct vfio_type1_test_cmd cmd = { + .domain = which, + .size = page, + .fail_after = fail, + }; + + return ioctl(fd, + add ? VFIO_TYPE1_TEST_DOMAIN_ADD : + VFIO_TYPE1_TEST_DOMAIN_DEL, + &cmd); +} + +static int rw(int fd, size_t offset, void *data, size_t size, + unsigned int flags) +{ + struct vfio_type1_test_cmd cmd = { + .iova = IOVA + offset, + .size = size, + .buffer = (uintptr_t)data, + .flags = flags, + }; + + return ioctl(fd, VFIO_TYPE1_TEST_RW, &cmd); +} + +static int map(int fd, void *data, size_t size) +{ + struct vfio_iommu_type1_dma_map cmd = { + .argsz = sizeof(cmd), + .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, + .iova = IOVA, + .vaddr = (uintptr_t)data, + .size = size, + }; + + return ioctl(fd, VFIO_IOMMU_MAP_DMA, &cmd); +} + +static int unmap(int fd, size_t size) +{ + struct vfio_iommu_type1_dma_unmap cmd = { + .argsz = sizeof(cmd), + .iova = IOVA, + .size = size, + }; + int ret = ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &cmd); + + return ret ? ret : cmd.size == size ? 0 : -2; +} + +static int dirty(int fd, unsigned int flags) +{ + struct vfio_iommu_type1_dirty_bitmap cmd = { .argsz = sizeof(cmd), + .flags = flags }; + + return ioctl(fd, VFIO_IOMMU_DIRTY_PAGES, &cmd); +} + +static int bitmap(int fd, size_t size, uint64_t *bits) +{ + unsigned char + buffer[sizeof(struct vfio_iommu_type1_dirty_bitmap) + + sizeof(struct vfio_iommu_type1_dirty_bitmap_get)] = {}; + struct vfio_iommu_type1_dirty_bitmap *cmd = (void *)buffer; + struct vfio_iommu_type1_dirty_bitmap_get *range = (void *)cmd->data; + + cmd->argsz = sizeof(buffer); + cmd->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP; + range->iova = IOVA; + range->size = size; + range->bitmap.pgsize = 4096; + range->bitmap.size = sizeof(*bits); + range->bitmap.data = (void *)bits; + *bits = 0; + return ioctl(fd, VFIO_IOMMU_DIRTY_PAGES, buffer); +} + +static int all_bytes(const unsigned char *data, size_t size, + unsigned char value) +{ + for (size_t i = 0; i < size; i++) + if (data[i] != value) + return 0; + return 1; +} + +static void external_span(int backing, size_t native) +{ + const size_t fine = 4096, count = native / fine; + uint64_t bits; + + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_dma_unmap remove = { + .argsz = sizeof(remove), + .iova = IOVA, + .size = fine, + }; + int fd = open("/dev/vfio-test", O_RDWR), status = 0; + pid_t child; + + result(fd >= 0 && (!mapped || !domain(fd, 0, 1, fine, 0)), + "span fixture opens lazy or mapped type1 container"); + if (fd < 0) + continue; + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), + "span dirty tracking starts before fine mappings"); + child = fork(); + if (!child) { + unsigned char *data = mmap(NULL, native, + PROT_READ | PROT_WRITE, + MAP_SHARED, backing, 0); + + if (data == MAP_FAILED) + _exit(1); + for (size_t i = 0; i < count; i++) { + struct vfio_iommu_type1_dma_map area = { + .argsz = sizeof(area), + .flags = VFIO_DMA_MAP_FLAG_READ | + VFIO_DMA_MAP_FLAG_WRITE, + .iova = IOVA + i * fine, + .vaddr = (uintptr_t)data + i * fine, + .size = fine, + }; + + if (ioctl(fd, VFIO_IOMMU_MAP_DMA, &area)) + _exit(2); + } + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 17 }; + _exit(ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) ? 3 : 0); + } + result(child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status), + "source pins native page across independent fine MAP ioctls"); + result(!state(fd, &cmd) && !cmd.owner_users && + cmd.external_refs == 1 && + cmd.locked_bytes == count * native && + cmd.owner_locked_bytes == count * native, + "every fine DMA area retains its own charge after source exit"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 31 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && + !state(fd, &cmd) && cmd.external_refs == 2 && + cmd.locked_bytes == count * native, + "cross-area duplicate pin reuses all cached constituent pins"); + result(!bitmap(fd, native, &bits) && + bits == ((1ULL << count) - 1), + "cross-area dirty bitmap marks only each mapping's fine extent"); + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &remove) && + remove.size == fine && !state(fd, &cmd) && + !cmd.external_refs && cmd.notifications == 1 && + cmd.locked_bytes == + (mapped ? (count - 1) * native : 0), + "unmapping one constituent drains references in all page constituents"); + remove.iova = IOVA + fine; + remove.size = native - fine; + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &remove) && + remove.size == native - fine && + !state(fd, &cmd) && !cmd.locked_bytes && + !cmd.mapped_bytes[0], + "remaining fine mappings unmap without pins or translations"); + close(fd); + } +} + +static void native_owner(int fd, int backing, unsigned char *data, + size_t native) +{ + struct vfio_type1_test_cmd cmd = {}; + struct vfio_iommu_type1_dma_unmap invalidate = { + .argsz = sizeof(invalidate), + .flags = VFIO_DMA_UNMAP_FLAG_VADDR, + .iova = IOVA, + .size = 2 * native, + }; + struct vfio_iommu_type1_dma_map update = { + .argsz = sizeof(update), + .flags = VFIO_DMA_MAP_FLAG_VADDR, + .iova = IOVA, + .size = 2 * native, + .vaddr = (uintptr_t)data, + }; + struct __user_cap_header_struct header = { + .version = _LINUX_CAPABILITY_VERSION_3 + }; + struct __user_cap_data_struct saved[2], reduced[2]; + struct rlimit limit, empty; + unsigned char bytes[64]; + int file_flags = fcntl(backing, F_GETFD), status = 0; + pid_t child; + int budget_ready, budget_captured, transferred; + + result(!ioctl(fd, VFIO_TYPE1_TEST_EMULATED, &cmd) && + !domain(fd, 0, 1, 4096, 0), + "VADDR fixture retains production group restrictions"); + result(file_flags >= 0 && + !fcntl(backing, F_SETFD, file_flags & ~FD_CLOEXEC), + "native source inherits exact backing descriptor"); + child = fork(); + if (!child) { + char fd_arg[24], file_arg[24], native_arg[24]; + + snprintf(fd_arg, sizeof(fd_arg), "%d", fd); + snprintf(file_arg, sizeof(file_arg), "%d", backing); + snprintf(native_arg, sizeof(native_arg), "%zu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL)) + _exit(98); + execl("/init", "/init", "--native-source", fd_arg, file_arg, + native_arg, NULL); + _exit(99); + } + if (file_flags >= 0) + fcntl(backing, F_SETFD, file_flags); + result(child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && + !state(fd, &cmd) && !cmd.owner_users && + !(cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && + cmd.mapped_bytes[0] == 2 * native, + "exited native source owns legacy GUP pins before transfer"); + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &invalidate), + "canonical VADDR invalidation leaves device translations alive"); + budget_captured = !getrlimit(RLIMIT_MEMLOCK, &limit) && + !syscall(SYS_capget, &header, saved); + budget_ready = budget_captured; + if (budget_ready) { + memcpy(reduced, saved, sizeof(saved)); + reduced[CAP_IPC_LOCK / 32].effective &= + ~(1U << (CAP_IPC_LOCK % 32)); + empty = (struct rlimit){ .rlim_cur = 0, + .rlim_max = limit.rlim_max }; + budget_ready = !setrlimit(RLIMIT_MEMLOCK, &empty) && + !syscall(SYS_capset, &header, reduced); + } + errno = 0; + result(budget_ready && ioctl(fd, VFIO_IOMMU_MAP_DMA, &update) < 0 && + errno == ENOMEM && !state(fd, &cmd) && + !cmd.owner_users && + !(cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && + cmd.mapped_bytes[0] == 2 * native, + "failed owner credit preserves native pins, metadata and charge"); + if (budget_captured) { + result(!syscall(SYS_capset, &header, saved) && + !setrlimit(RLIMIT_MEMLOCK, &limit), + "restore original memlock budget and effective capability"); + } else { + result(0, "owner budget setup failed"); + } + transferred = !ioctl(fd, VFIO_IOMMU_MAP_DMA, &update) && + !state(fd, &cmd) && cmd.owner_users && + (cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && + cmd.owner_locked_bytes == 2 * native; + result(transferred, + "canonical VADDR retry adopts native pins for alternative owner"); + /* Dependent CPU access is prohibited while VADDR remains invalid. */ + if (!transferred) + return; + result(!rw(fd, 0, bytes, sizeof(bytes), 0) && + all_bytes(bytes, sizeof(bytes), 0x64), + "new owner accesses retained native buffer with its own mm authorization"); + cmd = (struct vfio_type1_test_cmd){ .flags = 1 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_EMULATED, &cmd), + "external consumer rejoins after VADDR update"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 17 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == 2 * native && cmd.external_refs == 1, + "external page array reuses adopted pins without new charge"); + result(!unmap(fd, 2 * native) && !state(fd, &cmd) && + !cmd.locked_bytes && !cmd.owner_locked_bytes && + !cmd.external_refs && !cmd.mapped_bytes[0], + "owner transfer final cleanup refunds recipient and clears domain"); + result(!domain(fd, 0, 0, 0, 0), "owner transfer domain detaches"); +} + +static int exercise(size_t native) +{ + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_info info = { .argsz = sizeof(info) }; + struct vfio_iommu_type1_dma_map bad = { .argsz = 8 }; + unsigned char bytes[64], disk[64]; + uint64_t bits = 0; + int fd = open("/dev/vfio-test", O_RDWR), + backing = memfd_create("vfio", MFD_CLOEXEC); + unsigned char *data = MAP_FAILED; + int status = 0; + pid_t child; + size_t length = native + 4096; + + printf("TAP version 13\n# real type1 ioctl ABI=%luK native=%zuK\n", + getauxval(AT_PAGESZ) / 1024, native / 1024); + result(fd >= 0 && backing >= 0 && !ftruncate(backing, 2 * native), + "open privileged recording-domain fixture and backing"); + if (fd < 0 || backing < 0 || failures) + goto out; + data = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, MAP_SHARED, + backing, 0); + result(data != MAP_FAILED, + "map native-sized shmem backing in actual process mm"); + if (data == MAP_FAILED) + goto out; + memset(data, 0x64, 2 * native); + result(!ioctl(fd, VFIO_IOMMU_GET_INFO, &info) && + (info.iova_pgsizes & 4096), + "type1 GET_INFO reports fine IOVA capability"); + errno = 0; + result(ioctl(fd, VFIO_IOMMU_MAP_DMA, &bad) < 0 && errno == EINVAL, + "canonical MAP rejects truncated argument"); + errno = 0; + result(ioctl(fd, VFIO_IOMMU_MAP_DMA, (void *)1) < 0 && errno == EFAULT, + "canonical MAP rejects inaccessible userspace argument"); + result(!map(fd, data, 2 * native), "lazy full-range MAP from real EL0"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0], + "lazy MAP acquires no DMA pins or translations"); + result(!rw(fd, 0, bytes, sizeof(bytes), 0) && + all_bytes(bytes, sizeof(bytes), 0x64), + "CPU-only GET reads actual source bytes"); + result(!state(fd, &cmd) && !cmd.locked_bytes, + "CPU GET does not charge long-term pins"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), + "canonical dirty tracking starts"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 17 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == native && cmd.external_refs == 1, + "external page array preserves unaligned caller byte offset"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 31 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == native && cmd.external_refs == 2, + "duplicate external pin charges physical native page once"); + result(!bitmap(fd, 2 * native, &bits) && + bits == ((1ULL << (native / 4096)) - 1), + "dirty bitmap covers every fine unit in native external page"); + result(!unmap(fd, 2 * native), + "canonical UNMAP drains callback-held external refs"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.external_refs && + cmd.notifications == 1, + "external-unpin callback refunds all pins"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP), + "dirty tracking stops"); + result(!domain(fd, 0, 1, 4096, 0), + "recording4K domain attaches before child MAP"); + child = fork(); + if (!child) { + unsigned char *source = mmap(NULL, 2 * native, + PROT_READ | PROT_WRITE, MAP_SHARED, + backing, 0); + int ret = source == MAP_FAILED ? -1 : + map(fd, source + 4096, length); + + _exit(ret ? 1 : 0); + } + result(child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status), + "fine-offset MAP retains real child pins after exit"); + result(!state(fd, &cmd) && !cmd.owner_users && + cmd.locked_bytes == 2 * native && + cmd.owner_locked_bytes == 2 * native && + cmd.mapped_bytes[0] == length, + "exited source mm retains exact physical-page charge and translation length"); + errno = 0; + result(rw(fd, 0, bytes, sizeof(bytes), 0) < 0 && errno == EFAULT, + "foreign process CPU access preserves source-mm authorization"); + result(!rw(fd, 0, bytes, sizeof(bytes), VFIO_TYPE1_TEST_WORKER) && + all_bytes(bytes, sizeof(bytes), 0x64), + "kernel consumer reads exited source pins"); + errno = 0; + result(domain(fd, 1, 1, 4096, 4096) < 0 && errno == ENOMEM && + !state(fd, &cmd) && !cmd.mapped_bytes[1] && + cmd.mapped_bytes[0] == length && + cmd.locked_bytes == 2 * native, + "failed replay leaves existing domain and pins intact"); + result(!domain(fd, 1, 1, 4096, 0) && !state(fd, &cmd) && + cmd.mapped_bytes[0] == length && + cmd.mapped_bytes[1] == length && + cmd.locked_bytes == 2 * native, + "retry replays cached pins without resurrecting source mm"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), + "fine DMA dirty tracking starts"); + memset(bytes, 0x9a, sizeof(bytes)); + result(!rw(fd, 4096 - 32, bytes, sizeof(bytes), + VFIO_TYPE1_TEST_WRITE | VFIO_TYPE1_TEST_WORKER) && + pread(backing, disk, sizeof(disk), 8192 - 32) == + sizeof(disk) && + all_bytes(disk, sizeof(disk), 0x9a), + "cached worker write dirties original shmem bytes"); + result(!bitmap(fd, length, &bits) && bits == 3, + "canonical bitmap returns exact two4K units touched by worker write"); + result(!unmap(fd, length), + "fine UNMAP synchronizes both recording domains"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0] && + !cmd.mapped_bytes[1], + "source-exit cleanup refunds pins and clears all translations"); + result(!domain(fd, 1, 0, 0, 0) && !domain(fd, 0, 0, 0, 0), + "both domains detach after complete cleanup"); + external_span(backing, native); + if (getauxval(AT_PAGESZ) != native) + native_owner(fd, backing, data, native); +out: + if (data != MAP_FAILED) + munmap(data, 2 * native); + if (backing >= 0) + close(backing); + if (fd >= 0) + close(fd); + printf("1..%u\n# type1 ioctl failures=%u\n", number, failures); + return !!failures; +} + +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--native-source")) { + size_t native = strtoul(argv[4], NULL, 10); + int fd = atoi(argv[2]), backing = atoi(argv[3]); + void *data; + + if (getauxval(AT_PAGESZ) != native) + return 2; + data = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, + MAP_SHARED, backing, 0); + return data == MAP_FAILED || map(fd, data, 2 * native) ? 1 : 0; + } + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + size_t expected = strtoul(argv[3], NULL, 10); + + if (getauxval(AT_PAGESZ) != expected) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) + return 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + FILE *dev = fopen("/sys/class/misc/aurora-vfio-type1-test/dev", "r"); + unsigned int major, minor; + + if (!dev || fscanf(dev, "%u:%u", &major, &minor) != 2) + return 4; + fclose(dev); + if (mknod("/dev/vfio-test", S_IFCHR | 0600, makedev(major, minor))) + return 5; + size_t native = getauxval(AT_PAGESZ), + targets[] = { native, 4096, 16384 }; + int ok = 1; + + int compat_only = getenv("VFIO_COMPAT_ONLY") != NULL; + + for (unsigned int i = 0; i < 3; i++) { + if (compat_only) + break; + size_t target = targets[i]; + pid_t child; + int status = 0; + + if (target > native || (i && target == native)) + continue; + child = fork(); + if (!child) { + char native_arg[24], target_arg[24]; + + snprintf(native_arg, sizeof(native_arg), "%zu", native); + snprintf(target_arg, sizeof(target_arg), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + target == native ? 0UL : target, 0UL, 0UL, + 0UL)) + _exit(3); + execl("/init", "/init", "--test", native_arg, + target_arg, NULL); + _exit(4); + } + ok &= child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("# actual ABI=%zuK child_status=%d\n", target / 1024, + status); + } + const char *expected_arg = getenv("VFIO_COMPAT_PAGES"); + size_t expected = expected_arg ? strtoul(expected_arg, NULL, 10) : 4096; + pid_t compat = fork(); + int compat_status = 0; + + if (!compat) { + char native_arg[24], page_arg[24]; + + snprintf(native_arg, sizeof(native_arg), "%zu", native); + snprintf(page_arg, sizeof(page_arg), "%zu", expected); + execl("/compat-vfio", "/compat-vfio", native_arg, page_arg, + NULL); + _exit(98); + } + ok &= compat > 0 && waitpid(compat, &compat_status, 0) == compat && + WIFEXITED(compat_status) && !WEXITSTATUS(compat_status); + printf("# AArch32 expected=%zuK child_status=%d\n", expected / 1024, + compat_status); + printf("VFIO TYPE1 IOCTL %s\n", ok ? "PASS" : "FAIL"); + reboot(RB_POWER_OFF); + return !ok; +} diff --git a/tools/testing/selftests/arm64/user4k/vhost-log-init.c b/tools/testing/selftests/arm64/user4k/vhost-log-init.c new file mode 100644 index 0000000000000..07d6a8969bca0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/vhost-log-init.c @@ -0,0 +1,392 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#ifndef VHOST_SET_FORK_FROM_OWNER +#define VHOST_SET_FORK_FROM_OWNER _IOW(VHOST_VIRTIO, 0x84, __u8) +#define VHOST_GET_FORK_FROM_OWNER _IOR(VHOST_VIRTIO, 0x85, __u8) +#endif +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +#define N 8 +#define ROUNDS 32 +#define PACKET_SIZE 96 +#define PROTOCOL 0x88b5 +#define DATA_GPA (51ULL * 4096) +#define TX_USED_GPA (71ULL * 4096) +#define RX_USED_GPA (70ULL * 4096) + +struct queue { + struct vring_desc desc[N]; + struct { + uint16_t flags, idx, ring[N + 1]; + } avail; + struct { + uint16_t flags, idx; + struct vring_used_elem ring[N]; + uint16_t event; + } used; + int kick, call; +}; + +static int queue_setup(int fd, struct queue *q, unsigned index) +{ + struct vhost_vring_state state = { .index = index, .num = N }; + struct vhost_vring_addr addr = { + .index = index, + .flags = 1U << VHOST_VRING_F_LOG, + .desc_user_addr = (uintptr_t)q->desc, + .avail_user_addr = (uintptr_t)&q->avail, + .used_user_addr = (uintptr_t)&q->used, + .log_guest_addr = index ? TX_USED_GPA : RX_USED_GPA, + }; + q->kick = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + q->call = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + CHECK(q->kick >= 0 && q->call >= 0); + CHECK(!ioctl(fd, VHOST_SET_VRING_NUM, &state)); + state.num = 0; + CHECK(!ioctl(fd, VHOST_SET_VRING_BASE, &state)); + CHECK(!ioctl(fd, VHOST_SET_VRING_ADDR, &addr)); + struct vhost_vring_file file = { .index = index, .fd = q->kick }; + CHECK(!ioctl(fd, VHOST_SET_VRING_KICK, &file)); + file.fd = q->call; + CHECK(!ioctl(fd, VHOST_SET_VRING_CALL, &file)); + return 0; +} +static int submit(struct queue *q, unsigned round) +{ + uint64_t one = 1; + q->avail.ring[round % N] = 0; + __atomic_store_n(&q->avail.idx, round + 1, __ATOMIC_RELEASE); + return write(q->kick, &one, sizeof(one)) == sizeof(one); +} +static int complete(struct queue *q, unsigned round) +{ + for (unsigned i = 0; i < 2000; i++) { + if (__atomic_load_n(&q->used.idx, __ATOMIC_ACQUIRE) == + round + 1) + return 1; + usleep(1000); + } + printf("queue timeout round=%u used=%u avail=%u\n", round, q->used.idx, + q->avail.idx); + return 0; +} +static int bitmap_ok(unsigned char *leaf, size_t ps, int active, int missing) +{ + for (size_t i = 0; i < ps; i++) { + unsigned char expected = i < ps - 9 ? (missing ? 0 : 0x64) : 0; + if (active && i == ps - 3) + expected = 8; /* RX buffer, GPA page 51 */ + if (active && i == ps - 1) + expected = 0xc0; /* RX/TX rings, pages 70/71 */ + if (__atomic_load_n(leaf + i, __ATOMIC_ACQUIRE) != expected) { + printf("bitmap byte %zu got=%#x expected=%#x\n", i, + leaf[i], expected); + return 0; + } + } + return 1; +} +static int filled(unsigned char *p, size_t len, unsigned char byte) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != byte) + return 0; + return 1; +} +static int exercise_case(size_t ps, unsigned shared, unsigned forked, + uint8_t mode) +{ + int backing = -1; + if (shared) { + backing = memfd_create("vhost-log", 0); + CHECK(backing >= 0 && !ftruncate(backing, 4 * ps)); + } + unsigned char *src = mmap( + (void *)0x20000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), + backing, 0); + unsigned char *dst = + mmap((void *)0x30000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, 4 * ps); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps, *bitmap = leaf + ps - 9; + memset(bitmap, 0, 9); + int barrier[2]; + pid_t child = -1; + if (forked) { + CHECK(!pipe(barrier)); + child = fork(); + CHECK(child >= 0); + if (!child) { + close(barrier[1]); + char token; + int ok = read(barrier[0], &token, 1) == 1 && + bitmap_ok(leaf, ps, shared, 0); + _exit(ok ? 0 : 8); + } + close(barrier[0]); + } + int tap = open("/dev/net/tun", O_RDWR | O_CLOEXEC); + CHECK(tap >= 0); + struct ifreq ifr = { .ifr_flags = IFF_TAP | IFF_NO_PI }; + strcpy(ifr.ifr_name, "abi%d"); + CHECK(!ioctl(tap, TUNSETIFF, &ifr)); + int raw = socket(AF_PACKET, SOCK_RAW | SOCK_CLOEXEC, htons(PROTOCOL)); + CHECK(raw >= 0); + CHECK(!ioctl(raw, SIOCGIFINDEX, &ifr)); + int ifindex = ifr.ifr_ifindex; + CHECK(!ioctl(raw, SIOCGIFFLAGS, &ifr)); + ifr.ifr_flags |= IFF_UP; + CHECK(!ioctl(raw, SIOCSIFFLAGS, &ifr)); + int fd = open("/dev/vhost-net", O_RDWR | O_CLOEXEC); + CHECK(fd >= 0); + CHECK(!ioctl(fd, VHOST_SET_FORK_FROM_OWNER, &mode)); + uint8_t gotmode = 9; + CHECK(!ioctl(fd, VHOST_GET_FORK_FROM_OWNER, &gotmode) && + gotmode == mode); + CHECK(!ioctl(fd, VHOST_SET_OWNER, NULL)); + uint64_t log_base = (uintptr_t)bitmap; + CHECK(!ioctl(fd, VHOST_SET_LOG_BASE, &log_base)); + unsigned char *data = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(data != MAP_FAILED); + struct { + struct vhost_memory mem; + struct vhost_memory_region region; + } mem = { + .mem.nregions = 1, + .region = { .guest_phys_addr = DATA_GPA, + .memory_size = 4096, + .userspace_addr = (uintptr_t)data }, + }; + CHECK(!ioctl(fd, VHOST_SET_MEM_TABLE, &mem)); + uint64_t features = (1ULL << VHOST_F_LOG_ALL) | + (1ULL << VHOST_NET_F_VIRTIO_NET_HDR); + uint64_t available; + CHECK(!ioctl(fd, VHOST_GET_FEATURES, &available) && + (available & features) == features); + CHECK(!ioctl(fd, VHOST_SET_FEATURES, &features)); + struct queue *q = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(q != MAP_FAILED && sizeof(*q) * 2 < ps); + CHECK(!queue_setup(fd, q, 0) && !queue_setup(fd, q + 1, 1)); + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { .index = index, .fd = tap }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, &backend)); + } + struct sockaddr_ll dest = { .sll_family = AF_PACKET, + .sll_protocol = htons(PROTOCOL), + .sll_ifindex = ifindex, + .sll_halen = 6 }; + unsigned char packet[PACKET_SIZE]; + memset(packet, 0x5c, sizeof(packet)); + memset(packet, 0xff, 6); + packet[6] = 2; + packet[12] = PROTOCOL >> 8; + packet[13] = PROTOCOL & 0xff; + q[0].desc[0] = (struct vring_desc){ .addr = DATA_GPA, + .len = 512, + .flags = VRING_DESC_F_WRITE }; + q[1].desc[0] = + (struct vring_desc){ .addr = DATA_GPA + 1024, + .len = sizeof(struct virtio_net_hdr) + + PACKET_SIZE }; + memcpy(data + 1024 + sizeof(struct virtio_net_hdr), packet, + PACKET_SIZE); + for (unsigned phase = 0; phase < 2; phase++) { + if (phase) { + memset(bitmap, 0, 9); + CHECK(!madvise(leaf, ps, MADV_DONTNEED)); + int pagemap = open("/proc/self/pagemap", O_RDONLY); + uint64_t entry; + CHECK(pagemap >= 0 && + pread(pagemap, &entry, sizeof(entry), + (uintptr_t)leaf / ps * sizeof(entry)) == + sizeof(entry)); + CHECK(!(entry & (1ULL << 63))); + close(pagemap); + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { + .index = index, .fd = tap + }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, + &backend)); + } + } + for (unsigned round = phase * ROUNDS; + round < (phase + 1) * ROUNDS; round++) { + packet[20] = round; + memset(data, 0xaa, 512); + CHECK(submit(q, round)); + CHECK(sendto(raw, packet, sizeof(packet), 0, + (void *)&dest, + sizeof(dest)) == sizeof(packet)); + CHECK(submit(q + 1, round)); + CHECK(complete(q, round) && complete(q + 1, round)); + CHECK(q[0].used.ring[round % N].id == 0); + if (q[0].used.ring[round % N].len != + sizeof(struct virtio_net_hdr) + PACKET_SIZE) + printf("RX length=%u expected=%zu ethertype=%02x%02x\n", + q[0].used.ring[round % N].len, + sizeof(struct virtio_net_hdr) + + PACKET_SIZE, + data[sizeof(struct virtio_net_hdr) + 12], + data[sizeof(struct virtio_net_hdr) + 13]); + CHECK(q[0].used.ring[round % N].len == + sizeof(struct virtio_net_hdr) + PACKET_SIZE); + CHECK(!memcmp(data + sizeof(struct virtio_net_hdr), + packet, PACKET_SIZE)); + CHECK(filled(data + sizeof(struct virtio_net_hdr) + + PACKET_SIZE, + 512 - sizeof(struct virtio_net_hdr) - + PACKET_SIZE, + 0xaa)); + } + /* Stop and flush workers before checking the bitmap or discarding backing. */ + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { .index = index, + .fd = -1 }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, &backend)); + } + CHECK(bitmap_ok(leaf, ps, 1, phase && !shared)); + if (forked && !phase) { + CHECK(write(barrier[1], "x", 1) == 1); + close(barrier[1]); + int status; + CHECK(waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status)); + } + } + close(fd); + close(raw); + close(tap); + for (unsigned i = 0; i < 2; i++) { + close(q[i].kick); + close(q[i].call); + } + CHECK(filled(dst, ps, 0x91) && filled(dst + 2 * ps, 2 * ps, 0x91)); + for (unsigned i = 0; i < 3; i++) + CHECK(filled(src + i * ps, ps, 0x61 + i)); + CHECK(!munmap(src, 3 * ps) && !munmap(dst, 4 * ps)); + CHECK(!munmap(data, ps) && !munmap(q, ps)); + if (backing >= 0) + close(backing); + printf("ok - %zuK vhost %s fork=%u worker=%u: %u RX/TX packets, relocated dirty bits, missing-PTE repair and neighbors\n", + ps / 1024, shared ? "shared" : "private", forked, mode, + 2 * ROUNDS); + return 0; +} +static int exercise(size_t expected) +{ + CHECK(getauxval(AT_PAGESZ) == expected); + alarm(45); + for (uint8_t mode = 0; mode < 2; mode++) + for (unsigned shared = 0; shared < 2; shared++) + for (unsigned forked = 0; forked < 2; forked++) + CHECK(!exercise_case(expected, shared, forked, + mode)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + /* Keep unsolicited IPv6 DAD packets out of the isolated TAP queues. */ + int ipv6 = + open("/proc/sys/net/ipv6/conf/default/disable_ipv6", O_WRONLY); + if (ipv6 < 0 || write(ipv6, "1", 1) != 1) { + perror("disable IPv6"); + fail = 1; + goto done; + } + close(ipv6); + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24]; + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = 0; + for (unsigned ticks = 0; c > 0 && ticks < 600; ticks++) { + pid_t got = waitpid(c, &status, WNOHANG); + if (got == c) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 599) { + kill(c, SIGKILL); + waitpid(c, &status, 0); + break; + } + usleep(100000); + } + printf("%s - ABI %zuK vhost log status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("VHOST LOG %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/wine-kernel32.def b/tools/testing/selftests/arm64/user4k/wine-kernel32.def new file mode 100644 index 0000000000000..891e48c1fb83b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/wine-kernel32.def @@ -0,0 +1,20 @@ +LIBRARY KERNEL32.dll +EXPORTS +VirtualAlloc +VirtualFree +VirtualProtect +VirtualQuery +GetSystemInfo +GetStdHandle +WriteFile +ExitProcess +GetLastError +AddVectoredExceptionHandler +RemoveVectoredExceptionHandler +CreateThread +WaitForSingleObject +GetExitCodeThread +CloseHandle +CreateFileMappingA +MapViewOfFile +UnmapViewOfFile diff --git a/tools/testing/selftests/arm64/user4k/wine-page-contract.c b/tools/testing/selftests/arm64/user4k/wine-page-contract.c new file mode 100644 index 0000000000000..149f619de6651 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/wine-page-contract.c @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding Windows x64 PE; Win32 imports, no CRT or external SDK. */ +typedef unsigned long DWORD; +typedef int BOOL; +typedef unsigned long long SIZE_T; +typedef void *HANDLE; +#define API __declspec(dllimport) +API void *VirtualAlloc(void *, SIZE_T, DWORD, DWORD); +API BOOL VirtualFree(void *, SIZE_T, DWORD); +API BOOL VirtualProtect(void *, SIZE_T, DWORD, DWORD *); +API SIZE_T VirtualQuery(const void *, void *, SIZE_T); +API void GetSystemInfo(void *); +API HANDLE GetStdHandle(DWORD); +API BOOL WriteFile(HANDLE, const void *, DWORD, DWORD *, void *); +API void ExitProcess(DWORD); +API DWORD GetLastError(void); +API void *AddVectoredExceptionHandler(DWORD, void *); +API DWORD RemoveVectoredExceptionHandler(void *); +API HANDLE CreateThread(void *, SIZE_T, DWORD (*)(void *), void *, DWORD, + DWORD *); +API DWORD WaitForSingleObject(HANDLE, DWORD); +API BOOL GetExitCodeThread(HANDLE, DWORD *); +API BOOL CloseHandle(HANDLE); +API HANDLE CreateFileMappingA(HANDLE, void *, DWORD, DWORD, DWORD, + const char *); +API void *MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); +API BOOL UnmapViewOfFile(const void *); +struct system_info { + DWORD arch, page; + void *min, *max; + SIZE_T mask; + DWORD cpus, type, granule; + unsigned short level, revision; +}; +struct memory_info { + void *base, *allocation; + DWORD allocation_protect, pad; + SIZE_T size; + DWORD state, protect, type, pad2; +}; +struct exception_record { + DWORD code, flags; + void *record, *address; + DWORD count, pad; + SIZE_T info[15]; +}; +struct exception_pointers { + struct exception_record *record; + void *context; +}; +/* Keep the freestanding declarations consistent with the Windows x64 ABI. */ +_Static_assert(sizeof(DWORD) == 4, "DWORD must be 32 bits"); +_Static_assert(sizeof(SIZE_T) == 8, "SIZE_T must be 64 bits"); +_Static_assert(sizeof(struct system_info) == 48, "SYSTEM_INFO layout"); +_Static_assert(sizeof(struct memory_info) == 48, + "MEMORY_BASIC_INFORMATION layout"); +_Static_assert(sizeof(struct exception_record) == 152, + "EXCEPTION_RECORD layout"); +static volatile unsigned char *pages; +static volatile DWORD handled; +static void say(const char *s) +{ + DWORD n = 0, w = 0; + while (s[n]) + n++; + WriteFile(GetStdHandle((DWORD)-11), s, n, &w, 0); +} +static void number(DWORD n) +{ + char buf[11], *p = buf + 10; + *p = 0; + do { + *--p = '0' + n % 10; + n /= 10; + } while (n); + say(p); +} +static void fail(DWORD stage) +{ + DWORD e = GetLastError(); + say("WINE PAGE CONTRACT FAIL stage="); + number(stage); + say(" error="); + number(e); + say("\r\n"); + ExitProcess(1); +} +#define CHECK(x, n) \ + do { \ + if (!(x)) \ + fail(n); \ + } while (0) +static long handler(struct exception_pointers *e) +{ + DWORD old; + if (e->record->code != 0xc0000005 || e->record->count < 2 || + e->record->info[0] != 1 || + e->record->info[1] != (SIZE_T)(pages + 4103)) + return 0; + if (!VirtualProtect((void *)(pages + 4096), 4096, 4, &old)) + return 0; + handled++; + return -1; +} +static DWORD thread(void *arg) +{ + volatile unsigned char *p = arg; + for (unsigned int i = 0; i < 4096; i++) + p[i] = (unsigned char)(i ^ 0x5a); + return 37; +} +void mainCRTStartup(void) +{ + struct system_info si; + struct memory_info mi; + DWORD old, status; + GetSystemInfo(&si); + CHECK(si.page == 4096 && si.granule == 65536, 1); + say("ok - Wine reports 4K pages and 64K allocation granularity\r\n"); + pages = VirtualAlloc(0, 65536, 0x2000, 1); + CHECK(pages, 2); + CHECK(VirtualAlloc((void *)pages, 12288, 0x1000, 4) == pages, 3); + for (unsigned int i = 0; i < 12288; i++) + pages[i] = (unsigned char)(i ^ (i >> 12)); + CHECK(VirtualProtect((void *)(pages + 4096), 4096, 2, &old) && old == 4, + 4); + CHECK(VirtualQuery((void *)(pages + 4096), &mi, sizeof(mi)) == + sizeof(mi) && + mi.base == pages + 4096 && mi.size == 4096 && + mi.protect == 2, + 5); + CHECK(VirtualQuery((void *)(pages + 8192), &mi, sizeof(mi)) == + sizeof(mi) && + mi.protect == 4, + 6); + void *veh = AddVectoredExceptionHandler(1, handler); + CHECK(veh, 7); + pages[4103] = 0x6d; + CHECK(handled == 1 && pages[4103] == 0x6d, 8); + CHECK(RemoveVectoredExceptionHandler(veh), 9); + say("ok - Wine independent 4K protection and handled access violation\r\n"); + CHECK(VirtualFree((void *)(pages + 4096), 4096, 0x4000), 10); + CHECK(VirtualQuery((void *)(pages + 4096), &mi, sizeof(mi)) == + sizeof(mi) && + mi.state == 0x2000, + 11); + CHECK(VirtualAlloc((void *)(pages + 4096), 4096, 0x1000, 4) == + pages + 4096, + 12); + for (unsigned int i = 0; i < 4096; i++) + CHECK(!pages[4096 + i], 13); + for (unsigned int i = 0; i < 4096; i++) + CHECK(pages[i] == (unsigned char)i && + pages[8192 + i] == (unsigned char)(i ^ 2), + 14); + say("ok - Wine 4K decommit and zero-filled recommit preserve neighbours\r\n"); + HANDLE t = CreateThread(0, 0, thread, (void *)(pages + 4096), 0, 0); + CHECK(t, 15); + CHECK(WaitForSingleObject(t, 30000) == 0 && + GetExitCodeThread(t, &status) && status == 37, + 16); + CHECK(CloseHandle(t), 17); + for (unsigned int i = 0; i < 4096; i++) + CHECK(pages[4096 + i] == (unsigned char)(i ^ 0x5a), 18); + say("ok - Wine threads share committed 4K pages\r\n"); + HANDLE file = CreateFileMappingA((HANDLE)(SIZE_T)-1, 0, 4, 0, 16384, 0); + CHECK(file, 19); + unsigned char *a = MapViewOfFile(file, 2, 0, 0, 16384), + *b = MapViewOfFile(file, 2, 0, 0, 16384); + CHECK(a && b, 20); + a[4099] = 0xa7; + CHECK(b[4099] == 0xa7, 21); + CHECK(VirtualProtect(a + 4096, 4096, 2, &old), 22); + b[4099] = 0x53; + CHECK(a[4099] == 0x53, 23); + CHECK(UnmapViewOfFile(a) && UnmapViewOfFile(b) && CloseHandle(file), + 24); + CHECK(VirtualFree((void *)pages, 0, 0x8000), 25); + say("ok - Wine shared section aliases and 4K protection\r\nWINE PAGE CONTRACT PASS\r\n"); + ExitProcess(0); +} diff --git a/tools/testing/selftests/arm64/user4k/x18-probe.S b/tools/testing/selftests/arm64/user4k/x18-probe.S new file mode 100644 index 0000000000000..565f34e8323f5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/x18-probe.S @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include + + .text + .globl x18_syscall + .type x18_syscall, %function +/* (nr, arg0, arg1, arg2, observed_x18); no libc code inside the probe. */ +x18_syscall: + hint 34 /* bti c */ + stp x18, x19, [sp, #-16]! + mov x19, x4 + movz x18, #0x3210 + movk x18, #0x7654, lsl #16 + movk x18, #0xba98, lsl #32 + movk x18, #0xfedc, lsl #48 + mov x8, x0 + mov x0, x1 + mov x1, x2 + mov x2, x3 + svc #0 + str x18, [x19] + ldp x18, x19, [sp], #16 + ret + .size x18_syscall, .-x18_syscall + .section .note.GNU-stack,"",%progbits diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 9e0dfd3a85b0e..8d0377cda51f0 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -533,6 +533,9 @@ struct vm_area_desc { /* Mutable fields. Populated with initial state. */ pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif struct file *vm_file; vma_flags_t vma_flags; pgprot_t page_prot; @@ -1300,6 +1303,9 @@ static inline void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->end = vma->vm_end; desc->pgoff = vma->vm_pgoff; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = vma->vm_subpage_offset; +#endif desc->vm_file = vma->vm_file; desc->vma_flags = vma->flags; desc->page_prot = vma->vm_page_prot; @@ -1308,6 +1314,117 @@ static inline void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->action.type = MMAP_NOTHING; } +static inline unsigned int mm_page_shift(const struct mm_struct *mm) +{ + return PAGE_SHIFT; +} + +static inline unsigned long mm_page_size(const struct mm_struct *mm) +{ + return PAGE_SIZE; +} + +static inline unsigned long mm_page_mask(const struct mm_struct *mm) +{ + return PAGE_MASK; +} + +static inline unsigned long mm_page_align(const struct mm_struct *mm, unsigned long size) +{ + return PAGE_ALIGN(size); +} + +static inline unsigned long vma_user_pages(const struct vm_area_struct *vma) +{ + return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; +} + +#define mm_account_memory(mm, policy_mm, pages) security_vm_enough_memory_mm(policy_mm, pages) +#define mm_unacct_memory(mm, pages) vm_unacct_memory(pages) + +/* File-cache indices remain native-page units even for a 4K VMA. */ +struct vm_page_offset { + pgoff_t index; + unsigned int offset; +}; + +static inline struct vm_page_offset vm_page_offset_add(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = pos.offset + (bytes & ~PAGE_MASK); + + pos.index += (bytes >> PAGE_SHIFT) + (low >> PAGE_SHIFT); + pos.offset = low & ~PAGE_MASK; + return pos; +} + +static inline struct vm_page_offset vm_page_offset_sub(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = bytes & ~PAGE_MASK; + + pos.index -= (bytes >> PAGE_SHIFT) + (low > pos.offset); + pos.offset = (pos.offset - low) & ~PAGE_MASK; + return pos; +} + +static inline bool vm_page_offset_equal(struct vm_page_offset a, struct vm_page_offset b) +{ + return a.index == b.index && a.offset == b.offset; +} + +static inline unsigned int vma_subpage_offset(const struct vm_area_struct *vma) +{ +#ifdef CONFIG_MM_SUBPAGE + return vma->vm_subpage_offset; +#else + return 0; +#endif +} + +/* address must be >= vm_start; vm_end computes the next contiguous offset. */ +static inline struct vm_page_offset vma_page_offset_at(const struct vm_area_struct *vma, + unsigned long address) +{ + unsigned long delta = address - vma->vm_start; + unsigned long low = (delta & ~PAGE_MASK) + vma_subpage_offset(vma); + + return (struct vm_page_offset) { + .index = vma->vm_pgoff + (delta >> PAGE_SHIFT) + (low >> PAGE_SHIFT), + .offset = low & ~PAGE_MASK, + }; +} + +static inline void vma_set_page_offset(struct vm_area_struct *vma, struct vm_page_offset pos) +{ + vma->vm_pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = pos.offset; +#endif +} + +/* Inverse lookup of a single byte; an offset outside the VMA is not clamped. */ +static inline unsigned long vma_address_at_offset(const struct vm_area_struct *vma, + struct vm_page_offset pos) +{ + unsigned long len = vma->vm_end - vma->vm_start; + unsigned long index, delta, last; + unsigned int offset = vma_subpage_offset(vma); + + if (!len || pos.index < vma->vm_pgoff || pos.offset >= PAGE_SIZE) + return -EFAULT; + index = pos.index - vma->vm_pgoff; + last = ((len - 1) >> PAGE_SHIFT) + + ((((len - 1) & ~PAGE_MASK) + offset) >> PAGE_SHIFT); + if (index > last) + return -EFAULT; + delta = (index << PAGE_SHIFT) + pos.offset; + if (delta < offset || delta - offset >= len) + return -EFAULT; + return vma->vm_start + delta - offset; +} + + static inline unsigned long vma_pages(const struct vm_area_struct *vma) { return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; diff --git a/tools/testing/vma/tests/merge.c b/tools/testing/vma/tests/merge.c index 03b6f9820e0ac..0b0f4ae413973 100644 --- a/tools/testing/vma/tests/merge.c +++ b/tools/testing/vma/tests/merge.c @@ -227,7 +227,7 @@ static bool test_simple_shrink(void) ASSERT_FALSE(attach_vma(&mm, vma)); - ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, 0)); + ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, (struct vm_page_offset) { 0, 0 })); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x1000); diff --git a/tools/testing/vma/tests/mmap.c b/tools/testing/vma/tests/mmap.c index c85bc000d1cb7..c87baa48ea26e 100644 --- a/tools/testing/vma/tests/mmap.c +++ b/tools/testing/vma/tests/mmap.c @@ -12,19 +12,19 @@ static bool test_mmap_region_basic(void) current->mm = &mm; /* Map at 0x300000, length 0x3000. */ - addr = __mmap_region(NULL, 0x300000, 0x3000, vma_flags, 0x300, NULL); + addr = __mmap_region(NULL, 0x300000, 0x3000, vma_flags, (struct vm_page_offset) { 0x300, 0 }, NULL); ASSERT_EQ(addr, 0x300000); /* Map at 0x250000, length 0x3000. */ - addr = __mmap_region(NULL, 0x250000, 0x3000, vma_flags, 0x250, NULL); + addr = __mmap_region(NULL, 0x250000, 0x3000, vma_flags, (struct vm_page_offset) { 0x250, 0 }, NULL); ASSERT_EQ(addr, 0x250000); /* Map at 0x303000, merging to 0x300000 of length 0x6000. */ - addr = __mmap_region(NULL, 0x303000, 0x3000, vma_flags, 0x303, NULL); + addr = __mmap_region(NULL, 0x303000, 0x3000, vma_flags, (struct vm_page_offset) { 0x303, 0 }, NULL); ASSERT_EQ(addr, 0x303000); /* Map at 0x24d000, merging to 0x250000 of length 0x6000. */ - addr = __mmap_region(NULL, 0x24d000, 0x3000, vma_flags, 0x24d, NULL); + addr = __mmap_region(NULL, 0x24d000, 0x3000, vma_flags, (struct vm_page_offset) { 0x24d, 0 }, NULL); ASSERT_EQ(addr, 0x24d000); ASSERT_EQ(mm.map_count, 2); diff --git a/tools/testing/vma/tests/vma.c b/tools/testing/vma/tests/vma.c index 754a2da063215..38a7cc6702f23 100644 --- a/tools/testing/vma/tests/vma.c +++ b/tools/testing/vma/tests/vma.c @@ -38,7 +38,7 @@ static bool test_copy_vma(void) /* Move backwards and do not merge. */ vma = alloc_and_link_vma(&mm, 0x3000, 0x5000, 3, vma_flags); - vma_new = copy_vma(&vma, 0, 0x2000, 0, &need_locks); + vma_new = copy_vma(&vma, 0, 0x2000, (struct vm_page_offset) { 0, 0 }, &need_locks); ASSERT_NE(vma_new, vma); ASSERT_EQ(vma_new->vm_start, 0); ASSERT_EQ(vma_new->vm_end, 0x2000); @@ -51,7 +51,7 @@ static bool test_copy_vma(void) vma = alloc_and_link_vma(&mm, 0, 0x2000, 0, vma_flags); vma_next = alloc_and_link_vma(&mm, 0x6000, 0x8000, 6, vma_flags); - vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, &need_locks); + vma_new = copy_vma(&vma, 0x4000, 0x2000, (struct vm_page_offset) { 4, 0 }, &need_locks); vma_assert_attached(vma_new); ASSERT_EQ(vma_new, vma_next); From 6404f6a5b060c3b8730f7fcda19d4a72d59fab1e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:23 -0400 Subject: [PATCH 0430/1579] docs: describe mixed userspace page sizes and the development launcher Document native backing versus per-process mappings, architecture responsibilities, supported combinations and remaining limitations. Provide a launcher for inherited and one-shot exec preferences. Signed-off-by: Diljit Singh --- Documentation/arch/arm64/index.rst | 1 + Documentation/arch/arm64/user4k.rst | 187 ++++++++++++++++++++++++++++ Documentation/mm/index.rst | 1 + Documentation/mm/user_page_size.rst | 84 +++++++++++++ tools/aurora-userpages/.gitignore | 1 + tools/aurora-userpages/Makefile | 14 +++ tools/aurora-userpages/README.rst | 34 +++++ tools/aurora-userpages/userpages.c | 83 ++++++++++++ 8 files changed, 405 insertions(+) create mode 100644 Documentation/arch/arm64/user4k.rst create mode 100644 Documentation/mm/user_page_size.rst create mode 100644 tools/aurora-userpages/.gitignore create mode 100644 tools/aurora-userpages/Makefile create mode 100644 tools/aurora-userpages/README.rst create mode 100644 tools/aurora-userpages/userpages.c diff --git a/Documentation/arch/arm64/index.rst b/Documentation/arch/arm64/index.rst index 98052b4ef4a1e..e5d0d0c46bf3b 100644 --- a/Documentation/arch/arm64/index.rst +++ b/Documentation/arch/arm64/index.rst @@ -32,5 +32,6 @@ ARM64 Architecture sve tagged-address-abi tagged-pointers + user4k features diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst new file mode 100644 index 0000000000000..d483c565d853b --- /dev/null +++ b/Documentation/arch/arm64/user4k.rst @@ -0,0 +1,187 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +=========================================== +Experimental per-process page sizes on arm64 +=========================================== + +The mixed-page implementation lets an address space use a different user +translation granule from the kernel's native physical page size. The kernel +page size remains fixed for the boot. An address space's user granule remains +fixed for its lifetime; exec can create a new address space with another +supported granule. See :doc:`../../mm/user_page_size` for the generic launch +policy and architecture integration contract. + +This is an experimental ABI. The launch commands use private prctl numbers, +not assigned upstream UAPI. The implementation and tests do not establish +complete application, device or hardware qualification. + +Configuration and supported combinations +======================================== + +Enable ``CONFIG_ARM64_USER4K_EXPERIMENTAL`` with 48-bit VA and PA and a native +16K or 64K page size. The option defaults off and selects the generic +``MM_SUBPAGE`` backing and ``ARCH_HAS_USER_PAGE_SIZE`` exec policy support. +LPA2 and ``NO_PAGE_MAPCOUNT`` configurations are excluded by Kconfig. + +The exposed combinations are: + +=================== ================== ============================ +Native kernel size AArch64 default AArch64 alternatives +=================== ================== ============================ +4K 4K None in this implementation +16K 16K 4K, when supported by the CPUs +64K 64K 4K and 16K, subject to support +=================== ================== ============================ + +Capability detection uses the sanitized system-wide granule intersection. +AArch32 executables select 4K when that capability is available and retain +the native granule otherwise, regardless of launch preferences. The latter +case still enforces native mapping/protection alignment. AArch32 mmap2 file +offsets retain their 4K ABI unit in either case. + +An AArch32 executable running under an arm64 kernel is distinct from an +``arch/arm`` kernel. The latter retains its existing native ABI; this series +does not yet provide its alternative-granule backend. Exploiting extra +translation capabilities available only within an asymmetric AArch32 CPU +domain also remains unimplemented. + +Larger-than-native process granules are not exposed. Internal 64K-on-16K +fixtures exercise allocation, mappings and argument staging, but those +fixtures do not enable a public 64K process ABI on a 16K kernel. + +Selecting an executable's page size +=================================== + +Build the development launcher with:: + + make -C tools/aurora-userpages + +For example:: + + tools/aurora-userpages/userpages 4k COMMAND [ARGUMENTS...] + tools/aurora-userpages/userpages --once native COMMAND [ARGUMENTS...] + tools/aurora-userpages/userpages --status + +The ordinary selection establishes an inherited per-thread exec preference. +``--once`` overrides it for one successful exec. A failed exec retains the +pending request; changing a preference never changes existing mappings. +Fork copies the preferences and mappings. Threads sharing an mm have the +same current granule but may establish different preferences for later exec. +``native`` clears an inherited alternative preference; a one-shot native +request can override that preference without clearing it. + +The final ELF loader resolves the granule before committing the new image. +If arguments were staged with another granule, their bytes are transferred +into a new private mm. Temporary argument mappings are bounded by both the +user leaf and native page size. Logical argument-size limits continue to +use the process granule. The interpreter and executable share one mm and +therefore one final granule. ELF segment alignment alone does not opt a +program into an alternative ABI. + +``AT_PAGESZ`` reports the process granule. User virtual address alignment, +protection boundaries and mapping lengths use that granule. Native physical +allocation, PFN arithmetic and file-cache indices continue to use the kernel +page size. Application execution does not require any of the KUnit options +or the ``arm64.user4k_test=1`` diagnostic boot parameter. + +Memory and table ownership +========================== + +Keep the following units separate: + +* ``PAGE_SIZE`` and physical PFNs describe native physical pages. +* ``mm_page_size(mm)`` describes the user mapping/protection unit. +* A folio owns one or more native pages and carries their physical charge. + +Smaller anonymous mappings retain explicit subpage ownership. A reference to +the containing folio alone does not retain an individual reusable slot. +The slot's byte offset must survive fault, COW, reverse mapping, migration, +swap and pinning operations. Newly exposed bytes are zeroed, and changing +one mapping's protection must not widen that change to adjacent slots. + +File offsets use a native cache index plus an explicit byte offset. VMA +split/merge, truncation and reverse mapping must preserve both components. +Physical allocation charges count the actual backing once, independently +of how many user mappings expose it. + +Lower page tables can share a native allocation. Table tokens preserve the +fragment address, while locking and lifetime use the native table owner. +Unissued cached fragments retain owner references. Issued fragments are not +reused within a live owner. Deferred reclamation must preserve the fragment +token through TLB invalidation and the required RCU grace period, including +allocation-failure fallback paths. + +Translation context and invalidation +=================================== + +The arm64 backend supplies per-mm page-table geometry. TTBR1 retains native +kernel tables, while TTBR0 uses the selected user geometry. A granule change +installs and synchronizes a reserved TTBR0 before changing TG0, then +synchronizes the new geometry before installing the user root. Software PAN +also selects the user TG0 even when restoration of TTBR0 is deferred until +uaccess or return to userspace. + +The existing mm ASID is retained. Selecting another granule does not add +an unconditional context-switch TLB flush. User invalidation operands and +range strides use the target mm's geometry; kernel and stage-2 operations +retain their own native geometry. Transition/idmap paths restore native +TG0 and then the active mm's user geometry when returning. + +The current configuration retains the existing 48-bit user address size. +Independent TG0/TG1 fields do not make all other translation controls +independent; the LPA2 exclusion must not be removed merely by changing TG0. + +Sharing, pinning and interface limits +==================================== + +Mixed-granule sharing is permitted. Shared mappings refer to one coherent +backing object, with permissions belonging to each virtual mapping. Shared +futex identity includes the object and byte offset. Private copying is not +an implementation of shared-memory semantics. + +Some interfaces have additional layout or ownership requirements. For +example, a perf ring's first mapper fixes its userspace granule; a mapper +using another granule is rejected. This does not prohibit mixed-granule +memfd or System V shared-memory aliases. + +A legacy page-array pin cannot express every smaller-granule byte range. +Converted consumers use typed fragment references carrying backing, offset, +length and ownership. Unconverted paths must not silently round a pin or DMA +mapping into neighboring slots. Mock IOMMU/VFIO tests validate the recorded +translations and ownership paths, not physical-device DMA on arbitrary +hardware. + +Alternative-granule THP/KSM and hugetlb support, remaining legacy pin +consumers, larger-than-native process activation and broader device +integration still need work. Userfaultfd feature bits and per-range ioctl +masks must be consulted; success for a shmem operation does not imply +support for the corresponding hugetlb operation. Sparse individual fills +can retain an entire native folio; packing behavior is not a throughput +or memory-efficiency guarantee for every workload. + +Test coverage +============= + +The arm64 selftests are in ``tools/testing/selftests/arm64/user4k``. The +``page-contract`` executable checks the exposed userspace contract. Extended +fixtures cover exec policy, AArch32, shared mappings, accounting, reclaim, +I/O and device-facing interfaces. Several are disposable-guest PID1 programs +with companion executables or storage requirements; they are not ordinary +host test commands. + +``CONFIG_ARM64_USER4K_KUNIT_TEST`` and the data/TLB suites exercise internal +geometry and ownership. Some fixtures require ``arm64.user4k_test=1`` and +some hardware translation tests are disabled under KASAN. Check individual +KTAP results and skips rather than treating boot success as a suite pass. +Private IOMMUFD/VFIO test-device options are separate and should remain off +in application kernels. + +The NUMA COW batching case runs only at boot, before userspace can change +its policy prerequisite. Its results remain visible in KUnit debugfs, but +it has no runtime rerun control. Other suites retain their rerun controls. + +Validation must distinguish native/default operation, each exposed +alternative, and AArch32 fallback with 4K support unavailable. Native ARM32 +short-descriptor and LPAE controls check that generic changes preserve the +existing ABI; they do not validate a mixed-page ARM32 backend. Internal +larger-leaf tests likewise do not qualify a public process mode. diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst index 7aa2a88869083..fb6128d42aa84 100644 --- a/Documentation/mm/index.rst +++ b/Documentation/mm/index.rst @@ -13,6 +13,7 @@ see the :doc:`admin guide <../admin-guide/mm/index>`. physical_memory page_tables process_addrs + user_page_size bootmem page_allocation vmalloc diff --git a/Documentation/mm/user_page_size.rst b/Documentation/mm/user_page_size.rst new file mode 100644 index 0000000000000..62ef6ae39e0ac --- /dev/null +++ b/Documentation/mm/user_page_size.rst @@ -0,0 +1,84 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +============================== +Per-process user mapping sizes +============================== + +The experimental mixed-page implementation separates the user mapping +size of an address space from the kernel's native allocation size. +``PAGE_SIZE`` still describes native physical pages. ``mm_page_size(mm)`` +describes the minimum user mapping and protection unit of that address +space. A process retains one mapping size for the lifetime of its ``mm``. +Exec may create a new ``mm`` with a different size; changing a launch +preference never changes an existing mapping. + +The generic interfaces described here are internal development interfaces. +The experimental launch commands still use private prctl numbers; these +are not assigned upstream UAPI. The arm64 backend's supported modes and +current restrictions are documented in +:doc:`../arch/arm64/user4k`. + +Exec policy and inheritance +=========================== + +``ARCH_HAS_USER_PAGE_SIZE`` enables two per-thread policy fields in +``task_struct``: + +``exec_page_shift`` + A one-shot request for the next exec. Zero means no one-shot request. + +``default_exec_page_shift`` + An inherited preference used when no one-shot request overrides it. + Zero means the architecture's ordinary executable policy. + +Fork and thread creation copy both fields. Subsequent changes affect only +the calling thread, including when it shares an ``mm`` with other threads. +An exec by a non-leader uses the execing thread's policy. The fields do not +belong in ``mm_struct``: threads sharing one address space may launch new +images with different preferences. + +Generic prctl handling validates reserved arguments before calling the +architecture validator. A rejected request leaves both fields unchanged. +Setting a native one-shot request while an inherited alternative is active +stores ``PAGE_SHIFT`` explicitly, so the next exec can override that +alternative. Clearing the inherited preference does not consume a pending +one-shot request. + +Exec stages arguments in a private ``mm``. Its initial allocation receives +the one-shot shift, if any; the final executable loader resolves the +inherited preference and any executable-specific policy. An ELF interpreter +must be loaded using the same final mapping size as its executable. +Failure before committing the new image retains both preferences. +``setup_new_exec()`` consumes the one-shot field after the new image is +committed, while retaining the inherited preference. + +Architecture responsibilities +============================= + +An architecture selecting ``ARCH_HAS_USER_PAGE_SIZE`` must implement: + +* ``arch_exec_page_size_shift(size)``: return zero for a native request, a + validated alternative shift, or a negative errno. Validation includes + the supported hardware and kernel configuration, not just whether the + requested byte size is a power of two. +* ``arch_mm_init_exec(mm, shift)``: initialize the private address space's + geometry before allocating its page-table root. Zero denotes native + geometry. The hook receives the request explicitly and does not read + a separate architecture thread-policy field. +* Binary-loader integration that resolves the final executable policy + before exposing the address space. ``bprm_set_page_shift()`` transfers + staged argument bytes when the final choice changes the initial granule. +* Page-table allocation, walking, invalidation, context switching and MM + operations for every size the architecture exposes. + +The architecture retains control over executable compatibility policy. +For example, the arm64 AArch32 loader prefers 4K where the backend supports +it and retains the native ABI when that capability is absent. This is +separate from running an AArch32 kernel: ``arch/arm`` needs its own backend +integration before it can advertise alternative mapping sizes. The generic +exec policy alone does not provide that backend or expose new sizes there. + +Architectures without this capability retain native mappings and reject +the experimental launch-policy commands. ``ARCH_HAS_MM_PGTABLE`` describes +per-mm page-table accessors; it does not by itself promise an executable +userspace ABI. diff --git a/tools/aurora-userpages/.gitignore b/tools/aurora-userpages/.gitignore new file mode 100644 index 0000000000000..a27b2bf8b140d --- /dev/null +++ b/tools/aurora-userpages/.gitignore @@ -0,0 +1 @@ +/userpages diff --git a/tools/aurora-userpages/Makefile b/tools/aurora-userpages/Makefile new file mode 100644 index 0000000000000..c6d42e2d15efa --- /dev/null +++ b/tools/aurora-userpages/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +CC ?= cc +CFLAGS ?= -O2 -g +CFLAGS += -Wall -Wextra +LDFLAGS += -Wl,-z,max-page-size=65536 + +.PHONY: all clean +all: userpages + +userpages: userpages.c ../../include/uapi/linux/prctl.h + $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $< $(LDLIBS) -o $@ + +clean: + rm -f userpages diff --git a/tools/aurora-userpages/README.rst b/tools/aurora-userpages/README.rst new file mode 100644 index 0000000000000..64f4fcb0dd12a --- /dev/null +++ b/tools/aurora-userpages/README.rst @@ -0,0 +1,34 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Per-process userspace page-size launcher +======================================= + +Build with ``make`` in this directory. The running arm64 kernel must provide +Aurora's experimental per-process granule API. + +Examples:: + + userpages 4k FEXInterpreter ./x86-program + userpages 4k python3 script.py + userpages native program + userpages --status + +The choice becomes the calling thread's inherited exec preference: it applies +to the launched program and subsequent execs in its descendants. Fork retains +the address space and both exec preferences. Other threads and processes are +unchanged. ``native`` clears the inherited alternative preference. Ordinary +processes that never opt in continue to use the kernel's native default. + +``--once`` changes only the next successful exec; an existing inherited +preference remains available for later execs. For example, +``userpages --once native program`` runs one program with native pages and +retains any inherited 4K preference for its subsequent execs. Failed exec does +not consume the pending request. No selection changes a live address space. +AArch32 programs always use 4K when supported, otherwise native pages, +regardless of either preference. + +The kernel checks supported sizes and CPU capabilities. The current branch +supports 4K userspace on 16K/64K kernels and 16K userspace on 64K kernels. +64K userspace on a 16K kernel is not enabled. This launcher uses the private +PR_AURORA_{SET,GET}_{EXEC,DEFAULT}_PAGE_SIZE API in this tree; +it is not an upstream Linux ABI. diff --git a/tools/aurora-userpages/userpages.c b/tools/aurora-userpages/userpages.c new file mode 100644 index 0000000000000..02d9e4357224d --- /dev/null +++ b/tools/aurora-userpages/userpages.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include "../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include + +static int usage(const char *name, int status) +{ + fprintf(status ? stderr : stdout, + "Usage: %s [--once] {native|4k|16k|64k} COMMAND [ARG ...]\n" + " %s --status\n" + "Select the userspace page size for this program and its descendants.\n", + name, name); + return status; +} + +int main(int argc, char **argv) +{ + unsigned long size; + int once = argc > 1 && !strcmp(argv[1], "--once"); + int first = once ? 2 : 1; + + if (argc == 2 && !strcmp(argv[1], "--help")) + return usage(argv[0], 0); + if (argc == 2 && !strcmp(argv[1], "--status")) { + int pending = prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); + int error = errno; + + printf("Current process page size: %lu bytes\n", getauxval(AT_PAGESZ)); + if (pending < 0) { + fprintf(stderr, "Page-size selection unavailable: %s\n", strerror(error)); + return 1; + } + int inherited = prctl(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); + + if (inherited < 0) { + perror("Cannot read inherited page-size preference"); + return 1; + } + if (inherited) + printf("Inherited exec preference: %d bytes\n", inherited); + else + puts("Inherited exec preference: native default"); + if (pending) + printf("Next exec page size: %d bytes\n", pending); + else + puts("Next exec page size: inherited preference"); + return 0; + } + if (argc < first + 2) + return usage(argv[0], 2); + if (!strcmp(argv[first], "native")) + size = 0; + else if (!strcmp(argv[first], "4k")) + size = 4096; + else if (!strcmp(argv[first], "16k")) + size = 16384; + else if (!strcmp(argv[first], "64k")) + size = 65536; + else + return usage(argv[0], 2); + + if (!once && prctl(PR_AURORA_SET_DEFAULT_PAGE_SIZE, size, 0UL, 0UL, 0UL)) { + fprintf(stderr, "Cannot set inherited %s pages: %s\n", + argv[first], strerror(errno)); + return 1; + } + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL)) { + fprintf(stderr, "Cannot select %s userspace pages: %s\n", + argv[first], strerror(errno)); + return 1; + } + execvp(argv[first + 1], &argv[first + 1]); + int error = errno; + + fprintf(stderr, "Cannot execute %s: %s\n", argv[first + 1], strerror(error)); + return error == ENOENT ? 127 : 126; +} From 39b781e430cd503feb48b302e1b437a93236126a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:15:47 +1000 Subject: [PATCH 0431/1579] drm: apple: balance piodma child ownership across bind attempts Adopt the registered piodma child rather than creating a duplicate during component rebind. Keep its lookup reference until unbind and release it on every later unsuccessful bind; failed component binds have no unbind call. A missing IOMMU domain defers bind. One cleanup helper distinguishes an adopted registration from a child created by this bind and clears stale child/domain pointers. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 99 +++++++++++++++------------- 2 files changed, 54 insertions(+), 46 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 0a237c86e343e..883c70399fc05 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -158,6 +158,7 @@ struct apple_dcp_hw_data { struct apple_dcp { struct device *dev; struct platform_device *piodma; + bool piodma_created; struct iommu_domain *iommu_dom; /* which of the nine cumulative A031 notify-client states to send next */ unsigned int a031_step; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index fc28b87045dfc..061f68d865437 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1588,6 +1588,19 @@ static void dcp_work_update_backlight(struct work_struct *work) dcp_backlight_update(dcp); } +static void dcp_release_piodma_iommu_dev(struct apple_dcp *dcp) +{ + if (dcp->piodma) { + if (dcp->piodma_created) + of_platform_device_destroy(&dcp->piodma->dev, NULL); + else + put_device(&dcp->piodma->dev); + } + dcp->piodma = NULL; + dcp->iommu_dom = NULL; + dcp->piodma_created = false; +} + static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) { int ret; @@ -1597,31 +1610,13 @@ static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) return dev_err_probe(dcp->dev, -ENODEV, "Failed to get piodma child DT node\n"); - dcp->piodma = of_platform_device_create(node, NULL, dcp->dev); - if (!dcp->piodma) { - /* - * of_platform_device_create() returns NULL when the node already - * has a platform device, which happens whenever a previous bind - * attempt created it and then failed further down (the caller's - * later error paths do not destroy it). Probe is retried - a - * deferral, or a component re-bind - and would then fail here - * forever with -ENODEV even though nothing is actually wrong. - * Adopt the existing device instead of giving up. - */ - dcp->piodma = of_find_device_by_node(node); - if (!dcp->piodma) - return dev_err_probe(dcp->dev, -ENODEV, - "Failed to create piodma pdev for %pOF\n", - node); - /* - * Drop the lookup reference: the device stays registered until - * of_platform_device_destroy(), exactly like one created above, - * and that call only releases the registration reference. - */ - put_device(&dcp->piodma->dev); - dev_info(dcp->dev, "reusing existing piodma pdev for %pOF\n", - node); - } + dcp->piodma = of_find_device_by_node(node); + dcp->piodma_created = !dcp->piodma; + if (dcp->piodma_created) + dcp->piodma = of_platform_device_create(node, NULL, dcp->dev); + if (!dcp->piodma) + return dev_err_probe(dcp->dev, -ENODEV, + "Failed to create piodma pdev for %pOF\n", node); ret = dma_set_mask_and_coherent(&dcp->piodma->dev, DMA_BIT_MASK(42)); if (ret) @@ -1645,7 +1640,7 @@ static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) return 0; err_destroy_pdev: - of_platform_device_destroy(&dcp->piodma->dev, NULL); + dcp_release_piodma_iommu_dev(dcp); return ret; } @@ -2002,24 +1997,33 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) } ret = dcp_create_piodma_iommu_dev(dcp); - if (ret || !dcp->iommu_dom) + if (ret) return dev_err_probe(dev, ret, - "Failed to created PIODMA iommu child device"); + "Failed to create PIODMA iommu child device\n"); + if (!dcp->iommu_dom) { + ret = dev_err_probe(dev, -EPROBE_DEFER, + "PIODMA iommu domain is unavailable\n"); + goto err_piodma; + } ret = dcp_get_disp_regs(dcp); if (ret) { dev_err(dev, "failed to find display registers\n"); - return ret; + goto err_piodma; } dcp->clk = devm_clk_get(dev, NULL); - if (IS_ERR(dcp->clk)) - return dev_err_probe(dev, PTR_ERR(dcp->clk), - "Unable to find clock\n"); + if (IS_ERR(dcp->clk)) { + ret = dev_err_probe(dev, PTR_ERR(dcp->clk), + "Unable to find clock\n"); + goto err_piodma; + } dcp->clk_194 = devm_clk_get_optional(dev, "clock-194"); - if (IS_ERR(dcp->clk_194)) - return dev_err_probe(dev, PTR_ERR(dcp->clk_194), - "Unable to find clock 0x194\n"); + if (IS_ERR(dcp->clk_194)) { + ret = dev_err_probe(dev, PTR_ERR(dcp->clk_194), + "Unable to find clock 0x194\n"); + goto err_piodma; + } bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry @@ -2039,17 +2043,24 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) } dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); - if (IS_ERR(dcp->rtk)) - return dev_err_probe(dev, PTR_ERR(dcp->rtk), - "Failed to initialize RTKit\n"); + if (IS_ERR(dcp->rtk)) { + ret = dev_err_probe(dev, PTR_ERR(dcp->rtk), + "Failed to initialize RTKit\n"); + goto err_piodma; + } if (dcp->hw.adopt_live_session) dev_info(dev, "negotiating RTKit with the running DCP\n"); ret = apple_rtkit_wake(dcp->rtk); - if (ret) - return dev_err_probe(dev, ret, - "Failed to boot RTKit: %d\n", ret); + if (ret) { + ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); + goto err_piodma; + } + return 0; + +err_piodma: + dcp_release_piodma_iommu_dev(dcp); return ret; } @@ -2098,11 +2109,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->shmem) iomfb_shutdown(dcp); - if (dcp->piodma) { - dcp->iommu_dom = NULL; - of_platform_device_destroy(&dcp->piodma->dev, NULL); - dcp->piodma = NULL; - } + dcp_release_piodma_iommu_dev(dcp); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { cancel_work_sync(&dcp->bl_register_wq); From 5e7df8e492e6b1ead94e57d10870be450982331b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:16:49 +1000 Subject: [PATCH 0432/1579] drm: apple: look up display register windows by reg-names The number of display windows differs between DCP instances. Count the disp-* names and fetch each resource by name so unrelated registers do not shift the bandwidth window indexes. Retain the positional lookup for device trees without reg-names. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 70 +++++++++++++------------------------ 1 file changed, 24 insertions(+), 46 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 061f68d865437..53bc040a24942 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1741,67 +1741,45 @@ static int dcp_get_bw_doorbell_reg(struct apple_dcp *dcp, u32 expected) return ret; } -/* - * Count the "disp-*" entries in reg-names. - * - * The positional form below assumes the reg layout is [coproc, disp-0 ...], - * i.e. exactly one non-disp register, and derives the count as - * num_resources - 1. DCPs with H17-generation firmware list a trailing - * "iop-vbar" register after the display apertures, which would inflate that - * count, bind the last disp_registers[] entry to iop-vbar and make the - * "apple,bw-scratch" disp_reg index check reject a valid device tree. - * - * Returns a negative errno when reg-names is absent, so callers fall back. - */ -static int dcp_count_disp_regs(struct device *dev) -{ - int n = of_property_count_strings(dev->of_node, "reg-names"); - const char *name; - int i, count = 0; - - if (n <= 0) - return -EINVAL; - - for (i = 0; i < n; ++i) { - if (of_property_read_string_index(dev->of_node, "reg-names", i, - &name)) - return -EINVAL; - if (!strncmp(name, "disp-", 5)) - count++; - } - - return count ? count : -EINVAL; -} - static int dcp_get_disp_regs(struct apple_dcp *dcp) { struct platform_device *pdev = to_platform_device(dcp->dev); - int count = dcp_count_disp_regs(dcp->dev); - bool by_name = count > 0; + int count = 0; int i, ret; + char name[16]; + const char *reg_name; + struct resource *res; + + if (of_property_present(dcp->dev->of_node, "reg-names")) { + ret = of_property_count_strings(dcp->dev->of_node, "reg-names"); + if (ret < 0) + return ret; - /* Device trees without reg-names keep the historical positional form. */ - if (!by_name) + for (i = 0; i < ret; i++) { + if (of_property_read_string_index(dcp->dev->of_node, + "reg-names", i, ®_name)) + return -EINVAL; + if (!strncmp(reg_name, "disp-", 5)) + count++; + } + } else { count = pdev->num_resources - 1; + } if (count <= 0 || count > MAX_DISP_REGISTERS) return -EINVAL; for (i = 0; i < count; ++i) { - if (by_name) { - char name[8]; - + if (of_property_present(dcp->dev->of_node, "reg-names")) { snprintf(name, sizeof(name), "disp-%d", i); - dcp->disp_registers[i] = platform_get_resource_byname( - pdev, IORESOURCE_MEM, name); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + name); } else { - dcp->disp_registers[i] = platform_get_resource( - pdev, IORESOURCE_MEM, 1 + i); + res = platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); } - if (!dcp->disp_registers[i]) { - dev_err(dcp->dev, "missing display register %d\n", i); + if (!res) return -EINVAL; - } + dcp->disp_registers[i] = res; } /* load pmgr bandwidth scratch resource and offset */ From 3d4229405bba7f2d94ce0eb36d0daf3cbe88bafd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:22:27 +1000 Subject: [PATCH 0433/1579] drm: apple: map firmware-provided RTKit buffers write-combined The DCP writes its RTKit buffers without CPU cache maintenance. A write-back mapping can read stale crash-log contents after the firmware has filled the buffer. Use a write-combined mapping for firmware-provided physical pages. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 53bc040a24942..1e758bb1267ce 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -951,8 +951,8 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) return ret; } - // TODO: verify phy_addr, cache attribute - bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WB); + /* Firmware writes without CPU cache maintenance. */ + bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WC); if (!bfr->buffer) return -ENOMEM; From c07d044a03117705721c78500bc36489ea34e6c4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:28:52 +1000 Subject: [PATCH 0434/1579] soc: apple: rtkit: recognise an initial crash log in adopted sessions Map the first crash-buffer request and validate a complete public CLHE header before decoding it. A fresh session's zeroed buffer remains an ordinary allocation request. An inherited session has already negotiated its buffer: mapping failure or an unreadable header must still notify the client of the crash, with an empty log when necessary. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit-crashlog.c | 14 ++-------- drivers/soc/apple/rtkit-internal.h | 11 ++++++++ drivers/soc/apple/rtkit.c | 43 ++++++++++++++++-------------- 3 files changed, 36 insertions(+), 32 deletions(-) diff --git a/drivers/soc/apple/rtkit-crashlog.c b/drivers/soc/apple/rtkit-crashlog.c index 8319e365110b8..5defa12862b15 100644 --- a/drivers/soc/apple/rtkit-crashlog.c +++ b/drivers/soc/apple/rtkit-crashlog.c @@ -8,7 +8,6 @@ #define FOURCC(a, b, c, d) \ (((u32)(a) << 24) | ((u32)(b) << 16) | ((u32)(c) << 8) | ((u32)(d))) -#define APPLE_RTKIT_CRASHLOG_HEADER FOURCC('C', 'L', 'H', 'E') #define APPLE_RTKIT_CRASHLOG_STR FOURCC('C', 's', 't', 'r') #define APPLE_RTKIT_CRASHLOG_VERSION FOURCC('C', 'v', 'e', 'r') #define APPLE_RTKIT_CRASHLOG_MBOX FOURCC('C', 'm', 'b', 'x') @@ -25,15 +24,6 @@ #define PSR_MODE_MASK 0x0000000f #endif -struct apple_rtkit_crashlog_header { - u32 fourcc; - u32 version; - u32 size; - u32 flags; - u8 _unk[16]; -}; -static_assert(sizeof(struct apple_rtkit_crashlog_header) == 0x20); - struct apple_rtkit_crashlog_mbox_entry { u64 msg0; u64 msg1; @@ -190,7 +180,7 @@ void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) struct apple_rtkit_crashlog_header header; memcpy(&header, bfr, sizeof(header)); - if (header.fourcc != APPLE_RTKIT_CRASHLOG_HEADER) { + if (header.fourcc != APPLE_RTKIT_CRASHLOG_HEADER_FOURCC) { dev_warn(rtk->dev, "RTKit: Expected crashlog header but got %x", header.fourcc); return; @@ -210,7 +200,7 @@ void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) memcpy(§ion_size, bfr + offset + 12, 4); switch (section_fourcc) { - case APPLE_RTKIT_CRASHLOG_HEADER: + case APPLE_RTKIT_CRASHLOG_HEADER_FOURCC: dev_dbg(rtk->dev, "RTKit: End of crashlog reached"); return; case APPLE_RTKIT_CRASHLOG_STR: diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 56d7fea01fade..927f33f1642e6 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -21,6 +21,17 @@ #define APPLE_RTKIT_APP_ENDPOINT_START 0x20 #define APPLE_RTKIT_MAX_ENDPOINTS 0x100 +#define APPLE_RTKIT_CRASHLOG_HEADER_FOURCC 0x434c4845 + +struct apple_rtkit_crashlog_header { + u32 fourcc; + u32 version; + u32 size; + u32 flags; + u8 _unk[16]; +}; + +static_assert(sizeof(struct apple_rtkit_crashlog_header) == 0x20); struct apple_rtkit { void *cookie; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 0797bfbd3a024..74921d4c6e068 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -390,6 +390,7 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg); u8 *bfr __free(kfree) = NULL; + struct apple_rtkit_crashlog_header header; if (type != APPLE_RTKIT_CRASHLOG_CRASH) { dev_warn(rtk->dev, "RTKit: Unknown crashlog message: %llx\n", @@ -397,27 +398,19 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) return; } - if (rtk->crashlog_inherited && !rtk->crashlog_buffer.size) { - /* - * The previous owner negotiated the crashlog buffer, so this - * session never learned its geometry. A CRASH message on an - * inherited session is therefore a crash notification, not a - * buffer request, and must not be answered as one. - */ - dev_err(rtk->dev, - "RTKit: adopted co-processor has crashed (crashlog at %#llx, %#llx bytes, not mapped)\n", - (u64)FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg), - (u64)FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12); - rtk->crashed = true; - if (rtk->ops->crashed) - rtk->ops->crashed(rtk->cookie, NULL, 0); - return; - } - if (!rtk->crashlog_buffer.size) { - apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, - APPLE_RTKIT_EP_CRASHLOG, msg); - return; + if (apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, + APPLE_RTKIT_EP_CRASHLOG, msg)) + goto unmapped; + if (rtk->crashlog_buffer.size < sizeof(header)) + goto unmapped; + + apple_rtkit_memcpy(rtk, &header, &rtk->crashlog_buffer, 0, + sizeof(header)); + if (header.fourcc != APPLE_RTKIT_CRASHLOG_HEADER_FOURCC || + header.size < sizeof(header) || + header.size > rtk->crashlog_buffer.size) + goto unmapped; } dev_err(rtk->dev, "RTKit: co-processor has crashed\n"); @@ -440,6 +433,16 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) rtk->crashed = true; if (rtk->ops->crashed) rtk->ops->crashed(rtk->cookie, bfr, bfr ? rtk->crashlog_buffer.size : 0); + return; + +unmapped: + /* An inherited session already negotiated its crash buffer. */ + if (!rtk->crashlog_inherited) + return; + dev_err(rtk->dev, "RTKit: adopted co-processor crashed without a readable log\n"); + rtk->crashed = true; + if (rtk->ops->crashed) + rtk->ops->crashed(rtk->cookie, NULL, 0); } static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg) From 1ff8bb4e3bd81e9487b93a02642ed22dea52dce8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:36:09 +1000 Subject: [PATCH 0435/1579] drm: apple: afk: derive the ring header stride from INITRB AFK rings describe their payload size in the first word. Derive the three header blocks from the INITRB total instead of assuming 64 bytes per block. Keep the offset and size fields wide enough for the full wire range, and align queue entries to the derived stride. Synthetic KUnit cases cover the older and larger ring headers and malformed layouts (AFK-1). Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 10 ++++++ drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/afk.c | 55 +++++++++++++++----------------- drivers/gpu/drm/apple/afk.h | 49 ++++++++++++++++------------ drivers/gpu/drm/apple/afk_test.c | 44 +++++++++++++++++++++++++ drivers/gpu/drm/apple/trace.h | 4 +-- 6 files changed, 110 insertions(+), 53 deletions(-) create mode 100644 drivers/gpu/drm/apple/afk_test.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index ecf7afd76ceda..c4eff8fd0c5b3 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -28,3 +28,13 @@ config DRM_APPLE_DEBUG bool "Enable additional driver debugging" depends on DRM_APPLE depends on EXPERT # only for developers + +config DRM_APPLE_KUNIT_TEST + tristate "KUnit tests for Apple display controller" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Build tests for the Apple display driver's AFK ring parser. + The tests cover the header strides used by older firmware and + larger rings, as well as malformed ring size combinations. + This option is intended for developer test kernels. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index c511c53c5447c..90915b49e96c5 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -18,3 +18,4 @@ appledrm-y += plane.o appledrm-$(CONFIG_TRACING) += trace.o obj-$(CONFIG_DRM_APPLE) += appledrm.o +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk_test.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 4a4215e0af2df..daa27e1ad7238 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -152,7 +152,8 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, u32 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; u32 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; u16 tag = FIELD_GET(INITRB_TAG, message); - u32 bufsz, hdrsz, end, block; + u32 bufsz, end, stride; + u8 *hdr; if (tag != ep->bfr_tag) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: expected tag 0x%x but got 0x%x\n", @@ -166,10 +167,10 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, return; } - if (!ep->bfr || size < sizeof(__le32) || base >= ep->bfr_size) { + if (!ep->bfr || base >= ep->bfr_size || size < sizeof(u32)) { dev_err(ep->dcp->dev, - "AFK[ep:%02x]: requested base 0x%x >= max size 0x%lx\n", - ep->endpoint, base, ep->bfr_size); + "AFK[ep:%02x]: invalid ring base 0x%x or size 0x%x\n", + ep->endpoint, base, size); return; } @@ -181,23 +182,21 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, return; } - bfr->hdr = ep->bfr + base; + hdr = (u8 *)ep->bfr + base; dma_rmb(); - bufsz = le32_to_cpu(READ_ONCE(*(__le32 *)bfr->hdr)); - hdrsz = size - bufsz; - block = hdrsz / AFK_RB_BLOCKS; - if (bufsz >= size || hdrsz % AFK_RB_BLOCKS || - block < AFK_RB_BLOCK_MIN || !is_power_of_2(block) || - !IS_ALIGNED(bufsz, block) || bufsz < 2 * block) { + bufsz = le32_to_cpu(READ_ONCE(*(__le32 *)hdr)); + if (afk_ring_stride(size, bufsz, &stride)) { dev_err(ep->dcp->dev, - "AFK[ep:%02x]: ring bufsz %#x with size %#x gives no valid header (base %#x, msg %#llx)\n", - ep->endpoint, bufsz, size, base, message); + "AFK[ep:%02x]: invalid ring payload size 0x%x of 0x%x\n", + ep->endpoint, bufsz, size); return; } - bfr->block = block; - bfr->buf = (u8 *)bfr->hdr + hdrsz; + bfr->rptr = (__le32 *)(hdr + stride); + bfr->wptr = (__le32 *)(hdr + 2 * stride); + bfr->buf = hdr + 3 * stride; bfr->bufsz = bufsz; + bfr->stride = stride; bfr->ready = true; if (ep->rxbfr.ready && ep->txbfr.ready) @@ -683,8 +682,8 @@ static bool afk_recv(struct apple_dcp_afkep *ep) return false; } - rptr = le32_to_cpu(*afk_rb_rptr(&ep->rxbfr)); - wptr = le32_to_cpu(*afk_rb_wptr(&ep->rxbfr)); + rptr = le32_to_cpu(*ep->rxbfr.rptr); + wptr = le32_to_cpu(*ep->rxbfr.wptr); trace_afk_recv_rwptr_pre(ep, rptr, wptr); if (rptr == wptr) @@ -729,7 +728,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) return false; } - *afk_rb_rptr(&ep->rxbfr) = cpu_to_le32(rptr); + *ep->rxbfr.rptr = cpu_to_le32(rptr); } if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { @@ -742,15 +741,12 @@ static bool afk_recv(struct apple_dcp_afkep *ep) channel = le32_to_cpu(hdr->channel); type = le32_to_cpu(hdr->type); - rptr = ALIGN(rptr + sizeof(*hdr) + size, ep->rxbfr.block); - if (WARN_ON(rptr > ep->rxbfr.bufsz)) - rptr = 0; - if (rptr == ep->rxbfr.bufsz) - rptr = 0; + rptr = afk_ring_advance(rptr, sizeof(*hdr) + size, + ep->rxbfr.bufsz, ep->rxbfr.stride); dma_mb(); - *afk_rb_rptr(&ep->rxbfr) = cpu_to_le32(rptr); + *ep->rxbfr.rptr = cpu_to_le32(rptr); trace_afk_recv_rwptr_post(ep, rptr, wptr); /* @@ -858,8 +854,8 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, spin_lock_irqsave(&ep->lock, flags); dma_rmb(); - rptr = le32_to_cpu(*afk_rb_rptr(&ep->txbfr)); - wptr = le32_to_cpu(*afk_rb_wptr(&ep->txbfr)); + rptr = le32_to_cpu(*ep->txbfr.rptr); + wptr = le32_to_cpu(*ep->txbfr.wptr); trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; total_size = sizeof(*hdr) + total_epic_size; @@ -972,12 +968,11 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, memcpy(ep->txbfr.buf + wptr, payload, payload_len); wptr += payload_len; - wptr = ALIGN(wptr, ep->txbfr.block); - if (wptr == ep->txbfr.bufsz) - wptr = 0; + wptr = afk_ring_advance(wptr, 0, ep->txbfr.bufsz, + ep->txbfr.stride); trace_afk_send_rwptr_post(ep, rptr, wptr); - *afk_rb_wptr(&ep->txbfr) = cpu_to_le32(wptr); + *ep->txbfr.wptr = cpu_to_le32(wptr); afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_SEND) | FIELD_PREP(SEND_WPTR, wptr)); ret = 0; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 6ba47202e23a9..1e691ec47c85e 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -8,7 +8,9 @@ #define _DRM_APPLE_DCP_AFK_H #include +#include #include +#include #include #include "dcp.h" @@ -71,17 +73,29 @@ struct apple_epic_service_ops { void (*teardown)(struct apple_epic_service *service); }; -/* - * The ring header is three control blocks -- bufsz, rptr, wptr -- each padded - * out to the firmware's control-block size. That block is 0x40 bytes up to - * v13.5 but 0x80 on H17P, making the header 0xc0 or 0x180. The size is - * self-describing: the INITRB message gives the total ring size and the first - * word of block 0 gives the payload size, so the stride is (size - bufsz) / 3 - * and no firmware-version test is needed. Address the fields by stride rather - * than through a fixed struct. - */ -#define AFK_RB_BLOCK_MIN 0x40 -#define AFK_RB_BLOCKS 3 +static inline int afk_ring_stride(u32 total, u32 bufsz, u32 *stride) +{ + u32 header, block; + + if (bufsz >= total) + return -EINVAL; + header = total - bufsz; + block = header / 3; + if (header % 3 || block < 0x40 || block % 0x40 || + bufsz < block || bufsz % block) + return -EINVAL; + + *stride = block; + return 0; +} + +static inline u32 afk_ring_advance(u32 ptr, u32 bytes, u32 bufsz, + u32 stride) +{ + u32 next = roundup(ptr + bytes, stride); + + return next == bufsz ? 0 : next; +} struct afk_qe { #define QE_MAGIC 0x20504f49 // ' POI' @@ -169,20 +183,13 @@ enum epic_subtype { struct afk_ringbuffer { bool ready; - void *hdr; - u32 block; /* control-block stride, 0x40 or 0x80 */ - u32 rptr; + __le32 *rptr; + __le32 *wptr; void *buf; size_t bufsz; + u32 stride; }; -static inline __le32 *afk_rb_field(struct afk_ringbuffer *bfr, unsigned int idx) -{ - return (__le32 *)((u8 *)bfr->hdr + idx * bfr->block); -} - -#define afk_rb_rptr(bfr) afk_rb_field(bfr, 1) -#define afk_rb_wptr(bfr) afk_rb_field(bfr, 2) struct apple_dcp_afkep { struct apple_dcp *dcp; diff --git a/drivers/gpu/drm/apple/afk_test.c b/drivers/gpu/drm/apple/afk_test.c new file mode 100644 index 0000000000000..e58a952642045 --- /dev/null +++ b/drivers/gpu/drm/apple/afk_test.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include + +#include "afk.h" + +static void afk_ring_stride_test(struct kunit *test) +{ + u32 stride; + + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c0, 0x1000, &stride), 0); + KUNIT_EXPECT_EQ(test, stride, 0x40U); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x1180, 0x1000, &stride), 0); + KUNIT_EXPECT_EQ(test, stride, 0x80U); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x1000, 0x1000, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10bd, 0x1000, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c1, 0x1000, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0xc0, 0, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c3, 0x1000, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c0, 0x1001, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x11c0, 0x1040, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x1180, 0x1000, &stride), 0); + KUNIT_EXPECT_EQ(test, afk_ring_advance(0xf80, 0x80, 0x1000, stride), 0U); + KUNIT_EXPECT_EQ(test, afk_ring_advance(0xe80, 0x80, 0x1000, stride), + 0xf00U); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c0, 0x1000, &stride), 0); + KUNIT_EXPECT_EQ(test, afk_ring_advance(0xfc0, 0x40, 0x1000, stride), 0U); +} + +static struct kunit_case afk_ring_cases[] = { + KUNIT_CASE(afk_ring_stride_test), + {} +}; + +static struct kunit_suite afk_ring_suite = { + .name = "apple-afk-ring", + .test_cases = afk_ring_cases, +}; + +kunit_test_suite(afk_ring_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index c0acac6ee690a..d3c5106e5d66d 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -109,11 +109,11 @@ TRACE_EVENT(dcp_send_msg, show_dcp_endpoint(__entry->endpoint), __entry->message)); TRACE_EVENT( - afk_getbuf, TP_PROTO(struct apple_dcp_afkep *ep, u16 size, u16 tag), + afk_getbuf, TP_PROTO(struct apple_dcp_afkep *ep, u32 size, u16 tag), TP_ARGS(ep, size, tag), TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) - __field(u8, endpoint) __field(u16, size) + __field(u8, endpoint) __field(u32, size) __field(u16, tag)), TP_fast_assign(__assign_str(devname); From 8389f7685bd70cddec2a515c2c73507bdc83ad45 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:40:44 +1000 Subject: [PATCH 0436/1579] drm: apple: drop messages for endpoints without a handler Firmware may send on an endpoint that this DCP instance did not start. Log the endpoint with rate limiting and drop the message instead of calling AFK through a NULL pointer or WARNing. Refuse to start the DCP when RTKit is absent (RTK-2). Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 1e758bb1267ce..708d8cd0f8c03 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -848,6 +848,7 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) { struct apple_dcp *dcp = cookie; + struct apple_dcp_afkep *ep; trace_dcp_recv_msg(dcp, endpoint, message); @@ -867,23 +868,33 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) case IOMFB_ENDPOINT: return iomfb_recv_msg(dcp, message); case AV_ENDPOINT: - afk_receive_message(dcp->avep, message); - return; + ep = dcp->avep; + break; case SYSTEM_ENDPOINT: - afk_receive_message(dcp->systemep, message); - return; + ep = dcp->systemep; + break; case DISP0_ENDPOINT: - afk_receive_message(dcp->ibootep, message); - return; + ep = dcp->ibootep; + break; case DPAVSERV_ENDPOINT: - afk_receive_message(dcp->dcpavservep, message); - return; + ep = dcp->dcpavservep; + break; case DPTX_ENDPOINT: - afk_receive_message(dcp->dptxep, message); - return; + ep = dcp->dptxep; + break; default: - WARN(endpoint, "unknown DCP endpoint %hhu\n", endpoint); + ep = NULL; + break; + } + + if (!ep) { + dev_warn_ratelimited(dcp->dev, + "dropping message for unhandled endpoint %#x\n", + endpoint); + return; } + + afk_receive_message(ep, message); } static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_size) @@ -1307,6 +1318,9 @@ int dcp_start(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + if (!dcp->rtk) + return -ENODEV; + init_completion(&dcp->start_done); /* From 98d2055c1e1e4d29b45011baa84cd3b0cbe4bd6b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 6 Oct 2026 21:29:18 +1100 Subject: [PATCH 0437/1579] drm: apple: duplicate cached scanout geometry with plane state An inactive CRTC update can duplicate plane state without invoking the plane update hook. Copy its cached rectangles and IOVA along with the surface descriptor so a later activation presents the same scanout. DRM's atomic plane helpers can skip updates on inactive CRTCs. Keep the cached scanout fields consistent with the duplicated framebuffer state. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/plane.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index 51a6c2dbd7a39..6a4cea28db217 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -333,6 +333,9 @@ apple_plane_duplicate_state(struct drm_plane *plane) __drm_atomic_helper_plane_duplicate_state(plane, &apple_plane_state->base); apple_plane_state->surf = old_apple_plane_state->surf; + apple_plane_state->src_rect = old_apple_plane_state->src_rect; + apple_plane_state->dst_rect = old_apple_plane_state->dst_rect; + apple_plane_state->iova = old_apple_plane_state->iova; return &apple_plane_state->base; } From c529eadaf40a7a176b34645d5aec73c3a92a77d3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:47:31 +1000 Subject: [PATCH 0438/1579] drm: apple: retain displaced framebuffers until confirmed completion Allocate and reference displaced scanout during plane preparation, where allocation failure can still abort the atomic commit. Transfer prepared entries before disabling or updating CRTCs, including inactive commits. Cleanup and state destruction release only unconsumed reservations. Retain transferred references until a successful successor swap completes or a successful power response confirms scanout stop on existing firmware variants. Rejected swap starts never arm retirement; rejected, missing or malformed power responses, timeouts and late replies cannot release live scanout. H17P stop semantics remain unqualified, so its power replies do not retire references. Initialize command output to an invalid status and validate legacy reply lengths before dispatching responses. Initialize retention ownership once at probe and reject rebind while a previous session's references remain. This preserves quarantine ownership instead of losing list links after a timed-out shutdown. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 26 ++- drivers/gpu/drm/apple/dcp-internal.h | 14 +- drivers/gpu/drm/apple/dcp.c | 86 +++++++++- drivers/gpu/drm/apple/dcp.h | 6 + drivers/gpu/drm/apple/iomfb.c | 11 ++ drivers/gpu/drm/apple/iomfb_template.c | 226 ++++++++----------------- drivers/gpu/drm/apple/plane.c | 63 ++++++- drivers/gpu/drm/apple/plane.h | 3 + 8 files changed, 262 insertions(+), 173 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 4c6fa62321a25..85c76f3facd7d 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -122,10 +122,11 @@ static void apple_crtc_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *state) { struct drm_crtc_state *crtc_state; + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); if (crtc_state->active_changed && !crtc_state->active) { - struct apple_crtc *apple_crtc = to_apple_crtc(crtc); dcp_poweroff(apple_crtc->dcp); } @@ -230,8 +231,29 @@ static const struct drm_mode_config_funcs apple_mode_config_funcs = { .fb_create = drm_gem_fb_create, }; +static void apple_atomic_commit_tail(struct drm_atomic_state *state) +{ + struct drm_plane *plane; + struct drm_plane_state *old, *new; + int i; + + /* Retain displaced scanout even when the CRTC stays inactive. */ + for_each_oldnew_plane_in_state(state, plane, old, new, i) { + struct apple_plane_state *apple_state = to_apple_plane_state(new); + struct dcp_fb_reference *entry = apple_state->retirement; + + if (!entry) + continue; + + apple_state->retirement = NULL; + dcp_retain_framebuffer(to_apple_crtc(old->crtc)->dcp, entry); + } + + drm_atomic_helper_commit_tail_rpm(state); +} + static const struct drm_mode_config_helper_funcs apple_mode_config_helpers = { - .atomic_commit_tail = drm_atomic_helper_commit_tail_rpm, + .atomic_commit_tail = apple_atomic_commit_tail, }; static void appledrm_connector_cleanup(struct drm_connector *connector) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 883c70399fc05..3d45d07dacf07 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -94,6 +94,8 @@ struct dcp_channel { dcp_callback_t callbacks[DCP_MAX_CALL_DEPTH]; void *cookies[DCP_MAX_CALL_DEPTH]; void *output[DCP_MAX_CALL_DEPTH]; + u32 in_len[DCP_MAX_CALL_DEPTH]; + u32 out_len[DCP_MAX_CALL_DEPTH]; u16 end[DCP_MAX_CALL_DEPTH]; /* Current depth of the call stack. Less than DCP_MAX_CALL_DEPTH */ @@ -274,16 +276,12 @@ struct apple_dcp { /* Workqueue for sending vblank events when a dcp swap is not possible */ struct work_struct vblank_wq; - /* List of referenced drm_framebuffers which can be unreferenced - * on the next successfully completed swap. + /* List of referenced framebuffers pending a completed replacement swap. + * The commit and RTKit work queues share it under this lock. */ struct list_head swapped_out_fbs; - /* - * Protects swapped_out_fbs, which is appended to from the DRM atomic - * commit (dcp_flush -> .atomic_flush) and armed/drained from the RTKit - * workqueue that runs the DCP callbacks. - */ - spinlock_t swapped_out_lock; + /* Protects swapped_out_fbs. */ + struct mutex swapped_out_fbs_lock; struct dcp_brightness brightness; /* Workqueue for updating the initial brightness */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 708d8cd0f8c03..b28b201d64924 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1615,6 +1615,77 @@ static void dcp_release_piodma_iommu_dev(struct apple_dcp *dcp) dcp->piodma_created = false; } +void dcp_retain_framebuffer(struct platform_device *pdev, + struct dcp_fb_reference *entry) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_add_tail(&entry->head, &dcp->swapped_out_fbs); + mutex_unlock(&dcp->swapped_out_fbs_lock); +} + +void dcp_arm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id) +{ + struct dcp_fb_reference *entry; + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_for_each_entry(entry, &dcp->swapped_out_fbs, head) { + if (entry->armed) + continue; + entry->swap_id = swap_id; + entry->armed = true; + } + mutex_unlock(&dcp->swapped_out_fbs_lock); +} + +void dcp_unarm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id) +{ + struct dcp_fb_reference *entry; + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_for_each_entry(entry, &dcp->swapped_out_fbs, head) + if (entry->armed && entry->swap_id == swap_id) + entry->armed = false; + mutex_unlock(&dcp->swapped_out_fbs_lock); +} + +void dcp_release_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id) +{ + struct dcp_fb_reference *entry, *tmp; + LIST_HEAD(completed); + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_for_each_entry_safe(entry, tmp, &dcp->swapped_out_fbs, head) { + if (!entry->armed || entry->swap_id != swap_id) + continue; + list_move_tail(&entry->head, &completed); + } + mutex_unlock(&dcp->swapped_out_fbs_lock); + + list_for_each_entry_safe(entry, tmp, &completed, head) { + list_del(&entry->head); + drm_framebuffer_put(entry->fb); + kfree(entry); + } +} + +void dcp_release_all_retained_framebuffers(struct apple_dcp *dcp) +{ + struct dcp_fb_reference *entry, *tmp; + LIST_HEAD(completed); + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_splice_init(&dcp->swapped_out_fbs, &completed); + mutex_unlock(&dcp->swapped_out_fbs_lock); + + list_for_each_entry_safe(entry, tmp, &completed, head) { + list_del(&entry->head); + drm_framebuffer_put(entry->fb); + kfree(entry); + } +} + static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) { int ret; @@ -1921,6 +1992,14 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) u32 cpu_ctrl; int ret; + /* A timed-out prior session may still scan these mappings. */ + mutex_lock(&dcp->swapped_out_fbs_lock); + ret = list_empty(&dcp->swapped_out_fbs) ? 0 : -EBUSY; + mutex_unlock(&dcp->swapped_out_fbs_lock); + if (ret) + return dev_err_probe(dev, ret, + "Previous scanout has not been stopped\n"); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); if (ret) return ret; @@ -2023,10 +2102,6 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); - dcp->swapped_out_fbs = - (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); - spin_lock_init(&dcp->swapped_out_lock); - if (!dcp->hw.adopt_live_session) { cpu_ctrl = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); @@ -2147,6 +2222,9 @@ static int dcp_platform_probe(struct platform_device *pdev) if (!dcp) return -ENOMEM; + INIT_LIST_HEAD(&dcp->swapped_out_fbs); + mutex_init(&dcp->swapped_out_fbs_lock); + dcp->fw_compat = fw_compat; dcp->dev = dev; dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index c500c1b8a4434..e4dc34ef8256a 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -52,6 +52,12 @@ void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, int dcp_start(struct platform_device *pdev); int dcp_wait_ready(struct platform_device *pdev, u64 timeout); void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state); +void dcp_retain_framebuffer(struct platform_device *pdev, + struct dcp_fb_reference *entry); +void dcp_arm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id); +void dcp_unarm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id); +void dcp_release_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id); +void dcp_release_all_retained_framebuffers(struct apple_dcp *dcp); bool dcp_is_initialized(struct platform_device *pdev); void apple_crtc_vblank(struct apple_crtc *apple); void dcp_drm_crtc_vblank(struct apple_crtc *crtc); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 16f87163524d3..d034f2c83705b 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -193,12 +193,17 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca if (in_len > 0) memcpy(out_data, data, in_len); + /* An unwritten status must not look like a successful response. */ + if (out_len) + memset(out_data + in_len, 0xff, out_len); trace_iomfb_push(dcp, call, context, offset, depth); ch->callbacks[depth] = cb; ch->cookies[depth] = cookie; ch->output[depth] = out + sizeof(header) + in_len; + ch->in_len[depth] = in_len; + ch->out_len[depth] = out_len; ch->end[depth] = offset + ALIGN(data_len, DCP_PACKET_ALIGNMENT); dcp_send_message(dcp, IOMFB_ENDPOINT, @@ -405,6 +410,12 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, out = ch->output[ch->depth]; ch->output[ch->depth] = NULL; } else { + if (header->in_len != ch->in_len[ch->depth] || + header->out_len != ch->out_len[ch->depth]) { + dev_err(dcp->dev, "invalid command response lengths\n"); + dcp->crashed = true; + return; + } out = data + sizeof(*header) + header->in_len; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index bc9fb5041e26e..98c6c5824438c 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -41,6 +41,7 @@ static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), flags1) == 0x40); struct dcp_wait_cookie { struct kref refcount; struct completion done; + u32 status; }; static void release_wait_cookie(struct kref *ref) @@ -219,108 +220,15 @@ static u32 dcpep_cb_zero(struct apple_dcp *dcp) return 0; } -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * Bind every framebuffer displaced by the commit that produced this swap to - * that swap's id. - * - * The id only exists once the firmware answers swap_start, after the atomic - * commit has queued its entries, so they are queued unarmed and stamped here. - * Swap-start requests are serialized by the DCP command stack. - */ -static void dcp_arm_swapped_out_fbs(struct apple_dcp *dcp, u32 swap_id) -{ - struct dcp_fb_reference *entry; - unsigned long flags; - - spin_lock_irqsave(&dcp->swapped_out_lock, flags); - list_for_each_entry(entry, &dcp->swapped_out_fbs, head) { - if (entry->armed) - continue; - entry->swap_id = swap_id; - entry->armed = true; - } - spin_unlock_irqrestore(&dcp->swapped_out_lock, flags); -} - -/* - * Drop framebuffers whose unbinding swap has completed. - * - * Entries must be keyed to the swap that unbinds them, rather than the last - * swap that happened to have completed when the atomic commit ran. A later - * successful completion also retires entries whose own completion was lost; - * the signed difference keeps that comparison valid across a 32-bit wrap. - */ -static void dcp_release_swapped_out_fbs(struct apple_dcp *dcp) -{ - struct dcp_fb_reference *entry, *tmp; - unsigned long flags; - LIST_HEAD(done); - - /* - * last_swap_id / have_swap_complete are only ever touched by the RTKit - * worker, which is also the only caller of this function, so the lock - * is here for the list alone. - */ - spin_lock_irqsave(&dcp->swapped_out_lock, flags); - while (!list_empty(&dcp->swapped_out_fbs)) { - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - - /* Newest entries; their swap has not been started yet. */ - if (!entry->armed) - break; - /* last_swap_id is 0 from kzalloc until a swap really lands. */ - if (!dcp->have_swap_complete) - break; - /* Hold the old framebuffer for one additional completed swap. */ - if ((s32)(dcp->last_swap_id - entry->swap_id) <= 0) - break; - - list_move_tail(&entry->head, &done); - } - spin_unlock_irqrestore(&dcp->swapped_out_lock, flags); - - /* drm_framebuffer_put() can sleep, so never call it under the lock. */ - list_for_each_entry_safe(entry, tmp, &done, head) { - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } -} -#else -static void dcp_arm_swapped_out_fbs(struct apple_dcp *dcp, u32 swap_id) { } - -static void dcp_release_swapped_out_fbs(struct apple_dcp *dcp) -{ - while (!list_empty(&dcp->swapped_out_fbs)) { - struct dcp_fb_reference *entry; - - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - if (entry->swap_id == dcp->last_swap_id) - break; - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } -} -#endif - static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct DCP_FW_NAME(dc_swap_complete_resp) *resp) { ktime_t now = ktime_get(); trace_iomfb_swap_complete(dcp, resp->swap_id); dcp->last_swap_id = resp->swap_id; + dcp_release_retained_framebuffers(dcp, resp->swap_id); dcp_drm_crtc_page_flip(dcp, now); -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - dcp->have_swap_complete = true; - dcp_release_swapped_out_fbs(dcp); -#endif if (dcp->crc_enabled) { u32 crc32 = 0; drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); @@ -1220,6 +1128,7 @@ static u64 dcpep_cb_get_time(struct apple_dcp *dcp) struct dcp_swap_cookie { struct kref refcount; struct completion done; + u32 status; u32 swap_id; }; @@ -1299,33 +1208,42 @@ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) { } static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; + struct dcp_swap_cookie *info = cookie; + u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; + u32 status = resp ? resp->ret : ~0U; - if (cookie) { - struct dcp_swap_cookie *info = cookie; + if (status) { + dev_err(dcp->dev, "swap_clear failed! status %u\n", status); + dcp_unarm_retained_framebuffers(dcp, swap_id); + dcp_drm_crtc_vblank(dcp->crtc); + } + if (info) { + WRITE_ONCE(info->status, status); complete(&info->done); kref_put(&info->refcount, release_swap_cookie); } - - if (resp->ret) { - dev_err(dcp->dev, "swap_clear failed! status %u\n", resp->ret); - dcp_drm_crtc_vblank(dcp->crtc); - return; - } - - dcp_release_swapped_out_fbs(dcp); } static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; + struct dcp_swap_cookie *info = cookie; + + if (!resp || resp->ret) { + if (info) { + WRITE_ONCE(info->status, resp ? resp->ret : ~0U); + complete(&info->done); + kref_put(&info->refcount, release_swap_cookie); + } + dcp_drm_crtc_vblank(dcp->crtc); + return; + } DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; - dcp_arm_swapped_out_fbs(dcp, resp->swap_id); + dcp_arm_retained_framebuffers(dcp, resp->swap_id); - if (cookie) { - struct dcp_swap_cookie *info = cookie; + if (info) info->swap_id = resp->swap_id; - } dcp_h17p_fix_swap_surfaces(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); @@ -1376,6 +1294,7 @@ void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) if (!cookie) return; + cookie->status = ~0U; init_completion(&cookie->done); kref_init(&cookie->refcount); /* increase refcount to ensure the receiver has a reference */ @@ -1425,14 +1344,26 @@ void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) static void complete_set_powerstate(struct apple_dcp *dcp, void *out, void *cookie) { + struct dcp_set_power_state_resp *resp = out; struct dcp_wait_cookie *wait = cookie; if (wait) { + WRITE_ONCE(wait->status, resp ? resp->ret : ~0U); complete(&wait->done); kref_put(&wait->refcount, release_wait_cookie); } } +static bool dcp_power_stop_confirmed(struct dcp_wait_cookie *cookie) +{ + /* H17P stop semantics still require hardware qualification. */ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + return false; +#else + return READ_ONCE(cookie->status) == 0; +#endif +} + static void last_client_closed_poff(struct apple_dcp *dcp, void *out, void *cookie) { struct DCP_FW_NAME(dcp_set_power_state_req) power_req = { @@ -1465,6 +1396,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; + cookie->status = ~0U; init_completion(&cookie->done); kref_init(&cookie->refcount); /* increase refcount to ensure the receiver has a reference */ @@ -1506,6 +1438,8 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(50)); swap_id = cookie->swap_id; + if (READ_ONCE(cookie->status)) + ret = 0; kref_put(&cookie->refcount, release_swap_cookie); if (ret <= 0) { dcp->crashed = true; @@ -1517,6 +1451,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); if (!poff_cookie) return; + poff_cookie->status = ~0U; init_completion(&poff_cookie->done); kref_init(&poff_cookie->refcount); /* increase refcount to ensure the receiver has a reference */ @@ -1535,10 +1470,15 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) if (ret == 0) dev_warn(dcp->dev, "setPowerState(0) timeout %u ms\n", 1000); - else if (ret > 0) + else if (ret > 0) { dev_dbg(dcp->dev, "setPowerState(0) finished with %d ms to spare", jiffies_to_msecs(ret)); + if (dcp_power_stop_confirmed(poff_cookie)) + dcp_release_all_retained_framebuffers(dcp); + else + dev_warn(dcp->dev, "scanout stop was not confirmed\n"); + } kref_put(&poff_cookie->refcount, release_wait_cookie); @@ -1573,6 +1513,7 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; + cookie->status = ~0U; init_completion(&cookie->done); kref_init(&cookie->refcount); /* increase refcount to ensure the receiver has a reference */ @@ -1765,24 +1706,29 @@ TRAMPOLINE_OUT(trampoline_create_backlight_service, dcpep_cb_create_backlight_se static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; + u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; if (resp->ret) { dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); + dcp_unarm_retained_framebuffers(dcp, swap_id); dcp_drm_crtc_vblank(dcp->crtc); return; } dcp->swap_start = ktime_get(); dcp->swap_submit_timestamp = arch_timer_read_counter(); - - dcp_release_swapped_out_fbs(dcp); } static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; + if (!resp || resp->ret) { + dev_err(dcp->dev, "swap_start was rejected\n"); + dcp_drm_crtc_vblank(dcp->crtc); + return; + } DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; - dcp_arm_swapped_out_fbs(dcp, resp->swap_id); + dcp_arm_retained_framebuffers(dcp, resp->swap_id); trace_iomfb_swap_submit(dcp, resp->swap_id); dcp_h17p_fix_swap_surfaces(dcp); @@ -1900,6 +1846,7 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, return -ENOMEM; } + cookie->status = ~0U; init_completion(&cookie->done); kref_init(&cookie->refcount); /* increase refcount to ensure the receiver has a reference */ @@ -2014,44 +1961,6 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru l = apl_plane->iomfb_surf; req->swap.swap_enabled |= BIT(l); - if (old_state->fb && new_state->fb != old_state->fb) { - /* - * Race condition between a framebuffer unbind getting - * swapped out and GEM unreferencing a framebuffer. If - * we lose the race, the display gets IOVA faults and - * the DCP crashes. We need to extend the lifetime of - * the drm_framebuffer (and hence the GEM object) until - * after we get a swap complete for the swap unbinding - * it. - */ - struct dcp_fb_reference *entry = - kzalloc(sizeof(*entry), GFP_KERNEL); - if (entry) { -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - unsigned long flags; - - entry->fb = old_state->fb; - /* - * Left unarmed: the id of the swap that - * unbinds this framebuffer is not known until - * the firmware answers swap_start. - */ - spin_lock_irqsave(&dcp->swapped_out_lock, - flags); - list_add_tail(&entry->head, - &dcp->swapped_out_fbs); - spin_unlock_irqrestore(&dcp->swapped_out_lock, - flags); -#else - entry->fb = old_state->fb; - entry->swap_id = dcp->last_swap_id; - list_add_tail(&entry->head, - &dcp->swapped_out_fbs); -#endif - } - drm_framebuffer_get(old_state->fb); - } - if (!new_state->fb || !new_state->visible) { continue; } @@ -2331,6 +2240,21 @@ void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp) struct DCP_FW_NAME(dcp_set_power_state_req) req = { /* defaults are ok */ }; + struct dcp_wait_cookie *cookie; + int ret; - dcp_set_power_state(dcp, DCP_POWER_OOB, &req, NULL, NULL); + cookie = kzalloc_obj(*cookie); + if (!cookie) + return; + cookie->status = ~0U; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + kref_get(&cookie->refcount); + dcp_set_power_state(dcp, DCP_POWER_OOB, &req, complete_set_powerstate, cookie); + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); + if (ret > 0 && dcp_power_stop_confirmed(cookie)) + dcp_release_all_retained_framebuffers(dcp); + else + dev_warn(dcp->dev, "shutdown scanout stop was not confirmed\n"); + kref_put(&cookie->refcount, release_wait_cookie); } diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index 6a4cea28db217..3bac8687e83de 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -5,6 +5,7 @@ #include "plane.h" +#include "dcp-internal.h" #include "iomfb_internal.h" #include @@ -13,6 +14,7 @@ #include #include #include +#include #include #include @@ -292,13 +294,56 @@ static void apple_plane_atomic_update(struct drm_plane *plane, new_state->iova = obj->dma_addr + base->fb->offsets[0]; } +static void apple_plane_cleanup_fb(struct drm_plane *plane, + struct drm_plane_state *state) +{ + struct apple_plane_state *apple_state = to_apple_plane_state(state); + struct dcp_fb_reference *entry = apple_state->retirement; + + if (!entry) + return; + + apple_state->retirement = NULL; + drm_framebuffer_put(entry->fb); + kfree(entry); +} + +static int apple_plane_prepare_fb(struct drm_plane *plane, + struct drm_plane_state *state) +{ + struct drm_plane_state *old; + struct dcp_fb_reference *entry; + int ret; + + ret = drm_gem_plane_helper_prepare_fb(plane, state); + if (ret) + return ret; + + old = drm_atomic_get_old_plane_state(state->state, plane); + if (!old || !old->crtc || !old->fb || old->fb == state->fb) + return 0; + + entry = kzalloc_obj(*entry); + if (!entry) + return -ENOMEM; + + drm_framebuffer_get(old->fb); + entry->fb = old->fb; + to_apple_plane_state(state)->retirement = entry; + return 0; +} + static const struct drm_plane_helper_funcs apple_primary_plane_helper_funcs = { + .prepare_fb = apple_plane_prepare_fb, + .cleanup_fb = apple_plane_cleanup_fb, .atomic_check = apple_plane_atomic_check, .atomic_update = apple_plane_atomic_update, .get_scanout_buffer = drm_fb_dma_get_scanout_buffer, }; static const struct drm_plane_helper_funcs apple_plane_helper_funcs = { + .prepare_fb = apple_plane_prepare_fb, + .cleanup_fb = apple_plane_cleanup_fb, .atomic_check = apple_plane_atomic_check, .atomic_update = apple_plane_atomic_update, }; @@ -307,8 +352,10 @@ static const struct drm_plane_helper_funcs apple_plane_helper_funcs = { static void apple_plane_reset(struct drm_plane *plane) { struct apple_plane_state *state = to_apple_plane_state(plane->state); - if (state) + if (state) { + apple_plane_cleanup_fb(plane, &state->base); __drm_atomic_helper_plane_destroy_state(&state->base); + } kfree(state); plane->state = NULL; @@ -340,19 +387,19 @@ apple_plane_duplicate_state(struct drm_plane *plane) return &apple_plane_state->base; } -// void apple_plane_destroy_state(struct drm_plane *plane, -// struct drm_plane_state *state) -// { -// drm_atomic_helper_plane_destroy_state(plane, state); -// } +static void apple_plane_destroy_state(struct drm_plane *plane, + struct drm_plane_state *state) +{ + apple_plane_cleanup_fb(plane, state); + drm_atomic_helper_plane_destroy_state(plane, state); +} static const struct drm_plane_funcs apple_plane_funcs = { .update_plane = drm_atomic_helper_update_plane, .disable_plane = drm_atomic_helper_disable_plane, .reset = apple_plane_reset, .atomic_duplicate_state = apple_plane_duplicate_state, - // .atomic_destroy_state = apple_plane_destroy_state, - .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, + .atomic_destroy_state = apple_plane_destroy_state, }; /* diff --git a/drivers/gpu/drm/apple/plane.h b/drivers/gpu/drm/apple/plane.h index c5db78144d1f8..298edcd7c561e 100644 --- a/drivers/gpu/drm/apple/plane.h +++ b/drivers/gpu/drm/apple/plane.h @@ -19,12 +19,15 @@ struct apple_plane { #define to_apple_plane(x) container_of(x, struct apple_plane, base) +struct dcp_fb_reference; + struct apple_plane_state { struct drm_plane_state base; struct dcp_surface surf; struct dcp_rect src_rect; struct dcp_rect dst_rect; u64 iova; + struct dcp_fb_reference *retirement; }; #define to_apple_plane_state(x) container_of(x, struct apple_plane_state, base) From c361702757bfccb9183ed79be0c6200fb3722e6f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:52:48 +1000 Subject: [PATCH 0439/1579] iommu/apple-dart: defer publication until locked roots are known The IOMMU core installs firmware-requested mappings before the locked stream is attached. Its root pages are not mapped at that point, so publishing host entries fails with -EIO. Defer that stream during early TLB sync; attach still requires every root and publishes the complete page table (DART-2). Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index b04359eecc5e6..3fd8b9fb5bc18 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -751,7 +751,8 @@ apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx static int apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, - struct apple_dart_stream_map *stream_map) + struct apple_dart_stream_map *stream_map, + bool defer_unknown) { struct apple_dart *dart = stream_map->dart; unsigned long flags; @@ -759,6 +760,20 @@ apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, spin_lock_irqsave(&dart->lock, flags); for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + bool ready = true; + + if (defer_unknown) { + for (idx = 0; idx < cfg->apple_dart_cfg.n_ttbrs; idx++) { + if (!dart->locked_ttbr[sid][idx] || + !dart->locked_owned[sid][idx]) { + ready = false; + break; + } + } + if (!ready) + continue; + } + for (idx = 0; idx < cfg->apple_dart_cfg.n_ttbrs; idx++) { u64 *live = dart->locked_ttbr[sid][idx]; u64 *owned = dart->locked_owned[sid][idx]; @@ -1052,7 +1067,8 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, "failed to power DART for TLB synchronization: %d\n", ret); return ret; } - ret = stream.dart->locked ? apple_dart_hw_sync_locked(cfg, &stream) : 0; + ret = stream.dart->locked ? + apple_dart_hw_sync_locked(cfg, &stream, true) : 0; if (ret) dev_err_ratelimited(stream.dart->dev, "failed to publish host page-table entries: %d\n", ret); @@ -1280,7 +1296,7 @@ apple_dart_setup_translation_locked(struct apple_dart_domain *domain, if (ret) goto unmap; } - ret = apple_dart_hw_sync_locked(cfg, stream_map); + ret = apple_dart_hw_sync_locked(cfg, stream_map, false); if (!ret) ret = stream_map->dart->hw->invalidate_tlb(stream_map); if (!ret) From d957b0bd713bbf4617f54f5ed0d274964e9af6c9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:58:51 +1000 Subject: [PATCH 0440/1579] iommu/apple-dart: validate software mappings against inherited slots Firmware-owned root slots remain live after Linux attaches. Accept private host mappings only when every applicable stream agrees on the physical address, full-page coverage and read/write/cache attributes encoded by the public APPLE_DART2 format. Apply the same checks to attach-time leaf-table mirroring and to every later mapping, without replacing occupied roots. Keep translated reservations intact so the IOMMU core creates software translations with the correct physical/DVA identity and domain DMA offset, and preserves its translated-domain requirement. RTKit buffer queries use that host table rather than an unchecked first-stream firmware fallback. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 206 +++++++++++++++++++++++-------------- 1 file changed, 128 insertions(+), 78 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 3fd8b9fb5bc18..413c719e02303 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -40,6 +40,10 @@ #include "dma-iommu.h" +#if IS_ENABLED(CONFIG_KUNIT) && !defined(MODULE) +#include +#endif + #define DART_MAX_STREAMS 256 #define DART_MAX_TTBR 4 #define MAX_DARTS_PER_DEVICE 8 @@ -516,6 +520,28 @@ static bool apple_dart_fw_slot_owned(const struct apple_dart_fw_root *fw, return fw && fw->leaf[i] && READ_ONCE(live[i]) == fw->entry[i]; } +static bool apple_dart_fw_leaf_decode(u64 pte, dma_addr_t iova, + phys_addr_t *phys, int *prot) +{ + if (!(pte & BIT_ULL(0)) || (pte & GENMASK_ULL(63, 52)) || + (pte & GENMASK_ULL(51, 40)) != GENMASK_ULL(51, 40)) + return false; + + *phys = ((pte & GENMASK_ULL(37, 10)) << 4) | (iova & (SZ_16K - 1)); + *prot = (!(pte & BIT_ULL(3)) ? IOMMU_READ : 0) | + (!(pte & BIT_ULL(2)) ? IOMMU_WRITE : 0) | + (!(pte & BIT_ULL(1)) ? IOMMU_CACHE : 0); + return true; +} + +static bool apple_dart_fw_mapping_matches(phys_addr_t phys, int prot, + phys_addr_t expected, int required) +{ + const int mask = IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE; + + return phys == expected && (prot & mask) == (required & mask); +} + /* * A root slot firmware already uses can still back Linux mappings when the * firmware's leaf table translates every page Linux mapped there to the same @@ -536,11 +562,16 @@ static bool apple_dart_fw_slot_mirrors(const struct apple_dart *dart, u64 ours = READ_ONCE(ours_leaf[j]); u64 theirs = READ_ONCE(fw_leaf[j]); + phys_addr_t ours_phys, fw_phys; + int ours_prot, fw_prot; + if (!(ours & APPLE_DART_PTE_VALID)) continue; - if (!(theirs & APPLE_DART_PTE_VALID) || - apple_dart_pte_to_paddr(dart, ours) != - apple_dart_pte_to_paddr(dart, theirs)) + + if (!apple_dart_fw_leaf_decode(ours, 0, &ours_phys, &ours_prot) || + !apple_dart_fw_leaf_decode(theirs, 0, &fw_phys, &fw_prot) || + !apple_dart_fw_mapping_matches(fw_phys, fw_prot, + ours_phys, ours_prot)) return false; } return true; @@ -1115,88 +1146,64 @@ static int apple_dart_iotlb_sync_map(struct iommu_domain *domain, } /* - * Look up a domain address in the firmware-owned slots of its locked roots. - * Returns true if a firmware slot covers @dva; *phys is then the firmware's - * translation of that page, or 0 if the firmware leaves it unmapped. + * Hardware continues to use every firmware-owned slot. Accept a private + * software mapping only when every stream using such a slot agrees on the + * full-page translation and its protection/cache attributes. */ -static bool apple_dart_fw_lookup(struct apple_dart_domain *domain, u64 dva, - phys_addr_t *phys) +static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, + phys_addr_t paddr, size_t size, int prot) { struct apple_dart_atomic_stream_map *map; - bool owned = false; - int i, sid; + int i, sid, ret = 0; - *phys = 0; for_each_stream_map(i, domain, map) { struct apple_dart *dart = map->dart; u32 entries = dart->pgsize / sizeof(u64); u32 pgshift = ilog2(dart->pgsize); u32 bits = ilog2(entries); - u64 leaf = (dva >> pgshift) & (entries - 1); - u64 slot = (dva >> (pgshift + bits)) & (entries - 1); - u64 idx = dva >> (pgshift + 2 * bits); unsigned long flags; + size_t off; - if (!dart->fw_mirror || idx >= DART_MAX_TTBR) + if (!dart->fw_mirror) continue; spin_lock_irqsave(&dart->lock, flags); - for (sid = 0; sid < dart->num_streams && !owned; sid++) { - const struct apple_dart_fw_root *fw; - u64 pte; + for (off = 0; off < size; off += dart->pgsize) { + u64 addr = dva + off; + u64 leaf = (addr >> pgshift) & (entries - 1); + u64 slot = (addr >> (pgshift + bits)) & (entries - 1); + u64 idx = addr >> (pgshift + 2 * bits); - if (!(atomic_long_read(&map->sidmap[BIT_WORD(sid)]) & - BIT_MASK(sid))) - continue; - fw = dart->locked_fw[sid][idx]; - if (!dart->locked_ttbr[sid][idx] || - !apple_dart_fw_slot_owned(fw, dart->locked_ttbr[sid][idx], - slot)) + if (idx >= DART_MAX_TTBR) continue; - owned = true; - pte = READ_ONCE(fw->leaf[slot][leaf]); - if (pte & APPLE_DART_PTE_VALID) - *phys = apple_dart_pte_to_paddr(dart, pte); + for (sid = 0; sid < dart->num_streams; sid++) { + const struct apple_dart_fw_root *fw; + phys_addr_t phys; + u64 pte; + int fw_prot; + + if (!(atomic_long_read(&map->sidmap[BIT_WORD(sid)]) & + BIT_MASK(sid))) + continue; + fw = dart->locked_fw[sid][idx]; + if (!dart->locked_ttbr[sid][idx] || + !apple_dart_fw_slot_owned(fw, + dart->locked_ttbr[sid][idx], slot)) + continue; + pte = READ_ONCE(fw->leaf[slot][leaf]); + if (!apple_dart_fw_leaf_decode(pte, addr, + &phys, &fw_prot) || + !apple_dart_fw_mapping_matches(phys, fw_prot, + paddr + off, prot)) { + ret = -EBUSY; + goto unlock; + } + } } + unlock: spin_unlock_irqrestore(&dart->lock, flags); - if (owned) - return true; - } - return false; -} - -/* - * Firmware buffers the coprocessor later shares with the host live in the - * firmware's own tables, not in Linux's; resolve them there. - */ -static phys_addr_t apple_dart_fw_iova_to_phys(struct apple_dart_domain *domain, - u64 dva) -{ - phys_addr_t phys; - u32 pgsize = domain->domain.pgsize_bitmap; - - if (!apple_dart_fw_lookup(domain, dva, &phys) || !phys) - return 0; - return phys | (dva & (pgsize - 1)); -} - -/* - * Linux mappings into a firmware-owned slot are never published; the - * hardware keeps walking the firmware's table. Only accept them where the - * firmware already provides the same translation. - */ -static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, - phys_addr_t paddr, size_t size) -{ - u32 pgsize = domain->domain.pgsize_bitmap; - size_t off; - - for (off = 0; off < size; off += pgsize) { - phys_addr_t phys; - - if (apple_dart_fw_lookup(domain, dva + off, &phys) && - phys != paddr + off) - return -EBUSY; + if (ret) + return ret; } return 0; } @@ -1207,18 +1214,13 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; - phys_addr_t phys; - u64 dva; - if (!ops) return 0; - dva = (iova + dart_domain->dma_offset) & dart_domain->mask; - phys = ops->iova_to_phys(ops, dva); - if (phys) - return phys; - - return apple_dart_fw_iova_to_phys(dart_domain, dva); + /* Reserved firmware mappings are also represented in this software table. */ + return ops->iova_to_phys(ops, + (iova + dart_domain->dma_offset) & + dart_domain->mask); } static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, @@ -1236,7 +1238,8 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, return -ENODEV; dva = (iova + dart_domain->dma_offset) & dart_domain->mask; - ret = apple_dart_check_fw_map(dart_domain, dva, paddr, pgsize * pgcount); + ret = apple_dart_check_fw_map(dart_domain, dva, paddr, pgsize * pgcount, + prot); if (ret) return ret; @@ -1871,8 +1874,9 @@ static int apple_dart_def_domain_type(struct device *dev) static void apple_dart_get_resv_regions(struct device *dev, struct list_head *head) { + struct iommu_resv_region *region; + if (IS_ENABLED(CONFIG_PCIE_APPLE) && dev_is_pci(dev)) { - struct iommu_resv_region *region; int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; region = iommu_alloc_resv_region(DOORBELL_ADDR, @@ -1884,6 +1888,10 @@ static void apple_dart_get_resv_regions(struct device *dev, list_add_tail(®ion->list, head); } + /* Keep translated regions so the core retains their software mappings + * and the device's translated-domain requirement. Locked-root publication + * validates them against every live firmware stream without replacing it. + */ iommu_dma_get_resv_regions(dev, head); } @@ -2652,6 +2660,48 @@ static struct platform_driver apple_dart_driver = { module_platform_driver(apple_dart_driver); +#if IS_ENABLED(CONFIG_KUNIT) && !defined(MODULE) +static void apple_dart_fw_leaf_test(struct kunit *test) +{ + const u64 leaf = GENMASK_ULL(51, 40) | + ((u64)(0x280000 >> 4) & GENMASK_ULL(37, 10)) | + BIT_ULL(0); + const int rwc = IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE; + phys_addr_t phys; + int prot; + bool valid, matched; + + valid = apple_dart_fw_leaf_decode(0, 0x4000, &phys, &prot); + KUNIT_EXPECT_FALSE(test, valid); + valid = apple_dart_fw_leaf_decode(BIT_ULL(0), 0x4000, + &phys, &prot); + KUNIT_EXPECT_FALSE(test, valid); + valid = apple_dart_fw_leaf_decode(leaf, 0x4000, &phys, &prot); + KUNIT_ASSERT_TRUE(test, valid); + matched = apple_dart_fw_mapping_matches(phys, prot, 0x280000, rwc); + KUNIT_EXPECT_TRUE(test, matched); + matched = apple_dart_fw_mapping_matches(phys, prot, 0x284000, rwc); + KUNIT_EXPECT_FALSE(test, matched); + valid = apple_dart_fw_leaf_decode(leaf | BIT_ULL(2), 0x4000, + &phys, &prot); + KUNIT_ASSERT_TRUE(test, valid); + matched = apple_dart_fw_mapping_matches(phys, prot, 0x280000, rwc); + KUNIT_EXPECT_FALSE(test, matched); +} + +static struct kunit_case apple_dart_fw_cases[] = { + KUNIT_CASE(apple_dart_fw_leaf_test), + {} +}; + +static struct kunit_suite apple_dart_fw_suite = { + .name = "apple-dart-firmware-leaf", + .test_cases = apple_dart_fw_cases, +}; + +kunit_test_suite(apple_dart_fw_suite); +#endif + MODULE_DESCRIPTION("IOMMU API for Apple's DART"); MODULE_AUTHOR("Sven Peter "); MODULE_LICENSE("GPL v2"); From 69b649e82306d72ea50d433735cd264ab0906440 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 01:07:37 +1000 Subject: [PATCH 0441/1579] dt-bindings: display: apple,dcp: document the DCP and display subsystem Describe the existing Asahi display nodes and the T8140 internal and external DCP compatibles. The piodma child and panel properties are included, and apple,iomfb-surfaces retains its per-slot enable meaning. T8140 DCPs must not specify a reset, because resetting the inherited display CPU can hang the machine (DT-5, DT-6). Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/display/apple,dcp.yaml | 256 ++++++++++++++++++ .../display/apple,display-subsystem.yaml | 29 ++ MAINTAINERS | 2 + 3 files changed, 287 insertions(+) create mode 100644 Documentation/devicetree/bindings/display/apple,dcp.yaml create mode 100644 Documentation/devicetree/bindings/display/apple,display-subsystem.yaml diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml new file mode 100644 index 0000000000000..47610c176cfd2 --- /dev/null +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -0,0 +1,256 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/apple,dcp.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple display coprocessor + +maintainers: + - Ryan Murray + +description: + Apple's Display Co-Processor (DCP) drives internal and external displays + through an RTKit mailbox. Its piodma child uses a separate display DART + stream from the coprocessor itself. + +properties: + compatible: + oneOf: + - items: + - enum: + - apple,t8103-dcp + - apple,t8112-dcp + - apple,t6000-dcp + - apple,t6020-dcp + - apple,t8140-dcp + - const: apple,dcp + - items: + - enum: + - apple,t8103-dcpext + - apple,t8112-dcpext + - apple,t6000-dcpext + - apple,t6020-dcpext + - const: apple,dcpext + - items: + - const: apple,t8140-dcpext + - const: apple,dcp + + reg: + minItems: 2 + maxItems: 6 + + reg-names: + minItems: 2 + items: + - const: coproc + - const: disp-0 + - const: disp-1 + - const: disp-2 + - const: disp-3 + - const: disp-4 + + mboxes: + maxItems: 1 + + mbox-names: + items: + - const: mbox + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + resets: + maxItems: 1 + + clocks: + minItems: 1 + maxItems: 2 + + phys: + minItems: 1 + maxItems: 8 + + phy-names: + minItems: 1 + maxItems: 8 + items: + pattern: '^(dp-phy|typec[0-9]+)$' + + apple,bw-scratch: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: PMGR bandwidth scratch resource, display index and offset. + + apple,bw-doorbell: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: Optional PMGR bandwidth doorbell resource and display index. + + apple,iomfb-surfaces: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 4 + maxItems: 4 + items: + enum: [0, 1] + description: Per-slot enable flags, in slot order. + + apple,firmware-version: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 3 + maxItems: 5 + description: Version reported by the loaded display firmware. + + apple,firmware-compat: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 3 + maxItems: 5 + description: Loader-provided firmware ABI version used for IOMFB selection. + + apple,connector-type: + enum: [HDMI-A, DP, USB-C] + + apple,dcp-index: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Display coprocessor index on multi-DCP systems. + + apple,dptx-phy: + $ref: /schemas/types.yaml#/definitions/uint32 + description: DisplayPort transmitter PHY selector. + + apple,dptx-die: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Die containing the DisplayPort transmitter. + + apple,notch-height: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Height in pixels reserved for the display notch. + + apple,typec-mux-indices: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 4 + description: Crossbar state for each Type-C display route. + + apple,typec-dptx-phys: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 4 + description: DisplayPort PHY index for each Type-C display route. + + mux-controls: + $ref: /schemas/types.yaml#/definitions/phandle-array + minItems: 1 + maxItems: 8 + + mux-control-names: + minItems: 1 + maxItems: 8 + items: + pattern: '^(dp-xbar|typec[0-9]+)$' + + mux-index: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Crossbar state for a fixed output. + + hdmi-hpd-gpios: + maxItems: 1 + + hdmi-pwren-gpios: + maxItems: 1 + + dp2hdmi-pwren-gpios: + maxItems: 1 + + '#address-cells': + const: 2 + + '#size-cells': + const: 2 + + piodma: + type: object + properties: + iommus: + maxItems: 1 + required: + - iommus + additionalProperties: false + + panel: + type: object + properties: + compatible: + oneOf: + - const: apple,panel + - items: + - pattern: '^apple,panel-j[0-9]+$' + - const: apple,panel + - items: + - pattern: '^apple,panel-j[0-9]+$' + - const: apple,panel-mini-led + - const: apple,panel + width-mm: + description: Physical panel width. + height-mm: + description: Physical panel height. + adj-height-mm: + description: Visible height below the notch. + apple,max-brightness: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Maximum firmware backlight level. + required: + - compatible + additionalProperties: false + + typec-routes: + type: object + properties: + '#address-cells': + const: 1 + '#size-cells': + const: 0 + patternProperties: + '^route@[0-3]$': + type: object + properties: + reg: + maxItems: 1 + mode-switch: + type: boolean + port: + $ref: /schemas/graph.yaml#/properties/port + required: + - reg + - mode-switch + - port + additionalProperties: false + required: + - '#address-cells' + - '#size-cells' + additionalProperties: false + + ports: + $ref: /schemas/graph.yaml#/properties/ports + +required: + - compatible + - reg + - reg-names + - mboxes + - iommus + - apple,bw-scratch + +allOf: + - if: + properties: + compatible: + contains: + enum: ['apple,t8140-dcp', 'apple,t8140-dcpext'] + then: + properties: + resets: false + +unevaluatedProperties: false +... diff --git a/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml b/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml new file mode 100644 index 0000000000000..429e68b29d864 --- /dev/null +++ b/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/apple,display-subsystem.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple display subsystem + +maintainers: + - Ryan Murray + +description: + Component master for one or more Apple display coprocessors. Each IOMMU + entry describes a scanout DART stream owned by a DCP instance. + +properties: + compatible: + const: apple,display-subsystem + + iommus: + minItems: 1 + maxItems: 8 + +required: + - compatible + - iommus + +unevaluatedProperties: false +... diff --git a/MAINTAINERS b/MAINTAINERS index 90654e399c0d3..2cb1e5509f160 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1951,6 +1951,8 @@ M: Janne Grunau L: dri-devel@lists.freedesktop.org S: Maintained T: git git://anongit.freedesktop.org/drm/drm-misc +F: Documentation/devicetree/bindings/display/apple,dcp.yaml +F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: drivers/gpu/drm/apple/ APPLE PCIE CONTROLLER DRIVER From 83cf4b40b8b089c59054137256553efa6820202a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 01:09:44 +1000 Subject: [PATCH 0442/1579] dt-bindings: chosen: document apple,dcp-rtkit-quiesced The loader reports whether the DCP core was running at hand-off using a u32 in /chosen. Document both values and allow the property to be absent on a chainload. The kernel uses this label to select its start mode once the H17P path is implemented (DT-5, LIFE-1). Assisted-by: LLM Signed-off-by: Ryan Murray --- .../chosen/apple,dcp-rtkit-quiesced.yaml | 36 +++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 37 insertions(+) create mode 100644 Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml diff --git a/Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml b/Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml new file mode 100644 index 0000000000000..d086bdc05f5ee --- /dev/null +++ b/Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/chosen/apple,dcp-rtkit-quiesced.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple DCP RTKit loader hand-off + +maintainers: + - Ryan Murray + +description: + The loader records whether it handed a running display coprocessor to + the kernel. It omits this property on a chainload. + +select: + properties: + $nodename: + const: chosen + required: + - $nodename + +properties: + $nodename: + const: chosen + + apple,dcp-rtkit-quiesced: + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1] + description: + One if the DCP core was running at hand-off, zero if it was not. + The name reflects the loader's RTKit hand-off state, not a request + for Linux to quiesce the coprocessor. + +additionalProperties: true +... diff --git a/MAINTAINERS b/MAINTAINERS index 2cb1e5509f160..47ceb787d696d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1951,6 +1951,7 @@ M: Janne Grunau L: dri-devel@lists.freedesktop.org S: Maintained T: git git://anongit.freedesktop.org/drm/drm-misc +F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: drivers/gpu/drm/apple/ From cc7d8002d8a18a9b9e1d33712ded06f788bbe13a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 6 Oct 2026 23:02:48 +1100 Subject: [PATCH 0443/1579] dt-bindings: display: apple,dcp: describe named handoff resources Describe the H17-generation iop-vbar window independently of the numbered display windows and include the existing T8142 compatible. Document the clock names consumed by the driver and the loader's reserved-memory lists. Permit partial surface-enable arrays: the public driver reads one flag per slot and accepts up to four entries. This preserves its existing meaning instead of requiring every device tree to provide all four slots. The named resources follow the public Apple DRM driver and the Linux-side J700 hardware-description observation. No reset is added to a device tree. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/display/apple,dcp.yaml | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index 47610c176cfd2..ac2286832ba7d 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -24,6 +24,7 @@ properties: - apple,t6000-dcp - apple,t6020-dcp - apple,t8140-dcp + - apple,t8142-dcp - const: apple,dcp - items: - enum: @@ -38,17 +39,18 @@ properties: reg: minItems: 2 - maxItems: 6 + maxItems: 7 reg-names: minItems: 2 items: - const: coproc - const: disp-0 - - const: disp-1 - - const: disp-2 - - const: disp-3 - - const: disp-4 + - enum: [disp-1, iop-vbar] + - enum: [disp-2, iop-vbar] + - enum: [disp-3, iop-vbar] + - enum: [disp-4, iop-vbar] + - const: iop-vbar mboxes: maxItems: 1 @@ -70,6 +72,20 @@ properties: minItems: 1 maxItems: 2 + clock-names: + minItems: 1 + items: + - const: clock-154 + - const: clock-194 + + memory-region: + minItems: 1 + maxItems: 64 + + memory-region-names: + minItems: 1 + maxItems: 64 + phys: minItems: 1 maxItems: 8 @@ -90,7 +106,7 @@ properties: apple,iomfb-surfaces: $ref: /schemas/types.yaml#/definitions/uint32-array - minItems: 4 + minItems: 1 maxItems: 4 items: enum: [0, 1] From 707f28a4c399092ba01bd281a3f0b6623f7bb223 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 08:10:21 +1100 Subject: [PATCH 0444/1579] iommu/apple-dart: import firmware translations on T8140 Select the existing firmware-root import path for locked T8140 DARTs. The loader leaves occupied firmware translations in place; attach-time mirroring and later host maps validate full-page addresses and protection attributes across every applicable stream. Vacant slots remain available for Linux mappings without overwriting inherited roots. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 413c719e02303..d15fb45d28233 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -2411,9 +2411,10 @@ static int apple_dart_probe(struct platform_device *pdev) dart->locked = false; else dart->locked = apple_dart_is_locked(dart); - /* T8142 loaders re-export the firmware's own display translations. */ + /* These loaders re-export the firmware's own display translations. */ dart->fw_mirror = dart->locked && - of_device_is_compatible(pdev->dev.of_node, "apple,t8142-dart"); + (of_device_is_compatible(pdev->dev.of_node, "apple,t8140-dart") || + of_device_is_compatible(pdev->dev.of_node, "apple,t8142-dart")); if (!dart->locked) { ret = apple_dart_hw_reset(dart); if (ret) From a88aa68f9fd78119e909f37a7a3c146d0557dceb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 13:25:00 +1100 Subject: [PATCH 0445/1579] drm: apple: balance backlight modeset locking A brightness-only atomic commit can acquire more than the CRTC lock. Handle -EDEADLK by clearing the temporary state, backing off and retrying, then drop the acquired locks and finalize the acquire context on every exit. The previous path returned after backoff with the slow lock held and never finalized the context on success or interruption. Use the DRM single-lock helper when updating the cached brightness value. That path acquires only the CRTC mutex and does not need an acquire context. Release the mutex before starting the atomic brightness update. This follows the public DRM modeset-lock and atomic-state contracts and applies to every firmware variant without changing the brightness bytes. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.c | 51 ++++++++++++--------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 9eb0c7d4eb534..d98a483b20d93 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -144,39 +144,38 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) struct drm_crtc *crtc = &dcp->crtc->base; int ret = 0; - drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); - ret = drm_modeset_lock(&crtc->mutex, &ctx); - if (ret == -EDEADLK) { - drm_modeset_backoff(&ctx); - return -EDEADLK; - } else if (ret == -ERESTARTSYS) { - return -ERESTARTSYS; - } - - if (!dcp->brightness.update) - goto done; - state = drm_atomic_state_alloc(crtc->dev); - if (!state) { - ret = -ENOMEM; - goto done; - } + if (!state) + return -ENOMEM; + drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); state->acquire_ctx = &ctx; + +retry: crtc_state = drm_atomic_get_crtc_state(state, crtc); if (IS_ERR(crtc_state)) { ret = PTR_ERR(crtc_state); - goto fail; + goto out; } + if (!dcp->brightness.update) + goto out; + crtc_state->color_mgmt_changed |= true; ret = drm_atomic_commit(state); -fail: +out: + if (ret == -EDEADLK) { + drm_atomic_state_clear(state); + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry; + } + drm_atomic_state_put(state); -done: drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); return ret; } @@ -203,22 +202,16 @@ static int dcp_set_brightness(struct backlight_device *bd) { int ret = 0; struct apple_dcp *dcp = bl_get_data(bd); - struct drm_modeset_acquire_ctx ctx; int brightness = backlight_get_brightness(bd); - drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); - ret = drm_modeset_lock(&dcp->crtc->base.mutex, &ctx); - if (ret == -EDEADLK) { - drm_modeset_backoff(&ctx); - return -EDEADLK; - } else if (ret == -ERESTARTSYS) { - return -ERESTARTSYS; - } + ret = drm_modeset_lock_single_interruptible(&dcp->crtc->base.mutex); + if (ret) + return ret; dcp->brightness.dac = calculate_dac(dcp, brightness); dcp->brightness.update = true; - drm_modeset_drop_locks(&ctx); + drm_modeset_unlock(&dcp->crtc->base.mutex); return dcp_backlight_update(dcp); } From e14196f801aa70b8884d66b30595e408f84f8d37 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 14:14:32 +1100 Subject: [PATCH 0446/1579] drm: apple: preserve scanout in brightness-only commits The brightness update creates a CRTC-only atomic state and marks its colour state changed. The atomic helpers do not add unchanged planes for this non-modesetting transaction, but the DCP present builder clears all surfaces and repopulates them only from planes in the atomic state. Add the CRTC's affected planes before committing so a static-screen brightness update re-presents the current scanout instead of a request without surfaces. Use the existing cached plane-state duplication and framebuffer references. Plane-state allocation and locking errors follow the same atomic backoff and cleanup path as the CRTC state. This uses the public DRM atomic-state contract and the existing Asahi present path for all firmware variants, without changing record layouts. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index d98a483b20d93..72e53acf140e3 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -161,6 +161,11 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) if (!dcp->brightness.update) goto out; + /* Re-present the current scanout when brightness is the only change. */ + ret = drm_atomic_add_affected_planes(state, crtc); + if (ret) + goto out; + crtc_state->color_mgmt_changed |= true; ret = drm_atomic_commit(state); From 4088c3b1854cdbaf106d8bfe72510551bf9fd43c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 15:37:55 +1100 Subject: [PATCH 0447/1579] arm64: dts: apple: t8140: add disabled display power domains The Linux-side ADT record R-PMGR-static identifies six physical display power-state registers and their internal and external parent chains. Describe them using the existing PMGR power-state binding. The common fabric parent and DCP power-gate identifiers are virtual ADT devices without power-state registers, so do not create physical controllers for those identifiers. Keep every new domain disabled. The system domains require firmware notification, whose transition ordering has not yet been qualified. This preserves the inherited display power state while providing the resource descriptions needed for subsequent display integration. Source: Linux ADT /arm-io/pmgr, decoded with public Asahi m1n1. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 58 +++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 3fc3ea3ad8fb4..6329641b248fc 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -56,6 +56,24 @@ apple,always-on; /* handed over running, cannot be reset */ }; + ps_disp_sys: power-controller@280 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x280 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_sys"; + status = "disabled"; + }; + + ps_dispext0_sys: power-controller@288 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_sys"; + status = "disabled"; + }; + ps_apcie_sys_gp: power-controller@2a0 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x2a0 4>; @@ -82,6 +100,36 @@ power-domains = <&ps_ans>; }; + ps_disp_fe: power-controller@318 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x318 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_fe"; + power-domains = <&ps_disp_sys>; + status = "disabled"; + }; + + ps_dispext0_fe: power-controller@320 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x320 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_fe"; + power-domains = <&ps_dispext0_sys>; + status = "disabled"; + }; + + ps_dispext0_cpu: power-controller@328 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x328 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_cpu"; + power-domains = <&ps_dispext0_fe>; + status = "disabled"; + }; + ps_apcie_sys_st: power-controller@3a0 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x3a0 4>; @@ -258,6 +306,16 @@ #reset-cells = <0>; label = "isp_cvm"; }; + ps_disp_cpu: power-controller@10000 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x10000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_cpu"; + power-domains = <&ps_disp_fe>; + status = "disabled"; + }; + }; &pmgr_mini { From 004200ec1cc8005fa692d797d2c5fb009ec9e05a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 16:38:21 +1100 Subject: [PATCH 0448/1579] arm64: dts: apple: t8140: add disabled internal display DARTs Describe the internal display, display GRT and DCP translation controllers from the Linux-exported ADT. Translate their first register windows through the ADT bus ranges and retain the shared interrupt. These controllers use the existing T8110 DART register interface. Only the translation controller windows are needed by the public DART driver. Do not expose the additional ADT windows as DART registers. Keep all three nodes disabled while display power transitions and consumer lifetimes remain unqualified. DMA apertures and consumer stream assignments will be described with their qualified consumers. Source: Linux ADT /arm-io/dart-disp0, dart-dispgrt and dart-dcp, decoded with public Asahi m1n1 (R-ADT). Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 2dd921b23368c..f9e6851a17488 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -707,6 +707,33 @@ }; &soc { + disp0_dart: iommu@412300000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x12300000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + status = "disabled"; + }; + + dispgrt_dart: iommu@412320000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x12320000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + status = "disabled"; + }; + + dcp_dart: iommu@412340000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x12340000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + status = "disabled"; + }; + pinctrl_ap: pinctrl@31a000000 { compatible = "apple,t8140-pinctrl", "apple,t8103-pinctrl"; reg = <0x3 0x1a000000 0x0 0x100000>; From 4cd820156dae33d73a606c8913c39d5e3c604d39 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 16:59:37 +1100 Subject: [PATCH 0449/1579] drm: apple: initialise each parsed display mode A timing entry can populate its colour modes before being rejected as virtual. enumerate_modes() then reuses that output slot for the next entry. Clearing only the original allocation lets a later entry without ColorModes inherit those colours. Even the first such entry sees zero scores as valid, rather than reporting that no colour was selected. Reset the output and score for each timing entry and initialise the intermediate timing attributes. Initialise every colour entry with invalid field values so a partial dictionary cannot reuse stack data from a preceding colour entry. Complete dictionaries keep their existing colour selection and preferred-mode ranking. This follows the public TimingElements parser and the DRM-3 requirement that rejected entries cannot leak fields into subsequent modes. Sixteen own dictionary fixtures exercise the actual parser, including missing colour arrays, rejected entries, partial colours and preferred ranking. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 3ba9f5e4c4dd5..eb75524105ca1 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -370,7 +370,15 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, dcp_parse_foreach_in_array(handle, outer_it) { struct iterator it; bool is_virtual = true; - struct color_mode cmode; + struct color_mode cmode = { + .colorimetry = -1, + .depth = -1, + .dynamic_range = -1, + .eotf = -1, + .id = -1, + .pixel_encoding = -1, + .score = -1, + }; dcp_parse_foreach_in_dict(handle, it) { char *key = parse_string(it.handle); @@ -447,13 +455,21 @@ static int parse_mode(struct dcp_parse_ctx *handle, { int ret = 0; struct iterator it; - struct dimension horiz, vert; + struct dimension horiz = {}, vert = {}; s64 min_vrr = 0, max_vrr = 0; s64 id = -1; s64 best_color_mode = -1; bool is_virtual = false; struct drm_display_mode *mode = &out->mode; + *out = (struct dcp_display_mode) { + .sdr_rgb.score = -1, + .sdr_444.score = -1, + .sdr.score = -1, + .best.score = -1, + }; + *score = -1; + dcp_parse_foreach_in_dict(handle, it) { char *key = parse_string(it.handle); @@ -583,7 +599,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, if (ret) return ERR_PTR(ret); - /* Start with a worst case allocation; parse_mode() only sets vrr. */ + /* Start with a worst case allocation. */ modes = kcalloc(it.len, sizeof(*modes), GFP_KERNEL); *count = 0; From d2f162f6d7c24ed783657112086f7b56e1fb0cbe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 17:05:00 +1100 Subject: [PATCH 0450/1579] arm64: dts: apple: t8140: add disabled external display DARTs The Linux-exported ADT describes separate DART controllers for the external display engine and coprocessor. Describe their controller windows and shared interrupt alongside the internal display DARTs, using the existing T8140/T8110 binding. Public m1n1 ADTNode.get_reg() translates /arm-io/dart-dispext0 and /arm-io/dart-dcpext0 through the parent bus ranges. Only the controller windows are register resources here. Keep both nodes disabled until consumer, stream and power integration has been qualified. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index f9e6851a17488..d230c1becfe7c 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -734,6 +734,24 @@ status = "disabled"; }; + dispext0_dart: iommu@416300000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x16300000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + status = "disabled"; + }; + + dcpext0_dart: iommu@416340000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x16340000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + status = "disabled"; + }; + pinctrl_ap: pinctrl@31a000000 { compatible = "apple,t8140-pinctrl", "apple,t8103-pinctrl"; reg = <0x3 0x1a000000 0x0 0x100000>; From 408c42fa74df6ebe3612a52654eff67d19cd13f1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 17:09:51 +1100 Subject: [PATCH 0451/1579] drm: apple: skip failed timing entries before parsing the next enumerate_modes() treats a failure to parse one timing entry as recoverable, but parse_mode() can return before consuming the rest of that entry. The next iteration then starts inside the rejected dictionary instead of at the next mode. Save the entry's starting position and skip the complete entry after a parse failure. If even the generic dictionary walker cannot skip it, return that error instead of continuing from an unknown boundary. Own public-format dictionary fixtures reproduce the failure with a wrongly typed timing score or colour ID followed by a valid mode. The following modes are now retained, including when a valid mode precedes the rejected one. Complete dictionaries keep the existing parse path. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index eb75524105ca1..024ecce8d31a5 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -607,13 +607,23 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, return ERR_PTR(-ENOMEM); for (; it.idx < it.len; ++it.idx) { + struct dcp_parse_ctx entry = *it.handle; + mode = &modes[*count]; ret = parse_mode(it.handle, mode, &score, width_mm, height_mm, notch_height, internal); /* Errors for a single mode are recoverable -- just skip it. */ - if (ret) + if (ret) { + /* A failed parse can stop in the middle of the dictionary. */ + *it.handle = entry; + ret = skip(it.handle); + if (ret) { + kfree(modes); + return ERR_PTR(ret); + } continue; + } /* Process a successful mode */ (*count)++; From 51e1e56423b55f153335a63da08163860a61c797 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 17:21:48 +1100 Subject: [PATCH 0452/1579] drm: apple: avoid aligned pointers to packed dictionary integers Dictionary tags are aligned to four bytes, so the following eight-byte integer need not be naturally aligned. parse_int_bound() already copies that value into aligned storage, but its source pointer still has an s64 type. Keep the source as an untyped byte pointer for memcpy(). Own public-format dictionary fixtures place integers at both four- and eight-byte boundaries. The actual parser now passes the alignment sanitizer as well as the mode enumeration checks without changing any wire bytes or integer bounds. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 024ecce8d31a5..df179afc87039 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -181,7 +181,7 @@ static char *parse_string(struct dcp_parse_ctx *handle) static int parse_int_bound(struct dcp_parse_ctx *handle, s64 *value, s64 min, s64 max) { const void *tag = parse_tag_of_type(handle, DCP_TYPE_INT64); - const s64 *in; + const void *in; if (IS_ERR(tag)) return PTR_ERR(tag); From 8d89fafdf3717a0eacdb0fa4b6720edf9b08bbd3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:12:50 +1100 Subject: [PATCH 0453/1579] drm: apple: add present-coupled H17P backlight policy Implement the clean display SPEC BACKLIGHT-1..4 policy in the existing Asahi backlight path. Keep commanded nits and completed levels separate, preserve takeover, and cache requests while blanked or suspended. Internal H17P DPMS keeps the pipe and surface alive. Admit range, takeover and wire encoding through a narrow interface to the serialized present queue. No H17P brightness bytes or default nits are inferred. Keep policy storage at the end of the device and gate template changes at compile time so older firmware keeps its existing fields and generated code. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 9 ++ drivers/gpu/drm/apple/dcp.c | 18 +++ drivers/gpu/drm/apple/dcp_backlight.c | 157 ++++++++++++++++++++++- drivers/gpu/drm/apple/dcp_backlight.h | 165 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 12 +- drivers/gpu/drm/apple/systemep.c | 6 + 6 files changed, 365 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp_backlight.h diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 3d45d07dacf07..2262fb98f3cc8 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -17,6 +17,7 @@ #include #include "dptxep.h" +#include "dcp_backlight.h" #include "iomfb.h" #include "iomfb_h17p.h" #include "iomfb_v12_3.h" @@ -340,6 +341,14 @@ struct apple_dcp { struct apple_connector *fixed_connector; struct apple_connector *typec_connector; int hdmi_hpd_irq; + + /* H17P policy shared by backlight, commit, and reply workers. */ + struct { + /* Protects state and callback installation. */ + spinlock_t lock; + struct dcp_backlight_state state; + void (*kick)(struct apple_dcp *dcp); + } backlight; }; void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index b28b201d64924..6b6828db5b506 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1488,6 +1488,13 @@ void dcp_poweron(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + if (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) { + ret = dcp_backlight_dpms(dcp, true); + if (ret) + dev_warn(dcp->dev, "backlight restore unavailable: %d\n", ret); + return; + } + if (dcp_is_typec_output(dcp)) { WRITE_ONCE(dcp->typec_crtc_off, false); @@ -1536,6 +1543,14 @@ void dcp_poweroff(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + /* Internal H17P DPMS is a brightness present, never a pipe stop. */ + if (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) { + ret = dcp_backlight_dpms(dcp, false); + if (ret) + dev_warn(dcp->dev, "backlight blank unavailable: %d\n", ret); + return; + } + if (dcp->avep) av_service_disconnect(dcp); @@ -1584,6 +1599,8 @@ static void dcp_work_register_backlight(struct work_struct *work) /* try to register backlight device, */ ret = dcp_backlight_register(dcp); + if (ret == -ENODATA) + goto out_unlock; if (ret) { dev_err(dcp->dev, "Unable to register backlight device\n"); dcp->brightness.maximum = 0; @@ -2224,6 +2241,7 @@ static int dcp_platform_probe(struct platform_device *pdev) INIT_LIST_HEAD(&dcp->swapped_out_fbs); mutex_init(&dcp->swapped_out_fbs_lock); + spin_lock_init(&dcp->backlight.lock); dcp->fw_compat = fw_compat; dcp->dev = dev; diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 72e53acf140e3..5c7f296eaf798 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -82,13 +82,125 @@ static u32 brightness_part2[] = { }; +bool dcp_backlight_active(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && + READ_ONCE(dcp->backlight.state.ready); +} + static int dcp_get_brightness(struct backlight_device *bd) { struct apple_dcp *dcp = bl_get_data(bd); + unsigned long flags; + u32 actual; + + if (dcp_backlight_active(dcp)) { + spin_lock_irqsave(&dcp->backlight.lock, flags); + actual = dcp->backlight.state.actual; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return actual; + } return dcp->brightness.nits; } +int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits, + void (*kick)(struct apple_dcp *dcp)) +{ + unsigned long flags; + int ret; + + if (dcp->fw_compat != DCP_FIRMWARE_H17P || !dcp_has_panel(dcp) || + !dcp->crtc || !kick) + return -EINVAL; + + mutex_lock(&dcp->bl_register_mutex); + spin_lock_irqsave(&dcp->backlight.lock, flags); + if (dcp->backlight.state.ready || dcp->brightness.bl_dev) { + ret = -EBUSY; + } else { + ret = dcp_bl_init(&dcp->backlight.state, maximum, + inherited_valid, inherited, + default_valid, default_nits); + if (!ret) + dcp->backlight.kick = kick; + } + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + mutex_unlock(&dcp->bl_register_mutex); + if (!ret) + schedule_work(&dcp->bl_register_wq); + return ret; +} + +bool dcp_backlight_seed(struct apple_dcp *dcp, u32 nits) +{ + unsigned long flags; + bool seeded; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + seeded = dcp_bl_seed(&dcp->backlight.state, nits); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return seeded; +} + +static void dcp_backlight_kick(struct apple_dcp *dcp) +{ + void (*kick)(struct apple_dcp *dcp) = READ_ONCE(dcp->backlight.kick); + + if (kick) + kick(dcp); +} + +int dcp_backlight_dpms(struct apple_dcp *dcp, bool on) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + ret = dcp_bl_dpms(&dcp->backlight.state, on); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + if (!ret) + dcp_backlight_kick(dcp); + return ret; +} + +int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, + struct dcp_backlight_present *present) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + ret = dcp_bl_prepare(&dcp->backlight.state, have_surface, present); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return ret; +} + +bool dcp_backlight_complete(struct apple_dcp *dcp, u64 sequence, bool accepted) +{ + unsigned long flags; + bool completed; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + completed = dcp_bl_complete(&dcp->backlight.state, sequence, accepted); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + /* Queue retries after the central queue has finished this transaction. */ + return completed; +} + +bool dcp_backlight_pending(struct apple_dcp *dcp) +{ + unsigned long flags; + bool pending; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + pending = dcp->backlight.state.dirty && !dcp->backlight.state.in_flight; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return pending; +} + #define SCALE_FACTOR (1 << 10) static u32 interpolate(int val, int min, int max, u32 *tbl, size_t tbl_size) @@ -187,6 +299,11 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) int dcp_backlight_update(struct apple_dcp *dcp) { + if (dcp_backlight_active(dcp)) { + dcp_backlight_kick(dcp); + return 0; + } + /* * Do not actively try to change brightness if no mode is set. * TODO: should this be reflected the in backlight's power property? @@ -208,6 +325,20 @@ static int dcp_set_brightness(struct backlight_device *bd) int ret = 0; struct apple_dcp *dcp = bl_get_data(bd); int brightness = backlight_get_brightness(bd); + unsigned long flags; + + if (dcp_backlight_active(dcp)) { + /* Preserve the requested level even while the core forces zero. */ + spin_lock_irqsave(&dcp->backlight.lock, flags); + ret = dcp_bl_request(&dcp->backlight.state, + bd->props.brightness, + backlight_is_blank(bd), + bd->props.state & BL_CORE_SUSPENDED); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + if (!ret) + dcp_backlight_kick(dcp); + return ret; + } ret = drm_modeset_lock_single_interruptible(&dcp->crtc->base.mutex); if (ret) @@ -221,22 +352,45 @@ static int dcp_set_brightness(struct backlight_device *bd) return dcp_backlight_update(dcp); } +static bool dcp_backlight_controls_device(struct backlight_device *bd, + struct device *display_dev) +{ + struct apple_dcp *dcp = bl_get_data(bd); + + return !display_dev || display_dev == dcp->dev || + (dcp->crtc && display_dev == dcp->crtc->base.dev->dev); +} + static const struct backlight_ops dcp_backlight_ops = { .options = BL_CORE_SUSPENDRESUME, .get_brightness = dcp_get_brightness, .update_status = dcp_set_brightness, + .controls_device = dcp_backlight_controls_device, }; int dcp_backlight_register(struct apple_dcp *dcp) { struct device *dev = dcp->dev; struct backlight_device *bl_dev; + unsigned long flags; struct backlight_properties props = { .type = BACKLIGHT_PLATFORM, .brightness = dcp->brightness.nits, .scale = BACKLIGHT_SCALE_LINEAR, }; props.max_brightness = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + if (dcp_backlight_active(dcp)) { + spin_lock_irqsave(&dcp->backlight.lock, flags); + if (!dcp->backlight.state.ready) { + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return -ENODATA; + } + /* Freeze takeover before userspace can discover the device. */ + dcp->backlight.state.controlled = true; + props.brightness = dcp->backlight.state.target; + props.max_brightness = dcp->backlight.state.maximum; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + } bl_dev = devm_backlight_device_register(dev, "apple-panel-bl", dev, dcp, &dcp_backlight_ops, &props); @@ -244,7 +398,8 @@ int dcp_backlight_register(struct apple_dcp *dcp) return PTR_ERR(bl_dev); dcp->brightness.bl_dev = bl_dev; - dcp->brightness.dac = calculate_dac(dcp, dcp->brightness.nits); + if (!dcp_backlight_active(dcp)) + dcp->brightness.dac = calculate_dac(dcp, dcp->brightness.nits); return 0; } diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h new file mode 100644 index 0000000000000..6d52a8f9eb7fe --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Ryan Murray */ + +#ifndef __APPLE_DCP_BACKLIGHT_H__ +#define __APPLE_DCP_BACKLIGHT_H__ + +#include +#include +#include + +/* Commanded nits, independent of firmware encoding. No fade policy. */ +struct dcp_backlight_state { + u32 maximum; + u32 target; + u32 actual; + u32 sent_nits; + u64 sequence; + bool ready; + bool controlled; + bool dpms_off; + bool core_blank; + bool suspended; + bool dirty; + bool in_flight; +}; + +struct dcp_backlight_present { + u64 sequence; + u32 nits; +}; + +/* Inputs must come from admitted panel/loader records, never a guess. */ +static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits) +{ + u32 nits; + + if (!maximum || maximum > INT_MAX) + return -EINVAL; + if (!inherited_valid && !default_valid) + return -ENODATA; + nits = inherited_valid ? inherited : default_nits; + if (nits > maximum) + return -ERANGE; + + *state = (struct dcp_backlight_state) { + .maximum = maximum, + .target = nits, + .actual = nits, + .ready = true, + }; + return 0; +} + +/* A decoded powerlog sample can seed takeover only before Linux control. */ +static inline bool dcp_bl_seed(struct dcp_backlight_state *state, u32 nits) +{ + if (!state->ready || state->controlled || nits > state->maximum) + return false; + state->target = nits; + state->actual = nits; + return true; +} + +static inline u32 dcp_bl_effective(const struct dcp_backlight_state *state) +{ + if (state->dpms_off || state->core_blank || state->suspended) + return 0; + return state->target; +} + +static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, + bool core_blank, bool suspended) +{ + if (!state->ready) + return -ENODATA; + if (nits > state->maximum) + return -ERANGE; + state->controlled = true; + if (state->target != nits || state->core_blank != core_blank || + state->suspended != suspended) { + state->target = nits; + state->core_blank = core_blank; + state->suspended = suspended; + state->dirty = true; + } + return 0; +} + +static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) +{ + if (!state->ready) + return -ENODATA; + state->controlled = true; + if (state->dpms_off != !on) { + state->dpms_off = !on; + state->dirty = true; + } + return 0; +} + +/* + * Called by the single outbound queue, with a pinned, accepted scanout or a + * prepared replacement. Soft-off never releases it. A brightness-only present + * must use the last accepted surface, not a rejected replacement's DRM state. + */ +static inline int dcp_bl_prepare(struct dcp_backlight_state *state, + bool have_surface, + struct dcp_backlight_present *present) +{ + if (!state->ready || !have_surface) + return -ENODATA; + if (state->in_flight) + return -EBUSY; + if (!state->dirty) + return -EALREADY; + + state->controlled = true; + state->in_flight = true; + state->dirty = false; + state->sent_nits = dcp_bl_effective(state); + present->nits = state->sent_nits; + present->sequence = ++state->sequence; + return 0; +} + +/* + * accepted means both accepted submit and its matching completed present. + * Rejection/serialization failure cancels the reservation without changing + * actual. This result is NEVER a framebuffer retirement authorization. + */ +static inline bool dcp_bl_complete(struct dcp_backlight_state *state, + u64 sequence, bool accepted) +{ + if (!state->in_flight || sequence != state->sequence) + return false; + state->in_flight = false; + if (accepted) + state->actual = state->sent_nits; + else + state->dirty = true; + return true; +} + +struct apple_dcp; + +/* + * kick only queues work; it must not submit a top-level RPC from the caller. + * The central IOMFB queue owns surface references, retries, and wire encoding. + * Configure before registration and keep the callback valid until teardown. + */ +bool dcp_backlight_active(struct apple_dcp *dcp); +int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits, + void (*kick)(struct apple_dcp *dcp)); +bool dcp_backlight_seed(struct apple_dcp *dcp, u32 nits); +int dcp_backlight_dpms(struct apple_dcp *dcp, bool on); +int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, + struct dcp_backlight_present *present); +bool dcp_backlight_complete(struct apple_dcp *dcp, u64 sequence, bool accepted); +bool dcp_backlight_pending(struct apple_dcp *dcp); + +#endif /* __APPLE_DCP_BACKLIGHT_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 98c6c5824438c..f91e394446b11 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -395,6 +395,11 @@ static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *pr switch (prop->id) { case IOMFB_PROPERTY_NITS: { +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + /* Configured H17P takeover uses only the admitted sample interface. */ + if (dcp_backlight_active(dcp)) + break; +#endif if (dcp_has_panel(dcp)) { dcp->brightness.nits = prop->value / dcp->brightness.scale; /* notify backlight device of the initial brightness */ @@ -1924,6 +1929,7 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); int plane_idx, l; int has_surface = 0; + bool update_brightness; crtc_state = drm_atomic_get_new_crtc_state(state, crtc); @@ -2025,7 +2031,11 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru req->swap.swap_completed = req->swap.swap_enabled; /* update brightness if changed */ - if (dcp_has_panel(dcp) && dcp->brightness.update) { + update_brightness = dcp_has_panel(dcp) && dcp->brightness.update; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + update_brightness = update_brightness && !dcp_backlight_active(dcp); +#endif + if (update_brightness) { req->swap.bl_unk = 1; req->swap.bl_value = dcp->brightness.dac; req->swap.bl_power = 0x40; diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c index 9fe7a0ce495aa..3133cd539b7f1 100644 --- a/drivers/gpu/drm/apple/systemep.c +++ b/drivers/gpu/drm/apple/systemep.c @@ -75,6 +75,12 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype struct apple_dcp *dcp = service->ep->dcp; int ret; + /* H17P takeover decoding requires an admitted record. Later zero reports + * must not overwrite the cached target or completed level. + */ + if (dcp_backlight_active(dcp)) + return 0; + dev_dbg(dcp->dev, "systemep[ch:%u]: report type:%02x len:%zu\n", service->channel, type, data_size); From 0da3b48fc8f7599050dea2f29a98cd65d9cb85aa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:12:50 +1100 Subject: [PATCH 0454/1579] drm: apple: test H17P backlight takeover and DPMS policy Exercise the clean SPEC takeover, cached-level restore and suspend ordering requirements. Verify that missing surfaces, busy transactions and rejected or stale completions do not change the completed level or lose pending requests. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 3 +- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp_backlight_test.c | 208 +++++++++++++++++++++ 3 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/apple/dcp_backlight_test.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index c4eff8fd0c5b3..c225708dff19b 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -34,7 +34,8 @@ config DRM_APPLE_KUNIT_TEST depends on KUNIT default KUNIT_ALL_TESTS help - Build tests for the Apple display driver's AFK ring parser. + Build tests for the Apple display driver's AFK ring parser and + backlight policy (takeover, DPMS, suspend and present completion). The tests cover the header strides used by older firmware and larger rings, as well as malformed ring size combinations. This option is intended for developer test kernels. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 90915b49e96c5..dc9ff905f283f 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -19,3 +19,4 @@ appledrm-$(CONFIG_TRACING) += trace.o obj-$(CONFIG_DRM_APPLE) += appledrm.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk_test.o +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_backlight_test.o diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c new file mode 100644 index 0000000000000..e990ea3e0bf06 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include + +#include "dcp_backlight.h" + +static void backlight_takeover_test(struct kunit *test) +{ + struct dcp_backlight_state state = {}; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, true, 36), 0); + KUNIT_EXPECT_EQ(test, state.target, 140U); + KUNIT_EXPECT_EQ(test, state.actual, 140U); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_TRUE(test, dcp_bl_seed(&state, 400)); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 601)); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 0)); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_EXPECT_EQ(test, state.actual, 400U); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 0, true, 36), 0); + KUNIT_EXPECT_EQ(test, state.target, 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, false, 0, true, 36), 0); + KUNIT_EXPECT_EQ(test, state.target, 36U); +} + +static void backlight_range_test(struct kunit *test) +{ + struct dcp_backlight_state state = {}; + + KUNIT_EXPECT_EQ(test, dcp_bl_init(&state, 600, false, 0, false, 0), -ENODATA); + KUNIT_EXPECT_FALSE(test, state.ready); + KUNIT_EXPECT_EQ(test, dcp_bl_request(&state, 140, false, false), -ENODATA); + KUNIT_EXPECT_EQ(test, dcp_bl_dpms(&state, false), -ENODATA); + KUNIT_EXPECT_EQ(test, dcp_bl_init(&state, 0, true, 0, false, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_bl_init(&state, U32_MAX, true, 0, false, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_bl_init(&state, 600, true, 601, false, 0), -ERANGE); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_request(&state, 601, false, false), -ERANGE); + KUNIT_EXPECT_EQ(test, state.target, 140U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 600, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 600U); +} + +static void backlight_dpms_cycles_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + int i; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 400, false, 0), 0); + for (i = 0; i < 20; i++) { + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 0U); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 400U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 400U); + } + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_EXPECT_EQ(test, state.target, 400U); +} + +static void backlight_blanked_target_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 400, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 140, false, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.target, 140U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 140U); + KUNIT_EXPECT_EQ(test, state.actual, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 140U); +} + +static void backlight_suspend_test(struct kunit *test) +{ + struct dcp_backlight_state state; + int off_first; + + for (off_first = 0; off_first < 2; off_first++) { + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + if (off_first) + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 140, true, true), 0); + if (!off_first) + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 36, true, true), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 36, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 36U); + KUNIT_EXPECT_EQ(test, state.target, 36U); + } + /* A core blank also keeps the panel dark independently of DRM DPMS. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, true, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 400U); +} + +static void backlight_surface_and_busy_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present first, second; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &first), -EALREADY); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 36, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, false, &first), -ENODATA); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_EXPECT_FALSE(test, state.in_flight); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &first), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &second), -EBUSY); + KUNIT_EXPECT_EQ(test, state.actual, 140U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, first.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 36U); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &second), 0); + KUNIT_EXPECT_EQ(test, second.nits, 400U); + KUNIT_EXPECT_FALSE(test, dcp_bl_complete(&state, first.sequence, true)); + KUNIT_EXPECT_TRUE(test, state.in_flight); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, second.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 400U); +} + +static void backlight_reject_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, false)); + KUNIT_EXPECT_EQ(test, state.actual, 140U); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_EXPECT_FALSE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_EXPECT_EQ(test, state.actual, 0U); +} + +static void backlight_inflight_dpms_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 140U); +} + +static struct kunit_case backlight_cases[] = { + KUNIT_CASE(backlight_takeover_test), + KUNIT_CASE(backlight_range_test), + KUNIT_CASE(backlight_dpms_cycles_test), + KUNIT_CASE(backlight_blanked_target_test), + KUNIT_CASE(backlight_suspend_test), + KUNIT_CASE(backlight_surface_and_busy_test), + KUNIT_CASE(backlight_reject_test), + KUNIT_CASE(backlight_inflight_dpms_test), + {} +}; + +static struct kunit_suite backlight_suite = { + .name = "apple-dcp-backlight", + .test_cases = backlight_cases, +}; + +kunit_test_suite(backlight_suite); + +MODULE_LICENSE("Dual MIT/GPL"); From 8d3922331b2ca82dcc8cb7effd263332d0635c98 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:12:51 +1100 Subject: [PATCH 0455/1579] docs: gpu: describe Apple H17P backlight admission contract Document the clean SPEC backlight policy, central queue and surface ownership interface, missing admission records, and hardware gates. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/gpu/apple-dcp-backlight.rst | 129 ++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 Documentation/gpu/apple-dcp-backlight.rst diff --git a/Documentation/gpu/apple-dcp-backlight.rst b/Documentation/gpu/apple-dcp-backlight.rst new file mode 100644 index 0000000000000..e47e7f3b4dd4b --- /dev/null +++ b/Documentation/gpu/apple-dcp-backlight.rst @@ -0,0 +1,129 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +Apple DCP H17P backlight integration +=================================== + +The backlight policy implements the clean display specification's BACKLIGHT-1 +through BACKLIGHT-4, with IOMFB-3 and LIFETIME-1 through LIFETIME-3 constraints. +It extends the existing Asahi backlight driver. It contains no H17P brightness +wire layout, calibration table, property command, or guessed initial level. +Policy activation requires explicit successful configuration. Existing +unconfigured H17P sessions, including T8142, keep their established DAC, +property/powerlog and DPMS paths. Older firmware remains on its existing path. +Neo startup must configure the policy before exposing the backlight or starting +its DPMS/brightness traffic; a firmware-version match alone does not activate +this policy. + +Admission and startup +--------------------- + +The central display worker must call ``dcp_backlight_configure()`` from process +context after linking the CRTC and admitting all required records. Its arguments +are the validated maximum commanded nits, an optional inherited level, an +optional documented ADT default, and a callback that schedules the central +outbound queue. Maximum must fit the backlight core's signed integer range. +An inherited zero is valid and takes precedence over the default. With neither +level available, configuration returns ``-ENODATA``. The policy backlight is registered only after configuration succeeds. +Configuration must precede any existing backlight registration; it returns +``-EBUSY`` if a device has already been published. Configuration does not send a command. + +The device exposes ``apple-panel-bl`` with platform type, linear scale and nits +units. Its maximum is the supplied ceiling; the older 509-nit DAC cap does not +apply to configured H17P policy. Consumer matching accepts the DCP device and its DRM device. +Registration freezes takeover before publishing the backlight. A decoded +powerlog value may seed the state through ``dcp_backlight_seed()`` before that +point. All samples after Linux takes control, including per-frame zeros, are +ignored. The older property and powerlog parsers do not seed an active policy implicitly. +Gather a sample before configuration and pass it as the inherited argument if +asynchronous registration could otherwise win the race. + +``brightness`` is the requested restore target. ``actual_brightness`` is the +last completed commanded level, initially the inherited/default estimate. This +is not a photometric measurement. Updates step immediately; no kernel fade or +initial-brightness/blank-level override is introduced. + +Queue and lifetime contract +--------------------------- + +The callback only schedules/coalesces work. It must not submit an RPC directly, +take DRM modeset locks, or run a present inline in a receive/commit callback. +It remains valid until the central queue and backlight workers are drained at +teardown. Policy access is protected by the brightness spinlock; the callback +is called after dropping it. Configuration also takes the registration mutex. + +The single outbound queue handles normal and brightness-only presents. A static +or inactive screen uses the last *accepted* surface, including its cached +geometry and IOVA. It must hold that surface's framebuffer reference and mapping +before calling ``dcp_backlight_prepare()`` with ``have_surface = true``. A DRM +plane state containing a rejected replacement is not an accepted surface. +The policy's boolean is a caller assertion, not a framebuffer reference. +Allocate any retirement bookkeeping during atomic prepare, never here. + +Prepare reserves a sequence and returns commanded nits. ``-ENODATA`` means no +configuration or surface; ``-EBUSY`` means a prior brightness transaction is +outstanding; ``-EALREADY`` means no brightness change is pending. These results +do not clear pending work. The serializer places the returned level only in +admitted fields of the next valid present. A normal present without a brightness +change keeps the inherited/completed level; its clean layout must support that. +The sequence is a software cookie, not a wire present ID. + +Associate the cookie with the central transaction and its strict wire ID. Call +``dcp_backlight_complete(sequence, true)`` only after both accepted submission +and matching completion. On serialization/submission rejection call it with +``false``; actual remains unchanged and the update is pending again. Wrong or +duplicate sequence callbacks have no effect. On a firmware crash retain the +surface and stop new submissions; there is no local reset/retry ladder. +After a transaction drains, inspect ``dcp_backlight_pending()`` and schedule the +next update or defer until the pipe becomes usable. Requests received during a +transaction remain pending. The central queue must bound static-screen latency +to 200 ms; this still requires a hardware timing test. + +A brightness-only completion never authorizes framebuffer retirement, including +retirement associated with a rejected replacement. Existing generic retention +helpers must be called only for the corresponding accepted replacement. This +policy never arms, releases, or unmaps a framebuffer. + +DPMS and suspend +---------------- + +Internal-panel H17P ``dcp_poweroff()`` and ``dcp_poweron()`` change the policy and +schedule the retained-surface queue. They bypass the legacy pipe-off/on path, +including surface clearing and final release. Until configuration, they warn +and leave the pipe intact; they cannot claim a completed hardware blank. +Startup must independently establish/adopt the live pipe before enabling the +policy. External outputs retain their existing power handling. + +DPMS off, a backlight core blank, or backlight core suspend forces effective nits +to zero while preserving the user target. Brightness changes during any blank +update the target. DPMS on restores it only when core blank/suspend is cleared. +The backlight core's suspend/resume callbacks feed this state; the central PM +worker still owns queue draining, domain lifetime and resume order. Do not call +unqualified H17P sleep/pipe-off methods as a suspend fallback. Keep the last +accepted surface mapped across soft-off and resume. + +Required records and qualification +---------------------------------- + +The following SPEC admission records must supply facts before production use: + +* R-ADT: panel ceiling/calibration in commanded nits and a documented default + if no inherited sample is available. No expected 525-nit ceiling is hardcoded. +* R-BACKLIGHT: takeover sample source, units, validity and timing; decoded + powerlog semantics; commanded-nits conversion, valid update/enable fields, + field offsets, widths and byte order in the present. +* R-SWAP: complete surface layout and successful submit/completion correlation + for normal and retained-surface presents, including inactive CRTC handling. +* R-DPMS: which admitted brightness/property actions blank and restore the live + pipe without clearing its surface, plus wake ordering. No new numeric property + or guessed DPMS payload is emitted by this policy. +* R-SERIALIZE: ordered brightness/present/property/diagnostic timeline and busy + deferral through the central queue; static-screen update latency. +* R-TEARDOWN: safe final release and suspend/resume boundary. Soft-off supplies + no such release boundary. + +Run ``apple-dcp-backlight`` KUnit tests with ``CONFIG_DRM_APPLE_KUNIT_TEST``. +They cover takeover/default/zero handling, range validation, twenty DPMS cycles, +cached restore, both suspend/DPMS orders, static-surface deferral, busy sequencing, +rejection and stale completion. They do not qualify firmware fields or GNOME. +Hardware acceptance requires O-BACKLIGHT, O-BACKLIGHT-TAKEOVER (10 Hz PBwo with +no jump above 0.05 W), O-BACKLIGHT-CRC, O-DPMS (twenty cycles), and O-SUSPEND. From 8d52d0bd5c9f7397621f23313d4f6fef90e03783 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:53:23 +1100 Subject: [PATCH 0456/1579] drm: apple: reject truncated firmware dictionary values Dictionary skips must validate payload bounds just like parsed values. Stop at the first error, bound recursive nesting, and propagate invalid container and skipped-value errors to each caller. Reject length truncation and cursor overflow, and preserve the audio look-ahead cursor until a complete key matches. Handle string allocation failure and failed event strings explicitly. These checks follow the existing public dictionary format and do not change valid payloads. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 199 ++++++++++++++++++++++----------- 1 file changed, 136 insertions(+), 63 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index df179afc87039..bfa30954b7a5a 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -35,11 +35,12 @@ struct dcp_parse_tag { static const void *parse_bytes(struct dcp_parse_ctx *ctx, size_t count) { - const void *ptr = ctx->blob + ctx->pos; + const void *ptr; - if (ctx->pos + count > ctx->len) + if (ctx->pos > ctx->len || count > ctx->len - ctx->pos) return ERR_PTR(-EINVAL); + ptr = ctx->blob + ctx->pos; ctx->pos += count; return ptr; } @@ -54,6 +55,8 @@ static const struct dcp_parse_tag *parse_tag(struct dcp_parse_ctx *ctx) const struct dcp_parse_tag *tag; /* Align to 32-bits */ + if (ctx->pos > U32_MAX - 3) + return ERR_PTR(-EINVAL); ctx->pos = round_up(ctx->pos, 4); tag = parse_bytes(ctx, sizeof(struct dcp_parse_tag)); @@ -81,38 +84,45 @@ static const struct dcp_parse_tag *parse_tag_of_type(struct dcp_parse_ctx *ctx, return tag; } -static int skip(struct dcp_parse_ctx *handle) +/* Bound stack use when ignoring nested firmware dictionaries. */ +#define DCP_PARSE_MAX_DEPTH 32 + +static int skip_value(struct dcp_parse_ctx *handle, unsigned int depth) { - const struct dcp_parse_tag *tag = parse_tag(handle); - int ret = 0; - int i; + const struct dcp_parse_tag *tag; + const void *data; + unsigned int count, i; + int ret; + if (depth >= DCP_PARSE_MAX_DEPTH) + return -EINVAL; + + tag = parse_tag(handle); if (IS_ERR(tag)) return PTR_ERR(tag); switch (tag->type) { case DCP_TYPE_DICTIONARY: - for (i = 0; i < tag->size; ++i) { - ret |= skip(handle); /* key */ - ret |= skip(handle); /* value */ - } - - return ret; - case DCP_TYPE_ARRAY: - for (i = 0; i < tag->size; ++i) - ret |= skip(handle); + count = tag->size; + if (tag->type == DCP_TYPE_DICTIONARY) + count *= 2; - return ret; + for (i = 0; i < count; ++i) { + ret = skip_value(handle, depth + 1); + if (ret) + return ret; + } + return 0; case DCP_TYPE_INT64: - handle->pos += sizeof(s64); - return 0; + data = parse_bytes(handle, sizeof(s64)); + break; case DCP_TYPE_STRING: case DCP_TYPE_BLOB: - handle->pos += tag->size; - return 0; + data = parse_bytes(handle, tag->size); + break; case DCP_TYPE_BOOL: return 0; @@ -120,6 +130,13 @@ static int skip(struct dcp_parse_ctx *handle) default: return -EINVAL; } + + return IS_ERR(data) ? PTR_ERR(data) : 0; +} + +static int skip(struct dcp_parse_ctx *handle) +{ + return skip_value(handle, 0); } #if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) @@ -136,23 +153,19 @@ static int skip_pair(struct dcp_parse_ctx *handle) static bool consume_string(struct dcp_parse_ctx *ctx, const char *specimen) { + struct dcp_parse_ctx probe = *ctx; const struct dcp_parse_tag *tag; const char *key; - ctx->pos = round_up(ctx->pos, 4); - if (ctx->pos + sizeof(*tag) + strlen(specimen) - 1 > ctx->len) - return false; - tag = ctx->blob + ctx->pos; - key = ctx->blob + ctx->pos + sizeof(*tag); - if (tag->padding) + tag = parse_tag_of_type(&probe, DCP_TYPE_STRING); + if (IS_ERR(tag) || tag->size != strlen(specimen)) return false; - if (tag->type != DCP_TYPE_STRING || - tag->size != strlen(specimen) || - strncmp(key, specimen, tag->size)) + key = parse_bytes(&probe, tag->size); + if (IS_ERR(key) || memcmp(key, specimen, tag->size)) return false; - skip(ctx); + *ctx = probe; return true; } #endif @@ -172,6 +185,8 @@ static char *parse_string(struct dcp_parse_ctx *handle) return (void *)in; out = kmalloc(tag->size + 1, GFP_KERNEL); + if (!out) + return ERR_PTR(-ENOMEM); memcpy(out, in, tag->size); out[tag->size] = '\0'; @@ -261,15 +276,13 @@ static int iterator_begin(struct dcp_parse_ctx *handle, struct iterator *it, return 0; } -#define dcp_parse_foreach_in_array(handle, it) \ - for (iterator_begin(handle, &it, false); it.idx < it.len; ++it.idx) -#define dcp_parse_foreach_in_dict(handle, it) \ - for (iterator_begin(handle, &it, true); it.idx < it.len; ++it.idx) - int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx) { const u32 *header; + if (size > U32_MAX) + return -EINVAL; + *ctx = (struct dcp_parse_ctx) { .blob = blob, .len = size, @@ -291,7 +304,11 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) struct iterator it; int ret = 0; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) @@ -307,7 +324,7 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) else if (!strcmp(key, "PreciseSyncRate")) ret = parse_int(it.handle, &dim->precise_sync_rate); else - skip(it.handle); + ret = skip(it.handle); if (!IS_ERR_OR_NULL(key)) kfree(key); @@ -367,7 +384,11 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, out->sdr.score = -1; out->best.score = -1; - dcp_parse_foreach_in_array(handle, outer_it) { + ret = iterator_begin(handle, &outer_it, false); + if (ret) + return ret; + + for (; outer_it.idx < outer_it.len; ++outer_it.idx) { struct iterator it; bool is_virtual = true; struct color_mode cmode = { @@ -380,7 +401,11 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, .score = -1, }; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) @@ -402,7 +427,7 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, else if (!strcmp(key, "Score")) ret = parse_int(it.handle, &cmode.score); else - skip(it.handle); + ret = skip(it.handle); if (!IS_ERR_OR_NULL(key)) kfree(key); @@ -470,13 +495,17 @@ static int parse_mode(struct dcp_parse_ctx *handle, }; *score = -1; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) ret = PTR_ERR(key); else if (is_virtual) - skip(it.handle); + ret = skip(it.handle); else if (!strcmp(key, "HorizontalAttributes")) ret = parse_dimension(it.handle, &horiz); else if (!strcmp(key, "VerticalAttributes")) @@ -494,7 +523,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, else if (!strcmp(key, "Score")) ret = parse_int(it.handle, score); else - skip(it.handle); + ret = skip(it.handle); if (!IS_ERR_OR_NULL(key)) kfree(key); @@ -647,7 +676,11 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, struct iterator it; s64 width_cm = 0, height_cm = 0; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) @@ -657,7 +690,7 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, else if (!strcmp(key, "MaxVerticalImageSize")) ret = parse_int(it.handle, &height_cm); else - skip(it.handle); + ret = skip(it.handle); if (!IS_ERR_OR_NULL(key)) kfree(key); @@ -685,7 +718,11 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, *name = ERR_PTR(-ENOENT); *class = ERR_PTR(-ENOENT); - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) { @@ -727,7 +764,7 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, if (!ret) parsed_unit = true; } else { - skip(it.handle); + ret = skip(it.handle); } kfree(key); @@ -840,14 +877,16 @@ static int parse_chmap(struct dcp_parse_ctx *handle, struct snd_pcm_chmap_elem * struct iterator it; int i, ret; - if (!chmap) { - skip(handle); - return 0; - } + if (!chmap) + return skip(handle); chmap->channels = 0; - dcp_parse_foreach_in_array(handle, it) { + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { for (i = 0; i < ARRAY_SIZE(chan_position_names); i++) if (consume_string(it.handle, chan_position_names[i].label)) break; @@ -875,7 +914,11 @@ static int parse_chan_layout_element(struct dcp_parse_ctx *handle, int ret; s64 nchans = 0; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { if (consume_string(it.handle, "ActiveChannelCount")) ret = parse_int(it.handle, &nchans); else if (consume_string(it.handle, "ChannelLayout")) @@ -900,7 +943,11 @@ static int parse_nchans_mask(struct dcp_parse_ctx *handle, unsigned int *mask) *mask = 0; - dcp_parse_foreach_in_array(handle, it) { + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { int nchans; ret = parse_chan_layout_element(it.handle, &nchans, NULL); @@ -920,7 +967,11 @@ static int parse_avep_element(struct dcp_parse_ctx *handle, struct iterator it; int ret; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { if (consume_string(handle, "StreamSampleRate")) ret = parse_sample_rate_bit(it.handle, &mask.rates); else if (consume_string(handle, "SampleSize")) @@ -958,12 +1009,20 @@ static int parse_mode_in_avep_element(struct dcp_parse_ctx *handle, struct dcp_parse_ctx save_handle; int ret; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { if (consume_string(it.handle, "AudioChannelLayoutElements")) { struct iterator inner_it; int nchans; - dcp_parse_foreach_in_array(it.handle, inner_it) { + ret = iterator_begin(it.handle, &inner_it, false); + if (ret) + return ret; + + for (; inner_it.idx < inner_it.len; ++inner_it.idx) { save_handle = *it.handle; ret = parse_chan_layout_element(inner_it.handle, &nchans, NULL); @@ -1016,7 +1075,11 @@ int parse_sound_constraints(struct dcp_parse_ctx *handle, hits->nchans = 0; } - dcp_parse_foreach_in_array(handle, it) { + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { ret = parse_avep_element(it.handle, sieve, hits); if (ret < 0) @@ -1035,7 +1098,11 @@ int parse_sound_mode(struct dcp_parse_ctx *handle, struct iterator it; int ret; - dcp_parse_foreach_in_array(handle, it) { + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { save_handle = *it.handle; ret = parse_avep_element(it.handle, sieve, NULL); @@ -1065,7 +1132,11 @@ int parse_system_log_mnits(struct dcp_parse_ctx *handle, struct dcp_system_ev_mn s64 timestamp = -1; bool type_match = false; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) { ret = PTR_ERR(key); @@ -1074,15 +1145,17 @@ int parse_system_log_mnits(struct dcp_parse_ctx *handle, struct dcp_system_ev_mn } else if (!strcmp(key, "iDAC")) { ret = parse_int(it.handle, &idac); } else if (!strcmp(key, "logEvent")) { - const char * value = parse_string(it.handle); - if (!IS_ERR_OR_NULL(value)) { + const char *value = parse_string(it.handle); + + ret = IS_ERR(value) ? PTR_ERR(value) : 0; + if (!ret) { type_match = strcmp(value, "Display (Event Forward)") == 0; kfree(value); } } else if (!strcmp(key, "timestamp")) { ret = parse_int(it.handle, ×tamp); } else { - skip(it.handle); + ret = skip(it.handle); } if (!IS_ERR_OR_NULL(key)) From 7f3a1c24c4b3aa3e43f9e140c1225d7237d7635b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:55:19 +1100 Subject: [PATCH 0457/1579] drm: apple: validate firmware timings before creating modes Initialize timing fields as absent and require an active area, total, porches, vertical refresh rate, timing ID and score. Validate each dimension before calculating the clock or narrowing geometry to DRM mode fields. Reject notch underflow, unrepresentable IDs and clocks, and sync intervals beyond the advertised total. The existing public dictionary parser supplies all inputs. Invalid timing entries remain recoverable so a later valid mode can still be advertised; explicitly zero porches remain valid. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 45 +++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index bfa30954b7a5a..dfe5b905cc43f 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -304,6 +304,14 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) struct iterator it; int ret = 0; + *dim = (struct dimension) { + .active = -1, + .total = -1, + .front_porch = -1, + .sync_width = -1, + .precise_sync_rate = -1, + }; + ret = iterator_begin(handle, &it, true); if (ret) return ret; @@ -314,15 +322,15 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) if (IS_ERR(key)) ret = PTR_ERR(key); else if (!strcmp(key, "Active")) - ret = parse_int(it.handle, &dim->active); + ret = parse_int_bound(it.handle, &dim->active, 0, U16_MAX); else if (!strcmp(key, "Total")) - ret = parse_int(it.handle, &dim->total); + ret = parse_int_bound(it.handle, &dim->total, 0, U16_MAX); else if (!strcmp(key, "FrontPorch")) - ret = parse_int(it.handle, &dim->front_porch); + ret = parse_int_bound(it.handle, &dim->front_porch, 0, U16_MAX); else if (!strcmp(key, "SyncWidth")) - ret = parse_int(it.handle, &dim->sync_width); + ret = parse_int_bound(it.handle, &dim->sync_width, 0, U16_MAX); else if (!strcmp(key, "PreciseSyncRate")) - ret = parse_int(it.handle, &dim->precise_sync_rate); + ret = parse_int_bound(it.handle, &dim->precise_sync_rate, 1, U32_MAX); else ret = skip(it.handle); @@ -333,6 +341,11 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) return ret; } + if (dim->active <= 0 || dim->total < dim->active || + dim->front_porch < 0 || dim->sync_width < 0 || + dim->front_porch + dim->sync_width > dim->total - dim->active) + return -EINVAL; + return 0; } @@ -349,6 +362,9 @@ struct color_mode { static int fill_color_mode(struct dcp_color_mode *color, struct color_mode *cmode) { + if (cmode->id < 0 || cmode->id > U32_MAX) + return -EINVAL; + if (color->score >= cmode->score) return 0; @@ -466,7 +482,7 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, * specifies the clock in kHz. The intermediate result may overflow a u32, so * use a u64 where required. */ -static u32 calculate_clock(struct dimension *horiz, struct dimension *vert) +static u64 calculate_clock(struct dimension *horiz, struct dimension *vert) { u32 pixels = horiz->total * vert->total; u64 clock = mul_u32_u32(pixels, vert->precise_sync_rate); @@ -480,8 +496,9 @@ static int parse_mode(struct dcp_parse_ctx *handle, { int ret = 0; struct iterator it; - struct dimension horiz = {}, vert = {}; + struct dimension horiz = { .active = -1 }, vert = { .active = -1 }; s64 min_vrr = 0, max_vrr = 0; + u64 clock; s64 id = -1; s64 best_color_mode = -1; bool is_virtual = false; @@ -517,11 +534,11 @@ static int parse_mode(struct dcp_parse_ctx *handle, else if (!strcmp(key, "ColorModes")) ret = parse_color_modes(it.handle, out); else if (!strcmp(key, "ID")) - ret = parse_int(it.handle, &id); + ret = parse_int_bound(it.handle, &id, 0, U32_MAX); else if (!strcmp(key, "IsVirtual")) ret = parse_bool(it.handle, &is_virtual); else if (!strcmp(key, "Score")) - ret = parse_int(it.handle, score); + ret = parse_int_bound(it.handle, score, 0, S64_MAX); else ret = skip(it.handle); @@ -548,7 +565,9 @@ static int parse_mode(struct dcp_parse_ctx *handle, /* * Reject modes without valid color mode. */ - if (best_color_mode < 0) + if (best_color_mode < 0 || id < 0 || *score < 0 || + horiz.active <= 0 || vert.active <= notch_height || + vert.precise_sync_rate <= 0) return -EINVAL; /* @@ -577,13 +596,17 @@ static int parse_mode(struct dcp_parse_ctx *handle, out->vrr = true; } + clock = calculate_clock(&horiz, &vert); + if (!clock || clock > INT_MAX) + return -EINVAL; + vert.active -= notch_height; vert.sync_width += notch_height; /* From here we must succeed. Start filling out the mode. */ *mode = (struct drm_display_mode) { .type = DRM_MODE_TYPE_DRIVER, - .clock = calculate_clock(&horiz, &vert), + .clock = clock, .vdisplay = vert.active, .vsync_start = vert.active + vert.front_porch, From 9546c4a7eacdfad2237bab45bd86e0bf197a6b1a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:57:28 +1100 Subject: [PATCH 0458/1579] dt-bindings: display: apple,dcp: describe bandwidth argument tuples Constrain each bandwidth property to a single provider reference with its resource index, display register index and, for scratch registers, byte offset. These are the arguments consumed by the existing public DCP driver and supplied by the Apple device trees. Add an example of the existing T8103 internal display resources and piodma child so the binding checks also exercise the argument tuples. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/display/apple,dcp.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index ac2286832ba7d..33f1bf6e44359 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -98,10 +98,21 @@ properties: apple,bw-scratch: $ref: /schemas/types.yaml#/definitions/phandle-array + items: + - items: + - description: Phandle of the PMGR bandwidth register provider. + - description: Index into the provider's reg property. + - description: Index in the firmware display register table. + - description: Byte offset of the scratch register within the resource. description: PMGR bandwidth scratch resource, display index and offset. apple,bw-doorbell: $ref: /schemas/types.yaml#/definitions/phandle-array + items: + - items: + - description: Phandle of the PMGR bandwidth register provider. + - description: Index into the provider's reg property. + - description: Index in the firmware display register table. description: Optional PMGR bandwidth doorbell resource and display index. apple,iomfb-surfaces: @@ -269,4 +280,32 @@ allOf: resets: false unevaluatedProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + dcp@231c00000 { + compatible = "apple,t8103-dcp", "apple,dcp"; + reg = <0x2 0x31c00000 0x0 0x4000>, + <0x2 0x30000000 0x0 0x3e8000>, + <0x2 0x31320000 0x0 0x4000>, + <0x2 0x31344000 0x0 0x4000>, + <0x2 0x31800000 0x0 0x800000>, + <0x2 0x3b3d0000 0x0 0x4000>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 0>; + apple,bw-scratch = <&pmgr_dcp 0 5 0x14>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + + piodma { + iommus = <&disp_dart 0>; + }; + }; + }; ... From 0a89f86921266e267f478fab4c6d3642f8bba53e Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:49:28 -0500 Subject: [PATCH 0459/1579] arm64: asahi.config: build the RDMA core and the soft RDMA drivers as modules The Apple Silicon kernel has CONFIG_INFINIBAND unset, so no node has an RDMA device, ib_uverbs or the rdma tools' kernel side. Build the RDMA core, its userspace verbs and management interfaces, and the two software providers (rxe, siw) as modules. With these, rdma-core and perftest work on any Mac with a network interface: `modprobe rdma_rxe; rdma link add rxe0 type rxe netdev ` gives a verbs device that needs no RDMA hardware. Nothing loads unless a user asks for it. All five are =m. In `make defconfig asahi.config` they resolve with their dependencies (NET_UDP_TUNNEL=m, IRQ_POLL=y, DIMLIB=y, DMA_SHARED_BUFFER=y, INFINIBAND_VIRT_DMA=y). --- arch/arm64/configs/asahi.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 20a50b6fcb3a1..7ce1ee4346471 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -78,3 +78,8 @@ CONFIG_APPLE_M1_CPU_PMU=y CONFIG_NVMEM_APPLE_EFUSES=m CONFIG_NVMEM_APPLE_SPMI=m CONFIG_MUX_APPLE_DPXBAR=m +CONFIG_INFINIBAND=m +CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_INFINIBAND_USER_MAD=m +CONFIG_RDMA_RXE=m +CONFIG_RDMA_SIW=m From ffa0cd3b0bed1b3c3c05bc8a20a557cdf99710f6 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 7 Oct 2026 11:15:00 -0400 Subject: [PATCH 0460/1579] HID: magicmouse: click through each force stage on the way up A deep click fired once per click and nothing marked leaving it: pressing gave a click and then a deep click, but easing off went straight from the deep stage to the final release with no feedback in between. macOS treats Force Touch as two stages and clicks at each boundary in both directions: press, deep, back to the first stage, release. Track the deep stage as a level rather than a one-shot flag. It ends at haptic_deep_release_threshold, well below where it starts, for the same reason the first stage has a gap between press and release: a finger held near the threshold must not chatter. Leaving it plays the release waveform, and the stage can be entered again within the same click. The default of 350 sits above the 150-270 that ordinary clicks peak at on a J613 and below its 500 deep threshold. Play the actuator's own force-click waveform for the deep stage on MTP (haptic_deep_waveform, on by default) instead of the press waveform with deep-click shaping, which on MTP has the same defaults as a press and so felt identical. Report the deep-click key from a separate "Force Click" input device and hold it for as long as the deep stage lasts. On the trackpad's own device it never reached userspace: libinput drives a touchpad with a dispatcher that drops every key that is not a button. Assisted-by: Claude:claude-opus-5-5 --- drivers/hid/hid-magicmouse.c | 162 ++++++++++++++++++++++++++++------- 1 file changed, 133 insertions(+), 29 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index e67e4ce60ab92..8eb69d187ce87 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -118,6 +118,22 @@ static u16 haptic_deep_threshold = 500; module_param(haptic_deep_threshold, ushort, 0644); MODULE_PARM_DESC(haptic_deep_threshold, "summed finger pressure for a deep click, 0 disables (default 500)"); +/* + * Force Touch has two stages, and macOS clicks on the way back up through + * each of them as well as on the way down: easing off a deep click to an + * ordinary press is felt, and so is the final release. The deep stage ends + * well below where it starts, for the same reason the first one does: a + * finger held near the threshold must not chatter. J613 ordinary clicks peak + * at 150-270 and firm presses at 700-910, so this sits above the former. + */ +static u16 haptic_deep_release_threshold = 350; +module_param(haptic_deep_release_threshold, ushort, 0644); +MODULE_PARM_DESC(haptic_deep_release_threshold, "summed finger pressure at which a deep click ends, below haptic_deep_threshold (default 350)"); + +static bool haptic_deep_waveform = true; +module_param(haptic_deep_waveform, bool, 0644); +MODULE_PARM_DESC(haptic_deep_waveform, "play the actuator's force-click waveform for a deep click (default on; off: the press waveform)"); + static u8 haptic_deep_pulses = 1; module_param(haptic_deep_pulses, byte, 0644); MODULE_PARM_DESC(haptic_deep_pulses, "pulses played for a deep click (default 1)"); @@ -128,12 +144,15 @@ MODULE_PARM_DESC(haptic_deep_gap_ms, "milliseconds between deep-click pulses, 0 /* * Linux has no key for a force click, so which one to send is the user's - * choice; nothing is sent unless one is named. It is read at probe because the - * capability has to be advertised when the input device is set up. + * choice; nothing is sent unless one is named. It is held down for as long as + * the deep stage lasts. The key comes from a separate input device: libinput + * drives a touchpad with its own dispatcher, which drops every key that is not + * a button, so a key on the trackpad's own device never reaches userspace. It + * is read at probe because that device is set up there. */ static u16 haptic_deep_keycode; module_param(haptic_deep_keycode, ushort, 0444); -MODULE_PARM_DESC(haptic_deep_keycode, "key code reported on a deep click, 0 for none (default 0)"); +MODULE_PARM_DESC(haptic_deep_keycode, "key code held during a deep click on a separate \"Force Click\" input device, 0 for none (default 0)"); static u8 haptic_pulse_gap_ms = 25; module_param(haptic_pulse_gap_ms, byte, 0644); @@ -282,8 +301,10 @@ struct magicmouse_sc { struct work_struct haptic_press_work; struct work_struct haptic_release_work; struct work_struct haptic_deep_work; + struct work_struct haptic_deep_release_work; + struct input_dev *force_input; bool haptic_button_down; - bool haptic_deep_fired; + bool haptic_deep_down; struct hid_device *hdev; struct delayed_work work; @@ -1478,7 +1499,7 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, } hid_dbg(msc->hdev, "played %s pulse %u/%u at strength 0x%02x\n", usage == (HID_HP_WAVEFORMRELEASE & HID_USAGE) ? - "release" : "press", + "release" : deep ? "deep" : "press", i + 1, pulses, strength); if (i + 1 < pulses) msleep(gap); @@ -1531,8 +1552,32 @@ static void magicmouse_deep_worker(struct work_struct *ws) if (!haptic_press) return; - magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMPRESS & HID_USAGE, true, - haptic_deep_pulses); + magicmouse_haptic_pulses(msc, haptic_deep_waveform ? + APPLE_HP_WAVEFORMDEEPCLICK & HID_USAGE : + HID_HP_WAVEFORMPRESS & HID_USAGE, + true, haptic_deep_pulses); +} + +/* Easing off a deep click to an ordinary press: the first of two releases. */ +static void magicmouse_deep_release_worker(struct work_struct *ws) +{ + struct magicmouse_sc *msc = + container_of(ws, struct magicmouse_sc, haptic_deep_release_work); + + if (!haptic_press) + return; + + magicmouse_haptic_pulses(msc, HID_HP_WAVEFORMRELEASE & HID_USAGE, false, + max_t(u8, haptic_release_pulses, 1)); +} + +static void magicmouse_force_key(struct magicmouse_sc *msc, bool down) +{ + if (!msc->force_input) + return; + + input_report_key(msc->force_input, haptic_deep_keycode, down); + input_sync(msc->force_input); } static void magicmouse_release_worker(struct work_struct *ws) @@ -1575,7 +1620,10 @@ static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, if (!haptic_press && !host_mode) { /* The firmware owns the actuator and reports the button. */ msc->haptic_button_down = false; - msc->haptic_deep_fired = false; + if (msc->haptic_deep_down) { + msc->haptic_deep_down = false; + magicmouse_force_key(msc, false); + } return firmware_button; } @@ -1590,29 +1638,39 @@ static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, if (!haptic_press) queue_work(msc->haptics->wq, &msc->haptic_release_work); - if (!msc->haptic_button_down && pressure >= haptic_press_threshold) { - msc->haptic_button_down = true; - msc->haptic_deep_fired = false; - hid_dbg(msc->hdev, "click at pressure %u\n", pressure); - queue_work(msc->haptics->wq, &msc->haptic_press_work); - } else if (msc->haptic_button_down && - pressure <= haptic_release_threshold) { + /* + * Three levels, up, clicked and deep, each step with its own click on + * the way down and on the way back up. One step per frame: a press hard + * enough to reach the deep stage at once still clicks twice, a frame + * apart. + */ + if (!msc->haptic_button_down) { + if (pressure >= haptic_press_threshold) { + msc->haptic_button_down = true; + hid_dbg(msc->hdev, "click at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_press_work); + } + } else if (pressure <= haptic_release_threshold) { msc->haptic_button_down = false; - msc->haptic_deep_fired = false; + if (msc->haptic_deep_down) { + msc->haptic_deep_down = false; + magicmouse_force_key(msc, false); + } hid_dbg(msc->hdev, "release at pressure %u\n", pressure); queue_work(msc->haptics->wq, &msc->haptic_release_work); - } else if (msc->haptic_button_down && !msc->haptic_deep_fired && - haptic_deep_threshold && - pressure >= haptic_deep_threshold) { - /* Pressing harder without lifting: once per click, not while held. */ - msc->haptic_deep_fired = true; - hid_dbg(msc->hdev, "deep click at pressure %u\n", pressure); - queue_work(msc->haptics->wq, &msc->haptic_deep_work); - - if (haptic_deep_keycode) { - input_report_key(msc->input, haptic_deep_keycode, 1); - input_report_key(msc->input, haptic_deep_keycode, 0); + } else if (!msc->haptic_deep_down) { + if (haptic_deep_threshold && pressure >= haptic_deep_threshold) { + msc->haptic_deep_down = true; + hid_dbg(msc->hdev, "deep click at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_deep_work); + magicmouse_force_key(msc, true); } + } else if (pressure <= min(haptic_deep_release_threshold, + haptic_deep_threshold)) { + msc->haptic_deep_down = false; + hid_dbg(msc->hdev, "deep release at pressure %u\n", pressure); + queue_work(msc->haptics->wq, &msc->haptic_deep_release_work); + magicmouse_force_key(msc, false); } return msc->haptic_button_down; @@ -1633,7 +1691,10 @@ static void magicmouse_haptic_forget_mode(struct magicmouse_sc *msc) msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; msc->haptic_button_down = false; - msc->haptic_deep_fired = false; + if (msc->haptic_deep_down) { + msc->haptic_deep_down = false; + magicmouse_force_key(msc, false); + } } #else static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, @@ -1959,6 +2020,46 @@ static void apple_taptic_destroy(struct ff_device *ff) msc->haptic_effects = NULL; } +/* + * The deep-click key lives on its own input device (see haptic_deep_keycode). + * It is managed by devres, so it goes away with the trackpad. Failing to set + * it up costs only the key, not the click feedback. + */ +static void magicmouse_setup_force_input(struct magicmouse_sc *msc) +{ + struct hid_device *hdev = msc->hdev; + struct input_dev *input; + int ret; + + if (haptic_deep_keycode > KEY_MAX) + return; + + input = devm_input_allocate_device(&hdev->dev); + if (!input) + return; + + input->name = devm_kasprintf(&hdev->dev, GFP_KERNEL, "%s Force Click", + hdev->name); + if (!input->name) + return; + input->phys = hdev->phys; + input->uniq = hdev->uniq; + input->id.bustype = hdev->bus; + input->id.vendor = hdev->vendor; + input->id.product = hdev->product; + input->id.version = hdev->version; + input->dev.parent = &hdev->dev; + input_set_capability(input, EV_KEY, haptic_deep_keycode); + + ret = input_register_device(input); + if (ret) { + hid_warn(hdev, "no force click key (%d)\n", ret); + return; + } + + msc->force_input = input; +} + static int apple_taptic_init_mtp(struct magicmouse_sc *msc, struct hid_haptic_device *haptic_dev) { @@ -1989,9 +2090,11 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, INIT_WORK(&msc->haptic_press_work, magicmouse_press_worker); INIT_WORK(&msc->haptic_release_work, magicmouse_release_worker); INIT_WORK(&msc->haptic_deep_work, magicmouse_deep_worker); + INIT_WORK(&msc->haptic_deep_release_work, + magicmouse_deep_release_worker); if (haptic_deep_keycode) - input_set_capability(msc->input, EV_KEY, haptic_deep_keycode); + magicmouse_setup_force_input(msc); /* FF_HAPTIC has to be in ffbit before the FF device is created. */ input_set_capability(msc->input, EV_FF, FF_HAPTIC); @@ -2071,6 +2174,7 @@ static void magicmouse_teardown_haptics(struct magicmouse_sc *msc) cancel_work_sync(&msc->haptic_press_work); cancel_work_sync(&msc->haptic_release_work); cancel_work_sync(&msc->haptic_deep_work); + cancel_work_sync(&msc->haptic_deep_release_work); /* * Tear the force feedback device down while msc is still valid: an From f98c8a7b9a27eeaa0fb391193f04a5905049deb4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:39:03 +1100 Subject: [PATCH 0461/1579] drm: apple: use the measured H17P late-init input The Linux-side startup record R-IOMFB-startup-run21 observes A000 with four zero input bytes. The public template instead sends one, as required by older firmware. Use zero for H17P while keeping the older variants' late-init requests unchanged. Keep the existing H17G input as well: the record admits only the H17P method profile. This corrects one measured input in the existing Asahi boot chain; it does not qualify the rest of startup or enable another device. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index f91e394446b11..e8f422e59b0ed 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -904,8 +904,12 @@ static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) static void boot_4b(struct apple_dcp *dcp, void *out, void *cookie) { #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u32 v_true = 1; - dcp_late_init_signal(dcp, false, &v_true, boot_5, NULL); +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + u32 value = dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; +#else + u32 value = 1; +#endif + dcp_late_init_signal(dcp, false, &value, boot_5, NULL); #else dcp_late_init_signal(dcp, false, boot_5, NULL); #endif From c298c21495a37eb98197265ab87f1a8a32b2c345 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:43:41 +1100 Subject: [PATCH 0462/1579] drm: apple: validate measured H17P callback sizes H17P dispatch currently trusts the firmware's input and output lengths before running typed callback handlers. A short record can make a handler read outside its input or write beyond its reply. Check the envelope and channel stack bounds before reading the header, then require a recorded callback tuple before clearing or dispatching its output. R-IOMFB-startup-run21 supplies the startup and present tuples; R-ANALYTICS supplies the D114 tuple. Stop the H17P interface on a mismatch rather than advancing its callback stack or sending subsequent calls. Older firmware and the existing H17G tuple dispatch stay unchanged. Tuple admission establishes sizes, not callback semantics or full device support. H17P callbacks absent from the measured set fail closed until an interoperability record admits them. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 27 ++++++++++++- drivers/gpu/drm/apple/iomfb.h | 2 + drivers/gpu/drm/apple/iomfb_h17p.c | 62 ++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index d034f2c83705b..78f49bcf9b5d9 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -163,6 +163,9 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca { struct dcp_method_entry resolved = *call; + if (dcp->fw_compat == DCP_FIRMWARE_H17P && READ_ONCE(dcp->crashed)) + return; + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G && call->tag_h17g[0]) memcpy(resolved.tag, call->tag_h17g, sizeof(resolved.tag)); @@ -332,10 +335,29 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, struct device *dev = dcp->dev; struct dcp_packet_header *hdr = data; void *in, *out; - int tag = dcp_parse_tag(hdr->tag); + int tag; struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + (length < sizeof(*hdr) || !ch || ch->depth >= DCP_MAX_CALL_DEPTH || + offset >= 0x8000 || length > 0x8000 - offset)) { + dev_err(dev, "invalid IOMFB callback envelope\n"); + dcp->crashed = true; + return; + } + + tag = dcp_parse_tag(hdr->tag); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + ((u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || + (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !iomfb_validate_callback_h17p(tag, hdr->in_len, hdr->out_len)))) { + dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", + tag, hdr->in_len, hdr->out_len); + dcp->crashed = true; + return; + } + in = data + sizeof(*hdr); out = in + hdr->in_len; @@ -705,6 +727,9 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) { enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && READ_ONCE(dcp->crashed)) + return; + if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) { /* * H17P reports its interface version (bits 63:48) and a diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 482f6e61d33c3..d17704012b1c6 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -206,6 +206,8 @@ struct dcp_method_entry { }; #define IOMFB_MAX_CB (1000) + +bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len); struct apple_dcp; typedef bool (*iomfb_cb_handler)(struct apple_dcp *, int, void *, void *); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 28f219b8be14e..81681b8e66982 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -7,6 +7,68 @@ #include "iomfb_v13_3.h" #include "version_utils.h" +struct dcp_callback_size { + u32 in_len; + u32 out_len; + bool valid; +}; + +/* R-IOMFB-startup-run21 and R-ANALYTICS Linux boundary observations. */ +static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { + [0] = { 0x0, 0x4, true }, + [1] = { 0x0, 0x4, true }, + [3] = { 0x4, 0x14, true }, + [6] = { 0x54, 0x50, true }, + [100] = { 0x0, 0x0, true }, + [101] = { 0x0, 0x4, true }, + [102] = { 0x44, 0x0, true }, + [104] = { 0x44, 0x0, true }, + [108] = { 0x0, 0x4, true }, + [109] = { 0x0, 0x4, true }, + [110] = { 0x0, 0x4, true }, + [111] = { 0x0, 0x4, true }, + [112] = { 0x0, 0x4, true }, + [113] = { 0x0, 0x4, true }, + [114] = { 0x1044, 0x1004, true }, + [121] = { 0x0, 0x4, true }, + [123] = { 0x0, 0x4, true }, + [125] = { 0x64, 0x24, true }, + [127] = { 0x4, 0x4, true }, + [128] = { 0x1008, 0x4, true }, + [129] = { 0x40, 0x4, true }, + [201] = { 0xc, 0x10, true }, + [206] = { 0x0, 0x4, true }, + [207] = { 0x0, 0x4, true }, + [300] = { 0x10, 0x0, true }, + [400] = { 0x4c, 0xc04, true }, + [401] = { 0x50, 0xc, true }, + [406] = { 0x48, 0x0, true }, + [411] = { 0x10, 0x1c, true }, + [413] = { 0x1048, 0x4, true }, + [414] = { 0x50, 0x4, true }, + [415] = { 0x4c, 0x4, true }, + [451] = { 0x14, 0x1c, true }, + [552] = { 0x1044, 0x4, true }, + [561] = { 0x1044, 0x4, true }, + [563] = { 0x4c, 0x4, true }, + [565] = { 0x48, 0x4, true }, + [574] = { 0x4, 0x8, true }, + [575] = { 0x58, 0x4c, true }, + [582] = { 0x8, 0x4, true }, + [590] = { 0x730, 0x0, true }, + [599] = { 0x0, 0x0, true }, +}; + +bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len) +{ + const struct dcp_callback_size *size; + + if (tag < 0 || tag >= ARRAY_SIZE(callback_sizes)) + return false; + size = &callback_sizes[tag]; + return size->valid && size->in_len == in_len && size->out_len == out_len; +} + struct dcp_h17p_hotplug_request { __le64 connected; u8 tiled_display[0x4c]; From 91cd9f040ac73922facfcec15afc06d998094a76 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:44:54 +1100 Subject: [PATCH 0463/1579] drm: apple: answer H17P analytics with an empty dictionary R-ANALYTICS records four complete Linux-side D114 exchanges with a 0x1044-byte request and a 0x1004-byte reply. Every reply begins with the OSDictionary tag 'd' and has zero remaining bytes; the public Asahi OSDictionary decoder accepts it as an empty dictionary. Encode that reply for the measured H17P profile after callback-size validation. Keep the existing H17G response, whose dictionary contract was not measured. This extends the existing Asahi callback table without introducing an arbitrary-call interface or a recovery retry. Byte equivalence to the recorded replies is checked separately from the required two-hour survival gate, which remains to be run on the device. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 81681b8e66982..7555e01c9cc92 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -186,6 +186,19 @@ static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, return true; } +static bool trampoline_analytics_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_zero(dcp, tag, out, in); + + trace_iomfb_callback(dcp, tag, __func__); + /* The dispatcher has validated the measured D114 reply size. */ + memset(out, 0, 0x1004); + *(u8 *)out = 'd'; + return true; +} + static bool trampoline_hotplug_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -237,7 +250,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [111] = trampoline_true, /* create_iomfb_service */ [112] = trampoline_create_backlight_service, /* create_backlight_service */ [113] = trampoline_true, /* create_nvram_service */ - [114] = trampoline_zero, /* CoreAnalyticsSendEvent */ + [114] = trampoline_analytics_h17p, /* CoreAnalyticsSendEvent */ [117] = trampoline_nop, /* set_idle_caching_state_ap */ [118] = trampoline_zero, /* upload_trace_start */ [119] = trampoline_zero, /* upload_trace_chunk */ From ddcd3dc7af901d9383bbc80972eff5effc87f143 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:49:42 +1100 Subject: [PATCH 0464/1579] drm: apple: serialize H17P presents at the measured boundary The public H17 template starts its surfaces after a 0x58a-byte C swap record and repairs their fields in place. SPEC PRESENT-2 requires a 0x588-byte wire boundary. R-SWAP-controlled-surface-run21 independently locates format, width, height, stride and per-plane dimensions/byte count at offsets matching Asahi's existing surface record at that boundary. Serialize H17P into separate device-owned staging storage, copying the swap prefix, unchanged public surfaces and trailing arguments to their wire positions. Keep the total A408 input at 0xe9c bytes, zero the final two bytes and preserve the native atomic input for repeated encoding. H17G keeps its existing public packing path. Static size/offset checks and randomized byte maps cover every byte of the transfer. Nine populated fields match the controlled Linux capture. This does not establish unmeasured field semantics or qualify resource ownership, formats, backlight or a functional display boot. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/iomfb_h17p.c | 18 +++++++++++++++ drivers/gpu/drm/apple/iomfb_h17p.h | 14 +++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 32 ++++++++++++++++++++++---- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 2262fb98f3cc8..4e69415cc3658 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -228,6 +228,8 @@ struct apple_dcp { struct dcp_swap_submit_req_v13_3 v13_3; struct dcp_swap_submit_req_h17p h17p; } swap; + /* Staging wire record; serialization never mutates the atomic inputs. */ + struct dcp_present_h17p present_h17p; /* swap id of the last completed swap */ u32 last_swap_id; diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 7555e01c9cc92..ccc33b70fc3d5 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -88,6 +88,24 @@ static_assert(sizeof(struct dcp_h17p_swap_info_request) == 0xe4); static_assert(sizeof(struct dc_swap_complete_resp_h17p) == 0x730); static_assert(offsetof(struct dc_swap_complete_resp_h17p, swap_id) == 0); +void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, + const struct dcp_swap_submit_req_h17p *request) +{ + size_t tail_offset = offsetof(struct dcp_swap_submit_req_h17p, surf_iova); + size_t tail_size = sizeof(*request) - tail_offset; + + static_assert(sizeof(request->swap) == sizeof(wire->swap) + 2); + static_assert(sizeof(request->surf) == sizeof(wire->surf)); + static_assert(sizeof(*request) - + offsetof(struct dcp_swap_submit_req_h17p, surf_iova) == + sizeof(wire->tail) - 2); + + memcpy(wire->swap, &request->swap, sizeof(wire->swap)); + memcpy(wire->surf, request->surf, sizeof(wire->surf)); + memcpy(wire->tail, (const u8 *)request + tail_offset, tail_size); + memset(wire->tail + tail_size, 0, sizeof(wire->tail) - tail_size); +} + static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A000", dcpep_late_init_signal), IOMFB_METHOD_H17("A025", "A029", dcpep_setup_video_limits), /* (0,0) */ diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 97c0da9e05b94..e9f4dfb6dd855 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -16,6 +16,20 @@ static_assert(sizeof(struct dcp_surface_h17p) == 0x22c); static_assert(sizeof(struct dcp_swap_submit_req_h17p) == 0xe9c); static_assert(sizeof(struct dcp_swap_submit_resp_h17p) == 0x0c); +/* The H17P swap boundary is independent of the template's native layout. */ +struct dcp_present_h17p { + u8 swap[0x588]; + struct dcp_surface_h17p surf[SWAP_SURFACES]; + u8 tail[0x64]; +} __packed; + +static_assert(sizeof(struct dcp_present_h17p) == 0xe9c); +static_assert(offsetof(struct dcp_present_h17p, surf) == 0x588); +static_assert(offsetof(struct dcp_present_h17p, tail) == 0xe38); + +void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, + const struct dcp_swap_submit_req_h17p *request); + #undef DCP_FW_VER #undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index e8f422e59b0ed..aee23045a5a33 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -61,9 +61,26 @@ IOMFB_THUNK_INOUT(get_color_remap_mode); IOMFB_THUNK_INOUT(last_client_close); IOMFB_THUNK_INOUT(abort_swaps_dcp); +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER +static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, + struct dcp_swap_submit_req_h17p *request, + dcp_callback_t cb, void *cookie) +{ + const void *data = request; + + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + iomfb_serialize_present_h17p(&dcp->present_h17p, request); + data = &dcp->present_h17p; + } + + dcp_push(dcp, oob, &dcp_methods[dcpep_swap_submit], sizeof(*request), + sizeof(struct dcp_swap_submit_resp_h17p), (void *)data, cb, cookie); +} +#else DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct DCP_FW_NAME(dcp_swap_submit_req), struct DCP_FW_NAME(dcp_swap_submit_resp)); +#endif DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct DCP_FW_NAME(dcp_swap_start_req), struct DCP_FW_NAME(dcp_swap_start_resp)); @@ -1160,9 +1177,10 @@ static void release_swap_cookie(struct kref *ref) * format two bytes late and its surface check rejects the swap as an * unsupported format. * - * Shift each surface's tail down by two rather than fork the shared struct. + * H17G retains this public packing path. The measured H17P profile uses + * a separate wire record with the SPEC's 0x588 swap boundary. */ -static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) +static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); unsigned int i; @@ -1182,6 +1200,10 @@ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) req->swap.h17p_pre_swap_id[0x00] = 2; req->swap.h17p_pre_swap_id[0x28] = 3; + /* The measured H17P layout is serialized without shifting its surfaces. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return; + for (i = 0; i < SWAP_SURFACES; i++) { u8 *p = (u8 *)&req->surf[i]; size_t n = sizeof(req->surf[i]); @@ -1211,7 +1233,7 @@ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) } } #else -static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) { } +static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { } #endif static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) @@ -1254,7 +1276,7 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, if (info) info->swap_id = resp->swap_id; - dcp_h17p_fix_swap_surfaces(dcp); + dcp_h17p_prepare_swap(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); } @@ -1740,7 +1762,7 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) dcp_arm_retained_framebuffers(dcp, resp->swap_id); trace_iomfb_swap_submit(dcp, resp->swap_id); - dcp_h17p_fix_swap_surfaces(dcp); + dcp_h17p_prepare_swap(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swapped, NULL); } From b6491b0fb4db1189d929ac426cec8415e31d6ff6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:56:32 +1100 Subject: [PATCH 0465/1579] drm: apple: require accepted H17P present IDs before retirement R-SWAP's controlled captures correlate the A406-assigned ID, A408 input and D590 completion. Every retained run21 completion follows its matching submit acknowledgment. The existing completion handler retires buffers and signals a flip without checking whether that ID was accepted. Track one started and accepted H17P present in the RTKit receive thread. Reject overlapping starts, inconsistent submit state, early completion, wrong IDs and duplicate completion before touching framebuffer retirement or DRM flip state. Stop the interface on protocol inconsistency. Rejected submits cancel the reservation while the existing retention code unarms their references; a later unsolicited completion cannot retire them. Apply the same policy to ordinary and clear swaps, preserving older firmware and H17G behavior. This is a lifecycle guard, not a substitute for the outbound transaction queue or hardware lifetime qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 6 ++- drivers/gpu/drm/apple/iomfb_h17p.h | 40 ++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 63 +++++++++++++++++++++++++- 3 files changed, 105 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 4e69415cc3658..255a74d3ba266 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -228,8 +228,6 @@ struct apple_dcp { struct dcp_swap_submit_req_v13_3 v13_3; struct dcp_swap_submit_req_h17p h17p; } swap; - /* Staging wire record; serialization never mutates the atomic inputs. */ - struct dcp_present_h17p present_h17p; /* swap id of the last completed swap */ u32 last_swap_id; @@ -351,6 +349,10 @@ struct apple_dcp { struct dcp_backlight_state state; void (*kick)(struct apple_dcp *dcp); } backlight; + + /* Staging wire record; serialization never mutates the atomic inputs. */ + struct dcp_present_h17p present_h17p; + struct dcp_present_state_h17p present_state_h17p; }; void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index e9f4dfb6dd855..2b260ebb2c86b 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -16,6 +16,46 @@ static_assert(sizeof(struct dcp_surface_h17p) == 0x22c); static_assert(sizeof(struct dcp_swap_submit_req_h17p) == 0xe9c); static_assert(sizeof(struct dcp_swap_submit_resp_h17p) == 0x0c); +/* Owned by the RTKit receive thread, independently of DRM state lifetime. */ +struct dcp_present_state_h17p { + u32 swap_id; + bool pending; + bool accepted; +}; + +static inline bool +dcp_present_begin_h17p(struct dcp_present_state_h17p *state, u32 swap_id) +{ + if (state->pending) + return false; + state->swap_id = swap_id; + state->pending = true; + state->accepted = false; + return true; +} + +static inline bool +dcp_present_submit_h17p(struct dcp_present_state_h17p *state, u32 swap_id, + bool accepted) +{ + if (!state->pending || state->accepted || state->swap_id != swap_id) + return false; + state->accepted = accepted; + if (!accepted) + state->pending = false; + return true; +} + +static inline bool +dcp_present_complete_h17p(struct dcp_present_state_h17p *state, u32 swap_id) +{ + if (!state->pending || !state->accepted || state->swap_id != swap_id) + return false; + state->pending = false; + state->accepted = false; + return true; +} + /* The H17P swap boundary is independent of the template's native layout. */ struct dcp_present_h17p { u8 swap[0x588]; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index aee23045a5a33..52d1f7c5aa02a 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -241,6 +241,15 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct DCP_FW_NAME(dc_swap_complete_resp) *resp) { ktime_t now = ktime_get(); + +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_present_complete_h17p(&dcp->present_state_h17p, resp->swap_id)) { + dev_err(dcp->dev, "unexpected present completion %u\n", resp->swap_id); + dcp->crashed = true; + return; + } +#endif trace_iomfb_swap_complete(dcp, resp->swap_id); dcp->last_swap_id = resp->swap_id; dcp_release_retained_framebuffers(dcp, resp->swap_id); @@ -1236,6 +1245,32 @@ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { } #endif +static bool dcp_present_begin(struct apple_dcp *dcp, u32 swap_id) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_present_begin_h17p(&dcp->present_state_h17p, swap_id)) { + dev_err(dcp->dev, "overlapping present %u\n", swap_id); + dcp->crashed = true; + return false; + } +#endif + return true; +} + +static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_present_submit_h17p(&dcp->present_state_h17p, swap_id, accepted)) { + dev_err(dcp->dev, "unexpected present submission %u\n", swap_id); + dcp->crashed = true; + return false; + } +#endif + return true; +} + static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; @@ -1243,6 +1278,9 @@ static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; u32 status = resp ? resp->ret : ~0U; + if (!dcp_present_submit(dcp, swap_id, !status)) + status = ~0U; + if (status) { dev_err(dcp->dev, "swap_clear failed! status %u\n", status); dcp_unarm_retained_framebuffers(dcp, swap_id); @@ -1270,6 +1308,15 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, dcp_drm_crtc_vblank(dcp->crtc); return; } + if (!dcp_present_begin(dcp, resp->swap_id)) { + if (info) { + WRITE_ONCE(info->status, ~0U); + complete(&info->done); + kref_put(&info->refcount, release_swap_cookie); + } + dcp_drm_crtc_vblank(dcp->crtc); + return; + } DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; dcp_arm_retained_framebuffers(dcp, resp->swap_id); @@ -1738,9 +1785,17 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + u32 status = resp ? resp->ret : ~0U; +#else + u32 status = resp->ret; +#endif - if (resp->ret) { - dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); + if (!dcp_present_submit(dcp, swap_id, !status)) + status = ~0U; + + if (status) { + dev_err(dcp->dev, "swap failed! status %u\n", status); dcp_unarm_retained_framebuffers(dcp, swap_id); dcp_drm_crtc_vblank(dcp->crtc); return; @@ -1758,6 +1813,10 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) dcp_drm_crtc_vblank(dcp->crtc); return; } + if (!dcp_present_begin(dcp, resp->swap_id)) { + dcp_drm_crtc_vblank(dcp->crtc); + return; + } DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; dcp_arm_retained_framebuffers(dcp, resp->swap_id); From 5f9574c081d54a4aaa419bb2ee89d53ca2a9d253 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:23:57 +1100 Subject: [PATCH 0466/1579] drm: apple: serialize H17P command and present transactions RTKit replies can issue nested calls while atomic commits and power requests arrive from other threads. Channel depth alone cannot distinguish those callers, and preparing a second present can overwrite the request still awaiting the first submit reply. Queue H17P top-level operations in FIFO order and serialize transmit preparation with the receive thread. Copy standalone command inputs and hold atomic state through the reply chain and matching present completion. Keep nested calls in their originating transaction. A submit ACK alone does not let the next operation reuse the present record. Bound the pending queue and stop new operations on a ten-second timeout. Keep the active operation pinned on failure because a timeout does not prove firmware has stopped using it. An expired watchdog from a previous operation checks the new deadline before reporting a timeout. Older firmware and the H17G method profile keep their existing transport. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 25 +++ drivers/gpu/drm/apple/dcp.c | 2 + drivers/gpu/drm/apple/iomfb.c | 295 ++++++++++++++++++++++++++- 3 files changed, 319 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 255a74d3ba266..ae012bc3577ec 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -29,6 +29,13 @@ struct apple_dcp; struct apple_dcp_afkep; + +/* One operation, including all of its nested replies and present completion. */ +struct iomfb_transaction { + struct list_head link; + void (*start)(struct apple_dcp *dcp, struct iomfb_transaction *transaction); + void (*release)(struct iomfb_transaction *transaction); +}; struct apple_dcp_typec_port; struct apple_dcp_typec_route { @@ -353,8 +360,26 @@ struct apple_dcp { /* Staging wire record; serialization never mutates the atomic inputs. */ struct dcp_present_h17p present_h17p; struct dcp_present_state_h17p present_state_h17p; + + /* Serializes H17P transmit preparation with RTKit receive callbacks. */ + struct { + /* Protects queue, channel stacks, and the current owner. */ + struct mutex lock; + struct list_head pending; + struct iomfb_transaction *active; + struct task_struct *owner; + struct work_struct work; + struct delayed_work timeout; + unsigned long deadline; + unsigned int queued; + bool stopped; + } iomfb; }; +void iomfb_queue_init(struct apple_dcp *dcp); +void iomfb_queue_stop(struct apple_dcp *dcp); +int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); + void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); int dcp_backlight_register(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 6b6828db5b506..8578886688320 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2193,6 +2193,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->shmem) iomfb_shutdown(dcp); + iomfb_queue_stop(dcp); dcp_release_piodma_iommu_dev(dcp); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { @@ -2242,6 +2243,7 @@ static int dcp_platform_probe(struct platform_device *pdev) INIT_LIST_HEAD(&dcp->swapped_out_fbs); mutex_init(&dcp->swapped_out_fbs_lock); spin_lock_init(&dcp->backlight.lock); + iomfb_queue_init(dcp); dcp->fw_compat = fw_compat; dcp->dev = dev; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 78f49bcf9b5d9..bfcf7542ec8d7 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -156,6 +157,195 @@ static u8 dcp_pop_depth(u8 *depth) return --(*depth); } +/* Older firmware and the H17G method profile retain their existing transport. */ +static bool iomfb_uses_queue(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G; +} + +static bool iomfb_channels_idle(struct apple_dcp *dcp) +{ + return !dcp->ch_cmd.depth && !dcp->ch_cb.depth && + !dcp->ch_oobcmd.depth && !dcp->ch_oobcb.depth && + !dcp->ch_async.depth && !dcp->ch_oobasync.depth; +} + +static void iomfb_discard_pending(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction, *next; + + lockdep_assert_held(&dcp->iomfb.lock); + list_for_each_entry_safe(transaction, next, &dcp->iomfb.pending, link) { + list_del(&transaction->link); + dcp->iomfb.queued--; + transaction->release(transaction); + } +} + +static void iomfb_queue_advance(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction = dcp->iomfb.active; + + lockdep_assert_held(&dcp->iomfb.lock); + if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { + iomfb_discard_pending(dcp); + /* The active operation may still be visible to firmware. */ + return; + } + if (!iomfb_channels_idle(dcp) || dcp->present_state_h17p.pending) + return; + + if (transaction) { + dcp->iomfb.active = NULL; + cancel_delayed_work(&dcp->iomfb.timeout); + transaction->release(transaction); + } + if (!list_empty(&dcp->iomfb.pending)) + schedule_work(&dcp->iomfb.work); +} + +static void iomfb_queue_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, iomfb.work); + struct iomfb_transaction *transaction; + + mutex_lock(&dcp->iomfb.lock); + if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { + iomfb_discard_pending(dcp); + goto unlock; + } + if (dcp->iomfb.active || !iomfb_channels_idle(dcp) || + dcp->present_state_h17p.pending || list_empty(&dcp->iomfb.pending)) + goto unlock; + + transaction = list_first_entry(&dcp->iomfb.pending, + struct iomfb_transaction, link); + list_del(&transaction->link); + dcp->iomfb.queued--; + dcp->iomfb.active = transaction; + WRITE_ONCE(dcp->iomfb.owner, current); + dcp->iomfb.deadline = jiffies + msecs_to_jiffies(10000); + mod_delayed_work(system_wq, &dcp->iomfb.timeout, msecs_to_jiffies(10000)); + transaction->start(dcp, transaction); + WRITE_ONCE(dcp->iomfb.owner, NULL); + iomfb_queue_advance(dcp); +unlock: + mutex_unlock(&dcp->iomfb.lock); +} + +static void iomfb_queue_timeout(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), + struct apple_dcp, iomfb.timeout); + + mutex_lock(&dcp->iomfb.lock); + if (dcp->iomfb.active && time_before(jiffies, dcp->iomfb.deadline)) { + mod_delayed_work(system_wq, &dcp->iomfb.timeout, + dcp->iomfb.deadline - jiffies); + } else if (dcp->iomfb.active) { + WRITE_ONCE(dcp->crashed, true); + dev_err(dcp->dev, "IOMFB transaction timed out\n"); + iomfb_discard_pending(dcp); + schedule_work(&dcp->vblank_wq); + } + mutex_unlock(&dcp->iomfb.lock); +} + +void iomfb_queue_init(struct apple_dcp *dcp) +{ + mutex_init(&dcp->iomfb.lock); + INIT_LIST_HEAD(&dcp->iomfb.pending); + INIT_WORK(&dcp->iomfb.work, iomfb_queue_work); + INIT_DELAYED_WORK(&dcp->iomfb.timeout, iomfb_queue_timeout); +} + +void iomfb_queue_stop(struct apple_dcp *dcp) +{ + if (!iomfb_uses_queue(dcp)) + return; + + mutex_lock(&dcp->iomfb.lock); + dcp->iomfb.stopped = true; + iomfb_discard_pending(dcp); + mutex_unlock(&dcp->iomfb.lock); + cancel_work_sync(&dcp->iomfb.work); + cancel_delayed_work_sync(&dcp->iomfb.timeout); +} + +int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) +{ + int ret = 0; + + mutex_lock(&dcp->iomfb.lock); + if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { + ret = -EIO; + } else if (dcp->iomfb.queued >= 32) { + ret = -EBUSY; + } else { + dcp->iomfb.queued++; + list_add_tail(&transaction->link, &dcp->iomfb.pending); + schedule_work(&dcp->iomfb.work); + } + mutex_unlock(&dcp->iomfb.lock); + return ret; +} + +struct iomfb_command { + struct iomfb_transaction transaction; + struct dcp_method_entry method; + dcp_callback_t callback; + void *cookie; + u32 in_len; + u32 out_len; + bool oob; + u8 data[]; +}; + +static void iomfb_command_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + struct iomfb_command *command = container_of(transaction, + struct iomfb_command, transaction); + + dcp_push(dcp, command->oob, &command->method, command->in_len, + command->out_len, command->data, command->callback, command->cookie); +} + +static void iomfb_command_release(struct iomfb_transaction *transaction) +{ + kfree(container_of(transaction, struct iomfb_command, transaction)); +} + +static int iomfb_queue_command(struct apple_dcp *dcp, bool oob, + const struct dcp_method_entry *method, + u32 in_len, u32 out_len, void *data, + dcp_callback_t callback, void *cookie) +{ + struct iomfb_command *command; + int ret; + + if ((u64)sizeof(struct dcp_packet_header) + in_len + out_len > 0x8000) + return -EMSGSIZE; + command = kzalloc(struct_size(command, data, in_len), GFP_KERNEL); + if (!command) + return -ENOMEM; + command->transaction.start = iomfb_command_start; + command->transaction.release = iomfb_command_release; + command->method = *method; + command->callback = callback; + command->cookie = cookie; + command->in_len = in_len; + command->out_len = out_len; + command->oob = oob; + if (in_len) + memcpy(command->data, data, in_len); + ret = iomfb_queue(dcp, &command->transaction); + if (ret) + iomfb_command_release(&command->transaction); + return ret; +} + /* Call a DCP function given by a tag */ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, u32 in_len, u32 out_len, void *data, dcp_callback_t cb, @@ -163,6 +353,14 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca { struct dcp_method_entry resolved = *call; + /* Reply chains stay on the serialized receiver; callers enqueue copies. */ + if (iomfb_uses_queue(dcp) && READ_ONCE(dcp->iomfb.owner) != current) { + if (iomfb_queue_command(dcp, oob, call, in_len, out_len, + data, cb, cookie)) + WRITE_ONCE(dcp->crashed, true); + return; + } + if (dcp->fw_compat == DCP_FIRMWARE_H17P && READ_ONCE(dcp->crashed)) return; @@ -185,8 +383,21 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca .tag[3] = call->tag[0], }; - u8 depth = dcp_push_depth(&ch->depth); - u16 offset = dcp_packet_start(ch, depth); + u8 depth; + u16 offset; + + if (iomfb_uses_queue(dcp) && + (!ch || ch->depth >= DCP_MAX_CALL_DEPTH || + (u64)sizeof(header) + in_len + out_len > 0x8000 || + dcp_packet_start(ch, ch->depth) > + 0x8000 - ALIGN(sizeof(header) + in_len + out_len, + DCP_PACKET_ALIGNMENT))) { + dev_err(dcp->dev, "invalid IOMFB command envelope\n"); + WRITE_ONCE(dcp->crashed, true); + return; + } + depth = dcp_push_depth(&ch->depth); + offset = dcp_packet_start(ch, depth); void *out = dcp->shmem + dcp_tx_offset(context) + offset; void *out_data = out + sizeof(header); @@ -647,11 +858,63 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, } +struct iomfb_atomic_transaction { + struct iomfb_transaction transaction; + struct drm_atomic_state *state; + struct drm_crtc *crtc; +}; + +static void iomfb_atomic_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + struct iomfb_atomic_transaction *atomic = container_of(transaction, + struct iomfb_atomic_transaction, transaction); + + if (dcp->valid_mode && dcp->connector && dcp->connector->connected) + iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); + else + schedule_work(&dcp->vblank_wq); +} + +static void iomfb_atomic_release(struct iomfb_transaction *transaction) +{ + struct iomfb_atomic_transaction *atomic = container_of(transaction, + struct iomfb_atomic_transaction, transaction); + + drm_atomic_state_put(atomic->state); + kfree(atomic); +} + +static void iomfb_queue_atomic(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct iomfb_atomic_transaction *atomic; + + atomic = kzalloc_obj(*atomic); + if (!atomic) + goto failed; + atomic->transaction.start = iomfb_atomic_start; + atomic->transaction.release = iomfb_atomic_release; + atomic->state = drm_atomic_state_get(state); + atomic->crtc = crtc; + if (!iomfb_queue(dcp, &atomic->transaction)) + return; + iomfb_atomic_release(&atomic->transaction); +failed: + WRITE_ONCE(dcp->crashed, true); + schedule_work(&dcp->vblank_wq); +} + void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) { struct platform_device *pdev = to_apple_crtc(crtc)->dcp; struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (iomfb_uses_queue(dcp)) { + iomfb_queue_atomic(dcp, crtc, state); + return; + } + /* * DCP does not complete swaps after a link loss. A plane-only commit * arriving between the reconnect callback and the required modeset @@ -723,7 +986,7 @@ bool dcp_is_initialized(struct platform_device *pdev) return dcp->active; } -void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) +static void iomfb_recv_message(struct apple_dcp *dcp, u64 message) { enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); @@ -748,11 +1011,37 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) dev_warn(dcp->dev, "Ignoring unknown message %llx\n", message); } +void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) +{ + if (!iomfb_uses_queue(dcp)) { + iomfb_recv_message(dcp, message); + return; + } + + mutex_lock(&dcp->iomfb.lock); + WRITE_ONCE(dcp->iomfb.owner, current); + if (!dcp->iomfb.stopped) + iomfb_recv_message(dcp, message); + WRITE_ONCE(dcp->iomfb.owner, NULL); + iomfb_queue_advance(dcp); + mutex_unlock(&dcp->iomfb.lock); +} + int iomfb_start_rtkit(struct apple_dcp *dcp) { dma_addr_t shmem_iova; int ret; + if (iomfb_uses_queue(dcp)) { + mutex_lock(&dcp->iomfb.lock); + if (dcp->iomfb.active) { + mutex_unlock(&dcp->iomfb.lock); + return -EBUSY; + } + dcp->iomfb.stopped = false; + mutex_unlock(&dcp->iomfb.lock); + } + /* * H17P firmware expects the remote allocator endpoint to be started * before IOMFB. A firmware subsystem whose endpoint was never started From d3a90452d1896bad6ac9df20d1a86cc4470cd846 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:33:56 +1100 Subject: [PATCH 0467/1579] drm: apple: pin the last completed H17P scanout A rejected atomic replacement can leave DRM state referring to a surface that the display never accepted. Re-presenting that state for a backlight update would bypass the previous submit failure and retire the wrong framebuffers. Keep an independent snapshot with framebuffer references for each accepted surface. Stage replacements from the atomic operation and publish them only after the accepted submit's matching completion. Merge unchanged planes and preserve the background on partial updates. Rejected operations release only their staged snapshot, leaving the completed scanout pinned. Retain the snapshot on soft-off or firmware failure. It supplies no proof of a final DMA stop, and rebinding an unconfirmed session is refused. Older firmware and the H17G method profile retain their existing path. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 9 ++ drivers/gpu/drm/apple/iomfb.c | 117 ++++++++++++++++++++++++- drivers/gpu/drm/apple/iomfb_template.c | 2 + 3 files changed, 124 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index ae012bc3577ec..0a7b498711cb4 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -30,6 +30,12 @@ struct apple_dcp; struct apple_dcp_afkep; +/* Snapshot of a completed present, independent of the latest DRM state. */ +struct iomfb_scanout_h17p { + struct dcp_swap_submit_req_h17p request; + struct drm_framebuffer *fb[SWAP_SURFACES]; +}; + /* One operation, including all of its nested replies and present completion. */ struct iomfb_transaction { struct list_head link; @@ -373,9 +379,12 @@ struct apple_dcp { unsigned long deadline; unsigned int queued; bool stopped; + struct iomfb_scanout_h17p *scanout; + struct iomfb_scanout_h17p *next_scanout; } iomfb; }; +void iomfb_scanout_complete_h17p(struct apple_dcp *dcp); void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index bfcf7542ec8d7..99fc8894683af 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -858,10 +858,105 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, } +static void iomfb_scanout_release_h17p(struct iomfb_scanout_h17p *scanout) +{ + unsigned int i; + + if (!scanout) + return; + for (i = 0; i < SWAP_SURFACES; i++) + if (scanout->fb[i]) + drm_framebuffer_put(scanout->fb[i]); + kfree(scanout); +} + +static struct iomfb_scanout_h17p * +iomfb_scanout_prepare_h17p(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct iomfb_scanout_h17p *scanout; + struct drm_plane *plane; + struct drm_plane_state *old_state, *new_state; + unsigned int i, slot; + int index; + + lockdep_assert_held(&dcp->iomfb.lock); + scanout = kzalloc_obj(*scanout); + if (!scanout) + return NULL; + if (dcp->iomfb.scanout) { + scanout->request = dcp->iomfb.scanout->request; + for (i = 0; i < SWAP_SURFACES; i++) { + scanout->fb[i] = dcp->iomfb.scanout->fb[i]; + if (scanout->fb[i]) + drm_framebuffer_get(scanout->fb[i]); + } + } + + for_each_oldnew_plane_in_state(state, plane, old_state, new_state, index) { + if (old_state->crtc != crtc && new_state->crtc != crtc) + continue; + slot = to_apple_plane(plane)->iomfb_surf; + if (slot >= SWAP_SURFACES) { + iomfb_scanout_release_h17p(scanout); + return NULL; + } + if (scanout->fb[slot]) + drm_framebuffer_put(scanout->fb[slot]); + scanout->fb[slot] = NULL; + if (new_state->crtc == crtc && new_state->visible && new_state->fb) { + scanout->fb[slot] = new_state->fb; + drm_framebuffer_get(new_state->fb); + } + } + return scanout; +} + +void iomfb_scanout_complete_h17p(struct apple_dcp *dcp) +{ + struct iomfb_scanout_h17p *scanout = dcp->iomfb.next_scanout; + struct iomfb_scanout_h17p *previous = dcp->iomfb.scanout; + struct dcp_swap_submit_req_h17p *request = &dcp->swap.h17p; + unsigned int i; + + lockdep_assert_held(&dcp->iomfb.lock); + if (!scanout) + return; + + /* Keep common present arguments, including the null output pointers. */ + scanout->request = *request; + /* Preserve unchanged planes when only a subset was presented. */ + for (i = 0; i < SWAP_SURFACES; i++) { + if (!(request->swap.swap_enabled & BIT(i)) && previous) { + scanout->request.surf[i] = previous->request.surf[i]; + scanout->request.surf_iova[i] = previous->request.surf_iova[i]; + scanout->request.swap.src_rect[i] = previous->request.swap.src_rect[i]; + scanout->request.swap.dst_rect[i] = previous->request.swap.dst_rect[i]; + scanout->request.swap.surf_ids[i] = previous->request.swap.surf_ids[i]; + scanout->request.swap.surf_flags[i] = previous->request.swap.surf_flags[i]; + scanout->request.swap.surf_unk[i] = previous->request.swap.surf_unk[i]; + } + scanout->request.surf_null[i] = !scanout->fb[i]; + } + /* Brightness re-presents all pinned surfaces with a fresh swap ID. */ + scanout->request.swap.swap_enabled = IOMFB_SET_BACKGROUND; + for (i = 0; i < SWAP_SURFACES; i++) + if (scanout->fb[i]) + scanout->request.swap.swap_enabled |= BIT(i); + scanout->request.swap.swap_completed = scanout->request.swap.swap_enabled; + if (!(request->swap.swap_enabled & IOMFB_SET_BACKGROUND) && previous) + scanout->request.swap.bg_color = previous->request.swap.bg_color; + dcp->iomfb.scanout = scanout; + dcp->iomfb.next_scanout = NULL; + iomfb_scanout_release_h17p(previous); +} + struct iomfb_atomic_transaction { struct iomfb_transaction transaction; struct drm_atomic_state *state; struct drm_crtc *crtc; + struct apple_dcp *dcp; + struct iomfb_scanout_h17p *scanout; }; static void iomfb_atomic_start(struct apple_dcp *dcp, @@ -870,10 +965,18 @@ static void iomfb_atomic_start(struct apple_dcp *dcp, struct iomfb_atomic_transaction *atomic = container_of(transaction, struct iomfb_atomic_transaction, transaction); - if (dcp->valid_mode && dcp->connector && dcp->connector->connected) - iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); - else + if (!dcp->valid_mode || !dcp->connector || !dcp->connector->connected) { schedule_work(&dcp->vblank_wq); + return; + } + atomic->scanout = iomfb_scanout_prepare_h17p(dcp, atomic->crtc, atomic->state); + if (!atomic->scanout) { + WRITE_ONCE(dcp->crashed, true); + schedule_work(&dcp->vblank_wq); + return; + } + dcp->iomfb.next_scanout = atomic->scanout; + iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); } static void iomfb_atomic_release(struct iomfb_transaction *transaction) @@ -881,6 +984,11 @@ static void iomfb_atomic_release(struct iomfb_transaction *transaction) struct iomfb_atomic_transaction *atomic = container_of(transaction, struct iomfb_atomic_transaction, transaction); + if (atomic->scanout && atomic->dcp->iomfb.scanout != atomic->scanout) { + if (atomic->dcp->iomfb.next_scanout == atomic->scanout) + atomic->dcp->iomfb.next_scanout = NULL; + iomfb_scanout_release_h17p(atomic->scanout); + } drm_atomic_state_put(atomic->state); kfree(atomic); } @@ -897,6 +1005,7 @@ static void iomfb_queue_atomic(struct apple_dcp *dcp, struct drm_crtc *crtc, atomic->transaction.release = iomfb_atomic_release; atomic->state = drm_atomic_state_get(state); atomic->crtc = crtc; + atomic->dcp = dcp; if (!iomfb_queue(dcp, &atomic->transaction)) return; iomfb_atomic_release(&atomic->transaction); @@ -1034,7 +1143,7 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) if (iomfb_uses_queue(dcp)) { mutex_lock(&dcp->iomfb.lock); - if (dcp->iomfb.active) { + if (dcp->iomfb.active || dcp->iomfb.scanout) { mutex_unlock(&dcp->iomfb.lock); return -EBUSY; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 52d1f7c5aa02a..b3df548b52245 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -249,6 +249,8 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, dcp->crashed = true; return; } + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + iomfb_scanout_complete_h17p(dcp); #endif trace_iomfb_swap_complete(dcp, resp->swap_id); dcp->last_swap_id = resp->swap_id; From 701863f39f2857bb5ce6ffdbbfa027f2f669b116 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:37:53 +1100 Subject: [PATCH 0468/1579] drm: apple: encode H17P commanded nits without floating point Linux-side captures during ordinary backlight sysfs changes show integer commanded levels in a little-endian binary64 field at offset 0x35e of the present record. Controlled levels of 2, 36, 140 and 400 nits produce those exact bytes while a retained framebuffer's blend CRC remains constant. Encode integer nits by constructing the exponent and mantissa, avoiding floating-point operations in the kernel. Write only the measured eight bytes in the staging record. This helper does not enable the backlight or assign meaning to surrounding calibration and control fields. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 18 ++++++++++++++++++ drivers/gpu/drm/apple/iomfb_h17p.h | 2 ++ 2 files changed, 20 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index ccc33b70fc3d5..b49a587f34472 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright The Asahi Linux Contributors */ +#include +#include + #include "dcp-link.h" #include "iomfb_h17p.h" #include "iomfb_v12_3.h" @@ -106,6 +109,21 @@ void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, memset(wire->tail + tail_size, 0, sizeof(wire->tail) - tail_size); } +/* Integer commanded nits are exact binary64 values; no kernel FP is needed. */ +void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits) +{ + u64 value = 0; + unsigned int exponent; + + if (nits) { + exponent = fls(nits) - 1; + value = (u64)(exponent + 1023) << 52; + value |= ((u64)nits << (52 - exponent)) & GENMASK_ULL(51, 0); + } + /* Measured from Linux presents during ordinary sysfs brightness changes. */ + put_unaligned_le64(value, wire->swap + 0x35e); +} + static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A000", dcpep_late_init_signal), IOMFB_METHOD_H17("A025", "A029", dcpep_setup_video_limits), /* (0,0) */ diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 2b260ebb2c86b..0121d334c214a 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -67,6 +67,8 @@ static_assert(sizeof(struct dcp_present_h17p) == 0xe9c); static_assert(offsetof(struct dcp_present_h17p, surf) == 0x588); static_assert(offsetof(struct dcp_present_h17p, tail) == 0xe38); +void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits); + void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, const struct dcp_swap_submit_req_h17p *request); From 2ba26ced478d339a2887d60231bf71d8df982f25 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:46:59 +1100 Subject: [PATCH 0469/1579] drm: apple: couple H17P backlight updates to completed presents Backlight updates need a valid surface even on a static screen or while DRM considers the internal CRTC inactive. Use the transaction queue to re-present the pinned, completed scanout. Coalesce brightness requests and reserve the policy sequence when preparing the wire record. Update cached actual nits only after accepted submission and its matching completion. Rejection cancels the reservation without updating actual. Brightness-only completions neither arm retirement nor release displaced framebuffers or signal another commit's flip event. Requests made during a successful present remain pending for the next operation; rejection does not create an automatic firmware retry loop. Expose a configuration hook requiring admitted range and takeover inputs; do not enable it with guessed values. Internal H17P DPMS preserves the pipe and requests zero or the retained target, reporting unavailable policy until configured. Preserve older firmware and H17G handling. Prepare any external H17P clear record inside the serialized reply chain, so a power request cannot overwrite a present still in flight. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 15 +++ drivers/gpu/drm/apple/dcp.c | 12 +- drivers/gpu/drm/apple/dcp_backlight.c | 3 + drivers/gpu/drm/apple/iomfb.c | 120 ++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 147 +++++++++++++++++-------- drivers/gpu/drm/apple/iomfb_template.h | 4 +- 6 files changed, 252 insertions(+), 49 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 0a7b498711cb4..a73eaa90ee5ae 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -41,6 +41,10 @@ struct iomfb_transaction { struct list_head link; void (*start)(struct apple_dcp *dcp, struct iomfb_transaction *transaction); void (*release)(struct iomfb_transaction *transaction); + struct dcp_backlight_present backlight; + bool backlight_reserved; + bool brightness_only; + bool completed; }; struct apple_dcp_typec_port; @@ -379,12 +383,23 @@ struct apple_dcp { unsigned long deadline; unsigned int queued; bool stopped; + bool backlight_queued; struct iomfb_scanout_h17p *scanout; struct iomfb_scanout_h17p *next_scanout; } iomfb; }; void iomfb_scanout_complete_h17p(struct apple_dcp *dcp); +void iomfb_present_backlight_h17p(struct apple_dcp *dcp); +bool iomfb_present_brightness_only_h17p(struct apple_dcp *dcp); +bool iomfb_present_complete_h17p(struct apple_dcp *dcp); +void iomfb_present_failed_h17p(struct apple_dcp *dcp); +bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, + const struct dcp_swap_submit_req_h17p *request, + struct dcp_present_h17p *wire); +int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits); void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8578886688320..116c80f5d902c 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1483,12 +1483,20 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) } } +static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) +{ + return (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) || + (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + dcp->connector_type == DRM_MODE_CONNECTOR_eDP); +} + void dcp_poweron(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; - if (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) { + if (dcp_uses_soft_dpms(dcp)) { ret = dcp_backlight_dpms(dcp, true); if (ret) dev_warn(dcp->dev, "backlight restore unavailable: %d\n", ret); @@ -1544,7 +1552,7 @@ void dcp_poweroff(struct platform_device *pdev) int ret; /* Internal H17P DPMS is a brightness present, never a pipe stop. */ - if (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) { + if (dcp_uses_soft_dpms(dcp)) { ret = dcp_backlight_dpms(dcp, false); if (ret) dev_warn(dcp->dev, "backlight blank unavailable: %d\n", ret); diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 5c7f296eaf798..61d05b93c6698 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -174,6 +174,9 @@ int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, spin_lock_irqsave(&dcp->backlight.lock, flags); ret = dcp_bl_prepare(&dcp->backlight.state, have_surface, present); + /* No reservation: carry the last completed level on an ordinary present. */ + if (ret == -EALREADY) + present->nits = dcp->backlight.state.actual; spin_unlock_irqrestore(&dcp->backlight.lock, flags); return ret; } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 99fc8894683af..8288cad699aad 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -179,6 +179,8 @@ static void iomfb_discard_pending(struct apple_dcp *dcp) list_for_each_entry_safe(transaction, next, &dcp->iomfb.pending, link) { list_del(&transaction->link); dcp->iomfb.queued--; + if (transaction->brightness_only) + dcp->iomfb.backlight_queued = false; transaction->release(transaction); } } @@ -199,6 +201,11 @@ static void iomfb_queue_advance(struct apple_dcp *dcp) if (transaction) { dcp->iomfb.active = NULL; cancel_delayed_work(&dcp->iomfb.timeout); + if (transaction->brightness_only) + dcp->iomfb.backlight_queued = false; + /* A new level requested during a successful present stays pending. */ + if (transaction->completed && dcp_backlight_pending(dcp)) + schedule_work(&dcp->bl_update_wq); transaction->release(transaction); } if (!list_empty(&dcp->iomfb.pending)) @@ -280,9 +287,13 @@ int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) mutex_lock(&dcp->iomfb.lock); if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { ret = -EIO; + } else if (transaction->brightness_only && dcp->iomfb.backlight_queued) { + ret = -EALREADY; } else if (dcp->iomfb.queued >= 32) { ret = -EBUSY; } else { + if (transaction->brightness_only) + dcp->iomfb.backlight_queued = true; dcp->iomfb.queued++; list_add_tail(&transaction->link, &dcp->iomfb.pending); schedule_work(&dcp->iomfb.work); @@ -951,6 +962,115 @@ void iomfb_scanout_complete_h17p(struct apple_dcp *dcp) iomfb_scanout_release_h17p(previous); } +bool iomfb_present_brightness_only_h17p(struct apple_dcp *dcp) +{ + return dcp->iomfb.active && dcp->iomfb.active->brightness_only; +} + +bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, + const struct dcp_swap_submit_req_h17p *request, + struct dcp_present_h17p *wire) +{ + struct iomfb_transaction *transaction = dcp->iomfb.active; + struct dcp_backlight_present present; + bool have_surface = false; + unsigned int i; + int ret; + + lockdep_assert_held(&dcp->iomfb.lock); + if (!dcp_backlight_active(dcp)) + return true; + if (!transaction) + return false; + for (i = 0; i < SWAP_SURFACES; i++) + have_surface |= !request->surf_null[i]; + ret = dcp_backlight_prepare(dcp, have_surface, &present); + if (ret && ret != -EALREADY) + return false; + if (!ret) { + transaction->backlight = present; + transaction->backlight_reserved = true; + } + iomfb_encode_backlight_h17p(wire, present.nits); + return true; +} + +void iomfb_present_failed_h17p(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction = dcp->iomfb.active; + + if (transaction && transaction->backlight_reserved) { + dcp_backlight_complete(dcp, transaction->backlight.sequence, false); + transaction->backlight_reserved = false; + } +} + +bool iomfb_present_complete_h17p(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction = dcp->iomfb.active; + + lockdep_assert_held(&dcp->iomfb.lock); + if (transaction) { + transaction->completed = true; + if (transaction->backlight_reserved) { + dcp_backlight_complete(dcp, transaction->backlight.sequence, true); + transaction->backlight_reserved = false; + } + if (transaction->brightness_only) + return false; + } + iomfb_scanout_complete_h17p(dcp); + return true; +} + +static void iomfb_backlight_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + struct iomfb_scanout_h17p *scanout = dcp->iomfb.scanout; + bool have_surface = false; + unsigned int i; + + if (!dcp_backlight_pending(dcp) || !dcp->valid_mode || !scanout || + !dcp->connector || !dcp->connector->connected) + return; + for (i = 0; i < SWAP_SURFACES; i++) + have_surface |= !!scanout->fb[i]; + if (!have_surface) + return; + + dcp->swap.h17p = scanout->request; + iomfb_present_backlight_h17p(dcp); +} + +static void iomfb_backlight_release(struct iomfb_transaction *transaction) +{ + kfree(transaction); +} + +static void iomfb_backlight_kick(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction; + + transaction = kzalloc_obj(*transaction); + if (!transaction) + return; + transaction->start = iomfb_backlight_start; + transaction->release = iomfb_backlight_release; + transaction->brightness_only = true; + if (iomfb_queue(dcp, transaction)) + kfree(transaction); +} + +int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits) +{ + if (!iomfb_uses_queue(dcp)) + return -EINVAL; + return dcp_backlight_configure(dcp, maximum, inherited_valid, inherited, + default_valid, default_nits, iomfb_backlight_kick); +} + struct iomfb_atomic_transaction { struct iomfb_transaction transaction; struct drm_atomic_state *state; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index b3df548b52245..da73a2dee5ffa 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -70,6 +70,11 @@ static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { iomfb_serialize_present_h17p(&dcp->present_h17p, request); + if (!iomfb_apply_backlight_h17p(dcp, request, &dcp->present_h17p)) { + if (cb) + cb(dcp, NULL, cookie); + return; + } data = &dcp->present_h17p; } @@ -241,6 +246,7 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct DCP_FW_NAME(dc_swap_complete_resp) *resp) { ktime_t now = ktime_get(); + bool retire = true; #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && @@ -250,13 +256,14 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, return; } if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - iomfb_scanout_complete_h17p(dcp); + retire = iomfb_present_complete_h17p(dcp); #endif trace_iomfb_swap_complete(dcp, resp->swap_id); dcp->last_swap_id = resp->swap_id; - dcp_release_retained_framebuffers(dcp, resp->swap_id); - - dcp_drm_crtc_page_flip(dcp, now); + if (retire) { + dcp_release_retained_framebuffers(dcp, resp->swap_id); + dcp_drm_crtc_page_flip(dcp, now); + } if (dcp->crc_enabled) { u32 crc32 = 0; drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); @@ -1247,6 +1254,23 @@ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { } #endif +static bool dcp_present_retires(struct apple_dcp *dcp) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return !iomfb_present_brightness_only_h17p(dcp); +#endif + return true; +} + +static void dcp_present_failed(struct apple_dcp *dcp) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + iomfb_present_failed_h17p(dcp); +#endif +} + static bool dcp_present_begin(struct apple_dcp *dcp, u32 swap_id) { #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER @@ -1273,6 +1297,43 @@ static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted return true; } +static void dcp_prepare_clear_swap(struct apple_dcp *dcp, void *request) +{ + typeof(DCP_FW_UNION(dcp->swap)) *swap = request; + + /* Clear surfaces. */ + memset(swap, 0, sizeof(*swap)); + + swap->swap.swap_enabled = + swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0x7; + swap->swap.bg_color = 0xFF000000; + + /* + * Turn off the backlight. This matters because the DCP's idea of + * backlight brightness gets desynced after a power change, and it + * needs to be told it's going to turn off so it will consider the + * subsequent update on poweron an actual change and restore the + * brightness. + */ + if (dcp_has_panel(dcp)) { + swap->swap.bl_unk = 1; + swap->swap.bl_value = 0; + swap->swap.bl_power = 0; + } + + /* Null all surfaces */ + for (int l = 0; l < SWAP_SURFACES; l++) + swap->surf_null[l] = true; +#if DCP_FW_VERSION(13, 2, 0) <= DCP_FW_VER +#if DCP_FW_VERSION(26, 0, 0) > DCP_FW_VER + for (int l = 0; l < 5; l++) + swap->surf2_null[l] = true; +#endif + swap->unknown_pointer_null = true; + swap->unknown_output_null = true; +#endif +} + static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; @@ -1285,8 +1346,11 @@ static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) if (status) { dev_err(dcp->dev, "swap_clear failed! status %u\n", status); - dcp_unarm_retained_framebuffers(dcp, swap_id); - dcp_drm_crtc_vblank(dcp->crtc); + dcp_present_failed(dcp); + if (dcp_present_retires(dcp)) { + dcp_unarm_retained_framebuffers(dcp, swap_id); + dcp_drm_crtc_vblank(dcp->crtc); + } } if (info) { WRITE_ONCE(info->status, status); @@ -1319,8 +1383,13 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, dcp_drm_crtc_vblank(dcp->crtc); return; } +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + dcp_prepare_clear_swap(dcp, &DCP_FW_UNION(dcp->swap)); +#endif DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; - dcp_arm_retained_framebuffers(dcp, resp->swap_id); + if (dcp_present_retires(dcp)) + dcp_arm_retained_framebuffers(dcp, resp->swap_id); if (info) info->swap_id = resp->swap_id; @@ -1471,7 +1540,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) struct dcp_swap_cookie *cookie; struct dcp_wait_cookie *poff_cookie; struct DCP_FW_NAME(dcp_swap_start_req) swap_req = { 0 }; - struct DCP_FW_NAME(dcp_swap_submit_req) *swap = &DCP_FW_UNION(dcp->swap); + typeof(DCP_FW_UNION(dcp->swap)) *swap = &DCP_FW_UNION(dcp->swap); cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) @@ -1482,36 +1551,11 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) /* increase refcount to ensure the receiver has a reference */ kref_get(&cookie->refcount); - // clear surfaces - memset(swap, 0, sizeof(*swap)); - - swap->swap.swap_enabled = - swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0x7; - swap->swap.bg_color = 0xFF000000; - - /* - * Turn off the backlight. This matters because the DCP's idea of - * backlight brightness gets desynced after a power change, and it - * needs to be told it's going to turn off so it will consider the - * subsequent update on poweron an actual change and restore the - * brightness. - */ - if (dcp_has_panel(dcp)) { - swap->swap.bl_unk = 1; - swap->swap.bl_value = 0; - swap->swap.bl_power = 0; - } - - /* Null all surfaces */ - for (int l = 0; l < SWAP_SURFACES; l++) - swap->surf_null[l] = true; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) -#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) - for (int l = 0; l < 5; l++) - swap->surf2_null[l] = true; -#endif - swap->unkU32Ptr_null = true; - swap->unkU32out_null = true; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + dcp_prepare_clear_swap(dcp, swap); +#else + dcp_prepare_clear_swap(dcp, swap); #endif dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); @@ -1798,8 +1842,11 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) if (status) { dev_err(dcp->dev, "swap failed! status %u\n", status); - dcp_unarm_retained_framebuffers(dcp, swap_id); - dcp_drm_crtc_vblank(dcp->crtc); + dcp_present_failed(dcp); + if (dcp_present_retires(dcp)) { + dcp_unarm_retained_framebuffers(dcp, swap_id); + dcp_drm_crtc_vblank(dcp->crtc); + } return; } dcp->swap_start = ktime_get(); @@ -1812,15 +1859,18 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) if (!resp || resp->ret) { dev_err(dcp->dev, "swap_start was rejected\n"); - dcp_drm_crtc_vblank(dcp->crtc); + if (dcp_present_retires(dcp)) + dcp_drm_crtc_vblank(dcp->crtc); return; } if (!dcp_present_begin(dcp, resp->swap_id)) { - dcp_drm_crtc_vblank(dcp->crtc); + if (dcp_present_retires(dcp)) + dcp_drm_crtc_vblank(dcp->crtc); return; } DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; - dcp_arm_retained_framebuffers(dcp, resp->swap_id); + if (dcp_present_retires(dcp)) + dcp_arm_retained_framebuffers(dcp, resp->swap_id); trace_iomfb_swap_submit(dcp, resp->swap_id); dcp_h17p_prepare_swap(dcp); @@ -1843,6 +1893,13 @@ static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) * H17P expects A428 between set_matrix and swap_start, with 0x10000 (1.0 in * 16.16 fixed point), the same identity value the D006 reply carries. */ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER +void iomfb_present_backlight_h17p(struct apple_dcp *dcp) +{ + do_swap(dcp, NULL, NULL); +} +#endif + static void dcp_pre_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) { u32 identity = 0x10000; @@ -2029,8 +2086,8 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru for (l = 0; l < 5; l++) req->surf2_null[l] = true; #endif - req->unkU32Ptr_null = true; - req->unkU32out_null = true; + req->unknown_pointer_null = true; + req->unknown_output_null = true; #endif /* diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 2d8787fc884fe..24cb7bf548ec1 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -119,8 +119,8 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { #endif u8 unkoutbool_null; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u8 unkU32Ptr_null; - u8 unkU32out_null; + u8 unknown_pointer_null; + u8 unknown_output_null; #endif #if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 padding[1]; From 543ba6bdea4410b046ef5975eaa03016e61fb421 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:49:40 +1100 Subject: [PATCH 0470/1579] drm: apple: stop on H17P callbacks without reply semantics A measured callback length establishes buffer bounds but does not make an all-zero reply valid. The H17P dispatch table still lacks a handler for the measured property request; treating that omission as a successful empty reply would hide an incomplete protocol admission. Stop the measured H17P interface before modifying output or acknowledging an unhandled callback. Keep the active transaction and its surface references pinned for recovery by reboot. Explicitly supported handlers still receive zero-initialized output, and the existing H17G fallback is preserved. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 5 +++++ drivers/gpu/drm/apple/iomfb_h17p.c | 13 ++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8288cad699aad..190c64b628af2 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -587,6 +587,11 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, !dcp->cb_handlers[tag]) { dev_warn(dev, "received unknown callback %c%c%c%c\n", hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + if (iomfb_uses_queue(dcp)) { + /* A measured size does not establish a safe reply payload. */ + WRITE_ONCE(dcp->crashed, true); + return; + } /* * Leaving a callback unanswered wedges the coprocessor: it * waits for the ack forever and the outer call never returns. diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index b49a587f34472..772a5ec2e3ccf 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -258,17 +258,12 @@ static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, } /* - * D400 (get_property) is left unhandled: the zeroed reply means "no such - * property". The firmware then runs without a panel power model, as it does - * for its other optional power properties. + * D400 has a measured size but no admitted property-reply semantics. It stays + * unhandled, so the measured H17P profile stops instead of guessing a reply. + * H17G retains its existing missing-property response. */ -/* - * H17P callback numbering. It is not a uniform shift of the v13.5 table: the - * service-creation block is D108..D113 exactly as on v13.5, and the swap - * completion is D590. A callback with no entry here is answered with a - * zeroed reply by dcpep_handle_cb(). - */ +/* H17P callback numbering is not a uniform shift of the v13.5 table. */ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [0] = dcpep_cb_d000_h17p, /* acked after a nested A033 */ [1] = trampoline_true, From 9d4cac8c9ec03208432d542210479c4324bf9424 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 05:36:15 +1100 Subject: [PATCH 0471/1579] drm: apple: preserve unknown H17P background on replay The first completed present need not set the background. Always adding a background update to its cached replay would then send a value that the firmware had never accepted, potentially changing alpha composition during a brightness-only update. Keep the background-update bit only when this or a previous completed present supplied the background. Continue re-presenting the pinned surfaces without changing an unknown inherited background. An explicit accepted background remains available across partial-plane updates. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 190c64b628af2..4d3c0c129f49b 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -955,7 +955,12 @@ void iomfb_scanout_complete_h17p(struct apple_dcp *dcp) scanout->request.surf_null[i] = !scanout->fb[i]; } /* Brightness re-presents all pinned surfaces with a fresh swap ID. */ - scanout->request.swap.swap_enabled = IOMFB_SET_BACKGROUND; + /* Replay only a background established by a completed present. */ + scanout->request.swap.swap_enabled = + request->swap.swap_enabled & IOMFB_SET_BACKGROUND; + if (previous) + scanout->request.swap.swap_enabled |= + previous->request.swap.swap_enabled & IOMFB_SET_BACKGROUND; for (i = 0; i < SWAP_SURFACES; i++) if (scanout->fb[i]) scanout->request.swap.swap_enabled |= BIT(i); From ba37a448b15f7f990a57777a486689caec117d6f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 05:52:14 +1100 Subject: [PATCH 0472/1579] dt-bindings: display: apple,dcp: describe firmware power tables Linux startup callbacks request five power lookup table properties from the display provider. Their replies reproduce the corresponding disp0 ADT bytes: three four-byte values and two 52-byte arrays. The table indices include the firmware's power model axis, which does not define Linux's backlight range. Describe bootloader-supplied byte arrays so the driver can transfer these per-device firmware inputs without embedding panel data or interpreting the power model. Limit the properties to the measured T8140 internal DCP and document their little-endian representation. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/display/apple,dcp.yaml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index 33f1bf6e44359..aefb570de5983 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -123,6 +123,49 @@ properties: enum: [0, 1] description: Per-slot enable flags, in slot order. + apple,power-lut-data-x: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 4 + maxItems: 4 + description: | + First dimension of the display firmware power lookup table. The + bootloader copies the disp0 ADT property verbatim, as a little-endian + 32-bit word, for transfer to the display firmware. + + apple,power-lut-data-y: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 4 + maxItems: 4 + description: | + Second dimension of the display firmware power lookup table, copied + verbatim from disp0 in the ADT as a little-endian 32-bit word. + + apple,power-lut-data-xindex: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 52 + maxItems: 52 + description: | + First-axis indices of the display firmware power lookup table, copied + verbatim from disp0 in the ADT as thirteen little-endian 32-bit words. + These are firmware table inputs, not a Linux backlight range. + + apple,power-lut-data-yindex: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 4 + maxItems: 4 + description: | + Second-axis index of the display firmware power lookup table, copied + verbatim from disp0 in the ADT as a little-endian 32-bit word. + + apple,power-lut-data-lut: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 52 + maxItems: 52 + description: | + Display firmware power lookup table, copied verbatim from disp0 in + the ADT as thirteen little-endian 32-bit words. Linux passes these + values to the firmware without interpreting or changing them. + apple,firmware-version: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 3 @@ -270,6 +313,19 @@ required: - apple,bw-scratch allOf: + - if: + not: + properties: + compatible: + contains: + const: apple,t8140-dcp + then: + properties: + apple,power-lut-data-x: false + apple,power-lut-data-y: false + apple,power-lut-data-xindex: false + apple,power-lut-data-yindex: false + apple,power-lut-data-lut: false - if: properties: compatible: From 833f0ac1fa7455efd77b0ca63b47e6a3e1521ede Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:29:18 -0600 Subject: [PATCH 0473/1579] wifi: brcmfmac: add AWDL interface creation for BCM4387 BCM4387 firmware implements AWDL (Apple Wireless Direct Link) and accepts BRCMF_INTERFACE_TYPE_AWDL through interface_create, but nl80211 has no AWDL iftype, so the interface cannot be created via add_virtual_intf(). Drive it from a vendor command instead. Creation is asynchronous, and has to be. nl80211 dispatches vendor commands holding the wiphy mutex: NL80211_CMD_VENDOR sets NL80211_FLAG_NEED_WIPHY without NL80211_FLAG_NO_WIPHY_MTX, unlike NL80211_CMD_NEW_INTERFACE which sets NEED_RTNL | NO_WIPHY_MTX and takes the wiphy mutex itself, in that order. Registering a netdev from the vendor command therefore deadlocks: nl80211_vendor_cmd <- holds wiphy mutex brcmf_awdl_add_vif brcmf_net_attach register_netdev <- takes RTNL register_netdevice call_netdevice_notifiers_info cfg80211_netdev_notifier_call mutex_lock <- wiphy mutex, already held The task blocks in uninterruptible sleep while holding the RTNL, which takes down every netdev operation on the system; recovery needs a reboot. Using cfg80211_register_netdevice() instead is not a fix either: it requires both the RTNL and the wiphy mutex, and the vendor command holds only the latter, so it trips ASSERT_RTNL and registers into an inconsistent state. Taking the RTNL explicitly would invert cfg80211's RTNL-then-wiphy order. So brcmf_awdl_add_vif() only asks firmware for the interface and returns. The netdev is registered later by brcmf_cfg80211_awdl_attach_pending(), called from the fweh event worker, which runs on the system workqueue holding neither lock. The vif event stays armed across the request because brcmf_notify_vif_event() is what links the brcmf_if to the vif; without it ndev->ieee80211_ptr is never set and cfg80211 ignores the netdev. Userspace must wait for awdl0 to appear rather than assume the vendor command created it. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 141 ++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/cfg80211.h | 5 + .../broadcom/brcm80211/brcmfmac/fweh.c | 7 + .../brcm80211/brcmfmac/interface_create.c | 8 +- .../brcm80211/brcmfmac/interface_create.h | 1 + .../broadcom/brcm80211/brcmfmac/vendor.c | 50 +++++++ .../broadcom/brcm80211/brcmfmac/vendor.h | 12 ++ 7 files changed, 223 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 7768676f5eb58..a93e4109dfd43 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -605,6 +605,147 @@ brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev) * * Return: pointer to new vif on success, ERR_PTR(-errno) if not */ +/** + * brcmf_awdl_add_vif() - create an AWDL interface and give it a netdev. + * + * @wiphy: wiphy device of the new interface. + * @name: name of the new interface. + * + * AWDL is Apple Wireless Direct Link, the link layer AirDrop and AirPlay run + * over. BCM4387 firmware implements it and accepts BRCMF_INTERFACE_TYPE_AWDL + * through interface_create, but there is no nl80211 interface type for it, so + * this cannot go through add_virtual_intf(). It is driven from the vendor + * command instead. + * + * The vif is allocated as NL80211_IFTYPE_STATION because cfg80211 needs some + * valid type for its bookkeeping; the firmware side is what actually decides + * the interface behaviour, from the iftype passed to interface_create. + * + * Return: pointer to the new vif on success, ERR_PTR(-errno) on failure. + */ +int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name) +{ + struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); + struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg)); + struct brcmf_cfg80211_vif *vif; + int err; + + if (brcmf_cfg80211_vif_event_armed(cfg) || cfg->awdl_pending) + return -EBUSY; + + vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_STATION); + if (IS_ERR(vif)) + return PTR_ERR(vif); + + /* Arm before asking firmware: the BRCMF_E_IF event is what links the + * brcmf_if to this vif (brcmf_notify_vif_event() bails with -EBADF if + * nothing is armed, leaving ndev->ieee80211_ptr unset). + */ + brcmf_cfg80211_arm_vif_event(cfg, vif); + + /* Pass NULL rather than a derived address. Supplying a MAC address + * at interface_create time kills the firmware control channel about + * two seconds after create, so the address is set afterwards via + * cur_etheraddr instead. + * + */ + strscpy(cfg->awdl_ifname, name, sizeof(cfg->awdl_ifname)); + cfg->awdl_pending = true; + + err = brcmf_cfg80211_request_awdl_if(ifp, NULL); + if (err) { + cfg->awdl_pending = false; + brcmf_cfg80211_arm_vif_event(cfg, NULL); + brcmf_free_vif(vif); + return err; + } + + /* Deliberately do NOT wait for the interface here, and do NOT attach + * the netdev. nl80211 calls vendor commands holding the wiphy mutex + * (NL80211_CMD_VENDOR sets NEED_WIPHY without NO_WIPHY_MTX), and + * register_netdev() re-enters that mutex via the netdev notifier + * chain -> cfg80211_netdev_notifier_call(). Doing it here self- + * deadlocks and wedges the RTNL for the whole system. The attach is + * done from brcmf_cfg80211_awdl_attach_pending(), which the fweh + * event worker calls while holding no locks at all. + */ + return 0; +} + +/** + * brcmf_cfg80211_awdl_attach_pending() - finish AWDL interface creation. + * + * @ifp: interface the firmware just reported via BRCMF_E_IF_ADD. + * + * Called from the fweh event worker (brcmf_fweh_handle_if_event()), which + * runs on the system workqueue holding neither the RTNL nor the wiphy mutex. + * That is the only context in which register_netdev() is safe for an + * interface created outside cfg80211's own add_virtual_intf() path. + */ +void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp) +{ + struct brcmf_cfg80211_info *cfg = ifp->drvr->config; + struct brcmf_cfg80211_vif *vif; + int err; + + if (!cfg || !cfg->awdl_pending) + return; + cfg->awdl_pending = false; + vif = ifp->vif; + + if (!ifp->ndev || !vif) { + bphy_err(ifp->drvr, "AWDL if event without netdev/vif\n"); + goto done; + } + + strscpy(ifp->ndev->name, cfg->awdl_ifname, sizeof(ifp->ndev->name)); + err = brcmf_net_attach(ifp, false); + if (err) { + bphy_err(ifp->drvr, "registering AWDL netdevice failed\n"); + free_netdev(ifp->ndev); + brcmf_free_vif(vif); + goto done; + } + + brcmf_info("AWDL interface %s created (bsscfgidx %d)\n", + ifp->ndev->name, ifp->bsscfgidx); +done: + brcmf_cfg80211_arm_vif_event(cfg, NULL); +} + +/** + * brcmf_awdl_del_vif() - tear down an interface made by brcmf_awdl_add_vif(). + * + * @wiphy: wiphy device of the interface. + * @wdev: wireless device to remove. + * + * Return: 0 on success, negative errno on failure. + */ +int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev) +{ + struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); + struct brcmf_cfg80211_vif *vif = container_of(wdev, + struct brcmf_cfg80211_vif, + wdev); + struct brcmf_if *ifp = vif->ifp; + int err; + + brcmf_cfg80211_arm_vif_event(cfg, vif); + err = brcmf_fil_bsscfg_data_set(ifp, "interface_remove", NULL, 0); + if (err) { + brcmf_cfg80211_arm_vif_event(cfg, NULL); + return err; + } + + err = brcmf_cfg80211_wait_vif_event(cfg, BRCMF_E_IF_DEL, + BRCMF_VIF_EVENT_TIMEOUT); + brcmf_cfg80211_arm_vif_event(cfg, NULL); + if (!err) + return -EIO; + + return 0; +} + static struct wireless_dev *brcmf_apsta_add_vif(struct wiphy *wiphy, const char *name, struct vif_params *params, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index dec5ff1190f94..fdc9f7e627f0a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -397,6 +397,8 @@ struct brcmf_cfg80211_info { struct work_struct escan_timeout_work; struct list_head vif_list; struct brcmf_cfg80211_vif_event vif_event; + bool awdl_pending; + char awdl_ifname[IFNAMSIZ]; struct completion vif_disabled; struct brcmu_d11inf d11inf; struct brcmf_assoclist_le assoclist; @@ -508,6 +510,9 @@ bool brcmf_get_vif_state_any(struct brcmf_cfg80211_info *cfg, void brcmf_cfg80211_arm_vif_event(struct brcmf_cfg80211_info *cfg, struct brcmf_cfg80211_vif *vif); bool brcmf_cfg80211_vif_event_armed(struct brcmf_cfg80211_info *cfg); +int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name); +void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp); +int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev); int brcmf_cfg80211_wait_vif_event(struct brcmf_cfg80211_info *cfg, u8 action, ulong timeout); s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c index eec4b10e1f674..3e06c23defcb3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c @@ -183,6 +183,13 @@ static void brcmf_fweh_handle_if_event(struct brcmf_pub *drvr, brcmf_fweh_call_event_handler(drvr, ifp, emsg->event_code, emsg, data); + /* The handler above establishes the vif <-> ifp linkage. Only then can + * the AWDL netdev be registered, and only from here: this worker holds + * no locks, unlike the vendor command that requested the interface. + */ + if (ifp && ifevent->action == BRCMF_E_IF_ADD) + brcmf_cfg80211_awdl_attach_pending(ifp); + if (ifp && ifevent->action == BRCMF_E_IF_DEL) { bool armed = brcmf_cfg80211_vif_event_armed(drvr->config); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c index 1f40ff8d632c2..e9ebb90030575 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c @@ -52,7 +52,7 @@ enum brcmf_interface_type { BRCMF_INTERFACE_TYPE_STA = 0, BRCMF_INTERFACE_TYPE_AP = 1, - /* The missing number here is deliberate */ + BRCMF_INTERFACE_TYPE_AWDL = 2, BRCMF_INTERFACE_TYPE_NAN = 3, BRCMF_INTERFACE_TYPE_P2P_GO = 4, BRCMF_INTERFACE_TYPE_P2P_GC = 5, @@ -240,6 +240,12 @@ int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr) macaddr); } +int brcmf_cfg80211_request_awdl_if(struct brcmf_if *ifp, u8 *macaddr) +{ + return brcmf_cfg80211_request_if(ifp, BRCMF_INTERFACE_TYPE_AWDL, + macaddr); +} + int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp) { int err; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h index 669fa1508b67f..7fadbea590e28 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h @@ -9,5 +9,6 @@ int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr); int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp); +int brcmf_cfg80211_request_awdl_if(struct brcmf_if *ifp, u8 *macaddr); #endif /* _BRCMF_INTERFACE_CREATE_H_ */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index d07e7c7355d9d..6dd6a3d7231fe 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -104,6 +104,46 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, return ret; } +#define BRCMF_AWDL_IFNAME "awdl0" + +static int brcmf_cfg80211_vndr_cmds_awdl_handler(struct wiphy *wiphy, + struct wireless_dev *wdev, + const void *data, int len) +{ + struct net_device *ndev; + struct wireless_dev *awdl_wdev; + u32 op; + + if (len < sizeof(op)) + return -EINVAL; + op = *(const u32 *)data; + + switch (op) { + case BRCMF_VNDR_AWDL_OP_CREATE: + ndev = dev_get_by_name(wiphy_net(wiphy), BRCMF_AWDL_IFNAME); + if (ndev) { + dev_put(ndev); + return -EEXIST; + } + /* Asynchronous: returns as soon as firmware has been asked. + * The netdev appears once the fweh worker registers it, so + * userspace must wait for awdl0 rather than assume it exists. + */ + return brcmf_awdl_add_vif(wiphy, BRCMF_AWDL_IFNAME); + case BRCMF_VNDR_AWDL_OP_DESTROY: + ndev = dev_get_by_name(wiphy_net(wiphy), BRCMF_AWDL_IFNAME); + if (!ndev) + return -ENODEV; + awdl_wdev = ndev->ieee80211_ptr; + dev_put(ndev); + if (!awdl_wdev) + return -ENODEV; + return brcmf_awdl_del_vif(wiphy, awdl_wdev); + default: + return -EINVAL; + } +} + const struct wiphy_vendor_command brcmf_vendor_cmds[] = { { { @@ -115,4 +155,14 @@ const struct wiphy_vendor_command brcmf_vendor_cmds[] = { .policy = VENDOR_CMD_RAW_DATA, .doit = brcmf_cfg80211_vndr_cmds_dcmd_handler }, + { + { + .vendor_id = BROADCOM_OUI, + .subcmd = BRCMF_VNDR_CMDS_AWDL + }, + .flags = WIPHY_VENDOR_CMD_NEED_WDEV | + WIPHY_VENDOR_CMD_NEED_NETDEV, + .policy = VENDOR_CMD_RAW_DATA, + .doit = brcmf_cfg80211_vndr_cmds_awdl_handler + }, }; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h index 418f33ea6fd3f..74886de2ba78d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h @@ -11,9 +11,21 @@ enum brcmf_vndr_cmds { BRCMF_VNDR_CMDS_UNSPEC, BRCMF_VNDR_CMDS_DCMD, + BRCMF_VNDR_CMDS_AWDL, BRCMF_VNDR_CMDS_LAST }; +/** + * enum brcmf_vndr_awdl_op - operations for BRCMF_VNDR_CMDS_AWDL + * + * @BRCMF_VNDR_AWDL_OP_CREATE: create the AWDL interface + * @BRCMF_VNDR_AWDL_OP_DESTROY: remove the AWDL interface + */ +enum brcmf_vndr_awdl_op { + BRCMF_VNDR_AWDL_OP_CREATE, + BRCMF_VNDR_AWDL_OP_DESTROY, +}; + /** * enum brcmf_nlattrs - nl80211 message attributes * From d641674bcea6c9ec18434e2c5883fa79e2766dfe Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Wed, 26 Aug 2026 19:02:47 -0600 Subject: [PATCH 0474/1579] wifi: brcmfmac: use AWDL-specific netdev ops for awdl0 Bringing an AWDL interface up through brcmf_netdev_open() stops the firmware answering dcmds. Every subsequent command times out with -EIO, the infrastructure STA link drops with it, and only a brcmfmac reload recovers the device. The standard open path runs station bring-up against the interface: a toe_ol iovar query, brcmf_cfg80211_up(), and then the multicast and promisc configuration that ndo_set_rx_mode schedules via _brcmf_set_multicast_list(). An AWDL bsscfg is not a station, and issuing that sequence against one wedges the firmware control path. Give AWDL interfaces their own net_device_ops. brcmf_netdev_open_awdl() checks the bus is up, clears the 802.1x pending count, and leaves the carrier off, touching no firmware state. ndo_set_rx_mode is omitted entirely so the multicast worker never runs for these interfaces, and ndo_stop skips brcmf_cfg80211_down() for the same reason the open path skips brcmf_cfg80211_up(). awdl0 now comes up with the firmware still responsive and the STA link undisturbed. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 2 + .../broadcom/brcm80211/brcmfmac/core.c | 52 ++++++++++++++++++- .../broadcom/brcm80211/brcmfmac/core.h | 1 + 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index a93e4109dfd43..5326a3abae0a1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -698,6 +698,8 @@ void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp) goto done; } + /* Must be set before brcmf_net_attach(), which picks netdev_ops. */ + ifp->is_awdl = true; strscpy(ifp->ndev->name, cfg->awdl_ifname, sizeof(ifp->ndev->name)); err = brcmf_net_attach(ifp, false); if (err) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 3c4125c336355..8b01fcb1cad01 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -632,6 +632,55 @@ static int brcmf_netdev_open(struct net_device *ndev) return 0; } +/* AWDL interfaces are not STAs. brcmf_netdev_open() runs the standard station + * bring-up -- a toe_ol query, brcmf_cfg80211_up(), then the multicast and + * promisc configuration driven by ndo_set_rx_mode -- and issuing those against + * an AWDL bsscfg stops the firmware answering dcmds at all: every subsequent + * command times out and only a brcmfmac reload recovers it. Bring the netdev + * up without touching firmware, and deliberately omit ndo_set_rx_mode so the + * multicast worker never runs for these interfaces. + */ +static int brcmf_netdev_open_awdl(struct net_device *ndev) +{ + struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_bus *bus_if = drvr->bus_if; + + brcmf_dbg(TRACE, "Enter, bsscfgidx=%d (AWDL)\n", ifp->bsscfgidx); + + if (bus_if->state != BRCMF_BUS_UP) { + bphy_err(drvr, "failed bus is not ready\n"); + return -EAGAIN; + } + + atomic_set(&ifp->pend_8021x_cnt, 0); + + /* Carrier stays off until AWDL is actually running. */ + netif_carrier_off(ndev); + return 0; +} + +static int brcmf_netdev_stop_awdl(struct net_device *ndev) +{ + struct brcmf_if *ifp = netdev_priv(ndev); + + brcmf_dbg(TRACE, "Enter, bsscfgidx=%d (AWDL)\n", ifp->bsscfgidx); + + /* No brcmf_cfg80211_down(): this vif was never brought up through + * cfg80211, and tearing down the STA state would hit the same + * unresponsive-firmware path as the bring-up. + */ + brcmf_net_setcarrier(ifp, false); + return 0; +} + +static const struct net_device_ops brcmf_netdev_ops_awdl = { + .ndo_open = brcmf_netdev_open_awdl, + .ndo_stop = brcmf_netdev_stop_awdl, + .ndo_start_xmit = brcmf_netdev_start_xmit, + .ndo_set_mac_address = brcmf_netdev_set_mac_address, +}; + static const struct net_device_ops brcmf_netdev_ops_pri = { .ndo_open = brcmf_netdev_open, .ndo_stop = brcmf_netdev_stop, @@ -651,7 +700,8 @@ int brcmf_net_attach(struct brcmf_if *ifp, bool locked) ndev = ifp->ndev; /* set appropriate operations */ - ndev->netdev_ops = &brcmf_netdev_ops_pri; + ndev->netdev_ops = ifp->is_awdl ? &brcmf_netdev_ops_awdl + : &brcmf_netdev_ops_pri; ndev->needed_headroom += drvr->hdrlen; ndev->ethtool_ops = &brcmf_ethtool_ops; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index 4b52a3aa855de..451bcf85a4bdf 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -268,6 +268,7 @@ struct brcmf_if { struct in6_addr ipv6_addr_tbl[NDOL_MAX_ENTRIES]; u8 ipv6addr_idx; bool fwil_fwerr; + bool is_awdl; }; int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp); From 5442759f7f3316bbce4bb1fc46a69077089afab3 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:07:12 -0600 Subject: [PATCH 0475/1579] wifi: brcmfmac: return firmware error codes from the vendor dcmd brcmf_fil_cmd_data() collapses every firmware failure to -EBADE unless ifp->fwil_fwerr is set, so a userspace probe driving iovars through the Broadcom vendor dcmd passthrough cannot tell "no such iovar" from "bad argument" from "not up". Set fwil_fwerr around the call and log the raw BCME_* value. This makes the passthrough usable for firmware exploration: the same probe that previously reported EBADE for every failure now distinguishes BCME_BADOPTION, BCME_BUFTOOSHORT, BCME_NOTFOUND and BCME_NOTREADY. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../wireless/broadcom/brcm80211/brcmfmac/vendor.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index 6dd6a3d7231fe..d3211d092cf44 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -64,14 +64,25 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, *(char *)(dcmd_buf + len) = '\0'; } + /* Return the firmware's own BCME_* code instead of the -EBADE that + * brcmf_fil_cmd_data() collapses every failure to. Without this a + * probe cannot tell "no such iovar" from "bad argument" from "not + * up", which is most of what a probe needs to know. + */ + ifp->fwil_fwerr = true; if (cmdhdr->set) ret = brcmf_fil_cmd_data_set(ifp, cmdhdr->cmd, dcmd_buf, ret_len); else ret = brcmf_fil_cmd_data_get(ifp, cmdhdr->cmd, dcmd_buf, ret_len); - if (ret != 0) + ifp->fwil_fwerr = false; + if (ret != 0) { + brcmf_err("dcmd %u ifidx=%d %s: firmware error %d\n", + cmdhdr->cmd, ifp->ifidx, + cmdhdr->set ? "set" : "get", ret); goto exit; + } wr_pointer = dcmd_buf; while (ret_len > 0) { From cb16d0b2a6c0316f4ffc66d25397032c96134bc7 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:07:12 -0600 Subject: [PATCH 0476/1579] wifi: brcmfmac: create the AWDL interface brcmf_cfg80211_request_awdl_if() sent interface_create with MAC_USE alone: no BSSID and no requested interface index. On BCM4387 that yields an interface whose AWDL iovars stay unusable. Send flags 0x1a (MAC_USE | IF_INDEX_USE | BSSID_USE) with an explicit BSSID and if_index 2 instead. Firmware behaviour changes with those arguments: the interface is created at the requested index (wl0.2), awdl_peer_op returns data instead of BCME_NOMEM, and awdl_af_hdr, awdl_config and awdl_af_rssi are accepted where they previously were not. The three values are module parameters so they can be swept without rebuilding. Derive a MAC when the caller passes NULL rather than sending MAC_USE with an all-zero address. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../brcm80211/brcmfmac/interface_create.c | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c index e9ebb90030575..3a4f6b3be4470 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -22,6 +23,24 @@ #include "common.h" #include "interface_create.h" +/* Creating a usable AWDL interface requires interface_create flags + * MAC_USE | IF_INDEX_USE | BSSID_USE together with an explicit BSSID and + * an interface index; brcmfmac otherwise sends MAC_USE alone, which the + * firmware accepts but leaves the AWDL iovars unusable. These parameters + * exist so the three values can be swept without rebuilding the module. + */ +static uint brcmf_awdl_create_flags = 0x1a; +module_param_named(awdl_create_flags, brcmf_awdl_create_flags, uint, 0644); +MODULE_PARM_DESC(awdl_create_flags, "interface_create flags for AWDL"); + +static uint brcmf_awdl_if_index = 2; +module_param_named(awdl_if_index, brcmf_awdl_if_index, uint, 0644); +MODULE_PARM_DESC(awdl_if_index, "interface_create if_index for AWDL"); + +static char *brcmf_awdl_bssid = "00:25:00:ff:94:73"; +module_param_named(awdl_bssid, brcmf_awdl_bssid, charp, 0644); +MODULE_PARM_DESC(awdl_bssid, "interface_create BSSID for AWDL"); + #define BRCMF_INTERFACE_CREATE_VER_1 1 #define BRCMF_INTERFACE_CREATE_VER_2 2 #define BRCMF_INTERFACE_CREATE_VER_3 3 @@ -130,6 +149,8 @@ static int brcmf_cfg80211_request_if_internal(struct brcmf_if *ifp, u32 version, enum brcmf_interface_type if_type, u8 *macaddr) { + bool awdl = if_type == BRCMF_INTERFACE_TYPE_AWDL; + switch (version) { case BRCMF_INTERFACE_CREATE_VER_1: { struct brcmf_interface_create_v1 iface_v1 = {}; @@ -182,6 +203,27 @@ static int brcmf_cfg80211_request_if_internal(struct brcmf_if *ifp, u32 version, brcmf_set_vif_sta_macaddr(ifp, iface_v3.mac_addr); } + if (awdl) { + flags = brcmf_awdl_create_flags; + /* brcmf_awdl_add_vif() passes NULL, so derive a MAC + * here rather than send MAC_USE with an all-zero + * address. + */ + if ((flags & BRCMF_INTERFACE_MAC_USE) && + is_zero_ether_addr(iface_v3.mac_addr)) + brcmf_set_vif_sta_macaddr(ifp, + iface_v3.mac_addr); + iface_v3.if_index = brcmf_awdl_if_index; + if (mac_pton(brcmf_awdl_bssid, iface_v3.bssid)) { + brcmf_info("awdl interface_create: flags 0x%x, if_index %u, bssid %pM, mac %pM\n", + flags, iface_v3.if_index, + iface_v3.bssid, iface_v3.mac_addr); + } else { + bphy_err(ifp->drvr, "bad awdl_bssid \"%s\"\n", + brcmf_awdl_bssid); + return -EINVAL; + } + } iface_v3.flags = cpu_to_le32(flags); return brcmf_fil_iovar_data_get(ifp, "interface_create", &iface_v3, sizeof(iface_v3)); From 7eef9ddae6901d1088089726ddb9e3046c326cce Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:07:12 -0600 Subject: [PATCH 0477/1579] wifi: brcmfmac: do not tear down the AWDL netdev from the vendor command nl80211 calls vendor commands holding the wiphy mutex and not the RTNL (NL80211_CMD_VENDOR sets NEED_WIPHY without NO_WIPHY_MTX). Calling brcmf_remove_interface() from brcmf_awdl_del_vif() therefore reaches cfg80211_close_dependents(), which asserts the RTNL: RTNL: assertion failed at net/wireless/core.c (419) cfg80211_close_dependents brcmf_remove_interface brcmf_awdl_del_vif brcmf_cfg80211_vndr_cmds_awdl_handler and then deadlocks the netdev layer against any concurrent RTNL holder, leaving NetworkManager and ip(8) unkillable in D state. Ask the firmware to drop the BSS and return. The BRCMF_E_IF_DEL event lands in brcmf_fweh_event_worker(), a plain work item holding neither lock, and brcmf_fweh_handle_if_event() removes the interface from there. This mirrors brcmf_awdl_add_vif(), which defers brcmf_net_attach() to the same worker for the same reason. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 5326a3abae0a1..f7d188fe7de10 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -725,27 +725,33 @@ void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp) */ int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev) { - struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_cfg80211_vif *vif = container_of(wdev, struct brcmf_cfg80211_vif, wdev); struct brcmf_if *ifp = vif->ifp; int err; - brcmf_cfg80211_arm_vif_event(cfg, vif); - err = brcmf_fil_bsscfg_data_set(ifp, "interface_remove", NULL, 0); - if (err) { - brcmf_cfg80211_arm_vif_event(cfg, NULL); - return err; - } - - err = brcmf_cfg80211_wait_vif_event(cfg, BRCMF_E_IF_DEL, - BRCMF_VIF_EVENT_TIMEOUT); - brcmf_cfg80211_arm_vif_event(cfg, NULL); - if (!err) - return -EIO; + /* Asymmetric with the rest of brcmfmac on purpose, for the same reason + * brcmf_awdl_add_vif() does not call brcmf_net_attach(): nl80211 calls + * vendor commands holding the wiphy mutex and NOT the RTNL + * (NL80211_CMD_VENDOR sets NEED_WIPHY without NO_WIPHY_MTX). Tearing + * the netdev down here reaches cfg80211_close_dependents(), which + * asserts the RTNL, and then deadlocks the whole netdev layer against + * any concurrent RTNL holder. Do NOT call brcmf_remove_interface() or + * arm the vif event from this context. + * + * Just ask the firmware to drop the BSS and return. The BRCMF_E_IF_DEL + * event lands in brcmf_fweh_event_worker(), which is a plain work item + * holding neither lock, and brcmf_fweh_handle_if_event() removes the + * interface from there. + */ + err = brcmf_fil_iovar_data_set(ifp, "interface_remove", NULL, 0); + if (err) + err = brcmf_fil_bsscfg_data_set(ifp, "interface_remove", NULL, 0); + if (err) + bphy_err(ifp->drvr, "interface_remove failed %d\n", err); - return 0; + return err; } static From d93ee0a6a31e6464f9a3c285979528659b17dd42 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Sun, 6 Sep 2026 14:52:58 -0600 Subject: [PATCH 0478/1579] wifi: brcmfmac: translate AWDL data frames at the awdl0 boundary AWDL data frames on the air are 802.3 + LLC/SNAP (OUI 00:17:f2, protocol 0x0800) + a 6-byte data header (03 04 00 00) + the real ethertype. The kernel only speaks Ethernet II, so for is_awdl interfaces: - RX: brcmf_rx_eth_type_trans() strips the encapsulation in place before eth_type_trans(). It is called from both delivery paths -- brcmf_rx_hdrpull (BCDC) and the msgbuf RX completion (PCIe); a first attempt in hdrpull alone was dead code on PCIe. Frames that are already Ethernet II pass. - TX: brcmf_awdl_encap() wraps Ethernet II in the same header with a per-interface incrementing sequence; needed_headroom grows by 16. Frames that already carry a length field (raw AF_PACKET senders building the encapsulation themselves) pass untouched. Also log the destination MAC on each awdl0 TX completion so per-frame fates can be read per peer (unicast to a MAC without a firmware peer entry completes 0x0003 FW_TOSSED; with one, 0x0000 acked). With this, EUI-64 neighbour pinning and awdl_peer_op ADD, a remote peer completes TCP to us over firmware-native AWDL. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/core.c | 76 ++++++++++++++++++- .../broadcom/brcm80211/brcmfmac/core.h | 4 + .../broadcom/brcm80211/brcmfmac/msgbuf.c | 50 ++++++++++-- 3 files changed, 121 insertions(+), 9 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 8b01fcb1cad01..0392b2a8a9064 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,17 @@ #define D11_PHY_HDR_LEN 6 +/* AWDL data frames on the air are 802.3 + LLC/SNAP (OUI 00:17:f2, protocol + * 0x0800) + a 4-byte data header, then the real ethertype. The kernel only + * speaks Ethernet II, so the driver translates at the awdl0 boundary in + * both directions. + */ +#define BRCMF_AWDL_ENCAP_LEN 16 +#define BRCMF_AWDL_DATA_MIN_LEN (ETH_HLEN + BRCMF_AWDL_ENCAP_LEN) + +static const u8 brcmf_awdl_snap[] = { 0xaa, 0xaa, 0x03, 0x00, 0x17, 0xf2, + 0x08, 0x00, 0x03, 0x04 }; + struct d11rxhdr_le { __le16 RxFrameSize; u16 PAD; @@ -290,6 +302,55 @@ static bool brcmf_skb_is_iapp(struct sk_buff *skb) #endif } +/* Unwrap SNAP + AWDL data header into Ethernet II, in place. skb->data is + * the 802.3 header. Frames that are not AWDL-encapsulated pass unchanged. + */ +static void brcmf_awdl_decap(struct sk_buff *skb) +{ + const struct ethhdr *eh = (const struct ethhdr *)skb->data; + + if (skb->len < BRCMF_AWDL_DATA_MIN_LEN || + ntohs(eh->h_proto) > ETH_DATA_LEN || + memcmp(skb->data + ETH_HLEN, brcmf_awdl_snap, + sizeof(brcmf_awdl_snap))) + return; + + memmove(skb->data + BRCMF_AWDL_ENCAP_LEN, skb->data, 2 * ETH_ALEN); + skb_pull(skb, BRCMF_AWDL_ENCAP_LEN); +} + +/* Every received data frame passes through here, on the BCDC and the msgbuf + * (PCIe) paths alike, before the kernel classifies it. + */ +__be16 brcmf_rx_eth_type_trans(struct brcmf_if *ifp, struct sk_buff *skb) +{ + if (ifp->is_awdl) + brcmf_awdl_decap(skb); + return eth_type_trans(skb, ifp->ndev); +} + +/* Wrap Ethernet II into 802.3 + SNAP + AWDL data header, in place. Frames + * already carrying a length field (raw AF_PACKET senders that build the + * encapsulation themselves) pass unchanged. Caller guarantees headroom. + */ +static void brcmf_awdl_encap(struct brcmf_if *ifp, struct sk_buff *skb) +{ + const struct ethhdr *eh = (const struct ethhdr *)skb->data; + u8 *hdr; + + if (ntohs(eh->h_proto) < ETH_P_802_3_MIN) + return; + + hdr = skb_push(skb, BRCMF_AWDL_ENCAP_LEN); + memmove(hdr, hdr + BRCMF_AWDL_ENCAP_LEN, 2 * ETH_ALEN); + put_unaligned_be16(skb->len - ETH_HLEN, hdr + 2 * ETH_ALEN); + memcpy(hdr + ETH_HLEN, brcmf_awdl_snap, sizeof(brcmf_awdl_snap)); + put_unaligned_le16(ifp->awdl_tx_seq++, + hdr + ETH_HLEN + sizeof(brcmf_awdl_snap)); + hdr[ETH_HLEN + sizeof(brcmf_awdl_snap) + 2] = 0; + hdr[ETH_HLEN + sizeof(brcmf_awdl_snap) + 3] = 0; +} + static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, struct net_device *ndev) { @@ -298,6 +359,7 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, struct brcmf_pub *drvr = ifp->drvr; struct ethhdr *eh; int head_delta; + unsigned int hdrlen = drvr->hdrlen; unsigned int tx_bytes = skb->len; brcmf_dbg(DATA, "Enter, bsscfgidx=%d\n", ifp->bsscfgidx); @@ -328,9 +390,12 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, goto done; } + if (ifp->is_awdl) + hdrlen += BRCMF_AWDL_ENCAP_LEN; + /* Make sure there's enough writeable headroom */ - if (skb_headroom(skb) < drvr->hdrlen || skb_header_cloned(skb)) { - head_delta = max_t(int, drvr->hdrlen - skb_headroom(skb), 0); + if (skb_headroom(skb) < hdrlen || skb_header_cloned(skb)) { + head_delta = max_t(int, hdrlen - skb_headroom(skb), 0); brcmf_dbg(INFO, "%s: %s headroom\n", brcmf_ifname(ifp), head_delta ? "insufficient" : "unmodifiable"); @@ -365,6 +430,9 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, /* set pacing shift for packet aggregation */ sk_pacing_shift_update(skb->sk, 8); + if (ifp->is_awdl) + brcmf_awdl_encap(ifp, skb); + ret = brcmf_proto_tx_queue_data(drvr, ifp->ifidx, skb); if (ret < 0) brcmf_txfinalize(ifp, skb, false); @@ -495,7 +563,7 @@ static int brcmf_rx_hdrpull(struct brcmf_pub *drvr, struct sk_buff *skb, return -ENODATA; } - skb->protocol = eth_type_trans(skb, (*ifp)->ndev); + skb->protocol = brcmf_rx_eth_type_trans(*ifp, skb); return 0; } @@ -704,6 +772,8 @@ int brcmf_net_attach(struct brcmf_if *ifp, bool locked) : &brcmf_netdev_ops_pri; ndev->needed_headroom += drvr->hdrlen; + if (ifp->is_awdl) + ndev->needed_headroom += BRCMF_AWDL_ENCAP_LEN; ndev->ethtool_ops = &brcmf_ethtool_ops; /* set the mac address & netns */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index 451bcf85a4bdf..86c9ef09c9090 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -250,6 +250,8 @@ enum brcmf_netif_stop_reason { * @pend_8021x_cnt: tracks outstanding number of 802.1x frames. * @pend_8021x_wait: used for signalling change in count. * @fwil_fwerr: flag indicating fwil layer should return firmware error codes. + * @is_awdl: interface carries AWDL data frames (SNAP + AWDL data header). + * @awdl_tx_seq: sequence number written into the AWDL data header on TX. */ struct brcmf_if { struct brcmf_pub *drvr; @@ -269,6 +271,7 @@ struct brcmf_if { u8 ipv6addr_idx; bool fwil_fwerr; bool is_awdl; + u16 awdl_tx_seq; }; int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp); @@ -285,6 +288,7 @@ void brcmf_txflowblock_if(struct brcmf_if *ifp, enum brcmf_netif_stop_reason reason, bool state); void brcmf_txfinalize(struct brcmf_if *ifp, struct sk_buff *txp, bool success); void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb); +__be16 brcmf_rx_eth_type_trans(struct brcmf_if *ifp, struct sk_buff *skb); void brcmf_netif_mon_rx(struct brcmf_if *ifp, struct sk_buff *skb); void brcmf_net_detach(struct net_device *ndev, bool locked); int brcmf_net_mon_attach(struct brcmf_if *ifp); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index eb3985a85ca25..869018de9ebf3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -912,11 +913,22 @@ brcmf_msgbuf_process_ioctl_complete(struct brcmf_msgbuf *msgbuf, void *buf) } +/* Firmware-reported completion status of AWDL data frames. brcmfmac never + * looks at tx_status; it is the WLFC_CTL_PKTFLAG_* fate of the frame + * (0 acked, 4 sent-no-ack i.e. multicast on air, 7 dropped by the firmware). + * Observed: awdl0 multicast completes ~1:3 sent:dropped, template or not. + * Logged for every frame queued on a non-primary flowring. 2000/s so a + * 200-frame burst at 1000/s is logged in full and in order -- the sequence + * of fates is the measurement. + */ +static DEFINE_RATELIMIT_STATE(brcmf_awdl_txs_rs, HZ, 2000); + static void brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) { struct brcmf_commonring *commonring; struct msgbuf_tx_status *tx_status; + struct brcmf_if *ifp; u32 idx; struct sk_buff *skb; u16 flowid; @@ -934,8 +946,23 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) commonring = msgbuf->flowrings[flowid]; atomic_dec(&commonring->outstanding_tx); - brcmf_txfinalize(brcmf_get_ifp(msgbuf->drvr, tx_status->msg.ifidx), - skb, true); + ifp = brcmf_get_ifp(msgbuf->drvr, tx_status->msg.ifidx); + /* Key on the flowring's ifidx, which the host assigned when it queued + * the frame, not on what the firmware echoes: repeated runs + * logged nothing for 1227 awdl0 frames each, and a completion + * echoed under ifidx 0 would have been invisible to both filters. + */ + if (brcmf_flowring_ifidx_get(msgbuf->flow, flowid) && + __ratelimit(&brcmf_awdl_txs_rs)) + pr_info("brcmfmac: awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", + brcmf_flowring_ifidx_get(msgbuf->flow, flowid), + tx_status->msg.ifidx, ifp ? ifp->is_awdl : -1, flowid, + (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), + le16_to_cpu(tx_status->tx_status), + le16_to_cpu(tx_status->metadata_len), + skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); + + brcmf_txfinalize(ifp, skb, true); } @@ -1236,7 +1263,7 @@ brcmf_msgbuf_process_rx_complete(struct brcmf_msgbuf *msgbuf, void *buf) return; } - skb->protocol = eth_type_trans(skb, ifp->ndev); + skb->protocol = brcmf_rx_eth_type_trans(ifp, skb); brcmf_netif_rx(ifp, skb); } @@ -1584,14 +1611,25 @@ static int brcmf_msgbuf_stats_read(struct seq_file *seq, void *data) if (!msgbuf->flow->rings[i]) continue; ring = msgbuf->flow->rings[i]; - if (ring->status != RING_OPEN) + hash = &msgbuf->flow->hash[ring->hash_id]; + if (ring->status != RING_OPEN) { + /* A ring the firmware has not acknowledged holds every + * frame queued to it, invisibly, while tx_packets + * advances. Show it rather than skip it. + */ + seq_printf(seq, "id %3u: status %d (not open), qlen %4u\n" + " ifidx %u, fifo %u, da %pM\n", + i, ring->status, + skb_queue_len(&ring->skblist), + hash->ifidx, hash->fifo, hash->mac); continue; + } commonring = msgbuf->flowrings[i]; - hash = &msgbuf->flow->hash[ring->hash_id]; - seq_printf(seq, "id %3u: rp %4u, wp %4u, qlen %4u, blocked %u\n" + seq_printf(seq, "id %3u: rp %4u, wp %4u, qlen %4u, blocked %u, outstanding_tx %d\n" " ifidx %u, fifo %u, da %pM\n", i, commonring->r_ptr, commonring->w_ptr, skb_queue_len(&ring->skblist), ring->blocked, + atomic_read(&commonring->outstanding_tx), hash->ifidx, hash->fifo, hash->mac); } From d2a7ae4c1c3d43aaf7dac45484e3b3ca91cf08c5 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Mon, 7 Sep 2026 01:28:47 -0600 Subject: [PATCH 0479/1579] brcmfmac: msgbuf: tolerate txstatus for a freed flowring brcmf_msgbuf_delete_flowring() gives up on outstanding frames after ~75 ms and frees the ring; the firmware then completes them (tx_status 0x0006 EXPIRED) against a flow->rings[] slot that is already NULL. The awdl txstatus log line dereferenced it (oops in brcmf_flowring_ifidx_get from brcmf_msgbuf_process_rx on awdl-if destroy). Look the ring up only if it still exists, and reject a flowring id beyond max_flowrings before indexing. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/msgbuf.c | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 869018de9ebf3..1df1a1592ca4f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -932,6 +932,7 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) u32 idx; struct sk_buff *skb; u16 flowid; + u8 ring_ifidx; tx_status = (struct msgbuf_tx_status *)buf; idx = le32_to_cpu(tx_status->msg.request_id) - 1; @@ -942,6 +943,13 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) if (!skb) return; + if (flowid >= msgbuf->max_flowrings) { + bphy_err(msgbuf->drvr, "txstatus for flowring %u out of range\n", + flowid); + brcmu_pkt_buf_free_skb(skb); + return; + } + set_bit(flowid, msgbuf->txstatus_done_map); commonring = msgbuf->flowrings[flowid]; atomic_dec(&commonring->outstanding_tx); @@ -951,11 +959,18 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) * the frame, not on what the firmware echoes: repeated runs * logged nothing for 1227 awdl0 frames each, and a completion * echoed under ifidx 0 would have been invisible to both filters. + * + * The ring may already be gone: brcmf_msgbuf_delete_flowring() gives up + * on outstanding frames after ~75 ms and the firmware completes them + * (0x0006 EXPIRED) after the ring was freed -- awdl0's VO multicast + * never completes while the ring is open (observed as a NULL deref + * here on `awdl-if destroy`). The skb and its pktid are unaffected. */ - if (brcmf_flowring_ifidx_get(msgbuf->flow, flowid) && - __ratelimit(&brcmf_awdl_txs_rs)) + ring_ifidx = msgbuf->flow->rings[flowid] ? + brcmf_flowring_ifidx_get(msgbuf->flow, flowid) : 0; + if (ring_ifidx && __ratelimit(&brcmf_awdl_txs_rs)) pr_info("brcmfmac: awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", - brcmf_flowring_ifidx_get(msgbuf->flow, flowid), + ring_ifidx, tx_status->msg.ifidx, ifp ? ifp->is_awdl : -1, flowid, (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), le16_to_cpu(tx_status->tx_status), From 196c0bd46a19484f1e628bd118786d0e6888ceab Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 05:54:56 +1100 Subject: [PATCH 0480/1579] drm: apple: transfer measured H17P provider power tables During Linux startup the display firmware requests five provider power table properties. The returned bytes match disp0 in the ADT, followed by zero padding and a little-endian byte count. A sixth request refers to a property absent from that node and receives an empty reply. Firmware leaves bytes following the key's first NUL uninitialized. Handle these transfers using bootloader-supplied DT data. Validate the observed scope, capacity, trailing bytes, terminated key and exact data sizes before replying. Missing required data, unknown keys and unmeasured request variants stop the transaction instead of inventing a property. Keep the existing empty-reply behaviour for the H17G method profile. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 100 +++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 772a5ec2e3ccf..4951949d7cc91 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -257,11 +257,100 @@ static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, return true; } -/* - * D400 has a measured size but no admitted property-reply semantics. It stays - * unhandled, so the measured H17P profile stops instead of guessing a reply. - * H17G retains its existing missing-property response. - */ +struct dcp_h17p_provider_request { + u8 scope[4]; + char name[64]; + __le32 capacity; + u8 reserved[4]; +} __packed; + +struct dcp_h17p_provider_reply { + u8 data[0xc00]; + __le32 length; +} __packed; + +struct dcp_h17p_provider_property { + const char *name; + const char *dt_name; + u32 length; +}; + +static_assert(sizeof(struct dcp_h17p_provider_request) == 0x4c); +static_assert(sizeof(struct dcp_h17p_provider_reply) == 0xc04); + +/* Only these provider transfers have been measured against the disp0 ADT. */ +static const struct dcp_h17p_provider_property provider_properties[] = { + { "power-lut-data-x", "apple,power-lut-data-x", 4 }, + { "power-lut-data-y", "apple,power-lut-data-y", 4 }, + { "power-lut-data-xindex", "apple,power-lut-data-xindex", 52 }, + { "power-lut-data-yindex", "apple,power-lut-data-yindex", 4 }, + { "power-lut-data-lut", "apple,power-lut-data-lut", 52 }, + { "power-lut-vbatt-cur-nominal", "apple,power-lut-vbatt-cur-nominal", 0 }, +}; + +static const struct dcp_h17p_provider_property * +dcp_provider_property_h17p(const struct dcp_h17p_provider_request *request) +{ + unsigned int i; + + if (memcmp(request->scope, "VORP", sizeof(request->scope)) || + le32_to_cpu(request->capacity) != + sizeof_field(struct dcp_h17p_provider_reply, data) || + memchr_inv(request->reserved, 0, sizeof(request->reserved)) || + strnlen(request->name, sizeof(request->name)) == sizeof(request->name)) + return NULL; + + /* Bytes following the first NUL were not initialized by the firmware. */ + for (i = 0; i < ARRAY_SIZE(provider_properties); i++) + if (!strcmp(request->name, provider_properties[i].name)) + return &provider_properties[i]; + return NULL; +} + +static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_provider_property *property; + struct dcp_h17p_provider_reply *reply = out; + struct device_node *node = dcp->dev->of_node; + int length, ret; + + /* The dispatcher already cleared the exact H17G output length. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return true; + + trace_iomfb_callback(dcp, tag, __func__); + property = dcp_provider_property_h17p(in); + if (!property) { + dev_err(dcp->dev, "unqualified D400 provider property request\n"); + goto fail; + } + + length = of_property_count_u8_elems(node, property->dt_name); + if (!property->length) { + /* Only this absent property has a measured empty reply. */ + if (of_property_present(node, property->dt_name)) + goto fail; + } else if (length != (int)property->length) { + dev_err(dcp->dev, "invalid provider property %s: %d bytes\n", + property->dt_name, length); + goto fail; + } + + memset(reply, 0, sizeof(*reply)); + if (property->length) { + ret = of_property_read_u8_array(node, property->dt_name, + reply->data, property->length); + if (ret) + goto fail; + } + reply->length = cpu_to_le32(property->length); + return true; + +fail: + WRITE_ONCE(dcp->crashed, true); + return false; +} /* H17P callback numbering is not a uniform shift of the v13.5 table. */ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { @@ -302,6 +391,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [208] = trampoline_nop, /* update_backlight_factor_prop */ [209] = trampoline_get_time, /* get_calendar_time_ms */ [300] = trampoline_pr_publish, + [400] = trampoline_provider_property_h17p, [401] = trampoline_get_uint_prop, /* get_uint_prop */ [404] = trampoline_nop, /* set_uint_prop */ [406] = trampoline_set_fx_prop, /* set_fx_prop */ From b849064eeb2674ec33eed26ec239362e72b1db9d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:11:18 +1100 Subject: [PATCH 0481/1579] drm: apple: unwind failed firmware buffer allocations The firmware buffer callback reserves a descriptor before allocating its backing memory and ignores failures from both dma_alloc_coherent() and dma_get_sgtable(). A failed allocation can therefore reach scatterlist creation with a NULL buffer, or leave an occupied descriptor without a usable mapping. Release also discards the scatterlist without freeing it. Reject zero-sized and overflowing requests before either page-rounding operation. Build the descriptor locally and publish it only after both DMA operations succeed. Free the coherent allocation if scatterlist creation fails, and release the scatterlist together with its backing allocation. Successful requests retain the existing wire-size rounding and reply fields. Fault injection exercises both failure points, descriptor exhaustion, rounding boundaries, retry after failure and release for the older firmware and H17P template profiles. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 56 ++++++++++++++++---------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index da73a2dee5ffa..004e97b02c010 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -593,35 +594,45 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, struct dcp_allocate_buffer_req *req) { struct dcp_allocate_buffer_resp resp = { 0 }; - struct dcp_mem_descriptor *memdesc; - size_t size; + struct dcp_mem_descriptor allocated = { 0 }; + size_t size, rounded; u32 id; + int ret; + + /* Validate both wire-size and IOMMU-page rounding before allocating. */ + if (!req->size || req->size > SIZE_MAX || + check_add_overflow((size_t)req->size, (size_t)4095, &rounded)) + return resp; + allocated.size = round_down(rounded, 4096); + if (check_add_overflow(allocated.size, (size_t)SZ_16K - 1, &rounded)) + return resp; + size = round_down(rounded, SZ_16K); - resp.dva_size = ALIGN(req->size, 4096); - resp.mem_desc_id = - find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); - if (resp.mem_desc_id >= DCP_MAX_MAPPINGS) { + if (id >= DCP_MAX_MAPPINGS) { dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring\n"); - resp.dva_size = 0; - resp.mem_desc_id = 0; return resp; } - id = resp.mem_desc_id; - set_bit(id, dcp->memdesc_map); - memdesc = &dcp->memdesc[id]; + allocated.buf = dma_alloc_coherent(dcp->dev, size, &allocated.dva, + GFP_KERNEL); + if (!allocated.buf) + return resp; - memdesc->piodma_mapped = false; - memdesc->size = resp.dva_size; - /* HACK: align size to 16K since the iommu API only maps full pages */ - size = ALIGN(resp.dva_size, SZ_16K); - memdesc->buf = dma_alloc_coherent(dcp->dev, size, - &memdesc->dva, GFP_KERNEL); + ret = dma_get_sgtable(dcp->dev, &allocated.map, allocated.buf, + allocated.dva, size); + if (ret) { + dma_free_coherent(dcp->dev, size, allocated.buf, allocated.dva); + return resp; + } - dma_get_sgtable(dcp->dev, &memdesc->map, memdesc->buf, memdesc->dva, - size); - resp.dva = memdesc->dva; + /* Callbacks are serialized; publish only a fully initialized descriptor. */ + dcp->memdesc[id] = allocated; + set_bit(id, dcp->memdesc_map); + resp.mem_desc_id = id; + resp.dva_size = allocated.size; + resp.dva = allocated.dva; #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) /* @@ -629,8 +640,8 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, * microcode through it, and with a zero paddr the timing sequencer * comes up and is powered straight back down. */ - if (memdesc->map.sgl) - resp.paddr = sg_phys(memdesc->map.sgl); + if (allocated.map.sgl) + resp.paddr = sg_phys(allocated.map.sgl); #endif return resp; @@ -674,6 +685,7 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) } if (memdesc->buf) { + sg_free_table(&memdesc->map); dma_free_coherent(dcp->dev, size, memdesc->buf, memdesc->dva); memdesc->buf = NULL; memset(&memdesc->map, 0, sizeof(memdesc->map)); From c2dcd7b313b1d6b2b54db0d09b5fd37066813a8c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:13:59 +1100 Subject: [PATCH 0482/1579] drm: apple: qualify H17P firmware buffer allocation replies Linux startup measurements correlate seven firmware allocation replies with the core DMA allocation and IOMMU mapping events. The first reply address matches the first physical scatterlist page, the second matches the device DMA address, and the reported length is rounded to 4 KiB. Two allocations have physically noncontiguous backing pages. Route H17P allocation requests through a bounded callback that admits the two observed option combinations and zero trailing/null bytes. Reuse the existing Asahi allocator and encode its successful reply explicitly in little endian. Stop the transaction when validation or allocation fails, since no H17P error reply has been measured. The H17G profile keeps its existing callback path. Describe the first address as a measured scatterlist correlation rather than assigning an unmeasured firmware use to it. This does not establish secondary-domain ownership or enable unqualified teardown callbacks. Replay all seven complete replies against independent core trace inputs, and exercise option mutations, both DMA failure points, overflow and mapping-table exhaustion. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 45 +++++++++++++++++++++++++- drivers/gpu/drm/apple/iomfb_template.c | 5 ++- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 4951949d7cc91..ca17da74ed7af 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -352,6 +352,49 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, return false; } +static bool trampoline_allocate_buffer_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_allocate_buffer_req *wire = in; + struct dcp_allocate_buffer_req request = { 0 }; + struct dcp_allocate_buffer_resp reply; + u32 options; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_allocate_buffer(dcp, tag, out, in); + + static_assert(sizeof(*wire) == 0x14); + static_assert(sizeof(reply) == 0x1c); + static_assert(offsetof(struct dcp_allocate_buffer_req, size) == 4); + static_assert(offsetof(struct dcp_allocate_buffer_req, unk2) == 12); + static_assert(offsetof(struct dcp_allocate_buffer_req, paddr_null) == 16); + + trace_iomfb_callback(dcp, tag, __func__); + options = get_unaligned_le32(&wire->unk2); + /* Only these option combinations have observed successful replies. */ + if (get_unaligned_le32(&wire->unk0) != 0x703 || + (options != 4 && options != SZ_16K) || + memchr_inv(&wire->paddr_null, 0, 4)) + goto fail; + + request.size = get_unaligned_le64(&wire->size); + reply = dcpep_cb_allocate_buffer(dcp, &request); + if (!reply.dva_size) + goto fail; + + put_unaligned_le64(reply.paddr, out); + put_unaligned_le64(reply.dva, (u8 *)out + 8); + put_unaligned_le64(reply.dva_size, (u8 *)out + 16); + put_unaligned_le32(reply.mem_desc_id, (u8 *)out + 24); + return true; + +fail: + /* Allocation failures have no measured H17P reply encoding. */ + dev_err(dcp->dev, "unqualified or failed D451 buffer allocation\n"); + WRITE_ONCE(dcp->crashed, true); + return false; +} + /* H17P callback numbering is not a uniform shift of the v13.5 table. */ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [0] = dcpep_cb_d000_h17p, /* acked after a nested A033 */ @@ -400,7 +443,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [413] = trampoline_true, /* setProperty */ [414] = trampoline_sr_set_property_int, /* setProperty */ [415] = trampoline_true, /* setProperty */ - [451] = trampoline_allocate_buffer, /* allocate_buffer */ + [451] = trampoline_allocate_buffer_h17p, /* allocate_buffer */ [452] = trampoline_map_physical, /* prepare */ [454] = trampoline_release_mem_desc, /* release_descriptor */ [552] = trampoline_true, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 004e97b02c010..a580ccd582f73 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -636,9 +636,8 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) /* - * H17P firmware also needs the physical address: it loads the APT M3 - * microcode through it, and with a zero paddr the timing sequencer - * comes up and is powered straight back down. + * Linux allocation traces correlate the first H17P reply address with + * the first SG page. The remaining pages need not be contiguous. */ if (allocated.map.sgl) resp.paddr = sg_phys(allocated.map.sgl); From 86d5aac91309e4610a32cc76d23e3b3a3bcd4ace Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:35:59 +1100 Subject: [PATCH 0483/1579] drm: apple: validate compact H17P receive message lengths Linux startup capture on the internal display endpoint records seventeen compact messages. Their header length counts all bytes after the first eight bytes, and the service channel is carried at offset two. Announce and report payloads start after the compact 24-byte header. Check the recorded length against the actual queue-entry size before normalizing a measured H17P receive header or dispatching its payload. Keep the existing H17G framing path unchanged. No outgoing compact message encoding is introduced by this receive-side check. Captured messages, unaligned buffers, truncated headers and length-bit mutations exercise the actual validator under address/undefined checks. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index daa27e1ad7238..9440662d55788 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -554,6 +555,14 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, payload_size, true); } +static bool afk_validate_h17p_header(const u8 *data, size_t size) +{ + /* The measured length counts bytes after the first eight header bytes. */ + return size >= sizeof(struct epic_hdr) + + sizeof(struct epic_sub_hdr_h17p) && + get_unaligned_le32(data + 4) == size - 8; +} + static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, u8 *data, size_t data_size) { @@ -574,6 +583,13 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, hdr_len = sizeof(*ehdr) + sizeof(*c); if (data_size < hdr_len) goto too_small; + if (ep->dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !afk_validate_h17p_header(data, data_size)) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: invalid compact message length\n", + ep->endpoint); + return; + } /* * H17P multiplexes every service of an endpoint onto queue From 81baffa4a5dfb6dc07c4c580b09a3024d02029fe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:37:26 +1100 Subject: [PATCH 0484/1579] drm: apple: answer the measured H17P startup integer query Three Linux startup transactions query the PMU Temperature property and receive the same 64-bit value 3029 with a successful boolean result. This is the existing Asahi firmware reply value, but the generic handler only returns it for panels marked as mini-LED. The internal H17P panel also makes this startup query. Use the observed reply for the measured H17P property request, validating the object, bounded property name and zero null/trailing bytes. Bytes after the name terminator and the input's previous value are ignored. Stop on unmeasured queries rather than returning an invented zero value. The H17G profile retains the generic panel-dependent behavior. The scalar is a firmware interoperability value; its temperature units and any relationship to a physical sensor have not been established. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index ca17da74ed7af..47cf407089867 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -352,6 +352,30 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, return false; } +static bool trampoline_get_uint_prop_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_get_uint_prop_req *request = in; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_get_uint_prop(dcp, tag, out, in); + + trace_iomfb_callback(dcp, tag, __func__); + /* The observed startup query uses Asahi's existing Temperature reply. */ + if (memcmp(request->obj, "SUMP", sizeof(request->obj)) || + strnlen(request->key, sizeof(request->key)) == sizeof(request->key) || + strcmp(request->key, "Temperature") || request->value_null || + memchr_inv(request->padding, 0, sizeof(request->padding))) { + dev_err(dcp->dev, "unqualified D401 integer property query\n"); + WRITE_ONCE(dcp->crashed, true); + return false; + } + + put_unaligned_le64(3029, out); + *((u8 *)out + offsetof(struct dcp_get_uint_prop_resp, ret)) = true; + return true; +} + static bool trampoline_allocate_buffer_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -435,7 +459,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [209] = trampoline_get_time, /* get_calendar_time_ms */ [300] = trampoline_pr_publish, [400] = trampoline_provider_property_h17p, - [401] = trampoline_get_uint_prop, /* get_uint_prop */ + [401] = trampoline_get_uint_prop_h17p, /* get_uint_prop */ [404] = trampoline_nop, /* set_uint_prop */ [406] = trampoline_set_fx_prop, /* set_fx_prop */ [408] = trampoline_get_frequency_h17p, /* getClockFrequency */ From 6c23d77fd9dc0c2a0c326ec29732beffb98e6539 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:39:36 +1100 Subject: [PATCH 0485/1579] drm: apple: select the internal T8140 H17P method profile Linux startup measurements on T8140 identify the internal display's H17P IOMFB interface and a live bootloader-owned RTKit session. Falling back to the generic DCP match cannot select that firmware profile and would attempt to start the coprocessor instead of adopting its session. Match the internal T8140 DCP explicitly and reuse the existing H17 live-session adoption with the measured method profile. The match advertises no external display ports; external DCP qualification remains separate. The public DT still leaves display resources disabled. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 116c80f5d902c..b7d6100b3e3de 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2457,6 +2457,13 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { .num_dptx_ports = 2, }; +/* The internal T8140 endpoint uses the measured H17P method profile. */ +static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { + .num_dptx_ports = 0, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_H17P, +}; + /* * M5 (T8142) runs H17-generation DCP firmware with the H17G method numbering * and is left running by the bootloader. @@ -2479,6 +2486,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_dcpext = { static const struct of_device_id of_match[] = { { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, + { .compatible = "apple,t8140-dcp", .data = &apple_dcp_hw_t8140, }, { .compatible = "apple,t8142-dcp", .data = &apple_dcp_hw_t8142, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, From aad12fc0184db3d49d08a58a946f127232f47f79 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:52:22 +1100 Subject: [PATCH 0486/1579] drm: apple: answer the H17P luma-range startup callback The internal H17P display requests an eight-byte luma-range reply during client open. Selector zero receives the binary32 encoding of 219.0 and a zero trailing word. Leaving this measured callback unbound stops startup; a zero range also cannot provide the firmware's required colour scale. Provide the recorded reply for selector zero without kernel floating point. Reject unqualified selectors, and preserve the H17G profile's existing zero-filled unknown-callback result, including short output envelopes. The actual dispatcher reproduces the captured reply and rejects selector mutations without acknowledging them. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 47cf407089867..6ac01ee69fcf4 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -352,6 +352,25 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, return false; } +static bool trampoline_luma_span_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + /* Preserve the older profile's already-cleared unknown-callback reply. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return true; + + trace_iomfb_callback(dcp, tag, __func__); + if (get_unaligned_le32(in)) { + dev_err(dcp->dev, "unqualified D574 range selector\n"); + WRITE_ONCE(dcp->crashed, true); + return false; + } + + /* Selector zero receives binary32 219.0 followed by a zero word. */ + put_unaligned_le64(0x435b0000, out); + return true; +} + static bool trampoline_get_uint_prop_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -476,6 +495,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [565] = trampoline_true, [567] = trampoline_true, [572] = trampoline_zero, /* powerUpDART */ + [574] = trampoline_luma_span_h17p, [575] = trampoline_hotplug_h17p, [576] = trampoline_nop, [582] = iomfbep_cb_create_dfb_surface, /* create_default_fb_surface */ From 264f86d1a559628ad478938b4a24587c4685db94 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:57:31 +1100 Subject: [PATCH 0487/1579] drm: apple: register the measured internal H17P default surface The internal H17P default-framebuffer registration uses an ARGB format word, an opaque layout word and reserved surface handle four. The existing template sends the older profile's different values. Comparing the entire registered descriptor against a Linux startup capture exposes the mismatch before the firmware receives it. Override those three fields for the measured H17P profile using explicit byte/unaligned encoding. Preserve the established H17G descriptor and leave presentation surface construction on its separate path. The full 560-byte registration request now reproduces the retained Linux input. Hardware registration and presentation remain separate checks. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index a580ccd582f73..37d15754616e6 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -338,6 +338,13 @@ static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) s[0x035] = 0x02; s[0x051] = 0x01; s[0x149] = 0x01; + + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + /* Internal H17P default registration differs from the older profile. */ + memcpy(s + 11, "ARGB", 4); + put_unaligned_le32(0x00e44000, s + 41); + s[53] = BIT(2); + } } /* From 2c6e48a992b52fa993cc944039a0f6edbf206361 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 08:15:17 +1100 Subject: [PATCH 0488/1579] drm: apple: acknowledge bounded unknown H17P callbacks H17P waits for every firmware callback acknowledgment. Stopping on an unknown callback leaves the outer command outstanding and blocks all later presentation. Its generic unsupported-callback response is the zero-filled output area followed by the normal acknowledgment. Restore that bounded fallback for callbacks without a handler. Validate the header, tag range, stack depth, channel extent and complete record length before touching output. Typed handlers still require their known input/output sizes. Record unknown tags once per device, preserving the older firmware profiles' established dispatch behavior. Dispatcher fault injection covers malformed envelopes, output canaries, missing handlers and the complete measured startup callback set. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/iomfb.c | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index a73eaa90ee5ae..9d1db4fe27fba 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -370,6 +370,7 @@ struct apple_dcp { /* Staging wire record; serialization never mutates the atomic inputs. */ struct dcp_present_h17p present_h17p; struct dcp_present_state_h17p present_state_h17p; + DECLARE_BITMAP(unknown_callbacks, IOMFB_MAX_CB); /* Serializes H17P transmit preparation with RTKit receive callbacks. */ struct { diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 4d3c0c129f49b..81a5d40ed2d01 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -560,6 +560,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, int tag; struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; + bool handled; if (dcp->fw_compat == DCP_FIRMWARE_H17P && (length < sizeof(*hdr) || !ch || ch->depth >= DCP_MAX_CALL_DEPTH || @@ -570,9 +571,13 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, } tag = dcp_parse_tag(hdr->tag); + handled = tag >= 0 && tag < IOMFB_MAX_CB && dcp->cb_handlers && + dcp->cb_handlers[tag]; if (dcp->fw_compat == DCP_FIRMWARE_H17P && - ((u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || + (tag < 0 || tag >= IOMFB_MAX_CB || + (u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + handled && !iomfb_validate_callback_h17p(tag, hdr->in_len, hdr->out_len)))) { dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", tag, hdr->in_len, hdr->out_len); @@ -583,15 +588,11 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, in = data + sizeof(*hdr); out = in + hdr->in_len; - if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || - !dcp->cb_handlers[tag]) { - dev_warn(dev, "received unknown callback %c%c%c%c\n", - hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); - if (iomfb_uses_queue(dcp)) { - /* A measured size does not establish a safe reply payload. */ - WRITE_ONCE(dcp->crashed, true); - return; - } + if (!handled) { + if (!iomfb_uses_queue(dcp) || + !test_and_set_bit(tag, dcp->unknown_callbacks)) + dev_warn(dev, "received unknown callback %c%c%c%c\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); /* * Leaving a callback unanswered wedges the coprocessor: it * waits for the ack forever and the outer call never returns. From 0213f782787c9163a07dbd2637cd692830d5d541 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 08:18:56 +1100 Subject: [PATCH 0489/1579] drm: apple: describe T8140 firmware clock aperture in the hardware profile The internal T8140 firmware exposes a sixth register aperture at 0x302800000 with size 0xbc000. Its bandwidth callback names the scratch at offset 0x20000 and request doorbell at offset 0x68000. Using the AP PMGR window from an older loader tree supplies different addresses and breaks the display firmware's clock setup. Describe this firmware aperture in the T8140 hardware profile. Append it only after the five required display windows, validate both offsets and use the same resource for the firmware register and bandwidth replies. This does not map, prime or write the clock window from the AP. Older profiles retain their DT-provided bandwidth resources and probe checks. The resource matches the independently retained disp0 ADT register and Linux register reply. Resource tests cover the two bandwidth addresses, missing windows, bad offsets and the older DT path. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 4 ++++ drivers/gpu/drm/apple/dcp.c | 28 +++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 9d1db4fe27fba..7ccd11d7744e6 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -172,6 +172,10 @@ struct apple_dcp_hw_data { */ bool adopt_live_session; enum dcp_firmware_version firmware_compat; + /* Firmware-visible clock aperture, independent of the AP PMGR window. */ + const struct resource *firmware_clock; + u32 firmware_scratch; + u32 firmware_request; }; /* TODO: move IOMFB members to its own struct */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index b7d6100b3e3de..4083b21e50679 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1892,6 +1892,24 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) dcp->disp_registers[i] = res; } + if (dcp->hw.firmware_clock) { + /* The internal firmware's last aperture follows five display windows. */ + if (count != 5 || resource_size(dcp->hw.firmware_clock) < 4 || + dcp->hw.firmware_scratch > resource_size(dcp->hw.firmware_clock) - 4 || + dcp->hw.firmware_request > resource_size(dcp->hw.firmware_clock) - 4) + return -EINVAL; + + dcp->disp_bw_scratch_res = *dcp->hw.firmware_clock; + dcp->disp_bw_scratch_index = count; + dcp->disp_bw_scratch_offset = dcp->hw.firmware_scratch; + dcp->disp_bw_doorbell_res = dcp->disp_bw_scratch_res; + dcp->disp_bw_doorbell_res.start += dcp->hw.firmware_request; + dcp->disp_bw_doorbell_res.end = dcp->disp_bw_doorbell_res.start + 3; + dcp->disp_registers[count] = &dcp->disp_bw_scratch_res; + dcp->nr_disp_registers = count + 1; + return 0; + } + /* load pmgr bandwidth scratch resource and offset */ ret = dcp_get_bw_scratch_reg(dcp, count); if (ret < 0) @@ -2225,6 +2243,7 @@ static const struct component_ops dcp_comp_ops = { static int dcp_platform_probe(struct platform_device *pdev) { + const struct apple_dcp_hw_data *hw = of_device_get_match_data(&pdev->dev); enum dcp_firmware_version fw_compat; struct device *dev = &pdev->dev; struct apple_dcp *dcp; @@ -2240,7 +2259,8 @@ static int dcp_platform_probe(struct platform_device *pdev) * device trees. This prevents replacing simpledrm and ending up without * display. */ - if (!of_property_present(dev->of_node, "apple,bw-scratch")) + if (!of_property_present(dev->of_node, "apple,bw-scratch") && + !hw->firmware_clock) return dev_err_probe(dev, -ENODEV, "Incompatible devicetree! " "Use devicetree matching this kernel.\n"); @@ -2458,10 +2478,16 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { }; /* The internal T8140 endpoint uses the measured H17P method profile. */ +static const struct resource t8140_firmware_clock = + DEFINE_RES_MEM(0x302800000, 0xbc000); + static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { .num_dptx_ports = 0, .adopt_live_session = true, .firmware_compat = DCP_FIRMWARE_H17P, + .firmware_clock = &t8140_firmware_clock, + .firmware_scratch = 0x20000, + .firmware_request = 0x68000, }; /* From 4d993201a5ef5abf4252a25c1ae0cc428a9b6897 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:46:10 +1100 Subject: [PATCH 0490/1579] drm: apple: latch bounded H17P backlight takeover hints Linux powerlog observations contain millinits brightness hints before presentation. Later reports can differ from the first commanded level, so use the first valid panel-bounded hint and ignore subsequent reports once the policy is ready. Keep the existing panel maximum and 509-nit software clamp without inventing a fallback or calibration. Connect the public report parser to the queued backlight policy. Reject millinits overflow before narrowing, and keep registration and legacy brightness submission unavailable until the measured profile has a hint. Preserve the H17G and older firmware paths. Physical inherited-level continuity remains a hardware qualification gate. Add KUnit coverage for conversion bounds, zero hints and protection from late reports after control registration. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.c | 24 ++++++++++++++++++ drivers/gpu/drm/apple/dcp_backlight.h | 12 +++++++++ drivers/gpu/drm/apple/dcp_backlight_test.c | 29 ++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 4 +-- drivers/gpu/drm/apple/parser.c | 2 ++ drivers/gpu/drm/apple/systemep.c | 13 +++++++--- 6 files changed, 79 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 61d05b93c6698..6a2ad5bcf3fb7 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -85,9 +85,26 @@ static u32 brightness_part2[] = { bool dcp_backlight_active(struct apple_dcp *dcp) { return dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && READ_ONCE(dcp->backlight.state.ready); } +int dcp_backlight_takeover(struct apple_dcp *dcp, u32 millinits) +{ + u32 maximum, nits; + int ret; + + if (dcp_backlight_active(dcp)) + return 0; + maximum = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + ret = dcp_bl_takeover_nits(maximum, millinits, &nits); + if (ret) + return ret; + + /* Keep the first bounded hint. Later reports are not user requests. */ + return iomfb_configure_backlight_h17p(dcp, maximum, true, nits, false, 0); +} + static int dcp_get_brightness(struct backlight_device *bd) { struct apple_dcp *dcp = bl_get_data(bd); @@ -306,6 +323,9 @@ int dcp_backlight_update(struct apple_dcp *dcp) dcp_backlight_kick(dcp); return 0; } + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return -ENODATA; /* * Do not actively try to change brightness if no mode is set. @@ -382,6 +402,10 @@ int dcp_backlight_register(struct apple_dcp *dcp) .scale = BACKLIGHT_SCALE_LINEAR, }; props.max_brightness = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_backlight_active(dcp)) + return -ENODATA; if (dcp_backlight_active(dcp)) { spin_lock_irqsave(&dcp->backlight.lock, flags); if (!dcp->backlight.state.ready) { diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 6d52a8f9eb7fe..e93ee7511c626 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -29,6 +29,17 @@ struct dcp_backlight_present { u32 nits; }; +/* Public powerlog reports use millinits; validate before rounding to nits. */ +static inline int dcp_bl_takeover_nits(u32 maximum, u32 millinits, u32 *nits) +{ + if (!maximum || maximum > INT_MAX) + return -EINVAL; + if ((u64)millinits > (u64)maximum * 1000) + return -ERANGE; + *nits = millinits / 1000; + return 0; +} + /* Inputs must come from admitted panel/loader records, never a guess. */ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, bool inherited_valid, u32 inherited, @@ -151,6 +162,7 @@ struct apple_dcp; * Configure before registration and keep the callback valid until teardown. */ bool dcp_backlight_active(struct apple_dcp *dcp); +int dcp_backlight_takeover(struct apple_dcp *dcp, u32 millinits); int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, bool inherited_valid, u32 inherited, bool default_valid, u32 default_nits, diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index e990ea3e0bf06..a130f6f414712 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -6,6 +6,34 @@ #include "dcp_backlight.h" +static void backlight_millinits_takeover_test(struct kunit *test) +{ + struct dcp_backlight_state state; + u32 nits = 123; + + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(0, 100000, &nits), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(U32_MAX, 100000, &nits), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, 509001, &nits), -ERANGE); + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), -ERANGE); + KUNIT_EXPECT_EQ(test, nits, 123U); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 100999, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 100U); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 509, true, nits, false, 0), 0); + KUNIT_EXPECT_EQ(test, state.actual, 100U); + KUNIT_EXPECT_FALSE(test, state.dirty); + /* Registration freezes the hint before exposing the controls. */ + state.controlled = true; + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 400)); + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 0)); + KUNIT_EXPECT_EQ(test, state.target, 100U); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 0, &nits), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 509, true, nits, false, 0), 0); + KUNIT_EXPECT_EQ(test, state.target, 0U); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 509000, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 509U); +} + static void backlight_takeover_test(struct kunit *test) { struct dcp_backlight_state state = {}; @@ -187,6 +215,7 @@ static void backlight_inflight_dpms_test(struct kunit *test) } static struct kunit_case backlight_cases[] = { + KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), KUNIT_CASE(backlight_range_test), KUNIT_CASE(backlight_dpms_cycles_test), diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 37d15754616e6..cfc7ca924d842 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -439,8 +439,8 @@ static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *pr case IOMFB_PROPERTY_NITS: { #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - /* Configured H17P takeover uses only the admitted sample interface. */ - if (dcp_backlight_active(dcp)) + /* Measured H17P takeover comes from the powerlog hint interface. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) break; #endif if (dcp_has_panel(dcp)) { diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index dfe5b905cc43f..16027c1923369 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -1192,6 +1192,8 @@ int parse_system_log_mnits(struct dcp_parse_ctx *handle, struct dcp_system_ev_mn if (!type_match || mnits < 0 || idac < 0 || timestamp < 0) return -EINVAL; + if (mnits > U32_MAX) + return -ERANGE; entry->millinits = mnits; entry->idac = idac; diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c index 3133cd539b7f1..a32f0a0bcf6a0 100644 --- a/drivers/gpu/drm/apple/systemep.c +++ b/drivers/gpu/drm/apple/systemep.c @@ -75,9 +75,7 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype struct apple_dcp *dcp = service->ep->dcp; int ret; - /* H17P takeover decoding requires an admitted record. Later zero reports - * must not overwrite the cached target or completed level. - */ + /* Later reports must not overwrite a latched hint or Linux's target. */ if (dcp_backlight_active(dcp)) return 0; @@ -102,6 +100,15 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype dev_dbg(dcp->dev, "systemep: mNits event: Nits: %u.%03u, iDAC: %u\n", mnits.millinits / 1000, mnits.millinits % 1000, mnits.idac); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + if (!dcp_has_panel(dcp)) + return 0; + ret = dcp_backlight_takeover(dcp, mnits.millinits); + if (ret && ret != -EBUSY) + dev_dbg(dcp->dev, "backlight takeover hint unavailable: %d\n", ret); + return 0; + } dcp->brightness.nits = mnits.millinits / 1000; From cf69782abab85242968918ac5b8530299d8f723c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:53:14 +1100 Subject: [PATCH 0491/1579] drm: apple: encode H17P backlight activation and panel ceilings Linux brightness changes and soft DPMS presents use the same activation bytes and integer binary64 nits field. Encode the complete brightness block through the existing queued reservation, preserving the panel ceiling pair on every frame, including idle and pre-takeover frames. Refuse missing ceilings instead of submitting a zero calibration pair. An accepted matching completion updates the completed commanded level. Idle presents leave activation disabled; zero-nit enabled presents blank the retained scanout and unblank restores the user's target. Keep the wire ceiling separate from the existing exposed software brightness cap. Add a native actual-function fixture with captured packet fragments, full-range enabled/disabled replay, registration and reservation checks, rejection/coalescing and twenty software DPMS cycles. Document its mocked transport limits and the remaining integrated hardware qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/gpu/apple-dcp-backlight.rst | 56 +++- drivers/gpu/drm/apple/iomfb.c | 14 +- drivers/gpu/drm/apple/iomfb_h17p.c | 31 ++- drivers/gpu/drm/apple/iomfb_h17p.h | 3 +- drivers/gpu/drm/apple/iomfb_template.c | 3 +- tools/testing/apple/backlight-golden.json | 297 ++++++++++++++++++++++ tools/testing/apple/check-backlight.py | 279 ++++++++++++++++++++ 7 files changed, 664 insertions(+), 19 deletions(-) create mode 100644 tools/testing/apple/backlight-golden.json create mode 100644 tools/testing/apple/check-backlight.py diff --git a/Documentation/gpu/apple-dcp-backlight.rst b/Documentation/gpu/apple-dcp-backlight.rst index e47e7f3b4dd4b..e486fcf4840df 100644 --- a/Documentation/gpu/apple-dcp-backlight.rst +++ b/Documentation/gpu/apple-dcp-backlight.rst @@ -5,20 +5,19 @@ Apple DCP H17P backlight integration The backlight policy implements the clean display specification's BACKLIGHT-1 through BACKLIGHT-4, with IOMFB-3 and LIFETIME-1 through LIFETIME-3 constraints. -It extends the existing Asahi backlight driver. It contains no H17P brightness -wire layout, calibration table, property command, or guessed initial level. -Policy activation requires explicit successful configuration. Existing -unconfigured H17P sessions, including T8142, keep their established DAC, -property/powerlog and DPMS paths. Older firmware remains on its existing path. -Neo startup must configure the policy before exposing the backlight or starting -its DPMS/brightness traffic; a firmware-version match alone does not activate -this policy. +It extends the existing Asahi backlight driver. The measured internal H17P profile carries brightness activation through the +central present queue. It latches the first bounded powerlog hint without +sending a brightness update and leaves registration unavailable if there is +no hint. H17G and older firmware retain their existing paths. Physical loader +level continuity remains a measurement gate. Admission and startup --------------------- -The central display worker must call ``dcp_backlight_configure()`` from process -context after linking the CRTC and admitting all required records. Its arguments +The measured profile connects the public powerlog parser to +``iomfb_configure_backlight_h17p()`` after linking the CRTC. Generic users of +``dcp_backlight_configure()`` must still supply validated inputs from process +context. Its arguments are the validated maximum commanded nits, an optional inherited level, an optional documented ADT default, and a callback that schedules the central outbound queue. Maximum must fit the backlight core's signed integer range. @@ -64,7 +63,8 @@ configuration or surface; ``-EBUSY`` means a prior brightness transaction is outstanding; ``-EALREADY`` means no brightness change is pending. These results do not clear pending work. The serializer places the returned level only in admitted fields of the next valid present. A normal present without a brightness -change keeps the inherited/completed level; its clean layout must support that. +change disables the measured activation fragment, preserving the existing +physical level without resending a level. The sequence is a software cookie, not a wire present ID. Associate the cookie with the central transaction and its strict wire ID. Call @@ -127,3 +127,37 @@ cached restore, both suspend/DPMS orders, static-surface deferral, busy sequenci rejection and stale completion. They do not qualify firmware fields or GNOME. Hardware acceptance requires O-BACKLIGHT, O-BACKLIGHT-TAKEOVER (10 Hz PBwo with no jump above 0.05 W), O-BACKLIGHT-CRC, O-DPMS (twenty cycles), and O-SUSPEND. + +Measured activation and software replay +-------------------------------------- + +Linux-boundary measurements on the default J700 driver establish an 18-byte +fragment at A408 offsets 0x354 through 0x365. Brightness-enabled presents use +``01 00 00 00 01 01 01 00 01 00`` followed by integer little-endian binary64 +nits. Ordinary presents disable the activation and nits fragment. Its individual control +bytes have no separately inferred semantics. The two binary64 ceiling fields +at 0x36e and 0x376 carry the raw panel maximum on every internal present, +including ordinary frames and frames before a takeover hint arrives. Enabled +updates additionally carry unity at 0x37e and 0x3e6; other scalars are zero. +The complete owned block ends at 0x3ed. The observed 525-nit ceiling is distinct +from the 509-nit exposed software range. Missing ceilings refuse serialization. +The older DAC/power words at 0x32f through 0x33b stay zero on this profile, +including before a takeover hint arrives. Enabled zero nits blanks the +panel while keeping the pipe and retained surface alive. Requests while +blanked change the restore target; unblank restores it through the same queue. + +The measurements include sysfs changes, two GNOME Mutter SetBacklight API +steps and one PowerSaveMode off/on cycle. The latter reaches near-zero panel +power while display-pipe power stays at 605 mW. Cached sysfs actual brightness +on the observed driver is not physical readback while blanked. These records +do not establish clean-driver firmware acceptance, physical takeover continuity, +static-screen latency, suspend/resume or a twenty-cycle hardware gate. + +The native fixture extracts the actual registration, takeover and queue +reservation functions and replays captured fragments under AddressSanitizer +and UndefinedBehaviorSanitizer:: + + python3 tools/testing/apple/check-backlight.py --output /path/to/scratch + +It mocks locks, scheduling, DRM registration and transport. The fragment corpus +retains hashes of clean Linux observations, with runtime addresses omitted. diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 81a5d40ed2d01..044a6a7c675ef 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -989,9 +989,16 @@ bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, int ret; lockdep_assert_held(&dcp->iomfb.lock); - if (!dcp_backlight_active(dcp)) + if (!dcp_backlight_active(dcp)) { + if (!dcp_has_panel(dcp)) + return true; + if (!dcp->brightness.maximum) + return false; + iomfb_encode_backlight_h17p(wire, 0, + dcp->brightness.maximum, false); return true; - if (!transaction) + } + if (!transaction || !dcp->brightness.maximum) return false; for (i = 0; i < SWAP_SURFACES; i++) have_surface |= !request->surf_null[i]; @@ -1002,7 +1009,8 @@ bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, transaction->backlight = present; transaction->backlight_reserved = true; } - iomfb_encode_backlight_h17p(wire, present.nits); + iomfb_encode_backlight_h17p(wire, present.nits, + dcp->brightness.maximum, !ret); return true; } diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 6ac01ee69fcf4..85d1a4d13ceff 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -110,7 +110,7 @@ void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, } /* Integer commanded nits are exact binary64 values; no kernel FP is needed. */ -void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits) +static u64 iomfb_nits_binary64_h17p(u32 nits) { u64 value = 0; unsigned int exponent; @@ -120,8 +120,33 @@ void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits) value = (u64)(exponent + 1023) << 52; value |= ((u64)nits << (52 - exponent)) & GENMASK_ULL(51, 0); } - /* Measured from Linux presents during ordinary sysfs brightness changes. */ - put_unaligned_le64(value, wire->swap + 0x35e); + return value; +} + +void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, + u32 maximum, bool update) +{ + u64 ceiling = iomfb_nits_binary64_h17p(maximum); + u64 unity = iomfb_nits_binary64_h17p(1); + + /* This profile does not use the older DAC/power words. */ + memset(wire->swap + 0x32f, 0, 0xd); + memset(wire->swap + 0x354, 0, 0x9a); + /* The ceiling pair must remain valid even on a brightness-idle frame. */ + put_unaligned_le64(ceiling, wire->swap + 0x36e); + put_unaligned_le64(ceiling, wire->swap + 0x376); + if (!update) + return; + + /* Linux brightness changes and soft DPMS use the same activation bytes. */ + put_unaligned_le32(1, wire->swap + 0x354); + wire->swap[0x358] = 1; + wire->swap[0x359] = 1; + wire->swap[0x35a] = 1; + put_unaligned_le16(1, wire->swap + 0x35c); + put_unaligned_le64(iomfb_nits_binary64_h17p(nits), wire->swap + 0x35e); + put_unaligned_le64(unity, wire->swap + 0x37e); + put_unaligned_le64(unity, wire->swap + 0x3e6); } static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 0121d334c214a..e4eb5be2b0143 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -67,7 +67,8 @@ static_assert(sizeof(struct dcp_present_h17p) == 0xe9c); static_assert(offsetof(struct dcp_present_h17p, surf) == 0x588); static_assert(offsetof(struct dcp_present_h17p, tail) == 0xe38); -void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits); +void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, + u32 maximum, bool update); void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, const struct dcp_swap_submit_req_h17p *request); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index cfc7ca924d842..ea0d29d55fc59 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2195,7 +2195,8 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru /* update brightness if changed */ update_brightness = dcp_has_panel(dcp) && dcp->brightness.update; #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - update_brightness = update_brightness && !dcp_backlight_active(dcp); + update_brightness = update_brightness && + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; #endif if (update_brightness) { req->swap.bl_unk = 1; diff --git a/tools/testing/apple/backlight-golden.json b/tools/testing/apple/backlight-golden.json new file mode 100644 index 0000000000000..d1a8af2ab2011 --- /dev/null +++ b/tools/testing/apple/backlight-golden.json @@ -0,0 +1,297 @@ +[ + { + "nits": 197, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000a06840000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "59bc5374f6e05d6bcf2839b9522bdb9456074f80c7d542c0ecdc7e90cf7c1d16" + }, + { + "nits": 188, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000806740000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "05c226553a34876237f2eef41246a75d013cd29115e68437df03f594f2459941" + }, + { + "nits": 179, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000606640000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "e74693443c8a029a060cf3612556f93837a82cc3f3936975dcd4848432186067" + }, + { + "nits": 170, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000406540000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "67521cee14ce162f0a733a815b1f059f3fe394c0c5e87bfe178f14b3acceb52c" + }, + { + "nits": 161, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000206440000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "f2996dbaff64ac3d653cd4b420df919c418697fcf8dc4f995f562e5839efcef2" + }, + { + "nits": 152, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000006340000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "b134853efd740adc11f258a206b4deb098df1765191c073178e033df83d17148" + }, + { + "nits": 143, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000e06140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "1488c635d8d55ab49f73f9d9698a4b3e0f8fe95fcf925b73c21f4846fa7188f2" + }, + { + "nits": 134, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000c06040000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "132f2aba31bdc58731510361e6b934284ac81baeed5346dc235b615a43f6d798" + }, + { + "nits": 125, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000405f40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a4f3e35516ba00acbed9c6c04ebb564334dab983b1a016f570dfe739ea2a9ae3" + }, + { + "nits": 116, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000005d40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "837951003a3632bb5c063e3ee70606583af2f843ef722864cbf461ef01fc5a81" + }, + { + "nits": 106, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000805a40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a68f25f1ce7f63c335781888df47a8eb12f863c37795b2377d69c270dc063cac" + }, + { + "nits": 97, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000405840000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "6c5108c9aadd93c72b2f0d5b72adb09f55ca63159f909d0876714bbf62f80ea5" + }, + { + "nits": 88, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000005640000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "b2db893e953fda0c249ecf7c674acfd2018d1267985cc09d842b2f38910b2189" + }, + { + "nits": 79, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000c05340000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "321da86969fcfdb11f7d6e508a627bb0f9146defd48c4fa0d0cd76d7b63fd49b" + }, + { + "nits": 70, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000805140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "66443759f999549d7d9b076c338cc9f5ba7b2606ec5f764f14a17f8a23f28acb" + }, + { + "nits": 61, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000804e40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "4c3447c9290bdda6fc9f38d024f8c8482fc7692b61243680cf6dc2012dd8d8a1" + }, + { + "nits": 52, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000004a40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "ce727d2eb98cbdeba8be5215577c2486ad8d5ad1b7ca1db2358d7c27f0cc2f1f" + }, + { + "nits": 43, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000804540000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "00f94f04f7e431494dddf154cf8e68a1e3fe91afa3ab18d41c52223d582112ae" + }, + { + "nits": 36, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000004240000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "4ae2c36b9d9a773bcfaf68c74b81e3761f4b0da68ffff4f7a55194207b313828" + }, + { + "nits": 59, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000804d40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "aa653a1c3d85ad9cf3a487aebdfafd0332c6d81335e9bc051292cab4610997de" + }, + { + "nits": 92, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000005740000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a57f482514132d36aa5c46026b7c579bd74b1d21b84e5ed67e4123eab72e1054" + }, + { + "nits": 125, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000405f40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "6260e7be2d29193f51a0fe60337efcdc9c6e73b469f527afb9f0ff83765467ec" + }, + { + "nits": 140, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000806140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "0eab7f3ebd304ddb5f40d2c8bc1ecd2110b166faf70f89b98a2521e0b6c7ac95" + }, + { + "nits": 200, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000006940000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "98df21464117081715a48b0076304aad0f2b8307ee682c20a7feafd9a1be9202" + }, + { + "nits": 287, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000f07140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "7f5df7ed6f804a3340c9782698f5292e3b4117f2eb9943778010936a6b2746a4" + }, + { + "nits": 375, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000707740000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "994697bd5bb64697f4d2bbbf2311a7c383f356161184e2a9aeea610515dda25e" + }, + { + "nits": 400, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000007940000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a9fbdac983180e10843468b14c2cade1ee7d0e8711e02fce56489112c91b38e0" + }, + { + "nits": 335, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000f07440000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "4cf3e7680e17a183e40e54b69d54693f43f3643346de050beb5c9cb73214b8f8" + }, + { + "nits": 249, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000206f40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "9b174a609f7fda9e379923c5e1fba03952b6f44d3ad031dfa8660a015326725e" + }, + { + "nits": 163, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000606440000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "bea9a384a1ceb723ddda8416748e20057e67627d768e9d1a81207d9f1a21d812" + }, + { + "nits": 131, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000606040000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "2445a2a32d0cbf27f6eb394bc48ff08c58ca88a32e6a6f9cbe633e983ab05256" + }, + { + "nits": 185, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000206740000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "1a23772d33bedf58b4c566ae84267a0631f7ced0d881ac9470667ec2eab477bc" + }, + { + "nits": 264, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000807040000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "3bd83da635cd09c608e2124c75e9aaa9ed9337622e0a818c500ddf83d8146d33" + }, + { + "nits": 337, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000107540000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "d44097a52f6cef9ca7c4ddf39897bf67a9915b520b154d50d5210595997f0495" + }, + { + "nits": 357, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000507640000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "51f19fe9d95e6d95b8b81b158a3d5c2db391973a66695858610ae24ade44c0d8" + }, + { + "nits": 0, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000000000000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "3f3d1f6ae52991efa63ac64d74e3e9f28e8b000791732102a69a5c05452d2512" + }, + { + "nits": 140, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000806140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "5283a0beb8119ca257f5c00ae267598427c2bb01a472395c83b325f8c7a4b81a" + }, + { + "nits": 212, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000806a40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "d1bdce12296d366b8315c73d850fc80e92720748aff8ce3ff28a136e6104f0c8" + }, + { + "nits": 313, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000907340000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a86f277bb5b34a2b13301d2c3464db76325d0bacd6009b5e46329fe97c82df60" + }, + { + "nits": 413, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000d07940000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "bca47e8efde3063740bd65f174ccc0b502b5ef2350cc821397679724ecf7d2d6" + }, + { + "nits": 454, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000607c40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "17c4a25be2d6cefb43cb282859f500415e3902be3d9d9ef5ab9045e6e9c6af53" + }, + { + "nits": 100, + "maximum": 525, + "enabled": false, + "fragment": "00000000000000000000000000000000000000000000000000000000000000688040000000000068804000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "payload_sha256": "7de62e12dc678b02f3b47b68ab369efc651cc05d765d2e024652583e410461da", + "provenance": "run24 clean ordinary startup presents" + } +] diff --git a/tools/testing/apple/check-backlight.py b/tools/testing/apple/check-backlight.py new file mode 100644 index 0000000000000..e1df3dc3458de --- /dev/null +++ b/tools/testing/apple/check-backlight.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Native replay of clean brightness fragments and actual policy/queue functions. + +Run with --output on a scratch volume. This mocks kernel locks, DRM registration +and queue transport; it does not claim firmware or KUnit boot acceptance. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--output', required=True, type=Path) +parser.add_argument('--cc', default='clang') +args = parser.parse_args() +root = Path(__file__).resolve().parents[3] +src = root / 'drivers/gpu/drm/apple' +out = args.output.resolve() +out.mkdir(parents=True, exist_ok=True) + + +def function(file, name): + text = (src / file).read_text() + match = re.search(r'^\w[\w\s*]*?\b' + name + r'\(', text, re.M) + assert match, name + start = text.index('{', match.end()) + depth = 1 + pos = start + 1 + while depth: + depth += (text[pos] == '{') - (text[pos] == '}') + pos += 1 + return text[match.start():pos] + + +include = out / 'include/linux' +include.mkdir(parents=True, exist_ok=True) +(include / 'types.h').write_text('#include \n#include \ntypedef uint8_t u8;typedef uint32_t u32;typedef uint64_t u64;\n') +(include / 'limits.h').write_text('#include \n#define U32_MAX UINT32_MAX\n') +(include / 'errno.h').write_text('#include \n#ifndef ENODATA\n#define ENODATA 61\n#endif\n') +prelude = r''' +#include +#include +#include +#include +#include +#include +#include "dcp_backlight.h" +#define DCP_FIRMWARE_H17P 17 +#define DCP_IOMFB_METHODS_H17G 1 +#define MAX_BRIGHTNESS_PART2 510U +#define SWAP_SURFACES 4 +#define READ_ONCE(x) (x) +#define min(a,b) ((a)<(b)?(a):(b)) +#define lockdep_assert_held(x) ((void)(x)) +#define mutex_lock(x) ((void)(x)) +#define mutex_unlock(x) ((void)(x)) +#define spin_lock_irqsave(x,f) do { (void)(x);(f)=0; } while(0) +#define spin_unlock_irqrestore(x,f) do { (void)(x);(void)(f); } while(0) +#define GENMASK_ULL(h,l) ((~UINT64_C(0) >> (63-(h))) & (~UINT64_C(0) << (l))) +#define fls(x) (32-__builtin_clz(x)) +#define BACKLIGHT_PLATFORM 1 +#define BACKLIGHT_SCALE_LINEAR 2 +struct device { int unused; }; +struct backlight_device { int unused; }; +struct backlight_properties { int type;u32 brightness,max_brightness;int scale; }; +struct dcp_present_h17p { u8 swap[0x588];u8 guard[32]; }; +struct dcp_swap_submit_req_h17p { u8 surf_null[SWAP_SURFACES]; }; +struct iomfb_transaction { + struct dcp_backlight_present backlight; + bool backlight_reserved,brightness_only,completed; +}; +struct apple_dcp { + int fw_compat;struct { int iomfb_method_profile; } hw; + bool panel;void *crtc;struct device *dev; + int bl_register_mutex,bl_register_wq; + struct { u32 maximum;u32 nits;struct backlight_device *bl_dev;u32 dac; } brightness; + struct { int lock;struct dcp_backlight_state state;void (*kick)(struct apple_dcp *); } backlight; + struct { int lock;struct iomfb_transaction *active; } iomfb; +}; +static int schedule_count,register_count,retire_count,dac_count; +static struct backlight_properties registered; +static const int dcp_backlight_ops; +static bool dcp_has_panel(struct apple_dcp *dcp) { return dcp->panel; } +static void schedule_work(int *work) { (void)work;schedule_count++; } +static void iomfb_backlight_kick(struct apple_dcp *dcp) { (void)dcp; } +static void iomfb_scanout_complete_h17p(struct apple_dcp *dcp) { (void)dcp;retire_count++; } +static u32 calculate_dac(struct apple_dcp *dcp,u32 nits) { (void)dcp;dac_count++;return nits; } +static struct backlight_device *devm_backlight_device_register(struct device *dev,const char *name, + struct device *parent,struct apple_dcp *dcp,const int *ops,const struct backlight_properties *props) { + static struct backlight_device device; + (void)dev;(void)parent;(void)dcp;(void)ops; + assert(!strcmp(name,"apple-panel-bl"));registered=*props;register_count++;return &device; +} +#define IS_ERR(p) (!(p)) +#define PTR_ERR(p) ((void)(p),-ENOMEM) +static void put_unaligned_le64(u64 v,void *p) { for(int i=0;i<8;i++)((u8 *)p)[i]=v>>(8*i); } +static void put_unaligned_le32(u32 v,void *p) { for(int i=0;i<4;i++)((u8 *)p)[i]=v>>(8*i); } +static void put_unaligned_le16(uint16_t v,void *p) { for(int i=0;i<2;i++)((u8 *)p)[i]=v>>(8*i); } +int iomfb_configure_backlight_h17p(struct apple_dcp *,u32,bool,u32,bool,u32); +''' +functions = [ + ('iomfb_h17p.c', 'iomfb_nits_binary64_h17p'), + ('iomfb_h17p.c', 'iomfb_encode_backlight_h17p'), + ('iomfb.c', 'iomfb_uses_queue'), + ('dcp_backlight.c', 'dcp_backlight_active'), + ('dcp_backlight.c', 'dcp_backlight_configure'), + ('iomfb.c', 'iomfb_configure_backlight_h17p'), + ('dcp_backlight.c', 'dcp_backlight_takeover'), + ('dcp_backlight.c', 'dcp_backlight_register'), + ('dcp_backlight.c', 'dcp_backlight_prepare'), + ('dcp_backlight.c', 'dcp_backlight_complete'), + ('iomfb.c', 'iomfb_apply_backlight_h17p'), + ('iomfb.c', 'iomfb_present_failed_h17p'), + ('iomfb.c', 'iomfb_present_complete_h17p'), +] +main = r''' +static struct apple_dcp fixture(void) { + return (struct apple_dcp){.fw_compat=DCP_FIRMWARE_H17P,.panel=true, + .crtc=(void *)1,.brightness.maximum=525}; +} +static u64 level_bits(const struct dcp_present_h17p *wire) { + u64 value=0;for(int i=0;i<8;i++)value|=(u64)wire->swap[0x35e + i]<<(i*8);return value; +} +static double level(const struct dcp_present_h17p *wire) { + u64 bits=level_bits(wire);double value;memcpy(&value,&bits,sizeof(value));return value; +} +static void outside_unchanged(const struct dcp_present_h17p *wire) { + const u8 *data=(const u8 *)wire; + for(size_t i=0;i=0x32f && i<0x33c)assert(!data[i]); + else if(i<0x354||i>=0x3ee)assert(data[i]==0xa5); + } +} +int main(int argc,char **argv) { + struct dcp_present_h17p wire; + if(argc==3) { + memset(&wire,0xa5,sizeof(wire)); + iomfb_encode_backlight_h17p(&wire,strtoul(argv[1],NULL,0),525,atoi(argv[2])); + outside_unchanged(&wire); + for(int i=0x354;i<0x3ee;i++)printf("%02x",wire.swap[i]);puts("");return 0; + } + if(argc==2) { + for(u32 nits=0;nits<510;nits++)for(int update=0;update<2;update++) { + memset(&wire,0xa5,sizeof(wire)); + iomfb_encode_backlight_h17p(&wire,nits,525,update); + outside_unchanged(&wire); + printf("%u %d ",nits,update); + for(int i=0x354;i<0x3ee;i++)printf("%02x",wire.swap[i]);puts(""); + } + return 0; + } + struct apple_dcp dcp=fixture(); + assert(dcp_backlight_register(&dcp)==-ENODATA); + assert(!register_count); + assert(dcp_backlight_takeover(&dcp,509001)==-ERANGE); + assert(!dcp_backlight_active(&dcp)); + assert(dcp_backlight_takeover(&dcp,100000)==0); + assert(dcp.backlight.state.target==100 && dcp.backlight.state.actual==100); + assert(!dcp.backlight.state.dirty && schedule_count==1); + assert(dcp_backlight_takeover(&dcp,400000)==0); + assert(dcp_backlight_takeover(&dcp,0)==0); + assert(dcp.backlight.state.target==100 && schedule_count==1); + assert(dcp_backlight_register(&dcp)==0 && register_count==1); + assert(registered.brightness==100 && registered.max_brightness==509 && !dac_count); + assert(dcp.backlight.state.controlled); + struct iomfb_transaction transaction={.brightness_only=true}; + struct dcp_swap_submit_req_h17p request={.surf_null={0,1,1,1}}; + dcp.iomfb.active=&transaction; + struct apple_dcp unready=fixture(); + memset(&wire,0xa5,sizeof(wire)); + assert(iomfb_apply_backlight_h17p(&unready,&request,&wire)); + assert(!wire.swap[0x354] && !level_bits(&wire)); + assert(wire.swap[0x374] || wire.swap[0x375]); + unready.brightness.maximum=0; + memset(&wire,0xa5,sizeof(wire)); + assert(!iomfb_apply_backlight_h17p(&unready,&request,&wire)); + assert(wire.swap[0x354]==0xa5); + dcp.brightness.maximum=0; + assert(!iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + dcp.brightness.maximum=525; + memset(&wire,0xa5,sizeof(wire)); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + for(int i=0x354;i<0x366;i++)assert(!wire.swap[i]); + assert(!transaction.backlight_reserved); + assert(dcp_bl_request(&dcp.backlight.state,400,false,false)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==400 && dcp.backlight.state.actual==100); + assert(transaction.backlight_reserved); + assert(!dcp_backlight_complete(&dcp,transaction.backlight.sequence+1,true)); + assert(dcp_bl_request(&dcp.backlight.state,140,false,false)==0); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp.backlight.state.actual==400 && dcp.backlight.state.dirty && !retire_count); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==140); + iomfb_present_failed_h17p(&dcp); + assert(dcp.backlight.state.actual==400 && dcp.backlight.state.dirty); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp.backlight.state.actual==140 && !retire_count); + for(int i=0;i<20;i++) { + assert(dcp_bl_dpms(&dcp.backlight.state,false)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==0 && wire.swap[0x354]==1); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp.backlight.state.actual==0 && dcp.backlight.state.target==140); + assert(dcp_bl_dpms(&dcp.backlight.state,true)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==140); + assert(!iomfb_present_complete_h17p(&dcp)); + } + assert(!retire_count); + assert(dcp_bl_dpms(&dcp.backlight.state,false)==0); + assert(dcp_bl_request(&dcp.backlight.state,400,false,false)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==0 && dcp.backlight.state.target==400); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp_bl_dpms(&dcp.backlight.state,true)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==400); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp_bl_request(&dcp.backlight.state,36,false,false)==0); + memset(request.surf_null,1,sizeof(request.surf_null)); + memset(&wire,0xa5,sizeof(wire)); + assert(!iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(wire.swap[0x354]==0xa5 && dcp.backlight.state.dirty); + struct apple_dcp h17g=fixture();h17g.hw.iomfb_method_profile=DCP_IOMFB_METHODS_H17G; + assert(dcp_backlight_takeover(&h17g,100000)==-EINVAL); + assert(!h17g.backlight.state.ready); + struct apple_dcp external=fixture();external.panel=false; + assert(dcp_backlight_takeover(&external,100000)==-EINVAL); + struct apple_dcp unlinked=fixture();unlinked.crtc=NULL; + assert(dcp_backlight_takeover(&unlinked,100000)==-EINVAL); + struct apple_dcp zero=fixture(); + assert(dcp_backlight_takeover(&zero,0)==0); + assert(zero.backlight.state.target==0 && !zero.backlight.state.dirty); + puts("Actual takeover/registration/reservation/accepted completion/rejection/20 DPMS cycles passed"); + return 0; +} +''' +source = prelude + '\n'.join(function(f, n) for f, n in functions) + main +(out / 'fixture.c').write_text(source) +argv = [args.cc, '-std=gnu11', '-Wall', '-Wextra', '-Werror', '-fsanitize=address,undefined', + '-I' + str(out / 'include'), '-I' + str(src), str(out / 'fixture.c'), '-o', str(out / 'fixture')] +build = subprocess.run(argv, text=True, capture_output=True) +(out / 'compile.log').write_text(build.stdout + build.stderr) +assert not build.returncode, build.stderr +run = subprocess.run([str(out / 'fixture')], text=True, capture_output=True) +(out / 'run.log').write_text(run.stdout + run.stderr) +assert not run.returncode, run.stdout + run.stderr +# All measured brightness fragments, including DPMS zero and ordinary startup. +golden = json.loads(Path(__file__).with_name('backlight-golden.json').read_text()) +for case in golden: + fragment = subprocess.check_output([str(out / 'fixture'), str(case['nits']), str(int(case['enabled']))], text=True).strip() + assert fragment == case['fragment'], (case, fragment) +# Exact integer binary64 throughout the exposed range, plus disabled updates. +import struct +range_output = subprocess.check_output([str(out / 'fixture'), 'range'], text=True) +for line in range_output.splitlines(): + n, enabled, encoded = line.split() + nits, enabled = int(n), bool(int(enabled)) + expected = bytearray(0x9a) + expected[0x1a:0x22] = expected[0x22:0x2a] = struct.pack(' Date: Thu, 8 Oct 2026 08:34:48 +1100 Subject: [PATCH 0492/1579] drm: apple: enable the internal H17P pipe once before soft DPMS Internal H17P startup requires one out-of-band power-on request after client open. The soft-DPMS shortcut currently bypasses that first enable, although it is appropriate for later brightness-only transitions. Latch the matching successful enable reply and perform the initial enable before entering soft DPMS. Later unblank operations restore brightness without another power or display-device request. A missing or malformed enable reply stops presentation instead of treating the pipe as ready. Keep the measured H17P pipe live through off, sleep and shutdown entry points: surface-free presents and firmware power-down commands can cause null-base DMA faults or firmware NMI. Stop the software queue on shutdown and retain scanout ownership without claiming a final hardware stop. The older firmware profiles keep their established lifecycle paths. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 10 ++++++++++ drivers/gpu/drm/apple/iomfb_h17p.c | 1 + drivers/gpu/drm/apple/iomfb_template.c | 24 ++++++++++++++++++++++++ 4 files changed, 37 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 7ccd11d7744e6..916ee56d69ed0 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -272,6 +272,8 @@ struct apple_dcp { /* Is the DCP booted? */ bool active; + /* A successful initial enable keeps the H17P pipe running across DPMS. */ + bool pipe_enabled_h17p; /* eDP display without DP-HDMI conversion */ bool main_display; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4083b21e50679..82d910c1366ee 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1497,6 +1497,16 @@ void dcp_poweron(struct platform_device *pdev) int ret; if (dcp_uses_soft_dpms(dcp)) { + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !READ_ONCE(dcp->pipe_enabled_h17p)) { + iomfb_poweron_h17p(dcp); + if (!READ_ONCE(dcp->pipe_enabled_h17p)) { + dev_err(dcp->dev, "initial display pipe enable failed\n"); + WRITE_ONCE(dcp->crashed, true); + return; + } + } ret = dcp_backlight_dpms(dcp, true); if (ret) dev_warn(dcp->dev, "backlight restore unavailable: %d\n", ret); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 85d1a4d13ceff..b140d7ae385fa 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -539,6 +539,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { void iomfb_start_h17p(struct apple_dcp *dcp) { + WRITE_ONCE(dcp->pipe_enabled_h17p, false); dcp->cb_handlers = cb_handlers; dcp_start_signal(dcp, false, dcp_started, NULL); } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index ea0d29d55fc59..a61101520e547 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1420,6 +1420,12 @@ static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_wait_cookie *wait = cookie; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + out && get_unaligned_le64(out) == 1) + WRITE_ONCE(dcp->pipe_enabled_h17p, true); +#endif + if (wait) { complete(&wait->done); kref_put(&wait->refcount, release_wait_cookie); @@ -1560,6 +1566,12 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) struct DCP_FW_NAME(dcp_swap_start_req) swap_req = { 0 }; typeof(DCP_FW_UNION(dcp->swap)) *swap = &DCP_FW_UNION(dcp->swap); +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + /* The measured profile has no safe pipe-off or surface-free transition. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return; +#endif + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; @@ -1652,6 +1664,11 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) struct dcp_wait_cookie *cookie; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return; +#endif + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; @@ -2416,6 +2433,13 @@ void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp) struct dcp_wait_cookie *cookie; int ret; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + iomfb_queue_stop(dcp); + return; + } +#endif + cookie = kzalloc_obj(*cookie); if (!cookie) return; From 544eb67a1a17fd5f7ba0fff25280ff91542a0552 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:55:15 +1100 Subject: [PATCH 0493/1579] drm: apple: use ARGB for H17P fbdev H17P applies the stored fourth byte as straight alpha even when the framebuffer uses an X format. A depth-24 fbdev console therefore produces transparent pixels and appears black. Request depth 32 from the DMA fbdev helper only for the H17P internal-panel profile. Older firmware retains its existing XRGB console. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 85c76f3facd7d..b4e3b2a436c11 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -65,9 +65,30 @@ static int apple_drm_gem_dumb_create(struct drm_file *file_priv, return drm_gem_dma_dumb_create_internal(file_priv, drm, args); } +static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, + struct drm_fb_helper_surface_size *sizes) +{ + struct drm_crtc *crtc; + + drm_for_each_crtc(crtc, helper->dev) { + struct apple_dcp *dcp; + + dcp = platform_get_drvdata(to_apple_crtc(crtc)->dcp); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + /* H17P applies the stored fourth byte as straight alpha. */ + sizes->surface_bpp = 32; + sizes->surface_depth = 32; + break; + } + } + + return drm_fbdev_dma_driver_fbdev_probe(helper, sizes); +} + static const struct drm_driver apple_drm_driver = { DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), - DRM_FBDEV_DMA_DRIVER_OPS, + .fbdev_probe = apple_drm_fbdev_probe, .name = DRIVER_NAME, .desc = DRIVER_DESC, .major = 1, From ad45f2295793cb2a2d05987678f2ac5314d0b5c9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:56:15 +1100 Subject: [PATCH 0494/1579] drm: apple: configure opaque X formats on H17P H17P blends the stored X byte as alpha. Linux boundary measurements show that property 0x49 set to zero and the existing premultiplied surface marker make 8-bit XRGB scanout opaque without changing ARGB alpha. Describe the measured A352 request and send it as a priority item on the serialized H17P queue after the first completed present of each full power-on. Keep unqualified 10-bit XRGB off the H17P primary plane, and cover the admitted surface offsets, IOVA tail and property bytes with KUnit fixtures. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/apple_drv.c | 9 ++- drivers/gpu/drm/apple/dcp-internal.h | 4 ++ drivers/gpu/drm/apple/iomfb.c | 84 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_h17p.c | 13 ++++ drivers/gpu/drm/apple/iomfb_h17p.h | 15 +++++ drivers/gpu/drm/apple/iomfb_h17p_test.c | 84 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 5 ++ drivers/gpu/drm/apple/plane.c | 23 ++++++- drivers/gpu/drm/apple/plane.h | 1 + tools/testing/apple/check-backlight.py | 13 +++- 11 files changed, 249 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/drm/apple/iomfb_h17p_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index dc9ff905f283f..837aa713ad435 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -11,6 +11,7 @@ appledrm-y += ibootep.o appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o appledrm-y += iomfb_h17p.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index b4e3b2a436c11..db4abd811cb64 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -405,12 +405,19 @@ static int apple_probe_per_dcp(struct device *dev, u32 surf; int zpos = 0; bool supports_l10r = !dcp_fw_compat_is_12_x(dcp); + struct apple_dcp *dcp_data = platform_get_drvdata(dcp); + bool supports_xrgb2101010; enum drm_plane_type plane_type; + supports_xrgb2101010 = dcp_data->fw_compat != DCP_FIRMWARE_H17P || + dcp_data->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; + for_each_set_bit(surf, iomfb_surfaces, DCP_MAX_PLANES) { plane_type = (zpos == 0) ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY; planes[zpos] = apple_plane_init(drm, 1U << num, surf, - supports_l10r, plane_type); + supports_l10r, + supports_xrgb2101010, + plane_type); if (IS_ERR(planes[zpos])) return PTR_ERR(planes[zpos]); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 916ee56d69ed0..0720570ddd1ba 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -391,6 +391,7 @@ struct apple_dcp { unsigned int queued; bool stopped; bool backlight_queued; + atomic_t opaque_x_state; struct iomfb_scanout_h17p *scanout; struct iomfb_scanout_h17p *next_scanout; } iomfb; @@ -404,6 +405,9 @@ void iomfb_present_failed_h17p(struct apple_dcp *dcp); bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, const struct dcp_swap_submit_req_h17p *request, struct dcp_present_h17p *wire); +void iomfb_apply_opaque_x_h17p(struct apple_dcp *dcp, dcp_callback_t callback, + void *cookie); +void iomfb_opaque_x_reset_h17p(struct apple_dcp *dcp); int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, bool inherited_valid, u32 inherited, bool default_valid, u32 default_nits); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 044a6a7c675ef..dfcaaeb916996 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -171,6 +171,46 @@ static bool iomfb_channels_idle(struct apple_dcp *dcp) !dcp->ch_async.depth && !dcp->ch_oobasync.depth; } +enum iomfb_opaque_x_state { + IOMFB_OPAQUE_X_WAITING, + IOMFB_OPAQUE_X_NEEDED, + IOMFB_OPAQUE_X_QUEUED, + IOMFB_OPAQUE_X_READY, +}; + +struct iomfb_opaque_x_transaction { + struct iomfb_transaction transaction; +}; + +static void iomfb_opaque_x_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction); +static void iomfb_opaque_x_release(struct iomfb_transaction *transaction); + +static int iomfb_enqueue_opaque_x(struct apple_dcp *dcp) +{ + struct iomfb_opaque_x_transaction *opaque; + + lockdep_assert_held(&dcp->iomfb.lock); + if (atomic_read(&dcp->iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) + return 0; + if (dcp->iomfb.queued >= 32) + return -EBUSY; + opaque = kzalloc_obj(*opaque); + if (!opaque) + return -ENOMEM; + if (atomic_cmpxchg(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_NEEDED, + IOMFB_OPAQUE_X_QUEUED) != IOMFB_OPAQUE_X_NEEDED) { + kfree(opaque); + return 0; + } + + opaque->transaction.start = iomfb_opaque_x_start; + opaque->transaction.release = iomfb_opaque_x_release; + dcp->iomfb.queued++; + list_add(&opaque->transaction.link, &dcp->iomfb.pending); + return 0; +} + static void iomfb_discard_pending(struct apple_dcp *dcp) { struct iomfb_transaction *transaction, *next; @@ -208,6 +248,12 @@ static void iomfb_queue_advance(struct apple_dcp *dcp) schedule_work(&dcp->bl_update_wq); transaction->release(transaction); } + if (iomfb_enqueue_opaque_x(dcp)) { + WRITE_ONCE(dcp->crashed, true); + iomfb_discard_pending(dcp); + schedule_work(&dcp->vblank_wq); + return; + } if (!list_empty(&dcp->iomfb.pending)) schedule_work(&dcp->iomfb.work); } @@ -259,12 +305,48 @@ static void iomfb_queue_timeout(struct work_struct *work) mutex_unlock(&dcp->iomfb.lock); } +static void iomfb_opaque_x_complete(struct apple_dcp *dcp, void *out, + void *cookie) +{ + u32 status = out ? *(u32 *)out : ~0U; + + (void)cookie; + + if (status) { + dev_err(dcp->dev, "opaque X property failed: %u\n", status); + WRITE_ONCE(dcp->crashed, true); + schedule_work(&dcp->vblank_wq); + return; + } + + atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_READY); +} + +static void iomfb_opaque_x_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + (void)transaction; + iomfb_apply_opaque_x_h17p(dcp, iomfb_opaque_x_complete, NULL); +} + +static void iomfb_opaque_x_release(struct iomfb_transaction *transaction) +{ + kfree(container_of(transaction, struct iomfb_opaque_x_transaction, + transaction)); +} + +void iomfb_opaque_x_reset_h17p(struct apple_dcp *dcp) +{ + atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); +} + void iomfb_queue_init(struct apple_dcp *dcp) { mutex_init(&dcp->iomfb.lock); INIT_LIST_HEAD(&dcp->iomfb.pending); INIT_WORK(&dcp->iomfb.work, iomfb_queue_work); INIT_DELAYED_WORK(&dcp->iomfb.timeout, iomfb_queue_timeout); + atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); } void iomfb_queue_stop(struct apple_dcp *dcp) @@ -1038,6 +1120,8 @@ bool iomfb_present_complete_h17p(struct apple_dcp *dcp) if (transaction->brightness_only) return false; } + atomic_cmpxchg(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING, + IOMFB_OPAQUE_X_NEEDED); iomfb_scanout_complete_h17p(dcp); return true; } diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index b140d7ae385fa..fca112eb4ecf2 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -210,6 +210,19 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { #include "iomfb_template.c" +void iomfb_apply_opaque_x_h17p(struct apple_dcp *dcp, dcp_callback_t callback, + void *cookie) +{ + static const struct dcp_method_entry method = { + .name = "apply_property", + .tag = { 'A', '3', '5', '2' }, + }; + struct dcp_apply_property_h17p property = dcp_opaque_x_property_h17p(); + + dcp_push(dcp, false, &method, sizeof(property), sizeof(u32), &property, + callback, cookie); +} + static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index e4eb5be2b0143..9473940250297 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -67,6 +67,21 @@ static_assert(sizeof(struct dcp_present_h17p) == 0xe9c); static_assert(offsetof(struct dcp_present_h17p, surf) == 0x588); static_assert(offsetof(struct dcp_present_h17p, tail) == 0xe38); +struct dcp_apply_property_h17p { + u32 property; + u32 value; +} __packed; + +static_assert(sizeof(struct dcp_apply_property_h17p) == 0x8); + +static inline struct dcp_apply_property_h17p dcp_opaque_x_property_h17p(void) +{ + return (struct dcp_apply_property_h17p) { + .property = 0x49, + .value = 0, + }; +} + void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, u32 maximum, bool update); diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c new file mode 100644 index 0000000000000..c2a71ffa8cdbf --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include + +#include "iomfb_h17p.h" + +static void h17p_surface_wire_test(struct kunit *test) +{ + struct dcp_swap_submit_req_h17p *request; + struct dcp_present_h17p *wire; + struct dcp_surface *surface; + u64 iova = 0x123456789aULL; + + request = kunit_kzalloc(test, sizeof(*request), GFP_KERNEL); + wire = kunit_kmalloc(test, sizeof(*wire), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, request); + KUNIT_ASSERT_NOT_NULL(test, wire); + surface = &request->surf[0].base; + memset(wire, 0xa5, sizeof(*wire)); + surface->is_premultiplied = 1; + surface->plane_cnt = 1; + surface->plane_cnt2 = 1; + surface->format = DCP_FORMAT_BGRA; + surface->xfer_func = DCP_XFER_FUNC_SDR; + surface->colorspace = DCP_COLORSPACE_NATIVE; + surface->stride = 10176; + surface->width = 2408; + surface->height = 1506; + surface->buf_size = 10176 * 1506; + surface->planes[0].width = 2408; + surface->planes[0].height = 1506; + surface->planes[0].stride = 10176; + surface->planes[0].size = 10176 * 1506; + request->surf_iova[0] = iova; + request->surf_null[0] = false; + request->surf_null[1] = true; + request->surf_null[2] = true; + request->surf_null[3] = true; + + iomfb_serialize_present_h17p(wire, request); + + KUNIT_EXPECT_MEMEQ(test, wire->swap, &request->swap, sizeof(wire->swap)); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x593), + DCP_FORMAT_BGRA); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x59d), 10176U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5a9), 2408U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5ad), 1506U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5b1), + 10176U * 1506U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5e1), 2408U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5e5), 1506U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5f1), 10176U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5f5), + 10176U * 1506U); + KUNIT_EXPECT_EQ(test, get_unaligned_le64((u8 *)wire + 0xe38), iova); + KUNIT_EXPECT_EQ(test, wire->tail[sizeof(wire->tail) - 2], 0U); + KUNIT_EXPECT_EQ(test, wire->tail[sizeof(wire->tail) - 1], 0U); +} + +static void h17p_opaque_x_parameter_test(struct kunit *test) +{ + struct dcp_apply_property_h17p property = dcp_opaque_x_property_h17p(); + + KUNIT_EXPECT_EQ(test, property.property, 0x49U); + KUNIT_EXPECT_EQ(test, property.value, 0U); +} + +static struct kunit_case h17p_present_cases[] = { + KUNIT_CASE(h17p_surface_wire_test), + KUNIT_CASE(h17p_opaque_x_parameter_test), + {} +}; + +static struct kunit_suite h17p_present_suite = { + .name = "apple-dcp-h17p-present", + .test_cases = h17p_present_cases, +}; + +kunit_test_suite(h17p_present_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index a61101520e547..dacec823cb46d 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1463,6 +1463,11 @@ void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) u32 handle; dev_info(dcp->dev, "dcp_poweron() starting\n"); +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + iomfb_opaque_x_reset_h17p(dcp); +#endif + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index 3bac8687e83de..b128a7df971da 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -430,6 +430,23 @@ static const u32 dcp_primary_formats[] = { #endif }; +/* H17P 10-bit XRGB alpha semantics have not been qualified. */ +static const u32 dcp_primary_formats_h17p[] = { + DRM_FORMAT_ARGB2101010, + DRM_FORMAT_XRGB8888, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_NV12, + DRM_FORMAT_NV16, + DRM_FORMAT_NV24, + DRM_FORMAT_P010, + DRM_FORMAT_P210, +#if defined(DRM_FORMAT_P410) + DRM_FORMAT_P410, +#endif +}; + static const u32 dcp_overlay_formats[] = { DRM_FORMAT_ARGB2101010, DRM_FORMAT_ARGB8888, @@ -484,6 +501,7 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, unsigned long possible_crtcs, u32 iomfb_surf, bool supports_l10r, + bool supports_xrgb2101010, enum drm_plane_type type) { struct apple_plane *plane; @@ -492,9 +510,12 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, switch (type) { case DRM_PLANE_TYPE_PRIMARY: - if (supports_l10r) { + if (supports_l10r && supports_xrgb2101010) { fmts = dcp_primary_formats; num_fmts = ARRAY_SIZE(dcp_primary_formats); + } else if (supports_l10r) { + fmts = dcp_primary_formats_h17p; + num_fmts = ARRAY_SIZE(dcp_primary_formats_h17p); } else { fmts = dcp_primary_formats_12_x; num_fmts = ARRAY_SIZE(dcp_primary_formats_12_x); diff --git a/drivers/gpu/drm/apple/plane.h b/drivers/gpu/drm/apple/plane.h index 298edcd7c561e..3cc4db891004e 100644 --- a/drivers/gpu/drm/apple/plane.h +++ b/drivers/gpu/drm/apple/plane.h @@ -36,6 +36,7 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, unsigned long possible_crtcs, u32 iomfb_surf, bool supports_l10r, + bool supports_xrgb2101010, enum drm_plane_type type); #endif /* __APPLE_PLANE_H__ */ diff --git a/tools/testing/apple/check-backlight.py b/tools/testing/apple/check-backlight.py index e1df3dc3458de..2a1f6412c9639 100644 --- a/tools/testing/apple/check-backlight.py +++ b/tools/testing/apple/check-backlight.py @@ -79,7 +79,7 @@ def function(file, name): int bl_register_mutex,bl_register_wq; struct { u32 maximum;u32 nits;struct backlight_device *bl_dev;u32 dac; } brightness; struct { int lock;struct dcp_backlight_state state;void (*kick)(struct apple_dcp *); } backlight; - struct { int lock;struct iomfb_transaction *active; } iomfb; + struct { int lock;struct iomfb_transaction *active;int opaque_x_state; } iomfb; }; static int schedule_count,register_count,retire_count,dac_count; static struct backlight_properties registered; @@ -100,6 +100,10 @@ def function(file, name): static void put_unaligned_le64(u64 v,void *p) { for(int i=0;i<8;i++)((u8 *)p)[i]=v>>(8*i); } static void put_unaligned_le32(u32 v,void *p) { for(int i=0;i<4;i++)((u8 *)p)[i]=v>>(8*i); } static void put_unaligned_le16(uint16_t v,void *p) { for(int i=0;i<2;i++)((u8 *)p)[i]=v>>(8*i); } +enum { IOMFB_OPAQUE_X_WAITING, IOMFB_OPAQUE_X_NEEDED }; +static int atomic_cmpxchg(int *value,int expected,int desired) { + int old=*value;if(old==expected)*value=desired;return old; +} int iomfb_configure_backlight_h17p(struct apple_dcp *,u32,bool,u32,bool,u32); ''' functions = [ @@ -194,6 +198,7 @@ def function(file, name): assert(dcp_bl_request(&dcp.backlight.state,140,false,false)==0); assert(!iomfb_present_complete_h17p(&dcp)); assert(dcp.backlight.state.actual==400 && dcp.backlight.state.dirty && !retire_count); + assert(dcp.iomfb.opaque_x_state==IOMFB_OPAQUE_X_WAITING); assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); assert(level(&wire)==140); iomfb_present_failed_h17p(&dcp); @@ -237,6 +242,12 @@ def function(file, name): struct apple_dcp zero=fixture(); assert(dcp_backlight_takeover(&zero,0)==0); assert(zero.backlight.state.target==0 && !zero.backlight.state.dirty); + transaction = (struct iomfb_transaction){.brightness_only=false}; + dcp.iomfb.active = &transaction; + dcp.iomfb.opaque_x_state = IOMFB_OPAQUE_X_WAITING; + assert(iomfb_present_complete_h17p(&dcp)); + assert(dcp.iomfb.opaque_x_state == IOMFB_OPAQUE_X_NEEDED); + puts("Actual takeover/registration/reservation/accepted completion/rejection/20 DPMS cycles passed"); return 0; } From 9756022c265e0845c1db58a801a7c7de7bd0975d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:55:15 +1100 Subject: [PATCH 0495/1579] drm: apple: bound H17P firmware buffer mappings H17P allocates firmware buffers through the DCP device DMA domain, then requests the same scatterlist and IOVA in the piodma child domain. The secondary mapping is read/write. Allocation failures receive an all-zero reply; rejected maps echo the descriptor with a zero device address; normal lifetime unmaps the secondary translation before releasing backing pages. Decode and encode the packed H17P records explicitly in little endian. Admit the unmap and release envelopes, reject stale and duplicate descriptors, and roll back short map results. Keep ownership when unmap is short, and make release remove any live secondary translation before freeing the scatterlist and coherent allocation. This bounds descriptor exhaustion, allocation and scatterlist failures, partial maps and unmaps, duplicate operations, invalid IDs, and release without an earlier unmap while retaining the existing older-firmware paths. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 99 ++++++++++++++++---- drivers/gpu/drm/apple/iomfb_template.c | 124 +++++++++++++++++-------- 2 files changed, 164 insertions(+), 59 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index fca112eb4ecf2..6fc2567c17310 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -40,6 +40,7 @@ static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { [128] = { 0x1008, 0x4, true }, [129] = { 0x40, 0x4, true }, [201] = { 0xc, 0x10, true }, + [202] = { 0x1a, 0x0, true }, [206] = { 0x0, 0x4, true }, [207] = { 0x0, 0x4, true }, [300] = { 0x10, 0x0, true }, @@ -51,6 +52,7 @@ static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { [414] = { 0x50, 0x4, true }, [415] = { 0x4c, 0x4, true }, [451] = { 0x14, 0x1c, true }, + [454] = { 0x4, 0x1, true }, [552] = { 0x1044, 0x4, true }, [561] = { 0x1044, 0x4, true }, [563] = { 0x4c, 0x4, true }, @@ -439,7 +441,6 @@ static bool trampoline_allocate_buffer_h17p(struct apple_dcp *dcp, int tag, const struct dcp_allocate_buffer_req *wire = in; struct dcp_allocate_buffer_req request = { 0 }; struct dcp_allocate_buffer_resp reply; - u32 options; if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) return trampoline_allocate_buffer(dcp, tag, out, in); @@ -451,29 +452,91 @@ static bool trampoline_allocate_buffer_h17p(struct apple_dcp *dcp, int tag, static_assert(offsetof(struct dcp_allocate_buffer_req, paddr_null) == 16); trace_iomfb_callback(dcp, tag, __func__); - options = get_unaligned_le32(&wire->unk2); - /* Only these option combinations have observed successful replies. */ - if (get_unaligned_le32(&wire->unk0) != 0x703 || - (options != 4 && options != SZ_16K) || - memchr_inv(&wire->paddr_null, 0, 4)) - goto fail; - request.size = get_unaligned_le64(&wire->size); reply = dcpep_cb_allocate_buffer(dcp, &request); - if (!reply.dva_size) - goto fail; + if (!reply.dva_size) { + /* Exhaustion and allocation errors use the all-zero reply. */ + memset(out, 0, sizeof(reply)); + return true; + } put_unaligned_le64(reply.paddr, out); put_unaligned_le64(reply.dva, (u8 *)out + 8); put_unaligned_le64(reply.dva_size, (u8 *)out + 16); put_unaligned_le32(reply.mem_desc_id, (u8 *)out + 24); return true; +} -fail: - /* Allocation failures have no measured H17P reply encoding. */ - dev_err(dcp->dev, "unqualified or failed D451 buffer allocation\n"); - WRITE_ONCE(dcp->crashed, true); - return false; +static bool trampoline_map_piodma_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_map_buf_req *wire = in; + struct dcp_map_buf_req request = { 0 }; + struct dcp_map_buf_resp_h17p reply = { 0 }; + u64 buffer; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_map_piodma(dcp, tag, out, in); + + static_assert(sizeof(*wire) == 0xc); + static_assert(sizeof(reply) == 0x10); + static_assert(offsetof(struct dcp_map_buf_req, unk) == 8); + static_assert(offsetof(struct dcp_map_buf_req, dva_null) == 11); + + trace_iomfb_callback(dcp, tag, __func__); + buffer = get_unaligned_le64(&wire->buffer); + request.buffer = buffer; + if (!dcpep_map_piodma(dcp, &request, &reply)) { + /* A rejected mapping echoes the descriptor with a zero DVA. */ + put_unaligned_le32(buffer, out); + put_unaligned_le64(0, (u8 *)out + 4); + put_unaligned_le32(0, (u8 *)out + 12); + return true; + } + + put_unaligned_le32(reply.buffer, out); + put_unaligned_le64(reply.dva, (u8 *)out + 4); + put_unaligned_le32(reply.unk, (u8 *)out + 12); + return true; +} + +static bool trampoline_unmap_piodma_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_unmap_buf_resp *wire = in; + struct dcp_unmap_buf_resp request = { 0 }; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_unmap_piodma(dcp, tag, out, in); + + static_assert(sizeof(*wire) == 0x1a); + static_assert(offsetof(struct dcp_unmap_buf_resp, vaddr) == 8); + static_assert(offsetof(struct dcp_unmap_buf_resp, dva) == 16); + static_assert(offsetof(struct dcp_unmap_buf_resp, buf_null) == 25); + + trace_iomfb_callback(dcp, tag, __func__); + request.buffer = get_unaligned_le64(&wire->buffer); + request.vaddr = get_unaligned_le64(&wire->vaddr); + request.dva = get_unaligned_le64(&wire->dva); + request.unk = wire->unk; + request.buf_null = wire->buf_null; + dcpep_unmap_piodma(dcp, &request); + return true; +} + +static bool +trampoline_release_mem_desc_h17p(struct apple_dcp *dcp, int tag, void *out, + void *in) +{ + u32 id; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_release_mem_desc(dcp, tag, out, in); + + trace_iomfb_callback(dcp, tag, __func__); + id = get_unaligned_le32(in); + *(u8 *)out = dcpep_cb_release_mem_desc(dcp, &id); + return true; } /* H17P callback numbering is not a uniform shift of the v13.5 table. */ @@ -508,8 +571,8 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [128] = trampoline_prop_chunk, /* setDCPAVPropChunk */ [129] = trampoline_prop_end, /* setDCPAVPropEnd */ [130] = trampoline_allocate_bandwidth, /* allocate_bandwidth */ - [201] = trampoline_map_piodma, /* map_buf */ - [202] = trampoline_unmap_piodma, /* unmap_buf */ + [201] = trampoline_map_piodma_h17p, /* map_buf */ + [202] = trampoline_unmap_piodma_h17p, /* unmap_buf */ [206] = iomfbep_cb_match_pmu_service_2, /* match_pmu_service */ [207] = iomfbep_cb_match_backlight_service, /* match_backlight_service */ [208] = trampoline_nop, /* update_backlight_factor_prop */ @@ -526,7 +589,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [415] = trampoline_true, /* setProperty */ [451] = trampoline_allocate_buffer_h17p, /* allocate_buffer */ [452] = trampoline_map_physical, /* prepare */ - [454] = trampoline_release_mem_desc, /* release_descriptor */ + [454] = trampoline_release_mem_desc_h17p, /* release_descriptor */ [552] = trampoline_true, [561] = trampoline_true, [563] = trampoline_true, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index dacec823cb46d..a75ffb818edd4 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -100,7 +100,7 @@ DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, * channel. Sent on the ordinary command channel it is never answered, which * strands the power transition and every later call behind it. */ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER #define DCP_POWER_OOB true #else #define DCP_POWER_OOB false @@ -502,11 +502,13 @@ static void iomfbep_cb_set_fx_prop(struct apple_dcp *dcp, struct iomfb_set_fx_pr * PIODMA is separate from the main DCP and uses own IOVA space on a dedicated * stream of the display DART, rather than the expected DCP DART. */ -static struct DCP_FW_NAME(dcp_map_buf_resp) dcpep_cb_map_piodma(struct apple_dcp *dcp, - struct dcp_map_buf_req *req) +static bool dcpep_map_piodma(struct apple_dcp *dcp, + struct dcp_map_buf_req *req, + struct DCP_FW_NAME(dcp_map_buf_resp) * resp) { struct dcp_mem_descriptor *memdesc; struct sg_table *map; + size_t size; ssize_t ret; if (req->buffer >= ARRAY_SIZE(dcp->memdesc)) @@ -514,81 +516,112 @@ static struct DCP_FW_NAME(dcp_map_buf_resp) dcpep_cb_map_piodma(struct apple_dcp memdesc = &dcp->memdesc[req->buffer]; map = &memdesc->map; + size = ALIGN(memdesc->size, SZ_16K); - if (!map->sgl) + if (!test_bit(req->buffer, dcp->memdesc_map) || !map->sgl || + memdesc->piodma_mapped || !dcp->iommu_dom) goto reject; /* use the piodma iommu domain to map against the right IOMMU */ ret = iommu_map_sgtable(dcp->iommu_dom, memdesc->dva, map, IOMMU_READ | IOMMU_WRITE); - /* HACK: expect size to be 16K aligned since the iommu API only maps - * full pages - */ - if (ret < 0 || ret != ALIGN(memdesc->size, SZ_16K)) { - dev_err(dcp->dev, "iommu_map_sgtable() returned %zd instead of expected buffer size of %zu\n", ret, memdesc->size); + if (ret != (ssize_t)size) { + dev_err(dcp->dev, + "iommu_map_sgtable() returned %zd instead of %zu\n", + ret, size); + if (ret > 0) + iommu_unmap(dcp->iommu_dom, memdesc->dva, ret); goto reject; } #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* remembered so release_mem_desc() can tear a stale mapping down */ memdesc->piodma_mapped = true; /* the H17P reply echoes the buffer index */ - return (struct DCP_FW_NAME(dcp_map_buf_resp)){ + *resp = (struct DCP_FW_NAME(dcp_map_buf_resp)) { .buffer = req->buffer, .dva = memdesc->dva, }; #else - return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .dva = memdesc->dva }; + *resp = (struct DCP_FW_NAME(dcp_map_buf_resp)) { .dva = memdesc->dva }; #endif + return true; reject: dev_err(dcp->dev, "denying map of invalid buffer %llx for piodma\n", req->buffer); + return false; +} + +static struct DCP_FW_NAME(dcp_map_buf_resp) dcpep_cb_map_piodma(struct apple_dcp *dcp, + struct dcp_map_buf_req *req) +{ + struct DCP_FW_NAME(dcp_map_buf_resp) resp = { 0 }; + + if (dcpep_map_piodma(dcp, req, &resp)) + return resp; + #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - return (struct DCP_FW_NAME(dcp_map_buf_resp)){ - .buffer = req->buffer, - .dva = 0, - }; + resp.buffer = req->buffer; #else - return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .ret = EINVAL }; + resp.ret = EINVAL; #endif + return resp; } -static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, - struct dcp_unmap_buf_resp *resp) +static bool dcpep_unmap_piodma(struct apple_dcp *dcp, + struct dcp_unmap_buf_resp *request) { struct dcp_mem_descriptor *memdesc; + size_t size, unmapped; - if (resp->buffer >= ARRAY_SIZE(dcp->memdesc)) { + if (request->buffer >= ARRAY_SIZE(dcp->memdesc)) { dev_warn(dcp->dev, "unmap request for out of range buffer %llu\n", - resp->buffer); - return; + request->buffer); + return false; } - memdesc = &dcp->memdesc[resp->buffer]; + memdesc = &dcp->memdesc[request->buffer]; - if (!memdesc->buf) { - dev_warn(dcp->dev, - "unmap for non-mapped buffer %llu iova:0x%08llx\n", - resp->buffer, resp->dva); - return; - } + if (!test_bit(request->buffer, dcp->memdesc_map) || !memdesc->buf || + !dcp->iommu_dom) + goto not_mapped; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (!memdesc->piodma_mapped) + goto not_mapped; +#endif - if (memdesc->dva != resp->dva) { - dev_warn(dcp->dev, "unmap buffer %llu address mismatch " - "memdesc.dva:%llx dva:%llx\n", resp->buffer, - memdesc->dva, resp->dva); - return; + if (memdesc->dva != request->dva) { + dev_warn(dcp->dev, + "unmap buffer %llu address mismatch memdesc.dva:%llx dva:%llx\n", + request->buffer, + memdesc->dva, request->dva); + return false; } /* use the piodma iommu domain to unmap from the right IOMMU */ - /* HACK: expect size to be 16K aligned since the iommu API only maps - * full pages - */ - iommu_unmap(dcp->iommu_dom, memdesc->dva, ALIGN(memdesc->size, SZ_16K)); + size = ALIGN(memdesc->size, SZ_16K); + unmapped = iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + if (unmapped != size) { + dev_err(dcp->dev, "iommu_unmap() returned %zu instead of %zu\n", + unmapped, size); + return false; + } memdesc->piodma_mapped = false; + return true; + +not_mapped: + dev_warn(dcp->dev, + "unmap for non-mapped buffer %llu iova:0x%08llx\n", + request->buffer, request->dva); + return false; +} + +static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, + struct dcp_unmap_buf_resp *request) +{ + dcpep_unmap_piodma(dcp, request); } /* @@ -656,7 +689,7 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) { struct dcp_mem_descriptor *memdesc; - size_t size; + size_t size, unmapped; u32 id = *mem_desc_id; if (id >= DCP_MAX_MAPPINGS) { @@ -665,7 +698,7 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) return 0; } - if (!test_and_clear_bit(id, dcp->memdesc_map)) { + if (!test_bit(id, dcp->memdesc_map)) { dev_warn(dcp->dev, "unmap request for unused mem_desc_id %u\n", id); return 0; @@ -686,9 +719,18 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) dev_warn(dcp->dev, "releasing buffer %u while still mapped for piodma; unmapping first\n", id); - iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + if (!dcp->iommu_dom) + return 0; + unmapped = iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + if (unmapped != size) { + dev_err(dcp->dev, + "failed to unmap buffer %u before release: %zu of %zu\n", + id, unmapped, size); + return 0; + } memdesc->piodma_mapped = false; } + clear_bit(id, dcp->memdesc_map); if (memdesc->buf) { sg_free_table(&memdesc->map); From 22778725c2865885de5762e9ec5c496cd1e559cc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 08:46:32 +1100 Subject: [PATCH 0496/1579] drm: apple: retain J700 provider replies with older loaders The working J700 loader predates the display power-table DT properties. Requiring only those new properties prevents the firmware's default framebuffer setup before a compatible loader can be installed. Provide the independently observed J700 table bytes when the internal T8140 endpoint lacks a property and the machine is specifically J700. Explicit per-device DT data always wins; malformed present properties still fail validation. Other boards keep the loader-data requirement. The byte arrays match both the retained disp0 ADT and firmware replies; no physical brightness range or power-model interpretation is added. New loaders can forward all five properties verbatim using the existing DT interface, removing the need for this compatibility path. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 51 ++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 6fc2567c17310..2de0fbbd4620a 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -313,19 +313,49 @@ struct dcp_h17p_provider_property { const char *name; const char *dt_name; u32 length; + const u8 *legacy_data; }; static_assert(sizeof(struct dcp_h17p_provider_request) == 0x4c); static_assert(sizeof(struct dcp_h17p_provider_reply) == 0xc04); +/* Compatibility data matches the independently decoded J700 disp0 ADT. */ +static const u8 j700_provider_data_0[] = { + 0x0d, 0x00, 0x00, 0x00 +}; + +static const u8 j700_provider_data_1[] = { + 0x01, 0x00, 0x00, 0x00 +}; + +static const u8 j700_provider_data_2[] = { + 0x02, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, + 0x64, 0x00, 0x00, 0x00, 0x96, 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, + 0xfa, 0x00, 0x00, 0x00, 0x2c, 0x01, 0x00, 0x00, 0x5e, 0x01, 0x00, 0x00, + 0x90, 0x01, 0x00, 0x00, 0xc2, 0x01, 0x00, 0x00, 0xf4, 0x01, 0x00, 0x00, + 0x0d, 0x02, 0x00, 0x00 +}; + +static const u8 j700_provider_data_3[] = { + 0x64, 0x00, 0x00, 0x00 +}; + +static const u8 j700_provider_data_4[] = { + 0x34, 0x03, 0x00, 0x00, 0x84, 0x03, 0x00, 0x00, 0x24, 0x05, 0x00, 0x00, + 0x52, 0x07, 0x00, 0x00, 0x11, 0x09, 0x00, 0x00, 0xde, 0x0a, 0x00, 0x00, + 0xd2, 0x0c, 0x00, 0x00, 0xe2, 0x0e, 0x00, 0x00, 0xfb, 0x10, 0x00, 0x00, + 0x64, 0x13, 0x00, 0x00, 0xd5, 0x15, 0x00, 0x00, 0x68, 0x18, 0x00, 0x00, + 0xa2, 0x19, 0x00, 0x00 +}; + /* Only these provider transfers have been measured against the disp0 ADT. */ static const struct dcp_h17p_provider_property provider_properties[] = { - { "power-lut-data-x", "apple,power-lut-data-x", 4 }, - { "power-lut-data-y", "apple,power-lut-data-y", 4 }, - { "power-lut-data-xindex", "apple,power-lut-data-xindex", 52 }, - { "power-lut-data-yindex", "apple,power-lut-data-yindex", 4 }, - { "power-lut-data-lut", "apple,power-lut-data-lut", 52 }, - { "power-lut-vbatt-cur-nominal", "apple,power-lut-vbatt-cur-nominal", 0 }, + { "power-lut-data-x", "apple,power-lut-data-x", 4, j700_provider_data_0 }, + { "power-lut-data-y", "apple,power-lut-data-y", 4, j700_provider_data_1 }, + { "power-lut-data-xindex", "apple,power-lut-data-xindex", 52, j700_provider_data_2 }, + { "power-lut-data-yindex", "apple,power-lut-data-yindex", 4, j700_provider_data_3 }, + { "power-lut-data-lut", "apple,power-lut-data-lut", 52, j700_provider_data_4 }, + { "power-lut-vbatt-cur-nominal", "apple,power-lut-vbatt-cur-nominal", 0, NULL }, }; static const struct dcp_h17p_provider_property * @@ -371,6 +401,15 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, /* Only this absent property has a measured empty reply. */ if (of_property_present(node, property->dt_name)) goto fail; + } else if (!of_property_present(node, property->dt_name) && + of_machine_is_compatible("apple,j700") && + of_device_is_compatible(node, "apple,t8140-dcp")) { + /* Qualified older loaders lack the newly described table properties. */ + memset(reply, 0, sizeof(*reply)); + memcpy(reply->data, property->legacy_data, property->length); + reply->length = cpu_to_le32(property->length); + dev_warn_once(dcp->dev, "using J700 provider data for an older loader\n"); + return true; } else if (length != (int)property->length) { dev_err(dcp->dev, "invalid provider property %s: %d bytes\n", property->dt_name, length); From ebcff4ba64dd90983cc8150b0f46fda82b5fb9f0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 09:40:34 +1100 Subject: [PATCH 0497/1579] iommu: apple-dart: retain inherited translations before reservation setup A locked display DART already translates the boot firmware regions. When the core builds temporary read-write mappings for those reservations, their cache attributes and the firmware's read-only code region differ from the inherited leaves. Exact mirror validation then rejects the first default-domain attach with EBUSY. Snapshot the supported three-level firmware roots during domain setup, before the core checks translated reservations. Resolve existing firmware translations when the software table has no entry, requiring unchanged root ownership and consistent physical addresses across the domain's streams. The core can then keep the existing mappings without creating replacement leaves or widening firmware permissions. Continue to validate host mappings and publish only into vacant root slots. The lookup borrows mapped firmware tables; it does not install those tables in the software allocator or free them with its page tables. Unqualified layouts and conflicting stream translations remain refused. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 74 +++++++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index d15fb45d28233..0dc4fb3fef6a6 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -1208,19 +1208,66 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, return 0; } +static phys_addr_t +apple_dart_inherited_phys(struct apple_dart_domain *domain, dma_addr_t address) +{ + struct apple_dart_atomic_stream_map *map; + phys_addr_t result = 0; + int i, sid; + + for_each_stream_map(i, domain, map) { + struct apple_dart *dart = map->dart; + unsigned long flags; + size_t entries = dart->pgsize / sizeof(u64); + size_t page = (address / dart->pgsize) % entries; + size_t slot = address / dart->pgsize / entries; + + if (!dart->locked || !dart->fw_mirror || slot >= entries) + return 0; + spin_lock_irqsave(&dart->lock, flags); + for (sid = 0; sid < dart->num_streams; sid++) { + struct apple_dart_fw_root *fw; + phys_addr_t phys; + int prot; + + if (!(atomic_long_read(&map->sidmap[BIT_WORD(sid)]) & BIT_MASK(sid))) + continue; + fw = dart->locked_fw[sid][0]; + if (!apple_dart_fw_slot_owned(fw, dart->locked_ttbr[sid][0], slot) || + !apple_dart_fw_leaf_decode(READ_ONCE(fw->leaf[slot][page]), + address, &phys, &prot) || + (result && result != phys)) { + spin_unlock_irqrestore(&dart->lock, flags); + return 0; + } + result = phys; + } + spin_unlock_irqrestore(&dart->lock, flags); + } + return result; +} + static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; + struct io_pgtable_cfg *cfg; + phys_addr_t phys; + dma_addr_t address; if (!ops) return 0; - /* Reserved firmware mappings are also represented in this software table. */ - return ops->iova_to_phys(ops, - (iova + dart_domain->dma_offset) & - dart_domain->mask); + address = (iova + dart_domain->dma_offset) & dart_domain->mask; + phys = ops->iova_to_phys(ops, address); + if (phys) + return phys; + cfg = &io_pgtable_ops_to_pgtable(ops)->cfg; + if (cfg->apple_dart_cfg.n_levels != 3) + return 0; + /* Existing firmware translations need no replacement software mapping. */ + return apple_dart_inherited_phys(dart_domain, address); } static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, @@ -1426,6 +1473,25 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, dart_domain->domain.geometry.force_aperture = true; dart_domain->dma_offset = dart->dma_offset; + if (pgtbl_cfg.apple_dart_cfg.n_levels == 3) { + struct apple_dart_stream_map *stream; + int index; + + /* Make inherited translations visible before core reservation setup. */ + for_each_stream_map(i, cfg, stream) { + if (!stream->dart->locked || !stream->dart->fw_mirror) + continue; + for (index = 0; index < pgtbl_cfg.apple_dart_cfg.n_ttbrs; index++) { + ret = apple_dart_hw_map_locked_ttbr(stream, index); + if (ret) { + free_io_pgtable_ops(dart_domain->pgtbl_ops); + dart_domain->pgtbl_ops = NULL; + goto done; + } + } + } + } + dart_domain->finalized = true; done: From 35a8b58d13a8f910904223e2d417c835c8e85258 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 11:15:00 +1100 Subject: [PATCH 0498/1579] MAINTAINERS: include Apple display tests and documentation Cover the backlight documentation and generic display replay tests under the existing Apple display driver entry so changes reach its reviewers together with the driver. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 47ceb787d696d..1516f3e6ec4f4 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1954,7 +1954,9 @@ T: git git://anongit.freedesktop.org/drm/drm-misc F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml +F: Documentation/gpu/apple-dcp-backlight.rst F: drivers/gpu/drm/apple/ +F: tools/testing/apple/ APPLE PCIE CONTROLLER DRIVER M: Marc Zyngier From 383f3b5ed30a70d0e87deb41c47577122eeeb207 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 10:37:24 +1100 Subject: [PATCH 0499/1579] drm: apple: accept the legacy J700 primary surface selection The qualified J700 loader passes one zero in apple,iomfb-surfaces to select firmware surface zero. The per-slot flag parser instead leaves the bitmap empty, which registers a CRTC without a primary plane and crashes fbdev initialization. Recognize this exact legacy encoding only on J700 internal T8140 DCP, and reject empty surface masks before CRTC creation on every profile. Also check the CRTC allocation before use. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 4 ++++ drivers/gpu/drm/apple/dcp.c | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index db4abd811cb64..aa815a8b3bf4f 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -411,6 +411,8 @@ static int apple_probe_per_dcp(struct device *dev, supports_xrgb2101010 = dcp_data->fw_compat != DCP_FIRMWARE_H17P || dcp_data->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; + if (bitmap_empty(iomfb_surfaces, DCP_MAX_PLANES)) + return dev_err_probe(dev, -EINVAL, "No usable display surfaces\n"); for_each_set_bit(surf, iomfb_surfaces, DCP_MAX_PLANES) { plane_type = (zpos == 0) ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY; @@ -435,6 +437,8 @@ static int apple_probe_per_dcp(struct device *dev, * knows what to do with overlays. */ crtc = kzalloc(sizeof(*crtc), GFP_KERNEL); + if (!crtc) + return -ENOMEM; ret = drm_crtc_init_with_planes(drm, &crtc->base, planes[0], NULL, &apple_crtc_funcs, NULL); if (ret) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 82d910c1366ee..4f11ac29bf01d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2331,6 +2331,13 @@ static int dcp_platform_probe(struct platform_device *pdev) set_bit(surf, dcp->iomfb_surfaces); surf++; } + /* The qualified J700 loader names surface zero rather than enabling it. */ + if (of_machine_is_compatible("apple,j700") && + of_device_is_compatible(dev->of_node, "apple,t8140-dcp") && + num_surfs == 1 && + !of_property_read_u32(dev->of_node, "apple,iomfb-surfaces", &surf_en) && + surf_en == 0) + set_bit(0, dcp->iomfb_surfaces); if (dcp->phy) { int ret; From 44706aa64c1e999e54ddd74868608a6d84a96aa1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 10:37:25 +1100 Subject: [PATCH 0500/1579] dt-bindings: display: apple: document legacy J700 surface selection The qualified J700 bootloader uses one zero to select the primary surface, rather than the per-slot enable flags used by other loaders. Document that existing compatibility encoding. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/display/apple,dcp.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index aefb570de5983..bba5d64f50451 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -121,7 +121,9 @@ properties: maxItems: 4 items: enum: [0, 1] - description: Per-slot enable flags, in slot order. + description: | + Per-slot enable flags, in slot order. The legacy J700 bootloader + supplies a single zero to select surface zero instead. apple,power-lut-data-x: $ref: /schemas/types.yaml#/definitions/uint8-array From 43039912593a1f7130687a32792627e9d820dc87 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 11:54:21 +1100 Subject: [PATCH 0501/1579] drm: apple: retain adopted H17P DMA resources through reboot Internal H17P keeps its display pipe running when the panel is blanked. Its shutdown path stops the outbound queue without a firmware-acknowledged scanout stop. Deleting the component nevertheless releases the binding's devres group, including RTKit shared buffers, and destroys a host-created piodma device. The running firmware can still access those resources. During system shutdown, stop host submissions and backlight work while keeping the adopted component and its DMA resources allocated until reset. Keep the existing component teardown for the older firmware profiles. This does not qualify component removal or a firmware client-close method. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4f11ac29bf01d..de436ed971101 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2442,6 +2442,24 @@ static void dcp_platform_remove(struct platform_device *pdev) static void dcp_platform_shutdown(struct platform_device *pdev) { + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp && dcp->hw.adopt_live_session && + dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + /* + * This firmware keeps scanning across soft DPMS and has no + * qualified stop sequence. Component unbind releases the RTKit + * devres group and the piodma domain while firmware may still + * access them. Stop host submissions but keep the component and + * its DMA resources alive until the system resets. + */ + iomfb_queue_stop(dcp); + cancel_work_sync(&dcp->bl_register_wq); + cancel_work_sync(&dcp->bl_update_wq); + return; + } + component_del(&pdev->dev, &dcp_comp_ops); } From 41e7892e40eb21ddb08b2391a7940ed6499daf8d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 13:57:58 +1100 Subject: [PATCH 0502/1579] soc: apple: pmp: use a 42-bit DMA mask on T8140 The T8140 PMP DART DMA window starts at 1 TiB. The default 32-bit DMA mask leaves no usable IOVA, so the firmware's first buffer request fails with ENOMEM and probe times out. Set the mask before allocating any firmware buffers, using per-SoC match data. Retain the default mask on existing SoCs. Port Ryan Murray's Linux-observed T8140 fix onto the existing Asahi PMP driver by Sasha Finkelstein, including Janne Grunau's fixup. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/pmp.rs | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index 9a46852a97bc6..3356fc889b523 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -16,7 +16,9 @@ use kernel::{ devres::Devres, dma::{ Coherent, - CoherentBox, // + CoherentBox, + Device as _, + DmaMask, // }, io::{ mem::IoMem, @@ -399,20 +401,46 @@ impl rtkit::Operations for PmpData { #[allow(dead_code)] struct PmpDriver(Arc); +/// Per-SoC configuration. +struct PmpHwConfig { + /// DMA mask to set before the firmware allocates anything, or `None` to + /// keep the default. + dma_mask: Option, +} + +const HW_CFG_DEFAULT: PmpHwConfig = PmpHwConfig { dma_mask: None }; + +/// T8140: the DART translates 42 bits and its DMA window starts at 1 TiB. +const HW_CFG_T8140: PmpHwConfig = PmpHwConfig { + dma_mask: Some(DmaMask::new::<42>()), +}; + kernel::of_device_table!( OF_TABLE, MODULE_OF_TABLE, - (), - [(of::DeviceId::new(c"apple,t6000-pmp-v2"), ())] + ::IdInfo, + [ + (of::DeviceId::new(c"apple,t8140-pmp-v2"), &HW_CFG_T8140), + (of::DeviceId::new(c"apple,t6000-pmp-v2"), &HW_CFG_DEFAULT), + ] ); impl platform::Driver for PmpDriver { - type IdInfo = (); + type IdInfo = &'static PmpHwConfig; const OF_ID_TABLE: Option> = Some(&OF_TABLE); - fn probe(pdev: &platform::Device, _info: Option<&()>) -> impl PinInit { + fn probe( + pdev: &platform::Device, + info: Option<&Self::IdInfo>, + ) -> impl PinInit { let dev: ARef = pdev.as_ref().into(); + let cfg = info.ok_or(ENODEV)?; + if let Some(mask) = cfg.dma_mask { + // SAFETY: No DMA allocations or mappings exist for this device + // yet. + unsafe { pdev.dma_set_mask_and_coherent(mask)? }; + } let data = PmpData::new(pdev)?; let node = dev.fwnode().ok_or(EIO)?; let dvid = node From f898b116beaaab7add264a0f074b21743875d3ae Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 13:52:31 +1100 Subject: [PATCH 0503/1579] arm64: dts: apple: Describe additional T8140 peripheral power states Describe the remaining peripheral power-state registers from the J700 ADT, using its explicit PMGR group, offset and parent identifiers. Expose the supported local-state dependencies through the existing PMGR power-domain binding. Firmware-owned main fabric records marked no-ps are not instantiated as generic power domains. Keep critical states on and preserve the firmware ownership of active AOP peripherals. Describe the unused AOP2 and HE NRCS states disabled: the existing board measurements show that HE NRCS cannot reach ACTIVE. Leave the firmware-owned PMC state unmanaged. The register descriptions are independently derived from the ADT and public bindings. No private device-tree source was copied. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 868 ++++++++++++++++++++++ 1 file changed, 868 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 3fc3ea3ad8fb4..8994178318c77 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -480,3 +480,871 @@ apple,always-on; /* used by the SMC firmware */ }; }; + +&pmgr { + ps_msg: power-controller@108 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msg"; + status = "okay"; + }; + + ps_dwi: power-controller@118 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dwi"; + status = "okay"; + }; + + ps_pms_fpwm0: power-controller@140 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm0"; + status = "okay"; + }; + + ps_pms_fpwm1: power-controller@148 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm1"; + status = "okay"; + }; + + ps_pms_fpwm2: power-controller@150 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm2"; + status = "okay"; + }; + + ps_pms_fpwm3: power-controller@158 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm3"; + status = "okay"; + }; + + ps_pms_fpwm4: power-controller@160 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x160 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm4"; + status = "okay"; + }; + + ps_pms_c1ppt: power-controller@168 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_c1ppt"; + apple,always-on; + status = "okay"; + }; + + ps_soc_rc: power-controller@170 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x170 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_rc"; + status = "okay"; + }; + + ps_soc_dpe: power-controller@178 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x178 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_dpe"; + apple,always-on; + status = "okay"; + }; + + ps_pmgr_soc_ocla: power-controller@180 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x180 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmgr_soc_ocla"; + status = "okay"; + }; + + ps_ispsens0: power-controller@188 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x188 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens0"; + status = "okay"; + }; + + ps_ispsens1: power-controller@190 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens1"; + status = "okay"; + }; + + ps_ap_tmm: power-controller@1a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ap_tmm"; + status = "okay"; + }; + + ps_jpg: power-controller@230 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x230 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "jpg"; + status = "okay"; + }; + + ps_msr: power-controller@238 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x238 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr"; + status = "okay"; + }; + + ps_prores: power-controller@250 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x250 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "prores"; + status = "okay"; + }; + + ps_dispext0_sys: power-controller@288 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_sys"; + status = "okay"; + }; + + ps_ane_sys: power-controller@290 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x290 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_sys"; + status = "okay"; + }; + + ps_msr_ase_core: power-controller@298 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x298 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr_ase_core"; + power-domains = <&ps_msr>; + status = "okay"; + }; + + ps_sio_cpu: power-controller@2a8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sio_cpu"; + status = "okay"; + }; + + ps_fpwm0: power-controller@2b0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm0"; + status = "okay"; + }; + + ps_fpwm1: power-controller@2b8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm1"; + status = "okay"; + }; + + ps_fpwm2: power-controller@2c0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm2"; + status = "okay"; + }; + + ps_i2c1: power-controller@2d0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c1"; + status = "okay"; + }; + + ps_i2c2: power-controller@2d8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c2"; + status = "okay"; + }; + + ps_i2c3: power-controller@2e0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c3"; + status = "okay"; + }; + + ps_i2c4: power-controller@2e8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c4"; + status = "okay"; + }; + + ps_spi_p: power-controller@2f0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + status = "okay"; + }; + + ps_uart_p: power-controller@2f8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_p"; + status = "okay"; + }; + + ps_dpa_p: power-controller@300 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa_p"; + apple,always-on; + status = "okay"; + }; + + ps_aes: power-controller@308 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x308 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aes"; + status = "okay"; + }; + + ps_dispext0_fe: power-controller@320 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x320 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_fe"; + power-domains = <&ps_dispext0_sys>; + status = "okay"; + }; + + ps_dispext0_cpu: power-controller@328 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x328 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_cpu"; + power-domains = <&ps_dispext0_fe>; + status = "okay"; + }; + + ps_spi0: power-controller@330 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi0"; + power-domains = <&ps_spi_p>; + status = "okay"; + }; + + ps_spi1: power-controller@338 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x338 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi1"; + power-domains = <&ps_spi_p>; + status = "okay"; + }; + + ps_spi3: power-controller@348 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x348 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi3"; + power-domains = <&ps_spi_p>; + status = "okay"; + }; + + ps_qspi: power-controller@350 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x350 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "qspi"; + power-domains = <&ps_spi_p>; + status = "okay"; + }; + + ps_uart_n: power-controller@358 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x358 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_n"; + power-domains = <&ps_uart_p>; + status = "okay"; + }; + + ps_uart0: power-controller@360 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x360 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart0"; + power-domains = <&ps_uart_p>; + status = "okay"; + }; + + ps_uart1: power-controller@368 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x368 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart1"; + power-domains = <&ps_uart_p>; + status = "okay"; + }; + + ps_uart2: power-controller@370 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x370 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart2"; + power-domains = <&ps_uart_p>; + status = "okay"; + }; + + ps_uart3: power-controller@378 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x378 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart3"; + power-domains = <&ps_uart_p>; + status = "okay"; + }; + + ps_uart4: power-controller@380 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x380 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart4"; + power-domains = <&ps_uart_p>; + status = "okay"; + }; + + ps_uart5: power-controller@388 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x388 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart5"; + power-domains = <&ps_uart_p>; + status = "okay"; + }; + + ps_dpa0: power-controller@390 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa0"; + power-domains = <&ps_dpa_p>; + status = "okay"; + }; + + ps_dpa1: power-controller@398 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x398 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa1"; + power-domains = <&ps_dpa_p>; + status = "okay"; + }; + +}; + +&pmgr_mini { + ps_debug_gated: power-controller@0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_gated"; + apple,always-on; + status = "okay"; + }; + + ps_nub_spmi1: power-controller@60 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi1"; + apple,always-on; + status = "okay"; + }; + + ps_nub_aon_nogate: power-controller@78 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x78 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_aon_nogate"; + apple,always-on; + status = "okay"; + }; + + ps_nub_ocla: power-controller@80 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x80 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_ocla"; + apple,always-on; + status = "okay"; + }; + + ps_nub_gpio: power-controller@88 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x88 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_gpio"; + apple,always-on; + status = "okay"; + }; + + ps_nub_fabric: power-controller@a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_fabric"; + apple,always-on; + status = "okay"; + }; + + ps_nub_sram: power-controller@a8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_sram"; + apple,always-on; + status = "okay"; + }; + + ps_debug_switch: power-controller@b0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_switch"; + apple,always-on; + status = "okay"; + }; + + ps_aop_gpio: power-controller@4008 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_gpio"; + power-domains = <&ps_aop_filter>; + apple,always-on; + status = "okay"; + }; + + ps_aop_shim: power-controller@4028 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_shim"; + power-domains = <&ps_aop_base>; + apple,always-on; + status = "okay"; + }; + + ps_aop_piodma: power-controller@4030 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_piodma"; + power-domains = <&ps_aop_base>; + apple,always-on; + status = "okay"; + }; + + ps_aop2_piodma: power-controller@4038 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop2_piodma"; + power-domains = <&ps_aop_base>; + /* Firmware-owned or unable to reach ACTIVE on this board. */ + status = "disabled"; + }; + + ps_aop_i2cm0: power-controller@4040 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4040 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_i2cm0"; + power-domains = <&ps_aop_fr>; + apple,always-on; + status = "okay"; + }; + + ps_aop_i2cm1: power-controller@4048 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4048 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_i2cm1"; + power-domains = <&ps_aop_fr>; + apple,always-on; + status = "okay"; + }; + + ps_aop_i2cm2: power-controller@4050 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4050 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_i2cm2"; + power-domains = <&ps_aop_fr>; + apple,always-on; + status = "okay"; + }; + + ps_aop_i2cm3: power-controller@4058 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4058 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_i2cm3"; + power-domains = <&ps_aop_fr>; + /* Firmware-owned or unable to reach ACTIVE on this board. */ + status = "disabled"; + }; + + ps_aop_scm: power-controller@4060 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4060 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_scm"; + power-domains = <&ps_aop_base>, <&ps_aop_fr>; + apple,always-on; + status = "okay"; + }; + + ps_aop_spi0: power-controller@4068 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4068 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_spi0"; + power-domains = <&ps_aop_fr>; + apple,always-on; + status = "okay"; + }; + + ps_aop_spi1: power-controller@4070 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4070 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_spi1"; + power-domains = <&ps_aop_fr>; + /* Firmware-owned or unable to reach ACTIVE on this board. */ + status = "disabled"; + }; + + ps_aop_spmi0: power-controller@4078 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4078 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_spmi0"; + power-domains = <&ps_aop_filter>, <&ps_aop_fr>; + apple,always-on; + status = "okay"; + }; + + ps_aop_spmi1: power-controller@4080 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4080 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_spmi1"; + power-domains = <&ps_aop_filter>, <&ps_aop_fr>; + apple,always-on; + status = "okay"; + }; + + ps_aop_he_nrcs0: power-controller@4088 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4088 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_he_nrcs0"; + power-domains = <&ps_aop_fr>; + /* Firmware-owned or unable to reach ACTIVE on this board. */ + status = "disabled"; + }; + + ps_aop_he_nrcs1: power-controller@4090 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4090 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_he_nrcs1"; + power-domains = <&ps_aop_fr>; + /* Firmware-owned or unable to reach ACTIVE on this board. */ + status = "disabled"; + }; + + ps_aop2_cpu: power-controller@40a0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop2_cpu"; + power-domains = <&ps_aop_base>, <&ps_aop_fr>; + /* Firmware-owned or unable to reach ACTIVE on this board. */ + status = "disabled"; + }; + + ps_aop_uart0: power-controller@40b0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_uart0"; + power-domains = <&ps_aop_shim>; + apple,always-on; + status = "okay"; + }; + + ps_aop_uart1: power-controller@40b8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_uart1"; + power-domains = <&ps_aop_shim>; + apple,always-on; + status = "okay"; + }; + + ps_aop_uart2: power-controller@40c0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_uart2"; + power-domains = <&ps_aop_shim>; + apple,always-on; + status = "okay"; + }; + + ps_aop_sram: power-controller@40c8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aop_sram"; + power-domains = <&ps_aop_scm>, <&ps_aop_cpu>; + apple,always-on; + status = "okay"; + }; + + ps_audio_sense_pdmc: power-controller@40e0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_sense_pdmc"; + power-domains = <&ps_audio_fr>; + status = "okay"; + }; + + ps_audio_base_s_a: power-controller@40e8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_base_s_a"; + power-domains = <&ps_audio_p>; + status = "okay"; + }; + + ps_audio_mca2_m: power-controller@4110 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca2_m"; + power-domains = <&ps_audio_p>; + status = "okay"; + }; + + ps_audio_mca3_m: power-controller@4118 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca3_m"; + power-domains = <&ps_audio_p>; + status = "okay"; + }; + + ps_audio_mca4_m: power-controller@4120 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca4_m"; + power-domains = <&ps_audio_p>; + status = "okay"; + }; + + ps_audio_mca5_m: power-controller@4128 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca5_m"; + power-domains = <&ps_audio_p>; + status = "okay"; + }; + + ps_audio_leap_rx1: power-controller@4150 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_rx1"; + power-domains = <&ps_audio_leap_c>; + status = "okay"; + }; + + ps_audio_leap_rx2: power-controller@4158 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_rx2"; + power-domains = <&ps_audio_leap_c>; + status = "okay"; + }; + + ps_audio_leap_rx3: power-controller@4160 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4160 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_rx3"; + power-domains = <&ps_audio_leap_c>; + status = "okay"; + }; + + ps_audio_leap_rx4: power-controller@4168 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_rx4"; + power-domains = <&ps_audio_leap_c>; + status = "okay"; + }; + + ps_smc_fabric: power-controller@8000 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "smc_fabric"; + status = "okay"; + }; + + ps_smc_i2cm0: power-controller@8028 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "smc_i2cm0"; + power-domains = <&ps_smc_fabric>; + apple,always-on; + status = "okay"; + }; + +}; From d88816597f1fd190da7411ada67fbf40697971be Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 13:58:05 +1100 Subject: [PATCH 0504/1579] dt-bindings: rtc: describe the J700 Baku PMU alarm Describe the RTC function as a child of the existing Baku SPMI NVMEM provider. It shares the whole-second offset cell with the SMC RTC and has a dedicated alarm interrupt. Add the child to the PMIC schema. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/nvmem/apple,spmi-nvmem.yaml | 3 + .../bindings/rtc/apple,j700-pmu-rtc.yaml | 63 +++++++++++++++++++ MAINTAINERS | 1 + 3 files changed, 67 insertions(+) create mode 100644 Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml diff --git a/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml b/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml index 7a4114c7fbd19..09cdd776f8d8b 100644 --- a/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml +++ b/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml @@ -27,6 +27,9 @@ properties: reg: maxItems: 1 + rtc: + $ref: /schemas/rtc/apple,j700-pmu-rtc.yaml# + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml b/Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml new file mode 100644 index 0000000000000..b1e289c06fe33 --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rtc/apple,j700-pmu-rtc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple J700 Baku PMU real-time clock + +maintainers: + - Ryan Murray + +description: | + Baku exposes a 65536 Hz counter and an alarm comparator over SPMI. + The clock's whole-second offset is shared with the SMC RTC through an + eight-byte little-endian NVMEM cell. This function is a child of the + PMIC NVMEM provider and uses that provider's SPMI register map. + +allOf: + - $ref: rtc.yaml# + +properties: + compatible: + const: apple,j700-pmu-rtc + + interrupts: + maxItems: 1 + + nvmem-cells: + maxItems: 1 + description: The shared eight-byte whole-second RTC offset. + + nvmem-cell-names: + items: + - const: rtc_offset + + wakeup-source: true + +required: + - compatible + - interrupts + - nvmem-cells + - nvmem-cell-names + +unevaluatedProperties: false + +examples: + - | + #include + #include + + pmic@e { + compatible = "apple,baku-pmic", "apple,spmi-nvmem"; + reg = <0xe SPMI_USID>; + + rtc { + compatible = "apple,j700-pmu-rtc"; + interrupt-parent = <&spmi>; + interrupts = <2 IRQ_TYPE_EDGE_RISING>; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + wakeup-source; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index 90654e399c0d3..b240bab9104d9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2580,6 +2580,7 @@ F: Documentation/devicetree/bindings/power/apple* F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml F: Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml F: Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml +F: Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml F: Documentation/devicetree/bindings/spmi/apple,spmi.yaml From 344854db2e7946bbe7402860652806ac62d53497 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 13:59:04 +1100 Subject: [PATCH 0505/1579] dt-bindings: spi: describe the T8140 QSPIMC controller Describe the SPI-memory controller's register aperture, clock, interrupt and power domain. QSPIMC uses a packed command FIFO and differs from SPIMC. Originally written by DjDeveloperr; DjDeveloperr's Signed-off-by is outstanding. Port his publicly available controller binding. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/spi/apple,qspi.yaml | 62 +++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 63 insertions(+) create mode 100644 Documentation/devicetree/bindings/spi/apple,qspi.yaml diff --git a/Documentation/devicetree/bindings/spi/apple,qspi.yaml b/Documentation/devicetree/bindings/spi/apple,qspi.yaml new file mode 100644 index 0000000000000..5f526fe3a7970 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/apple,qspi.yaml @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/spi/apple,qspi.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple ARM SoC QSPIMC controller + +maintainers: + - Asahi Linux + +description: | + Apple QSPIMC is a SPI-memory-only controller with a packed command FIFO. + It is distinct from the ordinary Apple SPIMC controller despite serving a + similar set of SPI NOR devices. + +allOf: + - $ref: spi-controller.yaml# + +properties: + compatible: + const: apple,t8140-qspi + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + interrupts: + maxItems: 1 + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + - clocks + - interrupts + +unevaluatedProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + spi@385110000 { + compatible = "apple,t8140-qspi"; + reg = <0x3 0x85110000 0x0 0x4800>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clk_200m>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index 90654e399c0d3..63360aa82645f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2581,6 +2581,7 @@ F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml F: Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml F: Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml +F: Documentation/devicetree/bindings/spi/apple,qspi.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml F: Documentation/devicetree/bindings/spmi/apple,spmi.yaml F: Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml From 9057400a21c4633d53544085802b7a32c01d4eb8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 13:59:06 +1100 Subject: [PATCH 0506/1579] spi: apple: add T8140 QSPIMC controller Add PIO spi-mem support for the QSPIMC controller used by T8140. The controller uses packed command descriptors and is not register-compatible with the ordinary Apple SPIMC controller. Use the SPI NOR and MTD stack for flash discovery and access. Originally written by DjDeveloperr; DjDeveloperr's Signed-off-by is outstanding. Port his publicly available GPL controller and binding. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/spi/Kconfig | 9 + drivers/spi/Makefile | 1 + drivers/spi/spi-apple-qspi.c | 618 +++++++++++++++++++++++++++++++++++ 4 files changed, 629 insertions(+) create mode 100644 drivers/spi/spi-apple-qspi.c diff --git a/MAINTAINERS b/MAINTAINERS index 63360aa82645f..7f3cc384883d1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2617,6 +2617,7 @@ F: drivers/pwm/pwm-apple.c F: drivers/reset/reset-apple-cio.c F: drivers/rtc/rtc-macsmc.c F: drivers/soc/apple/* +F: drivers/spi/spi-apple-qspi.c F: drivers/spi/spi-apple.c F: drivers/spmi/spmi-apple-controller.c F: drivers/thunderbolt/apple.c diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index b563f49e21977..0171742dc6625 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -129,6 +129,15 @@ config SPI_APPLE to a single device like spi-nor (nvram), input device controller or fingerprint sensor. +config SPI_APPLE_QSPI + tristate "Apple SoC QSPIMC controller" + depends on ARCH_APPLE || COMPILE_TEST + depends on SPI_MEM + help + This enables support for Apple's command-FIFO QSPIMC controller. + Unlike the ordinary Apple SPIMC block, QSPIMC accepts packed SPI + memory descriptors and is used for the boot NOR on T8140 systems. + config SPI_AR934X tristate "Qualcomm Atheros AR934X/QCA95XX SPI controller driver" depends on ATH79 || COMPILE_TEST diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile index 9d36190a98848..b422044ca08a2 100644 --- a/drivers/spi/Makefile +++ b/drivers/spi/Makefile @@ -23,6 +23,7 @@ obj-$(CONFIG_SPI_AMLOGIC_SPIFC_A1) += spi-amlogic-spifc-a1.o obj-$(CONFIG_SPI_AMLOGIC_SPIFC_A4) += spi-amlogic-spifc-a4.o obj-$(CONFIG_SPI_AMLOGIC_SPISG) += spi-amlogic-spisg.o obj-$(CONFIG_SPI_APPLE) += spi-apple.o +obj-$(CONFIG_SPI_APPLE_QSPI) += spi-apple-qspi.o obj-$(CONFIG_SPI_AR934X) += spi-ar934x.o obj-$(CONFIG_SPI_ARMADA_3700) += spi-armada-3700.o obj-$(CONFIG_SPI_ASPEED_SMC) += spi-aspeed-smc.o diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c new file mode 100644 index 0000000000000..66a4d11dc0a2b --- /dev/null +++ b/drivers/spi/spi-apple-qspi.c @@ -0,0 +1,618 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Apple QSPIMC controller + * + * QSPIMC is not register-compatible with the ordinary Apple SPIMC block. + * It consumes packed transfer descriptors and payload words through a + * command FIFO. This driver intentionally implements the PIO spi-mem path + * only; that is sufficient for SPI NOR discovery and access. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_QSPI_C_OFFSET 0x800 +#define APPLE_QSPI_MIN_SIZE (APPLE_QSPI_C_OFFSET + 0xa0) + +/* P (queue) block. */ +#define APPLE_QSPI_P_TXFIFO 0x18 +#define APPLE_QSPI_P_RXFIFO 0x1c +#define APPLE_QSPI_P_QSTATUS 0x20 +#define APPLE_QSPI_P_QSTATUS_TX_USED GENMASK(15, 8) +#define APPLE_QSPI_P_QSTATUS_RX_AVAIL GENMASK(31, 24) +#define APPLE_QSPI_P_IRQ_ENABLE 0x2c +#define APPLE_QSPI_P_IRQ_STATUS 0x30 +#define APPLE_QSPI_P_VERSION 0x3c + +/* C (command) block. */ +#define APPLE_QSPI_C_CONTROL 0x00 +#define APPLE_QSPI_C_CONTROL_START BIT(0) +#define APPLE_QSPI_C_CONTROL_RESET BIT(8) +#define APPLE_QSPI_C_MODE 0x04 +#define APPLE_QSPI_C_MODE_CPHA BIT(8) +#define APPLE_QSPI_C_MODE_CPOL BIT(9) +#define APPLE_QSPI_C_PAYLOAD_COUNT 0x14 +#define APPLE_QSPI_C_RX_COUNT 0x18 +#define APPLE_QSPI_C_QTHRESHOLD 0x24 +#define APPLE_QSPI_C_IRQ_ENABLE 0x28 +#define APPLE_QSPI_C_IRQ_STATUS 0x2c +#define APPLE_QSPI_C_IRQ_COMPLETE BIT(0) +#define APPLE_QSPI_C_QDEPTH 0x38 +#define APPLE_QSPI_C_QDEPTH_TX GENMASK(7, 0) +#define APPLE_QSPI_C_QDEPTH_RX GENMASK(15, 8) +#define APPLE_QSPI_C_CONFIG_48 0x48 +#define APPLE_QSPI_C_CONFIG_58 0x58 +#define APPLE_QSPI_C_CONFIG_5C 0x5c +#define APPLE_QSPI_C_CLKDIV 0x64 +#define APPLE_QSPI_C_CONFIG_68 0x68 +#define APPLE_QSPI_C_CONFIG_6C 0x6c +#define APPLE_QSPI_C_CONFIG_7C 0x7c +#define APPLE_QSPI_C_CONFIG_80 0x80 +#define APPLE_QSPI_C_CONFIG_84 0x84 +#define APPLE_QSPI_C_CONFIG_90 0x90 +#define APPLE_QSPI_C_CONFIG_94 0x94 +#define APPLE_QSPI_C_CONFIG_98 0x98 +#define APPLE_QSPI_C_CONFIG_9C 0x9c + +/* Packed transfer descriptor, word 0. */ +#define APPLE_QSPI_CMD_MODE GENMASK(3, 0) +#define APPLE_QSPI_CMD_FRAME_A BIT(4) +#define APPLE_QSPI_CMD_FRAME_B BIT(5) +#define APPLE_QSPI_CMD_TAG GENMASK(15, 12) +#define APPLE_QSPI_CMD_SHARED_COUNT GENMASK(19, 16) +#define APPLE_QSPI_CMD_MULTI_COUNT GENMASK(23, 20) +#define APPLE_QSPI_CMD_TURNAROUND GENMASK(31, 24) +#define APPLE_QSPI_CMD_RX BIT(3) + +#define APPLE_QSPI_IRQ_MASK 0x3 +#define APPLE_QSPI_MAX_DATA 4096 +#define APPLE_QSPI_MAX_HZ 40000000 +#define APPLE_QSPI_TIMEOUT_MS 1000 + +struct apple_qspi { + void __iomem *base; + void __iomem *p; + void __iomem *c; + struct clk *clk; + struct completion done; + /* Protects the active transfer pointers shared with the IRQ handler. */ + spinlock_t lock; + int irq; + + const u8 *tx; + size_t tx_left; + u8 *rx; + size_t rx_left; + u8 tag; + u8 tx_depth; + u8 rx_depth; + bool active; +}; + +struct apple_qspi_op_shape { + u8 width; + u8 shared_count; + u8 multi_count; + u8 turnaround; +}; + +static inline u32 apple_qspi_p_read(struct apple_qspi *qspi, u32 reg) +{ + return readl_relaxed(qspi->p + reg); +} + +static inline void apple_qspi_p_write(struct apple_qspi *qspi, u32 reg, u32 val) +{ + writel_relaxed(val, qspi->p + reg); +} + +static inline u32 apple_qspi_c_read(struct apple_qspi *qspi, u32 reg) +{ + return readl_relaxed(qspi->c + reg); +} + +static inline void apple_qspi_c_write(struct apple_qspi *qspi, u32 reg, u32 val) +{ + writel_relaxed(val, qspi->c + reg); +} + +static int apple_qspi_width_mode(u8 width) +{ + switch (width) { + case 1: + return 1; + case 2: + return 2; + case 4: + return 3; + default: + return -EINVAL; + } +} + +static int apple_qspi_shape_op(const struct spi_mem_op *op, + struct apple_qspi_op_shape *shape) +{ + u32 dummy_cycles = 0; + u8 width; + + if (op->cmd.nbytes != 1 || op->cmd.buswidth != 1 || op->cmd.dtr) + return -EOPNOTSUPP; + if ((op->addr.nbytes && op->addr.dtr) || + (op->dummy.nbytes && op->dummy.dtr) || + (op->data.nbytes && (op->data.dtr || op->data.ecc || + op->data.swap16))) + return -EOPNOTSUPP; + + if (op->data.nbytes) + width = op->data.buswidth; + else if (op->dummy.nbytes) + width = op->dummy.buswidth; + else if (op->addr.nbytes) + width = op->addr.buswidth; + else + width = 1; + + if (apple_qspi_width_mode(width) < 0) + return -EOPNOTSUPP; + if (op->addr.nbytes && op->addr.buswidth != 1 && + op->addr.buswidth != width) + return -EOPNOTSUPP; + if (op->dummy.nbytes && op->dummy.buswidth != width) + return -EOPNOTSUPP; + if (op->data.nbytes && op->data.buswidth != width) + return -EOPNOTSUPP; + + shape->width = width; + shape->shared_count = op->cmd.nbytes; + shape->multi_count = 0; + if (op->addr.nbytes) { + if (op->addr.buswidth == 1) + shape->shared_count += op->addr.nbytes; + else + shape->multi_count = op->addr.nbytes; + } + + if (shape->shared_count > 15 || shape->multi_count > 15) + return -EOPNOTSUPP; + + if (op->dummy.nbytes) { + dummy_cycles = op->dummy.nbytes * 8 / op->dummy.buswidth; + if (dummy_cycles > 255) + return -EOPNOTSUPP; + } + shape->turnaround = dummy_cycles; + + return 0; +} + +static bool apple_qspi_supports_op(struct spi_mem *mem, + const struct spi_mem_op *op) +{ + struct apple_qspi_op_shape shape; + + if (spi_get_chipselect(mem->spi, 0) || spi_get_csgpiod(mem->spi, 0)) + return false; + if (op->addr.nbytes > 4 || op->data.nbytes > APPLE_QSPI_MAX_DATA) + return false; + if (apple_qspi_shape_op(op, &shape)) + return false; + + return spi_mem_default_supports_op(mem, op); +} + +static int apple_qspi_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op) +{ + op->data.nbytes = min_t(unsigned int, op->data.nbytes, APPLE_QSPI_MAX_DATA); + return 0; +} + +static void apple_qspi_fill_tx(struct apple_qspi *qspi) +{ + u32 status, used, available; + + status = apple_qspi_p_read(qspi, APPLE_QSPI_P_QSTATUS); + used = FIELD_GET(APPLE_QSPI_P_QSTATUS_TX_USED, status); + available = used < qspi->tx_depth ? qspi->tx_depth - used : 0; + + while (available-- && qspi->tx_left) { + u32 word = 0; + unsigned int shift; + + for (shift = 0; shift < 32 && qspi->tx_left; shift += 8) { + word |= (u32)*qspi->tx++ << shift; + qspi->tx_left--; + } + apple_qspi_p_write(qspi, APPLE_QSPI_P_TXFIFO, word); + } +} + +static void apple_qspi_drain_rx(struct apple_qspi *qspi) +{ + u32 status, available; + + status = apple_qspi_p_read(qspi, APPLE_QSPI_P_QSTATUS); + available = FIELD_GET(APPLE_QSPI_P_QSTATUS_RX_AVAIL, status); + + while (available-- && qspi->rx_left) { + u32 word = apple_qspi_p_read(qspi, APPLE_QSPI_P_RXFIFO); + unsigned int shift; + + for (shift = 0; shift < 32 && qspi->rx_left; shift += 8) { + *qspi->rx++ = word >> shift; + qspi->rx_left--; + } + } +} + +static irqreturn_t apple_qspi_irq(int irq, void *data) +{ + struct apple_qspi *qspi = data; + unsigned long flags; + u32 p_status, c_status; + bool complete_op = false; + + p_status = apple_qspi_p_read(qspi, APPLE_QSPI_P_IRQ_STATUS); + c_status = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); + if (!p_status && !c_status) + return IRQ_NONE; + + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_STATUS, p_status); + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_STATUS, c_status); + + spin_lock_irqsave(&qspi->lock, flags); + if (qspi->active) { + apple_qspi_drain_rx(qspi); + apple_qspi_fill_tx(qspi); + if (c_status & APPLE_QSPI_C_IRQ_COMPLETE) + complete_op = true; + } + spin_unlock_irqrestore(&qspi->lock, flags); + + if (complete_op) + complete(&qspi->done); + + return IRQ_HANDLED; +} + +static void apple_qspi_clear_irqs(struct apple_qspi *qspi) +{ + u32 status; + + status = apple_qspi_p_read(qspi, APPLE_QSPI_P_IRQ_STATUS); + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_STATUS, status); + status = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_STATUS, status); +} + +static void apple_qspi_init_config(struct apple_qspi *qspi) +{ + u32 val; + + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_7C, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_80, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_58, 11); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_5C, 10); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_94, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_98, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_9C, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_68, 11); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_6C, 10); + + val = apple_qspi_c_read(qspi, APPLE_QSPI_C_CONFIG_48); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_48, val | 0x00300000); + apple_qspi_c_write(qspi, APPLE_QSPI_C_QTHRESHOLD, + 1 | ((qspi->rx_depth >> 1) << 8)); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_90, 4); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONFIG_84, 0x00020000); +} + +static int apple_qspi_hw_reset(struct apple_qspi *qspi) +{ + u32 val; + int ret; + + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_ENABLE, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_ENABLE, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONTROL, + APPLE_QSPI_C_CONTROL_RESET); + ret = readl_poll_timeout(qspi->c + APPLE_QSPI_C_CONTROL, val, + !(val & APPLE_QSPI_C_CONTROL_RESET), 10, 10000); + if (ret) + return ret; + + apple_qspi_clear_irqs(qspi); + apple_qspi_init_config(qspi); + return 0; +} + +static int apple_qspi_set_frequency(struct apple_qspi *qspi, + struct spi_mem *mem, + const struct spi_mem_op *op) +{ + unsigned long input = clk_get_rate(qspi->clk); + u32 requested, divisor; + + requested = op->max_freq ?: mem->spi->max_speed_hz; + requested = min_t(u32, requested, APPLE_QSPI_MAX_HZ); + if (!input || !requested) + return -EINVAL; + + divisor = DIV_ROUND_UP(input, requested); + divisor = clamp_t(u32, divisor, 2, 256); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CLKDIV, divisor - 1); + return 0; +} + +static bool apple_qspi_wait_complete(struct apple_qspi *qspi) +{ + unsigned long deadline = jiffies + + msecs_to_jiffies(APPLE_QSPI_TIMEOUT_MS); + unsigned long flags; + u32 p_status, c_status; + + do { + if (try_wait_for_completion(&qspi->done)) + return true; + + /* + * TX-only commands do not reliably raise the wired interrupt on + * T8140, although the completion bit is latched in the controller. + * Poll both status banks as a fallback and service the FIFOs so a + * page program larger than the TX FIFO can continue making progress. + */ + p_status = apple_qspi_p_read(qspi, APPLE_QSPI_P_IRQ_STATUS); + c_status = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); + if (p_status) + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_STATUS, p_status); + if (c_status) + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_STATUS, c_status); + + spin_lock_irqsave(&qspi->lock, flags); + if (qspi->active) { + apple_qspi_drain_rx(qspi); + apple_qspi_fill_tx(qspi); + } + spin_unlock_irqrestore(&qspi->lock, flags); + + if (c_status & APPLE_QSPI_C_IRQ_COMPLETE) + return true; + if (time_after_eq(jiffies, deadline)) + break; + + if (wait_for_completion_timeout(&qspi->done, 1)) + return true; + } while (time_before(jiffies, deadline)); + + return false; +} + +static int apple_qspi_exec_op(struct spi_mem *mem, + const struct spi_mem_op *op) +{ + struct spi_controller *ctlr = mem->spi->controller; + struct apple_qspi *qspi = spi_controller_get_devdata(ctlr); + struct apple_qspi_op_shape shape; + unsigned long flags; + size_t header_len, tx_len; + u8 *tx_buf, *p; + u32 descriptor, mode; + u32 p_status, c_status; + int ret, retries; + bool complete; + + ret = apple_qspi_shape_op(op, &shape); + if (ret) + return ret; + + header_len = op->cmd.nbytes + op->addr.nbytes; + tx_len = header_len; + if (op->data.dir == SPI_MEM_DATA_OUT) + tx_len += op->data.nbytes; + + tx_buf = kmalloc(tx_len, GFP_KERNEL); + if (!tx_buf) + return -ENOMEM; + + p = tx_buf; + *p++ = op->cmd.opcode; + if (op->addr.nbytes) { + int i; + + for (i = op->addr.nbytes - 1; i >= 0; i--) + *p++ = op->addr.val >> (8 * i); + } + if (op->data.dir == SPI_MEM_DATA_OUT && op->data.nbytes) + memcpy(p, op->data.buf.out, op->data.nbytes); + + ret = apple_qspi_set_frequency(qspi, mem, op); + if (ret) + goto out_free; + + mode = (mem->spi->mode & SPI_CPOL ? APPLE_QSPI_C_MODE_CPOL : 0) | + (mem->spi->mode & SPI_CPHA ? APPLE_QSPI_C_MODE_CPHA : 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_MODE, mode); + + mode = apple_qspi_width_mode(shape.width); + if (op->data.dir == SPI_MEM_DATA_IN) + mode |= APPLE_QSPI_CMD_RX; + descriptor = FIELD_PREP(APPLE_QSPI_CMD_MODE, mode) | + APPLE_QSPI_CMD_FRAME_A | APPLE_QSPI_CMD_FRAME_B | + FIELD_PREP(APPLE_QSPI_CMD_TAG, qspi->tag++ & 0xf) | + FIELD_PREP(APPLE_QSPI_CMD_SHARED_COUNT, + shape.shared_count) | + FIELD_PREP(APPLE_QSPI_CMD_MULTI_COUNT, + shape.multi_count) | + FIELD_PREP(APPLE_QSPI_CMD_TURNAROUND, + shape.turnaround); + + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_ENABLE, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_ENABLE, 0); + apple_qspi_clear_irqs(qspi); + reinit_completion(&qspi->done); + + spin_lock_irqsave(&qspi->lock, flags); + qspi->tx = tx_buf; + qspi->tx_left = tx_len; + qspi->rx = op->data.dir == SPI_MEM_DATA_IN ? op->data.buf.in : NULL; + qspi->rx_left = op->data.dir == SPI_MEM_DATA_IN ? op->data.nbytes : 0; + qspi->active = true; + apple_qspi_p_write(qspi, APPLE_QSPI_P_TXFIFO, descriptor); + apple_qspi_p_write(qspi, APPLE_QSPI_P_TXFIFO, op->data.nbytes); + apple_qspi_fill_tx(qspi); + spin_unlock_irqrestore(&qspi->lock, flags); + + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_ENABLE, + APPLE_QSPI_IRQ_MASK); + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_ENABLE, + APPLE_QSPI_IRQ_MASK); + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONTROL, + APPLE_QSPI_C_CONTROL_START); + + complete = apple_qspi_wait_complete(qspi); + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_ENABLE, 0); + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_ENABLE, 0); + synchronize_irq(qspi->irq); + + spin_lock_irqsave(&qspi->lock, flags); + for (retries = 100; qspi->rx_left && retries; retries--) + apple_qspi_drain_rx(qspi); + qspi->active = false; + ret = qspi->tx_left || qspi->rx_left ? -EIO : 0; + spin_unlock_irqrestore(&qspi->lock, flags); + + apple_qspi_c_write(qspi, APPLE_QSPI_C_CONTROL, 0); + if (!complete) { + p_status = apple_qspi_p_read(qspi, APPLE_QSPI_P_QSTATUS); + c_status = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); + dev_err(&mem->spi->dev, + "transfer timed out: p=%#x c=%#x pcnt=%#x prxcnt=%#x tx=%zu rx=%zu\n", + p_status, c_status, + apple_qspi_c_read(qspi, APPLE_QSPI_C_PAYLOAD_COUNT), + apple_qspi_c_read(qspi, APPLE_QSPI_C_RX_COUNT), + qspi->tx_left, qspi->rx_left); + ret = -ETIMEDOUT; + if (apple_qspi_hw_reset(qspi)) + dev_err(&mem->spi->dev, + "controller reset failed after timeout\n"); + } else if (ret) { + dev_err(&mem->spi->dev, + "transfer completed with %zu tx and %zu rx bytes pending\n", + qspi->tx_left, qspi->rx_left); + } + +out_free: + kfree(tx_buf); + return ret; +} + +static const struct spi_controller_mem_ops apple_qspi_mem_ops = { + .adjust_op_size = apple_qspi_adjust_op_size, + .supports_op = apple_qspi_supports_op, + .exec_op = apple_qspi_exec_op, +}; + +static int apple_qspi_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct spi_controller *ctlr; + struct apple_qspi *qspi; + struct resource *res; + u32 depths, version; + int irq, ret; + + ctlr = devm_spi_alloc_host(dev, sizeof(*qspi)); + if (!ctlr) + return -ENOMEM; + + qspi = spi_controller_get_devdata(ctlr); + init_completion(&qspi->done); + spin_lock_init(&qspi->lock); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res || resource_size(res) < APPLE_QSPI_MIN_SIZE) + return dev_err_probe(dev, -EINVAL, + "QSPIMC aperture is too small\n"); + + qspi->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(qspi->base)) + return PTR_ERR(qspi->base); + qspi->p = qspi->base; + qspi->c = qspi->base + APPLE_QSPI_C_OFFSET; + + qspi->clk = devm_clk_get_enabled(dev, NULL); + if (IS_ERR(qspi->clk)) + return dev_err_probe(dev, PTR_ERR(qspi->clk), + "failed to enable controller clock\n"); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + qspi->irq = irq; + ret = devm_request_irq(dev, irq, apple_qspi_irq, 0, + dev_name(dev), qspi); + if (ret) + return dev_err_probe(dev, ret, "failed to request IRQ\n"); + + ret = apple_qspi_hw_reset(qspi); + if (ret) + return dev_err_probe(dev, ret, "controller reset timed out\n"); + + depths = apple_qspi_c_read(qspi, APPLE_QSPI_C_QDEPTH); + qspi->tx_depth = FIELD_GET(APPLE_QSPI_C_QDEPTH_TX, depths); + qspi->rx_depth = FIELD_GET(APPLE_QSPI_C_QDEPTH_RX, depths); + if (qspi->tx_depth < 3 || !qspi->rx_depth) + return dev_err_probe(dev, -ENODEV, + "invalid FIFO depths tx=%u rx=%u\n", + qspi->tx_depth, qspi->rx_depth); + apple_qspi_init_config(qspi); + + ctlr->num_chipselect = 1; + ctlr->max_native_cs = 1; + ctlr->mode_bits = SPI_CPHA | SPI_CPOL | + SPI_RX_DUAL | SPI_RX_QUAD | + SPI_TX_DUAL | SPI_TX_QUAD; + ctlr->bits_per_word_mask = SPI_BPW_MASK(8); + ctlr->mem_ops = &apple_qspi_mem_ops; + + platform_set_drvdata(pdev, ctlr); + ret = devm_spi_register_controller(dev, ctlr); + if (ret) + return dev_err_probe(dev, ret, + "failed to register SPI controller\n"); + + version = apple_qspi_p_read(qspi, APPLE_QSPI_P_VERSION); + dev_info(dev, "QSPIMC version %#x, %lu Hz clock, txq=%u rxq=%u\n", + version, clk_get_rate(qspi->clk), qspi->tx_depth, + qspi->rx_depth); + return 0; +} + +static const struct of_device_id apple_qspi_of_match[] = { + { .compatible = "apple,t8140-qspi" }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_qspi_of_match); + +static struct platform_driver apple_qspi_driver = { + .probe = apple_qspi_probe, + .driver = { + .name = "apple-qspi", + .of_match_table = apple_qspi_of_match, + }, +}; +module_platform_driver(apple_qspi_driver); + +MODULE_AUTHOR("The Asahi Linux Contributors"); +MODULE_DESCRIPTION("Apple QSPIMC controller driver"); +MODULE_LICENSE("GPL"); From 908b68bb91225d602fc5c282de90b92de02c7306 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:00:14 +1100 Subject: [PATCH 0507/1579] spi: apple-qspi: service transfer FIFOs with bounded microsecond polling T8140 can latch command completion without delivering the wired IRQ, and the RX FIFO threshold interrupt does not reliably fire. Waiting a jiffy between service attempts limits Linux reads to one FIFO per tick. The observed NVRAM read took 55 seconds at about 34 kB/s. Use the polling helper with a microsecond sleep interval to drain and refill the FIFOs. Preserve the one-second deadline and the IRQ completion path. Serialize polling FIFO service with the interrupt handler. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi.c | 66 ++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index 66a4d11dc0a2b..b831622753d55 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -354,47 +353,40 @@ static int apple_qspi_set_frequency(struct apple_qspi *qspi, return 0; } -static bool apple_qspi_wait_complete(struct apple_qspi *qspi) +static bool apple_qspi_poll_complete(struct apple_qspi *qspi) { - unsigned long deadline = jiffies + - msecs_to_jiffies(APPLE_QSPI_TIMEOUT_MS); unsigned long flags; - u32 p_status, c_status; - - do { - if (try_wait_for_completion(&qspi->done)) - return true; - - /* - * TX-only commands do not reliably raise the wired interrupt on - * T8140, although the completion bit is latched in the controller. - * Poll both status banks as a fallback and service the FIFOs so a - * page program larger than the TX FIFO can continue making progress. - */ - p_status = apple_qspi_p_read(qspi, APPLE_QSPI_P_IRQ_STATUS); - c_status = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); - if (p_status) - apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_STATUS, p_status); - if (c_status) - apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_STATUS, c_status); - - spin_lock_irqsave(&qspi->lock, flags); - if (qspi->active) { - apple_qspi_drain_rx(qspi); - apple_qspi_fill_tx(qspi); - } - spin_unlock_irqrestore(&qspi->lock, flags); + u32 queue_irqs, command_irqs; - if (c_status & APPLE_QSPI_C_IRQ_COMPLETE) - return true; - if (time_after_eq(jiffies, deadline)) - break; + spin_lock_irqsave(&qspi->lock, flags); + queue_irqs = apple_qspi_p_read(qspi, APPLE_QSPI_P_IRQ_STATUS); + command_irqs = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_STATUS, queue_irqs); + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_STATUS, command_irqs); + if (qspi->active) { + apple_qspi_drain_rx(qspi); + apple_qspi_fill_tx(qspi); + } + spin_unlock_irqrestore(&qspi->lock, flags); - if (wait_for_completion_timeout(&qspi->done, 1)) - return true; - } while (time_before(jiffies, deadline)); + return (command_irqs & APPLE_QSPI_C_IRQ_COMPLETE) || + try_wait_for_completion(&qspi->done); +} - return false; +static bool apple_qspi_wait_complete(struct apple_qspi *qspi) +{ + bool finished; + + /* + * T8140 can latch completion without delivering the wired IRQ, and + * the RX threshold IRQ does not reliably drain the FIFO. A jiffy + * between polls throttles reads to one FIFO per scheduler tick. + * Service the FIFOs at microsecond intervals while retaining the + * bounded transfer deadline and IRQ completion path. + */ + return !read_poll_timeout(apple_qspi_poll_complete, finished, finished, + 20, APPLE_QSPI_TIMEOUT_MS * USEC_PER_MSEC, + false, qspi); } static int apple_qspi_exec_op(struct spi_mem *mem, From f69f1b76fcb04e6bd7d7d066654e612be9761af0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:22:12 +1100 Subject: [PATCH 0508/1579] pmdomain: apple: pmp-report: expose T8140 fast-die effort Describe the T8140 report layout from the Apple device tree and expose its fast-die effort through the existing PMP report mapping. A stopped PMP reads zero, so check the running bit before accepting an effort. Report entry 0x1c1 contains byte 1 for the P cluster, observed under Linux load, and byte 0 for E, assigned from ADT lane order. The E byte has not yet been observed changing. Do not read entry 0x1c2 or scan the unpopulated part of the report aperture. Expose read-only debugfs status and lane effort values for diagnostics. Port the original Linux-derived implementation by Ryan Murray. Retain the Asahi report driver and use it as the sole owner of the mapping. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 3 + drivers/pmdomain/apple/pmp-report.c | 124 +++++++++++++++++++++++++++ include/linux/soc/apple/pmp-report.h | 26 ++++++ 3 files changed, 153 insertions(+) create mode 100644 include/linux/soc/apple/pmp-report.h diff --git a/MAINTAINERS b/MAINTAINERS index 90654e399c0d3..a1368b8fa0733 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1985,9 +1985,12 @@ F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ +F: drivers/pmdomain/apple/pmp-report.c F: drivers/soc/apple/aop.rs +F: drivers/soc/apple/pmp.rs F: drivers/thermal/apple-smc-cpu-thermal.c F: drivers/tty/serial/apple_dockchannel* +F: include/linux/soc/apple/pmp-report.h F: include/linux/soc/apple/smc-thermal.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 065d3296f5a7a..ae814fb33bc50 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -5,21 +5,38 @@ * Copyright The Asahi Linux Contributors */ +#include #include #include +#include #include #include #include #include #include +#include +#include +#include #define PMP_REPORT_READY 0x1 +/** + * struct apple_pmp_report_offsets - per-SoC layout of the report region + * @tgt_read: power state request, read side + * @tgt_write: power state request, write side + * @actual: power state acknowledgment + * @status: PMP status; bit 0 is set while the PMP is running + * @fast_die: 32-bit word holding the fast die-temperature effort of each + * CPU cluster, one byte per lane; 0 if the SoC has none + * @fast_die_lanes: number of lanes at @fast_die, at most 4 + */ struct apple_pmp_report_offsets { u32 tgt_read; u32 tgt_write; u32 actual; u32 status; + u32 fast_die; + u32 fast_die_lanes; }; struct apple_pmp_report { @@ -27,8 +44,89 @@ struct apple_pmp_report { const struct apple_pmp_report_offsets *offsets; void __iomem *base; spinlock_t lock; + struct list_head list; }; +/* + * Bound report regions, for apple_pmp_report_fast_die_effort(). Regions are + * added once they are set up and never removed: the driver cannot be unbound. + */ +static LIST_HEAD(apple_pmp_reports); +static DEFINE_MUTEX(apple_pmp_reports_lock); +static struct dentry *apple_pmp_report_debugfs; + +/** + * apple_pmp_report_fast_die_effort() - read the fast die-temperature effort + * of a CPU cluster + * @np: device tree node of the report region + * @lane: lane of the cluster in the fast die-temperature report + * @effort: set to the effort on success + * + * The PMP of some SoCs runs a control loop on its own die-temperature sensors + * and publishes an 8-bit effort for each CPU cluster: 0 requests no limit, + * and the effort rises towards 255 as the die exceeds the firmware's target. + * The effort is only valid while the PMP reports that it is running; a + * stopped PMP reads 0. + * + * Context: Any context. + * Return: 0 on success, -EPROBE_DEFER if the region is not bound (yet), + * -EOPNOTSUPP if the SoC's PMP publishes no such effort, -EINVAL if @lane is + * out of range, or -EAGAIN if the PMP does not report that it is running. + */ +int apple_pmp_report_fast_die_effort(const struct device_node *np, + unsigned int lane, u8 *effort) +{ + const struct apple_pmp_report_offsets *offs; + const struct apple_pmp_report *rep; + int ret = -EPROBE_DEFER; + + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node != np) + continue; + + offs = rep->offsets; + if (!offs->fast_die) { + ret = -EOPNOTSUPP; + } else if (lane >= offs->fast_die_lanes) { + ret = -EINVAL; + } else if (!(readl(rep->base + offs->status) & PMP_REPORT_READY)) { + ret = -EAGAIN; + } else { + *effort = readl(rep->base + offs->fast_die) >> + (lane * BITS_PER_BYTE); + ret = 0; + } + break; + } + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_fast_die_effort); + +/* Read only the documented status and fast-die entries, never the aperture. */ +static int apple_pmp_report_status_show(struct seq_file *s, void *unused) +{ + struct apple_pmp_report *rep = s->private; + const struct apple_pmp_report_offsets *offs = rep->offsets; + u32 status = readl(rep->base + offs->status); + unsigned int lane; + u32 effort; + + seq_printf(s, "running: %u\n", !!(status & PMP_REPORT_READY)); + if (!(status & PMP_REPORT_READY) || !offs->fast_die) + return 0; + + effort = readl(rep->base + offs->fast_die); + for (lane = 0; lane < offs->fast_die_lanes; lane++) + seq_printf(s, "lane%u: %u\n", lane, + (effort >> (lane * BITS_PER_BYTE)) & 0xff); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(apple_pmp_report_status); + static int apple_pmp_report_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -51,6 +149,14 @@ static int apple_pmp_report_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "failed to create child devices\n"); + mutex_lock(&apple_pmp_reports_lock); + list_add_tail_rcu(&rep->list, &apple_pmp_reports); + mutex_unlock(&apple_pmp_reports_lock); + + if (rep->offsets->fast_die) + debugfs_create_file(dev_name(dev), 0400, apple_pmp_report_debugfs, + rep, &apple_pmp_report_status_fops); + return 0; } @@ -75,10 +181,21 @@ static const struct apple_pmp_report_offsets apple_pmp_offsets_t8112 = { .status = 0x10, }; +/* The fast die-temperature report is the low two bytes of entry 0x1c1. */ +static const struct apple_pmp_report_offsets apple_pmp_offsets_t8140 = { + .tgt_read = 0x1880, + .tgt_write = 0x10c40, + .actual = 0x18c0, + .status = 0x10, + .fast_die = 0x1c10, + .fast_die_lanes = 2, +}; + static const struct of_device_id apple_pmp_report_of_match[] = { { .compatible = "apple,t6000-pmp-v2-report", .data = &apple_pmp_offsets_t600x }, { .compatible = "apple,t6020-pmp-v2-report", .data = &apple_pmp_offsets_t602x }, { .compatible = "apple,t8112-pmp-v2-report", .data = &apple_pmp_offsets_t8112 }, + { .compatible = "apple,t8140-pmp-v2-report", .data = &apple_pmp_offsets_t8140 }, {} }; @@ -87,6 +204,11 @@ static struct platform_driver apple_pmp_report_driver = { .driver = { .name = "apple-pmp-report", .of_match_table = apple_pmp_report_of_match, + /* + * Its child power domains and the region list keep using the + * mapping, and nothing tears them down. + */ + .suppress_bind_attrs = true, }, }; @@ -222,6 +344,7 @@ MODULE_DEVICE_TABLE(of, apple_pmp_report_entry_of_match); static int __init apple_pmp_report_init(void) { + apple_pmp_report_debugfs = debugfs_create_dir("apple-pmp-report", NULL); platform_driver_register(&apple_pmp_report_entry_driver); platform_driver_register(&apple_pmp_report_driver); return 0; @@ -229,6 +352,7 @@ static int __init apple_pmp_report_init(void) static void __exit apple_pmp_report_exit(void) { + debugfs_remove_recursive(apple_pmp_report_debugfs); platform_driver_unregister(&apple_pmp_report_entry_driver); platform_driver_unregister(&apple_pmp_report_driver); } diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h new file mode 100644 index 0000000000000..31c2e4b17a303 --- /dev/null +++ b/include/linux/soc/apple/pmp-report.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple SoC PMP report region + * Copyright (C) The Asahi Linux Contributors + */ + +#ifndef _LINUX_SOC_APPLE_PMP_REPORT_H_ +#define _LINUX_SOC_APPLE_PMP_REPORT_H_ + +#include +#include + +struct device_node; + +#if IS_ENABLED(CONFIG_APPLE_PMP_REPORT) +int apple_pmp_report_fast_die_effort(const struct device_node *np, + unsigned int lane, u8 *effort); +#else +static inline int apple_pmp_report_fast_die_effort(const struct device_node *np, + unsigned int lane, u8 *effort) +{ + return -ENODEV; +} +#endif + +#endif /* _LINUX_SOC_APPLE_PMP_REPORT_H_ */ From e43b3f4c3679c22ec96140720a7926bb5cc18304 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:16:43 +1100 Subject: [PATCH 0509/1579] Bluetooth: hci_bcm4377: drain recovery work before module exit Queue recovery on a dedicated workqueue. Unregister the PCI driver and its HCI timeout producers before draining that queue, so no callback can outlive the module text. A module reference released inside a callback alone would not guarantee that the callback has returned before unloading. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/hci_bcm4377.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index b2c99fa7d9f84..fff9c2005d6bf 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -2282,6 +2282,8 @@ static bool bcm4377_hci_wakeup(struct hci_dev *hdev) return device_may_wakeup(&bcm4377->pdev->dev); } +static struct workqueue_struct *bcm4377_reset_wq; + struct bcm4377_reset_work { struct work_struct work; struct pci_dev *pdev; @@ -2330,7 +2332,7 @@ static void bcm4377_hci_reset(struct hci_dev *hdev) bt_dev_err(hdev, "controller not responding, rebinding the driver"); rw->pdev = pci_dev_get(bcm4377->pdev); INIT_WORK(&rw->work, bcm4377_reset_work); - schedule_work(&rw->work); + queue_work(bcm4377_reset_wq, &rw->work); } static void bcm4377_hci_unregister_dev(void *data) @@ -2643,7 +2645,29 @@ static struct pci_driver bcm4377_pci_driver = { .probe = bcm4377_probe, .driver.pm = &bcm4377_ops, }; -module_pci_driver(bcm4377_pci_driver); +static int __init bcm4377_init(void) +{ + int ret; + + bcm4377_reset_wq = alloc_workqueue("bcm4377-reset", WQ_UNBOUND, 0); + if (!bcm4377_reset_wq) + return -ENOMEM; + + ret = pci_register_driver(&bcm4377_pci_driver); + if (ret) + destroy_workqueue(bcm4377_reset_wq); + + return ret; +} +module_init(bcm4377_init); + +static void __exit bcm4377_exit(void) +{ + /* Unregister HCI devices before draining the callbacks they queued. */ + pci_unregister_driver(&bcm4377_pci_driver); + destroy_workqueue(bcm4377_reset_wq); +} +module_exit(bcm4377_exit); MODULE_AUTHOR("Sven Peter "); MODULE_DESCRIPTION("Bluetooth support for Broadcom 4377/4378/4387/4388 devices"); From c8625feab8663c0c8df952d0580dedb693fa654c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:21:57 +1100 Subject: [PATCH 0510/1579] usb: typec: tipd: preserve shared and inherited debug mode Reject mode exit and relocking when they would invalidate another open file or the bootloader Debug USB route. Validate the DBMa boolean payload, and serialize command-result reads with writes on the controller lock. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/debugfs.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/drivers/usb/typec/tipd/debugfs.c b/drivers/usb/typec/tipd/debugfs.c index 33dbc9dca411d..5a8d576eff0dc 100644 --- a/drivers/usb/typec/tipd/debugfs.c +++ b/drivers/usb/typec/tipd/debugfs.c @@ -163,9 +163,21 @@ static ssize_t tipd_debugfs_command_write(struct file *file, return -EINVAL; if (!memcmp(cmd->cmd, "DVEn", 4) && len % 4) return -EINVAL; + if (!memcmp(cmd->cmd, "DBMa", 4) && buf[4] > 1) + return -EINVAL; mutex_lock(&tps->lock); + /* Preserve inherited mode and the leases held by other open files. */ + if ((!memcmp(cmd->cmd, "DBMa", 4) && !buf[4] && + (tps->dbma_at_probe || + tps->debugfs->dbma_holders > (unsigned int)f->holds_dbma)) || + (!memcmp(cmd->cmd, "LOCK", 4) && !memchr_inv(buf + 4, 0, 4) && + (tps->dbma_at_probe || tps->debugfs->dbma_holders))) { + ret = -EBUSY; + goto out; + } + ret = tps6598x_exec_cmd_tmo(tps, cmd->cmd, len, len ? &buf[4] : NULL, sizeof(out), out, cmd->timeout_ms, 0); @@ -186,6 +198,7 @@ static ssize_t tipd_debugfs_command_write(struct file *file, } } +out: mutex_unlock(&tps->lock); return ret ? ret : count; @@ -195,12 +208,17 @@ static ssize_t tipd_debugfs_command_read(struct file *file, char __user *ubuf, size_t count, loff_t *ppos) { struct tipd_debugfs_file *f = file->private_data; + ssize_t ret; + mutex_lock(&f->tps->lock); if (!f->have_result) - return -ENODATA; + ret = -ENODATA; + else + ret = simple_read_from_buffer(ubuf, count, ppos, f->result, + sizeof(f->result)); + mutex_unlock(&f->tps->lock); - return simple_read_from_buffer(ubuf, count, ppos, f->result, - sizeof(f->result)); + return ret; } static const struct file_operations tipd_debugfs_command_fops = { From 481cc7c0d515898158b01945b6edb8579de77da9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:25:52 +1100 Subject: [PATCH 0511/1579] drm/asahi: tie stats registration to a per-device snapshot owner Keep a snapshot Arc in the successfully bound driver and pass its address to a per-device C attribute wrapper. Register after GPU initialization and drain sysfs readers before dropping the owner. Remove early global pointer publication so competing or failing probes cannot replace another device snapshot. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/driver.rs | 17 +++-- drivers/gpu/drm/asahi/gpu.rs | 11 +-- drivers/gpu/drm/asahi/stats.rs | 23 +++--- drivers/gpu/drm/asahi/sysfs.c | 99 +++++++++++--------------- drivers/gpu/drm/asahi/sysfs.h | 6 +- drivers/gpu/drm/asahi/sysfs_exports.rs | 49 ++++++------- 6 files changed, 94 insertions(+), 111 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 3653ee155d2e7..5c1c7591bcfe5 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -51,6 +51,10 @@ pub(crate) struct AsahiDriver { /// Raw `struct device *` of the platform device. Stored once at probe /// time so the `Drop` impl can find the sysfs file on unregister. raw_dev: *mut bindings::device, + raw_stats: *mut core::ffi::c_void, + // Own the allocation until unregister has drained all sysfs readers. + #[expect(unused)] + stats: Arc, } unsafe impl Send for AsahiDriver {} @@ -61,7 +65,7 @@ impl Drop for AsahiDriver { // SAFETY: `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { - crate::sysfs_exports::unregister(self.raw_dev); + crate::sysfs_exports::unregister(self.raw_dev, self.raw_stats); } } } @@ -250,8 +254,13 @@ impl platform::Driver for AsahiDriver { // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); - crate::sysfs_exports::register(raw_dev, *module_parameters::stats_export.value() != 0)?; - - Ok(Self { drm, raw_dev }) + let stats = (*drm).gpu.stats_snapshot(); + let raw_stats = crate::sysfs_exports::register( + raw_dev, + Arc::as_ptr(&stats), + *module_parameters::stats_export.value() != 0, + )?; + + Ok(Self { drm, raw_dev, raw_stats, stats }) } } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index bcb3402cdf26a..39cb3c8cc99f3 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -83,7 +83,6 @@ use crate::{ queue, regs, stats, - sysfs_exports, workqueue, // }; @@ -267,6 +266,8 @@ pub(crate) struct GpuManager { /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. pub(crate) trait GpuManager: Send + Sync { + fn stats_snapshot(&self) -> Arc; + /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -732,10 +733,6 @@ impl GpuManager::ver { let event_manager_clone = event_manager.clone(); let buffer_mgr_clone = buffer_mgr.clone(); let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; - // Publish the snapshot pointer to the C sysfs shim. It is cleared - // (by the same shim) on registration removal; until then readers see - // a stable pointer and AtomicU* races that govern. - sysfs_exports::set_snapshot_ptr(Arc::as_ptr(&stats_snap)); let stats_snap_clone = stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( @@ -1252,6 +1249,10 @@ impl GpuManager::ver { #[versions(AGX)] impl GpuManager for GpuManager::ver { + fn stats_snapshot(&self) -> Arc { + self.stats.clone() + } + fn as_any(&self) -> &dyn Any { self } diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs index e3160e1a5c632..772d969d9baff 100644 --- a/drivers/gpu/drm/asahi/stats.rs +++ b/drivers/gpu/drm/asahi/stats.rs @@ -6,7 +6,7 @@ //! (see `fw::channels.rs::StatsMsg`) and exposes them to the sysfs file //! `agx_stats` (registered from `sysfs.c`). //! -//! The snapshot is process-local and process-private. The firmware-stat +//! The snapshot belongs to one GPU manager. The firmware-stat //! decoder that feeds it lives in `StatsChannel::poll` (`channel.rs`), inside //! the versioned context where `StatsMsg` is nameable; `note_job()` is called //! from the `recv_message` rtkit callback (so the firmware mailbox IRQ @@ -35,12 +35,10 @@ use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; -/// Process-wide snapshot of AGX firmware stats. +/// Per-device snapshot of AGX firmware stats. /// -/// Lives in the `GpuManager`; the C static `asahi_stats_snapshot_ptr` (see -/// `sysfs.c`) holds its address for the C `show` callback, so the field -/// layout is a cross-language contract: `#[repr(C)]` makes the declaration -/// order binding and `sysfs.c` asserts the offsets at build time. +/// The manager and bound driver retain Arcs. The driver's Arc outlives the +/// per-device C sysfs registration; repr(C) establishes the shared layout. #[derive(Default)] #[repr(C)] pub(crate) struct StatsSnapshot { @@ -62,12 +60,13 @@ pub(crate) struct StatsSnapshot { pub(crate) jobs: AtomicU64, } -/// Pointer to the live snapshot: the C static `asahi_stats_snapshot_ptr` -/// (see `sysfs.c`) is the single storage location, written through -/// `sysfs_exports::set_snapshot_ptr` and read by the C `show` callback via -/// `READ_ONCE`. The pointer is established before the DRM device registers -/// and removed on unregister; the pointed-to fields are `AtomicU*` and carry -/// the data races themselves. +// Keep the Rust half of the C sysfs layout contract checked as well. +const _: () = { + assert!(core::mem::offset_of!(StatsSnapshot, pstate) == 16); + assert!(core::mem::offset_of!(StatsSnapshot, busy_ns) == 40); + assert!(core::mem::offset_of!(StatsSnapshot, jobs) == 48); + assert!(core::mem::size_of::() == 56); +}; impl StatsSnapshot { /// Bump the completed-submission counter (called from the queue diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 923e4013804b7..961512bf7309d 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -1,22 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // -// Sysfs shim for the AGX firmware stats export. -// -// The Rust side keeps the stats state in `crate::stats::StatsSnapshot` and -// publishes its raw pointer to the static `asahi_stats_snapshot_ptr` below -// (an atomic `u64` so the writer and reader do not need locking). This file -// owns the actual `device_attribute` and the formatted output, because -// `device_create_file` and the `device_attribute` macros are not in the -// Rust bindgen bindings for this kernel tree. -// -// The contract is one read-only file: -// /sys/class/drm/cardX/device/agx_stats -// Format: `key value\n` per line, ASCII integers, owner-readable. +// Per-device sysfs shim. Rust owns the snapshot through an Arc in the bound +// driver. The attribute wrapper is removed and its readers drained before +// that Arc is dropped. Atomic fields may reflect different sample instants. #include #include #include #include +#include #include #include #include @@ -24,18 +16,7 @@ #include "sysfs.h" -/* - * Mirror of `#[repr(C)] crate::stats::StatsSnapshot` field layout. Rust sets - * `asahi_stats_snapshot_ptr` (an atomic u64) to the heap address of one of - * these, and clears it on unregister. Field reads are all `Relaxed` atomic - * u32 / u64 (the C side uses READ_ONCE). - * - * The Rust struct carries `#[repr(C)]` so this declaration order is binding; - * the BUILD_BUG_ONs in asahi_sysfs_register() turn any drift (a repr(Rust) - * struct is silently reordered by rustc, which once cross-aligned this - * readout: busy_ns bounced in 2^32 steps and jobs read 0 while incrementing) - * into a compile error. - */ +/* Mirror of repr(C) Rust atomics; READ_ONCE accesses the current values. */ struct asahi_stats_snapshot { u32 util1; u32 util2; @@ -51,35 +32,22 @@ struct asahi_stats_snapshot { u64 jobs; }; -static int asahi_stats_export_enabled; - -/* - * Set by Rust through asahi_stats_set_snapshot_ptr(); read via READ_ONCE. - * A NULL (0) pointer means the device has not yet exposed stats (or has been - * unregistered). The Rust side owns the lifetime of the pointed-to struct. - */ -unsigned long long asahi_stats_snapshot_ptr; - -void asahi_stats_set_snapshot_ptr(unsigned long long p) -{ - WRITE_ONCE(asahi_stats_snapshot_ptr, p); -} +struct asahi_stats_attribute { + struct device_attribute attr; + const struct asahi_stats_snapshot *snapshot; + bool export_enabled; +}; static ssize_t agx_stats_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct asahi_stats_snapshot __rcu *snap; + struct asahi_stats_attribute *stats = + container_of(attr, struct asahi_stats_attribute, attr); + const struct asahi_stats_snapshot *snap = stats->snapshot; ssize_t n = 0; - (void)attr; - - rcu_read_lock(); - snap = (struct asahi_stats_snapshot __rcu *) - READ_ONCE(asahi_stats_snapshot_ptr); - if (!snap || !READ_ONCE(asahi_stats_export_enabled)) { - rcu_read_unlock(); - return scnprintf(buf, PAGE_SIZE, "unsupported\n"); - } + if (!stats->export_enabled) + return sysfs_emit(buf, "unsupported\n"); n += scnprintf(buf + n, PAGE_SIZE - n, "busy_ns %llu\n", (unsigned long long)READ_ONCE(snap->busy_ns)); n += scnprintf(buf + n, PAGE_SIZE - n, "jobs %llu\n", @@ -100,22 +68,22 @@ static ssize_t agx_stats_show(struct device *dev, (u32)READ_ONCE(snap->temperature_raw)); n += scnprintf(buf + n, PAGE_SIZE - n, "temperature_scale %u\n", (u32)READ_ONCE(snap->temperature_scale)); - rcu_read_unlock(); return n; } -static DEVICE_ATTR_RO(agx_stats); /* * Called from Rust's `AsahiDriver::probe` after the DRM device is * registered. Returns 0 on success, or a negative errno. */ -int asahi_sysfs_register(struct device *dev, int export_enabled) +int asahi_sysfs_register(struct device *dev, const void *snapshot, + int export_enabled, void **handle) { + struct asahi_stats_attribute *stats; int ret; - if (!dev) + if (!dev || !snapshot || !handle) return -ENODEV; BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, pstate) != 16); @@ -123,23 +91,38 @@ int asahi_sysfs_register(struct device *dev, int export_enabled) BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, jobs) != 48); BUILD_BUG_ON(sizeof(struct asahi_stats_snapshot) != 56); - WRITE_ONCE(asahi_stats_export_enabled, export_enabled); + stats = kzalloc_obj(*stats); + if (!stats) + return -ENOMEM; + + sysfs_attr_init(&stats->attr.attr); + stats->attr.attr.name = "agx_stats"; + stats->attr.attr.mode = 0444; + stats->attr.show = agx_stats_show; + stats->snapshot = snapshot; + stats->export_enabled = export_enabled; - ret = device_create_file(dev, &dev_attr_agx_stats); - if (ret) + ret = device_create_file(dev, &stats->attr); + if (ret) { + kfree(stats); return ret; + } + *handle = stats; return 0; } EXPORT_SYMBOL_GPL(asahi_sysfs_register); -void asahi_sysfs_unregister(struct device *dev) +void asahi_sysfs_unregister(struct device *dev, void *handle) { - if (!dev) + struct asahi_stats_attribute *stats = handle; + + if (!dev || !stats) return; - device_remove_file(dev, &dev_attr_agx_stats); - WRITE_ONCE(asahi_stats_snapshot_ptr, 0); + /* Removal drains active show callbacks before either allocation dies. */ + device_remove_file(dev, &stats->attr); + kfree(stats); } EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index ed96fbd3afe50..5969a1718925f 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -5,8 +5,8 @@ struct device; -int asahi_sysfs_register(struct device *dev, int export_enabled); -void asahi_sysfs_unregister(struct device *dev); -void asahi_stats_set_snapshot_ptr(unsigned long long p); +int asahi_sysfs_register(struct device *dev, const void *snapshot, + int export_enabled, void **handle); +void asahi_sysfs_unregister(struct device *dev, void *handle); #endif /* _ASAHI_SYSFS_H */ diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index c9cdf1c9ef641..421f89dcff2e2 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -1,46 +1,37 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! FFI declarations for the C `sysfs.c` shim. -//! -//! The Rust driver keeps the firmware stats in [`crate::stats::StatsSnapshot`] -//! and uses these shims to register / unregister the sysfs file and to publish -//! the snapshot pointer for the C-side `show` callback. +//! Per-device sysfs registration. The caller retains the snapshot Arc until +//! unregister has drained active readers and freed the C attribute wrapper. -use core::ffi::{c_int, c_ulonglong}; +use core::ffi::{c_int, c_void}; +use crate::stats::StatsSnapshot; extern "C" { - /// Register the `agx_stats` sysfs file on `dev`; `export_enabled` 0 - /// makes the file print `unsupported`. Returns 0 on success. - fn asahi_sysfs_register(dev: *mut kernel::bindings::device, export_enabled: c_int) -> c_int; - - /// Unregister the `agx_stats` sysfs file and clear the snapshot pointer. - fn asahi_sysfs_unregister(dev: *mut kernel::bindings::device); - - /// Publish the snapshot pointer for the C-side `show` callback (the C - /// static `asahi_stats_snapshot_ptr` is the single storage location). - fn asahi_stats_set_snapshot_ptr(p: c_ulonglong); + fn asahi_sysfs_register( + dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, + export_enabled: c_int, + handle: *mut *mut c_void, + ) -> c_int; + fn asahi_sysfs_unregister(dev: *mut kernel::bindings::device, handle: *mut c_void); } -/// Register the sysfs file. Safe to call from `AsahiDriver::probe` after the -/// DRM device has been registered. +/// Register only after initialization succeeds, keeping the snapshot Arc alive. pub(crate) fn register( dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, export_enabled: bool, -) -> kernel::error::Result { - let ret = unsafe { asahi_sysfs_register(dev, export_enabled as c_int) }; +) -> kernel::error::Result<*mut c_void> { + let mut handle = core::ptr::null_mut(); + let ret = unsafe { asahi_sysfs_register(dev, snapshot, export_enabled as c_int, &mut handle) }; if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { - Ok(()) + Ok(handle) } } -/// Unregister the sysfs file. Safe to call from the device release path. -pub(crate) fn unregister(dev: *mut kernel::bindings::device) { - unsafe { asahi_sysfs_unregister(dev) }; -} - -/// Publish the address of the live `StatsSnapshot` to the C shim. -pub(crate) fn set_snapshot_ptr(ptr: *const crate::stats::StatsSnapshot) { - unsafe { asahi_stats_set_snapshot_ptr(ptr as c_ulonglong) }; +/// Drain readers before the caller drops its snapshot Arc. +pub(crate) fn unregister(dev: *mut kernel::bindings::device, handle: *mut c_void) { + unsafe { asahi_sysfs_unregister(dev, handle) }; } From 691ba4cea2597906a194b04615cb6833aaece7b9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:40:33 +1100 Subject: [PATCH 0512/1579] dt-bindings: apple: Describe T8140 idle and display crossbar controllers Document the T8140 CPU idle selector and the existing Apple display crossbar interface. T8140 uses only the WFI clock-gated state; CPU and cluster power-down are not supported. The crossbar compatible follows the controller type reported by the ADT and the existing Asahi driver. Keep CS42L42 supply phandles mandatory while permitting the established CS42L83 board model with platform-managed supplies. Existing public Apple board descriptions already use that model. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/cpu/apple,t8140-cpuidle.yaml | 33 +++++++++++++ .../bindings/mux/apple,display-crossbar.yaml | 47 +++++++++++++++++++ .../bindings/sound/cirrus,cs42l42.yaml | 19 ++++++-- MAINTAINERS | 2 + 4 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml create mode 100644 Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml diff --git a/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml b/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml new file mode 100644 index 0000000000000..aa27ec55309c8 --- /dev/null +++ b/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/cpu/apple,t8140-cpuidle.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 CPU idle controller + +maintainers: + - Ryan Murray + +description: + Selects the Apple CPU idle driver on T8140 systems. Only the WFI + clock-gated state is supported; CPU and cluster power-down states + are not enabled on this SoC. + +properties: + $nodename: + const: cpuidle + + compatible: + const: apple,t8140-cpuidle + +required: + - compatible + +additionalProperties: false + +examples: + - | + cpuidle { + compatible = "apple,t8140-cpuidle"; + }; diff --git a/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml b/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml new file mode 100644 index 0000000000000..eb80cf382a188 --- /dev/null +++ b/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mux/apple,display-crossbar.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple display crossbar + +maintainers: + - Sven Peter + +description: + Routes display pipelines to the DisplayPort transmitters and inputs + of an Apple Type-C PHY. Each crossbar exposes three mux controls, + selected by the consumer's mux specifier. + +allOf: + - $ref: mux-controller.yaml# + +properties: + compatible: + enum: + - apple,t8103-display-crossbar + - apple,t8112-display-crossbar + - apple,t6000-display-crossbar + - apple,t6020-display-crossbar + + reg: + maxItems: 1 + + '#mux-control-cells': + const: 1 + +required: + - compatible + - reg + - '#mux-control-cells' + +unevaluatedProperties: false + +examples: + - | + mux-controller@b04c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0xb04c000 0x4000>; + #mux-control-cells = <1>; + }; diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml index 28a5303cf7204..1d839e28c0f8a 100644 --- a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml +++ b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml @@ -14,6 +14,8 @@ description: It provides a high-dynamic range, stereo DAC for audio playback and a mono high-dynamic-range ADC for audio capture. There is an integrated headset detection block. + CS42L83 systems may have platform-managed supplies without regulator + phandles, as on Apple Silicon boards. properties: compatible: @@ -192,11 +194,18 @@ properties: required: - compatible - reg - - VP-supply - - VCP-supply - - VD_FILT-supply - - VL-supply - - VA-supply +allOf: + - if: + properties: + compatible: + const: cirrus,cs42l42 + then: + required: + - VP-supply + - VCP-supply + - VD_FILT-supply + - VL-supply + - VA-supply additionalProperties: false diff --git a/MAINTAINERS b/MAINTAINERS index 90654e399c0d3..e480d8a536878 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1969,9 +1969,11 @@ APPLE T8140 SOC AND J700 (MACBOOK NEO) SUPPORT M: Ryan Murray W: https://aurora-silicon.github.io S: Maintained +F: Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml +F: Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml F: Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml F: Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml From ff07618c82bb1831a8ef09135fc0a6c414d790bd Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Thu, 8 Oct 2026 14:43:19 +1100 Subject: [PATCH 0513/1579] rtc: add opt-in J700 Baku PMU alarms with retained s2idle interrupts Expose the Baku counter and alarm as a child of the existing SPMI PMIC provider. Share its regmap and NVMEM offset with the SMC RTC. Read the 65536 Hz counter as whole seconds and obtain the 64-bit seconds offset on each operation so that SMC clock updates are visible immediately. Keep time-setting on the existing SMC RTC and expose the PMU alarm here. The original opt-in alarm design was written by Eryk Wieliczko; Eryk's Signed-off-by is outstanding. Independently rewrite the implementation using register facts from private research and Linux observations, without copying private implementation code. Retain his authorship for the design. Retained SPMI/AIC power is required. Opt-in is explicit and alarms are limited to one hour until longer retention is qualified. Reject deeper suspend states and an alarm inherited from firmware. Acknowledge alarms in a threaded interrupt and fail closed on acknowledgment I/O errors. No off-domain wake or power-saving claim is made. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/nvmem/apple-spmi-nvmem.c | 10 +- drivers/rtc/Kconfig | 11 + drivers/rtc/Makefile | 1 + drivers/rtc/rtc-apple-pmu.c | 369 +++++++++++++++++++++++++++++++ 5 files changed, 391 insertions(+), 1 deletion(-) create mode 100644 drivers/rtc/rtc-apple-pmu.c diff --git a/MAINTAINERS b/MAINTAINERS index b240bab9104d9..1d365477614d0 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2615,6 +2615,7 @@ F: drivers/power/reset/macsmc-reboot.c F: drivers/power/supply/macsmc-power.c F: drivers/pwm/pwm-apple.c F: drivers/reset/reset-apple-cio.c +F: drivers/rtc/rtc-apple-pmu.c F: drivers/rtc/rtc-macsmc.c F: drivers/soc/apple/* F: drivers/spi/spi-apple.c diff --git a/drivers/nvmem/apple-spmi-nvmem.c b/drivers/nvmem/apple-spmi-nvmem.c index 7acb0c07d6abe..338e2e72f803d 100644 --- a/drivers/nvmem/apple-spmi-nvmem.c +++ b/drivers/nvmem/apple-spmi-nvmem.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,7 @@ static int apple_spmi_nvmem_write(void *priv, unsigned int offset, void *val, static int apple_spmi_nvmem_probe(struct spmi_device *sdev) { struct regmap *regmap; + struct nvmem_device *provider; struct nvmem_config nvmem_cfg = { .dev = &sdev->dev, .name = "spmi_nvmem", @@ -54,7 +56,13 @@ static int apple_spmi_nvmem_probe(struct spmi_device *sdev) nvmem_cfg.priv = regmap; - return PTR_ERR_OR_ZERO(devm_nvmem_register(&sdev->dev, &nvmem_cfg)); + provider = devm_nvmem_register(&sdev->dev, &nvmem_cfg); + if (IS_ERR(provider)) + return PTR_ERR(provider); + if (!of_device_is_compatible(sdev->dev.of_node, "apple,baku-pmic")) + return 0; + + return devm_of_platform_populate(&sdev->dev); } static const struct of_device_id apple_spmi_nvmem_id_table[] = { diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index 364afc73f8abd..5488a2dd5c86e 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -2100,6 +2100,17 @@ config RTC_DRV_WILCO_EC This can also be built as a module. If so, the module will be named "rtc_wilco_ec". +config RTC_DRV_APPLE_PMU + tristate "Apple J700 Baku PMU RTC alarm" + depends on ARCH_APPLE && NVMEM_APPLE_SPMI && PM_SLEEP + help + Read the Baku PMU clock and use alarms with retained-power + suspend-to-idle. The module requires experimental=1. Alarms are + limited to one hour; deeper suspend states are not supported. + Continue setting the shared clock through the SMC RTC. + + This driver can be built as the module rtc-apple-pmu. + config RTC_DRV_MACSMC tristate "Apple Mac System Management Controller RTC" depends on MFD_MACSMC diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile index 6cf7e066314e1..729ff56c871d3 100644 --- a/drivers/rtc/Makefile +++ b/drivers/rtc/Makefile @@ -94,6 +94,7 @@ obj-$(CONFIG_RTC_DRV_M48T35) += rtc-m48t35.o obj-$(CONFIG_RTC_DRV_M48T59) += rtc-m48t59.o obj-$(CONFIG_RTC_DRV_M48T86) += rtc-m48t86.o obj-$(CONFIG_RTC_DRV_MA35D1) += rtc-ma35d1.o +obj-$(CONFIG_RTC_DRV_APPLE_PMU) += rtc-apple-pmu.o obj-$(CONFIG_RTC_DRV_MACSMC) += rtc-macsmc.o obj-$(CONFIG_RTC_DRV_MAX31335) += rtc-max31335.o obj-$(CONFIG_RTC_DRV_MAX6900) += rtc-max6900.o diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c new file mode 100644 index 0000000000000..f398a1a802165 --- /dev/null +++ b/drivers/rtc/rtc-apple-pmu.c @@ -0,0 +1,369 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Apple Baku PMU clock and retained-power alarm. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BAKU_RTC_CONTROL 0xf800 +#define BAKU_RTC_COUNTER 0xf802 +#define BAKU_RTC_COMPARE 0xf808 +#define BAKU_RTC_EVENT 0xf80c +#define BAKU_RTC_MASK 0xf80e +#define BAKU_RTC_RUN BIT(0) +#define BAKU_RTC_ARM BIT(6) +#define BAKU_RTC_ALARM BIT(0) +#define BAKU_RTC_HZ_SHIFT 16 +#define BAKU_RTC_MAX_DELAY 3600 + +static bool experimental; +module_param(experimental, bool, 0444); +MODULE_PARM_DESC(experimental, "Enable alarms with retained SPMI/AIC power in s2idle"); + +struct baku_rtc { + struct regmap *map; + struct nvmem_cell *offset; + struct rtc_device *clock; + /* Serialize comparator updates and interrupt acknowledgment. */ + struct mutex transaction; + int interrupt; + bool io_failed; +}; + +static int baku_rtc_seconds(struct baku_rtc *baku, u64 *seconds) +{ + u8 count[8] = {}; + int error; + + error = regmap_bulk_read(baku->map, BAKU_RTC_COUNTER, count, 6); + if (error) + return error; + *seconds = get_unaligned_le64(count) >> BAKU_RTC_HZ_SHIFT; + return 0; +} + +static int baku_rtc_offset(struct baku_rtc *baku, s64 *seconds) +{ + size_t bytes; + void *value; + + /* The SMC RTC can update this cell, so do not cache it at probe. */ + value = nvmem_cell_read(baku->offset, &bytes); + if (IS_ERR(value)) + return PTR_ERR(value); + if (bytes != sizeof(__le64)) { + kfree(value); + return -EINVAL; + } + *seconds = get_unaligned_le64(value); + kfree(value); + if (*seconds < 0 || *seconds > RTC_TIMESTAMP_END_9999 - U32_MAX) + return -ERANGE; + return 0; +} + +static int baku_rtc_read_time(struct device *dev, struct rtc_time *time) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + u64 counter; + s64 epoch; + int error; + + guard(mutex)(&baku->transaction); + error = baku_rtc_offset(baku, &epoch); + if (error) + return error; + error = baku_rtc_seconds(baku, &counter); + if (error) + return error; + rtc_time64_to_tm(epoch + counter, time); + return 0; +} + +/* All comparator/control accesses are serialized with the IRQ thread. */ +static int baku_rtc_stop_alarm(struct baku_rtc *baku) +{ + int error; + + error = regmap_update_bits(baku->map, BAKU_RTC_MASK, + BAKU_RTC_ALARM, BAKU_RTC_ALARM); + if (error) + return error; + error = regmap_update_bits(baku->map, BAKU_RTC_CONTROL, BAKU_RTC_ARM, 0); + if (error) + return error; + return regmap_write(baku->map, BAKU_RTC_EVENT, BAKU_RTC_ALARM); +} + +static int baku_rtc_start_alarm(struct baku_rtc *baku) +{ + int error; + + error = regmap_write(baku->map, BAKU_RTC_EVENT, BAKU_RTC_ALARM); + if (!error) + error = regmap_update_bits(baku->map, BAKU_RTC_MASK, BAKU_RTC_ALARM, 0); + if (!error) + error = regmap_update_bits(baku->map, BAKU_RTC_CONTROL, + BAKU_RTC_RUN | BAKU_RTC_ARM, + BAKU_RTC_RUN | BAKU_RTC_ARM); + if (error) + baku_rtc_stop_alarm(baku); + return error; +} + +static int baku_rtc_validate_alarm(struct baku_rtc *baku, u64 comparator) +{ + u64 counter; + int error; + + error = baku_rtc_seconds(baku, &counter); + if (error) + return error; + if (comparator <= counter || comparator - counter > BAKU_RTC_MAX_DELAY) + return -ETIME; + return 0; +} + +static int baku_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alarm) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + unsigned int control, event, mask; + __le32 comparator; + s64 epoch; + int error; + + guard(mutex)(&baku->transaction); + error = baku_rtc_offset(baku, &epoch); + if (error) + return error; + error = regmap_bulk_read(baku->map, BAKU_RTC_COMPARE, + &comparator, sizeof(comparator)); + if (!error) + error = regmap_read(baku->map, BAKU_RTC_CONTROL, &control); + if (!error) + error = regmap_read(baku->map, BAKU_RTC_EVENT, &event); + if (!error) + error = regmap_read(baku->map, BAKU_RTC_MASK, &mask); + if (error) + return error; + rtc_time64_to_tm(epoch + le32_to_cpu(comparator), &alarm->time); + alarm->enabled = (control & BAKU_RTC_ARM) && !(mask & BAKU_RTC_ALARM); + alarm->pending = !!(event & BAKU_RTC_ALARM); + return 0; +} + +static int baku_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alarm) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + __le32 comparator, readback; + s64 epoch, deadline; + int error; + + guard(mutex)(&baku->transaction); + if (baku->io_failed) + return -EIO; + error = baku_rtc_offset(baku, &epoch); + if (error) + return error; + deadline = rtc_tm_to_time64(&alarm->time) - epoch; + if (deadline < 0 || deadline > U32_MAX) + return -ERANGE; + error = baku_rtc_validate_alarm(baku, deadline); + if (error) + return error; + error = baku_rtc_stop_alarm(baku); + if (error) + return error; + comparator = cpu_to_le32(deadline); + error = regmap_bulk_write(baku->map, BAKU_RTC_COMPARE, + &comparator, sizeof(comparator)); + if (error) + return error; + error = regmap_bulk_read(baku->map, BAKU_RTC_COMPARE, + &readback, sizeof(readback)); + if (error) + return error; + if (readback != comparator) + return -EIO; + if (alarm->enabled) + return baku_rtc_start_alarm(baku); + return 0; +} + +static int baku_rtc_alarm_enable(struct device *dev, unsigned int enable) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + __le32 comparator; + int error; + + guard(mutex)(&baku->transaction); + if (!enable) + return baku_rtc_stop_alarm(baku); + if (baku->io_failed) + return -EIO; + error = regmap_bulk_read(baku->map, BAKU_RTC_COMPARE, + &comparator, sizeof(comparator)); + if (error) + return error; + error = baku_rtc_validate_alarm(baku, le32_to_cpu(comparator)); + return error ?: baku_rtc_start_alarm(baku); +} + +static irqreturn_t baku_rtc_interrupt(int irq, void *context) +{ + struct device *dev = context; + struct baku_rtc *baku = dev_get_drvdata(dev); + unsigned int event; + int error; + + mutex_lock(&baku->transaction); + error = regmap_read(baku->map, BAKU_RTC_EVENT, &event); + if (!error && (event & BAKU_RTC_ALARM)) + error = baku_rtc_stop_alarm(baku); + if (error) { + baku->io_failed = true; + disable_irq_nosync(irq); + } + mutex_unlock(&baku->transaction); + if (error) { + dev_err(dev, "cannot acknowledge RTC interrupt: %d\n", error); + pm_wakeup_dev_event(dev, 0, true); + return IRQ_HANDLED; + } + if (!(event & BAKU_RTC_ALARM)) + return IRQ_NONE; + if (device_may_wakeup(dev)) + pm_wakeup_dev_event(dev, 0, true); + rtc_update_irq(baku->clock, 1, RTC_IRQF | RTC_AF); + return IRQ_HANDLED; +} + +static const struct rtc_class_ops baku_rtc_ops = { + .read_time = baku_rtc_read_time, + .read_alarm = baku_rtc_read_alarm, + .set_alarm = baku_rtc_set_alarm, + .alarm_irq_enable = baku_rtc_alarm_enable, +}; + +static int baku_rtc_prepare(struct device *dev) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE || baku->io_failed) + return -EBUSY; + return 0; +} + +static const struct dev_pm_ops baku_rtc_pm = { + .prepare = baku_rtc_prepare, +}; + +static void baku_rtc_release(void *context) +{ + struct device *dev = context; + struct baku_rtc *baku = dev_get_drvdata(dev); + + disable_irq(baku->interrupt); + mutex_lock(&baku->transaction); + if (baku_rtc_stop_alarm(baku)) + dev_err(dev, "cannot disable RTC alarm on removal\n"); + mutex_unlock(&baku->transaction); + device_init_wakeup(dev, false); +} + +static int baku_rtc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct baku_rtc *baku; + unsigned int control; + int error; + + /* Retention has only been measured on J700 with the explicit opt-in. */ + if (!experimental) + return -ENODEV; + baku = devm_kzalloc(dev, sizeof(*baku), GFP_KERNEL); + if (!baku) + return -ENOMEM; + baku->map = dev_get_regmap(dev->parent, NULL); + if (!baku->map) + return dev_err_probe(dev, -EPROBE_DEFER, "PMIC regmap unavailable\n"); + baku->offset = devm_nvmem_cell_get(dev, "rtc_offset"); + if (IS_ERR(baku->offset)) + return dev_err_probe(dev, PTR_ERR(baku->offset), "RTC offset unavailable\n"); + baku->interrupt = platform_get_irq(pdev, 0); + if (baku->interrupt < 0) + return baku->interrupt; + mutex_init(&baku->transaction); + platform_set_drvdata(pdev, baku); + error = regmap_read(baku->map, BAKU_RTC_CONTROL, &control); + if (error) + return error; + /* Respect an alarm left armed by firmware. */ + if (control & BAKU_RTC_ARM) + return -EBUSY; + baku->clock = devm_rtc_allocate_device(dev); + if (IS_ERR(baku->clock)) + return PTR_ERR(baku->clock); + baku->clock->ops = &baku_rtc_ops; + baku->clock->range_min = 0; + baku->clock->range_max = RTC_TIMESTAMP_END_9999; + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, baku->clock->features); + error = device_init_wakeup(dev, true); + if (error) + return error; + /* + * This interrupt remains live during retained-power s2idle. The + * thread uses the parent PMIC's SPMI regmap to acknowledge the alarm. + * Off-domain wake routing is not established by this driver. + */ + error = devm_request_threaded_irq(dev, baku->interrupt, NULL, + baku_rtc_interrupt, + IRQF_NO_AUTOEN | IRQF_NO_SUSPEND | IRQF_ONESHOT, + dev_name(dev), dev); + if (error) + goto disable_wakeup; + error = baku_rtc_stop_alarm(baku); + if (error) + goto disable_wakeup; + error = devm_rtc_register_device(baku->clock); + if (error) + goto disable_wakeup; + error = devm_add_action_or_reset(dev, baku_rtc_release, dev); + if (error) + return error; + enable_irq(baku->interrupt); + return 0; + +disable_wakeup: + device_init_wakeup(dev, false); + return error; +} + +static const struct of_device_id baku_rtc_match[] = { + { .compatible = "apple,j700-pmu-rtc" }, + { } +}; +MODULE_DEVICE_TABLE(of, baku_rtc_match); + +static struct platform_driver baku_rtc_driver = { + .probe = baku_rtc_probe, + .driver = { + .name = "apple-pmu-rtc", + .of_match_table = baku_rtc_match, + .pm = &baku_rtc_pm, + }, +}; +module_platform_driver(baku_rtc_driver); + +MODULE_AUTHOR("Eryk Wieliczko "); +MODULE_DESCRIPTION("Apple Baku PMU RTC alarm with retained-power s2idle"); +MODULE_LICENSE("GPL"); From b1dba8e165d45c78761dd5c7def4f1192b2e2e67 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:43:26 +1100 Subject: [PATCH 0514/1579] arm64: dts: apple: describe the J700 PMU alarm function Add the RTC function below the existing Baku SPMI PMIC and share the whole-second offset cell with the SMC RTC. Use peripheral interrupt 2 for the retained-power alarm. The driver requires an explicit opt-in. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 2dd921b23368c..818286ac18241 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -337,6 +337,15 @@ compatible = "apple,baku-pmic", "apple,spmi-nvmem"; reg = <0xe SPMI_USID>; + pmu_rtc: rtc { + compatible = "apple,j700-pmu-rtc"; + interrupt-parent = <&nub_spmi0>; + interrupts = <2 IRQ_TYPE_EDGE_RISING>; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + wakeup-source; + }; + nvmem-layout { compatible = "fixed-layout"; #address-cells = <1>; From 1b3813b8d50f3fb868b150dd2f06c80427290b7d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:44:55 +1100 Subject: [PATCH 0515/1579] accel/ane: keep the wedge until recovery proves DMA quiescent Closing any ANE file cleared the global wedge before recovery checked it, so the reset was skipped and live DMA backing could be freed or reused. Retry recovery with the wedge and module pin intact. Only the existing successful reset path may clear them; a failed reset requires a reboot and must preserve resources across unrelated file closes. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane.h | 9 --------- drivers/accel/ane/ane_drv.c | 24 +----------------------- drivers/accel/ane/ane_tm.c | 23 ----------------------- 3 files changed, 1 insertion(+), 55 deletions(-) diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index eac81bdab9f74..9a512e7568e60 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -135,15 +135,6 @@ struct ane_request { u32 bar[DRM_ANE_TILE_COUNT]; }; -/* - * Drop the wedge pin (module refcount) without requiring a successful - * recovery. Used by ane_drm_postclose so a wedged engine cannot leave - * the module un-unloadable after a clean session close. Recovery stays - * safe to call afterwards; if it returns error, the postclose path has - * already unlocked the module count so the operator can swap the ko. - */ -void ane_wedge_clear(struct ane_device *ane); - /* * Reclaim wedge-preserved BO ranges (unmap, release nodes and pages). * Only legal once DMA is provably quiescent: after a successful diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 70f0d9bd9b365..7fd0b9e717a91 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -796,21 +796,7 @@ static int ane_drm_open(struct drm_device *drm, struct drm_file *file) return 0; } -/* - * Per-session teardown: every file context open is a fresh boundary. - * Anything the previous session left on the device (wedged flag, tm - * completion counters, in-flight recovery state) belongs to that - * session and must not survive into the next one. - * - * drm_gem_release runs before us and has dropped every handle this - * file held. BOs leaked by `ane_gem_free_object under wedge` stay on - * ane->bo_list until module teardown (their backing `drm_gem_object`s - * are already gone, and touching them here would be use-after-free). - * We do not try to drain them; we only clear the session-leaving - * state (wedged + tm counters + recovering flag) so the next opener - * sees a clean device. The orphan BOs remain inert: they are not in - * any drm_file's handle table. - */ +/* Retry recovery on close, retaining quarantine if DMA cannot be quiesced. */ static void ane_drm_postclose(struct drm_device *drm, struct drm_file *file) { struct ane_device *ane = drm->dev_private; @@ -821,14 +807,6 @@ static void ane_drm_postclose(struct drm_device *drm, struct drm_file *file) return; } - /* Drop the wedge pin first: a wedged engine that cannot be - * recovered in software must NOT keep the module pinned, or the - * operator cannot unload the ko for an updated build without a - * reboot. ane_wedge_clear releases both the wedged flag and the - * module_refcount; ane_tm_recover then attempts a power-cycle - * recovery (succeeds only if the engine is actually idle-able). - */ - ane_wedge_clear(ane); ane_tm_recover(ane); mutex_unlock(&ane->engine_lock); diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 703d595b612a9..34f5144cea2a9 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -645,26 +645,3 @@ int ane_tm_recover(struct ane_device *ane) return 0; } -/* - * Drop the wedge pin on demand. The recovery path in ane_tm_recover - * does an atomic_xchg + module_put, but only on its success branch - - * a real -110 with the engine stuck leaves ane->wedged set and the - * module refcount one too high, so rmmod fails forever and the bug - * becomes a kernel-brick-without-reboot. ane_drm_postclose calls - * this unconditionally so every session close releases the pin, - * letting the operator unload the ko for an updated build even when - * the engine itself cannot be cleared in software. - * - * Pair count: must follow an ane_wedge_set() that called - * __module_get(THIS_MODULE). Today the only such set is in the wedge: - * branch of ane_tm_execute(); future sets must call this in their - * clean-up path too. - */ -void ane_wedge_clear(struct ane_device *ane) -{ - if (atomic_xchg(&ane->wedged, 0)) { - module_put(THIS_MODULE); - dev_info(ane->dev, - "wedge pin released (engine state unknown)\n"); - } -} From a10980d3ab632b04930ce668338d4d5efc95fac5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:44:57 +1100 Subject: [PATCH 0516/1579] accel/ane: pin H14 EXEC IO buffers through firmware completion The firmware reference flag prevented physical deallocation but did not stop BO_FREE or the last VMA close returning an in-flight IO BO to the global pool. Take one kref per IO element under bo_lock and release all acquired references only after the command completes. Validation errors unwind the same reference budget, and firmware failures latch quarantine before any final put can recycle storage. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane_t6021_rtclient_main.c | 31 +++++++++++++-------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index be21e4c1b0eb1..90bb89339486b 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -1314,11 +1314,12 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, const struct drm_ane_exec *user) { struct drm_ane_exec_io *ios; + struct ane_t6021_bo **exec_bos; struct ane_legacy_buffer *command; struct ane_t6021_fd *fd = file->driver_priv; size_t ios_size; size_t cmd_size; - int ret, i; + int ret, i, pinned = 0; if (user->pad || user->count < 1 || user->count > DRM_ANE_MAX_BINDS || user->priority < 2 || user->priority > 7 || !fd) @@ -1346,17 +1347,21 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, return -E2BIG; } + exec_bos = kcalloc(user->count, sizeof(*exec_bos), GFP_KERNEL); + if (!exec_bos) { + kfree(ios); + return -ENOMEM; + } + mutex_lock(&ane_t6021_fw_lock); if (atomic_read(&ane_t6021_quarantined)) { - mutex_unlock(&ane_t6021_fw_lock); - kfree(ios); - return -ETIMEDOUT; + ret = -ETIMEDOUT; + goto unlock; } command = ane->cmd_buf; if (!command) { - mutex_unlock(&ane_t6021_fw_lock); - kfree(ios); - return -ENODEV; + ret = -ENODEV; + goto unlock; } memset(command->cpu, 0, SZ_16K); { @@ -1385,11 +1390,9 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, ret = -EINVAL; goto unlock; } - /* The IOVA below is about to be published to the - * firmware, so mark the BO before the exchange. - * Under bo_lock the handle reference cannot go - * away, so no kref is needed here. - */ + /* Keep this IO element out of the reuse pool until completion. */ + kref_get(&bo->refcount); + exec_bos[pinned++] = bo; bo->fw_ref = true; iova = bo->dma; mutex_unlock(&ane_t6021_bo_lock); @@ -1416,6 +1419,10 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, user->timeout_ms : 5000); } unlock: + /* Command failures latch quarantine before these final puts. */ + while (pinned) + kref_put(&exec_bos[--pinned]->refcount, ane_t6021_bo_release); + kfree(exec_bos); mutex_unlock(&ane_t6021_fw_lock); kfree(ios); return ret; From dc5bd84dddaa394ccfea643b6ef5173a7b699110 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:44:58 +1100 Subject: [PATCH 0517/1579] accel/ane: retain fault-drain scratch mappings until quiescence A failed drain freed scratch pages and IOVA reservations before attempting to stop the DMA-active engine. Store the bounded scratch registry on the device, protected by engine_lock, so it survives the request's return. Reclaim it with the existing preserved BO registry only after successful recovery or a safe removal. Normal confirmed-idle completion still releases scratch immediately. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane.h | 21 ++++++++++++--------- drivers/accel/ane/ane_drv.c | 2 ++ drivers/accel/ane/ane_tm.c | 16 +++++++--------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index 9a512e7568e60..c9d18fb13f521 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -17,6 +17,15 @@ #include "ane_stats.h" +#define ANE_DART_MAX 3 +#define ANE_DART_SCRATCH_MAX 16 + +struct ane_dart_scratch { + u64 iova; + struct page *page; + struct drm_mm_node *node; +}; + struct ane_device { struct drm_device drm; struct device *dev; @@ -62,6 +71,9 @@ struct ane_device { * nodes (see ane_reclaim_preserved). Registered under engine_lock. */ struct list_head preserved_list; + /* Fault-drain DMA backing, retained until completion or a safe reset. */ + struct ane_dart_scratch scratch[ANE_DART_SCRATCH_MAX]; + int scratch_pages; bool removed; /* @@ -147,15 +159,6 @@ void ane_boost_exit(struct ane_device *ane); void ane_boost_begin(struct ane_device *ane); void ane_boost_end(struct ane_device *ane); -#define ANE_DART_MAX 3 -#define ANE_DART_SCRATCH_MAX 16 - -struct ane_dart_scratch { - u64 iova; - struct page *page; - struct drm_mm_node *node; -}; - int ane_dart_init(struct ane_device *ane); void ane_dart_mask(struct ane_device *ane); void ane_dart_unmask(struct ane_device *ane); diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 7fd0b9e717a91..c98f2348e82f1 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -360,6 +360,8 @@ void ane_reclaim_preserved(struct ane_device *ane) unsigned long count = 0; lockdep_assert_held(&ane->engine_lock); + ane_dart_release_scratch(ane, ane->scratch, ane->scratch_pages); + ane->scratch_pages = 0; list_for_each_entry_safe(p, tmp, &ane->preserved_list, list) { u64 iova = p->mm->start; diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 34f5144cea2a9..ba234d57cdf74 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -265,8 +265,7 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) u32 finished = 0; u32 fault_status = 0; u64 fault_iova = 0; - struct ane_dart_scratch scratch[ANE_DART_SCRATCH_MAX]; - int scratch_pages = 0; + struct ane_dart_scratch *scratch = ane->scratch; bool faulted = false; int err, status; @@ -298,10 +297,10 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) err = status; if (ane_dart_faulted(ane, &fault_iova, &fault_status)) { faulted = true; - if (scratch_pages >= ANE_DART_SCRATCH_MAX) + if (ane->scratch_pages >= ANE_DART_SCRATCH_MAX) break; if (ane_dart_drain_fault(ane, fault_iova, scratch, - &scratch_pages)) + &ane->scratch_pages)) break; /* The retried transaction now completes; poll on. */ continue; @@ -321,7 +320,7 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) if (err || ane_dart_faulted(ane, NULL, NULL)) { /* Drain failed or the drained program never reached * completion: the engine is not idle and access is - * off the table. wedge releases the scratch. + * off the table. Retain scratch backing until reset. */ err = err ?: -EIO; goto wedge; @@ -334,11 +333,12 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) */ rmb(); tq_write32(ane, TQ_STATUS(req->qid), 0x0); - ane_dart_release_scratch(ane, scratch, scratch_pages); + ane_dart_release_scratch(ane, scratch, ane->scratch_pages); ane_dart_unmask(ane); dev_err(ane->dev, "DART fault contained: status=%#x iova=%#llx scratch=%d\n", - fault_status, fault_iova, scratch_pages); + fault_status, fault_iova, ane->scratch_pages); + ane->scratch_pages = 0; return -EIO; } if (err) @@ -356,7 +356,6 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) return 0; wedge: - ane_dart_release_scratch(ane, scratch, scratch_pages); ane_dart_unmask(ane); if (atomic_xchg(&ane->wedged, 1) == 0) { __module_get(THIS_MODULE); @@ -644,4 +643,3 @@ int ane_tm_recover(struct ane_device *ane) } return 0; } - From f527149b2521930d037f8fead78c054f6563d4ec Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:43:48 +1100 Subject: [PATCH 0518/1579] nvme: apple: guard crypto cleanup on controllers without TCBs T8015 does not allocate NVMMU transaction control blocks. Complete its ordinary DMA unmapping before returning, and access crypto state only on controllers with LSQ NVMMU support. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 0513d120abffd..83d5924f640f3 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -876,6 +876,9 @@ static __always_inline void apple_nvme_unmap_rq(struct request *req) if (blk_rq_nr_phys_segments(req)) apple_nvme_unmap_data(anv, req); + if (!anv->hw->has_lsq_nvmmu) + return; + tag = nvme_tag_from_cid(iod->cmd.common.command_id); tcb = &iod->q->tcbs[tag]; if (unlikely(tcb->dma_flags & BIT(2))) { From 2be5ef83e7f974383175bf23e64f31573bac12aa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:46:56 +1100 Subject: [PATCH 0519/1579] cpufreq: apple: retain the unmanaged P-state ceiling on T8152 PR 43 replaces the private J700 thermal controller with thermal zones and cpufreq cooling. On the current base, T8152 also uses the private-policy prerequisite. Removing that check globally allows its high P-states even though no replacement policy has been provided for that SoC. Stop requiring the private policy on T8140, whose thermal zones this series supplies. Keep rejecting tables above the unmanaged ceiling for other SoCs that still require qualified thermal protection. Preserve the current SoC-specific transition timeout and verification paths. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index dbb47ece7dc3f..c4ac65da9f873 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -119,8 +119,6 @@ static const struct apple_soc_cpufreq_info soc_t8112_info = { */ static const struct apple_soc_cpufreq_info soc_t8140_info = { .verify_transition = true, - .needs_thermal_policy = true, - .max_unmanaged_pstate = 2, .min_pstate = 1, .max_pstate = 31, .ps1_mask = APPLE_DVFS_CMD_PS1, @@ -335,6 +333,7 @@ static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) { + struct cpufreq_frequency_table *p; int ret, i; unsigned int transition_latency; void __iomem *reg_base; @@ -436,6 +435,16 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) policy->dvfs_possible_from_any_cpu = true; policy->fast_switch_possible = !info->verify_transition; policy->suspend_freq = freq_table[0].frequency; + /* Unqualified SoCs still require protection above their safe ceiling. */ + if (info->needs_thermal_policy) { + cpufreq_for_each_valid_entry(p, policy->freq_table) { + if (p->driver_data <= info->max_unmanaged_pstate) + continue; + dev_err(cpu_dev, "higher P-states require a qualified thermal policy\n"); + ret = -ENODEV; + goto out_free_cpufreq_table; + } + } return 0; From 586ae061448db34bfd4f799ec9550dfb65c09165 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:47:16 +1100 Subject: [PATCH 0520/1579] soc: apple: sep: acquire the sensor enable GPIO by name The fingerprint binding uses enable-gpios. Request that named GPIO first while retaining the legacy unnamed property and board-specific fallback. Propagate acquisition errors and defer an unavailable GPIO supplier. Refuse binding when no usable power control can be obtained. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sensor_shim.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index 60cef6eee9e30..5738312663b79 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -218,7 +218,15 @@ static int sep_acquire_power(struct spi_device *spi) of_property_present(spi->dev.of_node, "apple,smc-power-key")) return sep_acquire_smc_power(spi); - sep_power = gpiod_get_index(&spi->dev, NULL, 0, GPIOD_OUT_LOW); + sep_power = gpiod_get(&spi->dev, "enable", GPIOD_OUT_LOW); + if (IS_ERR(sep_power) && PTR_ERR(sep_power) == -ENOENT) + sep_power = gpiod_get_index(&spi->dev, NULL, 0, GPIOD_OUT_LOW); + if (IS_ERR(sep_power) && PTR_ERR(sep_power) != -ENOENT) { + int error = PTR_ERR(sep_power); + + sep_power = NULL; + return error; + } if (!IS_ERR(sep_power)) { sep_power_source = SEP_POWER_NODE_PROPERTY; dev_info(&spi->dev, @@ -230,12 +238,12 @@ static int sep_acquire_power(struct spi_device *spi) /* * The fallback below is the j414s power line; a sibling board's sensor sits * on a different pin, so only j414s may use it. Every other machine must - * describe the power GPIO in its device node (the gpiod_get_index path). + * describe enable-gpios in its device node. */ if (!of_machine_is_compatible("apple,j414s")) { dev_warn(&spi->dev, - "sep sensor: no power GPIO in the device node; describe gpios in DT\n"); - return 0; + "sep sensor: no power GPIO in the device node; describe enable-gpios in DT\n"); + return -ENODEV; } np = of_find_node_by_path(SEP_SENSOR_GPIO_NODE); @@ -243,7 +251,7 @@ static int sep_acquire_power(struct spi_device *spi) dev_warn(&spi->dev, "sep sensor: no DT node at %s, no power line\n", SEP_SENSOR_GPIO_NODE); - return 0; + return -ENODEV; } gdev = gpio_device_find_by_fwnode(of_fwnode_handle(np)); @@ -252,7 +260,7 @@ static int sep_acquire_power(struct spi_device *spi) dev_warn(&spi->dev, "sep sensor: %s has no registered GPIO device\n", SEP_SENSOR_GPIO_NODE); - return 0; + return -EPROBE_DEFER; } gc = gpio_device_get_chip(gdev); @@ -269,7 +277,7 @@ static int sep_acquire_power(struct spi_device *spi) SEP_SENSOR_GPIO_LINE, SEP_SENSOR_GPIO_NODE, gpio_device_get_label(gdev)); gpio_device_put(gdev); - return 0; + return -ENODEV; } sep_power_source = SEP_POWER_CHIP_LINE; From 2c815e33eced0f47a8b12914d0403c5da25f6198 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:44:42 +1100 Subject: [PATCH 0521/1579] PCI: apple: preserve the retained bootstrap across suspend-to-idle sleep The retained PIODMA supplier prevents the radio configuration from being built with system sleep. Permit suspend-to-idle once the single bootstrap has completed and the command, FIFO and owned storage validate as idle. Reject sleep from a failed or incomplete bootstrap. Restore the existing register-bank configuration after its power domain resumes, while preserving the arena and never resubmitting its descriptor. Reject hibernation and deeper suspend, which cannot preserve the live endpoint and firmware ownership contract. Keep the supplier and PCI host built in: published DMA and supplier IRQ ownership still lack a removal contract. This retains the existing link boundary and requires no module export or parallel provider. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/PCI/controller/apple-t8140.rst | 10 ++- drivers/pci/controller/Kconfig | 9 ++- .../pci/controller/pcie-apple-piodma-diag.c | 66 ++++++++++++++++++- 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 90d4eab8f07f4..24213c2cc3aa7 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -11,11 +11,19 @@ This is an opt-in bring-up implementation, not general T8140 PCIe support. Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y`` and pass ``pcie_apple_piodma_diag.enumerate=1`` to the kernel. Without the opt-in, the root-only diagnostic path continues to reject downstream configuration access. -The option excludes suspend and kexec. Failed or uncertain bootstrap state +Build the host and its DART in as well: the supplier retains published DMA +and has no removal contract. Keeping the supplier and host built in avoids +a module dependency on a provider whose lifetime cannot end at unload. + +Suspend-to-idle suspend retains the arena after checking the completed request, +idle FIFO and command state. Resume restores the existing configuration +banks without issuing a second bootstrap. Hibernation, deeper suspend and +kexec are unsupported after activation. Failed or uncertain bootstrap state requires a full external hardware reset; there is no command retry. The PIODMA arena remains allocated until external reset. Controller removal, teardown, memory reuse and arbitrary downstream devices are unqualified. +Fresh hardware qualification must check both radio functions across sleep. Do not remove the controller or its IOMMU while this experiment is active. These lifecycle contracts require further work before production support. diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index ebdc821fea454..5ee9d27177f12 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -59,13 +59,16 @@ config PCIE_APPLE config PCIE_APPLE_PIODMA_DIAG bool "J700 PIODMA diagnostic (requires full reset after use)" depends on PCIE_APPLE=y && ARCH_APPLE && IOMMU_DMA - depends on !PM_SLEEP && !KEXEC_CORE + depends on !KEXEC_CORE help Build the experimental SID17 retained bootstrap provider. It submits one fixed request, validates typed ECAM and permits enumeration of the two Neo functions only with the explicit enumerate=1 parameter. - Its arena remains owned until a full reset. This is not production - support, and does not qualify memory reuse or radio operation. + The host and supplier must both be built in: supplier removal cannot + release a published DMA arena safely. Suspend-to-idle sleep retains that + arena and restores the configuration without repeating a request. + Hibernation and deeper suspend are rejected after activation. + This does not qualify memory reuse or arbitrary downstream devices. config PCI_VERSATILE bool "ARM Versatile PB PCI controller" diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 87caa91262e37..5b1e27ad29e52 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -8,7 +8,9 @@ * * This built-in driver deliberately retains its coherent arena and device * power reference until a coordinator-controlled full hardware reset. There - * is no unbind interface, suspend, kexec or retry. Native ECAM is only + * is no unbind interface, kexec or retry. Suspend-to-idle sleep preserves the + * arena and restores the register banks without resubmitting its request. + * Native ECAM is only * opened by the host after a successful bootstrap and typed read checks. */ #include @@ -29,6 +31,7 @@ #include #include #include +#include #include "pcie-apple-piodma-diag.h" @@ -92,6 +95,7 @@ struct apple_piodma_diag { u64 pointer_prefix; u8 secondary_bus; bool retained; + bool ready; }; static int apple_piodma_diag_root(struct apple_piodma_diag *diag, struct pci_dev *root) @@ -532,11 +536,13 @@ int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) ret = apple_piodma_diag_initialize(diag); if (!ret) ret = apple_piodma_diag_submit(diag, 0); - if (ret) + if (ret) { dev_err(supplier, "bootstrap failure=%d; retained ownership; no retry\n", ret); - else + } else { + WRITE_ONCE(diag->ready, true); dev_info(supplier, "BOOTSTRAP_VALIDATED word=%#x; one retained slot\n", diag->slots[0].result); + } return ret; } @@ -597,6 +603,59 @@ static void apple_piodma_diag_shutdown(struct platform_device *pdev) dev_warn(&pdev->dev, "diagnostic arena retained; require full hardware reset\n"); } +/* No new request is permitted once the host has admitted its endpoints. */ +static int apple_piodma_diag_sleep_check(struct apple_piodma_diag *diag) +{ + if (!READ_ONCE(diag->ready) || READ_ONCE(diag->irq_fault)) + return -EIO; + if (readl(diag->engine + 0x54) & BIT(0)) + return -EBUSY; + if ((readl(diag->engine + 0x0c) & PIODMA_FIFO_MASK) != 16) + return -EBUSY; + return apple_piodma_diag_check_slots(diag); +} + +static int apple_piodma_diag_suspend_noirq(struct device *dev) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(dev); + + if (!diag->retained) + return 0; + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) + return -EOPNOTSUPP; + return apple_piodma_diag_sleep_check(diag); +} + +static int apple_piodma_diag_resume_noirq(struct device *dev) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(dev); + int ret; + + if (!diag->retained) + return 0; + if (!READ_ONCE(diag->ready) || READ_ONCE(diag->irq_fault)) + return -EIO; + /* Genpd may gate this island; restore configuration, never a command. */ + ret = apple_piodma_diag_initialize(diag); + if (ret) + return ret; + return apple_piodma_diag_sleep_check(diag); +} + +static int apple_piodma_diag_freeze(struct device *dev) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(dev); + + return diag->retained ? -EOPNOTSUPP : 0; +} + +static const struct dev_pm_ops apple_piodma_diag_pm_ops = { + .suspend_noirq = apple_piodma_diag_suspend_noirq, + .resume_noirq = apple_piodma_diag_resume_noirq, + .freeze = apple_piodma_diag_freeze, + .poweroff = apple_piodma_diag_freeze, +}; + static const struct of_device_id apple_piodma_diag_match[] = { { .compatible = "apple,t8140-piodma-diagnostic" }, { } @@ -608,6 +667,7 @@ static struct platform_driver apple_piodma_diag_driver = { .driver = { .name = "apple-piodma-diag", .of_match_table = apple_piodma_diag_match, + .pm = pm_sleep_ptr(&apple_piodma_diag_pm_ops), .suppress_bind_attrs = true, }, }; From aca3d365aad20020392346780c8b65249f59ceff Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:54:44 +1100 Subject: [PATCH 0522/1579] arm64: configs: select the J700 radio transport and bootstrap parents The retained PIODMA bootstrap requires a built-in Apple PCI host, whose DART parent must also be built in. Select those providers in the Asahi fragment and request the existing Bluetooth transport as a module. Add a J700 radio fragment that selects the fullmac and modular Bluetooth paths, while preserving system sleep and disabling the kexec selectors that cannot preserve live firmware DMA ownership. It is applied after the general Asahi fragment. Runtime enumeration and Bluetooth admission remain opt-in and firmware/calibration remain external packages. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + arch/arm64/configs/asahi.config | 5 +++-- arch/arm64/configs/j700-radio.config | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/configs/j700-radio.config diff --git a/MAINTAINERS b/MAINTAINERS index 90654e399c0d3..68c81fecc4d16 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1981,6 +1981,7 @@ F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml F: Documentation/hid/apple-j700-multitouch.rst F: arch/arm64/boot/dts/apple/t8140* +F: arch/arm64/configs/j700-radio.config F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 20a50b6fcb3a1..d3fb24fe5fcaf 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -10,7 +10,8 @@ CONFIG_ARM_APPLE_SOC_CPUFREQ=m CONFIG_APPLE_SMC_CPU_THERMAL=y CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m -CONFIG_PCIE_APPLE=m +CONFIG_BT_MTK7932_PCIE=m +CONFIG_PCIE_APPLE=y CONFIG_NVME_APPLE=m CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y @@ -54,7 +55,7 @@ CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m CONFIG_MFD_MACSMC=m CONFIG_COMMON_CLK_APPLE_NCO=m -CONFIG_APPLE_DART=m +CONFIG_APPLE_DART=y CONFIG_APPLE_DOCKCHANNEL=m CONFIG_APPLE_MAILBOX=y CONFIG_APPLE_PMGR_MISC=y diff --git a/arch/arm64/configs/j700-radio.config b/arch/arm64/configs/j700-radio.config new file mode 100644 index 0000000000000..87b09a42d3cc6 --- /dev/null +++ b/arch/arm64/configs/j700-radio.config @@ -0,0 +1,19 @@ +# J700 retained radio bootstrap; merge after defconfig and asahi.config. +# System sleep is retained; active radio firmware cannot cross kexec. +# Firmware/calibration and runtime admission follow the MT7932 documentation. +# CONFIG_KEXEC is not set +# CONFIG_KEXEC_FILE is not set +# CONFIG_KEXEC_HANDOVER is not set +CONFIG_PCI=y +CONFIG_PCIE_APPLE=y +CONFIG_APPLE_DART=y +CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_WLAN=y +CONFIG_WLAN_VENDOR_MEDIATEK=y +CONFIG_MT7932_FULLMAC=m +CONFIG_BT=m +CONFIG_BT_BREDR=y +CONFIG_BT_LE=y +CONFIG_BT_MTK7932_PCIE=m +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_CMAC=y From bc1823a897a70ecbfaa0d4ecba548c7b0352bce5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:43:10 +1100 Subject: [PATCH 0523/1579] wifi: brcmfmac: bound AWDL creation and validate its data path Use the vif-event lock to arbitrate completion and timeout of asynchronous AWDL creation. Defer cleanup of a failed registration to wiphy work, and drain event and wiphy work before protocol teardown. Keep the netdevice reference and validate its wiphy and AWDL role when requesting deletion. Retain the vendor dcmd errno ABI rather than toggling shared fwil error state around commands. Account for the 16-byte AWDL envelope in the MTU and reject oversized Ethernet-II frames before writing an invalid 802.3 length. Route the per-frame measurement trace through dynamic debug. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 150 +++++++++++++++--- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 7 + .../broadcom/brcm80211/brcmfmac/core.c | 30 +++- .../broadcom/brcm80211/brcmfmac/msgbuf.c | 14 +- .../broadcom/brcm80211/brcmfmac/vendor.c | 23 +-- 5 files changed, 177 insertions(+), 47 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index f7d188fe7de10..55192634f1fd2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -605,6 +605,85 @@ brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev) * * Return: pointer to new vif on success, ERR_PTR(-errno) if not */ +static void brcmf_awdl_cleanup_locked(struct brcmf_cfg80211_info *cfg) +{ + struct brcmf_cfg80211_vif *vif; + struct brcmf_if *ifp; + + spin_lock(&cfg->vif_event.vif_event_lock); + ifp = cfg->awdl_failed_ifp; + vif = cfg->awdl_vif; + cfg->awdl_failed_ifp = NULL; + cfg->awdl_vif = NULL; + WRITE_ONCE(cfg->awdl_pending, false); + cfg->awdl_attaching = false; + if (cfg->vif_event.vif == vif) { + cfg->vif_event.vif = NULL; + cfg->vif_event.action = 0; + } + spin_unlock(&cfg->vif_event.vif_event_lock); + + if (ifp) { + brcmf_fil_iovar_data_set(ifp, "interface_remove", NULL, 0); + brcmf_proto_del_if(cfg->pub, ifp); + ifp->vif = NULL; + } + if (vif) + brcmf_free_vif(vif); + if (ifp) { + if (ifp->ndev) { + ifp->ndev->ieee80211_ptr = NULL; + free_netdev(ifp->ndev); + } else { + kfree(ifp); + } + } +} + +static void brcmf_awdl_cleanup(struct wiphy *wiphy, struct wiphy_work *work) +{ + struct brcmf_cfg80211_info *cfg = + container_of(work, struct brcmf_cfg80211_info, awdl_cleanup); + + brcmf_awdl_cleanup_locked(cfg); +} + +void brcmf_cfg80211_awdl_stop(struct brcmf_cfg80211_info *cfg) +{ + wiphy_lock(cfg->wiphy); + wiphy_delayed_work_cancel(cfg->wiphy, &cfg->awdl_timeout); + wiphy_work_cancel(cfg->wiphy, &cfg->awdl_cleanup); + brcmf_awdl_cleanup_locked(cfg); + wiphy_unlock(cfg->wiphy); +} + +static void brcmf_awdl_timeout(struct wiphy *wiphy, struct wiphy_work *work) +{ + struct brcmf_cfg80211_info *cfg = + container_of(work, struct brcmf_cfg80211_info, awdl_timeout.work); + struct brcmf_cfg80211_vif *vif; + + spin_lock(&cfg->vif_event.vif_event_lock); + vif = cfg->awdl_vif; + /* IF_ADD has already claimed the vif; its worker owns completion. */ + if (!cfg->awdl_pending || cfg->awdl_attaching || + (vif && vif->ifp)) { + spin_unlock(&cfg->vif_event.vif_event_lock); + return; + } + cfg->awdl_vif = NULL; + WRITE_ONCE(cfg->awdl_pending, false); + if (cfg->vif_event.vif == vif) { + cfg->vif_event.vif = NULL; + cfg->vif_event.action = 0; + } + spin_unlock(&cfg->vif_event.vif_event_lock); + + brcmf_err("AWDL interface creation timed out\n"); + if (vif) + brcmf_free_vif(vif); +} + /** * brcmf_awdl_add_vif() - create an AWDL interface and give it a netdev. * @@ -621,7 +700,7 @@ brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev) * valid type for its bookkeeping; the firmware side is what actually decides * the interface behaviour, from the iftype passed to interface_create. * - * Return: pointer to the new vif on success, ERR_PTR(-errno) on failure. + * Return: 0 when requested, or a negative error code. */ int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name) { @@ -630,7 +709,10 @@ int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name) struct brcmf_cfg80211_vif *vif; int err; - if (brcmf_cfg80211_vif_event_armed(cfg) || cfg->awdl_pending) + if (cfg->pub->bus_if->state != BRCMF_BUS_UP) + return -ENODEV; + if (brcmf_cfg80211_vif_event_armed(cfg) || + READ_ONCE(cfg->awdl_pending)) return -EBUSY; vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_STATION); @@ -650,13 +732,17 @@ int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name) * */ strscpy(cfg->awdl_ifname, name, sizeof(cfg->awdl_ifname)); - cfg->awdl_pending = true; + spin_lock(&cfg->vif_event.vif_event_lock); + cfg->awdl_vif = vif; + WRITE_ONCE(cfg->awdl_pending, true); + spin_unlock(&cfg->vif_event.vif_event_lock); + wiphy_delayed_work_queue(wiphy, &cfg->awdl_timeout, + BRCMF_VIF_EVENT_TIMEOUT); err = brcmf_cfg80211_request_awdl_if(ifp, NULL); if (err) { - cfg->awdl_pending = false; - brcmf_cfg80211_arm_vif_event(cfg, NULL); - brcmf_free_vif(vif); + /* An IF_ADD may already be in flight: retire under the same lock. */ + wiphy_delayed_work_queue(wiphy, &cfg->awdl_timeout, 0); return err; } @@ -688,31 +774,54 @@ void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp) struct brcmf_cfg80211_vif *vif; int err; - if (!cfg || !cfg->awdl_pending) + if (!cfg) return; - cfg->awdl_pending = false; - vif = ifp->vif; + spin_lock(&cfg->vif_event.vif_event_lock); + vif = cfg->awdl_vif; + if (!cfg->awdl_pending || cfg->awdl_attaching || !vif || + ifp->vif != vif) { + spin_unlock(&cfg->vif_event.vif_event_lock); + return; + } + cfg->awdl_attaching = true; + spin_unlock(&cfg->vif_event.vif_event_lock); - if (!ifp->ndev || !vif) { - bphy_err(ifp->drvr, "AWDL if event without netdev/vif\n"); - goto done; + if (!ifp->ndev) { + bphy_err(ifp->drvr, "AWDL if event without netdev\n"); + goto fail; } - /* Must be set before brcmf_net_attach(), which picks netdev_ops. */ ifp->is_awdl = true; strscpy(ifp->ndev->name, cfg->awdl_ifname, sizeof(ifp->ndev->name)); err = brcmf_net_attach(ifp, false); if (err) { bphy_err(ifp->drvr, "registering AWDL netdevice failed\n"); - free_netdev(ifp->ndev); - brcmf_free_vif(vif); - goto done; + goto fail; } + spin_lock(&cfg->vif_event.vif_event_lock); + cfg->awdl_vif = NULL; + WRITE_ONCE(cfg->awdl_pending, false); + cfg->awdl_attaching = false; + if (cfg->vif_event.vif == vif) { + cfg->vif_event.vif = NULL; + cfg->vif_event.action = 0; + } + spin_unlock(&cfg->vif_event.vif_event_lock); brcmf_info("AWDL interface %s created (bsscfgidx %d)\n", ifp->ndev->name, ifp->bsscfgidx); -done: - brcmf_cfg80211_arm_vif_event(cfg, NULL); + return; + +fail: + /* Do not free a vif while an event or wiphy operation can still see it. */ + spin_lock(&cfg->vif_event.vif_event_lock); + if (cfg->pub->iflist[ifp->bsscfgidx] == ifp) + cfg->pub->iflist[ifp->bsscfgidx] = NULL; + cfg->awdl_failed_ifp = ifp; + cfg->vif_event.vif = NULL; + cfg->vif_event.action = 0; + spin_unlock(&cfg->vif_event.vif_event_lock); + wiphy_work_queue(cfg->wiphy, &cfg->awdl_cleanup); } /** @@ -731,6 +840,9 @@ int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev) struct brcmf_if *ifp = vif->ifp; int err; + if (wdev->wiphy != wiphy || !ifp || !ifp->is_awdl) + return -ENODEV; + /* Asymmetric with the rest of brcmfmac on purpose, for the same reason * brcmf_awdl_add_vif() does not call brcmf_net_attach(): nl80211 calls * vendor commands holding the wiphy mutex and NOT the RTNL @@ -8711,6 +8823,8 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, cfg->wiphy = wiphy; cfg->pub = drvr; init_vif_event(&cfg->vif_event); + wiphy_delayed_work_init(&cfg->awdl_timeout, brcmf_awdl_timeout); + wiphy_work_init(&cfg->awdl_cleanup, brcmf_awdl_cleanup); INIT_LIST_HEAD(&cfg->vif_list); cfg->force_band_setup = true; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index fdc9f7e627f0a..ce00102086f67 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -397,7 +397,13 @@ struct brcmf_cfg80211_info { struct work_struct escan_timeout_work; struct list_head vif_list; struct brcmf_cfg80211_vif_event vif_event; + /* Protected by vif_event.vif_event_lock. */ bool awdl_pending; + bool awdl_attaching; + struct brcmf_cfg80211_vif *awdl_vif; + struct brcmf_if *awdl_failed_ifp; + struct wiphy_delayed_work awdl_timeout; + struct wiphy_work awdl_cleanup; char awdl_ifname[IFNAMSIZ]; struct completion vif_disabled; struct brcmu_d11inf d11inf; @@ -510,6 +516,7 @@ bool brcmf_get_vif_state_any(struct brcmf_cfg80211_info *cfg, void brcmf_cfg80211_arm_vif_event(struct brcmf_cfg80211_info *cfg, struct brcmf_cfg80211_vif *vif); bool brcmf_cfg80211_vif_event_armed(struct brcmf_cfg80211_info *cfg); +void brcmf_cfg80211_awdl_stop(struct brcmf_cfg80211_info *cfg); int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name); void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp); int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 0392b2a8a9064..a4460f06c1301 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -333,13 +333,15 @@ __be16 brcmf_rx_eth_type_trans(struct brcmf_if *ifp, struct sk_buff *skb) * already carrying a length field (raw AF_PACKET senders that build the * encapsulation themselves) pass unchanged. Caller guarantees headroom. */ -static void brcmf_awdl_encap(struct brcmf_if *ifp, struct sk_buff *skb) +static int brcmf_awdl_encap(struct brcmf_if *ifp, struct sk_buff *skb) { const struct ethhdr *eh = (const struct ethhdr *)skb->data; u8 *hdr; if (ntohs(eh->h_proto) < ETH_P_802_3_MIN) - return; + return 0; + if (skb->len - ETH_HLEN + BRCMF_AWDL_ENCAP_LEN > ETH_DATA_LEN) + return -EMSGSIZE; hdr = skb_push(skb, BRCMF_AWDL_ENCAP_LEN); memmove(hdr, hdr + BRCMF_AWDL_ENCAP_LEN, 2 * ETH_ALEN); @@ -349,6 +351,7 @@ static void brcmf_awdl_encap(struct brcmf_if *ifp, struct sk_buff *skb) hdr + ETH_HLEN + sizeof(brcmf_awdl_snap)); hdr[ETH_HLEN + sizeof(brcmf_awdl_snap) + 2] = 0; hdr[ETH_HLEN + sizeof(brcmf_awdl_snap) + 3] = 0; + return 0; } static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, @@ -430,8 +433,13 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, /* set pacing shift for packet aggregation */ sk_pacing_shift_update(skb->sk, 8); - if (ifp->is_awdl) - brcmf_awdl_encap(ifp, skb); + if (ifp->is_awdl) { + ret = brcmf_awdl_encap(ifp, skb); + if (ret) { + brcmf_txfinalize(ifp, skb, false); + goto done; + } + } ret = brcmf_proto_tx_queue_data(drvr, ifp->ifidx, skb); if (ret < 0) @@ -772,8 +780,12 @@ int brcmf_net_attach(struct brcmf_if *ifp, bool locked) : &brcmf_netdev_ops_pri; ndev->needed_headroom += drvr->hdrlen; - if (ifp->is_awdl) + if (ifp->is_awdl) { ndev->needed_headroom += BRCMF_AWDL_ENCAP_LEN; + /* The 802.3 length includes the AWDL envelope. */ + ndev->max_mtu = ETH_DATA_LEN - BRCMF_AWDL_ENCAP_LEN; + ndev->mtu = ndev->max_mtu; + } ndev->ethtool_ops = &brcmf_ethtool_ops; /* set the mac address & netns */ @@ -1612,14 +1624,18 @@ void brcmf_detach(struct device *dev) #endif brcmf_bus_change_state(bus_if, BRCMF_BUS_DOWN); + /* Stop event producers before draining their worker. */ + brcmf_bus_stop(drvr->bus_if); + /* Drain IF events before their interfaces and protocol state go away. */ + brcmf_fweh_detach(drvr); + if (drvr->config) + brcmf_cfg80211_awdl_stop(drvr->config); /* make sure primary interface removed last */ for (i = BRCMF_MAX_IFS - 1; i > -1; i--) { if (drvr->iflist[i]) brcmf_remove_interface(drvr->iflist[i], false); } - brcmf_bus_stop(drvr->bus_if); - brcmf_fweh_detach(drvr); brcmf_proto_detach(drvr); if (drvr->mon_if) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 1df1a1592ca4f..c3dfd76c4dbfa 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -969,13 +969,13 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) ring_ifidx = msgbuf->flow->rings[flowid] ? brcmf_flowring_ifidx_get(msgbuf->flow, flowid) : 0; if (ring_ifidx && __ratelimit(&brcmf_awdl_txs_rs)) - pr_info("brcmfmac: awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", - ring_ifidx, - tx_status->msg.ifidx, ifp ? ifp->is_awdl : -1, flowid, - (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), - le16_to_cpu(tx_status->tx_status), - le16_to_cpu(tx_status->metadata_len), - skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); + brcmf_dbg(MSGBUF, "awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", + ring_ifidx, tx_status->msg.ifidx, + ifp ? ifp->is_awdl : -1, flowid, + (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), + le16_to_cpu(tx_status->tx_status), + le16_to_cpu(tx_status->metadata_len), + skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); brcmf_txfinalize(ifp, skb, true); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index d3211d092cf44..25623533effa7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -64,25 +64,14 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, *(char *)(dcmd_buf + len) = '\0'; } - /* Return the firmware's own BCME_* code instead of the -EBADE that - * brcmf_fil_cmd_data() collapses every failure to. Without this a - * probe cannot tell "no such iovar" from "bad argument" from "not - * up", which is most of what a probe needs to know. - */ - ifp->fwil_fwerr = true; if (cmdhdr->set) ret = brcmf_fil_cmd_data_set(ifp, cmdhdr->cmd, dcmd_buf, ret_len); else ret = brcmf_fil_cmd_data_get(ifp, cmdhdr->cmd, dcmd_buf, ret_len); - ifp->fwil_fwerr = false; - if (ret != 0) { - brcmf_err("dcmd %u ifidx=%d %s: firmware error %d\n", - cmdhdr->cmd, ifp->ifidx, - cmdhdr->set ? "set" : "get", ret); + if (ret != 0) goto exit; - } wr_pointer = dcmd_buf; while (ret_len > 0) { @@ -124,6 +113,7 @@ static int brcmf_cfg80211_vndr_cmds_awdl_handler(struct wiphy *wiphy, struct net_device *ndev; struct wireless_dev *awdl_wdev; u32 op; + int err; if (len < sizeof(op)) return -EINVAL; @@ -146,10 +136,13 @@ static int brcmf_cfg80211_vndr_cmds_awdl_handler(struct wiphy *wiphy, if (!ndev) return -ENODEV; awdl_wdev = ndev->ieee80211_ptr; - dev_put(ndev); - if (!awdl_wdev) + if (!awdl_wdev || awdl_wdev->wiphy != wiphy) { + dev_put(ndev); return -ENODEV; - return brcmf_awdl_del_vif(wiphy, awdl_wdev); + } + err = brcmf_awdl_del_vif(wiphy, awdl_wdev); + dev_put(ndev); + return err; default: return -EINVAL; } From 0ef02b99f3570f26233c7b91f30f190a684735e0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:56:08 +1100 Subject: [PATCH 0524/1579] Bluetooth: hci_bcm4377: stop recovery when the PCI reset fails Reset must quiesce the controller before shared ring state is cleared and firmware is restarted. Preserve the reset error rather than reinitializing DMA state on a running controller. Leave needs_reset set until every recovery stage completes so a failed boot or RTI handshake cannot fall back to ring creation on the next open. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/hci_bcm4377.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 5fe775766e781..51b3e8bf9c135 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -1320,6 +1320,8 @@ static int bcm4377_pci_reset(struct bcm4377_data *bcm4377) int ret; int irq; + /* Keep retrying a full reset if any recovery stage fails. */ + bcm4377->needs_reset = true; dev_info(&bcm4377->pdev->dev, "performing hardware reset recovery\n"); irq = pci_irq_vector(bcm4377->pdev, 0); @@ -1330,10 +1332,11 @@ static int bcm4377_pci_reset(struct bcm4377_data *bcm4377) bcm4377_disable_aspm(bcm4377); ret = pci_reset_function(bcm4377->pdev); - if (ret) - dev_warn(&bcm4377->pdev->dev, - "function level reset failed with %d; trying to continue anyway\n", - ret); + if (ret) { + dev_err(&bcm4377->pdev->dev, + "function level reset failed: %d\n", ret); + goto out_irq; + } msleep(100); From b0a02dcbfded7f1485db7f37694712892025a4d9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:57:15 +1100 Subject: [PATCH 0525/1579] cpufreq: apple-soc: enforce PMP fast-die limits through QoS The PMP publishes a per-cluster effort faster than the SMC reports die temperature. Sample the effort and enforce a maximum-frequency QoS request. The cpufreq core combines it with thermal cooling limits and reports reduced capacity to the scheduler. Keep the existing IPA policy. Use the OPP power table to map increasing effort to a monotonic power ceiling. Until the PMP is running, retain a ceiling of one third of the maximum OPP power. Deferrable timers sample busy CPUs without waking idle clusters; an hrtimer follows a reduced ceiling so idle capacity can recover. Drain timers and update work before removing the QoS request. Port Ryan Murray's original Linux-tested direct limiter. Apply the public IPA series' removal of the obsolete SMC policy hooks while preserving command-state transition verification. Keep the firmware- limit lifecycle separate from the generic policy cleanup so the public thermal-series removals merge without overlapping new hooks. No additional thermal policy or experimental smoothing is introduced. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 13 +- drivers/cpufreq/apple-soc-cpufreq.c | 548 +++++++++++++++++++++++++--- 2 files changed, 515 insertions(+), 46 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index a1368b8fa0733..2d4c286726d87 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1965,6 +1965,14 @@ L: linux-hwmon@vger.kernel.org S: Odd fixes F: drivers/hwmon/applesmc.c +APPLE T8140 PMP SUPPORT +M: Ryan Murray +L: asahi@lists.linux.dev +S: Maintained +F: drivers/pmdomain/apple/pmp-report.c +F: drivers/soc/apple/pmp.rs +F: include/linux/soc/apple/pmp-report.h + APPLE T8140 SOC AND J700 (MACBOOK NEO) SUPPORT M: Ryan Murray W: https://aurora-silicon.github.io @@ -1985,13 +1993,8 @@ F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ -F: drivers/pmdomain/apple/pmp-report.c F: drivers/soc/apple/aop.rs -F: drivers/soc/apple/pmp.rs -F: drivers/thermal/apple-smc-cpu-thermal.c F: drivers/tty/serial/apple_dockchannel* -F: include/linux/soc/apple/pmp-report.h -F: include/linux/soc/apple/smc-thermal.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 4075f26706b0b..55ca3de616e63 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -7,6 +7,18 @@ * Based on scpi-cpufreq.c */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include #include #include @@ -21,7 +33,6 @@ #include #include #include -#include #define APPLE_DVFS_CMD 0x20 #define APPLE_DVFS_CMD_BUSY BIT(31) @@ -61,6 +72,118 @@ #define APPLE_DVFS_TRANSITION_TIMEOUT 400 +/* + * Fast die-temperature limit + * + * The power management processor (PMP) of some SoCs runs a control loop on + * its own die-temperature sensors and publishes, for each CPU cluster, an + * 8-bit effort: 0 requests no limit, and the effort rises towards 255 as the + * die exceeds the firmware's target. The firmware refreshes the effort every + * few milliseconds, far faster than the SMC temperature sensors, so a cluster + * can run at full speed until the die hot spot reaches the firmware's target. + * + * The firmware requests the limit and the OS enforces it, as scmi-cpufreq + * does with SCMI performance limits: the limit is a FREQ_QOS_MAX request on + * the policy, so the cpufreq core applies it to the governor and reports it + * to the scheduler as cpufreq pressure. apple_fast_die_limit() maps an + * effort to that limit. + * + * The limit fails safe. The effort is only valid while the PMP reports that + * it is running, and the PMP may start late or never, for instance when its + * driver is not loaded or the bootloader did not prepare it. Until it + * reports that it is running, the cluster is held at a fallback limit: the + * fastest state whose power is at most a third of the fastest state's. If + * it has not reported running 30 s after the limit was set up, a warning + * says that the fast limit is unavailable; the fallback limit stays until + * the PMP runs. + * + * The effort has to be sampled, since no interrupt is known to signal a new + * one under Linux. While the limit is at the cluster's maximum, every CPU + * of the cluster has a pinned, deferrable timer that samples the effort once + * a tick, and the first CPU to run in a tick takes the sample. Deferrable + * timers do not wake idle CPUs, so only busy CPUs sample, and an idle + * cluster, which produces no heat, costs nothing. Once the limit is below + * the maximum, a per-cluster hrtimer samples every 5 ms whether the cluster + * is busy or not, until the limit is back at the maximum: an idle cluster + * would otherwise keep a stale limit, which the scheduler sees as reduced + * capacity and so keeps work away from the cluster. While the PMP is not + * running, the hrtimer only checks every 100 ms, and every second once the + * warning has been given. A nohz_full CPU can run a task with its tick + * stopped, and so without sampling, so a cluster with nohz_full CPUs + * samples from the hrtimer all the time. + * + * A sample reads the effort through the PMP report driver; only when the + * limit it maps to changes is a work item queued to update the QoS request, + * which must be done in process context. + */ +#define APPLE_FAST_DIE_EFFORT_MAX 255 +#define APPLE_FAST_DIE_POLL_MS 5 +#define APPLE_FAST_DIE_WAIT_MS 100 +#define APPLE_FAST_DIE_GAVE_UP_MS 1000 +#define APPLE_FAST_DIE_WARN_S 30 + +/** + * struct apple_fast_die - fast die-temperature limit of one cluster + * @policy: policy of the cluster + * @report: PMP report region, or NULL if the limit is misdescribed + * @lane: lane of the cluster in the fast die-temperature report + * @lock: serialises samples and protects the fields below it + * @limit: last limit derived from a sample, in kHz; 0 forces an update + * @polling: @poll is armed + * @waiting: the last sample found no running PMP + * @warned: the fast limit has been reported unavailable + * @per_cpu: the per-CPU deferrable timers sample while the limit is at @max + * @stopping: ->exit() is tearing the limit down; do not arm @poll again + * @max: frequency of the fastest state, in kHz + * @fallback: limit while the PMP is not running, in kHz + * @start: jiffies when the limit was set up + * @last_poll: jiffies of the last sample by a per-CPU timer + * @poll: non-deferrable sampling timer, armed while the limit is below @max + * @work: updates @req to @limit + * @req: FREQ_QOS_MAX request that enforces @limit + * @cpus: every CPU of the cluster, online or not + * @nr_states: number of frequency-table entries + * @power: power of each frequency-table entry, in microwatts from the OPP + * table, or the frequency when the OPPs do not describe their power + */ +struct apple_fast_die { + struct cpufreq_policy *policy; + struct device_node *report; + unsigned int lane; + spinlock_t lock; /* serialises samples; protects the fields below */ + unsigned int limit; + bool polling; + bool waiting; + bool warned; + bool per_cpu; + bool stopping; + unsigned int max; + unsigned int fallback; + unsigned long start; + unsigned long last_poll; + struct hrtimer poll; + struct work_struct work; + struct freq_qos_request req; + cpumask_var_t cpus; + unsigned int nr_states; + unsigned long power[] __counted_by(nr_states); +}; + +/** + * struct apple_fast_die_timer - per-CPU sampling timer + * @timer: pinned, deferrable timer that samples the effort + * @fd: limit of the cluster of this CPU, NULL when it has none; written with + * the CPU hotplug lock held, like the hotplug callbacks that read it + */ +struct apple_fast_die_timer { + struct timer_list timer; + struct apple_fast_die *fd; +}; + +static DEFINE_PER_CPU(struct apple_fast_die_timer, apple_fast_die_timers); +/* Dynamic CPU hotplug state of the per-CPU timers, 0 if there is none. */ +static int apple_fast_die_hp_state; + struct apple_soc_cpufreq_info { bool has_ps2; bool verify_transition; @@ -76,15 +199,16 @@ struct apple_soc_cpufreq_info { }; struct apple_cpu_priv { + struct apple_fast_die *fast_die; struct device *cpu_dev; void __iomem *reg_base; const struct apple_soc_cpufreq_info *info; bool transition_failed; unsigned int expected_pstate; - struct apple_smc_thermal_cpu *thermal; }; static struct cpufreq_driver apple_soc_cpufreq_driver; +static void apple_soc_cpufreq_exit_with_limits(struct cpufreq_policy *policy); static const struct apple_soc_cpufreq_info soc_s5l8960x_info = { .has_ps2 = false, @@ -249,7 +373,6 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, apple_soc_cpufreq_timeout(priv->info))) { if (priv->info->verify_transition) { priv->transition_failed = true; - apple_smc_thermal_cpu_fault(priv->thermal); dev_err(priv->cpu_dev, "DVFS busy timeout, command=%#llx; stopping requests\n", reg); } @@ -262,7 +385,6 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, /* Do not fight an unexpected retained policy owner. */ if (previous != priv->expected_pstate) { priv->transition_failed = true; - apple_smc_thermal_cpu_fault(priv->thermal); dev_err(priv->cpu_dev, "unexpected DVFS state %u (expected %u); stopping requests\n", previous, priv->expected_pstate); return -EIO; @@ -288,7 +410,6 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, 2, apple_soc_cpufreq_timeout(priv->info))) { /* Do not issue another request or guess a rollback after failure. */ priv->transition_failed = true; - apple_smc_thermal_cpu_fault(priv->thermal); dev_err(priv->cpu_dev, "DVFS completion timeout, command=%#llx; stopping requests\n", reg); return -ETIMEDOUT; @@ -310,7 +431,8 @@ static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, void __iomem **reg_base, - const struct apple_soc_cpufreq_info **info) + const struct apple_soc_cpufreq_info **info, + struct device_node **cluster) { struct of_phandle_args args; const struct of_device_id *match; @@ -331,16 +453,340 @@ static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, *info = match->data; *reg_base = of_iomap(args.np, 0); - of_node_put(args.np); - if (!*reg_base) + if (!*reg_base) { + of_node_put(args.np); return -ENOMEM; + } + *cluster = args.np; + + return 0; +} + +/* The fastest state whose power is at most @budget, or the slowest. */ +static unsigned int apple_fast_die_fit(const struct apple_fast_die *fd, + u64 budget) +{ + unsigned int i, idx = 0; + + for (i = 0; i < fd->nr_states; i++) + if (fd->power[i] <= budget) + idx = i; + + return fd->policy->freq_table[idx].frequency; +} + +/* + * Map an effort to a frequency limit, in kHz: the fastest state whose power + * fits a budget of (255 - effort) / 255 of the fastest state's power. Any + * mapping that lowers the limit monotonically as the effort rises would do, + * since the firmware's loop is closed on the die temperature and corrects + * the error; power is the natural axis for the output of a temperature + * controller. + */ +static unsigned int apple_fast_die_limit(const struct apple_fast_die *fd, + unsigned int effort) +{ + /* + * Multiply first: dividing the fastest state's power by 255 first + * rounds it down, which would limit an unlimited cluster one state low. + */ + return apple_fast_die_fit(fd, + div_u64((u64)fd->power[fd->nr_states - 1] * + (APPLE_FAST_DIE_EFFORT_MAX - effort), + APPLE_FAST_DIE_EFFORT_MAX)); +} + +static ktime_t apple_fast_die_interval(const struct apple_fast_die *fd) +{ + if (!fd->waiting) + return ms_to_ktime(APPLE_FAST_DIE_POLL_MS); + if (!fd->warned) + return ms_to_ktime(APPLE_FAST_DIE_WAIT_MS); + return ms_to_ktime(APPLE_FAST_DIE_GAVE_UP_MS); +} + +/* Arm the non-deferrable timer if the limit needs it. Call with @lock held. */ +static void apple_fast_die_arm(struct apple_fast_die *fd) +{ + if (fd->stopping || fd->polling || + (fd->per_cpu && fd->limit >= fd->max)) + return; + + fd->polling = true; + hrtimer_start(&fd->poll, apple_fast_die_interval(fd), + HRTIMER_MODE_REL_SOFT); +} + +/* Take a sample and queue an update if the limit changes. Call with @lock held. */ +static void apple_fast_die_sample(struct apple_fast_die *fd) +{ + struct apple_cpu_priv *priv = fd->policy->driver_data; + unsigned int limit; + u8 effort; + int ret; + + ret = apple_pmp_report_fast_die_effort(fd->report, fd->lane, &effort); + if (!ret) { + limit = apple_fast_die_limit(fd, effort); + fd->waiting = false; + } else { + limit = fd->fallback; + fd->waiting = true; + if (!fd->warned && + time_after(jiffies, fd->start + APPLE_FAST_DIE_WARN_S * HZ)) { + fd->warned = true; + dev_warn(priv->cpu_dev, + "PMP not running after %d s (%d): fast die-temperature limit unavailable, CPUs %*pbl stay limited to %u kHz\n", + APPLE_FAST_DIE_WARN_S, ret, + cpumask_pr_args(fd->cpus), limit); + } + } + + if (limit != fd->limit) { + WRITE_ONCE(fd->limit, limit); + schedule_work(&fd->work); + } +} + +static void apple_fast_die_work(struct work_struct *work) +{ + struct apple_fast_die *fd = container_of(work, struct apple_fast_die, work); + struct apple_cpu_priv *priv = fd->policy->driver_data; + unsigned int limit = READ_ONCE(fd->limit); + int ret; + + ret = freq_qos_update_request(&fd->req, limit); + if (ret >= 0) + return; + + dev_warn_ratelimited(priv->cpu_dev, + "failed to update the fast die-temperature limit: %d\n", ret); + /* Make the next sample queue the update again, and make sure it comes. */ + spin_lock_bh(&fd->lock); + if (fd->limit == limit) { + fd->limit = 0; + apple_fast_die_arm(fd); + } + spin_unlock_bh(&fd->lock); +} + +static enum hrtimer_restart apple_fast_die_poll(struct hrtimer *t) +{ + struct apple_fast_die *fd = container_of(t, struct apple_fast_die, poll); + enum hrtimer_restart restart = HRTIMER_NORESTART; + + spin_lock(&fd->lock); + apple_fast_die_sample(fd); + if (!fd->stopping && (!fd->per_cpu || fd->limit < fd->max)) { + hrtimer_forward_now(t, apple_fast_die_interval(fd)); + restart = HRTIMER_RESTART; + } else { + fd->polling = false; + } + spin_unlock(&fd->lock); + + return restart; +} + +static void apple_fast_die_tick(struct timer_list *t) +{ + struct apple_fast_die_timer *ft = timer_container_of(ft, t, timer); + struct apple_fast_die *fd = ft->fd; + unsigned long now = jiffies; + + /* + * The first busy CPU of the cluster to run in a tick takes the sample, + * unless the non-deferrable timer is sampling. + */ + if (xchg(&fd->last_poll, now) != now && spin_trylock(&fd->lock)) { + if (!fd->polling) { + apple_fast_die_sample(fd); + apple_fast_die_arm(fd); + } + spin_unlock(&fd->lock); + } + mod_timer(t, now + 1); +} + +static int apple_fast_die_cpu_online(unsigned int cpu) +{ + struct apple_fast_die_timer *ft = per_cpu_ptr(&apple_fast_die_timers, cpu); + + if (!ft->fd) + return 0; + + /* Wait for a running sample, which rearms the timer, before arming it. */ + timer_delete_sync(&ft->timer); + ft->timer.expires = jiffies + 1; + add_timer_on(&ft->timer, cpu); return 0; } +static int apple_fast_die_cpu_offline(unsigned int cpu) +{ + struct apple_fast_die_timer *ft = per_cpu_ptr(&apple_fast_die_timers, cpu); + + if (ft->fd) + timer_delete_sync(&ft->timer); + + return 0; +} + +/* + * Called at the end of ->init(), with the CPU hotplug lock held. The core + * fills policy->related_cpus only after ->init() returns, and may call ->exit() + * before it does, so the limit keeps its own copy of the cluster's CPUs. + */ +static int apple_fast_die_init(struct cpufreq_policy *policy, + struct device_node *cluster) +{ + struct apple_cpu_priv *priv = policy->driver_data; + struct of_phandle_args args; + struct apple_fast_die *fd; + unsigned int cpu, i, n = 0; + bool have_power = true; + int parsed, ret; + + parsed = of_parse_phandle_with_fixed_args(cluster, "apple,fast-die-effort", + 1, 0, &args); + if (parsed == -ENOENT) + return 0; + + while (policy->freq_table[n].frequency != CPUFREQ_TABLE_END) + n++; + + fd = kzalloc_flex(*fd, power, n); + if (!fd) { + ret = -ENOMEM; + goto err_put; + } + fd->nr_states = n; + if (!zalloc_cpumask_var(&fd->cpus, GFP_KERNEL)) { + ret = -ENOMEM; + goto err_free; + } + + for (i = 0; i < n; i++) { + unsigned long rate = policy->freq_table[i].frequency * 1000UL + 999; + struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(priv->cpu_dev, &rate); + + if (IS_ERR(opp)) { + ret = PTR_ERR(opp); + goto err_free; + } + fd->power[i] = dev_pm_opp_get_power(opp); + dev_pm_opp_put(opp); + if (!fd->power[i]) + have_power = false; + } + /* Without an energy model, frequency stands in for power. */ + if (!have_power) + for (i = 0; i < n; i++) + fd->power[i] = policy->freq_table[i].frequency; + + fd->policy = policy; + spin_lock_init(&fd->lock); + INIT_WORK(&fd->work, apple_fast_die_work); + hrtimer_setup(&fd->poll, apple_fast_die_poll, CLOCK_MONOTONIC, + HRTIMER_MODE_REL_SOFT); + cpumask_copy(fd->cpus, policy->cpus); + fd->per_cpu = apple_fast_die_hp_state > 0; + for_each_cpu(cpu, fd->cpus) + if (tick_nohz_full_cpu(cpu)) + fd->per_cpu = false; + fd->max = policy->freq_table[n - 1].frequency; + fd->fallback = apple_fast_die_fit(fd, div_u64(fd->power[n - 1], 3)); + fd->start = jiffies; + + if (parsed) { + /* The limit is meant to be there, so keep the fallback. */ + dev_warn(priv->cpu_dev, + "invalid apple,fast-die-effort (%d): CPUs %*pbl stay limited to %u kHz\n", + parsed, cpumask_pr_args(fd->cpus), fd->fallback); + fd->warned = true; + } else { + fd->report = args.np; + fd->lane = args.args[0]; + } + + /* Hold the fallback limit until a sample finds the PMP running. */ + fd->limit = fd->fallback; + ret = freq_qos_add_request(&policy->constraints, &fd->req, FREQ_QOS_MAX, + fd->limit); + if (ret < 0) + goto err_free; + + priv->fast_die = fd; + + if (fd->per_cpu) { + for_each_cpu(cpu, fd->cpus) { + struct apple_fast_die_timer *ft = + per_cpu_ptr(&apple_fast_die_timers, cpu); + + timer_setup(&ft->timer, apple_fast_die_tick, + TIMER_DEFERRABLE | TIMER_PINNED); + ft->fd = fd; + if (cpu_online(cpu)) + apple_fast_die_cpu_online(cpu); + } + } + + spin_lock_bh(&fd->lock); + apple_fast_die_arm(fd); + spin_unlock_bh(&fd->lock); + + dev_dbg(priv->cpu_dev, "fast die-temperature limit set up, fallback %u kHz\n", + fd->fallback); + + return 0; + +err_free: + free_cpumask_var(fd->cpus); + kfree(fd); +err_put: + if (!parsed) + of_node_put(args.np); + return ret; +} + +static void apple_fast_die_exit(struct apple_cpu_priv *priv) +{ + struct apple_fast_die *fd = priv->fast_die; + unsigned int cpu; + + if (!fd) + return; + + /* + * Keep the update work from arming the hrtimer again, stop the per-CPU + * timers, which can arm it too, then the hrtimer, and then drain the + * update any of them may have queued. + */ + spin_lock_bh(&fd->lock); + fd->stopping = true; + spin_unlock_bh(&fd->lock); + if (fd->per_cpu) { + for_each_cpu(cpu, fd->cpus) { + struct apple_fast_die_timer *ft = + per_cpu_ptr(&apple_fast_die_timers, cpu); + + timer_shutdown_sync(&ft->timer); + ft->fd = NULL; + } + } + hrtimer_cancel(&fd->poll); + cancel_work_sync(&fd->work); + freq_qos_remove_request(&fd->req); + + priv->fast_die = NULL; + of_node_put(fd->report); + free_cpumask_var(fd->cpus); + kfree(fd); +} + static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) { - struct cpufreq_frequency_table *p; int ret, i; unsigned int transition_latency; void __iomem *reg_base; @@ -348,6 +794,7 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) struct apple_cpu_priv *priv; const struct apple_soc_cpufreq_info *info; struct cpufreq_frequency_table *freq_table; + struct device_node *cluster; cpu_dev = get_cpu_device(policy->cpu); if (!cpu_dev) { @@ -359,7 +806,7 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) if (!priv) return -ENOMEM; - ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info); + ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info, &cluster); if (ret) { dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret); goto out_free_priv; @@ -430,6 +877,13 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) policy->driver_data = priv; policy->freq_table = freq_table; + ret = apple_fast_die_init(policy, cluster); + if (ret) { + dev_err(cpu_dev, "failed to set up the fast die-temperature limit: %d\n", ret); + goto out_clear_policy; + } + of_node_put(cluster); + transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); if (!transition_latency) { /* Conservative transaction bound, not a measured transition time. */ @@ -442,35 +896,19 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) policy->dvfs_possible_from_any_cpu = true; policy->fast_switch_possible = !info->verify_transition; policy->suspend_freq = freq_table[0].frequency; - if (info->needs_thermal_policy) { - struct device_node *hwmon; - bool higher = false; - bool thermal; - - /* Higher states require the SMC thermal policy. */ - cpufreq_for_each_valid_entry(p, policy->freq_table) - if (p->driver_data > info->max_unmanaged_pstate) - higher = true; - hwmon = of_find_compatible_node(NULL, NULL, "apple,smc-hwmon"); - thermal = hwmon && of_property_read_bool(hwmon, "apple,cpu-thermal-policy"); - of_node_put(hwmon); - if (higher || thermal) { - priv->thermal = apple_smc_thermal_cpu_add(policy); - if (IS_ERR(priv->thermal)) { - ret = PTR_ERR(priv->thermal); - goto out_free_cpufreq_table; - } - } - } return 0; +out_clear_policy: + policy->driver_data = NULL; + policy->freq_table = NULL; out_free_cpufreq_table: dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); out_free_table: dev_pm_opp_of_cpumask_remove_table(policy->cpus); out_iounmap: iounmap(reg_base); + of_node_put(cluster); out_free_priv: kfree(priv); return ret; @@ -480,7 +918,6 @@ static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) { struct apple_cpu_priv *priv = policy->driver_data; - apple_smc_thermal_cpu_remove(priv->thermal); dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); /* * The core clears the last CPU from policy->cpus before calling @@ -491,13 +928,6 @@ static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) kfree(priv); } -static void apple_soc_cpufreq_ready(struct cpufreq_policy *policy) -{ - struct apple_cpu_priv *priv = policy->driver_data; - - apple_smc_thermal_cpu_ready(priv->thermal); -} - static struct cpufreq_driver apple_soc_cpufreq_driver = { .name = "apple-cpufreq", .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | @@ -505,27 +935,63 @@ static struct cpufreq_driver apple_soc_cpufreq_driver = { .verify = cpufreq_generic_frequency_table_verify, .get = apple_soc_cpufreq_get_rate, .init = apple_soc_cpufreq_init, - .exit = apple_soc_cpufreq_exit, + .exit = apple_soc_cpufreq_exit_with_limits, .target_index = apple_soc_cpufreq_set_target, .fast_switch = apple_soc_cpufreq_fast_switch, .register_em = cpufreq_register_em_with_opp, .set_boost = cpufreq_boost_set_sw, .suspend = cpufreq_generic_suspend, - .ready = apple_soc_cpufreq_ready, }; +/* Drain firmware-limit updates before releasing the policy's OPP table. */ +static void apple_soc_cpufreq_exit_with_limits(struct cpufreq_policy *policy) +{ + apple_fast_die_exit(policy->driver_data); + apple_soc_cpufreq_exit(policy); +} + static int __init apple_soc_cpufreq_module_init(void) { + struct device_node *np; + int ret; + if (!of_machine_is_compatible("apple,arm-platform")) return -ENODEV; - return cpufreq_register_driver(&apple_soc_cpufreq_driver); + /* + * Only SoCs with a fast die-temperature limit need the per-CPU timers + * to follow CPU hotplug. Either order against the cpufreq core's own + * hotplug state is fine: ->init() arms the timers of the CPUs that are + * online, and ->exit() stops them all. Without the state the limit + * still works, sampling from the hrtimer all the time. + */ + np = of_find_node_with_property(NULL, "apple,fast-die-effort"); + if (np) { + of_node_put(np); + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "cpufreq/apple-soc:online", + apple_fast_die_cpu_online, + apple_fast_die_cpu_offline); + if (ret < 0) + pr_warn("apple-cpufreq: no CPU hotplug state (%d), fast die-temperature limit always polls\n", + ret); + else + apple_fast_die_hp_state = ret; + } + + ret = cpufreq_register_driver(&apple_soc_cpufreq_driver); + if (ret && apple_fast_die_hp_state > 0) + cpuhp_remove_state_nocalls(apple_fast_die_hp_state); + + return ret; } module_init(apple_soc_cpufreq_module_init); static void __exit apple_soc_cpufreq_module_exit(void) { cpufreq_unregister_driver(&apple_soc_cpufreq_driver); + if (apple_fast_die_hp_state > 0) + cpuhp_remove_state_nocalls(apple_fast_die_hp_state); } module_exit(apple_soc_cpufreq_module_exit); From c484ec442525743679b4e36895e62e5ec5658f6f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:30:44 +1100 Subject: [PATCH 0526/1579] HID: magicmouse: serialize haptic effect updates and retain C1FE clicks Publish complete effect parameters under a per-effect spinlock only after the mode switch succeeds. Workers take one coherent snapshot and erase drains the affected job before clearing it. Failed updates leave the previous effect intact. Serialize actuator lookup, mode transactions and cached state with the existing haptic auto-trigger mutex. Drain workers before taking that mutex during erase and destroy, and retake host mode on playback after resume invalidates the mode cache. C1FE reports have no pressure, so they cannot emulate buttons when firmware click handling is disabled by host-controlled haptics. Keep those pads in firmware-controlled mode and omit force feedback and pressure-pad capabilities. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/hid-magicmouse.c | 140 ++++++++++++++++++++++++----------- 1 file changed, 97 insertions(+), 43 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 8eb69d187ce87..aa0e6cee6d8fd 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -247,6 +248,9 @@ struct magicmouse_input_ops { struct effect_job { struct work_struct work; + struct magicmouse_sc *msc; + /* Protects effect parameters shared with the worker. */ + spinlock_t lock; struct hid_device *taptic_hdev; u16 effect_type; u8 strength; @@ -1264,7 +1268,8 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, struct magicmouse_sc *msc = hid_get_drvdata(hdev); __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); - __set_bit(INPUT_PROP_PRESSUREPAD, input->propbit); + if (!msc->mtp_c1fe) + __set_bit(INPUT_PROP_PRESSUREPAD, input->propbit); __clear_bit(BTN_0, input->keybit); __clear_bit(BTN_RIGHT, input->keybit); __clear_bit(BTN_MIDDLE, input->keybit); @@ -1272,7 +1277,9 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, __clear_bit(REL_X, input->relbit); __clear_bit(REL_Y, input->relbit); - mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK | INPUT_MT_TOTAL_FORCE; + mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK; + if (!msc->mtp_c1fe) + mt_flags |= INPUT_MT_TOTAL_FORCE; /* * The C1FE report carries positions and the button only; advertise @@ -1445,6 +1452,7 @@ static int magicmouse_switch_mode(struct input_dev *trackpad_idev, int mode) static u8 magicmouse_haptic_softness(struct hid_device *hdev, bool deep); static u8 magicmouse_haptic_strength(struct hid_device *hdev, bool deep); static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc); +static void magicmouse_haptic_release_mode_locked(struct magicmouse_sc *msc); /* * Play the waveform @pulses times, @haptic_pulse_gap_ms apart. A failure here @@ -1454,7 +1462,7 @@ static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc); static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, bool deep, u8 pulses) { - struct hid_device *actuator = magicmouse_get_actuator(msc); + struct hid_device *actuator; u8 strength, softness, gap; unsigned int i; int ret; @@ -1462,6 +1470,9 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, if (!pulses) return; + guard(mutex)(&msc->haptics->auto_trigger_mutex); + actuator = magicmouse_get_actuator(msc); + if (!actuator) { hid_warn(msc->hdev, "no actuator for host click feedback; using the firmware's\n"); @@ -1469,7 +1480,7 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, return; } - if (msc->haptics->mode != HID_HAPTIC_MODE_HOST) { + if (READ_ONCE(msc->haptics->mode) != HID_HAPTIC_MODE_HOST) { ret = apple_taptic_switch_modes(actuator, HID_HAPTIC_MODE_HOST); if (ret) { hid_warn(msc->hdev, @@ -1478,7 +1489,7 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, haptic_press = false; return; } - msc->haptics->mode = HID_HAPTIC_MODE_HOST; + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_HOST); hid_info(msc->hdev, "host-driven click feedback active\n"); } @@ -1494,7 +1505,7 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, "click waveform failed (%d); returning click feedback to the firmware\n", ret); haptic_press = false; - magicmouse_haptic_release_mode(msc); + magicmouse_haptic_release_mode_locked(msc); return; } hid_dbg(msc->hdev, "played %s pulse %u/%u at strength 0x%02x\n", @@ -1510,24 +1521,30 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, * Hand click feedback back to the actuator's firmware, unless userspace still * has effects uploaded and so still wants host-controlled mode. */ -static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc) +static void magicmouse_haptic_release_mode_locked(struct magicmouse_sc *msc) { struct hid_device *actuator; unsigned int i; - if (msc->haptics->mode != HID_HAPTIC_MODE_HOST) + if (READ_ONCE(msc->haptics->mode) != HID_HAPTIC_MODE_HOST) return; if (msc->haptic_effects) { for (i = 0; i < FF_MAX_EFFECTS; i++) - if (msc->haptic_effects[i].effect_type) + if (READ_ONCE(msc->haptic_effects[i].effect_type)) return; } actuator = magicmouse_get_actuator(msc); if (actuator && !apple_taptic_switch_modes(actuator, HID_HAPTIC_MODE_DEVICE)) - msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_DEVICE); +} + +static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc) +{ + guard(mutex)(&msc->haptics->auto_trigger_mutex); + magicmouse_haptic_release_mode_locked(msc); } static void magicmouse_press_worker(struct work_struct *ws) @@ -1615,7 +1632,7 @@ static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, if (!msc->haptics || !msc->haptics->wq) return firmware_button; - host_mode = msc->haptics->mode == HID_HAPTIC_MODE_HOST; + host_mode = READ_ONCE(msc->haptics->mode) == HID_HAPTIC_MODE_HOST; if (!haptic_press && !host_mode) { /* The firmware owns the actuator and reports the button. */ @@ -1689,7 +1706,7 @@ static void magicmouse_haptic_forget_mode(struct magicmouse_sc *msc) if (!msc->haptics) return; - msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_DEVICE); msc->haptic_button_down = false; if (msc->haptic_deep_down) { msc->haptic_deep_down = false; @@ -1901,6 +1918,10 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); struct hid_haptic_device *haptics = msc->haptics; struct hid_device *actuator; + struct effect_job *job = &msc->haptic_effects[effect->id]; + unsigned long flags; + u8 strength, softness; + bool deep; int ret; switch (effect->u.haptic.hid_usage) { @@ -1912,30 +1933,31 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, return -EINVAL; } + guard(mutex)(&haptics->auto_trigger_mutex); actuator = magicmouse_get_actuator(msc); if (!actuator) return -ENODEV; - msc->haptic_effects[effect->id].taptic_hdev = actuator; - msc->haptic_effects[effect->id].effect_type = (effect->u.haptic.hid_usage) & HID_USAGE; - msc->haptic_effects[effect->id].strength = - (min(100, (effect->u.haptic.intensity)) * - magicmouse_haptic_strength(trackpad_hdev, - magicmouse_effect_is_deep(effect))) / 100; - msc->haptic_effects[effect->id].softness = - magicmouse_haptic_softness(trackpad_hdev, - magicmouse_effect_is_deep(effect)); + deep = magicmouse_effect_is_deep(effect); + strength = min_t(u16, 100, effect->u.haptic.intensity) * + magicmouse_haptic_strength(trackpad_hdev, deep) / 100; + softness = magicmouse_haptic_softness(trackpad_hdev, deep); /* The actuator may have been rebound or reset since the last upload. */ ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_HOST); if (ret) { dev_err(&msc->hdev->dev, "failed to switch to host-controlled mode: %d\n", ret); - msc->haptic_effects[effect->id].effect_type = 0; return ret; } - haptics->mode = HID_HAPTIC_MODE_HOST; + spin_lock_irqsave(&job->lock, flags); + job->taptic_hdev = actuator; + job->strength = strength; + job->softness = softness; + WRITE_ONCE(job->effect_type, effect->u.haptic.hid_usage & HID_USAGE); + spin_unlock_irqrestore(&job->lock, flags); + WRITE_ONCE(haptics->mode, HID_HAPTIC_MODE_HOST); hid_dbg(trackpad_hdev, "uploaded haptic effect %d\n", effect->id); return 0; @@ -1944,11 +1966,32 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, static void haptic_playback_worker(struct work_struct *ws) { struct effect_job *job = container_of(ws, struct effect_job, work); + struct hid_device *actuator; + unsigned long flags; + u16 type; + u8 strength, softness; + int ret; + + spin_lock_irqsave(&job->lock, flags); + actuator = job->taptic_hdev; + type = READ_ONCE(job->effect_type); + strength = job->strength; + softness = job->softness; + spin_unlock_irqrestore(&job->lock, flags); - if (job->effect_type && - apple_taptic_send(job->taptic_hdev, job->effect_type, - job->strength, job->softness)) - hid_warn(job->taptic_hdev, "cannot play the haptic effect\n"); + if (!type) + return; + guard(mutex)(&job->msc->haptics->auto_trigger_mutex); + if (READ_ONCE(job->msc->haptics->mode) != HID_HAPTIC_MODE_HOST) { + ret = apple_taptic_switch_modes(actuator, HID_HAPTIC_MODE_HOST); + if (ret) { + hid_warn(actuator, "cannot take host-controlled mode (%d)\n", ret); + return; + } + WRITE_ONCE(job->msc->haptics->mode, HID_HAPTIC_MODE_HOST); + } + if (apple_taptic_send(actuator, type, strength, softness)) + hid_warn(actuator, "cannot play the haptic effect\n"); } static int apple_taptic_playback(struct input_dev *trackpad_idev, int effect_id, int value) @@ -1968,16 +2011,19 @@ static int apple_taptic_erase(struct input_dev *trackpad_idev, int effect_id) struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); int i, ret = 0; - msc->haptic_effects[effect_id].effect_type = 0; + cancel_work_sync(&msc->haptic_effects[effect_id].work); + WRITE_ONCE(msc->haptic_effects[effect_id].effect_type, 0); for (i = 0; i < FF_MAX_EFFECTS; i++) { - if (msc->haptic_effects[i].effect_type != 0) + if (READ_ONCE(msc->haptic_effects[i].effect_type) != 0) return 0; } + /* Drain workers before taking the mutex they need. */ + flush_workqueue(msc->haptics->wq); + guard(mutex)(&msc->haptics->auto_trigger_mutex); /* Return to device-controlled mode if there are no effects left */ - if (msc->haptics->mode == HID_HAPTIC_MODE_HOST) { - flush_workqueue(msc->haptics->wq); + if (READ_ONCE(msc->haptics->mode) == HID_HAPTIC_MODE_HOST) { ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_DEVICE); if (ret) { @@ -1987,7 +2033,7 @@ static int apple_taptic_erase(struct input_dev *trackpad_idev, int effect_id) return ret; } - msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_DEVICE); } return 0; @@ -2002,15 +2048,18 @@ static void apple_taptic_destroy(struct ff_device *ff) /* ff->private is devm-allocated; do not let ff-core kfree() it. */ ff->private = NULL; - if (msc->haptics && msc->haptics->mode == HID_HAPTIC_MODE_HOST) { - flush_workqueue(msc->haptics->wq); - ret = magicmouse_switch_mode(msc->haptics->input_dev, HID_HAPTIC_MODE_DEVICE); - if (ret) - hid_warn(msc->hdev, - "failed to switch back to device-controlled mode: %d\n", - ret); - else - msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + flush_workqueue(haptic_dev->wq); + { + guard(mutex)(&haptic_dev->auto_trigger_mutex); + if (msc->haptics && READ_ONCE(msc->haptics->mode) == HID_HAPTIC_MODE_HOST) { + ret = magicmouse_switch_mode(msc->haptics->input_dev, HID_HAPTIC_MODE_DEVICE); + if (ret) + hid_warn(msc->hdev, + "failed to switch back to device-controlled mode: %d\n", + ret); + else + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_DEVICE); + } } destroy_workqueue(haptic_dev->wq); @@ -2069,6 +2118,7 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, haptic_dev->hdev = trackpad_hdev; haptic_dev->input_dev = msc->input; + mutex_init(&haptic_dev->auto_trigger_mutex); msc->haptic_effects = kzalloc_objs(struct effect_job, FF_MAX_EFFECTS); if (!msc->haptic_effects) { @@ -2084,8 +2134,11 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, return -ENOMEM; } - for (i = 0; i < FF_MAX_EFFECTS; i++) + for (i = 0; i < FF_MAX_EFFECTS; i++) { + msc->haptic_effects[i].msc = msc; + spin_lock_init(&msc->haptic_effects[i].lock); INIT_WORK(&msc->haptic_effects[i].work, haptic_playback_worker); + } INIT_WORK(&msc->haptic_press_work, magicmouse_press_worker); INIT_WORK(&msc->haptic_release_work, magicmouse_release_worker); @@ -2137,7 +2190,8 @@ static int magicmouse_init_haptics(struct magicmouse_sc *msc, struct hid_device struct hid_haptic_device *haptics; int ret; - if (msc->haptics) + /* C1FE has no pressure data to emulate clicks in host mode. */ + if (msc->haptics || msc->mtp_c1fe) return 0; /* From a1f9aabbe6a9771a4dbf3259bc808713f0daac46 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:00:11 +1100 Subject: [PATCH 0527/1579] dt-bindings: apple: describe PMP fast-die reporting on T8140 Document the existing PMP firmware driver resources and the report aperture consumed by the fast-die limiter. Define a report phandle and lane index on the cluster DVFS controller, using the measured P lane and ADT E lane ordering. Firmware calibration properties are populated by the bootloader from the Apple device tree. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../cpufreq/apple,cluster-cpufreq.yaml | 12 +++ .../power/apple,t8140-pmp-report.yaml | 46 +++++++++ .../bindings/soc/apple/apple,pmp.yaml | 94 +++++++++++++++++++ MAINTAINERS | 3 + 4 files changed, 155 insertions(+) create mode 100644 Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml diff --git a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml index d6618e159cf72..a27f4314e0387 100644 --- a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +++ b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml @@ -45,6 +45,18 @@ properties: reg: maxItems: 1 + apple,fast-die-effort: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: | + PMP report region and cluster effort lane. Increasing firmware effort + imposes a maximum-frequency QoS constraint alongside thermal cooling. + If the PMP is unavailable, cpufreq retains its fallback ceiling. + items: + - items: + - description: PMP report region + - description: Cluster lane (0 for E, 1 for P on T8140) + enum: [0, 1] + '#performance-domain-cells': const: 0 diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml new file mode 100644 index 0000000000000..9b515685eef00 --- /dev/null +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/power/apple,t8140-pmp-report.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 PMP report aperture + +maintainers: + - Ryan Murray + +description: | + The PMP report aperture contains a read window and an update window. + The PMP running bit qualifies the per-cluster fast-die effort. Lane 1 + tracks P-cluster load in Linux measurements. Lane 0 is assigned to the + E cluster from the Apple device tree order and has not been observed + changing. Only documented report entries may be read; the aperture is + not a contiguous readable array of registers. + +properties: + compatible: + const: apple,t8140-pmp-v2-report + + reg: + maxItems: 1 + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + pmp-report@3003c0000 { + compatible = "apple,t8140-pmp-v2-report"; + reg = <3 0x003c0000 0 0x14000>; + }; + }; diff --git a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml new file mode 100644 index 0000000000000..9784c62fba89e --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,pmp.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple power management co-processor + +maintainers: + - Ryan Murray + +description: | + The PMP runs firmware loaded by the bootloader. Linux starts it through + RTKit and supplies calibration copied from the Apple device tree. The + bootloader must initialize the PMP DART address protection filter before + Linux starts the firmware. Firmware-owned PMC power state must be retained. + +properties: + compatible: + oneOf: + - const: apple,t6000-pmp-v2 + - items: + - const: apple,t8140-pmp-v2 + - const: apple,t6000-pmp-v2 + + reg: + items: + - description: Firmware SRAM + - description: ASC control registers + + reg-names: + items: + - const: pmp + - const: asc + + mboxes: + maxItems: 1 + + mbox-names: + const: mbox + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + apple,board-id: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Board identifier copied by the bootloader. + + apple,dram-vendor-id: + $ref: /schemas/types.yaml#/definitions/uint32 + description: DRAM vendor identifier copied by the bootloader. + + apple,dram-capacity: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Firmware DRAM capacity value copied by the bootloader. + + apple,pio-ranges: + $ref: /schemas/types.yaml#/definitions/uint64-array + minItems: 2 + description: Host physical address and size pairs mapped into firmware PIO space. + +patternProperties: + '^apple,tunable-': + $ref: /schemas/types.yaml#/definitions/uint8-array + description: Opaque firmware calibration copied from the Apple device tree. + +required: + - compatible + - reg + - reg-names + - mboxes + - mbox-names + - iommus + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + pmp@300500000 { + compatible = "apple,t8140-pmp-v2", "apple,t6000-pmp-v2"; + reg = <3 0x00500000 0 0xa0000>, <3 0x00e00000 0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index 2d4c286726d87..2d587c8002676 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1969,6 +1969,9 @@ APPLE T8140 PMP SUPPORT M: Ryan Murray L: asahi@lists.linux.dev S: Maintained +F: Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +F: Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml F: drivers/pmdomain/apple/pmp-report.c F: drivers/soc/apple/pmp.rs F: include/linux/soc/apple/pmp-report.h From e6864ee9e567efc69a73910035a09b6093590084 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:58:01 +1100 Subject: [PATCH 0528/1579] drm/apple: expose and enforce the routed Type-C CRTC Dual-stream machines kept advertising every candidate CRTC after the fabric assigned a connector to one pipeline. An explicit atomic commit could select the other advertised CRTC and send its framebuffer to a different physical display. Publish the active pipeline mask for dual-stream ports as well as single-stream ports, including routes adopted during DRM binding. Restore the candidate mask only while the connector is disconnected and unrouted. Validate the requested CRTC against the actual connector owner in atomic encoder selection, so stale userspace topology fails rather than driving another output. The route planner still uses its separate candidate mask; clients can discover the valid assignment by re-reading encoder topology on hotplug. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 15 +++++++++----- drivers/gpu/drm/apple/dcp.c | 34 +++++++++++-------------------- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index d6a544dd1bac2..ac757ffa6e821 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -279,16 +279,21 @@ apple_connector_atomic_best_encoder(struct drm_connector *conn, struct drm_atomic_state *state) { struct apple_connector *apple_connector = to_apple_connector(conn); + struct drm_connector_state *conn_state; struct drm_encoder *encoder; /* - * A Type-C port has one encoder, whose possible_crtcs either already - * names the pipeline the fabric routed the port to or, on dual-stream - * machines, the fixed set the fabric only routes within, so it is the - * only answer there is. + * Type-C tunnels are routed before the modeset and cannot move here. + * Reject stale topology and explicit assignments to another pipeline. */ - if (apple_connector->port_encoder) + if (apple_connector->port_encoder) { + conn_state = drm_atomic_get_new_connector_state(state, conn); + if (!conn_state || !conn_state->crtc || + READ_ONCE(apple_connector->dcp) != + to_apple_crtc(conn_state->crtc)->dcp) + return NULL; return apple_connector->port_encoder; + } drm_connector_for_each_possible_encoder(conn, encoder) return encoder; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4c1a6f85eb4d6..c3f32133be425 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -278,12 +278,10 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * Offering it the choice makes it pair the port with a pipeline * holding a different monitor's mode list, and the modeset is * rejected with no way for it to recover. The hotplug that - * follows makes it re-read this. Dual-stream machines keep - * fixed possible_crtcs instead: compositors that read them once - * (e.g. aquamarine/Hyprland) never see the narrowing. + * follows publishes the actual binding on dual-stream machines + * too. Clients must re-read encoder topology after a hotplug. */ - if (connector->port_encoder && dcp->crtc && - !dcp_typec_dual_stream()) + if (connector->port_encoder && dcp->crtc) connector->port_encoder->possible_crtcs = drm_crtc_mask(&dcp->crtc->base); } @@ -368,7 +366,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) WRITE_ONCE(connector->dcp, NULL); /* Unrouted: the port could go to any of its pipelines again. */ - if (connector->port_encoder && !dcp_typec_dual_stream()) + if (connector->port_encoder) connector->port_encoder->possible_crtcs = connector->candidate_crtcs; } @@ -651,9 +649,7 @@ dcp_typec_lowest_free(struct apple_dcp_typec_port *port) if (!dcp_typec_route_available(candidate)) continue; /* - * Lowest free CRTC index first: on dual-stream machines that - * is what a compositor picks from the port's fixed - * possible_crtcs. + * Prefer the lowest free CRTC index for a stable allocation order. */ score = dcp_typec_route_score(candidate); if (score < best_score) { @@ -693,17 +689,12 @@ static void dcp_typec_route_waiting(void) } /* - * Keep the Type-C routes where a compositor starting now expects them. - * Dual-stream machines keep possible_crtcs fixed, and compositors read - * them once and pair connectors with CRTCs themselves: aquamarine - * (Hyprland) walks the CRTCs in index order and gives each to the first - * connected connector, in connector order, that can use it. A connector - * paired with a pipeline other than the one routed to its display has its - * modes checked against the other display's list, so its modesets fail. - * The Type-C and Thunderbolt events that route the ports come in no - * particular order, so until a compositor owns the display (see - * dcp_typec_frozen()) every route change re-runs that pairing from - * scratch (dcp_typec_plan()) and follows it. + * Prefer connector order when placing Type-C streams before a DRM master + * starts. This is an allocation policy, not a promise about the CRTC that + * userspace must choose: each connected encoder advertises only its actual + * route, and atomic encoder selection checks that ownership. Route changes + * publish the new mask through hotplug; the plan uses candidate_crtcs so + * narrowing the advertised mask does not remove hardware routing choices. * * Only direct DP-alt routes move: each goes to exactly its planned * pipeline. A Thunderbolt tunnel never moves once set up, so where one @@ -1448,8 +1439,7 @@ void dcp_typec_port_set_connector(unsigned int idx, bool secondary, if (owner) { struct apple_dcp *dcp = owner->dcp; - if (dcp->crtc && connector->port_encoder && - !dcp_typec_dual_stream()) + if (dcp->crtc && connector->port_encoder) connector->port_encoder->possible_crtcs = drm_crtc_mask(&dcp->crtc->base); From aef46b5a4ad4fe52a9f59ef9b8215ab653127a68 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:05:43 +1100 Subject: [PATCH 0529/1579] MAINTAINERS: add a dedicated Apple QSPIMC entry Track the QSPIMC driver and binding together. Keep the existing Apple machine binding list independent of the new SPI controller entry. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 7f3cc384883d1..1e9a28969bee2 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2581,7 +2581,6 @@ F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml F: Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml F: Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml -F: Documentation/devicetree/bindings/spi/apple,qspi.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml F: Documentation/devicetree/bindings/spmi/apple,spmi.yaml F: Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml @@ -2617,7 +2616,6 @@ F: drivers/pwm/pwm-apple.c F: drivers/reset/reset-apple-cio.c F: drivers/rtc/rtc-macsmc.c F: drivers/soc/apple/* -F: drivers/spi/spi-apple-qspi.c F: drivers/spi/spi-apple.c F: drivers/spmi/spmi-apple-controller.c F: drivers/thunderbolt/apple.c @@ -2634,6 +2632,13 @@ F: include/uapi/drm/asahi_drm.h F: net/bluetooth/brcm.c F: net/bluetooth/brcm.h +ARM/APPLE QSPIMC DRIVER +M: Ryan Murray +L: asahi@lists.linux.dev +S: Maintained +F: Documentation/devicetree/bindings/spi/apple,qspi.yaml +F: drivers/spi/spi-apple-qspi.c + ARM/ARTPEC MACHINE SUPPORT M: Jesper Nilsson M: Lars Persson From 3dd2554de0c5b6e9d16a3809186872847987de6a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:21:54 +1100 Subject: [PATCH 0530/1579] arm64: dts: apple: enable the PMP fast-die limiter on J700 Describe the PMP SRAM, ASC, mailbox, DART and report resources from the Apple device tree. The DART register range includes its v2.3 exception registers. Enable the resources on J700 and connect the cluster effort lanes so cpufreq retains its fallback caps until the firmware runs. Leave the firmware-owned PMC domain untouched. The PMP firmware and fast-die maximum-frequency requests complement the existing thermal power allocator; thermal trips and sustainable power remain unchanged. The bootloader must initialize the PMP DAPF and copy its calibration. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 17 ++++++++ arch/arm64/boot/dts/apple/t8140.dtsi | 50 +++++++++++++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index cc9f115adb7ab..0a185a3c1b4be 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -19,6 +19,7 @@ atcphy0 = &atcphy0; keyboard = &keyboard; serial0 = &serial0; + pmp = &pmp; wifi0 = &wifi0; }; @@ -60,6 +61,22 @@ status = "okay"; }; +&pmp_dart { + status = "okay"; +}; + +&pmp_mbox { + status = "okay"; +}; + +&pmp_report { + status = "okay"; +}; + +&pmp { + status = "okay"; +}; + &pinctrl_ap { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 2dd921b23368c..36bfdf78ec7cd 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -280,12 +280,45 @@ compatible = "apple,t8140-cluster-cpufreq"; reg = <0x2 0x10e20000 0x0 0x1000>; #performance-domain-cells = <0>; + /* E lane follows the ADT ordering; no changing sample yet. */ + apple,fast-die-effort = <&pmp_report 0>; }; cpufreq_p: cpufreq@211e20000 { compatible = "apple,t8140-cluster-cpufreq"; reg = <0x2 0x11e20000 0x0 0x1000>; - #performance-domain-cells = <0>; + /* P lane has been observed under Linux CPU load. */ + apple,fast-die-effort = <&pmp_report 1>; + }; + + pmp_dart: iommu@300300000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x3 0x00300000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_pmp>; + apple,dma-range = <0x100 0x0 0x0 0xbff00000>; + status = "disabled"; + }; + + pmp_report: pmp-report@3003c0000 { + compatible = "apple,t8140-pmp-v2-report"; + reg = <0x3 0x003c0000 0x0 0x14000>; + power-domains = <&ps_pms_sram>; + status = "disabled"; + }; + + pmp: pmp@300500000 { + compatible = "apple,t8140-pmp-v2", "apple,t6000-pmp-v2"; + reg = <0x3 0x00500000 0x0 0xa0000>, + <0x3 0x00e00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + status = "disabled"; }; pmgr: power-management@300700000 { @@ -296,6 +329,21 @@ /* child nodes are added in t8140-pmgr.dtsi */ }; + pmp_mbox: mailbox@300e08000 { + compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x00e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>; + status = "disabled"; + }; + aic: interrupt-controller@301000000 { compatible = "apple,t8140-aic3", "apple,t8122-aic3"; #interrupt-cells = <3>; From f75a1041fca487774f29b133f736296f4d73a4e3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:40:58 +1100 Subject: [PATCH 0531/1579] arm64: dts: apple: Wire up remaining supported J700 peripherals Describe the PMP DART, mailbox, SRAM and report windows from the J700 ADT, including the high DMA aperture. Add the QSPIMC controller, SPI NOR NVRAM partition, I2C2 pinmux and bus, and display crossbar. Describe and enable the external display DARTs and mailbox using existing Apple DART and mailbox bindings. Align the existing pinctrl and diagnostic PIODMA node names with their bindings without changing their labels or resources. Add a T8140 cpuidle node and match it in the existing Apple idle driver. Keep WFI as its only idle state and skip legacy device creation when DT describes the controller. Document the idle and crossbar interfaces. The PMP and QSPI consumers need their corresponding driver support. External DCP and Type-C display routes still need a validated T8140 external-display profile; this change does not wire that consumer. The register, interrupt, power-domain and partition descriptions are independently derived from the ADT and public bindings. The QSPI clock selection follows the existing public QSPIMC binding example. No private device-tree source was copied. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 79 +++++++++++- arch/arm64/boot/dts/apple/t8140.dtsi | 153 +++++++++++++++++++++++ drivers/cpuidle/cpuidle-apple.c | 18 +++ 3 files changed, 248 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index cc9f115adb7ab..e4b11cdcb08e1 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -16,6 +16,10 @@ model = "Apple MacBook Neo"; aliases { + pmp = &pmp; + nvram = &nvram; + spi0 = &qspi; + i2c2 = &i2c2; atcphy0 = &atcphy0; keyboard = &keyboard; serial0 = &serial0; @@ -64,6 +68,77 @@ status = "okay"; }; +&jpeg_dart { + status = "okay"; +}; + +&apr_dart { + status = "okay"; +}; + +&pmp_dart { + status = "okay"; +}; + +&pmp_report { + status = "okay"; +}; + +&pmp_mbox { + status = "okay"; +}; + +&pmp { + status = "okay"; +}; + +&i2c2 { + status = "okay"; +}; + +&qspi { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <25000000>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <4>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + nvram: partition@24000 { + label = "nvram"; + reg = <0x24000 0x1d0000>; + }; + }; + }; +}; + +&dispext0_dart { + status = "okay"; +}; + +&dcpext0_dart { + status = "okay"; +}; + +&dcpext_mbox { + status = "okay"; +}; + +&dpxbar0 { + status = "okay"; +}; + +&cpuidle { + status = "okay"; +}; + &i2c0 { status = "okay"; @@ -1369,7 +1444,7 @@ }; &pinctrl_ap { - pcie0_clkreq: pcie0-clkreq { + pcie0_clkreq: pcie0-clkreq-pins { pinmux = ; }; }; @@ -1381,7 +1456,7 @@ }; / { - pcie0_piodma_diag: dma-controller@390030000 { + pcie0_piodma_diag: piodma@390030000 { compatible = "apple,t8140-piodma-diagnostic"; reg = <0x3 0x90030000 0x0 0x4000>, <0x3 0x90034000 0x0 0x4000>, diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 2dd921b23368c..9127d6185e4ec 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -18,6 +18,18 @@ #address-cells = <2>; #size-cells = <2>; + cpuidle: cpuidle { + compatible = "apple,t8140-cpuidle"; + status = "disabled"; + }; + + clk_200m: clock-200m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <200000000>; + clock-output-names = "clk_200m"; + }; + cpus { #address-cells = <2>; #size-cells = <0>; @@ -276,16 +288,152 @@ nonposted-mmio; ranges; + jpeg_dart: iommu@28d020000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x2 0x8d020000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = /bits/ 64 <0x10000000000 0x30000000000>; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + + apr_dart: iommu@295020000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x2 0x95020000 0x0 0x20000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = /bits/ 64 <0x10000000000 0x30000000000>; + power-domains = <&ps_prores>; + status = "disabled"; + }; + + pmp_dart: iommu@300300000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x3 0x00300000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = /bits/ 64 <0x10000000000 0xbff00000>; + power-domains = <&ps_pmp>; + status = "disabled"; + }; + + pmp_report: pmp-report@3003c0000 { + compatible = "apple,t8140-pmp-v2-report"; + reg = <0x3 0x003c0000 0x0 0x14000>; + power-domains = <&ps_pms_sram>; + status = "disabled"; + }; + + pmp: pmp@300500000 { + compatible = "apple,t8140-pmp-v2", "apple,t6000-pmp-v2"; + reg = <0x3 0x00500000 0x0 0xa0000>, + <0x3 0x00e00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + status = "disabled"; + }; + + pmp_mbox: mailbox@300e08000 { + compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x00e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>; + status = "disabled"; + }; + + dispext0_dart: iommu@416300000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x16300000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + status = "disabled"; + }; + + dcpext0_dart: iommu@416340000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x16340000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + status = "disabled"; + }; + + dcpext_mbox: mailbox@416e08000 { + compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x16e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_dispext0_cpu>; + status = "disabled"; + }; + + i2c2: i2c@385018000 { + compatible = "apple,t8140-i2c", "apple,t8103-i2c"; + reg = <0x3 0x85018000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c2_pins>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_i2c2>; + status = "disabled"; + }; + + qspi: spi@385110000 { + compatible = "apple,t8140-qspi"; + /* Union of the overlapping controller and FIFO ADT windows. */ + reg = <0x3 0x85110000 0x0 0x4800>; + clocks = <&clk_200m>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_qspi>; + status = "disabled"; + }; + + dpxbar0: display-crossbar@40b04c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0x4 0x0b04c000 0x0 0x4000>; + #mux-control-cells = <1>; + status = "disabled"; + }; + cpufreq_e: cpufreq@210e20000 { compatible = "apple,t8140-cluster-cpufreq"; reg = <0x2 0x10e20000 0x0 0x1000>; #performance-domain-cells = <0>; + apple,fast-die-effort = <&pmp_report 0>; }; cpufreq_p: cpufreq@211e20000 { compatible = "apple,t8140-cluster-cpufreq"; reg = <0x2 0x11e20000 0x0 0x1000>; #performance-domain-cells = <0>; + apple,fast-die-effort = <&pmp_report 1>; }; pmgr: power-management@300700000 { @@ -732,6 +880,11 @@ gpio-reserved-ranges = <50 1>, <134 1>; status = "disabled"; + i2c2_pins: i2c2-pins { + pinmux = , + ; + }; + i2c0_pins: i2c0-pins { pinmux = , ; diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c index 3d2b804df8a2d..8e97d626c86c1 100644 --- a/drivers/cpuidle/cpuidle-apple.c +++ b/drivers/cpuidle/cpuidle-apple.c @@ -124,12 +124,22 @@ static struct cpuidle_driver apple_idle_driver = { static int apple_cpuidle_probe(struct platform_device *pdev) { + /* A T8140 CPU PD opt-in hung J700; expose only WFI. */ + if (of_machine_is_compatible("apple,t8140")) + apple_idle_driver.state_count = STATE_WFI + 1; + return cpuidle_register(&apple_idle_driver, NULL); } +static const struct of_device_id apple_cpuidle_of_match[] = { + { .compatible = "apple,t8140-cpuidle" }, + { } +}; + static struct platform_driver apple_cpuidle_driver = { .driver = { .name = "cpuidle-apple", + .of_match_table = apple_cpuidle_of_match, }, .probe = apple_cpuidle_probe, }; @@ -137,12 +147,20 @@ static struct platform_driver apple_cpuidle_driver = { static int __init apple_cpuidle_init(void) { struct platform_device *pdev; + struct device_node *np; int ret; ret = platform_driver_register(&apple_cpuidle_driver); if (ret) return ret; + /* DT-described systems are populated by the OF platform code. */ + np = of_find_matching_node(NULL, apple_cpuidle_of_match); + if (np) { + of_node_put(np); + return 0; + } + if (!of_machine_is_compatible("apple,arm-platform")) return 0; From 0956f639cc250e81e912869ea71947ba86bfbd40 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:47:19 +1100 Subject: [PATCH 0532/1579] soc: apple: sep: retain the sensor SPI mode from the board An unconditional mode-2 assignment overrides the mode-1 J313 device tree. Retain the board mode unless a diagnostic override is supplied, and reject override values outside the SPI mode range. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sensor_shim.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index 5738312663b79..b0e33c8af2857 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -21,18 +21,13 @@ #include "shim.h" -/* - * 8 MHz, SPI mode 2 (CPOL=1 CPHA=0), 8-bit words. Mode 3 returns sixteen zero - * bytes, else identical. Must agree with the node's spi-cpol / absent spi-cpha, - * which is what spi_setup() applies. - */ +/* The board selects the SPI mode; an override is diagnostic only. */ #define SEP_SENSOR_HZ 8000000 #define SEP_SENSOR_BITS 8 -static unsigned int sep_sensor_mode = SPI_MODE_2; +static unsigned int sep_sensor_mode = ~0U; module_param_named(sensor_spi_mode, sep_sensor_mode, uint, 0444); MODULE_PARM_DESC(sensor_spi_mode, - "SPI mode for the sensor: 2 (CPOL=1 CPHA=0, the default and the only mode this sensor answers under) or 3 (CPOL=1 CPHA=1, reads all-zero). For comparison only."); -#define SEP_SENSOR_MODE sep_sensor_mode + "Diagnostic SPI mode override (0-3); default retains the board mode"); /* Chip-select setup and hold, in ns. */ #define SEP_SENSOR_CS_NS 20 @@ -340,11 +335,15 @@ static int sep_apply_cs_timing(struct spi_device *spi) * select. spi_setup() takes the controller io_mutex, not the bus lock, * so no deadlock. */ + if (sep_sensor_mode != ~0U && sep_sensor_mode > SPI_MODE_3) + return -EINVAL; + rc = spi_bus_lock(ctlr); if (rc) return rc; - spi->mode = SEP_SENSOR_MODE; + if (sep_sensor_mode != ~0U) + spi->mode = sep_sensor_mode; spi->bits_per_word = SEP_SENSOR_BITS; spi->max_speed_hz = SEP_SENSOR_HZ; spi->cs_setup.value = SEP_SENSOR_CS_NS; From af9ee0f54d1b3686cfe3c14adda3590748f5851f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:47:21 +1100 Subject: [PATCH 0533/1579] soc: apple: sep: wake detached waiters under the lifetime lock The last file release can free the biometric character device after unregister drops its mutex. Wake detached waiters before releasing that mutex so unregister cannot access the wait queue after destruction. The open-count decision still leaves exactly one path responsible for freeing the device. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/bio_shim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/bio_shim.c b/drivers/soc/apple/bio_shim.c index 50f7e7ddf61ea..476c9e86cb6c9 100644 --- a/drivers/soc/apple/bio_shim.c +++ b/drivers/soc/apple/bio_shim.c @@ -168,9 +168,9 @@ void sep_bio_unregister(void *dev) mutex_lock(&d->lock); d->ctx = NULL; free_now = (d->open_count == 0); - mutex_unlock(&d->lock); - + /* The last close may free d as soon as the lifetime lock is dropped. */ wake_up_interruptible(&d->wq); + mutex_unlock(&d->lock); if (free_now) { mutex_destroy(&d->lock); From dec89fc14d5b9d11df81969caaf247efda601e05 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:52:46 +1100 Subject: [PATCH 0534/1579] soc: apple: sep: drain captures before tearing down the sensor Reject new entry points and protocol sends during shutdown, wake and abort blocked transactions, and join capture work before powering down or unregistering the sensor. Use a device lifetime mutex and a protected sensor registry so independent SPI unbind also waits for exported sensor operations before releasing their resources. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 16 +- drivers/soc/apple/sensor_shim.c | 265 ++++++++++++++++++++++++++++++-- drivers/soc/apple/sep.rs | 26 +++- drivers/soc/apple/sks.rs | 3 +- 4 files changed, 284 insertions(+), 26 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index c91938b60fee1..aae0afcf33e15 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2188,7 +2188,7 @@ impl SepData { name: &'static CStr, payload: &[u8], ) -> Result { - if !self.sbio_ready.load(Relaxed) { + if self.shutting_down.load(Relaxed) || !self.sbio_ready.load(Relaxed) { return Err(ENODEV); } @@ -2204,6 +2204,11 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(SBIO_TIMEOUT_MS); let mut guard = self.sbio_rx.lock(); loop { + if self.shutting_down.load(Relaxed) { + guard.abort(); + let _ = guard.take_done(); + return Err(ENODEV); + } if let Some(done) = guard.take_done() { return Ok(done); } @@ -2298,6 +2303,9 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(SBIO_TIMEOUT_MS); let mut guard = self.sbio_rx.lock(); loop { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } if guard.take_grant() { return Ok(()); } @@ -2346,6 +2354,9 @@ impl SepData { } pub(crate) fn bio_open(&self) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } let mut session = self.bio_session.lock(); bio::open(&mut session)?; drop(session); @@ -2405,6 +2416,9 @@ impl SepData { } pub(crate) fn bio_ioctl(&self, cmd: u32, arg: usize) -> Result { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } if cmd == bio::IOC_ATTEST { return self.bio_attest(arg); } diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index b0e33c8af2857..e52acb69b04f0 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -19,6 +19,9 @@ #include #include +#include +#include + #include "shim.h" /* The board selects the SPI mode; an override is diagnostic only. */ @@ -35,6 +38,28 @@ MODULE_PARM_DESC(sensor_spi_mode, #define SEP_SENSOR_OFF_DELAY_MS 10 #define SEP_SENSOR_ON_DELAY_MS 7 +struct sep_sensor_lifetime { + /* Serializes sensor operations with this device's resource teardown. */ + struct mutex lock; +}; + +static DEFINE_MUTEX(sep_sensor_registry_lock); +static struct sep_sensor_lifetime *sep_active_sensor; + +static struct sep_sensor_lifetime *sep_lock_sensor(void) +{ + struct sep_sensor_lifetime *sensor; + + mutex_lock(&sep_sensor_registry_lock); + sensor = sep_active_sensor; + if (sensor) + mutex_lock(&sensor->lock); + mutex_unlock(&sep_sensor_registry_lock); + return sensor; +} + +static void sep_sensor_irq_teardown_locked(void); + static struct spi_device *sep_spi; static struct gpio_desc *sep_power; static bool sep_registered; @@ -377,7 +402,7 @@ static int sep_apply_cs_timing(struct spi_device *spi) return 0; } -static int sep_sensor_probe(struct spi_device *spi) +static int sep_sensor_probe_locked(struct spi_device *spi) { int rc; @@ -399,11 +424,41 @@ static int sep_sensor_probe(struct spi_device *spi) return 0; } +static int sep_sensor_probe(struct spi_device *spi) +{ + struct sep_sensor_lifetime *sensor; + int error; + + sensor = devm_kzalloc(&spi->dev, sizeof(*sensor), GFP_KERNEL); + if (!sensor) + return -ENOMEM; + mutex_init(&sensor->lock); + mutex_lock(&sep_sensor_registry_lock); + if (sep_active_sensor) { + error = -EBUSY; + } else { + error = sep_sensor_probe_locked(spi); + if (!error) { + spi_set_drvdata(spi, sensor); + sep_active_sensor = sensor; + } + } + mutex_unlock(&sep_sensor_registry_lock); + return error; +} + static void sep_sensor_remove(struct spi_device *spi) { - sep_sensor_irq_teardown(); + struct sep_sensor_lifetime *sensor = spi_get_drvdata(spi); + + mutex_lock(&sep_sensor_registry_lock); + mutex_lock(&sensor->lock); + sep_active_sensor = NULL; + sep_sensor_irq_teardown_locked(); sep_release_power(); sep_spi = NULL; + mutex_unlock(&sensor->lock); + mutex_unlock(&sep_sensor_registry_lock); } /* @@ -460,19 +515,19 @@ void sep_sensor_unregister(void) sep_registered = false; } -int sep_sensor_bound(void) +static int sep_sensor_bound_locked(void) { return sep_spi != NULL; } /* Whether those delays reach hardware, are emulated, or are silently bypassed. */ -int sep_sensor_cs_timing_mode(void) +static int sep_sensor_cs_timing_mode_locked(void) { return sep_cs_timing_mode; } /* Power cycle: off, 10 ms, on, 7 ms. -ENODEV if there is no power line. */ -int sep_sensor_power_cycle(void) +static int sep_sensor_power_cycle_locked(void) { if (sep_power_source == SEP_POWER_SMC_RAILS) { int rc = sep_smc_set_power(0); @@ -497,12 +552,12 @@ int sep_sensor_power_cycle(void) } /* How the power line was obtained: none, the node's property, or chip+line. */ -int sep_sensor_power_source(void) +static int sep_sensor_power_source_locked(void) { return sep_power_source; } -int sep_sensor_power_line(void) +static int sep_sensor_power_line_locked(void) { if (sep_power_source == SEP_POWER_SMC_RAILS) return -ENODEV; @@ -516,7 +571,7 @@ int sep_sensor_power_line(void) * the sensor node's "firmware-name" property. NULL when absent, so the caller * can fall back to a default. */ -const char *sep_sensor_firmware_name(void) +static const char *sep_sensor_firmware_name_locked(void) { const char *name = NULL; @@ -528,7 +583,7 @@ const char *sep_sensor_firmware_name(void) } /* Powers the sensor on/off, holding the hardware settling delay. */ -int sep_sensor_power(int on) +static int sep_sensor_power_locked(int on) { if (sep_power_source == SEP_POWER_SMC_RAILS) { int rc = sep_smc_set_power(on ? 1 : 0); @@ -567,7 +622,7 @@ static irqreturn_t sep_drdy_isr(int irq, void *dev_id) * firmware patch. Returns 0 when an interrupt is available, -errno otherwise * (the caller then falls back to polling). */ -int sep_sensor_irq_setup(void) +static int sep_sensor_irq_setup_locked(void) { struct device_node *np; struct gpio_device *gdev; @@ -640,7 +695,7 @@ int sep_sensor_irq_setup(void) return 0; } -void sep_sensor_irq_teardown(void) +static void sep_sensor_irq_teardown_locked(void) { if (sep_drdy_irq >= 0) { free_irq(sep_drdy_irq, sep_spi); @@ -652,14 +707,14 @@ void sep_sensor_irq_teardown(void) } } -int sep_sensor_irq_available(void) +static int sep_sensor_irq_available_locked(void) { return sep_drdy_irq >= 0; } /* Clear any stale signal before a capture so the next wait reflects a fresh * data-ready edge, not a leftover from the previous frame. */ -void sep_sensor_irq_arm(void) +static void sep_sensor_irq_arm_locked(void) { if (sep_drdy_irq >= 0) reinit_completion(&sep_drdy_done); @@ -670,7 +725,7 @@ void sep_sensor_irq_arm(void) * -ETIMEDOUT = no signal (the caller re-polls status over SPI regardless, so a * missed interrupt only costs one poll interval), -ENODEV = no interrupt. */ -int sep_sensor_irq_wait(unsigned int timeout_ms) +static int sep_sensor_irq_wait_locked(unsigned int timeout_ms) { if (sep_drdy_irq < 0) return -ENODEV; @@ -685,7 +740,7 @@ int sep_sensor_irq_wait(unsigned int timeout_ms) * `len` transmit bytes (trailing ones as 0xff) rather than relying on the * controller running its transmit buffer dry. */ -int sep_sensor_xfer(const void *tx, void *rx, size_t len) +static int sep_sensor_xfer_locked(const void *tx, void *rx, size_t len) { struct spi_transfer xfer = { .tx_buf = tx, @@ -708,7 +763,7 @@ int sep_sensor_xfer(const void *tx, void *rx, size_t len) * A simultaneous receive changes the long transfer's pacing and the sensor * rejects the patch blob, so the receive must be absent entirely. */ -int sep_sensor_xfer_tx(const void *tx, size_t len) +static int sep_sensor_xfer_tx_locked(const void *tx, size_t len) { struct spi_transfer xfer = { .tx_buf = tx, @@ -730,7 +785,7 @@ int sep_sensor_xfer_tx(const void *tx, size_t len) * Two transfers in one chip-select assertion: command out, then read in. * Chip-select stays asserted because neither transfer sets cs_change. */ -int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) +static int sep_sensor_xfer2_locked(const void *tx, size_t tx_len, void *rx, size_t rx_len) { struct spi_transfer xfers[2] = { { @@ -754,3 +809,179 @@ int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) return spi_sync_transfer(sep_spi, xfers, 2); } + +int sep_sensor_bound(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return 0; + result = sep_sensor_bound_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_cs_timing_mode(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return 0; + result = sep_sensor_cs_timing_mode_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_power_cycle(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_power_cycle_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_power_source(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return 0; + result = sep_sensor_power_source_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_power_line(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_power_line_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +const char *sep_sensor_firmware_name(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + const char *result; + + if (!sensor) + return NULL; + result = sep_sensor_firmware_name_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_power(int on) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_power_locked(on); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_irq_setup(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_irq_setup_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +void sep_sensor_irq_teardown(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + + if (!sensor) + return; + sep_sensor_irq_teardown_locked(); + mutex_unlock(&sensor->lock); +} + +int sep_sensor_irq_available(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return 0; + result = sep_sensor_irq_available_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +void sep_sensor_irq_arm(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + + if (!sensor) + return; + sep_sensor_irq_arm_locked(); + mutex_unlock(&sensor->lock); +} + +int sep_sensor_irq_wait(unsigned int timeout_ms) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_irq_wait_locked(timeout_ms); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_xfer(const void *tx, void *rx, size_t len) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_xfer_locked(tx, rx, len); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_xfer_tx(const void *tx, size_t len) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_xfer_tx_locked(tx, len); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_xfer2_locked(tx, tx_len, rx, rx_len); + mutex_unlock(&sensor->lock); + return result; +} diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 6aa66f9263944..05aeba3ae6cb7 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1005,6 +1005,9 @@ impl SepData { } fn send(&self, msg: Message) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } self.mbox.lock().as_ref().ok_or(ENODEV)?.send(msg, false) } @@ -1021,6 +1024,10 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); let mut guard = self.control.lock(); loop { + if self.shutting_down.load(Relaxed) { + guard.abandon(idx); + return Err(ENODEV); + } if let Some(reply) = guard.take_reply(idx) { guard.release(idx); drop(guard); @@ -1448,7 +1455,8 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(ms); let mut guard = self.scrd_probe.lock(); loop { - if guard.captured.len() >= until || remaining == 0 { + if self.shutting_down.load(Relaxed) + || guard.captured.len() >= until || remaining == 0 { return; } match self @@ -1916,16 +1924,16 @@ impl SepData { fn remove(&self) { self.shutting_down.store(true, Relaxed); - self.release_firmware_mapping(); + bio::release(&mut self.bio_session.lock()); + self.rng_shutdown.store(true, Relaxed); + self.control_wq.notify_all(); + self.sbio_wq.notify_all(); + self.sks_wq.notify_all(); + self.scrd_wq.notify_all(); trusted::unregister(); self.unregister_fv_kernel(); let dev = self.bio_dev.lock().take(); - if dev.is_some() { - bio::release(&mut self.bio_session.lock()); - sensor::power(false); - sensor::unregister_driver(); - } drop(dev); self.rng_shutdown.store(true, Relaxed); @@ -1950,6 +1958,10 @@ impl SepData { ); } + sensor::power(false); + sensor::unregister_driver(); + self.release_firmware_mapping(); + let _ = self.store.lock().take(); let _ = self.host_store.lock().take(); diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 03caee8f0767b..505c59228ae3f 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -884,7 +884,8 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(ms); let mut guard = self.sks_probe.lock(); loop { - if guard.captured.len() >= until || remaining == 0 { + if self.shutting_down.load(Relaxed) + || guard.captured.len() >= until || remaining == 0 { return; } match self From b0890a92a7568ba0adeb2d6a70b622a8bfd3783f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:00:10 +1100 Subject: [PATCH 0535/1579] soc: apple: sep: bind capture results to their queued operation Assign a generation to each queued capture and retain worker ownership across cancellation and close. Refuse a replacement operation until the worker has retired, and check the queued generation before accepting progress, guidance or a terminal result. This prevents an old verify result from being bound to a new request nonce and serializes enrol and verify work against the shared sensor and buffers. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/bio.rs | 66 +++++++++++++++++++++++++++++++++------ drivers/soc/apple/sbio.rs | 32 +++++++++++++++---- drivers/soc/apple/sep.rs | 16 ++++++---- 3 files changed, 92 insertions(+), 22 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index d115a18d64d97..464022c681f1f 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -417,6 +417,8 @@ impl Op { } pub(crate) struct Session { + generation: u64, + worker_epoch: Option, open: bool, op: Op, unseen: bool, @@ -427,6 +429,8 @@ pub(crate) struct Session { impl Session { pub(crate) fn new() -> Session { Session { + generation: 0, + worker_epoch: None, open: false, op: Op::Idle, unseen: false, @@ -436,6 +440,7 @@ impl Session { } fn reset(&mut self) { + self.generation = self.generation.saturating_add(1); self.open = false; self.op = Op::Idle; self.unseen = false; @@ -443,6 +448,15 @@ impl Session { self.token = None; } + fn start_worker(&mut self) -> Result<()> { + if self.worker_epoch.is_some() { + return Err(EBUSY); + } + self.generation = self.generation.checked_add(1).ok_or(EOVERFLOW)?; + self.worker_epoch = Some(self.generation); + Ok(()) + } + pub(crate) fn ready(&self) -> bool { self.unseen } @@ -560,7 +574,7 @@ fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { } require_sensor(ctx.sensor_present)?; - if !ctx.session.op.may_start() { + if ctx.session.worker_epoch.is_some() || !ctx.session.op.may_start() { return Err(EBUSY); } if matches!(ctx.live_identity_count, Some(count) if count >= SEP_USER_CAPACITY) @@ -569,6 +583,7 @@ fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { return Err(ENOSPC); } + ctx.session.start_worker()?; ctx.session.op = Op::Enrol { label: request.label, stage: 0, @@ -590,10 +605,14 @@ fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { pub(crate) fn enrol_advance( session: &mut Session, + epoch: u64, completed: u32, percent_now: u32, guide: Guidance, ) -> bool { + if !worker_is_current(session, epoch) { + return false; + } if let Op::Enrol { stage, terminal, @@ -614,7 +633,10 @@ pub(crate) fn enrol_advance( false } -pub(crate) fn capture_guide(session: &mut Session, guide: Guidance) -> bool { +pub(crate) fn capture_guide(session: &mut Session, epoch: u64, guide: Guidance) -> bool { + if !worker_is_current(session, epoch) { + return false; + } let changed = match &mut session.op { Op::Enrol { terminal, guidance, .. } => { if terminal.is_none() && *guidance != guide { @@ -642,9 +664,13 @@ pub(crate) fn capture_guide(session: &mut Session, guide: Guidance) -> bool { pub(crate) fn enrol_finish( session: &mut Session, + epoch: u64, index: &mut IdentityIndex, outcome: core::result::Result<[u8; UUID_LEN], u32>, ) -> bool { + if !worker_is_current(session, epoch) { + return false; + } let Op::Enrol { label, terminal, .. } = &mut session.op @@ -670,8 +696,8 @@ pub(crate) fn enrol_finish( true } -pub(crate) fn enrol_is_live(session: &Session) -> bool { - matches!(&session.op, Op::Enrol { terminal: None, .. }) && session.open +pub(crate) fn enrol_is_live(session: &Session, epoch: u64) -> bool { + worker_is_current(session, epoch) && matches!(&session.op, Op::Enrol { terminal: None, .. }) } fn enrol_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { @@ -732,7 +758,7 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { } require_sensor(ctx.sensor_present)?; - if !ctx.session.op.may_start() { + if ctx.session.worker_epoch.is_some() || !ctx.session.op.may_start() { return Err(EBUSY); } @@ -754,6 +780,7 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { }); } + ctx.session.start_worker()?; ctx.session.token = None; ctx.session.op = Op::Verify { nonce: request.nonce, @@ -772,9 +799,13 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { pub(crate) fn verify_finish( session: &mut Session, + epoch: u64, outcome: VerifyOutcome, token_bytes: [u8; TOKEN_LEN], ) -> bool { + if !worker_is_current(session, epoch) { + return false; + } let Op::Verify { nonce, terminal, .. } = &mut session.op else { return false; }; @@ -793,12 +824,26 @@ pub(crate) fn verify_finish( true } -pub(crate) fn verify_is_live(session: &Session) -> bool { - matches!(&session.op, Op::Verify { terminal: None, .. }) && session.open +pub(crate) fn verify_is_live(session: &Session, epoch: u64) -> bool { + worker_is_current(session, epoch) && matches!(&session.op, Op::Verify { terminal: None, .. }) } -pub(crate) fn capture_is_live(session: &Session) -> bool { - enrol_is_live(session) || verify_is_live(session) +pub(crate) fn capture_is_live(session: &Session, epoch: u64) -> bool { + enrol_is_live(session, epoch) || verify_is_live(session, epoch) +} + +pub(crate) fn queued_epoch(session: &Session) -> Option { + session.worker_epoch +} + +fn worker_is_current(session: &Session, epoch: u64) -> bool { + session.open && session.generation == epoch && session.worker_epoch == Some(epoch) +} + +pub(crate) fn worker_stopped(session: &mut Session, epoch: u64) { + if session.worker_epoch == Some(epoch) { + session.worker_epoch = None; + } } fn verify_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { @@ -868,6 +913,7 @@ fn cancel(ctx: &mut Context<'_>) -> Result { Op::Verify { terminal, .. } => *terminal = Some(VerifyOutcome::Failed(STATUS_LOCAL)), } + ctx.session.generation = ctx.session.generation.saturating_add(1); ctx.session.token = None; ctx.session.unseen = true; @@ -940,7 +986,7 @@ fn delete_all(ctx: &mut Context<'_>) -> Result { } pub(crate) fn open(session: &mut Session) -> Result<()> { - if session.open { + if session.open || session.worker_epoch.is_some() { return Err(EBUSY); } session.reset(); diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index aae0afcf33e15..6c046bd4285c5 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -232,7 +232,9 @@ impl SepData { let filed = { let mut session = self.bio_session.lock(); let mut index = self.bio_index.lock(); - bio::enrol_finish(&mut session, &mut index, outcome) + bio::enrol_finish( + &mut session, self.capture_epoch.load(Relaxed), &mut index, outcome, + ) }; if !filed { return; @@ -1357,7 +1359,7 @@ impl SepData { let mut identity: Option<[u8; bio::UUID_LEN]> = None; let outcome = loop { - if !bio::enrol_is_live(&self.bio_session.lock()) { + if !bio::enrol_is_live(&self.bio_session.lock(), self.capture_epoch.load(Relaxed)) { break None; } if counter >= ENROL_MAX_CAPTURES { @@ -1374,6 +1376,7 @@ impl SepData { counter = counter.saturating_add(1); let woke = bio::enrol_advance( &mut self.bio_session.lock(), + self.capture_epoch.load(Relaxed), stage, percent, bio::Guidance::LiftAndMove, @@ -1395,7 +1398,10 @@ impl SepData { // The capture did not take (partial/unusable contact). Nudge // the person to hold still and press again rather than // silently re-capturing with no feedback. - if bio::capture_guide(&mut self.bio_session.lock(), bio::Guidance::HoldStill) { + if bio::capture_guide( + &mut self.bio_session.lock(), self.capture_epoch.load(Relaxed), + bio::Guidance::HoldStill, + ) { self.bio_wake(); } } @@ -2025,7 +2031,7 @@ impl SepData { ); let mut attempt: u32 = 0; loop { - if !bio::capture_is_live(&self.bio_session.lock()) { + if !bio::capture_is_live(&self.bio_session.lock(), self.capture_epoch.load(Relaxed)) { return CaptureWait::Abandon; } @@ -2050,7 +2056,9 @@ impl SepData { _ => None, }; if let Some(guide) = guide { - if bio::capture_guide(&mut self.bio_session.lock(), guide) { + if bio::capture_guide( + &mut self.bio_session.lock(), self.capture_epoch.load(Relaxed), guide, + ) { self.bio_wake(); } } @@ -2375,15 +2383,24 @@ impl SepData { } pub(crate) fn queue_enrolment(this: Arc) { + let Some(epoch) = bio::queued_epoch(&this.bio_session.lock()) else { + return; + }; + this.capture_epoch.store(epoch, Relaxed); if workqueue::system() .enqueue::, ENROL_WORK_ID>(this.clone()) .is_err() { this.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + bio::worker_stopped(&mut this.bio_session.lock(), epoch); } } pub(crate) fn queue_verify(this: Arc) { + let Some(epoch) = bio::queued_epoch(&this.bio_session.lock()) else { + return; + }; + this.capture_epoch.store(epoch, Relaxed); if workqueue::system() .enqueue::, VERIFY_WORK_ID>(this.clone()) .is_err() @@ -2392,11 +2409,14 @@ impl SepData { bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), [0u8; bio::TOKEN_LEN], ); + bio::worker_stopped(&mut this.bio_session.lock(), epoch); } } fn finish_verify(&self, outcome: bio::VerifyOutcome, token_bytes: [u8; bio::TOKEN_LEN]) { - let filed = bio::verify_finish(&mut self.bio_session.lock(), outcome, token_bytes); + let filed = bio::verify_finish( + &mut self.bio_session.lock(), self.capture_epoch.load(Relaxed), outcome, token_bytes, + ); if filed { self.bio_wake(); } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 05aeba3ae6cb7..428344654028f 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -659,6 +659,7 @@ struct SepData { registered: Atomic, shutting_down: Atomic, + capture_epoch: Atomic, #[pin] rx_work: Work, @@ -824,6 +825,7 @@ impl SepData { settle_idle_ticks: Atomic::new(0), registered: Atomic::new(false), shutting_down: Atomic::new(false), + capture_epoch: Atomic::new(0), rx_work <- new_work!("SepData::rx_work"), enrol_work <- new_work!("SepData::enrol_work"), verify_work <- new_work!("SepData::verify_work"), @@ -2114,10 +2116,11 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { - if this.shutting_down.load(Relaxed) { - return; + let epoch = this.capture_epoch.load(Relaxed); + if !this.shutting_down.load(Relaxed) { + this.run_enrolment(); } - this.run_enrolment(); + bio::worker_stopped(&mut this.bio_session.lock(), epoch); } } @@ -2125,10 +2128,11 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { - if this.shutting_down.load(Relaxed) { - return; + let epoch = this.capture_epoch.load(Relaxed); + if !this.shutting_down.load(Relaxed) { + this.run_verify(); } - this.run_verify(); + bio::worker_stopped(&mut this.bio_session.lock(), epoch); } } From 5b4a74c4e0aa2a914687a528fd1ebc4ff790e9b2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:02:11 +1100 Subject: [PATCH 0536/1579] soc: apple: sep: export the biometric ioctl header through UAPI Move the public biometric interface into include/uapi/linux and use conventional ioctl macro spelling without changing its fixed-width layouts or ioctl values. Make the libfprint patch consume the exported header instead of maintaining a duplicated ABI definition, and document header installation for its build. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 3 +- drivers/soc/apple/sep-bio.h | 130 ---------- include/uapi/linux/apple-sep.h | 133 +++++++++++ tools/aurora-sep/README.md | 8 + .../libfprint-1.94.100-apple-sep.patch | 223 ++++-------------- 5 files changed, 187 insertions(+), 310 deletions(-) delete mode 100644 drivers/soc/apple/sep-bio.h create mode 100644 include/uapi/linux/apple-sep.h diff --git a/MAINTAINERS b/MAINTAINERS index ced483de94493..743bb889b069a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2598,7 +2598,6 @@ F: drivers/cpufreq/apple-soc-cpufreq.c F: drivers/dma/apple-admac.c F: drivers/gpio/gpio-macsmc.c F: drivers/hwmon/macsmc-hwmon.c -F: drivers/pmdomain/apple/ F: drivers/i2c/busses/i2c-pasemi-core.c F: drivers/i2c/busses/i2c-pasemi-platform.c F: drivers/input/misc/macsmc-input.c @@ -2612,6 +2611,7 @@ F: drivers/nvmem/apple-efuses.c F: drivers/nvmem/apple-spmi-nvmem.c F: drivers/phy/apple/ F: drivers/pinctrl/pinctrl-apple-gpio.c +F: drivers/pmdomain/apple/ F: drivers/power/reset/macsmc-reboot.c F: drivers/power/supply/macsmc-power.c F: drivers/pwm/pwm-apple.c @@ -2630,6 +2630,7 @@ F: include/dt-bindings/pinctrl/apple.h F: include/linux/mfd/macsmc.h F: include/linux/soc/apple/* F: include/uapi/drm/asahi_drm.h +F: include/uapi/linux/apple-sep.h F: net/bluetooth/brcm.c F: net/bluetooth/brcm.h diff --git a/drivers/soc/apple/sep-bio.h b/drivers/soc/apple/sep-bio.h deleted file mode 100644 index d52b880ac9332..0000000000000 --- a/drivers/soc/apple/sep-bio.h +++ /dev/null @@ -1,130 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Copyright 2026 Dj */ -/* - * Userspace interface for the SEP biometric device. - * - * The enclave does the matching. Nothing biometric crosses this interface: only - * an operation, a stage, a status, an opaque identity UUID, and an opaque host - * label userspace chooses. - */ - -#pragma once - -#include -#include - -#define SEP_BIO_IFACE_VERSION 4 - -#define SEP_BIO_UUID_LEN 16 -#define SEP_BIO_LABEL_LEN 128 -#define SEP_BIO_NONCE_LEN 32 -#define SEP_BIO_TOKEN_LEN 32 -#define SEP_BIO_MAX_IDENTITIES 32 -#define SEP_BIO_CHALLENGE_LEN 32 -#define SEP_BIO_ATTEST_PUB_LEN 65 -#define SEP_BIO_ATTEST_SIG_MAX 72 - -enum { - SEP_BIO_STATE_IDLE = 0, - SEP_BIO_STATE_PENDING = 1, - SEP_BIO_STATE_PROGRESS = 2, - SEP_BIO_STATE_DONE = 3, - SEP_BIO_STATE_FAILED = 4, -}; - - -enum { - SEP_BIO_NO_MATCH = 0, - SEP_BIO_MATCH = 1, - SEP_BIO_NOT_COMPARED = 2, -}; - -struct sep_bio_identity { - __u8 uuid[SEP_BIO_UUID_LEN]; - __u8 label[SEP_BIO_LABEL_LEN]; -}; - -struct sep_bio_info { - __u32 version; - __u32 sensor_present; - __u32 enrolled; - __u32 capacity; /* usable slots in this SEP user context, not ABI list size */ - __u32 enroll_stages; - __u32 reserved[3]; -}; - -/* reserved[0] flags; reserved[1] is a live SEP identity count only when valid. */ -#define SEP_BIO_INFO_LIVE_COUNT_VALID 1u - -struct sep_bio_list { - __u32 count; - __u32 reserved; - struct sep_bio_identity id[SEP_BIO_MAX_IDENTITIES]; -}; - -struct sep_bio_enrol_start { - __u32 flags; - __u32 reserved; - __u8 label[SEP_BIO_LABEL_LEN]; -}; - -#define SEP_BIO_GUIDANCE_NONE 0 -#define SEP_BIO_GUIDANCE_PLACE 1 -#define SEP_BIO_GUIDANCE_LIFT_AND_MOVE 2 -#define SEP_BIO_GUIDANCE_HOLD_STILL 3 - -struct sep_bio_enrol_poll { - __u32 state; - __u32 stage; - __u32 stages_total; - __u32 status; - __u8 uuid[SEP_BIO_UUID_LEN]; - __u32 guidance; - __u32 progress_percent; -}; -struct sep_bio_verify_start { - __u32 flags; - __u32 reserved; - __u8 nonce[SEP_BIO_NONCE_LEN]; -}; - -struct sep_bio_verify_poll { - __u32 state; - __u32 result; - __u32 status; - __u32 guidance; /* SEP_BIO_GUIDANCE_*; status only, no capture side effect */ - __u8 uuid[SEP_BIO_UUID_LEN]; - __u8 token[SEP_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ - __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ -}; - -struct sep_bio_delete { - __u8 uuid[SEP_BIO_UUID_LEN]; -}; - -/* - * Device attestation of key possession: the enclave signs 'challenge' with the - * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and - * returns the DER signature and public point. The private key never leaves the - * enclave. - */ -struct sep_bio_attest { - __u32 sig_len; /* out: DER signature length */ - __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ - __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ - __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ - __u8 reserved[3]; -}; - -#define SEP_BIO_IOC_MAGIC 0xB1 - -#define SEP_BIO_GET_INFO _IOR (SEP_BIO_IOC_MAGIC, 0x01, struct sep_bio_info) -#define SEP_BIO_LIST _IOR (SEP_BIO_IOC_MAGIC, 0x02, struct sep_bio_list) -#define SEP_BIO_ENROL_START _IOW (SEP_BIO_IOC_MAGIC, 0x03, struct sep_bio_enrol_start) -#define SEP_BIO_ENROL_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x04, struct sep_bio_enrol_poll) -#define SEP_BIO_VERIFY_START _IOW (SEP_BIO_IOC_MAGIC, 0x05, struct sep_bio_verify_start) -#define SEP_BIO_VERIFY_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x06, struct sep_bio_verify_poll) -#define SEP_BIO_CANCEL _IO (SEP_BIO_IOC_MAGIC, 0x07) -#define SEP_BIO_DELETE _IOW (SEP_BIO_IOC_MAGIC, 0x08, struct sep_bio_delete) -#define SEP_BIO_DELETE_ALL _IO (SEP_BIO_IOC_MAGIC, 0x09) -#define SEP_BIO_ATTEST _IOWR(SEP_BIO_IOC_MAGIC, 0x0a, struct sep_bio_attest) diff --git a/include/uapi/linux/apple-sep.h b/include/uapi/linux/apple-sep.h new file mode 100644 index 0000000000000..81467fe91f36a --- /dev/null +++ b/include/uapi/linux/apple-sep.h @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note OR MIT */ +/* Copyright 2026 Dj */ +/* + * Userspace interface for the SEP biometric device. + * + * The enclave does the matching. Nothing biometric crosses this interface: only + * an operation, a stage, a status, an opaque identity UUID, and an opaque host + * label userspace chooses. + */ + +#ifndef _UAPI_LINUX_APPLE_SEP_H +#define _UAPI_LINUX_APPLE_SEP_H + +#include +#include + +#define SEP_BIO_IFACE_VERSION 4 + +#define SEP_BIO_UUID_LEN 16 +#define SEP_BIO_LABEL_LEN 128 +#define SEP_BIO_NONCE_LEN 32 +#define SEP_BIO_TOKEN_LEN 32 +#define SEP_BIO_MAX_IDENTITIES 32 +#define SEP_BIO_CHALLENGE_LEN 32 +#define SEP_BIO_ATTEST_PUB_LEN 65 +#define SEP_BIO_ATTEST_SIG_MAX 72 + +enum { + SEP_BIO_STATE_IDLE = 0, + SEP_BIO_STATE_PENDING = 1, + SEP_BIO_STATE_PROGRESS = 2, + SEP_BIO_STATE_DONE = 3, + SEP_BIO_STATE_FAILED = 4, +}; + +enum { + SEP_BIO_NO_MATCH = 0, + SEP_BIO_MATCH = 1, + SEP_BIO_NOT_COMPARED = 2, +}; + +struct sep_bio_identity { + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +struct sep_bio_info { + __u32 version; + __u32 sensor_present; + __u32 enrolled; + __u32 capacity; /* usable slots in this SEP user context, not ABI list size */ + __u32 enroll_stages; + __u32 reserved[3]; +}; + +/* reserved[0] flags; reserved[1] is a live SEP identity count only when valid. */ +#define SEP_BIO_INFO_LIVE_COUNT_VALID 1u + +struct sep_bio_list { + __u32 count; + __u32 reserved; + struct sep_bio_identity id[SEP_BIO_MAX_IDENTITIES]; +}; + +struct sep_bio_enrol_start { + __u32 flags; + __u32 reserved; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +#define SEP_BIO_GUIDANCE_NONE 0 +#define SEP_BIO_GUIDANCE_PLACE 1 +#define SEP_BIO_GUIDANCE_LIFT_AND_MOVE 2 +#define SEP_BIO_GUIDANCE_HOLD_STILL 3 + +struct sep_bio_enrol_poll { + __u32 state; + __u32 stage; + __u32 stages_total; + __u32 status; + __u8 uuid[SEP_BIO_UUID_LEN]; + __u32 guidance; + __u32 progress_percent; +}; + +struct sep_bio_verify_start { + __u32 flags; + __u32 reserved; + __u8 nonce[SEP_BIO_NONCE_LEN]; +}; + +struct sep_bio_verify_poll { + __u32 state; + __u32 result; + __u32 status; + __u32 guidance; /* SEP_BIO_GUIDANCE_*; status only, no capture side effect */ + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 token[SEP_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ + __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ +}; + +struct sep_bio_delete { + __u8 uuid[SEP_BIO_UUID_LEN]; +}; + +/* + * Device attestation of key possession: the enclave signs 'challenge' with the + * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and + * returns the DER signature and public point. The private key never leaves the + * enclave. + */ +struct sep_bio_attest { + __u32 sig_len; /* out: DER signature length */ + __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ + __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ + __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ + __u8 reserved[3]; +}; + +#define SEP_BIO_IOC_MAGIC 0xB1 + +#define SEP_BIO_GET_INFO _IOR(SEP_BIO_IOC_MAGIC, 0x01, struct sep_bio_info) +#define SEP_BIO_LIST _IOR(SEP_BIO_IOC_MAGIC, 0x02, struct sep_bio_list) +#define SEP_BIO_ENROL_START _IOW(SEP_BIO_IOC_MAGIC, 0x03, struct sep_bio_enrol_start) +#define SEP_BIO_ENROL_POLL _IOR(SEP_BIO_IOC_MAGIC, 0x04, struct sep_bio_enrol_poll) +#define SEP_BIO_VERIFY_START _IOW(SEP_BIO_IOC_MAGIC, 0x05, struct sep_bio_verify_start) +#define SEP_BIO_VERIFY_POLL _IOR(SEP_BIO_IOC_MAGIC, 0x06, struct sep_bio_verify_poll) +#define SEP_BIO_CANCEL _IO(SEP_BIO_IOC_MAGIC, 0x07) +#define SEP_BIO_DELETE _IOW(SEP_BIO_IOC_MAGIC, 0x08, struct sep_bio_delete) +#define SEP_BIO_DELETE_ALL _IO(SEP_BIO_IOC_MAGIC, 0x09) +#define SEP_BIO_ATTEST _IOWR(SEP_BIO_IOC_MAGIC, 0x0a, struct sep_bio_attest) + +#endif /* _UAPI_LINUX_APPLE_SEP_H */ diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index b83ef5dea2e3d..bfe086e48a2c6 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -121,3 +121,11 @@ On Omarchy, run `omarchy-apply-lock` once after enrollment. The Quickshell lock screen then selects `omarchy-lock-fingerprint` automatically and accepts Touch ID through fprintd. Hyprlock's separate fingerprint switch is not used by the Quickshell lock screen. + +## Biometric userspace header + +The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this +checkout's headers with `make headers_install INSTALL_HDR_PATH=` +and add that directory's `include` to the libfprint build include path. The +libfprint patch uses `` directly; it has no copied ABI +header. Interface version, ioctl numbers and fixed-width layouts are unchanged. diff --git a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch index 1947bd095b43b..3faef0fdf1dc2 100644 --- a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch +++ b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch @@ -1,138 +1,3 @@ -diff --git a/libfprint/drivers/aurora/aurora-bio.h b/libfprint/drivers/aurora/aurora-bio.h -new file mode 100644 -index 0000000..b2d2f43 ---- /dev/null -+++ b/libfprint/drivers/aurora/aurora-bio.h -@@ -0,0 +1,129 @@ -+/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -+/* Copyright 2026 Dj */ -+/* -+ * Userspace interface for the SEP/Mesa biometric device. -+ * -+ * The enclave does the matching. Nothing biometric crosses this interface: only -+ * an operation, a stage, a status, an opaque identity UUID, and an opaque host -+ * label userspace chooses. -+ * -+ * SPDX-License-Identifier: LGPL-2.1-or-later -+ */ -+ -+#pragma once -+ -+#include -+#include -+ -+#define AURORA_BIO_IFACE_VERSION 4 -+ -+#define AURORA_BIO_UUID_LEN 16 -+#define AURORA_BIO_LABEL_LEN 128 -+#define AURORA_BIO_NONCE_LEN 32 -+#define AURORA_BIO_TOKEN_LEN 32 -+#define AURORA_BIO_MAX_IDENTITIES 32 -+#define AURORA_BIO_CHALLENGE_LEN 32 -+#define AURORA_BIO_ATTEST_PUB_LEN 65 -+#define AURORA_BIO_ATTEST_SIG_MAX 72 -+ -+enum { -+ AURORA_BIO_STATE_IDLE = 0, -+ AURORA_BIO_STATE_PENDING = 1, -+ AURORA_BIO_STATE_PROGRESS = 2, -+ AURORA_BIO_STATE_DONE = 3, -+ AURORA_BIO_STATE_FAILED = 4, -+}; -+ -+ -+enum { -+ AURORA_BIO_NO_MATCH = 0, -+ AURORA_BIO_MATCH = 1, -+ AURORA_BIO_NOT_COMPARED = 2, -+}; -+ -+struct aurora_bio_identity { -+ __u8 uuid[AURORA_BIO_UUID_LEN]; -+ __u8 label[AURORA_BIO_LABEL_LEN]; -+}; -+ -+struct aurora_bio_info { -+ __u32 version; -+ __u32 sensor_present; -+ __u32 enrolled; -+ __u32 capacity; -+ __u32 enroll_stages; -+ __u32 reserved[3]; -+}; -+ -+struct aurora_bio_list { -+ __u32 count; -+ __u32 reserved; -+ struct aurora_bio_identity id[AURORA_BIO_MAX_IDENTITIES]; -+}; -+ -+struct aurora_bio_enrol_start { -+ __u32 flags; -+ __u32 reserved; -+ __u8 label[AURORA_BIO_LABEL_LEN]; -+}; -+ -+#define AURORA_BIO_GUIDANCE_NONE 0 -+#define AURORA_BIO_GUIDANCE_PLACE 1 -+#define AURORA_BIO_GUIDANCE_LIFT_AND_MOVE 2 -+#define AURORA_BIO_GUIDANCE_HOLD_STILL 3 -+ -+struct aurora_bio_enrol_poll { -+ __u32 state; -+ __u32 stage; -+ __u32 stages_total; -+ __u32 status; -+ __u8 uuid[AURORA_BIO_UUID_LEN]; -+ __u32 guidance; -+ __u32 progress_percent; -+}; -+struct aurora_bio_verify_start { -+ __u32 flags; -+ __u32 reserved; -+ __u8 nonce[AURORA_BIO_NONCE_LEN]; -+}; -+ -+struct aurora_bio_verify_poll { -+ __u32 state; -+ __u32 result; -+ __u32 status; -+ __u32 reserved; -+ __u8 uuid[AURORA_BIO_UUID_LEN]; -+ __u8 token[AURORA_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ -+ __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ -+}; -+ -+struct aurora_bio_delete { -+ __u8 uuid[AURORA_BIO_UUID_LEN]; -+}; -+ -+/* -+ * Device attestation of key possession: the enclave signs 'challenge' with the -+ * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and -+ * returns the DER signature and public point. The private key never leaves the -+ * enclave. -+ */ -+struct aurora_bio_attest { -+ __u32 sig_len; /* out: DER signature length */ -+ __u8 challenge[AURORA_BIO_CHALLENGE_LEN]; /* in */ -+ __u8 public[AURORA_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ -+ __u8 signature[AURORA_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ -+ __u8 reserved[3]; -+}; -+ -+#define AURORA_BIO_IOC_MAGIC 0xB1 -+ -+#define AURORA_BIO_GET_INFO _IOR (AURORA_BIO_IOC_MAGIC, 0x01, struct aurora_bio_info) -+#define AURORA_BIO_LIST _IOR (AURORA_BIO_IOC_MAGIC, 0x02, struct aurora_bio_list) -+#define AURORA_BIO_ENROL_START _IOW (AURORA_BIO_IOC_MAGIC, 0x03, struct aurora_bio_enrol_start) -+#define AURORA_BIO_ENROL_POLL _IOR (AURORA_BIO_IOC_MAGIC, 0x04, struct aurora_bio_enrol_poll) -+#define AURORA_BIO_VERIFY_START _IOW (AURORA_BIO_IOC_MAGIC, 0x05, struct aurora_bio_verify_start) -+#define AURORA_BIO_VERIFY_POLL _IOR (AURORA_BIO_IOC_MAGIC, 0x06, struct aurora_bio_verify_poll) -+#define AURORA_BIO_CANCEL _IO (AURORA_BIO_IOC_MAGIC, 0x07) -+#define AURORA_BIO_DELETE _IOW (AURORA_BIO_IOC_MAGIC, 0x08, struct aurora_bio_delete) -+#define AURORA_BIO_DELETE_ALL _IO (AURORA_BIO_IOC_MAGIC, 0x09) -+#define AURORA_BIO_ATTEST _IOWR(AURORA_BIO_IOC_MAGIC, 0x0a, struct aurora_bio_attest) diff --git a/libfprint/drivers/aurora/aurora.c b/libfprint/drivers/aurora/aurora.c new file mode 100644 index 0000000..80ace06 @@ -152,7 +17,7 @@ index 0000000..80ace06 +#define FP_COMPONENT "apple-sep" + +#include "drivers_api.h" -+#include "aurora-bio.h" ++#include + +#include +#include @@ -172,7 +37,7 @@ index 0000000..80ace06 + guint watch_id; + guint32 enroll_last_reported_stage; + -+ guint8 nonce[AURORA_BIO_NONCE_LEN]; ++ guint8 nonce[SEP_BIO_NONCE_LEN]; + gboolean nonce_valid; +}; + @@ -248,14 +113,14 @@ index 0000000..80ace06 + + user_id = fpi_print_generate_user_id (print); + -+ if (user_id == NULL || strlen (user_id) >= AURORA_BIO_LABEL_LEN) ++ if (user_id == NULL || strlen (user_id) >= SEP_BIO_LABEL_LEN) + { + g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, + "print label does not fit the device record"); + return FALSE; + } + -+ memset (label_out, 0, AURORA_BIO_LABEL_LEN); ++ memset (label_out, 0, SEP_BIO_LABEL_LEN); + memcpy (label_out, user_id, strlen (user_id)); + return TRUE; +} @@ -281,7 +146,7 @@ index 0000000..80ace06 + GVariant *data; + + data = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, uuid, -+ AURORA_BIO_UUID_LEN, 1); ++ SEP_BIO_UUID_LEN, 1); + + fpi_print_set_type (print, FPI_PRINT_RAW); + fpi_print_set_device_stored (print, TRUE); @@ -291,10 +156,10 @@ index 0000000..80ace06 + * the buffer before anything reads it as a string. */ + if (label != NULL) + { -+ gchar safe[AURORA_BIO_LABEL_LEN]; ++ gchar safe[SEP_BIO_LABEL_LEN]; + -+ memcpy (safe, label, AURORA_BIO_LABEL_LEN); -+ safe[AURORA_BIO_LABEL_LEN - 1] = '\0'; ++ memcpy (safe, label, SEP_BIO_LABEL_LEN); ++ safe[SEP_BIO_LABEL_LEN - 1] = '\0'; + + if (safe[0] != '\0') + fpi_print_fill_from_user_id (print, safe); @@ -320,14 +185,14 @@ index 0000000..80ace06 + } + + raw = g_variant_get_fixed_array (data, &len, 1); -+ if (raw == NULL || len != AURORA_BIO_UUID_LEN) ++ if (raw == NULL || len != SEP_BIO_UUID_LEN) + { + g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, + "enclave identity is the wrong size"); + return FALSE; + } + -+ memcpy (uuid_out, raw, AURORA_BIO_UUID_LEN); ++ memcpy (uuid_out, raw, SEP_BIO_UUID_LEN); + return TRUE; +} + @@ -342,7 +207,7 @@ index 0000000..80ace06 +static gboolean +aurora_print_has_uuid (FpPrint *print, const guint8 *uuid) +{ -+ guint8 stored[AURORA_BIO_UUID_LEN]; ++ guint8 stored[SEP_BIO_UUID_LEN]; + + if (print == NULL) + return FALSE; @@ -350,7 +215,7 @@ index 0000000..80ace06 + if (!aurora_uuid_from_print (print, stored, NULL)) + return FALSE; + -+ return memcmp (stored, uuid, AURORA_BIO_UUID_LEN) == 0; ++ return memcmp (stored, uuid, SEP_BIO_UUID_LEN) == 0; +} + +/* ------------------------------------------------------------------ */ @@ -360,7 +225,7 @@ index 0000000..80ace06 +static void +aurora_probe (FpDevice *device) +{ -+ struct aurora_bio_info info = { 0 }; ++ struct sep_bio_info info = { 0 }; + const gchar *path; + g_autofree gchar *serial = NULL; + gint fd; @@ -384,7 +249,7 @@ index 0000000..80ace06 + return; + } + -+ if (ioctl (fd, AURORA_BIO_GET_INFO, &info) < 0) ++ if (ioctl (fd, SEP_BIO_GET_INFO, &info) < 0) + { + gint err = errno; + @@ -398,13 +263,13 @@ index 0000000..80ace06 + + close (fd); + -+ if (info.version != AURORA_BIO_IFACE_VERSION) ++ if (info.version != SEP_BIO_IFACE_VERSION) + { + fpi_device_probe_complete (device, NULL, NULL, + fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, + "interface version %u, expected %u", + info.version, -+ AURORA_BIO_IFACE_VERSION)); ++ SEP_BIO_IFACE_VERSION)); + return; + } + @@ -488,7 +353,7 @@ index 0000000..80ace06 + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); + + if (self->fd >= 0) -+ ioctl (self->fd, AURORA_BIO_CANCEL); ++ ioctl (self->fd, SEP_BIO_CANCEL); +} + +/* ------------------------------------------------------------------ */ @@ -500,11 +365,11 @@ index 0000000..80ace06 +{ + FpDevice *device = FP_DEVICE (user_data); + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct aurora_bio_enrol_poll poll_res = { 0 }; ++ struct sep_bio_enrol_poll poll_res = { 0 }; + g_autoptr(GError) error = NULL; + FpPrint *print = NULL; + -+ if (!aurora_ioctl (self, AURORA_BIO_ENROL_POLL, &poll_res, &error)) ++ if (!aurora_ioctl (self, SEP_BIO_ENROL_POLL, &poll_res, &error)) + { + self->watch_id = 0; + fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); @@ -513,10 +378,10 @@ index 0000000..80ace06 + + switch (poll_res.state) + { -+ case AURORA_BIO_STATE_PENDING: ++ case SEP_BIO_STATE_PENDING: + return G_SOURCE_CONTINUE; + -+ case AURORA_BIO_STATE_PROGRESS: ++ case SEP_BIO_STATE_PROGRESS: + /* fprintd calls this "enroll-stage-passed". A guidance wakeup or an + * initial stage-zero event must not masquerade as a captured sample. */ + if (poll_res.stage > self->enroll_last_reported_stage) @@ -526,13 +391,13 @@ index 0000000..80ace06 + } + return G_SOURCE_CONTINUE; + -+ case AURORA_BIO_STATE_DONE: ++ case SEP_BIO_STATE_DONE: + fpi_device_get_enroll_data (device, &print); + if (print != NULL) + { + GVariant *data = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, + poll_res.uuid, -+ AURORA_BIO_UUID_LEN, 1); ++ SEP_BIO_UUID_LEN, 1); + fpi_print_set_type (print, FPI_PRINT_RAW); + fpi_print_set_device_stored (print, TRUE); + g_object_set (print, "fpi-data", data, NULL); @@ -545,7 +410,7 @@ index 0000000..80ace06 + "enrolment produced no print")); + return G_SOURCE_REMOVE; + -+ case AURORA_BIO_STATE_FAILED: ++ case SEP_BIO_STATE_FAILED: + default: + self->watch_id = 0; + fpi_device_enroll_complete (device, NULL, @@ -560,7 +425,7 @@ index 0000000..80ace06 +aurora_enroll (FpDevice *device) +{ + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct aurora_bio_enrol_start start = { 0 }; ++ struct sep_bio_enrol_start start = { 0 }; + g_autoptr(GError) error = NULL; + FpPrint *print = NULL; + @@ -572,7 +437,7 @@ index 0000000..80ace06 + return; + } + -+ if (!aurora_ioctl (self, AURORA_BIO_ENROL_START, &start, &error)) ++ if (!aurora_ioctl (self, SEP_BIO_ENROL_START, &start, &error)) + { + fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); + return; @@ -598,23 +463,23 @@ index 0000000..80ace06 + * might interpret loosely. */ +static gboolean +aurora_result_is_match (FpiDeviceAurora *self, -+ const struct aurora_bio_verify_poll *res) ++ const struct sep_bio_verify_poll *res) +{ + struct timespec now; + guint64 now_ns; + gboolean token_set = FALSE; + gsize i; + -+ if (res->state != AURORA_BIO_STATE_DONE) ++ if (res->state != SEP_BIO_STATE_DONE) + return FALSE; + -+ if (res->result != AURORA_BIO_MATCH) ++ if (res->result != SEP_BIO_MATCH) + return FALSE; + + if (!self->nonce_valid) + return FALSE; + -+ for (i = 0; i < AURORA_BIO_TOKEN_LEN; i++) ++ for (i = 0; i < SEP_BIO_TOKEN_LEN; i++) + if (res->token[i] != 0) + { + token_set = TRUE; @@ -642,7 +507,7 @@ index 0000000..80ace06 +{ + FpDevice *device = FP_DEVICE (user_data); + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct aurora_bio_verify_poll res = { 0 }; ++ struct sep_bio_verify_poll res = { 0 }; + FpiDeviceAction action = fpi_device_get_current_action (device); + g_autoptr(GError) error = NULL; + /* Not a g_autoptr: FpPrint is a GInitiallyUnowned and both report functions @@ -651,7 +516,7 @@ index 0000000..80ace06 + * match completes. */ + FpPrint *matched = NULL; + -+ if (!aurora_ioctl (self, AURORA_BIO_VERIFY_POLL, &res, &error)) ++ if (!aurora_ioctl (self, SEP_BIO_VERIFY_POLL, &res, &error)) + { + self->watch_id = 0; + /* Terminal path: void the per-operation nonce like every other exit, so @@ -665,8 +530,8 @@ index 0000000..80ace06 + return G_SOURCE_REMOVE; + } + -+ if (res.state == AURORA_BIO_STATE_PENDING || -+ res.state == AURORA_BIO_STATE_PROGRESS) ++ if (res.state == SEP_BIO_STATE_PENDING || ++ res.state == SEP_BIO_STATE_PROGRESS) + return G_SOURCE_CONTINUE; + + self->watch_id = 0; @@ -686,7 +551,7 @@ index 0000000..80ace06 + fpi_device_verify_report (device, FPI_MATCH_SUCCESS, matched, NULL); + fpi_device_verify_complete (device, NULL); + } -+ else if (res.result == AURORA_BIO_NOT_COMPARED) ++ else if (res.result == SEP_BIO_NOT_COMPARED) + { + /* No comparison happened, so there is no verdict to report. Saying + * FAIL here would tell the user their finger was rejected when it @@ -723,7 +588,7 @@ index 0000000..80ace06 + } + } + -+ if (matched == NULL && res.result == AURORA_BIO_NOT_COMPARED) ++ if (matched == NULL && res.result == SEP_BIO_NOT_COMPARED) + { + fpi_device_identify_complete (device, + fpi_device_error_new (FP_DEVICE_ERROR_GENERAL)); @@ -745,7 +610,7 @@ index 0000000..80ace06 +aurora_match_start (FpDevice *device) +{ + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct aurora_bio_verify_start start = { 0 }; ++ struct sep_bio_verify_start start = { 0 }; + FpiDeviceAction action = fpi_device_get_current_action (device); + g_autoptr(GError) error = NULL; + FpPrint *print = NULL; @@ -805,7 +670,7 @@ index 0000000..80ace06 + + memcpy (start.nonce, self->nonce, sizeof (start.nonce)); + -+ if (!aurora_ioctl (self, AURORA_BIO_VERIFY_START, &start, &error)) ++ if (!aurora_ioctl (self, SEP_BIO_VERIFY_START, &start, &error)) + { + self->nonce_valid = FALSE; + if (action == FPI_DEVICE_ACTION_VERIFY) @@ -827,18 +692,18 @@ index 0000000..80ace06 +aurora_list (FpDevice *device) +{ + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct aurora_bio_list list = { 0 }; ++ struct sep_bio_list list = { 0 }; + g_autoptr(GError) error = NULL; + GPtrArray *prints; + guint32 i; + -+ if (!aurora_ioctl (self, AURORA_BIO_LIST, &list, &error)) ++ if (!aurora_ioctl (self, SEP_BIO_LIST, &list, &error)) + { + fpi_device_list_complete (device, NULL, g_steal_pointer (&error)); + return; + } + -+ if (list.count > AURORA_BIO_MAX_IDENTITIES) ++ if (list.count > SEP_BIO_MAX_IDENTITIES) + { + fpi_device_list_complete (device, NULL, + fpi_device_error_new_msg (FP_DEVICE_ERROR_PROTO, @@ -863,7 +728,7 @@ index 0000000..80ace06 +aurora_delete (FpDevice *device) +{ + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct aurora_bio_delete del = { 0 }; ++ struct sep_bio_delete del = { 0 }; + g_autoptr(GError) error = NULL; + FpPrint *print = NULL; + @@ -878,7 +743,7 @@ index 0000000..80ace06 + return; + } + -+ if (!aurora_ioctl (self, AURORA_BIO_DELETE, &del, &error)) ++ if (!aurora_ioctl (self, SEP_BIO_DELETE, &del, &error)) + { + fpi_device_delete_complete (device, g_steal_pointer (&error)); + return; @@ -893,7 +758,7 @@ index 0000000..80ace06 + FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); + g_autoptr(GError) error = NULL; + -+ if (!aurora_ioctl (self, AURORA_BIO_DELETE_ALL, NULL, &error)) ++ if (!aurora_ioctl (self, SEP_BIO_DELETE_ALL, NULL, &error)) + { + fpi_device_clear_storage_complete (device, g_steal_pointer (&error)); + return; From 8896264380c3bb16e4f68364f82368b65cda7737 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:31:38 +1100 Subject: [PATCH 0537/1579] Bluetooth: mt7932: support modules and system suspend The MT7932 PCIe transport cannot be selected with modular Bluetooth or system sleep enabled. Make the existing transport tristate and coordinate firmware quiescence with HCI suspend before PCI power transitions. Wait for the firmware's system-sleep acknowledgment before completing suspend. Restore the transport before resuming HCI, and undo the sleep request if suspend fails. Keep ring publication stopped while quiesced and preserve the DMA arena across sleep. Invalid notifications and failed restoration close the transport instead of retrying an uncertain state. Unregister HCI, quiesce firmware and synchronize transport work and interrupts on removal. Release device-visible storage only after bus mastering is disabled and pending PCI transactions drain; retain it if that cannot be confirmed. Hibernation, firmware recovery and warm reprobe remain unsupported. This extends the existing MT7932 transport and preserves its contributor notices and firmware/calibration path. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 24 +- drivers/bluetooth/Kconfig | 19 +- drivers/bluetooth/mt7932_bt_pcie.c | 265 +++++++++++++++--- 3 files changed, 259 insertions(+), 49 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 594f513ba2a2e..c8d641aadfda2 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -21,17 +21,21 @@ Use the separately reviewed T8140 PCIe bootstrap described in ``wifi0`` endpoint's own ``local-mac-address``; a zero placeholder is rejected. The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, -``CONFIG_CFG80211=y``, ``CONFIG_BT_MTK7932_PCIE=y``, ``CONFIG_BT_BREDR=y``, +``CONFIG_CFG80211=y``, ``CONFIG_BT_MTK7932_PCIE=m``, ``CONFIG_BT_BREDR=y``, ``CONFIG_BT_LE=y``, ``CONFIG_CRYPTO_AES=y`` and ``CONFIG_CRYPTO_CMAC=y``. -The bootstrap and Bluetooth experiment exclude suspend and kexec. +The Bluetooth transport can be built with modular Bluetooth and system +sleep. The separately selected bootstrap provider has its own lifecycle +restrictions. Kexec is excluded while either retained experiment is enabled. Use the ordinary cfg80211 regulatory database and applicable country policy. The validated first-admission fallback is kernel country 00 with firmware XZ. After the root filesystem and the local firmware packages are available, select PCIe ASPM performance policy before loading ``mt7932-fullmac``. This matches the tested admission sequence. Bluetooth's gate defaults closed; -validate the cold, unbound ``14c3:793b`` function, enable -``/sys/module/mt7932_bt_pcie/parameters/enable``, then request its PCI probe. +validate the cold, unbound ``14c3:793b`` function, then load +``mt7932_bt_pcie`` with ``enable=1`` after its local inputs are installed. +A built-in transport can instead open its enable parameter before requesting +the PCI probe. Do not reprobe after a failed or uncertain Bluetooth admission. The tested Wi-Fi driver owns function 0 and Bluetooth owns function 1. @@ -90,9 +94,15 @@ Limitations * PCI bootstrap memory remains retained until external reset. Controller removal, memory reuse, suspend and kexec are unqualified. -* Bluetooth PCI removal/quiescence is incomplete. Its software queue limit - does not provide HCI backpressure; saturation can drop an accounted frame. - Both require correction before production use. +* Bluetooth system sleep uses a bounded firmware quiesce/restore handshake + around HCI suspend/resume, preserving its DMA arena. Hibernation is rejected. + Removal first quiesces firmware, then stops work and interrupts, disables + bus mastering and drains pending + PCI transactions before freeing DMA. An unconfirmed drain retains storage + until reset. Firmware recovery and warm reprobe are unsupported; use a fresh + cold boot after removing or faulting an active transport. +* The Bluetooth software queue limit does not provide HCI backpressure; + saturation can drop an accounted frame. * Arbitrary scan IEs, WPA3/SAE, required MFP, AP/P2P, general country-package generation, roaming and long-duration reliability are unqualified. * SCO/headset microphone, LE Audio/ISO and simultaneous headset audio are diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 1f71325c694be..3d350157a6123 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -24,16 +24,19 @@ config BT_MTK select FW_LOADER config BT_MTK7932_PCIE - bool "J700 MT7932 PCIe Bluetooth experiment" - depends on BT=y && PCI && ARCH_APPLE && IOMMU_DMA - depends on !PM_SLEEP && !KEXEC_CORE + tristate "J700 MT7932 PCIe Bluetooth transport" + depends on BT && PCI && ARCH_APPLE && IOMMU_DMA + depends on !KEXEC_CORE select FW_LOADER help - Build the standalone J700 MT7932 transport experiment. Its probe gate - defaults closed so board firmware can remain on the real root filesystem. - It retains all DMA memory after activation, including terminal failures, - until a full external reset. Unbind and automatic recovery are excluded. - This does not qualify suspend or physical removal. + Support the J700 Bluetooth PCIe function with locally installed firmware + and calibration. The late-probe enable parameter defaults off. + System suspend uses the firmware sleep handshake and preserves DMA + storage. Hibernation and automatic firmware recovery are unsupported. + Removal drains the PCI function before releasing device-visible memory; + an unconfirmed drain retains it until a full hardware reset. + + To build as a module, choose M. The module is mt7932_bt_pcie. config BT_HCIBTUSB tristate "HCI USB driver" diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index cb88d33873c90..493f95cbcda09 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -4,9 +4,9 @@ * protocol contracts and public Linux interfaces. Original transport research: * DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. * - * The built-in driver retains every device-visible allocation after mastering - * is enabled, including all terminal failures. No unbind, retry, suspend or - * quiescence claim is made. Firmware and board inputs are supplied locally. + * Firmware and board inputs are supplied locally. System sleep preserves the + * transport arena and uses the firmware quiesce/restore handshake. Unload + * releases DMA only after the PCI function has stopped bus mastering. */ #include #include @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -94,6 +95,13 @@ struct bt7932_ring { u16 previous; }; +enum bt7932_pm_state { + BT7932_RUNNING, + BT7932_QUIESCING, + BT7932_QUIESCED, + BT7932_RESTORING, +}; + struct bt7932 { struct pci_dev *pdev; void __iomem *bar; @@ -121,6 +129,8 @@ struct bt7932 { int irq; bool enabled, regions, vectors, irq_requested, retained; bool ipc_ready, opened, awake, waking, fault, registered; + bool stopping, runtime_forbidden; + enum bt7932_pm_state pm_state; u64 tx_packets, rx_packets, diagnostics; }; @@ -395,6 +405,13 @@ static int bt7932_arena(struct bt7932 *bt) return 0; } +/* Serialize control requests with sleep notifications and ring publication. */ +static void bt7932_request_sleep(struct bt7932 *bt, u32 control) +{ + writel(control, bt->bar + BT7932_SLEEP); + bt7932_ring_doorbell(bt, 12); +} + /* Caller owns lock; no HCI command wait can run from the IRQ thread. */ static int bt7932_sleep_locked(struct bt7932 *bt) { @@ -403,21 +420,46 @@ static int bt7932_sleep_locked(struct bt7932 *bt) dma_rmb(); value = le32_to_cpu(READ_ONCE(*notification)); + if (value == 3) { + if (bt->pm_state == BT7932_QUIESCING) { + WRITE_ONCE(bt->pm_state, BT7932_QUIESCED); + WRITE_ONCE(bt->awake, false); + wake_up_all(&bt->wake_wait); + } else if (bt->pm_state != BT7932_QUIESCED && + bt->pm_state != BT7932_RESTORING) { + return -EPROTO; + } + return 0; + } if (value > 1) return -EPROTO; - if (!value) { - if (bt->last_sleep != value || bt->waking) { - writel(0, bt->bar + BT7932_SLEEP); - bt7932_ring_doorbell(bt, 12); + /* A previous host-sleep notification must not cancel control 3. */ + if (bt->pm_state == BT7932_QUIESCING) + return 0; + if (bt->pm_state == BT7932_QUIESCED) + return -EPROTO; + if (bt->pm_state == BT7932_RESTORING) { + if (value) { + if (!bt->waking) { + bt->waking = true; + bt7932_request_sleep(bt, 1); + bt7932_request_sleep(bt, 2); + } + return 0; } + bt->waking = true; + WRITE_ONCE(bt->pm_state, BT7932_RUNNING); + } + if (!value) { + if (bt->last_sleep != value || bt->waking) + bt7932_request_sleep(bt, 0); WRITE_ONCE(bt->awake, true); bt->waking = false; wake_up_all(&bt->wake_wait); } else { WRITE_ONCE(bt->awake, false); if (!bt->waking && bt->last_sleep != value) { - writel(1, bt->bar + BT7932_SLEEP); - bt7932_ring_doorbell(bt, 12); + bt7932_request_sleep(bt, 1); } } bt->last_sleep = value; @@ -425,7 +467,7 @@ static int bt7932_sleep_locked(struct bt7932 *bt) } /* On success returns with lock held, covering the final check/publication. */ -static int bt7932_wake_and_lock(struct bt7932 *bt) +static int bt7932_wake_and_lock(struct bt7932 *bt, bool require_open) { unsigned long deadline = jiffies + HZ; unsigned long now; @@ -433,9 +475,10 @@ static int bt7932_wake_and_lock(struct bt7932 *bt) for (;;) { mutex_lock(&bt->lock); - if (bt->fault || !bt->opened) { + if (bt->fault || bt->stopping || bt->pm_state != BT7932_RUNNING || + (require_open && !bt->opened)) { mutex_unlock(&bt->lock); - return -EIO; + return -EHOSTDOWN; } ret = bt7932_sleep_locked(bt); if (ret) @@ -444,8 +487,7 @@ static int bt7932_wake_and_lock(struct bt7932 *bt) return 0; if (!bt->waking) { bt->waking = true; - writel(2, bt->bar + BT7932_SLEEP); - bt7932_ring_doorbell(bt, 12); + bt7932_request_sleep(bt, 2); } mutex_unlock(&bt->lock); now = jiffies; @@ -455,7 +497,9 @@ static int bt7932_wake_and_lock(struct bt7932 *bt) goto fail; } wait_event_timeout(bt->wake_wait, - READ_ONCE(bt->awake) || READ_ONCE(bt->fault) || !READ_ONCE(bt->opened), + READ_ONCE(bt->awake) || READ_ONCE(bt->fault) || + READ_ONCE(bt->stopping) || + (require_open && !READ_ONCE(bt->opened)), min_t(unsigned long, msecs_to_jiffies(10), deadline - now)); } fail: @@ -499,7 +543,7 @@ static void bt7932_tx_work(struct work_struct *work) u16 consumer; u8 *descriptor; - if (bt7932_wake_and_lock(bt)) { + if (bt7932_wake_and_lock(bt, true)) { kfree_skb(skb); skb_queue_purge(&bt->tx_queue); return; @@ -601,7 +645,7 @@ static irqreturn_t bt7932_irq_thread(int irq, void *data) int channel, ret; mutex_lock(&bt->lock); - if (!bt->ipc_ready || bt->fault) + if (!bt->ipc_ready || bt->fault || bt->stopping) goto out; if (READ_ONCE(*(u8 *)(bt->arena + bt->peripheral + 4)) == 4) { bt7932_fault_locked(bt, "firmware entered fault state4"); @@ -612,6 +656,9 @@ static irqreturn_t bt7932_irq_thread(int irq, void *data) bt7932_fault_locked(bt, "invalid sleep notification"); goto out; } + /* Drain the old offers until quiescence, then stop all publication. */ + if (bt->pm_state == BT7932_QUIESCED || bt->pm_state == BT7932_RESTORING) + goto out; producer = bt7932_counter(bt, bt->cr_producer, 0); if (producer != bt->cr_previous) { /* Counter retirement only; no invented per-record ACK interpretation. */ @@ -778,8 +825,9 @@ static int bt7932_open(struct hci_dev *hdev) int ret = 0; mutex_lock(&bt->lock); - if (bt->fault || !bt->ipc_ready) - ret = -EIO; + if (bt->fault || !bt->ipc_ready || bt->stopping || + bt->pm_state != BT7932_RUNNING) + ret = -EHOSTDOWN; else WRITE_ONCE(bt->opened, true); mutex_unlock(&bt->lock); @@ -811,8 +859,9 @@ static int bt7932_send(struct hci_dev *hdev, struct sk_buff *skb) unsigned long flags; int ret; - if (READ_ONCE(bt->fault) || !READ_ONCE(bt->opened)) - return -EIO; + if (READ_ONCE(bt->fault) || !READ_ONCE(bt->opened) || + READ_ONCE(bt->stopping) || READ_ONCE(bt->pm_state) != BT7932_RUNNING) + return -EHOSTDOWN; if (channel < 0 || skb->len > geometry[channel].payload) return -EMSGSIZE; ret = skb_linearize(skb); @@ -821,6 +870,10 @@ static int bt7932_send(struct hci_dev *hdev, struct sk_buff *skb) if (!bt7932_hci_length(hci_skb_pkt_type(skb), skb->data, skb->len)) return -EINVAL; spin_lock_irqsave(&bt->tx_queue.lock, flags); + if (READ_ONCE(bt->stopping) || READ_ONCE(bt->pm_state) != BT7932_RUNNING) { + spin_unlock_irqrestore(&bt->tx_queue.lock, flags); + return -EHOSTDOWN; + } if (bt->tx_queue.qlen >= BT7932_TX_QUEUE_MAX) { spin_unlock_irqrestore(&bt->tx_queue.lock, flags); return -EBUSY; @@ -954,10 +1007,8 @@ static int bt7932_iommu(struct bt7932 *bt) return ret; } -static void bt7932_unpublished_cleanup(struct bt7932 *bt) +static void bt7932_cleanup(struct bt7932 *bt, bool release_dma) { - if (WARN_ON_ONCE(bt->retained)) - return; if (bt->irq_requested) free_irq(bt->irq, bt); if (bt->vectors) @@ -966,9 +1017,9 @@ static void bt7932_unpublished_cleanup(struct bt7932 *bt) hci_free_dev(bt->hdev); if (bt->tx_wq) destroy_workqueue(bt->tx_wq); - if (bt->arena) + if (release_dma && bt->arena) dma_free_coherent(&bt->pdev->dev, bt->arena_size, bt->arena, bt->arena_dma); - if (bt->image) + if (release_dma && bt->image) dma_free_coherent(&bt->pdev->dev, bt->image_size, bt->image, bt->image_dma); kfree_sensitive(bt->calibration); kfree_sensitive(bt->ptx); @@ -976,6 +1027,8 @@ static void bt7932_unpublished_cleanup(struct bt7932 *bt) pci_iounmap(bt->pdev, bt->bar); if (bt->regions) pci_release_regions(bt->pdev); + if (bt->runtime_forbidden) + pm_runtime_allow(&bt->pdev->dev); if (bt->enabled) pci_disable_device(bt->pdev); kfree_sensitive(bt); @@ -1089,7 +1142,8 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) bt->irq_requested = true; pci_set_drvdata(pdev, bt); pm_runtime_forbid(&pdev->dev); - /* Every later outcome stays bound and retains all published ownership. */ + bt->runtime_forbidden = true; + /* Failed admission remains bound until removal or a full reset. */ bt->retained = true; pci_set_master(pdev); ret = bt7932_download(bt); @@ -1115,21 +1169,163 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) return 0; fail: dev_err(&pdev->dev, "BT_ADMISSION_FAILED error=%d; no DMA published\n", ret); - bt7932_unpublished_cleanup(bt); + bt7932_cleanup(bt, true); return ret; } -static void bt7932_shutdown(struct pci_dev *pdev) +/* Poll coherent notifications as well as IRQs, including a lost final edge. */ +static int bt7932_wait_pm(struct bt7932 *bt, enum bt7932_pm_state target) { - struct bt7932 *bt = pci_get_drvdata(pdev); + unsigned long limit = jiffies + HZ; + int ret; - WRITE_ONCE(bt->opened, false); + for (;;) { + mutex_lock(&bt->lock); + ret = bt->fault || bt->stopping ? -EIO : bt7932_sleep_locked(bt); + if (!ret && bt->pm_state == target) { + mutex_unlock(&bt->lock); + return 0; + } + mutex_unlock(&bt->lock); + if (ret) + return ret; + if (time_after_eq(jiffies, limit)) + return -ETIMEDOUT; + usleep_range(1000, 2000); + } +} + +static int bt7932_quiesce(struct bt7932 *bt) +{ + int ret; + + ret = bt7932_wake_and_lock(bt, false); + if (ret) + return ret; + WRITE_ONCE(bt->pm_state, BT7932_QUIESCING); + WRITE_ONCE(bt->awake, false); + bt7932_request_sleep(bt, 3); + mutex_unlock(&bt->lock); wake_up_all(&bt->wake_wait); cancel_work_sync(&bt->tx_work); skb_queue_purge(&bt->tx_queue); + return bt7932_wait_pm(bt, BT7932_QUIESCED); +} + +static int bt7932_restore_transport(struct bt7932 *bt) +{ + int ret; + + mutex_lock(&bt->lock); + if (bt->fault || bt->stopping) { + mutex_unlock(&bt->lock); + return -EIO; + } + WRITE_ONCE(bt->pm_state, BT7932_RESTORING); + bt->waking = false; + bt7932_request_sleep(bt, 0); + mutex_unlock(&bt->lock); + ret = bt7932_wait_pm(bt, BT7932_RUNNING); + if (ret) { + mutex_lock(&bt->lock); + bt7932_fault_locked(bt, "system sleep restoration failed"); + mutex_unlock(&bt->lock); + } + return ret; +} + +static int bt7932_suspend(struct device *dev) +{ + struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); + int ret; + + if (READ_ONCE(bt->fault)) + return -EIO; + if (!bt->registered) + return 0; + ret = hci_suspend_dev(bt->hdev); + if (ret) + return ret; + ret = bt7932_quiesce(bt); + if (!ret) { + dev_dbg(dev, "system sleep quiesce acknowledged\n"); + return 0; + } + dev_warn(dev, "system sleep quiesce failed: %d\n", ret); + /* Undo our request before the PM core rolls back the rest of the bus. */ + if (!READ_ONCE(bt->fault)) + bt7932_restore_transport(bt); + hci_resume_dev(bt->hdev); + return ret; +} + +static int bt7932_resume(struct device *dev) +{ + struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); + int ret; + + if (READ_ONCE(bt->fault)) + return -EIO; + if (!bt->registered) + return 0; + if (READ_ONCE(bt->pm_state) != BT7932_RUNNING) { + ret = bt7932_restore_transport(bt); + if (ret) + return ret; + } + dev_dbg(dev, "transport restored after system sleep\n"); + return hci_resume_dev(bt->hdev); +} + +static int bt7932_freeze(struct device *dev) +{ + struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); + + /* A restored image cannot adopt the running firmware's DMA pointers. */ + return bt->retained ? -EOPNOTSUPP : 0; +} + +static const struct dev_pm_ops bt7932_pm_ops = { + .suspend = bt7932_suspend, + .resume = bt7932_resume, + .freeze = bt7932_freeze, + .thaw = bt7932_resume, + .poweroff = bt7932_freeze, + .restore = bt7932_resume, +}; + +static void bt7932_remove(struct pci_dev *pdev) +{ + struct bt7932 *bt = pci_get_drvdata(pdev); + u16 command = U16_MAX; + bool quiesced, drained; + + /* Closing HCI may send Reset: keep the transport and IRQ alive for it. */ + if (bt->registered) + hci_unregister_dev(bt->hdev); + quiesced = bt->ipc_ready && !READ_ONCE(bt->fault) && !bt7932_quiesce(bt); mutex_lock(&bt->lock); - bt7932_fault_locked(bt, "shutdown retains DMA until external reset"); + WRITE_ONCE(bt->stopping, true); + WRITE_ONCE(bt->opened, false); mutex_unlock(&bt->lock); + wake_up_all(&bt->wake_wait); + cancel_work_sync(&bt->tx_work); + skb_queue_purge(&bt->tx_queue); + if (bt->irq_requested) { + free_irq(bt->irq, bt); + bt->irq_requested = false; + } + pci_clear_master(pdev); + drained = quiesced && !pci_read_config_word(pdev, PCI_COMMAND, &command) && + command != U16_MAX && !(command & PCI_COMMAND_MASTER) && + pci_wait_for_pending_transaction(pdev); + if (!drained && bt->retained) { + /* Keep the DMA allocations and their device alive until reset. */ + get_device(&pdev->dev); + dev_err(&pdev->dev, "PCI drain unconfirmed; DMA retained until reset\n"); + } + pci_set_drvdata(pdev, NULL); + bt7932_cleanup(bt, drained || !bt->retained); } static const struct pci_device_id bt7932_ids[] = { @@ -1142,9 +1338,10 @@ static struct pci_driver bt7932_driver = { .name = "mt7932_bt_pcie", .id_table = bt7932_ids, .probe = bt7932_probe, - .shutdown = bt7932_shutdown, + .remove = bt7932_remove, + .shutdown = bt7932_remove, .driver = { - .suppress_bind_attrs = true, + .pm = pm_sleep_ptr(&bt7932_pm_ops), }, }; module_pci_driver(bt7932_driver); From 955a9e4a6b6d9ce8364b36b4121432e61e01bbf3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:34:41 +1100 Subject: [PATCH 0538/1579] Bluetooth: hci_bcm4377: separate the driver and init declarations Add the declaration separator required by strict checkpatch. This is a formatting follow-up to the supplied recovery-work lifetime fix and does not change workqueue or module teardown behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/hci_bcm4377.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index fff9c2005d6bf..3347d877a1438 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -2645,6 +2645,7 @@ static struct pci_driver bcm4377_pci_driver = { .probe = bcm4377_probe, .driver.pm = &bcm4377_ops, }; + static int __init bcm4377_init(void) { int ret; From 039af3926ccff9ac2bc37b514b481589d4ba5732 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:34:39 +1100 Subject: [PATCH 0539/1579] HID: magicmouse: wrap the reviewed haptic mode restoration call Wrap the mode-restoration call to satisfy the kernel line-length check. This is a formatting follow-up to the supplied review fix; the haptic locking and firmware-controlled C1FE behavior are unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/hid-magicmouse.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index aa0e6cee6d8fd..c382f940ff094 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -2052,7 +2052,8 @@ static void apple_taptic_destroy(struct ff_device *ff) { guard(mutex)(&haptic_dev->auto_trigger_mutex); if (msc->haptics && READ_ONCE(msc->haptics->mode) == HID_HAPTIC_MODE_HOST) { - ret = magicmouse_switch_mode(msc->haptics->input_dev, HID_HAPTIC_MODE_DEVICE); + ret = magicmouse_switch_mode(msc->haptics->input_dev, + HID_HAPTIC_MODE_DEVICE); if (ret) hid_warn(msc->hdev, "failed to switch back to device-controlled mode: %d\n", From ac8a70914ef75b7a0b4c5352e6728a831f63b778 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:50:15 +1100 Subject: [PATCH 0540/1579] rtc: apple-pmu: leave the counter control state to firmware The alarm arm mask is described by the Linux-captured ADT. Setting an additional RUN bit is unnecessary for programming the comparator and changes the shared time counter's state. Update only the alarm arm bit, leaving the clock path and unrelated control bits unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/rtc/rtc-apple-pmu.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index f398a1a802165..7853b72a1b552 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -19,7 +19,6 @@ #define BAKU_RTC_COMPARE 0xf808 #define BAKU_RTC_EVENT 0xf80c #define BAKU_RTC_MASK 0xf80e -#define BAKU_RTC_RUN BIT(0) #define BAKU_RTC_ARM BIT(6) #define BAKU_RTC_ALARM BIT(0) #define BAKU_RTC_HZ_SHIFT 16 @@ -113,8 +112,7 @@ static int baku_rtc_start_alarm(struct baku_rtc *baku) error = regmap_update_bits(baku->map, BAKU_RTC_MASK, BAKU_RTC_ALARM, 0); if (!error) error = regmap_update_bits(baku->map, BAKU_RTC_CONTROL, - BAKU_RTC_RUN | BAKU_RTC_ARM, - BAKU_RTC_RUN | BAKU_RTC_ARM); + BAKU_RTC_ARM, BAKU_RTC_ARM); if (error) baku_rtc_stop_alarm(baku); return error; From 60ab6cd6b45e37b2f7f4828505760439b8a0fd0f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:12:21 +1100 Subject: [PATCH 0541/1579] MAINTAINERS: identify Aurora as the QSPIMC contact Point the QSPIMC maintenance entry and binding at Ryan Murray and the Aurora Silicon project site. Remove the mailing-list association from this controller's new maintenance entry. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/spi/apple,qspi.yaml | 2 +- MAINTAINERS | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/spi/apple,qspi.yaml b/Documentation/devicetree/bindings/spi/apple,qspi.yaml index 5f526fe3a7970..4d33549e7bf39 100644 --- a/Documentation/devicetree/bindings/spi/apple,qspi.yaml +++ b/Documentation/devicetree/bindings/spi/apple,qspi.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Apple ARM SoC QSPIMC controller maintainers: - - Asahi Linux + - Ryan Murray description: | Apple QSPIMC is a SPI-memory-only controller with a packed command FIFO. diff --git a/MAINTAINERS b/MAINTAINERS index 1e9a28969bee2..c684d47568a30 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2634,7 +2634,7 @@ F: net/bluetooth/brcm.h ARM/APPLE QSPIMC DRIVER M: Ryan Murray -L: asahi@lists.linux.dev +W: https://aurora-silicon.github.io S: Maintained F: Documentation/devicetree/bindings/spi/apple,qspi.yaml F: drivers/spi/spi-apple-qspi.c From a87814cc20907f021b4e63ce4d0b6f2a63932161 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:12:22 +1100 Subject: [PATCH 0542/1579] MAINTAINERS: place the J700 PMU RTC under Aurora support Track the new J700 PMU RTC driver and binding under the existing T8140 and J700 support entry. Keep the existing Apple machine maintenance entry focused on its original files. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 1d365477614d0..c7f105caa214a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1973,6 +1973,7 @@ F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml F: Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml +F: Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml F: Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml @@ -1985,6 +1986,7 @@ F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ +F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs F: drivers/thermal/apple-smc-cpu-thermal.c F: drivers/tty/serial/apple_dockchannel* @@ -2580,7 +2582,6 @@ F: Documentation/devicetree/bindings/power/apple* F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml F: Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml F: Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml -F: Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml F: Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml F: Documentation/devicetree/bindings/spmi/apple,spmi.yaml @@ -2615,7 +2616,6 @@ F: drivers/power/reset/macsmc-reboot.c F: drivers/power/supply/macsmc-power.c F: drivers/pwm/pwm-apple.c F: drivers/reset/reset-apple-cio.c -F: drivers/rtc/rtc-apple-pmu.c F: drivers/rtc/rtc-macsmc.c F: drivers/soc/apple/* F: drivers/spi/spi-apple.c From 1afd820cd7fcc775dc154ce4bfc1f955bc814bfc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:15:31 +1100 Subject: [PATCH 0543/1579] dt-bindings: mailbox: Describe the T8140 GPU ASCWrap interface Describe the existing GPU coprocessor mailbox interface, with wrapper and vector-base resources, two FIFO interrupts and firmware role names. The public T8140 GPU device tree already supplies these properties; without a binding, validation cannot decode the firmware-role string. The description follows the public GPU interface and J700 ADT resource layout. This does not change the GPU device tree or driver behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/mailbox/apple,ascwrap-v6.yaml | 82 +++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 83 insertions(+) create mode 100644 Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml diff --git a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml new file mode 100644 index 0000000000000..4798b62bd8430 --- /dev/null +++ b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mailbox/apple,ascwrap-v6.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 ASCWrap v6 mailbox + +maintainers: + - Ryan Murray + +description: + Mailbox interface of the T8140 GPU coprocessors. The wrapper resource + contains the coprocessor control registers and mailbox FIFOs. A separate + resource provides the coprocessor vector-base register. The firmware role + identifies which of the two GPU firmware instances uses the mailbox. + +properties: + compatible: + const: apple,t8140-ascwrap-v6 + + reg: + items: + - description: ASC wrapper registers + - description: Coprocessor vector-base register + + reg-names: + items: + - const: wrapper + - const: iop-vbar + + interrupts: + items: + - description: Send FIFO empty interrupt + - description: Receive FIFO not-empty interrupt + + interrupt-names: + items: + - const: send-empty + - const: recv-not-empty + + '#mbox-cells': + const: 0 + + power-domains: + maxItems: 1 + + apple,firmware-role: + enum: [GFX, GFX1] + +required: + - compatible + - reg + - reg-names + - interrupts + - interrupt-names + - '#mbox-cells' + - apple,firmware-role + +additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + mailbox@482600000 { + compatible = "apple,t8140-ascwrap-v6"; + reg = <0x4 0x82600000 0x0 0x88000>, + <0x4 0x82050000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; + #mbox-cells = <0>; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index e480d8a536878..6428fab298b2b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1972,6 +1972,7 @@ S: Maintained F: Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml +F: Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml F: Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml F: Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml From 97c4777bcb117201907dbcaf70f967bd24b25284 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:12:00 +1100 Subject: [PATCH 0544/1579] drm: apple: retain live H17P resources across component teardown An adopted H17P session keeps scanning after soft DPMS. Component unbind and bind-error unwind release the RTKit devres group even though firmware has not acknowledged a DMA stop. Driver removal can also drop its default IOMMU-domain usage and release AFK rings and consumer contexts. Export the existing default-domain usage helpers for module callers. Own that usage in the DCP driver and retain it, RTKit shared buffers, AFK rings, PIODMA and the referenced DRM objects until reset once negotiation with the live firmware starts. Quiesce host callbacks and AFK work without sending an unqualified firmware stop. Keep their contexts valid through managed cleanup and unplug the DRM device before teardown. Pin the DMA device, IOMMU group and module for that firmware lifetime, reject rebinding the same firmware node with an explicit reboot-required message, and keep the existing resource releases for profiles with a normal stop path. Pre-negotiation probe failures still release their resources. Add a native regression exercising the actual component/master unbind and resource-release functions through simulated component and driver devres cleanup, including partial initialization and legacy release behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 43 +++++- drivers/gpu/drm/apple/afk.h | 1 + drivers/gpu/drm/apple/apple_drv.c | 2 +- drivers/gpu/drm/apple/dcp-internal.h | 4 + drivers/gpu/drm/apple/dcp.c | 149 ++++++++++++++++++- drivers/gpu/drm/apple/dcp_backlight.c | 8 ++ drivers/iommu/iommu.c | 8 +- tools/testing/apple/check-lifetime.py | 199 ++++++++++++++++++++++++++ 8 files changed, 402 insertions(+), 12 deletions(-) create mode 100644 tools/testing/apple/check-lifetime.py diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 9440662d55788..27857ab068467 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -54,13 +55,22 @@ static void afk_send(struct apple_dcp_afkep *ep, u64 message) dcp_send_message(ep->dcp, ep->endpoint, message); } +static void afk_release_context(void *data) +{ + struct apple_dcp_afkep *afkep = data; + + afk_quiesce(afkep); + if (!afkep->dcp->retain_dma) + kfree(afkep); +} + struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, const struct apple_epic_service_ops *ops) { struct apple_dcp_afkep *afkep; int ret; - afkep = devm_kzalloc(dcp->dev, sizeof(*afkep), GFP_KERNEL); + afkep = kzalloc_obj(*afkep); if (!afkep) return ERR_PTR(-ENOMEM); @@ -74,19 +84,32 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, goto out_free_afkep; } - // TODO: devm_ for wq - init_completion(&afkep->started); init_completion(&afkep->stopped); spin_lock_init(&afkep->lock); + ret = devm_add_action_or_reset(dcp->dev, afk_release_context, afkep); + if (ret) + return ERR_PTR(ret); return afkep; out_free_afkep: - devm_kfree(dcp->dev, afkep); + kfree(afkep); return ERR_PTR(ret); } +void afk_quiesce(struct apple_dcp_afkep *afkep) +{ + if (IS_ERR_OR_NULL(afkep)) + return; + if (afkep->wq) { + destroy_workqueue(afkep->wq); + afkep->wq = NULL; + } + debugfs_remove_recursive(afkep->debugfs_entry); + afkep->debugfs_entry = NULL; +} + void afk_shutdown(struct apple_dcp_afkep *afkep) { afk_send(afkep, FIELD_PREP(RBEP_TYPE, RBEP_SHUTDOWN)); @@ -97,7 +120,7 @@ void afk_shutdown(struct apple_dcp_afkep *afkep) dev_err(afkep->dcp->dev, "Timed out shutting down AFK endpoint %02x", afkep->endpoint); } - destroy_workqueue(afkep->wq); + afk_quiesce(afkep); } int afk_start(struct apple_dcp_afkep *ep) @@ -131,8 +154,11 @@ static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) return; } - ep->bfr = dmam_alloc_coherent(ep->dcp->dev, size, &ep->bfr_dma, - GFP_KERNEL); + /* An unstopped adopted session owns this ring until the machine resets. */ + if (READ_ONCE(ep->dcp->retain_dma)) + ep->bfr = dma_alloc_coherent(ep->dcp->dev, size, &ep->bfr_dma, GFP_KERNEL); + else + ep->bfr = dmam_alloc_coherent(ep->dcp->dev, size, &ep->bfr_dma, GFP_KERNEL); if (!ep->bfr) { dev_err(ep->dcp->dev, "Failed to allocate %d bytes buffer\n", size); @@ -1011,6 +1037,9 @@ int afk_send_command(struct apple_epic_service *service, u8 type, struct apple_dcp_afkep *ep = service->ep; DECLARE_COMPLETION_ONSTACK(completion); + if (READ_ONCE(ep->dcp->quiescing)) + return -ENODEV; + rxbuf = dma_alloc_coherent(ep->dcp->dev, output_len, &rxbuf_dma, GFP_KERNEL); if (!rxbuf) diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 1e691ec47c85e..85085a3cc411c 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -224,6 +224,7 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, const struct apple_epic_service_ops *ops); int afk_start(struct apple_dcp_afkep *ep); void afk_shutdown(struct apple_dcp_afkep *ep); +void afk_quiesce(struct apple_dcp_afkep *ep); int afk_receive_message(struct apple_dcp_afkep *ep, u64 message); int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, enum epic_type etype, enum epic_category ecat, u8 stype, diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index aa815a8b3bf4f..ef3a45a73a075 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -752,7 +752,7 @@ static void apple_drm_uninit(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - drm_dev_unregister(&apple->drm); + drm_dev_unplug(&apple->drm); drm_atomic_helper_shutdown(&apple->drm); component_unbind_all(dev, NULL); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 0720570ddd1ba..5be5ba9c87387 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -180,6 +180,10 @@ struct apple_dcp_hw_data { /* TODO: move IOMFB members to its own struct */ struct apple_dcp { + /* An adopted session has no confirmed firmware DMA stop boundary. */ + bool retain_dma; + bool quiescing; + bool drm_retained; struct device *dev; struct platform_device *piodma; bool piodma_created; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index de436ed971101..e0082be652f3d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ #include #include +#include #include #include #include @@ -46,6 +48,83 @@ #define APPLE_DCP_COPROC_CPU_CONTROL 0x44 #define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) +struct dcp_session_ref { + struct list_head link; + struct device *dev; + struct iommu_group *group; +}; + +static LIST_HEAD(dcp_retained_sessions); +static DEFINE_MUTEX(dcp_retained_sessions_lock); + +static bool dcp_session_retained(struct device *dev) +{ + struct dcp_session_ref *session; + bool retained = false; + + mutex_lock(&dcp_retained_sessions_lock); + list_for_each_entry(session, &dcp_retained_sessions, link) { + if (dev_fwnode(session->dev) == dev_fwnode(dev)) { + retained = true; + break; + } + } + mutex_unlock(&dcp_retained_sessions_lock); + return retained; +} + +static int dcp_pin_live_session(struct apple_dcp *dcp) +{ + struct dcp_session_ref *session; + + if (!dcp->hw.adopt_live_session || dcp->fw_compat != DCP_FIRMWARE_H17P || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || dcp->retain_dma) + return 0; + session = kzalloc_obj(*session); + if (!session) + return -ENOMEM; + + /* Keep the DMA device, domain usage and driver code valid until reset. */ + session->dev = get_device(dcp->dev); + session->group = iommu_group_get(dcp->dev); + __module_get(THIS_MODULE); + WRITE_ONCE(dcp->retain_dma, true); + mutex_lock(&dcp_retained_sessions_lock); + list_add_tail(&session->link, &dcp_retained_sessions); + mutex_unlock(&dcp_retained_sessions_lock); + dev_info(dcp->dev, "live H17P DMA resources retained until reboot; rebind unsupported\n"); + return 0; +} + +static void dcp_release_dma_domain(void *data) +{ + struct apple_dcp *dcp = data; + + if (!dcp->retain_dma) + iommu_device_unuse_default_domain(dcp->dev); +} + +static void dcp_release_context(void *data) +{ + struct apple_dcp *dcp = data; + + if (!dcp->retain_dma) + kfree(dcp); +} + +static void dcp_release_rtkit(void *data) +{ + struct apple_dcp *dcp = data; + + if (dcp->retain_dma) { + WRITE_ONCE(dcp->quiescing, true); + apple_rtkit_free_retaining_buffers(dcp->rtk); + } else { + apple_rtkit_free(dcp->rtk); + } + dcp->rtk = NULL; +} + #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) static bool show_notch; @@ -850,6 +929,9 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) struct apple_dcp *dcp = cookie; struct apple_dcp_afkep *ep; + if (READ_ONCE(dcp->quiescing)) + return; + trace_dcp_recv_msg(dcp, endpoint, message); /* @@ -901,6 +983,9 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ { struct apple_dcp *dcp = cookie; + if (READ_ONCE(dcp->quiescing)) + return; + dcp->crashed = true; dev_err(dcp->dev, "DCP has crashed\n"); if (dcp->connector) { @@ -1004,6 +1089,8 @@ static struct apple_rtkit_ops rtkit_ops = { void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message) { + if (READ_ONCE(dcp->quiescing)) + return; trace_dcp_send_msg(dcp, endpoint, message); apple_rtkit_send_message(dcp->rtk, endpoint, message, NULL, true); @@ -1639,6 +1726,10 @@ static void dcp_work_update_backlight(struct work_struct *work) static void dcp_release_piodma_iommu_dev(struct apple_dcp *dcp) { + if (dcp->retain_dma) { + WRITE_ONCE(dcp->quiescing, true); + return; + } if (dcp->piodma) { if (dcp->piodma_created) of_platform_device_destroy(&dcp->piodma->dev, NULL); @@ -2045,6 +2136,10 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) u32 cpu_ctrl; int ret; + if (READ_ONCE(dcp->quiescing)) + return dev_err_probe(dev, -EBUSY, + "Live DCP session cannot be rebound; reboot required\n"); + /* A timed-out prior session may still scan these mappings. */ mutex_lock(&dcp->swapped_out_fbs_lock); ret = list_empty(&dcp->swapped_out_fbs) ? 0 : -EBUSY; @@ -2162,12 +2257,19 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); } - dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); + /* Registering the mailbox can already expose RTKit buffer requests. */ + ret = dcp_pin_live_session(dcp); + if (ret) + goto err_piodma; + dcp->rtk = apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); if (IS_ERR(dcp->rtk)) { ret = dev_err_probe(dev, PTR_ERR(dcp->rtk), "Failed to initialize RTKit\n"); goto err_piodma; } + ret = devm_add_action_or_reset(dev, dcp_release_rtkit, dcp); + if (ret) + goto err_piodma; if (dcp->hw.adopt_live_session) dev_info(dev, "negotiating RTKit with the running DCP\n"); @@ -2195,6 +2297,34 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (!dcp) return; + if (dcp->retain_dma) { + /* No firmware stop is acknowledged: detach host users, never DMA. */ + WRITE_ONCE(dcp->quiescing, true); + if (dcp->crtc && !dcp->drm_retained) { + drm_dev_get(dcp->crtc->base.dev); + dcp->drm_retained = true; + } + iomfb_queue_stop(dcp); + afk_quiesce(dcp->avep); + afk_quiesce(dcp->dptxep); + afk_quiesce(dcp->ibootep); + afk_quiesce(dcp->systemep); + afk_quiesce(dcp->dcpavservep); + if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { + cancel_work_sync(&dcp->bl_register_wq); + cancel_work_sync(&dcp->bl_update_wq); + } + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + cancel_work_sync(&dcp->vblank_wq); + if (dcp->connector) + cancel_work_sync(&dcp->connector->hotplug_wq); + if (dcp->typec_connector && dcp->typec_connector != dcp->connector) + cancel_work_sync(&dcp->typec_connector->hotplug_wq); + dev_warn(dev, "retaining live DCP DMA resources after unbind; reboot required\n"); + return; + } + if (dcp->hdmi_hpd_irq) disable_irq(dcp->hdmi_hpd_irq); @@ -2261,6 +2391,9 @@ static int dcp_platform_probe(struct platform_device *pdev) u32 surf_en; u32 mux_index; + if (dcp_session_retained(dev)) + return dev_err_probe(dev, -EBUSY, "Previous live DCP session requires a reboot\n"); + fw_compat = dcp_check_firmware_version(dev); if (fw_compat == DCP_FIRMWARE_UNKNOWN) return -ENODEV; @@ -2274,9 +2407,12 @@ static int dcp_platform_probe(struct platform_device *pdev) return dev_err_probe(dev, -ENODEV, "Incompatible devicetree! " "Use devicetree matching this kernel.\n"); - dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); + dcp = kzalloc_obj(*dcp); if (!dcp) return -ENOMEM; + ret = devm_add_action_or_reset(dev, dcp_release_context, dcp); + if (ret) + return ret; INIT_LIST_HEAD(&dcp->swapped_out_fbs); mutex_init(&dcp->swapped_out_fbs_lock); @@ -2286,6 +2422,14 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp->fw_compat = fw_compat; dcp->dev = dev; dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); + + /* The domain use must survive driver-core DMA cleanup for a live session. */ + ret = iommu_device_use_default_domain(dev); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, dcp_release_dma_domain, dcp); + if (ret) + return ret; dcp->fixed_connector_type = dcp_connector_type_from_dt(dev->of_node); dcp->connector_type = dcp->fixed_connector_type; of_property_read_u32(dev->of_node, "apple,dcp-index", &dcp->index); @@ -2556,6 +2700,7 @@ static const struct of_device_id of_match[] = { MODULE_DEVICE_TABLE(of, of_match); static struct platform_driver apple_platform_driver = { + .driver_managed_dma = true, .probe = dcp_platform_probe, .remove = dcp_platform_remove, .shutdown = dcp_platform_shutdown, diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 6a2ad5bcf3fb7..9d76154e13ce5 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -111,6 +111,9 @@ static int dcp_get_brightness(struct backlight_device *bd) unsigned long flags; u32 actual; + if (READ_ONCE(dcp->quiescing)) + return -ENODEV; + if (dcp_backlight_active(dcp)) { spin_lock_irqsave(&dcp->backlight.lock, flags); actual = dcp->backlight.state.actual; @@ -319,6 +322,8 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) int dcp_backlight_update(struct apple_dcp *dcp) { + if (READ_ONCE(dcp->quiescing)) + return -ENODEV; if (dcp_backlight_active(dcp)) { dcp_backlight_kick(dcp); return 0; @@ -350,6 +355,9 @@ static int dcp_set_brightness(struct backlight_device *bd) int brightness = backlight_get_brightness(bd); unsigned long flags; + if (READ_ONCE(dcp->quiescing)) + return -ENODEV; + if (dcp_backlight_active(dcp)) { /* Preserve the requested level even while the core forces zero. */ spin_lock_irqsave(&dcp->backlight.lock, flags); diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 87581d6778009..c8ca796ee542c 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -3350,10 +3350,12 @@ static ssize_t iommu_group_store_type(struct iommu_group *group, * * The device driver about to bind @dev wants to do DMA through the kernel * DMA API. Return 0 if it is allowed, otherwise an error. + * Drivers with driver_managed_dma may manage this usage themselves. The + * device lock must serialize probe and removal while holding the usage. */ int iommu_device_use_default_domain(struct device *dev) { - /* Caller is the driver core during the pre-probe path */ + /* Called with the device lock held during probe. */ struct iommu_group *group = dev->iommu_group; int ret = 0; @@ -3380,6 +3382,7 @@ int iommu_device_use_default_domain(struct device *dev) mutex_unlock(&group->mutex); return ret; } +EXPORT_SYMBOL_GPL(iommu_device_use_default_domain); /** * iommu_device_unuse_default_domain() - Device driver stops handling device @@ -3391,7 +3394,7 @@ int iommu_device_use_default_domain(struct device *dev) */ void iommu_device_unuse_default_domain(struct device *dev) { - /* Caller is the driver core during the post-probe path */ + /* Called with the device lock held during removal. */ struct iommu_group *group = dev->iommu_group; if (!group) @@ -3403,6 +3406,7 @@ void iommu_device_unuse_default_domain(struct device *dev) mutex_unlock(&group->mutex); } +EXPORT_SYMBOL_GPL(iommu_device_unuse_default_domain); static int __iommu_group_alloc_blocking_domain(struct iommu_group *group) { diff --git a/tools/testing/apple/check-lifetime.py b/tools/testing/apple/check-lifetime.py new file mode 100644 index 0000000000000..d7f82f276dc84 --- /dev/null +++ b/tools/testing/apple/check-lifetime.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Native regression for component/devres teardown of an unstopped session. + +Mocks kernel resource releases and the component framework. It executes the +actual unbind, PIODMA release, RTKit release, domain release and ring allocator. +No firmware calls or hardware acceptance are claimed. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('--output', type=Path, required=True) +p.add_argument('--cc', default='clang') +a = p.parse_args() +root = Path(__file__).resolve().parents[3] +src = root / 'drivers/gpu/drm/apple' +a.output.mkdir(parents=True, exist_ok=True) + + +def function(file, name): + text = (src / file).read_text() + match = re.search(r'^\w[\w\s*]*?\b' + name + r'\(', text, re.M) + assert match, name + start = text.index('{', match.end()) + pos, depth = start + 1, 1 + while depth: + depth += (text[pos] == '{') - (text[pos] == '}') + pos += 1 + return text[match.start():pos] + + +# This is the driver-core switch that transfers default-domain lifetime to us. +text = (src / 'dcp.c').read_text() +assert re.search(r'static struct platform_driver apple_platform_driver = \{\s*' + r'\.driver_managed_dma = true,', text) +functions = '\n'.join(function(f, n) for f, n in ( + ('dcp.c', 'dcp_release_dma_domain'), ('dcp.c', 'dcp_release_rtkit'), + ('dcp.c', 'dcp_release_piodma_iommu_dev'), ('afk.c', 'afk_quiesce'), + ('afk.c', 'afk_release_context'), ('dcp.c', 'dcp_release_context'), + ('afk.c', 'afk_getbuf'), ('dcp.c', 'dcp_comp_unbind'), + ('apple_drv.c', 'apple_drm_uninit'))) +prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint8_t u8;typedef uint16_t u16;typedef uint32_t u32;typedef uint64_t u64; +#define GFP_KERNEL 0 +#define BLOCK_SHIFT 6 +#define GETBUF_SIZE 0xffffU +#define GETBUF_TAG 0xffff0000U +#define RBEP_TYPE 0 +#define RBEP_GETBUF_ACK 0 +#define GETBUF_ACK_DVA 0 +#define FIELD_GET(mask,x) ((x)&(mask)) +#define FIELD_PREP(mask,x) ((void)(mask),(u64)(x)) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define IS_ERR_OR_NULL(x) (!(x)||(uintptr_t)(x)>(uintptr_t)-4096) +#define DRM_MODE_CONNECTOR_eDP 14 +struct work_struct {int id;};struct delayed_work {struct work_struct work;}; +struct drm_device {int refs;bool unplugged;}; +struct apple_drm_private {struct drm_device drm;}; +struct device {void *data;int domain_users;}; +struct platform_device {struct device dev;}; +struct apple_crtc {struct {struct drm_device *dev;} base;}; +struct apple_connector {struct work_struct hotplug_wq;}; +struct apple_rtkit {bool rx_stopped;}; +struct apple_dcp; +struct apple_dcp_afkep { + struct apple_dcp *dcp;void *wq;void *bfr;void *debugfs_entry; + u64 bfr_dma;u32 bfr_size;u16 bfr_tag; +}; +struct apple_dcp { + bool retain_dma,quiescing,drm_retained,piodma_created;int connector_type,hdmi_hpd_irq; + struct device *dev;struct apple_crtc *crtc; + struct apple_connector *connector,*typec_connector; + struct platform_device *piodma;void *iommu_dom,*typec_mux,*shmem,*clk,*clk_194; + struct apple_rtkit *rtk; + struct apple_dcp_afkep *avep,*dptxep,*ibootep,*systemep,*dcpavservep; + struct work_struct bl_register_wq,bl_update_wq,vblank_wq; + struct delayed_work typec_reconnect_wq,typec_fabric_retrain_wq; +}; +static int dma_frees,domain_drops,pio_destroys,pio_puts,firmware_stops,queue_stops; +static int raw_rings,managed_rings,work_drains; +static int context_frees; +static void *ring; +static struct device *component; +static void dcp_comp_unbind(struct device *,struct device *,void *); +static void kfree(void *p) {assert(p);context_frees++;} +static void debugfs_remove_recursive(void *p) {(void)p;} +static void dev_warn(struct device *d,const char *fmt) {(void)d;(void)fmt;} +static void dev_err(struct device *d,const char *fmt,...) {(void)d;(void)fmt;} +static void *dev_get_drvdata(struct device *d) {return d->data;} +static void dev_set_drvdata(struct device *d,void *p) {d->data=p;} +static void drm_dev_unplug(struct drm_device *d) {d->unplugged=true;} +static void drm_atomic_helper_shutdown(struct drm_device *d) {assert(d->unplugged);} +static void component_unbind_all(struct device *m,void *p) {dcp_comp_unbind(component,m,p);} +static void iommu_device_unuse_default_domain(struct device *d) {domain_drops++;d->domain_users--;} +static void apple_rtkit_free(struct apple_rtkit *r) {r->rx_stopped=true;dma_frees++;} +static void apple_rtkit_free_retaining_buffers(struct apple_rtkit *r) {r->rx_stopped=true;} +static void of_platform_device_destroy(struct device *d,void *p) {(void)d;(void)p;pio_destroys++;} +static void put_device(struct device *d) {(void)d;pio_puts++;} +static void drm_dev_get(struct drm_device *d) {d->refs++;} +static void iomfb_queue_stop(struct apple_dcp *d) {assert(d->quiescing||!d->retain_dma);queue_stops++;} +static void destroy_workqueue(void *w) {assert(w);work_drains++;} +static void cancel_work_sync(struct work_struct *w) {(void)w;} +static void cancel_delayed_work_sync(struct delayed_work *w) {(void)w;} +static void disable_irq(int i) {(void)i;} +static void typec_mux_put(void *p) {(void)p;} +static void av_service_disconnect(struct apple_dcp *d) {(void)d;firmware_stops++;} +static void afk_shutdown(struct apple_dcp_afkep *e) {(void)e;firmware_stops++;} +static void iomfb_shutdown(struct apple_dcp *d) {(void)d;firmware_stops++;} +static void devm_clk_put(struct device *d,void *c) {(void)d;(void)c;} +static void trace_afk_getbuf(struct apple_dcp_afkep *e,u32 size,u16 tag) {(void)e;(void)size;(void)tag;} +static void afk_send(struct apple_dcp_afkep *e,u64 msg) {(void)e;(void)msg;} +static void *dma_alloc_coherent(struct device *d,u32 size,u64 *addr,int flags) { + (void)d;(void)flags;raw_rings++;ring=calloc(1,size);assert(ring);*addr=123;return ring; +} +static void *dmam_alloc_coherent(struct device *d,u32 size,u64 *addr,int flags) { + (void)d;(void)flags;managed_rings++;ring=calloc(1,size);assert(ring);*addr=123;return ring; +} +''' +main = r''' +int main(void) { + unsigned cases=0; + for(int live=0;live<2;live++)for(int created=0;created<2;created++)for(int linked=0;linked<2;linked++)for(int master=0;master<2;master++) { + struct device dev={.domain_users=1};struct platform_device pio={0}; + struct apple_drm_private apple={.drm={.refs=1}}; + struct device main={.data=&apple};struct apple_crtc crtc={.base={&apple.drm}}; + struct apple_rtkit rtk={0};struct apple_connector con={0}; + struct apple_dcp d={.retain_dma=live,.dev=&dev,.crtc=linked?&crtc:NULL, + .connector=&con,.piodma=&pio,.piodma_created=created,.iommu_dom=&dev, + .rtk=&rtk,.shmem=&dev,.connector_type=DRM_MODE_CONNECTOR_eDP}; + struct apple_dcp_afkep ep[5]={0}; + for(int i=0;i<5;i++)ep[i]=(struct apple_dcp_afkep){.dcp=&d,.wq=&dev}; + d.avep=&ep[0];d.dptxep=&ep[1];d.ibootep=&ep[2];d.systemep=&ep[3];d.dcpavservep=&ep[4];dev.data=&d; + dma_frees=domain_drops=pio_destroys=pio_puts=firmware_stops=queue_stops=0; + raw_rings=managed_rings=work_drains=context_frees=0; + afk_getbuf(&ep[0],1);assert(ep[0].bfr_size==64); + afk_getbuf(&ep[0],1);assert(raw_rings+managed_rings==1); + component=&dev; + if(master) {apple_drm_uninit(&main);assert(!main.data&&apple.drm.unplugged);} + else dcp_comp_unbind(&dev,&dev,NULL); + /* Component devres release runs even after an early unbind return. */ + dcp_release_rtkit(&d);assert(rtk.rx_stopped&&!d.rtk); + /* Driver removal also releases probe resources and its domain action. */ + dcp_release_dma_domain(&d); + if(live) { + assert(d.quiescing&&!dma_frees&&!domain_drops&&dev.domain_users==1); + assert(d.piodma==&pio&&d.iommu_dom==&dev&&!pio_destroys&&!pio_puts); + assert(raw_rings==1&&!managed_rings&&!firmware_stops&&work_drains==5); + assert(apple.drm.refs==(linked?2:1)); + dcp_comp_unbind(&dev,&dev,NULL);assert(apple.drm.refs==(linked?2:1)); + dcp_release_piodma_iommu_dev(&d);assert(!pio_destroys&&!pio_puts); + } else { + assert(dma_frees==1&&domain_drops==1&&dev.domain_users==0); + assert(!d.piodma&&!d.iommu_dom&&managed_rings==1&&!raw_rings); + assert(pio_destroys==created&&pio_puts==!created&&firmware_stops>0); + assert(apple.drm.refs==1); + } + for(int i=0;i<5;i++)afk_release_context(&ep[i]); + dcp_release_context(&d);assert(context_frees==(live?0:6)); + afk_quiesce(NULL);afk_quiesce((struct apple_dcp_afkep *)(intptr_t)-12); + /* Simulated machine reset: test memory can now be reclaimed. */ + free(ring);cases++; + } + /* A failed bind has no unbind callback: resource cleanup must close rebind. */ + { + struct device dev={0};struct platform_device pio={0};struct apple_rtkit rtk={0}; + struct apple_dcp d={.dev=&dev,.retain_dma=true,.piodma=&pio,.rtk=&rtk}; + assert(!d.quiescing);dcp_release_piodma_iommu_dev(&d); + assert(d.quiescing&&d.piodma==&pio);cases++; + d.quiescing=false;dcp_release_rtkit(&d); + assert(d.quiescing&&rtk.rx_stopped&&!d.rtk);cases++; + } + puts("PASS: 18 actual-source component, master-unwind, driver-resource-release and failed-bind cases; live DMA/domain/rings retained; legacy releases preserved"); + assert(cases==18);return 0; +} +''' +code = a.output / 'lifetime.c' +code.write_text(prefix + functions + main) +command = [a.cc, '-std=gnu11', '-O1', '-g', '-Wall', '-Wextra', '-Werror', + '-Wno-unused-parameter', '-fsanitize=address,undefined', str(code), + '-o', str(a.output / 'lifetime')] +subprocess.run(command, check=True) +result = subprocess.check_output([str(a.output / 'lifetime')], text=True) +(a.output / 'receipt.json').write_text(json.dumps(dict(command=command, result=result, + functions_sha256=hashlib.sha256(functions.encode()).hexdigest(), + scope='Native resource-release simulation; not a live firmware unbind test'), indent=2) + '\n') +print(result) From 9a19868b10fa7a7bb7835fba6978abac3f059d6b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:32:00 +1100 Subject: [PATCH 0545/1579] drm: apple: retry rejected H17P brightness presents with a bound A rejected brightness present cancels its reservation and leaves the policy dirty, but it receives no successful completion. The queue's completion-only reschedule condition therefore drops the update on a static display until another user request or frame arrives. Track a rejected backlight reservation separately from a completed present. After its channel drains, schedule up to three deferred retries at 100, 200 and 400 ms through the existing outbound queue. Preserve the last accepted surface, leave an exhausted target dirty, reset the budget on a new user request or successful completion, and cancel delayed work when the queue stops. Crashed and stopped queues never retry. Add a native regression that executes the actual failure, queue-advance, retry-work and completion functions: one rejected present is retried and accepted without another user request. Check the finite retry budget, new-target rearming and stopped, crashed and undrained-channel guards. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp_backlight.c | 11 ++ drivers/gpu/drm/apple/dcp_backlight.h | 21 ++- drivers/gpu/drm/apple/iomfb.c | 29 +++- tools/testing/apple/check-backlight-retry.py | 162 +++++++++++++++++++ tools/testing/apple/check-backlight.py | 2 +- 6 files changed, 222 insertions(+), 5 deletions(-) create mode 100644 tools/testing/apple/check-backlight-retry.py diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 5be5ba9c87387..15945de4bcd22 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -43,6 +43,7 @@ struct iomfb_transaction { void (*release)(struct iomfb_transaction *transaction); struct dcp_backlight_present backlight; bool backlight_reserved; + bool backlight_failed; bool brightness_only; bool completed; }; @@ -391,6 +392,7 @@ struct apple_dcp { struct task_struct *owner; struct work_struct work; struct delayed_work timeout; + struct delayed_work backlight_retry; unsigned long deadline; unsigned int queued; bool stopped; diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 9d76154e13ce5..544c09964df38 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -224,6 +224,17 @@ bool dcp_backlight_pending(struct apple_dcp *dcp) return pending; } +unsigned int dcp_backlight_retry_delay(struct apple_dcp *dcp) +{ + unsigned long flags; + unsigned int delay; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + delay = dcp_bl_retry_delay(&dcp->backlight.state); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return delay; +} + #define SCALE_FACTOR (1 << 10) static u32 interpolate(int val, int min, int max, u32 *tbl, size_t tbl_size) diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index e93ee7511c626..2ad4abb4acd37 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -15,6 +15,7 @@ struct dcp_backlight_state { u32 actual; u32 sent_nits; u64 sequence; + u8 retries; bool ready; bool controlled; bool dpms_off; @@ -89,6 +90,7 @@ static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, if (nits > state->maximum) return -ERANGE; state->controlled = true; + state->retries = 0; if (state->target != nits || state->core_blank != core_blank || state->suspended != suspended) { state->target = nits; @@ -104,6 +106,7 @@ static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) if (!state->ready) return -ENODATA; state->controlled = true; + state->retries = 0; if (state->dpms_off != !on) { state->dpms_off = !on; state->dirty = true; @@ -147,13 +150,26 @@ static inline bool dcp_bl_complete(struct dcp_backlight_state *state, if (!state->in_flight || sequence != state->sequence) return false; state->in_flight = false; - if (accepted) + if (accepted) { state->actual = state->sent_nits; - else + state->retries = 0; + } else { state->dirty = true; + if (state->retries < 4) + state->retries++; + } return true; } +/* Three deferred retries per user request; a permanent rejection stays dirty. */ +static inline unsigned int dcp_bl_retry_delay(const struct dcp_backlight_state *state) +{ + if (!state->ready || !state->dirty || state->in_flight || + !state->retries || state->retries > 3) + return 0; + return 100U << (state->retries - 1); +} + struct apple_dcp; /* @@ -173,5 +189,6 @@ int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, struct dcp_backlight_present *present); bool dcp_backlight_complete(struct apple_dcp *dcp, u64 sequence, bool accepted); bool dcp_backlight_pending(struct apple_dcp *dcp); +unsigned int dcp_backlight_retry_delay(struct apple_dcp *dcp); #endif /* __APPLE_DCP_BACKLIGHT_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index dfcaaeb916996..fe6bcb9703366 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -244,8 +244,20 @@ static void iomfb_queue_advance(struct apple_dcp *dcp) if (transaction->brightness_only) dcp->iomfb.backlight_queued = false; /* A new level requested during a successful present stays pending. */ - if (transaction->completed && dcp_backlight_pending(dcp)) - schedule_work(&dcp->bl_update_wq); + if (dcp_backlight_pending(dcp)) { + if (transaction->completed) { + schedule_work(&dcp->bl_update_wq); + } else if (transaction->backlight_failed) { + unsigned int delay = dcp_backlight_retry_delay(dcp); + + if (delay) + mod_delayed_work(system_wq, &dcp->iomfb.backlight_retry, + msecs_to_jiffies(delay)); + else + dev_warn_ratelimited(dcp->dev, + "backlight retry limit reached\n"); + } + } transaction->release(transaction); } if (iomfb_enqueue_opaque_x(dcp)) { @@ -340,12 +352,23 @@ void iomfb_opaque_x_reset_h17p(struct apple_dcp *dcp) atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); } +static void iomfb_backlight_retry(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + iomfb.backlight_retry); + + if (!READ_ONCE(dcp->crashed) && !READ_ONCE(dcp->iomfb.stopped) && + dcp_backlight_pending(dcp)) + schedule_work(&dcp->bl_update_wq); +} + void iomfb_queue_init(struct apple_dcp *dcp) { mutex_init(&dcp->iomfb.lock); INIT_LIST_HEAD(&dcp->iomfb.pending); INIT_WORK(&dcp->iomfb.work, iomfb_queue_work); INIT_DELAYED_WORK(&dcp->iomfb.timeout, iomfb_queue_timeout); + INIT_DELAYED_WORK(&dcp->iomfb.backlight_retry, iomfb_backlight_retry); atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); } @@ -360,6 +383,7 @@ void iomfb_queue_stop(struct apple_dcp *dcp) mutex_unlock(&dcp->iomfb.lock); cancel_work_sync(&dcp->iomfb.work); cancel_delayed_work_sync(&dcp->iomfb.timeout); + cancel_delayed_work_sync(&dcp->iomfb.backlight_retry); } int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) @@ -1103,6 +1127,7 @@ void iomfb_present_failed_h17p(struct apple_dcp *dcp) if (transaction && transaction->backlight_reserved) { dcp_backlight_complete(dcp, transaction->backlight.sequence, false); transaction->backlight_reserved = false; + transaction->backlight_failed = true; } } diff --git a/tools/testing/apple/check-backlight-retry.py b/tools/testing/apple/check-backlight-retry.py new file mode 100644 index 0000000000000..32d1fc40580f2 --- /dev/null +++ b/tools/testing/apple/check-backlight-retry.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Native regression for rejected brightness transactions and bounded retries. + +Executes actual reservation, failure, queue-drain, retry-work and completion +functions with mocked scheduling and transport. Does not submit firmware RPCs. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('--output', type=Path, required=True) +p.add_argument('--cc', default='clang') +a = p.parse_args() +root = Path(__file__).resolve().parents[3] +src = root / 'drivers/gpu/drm/apple' +a.output.mkdir(parents=True, exist_ok=True) + + +def function(file, name): + text = (src / file).read_text() + match = re.search(r'^\w[\w\s*]*?\b' + name + r'\(', text, re.M) + assert match, name + start = text.index('{', match.end()) + pos, depth = start + 1, 1 + while depth: + depth += (text[pos] == '{') - (text[pos] == '}') + pos += 1 + return text[match.start():pos] + + +policy = re.sub(r'^#include ]+>\n', '', + (src / 'dcp_backlight.h').read_text(), flags=re.M) +functions = '\n'.join(function(f, n) for f, n in ( + ('dcp_backlight.c', 'dcp_backlight_complete'), + ('dcp_backlight.c', 'dcp_backlight_pending'), + ('dcp_backlight.c', 'dcp_backlight_retry_delay'), + ('iomfb.c', 'iomfb_present_failed_h17p'), + ('iomfb.c', 'iomfb_present_complete_h17p'), + ('iomfb.c', 'iomfb_queue_advance'), ('iomfb.c', 'iomfb_backlight_retry'))) +prefix = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8;typedef uint32_t u32;typedef uint64_t u64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define lockdep_assert_held(x) ((void)(x)) +#define spin_lock_irqsave(x,f) do { (void)(x);(f)=0; } while(0) +#define spin_unlock_irqrestore(x,f) do { (void)(x);(void)(f); } while(0) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define to_delayed_work(p) container_of(p,struct delayed_work,work) +#define msecs_to_jiffies(x) (x) +#define IOMFB_OPAQUE_X_WAITING 0 +#define IOMFB_OPAQUE_X_NEEDED 1 +struct work_struct {int id;};struct delayed_work {struct work_struct work;}; +struct list_head {int empty;}; +struct iomfb_transaction { + struct dcp_backlight_present backlight; + bool backlight_reserved,backlight_failed,brightness_only,completed; + void (*release)(struct iomfb_transaction *); +}; +struct apple_dcp { + bool crashed;void *dev;struct work_struct bl_update_wq,vblank_wq; + struct {int lock;struct dcp_backlight_state state;} backlight; + struct {bool pending;} present_state_h17p; + struct {int lock;bool stopped,backlight_queued;struct iomfb_transaction *active; + struct delayed_work timeout,backlight_retry;struct work_struct work; + struct list_head pending;int opaque_x_state; + } iomfb; +}; +static unsigned delay,scheduled,released,discarded;static bool idle=true; +static void *system_wq; +static bool iomfb_channels_idle(struct apple_dcp *d) {(void)d;return idle;} +static int iomfb_enqueue_opaque_x(struct apple_dcp *d) {(void)d;return 0;} +static void iomfb_discard_pending(struct apple_dcp *d) {(void)d;discarded++;} +static bool list_empty(struct list_head *l) {return l->empty;} +static void cancel_delayed_work(struct delayed_work *w) {(void)w;} +static void schedule_work(struct work_struct *w) {assert(w->id==1);scheduled++;} +static void mod_delayed_work(void *q,struct delayed_work *w,unsigned ms) { + (void)q;assert(w->work.id==2);delay=ms; +} +static void dev_warn_ratelimited(void *d,const char *msg) {(void)d;(void)msg;} +static void iomfb_scanout_complete_h17p(struct apple_dcp *d) {(void)d;} +static void atomic_cmpxchg(int *p,int old,int value) {if(*p==old)*p=value;} +static void release(struct iomfb_transaction *t) {(void)t;released++;} +''' +# The policy header defines structs before the fixture needs them. +prefix = prefix[:prefix.index('struct work_struct')] + policy + prefix[prefix.index('struct work_struct'):] +main = r''' +static void init(struct apple_dcp *d) { + u32 inherited; + memset(d,0,sizeof(*d));d->bl_update_wq.id=1;d->iomfb.backlight_retry.work.id=2; + assert(!dcp_bl_takeover_nits(509,36000,&inherited)&&inherited==36); + d->iomfb.pending.empty=1;assert(!dcp_bl_init(&d->backlight.state,509,true,inherited,false,0)); + assert(dcp_bl_seed(&d->backlight.state,36));assert(!dcp_bl_dpms(&d->backlight.state,true)); + assert(!dcp_bl_request(&d->backlight.state,140,false,false)); + delay=scheduled=released=discarded=0;idle=true; +} +static void reserve(struct apple_dcp *d,struct iomfb_transaction *t) { + memset(t,0,sizeof(*t));t->brightness_only=true;t->release=release; + assert(!dcp_bl_prepare(&d->backlight.state,true,&t->backlight)); + t->backlight_reserved=true;d->iomfb.active=t;d->iomfb.backlight_queued=true; +} +static void reject(struct apple_dcp *d) { + iomfb_present_failed_h17p(d);d->present_state_h17p.pending=false; + iomfb_queue_advance(d); +} +int main(void) { + struct apple_dcp d;struct iomfb_transaction t; + init(&d);reserve(&d,&t);reject(&d); + assert(delay==100&&!d.iomfb.active&&!d.iomfb.backlight_queued&&released==1); + assert(d.backlight.state.actual==36&&d.backlight.state.dirty&&!t.completed); + /* The deferred worker retries on retained pixels, with NO new user request. */ + iomfb_backlight_retry(&d.iomfb.backlight_retry.work);assert(scheduled==1); + reserve(&d,&t);assert(!iomfb_present_complete_h17p(&d));iomfb_queue_advance(&d); + assert(d.backlight.state.actual==140&&!d.backlight.state.dirty&&!d.backlight.state.retries); + init(&d); + for(unsigned i=0;i<4;i++) { + reserve(&d,&t);delay=0;reject(&d);assert(delay==(i<3?(100U< Date: Thu, 8 Oct 2026 16:12:48 +1100 Subject: [PATCH 0546/1579] MAINTAINERS: assign new DCP bindings and tests to Aurora support The new DCP bindings, backlight documentation and native tests belong to Aurora's T8140 and J700 support. Move their file patterns to that existing entry so reviews for those files reach their maintainer. Keep the existing Apple DRM driver source pattern under its current display-driver entry. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 1516f3e6ec4f4..f14b1110393a5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1951,12 +1951,7 @@ M: Janne Grunau L: dri-devel@lists.freedesktop.org S: Maintained T: git git://anongit.freedesktop.org/drm/drm-misc -F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml -F: Documentation/devicetree/bindings/display/apple,dcp.yaml -F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml -F: Documentation/gpu/apple-dcp-backlight.rst F: drivers/gpu/drm/apple/ -F: tools/testing/apple/ APPLE PCIE CONTROLLER DRIVER M: Marc Zyngier @@ -1974,6 +1969,9 @@ APPLE T8140 SOC AND J700 (MACBOOK NEO) SUPPORT M: Ryan Murray W: https://aurora-silicon.github.io S: Maintained +F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml +F: Documentation/devicetree/bindings/display/apple,dcp.yaml +F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml @@ -1984,6 +1982,7 @@ F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml +F: Documentation/gpu/apple-dcp-backlight.rst F: Documentation/hid/apple-j700-multitouch.rst F: arch/arm64/boot/dts/apple/t8140* F: arch/arm64/include/asm/apple-idle.h @@ -1997,6 +1996,7 @@ F: include/linux/soc/apple/smc-thermal.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs +F: tools/testing/apple/ APPLETALK NETWORK LAYER L: netdev@vger.kernel.org From 9aebc2952fb51f6b0b9c03222109fe7f0ac107c2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:19:25 +1100 Subject: [PATCH 0547/1579] arm64: dts: apple: retain the T8140 P-cluster domain cells The P-cluster DVFS controller provides a performance domain with no arguments. Retain its required cell count when connecting the PMP effort lane so the CPU performance-domain lookup remains valid. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 36bfdf78ec7cd..f90ab5435949d 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -287,6 +287,7 @@ cpufreq_p: cpufreq@211e20000 { compatible = "apple,t8140-cluster-cpufreq"; reg = <0x2 0x11e20000 0x0 0x1000>; + #performance-domain-cells = <0>; /* P lane has been observed under Linux CPU load. */ apple,fast-die-effort = <&pmp_report 1>; }; From 966e777fd568565b88c5f8ba4acb36463ae31e47 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:41:06 +1100 Subject: [PATCH 0548/1579] MAINTAINERS: keep T8140 PMP coverage under Aurora Place the PMP driver, report interface and related bindings under the existing J700 support entry owned by Ryan Murray. Keep project contact information together rather than creating a separate coordination entry. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 2d587c8002676..bc97a87c02de8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1965,9 +1965,9 @@ L: linux-hwmon@vger.kernel.org S: Odd fixes F: drivers/hwmon/applesmc.c -APPLE T8140 PMP SUPPORT +APPLE T8140 SOC AND J700 (MACBOOK NEO) SUPPORT M: Ryan Murray -L: asahi@lists.linux.dev +W: https://aurora-silicon.github.io S: Maintained F: Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml F: Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -1975,11 +1975,6 @@ F: Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml F: drivers/pmdomain/apple/pmp-report.c F: drivers/soc/apple/pmp.rs F: include/linux/soc/apple/pmp-report.h - -APPLE T8140 SOC AND J700 (MACBOOK NEO) SUPPORT -M: Ryan Murray -W: https://aurora-silicon.github.io -S: Maintained F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml From 5c7b86c58c8ffee4a08251901bb1b3dc15d69a85 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:46:49 +1100 Subject: [PATCH 0549/1579] spi: apple-qspi: reject clocks below the divider range Clamping a too-large divider to 256 makes the bus faster than requested. For a 200 MHz input, a 100 kHz request would run at about 781 kHz. Reject unrepresentable slow transfers instead of violating the device limit. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index b831622753d55..57a572c1e779d 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -348,7 +348,10 @@ static int apple_qspi_set_frequency(struct apple_qspi *qspi, return -EINVAL; divisor = DIV_ROUND_UP(input, requested); - divisor = clamp_t(u32, divisor, 2, 256); + /* Never exceed the requested clock when it is below our range. */ + if (divisor > 256) + return -EINVAL; + divisor = max_t(u32, divisor, 2); apple_qspi_c_write(qspi, APPLE_QSPI_C_CLKDIV, divisor - 1); return 0; } From 2aea427d740d614584e59a7f76c0e85f4589e4e6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:54:42 +1100 Subject: [PATCH 0550/1579] soc: apple: sep: disable the unmediated raw SBIO probe ioctl The raw probe accepts arbitrary state-changing opcodes before the normal bio command policy and host-index reconciliation. It also has no ownership over multi-request capture workflows, so an administrator check alone cannot make it safe between workqueue stages. Retain the ioctl number but return EOPNOTSUPP without copying or sending any firmware request. Remove the userspace raw-transfer handler. Normal in-kernel typed transfers and the regular bio ioctls keep their existing policy; a future diagnostic interface needs a constrained command set and explicit workflow ownership before it can be exposed. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 41 ++------------------------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 8ae6ba132285b..3b30e8ec73b4d 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2179,7 +2179,8 @@ impl SepData { return self.bio_attest(arg); } if cmd == bio::IOC_SBIO_PROBE { - return self.bio_sbio_probe(arg); + // Arbitrary firmware commands bypass command policy and capture state. + return Err(kernel::error::code::EOPNOTSUPP); } let handled = { let mut session = self.bio_session.lock(); @@ -2219,44 +2220,6 @@ impl SepData { Ok(handled) } - /// Diagnostics: perform one raw sbio transfer on behalf of userspace. - fn bio_sbio_probe(&self, arg: usize) -> Result { - let user = kernel::uaccess::UserPtr::from_addr(arg); - let mut req: bio::SbioProbe = - kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) - .reader() - .read()?; - let n = (req.len as usize).min(SBIO_MAX_PAYLOAD).min(req.payload.len()); - let done = self.sbio_transfer_raw(req.opcode, c"PROBE", &req.payload[..n])?; - req.status = match done.status.answered() { - Some(v) => v as i32, - None => i32::MIN, - }; - let m = done.payload.len().min(req.reply.len()); - req.reply = [0u8; 256]; - req.reply[..m].copy_from_slice(&done.payload[..m]); - req.reply_len = done.payload.len() as u32; - dev_info!( - self.dev, - "sbio: PROBE opcode {:#x} len {} -> status {:?}, reply {} bytes\n", - req.opcode, - n, - done.status.answered(), - done.payload.len() - ); - kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) - .writer() - .write(&req)?; - Ok(bio::Handled { - ret: 0, - wake: false, - start_enrol: false, - start_verify: false, - delete_identity: None, - delete_identities: KVec::new(), - }) - } - fn bio_attest(&self, arg: usize) -> Result { let user = kernel::uaccess::UserPtr::from_addr(arg); let req: bio::Attest = From 3a1dd30fb3ac52e0d8249dea2bd4a3ad5d4a075e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 17:04:00 +1100 Subject: [PATCH 0551/1579] cpufreq: apple-soc: keep required fallback limits with an older DT The new limiter replaces the old thermal admission hooks. A missing fast-die property must not skip the fallback on SoCs that require thermal policy. Treat that omission like other invalid limit descriptions while leaving optional legacy platforms unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 55ca3de616e63..27ec399ae8713 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -650,7 +650,8 @@ static int apple_fast_die_init(struct cpufreq_policy *policy, parsed = of_parse_phandle_with_fixed_args(cluster, "apple,fast-die-effort", 1, 0, &args); - if (parsed == -ENOENT) + /* Required thermal admission must fail closed with an older DT. */ + if (parsed == -ENOENT && !priv->info->needs_thermal_policy) return 0; while (policy->freq_table[n].frequency != CPUFREQ_TABLE_END) From a260ed62baeb152d16a9755f5c071875cd007d06 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:54:42 +1100 Subject: [PATCH 0552/1579] soc: apple: sep: disable the unmediated raw SBIO probe ioctl The raw probe accepts arbitrary state-changing opcodes before the normal bio command policy and host-index reconciliation. It also has no ownership over multi-request capture workflows, so an administrator check alone cannot make it safe between workqueue stages. Retain the ioctl number but return EOPNOTSUPP without copying or sending any firmware request. Remove the userspace raw-transfer handler. Normal in-kernel typed transfers and the regular bio ioctls keep their existing policy; a future diagnostic interface needs a constrained command set and explicit workflow ownership before it can be exposed. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 41 ++------------------------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 8ae6ba132285b..83708b5decef4 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2179,7 +2179,8 @@ impl SepData { return self.bio_attest(arg); } if cmd == bio::IOC_SBIO_PROBE { - return self.bio_sbio_probe(arg); + // Arbitrary firmware commands bypass command policy and capture state. + return Err(Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32))); } let handled = { let mut session = self.bio_session.lock(); @@ -2219,44 +2220,6 @@ impl SepData { Ok(handled) } - /// Diagnostics: perform one raw sbio transfer on behalf of userspace. - fn bio_sbio_probe(&self, arg: usize) -> Result { - let user = kernel::uaccess::UserPtr::from_addr(arg); - let mut req: bio::SbioProbe = - kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) - .reader() - .read()?; - let n = (req.len as usize).min(SBIO_MAX_PAYLOAD).min(req.payload.len()); - let done = self.sbio_transfer_raw(req.opcode, c"PROBE", &req.payload[..n])?; - req.status = match done.status.answered() { - Some(v) => v as i32, - None => i32::MIN, - }; - let m = done.payload.len().min(req.reply.len()); - req.reply = [0u8; 256]; - req.reply[..m].copy_from_slice(&done.payload[..m]); - req.reply_len = done.payload.len() as u32; - dev_info!( - self.dev, - "sbio: PROBE opcode {:#x} len {} -> status {:?}, reply {} bytes\n", - req.opcode, - n, - done.status.answered(), - done.payload.len() - ); - kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) - .writer() - .write(&req)?; - Ok(bio::Handled { - ret: 0, - wake: false, - start_enrol: false, - start_verify: false, - delete_identity: None, - delete_identities: KVec::new(), - }) - } - fn bio_attest(&self, arg: usize) -> Result { let user = kernel::uaccess::UserPtr::from_addr(arg); let req: bio::Attest = From 1f870b821c9e31a1da4fee576a0873f9a30def8a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 17:56:15 +1100 Subject: [PATCH 0553/1579] dt-bindings: npu: apple,ane: describe the T8140 topology T8140 describes two ANE interrupts, three DART clients and a single controllable system power domain. Describe that topology separately from the M1 and M2 requirements without selecting an older firmware interface. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/npu/apple,ane.yaml | 63 +++++++++++++++++-- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/npu/apple,ane.yaml b/Documentation/devicetree/bindings/npu/apple,ane.yaml index ebca460a81bc1..79cf942654f7b 100644 --- a/Documentation/devicetree/bindings/npu/apple,ane.yaml +++ b/Documentation/devicetree/bindings/npu/apple,ane.yaml @@ -14,7 +14,8 @@ description: SoCs. It accesses memory through three DARTs. On T8103 and T6000 the host drives the engine through its register window. On T602x and T8112 a coprocessor runs the ANE firmware and the host talks to it through an ASC - mailbox. + mailbox. T8140 has two interrupts and one system power domain; its + coprocessor uses a different firmware interface. properties: compatible: @@ -25,6 +26,7 @@ properties: - apple,t6021-ane - apple,t6022-ane - apple,t8112-ane + - apple,t8140-ane reg: minItems: 1 @@ -45,7 +47,8 @@ properties: - const: fuse interrupts: - maxItems: 1 + minItems: 1 + maxItems: 2 interrupt-names: items: @@ -57,8 +60,8 @@ properties: power-domains: description: The power states the engine needs, five on T8103 and T6000, eight on - T602x and seven on T8112. - minItems: 5 + T602x, seven on T8112 and one system domain on T8140. + minItems: 1 maxItems: 8 resets: @@ -80,11 +83,40 @@ required: - reg - reg-names - interrupts - - interrupt-names - iommus - power-domains allOf: + - if: + properties: + compatible: + contains: + const: apple,t8140-ane + then: + properties: + reg: + maxItems: 1 + reg-names: + maxItems: 1 + interrupts: + minItems: 2 + interrupt-names: false + iommus: + minItems: 3 + power-domains: + maxItems: 1 + resets: false + mboxes: false + memory-region: false + else: + properties: + interrupts: + maxItems: 1 + power-domains: + minItems: 5 + required: + - interrupt-names + - if: properties: compatible: @@ -149,6 +181,27 @@ allOf: additionalProperties: false examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + ane@400000000 { + compatible = "apple,t8140-ane"; + reg = <0x4 0x0 0x0 0x2000000>; + reg-names = "engine"; + interrupt-parent = <&aic>; + interrupts = , + ; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + }; + - | #include #include From b9a339703e95a6a65e0e4515caa4c4f6afb0d9ce Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 17:56:16 +1100 Subject: [PATCH 0554/1579] arm64: dts: apple: describe the disabled T8140 Neural Engine Describe the ANE cluster, its three DART instances and the ANE_SYS power domain from the device tree. Keep the engine and DARTs disabled until the T8140 firmware backend and child power sequence can initialize them safely. The M2 firmware backend is not compatible with this generation. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 13 +++++++ arch/arm64/boot/dts/apple/t8140.dtsi | 46 +++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index f9058fa8efc30..907f966959538 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -471,4 +471,17 @@ label = "smc_i2cm2"; apple,always-on; /* used by the SMC firmware */ }; + +}; + +&pmgr { + + ps_ane_sys: power-controller@290 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x290 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_sys"; + status = "okay"; + }; }; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 5ffe82f64ca3d..02bd006a33c6e 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1107,6 +1107,52 @@ apple,dart-vm-size = <0x0 0xa0000000>; status = "disabled"; }; + + ane_dart0: iommu@401800000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x01800000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + apple,dma-range = <0x100 0x0 0x300 0x0>; + status = "disabled"; + }; + + ane_dart1: iommu@401820000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x01820000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + apple,dma-range = <0x100 0x0 0x300 0x0>; + status = "disabled"; + }; + + ane_dart2: iommu@401840000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x4 0x01840000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + apple,dma-range = <0x100 0x0 0x300 0x0>; + status = "disabled"; + }; + + /* The T8140 firmware and child power sequence need their own backend. */ + ane: ane@400000000 { + compatible = "apple,t8140-ane"; + reg = <0x4 0x0 0x0 0x2000000>; + reg-names = "engine"; + interrupt-parent = <&aic>; + interrupts = , + ; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; }; #include "t8140-pmgr.dtsi" From fe68f33630c775b16bdf44f5628d1e7d5edc24e4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 18:03:41 +1100 Subject: [PATCH 0555/1579] usb: typec: tipd: descend according to the shared core The shared TIPD core can be built in for the SPMI transport while the I2C transport is a module. Descending with the I2C setting then omits the built-in core from the parent archive, leaving the I2C module with unresolved TIPD references at modpost and omitting the SPMI driver from vmlinux. Use TYPEC_TPS6598X_CORE for directory traversal, as in the current aurora-wip Makefile. The child Makefile already selects the core and both transports independently. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/usb/typec/Makefile b/drivers/usb/typec/Makefile index 8a6a1c663eb69..eebe658de6f8c 100644 --- a/drivers/usb/typec/Makefile +++ b/drivers/usb/typec/Makefile @@ -5,7 +5,7 @@ typec-$(CONFIG_ACPI) += port-mapper.o obj-$(CONFIG_TYPEC) += altmodes/ obj-$(CONFIG_TYPEC_TCPM) += tcpm/ obj-$(CONFIG_TYPEC_UCSI) += ucsi/ -obj-$(CONFIG_TYPEC_TPS6598X) += tipd/ +obj-$(CONFIG_TYPEC_TPS6598X_CORE) += tipd/ obj-$(CONFIG_TYPEC_ANX7411) += anx7411.o obj-$(CONFIG_TYPEC_HD3SS3220) += hd3ss3220.o obj-$(CONFIG_TYPEC_STUSB160X) += stusb160x.o From e3163ca622d6c55979d963929edbdf47f2c4c38c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 17:09:30 +1100 Subject: [PATCH 0556/1579] soc: apple: expose the T8140 PMP DVFS report for diagnosis The firmware describes DVFS-STATE as four report entries beginning at entry 8. Make those values available through a read-only debugfs file in the existing PMP report driver. Require firmware readiness before and after sampling, bound the reads to the mapped resource, and remove the debugfs directory before releasing the mapping. The entry encoding and rate conversion are not established. Document the raw values as diagnostics, without presenting them as temperatures or hardware clock rates. This complements firmware frequency limits and thermal cooling without issuing new requests or changing policy. The T8140 report match and platform resources are provided separately. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/soc/apple/t8140-pmp-dvfs.rst | 24 +++++++ MAINTAINERS | 1 + drivers/pmdomain/apple/pmp-report.c | 80 ++++++++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 Documentation/soc/apple/t8140-pmp-dvfs.rst diff --git a/Documentation/soc/apple/t8140-pmp-dvfs.rst b/Documentation/soc/apple/t8140-pmp-dvfs.rst new file mode 100644 index 0000000000000..7ec4ae7366ed9 --- /dev/null +++ b/Documentation/soc/apple/t8140-pmp-dvfs.rst @@ -0,0 +1,24 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +T8140 PMP DVFS diagnostics +========================= + +The existing PMP report provider exposes a read-only ``dvfs`` file under +``/sys/kernel/debug/apple-pmp-/`` on T8140. The provider must be +bound; the file is readable by root. Its output lists ADT report entries +8 through 11 and both raw 64-bit words of each 16-byte entry. Only those four entries +are read. The report aperture is not dumped. + +Reading returns ``EAGAIN`` if the PMP running bit is clear before or after +sampling. Entries are sampled individually, rather than as an atomic +firmware snapshot. A running PMP alone does not establish freshness of +each value. + +The field encoding and conversion to clock rates are not yet established. +Values must not be interpreted as Hz, temperatures, or verified hardware +clock states. This interface changes neither firmware requests nor cache +allocation nor thermal policy. + +CPU temperatures remain available through ``macsmc-hwmon``. CPU cooling +and firmware frequency limits remain under their existing drivers. The +DVFS diagnostic file is not a thermal sensor. diff --git a/MAINTAINERS b/MAINTAINERS index 90654e399c0d3..0e3c1142a1b4e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1980,6 +1980,7 @@ F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml F: Documentation/hid/apple-j700-multitouch.rst +F: Documentation/soc/apple/t8140-pmp-dvfs.rst F: arch/arm64/boot/dts/apple/t8140* F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 065d3296f5a7a..a83893c250cac 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -5,10 +5,12 @@ * Copyright The Asahi Linux Contributors */ +#include #include #include #include #include +#include #include #include #include @@ -22,6 +24,10 @@ struct apple_pmp_report_offsets { u32 status; }; +struct apple_pmp_report; + +static int apple_pmp_dvfs_create(struct apple_pmp_report *report); + struct apple_pmp_report { struct device *dev; const struct apple_pmp_report_offsets *offsets; @@ -47,6 +53,10 @@ static int apple_pmp_report_probe(struct platform_device *pdev) return PTR_ERR(rep->base); rep->offsets = of_device_get_match_data(dev); dev_set_drvdata(dev, rep); + ret = apple_pmp_dvfs_create(rep); + if (ret) + return ret; + ret = of_platform_populate(np, NULL, NULL, dev); if (ret) return dev_err_probe(dev, ret, "failed to create child devices\n"); @@ -135,6 +145,76 @@ static int apple_pmp_report_entry_power_off(struct generic_pm_domain *genpd) return apple_pmp_report_set_state(genpd, false); } +/* ADT DVFS-STATE: four 16-byte entries beginning at entry 8. */ +#define APPLE_PMP_DVFS_OFFSET 0x80 +#define APPLE_PMP_DVFS_STRIDE 16 +#define APPLE_PMP_DVFS_COUNT 4 + +static int apple_pmp_dvfs_show(struct seq_file *seq, void *unused) +{ + struct apple_pmp_report *report = seq->private; + u64 values[APPLE_PMP_DVFS_COUNT][2]; + unsigned int index; + + if (!(readl(report->base + report->offsets->status) & PMP_REPORT_READY)) + return -EAGAIN; + + for (index = 0; index < ARRAY_SIZE(values); index++) { + void __iomem *entry = report->base + APPLE_PMP_DVFS_OFFSET + + index * APPLE_PMP_DVFS_STRIDE; + + values[index][0] = readq(entry); + values[index][1] = readq(entry + sizeof(values[index][0])); + } + + if (!(readl(report->base + report->offsets->status) & PMP_REPORT_READY)) + return -EAGAIN; + + for (index = 0; index < ARRAY_SIZE(values); index++) + seq_printf(seq, "entry%u: 0x%016llx 0x%016llx\n", index + 8, + values[index][0], values[index][1]); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(apple_pmp_dvfs); + +static void apple_pmp_dvfs_remove(void *directory) +{ + debugfs_remove_recursive(directory); +} + +static int apple_pmp_dvfs_create(struct apple_pmp_report *report) +{ + struct device *dev = report->dev; + struct resource *resource; + struct dentry *directory; + char *name; + int ret; + + if (!of_device_is_compatible(dev->of_node, "apple,t8140-pmp-v2-report")) + return 0; + + resource = platform_get_resource(to_platform_device(dev), IORESOURCE_MEM, 0); + if (resource_size(resource) < APPLE_PMP_DVFS_OFFSET + + APPLE_PMP_DVFS_COUNT * APPLE_PMP_DVFS_STRIDE) + return dev_err_probe(dev, -EINVAL, "DVFS report exceeds resource\n"); + + name = devm_kasprintf(dev, GFP_KERNEL, "apple-pmp-%s", dev_name(dev)); + if (!name) + return -ENOMEM; + + directory = debugfs_create_dir(name, NULL); + if (IS_ERR(directory)) + return 0; + + ret = devm_add_action_or_reset(dev, apple_pmp_dvfs_remove, directory); + if (ret) + return ret; + + debugfs_create_file("dvfs", 0400, directory, report, &apple_pmp_dvfs_fops); + return 0; +} + static int apple_pmp_report_entry_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; From 42990988151ef145c01108d384143b9d474b0a7d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 18:03:41 +1100 Subject: [PATCH 0557/1579] usb: typec: tipd: descend according to the shared core The shared TIPD core can be built in for the SPMI transport while the I2C transport is a module. Descending with the I2C setting then omits the built-in core from the parent archive, leaving the I2C module with unresolved TIPD references at modpost and omitting the SPMI driver from vmlinux. Use TYPEC_TPS6598X_CORE for directory traversal, as in the current aurora-wip Makefile. The child Makefile already selects the core and both transports independently. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/usb/typec/Makefile b/drivers/usb/typec/Makefile index 8a6a1c663eb69..eebe658de6f8c 100644 --- a/drivers/usb/typec/Makefile +++ b/drivers/usb/typec/Makefile @@ -5,7 +5,7 @@ typec-$(CONFIG_ACPI) += port-mapper.o obj-$(CONFIG_TYPEC) += altmodes/ obj-$(CONFIG_TYPEC_TCPM) += tcpm/ obj-$(CONFIG_TYPEC_UCSI) += ucsi/ -obj-$(CONFIG_TYPEC_TPS6598X) += tipd/ +obj-$(CONFIG_TYPEC_TPS6598X_CORE) += tipd/ obj-$(CONFIG_TYPEC_ANX7411) += anx7411.o obj-$(CONFIG_TYPEC_HD3SS3220) += hd3ss3220.o obj-$(CONFIG_TYPEC_STUSB160X) += stusb160x.o From 4a7951a40178cb0f2158f54b3f3b8dd34766332c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 18:44:09 +1100 Subject: [PATCH 0558/1579] soc: apple: mailbox: propagate IRQ lookup errors Preserve the receive and send-empty IRQ lookup errors, including deferred probe, rather than replacing them with ENODEV. Keep ENXIO as the supported absence of the optional send-empty IRQ for polling transports. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mailbox.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 4a4df15707ce3..42fd1978421e1 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -395,7 +395,7 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->irq_recv_not_empty = platform_get_irq_byname(pdev, "recv-not-empty"); if (mbox->irq_recv_not_empty < 0) - return -ENODEV; + return mbox->irq_recv_not_empty; /* * Some coprocessors (e.g. the T6021 ANE ASC) have no send-empty line. @@ -404,7 +404,7 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->irq_send_empty = platform_get_irq_byname_optional(pdev, "send-empty"); if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) - return -ENODEV; + return mbox->irq_send_empty; spin_lock_init(&mbox->rx_lock); spin_lock_init(&mbox->tx_lock); From ed955e4b8282dc95819dca6611df4fbc49f96951 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:30:44 +1100 Subject: [PATCH 0559/1579] HID: magicmouse: serialize haptic effect updates and retain C1FE clicks Publish complete effect parameters under a per-effect spinlock only after the mode switch succeeds. Workers take one coherent snapshot and erase drains the affected job before clearing it. Failed updates leave the previous effect intact. Serialize actuator lookup, mode transactions and cached state with the existing haptic auto-trigger mutex. Drain workers before taking that mutex during erase and destroy, and retake host mode on playback after resume invalidates the mode cache. C1FE reports have no pressure, so they cannot emulate buttons when firmware click handling is disabled by host-controlled haptics. Keep those pads in firmware-controlled mode and omit force feedback and pressure-pad capabilities. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/hid-magicmouse.c | 140 ++++++++++++++++++++++++----------- 1 file changed, 97 insertions(+), 43 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 8eb69d187ce87..aa0e6cee6d8fd 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -247,6 +248,9 @@ struct magicmouse_input_ops { struct effect_job { struct work_struct work; + struct magicmouse_sc *msc; + /* Protects effect parameters shared with the worker. */ + spinlock_t lock; struct hid_device *taptic_hdev; u16 effect_type; u8 strength; @@ -1264,7 +1268,8 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, struct magicmouse_sc *msc = hid_get_drvdata(hdev); __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); - __set_bit(INPUT_PROP_PRESSUREPAD, input->propbit); + if (!msc->mtp_c1fe) + __set_bit(INPUT_PROP_PRESSUREPAD, input->propbit); __clear_bit(BTN_0, input->keybit); __clear_bit(BTN_RIGHT, input->keybit); __clear_bit(BTN_MIDDLE, input->keybit); @@ -1272,7 +1277,9 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, __clear_bit(REL_X, input->relbit); __clear_bit(REL_Y, input->relbit); - mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK | INPUT_MT_TOTAL_FORCE; + mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK; + if (!msc->mtp_c1fe) + mt_flags |= INPUT_MT_TOTAL_FORCE; /* * The C1FE report carries positions and the button only; advertise @@ -1445,6 +1452,7 @@ static int magicmouse_switch_mode(struct input_dev *trackpad_idev, int mode) static u8 magicmouse_haptic_softness(struct hid_device *hdev, bool deep); static u8 magicmouse_haptic_strength(struct hid_device *hdev, bool deep); static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc); +static void magicmouse_haptic_release_mode_locked(struct magicmouse_sc *msc); /* * Play the waveform @pulses times, @haptic_pulse_gap_ms apart. A failure here @@ -1454,7 +1462,7 @@ static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc); static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, bool deep, u8 pulses) { - struct hid_device *actuator = magicmouse_get_actuator(msc); + struct hid_device *actuator; u8 strength, softness, gap; unsigned int i; int ret; @@ -1462,6 +1470,9 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, if (!pulses) return; + guard(mutex)(&msc->haptics->auto_trigger_mutex); + actuator = magicmouse_get_actuator(msc); + if (!actuator) { hid_warn(msc->hdev, "no actuator for host click feedback; using the firmware's\n"); @@ -1469,7 +1480,7 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, return; } - if (msc->haptics->mode != HID_HAPTIC_MODE_HOST) { + if (READ_ONCE(msc->haptics->mode) != HID_HAPTIC_MODE_HOST) { ret = apple_taptic_switch_modes(actuator, HID_HAPTIC_MODE_HOST); if (ret) { hid_warn(msc->hdev, @@ -1478,7 +1489,7 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, haptic_press = false; return; } - msc->haptics->mode = HID_HAPTIC_MODE_HOST; + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_HOST); hid_info(msc->hdev, "host-driven click feedback active\n"); } @@ -1494,7 +1505,7 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, "click waveform failed (%d); returning click feedback to the firmware\n", ret); haptic_press = false; - magicmouse_haptic_release_mode(msc); + magicmouse_haptic_release_mode_locked(msc); return; } hid_dbg(msc->hdev, "played %s pulse %u/%u at strength 0x%02x\n", @@ -1510,24 +1521,30 @@ static void magicmouse_haptic_pulses(struct magicmouse_sc *msc, u16 usage, * Hand click feedback back to the actuator's firmware, unless userspace still * has effects uploaded and so still wants host-controlled mode. */ -static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc) +static void magicmouse_haptic_release_mode_locked(struct magicmouse_sc *msc) { struct hid_device *actuator; unsigned int i; - if (msc->haptics->mode != HID_HAPTIC_MODE_HOST) + if (READ_ONCE(msc->haptics->mode) != HID_HAPTIC_MODE_HOST) return; if (msc->haptic_effects) { for (i = 0; i < FF_MAX_EFFECTS; i++) - if (msc->haptic_effects[i].effect_type) + if (READ_ONCE(msc->haptic_effects[i].effect_type)) return; } actuator = magicmouse_get_actuator(msc); if (actuator && !apple_taptic_switch_modes(actuator, HID_HAPTIC_MODE_DEVICE)) - msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_DEVICE); +} + +static void magicmouse_haptic_release_mode(struct magicmouse_sc *msc) +{ + guard(mutex)(&msc->haptics->auto_trigger_mutex); + magicmouse_haptic_release_mode_locked(msc); } static void magicmouse_press_worker(struct work_struct *ws) @@ -1615,7 +1632,7 @@ static bool magicmouse_haptic_click(struct magicmouse_sc *msc, u32 pressure, if (!msc->haptics || !msc->haptics->wq) return firmware_button; - host_mode = msc->haptics->mode == HID_HAPTIC_MODE_HOST; + host_mode = READ_ONCE(msc->haptics->mode) == HID_HAPTIC_MODE_HOST; if (!haptic_press && !host_mode) { /* The firmware owns the actuator and reports the button. */ @@ -1689,7 +1706,7 @@ static void magicmouse_haptic_forget_mode(struct magicmouse_sc *msc) if (!msc->haptics) return; - msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_DEVICE); msc->haptic_button_down = false; if (msc->haptic_deep_down) { msc->haptic_deep_down = false; @@ -1901,6 +1918,10 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); struct hid_haptic_device *haptics = msc->haptics; struct hid_device *actuator; + struct effect_job *job = &msc->haptic_effects[effect->id]; + unsigned long flags; + u8 strength, softness; + bool deep; int ret; switch (effect->u.haptic.hid_usage) { @@ -1912,30 +1933,31 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, return -EINVAL; } + guard(mutex)(&haptics->auto_trigger_mutex); actuator = magicmouse_get_actuator(msc); if (!actuator) return -ENODEV; - msc->haptic_effects[effect->id].taptic_hdev = actuator; - msc->haptic_effects[effect->id].effect_type = (effect->u.haptic.hid_usage) & HID_USAGE; - msc->haptic_effects[effect->id].strength = - (min(100, (effect->u.haptic.intensity)) * - magicmouse_haptic_strength(trackpad_hdev, - magicmouse_effect_is_deep(effect))) / 100; - msc->haptic_effects[effect->id].softness = - magicmouse_haptic_softness(trackpad_hdev, - magicmouse_effect_is_deep(effect)); + deep = magicmouse_effect_is_deep(effect); + strength = min_t(u16, 100, effect->u.haptic.intensity) * + magicmouse_haptic_strength(trackpad_hdev, deep) / 100; + softness = magicmouse_haptic_softness(trackpad_hdev, deep); /* The actuator may have been rebound or reset since the last upload. */ ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_HOST); if (ret) { dev_err(&msc->hdev->dev, "failed to switch to host-controlled mode: %d\n", ret); - msc->haptic_effects[effect->id].effect_type = 0; return ret; } - haptics->mode = HID_HAPTIC_MODE_HOST; + spin_lock_irqsave(&job->lock, flags); + job->taptic_hdev = actuator; + job->strength = strength; + job->softness = softness; + WRITE_ONCE(job->effect_type, effect->u.haptic.hid_usage & HID_USAGE); + spin_unlock_irqrestore(&job->lock, flags); + WRITE_ONCE(haptics->mode, HID_HAPTIC_MODE_HOST); hid_dbg(trackpad_hdev, "uploaded haptic effect %d\n", effect->id); return 0; @@ -1944,11 +1966,32 @@ static int apple_upload_effects(struct input_dev *trackpad_idev, static void haptic_playback_worker(struct work_struct *ws) { struct effect_job *job = container_of(ws, struct effect_job, work); + struct hid_device *actuator; + unsigned long flags; + u16 type; + u8 strength, softness; + int ret; + + spin_lock_irqsave(&job->lock, flags); + actuator = job->taptic_hdev; + type = READ_ONCE(job->effect_type); + strength = job->strength; + softness = job->softness; + spin_unlock_irqrestore(&job->lock, flags); - if (job->effect_type && - apple_taptic_send(job->taptic_hdev, job->effect_type, - job->strength, job->softness)) - hid_warn(job->taptic_hdev, "cannot play the haptic effect\n"); + if (!type) + return; + guard(mutex)(&job->msc->haptics->auto_trigger_mutex); + if (READ_ONCE(job->msc->haptics->mode) != HID_HAPTIC_MODE_HOST) { + ret = apple_taptic_switch_modes(actuator, HID_HAPTIC_MODE_HOST); + if (ret) { + hid_warn(actuator, "cannot take host-controlled mode (%d)\n", ret); + return; + } + WRITE_ONCE(job->msc->haptics->mode, HID_HAPTIC_MODE_HOST); + } + if (apple_taptic_send(actuator, type, strength, softness)) + hid_warn(actuator, "cannot play the haptic effect\n"); } static int apple_taptic_playback(struct input_dev *trackpad_idev, int effect_id, int value) @@ -1968,16 +2011,19 @@ static int apple_taptic_erase(struct input_dev *trackpad_idev, int effect_id) struct magicmouse_sc *msc = hid_get_drvdata(trackpad_hdev); int i, ret = 0; - msc->haptic_effects[effect_id].effect_type = 0; + cancel_work_sync(&msc->haptic_effects[effect_id].work); + WRITE_ONCE(msc->haptic_effects[effect_id].effect_type, 0); for (i = 0; i < FF_MAX_EFFECTS; i++) { - if (msc->haptic_effects[i].effect_type != 0) + if (READ_ONCE(msc->haptic_effects[i].effect_type) != 0) return 0; } + /* Drain workers before taking the mutex they need. */ + flush_workqueue(msc->haptics->wq); + guard(mutex)(&msc->haptics->auto_trigger_mutex); /* Return to device-controlled mode if there are no effects left */ - if (msc->haptics->mode == HID_HAPTIC_MODE_HOST) { - flush_workqueue(msc->haptics->wq); + if (READ_ONCE(msc->haptics->mode) == HID_HAPTIC_MODE_HOST) { ret = magicmouse_switch_mode(trackpad_idev, HID_HAPTIC_MODE_DEVICE); if (ret) { @@ -1987,7 +2033,7 @@ static int apple_taptic_erase(struct input_dev *trackpad_idev, int effect_id) return ret; } - msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_DEVICE); } return 0; @@ -2002,15 +2048,18 @@ static void apple_taptic_destroy(struct ff_device *ff) /* ff->private is devm-allocated; do not let ff-core kfree() it. */ ff->private = NULL; - if (msc->haptics && msc->haptics->mode == HID_HAPTIC_MODE_HOST) { - flush_workqueue(msc->haptics->wq); - ret = magicmouse_switch_mode(msc->haptics->input_dev, HID_HAPTIC_MODE_DEVICE); - if (ret) - hid_warn(msc->hdev, - "failed to switch back to device-controlled mode: %d\n", - ret); - else - msc->haptics->mode = HID_HAPTIC_MODE_DEVICE; + flush_workqueue(haptic_dev->wq); + { + guard(mutex)(&haptic_dev->auto_trigger_mutex); + if (msc->haptics && READ_ONCE(msc->haptics->mode) == HID_HAPTIC_MODE_HOST) { + ret = magicmouse_switch_mode(msc->haptics->input_dev, HID_HAPTIC_MODE_DEVICE); + if (ret) + hid_warn(msc->hdev, + "failed to switch back to device-controlled mode: %d\n", + ret); + else + WRITE_ONCE(msc->haptics->mode, HID_HAPTIC_MODE_DEVICE); + } } destroy_workqueue(haptic_dev->wq); @@ -2069,6 +2118,7 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, haptic_dev->hdev = trackpad_hdev; haptic_dev->input_dev = msc->input; + mutex_init(&haptic_dev->auto_trigger_mutex); msc->haptic_effects = kzalloc_objs(struct effect_job, FF_MAX_EFFECTS); if (!msc->haptic_effects) { @@ -2084,8 +2134,11 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, return -ENOMEM; } - for (i = 0; i < FF_MAX_EFFECTS; i++) + for (i = 0; i < FF_MAX_EFFECTS; i++) { + msc->haptic_effects[i].msc = msc; + spin_lock_init(&msc->haptic_effects[i].lock); INIT_WORK(&msc->haptic_effects[i].work, haptic_playback_worker); + } INIT_WORK(&msc->haptic_press_work, magicmouse_press_worker); INIT_WORK(&msc->haptic_release_work, magicmouse_release_worker); @@ -2137,7 +2190,8 @@ static int magicmouse_init_haptics(struct magicmouse_sc *msc, struct hid_device struct hid_haptic_device *haptics; int ret; - if (msc->haptics) + /* C1FE has no pressure data to emulate clicks in host mode. */ + if (msc->haptics || msc->mtp_c1fe) return 0; /* From a75dad5f1c1ba3fdcc4447db02d10ffd38062bd2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:34:39 +1100 Subject: [PATCH 0560/1579] HID: magicmouse: wrap the reviewed haptic mode restoration call Wrap the mode-restoration call to satisfy the kernel line-length check. This is a formatting follow-up to the supplied review fix; the haptic locking and firmware-controlled C1FE behavior are unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/hid-magicmouse.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index aa0e6cee6d8fd..c382f940ff094 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -2052,7 +2052,8 @@ static void apple_taptic_destroy(struct ff_device *ff) { guard(mutex)(&haptic_dev->auto_trigger_mutex); if (msc->haptics && READ_ONCE(msc->haptics->mode) == HID_HAPTIC_MODE_HOST) { - ret = magicmouse_switch_mode(msc->haptics->input_dev, HID_HAPTIC_MODE_DEVICE); + ret = magicmouse_switch_mode(msc->haptics->input_dev, + HID_HAPTIC_MODE_DEVICE); if (ret) hid_warn(msc->hdev, "failed to switch back to device-controlled mode: %d\n", From aac959cdbfc5cea06d6b5ae32003da3193379aea Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:56:08 +1100 Subject: [PATCH 0561/1579] Bluetooth: hci_bcm4377: stop recovery when the PCI reset fails Reset must quiesce the controller before shared ring state is cleared and firmware is restarted. Preserve the reset error rather than reinitializing DMA state on a running controller. Leave needs_reset set until every recovery stage completes so a failed boot or RTI handshake cannot fall back to ring creation on the next open. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/hci_bcm4377.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 5fe775766e781..51b3e8bf9c135 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -1320,6 +1320,8 @@ static int bcm4377_pci_reset(struct bcm4377_data *bcm4377) int ret; int irq; + /* Keep retrying a full reset if any recovery stage fails. */ + bcm4377->needs_reset = true; dev_info(&bcm4377->pdev->dev, "performing hardware reset recovery\n"); irq = pci_irq_vector(bcm4377->pdev, 0); @@ -1330,10 +1332,11 @@ static int bcm4377_pci_reset(struct bcm4377_data *bcm4377) bcm4377_disable_aspm(bcm4377); ret = pci_reset_function(bcm4377->pdev); - if (ret) - dev_warn(&bcm4377->pdev->dev, - "function level reset failed with %d; trying to continue anyway\n", - ret); + if (ret) { + dev_err(&bcm4377->pdev->dev, + "function level reset failed: %d\n", ret); + goto out_irq; + } msleep(100); From 014734a12f83c44a4352a69a56af9745dd256e30 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:19:18 +1100 Subject: [PATCH 0562/1579] PCI: apple: retain preinitialized tunnels when cold init is disabled The board property permits experimental kernel cold initialization, but must not reject an existing successful m1n1 handoff. Admit the explicit preinit certificate in both the population gate and preparation helper without touching cold-init registers. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple.c | 7 +++++++ drivers/thunderbolt/apple.c | 19 ++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index f63837dc8014d..3362d0e8790d0 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2174,6 +2174,13 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) if (!np) return 0; + /* A qualified bootloader handoff does not require experimental init. */ + if (!of_property_read_u32(np, "apple,pciec-preinit-status", &stat) && + stat == 1) { + ret = 0; + goto out; + } + if (!apple_pcie_tunnel_kernel_init_enabled()) { dev_err(dev, "PCIe-C %pOF: kernel init disabled (pcie_apple.tunnel_kernel_init=0)\n", np); diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index e69bd9feeada3..f185099865568 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -206,6 +206,22 @@ static bool apple_cio_pcie_tunnel_kernel_init(struct device_node *parent) return false; } +static bool apple_cio_pcie_tunnel_has_handoff(struct device_node *parent) +{ + for_each_available_child_of_node_scoped(parent, child) { + u32 status; + + if (!of_device_is_compatible(child, "apple,t8103-pciec") && + !of_device_is_compatible(child, "apple,t6000-pciec")) + continue; + + return !of_property_read_u32(child, "apple,pciec-preinit-status", + &status) && status == 1; + } + + return false; +} + static bool apple_cio_pcie_tunnel_is_preinitialized(struct device_node *parent) { @@ -559,7 +575,8 @@ static int apple_cio_populate_pcie_tunnel(struct apple_cio *acio) /* Experimental: don't touch a cold PCIe-C port unless opted in. */ if (acio->pcie_tunnel_kernel_init && - !apple_pcie_tunnel_kernel_init_enabled()) + !apple_pcie_tunnel_kernel_init_enabled() && + !apple_cio_pcie_tunnel_has_handoff(acio->pcie_tunnel_np)) return dev_err_probe(acio->dev, -EPERM, "PCIe-C kernel init disabled (pcie_apple.tunnel_kernel_init=0)\n"); From 2f434459eb094edc046cd2ff71f1901302d7ef3d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:19:21 +1100 Subject: [PATCH 0563/1579] soc: apple: rtkit: reject an already owned mailbox receiver Keep receiver ownership reserved until teardown finishes. A competing initialization must return EBUSY instead of overwriting the callback and cookie, and must not stop the mailbox owned by the existing receiver. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 19ed2a0e8520e..fb0634d53b0ab 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -670,18 +670,21 @@ int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) } EXPORT_SYMBOL_GPL(apple_rtkit_start_ep); -static void apple_rtkit_claim_rx(struct apple_rtkit *rtk) +static int apple_rtkit_claim_rx(struct apple_rtkit *rtk) { struct apple_mbox *mbox = rtk->mbox; unsigned long flags; spin_lock_irqsave(&mbox->rx_lock, flags); - if (mbox->rx || mbox->cookie) - dev_warn(rtk->dev, - "RTKit: mailbox receiver already claimed, taking it over\n"); + if (mbox->rx || mbox->cookie) { + spin_unlock_irqrestore(&mbox->rx_lock, flags); + return -EBUSY; + } mbox->cookie = rtk; mbox->rx = apple_rtkit_rx; spin_unlock_irqrestore(&mbox->rx_lock, flags); + + return 0; } static void apple_rtkit_detach_rx(struct apple_rtkit *rtk) @@ -759,7 +762,9 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, goto free_rtk; } - apple_rtkit_claim_rx(rtk); + ret = apple_rtkit_claim_rx(rtk); + if (ret) + goto destroy_wq; ret = apple_mbox_start(rtk->mbox); if (ret) @@ -770,6 +775,9 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, stop_rx: apple_rtkit_stop_rx(rtk); apple_rtkit_release_rx(rtk); + goto free_rtk; +destroy_wq: + destroy_workqueue(rtk->wq); free_rtk: kfree(rtk); return ERR_PTR(ret); From a229707b394218e7c4e4d0348a6068b1433c4726 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:25:52 +1100 Subject: [PATCH 0564/1579] drm/asahi: tie stats registration to a per-device snapshot owner Keep a snapshot Arc in the successfully bound driver and pass its address to a per-device C attribute wrapper. Register after GPU initialization and drain sysfs readers before dropping the owner. Remove early global pointer publication so competing or failing probes cannot replace another device snapshot. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/driver.rs | 17 +++-- drivers/gpu/drm/asahi/gpu.rs | 11 +-- drivers/gpu/drm/asahi/stats.rs | 23 +++--- drivers/gpu/drm/asahi/sysfs.c | 99 +++++++++++--------------- drivers/gpu/drm/asahi/sysfs.h | 6 +- drivers/gpu/drm/asahi/sysfs_exports.rs | 49 ++++++------- 6 files changed, 94 insertions(+), 111 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 3653ee155d2e7..5c1c7591bcfe5 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -51,6 +51,10 @@ pub(crate) struct AsahiDriver { /// Raw `struct device *` of the platform device. Stored once at probe /// time so the `Drop` impl can find the sysfs file on unregister. raw_dev: *mut bindings::device, + raw_stats: *mut core::ffi::c_void, + // Own the allocation until unregister has drained all sysfs readers. + #[expect(unused)] + stats: Arc, } unsafe impl Send for AsahiDriver {} @@ -61,7 +65,7 @@ impl Drop for AsahiDriver { // SAFETY: `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { - crate::sysfs_exports::unregister(self.raw_dev); + crate::sysfs_exports::unregister(self.raw_dev, self.raw_stats); } } } @@ -250,8 +254,13 @@ impl platform::Driver for AsahiDriver { // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); - crate::sysfs_exports::register(raw_dev, *module_parameters::stats_export.value() != 0)?; - - Ok(Self { drm, raw_dev }) + let stats = (*drm).gpu.stats_snapshot(); + let raw_stats = crate::sysfs_exports::register( + raw_dev, + Arc::as_ptr(&stats), + *module_parameters::stats_export.value() != 0, + )?; + + Ok(Self { drm, raw_dev, raw_stats, stats }) } } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index bcb3402cdf26a..39cb3c8cc99f3 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -83,7 +83,6 @@ use crate::{ queue, regs, stats, - sysfs_exports, workqueue, // }; @@ -267,6 +266,8 @@ pub(crate) struct GpuManager { /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. pub(crate) trait GpuManager: Send + Sync { + fn stats_snapshot(&self) -> Arc; + /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -732,10 +733,6 @@ impl GpuManager::ver { let event_manager_clone = event_manager.clone(); let buffer_mgr_clone = buffer_mgr.clone(); let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; - // Publish the snapshot pointer to the C sysfs shim. It is cleared - // (by the same shim) on registration removal; until then readers see - // a stable pointer and AtomicU* races that govern. - sysfs_exports::set_snapshot_ptr(Arc::as_ptr(&stats_snap)); let stats_snap_clone = stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( @@ -1252,6 +1249,10 @@ impl GpuManager::ver { #[versions(AGX)] impl GpuManager for GpuManager::ver { + fn stats_snapshot(&self) -> Arc { + self.stats.clone() + } + fn as_any(&self) -> &dyn Any { self } diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs index e3160e1a5c632..772d969d9baff 100644 --- a/drivers/gpu/drm/asahi/stats.rs +++ b/drivers/gpu/drm/asahi/stats.rs @@ -6,7 +6,7 @@ //! (see `fw::channels.rs::StatsMsg`) and exposes them to the sysfs file //! `agx_stats` (registered from `sysfs.c`). //! -//! The snapshot is process-local and process-private. The firmware-stat +//! The snapshot belongs to one GPU manager. The firmware-stat //! decoder that feeds it lives in `StatsChannel::poll` (`channel.rs`), inside //! the versioned context where `StatsMsg` is nameable; `note_job()` is called //! from the `recv_message` rtkit callback (so the firmware mailbox IRQ @@ -35,12 +35,10 @@ use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; -/// Process-wide snapshot of AGX firmware stats. +/// Per-device snapshot of AGX firmware stats. /// -/// Lives in the `GpuManager`; the C static `asahi_stats_snapshot_ptr` (see -/// `sysfs.c`) holds its address for the C `show` callback, so the field -/// layout is a cross-language contract: `#[repr(C)]` makes the declaration -/// order binding and `sysfs.c` asserts the offsets at build time. +/// The manager and bound driver retain Arcs. The driver's Arc outlives the +/// per-device C sysfs registration; repr(C) establishes the shared layout. #[derive(Default)] #[repr(C)] pub(crate) struct StatsSnapshot { @@ -62,12 +60,13 @@ pub(crate) struct StatsSnapshot { pub(crate) jobs: AtomicU64, } -/// Pointer to the live snapshot: the C static `asahi_stats_snapshot_ptr` -/// (see `sysfs.c`) is the single storage location, written through -/// `sysfs_exports::set_snapshot_ptr` and read by the C `show` callback via -/// `READ_ONCE`. The pointer is established before the DRM device registers -/// and removed on unregister; the pointed-to fields are `AtomicU*` and carry -/// the data races themselves. +// Keep the Rust half of the C sysfs layout contract checked as well. +const _: () = { + assert!(core::mem::offset_of!(StatsSnapshot, pstate) == 16); + assert!(core::mem::offset_of!(StatsSnapshot, busy_ns) == 40); + assert!(core::mem::offset_of!(StatsSnapshot, jobs) == 48); + assert!(core::mem::size_of::() == 56); +}; impl StatsSnapshot { /// Bump the completed-submission counter (called from the queue diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 923e4013804b7..961512bf7309d 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -1,22 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // -// Sysfs shim for the AGX firmware stats export. -// -// The Rust side keeps the stats state in `crate::stats::StatsSnapshot` and -// publishes its raw pointer to the static `asahi_stats_snapshot_ptr` below -// (an atomic `u64` so the writer and reader do not need locking). This file -// owns the actual `device_attribute` and the formatted output, because -// `device_create_file` and the `device_attribute` macros are not in the -// Rust bindgen bindings for this kernel tree. -// -// The contract is one read-only file: -// /sys/class/drm/cardX/device/agx_stats -// Format: `key value\n` per line, ASCII integers, owner-readable. +// Per-device sysfs shim. Rust owns the snapshot through an Arc in the bound +// driver. The attribute wrapper is removed and its readers drained before +// that Arc is dropped. Atomic fields may reflect different sample instants. #include #include #include #include +#include #include #include #include @@ -24,18 +16,7 @@ #include "sysfs.h" -/* - * Mirror of `#[repr(C)] crate::stats::StatsSnapshot` field layout. Rust sets - * `asahi_stats_snapshot_ptr` (an atomic u64) to the heap address of one of - * these, and clears it on unregister. Field reads are all `Relaxed` atomic - * u32 / u64 (the C side uses READ_ONCE). - * - * The Rust struct carries `#[repr(C)]` so this declaration order is binding; - * the BUILD_BUG_ONs in asahi_sysfs_register() turn any drift (a repr(Rust) - * struct is silently reordered by rustc, which once cross-aligned this - * readout: busy_ns bounced in 2^32 steps and jobs read 0 while incrementing) - * into a compile error. - */ +/* Mirror of repr(C) Rust atomics; READ_ONCE accesses the current values. */ struct asahi_stats_snapshot { u32 util1; u32 util2; @@ -51,35 +32,22 @@ struct asahi_stats_snapshot { u64 jobs; }; -static int asahi_stats_export_enabled; - -/* - * Set by Rust through asahi_stats_set_snapshot_ptr(); read via READ_ONCE. - * A NULL (0) pointer means the device has not yet exposed stats (or has been - * unregistered). The Rust side owns the lifetime of the pointed-to struct. - */ -unsigned long long asahi_stats_snapshot_ptr; - -void asahi_stats_set_snapshot_ptr(unsigned long long p) -{ - WRITE_ONCE(asahi_stats_snapshot_ptr, p); -} +struct asahi_stats_attribute { + struct device_attribute attr; + const struct asahi_stats_snapshot *snapshot; + bool export_enabled; +}; static ssize_t agx_stats_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct asahi_stats_snapshot __rcu *snap; + struct asahi_stats_attribute *stats = + container_of(attr, struct asahi_stats_attribute, attr); + const struct asahi_stats_snapshot *snap = stats->snapshot; ssize_t n = 0; - (void)attr; - - rcu_read_lock(); - snap = (struct asahi_stats_snapshot __rcu *) - READ_ONCE(asahi_stats_snapshot_ptr); - if (!snap || !READ_ONCE(asahi_stats_export_enabled)) { - rcu_read_unlock(); - return scnprintf(buf, PAGE_SIZE, "unsupported\n"); - } + if (!stats->export_enabled) + return sysfs_emit(buf, "unsupported\n"); n += scnprintf(buf + n, PAGE_SIZE - n, "busy_ns %llu\n", (unsigned long long)READ_ONCE(snap->busy_ns)); n += scnprintf(buf + n, PAGE_SIZE - n, "jobs %llu\n", @@ -100,22 +68,22 @@ static ssize_t agx_stats_show(struct device *dev, (u32)READ_ONCE(snap->temperature_raw)); n += scnprintf(buf + n, PAGE_SIZE - n, "temperature_scale %u\n", (u32)READ_ONCE(snap->temperature_scale)); - rcu_read_unlock(); return n; } -static DEVICE_ATTR_RO(agx_stats); /* * Called from Rust's `AsahiDriver::probe` after the DRM device is * registered. Returns 0 on success, or a negative errno. */ -int asahi_sysfs_register(struct device *dev, int export_enabled) +int asahi_sysfs_register(struct device *dev, const void *snapshot, + int export_enabled, void **handle) { + struct asahi_stats_attribute *stats; int ret; - if (!dev) + if (!dev || !snapshot || !handle) return -ENODEV; BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, pstate) != 16); @@ -123,23 +91,38 @@ int asahi_sysfs_register(struct device *dev, int export_enabled) BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, jobs) != 48); BUILD_BUG_ON(sizeof(struct asahi_stats_snapshot) != 56); - WRITE_ONCE(asahi_stats_export_enabled, export_enabled); + stats = kzalloc_obj(*stats); + if (!stats) + return -ENOMEM; + + sysfs_attr_init(&stats->attr.attr); + stats->attr.attr.name = "agx_stats"; + stats->attr.attr.mode = 0444; + stats->attr.show = agx_stats_show; + stats->snapshot = snapshot; + stats->export_enabled = export_enabled; - ret = device_create_file(dev, &dev_attr_agx_stats); - if (ret) + ret = device_create_file(dev, &stats->attr); + if (ret) { + kfree(stats); return ret; + } + *handle = stats; return 0; } EXPORT_SYMBOL_GPL(asahi_sysfs_register); -void asahi_sysfs_unregister(struct device *dev) +void asahi_sysfs_unregister(struct device *dev, void *handle) { - if (!dev) + struct asahi_stats_attribute *stats = handle; + + if (!dev || !stats) return; - device_remove_file(dev, &dev_attr_agx_stats); - WRITE_ONCE(asahi_stats_snapshot_ptr, 0); + /* Removal drains active show callbacks before either allocation dies. */ + device_remove_file(dev, &stats->attr); + kfree(stats); } EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index ed96fbd3afe50..5969a1718925f 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -5,8 +5,8 @@ struct device; -int asahi_sysfs_register(struct device *dev, int export_enabled); -void asahi_sysfs_unregister(struct device *dev); -void asahi_stats_set_snapshot_ptr(unsigned long long p); +int asahi_sysfs_register(struct device *dev, const void *snapshot, + int export_enabled, void **handle); +void asahi_sysfs_unregister(struct device *dev, void *handle); #endif /* _ASAHI_SYSFS_H */ diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index c9cdf1c9ef641..421f89dcff2e2 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -1,46 +1,37 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! FFI declarations for the C `sysfs.c` shim. -//! -//! The Rust driver keeps the firmware stats in [`crate::stats::StatsSnapshot`] -//! and uses these shims to register / unregister the sysfs file and to publish -//! the snapshot pointer for the C-side `show` callback. +//! Per-device sysfs registration. The caller retains the snapshot Arc until +//! unregister has drained active readers and freed the C attribute wrapper. -use core::ffi::{c_int, c_ulonglong}; +use core::ffi::{c_int, c_void}; +use crate::stats::StatsSnapshot; extern "C" { - /// Register the `agx_stats` sysfs file on `dev`; `export_enabled` 0 - /// makes the file print `unsupported`. Returns 0 on success. - fn asahi_sysfs_register(dev: *mut kernel::bindings::device, export_enabled: c_int) -> c_int; - - /// Unregister the `agx_stats` sysfs file and clear the snapshot pointer. - fn asahi_sysfs_unregister(dev: *mut kernel::bindings::device); - - /// Publish the snapshot pointer for the C-side `show` callback (the C - /// static `asahi_stats_snapshot_ptr` is the single storage location). - fn asahi_stats_set_snapshot_ptr(p: c_ulonglong); + fn asahi_sysfs_register( + dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, + export_enabled: c_int, + handle: *mut *mut c_void, + ) -> c_int; + fn asahi_sysfs_unregister(dev: *mut kernel::bindings::device, handle: *mut c_void); } -/// Register the sysfs file. Safe to call from `AsahiDriver::probe` after the -/// DRM device has been registered. +/// Register only after initialization succeeds, keeping the snapshot Arc alive. pub(crate) fn register( dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, export_enabled: bool, -) -> kernel::error::Result { - let ret = unsafe { asahi_sysfs_register(dev, export_enabled as c_int) }; +) -> kernel::error::Result<*mut c_void> { + let mut handle = core::ptr::null_mut(); + let ret = unsafe { asahi_sysfs_register(dev, snapshot, export_enabled as c_int, &mut handle) }; if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { - Ok(()) + Ok(handle) } } -/// Unregister the sysfs file. Safe to call from the device release path. -pub(crate) fn unregister(dev: *mut kernel::bindings::device) { - unsafe { asahi_sysfs_unregister(dev) }; -} - -/// Publish the address of the live `StatsSnapshot` to the C shim. -pub(crate) fn set_snapshot_ptr(ptr: *const crate::stats::StatsSnapshot) { - unsafe { asahi_stats_set_snapshot_ptr(ptr as c_ulonglong) }; +/// Drain readers before the caller drops its snapshot Arc. +pub(crate) fn unregister(dev: *mut kernel::bindings::device, handle: *mut c_void) { + unsafe { asahi_sysfs_unregister(dev, handle) }; } From 14e101ee2b8b599cf627015368e16ff8b3d2b95f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 19:01:16 +1100 Subject: [PATCH 0565/1579] cpufreq: apple: retain the unmanaged T8152 P-state ceiling Removing the private thermal-policy prerequisite for T8140 must not admit high P-state tables on other SoCs without qualified protection. Keep the T8152 ceiling at state 3 while T8140 uses thermal zones. Carry the existing public T8152 metadata and per-SoC timeout convention into this older thermal branch, so the ceiling applies when the branch is integrated with the current Apple CPU frequency support. The T8152 metadata follows Acelogic's public implementation; no contributor certification is added on their behalf. Other SoCs retain the default transition timeout and their existing admission behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 41 ++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index f2a1e91371be4..eb8b683572231 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -63,6 +63,9 @@ struct apple_soc_cpufreq_info { bool has_ps2; bool verify_transition; + bool needs_thermal_policy; + u32 transition_timeout_us; + u64 max_unmanaged_pstate; u64 min_pstate; u64 max_pstate; u64 cur_pstate_mask; @@ -122,6 +125,18 @@ static const struct apple_soc_cpufreq_info soc_t8140_info = { .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, }; +/* T8152 ACC uses state 2 as the first OPP and shares P/M requests. */ +static const struct apple_soc_cpufreq_info soc_t8152_info = { + .verify_transition = true, + .needs_thermal_policy = true, + .transition_timeout_us = 2000, + .max_unmanaged_pstate = 3, + .min_pstate = 2, + .max_pstate = 31, + .ps1_mask = APPLE_DVFS_CMD_PS1, + .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, +}; + static const struct apple_soc_cpufreq_info soc_default_info = { .has_ps2 = false, .max_pstate = 15, @@ -151,9 +166,18 @@ static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { .compatible = "apple,t8140-cluster-cpufreq", .data = &soc_t8140_info, }, + { + .compatible = "apple,t8152-cluster-cpufreq", + .data = &soc_t8152_info, + }, {} }; +static u32 apple_soc_cpufreq_timeout(const struct apple_soc_cpufreq_info *info) +{ + return info->transition_timeout_us ?: APPLE_DVFS_TRANSITION_TIMEOUT; +} + static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu) { struct cpufreq_policy *policy; @@ -242,7 +266,7 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, !(reg & APPLE_DVFS_CMD_BUSY) && FIELD_GET(APPLE_DVFS_CMD_PS1, reg) == pstate, - 2, APPLE_DVFS_TRANSITION_TIMEOUT)) { + 2, apple_soc_cpufreq_timeout(priv->info))) { /* Do not issue another request or guess a rollback after failure. */ priv->transition_failed = true; dev_err(priv->cpu_dev, @@ -296,6 +320,7 @@ static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) { + struct cpufreq_frequency_table *p; int ret, i; unsigned int transition_latency; void __iomem *reg_base; @@ -364,7 +389,7 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) /* Registration must not silently reset an unknown inherited state. */ ret = readq_poll_timeout_atomic(reg_base + APPLE_DVFS_CMD, cmd, !(cmd & APPLE_DVFS_CMD_BUSY), 2, - APPLE_DVFS_TRANSITION_TIMEOUT); + apple_soc_cpufreq_timeout(info)); if (ret) goto out_free_cpufreq_table; state = FIELD_GET(APPLE_DVFS_CMD_PS1, cmd); @@ -388,7 +413,7 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); if (!transition_latency) { /* Conservative transaction bound, not a measured transition time. */ - transition_latency = APPLE_DVFS_TRANSITION_TIMEOUT * NSEC_PER_USEC; + transition_latency = apple_soc_cpufreq_timeout(info) * NSEC_PER_USEC; if (info->verify_transition) transition_latency *= 2; } @@ -397,6 +422,16 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) policy->dvfs_possible_from_any_cpu = true; policy->fast_switch_possible = !info->verify_transition; policy->suspend_freq = freq_table[0].frequency; + /* Unqualified SoCs still require protection above their safe ceiling. */ + if (info->needs_thermal_policy) { + cpufreq_for_each_valid_entry(p, policy->freq_table) { + if (p->driver_data <= info->max_unmanaged_pstate) + continue; + dev_err(cpu_dev, "higher P-states require a qualified thermal policy\n"); + ret = -ENODEV; + goto out_free_cpufreq_table; + } + } return 0; From 16fcece08bc32fb83d3e979b8d30ae0abab87d66 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 19:37:23 +1100 Subject: [PATCH 0566/1579] Bluetooth: mt7932: accept nonzero HCI registration indices hci_register_dev returns a nonnegative controller index on success. Treating every nonzero value as an error leaves a successfully published controller with registered=false. Removal then skips hci_unregister_dev and frees its callback state while the HCI core still owns the controller. Reject only negative registration results so every published controller is tracked and unregistered before transport and callback state cleanup. The PCI probe continues to return zero for a successful registration. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 493f95cbcda09..ab41a52ac656c 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1156,7 +1156,8 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) bt7932_channels(bt); mutex_unlock(&bt->lock); ret = hci_register_dev(bt->hdev); - if (ret) + /* Registration returns the nonnegative HCI index on success. */ + if (ret < 0) goto retained_fault; bt->registered = true; dev_info(&pdev->dev, "BT_HCI_REGISTERED; setup/discovery/connection remain separate gates\n"); From bdf09f513e0ebf2e794c72fc969829e19ea4b619 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 4 Aug 2026 14:05:23 +0200 Subject: [PATCH 0567/1579] mm: export vmf_insert_pfn_prot_mkwrite(), change variants to inline Right now, users of .pfn_mkwrite() have no way to create a PTE that has gone through maybe_mkwrite(). Because vma_set_page_prot() will have cleared the writable PTE bit, users of fixup_user_fault() will see a read-only PTE and have no clue that the page needs a *second* fault to reach its final status. Handling this in fixup_user_fault() is problematic: the information about the presence of *_mkwrite is only recorded in vma->vm_page_prot, which is an opaque pgprot_t, therefore only follow_pfnmap_start() knows how to retrieve it. There are actually some preexisting functions that suggest how this is supposed to be handled, namely vmf_insert_page_mkwrite() and vmf_insert_pfn_pmd(). Fixing the drivers requires similar variants of vm_insert_pfn(), namely vmf_insert_pfn_mkwrite() for the common case where vma->vm_page_prot is okay, and vmf_insert_pfn_prot_mkwrite() when really all parameters are needed. This makes it possible to fix drivers that use .pfn_mkwrite together with vmf_insert_pfn() and vmf_insert_pfn_prot(). Since vmf_insert_pfn_prot_mkwrite() is the most general variant and all the others are just special cases, turn them into inline functions in the header. Fixes: 28e3918179aa ("drm/gem-shmem: Track folio accessed/dirty status in mmap") Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini (cherry picked from commit c1fd108e821da130297fbfedd3305b15d34f2e35) Signed-off-by: Diljit Singh (cherry picked from commit b098e9ca8189ccc388a678561cb73b23f896b151) From 1ae0ac48652af7839d0c411ed961adae90875879 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 19:42:31 +1100 Subject: [PATCH 0568/1579] soc: apple: sep: establish authenticated sensor sessions on J700 The J700 enclave restores stored identities but cannot open a capture session without the sensor's nonce and signed root-key container. Repeated sensor registration must precede CLEAR_STATE, and communication initialization must run before restoring the biometric state. Describe the nonce session in the existing platform profile and perform its initialization, platform identity, sensor serial, nonce and root-key steps through the existing SBIO and SPI transports. Validate the complete nonce frame with the existing CRC reader and refuse a missing identity, firmware or unsuccessful exchange. Load the externally supplied signed key once per enclave lifetime; do not log identity or cryptographic payloads. For this protocol, apply encrypted sensor parameters before calibration and complete initialization afterwards. Use calibration source 6 while retaining the existing source and ordering for other profiles. Enable owner export and user-before-master saves for J700, then refresh the device view and match credential after restoring its components. Retain the existing xART write policy. Protocol facts come from our private research and Linux observations. This implementation is independently written; no private driver code or device material is included. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 2 +- drivers/soc/apple/profile.rs | 6 ++ drivers/soc/apple/sbio.rs | 190 ++++++++++++++++++++++++++++++----- drivers/soc/apple/sensor.rs | 12 +++ drivers/soc/apple/sep.rs | 3 + 5 files changed, 186 insertions(+), 27 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 743bb889b069a..099be49d2f658 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1988,6 +1988,7 @@ F: drivers/iio/common/aop_sensors/ F: drivers/soc/apple/aop.rs F: drivers/thermal/apple-smc-cpu-thermal.c F: drivers/tty/serial/apple_dockchannel* +F: include/uapi/linux/apple-sep.h F: include/linux/soc/apple/smc-thermal.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs @@ -2630,7 +2631,6 @@ F: include/dt-bindings/pinctrl/apple.h F: include/linux/mfd/macsmc.h F: include/linux/soc/apple/* F: include/uapi/drm/asahi_drm.h -F: include/uapi/linux/apple-sep.h F: net/bluetooth/brcm.c F: net/bluetooth/brcm.h diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index ee2f43096740d..91ad39a45b6d0 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj +// Copyright 2026 Ryan Murray //! Typed platform profiles for the SEP driver. //! @@ -124,6 +125,8 @@ pub(crate) struct PlatformProfile { pub(crate) firmware_region: &'static CStr, /// Identity keybag CREATE_KEYBAG field encoding (per-SoC; see [`KeybagCreate`]). pub(crate) keybag_create: KeybagCreate, + /// Require nonce authentication and signed sensor material before capture. + pub(crate) nonce_session: bool, } const T8103: PlatformProfile = PlatformProfile { @@ -151,6 +154,7 @@ const T8103: PlatformProfile = PlatformProfile { bag_type: 0x20000, arg: 0, }, + nonce_session: false, }; const T6020: PlatformProfile = PlatformProfile { @@ -178,6 +182,7 @@ const T6020: PlatformProfile = PlatformProfile { bag_type: 0, arg: -1, }, + nonce_session: false, }; /// MacBook Neo. The J700 ADT records a pre-booted SEP, a spi2 Mesa sensor @@ -209,6 +214,7 @@ const T8140: PlatformProfile = PlatformProfile { bag_type: 0, arg: -1, }, + nonce_session: true, }; static_assert!(T8103.shmem_capacity == 0x30000); diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 6c046bd4285c5..fd8bbe36ff764 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj +// Copyright 2026 Ryan Murray //! Touch ID: the biometric endpoint (SBIO) and its `/dev/sep-bio` interface. //! The enclave matches; no biometric image ever crosses to userspace. @@ -367,12 +368,9 @@ impl SepData { } } - // The J414s owner exported successfully from state 0x3 in a bounded - // no-finger probe, although an earlier pre-UUID boot saw the master - // refuse an early save from the same state. Keep this path opt-in - // until a new enrollment and reboot prove that the owner blob belongs - // to the new user context. - if *module_parameters::j414s_persistent_enrol.value() != 0 { + // J700 needs the owner component even when state 0x3 has no + // save-pending bit. Retain the opt-in policy on older profiles. + if self.profile.nonce_session || *module_parameters::j414s_persistent_enrol.value() != 0 { if owner_initial_state == Some(0x3) && !self.save_catacomb( crate::sbio::CatacombUser::OWNER, @@ -761,7 +759,7 @@ impl SepData { } fn save_all_components(&self, user: crate::sbio::UserId) -> bool { - if *module_parameters::j414s_persistent_enrol.value() != 0 { + if self.profile.nonce_session || *module_parameters::j414s_persistent_enrol.value() != 0 { return self.save_all_components_user_first(user); } // On m2fix11, a completed enrollment marked both master and user 0x7, @@ -1006,10 +1004,42 @@ impl SepData { self.register_ool(&self.ool_sbio)?; self.sbio_ready.store(true, Relaxed); + if self.profile.nonce_session { + let word = 6u32; + // Communication must be initialized before restoring biometric state. + let result = self.sbio_transfer_raw( + 0x73, c"INIT_SBIO_COMMUNICATION", &word.to_le_bytes(), + ); + match result { + Ok(done) if done.status.is_ok() => { + dev_info!(self.dev, "sbio: communication initialization accepted\n"); + } + Ok(done) => { + self.sbio_ready.store(false, Relaxed); + dev_err!(self.dev, "sbio: communication init refused: status {}, {} bytes\n", + done.status, done.payload.len()); + return Err(EIO); + } + Err(e) => { + self.sbio_ready.store(false, Relaxed); + dev_err!(self.dev, "sbio: communication init transport failed: {:?}\n", e); + return Err(e); + } + } + } Ok(()) } fn attach_bringup(&self) { + if self.profile.nonce_session { + if !self.bring_sensor_online() { + dev_warn!( + self.dev, + "sensor: authenticated attach incomplete; capture unavailable\n" + ); + } + return; + } if !sensor::is_bound() { dev_warn!( self.dev, @@ -1033,18 +1063,34 @@ impl SepData { return false; }; + // Newer sensors require the encrypted parameters and transparent + // channel before the calibration challenge can be serviced. + let parameters = if self.profile.nonce_session { + match self.apply_sensor_parameters() { + Some(parameters) => Some(parameters), + None => return false, + } + } else { + None + }; + if !self.sensor_calibrated.load(Relaxed) { if !self.calibrate_sensor() { return false; } self.sensor_calibrated.store(true, Relaxed); - let Some(reloaded_patch) = self.wake_sensor() else { - return false; - }; - patch = reloaded_patch; + if parameters.is_none() { + let Some(reloaded_patch) = self.wake_sensor() else { + return false; + }; + patch = reloaded_patch; + } } - let ok = self.complete_bringup(patch); + let ok = match parameters { + Some(parameters) => self.finish_sensor_setup(patch, parameters), + None => self.complete_bringup(patch), + }; // The sensor is patched and idle here -- the only safe moment to // configure the data-ready interrupt, which is then left alone for the // driver's life. Interrupt capture is the default; a machine that does @@ -1457,6 +1503,22 @@ impl SepData { let stage = self.bringup.load(Relaxed); dev_info!(self.dev, "sbio: registering sensor at bring-up stage {}\n", stage); + if self.profile.nonce_session { + // J700 needs identification before CLEAR_STATE on every reset. + let platform = match Self::sensor_platform_identity() { + Ok(platform) => platform, + Err(error) => { + dev_err!(self.dev, "sensor: platform identity unavailable: {:?}\n", error); + return false; + } + }; + if !self.sensor_session_request(0x48, c"PLATFORM_IDENTITY", &platform) + || self.sbio_expect_ok(&crate::sbio::sbio_register_sensor(id)).is_none() + { + return false; + } + } + if stage >= BRINGUP_ESTABLISHED { let _ = self.log_identity_count(c"before CLEAR_STATE for sensor re-registration"); if self @@ -1468,7 +1530,7 @@ impl SepData { let _ = self.log_identity_count(c"after CLEAR_STATE for sensor re-registration"); } - if stage == BRINGUP_FRESH { + if stage == BRINGUP_FRESH && !self.profile.nonce_session { let op = crate::sbio::sbio_register_sensor(id); match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { @@ -1490,6 +1552,9 @@ impl SepData { let op = crate::sbio::sbio_register_sensor_serial(&serial); match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { + if self.profile.nonce_session && !self.authenticate_sensor_session() { + return false; + } self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); let _ = self.log_identity_count(c"after sensor serial registration"); true @@ -1498,6 +1563,70 @@ impl SepData { } } + fn sensor_platform_identity() -> Result<[u8; 12]> { + let node = kernel::of::root().ok_or(ENODEV)?; + let property: KVec = node.get_property(c"serial-number")?; + let length = property.iter().position(|byte| *byte == 0).ok_or(EINVAL)?; + let mut identity = [0; 12]; + if length == 0 || length > identity.len() || property.len() != length + 1 { + return Err(EINVAL); + } + identity[..length].copy_from_slice(&property[..length]); + Ok(identity) + } + + fn sensor_session_request(&self, opcode: u16, name: &'static CStr, bytes: &[u8]) -> bool { + match self.sbio_transfer_raw(opcode, name, bytes) { + Ok(reply) if reply.status.is_ok() => true, + Ok(reply) => { + dev_err!(self.dev, "sensor: {} refused with status {}\n", name, reply.status); + false + } + Err(error) => { + dev_err!(self.dev, "sensor: {} transfer failed: {:?}\n", name, error); + false + } + } + } + + fn authenticate_sensor_session(&self) -> bool { + let nonce = match sensor::session_nonce() { + Ok(nonce) => nonce, + Err(error) => { + dev_err!(self.dev, "sensor: nonce read failed: {:?}\n", error); + return false; + } + }; + if !self.sensor_session_request(0x81, c"SENSOR_NONCE", &nonce) { + return false; + } + + if !self.sensor_key_loaded.load(Relaxed) { + let firmware = match kernel::firmware::Firmware::request( + c"apple/msrk-j700.bin", &self.dev, + ) { + Ok(firmware) => firmware, + Err(error) => { + dev_err!( + self.dev, + "sensor: signed root-key container unavailable: {:?}\n", + error + ); + return false; + } + }; + if firmware.data().is_empty() + || !self.sensor_session_request(0x86, c"SENSOR_ROOT_KEY", firmware.data()) + { + return false; + } + self.sensor_key_loaded.store(true, Relaxed); + } + // Each identification starts a new sensor-side exchange, even though + // the signed key remains loaded for the lifetime of the enclave. + self.establish_session() + } + fn wake_sensor(&self) -> Option { let _source = sensor::power_source(); let cycled = sensor::power_cycle(); @@ -1607,22 +1736,16 @@ impl SepData { return false; }; - let request = match crate::sbio::SbioCalibration::new(&blob) { + let source = if self.profile.nonce_session { 6 } else { SBIO_CALIBRATION_SOURCE }; + let request = match crate::sbio::SbioCalibration::new(&blob, source) { Ok(request) => request, Err(_) => { return false; } }; - if self - .sbio_transfer_raw(request.opcode(), request.name(), request.payload()) - .ok() - .filter(|done| done.status.is_ok()) - .is_none() - { - dev_err!( - self.dev, - "sensor: 0x5b LOAD_CALIBRATION failed. Without it every capture is refused with status 1.\n" - ); + if !self.sensor_session_request(request.opcode(), request.name(), request.payload()) { + dev_err!(self.dev, "sensor: calibration refused (source {}, {} bytes)\n", + source, request.payload().len()); return false; } @@ -1636,6 +1759,10 @@ impl SepData { return false; }; + self.finish_sensor_setup(patch, params) + } + + fn finish_sensor_setup(&self, patch: PatchLoaded, params: ParametersApplied) -> bool { let op = crate::sbio::sbio_complete_init(patch, params); if self.sbio_expect_ok(&op).is_none() { dev_err!( @@ -2542,6 +2669,17 @@ impl SepData { let restored = self.restore_all_components(); dev_info!(self.dev, "sbio: cold-match preparation {}, component restore {}\n", prepared, restored); + // Restore can replace the credential and the device's identity view. + // Refresh both after loading J700's persistent components, before + // admitting a verify operation. + let match_context = if self.profile.nonce_session { + let synced = restored && self.sync_device_view(); + self.device_view_synced.store(synced, Relaxed); + synced && crate::sbio::UserId::new(SBIO_PROBE_USER_ID) + .is_some_and(|user| self.establish_scrd_match_context(user)) + } else { + match_context + }; let identities = self.log_identity_count(c"after Catacomb restore with sensor registered"); let identity_ready = matches!(identities, Some(n) if n > 0); let ready = prepared && match_context && restored && identity_ready @@ -2949,10 +3087,10 @@ pub(crate) const SBIO_CALIBRATION_SOURCE: u32 = 3; pub(crate) struct SbioCalibration(KVec); impl SbioCalibration { - pub(crate) fn new(blob: &crate::CalibrationBlob) -> Result { + pub(crate) fn new(blob: &crate::CalibrationBlob, source: u32) -> Result { let bytes = blob.bytes(); let mut body = KVec::new(); - body.extend_from_slice(&SBIO_CALIBRATION_SOURCE.to_le_bytes(), GFP_KERNEL)?; + body.extend_from_slice(&source.to_le_bytes(), GFP_KERNEL)?; body.extend_from_slice(bytes, GFP_KERNEL)?; Ok(SbioCalibration(body)) } diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 8fec6b71c8715..3fcba35e39428 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj +// Copyright 2026 Ryan Murray //! The fingerprint sensor over SPI. @@ -581,6 +582,17 @@ pub(crate) fn read_sensor_serial() -> Result { Ok(SensorSerial(serial)) } +/// Read the nonce as one frame; a separate acknowledgment would consume it. +pub(crate) fn session_nonce() -> Result<[u8; 32]> { + command(&[0x80, 0x35, 0, 7, 0, 0, 0])?; + let mut value = [0; 32]; + read_framed_verified(9 + value.len() + CRC_LEN, 9, &mut value)?; + if value.iter().all(|byte| *byte == 0) { + return Err(EIO); + } + Ok(value) +} + pub(crate) fn read_session_reply() -> Result<[u8; SESSION_SHARE_LEN]> { let mut share = [0u8; SESSION_SHARE_LEN]; read_framed(SESSION_REPLY_LEN, SESSION_REPLY_AT, &mut share)?; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 428344654028f..392ad9f05d452 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj +// Copyright 2026 Ryan Murray //! Driver for the Apple SEP (Secure Enclave Processor): attaches to a running //! SEP over the AP mailbox, drives Touch ID enrol/verify via `/dev/sep-bio`, //! and registers a hwrng and a SEP-backed `trusted` key source. @@ -577,6 +578,7 @@ struct SepData { templates_restored: Atomic, sensor_calibrated: Atomic, + sensor_key_loaded: Atomic, enrol_open: Atomic, @@ -787,6 +789,7 @@ impl SepData { touchid_failed: Atomic::new(false), enrol_open: Atomic::new(false), sensor_calibrated: Atomic::new(false), + sensor_key_loaded: Atomic::new(false), templates_restored: Atomic::new(false), enrol_material <- new_mutex!(None), enrol_identity_candidates <- new_mutex!(KVec::new()), From 8dcfee75dee4e6590544e28f50b0131178b9f659 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 4 Aug 2026 14:05:24 +0200 Subject: [PATCH 0569/1579] drm/shmem_helper: use vmf_insert_pfn_mkwrite() This ensures that KVM or VFIO correctly see a writable PTE when they request one. Otherwise, a guest write to an unpopulated PTE from a mapping backed by a DRM GEM BO triggers a VM exit with EFAULT. The code actually is simpler, because the same logic already applied to the hugepage mapping case using vmf_insert_pfn_pmd(). Reported-by: Sergio Lopez Link: https://lore.kernel.org/kvm/20260729072044.25796-1-slp@redhat.com/ Tested-by: Sergio Lopez Reviewed-by: Boris Brezillon Fixes: 28e3918179aa ("drm/gem-shmem: Track folio accessed/dirty status in mmap") Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini (cherry picked from commit c2b3c978fe93b0146c70eb63362e6f88c757d8b0) Signed-off-by: Diljit Singh (cherry picked from commit b2f69c626877662885704b9ab01718f8188759a6) From 272ec02e42f88542708672697b8492068ab58337 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 4 Aug 2026 14:05:27 +0200 Subject: [PATCH 0570/1579] mm: pull writability check to follow_pfnmap_start() All callers of follow_pfnmap_start() except s390_pci_mmio_write() are following it, if they are doing a write, with a check that args.writable is true; for s390_pci_mmio_write() that's a bug. Also, most of them return -EFAULT if it is not. Pull the check directly into follow_pfnmap_start() through another input parameter args.write_fault, to eliminate the need to do it in the caller. This also fixes an issue where follow_pfnmap_start() would return 0 for a PFN that is mapped read-only, and the caller would not attempt to call fixup_user_fault() on it; this can happen with vm_ops that set .pfn_mkwrite(), for example. Instead, now the caller (for example hva_to_pfn_remapped()) sees an error, does attempt to fix it, and only returns -EFAULT if the fixup was fruitless. Reported-by: Sergio Lopez Fixes: 28e3918179aa ("drm/gem-shmem: Track folio accessed/dirty status in mmap") Link: https://lore.kernel.org/kvm/CAAiTLFU1ALsDoJoKW3d9bUvv990AozAoX=bEHmfnG54qyBAHFg@mail.gmail.com/ Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini (cherry picked from commit 2e835aa2526160cf112c5fe5257b4299d46107d1) Signed-off-by: Diljit Singh (cherry picked from commit 3607582ad7900e9a9d15ba53ecce85089da29d72) From af568c5f1c10da68fc03d0b411189b0f33cafb2e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 19:53:55 +1100 Subject: [PATCH 0571/1579] drm: apple: guard the custom fbdev probe with its configuration Apple DRM can be enabled without DRM_FBDEV_EMULATION. In that configuration the DMA fbdev helper does not declare its probe function, but the custom Apple probe still calls it and fails to compile. Compile the custom probe and select it only when fbdev emulation is enabled. Otherwise use the DMA helper driver-ops macro, which supplies a NULL probe. The H17P ARGB console policy is unchanged when enabled. Fixes: 544eb67a1a17 ("drm: apple: use ARGB for H17P fbdev") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index ef3a45a73a075..aabf7f29f96d8 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -65,6 +65,7 @@ static int apple_drm_gem_dumb_create(struct drm_file *file_priv, return drm_gem_dma_dumb_create_internal(file_priv, drm, args); } +#ifdef CONFIG_DRM_FBDEV_EMULATION static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, struct drm_fb_helper_surface_size *sizes) { @@ -85,10 +86,15 @@ static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, return drm_fbdev_dma_driver_fbdev_probe(helper, sizes); } +#endif static const struct drm_driver apple_drm_driver = { DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), +#ifdef CONFIG_DRM_FBDEV_EMULATION .fbdev_probe = apple_drm_fbdev_probe, +#else + DRM_FBDEV_DMA_DRIVER_OPS, +#endif .name = DRIVER_NAME, .desc = DRIVER_DESC, .major = 1, From 3a59b75b7071b7eab5cdd4315a5d41c8e1ead63f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0572/1579] mm: make VM operations aware of per-process mapping sizes Separate target-mm virtual geometry from native physical pages across faults, COW, reclaim, swap, GUP, VMAs and accounting. Supply explicit fragment interfaces and keep native defaults until the optional backend is enabled later in the series. Signed-off-by: Diljit Singh (cherry picked from commit 81fd8e0b21629d0fdcfd36bed3c684cf278b6113) --- arch/x86/kernel/uprobes.c | 6 +- fs/proc/array.c | 3 +- fs/proc/task_mmu.c | 295 ++++- fs/userfaultfd.c | 57 +- include/asm-generic/pgalloc.h | 17 +- include/asm-generic/tlb.h | 43 +- include/linux/binfmts.h | 6 + include/linux/coredump.h | 13 +- include/linux/highmem.h | 15 + include/linux/huge_mm.h | 4 + include/linux/khugepaged.h | 8 + include/linux/mm.h | 104 +- include/linux/mm_granule.h | 245 ++++ include/linux/mm_subpage.h | 277 ++++ include/linux/mm_types.h | 31 +- include/linux/mman.h | 11 + include/linux/mmu_context.h | 5 + include/linux/page_table_check.h | 4 + include/linux/pagemap.h | 5 +- include/linux/pgtable.h | 98 +- include/linux/rmap.h | 106 +- include/linux/sched.h | 5 + include/linux/sched/mm.h | 4 + include/linux/sched/signal.h | 2 +- include/linux/shmem_fs.h | 51 + include/linux/swap.h | 26 +- include/linux/syscalls.h | 5 + include/linux/uprobes.h | 1 + include/trace/events/kmem.h | 2 +- include/trace/events/mmap.h | 2 +- include/trace/events/oom.h | 9 +- include/uapi/linux/prctl.h | 18 + ipc/shm.c | 36 +- kernel/events/core.c | 83 +- kernel/events/internal.h | 37 +- kernel/events/ring_buffer.c | 47 +- kernel/events/uprobes.c | 212 ++- kernel/fork.c | 35 +- kernel/futex/core.c | 57 +- kernel/sched/fair.c | 6 +- kernel/sys.c | 52 +- kernel/trace/trace_events_user.c | 36 +- kernel/tsacct.c | 16 +- lib/vdso/datastore.c | 3 +- mm/Kconfig | 30 +- mm/damon/ops-common.c | 10 +- mm/filemap.c | 73 +- mm/gup.c | 590 ++++++++- mm/huge_memory.c | 11 +- mm/internal.h | 87 +- mm/khugepaged.c | 119 +- mm/ksm.c | 9 +- mm/madvise.c | 156 ++- mm/memory-failure.c | 45 + mm/memory.c | 2071 +++++++++++++++++++++++++----- mm/mempolicy.c | 57 +- mm/migrate.c | 85 +- mm/migrate_device.c | 2 +- mm/mincore.c | 43 +- mm/mlock.c | 42 +- mm/mm_init.c | 3 +- mm/mmap.c | 139 +- mm/mmu_gather.c | 67 +- mm/mprotect.c | 143 ++- mm/mremap.c | 89 +- mm/mseal.c | 4 +- mm/msync.c | 6 +- mm/oom_kill.c | 25 +- mm/page_idle.c | 7 +- mm/page_io.c | 61 +- mm/page_table_check.c | 142 +- mm/page_vma_mapped.c | 61 +- mm/pagewalk.c | 49 +- mm/pgtable-generic.c | 35 +- mm/process_vm_access.c | 96 +- mm/rmap.c | 519 +++++++- mm/shmem.c | 512 +++++++- mm/swap.h | 22 +- mm/swap_state.c | 6 +- mm/swapfile.c | 387 +++++- mm/truncate.c | 2 + mm/user-subpage-internal.h | 69 + mm/userfaultfd.c | 518 +++++++- mm/util.c | 106 +- mm/vma.c | 180 ++- mm/vma.h | 40 +- mm/vma_exec.c | 6 +- mm/vma_init.c | 5 +- mm/vmscan.c | 124 +- 89 files changed, 7676 insertions(+), 1275 deletions(-) create mode 100644 include/linux/mm_granule.h create mode 100644 include/linux/mm_subpage.h create mode 100644 mm/user-subpage-internal.h diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 3af979fb41d38..3155eed81c2ec 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -926,13 +926,15 @@ static int is_call_insn(uprobe_opcode_t *insn) * Verification callback used by int3_update uprobe_write calls to make sure * the underlying instruction is as expected - either int3 or call. */ -static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode, +static int verify_insn(struct page *page, unsigned long vaddr, unsigned int page_offset, + uprobe_opcode_t *new_opcode, int nbytes, void *data) { struct write_opcode_ctx *ctx = data; uprobe_opcode_t old_opcode[5]; - uprobe_copy_from_page(page, ctx->base, (uprobe_opcode_t *) &old_opcode, 5); + uprobe_copy_from_page(page, page_offset - (vaddr - ctx->base), + (uprobe_opcode_t *) &old_opcode, 5); switch (ctx->expect) { case EXPECT_SWBP: diff --git a/fs/proc/array.c b/fs/proc/array.c index 479ea8cb4ef43..5b45506942991 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -610,7 +610,8 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, seq_put_decimal_ull(m, " ", 0); seq_put_decimal_ull(m, " ", start_time); seq_put_decimal_ull(m, " ", vsize); - seq_put_decimal_ull(m, " ", mm ? get_mm_rss(mm) : 0); + seq_put_decimal_ull(m, " ", mm ? + mm_pages_for_reader(mm, current->mm, get_mm_rss(mm)) : 0); seq_put_decimal_ull(m, " ", rsslim); seq_put_decimal_ull(m, " ", mm ? (permitted ? mm->start_code : 1) : 0); seq_put_decimal_ull(m, " ", mm ? (permitted ? mm->end_code : 1) : 0); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 436e1c1d11f41..8938ae6758100 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include #include #include @@ -33,7 +34,7 @@ #define SENTINEL_VMA_GATE -2 #define SEQ_PUT_DEC(str, val) \ - seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8) + seq_put_decimal_ull_width(m, str, mm_pages_to_kb(mm, val), 8) void task_mem(struct seq_file *m, struct mm_struct *mm) { unsigned long text, lib, swap, anon, file, shmem; @@ -58,15 +59,18 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) hiwater_rss = mm->hiwater_rss; /* split executable areas between text and lib */ - text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK); - text = min(text, mm->exec_vm << PAGE_SHIFT); - lib = (mm->exec_vm << PAGE_SHIFT) - text; + text = mm_page_align(mm, mm->end_code) - + (mm->start_code & mm_page_mask(mm)); + text = min(text, mm_pages_to_bytes(mm, mm->exec_vm)); + lib = (mm_pages_to_bytes(mm, mm->exec_vm)) - text; swap = get_mm_counter_sum(mm, MM_SWAPENTS); SEQ_PUT_DEC("VmPeak:\t", hiwater_vm); SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm); - SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm); - SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm)); + seq_put_decimal_ull_width(m, " kB\nVmLck:\t", + mm_locked_vm_bytes(mm) >> 10, 8); + seq_put_decimal_ull_width(m, " kB\nVmPin:\t", + atomic64_read(&mm->pinned_vm) << (PAGE_SHIFT - 10), 8); SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss); SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss); SEQ_PUT_DEC(" kB\nRssAnon:\t", anon); @@ -88,20 +92,25 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) unsigned long task_vsize(struct mm_struct *mm) { - return PAGE_SIZE * mm->total_vm; + return mm_pages_to_bytes(mm, mm->total_vm); } unsigned long task_statm(struct mm_struct *mm, unsigned long *shared, unsigned long *text, unsigned long *data, unsigned long *resident) { - *shared = get_mm_counter_sum(mm, MM_FILEPAGES) + + struct mm_struct *reader = current->mm; + unsigned long file = get_mm_counter_sum(mm, MM_FILEPAGES) + get_mm_counter_sum(mm, MM_SHMEMPAGES); - *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) - >> PAGE_SHIFT; - *data = mm->data_vm + mm->stack_vm; - *resident = *shared + get_mm_counter_sum(mm, MM_ANONPAGES); - return mm->total_vm; + unsigned long code = (mm_page_align(mm, mm->end_code) - + (mm->start_code & mm_page_mask(mm))) >> mm_page_shift(mm); + + *shared = mm_pages_for_reader(mm, reader, file); + *text = mm_pages_for_reader(mm, reader, code); + *data = mm_pages_for_reader(mm, reader, mm->data_vm + mm->stack_vm); + *resident = mm_pages_for_reader(mm, reader, + file + get_mm_counter_sum(mm, MM_ANONPAGES)); + return mm_pages_for_reader(mm, reader, mm->total_vm); } #ifdef CONFIG_NUMA @@ -475,7 +484,8 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma) dev = inode->i_sb->s_dev; ino = inode->i_ino; - pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT; + pgoff = (((loff_t)vma->vm_pgoff) << PAGE_SHIFT) + + vma_subpage_offset(vma); } start = vma->vm_start; @@ -717,7 +727,8 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) if (vma->vm_file) { const struct inode *inode = file_user_inode(vma->vm_file); - karg.vma_offset = ((__u64)vma->vm_pgoff) << PAGE_SHIFT; + karg.vma_offset = (((__u64)vma->vm_pgoff) << PAGE_SHIFT) + + vma_subpage_offset(vma); karg.dev_major = MAJOR(inode->i_sb->s_dev); karg.dev_minor = MINOR(inode->i_sb->s_dev); karg.inode = inode->i_ino; @@ -857,6 +868,50 @@ const struct file_operations proc_pid_maps_operations = { */ #define PSS_SHIFT 12 +#if defined(CONFIG_MM_SUBPAGE) && (defined(CONFIG_PROC_PAGE_MONITOR) || defined(CONFIG_NUMA)) +/* Snapshot only: rmap and quarter counters can change in other address spaces. */ +static void file_quarter_mapcounts(struct page *page, struct mm_subpage_mapcounts *counts) +{ + int maps = folio_precise_page_mapcount(page_folio(page), page); + unsigned int small = mm_subpage_file_mapcounts(page, counts); + unsigned int native = maps > small ? maps - small : 0; + unsigned int i; + + for (i = 0; i < MM_SUBPAGES_PER_PAGE; i++) + counts->slots[i] += native; +} + +static unsigned int proc_leaf_mapcount(struct vm_area_struct *vma, pte_t pte, + struct page *page) +{ + struct folio *folio = page_folio(page); + struct mm_subpage_mapcounts counts; + unsigned int i, count = 0; + + if (!folio_test_anon(folio)) { + unsigned int first = mm_page_size(vma->vm_mm) < PAGE_SIZE ? + offset_in_page(pte_phys_mm(vma->vm_mm, pte)) / SZ_4K : 0; + unsigned int nr = mm_page_size(vma->vm_mm) / SZ_4K; + + file_quarter_mapcounts(page, &counts); + for (i = first; i < first + nr; i++) + count = max(count, counts.slots[i]); + return count; + } + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + + if (slot) { + count = mm_subpage_mapcount(slot); + mm_subpage_put(slot); + } + return count; + } + return folio_precise_page_mapcount(folio, page); +} + +#endif + #ifdef CONFIG_PROC_PAGE_MONITOR struct mem_size_stats { unsigned long resident; @@ -913,6 +968,73 @@ static void smaps_page_accumulate(struct mem_size_stats *mss, } } +#ifdef CONFIG_MM_SUBPAGE +static void smaps_account_file_extent(struct mem_size_stats *mss, struct page *page, + unsigned int offset, unsigned long size, bool dirty, bool locked, bool present) +{ + struct folio *folio = page_folio(page); + struct mm_subpage_mapcounts counts = {}; + unsigned int i; + + if (present) + file_quarter_mapcounts(page, &counts); + for (i = offset / SZ_4K; i < (offset + size) / SZ_4K; i++) { + unsigned long pss = SZ_4K << PSS_SHIFT; + + if (counts.slots[i] > 1) + pss /= counts.slots[i]; + smaps_page_accumulate(mss, folio, SZ_4K, pss, dirty, locked, + present && counts.slots[i] < 2); + } +} + +static void smaps_account_small(struct mem_size_stats *mss, struct vm_area_struct *vma, + struct page *page, pte_t pte, bool young, + bool dirty, bool locked, bool present) +{ + struct folio *folio = page_folio(page); + unsigned long size = mm_page_size(vma->vm_mm), pss = size << PSS_SHIFT; + unsigned int count = present && folio_test_anon(folio) ? + proc_leaf_mapcount(vma, pte, page) : 0; + + mss->resident += size; + if (folio_test_anon(folio)) { + mss->anonymous += size; + if (present && !dirty) { + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + + if (slot) { + if (mm_subpage_is_lazyfree(slot)) + mss->lazyfree += size; + mm_subpage_put(slot); + } + } + } + if (folio_test_ksm(folio)) + mss->ksm += size; + if (young || folio_test_young(folio) || folio_test_referenced(folio)) + mss->referenced += size; + if (present && !folio_test_anon(folio)) { + smaps_account_file_extent(mss, page, offset_in_page(pte_phys_mm(vma->vm_mm, pte)), + size, dirty, locked, present); + return; + } + if (count > 1) + pss /= count; + smaps_page_accumulate(mss, folio, size, pss, dirty, locked, present && count < 2); +} + +static bool smaps_account_file_quarters(struct mem_size_stats *mss, struct page *page, + bool dirty, bool locked) +{ + if (folio_test_anon(page_folio(page))) + return false; + smaps_account_file_extent(mss, page, 0, PAGE_SIZE, dirty, locked, true); + return true; +} + +#endif + static void smaps_account(struct mem_size_stats *mss, struct page *page, bool compound, bool young, bool dirty, bool locked, bool present) @@ -976,6 +1098,11 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page, for (i = 0; i < nr; i++, page++) { unsigned long pss = PAGE_SIZE << PSS_SHIFT; +#ifdef CONFIG_MM_SUBPAGE + if (smaps_account_file_quarters(mss, page, dirty, locked)) + continue; +#endif + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) { mapcount = folio_precise_page_mapcount(folio, page); exclusive = mapcount < 2; @@ -995,6 +1122,14 @@ static int smaps_pte_hole(unsigned long addr, unsigned long end, struct mem_size_stats *mss = walk->private; struct vm_area_struct *vma = walk->vma; + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + u64 start = ((u64)pos.index << PAGE_SHIFT) + pos.offset; + + mss->swap += shmem_partial_swap_usage_bytes(vma->vm_file->f_mapping, + start, start + end - addr); + return 0; + } mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping, linear_page_index(vma, addr), linear_page_index(vma, end)); @@ -1010,7 +1145,7 @@ static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk) #ifdef CONFIG_SHMEM if (walk->ops->pte_hole) { /* depth is not used */ - smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk); + smaps_pte_hole(addr, addr + mm_page_size(walk->mm), 0, walk); } #endif } @@ -1038,15 +1173,20 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr, if (softleaf_is_swap(entry)) { int mapcount; - mss->swap += PAGE_SIZE; - mapcount = swp_swapcount(entry); + mss->swap += mm_page_size(vma->vm_mm); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + mapcount = swap_subpage_count(entry, pte_swp_subpage_offset(ptent)); + else +#endif + mapcount = swp_swapcount(entry); if (mapcount >= 2) { - u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT; + u64 pss_delta = (u64)mm_page_size(vma->vm_mm) << PSS_SHIFT; do_div(pss_delta, mapcount); mss->swap_pss += pss_delta; } else { - mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT; + mss->swap_pss += (u64)mm_page_size(vma->vm_mm) << PSS_SHIFT; } } else if (softleaf_has_pfn(entry)) { if (softleaf_is_device_private(entry)) @@ -1058,6 +1198,12 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr, if (!page) return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + smaps_account_small(mss, vma, page, ptent, young, dirty, locked, present); + return; + } +#endif smaps_account(mss, page, false, young, dirty, locked, present); } @@ -1124,7 +1270,7 @@ static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte++, addr += PAGE_SIZE) + for (; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) smaps_pte_entry(pte, addr, walk); pte_unmap_unlock(pte - 1, ptl); out: @@ -1712,7 +1858,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte++, addr += PAGE_SIZE) { + for (; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) { ptent = ptep_get(pte); if (cp->type == CLEAR_REFS_SOFT_DIRTY) { @@ -1855,9 +2001,6 @@ struct pagemapread { bool show_pfn; }; -#define PAGEMAP_WALK_SIZE (PMD_SIZE) -#define PAGEMAP_WALK_MASK (PMD_MASK) - #define PM_ENTRY_BYTES sizeof(pagemap_entry_t) #define PM_PFRAME_BITS 55 #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0) @@ -1884,7 +2027,7 @@ static int add_to_pagemap(pagemap_entry_t *pme, struct pagemapread *pm) return 0; } -static bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page) +static inline bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page) { if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) return folio_precise_page_mapcount(folio, page) == 1; @@ -1909,7 +2052,7 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, else hole_end = end; - for (; addr < hole_end; addr += PAGE_SIZE) { + for (; addr < hole_end; addr += mm_page_size(walk->mm)) { err = add_to_pagemap(&pme, pm); if (err) goto out; @@ -1921,7 +2064,7 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, /* Addresses in the VMA. */ if (vma->vm_flags & VM_SOFTDIRTY) pme = make_pme(0, PM_SOFT_DIRTY); - for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) { + for (; addr < min(end, vma->vm_end); addr += mm_page_size(walk->mm)) { err = add_to_pagemap(&pme, pm); if (err) goto out; @@ -1931,6 +2074,16 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, return err; } +static bool pagemap_page_exclusive(struct vm_area_struct *vma, pte_t pte, + struct folio *folio, struct page *page) +{ +#ifdef CONFIG_MM_SUBPAGE + return proc_leaf_mapcount(vma, pte, page) == 1; +#else + return __folio_page_mapped_exclusively(folio, page); +#endif +} + static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, struct vm_area_struct *vma, unsigned long addr, pte_t pte) { @@ -1943,7 +2096,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, if (pte_present(pte)) { if (pm->show_pfn) - frame = pte_pfn(pte); + frame = pte_phys_mm(vma->vm_mm, pte) >> mm_page_shift(vma->vm_mm); flags |= PM_PRESENT; page = vm_normal_page(vma, addr, pte); if (pte_soft_dirty(pte)) @@ -1969,6 +2122,12 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, offset = softleaf_to_pfn(entry); else offset = swp_offset(entry); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE && + (softleaf_is_swap(entry) || softleaf_has_pfn(entry))) + offset = (offset << (PAGE_SHIFT - mm_page_shift(vma->vm_mm))) + + (pte_swp_subpage_offset(pte) >> mm_page_shift(vma->vm_mm)); +#endif frame = swp_type(entry) | (offset << MAX_SWAPFILES_SHIFT); } @@ -1985,8 +2144,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, folio = page_folio(page); if (!folio_test_anon(folio)) flags |= PM_FILE; - if ((flags & PM_PRESENT) && - __folio_page_mapped_exclusively(folio, page)) + if ((flags & PM_PRESENT) && pagemap_page_exclusive(vma, pte, folio, page)) flags |= PM_MMAP_EXCLUSIVE; } @@ -2102,7 +2260,7 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return err; } - for (; addr < end; pte++, addr += PAGE_SIZE) { + for (; addr < end; pte++, addr += mm_page_size(walk->mm)) { pagemap_entry_t pme; pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte)); @@ -2236,7 +2394,7 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, /* do not disclose physical addresses: attack vector */ pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN); - pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT); + pm.len = (pmd_size_mm(mm) >> mm_page_shift(mm)); pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL); ret = -ENOMEM; if (!pm.buffer) @@ -2248,16 +2406,16 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, /* watch out for wraparound */ start_vaddr = end_vaddr; - if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) { + if (svpfn <= (ULONG_MAX >> mm_page_shift(mm))) { unsigned long end; ret = mmap_read_lock_killable(mm); if (ret) goto out_free; - start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT); + start_vaddr = untagged_addr_remote(mm, svpfn << mm_page_shift(mm)); mmap_read_unlock(mm); - end = start_vaddr + ((count / PM_ENTRY_BYTES) << PAGE_SHIFT); + end = start_vaddr + ((count / PM_ENTRY_BYTES) << mm_page_shift(mm)); if (end >= start_vaddr && end < mm->task_size) end_vaddr = end; } @@ -2272,7 +2430,7 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, unsigned long end; pm.pos = 0; - end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK; + end = (start_vaddr + pmd_size_mm(mm)) & pmd_mask_mm(mm); /* overflow ? */ if (end < start_vaddr || end > end_vaddr) end = end_vaddr; @@ -2333,6 +2491,7 @@ static int pagemap_release(struct inode *inode, struct file *file) struct pagemap_scan_private { struct pm_scan_arg arg; + unsigned long page_size; unsigned long masks_of_interest, cur_vma_category; struct page_region *vec_buf; unsigned long vec_buf_len, vec_buf_index, found_pages; @@ -2559,7 +2718,7 @@ static void pagemap_scan_backout_range(struct pagemap_scan_private *p, else cur_buf->start = cur_buf->end = 0; - p->found_pages -= (end - addr) / PAGE_SIZE; + p->found_pages -= (end - addr) / p->page_size; } #endif @@ -2671,11 +2830,11 @@ static int pagemap_scan_output(unsigned long categories, categories &= p->arg.return_mask; - n_pages = (*end - addr) / PAGE_SIZE; + n_pages = (*end - addr) / p->page_size; if (check_add_overflow(p->found_pages, n_pages, &total_pages) || total_pages > p->arg.max_pages) { size_t n_too_much = total_pages - p->arg.max_pages; - *end -= n_too_much * PAGE_SIZE; + *end -= n_too_much * p->page_size; n_pages -= n_too_much; ret = -ENOSPC; } @@ -2769,7 +2928,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) { /* Fast path for performing exclusive WP */ - for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { + for (addr = start; addr != end; pte++, addr += p->page_size) { pte_t ptent = ptep_get(pte); if ((pte_present(ptent) && pte_uffd_wp(ptent)) || @@ -2778,7 +2937,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, make_uffd_wp_pte(vma, addr, pte, ptent); if (!flush_end) start = addr; - flush_end = addr + PAGE_SIZE; + flush_end = addr + p->page_size; } goto flush_and_return; } @@ -2786,8 +2945,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, if (!p->arg.category_anyof_mask && !p->arg.category_inverted && p->arg.category_mask == PAGE_IS_WRITTEN && p->arg.return_mask == PAGE_IS_WRITTEN) { - for (addr = start; addr < end; pte++, addr += PAGE_SIZE) { - unsigned long next = addr + PAGE_SIZE; + for (addr = start; addr < end; pte++, addr += p->page_size) { + unsigned long next = addr + p->page_size; pte_t ptent = ptep_get(pte); if ((pte_present(ptent) && pte_uffd_wp(ptent)) || @@ -2807,11 +2966,11 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, goto flush_and_return; } - for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { + for (addr = start; addr != end; pte++, addr += p->page_size) { pte_t ptent = ptep_get(pte); unsigned long categories = p->cur_vma_category | pagemap_page_category(p, vma, addr, ptent); - unsigned long next = addr + PAGE_SIZE; + unsigned long next = addr + p->page_size; if (!pagemap_scan_is_interesting_page(categories, p)) continue; @@ -3007,7 +3166,7 @@ static const struct mm_walk_ops pagemap_scan_ops = { .hugetlb_entry = pagemap_scan_hugetlb_entry, }; -static int pagemap_scan_get_args(struct pm_scan_arg *arg, +static int pagemap_scan_get_args(struct mm_struct *mm, struct pm_scan_arg *arg, unsigned long uarg) { if (copy_from_user(arg, (void __user *)uarg, sizeof(*arg))) @@ -3028,7 +3187,7 @@ static int pagemap_scan_get_args(struct pm_scan_arg *arg, arg->vec = untagged_addr((unsigned long)arg->vec); /* Validate memory pointers */ - if (!IS_ALIGNED(arg->start, PAGE_SIZE)) + if (!IS_ALIGNED(arg->start, mm_page_size(mm))) return -EINVAL; if (!access_ok((void __user *)(long)arg->start, arg->end - arg->start)) return -EFAULT; @@ -3041,7 +3200,7 @@ static int pagemap_scan_get_args(struct pm_scan_arg *arg, return -EFAULT; /* Fixup default values */ - arg->end = ALIGN(arg->end, PAGE_SIZE); + arg->end = ALIGN(arg->end, mm_page_size(mm)); arg->walk_end = 0; if (!arg->max_pages) arg->max_pages = ULONG_MAX; @@ -3060,12 +3219,13 @@ static int pagemap_scan_writeback_args(struct pm_scan_arg *arg, return 0; } -static int pagemap_scan_init_bounce_buffer(struct pagemap_scan_private *p) +static int pagemap_scan_init_bounce_buffer(struct mm_struct *mm, + struct pagemap_scan_private *p) { if (!p->arg.vec_len) return 0; - p->vec_buf_len = min_t(size_t, PAGEMAP_WALK_SIZE >> PAGE_SHIFT, + p->vec_buf_len = min_t(size_t, pmd_size_mm(mm) >> mm_page_shift(mm), p->arg.vec_len); p->vec_buf = kmalloc_objs(*p->vec_buf, p->vec_buf_len); if (!p->vec_buf) @@ -3106,18 +3266,18 @@ static long pagemap_scan_flush_buffer(struct pagemap_scan_private *p) static long do_pagemap_scan(struct mm_struct *mm, unsigned long uarg) { - struct pagemap_scan_private p = {0}; + struct pagemap_scan_private p = { .page_size = mm_page_size(mm) }; unsigned long walk_start; size_t n_ranges_out = 0; int ret; - ret = pagemap_scan_get_args(&p.arg, uarg); + ret = pagemap_scan_get_args(mm, &p.arg, uarg); if (ret) return ret; p.masks_of_interest = p.arg.category_mask | p.arg.category_anyof_mask | p.arg.return_mask; - ret = pagemap_scan_init_bounce_buffer(&p); + ret = pagemap_scan_init_bounce_buffer(mm, &p); if (ret) return ret; @@ -3219,16 +3379,17 @@ struct numa_maps_private { struct numa_maps md; }; +static int numa_page_mapcount(struct page *page) +{ + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + return folio_precise_page_mapcount(page_folio(page), page); + return folio_average_page_mapcount(page_folio(page)); +} + static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty, - unsigned long nr_pages) + unsigned long nr_pages, int count) { struct folio *folio = page_folio(page); - int count; - - if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) - count = folio_precise_page_mapcount(folio, page); - else - count = folio_average_page_mapcount(folio); md->pages += nr_pages; if (pte_dirty || folio_test_dirty(folio)) @@ -3318,7 +3479,7 @@ static int gather_pte_stats(pmd_t *pmd, unsigned long addr, page = can_gather_numa_stats_pmd(*pmd, vma, addr); if (page) gather_stats(page, md, pmd_dirty(*pmd), - HPAGE_PMD_SIZE/PAGE_SIZE); + HPAGE_PMD_SIZE/PAGE_SIZE, numa_page_mapcount(page)); spin_unlock(ptl); return 0; } @@ -3333,9 +3494,13 @@ static int gather_pte_stats(pmd_t *pmd, unsigned long addr, struct page *page = can_gather_numa_stats(ptent, vma, addr); if (!page) continue; - gather_stats(page, md, pte_dirty(ptent), 1); +#ifdef CONFIG_MM_SUBPAGE + gather_stats(page, md, pte_dirty(ptent), 1, proc_leaf_mapcount(vma, ptent, page)); +#else + gather_stats(page, md, pte_dirty(ptent), 1, numa_page_mapcount(page)); +#endif - } while (pte++, addr += PAGE_SIZE, addr != end); + } while (pte++, addr += mm_page_size(vma->vm_mm), addr != end); pte_unmap_unlock(orig_pte, ptl); cond_resched(); return 0; @@ -3357,7 +3522,7 @@ static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, page = pte_page(huge_pte); md = walk->private; - gather_stats(page, md, pte_dirty(huge_pte), 1); + gather_stats(page, md, pte_dirty(huge_pte), 1, numa_page_mapcount(page)); out: spin_unlock(ptl); return 0; diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index dba1172436b74..3174e3b6edb41 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -301,6 +301,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, pmd_t *pmd, _pmd; pte_t *pte; pte_t ptent; + spinlock_t *ptl; bool ret; assert_fault_locked(vmf); @@ -308,13 +309,13 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, pgd = pgd_offset(mm, address); if (!pgd_present(*pgd)) return true; - p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + p4d = p4d_offset_mm(mm, pgd, address); + if (!p4d_present_mm(mm, *p4d)) return true; - pud = pud_offset(p4d, address); + pud = pud_offset_mm(mm, p4d, address); if (!pud_present(*pud)) return true; - pmd = pmd_offset(pud, address); + pmd = pmd_offset_mm(mm, pud, address); again: _pmd = pmdp_get_lockless(pmd); if (pmd_none(_pmd)) @@ -331,7 +332,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, if (pmd_trans_huge(_pmd)) return !pmd_write(_pmd) && (reason & VM_UFFD_WP); - pte = pte_offset_map(pmd, address); + pte = pte_offset_map_ro_nolock(mm, pmd, address, &ptl); if (!pte) goto again; @@ -1251,13 +1252,11 @@ static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx, __wake_userfault(ctx, range); } -static __always_inline int validate_unaligned_range( +static __always_inline int validate_byte_range( struct mm_struct *mm, __u64 start, __u64 len) { __u64 task_size = mm->task_size; - if (len & ~PAGE_MASK) - return -EINVAL; if (!len) return -EINVAL; if (start >= task_size) @@ -1269,10 +1268,19 @@ static __always_inline int validate_unaligned_range( return 0; } +static __always_inline int validate_unaligned_range( + struct mm_struct *mm, __u64 start, __u64 len) +{ + if (len & ~mm_page_mask(mm)) + return -EINVAL; + + return validate_byte_range(mm, start, len); +} + static __always_inline int validate_range(struct mm_struct *mm, __u64 start, __u64 len) { - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; return validate_unaligned_range(mm, start, len); @@ -1428,6 +1436,13 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC : UFFD_API_RANGE_IOCTLS; + if (mm_page_size(mm) != PAGE_SIZE) { + ioctls_out &= (1ULL << _UFFDIO_WAKE) | (1ULL << _UFFDIO_WRITEPROTECT) | + (1ULL << _UFFDIO_CONTINUE) | (1ULL << _UFFDIO_COPY) | + (1ULL << _UFFDIO_ZEROPAGE) | (1ULL << _UFFDIO_POISON); + } + if (mm_page_size(mm) < PAGE_SIZE) + ioctls_out |= 1ULL << _UFFDIO_MOVE; /* * Declare the WP ioctl only if the WP mode is @@ -1645,8 +1660,12 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, sizeof(uffdio_copy)-sizeof(__s64))) goto out; - ret = validate_unaligned_range(ctx->mm, uffdio_copy.src, - uffdio_copy.len); + /* + * COPY reads the pager's address space, which can have a different + * task size and granule from the registered target. Only the target + * range below requires a page-aligned length. + */ + ret = validate_byte_range(current->mm, uffdio_copy.src, uffdio_copy.len); if (ret) goto out; ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len); @@ -2015,6 +2034,18 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, /* report all available features and ioctls to userland */ uffdio_api.features = UFFD_API_FEATURES; + /* Base-page anonymous and shared shmem operations support alternative leaves. */ + if (mm_page_size(ctx->mm) != PAGE_SIZE) + uffdio_api.features &= UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_MISSING_SHMEM | UFFD_FEATURE_MINOR_SHMEM | + UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_EVENT_REMAP | + UFFD_FEATURE_EVENT_REMOVE | UFFD_FEATURE_EVENT_UNMAP | + UFFD_FEATURE_SIGBUS | UFFD_FEATURE_THREAD_ID | UFFD_FEATURE_EXACT_ADDRESS | + UFFD_FEATURE_WP_UNPOPULATED | UFFD_FEATURE_WP_ASYNC | UFFD_FEATURE_POISON; + + if (mm_page_size(ctx->mm) < PAGE_SIZE) + uffdio_api.features |= UFFD_FEATURE_MOVE; + #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR uffdio_api.features &= ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); @@ -2062,6 +2093,10 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd, if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx)) return -EINVAL; + /* Larger-than-native MOVE still requires contiguous-owner support. */ + if (cmd == UFFDIO_MOVE && mm_page_size(ctx->mm) > PAGE_SIZE) + return -EOPNOTSUPP; + switch(cmd) { case UFFDIO_API: ret = userfaultfd_api(ctx, arg); diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h index 57137d3ac1592..619fd3ed40634 100644 --- a/include/asm-generic/pgalloc.h +++ b/include/asm-generic/pgalloc.h @@ -81,7 +81,7 @@ static inline pgtable_t __pte_alloc_one_noprof(struct mm_struct *mm, gfp_t gfp) return NULL; } - return ptdesc_page(ptdesc); + return pgtable_from_page(ptdesc_page(ptdesc)); } #define __pte_alloc_one(...) alloc_hooks(__pte_alloc_one_noprof(__VA_ARGS__)) @@ -111,17 +111,18 @@ static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm) * @mm: the mm_struct of the current context * @pte_page: the `struct page` referencing the ptdesc */ -static inline void pte_free(struct mm_struct *mm, struct page *pte_page) +#ifndef __HAVE_ARCH_PTE_FREE +static inline void pte_free(struct mm_struct *mm, pgtable_t pte_page) { - struct ptdesc *ptdesc = page_ptdesc(pte_page); + struct ptdesc *ptdesc = page_ptdesc(pgtable_to_page(pte_page)); pagetable_dtor_free(ptdesc); } +#endif #if CONFIG_PGTABLE_LEVELS > 2 -#ifndef __HAVE_ARCH_PMD_ALLOC_ONE /** * pmd_alloc_one - allocate memory for a PMD-level page table * @mm: the mm_struct of the current context @@ -133,7 +134,7 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page) * * Return: pointer to the allocated memory or %NULL on error */ -static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +static inline pmd_t *__pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) { struct ptdesc *ptdesc; gfp_t gfp = GFP_PGTABLE_USER; @@ -153,6 +154,12 @@ static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long ad return ptdesc_address(ptdesc); } + +#ifndef __HAVE_ARCH_PMD_ALLOC_ONE +static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + return __pmd_alloc_one_noprof(mm, addr); +} #define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__)) #endif diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index bdcc2778ac64f..900bee13b40cb 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -222,6 +222,15 @@ static inline void __tlb_remove_table(void *table) } #endif +/* Architectures may pass opaque table tokens instead of descriptors. */ +#ifndef tlb_table_is_ptdesc +static inline bool tlb_table_is_ptdesc(void *table) +{ + (void)table; + return true; +} +#endif + extern void tlb_remove_table(struct mmu_gather *tlb, void *table); #else /* !CONFIG_MMU_GATHER_TABLE_FREE */ @@ -295,6 +304,12 @@ extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, in bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page, unsigned int nr_pages, bool delay_rmap); +#ifdef CONFIG_MM_SUBPAGE +struct mm_subpage; +/* Consumes one slot reference; true requires flushing the full batch. */ +bool __tlb_remove_subpage(struct mmu_gather *tlb, struct mm_subpage *subpage); +#endif + #ifdef CONFIG_SMP /* * This both sets 'delayed_rmap', and returns true. It would be an inline @@ -543,15 +558,15 @@ static inline void tlb_change_page_size(struct mmu_gather *tlb, static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb) { if (tlb->cleared_ptes) - return PAGE_SHIFT; + return mm_page_shift(tlb->mm); if (tlb->cleared_pmds) - return PMD_SHIFT; + return pmd_shift_mm(tlb->mm); if (tlb->cleared_puds) - return PUD_SHIFT; + return pud_shift_mm(tlb->mm); if (tlb->cleared_p4ds) - return P4D_SHIFT; + return p4d_shift_mm(tlb->mm); - return PAGE_SHIFT; + return mm_page_shift(tlb->mm); } static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) @@ -658,7 +673,7 @@ static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, u */ #define tlb_remove_tlb_entry(tlb, ptep, address) \ do { \ - tlb_flush_pte_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pte_range(tlb, address, mm_page_size((tlb)->mm)); \ __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) @@ -672,13 +687,15 @@ static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, u static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, pte_t *ptep, unsigned int nr, unsigned long address) { - tlb_flush_pte_range(tlb, address, PAGE_SIZE * nr); + unsigned long page_size = mm_page_size(tlb->mm); + + tlb_flush_pte_range(tlb, address, page_size * nr); for (;;) { __tlb_remove_tlb_entry(tlb, ptep, address); if (--nr == 0) break; ptep++; - address += PAGE_SIZE; + address += page_size; } } @@ -739,13 +756,13 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, * architecture to do its own odd thing, not cause pain for others * http://lkml.kernel.org/r/CA+55aFzBggoXtNXQeng5d_mRoDnaMBE5Y+URs+PHR67nUpMtaw@mail.gmail.com * - * For now w.r.t page table cache, mark the range_size as PAGE_SIZE + * For now w.r.t page table cache, mark the range_size as one mm granule. */ #ifndef pte_free_tlb #define pte_free_tlb(tlb, ptep, address) \ do { \ - tlb_flush_pmd_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pmd_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pte_free_tlb(tlb, ptep, address); \ } while (0) @@ -754,7 +771,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef pmd_free_tlb #define pmd_free_tlb(tlb, pmdp, address) \ do { \ - tlb_flush_pud_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pud_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pmd_free_tlb(tlb, pmdp, address); \ } while (0) @@ -763,7 +780,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef pud_free_tlb #define pud_free_tlb(tlb, pudp, address) \ do { \ - tlb_flush_p4d_range(tlb, address, PAGE_SIZE); \ + tlb_flush_p4d_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pud_free_tlb(tlb, pudp, address); \ } while (0) @@ -772,7 +789,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef p4d_free_tlb #define p4d_free_tlb(tlb, pudp, address) \ do { \ - __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + __tlb_adjust_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __p4d_free_tlb(tlb, pudp, address); \ } while (0) diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h index 65abd5ab8836c..c242a5e9cc758 100644 --- a/include/linux/binfmts.h +++ b/include/linux/binfmts.h @@ -93,6 +93,8 @@ struct linux_binfmt { #ifdef CONFIG_COREDUMP int (*core_dump)(struct coredump_params *cprm); unsigned long min_coredump; /* minimal dump size */ + /* Optional per-mm replacement for formats with a variable page ABI. */ + unsigned long (*min_coredump_size)(void); #endif } __randomize_layout; @@ -121,6 +123,10 @@ static inline void insert_binfmt(struct linux_binfmt *fmt) extern void unregister_binfmt(struct linux_binfmt *); +#ifdef CONFIG_MM_SUBPAGE +int bprm_set_page_shift(struct linux_binprm *bprm, unsigned int page_shift); +#endif + extern int __must_check remove_arg_zero(struct linux_binprm *); extern int begin_new_exec(struct linux_binprm * bprm); extern void setup_new_exec(struct linux_binprm * bprm); diff --git a/include/linux/coredump.h b/include/linux/coredump.h index 68861da4cf7c2..5f85a12c7f998 100644 --- a/include/linux/coredump.h +++ b/include/linux/coredump.h @@ -12,10 +12,21 @@ struct core_vma_metadata { unsigned long start, end; vm_flags_t flags; unsigned long dump_size; - unsigned long pgoff; + unsigned long pgoff; /* File offset in the dumped mm's page units. */ struct file *file; }; +/* NT_FILE offsets are expressed in the dumped process's page units. */ +static inline unsigned long core_vma_pgoff(const struct vm_area_struct *vma) +{ + unsigned int shift = mm_page_shift(vma->vm_mm); + + if (shift > PAGE_SHIFT) + return vma->vm_pgoff >> (shift - PAGE_SHIFT); + return (vma->vm_pgoff << (PAGE_SHIFT - shift)) + + (vma_subpage_offset(vma) >> shift); +} + struct coredump_params { const kernel_siginfo_t *siginfo; struct file *file; diff --git a/include/linux/highmem.h b/include/linux/highmem.h index d7aac9de1c8a4..37812575512be 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -330,6 +330,21 @@ struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, } #endif +#ifndef vma_alloc_zeroed_movable_folio_order +static inline struct folio *vma_alloc_zeroed_movable_folio_order( + struct vm_area_struct *vma, unsigned long addr, unsigned int order) +{ + struct folio *folio; + + if (!order) + return vma_alloc_zeroed_movable_folio(vma, addr); + folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, order, vma, addr); + if (folio && user_alloc_needs_zeroing()) + clear_user_highpages(&folio->page, addr, folio_nr_pages(folio)); + return folio; +} +#endif + static inline void clear_highpage(struct page *page) { void *kaddr = kmap_local_page(page); diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 2949e5acff351..1f7f729b87868 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -3,6 +3,7 @@ #define _LINUX_HUGE_MM_H #include +#include #include /* only for vma_is_dax() */ #include @@ -331,6 +332,9 @@ struct thpsize { static inline bool vma_thp_disabled(struct vm_area_struct *vma, vm_flags_t vm_flags, bool forced_collapse) { + /* Native THP geometry and rmap cannot describe alternative user leaves. */ + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return true; /* Are THPs disabled for this VMA? */ if (vm_flags & VM_NOHUGEPAGE) return true; diff --git a/include/linux/khugepaged.h b/include/linux/khugepaged.h index d7a9053ff4fed..1510fdb048010 100644 --- a/include/linux/khugepaged.h +++ b/include/linux/khugepaged.h @@ -8,6 +8,8 @@ extern unsigned int khugepaged_max_ptes_none __read_mostly; #ifdef CONFIG_TRANSPARENT_HUGEPAGE extern struct attribute_group khugepaged_attr_group; +int collapse_file_user_page(struct mm_struct *mm, struct file *file, pgoff_t start); + extern int khugepaged_init(void); extern void khugepaged_destroy(void); extern int start_stop_khugepaged(void); @@ -32,6 +34,12 @@ static inline void khugepaged_exit(struct mm_struct *mm) __khugepaged_exit(mm); } #else /* CONFIG_TRANSPARENT_HUGEPAGE */ +static inline int collapse_file_user_page(struct mm_struct *mm, struct file *file, + pgoff_t start) +{ + return -EOPNOTSUPP; +} + static inline void khugepaged_fork(struct mm_struct *mm, struct mm_struct *oldmm) { } diff --git a/include/linux/mm.h b/include/linux/mm.h index f4481c338b36c..6c01377914fe8 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -750,6 +751,9 @@ struct vm_fault { }; struct page *cow_page; /* Page handler may use for COW fault */ +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *cow_subpage; /* Owns cow_page's selected fragment */ +#endif struct page *page; /* ->fault handlers should return a * page here, unless VM_FAULT_NOPAGE * is set (which is also implied by @@ -1615,7 +1619,7 @@ static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) { if (unlikely(vma->vm_ops && vma->vm_ops->pagesize)) return vma->vm_ops->pagesize(vma); - return PAGE_SIZE; + return mm_page_size(vma->vm_mm); } unsigned long vma_mmu_pagesize(struct vm_area_struct *vma); @@ -2010,6 +2014,10 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa void set_pte_range(struct vm_fault *vmf, struct folio *folio, struct page *page, unsigned int nr, unsigned long addr); +#ifdef CONFIG_MM_SUBPAGE +vm_fault_t map_file_user_pages(struct vm_fault *vmf, struct folio *folio, + pgoff_t first, pgoff_t last, bool fault); +#endif vm_fault_t finish_fault(struct vm_fault *vmf); #endif @@ -3143,10 +3151,13 @@ struct follow_pfnmap_args { * @address: the virtual address to walk * @write: if true, fail with -EFAULT unless the mapping is * writable + * @allow_subpage: caller handles exact byte offsets and user-leaf bounds; + * otherwise a non-native user granule is rejected */ struct vm_area_struct *vma; unsigned long address; bool write; + bool allow_subpage; /** * Internals: * @@ -3157,13 +3168,15 @@ struct follow_pfnmap_args { /** * Outputs: * - * @pfn: the PFN of the address - * @addr_mask: address mask covering pfn + * @pfn: native PFN containing the requested physical byte + * @offset: byte offset of that byte within the native physical page + * @addr_mask: virtual address mask of the leaf mapping * @pgprot: the pgprot_t of the mapping * @writable: whether the mapping is writable * @special: whether the mapping is a special mapping (real PFN maps) */ unsigned long pfn; + unsigned int offset; unsigned long addr_mask; pgprot_t pgprot; bool writable; @@ -3215,6 +3228,13 @@ static inline void unmap_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen, int even_cows) { } #endif +#ifdef CONFIG_MM_SUBPAGE +void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize); +void unmap_mapping_subpage_hole(struct address_space *mapping, loff_t start, u64 end); +#else +static inline void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize) { } +#endif + static inline void unmap_shared_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen) { @@ -3443,7 +3463,8 @@ static inline void reset_mm_hiwater_rss(struct mm_struct *mm) static inline void setmax_mm_hiwater_rss(unsigned long *maxrss, struct mm_struct *mm) { - unsigned long hiwater_rss = get_mm_hiwater_rss(mm); + /* signal->maxrss/cmaxrss survive exec and child reaping: store kB. */ + unsigned long hiwater_rss = mm_pages_to_kb(mm, get_mm_hiwater_rss(mm)); if (*maxrss < hiwater_rss) *maxrss = hiwater_rss; @@ -3514,14 +3535,14 @@ static inline void mm_inc_nr_puds(struct mm_struct *mm) { if (mm_pud_folded(mm)) return; - atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); + atomic_long_add(pud_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_puds(struct mm_struct *mm) { if (mm_pud_folded(mm)) return; - atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); + atomic_long_sub(pud_table_bytes_mm(mm), &mm->pgtables_bytes); } #endif @@ -3542,14 +3563,14 @@ static inline void mm_inc_nr_pmds(struct mm_struct *mm) { if (mm_pmd_folded(mm)) return; - atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); + atomic_long_add(pmd_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_pmds(struct mm_struct *mm) { if (mm_pmd_folded(mm)) return; - atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); + atomic_long_sub(pmd_table_bytes_mm(mm), &mm->pgtables_bytes); } #endif @@ -3566,12 +3587,12 @@ static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) static inline void mm_inc_nr_ptes(struct mm_struct *mm) { - atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); + atomic_long_add(pte_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_ptes(struct mm_struct *mm) { - atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); + atomic_long_sub(pte_table_bytes_mm(mm), &mm->pgtables_bytes); } #else @@ -3594,32 +3615,40 @@ static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) { return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ? - NULL : p4d_offset(pgd, address); + NULL : p4d_offset_mm(mm, pgd, address); } static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) { - return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? - NULL : pud_offset(p4d, address); + return (unlikely(p4d_none_mm(mm, *p4d)) && __pud_alloc(mm, p4d, address)) ? + NULL : pud_offset_mm(mm, p4d, address); } static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) { return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? - NULL: pmd_offset(pud, address); + NULL : pmd_offset_mm(mm, pud, address); } #endif /* CONFIG_MMU */ enum pt_flags { PT_kernel = PG_referenced, PT_reserved = PG_reserved, + PT_fragmented = PG_private, + PT_frag_defer = PG_active, /* High bits are used for zone/node/section */ }; static inline struct ptdesc *virt_to_ptdesc(const void *x) { - return page_ptdesc(virt_to_page(x)); + struct page *page = virt_to_page(x); + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* One hardware table may span multiple native allocator pages. */ + page = compound_head(page); +#endif + return page_ptdesc(page); } /** @@ -3836,12 +3865,25 @@ static inline bool pagetable_pte_ctor(struct mm_struct *mm, } pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, pmd_t *pmdvalp); +#else +#define __pte_offset_map_mm(mm, pmd, addr, pmdvalp) \ + __pte_offset_map(pmd, addr, pmdvalp) +#endif static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr) { return __pte_offset_map(pmd, addr, NULL); } +static inline pte_t *pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr) +{ + return __pte_offset_map_mm(mm, pmd, addr, NULL); +} + pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp); @@ -3859,7 +3901,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, #define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd)) #define pte_alloc_map(mm, pmd, address) \ - (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address)) + (pte_alloc(mm, pmd) ? NULL : pte_offset_map_mm(mm, pmd, address)) #define pte_alloc_map_lock(mm, pmd, address, ptlp) \ (pte_alloc(mm, pmd) ? \ @@ -3874,7 +3916,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, static inline struct page *pmd_pgtable_page(pmd_t *pmd) { unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); - return virt_to_page((void *)((unsigned long) pmd & mask)); + return ptdesc_page(virt_to_ptdesc((void *)((unsigned long)pmd & mask))); } static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd) @@ -4160,6 +4202,12 @@ extern unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf); +#ifdef CONFIG_MM_SUBPAGE +unsigned long do_mmap_offset(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, struct vm_page_offset pos, unsigned long *populate, + struct list_head *uf); +#endif extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf, bool unlock); @@ -4288,7 +4336,11 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) static inline unsigned long vma_last_pgoff(struct vm_area_struct *vma) { +#ifdef CONFIG_MM_SUBPAGE + return vma_page_offset_at(vma, vma->vm_end - 1).index; +#else return vma->vm_pgoff + vma_pages(vma) - 1; +#endif } static inline unsigned long vma_desc_size(const struct vm_area_desc *desc) @@ -4541,6 +4593,9 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t pgprot); int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); +int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long nr_pages, + unsigned long offset, unsigned long length); int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num); int map_kernel_pages_prepare(struct vm_area_desc *desc); @@ -4552,6 +4607,9 @@ int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, unsigned long num); vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, bool write); +vm_fault_t vmf_insert_pfn_prot_mkwrite_offset(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, unsigned int offset, + pgprot_t pgprot, bool mkwrite); vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot, bool mkwrite); vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, @@ -4648,6 +4706,18 @@ static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma, return VM_FAULT_NOPAGE; } +/* Insert one user granule from an explicit offset within a native page. */ +#ifdef CONFIG_MM_SUBPAGE +vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, unsigned long addr, + struct page *page, unsigned int offset); +#else +static inline vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, + unsigned long addr, struct page *page, unsigned int offset) +{ + return offset ? VM_FAULT_SIGBUS : vmf_insert_page(vma, addr, page); +} +#endif + #ifndef io_remap_pfn_range_pfn static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, unsigned long size) diff --git a/include/linux/mm_granule.h b/include/linux/mm_granule.h new file mode 100644 index 0000000000000..6e73216edcf96 --- /dev/null +++ b/include/linux/mm_granule.h @@ -0,0 +1,245 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MM_GRANULE_H +#define _LINUX_MM_GRANULE_H + +#include +#include +#include + +/* + * User virtual mapping granule. Never use these for physical PFNs, struct + * page/folio sizes, page-cache indices, or physical memory accounting. + * The granule is immutable once an mm has mappings or is visible to tasks. + */ +#ifndef arch_mm_page_shift +static inline unsigned int arch_mm_page_shift(const struct mm_struct *mm) +{ + return PAGE_SHIFT; +} +#endif + +static inline unsigned int mm_page_shift(const struct mm_struct *mm) +{ + return arch_mm_page_shift(mm); +} + +static inline unsigned long mm_page_size(const struct mm_struct *mm) +{ + return 1UL << mm_page_shift(mm); +} + +/* A byte-range reference must fit both the user leaf and a native page. */ +static inline unsigned long mm_user_fragment_size(const struct mm_struct *mm) +{ + return mm_page_shift(mm) < PAGE_SHIFT ? mm_page_size(mm) : PAGE_SIZE; +} + +/* Native physical pages covered by one base user PTE, including fragments. */ +static inline unsigned long mm_pte_native_pages(const struct mm_struct *mm) +{ + unsigned long pages = mm_page_size(mm) >> PAGE_SHIFT; + + return pages ?: 1; +} + +static inline unsigned long mm_page_mask(const struct mm_struct *mm) +{ + return ~(mm_page_size(mm) - 1); +} + +/* + * ASLR tunables count native-page bits. Retain their byte window when a + * process uses another granule, including the extra low user-page bits. + * native_mask must be a contiguous low-bit mask whose byte span fits ulong. + */ +static inline unsigned long mm_aslr_mask(const struct mm_struct *mm, + unsigned long native_mask) +{ + unsigned int shift = mm_page_shift(mm); + + if (shift < PAGE_SHIFT) { + unsigned int extra = PAGE_SHIFT - shift; + + return (native_mask << extra) | ((1UL << extra) - 1); + } + return native_mask >> (shift - PAGE_SHIFT); +} + +/* Virtual-range accounting. Physical pages and cache indices stay native. */ +static inline unsigned long vma_user_pages(const struct vm_area_struct *vma) +{ + return (vma->vm_end - vma->vm_start) >> mm_page_shift(vma->vm_mm); +} + +static inline unsigned long mm_page_align(const struct mm_struct *mm, unsigned long bytes) +{ + return (bytes + mm_page_size(mm) - 1) & mm_page_mask(mm); +} + +/* + * RSS and virtual counters use the target mm's user granule. Consumers + * comparing different mms or physical memory must convert explicitly. + */ +static inline unsigned long mm_pages_to_bytes(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << mm_page_shift(mm); +} + +static inline unsigned long mm_pages_to_kb(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << (mm_page_shift(mm) - 10); +} + +/* Round up only when reducing precision; do not hide a partial user page. */ +static inline unsigned long mm_pages_to_shift(const struct mm_struct *mm, + unsigned long pages, unsigned int shift) +{ + unsigned int source = mm_page_shift(mm); + + if (source >= shift) + return pages << (source - shift); + return (pages >> (shift - source)) + + !!(pages & ((1UL << (shift - source)) - 1)); +} + +static inline unsigned long mm_pages_to_native(const struct mm_struct *mm, + unsigned long pages) +{ + return mm_pages_to_shift(mm, pages, PAGE_SHIFT); +} + +/* A page-count proc ABI uses the reader's granule, including remote reads. */ +static inline unsigned long mm_pages_for_reader(const struct mm_struct *mm, + const struct mm_struct *reader, unsigned long pages) +{ + return mm_pages_to_shift(mm, pages, + reader ? mm_page_shift(reader) : PAGE_SHIFT); +} + +/* + * Time integrals need one unit across exec. Locked VM also uses this unit: + * it must represent both logical mappings and native physical charges exactly, + * including a single native-page pin in a larger-granule userspace mm. + */ +#ifdef CONFIG_MM_SUBPAGE +#define MM_ACCOUNT_SHIFT 12 +#else +#define MM_ACCOUNT_SHIFT PAGE_SHIFT +#endif + +static inline unsigned long mm_pages_to_account(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << (mm_page_shift(mm) - MM_ACCOUNT_SHIFT); +} + +static inline unsigned long mm_locked_vm_bytes(const struct mm_struct *mm) +{ + return READ_ONCE(mm->locked_vm) << MM_ACCOUNT_SHIFT; +} + +/* Physical-page limits must include any partial native page of locked VM. */ +static inline unsigned long mm_locked_vm_native_pages(const struct mm_struct *mm) +{ + unsigned long units = READ_ONCE(mm->locked_vm); + unsigned int shift = PAGE_SHIFT - MM_ACCOUNT_SHIFT; + + return (units >> shift) + !!(units & ((1UL << shift) - 1)); +} + +/* File-cache indices remain native-page units even for a 4K VMA. */ +struct vm_page_offset { + pgoff_t index; + unsigned int offset; +}; + +static inline struct vm_page_offset vm_page_offset_add(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = pos.offset + (bytes & ~PAGE_MASK); + + pos.index += (bytes >> PAGE_SHIFT) + (low >> PAGE_SHIFT); + pos.offset = low & ~PAGE_MASK; + return pos; +} + +static inline struct vm_page_offset vm_page_offset_sub(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = bytes & ~PAGE_MASK; + + pos.index -= (bytes >> PAGE_SHIFT) + (low > pos.offset); + pos.offset = (pos.offset - low) & ~PAGE_MASK; + return pos; +} + +static inline bool vm_page_offset_equal(struct vm_page_offset a, struct vm_page_offset b) +{ + return a.index == b.index && a.offset == b.offset; +} + +static inline unsigned int vma_subpage_offset(const struct vm_area_struct *vma) +{ +#ifdef CONFIG_MM_SUBPAGE + return vma->vm_subpage_offset; +#else + return 0; +#endif +} + +/* address must be >= vm_start; vm_end computes the next contiguous offset. */ +static inline struct vm_page_offset vma_page_offset_at(const struct vm_area_struct *vma, + unsigned long address) +{ + unsigned long delta = address - vma->vm_start; + unsigned long low = (delta & ~PAGE_MASK) + vma_subpage_offset(vma); + + return (struct vm_page_offset) { + .index = vma->vm_pgoff + (delta >> PAGE_SHIFT) + (low >> PAGE_SHIFT), + .offset = low & ~PAGE_MASK, + }; +} + +/* The start byte of a faulting granule must precede EOF. */ +static inline bool vma_file_offset_valid(const struct vm_area_struct *vma, + unsigned long address, loff_t size) +{ + struct vm_page_offset pos = vma_page_offset_at(vma, address); + pgoff_t end = size >> PAGE_SHIFT; + + return pos.index < end || + (pos.index == end && pos.offset < (size & ~PAGE_MASK)); +} + +static inline void vma_set_page_offset(struct vm_area_struct *vma, struct vm_page_offset pos) +{ + vma->vm_pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = pos.offset; +#endif +} + +/* Inverse lookup of a single byte; an offset outside the VMA is not clamped. */ +static inline unsigned long vma_address_at_offset(const struct vm_area_struct *vma, + struct vm_page_offset pos) +{ + unsigned long len = vma->vm_end - vma->vm_start; + unsigned long index, delta, last; + unsigned int offset = vma_subpage_offset(vma); + + if (!len || pos.index < vma->vm_pgoff || pos.offset >= PAGE_SIZE) + return -EFAULT; + index = pos.index - vma->vm_pgoff; + last = ((len - 1) >> PAGE_SHIFT) + + ((((len - 1) & ~PAGE_MASK) + offset) >> PAGE_SHIFT); + if (index > last) + return -EFAULT; + delta = (index << PAGE_SHIFT) + pos.offset; + if (delta < offset || delta - offset >= len) + return -EFAULT; + return vma->vm_start + delta - offset; +} + +#endif diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h new file mode 100644 index 0000000000000..13344e3a92384 --- /dev/null +++ b/include/linux/mm_subpage.h @@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MM_SUBPAGE_H +#define _LINUX_MM_SUBPAGE_H + +#include +#include +#include + +#define MM_SUBPAGE_SHIFT 12 +#define MM_SUBPAGE_SIZE (1UL << MM_SUBPAGE_SHIFT) +#define MM_SUBPAGES_PER_PAGE (PAGE_SIZE / MM_SUBPAGE_SIZE) + +struct mm_subpage_mapcounts { + unsigned int slots[MM_SUBPAGES_PER_PAGE]; +}; + +struct folio; +struct page; +struct mm_struct; +struct mm_subpage; +struct mm_subpage_pool; +struct page_ext_operations; +struct vm_area_struct; + +extern struct page_ext_operations mm_subpage_ext_ops; + +#ifdef CONFIG_MM_SUBPAGE +/* Per-native-page shmem validity; caller holds the containing folio lock. */ +unsigned long mm_subpage_shmem_missing(struct page *page); +void mm_subpage_shmem_set_missing(struct page *page, unsigned long missing); +#endif + +/* + * A pool is one caller-defined allocation/policy/charge domain. The caller + * supplies fresh, unmapped native folios using its normal NUMA and memcg + * allocation path. No mm, VMA, rmap, swap or folio-private field is repurposed. + * Fault paths transfer slot references to PTEs through the rmap helpers below. + * + * Each caller must hold a pool reference through pool operations. close() + * stops allocation and releases unused backing; it does not consume that + * reference. Existing slot references outlive close() and the originating mm. + */ +struct mm_subpage_pool *mm_subpage_pool_create(gfp_t gfp); +struct mm_subpage_pool *mm_subpage_pool_create_granule(gfp_t gfp, unsigned int shift); +void mm_subpage_pool_get(struct mm_subpage_pool *pool); +void mm_subpage_pool_put(struct mm_subpage_pool *pool); +void mm_subpage_pool_close(struct mm_subpage_pool *pool); + +/* Success consumes the folio reference. On error the caller retains it. */ +int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + gfp_t gfp); +int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int offset, gfp_t gfp); +unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool); + +/* -EAGAIN requires more compatible backing; -ESHUTDOWN means pool closed. */ +struct mm_subpage *mm_subpage_alloc(struct mm_subpage_pool *pool); +struct mm_subpage *mm_subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source); +/* Exact physical byte offset within a native folio; no other slot is substituted. */ +struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned int offset); +struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset); + +/* + * Reattach a slot to an uptodate, locked order-0 swap-cache folio, without + * zeroing or copying its bytes. Caller holds a native folio reference and + * retains it. The returned slot owns an additional reference. -EAGAIN means + * ownership changed concurrently; -ENOMEM means metadata allocation failed. + */ +struct mm_subpage *mm_subpage_restore(struct folio *folio, unsigned int offset, gfp_t gfp); +struct mm_subpage *mm_subpage_restore_granule(struct folio *folio, unsigned int offset, + unsigned int shift, gfp_t gfp); + +/* PTE lock held, one file-backed alternative leaf per call. Native mms are no-ops. */ +#ifdef CONFIG_MM_SUBPAGE +void __mm_subpage_file_map_add(phys_addr_t phys, unsigned long size); +void __mm_subpage_file_map_del(phys_addr_t phys, unsigned long size); +static inline void mm_subpage_file_map_add(struct vm_area_struct *vma, phys_addr_t phys) +{ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + __mm_subpage_file_map_add(phys, mm_page_size(vma->vm_mm)); +} +static inline void mm_subpage_file_map_del(struct vm_area_struct *vma, phys_addr_t phys) +{ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + __mm_subpage_file_map_del(phys, mm_page_size(vma->vm_mm)); +} +/* Fill 4K coverage counts; return alternative leaf count. Statistics only. */ +unsigned int mm_subpage_file_mapcounts(struct page *page, + struct mm_subpage_mapcounts *counts); +#else +static inline void mm_subpage_file_map_add(struct vm_area_struct *vma, phys_addr_t phys) { } +static inline void mm_subpage_file_map_del(struct vm_area_struct *vma, phys_addr_t phys) { } +#endif + +/* Both folios locked; preallocate destination metadata before removing PTEs. */ +#ifdef CONFIG_MM_SUBPAGE +int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst); +bool mm_subpage_migrating(struct folio *folio); +void mm_subpage_migrate_save_lazyfree(struct mm_subpage *subpage); +/* Infallible metadata reuse after prepare; caller still holds the folio lock. */ +struct mm_subpage *mm_subpage_migrate_restore(struct folio *folio, unsigned int offset); +void mm_subpage_migrate_finish(struct folio *folio); +#else +static inline int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) +{ + return 0; +} +static inline bool mm_subpage_migrating(struct folio *folio) +{ + return false; +} +static inline void mm_subpage_migrate_finish(struct folio *folio) { } +#endif + +/* + * get() requires a live reference. A mapping reference must not be put until + * its PTE is removed and TLB invalidation completes. Every pin/alias holds + * its own reference: the slot becomes reusable only after the final put. + * Every live slot reference also holds one reference to the native folio. + * copy() requires the caller to stabilise the source contents (as for COW). + * A failed data copy returns -EHWPOISON without publishing the destination. + */ +void mm_subpage_get(struct mm_subpage *subpage); +void mm_subpage_put(struct mm_subpage *subpage); +/* + * Acquire the live identity containing a 4K-aligned physical address, or NULL. + * Interior quarters of a larger identity return that same identity. + * The caller must stabilise the physical allocation, e.g. with a live PTE + * under its lock or a folio reference. A lockless walker additionally needs + * GUP's TLB/free ordering; RCU protects metadata, not arbitrary physical PFNs. + * This is not a mapping identity check: a lockless PTE reader must revalidate + * its PTE after acquisition, just as for speculative folio references. + */ +struct mm_subpage *mm_subpage_get_from_phys(phys_addr_t phys); +/* + * Optional shared sets for asynchronous consumers of native byte spans. + * add() requires the source slot identities to remain live for the call. + * The pointer being modified is exclusively owned by the caller; get() shares + * a read-only set, and mutation uses copy-on-write. Duplicate ranges add no + * references. prune() requires a stable, side-effect-free membership predicate. + * Allocation failure leaves the original set intact. put() accepts NULL. + */ +struct mm_subpage_refs; +#ifdef CONFIG_MM_SUBPAGE +struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs); +void mm_subpage_refs_put(struct mm_subpage_refs *refs); +int mm_subpage_refs_add(struct mm_subpage_refs **refs, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp); +int mm_subpage_refs_prune(struct mm_subpage_refs **refs, + bool (*keep)(const struct mm_subpage *, void *), + void *context, gfp_t gfp); +#else +static inline struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs) +{ + return refs; +} +static inline void mm_subpage_refs_put(struct mm_subpage_refs *refs) { } +static inline int mm_subpage_refs_add(struct mm_subpage_refs **refs, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp) +{ + return 0; +} +static inline int mm_subpage_refs_prune(struct mm_subpage_refs **refs, + bool (*keep)(const struct mm_subpage *, void *), void *context, gfp_t gfp) +{ + return 0; +} +#endif + +/* + * Anonymous rmap attachment: caller holds the folio lock and PTE lock, has + * prepared vma->anon_vma, and transfers an existing slot reference to its PTE. + * A folio may cover only one native linear index and anon_vma root; its slot + * offset must match the VMA's linear byte offset. Removal needs the PTE lock. + * These helpers do not install PTEs, update RSS, enqueue on LRU, or drop refs. + */ +int mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address); +int mm_subpage_add_new_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address); +void mm_subpage_remove_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma); +/* Both PTE locks and source write_protect_seq held; both mappings become R/O. */ +int mm_subpage_dup_anon_rmap(struct mm_subpage *subpage, + struct vm_area_struct *dst, struct vm_area_struct *src, unsigned long address); +/* + * Per-slot anonymous exclusivity. Setting requires the folio and PTE locks, + * and proof of a fresh private slot or an exclusive migration/swap entry. + * Sharing requires the PTE lock and fails while a typed pin exists. + */ +void mm_subpage_set_exclusive(struct mm_subpage *subpage); +bool mm_subpage_is_exclusive(const struct mm_subpage *subpage); +/* + * Folio and PTE locked; caller proves one remaining swap PTE for this slot. + * Rmap has been attached, but its PTE is not published yet. Reject old GET + * references or present aliases before recovering lost exclusive ownership. + */ +bool mm_subpage_try_reuse_swap(struct mm_subpage *subpage); +/* Present PTE plus caller lookup ref; caller proves no swapped aliases. */ +bool mm_subpage_try_reuse_anon(struct mm_subpage *subpage); +/* PTE locked, caller holds one GET; retire the PTE/TLB before dropping its ref. */ +bool mm_subpage_can_zap(struct mm_subpage *subpage); +bool mm_subpage_try_share(struct mm_subpage *subpage); +/* Folio/PTE locked, caller owns a lookup reference. */ +bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage); +bool mm_subpage_all_lazyfree(struct mm_subpage *subpage); +bool mm_subpage_discardable(struct mm_subpage *subpage); +bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage); +void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage); +/* + * Caller holds a live slot reference and its PTE lock. Success takes an + * additional slot reference plus a native FOLL_PIN reference. The pin keeps + * the exact fragment occupied after unmap; release only with subpage_unpin. + * This is not a GUP walker and does not validate user addresses/permissions. + */ +int mm_subpage_pin(struct mm_subpage *subpage); +void mm_subpage_unpin(struct mm_subpage *subpage); +unsigned int mm_subpage_pincount(const struct mm_subpage *subpage); +#ifdef CONFIG_MM_SUBPAGE +/* A referenced/pinned folio range; never infer offset from the user VA. */ +struct user_page_fragment { + struct folio *folio; + struct mm_subpage *subpage; + unsigned long offset; + unsigned int length; + bool pinned; +}; + +/* + * Takes/releases mmap_lock internally. Returns the number of records filled, + * or -errno if none. A positive partial result is valid; sum record lengths + * to advance the user address. Each record stops at the target mm's granule + * or a native physical-page boundary, whichever comes first. + * Caller owns the target mm lifetime. Release either kind of record with + * release_user_fragments(); copying a record does not acquire a reference. + * FOLL_LONGTERM pins migrate unpinnable backing before returning success. + * Device/PFN mappings are not accepted by this interface. + */ +long pin_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity); +long get_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity); +/* + * Single-record GET with a stable VMA. Caller holds mmap_lock throughout; + * this function never releases it. Returns 1 or -errno (0 for zero length). + * FOLL_NOWAIT is not supported. The returned VMA is valid until unlock. + */ +int get_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma); +/* Single-record PIN under the caller's mmap_lock, retained throughout. + * Same stable-VMA/partial-record contract as GET above. FOLL_LONGTERM and + * FOLL_NOWAIT are rejected: long-term migration can require dropping locks. + */ +int pin_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma); +/* Current mm, one aligned fragment; sparse zero pages return a negative errno. */ +int get_dump_fragment(unsigned long addr, struct user_page_fragment *fragment); +/* Dirty a retained reference without releasing its allocation identity. */ +void mark_user_fragment_dirty(struct user_page_fragment *fragment); +void release_user_fragments(struct user_page_fragment *fragments, + unsigned long count, bool dirty); +#endif + +int mm_subpage_mapcount(const struct mm_subpage *subpage); +/* Borrowed folio: a raw folio reference alone does not keep a slot occupied. */ +struct folio *mm_subpage_folio(const struct mm_subpage *subpage); +unsigned int mm_subpage_shift(const struct mm_subpage *subpage); +unsigned int mm_subpage_size(const struct mm_subpage *subpage); +unsigned int mm_subpage_offset(const struct mm_subpage *subpage); +phys_addr_t mm_subpage_phys(const struct mm_subpage *subpage); + +#endif diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index a308e2c23b829..7f201dda7a13b 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -558,7 +558,8 @@ FOLIO_MATCH(compound_info, _head_3); * @__page_mapping: Aliases with page->mapping. Unused for page tables. * @pt_index: Used for s390 gmap. * @pt_mm: Used for x86 pgds. - * @pt_frag_refcount: For fragmented page table tracking. Powerpc only. + * @pt_frag_refcount: For fragmented page table tracking (powerpc, arm64). + * @pt_frag_shift: Immutable arm64 fragment granule, valid through RCU release. * @pt_share_count: Used for HugeTLB PMD page table share count. * @_pt_pad_2: Padding to ensure proper alignment. * @ptl: Lock for the page table. @@ -585,7 +586,15 @@ struct ptdesc { union { pgoff_t pt_index; struct mm_struct *pt_mm; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* arm64 is 64-bit: use the spare half of this existing word. */ + struct { + atomic_t pt_frag_refcount; + u8 pt_frag_shift; + }; +#else atomic_t pt_frag_refcount; +#endif #ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING atomic_t pt_share_count; #endif @@ -905,6 +914,9 @@ struct vm_area_desc { /* Mutable fields. Populated with initial state. */ pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif struct file *vm_file; vma_flags_t vma_flags; pgprot_t page_prot; @@ -992,6 +1004,9 @@ struct vm_area_struct { /* Information about our backing store: */ unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE units */ +#ifdef CONFIG_MM_SUBPAGE + unsigned int vm_subpage_offset; /* Byte offset within vm_pgoff. */ +#endif struct file * vm_file; /* File we map to (can be NULL). */ void * vm_private_data; /* was vm_pte (shared mem) */ @@ -1281,12 +1296,16 @@ struct mm_struct { unsigned int __percpu *futex_ref; #endif - unsigned long hiwater_rss; /* High-watermark of RSS usage */ - unsigned long hiwater_vm; /* High-water virtual memory usage */ + unsigned long hiwater_rss; /* Peak RSS in user granules */ + unsigned long hiwater_vm; /* Peak VM in user granules */ - unsigned long total_vm; /* Total pages mapped */ - unsigned long locked_vm; /* Pages that have PG_mlocked set */ - atomic64_t pinned_vm; /* Refcount permanently increased */ + unsigned long total_vm; /* Mapped user granules */ +#ifdef CONFIG_MM_SUBPAGE + /* VM_ACCOUNT granules; global commitment rounds this sum once. */ + unsigned long committed_user_pages; +#endif + unsigned long locked_vm; /* Locked bytes >> MM_ACCOUNT_SHIFT */ + atomic64_t pinned_vm; /* Pinned native physical pages */ unsigned long data_vm; /* VM_WRITE & ~VM_SHARED & ~VM_STACK */ unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE & ~VM_STACK */ unsigned long stack_vm; /* VM_STACK */ diff --git a/include/linux/mman.h b/include/linux/mman.h index 389521594c69c..7b80c19f6aa5d 100644 --- a/include/linux/mman.h +++ b/include/linux/mman.h @@ -83,6 +83,17 @@ static inline void vm_unacct_memory(long pages) vm_acct_memory(-pages); } +/* VMA reservations use user granules; shmem/physical reservations use pages. */ +#ifdef CONFIG_MM_SUBPAGE +int mm_account_memory(struct mm_struct *mm, struct mm_struct *policy_mm, long pages); +void mm_acct_memory(struct mm_struct *mm, long pages); +void mm_unacct_memory(struct mm_struct *mm, unsigned long pages); +#else +#define mm_account_memory(mm, policy_mm, pages) security_vm_enough_memory_mm(policy_mm, pages) +#define mm_acct_memory(mm, pages) vm_acct_memory(pages) +#define mm_unacct_memory(mm, pages) vm_unacct_memory(pages) +#endif + /* * Allow architectures to handle additional protection and flag bits. The * overriding macros must be defined in the arch-specific asm/mman.h file. diff --git a/include/linux/mmu_context.h b/include/linux/mmu_context.h index ed3dd0f3fe194..a4260bc898205 100644 --- a/include/linux/mmu_context.h +++ b/include/linux/mmu_context.h @@ -5,6 +5,11 @@ #include #include +/* Select immutable exec geometry before allocating any page tables. */ +#ifndef arch_mm_init_exec +static inline void arch_mm_init_exec(struct mm_struct *mm, unsigned int page_shift) { } +#endif + /* Architectures that care about IRQ state in switch_mm can override this. */ #ifndef switch_mm_irqs_off # define switch_mm_irqs_off switch_mm diff --git a/include/linux/page_table_check.h b/include/linux/page_table_check.h index 12268a32e8be1..3ee1096fb0e53 100644 --- a/include/linux/page_table_check.h +++ b/include/linux/page_table_check.h @@ -12,6 +12,10 @@ extern struct static_key_true page_table_check_disabled; extern struct page_ext_operations page_table_check_ops; +#if IS_ENABLED(CONFIG_KUNIT) +int page_table_check_get_counts(struct page *page, unsigned int offset, + int *anon, int *file); +#endif void __page_table_check_zero(struct page *page, unsigned int order); void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 31a848485ad9d..9cde417795f00 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1068,10 +1068,7 @@ static inline pgoff_t folio_pgoff(const struct folio *folio) static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, const unsigned long address) { - pgoff_t pgoff; - pgoff = (address - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; - return pgoff; + return vma_page_offset_at(vma, address).index; } struct wait_page_key { diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index cdd68ed3ae1a9..f5ef44f25c52b 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -12,11 +12,18 @@ #ifdef CONFIG_MMU #include +#include #include #include #include #include +/* pgtable_t is an opaque allocation token, not necessarily struct page *. */ +#ifndef pgtable_to_page +#define pgtable_to_page(table) (table) +#define pgtable_from_page(page) (page) +#endif + #if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS #error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED @@ -155,6 +162,59 @@ static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address) */ #define pgd_offset_k(address) pgd_offset(&init_mm, (address)) +/* Explicit target-mm accessors; legacy accessors keep native geometry. */ +#ifndef pte_phys_mm +#define pte_phys_mm(mm, pte) PFN_PHYS(pte_pfn(pte)) +#endif +#ifndef phys_pte_mm +#define phys_pte_mm(mm, phys, prot) pfn_pte((phys) >> PAGE_SHIFT, prot) +#endif +#ifndef pte_pgprot_mm +#define pte_pgprot_mm(mm, pte) pte_pgprot(pte) +#endif +#ifndef pgd_offset_pgd_mm +#define pgd_offset_pgd_mm(mm, pgd, addr) pgd_offset_pgd(pgd, addr) +#endif +#ifndef p4d_offset_mm +#define p4d_offset_mm(mm, pgd, addr) p4d_offset(pgd, addr) +#endif +#ifndef pud_offset_mm +#define pud_offset_mm(mm, p4d, addr) pud_offset(p4d, addr) +#endif +#ifndef pmd_offset_mm +#define pmd_offset_mm(mm, pud, addr) pmd_offset(pud, addr) +#endif +#ifndef __pte_map_mm +#define __pte_map_mm(mm, pmd, addr) __pte_map(pmd, addr) +#endif +#ifndef pte_offset_kernel_mm +#define pte_offset_kernel_mm(mm, pmd, addr) pte_offset_kernel(pmd, addr) +#endif + +#ifndef pud_pgtable_mm +#define pud_pgtable_mm(mm, pud) pud_pgtable(pud) +#endif + +#ifndef pmd_size_mm +#define pmd_shift_mm(mm) PMD_SHIFT +#define pud_shift_mm(mm) PUD_SHIFT +#define p4d_shift_mm(mm) P4D_SHIFT +#define pmd_size_mm(mm) PMD_SIZE +#define pud_size_mm(mm) PUD_SIZE +#define p4d_size_mm(mm) P4D_SIZE +#define pgd_size_mm(mm) PGDIR_SIZE +#define pmd_mask_mm(mm) PMD_MASK +#define pud_mask_mm(mm) PUD_MASK +#define p4d_mask_mm(mm) P4D_MASK +#define pgd_mask_mm(mm) PGDIR_MASK +#endif + +#ifndef pte_table_bytes_mm +#define pte_table_bytes_mm(mm) (PTRS_PER_PTE * sizeof(pte_t)) +#define pmd_table_bytes_mm(mm) (PTRS_PER_PMD * sizeof(pmd_t)) +#define pud_table_bytes_mm(mm) (PTRS_PER_PUD * sizeof(pud_t)) +#endif + /* * In many cases it is known that a virtual address is mapped at PMD or PTE * level, so instead of traversing all the page table levels, we can get a @@ -388,6 +448,10 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) #define pte_next_pfn(pte) pte_advance_pfn(pte, 1) +#ifndef pte_advance_pfn_mm +#define pte_advance_pfn_mm(mm, pte, nr) pte_advance_pfn(pte, nr) +#endif + #ifndef set_ptes /** * set_ptes - Map consecutive pages to a contiguous range of addresses. @@ -1465,6 +1529,13 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) * and any p?d_bad entries - reporting the error before resetting to none. * Do the tests inline, but report and clear the bad entry in mm/memory.c. */ +#ifndef pgd_addr_end_mm +#define pgd_addr_end_mm(mm, addr, end) pgd_addr_end(addr, end) +#define p4d_addr_end_mm(mm, addr, end) p4d_addr_end(addr, end) +#define pud_addr_end_mm(mm, addr, end) pud_addr_end(addr, end) +#define pmd_addr_end_mm(mm, addr, end) pmd_addr_end(addr, end) +#endif + void pgd_clear_bad(pgd_t *); #ifndef __PAGETABLE_P4D_FOLDED @@ -1492,6 +1563,25 @@ static inline int pgd_none_or_clear_bad(pgd_t *pgd) return 0; } +#ifndef p4d_none_mm +#define p4d_none_mm(mm, p4d) p4d_none(p4d) +#define p4d_bad_mm(mm, p4d) p4d_bad(p4d) +#define p4d_present_mm(mm, p4d) p4d_present(p4d) +#define p4d_clear_mm(mm, p4d) p4d_clear(p4d) +#endif + +static inline int p4d_none_or_clear_bad_mm(struct mm_struct *mm, p4d_t *p4d) +{ + if (p4d_none_mm(mm, *p4d)) + return 1; + if (unlikely(p4d_bad_mm(mm, *p4d))) { + p4d_ERROR(*p4d); + p4d_clear_mm(mm, p4d); + return 1; + } + return 0; +} + static inline int p4d_none_or_clear_bad(p4d_t *p4d) { if (p4d_none(*p4d)) @@ -1979,11 +2069,15 @@ static inline int is_zero_pfn(unsigned long pfn) #define zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr)) #else +#ifndef ARCH_ZERO_PAGE_SIZE +#define ARCH_ZERO_PAGE_SIZE PAGE_SIZE +#endif + static inline int is_zero_pfn(unsigned long pfn) { extern unsigned long zero_page_pfn; - return pfn == zero_page_pfn; + return pfn - zero_page_pfn < (ARCH_ZERO_PAGE_SIZE >> PAGE_SHIFT); } static inline unsigned long zero_pfn(unsigned long addr) @@ -1993,7 +2087,7 @@ static inline unsigned long zero_pfn(unsigned long addr) return zero_page_pfn; } -extern uint8_t empty_zero_page[PAGE_SIZE]; +extern uint8_t empty_zero_page[ARCH_ZERO_PAGE_SIZE]; extern struct page *__zero_page; static inline struct page *_zero_page(unsigned long addr) diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 8dc0871e5f001..2bc2c986eb76e 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -40,6 +40,10 @@ struct anon_vma { * anon_vma if they are the last user on release */ atomic_t refcount; +#ifdef CONFIG_MM_SUBPAGE + /* Immutable base-page requirement shared by this anonymous lineage. */ + unsigned int min_folio_order; +#endif /* * Count of child anon_vmas. Equals to the count of all anon_vmas that @@ -398,29 +402,40 @@ static __always_inline void __folio_rmap_sanity_checks(const struct folio *folio } /* - * rmap interfaces called when adding or removing pte of page + * Single-PTE interfaces cover the complete native-page span of a user leaf. + * Multi-page interfaces below retain explicit native physical-page counts. */ void folio_move_anon_rmap(struct folio *, struct vm_area_struct *); void folio_add_anon_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *, unsigned long address, rmap_t flags); -#define folio_add_anon_rmap_pte(folio, page, vma, address, flags) \ - folio_add_anon_rmap_ptes(folio, page, 1, vma, address, flags) +static inline void folio_add_anon_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma, unsigned long address, rmap_t flags) +{ + folio_add_anon_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), + vma, address, flags); +} void folio_add_anon_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *, unsigned long address, rmap_t flags); void folio_add_new_anon_rmap(struct folio *, struct vm_area_struct *, unsigned long address, rmap_t flags); void folio_add_file_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *); -#define folio_add_file_rmap_pte(folio, page, vma) \ - folio_add_file_rmap_ptes(folio, page, 1, vma) +static inline void folio_add_file_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma) +{ + folio_add_file_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), vma); +} void folio_add_file_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *); void folio_add_file_rmap_pud(struct folio *, struct page *, struct vm_area_struct *); void folio_remove_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *); -#define folio_remove_rmap_pte(folio, page, vma) \ - folio_remove_rmap_ptes(folio, page, 1, vma) +static inline void folio_remove_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma) +{ + folio_remove_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), vma); +} void folio_remove_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *); void folio_remove_rmap_pud(struct folio *, struct page *, @@ -541,7 +556,8 @@ static inline void folio_dup_file_rmap_ptes(struct folio *folio, static __always_inline void folio_dup_file_rmap_pte(struct folio *folio, struct page *page, struct vm_area_struct *dst_vma) { - __folio_dup_file_rmap(folio, page, 1, dst_vma, PGTABLE_LEVEL_PTE); + __folio_dup_file_rmap(folio, page, mm_pte_native_pages(dst_vma->vm_mm), + dst_vma, PGTABLE_LEVEL_PTE); } /** @@ -665,8 +681,8 @@ static __always_inline int folio_try_dup_anon_rmap_pte(struct folio *folio, struct page *page, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) { - return __folio_try_dup_anon_rmap(folio, page, 1, dst_vma, src_vma, - PGTABLE_LEVEL_PTE); + return __folio_try_dup_anon_rmap(folio, page, mm_pte_native_pages(src_vma->vm_mm), + dst_vma, src_vma, PGTABLE_LEVEL_PTE); } /** @@ -708,13 +724,16 @@ static inline int folio_try_dup_anon_rmap_pmd(struct folio *folio, static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, struct page *page, int nr_pages, enum pgtable_level level) { + int i, exclusive_pages = level == PGTABLE_LEVEL_PTE ? nr_pages : 1; + VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); VM_WARN_ON_FOLIO(!PageAnonExclusive(page), folio); __folio_rmap_sanity_checks(folio, page, nr_pages, level); /* device private folios cannot get pinned via GUP. */ if (unlikely(folio_is_device_private(folio))) { - ClearPageAnonExclusive(page); + for (i = 0; i < exclusive_pages; i++) + ClearPageAnonExclusive(page + i); return 0; } @@ -766,7 +785,8 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, if (unlikely(folio_maybe_dma_pinned(folio))) return -EBUSY; - ClearPageAnonExclusive(page); + for (i = 0; i < exclusive_pages; i++) + ClearPageAnonExclusive(page + i); /* * This is conceptually a smp_wmb() paired with the smp_rmb() in @@ -782,7 +802,8 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, * mapped by a PTE possibly shared to prepare * for KSM or temporary unmapping * @folio: The folio to share a mapping of - * @page: The mapped exclusive page + * @page: The first native page covered by the exclusive user PTE + * @vma: The VMA defining the base user PTE granule * * The caller needs to hold the page table lock and has to have the page table * entries cleared/invalidated. @@ -799,9 +820,10 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, * -EBUSY otherwise. */ static inline int folio_try_share_anon_rmap_pte(struct folio *folio, - struct page *page) + struct page *page, struct vm_area_struct *vma) { - return __folio_try_share_anon_rmap(folio, page, 1, PGTABLE_LEVEL_PTE); + return __folio_try_share_anon_rmap(folio, page, mm_pte_native_pages(vma->vm_mm), + PGTABLE_LEVEL_PTE); } /** @@ -855,12 +877,23 @@ struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr, #define PVMW_SYNC (1 << 0) /* Look for migration entries rather than present PTEs */ #define PVMW_MIGRATION (1 << 1) +/* Match one present subpage PTE at address, including its physical offset. */ +#define PVMW_SUBPAGE (1 << 2) /* Result flags */ /* The page is mapped across page table boundary */ #define PVMW_PGTABLE_CROSSED (1 << 16) +/* One reverse-map callback, optionally restricted to one physical user leaf. */ +struct rmap_walk_range { + unsigned long address; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; + bool subpage; +#endif +}; + struct page_vma_mapped_walk { unsigned long pfn; unsigned long nr_pages; @@ -871,6 +904,10 @@ struct page_vma_mapped_walk { pte_t *pte; spinlock_t *ptl; unsigned int flags; +#ifdef CONFIG_MM_SUBPAGE + /* PVMW_SUBPAGE: byte offset within pfn; nr_pages must be one. */ + unsigned int subpage_offset; +#endif }; #define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \ @@ -883,6 +920,32 @@ struct page_vma_mapped_walk { .flags = _flags, \ } +static inline struct page_vma_mapped_walk folio_rmap_walk_init( + struct folio *folio, struct vm_area_struct *vma, + struct rmap_walk_range range, unsigned int flags) +{ + struct page_vma_mapped_walk walk = { + .pfn = folio_pfn(folio), + .nr_pages = folio_nr_pages(folio), + .pgoff = folio_pgoff(folio), + .vma = vma, + .address = range.address, + .flags = flags, + }; + +#ifdef CONFIG_MM_SUBPAGE + if (range.subpage) { + walk.flags |= PVMW_SUBPAGE; + walk.subpage_offset = range.subpage_offset; + } +#endif + return walk; +} + +#define DEFINE_FOLIO_RMAP_WALK(name, _folio, _vma, _range, _flags) \ + struct page_vma_mapped_walk name = \ + folio_rmap_walk_init(_folio, _vma, _range, _flags) + static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw) { /* HugeTLB pte is set to the relevant page table entry without pte_mapped. */ @@ -943,7 +1006,7 @@ void remove_migration_ptes(struct folio *src, struct folio *dst, * arg: passed to rmap_one() and invalid_vma() * try_lock: bail out if the rmap lock is contended * contended: indicate the rmap traversal bailed out due to lock contention - * rmap_one: executed on each vma where page is mapped + * rmap_one: executed for each mapped VMA range (possibly one user leaf) * done: for checking traversing termination condition * anon_lock: for getting anon_lock by optimized way rather than default * invalid_vma: for skipping uninterested vma @@ -957,7 +1020,7 @@ struct rmap_walk_control { * Otherwise, return true. */ bool (*rmap_one)(struct folio *folio, struct vm_area_struct *vma, - unsigned long addr, void *arg); + struct rmap_walk_range range, void *arg); int (*done)(struct folio *folio); struct anon_vma *(*anon_lock)(const struct folio *folio, struct rmap_walk_control *rwc); @@ -966,6 +1029,15 @@ struct rmap_walk_control { void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc); void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc); +#ifdef CONFIG_MM_SUBPAGE +unsigned int folio_anon_min_order(struct folio *folio); +#else +static inline unsigned int folio_anon_min_order(struct folio *folio) +{ + return 0; +} +#endif + struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio, struct rmap_walk_control *rwc); diff --git a/include/linux/sched.h b/include/linux/sched.h index eaf9751383596..31b894db80f11 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -961,6 +961,11 @@ struct task_struct { struct mm_struct *mm; struct mm_struct *active_mm; +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + /* Per-thread exec policy; fork inherits it, the live mm is immutable. */ + unsigned int exec_page_shift; + unsigned int default_exec_page_shift; +#endif int exit_state; int exit_code; diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 95d0040df5841..38d573026d7fb 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -14,6 +14,10 @@ * Routines for handling mm_structs */ extern struct mm_struct *mm_alloc(void); +extern struct mm_struct *mm_alloc_exec(void); +extern struct mm_struct *mm_alloc_exec_page_shift(unsigned int page_shift); +/* Return zero for native, an alternative shift, or a negative errno. */ +int arch_exec_page_size_shift(unsigned long size); /** * mmgrab() - Pin a &struct mm_struct. diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 584ae88b435e9..7af2aeb81e41d 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -194,7 +194,7 @@ struct signal_struct { unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw; unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt; unsigned long inblock, oublock, cinblock, coublock; - unsigned long maxrss, cmaxrss; + unsigned long maxrss, cmaxrss; /* kB, independent of each mm granule */ struct task_io_accounting ioac; /* diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 69b0177da1564..bf3a012954700 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -32,6 +32,8 @@ struct swap_iocb; * isn't directly visible to userspace. */ #define SHMEM_F_MAPPING_FROZEN BIT(2) +/* This inode has used sub-native shared-page validity. */ +#define SHMEM_F_UFFD_SUBPAGE BIT(3) struct shmem_inode_info { spinlock_t lock; @@ -52,6 +54,10 @@ struct shmem_inode_info { pgoff_t fallocend; /* highest fallocate endindex */ unsigned int fsflags; /* for FS_IOC_[SG]ETFLAGS */ atomic_t stop_eviction; /* hold when working on inode */ +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + /* Swapped missing 4K units, keyed by native cache index. */ + struct xarray uffd_missing; +#endif #ifdef CONFIG_TMPFS_QUOTA struct dquot __rcu *i_dquot[MAXQUOTAS]; #endif @@ -113,13 +119,56 @@ extern unsigned long shmem_get_unmapped_area(struct file *, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); extern int shmem_lock(struct file *file, int lock, struct ucounts *ucounts); #ifdef CONFIG_SHMEM +struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, struct inode *inode, + pgoff_t index, unsigned int order); bool shmem_mapping(const struct address_space *mapping); #else +static inline struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, + struct inode *inode, pgoff_t index, unsigned int order) +{ + return ERR_PTR(-EOPNOTSUPP); +} static inline bool shmem_mapping(const struct address_space *mapping) { return false; } #endif /* CONFIG_SHMEM */ +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +/* Return a locked cache folio and caller reference; new backing is all missing. */ +struct folio *shmem_uffd_prepare_folio(struct vm_area_struct *vma, + unsigned long address, bool *new); +void shmem_uffd_abort_folio(struct folio *folio, struct vm_area_struct *vma, bool new); +bool shmem_uffd_has_missing_native_page(struct folio *folio); +bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length); +/* Resident masks require no allocation; inode metadata persists only in swap. */ +void shmem_uffd_copy_page(struct page *dst, struct page *src); +void shmem_uffd_migrate(struct folio *dst, struct folio *src); +/* Locked, uptodate cache folio. */ +unsigned long shmem_uffd_missing_mask(struct folio *folio, pgoff_t index); +int shmem_uffd_set_missing(struct folio *folio, pgoff_t index, + unsigned long missing); +/* Clear populated units; no allocation. Caller holds the folio lock. */ +void shmem_uffd_mark_present(struct folio *folio, pgoff_t index, + unsigned long present); +/* Mark exactly the byte range exposed by a mapping; folio lock held. */ +void shmem_uffd_mark_mapped(struct folio *folio, unsigned long offset, + unsigned long length); +/* i_pages lock held; deletion, not replacement by swap or migration. */ +void shmem_uffd_forget(struct address_space *mapping, pgoff_t start, + unsigned long nr); +#else +static inline void shmem_uffd_copy_page(struct page *dst, struct page *src) { } +static inline void shmem_uffd_migrate(struct folio *dst, struct folio *src) { } +static inline bool shmem_uffd_has_missing_native_page(struct folio *folio) { return false; } +static inline bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length) { return false; } +static inline void shmem_uffd_mark_mapped(struct folio *folio, + unsigned long offset, unsigned long length) { } +static inline void shmem_uffd_forget(struct address_space *mapping, + pgoff_t start, unsigned long nr) { } +#endif + void shmem_unlock_mapping(struct address_space *mapping); struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); @@ -160,6 +209,8 @@ static inline void shmem_uncharge(struct inode *inode, long pages) { } #endif +unsigned long shmem_partial_swap_usage_bytes(struct address_space *mapping, + u64 start, u64 end); extern unsigned long shmem_partial_swap_usage(struct address_space *mapping, pgoff_t start, pgoff_t end); diff --git a/include/linux/swap.h b/include/linux/swap.h index 7a09df6977a5f..4e9136d3ae3f6 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -228,7 +228,7 @@ enum { */ #define SWAP_ENTRY_INVALID 0 -#ifdef CONFIG_THP_SWAP +#if defined(CONFIG_THP_SWAP) || defined(CONFIG_MM_SUBPAGE) #define SWAP_NR_ORDERS (PMD_ORDER + 1) #else #define SWAP_NR_ORDERS 1 @@ -443,6 +443,7 @@ extern int swp_swapcount(swp_entry_t entry); struct backing_dev_info; extern struct swap_info_struct *get_swap_device(swp_entry_t entry); sector_t swap_folio_sector(struct folio *folio); +sector_t swap_extent_sector(swp_entry_t entry, unsigned long *nr_pages); /* * If there is an existing swap slot reference (swap entry) and the caller @@ -455,6 +456,13 @@ sector_t swap_folio_sector(struct folio *folio); * a swap count > 1. See comments of folio_*_swap helpers for more info. */ int swap_dup_entry_direct(swp_entry_t entry); +int swap_dup_entries_direct(swp_entry_t entry, unsigned int nr); +#ifdef CONFIG_MM_SUBPAGE +/* Offset is bytes within one native swap entry; caller owns a PTE/cache ref. */ +int swap_subpage_dup(swp_entry_t entry, unsigned int offset); +void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache); +unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset); +#endif void swap_put_entries_direct(swp_entry_t entry, int nr); /* @@ -498,11 +506,27 @@ static inline void free_swap_cache(struct folio *folio) { } +#ifdef CONFIG_MM_SUBPAGE +static inline int swap_subpage_dup(swp_entry_t entry, unsigned int offset) +{ + return -EINVAL; +} +static inline void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache) { } +static inline unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset) +{ + return 0; +} +#endif static inline int swap_dup_entry_direct(swp_entry_t ent) { return 0; } +static inline int swap_dup_entries_direct(swp_entry_t ent, unsigned int nr) +{ + return 0; +} + static inline void swap_put_entries_direct(swp_entry_t ent, int nr) { } diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 4fb7291f54b62..29beb44f370fe 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1261,6 +1261,11 @@ static inline int ksys_fadvise64_64(int fd, loff_t offset, loff_t len, unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long fd, unsigned long pgoff); +#ifdef CONFIG_MM_SUBPAGE +unsigned long ksys_mmap_user(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long offset); +#endif + ssize_t ksys_readahead(int fd, loff_t offset, size_t count); int ksys_ipc(unsigned int call, int first, unsigned long second, unsigned long third, void __user * ptr, long fifth); diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index f548fea2adec8..237b530ed600f 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -192,6 +192,7 @@ struct uprobes_state { }; typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr, + unsigned int page_offset, uprobe_opcode_t *insn, int nbytes, void *data); extern void __init uprobes_init(void); diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h index cd7920c81f85b..fefbf65d61441 100644 --- a/include/trace/events/kmem.h +++ b/include/trace/events/kmem.h @@ -449,7 +449,7 @@ TRACE_EVENT(rss_stat, __entry->curr = current->mm == mm && !(current->flags & PF_KTHREAD); __entry->member = member; __entry->size = (percpu_counter_sum_positive(&mm->rss_stat[member]) - << PAGE_SHIFT); + << mm_page_shift(mm)); ), TP_printk("mm_id=%u curr=%d type=%s size=%ldB", diff --git a/include/trace/events/mmap.h b/include/trace/events/mmap.h index ee2843a5daef1..46df97560969d 100644 --- a/include/trace/events/mmap.h +++ b/include/trace/events/mmap.h @@ -26,7 +26,7 @@ TRACE_EVENT(vm_unmapped_area, TP_fast_assign( __entry->addr = addr; - __entry->total_vm = current->mm->total_vm; + __entry->total_vm = mm_pages_to_native(current->mm, current->mm->total_vm); __entry->flags = info->flags; __entry->length = info->length; __entry->low_limit = info->low_limit; diff --git a/include/trace/events/oom.h b/include/trace/events/oom.h index 9f0a5d1482c44..d5321b6f01ffa 100644 --- a/include/trace/events/oom.h +++ b/include/trace/events/oom.h @@ -7,7 +7,6 @@ #include #include -#define PG_COUNT_TO_KB(x) ((x) << (PAGE_SHIFT - 10)) TRACE_EVENT(oom_score_adj_update, @@ -93,10 +92,10 @@ TRACE_EVENT(mark_victim, TP_fast_assign( __entry->pid = task->pid; __assign_str(comm); - __entry->total_vm = PG_COUNT_TO_KB(task->mm->total_vm); - __entry->anon_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_ANONPAGES)); - __entry->file_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_FILEPAGES)); - __entry->shmem_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_SHMEMPAGES)); + __entry->total_vm = mm_pages_to_kb(task->mm, task->mm->total_vm); + __entry->anon_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_ANONPAGES)); + __entry->file_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_FILEPAGES)); + __entry->shmem_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_SHMEMPAGES)); __entry->uid = uid; __entry->pgtables = mm_pgtables_bytes(task->mm) >> 10; __entry->oom_score_adj = task->signal->oom_score_adj; diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index 1a06e7feb7dc9..6e08471fbe822 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -421,4 +421,22 @@ struct prctl_mm_map { # define PR_CFI_DISABLE _BITUL(1) # define PR_CFI_LOCK _BITUL(2) +/* + * Aurora experimental ABI: private numbers, not upstream assignments. + * SET_EXEC selects the calling thread's next successful exec granule. + * Zero requests native pages; with an alternative DEFAULT in effect this + * installs an explicit one-shot native override. GET_EXEC returns zero when + * no override remains, or its size in bytes. Successful exec consumes it. + * SET_DEFAULT selects the fallback for subsequent execs; zero restores native. + * GET_DEFAULT returns the alternative size in bytes, or zero for native. + * Fork inherits both policies; exec preserves DEFAULT. Failed exec preserves + * both. Neither setter changes the live address space or other threads. + * AArch32 always selects 4K when supported, otherwise the native granule, + * independently of these preferences. Sizes are capability-checked. + */ +#define PR_AURORA_SET_EXEC_PAGE_SIZE 0x41555001 +#define PR_AURORA_GET_EXEC_PAGE_SIZE 0x41555002 +#define PR_AURORA_SET_DEFAULT_PAGE_SIZE 0x41555003 +#define PR_AURORA_GET_DEFAULT_PAGE_SIZE 0x41555004 + #endif /* _LINUX_PRCTL_H */ diff --git a/ipc/shm.c b/ipc/shm.c index b3e8a58e177d3..2929b9243f8e8 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -51,6 +51,14 @@ #include "util.h" +#ifndef arch_shm_attach_align +static unsigned long arch_shm_attach_align(const struct mm_struct *mm, + unsigned long align) +{ + return align; +} +#endif + struct shmid_kernel /* private to the kernel */ { struct kern_ipc_perm shm_perm; @@ -1538,6 +1546,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, if (shmid < 0) goto out; + shmlba = arch_shm_attach_align(current->mm, shmlba); if (addr) { if (addr & (shmlba - 1)) { if (shmflg & SHM_RND) { @@ -1552,7 +1561,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, goto out; } else #ifndef __ARCH_FORCE_SHMLBA - if (addr & ~PAGE_MASK) + if (addr & ~mm_page_mask(current->mm)) #endif goto out; } @@ -1723,6 +1732,20 @@ COMPAT_SYSCALL_DEFINE3(shmat, int, shmid, compat_uptr_t, shmaddr, int, shmflg) } #endif +/* VMA splits retain native file indices plus a subpage byte offset. */ +#ifdef CONFIG_MMU +static bool shm_vma_matches_addr(struct vm_area_struct *vma, unsigned long addr) +{ + unsigned long offset; + + if (vma->vm_ops != &shm_vm_ops || vma->vm_start < addr) + return false; + offset = vma->vm_start - addr; + return (offset >> PAGE_SHIFT) == vma->vm_pgoff && + (offset & ~PAGE_MASK) == vma_subpage_offset(vma); +} +#endif + /* * detach and kill segment if marked destroyed. * The work is done in shm_close. @@ -1739,7 +1762,7 @@ long ksys_shmdt(char __user *shmaddr) VMA_ITERATOR(vmi, mm, addr); #endif - if (addr & ~PAGE_MASK) + if (addr & ~mm_page_mask(mm)) return retval; if (mmap_write_lock_killable(mm)) @@ -1774,8 +1797,7 @@ long ksys_shmdt(char __user *shmaddr) * a fragment created by mprotect() and/or munmap(), or it * otherwise it starts at this address with no hassles. */ - if ((vma->vm_ops == &shm_vm_ops) && - (vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) { + if (shm_vma_matches_addr(vma, addr)) { /* * Record the file of the shm segment being @@ -1804,12 +1826,10 @@ long ksys_shmdt(char __user *shmaddr) * could possibly have landed at. Also cast things to loff_t to * prevent overflows and make comparisons vs. equal-width types. */ - size = PAGE_ALIGN(size); + size = ALIGN(size, mm_page_size(mm)); while (vma && (loff_t)(vma->vm_end - addr) <= size) { /* finding a matching vma now does not alter retval */ - if ((vma->vm_ops == &shm_vm_ops) && - ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) && - (vma->vm_file == file)) { + if (shm_vma_matches_addr(vma, addr) && vma->vm_file == file) { do_vmi_align_munmap(&vmi, vma, mm, vma->vm_start, vma->vm_end, NULL, false); } diff --git a/kernel/events/core.c b/kernel/events/core.c index 1a23ae67c3cc8..343dc1af5a308 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6812,7 +6812,7 @@ static void perf_event_init_userpage(struct perf_event *event) /* Allow new userspace to detect that bit 0 is deprecated */ userpg->cap_bit0_is_deprecated = 1; userpg->size = offsetof(struct perf_event_mmap_page, __reserved); - userpg->data_offset = PAGE_SIZE; + userpg->data_offset = perf_metadata_size(rb); userpg->data_size = perf_data_size(rb); unlock: @@ -6997,6 +6997,11 @@ typedef void (*mapped_f)(struct perf_event *event, struct mm_struct *mm); f; \ }) +static u64 perf_mmap_offset(struct vm_area_struct *vma) +{ + return ((u64)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); +} + static void perf_mmap_open(struct vm_area_struct *vma) { struct perf_event *event = vma->vm_file->private_data; @@ -7005,7 +7010,7 @@ static void perf_mmap_open(struct vm_area_struct *vma) refcount_inc(&event->mmap_count); refcount_inc(&event->rb->mmap_count); - if (vma->vm_pgoff) + if (perf_mmap_offset(vma)) refcount_inc(&event->rb->aux_mmap_count); if (mapped) @@ -7039,7 +7044,7 @@ static void perf_mmap_close(struct vm_area_struct *vma) * The AUX buffer is strictly a sub-buffer, serialize using aux_mutex * to avoid complications. */ - if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff && + if (rb_has_aux(rb) && perf_mmap_offset(vma) == rb->aux_offset && refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) { /* * Stop all AUX events that are writing to this buffer, @@ -7132,7 +7137,9 @@ static void perf_mmap_close(struct vm_area_struct *vma) static vm_fault_t perf_mmap_pfn_mkwrite(struct vm_fault *vmf) { /* The first page is the user control page, others are read-only. */ - return vmf->pgoff == 0 ? 0 : VM_FAULT_SIGBUS; + return !perf_mmap_offset(vmf->vma) && + vmf->address - vmf->vma->vm_start < mm_page_size(vmf->vma->vm_mm) ? + 0 : VM_FAULT_SIGBUS; } static int perf_mmap_may_split(struct vm_area_struct *vma, unsigned long addr) @@ -7153,7 +7160,8 @@ static const struct vm_operations_struct perf_mmap_vmops = { static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) { - unsigned long nr_pages = vma_pages(vma); + unsigned long va = vma->vm_start; + bool aux = perf_mmap_offset(vma) != 0; int err = 0; unsigned long pagenum; @@ -7197,9 +7205,21 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) * perf_mmap_open() and perf_mmap_close()) so we ensure the lifetime of * this mapping is maintained correctly. */ - for (pagenum = 0; pagenum < nr_pages; pagenum++) { - unsigned long va = vma->vm_start + PAGE_SIZE * pagenum; - struct page *page = perf_mmap_to_page(rb, vma->vm_pgoff + pagenum); + for (pagenum = 0; va < vma->vm_end; pagenum++) { + unsigned long size = min(PAGE_SIZE, vma->vm_end - va); + struct page *page; + + if (aux) { + if (pagenum >= rb->aux_nr_pages) { + err = -EINVAL; + break; + } + page = virt_to_page(rb->aux_pages[pagenum]); + } else { + page = perf_mmap_to_page(rb, pagenum); + if (!pagenum) + size = perf_metadata_size(rb); + } if (page == NULL) { err = -EINVAL; @@ -7207,16 +7227,17 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) } /* Map readonly, perf_mmap_pfn_mkwrite() called on write fault. */ - err = remap_pfn_range(vma, va, page_to_pfn(page), PAGE_SIZE, + err = remap_pfn_range(vma, va, page_to_pfn(page), size, vm_get_page_prot(vma->vm_flags & ~VM_SHARED)); if (err) break; + va += size; } #ifdef CONFIG_MMU /* Clear any partial mappings on error. */ if (err) - zap_vma_range(vma, vma->vm_start, nr_pages * PAGE_SIZE); + zap_vma_range(vma, vma->vm_start, vma->vm_end - vma->vm_start); #endif return err; @@ -7269,7 +7290,7 @@ static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer * { struct user_struct *user = rb->mmap_user; - atomic_long_sub((perf_data_size(rb) >> PAGE_SHIFT) + 1 - rb->mmap_locked, + atomic_long_sub(data_page_nr(rb) + 1 - rb->mmap_locked, &user->locked_vm); atomic64_sub(rb->mmap_locked, &vma->vm_mm->pinned_vm); } @@ -7277,7 +7298,9 @@ static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer * static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, unsigned long nr_pages) { - long extra = 0, user_extra = nr_pages; + unsigned int page_shift = mm_page_shift(vma->vm_mm); + unsigned long data_size; + long extra = 0, user_extra; struct perf_buffer *rb; int rb_flags = 0; @@ -7289,11 +7312,14 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, */ if (nr_pages != 0 && !is_power_of_2(nr_pages)) return -EINVAL; + data_size = nr_pages << page_shift; + /* Physical allocation and locked-memory accounting stay in native pages. */ + user_extra = DIV_ROUND_UP(data_size, PAGE_SIZE) + 1; WARN_ON_ONCE(event->ctx->parent_ctx); if (event->rb) { - if (data_page_nr(event->rb) != nr_pages) + if (perf_data_size(event->rb) != data_size) return -EINVAL; /* @@ -7329,7 +7355,7 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, if (vma->vm_flags & VM_WRITE) rb_flags |= RING_BUFFER_WRITABLE; - rb = rb_alloc(nr_pages, + rb = rb_alloc(data_size, page_shift, event->attr.watermark ? event->attr.wakeup_watermark : 0, event->cpu, rb_flags); @@ -7350,9 +7376,10 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, return 0; } -static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, - unsigned long nr_pages) +static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event) { + unsigned long size = vma->vm_end - vma->vm_start; + unsigned long nr_pages = size >> PAGE_SHIFT; long extra = 0, user_extra = nr_pages; u64 aux_offset, aux_size; struct perf_buffer *rb; @@ -7361,6 +7388,9 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, rb = event->rb; if (!rb) return -EINVAL; + /* PMU setup_aux() consumes whole native pages, independent of mmap layout. */ + if (!IS_ALIGNED(size, PAGE_SIZE)) + return -EINVAL; guard(mutex)(&rb->aux_mutex); @@ -7372,14 +7402,14 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, aux_offset = READ_ONCE(rb->user_page->aux_offset); aux_size = READ_ONCE(rb->user_page->aux_size); - if (aux_offset < perf_data_size(rb) + PAGE_SIZE) + if (aux_offset < perf_data_size(rb) + perf_metadata_size(rb)) return -EINVAL; - if (aux_offset != vma->vm_pgoff << PAGE_SHIFT) + if (aux_offset != perf_mmap_offset(vma)) return -EINVAL; /* already mapped with a different offset */ - if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff) + if (rb_has_aux(rb) && rb->aux_offset != aux_offset) return -EINVAL; if (aux_size != nr_pages * PAGE_SIZE) @@ -7409,7 +7439,7 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, if (vma->vm_flags & VM_WRITE) rb_flags |= RING_BUFFER_WRITABLE; - ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, + ret = rb_alloc_aux(rb, event, aux_offset, nr_pages, event->attr.aux_watermark, rb_flags); if (ret) { refcount_dec(&rb->mmap_count); @@ -7449,12 +7479,13 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) return ret; vma_size = vma->vm_end - vma->vm_start; - nr_pages = vma_size / PAGE_SIZE; + nr_pages = vma_size >> mm_page_shift(vma->vm_mm); if (nr_pages > INT_MAX) return -ENOMEM; - if (vma_size != PAGE_SIZE * nr_pages) + if (vma_size != mm_page_size(vma->vm_mm) * nr_pages || + mm_page_shift(vma->vm_mm) > PAGE_SHIFT) return -EINVAL; scoped_guard (mutex, &event->mmap_mutex) { @@ -7466,10 +7497,14 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) if (event->state <= PERF_EVENT_STATE_REVOKED) return -ENODEV; - if (vma->vm_pgoff == 0) + /* One ring has one immutable metadata/data boundary for all aliases. */ + if (event->rb && event->rb->mmap_page_shift != mm_page_shift(vma->vm_mm)) + return -EINVAL; + + if (!perf_mmap_offset(vma)) ret = perf_mmap_rb(vma, event, nr_pages); else - ret = perf_mmap_aux(vma, event, nr_pages); + ret = perf_mmap_aux(vma, event); if (ret) return ret; diff --git a/kernel/events/internal.h b/kernel/events/internal.h index c03c4f2eea571..5238cfa3399b1 100644 --- a/kernel/events/internal.h +++ b/kernel/events/internal.h @@ -15,9 +15,10 @@ struct perf_buffer { struct rcu_head rcu_head; #ifdef CONFIG_PERF_USE_VMALLOC struct work_struct work; - int page_order; /* allocation order */ #endif int nr_pages; /* nr of data pages */ + unsigned int data_page_shift; /* bytes per output chunk */ + unsigned int mmap_page_shift; /* userspace metadata size */ int overwrite; /* can overwrite itself */ int paused; /* can write into ring buffer */ @@ -44,7 +45,7 @@ struct perf_buffer { long aux_head; unsigned int aux_nest; long aux_wakeup; /* last aux_watermark boundary crossed by aux_head */ - unsigned long aux_pgoff; + u64 aux_offset; int aux_nr_pages; int aux_overwrite; refcount_t aux_mmap_count; @@ -80,10 +81,11 @@ static inline void rb_toggle_paused(struct perf_buffer *rb, bool pause) } extern struct perf_buffer * -rb_alloc(int nr_pages, long watermark, int cpu, int flags); +rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags); extern void perf_event_wakeup(struct perf_event *event); extern int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, - pgoff_t pgoff, int nr_pages, long watermark, int flags); + u64 offset, int nr_pages, long watermark, int flags); extern void rb_free_aux(struct perf_buffer *rb); extern struct perf_buffer *ring_buffer_get(struct perf_event *event); extern void ring_buffer_put(struct perf_buffer *rb); @@ -99,34 +101,19 @@ void perf_event_aux_event(struct perf_event *event, unsigned long head, extern struct page * perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff); -#ifdef CONFIG_PERF_USE_VMALLOC -/* - * Back perf_mmap() with vmalloc memory. - * - * Required for architectures that have d-cache aliasing issues. - */ - -static inline int page_order(struct perf_buffer *rb) -{ - return rb->page_order; -} - -#else - -static inline int page_order(struct perf_buffer *rb) +static inline unsigned long perf_data_size(struct perf_buffer *rb) { - return 0; + return (unsigned long)rb->nr_pages << rb->data_page_shift; } -#endif static inline int data_page_nr(struct perf_buffer *rb) { - return rb->nr_pages << page_order(rb); + return DIV_ROUND_UP(perf_data_size(rb), PAGE_SIZE); } -static inline unsigned long perf_data_size(struct perf_buffer *rb) +static inline unsigned long perf_metadata_size(struct perf_buffer *rb) { - return rb->nr_pages << (PAGE_SHIFT + page_order(rb)); + return 1UL << rb->mmap_page_shift; } static inline unsigned long perf_aux_size(struct perf_buffer *rb) @@ -154,7 +141,7 @@ static inline unsigned long perf_aux_size(struct perf_buffer *rb) handle->page++; \ handle->page &= rb->nr_pages - 1; \ handle->addr = rb->data_pages[handle->page]; \ - handle->size = PAGE_SIZE << page_order(rb); \ + handle->size = 1UL << rb->data_page_shift; \ } \ } while (len && written == size); \ \ diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 9fe92161715e0..6c5de8fba83f2 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -238,7 +238,7 @@ __perf_output_begin(struct perf_output_handle *handle, if (unlikely(head - local_read(&rb->wakeup) > rb->watermark)) local_add(rb->watermark, &rb->wakeup); - page_shift = PAGE_SHIFT + page_order(rb); + page_shift = rb->data_page_shift; handle->page = (offset >> page_shift) & (rb->nr_pages - 1); offset &= (1UL << page_shift) - 1; @@ -677,7 +677,7 @@ static void __rb_free_aux(struct perf_buffer *rb) } int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, - pgoff_t pgoff, int nr_pages, long watermark, int flags) + u64 offset, int nr_pages, long watermark, int flags) { bool overwrite = !(flags & RING_BUFFER_WRITABLE); int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu); @@ -784,7 +784,7 @@ int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, out: if (!ret) - rb->aux_pgoff = pgoff; + rb->aux_offset = offset; else __rb_free_aux(rb); @@ -803,8 +803,8 @@ void rb_free_aux(struct perf_buffer *rb) * Back perf_mmap() with regular GFP_KERNEL-0 pages. */ -static struct page * -__perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) +struct page * +perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) { if (pgoff > rb->nr_pages) return NULL; @@ -835,10 +835,12 @@ static void perf_mmap_free_page(void *addr) __free_page(page); } -struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) +struct perf_buffer *rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags) { struct perf_buffer *rb; unsigned long size; + int nr_pages = DIV_ROUND_UP(data_size, PAGE_SIZE); int i, node; size = sizeof(struct perf_buffer); @@ -863,6 +865,10 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) } rb->nr_pages = nr_pages; + rb->mmap_page_shift = mmap_page_shift; + /* A ring smaller than a native page still wraps at its requested size. */ + rb->data_page_shift = data_size && data_size < PAGE_SIZE ? + ilog2(data_size) : PAGE_SHIFT; ring_buffer_init(rb, watermark, flags); @@ -892,8 +898,8 @@ void rb_free(struct perf_buffer *rb) } #else -static struct page * -__perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) +struct page * +perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) { /* The '>' counts in the user page. */ if (pgoff > data_page_nr(rb)) @@ -917,10 +923,12 @@ void rb_free(struct perf_buffer *rb) schedule_work(&rb->work); } -struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) +struct perf_buffer *rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags) { struct perf_buffer *rb; unsigned long size; + int nr_pages = DIV_ROUND_UP(data_size, PAGE_SIZE); void *all_buf; int node; @@ -940,9 +948,10 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) rb->user_page = all_buf; rb->data_pages[0] = all_buf + PAGE_SIZE; + rb->mmap_page_shift = mmap_page_shift; if (nr_pages) { rb->nr_pages = 1; - rb->page_order = ilog2(nr_pages); + rb->data_page_shift = ilog2(data_size); } ring_buffer_init(rb, watermark, flags); @@ -957,21 +966,3 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) } #endif - -struct page * -perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) -{ - if (rb->aux_nr_pages) { - /* above AUX space */ - if (pgoff > rb->aux_pgoff + rb->aux_nr_pages) - return NULL; - - /* AUX space */ - if (pgoff >= rb->aux_pgoff) { - int aux_pgoff = array_index_nospec(pgoff - rb->aux_pgoff, rb->aux_nr_pages); - return virt_to_page(rb->aux_pages[aux_pgoff]); - } - } - - return __perf_mmap_to_page(rb, pgoff); -} diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 6300b216012cb..640991b1c25cb 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -30,6 +30,7 @@ #include #include /* check_stable_address_space */ #include +#include #include @@ -144,12 +145,14 @@ static bool valid_vma(struct vm_area_struct *vma, bool is_register) static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset) { - return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT) - + vma_subpage_offset(vma); } static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) { - return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start); + return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma) + + (vaddr - vma->vm_start); } /** @@ -191,7 +194,8 @@ static void copy_to_page(struct page *page, unsigned long vaddr, const void *src kunmap_local(kaddr); } -static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *insn, +static int verify_opcode(struct page *page, unsigned long vaddr, unsigned int page_offset, + uprobe_opcode_t *insn, int nbytes, void *data) { uprobe_opcode_t old_opcode; @@ -206,7 +210,7 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t * is a trap variant; uprobes always wins over any other (gdb) * breakpoint. */ - uprobe_copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE); + uprobe_copy_from_page(page, page_offset, &old_opcode, UPROBE_SWBP_INSN_SIZE); is_swbp = is_swbp_insn(&old_opcode); if (is_swbp_insn(insn)) { @@ -307,14 +311,24 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) { void *kaddr; struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *vma; + bool alternative = mm_page_size(mm) != PAGE_SIZE; +#endif int ret; short *ptr; if (!vaddr || !d) return -EINVAL; - ret = get_user_pages_remote(mm, vaddr, 1, - FOLL_WRITE, &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + ret = get_user_fragment_vma_remote(mm, vaddr, sizeof(short), + FOLL_WRITE, &fragment, &vma); + else +#endif + ret = get_user_pages_remote(mm, vaddr, 1, FOLL_WRITE, &page, NULL); if (unlikely(ret <= 0)) { /* * We are asking for 1 page. If get_user_pages_remote() fails, @@ -323,8 +337,16 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) return ret == 0 ? -EBUSY : ret; } - kaddr = kmap_local_page(page); - ptr = kaddr + (vaddr & ~PAGE_MASK); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) { + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + ptr = kaddr; + } else +#endif + { + kaddr = kmap_local_page(page); + ptr = kaddr + (vaddr & ~PAGE_MASK); + } if (unlikely(*ptr + d < 0)) { pr_warn("ref_ctr going negative. vaddr: 0x%lx, " @@ -337,7 +359,12 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) ret = 0; out: kunmap_local(kaddr); - put_page(page); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + release_user_fragments(&fragment, 1, !ret); + else +#endif + put_page(page); return ret; } @@ -381,7 +408,8 @@ static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm, } static bool orig_page_is_identical(struct vm_area_struct *vma, - unsigned long vaddr, struct page *page, bool *pmd_mappable) + unsigned long vaddr, struct page *page, unsigned int page_offset, + bool *pmd_mappable) { const pgoff_t index = vaddr_to_offset(vma, vaddr) >> PAGE_SHIFT; struct folio *orig_folio = filemap_get_folio(vma->vm_file->f_mapping, @@ -393,19 +421,30 @@ static bool orig_page_is_identical(struct vm_area_struct *vma, return false; orig_page = folio_file_page(orig_folio, index); - *pmd_mappable = folio_test_pmd_mappable(orig_folio); - identical = folio_test_uptodate(orig_folio) && - pages_identical(page, orig_page); + *pmd_mappable = mm_page_size(vma->vm_mm) == PAGE_SIZE && + folio_test_pmd_mappable(orig_folio); + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + unsigned int offset = vma_page_offset_at(vma, vaddr).offset; + void *a = kmap_local_page(page), *b = kmap_local_page(orig_page); + + identical = folio_test_uptodate(orig_folio) && + !memcmp(a + page_offset, b + offset, mm_page_size(vma->vm_mm)); + kunmap_local(b); + kunmap_local(a); + } else { + identical = folio_test_uptodate(orig_folio) && + pages_identical(page, orig_page); + } folio_put(orig_folio); return identical; } static int __uprobe_write(struct vm_area_struct *vma, - struct folio_walk *fw, struct folio *folio, - unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, + struct folio_walk *fw, struct folio *folio, struct mm_subpage *slot, + unsigned int page_offset, unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, bool is_register) { - const unsigned long vaddr = insn_vaddr & PAGE_MASK; + const unsigned long vaddr = insn_vaddr & mm_page_mask(vma->vm_mm); bool pmd_mappable; /* For now, we'll only handle PTE-mapped folios. */ @@ -417,6 +456,12 @@ static int __uprobe_write(struct vm_area_struct *vma, * but the VMA might not be writable. */ if (!pte_write(fw->pte)) { +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + if (!mm_subpage_is_exclusive(slot)) + return -EFAULT; + } else +#endif if (!PageAnonExclusive(fw->page)) return -EFAULT; if (unlikely(userfaultfd_pte_wp(vma, fw->pte))) @@ -430,31 +475,49 @@ static int __uprobe_write(struct vm_area_struct *vma, */ flush_cache_page(vma, vaddr, pte_pfn(fw->pte)); fw->pte = ptep_clear_flush(vma, vaddr, fw->ptep); - copy_to_page(fw->page, insn_vaddr, insn, nbytes); + copy_to_page(fw->page, page_offset, insn, nbytes); /* * When unregistering, we may only zap a PTE if uffd is disabled and * there are no unexpected folio references ... */ - if (is_register || userfaultfd_missing(vma) || - (folio_ref_count(folio) != folio_expected_ref_count(folio) + 1)) + if (is_register || userfaultfd_missing(vma)) + goto remap; +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + if (!mm_subpage_can_zap(slot)) + goto remap; + } else +#endif + if (folio_ref_count(folio) != folio_expected_ref_count(folio) + 1) goto remap; /* * ... and the mapped page is identical to the original page that * would get faulted in on next access. */ - if (!orig_page_is_identical(vma, vaddr, fw->page, &pmd_mappable)) + if (!orig_page_is_identical(vma, vaddr, fw->page, + page_offset - (insn_vaddr - vaddr), &pmd_mappable)) goto remap; dec_mm_counter(vma->vm_mm, MM_ANONPAGES); - folio_remove_rmap_pte(folio, fw->page, vma); +#ifdef CONFIG_MM_SUBPAGE + if (slot) + mm_subpage_remove_anon_rmap(slot, vma); + else +#endif + folio_remove_rmap_pte(folio, fw->page, vma); if (!folio_mapped(folio) && folio_test_swapcache(folio) && folio_trylock(folio)) { folio_free_swap(folio); folio_unlock(folio); } - folio_put(folio); +#ifdef CONFIG_MM_SUBPAGE + if (slot) + mm_subpage_put(slot); /* Retired PTE reference; caller still has its GET. */ + else +#endif + folio_put(folio); return pmd_mappable; remap: @@ -494,13 +557,24 @@ int uprobe_write_opcode(struct arch_uprobe *auprobe, struct vm_area_struct *vma, verify_opcode, is_register, true /* do_update_ref_ctr */, NULL); } +static void uprobe_put_folio(struct folio *folio, struct mm_subpage *slot) +{ +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + mm_subpage_put(slot); + return; + } +#endif + folio_put(folio); +} + int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, const unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, uprobe_write_verify_t verify, bool is_register, bool do_update_ref_ctr, void *data) { - const unsigned long vaddr = insn_vaddr & PAGE_MASK; struct mm_struct *mm = vma->vm_mm; + const unsigned long vaddr = insn_vaddr & mm_page_mask(mm); struct uprobe *uprobe; int ret, ref_ctr_updated = 0; unsigned int gup_flags = FOLL_FORCE; @@ -508,7 +582,16 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, struct folio_walk fw; struct folio *folio; struct page *page; - + struct mm_subpage *slot = NULL; + unsigned int page_offset; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *found; +#endif + + /* Larger-than-native public mms are not enabled by the mixed-page ABI yet. */ + if (mm_page_size(mm) > PAGE_SIZE) + return -EOPNOTSUPP; uprobe = container_of(auprobe, struct uprobe, arch); if (WARN_ON_ONCE(!is_cow_mapping(vma->vm_flags))) @@ -526,14 +609,29 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, gup_flags |= FOLL_WRITE | FOLL_SPLIT_PMD; retry: - ret = get_user_pages_remote(mm, vaddr, 1, gup_flags, &page, NULL); - if (ret <= 0) - goto out; - folio = page_folio(page); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + ret = get_user_fragment_vma_remote(mm, vaddr, mm_page_size(mm), + gup_flags & ~FOLL_SPLIT_PMD, &fragment, &found); + if (ret <= 0) + goto out; + folio = fragment.folio; + slot = fragment.subpage; + page = folio_page(folio, fragment.offset >> PAGE_SHIFT); + page_offset = offset_in_page(fragment.offset) + insn_vaddr - vaddr; + } else +#endif + { + ret = get_user_pages_remote(mm, vaddr, 1, gup_flags, &page, NULL); + if (ret <= 0) + goto out; + folio = page_folio(page); + page_offset = offset_in_page(insn_vaddr); + } - ret = verify(page, insn_vaddr, insn, nbytes, data); + ret = verify(page, insn_vaddr, page_offset, insn, nbytes, data); if (ret <= 0) { - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -541,7 +639,7 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, if (do_update_ref_ctr && !ref_ctr_updated && uprobe->ref_ctr_offset) { ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1); if (ret) { - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -551,7 +649,7 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, ret = 0; if (unlikely(!folio_test_anon(folio) || folio_is_zone_device(folio))) { VM_WARN_ON_ONCE(is_register); - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -562,22 +660,29 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, * be able to do it under PTL. */ mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, - vaddr, vaddr + PAGE_SIZE); + vaddr, vaddr + mm_page_size(mm)); mmu_notifier_invalidate_range_start(&range); } ret = -EAGAIN; /* Walk the page tables again, to perform the actual update. */ if (folio_walk_start(&fw, vma, vaddr, 0)) { - if (fw.page == page) - ret = __uprobe_write(vma, &fw, folio, insn_vaddr, insn, nbytes, is_register); + bool same = fw.page == page; +#ifdef CONFIG_MM_SUBPAGE + if (slot) + same &= fw.level == FW_LEVEL_PTE && + pte_phys_mm(mm, fw.pte) == mm_subpage_phys(slot); +#endif + if (same) + ret = __uprobe_write(vma, &fw, folio, slot, page_offset, + insn_vaddr, insn, nbytes, is_register); folio_walk_end(&fw, vma); } if (!is_register) mmu_notifier_invalidate_range_end(&range); - folio_put(folio); + uprobe_put_folio(folio, slot); switch (ret) { case -EFAULT: gup_flags |= FOLL_WRITE | FOLL_SPLIT_PMD; @@ -1211,7 +1316,9 @@ build_map_info(struct address_space *mapping, loff_t offset, bool is_register) again: i_mmap_lock_read(mapping); vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { - if (!valid_vma(vma, is_register)) + if (!valid_vma(vma, is_register) || + offset < vaddr_to_offset(vma, vma->vm_start) || + offset >= vaddr_to_offset(vma, vma->vm_end)) continue; if (!prev && !more) { @@ -1482,7 +1589,7 @@ static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) file_inode(vma->vm_file) != uprobe->inode) continue; - offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; + offset = vaddr_to_offset(vma, vma->vm_start); if (uprobe->offset < offset || uprobe->offset >= offset + vma->vm_end - vma->vm_start) continue; @@ -1696,8 +1803,14 @@ static const struct vm_special_mapping xol_mapping = { unsigned long __weak arch_uprobe_get_xol_area(void) { - /* Try to map as high as possible, this is only a hint. */ - return get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0); + unsigned long size = PAGE_SIZE; + unsigned long addr; + + /* XOL arch copy hooks use native offsets into the one backing page. */ + if (mm_page_size(current->mm) < PAGE_SIZE) + size += PAGE_SIZE - mm_page_size(current->mm); + addr = get_unmapped_area(NULL, TASK_SIZE - size, size, 0, 0); + return IS_ERR_VALUE(addr) ? addr : ALIGN(addr, PAGE_SIZE); } /* Slot allocation for XOL */ @@ -2413,6 +2526,23 @@ static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr) if (likely(result == 0)) goto out; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + struct user_page_fragment fragment; + struct vm_area_struct *vma; + void *kaddr; + + result = get_user_fragment_vma_remote(mm, vaddr, UPROBE_SWBP_INSN_SIZE, + FOLL_FORCE, &fragment, &vma); + if (result != 1) + return result < 0 ? result : -EFAULT; + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + memcpy(&opcode, kaddr, UPROBE_SWBP_INSN_SIZE); + kunmap_local(kaddr); + release_user_fragments(&fragment, 1, false); + goto out; + } +#endif result = get_user_pages(vaddr, 1, FOLL_FORCE, &page); if (result < 0) return result; @@ -2453,7 +2583,7 @@ static struct uprobe *find_active_uprobe_speculative(unsigned long bp_vaddr) if (!vm_file) return NULL; - offset = (loff_t)(vma->vm_pgoff << PAGE_SHIFT) + (bp_vaddr - vma->vm_start); + offset = vaddr_to_offset(vma, bp_vaddr); uprobe = find_uprobe_rcu(vm_file->f_inode, offset); if (!uprobe) return NULL; diff --git a/kernel/fork.c b/kernel/fork.c index 5a4623b058a20..88788baaff964 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -112,7 +112,7 @@ #include #include -#include +#include #include #include @@ -639,6 +639,12 @@ static void check_mm(struct mm_struct *mm) } } +#ifdef CONFIG_MM_SUBPAGE + if (mm->committed_user_pages) + pr_alert("BUG: non-zero user commitment on freeing mm: %lu\n", + mm->committed_user_pages); +#endif + if (mm_pgtables_bytes(mm)) pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", mm_pgtables_bytes(mm)); @@ -1089,6 +1095,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, INIT_LIST_HEAD(&mm->mmlist); mm_pgtables_bytes_init(mm); mm->map_count = 0; +#ifdef CONFIG_MM_SUBPAGE + mm->committed_user_pages = 0; +#endif mm->locked_vm = 0; atomic64_set(&mm->pinned_vm, 0); memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); @@ -1159,7 +1168,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, /* * Allocate and initialize an mm_struct. */ -struct mm_struct *mm_alloc(void) +static struct mm_struct *__mm_alloc(bool for_exec, unsigned int page_shift) { struct mm_struct *mm; @@ -1168,10 +1177,32 @@ struct mm_struct *mm_alloc(void) return NULL; memset(mm, 0, sizeof(*mm)); + if (for_exec) + arch_mm_init_exec(mm, page_shift); return mm_init(mm, current, current_user_ns()); } + +struct mm_struct *mm_alloc(void) +{ + return __mm_alloc(false, 0); +} EXPORT_SYMBOL_IF_KUNIT(mm_alloc); +struct mm_struct *mm_alloc_exec(void) +{ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + return __mm_alloc(true, current->exec_page_shift); +#else + return __mm_alloc(true, 0); +#endif +} + +/* The caller has validated the requested exec granule. */ +struct mm_struct *mm_alloc_exec_page_shift(unsigned int page_shift) +{ + return __mm_alloc(true, page_shift); +} + static inline void __mmput(struct mm_struct *mm) { VM_BUG_ON(atomic_read(&mm->mm_users)); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 7ffe6dc79b28e..771c12983cd2a 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include "futex.h" @@ -512,6 +513,44 @@ static u64 get_inode_sequence_number(struct inode *inode) } } +/* Keep a typed slot reference alive until the mapping identity is sampled. */ +struct futex_page_ref { + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; +#endif +}; + +static int futex_get_page_ref(struct mm_struct *mm, unsigned long address, + unsigned long uaddr, unsigned int size, unsigned int flags, + struct futex_page_ref *ref) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + int ret = get_user_fragments_remote(mm, uaddr, size, flags, + &ref->fragment, 1); + + if (ret != 1) + return ret ?: -EFAULT; + ref->page = folio_page(ref->fragment.folio, + ref->fragment.offset >> PAGE_SHIFT); + return 1; + } +#endif + return get_user_pages_fast(address, 1, flags, &ref->page); +} + +static void futex_put_page_ref(struct futex_page_ref *ref) +{ +#ifdef CONFIG_MM_SUBPAGE + if (ref->fragment.folio) { + release_user_fragments(&ref->fragment, 1, false); + return; + } +#endif + put_page(ref->page); +} + /** * get_futex_key() - Get parameters which are the keys for a futex * @uaddr: virtual address of the futex @@ -545,6 +584,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, unsigned long address = (unsigned long)uaddr; struct mm_struct *mm = current->mm; struct page *page; + struct futex_page_ref ref = {}; struct folio *folio; struct address_space *mapping; int node, err, size, ro = 0; @@ -629,13 +669,13 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, if (unlikely(should_fail_futex(true))) return -EFAULT; - err = get_user_pages_fast(address, 1, FOLL_WRITE, &page); + err = futex_get_page_ref(mm, address, (unsigned long)uaddr, size, FOLL_WRITE, &ref); /* * If write access is not required (eg. FUTEX_WAIT), try * and get read-only access. */ if (err == -EFAULT && rw == FUTEX_READ) { - err = get_user_pages_fast(address, 1, 0, &page); + err = futex_get_page_ref(mm, address, (unsigned long)uaddr, size, 0, &ref); ro = 1; } if (err < 0) @@ -660,6 +700,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, * filesystem-backed pages, the precise page is required as the * index of the page determines the key. */ + page = ref.page; folio = page_folio(page); mapping = READ_ONCE(folio->mapping); @@ -689,7 +730,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, folio_lock(folio); shmem_swizzled = folio_test_swapcache(folio) || folio->mapping; folio_unlock(folio); - folio_put(folio); + futex_put_page_ref(&ref); if (shmem_swizzled) goto again; @@ -739,7 +780,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, if (READ_ONCE(folio->mapping) != mapping) { rcu_read_unlock(); - folio_put(folio); + futex_put_page_ref(&ref); goto again; } @@ -747,11 +788,15 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, inode = READ_ONCE(mapping->host); if (!inode) { rcu_read_unlock(); - folio_put(folio); + futex_put_page_ref(&ref); goto again; } +#ifdef CONFIG_MM_SUBPAGE + if (ref.fragment.folio) + key->both.offset = offset_in_page(ref.fragment.offset); +#endif key->both.offset |= FUT_OFF_INODE; /* inode-based key */ key->shared.i_seq = get_inode_sequence_number(inode); key->shared.pgoff = page_pgoff(folio, page); @@ -759,7 +804,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, } out: - folio_put(folio); + futex_put_page_ref(&ref); return err; } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index abb76775aa728..e06b05a08f7f4 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1694,7 +1694,7 @@ static unsigned int task_nr_scan_windows(struct task_struct *p) * on resident pages */ nr_scan_pages = MB_TO_PAGES(sysctl_numa_balancing_scan_size); - rss = get_mm_rss(p->mm); + rss = mm_pages_to_native(p->mm, get_mm_rss(p->mm)); if (!rss) rss = nr_scan_pages; @@ -3682,8 +3682,8 @@ static void task_numa_work(struct callback_head *work) * areas faster. */ if (nr_pte_updates) - pages -= (end - start) >> PAGE_SHIFT; - virtpages -= (end - start) >> PAGE_SHIFT; + pages -= DIV_ROUND_UP(end - start, PAGE_SIZE); + virtpages -= DIV_ROUND_UP(end - start, PAGE_SIZE); start = end; if (pages <= 0 || virtpages <= 0) diff --git a/kernel/sys.c b/kernel/sys.c index ec432b89e8540..09702e8f773ba 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -1927,7 +1927,7 @@ void getrusage(struct task_struct *p, int who, struct rusage *r) } out_children: - r->ru_maxrss = maxrss * (PAGE_SIZE / 1024); /* convert pages to KBs */ + r->ru_maxrss = maxrss; /* signal and mm high-water maxima are in kB */ r->ru_utime = ns_to_kernel_old_timeval(utime); r->ru_stime = ns_to_kernel_old_timeval(stime); } @@ -2532,6 +2532,40 @@ static int prctl_set_thp_disable(bool thp_disable, unsigned long flags, return 0; } +static int prctl_page_size_get(struct task_struct *task, bool persistent) +{ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + unsigned int shift = persistent ? task->default_exec_page_shift : + task->exec_page_shift; + + return shift ? 1U << shift : 0; +#else + return -EINVAL; +#endif +} + +static int prctl_page_size_set(struct task_struct *task, unsigned long size, + bool persistent) +{ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + int shift = arch_exec_page_size_shift(size); + + if (shift < 0) + return shift; + if (persistent) { + task->default_exec_page_shift = shift; + } else { + /* A native request overrides an inherited default for one exec. */ + if (!shift && task->default_exec_page_shift) + shift = PAGE_SHIFT; + task->exec_page_shift = shift; + } + return 0; +#else + return -EINVAL; +#endif +} + int __weak arch_prctl_mem_model_get(struct task_struct *t) { return -EINVAL; @@ -2555,6 +2589,22 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = 0; switch (option) { + case PR_AURORA_GET_EXEC_PAGE_SIZE: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_get(me, false); + case PR_AURORA_SET_EXEC_PAGE_SIZE: + if (arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_set(me, arg2, false); + case PR_AURORA_GET_DEFAULT_PAGE_SIZE: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_get(me, true); + case PR_AURORA_SET_DEFAULT_PAGE_SIZE: + if (arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_set(me, arg2, true); case PR_GET_MEM_MODEL: if (arg2 || arg3 || arg4 || arg5) return -EINVAL; diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 8c82ecb735f41..e8b622a8178cb 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include "trace_dynevent.h" @@ -554,6 +555,11 @@ static int user_event_enabler_write(struct user_event_mm *mm, unsigned long uaddr = enabler->addr; unsigned long *ptr; struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *vma; + bool alternative = mm_page_size(mm->mm) != PAGE_SIZE; +#endif void *kaddr; int bit = ENABLE_BIT(enabler); int ret; @@ -573,8 +579,14 @@ static int user_event_enabler_write(struct user_event_mm *mm, align_addr_bit(&uaddr, &bit, ENABLE_BITOPS(enabler)); - ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT, - &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + ret = pin_user_fragment_vma_remote(mm->mm, uaddr, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &vma); + else +#endif + ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT, + &page, NULL); if (unlikely(ret <= 0)) { if (!fixup_fault) @@ -586,8 +598,17 @@ static int user_event_enabler_write(struct user_event_mm *mm, return -EFAULT; } - kaddr = kmap_local_page(page); - ptr = kaddr + (uaddr & ~PAGE_MASK); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) { + /* align_addr_bit() keeps the complete atomic word in this fragment. */ + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + ptr = kaddr; + } else +#endif + { + kaddr = kmap_local_page(page); + ptr = kaddr + (uaddr & ~PAGE_MASK); + } /* Update bit atomically, user tracers must be atomic as well */ if (enabler->event && enabler->event->status) @@ -596,7 +617,12 @@ static int user_event_enabler_write(struct user_event_mm *mm, clear_bit(bit, ptr); kunmap_local(kaddr); - unpin_user_pages_dirty_lock(&page, 1, true); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + release_user_fragments(&fragment, 1, true); + else +#endif + unpin_user_pages_dirty_lock(&page, 1, true); return 0; } diff --git a/kernel/tsacct.c b/kernel/tsacct.c index 5c153106e6427..5f59d071ab565 100644 --- a/kernel/tsacct.c +++ b/kernel/tsacct.c @@ -91,16 +91,16 @@ void xacct_add_tsk(struct taskstats *stats, struct task_struct *p) { struct mm_struct *mm; - /* convert pages-nsec/1024 to Mbyte-usec, see __acct_update_integrals */ - stats->coremem = p->acct_rss_mem1 * PAGE_SIZE; + /* convert account-units-nsec/1024 to Mbyte-usec, see __acct_update_integrals */ + stats->coremem = p->acct_rss_mem1 * (1UL << MM_ACCOUNT_SHIFT); do_div(stats->coremem, 1000 * KB); - stats->virtmem = p->acct_vm_mem1 * PAGE_SIZE; + stats->virtmem = p->acct_vm_mem1 * (1UL << MM_ACCOUNT_SHIFT); do_div(stats->virtmem, 1000 * KB); mm = get_task_mm(p); if (mm) { /* adjust to KB unit */ - stats->hiwater_rss = get_mm_hiwater_rss(mm) * PAGE_SIZE / KB; - stats->hiwater_vm = get_mm_hiwater_vm(mm) * PAGE_SIZE / KB; + stats->hiwater_rss = mm_pages_to_kb(mm, get_mm_hiwater_rss(mm)); + stats->hiwater_vm = mm_pages_to_kb(mm, get_mm_hiwater_vm(mm)); mmput(mm); } stats->read_char = p->ioac.rchar & KB_MASK; @@ -140,8 +140,10 @@ static void __acct_update_integrals(struct task_struct *tsk, * The final unit reported to userspace is Mbyte-usecs, * the rest of the math is done in xacct_add_tsk. */ - tsk->acct_rss_mem1 += delta * get_mm_rss(tsk->mm) >> 10; - tsk->acct_vm_mem1 += delta * READ_ONCE(tsk->mm->total_vm) >> 10; + tsk->acct_rss_mem1 += delta * mm_pages_to_shift(tsk->mm, get_mm_rss(tsk->mm), + MM_ACCOUNT_SHIFT) >> 10; + tsk->acct_vm_mem1 += delta * mm_pages_to_shift(tsk->mm, READ_ONCE(tsk->mm->total_vm), + MM_ACCOUNT_SHIFT) >> 10; } /** diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c index 17d37b82ebc68..d808ddcb94b01 100644 --- a/lib/vdso/datastore.c +++ b/lib/vdso/datastore.c @@ -83,7 +83,8 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, vm_fault_t err; addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE; - err = vmf_insert_page(vma, addr, page); + err = vmf_insert_page_offset(vma, addr, page, + vma_page_offset_at(vma, vmf->address).offset); if (unlikely(err & VM_FAULT_ERROR)) return err; page = timens_page; diff --git a/mm/Kconfig b/mm/Kconfig index e8bf1e9e6ad90..b743b2290b309 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1164,6 +1164,28 @@ config IDLE_PAGE_TRACKING See Documentation/admin-guide/mm/idle_page_tracking.rst for more details. +config ARCH_HAS_MM_PGTABLE + bool + help + Architecture supplies target-mm page-table geometry and accessors. + This does not change the native physical allocation granule. + +config ARCH_HAS_USER_PAGE_SIZE + bool + help + Architecture supports selecting the user mapping granule at exec. + It supplies arch_exec_page_size_shift() to validate requests, and + integrates the selected granule into mm allocation and binary loading. + +config MM_SUBPAGE + bool + select PAGE_EXTENSION + select XARRAY_MULTI + help + Reference-counted user data slots smaller than a native folio. + Architectures must integrate rmap, faults, reclaim and pins before + exposing these backing primitives to userspace. + # Architectures which implement cpu_dcache_is_aliasing() to query # whether the data caches are aliased (VIVT or VIPT with dcache # aliasing) need to select this. @@ -1439,9 +1461,15 @@ config ARCH_HAS_USER_SHADOW_STACK config HAVE_ARCH_TLB_REMOVE_TABLE def_bool n +config ARCH_SUPPORTS_PT_RECLAIM + bool + help + The architecture's custom table-free callback supports PTE reclamation. + config PT_RECLAIM def_bool y - depends on MMU_GATHER_RCU_TABLE_FREE && !HAVE_ARCH_TLB_REMOVE_TABLE + depends on MMU_GATHER_RCU_TABLE_FREE + depends on !HAVE_ARCH_TLB_REMOVE_TABLE || ARCH_SUPPORTS_PT_RECLAIM help Try to reclaim empty user page table pages in paths other than munmap and exit_mmap path. diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index cca7014ab5f13..3d618828c2c8d 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -155,9 +155,10 @@ int damon_cold_score(struct damon_ctx *c, struct damon_region *r, } static bool damon_folio_mkold_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + unsigned long addr = rmap.address; + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); while (page_vma_mapped_walk(&pvmw)) { addr = pvmw.address; @@ -190,10 +191,11 @@ void damon_folio_mkold(struct folio *folio) } static bool damon_folio_young_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long addr = rmap.address; bool *accessed = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); pte_t pte; *accessed = false; diff --git a/mm/filemap.c b/mm/filemap.c index bbd400d893aa5..14e2656b6a9cc 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -225,6 +225,7 @@ void __filemap_remove_folio(struct folio *folio, void *shadow) trace_mm_filemap_delete_from_page_cache(folio); filemap_unaccount_folio(mapping, folio); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); page_cache_delete(mapping, folio, shadow); } @@ -308,6 +309,7 @@ static void page_cache_delete_batch(struct address_space *mapping, WARN_ON_ONCE(!folio_test_locked(folio)); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); folio->mapping = NULL; /* Leave folio->index set: truncation lookup relies on it */ @@ -827,6 +829,7 @@ void replace_page_cache_folio(struct folio *old, struct folio *new) mem_cgroup_replace_folio(old, new); xas_lock_irq(&xas); + shmem_uffd_migrate(new, old); xas_store(&xas, new); old->mapping = NULL; @@ -3387,7 +3390,7 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) */ struct vm_area_struct *vma = vmf->vma; unsigned long start = vma->vm_pgoff; - unsigned long end = start + vma_pages(vma); + unsigned long end = vma_last_pgoff(vma) + 1; unsigned long ra_end; ra->order = exec_folio_order(); @@ -3529,7 +3532,12 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) bool mapping_locked = false; max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - if (unlikely(index >= max_idx)) + if (unlikely(index >= max_idx +#ifdef CONFIG_MM_SUBPAGE + || (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) +#endif + )) return VM_FAULT_SIGBUS; trace_mm_filemap_fault(mapping, index); @@ -3633,7 +3641,12 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) * We must recheck i_size under page lock. */ max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - if (unlikely(index >= max_idx)) { + if (unlikely(index >= max_idx +#ifdef CONFIG_MM_SUBPAGE + || (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) +#endif + )) { folio_unlock(folio); folio_put(folio); return VM_FAULT_SIGBUS; @@ -3783,6 +3796,10 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, } do { + if (userfaultfd_missing(vmf->vma) && + shmem_uffd_range_missing(folio, folio->index + + folio_page_idx(folio, page + count), 0, PAGE_SIZE)) + goto skip; if (PageHWPoison(page + count)) goto skip; @@ -3847,6 +3864,9 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, vm_fault_t ret = 0; struct page *page = &folio->page; + if (userfaultfd_missing(vmf->vma) && + shmem_uffd_range_missing(folio, folio->index, 0, PAGE_SIZE)) + goto out; if (PageHWPoison(page)) goto out; @@ -3875,6 +3895,48 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, return ret; } +#ifdef CONFIG_MM_SUBPAGE +static vm_fault_t filemap_map_user_pages(struct vm_fault *vmf, + pgoff_t first, pgoff_t last) +{ + struct vm_area_struct *vma = vmf->vma; + struct address_space *mapping = vma->vm_file->f_mapping; + XA_STATE(xas, &mapping->i_pages, first); + struct folio *folio; + vm_fault_t ret = 0; + unsigned short miss = 0, saved; + + rcu_read_lock(); + folio = next_uptodate_folio(&xas, mapping, last); + if (!folio) + goto out; + if (pmd_none(*vmf->pmd) && vmf->prealloc_pte) + pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte); + vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, + vmf->address, &vmf->ptl); + if (!vmf->pte) { + folio_unlock(folio); + folio_put(folio); + goto out; + } + do { + ret |= map_file_user_pages(vmf, folio, first, last, false); + if (!folio_test_workingset(folio)) + miss += min(folio_next_index(folio) - 1, last) - + max(folio->index, first) + 1; + folio_unlock(folio); + folio_put(folio); + } while ((folio = next_uptodate_folio(&xas, mapping, last)) != NULL); + pte_unmap_unlock(vmf->pte, vmf->ptl); + trace_mm_filemap_map_pages(mapping, first, last); +out: + rcu_read_unlock(); + saved = READ_ONCE(vma->vm_file->f_ra.mmap_miss); + WRITE_ONCE(vma->vm_file->f_ra.mmap_miss, saved > miss ? saved - miss : 0); + return ret; +} +#endif + vm_fault_t filemap_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff) { @@ -3890,6 +3952,11 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, unsigned int nr_pages = 0, folio_type; unsigned short mmap_miss = 0, mmap_miss_saved; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return filemap_map_user_pages(vmf, start_pgoff, end_pgoff); +#endif + /* * Recalculate end_pgoff based on file_end before calling * next_uptodate_folio() to avoid races with concurrent diff --git a/mm/gup.c b/mm/gup.c index ad9ded39609cb..8d9bbc80546fa 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -1297,6 +1298,11 @@ static struct vm_area_struct *gup_vma_lookup(struct mm_struct *mm, #endif } +#ifdef CONFIG_MM_SUBPAGE +static long prefault_subpage_range(struct mm_struct *mm, unsigned long start, + unsigned long nr_pages, unsigned int flags, int *locked); +#endif + /** * __get_user_pages() - pin user pages in memory * @mm: mm_struct of target mm @@ -1363,6 +1369,15 @@ static long __get_user_pages(struct mm_struct *mm, if (!nr_pages) return 0; +#ifdef CONFIG_MM_SUBPAGE + /* A prefault has no page-array ownership or physical-offset ambiguity. */ + if (mm_page_size(mm) != PAGE_SIZE) { + if (pages || (gup_flags & (FOLL_GET | FOLL_PIN))) + return -EOPNOTSUPP; + return prefault_subpage_range(mm, start, nr_pages, gup_flags, locked); + } +#endif + start = untagged_addr_remote(mm, start); VM_WARN_ON_ONCE(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); @@ -1721,7 +1736,7 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm, */ if (likely(pages)) pages += ret; - start += ret << PAGE_SHIFT; + start += ret << mm_page_shift(mm); /* The lock was temporarily dropped, so we must unlock later */ must_unlock = true; @@ -1768,7 +1783,7 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm, break; if (likely(pages)) pages++; - start += PAGE_SIZE; + start += mm_page_size(mm); } if (must_unlock && *locked) { /* @@ -1814,13 +1829,13 @@ long populate_vma_page_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, int *locked) { struct mm_struct *mm = vma->vm_mm; - unsigned long nr_pages = (end - start) / PAGE_SIZE; + unsigned long nr_pages = (end - start) / mm_page_size(mm); int local_locked = 1; int gup_flags; long ret; - VM_WARN_ON_ONCE(!PAGE_ALIGNED(start)); - VM_WARN_ON_ONCE(!PAGE_ALIGNED(end)); + VM_WARN_ON_ONCE(!IS_ALIGNED(start, mm_page_size(mm))); + VM_WARN_ON_ONCE(!IS_ALIGNED(end, mm_page_size(mm))); VM_WARN_ON_ONCE_VMA(start < vma->vm_start, vma); VM_WARN_ON_ONCE_VMA(end > vma->vm_end, vma); mmap_assert_locked(mm); @@ -1887,12 +1902,12 @@ long populate_vma_page_range(struct vm_area_struct *vma, long faultin_page_range(struct mm_struct *mm, unsigned long start, unsigned long end, bool write, int *locked) { - unsigned long nr_pages = (end - start) / PAGE_SIZE; + unsigned long nr_pages = (end - start) / mm_page_size(mm); int gup_flags; long ret; - VM_WARN_ON_ONCE(!PAGE_ALIGNED(start)); - VM_WARN_ON_ONCE(!PAGE_ALIGNED(end)); + VM_WARN_ON_ONCE(!IS_ALIGNED(start, mm_page_size(mm))); + VM_WARN_ON_ONCE(!IS_ALIGNED(end, mm_page_size(mm))); mmap_assert_locked(mm); /* @@ -1968,7 +1983,7 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) } break; } - nend = nstart + ret * PAGE_SIZE; + nend = nstart + ret * mm_page_size(mm); ret = 0; } if (locked) @@ -2047,6 +2062,7 @@ size_t fault_in_writeable(char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; if (unlikely(size == 0)) @@ -2055,7 +2071,7 @@ size_t fault_in_writeable(char __user *uaddr, size_t size) return size; /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) unsafe_put_user(0, (char __user *)cur, out); out: user_write_access_end(); @@ -2116,6 +2132,7 @@ size_t fault_in_safe_writeable(const char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; struct mm_struct *mm = current->mm; bool unlocked = false; @@ -2125,7 +2142,7 @@ size_t fault_in_safe_writeable(const char __user *uaddr, size_t size) mmap_read_lock(mm); /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) if (fixup_user_fault(mm, cur, FAULT_FLAG_WRITE, &unlocked)) break; mmap_read_unlock(mm); @@ -2148,6 +2165,7 @@ size_t fault_in_readable(const char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; volatile char c; @@ -2157,7 +2175,7 @@ size_t fault_in_readable(const char __user *uaddr, size_t size) return size; /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) unsafe_get_user(c, (const char __user *)cur, out); out: user_read_access_end(); @@ -2211,11 +2229,18 @@ struct pages_or_folios { void **entries; }; bool has_folios; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *fragments; +#endif long nr_entries; }; static struct folio *pofs_get_folio(struct pages_or_folios *pofs, long i) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) + return pofs->fragments[i].folio; +#endif if (pofs->has_folios) return pofs->folios[i]; return page_folio(pofs->pages[i]); @@ -2223,22 +2248,56 @@ static struct folio *pofs_get_folio(struct pages_or_folios *pofs, long i) static void pofs_clear_entry(struct pages_or_folios *pofs, long i) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + pofs->fragments[i] = (struct user_page_fragment) {}; + return; + } +#endif pofs->entries[i] = NULL; } static void pofs_unpin(struct pages_or_folios *pofs) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + release_user_fragments(pofs->fragments, pofs->nr_entries, false); + return; + } +#endif if (pofs->has_folios) unpin_folios(pofs->folios, pofs->nr_entries); else unpin_user_pages(pofs->pages, pofs->nr_entries); } +static void pofs_unpin_entry(struct pages_or_folios *pofs, long i) +{ +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + release_user_fragments(&pofs->fragments[i], 1, false); + return; + } +#endif + unpin_folio(pofs_get_folio(pofs, i)); +} + static struct folio *pofs_next_folio(struct folio *folio, struct pages_or_folios *pofs, long *index_ptr) { long i = *index_ptr + 1; +#ifdef CONFIG_MM_SUBPAGE + /* Several adjacent user quarters can pin the same native folio. */ + if (pofs->fragments) { + while (i < pofs->nr_entries && pofs->fragments[i].folio == folio) + i++; + if (i == pofs->nr_entries) + return NULL; + *index_ptr = i; + return pofs->fragments[i].folio; + } +#endif if (!pofs->has_folios && folio_test_large(folio)) { const unsigned long start_pfn = folio_pfn(folio); const unsigned long end_pfn = start_pfn + folio_nr_pages(folio); @@ -2333,9 +2392,9 @@ migrate_longterm_unpinnable_folios(struct list_head *movable_folio_list, * convert the pin on the source folio to a normal * reference. */ - pofs_clear_entry(pofs, i); folio_get(folio); - gup_put_folio(folio, 1, FOLL_PIN); + pofs_unpin_entry(pofs, i); + pofs_clear_entry(pofs, i); if (migrate_device_coherent_folio(folio)) { ret = -EBUSY; @@ -2352,7 +2411,7 @@ migrate_longterm_unpinnable_folios(struct list_head *movable_folio_list, * calling folio_isolate_lru() which takes a reference so the * folio won't be freed if it's migrating. */ - unpin_folio(folio); + pofs_unpin_entry(pofs, i); pofs_clear_entry(pofs, i); } @@ -2458,6 +2517,23 @@ static long check_and_migrate_movable_folios(unsigned long nr_folios, } #endif /* CONFIG_MIGRATION */ +#ifdef CONFIG_MM_SUBPAGE +static long check_and_migrate_movable_fragments(unsigned long count, + struct user_page_fragment *fragments) +{ +#ifdef CONFIG_MIGRATION + struct pages_or_folios pofs = { + .fragments = fragments, + .nr_entries = count, + }; + + return check_and_migrate_movable_pages_or_folios(&pofs); +#else + return 0; +#endif +} +#endif + /* * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which * allows us to process the FOLL_LONGTERM flag. @@ -3133,6 +3209,11 @@ static unsigned long gup_fast(unsigned long start, unsigned long end, int nr_pinned = 0; unsigned seq; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return 0; +#endif + if (!IS_ENABLED(CONFIG_HAVE_GUP_FAST) || !gup_fast_permitted(start, end)) return 0; @@ -3355,6 +3436,464 @@ long pin_user_pages_remote(struct mm_struct *mm, } EXPORT_SYMBOL(pin_user_pages_remote); +#ifdef CONFIG_MM_SUBPAGE +/* 1: acquired, 0: fault required, -EMLINK: read pin needs private backing. */ +static int follow_subpage_fragment(struct vm_area_struct *vma, unsigned long addr, + unsigned int flags, struct user_page_fragment *fragment) +{ + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slot = NULL; + struct folio *folio; + struct page *page; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *ptep, pte; + spinlock_t *ptl; + phys_addr_t phys; + int ret = 0; + bool file_ref = false; + + if (pgd_none(*pgd) || pgd_bad(*pgd)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_mm(mm, *p4d) || p4d_bad_mm(mm, *p4d)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + pud = pud_offset_mm(mm, p4d, addr); + if (pud_none(*pud)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + if (pud_leaf(*pud) || pud_bad(*pud)) + return -EOPNOTSUPP; + pmd = pmd_offset_mm(mm, pud, addr); + if (pmd_none(*pmd)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + if (pmd_leaf(*pmd) || pmd_bad(*pmd)) + return -EOPNOTSUPP; + ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!ptep) + return 0; + pte = ptep_get(ptep); + if (pte_none(pte)) { + ret = PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + goto out; + } + if (!pte_present(pte) || (pte_protnone(pte) && !gup_can_follow_protnone(vma, flags))) + goto out; + phys = pte_phys_mm(mm, pte); + page = vm_normal_page(vma, addr, pte); + if (mm_page_size(mm) > PAGE_SIZE) { + unsigned long offset = (addr & (mm_page_size(mm) - 1)) & PAGE_MASK; + + phys += offset; + if (page) + page += offset >> PAGE_SHIFT; + } + if (!page) { + if (flags & FOLL_DUMP) { + ret = -EFAULT; + goto out; + } + if (!is_zero_pfn(pte_pfn(pte))) { + ret = -EOPNOTSUPP; + goto out; + } + if (flags & FOLL_WRITE) + goto out; + folio = page_folio(pfn_to_page(PHYS_PFN(phys))); + if (flags & FOLL_GET) + folio_get(folio); + } else { + folio = page_folio(page); + /* Coarse leaves use ordinary native-page ownership, not slots. */ + if (mm_page_size(mm) > PAGE_SIZE) { + if ((flags & FOLL_WRITE) && + !can_follow_write_pte(pte, page, vma, flags)) + goto out; + if (!pte_write(pte) && gup_must_unshare(vma, flags, page)) { + ret = -EMLINK; + goto out; + } + ret = try_grab_folio(folio, 1, flags); + if (ret) + goto out; + file_ref = true; + goto acquired; + } + if (!folio_test_anon(folio)) { + if (vma->vm_file && folio_mapping(folio) != vma->vm_file->f_mapping) { + ret = -EOPNOTSUPP; + goto out; + } + if ((flags & FOLL_WRITE) && !pte_write(pte)) + goto out; + /* Private non-anonymous read pins must first acquire COW ownership. */ + if ((flags & FOLL_PIN) && is_cow_mapping(vma->vm_flags)) { + ret = -EMLINK; + goto out; + } + ret = try_grab_folio(folio, 1, flags); + if (ret) + goto out; + file_ref = true; + goto acquired; + } + if (folio_test_ksm(folio)) { + ret = -EOPNOTSUPP; + goto out; + } + slot = mm_subpage_get_from_phys(phys); + if (!slot) { + ret = -EFAULT; + goto out; + } + if ((flags & FOLL_WRITE) && !pte_write(pte)) { + bool force = (flags & FOLL_FORCE) && + !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE | VM_WRITE)) && + (vma->vm_flags & VM_MAYWRITE) && mm_subpage_is_exclusive(slot) && + !pte_needs_soft_dirty_wp(vma, pte) && !userfaultfd_pte_wp(vma, pte); + + if (!force) + goto out; + } + if ((flags & FOLL_PIN) && !mm_subpage_is_exclusive(slot)) { + ret = flags & FOLL_WRITE ? 0 : -EMLINK; + goto out; + } + if (flags & FOLL_PIN) { + ret = mm_subpage_pin(slot); + if (ret) { + if (ret == -EAGAIN) + ret = 0; + goto out; + } + } + } +acquired: + if (slot && (flags & FOLL_WRITE)) + mm_subpage_cancel_lazyfree(slot); + ret = flags & FOLL_PIN ? arch_make_folio_accessible(folio) : 0; + if (ret) { + if (slot) + mm_subpage_unpin(slot); + else if (file_ref) + gup_put_folio(folio, 1, flags); + goto out; + } + if ((flags & FOLL_TOUCH) && (flags & FOLL_WRITE) && + !pte_dirty(pte) && !folio_test_dirty(folio)) + folio_mark_dirty(folio); + folio_mark_accessed(folio); + flush_anon_page(vma, &folio->page, addr); + flush_dcache_folio(folio); + fragment->folio = folio; + fragment->subpage = slot; + fragment->pinned = flags & FOLL_PIN; + if (flags & FOLL_GET) + slot = NULL; /* The returned record consumes the lookup reference. */ + fragment->offset = phys - PFN_PHYS(folio_pfn(folio)) + + (addr & (mm_user_fragment_size(mm) - 1)); + ret = 1; +out: + if (slot) + mm_subpage_put(slot); /* Lookup reference; success retains a separate pin. */ + pte_unmap_unlock(ptep, ptl); + return ret; +} + +/* No references escape: only ensure each user leaf can be followed. */ +static long prefault_subpage_range(struct mm_struct *mm, unsigned long start, + unsigned long nr_pages, unsigned int flags, int *locked) +{ + unsigned long done = 0; + long ret = 0; + + mmap_assert_locked(mm); + start = untagged_addr_remote(mm, start); + while (done < nr_pages) { + struct user_page_fragment unused; + struct vm_area_struct *vma = vma_lookup(mm, start); + + if (!vma) { + ret = flags & FOLL_MADV_POPULATE ? -ENOMEM : -EFAULT; + break; + } + ret = check_vma_flags(vma, flags); + if (ret) { + if (flags & FOLL_MADV_POPULATE) + ret = -EINVAL; + break; + } + if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { + ret = flags & FOLL_MADV_POPULATE ? -EINVAL : -EOPNOTSUPP; + break; + } + if (gup_signal_pending(flags)) { + ret = -EINTR; + break; + } + cond_resched(); + ret = follow_subpage_fragment(vma, start, flags, &unused); + if (!ret) { + ret = faultin_page(vma, start, flags, false, locked); + if (!ret) + continue; + if (ret == -EBUSY || ret == -EAGAIN) + ret = 0; /* Preserve GUP's caller-managed retry contract. */ + break; + } + if (ret < 0) + break; + done++; + start += mm_page_size(mm); + } + if (ret == -EOPNOTSUPP && (flags & FOLL_MADV_POPULATE)) + ret = -EINVAL; + return done ? (long)done : ret; +} + +static long user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity, bool pin, + struct vm_area_struct **vma_out, bool dump) +{ + const unsigned int allowed = FOLL_WRITE | FOLL_FORCE | FOLL_NOFAULT | + FOLL_NOWAIT | FOLL_INTERRUPTIBLE | FOLL_LONGTERM | FOLL_ANON; + unsigned int internal = flags | FOLL_TOUCH | FOLL_REMOTE | + (pin ? FOLL_PIN : FOLL_GET) | (dump ? FOLL_DUMP : 0); + unsigned long end, count = 0; + int locked = 0; + long ret; + + if (!length) + return 0; + if ((flags & FOLL_LONGTERM) && !pin) + return -EINVAL; + if (!fragments || !capacity || capacity > LONG_MAX || (flags & ~allowed)) + return -EINVAL; + if (vma_out) { + mmap_assert_locked(mm); + *vma_out = NULL; + } else { + internal |= FOLL_UNLOCKABLE; + ret = mmap_read_lock_killable(mm); + if (ret) + return ret; + } + locked = 1; + start = untagged_addr_remote(mm, start); + if (check_add_overflow(start, length, &end)) { + ret = -EFAULT; + goto out; + } + if (pin) + mm_set_has_pinned_flag(mm); + while (start < end && count < capacity) { + unsigned long bytes = min(end - start, + mm_user_fragment_size(mm) - (start & (mm_user_fragment_size(mm) - 1))); + struct vm_area_struct *vma; + + if (!locked) { + ret = mmap_read_lock_killable(mm); + if (ret) + break; + locked = 1; + } + if (gup_signal_pending(internal)) { + ret = -EINTR; + break; + } + cond_resched(); + if (mm_page_size(mm) == PAGE_SIZE) { + struct page *page; + + if (dump) + ret = __get_user_pages_locked(mm, start, 1, &page, &locked, internal); + else if (pin) + ret = pin_user_pages_remote(mm, start, 1, flags, &page, + vma_out ? NULL : &locked); + else + ret = get_user_pages_remote(mm, start, 1, flags, &page, + vma_out ? NULL : &locked); + if (ret != 1) { + if (!ret) + ret = -EAGAIN; + break; + } + fragments[count].folio = page_folio(page); + fragments[count].subpage = NULL; + fragments[count].pinned = pin; + fragments[count].offset = folio_page_idx(page_folio(page), page) * PAGE_SIZE + + offset_in_page(start); + } else { + vma = gup_vma_lookup(mm, start); + ret = vma ? check_vma_flags(vma, internal) : -EFAULT; + if (ret) + break; + /* Huge and special/device mappings need separate integration. */ + if (is_vm_hugetlb_page(vma) || + (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP | VM_IO))) { + ret = -EOPNOTSUPP; + break; + } + ret = follow_subpage_fragment(vma, start, internal, &fragments[count]); + if (!ret || ret == -EMLINK) { + ret = faultin_page(vma, start, internal, ret == -EMLINK, &locked); + if (!ret) + continue; + if (!locked && (ret == -EBUSY || ret == -EAGAIN)) { + internal |= FOLL_TRIED; + continue; + } + if (ret == -EBUSY) + ret = -EAGAIN; + break; + } + if (ret < 0) + break; + } + if (vma_out) + *vma_out = vma_lookup(mm, start); + fragments[count++].length = bytes; + start += bytes; + internal &= ~FOLL_TRIED; + } +out: + if (locked && !vma_out) + mmap_read_unlock(mm); + return count ? (long)count : ret; +} + +long pin_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity) +{ + unsigned int saved; + long ret, count; + + if (!(flags & FOLL_LONGTERM) || mm_page_size(mm) == PAGE_SIZE) + return user_fragments_remote(mm, start, length, flags, fragments, + capacity, true, NULL, false); + /* Match native long-term GUP: fault outside movable zones, migrate any + * existing unpinnable backing, then retry with fresh typed references. + */ + saved = memalloc_pin_save(); + do { + count = user_fragments_remote(mm, start, length, flags, fragments, + capacity, true, NULL, false); + if (count <= 0) { + ret = count; + break; + } + ret = check_and_migrate_movable_fragments(count, fragments); + } while (ret == -EAGAIN); + memalloc_pin_restore(saved); + return ret ? ret : count; +} +EXPORT_SYMBOL_GPL(pin_user_fragments_remote); + +long get_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity) +{ + return user_fragments_remote(mm, start, length, flags, fragments, capacity, + false, NULL, false); +} +EXPORT_SYMBOL_GPL(get_user_fragments_remote); + +int get_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma) +{ + int ret; + + /* As with get_user_page_vma_remote(), preserve the caller's VMA lock. */ + if (!vma) + return -EINVAL; + *vma = NULL; + if (flags & FOLL_NOWAIT) + return -EINVAL; + ret = user_fragments_remote(mm, start, length, flags, fragment, 1, false, vma, false); + if (ret > 0 && !*vma) { + release_user_fragments(fragment, 1, false); + return -EFAULT; + } + return ret; +} + +int pin_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma) +{ + int ret; + + if (!vma) + return -EINVAL; + *vma = NULL; + if (flags & (FOLL_NOWAIT | FOLL_LONGTERM)) + return -EINVAL; + ret = user_fragments_remote(mm, start, length, flags, fragment, 1, true, vma, false); + if (ret > 0 && !*vma) { + release_user_fragments(fragment, 1, false); + return -EFAULT; + } + return ret; +} + +#ifdef CONFIG_ELF_CORE +/* Like get_dump_page(), but preserves the selected user-page ownership. */ +int get_dump_fragment(unsigned long addr, struct user_page_fragment *fragment) +{ + return user_fragments_remote(current->mm, addr, mm_user_fragment_size(current->mm), + FOLL_FORCE, fragment, 1, false, NULL, true); +} +#endif + +void mark_user_fragment_dirty(struct user_page_fragment *fragment) +{ + struct folio *folio = fragment->folio; + + if (!folio) + return; + if (!is_zero_folio(folio) && !folio_test_dirty(folio)) { + folio_lock(folio); + folio_mark_dirty(folio); + folio_unlock(folio); + } + if (fragment->subpage) + mm_subpage_cancel_lazyfree(fragment->subpage); +} +EXPORT_SYMBOL_GPL(mark_user_fragment_dirty); + +void release_user_fragments(struct user_page_fragment *fragments, + unsigned long count, bool dirty) +{ + unsigned long i; + + for (i = 0; i < count; i++) { + struct user_page_fragment *fragment = &fragments[i]; + struct folio *folio = fragment->folio; + + if (!folio) + continue; + if (dirty) + mark_user_fragment_dirty(fragment); + if (fragment->subpage) { + if (fragment->pinned) + mm_subpage_unpin(fragment->subpage); + else + mm_subpage_put(fragment->subpage); + } else if (fragment->pinned) { + unpin_folio(folio); + } else { + folio_put(folio); + } + *fragment = (struct user_page_fragment) {}; + } +} +EXPORT_SYMBOL_GPL(release_user_fragments); +#endif + /** * pin_user_pages() - pin user pages in memory for use by other devices * @@ -3407,6 +3946,25 @@ long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, } EXPORT_SYMBOL(pin_user_pages_unlocked); +/* Direct cache pins have no PTE fault to publish fine shmem exposure. */ +static int memfd_pin_folio(struct file *memfd, struct folio *folio) +{ + bool lock = IS_ENABLED(CONFIG_MM_SUBPAGE) && + IS_ENABLED(CONFIG_USERFAULTFD) && shmem_file(memfd); + int ret; + + /* Serialize both acquisition and exposure against partial hole punch. */ + if (lock) + folio_lock(folio); + ret = try_grab_folio(folio, 1, FOLL_PIN); + if (lock) { + if (!ret) + shmem_uffd_mark_mapped(folio, 0, folio_size(folio)); + folio_unlock(folio); + } + return ret; +} + /** * memfd_pin_folios() - pin folios associated with a memfd * @memfd: the memfd whose folios are to be pinned @@ -3492,7 +4050,7 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, for (i = 0; i < nr_found; i++) { folio = fbatch.folios[i]; - if (try_grab_folio(folio, 1, FOLL_PIN)) { + if (memfd_pin_folio(memfd, folio)) { folio_batch_release(&fbatch); ret = -EINVAL; goto err; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index dd93f1b0c44a2..7ab369477fcc7 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2715,7 +2715,8 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, goto unlock; if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma, - vma_is_single_threaded_private(vma))) + vma_is_single_threaded_private(vma), + folio_maybe_mapped_shared(pmd_folio(*pmd)))) goto unlock; } /* @@ -4118,6 +4119,12 @@ static int __folio_split(struct folio *folio, unsigned int new_order, } anon_vma_lock_write(anon_vma); mapping = NULL; +#ifdef CONFIG_MM_SUBPAGE + if (new_order < anon_vma->min_folio_order) { + ret = -EBUSY; + goto out_unlock; + } +#endif } else { unsigned int min_order; gfp_t gfp; @@ -4394,7 +4401,7 @@ int folio_split(struct folio *folio, unsigned int new_order, unsigned int min_order_for_split(struct folio *folio) { if (folio_test_anon(folio)) - return 0; + return folio_anon_min_order(folio); /* * If the folio got truncated, we don't know the previous mapping and diff --git a/mm/internal.h b/mm/internal.h index 5a2ddcf68e0b6..5ad4e57daedc6 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -305,6 +305,50 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) return pte_mkold(pte); } +#ifdef CONFIG_MM_SUBPAGE +/* Batch contiguous user leaves without confusing their offsets with PFNs. */ +static inline unsigned int folio_subpage_pte_batch_flags(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t *ptentp, unsigned int max_nr, fpb_t flags) +{ + struct mm_struct *mm = vma->vm_mm; + pte_t pte = *ptentp; + phys_addr_t offset = pte_phys_mm(mm, pte) - PFN_PHYS(folio_pfn(folio)); + pte_t expected; + unsigned int nr; + + VM_BUG_ON_FOLIO(offset >= folio_size(folio), folio); + max_nr = min_t(unsigned long, max_nr, (folio_size(folio) - offset) >> mm_page_shift(mm)); + VM_BUG_ON_FOLIO(!max_nr, folio); + nr = min(max_nr, pte_batch_hint(ptep, pte)); + expected = pte_advance_pfn_mm(mm, __pte_batch_clear_ignored(pte, flags), nr); + while (nr < max_nr) { + unsigned int batch; + + pte = ptep_get(ptep + nr); + if (!pte_same(__pte_batch_clear_ignored(pte, flags), expected)) + break; + if ((flags & FPB_MERGE_WRITE) && pte_write(pte)) + *ptentp = pte_mkwrite(*ptentp, vma); + if (flags & FPB_MERGE_YOUNG_DIRTY) { + if (pte_young(pte)) + *ptentp = pte_mkyoung(*ptentp); + if (pte_dirty(pte)) + *ptentp = pte_mkdirty(*ptentp); + } + batch = min(max_nr - nr, pte_batch_hint(ptep + nr, pte)); + nr += batch; + expected = pte_advance_pfn_mm(mm, expected, batch); + } + return nr; +} + +static inline unsigned int folio_subpage_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t pte, unsigned int max_nr, fpb_t flags) +{ + return folio_subpage_pte_batch_flags(vma, folio, ptep, &pte, max_nr, flags); +} +#endif + /** * folio_pte_batch_flags - detect a PTE batch for a large folio * @folio: The large folio to detect a PTE batch for. @@ -416,6 +460,10 @@ static inline pte_t pte_move_swp_offset(pte_t pte, long delta) if (pte_swp_uffd_wp(pte)) new = pte_swp_mkuffd_wp(new); +#ifdef CONFIG_MM_SUBPAGE + new = pte_swp_set_subpage_offset(new, pte_swp_subpage_offset(pte)); +#endif + return new; } @@ -1158,6 +1206,11 @@ folio_within_range(struct folio *folio, struct vm_area_struct *vma, end = vma->vm_end; pgoff = folio_pgoff(folio); + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + addr = vma_address_at_offset(vma, (struct vm_page_offset) { .index = pgoff }); + return !IS_ERR_VALUE(addr) && addr >= start && addr <= end && + end - addr >= folio_size(folio); + } /* if folio start address is not in vma range */ if (!in_range(pgoff, vma->vm_pgoff, vma_pglen)) @@ -1240,6 +1293,8 @@ static inline unsigned long vma_address(const struct vm_area_struct *vma, if (pgoff >= vma->vm_pgoff) { address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + if (pgoff != vma->vm_pgoff) + address -= vma_subpage_offset(vma); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address >= vma->vm_end) address = -EFAULT; @@ -1262,15 +1317,24 @@ static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) pgoff_t pgoff; unsigned long address; +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) + return min(pvmw->address + mm_page_size(vma->vm_mm), vma->vm_end); +#endif + /* Common case, plus ->pgoff is invalid for KSM */ - if (pvmw->nr_pages == 1) - return pvmw->address + PAGE_SIZE; + if (pvmw->nr_pages == 1 && mm_page_size(vma->vm_mm) >= PAGE_SIZE) + return min((pvmw->address & mm_page_mask(vma->vm_mm)) + + mm_page_size(vma->vm_mm), vma->vm_end); pgoff = pvmw->pgoff + pvmw->nr_pages; address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + address -= vma_subpage_offset(vma); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address > vma->vm_end) address = vma->vm_end; + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + address = min(mm_page_align(vma->vm_mm, address), vma->vm_end); return address; } @@ -1568,11 +1632,11 @@ static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma) #ifdef CONFIG_NUMA_BALANCING bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, - bool is_private_single_threaded); + bool is_private_single_threaded, bool shared); #else static inline bool folio_can_map_prot_numa(struct folio *folio, - struct vm_area_struct *vma, bool is_private_single_threaded) + struct vm_area_struct *vma, bool is_private_single_threaded, bool shared) { return false; } @@ -1900,7 +1964,7 @@ static inline bool clear_flush_young_ptes_notify(struct vm_area_struct *vma, young = clear_flush_young_ptes(vma, addr, ptep, nr); young |= mmu_notifier_clear_flush_young(vma->vm_mm, addr, - addr + nr * PAGE_SIZE); + addr + nr * mm_page_size(vma->vm_mm)); return young; } @@ -1920,7 +1984,8 @@ static inline bool test_and_clear_young_ptes_notify(struct vm_area_struct *vma, bool young; young = test_and_clear_young_ptes(vma, addr, ptep, nr); - young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + nr * PAGE_SIZE); + young |= mmu_notifier_clear_young(vma->vm_mm, addr, + addr + nr * mm_page_size(vma->vm_mm)); return young; } @@ -1952,4 +2017,14 @@ static inline int get_sysctl_max_map_count(void) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages); +#ifdef CONFIG_MM_SUBPAGE +struct iov_iter; +int process_vm_rw_fragments(struct mm_struct *mm, unsigned long addr, + unsigned long len, struct iov_iter *iter, bool write); +#endif + +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SWAP) +vm_fault_t do_swap_coarse_page(struct vm_fault *vmf); +#endif + #endif /* __MM_INTERNAL_H */ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index d6e04041f5dc1..ad4832f64fc30 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1066,28 +1066,31 @@ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm, } static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_struct *mm, - struct collapse_control *cc) + struct collapse_control *cc, unsigned int order) { gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() : GFP_TRANSHUGE); int node = collapse_find_target_node(cc); struct folio *folio; - folio = __folio_alloc(gfp, HPAGE_PMD_ORDER, node, &cc->alloc_nmask); + folio = __folio_alloc(gfp, order, node, &cc->alloc_nmask); if (!folio) { *foliop = NULL; - count_vm_event(THP_COLLAPSE_ALLOC_FAILED); + if (is_pmd_order(order)) + count_vm_event(THP_COLLAPSE_ALLOC_FAILED); return SCAN_ALLOC_HUGE_PAGE_FAIL; } - count_vm_event(THP_COLLAPSE_ALLOC); + if (is_pmd_order(order)) + count_vm_event(THP_COLLAPSE_ALLOC); if (unlikely(mem_cgroup_charge(folio, mm, gfp))) { folio_put(folio); *foliop = NULL; return SCAN_CGROUP_CHARGE_FAIL; } - count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1); + if (is_pmd_order(order)) + count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1); *foliop = folio; return SCAN_SUCCEED; @@ -1116,7 +1119,7 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a */ mmap_read_unlock(mm); - result = alloc_charge_folio(&folio, mm, cc); + result = alloc_charge_folio(&folio, mm, cc, HPAGE_PMD_ORDER); if (result != SCAN_SUCCEED) goto out_nolock; @@ -1880,22 +1883,33 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) * + unlock and free huge page; */ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, - struct file *file, pgoff_t start, struct collapse_control *cc) + struct file *file, pgoff_t start, struct collapse_control *cc, + unsigned int order) { struct address_space *mapping = file->f_mapping; struct page *dst; struct folio *folio, *tmp, *new_folio; - pgoff_t index = 0, end = start + HPAGE_PMD_NR; + unsigned long nr_pages = 1UL << order; + pgoff_t index = 0, end = start + nr_pages; LIST_HEAD(pagelist); - XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); + XA_STATE_ORDER(xas, &mapping->i_pages, start, order); enum scan_result result = SCAN_SUCCEED; int nr_none = 0; bool is_shmem = shmem_file(file); VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); - VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); + VM_BUG_ON(start & (nr_pages - 1)); - result = alloc_charge_folio(&new_folio, mm, cc); + if (is_pmd_order(order)) { + result = alloc_charge_folio(&new_folio, mm, cc, order); + } else { + /* Sub-PMD user leaves must respect the shmem inode's allocation policy. */ + new_folio = shmem_alloc_collapse_folio(mm, mapping->host, start, order); + if (IS_ERR(new_folio)) { + new_folio = NULL; + result = SCAN_ALLOC_HUGE_PAGE_FAIL; + } + } if (result != SCAN_SUCCEED) goto out; @@ -2024,7 +2038,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * we locked the first folio, then a THP might be there already. * This will be discovered on the first iteration. */ - if (is_pmd_order(folio_order(folio))) { + if (folio_order(folio) >= order) { result = SCAN_PTE_MAPPED_HUGEPAGE; goto out_unlock; } @@ -2149,6 +2163,8 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, while (index < folio->index) { clear_highpage(dst); + if (is_shmem) + shmem_uffd_copy_page(dst, NULL); index++; dst++; } @@ -2158,12 +2174,16 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, result = SCAN_COPY_MC; goto rollback; } + if (is_shmem) + shmem_uffd_copy_page(dst, folio_page(folio, i)); index++; dst++; } } while (index < end) { clear_highpage(dst); + if (is_shmem) + shmem_uffd_copy_page(dst, NULL); index++; dst++; } @@ -2205,8 +2225,13 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * not be able to observe any missing pages due to the * previously inserted retry entries. */ + /* + * A required user leaf populates its whole granule, as a native + * fault does. Only optional PMD collapse must preserve cache holes + * merely because another VMA has missing-page registration. + */ vma_interval_tree_foreach(vma, &mapping->i_mmap, start, end) { - if (userfaultfd_missing(vma)) { + if (is_pmd_order(order) && userfaultfd_missing(vma)) { result = SCAN_EXCEED_NONE_PTE; goto immap_locked; } @@ -2229,12 +2254,13 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, } if (is_shmem) { - lruvec_stat_mod_folio(new_folio, NR_SHMEM, HPAGE_PMD_NR); - lruvec_stat_mod_folio(new_folio, NR_SHMEM_THPS, HPAGE_PMD_NR); - } else { - lruvec_stat_mod_folio(new_folio, NR_FILE_THPS, HPAGE_PMD_NR); + lruvec_stat_mod_folio(new_folio, NR_SHMEM, nr_pages); + if (is_pmd_order(order)) + lruvec_stat_mod_folio(new_folio, NR_SHMEM_THPS, nr_pages); + } else if (is_pmd_order(order)) { + lruvec_stat_mod_folio(new_folio, NR_FILE_THPS, nr_pages); } - lruvec_stat_mod_folio(new_folio, NR_FILE_PAGES, HPAGE_PMD_NR); + lruvec_stat_mod_folio(new_folio, NR_FILE_PAGES, nr_pages); /* * Mark new_folio as uptodate before inserting it into the @@ -2242,14 +2268,14 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * unwritten page. */ folio_mark_uptodate(new_folio); - folio_ref_add(new_folio, HPAGE_PMD_NR - 1); + folio_ref_add(new_folio, nr_pages - 1); if (is_shmem) folio_mark_dirty(new_folio); folio_add_lru(new_folio); /* Join all the small entries into a single multi-index entry. */ - xas_set_order(&xas, start, HPAGE_PMD_ORDER); + xas_set_order(&xas, start, order); xas_store(&xas, new_folio); WARN_ON_ONCE(xas_error(&xas)); xas_unlock_irq(&xas); @@ -2258,9 +2284,11 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * Remove pte page tables, so we can re-fault the page as huge. * If MADV_COLLAPSE, adjust result to call try_collapse_pte_mapped_thp(). */ - retract_page_tables(mapping, start); - if (cc && !cc->is_khugepaged) - result = SCAN_PTE_MAPPED_HUGEPAGE; + if (is_pmd_order(order)) { + retract_page_tables(mapping, start); + if (cc && !cc->is_khugepaged) + result = SCAN_PTE_MAPPED_HUGEPAGE; + } folio_unlock(new_folio); /* @@ -2317,7 +2345,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, folio_put(new_folio); out: VM_BUG_ON(!list_empty(&pagelist)); - trace_mm_khugepaged_collapse_file(mm, new_folio, index, addr, is_shmem, file, HPAGE_PMD_NR, result); + trace_mm_khugepaged_collapse_file(mm, new_folio, index, addr, is_shmem, file, nr_pages, result); return result; } @@ -2420,7 +2448,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, result = SCAN_EXCEED_NONE_PTE; count_vm_event(THP_SCAN_EXCEED_NONE_PTE); } else { - result = collapse_file(mm, addr, file, start, cc); + result = collapse_file(mm, addr, file, start, cc, HPAGE_PMD_ORDER); } } @@ -2831,6 +2859,47 @@ static int madvise_collapse_errno(enum scan_result r) } } +/* + * Promote the cache backing for one larger-than-native user leaf. The caller + * owns mm/file references; no folio or page-table lock may be held. Start is + * a native file index. This sub-PMD shmem path may run under the fault lock: + * it neither retracts page tables (which requires mmap write locking) nor + * invokes filesystem writeback. Source folios are trylocked and rolled back + * on contention. Ordinary PMD collapse retains its existing lock policy. + */ +int collapse_file_user_page(struct mm_struct *mm, struct file *file, pgoff_t start) +{ + struct collapse_control *cc; + unsigned int order; + enum scan_result result; + + if (mm_page_shift(mm) <= PAGE_SHIFT || !shmem_file(file)) + return -EOPNOTSUPP; + order = mm_page_shift(mm) - PAGE_SHIFT; + /* Shmem's order limit reflects THP policy, not an architectural leaf limit. */ + if (order >= HPAGE_PMD_ORDER || order > MAX_PAGECACHE_ORDER || + !IS_ALIGNED(start, 1UL << order) || start > ULONG_MAX - (1UL << order)) + return -EINVAL; + cc = kzalloc(sizeof(*cc), GFP_KERNEL); + if (!cc) + return -ENOMEM; + lru_add_drain_all(); + result = collapse_file(mm, 0, file, start, cc, order); + kfree(cc); + if (result == SCAN_SUCCEED || result == SCAN_PTE_MAPPED_HUGEPAGE) + return 0; + /* Retained references cannot be displaced by replacing their cache folio. */ + if (result == SCAN_PAGE_COUNT) + return -EBUSY; + if (result == SCAN_CGROUP_CHARGE_FAIL) + return -ENOMEM; + if (result == SCAN_COPY_MC) + return -EHWPOISON; + if (result == SCAN_TRUNCATED) + return -EAGAIN; + return madvise_collapse_errno(result); +} + int madvise_collapse(struct vm_area_struct *vma, unsigned long start, unsigned long end, bool *lock_dropped) { diff --git a/mm/ksm.c b/mm/ksm.c index 7d5b76478f0b6..cd0398e972941 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -804,6 +804,10 @@ static struct page *get_mergeable_page(struct ksm_rmap_item *rmap_item) struct folio_walk fw; struct folio *folio; + /* The KSM comparison and replacement paths still require native leaves. */ + if (mm_page_size(mm) != PAGE_SIZE) + return NULL; + mmap_read_lock(mm); vma = find_mergeable_vma(mm, addr); if (!vma) @@ -1338,7 +1342,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct folio *folio, /* See folio_try_share_anon_rmap_pte(): clear PTE first. */ if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, &folio->page)) { + folio_try_share_anon_rmap_pte(folio, &folio->page, vma)) { set_pte_at(mm, pvmw.address, pvmw.pte, entry); goto out_unlock; } @@ -3206,7 +3210,8 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc) if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, addr, rwc->arg)) { + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = addr }, rwc->arg)) { anon_vma_unlock_read(anon_vma); return; } diff --git a/mm/madvise.c b/mm/madvise.c index 5665785e5ac38..1d15da9d1200d 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -193,7 +194,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, spinlock_t *ptl; unsigned long addr; - for (addr = start; addr < end; addr += PAGE_SIZE) { + for (addr = start; addr < end; addr += mm_page_size(vma->vm_mm)) { pte_t pte; softleaf_t entry; struct folio *folio; @@ -236,7 +237,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma, struct address_space *mapping) { XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start)); - pgoff_t end_index = linear_page_index(vma, end) - 1; + pgoff_t end_index = linear_page_index(vma, end - 1); struct folio *folio; struct swap_iocb *splug = NULL; @@ -252,8 +253,11 @@ static void shmem_swapin_range(struct vm_area_struct *vma, if (!softleaf_is_swap(entry)) continue; - addr = vma->vm_start + - ((xas.xa_index - vma->vm_pgoff) << PAGE_SHIFT); + /* The first cache page may start before this VMA's byte offset. */ + addr = xas.xa_index == linear_page_index(vma, start) ? start : + vma_address_at_offset(vma, (struct vm_page_offset) { + .index = xas.xa_index, + }); xas_pause(&xas); rcu_read_unlock(); @@ -318,7 +322,7 @@ static long madvise_willneed(struct madvise_behavior *madv_behavior) mark_mmap_lock_dropped(madv_behavior); get_file(file); offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); mmap_read_unlock(mm); vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED); fput(file); @@ -340,12 +344,18 @@ static inline bool can_do_file_pageout(struct vm_area_struct *vma) file_permission(vma->vm_file, MAY_WRITE) == 0; } -static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end, +static inline int madvise_folio_pte_batch(struct vm_area_struct *vma, + unsigned long addr, unsigned long end, struct folio *folio, pte_t *ptep, pte_t *ptentp) { - int max_nr = (end - addr) / PAGE_SIZE; + int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch_flags(vma, folio, ptep, ptentp, + max_nr, FPB_MERGE_YOUNG_DIRTY); +#endif return folio_pte_batch_flags(folio, NULL, ptep, ptentp, max_nr, FPB_MERGE_YOUNG_DIRTY); } @@ -376,7 +386,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, #ifdef CONFIG_TRANSPARENT_HUGEPAGE if (pmd_trans_huge(*pmd)) { pmd_t orig_pmd; - unsigned long next = pmd_addr_end(addr, end); + unsigned long next = pmd_addr_end_mm(mm, addr, end); tlb_change_page_size(tlb, HPAGE_PMD_SIZE); ptl = pmd_trans_huge_lock(pmd, vma); @@ -446,14 +456,14 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, regular_folio: #endif - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, mm_page_size(mm)); restart: start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); if (!start_pte) return 0; flush_tlb_batched_pending(mm); lazy_mmu_mode_enable(); - for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) { + for (; addr < end; pte += nr, addr += nr * mm_page_size(mm)) { nr = 1; ptent = ptep_get(pte); @@ -484,9 +494,10 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, * fail to split a folio, leave it in place and advance to the * next pte in the range. */ - if (folio_test_large(folio)) { - nr = madvise_folio_pte_batch(addr, end, folio, pte, &ptent); - if (nr < folio_nr_pages(folio)) { + if (folio_test_large(folio) || mm_page_size(mm) != PAGE_SIZE) { + nr = madvise_folio_pte_batch(vma, addr, end, folio, pte, &ptent); + if (folio_test_large(folio) && + nr < (folio_size(folio) >> mm_page_shift(mm))) { int err; if (folio_maybe_mapped_shared(folio)) @@ -521,7 +532,8 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, * number of pages, it must be exclusive. */ if (!folio_test_lru(folio) || - folio_mapcount(folio) != folio_nr_pages(folio)) + folio_mapcount(folio) != + (mm_page_size(mm) != PAGE_SIZE ? nr : folio_nr_pages(folio))) continue; if (pageout_anon_only_filter && !folio_test_anon(folio)) @@ -649,6 +661,72 @@ static long madvise_pageout(struct madvise_behavior *madv_behavior) return 0; } +#ifdef CONFIG_MM_SUBPAGE +static int madvise_free_subpage_range(pmd_t *pmd, unsigned long addr, + unsigned long end, struct mm_walk *walk) +{ + struct vm_area_struct *vma = walk->vma; + struct mm_struct *mm = vma->vm_mm; + struct mmu_gather *tlb = walk->private; + spinlock_t *ptl; + pte_t *base, *ptep; + long swapped = 0; + + base = ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!base) + return 0; + flush_tlb_batched_pending(mm); + for (; addr < end; addr += mm_page_size(mm), ptep++) { + pte_t pte = ptep_get(ptep); + struct mm_subpage *slot; + struct folio *folio; + struct page *page; + + if (!pte_present(pte)) { + softleaf_t entry = softleaf_from_pte(pte); + + if (softleaf_is_swap(entry)) { + swap_subpage_put(entry, + pte_swp_subpage_offset(pte), true); + pte_clear_not_present_full(mm, addr, ptep, tlb->fullmm); + swapped--; + } + continue; + } + page = vm_normal_page(vma, addr, pte); + if (!page) + continue; + folio = page_folio(page); + if (!folio_test_anon(folio) || !folio_trylock(folio)) + continue; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (!slot) + goto unlock; + if (folio_test_swapcache(folio) && !folio_free_swap(folio)) + goto put; + if (!mm_subpage_mark_lazyfree(slot)) + goto put; + clear_young_dirty_ptes(vma, addr, ptep, 1, + CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY); + tlb_remove_tlb_entry(tlb, ptep, addr); + /* Partial hints must never clear a neighbor's dirty state. */ + if (mm_subpage_all_lazyfree(slot)) { + folio_clear_dirty(folio); + folio_mark_lazyfree(folio); + } +put: + mm_subpage_put(slot); +unlock: + folio_unlock(folio); + } + if (swapped) + add_mm_counter(mm, MM_SWAPENTS, swapped); + pte_unmap_unlock(base, ptl); + cond_resched(); + return 0; +} +#endif + static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct mm_walk *walk) @@ -664,6 +742,10 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, unsigned long next; int nr, max_nr; + #ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) + return madvise_free_subpage_range(pmd, addr, end, walk); +#endif next = pmd_addr_end(addr, end); if (pmd_trans_huge(*pmd)) if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next)) @@ -714,7 +796,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, * next pte in the range. */ if (folio_test_large(folio)) { - nr = madvise_folio_pte_batch(addr, end, folio, pte, &ptent); + nr = madvise_folio_pte_batch(vma, addr, end, folio, pte, &ptent); if (nr < folio_nr_pages(folio)) { int err; @@ -988,7 +1070,7 @@ static long madvise_populate(struct madvise_behavior *madv_behavior) return -ENOMEM; } } - start += pages * PAGE_SIZE; + start += pages * mm_page_size(mm); } return 0; } @@ -1022,7 +1104,7 @@ static long madvise_remove(struct madvise_behavior *madv_behavior) return -EACCES; offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); /* * Filesystem's fallocate may need to take i_rwsem. We need to @@ -1182,7 +1264,7 @@ static long madvise_guard_install(struct madvise_behavior *madv_behavior) if (err == 0) { unsigned long nr_expected_pages = - PHYS_PFN(range->end - range->start); + (range->end - range->start) >> mm_page_shift(vma->vm_mm); VM_WARN_ON(nr_pages != nr_expected_pages); return 0; @@ -1833,16 +1915,17 @@ static void madvise_finish_tlb(struct madvise_behavior *madv_behavior) tlb_finish_mmu(madv_behavior->tlb); } -static bool is_valid_madvise(unsigned long start, size_t len_in, int behavior) +static bool is_valid_madvise(struct mm_struct *mm, unsigned long start, + size_t len_in, int behavior) { size_t len; if (!madvise_behavior_valid(behavior)) return false; - if (!PAGE_ALIGNED(start)) + if (!IS_ALIGNED(start, mm_page_size(mm))) return false; - len = PAGE_ALIGN(len_in); + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero */ if (len_in && !len) @@ -1865,14 +1948,29 @@ static bool is_valid_madvise(unsigned long start, size_t len_in, int behavior) * operation. This function returns true in the cases, otherwise false. In * the former case we store an error on @err. */ -static bool madvise_should_skip(unsigned long start, size_t len_in, +static bool madvise_should_skip(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior, int *err) { - if (!is_valid_madvise(start, len_in, behavior)) { + if (!is_valid_madvise(mm, start, len_in, behavior)) { *err = -EINVAL; return true; } - if (start + PAGE_ALIGN(len_in) == start) { +#ifdef CONFIG_MM_SUBPAGE + /* These engines still assume whole native folios/tables. */ + if (mm_page_size(mm) != PAGE_SIZE) { + switch (behavior) { + case MADV_MERGEABLE: + case MADV_UNMERGEABLE: + case MADV_HUGEPAGE: + case MADV_COLLAPSE: + case MADV_HWPOISON: + case MADV_SOFT_OFFLINE: + *err = -EOPNOTSUPP; + return true; + } + } +#endif + if (start + mm_page_align(mm, len_in) == start) { *err = 0; return true; } @@ -1919,7 +2017,7 @@ static int madvise_do_behavior(unsigned long start, size_t len_in, } range->start = get_untagged_addr(madv_behavior->mm, start); - range->end = range->start + PAGE_ALIGN(len_in); + range->end = range->start + mm_page_align(madv_behavior->mm, len_in); blk_start_plug(&plug); if (is_madvise_populate(madv_behavior)) @@ -2012,7 +2110,7 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh .tlb = &tlb, }; - if (madvise_should_skip(start, len_in, behavior, &error)) + if (madvise_should_skip(mm, start, len_in, behavior, &error)) return error; error = madvise_lock(&madv_behavior); if (error) @@ -2055,7 +2153,7 @@ static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter, size_t len_in = iter_iov_len(iter); int error; - if (madvise_should_skip(start, len_in, behavior, &error)) + if (madvise_should_skip(mm, start, len_in, behavior, &error)) ret = error; else ret = madvise_do_behavior(start, len_in, &madv_behavior); @@ -2185,9 +2283,9 @@ static int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, .anon_name = anon_name, }; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; - len = (len_in + ~PAGE_MASK) & PAGE_MASK; + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero */ if (len_in && !len) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index d47aef256a324..964c002533131 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -67,6 +67,9 @@ #include "swap.h" #include "internal.h" +#ifdef CONFIG_MM_SUBPAGE +#include "user-subpage-internal.h" +#endif static int sysctl_memory_failure_early_kill __read_mostly; @@ -539,6 +542,37 @@ struct task_struct *task_early_kill(struct task_struct *tsk, int force_early) return find_early_kill_thread(tsk); } +#ifdef CONFIG_MM_SUBPAGE +struct subpage_collect_procs { + const struct page *page; + struct list_head *to_kill; + int force_early; +}; + +static bool collect_procs_subpage(struct folio *folio, struct vm_area_struct *vma, + struct rmap_walk_range range, void *arg) +{ + struct subpage_collect_procs *args = arg; + DEFINE_FOLIO_RMAP_WALK(walk, folio, vma, range, PVMW_SYNC); + struct task_struct *tsk; + unsigned long address; + + if (!page_vma_mapped_walk(&walk)) + return true; + address = walk.address; + page_vma_mapped_walk_done(&walk); + rcu_read_lock(); + for_each_process(tsk) { + struct task_struct *t = task_early_kill(tsk, args->force_early); + + if (t && t->mm == vma->vm_mm) + add_to_kill_anon_file(t, args->page, vma, args->to_kill, address); + } + rcu_read_unlock(); + return true; +} +#endif + /* * Collect processes when the error hit an anonymous page. */ @@ -549,6 +583,17 @@ static void collect_procs_anon(const struct folio *folio, struct task_struct *tsk; struct anon_vma *av; pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + struct subpage_collect_procs args = { page, to_kill, force_early }; + struct rmap_walk_control rwc = { + .rmap_one = collect_procs_subpage, + .arg = &args, + }; + + if (!folio_test_large(folio) && + rmap_walk_subpages((struct folio *)folio, &rwc, false)) + return; +#endif av = folio_lock_anon_vma_read(folio, NULL); if (av == NULL) /* Not actually mapped anymore */ diff --git a/mm/memory.c b/mm/memory.c index a903d5c1859cb..ade3dbadb409a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -87,6 +87,8 @@ #include #include "pgalloc-track.h" +#include + #include "internal.h" #include "swap.h" @@ -191,26 +193,26 @@ static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, unsigned long start; start = addr; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(tlb->mm, pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(tlb->mm, addr, end); if (pmd_none_or_clear_bad(pmd)) continue; free_pte_range(tlb, pmd, addr); } while (pmd++, addr = next, addr != end); - start &= PUD_MASK; + start &= pud_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= PUD_MASK; + ceiling &= pud_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - pmd = pmd_offset(pud, start); + pmd = pmd_offset_mm(tlb->mm, pud, start); pud_clear(pud); pmd_free_tlb(tlb, pmd, start); mm_dec_nr_pmds(tlb->mm); @@ -225,27 +227,27 @@ static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, unsigned long start; start = addr; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(tlb->mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(tlb->mm, addr, end); if (pud_none_or_clear_bad(pud)) continue; free_pmd_range(tlb, pud, addr, next, floor, ceiling); } while (pud++, addr = next, addr != end); - start &= P4D_MASK; + start &= p4d_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= P4D_MASK; + ceiling &= p4d_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - pud = pud_offset(p4d, start); - p4d_clear(p4d); + pud = pud_offset_mm(tlb->mm, p4d, start); + p4d_clear_mm(tlb->mm, p4d); pud_free_tlb(tlb, pud, start); mm_dec_nr_puds(tlb->mm); } @@ -259,26 +261,26 @@ static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd, unsigned long start; start = addr; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(tlb->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(tlb->mm, addr, end); + if (p4d_none_or_clear_bad_mm(tlb->mm, p4d)) continue; free_pud_range(tlb, p4d, addr, next, floor, ceiling); } while (p4d++, addr = next, addr != end); - start &= PGDIR_MASK; + start &= pgd_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= PGDIR_MASK; + ceiling &= pgd_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - p4d = p4d_offset(pgd, start); + p4d = p4d_offset_mm(tlb->mm, pgd, start); pgd_clear(pgd); p4d_free_tlb(tlb, p4d, start); } @@ -320,7 +322,7 @@ void free_pgd_range(struct mmu_gather *tlb, * Wherever addr is brought up or ceiling brought down, we must * be careful to reject "the opposite 0" before it confuses the * subsequent tests. But what about where end is brought down - * by PMD_SIZE below? no, end can't go down to 0 there. + * by one PMD span below? no, end can't go down to 0 there. * * Whereas we round start (addr) and ceiling down, by different * masks at different levels, in order to test whether a table @@ -328,19 +330,19 @@ void free_pgd_range(struct mmu_gather *tlb, * bother to round floor or end up - the tests don't need that. */ - addr &= PMD_MASK; + addr &= pmd_mask_mm(tlb->mm); if (addr < floor) { - addr += PMD_SIZE; + addr += pmd_size_mm(tlb->mm); if (!addr) return; } if (ceiling) { - ceiling &= PMD_MASK; + ceiling &= pmd_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) - end -= PMD_SIZE; + end -= pmd_size_mm(tlb->mm); if (addr > end - 1) return; /* @@ -350,7 +352,7 @@ void free_pgd_range(struct mmu_gather *tlb, tlb_change_page_size(tlb, PAGE_SIZE); pgd = pgd_offset(tlb->mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(tlb->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; free_p4d_range(tlb, pgd, addr, next, floor, ceiling); @@ -539,16 +541,16 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add return; } - p4dp = p4d_offset(pgdp, addr); + p4dp = p4d_offset_mm(mm, pgdp, addr); p4d = p4dp_get(p4dp); p4dv = p4d_val(p4d); - if (!p4d_present(p4d) || p4d_leaf(p4d)) { + if (!p4d_present_mm(mm, p4d) || p4d_leaf(p4d)) { pr_alert("pgd:%08llx p4d:%08llx\n", pgdv, p4dv); return; } - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_mm(mm, p4dp, addr); pud = pudp_get(pudp); pudv = pud_val(pud); @@ -557,7 +559,7 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add return; } - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_mm(mm, pudp, addr); pmd = pmdp_get(pmdp); pmdv = pmd_val(pmd); @@ -950,7 +952,15 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, struct page *page; if (likely(softleaf_is_swap(entry))) { - if (swap_dup_entry_direct(entry) < 0) + int err; + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) + err = swap_subpage_dup(entry, pte_swp_subpage_offset(orig_pte)); + else +#endif + err = swap_dup_entries_direct(entry, mm_pte_native_pages(src_mm)); + if (err < 0) return -EIO; /* make sure dst_mm is on swapoff's mmlist. */ @@ -982,6 +992,10 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, entry = make_readable_migration_entry( swp_offset(entry)); pte = softleaf_to_pte(entry); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) + pte = pte_swp_set_subpage_offset(pte, pte_swp_subpage_offset(orig_pte)); +#endif if (pte_swp_soft_dirty(orig_pte)) pte = pte_swp_mksoft_dirty(pte); if (pte_swp_uffd_wp(orig_pte)) @@ -1076,10 +1090,14 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma * over and copy the page & arm it. */ - if (copy_mc_user_highpage(&new_folio->page, page, addr, src_vma)) - return -EHWPOISON; + for (unsigned int i = 0; i < mm_pte_native_pages(src_vma->vm_mm); i++) { + if (copy_mc_user_highpage(folio_page(new_folio, i), page + i, + addr + i * PAGE_SIZE, src_vma)) + return -EHWPOISON; + } *prealloc = NULL; + folio_ref_add(new_folio, mm_pte_native_pages(dst_vma->vm_mm) - 1); __folio_mark_uptodate(new_folio); folio_add_new_anon_rmap(new_folio, dst_vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, dst_vma); @@ -1095,6 +1113,55 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma return 0; } +#ifdef CONFIG_MM_SUBPAGE +/* One new native folio may publish several independently locked user leaves. */ +static void subpage_add_new_lru(struct folio *folio, struct vm_area_struct *vma, + unsigned int mapped) +{ + folio_add_lru_vma(folio, vma); + if ((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED) + while (--mapped) + mlock_folio(folio); +} + +/* The destination slot and its charge/metadata were allocated outside PTLs. */ +static int copy_pinned_subpage(struct vm_area_struct *dst_vma, pte_t *dst_pte, + pte_t source_pte, unsigned long addr, int *rss, + struct mm_subpage *source, struct mm_subpage **prealloc) +{ + struct mm_subpage *dest = *prealloc; + struct folio *new; + pte_t pte; + int ret; + + if (!dest) + return -EAGAIN; + new = mm_subpage_folio(dest); + ret = copy_user_subpage_range(&new->page, mm_subpage_offset(dest), + &mm_subpage_folio(source)->page, mm_subpage_offset(source), + mm_subpage_size(source)); + if (ret) + return ret; + /* Fresh, unmapped backing is private to this preallocation. */ + if (WARN_ON_ONCE(!folio_trylock(new))) + return -EBUSY; + mm_subpage_set_exclusive(dest); + ret = mm_subpage_add_new_anon_rmap(dest, dst_vma, addr); + if (!ret) { + subpage_add_new_lru(new, dst_vma, 1); + rss[MM_ANONPAGES]++; + pte = phys_pte_mm(dst_vma->vm_mm, mm_subpage_phys(dest), dst_vma->vm_page_prot); + pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); + if (userfaultfd_pte_wp(dst_vma, source_pte)) + pte = pte_mkuffd_wp(pte); + set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); + *prealloc = NULL; + } + folio_unlock(new); + return ret; +} +#endif + static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr, int nr) @@ -1128,18 +1195,46 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, static inline int copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr, - int max_nr, int *rss, struct folio **prealloc) + int max_nr, int *rss, struct folio **prealloc, + struct mm_subpage **subprealloc) { fpb_t flags = FPB_MERGE_WRITE; struct page *page; struct folio *folio; - int err, nr; + int err, nr, nr_native; + unsigned int span = mm_pte_native_pages(src_vma->vm_mm); page = vm_normal_page(src_vma, addr, pte); if (unlikely(!page)) goto copy_pte; folio = page_folio(page); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_vma->vm_mm) < PAGE_SIZE) { + if (folio_test_anon(folio)) { + struct mm_subpage *slot; + + slot = mm_subpage_get_from_phys(pte_phys_mm(src_vma->vm_mm, pte)); + if (WARN_ON_ONCE(!slot)) + return -EINVAL; + err = mm_subpage_dup_anon_rmap(slot, dst_vma, src_vma, addr); + if (err == -EBUSY) { + err = copy_pinned_subpage(dst_vma, dst_pte, pte, addr, + rss, slot, subprealloc); + mm_subpage_put(slot); + return err ? err : 1; + } + if (err) { + mm_subpage_put(slot); + return err; + } + /* The lookup reference becomes the child's PTE reference. */ + rss[MM_ANONPAGES]++; + goto copy_pte; + } + max_nr = 1; + } +#endif /* * If we likely have to copy, just don't bother with batching. Make @@ -1152,18 +1247,25 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma if (vma_soft_dirty_enabled(src_vma)) flags |= FPB_RESPECT_SOFT_DIRTY; - nr = folio_pte_batch_flags(folio, src_vma, src_pte, &pte, max_nr, flags); - folio_ref_add(folio, nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_vma->vm_mm) > PAGE_SIZE) + nr = folio_subpage_pte_batch_flags(src_vma, folio, src_pte, + &pte, max_nr, flags); + else +#endif + nr = folio_pte_batch_flags(folio, src_vma, src_pte, &pte, max_nr, flags); + nr_native = nr * span; + folio_ref_add(folio, nr_native); if (folio_test_anon(folio)) { if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page, - nr, dst_vma, src_vma))) { - folio_ref_sub(folio, nr); + nr_native, dst_vma, src_vma))) { + folio_ref_sub(folio, nr_native); return -EAGAIN; } rss[MM_ANONPAGES] += nr; VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio); } else { - folio_dup_file_rmap_ptes(folio, page, nr, dst_vma); + folio_dup_file_rmap_ptes(folio, page, nr_native, dst_vma); rss[mm_counter_file(folio)] += nr; } __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, @@ -1171,7 +1273,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma return nr; } - folio_get(folio); + folio_ref_add(folio, span); if (folio_test_anon(folio)) { /* * If this page may have been pinned by the parent process, @@ -1181,7 +1283,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma */ if (unlikely(folio_try_dup_anon_rmap_pte(folio, page, dst_vma, src_vma))) { /* Page may be pinned, we have to copy. */ - folio_put(folio); + folio_put_refs(folio, span); err = copy_present_page(dst_vma, src_vma, dst_pte, src_pte, addr, rss, prealloc, page); return err ? err : 1; @@ -1189,6 +1291,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma rss[MM_ANONPAGES]++; VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio); } else { + mm_subpage_file_map_add(dst_vma, pte_phys_mm(src_vma->vm_mm, pte)); folio_dup_file_rmap_pte(folio, page, dst_vma); rss[mm_counter_file(folio)]++; } @@ -1202,11 +1305,12 @@ static inline struct folio *folio_prealloc(struct mm_struct *src_mm, struct vm_area_struct *vma, unsigned long addr, bool need_zero) { struct folio *new_folio; + unsigned int order = max_t(unsigned int, mm_page_shift(src_mm), PAGE_SHIFT) - PAGE_SHIFT; if (need_zero) - new_folio = vma_alloc_zeroed_movable_folio(vma, addr); + new_folio = vma_alloc_zeroed_movable_folio_order(vma, addr, order); else - new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr); + new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, order, vma, addr); if (!new_folio) return NULL; @@ -1220,6 +1324,34 @@ static inline struct folio *folio_prealloc(struct mm_struct *src_mm, return new_folio; } +#ifdef CONFIG_MM_SUBPAGE +static struct mm_subpage *anon_subpage_prealloc(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr) +{ + struct mm_subpage_pool *pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + struct mm_subpage *slot = ERR_PTR(-ENOMEM); + struct folio *folio; + int ret; + + if (!pool) + return slot; + folio = folio_prealloc(mm, vma, addr, false); + if (!folio) + goto out; + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + slot = ERR_PTR(ret); + goto out; + } + slot = mm_subpage_alloc_at(pool, vma_page_offset_at(vma, addr).offset); +out: + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + return slot; +} +#endif + static int copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, @@ -1236,6 +1368,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, int rss[NR_MM_COUNTERS]; softleaf_t entry = softleaf_mk_none(); struct folio *prealloc = NULL; + struct mm_subpage *subprealloc = NULL; int nr; again: @@ -1316,16 +1449,23 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, WARN_ON_ONCE(ret != -ENOENT); } /* copy_present_ptes() will clear `*prealloc' if consumed */ - max_nr = (end - addr) / PAGE_SIZE; + max_nr = (end - addr) >> mm_page_shift(src_mm); ret = copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, - ptent, addr, max_nr, rss, &prealloc); + ptent, addr, max_nr, rss, &prealloc, &subprealloc); /* * If we need a pre-allocated page for this pte, drop the * locks, allocate, and try again. * If copy failed due to hwpoison in source page, break out. */ - if (unlikely(ret == -EAGAIN || ret == -EHWPOISON)) + if (unlikely(ret == -EAGAIN || ret == -EHWPOISON || + (IS_ENABLED(CONFIG_MM_SUBPAGE) && ret < 0))) break; +#ifdef CONFIG_MM_SUBPAGE + if (subprealloc) { + mm_subpage_put(subprealloc); + subprealloc = NULL; + } +#endif if (unlikely(prealloc)) { /* * pre-alloc page cannot be reused by next time so as @@ -1338,7 +1478,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } nr = ret; progress += 8 * nr; - } while (dst_pte += nr, src_pte += nr, addr += PAGE_SIZE * nr, + } while (dst_pte += nr, src_pte += nr, addr += mm_page_size(src_mm) * nr, addr != end); lazy_mmu_mode_disable(); @@ -1357,10 +1497,22 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } else if (ret == -EBUSY || unlikely(ret == -EHWPOISON)) { goto out; } else if (ret == -EAGAIN) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) { + subprealloc = anon_subpage_prealloc(src_mm, src_vma, addr); + if (IS_ERR(subprealloc)) + return PTR_ERR(subprealloc); + ret = 0; + goto again; + } +#endif prealloc = folio_prealloc(src_mm, src_vma, addr, false); if (!prealloc) return -ENOMEM; } else if (ret < 0) { + /* Subpage ownership checks can return additional fatal errors. */ + if (IS_ENABLED(CONFIG_MM_SUBPAGE)) + goto out; VM_WARN_ON_ONCE(1); } @@ -1370,6 +1522,10 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, if (addr != end) goto again; out: +#ifdef CONFIG_MM_SUBPAGE + if (subprealloc) + mm_subpage_put(subprealloc); +#endif if (unlikely(prealloc)) folio_put(prealloc); return ret; @@ -1388,9 +1544,9 @@ copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_pmd = pmd_alloc(dst_mm, dst_pud, addr); if (!dst_pmd) return -ENOMEM; - src_pmd = pmd_offset(src_pud, addr); + src_pmd = pmd_offset_mm(src_vma->vm_mm, src_pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(src_vma->vm_mm, addr, end); if (pmd_is_huge(*src_pmd)) { int err; @@ -1425,9 +1581,9 @@ copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_pud = pud_alloc(dst_mm, dst_p4d, addr); if (!dst_pud) return -ENOMEM; - src_pud = pud_offset(src_p4d, addr); + src_pud = pud_offset_mm(src_vma->vm_mm, src_p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(src_vma->vm_mm, addr, end); if (pud_trans_huge(*src_pud)) { int err; @@ -1461,10 +1617,10 @@ copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr); if (!dst_p4d) return -ENOMEM; - src_p4d = p4d_offset(src_pgd, addr); + src_p4d = p4d_offset_mm(src_vma->vm_mm, src_pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(src_p4d)) + next = p4d_addr_end_mm(src_vma->vm_mm, addr, end); + if (p4d_none_or_clear_bad_mm(src_vma->vm_mm, src_p4d)) continue; if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d, addr, next)) @@ -1516,6 +1672,9 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) bool is_cow; int ret; + if (mm_page_shift(dst_mm) != mm_page_shift(src_mm)) + return -EINVAL; + if (!vma_needs_copy(dst_vma, src_vma)) return 0; @@ -1549,7 +1708,7 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) dst_pgd = pgd_offset(dst_mm, addr); src_pgd = pgd_offset(src_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(src_vma->vm_mm, addr, end); if (pgd_none_or_clear_bad(src_pgd)) continue; if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd, @@ -1629,7 +1788,7 @@ zap_install_uffd_wp_if_needed(struct vm_area_struct *vma, if (--nr == 0) break; pte++; - addr += PAGE_SIZE; + addr += mm_page_size(vma->vm_mm); } return was_installed; @@ -1642,10 +1801,12 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, bool *force_flush, bool *force_break, bool *any_skipped) { struct mm_struct *mm = tlb->mm; + unsigned int nr_native = nr * mm_pte_native_pages(mm); bool delay_rmap = false; if (!folio_test_anon(folio)) { ptent = get_and_clear_full_ptes(mm, addr, pte, nr, tlb->fullmm); + mm_subpage_file_map_del(vma, pte_phys_mm(mm, ptent)); if (pte_dirty(ptent)) { folio_mark_dirty(folio); if (tlb_delay_rmap(tlb)) { @@ -1669,16 +1830,44 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, nr, details, ptent); if (!delay_rmap) { - folio_remove_rmap_ptes(folio, page, nr, vma); + folio_remove_rmap_ptes(folio, page, nr_native, vma); if (unlikely(folio_mapcount(folio) < 0)) print_bad_pte(vma, addr, ptent, page); } - if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) { + /* Gather payloads count native references, not logical PTEs. */ + if (unlikely(__tlb_remove_folio_pages(tlb, page, nr_native, delay_rmap))) { + *force_flush = true; + *force_break = true; + } +} + +#ifdef CONFIG_MM_SUBPAGE +static void zap_present_subpage(struct mmu_gather *tlb, + struct vm_area_struct *vma, pte_t *pte, pte_t ptent, + unsigned long addr, int *rss, bool *force_flush, + bool *force_break, bool *any_skipped) +{ + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(tlb->mm, ptent)); + + /* All small-granule anonymous PTEs must carry slot ownership. */ + if (WARN_ON_ONCE(!slot)) { + *any_skipped = true; + return; + } + ptep_get_and_clear_full(tlb->mm, addr, pte, tlb->fullmm); + arch_check_zapped_pte(vma, ptent); + tlb_remove_tlb_entry(tlb, pte, addr); + mm_subpage_remove_anon_rmap(slot, vma); + rss[MM_ANONPAGES]--; + /* Drop the lookup ref, then transfer the PTE ref to the TLB batch. */ + mm_subpage_put(slot); + if (__tlb_remove_subpage(tlb, slot)) { *force_flush = true; *force_break = true; } } +#endif /* * Zap or skip at least one present PTE, trying to batch-process subsequent @@ -1716,12 +1905,30 @@ static inline int zap_present_ptes(struct mmu_gather *tlb, return 1; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + if (folio_test_anon(folio)) { + zap_present_subpage(tlb, vma, pte, ptent, addr, rss, + force_flush, force_break, any_skipped); + return 1; + } + /* Native folio batching still expresses physical PAGE_SIZE steps. */ + max_nr = 1; + } +#endif + /* * Make sure that the common "small folio" case is as fast as possible * by keeping the batching logic separate. */ if (unlikely(folio_test_large(folio) && max_nr != 1)) { - nr = folio_pte_batch(folio, pte, ptent, max_nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) > PAGE_SIZE) + nr = folio_subpage_pte_batch(vma, folio, pte, ptent, + max_nr, 0); + else +#endif + nr = folio_pte_batch(folio, pte, ptent, max_nr); zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr, addr, details, rss, force_flush, force_break, any_skipped); @@ -1764,9 +1971,20 @@ static inline int zap_nonpresent_ptes(struct mmu_gather *tlb, if (!should_zap_cows(details)) return 1; - nr = swap_pte_batch(pte, max_nr, ptent); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + swap_subpage_put(entry, pte_swp_subpage_offset(ptent), true); + } else +#endif + { + unsigned int span = mm_pte_native_pages(vma->vm_mm); + + /* Adjacent coarse PTEs advance by a full group of swap slots. */ + if (span == 1) + nr = swap_pte_batch(pte, max_nr, ptent); + swap_put_entries_direct(entry, nr * span); + } rss[MM_SWAPENTS] -= nr; - swap_put_entries_direct(entry, nr); } else if (softleaf_is_migration(entry)) { struct folio *folio = softleaf_to_folio(entry); @@ -1811,7 +2029,7 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, bool *any_skipped) { pte_t ptent = ptep_get(pte); - int max_nr = (end - addr) / PAGE_SIZE; + int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); int nr = 0; /* Skip all consecutive none ptes */ @@ -1825,7 +2043,7 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, if (!max_nr) return nr; pte += nr; - addr += nr * PAGE_SIZE; + addr += nr * mm_page_size(vma->vm_mm); } if (pte_present(ptent)) @@ -1839,13 +2057,14 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, return nr; } -static bool pte_table_reclaim_possible(unsigned long start, unsigned long end, +static bool pte_table_reclaim_possible(struct mm_struct *mm, + unsigned long start, unsigned long end, struct zap_details *details) { if (!IS_ENABLED(CONFIG_PT_RECLAIM)) return false; /* Only zap if we are allowed to and cover the full page table. */ - return details && details->reclaim_pt && (end - start >= PMD_SIZE); + return details && details->reclaim_pt && (end - start >= pmd_size_mm(mm)); } static bool zap_empty_pte_table(struct mm_struct *mm, pmd_t *pmd, @@ -1877,7 +2096,7 @@ static bool zap_pte_table_if_empty(struct mm_struct *mm, pmd_t *pmd, if (ptl != pml) spin_lock_nested(ptl, SINGLE_DEPTH_NESTING); - for (i = 0, pte = start_pte; i < PTRS_PER_PTE; i++, pte++) { + for (i = 0, pte = start_pte; i < pte_table_bytes_mm(mm) / sizeof(pte_t); i++, pte++) { if (!pte_none(ptep_get(pte))) goto out_ptl; } @@ -1902,7 +2121,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end, struct zap_details *details) { - bool can_reclaim_pt = pte_table_reclaim_possible(addr, end, details); + bool can_reclaim_pt = pte_table_reclaim_possible(tlb->mm, addr, end, details); bool force_flush = false, force_break = false; struct mm_struct *mm = tlb->mm; int rss[NR_MM_COUNTERS]; @@ -1915,7 +2134,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, int nr; retry: - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, mm_page_size(mm)); init_rss_vec(rss); start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl); if (!pte) @@ -1936,11 +2155,11 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, if (any_skipped) can_reclaim_pt = false; if (unlikely(force_break)) { - addr += nr * PAGE_SIZE; + addr += nr * mm_page_size(vma->vm_mm); direct_reclaim = false; break; } - } while (pte += nr, addr += PAGE_SIZE * nr, addr != end); + } while (pte += nr, addr += mm_page_size(mm) * nr, addr != end); /* * Fast path: try to hold the pmd lock and unmap the PTE page. @@ -1997,9 +2216,9 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, pmd_t *pmd; unsigned long next; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(tlb->mm, pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(tlb->mm, addr, end); if (pmd_is_huge(*pmd)) { if (next - addr != HPAGE_PMD_SIZE) __split_huge_pmd(vma, pmd, addr, false); @@ -2033,9 +2252,9 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb, pud_t *pud; unsigned long next; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(tlb->mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(tlb->mm, addr, end); if (pud_trans_huge(*pud)) { if (next - addr != HPAGE_PUD_SIZE) split_huge_pud(vma, pud, addr); @@ -2061,10 +2280,10 @@ static inline unsigned long zap_p4d_range(struct mmu_gather *tlb, p4d_t *p4d; unsigned long next; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(tlb->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(tlb->mm, addr, end); + if (p4d_none_or_clear_bad_mm(tlb->mm, p4d)) continue; next = zap_pud_range(tlb, vma, p4d, addr, next, details); } while (p4d++, addr = next, addr != end); @@ -2103,7 +2322,7 @@ static void __zap_vma_range(struct mmu_gather *tlb, struct vm_area_struct *vma, tlb_start_vma(tlb, vma); pgd = pgd_offset(vma->vm_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(tlb->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; next = zap_p4d_range(tlb, vma, pgd, addr, next, details); @@ -2350,7 +2569,7 @@ static int validate_page_before_insert(struct vm_area_struct *vma, static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, unsigned long addr, struct page *page, - pgprot_t prot, bool mkwrite) + pgprot_t prot, bool mkwrite, unsigned int offset) { struct folio *folio = page_folio(page); pte_t pteval = ptep_get(pte); @@ -2360,7 +2579,7 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, return -EBUSY; /* see insert_pfn(). */ - if (pte_pfn(pteval) != page_to_pfn(page)) { + if (pte_phys_mm(vma->vm_mm, pteval) != page_to_phys(page) + offset) { WARN_ON_ONCE(!is_zero_pfn(pte_pfn(pteval))); return -EFAULT; } @@ -2372,17 +2591,18 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, } /* Ok, finally just insert the thing.. */ - pteval = mk_pte(page, prot); + pteval = phys_pte_mm(vma->vm_mm, page_to_phys(page) + offset, prot); if (unlikely(is_zero_folio(folio))) { pteval = pte_mkspecial(pteval); } else { folio_get(folio); - pteval = mk_pte(page, prot); + pteval = phys_pte_mm(vma->vm_mm, page_to_phys(page) + offset, prot); if (mkwrite) { pteval = pte_mkyoung(pteval); pteval = maybe_mkwrite(pte_mkdirty(pteval), vma); } inc_mm_counter(vma->vm_mm, mm_counter_file(folio)); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, pteval)); folio_add_file_rmap_pte(folio, page, vma); } set_pte_at(vma->vm_mm, addr, pte, pteval); @@ -2390,7 +2610,7 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, } static int insert_page(struct vm_area_struct *vma, unsigned long addr, - struct page *page, pgprot_t prot, bool mkwrite) + struct page *page, pgprot_t prot, bool mkwrite, unsigned int offset) { int retval; pte_t *pte; @@ -2404,83 +2624,120 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr, if (!pte) goto out; retval = insert_page_into_pte_locked(vma, pte, addr, page, prot, - mkwrite); + mkwrite, offset); pte_unmap_unlock(pte, ptl); out: return retval; } -static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte, - unsigned long addr, struct page *page, pgprot_t prot) +/* Keep lock batching in units of hardware leaves, backing in native pages. */ +static int insert_page_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long offset, + unsigned long *length, pgprot_t prot) { - int err; + struct mm_struct *mm = vma->vm_mm; + unsigned long remaining = *length; + unsigned long size = mm_page_size(mm); + int ret = 0; - err = validate_page_before_insert(vma, page); - if (err) - return err; - return insert_page_into_pte_locked(vma, pte, addr, page, prot, false); + while (remaining) { + unsigned long in_pmd = min(remaining, + pmd_size_mm(mm) - (addr & ~pmd_mask_mm(mm))); + pmd_t *pmd = walk_to_pmd(mm, addr); + + if (!pmd) { + ret = -EFAULT; + break; + } + if (pte_alloc(mm, pmd)) { + ret = -ENOMEM; + break; + } + while (in_pmd) { + unsigned int batch = min_t(unsigned long, in_pmd / size, 8); + unsigned int i; + spinlock_t *ptl; + pte_t *start = pte_offset_map_lock(mm, pmd, addr, &ptl); + + if (!start) { + ret = -EFAULT; + goto out; + } + for (i = 0; i < batch; i++) { + struct page *page = pages[offset >> PAGE_SHIFT]; + + ret = validate_page_before_insert(vma, page); + if (!ret) + ret = insert_page_into_pte_locked(vma, start + i, + addr, page, prot, false, offset_in_page(offset)); + if (ret) + break; + addr += size; + offset += size; + remaining -= size; + in_pmd -= size; + } + pte_unmap_unlock(start, ptl); + if (ret) + goto out; + } + } +out: + *length = remaining; + return ret; } -/* insert_pages() amortizes the cost of spinlock operations - * when inserting pages in a loop. - */ static int insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num, pgprot_t prot) { - pmd_t *pmd = NULL; - pte_t *start_pte, *pte; - spinlock_t *pte_lock; - struct mm_struct *const mm = vma->vm_mm; - unsigned long curr_page_idx = 0; - unsigned long remaining_pages_total = *num; - unsigned long pages_to_write_in_pmd; + unsigned long length; int ret; -more: - ret = -EFAULT; - pmd = walk_to_pmd(mm, addr); - if (!pmd) - goto out; - pages_to_write_in_pmd = min_t(unsigned long, - remaining_pages_total, PTRS_PER_PTE - pte_index(addr)); - - /* Allocate the PTE if necessary; takes PMD lock once only. */ - ret = -ENOMEM; - if (pte_alloc(mm, pmd)) - goto out; + if (check_mul_overflow(*num, PAGE_SIZE, &length)) + return -EOVERFLOW; + ret = insert_page_range(vma, addr, pages, 0, &length, prot); + /* A partially inserted native page is still an incomplete input page. */ + *num = DIV_ROUND_UP(length, PAGE_SIZE); + return ret; +} - while (pages_to_write_in_pmd) { - int pte_idx = 0; - const int batch_size = min_t(int, pages_to_write_in_pmd, 8); +/** + * vm_insert_pages_range - map a byte range from a native-page array + * @vma: target VMA, held under the mmap write lock + * @addr: first user address + * @pages: native backing pages + * @nr_pages: size of @pages + * @offset: byte offset within the backing array + * @length: bytes to map + * + * Address, offset and length must match the target mm's granule. A failed + * insertion can leave a prefix mapped, as with vm_insert_pages(); the caller + * must unmap that prefix when abandoning the mapping. Physical ownership + * remains in native pages. Each installed user leaf acquires its own ref. + */ +int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long nr_pages, + unsigned long offset, unsigned long length) +{ + unsigned long bytes, end, size = mm_page_size(vma->vm_mm); - start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock); - if (!start_pte) { - ret = -EFAULT; - goto out; - } - for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) { - int err = insert_page_in_batch_locked(vma, pte, - addr, pages[curr_page_idx], prot); - if (unlikely(err)) { - pte_unmap_unlock(start_pte, pte_lock); - ret = err; - remaining_pages_total -= pte_idx; - goto out; - } - addr += PAGE_SIZE; - ++curr_page_idx; - } - pte_unmap_unlock(start_pte, pte_lock); - pages_to_write_in_pmd -= batch_size; - remaining_pages_total -= batch_size; + if (!IS_ALIGNED(addr | offset | length, size) || + check_mul_overflow(nr_pages, PAGE_SIZE, &bytes) || + offset > bytes || length > bytes - offset || + check_add_overflow(addr, length, &end) || + !range_in_vma(vma, addr, end)) + return -EINVAL; + if (!length) + return 0; + if (vma->vm_flags & VM_PFNMAP) + return -EINVAL; + if (!(vma->vm_flags & VM_MIXEDMAP)) { + mmap_assert_write_locked(vma->vm_mm); + vm_flags_set(vma, VM_MIXEDMAP); } - if (remaining_pages_total) - goto more; - ret = 0; -out: - *num = remaining_pages_total; - return ret; + return insert_page_range(vma, addr, pages, offset, &length, vma->vm_page_prot); } +EXPORT_SYMBOL_GPL(vm_insert_pages_range); /** * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock. @@ -2501,9 +2758,12 @@ int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num) { const unsigned long nr_pages = *num; - const unsigned long end = addr + PAGE_SIZE * nr_pages; + unsigned long length, end; - if (!range_in_vma(vma, addr, end)) + if (!IS_ALIGNED(addr, mm_page_size(vma->vm_mm)) || + check_mul_overflow(nr_pages, PAGE_SIZE, &length) || + check_add_overflow(addr, length, &end) || + !range_in_vma(vma, addr, end)) return -EFAULT; if (!(vma->vm_flags & VM_MIXEDMAP)) { VM_WARN_ON_ONCE(mmap_read_trylock(vma->vm_mm)); @@ -2528,8 +2788,10 @@ int map_kernel_pages_prepare(struct vm_area_desc *desc) } nr_pages = action->map_kernel.nr_pages; - end = addr + PAGE_SIZE * nr_pages; - if (!range_in_vma_desc(desc, addr, end)) + if (!IS_ALIGNED(addr, mm_page_size(desc->mm)) || + check_mul_overflow(nr_pages, PAGE_SIZE, &end) || + check_add_overflow(addr, end, &end) || + !range_in_vma_desc(desc, addr, end)) return -EFAULT; return 0; @@ -2588,10 +2850,35 @@ int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, BUG_ON(vma->vm_flags & VM_PFNMAP); vm_flags_set(vma, VM_MIXEDMAP); } - return insert_page(vma, addr, page, vma->vm_page_prot, false); + return insert_page(vma, addr, page, vma->vm_page_prot, false, 0); } EXPORT_SYMBOL(vm_insert_page); +#ifdef CONFIG_MM_SUBPAGE +/* Explicit physical offset; the VMA and supplied page retain native ownership. */ +vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, unsigned long addr, + struct page *page, unsigned int offset) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long size = mm_page_size(mm); + int err; + + if (!(vma->vm_flags & VM_MIXEDMAP) || (vma->vm_flags & VM_PFNMAP) || + (addr & (size - 1)) || (offset & (size - 1)) || + offset > PAGE_SIZE - size || addr < vma->vm_start || + addr >= vma->vm_end || vma->vm_end - addr < size) + return VM_FAULT_SIGBUS; + err = insert_page(vma, addr, page, vma->vm_page_prot, false, offset); + if (err == -ENOMEM) + return VM_FAULT_OOM; + if (err < 0 && err != -EBUSY) + return VM_FAULT_SIGBUS; + return VM_FAULT_NOPAGE; +} +EXPORT_SYMBOL_GPL(vmf_insert_page_offset); +#endif + + /* * __vm_map_pages - maps range of kernel pages into user vma * @vma: user vma to map to @@ -2606,20 +2893,22 @@ EXPORT_SYMBOL(vm_insert_page); * Return: 0 on success and error code otherwise. */ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, - unsigned long num, unsigned long offset) + unsigned long num, unsigned long offset, + unsigned int subpage_offset) { - unsigned long count = vma_pages(vma); - unsigned long uaddr = vma->vm_start; + unsigned long bytes, available; - /* Fail if the user requested offset is beyond the end of the object */ - if (offset >= num) + if (offset >= num || + check_mul_overflow(num - offset, PAGE_SIZE, &available) || + subpage_offset >= PAGE_SIZE || + check_mul_overflow(offset, PAGE_SIZE, &bytes) || + check_add_overflow(bytes, (unsigned long)subpage_offset, &bytes)) return -ENXIO; - - /* Fail if the user requested size exceeds available object size */ - if (count > num - offset) + if (vma->vm_end - vma->vm_start > + available - subpage_offset) return -ENXIO; - - return vm_insert_pages(vma, uaddr, pages + offset, &count); + return vm_insert_pages_range(vma, vma->vm_start, pages, num, bytes, + vma->vm_end - vma->vm_start); } /** @@ -2643,7 +2932,7 @@ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, int vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num) { - return __vm_map_pages(vma, pages, num, vma->vm_pgoff); + return __vm_map_pages(vma, pages, num, vma->vm_pgoff, vma_subpage_offset(vma)); } EXPORT_SYMBOL(vm_map_pages); @@ -2663,12 +2952,12 @@ EXPORT_SYMBOL(vm_map_pages); int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, unsigned long num) { - return __vm_map_pages(vma, pages, num, 0); + return __vm_map_pages(vma, pages, num, 0, 0); } EXPORT_SYMBOL(vm_map_pages_zero); static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, pgprot_t prot, bool mkwrite) + unsigned long pfn, pgprot_t prot, bool mkwrite, unsigned int offset) { struct mm_struct *mm = vma->vm_mm; pte_t *pte, entry; @@ -2690,7 +2979,7 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, * allocation and mapping invalidation so just skip the * update. */ - if (pte_pfn(entry) != pfn) { + if (pte_phys_mm(mm, entry) != PFN_PHYS(pfn) + offset) { WARN_ON_ONCE(!is_zero_pfn(pte_pfn(entry))); goto out_unlock; } @@ -2703,7 +2992,7 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, } /* Ok, finally just insert the thing.. */ - entry = pte_mkspecial(pfn_pte(pfn, prot)); + entry = pte_mkspecial(phys_pte_mm(mm, PFN_PHYS(pfn) + offset, prot)); if (mkwrite) { entry = pte_mkyoung(entry); @@ -2765,9 +3054,9 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, * Context: Process context. May allocate using %GFP_KERNEL. * Return: vm_fault_t value. */ -vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, +static vm_fault_t __vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot, - bool mkwrite) + bool mkwrite, unsigned int offset) { /* * Technically, architectures with pte_special can avoid all these @@ -2789,10 +3078,31 @@ vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, pfnmap_setup_cachemode_pfn(pfn, &pgprot); - return insert_pfn(vma, addr, pfn, pgprot, mkwrite); + return insert_pfn(vma, addr, pfn, pgprot, mkwrite, offset); +} +vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, pgprot_t pgprot, + bool mkwrite) +{ + return __vmf_insert_pfn_prot_mkwrite(vma, addr, pfn, pgprot, mkwrite, 0); } EXPORT_SYMBOL(vmf_insert_pfn_prot_mkwrite); +/* Explicit quarter offset; a native PFN alone cannot encode a 4K leaf. */ +vm_fault_t vmf_insert_pfn_prot_mkwrite_offset(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, unsigned int offset, + pgprot_t pgprot, bool mkwrite) +{ + unsigned long size = mm_page_size(vma->vm_mm); + + if (!IS_ALIGNED(addr | offset, size) || offset > PAGE_SIZE - size || + addr < vma->vm_start || addr >= vma->vm_end || + vma->vm_end - addr < size) + return VM_FAULT_SIGBUS; + return __vmf_insert_pfn_prot_mkwrite(vma, addr, pfn, pgprot, mkwrite, offset); +} +EXPORT_SYMBOL_GPL(vmf_insert_pfn_prot_mkwrite_offset); + static bool vm_mixed_ok(struct vm_area_struct *vma, unsigned long pfn, bool mkwrite) { @@ -2840,9 +3150,9 @@ static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma, * result in pfn_t_has_page() == false. */ page = pfn_to_page(pfn); - err = insert_page(vma, addr, page, pgprot, mkwrite); + err = insert_page(vma, addr, page, pgprot, mkwrite, 0); } else { - return insert_pfn(vma, addr, pfn, pgprot, mkwrite); + return insert_pfn(vma, addr, pfn, pgprot, mkwrite, 0); } if (err == -ENOMEM) @@ -2863,7 +3173,7 @@ vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, if (addr < vmf->vma->vm_start || addr >= vmf->vma->vm_end) return VM_FAULT_SIGBUS; - err = insert_page(vmf->vma, addr, page, pgprot, write); + err = insert_page(vmf->vma, addr, page, pgprot, write, 0); if (err == -ENOMEM) return VM_FAULT_OOM; if (err < 0 && err != -EBUSY) @@ -2898,7 +3208,7 @@ vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, */ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pte_t *pte, *mapped_pte; spinlock_t *ptl; @@ -2910,13 +3220,13 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, lazy_mmu_mode_enable(); do { BUG_ON(!pte_none(ptep_get(pte))); - if (!pfn_modify_allowed(pfn, prot)) { + if (!pfn_modify_allowed(PHYS_PFN(phys), prot)) { err = -EACCES; break; } - set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); - pfn++; - } while (pte++, addr += PAGE_SIZE, addr != end); + set_pte_at(mm, addr, pte, pte_mkspecial(phys_pte_mm(mm, phys, prot))); + phys += mm_page_size(mm); + } while (pte++, addr += mm_page_size(mm), addr != end); lazy_mmu_mode_disable(); pte_unmap_unlock(mapped_pte, ptl); return err; @@ -2924,67 +3234,67 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pmd_t *pmd; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; pmd = pmd_alloc(mm, pud, addr); if (!pmd) return -ENOMEM; VM_BUG_ON(pmd_trans_huge(*pmd)); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(mm, addr, end); err = remap_pte_range(mm, pmd, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pmd++, addr = next, addr != end); return 0; } static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pud_t *pud; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; pud = pud_alloc(mm, p4d, addr); if (!pud) return -ENOMEM; do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(mm, addr, end); err = remap_pmd_range(mm, pud, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pud++, addr = next, addr != end); return 0; } static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { p4d_t *p4d; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; p4d = p4d_alloc(mm, pgd, addr); if (!p4d) return -ENOMEM; do { - next = p4d_addr_end(addr, end); + next = p4d_addr_end_mm(mm, addr, end); err = remap_pud_range(mm, p4d, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (p4d++, addr = next, addr != end); return 0; } @@ -3013,25 +3323,26 @@ static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long ad { pgd_t *pgd; unsigned long next; - unsigned long end = addr + PAGE_ALIGN(size); struct mm_struct *mm = vma->vm_mm; + unsigned long end = addr + mm_page_align(mm, size); + phys_addr_t phys = PFN_PHYS(pfn); int err; - if (WARN_ON_ONCE(!PAGE_ALIGNED(addr))) + if (WARN_ON_ONCE(!IS_ALIGNED(addr, mm_page_size(mm)))) return -EINVAL; VM_WARN_ON_ONCE(!vma_test_all_mask(vma, VMA_REMAP_FLAGS)); BUG_ON(addr >= end); - pfn -= addr >> PAGE_SHIFT; pgd = pgd_offset(mm, addr); flush_cache_range(vma, addr, end); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(mm, addr, end); err = remap_p4d_range(mm, pgd, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pgd++, addr = next, addr != end); return 0; @@ -3093,7 +3404,7 @@ static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, struct pfnmap_track_ctx *ctx = NULL; int err; - size = PAGE_ALIGN(size); + size = mm_page_align(vma->vm_mm, size); /* * If we cover the full VMA, we'll perform actual tracking, and @@ -3162,7 +3473,7 @@ static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size) { - const unsigned long end = addr + PAGE_ALIGN(size); + const unsigned long end = addr + mm_page_align(vma->vm_mm, size); const bool is_cow = is_cow_mapping(vma->vm_flags); int err; @@ -3540,15 +3851,14 @@ static inline int pte_unmap_same(struct vm_fault *vmf) * -EHWPOISON: copy failed due to hwpoison in source page * -EAGAIN: copied failed (some other reason) */ -static inline int __wp_page_copy_user(struct page *dst, struct page *src, - struct vm_fault *vmf) +static inline int __wp_native_page_copy_user(struct page *dst, struct page *src, + struct vm_fault *vmf, unsigned long addr) { int ret; void *kaddr; void __user *uaddr; struct vm_area_struct *vma = vmf->vma; struct mm_struct *mm = vma->vm_mm; - unsigned long addr = vmf->address; if (likely(src)) { if (copy_mc_user_highpage(dst, src, addr, vma)) @@ -3638,6 +3948,22 @@ static inline int __wp_page_copy_user(struct page *dst, struct page *src, return ret; } +static int __wp_page_copy_user(struct page *dst, struct page *src, + struct vm_fault *vmf) +{ + unsigned int i, nr = mm_pte_native_pages(vmf->vma->vm_mm); + + for (i = 0; i < nr; i++) { + int ret = __wp_native_page_copy_user(dst + i, src ? src + i : NULL, + vmf, vmf->address + i * PAGE_SIZE); + + if (ret) + return ret; + kmsan_copy_page_meta(dst + i, src ? src + i : NULL); + } + return 0; +} + static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma) { struct file *vm_file = vma->vm_file; @@ -3883,14 +4209,13 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) delayacct_wpcopy_end(); return err == -EHWPOISON ? VM_FAULT_HWPOISON : 0; } - kmsan_copy_page_meta(&new_folio->page, vmf->page); } __folio_mark_uptodate(new_folio); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, - vmf->address & PAGE_MASK, - (vmf->address & PAGE_MASK) + PAGE_SIZE); + vmf->address & mm_page_mask(mm), + (vmf->address & mm_page_mask(mm)) + mm_page_size(mm)); mmu_notifier_invalidate_range_start(&range); /* @@ -3927,6 +4252,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) * some TLBs while the old PTE remains in others. */ ptep_clear_flush(vma, vmf->address, vmf->pte); + folio_ref_add(new_folio, mm_pte_native_pages(mm) - 1); folio_add_new_anon_rmap(new_folio, vma, vmf->address, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, vma); BUG_ON(unshare && pte_write(entry)); @@ -3970,7 +4296,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) mmu_notifier_invalidate_range_end(&range); if (new_folio) - folio_put(new_folio); + folio_put_refs(new_folio, page_copied ? mm_pte_native_pages(mm) : 1); if (old_folio) { if (page_copied) free_swap_cache(old_folio); @@ -4092,7 +4418,7 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf, struct folio *folio) return ret; } -#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#ifdef CONFIG_MM_ID static bool __wp_can_reuse_large_anon_folio(struct folio *folio, struct vm_area_struct *vma) { @@ -4158,18 +4484,18 @@ static bool __wp_can_reuse_large_anon_folio(struct folio *folio, folio_unlock_large_mapcount(folio); return exclusive; } -#else /* !CONFIG_TRANSPARENT_HUGEPAGE */ +#else /* !CONFIG_MM_ID */ static bool __wp_can_reuse_large_anon_folio(struct folio *folio, struct vm_area_struct *vma) { BUILD_BUG(); } -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif /* CONFIG_MM_ID */ static bool wp_can_reuse_anon_folio(struct folio *folio, struct vm_area_struct *vma) { - if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && folio_test_large(folio)) + if (IS_ENABLED(CONFIG_MM_ID) && folio_test_large(folio)) return __wp_can_reuse_large_anon_folio(folio, vma); /* @@ -4207,67 +4533,238 @@ static bool wp_can_reuse_anon_folio(struct folio *folio, return true; } -/* - * This routine handles present pages, when - * * users try to write to a shared page (FAULT_FLAG_WRITE) - * * GUP wants to take a R/O pin on a possibly shared anonymous page - * (FAULT_FLAG_UNSHARE) - * - * It is done by copying the page to a new address and decrementing the - * shared-page counter for the old page. - * - * Note that this routine assumes that the protection checks have been - * done by the caller (the low-level page fault routine in most cases). - * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've - * done any necessary COW. - * - * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even - * though the page will change only once the write actually happens. This - * avoids a few races, and potentially makes it more efficient. - * - * We enter with non-exclusive mmap_lock (to exclude vma changes, - * but allow concurrent faults), with pte both mapped and locked. - * We return with mmap_lock still held, but pte unmapped and unlocked. - */ -static vm_fault_t do_wp_page(struct vm_fault *vmf) +#ifdef CONFIG_MM_SUBPAGE +/* Enter with the original PTE locked; a slot, not its native folio, is copied. */ +static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) __releases(vmf->ptl) { - const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; struct vm_area_struct *vma = vmf->vma; - struct folio *folio = NULL; - pte_t pte; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *source = NULL, *dest = NULL; + struct mm_subpage_pool *pool = NULL; + struct folio *old = vmf->page ? page_folio(vmf->page) : NULL; + struct folio *new = NULL; + struct mmu_notifier_range range; + bool copied = false, unshare = vmf->flags & FAULT_FLAG_UNSHARE; + unsigned int offset; + vm_fault_t ret = VM_FAULT_SIGBUS; + pte_t entry; + int err; - if (likely(!unshare)) { - if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) { - if (!userfaultfd_wp_async(vma)) { - pte_unmap_unlock(vmf->pte, vmf->ptl); - return handle_userfault(vmf, VM_UFFD_WP); - } + if (old && folio_test_anon(old)) { + source = mm_subpage_get_from_phys(pte_phys_mm(mm, vmf->orig_pte)); + if (WARN_ON_ONCE(!source)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + return VM_FAULT_SIGBUS; + } + if (!mm_subpage_is_exclusive(source) && mm_subpage_mapcount(source) == 1 && + folio_trylock(old)) { + bool reusable = true; - /* - * Nothing needed (cache flush, TLB invalidations, - * etc.) because we're only removing the uffd-wp bit, - * which is completely invisible to the user. - */ - pte = pte_clear_uffd_wp(ptep_get(vmf->pte)); + if (folio_test_swapcache(old)) { + swp_entry_t entry = page_swap_entry(&old->page); - set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); - /* - * Update this to be prepared for following up CoW - * handling - */ - vmf->orig_pte = pte; + reusable = !swap_subpage_count(entry, mm_subpage_offset(source)) && + !(folio_test_writeback(old) && + (folio_swap_flags(old) & SWP_STABLE_WRITES)); + } + if (reusable) + mm_subpage_try_reuse_anon(source); + folio_unlock(old); } - - /* - * Userfaultfd write-protect can defer flushes. Ensure the TLB - * is flushed in this case before copying. - */ - if (unlikely(userfaultfd_wp(vmf->vma) && - mm_tlb_flush_pending(vmf->vma->vm_mm))) - flush_tlb_page(vmf->vma, vmf->address); + if (mm_subpage_is_exclusive(source)) { + if (unshare) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + } else { + folio_xchg_last_cpupid(old, (1 << LAST_CPUPID_SHIFT) - 1); + /* Exclusivity belongs to the slot, not PageAnonExclusive. */ + wp_page_reuse(vmf, NULL); + } + mm_subpage_put(source); + return 0; + } + } else if (old) { + folio_get(old); + } else if (!is_zero_pfn(pte_pfn(vmf->orig_pte))) { + /* Private device/PFN mappings need their own subpage contract. */ + pte_unmap_unlock(vmf->pte, vmf->ptl); + return VM_FAULT_SIGBUS; } - + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + delayacct_wpcopy_start(); + ret = vmf_anon_prepare(vmf); + if (ret) + goto out; + offset = vma_page_offset_at(vma, vmf->address).offset; + ret = VM_FAULT_OOM; + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + goto out; + new = folio_prealloc(mm, vma, vmf->address, false); + if (!new) + goto out; + err = mm_subpage_pool_add_folio(pool, new, GFP_KERNEL); + if (err) { + folio_put(new); + goto out; + } + dest = source ? mm_subpage_copy_at(pool, source, offset) : + mm_subpage_alloc_at(pool, offset); + if (IS_ERR(dest)) { + if (PTR_ERR(dest) == -EHWPOISON) + ret = VM_FAULT_HWPOISON; + dest = NULL; + goto out; + } + if (old && !source) { + err = copy_user_subpage_range(&new->page, offset, vmf->page, + pte_phys_mm(mm, vmf->orig_pte) & ~PAGE_MASK, + mm_page_size(mm)); + if (err) { + ret = VM_FAULT_HWPOISON; + goto out; + } + } + mm_subpage_pool_close(pool); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, + vmf->address, vmf->address + mm_page_size(mm)); + mmu_notifier_invalidate_range_start(&range); + /* Keep the folio alive after a concurrent zap can consume its PTE ref. */ + folio_get(new); + folio_lock(new); + ret = 0; + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock_pte; + } + mm_subpage_set_exclusive(dest); + err = mm_subpage_add_new_anon_rmap(dest, vma, vmf->address); + if (WARN_ON_ONCE(err)) { + ret = VM_FAULT_SIGBUS; + goto unlock_pte; + } + entry = pte_sw_mkyoung(phys_pte_mm(mm, mm_subpage_phys(dest), vma->vm_page_prot)); + if (unshare) { + if (pte_soft_dirty(vmf->orig_pte)) + entry = pte_mksoft_dirty(entry); + if (pte_uffd_wp(vmf->orig_pte)) + entry = pte_mkuffd_wp(entry); + } else { + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + } + BUG_ON(unshare && pte_write(entry)); + flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); + /* Break-before-make also orders removal of the old slot's rmap. */ + ptep_clear_flush(vma, vmf->address, vmf->pte); + subpage_add_new_lru(new, vma, 1); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + if (source) { + mm_subpage_remove_anon_rmap(source, vma); + } else if (old) { + mm_subpage_file_map_del(vma, pte_phys_mm(mm, vmf->orig_pte)); + folio_remove_rmap_pte(old, vmf->page, vma); + dec_mm_counter(mm, mm_counter_file(old)); + inc_mm_counter(mm, MM_ANONPAGES); + } else { + ksm_might_unmap_zero_page(mm, vmf->orig_pte); + inc_mm_counter(mm, MM_ANONPAGES); + } + dest = NULL; /* The new PTE owns the reference. */ + copied = true; +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + folio_unlock(new); + folio_put(new); + mmu_notifier_invalidate_range_end(&range); +out: + if (pool) { + mm_subpage_pool_close(pool); + if (dest) + mm_subpage_put(dest); + mm_subpage_pool_put(pool); + } + if (source) { + if (copied) + mm_subpage_put(source); /* Retired PTE reference, after TLBI. */ + mm_subpage_put(source); /* Source stabilisation reference. */ + } else if (old) { + if (copied) + folio_put(old); + folio_put(old); + } + delayacct_wpcopy_end(); + return ret; +} +#endif + +/* + * This routine handles present pages, when + * * users try to write to a shared page (FAULT_FLAG_WRITE) + * * GUP wants to take a R/O pin on a possibly shared anonymous page + * (FAULT_FLAG_UNSHARE) + * + * It is done by copying the page to a new address and decrementing the + * shared-page counter for the old page. + * + * Note that this routine assumes that the protection checks have been + * done by the caller (the low-level page fault routine in most cases). + * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've + * done any necessary COW. + * + * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even + * though the page will change only once the write actually happens. This + * avoids a few races, and potentially makes it more efficient. + * + * We enter with non-exclusive mmap_lock (to exclude vma changes, + * but allow concurrent faults), with pte both mapped and locked. + * We return with mmap_lock still held, but pte unmapped and unlocked. + */ +static vm_fault_t do_wp_page(struct vm_fault *vmf) + __releases(vmf->ptl) +{ + const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; + struct vm_area_struct *vma = vmf->vma; + struct folio *folio = NULL; + pte_t pte; + + if (likely(!unshare)) { + if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) { + if (!userfaultfd_wp_async(vma)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + return handle_userfault(vmf, VM_UFFD_WP); + } + + /* + * Nothing needed (cache flush, TLB invalidations, + * etc.) because we're only removing the uffd-wp bit, + * which is completely invisible to the user. + */ + pte = pte_clear_uffd_wp(ptep_get(vmf->pte)); + + set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); + /* + * Update this to be prepared for following up CoW + * handling + */ + vmf->orig_pte = pte; + } + + /* + * Userfaultfd write-protect can defer flushes. Ensure the TLB + * is flushed in this case before copying. + */ + if (unlikely(userfaultfd_wp(vmf->vma) && + mm_tlb_flush_pending(vmf->vma->vm_mm))) + flush_tlb_page(vmf->vma, vmf->address); + } + vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte); if (vmf->page) @@ -4292,6 +4789,11 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) return wp_page_shared(vmf, folio); } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return wp_subpage_copy(vmf); +#endif + /* * Private mapping: create an exclusive anonymous page copy if reuse * is impossible. We might miss VM_WRITE for FOLL_FORCE handling. @@ -4301,8 +4803,12 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) */ if (folio && folio_test_anon(folio) && (PageAnonExclusive(vmf->page) || wp_can_reuse_anon_folio(folio, vma))) { - if (!PageAnonExclusive(vmf->page)) - SetPageAnonExclusive(vmf->page); + if (!PageAnonExclusive(vmf->page)) { + unsigned int i, nr = mm_pte_native_pages(vma->vm_mm); + + for (i = 0; i < nr; i++) + SetPageAnonExclusive(vmf->page + i); + } if (unlikely(unshare)) { pte_unmap_unlock(vmf->pte, vmf->ptl); return 0; @@ -4340,12 +4846,116 @@ static inline void unmap_mapping_range_tree(struct rb_root_cached *root, start = vma->vm_start + ((start_idx - vma->vm_pgoff) << PAGE_SHIFT); size = (end_idx - start_idx) << PAGE_SHIFT; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + unsigned long end = vma->vm_end; + + start = vma_address(vma, first_index, last_index - first_index + 1); + /* The interval tree guarantees a nonempty intersection. */ + VM_BUG_ON_VMA(IS_ERR_VALUE(start), vma); + if (last_index < vma_last_pgoff(vma)) + end = vma_address_at_offset(vma, (struct vm_page_offset) { + .index = last_index + 1, + }); + size = end - start; + } else if (mm_page_size(vma->vm_mm) > PAGE_SIZE) { + unsigned long granule = mm_page_size(vma->vm_mm); + unsigned long end = min(start + size, vma->vm_end); + + /* A native-page invalidation removes every overlapping leaf. */ + start = max(round_down(start, granule), vma->vm_start); + end = min(round_up(end, granule), vma->vm_end); + size = end - start; + } +#endif + tlb_gather_mmu(&tlb, vma->vm_mm); zap_vma_range_batched(&tlb, vma, start, size, details); tlb_finish_mmu(&tlb); } } +#ifdef CONFIG_MM_SUBPAGE +/* Unmap affected user leaves, including coarse aliases; preserve private COW. */ +void unmap_mapping_subpage_hole(struct address_space *mapping, loff_t first, u64 last) +{ + struct vm_area_struct *vma; + struct zap_details details = { .skip_cows = true }; + + i_mmap_lock_read(mapping); + vma_interval_tree_foreach(vma, &mapping->i_mmap, + first >> PAGE_SHIFT, (last - 1) >> PAGE_SHIFT) { + unsigned long start, end, granule = mm_page_size(vma->vm_mm); + u64 low = round_down((u64)first, granule), high = round_up(last, granule); + struct vm_page_offset begin = { low >> PAGE_SHIFT, low & ~PAGE_MASK }; + struct vm_page_offset finish = { high >> PAGE_SHIFT, high & ~PAGE_MASK }; + struct vm_page_offset vbegin = vma_page_offset_at(vma, vma->vm_start); + struct vm_page_offset vend = vma_page_offset_at(vma, vma->vm_end); + struct mmu_gather tlb; + + if (begin.index < vbegin.index || + (begin.index == vbegin.index && begin.offset <= vbegin.offset)) + start = vma->vm_start; + else + start = vma_address_at_offset(vma, begin); + if (finish.index > vend.index || + (finish.index == vend.index && finish.offset >= vend.offset)) + end = vma->vm_end; + else + end = vma_address_at_offset(vma, finish); + if (IS_ERR_VALUE(start) || IS_ERR_VALUE(end) || start >= end) + continue; + tlb_gather_mmu(&tlb, vma->vm_mm); + zap_vma_range_batched(&tlb, vma, start, end - start, &details); + tlb_finish_mmu(&tlb); + } + i_mmap_unlock_read(mapping); +} + +/* + * Native truncation keeps its final partial page. Smaller-granule mms must + * additionally discard complete granules in that native page, including + * private COW copies. Call on both sides of cache truncation, as for the + * ordinary unmap_mapping_range() calls. + */ +void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize) +{ + struct vm_area_struct *vma; + struct zap_details details = { }; + pgoff_t index = newsize >> PAGE_SHIFT; + unsigned int offset = newsize & ~PAGE_MASK; + + if (!offset) + return; + i_mmap_lock_read(mapping); + vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) { + unsigned long start, end, granule = mm_page_size(vma->vm_mm); + struct vm_page_offset pos = { .index = index, .offset = round_up(offset, granule) }; + struct vm_page_offset vpos = vma_page_offset_at(vma, vma->vm_start); + struct mmu_gather tlb; + + if (pos.offset >= PAGE_SIZE) + continue; + if (pos.index < vpos.index || + (pos.index == vpos.index && pos.offset < vpos.offset)) + start = vma->vm_start; + else + start = vma_address_at_offset(vma, pos); + if (IS_ERR_VALUE(start)) + continue; + end = vma->vm_end; + if (index < vma_last_pgoff(vma)) + end = vma_address_at_offset(vma, (struct vm_page_offset) { + .index = index + 1, + }); + tlb_gather_mmu(&tlb, vma->vm_mm); + zap_vma_range_batched(&tlb, vma, start, end - start, &details); + tlb_finish_mmu(&tlb); + } + i_mmap_unlock_read(mapping); +} +#endif + /** * unmap_mapping_folio() - Unmap single folio from processes. * @folio: The locked folio to be unmapped. @@ -4772,6 +5382,134 @@ static void check_swap_exclusive(struct folio *folio, swp_entry_t entry, } while (--nr_pages); } +#ifdef CONFIG_MM_SUBPAGE +/* The swap-cache index remains native-sized; the PTE retains its slot offset. */ +static vm_fault_t do_swap_subpage(struct vm_fault *vmf, swp_entry_t entry) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct swap_info_struct *si; + struct mm_subpage *slot = NULL; + struct folio *folio; + unsigned int offset = pte_swp_subpage_offset(vmf->orig_pte); + vm_fault_t ret; + pte_t pte; + int err; + + ret = vmf_anon_prepare(vmf); + if (ret) + return ret; + /* Swap encodes the physical slot, independent of its virtual position. */ + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(mm))) + return VM_FAULT_SIGBUS; + si = get_swap_device(entry); + if (!si) + return 0; + folio = swap_cache_get_folio(entry); + if (folio) { + swap_update_readahead(folio, vma, vmf->address); + } else { + folio = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, vmf); + if (!folio) { + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (vmf->pte) { + if (pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + ret = VM_FAULT_OOM; + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + } + goto out_device; + } + ret = VM_FAULT_MAJOR; + count_vm_event(PGMAJFAULT); + count_memcg_event_mm(mm, PGMAJFAULT); + } + ret |= folio_lock_or_retry(folio, vmf); + if (ret & VM_FAULT_RETRY) + goto out_folio; + if (!folio_matches_swap_entry(folio, entry)) + goto unlock_folio; + if (folio_order(folio) || !folio_test_uptodate(folio) || folio_test_ksm(folio)) { + ret = VM_FAULT_SIGBUS; + goto unlock_folio; + } + if (PageHWPoison(&folio->page)) { + ret = VM_FAULT_HWPOISON; + goto unlock_folio; + } + arch_swap_restore(folio_swap(entry, folio), folio); + slot = mm_subpage_restore_granule(folio, offset, mm_page_shift(mm), GFP_KERNEL); + if (IS_ERR(slot)) { + err = PTR_ERR(slot); + slot = NULL; + if (err != -EAGAIN) + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_folio; + } + folio_throttle_swaprate(folio, GFP_KERNEL); + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + goto unlock_pte; + err = mm_subpage_add_anon_rmap(slot, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_pte; + } + /* + * A carried marker proves exclusivity directly. After the last fork + * sharer exits, recover it only with one swap reference to this exact + * quarter and no other mapping or escaped slot reference. Dropping a + * nonpresent PTE in zap flushes pending reclaim TLB invalidations first. + * Stable-write backends must still finish writeback before reuse. + */ + if (!(folio_test_writeback(folio) && data_race(si->flags & SWP_STABLE_WRITES))) { + if (pte_swp_exclusive(vmf->orig_pte)) + mm_subpage_set_exclusive(slot); + else if (swap_subpage_count(entry, offset) == 1) + mm_subpage_try_reuse_swap(slot); + } + pte = pte_sw_mkyoung(phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot)); + if (pte_swp_soft_dirty(vmf->orig_pte)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(vmf->orig_pte)) + pte = pte_mkuffd_wp(pte); + inc_mm_counter(mm, MM_ANONPAGES); + dec_mm_counter(mm, MM_SWAPENTS); + swap_subpage_put(entry, offset, false); + set_pte_at(mm, vmf->address, vmf->pte, pte); + arch_do_swap_page_nr(mm, vma, vmf->address, pte, pte, 1); + slot = NULL; /* The restored PTE owns the slot reference. */ + vmf->orig_pte = pte; + if (should_try_to_free_swap(si, folio, vma, 2, vmf->flags)) + folio_free_swap(folio); + folio_unlock(folio); + folio_put(folio); /* Drop the cache lookup, not the PTE reference. */ + put_swap_device(si); + if (vmf->flags & FAULT_FLAG_WRITE) { + ret |= do_wp_page(vmf); + return ret & VM_FAULT_ERROR ? ret & VM_FAULT_ERROR : ret; + } + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + return ret; +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + if (slot) + mm_subpage_put(slot); + folio_unlock(folio); +out_folio: + folio_put(folio); +out_device: + put_swap_device(si); + return ret; +} +#endif + /* * We enter with non-exclusive mmap_lock (to exclude vma changes, * but allow concurrent faults), and pte mapped but not yet locked. @@ -4780,6 +5518,209 @@ static void check_swap_exclusive(struct folio *folio, swp_entry_t entry, * We return with the mmap_lock locked or unlocked in the same cases * as does filemap_fault(). */ +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SWAP) +/* One coarse PTE owns a naturally aligned group of native swap entries. */ +vm_fault_t do_swap_coarse_page(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + /* Arm64's largest base user page is 64K, its smallest native page 4K. */ + struct folio *cache[SZ_64K / PAGE_SIZE]; + unsigned int nr = mm_pte_native_pages(mm), covered = 0, count = 0, i; + swp_entry_t entry = softleaf_from_pte(vmf->orig_pte), next; + struct swap_info_struct *si; + struct folio *backing = NULL; + struct page *page; + unsigned int start; + bool copied = false, exclusive, mapped = false; + vm_fault_t ret = 0; + pte_t pte; + + vmf->pte = NULL; + if (WARN_ON_ONCE(nr <= 1 || nr > ARRAY_SIZE(cache) || + !IS_ALIGNED(swp_offset(entry), nr))) + return VM_FAULT_SIGBUS; + si = get_swap_device(entry); + if (!si) + return 0; + + while (covered < nr) { + struct folio *folio; + unsigned int take; + vm_fault_t wait; + + next = entry; + next.val += covered; + folio = swap_cache_get_folio(next); + if (!folio) { + folio = read_swap_cache_async(next, GFP_HIGHUSER_MOVABLE, vma, + vmf->address + covered * PAGE_SIZE, NULL); + if (!(ret & VM_FAULT_MAJOR)) { + ret |= VM_FAULT_MAJOR; + count_vm_event(PGMAJFAULT); + count_memcg_event_mm(mm, PGMAJFAULT); + } + } + if (!folio) + goto oom; + if (!count) { + wait = folio_lock_or_retry(folio, vmf); + if (wait & VM_FAULT_RETRY) { + folio_put(folio); + ret |= wait; + goto out; + } + } else if (!folio_trylock(folio)) { + /* Drop earlier cache locks before waiting for this I/O or owner. */ + for (i = 0; i < count; i++) { + folio_unlock(cache[i]); + folio_put(cache[i]); + } + count = 0; + covered = 0; + wait = folio_lock_or_retry(folio, vmf); + if (wait & VM_FAULT_RETRY) { + folio_put(folio); + ret |= wait; + goto out; + } + folio_unlock(folio); + folio_put(folio); + cond_resched(); + continue; + } + cache[count++] = folio; + if (!folio_matches_swap_entry(folio, next)) + goto out; + if (!folio_test_uptodate(folio)) { + ret = VM_FAULT_SIGBUS; + goto out; + } + start = swp_offset(next) - swp_offset(folio->swap); + take = min(nr - covered, folio_nr_pages(folio) - start); + for (i = 0; i < take; i++) { + if (PageHWPoison(folio_page(folio, start + i))) { + ret = VM_FAULT_HWPOISON; + goto out; + } + } + arch_swap_restore(folio_swap(next, folio), folio); + covered += take; + } + + start = swp_offset(entry) - swp_offset(cache[0]->swap); + if (count == 1 && IS_ALIGNED(start, nr) && + (folio_test_anon(cache[0]) || folio_nr_pages(cache[0]) == nr)) { + backing = cache[0]; + page = folio_page(backing, start); + } else { + backing = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, ilog2(nr), vma, vmf->address); + if (!backing) + goto oom; + copied = true; + if (mem_cgroup_swapin_charge_folio(backing, mm, GFP_KERNEL, entry)) + goto oom; + covered = 0; + for (i = 0; i < count; i++) { + unsigned int j, take; + + start = swp_offset(entry) + covered - swp_offset(cache[i]->swap); + take = min(nr - covered, folio_nr_pages(cache[i]) - start); + for (j = 0; j < take; j++) { + struct page *dst = folio_page(backing, covered + j); + struct page *src = folio_page(cache[i], start + j); + + if (copy_mc_user_highpage(dst, src, + vmf->address + (covered + j) * PAGE_SIZE, vma)) { + ret = VM_FAULT_HWPOISON; + goto out; + } + kmsan_copy_page_meta(dst, src); + } + covered += take; + } + folio_set_dirty(backing); + __folio_mark_uptodate(backing); + page = &backing->page; + } + + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + goto out; + exclusive = copied || pte_swp_exclusive(vmf->orig_pte); + if (!copied && exclusive) { + check_swap_exclusive(backing, entry, nr); + if (folio_test_writeback(backing) && data_race(si->flags & SWP_STABLE_WRITES)) + exclusive = false; + } + pte = mk_pte(page, vma->vm_page_prot); + if (pte_swp_soft_dirty(vmf->orig_pte)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(vmf->orig_pte)) + pte = pte_mkuffd_wp(pte); + if (exclusive && (vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && + !pte_needs_soft_dirty_wp(vma, pte)) { + pte = pte_mkwrite(pte, vma); + if (vmf->flags & FAULT_FLAG_WRITE) { + pte = pte_mkdirty(pte); + vmf->flags &= ~FAULT_FLAG_WRITE; + } + } + folio_ref_add(backing, nr - copied); + if (!folio_test_anon(backing)) + folio_add_new_anon_rmap(backing, vma, vmf->address, + exclusive ? RMAP_EXCLUSIVE : RMAP_NONE); + else + folio_add_anon_rmap_pte(backing, page, vma, vmf->address, + exclusive ? RMAP_EXCLUSIVE : RMAP_NONE); + if (copied) + folio_add_lru_vma(backing, vma); + inc_mm_counter(mm, MM_ANONPAGES); + dec_mm_counter(mm, MM_SWAPENTS); + flush_icache_pages(vma, page, nr); + vmf->orig_pte = pte; + set_pte_at(mm, vmf->address, vmf->pte, pte); + arch_do_swap_page_nr(mm, vma, vmf->address, pte, pte, 1); + mapped = true; + covered = 0; + for (i = 0; i < count; i++) { + unsigned int take; + + start = swp_offset(entry) + covered - swp_offset(cache[i]->swap); + take = min(nr - covered, folio_nr_pages(cache[i]) - start); + folio_put_swap_range(cache[i], folio_page(cache[i], start), take); + covered += take; + } + goto out; + +oom: + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + ret |= VM_FAULT_OOM; +out: + if (copied && !mapped) + folio_put(backing); + for (i = 0; i < count; i++) { + if (mapped) + folio_free_swap(cache[i]); + folio_unlock(cache[i]); + folio_put(cache[i]); + } + if (mapped && (vmf->flags & FAULT_FLAG_WRITE)) { + ret |= do_wp_page(vmf); + vmf->pte = NULL; /* do_wp_page drops the PT lock. */ + } else if (mapped) { + update_mmu_cache(vma, vmf->address, vmf->pte); + } + if (vmf->pte) + pte_unmap_unlock(vmf->pte, vmf->ptl); + put_swap_device(si); + if (!mapped && !(ret & (VM_FAULT_ERROR | VM_FAULT_RETRY))) + cond_resched(); + return ret; +} +#endif + vm_fault_t do_swap_page(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -4854,6 +5795,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) goto out; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return do_swap_subpage(vmf, entry); +#ifdef CONFIG_SWAP + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + return do_swap_coarse_page(vmf); +#endif +#endif + /* Prevent swapoff from happening to us. */ si = get_swap_device(entry); if (unlikely(!si)) @@ -5291,6 +6241,7 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, bool uffd_wp) { const unsigned int nr_pages = folio_nr_pages(folio); + const unsigned int nr_ptes = folio_size(folio) >> mm_page_shift(vma->vm_mm); pte_t entry = folio_mk_pte(folio, vma->vm_page_prot); entry = pte_sw_mkyoung(entry); @@ -5303,8 +6254,8 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, folio_ref_add(folio, nr_pages - 1); folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); - set_ptes(vma->vm_mm, addr, pte, entry, nr_pages); - update_mmu_cache_range(NULL, vma, addr, pte, nr_pages); + set_ptes(vma->vm_mm, addr, pte, entry, nr_ptes); + update_mmu_cache_range(NULL, vma, addr, pte, nr_ptes); } static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, @@ -5313,10 +6264,123 @@ static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, const unsigned int order = folio_order(folio); map_anon_folio_pte_nopf(folio, pte, vma, addr, uffd_wp); - add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1L << order); + add_mm_counter(vma->vm_mm, MM_ANONPAGES, folio_size(folio) >> mm_page_shift(vma->vm_mm)); count_mthp_stat(order, MTHP_STAT_ANON_FAULT_ALLOC); } +#ifdef CONFIG_MM_SUBPAGE +static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slots[PAGE_SIZE / MM_SUBPAGE_SIZE] = {}; + struct vm_page_offset pos = vma_page_offset_at(vma, vmf->address); + struct mm_subpage_pool *pool; + struct folio *folio; + unsigned long start, end, lower; + unsigned int first_offset, nr, target, i, mapped = 0; + vm_fault_t ret = VM_FAULT_OOM; + pte_t *first; + int err; + + /* One native linear index, clipped to the VMA and this PTE table. */ + lower = max(vma->vm_start, vmf->address & pmd_mask_mm(mm)); + start = vmf->address - min_t(unsigned long, pos.offset, vmf->address - lower); + end = vmf->address + min(PAGE_SIZE - pos.offset, + pmd_addr_end_mm(mm, vmf->address, vma->vm_end) - vmf->address); + first_offset = pos.offset - (vmf->address - start); + nr = (end - start) >> mm_page_shift(mm); + target = (vmf->address - start) >> mm_page_shift(mm); + + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return VM_FAULT_OOM; + /* Supply initialises every byte once, including the unissued slots. */ + folio = folio_prealloc(mm, vma, vmf->address, false); + if (!folio) + goto release; + err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (err) { + folio_put(folio); + goto release; + } + for (i = 0; i < nr; i++) { + slots[i] = mm_subpage_alloc_at(pool, first_offset + i * mm_page_size(mm)); + if (IS_ERR(slots[i])) { + slots[i] = NULL; + goto release; + } + } + + /* PTE references may disappear as soon as we drop the PTL. */ + folio_get(folio); + folio_lock(folio); + ret = 0; + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (vmf_pte_changed(vmf)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock_pte; + } + ret = check_stable_address_space(mm); + if (ret) + goto unlock_pte; + if (userfaultfd_missing(vma)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + folio_unlock(folio); + folio_put(folio); + ret = handle_userfault(vmf, VM_UFFD_MISSING); + goto release; + } + + first = vmf->pte - target; + for (i = 0; i < nr; i++) { + unsigned long address = start + i * mm_page_size(mm); + pte_t entry; + + /* Do not replace existing mappings or userfaultfd markers nearby. */ + if (i != target && !pte_none(ptep_get(first + i))) + continue; + mm_subpage_set_exclusive(slots[i]); + err = mm_subpage_add_new_anon_rmap(slots[i], vma, address); + if (WARN_ON_ONCE(err)) { + ret = VM_FAULT_SIGBUS; + break; + } + entry = phys_pte_mm(mm, mm_subpage_phys(slots[i]), vma->vm_page_prot); + entry = pte_sw_mkyoung(entry); + if (vma->vm_flags & VM_WRITE) + entry = pte_mkwrite(pte_mkdirty(entry), vma); + if (i == target && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + set_pte_at(mm, address, first + i, entry); + update_mmu_cache(vma, address, first + i); + slots[i] = NULL; /* The PTE now owns the reference. */ + mapped++; + } + if (mapped) { + subpage_add_new_lru(folio, vma, mapped); + add_mm_counter(mm, MM_ANONPAGES, mapped); + count_mthp_stat(0, MTHP_STAT_ANON_FAULT_ALLOC); + } +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + folio_unlock(folio); + folio_put(folio); +release: + mm_subpage_pool_close(pool); + for (i = 0; i < ARRAY_SIZE(slots); i++) + if (slots[i]) + mm_subpage_put(slots[i]); + mm_subpage_pool_put(pool); + return ret; +} +#endif + /* * We enter with non-exclusive mmap_lock (to exclude vma changes, * but allow concurrent faults), and pte mapped but not yet locked. @@ -5376,6 +6440,10 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) ret = vmf_anon_prepare(vmf); if (ret) return ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return do_anonymous_subpage(vmf); +#endif /* Returns NULL on OOM or ERR_PTR(-EAGAIN) if we must retry the fault */ folio = alloc_anon_folio(vmf); if (IS_ERR(folio)) @@ -5383,8 +6451,8 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) if (!folio) goto oom; - nr_pages = folio_nr_pages(folio); - addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE); + nr_pages = folio_size(folio) >> mm_page_shift(vma->vm_mm); + addr = ALIGN_DOWN(vmf->address, folio_size(folio)); /* * The memory barrier inside __folio_mark_uptodate makes sure that @@ -5517,6 +6585,10 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa * PMD mappings if THPs are disabled. As we already have a THP, * behave as if we are forcing a collapse. */ + /* Match the native PTE fault policy: partial data exposes its zero subunits. */ + if (userfaultfd_missing(vma) && shmem_uffd_has_missing_native_page(folio)) + return ret; + if (thp_disabled_by_hw() || vma_thp_disabled(vma, vma->vm_flags, /* forced_collapse=*/ true)) return ret; @@ -5558,6 +6630,8 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); add_mm_counter(vma->vm_mm, mm_counter_file(folio), HPAGE_PMD_NR); + shmem_uffd_mark_mapped(folio, folio_page_idx(folio, page) << PAGE_SHIFT, + HPAGE_PMD_SIZE); folio_add_file_rmap_pmd(folio, page, vma); /* @@ -5620,6 +6694,8 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); } else { + shmem_uffd_mark_mapped(folio, folio_page_idx(folio, page) << PAGE_SHIFT, + (unsigned long)nr << PAGE_SHIFT); folio_add_file_rmap_ptes(folio, page, nr, vma); } set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr); @@ -5636,6 +6712,252 @@ static bool vmf_pte_changed(struct vm_fault *vmf) return !pte_none(ptep_get(vmf->pte)); } +#ifdef CONFIG_MM_SUBPAGE +static bool folio_user_range_valid(struct folio *folio, unsigned long offset, + unsigned long size) +{ + return offset <= folio_size(folio) && size <= folio_size(folio) - offset && + IS_ALIGNED(page_to_phys(&folio->page) + offset, size); +} + +/* + * Map the intersection of a locked cache folio, file-index range, VMA and + * the fault's PTE table. vmf->pte points at vmf->address and its lock is held. + * Each PTE owns a reference per covered native page (one for a subpage). + * The caller's reference is untouched, including when no PTE is installed. + */ +vm_fault_t map_file_user_pages(struct vm_fault *vmf, struct folio *folio, + pgoff_t first, pgoff_t last, bool fault) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + unsigned long addr, end, table_start = vmf->address & pmd_mask_mm(mm); + unsigned long size = mm_page_size(mm); + loff_t file_size = i_size_read(vma->vm_file->f_mapping->host); + vm_fault_t ret = 0; + unsigned int mapped = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + first = max(first, folio->index); + last = min(last, folio_next_index(folio) - 1); + if (first > last) + return 0; + addr = vma_address(vma, first, last - first + 1); + if (IS_ERR_VALUE(addr)) + return 0; + addr = ALIGN(max(addr, table_start), size); + end = pmd_addr_end_mm(mm, vmf->address, vma->vm_end); + if (userfaultfd_armed(vma)) { + addr = max(addr, vmf->address); + end = min(end, vmf->address + size); + } + for (; addr < end; addr += size) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + struct page *page; + unsigned long offset; + unsigned int i, nr = mm_pte_native_pages(mm); + bool poisoned = false; + pte_t entry, *ptep; + + if (pos.index > last || !vma_file_offset_valid(vma, addr, file_size)) + break; + offset = ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset; + /* + * A coarse leaf cannot span separate cache folios or a partial + * requested index range, even if neighboring PFNs happen to match. + */ + if (nr - 1 > last - pos.index || + !folio_user_range_valid(folio, offset, size)) + continue; + /* A native cache folio can contain independently missing leaves. */ + if (userfaultfd_missing(vma) && + shmem_uffd_range_missing(folio, pos.index, pos.offset, size)) + continue; + page = folio_file_page(folio, pos.index); + ptep = vmf->pte + ((long)(addr - vmf->address) >> mm_page_shift(mm)); + for (i = 0; i < nr; i++) + poisoned |= PageHWPoison(page + i); + if (poisoned) + continue; + if (!pte_none(ptep_get(ptep)) && + !(fault && addr == vmf->address && !vmf_pte_changed(vmf))) + continue; + flush_icache_pages(vma, page, nr); + entry = phys_pte_mm(mm, page_to_phys(page) + pos.offset, vma->vm_page_prot); + if (addr != vmf->address && arch_wants_old_prefaulted_pte()) + entry = pte_mkold(entry); + else + entry = pte_sw_mkyoung(entry); + if (vmf->flags & FAULT_FLAG_WRITE) + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + else if (pte_write(entry) && folio_test_dirty(folio)) + entry = pte_mkdirty(entry); + if (fault && addr == vmf->address && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_ref_add(folio, nr); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, entry)); + shmem_uffd_mark_mapped(folio, + ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset, size); + folio_add_file_rmap_pte(folio, page, vma); + set_pte_at(mm, addr, ptep, entry); + update_mmu_cache_range(vmf, vma, addr, ptep, 1); + mapped++; + if (addr == vmf->address) + ret = VM_FAULT_NOPAGE; + } + add_mm_counter(mm, mm_counter_file(folio), mapped); + return ret; +} + +/* + * A non-file fault callback supplies one native page, not a cache-indexed + * folio. In particular, VVAR callbacks may reorder native pages. Only map + * quarters of this supplied page; never infer adjacent pages from its folio. + */ +static vm_fault_t map_special_subpages(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct page *page = vmf->page; + struct folio *folio = page_folio(page); + unsigned int offset = vma_page_offset_at(vma, vmf->address).offset; + unsigned long size = mm_page_size(mm), addr, end; + vm_fault_t ret = 0; + unsigned int mapped = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (PageHWPoison(page)) + return 0; + addr = offset > vmf->address ? vma->vm_start : + max(vma->vm_start, vmf->address - offset); + addr = max(addr, vmf->address & pmd_mask_mm(mm)); + end = min(vmf->address + PAGE_SIZE - offset, + pmd_addr_end_mm(mm, vmf->address, vma->vm_end)); + if (userfaultfd_armed(vma)) { + addr = vmf->address; + end = vmf->address + size; + } + flush_icache_pages(vma, page, 1); + for (; addr < end; addr += size) { + unsigned int physical_offset = addr < vmf->address ? + offset - (vmf->address - addr) : offset + (addr - vmf->address); + pte_t *ptep = vmf->pte + ((long)(addr - vmf->address) >> mm_page_shift(mm)); + pte_t entry; + + if (!pte_none(ptep_get(ptep)) && + !(addr == vmf->address && !vmf_pte_changed(vmf))) + continue; + entry = phys_pte_mm(mm, page_to_phys(page) + physical_offset, vma->vm_page_prot); + entry = pte_sw_mkyoung(entry); + if (vmf->flags & FAULT_FLAG_WRITE) + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + if (addr == vmf->address && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_get(folio); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, entry)); + folio_add_file_rmap_pte(folio, page, vma); + set_pte_at(mm, addr, ptep, entry); + update_mmu_cache_range(vmf, vma, addr, ptep, 1); + mapped++; + if (addr == vmf->address) + ret = VM_FAULT_NOPAGE; + } + add_mm_counter(mm, mm_counter_file(folio), mapped); + return ret; +} + +static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slot = vmf->cow_subpage; + bool is_cow = (vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED); + struct folio *folio = page_folio(is_cow ? vmf->cow_page : vmf->page); + vm_fault_t ret; + pte_t entry; + + if (!(vma->vm_flags & VM_SHARED)) { + ret = check_stable_address_space(mm); + if (ret) + return ret; + } + if (pmd_none(*vmf->pmd)) { + if (vmf->prealloc_pte) + pmd_install(mm, vmf->pmd, &vmf->prealloc_pte); + else if (pte_alloc(mm, vmf->pmd)) + return VM_FAULT_OOM; + } + /* The private destination is not yet visible through a PTE. */ + if (is_cow) { + folio_get(folio); + folio_lock(folio); + } + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + ret = VM_FAULT_NOPAGE; + if (!vmf->pte) + goto unlock_folio; + if (vmf_pte_changed(vmf)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock; + } + if (slot) { + mm_subpage_set_exclusive(slot); + if (WARN_ON_ONCE(mm_subpage_add_new_anon_rmap(slot, vma, vmf->address))) { + ret = VM_FAULT_SIGBUS; + goto unlock; + } + entry = phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot); + entry = maybe_mkwrite(pte_mkdirty(pte_sw_mkyoung(entry)), vma); + if (vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + subpage_add_new_lru(folio, vma, 1); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + inc_mm_counter(mm, MM_ANONPAGES); + vmf->cow_subpage = NULL; + ret = 0; + } else if (is_cow) { + if (!folio_user_range_valid(folio, 0, mm_page_size(mm))) { + ret = VM_FAULT_SIGBUS; + goto unlock; + } + entry = phys_pte_mm(mm, page_to_phys(&folio->page), vma->vm_page_prot); + entry = maybe_mkwrite(pte_mkdirty(pte_sw_mkyoung(entry)), vma); + if (vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_ref_add(folio, mm_pte_native_pages(mm) - 1); + folio_add_new_anon_rmap(folio, vma, vmf->address, RMAP_EXCLUSIVE); + folio_add_lru_vma(folio, vma); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + inc_mm_counter(mm, MM_ANONPAGES); + ret = 0; + } else { + if (vma->vm_file) + ret = map_file_user_pages(vmf, folio, folio->index, + folio_next_index(folio) - 1, true); + else if (mm_page_size(mm) < PAGE_SIZE) + ret = map_special_subpages(vmf); + else + ret = VM_FAULT_SIGBUS; + if (ret & VM_FAULT_NOPAGE) { + folio_put(folio); /* Consume the fault callback's reference. */ + ret = 0; + } else { + ret = VM_FAULT_SIGBUS; + } + } +unlock: + pte_unmap_unlock(vmf->pte, vmf->ptl); +unlock_folio: + if (is_cow) { + folio_unlock(folio); + folio_put(folio); + } + return ret; +} +#endif + /** * finish_fault - finish page fault once we have prepared the page to fault * @@ -5663,6 +6985,11 @@ vm_fault_t finish_fault(struct vm_fault *vmf) unsigned long addr; bool needs_fallback = false; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return finish_user_page_fault(vmf); +#endif + fallback: addr = vmf->address; @@ -5768,13 +7095,12 @@ vm_fault_t finish_fault(struct vm_fault *vmf) return ret; } -static unsigned long fault_around_pages __read_mostly = - 65536 >> PAGE_SHIFT; +static unsigned long fault_around_bytes __read_mostly = SZ_64K; #ifdef CONFIG_DEBUG_FS static int fault_around_bytes_get(void *data, u64 *val) { - *val = fault_around_pages << PAGE_SHIFT; + *val = READ_ONCE(fault_around_bytes); return 0; } @@ -5788,11 +7114,13 @@ static int fault_around_bytes_set(void *data, u64 val) return -EINVAL; /* - * The minimum value is 1 page, however this results in no fault-around - * at all. See should_fault_around(). + * Keep the policy in bytes: one native page can still contain several + * user leaves. The smallest supported leaf disables fault-around for + * every mm; larger values are compared with each mm's own granule. */ - val = max(val, PAGE_SIZE); - fault_around_pages = rounddown_pow_of_two(val) >> PAGE_SHIFT; + val = max_t(u64, val, IS_ENABLED(CONFIG_MM_SUBPAGE) ? + MM_SUBPAGE_SIZE : PAGE_SIZE); + WRITE_ONCE(fault_around_bytes, rounddown_pow_of_two(val)); return 0; } @@ -5819,24 +7147,42 @@ late_initcall(fault_around_debugfs); * This function doesn't cross VMA or page table boundaries, in order to call * map_pages() and acquire a PTE lock only once. * - * fault_around_pages defines how many pages we'll try to map. - * do_fault_around() expects it to be set to a power of two less than or equal - * to PTRS_PER_PTE. - * - * The virtual address of the area that we map is naturally aligned to - * fault_around_pages * PAGE_SIZE rounded down to the machine page size - * (and therefore to page order). This way it's easier to guarantee - * that we don't cross page table boundaries. + * fault_around_bytes is a power-of-two byte policy, independent of either + * physical or user granules. The native path converts it to native pages; + * the alternative path clamps it to the target mm's PTE-table coverage. + * Natural alignment keeps the window inside one page table. */ static vm_fault_t do_fault_around(struct vm_fault *vmf) { - pgoff_t nr_pages = READ_ONCE(fault_around_pages); + unsigned long configured_bytes = READ_ONCE(fault_around_bytes); + pgoff_t nr_pages = max(1UL, configured_bytes >> PAGE_SHIFT); pgoff_t pte_off = pte_index(vmf->address); /* The page offset of vmf->address within the VMA. */ pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff; pgoff_t from_pte, to_pte; vm_fault_t ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) != PAGE_SIZE) { + struct vm_area_struct *vma = vmf->vma; + unsigned long bytes = min(configured_bytes, + pmd_size_mm(vma->vm_mm)); + unsigned long start = max(ALIGN_DOWN(vmf->address, bytes), vma->vm_start); + unsigned long end = min(ALIGN_DOWN(vmf->address, bytes) + bytes, vma->vm_end); + + if (pmd_none(*vmf->pmd)) { + vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); + if (!vmf->prealloc_pte) + return VM_FAULT_OOM; + } + rcu_read_lock(); + ret = vma->vm_ops->map_pages(vmf, vma_page_offset_at(vma, start).index, + vma_page_offset_at(vma, end - 1).index); + rcu_read_unlock(); + return ret; + } +#endif + /* The PTE offset of the start address, clamped to the VMA. */ from_pte = max(ALIGN_DOWN(pte_off, nr_pages), pte_off - min(pte_off, vma_off)); @@ -5871,7 +7217,7 @@ static inline bool should_fault_around(struct vm_fault *vmf) return false; /* A single page implies no faulting 'around' at all. */ - return fault_around_pages > 1; + return READ_ONCE(fault_around_bytes) > mm_page_size(vmf->vma->vm_mm); } static vm_fault_t do_read_fault(struct vm_fault *vmf) @@ -5918,10 +7264,26 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) if (ret) return ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + vmf->cow_subpage = anon_subpage_prealloc(vma->vm_mm, vma, vmf->address); + if (IS_ERR(vmf->cow_subpage)) { + vmf->cow_subpage = NULL; + return VM_FAULT_OOM; + } + folio = mm_subpage_folio(vmf->cow_subpage); + goto allocated; + } +#endif + folio = folio_prealloc(vma->vm_mm, vma, vmf->address, false); if (!folio) return VM_FAULT_OOM; +#ifdef CONFIG_MM_SUBPAGE +allocated: +#endif + vmf->cow_page = &folio->page; ret = __do_fault(vmf); @@ -5930,10 +7292,38 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) if (ret & VM_FAULT_DONE_COW) return ret; - if (copy_mc_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma)) { - ret = VM_FAULT_HWPOISON; +#ifdef CONFIG_MM_SUBPAGE + if (vmf->cow_subpage) { + if (copy_user_subpage_range(vmf->cow_page, mm_subpage_offset(vmf->cow_subpage), + vmf->page, vma_page_offset_at(vma, vmf->address).offset, + mm_page_size(vma->vm_mm))) { + ret = VM_FAULT_HWPOISON; + goto unlock; + } + goto copied; + } +#endif + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) > PAGE_SIZE && + !folio_user_range_valid(page_folio(vmf->page), + folio_page_idx(page_folio(vmf->page), vmf->page) << PAGE_SHIFT, + mm_page_size(vma->vm_mm))) { + ret = VM_FAULT_SIGBUS; goto unlock; } +#endif + for (unsigned int i = 0; i < mm_pte_native_pages(vma->vm_mm); i++) { + if (copy_mc_user_highpage(vmf->cow_page + i, vmf->page + i, + vmf->address + i * PAGE_SIZE, vma)) { + ret = VM_FAULT_HWPOISON; + goto unlock; + } + } +#ifdef CONFIG_MM_SUBPAGE +copied: +#endif + __folio_mark_uptodate(folio); ret |= finish_fault(vmf); @@ -5944,6 +7334,13 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) goto uncharge_out; return ret; uncharge_out: +#ifdef CONFIG_MM_SUBPAGE + if (vmf->cow_subpage) { + mm_subpage_put(vmf->cow_subpage); + vmf->cow_subpage = NULL; + return ret; + } +#endif folio_put(folio); return ret; } @@ -6104,31 +7501,33 @@ static void numa_rebuild_single_mapping(struct vm_fault *vmf, struct vm_area_str update_mmu_cache_range(vmf, vma, fault_addr, fault_pte, 1); } -static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, +static void numa_rebuild_folio_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, struct folio *folio, pte_t fault_pte, bool ignore_writable, bool pte_write_upgrade) { - int nr = pte_pfn(fault_pte) - folio_pfn(folio); + struct mm_struct *mm = vma->vm_mm; + unsigned int page_shift = mm_page_shift(mm); + unsigned long page_size = 1UL << page_shift; + unsigned long offset = pte_phys_mm(mm, fault_pte) - PFN_PHYS(folio_pfn(folio)); unsigned long start, end, addr = vmf->address; - unsigned long addr_start = addr - (nr << PAGE_SHIFT); - unsigned long pt_start = ALIGN_DOWN(addr, PMD_SIZE); + unsigned long lower = max(addr & pmd_mask_mm(mm), vma->vm_start); + unsigned long upper = pmd_addr_end_mm(mm, addr, vma->vm_end); pte_t *start_ptep; - /* Stay within the VMA and within the page table. */ - start = max3(addr_start, pt_start, vma->vm_start); - end = min3(addr_start + folio_size(folio), pt_start + PMD_SIZE, - vma->vm_end); - start_ptep = vmf->pte - ((addr - start) >> PAGE_SHIFT); + /* Stay within the VMA, page table and physical owner without wrapping. */ + start = addr - min(offset, addr - lower); + end = addr + min(folio_size(folio) - offset, upper - addr); + start_ptep = vmf->pte - ((addr - start) >> page_shift); - /* Restore all PTEs' mapping of the large folio */ - for (addr = start; addr != end; start_ptep++, addr += PAGE_SIZE) { + /* Restore the owner's PTEs, including smaller leaves in a native folio. */ + for (addr = start; addr != end; start_ptep++, addr += page_size) { pte_t ptent = ptep_get(start_ptep); bool writable = false; if (!pte_present(ptent) || !pte_protnone(ptent)) continue; - if (pfn_folio(pte_pfn(ptent)) != folio) + if (pfn_folio(PHYS_PFN(pte_phys_mm(mm, ptent))) != folio) continue; if (!ignore_writable) { @@ -6221,8 +7620,8 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) * Make it present again, depending on how arch implements * non-accessible ptes, some can allow access by kernel mode. */ - if (folio && folio_test_large(folio)) - numa_rebuild_large_mapping(vmf, vma, folio, pte, ignore_writable, + if (folio && (folio_test_large(folio) || mm_page_shift(vma->vm_mm) < PAGE_SHIFT)) + numa_rebuild_folio_mapping(vmf, vma, folio, pte, ignore_writable, pte_write_upgrade); else numa_rebuild_single_mapping(vmf, vma, vmf->address, vmf->pte, @@ -6455,7 +7854,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, { struct vm_fault vmf = { .vma = vma, - .address = address & PAGE_MASK, + .address = address & mm_page_mask(vma->vm_mm), .real_address = address, .flags = flags, .pgoff = linear_page_index(vma, address), @@ -6786,7 +8185,7 @@ int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) return -ENOMEM; spin_lock(&mm->page_table_lock); - if (!p4d_present(*p4d)) { + if (!p4d_present_mm(mm, *p4d)) { mm_inc_nr_puds(mm); smp_wmb(); /* See comment in pmd_install() */ p4d_populate(mm, p4d, new); @@ -6824,16 +8223,19 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) static inline int pfnmap_args_setup(struct follow_pfnmap_args *args, spinlock_t *lock, pte_t *ptep, - pgprot_t pgprot, unsigned long pfn_base, + pgprot_t pgprot, phys_addr_t phys_base, unsigned long addr_mask, bool writable, bool special) { + phys_addr_t phys = phys_base + (args->address & ~addr_mask); + if (!writable && args->write) return -EFAULT; args->lock = lock; args->ptep = ptep; - args->pfn = pfn_base + ((args->address & ~addr_mask) >> PAGE_SHIFT); + args->pfn = PHYS_PFN(phys); + args->offset = offset_in_page(phys); args->addr_mask = addr_mask; args->pgprot = pgprot; args->writable = writable; @@ -6885,8 +8287,14 @@ static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma) * * This function must not be used to modify PTE content. * + * Legacy callers retain the native-page contract. A caller setting + * allow_subpage must use pfn plus offset to identify the requested physical + * byte and must limit access to the leaf described by addr_mask. This does + * not pin the native page or grant access to neighbouring user mappings. + * * Return: zero on success, -EFAULT if @args->write was set but the - * mapping is not writable, -EINVAL if there is no mapping at all. + * mapping is not writable, -EINVAL if there is no mapping at all, or + * -EOPNOTSUPP for a non-native granule without allow_subpage. */ int follow_pfnmap_start(struct follow_pfnmap_args *args) { @@ -6908,17 +8316,20 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) goto out; + /* A legacy PFN consumer cannot represent partial native-page ownership. */ + if (mm_page_size(mm) != PAGE_SIZE && !args->allow_subpage) + return -EOPNOTSUPP; retry: pgdp = pgd_offset(mm, address); if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) goto out; - p4dp = p4d_offset(pgdp, address); + p4dp = p4d_offset_mm(mm, pgdp, address); p4d = p4dp_get(p4dp); - if (p4d_none(p4d) || unlikely(p4d_bad(p4d))) + if (p4d_none_mm(mm, p4d) || unlikely(p4d_bad_mm(mm, p4d))) goto out; - pudp = pud_offset(p4dp, address); + pudp = pud_offset_mm(mm, p4dp, address); pud = pudp_get(pudp); if (!pud_present(pud)) goto out; @@ -6934,14 +8345,14 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) goto retry; } r = pfnmap_args_setup(args, lock, NULL, pud_pgprot(pud), - pud_pfn(pud), PUD_MASK, pud_write(pud), + PFN_PHYS(pud_pfn(pud)), pud_mask_mm(mm), pud_write(pud), pud_special(pud)); if (r) spin_unlock(lock); return r; } - pmdp = pmd_offset(pudp, address); + pmdp = pmd_offset_mm(mm, pudp, address); pmd = pmdp_get_lockless(pmdp); if (!pmd_present(pmd)) goto out; @@ -6957,7 +8368,7 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) goto retry; } r = pfnmap_args_setup(args, lock, NULL, pmd_pgprot(pmd), - pmd_pfn(pmd), PMD_MASK, pmd_write(pmd), + PFN_PHYS(pmd_pfn(pmd)), pmd_mask_mm(mm), pmd_write(pmd), pmd_special(pmd)); if (r) spin_unlock(lock); @@ -6970,8 +8381,8 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) pte = ptep_get(ptep); if (!pte_present(pte)) goto unlock; - r = pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte), - pte_pfn(pte), PAGE_MASK, pte_write(pte), + r = pfnmap_args_setup(args, lock, ptep, pte_pgprot_mm(mm, pte), + pte_phys_mm(mm, pte), mm_page_mask(mm), pte_write(pte), pte_special(pte)); if (r) goto unlock; @@ -7018,20 +8429,30 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, resource_size_t phys_addr; pgprot_t prot = __pgprot(0); void __iomem *maddr; - int offset = offset_in_page(addr); + int offset; int ret = -EINVAL; bool writable; struct follow_pfnmap_args args = { .vma = vma, .address = addr, - .write = !!(write & FOLL_WRITE) + .write = !!(write & FOLL_WRITE), + .allow_subpage = true }; + if (len <= 0) + return 0; + if (addr < vma->vm_start || addr >= vma->vm_end) + return -EINVAL; + /* Each following leaf needs its own translation and permission check. */ + len = min_t(unsigned long, len, vma->vm_end - addr); + len = min_t(unsigned long, len, mm_page_size(vma->vm_mm) - + (addr & ~mm_page_mask(vma->vm_mm))); retry: if (follow_pfnmap_start(&args)) return -EINVAL; prot = args.pgprot; phys_addr = (resource_size_t)args.pfn << PAGE_SHIFT; + offset = args.offset; writable = args.writable; follow_pfnmap_end(&args); @@ -7043,7 +8464,8 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, goto out_unmap; if ((pgprot_val(prot) != pgprot_val(args.pgprot)) || - (phys_addr != (args.pfn << PAGE_SHIFT)) || + (phys_addr != ((resource_size_t)args.pfn << PAGE_SHIFT)) || + (offset != args.offset) || (writable != args.writable)) { follow_pfnmap_end(&args); iounmap(maddr); @@ -7089,8 +8511,19 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, void *maddr; struct folio *folio; struct vm_area_struct *vma = NULL; - struct page *page = get_user_page_vma_remote(mm, addr, - gup_flags, &vma); + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment = {}; + + if (mm_page_size(mm) != PAGE_SIZE) { + int got = get_user_fragment_vma_remote(mm, addr, len, + gup_flags, &fragment, &vma); + + page = got > 0 ? folio_page(fragment.folio, + fragment.offset >> PAGE_SHIFT) : ERR_PTR(got ?: -EFAULT); + } else +#endif + page = get_user_page_vma_remote(mm, addr, gup_flags, &vma); if (IS_ERR(page)) { /* We might need to expand the stack to access it */ @@ -7124,6 +8557,12 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, offset = addr & (PAGE_SIZE-1); if (bytes > PAGE_SIZE-offset) bytes = PAGE_SIZE-offset; +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) { + offset = offset_in_page(fragment.offset); + bytes = fragment.length; + } +#endif maddr = kmap_local_folio(folio, folio_page_idx(folio, page) * PAGE_SIZE); if (write) { @@ -7134,7 +8573,13 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, copy_from_user_page(vma, page, addr, buf, maddr + offset, bytes); } - folio_release_kmap(folio, maddr); +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) { + kunmap_local(maddr); + release_user_fragments(&fragment, 1, false); + } else +#endif + folio_release_kmap(folio, maddr); } len -= bytes; buf += bytes; diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 4e4421b22b59f..3b97c71c9d3cf 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -78,6 +78,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -688,6 +689,8 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, struct folio *folio; struct queue_pages *qp = walk->private; unsigned long flags = qp->flags; + unsigned int page_shift = mm_page_shift(walk->mm); + unsigned long page_size = 1UL << page_shift; pte_t *pte, *mapped_pte; pte_t ptent; spinlock_t *ptl; @@ -705,8 +708,8 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte += nr, addr += nr * PAGE_SIZE) { - max_nr = (end - addr) >> PAGE_SHIFT; + for (; addr != end; pte += nr, addr += nr * page_size) { + max_nr = (end - addr) >> page_shift; nr = 1; ptent = ptep_get(pte); if (pte_none(ptent)) @@ -721,7 +724,11 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, folio = vm_normal_folio(vma, addr, ptent); if (!folio || folio_is_zone_device(folio)) continue; - if (folio_test_large(folio) && max_nr != 1) +#ifdef CONFIG_MM_SUBPAGE + if (page_shift < PAGE_SHIFT) + nr = folio_subpage_pte_batch(vma, folio, pte, ptent, max_nr, 0); +#endif + if (page_shift >= PAGE_SHIFT && folio_test_large(folio) && max_nr != 1) nr = folio_pte_batch(folio, pte, ptent, max_nr); /* * vm_normal_folio() filters out zero pages, but there might @@ -731,9 +738,10 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, continue; if (!queue_folio_required(folio, qp)) continue; - if (folio_test_large(folio)) { + if (folio_test_large(folio) || page_shift < PAGE_SHIFT) { /* - * A large folio can only be isolated from LRU once, + * A native folio backing smaller leaves, like a large + * folio, can only be isolated from LRU once, * but may be mapped by many PTEs (and Copy-On-Write may * intersperse PTEs of other, order 0, folios). This is * a common case, so don't mistake it for failure (but @@ -755,7 +763,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) || !vma_migratable(vma) || !migrate_folio_add(folio, qp->pagelist, flags)) { - qp->nr_failed += nr; + qp->nr_failed += page_shift < PAGE_SHIFT ? 1 : nr; if (strictly_unmovable(flags)) break; } @@ -824,6 +832,7 @@ static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask, * @folio: The folio whose mapping considered for being made NUMA hintable * @vma: The VMA that the folio belongs to. * @is_private_single_threaded: Is this a single-threaded private VMA or not + * @shared: Is the mapped memory potentially shared between address spaces * * This function checks to see if the folio actually indicates that * we need to make the mapping one which causes a NUMA hinting fault, @@ -833,7 +842,7 @@ static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask, * Return: True if the mapping of the folio needs to be changed, false otherwise. */ bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, - bool is_private_single_threaded) + bool is_private_single_threaded, bool shared) { int nid; @@ -841,7 +850,7 @@ bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, return false; /* Also skip shared copy-on-write folios */ - if (is_cow_mapping(vma->vm_flags) && folio_maybe_mapped_shared(folio)) + if (is_cow_mapping(vma->vm_flags) && shared) return false; /* Folios are pinned and can't be migrated */ @@ -1050,6 +1059,12 @@ static int mbind_range(struct vma_iterator *vmi, struct vm_area_struct *vma, vmstart = vma->vm_start; } + /* Shared policy is indexed by native page-cache allocation units. */ + if (vma_is_shmem(vma) && + (vma_page_offset_at(vma, vmstart).offset || + ((vmend - vmstart) & ~PAGE_MASK))) + return -EINVAL; + if (mpol_equal(vma->vm_policy, new_pol)) { *prev = vma; return 0; @@ -1135,6 +1150,18 @@ static int lookup_node(struct mm_struct *mm, unsigned long addr) struct page *p = NULL; int ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_shift(mm) != PAGE_SHIFT) { + struct user_page_fragment fragment; + + ret = get_user_fragments_remote(mm, addr, 1, 0, &fragment, 1); + if (ret > 0) { + ret = folio_nid(fragment.folio); + release_user_fragments(&fragment, 1, false); + } + return ret; + } +#endif ret = get_user_pages_fast(addr & PAGE_MASK, 1, 0, &p); if (ret > 0) { ret = page_to_nid(p); @@ -1502,13 +1529,13 @@ static long do_mbind(unsigned long start, unsigned long len, if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) return -EPERM; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; if (mode == MPOL_DEFAULT) flags &= ~MPOL_MF_STRICT; - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); end = start + len; if (end < start) @@ -1768,7 +1795,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le VMA_ITERATOR(vmi, mm, start); start = untagged_addr(start); - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; /* * flags is used for future extension if any. @@ -1783,7 +1810,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le if (home_node >= MAX_NUMNODES || !node_online(home_node)) return -EINVAL; - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); end = start + len; if (end < start) @@ -2049,7 +2076,7 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, if (pol->mode == MPOL_INTERLEAVE || pol->mode == MPOL_WEIGHTED_INTERLEAVE) { *ilx += vma->vm_pgoff >> order; - *ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order); + *ilx += (vma_page_offset_at(vma, addr).index - vma->vm_pgoff) >> order; } return pol; } @@ -3254,6 +3281,10 @@ int mpol_set_shared_policy(struct shared_policy *sp, struct sp_node *new = NULL; unsigned long sz = vma_pages(vma); + /* Never truncate a partial cache-page policy update to zero or a neighbour. */ + if (vma_subpage_offset(vma) || ((vma->vm_end - vma->vm_start) & ~PAGE_MASK)) + return -EINVAL; + if (pol) { new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, pol); if (!new) diff --git a/mm/migrate.c b/mm/migrate.c index 8a64291ab5b44..795143e1b4cc4 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -14,6 +14,8 @@ */ #include +#include +#include #include #include #include @@ -340,14 +342,50 @@ struct rmap_walk_arg { bool map_unused_to_zeropage; }; +#ifdef CONFIG_MM_SUBPAGE +static void remove_migration_subpage(struct folio *folio, + struct page_vma_mapped_walk *pvmw, pte_t old) +{ + struct vm_area_struct *vma = pvmw->vma; + softleaf_t entry = softleaf_from_pte(old); + struct mm_subpage *slot; + pte_t pte; + int ret; + + slot = mm_subpage_migrate_restore(folio, pte_swp_subpage_offset(old)); + pte = phys_pte_mm(vma->vm_mm, mm_subpage_phys(slot), READ_ONCE(vma->vm_page_prot)); + if (!softleaf_is_migration_young(entry)) + pte = pte_mkold(pte); + if (folio_test_dirty(folio) && softleaf_is_migration_dirty(entry)) + pte = pte_mkdirty(pte); + if (pte_swp_soft_dirty(old)) + pte = pte_mksoft_dirty(pte); + else + pte = pte_clear_soft_dirty(pte); + if (softleaf_is_migration_write(entry)) + pte = pte_mkwrite(pte, vma); + else if (pte_swp_uffd_wp(old)) + pte = pte_mkuffd_wp(pte); + if (!softleaf_is_migration_read(entry)) + mm_subpage_set_exclusive(slot); + ret = mm_subpage_add_anon_rmap(slot, vma, pvmw->address); + VM_BUG_ON_FOLIO(ret, folio); + set_pte_at(vma->vm_mm, pvmw->address, pvmw->pte, pte); + if (READ_ONCE(vma->vm_flags) & VM_LOCKED) + mlock_drain_local(); + trace_remove_migration_pte(pvmw->address, pte_val(pte), 0); + update_mmu_cache(vma, pvmw->address, pvmw->pte); +} +#endif + /* * Restore a potential migration pte to a working pte entry */ static bool remove_migration_pte(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { struct rmap_walk_arg *rmap_walk_arg = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION); + DEFINE_FOLIO_RMAP_WALK(pvmw, rmap_walk_arg->folio, vma, rmap, PVMW_SYNC | PVMW_MIGRATION); while (page_vma_mapped_walk(&pvmw)) { rmap_t rmap_flags = RMAP_NONE; @@ -372,13 +410,29 @@ static bool remove_migration_pte(struct folio *folio, } #endif old_pte = ptep_get(pvmw.pte); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE && folio_test_anon(folio)) { + remove_migration_subpage(folio, &pvmw, old_pte); + continue; + } +#endif if (rmap_walk_arg->map_unused_to_zeropage && try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx)) continue; - folio_get(folio); + /* Match the native-page references dropped by try_to_migrate_one(). */ + if (!folio_test_hugetlb(folio)) + folio_ref_add(folio, mm_pte_native_pages(vma->vm_mm)); + else + folio_get(folio); pte = mk_pte(new, READ_ONCE(vma->vm_page_prot)); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + pte = phys_pte_mm(vma->vm_mm, page_to_phys(new) + + pte_swp_subpage_offset(old_pte), READ_ONCE(vma->vm_page_prot)); +#endif + entry = softleaf_from_pte(old_pte); if (!softleaf_is_migration_young(entry)) pte = pte_mkold(pte); @@ -431,8 +485,10 @@ static bool remove_migration_pte(struct folio *folio, if (folio_test_anon(folio)) folio_add_anon_rmap_pte(folio, new, vma, pvmw.address, rmap_flags); - else + else { + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, pte)); folio_add_file_rmap_pte(folio, new, vma); + } set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); } if (READ_ONCE(vma->vm_flags) & VM_LOCKED) @@ -640,10 +696,12 @@ static int __folio_migrate_mapping(struct address_space *mapping, folio_set_dirty(newfolio); } - if (folio_test_swapcache(folio)) + if (folio_test_swapcache(folio)) { __swap_cache_replace_folio(ci, folio, newfolio); - else + } else { + shmem_uffd_migrate(newfolio, folio); xas_store(&xas, newfolio); + } /* * Drop cache reference from old folio by unfreezing @@ -1170,8 +1228,10 @@ static void migrate_folio_undo_src(struct folio *src, /* Drop an anon_vma reference if we took one */ if (anon_vma) put_anon_vma(anon_vma); - if (locked) + if (locked) { + mm_subpage_migrate_finish(src); folio_unlock(src); + } if (ret) list_move_tail(&src->lru, ret); } @@ -1180,8 +1240,10 @@ static void migrate_folio_undo_src(struct folio *src, static void migrate_folio_undo_dst(struct folio *dst, bool locked, free_folio_t put_new_folio, unsigned long private) { - if (locked) + if (locked) { + mm_subpage_migrate_finish(dst); folio_unlock(dst); + } if (put_new_folio) put_new_folio(dst, private); else @@ -1306,6 +1368,11 @@ static int migrate_folio_unmap(new_folio_t get_new_folio, return 0; } + rc = mm_subpage_migrate_prepare(src, dst); + if (rc) + goto out; + rc = -EAGAIN; + /* * Corner case handling: * 1. When a new swap-cache page is read into, it is added to the LRU @@ -1411,6 +1478,8 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private, remove_migration_ptes(src, dst, 0); out_unlock_both: + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); folio_unlock(dst); folio_set_owner_migrate_reason(dst, reason); /* diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 17074c2573b94..bae35a327cf1b 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -408,7 +408,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, if (anon_exclusive) { pte = ptep_clear_flush(vma, addr, ptep); - if (folio_try_share_anon_rmap_pte(folio, page)) { + if (folio_try_share_anon_rmap_pte(folio, page, vma)) { set_pte_at(mm, addr, ptep, pte); if (fault_folio != folio) folio_unlock(folio); diff --git a/mm/mincore.c b/mm/mincore.c index c8757c5085bf9..66458b6bb2091 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -48,7 +48,7 @@ static int mincore_hugetlb(pte_t *pte, unsigned long hmask, unsigned long addr, present = 1; } - for (; addr != end; vec++, addr += PAGE_SIZE) + for (; addr != end; vec++, addr += mm_page_size(walk->mm)) *vec = present; walk->private = vec; spin_unlock(ptl); @@ -135,15 +135,22 @@ static unsigned char mincore_page(struct address_space *mapping, pgoff_t index) static int __mincore_unmapped_range(unsigned long addr, unsigned long end, struct vm_area_struct *vma, unsigned char *vec) { - unsigned long nr = (end - addr) >> PAGE_SHIFT; + unsigned long nr = (end - addr) >> mm_page_shift(vma->vm_mm); int i; if (vma->vm_file) { - pgoff_t pgoff; + unsigned int shift = mm_page_shift(vma->vm_mm); - pgoff = linear_page_index(vma, addr); - for (i = 0; i < nr; i++, pgoff++) - vec[i] = mincore_page(vma->vm_file->f_mapping, pgoff); + /* Cache indices are native; several user leaves can share one. */ + for (i = 0; i < nr; ) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + unsigned long batch = min(nr - i, (PAGE_SIZE - pos.offset) >> shift); + unsigned char present = mincore_page(vma->vm_file->f_mapping, pos.index); + + memset(vec + i, present, batch); + i += batch; + addr += batch << shift; + } } else { for (i = 0; i < nr; i++) vec[i] = 0; @@ -167,7 +174,7 @@ static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct vm_area_struct *vma = walk->vma; pte_t *ptep; unsigned char *vec = walk->private; - int nr = (end - addr) >> PAGE_SHIFT; + int nr = (end - addr) >> mm_page_shift(walk->mm); int step, i; ptl = pmd_trans_huge_lock(pmd, vma); @@ -182,19 +189,19 @@ static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; ptep += step, addr += step * PAGE_SIZE) { + for (; addr != end; ptep += step, addr += step * mm_page_size(walk->mm)) { pte_t pte = ptep_get(ptep); step = 1; /* We need to do cache lookup too for markers */ if (pte_none(pte) || pte_is_marker(pte)) - __mincore_unmapped_range(addr, addr + PAGE_SIZE, + __mincore_unmapped_range(addr, addr + mm_page_size(walk->mm), vma, vec); else if (pte_present(pte)) { unsigned int batch = pte_batch_hint(ptep, pte); if (batch > 1) { - unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + unsigned int max_nr = (end - addr) >> mm_page_shift(walk->mm); step = min_t(unsigned int, batch, max_nr); } @@ -252,16 +259,16 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v vma = vma_lookup(current->mm, addr); if (!vma) return -ENOMEM; - end = min(vma->vm_end, addr + (pages << PAGE_SHIFT)); + end = min(vma->vm_end, addr + (pages << mm_page_shift(vma->vm_mm))); if (!can_do_mincore(vma)) { - unsigned long pages = DIV_ROUND_UP(end - addr, PAGE_SIZE); + unsigned long pages = DIV_ROUND_UP(end - addr, mm_page_size(vma->vm_mm)); memset(vec, 1, pages); return pages; } err = walk_page_range(vma->vm_mm, addr, end, &mincore_walk_ops, vec); if (err < 0) return err; - return (end - addr) >> PAGE_SHIFT; + return (end - addr) >> mm_page_shift(vma->vm_mm); } /* @@ -281,7 +288,7 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v * return values: * zero - success * -EFAULT - vec points to an illegal address - * -EINVAL - addr is not a multiple of PAGE_SIZE + * -EINVAL - addr is not aligned to the process page size * -ENOMEM - Addresses in the range [addr, addr + len] are * invalid for the address space of this process, or * specify one or more pages which are not currently @@ -298,7 +305,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, start = untagged_addr(start); /* Check the start address: needs to be page-aligned.. */ - if (unlikely(start & ~PAGE_MASK)) + if (unlikely(start & ~mm_page_mask(current->mm))) return -EINVAL; /* ..and we need to be passed a valid user-space range */ @@ -306,8 +313,8 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, return -ENOMEM; /* This also avoids any overflows on PAGE_ALIGN */ - pages = len >> PAGE_SHIFT; - pages += (offset_in_page(len)) != 0; + pages = len >> mm_page_shift(current->mm); + pages += (len & ~mm_page_mask(current->mm)) != 0; if (!access_ok(vec, pages)) return -EFAULT; @@ -334,7 +341,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, } pages -= retval; vec += retval; - start += retval << PAGE_SHIFT; + start += retval << mm_page_shift(current->mm); retval = 0; } free_page((unsigned long) tmp); diff --git a/mm/mlock.c b/mm/mlock.c index 8c227fefa2df0..a6b67d6b20cac 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -304,15 +304,19 @@ void munlock_folio(struct folio *folio) local_unlock(&mlock_fbatch.lock); } -static inline unsigned int folio_mlock_step(struct folio *folio, +static inline unsigned int folio_mlock_step(struct vm_area_struct *vma, struct folio *folio, pte_t *pte, unsigned long addr, unsigned long end) { - unsigned int count = (end - addr) >> PAGE_SHIFT; + unsigned int count = (end - addr) >> mm_page_shift(vma->vm_mm); pte_t ptent = ptep_get(pte); if (!folio_test_large(folio)) return 1; + #ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch(vma, folio, pte, ptent, count, 0); +#endif return folio_pte_batch(folio, pte, ptent, count); } @@ -341,7 +345,7 @@ static inline bool allow_mlock_munlock(struct folio *folio, return false; /* folio is not fully mapped, skip mlock */ - if (step != folio_nr_pages(folio)) + if (step != (folio_size(folio) >> mm_page_shift(vma->vm_mm))) return false; return true; @@ -381,7 +385,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, return 0; } - for (pte = start_pte; addr != end; pte++, addr += PAGE_SIZE) { + for (pte = start_pte; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) { ptent = ptep_get(pte); if (!pte_present(ptent)) continue; @@ -389,7 +393,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, if (!folio || folio_is_zone_device(folio)) continue; - step = folio_mlock_step(folio, pte, addr, end); + step = folio_mlock_step(vma, folio, pte, addr, end); if (!allow_mlock_munlock(folio, vma, start, end, step)) goto next_entry; @@ -400,7 +404,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, next_entry: pte += step - 1; - addr += (step - 1) << PAGE_SHIFT; + addr += (step - 1) << mm_page_shift(vma->vm_mm); } pte_unmap(start_pte); out: @@ -471,7 +475,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags); const vma_flags_t old_vma_flags = vma->flags; struct mm_struct *mm = vma->vm_mm; - int nr_pages; + long nr_pages; int ret = 0; if (vma_flags_same_pair(&old_vma_flags, &new_vma_flags) || @@ -492,7 +496,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, /* * Keep track of amount of locked VM. */ - nr_pages = (end - start) >> PAGE_SHIFT; + nr_pages = (end - start) >> MM_ACCOUNT_SHIFT; if (!vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT)) nr_pages = -nr_pages; else if (vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) @@ -524,8 +528,8 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, struct vm_area_struct *vma, *prev; VMA_ITERATOR(vmi, current->mm, start); - VM_BUG_ON(offset_in_page(start)); - VM_BUG_ON(len != PAGE_ALIGN(len)); + VM_BUG_ON((start & ~mm_page_mask(current->mm))); + VM_BUG_ON(len != mm_page_align(current->mm, len)); end = start + len; if (end < start) return -EINVAL; @@ -572,7 +576,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, * vma pages, as return value. * Note deferred memory locking case(mlock2(,,MLOCK_ONFAULT) * is also counted. - * Return value: previously mlocked page counts + * Return value: previously mlocked MM_ACCOUNT_SHIFT units */ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, unsigned long start, size_t len) @@ -600,7 +604,7 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, } } - return count >> PAGE_SHIFT; + return count >> MM_ACCOUNT_SHIFT; } /* @@ -626,12 +630,12 @@ static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t fla if (!can_do_mlock()) return -EPERM; - len = PAGE_ALIGN(len + (offset_in_page(start))); - start &= PAGE_MASK; + len = mm_page_align(current->mm, len + (start & ~mm_page_mask(current->mm))); + start &= mm_page_mask(current->mm); lock_limit = rlimit(RLIMIT_MEMLOCK); - lock_limit >>= PAGE_SHIFT; - locked = len >> PAGE_SHIFT; + lock_limit >>= MM_ACCOUNT_SHIFT; + locked = len >> MM_ACCOUNT_SHIFT; if (mmap_write_lock_killable(current->mm)) return -EINTR; @@ -686,8 +690,8 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) start = untagged_addr(start); - len = PAGE_ALIGN(len + (offset_in_page(start))); - start &= PAGE_MASK; + len = mm_page_align(current->mm, len + (start & ~mm_page_mask(current->mm))); + start &= mm_page_mask(current->mm); if (mmap_write_lock_killable(current->mm)) return -EINTR; @@ -761,7 +765,7 @@ SYSCALL_DEFINE1(mlockall, int, flags) return -EPERM; lock_limit = rlimit(RLIMIT_MEMLOCK); - lock_limit >>= PAGE_SHIFT; + lock_limit >>= mm_page_shift(current->mm); if (mmap_write_lock_killable(current->mm)) return -EINTR; diff --git a/mm/mm_init.c b/mm/mm_init.c index c7e498fc1c4c0..8d14ce83be36b 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -57,7 +57,8 @@ unsigned long zero_page_pfn __ro_after_init; EXPORT_SYMBOL(zero_page_pfn); #ifndef __HAVE_COLOR_ZERO_PAGE -uint8_t empty_zero_page[PAGE_SIZE] __page_aligned_bss; +uint8_t empty_zero_page[ARCH_ZERO_PAGE_SIZE] __page_aligned_bss + __aligned(ARCH_ZERO_PAGE_SIZE); EXPORT_SYMBOL(empty_zero_page); struct page *__zero_page __ro_after_init; diff --git a/mm/mmap.c b/mm/mmap.c index 2311ae7c2ff45..097161e3e61ef 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -151,8 +151,8 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) mm->end_data, mm->start_data)) goto out; - newbrk = PAGE_ALIGN(brk); - oldbrk = PAGE_ALIGN(mm->brk); + newbrk = mm_page_align(mm, brk); + oldbrk = mm_page_align(mm, mm->brk); if (oldbrk == newbrk) { mm->brk = brk; goto success; @@ -186,8 +186,8 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) * expansion area */ vma_iter_init(&vmi, mm, oldbrk); - next = vma_find(&vmi, newbrk + PAGE_SIZE + stack_guard_gap); - if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) + next = vma_find(&vmi, newbrk + mm_page_size(mm) + stack_guard_gap); + if (next && newbrk + mm_page_size(mm) > vm_start_gap(next)) goto out; brkvma = vma_prev_limit(&vmi, mm->start_brk); @@ -218,12 +218,12 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) * If a hint addr is less than mmap_min_addr change hint to be as * low as possible but still greater than mmap_min_addr */ -static inline unsigned long round_hint_to_min(unsigned long hint) +static inline unsigned long round_hint_to_min(struct mm_struct *mm, unsigned long hint) { - hint &= PAGE_MASK; + hint &= mm_page_mask(mm); if (((void *)hint != NULL) && (hint < mmap_min_addr)) - return PAGE_ALIGN(mmap_min_addr); + return mm_page_align(mm, mmap_min_addr); return hint; } @@ -235,11 +235,11 @@ bool mlock_future_ok(const struct mm_struct *mm, bool is_vma_locked, if (!is_vma_locked || capable(CAP_IPC_LOCK)) return true; - locked_pages = bytes >> PAGE_SHIFT; + locked_pages = bytes >> MM_ACCOUNT_SHIFT; locked_pages += mm->locked_vm; limit_pages = rlimit(RLIMIT_MEMLOCK); - limit_pages >>= PAGE_SHIFT; + limit_pages >>= MM_ACCOUNT_SHIFT; return locked_pages <= limit_pages; } @@ -264,24 +264,25 @@ static inline u64 file_mmap_size_max(struct file *file, struct inode *inode) } static inline bool file_mmap_ok(struct file *file, struct inode *inode, - unsigned long pgoff, unsigned long len) + struct vm_page_offset pos, unsigned long len) { u64 maxsize = file_mmap_size_max(file, inode); if (maxsize && len > maxsize) return false; maxsize -= len; - if (pgoff > maxsize >> PAGE_SHIFT) + if (pos.index > maxsize >> PAGE_SHIFT || + (pos.index == maxsize >> PAGE_SHIFT && pos.offset > (maxsize & ~PAGE_MASK))) return false; return true; } /** - * do_mmap() - Perform a userland memory mapping into the current process + * do_mmap_internal() - Perform a userland memory mapping into the current process * address space of length @len with protection bits @prot, mmap flags @flags * (from which VMA flags will be inferred), and any additional VMA flags to * apply @vm_flags. If this is a file-backed mapping then the file is specified - * in @file and page offset into the file via @pgoff. + * in @file and its byte position via @pos. * * This function does not perform security checks on the file and assumes, if * @uf is non-NULL, the caller has provided a list head to track unmap events @@ -321,7 +322,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * @flags: Flags specifying how the mapping should be performed, see mmap (2) * for details. * @vm_flags: VMA flags which should be set by default, or 0 otherwise. - * @pgoff: Page offset into the @file if file-backed, should be 0 otherwise. + * @pos: Native cache index and byte offset; ignored for anonymous mappings. * @populate: A pointer to a value which will be set to 0 if no population of * the range is required, or the number of bytes to populate if it is. Must be * non-NULL. See mmap (2) for details as to under what circumstances population @@ -333,20 +334,22 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * Returns: Either an error, or the address at which the requested mapping has * been performed. */ -unsigned long do_mmap(struct file *file, unsigned long addr, +static unsigned long do_mmap_internal(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, vm_flags_t vm_flags, - unsigned long pgoff, unsigned long *populate, + struct vm_page_offset pos, unsigned long *populate, struct list_head *uf) { struct mm_struct *mm = current->mm; + unsigned long pgoff = pos.index; int pkey = 0; *populate = 0; mmap_assert_write_locked(mm); - if (!len) + if (!len || pos.offset >= PAGE_SIZE || + !IS_ALIGNED(pos.offset, mm_page_size(mm))) return -EINVAL; /* @@ -364,15 +367,15 @@ unsigned long do_mmap(struct file *file, unsigned long addr, flags |= MAP_FIXED; if (!(flags & MAP_FIXED)) - addr = round_hint_to_min(addr); + addr = round_hint_to_min(mm, addr); /* Careful about overflows.. */ - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); if (!len) return -ENOMEM; /* offset overflow? */ - if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) + if (vm_page_offset_add(pos, len).index < pgoff) return -EOVERFLOW; /* Too many mappings? */ @@ -426,7 +429,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long flags_mask; int err; - if (!file_mmap_ok(file, inode, pgoff, len)) + if (!file_mmap_ok(file, inode, pos, len)) return -EOVERFLOW; flags_mask = LEGACY_MAP_MASK; @@ -500,6 +503,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * Ignore pgoff. */ pgoff = 0; + pos.offset = 0; vm_flags |= VM_SHARED | VM_MAYSHARE; break; case MAP_DROPPABLE: @@ -537,6 +541,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * Set pgoff according to addr for anon_vma. */ pgoff = addr >> PAGE_SHIFT; + pos.offset = addr & ~PAGE_MASK; break; default: return -EINVAL; @@ -557,7 +562,11 @@ unsigned long do_mmap(struct file *file, unsigned long addr, vm_flags |= VM_NORESERVE; } - addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); + addr = mmap_region(file, addr, len, vm_flags, + (struct vm_page_offset) { + .index = pgoff, + .offset = pos.offset, + }, uf); if (!IS_ERR_VALUE(addr) && ((vm_flags & VM_LOCKED) || (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) @@ -565,9 +574,28 @@ unsigned long do_mmap(struct file *file, unsigned long addr, return addr; } -unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, +unsigned long do_mmap(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, + struct list_head *uf) +{ + return do_mmap_internal(file, addr, len, prot, flags, vm_flags, + (struct vm_page_offset) { pgoff, 0 }, populate, uf); +} + +#ifdef CONFIG_MM_SUBPAGE +unsigned long do_mmap_offset(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, struct vm_page_offset pos, unsigned long *populate, + struct list_head *uf) +{ + return do_mmap_internal(file, addr, len, prot, flags, vm_flags, pos, populate, uf); +} +#endif + +static unsigned long ksys_mmap_offset(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, - unsigned long fd, unsigned long pgoff) + unsigned long fd, unsigned long pgoff, unsigned int suboffset) { struct file *file = NULL; unsigned long retval; @@ -603,13 +631,34 @@ unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, return PTR_ERR(file); } - retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); + if (suboffset) + retval = vm_mmap(file, addr, len, prot, flags, + (pgoff << PAGE_SHIFT) | suboffset); + else + retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); out_fput: if (file) fput(file); return retval; } +unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long pgoff) +{ + return ksys_mmap_offset(addr, len, prot, flags, fd, pgoff, 0); +} + +#ifdef CONFIG_MM_SUBPAGE +unsigned long ksys_mmap_user(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long offset) +{ + if (!IS_ALIGNED(offset, mm_page_size(current->mm))) + return -EINVAL; + return ksys_mmap_offset(addr, len, prot, flags, fd, + offset >> PAGE_SHIFT, offset & ~PAGE_MASK); +} +#endif + SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, pgoff) @@ -703,7 +752,7 @@ generic_get_unmapped_area(struct file *filp, unsigned long addr, return addr; if (addr) { - addr = PAGE_ALIGN(addr); + addr = mm_page_align(mm, addr); vma = find_vma_prev(mm, addr, &prev); if (mmap_end - len >= addr && addr >= mmap_min_addr && (!vma || addr + len <= vm_start_gap(vma)) && @@ -754,7 +803,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, /* requesting a specific address */ if (addr) { - addr = PAGE_ALIGN(addr); + addr = mm_page_align(mm, addr); vma = find_vma_prev(mm, addr, &prev); if (mmap_end - len >= addr && addr >= mmap_min_addr && (!vma || addr + len <= vm_start_gap(vma)) && @@ -764,7 +813,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.length = len; - info.low_limit = PAGE_SIZE; + info.low_limit = mm_page_size(mm); info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); info.start_gap = stack_guard_placement(vm_flags); if (filp && is_file_hugepages(filp)) @@ -777,7 +826,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, * can happen with large stack limits and large mmap() * allocations. */ - if (offset_in_page(addr)) { + if (addr & ~mm_page_mask(mm)) { VM_BUG_ON(addr != -ENOMEM); info.flags = 0; info.low_limit = TASK_UNMAPPED_BASE; @@ -857,7 +906,7 @@ __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, if (addr > TASK_SIZE - len) return -ENOMEM; - if (offset_in_page(addr)) + if (addr & ~mm_page_mask(current->mm)) return -EINVAL; error = security_mmap_addr(addr); @@ -962,7 +1011,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon { struct vm_area_struct *vma, *prev; - addr &= PAGE_MASK; + addr &= mm_page_mask(mm); vma = find_vma_prev(mm, addr, &prev); if (vma && (vma->vm_start <= addr)) return vma; @@ -985,7 +1034,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon struct vm_area_struct *vma; unsigned long start; - addr &= PAGE_MASK; + addr &= mm_page_mask(mm); vma = find_vma(mm, addr); if (!vma) return NULL; @@ -1214,7 +1263,7 @@ int vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec) LIST_HEAD(uf); VMA_ITERATOR(vmi, mm, addr); - len = PAGE_ALIGN(request); + len = mm_page_align(mm, request); if (len < request) return -ENOMEM; if (!len) @@ -1257,7 +1306,7 @@ unsigned long tear_down_vmas(struct mm_struct *mm, struct vma_iterator *vmi, vma_iter_set(vmi, vma->vm_end); do { if (vma->vm_flags & VM_ACCOUNT) - nr_accounted += vma_pages(vma); + nr_accounted += vma_user_pages(vma); vma_mark_detached(vma); remove_vma(vma); count++; @@ -1322,8 +1371,8 @@ void exit_mmap(struct mm_struct *mm) destroy: __mt_destroy(&mm->mm_mt); trace_exit_mmap(mm); + mm_unacct_memory(mm, nr_accounted); mmap_write_unlock(mm); - vm_unacct_memory(nr_accounted); } /* @@ -1333,19 +1382,19 @@ void exit_mmap(struct mm_struct *mm) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages) { - if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT) + if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> mm_page_shift(mm)) return false; if (is_data_mapping_vma_flags(vma_flags) && - mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) { + mm->data_vm + npages > rlimit(RLIMIT_DATA) >> mm_page_shift(mm)) { /* Workaround for Valgrind */ if (rlimit(RLIMIT_DATA) == 0 && - mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT) + mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> mm_page_shift(mm)) return true; pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n", current->comm, current->pid, - (mm->data_vm + npages) << PAGE_SHIFT, + (mm->data_vm + npages) << mm_page_shift(mm), rlimit(RLIMIT_DATA), ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data"); @@ -1474,7 +1523,7 @@ static struct vm_area_struct *__install_special_mapping( if (ret) goto out; - vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); + vm_stat_account(mm, vma->vm_flags, len >> mm_page_shift(mm)); perf_event_mmap(vma); @@ -1771,14 +1820,14 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) if (retval) goto loop_out; - vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt)); + vm_stat_account(mm, mpnt->vm_flags, -vma_user_pages(mpnt)); continue; } charge = 0; if (mpnt->vm_flags & VM_ACCOUNT) { - unsigned long len = vma_pages(mpnt); + unsigned long len = vma_user_pages(mpnt); - if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ + if (mm_account_memory(mm, oldmm, len)) /* sic */ goto fail_nomem; charge = len; } @@ -1890,7 +1939,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) flush_cache_mm(mm); unmap_region(&unmap); charge = tear_down_vmas(mm, &vmi, tmp, end); - vm_unacct_memory(charge); + mm_unacct_memory(mm, charge); } __mt_destroy(&mm->mm_mt); /* @@ -1916,6 +1965,6 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) vm_area_free(tmp); fail_nomem: retval = -ENOMEM; - vm_unacct_memory(charge); + mm_unacct_memory(mm, charge); goto loop_out; } diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 3985d856de7f9..42419a47a1880 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -12,6 +12,9 @@ #include #include #include +#include + +#include "user-subpage-internal.h" #include @@ -100,6 +103,49 @@ void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma) */ #define MAX_NR_FOLIOS_PER_FREE 512 +#ifdef CONFIG_MM_SUBPAGE +/* Private to gather batches: never pass this token to encoded_page_ptr(). */ +#define TLB_SUBPAGE_TAG BIT(2) + +static void tlb_free_pages_and_subpages(struct encoded_page **pages, unsigned int nr) +{ + unsigned int start = 0, i = 0; + + while (i < nr) { + unsigned long token = (unsigned long)pages[i]; + + if (token & TLB_SUBPAGE_TAG) { + if (i != start) + free_pages_and_swap_cache(pages + start, i - start); + mm_subpage_put((struct mm_subpage *)(token & ~TLB_SUBPAGE_TAG)); + start = ++i; + } else { + /* The count payload is not a pointer and may have bit 2 set. */ + i += encoded_page_flags(pages[i]) & ENCODED_PAGE_BIT_NR_PAGES_NEXT ? 2 : 1; + } + } + if (i != start) + free_pages_and_swap_cache(pages + start, i - start); +} + +bool __tlb_remove_subpage(struct mmu_gather *tlb, struct mm_subpage *subpage) +{ + struct mmu_gather_batch *batch = tlb->active; + + BUILD_BUG_ON(__alignof__(struct page) < 8); + BUILD_BUG_ON(__alignof__(struct mm_subpage) < 8); + BUILD_BUG_ON(TLB_SUBPAGE_TAG & ENCODED_PAGE_BITS); + VM_BUG_ON(!tlb->end); + VM_BUG_ON(mm_page_shift(tlb->mm) != mm_subpage_shift(subpage)); + batch->encoded_pages[batch->nr++] = + (struct encoded_page *)((unsigned long)subpage | TLB_SUBPAGE_TAG); + /* Keep room for a following native page/count pair. */ + return batch->nr >= batch->max - 1 && !tlb_next_batch(tlb); +} +#else +#define tlb_free_pages_and_subpages free_pages_and_swap_cache +#endif + static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) { struct encoded_page **pages = batch->encoded_pages; @@ -135,7 +181,7 @@ static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) } } - free_pages_and_swap_cache(pages, nr); + tlb_free_pages_and_subpages(pages, nr); pages += nr; batch->nr -= nr; @@ -174,7 +220,15 @@ static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb, #ifdef CONFIG_MMU_GATHER_PAGE_SIZE VM_WARN_ON(tlb->page_size != page_size); - VM_WARN_ON_ONCE(nr_pages != 1 && page_size != PAGE_SIZE); + if (page_size > PAGE_SIZE && page_size == mm_page_size(tlb->mm)) { + /* A coarse base PTE owns one reference per covered native page. */ + unsigned long span = mm_pte_native_pages(tlb->mm); + + VM_WARN_ON_ONCE(!nr_pages || nr_pages % span || + !IS_ALIGNED(page_to_pfn(page), span)); + } else { + VM_WARN_ON_ONCE(nr_pages != 1 && page_size != PAGE_SIZE); + } VM_WARN_ON_ONCE(page_folio(page) != page_folio(page + nr_pages - 1)); #endif @@ -208,7 +262,7 @@ bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page, unsigned int nr_pages, bool delay_rmap) { return __tlb_remove_folio_pages_size(tlb, page, nr_pages, delay_rmap, - PAGE_SIZE); + mm_page_size(tlb->mm)); } bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size) @@ -352,6 +406,13 @@ static inline void __tlb_remove_table_one(void *table) { struct ptdesc *ptdesc; + /* Preserve opaque architecture tokens through the RCU grace period. */ + if (!tlb_table_is_ptdesc(table)) { + tlb_remove_table_sync_rcu(); + __tlb_remove_table(table); + return; + } + ptdesc = table; call_rcu(&ptdesc->pt_rcu_head, __tlb_remove_table_one_rcu); } diff --git a/mm/mprotect.c b/mm/mprotect.c index 9cbf932b028cf..0f5ea22bf4c90 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -58,6 +59,31 @@ static bool maybe_change_pte_writable(struct vm_area_struct *vma, pte_t pte) return true; } +#ifdef CONFIG_MM_SUBPAGE +static bool subpage_pte_exclusive(struct vm_area_struct *vma, pte_t pte) +{ + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + bool exclusive; + + if (!slot) + return false; + exclusive = mm_subpage_is_exclusive(slot); + mm_subpage_put(slot); + return exclusive; +} + +static int subpage_exclusive_pte_batch(struct vm_area_struct *vma, pte_t *ptep, + int nr_ptes, bool exclusive) +{ + int nr; + + for (nr = 1; nr < nr_ptes; nr++) + if (subpage_pte_exclusive(vma, ptep_get(ptep + nr)) != exclusive) + break; + return nr; +} +#endif + static bool can_change_private_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte) { @@ -73,6 +99,10 @@ static bool can_change_private_pte_writable(struct vm_area_struct *vma, * any additional checks while holding the PT lock. */ page = vm_normal_page(vma, addr, pte); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return page && PageAnon(page) && subpage_pte_exclusive(vma, pte); +#endif return page && PageAnon(page) && PageAnonExclusive(page); } @@ -103,13 +133,20 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, return can_change_shared_pte_writable(vma, pte); } -static int mprotect_folio_pte_batch(struct folio *folio, pte_t *ptep, - pte_t pte, int max_nr_ptes, fpb_t flags) + +static __always_inline int mprotect_folio_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t pte, int max_nr_ptes, + fpb_t flags, unsigned int page_shift) { /* No underlying folio, so cannot batch */ if (!folio) return 1; +#ifdef CONFIG_MM_SUBPAGE + if (page_shift != PAGE_SHIFT) + return folio_subpage_pte_batch(vma, folio, ptep, pte, max_nr_ptes, flags); +#endif + if (!folio_test_large(folio)) return 1; @@ -125,17 +162,17 @@ static __always_inline void prot_commit_flush_ptes(struct vm_area_struct *vma, * Advance the position in the batch by idx; note that if idx > 0, * then the nr_ptes passed here is <= batch size - idx. */ - addr += idx * PAGE_SIZE; + addr += idx * mm_page_size(vma->vm_mm); ptep += idx; - oldpte = pte_advance_pfn(oldpte, idx); - ptent = pte_advance_pfn(ptent, idx); + oldpte = pte_advance_pfn_mm(vma->vm_mm, oldpte, idx); + ptent = pte_advance_pfn_mm(vma->vm_mm, ptent, idx); if (set_write) ptent = pte_mkwrite(ptent, vma); modify_prot_commit_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes); if (pte_needs_flush(oldpte, ptent)) - tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE); + tlb_flush_pte_range(tlb, addr, nr_ptes * mm_page_size(vma->vm_mm)); } /* @@ -177,6 +214,25 @@ static __always_inline void commit_anon_folio_batch(struct vm_area_struct *vma, int sub_batch_idx = 0; int len; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + while (nr_ptes) { + expected_anon_exclusive = subpage_pte_exclusive(vma, + pte_advance_pfn_mm(vma->vm_mm, oldpte, sub_batch_idx)); + for (len = 1; len < nr_ptes; len++) + if (subpage_pte_exclusive(vma, + pte_advance_pfn_mm(vma->vm_mm, oldpte, sub_batch_idx + len)) != + expected_anon_exclusive) + break; + prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, len, + sub_batch_idx, expected_anon_exclusive, tlb); + sub_batch_idx += len; + nr_ptes -= len; + } + return; + } +#endif + while (nr_ptes) { expected_anon_exclusive = PageAnonExclusive(first_page + sub_batch_idx); len = page_anon_exclusive_sub_batch(sub_batch_idx, nr_ptes, @@ -265,6 +321,11 @@ static long change_softleaf_pte(struct vm_area_struct *vma, newpte = oldpte; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + newpte = pte_swp_set_subpage_offset(newpte, pte_swp_subpage_offset(oldpte)); +#endif + if (uffd_wp) newpte = pte_swp_mkuffd_wp(newpte); else if (uffd_wp_resolve) @@ -320,6 +381,9 @@ static long change_pte_range(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, pgprot_t newprot, unsigned long cp_flags) { + /* The mm's mapping granule cannot change during the walk. */ + const unsigned int page_shift = mm_page_shift(vma->vm_mm); + const unsigned long page_size = 1UL << page_shift; pte_t *pte, oldpte; spinlock_t *ptl; long pages = 0; @@ -328,7 +392,7 @@ static long change_pte_range(struct mmu_gather *tlb, bool uffd_wp = cp_flags & MM_CP_UFFD_WP; int nr_ptes; - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, page_size); pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); if (!pte) return -EAGAIN; @@ -343,7 +407,7 @@ static long change_pte_range(struct mmu_gather *tlb, oldpte = ptep_get(pte); if (pte_present(oldpte)) { const fpb_t flags = FPB_RESPECT_SOFT_DIRTY | FPB_RESPECT_WRITE; - int max_nr_ptes = (end - addr) >> PAGE_SHIFT; + int max_nr_ptes = (end - addr) >> page_shift; struct folio *folio = NULL; struct page *page; @@ -359,17 +423,32 @@ static long change_pte_range(struct mmu_gather *tlb, * Avoid trapping faults against the zero or KSM * pages. See similar comment in change_huge_pmd. */ - if (prot_numa && - !folio_can_map_prot_numa(folio, vma, - is_private_single_threaded)) { - - /* determine batch to skip */ - nr_ptes = mprotect_folio_pte_batch(folio, - pte, oldpte, max_nr_ptes, /* flags = */ 0); - continue; + if (prot_numa) { + bool shared = folio && folio_maybe_mapped_shared(folio); + +#ifdef CONFIG_MM_SUBPAGE + /* Multiple private leaves do not imply shared COW data. */ + if (folio && folio_test_anon(folio) && page_shift < PAGE_SHIFT) { + shared = !subpage_pte_exclusive(vma, oldpte); + max_nr_ptes = folio_subpage_pte_batch(vma, folio, + pte, oldpte, max_nr_ptes, flags); + /* COW sharing can differ within one physical owner. */ + max_nr_ptes = subpage_exclusive_pte_batch(vma, pte, + max_nr_ptes, !shared); + } +#endif + if (!folio_can_map_prot_numa(folio, vma, + is_private_single_threaded, shared)) { + /* determine batch to skip */ + nr_ptes = mprotect_folio_pte_batch(vma, folio, + pte, oldpte, max_nr_ptes, /* flags = */ 0, + page_shift); + continue; + } } - nr_ptes = mprotect_folio_pte_batch(folio, pte, oldpte, max_nr_ptes, flags); + nr_ptes = mprotect_folio_pte_batch(vma, folio, pte, oldpte, + max_nr_ptes, flags, page_shift); /* * Optimize for the small-folio common case by @@ -408,7 +487,7 @@ static long change_pte_range(struct mmu_gather *tlb, } else { pages += change_softleaf_pte(vma, addr, pte, oldpte, cp_flags); } - } while (pte += nr_ptes, addr += nr_ptes * PAGE_SIZE, addr != end); + } while (pte += nr_ptes, addr += nr_ptes * page_size, addr != end); lazy_mmu_mode_disable(); pte_unmap_unlock(pte - 1, ptl); @@ -487,12 +566,12 @@ static inline long change_pmd_range(struct mmu_gather *tlb, long pages = 0; unsigned long nr_huge_updates = 0; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(vma->vm_mm, pud, addr); do { long ret; pmd_t _pmd; again: - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(vma->vm_mm, addr, end); ret = change_pmd_prepare(vma, pmd, cp_flags); if (ret) { @@ -559,10 +638,10 @@ static inline long change_pud_range(struct mmu_gather *tlb, range.start = 0; - pudp = pud_offset(p4d, addr); + pudp = pud_offset_mm(vma->vm_mm, p4d, addr); do { again: - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, pudp, pmd, addr, cp_flags); if (ret) { pages = ret; @@ -615,13 +694,13 @@ static inline long change_p4d_range(struct mmu_gather *tlb, unsigned long next; long pages = 0, ret; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(vma->vm_mm, pgd, addr); do { - next = p4d_addr_end(addr, end); + next = p4d_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, p4d, pud, addr, cp_flags); if (ret) return ret; - if (p4d_none_or_clear_bad(p4d)) + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4d)) continue; pages += change_pud_range(tlb, vma, p4d, addr, next, newprot, cp_flags); @@ -643,7 +722,7 @@ static long change_protection_range(struct mmu_gather *tlb, pgd = pgd_offset(mm, addr); tlb_start_vma(tlb, vma); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, pgd, p4d, addr, cp_flags); if (ret) { pages = ret; @@ -729,7 +808,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, struct mm_struct *mm = vma->vm_mm; const vma_flags_t old_vma_flags = READ_ONCE(vma->flags); vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags); - long nrpages = (end - start) >> PAGE_SHIFT; + long nrpages = (end - start) >> mm_page_shift(mm); unsigned int mm_cp_flags = 0; unsigned long charged = 0; int error; @@ -777,7 +856,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, VMA_ACCOUNT_BIT, VMA_WRITE_BIT, VMA_HUGETLB_BIT, VMA_SHARED_BIT, VMA_NORESERVE_BIT)) { charged = nrpages; - if (security_vm_enough_memory_mm(mm, charged)) + if (mm_account_memory(mm, mm, charged)) return -ENOMEM; vma_flags_set(&new_vma_flags, VMA_ACCOUNT_BIT); } @@ -808,7 +887,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, if (vma_flags_test(&old_vma_flags, VMA_ACCOUNT_BIT) && !vma_flags_test(&new_vma_flags, VMA_ACCOUNT_BIT)) - vm_unacct_memory(nrpages); + mm_unacct_memory(mm, nrpages); /* * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major @@ -826,7 +905,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, return 0; fail: - vm_unacct_memory(charged); + mm_unacct_memory(mm, charged); return error; } @@ -851,11 +930,11 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ return -EINVAL; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(current->mm)) return -EINVAL; if (!len) return 0; - len = PAGE_ALIGN(len); + len = mm_page_align(current->mm, len); end = start + len; if (end <= start) return -ENOMEM; diff --git a/mm/mremap.c b/mm/mremap.c index e9c8b1d05832b..97d5ebf945285 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -82,11 +82,11 @@ static pud_t *get_old_pud(struct mm_struct *mm, unsigned long addr) if (pgd_none_or_clear_bad(pgd)) return NULL; - p4d = p4d_offset(pgd, addr); - if (p4d_none_or_clear_bad(p4d)) + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_or_clear_bad_mm(mm, p4d)) return NULL; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(mm, p4d, addr); if (pud_none_or_clear_bad(pud)) return NULL; @@ -102,7 +102,7 @@ static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr) if (!pud) return NULL; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(mm, pud, addr); if (pmd_none(*pmd)) return NULL; @@ -183,6 +183,14 @@ static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr if (max_nr == 1) return 1; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + folio = vm_normal_folio(vma, addr, pte); + return folio ? folio_subpage_pte_batch(vma, folio, ptep, pte, + max_nr, FPB_RESPECT_WRITE) : 1; + } +#endif + /* Avoid expensive folio lookup if we stand no chance of benefit. */ if (pte_batch_hint(ptep, pte) == 1) return 1; @@ -262,12 +270,12 @@ static int move_ptes(struct pagetable_move_control *pmc, flush_tlb_batched_pending(vma->vm_mm); lazy_mmu_mode_enable(); - for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE, - new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) { + for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * mm_page_size(mm), + new_ptep += nr_ptes, new_addr += nr_ptes * mm_page_size(mm)) { VM_WARN_ON_ONCE(!pte_none(*new_ptep)); nr_ptes = 1; - max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT; + max_nr_ptes = (old_end - old_addr) >> mm_page_shift(mm); old_pte = ptep_get(old_ptep); if (pte_none(old_pte)) continue; @@ -407,7 +415,7 @@ static bool move_normal_pmd(struct pagetable_move_control *pmc, VM_BUG_ON(!pmd_none(*new_pmd)); pmd_populate(mm, new_pmd, pmd_pgtable(pmd)); - flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PMD_SIZE); + flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + pmd_size_mm(mm)); out_unlock: if (new_ptl != old_ptl) spin_unlock(new_ptl); @@ -458,8 +466,8 @@ static bool move_normal_pud(struct pagetable_move_control *pmc, VM_BUG_ON(!pud_none(*new_pud)); - pud_populate(mm, new_pud, pud_pgtable(pud)); - flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PUD_SIZE); + pud_populate(mm, new_pud, pud_pgtable_mm(mm, pud)); + flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + pud_size_mm(mm)); if (new_ptl != old_ptl) spin_unlock(new_ptl); spin_unlock(old_ptl); @@ -549,13 +557,13 @@ static __always_inline unsigned long get_extent(enum pgt_entry entry, switch (entry) { case HPAGE_PMD: case NORMAL_PMD: - mask = PMD_MASK; - size = PMD_SIZE; + mask = pmd_mask_mm(pmc->old->vm_mm); + size = pmd_size_mm(pmc->old->vm_mm); break; case HPAGE_PUD: case NORMAL_PUD: - mask = PUD_MASK; - size = PUD_SIZE; + mask = pud_mask_mm(pmc->old->vm_mm); + size = pud_size_mm(pmc->old->vm_mm); break; default: BUILD_BUG(); @@ -811,7 +819,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) * If possible, realign addresses to PMD boundary for faster copy. * Only realign if the mremap copying hits a PMD boundary. */ - try_realign_addr(pmc, PMD_MASK); + try_realign_addr(pmc, pmd_mask_mm(mm)); flush_cache_range(pmc->old, pmc->old_addr, pmc->old_end); mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, mm, @@ -838,7 +846,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) /* We ignore and continue on error? */ continue; } - } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == PUD_SIZE) { + } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == pud_size_mm(mm)) { if (move_pgt_entry(pmc, NORMAL_PUD, old_pud, new_pud)) continue; } @@ -857,7 +865,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) continue; split_huge_pmd(pmc->old, old_pmd, pmc->old_addr); } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PMD) && - extent == PMD_SIZE) { + extent == pmd_size_mm(mm)) { /* * If the extent is PMD-sized, try to speed the move by * moving at the PMD level if possible. @@ -947,9 +955,7 @@ static unsigned long vrm_set_new_addr(struct vma_remap_struct *vrm) { struct vm_area_struct *vma = vrm->vma; unsigned long map_flags = 0; - /* Page Offset _into_ the VMA. */ - pgoff_t internal_pgoff = (vrm->addr - vma->vm_start) >> PAGE_SHIFT; - pgoff_t pgoff = vma->vm_pgoff + internal_pgoff; + pgoff_t pgoff = vma_page_offset_at(vma, vrm->addr).index; unsigned long new_addr = vrm_implies_new_addr(vrm) ? vrm->new_addr : 0; unsigned long res; @@ -985,13 +991,13 @@ static bool vrm_calc_charge(struct vma_remap_struct *vrm) * the length of the new one. Otherwise it's just the delta in size. */ if (vrm->flags & MREMAP_DONTUNMAP) - charged = vrm->new_len >> PAGE_SHIFT; + charged = vrm->new_len >> mm_page_shift(current->mm); else - charged = vrm->delta >> PAGE_SHIFT; + charged = vrm->delta >> mm_page_shift(current->mm); /* This accounts 'charged' pages of memory. */ - if (security_vm_enough_memory_mm(current->mm, charged)) + if (mm_account_memory(current->mm, current->mm, charged)) return false; vrm->charged = charged; @@ -1007,7 +1013,7 @@ static void vrm_uncharge(struct vma_remap_struct *vrm) if (!(vrm->vma->vm_flags & VM_ACCOUNT)) return; - vm_unacct_memory(vrm->charged); + mm_unacct_memory(current->mm, vrm->charged); vrm->charged = 0; } @@ -1019,13 +1025,13 @@ static void vrm_uncharge(struct vma_remap_struct *vrm) static void vrm_stat_account(struct vma_remap_struct *vrm, unsigned long bytes) { - unsigned long pages = bytes >> PAGE_SHIFT; + unsigned long pages = bytes >> mm_page_shift(current->mm); struct mm_struct *mm = current->mm; struct vm_area_struct *vma = vrm->vma; vm_stat_account(mm, vma->vm_flags, pages); if (vma->vm_flags & VM_LOCKED) - mm->locked_vm += pages; + mm->locked_vm += bytes >> MM_ACCOUNT_SHIFT; } static bool __check_map_count_against_split(struct mm_struct *mm, @@ -1198,7 +1204,7 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) vrm->vmi_needs_invalidate = true; if (err) { /* OOM: unable to split vma, just get accounts right */ - vm_acct_memory(len >> PAGE_SHIFT); + mm_acct_memory(current->mm, len >> mm_page_shift(current->mm)); return; } @@ -1255,16 +1261,14 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) static int copy_vma_and_data(struct vma_remap_struct *vrm, struct vm_area_struct **new_vma_ptr) { - unsigned long internal_offset = vrm->addr - vrm->vma->vm_start; - unsigned long internal_pgoff = internal_offset >> PAGE_SHIFT; - unsigned long new_pgoff = vrm->vma->vm_pgoff + internal_pgoff; + struct vm_page_offset pos = vma_page_offset_at(vrm->vma, vrm->addr); unsigned long moved_len; struct vm_area_struct *vma = vrm->vma; struct vm_area_struct *new_vma; int err = 0; PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len); - new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, new_pgoff, + new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, pos, &pmc.need_rmap_locks); if (!new_vma) { vrm_uncharge(vrm); @@ -1473,7 +1477,7 @@ static unsigned long mremap_to(struct vma_remap_struct *vrm) /* MREMAP_DONTUNMAP expands by old_len since old_len == new_len */ if (vrm->flags & MREMAP_DONTUNMAP) { vma_flags_t vma_flags = vrm->vma->flags; - unsigned long pages = vrm->old_len >> PAGE_SHIFT; + unsigned long pages = vrm->old_len >> mm_page_shift(current->mm); if (!may_expand_vm(mm, &vma_flags, pages)) return -ENOMEM; @@ -1495,7 +1499,7 @@ static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) if (find_vma_intersection(vma->vm_mm, vma->vm_end, end)) return 0; if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, - 0, MAP_FIXED) & ~PAGE_MASK) + 0, MAP_FIXED) & ~mm_page_mask(vma->vm_mm)) return 0; return 1; } @@ -1727,7 +1731,8 @@ static int check_prep_vma(struct vma_remap_struct *vrm) struct vm_area_struct *vma = vrm->vma; struct mm_struct *mm = current->mm; unsigned long addr = vrm->addr; - unsigned long old_len, new_len, pgoff; + unsigned long old_len, new_len; + struct vm_page_offset pos, end; if (!vma) return -EFAULT; @@ -1802,9 +1807,9 @@ static int check_prep_vma(struct vma_remap_struct *vrm) vrm->populate_expand = true; /* Need to be careful about a growing mapping */ - pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; - if (pgoff + (new_len >> PAGE_SHIFT) < pgoff) + pos = vma_page_offset_at(vma, addr); + end = vm_page_offset_add(pos, new_len); + if (end.index < pos.index) return -EINVAL; if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) @@ -1813,7 +1818,7 @@ static int check_prep_vma(struct vma_remap_struct *vrm) if (!mlock_future_ok(mm, vma->vm_flags & VM_LOCKED, vrm->delta)) return -EAGAIN; - if (!may_expand_vm(mm, &vma->flags, vrm->delta >> PAGE_SHIFT)) + if (!may_expand_vm(mm, &vma->flags, vrm->delta >> mm_page_shift(current->mm))) return -ENOMEM; return 0; @@ -1834,7 +1839,7 @@ static unsigned long check_mremap_params(struct vma_remap_struct *vrm) return -EINVAL; /* Start address must be page-aligned. */ - if (offset_in_page(addr)) + if (addr & ~mm_page_mask(current->mm)) return -EINVAL; /* @@ -1858,7 +1863,7 @@ static unsigned long check_mremap_params(struct vma_remap_struct *vrm) return -EINVAL; /* The new address must be page-aligned. */ - if (offset_in_page(vrm->new_addr)) + if (vrm->new_addr & ~mm_page_mask(current->mm)) return -EINVAL; /* A fixed address implies a move. */ @@ -1971,8 +1976,8 @@ static unsigned long do_mremap(struct vma_remap_struct *vrm) unsigned long res; bool failed; - vrm->old_len = PAGE_ALIGN(vrm->old_len); - vrm->new_len = PAGE_ALIGN(vrm->new_len); + vrm->old_len = mm_page_align(mm, vrm->old_len); + vrm->new_len = mm_page_align(mm, vrm->new_len); res = check_mremap_params(vrm); if (res) diff --git a/mm/mseal.c b/mm/mseal.c index e2093ae3d25cd..62df360b43545 100644 --- a/mm/mseal.c +++ b/mm/mseal.c @@ -151,10 +151,10 @@ int do_mseal(unsigned long start, size_t len_in, unsigned long flags) return -EINVAL; start = untagged_addr(start); - if (!PAGE_ALIGNED(start)) + if (start & ~mm_page_mask(mm)) return -EINVAL; - len = PAGE_ALIGN(len_in); + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero. */ if (len_in && !len) return -EINVAL; diff --git a/mm/msync.c b/mm/msync.c index ac4c9bfea2e7f..6d9027ee47246 100644 --- a/mm/msync.c +++ b/mm/msync.c @@ -41,12 +41,12 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) if (flags & ~(MS_ASYNC | MS_INVALIDATE | MS_SYNC)) goto out; - if (offset_in_page(start)) + if (start & ~mm_page_mask(mm)) goto out; if ((flags & MS_ASYNC) && (flags & MS_SYNC)) goto out; error = -ENOMEM; - len = (len + ~PAGE_MASK) & PAGE_MASK; + len = mm_page_align(mm, len); end = start + len; if (end < start) goto out; @@ -86,7 +86,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) } file = vma->vm_file; fstart = (start - vma->vm_start) + - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); fend = fstart + (min(end, vma->vm_end) - start) - 1; start = vma->vm_end; if ((flags & MS_SYNC) && file && diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 5f372f6e26fa3..5345f43c10b6f 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -225,7 +225,8 @@ long oom_badness(struct task_struct *p, unsigned long totalpages) * The baseline for the badness score is the proportion of RAM that each * task's rss, pagetable and swap space use. */ - points = get_mm_rss_sum(p->mm) + get_mm_counter_sum(p->mm, MM_SWAPENTS) + + points = mm_pages_to_native(p->mm, get_mm_rss_sum(p->mm) + + get_mm_counter_sum(p->mm, MM_SWAPENTS)) + mm_pgtables_bytes(p->mm) / PAGE_SIZE; task_unlock(p); @@ -399,10 +400,10 @@ static int dump_task(struct task_struct *p, void *arg) pr_info("[%7d] %5d %5d %8lu %8lu %8lu %8lu %9lu %8ld %8lu %5hd %s\n", task->pid, from_kuid(&init_user_ns, task_uid(task)), - task->tgid, task->mm->total_vm, get_mm_rss_sum(task->mm), - get_mm_counter_sum(task->mm, MM_ANONPAGES), get_mm_counter_sum(task->mm, MM_FILEPAGES), - get_mm_counter_sum(task->mm, MM_SHMEMPAGES), mm_pgtables_bytes(task->mm), - get_mm_counter_sum(task->mm, MM_SWAPENTS), + task->tgid, mm_pages_to_native(task->mm, task->mm->total_vm), mm_pages_to_native(task->mm, get_mm_rss_sum(task->mm)), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_ANONPAGES)), mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_FILEPAGES)), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_SHMEMPAGES)), mm_pgtables_bytes(task->mm), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_SWAPENTS)), task->signal->oom_score_adj, task->comm); task_unlock(task); @@ -588,9 +589,9 @@ static bool oom_reap_task_mm(struct task_struct *tsk, struct mm_struct *mm) pr_info("oom_reaper: reaped process %d (%s), now anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n", task_pid_nr(tsk), tsk->comm, - K(get_mm_counter_sum(mm, MM_ANONPAGES)), - K(get_mm_counter_sum(mm, MM_FILEPAGES)), - K(get_mm_counter_sum(mm, MM_SHMEMPAGES))); + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_ANONPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_FILEPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_SHMEMPAGES))); out_finish: trace_finish_task_reaping(tsk->pid); out_unlock: @@ -943,10 +944,10 @@ static void __oom_kill_process(struct task_struct *victim, const char *message) do_send_sig_info(SIGKILL, SEND_SIG_PRIV, victim, PIDTYPE_TGID); mark_oom_victim(victim); pr_err("%s: Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB, UID:%u pgtables:%lukB oom_score_adj:%d\n", - message, task_pid_nr(victim), victim->comm, K(mm->total_vm), - K(get_mm_counter_sum(mm, MM_ANONPAGES)), - K(get_mm_counter_sum(mm, MM_FILEPAGES)), - K(get_mm_counter_sum(mm, MM_SHMEMPAGES)), + message, task_pid_nr(victim), victim->comm, mm_pages_to_kb(mm, mm->total_vm), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_ANONPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_FILEPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_SHMEMPAGES)), from_kuid(&init_user_ns, task_uid(victim)), mm_pgtables_bytes(mm) >> 10, victim->signal->oom_score_adj); task_unlock(victim); diff --git a/mm/page_idle.c b/mm/page_idle.c index 9c67cbac29659..8af46b0d8f746 100644 --- a/mm/page_idle.c +++ b/mm/page_idle.c @@ -51,9 +51,10 @@ static struct folio *page_idle_get_folio(unsigned long pfn) static bool page_idle_clear_pte_refs_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long addr, void *arg) + struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + unsigned long addr = rmap.address; + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool referenced = false; while (page_vma_mapped_walk(&pvmw)) { @@ -69,7 +70,7 @@ static bool page_idle_clear_pte_refs_one(struct folio *folio, */ if (likely(pte_present(ptep_get(pvmw.pte)))) referenced |= ptep_test_and_clear_young(vma, addr, pvmw.pte); - referenced |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE); + referenced |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + mm_page_size(vma->vm_mm)); } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { pmd_t pmdval = pmdp_get(pvmw.pmd); diff --git a/mm/page_io.c b/mm/page_io.c index 70cea9e24d2fd..290706f931bdc 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "swap.h" static void __end_swap_bio_write(struct bio *bio) @@ -411,6 +412,58 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug) *swap_plug = sio; } +static DEFINE_MUTEX(swap_extent_io_mutex); +static struct bio_set swap_extent_bioset; + +int swap_extent_io_init(void) +{ + int ret = 0; + + /* Swapon prepares the reserve before any reclaim can use this file. */ + mutex_lock(&swap_extent_io_mutex); + if (!bioset_initialized(&swap_extent_bioset)) + ret = bioset_init(&swap_extent_bioset, SWAP_CLUSTER_MAX, 0, + BIOSET_NEED_RESCUER); + mutex_unlock(&swap_extent_io_mutex); + return ret; +} + +/* + * A mandatory large base page cannot split to fit a swap-file extent. Chain + * each physical run to the original bio: its folio completion must wait for + * every run, including on partial I/O or allocation failure. + */ +static void swap_submit_bio(struct bio *bio, void *private) +{ + struct folio *folio = private; + struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); + swp_entry_t entry = folio->swap; + + if (!(READ_ONCE(sis->flags) & SWP_BLKDEV) && folio_test_large(folio)) { + while (true) { + unsigned long nr = bio->bi_iter.bi_size >> PAGE_SHIFT; + unsigned int bytes; + struct bio *split; + + bio->bi_iter.bi_sector = swap_extent_sector(entry, &nr); + bytes = nr << PAGE_SHIFT; + if (bytes == bio->bi_iter.bi_size) + break; + /* The unsubmitted parent may own a bio from fs_bio_set. */ + split = bio_split(bio, bytes >> 9, GFP_NOIO, &swap_extent_bioset); + if (IS_ERR(split)) { + bio->bi_status = BLK_STS_RESOURCE; + bio_endio(bio); + return; + } + bio_chain(split, bio); + submit_bio(split); + entry.val += nr; + } + } + submit_bio(bio); +} + static void swap_writepage_bdev_sync(struct folio *folio, struct swap_info_struct *sis) { @@ -427,7 +480,7 @@ static void swap_writepage_bdev_sync(struct folio *folio, folio_start_writeback(folio); folio_unlock(folio); - submit_bio_wait(&bio); + bio_await(&bio, folio, swap_submit_bio); __end_swap_bio_write(&bio); } @@ -445,7 +498,7 @@ static void swap_writepage_bdev_async(struct folio *folio, count_swpout_vm_event(folio); folio_start_writeback(folio); folio_unlock(folio); - submit_bio(bio); + swap_submit_bio(bio, folio); } void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug) @@ -590,7 +643,7 @@ static void swap_read_folio_bdev_sync(struct folio *folio, count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio_wait(&bio); + bio_await(&bio, folio, swap_submit_bio); __end_swap_bio_read(&bio); put_task_struct(current); } @@ -607,7 +660,7 @@ static void swap_read_folio_bdev_async(struct folio *folio, count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio(bio); + swap_submit_bio(bio, folio); } void swap_read_folio(struct folio *folio, struct swap_iocb **plug) diff --git a/mm/page_table_check.c b/mm/page_table_check.c index 2403f5a114105..dc6797a41daaa 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -4,6 +4,7 @@ * Copyright (c) 2021, Google LLC. * Pasha Tatashin */ +#include #include #include #include @@ -13,11 +14,21 @@ #undef pr_fmt #define pr_fmt(fmt) "page_table_check: " fmt -struct page_table_check { +#ifdef CONFIG_MM_SUBPAGE +#define PAGE_TABLE_CHECK_SLOTS (PAGE_SIZE / SZ_4K) +#else +#define PAGE_TABLE_CHECK_SLOTS 1 +#endif + +struct page_table_check_slot { atomic_t anon_map_count; atomic_t file_map_count; }; +struct page_table_check { + struct page_table_check_slot slots[PAGE_TABLE_CHECK_SLOTS]; +}; + static bool __page_table_check_enabled __initdata = IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED); @@ -56,6 +67,74 @@ static struct page_table_check *get_page_table_check(struct page_ext *page_ext) return page_ext_data(page_ext, &page_table_check_ops); } +#if IS_ENABLED(CONFIG_KUNIT) +/* Diagnostic snapshot for private test allocations, not a synchronization API. */ +int page_table_check_get_counts(struct page *page, unsigned int offset, + int *anon, int *file) +{ + struct page_ext *ext; + struct page_table_check *ptc; + unsigned int slot = 0; + + if (static_branch_likely(&page_table_check_disabled)) + return -EOPNOTSUPP; + if (offset >= PAGE_SIZE) + return -EINVAL; +#ifdef CONFIG_MM_SUBPAGE + slot = offset / SZ_4K; +#endif + ext = page_ext_get(page); + if (!ext) + return -ENOENT; + ptc = get_page_table_check(ext); + *anon = atomic_read(&ptc->slots[slot].anon_map_count); + *file = atomic_read(&ptc->slots[slot].file_map_count); + page_ext_put(ext); + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(page_table_check_get_counts); +#endif + +static void page_table_check_update(struct page_table_check_slot *slot, + bool anon, bool clear, bool rw) +{ + if (anon) { + BUG_ON(atomic_read(&slot->file_map_count)); + if (clear) + BUG_ON(atomic_dec_return(&slot->anon_map_count) < 0); + else + BUG_ON(atomic_inc_return(&slot->anon_map_count) > 1 && rw); + } else { + BUG_ON(atomic_read(&slot->anon_map_count)); + if (clear) + BUG_ON(atomic_dec_return(&slot->file_map_count) < 0); + else + BUG_ON(atomic_inc_return(&slot->file_map_count) < 0); + } +} + +#ifdef CONFIG_MM_SUBPAGE +static void page_table_check_subpage(phys_addr_t phys, unsigned long size, + bool clear, bool rw) +{ + unsigned long pfn = PHYS_PFN(phys); + struct page_table_check *ptc; + struct page_ext *ext; + struct page *page; + unsigned int first = offset_in_page(phys) / SZ_4K, i; + + if (!pfn_valid(pfn)) + return; + page = pfn_to_page(pfn); + BUG_ON(PageSlab(page)); + ext = page_ext_get(page); + ptc = get_page_table_check(ext); + for (i = first; i < first + size / SZ_4K; i++) + page_table_check_update(&ptc->slots[i], PageAnon(page), clear, rw); + page_ext_put(ext); +} +#endif + /* * An entry is removed from the page table, decrement the counters for that page * verify that it is of correct type and counters do not become negative. @@ -78,13 +157,8 @@ static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt) for_each_page_ext(page, pgcnt, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - if (anon) { - BUG_ON(atomic_read(&ptc->file_map_count)); - BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0); - } else { - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0); - } + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) + page_table_check_update(&ptc->slots[i], anon, true, false); } rcu_read_unlock(); } @@ -113,13 +187,8 @@ static void page_table_check_set(unsigned long pfn, unsigned long pgcnt, for_each_page_ext(page, pgcnt, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - if (anon) { - BUG_ON(atomic_read(&ptc->file_map_count)); - BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw); - } else { - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0); - } + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) + page_table_check_update(&ptc->slots[i], anon, false, rw); } rcu_read_unlock(); } @@ -139,8 +208,10 @@ void __page_table_check_zero(struct page *page, unsigned int order) for_each_page_ext(page, 1 << order, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_read(&ptc->file_map_count)); + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) { + BUG_ON(atomic_read(&ptc->slots[i].anon_map_count)); + BUG_ON(atomic_read(&ptc->slots[i].file_map_count)); + } } rcu_read_unlock(); } @@ -151,8 +222,15 @@ void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, if (&init_mm == mm) return; - if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte)) - page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT); + if (!pte_user_accessible_page(mm, addr, pte) || pte_special(pte)) + return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + page_table_check_subpage(pte_phys_mm(mm, pte), mm_page_size(mm), true, false); + return; + } +#endif + page_table_check_clear(pte_pfn(pte), mm_pte_native_pages(mm)); } EXPORT_SYMBOL(__page_table_check_pte_clear); @@ -218,9 +296,20 @@ void __page_table_check_ptes_set(struct mm_struct *mm, unsigned long addr, page_table_check_pte_flags(pte); for (i = 0; i < nr; i++) - __page_table_check_pte_clear(mm, addr + PAGE_SIZE * i, ptep_get(ptep + i)); - if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte)) - page_table_check_set(pte_pfn(pte), nr, pte_write(pte)); + __page_table_check_pte_clear(mm, addr + mm_page_size(mm) * i, ptep_get(ptep + i)); + if (!pte_user_accessible_page(mm, addr, pte) || pte_special(pte)) + return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + phys_addr_t phys = pte_phys_mm(mm, pte); + + for (i = 0; i < nr; i++) + page_table_check_subpage(phys + i * mm_page_size(mm), mm_page_size(mm), false, + pte_write(pte)); + return; + } +#endif + page_table_check_set(pte_pfn(pte), nr * mm_pte_native_pages(mm), pte_write(pte)); } EXPORT_SYMBOL(__page_table_check_ptes_set); @@ -279,16 +368,17 @@ void __page_table_check_pte_clear_range(struct mm_struct *mm, return; if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { - pte_t *ptep = pte_offset_map(&pmd, addr); + pte_t *ptep = pte_offset_map_mm(mm, &pmd, addr); + unsigned long nr = pte_table_bytes_mm(mm) / sizeof(pte_t); unsigned long i; if (WARN_ON(!ptep)) return; - for (i = 0; i < PTRS_PER_PTE; i++) { + for (i = 0; i < nr; i++) { __page_table_check_pte_clear(mm, addr, ptep_get(ptep)); - addr += PAGE_SIZE; + addr += mm_page_size(mm); ptep++; } - pte_unmap(ptep - PTRS_PER_PTE); + pte_unmap(ptep - nr); } } diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index ccd1c622c8b24..e3b0b64fbfba2 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -87,7 +87,7 @@ static bool map_pte(struct page_vma_mapped_walk *pvmw, pmd_t *pmdvalp, * mapped at the @pvmw->pte * @pvmw: page_vma_mapped_walk struct, includes a pair pte and pfn range * for checking - * @pte_nr: the number of small pages described by @pvmw->pte. + * @pte_nr: the number of native physical pages covered by @pvmw->pte. * * page_vma_mapped_walk() found a place where pfn range is *potentially* * mapped. check_pte() has to validate this. @@ -109,6 +109,18 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr) unsigned long pfn; pte_t ptent = ptep_get(pvmw->pte); +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) { + if (!(pvmw->flags & PVMW_MIGRATION)) + return pte_present(ptent) && + pte_phys_mm(pvmw->vma->vm_mm, ptent) == + PFN_PHYS(pvmw->pfn) + pvmw->subpage_offset; + if (pte_swp_subpage_offset(ptent) != pvmw->subpage_offset) + return false; + /* The common migration check below validates the native PFN. */ + } +#endif + if (pvmw->flags & PVMW_MIGRATION) { const softleaf_t entry = softleaf_from_pte(ptent); @@ -188,6 +200,18 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) pud_t *pud; pmd_t pmde; +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) { + if (WARN_ON_ONCE(pvmw->nr_pages != 1 || + mm_page_size(mm) >= PAGE_SIZE || + !IS_ALIGNED(pvmw->address, mm_page_size(mm)) || + pvmw->subpage_offset >= PAGE_SIZE || + !IS_ALIGNED(pvmw->subpage_offset, mm_page_size(mm)) || + is_vm_hugetlb_page(vma))) + return not_found(pvmw); + } +#endif + /* The only possible pmd mapping has been handled on last iteration */ if (pvmw->pmd && !pvmw->pte) return not_found(pvmw); @@ -213,6 +237,9 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) return true; } + /* A query for an interior native page still visits one whole user leaf. */ + if (mm_page_size(mm) > PAGE_SIZE) + pvmw->address &= mm_page_mask(mm); end = vma_address_end(pvmw); if (pvmw->pte) goto next_pte; @@ -220,21 +247,21 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) do { pgd = pgd_offset(mm, pvmw->address); if (!pgd_present(*pgd)) { - step_forward(pvmw, PGDIR_SIZE); + step_forward(pvmw, pgd_size_mm(mm)); continue; } - p4d = p4d_offset(pgd, pvmw->address); - if (!p4d_present(*p4d)) { - step_forward(pvmw, P4D_SIZE); + p4d = p4d_offset_mm(mm, pgd, pvmw->address); + if (!p4d_present_mm(mm, *p4d)) { + step_forward(pvmw, p4d_size_mm(mm)); continue; } - pud = pud_offset(p4d, pvmw->address); + pud = pud_offset_mm(mm, p4d, pvmw->address); if (!pud_present(*pud)) { - step_forward(pvmw, PUD_SIZE); + step_forward(pvmw, pud_size_mm(mm)); continue; } - pvmw->pmd = pmd_offset(pud, pvmw->address); + pvmw->pmd = pmd_offset_mm(mm, pud, pvmw->address); /* * Make sure the pmd value isn't cached in a register by the * compiler and used as a stale value after we've observed a @@ -245,6 +272,9 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && (pmd_trans_huge(pmde) || pmd_is_migration_entry(pmde) || pmd_is_device_private_entry(pmde))) { + /* Subpage PMD mappings need a distinct huge-page implementation. */ + if (mm_page_size(mm) != PAGE_SIZE) + return not_found(pvmw); pvmw->ptl = pmd_lock(mm, pvmw->pmd); pmde = *pvmw->pmd; if (pmd_is_migration_entry(pmde)) { @@ -285,7 +315,7 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) (pvmw->nr_pages >= HPAGE_PMD_NR)) sync_with_folio_pmd_zap(mm, pvmw->pmd); - step_forward(pvmw, PMD_SIZE); + step_forward(pvmw, pmd_size_mm(mm)); continue; } if (!map_pte(pvmw, &pmde, &ptl)) { @@ -294,15 +324,15 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) goto next_pte; } this_pte: - if (check_pte(pvmw, 1)) + if (check_pte(pvmw, mm_pte_native_pages(mm))) return true; next_pte: do { - pvmw->address += PAGE_SIZE; + pvmw->address += mm_page_size(mm); if (pvmw->address >= end) return not_found(pvmw); /* Did we cross page table boundary? */ - if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) { + if ((pvmw->address & (pmd_size_mm(mm) - mm_page_size(mm))) == 0) { if (pvmw->ptl) { spin_unlock(pvmw->ptl); pvmw->ptl = NULL; @@ -345,6 +375,7 @@ unsigned long page_mapped_in_vma(const struct page *page, struct vm_area_struct *vma) { const struct folio *folio = page_folio(page); + unsigned long address; struct page_vma_mapped_walk pvmw = { .pfn = page_to_pfn(page), .nr_pages = 1, @@ -352,13 +383,15 @@ unsigned long page_mapped_in_vma(const struct page *page, .flags = PVMW_SYNC, }; - pvmw.address = vma_address(vma, page_pgoff(folio, page), 1); + address = vma_address(vma, page_pgoff(folio, page), 1); + pvmw.address = address; if (pvmw.address == -EFAULT) goto out; if (!page_vma_mapped_walk(&pvmw)) return -EFAULT; page_vma_mapped_walk_done(&pvmw); out: - return pvmw.address; + /* Report this native page's byte address, not the covering leaf start. */ + return address; } #endif diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 5d87c632a2550..c3216b4852399 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -30,13 +30,14 @@ static int walk_pte_range_inner(pte_t *pte, unsigned long addr, unsigned long end, struct mm_walk *walk) { const struct mm_walk_ops *ops = walk->ops; + const unsigned long page_size = mm_page_size(walk->mm); int err = 0; for (;;) { if (ops->install_pte && pte_none(ptep_get(pte))) { pte_t new_pte; - err = ops->install_pte(addr, addr + PAGE_SIZE, &new_pte, + err = ops->install_pte(addr, addr + page_size, &new_pte, walk); if (err) break; @@ -46,13 +47,13 @@ static int walk_pte_range_inner(pte_t *pte, unsigned long addr, if (!WARN_ON_ONCE(walk->no_vma)) update_mmu_cache(walk->vma, addr, pte); } else { - err = ops->pte_entry(pte, addr, addr + PAGE_SIZE, walk); + err = ops->pte_entry(pte, addr, addr + page_size, walk); if (err) break; } - if (addr >= end - PAGE_SIZE) + if (addr >= end - page_size) break; - addr += PAGE_SIZE; + addr += page_size; pte++; } return err; @@ -67,15 +68,15 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, if (walk->no_vma) { /* - * pte_offset_map() might apply user-specific validation. + * pte_offset_map_mm() might apply user-specific validation. * Indeed, on x86_64 the pmd entries set up by init_espfix_ap() * fit its pmd_bad() check (_PAGE_NX set and _PAGE_RW clear), * and CONFIG_EFI_PGT_DUMP efi_mm goes so far as to walk them. */ if (walk->mm == &init_mm || addr >= TASK_SIZE) - pte = pte_offset_kernel(pmd, addr); + pte = pte_offset_kernel_mm(walk->mm, pmd, addr); else - pte = pte_offset_map(pmd, addr); + pte = pte_offset_map_mm(walk->mm, pmd, addr); if (pte) { err = walk_pte_range_inner(pte, addr, end, walk); if (walk->mm != &init_mm && addr < TASK_SIZE) @@ -123,10 +124,10 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, return 0; } - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(walk->mm, pud, addr); do { again: - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(walk->mm, addr, end); if (pmd_none(*pmd)) { if (has_install) err = __pte_alloc(walk->mm, pmd); @@ -193,10 +194,10 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, int err = 0; int depth = real_depth(2); - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(walk->mm, p4d, addr); do { again: - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(walk->mm, addr, end); if (pud_none(*pud)) { if (has_install) err = __pmd_alloc(walk->mm, pud, addr); @@ -258,10 +259,10 @@ static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, int err = 0; int depth = real_depth(1); - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(walk->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) { + next = p4d_addr_end_mm(walk->mm, addr, end); + if (p4d_none_or_clear_bad_mm(walk->mm, p4d)) { if (has_install) err = __pud_alloc(walk->mm, p4d, addr); else if (ops->pte_hole) @@ -297,11 +298,11 @@ static int walk_pgd_range(unsigned long addr, unsigned long end, int err = 0; if (walk->pgd) - pgd = walk->pgd + pgd_index(addr); + pgd = pgd_offset_pgd_mm(walk->mm, walk->pgd, addr); else pgd = pgd_offset(walk->mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(walk->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) { if (has_install) err = __p4d_alloc(walk->mm, pgd, addr); @@ -924,11 +925,11 @@ struct folio *folio_walk_start(struct folio_walk *fw, if (pgd_none_or_clear_bad(pgdp)) goto not_found; - p4dp = p4d_offset(pgdp, addr); - if (p4d_none_or_clear_bad(p4dp)) + p4dp = p4d_offset_mm(vma->vm_mm, pgdp, addr); + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4dp)) goto not_found; - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_mm(vma->vm_mm, p4dp, addr); pud = pudp_get(pudp); if (pud_none(pud)) goto not_found; @@ -937,7 +938,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, ptl = pud_lock(vma->vm_mm, pudp); pud = pudp_get(pudp); - entry_size = PUD_SIZE; + entry_size = pud_size_mm(vma->vm_mm); fw->level = FW_LEVEL_PUD; fw->pudp = pudp; fw->pud = pud; @@ -959,7 +960,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, pmd_table: VM_WARN_ON_ONCE(!pud_present(pud) || pud_leaf(pud)); - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_mm(vma->vm_mm, pudp, addr); pmd = pmdp_get_lockless(pmdp); if (pmd_none(pmd)) goto not_found; @@ -968,7 +969,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, ptl = pmd_lock(vma->vm_mm, pmdp); pmd = pmdp_get(pmdp); - entry_size = PMD_SIZE; + entry_size = pmd_size_mm(vma->vm_mm); fw->level = FW_LEVEL_PMD; fw->pmdp = pmdp; fw->pmd = pmd; @@ -1001,7 +1002,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, goto not_found; pte = ptep_get(ptep); - entry_size = PAGE_SIZE; + entry_size = mm_page_size(vma->vm_mm); fw->level = FW_LEVEL_PTE; fw->ptep = ptep; fw->pte = pte; @@ -1028,5 +1029,5 @@ struct folio *folio_walk_start(struct folio_walk *fw, else fw->page = NULL; fw->ptl = ptl; - return page_folio(page); + return page_folio(zeropage ? page : fw->page); } diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index b91b1a98029c7..7a9eef9b4f5b7 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -168,13 +168,15 @@ pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable) { + struct page *page = pgtable_to_page(pgtable); + assert_spin_locked(pmd_lockptr(mm, pmdp)); /* FIFO */ if (!pmd_huge_pte(mm, pmdp)) - INIT_LIST_HEAD(&pgtable->lru); + INIT_LIST_HEAD(&page->lru); else - list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru); + list_add(&page->lru, &pgtable_to_page(pmd_huge_pte(mm, pmdp))->lru); pmd_huge_pte(mm, pmdp) = pgtable; } #endif @@ -184,15 +186,17 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) { pgtable_t pgtable; + struct page *page, *next; assert_spin_locked(pmd_lockptr(mm, pmdp)); /* FIFO */ pgtable = pmd_huge_pte(mm, pmdp); - pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru, - struct page, lru); + page = pgtable_to_page(pgtable); + next = list_first_entry_or_null(&page->lru, struct page, lru); + pmd_huge_pte(mm, pmdp) = pgtable_from_page(next); if (pmd_huge_pte(mm, pmdp)) - list_del(&pgtable->lru); + list_del(&page->lru); return pgtable; } #endif @@ -282,7 +286,12 @@ static unsigned long pmdp_get_lockless_start(void) { return 0; } static void pmdp_get_lockless_end(unsigned long irqflags) { } #endif +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, pmd_t *pmdvalp) +#else pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) +#endif { unsigned long irqflags; pmd_t pmdval; @@ -302,19 +311,27 @@ pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) pmd_clear_bad(pmd); goto nomap; } - return __pte_map(&pmdval, addr); + return __pte_map_mm(mm, &pmdval, addr); nomap: rcu_read_unlock(); return NULL; } +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) +{ + /* NULL retains native geometry for callers not yet converted. */ + return __pte_offset_map_mm(NULL, pmd, addr, pmdvalp); +} +#endif + pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp) { pmd_t pmdval; pte_t *pte; - pte = __pte_offset_map(pmd, addr, &pmdval); + pte = __pte_offset_map_mm(mm, pmd, addr, &pmdval); if (likely(pte)) *ptlp = pte_lockptr(mm, &pmdval); return pte; @@ -327,7 +344,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, pte_t *pte; VM_WARN_ON_ONCE(!pmdvalp); - pte = __pte_offset_map(pmd, addr, pmdvalp); + pte = __pte_offset_map_mm(mm, pmd, addr, pmdvalp); if (likely(pte)) *ptlp = pte_lockptr(mm, pmdvalp); return pte; @@ -397,7 +414,7 @@ pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, pmd_t pmdval; pte_t *pte; again: - pte = __pte_offset_map(pmd, addr, &pmdval); + pte = __pte_offset_map_mm(mm, pmd, addr, &pmdval); if (unlikely(!pte)) return pte; ptl = pte_lockptr(mm, &pmdval); diff --git a/mm/process_vm_access.c b/mm/process_vm_access.c index 656d3e88755b6..f178b5fdb6589 100644 --- a/mm/process_vm_access.c +++ b/mm/process_vm_access.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,8 @@ #include #include +#include "internal.h" + /** * process_vm_rw_pages - read/write pages from task specified * @pages: array of pointers to pages we want to copy @@ -135,6 +138,63 @@ static int process_vm_rw_single_vec(unsigned long addr, which lives on stack */ #define PVM_MAX_PP_ARRAY_COUNT 16 +#ifdef CONFIG_MM_SUBPAGE +/* Same pin/copy/dirty lifetime as the page-array path, with explicit offsets. */ +int process_vm_rw_fragments(struct mm_struct *mm, unsigned long addr, + unsigned long len, struct iov_iter *iter, bool write) +{ + struct user_page_fragment stack[PVM_MAX_PP_ARRAY_COUNT]; + struct user_page_fragment *fragments = stack; + unsigned long capacity, nr; + int ret = 0; + + if (!len || !iov_iter_count(iter)) + return 0; + capacity = 1 + ((len - 1) >> MM_SUBPAGE_SHIFT) + + ((((len - 1) & (MM_SUBPAGE_SIZE - 1)) + + (addr & (MM_SUBPAGE_SIZE - 1))) >> MM_SUBPAGE_SHIFT); + capacity = min_t(unsigned long, capacity, + PVM_MAX_KMALLOC_PAGES * PAGE_SIZE / sizeof(*fragments)); + if (capacity > ARRAY_SIZE(stack)) { + fragments = kmalloc_array(capacity, sizeof(*fragments), GFP_KERNEL); + if (!fragments) + return -ENOMEM; + } + while (len && iov_iter_count(iter)) { + long got = pin_user_fragments_remote(mm, addr, len, + write ? FOLL_WRITE : 0, fragments, capacity); + + if (got <= 0) { + ret = -EFAULT; + break; + } + for (nr = 0; nr < got && iov_iter_count(iter); nr++) { + struct user_page_fragment *f = &fragments[nr]; + struct page *page = folio_page(f->folio, f->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(f->offset); + size_t copied; + + if (write) + copied = copy_page_from_iter(page, offset, f->length, iter); + else + copied = copy_page_to_iter(page, offset, f->length, iter); + addr += copied; + len -= copied; + if (copied < f->length && iov_iter_count(iter)) { + ret = -EFAULT; + break; + } + } + release_user_fragments(fragments, got, write); + if (ret) + break; + } + if (fragments != stack) + kfree(fragments); + return ret; +} +#endif + /** * process_vm_rw_core - core of reading/writing pages from task specified * @pid: PID of process to read/write from/to @@ -182,17 +242,6 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, if (nr_pages == 0) return 0; - if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) { - /* For reliability don't try to kmalloc more than - 2 pages worth */ - process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES * PAGE_SIZE, - sizeof(struct page *)*nr_pages), - GFP_KERNEL); - - if (!process_pages) - return -ENOMEM; - } - /* Get process information */ task = find_get_task_by_vpid(pid); if (!task) { @@ -212,11 +261,35 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, goto put_task_struct; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + for (i = 0; i < riovcnt && iov_iter_count(iter) && !rc; i++) + rc = process_vm_rw_fragments(mm, (unsigned long)rvec[i].iov_base, + rvec[i].iov_len, iter, vm_write); + goto copied; + } +#endif + if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) { + /* For reliability don't try to kmalloc more than + 2 pages worth */ + process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES * PAGE_SIZE, + sizeof(struct page *)*nr_pages), + GFP_KERNEL); + + if (!process_pages) { + rc = -ENOMEM; + goto put_mm; + } + } + for (i = 0; i < riovcnt && iov_iter_count(iter) && !rc; i++) rc = process_vm_rw_single_vec( (unsigned long)rvec[i].iov_base, rvec[i].iov_len, iter, process_pages, mm, task, vm_write); +#ifdef CONFIG_MM_SUBPAGE +copied: +#endif /* copied = space before - space after */ total_len -= iov_iter_count(iter); @@ -226,6 +299,7 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, if (total_len) rc = total_len; +put_mm: mmput(mm); put_task_struct: diff --git a/mm/rmap.c b/mm/rmap.c index 99e1b3dc390b7..795b568b9f7c5 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -75,6 +75,7 @@ #include #include #include +#include #include @@ -83,17 +84,22 @@ #include "internal.h" #include "swap.h" +#include "user-subpage-internal.h" static struct kmem_cache *anon_vma_cachep; static struct kmem_cache *anon_vma_chain_cachep; -static inline struct anon_vma *anon_vma_alloc(void) +static inline struct anon_vma *anon_vma_alloc(struct vm_area_struct *vma) { struct anon_vma *anon_vma; anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); if (anon_vma) { atomic_set(&anon_vma->refcount, 1); +#ifdef CONFIG_MM_SUBPAGE + anon_vma->min_folio_order = + max_t(unsigned int, mm_page_shift(vma->vm_mm), PAGE_SHIFT) - PAGE_SHIFT; +#endif anon_vma->num_children = 0; anon_vma->num_active_vmas = 0; anon_vma->parent = anon_vma; @@ -198,7 +204,7 @@ int __anon_vma_prepare(struct vm_area_struct *vma) anon_vma = find_mergeable_anon_vma(vma); allocated = NULL; if (!anon_vma) { - anon_vma = anon_vma_alloc(); + anon_vma = anon_vma_alloc(vma); if (unlikely(!anon_vma)) goto out_enomem_free_avc; anon_vma->num_children++; /* self-parent link for new root */ @@ -388,7 +394,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) /* Drop inherited anon_vma, we'll reuse existing or allocate new. */ vma->anon_vma = NULL; - anon_vma = anon_vma_alloc(); + anon_vma = anon_vma_alloc(vma); if (!anon_vma) return -ENOMEM; avc = anon_vma_chain_alloc(GFP_KERNEL); @@ -622,6 +628,35 @@ struct anon_vma *folio_get_anon_vma(const struct folio *folio) return anon_vma; } +#ifdef CONFIG_MM_SUBPAGE +/* The caller retains the folio; splitting rechecks the order under its lock. */ +unsigned int folio_anon_min_order(struct folio *folio) +{ + struct anon_vma *anon_vma = NULL; + unsigned long mapping; + unsigned int order = 0; + + /* Also used by memory-failure callers before they lock the folio. */ + rcu_read_lock(); + mapping = (unsigned long)READ_ONCE(folio->mapping); + if ((mapping & FOLIO_MAPPING_FLAGS) != FOLIO_MAPPING_ANON || !folio_mapped(folio)) + goto out; + anon_vma = (struct anon_vma *)(mapping - FOLIO_MAPPING_ANON); + if (!atomic_inc_not_zero(&anon_vma->refcount)) { + anon_vma = NULL; + goto out; + } + /* Revalidate against unmap/reuse of the RCU-typesafe anon_vma slab. */ + if (folio_mapped(folio) && mapping == (unsigned long)READ_ONCE(folio->mapping)) + order = anon_vma->min_folio_order; +out: + rcu_read_unlock(); + if (anon_vma) + put_anon_vma(anon_vma); + return order; +} +#endif + /* * Similar to folio_get_anon_vma() except it locks the anon_vma. * @@ -856,6 +891,13 @@ static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) unsigned long page_address_in_vma(const struct folio *folio, const struct page *page, const struct vm_area_struct *vma) { +#ifdef CONFIG_MM_SUBPAGE + unsigned long address; + + if (folio_test_anon(folio) && !folio_test_large(folio) && + mm_subpage_address_in_vma(folio, vma, &address)) + return address; +#endif if (folio_test_anon(folio)) { struct anon_vma *anon_vma = folio_anon_vma(folio); /* @@ -891,15 +933,15 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) if (!pgd_present(*pgd)) goto out; - p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + p4d = p4d_offset_mm(mm, pgd, address); + if (!p4d_present_mm(mm, *p4d)) goto out; - pud = pud_offset(p4d, address); + pud = pud_offset_mm(mm, p4d, address); if (!pud_present(*pud)) goto out; - pmd = pmd_offset(pud, address); + pmd = pmd_offset_mm(mm, pud, address); out: return pmd; } @@ -915,10 +957,11 @@ struct folio_referenced_arg { * arg: folio_referenced_arg will be passed */ static bool folio_referenced_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long address, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct folio_referenced_arg *pra = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); int ptes = 0, referenced = 0; unsigned int nr; @@ -928,10 +971,11 @@ static bool folio_referenced_one(struct folio *folio, if (vma->vm_flags & VM_LOCKED) { ptes++; - pra->mapcount--; + pra->mapcount -= pvmw.pte ? mm_pte_native_pages(vma->vm_mm) : 1; /* Only mlock fully mapped pages */ - if (pvmw.pte && ptes != pvmw.nr_pages) + if (pvmw.pte && ptes != (folio_test_large(folio) ? + ((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm) : 1)) continue; /* @@ -966,11 +1010,18 @@ static bool folio_referenced_one(struct folio *folio, } if (pvmw.pte && folio_test_large(folio)) { - const unsigned long end_addr = pmd_addr_end(address, vma->vm_end); - const unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT; + const unsigned long end_addr = pmd_addr_end_mm(vma->vm_mm, + address, vma->vm_end); + const unsigned int max_nr = (end_addr - address) >> mm_page_shift(vma->vm_mm); pte_t pteval = ptep_get(pvmw.pte); - nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + nr = folio_subpage_pte_batch(vma, folio, pvmw.pte, + pteval, max_nr, 0); + else +#endif + nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr); } /* @@ -994,19 +1045,19 @@ static bool folio_referenced_one(struct folio *folio, } ptes += nr; - pra->mapcount -= nr; + pra->mapcount -= nr * (pvmw.pte ? mm_pte_native_pages(vma->vm_mm) : 1); /* * If we are sure that we batched the entire folio, * we can just optimize and stop right here. */ - if (ptes == pvmw.nr_pages) { + if (ptes == (((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm))) { page_vma_mapped_walk_done(&pvmw); break; } /* Skip the batched PTEs */ pvmw.pte += nr - 1; - pvmw.address += (nr - 1) * PAGE_SIZE; + pvmw.address += (nr - 1) * mm_page_size(vma->vm_mm); } if (referenced) @@ -1178,9 +1229,9 @@ static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw) } static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, PVMW_SYNC); int *cleaned = arg; *cleaned += page_vma_mkclean_one(&pvmw); @@ -1229,8 +1280,9 @@ struct wrprotect_file_state { }; static bool mapping_wrprotect_range_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long address, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct wrprotect_file_state *state = (struct wrprotect_file_state *)arg; struct page_vma_mapped_walk pvmw = { .pfn = state->pfn, @@ -1515,7 +1567,7 @@ static void __page_check_anon_rmap(const struct folio *folio, static __always_inline void __folio_add_anon_rmap(struct folio *folio, struct page *page, int nr_pages, struct vm_area_struct *vma, - unsigned long address, rmap_t flags, enum pgtable_level level) + unsigned long address, rmap_t flags, enum pgtable_level level, bool new) { int i; @@ -1572,7 +1624,7 @@ static __always_inline void __folio_add_anon_rmap(struct folio *folio, * Partially mapped folios can be split on reclaim and part outside * of mlocked VMA can be evicted or freed. */ - if (folio_nr_pages(folio) == nr_pages) + if (!new && folio_nr_pages(folio) == nr_pages) mlock_vma_folio(folio, vma); } @@ -1597,7 +1649,7 @@ void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page, rmap_t flags) { __folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags, - PGTABLE_LEVEL_PTE); + PGTABLE_LEVEL_PTE, false); } /** @@ -1618,7 +1670,7 @@ void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page, { #ifdef CONFIG_TRANSPARENT_HUGEPAGE __folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags, - PGTABLE_LEVEL_PMD); + PGTABLE_LEVEL_PMD, false); #else WARN_ON_ONCE(true); #endif @@ -1697,6 +1749,112 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1); } +#ifdef CONFIG_MM_SUBPAGE +static int __mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address, bool new) +{ + struct folio *folio = mm_subpage_folio(subpage); + int ret; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (mm_page_shift(vma->vm_mm) != mm_subpage_shift(subpage) || !vma->anon_vma || + address < vma->vm_start || address >= vma->vm_end || + !IS_ALIGNED(address, mm_subpage_size(subpage))) + return -EINVAL; + if (mm_subpage_anon_root(subpage)) { + if (mm_subpage_is_droppable(subpage) != !!(vma->vm_flags & VM_DROPPABLE)) + return -EINVAL; + if (!folio_test_anon(folio) || folio_test_ksm(folio) || + PageAnonExclusive(&folio->page)) + return -EINVAL; + } else { + bool was_anon = folio_test_anon(folio); + + /* Swap-cache folios can retain a stale, unmapped anon pointer. */ + if (folio_mapped(folio) || folio_test_ksm(folio) || + (folio->mapping && !was_anon) || + (was_anon && PageAnonExclusive(&folio->page))) + return -EINVAL; + /* Slot references can survive the last VMA in this root. */ + mm_subpage_hold_anon_root(subpage, vma->anon_vma->root, + vma->vm_flags & VM_DROPPABLE); + /* + * A restored folio is already visible in the swap cache. Other + * faults may set PG_waiters while we hold its lock; preserve those + * concurrent flag updates even when PG_swapbacked is already set. + */ + if (!(vma->vm_flags & VM_DROPPABLE)) + folio_set_swapbacked(folio); + /* Neighboring slots may acquire different fork/COW relationships. */ + __folio_set_anon(folio, vma, address, false); + if (!was_anon) + mod_mthp_stat(0, MTHP_STAT_NR_ANON, 1); + } + ret = mm_subpage_bind_rmap(subpage, vma, address); + if (ret) + return ret; + atomic_inc(&subpage->mapcount); + /* Identity was checked per slot, not against the folio's linear anchor. */ + __folio_add_rmap(folio, &folio->page, 1, vma, PGTABLE_LEVEL_PTE); + if (!new) + mlock_vma_folio(folio, vma); + return 0; +} + +int mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address) +{ + return __mm_subpage_add_anon_rmap(subpage, vma, address, false); +} + +/* Caller adds this fresh folio to LRU and accounts its locked leaves together. */ +int mm_subpage_add_new_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address) +{ + return __mm_subpage_add_anon_rmap(subpage, vma, address, true); +} + +void mm_subpage_remove_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma) +{ + struct folio *folio = mm_subpage_folio(subpage); + + VM_BUG_ON_FOLIO(atomic_read(&subpage->mapcount) <= 0, folio); + atomic_dec(&subpage->mapcount); + folio_remove_rmap_pte(folio, &folio->page, vma); +} + +int mm_subpage_dup_anon_rmap(struct mm_subpage *subpage, + struct vm_area_struct *dst, struct vm_area_struct *src, unsigned long address) +{ + struct folio *folio = mm_subpage_folio(subpage); + struct vm_page_offset pos; + int ret; + + mmap_assert_write_locked(src->vm_mm); + mmap_assert_write_locked(dst->vm_mm); + if (mm_page_shift(src->vm_mm) != mm_subpage_shift(subpage) || + mm_page_shift(dst->vm_mm) != mm_subpage_shift(subpage) || + !src->anon_vma || !dst->anon_vma || !folio_test_anon(folio) || + folio_test_ksm(folio) || PageAnonExclusive(&folio->page) || + address < dst->vm_start || address >= dst->vm_end || + address < src->vm_start || address >= src->vm_end) + return -EINVAL; + pos = vma_page_offset_at(dst, address); + /* Both mmap locks and the source PTE stabilize this slot's identity. */ + if (!mm_subpage_match_rmap(subpage, dst, address) || + src->anon_vma->root != dst->anon_vma->root || + !vm_page_offset_equal(pos, vma_page_offset_at(src, address))) + return -EINVAL; + /* A pin in a neighboring physical slot does not share this slot. */ + if (!mm_subpage_try_share(subpage)) + return -EBUSY; + ret = folio_try_dup_anon_rmap_pte(folio, &folio->page, dst, src); + if (!ret) + atomic_inc(&subpage->mapcount); + return ret; +} +#endif + static __always_inline void __folio_add_file_rmap(struct folio *folio, struct page *page, int nr_pages, struct vm_area_struct *vma, enum pgtable_level level) @@ -1949,14 +2107,20 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, struct vm_area_struct *vma = pvmw->vma; unsigned int max_nr; +#ifdef CONFIG_MM_SUBPAGE + /* Native PFN batching advances physical pages, not 4K file quarters. */ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return 1; +#endif + if (flags & TTU_HWPOISON) return 1; if (!folio_test_large(folio)) return 1; /* We may only batch within a single VMA and a single page table. */ - end_addr = pmd_addr_end(addr, vma->vm_end); - max_nr = (end_addr - addr) >> PAGE_SHIFT; + end_addr = pmd_addr_end_mm(vma->vm_mm, addr, vma->vm_end); + max_nr = (end_addr - addr) >> mm_page_shift(vma->vm_mm); /* We only support lazyfree or file folios batching for now ... */ if (folio_test_anon(folio) && folio_test_swapbacked(folio)) @@ -1978,20 +2142,116 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY); } +#ifdef CONFIG_MM_SUBPAGE +/* Folio and PTE locks held. Each PTE takes one reference to a native swap entry. */ +static bool try_to_unmap_subpage(struct folio *folio, struct vm_area_struct *vma, + struct page_vma_mapped_walk *pvmw, enum ttu_flags flags) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long address = pvmw->address, end = address + mm_page_size(mm); + struct mm_subpage *slot; + pte_t pte = ptep_get(pvmw->pte), swapped; + swp_entry_t entry; + bool exclusive, ret = false; + + if (!pte_present(pte) || (flags & TTU_HWPOISON)) + return false; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (WARN_ON_ONCE(!slot)) + return false; + if ((vma->vm_flags & VM_DROPPABLE) || mm_subpage_is_lazyfree(slot)) { + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + /* A released quarter can be reused before its native folio is freed. */ + flush_tlb_range(vma, address, end); + smp_mb(); + if (((vma->vm_flags & VM_DROPPABLE) || !pte_dirty(pte)) && + mm_subpage_discardable(slot) && + arch_unmap_one(mm, vma, address, pte) >= 0) { + update_hiwater_rss(mm); + dec_mm_counter(mm, MM_ANONPAGES); + mm_subpage_remove_anon_rmap(slot, vma); + mm_subpage_put(slot); /* Removed PTE reference. */ + ret = true; + goto out; + } + set_pte_at(mm, address, pvmw->pte, pte); + /* Retained references must not turn a droppable folio into swap data. */ + if (vma->vm_flags & VM_DROPPABLE) + goto out; + if (pte_dirty(pte)) { + mm_subpage_cancel_lazyfree(slot); + folio_mark_dirty(folio); + } + } + if (!folio_test_swapbacked(folio)) { + /* A write or retained reference cancelled whole-folio lazy reclaim. */ + folio_set_swapbacked(folio); + goto out; + } + if (!folio_test_swapcache(folio)) + goto out; + entry = page_swap_entry(&folio->page); + if (swap_subpage_dup(entry, mm_subpage_offset(slot)) < 0) + goto out; + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + if (should_defer_flush(mm, flags)) + set_tlb_ubc_flush_pending(mm, pte, address, end); + else + flush_tlb_range(vma, address, end); + if (pte_dirty(pte)) + folio_mark_dirty(folio); + /* The PTE lock serializes fork and this slot's ownership transfer. */ + exclusive = mm_subpage_is_exclusive(slot); + if (arch_unmap_one(mm, vma, address, pte) < 0 || !mm_subpage_try_share(slot)) { + swap_subpage_put(entry, mm_subpage_offset(slot), false); + set_pte_at(mm, address, pvmw->pte, pte); + goto out; + } + if (list_empty(&mm->mmlist)) { + spin_lock(&mmlist_lock); + if (list_empty(&mm->mmlist)) + list_add(&mm->mmlist, &init_mm.mmlist); + spin_unlock(&mmlist_lock); + } + swapped = pte_swp_set_subpage_offset(swp_entry_to_pte(entry), mm_subpage_offset(slot)); + if (exclusive) + swapped = pte_swp_mkexclusive(swapped); + if (pte_soft_dirty(pte)) + swapped = pte_swp_mksoft_dirty(swapped); + if (pte_uffd_wp(pte)) + swapped = pte_swp_mkuffd_wp(swapped); + update_hiwater_rss(mm); + dec_mm_counter(mm, MM_ANONPAGES); + inc_mm_counter(mm, MM_SWAPENTS); + set_pte_at(mm, address, pvmw->pte, swapped); + mm_subpage_remove_anon_rmap(slot, vma); + /* Swap-cache backing cannot be recycled while a deferred TLBI is pending. */ + mm_subpage_put(slot); + ret = true; +out: + mm_subpage_put(slot); + return ret; +} +#endif + /* * @arg: enum ttu_flags will be passed to this argument */ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct mm_struct *mm = vma->vm_mm; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool anon_exclusive, ret = true; pte_t pteval; + phys_addr_t file_phys = 0; struct page *subpage; struct mmu_notifier_range range; enum ttu_flags flags = (enum ttu_flags)(long)arg; - unsigned long nr_pages = 1, end_addr; + unsigned long nr_pages = 1, end_addr, nr_refs; unsigned long pfn; unsigned long hsz = 0; int ptes = 0; @@ -2003,7 +2263,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * if page table locking is skipped: use TTU_SYNC to wait for that. */ if (flags & TTU_SYNC) - pvmw.flags = PVMW_SYNC; + pvmw.flags |= PVMW_SYNC; /* * For THP, we have to assume the worse case ie pmd for invalidation. @@ -2049,7 +2309,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, ret = false; /* Only mlock fully mapped pages */ - if (pvmw.pte && ptes != pvmw.nr_pages) + if (pvmw.pte && ptes != (folio_test_large(folio) ? + ((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm) : 1)) continue; /* @@ -2100,6 +2361,15 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * actually map pages. */ pteval = ptep_get(pvmw.pte); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE && folio_test_anon(folio)) { + if (!try_to_unmap_subpage(folio, vma, &pvmw, flags)) + goto walk_abort; + /* One native folio may cover four independently mapped slots. */ + continue; + } +#endif + if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { @@ -2110,6 +2380,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, } subpage = folio_page(folio, pfn - folio_pfn(folio)); + if (!folio_test_anon(folio)) + file_phys = pte_phys_mm(mm, ptep_get(pvmw.pte)); address = pvmw.address; anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(subpage); @@ -2167,7 +2439,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, folio_mark_dirty(folio); } else if (likely(pte_present(pteval))) { nr_pages = folio_unmap_pte_batch(folio, &pvmw, flags, pteval); - end_addr = address + nr_pages * PAGE_SIZE; + end_addr = address + nr_pages * mm_page_size(mm); flush_cache_range(vma, address, end_addr); /* Nuke the page table entry. */ @@ -2281,7 +2553,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, goto discard; } - if (folio_dup_swap(folio, subpage) < 0) { + if (folio_dup_swap_range(folio, subpage, mm_pte_native_pages(mm)) < 0) { set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } @@ -2292,15 +2564,15 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * so we'll not check/care. */ if (arch_unmap_one(mm, vma, address, pteval) < 0) { - folio_put_swap(folio, subpage); + folio_put_swap_range(folio, subpage, mm_pte_native_pages(mm)); set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } /* See folio_try_share_anon_rmap(): clear PTE first. */ if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, subpage)) { - folio_put_swap(folio, subpage); + folio_try_share_anon_rmap_pte(folio, subpage, vma)) { + folio_put_swap_range(folio, subpage, mm_pte_native_pages(mm)); set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } @@ -2342,20 +2614,26 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, add_mm_counter(mm, mm_counter_file(folio), -nr_pages); } discard: + nr_refs = nr_pages; + /* A base user leaf can own references to several native pages. */ + if (!folio_test_hugetlb(folio)) + nr_refs *= mm_pte_native_pages(mm); + if (!folio_test_anon(folio)) + mm_subpage_file_map_del(vma, file_phys); if (unlikely(folio_test_hugetlb(folio))) { hugetlb_remove_rmap(folio); } else { - folio_remove_rmap_ptes(folio, subpage, nr_pages, vma); + folio_remove_rmap_ptes(folio, subpage, nr_refs, vma); } if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); - folio_put_refs(folio, nr_pages); + folio_put_refs(folio, nr_refs); /* * If we are sure that we batched the entire folio and cleared * all PTEs, we can just optimize and stop right here. */ - if (nr_pages == folio_nr_pages(folio)) + if (nr_pages == folio_nr_pages(folio) && mm_page_size(mm) == PAGE_SIZE) goto walk_done; continue; walk_abort: @@ -2412,11 +2690,72 @@ void try_to_unmap(struct folio *folio, enum ttu_flags flags) * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs * containing migration entries. */ +#ifdef CONFIG_MM_SUBPAGE +static bool try_to_migrate_subpage(struct folio *folio, struct vm_area_struct *vma, + struct page_vma_mapped_walk *pvmw, enum ttu_flags flags) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long address = pvmw->address, end = address + mm_page_size(mm); + struct mm_subpage *slot; + pte_t pte = ptep_get(pvmw->pte), migration; + swp_entry_t entry; + bool ret = false; + + /* Other migration drivers must prepare ownership before removing PTEs. */ + if (!pte_present(pte) || !mm_subpage_migrating(folio) || + folio_test_hwpoison(folio) || folio_is_zone_device(folio)) + return false; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (WARN_ON_ONCE(!slot)) + return false; + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + if (should_defer_flush(mm, flags)) + set_tlb_ubc_flush_pending(mm, pte, address, end); + else + flush_tlb_range(vma, address, end); + if (pte_dirty(pte)) + folio_mark_dirty(folio); + if (arch_unmap_one(mm, vma, address, pte) < 0) { + set_pte_at(mm, address, pvmw->pte, pte); + goto out; + } + if (pte_write(pte)) + entry = make_writable_migration_entry(folio_pfn(folio)); + else if (mm_subpage_is_exclusive(slot)) + entry = make_readable_exclusive_migration_entry(folio_pfn(folio)); + else + entry = make_readable_migration_entry(folio_pfn(folio)); + if (pte_young(pte)) + entry = make_migration_entry_young(entry); + if (pte_dirty(pte)) + entry = make_migration_entry_dirty(entry); + migration = pte_swp_set_subpage_offset(swp_entry_to_pte(entry), mm_subpage_offset(slot)); + if (pte_soft_dirty(pte)) + migration = pte_swp_mksoft_dirty(migration); + if (pte_uffd_wp(pte)) + migration = pte_swp_mkuffd_wp(migration); + set_pte_at(mm, address, pvmw->pte, migration); + trace_set_migration_pte(address, pte_val(migration), 0); + mm_subpage_migrate_save_lazyfree(slot); + mm_subpage_remove_anon_rmap(slot, vma); + if (vma->vm_flags & VM_LOCKED) + mlock_drain_local(); + /* The driver holds backing until its batched invalidation completes. */ + mm_subpage_put(slot); + ret = true; +out: + mm_subpage_put(slot); + return ret; +} +#endif + static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct mm_struct *mm = vma->vm_mm; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool anon_exclusive, writable, ret = true; pte_t pteval; struct page *subpage; @@ -2424,6 +2763,11 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, enum ttu_flags flags = (enum ttu_flags)(long)arg; unsigned long pfn; unsigned long hsz = 0; + unsigned int nr_refs = 1; + + /* A base PTE owns one reference for each covered native page. */ + if (!folio_test_hugetlb(folio)) + nr_refs = mm_pte_native_pages(mm); /* * When racing against e.g. zap_pte_range() on another cpu, @@ -2432,7 +2776,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, * if page table locking is skipped: use TTU_SYNC to wait for that. */ if (flags & TTU_SYNC) - pvmw.flags = PVMW_SYNC; + pvmw.flags |= PVMW_SYNC; /* * For THP, we have to assume the worse case ie pmd for invalidation. @@ -2501,6 +2845,17 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, /* Unexpected PMD-mapped THP? */ VM_BUG_ON_FOLIO(!pvmw.pte, folio); + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE && folio_test_anon(folio)) { + if (!try_to_migrate_subpage(folio, vma, &pvmw, flags)) { + ret = false; + page_vma_mapped_walk_done(&pvmw); + break; + } + continue; + } +#endif /* * Handle PFN swap PTEs, such as device-exclusive ones, that * actually map pages. @@ -2586,7 +2941,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ pteval = ptep_get_and_clear(mm, address, pvmw.pte); - set_tlb_ubc_flush_pending(mm, pteval, address, address + PAGE_SIZE); + set_tlb_ubc_flush_pending(mm, pteval, address, address + mm_page_size(mm)); } else { pteval = ptep_clear_flush(vma, address, pvmw.pte); } @@ -2663,7 +3018,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, break; } } else if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, subpage)) { + folio_try_share_anon_rmap_pte(folio, subpage, vma)) { set_pte_at(mm, address, pvmw.pte, pteval); ret = false; page_vma_mapped_walk_done(&pvmw); @@ -2701,6 +3056,13 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, if (pte_swp_uffd_wp(pteval)) swp_pte = pte_swp_mkuffd_wp(swp_pte); } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) + swp_pte = pte_swp_set_subpage_offset(swp_pte, + pte_present(pteval) ? pte_phys_mm(mm, pteval) & ~PAGE_MASK : + pte_swp_subpage_offset(pteval)); +#endif + if (folio_test_hugetlb(folio)) set_huge_pte_at(mm, address, pvmw.pte, swp_pte, hsz); @@ -2714,13 +3076,15 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ } + if (!folio_test_anon(folio)) + mm_subpage_file_map_del(vma, pte_phys_mm(mm, pteval)); if (unlikely(folio_test_hugetlb(folio))) hugetlb_remove_rmap(folio); else folio_remove_rmap_pte(folio, subpage, vma); if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); - folio_put(folio); + folio_put_refs(folio, nr_refs); } mmu_notifier_invalidate_range_end(&range); @@ -2951,6 +3315,61 @@ static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio, return anon_vma; } +#ifdef CONFIG_MM_SUBPAGE +noinline bool rmap_walk_subpages(struct folio *folio, + struct rmap_walk_control *rwc, bool locked) +{ + struct mm_subpage_rmap maps[MM_SUBPAGES_PER_PAGE]; + int i, nr = mm_subpage_snapshot_rmap(folio, maps); + + if (nr < 0) + return false; + /* Existing locked anonymous walks operate on large folios. */ + VM_BUG_ON_FOLIO(locked, folio); + for (i = 0; i < nr; i++) { + struct mm_subpage_rmap *map = &maps[i]; + struct anon_vma_chain *avc; + bool stop = false; + + if (!anon_vma_trylock_read(map->root)) { + if (rwc->try_lock) { + rwc->contended = true; + break; + } + anon_vma_lock_read(map->root); + } + anon_vma_interval_tree_foreach(avc, &map->root->rb_root, + map->pos.index, map->pos.index) { + struct vm_area_struct *vma = avc->vma; + unsigned long address = vma_address_at_offset(vma, map->pos); + struct rmap_walk_range range = { + .address = address, + .subpage = true, + .subpage_offset = map->offset, + }; + + /* Native-index overlap need not include this exact leaf. */ + if (address == -EFAULT) + continue; + cond_resched(); + if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) + continue; + if (!rwc->rmap_one(folio, vma, range, rwc->arg) || + (rwc->done && rwc->done(folio))) { + stop = true; + break; + } + } + anon_vma_unlock_read(map->root); + if (stop) + break; + } + for (i = 0; i < nr; i++) + put_anon_vma(maps[i].root); + return true; +} +#endif + /* * rmap_walk_anon - do something to anonymous page using the object-based * rmap method @@ -2973,6 +3392,10 @@ static void rmap_walk_anon(struct folio *folio, * to an anon_vma with different root. */ VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); +#ifdef CONFIG_MM_SUBPAGE + if (!folio_test_large(folio) && rmap_walk_subpages(folio, rwc, locked)) + return; +#endif if (locked) { anon_vma = folio_anon_vma(folio); @@ -2998,7 +3421,8 @@ static void rmap_walk_anon(struct folio *folio, if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, address, rwc->arg)) + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = address }, rwc->arg)) break; if (rwc->done && rwc->done(folio)) break; @@ -3061,7 +3485,8 @@ static void __rmap_walk_file(struct folio *folio, struct address_space *mapping, if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, address, rwc->arg)) + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = address }, rwc->arg)) goto done; if (rwc->done && rwc->done(folio)) goto done; diff --git a/mm/shmem.c b/mm/shmem.c index 7b1ea9fb598f4..705765dddf1d2 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -30,6 +30,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -875,6 +878,266 @@ static void shmem_update_stats(struct folio *folio, int nr_pages) lruvec_stat_mod_folio(folio, NR_SHMEM, nr_pages); } +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +#define SHMEM_UFFD_ALL_UNITS GENMASK(PAGE_SIZE / SZ_4K - 1, 0) + +/* + * Resident masks live in page_ext, allowing partial truncate to update them + * without allocation. All cache insertion/replacement paths initialize or + * transfer them under the folio lock. The inode XArray persists masks only + * for swapped indices. Its publication/removal additionally holds i_pages, + * before taking uffd_missing's XArray lock. + */ +static void shmem_uffd_enable(struct inode *inode) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + + spin_lock(&info->lock); + info->flags |= SHMEM_F_UFFD_SUBPAGE; + spin_unlock(&info->lock); +} + +static unsigned long shmem_uffd_mask(unsigned int offset, unsigned long length) +{ + return GENMASK((offset + length - 1) / SZ_4K, offset / SZ_4K); +} + +unsigned long shmem_uffd_missing_mask(struct folio *folio, pgoff_t index) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); + VM_BUG_ON_FOLIO(!folio_mapping(folio) || !shmem_mapping(folio_mapping(folio)), folio); + return mm_subpage_shmem_missing(folio_page(folio, index - folio->index)); +} + +int shmem_uffd_set_missing(struct folio *folio, pgoff_t index, + unsigned long missing) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!folio_mapping(folio) || !shmem_mapping(folio_mapping(folio)) || + !folio_contains(folio, index) || + !folio_test_uptodate(folio) || (missing & ~SHMEM_UFFD_ALL_UNITS)) + return -EINVAL; + if (missing) + shmem_uffd_enable(folio->mapping->host); + mm_subpage_shmem_set_missing(folio_page(folio, index - folio->index), missing); + return 0; +} + +bool shmem_uffd_has_missing_native_page(struct folio *folio) +{ + struct address_space *mapping = folio_mapping(folio); + unsigned long i; + + if (!mapping || !shmem_mapping(mapping) || + !(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return false; + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + /* Native PMD exposure is allowed when every native leaf has some data. */ + for (i = 0; i < folio_nr_pages(folio); i++) + if (mm_subpage_shmem_missing(folio_page(folio, i)) == SHMEM_UFFD_ALL_UNITS) + return true; + return false; +} + +bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length) +{ + struct address_space *mapping = folio_mapping(folio); + unsigned long mask; + + if (!mapping || !shmem_mapping(mapping)) + return false; + mask = shmem_uffd_mask(offset, length); + return (shmem_uffd_missing_mask(folio, index) & mask) == mask; +} + +/* A coarse alias with any populated units preserves them and exposes zero holes. */ +static unsigned long shmem_uffd_reason(struct vm_fault *vmf, unsigned long missing) +{ + struct vm_area_struct *vma = vmf->vma; + struct vm_page_offset pos = vma_page_offset_at(vma, vmf->address); + unsigned long mask = shmem_uffd_mask(pos.offset, mm_page_size(vma->vm_mm)); + + if ((missing & mask) == mask) + return userfaultfd_missing(vma) ? VM_UFFD_MISSING : 0; + return userfaultfd_minor(vma) ? VM_UFFD_MINOR : 0; +} + +void shmem_uffd_mark_present(struct folio *folio, pgoff_t index, + unsigned long present) +{ + unsigned long missing = shmem_uffd_missing_mask(folio, index); + + if (missing & present) + mm_subpage_shmem_set_missing(folio_page(folio, index - folio->index), + missing & ~present); +} + +void shmem_uffd_mark_mapped(struct folio *folio, unsigned long offset, + unsigned long length) +{ + struct address_space *mapping = folio_mapping(folio); + + if (!mapping || !shmem_mapping(mapping) || + !(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return; + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(offset + length > folio_size(folio), folio); + while (length) { + unsigned int low = offset & ~PAGE_MASK; + unsigned long size = min(length, PAGE_SIZE - low); + unsigned long mask = GENMASK((low + size - 1) / SZ_4K, low / SZ_4K); + + shmem_uffd_mark_present(folio, folio->index + (offset >> PAGE_SHIFT), mask); + offset += size; + length -= size; + } +} + +void shmem_uffd_forget(struct address_space *mapping, pgoff_t start, + unsigned long nr) +{ + struct xarray *xa; + unsigned long index; + void *entry; + + if (!shmem_mapping(mapping)) + return; + lockdep_assert_held(&mapping->i_pages.xa_lock); + xa = &SHMEM_I(mapping->host)->uffd_missing; + if (xa_empty(xa)) + return; + xa_lock(xa); + xa_for_each_range(xa, index, entry, start, start + nr - 1) + __xa_erase(xa, index); + xa_unlock(xa); +} + +/* New cache identity or swap-in: initialize every native page before unlock. */ +static void shmem_uffd_cache_init(struct folio *folio, bool from_swap) +{ + struct address_space *mapping = folio->mapping; + struct xarray *xa = &SHMEM_I(mapping->host)->uffd_missing; + unsigned long i; + + lockdep_assert_held(&mapping->i_pages.xa_lock); + for (i = 0; i < folio_nr_pages(folio); i++) { + void *entry = from_swap ? xa_load(xa, folio->index + i) : NULL; + + mm_subpage_shmem_set_missing(folio_page(folio, i), + entry ? xa_to_value(entry) : 0); + } + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); +} + +/* + * Reserve persistence before committing swap. On failure the cache folio and + * its resident mask stay intact; reclaim may retry once memory is available. + */ +static int shmem_uffd_swap_prepare(struct folio *folio) +{ + struct address_space *mapping = folio->mapping; + struct xarray *xa = &SHMEM_I(mapping->host)->uffd_missing; + struct mem_cgroup *memcg, *saved; + unsigned long i; + int err = 0; + + if (!(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return 0; + /* Reclaim may run outside the backing folio's charge domain. */ + rcu_read_lock(); + do { + memcg = folio_memcg(folio) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + for (i = 0; i < folio_nr_pages(folio); i++) { + unsigned long mask = mm_subpage_shmem_missing(folio_page(folio, i)); + + if (!mask) + continue; + err = xa_err(xa_store(xa, folio->index + i, xa_mk_value(mask), + GFP_NOWAIT | __GFP_ACCOUNT | __GFP_NOWARN)); + if (err) + break; + } + set_active_memcg(saved); + mem_cgroup_put(memcg); + if (err) { + xa_lock_irq(&mapping->i_pages); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); + xa_unlock_irq(&mapping->i_pages); + } + return err; +} + +void shmem_uffd_copy_page(struct page *dst, struct page *src) +{ + mm_subpage_shmem_set_missing(dst, src ? mm_subpage_shmem_missing(src) : 0); +} + +void shmem_uffd_migrate(struct folio *dst, struct folio *src) +{ + unsigned long i; + + if (!shmem_mapping(src->mapping)) + return; + VM_BUG_ON_FOLIO(!folio_test_locked(src), src); + VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst); + for (i = 0; i < folio_nr_pages(src); i++) + shmem_uffd_copy_page(folio_page(dst, i), folio_page(src, i)); +} + +/* Only fully removed 4K units become holes; sub-unit edge bytes stay valid. */ +static void shmem_uffd_partial_hole(struct folio *folio, loff_t start, uoff_t end) +{ + loff_t first = max(start, folio_pos(folio)); + u64 last = min_t(u64, end, folio_next_pos(folio) - 1); + unsigned long offset, length; + bool pinned; + + if (first >= last || + (first == folio_pos(folio) && last == folio_next_pos(folio) - 1)) + return; + first = round_up(first, SZ_4K); + last = round_down(last + 1, SZ_4K); + if (first >= last) + return; + shmem_uffd_enable(folio->mapping->host); + /* Revoke covering coarse aliases before their holes can be refilled. */ + unmap_mapping_subpage_hole(folio->mapping, first, last); + /* Pair with fast GUP's pin-before-PTE-recheck barrier. */ + smp_mb(); + pinned = folio_maybe_dma_pinned(folio); + offset = first - folio_pos(folio); + length = last - first; + while (length) { + unsigned int low = offset & ~PAGE_MASK; + unsigned long nr = min(length, PAGE_SIZE - low); + struct page *page = folio_page(folio, offset >> PAGE_SHIFT); + unsigned long mask = shmem_uffd_mask(low, nr); + unsigned long missing = mm_subpage_shmem_missing(page); + + /* + * A pin survives PTE removal and can modify retained backing. + * Preserve native partial-hole semantics: zero it, but do not + * let a missing-page COPY overwrite a subsequent device write. + * Pins have folio granularity, so exposure is conservative when + * only a neighbouring range is pinned. Full-folio removal still + * detaches old pins from the file's new cache identity. + */ + mm_subpage_shmem_set_missing(page, pinned ? missing & ~mask : missing | mask); + offset += nr; + length -= nr; + } +} +#else +static void shmem_uffd_cache_init(struct folio *folio, bool from_swap) { } +static int shmem_uffd_swap_prepare(struct folio *folio) { return 0; } +static void shmem_uffd_partial_hole(struct folio *folio, loff_t start, uoff_t end) { } +#endif + /* * Somewhat like filemap_add_folio, but error if expected item has gone. */ @@ -922,6 +1185,7 @@ int shmem_add_to_page_cache(struct folio *folio, xas_store(&xas, folio); if (xas_error(&xas)) goto unlock; + shmem_uffd_cache_init(folio, expected != NULL); shmem_update_stats(folio, nr); mapping->nrpages += nr; unlock: @@ -976,8 +1240,10 @@ static long shmem_free_swap(struct address_space *mapping, base = round_down(xas.xa_index, nr_pages); if (base < index || base + nr_pages - 1 > end) nr_pages = 0; - else + else { + shmem_uffd_forget(mapping, base, nr_pages); xas_store(&xas, NULL); + } } xas_unlock_irq(&xas); @@ -1020,6 +1286,39 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping, return swapped << PAGE_SHIFT; } +/* Byte-clipped variant for mappings smaller than a native cache page. */ +unsigned long shmem_partial_swap_usage_bytes(struct address_space *mapping, + u64 start, u64 end) +{ + XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT); + void *entry; + unsigned long swapped = 0, last; + + if (start >= end) + return 0; + last = (end - 1) >> PAGE_SHIFT; + rcu_read_lock(); + xas_for_each(&xas, entry, last) { + if (xas_retry(&xas, entry)) + continue; + if (xa_is_value(entry)) { + unsigned int order = xas_get_order(&xas); + u64 first = ((u64)xas.xa_index & ~((1UL << order) - 1)) << PAGE_SHIFT; + u64 stop = first + ((u64)PAGE_SIZE << order); + + swapped += min(stop, end) - max(first, start); + } + if (xas.xa_index == last) + break; + if (need_resched()) { + xas_pause(&xas); + cond_resched_rcu(); + } + } + rcu_read_unlock(); + return swapped; +} + /* * Determine (in bytes) how many of the shmem object's pages mapped by the * given vma is swapped out. @@ -1044,6 +1343,12 @@ unsigned long shmem_swap_usage(struct vm_area_struct *vma) */ if (!swapped) return 0; + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + u64 start = ((u64)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); + + return shmem_partial_swap_usage_bytes(mapping, start, + start + vma->vm_end - vma->vm_start); + } if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) return swapped << PAGE_SHIFT; @@ -1101,6 +1406,12 @@ static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index) return folio; } +static bool shmem_truncate_partial_folio(struct folio *folio, loff_t start, uoff_t end) +{ + shmem_uffd_partial_hole(folio, start, end); + return truncate_inode_partial_folio(folio, start, end); +} + /* * Remove range of pages and swap entries from page cache, and free them. * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. @@ -1164,7 +1475,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, if (folio) { same_folio = lend < folio_next_pos(folio); folio_mark_dirty(folio); - if (!truncate_inode_partial_folio(folio, lstart, lend)) { + if (!shmem_truncate_partial_folio(folio, lstart, lend)) { start = folio_next_index(folio); if (same_folio) end = folio->index; @@ -1178,7 +1489,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT); if (folio) { folio_mark_dirty(folio); - if (!truncate_inode_partial_folio(folio, lstart, lend)) + if (!shmem_truncate_partial_folio(folio, lstart, lend)) end = folio->index; folio_unlock(folio); folio_put(folio); @@ -1247,7 +1558,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, if (!folio_test_large(folio)) { truncate_inode_folio(mapping, folio); - } else if (truncate_inode_partial_folio(folio, lstart, lend)) { + } else if (shmem_truncate_partial_folio(folio, lstart, lend)) { /* * If we split a page, reset the loop so * that we pick up the new sub pages. @@ -1355,6 +1666,7 @@ static int shmem_setattr(struct mnt_idmap *idmap, } if (newsize <= oldsize) { loff_t holebegin = round_up(newsize, PAGE_SIZE); + unmap_mapping_subpage_tail(inode->i_mapping, newsize); if (oldsize > holebegin) unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); @@ -1362,6 +1674,7 @@ static int shmem_setattr(struct mnt_idmap *idmap, shmem_truncate_range(inode, newsize, (loff_t)-1); /* unmap again to remove racily COWed private pages */ + unmap_mapping_subpage_tail(inode->i_mapping, newsize); if (oldsize > holebegin) unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); @@ -1677,6 +1990,8 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, folio_mark_uptodate(folio); } + if (shmem_uffd_swap_prepare(folio)) + goto redirty; if (!folio_alloc_swap(folio)) { bool first_swapped = shmem_recalc_inode(inode, 0, nr_pages); int error; @@ -1730,6 +2045,10 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, swap_cache_del_folio(folio); goto redirty; } + /* Allocation failed: no swap entry owns the reserved inode metadata. */ + xa_lock_irq(&mapping->i_pages); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); + xa_unlock_irq(&mapping->i_pages); if (nr_pages > 1) goto try_split; redirty: @@ -1934,6 +2253,23 @@ static struct folio *shmem_alloc_folio(gfp_t gfp, int order, return folio; } +/* Allocate unexposed promotion backing with the inode's zone/NUMA policy. */ +struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, struct inode *inode, + pgoff_t index, unsigned int order) +{ + gfp_t gfp = limit_gfp_mask(GFP_TRANSHUGE, mapping_gfp_mask(inode->i_mapping)); + struct folio *folio; + + folio = shmem_alloc_folio(gfp, order, SHMEM_I(inode), index); + if (!folio) + return ERR_PTR(-ENOMEM); + if (mem_cgroup_charge(folio, mm, gfp)) { + folio_put(folio); + return ERR_PTR(-ENOMEM); + } + return folio; +} + static struct folio *shmem_alloc_and_add_folio(struct vm_fault *vmf, gfp_t gfp, struct inode *inode, pgoff_t index, struct mm_struct *fault_mm, unsigned long orders) @@ -2484,13 +2820,36 @@ static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index, fault_mm = vma ? vma->vm_mm : NULL; folio = filemap_get_entry(inode->i_mapping, index); - if (folio && vma && userfaultfd_minor(vma)) { + if (folio && vma && userfaultfd_minor(vma) && + !(READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { if (!xa_is_value(folio)) folio_put(folio); *fault_type = handle_userfault(vmf, VM_UFFD_MINOR); return 0; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + if (xa_is_value(folio) && vma && userfaultfd_armed(vma) && + (READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { + unsigned long reason; + void *partial; + + /* Cache identity and its persisted mask must describe the same entry. */ + xa_lock_irq(&inode->i_mapping->i_pages); + if (xa_load(&inode->i_mapping->i_pages, index) != folio) { + xa_unlock_irq(&inode->i_mapping->i_pages); + goto repeat; + } + partial = xa_load(&SHMEM_I(inode)->uffd_missing, index); + reason = shmem_uffd_reason(vmf, partial ? xa_to_value(partial) : 0); + xa_unlock_irq(&inode->i_mapping->i_pages); + if (reason) { + *fault_type = handle_userfault(vmf, reason); + return 0; + } + } +#endif + if (xa_is_value(folio)) { error = shmem_swapin_folio(inode, index, &folio, sgp, gfp, vma, fault_type); @@ -2510,6 +2869,20 @@ static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index, folio_put(folio); goto repeat; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + if (vma && userfaultfd_armed(vma) && + (READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { + unsigned long reason = shmem_uffd_reason(vmf, + shmem_uffd_missing_mask(folio, index)); + + if (reason) { + folio_unlock(folio); + folio_put(folio); + *fault_type = handle_userfault(vmf, reason); + return 0; + } + } +#endif if (sgp == SGP_WRITE) folio_mark_accessed(folio); if (folio_test_uptodate(folio)) @@ -2748,15 +3121,33 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) struct folio *folio = NULL; vm_fault_t ret = 0; int err; +#ifdef CONFIG_MM_SUBPAGE + bool promoted = false; + unsigned long user_size = mm_page_size(vmf->vma->vm_mm); + + /* Missing coarse PTEs require a blocking caller to prepare backing. */ + if (user_size > PAGE_SIZE && (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) + return VM_FAULT_RETRY; +#endif +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) + return VM_FAULT_SIGBUS; +#endif + +#ifdef CONFIG_MM_SUBPAGE +retry: +#endif /* * Trinity finds that probing a hole which tmpfs is punching can * prevent the hole-punch from ever completing: noted in i_private. */ if (unlikely(inode->i_private)) { - ret = shmem_falloc_wait(vmf, inode); - if (ret) - return ret; + vm_fault_t wait = shmem_falloc_wait(vmf, inode); + + if (wait) + return ret | wait; } WARN_ON_ONCE(vmf->page != NULL); @@ -2765,6 +3156,45 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) if (err) return vmf_error(err); if (folio) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) { + folio_unlock(folio); + folio_put(folio); + return VM_FAULT_SIGBUS; + } + if (user_size > PAGE_SIZE) { + struct vm_page_offset pos = vma_page_offset_at(vmf->vma, vmf->address); + unsigned long nr = user_size >> PAGE_SHIFT; + + if (pos.offset || !IS_ALIGNED(pos.index, nr)) { + folio_unlock(folio); + folio_put(folio); + return VM_FAULT_SIGBUS; + } + if (folio_size(folio) < user_size) { + folio_unlock(folio); + folio_put(folio); + folio = NULL; + /* Bound work if splitting races a successful promotion. */ + if (promoted) { + cond_resched(); + return ret | VM_FAULT_NOPAGE; + } + err = collapse_file_user_page(vmf->vma->vm_mm, + vmf->vma->vm_file, pos.index); + if (err == -EAGAIN) { + cond_resched(); + return ret | VM_FAULT_NOPAGE; + } + if (err) + return vmf_error(err); + promoted = true; + goto retry; + } + } +#endif + vmf->page = folio_file_page(folio, vmf->pgoff); ret |= VM_FAULT_LOCKED; } @@ -2899,7 +3329,7 @@ static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, * by page order, as in shmem_get_pgoff_policy() and get_vma_policy()). */ *ilx = inode->i_ino; - index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + index = vma_page_offset_at(vma, addr).index; return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); } @@ -3086,6 +3516,9 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap, memset(info, 0, (char *)inode - (char *)info); INIT_LIST_HEAD_RCU(&info->xattrs); spin_lock_init(&info->lock); +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + xa_init_flags(&info->uffd_missing, XA_FLAGS_ACCOUNT); +#endif atomic_set(&info->stop_eviction, 0); info->seals = F_SEAL_SEAL; info->flags = vma_flags_test(&flags, VMA_NORESERVE_BIT) @@ -3245,6 +3678,57 @@ static void shmem_mfill_filemap_remove(struct folio *folio, folio_unlock(folio); } +#ifdef CONFIG_MM_SUBPAGE +struct folio *shmem_uffd_prepare_folio(struct vm_area_struct *vma, + unsigned long address, bool *new) +{ + struct inode *inode = file_inode(vma->vm_file); + pgoff_t index = vma_page_offset_at(vma, address).index; + struct folio *folio; + int err; + + *new = false; + /* Must precede cache insertion: minor faults must wait for initialization. */ + shmem_uffd_enable(inode); +retry: + err = shmem_get_folio(inode, index, 0, &folio, SGP_NOALLOC); + if (!err) + return folio; + if (err != -ENOENT) + return ERR_PTR(err); + folio = shmem_mfill_folio_alloc(vma, address); + if (!folio) + return ERR_PTR(-ENOMEM); + clear_highpage(&folio->page); + flush_dcache_folio(folio); + __folio_mark_uptodate(folio); + err = shmem_mfill_filemap_add(folio, vma, address); + if (err) { + folio_put(folio); + if (err == -EEXIST) + goto retry; + return ERR_PTR(err); + } + err = shmem_uffd_set_missing(folio, index, SHMEM_UFFD_ALL_UNITS); + if (err) { + shmem_mfill_filemap_remove(folio, vma); + folio_put(folio); + return ERR_PTR(err); + } + *new = true; + return folio; +} + +void shmem_uffd_abort_folio(struct folio *folio, struct vm_area_struct *vma, bool new) +{ + if (new) + shmem_mfill_filemap_remove(folio, vma); + else + folio_unlock(folio); + folio_put(folio); +} +#endif + static struct folio *shmem_get_folio_noalloc(struct inode *inode, pgoff_t pgoff) { struct folio *folio; @@ -3331,6 +3815,7 @@ shmem_write_end(const struct kiocb *iocb, struct address_space *mapping, } folio_mark_uptodate(folio); } + shmem_uffd_mark_mapped(folio, offset_in_folio(folio, pos), copied); folio_mark_dirty(folio); folio_unlock(folio); folio_put(folio); @@ -5143,6 +5628,10 @@ static struct inode *shmem_alloc_inode(struct super_block *sb) info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL); if (!info) return NULL; +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + /* Also cover inode-init failure before __shmem_get_inode() runs. */ + xa_init_flags(&info->uffd_missing, XA_FLAGS_ACCOUNT); +#endif return &info->vfs_inode; } @@ -5155,6 +5644,9 @@ static void shmem_free_in_core_inode(struct inode *inode) static void shmem_destroy_inode(struct inode *inode) { +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + xa_destroy(&SHMEM_I(inode)->uffd_missing); +#endif if (S_ISREG(inode->i_mode)) mpol_free_shared_policy(&SHMEM_I(inode)->policy); if (S_ISDIR(inode->i_mode)) @@ -5972,6 +6464,8 @@ struct folio *shmem_read_folio_gfp(struct address_space *mapping, if (error) return ERR_PTR(error); + /* A kernel caller receives and may modify the complete native folio. */ + shmem_uffd_mark_mapped(folio, 0, folio_size(folio)); folio_unlock(folio); return folio; #else diff --git a/mm/swap.h b/mm/swap.h index a77016f2423b9..5d064cab36f4c 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -10,9 +10,12 @@ extern int page_cluster; #ifdef CONFIG_THP_SWAP #define SWAPFILE_CLUSTER HPAGE_PMD_NR -#define swap_entry_order(order) (order) #else #define SWAPFILE_CLUSTER 256 +#endif +#if defined(CONFIG_THP_SWAP) || defined(CONFIG_MM_SUBPAGE) +#define swap_entry_order(order) (order) +#else #define swap_entry_order(order) 0 #endif @@ -38,6 +41,9 @@ struct swap_cluster_info { u8 order; atomic_long_t __rcu *table; /* Swap table entries, see mm/swap_table.h */ unsigned int *extend_table; /* For large swap count, protected by ci->lock */ +#ifdef CONFIG_MM_SUBPAGE + unsigned int *subpage_counts; /* Per-quarter PTE refs, protected by ci->lock */ +#endif struct list_head list; }; @@ -207,6 +213,8 @@ extern int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp); */ int folio_alloc_swap(struct folio *folio); int folio_dup_swap(struct folio *folio, struct page *subpage); +int folio_dup_swap_range(struct folio *folio, struct page *page, unsigned int nr); +void folio_put_swap_range(struct folio *folio, struct page *page, unsigned int nr); void folio_put_swap(struct folio *folio, struct page *subpage); /* For internal use */ @@ -216,6 +224,7 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si, /* linux/mm/page_io.c */ int sio_pool_init(void); +int swap_extent_io_init(void); struct swap_iocb; void swap_read_folio(struct folio *folio, struct swap_iocb **plug); void __swap_read_unplug(struct swap_iocb *plug); @@ -399,6 +408,17 @@ static inline void folio_put_swap(struct folio *folio, struct page *page) { } +static inline int folio_dup_swap_range(struct folio *folio, struct page *page, + unsigned int nr) +{ + return -EINVAL; +} + +static inline void folio_put_swap_range(struct folio *folio, struct page *page, + unsigned int nr) +{ +} + static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug) { } diff --git a/mm/swap_state.c b/mm/swap_state.c index 1415a5c54a43b..db1285bc1880d 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -427,6 +427,10 @@ void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, if (unlikely(folio_test_large(folio))) return; + /* Small-granule PTEs currently use native swap-cluster readahead. */ + if (vma && mm_page_size(vma->vm_mm) != PAGE_SIZE) + vma = NULL; + readahead = folio_test_clear_readahead(folio); if (vma && vma_ra) { unsigned long ra_val; @@ -896,7 +900,7 @@ struct folio *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, struct folio *folio; mpol = get_vma_policy(vmf->vma, vmf->address, 0, &ilx); - folio = swap_use_vma_readahead() ? + folio = swap_use_vma_readahead() && mm_page_size(vmf->vma->vm_mm) == PAGE_SIZE ? swap_vma_readahead(entry, gfp_mask, mpol, ilx, vmf) : swap_cluster_readahead(entry, gfp_mask, mpol, ilx); mpol_cond_put(mpol); diff --git a/mm/swapfile.c b/mm/swapfile.c index 74a1e324449dc..e5faba70227c0 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -329,17 +330,22 @@ offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset) BUG(); } +/* The extent tree is immutable while the swap entry is owned. */ +sector_t swap_extent_sector(swp_entry_t entry, unsigned long *nr_pages) +{ + struct swap_info_struct *sis = __swap_entry_to_info(entry); + pgoff_t offset = swp_offset(entry); + struct swap_extent *se = offset_to_swap_extent(sis, offset); + + *nr_pages = min(*nr_pages, se->start_page + se->nr_pages - offset); + return (se->start_block + offset - se->start_page) << (PAGE_SHIFT - 9); +} + sector_t swap_folio_sector(struct folio *folio) { - struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); - struct swap_extent *se; - sector_t sector; - pgoff_t offset; + unsigned long nr = folio_nr_pages(folio); - offset = swp_offset(folio->swap); - se = offset_to_swap_extent(sis, offset); - sector = se->start_block + (offset - se->start_page); - return sector << (PAGE_SHIFT - 9); + return swap_extent_sector(folio->swap, &nr); } /* @@ -479,6 +485,16 @@ static void swap_cluster_free_table(struct swap_cluster_info *ci) lockdep_assert_held(&ci->lock); table = (void *)rcu_dereference_protected(ci->table, true); rcu_assign_pointer(ci->table, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (ci->subpage_counts) { + unsigned int i; + + for (i = 0; i < SWAPFILE_CLUSTER * (PAGE_SIZE / MM_SUBPAGE_SIZE); i++) + VM_WARN_ON_ONCE(ci->subpage_counts[i]); + kfree(ci->subpage_counts); + ci->subpage_counts = NULL; + } +#endif swap_table_free(table); } @@ -1067,6 +1083,19 @@ static void swap_reclaim_work(struct work_struct *work) swap_reclaim_full_clusters(si, true); } +static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio *folio) +{ + unsigned int flags; + + if (!folio || !folio_order(folio)) + return true; + flags = READ_ONCE(si->flags); + if (flags & SWP_BLKDEV) + return true; + /* Filesystem swap_rw implementations retain their existing contract. */ + return si->bdev && !(flags & SWP_FS_OPS) && folio_anon_min_order(folio); +} + /* * Try to allocate swap entries with specified order and try set a new * cluster for current CPU too. @@ -1078,11 +1107,7 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, unsigned int order = likely(folio) ? folio_order(folio) : 0; unsigned int offset = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID; - /* - * Swapfile is not block device so unable - * to allocate large entries. - */ - if (order && !(si->flags & SWP_BLKDEV)) + if (!swap_folio_alloc_allowed(si, folio)) return 0; if (!(si->flags & SWP_SOLIDSTATE)) { @@ -1361,6 +1386,10 @@ static bool swap_alloc_fast(struct folio *folio) offset = this_cpu_read(percpu_swap_cluster.offset[order]); if (!si || !offset || !get_swap_device_info(si)) return false; + if (!swap_folio_alloc_allowed(si, folio)) { + put_swap_device(si); + return false; + } ci = swap_cluster_lock(si, offset); if (cluster_is_usable(ci, order)) { @@ -1543,7 +1572,7 @@ static void __swap_cluster_put_entry(struct swap_cluster_info *ci, */ static void swap_put_entries_cluster(struct swap_info_struct *si, pgoff_t offset, int nr, - bool reclaim_cache) + bool reclaim_cache, int subpage) { struct swap_cluster_info *ci; unsigned int ci_off, ci_end; @@ -1556,6 +1585,17 @@ static void swap_put_entries_cluster(struct swap_info_struct *si, ci = swap_cluster_lock(si, offset); ci_off = offset % SWAPFILE_CLUSTER; ci_end = ci_off + nr; +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) { + unsigned int index = ci_off * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage; + + if (WARN_ON_ONCE(nr != 1 || !ci->subpage_counts || !ci->subpage_counts[index])) { + swap_cluster_unlock(ci); + return; + } + ci->subpage_counts[index]--; + } +#endif do { swp_tb = __swap_table_get(ci, ci_off); if (swp_tb_get_count(swp_tb) == 1) { @@ -1649,7 +1689,7 @@ static int __swap_cluster_dup_entry(struct swap_cluster_info *ci, * and failed to allocate an extended table, -EINVAL if any entry is bad entry. */ static int swap_dup_entries_cluster(struct swap_info_struct *si, - pgoff_t offset, int nr) + pgoff_t offset, int nr, int subpage) { int err; struct swap_cluster_info *ci; @@ -1659,6 +1699,25 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, ci_end = ci_start + nr; ci_off = ci_start; ci = swap_cluster_lock(si, offset); +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) { + unsigned int index = ci_start * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage; + + VM_BUG_ON(nr != 1); + if (!ci->subpage_counts) + ci->subpage_counts = kcalloc(SWAPFILE_CLUSTER, + (PAGE_SIZE / MM_SUBPAGE_SIZE) * sizeof(*ci->subpage_counts), + GFP_ATOMIC | __GFP_NOWARN); + if (!ci->subpage_counts) { + err = -ENOMEM; + goto failed; + } + if (ci->subpage_counts[index] == UINT_MAX) { + err = -EOVERFLOW; + goto failed; + } + } +#endif restart: do { err = __swap_cluster_dup_entry(ci, ci_off); @@ -1673,6 +1732,10 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, goto failed; } } while (++ci_off < ci_end); +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) + ci->subpage_counts[ci_start * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage]++; +#endif swap_cluster_unlock(ci); return 0; failed: @@ -1683,6 +1746,132 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, return err; } +#ifdef CONFIG_MM_SUBPAGE +static bool swap_subpage_offset_valid(unsigned int offset) +{ + return offset < PAGE_SIZE && IS_ALIGNED(offset, MM_SUBPAGE_SIZE); +} + +int swap_subpage_dup(swp_entry_t entry, unsigned int offset) +{ + struct swap_info_struct *si = swap_entry_to_info(entry); + + if (WARN_ON_ONCE(!si || !swap_subpage_offset_valid(offset))) + return -EINVAL; + /* A locked swap-cache folio also permits the initial zero-to-one ref. */ + return swap_dup_entries_cluster(si, swp_offset(entry), 1, offset >> MM_SUBPAGE_SHIFT); +} + +void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache) +{ + struct swap_info_struct *si; + + if (WARN_ON_ONCE(!swap_subpage_offset_valid(offset))) + return; + si = get_swap_device(entry); + if (WARN_ON_ONCE(!si)) + return; + swap_put_entries_cluster(si, swp_offset(entry), 1, reclaim_cache, + offset >> MM_SUBPAGE_SHIFT); + put_swap_device(si); +} + +unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset) +{ + struct swap_cluster_info *ci; + struct swap_info_struct *si; + unsigned int count = 0; + + if (!swap_subpage_offset_valid(offset)) + return 0; + si = get_swap_device(entry); + if (!si) + return 0; + ci = swap_cluster_lock(si, swp_offset(entry)); + if (ci->subpage_counts) + count = ci->subpage_counts[swp_cluster_offset(entry) * + (PAGE_SIZE / MM_SUBPAGE_SIZE) + (offset >> MM_SUBPAGE_SHIFT)]; + swap_cluster_unlock(ci); + put_swap_device(si); + return count; +} +#endif + +/* + * Optional THPs can split if their order-specific cluster lists have no room. + * A mandatory base user page cannot: find a free aligned group without keeping + * the per-CPU or device allocation lock held over a device-wide scan. + */ +static void swap_alloc_required_from_device(struct swap_info_struct *si, + struct folio *folio) +{ + unsigned int order = folio_order(folio), nr = folio_nr_pages(folio); + unsigned long start; + + if (!swap_folio_alloc_allowed(si, folio)) + return; + for (start = 0; start < si->max; start += SWAPFILE_CLUSTER) { + struct swap_cluster_info *ci = __swap_offset_to_cluster(si, start); + unsigned long offset, end = min(start + SWAPFILE_CLUSTER, si->max); + bool found = false; + + if (!(READ_ONCE(si->flags) & SWP_WRITEOK)) + break; + if (!spin_trylock(&ci->lock)) + goto next; + /* Order zero's usability test permits occupied clusters of any order. */ + if (!cluster_is_usable(ci, 0) || ci->count + nr > SWAPFILE_CLUSTER) + goto unlock; + for (offset = start; offset + nr <= end; offset += nr) { + bool need_reclaim = false; + + /* Only genuinely free slots qualify; retain every existing owner. */ + if (!cluster_scan_range(si, ci, offset, nr, &need_reclaim) || need_reclaim) + continue; + if (!__swap_cluster_alloc_entries(si, ci, folio, offset - start)) + break; + ci->order = order; + if (ci->count < SWAPFILE_CLUSTER && ci->flags == CLUSTER_FLAG_FRAG) { + /* Same flag, different order: relocate the list explicitly. */ + spin_lock(&si->lock); + list_move_tail(&ci->list, &si->frag_clusters[order]); + spin_unlock(&si->lock); + } else { + relocate_cluster(si, ci); + } + found = true; + break; + } +unlock: + swap_cluster_unlock(ci); + if (found) + return; +next: + cond_resched(); + } +} + +static void swap_alloc_required(struct folio *folio) +{ + struct swap_info_struct *devices[MAX_SWAPFILES], *si; + unsigned int count = 0, i; + + if (get_nr_swap_pages() < folio_nr_pages(folio)) + return; + /* Pin a bounded snapshot in the allocator's existing priority order. */ + spin_lock(&swap_avail_lock); + plist_for_each_entry(si, &swap_avail_head, avail_list) { + if (get_swap_device_info(si)) + devices[count++] = si; + } + spin_unlock(&swap_avail_lock); + for (i = 0; i < count; i++) { + if (!folio_test_swapcache(folio)) + swap_alloc_required_from_device(devices[i], folio); + put_swap_device(devices[i]); + } +} + /** * folio_alloc_swap - allocate swap space for a folio * @folio: folio we want to move to swap @@ -1697,16 +1886,17 @@ int folio_alloc_swap(struct folio *folio) { unsigned int order = folio_order(folio); unsigned int size = 1 << order; + bool mandatory = order && folio_anon_min_order(folio); VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); VM_BUG_ON_FOLIO(!folio_test_uptodate(folio), folio); if (order) { /* - * Reject large allocation when THP_SWAP is disabled, - * the caller should split the folio and try again. + * Optional THP backing may split when THP_SWAP is disabled. + * A larger base user page must retain contiguous swap ownership. */ - if (!IS_ENABLED(CONFIG_THP_SWAP)) + if (!IS_ENABLED(CONFIG_THP_SWAP) && !mandatory) return -EAGAIN; /* @@ -1725,6 +1915,9 @@ int folio_alloc_swap(struct folio *folio) swap_alloc_slow(folio); local_unlock(&percpu_swap_cluster.lock); + if (mandatory && unlikely(!folio_test_swapcache(folio))) + swap_alloc_required(folio); + if (!order && unlikely(!folio_test_swapcache(folio))) { if (swap_sync_discard()) goto again; @@ -1757,6 +1950,24 @@ int folio_alloc_swap(struct folio *folio) * swap_put_entries_direct on its swap entry before this helper returns, or * the swap count may underflow. */ +int folio_dup_swap_range(struct folio *folio, struct page *page, unsigned int nr) +{ + swp_entry_t entry = page_swap_entry(page); + + VM_BUG_ON_FOLIO(!folio_test_locked(folio) || !folio_test_swapcache(folio), folio); + VM_BUG_ON_FOLIO(!nr || folio_page_idx(folio, page) + nr > folio_nr_pages(folio), folio); + return swap_dup_entries_cluster(swap_entry_to_info(entry), swp_offset(entry), nr, -1); +} + +void folio_put_swap_range(struct folio *folio, struct page *page, unsigned int nr) +{ + swp_entry_t entry = page_swap_entry(page); + + VM_BUG_ON_FOLIO(!folio_test_locked(folio) || !folio_test_swapcache(folio), folio); + VM_BUG_ON_FOLIO(!nr || folio_page_idx(folio, page) + nr > folio_nr_pages(folio), folio); + swap_put_entries_cluster(__swap_entry_to_info(entry), swp_offset(entry), nr, false, -1); +} + int folio_dup_swap(struct folio *folio, struct page *subpage) { swp_entry_t entry = folio->swap; @@ -1771,7 +1982,7 @@ int folio_dup_swap(struct folio *folio, struct page *subpage) } return swap_dup_entries_cluster(swap_entry_to_info(entry), - swp_offset(entry), nr_pages); + swp_offset(entry), nr_pages, -1); } /** @@ -1797,7 +2008,7 @@ void folio_put_swap(struct folio *folio, struct page *subpage) nr_pages = 1; } - swap_put_entries_cluster(si, swp_offset(entry), nr_pages, false); + swap_put_entries_cluster(si, swp_offset(entry), nr_pages, false, -1); } /* @@ -2070,7 +2281,7 @@ void swap_put_entries_direct(swp_entry_t entry, int nr) offset = start_offset; do { cluster_end = min(round_up(offset + 1, SWAPFILE_CLUSTER), end_offset); - swap_put_entries_cluster(si, offset, cluster_end - offset, true); + swap_put_entries_cluster(si, offset, cluster_end - offset, true, -1); offset = cluster_end; } while (offset < end_offset); out: @@ -2239,14 +2450,80 @@ static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte) return pte_same(pte_swp_clear_flags(pte), swp_pte); } +#ifdef CONFIG_MM_SUBPAGE +/* Caller holds the swap-cache folio lock and the VMA's mmap read lock. */ +static int unuse_subpage(struct vm_area_struct *vma, pmd_t *pmd, + unsigned long addr, swp_entry_t entry, struct folio *folio, + unsigned int offset) +{ + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slot = NULL; + spinlock_t *ptl; + pte_t *ptep, old, pte; + bool poisoned = PageHWPoison(&folio->page) || !folio_test_uptodate(folio); + int ret = 0; + + if (folio_order(folio) || folio_test_ksm(folio)) + return -EOPNOTSUPP; + if (!poisoned) { + arch_swap_restore(folio_swap(entry, folio), folio); + /* Allocate metadata before taking the PTE spinlock. */ + slot = mm_subpage_restore_granule(folio, offset, mm_page_shift(mm), GFP_KERNEL); + if (IS_ERR(slot)) + return PTR_ERR(slot) == -EAGAIN ? 0 : PTR_ERR(slot); + } + ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!ptep) + goto out; + old = ptep_get(ptep); + if (softleaf_from_pte(old).val != entry.val) + goto unlock; + /* A moved swap PTE retains its backing offset, not its VMA offset. */ + if (pte_swp_subpage_offset(old) != offset) + goto unlock; + if (poisoned) { + pte = swp_entry_to_pte(PageHWPoison(&folio->page) ? + make_hwpoison_entry(&folio->page) : make_poisoned_swp_entry()); + } else { + ret = mm_subpage_add_anon_rmap(slot, vma, addr); + if (ret) + goto unlock; + /* unuse_pte_range() waited for writeback and holds the folio lock. */ + VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio); + if (pte_swp_exclusive(old)) + mm_subpage_set_exclusive(slot); + else if (swap_subpage_count(entry, offset) == 1) + mm_subpage_try_reuse_swap(slot); + pte = pte_mkold(phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot)); + if (pte_swp_soft_dirty(old)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(old)) + pte = pte_mkuffd_wp(pte); + inc_mm_counter(mm, MM_ANONPAGES); + slot = NULL; + ret = 1; + } + dec_mm_counter(mm, MM_SWAPENTS); + set_pte_at(mm, addr, ptep, pte); + swap_subpage_put(entry, offset, false); +unlock: + pte_unmap_unlock(ptep, ptl); +out: + if (slot) + mm_subpage_put(slot); + return ret; +} +#endif + /* * No need to decide whether this PTE shares the swap entry with others, * just let do_wp_page work it out if a write is requested later - to * force COW, vm_page_prot omits write permission from any private vma. */ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, - unsigned long addr, swp_entry_t entry, struct folio *folio) + unsigned long addr, pte_t orig_pte, struct folio *folio) { + swp_entry_t entry = softleaf_from_pte(orig_pte); struct page *page; struct folio *swapcache; spinlock_t *ptl; @@ -2261,6 +2538,12 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, if (!folio_matches_swap_entry(folio, entry)) return 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return unuse_subpage(vma, pmd, addr, entry, folio, + pte_swp_subpage_offset(orig_pte)); +#endif + swapcache = folio; folio = ksm_might_need_to_copy(folio, vma, addr); if (unlikely(!folio)) @@ -2366,7 +2649,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, pte_t ptent; if (!pte++) { - pte = pte_offset_map(pmd, addr); + pte = pte_offset_map_mm(vma->vm_mm, pmd, addr); if (!pte) break; } @@ -2382,6 +2665,22 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, pte_unmap(pte); pte = NULL; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) { + struct vm_fault vmf = { + .vma = vma, .address = addr, .real_address = addr, + .pmd = pmd, .orig_pte = ptent, + }; + vm_fault_t fault = do_swap_coarse_page(&vmf); + + if (fault & VM_FAULT_OOM) + return -ENOMEM; + if (fault & VM_FAULT_ERROR) + return -EIO; + continue; + } +#endif + folio = swap_cache_get_folio(entry); if (!folio) { struct vm_fault vmf = { @@ -2404,7 +2703,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, folio_lock(folio); folio_wait_writeback(folio); - ret = unuse_pte(vma, pmd, addr, entry, folio); + ret = unuse_pte(vma, pmd, addr, ptent, folio); if (ret < 0) { folio_unlock(folio); folio_put(folio); @@ -2414,7 +2713,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, folio_free_swap(folio); folio_unlock(folio); folio_put(folio); - } while (addr += PAGE_SIZE, addr != end); + } while (addr += mm_page_size(vma->vm_mm), addr != end); if (pte) pte_unmap(pte); @@ -2429,10 +2728,10 @@ static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, unsigned long next; int ret; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(vma->vm_mm, pud, addr); do { cond_resched(); - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(vma->vm_mm, addr, end); ret = unuse_pte_range(vma, pmd, addr, next, type); if (ret) return ret; @@ -2448,9 +2747,9 @@ static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, unsigned long next; int ret; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(vma->vm_mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(vma->vm_mm, addr, end); if (pud_none_or_clear_bad(pud)) continue; ret = unuse_pmd_range(vma, pud, addr, next, type); @@ -2468,10 +2767,10 @@ static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, unsigned long next; int ret; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(vma->vm_mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(vma->vm_mm, addr, end); + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4d)) continue; ret = unuse_pud_range(vma, p4d, addr, next, type); if (ret) @@ -2491,7 +2790,7 @@ static int unuse_vma(struct vm_area_struct *vma, unsigned int type) pgd = pgd_offset(vma->vm_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(vma->vm_mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; ret = unuse_p4d_range(vma, pgd, addr, next, type); @@ -2798,10 +3097,15 @@ static int setup_swap_extents(struct swap_info_struct *sis, destroy_swap_extents(sis, swap_file); return -ENOMEM; } - return ret; + } else { + ret = generic_swapfile_activate(sis, swap_file, span); } - - return generic_swapfile_activate(sis, swap_file, span); + if (ret >= 0 && sis->bdev && + !(sis->flags & SWP_FS_OPS) && swap_extent_io_init()) { + destroy_swap_extents(sis, swap_file); + return -ENOMEM; + } + return ret; } static void _enable_swap_info(struct swap_info_struct *si) @@ -3708,7 +4012,7 @@ void si_swapinfo(struct sysinfo *val) * Also the swap entry must have a count >= 1. Otherwise folio_dup_swap should * be used. */ -int swap_dup_entry_direct(swp_entry_t entry) +int swap_dup_entries_direct(swp_entry_t entry, unsigned int nr) { struct swap_info_struct *si; @@ -3725,7 +4029,14 @@ int swap_dup_entry_direct(swp_entry_t entry) */ VM_WARN_ON_ONCE(!swap_entry_swapped(si, entry)); - return swap_dup_entries_cluster(si, swp_offset(entry), 1); + if (WARN_ON_ONCE(!nr || swp_cluster_offset(entry) + nr > SWAPFILE_CLUSTER)) + return -EINVAL; + return swap_dup_entries_cluster(si, swp_offset(entry), nr, -1); +} + +int swap_dup_entry_direct(swp_entry_t entry) +{ + return swap_dup_entries_direct(entry, 1); } #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) diff --git a/mm/truncate.c b/mm/truncate.c index 12cc89f89afcf..ca1830b647a97 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -794,8 +794,10 @@ void truncate_pagecache(struct inode *inode, loff_t newsize) * truncate_inode_pages finishes, hence the second * unmap_mapping_range call must be made for correctness. */ + unmap_mapping_subpage_tail(mapping, newsize); unmap_mapping_range(mapping, holebegin, 0, 1); truncate_inode_pages(mapping, newsize); + unmap_mapping_subpage_tail(mapping, newsize); unmap_mapping_range(mapping, holebegin, 0, 1); } EXPORT_SYMBOL(truncate_pagecache); diff --git a/mm/user-subpage-internal.h b/mm/user-subpage-internal.h new file mode 100644 index 0000000000000..bddb5eeb2ca7b --- /dev/null +++ b/mm/user-subpage-internal.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _MM_USER_SUBPAGE_INTERNAL_H +#define _MM_USER_SUBPAGE_INTERNAL_H + +#include +#include +#include +#include + +struct mm_subpage_owner; +struct rmap_walk_control; +struct anon_vma; + +struct mm_subpage { + struct mm_subpage_owner *owner; + refcount_t refs; + atomic_t mapcount; + unsigned int index; + u8 shift; + unsigned int pins; + bool exclusive; + bool lazyfree; + bool rmap_bound; +}; + +/* Logical identity is independent of the physical slot after relocation. */ +struct mm_subpage_rmap { + struct anon_vma *root; + struct vm_page_offset pos; + unsigned int offset; +}; + +struct subpage_retired_root; +struct mm_subpage_move { + struct mm_subpage *slot; + struct mm_subpage_rmap map; + struct subpage_retired_root *retired; + bool swap; + bool root_changed; + bool committed; +}; + +/* Folio and both PTE locks stay held across prepare, commit and finish. */ +int mm_subpage_move_prepare(struct mm_subpage_move *move, struct mm_subpage *slot, + struct vm_area_struct *src, unsigned long src_address, + struct vm_area_struct *dst, unsigned long dst_address, bool swap); +/* Present source PTE must be cleared and its TLB invalidated before commit. */ +int mm_subpage_move_commit(struct mm_subpage_move *move); +void mm_subpage_move_finish(struct mm_subpage_move *move); + +int mm_subpage_bind_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address); +bool mm_subpage_match_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address); +/* Return -1 for ordinary linear rmap; otherwise return referenced identities. */ +int mm_subpage_snapshot_rmap(struct folio *folio, struct mm_subpage_rmap *maps); +bool mm_subpage_address_in_vma(const struct folio *folio, + const struct vm_area_struct *vma, unsigned long *address); +bool rmap_walk_subpages(struct folio *folio, struct rmap_walk_control *rwc, + bool locked); + +struct anon_vma *mm_subpage_anon_root(const struct mm_subpage *subpage); + +/* Folio lock held; called once before publishing anonymous folio mapping. */ +void mm_subpage_hold_anon_root(struct mm_subpage *subpage, struct anon_vma *root, + bool droppable); +bool mm_subpage_is_droppable(const struct mm_subpage *subpage); + +#endif diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 01b151dca3085..8d72d991b05e5 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -16,10 +16,19 @@ #include #include #include +#include +#include #include #include #include "internal.h" #include "swap.h" +#ifdef CONFIG_MM_SUBPAGE +#include "user-subpage-internal.h" +#endif + +struct mfill_shared_data { + struct folio *source; /* NULL requests a zero-fill. */ +}; struct mfill_state { struct userfaultfd_ctx *ctx; @@ -32,6 +41,10 @@ struct mfill_state { unsigned long src_addr; unsigned long dst_addr; pmd_t *pmd; + struct folio *copy_buffer; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *slots[MM_SUBPAGES_PER_PAGE]; +#endif }; static bool anon_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags) @@ -194,8 +207,24 @@ static void uffd_mfill_unlock(struct vm_area_struct *vma) } #endif +/* Reservations keep the uninstalled leaves private to this operation. */ +static void mfill_free_subpages(struct mfill_state *state) +{ +#ifdef CONFIG_MM_SUBPAGE + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->slots); i++) { + if (state->slots[i]) { + mm_subpage_put(state->slots[i]); + state->slots[i] = NULL; + } + } +#endif +} + static void mfill_put_vma(struct mfill_state *state) { + mfill_free_subpages(state); if (!state->vma) return; @@ -248,6 +277,7 @@ static int mfill_get_vma(struct mfill_state *state) if ((flags & MFILL_ATOMIC_WP) && !(dst_vma->vm_flags & VM_UFFD_WP)) goto out_unlock; + if (is_vm_hugetlb_page(dst_vma)) return 0; @@ -320,16 +350,11 @@ static int mfill_establish_pmd(struct mfill_state *state) static bool mfill_file_over_size(struct vm_area_struct *dst_vma, unsigned long dst_addr) { - struct inode *inode; - pgoff_t offset, max_off; - if (!dst_vma->vm_file) return false; - inode = dst_vma->vm_file->f_inode; - offset = linear_page_index(dst_vma, dst_addr); - max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - return offset >= max_off; + return !vma_file_offset_valid(dst_vma, dst_addr, + i_size_read(file_inode(dst_vma->vm_file))); } /* @@ -341,7 +366,8 @@ static bool mfill_file_over_size(struct vm_area_struct *dst_vma, static int mfill_atomic_install_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, struct page *page, - uffd_flags_t flags) + uffd_flags_t flags, + const struct mfill_shared_data *shared_data) { int ret; struct mm_struct *dst_mm = dst_vma->vm_mm; @@ -354,6 +380,11 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, pte_t dst_ptep; _dst_pte = mk_pte(page, dst_vma->vm_page_prot); +#ifdef CONFIG_MM_SUBPAGE + if (page_in_cache && mm_page_size(dst_mm) < PAGE_SIZE) + _dst_pte = phys_pte_mm(dst_mm, page_to_phys(page) + + vma_page_offset_at(dst_vma, dst_addr).offset, dst_vma->vm_page_prot); +#endif _dst_pte = pte_mkdirty(_dst_pte); if (page_in_cache && !vm_shared) writable = false; @@ -384,7 +415,39 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, if (!pte_none(dst_ptep) && !pte_is_uffd_marker(dst_ptep)) goto out_unlock; +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + if (shared_data) { + struct vm_page_offset pos = vma_page_offset_at(dst_vma, dst_addr); + unsigned long offset = ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset; + unsigned long size = mm_page_size(dst_mm); + void *dst, *src; + + if (PageHWPoison(page)) { + ret = -EIO; + goto out_unlock; + } + if (!shmem_uffd_range_missing(folio, pos.index, pos.offset, size)) + goto out_unlock; + dst = kmap_local_folio(folio, offset); + if (shared_data->source) { + src = kmap_local_folio(shared_data->source, 0); + memcpy(dst, src, size); + kunmap_local(src); + } else { + memset(dst, 0, size); + } + kunmap_local(dst); + flush_dcache_folio(folio); + __folio_mark_uptodate(folio); + folio_mark_dirty(folio); + } +#endif + if (page_in_cache) { + mm_subpage_file_map_add(dst_vma, pte_phys_mm(dst_mm, _dst_pte)); + shmem_uffd_mark_mapped(folio, + (folio_page_idx(folio, page) << PAGE_SHIFT) + + vma_page_offset_at(dst_vma, dst_addr).offset, mm_page_size(dst_mm)); folio_add_file_rmap_pte(folio, page, dst_vma); } else { folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE); @@ -411,12 +474,13 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, return ret; } -static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) +static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr, + unsigned int offset, unsigned long len) { void *kaddr; int ret; - kaddr = kmap_local_folio(folio, 0); + kaddr = kmap_local_folio(folio, offset); /* * The read mmap_lock is held here. Despite the * mmap_lock being read recursive a deadlock is still @@ -433,8 +497,7 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) * and retry the copy outside the mmap_lock. */ pagefault_disable(); - ret = copy_from_user(kaddr, (const void __user *) src_addr, - PAGE_SIZE); + ret = copy_from_user(kaddr, (const void __user *)src_addr, len); pagefault_enable(); kunmap_local(kaddr); @@ -507,8 +570,11 @@ DEFINE_FREE(retry_put, struct mfill_retry_state *, if (_T) mfill_retry_state_put(_T)); static int mfill_copy_folio_retry(struct mfill_state *mfill_state, - struct folio *folio) + struct folio *folio, unsigned int offset, + unsigned long len) { + struct vm_page_offset pos = vma_page_offset_at(mfill_state->vma, + mfill_state->dst_addr); struct mfill_retry_state retry_state = { 0 }; struct mfill_retry_state *for_free __free(retry_put) = &retry_state; unsigned long src_addr = mfill_state->src_addr; @@ -520,8 +586,8 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, /* retry copying with mm_lock dropped */ mfill_put_vma(mfill_state); - kaddr = kmap_local_folio(folio, 0); - err = copy_from_user(kaddr, (const void __user *) src_addr, PAGE_SIZE); + kaddr = kmap_local_folio(folio, offset); + err = copy_from_user(kaddr, (const void __user *) src_addr, len); kunmap_local(kaddr); if (unlikely(err)) return -EFAULT; @@ -533,7 +599,10 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, if (err) return err; - if (mfill_retry_state_changed(&retry_state, mfill_state->vma)) + if (mfill_retry_state_changed(&retry_state, mfill_state->vma) || + (mm_page_size(mfill_state->ctx->mm) < PAGE_SIZE && + !vm_page_offset_equal(pos, vma_page_offset_at(mfill_state->vma, + mfill_state->dst_addr)))) return -EAGAIN; err = mfill_establish_pmd(mfill_state); @@ -543,6 +612,28 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, return 0; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) +/* Fill one missing leaf in new or retained shmem backing using staged data. */ +static int mfill_atomic_shmem_hole(struct mfill_state *state, + const struct mfill_shared_data *data) +{ + struct vm_page_offset pos; + struct folio *folio; + bool new; + int err; + + folio = shmem_uffd_prepare_folio(state->vma, state->dst_addr, &new); + if (IS_ERR(folio)) + return PTR_ERR(folio); + pos = vma_page_offset_at(state->vma, state->dst_addr); + err = mfill_atomic_install_pte(state->pmd, state->vma, state->dst_addr, + folio_file_page(folio, pos.index), state->flags, data); + if (err) + shmem_uffd_abort_folio(folio, state->vma, new); + return err; +} +#endif + static int __mfill_atomic_pte(struct mfill_state *state, const struct vm_uffd_ops *ops) { @@ -562,7 +653,7 @@ static int __mfill_atomic_pte(struct mfill_state *state, return -ENOMEM; if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) { - ret = mfill_copy_folio_locked(folio, src_addr); + ret = mfill_copy_folio_locked(folio, src_addr, 0, PAGE_SIZE); /* * Fallback to copy_from_user outside mmap_lock. * If retry is successful, mfill_copy_folio_locked() returns @@ -571,7 +662,7 @@ static int __mfill_atomic_pte(struct mfill_state *state, * will take care of unlocking if needed. */ if (unlikely(ret)) { - ret = mfill_copy_folio_retry(state, folio); + ret = mfill_copy_folio_retry(state, folio, 0, PAGE_SIZE); if (ret) goto err_folio_put; } @@ -590,12 +681,29 @@ static int __mfill_atomic_pte(struct mfill_state *state, if (ops->filemap_add) { ret = ops->filemap_add(folio, state->vma, state->dst_addr); +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + /* + * A pinned large shmem folio can survive a partial hole punch. + * Its cache entry exists while this native-sized range is missing. + * Reuse the prepared source and preserve all neighbouring bytes. + * Ordinary native insertion and non-shmem paths stay unchanged. + */ + if (ret == -EEXIST && (state->vma->vm_flags & VM_SHARED) && + shmem_file(state->vma->vm_file) && + (READ_ONCE(SHMEM_I(file_inode(state->vma->vm_file))->flags) & + SHMEM_F_UFFD_SUBPAGE)) { + struct mfill_shared_data data = { .source = folio }; + + ret = mfill_atomic_shmem_hole(state, &data); + goto err_folio_put; + } +#endif if (ret) goto err_folio_put; } ret = mfill_atomic_install_pte(state->pmd, state->vma, dst_addr, - &folio->page, flags); + &folio->page, flags, NULL); if (ret) goto err_filemap_remove; @@ -609,10 +717,184 @@ static int __mfill_atomic_pte(struct mfill_state *state, return ret; } +#ifdef CONFIG_MM_SUBPAGE +/* Pack only this request's leaves in one native linear index. */ +static struct mm_subpage *mfill_alloc_subpage(struct mfill_state *state) +{ + struct vm_area_struct *vma = state->vma; + struct mm_struct *mm = vma->vm_mm; + struct vm_page_offset pos = vma_page_offset_at(vma, state->dst_addr); + unsigned long size = mm_page_size(mm); + unsigned int first = pos.offset >> mm_page_shift(mm), i, nr; + struct mm_subpage_pool *pool; + struct mm_subpage *slot; + struct folio *folio; + int err = -ENOMEM; + + if (state->slots[first]) + goto take; + + mfill_free_subpages(state); + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return ERR_PTR(-ENOMEM); + folio = anon_alloc_folio(vma, state->dst_addr); + if (!folio) + goto out; + folio_throttle_swaprate(folio, GFP_KERNEL); + err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (err) { + folio_put(folio); + goto out; + } + nr = min_t(unsigned long, PAGE_SIZE - pos.offset, + state->dst_start + state->len - state->dst_addr) / size; + for (i = first; i < first + nr; i++) { + slot = mm_subpage_alloc_at(pool, i * size); + if (IS_ERR(slot)) { + err = PTR_ERR(slot); + mfill_free_subpages(state); + goto out; + } + state->slots[i] = slot; + } +out: + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + if (err) + return ERR_PTR(err); +take: + slot = state->slots[first]; + state->slots[first] = NULL; + return slot; +} + +static int mfill_atomic_subpage(struct mfill_state *state) +{ + struct mm_struct *mm = state->ctx->mm; + struct mm_subpage *slot = mfill_alloc_subpage(state); + struct vm_area_struct *vma; + struct folio *folio; + pte_t entry, *pte; + spinlock_t *ptl; + bool fresh; + int err; + + if (IS_ERR(slot)) + return PTR_ERR(slot); + folio = mm_subpage_folio(slot); + folio_lock(folio); + /* A previous leaf may have entered reclaim between iterations. */ + err = -EAGAIN; + if (folio_test_swapcache(folio)) + goto unlock_folio; + if (uffd_flags_mode_is(state->flags, MFILL_ATOMIC_COPY)) { + err = mfill_copy_folio_locked(folio, state->src_addr, + mm_subpage_offset(slot), mm_page_size(mm)); + if (err) { + /* + * The faulting retry must not hold the folio lock, or write + * a folio whose already-visible neighbours can be reclaimed. + * Use fresh backing and discard the remaining reservations + * before dropping the VMA lock. + */ + folio_unlock(folio); + mm_subpage_put(slot); + mfill_free_subpages(state); + slot = mfill_alloc_subpage(state); + if (IS_ERR(slot)) + return PTR_ERR(slot); + folio = mm_subpage_folio(slot); + err = mfill_copy_folio_retry(state, folio, + mm_subpage_offset(slot), mm_page_size(mm)); + if (err) + goto put_slot; + folio_lock(folio); + } + } + /* The allocator zeroed every byte, including unissued neighbours. */ + __folio_mark_uptodate(folio); + vma = state->vma; + pte = pte_offset_map_lock(mm, state->pmd, state->dst_addr, &ptl); + err = -EAGAIN; + if (!pte) + goto unlock_folio; + err = -EFAULT; + if (mfill_file_over_size(vma, state->dst_addr)) + goto unlock_pte; + err = -EEXIST; + if (!pte_none(ptep_get(pte)) && !pte_is_uffd_marker(ptep_get(pte))) + goto unlock_pte; + fresh = !folio_test_anon(folio); + mm_subpage_set_exclusive(slot); + err = mm_subpage_add_new_anon_rmap(slot, vma, state->dst_addr); + if (err) + goto unlock_pte; + if (fresh) + folio_add_lru_vma(folio, vma); + else if ((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED) + mlock_folio(folio); + inc_mm_counter(mm, MM_ANONPAGES); + entry = phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot); + entry = pte_mkdirty(entry); + if (vma->vm_flags & VM_WRITE) + entry = pte_mkwrite(entry, vma); + if (state->flags & MFILL_ATOMIC_WP) + entry = pte_mkuffd_wp(entry); + set_pte_at(mm, state->dst_addr, pte, entry); + update_mmu_cache(vma, state->dst_addr, pte); + /* The PTE owns slot now; release the folio lock before dropping PTL. */ + folio_unlock(folio); + pte_unmap_unlock(pte, ptl); + return 0; +unlock_pte: + pte_unmap_unlock(pte, ptl); +unlock_folio: + folio_unlock(folio); +put_slot: + mm_subpage_put(slot); + return err; +} +#endif + +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) +static int mfill_atomic_shared_subpage(struct mfill_state *state) +{ + struct mfill_shared_data data = {}; + int err; + + if (uffd_flags_mode_is(state->flags, MFILL_ATOMIC_COPY)) { + if (!state->copy_buffer) + state->copy_buffer = folio_alloc(GFP_KERNEL_ACCOUNT, 0); + data.source = state->copy_buffer; + if (!data.source) + return -ENOMEM; + err = mfill_copy_folio_locked(data.source, state->src_addr, 0, + mm_page_size(state->ctx->mm)); + if (err) + err = mfill_copy_folio_retry(state, data.source, 0, + mm_page_size(state->ctx->mm)); + if (err) + return err; + } + return mfill_atomic_shmem_hole(state, &data); +} +#endif + static int mfill_atomic_pte_copy(struct mfill_state *state) { const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(state->ctx->mm) < PAGE_SIZE) { +#ifdef CONFIG_SHMEM + if (state->vma->vm_flags & VM_SHARED) + return mfill_atomic_shared_subpage(state); +#endif + return mfill_atomic_subpage(state); + } +#endif + /* * The normal page fault path for a MAP_PRIVATE mapping in a * file-backed VMA will invoke the fault, fill the hole in the file and @@ -632,6 +914,16 @@ static int mfill_atomic_pte_zeroed_folio(struct mfill_state *state) { const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(state->ctx->mm) < PAGE_SIZE) { +#ifdef CONFIG_SHMEM + if (state->vma->vm_flags & VM_SHARED) + return mfill_atomic_shared_subpage(state); +#endif + return mfill_atomic_subpage(state); + } +#endif + return __mfill_atomic_pte(state, ops); } @@ -695,6 +987,13 @@ static int mfill_atomic_pte_continue(struct mfill_state *state) if (IS_ERR_OR_NULL(folio)) return -EFAULT; +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + if (vma_is_shmem(dst_vma) && shmem_uffd_range_missing(folio, pgoff, + vma_page_offset_at(dst_vma, dst_addr).offset, mm_page_size(dst_vma->vm_mm))) { + ret = -EFAULT; + goto out_release; + } +#endif page = folio_file_page(folio, pgoff); if (PageHWPoison(page)) { ret = -EIO; @@ -702,7 +1001,7 @@ static int mfill_atomic_pte_continue(struct mfill_state *state) } ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, - page, flags); + page, flags, NULL); if (ret) goto out_release; @@ -964,8 +1263,8 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, /* * Sanitize the command parameters: */ - VM_WARN_ON_ONCE(dst_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(dst_start & ~mm_page_mask(ctx->mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(ctx->mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(src_start + len <= src_start); @@ -998,9 +1297,9 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, cond_resched(); if (!err) { - state.dst_addr += PAGE_SIZE; - state.src_addr += PAGE_SIZE; - copied += PAGE_SIZE; + state.dst_addr += mm_page_size(ctx->mm); + state.src_addr += mm_page_size(ctx->mm); + copied += mm_page_size(ctx->mm); if (fatal_signal_pending(current)) err = -EINTR; @@ -1011,6 +1310,8 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, mfill_put_vma(&state); out: + if (state.copy_buffer) + folio_put(state.copy_buffer); VM_WARN_ON_ONCE(copied < 0); VM_WARN_ON_ONCE(err > 0); VM_WARN_ON_ONCE(!copied && !err); @@ -1099,8 +1400,8 @@ int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, /* * Sanitize the command parameters: */ - VM_WARN_ON_ONCE(start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(start & ~mm_page_mask(dst_mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(dst_mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(start + len <= start); @@ -1303,7 +1604,102 @@ static long move_present_ptes(struct mm_struct *mm, return src_addr > src_start ? src_addr - src_start : err; } +#ifdef CONFIG_MM_SUBPAGE +static long move_present_subpage(struct mm_struct *mm, + struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, + unsigned long dst_addr, unsigned long src_addr, + pte_t *dst_pte, pte_t *src_pte, pte_t orig_dst, pte_t orig_src, + pmd_t *dst_pmd, pmd_t dst_pmdval, + spinlock_t *dst_ptl, spinlock_t *src_ptl, struct folio *folio) +{ + struct mm_subpage_move move; + struct mm_subpage *slot = NULL; + pte_t pte; + long ret = -EAGAIN; + + flush_cache_range(src_vma, src_addr, src_addr + mm_page_size(mm)); + double_pt_lock(dst_ptl, src_ptl); + if (!is_pte_pages_stable(dst_pte, src_pte, orig_dst, orig_src, + dst_pmd, dst_pmdval)) + goto out; + ret = -EBUSY; + if (!folio_test_anon(folio) || folio_test_large(folio)) + goto out; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, orig_src)); + if (!slot || mm_subpage_folio(slot) != folio) + goto out; + ret = mm_subpage_move_prepare(&move, slot, src_vma, src_addr, + dst_vma, dst_addr, false); + if (ret) + goto out; + orig_src = ptep_clear_flush(src_vma, src_addr, src_pte); + ret = mm_subpage_move_commit(&move); + if (ret) { + set_pte_at(mm, src_addr, src_pte, orig_src); + } else { + pte = phys_pte_mm(mm, mm_subpage_phys(slot), dst_vma->vm_page_prot); + if (pgtable_supports_soft_dirty()) + pte = pte_mksoft_dirty(pte); + if (pte_dirty(orig_src)) + pte = pte_mkdirty(pte); + pte = pte_mkwrite(pte, dst_vma); + set_pte_at(mm, dst_addr, dst_pte, pte); + ret = mm_page_size(mm); + } + mm_subpage_move_finish(&move); +out: + if (slot) + mm_subpage_put(slot); + double_pt_unlock(dst_ptl, src_ptl); + return ret; +} + +/* Both PTLs and, if present, the swap-cache folio lock are held. */ +static int move_swap_subpage(struct mm_struct *mm, struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, unsigned long dst_addr, + unsigned long src_addr, pte_t *dst_pte, pte_t *src_pte, + pte_t orig_src, struct folio *folio, swp_entry_t entry) +{ + unsigned int offset = pte_swp_subpage_offset(orig_src); + struct mm_subpage_move move; + struct mm_subpage *slot = NULL; + int ret = -EBUSY; + + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(mm))) + return -EFAULT; + if (swap_subpage_count(entry, offset) != 1) + return -EBUSY; + if (folio) { + slot = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(folio)) + offset); + if (slot) { + ret = mm_subpage_move_prepare(&move, slot, src_vma, src_addr, + dst_vma, dst_addr, true); + if (ret) + goto out; + ret = mm_subpage_move_commit(&move); + if (ret) { + mm_subpage_move_finish(&move); + goto out; + } + } + } + /* No mapping count changes: the same-mm exclusive swap PTE is transferred. */ + orig_src = ptep_get_and_clear(mm, src_addr, src_pte); + if (pgtable_supports_soft_dirty()) + orig_src = pte_swp_mksoft_dirty(orig_src); + set_pte_at(mm, dst_addr, dst_pte, orig_src); + if (slot) + mm_subpage_move_finish(&move); + ret = mm_page_size(mm); +out: + if (slot) + mm_subpage_put(slot); + return ret; +} +#endif + static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, unsigned long dst_addr, unsigned long src_addr, pte_t *dst_pte, pte_t *src_pte, pte_t orig_dst_pte, pte_t orig_src_pte, @@ -1329,6 +1725,21 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, return -EAGAIN; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + int ret; + + if (!src_folio && swap_cache_has_folio(entry)) { + double_pt_unlock(dst_ptl, src_ptl); + return -EAGAIN; + } + ret = move_swap_subpage(mm, dst_vma, src_vma, dst_addr, src_addr, + dst_pte, src_pte, orig_src_pte, src_folio, entry); + double_pt_unlock(dst_ptl, src_ptl); + return ret; + } +#endif + /* * The src_folio resides in the swapcache, requiring an update to its * index and mapping to align with the dst_vma, where a swap-in may @@ -1381,13 +1792,13 @@ static int move_zeropage_pte(struct mm_struct *mm, return -EAGAIN; } - zero_pte = pte_mkspecial(pfn_pte(zero_pfn(dst_addr), - dst_vma->vm_page_prot)); + zero_pte = pte_mkspecial(phys_pte_mm(mm, PFN_PHYS(zero_pfn(dst_addr)), + dst_vma->vm_page_prot)); ptep_clear_flush(src_vma, src_addr, src_pte); set_pte_at(mm, dst_addr, dst_pte, zero_pte); double_pt_unlock(dst_ptl, src_ptl); - return PAGE_SIZE; + return mm_page_size(mm); } @@ -1474,7 +1885,7 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES)) ret = -ENOENT; else /* nothing to do to move a hole */ - ret = PAGE_SIZE; + ret = mm_page_size(mm); goto out; } @@ -1514,7 +1925,8 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd } folio = vm_normal_folio(src_vma, src_addr, orig_src_pte); - if (!folio || !PageAnonExclusive(&folio->page)) { + if (!folio || (mm_page_size(mm) >= PAGE_SIZE && + !PageAnonExclusive(&folio->page))) { spin_unlock(src_ptl); ret = -EBUSY; goto out; @@ -1554,6 +1966,15 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd } } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + ret = move_present_subpage(mm, dst_vma, src_vma, dst_addr, src_addr, + dst_pte, src_pte, orig_dst_pte, orig_src_pte, + dst_pmd, dst_pmdval, dst_ptl, src_ptl, src_folio); + goto out; + } +#endif + /* at this point we have src_folio locked */ if (folio_test_large(src_folio)) { /* split_folio() can block */ @@ -1592,7 +2013,8 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd goto out; } - if (!pte_swp_exclusive(orig_src_pte)) { + /* Smaller leaves validate per-slot swap and mapping counts under PTL. */ + if (mm_page_size(mm) >= PAGE_SIZE && !pte_swp_exclusive(orig_src_pte)) { ret = -EBUSY; goto out; } @@ -1635,7 +2057,7 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd goto retry; } } - ret = move_swap_pte(mm, dst_vma, dst_addr, src_addr, dst_pte, src_pte, + ret = move_swap_pte(mm, dst_vma, src_vma, dst_addr, src_addr, dst_pte, src_pte, orig_dst_pte, orig_src_pte, dst_pmd, dst_pmdval, dst_ptl, src_ptl, src_folio, si, entry); } @@ -1694,6 +2116,11 @@ static int validate_move_areas(struct userfaultfd_ctx *ctx, pgprot_val(src_vma->vm_page_prot) != pgprot_val(dst_vma->vm_page_prot)) return -EINVAL; + /* Native backing cannot mix droppable and swap-backed slot policies. */ + if (mm_page_size(src_vma->vm_mm) < PAGE_SIZE && + ((src_vma->vm_flags ^ dst_vma->vm_flags) & VM_DROPPABLE)) + return -EINVAL; + /* Only allow moving if both are mlocked or both aren't */ if ((src_vma->vm_flags & VM_LOCKED) != (dst_vma->vm_flags & VM_LOCKED)) return -EINVAL; @@ -1926,9 +2353,9 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, ssize_t moved = 0; /* Sanitize the command parameters. */ - VM_WARN_ON_ONCE(src_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(dst_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(src_start & ~mm_page_mask(mm)); + VM_WARN_ON_ONCE(dst_start & ~mm_page_mask(mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(src_start + len < src_start); @@ -2053,10 +2480,12 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, ret = move_pages_ptes(mm, dst_pmd, src_pmd, dst_vma, src_vma, dst_addr, src_addr, src_end - src_addr, mode); - if (ret < 0) + if (ret < 0) { err = ret; - else + } else { + err = 0; step_size = ret; + } } cond_resched(); @@ -2095,6 +2524,17 @@ bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags, { const struct vm_uffd_ops *ops = vma_uffd_ops(vma); + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + return false; + if (vma_is_anonymous(vma)) { + if (vm_flags & VM_UFFD_MINOR) + return false; + } else if (!vma_is_shmem(vma)) { + return false; + } + } + if (vma->vm_flags & (VM_DROPPABLE | VM_SHADOW_STACK)) return false; diff --git a/mm/util.c b/mm/util.c index 3b26f5bf92012..1f41dfdd3ad13 100644 --- a/mm/util.c +++ b/mm/util.c @@ -344,21 +344,22 @@ unsigned long randomize_stack_top(unsigned long stack_top) if (current->flags & PF_RANDOMIZE) { random_variable = get_random_long(); - random_variable &= STACK_RND_MASK; - random_variable <<= PAGE_SHIFT; + random_variable &= mm_aslr_mask(current->mm, STACK_RND_MASK); + random_variable <<= mm_page_shift(current->mm); } #ifdef CONFIG_STACK_GROWSUP - return PAGE_ALIGN(stack_top) + random_variable; + return mm_page_align(current->mm, stack_top) + random_variable; #else - return PAGE_ALIGN(stack_top) - random_variable; + return mm_page_align(current->mm, stack_top) - random_variable; #endif } /** - * randomize_page - Generate a random, page aligned address + * randomize_page_shift - Generate a random, granule aligned address * @start: The smallest acceptable address the caller will take. * @range: The size of the area, starting at @start, within which the * random address must fall. + * @shift: Log2 of the required address alignment. * * If @start + @range would overflow, @range is capped. * @@ -368,22 +369,31 @@ unsigned long randomize_stack_top(unsigned long stack_top) * Return: A page aligned address within [start, start + range). On error, * @start is returned. */ -unsigned long randomize_page(unsigned long start, unsigned long range) +static unsigned long randomize_page_shift(unsigned long start, + unsigned long range, unsigned int shift) { - if (!PAGE_ALIGNED(start)) { - range -= PAGE_ALIGN(start) - start; - start = PAGE_ALIGN(start); + unsigned long page_size = 1UL << shift; + + if (!IS_ALIGNED(start, page_size)) { + range -= ALIGN(start, page_size) - start; + start = ALIGN(start, page_size); } if (start > ULONG_MAX - range) range = ULONG_MAX - start; - range >>= PAGE_SHIFT; + range >>= shift; if (range == 0) return start; - return start + (get_random_long() % range << PAGE_SHIFT); + return start + (get_random_long() % range << shift); +} + +/* Existing callers retain native-page alignment. */ +unsigned long randomize_page(unsigned long start, unsigned long range) +{ + return randomize_page_shift(start, range, PAGE_SHIFT); } #ifdef CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT @@ -391,23 +401,25 @@ unsigned long __weak arch_randomize_brk(struct mm_struct *mm) { /* Is the current task 32bit ? */ if (!IS_ENABLED(CONFIG_64BIT) || is_compat_task()) - return randomize_page(mm->brk, SZ_32M); + return randomize_page_shift(mm->brk, SZ_32M, mm_page_shift(mm)); - return randomize_page(mm->brk, SZ_1G); + return randomize_page_shift(mm->brk, SZ_1G, mm_page_shift(mm)); } unsigned long arch_mmap_rnd(void) { - unsigned long rnd; + unsigned int bits; + unsigned long mask; #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS if (is_compat_task()) - rnd = get_random_long() & ((1UL << mmap_rnd_compat_bits) - 1); + bits = mmap_rnd_compat_bits; else #endif /* CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS */ - rnd = get_random_long() & ((1UL << mmap_rnd_bits) - 1); + bits = mmap_rnd_bits; - return rnd << PAGE_SHIFT; + mask = mm_aslr_mask(current->mm, (1UL << bits) - 1); + return (get_random_long() & mask) << mm_page_shift(current->mm); } static int mmap_is_legacy(const struct rlimit *rlim_stack) @@ -440,14 +452,15 @@ static unsigned long mmap_base(const unsigned long rnd, const struct rlimit *rli * task. mmap_base starts directly below the stack and grows * downwards. */ - return PAGE_ALIGN_DOWN(mmap_upper_limit(rlim_stack) - rnd); + return (mmap_upper_limit(rlim_stack) - rnd) & mm_page_mask(current->mm); #else unsigned long gap = rlim_stack->rlim_cur; unsigned long pad = stack_guard_gap; /* Account for stack randomization if necessary */ if (current->flags & PF_RANDOMIZE) - pad += (STACK_RND_MASK << PAGE_SHIFT); + pad += mm_aslr_mask(current->mm, STACK_RND_MASK) << + mm_page_shift(current->mm); /* Values close to RLIM_INFINITY can overflow. */ if (gap + pad > gap) @@ -458,7 +471,7 @@ static unsigned long mmap_base(const unsigned long rnd, const struct rlimit *rli else if (gap > MAX_GAP) gap = MAX_GAP; - return PAGE_ALIGN(STACK_TOP - gap - rnd); + return mm_page_align(current->mm, STACK_TOP - gap - rnd); #endif } @@ -491,7 +504,7 @@ EXPORT_SYMBOL_IF_KUNIT(arch_pick_mmap_layout); /** * __account_locked_vm - account locked pages to an mm's locked_vm * @mm: mm to account against - * @pages: number of pages to account + * @pages: number of native physical pages to account * @inc: %true if @pages should be considered positive, %false if not * @task: task used to check RLIMIT_MEMLOCK * @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped @@ -507,27 +520,35 @@ int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, const struct task_struct *task, bool bypass_rlim) { unsigned long locked_vm, limit; + unsigned int shift = PAGE_SHIFT - MM_ACCOUNT_SHIFT; int ret = 0; mmap_assert_write_locked(mm); + /* This exported API receives native physical-page counts. */ + if (pages > ULONG_MAX >> shift) + return -ENOMEM; + pages <<= shift; locked_vm = mm->locked_vm; if (inc) { if (!bypass_rlim) { - limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT; - if (locked_vm + pages > limit) + limit = task_rlimit(task, RLIMIT_MEMLOCK) >> MM_ACCOUNT_SHIFT; + if (pages > limit || locked_vm > limit - pages) ret = -ENOMEM; } + if (!ret && pages > ULONG_MAX - locked_vm) + ret = -ENOMEM; if (!ret) mm->locked_vm = locked_vm + pages; } else { - WARN_ON_ONCE(pages > locked_vm); + if (WARN_ON_ONCE(pages > locked_vm)) + return -EINVAL; mm->locked_vm = locked_vm - pages; } pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid, - (void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT, - locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK), + (void *)_RET_IP_, (inc) ? '+' : '-', pages << MM_ACCOUNT_SHIFT, + locked_vm << MM_ACCOUNT_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK), ret ? " - exceeded" : ""); return ret; @@ -562,11 +583,11 @@ int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc) } EXPORT_SYMBOL_GPL(account_locked_vm); -unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, +static unsigned long vm_mmap_offset_internal(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, - unsigned long flag, unsigned long pgoff) + unsigned long flag, struct vm_page_offset pos) { - loff_t off = (loff_t)pgoff << PAGE_SHIFT; + loff_t off = ((loff_t)pos.index << PAGE_SHIFT) | pos.offset; unsigned long ret; struct mm_struct *mm = current->mm; unsigned long populate; @@ -578,8 +599,11 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, if (!ret) { if (mmap_write_lock_killable(mm)) return -EINTR; - ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate, - &uf); +#ifdef CONFIG_MM_SUBPAGE + ret = do_mmap_offset(file, addr, len, prot, flag, 0, pos, &populate, &uf); +#else + ret = do_mmap(file, addr, len, prot, flag, 0, pos.index, &populate, &uf); +#endif mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate) @@ -588,6 +612,13 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, return ret; } +unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flag, unsigned long pgoff) +{ + return vm_mmap_offset_internal(file, addr, len, prot, flag, + (struct vm_page_offset) { pgoff, 0 }); +} + /* * Perform a userland memory mapping into the current process address space. See * the comment for do_mmap() for more details on this operation in general. @@ -609,12 +640,13 @@ unsigned long vm_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flag, unsigned long offset) { - if (unlikely(offset + PAGE_ALIGN(len) < offset)) + if (unlikely(offset + mm_page_align(current->mm, len) < offset)) return -EINVAL; - if (unlikely(offset_in_page(offset))) + if (unlikely(offset & ~mm_page_mask(current->mm))) return -EINVAL; - return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT); + return vm_mmap_offset_internal(file, addr, len, prot, flag, + (struct vm_page_offset) { offset >> PAGE_SHIFT, offset & ~PAGE_MASK }); } EXPORT_SYMBOL(vm_mmap); @@ -988,7 +1020,8 @@ int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin if (mm) { long reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10); - allowed -= min_t(long, mm->total_vm / 32, reserve); + allowed -= min_t(long, + mm_pages_to_bytes(mm, mm->total_vm) >> (PAGE_SHIFT + 5), reserve); } if (percpu_counter_read_positive(&vm_committed_as) < allowed) @@ -1189,6 +1222,9 @@ void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->end = vma->vm_end; desc->pgoff = vma->vm_pgoff; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = vma->vm_subpage_offset; +#endif desc->vm_file = vma->vm_file; desc->vma_flags = vma->flags; desc->page_prot = vma->vm_page_prot; diff --git a/mm/vma.c b/mm/vma.c index d90791b00a7b8..4161f7739a10e 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -13,7 +13,7 @@ struct mmap_state { unsigned long addr; unsigned long end; - pgoff_t pgoff; + struct vm_page_offset offset; unsigned long pglen; union { vm_flags_t vm_flags; @@ -42,19 +42,25 @@ struct mmap_state { bool file_doesnt_need_get :1; }; -#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, vma_flags_, file_) \ +#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pos_, vma_flags_, file_) \ struct mmap_state name = { \ .mm = mm_, \ .vmi = vmi_, \ .addr = addr_, \ .end = (addr_) + (len_), \ - .pgoff = pgoff_, \ - .pglen = PHYS_PFN(len_), \ + .offset = pos_, \ + .pglen = (len_) >> mm_page_shift(mm_), \ .vma_flags = vma_flags_, \ .file = file_, \ .page_prot = vma_get_page_prot(vma_flags_), \ } +#ifdef CONFIG_MM_SUBPAGE +#define VMG_MMAP_SUBPAGE_STATE(map_) .subpage_offset = (map_)->offset.offset, +#else +#define VMG_MMAP_SUBPAGE_STATE(map_) +#endif + #define VMG_MMAP_STATE(name, map_, vma_) \ struct vma_merge_struct name = { \ .mm = (map_)->mm, \ @@ -62,7 +68,8 @@ struct mmap_state { .start = (map_)->addr, \ .end = (map_)->end, \ .vma_flags = (map_)->vma_flags, \ - .pgoff = (map_)->pgoff, \ + .pgoff = (map_)->offset.index, \ + VMG_MMAP_SUBPAGE_STATE(map_) \ .file = (map_)->file, \ .prev = (map_)->prev, \ .middle = vma_, \ @@ -197,11 +204,12 @@ static void init_multi_vma_prep(struct vma_prepare *vp, */ static bool can_vma_merge_before(struct vma_merge_struct *vmg) { - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); + struct vm_page_offset end = vm_page_offset_add(vmg_page_offset(vmg), + vmg->end - vmg->start); if (is_mergeable_vma(vmg, /* merge_next = */ true) && is_mergeable_anon_vma(vmg, /* merge_next = */ true)) { - if (vmg->next->vm_pgoff == vmg->pgoff + pglen) + if (vm_page_offset_equal(vma_page_offset_at(vmg->next, vmg->next->vm_start), end)) return true; } @@ -221,7 +229,8 @@ static bool can_vma_merge_after(struct vma_merge_struct *vmg) { if (is_mergeable_vma(vmg, /* merge_next = */ false) && is_mergeable_anon_vma(vmg, /* merge_next = */ false)) { - if (vmg->prev->vm_pgoff + vma_pages(vmg->prev) == vmg->pgoff) + if (vm_page_offset_equal(vma_page_offset_at(vmg->prev, vmg->prev->vm_end), + vmg_page_offset(vmg))) return true; } return false; @@ -521,7 +530,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, new->vm_end = addr; } else { new->vm_start = addr; - new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); + vma_set_page_offset(new, vma_page_offset_at(vma, addr)); } err = -ENOMEM; @@ -560,7 +569,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, if (new_below) { vma->vm_start = addr; - vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT; + vma_set_page_offset(vma, vma_page_offset_at(new, addr)); } else { vma->vm_end = addr; } @@ -705,19 +714,21 @@ void validate_mm(struct mm_struct *mm) static void vmg_adjust_set_range(struct vma_merge_struct *vmg) { struct vm_area_struct *adjust; - pgoff_t pgoff; + struct vm_page_offset pos; if (vmg->__adjust_middle_start) { adjust = vmg->middle; - pgoff = adjust->vm_pgoff + PHYS_PFN(vmg->end - adjust->vm_start); + pos = vma_page_offset_at(adjust, vmg->end); } else if (vmg->__adjust_next_start) { adjust = vmg->next; - pgoff = adjust->vm_pgoff - PHYS_PFN(adjust->vm_start - vmg->end); + pos = vm_page_offset_sub(vma_page_offset_at(adjust, adjust->vm_start), + adjust->vm_start - vmg->end); } else { return; } - vma_set_range(adjust, vmg->end, adjust->vm_end, pgoff); + vma_set_range(adjust, vmg->end, adjust->vm_end, pos.index); + vma_set_page_offset(adjust, pos); } /* @@ -762,6 +773,7 @@ static int commit_merge(struct vma_merge_struct *vmg) vma_adjust_trans_huge(vma, vmg->start, vmg->end, vmg->__adjust_middle_start ? vmg->middle : NULL); vma_set_range(vma, vmg->start, vmg->end, vmg->pgoff); + vma_set_page_offset(vma, vmg_page_offset(vmg)); vmg_adjust_set_range(vmg); vma_iter_store_overwrite(vmg->vmi, vmg->target); @@ -931,7 +943,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( vmg->start = prev->vm_start; vmg->end = next->vm_end; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); /* * We already ensured anon_vma compatibility above, so now it's @@ -950,7 +962,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( */ vmg->start = prev->vm_start; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); if (!vmg->__remove_middle) vmg->__adjust_middle_start = true; @@ -965,7 +977,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( * shrink/delete extend */ - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); + unsigned long len = vmg->end - vmg->start; VM_WARN_ON_VMG(!merge_right, vmg); /* If we are offset into a VMA, then prev must be middle. */ @@ -973,13 +985,14 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( if (vmg->__remove_middle) { vmg->end = next->vm_end; - vmg->pgoff = next->vm_pgoff - pglen; + vmg_set_page_offset(vmg, + vm_page_offset_sub(vma_page_offset_at(next, next->vm_start), len)); } else { /* We shrink middle and expand next. */ vmg->__adjust_next_start = true; vmg->start = middle->vm_start; vmg->end = start; - vmg->pgoff = middle->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(middle, middle->vm_start)); } err = dup_anon_vma(next, middle, &anon_dup); @@ -1088,7 +1101,7 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg) if (can_merge_left) { vmg->start = prev->vm_start; vmg->target = prev; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); /* * If this merge would result in removal of the next VMA but we @@ -1245,7 +1258,7 @@ int vma_expand(struct vma_merge_struct *vmg) * Returns: 0 on success, -ENOMEM otherwise */ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff) + unsigned long start, unsigned long end, struct vm_page_offset pos) { struct vma_prepare vp; @@ -1266,7 +1279,8 @@ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_adjust_trans_huge(vma, start, end, NULL); vma_iter_clear(vmi); - vma_set_range(vma, start, end, pgoff); + vma_set_range(vma, start, end, pos.index); + vma_set_page_offset(vma, pos); vma_complete(&vp, vmi, vma->vm_mm); validate_mm(vma->vm_mm); return 0; @@ -1335,7 +1349,7 @@ static void vms_complete_munmap_vmas(struct vma_munmap_struct *vms, mm = current->mm; mm->map_count -= vms->vma_count; - mm->locked_vm -= vms->locked_vm; + mm->locked_vm -= mm_pages_to_account(mm, vms->locked_vm); if (vms->unlock) mmap_write_downgrade(mm); @@ -1360,7 +1374,7 @@ static void vms_complete_munmap_vmas(struct vma_munmap_struct *vms, mas_for_each(mas_detach, vma, ULONG_MAX) remove_vma(vma); - vm_unacct_memory(vms->nr_accounted); + mm_unacct_memory(mm, vms->nr_accounted); validate_mm(mm); if (vms->unlock) mmap_read_unlock(mm); @@ -1456,7 +1470,7 @@ static int vms_gather_munmap_vmas(struct vma_munmap_struct *vms, goto munmap_gather_failed; vma_mark_detached(next); - nrpages = vma_pages(next); + nrpages = vma_user_pages(next); vms->nr_pages += nrpages; if (vma_test(next, VMA_LOCKED_BIT)) @@ -1634,10 +1648,10 @@ int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long end; struct vm_area_struct *vma; - if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) + if ((start & ~mm_page_mask(mm)) || start > TASK_SIZE || len > TASK_SIZE-start) return -EINVAL; - end = start + PAGE_ALIGN(len); + end = start + mm_page_align(mm, len); if (end == start) return -EINVAL; @@ -1862,7 +1876,7 @@ static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) * prior to moving page table entries, to effect an mremap move. */ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - unsigned long addr, unsigned long len, pgoff_t pgoff, + unsigned long addr, unsigned long len, struct vm_page_offset pos, bool *need_rmap_locks) { struct vm_area_struct *vma = *vmap; @@ -1878,7 +1892,10 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, * to match new location, to increase its chance of merging. */ if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { - pgoff = addr >> PAGE_SHIFT; + pos = (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, + .offset = addr & ~PAGE_MASK, + }; faulted_in_anon_vma = false; } @@ -1894,7 +1911,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, if (new_vma && new_vma->vm_start < addr + len) return NULL; /* should never get here */ - vmg.pgoff = pgoff; + vmg_set_page_offset(&vmg, pos); vmg.next = vma_iter_next_rewind(&vmi, NULL); new_vma = vma_merge_copied_range(&vmg); @@ -1924,7 +1941,8 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, new_vma = vm_area_dup(vma); if (!new_vma) goto out; - vma_set_range(new_vma, addr, addr + len, pgoff); + vma_set_range(new_vma, addr, addr + len, pos.index); + vma_set_page_offset(new_vma, pos); if (vma_dup_policy(vma, new_vma)) goto out_free_vma; if (anon_vma_clone(new_vma, vma, VMA_OP_REMAP)) @@ -1979,7 +1997,8 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct * mpol_equal(vma_policy(a), vma_policy(b)) && a->vm_file == b->vm_file && vma_flags_empty(&diff) && - b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); + vm_page_offset_equal(vma_page_offset_at(a, b->vm_start), + vma_page_offset_at(b, b->vm_start)); } /* @@ -2398,7 +2417,10 @@ static void set_desc_from_map(struct vm_area_desc *desc, desc->start = map->addr; desc->end = map->end; - desc->pgoff = map->pgoff; + desc->pgoff = map->offset.index; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = map->offset.offset; +#endif desc->vm_file = map->file; desc->vma_flags = map->vma_flags; desc->page_prot = map->page_prot; @@ -2458,7 +2480,7 @@ static int __mmap_setup(struct mmap_state *map, struct vm_area_desc *desc, map->charged = map->pglen; map->charged -= vms->nr_accounted; if (map->charged) { - error = security_vm_enough_memory_mm(map->mm, map->charged); + error = mm_account_memory(map->mm, map->mm, map->charged); if (error) return error; } @@ -2549,7 +2571,8 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, return -ENOMEM; vma_iter_config(vmi, map->addr, map->end); - vma_set_range(vma, map->addr, map->end, map->pgoff); + vma_set_range(vma, map->addr, map->end, map->offset.index); + vma_set_page_offset(vma, map->offset); vma->flags = map->vma_flags; vma->vm_page_prot = map->page_prot; @@ -2621,7 +2644,7 @@ static void __mmap_complete(struct mmap_state *map, struct vm_area_struct *vma) if (!vma_supports_mlock(vma)) vma_clear_flags_mask(vma, VMA_LOCKED_MASK); else - mm->locked_vm += map->pglen; + mm->locked_vm += mm_pages_to_account(mm, map->pglen); } if (vma->vm_file) @@ -2678,7 +2701,10 @@ static int call_mmap_prepare(struct mmap_state *map, return err; /* Update fields permitted to be changed. */ - map->pgoff = desc->pgoff; + map->offset.index = desc->pgoff; +#ifdef CONFIG_MM_SUBPAGE + map->offset.offset = desc->subpage_offset; +#endif if (desc->vm_file != map->file) { map->file_doesnt_need_get = true; map->file = desc->vm_file; @@ -2689,6 +2715,12 @@ static int call_mmap_prepare(struct mmap_state *map, map->vm_ops = desc->vm_ops; map->vm_private_data = desc->private_data; +#ifdef CONFIG_MM_SUBPAGE + if (map->offset.offset >= PAGE_SIZE || + !IS_ALIGNED(map->offset.offset, mm_page_size(map->mm))) + return -EINVAL; +#endif + return 0; } @@ -2731,13 +2763,13 @@ static bool can_set_ksm_flags_early(struct mmap_state *map) static unsigned long __mmap_region(struct file *file, unsigned long addr, unsigned long len, vma_flags_t vma_flags, - unsigned long pgoff, struct list_head *uf) + struct vm_page_offset pos, struct list_head *uf) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma = NULL; bool have_mmap_prepare = file && file->f_op->mmap_prepare; VMA_ITERATOR(vmi, mm, addr); - MMAP_STATE(map, mm, &vmi, addr, len, pgoff, vma_flags, file); + MMAP_STATE(map, mm, &vmi, addr, len, pos, vma_flags, file); struct vm_area_desc desc = { .mm = mm, .file = file, @@ -2791,7 +2823,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, /* Accounting was done by __mmap_setup(). */ unacct_error: if (map.charged) - vm_unacct_memory(map.charged); + mm_unacct_memory(mm, map.charged); abort_munmap: /* * This indicates that .mmap_prepare has set a new file, differing from @@ -2807,7 +2839,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, /** * mmap_region() - Actually perform the userland mapping of a VMA into * current->mm with known, aligned and overflow-checked @addr and @len, and - * correctly determined VMA flags @vm_flags and page offset @pgoff. + * correctly determined VMA flags @vm_flags and file byte position @pos. * * This is an internal memory management function, and should not be used * directly. @@ -2819,8 +2851,8 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, * @addr: The page-aligned address at which to perform the mapping. * @len: The page-aligned, non-zero, length of the mapping. * @vm_flags: The VMA flags which should be applied to the mapping. - * @pgoff: If @file is specified, the page offset into the file, if not then - * the virtual page offset in memory of the anonymous mapping. + * @pos: Native page-cache index and byte offset within it. For anonymous + * mappings, the position is derived from the virtual address. * @uf: Optionally, a pointer to a list head used for tracking userfaultfd unmap * events. * @@ -2829,13 +2861,15 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, */ unsigned long mmap_region(struct file *file, unsigned long addr, unsigned long len, vm_flags_t vm_flags, - unsigned long pgoff, struct list_head *uf) + struct vm_page_offset pos, struct list_head *uf) { unsigned long ret; bool writable_file_mapping = false; const vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags); mmap_assert_write_locked(current->mm); + if (pos.offset >= PAGE_SIZE || !IS_ALIGNED(pos.offset, mm_page_size(current->mm))) + return -EINVAL; /* Check to see if MDWE is applicable. */ if (map_deny_write_exec(&vma_flags, &vma_flags)) @@ -2854,7 +2888,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, writable_file_mapping = true; } - ret = __mmap_region(file, addr, len, vma_flags, pgoff, uf); + ret = __mmap_region(file, addr, len, vma_flags, pos, uf); /* Clear our write mapping regardless of error. */ if (writable_file_mapping) @@ -2892,13 +2926,13 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_flags_set_mask(&vma_flags, mm->def_vma_flags); vma_flags = ksm_vma_flags(mm, NULL, vma_flags); - if (!may_expand_vm(mm, &vma_flags, len >> PAGE_SHIFT)) + if (!may_expand_vm(mm, &vma_flags, len >> mm_page_shift(mm))) return -ENOMEM; if (mm->map_count > get_sysctl_max_map_count()) return -ENOMEM; - if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) + if (mm_account_memory(mm, mm, len >> mm_page_shift(mm))) return -ENOMEM; /* @@ -2908,6 +2942,10 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, if (vma && vma->vm_end == addr) { VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, PHYS_PFN(addr)); + vmg_set_page_offset(&vmg, (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, .offset = addr & ~PAGE_MASK, + }); + vmg.prev = vma; /* vmi is positioned at prev, which this mode expects. */ vmg.just_expand = true; @@ -2927,6 +2965,9 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_set_anonymous(vma); vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT); + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, .offset = addr & ~PAGE_MASK, + }); vma->flags = vma_flags; vma->vm_page_prot = vm_get_page_prot(vma_flags_to_legacy(vma_flags)); vma_start_write(vma); @@ -2937,10 +2978,10 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, validate_mm(mm); out: perf_event_mmap(vma); - mm->total_vm += len >> PAGE_SHIFT; - mm->data_vm += len >> PAGE_SHIFT; + mm->total_vm += len >> mm_page_shift(mm); + mm->data_vm += len >> mm_page_shift(mm); if (vma_flags_test(&vma_flags, VMA_LOCKED_BIT)) - mm->locked_vm += (len >> PAGE_SHIFT); + mm->locked_vm += len >> MM_ACCOUNT_SHIFT; if (pgtable_supports_soft_dirty()) vma_set_flags(vma, VMA_SOFTDIRTY_BIT); return 0; @@ -2948,7 +2989,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, mas_store_fail: vm_area_free(vma); unacct_fail: - vm_unacct_memory(len >> PAGE_SHIFT); + mm_unacct_memory(mm, len >> mm_page_shift(mm)); return -ENOMEM; } @@ -3084,7 +3125,7 @@ static int acct_stack_growth(struct vm_area_struct *vma, /* mlock limit tests */ if (!mlock_future_ok(mm, vma_test(vma, VMA_LOCKED_BIT), - grow << PAGE_SHIFT)) + grow << mm_page_shift(mm))) return -ENOMEM; /* Check to ensure the stack will not grow into a hugetlb-only region */ @@ -3100,7 +3141,7 @@ static int acct_stack_growth(struct vm_area_struct *vma, * Overcommit.. This must be the final test, as it will * update security statistics. */ - if (security_vm_enough_memory_mm(mm, grow)) + if (mm_account_memory(mm, mm, grow)) return -ENOMEM; return 0; @@ -3125,10 +3166,10 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) mmap_assert_write_locked(mm); /* Guard against exceeding limits of the address space. */ - address &= PAGE_MASK; - if (address >= (TASK_SIZE & PAGE_MASK)) + address &= mm_page_mask(mm); + if (address >= (TASK_SIZE & mm_page_mask(mm))) return -ENOMEM; - address += PAGE_SIZE; + address += mm_page_size(mm); /* Enforce stack_guard_gap */ gap_addr = address + stack_guard_gap; @@ -3167,14 +3208,14 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) unsigned long size, grow; size = address - vma->vm_start; - grow = (address - vma->vm_end) >> PAGE_SHIFT; + grow = (address - vma->vm_end) >> mm_page_shift(mm); error = -ENOMEM; if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) - mm->locked_vm += grow; + mm->locked_vm += mm_pages_to_account(mm, grow); vm_stat_account(mm, vma->vm_flags, grow); anon_vma_interval_tree_pre_update_vma(vma); vma->vm_end = address; @@ -3209,7 +3250,7 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) mmap_assert_write_locked(mm); - address &= PAGE_MASK; + address &= mm_page_mask(mm); if (address < mmap_min_addr || address < FIRST_USER_ADDRESS) return -EPERM; @@ -3244,20 +3285,22 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) /* Somebody else might have raced and expanded it already */ if (address < vma->vm_start) { unsigned long size, grow; + struct vm_page_offset pos = vm_page_offset_sub( + vma_page_offset_at(vma, vma->vm_start), vma->vm_start - address); size = vma->vm_end - address; - grow = (vma->vm_start - address) >> PAGE_SHIFT; + grow = (vma->vm_start - address) >> mm_page_shift(mm); error = -ENOMEM; - if (grow <= vma->vm_pgoff) { + if (pos.index <= vma->vm_pgoff) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) - mm->locked_vm += grow; + mm->locked_vm += mm_pages_to_account(mm, grow); vm_stat_account(mm, vma->vm_flags, grow); anon_vma_interval_tree_pre_update_vma(vma); vma->vm_start = address; - vma->vm_pgoff -= grow; + vma_set_page_offset(vma, pos); /* Overwrite old entry in mtree. */ vma_iter_store_overwrite(&vmi, vma); anon_vma_interval_tree_post_update_vma(vma); @@ -3296,13 +3339,13 @@ int __vm_munmap(unsigned long start, size_t len, bool unlock) */ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) { - unsigned long charged = vma_pages(vma); + unsigned long charged = vma_user_pages(vma); if (find_vma_intersection(mm, vma->vm_start, vma->vm_end)) return -ENOMEM; if (vma_test(vma, VMA_ACCOUNT_BIT) && - security_vm_enough_memory_mm(mm, charged)) + mm_account_memory(mm, mm, charged)) return -ENOMEM; /* @@ -3319,12 +3362,15 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) */ if (vma_is_anonymous(vma)) { BUG_ON(vma->anon_vma); - vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = vma->vm_start >> PAGE_SHIFT, + .offset = vma->vm_start & ~PAGE_MASK, + }); } if (vma_link(mm, vma)) { if (vma_test(vma, VMA_ACCOUNT_BIT)) - vm_unacct_memory(charged); + mm_unacct_memory(mm, charged); return -ENOMEM; } diff --git a/mm/vma.h b/mm/vma.h index 8e4b61a7304c6..3d9c8ee4be05d 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -97,6 +97,9 @@ struct vma_merge_struct { unsigned long start; unsigned long end; pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif union { /* Temporary while VMA flags are being converted. */ @@ -234,9 +237,34 @@ static inline bool vmg_nomem(struct vma_merge_struct *vmg) static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, unsigned long addr) { - return vma->vm_pgoff + PHYS_PFN(addr - vma->vm_start); + return vma_page_offset_at(vma, addr).index; +} + +static inline struct vm_page_offset vmg_page_offset(const struct vma_merge_struct *vmg) +{ + return (struct vm_page_offset) { + .index = vmg->pgoff, +#ifdef CONFIG_MM_SUBPAGE + .offset = vmg->subpage_offset, +#endif + }; +} + +static inline void vmg_set_page_offset(struct vma_merge_struct *vmg, struct vm_page_offset pos) +{ + vmg->pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vmg->subpage_offset = pos.offset; +#endif } +#ifdef CONFIG_MM_SUBPAGE +#define VMG_SUBPAGE_STATE(vma, start) \ + .subpage_offset = vma_page_offset_at(vma, start).offset, +#else +#define VMG_SUBPAGE_STATE(vma, start) +#endif + #define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_) \ struct vma_merge_struct name = { \ .mm = mm_, \ @@ -259,6 +287,7 @@ static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, .end = end_, \ .vm_flags = vma_->vm_flags, \ .pgoff = vma_pgoff_offset(vma_, start_), \ + VMG_SUBPAGE_STATE(vma_, start_) \ .file = vma_->vm_file, \ .anon_vma = vma_->anon_vma, \ .policy = vma_policy(vma_), \ @@ -276,7 +305,7 @@ void validate_mm(struct mm_struct *mm); __must_check int vma_expand(struct vma_merge_struct *vmg); __must_check int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff); + unsigned long start, unsigned long end, struct vm_page_offset pos); static inline int vma_iter_store_gfp(struct vma_iterator *vmi, struct vm_area_struct *vma, gfp_t gfp) @@ -311,6 +340,9 @@ static inline void compat_set_vma_from_desc(struct vm_area_struct *vma, /* Mutable fields. Populated with initial state. */ vma->vm_pgoff = desc->pgoff; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = desc->subpage_offset; +#endif if (desc->vm_file != vma->vm_file) vma_set_file(vma, desc->vm_file); vma->flags = desc->vma_flags; @@ -448,7 +480,7 @@ void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, struct vm_area_struct *vma); struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - unsigned long addr, unsigned long len, pgoff_t pgoff, + unsigned long addr, unsigned long len, struct vm_page_offset pos, bool *need_rmap_locks); struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma); @@ -460,7 +492,7 @@ int mm_take_all_locks(struct mm_struct *mm); void mm_drop_all_locks(struct mm_struct *mm); unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, + unsigned long len, vm_flags_t vm_flags, struct vm_page_offset pos, struct list_head *uf); int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, diff --git a/mm/vma_exec.c b/mm/vma_exec.c index 5cee8b7efa0f2..8394f45a91113 100644 --- a/mm/vma_exec.c +++ b/mm/vma_exec.c @@ -42,6 +42,8 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) struct mmu_gather tlb; PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length); + vmg_set_page_offset(&vmg, vma_page_offset_at(vma, old_start)); + BUG_ON(new_start > new_end); /* @@ -89,7 +91,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) vma_prev(&vmi); /* Shrink the vma to just the new range */ - return vma_shrink(&vmi, vma, new_start, new_end, vma->vm_pgoff); + return vma_shrink(&vmi, vma, new_start, new_end, vma_page_offset_at(vma, vma->vm_start)); } /* @@ -138,7 +140,7 @@ int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, */ VM_WARN_ON_ONCE(VM_STACK_FLAGS & VM_STACK_INCOMPLETE_SETUP); vma->vm_end = STACK_TOP_MAX; - vma->vm_start = vma->vm_end - PAGE_SIZE; + vma->vm_start = vma->vm_end - mm_page_size(mm); if (pgtable_supports_soft_dirty()) flags |= VM_SOFTDIRTY; vm_flags_init(vma, flags); diff --git a/mm/vma_init.c b/mm/vma_init.c index 3c0b65950510e..a41ba35de7320 100644 --- a/mm/vma_init.c +++ b/mm/vma_init.c @@ -46,7 +46,10 @@ static void vm_area_init_from(const struct vm_area_struct *src, dest->vm_start = src->vm_start; dest->vm_end = src->vm_end; dest->anon_vma = src->anon_vma; - dest->vm_pgoff = src->vm_pgoff; + vma_set_page_offset(dest, (struct vm_page_offset) { + .index = src->vm_pgoff, + .offset = vma_subpage_offset(src), + }); dest->vm_file = src->vm_file; dest->vm_private_data = src->vm_private_data; vm_flags_init(dest, src->vm_flags); diff --git a/mm/vmscan.c b/mm/vmscan.c index 1af6c96d1bc14..fe28f70fd51b1 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3462,6 +3462,46 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, } } +static unsigned long lru_pte_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_page_size(mm) == PAGE_SIZE) + return pte_index(addr); + return (addr >> mm_page_shift(mm)) & (pte_table_bytes_mm(mm) / sizeof(pte_t) - 1); +} + +static unsigned long lru_pmd_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_page_size(mm) == PAGE_SIZE) + return pmd_index(addr); + return (addr >> pmd_shift_mm(mm)) & (pmd_table_bytes_mm(mm) / sizeof(pmd_t) - 1); +} + +static unsigned long lru_pud_entries(struct mm_struct *mm) +{ + return mm_pud_folded(mm) ? 1 : pud_table_bytes_mm(mm) / sizeof(pud_t); +} + +static unsigned long lru_pud_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_pud_folded(mm)) + return 0; + if (mm_page_size(mm) == PAGE_SIZE) + return pud_index(addr); + return (addr >> pud_shift_mm(mm)) & (lru_pud_entries(mm) - 1); +} + +static unsigned int lru_folio_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t *entry, unsigned int max_nr) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch_flags(vma, folio, ptep, entry, + max_nr, FPB_MERGE_YOUNG_DIRTY); +#endif + return folio_pte_batch_flags(folio, NULL, ptep, entry, max_nr, + FPB_MERGE_YOUNG_DIRTY); +} + static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, struct mm_walk *args) { @@ -3481,7 +3521,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, unsigned int nr; pmd_t pmdval; - pte = pte_offset_map_rw_nolock(args->mm, pmd, start & PMD_MASK, &pmdval, &ptl); + pte = pte_offset_map_rw_nolock(args->mm, pmd, start & pmd_mask_mm(args->mm), &pmdval, &ptl); if (!pte) return false; @@ -3497,7 +3537,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, lazy_mmu_mode_enable(); restart: - for (i = pte_index(start), addr = start; addr != end; i += nr, addr += nr * PAGE_SIZE) { + for (i = lru_pte_index(args->mm, start), addr = start; addr != end; i += nr, addr += nr * mm_page_size(args->mm)) { unsigned long pfn; struct folio *folio; pte_t *cur_pte = pte + i; @@ -3516,10 +3556,9 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, continue; if (folio_test_large(folio)) { - const unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + const unsigned int max_nr = (end - addr) >> mm_page_shift(args->mm); - nr = folio_pte_batch_flags(folio, NULL, cur_pte, &ptent, - max_nr, FPB_MERGE_YOUNG_DIRTY); + nr = lru_folio_pte_batch(args->vma, folio, cur_pte, &ptent, max_nr); total += nr - 1; walk->mm_stats[MM_LEAF_TOTAL] += nr - 1; } @@ -3544,7 +3583,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, walk_update_folio(walk, last, gen, dirty); last = NULL; - if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) + if (i < pte_table_bytes_mm(args->mm) / sizeof(pte_t) && + get_next_vma(pmd_mask_mm(args->mm), mm_page_size(args->mm), args, &start, &end)) goto restart; lazy_mmu_mode_disable(); @@ -3576,13 +3616,13 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area return; } - i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first); + i = addr == -1 ? 0 : lru_pmd_index(args->mm, addr) - lru_pmd_index(args->mm, *first); if (i && i <= MIN_LRU_BATCH) { __set_bit(i - 1, bitmap); return; } - pmd = pmd_offset(pud, *first); + pmd = pmd_offset_mm(args->mm, pud, *first); ptl = pmd_lockptr(args->mm, pmd); if (!spin_trylock(ptl)) @@ -3595,7 +3635,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area struct folio *folio; /* don't round down the first address */ - addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first; + addr = i ? (*first & pmd_mask_mm(args->mm)) + i * pmd_size_mm(args->mm) : *first; if (!pmd_present(pmd[i])) goto next; @@ -3661,14 +3701,14 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, * tables to avoid taking the PMD lock; the second, if necessary, takes * the PMD lock to clear the accessed bit in PMD entries. */ - pmd = pmd_offset(pud, start & PUD_MASK); + pmd = pmd_offset_mm(args->mm, pud, start & pud_mask_mm(args->mm)); restart: /* walk_pte_range() may call get_next_vma() */ vma = args->vma; - for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { + for (i = lru_pmd_index(args->mm, start), addr = start; addr != end; i++, addr = next) { pmd_t val = pmdp_get_lockless(pmd + i); - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(args->mm, addr, end); if (!pmd_present(val) || is_huge_zero_pmd(val)) { walk->mm_stats[MM_LEAF_TOTAL]++; @@ -3710,7 +3750,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first); - if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) + if (i < pmd_table_bytes_mm(args->mm) / sizeof(pmd_t) && + get_next_vma(pud_mask_mm(args->mm), pmd_size_mm(args->mm), args, &start, &end)) goto restart; } @@ -3725,12 +3766,12 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, VM_WARN_ON_ONCE(p4d_leaf(*p4d)); - pud = pud_offset(p4d, start & P4D_MASK); + pud = pud_offset_mm(args->mm, p4d, start & p4d_mask_mm(args->mm)); restart: - for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { + for (i = lru_pud_index(args->mm, start), addr = start; addr != end; i++, addr = next) { pud_t val = pudp_get(pud + i); - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(args->mm, addr, end); if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) continue; @@ -3738,15 +3779,16 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, walk_pmd_range(&val, addr, next, args); if (need_resched() || walk->batched >= MAX_LRU_BATCH) { - end = (addr | ~PUD_MASK) + 1; + end = (addr | ~pud_mask_mm(args->mm)) + 1; goto done; } } - if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) + if (i < lru_pud_entries(args->mm) && + get_next_vma(p4d_mask_mm(args->mm), pud_size_mm(args->mm), args, &start, &end)) goto restart; - end = round_up(end, P4D_SIZE); + end = round_up(end, p4d_size_mm(args->mm)); done: if (!end || !args->vma) return 1; @@ -4215,20 +4257,20 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) /* avoid taking the LRU lock under the PTL when possible */ walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; - start = max(addr & PMD_MASK, vma->vm_start); - end = min(addr | ~PMD_MASK, vma->vm_end - 1) + 1; + start = max(addr & pmd_mask_mm(vma->vm_mm), vma->vm_start); + end = min(addr | ~pmd_mask_mm(vma->vm_mm), vma->vm_end - 1) + 1; - if (end - start == PAGE_SIZE) + if (end - start == mm_page_size(vma->vm_mm)) return true; - if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { - if (addr - start < MIN_LRU_BATCH * PAGE_SIZE / 2) - end = start + MIN_LRU_BATCH * PAGE_SIZE; - else if (end - addr < MIN_LRU_BATCH * PAGE_SIZE / 2) - start = end - MIN_LRU_BATCH * PAGE_SIZE; + if (end - start > MIN_LRU_BATCH * mm_page_size(vma->vm_mm)) { + if (addr - start < MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2) + end = start + MIN_LRU_BATCH * mm_page_size(vma->vm_mm); + else if (end - addr < MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2) + start = end - MIN_LRU_BATCH * mm_page_size(vma->vm_mm); else { - start = addr - MIN_LRU_BATCH * PAGE_SIZE / 2; - end = addr + MIN_LRU_BATCH * PAGE_SIZE / 2; + start = addr - MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2; + end = addr + MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2; } } @@ -4240,10 +4282,10 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) lazy_mmu_mode_enable(); - pte -= (addr - start) / PAGE_SIZE; + pte -= (addr - start) / mm_page_size(vma->vm_mm); for (i = 0, addr = start; addr != end; - i += nr, pte += nr, addr += nr * PAGE_SIZE) { + i += nr, pte += nr, addr += nr * mm_page_size(vma->vm_mm)) { unsigned long pfn; pte_t ptent = ptep_get(pte); @@ -4257,10 +4299,9 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) continue; if (folio_test_large(folio)) { - const unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + const unsigned int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); - nr = folio_pte_batch_flags(folio, NULL, pte, &ptent, - max_nr, FPB_MERGE_YOUNG_DIRTY); + nr = lru_folio_pte_batch(vma, folio, pte, &ptent, max_nr); } if (!test_and_clear_young_ptes_notify(vma, addr, pte, nr)) @@ -4634,6 +4675,16 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c sc->nr.unqueued_dirty += delta; } + /* + * Legacy memcg reclaim relies on shrink_folio_list() waiting for + * writeback when reclaim-marked folios cycle through the LRU. Moving + * all of them to a younger generation can exhaust the charge limit + * before I/O completes. Let the common path apply its GFP and mapping + * deadlock checks instead of hiding those folios from it. + */ + if (writeback && !writeback_throttling_sane(sc)) + return false; + /* waiting for writeback */ if (writeback || (type == LRU_GEN_FILE && dirty)) { gen = folio_inc_gen(lruvec, folio, true); @@ -4668,8 +4719,9 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca if (!folio_test_referenced(folio)) set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0); - /* for shrink_folio_list() */ - folio_clear_reclaim(folio); + /* Preserve legacy memcg's repeated-writeback marker for shrink_folio_list(). */ + if (writeback_throttling_sane(sc) || !folio_test_writeback(folio)) + folio_clear_reclaim(folio); success = lru_gen_del_folio(lruvec, folio, true); VM_WARN_ON_ONCE_FOLIO(!success, folio); From 1669c4b8ab87f9da420c863f1ea4209557f0cda6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0573/1579] mm: add retained subpage backing ownership Implement reference-counted user slots within native folios, including zeroing, mapping and pin ownership, migration and deferred lifetime. The backend remains disabled until its architecture integration. Signed-off-by: Diljit Singh (cherry picked from commit c1a5f4938d74e70de292adee44236d69f7b14a2b) --- mm/Makefile | 1 + mm/page_ext.c | 4 + mm/user-subpage.c | 1673 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1678 insertions(+) create mode 100644 mm/user-subpage.c diff --git a/mm/Makefile b/mm/Makefile index 8ad2ab08244eb..f531742956b8e 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -150,3 +150,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o obj-$(CONFIG_EXECMEM) += execmem.o obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o +obj-$(CONFIG_MM_SUBPAGE) += user-subpage.o diff --git a/mm/page_ext.c b/mm/page_ext.c index e2e92bd27ebdd..a73aab719c040 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include @@ -78,6 +79,9 @@ static struct page_ext_operations page_idle_ops __initdata = { #endif static struct page_ext_operations *page_ext_ops[] __initdata = { +#ifdef CONFIG_MM_SUBPAGE + &mm_subpage_ext_ops, +#endif #ifdef CONFIG_PAGE_OWNER &page_owner_ops, #endif diff --git a/mm/user-subpage.c b/mm/user-subpage.c new file mode 100644 index 0000000000000..133d75241485a --- /dev/null +++ b/mm/user-subpage.c @@ -0,0 +1,1673 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Reference-counted user data slots in native folios. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "user-subpage-internal.h" +#include "internal.h" + +#define SUBPAGES_PER_FOLIO (PAGE_SIZE / MM_SUBPAGE_SIZE) +static bool subpage_shift_valid(unsigned int shift) +{ + return shift >= MM_SUBPAGE_SHIFT && shift < PAGE_SHIFT; +} + +static unsigned long subpage_mask(unsigned int shift) +{ + return GENMASK((PAGE_SIZE >> shift) - 1, 0); +} + +static unsigned long offset_mask(unsigned int shift, unsigned int offset) +{ + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, 1U << shift)) + return 0; + return BIT(offset >> shift); +} + +struct mm_subpage_owner { + struct rcu_head rcu; + struct work_struct free_work; + struct mm_subpage_pool *pool; + struct folio *folio; + struct anon_vma *anon_root; + struct mm_subpage_owner *migration_target; + /* Allocated only when slots need independent logical identities. */ + struct mm_subpage_rmap *rmap; + struct list_head all; + struct list_head available; + unsigned long used; + unsigned long zeroed; + u8 shift; + bool detached; + bool migrating; + bool droppable; + struct mm_subpage slots[SUBPAGES_PER_FOLIO]; +}; + +struct mm_subpage_ext { + struct mm_subpage_owner __rcu *owner; + atomic_t file_maps[SUBPAGES_PER_FOLIO]; + atomic_t file_leaves; + /* Folio lock protects validity; fits the 16K/64K layout padding. */ + u16 shmem_missing; +}; +static_assert(SUBPAGES_PER_FOLIO == 1 || sizeof(struct mm_subpage_ext) == + ALIGN(sizeof(void *) + sizeof(atomic_t) * (SUBPAGES_PER_FOLIO + 1), + sizeof(void *))); +static_assert(SUBPAGES_PER_FOLIO <= sizeof(u16) * BITS_PER_BYTE); + +unsigned long mm_subpage_shmem_missing(struct page *page) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + unsigned long missing; + + /* A referenced native page keeps its initialized section online. */ + VM_BUG_ON(!ext); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + missing = subext->shmem_missing; + page_ext_put(ext); + return missing; +} + +void mm_subpage_shmem_set_missing(struct page *page, unsigned long missing) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + + VM_BUG_ON(!ext); + VM_BUG_ON(missing & ~GENMASK(SUBPAGES_PER_FOLIO - 1, 0)); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + subext->shmem_missing = missing; + page_ext_put(ext); +} + +static bool __init subpage_ext_needed(void) +{ + return true; +} + +struct page_ext_operations mm_subpage_ext_ops = { + .size = sizeof(struct mm_subpage_ext), + .need = subpage_ext_needed, +}; + +/* Only alternative PTE mappings contribute; native rmap counts stay unchanged. */ +static void subpage_file_map_update(phys_addr_t phys, unsigned long size, int delta) +{ + struct page_ext *ext; + struct mm_subpage_ext *subext; + unsigned int index, nr, i; + int count; + + VM_BUG_ON(!is_power_of_2(size) || size < MM_SUBPAGE_SIZE || + size >= PAGE_SIZE || !IS_ALIGNED(phys, size)); + ext = page_ext_from_phys(phys); + if (WARN_ON_ONCE(!ext)) + return; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + index = offset_in_page(phys) >> MM_SUBPAGE_SHIFT; + nr = size >> MM_SUBPAGE_SHIFT; + for (i = index; i < index + nr; i++) { + count = atomic_add_return(delta, &subext->file_maps[i]); + VM_WARN_ON(count < 0); + } + /* Rmap counts one mapping, regardless of its coverage in 4K units. */ + count = atomic_add_return(delta, &subext->file_leaves); + VM_WARN_ON(count < 0); + page_ext_put(ext); +} + +void __mm_subpage_file_map_add(phys_addr_t phys, unsigned long size) +{ + subpage_file_map_update(phys, size, 1); +} + +void __mm_subpage_file_map_del(phys_addr_t phys, unsigned long size) +{ + subpage_file_map_update(phys, size, -1); +} + +/* Snapshot for statistics only, never an ownership or pinning decision. */ +unsigned int mm_subpage_file_mapcounts(struct page *page, struct mm_subpage_mapcounts *counts) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + unsigned int i, leaves; + + if (WARN_ON_ONCE(!ext)) { + memset(counts, 0, sizeof(*counts)); + return 0; + } + subext = page_ext_data(ext, &mm_subpage_ext_ops); + for (i = 0; i < SUBPAGES_PER_FOLIO; i++) + counts->slots[i] = max(atomic_read(&subext->file_maps[i]), 0); + leaves = max(atomic_read(&subext->file_leaves), 0); + page_ext_put(ext); + return leaves; +} + +static int owner_bind(struct mm_subpage_owner *owner) +{ + struct page_ext *ext = page_ext_get(&owner->folio->page); + struct mm_subpage_ext *subext; + int ret; + + if (!ext) + return -EOPNOTSUPP; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + ret = cmpxchg(&subext->owner, NULL, owner) ? -EEXIST : 0; + page_ext_put(ext); + return ret; +} + +static void owner_unbind(struct mm_subpage_owner *owner) +{ + struct page_ext *ext = page_ext_get(&owner->folio->page); + struct mm_subpage_ext *subext; + + /* The allocated, referenced folio prevents its section going offline. */ + VM_BUG_ON(!ext); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + VM_BUG_ON(rcu_access_pointer(subext->owner) != owner); + rcu_assign_pointer(subext->owner, NULL); + page_ext_put(ext); +} + +struct mm_subpage *mm_subpage_get_from_phys(phys_addr_t phys) +{ + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + struct mm_subpage *slot = NULL; + struct page_ext *ext; + unsigned int index; + + if (!IS_ALIGNED(phys, MM_SUBPAGE_SIZE)) + return NULL; + ext = page_ext_from_phys(phys); + if (!ext) + return NULL; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + index = owner ? offset_in_page(phys) >> owner->shift : 0; + /* Keep the native allocation alive before publishing another slot ref. */ + if (owner && folio_try_get(owner->folio)) { + if (refcount_inc_not_zero_acquire(&owner->slots[index].refs)) + slot = &owner->slots[index]; + else + folio_put(owner->folio); + } + page_ext_put(ext); + return slot; +} + +struct mm_subpage_pool { + struct rcu_head rcu; + spinlock_t lock; + refcount_t refs; + struct list_head owners; + struct list_head available; + struct mm_subpage_owner *unissued; + unsigned long backing_pages; + unsigned long all_slots; + u8 shift; + bool closed; +}; + +static void owner_init(struct mm_subpage_owner *owner, struct mm_subpage_pool *pool, + struct folio *folio) +{ + unsigned int i; + + owner->pool = pool; + owner->shift = pool->shift; + owner->folio = folio; + INIT_LIST_HEAD(&owner->available); + for (i = 0; i < (PAGE_SIZE >> pool->shift); i++) { + owner->slots[i].owner = owner; + owner->slots[i].index = i; + owner->slots[i].shift = pool->shift; + refcount_set(&owner->slots[i].refs, 0); + atomic_set(&owner->slots[i].mapcount, 0); + } +} + +struct mm_subpage_pool *mm_subpage_pool_create_granule(gfp_t gfp, unsigned int shift) +{ + struct mm_subpage_pool *pool; + + BUILD_BUG_ON(PAGE_SHIFT <= MM_SUBPAGE_SHIFT); + BUILD_BUG_ON(SUBPAGES_PER_FOLIO > BITS_PER_LONG); + if (!subpage_shift_valid(shift)) + return NULL; + pool = kzalloc(sizeof(*pool), gfp | __GFP_ACCOUNT); + if (!pool) + return NULL; + pool->shift = shift; + pool->all_slots = subpage_mask(shift); + spin_lock_init(&pool->lock); + refcount_set(&pool->refs, 1); + INIT_LIST_HEAD(&pool->owners); + INIT_LIST_HEAD(&pool->available); + return pool; +} + +struct mm_subpage_pool *mm_subpage_pool_create(gfp_t gfp) +{ + return mm_subpage_pool_create_granule(gfp, MM_SUBPAGE_SHIFT); +} + +void mm_subpage_pool_get(struct mm_subpage_pool *pool) +{ + refcount_inc(&pool->refs); +} + +void mm_subpage_pool_put(struct mm_subpage_pool *pool) +{ + if (!refcount_dec_and_test(&pool->refs)) + return; + WARN_ON_ONCE(!pool->closed || !list_empty(&pool->owners)); + kfree_rcu(pool, rcu); +} + +static void owner_remove(struct mm_subpage_owner *owner) +{ + lockdep_assert_held(&owner->pool->lock); + owner->detached = true; + list_del(&owner->all); + list_del_init(&owner->available); + if (owner->pool->unissued == owner) + owner->pool->unissued = NULL; + owner->pool->backing_pages--; +} + +static void owner_free_rmap(struct mm_subpage_owner *owner) +{ + unsigned int i; + + if (!owner->rmap) + return; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (owner->rmap[i].root) + put_anon_vma(owner->rmap[i].root); + kfree(owner->rmap); +} + +static void owner_free_metadata(struct mm_subpage_owner *owner) +{ + struct mm_subpage_pool *pool = owner->pool; + + if (owner->anon_root) + put_anon_vma(owner->anon_root); + owner_free_rmap(owner); + kfree_rcu(owner, rcu); + mm_subpage_pool_put(pool); +} + +static void owner_free_work(struct work_struct *work) +{ + owner_free_metadata(container_of(work, struct mm_subpage_owner, free_work)); +} + +static void owner_free(struct mm_subpage_owner *owner, bool put_folio) +{ + owner_unbind(owner); + if (put_folio) { + if (owner->anon_root) + free_swap_cache(owner->folio); + /* Last slot reference, or wholly unissued supply. */ + folio_put(owner->folio); + } + if (owner->rmap) { + /* Historical roots can have no live VMA; their final put may sleep. */ + INIT_WORK(&owner->free_work, owner_free_work); + queue_work(system_unbound_wq, &owner->free_work); + } else { + owner_free_metadata(owner); + } +} + +struct anon_vma *mm_subpage_anon_root(const struct mm_subpage *subpage) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + return subpage->owner->anon_root; +} + +void mm_subpage_hold_anon_root(struct mm_subpage *subpage, struct anon_vma *root, + bool droppable) +{ + struct mm_subpage_owner *owner = subpage->owner; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + VM_BUG_ON(owner->anon_root || root != root->root); + get_anon_vma(root); + owner->droppable = droppable; + owner->anon_root = root; +} + +static struct mm_subpage_rmap slot_rmap(struct mm_subpage *slot) +{ + struct mm_subpage_owner *owner = slot->owner; + + lockdep_assert_held(&owner->pool->lock); + if (owner->rmap) + return owner->rmap[slot->index]; + return (struct mm_subpage_rmap) { + .root = owner->anon_root, + .pos = { .index = owner->folio->index, + .offset = mm_subpage_offset(slot) }, + .offset = mm_subpage_offset(slot), + }; +} + +static bool rmap_matches(struct mm_subpage_rmap map, struct vm_area_struct *vma, + unsigned long address) +{ + return map.root == vma->anon_vma->root && + vm_page_offset_equal(map.pos, vma_page_offset_at(vma, address)); +} + +bool mm_subpage_match_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address) +{ + struct mm_subpage_pool *pool = slot->owner->pool; + unsigned long flags; + bool matches; + + spin_lock_irqsave(&pool->lock, flags); + matches = slot->rmap_bound && rmap_matches(slot_rmap(slot), vma, address); + spin_unlock_irqrestore(&pool->lock, flags); + return matches; +} + +static void *owner_kcalloc(struct mm_subpage_owner *owner, size_t nr, size_t size) +{ + struct mem_cgroup *memcg, *saved; + void *maps; + + rcu_read_lock(); + do { + memcg = mem_cgroup_from_virt(owner) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + maps = kcalloc(nr, size, GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + set_active_memcg(saved); + mem_cgroup_put(memcg); + return maps; +} + +struct subpage_retired_root { + struct work_struct work; + struct anon_vma *root; +}; + +static void subpage_put_retired_root(struct work_struct *work) +{ + struct subpage_retired_root *retired = + container_of(work, struct subpage_retired_root, work); + + put_anon_vma(retired->root); + kfree(retired); +} + +/* The folio lock serializes binding and sidecar creation against migration. */ +int mm_subpage_bind_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address) +{ + struct mm_subpage_owner *owner = slot->owner; + struct mm_subpage_pool *pool = owner->pool; + struct mm_subpage_rmap *maps = NULL; + struct subpage_retired_root *retired = NULL; + unsigned long flags; + unsigned int i; + int ret = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + spin_lock_irqsave(&pool->lock, flags); + if (slot->rmap_bound) { + ret = rmap_matches(slot_rmap(slot), vma, address) ? 0 : -EINVAL; + goto unlock; + } + if (!owner->rmap && !rmap_matches(slot_rmap(slot), vma, address)) { + spin_unlock_irqrestore(&pool->lock, flags); + /* The caller may also hold a PTE lock. Never enter reclaim here. */ + maps = owner_kcalloc(owner, PAGE_SIZE >> owner->shift, sizeof(*maps)); + if (!maps) + return -ENOMEM; + spin_lock_irqsave(&pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + maps[i].offset = i << owner->shift; + if (!owner->slots[i].rmap_bound) + continue; + maps[i] = slot_rmap(&owner->slots[i]); + get_anon_vma(maps[i].root); + } + owner->rmap = maps; + } + if (owner->rmap) { + struct mm_subpage_rmap *map = &owner->rmap[slot->index]; + + if (map->root && map->root != vma->anon_vma->root) { + /* A final anon_vma put may sleep; the caller can hold a PTL. */ + spin_unlock_irqrestore(&pool->lock, flags); + retired = owner_kcalloc(owner, 1, sizeof(*retired)); + if (!retired) + return -ENOMEM; + INIT_WORK(&retired->work, subpage_put_retired_root); + spin_lock_irqsave(&pool->lock, flags); + retired->root = map->root; + } + if (map->root != vma->anon_vma->root) { + get_anon_vma(vma->anon_vma->root); + map->root = vma->anon_vma->root; + } + map->pos = vma_page_offset_at(vma, address); + map->offset = mm_subpage_offset(slot); + } + slot->rmap_bound = true; +unlock: + spin_unlock_irqrestore(&pool->lock, flags); + if (retired) + queue_work(system_unbound_wq, &retired->work); + return ret; +} + +static bool subpage_move_allowed(struct mm_subpage *slot, bool swap) +{ + lockdep_assert_held(&slot->owner->pool->lock); + return !slot->owner->migrating && !slot->pins && + atomic_read(&slot->mapcount) == (swap ? 0 : 1) && + (swap || (slot->rmap_bound && slot->exclusive)); +} + +int mm_subpage_move_prepare(struct mm_subpage_move *move, struct mm_subpage *slot, + struct vm_area_struct *src, unsigned long src_address, + struct vm_area_struct *dst, unsigned long dst_address, bool swap) +{ + struct mm_subpage_owner *owner = slot->owner; + struct mm_subpage_rmap *maps, old; + unsigned long flags; + unsigned int i; + int ret = -EBUSY; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + memset(move, 0, sizeof(*move)); + if (!src->anon_vma || !dst->anon_vma || !owner->anon_root || + mm_page_shift(src->vm_mm) != slot->shift || + mm_page_shift(dst->vm_mm) != slot->shift || + owner->droppable != !!(dst->vm_flags & VM_DROPPABLE)) + return -EINVAL; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!subpage_move_allowed(slot, swap) || + (slot->rmap_bound && !rmap_matches(slot_rmap(slot), src, src_address))) + goto unlock; + spin_unlock_irqrestore(&owner->pool->lock, flags); + + /* Allocate before touching either PTE. The folio lock serializes writers. */ + if (!owner->rmap) { + maps = owner_kcalloc(owner, PAGE_SIZE >> owner->shift, sizeof(*maps)); + if (!maps) + return -ENOMEM; + spin_lock_irqsave(&owner->pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + maps[i].offset = i << owner->shift; + if (!owner->slots[i].rmap_bound) + continue; + maps[i] = slot_rmap(&owner->slots[i]); + get_anon_vma(maps[i].root); + } + owner->rmap = maps; + spin_unlock_irqrestore(&owner->pool->lock, flags); + } + old = owner->rmap[slot->index]; + move->root_changed = old.root != dst->anon_vma->root; + if (move->root_changed && old.root) { + move->retired = owner_kcalloc(owner, 1, sizeof(*move->retired)); + if (!move->retired) + return -ENOMEM; + INIT_WORK(&move->retired->work, subpage_put_retired_root); + move->retired->root = old.root; + } + if (move->root_changed) + get_anon_vma(dst->anon_vma->root); + move->slot = slot; + move->swap = swap; + move->map = (struct mm_subpage_rmap) { + .root = dst->anon_vma->root, + .pos = vma_page_offset_at(dst, dst_address), + .offset = mm_subpage_offset(slot), + }; + return 0; +unlock: + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +int mm_subpage_move_commit(struct mm_subpage_move *move) +{ + struct mm_subpage *slot = move->slot; + struct mm_subpage_owner *owner = slot->owner; + unsigned long flags; + int ret = -EBUSY; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + spin_lock_irqsave(&owner->pool->lock, flags); + /* Recheck pins after source-PTE clearing and the matching TLB barrier. */ + if (subpage_move_allowed(slot, move->swap)) { + owner->rmap[slot->index] = move->map; + slot->rmap_bound = true; + move->committed = true; + ret = 0; + } + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +void mm_subpage_move_finish(struct mm_subpage_move *move) +{ + if (move->committed) { + if (move->retired) + queue_work(system_unbound_wq, &move->retired->work); + } else { + /* The locked destination VMA still holds this root. */ + if (move->root_changed) + put_anon_vma(move->map.root); + kfree(move->retired); + } +} + +int mm_subpage_snapshot_rmap(struct folio *folio, struct mm_subpage_rmap *maps) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner; + unsigned long flags; + unsigned int i; + int nr = -1; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return nr; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->detached && owner->rmap) { + nr = 0; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + if (!owner->slots[i].rmap_bound) + continue; + maps[nr] = owner->rmap[i]; + get_anon_vma(maps[nr].root); + nr++; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); +out: + page_ext_put(ext); + return nr; +} + +/* A referenced folio plus a stable VMA suffices; no folio lock required. */ +bool mm_subpage_address_in_vma(const struct folio *folio, + const struct vm_area_struct *vma, unsigned long *address) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner; + unsigned long flags; + unsigned int i; + bool nonlinear = false; + + if (!ext) + return false; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->detached && owner->rmap) { + nonlinear = true; + *address = -EFAULT; + for (i = 0; vma->anon_vma && i < (PAGE_SIZE >> owner->shift); i++) { + if (!owner->slots[i].rmap_bound || + owner->rmap[i].root != vma->anon_vma->root) + continue; + *address = vma_address_at_offset(vma, owner->rmap[i].pos); + if (*address != -EFAULT) + break; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); +out: + page_ext_put(ext); + return nonlinear; +} + +bool mm_subpage_is_droppable(const struct mm_subpage *subpage) +{ + return subpage->owner->droppable; +} + +void mm_subpage_pool_close(struct mm_subpage_pool *pool) +{ + struct mm_subpage_owner *owner; + unsigned long flags; + + spin_lock_irqsave(&pool->lock, flags); + pool->closed = true; + /* At most one newly supplied owner has no issued slots. Close is O(1). */ + owner = pool->unissued; + if (owner) + owner_remove(owner); + spin_unlock_irqrestore(&pool->lock, flags); + if (owner) + owner_free(owner, true); +} + +static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, + unsigned long eligible) +{ + struct mm_subpage_owner *owner; + + lockdep_assert_held(&pool->lock); + list_for_each_entry(owner, &pool->available, available) + /* A nonpresent PTE may still need any of this folio's slot bytes. */ + if (!owner->migrating && (~owner->used & eligible) && + !folio_test_swapcache(owner->folio)) + return owner; + return NULL; +} + +static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + unsigned long eligible, gfp_t gfp) +{ + struct mm_subpage_owner *owner; + unsigned long flags; + int ret = 0; + + /* Only an unexposed order-0 allocation can become a slot owner. */ + if (folio_order(folio) || folio_ref_count(folio) != 1 || + folio->mapping || folio_mapped(folio) || folio_test_lru(folio) || + folio_test_private(folio) || folio_test_swapcache(folio)) + return -EINVAL; + owner = kzalloc(sizeof(*owner), gfp | __GFP_ACCOUNT); + if (!owner) + return -ENOMEM; + owner_init(owner, pool, folio); + /* Reject duplicate ownership before touching any existing data. */ + ret = owner_bind(owner); + if (ret) { + kfree(owner); + return ret; + } + /* Initialise even unissued bytes before any future whole-folio operation. */ + clear_user_subpage_range(&folio->page, 0, PAGE_SIZE); + __folio_mark_uptodate(folio); + owner->zeroed = pool->all_slots; + spin_lock_irqsave(&pool->lock, flags); + if (pool->closed) + ret = -ESHUTDOWN; + else if (find_available(pool, eligible)) + /* Another allocator supplied backing while we allocated metadata. */ + ret = -EEXIST; + else { + mm_subpage_pool_get(pool); + list_add_tail(&owner->all, &pool->owners); + list_add_tail(&owner->available, &pool->available); + pool->backing_pages++; + pool->unissued = owner; + } + spin_unlock_irqrestore(&pool->lock, flags); + if (ret) { + owner_unbind(owner); + kfree_rcu(owner, rcu); + } + return ret; +} + +int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + gfp_t gfp) +{ + return pool_add_folio(pool, folio, pool->all_slots, gfp); +} + +int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int offset, gfp_t gfp) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return -EINVAL; + return pool_add_folio(pool, folio, eligible, gfp); +} + +unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) +{ + unsigned long pages, flags; + + spin_lock_irqsave(&pool->lock, flags); + pages = pool->backing_pages; + spin_unlock_irqrestore(&pool->lock, flags); + return pages; +} + +static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, + unsigned long eligible, bool *needs_zero) +{ + struct mm_subpage_owner *owner; + struct mm_subpage *subpage; + unsigned long flags; + unsigned int index; + + spin_lock_irqsave(&pool->lock, flags); + if (pool->closed) { + subpage = ERR_PTR(-ESHUTDOWN); + goto out; + } + owner = find_available(pool, eligible); + if (!owner) { + subpage = ERR_PTR(-EAGAIN); + goto out; + } + if (pool->unissued == owner) + /* Transfer the supplied reference to the first reservation. */ + pool->unissued = NULL; + else + folio_get(owner->folio); + index = __ffs(~owner->used & eligible); + __set_bit(index, &owner->used); + if (needs_zero) + *needs_zero = !test_bit(index, &owner->zeroed); + __clear_bit(index, &owner->zeroed); + if (owner->used == pool->all_slots) + list_del_init(&owner->available); + subpage = &owner->slots[index]; + WRITE_ONCE(subpage->lazyfree, false); + /* The used bit keeps backing alive while data is initialised. */ + VM_BUG_ON(refcount_read(&subpage->refs)); + VM_BUG_ON(atomic_read(&subpage->mapcount)); +out: + spin_unlock_irqrestore(&pool->lock, flags); + return subpage; +} + +static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible) +{ + bool needs_zero; + struct mm_subpage *subpage = subpage_reserve(pool, eligible, &needs_zero); + + if (IS_ERR(subpage)) + return subpage; + /* Fresh supply was already zeroed as a whole; reused slots must be cleared. */ + if (needs_zero) + clear_user_subpage_range(&subpage->owner->folio->page, + mm_subpage_offset(subpage), mm_subpage_size(subpage)); + /* Data stores must precede publication in a PTE or another CPU's reference. */ + refcount_set_release(&subpage->refs, 1); + return subpage; +} + +struct mm_subpage *mm_subpage_alloc(struct mm_subpage_pool *pool) +{ + return subpage_alloc(pool, pool->all_slots); +} + +struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return ERR_PTR(-EINVAL); + return subpage_alloc(pool, eligible); +} + +static void subpage_release_locked(struct mm_subpage *subpage, unsigned long flags) +{ + struct mm_subpage_owner *owner = subpage->owner; + struct mm_subpage_pool *pool = owner->pool; + struct folio *folio = owner->folio; + bool release; + + lockdep_assert_held(&pool->lock); + VM_BUG_ON(refcount_read(&subpage->refs)); + VM_BUG_ON(atomic_read(&subpage->mapcount)); + VM_BUG_ON(subpage->pins); + WRITE_ONCE(subpage->exclusive, false); + if (!owner->migrating) { + WRITE_ONCE(subpage->lazyfree, false); + subpage->rmap_bound = false; + } + __clear_bit(subpage->index, &owner->used); + release = !owner->used && !owner->migrating; + if (release) + owner_remove(owner); + else if (!pool->closed && !owner->migrating && list_empty(&owner->available)) + list_add(&owner->available, &pool->available); + spin_unlock_irqrestore(&pool->lock, flags); + if (release) + owner_free(owner, true); + else + folio_put(folio); +} + +static void subpage_release(struct mm_subpage *subpage) +{ + unsigned long flags; + + spin_lock_irqsave(&subpage->owner->pool->lock, flags); + subpage_release_locked(subpage, flags); +} + +static struct mm_subpage *subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned long eligible) +{ + struct mm_subpage *dest; + int err; + + if (pool->shift != mm_subpage_shift(source)) + return ERR_PTR(-EINVAL); + dest = subpage_reserve(pool, eligible, NULL); + + if (IS_ERR(dest)) + return dest; + err = copy_user_subpage_range(&dest->owner->folio->page, mm_subpage_offset(dest), + &source->owner->folio->page, mm_subpage_offset(source), + mm_subpage_size(source)); + if (err) { + /* Never publish a reservation whose copy did not finish. */ + subpage_release(dest); + return ERR_PTR(err); + } + refcount_set_release(&dest->refs, 1); + return dest; +} + +struct mm_subpage *mm_subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source) +{ + return subpage_copy(pool, source, pool->all_slots); +} + +struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return ERR_PTR(-EINVAL); + return subpage_copy(pool, source, eligible); +} + +/* The page-extension RCU section also protects the pool until its lock is held. */ +static struct mm_subpage *restore_bound_slot(struct folio *folio, unsigned int offset, + unsigned int shift) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_pool *pool; + struct mm_subpage_ext *subext; + struct mm_subpage *slot = NULL; + unsigned int index; + unsigned long flags; + + if (!ext) + return ERR_PTR(-EOPNOTSUPP); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + if ((shift && shift != owner->shift) || !offset_mask(owner->shift, offset)) { + slot = ERR_PTR(-EINVAL); + goto out; + } + index = offset >> owner->shift; + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->detached) { + slot = ERR_PTR(-EAGAIN); + goto unlock; + } + slot = &owner->slots[index]; + if (test_bit(index, &owner->used)) { + folio_get(folio); + if (!refcount_inc_not_zero_acquire(&slot->refs)) { + /* Its final put or initial publication has not finished. */ + folio_put(folio); + slot = ERR_PTR(-EAGAIN); + } + goto unlock; + } + VM_BUG_ON(refcount_read(&slot->refs) || atomic_read(&slot->mapcount)); + if (pool->unissued == owner) + pool->unissued = NULL; + else + folio_get(folio); + __set_bit(index, &owner->used); + __clear_bit(index, &owner->zeroed); + if (owner->used == pool->all_slots) + list_del_init(&owner->available); + /* The folio lock and uptodate state stabilise the restored bytes. */ + refcount_set_release(&slot->refs, 1); +unlock: + spin_unlock_irqrestore(&pool->lock, flags); +out: + page_ext_put(ext); + return slot; +} + +struct mm_subpage *mm_subpage_restore_granule(struct folio *folio, unsigned int offset, + unsigned int shift, gfp_t gfp) +{ + struct mm_subpage_pool *pool; + struct mm_subpage_owner *owner; + struct mm_subpage *slot; + unsigned int index; + int err; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!subpage_shift_valid(shift) || !offset_mask(shift, offset) || folio_order(folio) || !folio_test_swapcache(folio) || + !folio_test_uptodate(folio) || folio_test_ksm(folio) || + (folio->mapping && !folio_test_anon(folio)) || + (folio_test_anon(folio) && PageAnonExclusive(&folio->page))) + return ERR_PTR(-EINVAL); + index = offset >> shift; + slot = restore_bound_slot(folio, offset, shift); + if (slot) + return slot; + + /* A reclaimed folio has no surviving owner; do not initialise its data. */ + pool = mm_subpage_pool_create_granule(gfp, shift); + if (!pool) + return ERR_PTR(-ENOMEM); + pool->closed = true; + owner = kzalloc(sizeof(*owner), gfp | __GFP_ACCOUNT); + if (!owner) { + mm_subpage_pool_put(pool); + return ERR_PTR(-ENOMEM); + } + owner_init(owner, pool, folio); + owner->used = BIT(index); + refcount_set(&owner->slots[index].refs, 1); + list_add(&owner->all, &pool->owners); + pool->backing_pages = 1; + folio_get(folio); + err = owner_bind(owner); + if (err) { + folio_put(folio); + list_del(&owner->all); + pool->backing_pages = 0; + kfree(owner); + mm_subpage_pool_put(pool); + return ERR_PTR(err == -EEXIST ? -EAGAIN : err); + } + /* The private closed pool's initial reference belongs to this owner. */ + return &owner->slots[index]; +} + +struct mm_subpage *mm_subpage_restore(struct folio *folio, unsigned int offset, gfp_t gfp) +{ + return mm_subpage_restore_granule(folio, offset, MM_SUBPAGE_SHIFT, gfp); +} + +/* Preserve the metadata charge when migration runs outside its owning task. */ +static struct mm_subpage_owner *owner_alloc_migration(struct mm_subpage_owner *owner) +{ + struct mem_cgroup *memcg, *saved; + struct mm_subpage_owner *new; + + rcu_read_lock(); + do { + memcg = mem_cgroup_from_virt(owner) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + /* Folio locks are held: do not recurse into reclaim or trigger memcg OOM. */ + new = kzalloc(sizeof(*new), GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + if (new && owner->rmap) { + new->rmap = kcalloc(PAGE_SIZE >> owner->shift, sizeof(*new->rmap), + GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + if (!new->rmap) { + kfree(new); + new = NULL; + } + } + set_active_memcg(saved); + mem_cgroup_put(memcg); + return new; +} + +/* A migration hold pins metadata, not the native folio: the driver pins it. */ +int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) +{ + struct page_ext *ext; + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner, *new; + struct mm_subpage_pool *pool; + unsigned long flags; + int ret = 0; + unsigned int i; + + VM_BUG_ON_FOLIO(!folio_test_locked(src), src); + VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst); + ext = page_ext_get(&src->page); + if (!ext) + return 0; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) { + page_ext_put(ext); + return 0; + } + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->detached) { + owner = NULL; + } else if (src == dst || folio_order(src) || folio_order(dst) || + owner->migrating || pool->unissued == owner) { + ret = -EBUSY; + } else { + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (owner->slots[i].pins) + ret = -EBUSY; + if (!ret) + owner->migrating = true; + } + spin_unlock_irqrestore(&pool->lock, flags); + page_ext_put(ext); + if (!owner || ret) + return ret; + + /* Async migration must not recurse into reclaim while holding folio locks. */ + new = owner_alloc_migration(owner); + if (!new) { + ret = -ENOMEM; + goto undo; + } + owner_init(new, pool, dst); + new->migrating = true; + new->zeroed = owner->zeroed; + spin_lock_irqsave(&pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + new->slots[i].rmap_bound = owner->slots[i].rmap_bound; + if (new->rmap) { + new->rmap[i] = owner->rmap[i]; + if (new->rmap[i].root) + get_anon_vma(new->rmap[i].root); + } + } + spin_unlock_irqrestore(&pool->lock, flags); + + new->droppable = owner->droppable; + new->anon_root = owner->anon_root; + if (new->anon_root) + get_anon_vma(new->anon_root); + mm_subpage_pool_get(pool); + spin_lock_irqsave(&pool->lock, flags); + list_add_tail(&new->all, &pool->owners); + pool->backing_pages++; + ret = owner_bind(new); + if (ret) + owner_remove(new); + else + owner->migration_target = new; + spin_unlock_irqrestore(&pool->lock, flags); + if (!ret) + return 0; + if (new->anon_root) + put_anon_vma(new->anon_root); + owner_free_rmap(new); + kfree(new); + mm_subpage_pool_put(pool); +undo: + mm_subpage_migrate_finish(src); + return ret; +} + +bool mm_subpage_migrating(struct folio *folio) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + bool active = false; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return false; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (owner) + active = READ_ONCE(owner->migrating); + page_ext_put(ext); + return active; +} + +/* Source folio/PTE locked: writable GETs cannot race this final snapshot. */ +void mm_subpage_migrate_save_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + + VM_BUG_ON_FOLIO(!owner->migrating || !owner->migration_target, owner->folio); + WRITE_ONCE(owner->migration_target->slots[subpage->index].lazyfree, + READ_ONCE(subpage->lazyfree)); +} + +struct mm_subpage *mm_subpage_migrate_restore(struct folio *folio, unsigned int offset) +{ + struct mm_subpage *slot; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(!offset_mask(MM_SUBPAGE_SHIFT, offset) || !mm_subpage_migrating(folio), folio); + /* No allocation or data initialisation can fail during migration recovery. */ + slot = restore_bound_slot(folio, offset, 0); + VM_BUG_ON_FOLIO(IS_ERR_OR_NULL(slot), folio); + return slot; +} + +void mm_subpage_migrate_finish(struct folio *folio) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + struct mm_subpage_pool *pool; + unsigned long flags; + unsigned int i; + bool release = false; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->migrating) { + VM_BUG_ON(owner->detached); + owner->migrating = false; + owner->migration_target = NULL; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (!(owner->used & BIT(i))) + owner->slots[i].rmap_bound = false; + release = !owner->used; + if (release) + owner_remove(owner); + else if (!pool->closed && owner->used != pool->all_slots && + list_empty(&owner->available)) + list_add(&owner->available, &pool->available); + } + spin_unlock_irqrestore(&pool->lock, flags); +out: + page_ext_put(ext); + if (release) + /* No slot references remain; only the migration driver pins backing. */ + owner_free(owner, false); +} + +void mm_subpage_get(struct mm_subpage *subpage) +{ + folio_get(subpage->owner->folio); + refcount_inc(&subpage->refs); +} + +void mm_subpage_put(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + struct mm_subpage_pool *pool = owner->pool; + struct folio *folio = owner->folio; + unsigned long flags; + + /* + * Snapshot the backing while our slot reference still retains owner. + * A non-final decrement can race the final put and metadata reclamation; + * only our physical folio reference survives that decrement. + */ + if (refcount_dec_and_lock_irqsave(&subpage->refs, &pool->lock, &flags)) + subpage_release_locked(subpage, flags); + else + folio_put(folio); +} + +void mm_subpage_set_exclusive(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + VM_BUG_ON(atomic_read(&subpage->mapcount) > 1 || !refcount_read(&subpage->refs)); + WRITE_ONCE(subpage->exclusive, true); + spin_unlock_irqrestore(&pool->lock, flags); +} + +bool mm_subpage_is_exclusive(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->exclusive); +} + +static bool subpage_try_reuse(struct mm_subpage *subpage, unsigned int references) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool reusable; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + reusable = atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == references && !subpage->pins; + if (reusable) + WRITE_ONCE(subpage->exclusive, true); + spin_unlock_irqrestore(&pool->lock, flags); + return reusable; +} + +bool mm_subpage_try_reuse_swap(struct mm_subpage *subpage) +{ + return subpage_try_reuse(subpage, 1); +} + +bool mm_subpage_try_reuse_anon(struct mm_subpage *subpage) +{ + return subpage_try_reuse(subpage, 2); +} + +bool mm_subpage_can_zap(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + spin_lock_irqsave(&pool->lock, flags); + eligible = !subpage->owner->migrating && !subpage->pins && + atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == 2; + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +bool mm_subpage_try_share(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool shared; + + spin_lock_irqsave(&pool->lock, flags); + shared = !subpage->pins; + if (shared) + WRITE_ONCE(subpage->exclusive, false); + spin_unlock_irqrestore(&pool->lock, flags); + return shared; +} + +bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->lazyfree); +} + +void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage) +{ + WRITE_ONCE(subpage->lazyfree, false); +} + +bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + eligible = atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == 2 && !subpage->pins; + if (eligible) + WRITE_ONCE(subpage->lazyfree, true); + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +bool mm_subpage_all_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + unsigned int i; + bool all = true; + + spin_lock_irqsave(&owner->pool->lock, flags); + for (i = 0; i < SUBPAGES_PER_FOLIO; i++) { + if (!(owner->used & BIT(i))) + continue; + if (!READ_ONCE(owner->slots[i].lazyfree)) { + all = false; + break; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); + return all; +} + +bool mm_subpage_discardable(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + spin_lock_irqsave(&pool->lock, flags); + eligible = (subpage->owner->droppable || READ_ONCE(subpage->lazyfree)) && + !subpage->pins && + refcount_read(&subpage->refs) == atomic_read(&subpage->mapcount) + 1; + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +int mm_subpage_pin(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + int ret = -EAGAIN; + + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->anon_root || owner->migrating || !subpage->exclusive || + !atomic_read(&subpage->mapcount)) + goto out; + ret = try_grab_folio(owner->folio, 1, FOLL_PIN); + if (ret) + goto out; + mm_subpage_get(subpage); + subpage->pins++; +out: + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +void mm_subpage_unpin(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + + spin_lock_irqsave(&owner->pool->lock, flags); + VM_BUG_ON(!subpage->pins || !subpage->exclusive); + subpage->pins--; + spin_unlock_irqrestore(&owner->pool->lock, flags); + unpin_folio(owner->folio); + mm_subpage_put(subpage); +} + +unsigned int mm_subpage_pincount(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->pins); +} + +int mm_subpage_mapcount(const struct mm_subpage *subpage) +{ + return atomic_read(&subpage->mapcount); +} + +struct folio *mm_subpage_folio(const struct mm_subpage *subpage) +{ + return subpage->owner->folio; +} + +unsigned int mm_subpage_shift(const struct mm_subpage *subpage) +{ + return subpage->shift; +} + +unsigned int mm_subpage_size(const struct mm_subpage *subpage) +{ + return 1U << subpage->shift; +} + +unsigned int mm_subpage_offset(const struct mm_subpage *subpage) +{ + return subpage->index << subpage->shift; +} + +phys_addr_t mm_subpage_phys(const struct mm_subpage *subpage) +{ + return PFN_PHYS(folio_pfn(subpage->owner->folio)) + mm_subpage_offset(subpage); +} + +/* + * Round the mm's aggregate reservation, never each VMA: splitting a VMA + * must not create extra charges or refunds. mmap writers serialize updates; + * munmap may retain the lock downgraded while finishing its refund. + */ +static int mm_commit_delta(struct mm_struct *mm, long pages, + unsigned long *next, long *delta) +{ + unsigned long old = mm->committed_user_pages; + unsigned int shift = mm_page_shift(mm); + + mmap_assert_locked(mm); + if (pages < 0 && WARN_ON_ONCE(pages < -(long)old)) + return -EINVAL; + if (pages > 0 && (unsigned long)pages > LONG_MAX - old) + return -ENOMEM; + *next = old + pages; + if (shift > PAGE_SHIFT) { + unsigned int extra = shift - PAGE_SHIFT; + + /* A logical reservation may span several native commit units. */ + if (*next > (LONG_MAX >> extra) || old > (LONG_MAX >> extra)) + return -ENOMEM; + *delta = (long)(*next << extra) - (long)(old << extra); + } else { + unsigned long per_native = 1UL << (PAGE_SHIFT - shift); + + *delta = DIV_ROUND_UP(*next, per_native) - DIV_ROUND_UP(old, per_native); + } + return 0; +} + +int mm_account_memory(struct mm_struct *mm, struct mm_struct *policy_mm, long pages) +{ + unsigned long next; + long delta; + int ret; + + if (mm_page_size(mm) == PAGE_SIZE) + return security_vm_enough_memory_mm(policy_mm, pages); + ret = mm_commit_delta(mm, pages, &next, &delta); + if (ret) + return ret; + ret = security_vm_enough_memory_mm(policy_mm, delta); + if (!ret) + mm->committed_user_pages = next; + return ret; +} + +void mm_acct_memory(struct mm_struct *mm, long pages) +{ + unsigned long next; + long delta; + + if (mm_page_size(mm) == PAGE_SIZE) { + vm_acct_memory(pages); + return; + } + if (WARN_ON_ONCE(mm_commit_delta(mm, pages, &next, &delta))) + return; + vm_acct_memory(delta); + mm->committed_user_pages = next; +} + +void mm_unacct_memory(struct mm_struct *mm, unsigned long pages) +{ + mm_acct_memory(mm, -(long)pages); +} + +/* Shared ownership for consumers whose native page spans contain user slots. */ +struct mm_subpage_refs { + refcount_t refs; + unsigned int count; + unsigned int capacity; + struct mm_subpage *slots[]; +}; + +struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs) +{ + if (refs) + refcount_inc(&refs->refs); + return refs; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_get); + +void mm_subpage_refs_put(struct mm_subpage_refs *refs) +{ + unsigned int i; + + if (!refs || !refcount_dec_and_test(&refs->refs)) + return; + for (i = 0; i < refs->count; i++) + mm_subpage_put(refs->slots[i]); + kfree(refs); +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_put); + +static struct mm_subpage_refs *subpage_refs_new(unsigned int capacity, gfp_t gfp) +{ + struct mm_subpage_refs *refs = kmalloc(struct_size(refs, slots, capacity), gfp); + + if (!refs) + return NULL; + refcount_set(&refs->refs, 1); + refs->count = 0; + refs->capacity = capacity; + return refs; +} + +static int subpage_refs_prepare(struct mm_subpage_refs **refsp, + unsigned int needed, gfp_t gfp) +{ + struct mm_subpage_refs *old = *refsp, *refs; + unsigned int i; + + if (old && refcount_read(&old->refs) == 1 && old->capacity >= needed) + return 0; + refs = subpage_refs_new(max(needed, 4U), gfp); + if (!refs) + return -ENOMEM; + if (old) { + for (i = 0; i < old->count; i++) { + mm_subpage_get(old->slots[i]); + refs->slots[refs->count++] = old->slots[i]; + } + } + *refsp = refs; + mm_subpage_refs_put(old); + return 0; +} + +int mm_subpage_refs_add(struct mm_subpage_refs **refsp, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp) +{ + struct mm_subpage *slots[PAGE_SIZE / MM_SUBPAGE_SIZE]; + struct mm_subpage_refs *refs = *refsp; + unsigned int pos, count = 0, i, j; + int ret = 0; + bool found = false; + + if (!len || folio_order(page_folio(page)) || !folio_test_anon(page_folio(page))) + return 0; + if (offset >= PAGE_SIZE || len > PAGE_SIZE - offset) + return -EINVAL; + for (pos = round_down(offset, MM_SUBPAGE_SIZE); pos < offset + len; + pos += MM_SUBPAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(page_to_phys(page) + pos); + + if (!slot) { + /* A native anonymous allocation has no quarter owner. */ + ret = found ? -EFAULT : 0; + goto out; + } + found = true; + for (j = 0; refs && j < refs->count; j++) + if (refs->slots[j] == slot) + break; + if (refs && j < refs->count) { + mm_subpage_put(slot); + continue; + } + for (j = 0; j < count; j++) + if (slots[j] == slot) + break; + if (j < count) + mm_subpage_put(slot); + else + slots[count++] = slot; + } + if (!count) + return 0; + if (refs && refs->count > UINT_MAX - count) { + ret = -ENOMEM; + goto out; + } + ret = subpage_refs_prepare(refsp, (refs ? refs->count : 0) + count, gfp); + if (ret) + goto out; + refs = *refsp; + for (i = 0; i < count; i++) + refs->slots[refs->count++] = slots[i]; + return 0; +out: + for (i = 0; i < count; i++) + mm_subpage_put(slots[i]); + return ret; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_add); + +int mm_subpage_refs_prune(struct mm_subpage_refs **refsp, + bool (*keep)(const struct mm_subpage *, void *), + void *context, gfp_t gfp) +{ + struct mm_subpage_refs *refs = *refsp, *new; + unsigned int i, count = 0; + + if (!refs) + return 0; + for (i = 0; i < refs->count; i++) + count += keep(refs->slots[i], context); + if (count == refs->count) + return 0; + if (!count) { + *refsp = NULL; + mm_subpage_refs_put(refs); + return 0; + } + if (refcount_read(&refs->refs) == 1) { + for (i = 0, count = 0; i < refs->count; i++) { + if (keep(refs->slots[i], context)) + refs->slots[count++] = refs->slots[i]; + else + mm_subpage_put(refs->slots[i]); + } + refs->count = count; + return 0; + } + new = subpage_refs_new(count, gfp); + if (!new) + return -ENOMEM; + for (i = 0; i < refs->count; i++) { + if (!keep(refs->slots[i], context)) + continue; + mm_subpage_get(refs->slots[i]); + new->slots[new->count++] = refs->slots[i]; + } + *refsp = new; + mm_subpage_refs_put(refs); + return 0; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_prune); From 24d35a6f9ab884e583c4d94dda55ab0683c28e36 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0574/1579] exec: stage arguments and ELF mappings at the selected user page size Resolve the per-thread exec preference before exposing the new address space. Transfer staged arguments across granules, bound temporary copies by their native mappings, and use the selected ABI for ELF and core dumps. Signed-off-by: Diljit Singh (cherry picked from commit c16faad6c36dd65a3cf47cbde2e89b7ac84e46a5) --- fs/binfmt_elf.c | 42 ++++++-- fs/coredump.c | 81 ++++++++++++--- fs/exec.c | 265 +++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 313 insertions(+), 75 deletions(-) diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 16a56b6b3f6ca..826235fd977c2 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -79,11 +79,11 @@ static int elf_core_dump(struct coredump_params *cprm); #define elf_core_dump NULL #endif -#if ELF_EXEC_PAGESIZE > PAGE_SIZE -#define ELF_MIN_ALIGN ELF_EXEC_PAGESIZE -#else -#define ELF_MIN_ALIGN PAGE_SIZE +#ifndef ELF_EXEC_PAGESIZE_MM +#define ELF_EXEC_PAGESIZE_MM(mm) ELF_EXEC_PAGESIZE #endif +#define ELF_MIN_ALIGN max_t(unsigned long, mm_page_size(current->mm), \ + ELF_EXEC_PAGESIZE_MM(current->mm)) #ifndef ELF_CORE_EFLAGS #define ELF_CORE_EFLAGS 0 @@ -93,12 +93,20 @@ static int elf_core_dump(struct coredump_params *cprm); #define ELF_PAGEOFFSET(_v) ((_v) & (ELF_MIN_ALIGN-1)) #define ELF_PAGEALIGN(_v) (((_v) + ELF_MIN_ALIGN - 1) & ~(ELF_MIN_ALIGN - 1)) +#ifdef CONFIG_COREDUMP +static unsigned long elf_min_coredump_size(void) +{ + return ELF_EXEC_PAGESIZE_MM(current->mm); +} +#endif + static struct linux_binfmt elf_format = { .module = THIS_MODULE, .load_binary = load_elf_binary, #ifdef CONFIG_COREDUMP .core_dump = elf_core_dump, .min_coredump = ELF_EXEC_PAGESIZE, + .min_coredump_size = elf_min_coredump_size, #endif }; @@ -248,7 +256,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec, ARCH_DLINFO; #endif NEW_AUX_ENT(AT_HWCAP, ELF_HWCAP); - NEW_AUX_ENT(AT_PAGESZ, ELF_EXEC_PAGESIZE); + NEW_AUX_ENT(AT_PAGESZ, ELF_EXEC_PAGESIZE_MM(mm)); NEW_AUX_ENT(AT_CLKTCK, CLOCKS_PER_SEC); NEW_AUX_ENT(AT_PHDR, phdr_addr); NEW_AUX_ENT(AT_PHENT, sizeof(struct elf_phdr)); @@ -622,6 +630,13 @@ static inline int arch_check_elf(struct elfhdr *ehdr, bool has_interp, #endif /* !CONFIG_ARCH_BINFMT_ELF_STATE */ +#ifndef arch_bprm_prepare_mm +static inline int arch_bprm_prepare_mm(struct linux_binprm *bprm, bool compat) +{ + return 0; +} +#endif + static inline int make_prot(u32 p_flags, struct arch_elf_state *arch_state, bool has_interp, bool is_interp) { @@ -1006,6 +1021,11 @@ static int load_elf_binary(struct linux_binprm *bprm) if (retval) goto out_free_dentry; + retval = arch_bprm_prepare_mm(bprm, + elf_ex->e_ident[EI_CLASS] == ELFCLASS32); + if (retval) + goto out_free_dentry; + /* Flush all traces of the currently running executable */ retval = begin_new_exec(bprm); if (retval) @@ -1335,7 +1355,7 @@ static int load_elf_binary(struct linux_binprm *bprm) * leave a gap between .bss and brk. */ if (!brk_moved) - mm->brk = mm->start_brk = mm->brk + PAGE_SIZE; + mm->brk = mm->start_brk = mm->brk + mm_page_size(mm); mm->brk = mm->start_brk = arch_randomize_brk(mm); brk_moved = true; @@ -1351,10 +1371,10 @@ static int load_elf_binary(struct linux_binprm *bprm) and some applications "depend" upon this behavior. Since we do not have the power to recompile these, we emulate the SVr4 behavior. Sigh. */ - error = vm_mmap(NULL, 0, PAGE_SIZE, PROT_READ | PROT_EXEC, + error = vm_mmap(NULL, 0, mm_page_size(mm), PROT_READ | PROT_EXEC, MAP_FIXED | MAP_PRIVATE, 0); - retval = do_mseal(0, PAGE_SIZE, 0); + retval = do_mseal(0, mm_page_size(mm), 0); if (retval) pr_warn_ratelimited("pid=%d, couldn't seal address 0, ret=%d.\n", task_pid_nr(current), retval); @@ -1658,7 +1678,7 @@ static int fill_files_note(struct memelfnote *note, struct coredump_params *cprm /* Now we know exact count of files, can store it */ data[0] = count; - data[1] = PAGE_SIZE; + data[1] = mm_page_size(current->mm); /* * Count usually is less than mm->map_count, * we need to move filenames down. @@ -2043,7 +2063,7 @@ static int elf_core_dump(struct coredump_params *cprm) offset += sz; } - dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE); + dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE_MM(current->mm)); offset += cprm->vma_data_size; offset += elf_core_extra_data_size(cprm); @@ -2083,7 +2103,7 @@ static int elf_core_dump(struct coredump_params *cprm) phdr.p_flags |= PF_W; if (meta->flags & VM_EXEC) phdr.p_flags |= PF_X; - phdr.p_align = ELF_EXEC_PAGESIZE; + phdr.p_align = ELF_EXEC_PAGESIZE_MM(current->mm); if (!dump_emit(cprm, &phdr, sizeof(phdr))) goto end_coredump; diff --git a/fs/coredump.c b/fs/coredump.c index bb6fdb1f458e9..6ab5f2e076698 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -884,7 +885,8 @@ static bool coredump_file(struct core_name *cn, struct coredump_params *cprm, struct file *file __free(fput) = NULL; int open_flags = O_CREAT | O_WRONLY | O_NOFOLLOW | O_LARGEFILE | O_EXCL; - if (cprm->limit < binfmt->min_coredump) + if (cprm->limit < (binfmt->min_coredump_size ? + binfmt->min_coredump_size() : binfmt->min_coredump)) return false; if (coredump_force_suid_safe(cprm) && cn->corename[0] != '/') { @@ -1278,7 +1280,8 @@ void dump_skip(struct coredump_params *cprm, size_t nr) EXPORT_SYMBOL(dump_skip); #ifdef CONFIG_ELF_CORE -static int dump_emit_page(struct coredump_params *cprm, struct page *page) +static int dump_emit_page(struct coredump_params *cprm, struct page *page, + unsigned int offset, unsigned int size) { struct bio_vec bvec; struct iov_iter iter; @@ -1294,19 +1297,19 @@ static int dump_emit_page(struct coredump_params *cprm, struct page *page) return 0; cprm->to_skip = 0; } - if (cprm->written + PAGE_SIZE > cprm->limit) + if (cprm->written + size > cprm->limit) return 0; if (dump_interrupted()) return 0; pos = file->f_pos; - bvec_set_page(&bvec, page, PAGE_SIZE, 0); - iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, PAGE_SIZE); + bvec_set_page(&bvec, page, size, offset); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size); n = __kernel_write_iter(cprm->file, &iter, &pos); - if (n != PAGE_SIZE) + if (n != size) return 0; file->f_pos = pos; - cprm->written += PAGE_SIZE; - cprm->pos += PAGE_SIZE; + cprm->written += size; + cprm->pos += size; return 1; } @@ -1321,10 +1324,11 @@ static int dump_emit_page(struct coredump_params *cprm, struct page *page) #define dump_page_alloc() alloc_page(GFP_KERNEL) #define dump_page_free(x) __free_page(x) -static struct page *dump_page_copy(struct page *src, struct page *dst) +static struct page *dump_page_copy(struct page *src, struct page *dst, + unsigned int offset, unsigned int size) { void *buf = kmap_local_page(src); - size_t left = copy_mc_to_kernel(page_address(dst), buf, PAGE_SIZE); + size_t left = copy_mc_to_kernel(page_address(dst) + offset, buf + offset, size); kunmap_local(buf); return left ? NULL : dst; } @@ -1334,12 +1338,51 @@ static struct page *dump_page_copy(struct page *src, struct page *dst) /* We just want to return non-NULL; it's never used. */ #define dump_page_alloc() ERR_PTR(-EINVAL) #define dump_page_free(x) ((void)(x)) -static inline struct page *dump_page_copy(struct page *src, struct page *dst) +static inline struct page *dump_page_copy(struct page *src, struct page *dst, + unsigned int offset, unsigned int size) { return src; } #endif +#ifdef CONFIG_MM_SUBPAGE +static int dump_subpage_range(struct coredump_params *cprm, unsigned long start, + unsigned long len) +{ + unsigned long size = mm_user_fragment_size(current->mm), addr; + struct page *scratch = dump_page_alloc(); + int ret = 0; + + if (!scratch) + return 0; + for (addr = start; addr < start + len; addr += size) { + struct user_page_fragment fragment; + + if (get_dump_fragment(addr, &fragment) == 1) { + struct page *page = folio_page(fragment.folio, + fragment.offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(fragment.offset); + int emitted; + + emitted = dump_emit_page(cprm, + dump_page_copy(page, scratch, offset, size), offset, size); + release_user_fragments(&fragment, 1, false); + if (!emitted) + goto out; + } else { + dump_skip(cprm, size); + } + if (dump_interrupted()) + goto out; + cond_resched(); + } + ret = 1; +out: + dump_page_free(scratch); + return ret; +} +#endif + int dump_user_range(struct coredump_params *cprm, unsigned long start, unsigned long len) { @@ -1347,6 +1390,10 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start, struct page *dump_page; int locked, ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return dump_subpage_range(cprm, start, len); +#endif dump_page = dump_page_alloc(); if (!dump_page) return 0; @@ -1375,7 +1422,8 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start, mmap_read_unlock(current->mm); locked = 0; } - int stop = !dump_emit_page(cprm, dump_page_copy(page, dump_page)); + int stop = !dump_emit_page(cprm, + dump_page_copy(page, dump_page, 0, PAGE_SIZE), 0, PAGE_SIZE); put_page(page); if (stop) goto out; @@ -1652,9 +1700,10 @@ static unsigned long vma_dump_size(struct vm_area_struct *vma, * dump the first page to aid in determining what was mapped here. */ if (FILTER(ELF_HEADERS) && - vma->vm_pgoff == 0 && (vma->vm_flags & VM_READ)) { + vma->vm_pgoff == 0 && !vma_subpage_offset(vma) && + (vma->vm_flags & VM_READ)) { if ((READ_ONCE(file_inode(vma->vm_file)->i_mode) & 0111) != 0) - return PAGE_SIZE; + return min(mm_page_size(vma->vm_mm), vma->vm_end - vma->vm_start); /* * ELF libraries aren't always executable. @@ -1754,7 +1803,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) m->end = vma->vm_end; m->flags = vma->vm_flags; m->dump_size = vma_dump_size(vma, cprm->mm_flags); - m->pgoff = vma->vm_pgoff; + m->pgoff = core_vma_pgoff(vma); m->file = vma->vm_file; if (m->file) get_file(m->file); @@ -1773,7 +1822,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) memcmp(elfmag, ELFMAG, SELFMAG) != 0) { m->dump_size = 0; } else { - m->dump_size = PAGE_SIZE; + m->dump_size = min(mm_page_size(mm), m->end - m->start); } } diff --git a/fs/exec.c b/fs/exec.c index 2eacc1441c59f..cb5cda32f8e30 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -122,6 +123,54 @@ bool path_noexec(const struct path *path) (path->mnt->mnt_sb->s_iflags & SB_I_NOEXEC); } +/* A temporary argument mapping owns one native page or an exact subpage slot. */ +struct arg_page { + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *subpage; + unsigned int offset; +#endif +}; + +static unsigned long bprm_page_size(const struct linux_binprm *bprm) +{ +#ifdef CONFIG_MMU + return mm_page_size(bprm->mm); +#else + return PAGE_SIZE; +#endif +} + +/* kmap_local_page() maps one native page, even for a larger user leaf. */ +static unsigned long bprm_copy_size(const struct linux_binprm *bprm) +{ + return min(bprm_page_size(bprm), PAGE_SIZE); +} + +static unsigned long bprm_copy_mask(const struct linux_binprm *bprm) +{ + return ~(bprm_copy_size(bprm) - 1); +} + +static unsigned long bprm_max_arg_strlen(const struct linux_binprm *bprm) +{ + return (MAX_ARG_STRLEN / PAGE_SIZE) * bprm_page_size(bprm); +} + +static unsigned int arg_page_offset(struct arg_page page) +{ +#ifdef CONFIG_MM_SUBPAGE + return page.offset; +#else + return 0; +#endif +} + +static void *map_arg_page(struct arg_page page) +{ + return kmap_local_page(page.page) + arg_page_offset(page); +} + #ifdef CONFIG_MMU /* * The nascent bprm->mm is not visible until exec_mmap() but it can @@ -141,10 +190,10 @@ static void acct_arg_size(struct linux_binprm *bprm, unsigned long pages) add_mm_counter(mm, MM_ANONPAGES, diff); } -static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, +static struct arg_page get_arg_page(struct linux_binprm *bprm, unsigned long pos, int write) { - struct page *page; + struct arg_page page = {}; struct vm_area_struct *vma = bprm->vma; struct mm_struct *mm = bprm->mm; int ret; @@ -155,28 +204,47 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, * ahead of time. */ if (!mmap_read_lock_maybe_expand(mm, vma, pos, write)) - return NULL; + return page; /* * We are doing an exec(). 'current' is the process * doing the exec and 'mm' is the new process's mm. */ - ret = get_user_pages_remote(mm, pos, 1, - write ? FOLL_WRITE : 0, - &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + struct vm_area_struct *area; + + ret = get_user_fragment_vma_remote(mm, pos, 1, write ? FOLL_WRITE : 0, + &fragment, &area); + if (ret > 0) { + page.page = folio_page(fragment.folio, fragment.offset >> PAGE_SHIFT); + page.subpage = fragment.subpage; + page.offset = offset_in_page(fragment.offset) & mm_page_mask(mm); + } + } else +#endif + ret = get_user_pages_remote(mm, pos, 1, + write ? FOLL_WRITE : 0, &page.page, NULL); mmap_read_unlock(mm); if (ret <= 0) - return NULL; + return page; if (write) - acct_arg_size(bprm, vma_pages(vma)); + acct_arg_size(bprm, mm_pages_for_reader(mm, current->mm, vma_user_pages(vma))); return page; } -static void put_arg_page(struct page *page) +static void put_arg_page(struct arg_page page) { - put_page(page); +#ifdef CONFIG_MM_SUBPAGE + if (page.subpage) { + mm_subpage_put(page.subpage); + return; + } +#endif + put_page(page.page); } static void free_arg_pages(struct linux_binprm *bprm) @@ -184,14 +252,14 @@ static void free_arg_pages(struct linux_binprm *bprm) } static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos, - struct page *page) + struct arg_page page) { - flush_cache_page(bprm->vma, pos, page_to_pfn(page)); + flush_cache_page(bprm->vma, pos, page_to_pfn(page.page)); } static bool valid_arg_len(struct linux_binprm *bprm, long len) { - return len <= MAX_ARG_STRLEN; + return len <= bprm_max_arg_strlen(bprm); } #else @@ -200,7 +268,7 @@ static inline void acct_arg_size(struct linux_binprm *bprm, unsigned long pages) { } -static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, +static struct arg_page get_arg_page(struct linux_binprm *bprm, unsigned long pos, int write) { struct page *page; @@ -209,14 +277,14 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, if (!page && write) { page = alloc_page(GFP_HIGHUSER|__GFP_ZERO); if (!page) - return NULL; + return (struct arg_page) {}; bprm->page[pos / PAGE_SIZE] = page; } - return page; + return (struct arg_page) { .page = page }; } -static void put_arg_page(struct page *page) +static void put_arg_page(struct arg_page page) { } @@ -237,7 +305,7 @@ static void free_arg_pages(struct linux_binprm *bprm) } static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos, - struct page *page) + struct arg_page page) { } @@ -259,7 +327,7 @@ static int bprm_mm_init(struct linux_binprm *bprm) int err; struct mm_struct *mm = NULL; - bprm->mm = mm = mm_alloc(); + bprm->mm = mm = mm_alloc_exec(); err = -ENOMEM; if (!mm) goto err; @@ -288,6 +356,102 @@ static int bprm_mm_init(struct linux_binprm *bprm) return err; } +#ifdef CONFIG_MM_SUBPAGE +/* + * Final ELF selection may change the granule after argv, envp and script + * arguments have been staged. Table geometry is immutable: build a second mm + * and transfer bytes at the same temporary stack addresses before committing + * exec. Until success, the original bprm and its argument charge remain intact. + */ +int bprm_set_page_shift(struct linux_binprm *bprm, unsigned int page_shift) +{ + struct linux_binprm replacement = {}; + struct mm_struct *old_mm; + unsigned long pos, end, new_top, string_len = 0; + unsigned long old_size = mm_user_fragment_size(bprm->mm), new_size; + int ret; + + if (mm_page_shift(bprm->mm) == page_shift) + return 0; + if (WARN_ON_ONCE(bprm->point_of_no_return)) + return -EINVAL; + + replacement.mm = mm_alloc_exec_page_shift(page_shift); + if (!replacement.mm) + return -ENOMEM; + ret = -EINVAL; + if (mm_page_shift(replacement.mm) != page_shift) + goto out; + new_size = mm_user_fragment_size(replacement.mm); + ret = create_init_stack_vma(replacement.mm, &replacement.vma, &new_top); + if (ret) + goto out; + end = bprm->vma->vm_end - sizeof(void *); + ret = -EINVAL; + if (new_top != end || bprm->p > end) + goto out; + + for (pos = bprm->p; pos < end;) { + struct arg_page src, dst; + unsigned long src_off = pos & (old_size - 1); + unsigned long dst_off = pos & (new_size - 1); + unsigned long len = min3(end - pos, old_size - src_off, + new_size - dst_off); + const char *from; + char *to; + + ret = -ERESTARTNOHAND; + if (fatal_signal_pending(current)) + goto out; + src = get_arg_page(bprm, pos, 0); + ret = -EFAULT; + if (!src.page) + goto out; + dst = get_arg_page(&replacement, pos, 1); + if (!dst.page) { + put_arg_page(src); + ret = -ENOMEM; + goto out; + } + from = map_arg_page(src); + to = map_arg_page(dst); + /* Recheck per-string limits if the selected granule shrank. */ + ret = 0; + for (unsigned long i = 0; i < len; i++) { + if (++string_len > bprm_max_arg_strlen(&replacement)) { + ret = -E2BIG; + break; + } + if (!from[src_off + i]) + string_len = 0; + } + if (!ret) + memcpy(to + dst_off, from + src_off, len); + kunmap_local(to); + kunmap_local(from); + flush_arg_page(&replacement, pos, dst); + put_arg_page(dst); + put_arg_page(src); + if (ret) + goto out; + pos += len; + cond_resched(); + } + + old_mm = bprm->mm; + acct_arg_size(bprm, 0); + bprm->mm = replacement.mm; + bprm->vma = replacement.vma; + bprm->vma_pages = replacement.vma_pages; + mmput(old_mm); + return 0; +out: + acct_arg_size(&replacement, 0); + mmput(replacement.mm); + return ret; +} +#endif + struct user_arg_ptr { #ifdef CONFIG_COMPAT bool is_compat; @@ -449,7 +613,7 @@ static int bprm_stack_limits(struct linux_binprm *bprm) static int copy_strings(int argc, struct user_arg_ptr argv, struct linux_binprm *bprm) { - struct page *kmapped_page = NULL; + struct arg_page kmapped_page = {}; char *kaddr = NULL; unsigned long kpos = 0; int ret; @@ -464,7 +628,7 @@ static int copy_strings(int argc, struct user_arg_ptr argv, if (IS_ERR(str)) goto out; - len = strnlen_user(str, MAX_ARG_STRLEN); + len = strnlen_user(str, bprm_max_arg_strlen(bprm)); if (!len) goto out; @@ -488,9 +652,9 @@ static int copy_strings(int argc, struct user_arg_ptr argv, } cond_resched(); - offset = pos % PAGE_SIZE; + offset = pos & ~bprm_copy_mask(bprm); if (offset == 0) - offset = PAGE_SIZE; + offset = bprm_copy_size(bprm); bytes_to_copy = offset; if (bytes_to_copy > len) @@ -501,23 +665,23 @@ static int copy_strings(int argc, struct user_arg_ptr argv, str -= bytes_to_copy; len -= bytes_to_copy; - if (!kmapped_page || kpos != (pos & PAGE_MASK)) { - struct page *page; + if (!kmapped_page.page || kpos != (pos & bprm_copy_mask(bprm))) { + struct arg_page page; page = get_arg_page(bprm, pos, 1); - if (!page) { + if (!page.page) { ret = -E2BIG; goto out; } - if (kmapped_page) { - flush_dcache_page(kmapped_page); + if (kmapped_page.page) { + flush_dcache_page(kmapped_page.page); kunmap_local(kaddr); put_arg_page(kmapped_page); } kmapped_page = page; - kaddr = kmap_local_page(kmapped_page); - kpos = pos & PAGE_MASK; + kaddr = map_arg_page(kmapped_page); + kpos = pos & bprm_copy_mask(bprm); flush_arg_page(bprm, kpos, kmapped_page); } if (copy_from_user(kaddr+offset, str, bytes_to_copy)) { @@ -528,8 +692,8 @@ static int copy_strings(int argc, struct user_arg_ptr argv, } ret = 0; out: - if (kmapped_page) { - flush_dcache_page(kmapped_page); + if (kmapped_page.page) { + flush_dcache_page(kmapped_page.page); kunmap_local(kaddr); put_arg_page(kmapped_page); } @@ -541,7 +705,7 @@ static int copy_strings(int argc, struct user_arg_ptr argv, */ int copy_string_kernel(const char *arg, struct linux_binprm *bprm) { - int len = strnlen(arg, MAX_ARG_STRLEN) + 1 /* terminating NUL */; + int len = strnlen(arg, bprm_max_arg_strlen(bprm)) + 1 /* terminating NUL */; unsigned long pos = bprm->p; if (len == 0) @@ -557,18 +721,19 @@ int copy_string_kernel(const char *arg, struct linux_binprm *bprm) while (len > 0) { unsigned int bytes_to_copy = min(len, - min_not_zero(offset_in_page(pos), PAGE_SIZE)); - struct page *page; + min_not_zero(pos & ~bprm_copy_mask(bprm), bprm_copy_size(bprm))); + struct arg_page page; pos -= bytes_to_copy; arg -= bytes_to_copy; len -= bytes_to_copy; page = get_arg_page(bprm, pos, 1); - if (!page) + if (!page.page) return -E2BIG; - flush_arg_page(bprm, pos & PAGE_MASK, page); - memcpy_to_page(page, offset_in_page(pos), arg, bytes_to_copy); + flush_arg_page(bprm, pos & bprm_copy_mask(bprm), page); + memcpy_to_page(page.page, arg_page_offset(page) + + (pos & ~bprm_copy_mask(bprm)), arg, bytes_to_copy); put_arg_page(page); } @@ -621,20 +786,20 @@ int setup_arg_pages(struct linux_binprm *bprm, /* Add space for stack randomization. */ if (current->flags & PF_RANDOMIZE) - stack_base += (STACK_RND_MASK << PAGE_SHIFT); + stack_base += (STACK_RND_MASK << mm_page_shift(mm)); /* Make sure we didn't let the argument array grow too large. */ if (vma->vm_end - vma->vm_start > stack_base) return -ENOMEM; - stack_base = PAGE_ALIGN(stack_top - stack_base); + stack_base = mm_page_align(mm, stack_top - stack_base); stack_shift = vma->vm_start - stack_base; mm->arg_start = bprm->p - stack_shift; bprm->p = vma->vm_end - stack_shift; #else stack_top = arch_align_stack(stack_top); - stack_top = PAGE_ALIGN(stack_top); + stack_top = mm_page_align(mm, stack_top); if (unlikely(stack_top < mmap_min_addr) || unlikely(vma->vm_end - vma->vm_start >= stack_top - mmap_min_addr)) @@ -702,7 +867,7 @@ int setup_arg_pages(struct linux_binprm *bprm, * Align this down to a page boundary as expand_stack * will align it up. */ - rlim_stack = bprm->rlim_stack.rlim_cur & PAGE_MASK; + rlim_stack = bprm->rlim_stack.rlim_cur & mm_page_mask(mm); stack_expand = min(rlim_stack, stack_size + stack_expand); @@ -1323,6 +1488,10 @@ void setup_new_exec(struct linux_binprm * bprm) arch_pick_mmap_layout(me->mm, &bprm->rlim_stack); + /* Consume the one-shot request only after committing to the new image. */ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + me->exec_page_shift = 0; +#endif arch_setup_new_exec(); /* Set the new mm task size. We have to do that late because it may @@ -1616,25 +1785,25 @@ int remove_arg_zero(struct linux_binprm *bprm) { unsigned long offset; char *kaddr; - struct page *page; + struct arg_page page; if (!bprm->argc) return 0; do { - offset = bprm->p & ~PAGE_MASK; + offset = bprm->p & ~bprm_copy_mask(bprm); page = get_arg_page(bprm, bprm->p, 0); - if (!page) + if (!page.page) return -EFAULT; - kaddr = kmap_local_page(page); + kaddr = map_arg_page(page); - for (; offset < PAGE_SIZE && kaddr[offset]; + for (; offset < bprm_copy_size(bprm) && kaddr[offset]; offset++, bprm->p++) ; kunmap_local(kaddr); put_arg_page(page); - } while (offset == PAGE_SIZE); + } while (offset == bprm_copy_size(bprm)); bprm->p++; bprm->argc--; From f0a266fee9aeffbe1daa820763e655b4ff8cc6c0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0575/1579] block: preserve user fragment ownership across I/O Carry the backing offset and retained fragment reference through bio, splice and io_uring paths so a smaller user mapping is not treated as an entire native physical page. Signed-off-by: Diljit Singh (cherry picked from commit c5f12989feb7fadc77031ea60829bdc879112864) --- block/bio.c | 150 +++++++++++++++++++++++++++++++++ block/blk-map.c | 2 +- fs/splice.c | 172 ++++++++++++++++++++++++++++++++++++++ include/linux/bio.h | 8 ++ include/linux/blk_types.h | 4 + io_uring/memmap.c | 8 +- io_uring/rsrc.c | 124 +++++++++++++++++++++++++++ 7 files changed, 466 insertions(+), 2 deletions(-) diff --git a/block/bio.c b/block/bio.c index 5f10900b3f42a..846417dec7d16 100644 --- a/block/bio.c +++ b/block/bio.c @@ -3,6 +3,7 @@ * Copyright (C) 2001 Jens Axboe */ #include +#include #include #include #include @@ -246,6 +247,10 @@ void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, #endif #ifdef CONFIG_BLK_DEV_INTEGRITY bio->bi_integrity = NULL; +#endif +#ifdef CONFIG_MM_SUBPAGE + bio->bi_user_fragments = NULL; + bio->bi_nr_user_fragments = 0; #endif bio->bi_vcnt = 0; @@ -1159,6 +1164,18 @@ void __bio_release_pages(struct bio *bio, bool mark_dirty) { struct folio_iter fi; +#ifdef CONFIG_MM_SUBPAGE + if (bio->bi_user_fragments) { + release_user_fragments(bio->bi_user_fragments, + bio->bi_nr_user_fragments, mark_dirty); + kfree(bio->bi_user_fragments); + bio->bi_user_fragments = NULL; + bio->bi_nr_user_fragments = 0; + bio_clear_flag(bio, BIO_PAGE_PINNED); + return; + } +#endif + bio_for_each_folio_all(fi, bio) { size_t nr_pages; @@ -1220,6 +1237,134 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, return 0; } +#ifdef CONFIG_MM_SUBPAGE +static bool bio_subpage_iter(const struct iov_iter *iter) +{ + return user_backed_iter(iter) && current->mm && + mm_page_size(current->mm) < PAGE_SIZE; +} + +int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs) +{ + struct iov_iter walk = *iter; + unsigned int count = 0; + + if (!bio_subpage_iter(iter)) + return iov_iter_npages(iter, max_vecs); + while (iov_iter_count(&walk) && count < max_vecs) { + size_t len, offset, nr; + + iov_iter_advance(&walk, 0); + len = min(iter_iov_len(&walk), iov_iter_count(&walk)); + if (WARN_ON_ONCE(!len)) + return max_vecs; + offset = (unsigned long)iter_iov_addr(&walk) & (MM_SUBPAGE_SIZE - 1); + nr = (len >> MM_SUBPAGE_SHIFT) + + DIV_ROUND_UP(offset + (len & (MM_SUBPAGE_SIZE - 1)), MM_SUBPAGE_SIZE); + if (nr >= max_vecs - count) + return max_vecs; + count += nr; + iov_iter_advance(&walk, len); + } + return count; +} +EXPORT_SYMBOL_GPL(bio_iov_user_vecs); + +/* Pins belong to the submitting bio, just as with ordinary BIO_PAGE_PINNED. + * Split/cloned bios borrow its vectors and complete before their owner does. + */ +static int bio_iov_iter_get_fragments(struct bio *bio, struct iov_iter *iter, + unsigned int align_mask) +{ + unsigned int capacity = bio->bi_max_vecs, total = 0; + unsigned int flags = iov_iter_rw(iter) == READ ? FOLL_WRITE : 0; + int ret = 0; + + if (WARN_ON_ONCE(bio->bi_vcnt || bio->bi_user_fragments || !capacity)) + return -EINVAL; + bio->bi_user_fragments = kmalloc_array(capacity, + sizeof(*bio->bi_user_fragments), GFP_KERNEL); + if (!bio->bi_user_fragments) + return -ENOMEM; + while (iov_iter_count(iter) && bio->bi_nr_user_fragments < capacity && + total < BIO_MAX_SIZE) { + unsigned int i, bytes = 0; + long nr; + + iov_iter_advance(iter, 0); + nr = pin_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(iter), + min3(iter_iov_len(iter), iov_iter_count(iter), + (size_t)(BIO_MAX_SIZE - total)), flags, + bio->bi_user_fragments + bio->bi_nr_user_fragments, + capacity - bio->bi_nr_user_fragments); + if (nr <= 0) { + ret = nr ?: -EFAULT; + break; + } + for (i = 0; i < nr; i++) { + struct user_page_fragment *fragment = + &bio->bi_user_fragments[bio->bi_nr_user_fragments++]; + struct page *page = folio_page(fragment->folio, + fragment->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(fragment->offset); + struct bio_vec *prev = bio->bi_vcnt ? + &bio->bi_io_vec[bio->bi_vcnt - 1] : NULL; + + /* Coalesce physically adjacent quarters within one folio. */ + if (prev && page_folio(prev->bv_page) == fragment->folio && + page_to_phys(prev->bv_page) + prev->bv_offset + prev->bv_len == + page_to_phys(page) + offset) + prev->bv_len += fragment->length; + else + bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt++], page, + fragment->length, offset); + bytes += fragment->length; + } + iov_iter_advance(iter, bytes); + total += bytes; + } + /* Trim pins separately from vectors: several pins may share one bvec. */ + if (total & align_mask) { + unsigned int trim = total & align_mask, left = trim; + + while (left) { + struct user_page_fragment *fragment = + &bio->bi_user_fragments[bio->bi_nr_user_fragments - 1]; + unsigned int bytes = min(left, fragment->length); + + if (bytes == fragment->length) { + release_user_fragments(fragment, 1, false); + bio->bi_nr_user_fragments--; + } else { + fragment->length -= bytes; + } + left -= bytes; + } + left = trim; + while (left) { + struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; + unsigned int bytes = min(left, bv->bv_len); + + bv->bv_len -= bytes; + if (!bv->bv_len) + bio->bi_vcnt--; + left -= bytes; + } + iov_iter_revert(iter, trim); + total -= trim; + } + bio->bi_iter.bi_size = total; + if (!total) { + kfree(bio->bi_user_fragments); + bio->bi_user_fragments = NULL; + return ret ?: -EFAULT; + } + bio_set_flag(bio, BIO_PAGE_PINNED); + return 0; +} +#endif + /** * bio_iov_iter_get_pages - add user or kernel pages to a bio * @bio: bio to add pages to @@ -1255,6 +1400,11 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, return 0; } +#ifdef CONFIG_MM_SUBPAGE + if (bio_subpage_iter(iter)) + return bio_iov_iter_get_fragments(bio, iter, len_align_mask); +#endif + if (iov_iter_extract_will_pin(iter)) bio_set_flag(bio, BIO_PAGE_PINNED); if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue)) diff --git a/block/blk-map.c b/block/blk-map.c index 768549f19f97e..b98eed276aea2 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -260,7 +260,7 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, gfp_t gfp_mask) { - unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS); + unsigned int nr_vecs = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS); struct bio *bio; int ret; diff --git a/fs/splice.c b/fs/splice.c index 9d8f63e2fd1ab..4b707ce00e86d 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -183,6 +185,79 @@ static void wakeup_pipe_readers(struct pipe_inode_info *pipe) kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); } +#ifdef CONFIG_MM_SUBPAGE +/* A shared lifetime for all user leaves coalesced into one native pipe page. */ +struct subpage_pipe_refs { + refcount_t refs; + unsigned int count; + struct user_page_fragment fragments[]; +}; + +static void subpage_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs = (void *)buf->private; + + if (refcount_dec_and_test(&refs->refs)) { + release_user_fragments(refs->fragments, refs->count, false); + kfree(refs); + } +} + +static bool subpage_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs = (void *)buf->private; + + return refcount_inc_not_zero(&refs->refs); +} + +static const struct pipe_buf_operations subpage_pipe_buf_ops = { + .release = subpage_pipe_buf_release, + .get = subpage_pipe_buf_get, + /* A gift of user leaves is not ownership of the containing native page. */ +}; + +static int subpage_pipe_buf_retain(struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs; + unsigned int pos, count = 0; + + if (!buf->len || buf->ops != &nosteal_pipe_buf_ops || + !folio_test_anon(page_folio(buf->page))) + return 0; + refs = kmalloc(struct_size(refs, fragments, PAGE_SIZE / MM_SUBPAGE_SIZE), GFP_KERNEL); + if (!refs) + return -ENOMEM; + for (pos = round_down(buf->offset, MM_SUBPAGE_SIZE); + pos < buf->offset + buf->len; pos += MM_SUBPAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(page_to_phys(buf->page) + pos); + + if (!slot) { + release_user_fragments(refs->fragments, count, false); + kfree(refs); + return count ? -EFAULT : 0; + } + refs->fragments[count++] = (struct user_page_fragment) { + .folio = page_folio(buf->page), + .subpage = slot, + .offset = pos, + .length = MM_SUBPAGE_SIZE, + }; + } + refcount_set(&refs->refs, 1); + refs->count = count; + /* Replace the incoming native reference with the typed references. */ + put_page(buf->page); + buf->ops = &subpage_pipe_buf_ops; + buf->private = (unsigned long)refs; + return 0; +} +#else +static int subpage_pipe_buf_retain(struct pipe_buffer *buf) +{ + return 0; +} +#endif + /** * splice_to_pipe - fill passed data into a pipe * @pipe: pipe to fill @@ -221,6 +296,16 @@ ssize_t splice_to_pipe(struct pipe_inode_info *pipe, buf->private = spd->partial[page_nr].private; buf->ops = spd->ops; buf->flags = 0; + { + int err = subpage_pipe_buf_retain(buf); + + if (err) { + buf->ops = NULL; + if (!ret) + ret = err; + break; + } + } head++; pipe->head = head; @@ -852,6 +937,16 @@ ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out, continue; } +#ifdef CONFIG_MM_SUBPAGE + /* Other transports still need their downstream lifetime audit. */ + if (buf->ops == &subpage_pipe_buf_ops && + sock->ops->family != AF_UNIX && + sock->ops->family != AF_INET && + sock->ops->family != AF_INET6) { + ret = -EOPNOTSUPP; + break; + } +#endif seg = min_t(size_t, remain, buf->len); ret = pipe_buf_confirm(pipe, buf); @@ -1440,6 +1535,79 @@ static ssize_t __do_splice(struct file *in, loff_t __user *off_in, return ret; } +#ifdef CONFIG_MM_SUBPAGE +static ssize_t subpage_iter_to_pipe(struct iov_iter *from, + struct pipe_inode_info *pipe, unsigned int flags) +{ + struct user_page_fragment fragments[16]; + size_t total = 0; + long ret = 0; + + if (WARN_ON_ONCE(!user_backed_iter(from))) + return -EINVAL; + while (iov_iter_count(from)) { + unsigned int i = 0; + long count; + + /* Normalize empty iovecs before taking the next address. */ + iov_iter_advance(from, 0); + count = get_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(from), + min(iov_iter_count(from), iter_iov_len(from)), 0, + fragments, ARRAY_SIZE(fragments)); + if (count <= 0) { + ret = count; + break; + } + while (i < count) { + struct user_page_fragment *first = &fragments[i]; + struct page *page = folio_page(first->folio, first->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(first->offset); + unsigned int length = first->length, nr = 1; + struct subpage_pipe_refs *refs; + struct pipe_buffer buf; + + /* Preserve native pipe capacity for contiguous packed quarters. */ + while (i + nr < count && length < PAGE_SIZE - offset) { + struct user_page_fragment *next = &fragments[i + nr]; + + if (folio_page(next->folio, next->offset >> PAGE_SHIFT) != page || + offset_in_page(next->offset) != offset + length) + break; + length += next->length; + nr++; + } + refs = kmalloc(struct_size(refs, fragments, nr), GFP_KERNEL); + if (!refs) { + ret = -ENOMEM; + break; + } + refcount_set(&refs->refs, 1); + refs->count = nr; + memcpy(refs->fragments, first, sizeof(*first) * nr); + buf = (struct pipe_buffer) { + .page = page, + .offset = offset, + .len = length, + .ops = &subpage_pipe_buf_ops, + .flags = flags, + .private = (unsigned long)refs, + }; + i += nr; /* add_to_pipe consumes these references even on failure. */ + ret = add_to_pipe(pipe, &buf); + if (ret < 0) + break; + iov_iter_advance(from, ret); + total += ret; + } + release_user_fragments(fragments + i, count - i, false); + if (ret < 0) + break; + } + return total ? total : ret; +} +#endif + static ssize_t iter_to_pipe(struct iov_iter *from, struct pipe_inode_info *pipe, unsigned int flags) @@ -1451,6 +1619,10 @@ static ssize_t iter_to_pipe(struct iov_iter *from, size_t total = 0; ssize_t ret = 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return subpage_iter_to_pipe(from, pipe, flags); +#endif while (iov_iter_count(from)) { struct page *pages[16]; ssize_t left; diff --git a/include/linux/bio.h b/include/linux/bio.h index dc17780d6c1e3..714e44044373a 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -389,11 +389,19 @@ static inline void bio_wouldblock_error(struct bio *bio) * pointed by @iter. If @iter is backed by bvec it's going to be reused * instead of allocating a new one. */ +#ifdef CONFIG_MM_SUBPAGE +int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs); +#endif + static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs) { if (iov_iter_is_bvec(iter)) return 0; +#ifdef CONFIG_MM_SUBPAGE + return bio_iov_user_vecs(iter, max_segs); +#else return iov_iter_npages(iter, max_segs); +#endif } /** diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 8808ee76e73c0..cc9b1eeddd415 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -267,6 +267,10 @@ struct bio { struct bio_integrity_payload *bi_integrity; /* data integrity */ #endif +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *bi_user_fragments; + unsigned short bi_nr_user_fragments; +#endif unsigned short bi_vcnt; /* how many bio_vec's */ /* diff --git a/io_uring/memmap.c b/io_uring/memmap.c index 4f9b439319c46..72e123757bea4 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -289,7 +289,10 @@ static int io_region_mmap(struct io_ring_ctx *ctx, unsigned long nr_pages = min(mr->nr_pages, max_pages); vm_flags_set(vma, VM_DONTEXPAND); - return vm_insert_pages(vma, vma->vm_start, mr->pages, &nr_pages); + if (mm_page_size(vma->vm_mm) == PAGE_SIZE) + return vm_insert_pages(vma, vma->vm_start, mr->pages, &nr_pages); + return vm_insert_pages_range(vma, vma->vm_start, mr->pages, nr_pages, 0, + min_t(unsigned long, vma->vm_end - vma->vm_start, nr_pages << PAGE_SHIFT)); } __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma) @@ -301,6 +304,9 @@ __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma) struct io_mapped_region *region; void *ptr; + if (vma_subpage_offset(vma)) + return -EINVAL; + guard(mutex)(&ctx->mmap_lock); ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff); diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 2dfc26233e286..33cc1e2688480 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -762,6 +763,95 @@ bool io_check_coalesce_buffer(struct page **page_array, int nr_pages, return true; } +#ifdef CONFIG_MM_SUBPAGE +struct io_fragment_buffer { + unsigned int count; + struct user_page_fragment fragments[]; +}; + +static void io_release_fragment_buffer(void *priv) +{ + struct io_fragment_buffer *buffer = priv; + + release_user_fragments(buffer->fragments, buffer->count, false); + kvfree(buffer); +} + +static struct io_rsrc_node *io_register_fragment_buffer(struct io_ring_ctx *ctx, + struct iovec *iov) +{ + unsigned long size = mm_page_size(current->mm); + unsigned long first = (unsigned long)iov->iov_base & (size - 1); + unsigned int count = DIV_ROUND_UP(first + iov->iov_len, size); + struct io_fragment_buffer *buffer; + struct io_mapped_ubuf *imu = NULL; + struct io_rsrc_node *node; + unsigned long charged = 0; + unsigned int i; + struct xarray seen; + long nr; + int ret = -ENOMEM; + + node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); + if (!node) + return ERR_PTR(-ENOMEM); + buffer = kvmalloc(struct_size(buffer, fragments, count), GFP_KERNEL_ACCOUNT); + if (!buffer) + goto free_node; + buffer->count = 0; + nr = pin_user_fragments_remote(current->mm, (unsigned long)iov->iov_base, + iov->iov_len, FOLL_WRITE | FOLL_LONGTERM, buffer->fragments, count); + if (nr > 0) + buffer->count = nr; + if (nr != count) { + ret = nr < 0 ? nr : -EFAULT; + goto free_buffer; + } + imu = io_alloc_imu(ctx, count); + if (!imu) + goto free_buffer; + imu->nr_bvecs = count; + imu->acct_pages = 0; + /* Several user leaves can share native backing, including aliases. */ + xa_init(&seen); + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &buffer->fragments[i]; + + ret = xa_insert(&seen, folio_pfn(f->folio), xa_mk_value(1), GFP_KERNEL); + if (!ret) + charged += folio_nr_pages(f->folio); + else if (ret != -EBUSY) + break; + bvec_set_page(&imu->bvec[i], folio_page(f->folio, f->offset >> PAGE_SHIFT), + f->length, offset_in_page(f->offset)); + } + xa_destroy(&seen); + if (ret && ret != -EBUSY) + goto free_imu; + ret = io_account_mem(ctx->user, ctx->mm_account, charged); + if (ret) + goto free_imu; + imu->acct_pages = charged; + imu->ubuf = (unsigned long)iov->iov_base; + imu->len = iov->iov_len; + imu->folio_shift = mm_page_shift(current->mm); + imu->release = io_release_fragment_buffer; + imu->priv = buffer; + imu->flags = 0; + imu->dir = IO_IMU_DEST | IO_IMU_SOURCE; + refcount_set(&imu->refs, 1); + node->buf = imu; + return node; +free_imu: + io_free_imu(ctx, imu); +free_buffer: + io_release_fragment_buffer(buffer); +free_node: + io_cache_free(&ctx->node_cache, node); + return ERR_PTR(ret); +} +#endif + static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, struct page **last_hpage) @@ -787,6 +877,11 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, if (ret) return ERR_PTR(ret); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) < PAGE_SIZE) + return io_register_fragment_buffer(ctx, iov); +#endif + node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); if (!node) return ERR_PTR(-ENOMEM); @@ -1095,6 +1190,14 @@ static int io_import_fixed(int ddir, struct iov_iter *iter, bvec += seg_skip; offset &= folio_mask; } +#ifdef CONFIG_MM_SUBPAGE + if (imu->folio_shift < PAGE_SHIFT) { + /* Physical fragment offsets are independent of the user address. */ + size_t first = min_t(size_t, len, bvec->bv_len - offset); + + nr_segs = 1 + DIV_ROUND_UP(len - first, 1UL << imu->folio_shift); + } else +#endif nr_segs = (offset + len + bvec->bv_offset + folio_mask) >> imu->folio_shift; iov_iter_bvec(iter, ddir, bvec, nr_segs, len); iter->iov_offset = offset; @@ -1356,6 +1459,27 @@ static int io_vec_fill_bvec(int ddir, struct iov_iter *iter, return -EOVERFLOW; offset = buf_addr - imu->ubuf; +#ifdef CONFIG_MM_SUBPAGE + if (imu->folio_shift < PAGE_SHIFT) { + unsigned int index; + unsigned long first = imu->ubuf & folio_mask; + + offset += first; + index = offset >> imu->folio_shift; + offset &= folio_mask; + if (!index) + offset -= first; + src_bvec = imu->bvec + index; + for (; iov_len; offset = 0, bvec_idx++, src_bvec++) { + size_t bytes = min_t(size_t, iov_len, src_bvec->bv_len - offset); + + bvec_set_page(&res_bvec[bvec_idx], src_bvec->bv_page, + bytes, src_bvec->bv_offset + offset); + iov_len -= bytes; + } + continue; + } +#endif /* * Only the first bvec can have non zero bv_offset, account it * here and work with full folios below. From 6b0a280a4c5913d285d0fcfaf63ec3a817ac1e74 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0576/1579] net: retain user fragments in networking and BPF consumers Preserve fragment ownership and byte ranges in socket, TLS, BPF and vhost consumers of user memory. Signed-off-by: Diljit Singh (cherry picked from commit e6bf191ec8db185fcc908e5a5410497aac3a7543) --- drivers/vhost/vhost.c | 21 +++ include/linux/bpf.h | 22 +++ include/linux/skmsg.h | 40 +++++- include/net/tls.h | 7 + kernel/bpf/bpf_local_storage.c | 23 ++-- kernel/bpf/syscall.c | 128 ++++++++++++++++-- net/core/skbuff.c | 119 ++++++++++++++++- net/core/skmsg.c | 88 +++++++++++- net/ipv4/tcp_bpf.c | 5 + net/tls/tls_device.c | 22 +++ net/tls/tls_sw.c | 238 +++++++++++++++++++++++++++++++-- 11 files changed, 673 insertions(+), 40 deletions(-) diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index e4c44f2ea413a..4c3577ce7b85b 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -2436,6 +2437,26 @@ static int set_bit_to_user(int nr, void __user *addr) int bit = nr + (log % PAGE_SIZE) * 8; int r; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + unsigned long word = log & ~(sizeof(unsigned long) - 1); + + /* Pin the complete atomic word, which stays inside one leaf. + * A relocated leaf's physical offset need not match its VA. + */ + r = pin_user_fragments_remote(current->mm, word, + sizeof(unsigned long), FOLL_WRITE, &fragment, 1); + if (r != 1) + return r < 0 ? r : -EFAULT; + base = kmap_local_folio(fragment.folio, fragment.offset); + set_bit(nr + (log - word) * 8, base); + kunmap_local(base); + release_user_fragments(&fragment, 1, true); + return 0; + } +#endif + r = pin_user_pages_fast(log, 1, FOLL_WRITE, &page); if (r < 0) return r; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index a2cf24d168802..cfec2e3b66979 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -263,6 +263,11 @@ struct btf_record { int wq_off; int refcount_off; int task_work_off; +#ifdef CONFIG_MM_SUBPAGE + /* Private ownership pointer appended to task-storage values with UPTRs. */ + u32 uptr_pins_off; + u32 uptr_extra_size; +#endif struct btf_field fields[]; }; @@ -563,6 +568,15 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src bpf_obj_memcpy(map->record, dst, src, map->value_size, true); } +static inline u32 bpf_obj_uptr_extra_size(const struct btf_record *rec) +{ +#ifdef CONFIG_MM_SUBPAGE + return IS_ERR_OR_NULL(rec) ? 0 : rec->uptr_extra_size; +#else + return 0; +#endif +} + static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) { unsigned long *src_uptr, *dst_uptr; @@ -580,6 +594,14 @@ static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, v dst_uptr = dst + field->offset; swap(*src_uptr, *dst_uptr); } +#ifdef CONFIG_MM_SUBPAGE + if (rec->uptr_extra_size) { + void **src_pins = src + rec->uptr_pins_off; + void **dst_pins = dst + rec->uptr_pins_off; + + swap(*src_pins, *dst_pins); + } +#endif } static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size) diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 937823856de59..30406f8cee116 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -41,6 +41,8 @@ struct sk_msg_sg { }; /* UAPI in filter.c depends on struct sk_msg_sg being first element. */ +struct mm_subpage_refs; + struct sk_msg { struct sk_msg_sg sg; void *data; @@ -49,6 +51,9 @@ struct sk_msg { u32 cork_bytes; u32 flags; struct sk_buff *skb; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage_refs *subpage_refs; +#endif struct sock *sk_redir; struct sock *sk; struct list_head list; @@ -124,6 +129,33 @@ struct sk_psock { struct rcu_work rwork; }; +#ifdef CONFIG_MM_SUBPAGE +int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len); +void sk_msg_subpages_get(struct sk_msg *msg); +void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src); +void sk_msg_subpages_release(struct sk_msg *msg); +static inline void sk_msg_subpages_transfer(struct sk_msg *dst, struct sk_msg *src) +{ + dst->subpage_refs = src->subpage_refs; + src->subpage_refs = NULL; +} +#else +static inline int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len) +{ + return 0; +} +static inline void sk_msg_subpages_get(struct sk_msg *msg) { } +static inline void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src) { } +static inline void sk_msg_subpages_release(struct sk_msg *msg) { } +static inline void sk_msg_subpages_transfer(struct sk_msg *dst, struct sk_msg *src) { } +#endif + +static inline void sk_msg_subpages_drop_empty(struct sk_msg *msg) +{ + if (!msg->sg.size) + sk_msg_subpages_release(msg); +} + int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, int elem_first_coalesce); int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, @@ -259,10 +291,14 @@ static inline void sk_msg_compute_data_pointers(struct sk_msg *msg) } } -static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, +static inline int sk_msg_page_add(struct sk_msg *msg, struct page *page, u32 len, u32 offset) { struct scatterlist *sge; + int ret = sk_msg_subpages_add(msg, page, offset, len); + + if (ret) + return ret; get_page(page); sge = sk_msg_elem(msg, msg->sg.end); @@ -272,6 +308,7 @@ static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, __set_bit(msg->sg.end, msg->sg.copy); msg->sg.size += len; sk_msg_iter_next(msg, end); + return 0; } static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state) @@ -420,6 +457,7 @@ static inline bool sk_psock_queue_empty(const struct sk_psock *psock) static inline void kfree_sk_msg(struct sk_msg *msg) { + sk_msg_subpages_release(msg); if (msg->skb) consume_skb(msg->skb); kfree(msg); diff --git a/include/net/tls.h b/include/net/tls.h index ebd2550280ae2..79d358a5fe887 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -34,6 +34,7 @@ #ifndef _TLS_OFFLOAD_H #define _TLS_OFFLOAD_H +#include #include #include #include @@ -139,6 +140,9 @@ struct tls_sw_context_rx { struct tls_strparser strp; atomic_t decrypt_pending; +#ifdef CONFIG_MM_SUBPAGE + struct llist_head subpage_cleanup; +#endif struct sk_buff_head async_hold; struct wait_queue_head wq; }; @@ -148,6 +152,9 @@ struct tls_record_info { u32 end_seq; int len; int num_frags; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage_refs *subpage_refs; +#endif skb_frag_t frags[MAX_SKB_FRAGS]; }; diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 6fc6a4b672b55..9abd4972966ff 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -24,6 +24,11 @@ select_bucket(struct bpf_local_storage_map *smap, return &smap->buckets[hash_ptr(local_storage, smap->bucket_log)]; } +static u32 bpf_selem_size(const struct bpf_local_storage_map *smap) +{ + return smap->elem_size + bpf_obj_uptr_extra_size(smap->map.record); +} + static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size) { struct bpf_map *map = &smap->map; @@ -72,10 +77,10 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, { struct bpf_local_storage_elem *selem; - if (mem_charge(smap, owner, smap->elem_size)) + if (mem_charge(smap, owner, bpf_selem_size(smap))) return NULL; - selem = bpf_map_kmalloc_nolock(&smap->map, smap->elem_size, + selem = bpf_map_kmalloc_nolock(&smap->map, bpf_selem_size(smap), __GFP_ZERO, NUMA_NO_NODE); if (selem) { @@ -91,7 +96,7 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, return selem; } - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); return NULL; } @@ -178,7 +183,7 @@ static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem * bool free_local_storage, bool pin_owner) { void *owner = local_storage->owner; - u32 uncharge = smap->elem_size; + u32 uncharge = bpf_selem_size(smap); if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) == SDATA(selem)) @@ -204,7 +209,7 @@ static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem * /* local_storage->lock must be held and selem->local_storage == local_storage. * The caller must ensure selem->smap is still valid to be - * dereferenced for its smap->elem_size and smap->cache_idx. + * dereferenced for its element size and smap->cache_idx. */ static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, struct bpf_local_storage_elem *selem, @@ -234,7 +239,7 @@ void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage, struct bpf_local_storage_map *smap; smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); - local_storage->mem_charge += smap->elem_size; + local_storage->mem_charge += bpf_selem_size(smap); RCU_INIT_POINTER(selem->local_storage, local_storage); hlist_add_head_rcu(&selem->snode, &local_storage->list); @@ -577,7 +582,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, err = bpf_local_storage_alloc(owner, smap, selem); if (err) { bpf_selem_free(selem, true); - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); return ERR_PTR(err); } @@ -661,7 +666,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, free_selem: bpf_selem_free_list(&old_selem_free_list, false); if (alloc_selem) { - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); bpf_selem_free(alloc_selem, true); } return err ? ERR_PTR(err) : SDATA(selem); @@ -862,7 +867,7 @@ void bpf_local_storage_map_free(struct bpf_map *map, * exit immediately. * * However, while freeing the storage one still needs to access the - * smap->elem_size to do the uncharging in + * bpf_selem_size(smap) to do the uncharging in * bpf_selem_unlink_storage_nolock(). * * Hence, wait another rcu grace period for the storage to be freed. diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 96d1b797e1523..c084c1cad03d8 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include #include @@ -164,6 +166,43 @@ static void maybe_wait_bpf_programs(struct bpf_map *map) synchronize_rcu_expedited(); } +#ifdef CONFIG_MM_SUBPAGE +struct bpf_uptr_pins { + struct work_struct work; + u32 count; + struct user_page_fragment fragments[]; +}; + +static void bpf_uptr_release_work(struct work_struct *work) +{ + struct bpf_uptr_pins *pins = container_of(work, struct bpf_uptr_pins, work); + + release_user_fragments(pins->fragments, pins->count, true); + kfree(pins); +} +#endif + +/* May run from an RCU callback. The bundle owns its pins independently of + * the map, task and value, so dirty release can wait for folio locks in work. + */ +static bool bpf_obj_release_uptr_pins(const struct btf_record *rec, void *obj) +{ +#ifdef CONFIG_MM_SUBPAGE + struct bpf_uptr_pins **owner, *pins; + + if (!bpf_obj_uptr_extra_size(rec)) + return false; + owner = obj + rec->uptr_pins_off; + pins = *owner; + if (pins) { + *owner = NULL; + queue_work(system_unbound_wq, &pins->work); + return true; + } +#endif + return false; +} + static void unpin_uptr_kaddr(void *kaddr) { if (kaddr) @@ -176,6 +215,9 @@ static void __bpf_obj_unpin_uptrs(struct btf_record *rec, u32 cnt, void *obj) void **uptr_addr; int i; + if (bpf_obj_release_uptr_pins(rec, obj)) + return; + for (i = 0, field = rec->fields; i < cnt; i++, field++) { if (field->type != BPF_UPTR) continue; @@ -193,8 +235,12 @@ static void bpf_obj_unpin_uptrs(struct btf_record *rec, void *obj) __bpf_obj_unpin_uptrs(rec, rec->cnt, obj); } -static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) +static int bpf_obj_pin_uptrs(struct bpf_map *map, void *obj) { + struct btf_record *rec = map->record; +#ifdef CONFIG_MM_SUBPAGE + struct bpf_uptr_pins *pins = NULL; +#endif const struct btf_field *field; const struct btf_type *t; unsigned long start, end; @@ -205,6 +251,18 @@ static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) if (!btf_record_has_field(rec, BPF_UPTR)) return 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + pins = bpf_map_kzalloc(map, struct_size(pins, fragments, rec->cnt), + GFP_KERNEL); + if (!pins) + return -ENOMEM; + INIT_WORK(&pins->work, bpf_uptr_release_work); + pins->count = rec->cnt; + *(struct bpf_uptr_pins **)(obj + rec->uptr_pins_off) = pins; + } +#endif + for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) { if (field->type != BPF_UPTR) continue; @@ -221,12 +279,35 @@ static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) goto unpin_all; } - /* The uptr's struct cannot span across two pages */ - if ((start & PAGE_MASK) != (end & PAGE_MASK)) { + /* The object must fit in one page of the supplying process. */ + if ((start & mm_page_mask(current->mm)) != + (end & mm_page_mask(current->mm))) { err = -EOPNOTSUPP; goto unpin_all; } +#ifdef CONFIG_MM_SUBPAGE + if (pins) { + struct user_page_fragment *fragment = &pins->fragments[i]; + + err = pin_user_fragments_remote(current->mm, start, t->size, + FOLL_LONGTERM | FOLL_WRITE, fragment, 1); + if (err != 1) { + if (!err) + err = -EFAULT; + goto unpin_all; + } + if (fragment->length != t->size || + PageHighMem(folio_page(fragment->folio, + fragment->offset >> PAGE_SHIFT))) { + err = -EOPNOTSUPP; + goto unpin_all; + } + *uptr_addr = folio_address(fragment->folio) + fragment->offset; + continue; + } +#endif + err = pin_user_pages_fast(start, 1, FOLL_LONGTERM | FOLL_WRITE, &page); if (err != 1) goto unpin_all; @@ -291,7 +372,7 @@ static int bpf_map_update_value(struct bpf_map *map, struct file *map_file, map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) { err = map->ops->map_push_elem(map, value, flags); } else { - err = bpf_obj_pin_uptrs(map->record, value); + err = bpf_obj_pin_uptrs(map, value); if (!err) { rcu_read_lock(); err = map->ops->map_update_elem(map, key, value, flags); @@ -815,10 +896,12 @@ void bpf_obj_cancel_fields(struct bpf_map *map, void *obj) void bpf_obj_free_fields(const struct btf_record *rec, void *obj) { const struct btf_field *fields; + bool fragment_pins; int i; if (IS_ERR_OR_NULL(rec)) return; + fragment_pins = bpf_obj_release_uptr_pins(rec, obj); fields = rec->fields; for (i = 0; i < rec->cnt; i++) { struct btf_struct_meta *pointee_struct_meta; @@ -859,8 +942,9 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj) } break; case BPF_UPTR: - /* The caller ensured that no one is using the uptr */ - unpin_uptr_kaddr(*(void **)field_ptr); + /* The caller ensured that no one is using the uptr. */ + if (!fragment_pins) + unpin_uptr_kaddr(*(void **)field_ptr); break; case BPF_LIST_HEAD: if (WARN_ON_ONCE(rec->spin_lock_off < 0)) @@ -1350,6 +1434,14 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, if (ret < 0) goto free_map_tab; +#ifdef CONFIG_MM_SUBPAGE + if (btf_record_has_field(map->record, BPF_UPTR)) { + map->record->uptr_pins_off = ALIGN(map->value_size, sizeof(void *)); + map->record->uptr_extra_size = map->record->uptr_pins_off + + sizeof(void *) - map->value_size; + } +#endif + if (map->ops->map_check_btf) { ret = map->ops->map_check_btf(map, btf, key_type, value_type); if (ret < 0) @@ -1775,6 +1867,24 @@ static int map_lookup_elem(union bpf_attr *attr) #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags +static void *bpf_map_memdup_value(struct bpf_map *map, bpfptr_t src, u32 size) +{ + u32 extra = bpf_obj_uptr_extra_size(map->record); + void *value; + + if (!extra) + return kvmemdup_bpfptr(src, size); + value = kvmalloc(size + extra, GFP_USER); + if (!value) + return ERR_PTR(-ENOMEM); + memset(value + size, 0, extra); + if (copy_from_bpfptr(value, src, size)) { + kvfree(value); + return ERR_PTR(-EFAULT); + } + return value; +} + static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) { bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel); @@ -1808,7 +1918,7 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) } value_size = bpf_map_value_size(map, attr->flags); - value = kvmemdup_bpfptr(uvalue, value_size); + value = bpf_map_memdup_value(map, uvalue, value_size); if (IS_ERR(value)) { err = PTR_ERR(value); goto free_key; @@ -2021,7 +2131,9 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file, if (!key) return -ENOMEM; - value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); + value = kvmalloc(value_size + bpf_obj_uptr_extra_size(map->record), + GFP_USER | __GFP_NOWARN | + (bpf_obj_uptr_extra_size(map->record) ? __GFP_ZERO : 0)); if (!value) { kvfree(key); return -ENOMEM; diff --git a/net/core/skbuff.c b/net/core/skbuff.c index a5cdcc0fc55e1..0bc6467aae5d4 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -7347,8 +7348,113 @@ nodefer: kfree_skb_napi_cache(skb); kick_defer_list_purge(cpu); } +#ifdef CONFIG_MM_SUBPAGE +/* The ordinary frag page ref protects allocation, not an anonymous quarter. */ +struct skb_subpage_refs { + struct ubuf_info ubuf; + struct mm_subpage_refs *owner; +}; + +static void skb_subpage_complete(struct sk_buff *skb, struct ubuf_info *ubuf, + bool success) +{ + struct skb_subpage_refs *refs = container_of(ubuf, struct skb_subpage_refs, ubuf); + + if (!refcount_dec_and_test(&ubuf->refcnt)) + return; + mm_subpage_refs_put(refs->owner); + kfree(refs); +} + +static const struct ubuf_info_ops skb_subpage_ops = { + .complete = skb_subpage_complete, +}; + +static bool skb_contains_subpage(const struct sk_buff *skb, phys_addr_t phys, + unsigned int size, unsigned int depth) +{ + struct sk_buff *child; + unsigned int i; + + /* Match the networking recursion bound, conservatively keeping unknowns. */ + if (depth >= 24) + return true; + if (skb->head_frag && skb_headlen(skb)) { + phys_addr_t start = page_to_phys(virt_to_page(skb->data)) + offset_in_page(skb->data); + + if (start < phys + size && phys < start + skb_headlen(skb)) + return true; + } + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + struct page *page = skb_frag_page(frag); + phys_addr_t start; + + if (!page) + continue; + start = page_to_phys(page) + skb_frag_off(frag); + if (start < phys + size && phys < start + skb_frag_size(frag)) + return true; + } + skb_walk_frags(skb, child) + if (skb_contains_subpage(child, phys, size, depth + 1)) + return true; + return false; +} + +static bool skb_keeps_subpage(const struct mm_subpage *slot, void *context) +{ + return skb_contains_subpage(context, mm_subpage_phys(slot), mm_subpage_size(slot), 0); +} + +/* Caller owns the writable skb and keeps the source slot identity alive. */ +static int skb_splice_hold_subpages(struct sk_buff *skb, struct page *page, + unsigned int off, unsigned int len) +{ + struct skb_subpage_refs *refs, *new; + struct ubuf_info *old = skb_zcopy(skb); + struct mm_subpage *probe; + int ret; + + if (!len || folio_order(page_folio(page)) || !folio_test_anon(page_folio(page))) + return 0; + /* Preserve the allocation-free native anonymous path. */ + probe = mm_subpage_get_from_phys(page_to_phys(page) + round_down(off, MM_SUBPAGE_SIZE)); + if (!probe) + return 0; + mm_subpage_put(probe); + if (old && (skb_zcopy_is_nouarg(skb) || old->ops != &skb_subpage_ops)) + return -EOPNOTSUPP; + refs = old ? container_of(old, struct skb_subpage_refs, ubuf) : NULL; + if (!refs || refcount_read(&old->refcnt) != 1) { + new = kzalloc(sizeof(*new), GFP_ATOMIC); + if (!new) + return -ENOMEM; + new->ubuf.ops = &skb_subpage_ops; + new->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; + refcount_set(&new->ubuf.refcnt, 1); + if (refs) + new->owner = mm_subpage_refs_get(refs->owner); + skb_zcopy_init(skb, &new->ubuf); + net_zcopy_put(old); + refs = new; + } + /* Repeated trim/append must not accumulate references to discarded data. */ + ret = mm_subpage_refs_prune(&refs->owner, skb_keeps_subpage, skb, GFP_ATOMIC); + if (ret) + return ret; + return mm_subpage_refs_add(&refs->owner, page, off, len, GFP_ATOMIC); +} +#else +static int skb_splice_hold_subpages(struct sk_buff *skb, struct page *page, + unsigned int off, unsigned int len) +{ + return 0; +} +#endif + static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, - size_t offset, size_t len) + size_t offset, size_t len, size_t progress) { const char *kaddr; __wsum csum; @@ -7356,7 +7462,8 @@ static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, kaddr = kmap_local_page(page); csum = csum_partial(kaddr + offset, len, 0); kunmap_local(kaddr); - skb->csum = csum_block_add(skb->csum, csum, skb->len); + /* skb->len is updated only after the entire iterator batch. */ + skb->csum = csum_block_add(skb->csum, csum, skb->len + progress); } /** @@ -7408,6 +7515,12 @@ ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, if (WARN_ON_ONCE(!sendpage_ok(page))) goto out; + ret = skb_splice_hold_subpages(skb, page, off, part); + if (ret < 0) { + iov_iter_revert(iter, len); + goto out; + } + ret = skb_append_pagefrags(skb, page, off, part, frag_limit); if (ret < 0) { @@ -7416,7 +7529,7 @@ ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, } if (skb->ip_summed == CHECKSUM_NONE) - skb_splice_csum_page(skb, page, off, part); + skb_splice_csum_page(skb, page, off, part, spliced); off = 0; spliced += part; diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 30c3b9a2681c4..10eb255be0ffa 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */ +#include +#include #include #include #include @@ -24,6 +26,65 @@ static bool sk_msg_try_coalesce_ok(struct sk_msg *msg, int elem_first_coalesce) return false; } +#ifdef CONFIG_MM_SUBPAGE +static bool sk_msg_keeps_subpage(const struct mm_subpage *slot, void *context) +{ + struct sk_msg *msg = context; + phys_addr_t phys = mm_subpage_phys(slot); + phys_addr_t end = phys + mm_subpage_size(slot); + u32 i = msg->sg.start; + + if (!msg->sg.size) + return false; + while (i != msg->sg.end) { + struct scatterlist *sg = &msg->sg.data[i]; + + if (sg->length) { + phys_addr_t start = page_to_phys(sg_page(sg)) + sg->offset; + + if (start < end && phys < start + sg->length) + return true; + } + sk_msg_iter_var_next(i); + } + return false; +} + +int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len) +{ + int ret; + + ret = mm_subpage_refs_prune(&msg->subpage_refs, sk_msg_keeps_subpage, msg, GFP_ATOMIC); + if (ret) + return ret; + return mm_subpage_refs_add(&msg->subpage_refs, page, offset, len, GFP_ATOMIC); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_add); + +/* Called after a borrowed descriptor copy whose source remains live. */ +void sk_msg_subpages_get(struct sk_msg *msg) +{ + mm_subpage_refs_get(msg->subpage_refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_get); + +void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src) +{ + WARN_ON_ONCE(dst->subpage_refs); + dst->subpage_refs = mm_subpage_refs_get(src->subpage_refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_share); + +void sk_msg_subpages_release(struct sk_msg *msg) +{ + struct mm_subpage_refs *refs = msg->subpage_refs; + + msg->subpage_refs = NULL; + mm_subpage_refs_put(refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_release); +#endif + int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, int elem_first_coalesce) { @@ -88,7 +149,7 @@ EXPORT_SYMBOL_GPL(sk_msg_alloc); int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, u32 off, u32 len) { - int i = src->sg.start; + int i = src->sg.start, ret; struct scatterlist *sge = sk_msg_elem(src, i); struct scatterlist *sgd = NULL; u32 sge_len, sge_off; @@ -114,11 +175,16 @@ int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, if (sgd && (sg_page(sge) == sg_page(sgd)) && (sg_virt(sge) + off == sg_virt(sgd) + sgd->length)) { + ret = sk_msg_subpages_add(dst, sg_page(sge), sge->offset + off, sge_len); + if (ret) + return ret; sgd->length += sge_len; dst->sg.size += sge_len; } else if (!sk_msg_full(dst)) { sge_off = sge->offset + off; - sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off); + ret = sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off); + if (ret) + return ret; } else { return -ENOSPC; } @@ -205,6 +271,7 @@ static int __sk_msg_free(struct sock *sk, struct sk_msg *msg, u32 i, sk_msg_check_to_free(msg, i, msg->sg.size); sge = sk_msg_elem(msg, i); } + sk_msg_subpages_release(msg); consume_skb(msg->skb); sk_msg_init(msg); return freed; @@ -248,6 +315,7 @@ static void __sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, sk_msg_check_to_free(msg, i, bytes); } msg->sg.start = i; + sk_msg_subpages_drop_empty(msg); } void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes) @@ -267,6 +335,7 @@ void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len) int trim = msg->sg.size - len; u32 i = msg->sg.end; + sk_msg_subpages_drop_empty(msg); if (trim <= 0) { WARN_ON(trim < 0); return; @@ -299,6 +368,7 @@ void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len) * does not require an update. */ if (!msg->sg.size) { + sk_msg_subpages_release(msg); msg->sg.curr = msg->sg.start; msg->sg.copybreak = 0; } else if (sk_msg_iter_dist(msg->sg.start, msg->sg.curr) >= @@ -334,11 +404,19 @@ int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from, goto out; } - bytes -= copied; - msg->sg.size += copied; - while (copied) { use = min_t(int, copied, PAGE_SIZE - offset); + ret = sk_msg_subpages_add(msg, pages[i], offset, use); + if (ret) { + unsigned int nr = DIV_ROUND_UP(offset + copied, PAGE_SIZE); + + while (nr--) + put_page(pages[i++]); + iov_iter_revert(from, copied); + goto out; + } + bytes -= use; + msg->sg.size += use; sg_set_page(&msg->sg.data[msg->sg.end], pages[i], use, offset); sg_unmark_end(&msg->sg.data[msg->sg.end]); diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index 18b45950d1acc..d082be17d76fc 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -44,6 +44,7 @@ static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, return -ENOMEM; lock_sock(sk); + sk_msg_subpages_share(tmp, msg); tmp->sg.start = msg->sg.start; i = msg->sg.start; do { @@ -76,6 +77,7 @@ static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, if (!ret) { msg->sg.start = i; + sk_msg_subpages_drop_empty(msg); if (!sk_psock_queue_msg(psock, tmp)) atomic_sub(copied, &sk->sk_rmem_alloc); sk_psock_data_ready(sk, psock); @@ -147,6 +149,7 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, break; } + sk_msg_subpages_drop_empty(msg); return 0; } @@ -434,6 +437,8 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, } } memcpy(psock->cork, msg, sizeof(*msg)); + if (psock->cork != msg) + sk_msg_subpages_transfer(psock->cork, msg); return 0; } diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 33c42f65364a2..10da3de523b93 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -32,6 +32,8 @@ #include #include #include +#include +#include #include #include #include @@ -144,6 +146,10 @@ static void destroy_record(struct tls_record_info *record) for (i = 0; i < record->num_frags; i++) __skb_frag_unref(&record->frags[i], false); + +#ifdef CONFIG_MM_SUBPAGE + mm_subpage_refs_put(record->subpage_refs); +#endif kfree(record); } @@ -349,6 +355,9 @@ static int tls_create_new_record(struct tls_offload_context_tx *offload_ctx, record = kmalloc_obj(*record); if (!record) return -ENOMEM; +#ifdef CONFIG_MM_SUBPAGE + record->subpage_refs = NULL; +#endif frag = &record->frags[0]; skb_frag_fill_page_desc(frag, pfrag->page, pfrag->offset, @@ -513,6 +522,15 @@ static int tls_push_data(struct sock *sk, goto handle_error; } +#ifdef CONFIG_MM_SUBPAGE + rc = mm_subpage_refs_add(&record->subpage_refs, zc_pfrag.page, + off, copy, GFP_KERNEL); + if (rc) { + iov_iter_revert(iter, copy); + goto handle_error; + } +#endif + zc_pfrag.offset = off; zc_pfrag.size = copy; tls_append_frag(record, &zc_pfrag, copy); @@ -1118,6 +1136,10 @@ int tls_set_device_offload(struct sock *sk) goto release_netdev; } + +#ifdef CONFIG_MM_SUBPAGE + start_marker_record->subpage_refs = NULL; +#endif offload_ctx = alloc_offload_ctx_tx(ctx); if (!offload_ctx) { rc = -ENOMEM; diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index ca0f657121a3b..f4f708533f79b 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -35,8 +35,11 @@ * SOFTWARE. */ +#include #include #include +#include +#include #include #include #include @@ -65,9 +68,36 @@ struct tls_decrypt_ctx { u8 aad[TLS_MAX_AAD_SIZE]; u8 tail; bool free_sgout; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *fragments; + unsigned int nr_fragments; + bool fragments_dirty; + struct llist_node cleanup; + void *allocation; +#endif struct scatterlist sg[]; }; +#if IS_ENABLED(CONFIG_KUNIT) && IS_ENABLED(CONFIG_MM_SUBPAGE) +/* Explicit test-only async provider selection; normal cipher selection stays intact. */ +static bool user4k_test_async; +module_param_named(user4k_test_async, user4k_test_async, bool, 0400); +MODULE_PARM_DESC(user4k_test_async, "KUnit: select cryptd for TLS receive testing"); +static atomic_long_t user4k_rx_deferred = ATOMIC_LONG_INIT(0); +static atomic_long_t user4k_rx_released = ATOMIC_LONG_INIT(0); + +static int user4k_read_count(char *buffer, const struct kernel_param *param) +{ + return scnprintf(buffer, PAGE_SIZE, "%ld\n", atomic_long_read(param->arg)); +} + +static const struct kernel_param_ops user4k_count_ops = { + .get = user4k_read_count, +}; +module_param_cb(user4k_rx_deferred, &user4k_count_ops, &user4k_rx_deferred, 0444); +module_param_cb(user4k_rx_released, &user4k_count_ops, &user4k_rx_released, 0444); +#endif + noinline void tls_err_abort(struct sock *sk, int err) { WARN_ON_ONCE(err >= 0); @@ -184,6 +214,35 @@ static int tls_padding_length(struct tls_prot_info *prot, struct sk_buff *skb, return sub; } +#ifdef CONFIG_MM_SUBPAGE +static void tls_release_user_fragments(struct tls_decrypt_ctx *dctx) +{ + release_user_fragments(dctx->fragments, dctx->nr_fragments, dctx->fragments_dirty); + kfree(dctx->fragments); + dctx->fragments = NULL; + dctx->nr_fragments = 0; +} +#endif + +/* Dirty pin release can take folio locks, so it belongs to the waiting task. */ +static void tls_finish_subpage_decrypts(struct tls_sw_context_rx *ctx) +{ +#ifdef CONFIG_MM_SUBPAGE + struct tls_decrypt_ctx *dctx, *next; + struct llist_node *head = llist_del_all(&ctx->subpage_cleanup); + + llist_for_each_entry_safe(dctx, next, head, cleanup) { + void *allocation = dctx->allocation; + + tls_release_user_fragments(dctx); +#if IS_ENABLED(CONFIG_KUNIT) + atomic_long_inc(&user4k_rx_released); +#endif + kfree(allocation); + } +#endif +} + static void tls_decrypt_done(void *data, int err) { struct aead_request *aead_req = data; @@ -196,6 +255,7 @@ static void tls_decrypt_done(void *data, int err) unsigned int pages; struct sock *sk; int aead_size; + bool deferred = false; /* If requests get too backlogged crypto API returns -EBUSY and calls * ->complete(-EINPROGRESS) immediately followed by ->complete(0) @@ -224,17 +284,29 @@ static void tls_decrypt_done(void *data, int err) tls_err_abort(sk, err); } - /* Free the destination pages if skb was not decrypted inplace */ - if (dctx->free_sgout) { - /* Skip the first S/G entry as it points to AAD */ - for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) { - if (!sg) - break; - put_page(sg_page(sg)); +#ifdef CONFIG_MM_SUBPAGE + if (dctx->fragments) { + dctx->allocation = aead_req; +#if IS_ENABLED(CONFIG_KUNIT) + atomic_long_inc(&user4k_rx_deferred); +#endif + llist_add(&dctx->cleanup, &ctx->subpage_cleanup); + deferred = true; + } +#endif + if (!deferred) { + /* Free the destination pages if skb was not decrypted inplace */ + if (dctx->free_sgout) { + /* Skip the first S/G entry as it points to AAD */ + for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) { + if (!sg) + break; + put_page(sg_page(sg)); + } } - } - kfree(aead_req); + kfree(aead_req); + } if (atomic_dec_and_test(&ctx->decrypt_pending)) complete(&ctx->async_wait.completion); @@ -246,6 +318,7 @@ static int tls_decrypt_async_wait(struct tls_sw_context_rx *ctx) crypto_wait_req(-EINPROGRESS, &ctx->async_wait); atomic_inc(&ctx->decrypt_pending); + tls_finish_subpage_decrypts(ctx); __skb_queue_purge(&ctx->async_hold); return ctx->async_wait.err; } @@ -670,6 +743,7 @@ static int tls_split_open_record(struct sock *sk, struct tls_rec *from, msg_opl->sg.size = bytes; msg_npl = &new->msg_plaintext; + sk_msg_subpages_share(msg_npl, msg_opl); msg_npl->apply_bytes = apply; msg_npl->sg.size = orig_size - bytes; @@ -733,6 +807,7 @@ static void tls_merge_open_record(struct sock *sk, struct tls_rec *to, sk_msg_free(sk, &to->msg_encrypted); sk_msg_xfer_full(&to->msg_encrypted, &from->msg_encrypted); + sk_msg_subpages_release(msg_npl); kfree(from); } @@ -935,12 +1010,14 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, redir_ingress = psock->redir_ingress; sk_redir = psock->sk_redir; memcpy(&msg_redir, msg, sizeof(*msg)); + sk_msg_subpages_get(&msg_redir); if (msg->apply_bytes < send) msg->apply_bytes = 0; else msg->apply_bytes -= send; sk_msg_return_zero(sk, msg, send); msg->sg.size -= send; + sk_msg_subpages_drop_empty(msg); release_sock(sk); err = tcp_bpf_sendmsg_redir(sk_redir, redir_ingress, &msg_redir, send, flags); @@ -956,6 +1033,7 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, *copied -= sk_msg_free_nocharge(sk, &msg_redir); msg->sg.size = 0; } + sk_msg_subpages_release(&msg_redir); if (msg->sg.size == 0) tls_free_open_rec(sk); break; @@ -1036,7 +1114,14 @@ static int tls_sw_sendmsg_splice(struct sock *sk, struct msghdr *msg, return -EIO; } - sk_msg_page_add(msg_pl, page, part, off); + { + int ret = sk_msg_page_add(msg_pl, page, part, off); + + if (ret) { + iov_iter_revert(&msg->msg_iter, part); + return ret; + } + } msg_pl->sg.copybreak = 0; msg_pl->sg.curr = msg_pl->sg.end; sk_mem_charge(sk, part); @@ -1454,10 +1539,96 @@ tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock, return 1; } +#ifdef CONFIG_MM_SUBPAGE +static bool tls_subpage_iter(const struct iov_iter *iter) +{ + return user_backed_iter(iter) && current->mm && mm_page_size(current->mm) < PAGE_SIZE; +} +#endif + +static int tls_iter_sg_capacity(struct iov_iter *iter, size_t length) +{ +#ifdef CONFIG_MM_SUBPAGE + if (tls_subpage_iter(iter)) { + struct iov_iter walk = *iter; + unsigned int count = 0; + + length = min(length, iov_iter_count(iter)); + while (length) { + size_t bytes, offset; + + iov_iter_advance(&walk, 0); + bytes = min(length, iter_iov_len(&walk)); + if (WARN_ON_ONCE(!bytes)) + return INT_MAX; + offset = (unsigned long)iter_iov_addr(&walk) & (MM_SUBPAGE_SIZE - 1); + count += DIV_ROUND_UP(offset + bytes, MM_SUBPAGE_SIZE); + iov_iter_advance(&walk, bytes); + length -= bytes; + } + return count; + } +#endif + return iov_iter_npages_cap(iter, INT_MAX, length); +} + +#ifdef CONFIG_MM_SUBPAGE +static int tls_setup_subpage_iter(struct iov_iter *from, int length, int *pages_used, + struct scatterlist *to, int capacity, + struct tls_decrypt_ctx *dctx) +{ + unsigned int used = 0, total = 0; + int ret = 0; + + if (!length) + return 0; + if (WARN_ON_ONCE(*pages_used || capacity <= 0)) + return -EINVAL; + dctx->fragments = kmalloc_array(capacity, sizeof(*dctx->fragments), GFP_KERNEL); + if (!dctx->fragments) + return -ENOMEM; + while (length) { + long nr; + unsigned int i, bytes = 0; + + iov_iter_advance(from, 0); + nr = pin_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(from), + min_t(size_t, length, iter_iov_len(from)), FOLL_WRITE, + dctx->fragments + used, capacity - used); + if (nr <= 0) { + ret = nr ?: -EFAULT; + goto out; + } + for (i = 0; i < nr; i++) { + struct user_page_fragment *fragment = &dctx->fragments[used]; + + sg_set_page(&to[used], + folio_page(fragment->folio, fragment->offset >> PAGE_SHIFT), + fragment->length, offset_in_page(fragment->offset)); + sg_unmark_end(&to[used]); + bytes += fragment->length; + used++; + } + dctx->nr_fragments = used; + iov_iter_advance(from, bytes); + length -= bytes; + total += bytes; + } + if (used) + sg_mark_end(&to[used - 1]); +out: + if (ret) + iov_iter_revert(from, total); + *pages_used = used; + return ret; +} +#endif + static int tls_setup_from_iter(struct iov_iter *from, int length, int *pages_used, struct scatterlist *to, - int to_max_pages) + int to_max_pages, struct tls_decrypt_ctx *dctx) { int rc = 0, i = 0, num_elem = *pages_used, maxpages; struct page *pages[MAX_SKB_FRAGS]; @@ -1465,6 +1636,10 @@ static int tls_setup_from_iter(struct iov_iter *from, ssize_t copied, use; size_t offset; +#ifdef CONFIG_MM_SUBPAGE + if (tls_subpage_iter(from)) + return tls_setup_subpage_iter(from, length, pages_used, to, to_max_pages, dctx); +#endif while (length > 0) { i = 0; maxpages = to_max_pages - num_elem; @@ -1582,7 +1757,7 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, if (out_iov) n_sgout = 1 + tail_pages + - iov_iter_npages_cap(out_iov, INT_MAX, data_len); + tls_iter_sg_capacity(out_iov, data_len); else n_sgout = sg_nents(out_sg); } else { @@ -1615,6 +1790,11 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, aead_req = (struct aead_request *)mem; dctx = (struct tls_decrypt_ctx *)(mem + aead_size); dctx->sk = sk; +#ifdef CONFIG_MM_SUBPAGE + dctx->fragments = NULL; + dctx->nr_fragments = 0; + dctx->fragments_dirty = false; +#endif sgin = &dctx->sg[0]; sgout = &dctx->sg[n_sgin]; @@ -1672,7 +1852,7 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, sg_set_buf(&sgout[0], dctx->aad, prot->aad_size); err = tls_setup_from_iter(out_iov, data_len, &pages, &sgout[1], - (n_sgout - 1 - tail_pages)); + (n_sgout - 1 - tail_pages), dctx); if (err < 0) goto exit_free_pages; @@ -1687,6 +1867,10 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, } dctx->free_sgout = !!pages; +#ifdef CONFIG_MM_SUBPAGE + /* An error from crypto may still leave writes in the destination. */ + dctx->fragments_dirty = true; +#endif /* Prepare and submit AEAD request */ err = tls_do_decryption(sk, sgin, sgout, dctx->iv, data_len + prot->tail_size, aead_req, darg); @@ -1715,6 +1899,12 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, darg->tail = dctx->tail; exit_free_pages: +#ifdef CONFIG_MM_SUBPAGE + if (dctx->fragments) { + tls_release_user_fragments(dctx); + pages = 0; + } +#endif /* Release the pages in case iov was mapped to pages */ for (; pages > 0; pages--) put_page(sg_page(&sgout[pages])); @@ -2795,6 +2985,9 @@ static struct tls_sw_context_rx *init_ctx_rx(struct tls_context *ctx) crypto_init_wait(&sw_ctx_rx->async_wait); atomic_set(&sw_ctx_rx->decrypt_pending, 1); +#ifdef CONFIG_MM_SUBPAGE + init_llist_head(&sw_ctx_rx->subpage_cleanup); +#endif init_waitqueue_head(&sw_ctx_rx->wq); skb_queue_head_init(&sw_ctx_rx->rx_list); skb_queue_head_init(&sw_ctx_rx->async_hold); @@ -2903,7 +3096,24 @@ int tls_set_sw_offload(struct sock *sk, int tx, rec_seq = crypto_info_rec_seq(src_crypto_info, cipher_desc); if (!*aead) { - *aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0); +#if IS_ENABLED(CONFIG_KUNIT) && IS_ENABLED(CONFIG_MM_SUBPAGE) + if (!tx && user4k_test_async) { + struct crypto_aead *sync; + char name[CRYPTO_MAX_ALG_NAME]; + + /* cryptd instances are named for the concrete child driver. */ + sync = crypto_alloc_aead(cipher_desc->cipher_name, 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(sync)) { + *aead = sync; + } else { + snprintf(name, sizeof(name), "cryptd(%s)", + crypto_tfm_alg_driver_name(crypto_aead_tfm(sync))); + *aead = crypto_alloc_aead(name, CRYPTO_ALG_ASYNC, CRYPTO_ALG_ASYNC); + crypto_free_aead(sync); + } + } else +#endif + *aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0); if (IS_ERR(*aead)) { rc = PTR_ERR(*aead); *aead = NULL; From 332aad3d4111a6f82c53dfa190fa2bb1b62ae1b9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0577/1579] iommu: carry exact user fragments through device mappings Convert IOMMUFD, VFIO and GEM/DMA-BUF paths to preserve the backing offset, mapping span and ownership of smaller user pages. Signed-off-by: Diljit Singh (cherry picked from commit 25dc8bcddb0298f46cbe5d335f6b56388e43e198) --- drivers/dma-buf/dma-buf.c | 35 +- drivers/gpu/drm/drm_gem.c | 7 +- drivers/gpu/drm/drm_gem_shmem_helper.c | 32 +- drivers/iommu/iommufd/Makefile | 2 + drivers/iommu/iommufd/device.c | 102 +++- drivers/iommu/iommufd/fragments.c | 421 +++++++++++++++ drivers/iommu/iommufd/io_pagetable.c | 15 +- drivers/iommu/iommufd/io_pagetable.h | 77 ++- drivers/iommu/iommufd/iommufd_private.h | 4 + drivers/iommu/iommufd/iova_bitmap.c | 131 ++++- drivers/iommu/iommufd/pages.c | 99 +++- drivers/iommu/iommufd/selftest.c | 61 ++- drivers/iommu/iommufd/viommu.c | 56 +- drivers/vfio/vfio_iommu_type1.c | 534 +++++++++++++------ drivers/vfio/vfio_iommu_type1_fragments.c | 609 ++++++++++++++++++++++ 15 files changed, 1918 insertions(+), 267 deletions(-) create mode 100644 drivers/iommu/iommufd/fragments.c create mode 100644 drivers/vfio/vfio_iommu_type1_fragments.c diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index d504c636dc294..0cb790ab2b6a3 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -225,9 +225,23 @@ static struct file_system_type dma_buf_fs_type = { .kill_sb = kill_anon_super, }; +static int dma_buf_mmap_range_check(struct dma_buf *dmabuf, + struct vm_area_struct *vma, unsigned long pgoff, unsigned int suboffset) +{ + unsigned long offset; + + if (check_mul_overflow(pgoff, PAGE_SIZE, &offset) || + check_add_overflow(offset, (unsigned long)suboffset, &offset)) + return -EOVERFLOW; + if (offset > dmabuf->size || vma->vm_end - vma->vm_start > dmabuf->size - offset) + return -EINVAL; + return 0; +} + static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) { struct dma_buf *dmabuf; + int ret; if (!is_dma_buf_file(file)) return -EINVAL; @@ -239,9 +253,9 @@ static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) return -EINVAL; /* check for overflowing the buffer's size */ - if (vma->vm_pgoff + vma_pages(vma) > - dmabuf->size >> PAGE_SHIFT) - return -EINVAL; + ret = dma_buf_mmap_range_check(dmabuf, vma, vma->vm_pgoff, vma_subpage_offset(vma)); + if (ret) + return ret; DMA_BUF_TRACE(trace_dma_buf_mmap_internal, dmabuf); @@ -1583,6 +1597,8 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_end_cpu_access, "DMA_BUF"); int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, unsigned long pgoff) { + int ret; + if (WARN_ON(!dmabuf || !vma)) return -EINVAL; @@ -1590,18 +1606,13 @@ int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, if (!dmabuf->ops->mmap) return -EINVAL; - /* check for offset overflow */ - if (pgoff + vma_pages(vma) < pgoff) - return -EOVERFLOW; - - /* check for overflowing the buffer's size */ - if (pgoff + vma_pages(vma) > - dmabuf->size >> PAGE_SHIFT) - return -EINVAL; + ret = dma_buf_mmap_range_check(dmabuf, vma, pgoff, 0); + if (ret) + return ret; /* readjust the vma */ vma_set_file(vma, dmabuf->file); - vma->vm_pgoff = pgoff; + vma_set_page_offset(vma, (struct vm_page_offset) { .index = pgoff }); DMA_BUF_TRACE(trace_dma_buf_mmap, dmabuf); diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index 3b2448a3a9de8..2176ab547bfb6 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -1343,7 +1343,7 @@ unsigned long drm_gem_get_unmapped_area(struct file *filp, unsigned long uaddr, struct drm_gem_object *obj; unsigned long ret; - obj = drm_gem_object_lookup_at_offset(filp, pgoff, len >> PAGE_SHIFT); + obj = drm_gem_object_lookup_at_offset(filp, pgoff, DIV_ROUND_UP(len, PAGE_SIZE)); if (IS_ERR(obj)) obj = NULL; @@ -1379,8 +1379,11 @@ int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma) struct drm_gem_object *obj; int ret; + /* GEM fake offsets identify an exact object start, not an interior byte. */ + if (vma_subpage_offset(vma)) + return -EINVAL; obj = drm_gem_object_lookup_at_offset(filp, vma->vm_pgoff, - vma_pages(vma)); + DIV_ROUND_UP(vma->vm_end - vma->vm_start, PAGE_SIZE)); if (IS_ERR(obj)) return PTR_ERR(obj); diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 5b8044508cfcf..7d019582c0cc2 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -554,13 +554,23 @@ int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev, } EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create); +/* GEM and PRIME add the object's fake offset; VMA splits retain it. + * Direct helper callers may instead supply an object-relative offset. + */ +static pgoff_t drm_gem_shmem_object_offset(struct drm_gem_object *obj, pgoff_t pgoff) +{ + pgoff_t base = drm_vma_node_start(&obj->vma_node); + + return pgoff >= base ? pgoff - base : pgoff; +} + static void drm_gem_shmem_record_mkwrite(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *obj = vma->vm_private_data; struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); loff_t num_pages = obj->size >> PAGE_SHIFT; - pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */ + pgoff_t page_offset = drm_gem_shmem_object_offset(obj, vmf->pgoff); if (drm_WARN_ON(obj->dev, !shmem->pages || page_offset >= num_pages)) return; @@ -587,7 +597,10 @@ static vm_fault_t try_insert_pfn(struct vm_fault *vmf, unsigned int order, vm_fault_t ret = VM_FAULT_FALLBACK; if (!order) { - ret = vmf_insert_pfn_mkwrite(vmf->vma, vmf->address, pfn, write); + struct vm_page_offset pos = vma_page_offset_at(vmf->vma, vmf->address); + + ret = vmf_insert_pfn_prot_mkwrite_offset(vmf->vma, vmf->address, + pfn, pos.offset, vmf->vma->vm_page_prot, write); #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP } else if (order == PMD_ORDER) { unsigned long paddr = pfn << PAGE_SHIFT; @@ -619,12 +632,12 @@ static vm_fault_t drm_gem_shmem_any_fault(struct vm_fault *vmf, unsigned int ord loff_t num_pages = obj->size >> PAGE_SHIFT; vm_fault_t ret = VM_FAULT_SIGBUS; struct page **pages = shmem->pages; - pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */ + pgoff_t page_offset = drm_gem_shmem_object_offset(obj, vmf->pgoff); struct page *page; struct folio *folio; unsigned long pfn; - if (order && order != PMD_ORDER) + if (order && (order != PMD_ORDER || mm_page_size(vma->vm_mm) != PAGE_SIZE)) return VM_FAULT_FALLBACK; dma_resv_lock(obj->resv, NULL); @@ -742,6 +755,17 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct if (is_cow_mapping(vma->vm_flags)) return -EINVAL; + /* Validate bytes, including a partial native-page offset from PRIME. */ + { + pgoff_t index = drm_gem_shmem_object_offset(obj, vma->vm_pgoff); + unsigned long offset, length = vma->vm_end - vma->vm_start; + + if (check_mul_overflow(index, PAGE_SIZE, &offset) || + check_add_overflow(offset, (unsigned long)vma_subpage_offset(vma), &offset) || + offset > obj->size || length > obj->size - offset) + return -EINVAL; + } + dma_resv_lock(shmem->base.resv, NULL); ret = drm_gem_shmem_get_pages_locked(shmem); dma_resv_unlock(shmem->base.resv); diff --git a/drivers/iommu/iommufd/Makefile b/drivers/iommu/iommufd/Makefile index 71d692c9a8f49..a868155e208d5 100644 --- a/drivers/iommu/iommufd/Makefile +++ b/drivers/iommu/iommufd/Makefile @@ -10,6 +10,8 @@ iommufd-y := \ vfio_compat.o \ viommu.o +iommufd-$(CONFIG_MM_SUBPAGE) += fragments.o + iommufd-$(CONFIG_IOMMUFD_TEST) += selftest.o obj-$(CONFIG_IOMMUFD) += iommufd.o diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c index 2895e5370910f..3ccb352bb3e50 100644 --- a/drivers/iommu/iommufd/device.c +++ b/drivers/iommu/iommufd/device.c @@ -1171,6 +1171,18 @@ struct iommufd_access *iommufd_access_create_internal(struct iommufd_ctx *ictx) return access; } +/* Physical byte-span consumers do not impose native-page IOVA alignment. */ +struct iommufd_access *iommufd_access_create_internal_phys(struct iommufd_ctx *ictx) +{ + struct iommufd_access *access = __iommufd_access_create(ictx); + + if (IS_ERR(access)) + return access; + access->iova_alignment = 1; + iommufd_object_finalize(ictx, &access->obj); + return access; +} + /** * iommufd_access_create - Create an iommufd_access * @ictx: iommufd file descriptor @@ -1404,6 +1416,12 @@ static bool check_area_prot(struct iopt_area *area, unsigned int flags) * ioas alignment is >= PAGE_SIZE and the iova is PAGE_SIZE aligned. However * smaller alignments have corner cases where this API can fail on otherwise * aligned iova. + * + * Alternative user granules are accepted only if each native output page has + * an aligned physical base and consecutive backing bytes within the requested + * range. The driver must access only @length bytes: a partial final page does + * not grant access to its remaining bytes. Fragment ownership stays in the IOAS + * until unpin; incompatible physical layouts return -EINVAL. */ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, unsigned long length, struct page **out_pages, @@ -1456,7 +1474,7 @@ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, flags, internal); if (rc) goto err_remove; - out_pages += last_index - index + 1; + out_pages += (last - iter.cur_iova) / PAGE_SIZE + 1; } if (!iopt_area_contig_done(&iter)) { rc = -ENOENT; @@ -1485,6 +1503,88 @@ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, } EXPORT_SYMBOL_NS_GPL(iommufd_access_pin_pages, "IOMMUFD"); +/** + * iommufd_access_pin_phys - Pin an internal contiguous physical byte span + * @access: Internal access created for byte-span pinning + * @iova: First IOVA byte + * @length: Exact byte length + * @out_phys: Physical address of the first byte, set only on success + * + * Retains native or fragment pin ownership and prevents IOAS unmap until the + * matching iommufd_access_unpin_pages(). No surrounding IOVA bytes are required. + * The caller must stop hardware access before unpinning or destroying access. + */ +int iommufd_access_pin_phys(struct iommufd_access *access, unsigned long iova, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_area_contig_iter iter; + struct io_pagetable *iopt; + struct iopt_area *area; + unsigned long last_iova; + phys_addr_t base = 0; + int rc; + + if (!iommufd_access_is_internal(access) || !out_phys || !length) + return -EINVAL; + if (check_add_overflow(iova, length - 1, &last_iova)) + return -EOVERFLOW; + + mutex_lock(&access->ioas_lock); + if (!access->ioas) { + rc = -ENOENT; + goto out_unlock; + } + iopt = &access->ioas->iopt; + down_read(&iopt->iova_rwsem); + iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) { + unsigned long last = min(last_iova, iopt_area_last_iova(area)); + phys_addr_t phys; + + if (area->prevent_access || !(area->iommu_prot & IOMMU_READ)) { + rc = -EPERM; + goto out_remove; + } + rc = iopt_area_add_phys_access(area, + iopt_area_start_byte(area, iter.cur_iova), + last - iter.cur_iova + 1, &phys); + if (rc) + goto out_remove; + if (iter.cur_iova == iova) { + base = phys; + } else if (phys != base + (iter.cur_iova - iova)) { + iopt_area_remove_access(area, + iopt_area_iova_to_index(area, iter.cur_iova), + iopt_area_iova_to_index(area, last), true); + rc = -EFAULT; + goto out_remove; + } + } + if (!iopt_area_contig_done(&iter)) { + rc = -ENOENT; + goto out_remove; + } + *out_phys = base; + up_read(&iopt->iova_rwsem); + rc = 0; + goto out_unlock; + +out_remove: + /* iter.cur_iova is the first byte whose access was not retained. */ + if (iter.cur_iova > iova) { + unsigned long end = iter.cur_iova - 1; + + iopt_for_each_contig_area(&iter, area, iopt, iova, end) + iopt_area_remove_access(area, + iopt_area_iova_to_index(area, iter.cur_iova), + iopt_area_iova_to_index(area, min(end, iopt_area_last_iova(area))), + true); + } + up_read(&iopt->iova_rwsem); +out_unlock: + mutex_unlock(&access->ioas_lock); + return rc; +} + /** * iommufd_access_rw - Read or write data under the iova * @access: IOAS access to act on diff --git a/drivers/iommu/iommufd/fragments.c b/drivers/iommu/iommufd/fragments.c new file mode 100644 index 0000000000000..5229605852133 --- /dev/null +++ b/drivers/iommu/iommufd/fragments.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Alternative user granules need an ownership record in addition to a physical + * address. A device translation cannot retain a subpage allocation identity. + * Keep those records until the final domain interval has gone away. The native + * PFN reader remains in pages.c; file and dma-buf suppliers retain its geometry. + */ +#include +#include +#include +#include + +#include "iommufd_private.h" +#include "io_pagetable.h" + +static phys_addr_t fragment_phys(const struct user_page_fragment *fragment) +{ + return PFN_PHYS(folio_pfn(fragment->folio)) + fragment->offset; +} + +/* Charge each native physical page once, even when several slots pin it. */ +static int fragment_store(struct iopt_pages *pages, unsigned long index, + struct user_page_fragment *fragment) +{ + unsigned long pfn = PHYS_PFN(fragment_phys(fragment)); + unsigned long count = xa_to_value(xa_load(&pages->fragment_pages, pfn)); + struct user_page_fragment *record; + int rc; + + record = kmemdup(fragment, sizeof(*record), GFP_KERNEL_ACCOUNT); + if (!record) + return -ENOMEM; + rc = xa_reserve(&pages->fragments, index, GFP_KERNEL_ACCOUNT); + if (rc) + goto out_free; + rc = xa_err(xa_store(&pages->fragment_pages, pfn, xa_mk_value(count + 1), + GFP_KERNEL_ACCOUNT)); + if (rc) + goto out_release; + /* xa_reserve() made this infallible. Ownership transfers here. */ + xa_store(&pages->fragments, index, record, GFP_NOWAIT); + if (!count) + pages->npinned++; + *fragment = (struct user_page_fragment) {}; + return 0; + +out_release: + xa_release(&pages->fragments, index); +out_free: + kfree(record); + return rc; +} + +/* No allocation on teardown, and callers have already unmapped all devices. */ +static void fragments_release_unused(struct iopt_pages *pages, + unsigned long first, unsigned long last) +{ + struct user_page_fragment *record; + unsigned long index = first; + + lockdep_assert_held(&pages->mutex); + xa_for_each_range(&pages->fragments, index, record, first, last) { + unsigned long pfn, count; + + if (interval_tree_iter_first(&pages->domains_itree, index, index) || + interval_tree_iter_first(&pages->access_itree, index, index)) + continue; + pfn = PHYS_PFN(fragment_phys(record)); + count = xa_to_value(xa_load(&pages->fragment_pages, pfn)); + if (WARN_ON(!count)) + continue; + if (count == 1) { + xa_erase(&pages->fragment_pages, pfn); + pages->npinned--; + } else { + xa_store(&pages->fragment_pages, pfn, xa_mk_value(count - 1), + GFP_NOWAIT); + } + xa_erase(&pages->fragments, index); + release_user_fragments(record, 1, pages->writable); + kfree(record); + cond_resched(); + } + if (pages->npinned < pages->last_npinned) + iopt_pages_update_pinned(pages, pages->last_npinned - pages->npinned, + false, NULL); +} + +/* Reuse existing records even if the original VA was unmapped or replaced. */ +static int fragments_acquire(struct iopt_pages *pages, unsigned long first, + unsigned long last) +{ + unsigned long size = 1UL << pages->page_shift, index = first; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + struct user_page_fragment *batch = NULL; + bool mm_live = false; + int rc = 0; + + lockdep_assert_held(&pages->mutex); + while (index <= last) { + unsigned long next = index, end, i; + long count; + + if (xa_load(&pages->fragments, index)) { + index++; + continue; + } + if (!mm_live) { + if (!mmget_not_zero(pages->source_mm)) { + rc = -EFAULT; + goto out; + } + mm_live = true; + batch = kmalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!batch) { + rc = -ENOMEM; + goto out; + } + } + /* Stop before any cached pin; never replace its ownership record. */ + end = xa_find(&pages->fragments, &next, last, XA_PRESENT) ? next : last + 1; + end = min(end, index + capacity); + count = pin_user_fragments_remote(pages->source_mm, + (unsigned long)pages->uptr + (index << pages->page_shift), + (end - index) << pages->page_shift, + FOLL_LONGTERM | (pages->writable ? FOLL_WRITE : 0), batch, capacity); + if (count <= 0) { + rc = count ?: -EFAULT; + goto out; + } + for (i = 0; i < count; i++) { + /* Each aligned logical index must have exactly one full pin. */ + if (WARN_ON(batch[i].length != size || + !IS_ALIGNED(fragment_phys(&batch[i]), size))) + rc = -EFAULT; + else + rc = fragment_store(pages, index, &batch[i]); + if (rc) { + release_user_fragments(batch + i, count - i, false); + goto out; + } + index++; + } + cond_resched(); + } + if (pages->npinned != pages->last_npinned) { + if (iommufd_should_fail()) + rc = -ENOMEM; + else + rc = iopt_pages_update_pinned(pages, + pages->npinned - pages->last_npinned, true, NULL); + } +out: + kfree(batch); + if (mm_live) + mmput(pages->source_mm); + if (rc) + fragments_release_unused(pages, first, last); + return rc; +} + +static void fragments_unmap(struct iommu_domain *domain, unsigned long iova, + unsigned long length) +{ + if (length) + WARN_ON(iommu_unmap(domain, iova, length) != length); +} + +/* All pins are present. Merge physically consecutive fragments before mapping. */ +static int fragments_map(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + unsigned long size = 1UL << pages->page_shift; + unsigned long index = iopt_area_index(area), last = iopt_area_last_index(area); + unsigned long remaining = iopt_area_length(area), done = 0; + unsigned long offset = area->page_offset; + unsigned long min_size = 1UL << __ffs(domain->pgsize_bitmap); + int rc; + + while (remaining) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + phys_addr_t phys = fragment_phys(record) + offset; + unsigned long bytes = min(size - offset, remaining); + + index++; + /* Legacy splitting forbids mappings larger than a native page. */ + while (index <= last && bytes < remaining && + (!area->iopt->disable_large_pages || bytes < PAGE_SIZE)) { + record = xa_load(&pages->fragments, index); + if (fragment_phys(record) != phys + bytes) + break; + bytes += min(size, remaining - bytes); + index++; + } + /* Never widen a device mapping to include an unowned neighbour. */ + if (!IS_ALIGNED(phys | (iopt_area_iova(area) + done) | bytes, min_size)) { + rc = -EINVAL; + goto out_unmap; + } + rc = iommu_map(domain, iopt_area_iova(area) + done, phys, bytes, + area->iommu_prot, GFP_KERNEL_ACCOUNT); + if (rc) + goto out_unmap; + done += bytes; + remaining -= bytes; + offset = 0; + cond_resched(); + } + return 0; + +out_unmap: + fragments_unmap(domain, iopt_area_iova(area), done); + return rc; +} + +int iopt_fragments_fill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + int rc; + + lockdep_assert_held(&pages->mutex); + rc = fragments_acquire(pages, iopt_area_index(area), iopt_area_last_index(area)); + if (rc) + return rc; + rc = fragments_map(area, pages, domain); + if (rc) + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); + return rc; +} + +int iopt_fragments_fill_domains(struct iopt_area *area, struct iopt_pages *pages) +{ + struct iommu_domain *domain; + unsigned long index, failed; + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, iopt_area_index(area), iopt_area_last_index(area)); + if (rc) + return rc; + xa_for_each(&area->iopt->domains, index, domain) { + rc = fragments_map(area, pages, domain); + if (rc) + goto out_unmap; + } + area->storage_domain = xa_load(&area->iopt->domains, 0); + interval_tree_insert(&area->pages_node, &pages->domains_itree); + return 0; + +out_unmap: + failed = index; + xa_for_each(&area->iopt->domains, index, domain) { + if (index == failed) + break; + fragments_unmap(domain, iopt_area_iova(area), iopt_area_length(area)); + } + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); + return rc; +} + +void iopt_fragments_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + lockdep_assert_held(&pages->mutex); + fragments_unmap(domain, iopt_area_iova(area), iopt_area_length(area)); + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); +} + +int iopt_fragments_rw(struct iopt_pages *pages, unsigned long start_byte, + void *data, unsigned long length, unsigned int flags) +{ + unsigned long size = 1UL << pages->page_shift; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + unsigned long index, offset = start_byte & (size - 1); + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + for (index = first; index <= last; index++) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + unsigned long bytes = min(size - offset, length); + void *addr = kmap_local_folio(record->folio, record->offset + offset); + + if (flags & IOMMUFD_ACCESS_RW_WRITE) + memcpy(addr, data, bytes); + else + memcpy(data, addr, bytes); + kunmap_local(addr); + data += bytes; + length -= bytes; + offset = 0; + } + fragments_release_unused(pages, first, last); + return 0; +} + +/* The retained typed records, not the output page array, own the pins. */ +static int fragments_register_access(struct iopt_area *area, unsigned long first, + unsigned long last, bool lock_area) +{ + struct iopt_pages *pages = area->pages; + struct iopt_pages_access *access; + struct interval_tree_node *node; + + lockdep_assert_held(&pages->mutex); + for (node = interval_tree_iter_first(&pages->access_itree, first, last); + node; node = interval_tree_iter_next(node, first, last)) { + if (node->start == first && node->last == last) { + access = container_of(node, struct iopt_pages_access, node); + access->users++; + goto out_count; + } + } + access = kzalloc_obj(*access, GFP_KERNEL_ACCOUNT); + if (!access) + return -ENOMEM; + access->node.start = first; + access->node.last = last; + access->users = 1; + interval_tree_insert(&access->node, &pages->access_itree); +out_count: + area->num_accesses++; + if (lock_area) + area->num_locks++; + return 0; +} + +/* Register a physical byte span without discarding fragment ownership. */ +int iopt_fragments_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_pages *pages = area->pages; + unsigned long size = 1UL << pages->page_shift; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + struct user_page_fragment *record; + unsigned long index; + phys_addr_t base; + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + record = xa_load(&pages->fragments, first); + base = fragment_phys(record); + for (index = first + 1; index <= last; index++) { + record = xa_load(&pages->fragments, index); + if (fragment_phys(record) != + base + ((index - first) << pages->page_shift)) { + rc = -EFAULT; + goto out_release; + } + cond_resched(); + } + rc = fragments_register_access(area, first, last, true); + if (rc) + goto out_release; + *out_phys = base + (start_byte & (size - 1)); + return 0; + +out_release: + fragments_release_unused(pages, first, last); + return rc; +} + +/* + * A native page pointer has no fragment offset. Require an aligned physical + * base and consecutive fragments within each output page; different output + * pages may be scattered. Only the requested logical interval is retained. + */ +int iopt_fragments_add_access(struct iopt_area *area, unsigned long first, + unsigned long last, struct page **out_pages, + bool lock_area) +{ + struct iopt_pages *pages = area->pages; + unsigned long per_page = 1UL << (PAGE_SHIFT - pages->page_shift); + unsigned long index; + phys_addr_t base = 0; + int rc; + + if (first & (per_page - 1)) + return -EINVAL; + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + for (index = first; index <= last; index++) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + unsigned long offset = (index - first) & (per_page - 1); + phys_addr_t phys = fragment_phys(record); + + if (!offset) { + base = phys; + if (!IS_ALIGNED(base, PAGE_SIZE)) { + rc = -EINVAL; + goto out_release; + } + *out_pages++ = pfn_to_page(PHYS_PFN(base)); + } else if (phys != base + (offset << pages->page_shift)) { + rc = -EINVAL; + goto out_release; + } + cond_resched(); + } + rc = fragments_register_access(area, first, last, lock_area); + if (!rc) + return 0; +out_release: + fragments_release_unused(pages, first, last); + return rc; +} + +void iopt_fragments_unfill_access(struct iopt_pages *pages, unsigned long first, + unsigned long last) +{ + fragments_release_unused(pages, first, last); +} diff --git a/drivers/iommu/iommufd/io_pagetable.c b/drivers/iommu/iommufd/io_pagetable.c index 24d4917105d94..bf8cabee53fb6 100644 --- a/drivers/iommu/iommufd/io_pagetable.c +++ b/drivers/iommu/iommufd/io_pagetable.c @@ -217,7 +217,8 @@ static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area, return -EPERM; area->iommu_prot = iommu_prot; - area->page_offset = start_byte % PAGE_SIZE; + area->page_shift = pages->page_shift; + area->page_offset = start_byte & ((1UL << area->page_shift) - 1); if (area->page_offset & (iopt->iova_alignment - 1)) return -EINVAL; @@ -225,10 +226,10 @@ static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area, if (check_add_overflow(iova, length - 1, &area->node.last)) return -EOVERFLOW; - area->pages_node.start = start_byte / PAGE_SIZE; + area->pages_node.start = start_byte >> area->page_shift; if (check_add_overflow(start_byte, length - 1, &area->pages_node.last)) return -EOVERFLOW; - area->pages_node.last = area->pages_node.last / PAGE_SIZE; + area->pages_node.last >>= area->page_shift; if (WARN_ON(area->pages_node.last >= pages->npages)) return -EOVERFLOW; @@ -572,7 +573,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, struct iommu_dirty_bitmap dirty; struct iova_bitmap_fn_arg arg; struct iova_bitmap *iter; - int ret = 0; + int ret; if (!ops || !ops->read_and_clear_dirty) return -EOPNOTSUPP; @@ -581,7 +582,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, bitmap->page_size, u64_to_user_ptr(bitmap->data)); if (IS_ERR(iter)) - return -ENOMEM; + return PTR_ERR(iter); iommu_dirty_bitmap_init(&dirty, iter, &gather); @@ -589,7 +590,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, arg.iopt = iopt; arg.domain = domain; arg.dirty = &dirty; - iova_bitmap_for_each(iter, &arg, __iommu_read_and_clear_dirty); + ret = iova_bitmap_for_each(iter, &arg, __iommu_read_and_clear_dirty); if (!(flags & IOMMU_DIRTY_NO_CLEAR)) iommu_iotlb_sync(domain, &gather); @@ -615,7 +616,7 @@ int iommufd_check_iova_range(struct io_pagetable *iopt, ((last_iova + 1) & (iommu_pgsize - 1))) return -EINVAL; - if (!bitmap->page_size) + if (!is_power_of_2(bitmap->page_size)) return -EINVAL; if ((bitmap->iova & (bitmap->page_size - 1)) || diff --git a/drivers/iommu/iommufd/io_pagetable.h b/drivers/iommu/iommufd/io_pagetable.h index 27e3e311d395b..d15e51fee0c10 100644 --- a/drivers/iommu/iommufd/io_pagetable.h +++ b/drivers/iommu/iommufd/io_pagetable.h @@ -5,6 +5,7 @@ #ifndef __IO_PAGETABLE_H #define __IO_PAGETABLE_H +#include #include #include #include @@ -45,6 +46,7 @@ struct iopt_area { struct iommu_domain *storage_domain; /* How many bytes into the first page the area starts */ unsigned int page_offset; + u8 page_shift; /* IOMMU_READ, IOMMU_WRITE, etc */ int iommu_prot; bool prevent_access : 1; @@ -107,8 +109,8 @@ static inline size_t iopt_area_length(struct iopt_area *area) /* * Number of bytes from the start of the iopt_pages that the iova begins. - * iopt_area_start_byte() / PAGE_SIZE encodes the starting page index - * iopt_area_start_byte() % PAGE_SIZE encodes the offset within that page + * iopt_area_start_byte() >> area->page_shift encodes the starting page index + * iopt_area_start_byte() % (1UL << area->page_shift) encodes the offset within that page */ static inline unsigned long iopt_area_start_byte(struct iopt_area *area, unsigned long iova) @@ -117,13 +119,13 @@ static inline unsigned long iopt_area_start_byte(struct iopt_area *area, WARN_ON(iova < iopt_area_iova(area) || iova > iopt_area_last_iova(area)); return (iova - iopt_area_iova(area)) + area->page_offset + - iopt_area_index(area) * PAGE_SIZE; + (iopt_area_index(area) << area->page_shift); } static inline unsigned long iopt_area_iova_to_index(struct iopt_area *area, unsigned long iova) { - return iopt_area_start_byte(area, iova) / PAGE_SIZE; + return iopt_area_start_byte(area, iova) >> area->page_shift; } #define __make_iopt_iter(name) \ @@ -214,7 +216,7 @@ struct iopt_pages_dmabuf { * io_pagetable's, through their iopt_area's, can share a single iopt_pages * which avoids multi-pinning and double accounting of page consumption. * - * indexes in this structure are measured in PAGE_SIZE units, are 0 based from + * Indexes use (1UL << page_shift) units and are 0 based from * the start of the uptr and extend to npages. pages are pinned dynamically * according to the intervals in the access_itree and domains_itree, npinned * records the current number of pages pinned. @@ -222,6 +224,8 @@ struct iopt_pages_dmabuf { struct iopt_pages { struct kref kref; struct mutex mutex; + /* Logical indices use page_shift; pin accounting remains native pages. */ + u8 page_shift; size_t npages; size_t npinned; size_t last_npinned; @@ -242,12 +246,73 @@ struct iopt_pages { u8 account_mode; struct xarray pinned_pfns; +#ifdef CONFIG_MM_SUBPAGE + /* Exact fragment pins, indexed independently of device translations. */ + struct xarray fragments; + /* Native physical PFN -> fragment count, for pin accounting. */ + struct xarray fragment_pages; +#endif /* Of iopt_pages_access::node */ struct rb_root_cached access_itree; /* Of iopt_area::pages_node */ struct rb_root_cached domains_itree; }; +static inline bool iopt_pages_use_fragments(struct iopt_pages *pages) +{ + return IS_ENABLED(CONFIG_MM_SUBPAGE) && pages->type == IOPT_ADDRESS_USER && + mm_page_size(pages->source_mm) != PAGE_SIZE; +} + +#ifdef CONFIG_MM_SUBPAGE +int iopt_fragments_fill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain); +int iopt_fragments_fill_domains(struct iopt_area *area, struct iopt_pages *pages); +void iopt_fragments_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain); +int iopt_fragments_add_access(struct iopt_area *area, unsigned long first, + unsigned long last, struct page **out_pages, + bool lock_area); +int iopt_fragments_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys); +void iopt_fragments_unfill_access(struct iopt_pages *pages, unsigned long first, + unsigned long last); +int iopt_fragments_rw(struct iopt_pages *pages, unsigned long start_byte, + void *data, unsigned long length, unsigned int flags); +#else +static inline int iopt_fragments_fill_domain(struct iopt_area *area, + struct iopt_pages *pages, struct iommu_domain *domain) +{ + return -EOPNOTSUPP; +} +static inline int iopt_fragments_fill_domains(struct iopt_area *area, + struct iopt_pages *pages) +{ + return -EOPNOTSUPP; +} +static inline void iopt_fragments_unfill_domain(struct iopt_area *area, + struct iopt_pages *pages, struct iommu_domain *domain) { } +static inline int iopt_fragments_add_access(struct iopt_area *area, + unsigned long first, unsigned long last, struct page **out_pages, + bool lock_area) +{ + return -EOPNOTSUPP; +} +static inline int iopt_fragments_add_phys_access(struct iopt_area *area, + unsigned long start_byte, unsigned long length, phys_addr_t *out_phys) +{ + return -EOPNOTSUPP; +} +static inline void iopt_fragments_unfill_access(struct iopt_pages *pages, + unsigned long first, unsigned long last) { } +static inline int iopt_fragments_rw(struct iopt_pages *pages, + unsigned long start_byte, void *data, unsigned long length, + unsigned int flags) +{ + return -EOPNOTSUPP; +} +#endif + static inline bool iopt_is_dmabuf(struct iopt_pages *pages) { if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) @@ -290,6 +355,8 @@ void iopt_pages_unfill_xarray(struct iopt_pages *pages, unsigned long start, int iopt_area_add_access(struct iopt_area *area, unsigned long start, unsigned long last, struct page **out_pages, unsigned int flags, bool lock_area); +int iopt_area_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys); void iopt_area_remove_access(struct iopt_area *area, unsigned long start, unsigned long last, bool unlock_area); int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h index 43fbc5bed8de3..d1a14e51df55e 100644 --- a/drivers/iommu/iommufd/iommufd_private.h +++ b/drivers/iommu/iommufd/iommufd_private.h @@ -531,6 +531,10 @@ static inline bool iommufd_access_is_internal(struct iommufd_access *access) } struct iommufd_access *iommufd_access_create_internal(struct iommufd_ctx *ictx); +struct iommufd_access *iommufd_access_create_internal_phys(struct iommufd_ctx *ictx); +int iommufd_access_pin_phys(struct iommufd_access *access, unsigned long iova, + unsigned long length, phys_addr_t *out_phys); + static inline void iommufd_access_destroy_internal(struct iommufd_ctx *ictx, diff --git a/drivers/iommu/iommufd/iova_bitmap.c b/drivers/iommu/iommufd/iova_bitmap.c index dac3e657d498d..12e1dd035871a 100644 --- a/drivers/iommu/iommufd/iova_bitmap.c +++ b/drivers/iommu/iommufd/iova_bitmap.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #define BITS_PER_PAGE (PAGE_SIZE * BITS_PER_BYTE) @@ -44,11 +45,16 @@ struct iova_bitmap_map { /* page offset of the first user page pinned */ unsigned long pgoff; - /* number of pages pinned */ + /* number of native pages or typed fragments pinned */ unsigned long npages; /* pinned pages representing the bitmap data */ struct page **pages; +#ifdef CONFIG_MM_SUBPAGE + /* Exact byte ranges, including offsets unrelated to the user VA. */ + struct user_page_fragment *fragments; + unsigned long pinned_bytes; +#endif }; /* @@ -63,7 +69,10 @@ struct iova_bitmap_map { * The bitmap object uses one base page to store all the pinned pages * pointers related to the bitmap. For sizeof(struct page*) == 8 it stores * 512 struct page pointers which, if the base page size is 4K, it means - * 2M of bitmap data is pinned at a time. If the iova_bitmap page size is + * 2M of bitmap data is pinned at a time. Alternative-granule processes use + * one native page of fragment descriptors instead, retaining each physical + * byte offset and bounding the window by the sum of fragment lengths. + * If the iova_bitmap page size is * also 4K then the range window to iterate is 64G. * * For example iterating on a total IOVA range of 4G..128G, it will walk @@ -116,6 +125,9 @@ struct iova_bitmap { /* length of the IOVA range for the whole bitmap */ size_t length; + + /* First lazy pin/write error during an iteration. */ + int error; }; /* @@ -192,6 +204,26 @@ static int iova_bitmap_get(struct iova_bitmap *bitmap) npages = min(npages + !!offset_in_page(addr), PAGE_SIZE / sizeof(struct page *)); +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + unsigned long bytes = bitmap->mapped_total_index - bitmap->mapped_base_index; + unsigned long i; + + ret = pin_user_fragments_remote(current->mm, (unsigned long)addr, + bytes, FOLL_WRITE, mapped->fragments, + PAGE_SIZE / sizeof(*mapped->fragments)); + if (ret <= 0) + return -EFAULT; + mapped->npages = ret; + mapped->pinned_bytes = 0; + for (i = 0; i < mapped->npages; i++) + mapped->pinned_bytes += mapped->fragments[i].length; + mapped->pgoff = 0; + mapped->iova = iova_bitmap_mapped_iova(bitmap); + mapped->length = iova_bitmap_mapped_length(bitmap); + return 0; + } +#endif ret = pin_user_pages_fast((unsigned long)addr, npages, FOLL_WRITE, mapped->pages); if (ret <= 0) @@ -221,7 +253,13 @@ static void iova_bitmap_put(struct iova_bitmap *bitmap) struct iova_bitmap_map *mapped = &bitmap->mapped; if (mapped->npages) { - unpin_user_pages(mapped->pages, mapped->npages); +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + release_user_fragments(mapped->fragments, mapped->npages, true); + mapped->pinned_bytes = 0; + } else +#endif + unpin_user_pages_dirty_lock(mapped->pages, mapped->npages, true); mapped->npages = 0; } } @@ -259,6 +297,16 @@ struct iova_bitmap *iova_bitmap_alloc(unsigned long iova, size_t length, bitmap->iova = iova; bitmap->length = length; mapped->iova = iova; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + mapped->fragments = (void *)__get_free_page(GFP_KERNEL); + if (!mapped->fragments) { + rc = -ENOMEM; + goto err; + } + return bitmap; + } +#endif mapped->pages = (struct page **)__get_free_page(GFP_KERNEL); if (!mapped->pages) { rc = -ENOMEM; @@ -291,6 +339,10 @@ void iova_bitmap_free(struct iova_bitmap *bitmap) mapped->pages = NULL; } +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) + free_page((unsigned long)mapped->fragments); +#endif kfree(bitmap); } EXPORT_SYMBOL_NS_GPL(iova_bitmap_free, "IOMMUFD"); @@ -303,7 +355,12 @@ static unsigned long iova_bitmap_mapped_remaining(struct iova_bitmap *bitmap) { unsigned long remaining, bytes; - bytes = (bitmap->mapped.npages << PAGE_SHIFT) - bitmap->mapped.pgoff; +#ifdef CONFIG_MM_SUBPAGE + if (bitmap->mapped.fragments) + bytes = bitmap->mapped.pinned_bytes; + else +#endif + bytes = (bitmap->mapped.npages << PAGE_SHIFT) - bitmap->mapped.pgoff; remaining = bitmap->mapped_total_index - bitmap->mapped_base_index; remaining = min_t(unsigned long, remaining, @@ -356,16 +413,21 @@ static int iova_bitmap_advance_to(struct iova_bitmap *bitmap, unsigned long iova) { unsigned long index; + int ret = -EINVAL; index = iova_bitmap_offset_to_index(bitmap, iova - bitmap->iova); if (index >= bitmap->mapped_total_index) - return -EINVAL; + goto out; bitmap->mapped_base_index = index; iova_bitmap_put(bitmap); /* Pin the next set of bitmap pages */ - return iova_bitmap_get(bitmap); + ret = iova_bitmap_get(bitmap); +out: + if (ret && !bitmap->error) + bitmap->error = ret; + return ret; } /** @@ -379,15 +441,58 @@ static int iova_bitmap_advance_to(struct iova_bitmap *bitmap, * mapped bitmap user pages into IOVA ranges to process. * * Return: 0 on success, and an error on failure either upon - * iteration or when the callback returns an error. + * iteration, a lazy bitmap write, or when the callback returns an error. */ int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque, iova_bitmap_fn_t fn) { - return fn(bitmap, bitmap->iova, bitmap->length, opaque); + int ret; + + bitmap->error = 0; + ret = fn(bitmap, bitmap->iova, bitmap->length, opaque); + return ret ?: bitmap->error; } EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, "IOMMUFD"); +#ifdef CONFIG_MM_SUBPAGE +static void iova_bitmap_set_fragments(struct iova_bitmap *bitmap, + unsigned long iova, size_t length) +{ + struct iova_bitmap_map *mapped = &bitmap->mapped; + unsigned long bit = (iova - bitmap->iova) >> mapped->pgshift; + unsigned long end = ((iova + length - 1 - bitmap->iova) >> mapped->pgshift) + 1; + + while (bit < end) { + unsigned long base, relative, i; + + iova = bitmap->iova + (bit << mapped->pgshift); + if (!iova_bitmap_mapped_range(mapped, iova, 1) && + iova_bitmap_advance_to(bitmap, iova)) + return; + base = bitmap->mapped_base_index * BITS_PER_BYTE; + relative = bit - base; + for (i = 0; i < mapped->npages && bit < end; i++) { + struct user_page_fragment *fragment = &mapped->fragments[i]; + unsigned long bits = fragment->length * BITS_PER_BYTE; + unsigned long count; + void *addr; + + if (relative >= bits) { + relative -= bits; + continue; + } + count = min(end - bit, bits - relative); + addr = kmap_local_folio(fragment->folio, fragment->offset & PAGE_MASK); + bitmap_set(addr, offset_in_page(fragment->offset) * BITS_PER_BYTE + + relative, count); + kunmap_local(addr); + bit += count; + relative = 0; + } + } +} +#endif + /** * iova_bitmap_set() - Records an IOVA range in bitmap * @bitmap: IOVA bitmap @@ -395,7 +500,8 @@ EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, "IOMMUFD"); * @length: IOVA range length * * Set the bits corresponding to the range [iova .. iova+length-1] in - * the user bitmap. + * the user bitmap. A lazy pin failure is retained for the enclosing + * iova_bitmap_for_each() to report; successful prefix writes are not undone. * */ void iova_bitmap_set(struct iova_bitmap *bitmap, @@ -404,6 +510,13 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, struct iova_bitmap_map *mapped = &bitmap->mapped; unsigned long cur_bit, last_bit, last_page_idx; +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + iova_bitmap_set_fragments(bitmap, iova, length); + return; + } +#endif + update_indexes: if (unlikely(!iova_bitmap_mapped_range(mapped, iova, length))) { /* diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c index 404f31d8f7291..0839878bfb67c 100644 --- a/drivers/iommu/iommufd/pages.c +++ b/drivers/iommu/iommufd/pages.c @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -207,7 +208,7 @@ static unsigned long iopt_area_index_to_iova(struct iopt_area *area, index -= iopt_area_index(area); if (index == 0) return iopt_area_iova(area); - return iopt_area_iova(area) - area->page_offset + index * PAGE_SIZE; + return iopt_area_iova(area) - area->page_offset + (index << area->page_shift); } static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area, @@ -219,7 +220,7 @@ static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area, if (index == iopt_area_last_index(area)) return iopt_area_last_iova(area); return iopt_area_iova(area) - area->page_offset + - (index - iopt_area_index(area) + 1) * PAGE_SIZE - 1; + ((index - iopt_area_index(area) + 1) << area->page_shift) - 1; } static void iommu_unmap_nofail(struct iommu_domain *domain, unsigned long iova, @@ -993,9 +994,11 @@ static int update_mm_locked_vm(struct iopt_pages *pages, unsigned long npages, } else if ((!user || (!user->upages && !user->ufolios)) && pages->source_mm != current->mm) { - if (!mmget_not_zero(pages->source_mm)) + if (mmget_not_zero(pages->source_mm)) + do_put = true; + else if (inc) return -EINVAL; - do_put = true; + /* mmgrab() keeps the lock and counters alive for final uncharge. */ } mmap_write_lock(pages->source_mm); @@ -1378,10 +1381,15 @@ static struct iopt_pages *iopt_alloc_pages(unsigned long start_byte, kref_init(&pages->kref); xa_init_flags(&pages->pinned_pfns, XA_FLAGS_ACCOUNT); +#ifdef CONFIG_MM_SUBPAGE + xa_init_flags(&pages->fragments, XA_FLAGS_ACCOUNT); + xa_init_flags(&pages->fragment_pages, XA_FLAGS_ACCOUNT); +#endif + pages->page_shift = PAGE_SHIFT; mutex_init(&pages->mutex); pages->source_mm = current->mm; mmgrab(pages->source_mm); - pages->npages = DIV_ROUND_UP(length + start_byte, PAGE_SIZE); + pages->npages = ((length - 1 + start_byte) >> PAGE_SHIFT) + 1; pages->access_itree = RB_ROOT_CACHED; pages->domains_itree = RB_ROOT_CACHED; pages->writable = writable; @@ -1400,8 +1408,10 @@ struct iopt_pages *iopt_alloc_user_pages(void __user *uptr, { struct iopt_pages *pages; unsigned long end; + unsigned int shift = min_t(unsigned int, mm_page_shift(current->mm), PAGE_SHIFT); + unsigned long granule = 1UL << shift; void __user *uptr_down = - (void __user *)ALIGN_DOWN((uintptr_t)uptr, PAGE_SIZE); + (void __user *)ALIGN_DOWN((uintptr_t)uptr, granule); if (check_add_overflow((unsigned long)uptr, length, &end)) return ERR_PTR(-EOVERFLOW); @@ -1409,6 +1419,8 @@ struct iopt_pages *iopt_alloc_user_pages(void __user *uptr, pages = iopt_alloc_pages(uptr - uptr_down, length, writable); if (IS_ERR(pages)) return pages; + pages->page_shift = shift; + pages->npages = ((length - 1 + (uptr - uptr_down)) >> shift) + 1; pages->uptr = uptr_down; pages->type = IOPT_ADDRESS_USER; return pages; @@ -1656,6 +1668,12 @@ void iopt_release_pages(struct kref *kref) WARN_ON(!RB_EMPTY_ROOT(&pages->domains_itree.rb_root)); WARN_ON(pages->npinned); WARN_ON(!xa_empty(&pages->pinned_pfns)); +#ifdef CONFIG_MM_SUBPAGE + WARN_ON(!xa_empty(&pages->fragments)); + WARN_ON(!xa_empty(&pages->fragment_pages)); + xa_destroy(&pages->fragments); + xa_destroy(&pages->fragment_pages); +#endif if (iopt_is_dmabuf(pages) && pages->dmabuf.attach) { struct dma_buf *dmabuf = pages->dmabuf.attach->dmabuf; @@ -1832,6 +1850,9 @@ void iopt_area_unmap_domain(struct iopt_area *area, struct iommu_domain *domain) void iopt_area_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, struct iommu_domain *domain) { + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_unfill_domain(area, pages, domain); + if (iopt_dmabuf_revoked(pages)) return; @@ -1855,6 +1876,9 @@ int iopt_area_fill_domain(struct iopt_area *area, struct iommu_domain *domain) lockdep_assert_held(&area->pages->mutex); + if (iopt_pages_use_fragments(area->pages)) + return iopt_fragments_fill_domain(area, area->pages, domain); + if (iopt_dmabuf_revoked(area->pages)) return 0; @@ -1916,6 +1940,9 @@ int iopt_area_fill_domains(struct iopt_area *area, struct iopt_pages *pages) if (xa_empty(&area->iopt->domains)) return 0; + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_fill_domains(area, pages); + mutex_lock(&pages->mutex); if (iopt_is_dmabuf(pages)) { rc = iopt_dmabuf_track_all_domains(area, pages); @@ -2365,6 +2392,10 @@ int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, if (iopt_is_dmabuf(pages)) return -EINVAL; + if (iopt_pages_use_fragments(pages) && change_mm && + !(flags & IOMMUFD_ACCESS_RW_KTHREAD)) + return iopt_fragments_rw(pages, start_byte, data, length, flags); + if (pages->type != IOPT_ADDRESS_USER) return iopt_pages_rw_slow(pages, start_index, last_index, start_byte % PAGE_SIZE, data, length, @@ -2385,11 +2416,14 @@ int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, * ignore any pinning inconsistencies, unlike a real DMA path. */ if (change_mm) { - if (!mmget_not_zero(pages->source_mm)) + if (!mmget_not_zero(pages->source_mm)) { + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_rw(pages, start_byte, data, length, flags); return iopt_pages_rw_slow(pages, start_index, last_index, start_byte % PAGE_SIZE, data, length, flags); + } kthread_use_mm(pages->source_mm); } @@ -2454,6 +2488,11 @@ int iopt_area_add_access(struct iopt_area *area, unsigned long start_index, if (iopt_is_dmabuf(pages)) return -EINVAL; + /* Preserve typed ownership and validate the native output geometry. */ + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_add_access(area, start_index, last_index, + out_pages, lock_area); + mutex_lock(&pages->mutex); access = iopt_pages_get_exact_access(pages, start_index, last_index); if (access) { @@ -2494,6 +2533,47 @@ int iopt_area_add_access(struct iopt_area *area, unsigned long start_index, return rc; } +/* Internal hardware queues consume a contiguous physical byte range. */ +int iopt_area_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_pages *pages = area->pages; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + unsigned long count = last - first + 1, i; + struct page **list; + phys_addr_t base; + int rc; + + if (iopt_is_dmabuf(pages)) + return -EINVAL; + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_add_phys_access(area, start_byte, length, out_phys); + + /* Retain the native page supplier and its accounting/batching rules. */ + list = kvcalloc(count, sizeof(*list), GFP_KERNEL_ACCOUNT); + if (!list) + return -ENOMEM; + rc = iopt_area_add_access(area, first, last, list, 0, true); + if (rc) + goto out_free; + base = PFN_PHYS(page_to_pfn(list[0])); + for (i = 1; i < count; i++) { + if (page_to_pfn(list[i]) != page_to_pfn(list[0]) + i) { + rc = -EFAULT; + goto out_unpin; + } + } + *out_phys = base + offset_in_page(start_byte); + goto out_free; + +out_unpin: + iopt_area_remove_access(area, first, last, true); +out_free: + kvfree(list); + return rc; +} + /** * iopt_area_remove_access() - Release an in-kernel access for PFNs * @area: The source of PFNs @@ -2526,7 +2606,10 @@ void iopt_area_remove_access(struct iopt_area *area, unsigned long start_index, goto out_unlock; interval_tree_remove(&access->node, &pages->access_itree); - iopt_pages_unfill_xarray(pages, start_index, last_index); + if (iopt_pages_use_fragments(pages)) + iopt_fragments_unfill_access(pages, start_index, last_index); + else + iopt_pages_unfill_xarray(pages, start_index, last_index); kfree(access); out_unlock: mutex_unlock(&pages->mutex); diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c index af07c642a5260..146b1fefa0475 100644 --- a/drivers/iommu/iommufd/selftest.c +++ b/drivers/iommu/iommufd/selftest.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -1201,6 +1202,28 @@ static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd, unsigned long pfn; long npages; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + phys_addr_t user_phys; + + npages = get_user_fragments_remote(current->mm, + (uintptr_t)uptr, page_size, 0, &fragment, 1); + if (npages != 1) { + rc = npages < 0 ? npages : -EFAULT; + goto out_put; + } + user_phys = PFN_PHYS(folio_pfn(fragment.folio)) + fragment.offset; + io_phys = iommu_iova_to_phys(&mock->domain, iova); + rc = fragment.length == page_size && io_phys == user_phys ? 0 : -EINVAL; + release_user_fragments(&fragment, 1, false); + if (rc) + goto out_put; + iova += page_size; + uptr += page_size; + continue; + } +#endif npages = get_user_pages_fast((uintptr_t)uptr & PAGE_MASK, 1, 0, pages); if (npages < 0) { @@ -1506,9 +1529,39 @@ static int iommufd_test_access_replace_ioas(struct iommufd_ucmd *ucmd, /* Check that the pages in a page array match the pages in the user VA */ static int iommufd_test_check_pages(void __user *uptr, struct page **pages, - size_t npages) -{ - for (; npages; npages--) { + size_t length) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + unsigned long offset = 0; + + while (offset < length) { + struct user_page_fragment fragment; + unsigned long bytes = min_t(unsigned long, length - offset, + mm_user_fragment_size(current->mm) - + (((unsigned long)uptr + offset) & + (mm_user_fragment_size(current->mm) - 1))); + phys_addr_t actual, expected; + long rc; + + rc = get_user_fragments_remote(current->mm, + (unsigned long)uptr + offset, bytes, 0, &fragment, 1); + if (rc <= 0) + return rc ?: -EFAULT; + actual = PFN_PHYS(folio_pfn(fragment.folio)) + fragment.offset; + expected = PFN_PHYS(page_to_pfn(pages[offset / PAGE_SIZE])) + + offset_in_page(offset); + bytes = fragment.length; + release_user_fragments(&fragment, 1, false); + if (actual != expected || !bytes || + bytes > PAGE_SIZE - offset_in_page(offset)) + return -EBADE; + offset += bytes; + } + return 0; + } +#endif + for (size_t npages = DIV_ROUND_UP(length, PAGE_SIZE); npages; npages--) { struct page *tmp_pages[1]; long rc; @@ -1586,7 +1639,7 @@ static int iommufd_test_access_pages(struct iommufd_ucmd *ucmd, if (uptr) { rc = iommufd_test_check_pages( uptr - (iova - ALIGN_DOWN(iova, PAGE_SIZE)), pages, - npages); + length + offset_in_page(iova)); if (rc) goto out_unaccess; } diff --git a/drivers/iommu/iommufd/viommu.c b/drivers/iommu/iommufd/viommu.c index fc13cf4737ea3..fa8296b15ef59 100644 --- a/drivers/iommu/iommufd/viommu.c +++ b/drivers/iommu/iommufd/viommu.c @@ -263,11 +263,7 @@ static void iommufd_hw_queue_destroy_access(struct iommufd_ctx *ictx, struct iommufd_access *access, u64 base_iova, size_t length) { - u64 aligned_iova = PAGE_ALIGN_DOWN(base_iova); - u64 offset = base_iova - aligned_iova; - - iommufd_access_unpin_pages(access, aligned_iova, - PAGE_ALIGN(length + offset)); + iommufd_access_unpin_pages(access, base_iova, length); iommufd_access_detach_internal(access); iommufd_access_destroy_internal(ictx, access); } @@ -300,66 +296,28 @@ static struct iommufd_access * iommufd_hw_queue_alloc_phys(struct iommu_hw_queue_alloc *cmd, struct iommufd_viommu *viommu, phys_addr_t *base_pa) { - u64 aligned_iova = PAGE_ALIGN_DOWN(cmd->nesting_parent_iova); - u64 offset = cmd->nesting_parent_iova - aligned_iova; struct iommufd_access *access; - struct page **pages; - size_t max_npages; - size_t length; - size_t i; int rc; - /* max_npages = DIV_ROUND_UP(offset + cmd->length, PAGE_SIZE) */ - if (check_add_overflow(offset, cmd->length, &length)) - return ERR_PTR(-ERANGE); - if (check_add_overflow(length, PAGE_SIZE - 1, &length)) + if (cmd->nesting_parent_iova > ULONG_MAX || cmd->length > ULONG_MAX) return ERR_PTR(-ERANGE); - max_npages = length / PAGE_SIZE; - /* length needs to be page aligned too */ - length = max_npages * PAGE_SIZE; - - /* - * Use kvcalloc() to avoid memory fragmentation for a large page array. - * Set __GFP_NOWARN to avoid syzkaller blowups - */ - pages = kvzalloc_objs(*pages, max_npages, GFP_KERNEL | __GFP_NOWARN); - if (!pages) - return ERR_PTR(-ENOMEM); - - access = iommufd_access_create_internal(viommu->ictx); - if (IS_ERR(access)) { - rc = PTR_ERR(access); - goto out_free; - } + access = iommufd_access_create_internal_phys(viommu->ictx); + if (IS_ERR(access)) + return access; rc = iommufd_access_attach_internal(access, viommu->hwpt->ioas); if (rc) goto out_destroy; - - rc = iommufd_access_pin_pages(access, aligned_iova, length, pages, 0); + rc = iommufd_access_pin_phys(access, cmd->nesting_parent_iova, + cmd->length, base_pa); if (rc) goto out_detach; - - /* Validate if the underlying physical pages are contiguous */ - for (i = 1; i < max_npages; i++) { - if (page_to_pfn(pages[i]) == page_to_pfn(pages[i - 1]) + 1) - continue; - rc = -EFAULT; - goto out_unpin; - } - - *base_pa = (page_to_pfn(pages[0]) << PAGE_SHIFT) + offset; - kvfree(pages); return access; -out_unpin: - iommufd_access_unpin_pages(access, aligned_iova, length); out_detach: iommufd_access_detach_internal(access); out_destroy: iommufd_access_destroy_internal(viommu->ictx, access); -out_free: - kvfree(pages); return ERR_PTR(rc); } diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index e6d3a2311a99b..df079de9ee5de 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -28,11 +28,14 @@ #include #include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -84,6 +87,8 @@ struct vfio_domain { bool enforce_cache_coherency : 1; }; +struct vfio_fragment_provider; + struct vfio_dma { struct rb_node node; dma_addr_t iova; /* Device address */ @@ -99,6 +104,7 @@ struct vfio_dma { unsigned long *bitmap; struct mm_struct *mm; size_t locked_vm; + struct vfio_fragment_provider *fragments; }; struct vfio_batch { @@ -128,7 +134,9 @@ struct vfio_pfn { struct rb_node node; dma_addr_t iova; /* Device address */ unsigned long pfn; /* Host pfn */ + unsigned long length; /* Bytes retained in this DMA */ unsigned int ref_count; + bool native_pin; /* separately adopted legacy external PIN */ }; struct vfio_regions { @@ -152,6 +160,9 @@ struct vfio_regions { #define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX) static int put_pfn(unsigned long pfn, int prot); +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn); +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length); static struct vfio_iommu_group* vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, @@ -238,6 +249,12 @@ static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old) } +/* Software dirty granularity must survive domain attach/detach unchanged. */ +static size_t vfio_dirty_pgsize(struct vfio_iommu *iommu) +{ + return IS_ENABLED(CONFIG_MM_SUBPAGE) ? SZ_4K : 1UL << __ffs(iommu->pgsize_bitmap); +} + static int vfio_dma_bitmap_alloc(struct vfio_dma *dma, size_t pgsize) { uint64_t npages = dma->size / pgsize; @@ -272,14 +289,15 @@ static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize) for (p = rb_first(&dma->pfn_list); p; p = rb_next(p)) { struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, node); - bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, 1); + bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, + vpfn->length >> pgshift); } } static void vfio_iommu_populate_bitmap_full(struct vfio_iommu *iommu) { struct rb_node *n; - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); @@ -383,7 +401,7 @@ static void vfio_unlink_pfn(struct vfio_dma *dma, struct vfio_pfn *old) } static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova, - unsigned long pfn) + unsigned long pfn, unsigned long length) { struct vfio_pfn *vpfn; @@ -393,6 +411,7 @@ static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova, vpfn->iova = iova; vpfn->pfn = pfn; + vpfn->length = length; vpfn->ref_count = 1; vfio_link_pfn(dma, vpfn); return 0; @@ -421,8 +440,21 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) vpfn->ref_count--; if (!vpfn->ref_count) { - ret = put_pfn(vpfn->pfn, dma->prot); - vfio_remove_from_pfn_list(dma, vpfn); + if (dma->fragments) { + unsigned long offset = vpfn->iova - dma->iova; + unsigned long length = vpfn->length; + + unsigned long pfn = vpfn->pfn; + bool native_pin = vpfn->native_pin; + + vfio_remove_from_pfn_list(dma, vpfn); + if (native_pin) + vfio_fragments_put_native_pin(dma, pfn); + vfio_fragments_release_range(dma, offset, length); + } else { + ret = put_pfn(vpfn->pfn, dma->prot); + vfio_remove_from_pfn_list(dma, vpfn); + } } return ret; } @@ -430,10 +462,19 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) static int mm_lock_acct(struct task_struct *task, struct mm_struct *mm, bool lock_cap, long npage) { - int ret = mmap_write_lock_killable(mm); + int ret; - if (ret) - return ret; + if (!npage) + return 0; + + /* A released pin must be refunded even if the caller is interrupted. */ + if (npage < 0) { + mmap_write_lock(mm); + } else { + ret = mmap_write_lock_killable(mm); + if (ret) + return ret; + } ret = __account_locked_vm(mm, abs(npage), npage > 0, task, lock_cap); mmap_write_unlock(mm); @@ -443,25 +484,28 @@ static int mm_lock_acct(struct task_struct *task, struct mm_struct *mm, static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) { struct mm_struct *mm; + bool get_mm = async && npage > 0; int ret; if (!npage) return 0; mm = dma->mm; - if (async && !mmget_not_zero(mm)) + if (get_mm && !mmget_not_zero(mm)) return -ESRCH; /* process exited */ ret = mm_lock_acct(dma->task, mm, dma->lock_cap, npage); if (!ret) dma->locked_vm += npage; - if (async) + if (get_mm) mmput(mm); return ret; } +#include "vfio_iommu_type1_fragments.c" + /* * Some mappings aren't backed by a struct page, for example an mmap'd * MMIO range for our own or another device. These use a different @@ -765,7 +809,8 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, */ if (acct_pages) { if (!dma->lock_cap && - mm->locked_vm + lock_acct + acct_pages > limit) { + mm_locked_vm_native_pages(mm) + + lock_acct + acct_pages > limit) { pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__, limit << PAGE_SHIFT); ret = -ENOMEM; @@ -889,6 +934,78 @@ static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, return unlocked; } +/* A native page may contain independently owned fine-grained DMA areas. */ +static unsigned long vfio_unpin_external_range(struct vfio_iommu *iommu, + dma_addr_t start, unsigned long length, + bool do_accounting) +{ + unsigned long done = 0; + + while (done < length) { + dma_addr_t iova = start + done; + struct vfio_dma *dma = vfio_find_dma(iommu, iova, 1); + unsigned long bytes; + + if (!dma || iova < dma->iova) + break; + bytes = min_t(unsigned long, length - done, + PAGE_SIZE - offset_in_page(iova)); + bytes = min(bytes, dma->size - (iova - dma->iova)); + vfio_unpin_page_external(dma, iova, do_accounting); + done += bytes; + } + return done; +} + +static int vfio_pin_external_segment(struct vfio_iommu *iommu, + struct vfio_dma *dma, dma_addr_t iova, unsigned long length, + int prot, bool do_accounting, unsigned long *pfn) +{ + struct vfio_pfn *vpfn; + int ret; + + if ((dma->prot & prot) != prot) + return -EPERM; + vpfn = vfio_iova_get_vfio_pfn(dma, iova); + if (vpfn) { + if (vpfn->length != length) { + vfio_unpin_page_external(dma, iova, do_accounting); + return -EINVAL; + } + *pfn = vpfn->pfn; + } else if (dma->fragments) { + ret = vfio_fragments_external(dma, iova, length, pfn); + if (ret) + return ret; + } else { + /* Ordinary native mappings still use one native GUP per page. */ + if (length != PAGE_SIZE) + return -EINVAL; + ret = vfio_pin_page_external(dma, dma->vaddr + iova - dma->iova, + pfn, do_accounting); + if (ret) + return ret; + if (!pfn_valid(*pfn)) { + ret = -EINVAL; + goto put_native; + } + ret = vfio_add_to_pfn_list(dma, iova, *pfn, length); + if (ret) + goto put_native; + } + if (iommu->dirty_page_tracking) { + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); + + bitmap_set(dma->bitmap, (iova - dma->iova) >> pgshift, + length >> pgshift); + } + return 0; +put_native: + if (put_pfn(*pfn, dma->prot) && do_accounting) + vfio_lock_acct(dma, -1, true); + return ret; +} + static int vfio_iommu_type1_pin_pages(void *iommu_data, struct iommu_group *iommu_group, dma_addr_t user_iova, @@ -897,119 +1014,72 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, { struct vfio_iommu *iommu = iommu_data; struct vfio_iommu_group *group; - int i, j, ret; - unsigned long remote_vaddr; - struct vfio_dma *dma; + unsigned long pinned = 0; bool do_accounting; dma_addr_t iova_end; size_t iova_size; + int i, ret; if (!iommu || !pages || npage <= 0) return -EINVAL; - - /* Supported for v2 version only */ + /* Callers recover the original byte offset from the returned page. */ + user_iova = ALIGN_DOWN(user_iova, PAGE_SIZE); if (!iommu->v2) return -EACCES; - if (check_mul_overflow(npage, PAGE_SIZE, &iova_size) || check_add_overflow(user_iova, iova_size - 1, &iova_end)) return -EOVERFLOW; mutex_lock(&iommu->lock); - if (WARN_ONCE(iommu->vaddr_invalid_count, "vfio_pin_pages not allowed with VFIO_UPDATE_VADDR\n")) { ret = -EBUSY; goto pin_done; } - - /* Fail if no dma_umap notifier is registered */ if (list_empty(&iommu->device_list)) { ret = -EINVAL; goto pin_done; } - - /* - * If iommu capable domain exist in the container then all pages are - * already pinned and accounted. Accounting should be done if there is no - * iommu capable domain in the container. - */ do_accounting = list_empty(&iommu->domain_list); - for (i = 0; i < npage; i++) { - unsigned long phys_pfn; - dma_addr_t iova; - struct vfio_pfn *vpfn; - - iova = user_iova + PAGE_SIZE * i; - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - if (!dma) { - ret = -EINVAL; - goto pin_unwind; - } - - if ((dma->prot & prot) != prot) { - ret = -EPERM; - goto pin_unwind; - } - - vpfn = vfio_iova_get_vfio_pfn(dma, iova); - if (vpfn) { - pages[i] = pfn_to_page(vpfn->pfn); - continue; - } - - remote_vaddr = dma->vaddr + (iova - dma->iova); - ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn, - do_accounting); - if (ret) - goto pin_unwind; - - if (!pfn_valid(phys_pfn)) { - ret = -EINVAL; - goto pin_unwind; - } - - ret = vfio_add_to_pfn_list(dma, iova, phys_pfn); - if (ret) { - if (put_pfn(phys_pfn, dma->prot) && do_accounting) - vfio_lock_acct(dma, -1, true); - goto pin_unwind; - } + unsigned long done = 0; - pages[i] = pfn_to_page(phys_pfn); + pages[i] = NULL; + while (done < PAGE_SIZE) { + dma_addr_t iova = user_iova + PAGE_SIZE * i + done; + struct vfio_dma *dma = vfio_find_dma(iommu, iova, 1); + unsigned long length, pfn; - if (iommu->dirty_page_tracking) { - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); - - /* - * Bitmap populated with the smallest supported page - * size - */ - bitmap_set(dma->bitmap, - (iova - dma->iova) >> pgshift, 1); + if (!dma || iova < dma->iova) { + ret = -EINVAL; + goto pin_unwind; + } + length = min_t(unsigned long, PAGE_SIZE - done, + dma->size - (iova - dma->iova)); + ret = vfio_pin_external_segment(iommu, dma, iova, length, + prot, do_accounting, &pfn); + if (ret) + goto pin_unwind; + pinned += length; + if (pages[i] && page_to_pfn(pages[i]) != pfn) { + ret = -EINVAL; + goto pin_unwind; + } + pages[i] = pfn_to_page(pfn); + done += length; } } - ret = i; - + ret = npage; group = vfio_iommu_find_iommu_group(iommu, iommu_group); if (!group->pinned_page_dirty_scope) { group->pinned_page_dirty_scope = true; iommu->num_non_pinned_groups--; } - goto pin_done; - pin_unwind: - pages[i] = NULL; - for (j = 0; j < i; j++) { - dma_addr_t iova; - - iova = user_iova + PAGE_SIZE * j; - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - vfio_unpin_page_external(dma, iova, do_accounting); - pages[j] = NULL; - } + vfio_unpin_external_range(iommu, user_iova, pinned, do_accounting); + while (i >= 0) + pages[i--] = NULL; pin_done: mutex_unlock(&iommu->lock); return ret; @@ -1019,39 +1089,21 @@ static void vfio_iommu_type1_unpin_pages(void *iommu_data, dma_addr_t user_iova, int npage) { struct vfio_iommu *iommu = iommu_data; - bool do_accounting; dma_addr_t iova_end; size_t iova_size; - int i; - - /* Supported for v2 version only */ - if (WARN_ON(!iommu->v2)) - return; + unsigned long unpinned; - if (WARN_ON(npage <= 0)) + if (WARN_ON(!iommu->v2) || WARN_ON(npage <= 0)) return; - + user_iova = ALIGN_DOWN(user_iova, PAGE_SIZE); if (WARN_ON(check_mul_overflow(npage, PAGE_SIZE, &iova_size) || check_add_overflow(user_iova, iova_size - 1, &iova_end))) return; - mutex_lock(&iommu->lock); - - do_accounting = list_empty(&iommu->domain_list); - for (i = 0; i < npage; i++) { - dma_addr_t iova = user_iova + PAGE_SIZE * i; - struct vfio_dma *dma; - - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - if (!dma) - break; - - vfio_unpin_page_external(dma, iova, do_accounting); - } - + unpinned = vfio_unpin_external_range(iommu, user_iova, iova_size, + list_empty(&iommu->domain_list)); mutex_unlock(&iommu->lock); - - WARN_ON(i != npage); + WARN_ON(unpinned != iova_size); } static long vfio_sync_unpin(struct vfio_dma *dma, struct vfio_domain *domain, @@ -1155,6 +1207,10 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, if (!dma->size) return 0; + if (dma->fragments) { + vfio_fragments_unmap(iommu, dma); + return 0; + } if (list_empty(&iommu->domain_list)) return 0; @@ -1233,6 +1289,7 @@ static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) { WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list)); vfio_unmap_unpin(iommu, dma, true); + vfio_fragments_destroy(dma); vfio_unlink_dma(iommu, dma); put_task_struct(dma->task); mmdrop(dma->mm); @@ -1246,21 +1303,18 @@ static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu) { struct vfio_domain *domain; + unsigned long minimum = SZ_4K; iommu->pgsize_bitmap = ULONG_MAX; - - list_for_each_entry(domain, &iommu->domain_list, next) + list_for_each_entry(domain, &iommu->domain_list, next) { iommu->pgsize_bitmap &= domain->domain->pgsize_bitmap; - - /* - * In case the IOMMU supports page sizes smaller than PAGE_SIZE - * we pretend PAGE_SIZE is supported and hide sub-PAGE_SIZE sizes. - * That way the user will be able to map/unmap buffers whose size/ - * start address is aligned with PAGE_SIZE. Pinning code uses that - * granularity while iommu driver can use the sub-PAGE_SIZE size - * to map the buffer. - */ - if (iommu->pgsize_bitmap & ~PAGE_MASK) { + minimum = max(minimum, 1UL << __ffs(domain->domain->pgsize_bitmap)); + } + if (IS_ENABLED(CONFIG_MM_SUBPAGE)) { + /* Every domain can compose its minimum pages into this common unit. */ + iommu->pgsize_bitmap &= ~(minimum - 1); + iommu->pgsize_bitmap |= minimum; + } else if (iommu->pgsize_bitmap & ~PAGE_MASK) { iommu->pgsize_bitmap &= PAGE_MASK; iommu->pgsize_bitmap |= PAGE_SIZE; } @@ -1367,6 +1421,8 @@ static void vfio_notify_dma_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) { struct vfio_device *device; + dma_addr_t iova = dma->iova; + size_t size = dma->size; if (list_empty(&iommu->device_list)) return; @@ -1377,11 +1433,12 @@ static void vfio_notify_dma_unmap(struct vfio_iommu *iommu, * call back down to this code and try to obtain the iommu->lock we must * drop it. */ - mutex_lock(&iommu->device_list_lock); + /* The range is a snapshot: another unmap may free dma while unlocked. */ mutex_unlock(&iommu->lock); + mutex_lock(&iommu->device_list_lock); list_for_each_entry(device, &iommu->device_list, iommu_entry) - device->ops->dma_unmap(device, dma->iova, dma->size); + device->ops->dma_unmap(device, iova, size); mutex_unlock(&iommu->device_list_lock); mutex_lock(&iommu->lock); @@ -1437,11 +1494,10 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, /* When dirty tracking is enabled, allow only min supported pgsize */ if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && - (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) { + (!iommu->dirty_page_tracking || (bitmap->pgsize != vfio_dirty_pgsize(iommu)))) { goto unlock; } - WARN_ON((pgsize - 1) & PAGE_MASK); again: /* * vfio-iommu-type1 (v1) - User mappings were coalesced together to @@ -1530,7 +1586,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { ret = update_user_bitmap(bitmap->data, iommu, dma, - iova, pgsize); + iova, vfio_dirty_pgsize(iommu)); if (ret) break; } @@ -1587,6 +1643,13 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret = 0; + if (dma->fragments) { + dma->size = map_size; + ret = vfio_fragments_map_all(iommu, dma); + if (ret) + vfio_remove_dma(iommu, dma); + return ret; + } vfio_batch_init(&batch); while (size) { @@ -1644,27 +1707,30 @@ static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu, return list_empty(iova); } -static int vfio_change_dma_owner(struct vfio_dma *dma) +static int vfio_change_dma_mm_pages(struct vfio_dma *dma, struct task_struct *task, + struct mm_struct *mm, bool lock_cap, unsigned long pages) { - struct task_struct *task = current->group_leader; - struct mm_struct *mm = current->mm; - long npage = dma->locked_vm; - bool lock_cap; + long old_pages = dma->locked_vm; int ret; - if (mm == dma->mm) - return 0; - - lock_cap = capable(CAP_IPC_LOCK); - ret = mm_lock_acct(task, mm, lock_cap, npage); + if (mm == dma->mm) { + if (pages == old_pages) + return 0; + ret = mm_lock_acct(task, mm, lock_cap, (long)pages - old_pages); + if (!ret) + dma->locked_vm = pages; + return ret; + } + ret = mm_lock_acct(task, mm, lock_cap, pages); if (ret) return ret; - if (mmget_not_zero(dma->mm)) { - mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -npage); - mmput(dma->mm); + /* Metadata adoption can deduplicate charges, but not original PIN refs. */ + ret = mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -old_pages); + if (ret) { + mm_lock_acct(task, mm, lock_cap, -(long)pages); + return ret; } - if (dma->task != task) { put_task_struct(dma->task); dma->task = get_task_struct(task); @@ -1673,9 +1739,40 @@ static int vfio_change_dma_owner(struct vfio_dma *dma) dma->mm = mm; mmgrab(dma->mm); dma->lock_cap = lock_cap; + dma->locked_vm = pages; return 0; } +static int vfio_change_dma_mm(struct vfio_dma *dma, struct task_struct *task, + struct mm_struct *mm, bool lock_cap) +{ + return vfio_change_dma_mm_pages(dma, task, mm, lock_cap, dma->locked_vm); +} + +static int vfio_change_dma_owner(struct vfio_iommu *iommu, struct vfio_dma *dma, + unsigned long vaddr) +{ + struct vfio_fragment_provider *provider; + int ret; + + if (dma->fragments || (mm_page_size(current->mm) == PAGE_SIZE && + IS_ALIGNED(vaddr, PAGE_SIZE))) + return vfio_change_dma_mm(dma, current->group_leader, current->mm, + capable(CAP_IPC_LOCK)); + + /* Prepare only metadata: failure must preserve every legacy pin/charge. */ + provider = vfio_fragments_prepare_native(iommu, dma); + if (IS_ERR(provider)) + return PTR_ERR(provider); + ret = vfio_change_dma_mm_pages(dma, current->group_leader, current->mm, + capable(CAP_IPC_LOCK), vfio_fragments_native_pages(provider)); + if (ret) + vfio_fragments_abort_native(provider); + else + vfio_fragments_commit_native(dma, provider); + return ret; +} + static int vfio_dma_do_map(struct vfio_iommu *iommu, struct vfio_iommu_type1_dma_map *map) { @@ -1713,8 +1810,6 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); - WARN_ON((pgsize - 1) & PAGE_MASK); - if ((size | iova | vaddr) & (pgsize - 1)) { ret = -EINVAL; goto out_unlock; @@ -1728,7 +1823,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, dma->size != size) { ret = -EINVAL; } else { - ret = vfio_change_dma_owner(dma); + ret = vfio_change_dma_owner(iommu, dma, vaddr); if (ret) goto out_unlock; dma->vaddr = vaddr; @@ -1781,6 +1876,11 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, /* Insert zero-sized and grow as we map chunks of it */ vfio_link_dma(iommu, dma); + ret = vfio_fragments_init(dma, size); + if (ret) { + vfio_remove_dma(iommu, dma); + goto out_unlock; + } /* Don't pin and map if container doesn't contain IOMMU capable domain*/ if (list_empty(&iommu->domain_list)) @@ -1789,7 +1889,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, ret = vfio_pin_map_dma(iommu, dma, size); if (!ret && iommu->dirty_page_tracking) { - ret = vfio_dma_bitmap_alloc(dma, pgsize); + ret = vfio_dma_bitmap_alloc(dma, vfio_dirty_pgsize(iommu)); if (ret) vfio_remove_dma(iommu, dma); } @@ -1822,6 +1922,14 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, size_t pos = 0; dma = rb_entry(n, struct vfio_dma, node); + if (dma->fragments) { + ret = vfio_fragments_acquire(dma, 0, dma->size); + if (!ret) + ret = vfio_fragments_map_domain(dma, domain->domain); + if (ret) + goto unwind; + continue; + } while (pos < dma->size) { dma_addr_t iova = dma->iova + pos; @@ -1906,6 +2014,11 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); size_t pos = 0; + if (dma->fragments) { + iommu_unmap(domain->domain, dma->iova, dma->size); + vfio_fragments_release_unused(dma); + continue; + } if (dma->iommu_mapped) { iommu_unmap(domain->domain, dma->iova, dma->size); continue; @@ -2442,6 +2555,10 @@ static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu) long locked = 0, unlocked = 0; dma = rb_entry(n, struct vfio_dma, node); + if (dma->fragments) { + vfio_unmap_unpin(iommu, dma, true); + continue; + } unlocked += vfio_unmap_unpin(iommu, dma, false); p = rb_first(&dma->pfn_list); for (; p; p = rb_next(p)) { @@ -2646,7 +2763,8 @@ static void *vfio_iommu_type1_open(unsigned long arg) mutex_init(&iommu->lock); mutex_init(&iommu->device_list_lock); INIT_LIST_HEAD(&iommu->device_list); - iommu->pgsize_bitmap = PAGE_MASK; + iommu->pgsize_bitmap = IS_ENABLED(CONFIG_MM_SUBPAGE) ? + GENMASK(BITS_PER_LONG - 1, 12) : PAGE_MASK; INIT_LIST_HEAD(&iommu->emulated_iommu_groups); return iommu; @@ -2812,7 +2930,7 @@ static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu, cap_mig.flags = 0; /* support minimum pgsize */ - cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap); + cap_mig.pgsize_bitmap = vfio_dirty_pgsize(iommu); cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX; return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig)); @@ -2910,6 +3028,63 @@ static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu, return vfio_dma_do_map(iommu, &map); } +#ifdef CONFIG_COMPAT +struct compat_vfio_bitmap { + compat_u64 pgsize; + compat_u64 size; + compat_uptr_t data; +}; +#endif + +static size_t vfio_bitmap_size(void) +{ +#ifdef CONFIG_COMPAT + /* IA32 differs from AArch32/x32 in its u64 alignment and tail padding. */ + if (in_compat_syscall() && compat_need_64bit_alignment_fixup()) + return sizeof(struct compat_vfio_bitmap); +#endif + return sizeof(struct vfio_bitmap); +} + +static size_t vfio_dirty_bitmap_get_size(void) +{ + return offsetof(struct vfio_iommu_type1_dirty_bitmap_get, bitmap) + + vfio_bitmap_size(); +} + +static int vfio_bitmap_from_user(struct vfio_bitmap *bitmap, void __user *user) +{ +#ifdef CONFIG_COMPAT + if (in_compat_syscall()) { + compat_uptr_t data; + unsigned long offset = offsetof(struct vfio_bitmap, data); + + /* Read only the pointer field, never its trailing compat padding. */ + if (copy_from_user(bitmap, user, offset) || + get_user(data, (compat_uptr_t __user *)(user + offset))) + return -EFAULT; + bitmap->data = compat_ptr(data); + return 0; + } +#endif + return copy_from_user(bitmap, user, sizeof(*bitmap)) ? -EFAULT : 0; +} + +static int vfio_dirty_bitmap_get_from_user( + struct vfio_iommu_type1_dirty_bitmap_get *range, void __user *user) +{ +#ifdef CONFIG_COMPAT + if (in_compat_syscall()) { + unsigned long offset = offsetof(typeof(*range), bitmap); + + if (copy_from_user(range, user, offset)) + return -EFAULT; + return vfio_bitmap_from_user(&range->bitmap, user + offset); + } +#endif + return copy_from_user(range, user, sizeof(*range)) ? -EFAULT : 0; +} + static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, unsigned long arg) { @@ -2937,13 +3112,12 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { unsigned long pgshift; - if (unmap.argsz < (minsz + sizeof(bitmap))) + if (unmap.argsz < minsz + vfio_bitmap_size()) return -EINVAL; - if (copy_from_user(&bitmap, - (void __user *)(arg + minsz), - sizeof(bitmap))) - return -EFAULT; + ret = vfio_bitmap_from_user(&bitmap, (void __user *)(arg + minsz)); + if (ret) + return ret; if (!access_ok((void __user *)bitmap.data, bitmap.size)) return -EINVAL; @@ -2992,7 +3166,7 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, size_t pgsize; mutex_lock(&iommu->lock); - pgsize = 1 << __ffs(iommu->pgsize_bitmap); + pgsize = vfio_dirty_pgsize(iommu); if (!iommu->dirty_page_tracking) { ret = vfio_dma_bitmap_alloc_all(iommu, pgsize); if (!ret) @@ -3015,12 +3189,12 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, size_t size, iommu_pgsize; dma_addr_t iova, iova_end; - if (!data_size || data_size < sizeof(range)) + if (data_size < vfio_dirty_bitmap_get_size()) return -EINVAL; - if (copy_from_user(&range, (void __user *)(arg + minsz), - sizeof(range))) - return -EFAULT; + ret = vfio_dirty_bitmap_get_from_user(&range, (void __user *)(arg + minsz)); + if (ret) + return ret; iova = range.iova; size = range.size; @@ -3046,7 +3220,7 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, mutex_lock(&iommu->lock); - iommu_pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); + iommu_pgsize = vfio_dirty_pgsize(iommu); /* allow only smallest supported pgsize */ if (range.bitmap.pgsize != iommu_pgsize) { @@ -3112,11 +3286,12 @@ static void vfio_iommu_type1_register_device(void *iommu_data, * Holding both locks here allows avoiding the device_list_lock in * several fast paths. See vfio_notify_dma_unmap() */ - mutex_lock(&iommu->lock); + /* Unmap callbacks hold device_list_lock while reentering iommu->lock. */ mutex_lock(&iommu->device_list_lock); + mutex_lock(&iommu->lock); list_add(&vdev->iommu_entry, &iommu->device_list); - mutex_unlock(&iommu->device_list_lock); mutex_unlock(&iommu->lock); + mutex_unlock(&iommu->device_list_lock); } static void vfio_iommu_type1_unregister_device(void *iommu_data, @@ -3127,11 +3302,12 @@ static void vfio_iommu_type1_unregister_device(void *iommu_data, if (!vdev->ops->dma_unmap) return; - mutex_lock(&iommu->lock); + /* Unmap callbacks hold device_list_lock while reentering iommu->lock. */ mutex_lock(&iommu->device_list_lock); + mutex_lock(&iommu->lock); list_del(&vdev->iommu_entry); - mutex_unlock(&iommu->device_list_lock); mutex_unlock(&iommu->lock); + mutex_unlock(&iommu->device_list_lock); } static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, @@ -3156,6 +3332,22 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, return -EPERM; mm = dma->mm; + if (dma->fragments) { + int ret; + + if (!kthread && current->mm != mm) + return -EFAULT; + offset = user_iova - dma->iova; + count = min(count, dma->size - offset); + ret = vfio_fragments_rw(dma, offset, data, count, write, copied); + if (*copied && write && iommu->dirty_page_tracking) { + unsigned long shift = __ffs(vfio_dirty_pgsize(iommu)); + + bitmap_set(dma->bitmap, offset >> shift, + ((offset + *copied - 1) >> shift) - (offset >> shift) + 1); + } + return ret; + } if (!mmget_not_zero(mm)) return -EPERM; @@ -3175,7 +3367,7 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, *copied = copy_to_user((void __user *)vaddr, data, count) ? 0 : count; if (*copied && iommu->dirty_page_tracking) { - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); /* * Bitmap populated with the smallest supported page * size @@ -3281,3 +3473,13 @@ MODULE_VERSION(DRIVER_VERSION); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR(DRIVER_AUTHOR); MODULE_DESCRIPTION(DRIVER_DESC); + +#ifdef CONFIG_VFIO_TYPE1_ACCOUNT_KUNIT_TEST +#include "vfio_iommu_type1_account_test.c" +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_ARM64_USER4K_EXPERIMENTAL) +#include "vfio_iommu_type1_fragments_test.c" +#ifdef CONFIG_ARM64_USER4K_VFIO_IOCTL_TEST +#include "vfio_iommu_type1_ioctl_test.c" +#endif +#endif +#endif diff --git a/drivers/vfio/vfio_iommu_type1_fragments.c b/drivers/vfio/vfio_iommu_type1_fragments.c new file mode 100644 index 0000000000000..22d46a1b805a7 --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_fragments.c @@ -0,0 +1,609 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by type1: retained byte ranges, rather than IOMMU PFNs, own pins. */ +#ifdef CONFIG_MM_SUBPAGE +struct vfio_fragment { + struct rb_node node; + unsigned long start; + struct user_page_fragment pin; +}; + +struct vfio_fragment_provider { + struct rb_root records; + struct xarray physical_pages; + unsigned long npinned; +}; + +static phys_addr_t vfio_fragment_phys(const struct user_page_fragment *pin) +{ + return PFN_PHYS(folio_pfn(pin->folio)) + pin->offset; +} + +/* Returns a containing range, or the first range after the requested byte. */ +static struct vfio_fragment *vfio_fragment_find(struct vfio_dma *dma, + unsigned long offset) +{ + struct rb_node *node = dma->fragments->records.rb_node; + struct vfio_fragment *next = NULL; + + while (node) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + + if (offset < record->start) { + next = record; + node = node->rb_left; + } else if (offset - record->start >= record->pin.length) { + node = node->rb_right; + } else { + return record; + } + } + return next; +} + +static int vfio_fragments_init(struct vfio_dma *dma, size_t length) +{ + struct vfio_fragment_provider *provider; + + if (mm_page_size(dma->mm) == PAGE_SIZE && + IS_ALIGNED(dma->vaddr | dma->iova | length, PAGE_SIZE)) + return 0; + provider = kzalloc_obj(*provider, GFP_KERNEL_ACCOUNT); + if (!provider) + return -ENOMEM; + provider->records = RB_ROOT; + xa_init(&provider->physical_pages); + dma->fragments = provider; + return 0; +} + +static int vfio_fragment_store(struct vfio_dma *dma, unsigned long start, + struct user_page_fragment *pin) +{ + struct vfio_fragment_provider *provider = dma->fragments; + struct rb_node **link = &provider->records.rb_node, *parent = NULL; + unsigned long pfn = PHYS_PFN(vfio_fragment_phys(pin)); + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + struct vfio_fragment *record; + int ret; + + record = kmalloc_obj(*record, GFP_KERNEL_ACCOUNT); + if (!record) + return -ENOMEM; + ret = xa_err(xa_store(&provider->physical_pages, pfn, xa_mk_value(count + 1), + GFP_KERNEL_ACCOUNT)); + if (ret) { + kfree(record); + return ret; + } + while (*link) { + struct vfio_fragment *cur = rb_entry(*link, struct vfio_fragment, node); + + parent = *link; + link = start < cur->start ? &(*link)->rb_left : &(*link)->rb_right; + } + record->start = start; + record->pin = *pin; + rb_link_node(&record->node, parent, link); + rb_insert_color(&record->node, &provider->records); + if (!count) + provider->npinned++; + *pin = (struct user_page_fragment) {}; + return 0; +} + +/* These descriptors borrow existing native GUP pins until owner commit. */ +static int vfio_fragment_native_pin(unsigned long pfn, struct user_page_fragment *pin) +{ + struct page *page; + + if (!pfn_valid(pfn)) + return -EOPNOTSUPP; + page = pfn_to_page(pfn); + if (PageReserved(page)) + return -EOPNOTSUPP; + *pin = (struct user_page_fragment) { + .folio = page_folio(page), + .offset = page_to_phys(page) - PFN_PHYS(folio_pfn(page_folio(page))), + .length = PAGE_SIZE, + .pinned = true, + }; + return 0; +} + +/* No PIN, GET or account reference was acquired while preparing metadata. */ +static void vfio_fragments_abort_native(struct vfio_fragment_provider *provider) +{ + struct rb_node *node; + + while ((node = rb_first(&provider->records))) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + + rb_erase(node, &provider->records); + kfree(record); + } + xa_destroy(&provider->physical_pages); + kfree(provider); +} + +static struct vfio_fragment_provider * +vfio_fragments_prepare_native(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_fragment_provider *provider; + struct vfio_dma stage = {}; + struct vfio_domain *domain = NULL; + struct rb_node *node; + int ret; + + if (mm_page_size(dma->mm) != PAGE_SIZE || dma->has_rsvd) + return ERR_PTR(-EOPNOTSUPP); + if (dma->iommu_mapped) { + if (list_empty(&iommu->domain_list)) + return ERR_PTR(-EFAULT); + domain = list_first_entry(&iommu->domain_list, struct vfio_domain, next); + } + provider = kzalloc_obj(*provider, GFP_KERNEL_ACCOUNT); + if (!provider) + return ERR_PTR(-ENOMEM); + provider->records = RB_ROOT; + xa_init(&provider->physical_pages); + stage.fragments = provider; + /* One descriptor owns each original native GUP pin, including aliases. */ + if (domain) { + for (unsigned long offset = 0; offset < dma->size; offset += PAGE_SIZE) { + phys_addr_t phys = iommu_iova_to_phys(domain->domain, dma->iova + offset); + struct user_page_fragment pin; + + if (!phys || !IS_ALIGNED(phys, PAGE_SIZE)) { + ret = -EFAULT; + goto out; + } + ret = vfio_fragment_native_pin(PHYS_PFN(phys), &pin); + if (!ret) + ret = vfio_fragment_store(&stage, offset, &pin); + if (ret) + goto out; + cond_resched(); + } + } + for (node = rb_first(&dma->pfn_list); node; node = rb_next(node)) { + struct vfio_pfn *vpfn = rb_entry(node, struct vfio_pfn, node); + struct user_page_fragment pin; + + ret = vfio_fragment_native_pin(vpfn->pfn, &pin); + if (ret) + goto out; + if (!domain) { + /* Without a domain, the external pin is the cached buffer. */ + ret = vfio_fragment_store(&stage, vpfn->iova - dma->iova, &pin); + } else { + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, + vpfn->pfn)); + + /* The external GUP reference may identify a different page. */ + ret = xa_err(xa_store(&provider->physical_pages, vpfn->pfn, + xa_mk_value(count + 1), GFP_KERNEL_ACCOUNT)); + if (!ret && !count) + provider->npinned++; + } + if (ret) + goto out; + } + return provider; +out: + vfio_fragments_abort_native(provider); + return ERR_PTR(ret); +} + +static unsigned long vfio_fragments_native_pages(struct vfio_fragment_provider *provider) +{ + return provider->npinned; +} + +static void vfio_fragments_commit_native(struct vfio_dma *dma, + struct vfio_fragment_provider *provider) +{ + struct rb_node *node; + + /* Mapped legacy external consumers own separate native GUP references. */ + if (dma->iommu_mapped) { + for (node = rb_first(&dma->pfn_list); node; node = rb_next(node)) + rb_entry(node, struct vfio_pfn, node)->native_pin = true; + } + dma->fragments = provider; +} + +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn) +{ + struct vfio_fragment_provider *provider = dma->fragments; + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + struct user_page_fragment pin; + + if (WARN_ON(!count || vfio_fragment_native_pin(pfn, &pin))) + return; + if (count == 1) { + xa_erase(&provider->physical_pages, pfn); + provider->npinned--; + } else { + xa_store(&provider->physical_pages, pfn, xa_mk_value(count - 1), GFP_NOWAIT); + } + release_user_fragments(&pin, 1, dma->prot & IOMMU_WRITE); + if (provider->npinned < dma->locked_vm) + vfio_lock_acct(dma, -(long)(dma->locked_vm - provider->npinned), true); +} + +static bool vfio_fragments_external_overlap(struct vfio_dma *dma, + dma_addr_t start, unsigned long length) +{ + struct rb_node *node = dma->pfn_list.rb_node; + + while (node) { + struct vfio_pfn *vpfn = rb_entry(node, struct vfio_pfn, node); + + if (start + length <= vpfn->iova) + node = node->rb_left; + else if (start >= vpfn->iova + vpfn->length) + node = node->rb_right; + else + return true; + } + return false; +} + +/* Device unmaps and their IOTLB synchronization must precede this call. */ +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + struct vfio_fragment_provider *provider = dma->fragments; + struct vfio_fragment *first = vfio_fragment_find(dma, start); + struct rb_node *node = first ? &first->node : NULL; + unsigned long end = start + length; + + if (dma->iommu_mapped) + return; + while (node) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + dma_addr_t iova = dma->iova + record->start; + unsigned long pfn, count; + + if (record->start >= end) + break; + node = rb_next(node); + /* Native-page consumers retain all records overlapping their page. */ + if (vfio_fragments_external_overlap(dma, iova, record->pin.length)) + continue; + pfn = PHYS_PFN(vfio_fragment_phys(&record->pin)); + count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + if (WARN_ON(!count)) + continue; + if (count == 1) { + xa_erase(&provider->physical_pages, pfn); + provider->npinned--; + } else { + xa_store(&provider->physical_pages, pfn, xa_mk_value(count - 1), + GFP_NOWAIT); + } + rb_erase(&record->node, &provider->records); + release_user_fragments(&record->pin, 1, dma->prot & IOMMU_WRITE); + kfree(record); + cond_resched(); + } + if (provider->npinned < dma->locked_vm) + vfio_lock_acct(dma, -(long)(dma->locked_vm - provider->npinned), true); +} + +static void vfio_fragments_release_unused(struct vfio_dma *dma) +{ + vfio_fragments_release_range(dma, 0, dma->size); +} + +/* Cached records survive source VA replacement and the last source mm user. */ +static int vfio_fragments_acquire(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + unsigned long end = start + length, original = start; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + struct user_page_fragment *batch = NULL; + bool live = false; + int ret = 0; + + while (start < end) { + struct vfio_fragment *record = vfio_fragment_find(dma, start); + unsigned long stop = end; + long count; + + if (record && record->start <= start) { + start = record->start + record->pin.length; + continue; + } + if (record) + stop = min(stop, record->start); + if (!live) { + if (!mmget_not_zero(dma->mm)) { + ret = -EFAULT; + goto out; + } + live = true; + batch = kmalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!batch) { + ret = -ENOMEM; + goto out; + } + } + count = pin_user_fragments_remote(dma->mm, dma->vaddr + start, + stop - start, FOLL_LONGTERM | (dma->prot & IOMMU_WRITE ? FOLL_WRITE : 0), + batch, capacity); + if (count <= 0) { + ret = count ?: -EFAULT; + goto out; + } + for (unsigned long i = 0; i < count; i++) { + unsigned long bytes = batch[i].length; + + if (WARN_ON(!bytes || bytes > stop - start || + offset_in_page(vfio_fragment_phys(&batch[i])) > + PAGE_SIZE - bytes)) + ret = -EFAULT; + else + ret = vfio_fragment_store(dma, start, &batch[i]); + if (ret) { + release_user_fragments(batch + i, count - i, false); + goto out; + } + start += bytes; + } + cond_resched(); + } + if (dma->fragments->npinned > dma->locked_vm) + ret = vfio_lock_acct(dma, dma->fragments->npinned - dma->locked_vm, false); +out: + kfree(batch); + if (live) + mmput(dma->mm); + if (ret) + vfio_fragments_release_range(dma, original, length); + return ret; +} + +/* Merge physical byte runs while retaining each original pin identity. */ +static int vfio_fragments_map_domain(struct vfio_dma *dma, struct iommu_domain *domain) +{ + unsigned long done = 0, min_size = 1UL << __ffs(domain->pgsize_bitmap); + int ret = 0; + + while (done < dma->size) { + struct vfio_fragment *record = vfio_fragment_find(dma, done); + unsigned long bytes, limit = dma->size - done; + phys_addr_t phys; + + if (WARN_ON(!record || record->start > done)) { + ret = -EFAULT; + goto out; + } + if (disable_hugepages) + limit = min(limit, PAGE_SIZE); + phys = vfio_fragment_phys(&record->pin) + done - record->start; + bytes = min(record->pin.length - (done - record->start), limit); + while (bytes < limit) { + record = vfio_fragment_find(dma, done + bytes); + if (!record || record->start != done + bytes || + vfio_fragment_phys(&record->pin) != phys + bytes) + break; + bytes += min_t(unsigned long, record->pin.length, limit - bytes); + } + if (!IS_ALIGNED((dma->iova + done) | phys | bytes, min_size)) { + ret = -EINVAL; + goto out; + } + ret = iommu_map(domain, dma->iova + done, phys, bytes, + dma->prot | IOMMU_CACHE, GFP_KERNEL_ACCOUNT); + if (ret) + goto out; + done += bytes; + cond_resched(); + } +out: + if (ret && done) + iommu_unmap(domain, dma->iova, done); + return ret; +} + +static int vfio_fragments_map_all(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_domain *domain; + int ret; + + ret = vfio_fragments_acquire(dma, 0, dma->size); + if (ret) + return ret; + list_for_each_entry(domain, &iommu->domain_list, next) { + ret = vfio_fragments_map_domain(dma, domain->domain); + if (ret) + goto unwind; + } + dma->iommu_mapped = true; + return 0; +unwind: + list_for_each_entry_continue_reverse(domain, &iommu->domain_list, next) + iommu_unmap(domain->domain, dma->iova, dma->size); + vfio_fragments_release_unused(dma); + return ret; +} + +static void vfio_fragments_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_domain *domain; + + if (dma->iommu_mapped) { + list_for_each_entry(domain, &iommu->domain_list, next) + WARN_ON(iommu_unmap(domain->domain, dma->iova, dma->size) != dma->size); + dma->iommu_mapped = false; + } + vfio_fragments_release_unused(dma); +} + +static void vfio_fragments_destroy(struct vfio_dma *dma) +{ + if (!dma->fragments) + return; + vfio_fragments_release_unused(dma); + if (WARN_ON(!RB_EMPTY_ROOT(&dma->fragments->records))) + return; + xa_destroy(&dma->fragments->physical_pages); + kfree(dma->fragments); + dma->fragments = NULL; +} + +/* The output page has no offset field: validate every byte and its native offset. */ +static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, + unsigned long length, unsigned long *pfn) +{ + unsigned long first = iova - dma->iova, done = 0; + phys_addr_t phys = 0; + int ret; + + ret = vfio_fragments_acquire(dma, first, length); + if (ret) + return ret; + while (done < length) { + struct vfio_fragment *record = vfio_fragment_find(dma, first + done); + phys_addr_t next; + unsigned long offset; + + if (WARN_ON(!record || record->start > first + done)) { + ret = -EFAULT; + goto out; + } + offset = first + done - record->start; + next = vfio_fragment_phys(&record->pin) + offset; + if (!done) + phys = next; + if (offset_in_page(phys) != offset_in_page(iova) || next != phys + done) { + ret = -EINVAL; + goto out; + } + done += min_t(unsigned long, record->pin.length - offset, length - done); + } + *pfn = PHYS_PFN(phys); + ret = vfio_add_to_pfn_list(dma, iova, *pfn, length); +out: + if (ret) + vfio_fragments_release_range(dma, first, length); + return ret; +} + +/* CPU-only missing ranges take GET references, not new long-term DMA pins. */ +static int vfio_fragments_rw(struct vfio_dma *dma, unsigned long offset, void *data, + unsigned long length, bool write, size_t *copied) +{ + bool live = false; + int ret = 0; + + while (length) { + struct vfio_fragment *record = vfio_fragment_find(dma, offset); + struct user_page_fragment temp = {}, *pin; + unsigned long in_record = 0, bytes = length; + void *addr; + + if (record && record->start <= offset) { + pin = &record->pin; + in_record = offset - record->start; + } else { + long count; + + if (record) + bytes = min(bytes, record->start - offset); + if (!live) { + if (!mmget_not_zero(dma->mm)) { + ret = -EFAULT; + break; + } + live = true; + } + count = get_user_fragments_remote(dma->mm, dma->vaddr + offset, bytes, + write ? FOLL_WRITE : 0, &temp, 1); + if (count <= 0) { + ret = count ?: -EFAULT; + break; + } + pin = &temp; + } + bytes = min_t(unsigned long, bytes, pin->length - in_record); + addr = kmap_local_folio(pin->folio, pin->offset + in_record); + if (write) + memcpy(addr, data, bytes); + else + memcpy(data, addr, bytes); + kunmap_local(addr); + if (write && record && record->start <= offset) + mark_user_fragment_dirty(pin); + release_user_fragments(&temp, 1, write); + data += bytes; + offset += bytes; + length -= bytes; + *copied += bytes; + } + if (live) + mmput(dma->mm); + return ret; +} +#else +static struct vfio_fragment_provider * +vfio_fragments_prepare_native(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static void vfio_fragments_abort_native(struct vfio_fragment_provider *provider) +{ +} +static unsigned long vfio_fragments_native_pages(struct vfio_fragment_provider *provider) +{ + return 0; +} +static void vfio_fragments_commit_native(struct vfio_dma *dma, + struct vfio_fragment_provider *provider) +{ +} +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn) +{ +} +static int vfio_fragments_init(struct vfio_dma *dma, size_t length) +{ + return 0; +} +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ +} +static void vfio_fragments_release_unused(struct vfio_dma *dma) +{ +} +static int vfio_fragments_acquire(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_map_domain(struct vfio_dma *dma, struct iommu_domain *domain) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_map_all(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + return -EOPNOTSUPP; +} +static void vfio_fragments_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ +} +static void vfio_fragments_destroy(struct vfio_dma *dma) +{ +} +static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, + unsigned long length, unsigned long *pfn) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_rw(struct vfio_dma *dma, unsigned long offset, void *data, + unsigned long length, bool write, size_t *copied) +{ + return -EOPNOTSUPP; +} +#endif From df753d48d26928a122b9da35be9994de8428f8e7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0578/1579] arm64: support smaller userspace translation granules per mm Keep kernel mappings at their native granule while selecting TTBR0 geometry per address space. Add capability checks, table-fragment lifetime, TLB invalidation and exec selection for 4K on 16K/64K and 16K on 64K. The backend remains opt-in; private launch commands are experimental. Signed-off-by: Diljit Singh (cherry picked from commit d2bfa3d271d01d3204306eb4c8cb6875a1310e35) --- arch/arm64/Kconfig | 1 + arch/arm64/Kconfig.debug | 21 +++ arch/arm64/include/asm/cpucaps.h | 5 + arch/arm64/include/asm/elf.h | 9 ++ arch/arm64/include/asm/kernel-pgtable.h | 2 +- arch/arm64/include/asm/mmu.h | 40 ++++++ arch/arm64/include/asm/mmu_context.h | 61 +++++++- arch/arm64/include/asm/mte.h | 8 ++ arch/arm64/include/asm/page.h | 33 ++++- arch/arm64/include/asm/pgalloc.h | 35 ++++- arch/arm64/include/asm/pgtable-hwdef.h | 12 +- arch/arm64/include/asm/pgtable.h | 51 +++++-- arch/arm64/include/asm/shmparam.h | 6 + arch/arm64/include/asm/tlb.h | 27 +++- arch/arm64/include/asm/tlbflush.h | 124 +++++++++++------ arch/arm64/include/asm/user-pgalloc.h | 59 ++++++++ arch/arm64/include/asm/user-pgtable.h | 164 ++++++++++++++++++++++ arch/arm64/include/asm/user-pte.h | 116 ++++++++++++++++ arch/arm64/kernel/cpufeature.c | 83 ++++++++++- arch/arm64/kernel/pi/idreg-override.c | 32 +++++ arch/arm64/kernel/pi/map_kernel.c | 2 +- arch/arm64/kernel/process.c | 33 ++++- arch/arm64/kernel/sys.c | 5 + arch/arm64/kernel/sys32.c | 6 + arch/arm64/kernel/vmlinux.lds.S | 5 +- arch/arm64/lib/mte.S | 46 +++++++ arch/arm64/mm/Makefile | 1 + arch/arm64/mm/context.c | 43 ++++++ arch/arm64/mm/contpte.c | 143 +++++++++++-------- arch/arm64/mm/copypage.c | 52 +++++++ arch/arm64/mm/fault.c | 30 ++-- arch/arm64/mm/fixmap.c | 2 +- arch/arm64/mm/init.c | 5 +- arch/arm64/mm/mmu.c | 4 +- arch/arm64/mm/pgd.c | 41 +++++- arch/arm64/mm/proc.S | 16 +-- arch/arm64/mm/user4k-pgalloc.c | 176 ++++++++++++++++++++++++ arch/arm64/tools/cpucaps | 2 + 38 files changed, 1349 insertions(+), 152 deletions(-) create mode 100644 arch/arm64/include/asm/user-pgalloc.h create mode 100644 arch/arm64/include/asm/user-pgtable.h create mode 100644 arch/arm64/include/asm/user-pte.h create mode 100644 arch/arm64/mm/user4k-pgalloc.c diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index a582d84151537..31b402de5245b 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -362,6 +362,7 @@ config FIX_EARLYCON_MEM config PGTABLE_LEVELS int + default 4 if ARM64_USER4K_EXPERIMENTAL default 2 if ARM64_16K_PAGES && ARM64_VA_BITS_36 default 2 if ARM64_64K_PAGES && ARM64_VA_BITS_42 default 3 if ARM64_64K_PAGES && (ARM64_VA_BITS_48 || ARM64_VA_BITS_52) diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 265c4461031f4..d15ccdeee2125 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -21,3 +21,24 @@ config ARM64_RELOC_TEST tristate "Relocation testing module" source "drivers/hwtracing/coresight/Kconfig" + +config ARM64_USER4K_EXPERIMENTAL + bool "Experimental mixed-granule userspace" + select ARCH_HAS_MM_PGTABLE + select ARCH_HAS_USER_PAGE_SIZE + select MM_SUBPAGE + select MM_ID + select HAVE_ARCH_TLB_REMOVE_TABLE + select ARCH_SUPPORTS_PT_RECLAIM + depends on (ARM64_16K_PAGES || ARM64_64K_PAGES) && ARM64_VA_BITS_48 && ARM64_PA_BITS_48 + depends on !ARM64_LPA2 && !NO_PAGE_MAPCOUNT + default n + help + Build the in-development MM support for alternative 4K user tables + on 16K/64K kernels and 16K user tables on 64K kernels. + Ordinary address spaces and kernel tables retain the native granule. An experimental + one-shot prctl selects a supported granule for the next exec. + This is a development ABI: several MM interfaces and device mappings + still require integration and validation. The private prctl numbers + are not upstream assignments or a stable ABI promise. + Say N for normal kernels. diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index d0d3cdd5763ca..fff3a9570ad65 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -19,6 +19,11 @@ cpucap_is_possible(const unsigned int cap) "cap must be < ARM64_NCAPS"); switch (cap) { + case ARM64_HAS_USER16K_GRANULE: + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && + IS_ENABLED(CONFIG_ARM64_64K_PAGES); + case ARM64_HAS_USER4K_GRANULE: + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL); case ARM64_HAS_EPAN: return IS_ENABLED(CONFIG_ARM64_EPAN); case ARM64_SVE: diff --git a/arch/arm64/include/asm/elf.h b/arch/arm64/include/asm/elf.h index d2779d604c7be..23320b49c3208 100644 --- a/arch/arm64/include/asm/elf.h +++ b/arch/arm64/include/asm/elf.h @@ -121,6 +121,9 @@ #define CORE_DUMP_USE_REGSET #define ELF_EXEC_PAGESIZE PAGE_SIZE +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define ELF_EXEC_PAGESIZE_MM(mm) mm_page_size(mm) +#endif /* * This is the base location for PIE (ET_DYN with INTERP) loads. On @@ -181,6 +184,12 @@ do { \ NEW_AUX_ENT(AT_IGNORE, 0); \ } while (0) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +struct linux_binprm; +int arm64_bprm_prepare_mm(struct linux_binprm *bprm, bool compat); +#define arch_bprm_prepare_mm arm64_bprm_prepare_mm +#endif + #define ARCH_HAS_SETUP_ADDITIONAL_PAGES struct linux_binprm; extern int arch_setup_additional_pages(struct linux_binprm *bprm, diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h index 229ee7976f693..2bd73e773095c 100644 --- a/arch/arm64/include/asm/kernel-pgtable.h +++ b/arch/arm64/include/asm/kernel-pgtable.h @@ -27,7 +27,7 @@ #endif #define SWAPPER_BLOCK_SIZE (UL(1) << SWAPPER_BLOCK_SHIFT) -#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - SWAPPER_SKIP_LEVEL) +#define SWAPPER_PGTABLE_LEVELS (ARM64_NATIVE_PGTABLE_LEVELS - SWAPPER_SKIP_LEVEL) #define INIT_IDMAP_PGTABLE_LEVELS (IDMAP_LEVELS - SWAPPER_SKIP_LEVEL) #define IDMAP_VA_BITS 48 diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 5e1211c540ab2..15943bec53392 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -8,6 +8,10 @@ #include #define MMCF_AARCH32 0x1 /* mm context flag for AArch32 executables */ +#define MMCF_USER_4K 0x2 /* experimental, immutable mm table geometry */ +#define MMCF_USER_16K 0x4 /* experimental, immutable mm table geometry */ +#define MMCF_USER_64K 0x8 /* internal geometry; larger user ABI not enabled */ +#define MMCF_USER_GRANULE (MMCF_USER_4K | MMCF_USER_16K | MMCF_USER_64K) #define USER_ASID_BIT 48 #define USER_ASID_FLAG (UL(1) << USER_ASID_BIT) @@ -16,6 +20,15 @@ #include #include +/* AArch32 falls back to the native ABI when 4K translation is unavailable. */ +static inline unsigned int arm64_exec_page_shift(bool compat, bool has_4k, + unsigned int native_shift, unsigned int requested_shift) +{ + if (compat) + return has_4k ? 12 : native_shift; + return requested_shift ?: native_shift; +} + typedef struct { atomic64_t id; #ifdef CONFIG_COMPAT @@ -25,8 +38,35 @@ typedef struct { void *vdso; unsigned long flags; u8 pkey_allocation_map; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* Unissued PTE/PMD/PUD fragments; protected by page_table_lock. */ + void *user4k_pt_frag[3]; +#endif } mm_context_t; +static inline bool arm64_context_user4k(const mm_context_t *context) +{ + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && + (context->flags & MMCF_USER_4K); +} + +static inline unsigned int arm64_context_page_shift(const mm_context_t *context) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL)) { + if (context->flags & MMCF_USER_4K) + return 12; + if (context->flags & MMCF_USER_16K) + return 14; + if (context->flags & MMCF_USER_64K) + return 16; + } + return PAGE_SHIFT; +} + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_mm_page_shift(mm) arm64_context_page_shift(&(mm)->context) +#endif + /* * We use atomic64_read() here because the ASID for an 'mm_struct' can * be reallocated when scheduling one of its threads following a diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index 803b687581527..db27ff606555b 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -26,6 +26,17 @@ #include #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_mm_init_exec arch_mm_init_exec +static inline void arch_mm_init_exec(struct mm_struct *mm, unsigned int page_shift) +{ + if (page_shift == 12) + mm->context.flags |= MMCF_USER_4K; + else if (page_shift == 14 && PAGE_SHIFT > 14) + mm->context.flags |= MMCF_USER_16K; +} +#endif + extern bool rodata_full; static inline void contextidr_thread_switch(struct task_struct *next) @@ -53,6 +64,29 @@ static inline void cpu_set_reserved_ttbr0(void) isb(); } +/* + * The caller has installed reserved_pg_dir in TTBR0 and excludes preemption. + * On a granule transition, complete that installation before changing TG0, + * then synchronise the new walk geometry before installing a user root. + * ASIDs remain unique across both granules; switching does not flush them. + */ +static inline void cpu_set_mm_tg0(struct mm_struct *mm) +{ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + unsigned long tcr = read_sysreg(tcr_el1); + unsigned int shift = mm_page_shift(mm); + unsigned long tg0 = shift == 12 ? TCR_TG0_4K : + (shift == 14 ? TCR_TG0_16K : TCR_TG0_64K); + + if ((tcr & TCR_EL1_TG0_MASK) == tg0) + return; + + isb(); + write_sysreg((tcr & ~TCR_EL1_TG0_MASK) | tg0, tcr_el1); + isb(); +#endif +} + void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm) @@ -62,16 +96,21 @@ static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm) } /* - * Ensure TCR.T0SZ is set to the provided value. + * Set TCR.T0SZ and restore native TG0 for the idmap/transition tables. */ static inline void __cpu_set_tcr_t0sz(unsigned long t0sz) { unsigned long tcr = read_sysreg(tcr_el1); + unsigned long mask = TCR_EL1_T0SZ_MASK; - if ((tcr & TCR_EL1_T0SZ_MASK) == t0sz) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + mask |= TCR_EL1_TG0_MASK; + t0sz |= IS_ENABLED(CONFIG_ARM64_64K_PAGES) ? TCR_TG0_64K : TCR_TG0_16K; +#endif + if ((tcr & mask) == t0sz) return; - tcr &= ~TCR_EL1_T0SZ_MASK; + tcr &= ~mask; tcr |= t0sz; write_sysreg(tcr, tcr_el1); isb(); @@ -97,6 +136,8 @@ static inline void cpu_uninstall_idmap(void) local_flush_tlb_all(); __cpu_set_tcr_t0sz(TCR_T0SZ(vabits_actual)); + /* SW PAN will restore TTBR0 later, but already needs the mm's TG0. */ + cpu_set_mm_tg0(mm); if (mm != &init_mm && !system_uses_ttbr0_pan()) cpu_switch_mm(mm->pgd, mm); } @@ -168,6 +209,9 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) { atomic64_set(&mm->context.id, 0); refcount_set(&mm->context.pinned, 0); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + memset(mm->context.user4k_pt_frag, 0, sizeof(mm->context.user4k_pt_frag)); +#endif /* pkey 0 is the default, so always reserve it. */ mm->context.pkey_allocation_map = BIT(0); @@ -175,6 +219,16 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) return 0; } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm); + +static inline void destroy_context(struct mm_struct *mm) +{ + arm64_user4k_pt_cache_destroy(mm); +} +#define destroy_context destroy_context +#endif + static inline void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm) { @@ -241,6 +295,7 @@ static inline void __switch_mm(struct mm_struct *next) */ if (next == &init_mm) { cpu_set_reserved_ttbr0(); + cpu_set_mm_tg0(next); return; } diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 7f7b97e099968..890d49a4398b2 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -98,6 +98,8 @@ static inline bool try_page_mte_tagging(struct page *page) void mte_zero_clear_page_tags(void *addr); void mte_sync_tags(pte_t pte, unsigned int nr_pages); void mte_copy_page_tags(void *kto, const void *kfrom); +void mte_copy_subpage_tags_range(void *kto, const void *kfrom, unsigned long size); +void mte_clear_subpage_tags_range(void *addr, unsigned long size); void mte_thread_init_user(void); void mte_thread_switch(struct task_struct *next); void mte_cpu_setup(void); @@ -134,6 +136,12 @@ static inline void mte_sync_tags(pte_t pte, unsigned int nr_pages) static inline void mte_copy_page_tags(void *kto, const void *kfrom) { } +static inline void mte_copy_subpage_tags_range(void *kto, const void *kfrom, unsigned long size) +{ +} +static inline void mte_clear_subpage_tags_range(void *addr, unsigned long size) +{ +} static inline void mte_thread_init_user(void) { } diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h index 58200de8a221a..96fa9d8ea23c5 100644 --- a/arch/arm64/include/asm/page.h +++ b/arch/arm64/include/asm/page.h @@ -10,6 +10,11 @@ #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +/* A shared zero mapping must cover the largest supported user granule. */ +#define ARCH_ZERO_PAGE_SIZE (1UL << 16) +#endif + #ifndef __ASSEMBLER__ #include /* for READ_IMPLIES_EXEC */ @@ -26,11 +31,28 @@ void copy_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma); #define __HAVE_ARCH_COPY_USER_HIGHPAGE +#ifdef CONFIG_MM_SUBPAGE +void clear_user_subpage_range(struct page *page, unsigned int offset, + unsigned int size); +int copy_user_subpage_range(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset, + unsigned int size); +void clear_user_subpage(struct page *page, unsigned int offset); +int copy_user_subpage(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset); +#endif + void copy_highpage(struct page *to, struct page *from); #define __HAVE_ARCH_COPY_HIGHPAGE -struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, - unsigned long vaddr); +struct folio *vma_alloc_zeroed_movable_folio_order(struct vm_area_struct *vma, + unsigned long vaddr, unsigned int order); +#define vma_alloc_zeroed_movable_folio_order vma_alloc_zeroed_movable_folio_order +static inline struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, + unsigned long vaddr) +{ + return vma_alloc_zeroed_movable_folio_order(vma, vaddr, 0); +} #define vma_alloc_zeroed_movable_folio vma_alloc_zeroed_movable_folio bool tag_clear_highpages(struct page *to, int numpages, bool clear_pages); @@ -38,7 +60,14 @@ bool tag_clear_highpages(struct page *to, int numpages, bool clear_pages); #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +/* A table may occupy a 4K fragment of a native 16K physical page. */ +typedef pte_t *pgtable_t; +#define pgtable_to_page(table) virt_to_page(table) +#define pgtable_from_page(page) ((page) ? (pgtable_t)page_address(page) : NULL) +#else typedef struct page *pgtable_t; +#endif int pfn_is_map_memory(unsigned long pfn); diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h index 1b4509d3382c6..41b81203e575c 100644 --- a/arch/arm64/include/asm/pgalloc.h +++ b/arch/arm64/include/asm/pgalloc.h @@ -15,7 +15,15 @@ #define __HAVE_ARCH_PGD_FREE #define __HAVE_ARCH_PUD_FREE +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define __HAVE_ARCH_PTE_ALLOC_ONE +#define __HAVE_ARCH_PTE_FREE +#define __HAVE_ARCH_PMD_ALLOC_ONE +#define __HAVE_ARCH_PMD_FREE +#define __HAVE_ARCH_PUD_ALLOC_ONE +#endif #include +#include #define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) @@ -31,6 +39,12 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_AF; pudval |= (mm == &init_mm) ? PUD_TABLE_UXN : PUD_TABLE_PXN; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_pud(pudp, __pud(__pa(pmdp) | pudval)); + return; + } +#endif __pud_populate(pudp, __pa(pmdp), pudval); } #else @@ -53,14 +67,24 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_AF; p4dval |= (mm == &init_mm) ? P4D_TABLE_UXN : P4D_TABLE_PXN; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_p4d(p4dp, __p4d(__pa(pudp) | p4dval)); + return; + } +#endif __p4d_populate(p4dp, __pa(pudp), p4dval); } static inline void pud_free(struct mm_struct *mm, pud_t *pud) { - if (!pgtable_l4_enabled()) + if (mm_pud_folded(mm)) return; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + arm64_pgtable_free(pud); +#else __pud_free(mm, pud); +#endif } #else static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) @@ -117,7 +141,16 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) { VM_BUG_ON(mm == &init_mm); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_pmd(pmdp, __pmd(__pa(ptep) | PMD_TYPE_TABLE | + PMD_TABLE_AF | PMD_TABLE_PXN)); + return; + } + __pmd_populate(pmdp, __pa(ptep), +#else __pmd_populate(pmdp, page_to_phys(ptep), +#endif PMD_TYPE_TABLE | PMD_TABLE_AF | PMD_TABLE_PXN); } diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h index 72f31800c7030..cb211cbcdb7fd 100644 --- a/arch/arm64/include/asm/pgtable-hwdef.h +++ b/arch/arm64/include/asm/pgtable-hwdef.h @@ -28,8 +28,10 @@ * * which gets simplified as : */ +#define ARM64_HW_PGTABLE_LEVELS_FOR(va_bits, page_shift) \ + (((va_bits) - PTDESC_ORDER - 1) / ((page_shift) - PTDESC_ORDER)) #define ARM64_HW_PGTABLE_LEVELS(va_bits) \ - (((va_bits) - PTDESC_ORDER - 1) / PTDESC_TABLE_SHIFT) + ARM64_HW_PGTABLE_LEVELS_FOR(va_bits, PAGE_SHIFT) /* * Size mapped by an entry at level n ( -1 <= n <= 3) @@ -44,7 +46,10 @@ * Rearranging it a bit we get : * (4 - n) * PTDESC_TABLE_SHIFT + PTDESC_ORDER */ -#define ARM64_HW_PGTABLE_LEVEL_SHIFT(n) (PTDESC_TABLE_SHIFT * (4 - (n)) + PTDESC_ORDER) +#define ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(n, page_shift) \ + (((page_shift) - PTDESC_ORDER) * (4 - (n)) + PTDESC_ORDER) +#define ARM64_HW_PGTABLE_LEVEL_SHIFT(n) \ + ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(n, PAGE_SHIFT) #define PTRS_PER_PTE (1 << PTDESC_TABLE_SHIFT) @@ -79,7 +84,8 @@ * PGDIR_SHIFT determines the size a top-level page table entry can map * (depending on the configuration, this level can be -1, 0, 1 or 2). */ -#define PGDIR_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - CONFIG_PGTABLE_LEVELS) +#define ARM64_NATIVE_PGTABLE_LEVELS ARM64_HW_PGTABLE_LEVELS(VA_BITS) +#define PGDIR_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - ARM64_NATIVE_PGTABLE_LEVELS) #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) #define PTRS_PER_PGD (1 << (VA_BITS - PGDIR_SHIFT)) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 4dfa42b7d0535..1ce97b836c8ac 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -467,6 +467,8 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte)); } +#include + /* * Hugetlb definitions. */ @@ -711,6 +713,23 @@ static inline void __set_ptes_anysz(struct mm_struct *mm, unsigned long addr, static inline void __set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte, unsigned int nr) { + if (arm64_mm_alt_granule(mm)) { + unsigned int native_pages = DIV_ROUND_UP((pte_phys_mm(mm, pte) & ~PAGE_MASK) + + ((unsigned long)nr << mm_page_shift(mm)), PAGE_SIZE); + + page_table_check_ptes_set(mm, addr, ptep, pte, nr); + __sync_cache_and_tags(pte, native_pages); + for (;;) { + __check_safe_pte_update(mm, ptep, pte); + __set_pte_nosync(ptep, pte); + if (--nr == 0) + break; + ptep++; + pte = pte_advance_pfn_mm(mm, pte, 1); + } + __set_pte_complete(pte); + return; + } __set_ptes_anysz(mm, addr, ptep, pte, nr, PAGE_SIZE); } @@ -946,6 +965,8 @@ static inline pmd_t *pud_pgtable(pud_t pud) static __always_inline bool pgtable_l4_enabled(void) { + if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) + return false; if (CONFIG_PGTABLE_LEVELS > 4 || !IS_ENABLED(CONFIG_ARM64_LPA2)) return true; if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT)) @@ -955,6 +976,8 @@ static __always_inline bool pgtable_l4_enabled(void) static inline bool mm_pud_folded(const struct mm_struct *mm) { + if (arm64_mm_alt_granule(mm)) + return mm_page_shift(mm) == 16; return !pgtable_l4_enabled(); } #define mm_pud_folded mm_pud_folded @@ -994,6 +1017,8 @@ static inline phys_addr_t p4d_page_paddr(p4d_t p4d) static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr) { + if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) + return (pud_t *)p4dp; /* Ensure that 'p4dp' indexes a page table according to 'addr' */ VM_BUG_ON(((addr >> P4D_SHIFT) ^ ((u64)p4dp >> 3)) % PTRS_PER_P4D); @@ -1216,6 +1241,8 @@ p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr) #endif /* CONFIG_PGTABLE_LEVELS > 4 */ +#include + #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) @@ -1261,7 +1288,7 @@ static inline int __ptep_set_access_flags(struct vm_area_struct *vma, pte_t entry, int dirty) { return __ptep_set_access_flags_anysz(vma, address, ptep, entry, dirty, - PAGE_SIZE); + mm_page_size(vma->vm_mm)); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -1389,7 +1416,7 @@ static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr, if (--nr == 0) break; ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(mm); } } @@ -1402,7 +1429,7 @@ static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm, pte = __ptep_get_and_clear(mm, addr, ptep); while (--nr) { ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(mm); tmp_pte = __ptep_get_and_clear(mm, addr, ptep); if (pte_dirty(tmp_pte)) pte = pte_mkdirty(pte); @@ -1450,7 +1477,7 @@ static inline void __wrprotect_ptes(struct mm_struct *mm, unsigned long address, { unsigned int i; - for (i = 0; i < nr; i++, address += PAGE_SIZE, ptep++) + for (i = 0; i < nr; i++, address += mm_page_size(mm), ptep++) __ptep_set_wrprotect(mm, address, ptep); } @@ -1490,7 +1517,7 @@ static inline void __clear_young_dirty_ptes(struct vm_area_struct *vma, if (--nr == 0) break; ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(vma->vm_mm); } } @@ -1516,9 +1543,11 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, * bits 0-1: present (must be zero) * bits 2: remember PG_anon_exclusive * bit 3: remember uffd-wp state + * bits 4-5: 4K slot offset within a native swap page (MM_SUBPAGE) * bits 6-10: swap type * bit 11: PTE_PRESENT_INVALID (must be zero) * bits 12-61: swap offset + * bits 62-63: high 4K slot offset bits (MM_SUBPAGE, up to 64K) */ #define __SWP_TYPE_SHIFT 6 #define __SWP_TYPE_BITS 5 @@ -1688,11 +1717,11 @@ static __always_inline void contpte_try_fold(struct mm_struct *mm, * We can't fold special mappings, because there is no associated folio. */ - const unsigned long contmask = CONT_PTES - 1; - bool valign = ((addr >> PAGE_SHIFT) & contmask) == contmask; + const unsigned long contmask = cont_ptes_mm(mm) - 1; + bool valign = ((addr >> mm_page_shift(mm)) & contmask) == contmask; if (unlikely(valign)) { - bool palign = (pte_pfn(pte) & contmask) == contmask; + bool palign = ((pte_phys_mm(mm, pte) >> mm_page_shift(mm)) & contmask) == contmask; if (unlikely(palign && pte_valid(pte) && !pte_cont(pte) && !pte_special(pte))) @@ -1713,7 +1742,11 @@ static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte) if (!pte_valid_cont(pte)) return 1; - return CONT_PTES - (((unsigned long)ptep >> 3) & (CONT_PTES - 1)); + { + unsigned int nr = arm64_cont_ptes(arm64_ptep_page_shift(ptep)); + + return nr - (((unsigned long)ptep >> 3) & (nr - 1)); + } } /* diff --git a/arch/arm64/include/asm/shmparam.h b/arch/arm64/include/asm/shmparam.h index f920e22ec677c..75120d3a71889 100644 --- a/arch/arm64/include/asm/shmparam.h +++ b/arch/arm64/include/asm/shmparam.h @@ -12,6 +12,12 @@ */ #define COMPAT_SHMLBA (4 * PAGE_SIZE) +/* SHMLBA also aligns kernel allocations; scale only SysV attachments. */ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_shm_attach_align(mm, align) \ + (((align) >> PAGE_SHIFT) << mm_page_shift(mm)) +#endif + #include #endif /* __ASM_SHMPARAM_H */ diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index 751bd57bc3bac..21453f3125e0f 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -10,6 +10,11 @@ #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +void __tlb_remove_table(void *token); +/* Fragment tokens do not carry a private descriptor callback head. */ +#define tlb_table_is_ptdesc(token) (!((unsigned long)(token) & 1UL)) +#endif #define tlb_flush tlb_flush static void tlb_flush(struct mmu_gather *tlb); @@ -76,8 +81,14 @@ static inline void tlb_flush(struct mmu_gather *tlb) static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr) { - struct ptdesc *ptdesc = page_ptdesc(pte); + struct ptdesc *ptdesc = page_ptdesc(pgtable_to_page(pte)); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pte | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } @@ -87,6 +98,12 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, { struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pmdp | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } #endif @@ -97,9 +114,15 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, { struct ptdesc *ptdesc = virt_to_ptdesc(pudp); - if (!pgtable_l4_enabled()) + if (mm_pud_folded(tlb->mm)) return; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pudp | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } #endif diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 57b4eda6a72bb..56062bf906cd4 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -56,8 +57,8 @@ }) /* - * Get translation granule of the system, which is decided by - * PAGE_SIZE. Used by TTL. + * Encode a translation granule for TTL and range operands. Native kernel + * and stage-2 callers use PAGE_SHIFT; userspace callers use their mm. * - 4KB : 1 * - 16KB : 2 * - 64KB : 3 @@ -66,20 +67,25 @@ #define TLBI_TTL_TG_16K 2 #define TLBI_TTL_TG_64K 3 -static inline unsigned long get_trans_granule(void) +static inline unsigned long get_trans_granule_shift(unsigned int page_shift) { - switch (PAGE_SIZE) { - case SZ_4K: + switch (page_shift) { + case 12: return TLBI_TTL_TG_4K; - case SZ_16K: + case 14: return TLBI_TTL_TG_16K; - case SZ_64K: + case 16: return TLBI_TTL_TG_64K; default: return 0; } } +static inline unsigned long get_trans_granule(void) +{ + return get_trans_granule_shift(PAGE_SHIFT); +} + #ifdef CONFIG_ARM64_ERRATUM_4193714 extern cpumask_t sme_active_cpus; @@ -175,13 +181,14 @@ static __always_inline void ipas2e1is(u64 arg) __tlbi(ipas2e1is, arg); } -static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, - u16 asid) +static __always_inline void __tlbi_level_asid_granule(tlbi_op op, u64 addr, + u32 level, u16 asid, + unsigned int page_shift) { u64 arg = __TLBI_VADDR(addr, asid); if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && level <= 3) { - u64 ttl = level | (get_trans_granule() << 2); + u64 ttl = level | (get_trans_granule_shift(page_shift) << 2); FIELD_MODIFY(TLBI_TTL_MASK, &arg, ttl); } @@ -189,6 +196,12 @@ static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, op(arg); } +static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, + u16 asid) +{ + __tlbi_level_asid_granule(op, addr, level, asid, PAGE_SHIFT); +} + static inline void __tlbi_level(tlbi_op op, u64 addr, u32 level) { __tlbi_level_asid(op, addr, level, 0); @@ -471,36 +484,45 @@ static __always_inline void ripas2e1is(u64 arg) __tlbi(ripas2e1is, arg); } -static __always_inline void __tlbi_range(tlbi_op op, u64 addr, - u16 asid, int scale, int num, - u32 level, bool lpa2) +static __always_inline void __tlbi_range_granule(tlbi_op op, u64 addr, + u16 asid, int scale, int num, + u32 level, bool lpa2, + unsigned int page_shift) { u64 arg = 0; - arg |= FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : PAGE_SHIFT)); + arg |= FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : page_shift)); arg |= FIELD_PREP(TLBIR_TTL_MASK, level > 3 ? 0 : level); arg |= FIELD_PREP(TLBIR_NUM_MASK, num); arg |= FIELD_PREP(TLBIR_SCALE_MASK, scale); - arg |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); + arg |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule_shift(page_shift)); arg |= FIELD_PREP(TLBIR_ASID_MASK, asid); op(arg); } -static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, +/* + * Production callers supply the constant system_supports_tlb_range function, + * which inlines to the existing alternative branch inside the loop. Do not + * evaluate it into a bool outside the loop: that adds runtime flag handling + * to the native path. Recording tests supply constant true/false predicates. + */ +static __always_inline void __flush_tlb_range_op_granule(tlbi_op lop, tlbi_op rop, u64 start, size_t pages, u64 stride, u16 asid, - u32 level, bool lpa2) + u32 level, bool lpa2, + unsigned int page_shift, + bool (*range_supported)(void)) { - u64 addr = start, end = start + pages * PAGE_SIZE; + u64 addr = start, end = start + (pages << page_shift); int scale = 3; while (addr != end) { int num; - pages = (end - addr) >> PAGE_SHIFT; + pages = (end - addr) >> page_shift; - if (!system_supports_tlb_range() || pages == 1) + if (!range_supported() || pages == 1) goto invalidate_one; if (lpa2 && !IS_ALIGNED(addr, SZ_64K)) @@ -508,26 +530,41 @@ static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, num = __TLBI_RANGE_NUM(pages, scale); if (num >= 0) { - __tlbi_range(rop, addr, asid, scale, num, level, lpa2); - addr += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; + __tlbi_range_granule(rop, addr, asid, scale, num, level, + lpa2, page_shift); + addr += __TLBI_RANGE_PAGES(num, scale) << page_shift; } scale--; continue; invalidate_one: - __tlbi_level_asid(lop, addr, level, asid); + __tlbi_level_asid_granule(lop, addr, level, asid, page_shift); addr += stride; } } +static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, + u64 start, size_t pages, + u64 stride, u16 asid, + u32 level, bool lpa2) +{ + __flush_tlb_range_op_granule(lop, rop, start, pages, stride, asid, + level, lpa2, PAGE_SHIFT, system_supports_tlb_range); +} + #define __flush_s1_tlb_range_op(op, start, pages, stride, asid, tlb_level) \ __flush_tlb_range_op(op, r##op, start, pages, stride, asid, tlb_level, lpa2_is_enabled()) #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ __flush_tlb_range_op(op, r##op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_enabled()) -static inline bool __flush_tlb_range_limit_excess(unsigned long pages, - unsigned long stride) +#define __flush_user_tlb_range_op(op, start, pages, stride, asid, level, shift) \ + __flush_tlb_range_op_granule(op, r##op, start, pages, stride, asid, \ + level, lpa2_is_enabled(), shift, system_supports_tlb_range) + +static inline bool __flush_tlb_range_limit_excess_granule(unsigned long pages, + unsigned long stride, + unsigned int page_shift) { /* * Assume that the worst case number of DVM ops required to flush a @@ -538,7 +575,13 @@ static inline bool __flush_tlb_range_limit_excess(unsigned long pages, if (system_supports_tlb_range()) return pages > MAX_TLBI_RANGE_PAGES; - return pages >= (MAX_DVM_OPS * stride) >> PAGE_SHIFT; + return pages >= (MAX_DVM_OPS * stride) >> page_shift; +} + +static inline bool __flush_tlb_range_limit_excess(unsigned long pages, + unsigned long stride) +{ + return __flush_tlb_range_limit_excess_granule(pages, stride, PAGE_SHIFT); } typedef unsigned __bitwise tlbf_t; @@ -565,10 +608,11 @@ static __always_inline void __do_flush_tlb_range(struct vm_area_struct *vma, { struct mm_struct *mm = vma->vm_mm; unsigned long asid, pages; + unsigned int page_shift = mm_page_shift(mm); - pages = (end - start) >> PAGE_SHIFT; + pages = (end - start) >> page_shift; - if (__flush_tlb_range_limit_excess(pages, stride)) { + if (__flush_tlb_range_limit_excess_granule(pages, stride, page_shift)) { flush_tlb_mm(mm); return; } @@ -582,20 +626,20 @@ static __always_inline void __do_flush_tlb_range(struct vm_area_struct *vma, switch (flags & (TLBF_NOWALKCACHE | TLBF_NOBROADCAST)) { case TLBF_NONE: - __flush_s1_tlb_range_op(vae1is, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vae1is, start, pages, stride, + asid, tlb_level, page_shift); break; case TLBF_NOWALKCACHE: - __flush_s1_tlb_range_op(vale1is, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vale1is, start, pages, stride, + asid, tlb_level, page_shift); break; case TLBF_NOBROADCAST: /* Combination unused */ BUG(); break; case TLBF_NOWALKCACHE | TLBF_NOBROADCAST: - __flush_s1_tlb_range_op(vale1, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vale1, start, pages, stride, + asid, tlb_level, page_shift); break; } @@ -629,16 +673,18 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, * Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough * information here. */ - __flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE); + __flush_tlb_range(vma, start, end, mm_page_size(vma->vm_mm), + TLBI_TTL_UNKNOWN, TLBF_NONE); } static inline void __flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr, tlbf_t flags) { - unsigned long start = round_down(uaddr, PAGE_SIZE); - unsigned long end = start + PAGE_SIZE; + unsigned long page_size = mm_page_size(vma->vm_mm); + unsigned long start = round_down(uaddr, page_size); + unsigned long end = start + page_size; - __do_flush_tlb_range(vma, start, end, PAGE_SIZE, 3, + __do_flush_tlb_range(vma, start, end, page_size, 3, TLBF_NOWALKCACHE | flags); } @@ -688,7 +734,7 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b { struct vm_area_struct vma = { .vm_mm = mm, .vm_flags = 0 }; - __flush_tlb_range(&vma, start, end, PAGE_SIZE, 3, + __flush_tlb_range(&vma, start, end, mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); } diff --git a/arch/arm64/include/asm/user-pgalloc.h b/arch/arm64/include/asm/user-pgalloc.h new file mode 100644 index 0000000000000..3b4dbde0c7736 --- /dev/null +++ b/arch/arm64/include/asm/user-pgalloc.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PGALLOC_H +#define __ASM_USER_PGALLOC_H + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + +enum arm64_user4k_pt_level { + USER4K_PT_PTE, + USER4K_PT_PMD, + USER4K_PT_PUD, +}; + +/* Explicit geometry entry point; callers must keep each mm's granule fixed. */ +void *arm64_user_pt_alloc_granule(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift); +void *arm64_user4k_pt_alloc(struct mm_struct *mm, enum arm64_user4k_pt_level level); +void arm64_user4k_pt_free(void *table); +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm); +void arm64_pgtable_free(void *table); +void arm64_pte_free_defer(struct mm_struct *mm, pgtable_t table); + +static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PTE, mm_page_shift(mm)); + return __pte_alloc_one_noprof(mm, GFP_PGTABLE_USER); +} +#define pte_alloc_one(...) alloc_hooks(pte_alloc_one_noprof(__VA_ARGS__)) + +static inline void pte_free(struct mm_struct *mm, pgtable_t table) +{ + arm64_pgtable_free(table); +} + +static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PMD, mm_page_shift(mm)); + return __pmd_alloc_one_noprof(mm, addr); +} +#define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__)) + +static inline void pmd_free(struct mm_struct *mm, pmd_t *table) +{ + arm64_pgtable_free(table); +} + +static inline pud_t *pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PUD, mm_page_shift(mm)); + return __pud_alloc_one_noprof(mm, addr); +} +#define pud_alloc_one(...) alloc_hooks(pud_alloc_one_noprof(__VA_ARGS__)) + +#define pte_free_defer arm64_pte_free_defer + +#endif +#endif diff --git a/arch/arm64/include/asm/user-pgtable.h b/arch/arm64/include/asm/user-pgtable.h new file mode 100644 index 0000000000000..b244f2387a7b4 --- /dev/null +++ b/arch/arm64/include/asm/user-pgtable.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PGTABLE_H +#define __ASM_USER_PGTABLE_H + +/* Included by asm/pgtable.h after the native accessors. */ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + +/* Alternative tables use 48-bit VA; 64K folds the PUD level. */ +static inline unsigned int arm64_user_level_shift(const struct mm_struct *mm, + unsigned int level) +{ + unsigned int shift = mm_page_shift(mm); + + if (shift == 16 && !level) + level = 1; + return shift + (3 - level) * (shift - 3); +} + +static inline unsigned long arm64_user_table_index(const struct mm_struct *mm, + unsigned long addr, unsigned int level) +{ + unsigned int shift = arm64_user_level_shift(mm, level); + unsigned int bits = level ? mm_page_shift(mm) - 3 : 48 - shift; + + return (addr >> shift) & ((1UL << bits) - 1); +} + +#define p4d_none_mm(mm, p4d) \ + (arm64_mm_alt_granule(mm) ? (!mm_pud_folded(mm) && !p4d_val(p4d)) : p4d_none(p4d)) +#define p4d_bad_mm(mm, p4d) \ + (arm64_mm_alt_granule(mm) ? (!mm_pud_folded(mm) && ((p4d_val(p4d) & P4D_TYPE_MASK) != P4D_TYPE_TABLE)) : p4d_bad(p4d)) +#define p4d_present_mm(mm, p4d) (!p4d_none_mm(mm, p4d)) +static inline void p4d_clear_mm(struct mm_struct *mm, p4d_t *p4d) +{ + if (arm64_mm_alt_granule(mm)) { + if (!mm_pud_folded(mm)) + set_p4d(p4d, __p4d(0)); + } else { + p4d_clear(p4d); + } +} +#define p4d_clear_mm p4d_clear_mm + + +/* pgtable_t preserves the fragment offset; pmd_page() remains its owner. */ +#define pmd_pgtable(pmd) ((pgtable_t)__va(pmd_val(pmd) & ARM64_USER4K_TABLE_ADDR_MASK)) + +static inline void *arm64_user4k_table(u64 descriptor) +{ + /* Unlike native __pXd_to_phys(), retain sub-native table address bits. */ + return __va(descriptor & ARM64_USER4K_TABLE_ADDR_MASK); +} + +static inline pgd_t *pgd_offset_pgd_mm(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return pgd + arm64_user_table_index(mm, addr, 0); + return pgd + ((addr >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)); +} + +#define pgd_offset_pgd_mm pgd_offset_pgd_mm + +static inline pgd_t *pgd_offset_mm(struct mm_struct *mm, unsigned long addr) +{ + return pgd_offset_pgd_mm(mm, mm->pgd, addr); +} + +#define pgd_offset(mm, addr) pgd_offset_mm(mm, addr) + +static inline p4d_t *p4d_offset_mm(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (p4d_t *)pgd; + return p4d_offset(pgd, addr); +} + +#define p4d_offset_mm p4d_offset_mm + +static inline pud_t *pud_offset_mm(struct mm_struct *mm, p4d_t *p4d, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) { + if (mm_pud_folded(mm)) + return (pud_t *)p4d; + return (pud_t *)arm64_user4k_table(p4d_val(READ_ONCE(*p4d))) + + arm64_user_table_index(mm, addr, 1); + } + return pud_offset(p4d, addr); +} + +#define pud_offset_mm pud_offset_mm + +static inline pmd_t *pud_pgtable_mm(struct mm_struct *mm, pud_t pud) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user4k_table(pud_val(pud)); + return pud_pgtable(pud); +} + +#define pud_pgtable_mm pud_pgtable_mm + +static inline pmd_t *pmd_offset_mm(struct mm_struct *mm, pud_t *pud, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (pmd_t *)arm64_user4k_table(pud_val(READ_ONCE(*pud))) + + arm64_user_table_index(mm, addr, 2); + return (pmd_t *)pud_pgtable(READ_ONCE(*pud)) + + ((addr >> PMD_SHIFT) & (PTRS_PER_PMD - 1)); +} + +#define pmd_offset_mm pmd_offset_mm + +static inline pte_t *pte_offset_kernel_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (pte_t *)arm64_user4k_table(pmd_val(READ_ONCE(*pmd))) + + arm64_user_table_index(mm, addr, 3); + return (pte_t *)pmd_page_vaddr(READ_ONCE(*pmd)) + + ((addr >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); +} + +#define pte_offset_kernel_mm pte_offset_kernel_mm +#define __pte_map_mm(mm, pmd, addr) pte_offset_kernel_mm(mm, pmd, addr) + +#define pmd_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 2) : PMD_SHIFT) +#define pud_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 1) : PUD_SHIFT) +#define p4d_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 0) : P4D_SHIFT) +#define pmd_size_mm(mm) (1UL << pmd_shift_mm(mm)) +#define pud_size_mm(mm) (1UL << pud_shift_mm(mm)) +#define p4d_size_mm(mm) (1UL << p4d_shift_mm(mm)) +#define pgd_size_mm(mm) (arm64_mm_alt_granule(mm) ? (1UL << arm64_user_level_shift(mm, 0)) : PGDIR_SIZE) +#define pte_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PTE * sizeof(pte_t)) +#define pmd_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PMD * sizeof(pmd_t)) +#define pud_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PUD * sizeof(pud_t)) +#define pmd_mask_mm(mm) (~(pmd_size_mm(mm) - 1)) +#define pud_mask_mm(mm) (~(pud_size_mm(mm) - 1)) +#define p4d_mask_mm(mm) (~(p4d_size_mm(mm) - 1)) +#define pgd_mask_mm(mm) (~(pgd_size_mm(mm) - 1)) + +static inline unsigned long arm64_user4k_addr_end(unsigned long addr, + unsigned long end, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned long boundary = (addr + size) & ~(size - 1); + + return boundary - 1 < end - 1 ? boundary : end; +} + +/* Native fallbacks retain their architecture-specific folding semantics. */ +#define pgd_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 0)) : pgd_addr_end(addr, end)) +#define p4d_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? (end) : p4d_addr_end(addr, end)) +#define pud_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 1)) : pud_addr_end(addr, end)) +#define pmd_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 2)) : pmd_addr_end(addr, end)) + +#endif /* CONFIG_ARM64_USER4K_EXPERIMENTAL */ +#endif diff --git a/arch/arm64/include/asm/user-pte.h b/arch/arm64/include/asm/user-pte.h new file mode 100644 index 0000000000000..aa4c1e3a5aab9 --- /dev/null +++ b/arch/arm64/include/asm/user-pte.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PTE_H +#define __ASM_USER_PTE_H + +/* Included after native leaf accessors, before their batched setters. */ +#define ARM64_USER4K_TABLE_ADDR_MASK GENMASK_ULL(47, 12) + +static inline bool arm64_mm_user4k(const struct mm_struct *mm) +{ + return mm && arm64_context_user4k(&mm->context); +} + +static inline bool arm64_mm_alt_granule(const struct mm_struct *mm) +{ + return mm && arm64_context_page_shift(&mm->context) != PAGE_SHIFT; +} + +static inline u64 arm64_user_leaf_addr_mask(const struct mm_struct *mm) +{ + return ARM64_USER4K_TABLE_ADDR_MASK & mm_page_mask(mm); +} + +/* PFNs still identify native pages; leaf physical addresses identify slots. */ +static inline phys_addr_t pte_phys_mm(const struct mm_struct *mm, pte_t pte) +{ + if (arm64_mm_alt_granule(mm)) + return pte_val(pte) & arm64_user_leaf_addr_mask(mm); + return __pte_to_phys(pte); +} +#define pte_phys_mm pte_phys_mm + +static inline pte_t phys_pte_mm(const struct mm_struct *mm, phys_addr_t phys, + pgprot_t prot) +{ + if (arm64_mm_alt_granule(mm)) { + VM_BUG_ON(phys & ~arm64_user_leaf_addr_mask(mm)); + return __pte(phys | pgprot_val(prot)); + } + return pfn_pte(phys >> PAGE_SHIFT, prot); +} +#define phys_pte_mm phys_pte_mm + +static inline pgprot_t pte_pgprot_mm(const struct mm_struct *mm, pte_t pte) +{ + if (arm64_mm_alt_granule(mm)) + return __pgprot(pte_val(pte) & ~arm64_user_leaf_addr_mask(mm)); + return pte_pgprot(pte); +} +#define pte_pgprot_mm pte_pgprot_mm + + +#ifdef CONFIG_MM_SUBPAGE +/* Slot bits belong to the PTE, not the native swap-cache index. */ +#define ARM64_SWP_SUBPAGE_MASK (GENMASK_ULL(5, 4) | GENMASK_ULL(63, 62)) + +static inline unsigned int pte_swp_subpage_offset(pte_t pte) +{ + return ((((pte_val(pte) >> 4) & 3) | + ((pte_val(pte) >> 60) & 12)) << 12); +} + +static inline pte_t pte_swp_set_subpage_offset(pte_t pte, unsigned int offset) +{ + VM_BUG_ON(!IS_ALIGNED(offset, SZ_4K) || offset >= PAGE_SIZE); + return __pte((pte_val(pte) & ~ARM64_SWP_SUBPAGE_MASK) | + (((u64)(offset >> 12) & 3) << 4) | + (((u64)(offset >> 12) & 12) << 60)); +} +#endif + +static inline pte_t pte_advance_pfn_mm(const struct mm_struct *mm, pte_t pte, + unsigned long nr) +{ + if (arm64_mm_alt_granule(mm)) + return phys_pte_mm(mm, pte_phys_mm(mm, pte) + (nr << mm_page_shift(mm)), + pte_pgprot_mm(mm, pte)); + return pte_advance_pfn(pte, nr); +} + +#define pte_advance_pfn_mm pte_advance_pfn_mm + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +unsigned int arm64_ptep_page_shift(pte_t *ptep); +#else +static inline unsigned int arm64_ptep_page_shift(pte_t *ptep) +{ + return PAGE_SHIFT; +} +#endif + +static inline unsigned int arm64_cont_ptes(unsigned int shift) +{ + /* Architectural contiguous group lengths for 4K, 16K and 64K. */ + return shift == 12 ? 16 : shift == 14 ? 128 : 32; +} + +static inline unsigned int cont_ptes_mm(const struct mm_struct *mm) +{ + return arm64_cont_ptes(mm_page_shift(mm)); +} + +static inline unsigned long cont_pte_size_mm(const struct mm_struct *mm) +{ + return (unsigned long)cont_ptes_mm(mm) << mm_page_shift(mm); +} + +static inline unsigned long pte_cont_addr_end_mm(const struct mm_struct *mm, + unsigned long addr, unsigned long end) +{ + unsigned long size = cont_pte_size_mm(mm); + unsigned long boundary = (addr + size) & ~(size - 1); + + return boundary - 1 < end - 1 ? boundary : end; +} + +#endif diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 432d35b2714cd..e30ef03e87b3e 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1661,6 +1661,62 @@ has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) return cpufeature_matches(val, entry); } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST +/* Removal-only model for late-CPU verification in a disposable test VM. */ +static int user4k_deny_late_cpu __ro_after_init = -1; + +static int __init user4k_deny_late_cpu_setup(char *str) +{ + int cpu; + + if (kstrtoint(str, 0, &cpu) || cpu <= 0 || cpu >= NR_CPUS) + return -EINVAL; + user4k_deny_late_cpu = cpu; + return 0; +} +early_param("arm64.user4k_deny_late_cpu", user4k_deny_late_cpu_setup); +#ifdef CONFIG_ARM64_64K_PAGES +static int user16k_deny_late_cpu __ro_after_init = -1; + +static int __init user16k_deny_late_cpu_setup(char *str) +{ + int cpu; + + if (kstrtoint(str, 0, &cpu) || cpu <= 0 || cpu >= NR_CPUS) + return -EINVAL; + user16k_deny_late_cpu = cpu; + return 0; +} +early_param("arm64.user16k_deny_late_cpu", user16k_deny_late_cpu_setup); +#endif +#endif + +static bool has_user4k_granule(const struct arm64_cpu_capabilities *entry, + int scope) +{ +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (scope == SCOPE_LOCAL_CPU && system_capabilities_finalized() && + smp_processor_id() == user4k_deny_late_cpu) + return false; +#endif + return has_cpuid_feature(entry, scope); +} + +#ifdef CONFIG_ARM64_64K_PAGES +static bool has_user16k_granule(const struct arm64_cpu_capabilities *entry, + int scope) +{ +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (scope == SCOPE_LOCAL_CPU && system_capabilities_finalized() && + smp_processor_id() == user16k_deny_late_cpu) + return false; +#endif + return has_cpuid_feature(entry, scope); +} +#endif +#endif + const struct cpumask *system_32bit_el0_cpumask(void) { if (!system_supports_32bit_el0()) @@ -2391,8 +2447,11 @@ static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap) * clear the tags or set PG_mte_tagged. */ if (!cleared_zero_page) { + unsigned long offset; + cleared_zero_page = true; - mte_clear_page_tags(lm_alias(empty_zero_page)); + for (offset = 0; offset < sizeof(empty_zero_page); offset += PAGE_SIZE) + mte_clear_page_tags(lm_alias(empty_zero_page + offset)); } kasan_init_hw_tags_cpu(); @@ -2529,6 +2588,28 @@ test_has_gicv5_legacy(const struct arm64_cpu_capabilities *entry, int scope) } static const struct arm64_cpu_capabilities arm64_features[] = { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + { + .desc = "4K userspace translation granule", + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .capability = ARM64_HAS_USER4K_GRANULE, + .matches = has_user4k_granule, + __ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN4, + ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN, + ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MAX) + }, +#ifdef CONFIG_ARM64_64K_PAGES + { + .desc = "16K userspace translation granule", + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .capability = ARM64_HAS_USER16K_GRANULE, + .matches = has_user16k_granule, + __ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN16, + ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MIN, + ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MAX) + }, +#endif +#endif { .capability = ARM64_ALWAYS_BOOT, .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, diff --git a/arch/arm64/kernel/pi/idreg-override.c b/arch/arm64/kernel/pi/idreg-override.c index 1288766a68bf6..622dc2034664c 100644 --- a/arch/arm64/kernel/pi/idreg-override.c +++ b/arch/arm64/kernel/pi/idreg-override.c @@ -38,11 +38,32 @@ struct ftr_set_desc { #define FIELD(n, s, f) { .name = n, .shift = s, .width = 4, .filter = f } +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && !defined(CONFIG_ARM64_4K_PAGES) +/* VM capability-model test: only remove 4K support, never fabricate it. */ +static bool __init user4k_granule_test_filter(u64 val) +{ + return val == ID_AA64MMFR0_EL1_TGRAN4_NI; +} + +#ifdef CONFIG_ARM64_64K_PAGES +static bool __init user16k_granule_test_filter(u64 val) +{ + return val == ID_AA64MMFR0_EL1_TGRAN16_NI; +} +#endif +#endif + static const struct ftr_set_desc mmfr0 __prel64_initconst = { .name = "id_aa64mmfr0", .override = &id_aa64mmfr0_override, .fields = { FIELD("ecv", ID_AA64MMFR0_EL1_ECV_SHIFT, NULL), +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && !defined(CONFIG_ARM64_4K_PAGES) + FIELD("tgran4", ID_AA64MMFR0_EL1_TGRAN4_SHIFT, user4k_granule_test_filter), +#ifdef CONFIG_ARM64_64K_PAGES + FIELD("tgran16", ID_AA64MMFR0_EL1_TGRAN16_SHIFT, user16k_granule_test_filter), +#endif +#endif {} }, }; @@ -59,11 +80,22 @@ static bool __init mmfr1_vh_filter(u64 val) val == 0); } +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && defined(CONFIG_ARM64_SW_TTBR0_PAN) +/* VM-only coverage of deferred TTBR0 restore on CPUs which implement PAN. */ +static bool __init user4k_pan_test_filter(u64 val) +{ + return val == 0; +} +#endif + static const struct ftr_set_desc mmfr1 __prel64_initconst = { .name = "id_aa64mmfr1", .override = &id_aa64mmfr1_override, .fields = { FIELD("vh", ID_AA64MMFR1_EL1_VH_SHIFT, mmfr1_vh_filter), +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && defined(CONFIG_ARM64_SW_TTBR0_PAN) + FIELD("pan", ID_AA64MMFR1_EL1_PAN_SHIFT, user4k_pan_test_filter), +#endif {} }, }; diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c index a852264958c36..c7dd0ba8d800e 100644 --- a/arch/arm64/kernel/pi/map_kernel.c +++ b/arch/arm64/kernel/pi/map_kernel.c @@ -243,7 +243,7 @@ asmlinkage void __init early_map_kernel(u64 boot_status, phys_addr_t fdt) static char const chosen_str[] __initconst = "/chosen"; u64 va_base, pa_base = (u64)&_text; u64 kaslr_offset = pa_base % MIN_KIMG_ALIGN; - int root_level = 4 - CONFIG_PGTABLE_LEVELS; + int root_level = 4 - ARM64_NATIVE_PGTABLE_LEVELS; int va_bits = VA_BITS; int chosen; void *fdt_mapped = map_fdt(fdt); diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index ab1beffa8d7a4..a326135b164d4 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -7,11 +7,13 @@ * Copyright (C) 2012 ARM Ltd. */ #include +#include #include #include #include #include #include +#include #include #include #include @@ -884,6 +886,33 @@ unsigned long arch_align_stack(unsigned long sp) return sp & ~0xf; } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +int arm64_bprm_prepare_mm(struct linux_binprm *bprm, bool compat) +{ + unsigned int requested = current->exec_page_shift ?: + current->default_exec_page_shift; + unsigned int shift = arm64_exec_page_shift(compat, + cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE), PAGE_SHIFT, + requested); + + return bprm_set_page_shift(bprm, shift); +} + +/* Return zero for the native default, or a validated alternative shift. */ +int arch_exec_page_size_shift(unsigned long size) +{ + if (!size || size == PAGE_SIZE) + return 0; + if (size == SZ_4K) + return cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE) ? + 12 : -EOPNOTSUPP; + if (PAGE_SIZE > SZ_16K && size == SZ_16K) + return cpus_have_final_cap(ARM64_HAS_USER16K_GRANULE) ? + 14 : -EOPNOTSUPP; + return -EINVAL; +} +#endif + #ifdef CONFIG_COMPAT int compat_elf_check_arch(const struct elf32_hdr *hdr) { @@ -911,10 +940,10 @@ int compat_elf_check_arch(const struct elf32_hdr *hdr) */ void arch_setup_new_exec(void) { - unsigned long mmflags = 0; + unsigned long mmflags = current->mm->context.flags & MMCF_USER_GRANULE; if (is_compat_task()) { - mmflags = MMCF_AARCH32; + mmflags |= MMCF_AARCH32; /* * Restrict the CPU affinity mask for a 32-bit task so that diff --git a/arch/arm64/kernel/sys.c b/arch/arm64/kernel/sys.c index f08408b6e8263..bdd2531554f27 100644 --- a/arch/arm64/kernel/sys.c +++ b/arch/arm64/kernel/sys.c @@ -22,6 +22,11 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, off) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return ksys_mmap_user(addr, len, prot, flags, fd, off); +#endif + if (offset_in_page(off) != 0) return -EINVAL; diff --git a/arch/arm64/kernel/sys32.c b/arch/arm64/kernel/sys32.c index 12a948f3a5043..ba66b65f506f4 100644 --- a/arch/arm64/kernel/sys32.c +++ b/arch/arm64/kernel/sys32.c @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -51,6 +52,11 @@ COMPAT_SYSCALL_DEFINE6(aarch32_mmap2, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, off_4k) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return ksys_mmap_user(addr, len, prot, flags, fd, off_4k << 12); +#endif + if (off_4k & (~PAGE_MASK >> 12)) return -EINVAL; diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index e1ac876200a3d..688b82c0cf30d 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -66,9 +66,10 @@ * We require that __hyp_bss_start and __bss_start are aligned, and enforce it * with an assertion. But the BSS_SECTION macro places an empty .sbss section * between them, which can in some cases cause the linker to misalign them. To - * work around the issue, force a page alignment for __bss_start. + * work around the issue, align __bss_start to the BSS section's alignment, + * which can exceed PAGE_SIZE for the mixed-granule zero page. */ -#define SBSS_ALIGN PAGE_SIZE +#define SBSS_ALIGN MAX(PAGE_SIZE, ALIGNOF(.bss)) #else /* CONFIG_KVM */ #define HYPERVISOR_EXTABLE #define HYPERVISOR_RODATA_SECTIONS diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S index 5018ac03b6bf3..c1ce370068cf4 100644 --- a/arch/arm64/lib/mte.S +++ b/arch/arm64/lib/mte.S @@ -80,6 +80,52 @@ SYM_FUNC_START(mte_copy_page_tags) ret SYM_FUNC_END(mte_copy_page_tags) +#ifdef CONFIG_MM_SUBPAGE +/* Copy one aligned power-of-two extent, preserving neighbouring tags. */ +SYM_FUNC_START(mte_copy_subpage_tags_range) + mov x6, x2 + add x2, x0, x2 + multitag_transfer_size x4, x5 + cmp x4, x6 + b.hi 2f +1: ldgm x3, [x1] + stgm x3, [x0] + add x0, x0, x4 + add x1, x1, x4 + cmp x0, x2 + b.ne 1b + ret +2: mov x3, xzr +3: ldg x3, [x1] + stg x3, [x0] + add x0, x0, #MTE_GRANULE_SIZE + add x1, x1, #MTE_GRANULE_SIZE + cmp x0, x2 + b.ne 3b + ret +SYM_FUNC_END(mte_copy_subpage_tags_range) + +/* Clear an aligned extent without changing data or neighbouring tags. */ +SYM_FUNC_START(mte_clear_subpage_tags_range) + mov x4, x1 + add x1, x0, x1 + multitag_transfer_size x2, x3 + cmp x2, x4 + b.hi 2f +1: stgm xzr, [x0] + add x0, x0, x2 + cmp x0, x1 + b.ne 1b + ret +2: mov x2, xzr +3: stg x2, [x0] + add x0, x0, #MTE_GRANULE_SIZE + cmp x0, x1 + b.ne 3b + ret +SYM_FUNC_END(mte_clear_subpage_tags_range) +#endif + /* * Read tags from a user buffer (one tag per byte) and set the corresponding * tags at the given kernel address. Used by PTRACE_POKEMTETAGS. diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index c26489cf96cd9..32d18488e527c 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_TRANS_TABLE) += trans_pgd-asm.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ARM64_MTE) += mteswap.o obj-$(CONFIG_ARM64_GCS) += gcs.o +obj-$(CONFIG_ARM64_USER4K_EXPERIMENTAL) += user4k-pgalloc.o KASAN_SANITIZE_physaddr.o += n obj-$(CONFIG_KASAN) += kasan_init.o diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index 0f4a28b874692..cd3f56b942708 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -38,6 +39,34 @@ static unsigned long *pinned_asid_map; #define ctxid2asid(asid) ((asid) & ~ASID_MASK) #define asid2ctxid(asid, genid) ((asid) | (genid)) +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST +/* Test-only capacity reduction; hardware ASID width and normal boots are unchanged. */ +static bool user4k_test_small_asids; + +static int __init user4k_test_asid_bits_setup(char *str) +{ + unsigned int bits; + + if (kstrtouint(str, 0, &bits) || bits != 8) + return -EINVAL; + user4k_test_small_asids = true; + return 0; +} +early_param("arm64.user4k_test_asid_bits", user4k_test_asid_bits_setup); + +/* Read-only evidence that an EL0 stress run actually crossed ASID rollover. */ +static int user4k_asid_generation_get(char *buffer, const struct kernel_param *kp) +{ + return sysfs_emit(buffer, "%llu\n", (unsigned long long)atomic64_read(&asid_generation)); +} + +static const struct kernel_param_ops user4k_asid_generation_ops = { + .get = user4k_asid_generation_get, +}; +module_param_cb(user4k_asid_generation, &user4k_asid_generation_ops, NULL, 0444); +MODULE_PARM_DESC(user4k_asid_generation, "Current ASID generation for mixed-granule VM tests"); +#endif + /* Get the ASIDBits supported by the current CPU */ static u32 get_cpu_asid_bits(void) { @@ -268,6 +297,13 @@ void check_and_switch_context(struct mm_struct *mm) */ if (!system_uses_ttbr0_pan()) cpu_switch_mm(mm->pgd, mm); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + else { + /* uaccess/EL0 return restore only TTBRs, so select TG0 here. */ + cpu_set_reserved_ttbr0_nosync(); + cpu_set_mm_tg0(mm); + } +#endif } unsigned long arm64_mm_context_get(struct mm_struct *mm) @@ -365,6 +401,7 @@ void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) ttbr1 |= FIELD_PREP(TTBRx_EL1_ASID_MASK, asid); cpu_set_reserved_ttbr0_nosync(); + cpu_set_mm_tg0(mm); write_sysreg(ttbr1, ttbr1_el1); write_sysreg(ttbr0, ttbr0_el1); isb(); @@ -401,6 +438,12 @@ arch_initcall(asids_update_limit); static int asids_init(void) { asid_bits = get_cpu_asid_bits(); +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (user4k_test_small_asids) { + pr_info("user4k test: limiting ASID allocator to 8 bits\n"); + asid_bits = 8; + } +#endif atomic64_set(&asid_generation, ASID_FIRST_VERSION); asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); if (!asid_map) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index 2de12656b4d8b..f8faf1a99eeca 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -8,6 +8,25 @@ #include #include +static inline unsigned int contpte_nr(pte_t *ptep) +{ + return arm64_cont_ptes(arm64_ptep_page_shift(ptep)); +} + +static inline phys_addr_t contpte_phys(pte_t pte, unsigned int shift) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && shift != PAGE_SHIFT) + return pte_val(pte) & (ARM64_USER4K_TABLE_ADDR_MASK & ~((1ULL << shift) - 1)); + return __pte_to_phys(pte); +} + +static inline pgprot_t contpte_prot(pte_t pte, unsigned int shift) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && shift != PAGE_SHIFT) + return __pgprot(pte_val(pte) & ~(ARM64_USER4K_TABLE_ADDR_MASK & ~((1ULL << shift) - 1))); + return pte_pgprot(pte); +} + static inline bool mm_is_user(struct mm_struct *mm) { /* @@ -23,10 +42,10 @@ static inline bool mm_is_user(struct mm_struct *mm) static inline pte_t *contpte_align_down(pte_t *ptep) { - return PTR_ALIGN_DOWN(ptep, sizeof(*ptep) * CONT_PTES); + return PTR_ALIGN_DOWN(ptep, sizeof(*ptep) * contpte_nr(ptep)); } -static inline pte_t *contpte_align_addr_ptep(unsigned long *start, +static inline pte_t *contpte_align_addr_ptep(struct mm_struct *mm, unsigned long *start, unsigned long *end, pte_t *ptep, unsigned int nr) { @@ -36,10 +55,10 @@ static inline pte_t *contpte_align_addr_ptep(unsigned long *start, * single VMA and a single page table. */ if (pte_cont(__ptep_get(ptep + nr - 1))) - *end = ALIGN(*end, CONT_PTE_SIZE); + *end = ALIGN(*end, cont_pte_size_mm(mm)); if (pte_cont(__ptep_get(ptep))) { - *start = ALIGN_DOWN(*start, CONT_PTE_SIZE); + *start = ALIGN_DOWN(*start, cont_pte_size_mm(mm)); ptep = contpte_align_down(ptep); } @@ -54,11 +73,11 @@ static void contpte_try_unfold_partial(struct mm_struct *mm, unsigned long addr, * of the range. */ - if (ptep != contpte_align_down(ptep) || nr < CONT_PTES) + if (ptep != contpte_align_down(ptep) || nr < cont_ptes_mm(mm)) contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); if (ptep + nr != contpte_align_down(ptep + nr)) { - unsigned long last_addr = addr + PAGE_SIZE * (nr - 1); + unsigned long last_addr = addr + mm_page_size(mm) * (nr - 1); pte_t *last_ptep = ptep + nr - 1; contpte_try_unfold(mm, last_addr, last_ptep, @@ -75,10 +94,11 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, int i; start_ptep = ptep = contpte_align_down(ptep); - start_addr = addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); - pte = pfn_pte(ALIGN_DOWN(pte_pfn(pte), CONT_PTES), pte_pgprot(pte)); + start_addr = addr = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); + pte = phys_pte_mm(mm, ALIGN_DOWN(pte_phys_mm(mm, pte), cont_pte_size_mm(mm)), + pte_pgprot_mm(mm, pte)); - for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) { + for (i = 0; i < cont_ptes_mm(mm); i++, ptep++, addr += mm_page_size(mm)) { pte_t ptent = __ptep_get_and_clear(mm, addr, ptep); if (pte_dirty(ptent)) @@ -225,10 +245,10 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, */ if (!system_supports_bbml2_noabort()) - __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, 3, + __flush_tlb_range(&vma, start_addr, addr, mm_page_size(mm), 3, TLBF_NOWALKCACHE); - __set_ptes(mm, start_addr, start_ptep, pte, CONT_PTES); + __set_ptes(mm, start_addr, start_ptep, pte, cont_ptes_mm(mm)); } void __contpte_try_fold(struct mm_struct *mm, unsigned long addr, @@ -269,24 +289,26 @@ void __contpte_try_fold(struct mm_struct *mm, unsigned long addr, page = pte_page(pte); folio = page_folio(page); folio_start = addr - (page - &folio->page) * PAGE_SIZE; + if (arm64_mm_alt_granule(mm)) + folio_start -= pte_phys_mm(mm, pte) & ~PAGE_MASK; folio_end = folio_start + folio_nr_pages(folio) * PAGE_SIZE; - cont_start = ALIGN_DOWN(addr, CONT_PTE_SIZE); - cont_end = cont_start + CONT_PTE_SIZE; + cont_start = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); + cont_end = cont_start + cont_pte_size_mm(mm); if (folio_start > cont_start || folio_end < cont_end) return; - pfn = ALIGN_DOWN(pte_pfn(pte), CONT_PTES); - prot = pte_pgprot(pte_mkold(pte_mkclean(pte))); - expected_pte = pfn_pte(pfn, prot); + pfn = ALIGN_DOWN(pte_phys_mm(mm, pte) >> mm_page_shift(mm), cont_ptes_mm(mm)); + prot = pte_pgprot_mm(mm, pte_mkold(pte_mkclean(pte))); + expected_pte = phys_pte_mm(mm, pfn << mm_page_shift(mm), prot); orig_ptep = ptep; ptep = contpte_align_down(ptep); - for (i = 0; i < CONT_PTES; i++) { + for (i = 0; i < cont_ptes_mm(mm); i++) { subpte = pte_mkold(pte_mkclean(__ptep_get(ptep))); if (!pte_same(subpte, expected_pte)) return; - expected_pte = pte_advance_pfn(expected_pte, 1); + expected_pte = pte_advance_pfn_mm(mm, expected_pte, 1); ptep++; } @@ -319,17 +341,18 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) * feet. */ + unsigned int nr = contpte_nr(ptep); pte_t pte; int i; ptep = contpte_align_down(ptep); - for (i = 0; i < CONT_PTES; i++, ptep++) { + for (i = 0; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); - for (; i < CONT_PTES; i++, ptep++) { + for (; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_young(pte)) { orig_pte = pte_mkyoung(orig_pte); @@ -343,7 +366,7 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) orig_pte = pte_mkyoung(orig_pte); i++; ptep++; - for (; i < CONT_PTES; i++, ptep++) { + for (; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); @@ -359,11 +382,12 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) EXPORT_SYMBOL_GPL(contpte_ptep_get); static inline bool contpte_is_consistent(pte_t pte, unsigned long pfn, + unsigned int shift, pgprot_t orig_prot) { - pgprot_t prot = pte_pgprot(pte_mkold(pte_mkclean(pte))); + pgprot_t prot = contpte_prot(pte_mkold(pte_mkclean(pte)), shift); - return pte_valid_cont(pte) && pte_pfn(pte) == pfn && + return pte_valid_cont(pte) && (contpte_phys(pte, shift) >> shift) == pfn && pgprot_val(prot) == pgprot_val(orig_prot); } @@ -386,6 +410,8 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) * because it is not part of a contpte range. */ + unsigned int shift = arm64_ptep_page_shift(orig_ptep); + unsigned int nr = arm64_cont_ptes(shift); pgprot_t orig_prot; unsigned long pfn; pte_t orig_pte; @@ -399,22 +425,22 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) if (!pte_valid_cont(orig_pte)) return orig_pte; - orig_prot = pte_pgprot(pte_mkold(pte_mkclean(orig_pte))); + orig_prot = contpte_prot(pte_mkold(pte_mkclean(orig_pte)), shift); ptep = contpte_align_down(orig_ptep); - pfn = pte_pfn(orig_pte) - (orig_ptep - ptep); + pfn = (contpte_phys(orig_pte, shift) >> shift) - (orig_ptep - ptep); - for (i = 0; i < CONT_PTES; i++, ptep++, pfn++) { + for (i = 0; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); - for (; i < CONT_PTES; i++, ptep++, pfn++) { + for (; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_young(pte)) { @@ -430,10 +456,10 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) i++; ptep++; pfn++; - for (; i < CONT_PTES; i++, ptep++, pfn++) { + for (; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_dirty(pte)) { @@ -468,16 +494,16 @@ void contpte_set_ptes(struct mm_struct *mm, unsigned long addr, if (!mm_is_user(mm)) return __set_ptes(mm, addr, ptep, pte, nr); - end = addr + (nr << PAGE_SHIFT); - pfn = pte_pfn(pte); - prot = pte_pgprot(pte); + end = addr + (nr << mm_page_shift(mm)); + pfn = pte_phys_mm(mm, pte) >> mm_page_shift(mm); + prot = pte_pgprot_mm(mm, pte); do { - next = pte_cont_addr_end(addr, end); - nr = (next - addr) >> PAGE_SHIFT; - pte = pfn_pte(pfn, prot); + next = pte_cont_addr_end_mm(mm, addr, end); + nr = (next - addr) >> mm_page_shift(mm); + pte = phys_pte_mm(mm, pfn << mm_page_shift(mm), prot); - if (((addr | next | (pfn << PAGE_SHIFT)) & ~CONT_PTE_MASK) == 0) + if (((addr | next | (pfn << mm_page_shift(mm))) & (cont_pte_size_mm(mm) - 1)) == 0) pte = pte_mkcont(pte); else pte = pte_mknoncont(pte); @@ -512,6 +538,8 @@ EXPORT_SYMBOL_GPL(contpte_get_and_clear_full_ptes); bool contpte_test_and_clear_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr) { + struct mm_struct *mm = vma->vm_mm; + /* * ptep_clear_flush_young() technically requires us to clear the access * flag for a _single_ pte. However, the core-mm code actually tracks @@ -524,11 +552,11 @@ bool contpte_test_and_clear_young_ptes(struct vm_area_struct *vma, * of the same large folio in a single VMA and a single page table. */ - unsigned long end = addr + nr * PAGE_SIZE; + unsigned long end = addr + nr * mm_page_size(mm); bool young = false; - ptep = contpte_align_addr_ptep(&addr, &end, ptep, nr); - for (; addr != end; ptep++, addr += PAGE_SIZE) + ptep = contpte_align_addr_ptep(vma->vm_mm, &addr, &end, ptep, nr); + for (; addr != end; ptep++, addr += mm_page_size(mm)) young |= __ptep_test_and_clear_young(vma, addr, ptep); return young; @@ -538,19 +566,21 @@ EXPORT_SYMBOL_GPL(contpte_test_and_clear_young_ptes); bool contpte_clear_flush_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr) { + struct mm_struct *mm = vma->vm_mm; + bool young; young = contpte_test_and_clear_young_ptes(vma, addr, ptep, nr); if (young) { - unsigned long end = addr + nr * PAGE_SIZE; + unsigned long end = addr + nr * mm_page_size(mm); - contpte_align_addr_ptep(&addr, &end, ptep, nr); + contpte_align_addr_ptep(vma->vm_mm, &addr, &end, ptep, nr); /* * See comment in __ptep_clear_flush_young(); same rationale for * eliding the trailing DSB applies here. */ - __flush_tlb_range(vma, addr, end, PAGE_SIZE, 3, + __flush_tlb_range(vma, addr, end, mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); } @@ -580,6 +610,8 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr, cydp_t flags) { + struct mm_struct *mm = vma->vm_mm; + /* * We can safely clear access/dirty without needing to unfold from * the architectures perspective, even when contpte is set. If the @@ -591,14 +623,15 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, * clearing access/dirty for the whole block. */ unsigned long start = addr; - unsigned long end = start + nr * PAGE_SIZE; + unsigned long end = start + nr * mm_page_size(mm); - ptep = contpte_align_addr_ptep(&start, &end, ptep, nr); - __clear_young_dirty_ptes(vma, start, ptep, (end - start) / PAGE_SIZE, flags); + ptep = contpte_align_addr_ptep(vma->vm_mm, &start, &end, ptep, nr); + __clear_young_dirty_ptes(vma, start, ptep, (end - start) >> mm_page_shift(mm), flags); } EXPORT_SYMBOL_GPL(contpte_clear_young_dirty_ptes); -static bool contpte_all_subptes_match_access_flags(pte_t *ptep, pte_t entry) +static bool contpte_all_subptes_match_access_flags(struct mm_struct *mm, + pte_t *ptep, pte_t entry) { pte_t *cont_ptep = contpte_align_down(ptep); /* @@ -609,7 +642,7 @@ static bool contpte_all_subptes_match_access_flags(pte_t *ptep, pte_t entry) pteval_t entry_cmp = pte_val(entry) & cmp_mask; int i; - for (i = 0; i < CONT_PTES; i++) { + for (i = 0; i < cont_ptes_mm(mm); i++) { pteval_t pte_cmp = pte_val(__ptep_get(cont_ptep + i)) & cmp_mask; if (pte_cmp != entry_cmp) @@ -623,6 +656,8 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t entry, int dirty) { + struct mm_struct *mm = vma->vm_mm; + unsigned long start_addr; pte_t orig_pte; int i; @@ -651,7 +686,7 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * no-op, and when any sub-PTE mismatches, proceed to update the whole * range. */ - if (contpte_all_subptes_match_access_flags(ptep, entry)) + if (contpte_all_subptes_match_access_flags(mm, ptep, entry)) return 0; /* @@ -672,7 +707,7 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * and instead flush the whole range at the end. */ ptep = contpte_align_down(ptep); - start_addr = addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); + start_addr = addr = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); /* * We are not advancing entry because __ptep_set_access_flags() @@ -680,13 +715,13 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * for the whole contpte block and returned early, pte_same() * within __ptep_set_access_flags() is likely false. */ - for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) + for (i = 0; i < cont_ptes_mm(mm); i++, ptep++, addr += mm_page_size(mm)) __ptep_set_access_flags(vma, addr, ptep, entry, 0); if (dirty) __flush_tlb_range(vma, start_addr, - start_addr + CONT_PTE_SIZE, - PAGE_SIZE, 3, + start_addr + cont_pte_size_mm(mm), + mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOBROADCAST); } else { __contpte_try_unfold(vma->vm_mm, addr, ptep, orig_pte); diff --git a/arch/arm64/mm/copypage.c b/arch/arm64/mm/copypage.c index cd5912ba617b7..b81ccf42b22d1 100644 --- a/arch/arm64/mm/copypage.c +++ b/arch/arm64/mm/copypage.c @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -72,3 +73,54 @@ void copy_user_highpage(struct page *to, struct page *from, flush_dcache_page(to); } EXPORT_SYMBOL_GPL(copy_user_highpage); + +#ifdef CONFIG_MM_SUBPAGE +void clear_user_subpage_range(struct page *page, unsigned int offset, unsigned int size) +{ + void *addr = page_address(page) + offset; + + VM_BUG_ON(!is_power_of_2(size) || size < SZ_4K || size > PAGE_SIZE); + VM_BUG_ON(!IS_ALIGNED(offset, size) || offset > PAGE_SIZE - size); + memset(addr, 0, size); + if (system_supports_mte() && page_mte_tagged(page)) + mte_clear_subpage_tags_range(addr, size); + flush_dcache_page(page); +} + +int copy_user_subpage_range(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset, unsigned int size) +{ + void *kto = page_address(to), *kfrom = page_address(from); + + VM_BUG_ON(!is_power_of_2(size) || size < SZ_4K || size > PAGE_SIZE); + VM_BUG_ON(!IS_ALIGNED(to_offset, size) || to_offset > PAGE_SIZE - size); + VM_BUG_ON(!IS_ALIGNED(from_offset, size) || from_offset > PAGE_SIZE - size); + if (PageHWPoison(from) || copy_mc_to_kernel(kto + to_offset, kfrom + from_offset, size)) + return -EHWPOISON; + if (kasan_hw_tags_enabled()) + page_kasan_tag_reset(to); + if (system_supports_mte()) { + if (page_mte_tagged(from)) { + if (try_page_mte_tagging(to)) + mte_clear_page_tags(kto); + mte_copy_subpage_tags_range(kto + to_offset, kfrom + from_offset, size); + set_page_mte_tagged(to); + } else if (page_mte_tagged(to)) { + mte_clear_subpage_tags_range(kto + to_offset, size); + } + } + flush_dcache_page(to); + return 0; +} +/* Existing 4K fault and ownership callers keep their fixed-size contract. */ +void clear_user_subpage(struct page *page, unsigned int offset) +{ + clear_user_subpage_range(page, offset, SZ_4K); +} + +int copy_user_subpage(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset) +{ + return copy_user_subpage_range(to, to_offset, from, from_offset, SZ_4K); +} +#endif diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 739800835920c..6a37063417c4c 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -150,7 +150,7 @@ static void show_pte(unsigned long addr) } pr_alert("%s pgtable: %luk pages, %llu-bit VAs, pgdp=%016lx\n", - mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K, + mm == &init_mm ? "swapper" : "user", mm_page_size(mm) / SZ_1K, vabits_actual, mm_to_pgd_phys(mm)); pgdp = pgd_offset(mm, addr); pgd = READ_ONCE(*pgdp); @@ -165,25 +165,25 @@ static void show_pte(unsigned long addr) if (pgd_none(pgd) || pgd_bad(pgd)) break; - p4dp = p4d_offset(pgdp, addr); + p4dp = p4d_offset_mm(mm, pgdp, addr); p4d = READ_ONCE(*p4dp); pr_cont(", p4d=%016llx", p4d_val(p4d)); - if (p4d_none(p4d) || p4d_bad(p4d)) + if (p4d_none_mm(mm, p4d) || p4d_bad_mm(mm, p4d)) break; - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_mm(mm, p4dp, addr); pud = READ_ONCE(*pudp); pr_cont(", pud=%016llx", pud_val(pud)); if (pud_none(pud) || pud_bad(pud)) break; - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_mm(mm, pudp, addr); pmd = READ_ONCE(*pmdp); pr_cont(", pmd=%016llx", pmd_val(pmd)); if (pmd_none(pmd) || pmd_bad(pmd)) break; - ptep = pte_offset_map(pmdp, addr); + ptep = pte_offset_map_mm(mm, pmdp, addr); if (!ptep) break; @@ -241,19 +241,15 @@ int __ptep_set_access_flags_anysz(struct vm_area_struct *vma, * flush_tlb_fix_spurious_fault(). */ if (dirty) { - switch (pgsize) { - case PAGE_SIZE: + if (pgsize == mm_page_size(vma->vm_mm)) { level = 3; - break; - case PMD_SIZE: + } else if (pgsize == pmd_size_mm(vma->vm_mm)) { level = 2; - break; #ifndef __PAGETABLE_PMD_FOLDED - case PUD_SIZE: + } else if (pgsize == pud_size_mm(vma->vm_mm)) { level = 1; - break; #endif - default: + } else { level = TLBI_TTL_UNKNOWN; WARN_ON(1); } @@ -1002,8 +998,8 @@ NOKPROBE_SYMBOL(do_sp_pc_abort); /* * Used during anonymous page fault handling. */ -struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, - unsigned long vaddr) +struct folio *vma_alloc_zeroed_movable_folio_order(struct vm_area_struct *vma, + unsigned long vaddr, unsigned int order) { gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO; @@ -1015,7 +1011,7 @@ struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, if (vma->vm_flags & VM_MTE) flags |= __GFP_ZEROTAGS; - return vma_alloc_folio(flags, 0, vma, vaddr); + return vma_alloc_folio(flags, order, vma, vaddr); } bool tag_clear_highpages(struct page *page, int numpages, bool clear_pages) diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c index c5c5425791da7..b670d27a898cc 100644 --- a/arch/arm64/mm/fixmap.c +++ b/arch/arm64/mm/fixmap.c @@ -77,7 +77,7 @@ static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr, p4d_t p4d = READ_ONCE(*p4dp); pud_t *pudp; - if (CONFIG_PGTABLE_LEVELS > 3 && !p4d_none(p4d) && + if (pgtable_l4_enabled() && !p4d_none(p4d) && p4d_page_paddr(p4d) != __pa_symbol(bm_pud)) { /* * We only end up here if the kernel mapping and the fixmap diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 97987f850a33c..1e5f314b542d5 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -368,8 +368,9 @@ void __init arch_mm_preinit(void) * Selected page table levels should match when derived from * scratch using the virtual address range and page size. */ - BUILD_BUG_ON(ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS) != - CONFIG_PGTABLE_LEVELS); + BUILD_BUG_ON(CONFIG_PGTABLE_LEVELS != + (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) ? 4 : + ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS))); if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) { extern int sysctl_overcommit_memory; diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 09fd9da4eeed4..976d115d72f3f 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -2253,8 +2253,8 @@ pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr, * in cases where cpu is affected with errata #2645198. */ if (pte_accessible(vma->vm_mm, pte) && pte_user_exec(pte)) - __flush_tlb_range(vma, addr, nr * PAGE_SIZE, - PAGE_SIZE, 3, TLBF_NOWALKCACHE); + __flush_tlb_range(vma, addr, addr + nr * mm_page_size(vma->vm_mm), + mm_page_size(vma->vm_mm), 3, TLBF_NOWALKCACHE); } return pte; diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c index bf5110b91e2fa..3b361b95322e1 100644 --- a/arch/arm64/mm/pgd.c +++ b/arch/arm64/mm/pgd.c @@ -16,14 +16,19 @@ #include static struct kmem_cache *pgd_cache __ro_after_init; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +static struct kmem_cache *user4k_pgd_cache __ro_after_init; +static struct kmem_cache *user16k_pgd_cache __ro_after_init; +static struct kmem_cache *user64k_pgd_cache __ro_after_init; +#endif static bool pgdir_is_page_size(void) { if (PGD_SIZE == PAGE_SIZE) return true; - if (CONFIG_PGTABLE_LEVELS == 4) + if (ARM64_NATIVE_PGTABLE_LEVELS == 4) return !pgtable_l4_enabled(); - if (CONFIG_PGTABLE_LEVELS == 5) + if (ARM64_NATIVE_PGTABLE_LEVELS == 5) return !pgtable_l5_enabled(); return false; } @@ -32,6 +37,14 @@ pgd_t *pgd_alloc(struct mm_struct *mm) { gfp_t gfp = GFP_PGTABLE_USER; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_context_user4k(&mm->context)) + return kmem_cache_alloc(user4k_pgd_cache, gfp); + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 14) + return kmem_cache_alloc(user16k_pgd_cache, gfp); + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 16) + return kmem_cache_alloc(user64k_pgd_cache, gfp); +#endif if (pgdir_is_page_size()) return __pgd_alloc(mm, 0); else @@ -40,6 +53,20 @@ pgd_t *pgd_alloc(struct mm_struct *mm) void pgd_free(struct mm_struct *mm, pgd_t *pgd) { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_context_user4k(&mm->context)) { + kmem_cache_free(user4k_pgd_cache, pgd); + return; + } + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 14) { + kmem_cache_free(user16k_pgd_cache, pgd); + return; + } + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 16) { + kmem_cache_free(user64k_pgd_cache, pgd); + return; + } +#endif if (pgdir_is_page_size()) __pgd_free(mm, pgd); else @@ -48,6 +75,16 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd) void __init pgtable_cache_init(void) { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (PAGE_SHIFT < 16) + user64k_pgd_cache = kmem_cache_create("user64k_pgd_cache", SZ_64K, SZ_64K, + SLAB_PANIC, NULL); + if (PAGE_SHIFT != 14) + user16k_pgd_cache = kmem_cache_create("user16k_pgd_cache", SZ_16K, SZ_16K, + SLAB_PANIC, NULL); + user4k_pgd_cache = kmem_cache_create("user4k_pgd_cache", SZ_4K, SZ_4K, + SLAB_PANIC, NULL); +#endif if (pgdir_is_page_size()) return; diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 22866b49be372..7f947589a3b4d 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -291,7 +291,7 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) cbnz cpu, __idmap_kpti_secondary -#if CONFIG_PGTABLE_LEVELS > 4 +#if ARM64_NATIVE_PGTABLE_LEVELS > 4 stp x29, x30, [sp, #-32]! mov x29, sp stp x19, x20, [sp, #16] @@ -352,7 +352,7 @@ alternative_else_nop_endif /* Set the flag to zero to indicate that we're all done */ str wzr, [flag_ptr] -#if CONFIG_PGTABLE_LEVELS > 4 +#if ARM64_NATIVE_PGTABLE_LEVELS > 4 ldp x19, x20, [sp, #16] ldp x29, x30, [sp], #32 #endif @@ -360,39 +360,39 @@ alternative_else_nop_endif .Lderef_pgd: /* P4D */ - .if CONFIG_PGTABLE_LEVELS > 4 + .if ARM64_NATIVE_PGTABLE_LEVELS > 4 p4d .req x30 pte_to_phys cur_p4dp, pgd kpti_map_pgtbl p4d, 0 kpti_mk_tbl_ng p4d, PTRS_PER_P4D b .Lnext_pgd - .else /* CONFIG_PGTABLE_LEVELS <= 4 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 4 */ p4d .req pgd .set .Lnext_p4d, .Lnext_pgd .endif .Lderef_p4d: /* PUD */ - .if CONFIG_PGTABLE_LEVELS > 3 + .if ARM64_NATIVE_PGTABLE_LEVELS > 3 pud .req x10 pte_to_phys cur_pudp, p4d kpti_map_pgtbl pud, 1 kpti_mk_tbl_ng pud, PTRS_PER_PUD b .Lnext_p4d - .else /* CONFIG_PGTABLE_LEVELS <= 3 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 3 */ pud .req pgd .set .Lnext_pud, .Lnext_pgd .endif .Lderef_pud: /* PMD */ - .if CONFIG_PGTABLE_LEVELS > 2 + .if ARM64_NATIVE_PGTABLE_LEVELS > 2 pmd .req x13 pte_to_phys cur_pmdp, pud kpti_map_pgtbl pmd, 2 kpti_mk_tbl_ng pmd, PTRS_PER_PMD b .Lnext_pud - .else /* CONFIG_PGTABLE_LEVELS <= 2 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 2 */ pmd .req pgd .set .Lnext_pmd, .Lnext_pgd .endif diff --git a/arch/arm64/mm/user4k-pgalloc.c b/arch/arm64/mm/user4k-pgalloc.c new file mode 100644 index 0000000000000..0bcaab5c3a8d2 --- /dev/null +++ b/arch/arm64/mm/user4k-pgalloc.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Hardware tables have immutable granule/owner metadata. Smaller tables share + * a native page; larger tables own a compound allocation. Only unissued slots + * in a shared native page are cached by the mm. + */ +#include +#include +#include +#include +#include + +unsigned int arm64_ptep_page_shift(pte_t *ptep) +{ + struct ptdesc *pt = virt_to_ptdesc(ptep); + + /* Table owners retain geometry through their last RCU reader. */ + return test_bit(PT_fragmented, &pt->pt_flags.f) ? pt->pt_frag_shift : PAGE_SHIFT; +} +EXPORT_SYMBOL_GPL(arm64_ptep_page_shift); + +static void fragment_owner_free(struct ptdesc *pt) +{ + clear_bit(PT_fragmented, &pt->pt_flags.f); + pagetable_dtor_free(pt); +} + +static void fragment_owner_free_rcu(struct rcu_head *rcu) +{ + fragment_owner_free(container_of(rcu, struct ptdesc, pt_rcu_head)); +} + +static void fragment_put(struct ptdesc *pt, unsigned int nr) +{ + if (!atomic_sub_and_test(nr, &pt->pt_frag_refcount)) + return; + if (test_and_clear_bit(PT_frag_defer, &pt->pt_flags.f)) + call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); + else + fragment_owner_free(pt); +} + +void *arm64_user_pt_alloc_granule(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift) +{ + struct ptdesc *pt; + void *table, *next; + bool constructed; + unsigned int frag_size, nr_fragments, order; + + if (shift != 12 && shift != 14 && shift != 16) + return NULL; + frag_size = 1U << shift; + order = shift > PAGE_SHIFT ? shift - PAGE_SHIFT : 0; + nr_fragments = (PAGE_SIZE << order) >> shift; + if (WARN_ON_ONCE((unsigned int)level >= ARRAY_SIZE(mm->context.user4k_pt_frag))) + return NULL; + + spin_lock(&mm->page_table_lock); + table = mm->context.user4k_pt_frag[level]; + if (table) { + /* An mm's geometry is immutable; never consume another granule. */ + if (virt_to_ptdesc(table)->pt_frag_shift != shift) { + spin_unlock(&mm->page_table_lock); + return NULL; + } + next = table + frag_size; + mm->context.user4k_pt_frag[level] = + ((unsigned long)next & ~PAGE_MASK) ? next : NULL; + } + spin_unlock(&mm->page_table_lock); + if (table) + return table; + + pt = pagetable_alloc(GFP_PGTABLE_USER, order); + if (!pt) + return NULL; + switch (level) { + case USER4K_PT_PTE: + constructed = pagetable_pte_ctor(mm, pt); + break; + case USER4K_PT_PMD: + constructed = pagetable_pmd_ctor(mm, pt); + break; + case USER4K_PT_PUD: + pagetable_pud_ctor(pt); + constructed = true; + break; + default: + constructed = false; + } + if (!constructed) { + pagetable_free(pt); + return NULL; + } + pt->pt_frag_shift = shift; + set_bit(PT_fragmented, &pt->pt_flags.f); + atomic_set(&pt->pt_frag_refcount, 1); + table = ptdesc_address(pt); + spin_lock(&mm->page_table_lock); + if (nr_fragments > 1 && !mm->context.user4k_pt_frag[level]) { + atomic_set(&pt->pt_frag_refcount, nr_fragments); + mm->context.user4k_pt_frag[level] = table + frag_size; + } + /* A racing allocation may own the cache; this page then has one user. */ + spin_unlock(&mm->page_table_lock); + return table; +} + +void *arm64_user4k_pt_alloc(struct mm_struct *mm, enum arm64_user4k_pt_level level) +{ + return arm64_user_pt_alloc_granule(mm, level, 12); +} + +void arm64_user4k_pt_free(void *table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + VM_BUG_ON(!test_bit(PT_fragmented, &pt->pt_flags.f)); + VM_BUG_ON(!IS_ALIGNED((unsigned long)table, 1UL << pt->pt_frag_shift)); + VM_BUG_ON(atomic_read(&pt->pt_frag_refcount) <= 0); + fragment_put(pt, 1); +} + +void arm64_pgtable_free(void *table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + if (test_bit(PT_fragmented, &pt->pt_flags.f)) + arm64_user4k_pt_free(table); + else { + VM_BUG_ON(!PAGE_ALIGNED(table)); + pagetable_dtor_free(pt); + } +} + +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm) +{ + unsigned int level; + + /* The mm is private or dying; no allocator may still access its cache. */ + for (level = 0; level < ARRAY_SIZE(mm->context.user4k_pt_frag); level++) { + void *next = mm->context.user4k_pt_frag[level]; + struct ptdesc *pt; + unsigned int issued, shift; + + if (!next) + continue; + mm->context.user4k_pt_frag[level] = NULL; + pt = virt_to_ptdesc(next); + shift = pt->pt_frag_shift; + issued = ((unsigned long)next & ~PAGE_MASK) >> shift; + fragment_put(pt, (PAGE_SIZE >> shift) - issued); + } +} + +void arm64_pte_free_defer(struct mm_struct *mm, pgtable_t table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + if (test_bit(PT_fragmented, &pt->pt_flags.f)) { + /* No freed slot is reused. Delay the final owner release for RCU. */ + set_bit(PT_frag_defer, &pt->pt_flags.f); + arm64_user4k_pt_free(table); + } else { + call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); + } +} + +void __tlb_remove_table(void *token) +{ + if ((unsigned long)token & 1) + arm64_user4k_pt_free((void *)((unsigned long)token & ~1UL)); + else + pagetable_dtor_free(token); +} diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index de92d63fdbc76..432df2ce9ddfd 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -69,6 +69,8 @@ HAS_TIDCP1 HAS_TLB_RANGE HAS_TSO_APPLE HAS_TSO_FIXED +HAS_USER4K_GRANULE +HAS_USER16K_GRANULE HAS_VA52 HAS_VIRT_HOST_EXTN HAS_WFXT From 2ddf173cb6ccbb896bf495bf332cb8f325f7efba Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 0579/1579] kunit: cover mixed-page ownership and translation lifetimes Add internal MM, table, subpage and recorded VFIO tests with separate opt-in controls. Translation fixtures require a disposable guest and recorded device mappings do not establish physical DMA qualification. Signed-off-by: Diljit Singh (cherry picked from commit e86efe2f2aaec4012ba91165f5211ca8c9c7716c) --- arch/arm64/Kconfig.debug | 70 + arch/arm64/mm/Makefile | 2 + arch/arm64/mm/user4k-data-test.c | 1835 ++ arch/arm64/mm/user4k-test.c | 14237 ++++++++++++++++ arch/arm64/mm/user4k-tlb-test.c | 184 + drivers/vfio/Kconfig | 12 + drivers/vfio/vfio_iommu_type1_account_test.c | 192 + .../vfio/vfio_iommu_type1_fragments_test.c | 1177 ++ drivers/vfio/vfio_iommu_type1_ioctl_test.c | 305 + drivers/vfio/vfio_iommu_type1_test.h | 36 + 10 files changed, 18050 insertions(+) create mode 100644 arch/arm64/mm/user4k-data-test.c create mode 100644 arch/arm64/mm/user4k-test.c create mode 100644 arch/arm64/mm/user4k-tlb-test.c create mode 100644 drivers/vfio/vfio_iommu_type1_account_test.c create mode 100644 drivers/vfio/vfio_iommu_type1_fragments_test.c create mode 100644 drivers/vfio/vfio_iommu_type1_ioctl_test.c create mode 100644 drivers/vfio/vfio_iommu_type1_test.h diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index d15ccdeee2125..58d7a1dc98405 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -42,3 +42,73 @@ config ARM64_USER4K_EXPERIMENTAL still require integration and validation. The private prctl numbers are not upstream assignments or a stable ABI promise. Say N for normal kernels. + +config ARM64_USER4K_KUNIT_TEST + bool "Test mixed userspace granules with larger kernel pages" + depends on KUNIT=y && ARM64_USER4K_EXPERIMENTAL + depends on ARM64_PA_BITS_48 + select ARM64_USER4K_DATA_KUNIT_TEST + select IOMMUFD + select IOMMUFD_DRIVER + default n + help + Exercise independent TTBR0 translation granules, subpage permissions + and invalidation while TTBR1 retains the normal native kernel tables. + Hardware translation/uaccess fixtures require arm64.user4k_test=1 + as a second opt-in and are disabled with KASAN. Software MM fixtures + can run with KASAN without borrowing or replacing TTBR0. + Run only in a disposable test VM. This regression harness does not + establish hardware qualification or production ABI readiness. + Say N for normal kernels. + +config ARM64_USER4K_IOMMUFD_TEST + bool "IOMMUFD ioctl harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && DEBUG_KERNEL + depends on FAULT_INJECTION && RUNTIME_TESTING_MENU + select GENERIC_PT + select IOMMU_PT + select IOMMU_PT_AMDV1 + select IOMMUFD + select IOMMUFD_TEST + default n + help + Enable the IOMMUFD mock-device ioctls and generic AMDv1 page tables + for the mixed-granule userspace ioctl fixture. This exposes privileged + test interfaces; use only in a disposable test VM. + Say N for normal kernels. + +config ARM64_USER4K_VFIO_ACCOUNT_TEST + bool "VFIO type1 pinning harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y + select VFIO + select VFIO_GROUP + select VFIO_CONTAINER + select VFIO_IOMMU_TYPE1 + select VFIO_TYPE1_ACCOUNT_KUNIT_TEST + default n + help + Enable built-in VFIO type1 accounting and retained-fragment tests + in a disposable mixed-granule guest. Recorded IOMMU translations + do not establish physical-device DMA support. + Say N for normal kernels. + +config ARM64_USER4K_VFIO_IOCTL_TEST + bool "Private VFIO type1 ioctl harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y && DEBUG_KERNEL + depends on RUNTIME_TESTING_MENU + select ARM64_USER4K_VFIO_ACCOUNT_TEST + default n + help + Expose a root-only disposable-guest misc device forwarding real EL0 + arguments to type1 ioctls and supplying recording IOMMU domains. + These privileged test controls perform no physical device DMA. + Say N for normal kernels. + +config ARM64_USER4K_DATA_KUNIT_TEST + bool "Test packed userspace allocation ownership" + depends on KUNIT=y && ARM64_USER4K_EXPERIMENTAL + default n + help + Exercise packed allocation, slot reuse, copying and deferred lifetime. + These tests use native kernel addresses without changing TTBR0, and + can run with KASAN even when the architectural tests are disabled. diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index 32d18488e527c..757f7eb07ccac 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -12,6 +12,8 @@ obj-$(CONFIG_TRANS_TABLE) += trans_pgd-asm.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ARM64_MTE) += mteswap.o obj-$(CONFIG_ARM64_GCS) += gcs.o +obj-$(CONFIG_ARM64_USER4K_KUNIT_TEST) += user4k-test.o user4k-tlb-test.o +obj-$(CONFIG_ARM64_USER4K_DATA_KUNIT_TEST) += user4k-data-test.o obj-$(CONFIG_ARM64_USER4K_EXPERIMENTAL) += user4k-pgalloc.o KASAN_SANITIZE_physaddr.o += n diff --git a/arch/arm64/mm/user4k-data-test.c b/arch/arm64/mm/user4k-data-test.c new file mode 100644 index 0000000000000..d7d550c9e1420 --- /dev/null +++ b/arch/arm64/mm/user4k-data-test.c @@ -0,0 +1,1835 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Backing ownership tests; these slots are not fault/rmap-owned mappings. */ +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_NET +#include +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) +#include +#endif +#ifdef CONFIG_INET +#include +#include +#endif +#include +#endif +#include +#include + +#define DATA_SLOTS MM_SUBPAGES_PER_PAGE + +struct data_fixture { + struct mm_subpage_pool *pool; + struct mm_subpage *held[2 * DATA_SLOTS + 2]; +}; + +static void data_fixture_free(void *arg) +{ + struct data_fixture *fixture = arg; + unsigned int i; + + if (fixture->pool) + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < ARRAY_SIZE(fixture->held); i++) + if (fixture->held[i]) + mm_subpage_put(fixture->held[i]); + if (fixture->pool) + mm_subpage_pool_put(fixture->pool); +} + +static struct data_fixture *data_fixture_create_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + + if (!fixture) + return NULL; + fixture->pool = mm_subpage_pool_create_granule(GFP_KERNEL, shift); + if (!fixture->pool) + return NULL; + if (kunit_add_action_or_reset(test, data_fixture_free, fixture)) + return NULL; + return fixture; +} + +static struct data_fixture *data_fixture_create(struct kunit *test) +{ + return data_fixture_create_granule(test, MM_SUBPAGE_SHIFT); +} + +static void put_held(struct data_fixture *fixture, unsigned int index) +{ + mm_subpage_put(fixture->held[index]); + fixture->held[index] = NULL; +} + +static struct mm_subpage *hold_alloc(struct data_fixture *fixture, unsigned int index) +{ + struct mm_subpage *slot = mm_subpage_alloc(fixture->pool); + + if (!IS_ERR(slot)) + fixture->held[index] = slot; + return slot; +} + +static int supply_backing(struct mm_subpage_pool *pool) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + int ret; + + if (!folio) + return -ENOMEM; + /* Check that stale allocation contents never become exposed. */ + memset(folio_address(folio), 0xcc, PAGE_SIZE); + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) + folio_put(folio); + return ret; +} + +static int supply_backing_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + int ret; + + if (!folio) + return -ENOMEM; + ret = mm_subpage_pool_add_folio_at(pool, folio, offset, GFP_KERNEL); + if (ret) + folio_put(folio); + return ret; +} + +static struct mm_subpage *hold_alloc_at(struct data_fixture *fixture, unsigned int index, + unsigned int offset) +{ + struct mm_subpage *slot = mm_subpage_alloc_at(fixture->pool, offset); + + if (!IS_ERR(slot)) + fixture->held[index] = slot; + return slot; +} + +static void *slot_address(struct mm_subpage *slot) +{ + return folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot); +} + +static void user4k_data_reuse_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct folio *folio; + struct mm_subpage *slot; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -EEXIST); + for (i = 0; i < DATA_SLOTS; i++) { + slot = hold_alloc(fixture, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)i + 1); + KUNIT_EXPECT_PTR_EQ(test, folio, mm_subpage_folio(fixture->held[0])); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), PFN_PHYS(folio_pfn(folio)) + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_4K), NULL); + memset(slot_address(slot), 0x10 + i, SZ_4K); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + /* A pin, alias or deferred-TLB holder keeps this slot unavailable. */ + fixture->held[DATA_SLOTS] = fixture->held[1]; + mm_subpage_get(fixture->held[DATA_SLOTS]); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS + 1); + put_held(fixture, 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + put_held(fixture, DATA_SLOTS); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS - 1); + slot = hold_alloc(fixture, DATA_SLOTS + 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_4K), NULL); + for (i = 0; i < DATA_SLOTS; i++) { + if (i != 1) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i]), + 0x10 + i, SZ_4K), NULL); + } + mm_subpage_pool_close(fixture->pool); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -ESHUTDOWN); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -ESHUTDOWN); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 1) + put_held(fixture, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + put_held(fixture, DATA_SLOTS + 1); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_copy_test(struct kunit *test) +{ + struct data_fixture *source = data_fixture_create(test); + struct data_fixture *dest = data_fixture_create(test); + struct mm_subpage *copy; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, source); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_EQ(test, supply_backing(source->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(dest->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(source, i)); + memset(slot_address(source->held[i]), 0x50 + i, SZ_4K); + } + source->held[DATA_SLOTS] = source->held[1]; + mm_subpage_get(source->held[DATA_SLOTS]); + copy = mm_subpage_copy(dest->pool, source->held[1]); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + dest->held[0] = copy; + KUNIT_EXPECT_NE(test, mm_subpage_phys(copy), mm_subpage_phys(source->held[1])); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x51, SZ_4K), NULL); + memset(slot_address(copy), 0xee, SZ_4K); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[i]), 0x50 + i, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(source->pool, source->held[1])), -EAGAIN); + /* Copy into a free neighboring slot in the same physical owner. */ + put_held(source, 2); + copy = mm_subpage_copy(source->pool, source->held[1]); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + source->held[2] = copy; + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(copy), mm_subpage_folio(source->held[1])); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(copy), 2U * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x51, SZ_4K), NULL); + memset(slot_address(copy), 0xdd, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[1]), 0x51, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[3]), 0x53, SZ_4K), NULL); +} + +static void user4k_data_lookup_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *slot, *found; + struct folio *folio; + phys_addr_t phys; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + phys = PFN_PHYS(folio_pfn(folio)); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys), NULL); + if (mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL)) { + folio_put(folio); + KUNIT_FAIL(test, "failed to supply lookup backing"); + return; + } + /* A published owner does not expose uninitialised or unissued slots. */ + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K), NULL); + for (i = 0; i < DATA_SLOTS; i++) { + slot = hold_alloc(fixture, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memset(slot_address(slot), 0x90 + i, SZ_4K); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2 * (int)i + 1); + if (!i) + KUNIT_EXPECT_EQ(test, mm_subpage_pool_add_folio(fixture->pool, folio, + GFP_KERNEL), -EEXIST); + found = mm_subpage_get_from_phys(phys + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, found); + fixture->held[i + DATA_SLOTS] = found; + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2 * (int)i + 2); + KUNIT_EXPECT_PTR_EQ(test, found, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K + 1), NULL); + } + /* A second pool must not zero or acquire already owned backing. */ + KUNIT_EXPECT_EQ(test, mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL), + -EINVAL); + for (i = 0; i < DATA_SLOTS; i++) { + put_held(fixture, i); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i + DATA_SLOTS]), + 0x90 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + put_held(fixture, DATA_SLOTS + 1); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + SZ_4K), NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[1]), 0, SZ_4K), NULL); + mm_subpage_pool_close(fixture->pool); + /* Keep the physical allocation stable while checking final unbinding. */ + folio_get(folio); + put_held(fixture, 1); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 1) + put_held(fixture, i + DATA_SLOTS); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); +} + +static void user4k_data_exact_slot_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *source, *copy, *reused; + struct folio *first; + static const unsigned int neighbors[] = { 0, SZ_4K, 3 * SZ_4K }; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_EXPECT_EQ(test, supply_backing_at(fixture->pool, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(fixture->pool, PAGE_SIZE)), -EINVAL); + KUNIT_ASSERT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), 0); + source = hold_alloc_at(fixture, 0, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, source); + first = mm_subpage_folio(source); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(source), (unsigned int)SZ_8K); + memset(slot_address(source), 0xa7, SZ_4K); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy_at(fixture->pool, source, 1)), -EINVAL); + /* Three free neighbors cannot satisfy a request for this occupied slot. */ + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy_at(fixture->pool, source, SZ_8K)), -EAGAIN); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -EEXIST); + KUNIT_ASSERT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), 0); + KUNIT_EXPECT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), -EEXIST); + copy = mm_subpage_copy_at(fixture->pool, source, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + fixture->held[1] = copy; + KUNIT_EXPECT_PTR_NE(test, mm_subpage_folio(copy), first); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(copy), (unsigned int)SZ_8K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0xa7, SZ_4K), NULL); + memset(slot_address(copy), 0x4b, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source), 0xa7, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 2UL); + for (i = 0; i < ARRAY_SIZE(neighbors); i++) { + struct mm_subpage *slot = hold_alloc_at(fixture, i + 2, neighbors[i]); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), first); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), neighbors[i]); + memset(slot_address(slot), 0x30 + i, SZ_4K); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(first), 4); + fixture->held[5] = source; + mm_subpage_get(source); + put_held(fixture, 0); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(fixture->pool, SZ_8K)), -EAGAIN); + put_held(fixture, 5); + reused = hold_alloc_at(fixture, 6, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, reused); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(reused), first); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(reused), 0, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(first), 4); + for (i = 0; i < ARRAY_SIZE(neighbors); i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i + 2]), + 0x30 + i, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x4b, SZ_4K), NULL); + mm_subpage_pool_close(fixture->pool); + for (i = 2; i <= 4; i++) + put_held(fixture, i); + put_held(fixture, 6); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + put_held(fixture, 1); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_close_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *slot; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + /* Closing with no issued slots must release that entire unused owner. */ + mm_subpage_pool_close(fixture->pool); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); + fixture = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + slot = hold_alloc(fixture, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memset(slot_address(slot), 0xa7, SZ_4K); + mm_subpage_pool_close(fixture->pool); + mm_subpage_pool_put(fixture->pool); + fixture->pool = NULL; + /* The originating pool/mm reference is gone; a child's slot remains live. */ + fixture->held[1] = slot; + mm_subpage_get(slot); + put_held(fixture, 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0xa7, SZ_4K), NULL); + put_held(fixture, 1); +} + +struct deferred_slot { + struct rcu_head rcu; + struct mm_subpage *slot; + struct completion done; +}; + +static void deferred_slot_put(struct rcu_head *rcu) +{ + struct deferred_slot *pending = container_of(rcu, struct deferred_slot, rcu); + + mm_subpage_put(pending->slot); + complete(&pending->done); +} + +static void data_rcu_barrier(void *unused) +{ + rcu_barrier(); +} + +static void user4k_data_deferred_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct deferred_slot *pending = kunit_kzalloc(test, sizeof(*pending), GFP_KERNEL); + struct mm_subpage *probe; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, pending); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, i)); + init_completion(&pending->done); + pending->slot = fixture->held[0]; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, data_rcu_barrier, NULL), 0); + mm_subpage_get(pending->slot); + rcu_read_lock(); + call_rcu(&pending->rcu, deferred_slot_put); + put_held(fixture, 0); + /* The delayed reference must prevent recycling before the grace period. */ + probe = mm_subpage_alloc(fixture->pool); + for (i = 1; i < DATA_SLOTS; i++) + put_held(fixture, i); + rcu_read_unlock(); + if (!IS_ERR(probe)) + fixture->held[DATA_SLOTS] = probe; + KUNIT_EXPECT_EQ(test, PTR_ERR(probe), -EAGAIN); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&pending->done, 5 * HZ), 0UL); + /* The callback retired the final live slot and the whole native owner. */ + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_backing_reject_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + folio = folio_alloc(GFP_KERNEL, 1); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + if (ret) + folio_put(folio); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_get(folio); + ret = mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2); + folio_put(folio); + if (ret) + folio_put(folio); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +struct data_worker { + struct mm_subpage_pool *pool; + struct task_struct *task; + struct completion ready, go, done; + unsigned int id, checks; + int error; +}; + +static struct mm_subpage *alloc_with_backing(struct mm_subpage_pool *pool) +{ + unsigned int tries; + + for (tries = 0; tries < 128; tries++) { + struct mm_subpage *slot = mm_subpage_alloc(pool); + int ret; + + if (!IS_ERR(slot) || PTR_ERR(slot) != -EAGAIN) + return slot; + ret = supply_backing(pool); + if (ret && ret != -EEXIST) + return ERR_PTR(ret); + cond_resched(); + } + return ERR_PTR(-EAGAIN); +} + +static int data_worker_run(void *arg) +{ + struct data_worker *worker = arg; + struct mm_subpage *slot = NULL; + unsigned int round; + u8 pattern = 0; + + for (round = 0; round < 64; round++) { + slot = alloc_with_backing(worker->pool); + if (IS_ERR(slot)) { + worker->error = PTR_ERR(slot); + slot = NULL; + break; + } + pattern = 1 + round + worker->id * 64; + memset(slot_address(slot), pattern, SZ_4K); + mm_subpage_get(slot); + mm_subpage_put(slot); + schedule_timeout_uninterruptible(1); + if (memchr_inv(slot_address(slot), pattern, SZ_4K)) { + worker->error = -EIO; + break; + } + worker->checks++; + if (round != 63) { + mm_subpage_put(slot); + slot = NULL; + } + } + complete(&worker->ready); + wait_for_completion(&worker->go); + if (slot) { + if (memchr_inv(slot_address(slot), pattern, SZ_4K)) + worker->error = -EIO; + mm_subpage_put(slot); + } + complete(&worker->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void data_worker_stop(void *arg) +{ + struct data_worker *worker = arg; + + complete_all(&worker->go); + kthread_stop(worker->task); +} + +static void user4k_data_concurrent_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct data_worker *workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].pool = fixture->pool; + workers[i].id = i; + init_completion(&workers[i].ready); + init_completion(&workers[i].go); + init_completion(&workers[i].done); + workers[i].task = kthread_run(data_worker_run, &workers[i], "user4k-data-%u", i); + KUNIT_ASSERT_FALSE(test, IS_ERR(workers[i].task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, data_worker_stop, &workers[i]), 0); + } + for (i = 0; i < 2; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < 2; i++) + complete(&workers[i].go); + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_EQ(test, workers[i].checks, 64U); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +struct lookup_worker { + struct task_struct *task; + struct completion observed; + phys_addr_t phys; + unsigned long checks; + int error; +}; + +static int lookup_worker_run(void *arg) +{ + struct lookup_worker *worker = arg; + bool observed = false; + + while (!kthread_should_stop()) { + struct mm_subpage *slot = mm_subpage_get_from_phys(worker->phys); + + if (slot) { + unsigned char *data = slot_address(slot); + unsigned char value = READ_ONCE(*data); + + if (mm_subpage_phys(slot) != worker->phys || + (value != 0x5a && value != 0xab) || + memchr_inv(data, value, SZ_4K)) + worker->error = -EINVAL; + worker->checks++; + mm_subpage_put(slot); + if (!observed) { + complete(&worker->observed); + observed = true; + } + if (worker->error) + break; + } + cond_resched(); + } + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void lookup_worker_stop(void *arg) +{ + struct lookup_worker *worker = arg; + + if (worker->task) { + kthread_stop(worker->task); + worker->task = NULL; + } +} + +static void user4k_data_lookup_race_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct lookup_worker *worker = kunit_kzalloc(test, sizeof(*worker), GFP_KERNEL); + unsigned int copied = 0, i; + unsigned long deadline; + struct folio *folio; + struct mm_subpage *found; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, worker); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, i)); + memset(slot_address(fixture->held[0]), 0x5a, SZ_4K); + memset(slot_address(fixture->held[1]), 0xab, SZ_4K); + worker->phys = mm_subpage_phys(fixture->held[2]); + put_held(fixture, 2); + init_completion(&worker->observed); + worker->task = kthread_run(lookup_worker_run, worker, "user4k-lookup"); + KUNIT_ASSERT_FALSE(test, IS_ERR(worker->task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, lookup_worker_stop, worker), 0); + if (num_online_cpus() > 1) { + unsigned int cpu = cpumask_next(cpumask_first(cpu_online_mask), cpu_online_mask); + + KUNIT_ASSERT_EQ(test, set_cpus_allowed_ptr(worker->task, cpumask_of(cpu)), 0); + } + /* Alternate complete contents; a reader must never see a partial copy. */ + deadline = jiffies + 30 * HZ; + while (copied < 2048 && time_before(jiffies, deadline)) { + struct mm_subpage *dest = mm_subpage_copy(fixture->pool, + fixture->held[copied & 1]); + bool observed = true; + phys_addr_t phys; + + if (PTR_ERR_OR_ZERO(dest) == -EAGAIN) { + /* Let a scheduled-out lookup holder retire its reference. */ + schedule_timeout_uninterruptible(1); + continue; + } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dest); + phys = mm_subpage_phys(dest); + if (!copied) + observed = wait_for_completion_timeout(&worker->observed, 5 * HZ) != 0; + mm_subpage_put(dest); + KUNIT_ASSERT_TRUE(test, observed); + KUNIT_ASSERT_EQ(test, phys, worker->phys); + copied++; + cond_resched(); + } + /* Retire the owner while lookup is still running, then verify unbinding. */ + folio = mm_subpage_folio(fixture->held[0]); + folio_get(folio); + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 2) + put_held(fixture, i); + lookup_worker_stop(worker); + found = mm_subpage_get_from_phys(worker->phys); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + if (found) + mm_subpage_put(found); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); + KUNIT_EXPECT_EQ(test, worker->error, 0); + KUNIT_EXPECT_GT(test, worker->checks, 0UL); + KUNIT_EXPECT_EQ(test, copied, 2048U); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_swapcache_reuse_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct mm_subpage *attempt; + struct folio *folio; + unsigned int i; + bool intact; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * MM_SUBPAGE_SIZE)); + memset(slot_address(f->held[i]), 0x60 + i, MM_SUBPAGE_SIZE); + } + folio = mm_subpage_folio(f->held[0]); + put_held(f, 1); + /* Synthetic cache membership only; no swap entry or I/O is fabricated. */ + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + attempt = mm_subpage_alloc_at(f->pool, MM_SUBPAGE_SIZE); + intact = !memchr_inv(folio_address(folio) + MM_SUBPAGE_SIZE, 0x61, MM_SUBPAGE_SIZE); + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + /* Restore the real state before assertions can unwind the fixture. */ + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_EXPECT_TRUE(test, intact); + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, MM_SUBPAGE_SIZE)); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x60 + i, MM_SUBPAGE_SIZE), NULL); +} + +static void restore_folio_free(void *arg) +{ + struct folio *folio = arg; + + /* Data-only fixtures never create a real swap entry or mapping. */ + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + folio_put(folio); +} + +static struct mm_subpage *restore_test_slot_granule(struct folio *folio, unsigned int offset, unsigned int shift) +{ + struct mm_subpage *slot; + + folio_lock(folio); + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + slot = mm_subpage_restore_granule(folio, offset, shift, GFP_KERNEL); + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + folio_unlock(folio); + return slot; +} + +static struct mm_subpage *restore_test_slot(struct folio *folio, unsigned int offset) +{ + return restore_test_slot_granule(folio, offset, MM_SUBPAGE_SHIFT); +} + +static void test_data_restore_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct folio *folio; + struct mm_subpage *slot; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + memset(slot_address(f->held[i]), 0x70 + i, size); + } + folio = mm_subpage_folio(f->held[0]); + folio_get(folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, folio), 0); + put_held(f, 1); + slot = restore_test_slot_granule(folio, size, shift); + if (!IS_ERR(slot)) + f->held[1] = slot; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)size); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 5); + slot = restore_test_slot_granule(folio, 0, shift); + if (!IS_ERR(slot)) + f->held[4] = slot; + KUNIT_ASSERT_PTR_EQ(test, slot, f->held[0]); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 6); + put_held(f, 4); + /* Retire the entire old owner, retaining only a native backing reference. */ + for (i = 0; i < 4; i++) + put_held(f, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), 0UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + for (i = 0; i < 4; i++) { + slot = restore_test_slot_granule(folio, i * size, shift); + if (!IS_ERR(slot)) + f->held[i] = slot; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0x70 + i, + size), NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 5); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, 1, shift)), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, PAGE_SIZE, shift)), -EINVAL); + if (shift != MM_SUBPAGE_SHIFT) { + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot(folio, 0)), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, SZ_4K, shift)), -EINVAL); + } + +} + +static void user4k_data_restore_test(struct kunit *test) +{ + test_data_restore_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_restore_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_restore_granule(test, 14); +} + +struct restore_worker { + struct folio *folio; + struct task_struct *task; + struct completion done; + unsigned int checks; + int error; +}; + +static int restore_worker_run(void *arg) +{ + struct restore_worker *w = arg; + unsigned long deadline = jiffies + 20 * HZ; + + while (!kthread_should_stop() && w->checks < 1024 && time_before(jiffies, deadline)) { + struct mm_subpage *slot; + + folio_lock(w->folio); + slot = mm_subpage_restore(w->folio, MM_SUBPAGE_SIZE, GFP_KERNEL); + folio_unlock(w->folio); + if (PTR_ERR_OR_ZERO(slot) == -EAGAIN) { + cond_resched(); + continue; + } + if (IS_ERR(slot)) { + w->error = PTR_ERR(slot); + break; + } + if (memchr_inv(slot_address(slot), 0x83, MM_SUBPAGE_SIZE)) + w->error = -EIO; + /* Retirement deliberately occurs without the folio lock. */ + mm_subpage_put(slot); + if (w->error) + break; + w->checks++; + cond_resched(); + } + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void restore_worker_stop(void *arg) +{ + struct restore_worker *w = arg; + + if (w->task) { + kthread_stop(w->task); + w->task = NULL; + } +} + +static void user4k_data_restore_race_test(struct kunit *test) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + struct restore_worker *workers; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, folio), 0); + memset(folio_address(folio), 0x83, PAGE_SIZE); + __folio_mark_uptodate(folio); + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].folio = folio; + init_completion(&workers[i].done); + workers[i].task = kthread_run(restore_worker_run, &workers[i], "user4k-restore-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "restore worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_worker_stop, &workers[i]), 0); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 25 * HZ), 0UL); + restore_worker_stop(&workers[i]); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_EQ(test, workers[i].checks, 1024U); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); +} + +static void test_data_migration_granule(struct kunit *test, unsigned int shift) +{ + unsigned int mode, size = 1U << shift; + + /* Success, rollback, all PTEs discarded, and partial-folio success. */ + for (mode = 0; mode < 4; mode++) { + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct folio *src, *dst, *target; + unsigned int i, nr = mode == 3 ? 3 : 4; + int ret, src_refs = 0, dst_refs = 0, blocked = 0; + bool unpublished = true, active = false; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, index * size)); + memset(slot_address(f->held[i]), 0x80 + index, size); + } + src = mm_subpage_folio(f->held[0]); + folio_get(src); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, src), 0); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, dst), 0); + folio_lock(src); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(src, dst); + if (!ret) { + struct mm_subpage *probe; + + src_refs = folio_ref_count(src); + dst_refs = folio_ref_count(dst); + active = mm_subpage_migrating(src) && mm_subpage_migrating(dst); + probe = mm_subpage_alloc_at(f->pool, size); + blocked = PTR_ERR_OR_ZERO(probe); + if (!IS_ERR(probe)) + mm_subpage_put(probe); + /* Emulate migration entries: their old mapping refs are gone. */ + for (i = 0; i < nr; i++) + put_held(f, i); + probe = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(src))); + unpublished &= !probe; + if (probe) + mm_subpage_put(probe); + probe = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(dst))); + unpublished &= !probe; + if (probe) + mm_subpage_put(probe); + if (mode != 1 && mode != 2) { + copy_user_highpage(&dst->page, &src->page, 0, NULL); + __folio_mark_uptodate(dst); + } + target = mode == 1 ? src : dst; + if (mode != 2) + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + f->held[i] = mm_subpage_migrate_restore(target, index * size); + } + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, active); + KUNIT_EXPECT_TRUE(test, unpublished); + KUNIT_EXPECT_EQ(test, src_refs, (int)nr + 1); + KUNIT_EXPECT_EQ(test, dst_refs, 1); + KUNIT_EXPECT_EQ(test, blocked, -EAGAIN); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), mode == 2 ? 0UL : 1UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(src), mode == 1 ? (int)nr + 1 : 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(dst), mode == 0 || mode == 3 ? (int)nr + 1 : 1); + if (mode != 2) + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->held[i]), mode == 1 ? src : dst); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + 0x80 + index, size), NULL); + } + if (mode == 3) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 3, size)); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->held[3]), dst); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[3]), 0, + size), NULL); + } + } +} + +static void user4k_data_migration_owner_test(struct kunit *test) +{ + test_data_migration_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_migration_owner_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_migration_granule(test, 14); +} + +#ifdef CONFIG_ARM64_MTE +static void test_slot_tags_set(void *addr, unsigned int tag) +{ + unsigned long value = (unsigned long)tag << MTE_TAG_SHIFT; + unsigned int offset; + + for (offset = 0; offset < MM_SUBPAGE_SIZE; offset += MTE_GRANULE_SIZE) + asm volatile(".arch_extension memtag\n\tstg %0, [%1]" + : : "r" (value), "r" (addr + offset) : "memory"); +} + +static bool test_slot_tags_equal(void *addr, unsigned int expected) +{ + unsigned int offset; + + for (offset = 0; offset < MM_SUBPAGE_SIZE; offset += MTE_GRANULE_SIZE) { + unsigned long value = 0; + + asm volatile(".arch_extension memtag\n\tldg %0, [%1]" + : "+r" (value) : "r" (addr + offset) : "memory"); + if (((value >> MTE_TAG_SHIFT) & 0xf) != expected) + return false; + } + return true; +} +#endif + +static void user4k_data_mte_test(struct kunit *test) +{ +#ifdef CONFIG_ARM64_MTE + struct data_fixture *f; + struct folio *folio; + unsigned int i; + + if (!system_supports_mte()) { + kunit_skip(test, "MTE is not available on this CPU"); + return; + } + f = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * MM_SUBPAGE_SIZE)); + folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_TRUE(test, try_page_mte_tagging(&folio->page)); + mte_clear_page_tags(folio_address(folio)); + for (i = 0; i < 4; i++) { + memset(slot_address(f->held[i]), 0x40 + i, MM_SUBPAGE_SIZE); + test_slot_tags_set(slot_address(f->held[i]), i + 1); + } + set_page_mte_tagged(&folio->page); + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, MM_SUBPAGE_SIZE)); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(slot_address(f->held[i]), i == 1 ? 0 : i + 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x40 + i, MM_SUBPAGE_SIZE), NULL); + } + put_held(f, 1); + f->held[1] = mm_subpage_copy_at(f->pool, f->held[0], MM_SUBPAGE_SIZE); + if (IS_ERR(f->held[1])) { + f->held[1] = NULL; + KUNIT_FAIL(test, "tagged slot copy failed"); + return; + } + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(slot_address(f->held[i]), i == 1 ? 1 : i + 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0x40 : 0x40 + i, MM_SUBPAGE_SIZE), NULL); + } + /* A fresh destination must initialise tags outside the copied slot too. */ + KUNIT_ASSERT_EQ(test, supply_backing_at(f->pool, 3 * MM_SUBPAGE_SIZE), 0); + f->held[4] = mm_subpage_copy_at(f->pool, f->held[2], 3 * MM_SUBPAGE_SIZE); + if (IS_ERR(f->held[4])) { + f->held[4] = NULL; + KUNIT_FAIL(test, "fresh tagged slot copy failed"); + return; + } + folio = mm_subpage_folio(f->held[4]); + KUNIT_EXPECT_TRUE(test, page_mte_tagged(&folio->page)); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(folio_address(folio) + + i * MM_SUBPAGE_SIZE, i == 3 ? 3 : 0)); +#else + kunit_skip(test, "MTE is not configured"); +#endif +} + +static void extent_folio_put(void *arg) +{ + folio_put(arg); +} + +static void user4k_data_copy_extent_test(struct kunit *test) +{ + const unsigned int sizes[] = { SZ_4K, SZ_16K, SZ_64K }; + + for (unsigned int n = 0; n < ARRAY_SIZE(sizes); n++) { + struct folio *source, *dest; + unsigned int size = sizes[n], to_offset; + void *from, *to; + + if (size > PAGE_SIZE) + continue; + source = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, source); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, source), 0); + dest = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, dest), 0); + from = folio_address(source); + to = folio_address(dest); + to_offset = PAGE_SIZE - size; + for (unsigned int i = 0; i < DATA_SLOTS; i++) + memset(from + i * SZ_4K, 0x40 + i, SZ_4K); + for (unsigned int tagged = 0; tagged < 2; tagged++) { + memset(to, 0xac, PAGE_SIZE); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) { + if (try_page_mte_tagging(&dest->page)) + mte_clear_page_tags(to); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(to + i * SZ_4K, 9); + set_page_mte_tagged(&dest->page); + if (tagged) { + if (try_page_mte_tagging(&source->page)) + mte_clear_page_tags(from); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(from + i * SZ_4K, 1 + i % 15); + set_page_mte_tagged(&source->page); + } + } +#endif + KUNIT_ASSERT_EQ(test, copy_user_subpage_range(&dest->page, to_offset, + &source->page, 0, size), 0); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int offset = i * SZ_4K; + bool copied = offset >= to_offset; + unsigned int source_index = copied ? (offset - to_offset) / SZ_4K : 0; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(to + offset, + copied ? 0x40 + source_index : 0xac, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(from + offset, 0x40 + i, SZ_4K), NULL); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(to + offset, + copied ? (tagged ? 1 + source_index % 15 : 0) : 9)); +#endif + } + clear_user_subpage_range(&dest->page, to_offset, size); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int offset = i * SZ_4K; + bool cleared = offset >= to_offset; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(to + offset, cleared ? 0 : 0xac, + SZ_4K), NULL); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(to + offset, cleared ? 0 : 9)); +#endif + } + } + } +} + +#ifdef CONFIG_NET +struct skb_test_state { + struct folio *folio; +#ifdef CONFIG_INET + struct socket *socket; +#endif + struct sk_buff *skbs[5]; +}; + +static void skb_test_cleanup(void *arg) +{ + struct skb_test_state *state = arg; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->skbs); i++) { + if (state->skbs[i]) + state->skbs[i]->sk = NULL; + kfree_skb(state->skbs[i]); + } +#ifdef CONFIG_INET + if (state->socket) + sock_release(state->socket); +#endif + /* This fixture marks ownership only; it installs no anonymous rmap. */ + if (state->folio) + state->folio->mapping = NULL; +} + +static void user4k_data_skb_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct iov_iter iter; + struct bio_vec bvec; + unsigned char *bytes = kunit_kmalloc(test, SZ_4K, GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_NOT_NULL(test, bytes); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + memset(slot_address(f->held[i]), 0x51 + i, SZ_4K); + } + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + state->skbs[0]->ip_summed = CHECKSUM_UNNECESSARY; + bvec_set_page(&bvec, &state->folio->page, SZ_4K, SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(state->skbs[0], &iter, SZ_4K), (ssize_t)SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, skb_zcopy(state->skbs[0])); + state->skbs[1] = pskb_copy(state->skbs[0], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[1]); + /* Append a distinct quarter while the callback object has two owners. */ + bvec_set_page(&bvec, &state->folio->page, SZ_4K, 2 * SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(state->skbs[0], &iter, SZ_4K), (ssize_t)SZ_4K); + KUNIT_EXPECT_PTR_NE(test, skb_zcopy(state->skbs[0]), skb_zcopy(state->skbs[1])); + state->skbs[2] = skb_clone(state->skbs[1], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[2]); + state->skbs[3] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[3]); + skb_split(state->skbs[0], state->skbs[3], SZ_2K); + state->skbs[4] = skb_copy(state->skbs[3], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[4]); + put_held(f, 1); + put_held(f, 2); + attempt = mm_subpage_alloc_at(f->pool, SZ_4K); + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_EXPECT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + for (i = 0; i < 2; i++) { + kfree_skb(state->skbs[i]); + state->skbs[i] = NULL; + } + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[2], 0, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x52, SZ_4K), NULL); + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[3], SZ_2K, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x53, SZ_4K), NULL); + for (i = 2; i < 4; i++) { + kfree_skb(state->skbs[i]); + state->skbs[i] = NULL; + } + /* A full data copy must not keep ownership of the original quarters. */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_4K)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 2, 2 * SZ_4K)); + memset(slot_address(f->held[1]), 0xe1, SZ_4K); + memset(slot_address(f->held[2]), 0xe2, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[4], SZ_2K, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x53, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} +#ifdef CONFIG_INET +static void test_data_skb_prune_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct sk_buff *skb; + struct bio_vec bvec; + struct iov_iter iter; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + KUNIT_ASSERT_EQ(test, sock_create_kern(&init_net, AF_INET, SOCK_STREAM, 0, &state->socket), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + skb = state->skbs[0]; + /* A live socket association prevents trim from condensing data into the head. */ + skb->sk = state->socket->sk; + skb->ip_summed = CHECKSUM_UNNECESSARY; + bvec_set_page(&bvec, &state->folio->page, 16, size - SZ_4K + 16); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, 16); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, 16), (ssize_t)16); + for (i = 0; i < 1024; i++) { + bvec_set_page(&bvec, &state->folio->page, size, (1 + i % 3) * size); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, size), (ssize_t)size); + /* Four fixture refs, two frag refs, two current quarter refs. */ + KUNIT_ASSERT_EQ(test, folio_ref_count(state->folio), 8); + KUNIT_ASSERT_EQ(test, pskb_trim(skb, 16), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 7); + } + skb->sk = NULL; + kfree_skb(skb); + state->skbs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} + +static void user4k_data_skb_prune_test(struct kunit *test) +{ + test_data_skb_prune_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_skb_prune_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_skb_prune_granule(test, 14); +} + +#endif + +static void user4k_data_skb_checksum_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct bio_vec bvec[2]; + struct iov_iter iter; + unsigned char *bytes; + unsigned int i, owned; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + bytes = folio_address(state->folio); + for (i = 0; i < PAGE_SIZE; i++) + bytes[i] = i * 13 + (i >> 12) * 31; + for (owned = 0; owned < 2; owned++) { + struct sk_buff *skb; + + state->folio->mapping = owned ? (void *)FOLIO_MAPPING_ANON : NULL; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + skb = state->skbs[0]; + skb->ip_summed = CHECKSUM_NONE; + bvec_set_page(&bvec[0], &state->folio->page, SZ_4K - 13, 13); + bvec_set_page(&bvec[1], &state->folio->page, SZ_4K, SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, bvec, 2, SZ_8K - 13); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, SZ_8K - 13), + (ssize_t)(SZ_8K - 13)); + KUNIT_EXPECT_EQ(test, (__force u16)csum_fold(skb->csum), + (__force u16)csum_fold(skb_checksum(skb, 0, skb->len, 0))); + kfree_skb(skb); + state->skbs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + } +} + +#endif + +struct refs_test_state { + struct folio *folio; + struct mm_subpage_refs *refs[3]; +}; + +static void refs_test_cleanup(void *arg) +{ + struct refs_test_state *state = arg; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->refs); i++) + mm_subpage_refs_put(state->refs[i]); + if (state->folio) + state->folio->mapping = NULL; +} + +static bool refs_test_keep(const struct mm_subpage *slot, void *context) +{ + unsigned long mask = *(unsigned long *)context; + + return mask & BIT(mm_subpage_offset(slot) >> MM_SUBPAGE_SHIFT); +} + +static void user4k_data_refset_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct refs_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct page *page; + unsigned long mask; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + state->folio = mm_subpage_folio(f->held[0]); + page = &state->folio->page; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, refs_test_cleanup, state), 0); + /* Ownership fixture only; no anonymous rmap is installed. */ + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, + SZ_4K + 17, SZ_8K - 34, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, SZ_4K, SZ_4K, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + state->refs[1] = mm_subpage_refs_get(state->refs[0]); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[1], page, 3 * SZ_4K, SZ_4K, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_NE(test, state->refs[0], state->refs[1]); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 5); + mask = BIT(2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 3); + state->refs[2] = mm_subpage_refs_get(state->refs[0]); + mask = BIT(1); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[2], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_NE(test, state->refs[0], state->refs[2]); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 4); + KUNIT_EXPECT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, PAGE_SIZE, 1, GFP_KERNEL), -EINVAL); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 4); + mask = 0; + for (i = 1; i < 3; i++) { + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[i], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_EQ(test, state->refs[i], NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + put_held(f, 1); + attempt = mm_subpage_alloc_at(f->pool, SZ_4K); + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + mm_subpage_refs_put(state->refs[0]); + state->refs[0] = NULL; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_4K)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS); + for (i = 0; i < 1024; i++) { + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 2 * (int)DATA_SLOTS); + mask = BIT(i % DATA_SLOTS); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[0], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 1); + } + mm_subpage_refs_put(state->refs[0]); + state->refs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS); +} + +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) +struct skmsg_test_state { + struct folio *folio; + struct socket *socket; + struct sk_msg messages[2]; + struct sk_msg borrowed; +}; + +static void skmsg_test_cleanup(void *arg) +{ + struct skmsg_test_state *state = arg; + unsigned int i; + + if (state->socket) { + for (i = 0; i < ARRAY_SIZE(state->messages); i++) + sk_msg_free(state->socket->sk, &state->messages[i]); + sk_msg_subpages_release(&state->borrowed); + sock_release(state->socket); + } + if (state->folio) + state->folio->mapping = NULL; +} + +static void test_data_skmsg_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct skmsg_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct sk_msg *src, *dst; + struct page *page; + struct sock *sk; + struct bio_vec bvec; + struct iov_iter iter; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + memset(slot_address(f->held[i]), 0x60 + i, size); + } + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skmsg_test_cleanup, state), 0); + KUNIT_ASSERT_EQ(test, sock_create_kern(&init_net, AF_INET, SOCK_STREAM, 0, &state->socket), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + page = &state->folio->page; + sk = state->socket->sk; + src = &state->messages[0]; + dst = &state->messages[1]; + sk_msg_init(src); + sk_msg_init(dst); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, (size / 2) - 13, size + (size / 2) + 13), 0); + sk_mem_charge(sk, (size / 2) - 13); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 2 * size), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + KUNIT_ASSERT_EQ(test, sk_msg_clone(sk, dst, src, 0, size + (size / 2) - 13), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 11); + sk_msg_free_partial(sk, src, (size / 2) - 13); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 0), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 11); + sk_msg_free(sk, dst); + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, size)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + sk_msg_free_partial(sk, src, size); + for (i = 0; i < 1024; i++) { + unsigned int index = (i + 1) & 3; + + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, index * size), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + sk_msg_free_partial(sk, src, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 7); + } + sk_msg_free(sk, src); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + /* A borrowed descriptor copy has an independent ownership lifetime. */ + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 0), 0); + sk_mem_charge(sk, size); + memcpy(&state->borrowed, src, sizeof(*src)); + sk_msg_subpages_get(&state->borrowed); + sk_msg_free(sk, src); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 5); + put_held(f, 0); + attempt = mm_subpage_alloc_at(f->pool, 0); + if (!IS_ERR(attempt)) + f->held[0] = attempt; + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(sg_virt(&state->borrowed.sg.data[0]), 0x60, size), NULL); + sk_msg_subpages_release(&state->borrowed); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 0, 0)); + /* Full transfer moves the set, and zero-copy iterator trim releases its references. */ + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 3 * size), 0); + sk_mem_charge(sk, size); + sk_msg_xfer_full(dst, src); + KUNIT_EXPECT_EQ(test, src->sg.size, 0U); + sk_msg_free(sk, dst); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + bvec_set_page(&bvec, page, size - 17, 3 * size + 17); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size - 17); + KUNIT_ASSERT_EQ(test, sk_msg_zerocopy_from_iter(sk, &iter, src, size - 17), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 6); + sk_msg_trim(sk, src, 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} + +static void user4k_data_skmsg_test(struct kunit *test) +{ + test_data_skmsg_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_skmsg_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_skmsg_granule(test, 14); +} +#endif + +static void user16k_data_owner_test(struct kunit *test) +{ + struct data_fixture *f, *dest, *small; + struct mm_subpage *slot, *found; + unsigned int count = PAGE_SIZE / SZ_16K; + + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + dest = data_fixture_create_granule(test, 14); + small = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(dest->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(small->pool), 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(small, 0)); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_4K)), -EINVAL); + for (unsigned int i = 0; i < count; i++) { + slot = hold_alloc_at(f, i, i * SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_shift(slot), 14U); + KUNIT_EXPECT_EQ(test, mm_subpage_size(slot), (unsigned int)SZ_16K); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)SZ_16K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_16K), NULL); + memset(slot_address(slot), 0x30 + i, SZ_16K); + for (unsigned int quarter = 0; quarter < 4; quarter++) { + found = mm_subpage_get_from_phys(mm_subpage_phys(slot) + quarter * SZ_4K); + KUNIT_ASSERT_PTR_EQ(test, found, slot); + mm_subpage_put(found); + } + } + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(f->pool)), -EAGAIN); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(dest->pool, small->held[0])), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(small->pool, f->held[0])), -EINVAL); + slot = mm_subpage_copy_at(dest->pool, f->held[count - 1], SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + dest->held[0] = slot; + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0x30 + count - 1, SZ_16K), NULL); + /* An interior-byte lookup holds the whole identity across release/reuse. */ + f->held[count] = mm_subpage_get_from_phys(mm_subpage_phys(f->held[1]) + 3 * SZ_4K); + KUNIT_ASSERT_PTR_EQ(test, f->held[count], f->held[1]); + put_held(f, 1); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_16K)), -EAGAIN); + put_held(f, count); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + for (unsigned int i = 0; i < count; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x30 + i, SZ_16K), NULL); + mm_subpage_pool_close(f->pool); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(f->pool)), -ESHUTDOWN); + for (unsigned int i = 0; i < count; i++) + put_held(f, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), 0UL); +} + +static void user16k_data_refset_test(struct kunit *test) +{ + struct data_fixture *f; + struct refs_test_state *state; + unsigned int count = PAGE_SIZE / SZ_16K; + unsigned long mask; + struct page *page; + + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (unsigned int i = 0; i < count; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_16K)); + state->folio = mm_subpage_folio(f->held[0]); + page = &state->folio->page; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, refs_test_cleanup, state), 0); + /* Metadata-only ownership fixture: no VMA or rmap is published. */ + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, + SZ_16K - 1024, 2048, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count + 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2); + state->refs[1] = mm_subpage_refs_get(state->refs[0]); + mask = BIT(SZ_16K >> MM_SUBPAGE_SHIFT); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2 + 1); + mask = 0; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[0], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count + 1); + put_held(f, 1); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_16K)), -EAGAIN); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count); +} + +static void user16k_data_mte_test(struct kunit *test) +{ +#ifdef CONFIG_ARM64_MTE + struct data_fixture *f; + struct folio *folio; + struct mm_subpage *copy; + unsigned int count = PAGE_SIZE / SZ_16K; + int ret; + + if (PAGE_SIZE <= SZ_16K || !system_supports_mte()) { + kunit_skip(test, "requires MTE and native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + KUNIT_ASSERT_NOT_NULL(test, f); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + if (try_page_mte_tagging(&folio->page)) + mte_clear_page_tags(folio_address(folio)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(folio_address(folio) + i * SZ_4K, 9); + set_page_mte_tagged(&folio->page); + ret = mm_subpage_pool_add_folio(f->pool, folio, GFP_KERNEL); + if (ret) + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + for (unsigned int i = 0; i < count; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_16K)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, 0)); + memset(address, 0x50 + i, SZ_4K); + test_slot_tags_set(address, 1 + i % 15); + } + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + bool cleared = i >= 4 && i < 8; + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, cleared ? 0 : 1 + i % 15)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(address, cleared ? 0 : 0x50 + i, SZ_4K), NULL); + } + put_held(f, 1); + copy = mm_subpage_copy_at(f->pool, f->held[count - 1], SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + f->held[1] = copy; + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int source = i >= 4 && i < 8 ? i + DATA_SLOTS - 8 : i; + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, 1 + source % 15)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(address, 0x50 + source, SZ_4K), NULL); + } +#else + kunit_skip(test, "MTE is not configured"); +#endif +} + +static void expect_file_coverage(struct kunit *test, struct folio *folio, + unsigned int leaves, unsigned int first, unsigned int tail, + unsigned int final) +{ + struct mm_subpage_mapcounts counts; + unsigned int i; + + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(&folio->page, &counts), leaves); + for (i = 0; i < DATA_SLOTS; i++) { + unsigned int expected = i == 0 ? first : + i == DATA_SLOTS - 1 ? final : i >= DATA_SLOTS - 4 ? tail : 0; + + KUNIT_EXPECT_EQ_MSG(test, counts.slots[i], expected, "coverage slot %u", i); + } +} + +static void user16k_file_coverage_test(struct kunit *test) +{ + struct folio *folio; + phys_addr_t base, tail, last; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative file coverage requires larger native backing"); + return; + } + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, folio), 0); + base = PFN_PHYS(folio_pfn(folio)); + tail = base + PAGE_SIZE - SZ_16K; + last = base + PAGE_SIZE - SZ_4K; + expect_file_coverage(test, folio, 0, 0, 0, 0); + __mm_subpage_file_map_add(tail, SZ_16K); + expect_file_coverage(test, folio, 1, 0, 1, 1); + __mm_subpage_file_map_add(last, SZ_4K); + expect_file_coverage(test, folio, 2, 0, 1, 2); + __mm_subpage_file_map_add(tail, SZ_16K); + expect_file_coverage(test, folio, 3, 0, 2, 3); + __mm_subpage_file_map_add(base, SZ_4K); + expect_file_coverage(test, folio, 4, 1, 2, 3); + __mm_subpage_file_map_del(tail, SZ_16K); + expect_file_coverage(test, folio, 3, 1, 1, 2); + __mm_subpage_file_map_del(last, SZ_4K); + expect_file_coverage(test, folio, 2, 1, 1, 1); + __mm_subpage_file_map_del(tail, SZ_16K); + expect_file_coverage(test, folio, 1, 1, 0, 0); + __mm_subpage_file_map_del(base, SZ_4K); + expect_file_coverage(test, folio, 0, 0, 0, 0); +} + +static struct kunit_case user4k_data_cases[] = { + KUNIT_CASE(user4k_data_reuse_test), + KUNIT_CASE(user4k_data_refset_test), +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) + KUNIT_CASE(user4k_data_skmsg_test), + KUNIT_CASE(user16k_data_skmsg_test), +#endif +#ifdef CONFIG_NET + KUNIT_CASE(user4k_data_skb_test), + KUNIT_CASE(user4k_data_skb_checksum_test), + +#ifdef CONFIG_INET + KUNIT_CASE(user4k_data_skb_prune_test), + KUNIT_CASE(user16k_data_skb_prune_test), +#endif +#endif + KUNIT_CASE(user4k_data_mte_test), + KUNIT_CASE(user4k_data_copy_extent_test), + KUNIT_CASE(user16k_data_owner_test), + KUNIT_CASE(user16k_file_coverage_test), + KUNIT_CASE(user16k_data_refset_test), + KUNIT_CASE(user16k_data_mte_test), + KUNIT_CASE(user4k_data_swapcache_reuse_test), + KUNIT_CASE(user4k_data_restore_test), + KUNIT_CASE(user16k_data_restore_test), + KUNIT_CASE(user4k_data_restore_race_test), + KUNIT_CASE(user4k_data_migration_owner_test), + KUNIT_CASE(user16k_data_migration_owner_test), + KUNIT_CASE(user4k_data_copy_test), + KUNIT_CASE(user4k_data_lookup_test), + KUNIT_CASE(user4k_data_exact_slot_test), + KUNIT_CASE(user4k_data_close_test), + KUNIT_CASE(user4k_data_deferred_test), + KUNIT_CASE(user4k_data_backing_reject_test), + KUNIT_CASE(user4k_data_concurrent_test), + KUNIT_CASE(user4k_data_lookup_race_test), + {} +}; + +static void user4k_data_suite_exit(struct kunit_suite *suite) +{ + /* Drain batched metadata frees before the disposable guest powers off. */ + kvfree_rcu_barrier(); +} + +static struct kunit_suite user4k_data_suite = { + .name = "arm64-user4k-data", + .test_cases = user4k_data_cases, + .suite_exit = user4k_data_suite_exit, +}; + +kunit_test_suite(user4k_data_suite); +MODULE_LICENSE("GPL"); diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c new file mode 100644 index 0000000000000..e406b97ea6235 --- /dev/null +++ b/arch/arm64/mm/user4k-test.c @@ -0,0 +1,14237 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Architectural prerequisite for mixed-granule userspace. + * + * Packed architectural tables and isolated Linux mms are separate fixtures. + * PAGE_SIZE, struct page, all ordinary mms and TTBR1 retain native geometry. Only bounded + * kernel workers may borrow the isolated Linux mms. Never pass + * packed architectural tables to generic MM walkers or expose them to a task. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../../drivers/iommu/iommufd/iommufd_private.h" +#include "../../../drivers/iommu/iommufd/io_pagetable.h" +#include "../../../mm/internal.h" +#include "../../../mm/vma.h" +#include "../../../mm/swap.h" +#ifdef CONFIG_FUTEX +#include "../../../kernel/futex/futex.h" +#endif + +#define USER4K_SHIFT 12 +#define TEST_SLOTS MM_SUBPAGES_PER_PAGE +#define USER4K_SIZE BIT(USER4K_SHIFT) +#define TEST_VA (4UL * SZ_1M) +#define TEST_LEVELS 4 +#define PROBE_SLOTS 4 +#define TEST_ROUNDS 8 +#define TEST_PROT (PTE_TYPE_PAGE | PTE_USER | PTE_AF | PTE_NG | \ + PTE_SHARED | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL)) + +static bool user4k_test_enabled; + +static int __init user4k_test_setup(char *str) +{ + bool enabled; + int ret = kstrtobool(str, &enabled); + + if (ret) + return ret; + /* Software MM tests are sanitizable; raw/borrowed TTBR0 tests stay gated. */ + if (enabled && IS_ENABLED(CONFIG_KASAN)) + return -EINVAL; + user4k_test_enabled = enabled; + return 0; +} +early_param("arm64.user4k_test", user4k_test_setup); + +struct test_tables { + u64 *level[TEST_LEVELS]; + unsigned int shift; +}; + +struct test_result { + u64 native[PROBE_SLOTS]; + u64 read[PROBE_SLOTS]; + u64 write[PROBE_SLOTS]; + u64 replaced[PROBE_SLOTS]; + u64 replaced_read; + u64 kernel; + u64 tcr; + u64 ttbr1; + u64 restored_tcr; + u64 restored_ttbr0; +}; + +static void test_free_page(void *ptr) +{ + free_page((unsigned long)ptr); +} + +static void *test_alloc_page(struct kunit *test) +{ + void *ptr = (void *)get_zeroed_page(GFP_KERNEL); + + if (!ptr) + return NULL; + if (kunit_add_action_or_reset(test, test_free_page, ptr)) + return NULL; + return ptr; +} + +static void test_put_compound(void *ptr) +{ + put_page(ptr); +} + +static void *test_alloc_granule(struct kunit *test, unsigned int shift) +{ + struct page *page; + + if (shift <= PAGE_SHIFT) + return test_alloc_page(test); + page = alloc_pages(GFP_KERNEL | __GFP_ZERO | __GFP_COMP, shift - PAGE_SHIFT); + if (!page) + return NULL; + if (kunit_add_action_or_reset(test, test_put_compound, page)) + return NULL; + return page_address(page); +} + +static unsigned int test_index(unsigned long va, unsigned int shift, + unsigned int level) +{ + unsigned int bits = shift - PTDESC_ORDER; + + return (va >> ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, shift)) & (BIT(bits) - 1); +} + +static int test_alloc_tables(struct kunit *test, struct test_tables *tables, + unsigned int shift) +{ + int level; + void *packed = NULL; + + tables->shift = shift; + if (shift == USER4K_SHIFT) { + /* Also test table address bits 13:12, not just leaf addresses. */ + packed = test_alloc_page(test); + if (!packed) + return -ENOMEM; + } + for (level = 0; level < TEST_LEVELS; level++) { + tables->level[level] = packed ? packed + level * USER4K_SIZE : + test_alloc_page(test); + if (!tables->level[level]) + return -ENOMEM; + } + for (level = 0; level < TEST_LEVELS - 1; level++) { + unsigned int index = test_index(TEST_VA, shift, level); + + tables->level[level][index] = + virt_to_phys(tables->level[level + 1]) | PMD_TYPE_TABLE; + } + return 0; +} + +static void user4k_geometry_test(struct kunit *test) +{ + static const unsigned int shifts4k[] = { 39, 30, 21, 12 }; + static const unsigned int shifts16k[] = { 47, 36, 25, 14 }; + static const unsigned int shifts64k[] = { 55, 42, 29, 16 }; + unsigned int level; + + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 12), 4); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 14), 4); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 16), 3); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(42, 16), 2); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(39, 12), 3); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(47, 14), 3); + for (level = 0; level < TEST_LEVELS; level++) { + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, 12), + shifts4k[level]); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, 14), + shifts16k[level]); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT(level), + PAGE_SHIFT == 16 ? shifts64k[level] : shifts16k[level]); + KUNIT_EXPECT_EQ(test, test_index(BIT(shifts4k[level]), 12, level), 1U); + KUNIT_EXPECT_EQ(test, test_index(BIT(shifts4k[level]) - 1, 12, level), 0U); + } +} + +static void test_leaf_encoding(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + const phys_addr_t base = GENMASK_ULL(47, PAGE_SHIFT); + pgprot_t prot = __pgprot(TEST_PROT | PTE_RDONLY); + unsigned int i; + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, mm); + pte = phys_pte_mm(mm, base, prot); + KUNIT_EXPECT_EQ(test, pte_val(pte), pte_val(pfn_pte(base >> PAGE_SHIFT, prot))); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), base); + KUNIT_EXPECT_EQ(test, pgprot_val(pte_pgprot_mm(mm, pte)), pgprot_val(prot)); + mm->context.flags |= shift == 12 ? MMCF_USER_4K : MMCF_USER_16K; + for (i = 0; i < (PAGE_SIZE >> shift); i++) { + phys_addr_t phys = base + i * (1UL << shift); + + pte = phys_pte_mm(mm, phys, prot); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys); + KUNIT_EXPECT_EQ(test, pte_pfn(pte), (unsigned long)(base >> PAGE_SHIFT)); + KUNIT_EXPECT_EQ(test, pgprot_val(pte_pgprot_mm(mm, pte)), pgprot_val(prot)); + if (i + 1 < (PAGE_SIZE >> shift)) + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte_advance_pfn_mm(mm, pte, 1)), + phys + (1UL << shift)); + pte = pte_modify(pte, __pgprot(TEST_PROT)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys); + KUNIT_EXPECT_FALSE(test, pte_val(pte) & PTE_RDONLY); + } +} + +static void user4k_leaf_encoding_test(struct kunit *test) +{ + test_leaf_encoding(test, 12); +} + +static void user16k_leaf_encoding_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative leaves require larger native backing"); + return; + } + test_leaf_encoding(test, 14); +} + + +/* Caller masks all asynchronous exceptions; this function cannot sleep. */ +static void test_switch_ttbr0(u64 ttbr, u64 tcr) +{ + /* The zero root is valid for both tested granules. */ + cpu_set_reserved_ttbr0(); + local_flush_tlb_all(); + write_sysreg(tcr, tcr_el1); + isb(); + write_sysreg(ttbr, ttbr0_el1); + isb(); +} + +static u64 test_translate(unsigned long va, bool write) +{ + if (write) + asm volatile("at s1e0w, %0" : : "r" (va) : "memory"); + else + asm volatile("at s1e0r, %0" : : "r" (va) : "memory"); + isb(); + return read_sysreg_par(); +} + +#define MM_TEST_MAPPINGS 11 + +struct test_mm { + struct kunit *test; + struct mm_struct *mm; + unsigned long address[MM_TEST_MAPPINGS]; + phys_addr_t phys[MM_TEST_MAPPINGS]; +}; + +static pte_t *test_mm_lookup(struct mm_struct *mm, unsigned long addr, + spinlock_t **ptl); + +static void test_mm_hardware(struct test_mm *ctx) +{ + struct kunit *test = ctx->test; + struct vm_area_struct vma = TLB_FLUSH_VMA(ctx->mm, 0); + struct arch_tlbflush_unmap_batch batch = {}; + pte_t *leaves[4]; + u64 invalidated[3][4]; + u64 result[MM_TEST_MAPPINGS], writes[MM_TEST_MAPPINGS]; + u64 old_tcr, old_ttbr0, old_ttbr1, old_par, tcr, kernel; + u64 old_id = atomic64_read(&ctx->mm->context.id); + u64 restored_tcr, restored_ttbr0, retained_ttbr1; + unsigned long flags; + unsigned int i, step; + + for (i = 0; i < ARRAY_SIZE(leaves); i++) { + spinlock_t *ptl; + + leaves[i] = test_mm_lookup(ctx->mm, ctx->address[i], &ptl); + if (leaves[i]) + pte_unmap_unlock(leaves[i], ptl); + KUNIT_ASSERT_NOT_NULL(test, leaves[i]); + } + + preempt_disable(); + flags = local_daif_save(); + old_tcr = read_sysreg(tcr_el1); + old_ttbr0 = read_sysreg(ttbr0_el1); + old_ttbr1 = read_sysreg(ttbr1_el1); + old_par = read_sysreg_par(); + if ((old_tcr & (TCR_EL1_DS | TCR_EL1_TG1_MASK | TCR_EL1_A1)) != + (TCR_TG1_16K | TCR_EL1_A1)) { + local_daif_restore(flags); + preempt_enable(); + kunit_skip(test, "requires 16K TTBR1, DS=0, and A1=1"); + return; + } + tcr = (old_tcr & ~(TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK | TCR_EL1_EPD0)) | + TCR_TG0_4K | TCR_T0SZ(48); + test_switch_ttbr0(virt_to_phys(ctx->mm->pgd), tcr); + /* A1 selects the saved TTBR1 ASID for these private test translations. */ + atomic64_set(&ctx->mm->context.id, old_ttbr1 >> 48); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + result[i] = test_translate(ctx->address[i], false); + writes[i] = test_translate(ctx->address[i], true); + } + for (step = 0; step < ARRAY_SIZE(invalidated); step++) { + unsigned int first = step == 2 ? 3 : 1; + unsigned int last = step == 1 ? 3 : first + 1; + + for (i = first; i < last; i++) + WRITE_ONCE(*leaves[i], __pte(0)); + switch (step) { + case 0: + __flush_tlb_page(&vma, ctx->address[first] + 1, + TLBF_NONOTIFY | TLBF_NOBROADCAST); + break; + case 1: + __flush_tlb_range(&vma, ctx->address[first], ctx->address[last], + SZ_4K, 3, TLBF_NONOTIFY); + break; + case 2: + arch_tlbbatch_add_pending(&batch, ctx->mm, ctx->address[first], + ctx->address[first] + SZ_4K); + arch_tlbbatch_flush(&batch); + break; + } + isb(); + for (i = 0; i < ARRAY_SIZE(leaves); i++) + invalidated[step][i] = test_translate(ctx->address[i], false); + for (i = first; i < last; i++) + WRITE_ONCE(*leaves[i], __pte(ctx->phys[i] | TEST_PROT)); + dsb(ishst); + isb(); + /* Repopulate before the next independent invalidation. */ + for (i = 0; i < ARRAY_SIZE(leaves); i++) + test_translate(ctx->address[i], false); + } + asm volatile("at s1e1r, %0" : : "r" (ctx->mm->pgd) : "memory"); + isb(); + kernel = read_sysreg_par(); + retained_ttbr1 = read_sysreg(ttbr1_el1); + test_switch_ttbr0(old_ttbr0, old_tcr); + restored_tcr = read_sysreg(tcr_el1); + restored_ttbr0 = read_sysreg(ttbr0_el1); + write_sysreg(old_par, par_el1); + isb(); + atomic64_set(&ctx->mm->context.id, old_id); + local_daif_restore(flags); + preempt_enable(); + + KUNIT_EXPECT_EQ(test, restored_tcr, old_tcr); + KUNIT_EXPECT_EQ(test, restored_ttbr0, old_ttbr0); + KUNIT_EXPECT_EQ(test, retained_ttbr1, old_ttbr1); + KUNIT_EXPECT_EQ(test, kernel & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, kernel & SYS_PAR_EL1_PA, + virt_to_phys(ctx->mm->pgd) & SYS_PAR_EL1_PA); + for (step = 0; step < ARRAY_SIZE(invalidated); step++) { + for (i = 0; i < ARRAY_SIZE(leaves); i++) { + bool hole = step == 0 ? i == 1 : step == 1 ? i == 1 || i == 2 : i == 3; + + KUNIT_EXPECT_EQ(test, invalidated[step][i] & SYS_PAR_EL1_F, (u64)hole); + if (!hole) + KUNIT_EXPECT_EQ(test, invalidated[step][i] & SYS_PAR_EL1_PA, + ctx->phys[i]); + } + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + KUNIT_EXPECT_EQ(test, result[i] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, result[i] & SYS_PAR_EL1_PA, ctx->phys[i]); + KUNIT_EXPECT_EQ(test, writes[i] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, writes[i] & SYS_PAR_EL1_PA, ctx->phys[i]); + } +} + +struct test_walk { + struct test_mm *ctx; + unsigned long size; + unsigned long holes; + unsigned int present; +}; + +/* Callbacks run under the PTE lock; record errors without invoking KUnit. */ +static int test_walk_pte(pte_t *pte, unsigned long addr, unsigned long next, + struct mm_walk *walk) +{ + struct test_walk *state = walk->private; + u64 value = pte_val(ptep_get(pte)); + unsigned int i; + + if (next - addr != state->size) + return -EINVAL; + if (!value) { + state->holes += next - addr; + return 0; + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + if (state->ctx->address[i] != addr) + continue; + if (value != (state->ctx->phys[i] | TEST_PROT)) + return -EINVAL; + state->present++; + return 0; + } + return -EINVAL; +} + +static int test_walk_hole(unsigned long addr, unsigned long next, int depth, + struct mm_walk *walk) +{ + struct test_walk *state = walk->private; + + state->holes += next - addr; + return 0; +} + +static const struct mm_walk_ops test_walk_ops = { + .pte_entry = test_walk_pte, + .pte_hole = test_walk_hole, +}; + +static pte_t *test_mm_lookup(struct mm_struct *mm, unsigned long addr, + spinlock_t **ptl) +{ + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + if (pgd_none(*pgd) || pgd_bad(*pgd)) + return NULL; + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_mm(mm, *p4d) || p4d_bad_mm(mm, *p4d)) + return NULL; + pud = pud_offset_mm(mm, p4d, addr); + if (pud_none(*pud) || pud_bad(*pud)) + return NULL; + pmd = pmd_offset_mm(mm, pud, addr); + if (pmd_none(*pmd) || pmd_bad(*pmd)) + return NULL; + return pte_offset_map_lock(mm, pmd, addr, ptl); +} + +static void test_mm_destroy(void *ptr) +{ + struct test_mm *ctx = ptr; + struct mm_struct *mm = ctx->mm; + struct mmu_gather tlb; + unsigned int i; + + if (!mm) + return; + mmap_write_lock(mm); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + + if (pte) { + /* Test-owned physical pages, not faulted/rmap-owned pages. */ + pte_clear(mm, ctx->address[i], pte); + pte_unmap_unlock(pte, ptl); + } + } + /* Exercise the real generic gather/teardown path, not a test walker. */ + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(ctx->test, mm_pgtables_bytes(mm), 0UL); + mmput(mm); + ctx->mm = NULL; +} + +static unsigned int test_mm_backing_pages(struct test_mm *ctx) +{ + struct page *owners[3 * MM_TEST_MAPPINGS]; + unsigned int count = 0, total = 0, i, j, level; + struct mm_struct *mm = ctx->mm; + + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + unsigned long addr = ctx->address[i]; + p4d_t *p4d = p4d_offset_mm(mm, pgd_offset(mm, addr), addr); + pud_t *pud = pud_offset_mm(mm, p4d, addr); + pmd_t *pmd = pmd_offset_mm(mm, pud, addr); + pte_t *pte = pte_offset_kernel_mm(mm, pmd, addr); + struct page *pages[] = { virt_to_page(pud), virt_to_page(pmd), virt_to_page(pte) }; + + for (level = mm_pud_folded(mm) ? 1 : 0; level < ARRAY_SIZE(pages); level++) { + pages[level] = compound_head(pages[level]); + for (j = 0; j < count; j++) + if (owners[j] == pages[level]) + break; + if (j == count) { + owners[count++] = pages[level]; + total += 1U << compound_order(pages[level]); + } + } + } + return total; +} + +/* Only valid for a private, empty mm with no ASID or hardware references. */ +static int test_mm_select_granule(struct mm_struct *mm, unsigned int shift) +{ + pgd_t *native_pgd = mm->pgd, *new_pgd; + unsigned long flag = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + + mm->context.flags |= flag; + new_pgd = pgd_alloc(mm); + mm->context.flags &= ~flag; + if (!new_pgd) + return -ENOMEM; + pgd_free(mm, native_pgd); + mm->pgd = new_pgd; + mm->context.flags |= flag; + return 0; +} + +static int test_mm_select4k(struct mm_struct *mm) +{ + return test_mm_select_granule(mm, 12); +} + +static void test_mm_tables(struct kunit *test, unsigned int shift, bool hardware) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + bool small = shift != PAGE_SHIFT; + unsigned int pmd_shift = small ? shift + (shift - 3) : PMD_SHIFT; + unsigned int pud_shift = small ? shift + 2 * (shift - 3) : PUD_SHIFT; + unsigned int pgd_shift = small ? (shift == 16 ? pud_shift : + shift + 3 * (shift - 3)) : PGDIR_SHIFT; + unsigned long size = 1UL << shift; + struct mm_struct *mm; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + for (i = 0; i < 4; i++) + ctx->address[i] = TEST_VA + i * size; + ctx->address[4] = (1UL << pmd_shift) - size; + ctx->address[5] = 1UL << pmd_shift; + ctx->address[6] = (1UL << pud_shift) - size; + ctx->address[7] = 1UL << pud_shift; + ctx->address[8] = (1UL << pgd_shift) - size; + ctx->address[9] = 1UL << pgd_shift; + ctx->address[10] = (1UL << 48) - size; + if (pud_shift == pgd_shift) { + ctx->address[6] -= size; + ctx->address[7] += size; + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + void *data = test_alloc_granule(test, shift); + + KUNIT_ASSERT_NOT_NULL(test, data); + ctx->phys[i] = virt_to_phys(data) + (shift < PAGE_SHIFT ? (i % (PAGE_SIZE >> shift)) * size : 0); + } + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + ctx->mm = mm; + /* Register after the data allocations: detach tables before freeing data. */ + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_EXPECT_EQ(test, mm_page_shift(mm), (unsigned int)PAGE_SHIFT); + if (small) { + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shift), 0); + KUNIT_EXPECT_EQ(test, (unsigned long)mm->pgd & (size - 1), 0UL); + } + KUNIT_EXPECT_EQ(test, mm_page_shift(mm), shift); + if (small) + KUNIT_EXPECT_EQ(test, mm_pud_folded(mm), shift == 16); + KUNIT_EXPECT_EQ(test, mm_page_size(mm), size); + KUNIT_EXPECT_EQ(test, mm_page_mask(mm), ~(size - 1)); + KUNIT_EXPECT_EQ(test, mm_page_shift(&init_mm), (unsigned int)PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, pmd_addr_end_mm(mm, (1UL << pmd_shift) - 1, + (1UL << pmd_shift) + size), + 1UL << pmd_shift); + KUNIT_EXPECT_EQ(test, pgd_addr_end_mm(mm, ULONG_MAX - size, 0), 0UL); + + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + unsigned long addr = ctx->address[i]; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d = p4d_alloc(mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + KUNIT_ASSERT_NOT_NULL(test, p4d); + pud = pud_alloc(mm, p4d, addr); + KUNIT_ASSERT_NOT_NULL(test, pud); + pmd = pmd_alloc(mm, pud, addr); + KUNIT_ASSERT_NOT_NULL(test, pmd); + pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + /* Do not assert while holding the spinlock/RCU section. */ + set_pte(pte, __pte(ctx->phys[i] | TEST_PROT)); + pte_unmap_unlock(pte, ptl); + } + /* Installed table bytes and physical backing (including cached slots) differ. */ + { + unsigned int shifts[] = { pmd_shift, pud_shift, pgd_shift }; + unsigned int tables = 0, owners = 0; + + for (unsigned int level = 0; level < (mm_pud_folded(mm) ? 2 : 3); level++) { + unsigned int unique = 0; + + for (unsigned int a = 0; a < MM_TEST_MAPPINGS; a++) { + unsigned int b; + + for (b = 0; b < a; b++) + if ((ctx->address[a] >> shifts[level]) == + (ctx->address[b] >> shifts[level])) + break; + unique += b == a; + } + tables += unique; + if (shift < PAGE_SHIFT) + owners += DIV_ROUND_UP(unique, PAGE_SIZE >> shift); + else + owners += unique << (shift - PAGE_SHIFT); + } + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), tables * size); + KUNIT_EXPECT_EQ(test, test_mm_backing_pages(ctx), owners); + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + u64 value; + + KUNIT_ASSERT_NOT_NULL(test, pte); + value = pte_val(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, value, ctx->phys[i] | TEST_PROT); + } + { + struct vm_area_struct *vma = vm_area_alloc(mm); + struct test_walk state = { .ctx = ctx, .size = size }; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_start = 0; + vma->vm_end = 1UL << 48; + mmap_write_lock(mm); + ret = walk_page_range_vma(vma, 0, 1UL << 48, &test_walk_ops, &state); + mmap_write_unlock(mm); + vm_area_free(vma); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, state.present, MM_TEST_MAPPINGS); + KUNIT_EXPECT_EQ(test, state.holes, (1UL << 48) - MM_TEST_MAPPINGS * size); + } + if (hardware) + test_mm_hardware(ctx); + { + struct mmu_gather tlb; + unsigned long before = mm_pgtables_bytes(mm); + unsigned long boundary = 1UL << pmd_shift; + + mmap_write_lock(mm); + /* An adjacent VMA's unaligned ceiling must keep the shared table. */ + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, boundary - size, 0, boundary - size); + tlb_finish_mmu(&tlb); + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), before); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte; + + if (ctx->address[i] >= boundary) + continue; + pte = test_mm_lookup(mm, ctx->address[i], &ptl); + if (pte) { + pte_clear(mm, ctx->address[i], pte); + pte_unmap_unlock(pte, ptl); + } + } + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, boundary, 0, boundary); + tlb_finish_mmu(&tlb); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), before - size); + /* The next table and the higher levels must still be walkable. */ + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + u64 value; + + if (ctx->address[i] < boundary) { + if (pte) + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_PTR_EQ(test, pte, NULL); + continue; + } + KUNIT_ASSERT_NOT_NULL(test, pte); + value = pte_val(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, value, ctx->phys[i] | TEST_PROT); + } + } + test_mm_destroy(ctx); +} + +static void user4k_mm_tables_test(struct kunit *test) +{ + test_mm_tables(test, 12, false); +} + +static void native_mm_tables_test(struct kunit *test) +{ + test_mm_tables(test, PAGE_SHIFT, false); +} + +static void user16k_mm_tables_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative tables require larger native backing"); + return; + } + test_mm_tables(test, 14, false); +} + +static void user64k_mm_tables_test(struct kunit *test) +{ + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "64K is already the native geometry"); + return; + } + test_mm_tables(test, 16, false); +} + +static void test_free_vma(void *ptr) +{ + vm_area_free(ptr); +} + +static void user4k_anon_read_fault_test(struct kunit *test) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + struct vm_area_struct *vma; + vm_flags_t vm_flags = VM_READ | VM_MAYREAD; + unsigned int i, j; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + ctx->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, ctx->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_ASSERT_EQ(test, test_mm_select4k(ctx->mm), 0); + for (i = 0; i < 4; i++) + ctx->address[i] = TEST_VA + i * SZ_4K; + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + PAGE_SIZE; + vma_set_anonymous(vma); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + vm_flags |= VM_NOHUGEPAGE; +#endif + vm_flags_init(vma, vm_flags); + vma->vm_page_prot = vm_get_page_prot(vm_flags); + /* This VMA is a private fault fixture, not inserted into the VMA tree. */ + for (i = 0; i < 4; i++) { + vm_fault_t ret; + + mmap_read_lock(ctx->mm); + ret = handle_mm_fault(vma, ctx->address[i] + 1, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(ctx->mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + for (j = 0; j < 4; j++) { + spinlock_t *ptl; + pte_t *ptep = test_mm_lookup(ctx->mm, ctx->address[j], &ptl); + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, ptep); + pte = ptep_get(ptep); + pte_unmap_unlock(ptep, ptl); + if (j > i) { + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + continue; + } + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_special(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_TRUE(test, is_zero_pfn(pte_pfn(pte))); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(ctx->mm, MM_ANONPAGES), 0L); +} + +static void user4k_mm_hardware_test(struct kunit *test) +{ + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE)) { + kunit_skip(test, "requires 4K, no CnP, and direct PTE permissions"); + return; + } + test_mm_tables(test, 12, true); +} + +struct fragment_fixture { + struct mm_struct *mm; + void *table[TEST_SLOTS + 2]; + struct page *pinned[3]; +}; + +static void fragment_fixture_destroy(void *data) +{ + struct fragment_fixture *f = data; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(f->table); i++) + if (f->table[i]) + arm64_pgtable_free(f->table[i]); + if (f->mm) + mmput(f->mm); + rcu_barrier(); + for (i = 0; i < ARRAY_SIZE(f->pinned); i++) + if (f->pinned[i]) + put_page(f->pinned[i]); +} + +static void test_fragment_lifetime(struct kunit *test, unsigned int shift) +{ + struct fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct ptdesc *pte_owner; + unsigned int i, slots = PAGE_SIZE >> shift; + unsigned long size = 1UL << shift; + unsigned int read_shift = 0; + bool live; + + KUNIT_ASSERT_NOT_NULL(test, f); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, fragment_fixture_destroy, f), 0); + /* Direct allocator fixture: the mm's native root is never changed or used. */ + for (i = 0; i < slots + 2; i++) { + enum arm64_user4k_pt_level level = i < slots ? USER4K_PT_PTE : i - slots + 1; + + f->table[i] = arm64_user_pt_alloc_granule(f->mm, level, shift); + KUNIT_ASSERT_NOT_NULL(test, f->table[i]); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)f->table[i], size)); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(f->table[i]), shift); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(f->table[i] + size - sizeof(pte_t)), shift); + KUNIT_EXPECT_EQ(test, pte_batch_hint(f->table[i], __pte(TEST_PROT | PTE_CONT)), + IS_ENABLED(CONFIG_ARM64_CONTPTE) ? arm64_cont_ptes(shift) : 1U); + if (!i && PAGE_SHIFT == 16) { + void *cached = f->mm->context.user4k_pt_frag[level]; + + KUNIT_EXPECT_PTR_EQ(test, arm64_user_pt_alloc_granule(f->mm, level, + shift == 12 ? 14 : 12), NULL); + KUNIT_EXPECT_PTR_EQ(test, f->mm->context.user4k_pt_frag[level], cached); + } + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0, size), NULL); + if (i == 0 || i >= slots) { + unsigned int slot = i == 0 ? 0 : i - slots + 1; + + f->pinned[slot] = virt_to_page(f->table[i]); + get_page(f->pinned[slot]); + } else { + KUNIT_EXPECT_PTR_EQ(test, virt_to_page(f->table[i]), f->pinned[0]); + KUNIT_EXPECT_EQ(test, (unsigned long)f->table[i] - + (unsigned long)f->table[0], i * (unsigned long)size); + } + memset(f->table[i], 0x50 + i, size); + } + for (i = 0; i < slots + 2; i++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0x50 + i, size), NULL); + memset(f->table[i], 0, size); + } + /* Pointer-only walkers must inspect all 128 entries of a 16K group. */ + if (IS_ENABLED(CONFIG_ARM64_CONTPTE)) { + pte_t *ptes = f->table[0]; + unsigned int nr = arm64_cont_ptes(shift), j; + pte_t observed; + + for (j = 0; j < nr; j++) + ptes[j] = pte_mkold(pte_mkclean(__pte(TEST_PROT | PTE_CONT | + (SZ_4M + (j << shift))))); + ptes[nr - 1] = pte_mkyoung(pte_mkdirty(ptes[nr - 1])); + observed = ptep_get(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + observed = ptep_get_lockless(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + KUNIT_EXPECT_EQ(test, pte_val(observed) & + (ARM64_USER4K_TABLE_ADDR_MASK & ~(size - 1)), + (u64)(SZ_4M + size)); + memset(ptes, 0, size); + } + KUNIT_EXPECT_PTR_NE(test, f->pinned[0], f->pinned[1]); + KUNIT_EXPECT_PTR_NE(test, f->pinned[1], f->pinned[2]); + pte_owner = page_ptdesc(f->pinned[0]); + KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots); + arm64_pte_free_defer(f->mm, f->table[0]); + f->table[0] = NULL; + KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots - 1); + for (i = 1; i < slots - 1; i++) { + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + } + /* A pre-existing software walker must survive the last fragment release. */ + rcu_read_lock(); + arm64_pgtable_free(f->table[slots - 1]); + f->table[slots - 1] = NULL; + live = folio_test_pgtable(page_folio(f->pinned[0])); + read_shift = arm64_ptep_page_shift(page_address(f->pinned[0])); + rcu_read_unlock(); + KUNIT_EXPECT_TRUE(test, live); + KUNIT_EXPECT_EQ(test, read_shift, shift); + rcu_barrier(); + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[0]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[0]), 1); + for (i = slots; i < slots + 2; i++) { + struct ptdesc *pt = virt_to_ptdesc(f->table[i]); + + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), (int)slots - 1); + KUNIT_EXPECT_TRUE(test, folio_test_pgtable(ptdesc_folio(pt))); + } + mmput(f->mm); + f->mm = NULL; + for (i = 1; i < 3; i++) { + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[i]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[i]), 1); + } +} + +/* Direct table-owner test; no larger-than-native user mm is enabled here. */ +static void user64k_table_owner_test(struct kunit *test) +{ + struct fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + unsigned int i, offset, order = 16 - PAGE_SHIFT; + unsigned int shift; + bool live; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, fragment_fixture_destroy, f), 0); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + for (i = 0; i < 3; i++) { + struct ptdesc *pt; + + f->table[i] = arm64_user_pt_alloc_granule(f->mm, i, 16); + KUNIT_ASSERT_NOT_NULL(test, f->table[i]); + pt = virt_to_ptdesc(f->table[i]); + f->pinned[i] = ptdesc_page(pt); + get_page(f->pinned[i]); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)f->table[i], SZ_64K)); + KUNIT_EXPECT_EQ(test, compound_order(f->pinned[i]), order); + KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), 1); + KUNIT_EXPECT_PTR_EQ(test, f->mm->context.user4k_pt_frag[i], NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0, SZ_64K), NULL); + /* Include the last entry of every constituent native page. */ + for (offset = PAGE_SIZE - sizeof(pte_t); offset < SZ_64K; offset += PAGE_SIZE) { + void *entry = f->table[i] + offset; + + KUNIT_EXPECT_PTR_EQ(test, virt_to_ptdesc(entry), pt); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(entry), 16U); + KUNIT_EXPECT_EQ(test, pte_batch_hint(entry, __pte(TEST_PROT | PTE_CONT)), + 32U - ((offset / sizeof(pte_t)) % 32)); + if (i == USER4K_PT_PTE) + KUNIT_EXPECT_PTR_EQ(test, ptep_lockptr(f->mm, entry), + ptep_lockptr(f->mm, f->table[i])); + if (i == USER4K_PT_PMD) + KUNIT_EXPECT_PTR_EQ(test, pmd_lockptr(f->mm, entry), + pmd_lockptr(f->mm, f->table[i])); + } + memset(f->table[i], 0x51 + i, SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0x51 + i, SZ_64K), NULL); + memset(f->table[i], 0, SZ_64K); + } + /* Contiguous readers on the last native page must still use 64K geometry. */ + if (IS_ENABLED(CONFIG_ARM64_CONTPTE)) { + pte_t *ptes = f->table[0] + SZ_64K - 32 * sizeof(pte_t); + pte_t observed; + unsigned int j; + + for (j = 0; j < 32; j++) + ptes[j] = pte_mkold(pte_mkclean(__pte(TEST_PROT | PTE_CONT | + (SZ_4M + (j << 16))))); + ptes[31] = pte_mkyoung(pte_mkdirty(ptes[31])); + observed = ptep_get(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + observed = ptep_get_lockless(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + KUNIT_EXPECT_EQ(test, pte_val(observed) & + (ARM64_USER4K_TABLE_ADDR_MASK & ~(SZ_64K - 1UL)), + (u64)(SZ_4M + SZ_64K)); + memset(f->table[0], 0, SZ_64K); + } + rcu_read_lock(); + arm64_pte_free_defer(f->mm, f->table[0]); + live = folio_test_pgtable(page_folio(f->pinned[0])); + shift = arm64_ptep_page_shift(f->table[0] + SZ_64K - sizeof(pte_t)); + f->table[0] = NULL; + rcu_read_unlock(); + KUNIT_EXPECT_TRUE(test, live); + KUNIT_EXPECT_EQ(test, shift, 16U); + rcu_barrier(); + for (i = 0; i < 3; i++) { + if (f->table[i]) { + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + } + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[i]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[i]), 1); + KUNIT_EXPECT_EQ(test, compound_order(f->pinned[i]), order); + } +} + +static void user4k_fragment_lifetime_test(struct kunit *test) +{ + test_fragment_lifetime(test, 12); +} + +static void user16k_fragment_lifetime_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K fragments require larger native backing"); + return; + } + test_fragment_lifetime(test, 14); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static void test_put_mm(void *ptr) +{ + mmput(ptr); +} + +static void native_pgtable_deposit_test(struct kunit *test) +{ + struct mm_struct *mm = mm_alloc(); + pgtable_t first, second, out1, out2; + pmd_t *pmd; + spinlock_t *ptl; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_put_mm, mm), 0); + pmd = pmd_alloc_one(mm, 0); + KUNIT_ASSERT_NOT_NULL(test, pmd); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, pmd), 0); + first = pte_alloc_one(mm); + KUNIT_ASSERT_NOT_NULL(test, first); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, first), 0); + second = pte_alloc_one(mm); + KUNIT_ASSERT_NOT_NULL(test, second); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, second), 0); + ptl = pmd_lock(mm, pmd); + pgtable_trans_huge_deposit(mm, pmd, first); + pgtable_trans_huge_deposit(mm, pmd, second); + out1 = pgtable_trans_huge_withdraw(mm, pmd); + out2 = pgtable_trans_huge_withdraw(mm, pmd); + spin_unlock(ptl); + KUNIT_EXPECT_PTR_EQ(test, out1, second); + KUNIT_EXPECT_PTR_EQ(test, out2, first); +} +#endif + +static void test_data_slot_put(void *ptr) +{ + mm_subpage_put(ptr); +} + +static void test_data_pool_free(void *ptr) +{ + mm_subpage_pool_close(ptr); + mm_subpage_pool_put(ptr); +} + +static bool test_alloc_data_slots(struct kunit *test, struct test_mm *ctx) +{ + struct mm_subpage_pool *pool = mm_subpage_pool_create(GFP_KERNEL); + struct folio *folio; + unsigned int i; + int ret; + + if (!pool) + return false; + if (kunit_add_action_or_reset(test, test_data_pool_free, pool)) + return false; + folio = folio_alloc(GFP_KERNEL, 0); + if (!folio) + return false; + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + return false; + } + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc(pool); + + if (IS_ERR(slot)) + return false; + if (kunit_add_action_or_reset(test, test_data_slot_put, slot)) + return false; + ctx->phys[i] = mm_subpage_phys(slot); + } + return true; +} + +/* These private mappings are borrowed only for bounded, fault-disabled uaccess. */ +static struct test_mm *test_usercopy_mm(struct kunit *test, bool small) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + unsigned long size = small ? SZ_4K : PAGE_SIZE; + unsigned int i; + + if (!ctx) + return NULL; + ctx->test = test; + if (small && !test_alloc_data_slots(test, ctx)) + return NULL; + for (i = 0; i < 4; i++) { + if (!small) { + void *data = test_alloc_page(test); + + if (!data) + return NULL; + ctx->phys[i] = virt_to_phys(data); + } + ctx->address[i] = TEST_VA + i * size; + memset(phys_to_virt(ctx->phys[i]), (small ? 0x40 : 0x80) + i, 64); + } + ctx->mm = mm_alloc(); + if (!ctx->mm) + return NULL; + if (kunit_add_action_or_reset(test, test_mm_destroy, ctx)) + return NULL; + if (small && test_mm_select4k(ctx->mm)) + return NULL; + for (i = 0; i < 4; i++) { + unsigned long addr = ctx->address[i]; + pgd_t *pgd = pgd_offset(ctx->mm, addr); + p4d_t *p4d = p4d_alloc(ctx->mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + if (!p4d) + return NULL; + pud = pud_alloc(ctx->mm, p4d, addr); + if (!pud) + return NULL; + pmd = pmd_alloc(ctx->mm, pud, addr); + if (!pmd) + return NULL; + pte = pte_alloc_map_lock(ctx->mm, pmd, addr, &ptl); + if (!pte) + return NULL; + if (i != 2) + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[i], + __pgprot(TEST_PROT | (i ? 0 : PTE_RDONLY)))); + pte_unmap_unlock(pte, ptl); + } + return ctx; +} + +static void user4k_rmap_walk_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, true); + struct vm_area_struct *vma; + unsigned int i, j, sync; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * PAGE_SIZE; + /* A later alias must not extend a native-owner range walk's bound. */ + ctx->address[4] = TEST_VA + PAGE_SIZE; + ctx->phys[4] = ctx->phys[0]; + pte = test_mm_lookup(ctx->mm, ctx->address[4], &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[4], __pgprot(TEST_PROT))); + pte_unmap_unlock(pte, ptl); + /* Same physical slot at a different virtual byte offset and native index. */ + ctx->address[5] = TEST_VA + PAGE_SIZE + SZ_4K; + ctx->phys[5] = ctx->phys[3]; + pte = test_mm_lookup(ctx->mm, ctx->address[5], &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[5], __pgprot(TEST_PROT))); + pte_unmap_unlock(pte, ptl); + for (sync = 0; sync < 2; sync++) { + struct page_vma_mapped_walk whole = { + .pfn = ctx->phys[0] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = TEST_VA, + .flags = sync ? PVMW_SYNC : 0, + }; + unsigned int count = 0; + bool valid = true; + + /* A native-owner walk must visit every mapped 4K occupant. */ + while (page_vma_mapped_walk(&whole)) { + unsigned int index = (whole.address - TEST_VA) / SZ_4K; + + if (index >= 4 || index == 2 || !whole.pte || + pte_phys_mm(ctx->mm, ptep_get(whole.pte)) != ctx->phys[index]) + valid = false; + if (++count > 4) { + page_vma_mapped_walk_done(&whole); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 3U); + for (i = 0; i < 4; i++) { + for (j = 0; j < 6; j++) { + struct rmap_walk_range range = { + .address = ctx->address[j], + .subpage = true, + .subpage_offset = offset_in_page(ctx->phys[i]), + }; + DEFINE_FOLIO_RMAP_WALK(one, page_folio(pfn_to_page(PHYS_PFN(ctx->phys[i]))), + vma, range, sync ? PVMW_SYNC : 0); + struct mm_subpage *slot = NULL; + bool found = page_vma_mapped_walk(&one); + phys_addr_t phys = 0; + + if (found) { + phys = pte_phys_mm(ctx->mm, ptep_get(one.pte)); + slot = mm_subpage_get_from_phys(phys); + page_vma_mapped_walk_done(&one); + } + KUNIT_EXPECT_EQ(test, found, (i == j && j != 2) || + (j == 4 && i == 0) || (j == 5 && i == 3)); + if (found) { + KUNIT_EXPECT_EQ(test, phys, ctx->phys[i]); + KUNIT_EXPECT_NOT_NULL(test, slot); + if (slot) { + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), phys); + mm_subpage_put(slot); + } + } + } + } + } +} + +static int test_map_alias(struct test_mm *ctx, unsigned int index, + unsigned long addr, phys_addr_t phys) +{ + struct mm_struct *mm = ctx->mm; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d = p4d_alloc(mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + ctx->address[index] = addr; + ctx->phys[index] = phys; + if (!p4d) + return -ENOMEM; + pud = pud_alloc(mm, p4d, addr); + if (!pud) + return -ENOMEM; + pmd = pmd_alloc(mm, pud, addr); + if (!pmd) + return -ENOMEM; + pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); + if (!pte) + return -ENOMEM; + set_pte(pte, pte_mkspecial(phys_pte_mm(mm, phys, __pgprot(TEST_PROT)))); + pte_unmap_unlock(pte, ptl); + return 0; +} + +/* Physical-range lookup only; coarse anonymous ownership is a separate step. */ +static void user64k_rmap_lookup_test(struct kunit *test) +{ + struct test_mm *ctx; + struct vm_area_struct *vma; + void *data; + unsigned long pfn, per_leaf = SZ_64K / PAGE_SIZE; + unsigned int mode, sync, i; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace leaf larger than native page"); + return; + } + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + data = test_alloc_granule(test, 17); + KUNIT_ASSERT_NOT_NULL(test, data); + pfn = virt_to_phys(data) >> PAGE_SHIFT; + ctx->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, ctx->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(ctx->mm, 16), 0); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = (1UL << 29) - SZ_64K; + vma->vm_end = vma->vm_start + 4 * SZ_64K; + vma->vm_pgoff = 4096; + page_folio(virt_to_page(data))->index = vma->vm_pgoff; + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, test_map_alias(ctx, i, vma->vm_start + i * SZ_64K, + virt_to_phys(data) + (i == 1 ? SZ_64K : 0)), 0); + + for (mode = 0; mode < 2; mode++) { + if (mode) { + for (i = 0; i < 3; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(ctx->mm, ctx->address[i], &ptl); + + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, swp_entry_to_pte(make_readable_migration_entry( + ctx->phys[i] >> PAGE_SHIFT))); + pte_unmap_unlock(pte, ptl); + } + } + for (sync = 0; sync < 2; sync++) { + unsigned int flags = (sync ? PVMW_SYNC : 0) | + (mode ? PVMW_MIGRATION : 0); + + for (i = 0; i < 2 * per_leaf; i++) { + struct page_vma_mapped_walk walk = { + .pfn = pfn + i, .nr_pages = 1, + /* Deliberately invalid pgoff: single-page callers may omit it. */ + .pgoff = 0, .vma = vma, + .address = vma->vm_start + i * PAGE_SIZE, + .flags = flags, + }; + unsigned int count = 0; + bool valid = true; + + while (page_vma_mapped_walk(&walk)) { + valid &= walk.pte && walk.address == + vma->vm_start + (i / per_leaf) * SZ_64K; + if (++count > 1) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 1U); +#ifdef CONFIG_MEMORY_FAILURE + if (!mode) + KUNIT_EXPECT_EQ(test, page_mapped_in_vma(pfn_to_page(pfn + i), vma), + vma->vm_start + i * PAGE_SIZE); +#endif + } + /* Two native pages straddle a coarse-leaf and PTE-table boundary. */ + { + struct page_vma_mapped_walk walk = { + .pfn = pfn + per_leaf - 1, .nr_pages = 2, + .pgoff = vma->vm_pgoff + per_leaf - 1, .vma = vma, + .address = vma->vm_start + SZ_64K - PAGE_SIZE, + .flags = flags, + }; + unsigned int count = 0; + bool valid = true; + + KUNIT_EXPECT_EQ(test, vma_address_end(&walk), vma->vm_start + 2 * SZ_64K); + while (page_vma_mapped_walk(&walk)) { + valid &= walk.address == vma->vm_start + count * SZ_64K; + if (++count > 2) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 2U); + KUNIT_EXPECT_TRUE(test, walk.flags & PVMW_PGTABLE_CROSSED); + } + for (i = 0; i < 2; i++) { + struct page_vma_mapped_walk walk = { + .pfn = i ? pfn + 2 * per_leaf : pfn - 1, + .nr_pages = 1, .vma = vma, + .address = vma->vm_start, .flags = flags, + }; + bool found = page_vma_mapped_walk(&walk); + + if (found) + page_vma_mapped_walk_done(&walk); + KUNIT_EXPECT_FALSE(test, found); + } + } + } +} + +static void user4k_rmap_boundary_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, true); + struct vm_area_struct *vma; + unsigned int i, count = 0; + struct page_vma_mapped_walk walk; + bool valid = true; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = SZ_2M - SZ_4K; + vma->vm_end = vma->vm_start + PAGE_SIZE; + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, test_map_alias(ctx, 4 + i, vma->vm_start + i * SZ_4K, + ctx->phys[i]), 0); + walk = (struct page_vma_mapped_walk) { + .pfn = ctx->phys[0] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = vma->vm_start, + .flags = PVMW_SYNC, + }; + while (page_vma_mapped_walk(&walk)) { + if (count >= 3 || walk.address != vma->vm_start + count * SZ_4K || + pte_phys_mm(ctx->mm, ptep_get(walk.pte)) != ctx->phys[count]) + valid = false; + if (++count > 3) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 3U); + KUNIT_EXPECT_TRUE(test, walk.flags & PVMW_PGTABLE_CROSSED); +} + +static void native_rmap_walk_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, false); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 4 * PAGE_SIZE; + for (i = 0; i < 4; i++) { + struct page_vma_mapped_walk walk = { + .pfn = ctx->phys[i] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = ctx->address[i], + .flags = PVMW_SYNC, + }; + unsigned int count = 0; + bool valid = true; + + while (page_vma_mapped_walk(&walk)) { + if (walk.address != ctx->address[i] || + pte_phys_mm(ctx->mm, ptep_get(walk.pte)) != ctx->phys[i]) + valid = false; + if (++count > 1) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, i != 2 ? 1U : 0U); +#ifdef CONFIG_MEMORY_FAILURE + page_folio(pfn_to_page(ctx->phys[i] >> PAGE_SHIFT))->index = i; + KUNIT_EXPECT_EQ(test, page_mapped_in_vma(pfn_to_page(ctx->phys[i] >> PAGE_SHIFT), vma), + i != 2 ? ctx->address[i] : (unsigned long)-EFAULT); +#endif + } +} + +static void user4k_vma_offsets_test(struct kunit *test) +{ + struct vm_area_struct *vma = kunit_kzalloc(test, sizeof(*vma), GFP_KERNEL); + unsigned int slot, length, step; + + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_start = (1UL << 48) - 8 * PAGE_SIZE; + for (slot = 0; slot < TEST_SLOTS; slot++) { + for (length = 1; length <= 8; length++) { + struct vm_page_offset initial = { ULONG_MAX - 3, slot * SZ_4K }; + struct vm_page_offset pos; + + vma->vm_end = vma->vm_start + length * SZ_4K; + vma_set_page_offset(vma, initial); + KUNIT_EXPECT_EQ(test, vma_last_pgoff(vma), + initial.index + (slot + length - 1) / TEST_SLOTS); + for (step = 0; step < length; step++) { + unsigned long addr = vma->vm_start + step * SZ_4K; + + pos = vma_page_offset_at(vma, addr); + KUNIT_EXPECT_EQ(test, pos.index, initial.index + (slot + step) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, pos.offset, ((slot + step) % TEST_SLOTS) * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, linear_page_index(vma, addr), pos.index); + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), addr); + } + pos = vma_page_offset_at(vma, vma->vm_end); + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + pos = initial; + pos.index--; + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + if (slot) { + pos = initial; + pos.offset--; + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + } + } + } +} + +static void user4k_vma_intervals_test(struct kunit *test) +{ + struct vm_area_struct *vmas = kunit_kcalloc(test, 3, sizeof(*vmas), GFP_KERNEL); + struct rb_root_cached root = RB_ROOT_CACHED; + struct vm_area_struct *vma; + unsigned int i, count = 0; + + KUNIT_ASSERT_NOT_NULL(test, vmas); + for (i = 0; i < 3; i++) { + vmas[i].vm_start = TEST_VA + i * SZ_8K; + vmas[i].vm_end = vmas[i].vm_start + SZ_4K; + vma_set_page_offset(&vmas[i], (struct vm_page_offset) { 77, i * SZ_4K }); + vma_interval_tree_insert(&vmas[i], &root); + } + vma_interval_tree_foreach(vma, &root, 77, 77) { + unsigned long address = vma_address_at_offset(vma, + (struct vm_page_offset) { 77, SZ_4K }); + + KUNIT_EXPECT_EQ(test, address, + vma == &vmas[1] ? vma->vm_start : (unsigned long)-EFAULT); + count++; + } + KUNIT_EXPECT_EQ(test, count, 3U); + KUNIT_EXPECT_PTR_EQ(test, vma_interval_tree_iter_first(&root, 76, 76), NULL); + KUNIT_EXPECT_PTR_EQ(test, vma_interval_tree_iter_first(&root, 78, 78), NULL); + for (i = 0; i < 3; i++) + vma_interval_tree_remove(&vmas[i], &root); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&root.rb_root)); +} + +static void user4k_vma_mm_free(void *arg) +{ + mmput(arg); +} + +/* These private mms have no concurrent VMA writers after dropping the lock. */ +static struct vm_area_struct *test_vma_lookup(struct mm_struct *mm, unsigned long addr) +{ + struct vm_area_struct *vma; + + mmap_read_lock(mm); + vma = vma_lookup(mm, addr); + mmap_read_unlock(mm); + return vma; +} + +static void user4k_vma_split_test(struct kunit *test) +{ + struct mm_struct *mm = mm_alloc(); + struct vm_area_struct *vma, *middle, *part; + unsigned long start = TEST_VA + SZ_4K; + vma_flags_t flags; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select4k(mm), 0); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Dummy vm_ops keep insert_vm_struct from replacing this file-like offset. */ + vma->vm_start = start; + vma->vm_end = start + 3 * SZ_4K; + vm_flags_init(vma, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_offset(vma, (struct vm_page_offset) { 77, 3 * SZ_4K }); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + if (ret) { + mmap_write_unlock(mm); + vm_area_free(vma); + KUNIT_FAIL(test, "insert_vm_struct: %d", ret); + return; + } + { + VMA_ITERATOR(vmi, mm, start); + + vma_iter_load(&vmi); + flags = vma->flags; + vma_flags_set(&flags, VMA_WRITE_BIT); + middle = vma_modify_flags(&vmi, vma, vma, start + SZ_4K, + start + 2 * SZ_4K, &flags); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, middle); + KUNIT_EXPECT_EQ(test, mm->map_count, 3); + for (i = 0; i < 3; i++) { + part = test_vma_lookup(mm, start + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, part); + KUNIT_EXPECT_EQ(test, part->vm_start, start + i * SZ_4K); + KUNIT_EXPECT_EQ(test, part->vm_end, start + (i + 1) * SZ_4K); + KUNIT_EXPECT_EQ(test, part->vm_pgoff, 77UL + (i + 3) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(part), ((i + 3) % TEST_SLOTS) * (unsigned int)SZ_4K); + } +} + +static struct mm_struct *test_vma_mm_granule(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm = mm_alloc(); + + if (!mm) + return NULL; + if (kunit_add_action_or_reset(test, user4k_vma_mm_free, mm)) + return NULL; + if (test_mm_select_granule(mm, shift)) + return NULL; + return mm; +} + +static struct mm_struct *test_vma_mm(struct kunit *test) +{ + return test_vma_mm_granule(test, 12); +} + +static struct vm_area_struct *test_vma_add(struct mm_struct *mm, unsigned long start, + unsigned long len, struct vm_page_offset pos, + vm_flags_t flags) +{ + struct vm_area_struct *vma = vm_area_alloc(mm); + int ret; + + if (!vma) + return ERR_PTR(-ENOMEM); + vma->vm_start = start; + vma->vm_end = start + len; + vm_flags_init(vma, flags | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_offset(vma, pos); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + mmap_write_unlock(mm); + if (ret) { + vm_area_free(vma); + return ERR_PTR(ret); + } + return vma; +} + +static void user4k_vma_merge_test(struct kunit *test) +{ + unsigned long start = TEST_VA + SZ_4K; + unsigned int scenario, i; + + for (scenario = 0; scenario < 4; scenario++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vmas[3], *merged; + struct vm_page_offset initial = { 77, 3 * SZ_4K }; + struct vm_page_offset expected; + unsigned long left = scenario >= 2 ? SZ_4K : 0; + unsigned long right = scenario == 1 || scenario == 3 ? 2 * SZ_4K : 3 * SZ_4K; + vma_flags_t flags; + + KUNIT_ASSERT_NOT_NULL(test, mm); + for (i = 0; i < 3; i++) { + struct vm_page_offset pos = vm_page_offset_add(initial, i * SZ_4K); + vm_flags_t bits = VM_READ; + + if (i == 1) + bits |= VM_WRITE; + if ((!i && scenario >= 2) || (i == 2 && scenario == 1)) + bits |= VM_EXEC; + /* Same native index, wrong byte offset: this must not merge. */ + if (i == 2 && scenario == 3) + pos.offset = 0; + vmas[i] = test_vma_add(mm, start + i * SZ_4K, SZ_4K, pos, bits); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vmas[i]); + } + flags = vmas[1]->flags; + vma_flags_clear(&flags, VMA_WRITE_BIT); + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, start + SZ_4K); + + vma_iter_load(&vmi); + merged = vma_modify_flags(&vmi, vmas[0], vmas[1], start + SZ_4K, + start + 2 * SZ_4K, &flags); + if (!IS_ERR(merged)) { + vma_start_write(merged); + merged->flags = flags; + } + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, merged); + expected = vm_page_offset_add(initial, left); + KUNIT_EXPECT_EQ(test, mm->map_count, scenario == 0 ? 1 : scenario == 3 ? 3 : 2); + KUNIT_EXPECT_EQ(test, merged->vm_start, start + left); + KUNIT_EXPECT_EQ(test, merged->vm_end, start + right); + KUNIT_EXPECT_EQ(test, merged->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(merged), expected.offset); + } +} + +static void user4k_vma_partial_merge_test(struct kunit *test) +{ + unsigned long start = TEST_VA; + unsigned int side; + + for (side = 0; side < 2; side++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_page_offset initial = { 77, 2 * SZ_4K }; + struct vm_area_struct *first, *second, *merged, *remaining; + unsigned long first_len = side ? SZ_4K : 2 * SZ_4K; + vma_flags_t flags; + struct vm_page_offset expected; + + KUNIT_ASSERT_NOT_NULL(test, mm); + first = test_vma_add(mm, start, first_len, initial, VM_READ | (side ? 0 : VM_WRITE)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, first); + second = test_vma_add(mm, start + first_len, 3 * SZ_4K - first_len, + vm_page_offset_add(initial, first_len), VM_READ | (side ? VM_WRITE : 0)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, second); + flags = side ? first->flags : second->flags; + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, start + SZ_4K); + + vma_iter_load(&vmi); + merged = vma_modify_flags(&vmi, first, side ? second : first, + start + SZ_4K, start + 2 * SZ_4K, &flags); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, merged); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + KUNIT_EXPECT_EQ(test, merged->vm_start, start + (side ? 0 : SZ_4K)); + KUNIT_EXPECT_EQ(test, merged->vm_end, start + (side ? 2 : 3) * SZ_4K); + expected = vm_page_offset_add(initial, side ? 0 : SZ_4K); + KUNIT_EXPECT_EQ(test, merged->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(merged), expected.offset); + remaining = test_vma_lookup(mm, start + (side ? 2 * SZ_4K : 0)); + KUNIT_ASSERT_NOT_NULL(test, remaining); + expected = vm_page_offset_add(initial, side ? 2 * SZ_4K : 0); + KUNIT_EXPECT_EQ(test, remaining->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(remaining), expected.offset); + } +} + +static void user4k_vma_copy_shrink_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *source, *dest; + struct vm_page_offset initial = { 77, 3 * SZ_4K }; + unsigned long start = TEST_VA + SZ_4K; + unsigned long target = TEST_VA + 16 * PAGE_SIZE + SZ_4K; + bool need_locks = false; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + source = test_vma_add(mm, start, PAGE_SIZE, initial, VM_READ); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, source); + for (i = 0; i < 3; i++) { + unsigned long addr = i == 2 ? target - SZ_4K : target + i * SZ_4K; + struct vm_page_offset pos = vm_page_offset_add(initial, i == 2 ? 0 : (i + 1) * SZ_4K); + + mmap_write_lock(mm); + dest = copy_vma(&source, addr, SZ_4K, pos, &need_locks); + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + KUNIT_EXPECT_EQ(test, dest->vm_start, i == 2 ? target - SZ_4K : target); + KUNIT_EXPECT_EQ(test, dest->vm_end, i ? target + 2 * SZ_4K : target + SZ_4K); + KUNIT_EXPECT_EQ(test, dest->vm_pgoff, 77UL + (i == 2 ? 3UL : 4UL) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(dest), ((i == 2 ? 3U : 4U) % TEST_SLOTS) * SZ_4K); + KUNIT_EXPECT_EQ(test, source->vm_pgoff, initial.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(source), initial.offset); + } + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, dest->vm_start); + struct vm_page_offset pos = vma_page_offset_at(dest, target); + + vma_iter_load(&vmi); + ret = vma_shrink(&vmi, dest, target, dest->vm_end, pos); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, dest->vm_start, target); + KUNIT_EXPECT_EQ(test, dest->vm_end, target + 2 * SZ_4K); + KUNIT_EXPECT_EQ(test, dest->vm_pgoff, 77UL + 4 / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(dest), (4 % TEST_SLOTS) * SZ_4K); +} + +struct anon_slot_fixture { + struct mm_struct *mm; + struct vm_area_struct *vma; + struct mm_subpage_pool *pool; + struct mm_subpage *slots[4]; + bool mapped[4]; +}; + +static void anon_slot_fixture_free(void *arg) +{ + struct anon_slot_fixture *f = arg; + struct mmu_gather tlb; + unsigned int i; + + if (f->pool) + mm_subpage_pool_close(f->pool); + if (f->mm) { + mmap_write_lock(f->mm); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, addr, &ptl); + + if (!pte) + continue; + ptep_get_and_clear_full(f->mm, addr, pte, 0); + if (f->mapped[i]) { + struct vm_area_struct *vma = vma_lookup(f->mm, addr); + + mm_subpage_remove_anon_rmap(f->slots[i], vma); + dec_mm_counter(f->mm, MM_ANONPAGES); + f->mapped[i] = false; + } + pte_unmap_unlock(pte, ptl); + } + tlb_gather_mmu_fullmm(&tlb, f->mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(f->mm); + } + /* No slot reference is released before the PTEs and TLBs are retired. */ + for (i = 0; i < 4; i++) + if (f->slots[i]) + mm_subpage_put(f->slots[i]); + if (f->pool) + mm_subpage_pool_put(f->pool); + if (f->mm) + mmput(f->mm); + memset(f, 0, sizeof(*f)); +} + +static struct anon_slot_fixture *anon_slot_fixture_create_from(struct kunit *test, + struct anon_slot_fixture *parent) +{ + struct anon_slot_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct vm_area_struct *vma; + struct folio *folio; + int ret; + + if (!f || kunit_add_action_or_reset(test, anon_slot_fixture_free, f)) + return NULL; + f->mm = mm_alloc(); + if (!f->mm || test_mm_select4k(f->mm)) + return NULL; + f->pool = mm_subpage_pool_create(GFP_KERNEL); + if (!f->pool) + return NULL; + folio = folio_alloc(GFP_KERNEL, 0); + if (!folio) + return NULL; + ret = mm_subpage_pool_add_folio(f->pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + return NULL; + } + vma = vm_area_alloc(f->mm); + if (!vma) + return NULL; + vma_set_anonymous(vma); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * PAGE_SIZE; + vm_flags_init(vma, VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_prot(vma); + if (parent) { + mmap_write_lock(parent->mm); + mmap_write_lock_nested(f->mm, SINGLE_DEPTH_NESTING); + vma_set_page_offset(vma, vma_page_offset_at(parent->vma, TEST_VA)); + ret = anon_vma_fork(vma, parent->vma); + } else { + mmap_write_lock(f->mm); + ret = 0; + } + if (!ret && parent) { + VMA_ITERATOR(vmi, f->mm, TEST_VA); + + vma_start_write(vma); + ret = vma_iter_store_gfp(&vmi, vma, GFP_KERNEL); + if (!ret) + f->mm->map_count++; + } else if (!ret) { + ret = insert_vm_struct(f->mm, vma); + } + if (!ret) { + f->vma = vma; + ret = anon_vma_prepare(vma); + } + if (!f->vma) + unlink_anon_vmas(vma); + mmap_write_unlock(f->mm); + if (parent) + mmap_write_unlock(parent->mm); + if (!f->vma) + vm_area_free(vma); + return ret ? NULL : f; +} + +static struct anon_slot_fixture *anon_slot_fixture_create(struct kunit *test) +{ + return anon_slot_fixture_create_from(test, NULL); +} + +static void user4k_anon_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct folio *folio = NULL; + unsigned int i; + vm_flags_t flags; + int ret, bad_offset = 0, bad_index = 0; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + struct test_mm tables = { .mm = f->mm }; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + f->slots[i] = slot; + folio = mm_subpage_folio(slot); + /* Allocate the table first; clear the test leaf before rmap attachment. */ + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, addr, mm_subpage_phys(slot)), 0); + mmap_read_lock(f->mm); + folio_lock(folio); + pte = test_mm_lookup(f->mm, addr, &ptl); + if (!pte) { + folio_unlock(folio); + mmap_read_unlock(f->mm); + KUNIT_FAIL(test, "missing allocated PTE"); + return; + } + pte_clear(f->mm, addr, pte); + if (!i) + bad_offset = mm_subpage_add_anon_rmap(slot, f->vma, addr + 1); + ret = mm_subpage_add_anon_rmap(slot, f->vma, addr); + if (!ret) { + f->mapped[i] = true; + inc_mm_counter(f->mm, MM_ANONPAGES); + set_pte_at(f->mm, addr, pte, phys_pte_mm(f->mm, mm_subpage_phys(slot), + __pgprot(TEST_PROT | PTE_RDONLY))); + if (!i) + bad_index = mm_subpage_add_anon_rmap(slot, f->vma, addr + PAGE_SIZE); + } + pte_unmap_unlock(pte, ptl); + folio_unlock(folio); + mmap_read_unlock(f->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)i + 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)i + 1); + } + KUNIT_EXPECT_EQ(test, bad_offset, -EINVAL); + KUNIT_EXPECT_EQ(test, bad_index, -EINVAL); + KUNIT_EXPECT_EQ(test, folio->index, TEST_VA >> PAGE_SHIFT); + KUNIT_EXPECT_FALSE(test, PageAnonExclusive(&folio->page)); + KUNIT_EXPECT_EQ(test, folio_expected_ref_count(folio), folio_ref_count(folio)); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 1); + for (i = 0; i < 4; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, TEST_VA + i * SZ_4K, &ptl); + bool young; + + KUNIT_ASSERT_NOT_NULL(test, pte); + young = pte_young(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_FALSE(test, young); + } + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); +} + +/* The fixture takes one existing slot reference, including on failure. */ +static int anon_slot_fixture_map(struct anon_slot_fixture *f, + unsigned int index, struct mm_subpage *slot) +{ + unsigned long addr = TEST_VA + index * SZ_4K; + struct test_mm tables = { .mm = f->mm }; + struct folio *folio = mm_subpage_folio(slot); + spinlock_t *ptl; + pte_t *pte; + int ret; + + f->slots[index] = slot; + ret = test_map_alias(&tables, 0, addr, mm_subpage_phys(slot)); + if (ret) + return ret; + mmap_read_lock(f->mm); + folio_lock(folio); + pte = test_mm_lookup(f->mm, addr, &ptl); + if (!pte) { + ret = -ENOMEM; + goto unlock; + } + pte_clear(f->mm, addr, pte); + ret = mm_subpage_add_anon_rmap(slot, f->vma, addr); + if (!ret) { + f->mapped[index] = true; + inc_mm_counter(f->mm, MM_ANONPAGES); + set_pte_at(f->mm, addr, pte, phys_pte_mm(f->mm, mm_subpage_phys(slot), + __pgprot(TEST_PROT | PTE_RDONLY))); + } + pte_unmap_unlock(pte, ptl); +unlock: + folio_unlock(folio); + mmap_read_unlock(f->mm); + return ret; +} + +static void user4k_anon_fork_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *parent = anon_slot_fixture_create(test); + struct anon_slot_fixture *child; + struct mm_subpage *slot; + struct folio *folio; + vm_flags_t flags; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, parent); + child = anon_slot_fixture_create_from(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + for (i = 0; i < 4; i++) { + slot = mm_subpage_alloc_at(parent->pool, i * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(parent, i, slot), 0); + mm_subpage_get(slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(child, i, slot), 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 2); + } + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 8); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 8); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + anon_slot_fixture_free(parent); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 4); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(child->slots[i]), 1); + /* The child mappings and root remain valid after the parent disappears. */ + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); +} + +static void user4k_nonlinear_rmap_reuse_test(struct kunit *test) +{ + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct anon_slot_fixture *c = anon_slot_fixture_create(test); + struct mm_subpage *slot; + struct folio *folio; + struct anon_vma_chain *avc; + phys_addr_t phys; + vm_flags_t flags; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, c); + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + folio = mm_subpage_folio(slot); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + phys = mm_subpage_phys(slot); +#ifdef CONFIG_FAILSLAB + { + unsigned int pending; + + mmap_read_lock(b->mm); + folio_lock(folio); + WRITE_ONCE(current->fail_nth, 1); + ret = mm_subpage_add_anon_rmap(slot, b->vma, TEST_VA + SZ_4K); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + /* Keep cleanup valid even if allocation unexpectedly succeeds. */ + if (!ret) + mm_subpage_remove_anon_rmap(slot, b->vma); + folio_unlock(folio); + mmap_read_unlock(b->mm); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); + } +#endif + /* Allocation failure left the slot available for the same operation. */ + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + anon_slot_fixture_free(b); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), phys); + + /* Give this empty VMA a different logical native index as well. */ + mmap_write_lock(c->mm); + anon_vma_lock_write(c->vma->anon_vma); + list_for_each_entry(avc, &c->vma->anon_vma_chain, same_vma) + anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); + c->vma->vm_pgoff += 9; + list_for_each_entry(avc, &c->vma->anon_vma_chain, same_vma) + anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); + anon_vma_unlock_write(c->vma->anon_vma); + mmap_write_unlock(c->mm); +#ifdef CONFIG_FAILSLAB + /* Retiring a previous root also needs failure-atomic bookkeeping. */ + mmap_read_lock(c->mm); + folio_lock(folio); + WRITE_ONCE(current->fail_nth, 1); + ret = mm_subpage_add_anon_rmap(slot, c->vma, TEST_VA + 2 * SZ_4K); + WRITE_ONCE(current->fail_nth, 0); + if (!ret) + mm_subpage_remove_anon_rmap(slot, c->vma); + folio_unlock(folio); + mmap_read_unlock(c->mm); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); +#endif + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(c, 2, slot), 0); + mmap_read_lock(c->mm); + KUNIT_EXPECT_EQ(test, page_address_in_vma(folio, &folio->page, c->vma), + TEST_VA + 2 * SZ_4K); + mmap_read_unlock(c->mm); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + /* The original root may disappear before the other slot's last mapping. */ + anon_slot_fixture_free(a); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(c->slots[2]), 1); +} + +static void user4k_dup_mmap_test(struct kunit *test) +{ + struct anon_slot_fixture *parent = anon_slot_fixture_create(test); + struct mm_struct *child; + struct folio *folio; + unsigned int i; + int ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, parent); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(parent->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(parent, i, slot), 0); + pte = test_mm_lookup(parent->mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte_at(parent->mm, addr, pte, + pte_mkwrite_novma(pte_mkdirty(ptep_get(pte)))); + pte_unmap_unlock(pte, ptl); + } + /* Split into four 4K VMAs plus a trailing hole, preserving COW flags. */ + mmap_write_lock(parent->mm); + for (i = 1; i < 4; i += 2) { + unsigned long addr = TEST_VA + i * SZ_4K; + VMA_ITERATOR(vmi, parent->mm, addr); + struct vm_area_struct *vma = vma_iter_load(&vmi); + vma_flags_t flags = vma->flags; + + vma_flags_set(&flags, VMA_DONTDUMP_BIT); + vma = vma_modify_flags(&vmi, vma, vma, addr, addr + SZ_4K, &flags); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + break; + } + } + mmap_write_unlock(parent->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, parent->mm->map_count, 5); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, child->map_count, 5); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 4UL); + folio = mm_subpage_folio(parent->slots[0]); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 8); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 8); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + struct vm_area_struct *vma = test_vma_lookup(child, addr); + unsigned int side; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(parent->slots[i]), 2); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->mm; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, addr, &ptl); + pte_t entry; + + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_FALSE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), + mm_subpage_phys(parent->slots[i])); + } + } + anon_slot_fixture_free(parent); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 4); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + /* KUnit's child-mm cleanup now exercises ordinary exit_mmap(). */ +} + +static void test_subpage_put(void *arg) +{ + mm_subpage_put(arg); +} + +static void user4k_anon_root_lifetime_test(struct kunit *test) +{ + struct anon_slot_fixture *old = anon_slot_fixture_create(test); + struct anon_slot_fixture *other = anon_slot_fixture_create(test); + struct mm_subpage *slot; + struct anon_vma *root; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, other); + slot = mm_subpage_alloc_at(old->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(old, 0, slot), 0); + folio = mm_subpage_folio(slot); + root = old->vma->anon_vma->root; + mm_subpage_get(slot); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slot), 0); + anon_slot_fixture_free(old); + /* Only the owner now pins the root; no VMA or PTE survives. */ + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + mm_subpage_get(slot); + ret = anon_slot_fixture_map(other, 0, slot); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); +} + +static void test_folio_put(void *arg) +{ + folio_put(arg); +} + +static void user4k_data_tlb_batch_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct folio *native = folio_alloc(GFP_KERNEL, 2); + struct mmu_gather tlb; + struct folio *backing; + unsigned int i; + bool full = false; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, native), 0); + for (i = 0; i < 4; i++) { + f->slots[i] = mm_subpage_alloc_at(f->pool, i * SZ_4K); + if (IS_ERR(f->slots[i])) { + f->slots[i] = NULL; + KUNIT_FAIL(test, "slot allocation failed"); + return; + } + } + backing = mm_subpage_folio(f->slots[0]); + memset(folio_address(backing), 0x5a, SZ_4K); + tlb_gather_mmu(&tlb, f->mm); + tlb_change_page_size(&tlb, MM_SUBPAGE_SIZE); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + for (i = 0; i < 64; i++) { + mm_subpage_get(f->slots[i % 4]); + if (__tlb_remove_subpage(&tlb, f->slots[i % 4])) { + full = true; + tlb_flush_mmu(&tlb); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + } + if (i == 12) { + /* Exercise the native count payload (3 << 2 has tag bit 2). */ + folio_ref_add(native, 3); + if (__tlb_remove_folio_pages(&tlb, &native->page, 3, false)) { + full = true; + tlb_flush_mmu(&tlb); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + } + } + } + mm_subpage_put(f->slots[0]); + f->slots[0] = NULL; + /* Without memory pressure these fit in the local plus one dynamic batch. */ + if (!full) { + struct mm_subpage *extra = mm_subpage_alloc_at(f->pool, 0); + + KUNIT_EXPECT_EQ(test, PTR_ERR(extra), -EAGAIN); + if (!IS_ERR(extra)) + mm_subpage_put(extra); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), 67); + KUNIT_EXPECT_EQ(test, folio_ref_count(native), 4); + KUNIT_EXPECT_GT(test, tlb.batch_count, 0U); + } + tlb_finish_mmu(&tlb); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), 3); + KUNIT_EXPECT_EQ(test, folio_ref_count(native), 1); + f->slots[0] = mm_subpage_alloc_at(f->pool, 0); + if (IS_ERR(f->slots[0])) { + f->slots[0] = NULL; + KUNIT_FAIL(test, "slot was not reusable after TLB completion"); + return; + } + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->slots[0]), backing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(backing), 0, SZ_4K), NULL); + for (i = 0; i < 4; i++) { + struct mm_subpage *extra = mm_subpage_alloc_at(f->pool, i * SZ_4K); + + KUNIT_EXPECT_EQ(test, PTR_ERR(extra), -EAGAIN); + if (!IS_ERR(extra)) + mm_subpage_put(extra); + } +} + +static void user4k_anon_zap_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct mm_subpage *slots[4]; + struct folio *folio; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + slots[i] = mm_subpage_alloc_at(f->pool, i * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slots[i]); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slots[i]), 0); + mm_subpage_get(slots[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slots[i]), 0); + } + folio = mm_subpage_folio(slots[0]); + mmap_read_lock(f->mm); + zap_vma_range(f->vma, TEST_VA + SZ_4K, SZ_4K); + f->mapped[1] = false; + f->slots[1] = NULL; + mmap_read_unlock(f->mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 3); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 7); + for (i = 0; i < 4; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, TEST_VA + i * SZ_4K, &ptl); + bool present; + + KUNIT_ASSERT_NOT_NULL(test, pte); + present = pte_present(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, present, i != 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), i == 1 ? 0 : 1); + } + /* Include both the existing hole and a trailing run of empty entries. */ + mmap_read_lock(f->mm); + zap_vma(f->vma); + for (i = 0; i < 4; i++) { + f->mapped[i] = false; + f->slots[i] = NULL; + } + mmap_read_unlock(f->mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm, MM_ANONPAGES), 0UL); +} + +static void user4k_anon_writable_slots_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte; + bool writable; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + pte = test_mm_lookup(f->mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte_at(f->mm, addr, pte, + pte_mkwrite_novma(pte_mkdirty(ptep_get(pte)))); + writable = pte_write(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, writable); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + } + KUNIT_EXPECT_EQ(test, folio_mapcount(mm_subpage_folio(f->slots[0])), 4); +} + +static struct vm_area_struct *test_fault_vma_in_mm(struct mm_struct *mm, + unsigned long start, unsigned long len) +{ + struct vm_area_struct *vma; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + int ret; + + if (!mm) + return NULL; + vma = vm_area_alloc(mm); + if (!vma) + return NULL; + vma_set_anonymous(vma); + vma->vm_start = start; + vma->vm_end = start + len; + vm_flags_init(vma, flags); + vma->vm_page_prot = vm_get_page_prot(flags); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + mmap_write_unlock(mm); + if (ret) { + vm_area_free(vma); + return NULL; + } + return vma; +} + +struct coarse_anon_fixture { + struct mm_struct *mm[2]; + bool attached[2]; + struct vm_area_struct *vma[2]; + struct folio *folio; + unsigned long mapped[2]; + unsigned int leaves; + bool pinned; +}; + +static void coarse_anon_fixture_free(void *ptr) +{ + struct coarse_anon_fixture *f = ptr; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + + if (f->pinned) { + unpin_user_page(folio_page(f->folio, nr - 1)); + f->pinned = false; + } + for (i = 0; i < 2; i++) { + unsigned int leaf; + + if (!f->mm[i]) + continue; + mmap_write_lock(f->mm[i]); + for (leaf = 0; leaf < (f->leaves ?: 1); leaf++) { + unsigned long addr = TEST_VA + leaf * SZ_64K; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm[i], addr, &ptl); + + if (pte) { + ptep_get_and_clear_full(f->mm[i], addr, pte, 0); + if (f->mapped[i] & BIT(leaf)) { + folio_remove_rmap_pte(f->folio, + folio_page(f->folio, leaf * nr), f->vma[i]); + dec_mm_counter(f->mm[i], MM_ANONPAGES); + } + pte_unmap_unlock(pte, ptl); + } + if (f->mapped[i] & BIT(leaf)) { + folio_put_refs(f->folio, nr); + f->mapped[i] &= ~BIT(leaf); + } + } + mmap_write_unlock(f->mm[i]); + if (f->vma[i] && !f->attached[i]) { + unlink_anon_vmas(f->vma[i]); + vm_area_free(f->vma[i]); + } + mmput(f->mm[i]); + f->vma[i] = NULL; + f->mm[i] = NULL; + } + if (f->folio) { + folio_put(f->folio); + f->folio = NULL; + } +} + +static void expect_coarse_anon_state(struct kunit *test, struct folio *folio, + unsigned int maps, bool exclusive) +{ + unsigned int i, nr = folio_nr_pages(folio); + + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(maps * nr)); + for (i = 0; i < nr; i++) { + struct page *page = folio_page(folio, i); + + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, (atomic_read(&page->_mapcount) + 1), (int)maps); + KUNIT_EXPECT_EQ(test, PageAnonExclusive(page), exclusive); + } +} + +static void user64k_anon_rmap_span_test(struct kunit *test) +{ + struct coarse_anon_fixture *f; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + pte_t entry, *pte; + spinlock_t *ptl; + int ret, dup_ret, share_ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous leaf"); + return; + } + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_anon_fixture_free, f), 0); + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 16 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + for (i = 0; i < 2; i++) { + struct test_mm tables = {}; + + f->mm[i] = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm[i]); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(f->mm[i], 16), 0); + if (!i) { + f->vma[i] = test_fault_vma_in_mm(f->mm[i], TEST_VA, SZ_64K); + f->attached[i] = !!f->vma[i]; + } else { + f->vma[i] = vm_area_alloc(f->mm[i]); + if (f->vma[i]) { + vma_set_anonymous(f->vma[i]); + f->vma[i]->vm_start = TEST_VA; + f->vma[i]->vm_end = TEST_VA + SZ_64K; + vm_flags_init(f->vma[i], f->vma[0]->vm_flags); + f->vma[i]->vm_page_prot = f->vma[0]->vm_page_prot; + vma_set_page_offset(f->vma[i], vma_page_offset_at(f->vma[0], TEST_VA)); + } + } + KUNIT_ASSERT_NOT_NULL(test, f->vma[i]); + tables.mm = f->mm[i]; + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, TEST_VA, PFN_PHYS(folio_pfn(f->folio))), 0); + pte = test_mm_lookup(f->mm[i], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + pte_clear(f->mm[i], TEST_VA, pte); + pte_unmap_unlock(pte, ptl); + } + mmap_write_lock(f->mm[0]); + ret = anon_vma_prepare(f->vma[0]); + mmap_write_unlock(f->mm[0]); + KUNIT_ASSERT_EQ(test, ret, 0); + mmap_write_lock(f->mm[0]); + mmap_write_lock_nested(f->mm[1], SINGLE_DEPTH_NESTING); + ret = anon_vma_fork(f->vma[1], f->vma[0]); + if (!ret) { + VMA_ITERATOR(vmi, f->mm[1], TEST_VA); + + vma_start_write(f->vma[1]); + ret = vma_iter_store_gfp(&vmi, f->vma[1], GFP_KERNEL); + if (!ret) { + f->mm[1]->map_count++; + f->attached[1] = true; + } + } + mmap_write_unlock(f->mm[1]); + mmap_write_unlock(f->mm[0]); + KUNIT_ASSERT_EQ(test, ret, 0); + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_new_anon_rmap(f->folio, f->vma[0], TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, nr); /* Fixture retains its original allocation ref. */ + inc_mm_counter(f->mm[0], MM_ANONPAGES); + f->mapped[0] = true; + entry = pte_mkwrite(pte_mkdirty(phys_pte_mm(f->mm[0], PFN_PHYS(folio_pfn(f->folio)), + f->vma[0]->vm_page_prot)), f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + expect_coarse_anon_state(test, f->folio, 1, true); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)nr + 1); + + mm_flags_set(MMF_HAS_PINNED, f->mm[0]); + ret = try_grab_folio(page_folio(folio_page(f->folio, nr - 1)), 1, FOLL_PIN); + f->pinned = !ret; + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + ptep_set_wrprotect(f->mm[0], TEST_VA, pte); + raw_write_seqcount_begin(&f->mm[0]->write_protect_seq); + dup_ret = folio_try_dup_anon_rmap_pte(f->folio, &f->folio->page, f->vma[1], f->vma[0]); + raw_write_seqcount_end(&f->mm[0]->write_protect_seq); + entry = ptep_get_and_clear(f->mm[0], TEST_VA, pte); + share_ret = folio_try_share_anon_rmap_pte(f->folio, &f->folio->page, f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, dup_ret, -EBUSY); + KUNIT_EXPECT_EQ(test, share_ret, -EBUSY); + expect_coarse_anon_state(test, f->folio, 1, true); + unpin_user_page(folio_page(f->folio, nr - 1)); + f->pinned = false; + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get_and_clear(f->mm[0], TEST_VA, pte); + share_ret = folio_try_share_anon_rmap_pte(f->folio, &f->folio->page, f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, pte_wrprotect(entry), 1); + pte_unmap_unlock(pte, ptl); + KUNIT_ASSERT_EQ(test, share_ret, 0); + expect_coarse_anon_state(test, f->folio, 1, false); + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + raw_write_seqcount_begin(&f->mm[0]->write_protect_seq); + dup_ret = folio_try_dup_anon_rmap_pte(f->folio, &f->folio->page, f->vma[1], f->vma[0]); + raw_write_seqcount_end(&f->mm[0]->write_protect_seq); + if (!dup_ret) { + folio_ref_add(f->folio, nr); + inc_mm_counter(f->mm[1], MM_ANONPAGES); + f->mapped[1] = true; + } + pte_unmap_unlock(pte, ptl); + KUNIT_ASSERT_EQ(test, dup_ret, 0); + pte = test_mm_lookup(f->mm[1], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_ptes(f->mm[1], TEST_VA, pte, pte_wrprotect(entry), 1); + pte_unmap_unlock(pte, ptl); + expect_coarse_anon_state(test, f->folio, 2, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)(2 * nr + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[0], MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[1], MM_ANONPAGES), 1L); + + /* Exercise ordinary unmapping of each shared coarse leaf. */ + for (i = 0; i < 2; i++) { + mmap_read_lock(f->mm[i]); + zap_vma_range(f->vma[i], TEST_VA, SZ_64K); + f->mapped[i] = 0; + mmap_read_unlock(f->mm[i]); + expect_coarse_anon_state(test, f->folio, 1 - i, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)((1 - i) * nr + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[i], MM_ANONPAGES), 0L); + } + + /* Keep an observation ref while cleanup releases the empty address spaces. */ + { + struct folio *folio = f->folio; + + folio_get(folio); + coarse_anon_fixture_free(f); + expect_coarse_anon_state(test, folio, 0, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); + } +} + +static void user64k_anon_zap_batch_test(struct kunit *test) +{ + struct coarse_anon_fixture *f; + struct test_mm tables = {}; + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned int i, nr = SZ_64K / PAGE_SIZE, batch; + pte_t entry, *pte; + spinlock_t *ptl; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous leaves"); + return; + } + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_anon_fixture_free, f), 0); + f->leaves = 2; + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 17 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + mm = f->mm[0] = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + /* Partial unmap can enqueue this folio on the memcg split queue. */ + KUNIT_ASSERT_EQ(test, mem_cgroup_charge(f->folio, mm, GFP_KERNEL), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, 16), 0); + vma = f->vma[0] = test_fault_vma_in_mm(mm, TEST_VA, 2 * SZ_64K); + f->attached[0] = !!vma; + KUNIT_ASSERT_NOT_NULL(test, vma); + tables.mm = mm; + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, i, addr, + PFN_PHYS(folio_pfn(f->folio)) + i * SZ_64K), 0); + pte = test_mm_lookup(mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + pte_clear(mm, addr, pte); + pte_unmap_unlock(pte, ptl); + } + mmap_write_lock(mm); + ret = anon_vma_prepare(vma); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_new_anon_rmap(f->folio, vma, TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, 2 * nr); + add_mm_counter(mm, MM_ANONPAGES, 2); + f->mapped[0] = BIT(0) | BIT(1); + entry = pte_mkwrite(pte_mkdirty(phys_pte_mm(mm, + PFN_PHYS(folio_pfn(f->folio)), vma->vm_page_prot)), vma); + set_ptes(mm, TEST_VA, pte, entry, 2); + batch = folio_subpage_pte_batch(vma, f->folio, pte, entry, 2, 0); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, batch, 2U); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)(2 * nr)); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)(2 * nr + 1)); + + /* The reference walker must consume logical PTEs and native mapcounts. */ + { + vm_flags_t flags; + bool young; + + folio_lock(f->folio); + ret = folio_referenced(f->folio, 1, NULL, &flags); + folio_unlock(f->folio); + KUNIT_EXPECT_EQ(test, ret, 1); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + young = pte_young(ptep_get(pte)) || pte_young(ptep_get(pte + 1)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_FALSE(test, young); + folio_lock(f->folio); + ret = folio_referenced(f->folio, 1, NULL, &flags); + folio_unlock(f->folio); + KUNIT_EXPECT_EQ(test, ret, 0); + } + + /* Removing one logical leaf must leave the adjacent leaf's native span. */ + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, SZ_64K); + f->mapped[0] &= ~BIT(0); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)nr); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)nr + 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1L); + for (i = 0; i < 2 * nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(f->folio, i)->_mapcount), + i < nr ? -1 : 0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + ret = pte_none(ptep_get(pte)) && pte_present(ptep_get(pte + 1)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, ret); + + /* Restore the first leaf, then remove both through one batched walk. */ + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_anon_rmap_pte(f->folio, &f->folio->page, vma, + TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, nr); + inc_mm_counter(mm, MM_ANONPAGES); + f->mapped[0] |= BIT(0); + set_ptes(mm, TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)(2 * nr)); + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, 2 * SZ_64K); + f->mapped[0] = 0; + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + for (i = 0; i < 2 * nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(f->folio, i)->_mapcount), -1); +} + +static void user64k_file_rmap_span_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse nonanonymous leaf"); + return; + } + mm = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * SZ_64K; + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 16 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, folio); + /* Private metadata-only fixture: no file-cache promotion or fault is implied. */ + folio_lock(folio); + folio_add_file_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount) + 1, 1); + folio_dup_file_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(2 * nr)); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount) + 1, 2); + folio_remove_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + folio_remove_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount), -1); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); +} + +static struct vm_area_struct *test_fault_vma(struct kunit *test, + unsigned long start, unsigned long len) +{ + return test_fault_vma_in_mm(test_vma_mm(test), start, len); +} + +static struct vm_area_struct *test_fault_vma_granule(struct kunit *test, + unsigned int shift, unsigned long start, unsigned long len) +{ + return test_fault_vma_in_mm(test_vma_mm_granule(test, shift), start, len); +} + +static void test_anon_write_fault_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned int shape; + + /* Swap/reclaim integration is a later gate, before any ABI activation. */ + KUNIT_ASSERT_EQ(test, total_swap_pages, 0UL); + for (shape = 0; shape < 2; shape++) { + unsigned long start = TEST_VA + (shape ? size : 0); + unsigned long len = shape ? size : 2 * PAGE_SIZE; + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, start, len); + struct folio *backing = NULL; + unsigned int i, mapped = 0; + vm_fault_t ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + ret = handle_mm_fault(vma, TEST_VA + size + 3, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + /* Remove transient LRU-add references before checking ownership. */ + lru_add_drain_all(); + for (i = 0; i < (PAGE_SIZE >> shift); i++) { + unsigned long addr = TEST_VA + i * size; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(vma->vm_mm, addr, &ptl); + struct mm_subpage *slot = NULL; + bool expected = !shape || i == 1; + bool present, writable = false, zero = false, identity = false; + + KUNIT_ASSERT_NOT_NULL(test, pte); + present = pte_present(ptep_get(pte)); + if (present) { + slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, ptep_get(pte))); + writable = pte_write(ptep_get(pte)); + if (slot) { + struct folio *folio = mm_subpage_folio(slot); + + if (!backing) + backing = folio; + identity = folio == backing && + mm_subpage_offset(slot) == i * size; + zero = !memchr_inv(folio_address(folio) + mm_subpage_offset(slot), + 0, size); + mm_subpage_put(slot); + } + mapped++; + } + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, present, expected); + if (expected) { + KUNIT_EXPECT_TRUE(test, writable); + KUNIT_EXPECT_TRUE(test, identity); + KUNIT_EXPECT_TRUE(test, zero); + } + } + KUNIT_ASSERT_NOT_NULL(test, backing); + KUNIT_EXPECT_EQ(test, mapped, shape ? 1U : (PAGE_SIZE >> shift)); + KUNIT_EXPECT_EQ(test, folio_mapcount(backing), (int)mapped); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), (int)mapped); + KUNIT_EXPECT_TRUE(test, folio_test_lru(backing)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), + (unsigned long)mapped); + } +} + +static void user4k_anon_write_fault_test(struct kunit *test) +{ + test_anon_write_fault_granule(test, 12); +} + +static void user16k_anon_write_fault_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_anon_write_fault_granule(test, 14); +} + +static void user4k_swap_encoding_test(struct kunit *test) +{ + const unsigned long offsets[] = { 1, 2, (1UL << __SWP_OFFSET_BITS) - 2 }; + const unsigned int types[] = { 0, 1, __SWP_TYPE_MASK }; + unsigned int i, j, slot, flags; + + for (i = 0; i < ARRAY_SIZE(offsets); i++) { + for (j = 0; j < ARRAY_SIZE(types); j++) { + for (slot = 0; slot < PAGE_SIZE / SZ_4K; slot++) { + for (flags = 0; flags < 4; flags++) { + softleaf_t entry = swp_entry(types[j], offsets[i]); + pte_t pte = softleaf_to_pte(entry), moved; + + pte = pte_swp_set_subpage_offset(pte, slot * SZ_4K); + if (flags & 1) + pte = pte_swp_mkexclusive(pte); + if (flags & 2) + pte = pte_swp_mkuffd_wp(pte); + KUNIT_EXPECT_FALSE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), slot * (unsigned int)SZ_4K); + moved = pte_move_swp_offset(pte, 1); + KUNIT_EXPECT_EQ(test, swp_offset(softleaf_from_pte(moved)), offsets[i] + 1); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(moved), slot * (unsigned int)SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_same(pte_move_swp_offset(moved, -1), pte)); + moved = pte_swp_set_subpage_offset(pte, ((slot + 1) % (PAGE_SIZE / SZ_4K)) * SZ_4K); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(moved).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(moved), ((slot + 1) % (PAGE_SIZE / SZ_4K)) * (unsigned int)SZ_4K); + } + } + } + } +} + +static pte_t test_fault_entry(struct kunit *test, struct mm_struct *mm, unsigned long addr) +{ + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, addr, &ptl); + pte_t entry = __pte(0); + + KUNIT_EXPECT_NOT_NULL(test, pte); + if (pte) { + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + } + return entry; +} + + +static void test_file_put(void *file) +{ + fput(file); +} + +static struct file *test_cache_file(struct kunit *test, unsigned long size, bool fill) +{ + struct file *file = shmem_file_setup("user4k-file-test", size, EMPTY_VMA_FLAGS); + unsigned char *buf; + loff_t pos = 0; + unsigned int i; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, test_file_put, file)) + return ERR_PTR(-ENOMEM); + if (!fill) + return file; + buf = kunit_kmalloc(test, size, GFP_KERNEL); + if (!buf) + return ERR_PTR(-ENOMEM); + for (i = 0; i < size; i++) + buf[i] = 0x31 + (i / SZ_4K); + if (kernel_write(file, buf, size, &pos) != size) + return ERR_PTR(-EIO); + return file; +} + +static struct vm_area_struct *test_file_vma_granule(struct kunit *test, struct file *file, + unsigned int shift, bool shared, unsigned long start, unsigned long len, + unsigned long offset) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + int ret; + + mm = test_vma_mm_granule(test, shift); + if (!mm) + return NULL; + vma = vm_area_alloc(mm); + if (!vma) + return NULL; + vma->vm_start = start; + vma->vm_end = start + len; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = offset >> PAGE_SHIFT, .offset = offset & ~PAGE_MASK, + }); + if (shared) + flags |= VM_SHARED | VM_MAYSHARE; + vm_flags_init(vma, flags); + vma->vm_file = get_file(file); + vma->vm_page_prot = vm_get_page_prot(flags); + mmap_write_lock(mm); + ret = vfs_mmap(file, vma); + if (!ret) { + vma_set_page_prot(vma); + ret = insert_vm_struct(mm, vma); + } + mmap_write_unlock(mm); + if (ret) { + fput(vma->vm_file); + vm_area_free(vma); + return NULL; + } + return vma; +} + +static struct vm_area_struct *test_file_vma(struct kunit *test, struct file *file, + bool native, bool shared, unsigned long start, unsigned long len, + unsigned long offset) +{ + return test_file_vma_granule(test, file, native ? PAGE_SHIFT : 12, + shared, start, len, offset); +} + +static void user4k_mmap_descriptor_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct vm_area_desc desc; + + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + PAGE_SIZE; + vma_set_page_offset(vma, (struct vm_page_offset) { 7, SZ_4K }); + compat_set_desc_from_vma(&desc, NULL, vma); + KUNIT_EXPECT_EQ(test, desc.pgoff, 7UL); + KUNIT_EXPECT_EQ(test, desc.subpage_offset, (unsigned int)SZ_4K); + desc.pgoff = 19; + desc.subpage_offset = 3 * SZ_4K; + KUNIT_ASSERT_EQ(test, __compat_vma_mmap(&desc, vma), 0); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, TEST_VA), + (struct vm_page_offset) { 19, 3 * SZ_4K })); +} + +static void user4k_mmap_region_offsets_test(struct kunit *test) +{ + struct file *file; + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_SHARED | VM_MAYSHARE | VM_NORESERVE; + unsigned long result[4]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 8 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + /* Only the full native-sized lengths are used before accounting conversion. */ + kthread_use_mm(mm); + mmap_write_lock(mm); + result[0] = mmap_region(file, start + PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 1, SZ_4K }, NULL); + result[1] = mmap_region(file, start + 2 * PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 2, SZ_4K }, NULL); + result[2] = mmap_region(file, start, PAGE_SIZE, flags, + (struct vm_page_offset) { 0, SZ_4K }, NULL); + /* Adjacent virtual bytes with a different file offset must not merge. */ + result[3] = mmap_region(file, start + 3 * PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 3, 2 * SZ_4K }, NULL); + mmap_write_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, result[0], start + PAGE_SIZE); + KUNIT_ASSERT_EQ(test, result[1], start + 2 * PAGE_SIZE); + KUNIT_ASSERT_EQ(test, result[2], start); + KUNIT_ASSERT_EQ(test, result[3], start + 3 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_start, start); + KUNIT_EXPECT_EQ(test, vma->vm_end, start + 3 * PAGE_SIZE); + for (i = 0; i < 12; i++) + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, start + i * SZ_4K), + (struct vm_page_offset) { (i + 1) / TEST_SLOTS, ((i + 1) % TEST_SLOTS) * SZ_4K })); + vma = test_vma_lookup(mm, start + 3 * PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 2U * SZ_4K); +} + +#ifdef CONFIG_PROC_FS +/* Exercise the actual status/statm producers without publishing a fake task. */ +extern void task_mem(struct seq_file *m, struct mm_struct *mm); +extern unsigned long task_vsize(struct mm_struct *mm); +extern unsigned long task_statm(struct mm_struct *mm, unsigned long *shared, + unsigned long *text, unsigned long *data, unsigned long *resident); + +static void expect_status_kb(struct kunit *test, struct seq_file *seq, + const char *name, unsigned long expected) +{ + const char *field = strstr(seq->buf, name); + unsigned long value = ULONG_MAX; + + KUNIT_ASSERT_NOT_NULL(test, field); + KUNIT_EXPECT_EQ(test, sscanf(field + strlen(name), "%lu", &value), 1); + KUNIT_EXPECT_EQ_MSG(test, value, expected, "%s", name); +} + +static void user4k_accounting_readers_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test); + struct mm_struct *native = mm_alloc(); + struct seq_file seq = {}; + unsigned long shared, text, data, resident, size, maxrss = 0; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + seq.size = 2048; + seq.buf = kunit_kzalloc(test, seq.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, seq.buf); + /* Same logical counts, deliberately not multiples of a native page. */ + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? native : small; + unsigned long kb = i ? PAGE_SIZE / 1024 : 4; + + mm->total_vm = 11; + mm->hiwater_vm = 13; + mm->data_vm = 5; + mm->stack_vm = 2; + mm->exec_vm = 3; + mm->locked_vm = mm_pages_to_account(mm, 1); + atomic64_set(&mm->pinned_vm, 2); + mm->hiwater_rss = 10; + mm->start_code = TEST_VA + mm_page_size(mm); + mm->end_code = mm->start_code + 2 * mm_page_size(mm); + percpu_counter_set(&mm->rss_stat[MM_ANONPAGES], 5); + percpu_counter_set(&mm->rss_stat[MM_FILEPAGES], 2); + percpu_counter_set(&mm->rss_stat[MM_SHMEMPAGES], 1); + percpu_counter_set(&mm->rss_stat[MM_SWAPENTS], 3); + seq.count = 0; + task_mem(&seq, mm); + KUNIT_EXPECT_FALSE(test, seq_has_overflowed(&seq)); + seq.buf[min(seq.count, seq.size - 1)] = '\0'; + expect_status_kb(test, &seq, "VmPeak:", 13 * kb); + expect_status_kb(test, &seq, "VmSize:", 11 * kb); + expect_status_kb(test, &seq, "VmLck:", kb); + expect_status_kb(test, &seq, "VmPin:", 2 * PAGE_SIZE / 1024); + expect_status_kb(test, &seq, "VmHWM:", 10 * kb); + expect_status_kb(test, &seq, "VmRSS:", 8 * kb); + expect_status_kb(test, &seq, "RssAnon:", 5 * kb); + expect_status_kb(test, &seq, "RssFile:", 2 * kb); + expect_status_kb(test, &seq, "RssShmem:", kb); + expect_status_kb(test, &seq, "VmData:", 5 * kb); + expect_status_kb(test, &seq, "VmStk:", 2 * kb); + expect_status_kb(test, &seq, "VmExe:", 2 * kb); + expect_status_kb(test, &seq, "VmLib:", kb); + expect_status_kb(test, &seq, "VmSwap:", 3 * kb); + KUNIT_EXPECT_EQ(test, task_vsize(mm), 11 * kb * 1024); + { + struct vm_area_struct vma = { .vm_mm = mm }; + + KUNIT_EXPECT_EQ(test, vma_kernel_pagesize(&vma), kb * 1024); + KUNIT_EXPECT_EQ(test, vma_mmu_pagesize(&vma), kb * 1024); + } + } + /* Maxima must compare bytes, not incompatible 4K/16K counts. */ + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, 40UL); + setmax_mm_hiwater_rss(&maxrss, native); + KUNIT_EXPECT_EQ(test, maxrss, 10UL * PAGE_SIZE / 1024); + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, 10UL * PAGE_SIZE / 1024); + small->hiwater_rss = 10 * TEST_SLOTS + 1; + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, (10UL * TEST_SLOTS + 1) * 4); + + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 0), 0UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 1), 1UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 5), DIV_ROUND_UP(5UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(native, small, 5), 5UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(small, native, 5), DIV_ROUND_UP(5UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(small, small, 5), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(native, native, 5), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_shift(small, 5, MM_ACCOUNT_SHIFT), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_shift(native, 5, MM_ACCOUNT_SHIFT), 5UL * TEST_SLOTS); + + /* A kernel reader defaults to native units; partial pages round up. */ + size = task_statm(small, &shared, &text, &data, &resident); + KUNIT_EXPECT_EQ(test, size, DIV_ROUND_UP(11UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, shared, 1UL); + KUNIT_EXPECT_EQ(test, text, 1UL); + KUNIT_EXPECT_EQ(test, data, DIV_ROUND_UP(7UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, resident, DIV_ROUND_UP(8UL, TEST_SLOTS)); + if (user4k_test_enabled || IS_ENABLED(CONFIG_KASAN)) { + kthread_use_mm(small); + size = task_statm(native, &shared, &text, &data, &resident); + kthread_unuse_mm(small); + KUNIT_EXPECT_EQ(test, size, 11UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, shared, 3UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, text, 2UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, data, 7UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, resident, 8UL * TEST_SLOTS); + } + /* Synthetic counters have no page references; restore before mmput. */ + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? native : small; + unsigned int counter; + + for (counter = 0; counter < NR_MM_COUNTERS; counter++) + percpu_counter_set(&mm->rss_stat[counter], 0); + mm->total_vm = mm->data_vm = mm->stack_vm = mm->exec_vm = 0; + mm->locked_vm = 0; + atomic64_set(&mm->pinned_vm, 0); + } +} +#endif + +static void mixed_locked_account_units_test(struct kunit *test) +{ + unsigned int shift; + unsigned long native_units = PAGE_SIZE >> MM_ACCOUNT_SHIFT; + + for (shift = 12; shift <= 16; shift += 2) { + struct mm_struct *mm = mm_alloc(); + unsigned long size = 1UL << shift; + unsigned long base, limit; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + if (shift != PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shift), 0); + base = mm_pages_to_account(mm, 1); + mmap_write_lock(mm); + /* Model one locked VMA leaf and two independently charged native pins. */ + mm->locked_vm = base; + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base + native_units); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size + PAGE_SIZE); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(mm), + DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE)); + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base + 2 * native_units); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size + PAGE_SIZE); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size); + + /* A refused charge must leave the exact pre-existing amount intact. */ + limit = task_rlimit(current, RLIMIT_MEMLOCK) >> MM_ACCOUNT_SHIFT; + mm->locked_vm = limit; + ret = __account_locked_vm(mm, 1, true, current, false); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, limit); + mm->locked_vm = ULONG_MAX - native_units + 1; + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, ULONG_MAX - native_units + 1); + mm->locked_vm = 0; + if (native_units > 1) { + ret = __account_locked_vm(mm, + (ULONG_MAX >> (PAGE_SHIFT - MM_ACCOUNT_SHIFT)) + 1, + true, current, true); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + } + mmap_write_unlock(mm); + } +} + +/* Prior shmem fixtures can refund global commit from deferred fput work. */ +static s64 test_commit_baseline(struct kunit *test) +{ + s64 before = percpu_counter_sum(&vm_committed_as), after; + + /* No mmap, file, or filesystem locks are held at these call sites. */ + flush_delayed_fput(); + after = percpu_counter_sum(&vm_committed_as); + if (after != before) + kunit_info(test, "drained deferred file refunds: commit delta %lld", after - before); + return after; +} + +static void user64k_commit_accounting_test(struct kunit *test) +{ + struct mm_struct *mm; + s64 before; + unsigned int i; + unsigned long native_per_leaf; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace granule larger than native"); + return; + } + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, 16), 0); + native_per_leaf = SZ_64K / PAGE_SIZE; + before = test_commit_baseline(test); + mmap_write_lock(mm); + for (i = 0; i < 9; i++) { + ret = mm_account_memory(mm, mm, 1); + KUNIT_EXPECT_EQ(test, ret, 0); + if (ret) + break; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i + 1); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)((i + 1) * native_per_leaf)); + } + /* Multiplication overflow must not mutate either reservation counter. */ + ret = mm_account_memory(mm, mm, LONG_MAX / native_per_leaf + 1); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + mm_acct_memory(mm, 3); + i += 3; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + while (i) { + mm_unacct_memory(mm, 1); + i--; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + } + mmap_write_unlock(mm); +} + +#ifdef CONFIG_PAGE_TABLE_CHECK +static void expect_coarse_file_counts(struct kunit *test, void *data, + unsigned int phase) +{ + unsigned int offset; + + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_4K) { + int anon = -1, file = -1, expected; + int ret = page_table_check_get_counts(virt_to_page(data + offset), + offset & ~PAGE_MASK, &anon, &file); + + /* Phase0: both coarse leaves;1:plusalias;2:firstclear;3:bothclear;4:allclear. */ + expected = phase < 2 || (phase == 2 && offset >= SZ_64K); + if (phase >= 1 && phase <= 3 && offset == 2 * SZ_64K - SZ_4K) + expected++; + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, anon, 0); + KUNIT_EXPECT_EQ_MSG(test, file, expected, "phase=%u offset=%u", phase, offset); + } +} +#endif + +static void user64k_page_table_check_test(struct kunit *test) +{ +#ifdef CONFIG_PAGE_TABLE_CHECK + struct mm_struct *coarse, *small; + void *data; + pte_t ptes[2] = {}, alias = __pte(0), first, last; + int anon, file; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace leaf larger than native page"); + return; + } + data = test_alloc_granule(test, 17); + KUNIT_ASSERT_NOT_NULL(test, data); + if (page_table_check_get_counts(virt_to_page(data), 0, &anon, &file)) { + kunit_skip(test, "requires page-table-check enabled at boot"); + return; + } + coarse = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, coarse), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(coarse, 16), 0); + small = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, small), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(small, 12), 0); + first = phys_pte_mm(coarse, virt_to_phys(data), __pgprot(TEST_PROT)); + last = phys_pte_mm(small, virt_to_phys(data) + 2 * SZ_64K - SZ_4K, + __pgprot(TEST_PROT)); + + /* Exercise checker hooks directly; no actual coarse file fault is claimed. */ + page_table_check_ptes_set(coarse, TEST_VA, ptes, first, 2); + ptes[0] = first; + ptes[1] = pte_advance_pfn_mm(coarse, first, 1); + expect_coarse_file_counts(test, data, 0); + page_table_check_ptes_set(small, TEST_VA, &alias, last, 1); + alias = last; + expect_coarse_file_counts(test, data, 1); + page_table_check_pte_clear(coarse, TEST_VA, ptes[0]); + ptes[0] = __pte(0); + expect_coarse_file_counts(test, data, 2); + page_table_check_pte_clear(coarse, TEST_VA + SZ_64K, ptes[1]); + ptes[1] = __pte(0); + expect_coarse_file_counts(test, data, 3); + page_table_check_pte_clear(small, TEST_VA, alias); + expect_coarse_file_counts(test, data, 4); +#else + kunit_skip(test, "requires CONFIG_PAGE_TABLE_CHECK"); +#endif +} + +static void user4k_commit_accounting_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + s64 before; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + mmap_write_lock(mm); + for (i = 0; i < 9; i++) { + ret = mm_account_memory(mm, mm, 1); + KUNIT_EXPECT_EQ(test, ret, 0); + if (ret) + break; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i + 1); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)DIV_ROUND_UP(i + 1, TEST_SLOTS)); + } + if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) { + unsigned long old = mm->committed_user_pages; + s64 committed = percpu_counter_sum(&vm_committed_as); + + ret = mm_account_memory(mm, mm, (totalram_pages() + total_swap_pages + 1) * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, old); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as), committed); + } + /* Native physical-page callers must charge four user granules per page. */ + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, (unsigned long)TEST_SLOTS); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + + /* Refund one granule at a time: splitting must not multiply refunds. */ + while (i) { + mm_unacct_memory(mm, 1); + i--; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)DIV_ROUND_UP(i, TEST_SLOTS)); + } + mmap_write_unlock(mm); +} + +/* Prefaulting must stop at each process-page boundary, not native pages. */ +static void user_granule_prefault_test(struct kunit *test) +{ + unsigned int shift; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + for (shift = 12; shift <= PAGE_SHIFT; shift += 2) { + struct mm_struct *mm = test_vma_mm_granule(test, shift); + unsigned long size = 1UL << shift, start = TEST_VA; + unsigned long flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long mapped[3]; + size_t left[3][4] = {}; + int phase, ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped[0] = vm_mmap(NULL, start, size, PROT_READ | PROT_WRITE, flags, 0); + mapped[2] = vm_mmap(NULL, start + 2 * size, size, + PROT_READ | PROT_WRITE, flags, 0); + mapped[1] = start + size; + if (mapped[0] == start && mapped[2] == start + 2 * size) { + for (phase = 0; phase < 3; phase++) { + char __user *addr = (char __user *)(start + 17); + size_t len = 3 * size - 17; + + /* Missing middle, then read-only, then fully writable. */ + if (phase) { + mapped[1] = vm_mmap(NULL, start + size, size, + PROT_READ | (phase == 2 ? PROT_WRITE : 0), + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + if (mapped[1] != start + size) + break; + } + left[phase][0] = fault_in_safe_writeable(addr, len); + left[phase][1] = fault_in_readable(addr, len); + left[phase][2] = fault_in_writeable(addr, len); + left[phase][3] = fault_in_subpage_writeable(addr, len); + } + ret = vm_munmap(start, 3 * size); + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped[0], start); + KUNIT_ASSERT_EQ(test, mapped[1], start + size); + KUNIT_ASSERT_EQ(test, mapped[2], start + 2 * size); + KUNIT_EXPECT_EQ(test, ret, 0); + for (phase = 0; phase < 3; phase++) { + KUNIT_EXPECT_EQ(test, left[phase][0], phase < 2 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][1], phase == 0 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][2], phase < 2 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][3], phase < 2 ? 2 * size : 0UL); + } + } +} + +static void user4k_mmap_accounting_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K, mapped; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + s64 before; + unsigned int i; + int ret = 0; + vm_fault_t fault = 0; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + kthread_use_mm(mm); + mmap_write_lock(mm); + mapped = mmap_region(NULL, start, 3 * SZ_4K, flags, + (struct vm_page_offset) { start >> PAGE_SHIFT, SZ_4K }, NULL); + mmap_write_unlock(mm); + if (!IS_ERR_VALUE(mapped)) { + mmap_read_lock(mm); + vma = vma_lookup(mm, start); + fault = handle_mm_fault(vma, start, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->data_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, 1LL); + /* Remove the middle first, then the two neighbors. */ + for (i = 0; i < 3; i++) { + unsigned int slot = i == 0 ? 1 : i == 1 ? 0 : 2; + VMA_ITERATOR(vmi, mm, start + slot * SZ_4K); + + kthread_use_mm(mm); + mmap_write_lock(mm); + ret = do_vmi_munmap(&vmi, mm, start + slot * SZ_4K, SZ_4K, NULL, false); + mmap_write_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 2UL - i); + KUNIT_EXPECT_EQ(test, mm->data_vm, 2UL - i); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 2UL - i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, i == 2 ? 0LL : 1LL); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); +} + +static vm_fault_t test_file_fault(struct vm_area_struct *vma, unsigned long addr, bool write) +{ + vm_fault_t ret; + + mmap_read_lock(vma->vm_mm); + ret = handle_mm_fault(vma, addr, FAULT_FLAG_REMOTE | + (write ? FAULT_FLAG_WRITE : 0), NULL); + mmap_read_unlock(vma->vm_mm); + return ret; +} + +static void user4k_vm_mmap_test(struct kunit *test) +{ + struct mm_struct *mm; + struct file *file; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + unsigned long anon_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long file_flags = MAP_SHARED | MAP_FIXED_NOREPLACE; + unsigned long result[7]; + int unmap[4]; + pte_t pte; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + mm->mmap_base = TEST_VA + 2 * PAGE_SIZE + SZ_4K; + mm_flags_clear(MMF_TOPDOWN, mm); + kthread_use_mm(mm); + result[0] = vm_mmap(NULL, start, SZ_4K, PROT_READ | PROT_WRITE, anon_flags, 0); + result[1] = vm_mmap(file, start + 2 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE, + file_flags, 3 * SZ_4K); + result[2] = vm_mmap(file, start + 3 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE, + file_flags, 4 * SZ_4K); + result[3] = vm_mmap(NULL, start, SZ_4K, PROT_READ, anon_flags, 0); + result[4] = vm_mmap(NULL, start + 17, SZ_4K, PROT_READ, anon_flags, 0); + result[5] = vm_mmap(file, start + SZ_4K, SZ_4K, PROT_READ, file_flags, SZ_4K + 1); + result[6] = vm_mmap(NULL, 0, SZ_4K + 1, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, result[0], start); + KUNIT_ASSERT_EQ(test, result[1], start + 2 * SZ_4K); + KUNIT_ASSERT_EQ(test, result[2], start + 3 * SZ_4K); + KUNIT_EXPECT_EQ(test, result[3], (unsigned long)-EEXIST); + KUNIT_EXPECT_EQ(test, result[4], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, result[5], (unsigned long)-EINVAL); + KUNIT_ASSERT_EQ(test, result[6], mm->mmap_base); + KUNIT_EXPECT_EQ(test, mm->total_vm, 5UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start, true) & VM_FAULT_ERROR, 0U); + vma = test_vma_lookup(mm, start + 2 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_end - vma->vm_start, 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 3U * SZ_4K); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + 2 * SZ_4K, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, mm, start + 2 * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(mm, pte)), (u8)0x34); + pte = test_fault_entry(test, mm, start + 3 * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(mm, pte)), (u8)0x35); + kthread_use_mm(mm); + unmap[0] = vm_munmap(start + 2 * SZ_4K, 1); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, unmap[0], 0); + vma = test_vma_lookup(mm, start + 3 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_pgoff, 4UL / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), (4U % TEST_SLOTS) * SZ_4K); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + kthread_use_mm(mm); + unmap[1] = vm_munmap(start, SZ_4K); + unmap[2] = vm_munmap(start + 3 * SZ_4K, SZ_4K); + unmap[3] = vm_munmap(result[6], 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, unmap[1], 0); + KUNIT_EXPECT_EQ(test, unmap[2], 0); + KUNIT_EXPECT_EQ(test, unmap[3], 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 0UL); + KUNIT_EXPECT_EQ(test, mm->map_count, 0); +} + +static void test_mlock_drain_all(struct kunit *test, struct folio *folio) +{ + unsigned int before, after; + bool counted; + + lru_add_drain(); + counted = folio_test_unevictable(folio); + before = counted ? folio->mlock_count : 0; + /* A sleeping fixture can enqueue mlock work on more than one CPU. */ + lru_add_drain_all(); + after = folio_test_unevictable(folio) ? folio->mlock_count : 0; + if (counted && before != after) + kunit_info(test, "remote mlock drain changed count %u -> %u", before, after); +} + +static void user4k_mlock_counts_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct mm_subpage *slot; + struct folio *folio; + unsigned long addr; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE | MAP_LOCKED, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, test_fault_entry(test, mm, TEST_VA))); + KUNIT_ASSERT_NOT_NULL(test, slot); + folio = mm_subpage_folio(slot); + folio_get(folio); + mm_subpage_put(slot); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm->locked_vm, (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + int ret; + + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA + i * SZ_4K, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + /* mlock_count shares storage with the evictable LRU links. */ + if (i != TEST_SLOTS - 1) + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS - 1 - i); + KUNIT_EXPECT_EQ(test, mm->locked_vm, TEST_SLOTS - 1UL - i); + KUNIT_EXPECT_EQ(test, folio_test_mlocked(folio), i != TEST_SLOTS - 1); + } +} + +static void test_mlock_enable_lru_cache(void *unused) +{ + lru_cache_enable(); +} + +static void user4k_mlock_shared_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *folio; + unsigned long addr; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + if (native) + mmput(native); + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + /* Exact counts require synchronous LRU/lock accounting. Normal deferred + * batches may legitimately miss locks while the folio is off LRU; + * user4k_mlock_repair_test checks that reclaim repairs that case. + */ + lru_cache_disable(); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_mlock_enable_lru_cache, NULL), 0); + kthread_use_mm(small); + addr = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + kthread_use_mm(native); + addr = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(native); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS + 1); + kthread_use_mm(small); + ret = vm_munmap(TEST_VA + SZ_4K, SZ_4K); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS); + kthread_use_mm(native); + ret = vm_munmap(TEST_VA, PAGE_SIZE); + kthread_unuse_mm(native); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS - 1); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + kthread_use_mm(small); + ret = vm_munmap(TEST_VA, PAGE_SIZE); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); +} + +static void test_mlock_putback(void *folio) +{ + folio_putback_lru(folio); +} + +static void user4k_mlock_repair_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode, walker; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + unsigned long size = 1UL << shifts[mode]; + + for (walker = 0; walker < 2; walker++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *folio; + unsigned long addr, offset; + vm_flags_t flags = 0; + int maps, ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_folio_put, folio), 0); + lru_add_drain_all(); + KUNIT_ASSERT_TRUE(test, folio_isolate_lru(folio)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_mlock_putback, folio), 0); + /* Two aliases leave a locked mapping even after native unmap. + * Force deferred mlocks to run while the folio is isolated. + */ + for (offset = 0; offset < 2 * PAGE_SIZE; offset += PAGE_SIZE) { + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA + offset, PAGE_SIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + offset); + } + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_lru(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + kunit_release_action(test, test_mlock_putback, folio); + lru_add_drain_all(); + KUNIT_ASSERT_TRUE(test, folio_test_unevictable(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, 0U); + + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); + maps = folio_mapcount(folio); + KUNIT_EXPECT_EQ(test, maps, (int)(2 * PAGE_SIZE / size) - 1); + folio_lock(folio); + if (walker) + ret = folio_referenced(folio, 1, NULL, &flags); + else + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + lru_add_drain_all(); + if (walker) { + KUNIT_EXPECT_GE(test, ret, 0); + KUNIT_EXPECT_TRUE(test, flags & VM_LOCKED); + } + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), maps); + for (offset = size; offset < 2 * PAGE_SIZE; offset += size) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + offset); + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_PTR_EQ(test, + page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(mm, pte)))), + folio); + } + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, 2 * PAGE_SIZE); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + kunit_info(test, "%uK %s repairs deferred mlock without unmapping", + 1U << (shifts[mode] - 10), walker ? "referenced" : "unmap"); + } + } +} + +asmlinkage long __arm64_sys_mremap(const struct pt_regs *regs); + +static unsigned long test_mremap(unsigned long old, unsigned long old_len, + unsigned long new_len, unsigned long flags, unsigned long new) +{ + struct pt_regs regs = { .regs = { old, old_len, new_len, flags, new } }; + + return __arm64_sys_mremap(®s); +} + +static int test_iova_bitmap_fill(struct iova_bitmap *bitmap, unsigned long iova, + size_t length, void *opaque) +{ + unsigned long unit = *(unsigned long *)opaque; + + iova_bitmap_set(bitmap, iova + 3 * unit, length - 10 * unit); + /* A second write advances backwards into an already populated range. */ + iova_bitmap_set(bitmap, iova, 5 * unit); + return 0; +} + +/* Real IOMMUFD lifecycle with a translation-recording IOMMU, no physical DMA. */ +struct test_fragment_domain { + struct iommu_domain domain; + struct xarray translations; + unsigned long fail_after; + unsigned long largest_map; + unsigned long mapped; + unsigned long map_calls; +}; + +struct test_fragment_iopt { + struct io_pagetable table; + struct test_fragment_domain domains[2]; + struct kunit *test; +}; + +static int test_fragment_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t phys, size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + unsigned long length = pgsize * pgcount, done; + int rc; + + mock->map_calls++; + mock->largest_map = max(mock->largest_map, length); + for (done = 0; done < length; done += SZ_4K) { + if (mock->fail_after && mock->mapped >= mock->fail_after) + return -ENOMEM; + rc = xa_insert(&mock->translations, (iova + done) >> 12, + xa_mk_value((phys + done) >> 12), gfp); + if (rc) + return rc; + *mapped += SZ_4K; + mock->mapped += SZ_4K; + } + return 0; +} + +static size_t test_fragment_unmap(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + unsigned long length = pgsize * pgcount, done; + + for (done = 0; done < length; done += SZ_4K) { + if (!xa_erase(&mock->translations, (iova + done) >> 12)) + break; + mock->mapped -= SZ_4K; + } + return done; +} + +static phys_addr_t test_fragment_translate(struct iommu_domain *domain, dma_addr_t iova) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + void *entry = xa_load(&mock->translations, iova >> 12); + + return entry ? ((phys_addr_t)xa_to_value(entry) << 12) + (iova & (SZ_4K - 1)) : 0; +} + +static void test_fragment_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) +{ +} + +static const struct iommu_domain_ops test_fragment_domain_ops = { + .map_pages = test_fragment_map, + .unmap_pages = test_fragment_unmap, + .iova_to_phys = test_fragment_translate, + .iotlb_sync = test_fragment_sync, +}; + +static void test_fragment_iopt_free(void *data) +{ + struct test_fragment_iopt *fixture = data; + struct iommu_domain *domain; + unsigned long unmapped, index; + unsigned int i; + + iopt_unmap_all(&fixture->table, &unmapped); + while ((domain = xa_find(&fixture->table.domains, &(unsigned long){0}, + ULONG_MAX, XA_PRESENT))) + iopt_table_remove_domain(&fixture->table, domain); + iopt_destroy_table(&fixture->table); + for (i = 0; i < ARRAY_SIZE(fixture->domains); i++) { + struct test_fragment_domain *mock = &fixture->domains[i]; + void *entry; + + KUNIT_EXPECT_TRUE(fixture->test, xa_empty(&mock->translations)); + xa_for_each(&mock->translations, index, entry) + xa_erase(&mock->translations, index); + xa_destroy(&mock->translations); + } +} + +static struct test_fragment_iopt *test_fragment_iopt_alloc(struct kunit *test) +{ + struct test_fragment_iopt *fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + unsigned int i; + + if (!fixture) + return NULL; + fixture->test = test; + iopt_init_table(&fixture->table); + for (i = 0; i < ARRAY_SIZE(fixture->domains); i++) { + struct test_fragment_domain *mock = &fixture->domains[i]; + + xa_init(&mock->translations); + mock->domain.ops = &test_fragment_domain_ops; + mock->domain.type = IOMMU_DOMAIN_UNMANAGED; + mock->domain.pgsize_bitmap = SZ_4K; + mock->domain.geometry.aperture_end = ULONG_MAX; + } + if (kunit_add_action_or_reset(test, test_fragment_iopt_free, fixture)) + return NULL; + return fixture; +} + +static int test_fragment_map_user(struct test_fragment_iopt *fixture, + struct mm_struct *mm, unsigned long va, unsigned long iova, unsigned long length) +{ + struct iommufd_ctx ctx = {}; + int rc; + + kthread_use_mm(mm); + rc = iopt_map_user_pages(&ctx, &fixture->table, &iova, (void __user *)va, + length, IOMMU_READ | IOMMU_WRITE, 0); + kthread_unuse_mm(mm); + return rc; +} + +static void user4k_iommufd_lifetime_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14, 16 }, mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + for (mode = 0; mode < ARRAY_SIZE(shifts); mode++) { + unsigned long size = 1UL << shifts[mode]; + unsigned long len = 2 * max(size, PAGE_SIZE); + unsigned long dest = TEST_VA + 32 * PAGE_SIZE + size; + unsigned long iova = SZ_1M, alias = SZ_16M, addr, moved, missed, unmapped; + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct iopt_pages *pages; + struct iopt_area *area; + unsigned char *buffer; + unsigned long off, pinned; + LIST_HEAD(copy); + int rc; + + if (mode && shifts[mode] == PAGE_SHIFT) + continue; + mm = test_vma_mm_granule(test, shifts[mode]); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + buffer = kunit_kmalloc(test, len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + memset(buffer, 0x31, len); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, buffer, len); + moved = missed ? 0 : test_mremap(addr, len, len, MREMAP_FIXED | MREMAP_MAYMOVE, dest); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, dest); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, dest, iova, len), 0); + pinned = atomic64_read(&mm->pinned_vm); + KUNIT_EXPECT_EQ(test, pinned, len / PAGE_SIZE); + KUNIT_EXPECT_GE(test, fixture->domains[0].largest_map, PAGE_SIZE); + for (off = 0; off < len; off += size) { + pte_t pte = test_fault_entry(test, mm, dest + off); + phys_addr_t actual = iommu_iova_to_phys(&fixture->domains[0].domain, iova + off); + + KUNIT_EXPECT_EQ(test, actual, pte_phys_mm(mm, pte)); + } + KUNIT_ASSERT_EQ(test, iopt_get_pages(&fixture->table, iova + size, len - size, ©), 0); + rc = iopt_map_pages(&fixture->table, ©, len - size, &alias, IOMMU_READ | IOMMU_WRITE, 0); + iopt_free_pages_list(©); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), pinned); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), pinned); + + /* Replace the original VA: mappings and copied IOAS must retain old pins. */ + kthread_use_mm(mm); + addr = vm_mmap(NULL, dest, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + memset(buffer, 0x5a, len); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)dest, buffer, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, missed, 0UL); + down_read(&fixture->table.iova_rwsem); + area = iopt_area_iter_first(&fixture->table, iova, iova); + pages = area->pages; + /* Force the domain-backed reader for the native control too. */ + memset(buffer, 0x77, len); + rc = iopt_pages_rw_access(pages, 0, buffer, len, IOMMUFD_ACCESS_RW_WRITE); + up_read(&fixture->table.iova_rwsem); + KUNIT_ASSERT_EQ(test, rc, 0); + for (off = 0; off < len; off += SZ_4K) { + phys_addr_t phys = iommu_iova_to_phys(&fixture->domains[1].domain, iova + off); + void *ptr = kmap_local_page(pfn_to_page(PHYS_PFN(phys))); + bool correct = !memchr_inv(ptr + offset_in_page(phys), 0x77, SZ_4K); + + kunmap_local(ptr); + KUNIT_EXPECT_TRUE(test, correct); + } + kthread_use_mm(mm); + missed = copy_from_user(buffer, (void __user *)dest, len); + /* Local CPU access retains the existing copy_from_user fast path. */ + rc = iopt_pages_rw_access(pages, 0, buffer, len, 0); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0x5a, len), NULL); + KUNIT_ASSERT_EQ(test, iopt_unmap_iova(&fixture->table, iova, len, &unmapped), 0); + KUNIT_EXPECT_EQ(test, unmapped, len); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), (len - (size < PAGE_SIZE ? 0 : size)) / PAGE_SIZE); + iopt_table_remove_domain(&fixture->table, &fixture->domains[0].domain); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, len - size); + KUNIT_ASSERT_EQ(test, iopt_unmap_iova(&fixture->table, alias, len - size, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOMMUFD %luK: relocated PA, batching, copy, two domains, VA replacement and unpin passed", size / SZ_1K); + } +} + +static void user4k_iommufd_rollback_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + unsigned long addr, iova = SZ_1M, size = SZ_4K; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 3 * size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + rc = vm_munmap(TEST_VA + size, size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, addr, iova, 3 * size), -EFAULT); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA + size, size, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + size); + KUNIT_EXPECT_LT(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA + size, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + size); + fixture->domains[1].fail_after = size; + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), -ENOMEM); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + fixture->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), 0); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 3 * size); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 3 * size); + kunit_release_action(test, test_fragment_iopt_free, fixture); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void user4k_iommufd_granule_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + unsigned long addr, unmapped, cut = SZ_1M + PAGE_SIZE - 1; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + fixture->domains[0].domain.pgsize_bitmap = PAGE_SIZE; + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 3 * PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + /* The device cannot cover a misaligned fragment by widening the mapping. */ + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA + SZ_4K, SZ_1M, PAGE_SIZE), -EINVAL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, SZ_4K), -EINVAL); + /* Full contiguous native spans still work, including legacy area splitting. */ + KUNIT_ASSERT_EQ(test, iopt_disable_large_pages(&fixture->table), 0); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, 2 * PAGE_SIZE), 0); + KUNIT_EXPECT_EQ(test, fixture->domains[0].largest_map, PAGE_SIZE); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 2LL); + KUNIT_ASSERT_EQ(test, iopt_cut_iova(&fixture->table, &cut, 1), 0); + rc = iopt_unmap_iova(&fixture->table, SZ_1M, PAGE_SIZE, &unmapped); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + /* A closed IOVA interval ending at ULONG_MAX must not wrap at map/unmap. */ + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, + ULONG_MAX - 2 * PAGE_SIZE + 1, 2 * PAGE_SIZE), 0); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, unmapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void user4k_iommufd_shared_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct file *file; + struct folio *folio; + unsigned long addr, unmapped, iova = SZ_1M; + unsigned char *buffer; + loff_t pos = 0; + phys_addr_t phys; + void *ptr; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + file = test_cache_file(test, 2 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + buffer = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA, 2 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 2 * SZ_4K), 0); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + folio_lock(folio); + folio_clear_dirty_for_io(folio); + folio_unlock(folio); + phys = iommu_iova_to_phys(&fixture->domains[0].domain, iova); + KUNIT_EXPECT_EQ(test, phys, PFN_PHYS(folio_pfn(folio)) + SZ_4K); + ptr = kmap_local_folio(folio, SZ_4K); + memset(ptr, 0x93, 2 * SZ_4K); + kunmap_local(ptr); + kthread_use_mm(mm); + rc = vm_munmap(TEST_VA, 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, rc, 0); + /* A failed second-domain attachment must retain the first domain's pins. */ + fixture->domains[1].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), -ENOMEM); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 2 * SZ_4K); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(folio)); + fixture->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_ASSERT_EQ(test, kernel_read(file, buffer, PAGE_SIZE, &pos), (ssize_t)PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + SZ_4K, 0x93, 2 * SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 3 * SZ_4K, 0, PAGE_SIZE - 3 * SZ_4K), NULL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void test_iommufd_exit_account(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct iopt_pages *pages; + struct iopt_area *area; + unsigned long addr, unmapped; + LIST_HEAD(keep); + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 2 * PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, 2 * PAGE_SIZE), 0); + down_read(&fixture->table.iova_rwsem); + area = iopt_area_iter_first(&fixture->table, SZ_1M, SZ_1M); + pages = area->pages; + mutex_lock(&pages->mutex); + /* Select VFIO-compatible accounting before the lazy range acquires pins. */ + pages->account_mode = IOPT_PAGES_ACCOUNT_MM; + mutex_unlock(&pages->mutex); + up_read(&fixture->table.iova_rwsem); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 2LL); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 2 * (PAGE_SIZE >> MM_ACCOUNT_SHIFT)); + /* Keep the provider, and thus mm_count, after mm_users reaches zero. */ + KUNIT_ASSERT_EQ(test, iopt_get_pages(&fixture->table, SZ_1M, 2 * PAGE_SIZE, &keep), 0); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, atomic_read(&mm->mm_users), 0); + KUNIT_EXPECT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_EXPECT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, pages->npinned, 0UL); + KUNIT_EXPECT_EQ(test, pages->last_npinned, 0UL); + iopt_free_pages_list(&keep); + kunit_release_action(test, test_fragment_iopt_free, fixture); +} + +static void user4k_iommufd_exit_account_test(struct kunit *test) +{ + test_iommufd_exit_account(test, PAGE_SHIFT); + test_iommufd_exit_account(test, 12); +} + +static void user4k_iova_bitmap_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 4 * size; + unsigned long old = TEST_VA, dest = TEST_VA + 32 * PAGE_SIZE + size; + unsigned long start = dest + size - sizeof(u64), bytes = 2 * size; + unsigned long unit = SZ_4K, iova = 1UL << 32, addr, moved; + unsigned char *actual, *expected; + struct iova_bitmap *bitmap; + unsigned long missed; + int ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + actual = kunit_kzalloc(test, len, GFP_KERNEL); + expected = kunit_kzalloc(test, len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, actual); + KUNIT_ASSERT_NOT_NULL(test, expected); + kthread_use_mm(mm); + addr = vm_mmap(NULL, old, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, actual, len); + moved = missed ? 0 : test_mremap(old, len, len, + MREMAP_FIXED | MREMAP_MAYMOVE, dest); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, old); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_EQ(test, moved, dest); + /* VA and PA offsets differ after moving by one alternative leaf. */ + if (size < PAGE_SIZE) { + pte_t pte = test_fault_entry(test, mm, dest); + + KUNIT_EXPECT_NE(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + dest & ~PAGE_MASK); + } + kthread_use_mm(mm); + bitmap = iova_bitmap_alloc(iova, bytes * 8 * unit, unit, + (u64 __user *)start); + if (!IS_ERR(bitmap)) { + ret = iova_bitmap_for_each(bitmap, &unit, test_iova_bitmap_fill); + iova_bitmap_free(bitmap); + } else { + ret = PTR_ERR(bitmap); + } + missed = copy_from_user(actual, (void __user *)dest, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + memset(expected + size - sizeof(u64), 0xff, bytes); + expected[size - sizeof(u64) + bytes - 1] = 1; + KUNIT_EXPECT_EQ(test, memcmp(actual, expected, len), 0); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap relocated %luK user leaves checked", size / SZ_1K); + } +} + +static void test_iova_free(void *bitmap) +{ + iova_bitmap_free(bitmap); +} + +static void test_iova_kvfree(void *buffer) +{ + kvfree(buffer); +} + +static void user4k_iova_bitmap_window_test(struct kunit *test) +{ + struct mm_struct *mm; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + unsigned long bytes = (capacity + 3) * SZ_4K, len = bytes + 2 * SZ_4K; + unsigned long start = TEST_VA + SZ_4K - sizeof(u64), unit = SZ_4K; + unsigned long addr, missed; + unsigned char *actual; + struct iova_bitmap *bitmap; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + actual = kvzalloc(len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, actual); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_kvfree, actual), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + bitmap = iova_bitmap_alloc(0, bytes * 8 * unit, unit, (u64 __user *)start); + ret = IS_ERR(bitmap) ? PTR_ERR(bitmap) : + iova_bitmap_for_each(bitmap, &unit, test_iova_bitmap_fill); + if (!IS_ERR(bitmap)) + iova_bitmap_free(bitmap); + missed = IS_ERR_VALUE(addr) ? len : copy_from_user(actual, (void __user *)addr, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual, 0, SZ_4K - sizeof(u64)), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual + SZ_4K - sizeof(u64), 0xff, bytes - 1), NULL); + KUNIT_EXPECT_EQ(test, actual[SZ_4K - sizeof(u64) + bytes - 1], (unsigned char)1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual + SZ_4K - sizeof(u64) + bytes, + 0, SZ_4K + sizeof(u64)), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap crossed %lu-record window and revisited first byte", capacity); +} + +static void user4k_iova_bitmap_retained_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 2 * size; + struct file *file = test_cache_file(test, 4 * PAGE_SIZE, false); + unsigned char *buffer = kunit_kzalloc(test, len, GFP_KERNEL); + unsigned long addr, replacement = 0, missed = 0; + struct iova_bitmap *bitmap; + struct folio *folio; + loff_t pos = size; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_NOT_NULL(test, buffer); + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, size); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, + (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) + iova_bitmap_set(bitmap, 3 * SZ_4K, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_free, bitmap), 0); + folio = filemap_get_folio(file->f_mapping, size >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, len); + if (!ret) { + replacement = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + if (!IS_ERR_VALUE(replacement)) { + memset(buffer, 0x5a, len); + missed = copy_to_user((void __user *)replacement, buffer, len); + } + } + /* The cached pins still refer to the old file, not the new VMA. */ + iova_bitmap_set(bitmap, 9 * SZ_4K, 2 * SZ_4K); + if (!ret && !IS_ERR_VALUE(replacement)) + missed |= copy_from_user(buffer, (void __user *)replacement, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, replacement, TEST_VA); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0x5a, len), NULL); + kunit_release_action(test, test_iova_free, bitmap); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_ASSERT_EQ(test, kernel_read(file, buffer, len, &pos), (ssize_t)len); + KUNIT_EXPECT_EQ(test, buffer[0], (unsigned char)8); + KUNIT_EXPECT_EQ(test, buffer[1], (unsigned char)6); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 2, 0, len - 2), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap %luK shared backing retained across unmap/replacement", size / SZ_1K); + } +} + +static long test_mprotect(unsigned long start, unsigned long len, unsigned long prot); + +static void user4k_iova_bitmap_partial_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 3 * size; + unsigned char *buffer = kunit_kzalloc(test, len, GFP_KERNEL); + unsigned long addr, refill, missed; + struct iova_bitmap *bitmap; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, buffer); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, buffer, len); + ret = vm_munmap(TEST_VA + size, size); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + missed |= copy_from_user(buffer, (void __user *)TEST_VA, size); + missed |= copy_from_user(buffer + 2 * size, (void __user *)(TEST_VA + 2 * size), size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_free, bitmap), 0); + /* Preserve the existing void API: pin failure stops at the valid prefix. */ + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0xff, size), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 2 * size, 0, size), NULL); + kthread_use_mm(mm); + refill = vm_mmap(NULL, TEST_VA + size, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + missed = copy_from_user(buffer, (void __user *)TEST_VA, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, refill, TEST_VA + size); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0xff, len), NULL); + kunit_release_action(test, test_iova_free, bitmap); + memset(buffer, 0, len); + kthread_use_mm(mm); + missed = copy_to_user((void __user *)TEST_VA, buffer, len); + ret = test_mprotect(TEST_VA, len, PROT_READ); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) { + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + iova_bitmap_free(bitmap); + } + missed |= copy_from_user(buffer, (void __user *)TEST_VA, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, len), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap %luK partial pin, retry and read-only checks passed", size / SZ_1K); + } +} + +static u64 test_table_descriptor(struct mm_struct *mm, unsigned long addr, bool pud) +{ + pgd_t *pgdp = pgd_offset(mm, addr); + p4d_t *p4dp = p4d_offset_mm(mm, pgdp, addr); + pud_t *pudp = pud_offset_mm(mm, p4dp, addr); + + return pud ? pud_val(*pudp) : pmd_val(*pmd_offset_mm(mm, pudp, addr)); +} + +static void user4k_mremap_move_test(struct kunit *test) +{ + static const struct { unsigned long old, new, len; unsigned int level; } cases[] = { + { TEST_VA + SZ_4K, 3 * SZ_2M + 2 * SZ_4K, 3 * SZ_4K, 0 }, + { SZ_2M - SZ_4K, SZ_1G - 2 * SZ_4K, 3 * SZ_4K, 0 }, + { SZ_1G - SZ_4K, BIT_ULL(39) - 2 * SZ_4K, 3 * SZ_4K, 0 }, + { 4 * SZ_2M, 8 * SZ_2M, SZ_2M, 1 }, + { 2UL * SZ_1G, 4UL * SZ_1G, SZ_1G, 2 }, + }; + unsigned int shape; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + for (shape = 0; shape < ARRAY_SIZE(cases); shape++) { + struct mm_struct *mm = test_vma_mm(test); + unsigned long old = cases[shape].old, new = cases[shape].new; + unsigned long len = cases[shape].len; + unsigned long offsets[] = { 0, SZ_4K, len - SZ_4K }; + unsigned long result, warm, moved, vm_before, commit_before, rss_before; + unsigned long flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long old_left[3], new_left[3]; + phys_addr_t phys[3]; + u8 readback[3], value = 0x79, byte; + u64 table = 0; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (cases[shape].level) + flags |= MAP_NORESERVE; + kthread_use_mm(mm); + warm = vm_mmap(NULL, SZ_1M, SZ_4K, PROT_READ | PROT_WRITE, flags, 0); + result = vm_mmap(NULL, old, len, PROT_READ | PROT_WRITE, flags, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, warm, SZ_1M); + KUNIT_ASSERT_EQ(test, result, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, warm, &value, 1, FOLL_WRITE), 1); + for (i = 0; i < 3; i++) { + value = 0x80 + i; + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + offsets[i], &value, 1, FOLL_WRITE), 1); + phys[i] = pte_phys_mm(mm, test_fault_entry(test, mm, old + offsets[i])); + } + if (cases[shape].level) { + table = test_table_descriptor(mm, old, cases[shape].level == 2); + /* Warm-up consumed the first fragment; the moved table is packed. */ + KUNIT_EXPECT_NE(test, table & (3UL * SZ_4K), 0ULL); + } + vm_before = mm->total_vm; + commit_before = mm->committed_user_pages; + rss_before = get_mm_rss_sum(mm); + kthread_use_mm(mm); + pagefault_disable(); + for (i = 0; i < 3; i++) + old_left[i] = copy_from_user(&byte, (void __user *)(old + offsets[i]), 1); + pagefault_enable(); + moved = test_mremap(old, len, len, MREMAP_FIXED | MREMAP_MAYMOVE, new); + pagefault_disable(); + for (i = 0; i < 3; i++) { + new_left[i] = copy_from_user(&readback[i], (void __user *)(new + offsets[i]), 1); + old_left[i] += !copy_from_user(&byte, (void __user *)(old + offsets[i]), 1); + } + pagefault_enable(); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + KUNIT_EXPECT_EQ(test, mm->total_vm, vm_before); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, commit_before); + KUNIT_EXPECT_EQ(test, get_mm_rss_sum(mm), rss_before); + for (i = 0; i < 3; i++) { + KUNIT_EXPECT_EQ(test, old_left[i], 0UL); + KUNIT_EXPECT_EQ(test, new_left[i], 0UL); + KUNIT_EXPECT_EQ(test, readback[i], (u8)(0x80 + i)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, new + offsets[i])), phys[i]); + } + if (cases[shape].level) + KUNIT_EXPECT_EQ(test, test_table_descriptor(mm, new, cases[shape].level == 2), table); + } +} + +static void user4k_stack_relocate_test(struct kunit *test) +{ + const unsigned long shifts[] = { SZ_4K, SZ_2M + SZ_4K, SZ_1G + 3 * SZ_4K }; + unsigned int shape, i; + + for (shape = 0; shape < ARRAY_SIZE(shifts); shape++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct vm_page_offset pos; + unsigned long top, old, new; + phys_addr_t phys[3]; + u8 bytes[3] = { 0x42, 0x73, 0xb5 }; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, create_init_stack_vma(mm, &vma, &top), 0); + old = vma->vm_end - 3 * SZ_4K; + new = old - shifts[shape]; + mmap_write_lock(mm); + ret = expand_stack_locked(vma, old); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, + &bytes[i], 1, FOLL_WRITE), 1); + phys[i] = pte_phys_mm(mm, test_fault_entry(test, mm, old + i * SZ_4K)); + } + pos = vma_page_offset_at(vma, old); + mmap_write_lock(mm); + ret = relocate_vma_down(vma, shifts[shape]); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, vma->vm_start, new); + KUNIT_EXPECT_EQ(test, vma->vm_end, new + 3 * SZ_4K); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, new), pos)); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, new + i * SZ_4K)), phys[i]); + } + /* Grow after relocation: keep the file-like anon offset linear. */ + mmap_write_lock(mm); + vm_flags_clear(vma, VM_STACK_INCOMPLETE_SETUP); + ret = expand_stack_locked(vma, new - SZ_4K); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 4UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new - SZ_4K, &bytes[0], 1, FOLL_WRITE), 1); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, new), pos)); + } +} + +static void user4k_mremap_cow_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + unsigned long old = TEST_VA + SZ_4K, new = SZ_2M - 2 * SZ_4K, mapped; + unsigned long moved; + u8 value = 0x29, byte; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + parent = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + kthread_use_mm(parent); + mapped = vm_mmap(NULL, old, 3 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(parent); + KUNIT_ASSERT_EQ(test, mapped, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(parent, old + SZ_4K, &value, 1, FOLL_WRITE), 1); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(child); + /* Move only the middle quarter, leaving both neighbors mapped. */ + moved = test_mremap(old + SZ_4K, SZ_4K, SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(child); + KUNIT_ASSERT_EQ(test, moved, new); + value = 0xa4; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, new, &value, 1, FOLL_WRITE), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(parent, old + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x29); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, new, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, value); + KUNIT_EXPECT_EQ(test, mm_pages_to_bytes(child, child->total_vm), 3UL * SZ_4K); + KUNIT_EXPECT_EQ(test, child->committed_user_pages, 3UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, old, &value, 1, 0), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, old + 2 * SZ_4K, &value, 1, 0), 1); +} + +static void user4k_mremap_resize_test(struct kunit *test) +{ + struct mm_struct *mm; + unsigned long old = TEST_VA + SZ_4K, new = TEST_VA + 8 * PAGE_SIZE + 2 * SZ_4K; + unsigned long res[7]; + u8 value = 0x98, byte = 0; + int unmap[2]; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + res[0] = vm_mmap(NULL, old, SZ_4K, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + res[1] = test_mremap(old, SZ_4K, 3 * SZ_4K - 1, 0, 0); + res[2] = test_mremap(old + 1, SZ_4K, SZ_4K, 0, 0); + res[3] = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new + 1); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, res[0], old); + KUNIT_ASSERT_EQ(test, res[1], old); + KUNIT_EXPECT_EQ(test, res[2], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, res[3], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + SZ_4K, &value, 1, FOLL_WRITE), 1); + kthread_use_mm(mm); + res[4] = test_mremap(old, 3 * SZ_4K, 2 * SZ_4K, 0, 0); + res[5] = test_mremap(old, 2 * SZ_4K, 2 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, res[4], old); + KUNIT_ASSERT_EQ(test, res[5], new); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 4UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, value); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, old + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0); + kthread_use_mm(mm); + unmap[0] = vm_munmap(old, 2 * SZ_4K); + unmap[1] = vm_munmap(new, 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, unmap[0], 0); + KUNIT_EXPECT_EQ(test, unmap[1], 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 0UL); +} + +static int test_reject_mremap(struct vm_area_struct *vma) +{ + return -EIO; +} + +static void user4k_cold_batch_test(struct kunit *test) +{ + struct mm_struct *mm, *child; + struct vm_area_struct *vma; + unsigned long start = TEST_VA, mapped; + spinlock_t *ptl; + pte_t *ptep, entry; + u8 byte = 0x68; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm cold fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start, &byte, 1, FOLL_WRITE), 1); + /* Only the last quarter is young: batching must aggregate that bit. */ + mmap_read_lock(mm); + vma = vma_lookup(mm, start); + for (i = 0; i < 3; i++) { + unsigned long addr = start + i * SZ_4K; + + ptep = test_mm_lookup(mm, addr, &ptl); + if (!ptep) { + mmap_read_unlock(mm); + KUNIT_FAIL(test, "missing cold test PTE"); + return; + } + entry = ptep_clear_flush(vma, addr, ptep); + set_pte_at(mm, addr, ptep, pte_mkold(entry)); + pte_unmap_unlock(ptep, ptl); + } + mmap_read_unlock(mm); + kthread_use_mm(mm); + ret = do_madvise(mm, start + 3 * SZ_4K, SZ_4K, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, pte_young(test_fault_entry(test, mm, start + 3 * SZ_4K))); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(mm); + ret = do_madvise(mm, start, PAGE_SIZE, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, pte_young(test_fault_entry(test, mm, start + 3 * SZ_4K))); + kunit_release_action(test, user4k_vma_mm_free, child); + kthread_use_mm(mm); + ret = do_madvise(mm, start, PAGE_SIZE, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_FALSE(test, pte_young(test_fault_entry(test, mm, start + i * SZ_4K))); + byte = 0; + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, start, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x68); +} + +static void user4k_prefault_dirty_test(struct kunit *test) +{ + struct mm_struct *mm; + struct folio *folio; + unsigned long start = TEST_VA + SZ_4K, mapped; + spinlock_t *ptl; + pte_t *ptep, entry; + u8 byte = 0x7a; + int locked = 1; + long ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm prefault fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, SZ_4K, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start, &byte, 1, FOLL_WRITE), 1); + mmap_read_lock(mm); + ptep = test_mm_lookup(mm, start, &ptl); + if (!ptep) { + mmap_read_unlock(mm); + KUNIT_FAIL(test, "missing prefault test PTE"); + return; + } + entry = ptep_clear_flush(vma_lookup(mm, start), start, ptep); + folio = page_folio(pte_page(entry)); + folio_get(folio); + set_pte_at(mm, start, ptep, pte_mkclean(entry)); + folio_clear_dirty(folio); + pte_unmap_unlock(ptep, ptl); + mmap_read_unlock(mm); + kthread_use_mm(mm); + mmap_read_lock(mm); + ret = faultin_page_range(mm, start, start + SZ_4K, true, &locked); + if (locked) + mmap_read_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 1L); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + folio_put(folio); +} + +struct test_fsync_record { + loff_t start, end; + unsigned int calls; + int datasync; +}; + +static int test_fsync_range(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct test_fsync_record *record = file->private_data; + + *record = (struct test_fsync_record) { start, end, record->calls + 1, datasync }; + return 0; +} + +asmlinkage long __arm64_sys_msync(const struct pt_regs *regs); + +static void user4k_msync_offsets_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vma; + struct file_operations ops; + const struct file_operations *original_ops; + void *original_private; + struct test_fsync_record records[3] = {}, record = {}; + struct pt_regs regs = { .regs = { TEST_VA + SZ_4K, 1, MS_SYNC } }; + long result[3]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm sync fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 3 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 5 * SZ_4K, 3 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + original_ops = file->f_op; + original_private = file->private_data; + ops = *original_ops; + ops.fsync = test_fsync_range; + file->f_op = &ops; + file->private_data = &record; + kthread_use_mm(vma->vm_mm); + for (i = 0; i < ARRAY_SIZE(result); i++) { + regs.regs[0] = vma->vm_start + i * SZ_4K; + result[i] = __arm64_sys_msync(®s); + records[i] = record; + } + kthread_unuse_mm(vma->vm_mm); + file->f_op = original_ops; + file->private_data = original_private; + for (i = 0; i < ARRAY_SIZE(result); i++) { + KUNIT_EXPECT_EQ(test, result[i], 0L); + KUNIT_EXPECT_EQ(test, records[i].calls, i + 1); + KUNIT_EXPECT_EQ(test, records[i].start, (loff_t)(3 + i) * SZ_4K); + KUNIT_EXPECT_EQ(test, records[i].end, (loff_t)(4 + i) * SZ_4K - 1); + KUNIT_EXPECT_EQ(test, records[i].datasync, 1); + } +} + +static void user4k_mremap_file_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + struct file *file; + struct vm_operations_struct rejecting_ops; + const struct vm_operations_struct *original_ops; + unsigned long old = TEST_VA + SZ_4K, new = SZ_1G - SZ_4K; + unsigned long mapped, moved; + u8 bytes[3]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(file, old, 3 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, 3 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, old); + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, &bytes[i], 1, 0), 1); + vma = test_vma_lookup(mm, old); + KUNIT_ASSERT_NOT_NULL(test, vma); + original_ops = vma->vm_ops; + rejecting_ops = *original_ops; + rejecting_ops.mremap = test_reject_mremap; + mmap_write_lock(mm); + vma->vm_ops = &rejecting_ops; + mmap_write_unlock(mm); + kthread_use_mm(mm); + moved = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + /* Restore stack-owned callbacks before any assertion can leave this scope. */ + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *part; + + for_each_vma(vmi, part) + if (part->vm_ops == &rejecting_ops) + part->vm_ops = original_ops; + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, moved, (unsigned long)-EIO); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 1); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + kthread_use_mm(mm); + moved = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + vma = test_vma_lookup(mm, new); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 3U * SZ_4K); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + } +} + +static void user4k_exec_arguments_test(struct kunit *test) +{ + unsigned int native; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm stack setup requires hardware opt-in"); + return; + } + for (native = 0; native < 2; native++) { + struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); + struct mm_struct *mm; + unsigned long size, mask, initial, copied, new_top; + unsigned long a_len = 2 * PAGE_SIZE + SZ_4K + 31, b_len = SZ_4K + 43; + char *a, *b, *buf, *too_long; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, bprm); + mm = native ? mm_alloc() : test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + size = mm_page_size(mm); + mask = mm_page_mask(mm); + a_len = min(a_len, 16 * size + 31); + bprm->mm = mm; + KUNIT_ASSERT_EQ(test, create_init_stack_vma(mm, &bprm->vma, &bprm->p), 0); + initial = bprm->p; + bprm->argmin = initial - SZ_2M; + bprm->rlim_stack.rlim_cur = SZ_8M; + bprm->rlim_stack.rlim_max = RLIM_INFINITY; + a = kunit_kmalloc(test, a_len + 1, GFP_KERNEL); + b = kunit_kmalloc(test, b_len + 1, GFP_KERNEL); + buf = kunit_kmalloc(test, a_len + b_len + 7, GFP_KERNEL); + too_long = kunit_kmalloc(test, 32 * size + 1, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, buf); + KUNIT_ASSERT_NOT_NULL(test, too_long); + memset(a, 'A', a_len); + a[a_len] = 0; + memset(b, 'b', b_len); + b[b_len] = 0; + memset(too_long, 'x', 32 * size); + too_long[32 * size] = 0; + KUNIT_EXPECT_EQ(test, copy_string_kernel(too_long, bprm), -E2BIG); + KUNIT_EXPECT_EQ(test, bprm->p, initial); + KUNIT_ASSERT_EQ(test, copy_string_kernel("tail", bprm), 0); + bprm->exec = bprm->p; + KUNIT_ASSERT_EQ(test, copy_string_kernel(a, bprm), 0); + KUNIT_ASSERT_EQ(test, copy_string_kernel(b, bprm), 0); + bprm->argc = 3; + copied = initial - bprm->p; + KUNIT_EXPECT_EQ(test, copied, a_len + b_len + 7); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, 0), (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_MEMEQ(test, buf + b_len + 1, a, a_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + new_top = (bprm->vma->vm_end - SZ_2M - size) & mask; + kthread_use_mm(mm); + ret = setup_arg_pages(bprm, new_top, EXSTACK_DISABLE_X); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Subtract < PAGE_SIZE, then round down to 16 bytes and up to a leaf. */ + KUNIT_EXPECT_LE(test, bprm->vma->vm_end, new_top); + KUNIT_EXPECT_LE(test, new_top - bprm->vma->vm_end, PAGE_SIZE); + KUNIT_EXPECT_EQ(test, bprm->vma->vm_end & ~mask, 0UL); + KUNIT_EXPECT_EQ(test, mm->start_stack, bprm->p); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, 0), (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_MEMEQ(test, buf + b_len + 1, a, a_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + /* /proc/PID/cmdline and environ read the stack with FOLL_ANON. */ + memset(buf, 0, copied); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, FOLL_ANON), + (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + /* Interpreter argument removal scans several user granules. */ + initial = bprm->p; + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + 1); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + a_len + 2); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + copied); + KUNIT_EXPECT_EQ(test, bprm->argc, 0); + KUNIT_EXPECT_EQ(test, bprm->vma_pages, 0UL); + } +} + +/* The transfer can replace bprm->mm, so release its final owner. */ +static void test_exec_mm_free(void *data) +{ + struct linux_binprm *bprm = data; + + mmput(bprm->mm); +} + +static void user64k_exec_arguments_test(struct kunit *test) +{ + unsigned int shifts[] = { 16, 12, PAGE_SHIFT }; + unsigned int mode; + + if (PAGE_SHIFT >= 16 || + (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { + kunit_skip(test, "requires internal larger user granule and MM opt-in"); + return; + } + for (mode = 0; mode < ARRAY_SIZE(shifts); mode++) { + struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); + unsigned long a_len = mode ? SZ_64K + SZ_16K + 31 : 32 * PAGE_SIZE + 31; + unsigned long b_len = PAGE_SIZE + 17, initial, copied; + char *a, *b, *buf; + unsigned long i; + + KUNIT_ASSERT_NOT_NULL(test, bprm); + bprm->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, bprm->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_exec_mm_free, bprm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(bprm->mm, 16), 0); + KUNIT_ASSERT_EQ(test, create_init_stack_vma(bprm->mm, &bprm->vma, &bprm->p), 0); + initial = bprm->p; + bprm->argmin = initial - SZ_2M; + bprm->rlim_stack.rlim_cur = SZ_8M; + bprm->rlim_stack.rlim_max = RLIM_INFINITY; + a = kunit_kmalloc(test, a_len + 1, GFP_KERNEL); + b = kunit_kmalloc(test, b_len + 1, GFP_KERNEL); + buf = kunit_kmalloc(test, a_len + b_len + 7, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, buf); + for (i = 0; i < a_len; i++) + a[i] = 'A' + i % 23; + a[a_len] = 0; + memset(b, 'b', b_len); + b[b_len] = 0; + KUNIT_ASSERT_EQ(test, copy_string_kernel("tail", bprm), 0); + KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, 5, 0), 5); + KUNIT_ASSERT_STREQ(test, buf, "tail"); + KUNIT_ASSERT_EQ(test, copy_string_kernel(a, bprm), 0); + KUNIT_ASSERT_EQ(test, copy_string_kernel(b, bprm), 0); + bprm->argc = 3; + copied = initial - bprm->p; + KUNIT_ASSERT_EQ(test, copied, a_len + b_len + 7); + KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), + (int)copied); + KUNIT_ASSERT_EQ(test, memcmp(buf, b, b_len + 1), 0); + KUNIT_ASSERT_EQ(test, memcmp(buf + b_len + 1, a, a_len + 1), 0); + KUNIT_ASSERT_STREQ(test, buf + b_len + a_len + 2, "tail"); + /* Exercise copying out of every native page within a coarse leaf. */ + if (shifts[mode] != 16) { + KUNIT_ASSERT_EQ(test, bprm_set_page_shift(bprm, shifts[mode]), 0); + KUNIT_ASSERT_EQ(test, mm_page_shift(bprm->mm), shifts[mode]); + memset(buf, 0, copied); + KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), + (int)copied); + KUNIT_ASSERT_EQ(test, memcmp(buf, b, b_len + 1), 0); + KUNIT_ASSERT_EQ(test, memcmp(buf + b_len + 1, a, a_len + 1), 0); + KUNIT_ASSERT_STREQ(test, buf + b_len + a_len + 2, "tail"); + } + initial = bprm->p; + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + 1); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + a_len + 2); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + copied); + KUNIT_EXPECT_EQ(test, bprm->argc, 0); + KUNIT_EXPECT_EQ(test, bprm->vma_pages, 0UL); + } +} + +static struct vm_area_struct *test_special_vma(struct mm_struct *mm, + const struct vm_special_mapping *sm, unsigned long start, unsigned long len, + vm_flags_t flags) +{ + struct vm_area_struct *vma; + + mmap_write_lock(mm); + vma = _install_special_mapping(mm, start, len, flags, sm); + mmap_write_unlock(mm); + return vma; +} + +static void user4k_special_mapping_test(struct kunit *test) +{ + struct vm_special_mapping *sm = kunit_kzalloc(test, sizeof(*sm), GFP_KERNEL); + struct page **pages = kunit_kcalloc(test, 3, sizeof(*pages), GFP_KERNEL); + void *data[2] = { test_alloc_page(test), test_alloc_page(test) }; + struct mm_struct *small = test_vma_mm(test), *cold = test_vma_mm(test); + struct mm_struct *native = mm_alloc(); + struct vm_area_struct *vma[3]; + unsigned long start = SZ_2M - SZ_4K; + unsigned int i, side; + u8 byte; + + KUNIT_ASSERT_NOT_NULL(test, sm); + KUNIT_ASSERT_NOT_NULL(test, pages); + KUNIT_ASSERT_NOT_NULL(test, data[0]); + KUNIT_ASSERT_NOT_NULL(test, data[1]); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, cold); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + /* Reverse the supplied native pages: folio indices cannot identify them. */ + pages[0] = virt_to_page(data[1]); + pages[1] = virt_to_page(data[0]); + sm->pages = pages; + sm->name = "[user4k-special-test]"; + for (i = 0; i < 2 * TEST_SLOTS; i++) + memset(page_address(pages[i / TEST_SLOTS]) + (i % TEST_SLOTS) * SZ_4K, 0x41 + i, SZ_4K); + vma[0] = test_special_vma(small, sm, start, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma[1] = test_special_vma(cold, sm, start, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma[2] = test_special_vma(native, sm, TEST_VA, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + for (i = 0; i < 3; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma[i]); + for (i = 0; i < 2 * TEST_SLOTS; i++) { + unsigned long addr = start + i * SZ_4K; + + KUNIT_ASSERT_EQ(test, access_remote_vm(small, addr, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0x41 + i)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small, test_fault_entry(test, small, addr)), + page_to_phys(pages[i / TEST_SLOTS]) + (i % TEST_SLOTS) * SZ_4K); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(small, MM_FILEPAGES), 2UL * TEST_SLOTS); + /* Both present-PTE and cold-PTE forced writes copy only their quarter. */ + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? cold : small; + + byte = 0xe1 + side; + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start + 2 * SZ_4K + 7, + &byte, 1, FOLL_WRITE | FOLL_FORCE), 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, start + 2 * SZ_4K + 7, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0xe1 + side)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native, TEST_VA + 2 * SZ_4K + 7, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x43); + } + { + struct user_page_fragment fragment; + long ret = pin_user_fragments_remote(small, start + 5 * SZ_4K + 13, + 7, 0, &fragment, 1); + + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_NOT_NULL(test, fragment.subpage); + KUNIT_EXPECT_EQ(test, fragment.offset, (5 * SZ_4K + 13UL) & ~PAGE_MASK); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment.folio) + fragment.offset, + 0x46, fragment.length), NULL); + release_user_fragments(&fragment, 1, false); + } +} + +static void user4k_insert_range_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + void *data[2] = { test_alloc_page(test), test_alloc_page(test) }; + struct page *pages[2]; + struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long size, start, offset, length, num; + unsigned int i; + int ret, invalid[4]; + + KUNIT_ASSERT_NOT_NULL(test, data[0]); + KUNIT_ASSERT_NOT_NULL(test, data[1]); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + pages[0] = virt_to_page(data[1]); + pages[1] = virt_to_page(data[0]); + size = mm_page_size(mm); + start = pmd_size_mm(mm) - size; + offset = native ? 0 : 3 * SZ_4K; + length = 2 * PAGE_SIZE - offset; + vma = test_vma_add(mm, start, length, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + invalid[0] = vm_insert_pages_range(vma, start + 1, pages, 2, offset, length); + invalid[1] = vm_insert_pages_range(vma, start, pages, 2, offset + 1, length); + invalid[2] = vm_insert_pages_range(vma, start, pages, 2, offset, length + size); + invalid[3] = vm_insert_pages_range(vma, start, pages, ULONG_MAX, 0, length); + ret = vm_insert_pages_range(vma, start, pages, 2, offset, length); + mmap_write_unlock(mm); + for (i = 0; i < ARRAY_SIZE(invalid); i++) + KUNIT_EXPECT_EQ(test, invalid[i], -EINVAL); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < length / size; i++) { + unsigned long source = offset + i * size; + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[source >> PAGE_SHIFT]) + offset_in_page(source)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), length / size); + /* The original native-page-count API also expands all user leaves. */ + start += 4 * PAGE_SIZE; + vma = test_vma_add(mm, start, 2 * PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + num = 2; + mmap_write_lock(mm); + ret = vm_insert_pages(vma, start, pages, &num); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, num, 0UL); + for (i = 0; i < 2 * PAGE_SIZE / size; i++) + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[(i * size) >> PAGE_SHIFT]) + offset_in_page(i * size)); + num = 2; + mmap_write_lock(mm); + ret = vm_insert_pages(vma, start, pages, &num); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EBUSY); + KUNIT_EXPECT_EQ(test, num, 2UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), + (length + 2 * PAGE_SIZE) / size); + start += 4 * PAGE_SIZE; + offset = native ? PAGE_SIZE : 3 * SZ_4K; + length = 2 * PAGE_SIZE - offset; + vma = test_vma_add(mm, start, length, + (struct vm_page_offset) { .index = offset >> PAGE_SHIFT, + .offset = offset_in_page(offset) }, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + ret = vm_map_pages(vma, pages, 2); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < length / size; i++) { + unsigned long source = offset + i * size; + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[source >> PAGE_SHIFT]) + offset_in_page(source)); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(pages[0]), 1); + KUNIT_EXPECT_EQ(test, page_ref_count(pages[1]), 1); + } +} + +static void user4k_remap_pfn_range_test(struct kunit *test) +{ + struct folio *folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 1); + + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (unsigned int native = 0; native < 2; native++) { + struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); + unsigned long size, boundaries[3]; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, mm), 0); + size = mm_page_size(mm); + boundaries[0] = pmd_size_mm(mm); + boundaries[1] = pud_size_mm(mm); + boundaries[2] = pgd_size_mm(mm); + for (unsigned int shape = 0; shape < ARRAY_SIZE(boundaries); shape++) { + unsigned long start = boundaries[shape] - size; + unsigned long length = PAGE_SIZE + size; + struct vm_area_struct *vma; + int ret; + + if (shape && boundaries[shape] == boundaries[shape - 1]) + continue; + vma = test_vma_add(mm, start, length, (struct vm_page_offset) {}, + VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_SHARED | VM_MAYSHARE); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + ret = remap_pfn_range(vma, start, folio_pfn(folio), length, + vma->vm_page_prot); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + for (unsigned long offset = 0; offset < length; offset += size) { + pte_t pte = test_fault_entry(test, mm, start + offset); + + struct follow_pfnmap_args args = { + .vma = vma, .address = start + offset + 31, + }; + phys_addr_t expected = PFN_PHYS(folio_pfn(folio)) + offset + 31; + int ret; + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_special(pte)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), + PFN_PHYS(folio_pfn(folio)) + offset); + + mmap_read_lock(mm); + ret = follow_pfnmap_start(&args); + if (!ret) + follow_pfnmap_end(&args); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, native ? 0 : -EOPNOTSUPP); + args.allow_subpage = true; + mmap_read_lock(mm); + ret = follow_pfnmap_start(&args); + if (!ret) { + KUNIT_EXPECT_EQ(test, args.pfn, (unsigned long)PHYS_PFN(expected)); + KUNIT_EXPECT_EQ(test, args.offset, (unsigned int)offset_in_page(expected)); + KUNIT_EXPECT_EQ(test, args.addr_mask, mm_page_mask(mm)); + KUNIT_EXPECT_TRUE(test, args.special); + follow_pfnmap_end(&args); + } + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + } + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, start - size))); + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, start + length))); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0UL); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + } +} + +#ifdef CONFIG_HAVE_IOREMAP_PROT +static void test_iounmap(void *arg) +{ + iounmap((void __iomem *)arg); +} + +static const struct vm_operations_struct test_iomem_ops = { + .access = generic_access_phys, +}; +#endif + +static void user4k_iomem_access_test(struct kunit *test) +{ +#if defined(CONFIG_HAVE_IOREMAP_PROT) && defined(CONFIG_OF) + struct device_node *node; + struct resource resource; + struct mm_struct *mm; + struct vm_area_struct *vma; + void __iomem *uart; + unsigned long start = SZ_2M + PAGE_SIZE - SZ_4K; + u8 reference[16], buffer[32]; + vm_fault_t fault; + int ret; + + if (!of_machine_is_compatible("linux,dummy-virt")) { + kunit_skip(test, "requires disposable QEMU virt PL011 identification registers"); + return; + } + node = of_find_compatible_node(NULL, NULL, "arm,pl011"); + KUNIT_ASSERT_NOT_NULL(test, node); + ret = of_address_to_resource(node, 0, &resource); + uart = ret ? NULL : of_iomap(node, 0); + of_node_put(node); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_NOT_NULL(test, uart); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iounmap, + (void *)uart), 0); + KUNIT_ASSERT_GE(test, resource_size(&resource), (resource_size_t)SZ_4K); + memcpy_fromio(reference, uart + 0xff0, sizeof(reference)); + KUNIT_ASSERT_NOT_NULL(test, memchr_inv(reference, 0, sizeof(reference))); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = test_vma_add(mm, start, 2 * SZ_4K, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_SHARED | VM_MAYSHARE | VM_IO | VM_PFNMAP); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_ops = &test_iomem_ops; + vma->vm_page_prot = pgprot_device(vm_get_page_prot(vma->vm_flags)); + mmap_read_lock(mm); + fault = vmf_insert_pfn_prot_mkwrite_offset(vma, start, + PHYS_PFN(resource.start), offset_in_page(resource.start), + vma->vm_page_prot, false); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)VM_FAULT_NOPAGE); + memset(buffer, 0x6d, sizeof(buffer)); + mmap_read_lock(mm); + ret = generic_access_phys(vma, start + 0xff0, buffer, sizeof(buffer), 0); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 16); + KUNIT_EXPECT_MEMEQ(test, buffer, reference, sizeof(reference)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 16, 0x6d, 16), NULL); + memset(buffer, 0x5a, sizeof(buffer)); + ret = access_remote_vm(mm, start + 0xff0, buffer, sizeof(buffer), 0); + KUNIT_EXPECT_EQ(test, ret, 16); + KUNIT_EXPECT_MEMEQ(test, buffer, reference, sizeof(reference)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 16, 0x5a, 16), NULL); + /* Writes must fail before touching the read-only device mapping. */ + mmap_read_lock(mm); + ret = generic_access_phys(vma, start + 0xff0, buffer, 1, FOLL_WRITE); + mmap_read_unlock(mm); + KUNIT_EXPECT_LT(test, ret, 0); +#else + kunit_skip(test, "requires OF and iomem access support"); +#endif +} + +static void user4k_pfn_offset_test(struct kunit *test) +{ + void *data = test_alloc_page(test); + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long start = SZ_2M - SZ_4K; + unsigned long pfn; + unsigned int i; + vm_fault_t result[4], invalid[3], write; + + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, mm); + pfn = page_to_pfn(virt_to_page(data)); + vma = test_vma_add(mm, start, PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | VM_SHARED | VM_MAYSHARE | + VM_PFNMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + /* Model a write-notify VMA: initial PTEs are read-only. */ + vma->vm_page_prot = vm_get_page_prot(VM_READ | VM_SHARED); + mmap_read_lock(mm); + for (i = 0; i < 4; i++) + result[i] = vmf_insert_pfn_prot_mkwrite_offset(vma, start + i * SZ_4K, + pfn, (3 - i) * SZ_4K, vma->vm_page_prot, false); + invalid[0] = vmf_insert_pfn_prot_mkwrite_offset(vma, start + 1, pfn, + 0, vma->vm_page_prot, false); + invalid[1] = vmf_insert_pfn_prot_mkwrite_offset(vma, start, pfn, + 1, vma->vm_page_prot, false); + invalid[2] = vmf_insert_pfn_prot_mkwrite_offset(vma, start, pfn, + PAGE_SIZE, vma->vm_page_prot, false); + write = vmf_insert_pfn_prot_mkwrite_offset(vma, start + SZ_4K, pfn, + 2 * SZ_4K, vma->vm_page_prot, true); + mmap_read_unlock(mm); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, mm, start + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, result[i], (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), PFN_PHYS(pfn) + (3 - i) * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_special(entry)); + KUNIT_EXPECT_EQ(test, !!pte_write(entry), i == 1); + } + for (i = 0; i < ARRAY_SIZE(invalid); i++) + KUNIT_EXPECT_EQ(test, invalid[i], (vm_fault_t)VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, write, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 0UL); + KUNIT_EXPECT_EQ(test, page_ref_count(virt_to_page(data)), 1); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(virt_to_page(data)), 1); +} + +static void user16k_insert_shared_test(struct kunit *test) +{ + struct mm_struct *middle, *small, *native; + struct vm_area_struct *vmas[3]; + struct mm_subpage_mapcounts counts; + struct page *page; + void *data; + vm_fault_t result; + unsigned int i; + vm_flags_t flags = VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "mixed16K file insertion requires larger native backing"); + return; + } + data = test_alloc_page(test); + KUNIT_ASSERT_NOT_NULL(test, data); + page = virt_to_page(data); + middle = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, middle); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, middle), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(middle, 14), 0); + small = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, small); + native = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + vmas[0] = test_vma_add(middle, TEST_VA, PAGE_SIZE, (struct vm_page_offset) {}, flags); + vmas[1] = test_vma_add(small, TEST_VA + SZ_4K, SZ_4K, (struct vm_page_offset) {}, flags); + vmas[2] = test_vma_add(native, TEST_VA, PAGE_SIZE, (struct vm_page_offset) {}, flags); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vmas[i]); + vmas[i]->vm_page_prot = vm_get_page_prot(vmas[i]->vm_flags); + } + for (i = 0; i < PAGE_SIZE / SZ_16K; i++) { + unsigned long address = TEST_VA + i * SZ_16K; + + mmap_read_lock(middle); + result = vmf_insert_page_offset(vmas[0], address, page, i * SZ_16K); + mmap_read_unlock(middle); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, pte_phys_mm(middle, test_fault_entry(test, middle, address)), + page_to_phys(page) + i * SZ_16K); + } + mmap_read_lock(small); + result = vmf_insert_page_offset(vmas[1], vmas[1]->vm_start, page, PAGE_SIZE - SZ_4K); + mmap_read_unlock(small); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + mmap_read_lock(native); + result = vmf_insert_page_offset(vmas[2], vmas[2]->vm_start, page, 0); + mmap_read_unlock(native); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 5U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], i == TEST_SLOTS - 1 ? 2U : 1U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 6); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 7); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(middle, MM_FILEPAGES), 4UL); + /* Real mm teardown removes whole16K leaves and preserves both aliases. */ + kunit_release_action(test, user4k_vma_mm_free, middle); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 1U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], i == TEST_SLOTS - 1 ? 1U : 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 2); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 3); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small, test_fault_entry(test, small, vmas[1]->vm_start)), + page_to_phys(page) + PAGE_SIZE - SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(native, test_fault_entry(test, native, vmas[2]->vm_start)), + page_to_phys(page)); + kunit_release_action(test, user4k_vma_mm_free, small); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 1); + kunit_release_action(test, user4k_vma_mm_free, native); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 0); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 1); +} + +static void user4k_insert_fragment_test(struct kunit *test) +{ + void *data = test_alloc_page(test); + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + struct page *page; + vm_fault_t result[4], invalid[4]; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, mm); + page = virt_to_page(data); + vma = test_vma_add(mm, start, PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_read_lock(mm); + for (i = 0; i < 4; i++) + result[i] = vmf_insert_page_offset(vma, start + i * SZ_4K, page, (3 - i) * SZ_4K); + invalid[0] = vmf_insert_page_offset(vma, start + 1, page, 0); + invalid[1] = vmf_insert_page_offset(vma, start, page, 1); + invalid[2] = vmf_insert_page_offset(vma, start, page, PAGE_SIZE); + invalid[3] = vmf_insert_page_offset(vma, vma->vm_end, page, 0); + mmap_read_unlock(mm); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, result[i], (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, invalid[i], (vm_fault_t)VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, start + i * SZ_4K)), + page_to_phys(page) + (3 - i) * SZ_4K); + } + KUNIT_EXPECT_EQ(test, page_ref_count(page), 5); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 4UL); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 1); +} + +#if defined(CONFIG_GENERIC_GETTIMEOFDAY) && defined(CONFIG_VDSO_GETRANDOM) +static void user4k_vdso_mapping_test(struct kunit *test) +{ + struct mm_struct *mm; + struct linux_binprm bprm = {}; + unsigned long code, base, left[8]; + u8 bytes[8], magic[4]; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm vDSO fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + mm->mmap_base = TEST_VA + SZ_4K; + mm_flags_clear(MMF_TOPDOWN, mm); + kthread_use_mm(mm); + ret = arch_setup_additional_pages(&bprm, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + code = (unsigned long)mm->context.vdso; + KUNIT_ASSERT_NE(test, code, 0UL); + base = code - VDSO_NR_PAGES * PAGE_SIZE; + KUNIT_EXPECT_EQ(test, base & ~PAGE_MASK, SZ_4K); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, code, magic, sizeof(magic), 0), (int)sizeof(magic)); + KUNIT_EXPECT_MEMEQ(test, magic, "\177ELF", 4); + /* VVAR forbids remote GUP; ordinary uaccess must still fault it in. */ + kthread_use_mm(mm); + for (i = 0; i < 4; i++) { + left[i] = copy_from_user(&bytes[i], (void __user *)(base + i * SZ_4K), 1); + left[4 + i] = copy_from_user(&bytes[4 + i], + (void __user *)(base + VDSO_RNG_PAGE_OFFSET * PAGE_SIZE + i * SZ_4K), 1); + } + kthread_unuse_mm(mm); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, left[i], 0UL); + KUNIT_EXPECT_EQ(test, left[4 + i], 0UL); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, base + i * SZ_4K)), + page_to_phys(virt_to_page(vdso_k_time_data)) + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, + base + VDSO_RNG_PAGE_OFFSET * PAGE_SIZE + i * SZ_4K)), + page_to_phys(virt_to_page(vdso_k_rng_data)) + i * SZ_4K); + } + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, base, magic, sizeof(magic), 0), 0); +} + +#endif + +static void user4k_remote_access_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA + SZ_4K, 3 * SZ_4K); + struct mm_struct *child = test_vma_mm(test); + struct vm_area_struct *child_vma; + unsigned long start = TEST_VA + SZ_4K; + u8 *buf = kunit_kmalloc(test, 3 * SZ_4K, GFP_KERNEL); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + KUNIT_ASSERT_NOT_NULL(test, buf); + memset(buf, 0x5a, 3 * SZ_4K); + KUNIT_ASSERT_EQ(test, access_remote_vm(vma->vm_mm, start, buf, 3 * SZ_4K, FOLL_WRITE), + 3 * (int)SZ_4K); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Deliberately straddle quarters whose physical/virtual offsets differ. */ + memset(buf, 0x96, SZ_4K + 34); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + SZ_4K - 17, + buf, SZ_4K + 34, FOLL_WRITE), (int)SZ_4K + 34); + memset(buf, 0, 3 * SZ_4K); + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, start, buf, 3 * SZ_4K, 0), + 3 * (int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x5a, 3 * SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start, buf, 3 * SZ_4K, 0), 3 * (int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x5a, SZ_4K - 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + SZ_4K - 17, 0x96, SZ_4K + 34), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 2 * SZ_4K + 17, 0x5a, SZ_4K - 17), NULL); + child_vma = test_vma_lookup(child, start); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + mmap_write_lock(child); + vm_flags_clear(child_vma, VM_WRITE); + mmap_write_unlock(child); + buf[0] = 0x27; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, FOLL_WRITE), 0); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, + FOLL_WRITE | FOLL_FORCE), 1); + buf[0] = 0; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, 0), 1); + KUNIT_EXPECT_EQ(test, buf[0], (u8)0x27); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 3 * SZ_4K - 5, buf, 20, 0), 5); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 3 * SZ_4K, buf, 20, 0), 0); +} + +static void user4k_remote_file_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *small, *native; + u8 buf[34]; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, 2 * SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 3 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x34, 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 17, 0x35, 17), NULL); + memset(buf, 0xa6, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 3 * SZ_4K - 17, + buf, sizeof(buf), FOLL_WRITE), (int)sizeof(buf)); + memset(buf, 0, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 4 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0xa6, sizeof(buf)), NULL); +#ifdef CONFIG_CROSS_MEMORY_ATTACH + { + struct kvec vec = { .iov_base = buf, .iov_len = sizeof(buf) }; + struct iov_iter iter; + + memset(buf, 0xb7, sizeof(buf)); + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, sizeof(buf)); + KUNIT_EXPECT_EQ(test, process_vm_rw_fragments(small->vm_mm, + TEST_VA + 3 * SZ_4K - 17, sizeof(buf), &iter, true), 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + memset(buf, 0, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 4 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0xb7, sizeof(buf)), NULL); + } +#endif + mmap_write_lock(small->vm_mm); + vm_flags_clear(small, VM_WRITE); + mmap_write_unlock(small->vm_mm); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, small->vm_start, buf, 1, + FOLL_WRITE | FOLL_FORCE), 0); +#ifdef CONFIG_CROSS_MEMORY_ATTACH + { + struct kvec vec = { .iov_base = buf, .iov_len = sizeof(buf) }; + struct iov_iter iter; + + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, sizeof(buf)); + KUNIT_EXPECT_EQ(test, process_vm_rw_fragments(small->vm_mm, + small->vm_start, sizeof(buf), &iter, true), -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), sizeof(buf)); + } +#endif +} + +#ifdef CONFIG_CROSS_MEMORY_ATTACH +static void user4k_process_vm_test(struct kunit *test) +{ + /* Exceed the stack batch, cross table boundaries and include a partial tail. */ + unsigned long start = SZ_2M - SZ_4K, len = 21 * SZ_4K; + struct vm_area_struct *vma = test_fault_vma(test, start, len); + struct mm_struct *child = test_vma_mm(test); + struct iov_iter iter; + struct kvec vec; + u8 *buf = kunit_kmalloc(test, len + 17, GFP_KERNEL); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + KUNIT_ASSERT_NOT_NULL(test, buf); + memset(buf, 0x61, len); + vec = (struct kvec) { .iov_base = buf, .iov_len = len }; + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, len); + ret = process_vm_rw_fragments(vma->vm_mm, start, len, &iter, true); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, iov_iter_count(&iter), 0UL); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(buf, 0xc7, len); + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, len - 34); + ret = process_vm_rw_fragments(child, start + 17, len - 34, &iter, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + memset(buf, 0, len); + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, len); + ret = process_vm_rw_fragments(vma->vm_mm, start, len, &iter, false); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x61, len), NULL); + vec.iov_len = len + 17; + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, len + 17); + ret = process_vm_rw_fragments(child, start, len + 17, &iter, false); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 17UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x61, 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 17, 0xc7, len - 34), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + len - 17, 0x61, 17), NULL); + /* A shorter local iterator ends successfully before the remote range. */ + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, 23); + ret = process_vm_rw_fragments(child, start + 13, len, &iter, false); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); +} +#endif + +#ifdef CONFIG_FUTEX +static void test_stop_kthread(void *arg); + +struct test_futex_waiter { + struct mm_struct *mm; + unsigned long address; + struct completion ready; + struct completion done; + int result; +}; + +static int test_futex_wait_run(void *arg) +{ + struct test_futex_waiter *w = arg; + ktime_t deadline = ktime_add_ms(ktime_get(), 10000); + + kthread_use_mm(w->mm); + complete(&w->ready); + w->result = futex_wait((u32 __user *)w->address, FLAGS_SIZE_32 | FLAGS_SHARED, + 0x33333333, &deadline, FUTEX_BITSET_MATCH_ANY); + kthread_unuse_mm(w->mm); + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void user4k_futex_wake_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vmas[2]; + unsigned long addresses[2] = { TEST_VA + SZ_4K + 64, TEST_VA + 2 * SZ_4K + 64 }; + unsigned int round; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm futex fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vmas[0] = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, SZ_4K, 2 * SZ_4K); + vmas[1] = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, vmas[0]); + KUNIT_ASSERT_NOT_NULL(test, vmas[1]); + /* Both directions use the real wait queue and atomic user-value read. */ + for (round = 0; round < 2; round++) { + struct test_futex_waiter *w = kunit_kzalloc(test, sizeof(*w), GFP_KERNEL); + struct task_struct *task; + unsigned long until; + int woke = 0, wrong; + + KUNIT_ASSERT_NOT_NULL(test, w); + w->mm = vmas[round]->vm_mm; + w->address = addresses[round]; + init_completion(&w->ready); + init_completion(&w->done); + task = kthread_run(test_futex_wait_run, w, "user4k-futex-%u", round); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->ready, 5 * HZ), 0UL); + until = jiffies + 5 * HZ; + kthread_use_mm(vmas[round ^ 1]->vm_mm); + wrong = futex_wake((u32 __user *)(addresses[round ^ 1] + 4), + FLAGS_SIZE_32 | FLAGS_SHARED, 1, FUTEX_BITSET_MATCH_ANY); + do { + woke = futex_wake((u32 __user *)addresses[round ^ 1], + FLAGS_SIZE_32 | FLAGS_SHARED, 1, FUTEX_BITSET_MATCH_ANY); + if (woke || completion_done(&w->done)) + break; + schedule_timeout_uninterruptible(1); + } while (time_before(jiffies, until)); + kthread_unuse_mm(vmas[round ^ 1]->vm_mm); + KUNIT_EXPECT_EQ(test, wrong, 0); + KUNIT_EXPECT_EQ(test, woke, 1); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->done, 12 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, w->result, 0); + kunit_release_action(test, test_stop_kthread, task); + } +} + +static void user4k_futex_keys_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *small, *native; + union futex_key keys[2], other; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm futex fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, 2 * SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + for (i = 0; i < 3; i++) { + unsigned long va = small->vm_start + i * SZ_4K + 64; + unsigned long fileoff = (i + 2) * SZ_4K + 64; + + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)va, FLAGS_SIZE_32 | FLAGS_SHARED, + &keys[0], FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(native->vm_mm); + ret = get_futex_key((u32 __user *)(TEST_VA + fileoff), + FLAGS_SIZE_32 | FLAGS_SHARED, &keys[1], FUTEX_READ); + kthread_unuse_mm(native->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, futex_match(&keys[0], &keys[1])); + KUNIT_EXPECT_EQ(test, keys[0].shared.pgoff, fileoff / PAGE_SIZE); + KUNIT_EXPECT_EQ(test, keys[0].both.offset, (unsigned int)(fileoff % PAGE_SIZE) | FUT_OFF_INODE); + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)(va + 4), FLAGS_SIZE_32 | FLAGS_SHARED, + &other, FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_FALSE(test, futex_match(&keys[0], &other)); + } + mmap_write_lock(small->vm_mm); + vm_flags_clear(small, VM_WRITE); + mmap_write_unlock(small->vm_mm); + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)(small->vm_start + 2 * SZ_4K + 64), + FLAGS_SIZE_32 | FLAGS_SHARED, &other, FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, futex_match(&keys[1], &other)); +} +#endif + +static void test_swap_usage_mapping_free(void *arg) +{ + struct address_space *mapping = arg; + + xa_destroy(&mapping->i_pages); +} + +static void user4k_shmem_swap_usage_test(struct kunit *test) +{ + struct address_space *mapping = kunit_kzalloc(test, sizeof(*mapping), GFP_KERNEL); + void *ret; + + KUNIT_ASSERT_NOT_NULL(test, mapping); + xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_swap_usage_mapping_free, mapping), 0); + /* Synthetic swap values exercise accounting without allocating swap slots. */ + ret = xa_store(&mapping->i_pages, 1, xa_mk_value(1), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + if (IS_ENABLED(CONFIG_XARRAY_MULTI)) { + ret = xa_store_range(&mapping->i_pages, 4, 7, + xa_mk_value(2), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + } else { + unsigned long index; + + /* Without multi-index entries, shmem stores one value per page. */ + for (index = 4; index <= 7; index++) { + ret = xa_store(&mapping->i_pages, index, + xa_mk_value(2), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + } + } + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, PAGE_SIZE + SZ_4K, + PAGE_SIZE + 3 * SZ_4K), 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 3 * SZ_4K, + 2 * PAGE_SIZE + SZ_4K), PAGE_SIZE); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 4 * PAGE_SIZE + SZ_4K, + 7 * PAGE_SIZE + 3 * SZ_4K), 3 * PAGE_SIZE + 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 5 * PAGE_SIZE + SZ_4K, + 5 * PAGE_SIZE + 2 * SZ_4K), (unsigned long)SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 0, SZ_4K), 0UL); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, PAGE_SIZE, PAGE_SIZE), 0UL); +} + +static void test_file_mapcounts(struct kunit *test, struct folio *folio, + const unsigned int expected[TEST_SLOTS], unsigned int native) +{ + struct mm_subpage_mapcounts counts; + unsigned int i, total = 0; + + for (i = 0; i < TEST_SLOTS; i++) + total += expected[i]; + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(&folio->page, &counts), total); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], expected[i]); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(total + native)); +} + +static void user4k_file_mapcounts_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *shared, *private, *native, *cvma; + struct mm_struct *child; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA, 3 * SZ_4K, SZ_4K); + private = test_file_vma(test, file, false, false, TEST_VA, 3 * SZ_4K, SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 2 }, 1); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA + 2 * SZ_4K, true) & + VM_FAULT_ERROR, 0U); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 1 }, 1); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, private->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 3, 3, 1 }, 1); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 3, 2, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, child); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, private->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 1, 1, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 1, 1, 1 }, 0); + kunit_release_action(test, user4k_vma_mm_free, shared->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 0, 0, 0 }, 0); +} + +/* A unique RAM-backed test inode: exercise generic file faults without + * making unsupported filesystem promotion part of this mapping fixture. + */ +static int test_coarse_read_folio(struct file *file, struct folio *folio) +{ + unsigned long i; + loff_t base = (loff_t)folio->index << PAGE_SHIFT; + loff_t size = i_size_read(file_inode(file)); + u8 *data = folio_address(folio); + + for (i = 0; i < folio_size(folio); i++) + data[i] = base + i < size ? 0x31 + ((base + i) / SZ_4K) : 0; + __folio_mark_uptodate(folio); + folio_unlock(folio); + return 0; +} + +static struct file *test_coarse_cache_file(struct kunit *test, unsigned int order, + unsigned long size) +{ + static const struct address_space_operations aops = { + .read_folio = test_coarse_read_folio, + .dirty_folio = filemap_dirty_folio, + }; + static const struct file_operations fops = { + .mmap = generic_file_mmap, + .read_iter = generic_file_read_iter, + .llseek = generic_file_llseek, + }; + struct file *file = anon_inode_create_getfile("user-coarse-file", &fops, + NULL, O_RDWR, NULL); + struct folio *folio; + int ret; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, test_file_put, file)) + return ERR_PTR(-ENOMEM); + file->f_mapping->a_ops = &aops; + mapping_set_large_folios(file->f_mapping); + i_size_write(file_inode(file), size); + folio = folio_alloc(GFP_KERNEL, order); + if (!folio) + return ERR_PTR(-ENOMEM); + ret = filemap_add_folio(file->f_mapping, folio, 0, GFP_KERNEL); + if (!ret) + test_coarse_read_folio(file, folio); /* Unlocks the installed folio. */ + folio_put(folio); + return ret ? ERR_PTR(ret) : file; +} + +static bool test_coarse_file_enabled(struct kunit *test) +{ + if (PAGE_SHIFT >= 16 || + (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { + kunit_skip(test, "requires internal larger user granule and borrowed-mm opt-in"); + return false; + } + return true; +} + +static void test_promote_unpin(void *data) +{ + struct user_page_fragment *fragment = data; + + if (fragment->folio) { + release_user_fragments(fragment, 1, true); + fragment->folio = NULL; + } +} + +static void user64k_file_promote_test(struct kunit *test) +{ + unsigned int pinned; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "cache promotion currently uses the collapse implementation"); + return; + } + for (pinned = 0; pinned < 2; pinned++) { + struct file *file = test_cache_file(test, 2 * SZ_64K, true); + struct vm_area_struct *coarse, *small; + struct user_page_fragment *fragment; + struct folio *folio; + phys_addr_t old[SZ_64K / PAGE_SIZE], untouched; + unsigned int nr = SZ_64K / PAGE_SIZE, i; + unsigned long offset, modified = SZ_64K / 2 + 19; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, SZ_64K); + small = test_file_vma(test, file, false, true, TEST_VA, SZ_64K, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, small); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + untouched = page_to_phys(&folio->page); + folio_put(folio); + for (i = 0; i < nr; i++) { + folio = filemap_get_folio(file->f_mapping, nr + i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + old[i] = page_to_phys(&folio->page); + folio_put(folio); + } + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + if (pinned) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_promote_unpin, + fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + TEST_VA + modified, 1, FOLL_WRITE, fragment, 1), 1L); + *(u8 *)(folio_address(fragment->folio) + fragment->offset) = 0xa7; + KUNIT_EXPECT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), -EBUSY); + /* A pin in the middle forces rollback of earlier isolated folios. */ + for (i = 0; i < nr; i++) { + folio = filemap_get_folio(file->f_mapping, nr + i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, page_to_phys(&folio->page), old[i]); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), (u8)0xa7); + kunit_release_action(test, test_promote_unpin, fragment); + } + KUNIT_ASSERT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), 0); + folio = filemap_get_folio(file->f_mapping, nr); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_order(folio), 16U - PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, folio->index, (pgoff_t)nr); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, (unsigned long)2 * nr); + for (offset = 0; offset < SZ_64K; offset++) { + u8 expected = pinned && offset == modified ? 0xa7 : 0x41 + offset / SZ_4K; + + if (((u8 *)folio_address(folio))[offset] != expected) { + KUNIT_FAIL(test, "promoted data mismatch at %lu", offset); + break; + } + } + /* An already suitable folio is an idempotent success, including with a GET. */ + KUNIT_EXPECT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), 0); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, coarse->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse->vm_mm, entry), page_to_phys(&folio->page)); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) { + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(&folio->page) + offset); + } + { + struct folio *first = filemap_get_folio(file->f_mapping, 0); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, first); + KUNIT_EXPECT_EQ(test, page_to_phys(&first->page), untouched); + folio_put(first); + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); + } +} + +static void user64k_shmem_fault_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 4; mode++) { + bool seeded = mode & 1, shared = mode < 2; + struct file *file = test_cache_file(test, SZ_64K, seeded); + struct vm_area_struct *vma; + struct folio *cache, *mapped; + unsigned long offset; + pte_t entry; + vm_fault_t fault; + unsigned int attempt; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, shared, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (attempt = 0; attempt < 8; attempt++) { + fault = test_file_fault(vma, TEST_VA, true); + if (!(fault & VM_FAULT_NOPAGE)) + break; + } + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + mapped = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, entry)))); + KUNIT_EXPECT_EQ(test, folio_size(mapped), SZ_64K); + KUNIT_EXPECT_EQ(test, !!folio_test_anon(mapped), !shared); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mapped) + offset, + seeded ? 0x31 + offset / SZ_4K : 0, SZ_4K), NULL); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, cache), 0); + KUNIT_EXPECT_EQ(test, folio_size(cache), SZ_64K); + if (shared) + KUNIT_EXPECT_PTR_EQ(test, mapped, cache); + else { + KUNIT_EXPECT_PTR_NE(test, mapped, cache); + memset(folio_address(mapped), 0xc9, SZ_64K); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + offset, + seeded ? 0x31 + offset / SZ_4K : 0, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, + shared ? MM_SHMEMPAGES : MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, SZ_64K / PAGE_SIZE); + { + u8 byte = 0x83, observed = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, TEST_VA + SZ_64K - 1, + &byte, 1, FOLL_WRITE), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, TEST_VA + SZ_64K - 1, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + KUNIT_EXPECT_EQ(test, ((u8 *)folio_address(cache))[SZ_64K - 1], + shared ? byte : (u8)(seeded ? 0x40 : 0)); + } + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(cache), (int)(SZ_64K / PAGE_SIZE) + 1); + } +} + +static void user64k_shmem_fault_busy_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 3; mode++) { + struct file *file = test_cache_file(test, SZ_64K, mode != 0); + struct vm_area_struct *vma; + struct folio *held; + vm_fault_t fault; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!mode) { + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_REMOTE | + FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT, NULL); + mmap_assert_locked(vma->vm_mm); + mmap_read_unlock(vma->vm_mm); + KUNIT_EXPECT_EQ(test, fault, (vm_fault_t)VM_FAULT_RETRY); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, 0UL); + } else if (mode == 1) { + held = filemap_get_folio(file->f_mapping, SZ_64K / (2 * PAGE_SIZE)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, held); + folio_lock(held); + fault = test_file_fault(vma, TEST_VA, false); + folio_unlock(held); + folio_put(held); + KUNIT_EXPECT_EQ(test, fault, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SHMEMPAGES), 0L); + } else { + struct vm_area_struct *small = test_file_vma(test, file, false, true, + TEST_VA, SZ_64K, 0); + struct user_page_fragment *fragment; + + KUNIT_ASSERT_NOT_NULL(test, small); + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_promote_unpin, + fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + TEST_VA + SZ_64K / 2, 1, FOLL_WRITE, fragment, 1), 1L); + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), (u8)0x39); + kunit_release_action(test, test_promote_unpin, fragment); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + } + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } +} + +static void user64k_shmem_fault_hole_test(struct kunit *test) +{ +#ifdef CONFIG_USERFAULTFD + unsigned int adjacent; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (adjacent = 0; adjacent < 2; adjacent++) { + struct file *file = test_cache_file(test, 2 * SZ_64K, false); + struct vm_area_struct *coarse, *missing, *seed; + struct folio *folio; + vm_fault_t fault; + bool new; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + missing = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, + adjacent ? SZ_64K : PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, missing); + seed = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, 0); + KUNIT_ASSERT_NOT_NULL(test, seed); + mmap_read_lock(seed->vm_mm); + folio = shmem_uffd_prepare_folio(seed, TEST_VA, &new); + mmap_read_unlock(seed->vm_mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, new); + memset(folio_address(folio), 0x6f, SZ_4K); + shmem_uffd_mark_present(folio, 0, BIT(0)); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), + GENMASK(PAGE_SIZE / SZ_4K - 1, 1)); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + /* Model the registration flag used by the collapse interval check. */ + mmap_write_lock(missing->vm_mm); + vma_start_write(missing); + vm_flags_set(missing, VM_UFFD_MISSING); + mmap_write_unlock(missing->vm_mm); + fault = test_file_fault(coarse, TEST_VA, false); + mmap_write_lock(missing->vm_mm); + vma_start_write(missing); + vm_flags_clear(missing, VM_UFFD_MISSING); + mmap_write_unlock(missing->vm_mm); + /* A real coarse fault may populate holes covered by its entire leaf. */ + KUNIT_EXPECT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 1, 0, SZ_4K)); + folio_unlock(folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x6f, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, + SZ_64K - SZ_4K), NULL); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, missing->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, seed->vm_mm); + } +#else + kunit_skip(test, "requires userfaultfd registration flags"); +#endif +} + +static void user64k_file_leaf_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *coarse, *native, *small, *child_vma; + struct mm_struct *coarse_mm, *child; + struct folio *folio; + unsigned long offset; + unsigned int nr = (2 * SZ_64K) / PAGE_SIZE; + unsigned int small_nr = 2 * SZ_64K / SZ_4K - 1; + unsigned int maps = 2 * nr + small_nr; + u8 byte = 0xc7, observed; + int ret; + + if (!test_coarse_file_enabled(test)) + return; + file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, 2 * SZ_64K, 0); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * SZ_64K, 0); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 2 * SZ_64K - SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + coarse_mm = coarse->vm_mm; + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + /* One cache folio supplies two user leaves, with native references per leaf. */ + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_64K) { + pte_t entry = test_fault_entry(test, coarse_mm, TEST_VA + offset); + + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse_mm, entry), + page_to_phys(&folio->page) + offset); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 2L); + for (offset = 0; offset < 2 * SZ_64K; offset += PAGE_SIZE) + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + for (offset = SZ_4K; offset < 2 * SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + KUNIT_ASSERT_EQ(test, access_remote_vm(coarse_mm, TEST_VA + SZ_64K + 3 * SZ_4K, + &byte, 1, FOLL_WRITE), 1); + KUNIT_ASSERT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + SZ_64K + 3 * SZ_4K, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + byte = 0x98; + KUNIT_ASSERT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 7 * SZ_4K, + &byte, 1, FOLL_WRITE), 1); + KUNIT_ASSERT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 7 * SZ_4K, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + KUNIT_EXPECT_EQ(test, ((u8 *)folio_address(folio))[7 * SZ_4K], byte); + + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, coarse_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Ordinary file VMAs are lazily populated in the child after fork. */ + mmap_read_lock(child); + child_vma = find_vma(child, TEST_VA); + mmap_read_unlock(child); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + maps += nr; + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + kthread_use_mm(coarse_mm); + ret = vm_munmap(TEST_VA, SZ_64K); + kthread_unuse_mm(coarse_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + maps -= SZ_64K / PAGE_SIZE; + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 1L); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, child, TEST_VA))); + + /* The native-rounded range must remove every overlapping coarse alias. */ + unmap_mapping_range(file->f_mapping, SZ_64K + SZ_4K, SZ_4K, 0); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, coarse_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, TEST_VA + SZ_64K + SZ_4K))); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + SZ_64K + PAGE_SIZE))); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 0L); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), + (int)(maps - 2 * SZ_64K / PAGE_SIZE - 1 - PAGE_SIZE / SZ_4K)); + + /* An exact subpage hole keeps neighboring small leaves in that native page. */ + mmap_read_lock(coarse_mm); + coarse = find_vma(coarse_mm, TEST_VA + SZ_64K); + mmap_read_unlock(coarse_mm); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < PAGE_SIZE; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + SZ_64K + offset, + false) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + unmap_mapping_subpage_hole(file->f_mapping, SZ_64K + SZ_4K, SZ_64K + 2 * SZ_4K); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, coarse_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA + SZ_64K))); + for (offset = 0; offset < PAGE_SIZE; offset += SZ_4K) + KUNIT_EXPECT_EQ(test, !!pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + SZ_64K + offset)), offset != SZ_4K); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(maps - 2 * SZ_64K / PAGE_SIZE - 2)); + kunit_release_action(test, user4k_vma_mm_free, coarse_mm); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); +} + +static void user64k_file_cow_test(struct kunit *test) +{ + unsigned int cold; + + if (!test_coarse_file_enabled(test)) + return; + for (cold = 0; cold < 2; cold++) { + struct file *file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + struct vm_area_struct *vma; + struct folio *cache, *anon; + unsigned long offset; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, false, TEST_VA, SZ_64K, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!cold) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + anon = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, entry)))); + KUNIT_EXPECT_TRUE(test, folio_test_anon(anon)); + KUNIT_EXPECT_EQ(test, folio_size(anon), SZ_64K); + folio_get(anon); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, anon), 0); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(anon) + offset, + 0x41 + offset / SZ_4K, SZ_4K), NULL); + memset(folio_address(anon), 0x93, SZ_64K); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + offset, + 0x31 + offset / SZ_4K, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + folio_put(cache); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_FILEPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(anon), (int)(SZ_64K / PAGE_SIZE)); + KUNIT_EXPECT_EQ(test, folio_ref_count(anon), (int)(SZ_64K / PAGE_SIZE) + 1); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(anon), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(anon), 1); + } +} + +static void user64k_file_bounds_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int mode; + unsigned long size = SZ_64K + PAGE_SIZE + 19; + pte_t entry; + + if (!test_coarse_file_enabled(test)) + return; + /* Too-small backing and a physically unaligned source cannot form a leaf. */ + for (mode = 0; mode < 2; mode++) { + file = test_coarse_cache_file(test, mode ? 17 - PAGE_SHIFT : 0, 2 * SZ_64K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, false, TEST_VA, SZ_64K, + mode ? PAGE_SIZE : 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } + file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, size); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, 3 * SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K, false) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_64K); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)) + + PAGE_SIZE + 19, 0, SZ_64K - PAGE_SIZE - 19), NULL); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA + 2 * SZ_64K, false) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_FILEPAGES), 2L); +} + +static void user4k_file_shared_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 4 * PAGE_SIZE, true); + struct vm_area_struct *small, *native; + struct folio *folio; + unsigned int i; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + native = test_file_vma(test, file, true, true, TEST_VA, 4 * PAGE_SIZE, 0); + /* Virtual and file offsets deliberately have different native alignment. */ + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 3 * PAGE_SIZE, 2 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + SZ_4K, false) & + VM_FAULT_ERROR, 0U); + /* Fault-around keeps its 64K window and populates all ready cache folios. */ + for (i = 0; i < 12; i++) { + unsigned long addr = small->vm_start + i * SZ_4K; + struct vm_page_offset pos = vma_page_offset_at(small, addr); + + entry = test_fault_entry(test, small->vm_mm, addr); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + folio = filemap_get_folio(file->f_mapping, pos.index); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(folio_file_page(folio, pos.index)) + pos.offset); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(small->vm_mm, entry)), + (u8)(0x33 + i)); + folio_put(folio); + } + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, native->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + memset(phys_to_virt(pte_phys_mm(native->vm_mm, entry)) + 2 * SZ_4K, 0x72, SZ_4K); + entry = test_fault_entry(test, small->vm_mm, small->vm_start); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(small->vm_mm, entry)), + 0x72, SZ_4K), NULL); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + SZ_4K, true) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, small->vm_start + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + memset(phys_to_virt(pte_phys_mm(small->vm_mm, entry)), 0x84, SZ_4K); + entry = test_fault_entry(test, native->vm_mm, TEST_VA); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(native->vm_mm, entry)) + + 3 * SZ_4K, 0x84, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + folio_put(folio); +} + +static void user4k_file_cow_test(struct kunit *test) +{ + unsigned int cold; + + for (cold = 0; cold < 2; cold++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *vma; + struct folio *cache; + pte_t entry; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, false, TEST_VA + SZ_4K, + 3 * SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!cold) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start + SZ_4K, true) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, vma->vm_start + SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + KUNIT_EXPECT_TRUE(test, folio_test_anon(page_folio(pte_page(entry)))); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, entry) & ~PAGE_MASK, 2UL * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), + 0x33, SZ_4K), NULL); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), 0x91, SZ_4K); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + folio_put(cache); + } +} + +static void user4k_file_eof_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, SZ_4K + 37, false); + struct vm_area_struct *small, *native; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, 0U); + for (i = 0; i < 4; i++) { + vm_fault_t ret = test_file_fault(small, TEST_VA + i * SZ_4K, false); + + KUNIT_EXPECT_EQ(test, ret & VM_FAULT_ERROR, i < 2 ? 0U : VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + i * SZ_4K)), i < 2); + } +} + + +static const struct vm_operations_struct test_filemap_fault_ops = { + .fault = filemap_fault, + .page_mkwrite = filemap_page_mkwrite, +}; + +static void user4k_filemap_fault_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * SZ_4K + 17, true); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + 3 * SZ_4K, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Exercise generic filemap_fault against an already populated cache. */ + vma->vm_ops = &test_filemap_fault_ops; + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start + SZ_4K, true) & VM_FAULT_ERROR, 0U); + for (i = 0; i < 3; i++) { + pte_t pte = test_fault_entry(test, vma->vm_mm, vma->vm_start + i * SZ_4K); + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), (u8)(0x31 + i)); + } + KUNIT_EXPECT_EQ(test, test_file_fault(vma, vma->vm_start + 3 * SZ_4K, false) & + VM_FAULT_ERROR, VM_FAULT_SIGBUS); +} + +static void user4k_file_truncate_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *small, *cow, *native; + struct iattr attr = { .ia_valid = ATTR_SIZE, .ia_size = SZ_4K + 37 }; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + PAGE_SIZE, SZ_4K); + cow = test_file_vma(test, file, false, false, TEST_VA, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, false, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, cow); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(cow, TEST_VA + 3 * SZ_4K, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + inode_lock(file_inode(file)); + ret = notify_change(&nop_mnt_idmap, file->f_path.dentry, &attr, NULL); + inode_unlock(file_inode(file)); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) { + unsigned long addr = small->vm_start + i * SZ_4K; + + if (i) + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, addr))); + KUNIT_EXPECT_EQ(test, test_file_fault(small, addr, false) & VM_FAULT_ERROR, + i ? VM_FAULT_SIGBUS : 0U); + } + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, cow->vm_mm, TEST_VA + 3 * SZ_4K))); + KUNIT_EXPECT_EQ(test, test_file_fault(cow, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, + VM_FAULT_SIGBUS); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA))); + KUNIT_EXPECT_EQ(test, test_file_fault(native, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, 0U); +} + +static void user4k_file_invalidate_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 4 * SZ_4K, PAGE_SIZE - SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & VM_FAULT_ERROR, 0U); + unmap_mapping_pages(file->f_mapping, 0, 1, false); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, vma->vm_mm, + vma->vm_start + i * SZ_4K)), i != 0); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & VM_FAULT_ERROR, 0U); + unmap_mapping_pages(file->f_mapping, 1, 1, false); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, vma->vm_mm, + vma->vm_start + i * SZ_4K)), i == 0); +} + +static void user4k_file_fork_hole_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *shared, *private, *cvma; + struct mm_struct *child; + struct inode *inode; + unsigned int i; + pte_t copied; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA, 2 * PAGE_SIZE, 0); + private = test_file_vma(test, file, false, false, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA, false) & VM_FAULT_ERROR, 0U); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, private->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + mmap_read_lock(child); + cvma = find_vma(child, TEST_VA); + mmap_read_unlock(child); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + copied = test_fault_entry(test, child, TEST_VA + SZ_4K); + memset(phys_to_virt(pte_phys_mm(child, copied)), 0x82, SZ_4K); + inode = file_inode(file); + /* A sub-native-page hole zeros shared bytes but retains private COW. */ + ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, SZ_4K, SZ_4K); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, i_size_read(inode), (loff_t)(2 * PAGE_SIZE)); + for (i = 0; i < 2 * TEST_SLOTS; i++) { + pte_t pte; + + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA + i * SZ_4K, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, shared->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(shared->vm_mm, pte)), + i == 1 ? 0 : 0x31 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, test_fault_entry(test, child, TEST_VA + SZ_4K)), + pte_phys_mm(child, copied)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(child, copied)), 0x82, SZ_4K), NULL); + /* A full cache-page hole exercises the file interval-tree invalidation. */ + ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, PAGE_SIZE, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = TEST_SLOTS; i < 2 * TEST_SLOTS; i++) { + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, shared->vm_mm, TEST_VA + i * SZ_4K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + i * SZ_4K))); + } +} + +static void test_zero_cow_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, TEST_VA, PAGE_SIZE); + struct mm_struct *mm; + struct mm_subpage *slot; + pte_t before[4], after; + unsigned int i; + vm_fault_t ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + for (i = 0; i < 4; i++) { + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + i * size, + FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + before[i] = test_fault_entry(test, mm, TEST_VA + i * size); + KUNIT_ASSERT_TRUE(test, pte_present(before[i])); + KUNIT_ASSERT_TRUE(test, is_zero_pfn(pte_pfn(before[i]))); + KUNIT_ASSERT_FALSE(test, pte_write(before[i])); + } + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + size + 7, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + lru_add_drain_all(); + for (i = 0; i < 4; i++) { + after = test_fault_entry(test, mm, TEST_VA + i * size); + if (i != 1) { + KUNIT_EXPECT_TRUE(test, pte_same(before[i], after)); + continue; + } + KUNIT_ASSERT_TRUE(test, pte_write(after)); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)size); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + size, 0, size), NULL); + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1UL); +} + +static void user4k_zero_cow_test(struct kunit *test) +{ + test_zero_cow_granule(test, 12); +} + +static void user16k_zero_cow_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_zero_cow_granule(test, 14); +} + +static void test_cow_reuse_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned int mode; + + for (mode = 0; mode < 2; mode++) { + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, TEST_VA, PAGE_SIZE); + struct mm_subpage *held = NULL; + struct mm_struct *child; + phys_addr_t original[4]; + unsigned int i; + vm_fault_t fault; + int ret; + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * size); + original[i] = pte_phys_mm(vma->vm_mm, pte); + memset(phys_to_virt(original[i]), 0xb0 + i, size); + } + if (mode) { + held = mm_subpage_get_from_phys(original[2]); + KUNIT_ASSERT_NOT_NULL(test, held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + } + child = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + for (i = 0; i < 4; i++) { + unsigned long address = TEST_VA + i * size; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte) == original[i], + !(mode && i == 2)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), + 0xb0 + i, size), NULL); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), 0xd0 + i, size); + } + if (held) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(mm_subpage_phys(held)), + 0xb2, size), NULL); + } +} + +static void user4k_cow_reuse_test(struct kunit *test) +{ + test_cow_reuse_granule(test, 12); +} + +static void user16k_cow_reuse_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_cow_reuse_granule(test, 14); +} + +static void test_fork_cow_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + struct vm_area_struct *parent = test_fault_vma_granule(test, shift, TEST_VA, 4 * size); + struct mm_struct *child = test_vma_mm_granule(test, shift); + struct vm_area_struct *cvma; + struct mm_subpage *held[4]; + pte_t before[4], after; + unsigned int i, side; + vm_fault_t fault; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + before[i] = test_fault_entry(test, parent->vm_mm, TEST_VA + i * size); + held[i] = mm_subpage_get_from_phys(pte_phys_mm(parent->vm_mm, before[i])); + KUNIT_ASSERT_NOT_NULL(test, held[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held[i]), 0); + memset(folio_address(mm_subpage_folio(held[i])) + mm_subpage_offset(held[i]), + 0x31 + i, size); + } + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + size + 15, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + for (i = 0; i < 4; i++) { + struct mm_subpage *slot; + bool changed = side && i == 1; + + after = test_fault_entry(test, mm, TEST_VA + i * size); + KUNIT_EXPECT_EQ(test, !!pte_write(after), changed); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after) != mm_subpage_phys(held[i]), changed); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)size); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0x31 + i, size), NULL); + if (changed) { + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); + memset(folio_address(mm_subpage_folio(slot)) + size, 0x77, size); + } + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 4UL); + } + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(held[i]), i == 1 ? 1 : 2); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(held[i])) + + mm_subpage_offset(held[i]), 0x31 + i, size), NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(held[0])), 11); +} + +static void user4k_fork_cow_test(struct kunit *test) +{ + test_fork_cow_granule(test, 12); +} + +static void user16k_fork_cow_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_fork_cow_granule(test, 14); +} + +static void user4k_folio_walk_geometry_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, PAGE_SHIFT, 14 }; + unsigned int which; + + for (which = 0; which < ARRAY_SIZE(shifts); which++) { + unsigned int shift = shifts[which], leaf; + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned long start, size = 1UL << shift; + vm_fault_t fault; + + if (which == 2 && PAGE_SHIFT <= 14) + continue; + mm = shift == PAGE_SHIFT ? mm_alloc() : test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shift == PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + start = pmd_size_mm(mm) - size; + vma = test_fault_vma_in_mm(mm, start, 2 * size); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (leaf = 0; leaf < 2; leaf++) { + unsigned long address = start + leaf * size; + struct folio_walk fw; + struct folio *folio; + struct page *page = NULL; + pte_t expected, walked = __pte(0); + bool zero = false; + + mmap_read_lock(mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address, 0); + if (folio) + folio_walk_end(&fw, vma); + mmap_read_unlock(mm); + KUNIT_EXPECT_PTR_EQ(test, folio, NULL); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address, FW_ZEROPAGE); + if (folio) { + zero = is_zero_folio(folio) && !fw.page && fw.level == FW_LEVEL_PTE; + folio_walk_end(&fw, vma); + } + mmap_read_unlock(mm); + KUNIT_EXPECT_TRUE(test, zero); + + mmap_read_lock(mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + expected = test_fault_entry(test, mm, address); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address + size - 1, 0); + if (folio) { + page = fw.page; + walked = fw.pte; + folio_walk_end(&fw, vma); + } + mmap_read_unlock(mm); + KUNIT_EXPECT_PTR_EQ(test, page, pfn_to_page(pte_pfn(expected))); + KUNIT_EXPECT_TRUE(test, pte_same(walked, expected)); + KUNIT_EXPECT_PTR_EQ(test, folio, page_folio(pfn_to_page(pte_pfn(expected)))); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_fragment_locked_pin_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, PAGE_SHIFT, 14 }; + unsigned int which; + + for (which = 0; which < ARRAY_SIZE(shifts); which++) { + unsigned int shift = shifts[which]; + struct mm_struct *mm; + struct vm_area_struct *vma, *found; + struct user_page_fragment fragment = { }; + unsigned long address; + int ret; + + if (which == 2 && PAGE_SHIFT <= 14) + continue; + mm = shift == PAGE_SHIFT ? mm_alloc() : test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shift == PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + vma = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + address = TEST_VA + (1UL << shift) - sizeof(long); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_LONGTERM, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOWAIT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + /* Only the first fragment is returned, while retaining the VMA lock. */ + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, 2 * sizeof(long), + FOLL_WRITE, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 1); + if (ret != 1) + continue; + KUNIT_EXPECT_PTR_EQ(test, found, vma); + KUNIT_EXPECT_TRUE(test, fragment.pinned); + KUNIT_EXPECT_EQ(test, fragment.length, (unsigned int)sizeof(long)); + KUNIT_EXPECT_EQ(test, !!fragment.subpage, shift != PAGE_SHIFT); + release_user_fragments(&fragment, 1, false); + /* With backing present, NOFAULT pins the exact same leaf successfully. */ + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 1); + if (ret == 1) { + KUNIT_EXPECT_PTR_EQ(test, found, vma); + release_user_fragments(&fragment, 1, true); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user16k_fragment_offset_test(struct kunit *test) +{ + struct vm_area_struct *vma; + struct user_page_fragment fragment = {}; + unsigned long address = TEST_VA + 3 * SZ_16K + SZ_4K + 13; + long ret; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative fragments require larger native backing"); + return; + } + vma = test_fault_vma_granule(test, 14, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + ret = pin_user_fragments_remote(vma->vm_mm, address, 31, FOLL_WRITE, &fragment, 1); + KUNIT_EXPECT_EQ(test, ret, 1L); + if (ret != 1) + return; + KUNIT_EXPECT_NOT_NULL(test, fragment.subpage); + KUNIT_EXPECT_EQ(test, fragment.offset, 3UL * SZ_16K + SZ_4K + 13); + KUNIT_EXPECT_EQ(test, fragment.length, 31U); + if (fragment.subpage) + KUNIT_EXPECT_EQ(test, mm_subpage_shift(fragment.subpage), 14U); + memset(folio_address(fragment.folio) + fragment.offset, 0x9c, fragment.length); + release_user_fragments(&fragment, 1, true); + { + u8 buf[33]; + + ret = access_remote_vm(vma->vm_mm, address - 1, buf, sizeof(buf), 0); + KUNIT_ASSERT_EQ(test, ret, (long)sizeof(buf)); + KUNIT_EXPECT_EQ(test, buf[0], (u8)0); + KUNIT_EXPECT_EQ(test, buf[32], (u8)0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 1, 0x9c, 31), NULL); + } +} + +static void user4k_anon_exit_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct mm_subpage *held; + struct anon_vma *root; + struct folio *folio; + struct mm_struct *mm; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + } + held = f->slots[0]; + folio = mm_subpage_folio(held); + root = f->vma->anon_vma->root; + mm_subpage_get(held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + mm_subpage_pool_close(f->pool); + mm_subpage_pool_put(f->pool); + /* Leave all four mapping refs and the tables to the real exit path. */ + mm = f->mm; + memset(f, 0, sizeof(*f)); + mmput(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(held), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); + for (i = 1; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_get_from_phys(mm_subpage_phys(held) + i * SZ_4K); + + KUNIT_EXPECT_PTR_EQ(test, slot, NULL); + if (slot) + mm_subpage_put(slot); + } +} + +#ifdef CONFIG_ARM64_CONTPTE +struct contpte_fixture { + struct mm_struct *mm; + struct folio *folio; + unsigned int nr; +}; + +static void contpte_fixture_free(void *arg) +{ + struct contpte_fixture *f = arg; + struct mmu_gather tlb; + unsigned int i; + + if (f->mm) { + mmap_write_lock(f->mm); + for (i = 0; i <= f->nr; i++) { + unsigned long addr = TEST_VA + i * mm_page_size(f->mm); + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, addr, &ptl); + + if (pte) { + pte_clear(f->mm, addr, pte); + pte_unmap_unlock(pte, ptl); + } + } + tlb_gather_mmu_fullmm(&tlb, f->mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(f->mm); + mmput(f->mm); + } + if (f->folio) + folio_put(f->folio); +} + +static void test_contpte_granule(struct kunit *test, bool small) +{ + struct contpte_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct vm_area_struct vma = {}; + struct test_mm tables = {}; + struct { + bool initial, addresses, hints, gathered, aged, access_all, unfolded; + bool permissions, refolded, holes, full_clear, sentinel; + } ok; + unsigned long size; + phys_addr_t phys; + pte_t base, saved, observed; + pte_t *pte; + spinlock_t *ptl; + unsigned int i; + int access_ret, access_same_ret, access_noop_ret; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, contpte_fixture_free, f), 0); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + if (small) + KUNIT_ASSERT_EQ(test, test_mm_select4k(f->mm), 0); + f->nr = cont_ptes_mm(f->mm); + size = mm_page_size(f->mm); + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, + get_order(cont_pte_size_mm(f->mm))); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + phys = PFN_PHYS(folio_pfn(f->folio)); + tables.mm = f->mm; + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, TEST_VA, phys), 0); + vma.vm_mm = f->mm; + base = phys_pte_mm(f->mm, phys, __pgprot(TEST_PROT | PTE_RDONLY)); + memset(&ok, 1, sizeof(ok)); + mmap_read_lock(f->mm); + pte = test_mm_lookup(f->mm, TEST_VA, &ptl); + if (!pte) { + mmap_read_unlock(f->mm); + KUNIT_FAIL(test, "missing PTE table"); + return; + } + pte_clear(f->mm, TEST_VA, pte); + /* The adjacent entry must never be included in this contiguous group. */ + saved = pte_mkspecial(base); + set_pte(pte + f->nr, saved); + set_ptes(f->mm, TEST_VA, pte, base, f->nr); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.initial &= pte_cont(observed); + ok.addresses &= pte_phys_mm(f->mm, observed) == phys + i * size; + ok.hints &= pte_batch_hint(pte + i, observed) == f->nr - i; + } + __set_pte(pte + 3, pte_mkdirty(__ptep_get(pte + 3))); + observed = ptep_get(pte + 1); + ok.gathered &= pte_dirty(observed) && pte_young(observed); + observed = ptep_get_lockless(pte + 2); + ok.gathered &= pte_dirty(observed) && pte_young(observed) && + pte_phys_mm(f->mm, observed) == phys + 2 * size; + clear_young_dirty_ptes(&vma, TEST_VA + 3 * size, pte + 3, 1, + CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.aged &= !pte_young(observed) && !pte_dirty(observed) && pte_cont(observed); + } + observed = pte_mkdirty(pte_mkyoung(__ptep_get(pte + 3))); + access_same_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, + pte + 3, observed, 1); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.access_all &= pte_cont(observed) && pte_young(observed) && + pte_dirty(observed) && !pte_write(observed); + } + access_noop_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, + pte + 3, ptep_get(pte + 3), 1); + observed = pte_mkwrite_novma(pte_mkdirty(pte_mkyoung(__ptep_get(pte + 3)))); + access_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, pte + 3, + observed, 1); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.unfolded &= !pte_cont(observed); + ok.permissions &= pte_write(observed) == (i == 3); + } + set_pte_at(f->mm, TEST_VA + 3 * size, pte + 3, + pte_advance_pfn_mm(f->mm, base, 3)); + /* Last-entry installation must fold using the physical byte offset too. */ + set_pte_at(f->mm, TEST_VA + (f->nr - 1) * size, pte + f->nr - 1, + pte_advance_pfn_mm(f->mm, base, f->nr - 1)); + for (i = 0; i < f->nr; i++) + ok.refolded &= pte_cont(__ptep_get(pte + i)); + get_and_clear_full_ptes(f->mm, TEST_VA + 3 * size, pte + 3, 2, 0); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.holes &= pte_none(observed) == (i == 3 || i == 4); + if (i != 3 && i != 4) + ok.holes &= !pte_cont(observed) && + pte_phys_mm(f->mm, observed) == phys + i * size; + } + set_ptes(f->mm, TEST_VA + 3 * size, pte + 3, + pte_advance_pfn_mm(f->mm, base, 3), 2); + set_pte_at(f->mm, TEST_VA + (f->nr - 1) * size, pte + f->nr - 1, + pte_advance_pfn_mm(f->mm, base, f->nr - 1)); + for (i = 0; i < f->nr; i++) + ok.refolded &= pte_cont(__ptep_get(pte + i)); + clear_full_ptes(f->mm, TEST_VA, pte, f->nr, 0); + for (i = 0; i < f->nr; i++) + ok.full_clear &= pte_none(__ptep_get(pte + i)); + ok.sentinel = pte_same(__ptep_get(pte + f->nr), saved); + pte_unmap_unlock(pte, ptl); + mmap_read_unlock(f->mm); + KUNIT_EXPECT_TRUE(test, ok.initial); + KUNIT_EXPECT_TRUE(test, ok.addresses); + KUNIT_EXPECT_TRUE(test, ok.hints); + KUNIT_EXPECT_TRUE(test, ok.gathered); + KUNIT_EXPECT_TRUE(test, ok.aged); + KUNIT_EXPECT_TRUE(test, ok.access_all); + KUNIT_EXPECT_EQ(test, access_same_ret, 1); + KUNIT_EXPECT_EQ(test, access_noop_ret, 0); + KUNIT_EXPECT_EQ(test, access_ret, 1); + KUNIT_EXPECT_TRUE(test, ok.unfolded); + KUNIT_EXPECT_TRUE(test, ok.permissions); + KUNIT_EXPECT_TRUE(test, ok.refolded); + KUNIT_EXPECT_TRUE(test, ok.holes); + KUNIT_EXPECT_TRUE(test, ok.full_clear); + KUNIT_EXPECT_TRUE(test, ok.sentinel); +} + +static void user4k_contpte_test(struct kunit *test) +{ + test_contpte_granule(test, true); +} + +static void native_contpte_test(struct kunit *test) +{ + test_contpte_granule(test, false); +} +#endif + +struct usercopy_worker { + struct test_mm *ctx[2]; + struct completion done; + unsigned int id; + unsigned int checks; + unsigned int idmaps; + int error; + u64 tcr; +}; + +static int test_usercopy_worker(void *arg) +{ + struct usercopy_worker *worker = arg; + u64 kernel_ttbr = read_sysreg(ttbr1_el1) & ~TTBRx_EL1_ASID_MASK; + unsigned int round, i, byte; + bool borrowed = false; + struct test_mm *ctx = NULL; + + for (round = 0; round < 16; round++) { + unsigned int small = (round + worker->id) & 1; + unsigned int cpu = cpumask_first(cpu_online_mask); + + ctx = worker->ctx[small]; + kthread_use_mm(ctx->mm); + borrowed = true; + if ((round + worker->id) & 2) { + unsigned int next = cpumask_next(cpu, cpu_online_mask); + + if (next < nr_cpu_ids) + cpu = next; + } + if (set_cpus_allowed_ptr(current, cpumask_of(cpu))) { + worker->error = 1; + break; + } + /* Exercise a real scheduler switch while borrowing this mm. */ + schedule_timeout_uninterruptible(1); + if (small && round < 4) { + unsigned long flags; + bool native; + + preempt_disable(); + flags = local_daif_save(); + if (round & 2) + cpu_install_ttbr0(phys_to_ttbr(__pa_symbol(idmap_pg_dir)), + TCR_T0SZ(IDMAP_VA_BITS)); + else + cpu_install_idmap(); + native = (read_sysreg(tcr_el1) & (TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK)) == + (TCR_TG0_16K | TCR_T0SZ(IDMAP_VA_BITS)); + cpu_uninstall_idmap(); + local_daif_restore(flags); + preempt_enable(); + if (!native) { + worker->error = 6; + break; + } + worker->idmaps++; + } + worker->tcr = read_sysreg(tcr_el1); + if ((worker->tcr & TCR_EL1_TG0_MASK) != (small ? TCR_TG0_4K : TCR_TG0_16K) || + (worker->tcr & TCR_EL1_T0SZ_MASK) != TCR_T0SZ(48) || + (worker->tcr & TCR_EL1_TG1_MASK) != TCR_TG1_16K || + (read_sysreg(ttbr1_el1) & ~TTBRx_EL1_ASID_MASK) != kernel_ttbr) { + worker->error = 2; + break; + } + for (i = 0; i < 4; i++) { + unsigned long offset = 64 + worker->id * sizeof(u64); + void __user *address = (void __user *)ctx->address[i]; + u64 value = 0xa5000000 | (worker->id << 16) | (round << 8) | i; + u64 *backing = phys_to_virt(ctx->phys[i]) + offset; + unsigned long read_left, write_left; + u8 data[8]; + + /* No VMA/fault path may be entered for these private mappings. */ + pagefault_disable(); + read_left = copy_from_user(data, address, sizeof(data)); + write_left = copy_to_user(address + offset, &value, sizeof(value)); + pagefault_enable(); + if (read_left != (i == 2 ? sizeof(data) : 0) || + write_left != (i == 0 || i == 2 ? sizeof(value) : 0)) { + worker->error = 3; + goto out; + } + if (i != 2) { + for (byte = 0; byte < sizeof(data); byte++) { + if (data[byte] != (small ? 0x40 : 0x80) + i) { + worker->error = 4; + goto out; + } + } + } + if (READ_ONCE(*backing) != (i == 0 || i == 2 ? 0 : value)) { + worker->error = 5; + goto out; + } + worker->checks++; + } + kthread_unuse_mm(ctx->mm); + borrowed = false; + } +out: + if (borrowed) + kthread_unuse_mm(ctx->mm); + complete(&worker->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void test_stop_kthread(void *arg) +{ + kthread_stop(arg); +} + +struct cow_reader { + struct mm_struct *parent; + struct mm_struct *child; + struct completion ready; + struct completion copied; + struct completion done; + u8 observed[8]; + unsigned long read_left[8], write_left[8]; + int error; + unsigned int checks; +}; + +static int test_cow_reader(void *arg) +{ + struct cow_reader *r = arg; + unsigned int i, side; + + kthread_use_mm(r->child); + /* Resolve fork's old PTEs normally, then cache pre-COW translations. */ + for (i = 0; i < 4; i++) { + u8 value = 0; + unsigned long left; + + left = copy_from_user(&value, (void __user *)(TEST_VA + i * SZ_4K + 64), 1); + if (left || value != 0x41 + i) + r->error = 1; + } + complete(&r->ready); + if (!wait_for_completion_timeout(&r->copied, 10 * HZ)) { + r->error = 2; + goto out; + } + for (side = 0; side < 2; side++) { + if (side) { + kthread_unuse_mm(r->child); + kthread_use_mm(r->parent); + } + for (i = 0; i < 4; i++) { + void __user *addr = (void __user *)(TEST_VA + i * SZ_4K + 64); + u8 value = 0, changed = 0x99; + unsigned long read_left, write_left; + + pagefault_disable(); + read_left = copy_from_user(&value, addr, 1); + write_left = copy_to_user(addr, &changed, 1); + pagefault_enable(); + r->observed[side * 4 + i] = value; + r->read_left[side * 4 + i] = read_left; + r->write_left[side * 4 + i] = write_left; + if (read_left || value != 0x41 + i || + write_left != (side == 0 && i == 1 ? 0 : 1)) + r->error = 3; + r->checks++; + } + } + kthread_unuse_mm(r->parent); + goto done; +out: + kthread_unuse_mm(r->child); +done: + complete(&r->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void user4k_fault_uaccess_test(struct kunit *test) +{ + struct vm_area_struct *parent, *child_vma; + struct mm_struct *child; + struct cow_reader *reader; + struct task_struct *task; + unsigned int i, cpu; + vm_fault_t fault; + int ret; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled || !system_supports_4kb_granule() || + system_supports_cnp() || cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires opt-in and compatible 4K TTBR0 controls"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + + memset(phys_to_virt(pte_phys_mm(parent->vm_mm, entry)), 0x41 + i, SZ_4K); + } + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + child_vma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + reader = kunit_kzalloc(test, sizeof(*reader), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, reader); + reader->parent = parent->vm_mm; + reader->child = child; + init_completion(&reader->ready); + init_completion(&reader->copied); + init_completion(&reader->done); + task = kthread_create(test_cow_reader, reader, "user4k-cow-reader"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + cpu = cpumask_last(cpu_online_mask); + kthread_bind(task, cpu); + wake_up_process(task); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&reader->ready, 10 * HZ), 0UL); + mmap_read_lock(child); + fault = handle_mm_fault(child_vma, TEST_VA + SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + complete(&reader->copied); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&reader->done, 10 * HZ), 0UL); + if (reader->error) + for (i = 0; i < 8; i++) + kunit_info(test, "access %u value=%#x read_left=%lu write_left=%lu\n", + i, reader->observed[i], reader->read_left[i], reader->write_left[i]); + KUNIT_EXPECT_EQ(test, reader->error, 0); + KUNIT_EXPECT_EQ(test, reader->checks, 8U); + { + pte_t entry = test_fault_entry(test, child, TEST_VA + SZ_4K); + + KUNIT_EXPECT_EQ(test, *(u8 *)(phys_to_virt(pte_phys_mm(child, entry)) + 64), (u8)0x99); + entry = test_fault_entry(test, parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, *(u8 *)(phys_to_virt(pte_phys_mm(parent->vm_mm, entry)) + 64), (u8)0x42); + } +} + +static void user4k_context_uaccess_test(struct kunit *test) +{ + struct test_mm *ctx[2]; + struct usercopy_worker *workers; + unsigned int i; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires 4K, no CnP/PIE/POE, DS=0, A1=1 and 48-bit TTBR0"); + return; + } + ctx[0] = test_usercopy_mm(test, false); + KUNIT_ASSERT_NOT_NULL(test, ctx[0]); + ctx[1] = test_usercopy_mm(test, true); + KUNIT_ASSERT_NOT_NULL(test, ctx[1]); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + struct task_struct *task; + + workers[i].ctx[0] = ctx[0]; + workers[i].ctx[1] = ctx[1]; + workers[i].id = i; + init_completion(&workers[i].done); + task = kthread_run(test_usercopy_worker, &workers[i], "user4k-copy-%u", i); + KUNIT_ASSERT_FALSE(test, IS_ERR(task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ_MSG(test, workers[i].error, 0, "worker %u TCR=%#llx", i, workers[i].tcr); + KUNIT_EXPECT_EQ(test, workers[i].checks, 64U); + KUNIT_EXPECT_EQ(test, workers[i].idmaps, 2U); + } + kunit_info(test, "two migrating kthreads, 32 borrowed-mm switches, %s PAN\n", + system_uses_ttbr0_pan() ? "software" : "hardware"); +} + +struct zap_reader { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready; + struct completion proceed; + struct completion done; + unsigned int cpu; + unsigned int checks; + bool zapped; + int error; +}; + +static int test_zap_reader(void *arg) +{ + struct zap_reader *reader = arg; + unsigned int phase, i; + + reader->error = set_cpus_allowed_ptr(current, cpumask_of(reader->cpu)); + if (reader->error) { + complete(&reader->ready); + goto out; + } + kthread_use_mm(reader->mm); + for (phase = 0; phase < 2; phase++) { + if (phase) { + complete(&reader->ready); + wait_for_completion(&reader->proceed); + if (!reader->zapped) + break; + } + for (i = 0; i < 4; i++) { + u64 value = 0, expected = 0xa5a50000UL + i; + unsigned long left; + + pagefault_disable(); + left = copy_from_user(&value, + (void __user *)((unsigned long)TEST_VA + i * SZ_4K), sizeof(value)); + pagefault_enable(); + if (phase && i == 1) { + if (!left) + reader->error = -EUCLEAN; + } else if (left || value != expected) { + reader->error = -EFAULT; + } + reader->checks++; + } + } + kthread_unuse_mm(reader->mm); +out: + complete(&reader->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void test_stop_zap_reader(void *arg) +{ + struct zap_reader *reader = arg; + + complete(&reader->proceed); + kthread_stop(reader->task); +} + +static void user4k_anon_remote_zap_test(struct kunit *test) +{ + struct anon_slot_fixture *f; + struct zap_reader *readers; + unsigned int i, cpu; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled || num_online_cpus() < 2) { + kunit_skip(test, "requires arm64.user4k_test=1 and two CPUs in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires 4K, no CnP/PIE/POE, DS=0, A1=1 and 48-bit TTBR0"); + return; + } + f = anon_slot_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + u64 value = 0xa5a50000UL + i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memcpy(folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot), + &value, sizeof(value)); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + } + readers = kunit_kcalloc(test, 2, sizeof(*readers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, readers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 2; i++) { + readers[i].mm = f->mm; + readers[i].cpu = cpu; + init_completion(&readers[i].ready); + init_completion(&readers[i].proceed); + init_completion(&readers[i].done); + readers[i].task = kthread_run(test_zap_reader, &readers[i], "user4k-zap-%u", i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, readers[i].task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_zap_reader, + &readers[i]), 0); + cpu = cpumask_next(cpu, cpu_online_mask); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].ready, 10 * HZ), 0UL); + KUNIT_ASSERT_EQ(test, readers[i].error, 0); + } + /* Keep an explicit non-mapping reference for fixture cleanup. */ + mm_subpage_get(f->slots[1]); + mmap_read_lock(f->mm); + zap_vma_range(f->vma, TEST_VA + SZ_4K, SZ_4K); + f->mapped[1] = false; + mmap_read_unlock(f->mm); + for (i = 0; i < 2; i++) { + readers[i].zapped = true; + complete(&readers[i].proceed); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, readers[i].error, 0); + KUNIT_EXPECT_EQ(test, readers[i].checks, 8U); + } +} + +static u64 *test_leaf(struct test_tables *tables, unsigned long va) +{ + return &tables->level[3][test_index(va, tables->shift, 3)]; +} + +/* Explicitly 4K, without the native PAGE_SHIFT or a granule/level hint. */ +static void test_invalidate_4k(unsigned long va, u64 ttbr1) +{ + u64 operand = (va >> USER4K_SHIFT) | (ttbr1 & GENMASK_ULL(63, 48)); + + dsb(ishst); + __tlbi(vae1, operand); + dsb(ish); + isb(); +} + +static void user4k_translation_test(struct kunit *test) +{ + struct test_tables small, native; + struct test_result *result; + unsigned long flags; + void *data, *other; + phys_addr_t data_pa, other_pa; + u64 old_tcr, old_ttbr0, old_ttbr1, old_par, tcr4k, tcr16k; + unsigned int round, slot; + + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE)) { + kunit_skip(test, "requires 4K, no CnP, and direct PTE permissions"); + return; + } + + KUNIT_ASSERT_EQ(test, PAGE_SHIFT, 14); + KUNIT_ASSERT_EQ(test, test_alloc_tables(test, &small, USER4K_SHIFT), 0); + KUNIT_ASSERT_EQ(test, test_alloc_tables(test, &native, PAGE_SHIFT), 0); + data = test_alloc_page(test); + other = test_alloc_page(test); + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, other); + data_pa = virt_to_phys(data); + other_pa = virt_to_phys(other); + result = kunit_kcalloc(test, TEST_ROUNDS, sizeof(*result), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, result); + + /* One native mapping and four independently described subpages. */ + *test_leaf(&native, TEST_VA) = other_pa | TEST_PROT; + *test_leaf(&small, TEST_VA) = data_pa | TEST_PROT | PTE_RDONLY; + *test_leaf(&small, TEST_VA + USER4K_SIZE) = + (data_pa + USER4K_SIZE) | TEST_PROT; + /* Slot 2 deliberately unmapped. Slot 3 aliases slot 0, writable. */ + *test_leaf(&small, TEST_VA + 3 * USER4K_SIZE) = data_pa | TEST_PROT; + + preempt_disable(); + flags = local_daif_save(); + old_tcr = read_sysreg(tcr_el1); + old_ttbr0 = read_sysreg(ttbr0_el1); + old_ttbr1 = read_sysreg(ttbr1_el1); + old_par = read_sysreg_par(); + /* DS affects both address ranges: never toggle it under a live TTBR1. */ + if ((old_tcr & (TCR_EL1_DS | TCR_EL1_TG1_MASK | TCR_EL1_A1)) != + (TCR_TG1_16K | TCR_EL1_A1)) { + local_daif_restore(flags); + preempt_enable(); + kunit_skip(test, "requires 16K TTBR1, DS=0, and A1=1"); + return; + } + tcr4k = (old_tcr & ~(TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK | + TCR_EL1_EPD0)) | TCR_TG0_4K | TCR_T0SZ(48); + tcr16k = (tcr4k & ~TCR_EL1_TG0_MASK) | TCR_TG0_16K; + + /* No allocations, logging, KUnit assertions or user access in this window. */ + for (round = 0; round < TEST_ROUNDS; round++) { + struct test_result *r = &result[round]; + + test_switch_ttbr0(virt_to_phys(native.level[0]), tcr16k); + for (slot = 0; slot < PROBE_SLOTS; slot++) + r->native[slot] = test_translate(TEST_VA + slot * USER4K_SIZE, false); + test_switch_ttbr0(virt_to_phys(small.level[0]), tcr4k); + r->tcr = read_sysreg(tcr_el1); + r->ttbr1 = read_sysreg(ttbr1_el1); + for (slot = 0; slot < PROBE_SLOTS; slot++) { + unsigned long va = TEST_VA + slot * USER4K_SIZE; + + r->read[slot] = test_translate(va, false); + r->write[slot] = test_translate(va, true); + } + + /* Break-before-make one 4K PTE and invalidate exactly that VA. */ + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), 0); + test_invalidate_4k(TEST_VA + USER4K_SIZE, old_ttbr1); + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), + (other_pa + USER4K_SIZE) | TEST_PROT | PTE_RDONLY); + dsb(ishst); + isb(); + for (slot = 0; slot < PROBE_SLOTS; slot++) + r->replaced[slot] = test_translate(TEST_VA + slot * USER4K_SIZE, true); + r->replaced_read = test_translate(TEST_VA + USER4K_SIZE, false); + asm volatile("at s1e1r, %0" : : "r" (data) : "memory"); + isb(); + r->kernel = read_sysreg_par(); + + test_switch_ttbr0(old_ttbr0, old_tcr); + r->restored_tcr = read_sysreg(tcr_el1); + r->restored_ttbr0 = read_sysreg(ttbr0_el1); + /* Private tables are inactive and their translations have been flushed. */ + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), + (data_pa + USER4K_SIZE) | TEST_PROT); + } + write_sysreg(old_par, par_el1); + isb(); + local_daif_restore(flags); + preempt_enable(); + + for (round = 0; round < TEST_ROUNDS; round++) { + struct test_result *r = &result[round]; + + KUNIT_EXPECT_EQ(test, r->tcr, tcr4k); + KUNIT_EXPECT_EQ(test, r->ttbr1, old_ttbr1); + KUNIT_EXPECT_EQ(test, r->restored_tcr, old_tcr); + KUNIT_EXPECT_EQ(test, r->restored_ttbr0, old_ttbr0); + KUNIT_EXPECT_EQ(test, r->kernel & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->kernel & SYS_PAR_EL1_PA, data_pa); + KUNIT_EXPECT_EQ(test, r->replaced_read & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->replaced_read & SYS_PAR_EL1_PA, + other_pa + USER4K_SIZE); + for (slot = 0; slot < PROBE_SLOTS; slot++) { + KUNIT_EXPECT_EQ(test, r->native[slot] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->native[slot] & SYS_PAR_EL1_PA, + other_pa + slot * USER4K_SIZE); + if (slot == 2) { + KUNIT_EXPECT_EQ(test, + FIELD_GET(SYS_PAR_EL1_FST, r->read[slot]), 7ULL); + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_F, 1ULL); + KUNIT_EXPECT_EQ(test, r->write[slot] & SYS_PAR_EL1_F, 1ULL); + KUNIT_EXPECT_EQ(test, r->replaced[slot] & SYS_PAR_EL1_F, 1ULL); + continue; + } + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_PA, + data_pa + (slot == 1 ? USER4K_SIZE : 0)); + KUNIT_EXPECT_EQ(test, r->write[slot] & SYS_PAR_EL1_F, + slot == 0 ? 1ULL : 0ULL); + KUNIT_EXPECT_EQ(test, r->replaced[slot] & SYS_PAR_EL1_F, + slot == 3 ? 0ULL : 1ULL); + } + } + kunit_info(test, "16K kernel, 4K EL0 translation/permissions; %u switch cycles\n", + TEST_ROUNDS); +} + +static struct folio *test_fault_folio(struct mm_struct *mm, unsigned long address) +{ + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, address, &ptl); + struct folio *folio = NULL; + + if (pte) { + pte_t entry = ptep_get(pte); + + if (pte_present(entry) && !pte_special(entry)) { + folio = page_folio(pfn_to_page(pte_pfn(entry))); + folio_get(folio); + } + pte_unmap_unlock(pte, ptl); + } + return folio; +} + +static bool test_reclaim_swapcache(swp_entry_t entry) +{ + unsigned int tries; + + for (tries = 0; tries < 100; tries++) { + struct folio *folio = swap_cache_get_folio(entry); + LIST_HEAD(folios); + + if (!folio) + return true; + folio_wait_writeback(folio); + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Only the swap cache and reclaim isolation may retain this folio. */ + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + return false; +} + +static void test_swap_tags_set(struct folio *folio) +{ +#ifdef CONFIG_ARM64_MTE + unsigned int offset; + + if (!system_supports_mte()) + return; + if (try_page_mte_tagging(&folio->page)) + mte_clear_page_tags(folio_address(folio)); + for (offset = 0; offset < PAGE_SIZE; offset += MTE_GRANULE_SIZE) { + unsigned long tag = (1UL + offset / SZ_4K) << MTE_TAG_SHIFT; + + asm volatile(".arch_extension memtag\n\tstg %0, [%1]" + : : "r" (tag), "r" (folio_address(folio) + offset) : "memory"); + } + set_page_mte_tagged(&folio->page); +#endif +} + +static bool test_swap_tags_match(struct folio *folio, unsigned int slot) +{ +#ifdef CONFIG_ARM64_MTE + unsigned int offset; + + if (!system_supports_mte()) + return true; + if (!page_mte_tagged(&folio->page)) + return false; + for (offset = slot * SZ_4K; offset < (slot + 1) * SZ_4K; offset += MTE_GRANULE_SIZE) { + unsigned long tag = 0; + + asm volatile(".arch_extension memtag\n\tldg %0, [%1]" + : "+r" (tag) : "r" (folio_address(folio) + offset) : "memory"); + if (((tag >> MTE_TAG_SHIFT) & 0xf) != ((slot + 1) & 0xf)) + return false; + } +#endif + return true; +} + +struct fragment_test_pins { + struct user_page_fragment fragments[2 * TEST_SLOTS]; + unsigned long count; +}; + +static void fragment_test_unpin(void *arg) +{ + struct fragment_test_pins *pins = arg; + + release_user_fragments(pins->fragments, pins->count, true); + pins->count = 0; +} + +static long fragment_test_pin(struct fragment_test_pins *pins, struct mm_struct *mm, + unsigned long start, unsigned long length, unsigned int flags, + unsigned long capacity) +{ + long ret; + + fragment_test_unpin(pins); + ret = pin_user_fragments_remote(mm, start, length, flags, pins->fragments, capacity); + pins->count = ret > 0 ? ret : 0; + return ret; +} + +static void user4k_fragment_pin_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + struct fragment_test_pins *pins; + struct vm_area_struct *vma; + struct mm_struct *mm, *child; + unsigned long size, total = 0; + unsigned int i, nr; + long ret; + + if (native) { + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + vma = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + } else { + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + } + size = mm_page_size(mm); + nr = PAGE_SIZE / size; + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, mm, TEST_VA, 1, FOLL_WRITE | FOLL_NOFAULT, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, TEST_VA + 123, PAGE_SIZE - 200, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + unsigned int offset = i ? i * size : 123; + unsigned int length = size - (!i ? 123 : 0) - (i == nr - 1 ? 77 : 0); + + KUNIT_EXPECT_EQ(test, f->offset, (unsigned long)offset); + KUNIT_EXPECT_EQ(test, f->length, length); + KUNIT_EXPECT_EQ(test, !!f->subpage, !native); + memset(folio_address(f->folio) + f->offset, 0x71 + i, f->length); + total += f->length; + } + KUNIT_EXPECT_EQ(test, total, PAGE_SIZE - 200); + fragment_test_unpin(pins); + child = native ? mm_alloc() : test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, child), 0); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + /* Ordinary references retain shared COW backing, as native FOLL_GET does. */ + ret = get_user_fragments_remote(child, TEST_VA + 123, PAGE_SIZE - 200, + 0, pins->fragments, ARRAY_SIZE(pins->fragments)); + pins->count = ret > 0 ? ret : 0; + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + pte_t parent_pte = test_fault_entry(test, mm, TEST_VA + i * size); + pte_t child_pte = test_fault_entry(test, child, TEST_VA + i * size); + + KUNIT_EXPECT_FALSE(test, f->pinned); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, parent_pte), pte_phys_mm(child, child_pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x71 + i, f->length), NULL); + } + fragment_test_unpin(pins); + /* Read pins must unshare COW mappings without enabling PTE writes. */ + ret = fragment_test_pin(pins, child, TEST_VA + 123, PAGE_SIZE - 200, FOLL_NOFAULT, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, child, TEST_VA + 123, PAGE_SIZE - 200, 0, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + pte_t pte = test_fault_entry(test, child, TEST_VA + i * size); + + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x71 + i, f->length), NULL); + if (!native) { + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(f->subpage)); + if (i) + KUNIT_EXPECT_PTR_NE(test, f->folio, pins->fragments[0].folio); + } + } + fragment_test_unpin(pins); + /* Return a valid partial result when the byte range reaches an unmapped VMA. */ + ret = fragment_test_pin(pins, mm, TEST_VA, PAGE_SIZE + size, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, (long)nr); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 1); + KUNIT_EXPECT_EQ(test, ret, 1L); + KUNIT_EXPECT_EQ(test, pins->fragments[0].length, (unsigned int)size); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, TEST_VA, size, FOLL_LONGTERM, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, 1L); + ret = fragment_test_pin(pins, mm, ULONG_MAX - 15, ULONG_MAX, 0, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +#ifdef CONFIG_BLOCK +struct bio_fragment_test { + struct bio *bio; + struct bio *clone; + struct user_page_fragment held[TEST_SLOTS]; + unsigned int count; +}; + +static void bio_fragment_test_free(void *arg) +{ + struct bio_fragment_test *state = arg; + + if (state->clone) + bio_put(state->clone); + if (state->bio) { + bio_release_pages(state->bio, false); + bio_put(state->bio); + } + release_user_fragments(state->held, state->count, false); +} + +static void user4k_bio_fragments_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct bio_fragment_test *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct iov_iter iter; + unsigned int i, bytes = round_down(PAGE_SIZE - 200, 512); + long nr; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bio fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, bio_fragment_test_free, state), 0); + /* Populate real writable anonymous mappings and retain independent GETs. */ + nr = get_user_fragments_remote(vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, + state->held, ARRAY_SIZE(state->held)); + state->count = nr > 0 ? nr : 0; + KUNIT_ASSERT_EQ(test, nr, (long)TEST_SLOTS); + state->bio = bio_alloc(NULL, 8, REQ_OP_READ, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->bio); + iov_iter_ubuf(&iter, ITER_DEST, (void __user *)(TEST_VA + 123), PAGE_SIZE - 200); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_vecs_to_alloc(&iter, BIO_MAX_VECS); + kthread_unuse_mm(vma->vm_mm); + KUNIT_EXPECT_EQ(test, ret, (int)TEST_SLOTS); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_iter_get_pages(state->bio, &iter, 511); + kthread_unuse_mm(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, state->bio->bi_iter.bi_size, bytes); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), PAGE_SIZE - 200 - bytes); + KUNIT_EXPECT_EQ(test, state->bio->bi_nr_user_fragments, (int)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, state->bio->bi_vcnt, 1); + KUNIT_EXPECT_EQ(test, state->bio->bi_io_vec[0].bv_offset, 123U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 1U); + state->clone = bio_alloc_clone(NULL, state->bio, GFP_KERNEL, &fs_bio_set); + KUNIT_ASSERT_NOT_NULL(test, state->clone); + KUNIT_EXPECT_PTR_EQ(test, state->clone->bi_user_fragments, NULL); + KUNIT_EXPECT_PTR_EQ(test, state->clone->bi_io_vec, state->bio->bi_io_vec); + /* Completion ownership must outlive all originating VMAs. */ + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 1U); + bio_put(state->clone); + state->clone = NULL; + bio_release_pages(state->bio, false); + KUNIT_EXPECT_PTR_EQ(test, state->bio->bi_user_fragments, NULL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 0U); +} + +static void user4k_bio_trim_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, SZ_4K); + struct bio_fragment_test *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct iov_iter iter; + unsigned int i; + long nr; + int ret, refs; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bio fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, bio_fragment_test_free, state), 0); + nr = get_user_fragments_remote(vma->vm_mm, TEST_VA, SZ_4K, FOLL_WRITE, state->held, 1); + state->count = nr > 0 ? nr : 0; + KUNIT_ASSERT_EQ(test, nr, 1L); + refs = folio_ref_count(state->held[0].folio); + state->bio = bio_alloc(NULL, 4, REQ_OP_READ, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->bio); + for (i = 0; i < 1024; i++) { + iov_iter_ubuf(&iter, ITER_DEST, (void __user *)(TEST_VA + SZ_4K - 256), 512); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_iter_get_pages(state->bio, &iter, 511); + kthread_unuse_mm(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 512UL); + KUNIT_EXPECT_EQ(test, state->bio->bi_vcnt, 0); + KUNIT_EXPECT_PTR_EQ(test, state->bio->bi_user_fragments, NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[0].subpage), 0U); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->held[0].folio), refs); + bio_reset(state->bio, NULL, REQ_OP_READ); + } +} +#endif + +static void user4k_fragment_longterm_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct fragment_test_pins *pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + phys_addr_t before; + bool movable; + unsigned int i; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)TEST_SLOTS); + before = PFN_PHYS(folio_pfn(pins->fragments[0].folio)); + movable = !folio_is_longterm_pinnable(pins->fragments[0].folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0x91 + i, pins->fragments[i].length); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, + FOLL_WRITE | FOLL_LONGTERM, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_TRUE(test, folio_is_longterm_pinnable(f->folio)); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(f->subpage), 1U); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x91 + i, f->length), NULL); + } + if (movable) + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(pins->fragments[0].folio)), before); + kunit_info(test, "long-term source movable=%u migrated=%u\n", movable, + PFN_PHYS(folio_pfn(pins->fragments[0].folio)) != before); + fragment_test_unpin(pins); + /* A long-term flag must never silently create an ordinary GET. */ + ret = get_user_fragments_remote(vma->vm_mm, TEST_VA, SZ_4K, FOLL_LONGTERM, + pins->fragments, 1); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void user4k_file_fragments_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *shared, *private; + struct fragment_test_pins *pins; + phys_addr_t shared_phys; + long ret; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, SZ_4K); + private = test_file_vma(test, file, false, false, TEST_VA + SZ_4K, 3 * SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, shared->vm_mm, shared->vm_start + 17, + 3 * SZ_4K - 33, FOLL_WRITE, 8); + KUNIT_ASSERT_EQ(test, ret, 3L); + shared_phys = PFN_PHYS(folio_pfn(pins->fragments[0].folio)); + for (i = 0; i < 3; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_PTR_EQ(test, f->subpage, NULL); + KUNIT_EXPECT_EQ(test, f->offset, (unsigned long)((i + 1) * SZ_4K + (!i ? 17 : 0))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x32 + i, f->length), NULL); + } + /* File pins stay live across PTE removal and mm exit. */ + kunit_release_action(test, user4k_vma_mm_free, shared->vm_mm); + for (i = 0; i < 3; i++) + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(pins->fragments[i].folio)); + fragment_test_unpin(pins); + ret = get_user_fragments_remote(private->vm_mm, private->vm_start, 3 * SZ_4K, + 0, pins->fragments, 8); + pins->count = ret > 0 ? ret : 0; + KUNIT_ASSERT_EQ(test, ret, 3L); + for (i = 0; i < 3; i++) { + KUNIT_EXPECT_PTR_EQ(test, pins->fragments[i].subpage, NULL); + KUNIT_EXPECT_EQ(test, PFN_PHYS(folio_pfn(pins->fragments[i].folio)), shared_phys); + } + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, private->vm_mm, private->vm_start, 3 * SZ_4K, 0, 8); + KUNIT_ASSERT_EQ(test, ret, 3L); + for (i = 0; i < 3; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_NOT_NULL(test, f->subpage); + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(f->folio)), shared_phys); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x32 + i, f->length), NULL); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, private->vm_mm, + private->vm_start + i * SZ_4K))); + } + fragment_test_unpin(pins); +} + +/* Exercise the same implementation used by the arm64 syscall entry. */ +asmlinkage long __arm64_sys_mprotect(const struct pt_regs *regs); + +static long test_mprotect(unsigned long start, unsigned long len, unsigned long prot) +{ + struct pt_regs regs = { .regs = { start, len, prot } }; + + return __arm64_sys_mprotect(®s); +} + +static unsigned long test_copy_word(unsigned long addr, u64 *value, bool write) +{ + unsigned long left; + + pagefault_disable(); + if (write) + left = copy_to_user((void __user *)addr, value, sizeof(*value)); + else + left = copy_from_user(value, (void __user *)addr, sizeof(*value)); + pagefault_enable(); + return left; +} + +static void user4k_mprotect_test(struct kunit *test) +{ + const unsigned long starts[] = { + TEST_VA, SZ_2M - SZ_4K, SZ_1G - SZ_4K, BIT(39) - SZ_4K, TEST_VA, + }; + unsigned int shape; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + for (shape = 0; shape < ARRAY_SIZE(starts); shape++) { + struct file *file = shape == 4 ? test_cache_file(test, PAGE_SIZE, true) : NULL; + struct mm_struct *mm = test_vma_mm(test); + unsigned long start = starts[shape], mapped; + unsigned long flags = MAP_FIXED_NOREPLACE | (file ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS); + phys_addr_t before[4] = { }; + long protect[4] = { }; + unsigned long initial = 0, final = 0, read_ro = 0, write_ro = 0, read_none = 0; + u64 value, ro_value = 0; + unsigned int i; + vm_fault_t fault = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shape == 4) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + kthread_use_mm(mm); + mapped = vm_mmap(file, start, PAGE_SIZE, PROT_READ | PROT_WRITE, flags, 0); + if (IS_ERR_VALUE(mapped)) + goto detach; + for (i = 0; i < 4; i++) { + struct vm_area_struct *vma = test_vma_lookup(mm, start + i * SZ_4K); + + fault |= test_file_fault(vma, start + i * SZ_4K, true); + if (fault & VM_FAULT_ERROR) + goto detach; + before[i] = pte_phys_mm(mm, test_fault_entry(test, mm, start + i * SZ_4K)); + value = 0x88110000 + i; + initial |= test_copy_word(start + i * SZ_4K, &value, true); + } + /* Keep the mm attached across permission changes and cached accesses. */ + protect[0] = test_mprotect(start + SZ_4K, SZ_4K, PROT_READ); + value = 0xaa; + write_ro = test_copy_word(start + SZ_4K, &value, true); + final |= test_copy_word(start, &value, true); + final |= test_copy_word(start + 3 * SZ_4K, &value, true); + protect[1] = test_mprotect(start + 2 * SZ_4K, SZ_4K, PROT_NONE); + read_none = test_copy_word(start + 2 * SZ_4K, &value, false); + read_ro = test_copy_word(start + SZ_4K, &ro_value, false); + protect[2] = test_mprotect(start, PAGE_SIZE, PROT_READ); + protect[3] = test_mprotect(start, PAGE_SIZE, PROT_READ | PROT_WRITE); + for (i = 0; i < 4; i++) { + value = 0x99220000 + i; + final |= test_copy_word(start + i * SZ_4K, &value, true); + } +detach: + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, initial, 0UL); + KUNIT_EXPECT_EQ(test, final, 0UL); + KUNIT_EXPECT_EQ(test, write_ro, (unsigned long)sizeof(u64)); + KUNIT_EXPECT_EQ(test, read_none, (unsigned long)sizeof(u64)); + KUNIT_EXPECT_EQ(test, read_ro, 0UL); + KUNIT_EXPECT_EQ(test, ro_value, 0x88110001ULL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, start + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, protect[i], 0L); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), before[i]); + } + KUNIT_EXPECT_EQ(test, mm->total_vm, (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm->map_count, 1); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +#ifdef CONFIG_NUMA_BALANCING +static void user_subpage_numa_cow_batch_test(struct kunit *test) +{ + unsigned int shift; + + if (!(READ_ONCE(sysctl_numa_balancing_mode) & NUMA_BALANCING_NORMAL)) { + kunit_skip(test, "requires normal NUMA balancing; use numa_balancing=enable"); + return; + } + for (shift = 12; shift < PAGE_SHIFT; shift += 2) { + unsigned long size = 1UL << shift, start = TEST_VA; + struct vm_area_struct *vma; + struct mm_struct *child = test_vma_mm_granule(test, shift); + struct mm_struct *mm = test_vma_mm_granule(test, shift); + phys_addr_t before; + unsigned long marked, mapped; + long ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + /* Use mmap accounting because the child will partially unmap. */ + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, 4 * size, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + i * size, true), + (vm_fault_t)0); + before = pte_phys_mm(mm, test_fault_entry(test, mm, start + size)); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + kthread_use_mm(child); + ret = vm_munmap(start + size, size); + kthread_unuse_mm(child); + KUNIT_ASSERT_EQ(test, ret, 0L); + /* Reuse the now-private slot without replacing its native owner. */ + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + size, true), (vm_fault_t)0); + KUNIT_ASSERT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + size)), before); + kthread_use_mm(mm); + ret = test_mprotect(start, 4 * size, PROT_READ); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Permit local-node hinting, which normally skips a single user. */ + mmget(mm); + mmap_read_lock(mm); + marked = change_prot_numa(vma, start, start + 4 * size); + mmap_read_unlock(mm); + mmput(mm); + KUNIT_EXPECT_EQ(test, marked, 1UL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, start + i * size); + + KUNIT_EXPECT_EQ(test, !!pte_protnone(pte), i == 1); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + } + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} +#endif + +static void user4k_mprotect_cow_batch_test(struct kunit *test) +{ + struct mm_struct *mm, *child; + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + phys_addr_t before[4]; + unsigned int i; + unsigned long mapped; + long ret[2]; + int copied; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, fragment_test_pin(pins, mm, TEST_VA + SZ_4K, SZ_4K, FOLL_WRITE, 8), 1L); + for (i = 0; i < 4; i++) + before[i] = pte_phys_mm(mm, test_fault_entry(test, mm, TEST_VA + i * SZ_4K)); + uprobe_start_dup_mmap(); + copied = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, copied, 0); + kthread_use_mm(mm); + ret[0] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + ret[1] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret[0], 0L); + KUNIT_EXPECT_EQ(test, ret[1], 0L); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), before[i]); + /* One native folio contains both exclusive and shared COW slots. */ + KUNIT_EXPECT_EQ(test, pte_write(pte), i == 1); + } + fragment_test_unpin(pins); +} + +static void user4k_mprotect_accounting_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + unsigned long mapped, committed[4], data[4]; + s64 global[4], before; + long ret[4]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + if (!IS_ERR_VALUE(mapped)) { + ret[0] = test_mprotect(TEST_VA + SZ_4K, SZ_4K, PROT_READ); + committed[0] = mm->committed_user_pages; + data[0] = mm->data_vm; + global[0] = percpu_counter_sum(&vm_committed_as) - before; + ret[1] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + committed[1] = mm->committed_user_pages; + data[1] = mm->data_vm; + global[1] = percpu_counter_sum(&vm_committed_as) - before; + ret[2] = test_mprotect(TEST_VA + 2 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE); + committed[2] = mm->committed_user_pages; + data[2] = mm->data_vm; + global[2] = percpu_counter_sum(&vm_committed_as) - before; + ret[3] = vm_munmap(TEST_VA, PAGE_SIZE); + committed[3] = mm->committed_user_pages; + data[3] = mm->data_vm; + global[3] = percpu_counter_sum(&vm_committed_as) - before; + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + for (i = 0; i < 4; i++) { + unsigned long expected = i == 0 ? TEST_SLOTS - 1 : i == 2 ? 1 : 0; + + KUNIT_EXPECT_EQ(test, ret[i], 0L); + KUNIT_EXPECT_EQ(test, committed[i], expected); + KUNIT_EXPECT_EQ(test, data[i], expected); + KUNIT_EXPECT_EQ(test, global[i], expected ? 1LL : 0LL); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 0); +} + +static void user4k_fragment_permissions_test(struct kunit *test) +{ + unsigned long start = TEST_VA + SZ_4K; + struct vm_area_struct *vma = test_fault_vma(test, start, SZ_4K); + struct fragment_test_pins *pins; + struct user_page_fragment *fragment; + struct mm_struct *mm; + spinlock_t *ptl; + pte_t *ptep; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + fragment = &pins->fragments[0]; + { + struct page *page = NULL; + int locked = 1; + + mmap_read_lock(mm); + ret = pin_user_pages_remote(mm, start, 1, 0, &page, &locked); + if (locked) + mmap_read_unlock(mm); + if (ret == 1) + unpin_user_page(page); + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + } + ret = fragment_test_pin(pins, mm, start + 17, 31, 0, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_TRUE(test, is_zero_folio(fragment->folio)); + KUNIT_EXPECT_PTR_EQ(test, fragment->subpage, NULL); + /* Zero-page PTEs alias the first physical 4K despite this VA's slot 1. */ + KUNIT_EXPECT_EQ(test, fragment->offset, 17UL); + KUNIT_EXPECT_EQ(test, fragment->length, 31U); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment->folio) + fragment->offset, + 0, fragment->length), NULL); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_NOT_NULL(test, fragment->subpage); + KUNIT_EXPECT_EQ(test, fragment->offset, SZ_4K + 17UL); + memset(folio_address(fragment->folio) + fragment->offset, 0xa4, fragment->length); + fragment_test_unpin(pins); + mmap_write_lock(mm); + vm_flags_clear(vma, VM_WRITE); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + ptep = test_mm_lookup(mm, start, &ptl); + if (ptep) { + ptep_set_wrprotect(mm, start, ptep); + pte_unmap_unlock(ptep, ptl); + } + flush_tlb_range(vma, start, start + SZ_4K); + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE, 8); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE | FOLL_FORCE, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, mm, start))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment->folio) + fragment->offset, + 0xa4, fragment->length), NULL); + fragment_test_unpin(pins); + mmap_write_lock(mm); + vm_flags_clear(vma, VM_READ); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_unlock(mm); + ret = fragment_test_pin(pins, mm, start + 17, 31, 0, 8); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_FORCE, 8); + KUNIT_EXPECT_EQ(test, ret, 1L); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, start, 1, FOLL_GET, 8); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void test_subpage_unpin(void *arg) +{ + mm_subpage_unpin(arg); +} + +static void user4k_slot_pin_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *child; + struct mm_subpage *slots[4]; + struct folio *folio, *dst; + unsigned int i; + vm_fault_t fault; + spinlock_t *ptl; + pte_t *ptep, pte; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + slots[i] = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slots[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slots[i]), 0); + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(slots[i])); + memset(folio_address(mm_subpage_folio(slots[i])) + i * SZ_4K, 0x61 + i, SZ_4K); + } + folio = mm_subpage_folio(slots[1]); + test_swap_tags_set(folio); + for (i = 0; i < 2; i++) { + ptep = test_mm_lookup(vma->vm_mm, TEST_VA + SZ_4K, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = mm_subpage_pin(slots[1]); + pte_unmap_unlock(ptep, ptl); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_unpin, slots[1]), 0); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 2U); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_FALSE(test, mm_subpage_try_share(slots[1])); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + /* Finish even unexpected success so a failing assertion cannot leak metadata. */ + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + KUNIT_EXPECT_EQ(test, ret, -EBUSY); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) { + struct mm_subpage *copy; + + pte = test_fault_entry(test, child, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, pte) == mm_subpage_phys(slots[i]), i != 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), i == 1 ? 1 : 2); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(slots[i]), i == 1); + copy = mm_subpage_get_from_phys(pte_phys_mm(child, pte)); + KUNIT_ASSERT_NOT_NULL(test, copy); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(copy), i == 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(copy)) + i * SZ_4K, + 0x61 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(copy), i)); + mm_subpage_put(copy); + } + /* A read pin on a shared slot must request unsharing, not pin shared COW data. */ + ptep = test_mm_lookup(vma->vm_mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = mm_subpage_pin(slots[0]); + pte_unmap_unlock(ptep, ptl); + if (!ret) + mm_subpage_unpin(slots[0]); + KUNIT_EXPECT_EQ(test, ret, -EAGAIN); + /* Re-enabling writes must retain the exact backing of an exclusive pinned slot. */ + ptep = test_mm_lookup(vma->vm_mm, TEST_VA + SZ_4K, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ptep_set_wrprotect(vma->vm_mm, TEST_VA + SZ_4K, ptep); + pte_unmap_unlock(ptep, ptl); + flush_tlb_range(vma, TEST_VA + SZ_4K, TEST_VA + 2 * SZ_4K); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA + SZ_4K, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), mm_subpage_phys(slots[1])); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + *((u8 *)folio_address(folio) + SZ_4K) = 0xab; + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(child, pte)), (u8)0x62); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[1]), 0); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 2U); + KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + SZ_4K), (u8)0xab); + kunit_release_action(test, test_subpage_unpin, slots[1]); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 1U); + kunit_release_action(test, test_subpage_unpin, slots[1]); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 0U); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); +} + +#ifdef CONFIG_MIGRATION +static struct folio *test_migration_alloc(struct folio *src, unsigned long private) +{ + return folio_alloc(GFP_HIGHUSER_MOVABLE, 0); +} + +asmlinkage long __arm64_sys_madvise(const struct pt_regs *regs); + +static long test_madvise(unsigned long addr, unsigned long size, unsigned long behavior) +{ + struct pt_regs regs = { .regs = { addr, size, behavior } }; + + return __arm64_sys_madvise(®s); +} + +static void user_granule_large_vma_split_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14 }, i; + + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) || + (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { + kunit_skip(test, "requires THP boundary splitting and borrowed-mm execution"); + return; + } + for (i = 0; i < ARRAY_SIZE(shifts); i++) { + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long size = 1UL << shifts[i]; + unsigned long base = 4UL * HPAGE_PMD_SIZE, mapped; + pte_t pte; + long ret; + + if (shifts[i] >= PAGE_SHIFT) + continue; + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shifts[i]), 0); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, base, 2UL * HPAGE_PMD_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE | MAP_NORESERVE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, base); + vma = test_vma_lookup(mm, base); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, base, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, base + size, true) & VM_FAULT_ERROR, 0U); + /* The generic VMA split probes the native THP boundary for this range. */ + kthread_use_mm(mm); + ret = test_mprotect(base + size, size, PROT_READ); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, mm, base + size); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + pte = test_fault_entry(test, mm, base); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + kthread_use_mm(mm); + ret = test_mprotect(base + size, size, PROT_READ | PROT_WRITE); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0L); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user_granule_brk_extent_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int count = PAGE_SHIFT > 14 ? 3 : 2, i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm brk fixture requires hardware opt-in"); + return; + } + for (i = 0; i < count; i++) { + struct mm_struct *mm = mm_alloc(); + struct vm_area_struct *vma; + unsigned long size = 1UL << shifts[i], neighbor; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shifts[i]), 0); + kthread_use_mm(mm); + ret = vm_brk_flags(TEST_VA, size + 1, false); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_start, TEST_VA); + KUNIT_EXPECT_EQ(test, vma->vm_end, TEST_VA + 2 * size); + KUNIT_EXPECT_EQ(test, mm->total_vm, 2UL); + /* FEX reserves exactly this page after its initial ELF/BSS break. */ + kthread_use_mm(mm); + neighbor = vm_mmap(NULL, TEST_VA + 2 * size, size, PROT_NONE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, neighbor, TEST_VA + 2 * size); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_droppable_reclaim_test(struct kunit *test) +{ + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm droppable fixture requires hardware opt-in"); + return; + } + /* Dirty discard, retained GET, retained PIN, migration, zero-page COW. */ + for (mode = 0; mode < 5; mode++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct user_page_fragment held = {}; + struct folio *folio; + unsigned int i, succeeded = 0; + unsigned long mapped; + long ret; + pte_t pte; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_DROPPABLE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_TRUE(test, vma->vm_flags & VM_WIPEONFORK); + KUNIT_EXPECT_TRUE(test, vma->vm_flags & VM_DONTDUMP); + if (mode == 4) { + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_TRUE(test, is_zero_pfn(pte_pfn(pte))); + } + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + if (mode != 4) + for (i = 1; i < PAGE_SIZE / SZ_4K; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + i * SZ_4K, + true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + memset(folio_address(folio), 0xa7, PAGE_SIZE); + folio_put(folio); + if (mode == 1 || mode == 2) { + ret = mode == 1 ? get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1) : + pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + if (mode == 3) { + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 1L); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_EQ(test, ret, 0L); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + } + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + if (mode == 1 || mode == 2) { + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(unsigned char *)folio_address(folio), 0xa7); + release_user_fragments(&held, 1, false); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + } + folio_put(folio); + for (i = 0; i < (mode == 4 ? 1 : PAGE_SIZE / SZ_4K); i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0L); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, PAGE_SIZE), NULL); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_lazyfree_test(struct kunit *test) +{ + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm advice requires hardware opt-in"); + return; + } + /* Partial, redirty, retained GET, typed write, all leaves, migration. */ + for (mode = 0; mode < 10; mode++) { + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *mm; + struct user_page_fragment held = {}; + struct folio *folio; + struct mm_subpage *slot; + unsigned int i, succeeded = 0; + long ret; + pte_t pte; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + if (mode == 6) { + ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + kthread_use_mm(mm); + ret = test_madvise(TEST_VA, + (mode == 4 || mode == 9) ? PAGE_SIZE : SZ_4K, MADV_FREE); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_EQ(test, pte_dirty(pte), mode == 6); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_is_lazyfree(slot), mode != 6); + mm_subpage_put(slot); + if (mode == 1) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + + if (mode == 2 || mode == 3 || mode == 8) { + ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, + mode == 3 ? FOLL_WRITE : 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + if (mode == 3 || mode == 8) { + memset(folio_address(held.folio) + held.offset, 0xd3, SZ_4K); + release_user_fragments(&held, 1, true); + } + } + if (mode == 7) { + ret = pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + if (mode == 5 || mode == 9) { + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 1L); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_EQ(test, ret, 0L); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + } + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_none(pte), (mode == 4 || mode == 9) || + (i == 0 && (mode == 0 || mode == 5))); + } + if (mode == 2 || mode == 6 || mode == 7) + release_user_fragments(&held, 1, false); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_mprotect_migration_test(struct kunit *test) +{ + unsigned int file_case; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + for (file_case = 0; file_case < 2; file_case++) { + struct mm_struct *mm = test_vma_mm(test); + struct file *file = file_case ? test_cache_file(test, PAGE_SIZE, true) : NULL; + struct vm_area_struct *vma; + struct folio *src, *dst; + unsigned long mapped; + unsigned int i; + int prepared, unmapped = -1; + long protected = -EINVAL; + bool offsets = true; + phys_addr_t phys; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (file_case) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + kthread_use_mm(mm); + mapped = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | (file ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + src = test_fault_folio(mm, TEST_VA); + if (!src) { + folio_put(dst); + KUNIT_FAIL(test, "missing migration source"); + return; + } + phys = PFN_PHYS(folio_pfn(src)); + folio_lock(src); + folio_lock(dst); + prepared = mm_subpage_migrate_prepare(src, dst); + if (!prepared) { + try_to_migrate(src, TTU_SYNC); + unmapped = folio_mapcount(src); + kthread_use_mm(mm); + protected = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + kthread_unuse_mm(mm); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + offsets &= softleaf_is_migration(softleaf_from_pte(pte)) && + !softleaf_is_migration_write(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + } + remove_migration_ptes(src, src, 0); + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + folio_put(dst); + folio_put(src); + KUNIT_ASSERT_EQ(test, prepared, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_EQ(test, protected, 0L); + KUNIT_EXPECT_TRUE(test, offsets); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys + i * SZ_4K); + } + } +} + +/* Independent roots and VA offsets must not change physical slot identity. */ +static void user4k_nonlinear_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct mm_struct *child = test_vma_mm(test); + struct mm_subpage *slot; + struct folio *folio, *dst; + phys_addr_t phys; + vm_flags_t flags; + unsigned int succeeded = 0, side; + int ret, unmapped = -1; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, child); + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + /* Unlike the lightweight rmap fixture, real LRU migration needs a charge. */ + ret = mem_cgroup_charge(folio, a->mm, GFP_KERNEL); + if (ret) { + mm_subpage_put(slot); + KUNIT_FAIL(test, "cannot charge nonlinear fixture folio"); + return; + } + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + phys = mm_subpage_phys(slot); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + /* Same native folio, different root and different VA/physical offset. */ + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), folio); + memset(folio_address(folio), 0x53, SZ_4K); + memset(folio_address(folio) + 3 * SZ_4K, 0xa7, SZ_4K); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, b->mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 2); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, + test_fault_entry(test, child, TEST_VA + SZ_4K)), phys + 3 * SZ_4K); + kunit_release_action(test, user4k_vma_mm_free, child); + + /* Rollback must find migration entries through both roots. */ + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + folio_get(folio); + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + if (!ret) { + try_to_migrate(folio, TTU_SYNC); + unmapped = folio_mapcount(folio); + remove_migration_ptes(folio, folio, 0); + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 2); + + /* Successful migration exercises copied identities in the new owner. */ + folio_add_lru(folio); + lru_add_drain_all(); + folio_get(folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (ret) { + KUNIT_FAIL(test, "nonlinear migration failed: %d", ret); + return; + } + /* Refresh fixture references before assertions/cleanup after migration. */ + for (side = 0; side < 2; side++) { + struct anon_slot_fixture *f = side ? b : a; + pte_t pte = test_fault_entry(test, f->mm, TEST_VA + side * SZ_4K); + + f->slots[side] = NULL; + f->mapped[side] = false; + if (!pte_present(pte)) + continue; + slot = mm_subpage_get_from_phys(pte_phys_mm(f->mm, pte)); + if (!slot) + continue; + f->slots[side] = slot; + f->mapped[side] = true; + mm_subpage_put(slot); + } + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + for (side = 0; side < 2; side++) { + struct anon_slot_fixture *f = side ? b : a; + + slot = f->slots[side]; + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), side ? 3U * SZ_4K : 0U); + KUNIT_EXPECT_NE(test, mm_subpage_phys(slot), phys + (side ? 3 * SZ_4K : 0)); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), side ? 0xa7 : 0x53, SZ_4K), NULL); + } +} + +static void user4k_migration_entries_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *child = test_vma_mm(test); + struct folio *src, *dst; + unsigned int i, side; + vm_fault_t fault; + bool encoded = true, copied = true, exact = true; + int ret, fork_ret = -EINVAL, unmapped = -1; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + src = test_fault_folio(vma->vm_mm, TEST_VA); + if (!src) { + folio_put(dst); + KUNIT_FAIL(test, "missing source folio"); + return; + } + folio_lock(src); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(src, dst); + if (!ret) { + try_to_migrate(src, TTU_SYNC); + unmapped = folio_mapcount(src); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + struct rmap_walk_range range = { + .address = TEST_VA + i * SZ_4K, + .subpage = true, + .subpage_offset = i * SZ_4K, + }; + DEFINE_FOLIO_RMAP_WALK(walk, src, vma, range, + PVMW_SYNC | PVMW_MIGRATION); + unsigned int count = 0; + + encoded &= softleaf_is_migration_write(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + while (page_vma_mapped_walk(&walk)) { + exact &= walk.address == TEST_VA + i * SZ_4K; + count++; + } + exact &= count == 1; + } + uprobe_start_dup_mmap(); + fork_ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + if (!fork_ret) { + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + copied &= softleaf_is_migration_read(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + } + } + } + /* Drop one migration entry while the other mm still retains it. */ + mmap_read_lock(vma->vm_mm); + zap_vma_range(vma, TEST_VA + SZ_4K, SZ_4K); + mmap_read_unlock(vma->vm_mm); + remove_migration_ptes(src, src, 0); + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + folio_put(dst); + folio_put(src); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, fork_ret, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_TRUE(test, encoded); + KUNIT_EXPECT_TRUE(test, copied); + KUNIT_EXPECT_TRUE(test, exact); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), side ? (unsigned long)TEST_SLOTS : TEST_SLOTS - 1UL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + struct mm_subpage *slot; + + if (!side && i == 1) { + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + continue; + } + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), i == 1 ? 1 : 2); + mm_subpage_put(slot); + } + } +} + +static struct folio *test_file_migration_alloc(struct folio *src, unsigned long private) +{ + return folio_alloc(GFP_HIGHUSER_MOVABLE, folio_order(src)); +} + +static void user64k_file_migration_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 6; mode++) { + bool shmem = mode & 1, fail = mode >= 2, pin = mode >= 4; + bool isolated; + unsigned long len = shmem ? SZ_64K : 2 * SZ_64K; + unsigned long base = shmem ? SZ_64K : 0, offset; + unsigned int nr = len / PAGE_SIZE, succeeded = 0; + unsigned int maps = 2 * nr + len / SZ_4K - 1; + struct file *file = shmem ? test_cache_file(test, 2 * SZ_64K, true) : + test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + struct vm_area_struct *coarse, *native, *small; + struct folio *folio; + struct user_page_fragment *fragment = NULL; + phys_addr_t old_phys, phys; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, len, base); + native = test_file_vma(test, file, true, true, TEST_VA, len, base); + /* Shifted file offset and a small-page table boundary in the alias. */ + small = test_file_vma(test, file, false, true, SZ_2M - SZ_4K, + len - SZ_4K, base + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + for (offset = 0; offset < len; offset += SZ_64K) + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + offset, shmem) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < len; offset += PAGE_SIZE) + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + offset, shmem) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < len - SZ_4K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + offset, + shmem) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, base >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), len); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + old_phys = page_to_phys(&folio->page); + if (pin) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_promote_unpin, fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + small->vm_end - SZ_4K + 37, 1, 0, fragment, 1), 1L); + KUNIT_EXPECT_PTR_EQ(test, fragment->folio, folio); + KUNIT_EXPECT_EQ(test, fragment->offset, len - SZ_4K + 37); + } + isolated = isolate_folio_to_list(folio, &folios); + /* Retain either a GET or a typed PIN to force migration rollback. */ + if (!fail || pin || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_file_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (pin) { + KUNIT_EXPECT_EQ(test, page_to_phys(&fragment->folio->page), old_phys); + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), + (u8)(0x31 + (base + len - SZ_4K) / SZ_4K)); + kunit_release_action(test, test_promote_unpin, fragment); + } else if (fail) { + folio_put(folio); + } + KUNIT_EXPECT_EQ_MSG(test, ret, fail ? 1 : 0, "mode=%u", mode); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : nr); + folio = filemap_get_folio(file->f_mapping, base >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + phys = page_to_phys(&folio->page); + KUNIT_EXPECT_EQ(test, phys == old_phys, fail); + KUNIT_EXPECT_EQ(test, folio_size(folio), len); + KUNIT_EXPECT_EQ(test, folio->index, base >> PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse->vm_mm, + shmem ? MM_SHMEMPAGES : MM_FILEPAGES), (long)(len / SZ_64K)); + for (offset = 0; offset < len; offset += SZ_4K) { + pte_t pte; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0x31 + (base + offset) / SZ_4K, SZ_4K), NULL); + if (offset) { + pte = test_fault_entry(test, small->vm_mm, + small->vm_start + offset - SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, pte), phys + offset); + } + if (!(offset % PAGE_SIZE)) { + pte = test_fault_entry(test, native->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(native->vm_mm, pte), phys + offset); + } + if (!(offset % SZ_64K)) { + pte = test_fault_entry(test, coarse->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse->vm_mm, pte), phys + offset); + } + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); + } +} + +static void user64k_anon_migration_test(struct kunit *test) +{ + unsigned int mode, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_file_enabled(test)) + return; + for (mode = 0; mode < 5; mode++) { + bool forked = mode & 1, fail = mode >= 2, pin = mode == 4; + struct vm_area_struct *vma, *cvma = NULL; + struct mm_struct *child = NULL; + struct user_page_fragment *fragment = NULL; + struct folio *folio; + phys_addr_t old_phys, phys; + unsigned int i, succeeded = 0; + pte_t pte; + bool isolated; + int ret; + LIST_HEAD(folios); + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, true) & + VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + old_phys = pte_phys_mm(vma->vm_mm, pte); + folio = page_folio(pfn_to_page(PHYS_PFN(old_phys))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(folio, i)), 0x51 + i, PAGE_SIZE); + if (forked) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + } + if (pin) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_promote_unpin, fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(vma->vm_mm, + TEST_VA + SZ_64K - 1, 1, FOLL_WRITE, fragment, 1), 1L); + KUNIT_EXPECT_PTR_EQ(test, fragment->folio, folio); + KUNIT_EXPECT_EQ(test, fragment->offset, (unsigned int)SZ_64K - 1); + } + lru_add_drain_all(); + folio_get(folio); + isolated = isolate_folio_to_list(folio, &folios); + if (!fail || pin || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_file_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (pin) + kunit_release_action(test, test_promote_unpin, fragment); + else if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ_MSG(test, ret, fail ? 1 : 0, "mode=%u", mode); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : nr); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), !forked); + phys = pte_phys_mm(vma->vm_mm, pte); + KUNIT_EXPECT_EQ(test, phys == old_phys, fail); + folio = page_folio(pfn_to_page(PHYS_PFN(phys))); + folio_get(folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + expect_coarse_anon_state(test, folio, forked ? 2 : 1, !forked); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr * (forked ? 2 : 1) + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + if (forked) { + struct folio *copied; + + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, pte), phys); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1L); + /* A restored shared migration entry must still perform full-leaf COW. */ + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_64K - 1, + true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), phys); + copied = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(child, pte)))); + expect_coarse_anon_state(test, copied, 1, true); + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(copied, i)), + 0x51 + i, PAGE_SIZE), NULL); + memset(folio_address(copied), 0xa7, SZ_64K); + kunit_release_action(test, user4k_vma_mm_free, child); + } + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(folio, i)), + 0x51 + i, PAGE_SIZE), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + } +} + +static void user4k_file_migration_test(struct kunit *test) +{ + unsigned int fail; + + for (fail = 0; fail < 2; fail++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *small, *native; + struct folio *folio; + phys_addr_t old_phys; + unsigned int i, succeeded = 0; + int ret; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, SZ_2M - SZ_4K, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + i * SZ_4K, + true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { [0 ... TEST_SLOTS - 1] = 1 }, 1); + old_phys = PFN_PHYS(folio_pfn(folio)); + isolated = isolate_folio_to_list(folio, &folios); + if (!fail || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ(test, ret, fail ? 1 : 0); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : 1U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { [0 ... TEST_SLOTS - 1] = 1 }, 1); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, small->vm_mm, small->vm_start + i * SZ_4K); + + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + PFN_PHYS(folio_pfn(folio)) + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry) == old_phys + i * SZ_4K, !!fail); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, pte_phys_mm(native->vm_mm, + test_fault_entry(test, native->vm_mm, TEST_VA)), PFN_PHYS(folio_pfn(folio))); + /* File reclaim must visit every 4K alias, including across PTE tables. */ + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + KUNIT_EXPECT_FALSE(test, folio_mapped(folio)); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 0, 0, 0 }, 0); + folio_unlock(folio); + folio_put(folio); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, + small->vm_start + i * SZ_4K))); + } + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + i * SZ_4K, + false) & VM_FAULT_ERROR, 0U); + } +} + +static void user4k_migration_test(struct kunit *test) +{ + const unsigned long boundaries[] = { + SZ_2M - SZ_4K, SZ_2M, SZ_1G - SZ_4K, SZ_1G, + BIT(39) - SZ_4K, BIT(39), + }; + unsigned int mode; + + /* Dense/sparse, forks, refcount rollback, batched TLBI and table edges. */ + for (mode = 0; mode < 16; mode++) { + bool edge = mode >= 6 && mode < 12; + bool locked = mode >= 12; + bool sparse = mode == 1 || edge || mode == 13; + bool forked = mode == 2 || mode == 4 || edge || mode == 14; + bool fail = mode == 3 || mode == 4 || mode == 15; + unsigned long start = edge ? boundaries[mode - 6] : + TEST_VA + (sparse ? SZ_4K : 0); + unsigned int nr = sparse ? 1 : 4, i, side, succeeded = 0; + struct vm_area_struct *vma = test_fault_vma(test, start, nr * SZ_4K); + struct mm_struct *child = NULL; + struct folio *folio; + phys_addr_t old_phys; + vm_fault_t fault; + int ret; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, vma); + if (locked) { + mmap_write_lock(vma->vm_mm); + vm_flags_set(vma, VM_LOCKED); + vma->vm_mm->locked_vm = nr; + mmap_write_unlock(vma->vm_mm); + } + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, start, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + old_phys = PFN_PHYS(folio_pfn(folio)); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x41 + i + mode, SZ_4K); + test_swap_tags_set(folio); + folio_put(folio); + if (forked) { + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + } + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + /* An extra native reference forces the normal migration rollback. */ + if (!fail || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + mode == 5 ? MIGRATE_ASYNC : MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ(test, ret, fail ? 1 : 0); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : 1U); + for (side = 0; side <= (unsigned int)forked; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + for (i = 0; i < nr; i++) { + unsigned int index = sparse ? offset_in_page(start) / SZ_4K : i; + pte_t pte = test_fault_entry(test, mm, start + i * SZ_4K); + struct mm_subpage *slot; + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) == old_phys + index * SZ_4K, fail); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), !forked); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), index * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), forked ? 2 : 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + index * SZ_4K, + 0x41 + index + mode, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, index)); + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)nr); + } + if (locked) { + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + if (!fail) + KUNIT_EXPECT_EQ(test, folio->mlock_count, nr); + folio_put(folio); + } + if (child) + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } +} +struct fragment_pin_worker { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready; + unsigned long checks; + int error; +}; + +static int fragment_pin_worker_run(void *arg) +{ + struct fragment_pin_worker *worker = arg; + struct user_page_fragment fragments[4]; + + while (!kthread_should_stop()) { + long nr = pin_user_fragments_remote(worker->mm, TEST_VA, PAGE_SIZE, + 0, fragments, ARRAY_SIZE(fragments)); + unsigned int i; + + if (nr != 4) { + worker->error = nr < 0 ? nr : -EIO; + } else { + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &fragments[i]; + + if (memchr_inv(folio_address(f->folio) + f->offset, 0x51 + i, f->length)) + worker->error = -EILSEQ; + worker->checks++; + } + } + if (nr > 0) + release_user_fragments(fragments, nr, false); + complete_all(&worker->ready); + if (worker->error) + break; + cond_resched(); + } + return 0; +} + +static void fragment_pin_worker_stop(void *arg) +{ + struct fragment_pin_worker *worker = arg; + + if (worker->task) { + kthread_stop(worker->task); + worker->task = NULL; + } +} + +static void user4k_fragment_pin_race_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct fragment_test_pins *pins; + struct fragment_pin_worker *workers; + unsigned int i, round, migrated = 0, busy = 0; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 4); + KUNIT_ASSERT_EQ(test, ret, 4L); + for (i = 0; i < 4; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0x51 + i, pins->fragments[i].length); + fragment_test_unpin(pins); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].mm = vma->vm_mm; + init_completion(&workers[i].ready); + workers[i].task = kthread_run(fragment_pin_worker_run, &workers[i], "user4k-pin-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "pin worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_pin_worker_stop, &workers[i]), 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + } + for (round = 0; round < 32; round++) { + struct mm_struct *child = test_vma_mm(test); + unsigned long addr = TEST_VA + (round % 4) * SZ_4K; + struct vm_area_struct *cvma; + struct folio *folio; + unsigned int succeeded = 0; + vm_fault_t fault; + struct user_page_fragment write_fragment; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + cvma = test_vma_lookup(child, addr); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + ret = pin_user_fragments_remote(child, addr, 1, FOLL_WRITE, &write_fragment, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + *((u8 *)folio_address(write_fragment.folio) + write_fragment.offset) = 0xa0 + round; + release_user_fragments(&write_fragment, 1, true); + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + if (ret) { + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_ASYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_TRUE(test, ret == 0 || ret == 1); + migrated += succeeded; + busy += ret == 1; + } + kunit_release_action(test, user4k_vma_mm_free, child); + } + for (i = 0; i < 2; i++) { + fragment_pin_worker_stop(&workers[i]); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_GE(test, workers[i].checks, 4UL); + } + kunit_info(test, "32 fork/COW/exit cycles against two pin workers; migrations=%u not_migrated=%u\n", + migrated, busy); +} + +#endif + +static int test_swap_control(const char *operation) +{ + char *argv[] = { "/init", (char *)operation, NULL }; + char *envp[] = { NULL }; + + return call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC); +} + +static void test_reenable_swap(void *arg) +{ + bool *disabled = arg; + + if (*disabled) + test_swap_control("--swapon-helper"); +} + +/* Exclusive slots must remain packed after read swap-in and later writes. */ +static void test_user4k_swap_exclusive(struct kunit *test, unsigned int mode) +{ + struct vm_area_struct *vma; + struct folio *folio; + struct mm_subpage *held = NULL; + bool *disabled; + phys_addr_t restored[TEST_SLOTS]; + swp_entry_t entry; + vm_fault_t fault; + unsigned int i; + int ret, mapped, hold_err = 0; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0xc0 + i, SZ_4K); + test_swap_tags_set(folio); + if (mode == 5) { + held = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(folio)) + 2 * SZ_4K); + if (held) { + hold_err = kunit_add_action_or_reset(test, test_subpage_put, held); + if (hold_err) + held = NULL; + } + } + ret = folio_alloc_swap(folio); + entry = folio->swap; + if (!ret) + try_to_unmap(folio, TTU_SYNC); + mapped = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + if (mode == 5) { + KUNIT_ASSERT_EQ(test, hold_err, 0); + KUNIT_ASSERT_NOT_NULL(test, held); + } + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(pte))); + KUNIT_EXPECT_TRUE(test, pte_swp_exclusive(pte)); + } + if (mode >= 3) { + struct mm_struct *child = test_vma_mm(test); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + } + } + if (mode == 1 || mode == 2 || mode == 4 || mode == 6) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + if (mode == 2 || mode == 6) { + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, *disabled); + } + for (i = 0; i < TEST_SLOTS; i++) { + struct mm_subpage *slot; + unsigned long address = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + restored[i] = pte_phys_mm(vma->vm_mm, pte); + slot = mm_subpage_get_from_phys(restored[i]); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(slot), !(held && i == 2)); + KUNIT_EXPECT_EQ(test, restored[i], restored[0] + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xc0 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(slot), i)); + mm_subpage_put(slot); + } + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long address = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + if (held && i == 2) { + KUNIT_EXPECT_NE(test, pte_phys_mm(vma->vm_mm, pte), restored[i]); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), 0xdd, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(held)) + + mm_subpage_offset(held), 0xc2, SZ_4K), NULL); + } else { + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), restored[i]); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0UL); + if (*disabled) { + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_EXPECT_EQ(test, ret, 0); + } +} + +static void user4k_swap_exclusive_cached_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 0); +} + +static void user4k_swap_exclusive_disk_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 1); +} + +static void user4k_swap_exclusive_swapoff_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 2); +} + +static void user4k_swap_reuse_cached_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 3); +} + +static void user4k_swap_reuse_disk_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 4); +} + +static void user4k_swap_reuse_get_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 5); +} + +static void user4k_swap_reuse_swapoff_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 6); +} + +static void test_user4k_swap_roundtrip(struct kunit *test, unsigned int mode, u8 base) +{ + const bool evict = mode != 0; + struct vm_area_struct *parent, *cvma; + struct mm_struct *child; + struct folio *folio; + swp_entry_t entry = { 0 }; + unsigned int i, count; + int err, mapped; + vm_fault_t fault; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "rerun this suite after enabling the disposable guest swap disk"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, base + i, SZ_4K); + test_swap_tags_set(folio); + if (mode == 2) { + LIST_HEAD(folios); + + folio_unlock(folio); + err = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!err) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Let the normal reclaim path allocate swap and unmap all slots. */ + reclaim_pages(&folios); + mapped = 0; + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + mapped += pte_present(pte); + } + entry = softleaf_from_pte(pte); + } else { + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC | (evict ? TTU_BATCH_FLUSH : 0)); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + try_to_unmap_flush(); + folio_put(folio); + } + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_ANONPAGES), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_swp_exclusive(pte)); + } + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, (unsigned int)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + if (evict) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + + /* Fork while all slots are nonpresent, preserving each byte offset. */ + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, 2U * TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 2U); + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long address = TEST_VA + ((i * 5 + 2) % TEST_SLOTS) * SZ_4K; + struct mm_subpage *slot; + spinlock_t *ptl; + pte_t *ptep; + bool intact = false; + + pte = test_fault_entry(test, parent->vm_mm, address); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + pte = test_fault_entry(test, child, address); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), ((i * 5 + 2) % TEST_SLOTS) * (unsigned int)SZ_4K); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + KUNIT_EXPECT_EQ(test, !!(fault & VM_FAULT_MAJOR), evict && !i); + ptep = test_mm_lookup(parent->vm_mm, address, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + pte = ptep_get(ptep); + slot = pte_present(pte) ? mm_subpage_get_from_phys(pte_phys_mm(parent->vm_mm, pte)) : NULL; + if (slot) { + intact = !memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), base + ((i * 5 + 2) % TEST_SLOTS), SZ_4K) && + test_swap_tags_match(mm_subpage_folio(slot), ((i * 5 + 2) % TEST_SLOTS)); + mm_subpage_put(slot); + } + pte_unmap_unlock(ptep, ptl); + KUNIT_EXPECT_TRUE(test, intact); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), 0UL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + /* A write fault must swap in and then COW just the child's selected slot. */ + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + SZ_4K, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, 1)); + memset(folio_address(folio) + SZ_4K, 0xa7, SZ_4K); + folio_put(folio); + folio = test_fault_folio(parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), TEST_SLOTS - 1UL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), i == 1 ? 0U : 1U); + /* Exit with three swapped slots, then tear down the restored parent. */ + kunit_release_action(test, user4k_vma_mm_free, child); + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, 0U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 0U); + kunit_release_action(test, user4k_vma_mm_free, parent->vm_mm); +} + +static void user4k_swap_mremap_test(struct kunit *test) +{ + struct mm_struct *mm; + struct folio *folio; + unsigned long old = TEST_VA, new = SZ_1G - SZ_4K, mapped, moved; + pte_t saved[TEST_SLOTS]; + u8 byte = 0x55; + int ret; + unsigned int i; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, old, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old, &byte, 1, FOLL_WRITE), 1); + folio = test_fault_folio(mm, old); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x81 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < TEST_SLOTS; i++) { + saved[i] = test_fault_entry(test, mm, old + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(saved[i]))); + } + kthread_use_mm(mm); + moved = test_mremap(old, PAGE_SIZE, PAGE_SIZE, MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte_t entry = test_fault_entry(test, mm, new + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, softleaf_from_pte(entry).val, softleaf_from_pte(saved[i]).val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(entry), i * (unsigned int)SZ_4K); + } + for (i = 0; i < TEST_SLOTS; i++) { + unsigned int slot = (i + 2) % TEST_SLOTS; + unsigned long addr = new + slot * SZ_4K; + + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, addr, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0x81 + slot)); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, slot)); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); +} + +static void user4k_swap_pin_test(struct kunit *test) +{ + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + struct folio *folio; + phys_addr_t pinned_phys; + unsigned int i; + vm_fault_t fault; + long ret; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 4); + KUNIT_ASSERT_EQ(test, ret, 4L); + for (i = 0; i < 4; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0xb1 + i, pins->fragments[i].length); + test_swap_tags_set(pins->fragments[0].folio); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA + SZ_4K, SZ_4K, FOLL_WRITE, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + pinned_phys = mm_subpage_phys(pins->fragments[0].subpage); + folio = pins->fragments[0].folio; + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), pinned_phys); + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(pins->fragments[0].subpage)); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(pins->fragments[0].subpage), 1U); + memset(folio_address(folio) + SZ_4K, 0xd2, SZ_4K); + fragment_test_unpin(pins); + /* Once released, the formerly pinned PTE can be swapped normally. */ + folio = test_fault_folio(vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + ret = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0L); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + i == 1 ? 0xd2 : 0xb1 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } +} + +#ifdef CONFIG_MIGRATION +static void user4k_swap_migration_test(struct kunit *test) +{ + struct vm_area_struct *vma, *cvma; + struct mm_struct *child; + struct folio *folio; + phys_addr_t old_phys; + unsigned int i, side, succeeded = 0; + vm_fault_t fault; + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + old_phys = PFN_PHYS(folio_pfn(folio)); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0xa1 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + SZ_4K, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_swapcache(folio)); + ret = isolate_folio_to_list(folio, &folios) ? 0 : -EBUSY; + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + /* One present slot and seven swap entries must all follow the new cache folio. */ + for (side = 0; side < 2; side++) { + struct vm_area_struct *area = side ? cvma : vma; + struct mm_struct *mm = area->vm_mm; + + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + + mmap_read_lock(mm); + fault = handle_mm_fault(area, addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(folio)), old_phys); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0xa1 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + } +} +#endif + +static void user4k_swap_cached_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 0, 0x51); +} + +static void user4k_swap_disk_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 1, 0x61); +} + +static void user4k_swap_reclaim_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 2, 0x71); +} + +static void user4k_swap_stress_test(struct kunit *test) +{ + unsigned int i; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (i = 0; i < 32; i++) + test_user4k_swap_roundtrip(test, i % 3, 0x21 + i * 5); + kunit_info(test, "32 swap/fork/COW/exit cycles with distinct data patterns\n"); +} + +static void user4k_swapoff_test(struct kunit *test) +{ + struct vm_area_struct *parent, *cvma; + struct mm_struct *child; + struct folio *folio; + bool *disabled; + vm_fault_t fault; + unsigned int i, side; + swp_entry_t entry; + int ret; + pte_t pte; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x91 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_fault_entry(test, parent->vm_mm, TEST_VA); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, *disabled); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + i * (unsigned long)SZ_4K); + folio = test_fault_folio(mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x91 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } + } + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + 3 * SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + 3 * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), pte_phys_mm(parent->vm_mm, + test_fault_entry(test, parent->vm_mm, TEST_VA + 3 * SZ_4K))); + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_EXPECT_EQ(test, ret, 0); +} + +static void user4k_swapoff_boundaries_test(struct kunit *test) +{ + const unsigned long boundaries[] = { SZ_2M, SZ_1G, 1UL << 39 }; + bool *disabled; + unsigned int boundary; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + for (boundary = 0; boundary < ARRAY_SIZE(boundaries); boundary++) { + unsigned long start = boundaries[boundary] - SZ_4K; + struct vm_area_struct *parent = test_fault_vma(test, start, 2 * SZ_4K); + struct mm_struct *child; + unsigned int i, side; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, parent); + for (i = 0; i < 2; i++) { + unsigned long addr = start + i * SZ_4K; + unsigned int offset = vma_page_offset_at(parent, addr).offset; + struct folio *folio; + vm_fault_t fault; + swp_entry_t entry; + LIST_HEAD(folios); + + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + memset(folio_address(folio) + offset, 0xb0 + boundary * 2 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = softleaf_from_pte(test_fault_entry(test, parent->vm_mm, addr)); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + } + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 2UL); + for (i = 0; i < 2; i++) { + unsigned long addr = start + i * SZ_4K; + unsigned int offset = vma_page_offset_at(parent, addr).offset; + pte_t pte = test_fault_entry(test, mm, addr); + struct folio *folio; + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + (unsigned long)offset); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0xb0 + boundary * 2 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, offset / SZ_4K)); + folio_put(folio); + } + } + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, parent->vm_mm); + } +} + +struct live_swap_reader { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready, resume, entered, done; + unsigned int id, checks; + int error; +}; + +static int live_swap_reader_run(void *arg) +{ + struct live_swap_reader *r = arg; + unsigned int phase, i; + + kthread_use_mm(r->mm); + for (phase = 0; phase < 2; phase++) { + if (phase) + complete(&r->entered); + for (i = 0; i < 4; i++) { + u8 value = 0; + void __user *addr = (void __user *)(TEST_VA + i * SZ_4K + 64 + r->id); + + /* Resolve AF, swap and COW faults through the real uaccess path. */ + if (copy_from_user(&value, addr, 1) || value != 0xd1 + i) + r->error = phase ? 2 : 1; + r->checks++; + } + if (!phase) { + complete(&r->ready); + if (!wait_for_completion_timeout(&r->resume, 15 * HZ)) { + r->error = 3; + goto out; + } + } + } + { + u8 value = 0xe0 + r->id; + void __user *addr = (void __user *)(TEST_VA + SZ_4K + 64 + r->id); + + if (copy_to_user(addr, &value, 1)) + r->error = 4; + r->checks++; + } +out: + kthread_unuse_mm(r->mm); + complete(&r->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void live_swap_reader_stop(void *arg) +{ + struct live_swap_reader *r = arg; + + if (r->task) { + complete_all(&r->resume); + kthread_stop(r->task); + r->task = NULL; + } +} + +static void test_live_fault_roundtrip(struct kunit *test, unsigned int mode) +{ + bool migration = mode != 0; + struct vm_area_struct *parent; + struct mm_struct *child; + struct live_swap_reader *readers; + struct folio *folio; + vm_fault_t fault; + unsigned int i, cpu; + swp_entry_t entry; + u64 tcr = read_sysreg(tcr_el1); + int ret; + LIST_HEAD(folios); + + if ((!migration && !total_swap_pages) || !user4k_test_enabled || num_online_cpus() < 2 || + !system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires two CPUs, compatible opt-in 4K TTBR0 controls, and swap for reclaim"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < 4; i++) + memset(folio_address(folio) + i * SZ_4K, 0xd1 + i, SZ_4K); + folio_put(folio); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + readers = kunit_kcalloc(test, 2, sizeof(*readers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, readers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 2; i++) { + readers[i].mm = child; + readers[i].id = i; + init_completion(&readers[i].ready); + init_completion(&readers[i].resume); + init_completion(&readers[i].entered); + init_completion(&readers[i].done); + readers[i].task = kthread_create(live_swap_reader_run, &readers[i], + "user4k-swap-read-%u", i); + if (IS_ERR(readers[i].task)) { + readers[i].task = NULL; + KUNIT_FAIL(test, "live swap worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, live_swap_reader_stop, + &readers[i]), 0); + kthread_bind(readers[i].task, cpu); + cpu = cpumask_next(cpu, cpu_online_mask); + wake_up_process(readers[i].task); + } + for (i = 0; i < 2; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].ready, 10 * HZ), 0UL); + lru_add_drain_all(); + folio = test_fault_folio(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + if (mode == 2) { +#ifdef CONFIG_MIGRATION + struct folio *dst = folio_alloc(GFP_KERNEL, 0); + bool entered = true, blocked = true, unmapped = false; + + if (!dst) { + folio_put(folio); + KUNIT_FAIL(test, "migration waiter destination allocation failed"); + return; + } + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + if (!ret) { + try_to_migrate(folio, TTU_SYNC); + unmapped = !folio_mapped(folio); + for (i = 0; i < 2; i++) + complete(&readers[i].resume); + for (i = 0; i < 2; i++) { + entered &= !!wait_for_completion_timeout(&readers[i].entered, HZ); + if (wait_for_completion_timeout(&readers[i].done, max(1UL, HZ / 20UL))) { + blocked = false; + complete(&readers[i].done); + } + } + remove_migration_ptes(folio, folio, 0); + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, entered); + KUNIT_EXPECT_TRUE(test, blocked); + KUNIT_EXPECT_TRUE(test, unmapped); +#endif + } else if (migration) { +#ifdef CONFIG_MIGRATION + unsigned int succeeded = 0; + phys_addr_t old_phys = PFN_PHYS(folio_pfn(folio)); + + ret = isolate_folio_to_list(folio, &folios) ? 0 : -EBUSY; + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_ASYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, test_fault_entry(test, child, TEST_VA)), old_phys); +#endif + } else { + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = softleaf_from_pte(test_fault_entry(test, child, TEST_VA)); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + } + for (i = 0; i < 2; i++) + complete(&readers[i].resume); + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].done, 15 * HZ), 0UL); + live_swap_reader_stop(&readers[i]); + KUNIT_EXPECT_EQ(test, readers[i].error, 0); + KUNIT_EXPECT_EQ(test, readers[i].checks, 9U); + } + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + SZ_4K + 64 + i), (u8)(0xe0 + i)); + folio_put(folio); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA + SZ_4K, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xd2, SZ_4K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 4UL); + kunit_info(test, "two CPUs access through uaccess after %s, then write one child slot\n", + mode == 2 ? "migration-entry rollback" : migration ? "migration" : "reclaim"); +} + +static void user4k_live_swap_fault_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 0); +} + +#ifdef CONFIG_MIGRATION +static void user4k_live_migration_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 1); +} + +static void user4k_migration_wait_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 2); +} +#endif + +/* Transfer PTE ownership to normal mm teardown, which also handles swap. */ +static int anon_slot_fixture_handoff(struct kunit *test, struct anon_slot_fixture *f) +{ + struct mm_struct *mm = f->mm; + + f->mm = NULL; + f->vma = NULL; + memset(f->slots, 0, sizeof(f->slots)); + memset(f->mapped, 0, sizeof(f->mapped)); + return kunit_add_action_or_reset(test, user4k_vma_mm_free, mm); +} + +static void test_nonlinear_swap(struct kunit *test, bool evict) +{ + unsigned int first; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (first = 0; first < 2; first++) { + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct vm_area_struct *vmas[2]; + struct mm_struct *child; + struct mm_subpage *slot; + struct folio *folio; + swp_entry_t entry = { }; + unsigned int side; + vm_fault_t fault; + int err, mapped; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + vmas[0] = a->vma; + vmas[1] = b->vma; + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + err = mem_cgroup_charge(folio, a->mm, GFP_KERNEL); + if (err) { + mm_subpage_put(slot); + KUNIT_FAIL(test, "cannot charge swap fixture: %d", err); + return; + } + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + folio_lock(folio); + mm_subpage_set_exclusive(slot); + folio_unlock(folio); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + folio_lock(folio); + mm_subpage_set_exclusive(slot); + folio_unlock(folio); + memset(folio_address(folio), 0x5b, SZ_4K); + memset(folio_address(folio) + 3 * SZ_4K, 0xd3, SZ_4K); + test_swap_tags_set(folio); + folio_mark_dirty(folio); + folio_add_lru(folio); + lru_add_drain_all(); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_handoff(test, a), 0); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_handoff(test, b), 0); + folio_get(folio); + folio_lock(folio); + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + for (side = 0; side < 2; side++) { + pte_t pte = test_fault_entry(test, vmas[side]->vm_mm, + TEST_VA + side * SZ_4K); + + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), side ? 3U * SZ_4K : 0U); + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, side ? 3 * SZ_4K : 0), 1U); + } + if (evict) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + /* Preserve physical offsets when a nonpresent nonlinear slot is forked. */ + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vmas[1]->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + for (side = 0; side < 2; side++) { + unsigned int which = side ^ first, physical = which ? 3 : 0; + struct vm_area_struct *vma = vmas[which]; + unsigned long address = TEST_VA + which * SZ_4K; + pte_t pte; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ_MSG(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), + (vm_fault_t)0, "evict=%d first=%u side=%u", evict, first, which); + KUNIT_EXPECT_EQ(test, !!(fault & VM_FAULT_MAJOR), evict && !side); + pte = test_fault_entry(test, vma->vm_mm, address); + slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), physical * (unsigned int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), which ? 0xd3 : 0x5b, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(slot), physical)); + mm_subpage_put(slot); + } + /* A child's write must copy only its slot at the new virtual offset. */ + { + struct vm_area_struct *vma = test_vma_lookup(child, TEST_VA + SZ_4K); + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(child); + fault = handle_mm_fault(vma, TEST_VA + SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + slot = mm_subpage_get_from_phys(pte_phys_mm(child, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xd3, SZ_4K), NULL); + memset(folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot), 0x91, SZ_4K); + mm_subpage_put(slot); + pte = test_fault_entry(test, vmas[1]->vm_mm, TEST_VA + SZ_4K); + slot = mm_subpage_get_from_phys(pte_phys_mm(vmas[1]->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xd3, SZ_4K), NULL); + mm_subpage_put(slot); + } + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vmas[1]->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, vmas[0]->vm_mm); + } +} + +static void user4k_nonlinear_swap_cached_test(struct kunit *test) +{ + test_nonlinear_swap(test, false); +} + +static void user4k_nonlinear_swap_disk_test(struct kunit *test) +{ + test_nonlinear_swap(test, true); +} + +#ifdef CONFIG_USERFAULTFD +/* The context is attached only during the call; ordinary mm cleanup owns the VMAs. */ +static ssize_t test_uffd_move(struct vm_area_struct *dst, unsigned long dst_addr, + unsigned long src_addr, unsigned long size) +{ + struct userfaultfd_ctx ctx = { .mm = dst->vm_mm }; + ssize_t ret; + + init_rwsem(&ctx.map_changing_lock); + atomic_set(&ctx.mmap_changing, 0); + mmap_write_lock(ctx.mm); + vma_start_write(dst); + dst->vm_userfaultfd_ctx.ctx = &ctx; + mmap_write_unlock(ctx.mm); + ret = move_pages(&ctx, dst_addr, src_addr, size, 0); + mmap_write_lock(ctx.mm); + vma_start_write(dst); + dst->vm_userfaultfd_ctx.ctx = NULL; + mmap_write_unlock(ctx.mm); + return ret; +} + +struct move_migration_worker { + struct userfaultfd_ctx ctx; + struct vm_area_struct *vma[2]; + struct task_struct *task; + struct completion ready; + unsigned long address[2], size; + unsigned long moves, busy; + int error; +}; + +static int move_migration_worker_run(void *arg) +{ + struct move_migration_worker *w = arg; + + while (!kthread_should_stop()) { + unsigned int side = w->moves & 1; + ssize_t ret = move_pages(&w->ctx, w->address[side ^ 1], + w->address[side], w->size, 0); + + if (ret == w->size) + WRITE_ONCE(w->moves, w->moves + 1); + else if (ret == -EAGAIN || ret == -EBUSY) + w->busy++; + else + w->error = ret < 0 ? ret : -EIO; + if (w->moves || w->error) + complete_all(&w->ready); + if (w->error) { + /* Keep the task valid until the owner joins it. */ + set_current_state(TASK_INTERRUPTIBLE); + if (!kthread_should_stop()) + schedule(); + __set_current_state(TASK_RUNNING); + } + /* Leave bounded gaps so migration can complete between contended moves. */ + if (ret == w->size) + usleep_range(1000, 2000); + else + cond_resched(); + } + return 0; +} + +static void move_migration_worker_stop(void *arg) +{ + struct move_migration_worker *w = arg; + unsigned int i; + + if (w->task) { + kthread_stop(w->task); + w->task = NULL; + } + mmap_write_lock(w->ctx.mm); + for (i = 0; i < 2; i++) { + vma_start_write(w->vma[i]); + w->vma[i]->vm_userfaultfd_ctx.ctx = NULL; + } + mmap_write_unlock(w->ctx.mm); +} + +static void user4k_move_migration_race_test(struct kunit *test) +{ + unsigned int shift; + + for (shift = 12; shift < PAGE_SHIFT; shift += 2) { + struct mm_struct *mm = test_vma_mm_granule(test, shift); + struct move_migration_worker *w; + unsigned long size = 1UL << shift, i, initial_moves; + unsigned int round, migrated = 0, failed = 0; + pte_t pte; + vm_fault_t fault; + + KUNIT_ASSERT_NOT_NULL(test, mm); + w = kunit_kzalloc(test, sizeof(*w), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, w); + w->ctx.mm = mm; + init_rwsem(&w->ctx.map_changing_lock); + atomic_set(&w->ctx.mmap_changing, 0); + init_completion(&w->ready); + w->size = size; + w->address[0] = TEST_VA + 3 * size; + w->address[1] = TEST_VA + SZ_128M + size; + w->vma[0] = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + w->vma[1] = test_fault_vma_in_mm(mm, TEST_VA + SZ_128M, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, w->vma[0]); + KUNIT_ASSERT_NOT_NULL(test, w->vma[1]); + for (i = 0; i < PAGE_SIZE; i += size) { + mmap_read_lock(mm); + fault = handle_mm_fault(w->vma[0], TEST_VA + i, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, mm, TEST_VA + i); + memset(phys_to_virt(pte_phys_mm(mm, pte)), 0x60 + i / size, size); + } + /* Both directions are registered before starting the concurrent mover. */ + mmap_write_lock(mm); + for (i = 0; i < 2; i++) { + vma_start_write(w->vma[i]); + w->vma[i]->vm_userfaultfd_ctx.ctx = &w->ctx; + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, move_migration_worker_stop, w), 0); + w->task = kthread_create(move_migration_worker_run, w, "user4k-move"); + if (IS_ERR(w->task)) { + w->task = NULL; + KUNIT_FAIL(test, "MOVE worker creation failed"); + return; + } + if (num_online_cpus() > 1) + kthread_bind(w->task, cpumask_last(cpu_online_mask)); + wake_up_process(w->task); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->ready, 10 * HZ), 0UL); + initial_moves = READ_ONCE(w->moves); + lru_add_drain_all(); + for (round = 0; round < 64; round++) { + struct folio *folio; + unsigned int succeeded = 0; + bool isolated; + int ret; + LIST_HEAD(folios); + + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE_MSG(test, isolated, "round=%u", round); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_GE(test, ret, 0); + migrated += succeeded; + failed += ret > 0; + cond_resched(); + } + kunit_release_action(test, move_migration_worker_stop, w); + kunit_info(test, "%uK MOVE/migration: moves=%lu retries=%lu migrated=%u busy=%u\n", + 1U << (shift - 10), w->moves, w->busy, migrated, failed); + KUNIT_EXPECT_EQ(test, w->error, 0); + KUNIT_EXPECT_GT(test, w->moves, initial_moves); + KUNIT_EXPECT_GT(test, migrated, 0U); + for (i = 0; i < PAGE_SIZE; i += size) { + unsigned long addr = i == 3 * size ? w->address[w->moves & 1] : TEST_VA + i; + + pte = test_fault_entry(test, mm, addr); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(mm, pte)), + 0x60 + i / size, size), NULL); + } + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, w->address[(w->moves & 1) ^ 1]))); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_move_swap_get_test(struct kunit *test) +{ + unsigned int variant; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (variant = 0; variant < PAGE_SHIFT - 12; variant++) { + unsigned int shift = 12 + (variant / 2) * 2; + bool forked = variant & 1; + unsigned long size = 1UL << shift; + unsigned long src_addr = TEST_VA + 3 * size; + unsigned long dst_addr = TEST_VA + SZ_128M + size; + struct mm_struct *mm = test_vma_mm_granule(test, shift); + struct vm_area_struct *src, *dst; + struct mm_subpage *held; + struct folio *folio; + phys_addr_t physical; + swp_entry_t entry = { }; + pte_t before, after; + vm_fault_t fault; + unsigned long i; + int err, mapped; + + KUNIT_ASSERT_NOT_NULL(test, mm); + src = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + dst = test_fault_vma_in_mm(mm, TEST_VA + SZ_128M, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, src); + KUNIT_ASSERT_NOT_NULL(test, dst); + for (i = 0; i < PAGE_SIZE; i += size) { + mmap_read_lock(mm); + fault = handle_mm_fault(src, TEST_VA + i, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + before = test_fault_entry(test, mm, TEST_VA + i); + memset(phys_to_virt(pte_phys_mm(mm, before)), 0x50 + i / size, size); + } + before = test_fault_entry(test, mm, src_addr); + physical = pte_phys_mm(mm, before); + held = mm_subpage_get_from_phys(physical); + KUNIT_ASSERT_NOT_NULL(test, held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + KUNIT_ASSERT_EQ(test, mm_subpage_offset(held), (unsigned int)(3 * size)); + folio = mm_subpage_folio(held); + lru_add_drain_all(); + folio_lock(folio); + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + KUNIT_ASSERT_EQ(test, mm_subpage_mapcount(held), 0); + KUNIT_ASSERT_EQ(test, swap_subpage_count(entry, 3 * size), 1U); + before = test_fault_entry(test, mm, src_addr); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(before))); + if (forked) { + struct mm_struct *child = test_vma_mm_granule(test, shift); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, test_uffd_move(dst, dst_addr, src_addr, size), + (ssize_t)-EBUSY); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(test_fault_entry(test, mm, + src_addr)).val, entry.val); + kunit_release_action(test, user4k_vma_mm_free, child); + before = test_fault_entry(test, mm, src_addr); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(before)); + KUNIT_ASSERT_EQ(test, swap_subpage_count(entry, 3 * size), 1U); + } + KUNIT_ASSERT_EQ(test, test_uffd_move(dst, dst_addr, src_addr, size), (ssize_t)size); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(after).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(after), (unsigned int)(3 * size)); + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, src_addr))); + /* Restore with the retained metadata already rebound to the new root. */ + mmap_read_lock(mm); + fault = handle_mm_fault(dst, dst_addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_ASSERT_TRUE(test, pte_present(after)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after), physical); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(physical), 0x53, size), NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(held), !forked); + /* Fork-cleared exclusivity requires COW while a GET is retained. */ + mmap_read_lock(mm); + fault = handle_mm_fault(dst, dst_addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_ASSERT_TRUE(test, pte_write(after)); + if (forked) + KUNIT_EXPECT_NE(test, pte_phys_mm(mm, after), physical); + else + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after), physical); + memset(phys_to_virt(pte_phys_mm(mm, after)), 0xa9, size); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(physical), + forked ? 0x53 : 0xa9, size), NULL); + for (i = 0; i < PAGE_SIZE; i += size) { + if (i == 3 * size) + continue; + mmap_read_lock(mm); + fault = handle_mm_fault(src, TEST_VA + i, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, TEST_VA + i); + KUNIT_ASSERT_TRUE(test, pte_present(after)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(mm, after)), + 0x50 + i / size, size), NULL); + } + kunit_release_action(test, test_subpage_put, held); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} +#endif + +static struct kunit_case user4k_swap_cases[] = { + KUNIT_CASE(user4k_swap_reuse_get_test), + KUNIT_CASE(user4k_swap_reuse_swapoff_test), + KUNIT_CASE(user4k_swap_reuse_cached_test), + KUNIT_CASE(user4k_swap_reuse_disk_test), + KUNIT_CASE(user4k_swap_exclusive_cached_test), + KUNIT_CASE(user4k_swap_exclusive_disk_test), + KUNIT_CASE(user4k_swap_exclusive_swapoff_test), + KUNIT_CASE(user4k_swap_cached_test), + KUNIT_CASE(user4k_swap_pin_test), + KUNIT_CASE(user4k_swap_mremap_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_swap_migration_test), +#endif + KUNIT_CASE(user4k_swap_disk_test), + KUNIT_CASE(user4k_swap_reclaim_test), + KUNIT_CASE(user4k_swapoff_test), + KUNIT_CASE(user4k_swap_stress_test), + KUNIT_CASE(user4k_swapoff_boundaries_test), + KUNIT_CASE(user4k_live_swap_fault_test), + {} +}; + +static void user4k_swap_suite_exit(struct kunit_suite *suite) +{ + kvfree_rcu_barrier(); +} + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +static void test_shmem_uffd_swap(struct kunit *test, unsigned int mode) +{ + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, mode == 3 ? 2 : 1); + struct swap_iocb *plug = NULL; + struct file *file; + struct folio *folio; + void *entry; + bool *disabled; + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires disposable guest swap"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + file = test_cache_file(test, PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xb6, mode == 3 ? 2 * SZ_4K : SZ_4K); + ret = shmem_uffd_set_missing(folio, 0, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + /* Match pageout(): submitted data is clean unless writeout redirties it. */ + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + ret = shmem_writeout(folio, &plug, &folios); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_TRUE(test, list_empty(&folios)); + entry = filemap_get_entry(file->f_mapping, 0); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + KUNIT_EXPECT_EQ(test, xa_to_value(xa_load(&SHMEM_I(file_inode(file))->uffd_missing, 0)), missing); + if (mode == 1) { + shmem_truncate_range(file_inode(file), 0, PAGE_SIZE - 1); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + } else if (mode == 3) { + shmem_truncate_range(file_inode(file), 0, SZ_4K - 1); + } else if (mode == 2) { + ret = test_swap_control("--swapoff-helper"); + *disabled = ret == 0; + KUNIT_ASSERT_EQ(test, ret, 0); + } + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), + mode == 1 ? 0UL : mode == 3 ? missing | BIT(0) : missing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), + mode == 1 || mode == 3 ? 0 : 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, + mode == 3 ? 0xb6 : 0, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + 2 * SZ_4K, + 0, PAGE_SIZE - 2 * SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); +} + +static void user4k_shmem_uffd_swapin_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 0); +} + +static void user4k_shmem_uffd_swapped_truncate_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 1); +} + +static void user4k_shmem_uffd_swapped_partial_hole_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 3); +} + +static void user4k_shmem_uffd_swapoff_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 2); +} + +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAULT_INJECTION_DEBUG_FS) +static void test_disable_failslab(void *unused) +{ + test_swap_control("--failslab-off-helper"); +} +#endif + +static void user4k_shmem_uffd_swap_alloc_fail_test(struct kunit *test) +{ +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAULT_INJECTION_DEBUG_FS) + unsigned int attempt; + + if (!total_swap_pages) { + kunit_skip(test, "requires disposable guest swap"); + return; + } + KUNIT_ASSERT_EQ(test, test_swap_control("--failslab-on-helper"), 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_disable_failslab, NULL), 0); + for (attempt = 1; attempt <= 2; attempt++) { + pgoff_t index = attempt == 1 ? 1 : 64; + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + struct file *file = test_cache_file(test, (index + 1) * PAGE_SIZE, false); + struct swap_iocb *plug = NULL; + struct folio *folio; + unsigned int pending; + unsigned long pfn; + void *entry; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), index, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xb6, SZ_4K); + ret = shmem_uffd_set_missing(folio, index, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, index); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + pfn = folio_pfn(folio); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + /* Fail this task's selected allocation and its allocator retries. */ + WRITE_ONCE(current->make_it_fail, 1); + WRITE_ONCE(current->fail_nth, attempt); + ret = shmem_writeout(folio, &plug, &folios); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + WRITE_ONCE(current->make_it_fail, 0); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_EXPECT_EQ(test, ret, AOP_WRITEPAGE_ACTIVATE); + if (ret != AOP_WRITEPAGE_ACTIVATE) { + if (plug) + swap_write_unplug(plug); + folio_wait_writeback(folio); + folio_put(folio); + return; + } + KUNIT_EXPECT_TRUE(test, folio_test_locked(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + KUNIT_EXPECT_PTR_EQ(test, folio->mapping, file->f_mapping); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, index), missing); + /* A failed multi-level insertion may retain an empty internal node. */ + { + unsigned long cursor = 0; + + KUNIT_EXPECT_PTR_EQ(test, xa_find(&SHMEM_I(file_inode(file))->uffd_missing, + &cursor, ULONG_MAX, XA_PRESENT), NULL); + } + KUNIT_EXPECT_PTR_EQ(test, plug, NULL); + KUNIT_EXPECT_TRUE(test, list_empty(&folios)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, PAGE_SIZE - SZ_4K), NULL); + /* No restart or cache repair: retry exactly the retained folio. */ + KUNIT_EXPECT_EQ(test, folio_pfn(folio), pfn); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + ret = shmem_writeout(folio, &plug, &folios); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = filemap_get_entry(file->f_mapping, index); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + KUNIT_EXPECT_EQ(test, xa_to_value(xa_load(&SHMEM_I(file_inode(file))->uffd_missing, index)), missing); + ret = shmem_get_folio(file_inode(file), index, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, index), missing); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, PAGE_SIZE - SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); + kunit_info(test, "failed metadata allocation %u at index %lu; same-folio retry and disk swap-in preserved bytes/mask", attempt, index); + } +#else + kunit_skip(test, "requires FAILSLAB and fault-injection debugfs"); +#endif +} + +#ifdef CONFIG_MEMCG +static void test_memcg_put(void *memcg) +{ + mem_cgroup_put(memcg); +} +#endif + +static void user4k_shmem_uffd_swap_charge_test(struct kunit *test) +{ +#ifdef CONFIG_MEMCG + struct cgroup *cgroup; + struct cgroup_subsys_state *css; + struct mem_cgroup *memcg, *saved; + struct file *file; + struct folio *folio; + struct swap_iocb *plug = NULL; + void *entry; + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages || mem_cgroup_disabled()) { + kunit_skip(test, "requires disposable guest swap and memcg"); + return; + } + KUNIT_ASSERT_EQ(test, test_swap_control("--memcg-setup-helper"), 0); + cgroup = cgroup_get_from_path("/uffd-metadata"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cgroup); + css = cgroup_get_e_css(cgroup, &memory_cgrp_subsys); + cgroup_put(cgroup); + KUNIT_ASSERT_NOT_NULL(test, css); + memcg = mem_cgroup_from_css(css); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_memcg_put, memcg), 0); + KUNIT_ASSERT_PTR_NE(test, memcg, root_mem_cgroup); + file = test_cache_file(test, 65 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + saved = set_active_memcg(memcg); + ret = shmem_get_folio(file_inode(file), 64, 0, &folio, SGP_CACHE); + set_active_memcg(saved); + KUNIT_ASSERT_EQ(test, ret, 0); + rcu_read_lock(); + KUNIT_EXPECT_PTR_EQ(test, folio_memcg(folio), memcg); + rcu_read_unlock(); + ret = shmem_uffd_set_missing(folio, 64, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, 64); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + /* Reclaim from the root charge domain, not the folio's cgroup. */ + saved = set_active_memcg(root_mem_cgroup); + ret = shmem_writeout(folio, &plug, &folios); + set_active_memcg(saved); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + rcu_read_lock(); + entry = rcu_dereference(SHMEM_I(file_inode(file))->uffd_missing.xa_head); + if (xa_is_node(entry)) + KUNIT_EXPECT_PTR_EQ(test, mem_cgroup_from_virt(xa_to_node(entry)), memcg); + rcu_read_unlock(); + KUNIT_ASSERT_TRUE(test, xa_is_node(entry)); + entry = filemap_get_entry(file->f_mapping, 64); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + ret = shmem_get_folio(file_inode(file), 64, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 64), missing); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); +#else + kunit_skip(test, "requires memcg"); +#endif +} + +static bool test_coarse_swap_enabled(struct kunit *test) +{ + if (!test_coarse_file_enabled(test)) + return false; + if (!total_swap_pages || !IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "requires disposable coarse-shmem swap runner"); + return false; + } + return true; +} + +static bool test_coarse_swap_out(struct kunit *test, struct file *file, pgoff_t index) +{ + unsigned int tries; + + for (tries = 0; tries < 100; tries++) { + struct folio *folio; + void *entry = filemap_get_entry(file->f_mapping, index); + LIST_HEAD(folios); + + if (xa_is_value(entry)) + return test_reclaim_swapcache(radix_to_swp_entry(entry)); + if (!entry) { + KUNIT_FAIL(test, "reclaim removed data without a swap entry"); + return false; + } + folio = entry; + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Real reclaim owns any folios produced by a large-folio split. */ + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + KUNIT_FAIL(test, "file index %lu did not reach swap", index); + return false; +} + +static void test_coarse_cold_shmem(struct kunit *test, bool promoted) +{ + struct file *file; + struct vm_area_struct *coarse, *small; + struct folio *folio; + unsigned int nr = SZ_64K / PAGE_SIZE, i; + unsigned long offset; + vm_fault_t fault; + + if (!test_coarse_swap_enabled(test)) + return; + file = test_cache_file(test, SZ_64K, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + SZ_64K - SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, small); + if (promoted) { + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + folio_put(folio); + } + for (offset = SZ_4K; offset < SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + for (i = 0; i < nr; i++) + KUNIT_ASSERT_TRUE(test, test_coarse_swap_out(test, file, i)); + /* XArray swap entries and absent swapcache prove this is a cold refault. */ + for (i = 0; i < nr; i++) { + void *entry = filemap_get_entry(file->f_mapping, i); + struct folio *cached = NULL; + + KUNIT_EXPECT_TRUE(test, xa_is_value(entry)); + if (entry && !xa_is_value(entry)) { + folio_put(entry); + return; + } + KUNIT_ASSERT_NOT_NULL(test, entry); + cached = swap_cache_get_folio(radix_to_swp_entry(entry)); + KUNIT_EXPECT_PTR_EQ(test, cached, NULL); + if (cached) + folio_put(cached); + } + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, 0UL); + KUNIT_EXPECT_EQ(test, SHMEM_I(file_inode(file))->swapped, (unsigned long)nr); + fault = test_file_fault(coarse, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0x31 + offset / SZ_4K, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, (unsigned long)nr); + KUNIT_EXPECT_EQ(test, SHMEM_I(file_inode(file))->swapped, 0UL); + for (offset = SZ_4K; offset < SZ_64K; offset += SZ_4K) { + pte_t entry; + + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(&folio->page) + offset); + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); +} + +static void user64k_shmem_cold_constituents_test(struct kunit *test) +{ + test_coarse_cold_shmem(test, false); +} + +static void user64k_shmem_cold_promoted_test(struct kunit *test) +{ + test_coarse_cold_shmem(test, true); +} + +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_MEMCG) +static void test_coarse_injection_off(void *unused) +{ + test_swap_control("--coarse-injection-off"); +} + +#endif + +static void user64k_shmem_fault_alloc_fail_test(struct kunit *test) +{ +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_MEMCG) + unsigned int mode; + + if (!test_coarse_swap_enabled(test)) + return; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_injection_off, NULL), 0); + for (mode = 0; mode < 3; mode++) { + struct file *file = test_cache_file(test, SZ_64K, false); + struct vm_area_struct *vma; + struct folio *folio; + phys_addr_t original[SZ_64K / PAGE_SIZE]; + unsigned char *buffer; + loff_t pos = 0; + unsigned int i, pending = 0; + vm_fault_t result; + bool prior_fail; + char operation[80]; + int enter = 0, leave = 0; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + /* Make the zero-limit charge case nonblocking and avoid an OOM kill. */ + if (mode == 2) + mapping_set_gfp_mask(file->f_mapping, + mapping_gfp_mask(file->f_mapping) & ~__GFP_DIRECT_RECLAIM); + buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + memset(buffer, 0x95, SZ_64K); + KUNIT_ASSERT_EQ(test, kernel_write(file, buffer, SZ_64K, &pos), (ssize_t)SZ_64K); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + folio = filemap_get_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + original[i] = page_to_phys(&folio->page); + folio_put(folio); + } + if (mode < 2) { + KUNIT_ASSERT_EQ(test, test_swap_control(mode ? "--coarse-page-on" : + "--coarse-slab-on"), 0); + } else { + snprintf(operation, sizeof(operation), "--coarse-memcg-enter=%d", + task_pid_nr(current)); + enter = test_swap_control(operation); + } + /* Direct callback isolates backing allocation from page-table allocation. */ + if (!enter) { + struct vm_fault vmf = { .vma = vma, .address = TEST_VA, + .pgoff = 0, .flags = FAULT_FLAG_REMOTE }; + + mmap_read_lock(vma->vm_mm); + prior_fail = current->make_it_fail; + if (!mode) + WRITE_ONCE(current->fail_nth, 1); + else if (mode == 1) + current->make_it_fail = true; + result = vma->vm_ops->fault(&vmf); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + current->make_it_fail = prior_fail; + mmap_read_unlock(vma->vm_mm); + /* Keep unexpected success safe for test cleanup. */ + if (vmf.page) { + if (result & VM_FAULT_LOCKED) + unlock_page(vmf.page); + put_page(vmf.page); + } + } else { + result = 0; + } + if (mode == 2) { + snprintf(operation, sizeof(operation), "--coarse-memcg-leave=%d", + task_pid_nr(current)); + leave = test_swap_control(operation); + } else { + leave = test_swap_control("--coarse-injection-off"); + } + KUNIT_ASSERT_EQ(test, enter, 0); + KUNIT_ASSERT_EQ(test, leave, 0); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_ASSERT_TRUE(test, result & VM_FAULT_OOM); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + folio = filemap_get_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + KUNIT_EXPECT_EQ(test, page_to_phys(&folio->page), original[i]); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x95, PAGE_SIZE), NULL); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, SZ_64K / PAGE_SIZE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SHMEMPAGES), 0L); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x95, SZ_64K), NULL); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + kunit_info(test, "coarse allocation failure mode %u preserved data and recovered", mode); + } +#else + kunit_skip(test, "requires slab/page fault injection and memcg"); +#endif +} + +static void user64k_anon_swap_test(struct kunit *test) +{ + unsigned int mode, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_swap_enabled(test)) + return; + for (mode = 0; mode < 7; mode++) { + struct vm_area_struct *vma, *cvma = NULL; + struct mm_struct *child = NULL; + struct folio *folio, *held = NULL; + swp_entry_t entry, part; + unsigned int i, tries; + pte_t pte; + bool *disabled; + bool warm = !mode || mode == 6; + vm_fault_t fault; + int err; + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x31 + i, SZ_4K); + if (mode == 2 || mode == 6) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + } + if (warm) { + folio_get(folio); + held = folio; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, held), 0); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_EXPECT_EQ(test, min_order_for_split(folio), 16U - PAGE_SHIFT); +#endif + folio_lock(folio); + KUNIT_EXPECT_EQ(test, folio_anon_min_order(folio), 16U - PAGE_SHIFT); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_EXPECT_EQ(test, split_huge_page_to_list_to_order(&folio->page, NULL, 0), + -EBUSY); + KUNIT_EXPECT_EQ(test, folio_order(folio), 16U - PAGE_SHIFT); + expect_coarse_anon_state(test, folio, child ? 2 : 1, !child); +#endif + err = folio_alloc_swap(folio); + if (!err) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, err, 0); + } else { + for (tries = 0; tries < 100; tries++) { + LIST_HEAD(folios); + + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + if (!pte_present(pte)) + break; + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + folio_get(folio); + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + } + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE_MSG(test, pte_present(pte), "mode=%u", mode); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(swp_offset(entry), nr)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 1L); + if (mode == 3) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + } + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), child ? 2 : 1); + if (!warm) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + if (mode == 3) { + /* Exit a swapped child: every constituent count must be released. */ + kunit_release_action(test, user4k_vma_mm_free, child); + child = NULL; + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + } + } + if (mode == 5) { +#if defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_FAULT_INJECTION) + bool old = current->make_it_fail; + + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-page-on"), 0); + current->make_it_fail = true; + fault = test_file_fault(vma, TEST_VA, false); + current->make_it_fail = old; + err = test_swap_control("--coarse-injection-off"); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_OOM); + KUNIT_EXPECT_TRUE(test, pte_same(test_fault_entry(test, vma->vm_mm, TEST_VA), pte)); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + } +#endif + } + if (mode == 4) { + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + err = test_swap_control("--swapoff-helper"); + if (!err) + *disabled = true; + KUNIT_ASSERT_EQ(test, err, 0); + } else { + fault = test_file_fault(vma, TEST_VA + SZ_64K - 1, false); + KUNIT_ASSERT_EQ_MSG(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U, + "mode=%u fault=%x", mode, fault); + if (!warm && mode != 5) + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + } + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE_MSG(test, pte_present(pte), "mode=%u", mode); + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + if (held) { + KUNIT_EXPECT_PTR_EQ(test, folio, held); /* Warm aligned cache: no copy. */ + } else { + folio_get(folio); + held = folio; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, held), 0); + } + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + if (child) { + struct folio *copied; + + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), page_to_phys(&folio->page)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + copied = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(child, pte)))); + expect_coarse_anon_state(test, copied, 1, true); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(copied) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + memset(folio_address(copied), 0xa7, SZ_64K); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, child); + } + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + if (held) { + KUNIT_EXPECT_EQ(test, folio_mapcount(held), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(held), 1); + } + for (i = 0; i < nr; i++) { + struct swap_info_struct *si; + + part = entry; + part.val += i; + si = get_swap_device(part); + if (si) { + KUNIT_EXPECT_FALSE(test, swap_entry_swapped(si, part)); + put_swap_device(si); + } else { + KUNIT_EXPECT_EQ(test, mode, 4U); + } + } + if (mode == 4) + kunit_release_action(test, test_reenable_swap, disabled); + kunit_info(test, "completed coarse anonymous swap mode %u", mode); + } +} + +/* The caller's private fixture has no concurrent PTE writers during reclaim. */ +static void test_coarse_anon_reclaim_once(struct vm_area_struct *vma) +{ + struct folio *folio = test_fault_folio(vma->vm_mm, vma->vm_start); + LIST_HEAD(folios); + + if (!folio) + return; + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); +} + +static void user64k_anon_swap_limit_test(struct kunit *test) +{ +#ifdef CONFIG_MEMCG + struct vm_area_struct *vma; + struct cgroup *cgroup; + struct cgroup_subsys_state *css; + struct mem_cgroup *memcg; + struct folio *folio; + swp_entry_t entry, part; + phys_addr_t original; + unsigned int i, tries, nr = SZ_64K / PAGE_SIZE; + long before, denied; + char operation[80]; + vm_fault_t fault; + pte_t pte; + int ret, leave; + + if (!test_coarse_swap_enabled(test)) + return; + if (mem_cgroup_disabled()) { + kunit_skip(test, "requires memory cgroups"); + return; + } + snprintf(operation, sizeof(operation), "--coarse-swap-enter=%d", task_pid_nr(current)); + KUNIT_ASSERT_EQ(test, test_swap_control(operation), 0); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + fault = vma ? test_file_fault(vma, TEST_VA, true) : VM_FAULT_OOM; + snprintf(operation, sizeof(operation), "--coarse-swap-leave=%d", task_pid_nr(current)); + leave = test_swap_control(operation); + KUNIT_ASSERT_EQ(test, leave, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + cgroup = cgroup_get_from_path("/coarse-swap-limit"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cgroup); + css = cgroup_get_e_css(cgroup, &memory_cgrp_subsys); + cgroup_put(cgroup); + KUNIT_ASSERT_NOT_NULL(test, css); + memcg = mem_cgroup_from_css(css); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_memcg_put, memcg), 0); + KUNIT_ASSERT_PTR_NE(test, memcg, root_mem_cgroup); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + rcu_read_lock(); + KUNIT_EXPECT_PTR_EQ(test, folio_memcg(folio), memcg); + rcu_read_unlock(); + original = page_to_phys(&folio->page); + memset(folio_address(folio), 0xc9, SZ_64K); + before = atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]); + folio_lock(folio); + ret = folio_alloc_swap(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + denied = atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]); + KUNIT_EXPECT_GT(test, denied, before); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); + /* Exercise vmscan's failed-allocation/split fallback without a retained GET. */ + for (tries = 0; tries < 8; tries++) + test_coarse_anon_reclaim_once(vma); + KUNIT_EXPECT_GT(test, atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]), denied); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), original); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xc9, SZ_64K), NULL); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-swap-unlimit"), 0); + for (tries = 0; tries < 100; tries++) { + test_coarse_anon_reclaim_once(vma); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + if (!pte_present(pte)) + break; + } + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), (unsigned long)nr); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xc9, SZ_64K), NULL); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); +#else + kunit_skip(test, "requires memory cgroups"); +#endif +} + +struct coarse_swap_worker { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready, go, entered, done; + u8 *buffer; + unsigned int id; + bool *disabled; + int error; +}; + +static int coarse_swap_worker_run(void *arg) +{ + struct coarse_swap_worker *w = arg; + unsigned int i; + u8 value = 0x91 + w->id; + + complete(&w->ready); + if (!wait_for_completion_timeout(&w->go, 15 * HZ) || kthread_should_stop()) { + w->error = -ETIMEDOUT; + goto out; + } + if (!w->mm) { + complete(&w->entered); + w->error = test_swap_control("--swapoff-helper"); + if (!w->error) + WRITE_ONCE(*w->disabled, true); + goto out; + } + kthread_use_mm(w->mm); + complete(&w->entered); + if (copy_from_user(w->buffer, (void __user *)TEST_VA, SZ_64K)) { + w->error = 1; + goto unuse; + } + for (i = 0; i < SZ_64K; i++) { + if (w->buffer[i] != (u8)(0x31 + i / SZ_4K)) { + w->error = 2; + goto unuse; + } + } + if (copy_to_user((void __user *)(TEST_VA + SZ_64K - 1), &value, 1) || + copy_from_user(w->buffer, (void __user *)TEST_VA, SZ_64K)) { + w->error = 3; + goto unuse; + } + for (i = 0; i < SZ_64K; i++) { + u8 expected = i == SZ_64K - 1 ? value : 0x31 + i / SZ_4K; + + if (w->buffer[i] != expected) { + w->error = 4; + break; + } + } +unuse: + kthread_unuse_mm(w->mm); +out: + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void coarse_swap_worker_stop(void *arg) +{ + struct coarse_swap_worker *w = arg; + + if (w->task) { + complete_all(&w->go); + kthread_stop(w->task); + w->task = NULL; + } +} + +static void user64k_anon_swap_race_test(struct kunit *test) +{ + unsigned int round, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_swap_enabled(test)) + return; + if (num_online_cpus() < 2 || !system_supports_64kb_granule()) { + kunit_skip(test, "requires two CPUs and 64K TTBR0 support"); + return; + } + for (round = 0; round < 8; round++) { + struct vm_area_struct *vma; + struct mm_struct *child; + struct coarse_swap_worker *workers; + struct folio *folio, *blocker; + struct swap_info_struct *si; + swp_entry_t entry, part; + bool *disabled, started = true, blocked; + bool off_started = false, cold = round & 1; + unsigned int i, j, tries; + pte_t pte; + int ret, cpu; + + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x31 + i, SZ_4K); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + if (!ret) { + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + } + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 2); + if (cold) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + part = entry; + part.val += nr - 1; + blocker = read_swap_cache_async(part, GFP_KERNEL, vma, + TEST_VA + SZ_64K - PAGE_SIZE, NULL); + KUNIT_ASSERT_NOT_NULL(test, blocker); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, blocker), 0); + KUNIT_EXPECT_EQ(test, folio_size(blocker), cold ? PAGE_SIZE : (unsigned long)SZ_64K); + workers = kunit_kcalloc(test, 3, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 3; i++) { + workers[i].mm = i < 2 ? (i ? child : vma->vm_mm) : NULL; + workers[i].id = i; + workers[i].disabled = disabled; + workers[i].buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers[i].buffer); + init_completion(&workers[i].ready); + init_completion(&workers[i].go); + init_completion(&workers[i].entered); + init_completion(&workers[i].done); + workers[i].task = kthread_create(coarse_swap_worker_run, &workers[i], + "user64k-swap-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "coarse swap worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + coarse_swap_worker_stop, &workers[i]), 0); + if (i < 2) { + kthread_bind(workers[i].task, cpu); + cpu = cpumask_next(cpu, cpu_online_mask); + } + wake_up_process(workers[i].task); + } + for (i = 0; i < 3; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + folio_lock(blocker); + for (i = 0; i < 3; i++) + complete(&workers[i].go); + for (i = 0; i < 3; i++) + started &= !!wait_for_completion_timeout(&workers[i].entered, HZ); + for (tries = 0; tries < 3 * HZ; tries++) { + si = get_swap_device(entry); + if (si) { + off_started = !(READ_ONCE(si->flags) & SWP_WRITEOK); + put_swap_device(si); + } + if (off_started) + break; + schedule_timeout_uninterruptible(1); + } + blocked = !completion_done(&workers[0].done) && + !completion_done(&workers[1].done) && !completion_done(&workers[2].done); + /* Never execute a fatal assertion while retaining the blocking lock. */ + folio_unlock(blocker); + KUNIT_EXPECT_TRUE(test, started); + KUNIT_EXPECT_TRUE(test, off_started); + KUNIT_EXPECT_TRUE(test, blocked); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 20 * HZ), 0UL); + KUNIT_EXPECT_EQ_MSG(test, workers[i].error, 0, "round=%u worker=%u", round, i); + kunit_release_action(test, coarse_swap_worker_stop, &workers[i]); + } + KUNIT_EXPECT_TRUE(test, *disabled); + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? child : vma->vm_mm; + + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (j = 0; j < SZ_64K; j++) { + u8 expected = j == SZ_64K - 1 ? 0x91 + i : 0x31 + j / SZ_4K; + + if (((u8 *)folio_address(folio))[j] != expected) { + KUNIT_FAIL(test, "round=%u side=%u byte=%u", round, i, j); + break; + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1L); + kunit_release_action(test, user4k_vma_mm_free, mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), folio == blocker ? 2 : 1); + kunit_release_action(test, test_folio_put, folio); + } + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(blocker)); + KUNIT_EXPECT_EQ(test, folio_ref_count(blocker), 1); + kunit_release_action(test, test_folio_put, blocker); + kunit_release_action(test, test_reenable_swap, disabled); + KUNIT_ASSERT_GT(test, total_swap_pages, 0UL); + } + kunit_info(test, "8 controlled warm/cold fault+COW+swapoff rounds completed"); +} + +struct coarse_swap_reservation { + struct folio **folios; + unsigned int count; +}; + +static void test_coarse_release_reservation(struct folio *folio) +{ + folio_lock(folio); + folio_put_swap(folio, NULL); + folio_free_swap(folio); + folio_unlock(folio); + folio_put(folio); +} + +static void test_coarse_release_reservations(void *arg) +{ + struct coarse_swap_reservation *r = arg; + unsigned int i; + + for (i = 0; i < r->count; i++) { + if (!r->folios[i]) + continue; + test_coarse_release_reservation(r->folios[i]); + r->folios[i] = NULL; + cond_resched(); + } +} + +static void test_coarse_swap_refusal(struct kunit *test, struct vm_area_struct *vma, + phys_addr_t original) +{ + struct folio *folio = test_fault_folio(vma->vm_mm, TEST_VA); + pte_t pte; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + for (i = 0; i < 4; i++) + test_coarse_anon_reclaim_once(vma); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), original); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xd6, SZ_64K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); +} + +static void user64k_anon_swap_fragment_test(struct kunit *test) +{ + struct coarse_swap_reservation *r; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int i, nr = SZ_64K / PAGE_SIZE, released = 0; + unsigned long hole, free_before; + phys_addr_t original; + swp_entry_t entry, part; + vm_fault_t fault; + pte_t pte; + int ret = 0; + + if (!test_coarse_swap_enabled(test)) + return; + /* Exhaust only the bounded disposable disk owned by this test's PID1. */ + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-fixture-check"), 0); + KUNIT_ASSERT_LT(test, total_swap_pages, (unsigned long)(SZ_256M / PAGE_SIZE)); + KUNIT_ASSERT_GT(test, total_swap_pages, 2UL * SWAPFILE_CLUSTER); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + original = page_to_phys(&folio->page); + memset(folio_address(folio), 0xd6, SZ_64K); + folio_put(folio); + r = kunit_kzalloc(test, sizeof(*r), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, r); + r->folios = kunit_kcalloc(test, total_swap_pages + 1, sizeof(*r->folios), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, r->folios); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_release_reservations, r), 0); + /* Each cache-only folio has one explicit slot owner, so allocation cannot reclaim it. */ + for (i = 0; i <= total_swap_pages; i++) { + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL); + if (ret) { + folio_put(folio); + KUNIT_FAIL(test, "reservation memory charge failed"); + return; + } + __folio_set_swapbacked(folio); + __folio_mark_uptodate(folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) { + ret = folio_dup_swap(folio, NULL); + if (ret) + folio_free_swap(folio); + } + folio_unlock(folio); + if (ret) { + folio_put(folio); + break; + } + r->folios[r->count++] = folio; + cond_resched(); + } + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + KUNIT_ASSERT_EQ(test, atomic_long_read(&nr_swap_pages), 0L); + test_coarse_swap_refusal(test, vma, original); + /* Retain exactly one native entry in each naturally aligned coarse group. */ + for (i = 0; i < r->count; i++) { + folio = r->folios[i]; + if (swp_offset(folio->swap) % nr == 1) + continue; + test_coarse_release_reservation(folio); + r->folios[i] = NULL; + } + free_before = atomic_long_read(&nr_swap_pages); + KUNIT_EXPECT_GT(test, free_before, total_swap_pages / 2); + KUNIT_EXPECT_LT(test, free_before, total_swap_pages); + test_coarse_swap_refusal(test, vma, original); + /* Free one aligned group near the end, without freeing a whole native-order cluster. */ + hole = round_down(total_swap_pages + 1, (unsigned long)SWAPFILE_CLUSTER) - nr; + for (i = 0; i < r->count; i++) { + folio = r->folios[i]; + if (!folio || swp_offset(folio->swap) < hole || swp_offset(folio->swap) >= hole + nr) + continue; + test_coarse_release_reservation(folio); + r->folios[i] = NULL; + released++; + } + KUNIT_ASSERT_EQ(test, released, 1U); + KUNIT_EXPECT_EQ(test, atomic_long_read(&nr_swap_pages), (long)free_before + 1); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ_MSG(test, ret, 0, "a complete aligned group is free in an order-0 cluster"); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, swp_offset(entry), hole); + for (i = 0; i < r->count; i++) { + if (r->folios[i]) + KUNIT_EXPECT_EQ(test, swp_swapcount(r->folios[i]->swap), 1); + } + kunit_release_action(test, test_coarse_release_reservations, r); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xd6, SZ_64K), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + kunit_info(test, "exhaustion, scattered free space and one-group recovery checked"); +} + +struct coarse_swap_io_mode { + struct swap_info_struct *si; + bool synchronous; +}; + +static void test_coarse_restore_io_mode(void *arg) +{ + struct coarse_swap_io_mode *mode = arg; + + spin_lock(&mode->si->lock); + if (mode->synchronous) + mode->si->flags |= SWP_SYNCHRONOUS_IO; + else + mode->si->flags &= ~SWP_SYNCHRONOUS_IO; + spin_unlock(&mode->si->lock); + put_swap_device(mode->si); +} + +static void user64k_anon_swap_extent_test(struct kunit *test) +{ + unsigned int mode, i; + + if (!test_coarse_swap_enabled(test)) + return; + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-fixture-check"), 0); + for (mode = 0; mode < 2; mode++) { + struct vm_area_struct *vma; + struct coarse_swap_io_mode *saved; + struct folio *folio; + unsigned long nr = SZ_64K / PAGE_SIZE, run = nr; + swp_entry_t entry; + sector_t first, next; + bool file; + int ret; + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x63 + i, SZ_4K); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, folio_mapcount(folio), 0); + entry = folio->swap; + saved = kunit_kzalloc(test, sizeof(*saved), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, saved); + saved->si = get_swap_device(entry); + KUNIT_ASSERT_NOT_NULL(test, saved->si); + saved->synchronous = saved->si->flags & SWP_SYNCHRONOUS_IO; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_restore_io_mode, saved), 0); + file = !(saved->si->flags & SWP_BLKDEV); + KUNIT_ASSERT_FALSE(test, saved->si->flags & SWP_FS_OPS); + first = swap_extent_sector(entry, &run); + if (file) { + unsigned long remaining; + swp_entry_t part = entry; + + KUNIT_ASSERT_LT_MSG(test, run, nr, "fragmented fixture must cross an extent"); + part.val += run; + remaining = nr - run; + next = swap_extent_sector(part, &remaining); + KUNIT_EXPECT_NE(test, next, first + (run << (PAGE_SHIFT - 9))); + } else { + KUNIT_EXPECT_EQ(test, run, nr); + } + /* The verified disposable fixture owns this otherwise idle swap area. */ + spin_lock(&saved->si->lock); + if (mode) + saved->si->flags |= SWP_SYNCHRONOUS_IO; + else + saved->si->flags &= ~SWP_SYNCHRONOUS_IO; + spin_unlock(&saved->si->lock); + folio_lock(folio); + folio_clear_dirty_for_io(folio); + __swap_writepage(folio, NULL); + folio_wait_writeback(folio); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, folio_test_dirty(folio)); + /* A full-folio read must reconstruct all physical runs, not warm cache. */ + memset(folio_address(folio), 0, SZ_64K); + folio_clear_uptodate(folio); + swap_read_folio(folio, NULL); + folio_lock(folio); + KUNIT_EXPECT_TRUE(test, folio_test_uptodate(folio)); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x63 + i, SZ_4K), NULL); + folio_unlock(folio); + kunit_release_action(test, test_coarse_restore_io_mode, saved); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + kunit_info(test, "64K full-folio %s I/O: file=%u first-run=%lu/%lu native pages", + mode ? "synchronous" : "asynchronous", file, run, nr); + } +} + +static struct kunit_case user4k_coarse_shmem_cases[] = { + KUNIT_CASE(user64k_anon_swap_extent_test), + KUNIT_CASE(user64k_anon_swap_test), + KUNIT_CASE(user64k_anon_swap_limit_test), + KUNIT_CASE(user64k_anon_swap_fragment_test), + KUNIT_CASE(user64k_anon_swap_race_test), + KUNIT_CASE(user64k_shmem_cold_constituents_test), + KUNIT_CASE(user64k_shmem_cold_promoted_test), + KUNIT_CASE(user64k_shmem_fault_alloc_fail_test), + {} +}; + +static struct kunit_suite user4k_coarse_shmem_suite = { + .name = "arm64-user4k-coarse-shmem", + .test_cases = user4k_coarse_shmem_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static struct kunit_case user4k_shmem_swap_cases[] = { + KUNIT_CASE(user4k_shmem_uffd_swap_alloc_fail_test), + KUNIT_CASE(user4k_shmem_uffd_swap_charge_test), + KUNIT_CASE(user4k_shmem_uffd_swapin_test), + KUNIT_CASE(user4k_shmem_uffd_swapped_truncate_test), + KUNIT_CASE(user4k_shmem_uffd_swapoff_test), + KUNIT_CASE(user4k_shmem_uffd_swapped_partial_hole_test), + {} +}; + +static struct kunit_suite user4k_shmem_swap_suite = { + .name = "arm64-user4k-shmem-swap", + .test_cases = user4k_shmem_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; +#endif + +static struct kunit_case user4k_nonlinear_swap_cases[] = { +#ifdef CONFIG_USERFAULTFD + KUNIT_CASE(user4k_move_swap_get_test), + KUNIT_CASE(user4k_move_migration_race_test), +#endif + KUNIT_CASE(user4k_nonlinear_swap_cached_test), + KUNIT_CASE(user4k_nonlinear_swap_disk_test), + {} +}; + +static struct kunit_suite user4k_nonlinear_swap_suite = { + .name = "arm64-user4k-nonlinear-swap", + .test_cases = user4k_nonlinear_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static struct kunit_suite user4k_swap_suite = { + .name = "arm64-user4k-swap", + .test_cases = user4k_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static void user4k_thp_eligibility_test(struct kunit *test) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + const enum tva_type types[] = { + TVA_PAGEFAULT, TVA_SMAPS, TVA_KHUGEPAGED, TVA_FORCED_COLLAPSE, + }; + struct vm_area_struct *vma, *normal; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_HUGEPAGE; + + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + vma = test_vma_add(small, SZ_1G, SZ_1G, (struct vm_page_offset) {}, flags); + normal = test_vma_add(native, SZ_1G, SZ_1G, (struct vm_page_offset) {}, flags); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, normal); + /* Neither user advice nor forced collapse can bypass geometry limits. */ + KUNIT_EXPECT_TRUE(test, vma_thp_disabled(vma, flags, false)); + KUNIT_EXPECT_TRUE(test, vma_thp_disabled(vma, flags, true)); + KUNIT_EXPECT_FALSE(test, vma_thp_disabled(normal, flags, false)); + KUNIT_EXPECT_FALSE(test, vma_thp_disabled(normal, flags, true)); + for (unsigned int i = 0; i < ARRAY_SIZE(types); i++) + KUNIT_EXPECT_EQ(test, __thp_vma_allowable_orders(vma, flags, + types[i], THP_ORDERS_ALL_ANON), 0UL); +#else + kunit_skip(test, "requires transparent hugepage support"); +#endif +} + +static void user4k_aslr_window_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + const unsigned long windows[] = { SZ_8M, SZ_32M, SZ_1G }; + + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + for (unsigned int i = 0; i < ARRAY_SIZE(windows); i++) { + unsigned long mask = (windows[i] >> PAGE_SHIFT) - 1; + unsigned long converted = mm_aslr_mask(small, mask); + + KUNIT_EXPECT_EQ(test, mm_aslr_mask(native, mask), mask); + KUNIT_EXPECT_EQ(test, (converted + 1) << mm_page_shift(small), + windows[i]); + KUNIT_EXPECT_EQ(test, converted & (TEST_SLOTS - 1), + (unsigned long)TEST_SLOTS - 1); + } +} + +static void user4k_exec_granule_policy_test(struct kunit *test) +{ + /* Model all native granules, including AArch32 with NO 4K support. */ + const unsigned int native[] = { 12, 14, 16 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(native); i++) { + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, false, native[i], 0), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, false, native[i], 12), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, true, native[i], 0), + 12U); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(false, true, native[i], 0), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(false, true, native[i], 12), + 12U); + } +} + +static void user_granule_zero_region_test(struct kunit *test) +{ + unsigned long base_pfn = zero_pfn(0); + unsigned int shift, i; + u8 *buffer; + + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(PFN_PHYS(base_pfn), SZ_64K)); + KUNIT_EXPECT_FALSE(test, is_zero_pfn(base_pfn - 1)); + KUNIT_EXPECT_FALSE(test, is_zero_pfn(base_pfn + SZ_64K / PAGE_SIZE)); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + KUNIT_EXPECT_TRUE(test, is_zero_pfn(base_pfn + i)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(pfn_to_page(base_pfn + i)), + 0, PAGE_SIZE), NULL); + } + buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + for (shift = 12; shift <= 16; shift += 2) { + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned long size = 1UL << shift, left; + vm_fault_t ret; + spinlock_t *ptl; + pte_t *pte, entry; + + if ((shift == 12 && !system_supports_4kb_granule()) || + (shift == 14 && !system_supports_16kb_granule()) || + (shift == 16 && !system_supports_64kb_granule())) { + kunit_info(test, "hardware zero mapping skipped for shift %u", shift); + continue; + } + /* Internal coarse mm: the public opt-in remains disabled. */ + mm = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = test_fault_vma_in_mm(mm, TEST_VA, size); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + size - 1, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_special(entry)); + KUNIT_EXPECT_FALSE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), PFN_PHYS(base_pfn)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + + memset(buffer, 0xa5, size); + kthread_use_mm(mm); + left = copy_from_user(buffer, (void __user *)TEST_VA, size); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, left, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, size), NULL); + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, size); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + } +} + +#ifdef CONFIG_COREDUMP +static void user_granule_core_offsets_test(struct kunit *test) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + struct vm_area_struct *vma = kunit_kzalloc(test, sizeof(*vma), GFP_KERNEL); + unsigned int shift; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_mm = mm; + for (shift = 12; shift <= 16; shift += 2) { + unsigned long bytes = (1UL << 55) + 3 * SZ_64K + (1UL << shift); + + mm->context.flags = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = bytes >> PAGE_SHIFT, .offset = bytes & ~PAGE_MASK, + }); + KUNIT_EXPECT_EQ(test, core_vma_pgoff(vma), bytes >> shift); + KUNIT_EXPECT_EQ(test, mm_user_fragment_size(mm), min(1UL << shift, PAGE_SIZE)); + vma_set_page_offset(vma, (struct vm_page_offset) {}); + KUNIT_EXPECT_EQ(test, core_vma_pgoff(vma), 0UL); + } +} +#endif + +static void user64k_fragment_ownership_test(struct kunit *test) +{ + struct user_page_fragment fragments[SZ_64K / PAGE_SIZE] = {}; + struct mm_struct *parent, *child; + struct vm_area_struct *vma, *cvma; + struct folio *original, *copied; + unsigned int i, nr = ARRAY_SIZE(fragments); + long count; + int ret, refs; + pte_t entry; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires a user leaf larger than native pages"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + count = get_user_fragments_remote(parent, TEST_VA + 17, SZ_64K - 34, + 0, fragments, nr); + if (count > 0) { + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &fragments[i]; + unsigned int bytes = PAGE_SIZE - (i == 0 || i == nr - 1 ? 17 : 0); + + KUNIT_EXPECT_TRUE(test, is_zero_folio(f->folio)); + KUNIT_EXPECT_EQ(test, folio_pfn(f->folio), zero_pfn(TEST_VA) + i); + KUNIT_EXPECT_PTR_EQ(test, f->subpage, NULL); + KUNIT_EXPECT_EQ(test, f->offset, i == 0 ? 17UL : 0UL); + KUNIT_EXPECT_EQ(test, f->length, bytes); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0, f->length), NULL); + } + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, parent, TEST_VA); + original = page_folio(pfn_to_page(pte_pfn(entry))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(original, i)), 0x51 + i, PAGE_SIZE); + lru_add_drain_all(); + refs = folio_ref_count(original); + count = get_user_fragments_remote(parent, TEST_VA + 17, SZ_64K - 34, + 0, fragments, nr); + if (count > 0) { + KUNIT_EXPECT_EQ(test, folio_ref_count(original), refs + (int)count); + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &fragments[i]; + + KUNIT_EXPECT_PTR_EQ(test, f->folio, original); + KUNIT_EXPECT_EQ(test, f->offset, i * PAGE_SIZE + (i == 0 ? 17UL : 0UL)); + KUNIT_EXPECT_LE(test, offset_in_page(f->offset) + f->length, PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x51 + i, f->length), NULL); + } + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + KUNIT_EXPECT_EQ(test, folio_ref_count(original), refs); + + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + /* A read pin after fork must unshare the complete coarse leaf. */ + count = pin_user_fragments_remote(child, TEST_VA, SZ_64K, 0, fragments, nr); + if (count > 0) { + copied = fragments[0].folio; + KUNIT_EXPECT_PTR_NE(test, copied, original); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(copied)); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, child, TEST_VA))); + for (i = 0; i < count; i++) { + KUNIT_EXPECT_PTR_EQ(test, fragments[i].folio, copied); + KUNIT_EXPECT_PTR_EQ(test, fragments[i].subpage, NULL); + KUNIT_EXPECT_TRUE(test, fragments[i].pinned); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(copied, i))); + KUNIT_EXPECT_EQ(test, fragments[i].offset, i * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, fragments[i].length, (unsigned int)PAGE_SIZE); + } + mmap_read_lock(child); + zap_vma_range(cvma, TEST_VA, SZ_64K); + mmap_read_unlock(child); + KUNIT_EXPECT_EQ(test, folio_mapcount(copied), 0); + for (i = 0; i < count; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(copied) + i * PAGE_SIZE, + 0x51 + i, PAGE_SIZE), NULL); + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(original) + i * PAGE_SIZE, + 0x51 + i, PAGE_SIZE), NULL); +} + +static void user64k_anon_fault_cow_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + struct vm_area_struct *vma, *cvma; + pte_t orig[2], copied; + struct folio *folio; + unsigned int i, j, nr = SZ_64K / PAGE_SIZE; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous faults"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, 2 * SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Leaf0 exercises zero-page COW; leaf1 exercises a direct write fault. */ + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, false) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_TRUE(test, pte_special(test_fault_entry(test, parent, TEST_VA))); + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + + KUNIT_ASSERT_EQ(test, test_file_fault(vma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + orig[i] = test_fault_entry(test, parent, addr); + KUNIT_ASSERT_TRUE(test, pte_present(orig[i])); + KUNIT_EXPECT_TRUE(test, pte_write(orig[i])); + KUNIT_EXPECT_FALSE(test, pte_special(orig[i])); + folio = page_folio(pfn_to_page(pte_pfn(orig[i]))); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, SZ_64K), NULL); + for (j = 0; j < nr; j++) { + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(folio, j))); + memset(page_address(folio_page(folio, j)), 0x31 + i * nr + j, PAGE_SIZE); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 2L); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 2L); + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + struct folio *old = page_folio(pfn_to_page(pte_pfn(orig[i]))); + + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, test_fault_entry(test, child, addr)), + pte_phys_mm(parent, orig[i])); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, parent, addr))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)(2 * nr)); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + copied = test_fault_entry(test, child, addr); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, copied), pte_phys_mm(parent, orig[i])); + folio = page_folio(pfn_to_page(pte_pfn(copied))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)nr); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + for (j = 0; j < nr; j++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(folio, j)), + 0x31 + i * nr + j, PAGE_SIZE), NULL); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(folio, j))); + } + memset(folio_address(folio), 0xa5, SZ_64K); + for (j = 0; j < nr; j++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(old, j)), + 0x31 + i * nr + j, PAGE_SIZE), NULL); + /* Parent reuse must restore exclusivity across the entire leaf. */ + lru_add_drain_all(); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, pte_phys_mm(parent, test_fault_entry(test, parent, addr)), + pte_phys_mm(parent, orig[i])); + for (j = 0; j < nr; j++) + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(old, j))); + } + if (system_supports_64kb_granule()) { + long protect[2]; + bool writable = false; + + kthread_use_mm(parent); + protect[0] = test_mprotect(TEST_VA, 2 * SZ_64K, PROT_READ); + kthread_unuse_mm(parent); + for (i = 0; i < 2; i++) + writable |= pte_write(test_fault_entry(test, parent, TEST_VA + i * SZ_64K)); + KUNIT_EXPECT_EQ(test, protect[0], 0L); + KUNIT_EXPECT_FALSE(test, writable); + kthread_use_mm(parent); + protect[1] = test_mprotect(TEST_VA, 2 * SZ_64K, PROT_READ | PROT_WRITE); + kthread_unuse_mm(parent); + KUNIT_EXPECT_EQ(test, protect[1], 0L); + for (i = 0; i < 2; i++) + KUNIT_EXPECT_TRUE(test, pte_write(test_fault_entry(test, parent, TEST_VA + i * SZ_64K))); + vma = test_vma_lookup(parent, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + } + mmap_read_lock(child); + zap_vma_range(cvma, TEST_VA, 2 * SZ_64K); + mmap_read_unlock(child); + mmap_read_lock(parent); + zap_vma_range(vma, TEST_VA, 2 * SZ_64K); + mmap_read_unlock(parent); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 0L); +} + +struct coarse_test_pin { + struct page *page; +}; + +static void coarse_test_unpin(void *arg) +{ + struct coarse_test_pin *pin = arg; + + if (pin->page) { + unpin_user_page(pin->page); + pin->page = NULL; + } +} + +static void user64k_pinned_fork_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + struct vm_area_struct *vma; + struct coarse_test_pin *pin; + struct folio *old, *new; + pte_t original, copied; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires pinned coarse leaf"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + original = test_fault_entry(test, parent, TEST_VA); + old = page_folio(pfn_to_page(pte_pfn(original))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(old, i)), 0x70 + i, PAGE_SIZE); + pin = kunit_kzalloc(test, sizeof(*pin), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pin); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_test_unpin, pin), 0); + mm_flags_set(MMF_HAS_PINNED, parent); + ret = try_grab_folio(old, 1, FOLL_PIN); + if (!ret) + pin->page = folio_page(old, nr - 1); + KUNIT_ASSERT_EQ(test, ret, 0); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + copied = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, copied), pte_phys_mm(parent, original)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(parent, test_fault_entry(test, parent, TEST_VA)), + pte_phys_mm(parent, original)); + new = page_folio(pfn_to_page(pte_pfn(copied))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)nr); + KUNIT_EXPECT_EQ(test, folio_mapcount(new), (int)nr); + for (i = 0; i < nr; i++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(new, i)), + 0x70 + i, PAGE_SIZE), NULL); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(new, i))); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(old, i))); + } + memset(page_address(pin->page), 0xe3, PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(new, nr - 1)), + 0x70 + nr - 1, PAGE_SIZE), NULL); + coarse_test_unpin(pin); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1L); +} + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +static void user4k_shmem_partial_hole_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 3 * PAGE_SIZE, true); + struct folio *folio; + loff_t start = PAGE_SIZE + SZ_4K + 17, end = PAGE_SIZE + 3 * SZ_4K + 16; + unsigned int i; + unsigned char byte = 0xa5; + loff_t write_pos = PAGE_SIZE + 2 * SZ_4K + 23; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), start, end); + folio = filemap_lock_folio(file->f_mapping, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), BIT(2)); + for (i = 0; i < PAGE_SIZE; i++) { + unsigned char expected = (i + PAGE_SIZE >= start && i + PAGE_SIZE <= end) ? + 0 : 0x31 + (PAGE_SIZE + i) / SZ_4K; + + if (((unsigned char *)folio_address(folio))[i] != expected) { + KUNIT_FAIL(test, "partial hole changed byte %u outside its range", i); + break; + } + } + /* Resident hole metadata has no dynamically allocated inode record. */ + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); + shmem_truncate_range(file_inode(file), PAGE_SIZE + SZ_4K, PAGE_SIZE + 2 * SZ_4K - 1); + KUNIT_ASSERT_EQ(test, kernel_write(file, &byte, 1, &write_pos), 1L); + folio = filemap_lock_folio(file->f_mapping, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), BIT(1)); + KUNIT_EXPECT_EQ(test, ((unsigned char *)folio_address(folio))[2 * SZ_4K + 23], byte); + folio_unlock(folio); + folio_put(folio); +} + +/* A surviving writable pin exposes bytes even after its PTE is revoked. */ +static void user4k_shmem_pinned_hole_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + struct folio *folio; + unsigned long offset = native ? 0 : SZ_4K; + unsigned long length = native ? PAGE_SIZE : SZ_4K; + unsigned long pfn; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, native, true, TEST_VA, length, offset); + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + KUNIT_ASSERT_EQ(test, fragment_test_pin(pins, vma->vm_mm, TEST_VA, + length, FOLL_WRITE | FOLL_LONGTERM, 1), 1L); + folio = pins->fragments[0].folio; + pfn = folio_pfn(folio); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, SZ_4K), NULL); + /* Model a completed device write through the original pin. */ + memset(folio_address(folio) + SZ_4K, 0xb7, SZ_4K); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + folio_unlock(folio); + fragment_test_unpin(pins); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_pfn(folio), pfn); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xb7, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x31, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + 2 * SZ_4K, 0x33, SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); + /* With the external writer gone, another hole can be missing. */ + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + folio_unlock(folio); + folio_put(folio); + } +} + +static void test_unpin_folio(void *folio) +{ + unpin_folio(folio); +} + +/* The direct memfd API must publish cache exposure without a user PTE. */ +static void user4k_shmem_memfd_pin_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *pin, *folio; + pgoff_t offset; + long ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + ret = memfd_pin_folios(file, SZ_4K, 2 * SZ_4K - 1, &pin, 1, &offset); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_unpin_folio, pin), 0); + KUNIT_EXPECT_EQ(test, offset, (pgoff_t)SZ_4K); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(pin)); + memset(folio_address(pin) + offset, 0xc8, SZ_4K); + folio_lock(pin); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(pin, 0, SZ_4K, SZ_4K)); + folio_unlock(pin); + kunit_release_action(test, test_unpin_folio, pin); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xc8, SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); +} + +static void user4k_shmem_replace_validity_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *old, *new; + unsigned long old_pfn; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + new = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, new); + folio_lock(new); + old = filemap_lock_folio(file->f_mapping, 0); + if (IS_ERR(old)) { + folio_unlock(new); + folio_put(new); + KUNIT_FAIL(test, "could not lock source cache folio"); + return; + } + old_pfn = folio_pfn(old); + memcpy(folio_address(new), folio_address(old), PAGE_SIZE); + __folio_set_swapbacked(new); + __folio_mark_uptodate(new); + folio_mark_dirty(new); + replace_page_cache_folio(old, new); + folio_add_lru(new); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(new, 0), BIT(1)); + KUNIT_EXPECT_NE(test, folio_pfn(new), old_pfn); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(new), 0x31, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(new) + SZ_4K, 0, SZ_4K), NULL); + folio_unlock(old); + folio_put(old); + folio_unlock(new); + folio_put(new); +} + +static void user4k_shmem_uffd_alias_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, false); + const unsigned long all = GENMASK(PAGE_SIZE / SZ_4K - 1, 0); + struct vm_area_struct *small, *native; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = shmem_uffd_set_missing(folio, 0, all); + KUNIT_EXPECT_TRUE(test, shmem_uffd_has_missing_native_page(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + small = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), all & ~BIT(1)); + KUNIT_EXPECT_FALSE(test, shmem_uffd_has_missing_native_page(folio)); + folio_unlock(folio); + folio_put(folio); + + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), 0UL); + folio_unlock(folio); + folio_put(folio); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); +} + +static void user4k_shmem_uffd_state_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 3 * PAGE_SIZE, false); + const unsigned long all = GENMASK(PAGE_SIZE / SZ_4K - 1, 0); + struct folio *folio; + unsigned long mask; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + for (i = 0; i < 3; i++) { + ret = shmem_get_folio(file_inode(file), i, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), 0UL); + ret = shmem_uffd_set_missing(folio, i, all & ~BIT(0)); + KUNIT_EXPECT_EQ(test, ret, 0); + shmem_uffd_mark_present(folio, i, BIT(1)); + mask = shmem_uffd_missing_mask(folio, i); + KUNIT_EXPECT_EQ(test, mask, all & ~GENMASK(1, 0)); + ret = shmem_uffd_set_missing(folio, i, BIT(PAGE_SIZE / SZ_4K)); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), mask); + folio_unlock(folio); + folio_put(folio); + } + /* Real truncate removes one index without discarding neighbouring state. */ + shmem_truncate_range(file_inode(file), PAGE_SIZE, 2 * PAGE_SIZE - 1); + KUNIT_EXPECT_PTR_EQ(test, xa_load(&SHMEM_I(file_inode(file))->uffd_missing, 1), NULL); + for (i = 0; i < 3; i += 2) { + folio = filemap_lock_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), all & ~GENMASK(1, 0)); + folio_unlock(folio); + folio_put(folio); + } + /* Generic batch deletion must not strand inode metadata either. */ + truncate_inode_pages(file->f_mapping, 0); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + ret = shmem_get_folio(file_inode(file), 1, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, PAGE_SIZE), NULL); + folio_unlock(folio); + folio_put(folio); +} + +#ifdef CONFIG_MIGRATION +static void user4k_shmem_uffd_migration_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, false); + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + struct folio *folio; + unsigned long old_pfn; + unsigned int succeeded = 0; + bool isolated; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xa5, SZ_4K); + folio_mark_dirty(folio); + ret = shmem_uffd_set_missing(folio, 0, missing); + old_pfn = folio_pfn(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, succeeded, 1U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_NE(test, folio_pfn(folio), old_pfn); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), missing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xa5, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, + PAGE_SIZE - SZ_4K), NULL); + shmem_uffd_mark_present(folio, 0, missing); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), 0UL); + folio_unlock(folio); + folio_put(folio); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); +} +#endif +#endif + +static struct kunit_case user4k_test_cases[] = { + KUNIT_CASE(user4k_exec_granule_policy_test), + KUNIT_CASE(user4k_aslr_window_test), + KUNIT_CASE(user4k_thp_eligibility_test), + KUNIT_CASE(user4k_geometry_test), + KUNIT_CASE(user4k_leaf_encoding_test), + KUNIT_CASE(user16k_leaf_encoding_test), + KUNIT_CASE(user4k_rmap_walk_test), + KUNIT_CASE(user4k_rmap_boundary_test), + KUNIT_CASE(native_rmap_walk_test), + KUNIT_CASE(user64k_rmap_lookup_test), + KUNIT_CASE(user64k_anon_rmap_span_test), + KUNIT_CASE(user64k_anon_zap_batch_test), + KUNIT_CASE(user_granule_zero_region_test), +#ifdef CONFIG_COREDUMP + KUNIT_CASE(user_granule_core_offsets_test), +#endif + KUNIT_CASE(user64k_fragment_ownership_test), + KUNIT_CASE(user64k_anon_fault_cow_test), + KUNIT_CASE(user64k_pinned_fork_test), + KUNIT_CASE(user64k_file_rmap_span_test), + KUNIT_CASE(user4k_vma_offsets_test), + KUNIT_CASE(user4k_vma_intervals_test), + KUNIT_CASE(user4k_vma_split_test), + KUNIT_CASE(user4k_vma_merge_test), + KUNIT_CASE(user4k_vma_partial_merge_test), + KUNIT_CASE(user4k_vma_copy_shrink_test), + KUNIT_CASE(user4k_anon_rmap_test), + KUNIT_CASE(user4k_anon_fork_rmap_test), + KUNIT_CASE(user4k_nonlinear_rmap_reuse_test), + KUNIT_CASE(user4k_dup_mmap_test), + KUNIT_CASE(user4k_anon_root_lifetime_test), + KUNIT_CASE(user4k_data_tlb_batch_test), + KUNIT_CASE(user4k_anon_zap_test), + KUNIT_CASE(user4k_anon_writable_slots_test), + KUNIT_CASE(user4k_anon_write_fault_test), + KUNIT_CASE(user16k_anon_write_fault_test), + KUNIT_CASE(user4k_mlock_counts_test), + KUNIT_CASE(user4k_mlock_shared_test), + KUNIT_CASE(user4k_mlock_repair_test), + KUNIT_CASE(user_granule_large_vma_split_test), + KUNIT_CASE(user_granule_brk_extent_test), + KUNIT_CASE(user4k_droppable_reclaim_test), + KUNIT_CASE(user4k_lazyfree_test), + KUNIT_CASE(user4k_zero_cow_test), + KUNIT_CASE(user16k_zero_cow_test), + KUNIT_CASE(user4k_shmem_swap_usage_test), +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + KUNIT_CASE(user4k_shmem_partial_hole_test), + KUNIT_CASE(user4k_shmem_pinned_hole_test), + KUNIT_CASE(user4k_shmem_memfd_pin_test), + KUNIT_CASE(user4k_shmem_replace_validity_test), + KUNIT_CASE(user4k_shmem_uffd_alias_test), + KUNIT_CASE(user4k_shmem_uffd_state_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_shmem_uffd_migration_test), +#endif +#endif + KUNIT_CASE(user4k_file_mapcounts_test), + KUNIT_CASE(user64k_file_promote_test), + KUNIT_CASE(user64k_shmem_fault_test), + KUNIT_CASE(user64k_shmem_fault_busy_test), + KUNIT_CASE(user64k_shmem_fault_hole_test), + KUNIT_CASE(user64k_file_leaf_test), + KUNIT_CASE(user64k_file_cow_test), + KUNIT_CASE(user64k_file_bounds_test), + KUNIT_CASE(user4k_file_shared_test), + KUNIT_CASE(user4k_iommufd_lifetime_test), + KUNIT_CASE(user4k_iommufd_rollback_test), + KUNIT_CASE(user4k_iommufd_granule_test), + KUNIT_CASE(user4k_iommufd_shared_test), + KUNIT_CASE(user4k_iommufd_exit_account_test), + KUNIT_CASE(user4k_iova_bitmap_test), + KUNIT_CASE(user4k_iova_bitmap_window_test), + KUNIT_CASE(user4k_iova_bitmap_retained_test), + KUNIT_CASE(user4k_iova_bitmap_partial_test), + KUNIT_CASE(user4k_mremap_move_test), + KUNIT_CASE(user4k_mremap_resize_test), + KUNIT_CASE(user4k_stack_relocate_test), + KUNIT_CASE(user4k_exec_arguments_test), + KUNIT_CASE(user64k_exec_arguments_test), + KUNIT_CASE(user4k_special_mapping_test), + KUNIT_CASE(user4k_insert_fragment_test), + KUNIT_CASE(user16k_insert_shared_test), + KUNIT_CASE(user4k_insert_range_test), + KUNIT_CASE(user4k_pfn_offset_test), + KUNIT_CASE(user4k_remap_pfn_range_test), + KUNIT_CASE(user4k_iomem_access_test), +#if defined(CONFIG_GENERIC_GETTIMEOFDAY) && defined(CONFIG_VDSO_GETRANDOM) + KUNIT_CASE(user4k_vdso_mapping_test), +#endif + KUNIT_CASE(user4k_mremap_cow_test), + KUNIT_CASE(user4k_mremap_file_test), + KUNIT_CASE(user4k_msync_offsets_test), + KUNIT_CASE(user4k_prefault_dirty_test), + KUNIT_CASE(user4k_cold_batch_test), + KUNIT_CASE(user4k_remote_access_test), + KUNIT_CASE(user4k_remote_file_test), +#ifdef CONFIG_CROSS_MEMORY_ATTACH + KUNIT_CASE(user4k_process_vm_test), +#endif +#ifdef CONFIG_FUTEX + KUNIT_CASE(user4k_futex_keys_test), + KUNIT_CASE(user4k_futex_wake_test), +#endif + KUNIT_CASE(user4k_mmap_descriptor_test), + KUNIT_CASE(user4k_mmap_region_offsets_test), + KUNIT_CASE(user4k_commit_accounting_test), + KUNIT_CASE(mixed_locked_account_units_test), + KUNIT_CASE(user64k_commit_accounting_test), + KUNIT_CASE(user64k_page_table_check_test), +#ifdef CONFIG_PROC_FS + KUNIT_CASE(user4k_accounting_readers_test), +#endif + KUNIT_CASE(user_granule_prefault_test), + KUNIT_CASE(user4k_mmap_accounting_test), + KUNIT_CASE(user4k_vm_mmap_test), + KUNIT_CASE(user4k_mprotect_test), + KUNIT_CASE(user4k_mprotect_cow_batch_test), + KUNIT_CASE(user4k_mprotect_accounting_test), + KUNIT_CASE(user4k_file_cow_test), + KUNIT_CASE(user4k_file_eof_test), + KUNIT_CASE(user4k_file_truncate_test), + KUNIT_CASE(user4k_filemap_fault_test), + KUNIT_CASE(user4k_file_invalidate_test), + KUNIT_CASE(user4k_file_fragments_test), + KUNIT_CASE(user4k_file_fork_hole_test), + KUNIT_CASE(user4k_swap_encoding_test), + KUNIT_CASE(user4k_cow_reuse_test), + KUNIT_CASE(user16k_cow_reuse_test), + KUNIT_CASE(user4k_fork_cow_test), + KUNIT_CASE(user16k_fork_cow_test), + KUNIT_CASE(user4k_slot_pin_test), + KUNIT_CASE(user4k_fragment_pin_test), + KUNIT_CASE(user4k_fragment_locked_pin_test), + KUNIT_CASE(user4k_folio_walk_geometry_test), + KUNIT_CASE(user4k_fragment_longterm_test), +#ifdef CONFIG_BLOCK + KUNIT_CASE(user4k_bio_fragments_test), + KUNIT_CASE(user4k_bio_trim_test), +#endif + KUNIT_CASE(user4k_fragment_permissions_test), + KUNIT_CASE(user16k_fragment_offset_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_migration_test), + KUNIT_CASE(user4k_file_migration_test), + KUNIT_CASE(user64k_file_migration_test), + KUNIT_CASE(user64k_anon_migration_test), + KUNIT_CASE(user4k_fragment_pin_race_test), + KUNIT_CASE(user4k_migration_entries_test), + KUNIT_CASE(user4k_nonlinear_rmap_test), + KUNIT_CASE(user4k_mprotect_migration_test), + KUNIT_CASE(user4k_live_migration_test), + KUNIT_CASE(user4k_migration_wait_test), +#endif + KUNIT_CASE(user4k_anon_exit_test), + KUNIT_CASE(user4k_anon_remote_zap_test), +#ifdef CONFIG_ARM64_CONTPTE + KUNIT_CASE(user4k_contpte_test), + KUNIT_CASE(native_contpte_test), +#endif + KUNIT_CASE(user4k_mm_tables_test), + KUNIT_CASE(user16k_mm_tables_test), + KUNIT_CASE(user64k_mm_tables_test), + KUNIT_CASE(native_mm_tables_test), + KUNIT_CASE(user4k_anon_read_fault_test), + KUNIT_CASE(user4k_mm_hardware_test), + KUNIT_CASE(user4k_fragment_lifetime_test), + KUNIT_CASE(user16k_fragment_lifetime_test), + KUNIT_CASE(user64k_table_owner_test), +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_CASE(native_pgtable_deposit_test), +#endif + KUNIT_CASE(user4k_translation_test), + KUNIT_CASE(user4k_context_uaccess_test), + KUNIT_CASE(user4k_fault_uaccess_test), + {} +}; + +#ifdef CONFIG_NUMA_BALANCING +/* + * The COW hinting expectations require a stable normal-balancing policy. + * Runtime sysctl writers do not take the fixture's mm or PTE locks, so run + * this case at boot and do not expose it through the debugfs rerun control. + */ +static struct kunit_case user4k_numa_init_cases[] = { + KUNIT_CASE(user_subpage_numa_cow_batch_test), + {} +}; + +static struct kunit_suite user4k_numa_init_suite = { + .name = "arm64-user4k-numa-init", + .test_cases = user4k_numa_init_cases, +}; + +kunit_test_init_section_suite(user4k_numa_init_suite); +#endif + +static struct kunit_suite user4k_test_suite = { + .name = "arm64-user4k", + .test_cases = user4k_test_cases, +}; + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_shmem_swap_suite, + &user4k_coarse_shmem_suite, + &user4k_nonlinear_swap_suite); +#else +kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_nonlinear_swap_suite); +#endif + +MODULE_LICENSE("GPL"); diff --git a/arch/arm64/mm/user4k-tlb-test.c b/arch/arm64/mm/user4k-tlb-test.c new file mode 100644 index 0000000000000..e5671256b7c76 --- /dev/null +++ b/arch/arm64/mm/user4k-tlb-test.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Decode emitted operands without executing TLBI instructions. */ +#include +#include +#include +#include + +struct tlbi_test_state { + u64 next; + u64 end; + u64 stride; + u16 asid; + u32 level; + unsigned int shift; + unsigned int single_ops; + unsigned int range_ops; + bool lpa2; +}; + +static unsigned int test_tg(unsigned int shift) +{ + return shift == 12 ? 1 : shift == 14 ? 2 : 3; +} + +static void record_single(u64 arg) +{ + struct kunit *test = kunit_get_current_test(); + struct tlbi_test_state *state = test->priv; + u64 address = (arg & GENMASK_ULL(43, 0)) << 12; + u64 hint = 0; + + if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && state->level <= 3) + hint = (test_tg(state->shift) << 2) | state->level; + KUNIT_EXPECT_EQ(test, address, state->next); + KUNIT_EXPECT_EQ(test, arg >> 48, (u64)state->asid); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBI_TTL_MASK, arg), hint); + state->next += state->stride; + state->single_ops++; + KUNIT_EXPECT_LE(test, state->next, state->end); +} + +static void record_range(u64 arg) +{ + struct kunit *test = kunit_get_current_test(); + struct tlbi_test_state *state = test->priv; + u64 address = FIELD_GET(TLBIR_BADDR_MASK, arg) << (state->lpa2 ? 16 : state->shift); + u64 count = (FIELD_GET(TLBIR_NUM_MASK, arg) + 1) << + (5 * FIELD_GET(TLBIR_SCALE_MASK, arg) + 1); + + KUNIT_EXPECT_EQ(test, address, state->next); + KUNIT_EXPECT_EQ(test, arg >> 48, (u64)state->asid); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBIR_TG_MASK, arg), (u64)test_tg(state->shift)); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBIR_TTL_MASK, arg), + state->level > 3 ? 0ULL : (u64)state->level); + if (state->lpa2) + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(address, SZ_64K)); + state->next += count << state->shift; + state->range_ops++; + KUNIT_EXPECT_LE(test, state->next, state->end); +} + +static bool range_supported(void) +{ + return true; +} + +static bool range_unsupported(void) +{ + return false; +} + +static void test_flush(struct kunit *test, unsigned int shift, u64 start, + unsigned long pages, u64 stride, u32 level, bool lpa2, + bool has_range) +{ + struct tlbi_test_state state = { + .next = start, + .end = start + (pages << shift), + .stride = stride, + .asid = 0xa55a, + .level = level, + .shift = shift, + .lpa2 = lpa2, + }; + + test->priv = &state; + __flush_tlb_range_op_granule(record_single, record_range, start, pages, + stride, state.asid, level, lpa2, shift, + has_range ? range_supported : range_unsupported); + KUNIT_EXPECT_EQ(test, state.next, state.end); + if (has_range) { + KUNIT_EXPECT_LE(test, state.single_ops + state.range_ops, 20U); + if (pages >= 32) + KUNIT_EXPECT_GT(test, state.range_ops, 0U); + } else { + KUNIT_EXPECT_EQ(test, state.range_ops, 0U); + KUNIT_EXPECT_EQ(test, state.single_ops, (unsigned int)((pages << shift) / stride)); + } + test->priv = NULL; +} + +static void user4k_tlbi_operands_test(struct kunit *test) +{ + static const unsigned long counts[] = { + 0, 1, 2, 3, 31, 32, 33, 63, 64, 65, 2047, 2048, 2049, + 65535, 65536, 65537, MAX_TLBI_RANGE_PAGES - 1, MAX_TLBI_RANGE_PAGES, + }; + static const u32 levels[] = { 2, 3, TLBI_TTL_UNKNOWN }; + unsigned int shift, i, j; + + /* Force both algorithms using recording callbacks, independent of CPU caps. */ + for (shift = 12; shift <= 16; shift += 2) { + u64 size = 1UL << shift; + + for (j = 0; j < ARRAY_SIZE(levels); j++) { + for (i = 0; i < ARRAY_SIZE(counts); i++) { + unsigned long pages = counts[i]; + + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], false, true); + if (shift != 16) + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], true, true); + if (pages <= 65) + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], false, false); + } + } + /* Preserve a larger leaf stride when range instructions are absent. */ + test_flush(test, shift, SZ_1G, 3 * 512, 512 * size, 2, false, false); + /* Last virtual page must not lose high address bits. */ + test_flush(test, shift, (1ULL << 48) - size, 1, size, 3, false, false); + } +} + +static void user4k_tlb_gather_test(struct kunit *test) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + pte_t entries[3] = {}; + unsigned int small; + + KUNIT_ASSERT_NOT_NULL(test, mm); + for (small = 0; small < 2; small++) { + struct mmu_gather tlb = { .mm = mm, .start = ULONG_MAX }; + unsigned long size = small ? SZ_4K : PAGE_SIZE; + unsigned long address = SZ_4M + size; + + mm->context.flags = small ? MMCF_USER_4K : 0; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + tlb_remove_tlb_entries(&tlb, entries, 3, address); + KUNIT_EXPECT_EQ(test, tlb.start, address); + KUNIT_EXPECT_EQ(test, tlb.end, address + 3 * size); + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), 3); + tlb.cleared_pmds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), TLBI_TTL_UNKNOWN); + tlb.cleared_ptes = 0; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? SZ_2M : PMD_SIZE); + tlb.cleared_pmds = 0; + tlb.cleared_puds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? SZ_1G : PUD_SIZE); + tlb.cleared_puds = 0; + tlb.cleared_p4ds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? 1UL << 39 : P4D_SIZE); + tlb.freed_tables = 1; + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), TLBI_TTL_UNKNOWN); + } +} + +static struct kunit_case user4k_tlb_test_cases[] = { + KUNIT_CASE(user4k_tlbi_operands_test), + KUNIT_CASE(user4k_tlb_gather_test), + {} +}; + +static struct kunit_suite user4k_tlb_test_suite = { + .name = "arm64-user4k-tlb", + .test_cases = user4k_tlb_test_cases, +}; + +kunit_test_suite(user4k_tlb_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index ceae52fd7586d..33f6c7e9608d5 100644 --- a/drivers/vfio/Kconfig +++ b/drivers/vfio/Kconfig @@ -54,6 +54,18 @@ config VFIO_IOMMU_TYPE1 tristate default n +config VFIO_TYPE1_ACCOUNT_KUNIT_TEST + bool "Test VFIO type1 pin accounting and retained fragments" if !KUNIT_ALL_TESTS + depends on KUNIT=y && VFIO_IOMMU_TYPE1=y + default KUNIT_ALL_TESTS + help + Exercise locked-memory refunds after source-mm exit and DMA owner + changes. Arm64 mixed-granule configurations also test accounting + conversions between user and native page sizes. + Mixed-granule tests exercise retained pins, mappings, replay and + external page arrays with recorded IOMMU translations. No physical + device DMA is performed. + config VFIO_IOMMU_SPAPR_TCE tristate depends on SPAPR_TCE_IOMMU diff --git a/drivers/vfio/vfio_iommu_type1_account_test.c b/drivers/vfio/vfio_iommu_type1_account_test.c new file mode 100644 index 0000000000000..5447258ce49b8 --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_account_test.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by type1 so the tests exercise the actual accounting helpers. */ +#include +#include + +struct vfio_account_mm { + struct mm_struct *mm; + bool live; +}; + +static void vfio_account_mm_free(void *data) +{ + struct vfio_account_mm *ctx = data; + + if (ctx->live) + mmput(ctx->mm); + mmdrop(ctx->mm); +} + +static struct vfio_account_mm *vfio_account_mm_new(struct kunit *test, + unsigned int shift) +{ + struct vfio_account_mm *ctx; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return NULL; + ctx->mm = mm_alloc(); + if (!ctx->mm) + return NULL; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (shift != PAGE_SHIFT) { + pgd_t *old = ctx->mm->pgd, *pgd; + unsigned long flag = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + + ctx->mm->context.flags |= flag; + pgd = pgd_alloc(ctx->mm); + ctx->mm->context.flags &= ~flag; + if (!pgd) { + mmput(ctx->mm); + return NULL; + } + pgd_free(ctx->mm, old); + ctx->mm->pgd = pgd; + ctx->mm->context.flags |= flag; + } +#endif + ctx->live = true; + mmgrab(ctx->mm); + if (kunit_add_action_or_reset(test, vfio_account_mm_free, ctx)) + return NULL; + return ctx; +} + +static void vfio_account_dma_free(void *data) +{ + struct vfio_dma *dma = data; + + /* Cleanup must also work with the negative-control implementation. */ + if (dma->locked_vm) + mm_lock_acct(dma->task, dma->mm, true, -dma->locked_vm); + mmdrop(dma->mm); + put_task_struct(dma->task); +} + +static struct vfio_dma *vfio_account_dma_new(struct kunit *test, + struct vfio_account_mm *ctx) +{ + struct vfio_dma *dma = kunit_kzalloc(test, sizeof(*dma), GFP_KERNEL); + + if (!dma) + return NULL; + dma->mm = ctx->mm; + mmgrab(dma->mm); + dma->task = get_task_struct(current->group_leader); + dma->lock_cap = true; + if (kunit_add_action_or_reset(test, vfio_account_dma_free, dma)) + return NULL; + return dma; +} + +static void vfio_account_exit_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + 12, 14, 16, +#endif + }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_dma *dma; + + if (i && shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_EQ(test, mm_page_shift(ctx->mm), shifts[i]); + dma = vfio_account_dma_new(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, 2, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(ctx->mm), 2UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 2UL * PAGE_SIZE); + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, 1, true), -ESRCH); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, -1, true), 0); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(ctx->mm), 1UL); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, -1, true), 0); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + kunit_info(test, "VFIO %uK user/%luK native dead-mm refunds checked", + 1U << (shifts[i] - 10), PAGE_SIZE >> 10); + } +} + +static void vfio_account_owner_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + 12, 14, 16, +#endif + }; + + for (unsigned int reverse = 0; reverse < 2; reverse++) { + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *old, *next; + struct vfio_dma *dma; + + if ((i || reverse) && shifts[i] == PAGE_SHIFT) + continue; + old = vfio_account_mm_new(test, reverse ? shifts[i] : PAGE_SHIFT); + next = vfio_account_mm_new(test, reverse ? PAGE_SHIFT : shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + dma = vfio_account_dma_new(test, old); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, 3, true), 0); + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&old->mm->mm_users), 0); + if (task_rlimit(dma->task, RLIMIT_MEMLOCK) != RLIM_INFINITY) { + unsigned long budget = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> + MM_ACCOUNT_SHIFT; + + /* Model a recipient whose existing pins consume its budget. */ + next->mm->locked_vm = budget; + KUNIT_EXPECT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, false), + -ENOMEM); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, old->mm); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 3UL); + KUNIT_EXPECT_EQ(test, next->mm->locked_vm, budget); + next->mm->locked_vm = 0; + } + KUNIT_ASSERT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, next->mm); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 3UL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 3UL * PAGE_SIZE); + /* Same-mm ownership is idempotent; it must not double the charge. */ + KUNIT_ASSERT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 3UL); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, -3, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 0UL); + kunit_info(test, "VFIO dead %uK mm -> %uK mm ownership refund checked", + 1U << (mm_page_shift(old->mm) - 10), + 1U << (mm_page_shift(next->mm) - 10)); + } + } +} + +static struct kunit_case vfio_account_cases[] = { + KUNIT_CASE(vfio_account_exit_test), + KUNIT_CASE(vfio_account_owner_test), + {} +}; + +static struct kunit_suite vfio_account_suite = { + .name = "vfio-type1-account", + .test_cases = vfio_account_cases, +}; + +kunit_test_suite(vfio_account_suite); diff --git a/drivers/vfio/vfio_iommu_type1_fragments_test.c b/drivers/vfio/vfio_iommu_type1_fragments_test.c new file mode 100644 index 0000000000000..60fffff10c6be --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_fragments_test.c @@ -0,0 +1,1177 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Actual type1 lifecycle with recorded translations; no physical device DMA. */ +#include +#include +#include +#include + +#define VFIO_TEST_VA SZ_64M +#define VFIO_TEST_IOVA SZ_256M + +struct vfio_test_domain { + struct iommu_domain core; + struct vfio_domain type1; + struct xarray translations; + unsigned long mapped; + unsigned long fail_after; + unsigned long largest_map; + unsigned int syncs; + struct folio *observed; + struct kunit *test; +}; + +struct vfio_fragment_fixture { + struct vfio_iommu *iommu; + struct vfio_test_domain domains[2]; + struct vfio_iommu_group group; + struct vfio_device device; + struct kunit *test; + unsigned int external_refs; + unsigned int notifications; +}; + +static int vfio_test_domain_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t phys, size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + unsigned long length = pgsize * pgcount; + + mock->largest_map = max(mock->largest_map, length); + for (unsigned long done = 0; done < length; done += SZ_4K) { + int ret; + + if (mock->fail_after && mock->mapped >= mock->fail_after) + return -ENOMEM; + ret = xa_insert(&mock->translations, (iova + done) >> 12, + xa_mk_value((phys + done) >> 12), gfp); + if (ret) + return ret; + mock->mapped += SZ_4K; + *mapped += SZ_4K; + } + return 0; +} + +static size_t vfio_test_domain_unmap(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + unsigned long length = pgsize * pgcount, done; + + for (done = 0; done < length; done += SZ_4K) { + if (!xa_erase(&mock->translations, (iova + done) >> 12)) + break; + mock->mapped -= SZ_4K; + } + return done; +} + +static phys_addr_t vfio_test_domain_translate(struct iommu_domain *domain, dma_addr_t iova) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + void *entry = xa_load(&mock->translations, iova >> 12); + + return entry ? ((phys_addr_t)xa_to_value(entry) << 12) + (iova & (SZ_4K - 1)) : 0; +} + +static void vfio_test_domain_sync(struct iommu_domain *domain, + struct iommu_iotlb_gather *gather) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + + mock->syncs++; + if (mock->observed) + KUNIT_EXPECT_TRUE(mock->test, folio_maybe_dma_pinned(mock->observed)); +} + +static const struct iommu_domain_ops vfio_test_domain_ops = { + .map_pages = vfio_test_domain_map, + .unmap_pages = vfio_test_domain_unmap, + .iova_to_phys = vfio_test_domain_translate, + .iotlb_sync = vfio_test_domain_sync, +}; + +static void vfio_test_notify(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_fragment_fixture *f = container_of(device, typeof(*f), device); + + f->notifications++; + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } +} + +static const struct vfio_device_ops vfio_test_device_ops = { + .dma_unmap = vfio_test_notify, +}; + +static void vfio_fragment_fixture_free(void *ptr) +{ + struct vfio_fragment_fixture *f = ptr; + struct rb_node *node; + + while ((node = rb_first(&f->iommu->dma_list))) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + struct rb_node *pin; + + while ((pin = rb_first(&dma->pfn_list))) { + struct vfio_pfn *vpfn = rb_entry(pin, struct vfio_pfn, node); + + vfio_unpin_page_external(dma, vpfn->iova, + list_empty(&f->iommu->domain_list)); + } + vfio_remove_dma(f->iommu, dma); + } + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + KUNIT_EXPECT_EQ(f->test, f->domains[i].mapped, 0UL); + KUNIT_EXPECT_TRUE(f->test, xa_empty(&f->domains[i].translations)); + xa_destroy(&f->domains[i].translations); + } + kfree(f->iommu); +} + +static struct vfio_fragment_fixture *vfio_fragment_fixture_new(struct kunit *test) +{ + struct vfio_fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + + if (!f) + return NULL; + f->test = test; + f->iommu = vfio_iommu_type1_open(VFIO_TYPE1v2_IOMMU); + if (IS_ERR(f->iommu)) + return NULL; + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + struct vfio_test_domain *mock = &f->domains[i]; + + xa_init(&mock->translations); + mock->test = test; + mock->core.ops = &vfio_test_domain_ops; + mock->core.type = IOMMU_DOMAIN_UNMANAGED; + mock->core.pgsize_bitmap = SZ_4K | PAGE_SIZE; + mock->type1.domain = &mock->core; + INIT_LIST_HEAD(&mock->type1.next); + INIT_LIST_HEAD(&mock->type1.group_list); + } + f->group.iommu_group = (struct iommu_group *)f; + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + f->iommu->num_non_pinned_groups = 1; + f->device.ops = &vfio_test_device_ops; + vfio_iommu_type1_register_device(f->iommu, &f->device); + if (kunit_add_action_or_reset(test, vfio_fragment_fixture_free, f)) + return NULL; + return f; +} + +static void vfio_test_file_put(void *ptr) +{ + fput(ptr); +} + +static struct file *vfio_fragment_file(struct kunit *test, unsigned long length, + unsigned char value) +{ + struct file *file = shmem_file_setup("vfio-fragments", length, EMPTY_VMA_FLAGS); + unsigned char *data; + loff_t pos = 0; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, vfio_test_file_put, file)) + return ERR_PTR(-ENOMEM); + data = kunit_kmalloc(test, length, GFP_KERNEL); + if (!data) + return ERR_PTR(-ENOMEM); + memset(data, value, length); + if (kernel_write(file, data, length, &pos) != length) + return ERR_PTR(-EIO); + return file; +} + +static unsigned long vfio_fragment_source(struct vfio_account_mm *ctx, + struct file *file, unsigned long length) +{ + unsigned long addr; + + kthread_use_mm(ctx->mm); + addr = vm_mmap(file, VFIO_TEST_VA, length, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(ctx->mm); + return addr; +} + +static int vfio_fragment_test_map(struct vfio_fragment_fixture *f, + struct vfio_account_mm *ctx, + unsigned long offset, unsigned long length) +{ + struct vfio_iommu_type1_dma_map map = { + .argsz = sizeof(map), + .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, + .vaddr = VFIO_TEST_VA + offset, + .iova = VFIO_TEST_IOVA, + .size = length, + }; + int ret; + + kthread_use_mm(ctx->mm); + ret = vfio_dma_do_map(f->iommu, &map); + kthread_unuse_mm(ctx->mm); + return ret; +} + +static void vfio_fragments_lifetime_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + unsigned char data[64]; + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, .size = 2 * PAGE_SIZE, + }; + struct vfio_bitmap bitmap = {}; + + if (shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x71); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + for (unsigned int d = 0; d < 2; d++) + f->domains[d].observed = vfio_fragment_find(dma, 0)->pin.folio; + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_GE(test, f->domains[0].largest_map, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 17, 1, IOMMU_READ | IOMMU_WRITE, &page), 1); + f->external_refs++; + KUNIT_EXPECT_EQ(test, page_to_phys(page), + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* The user VA goes away; retained pins are still the DMA buffer. */ + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x71, sizeof(data)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + KUNIT_EXPECT_EQ(test, f->domains[1].mapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA), + iommu_iova_to_phys(&f->domains[1].core, VFIO_TEST_IOVA)); + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, f->notifications, 1U); + KUNIT_EXPECT_GT(test, f->domains[0].syncs, 0U); + KUNIT_EXPECT_GT(test, f->domains[1].syncs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + kunit_info(test, "type1 retained %uK user/%luK native pins after mm exit and replay", + 1U << (shifts[i] - 10), PAGE_SIZE >> 10); + } +} + +static void vfio_fragments_lazy_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct file *file; + struct vfio_dma *dma; + struct page *page; + unsigned char data[64]; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x52); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x52, sizeof(data)), NULL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 37, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + PAGE_SIZE, 1, IOMMU_READ, &page), -EFAULT); + for (unsigned int i = 0; i < 3; i++) + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 37, 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->fragments->records)); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); +} + +static void vfio_fragments_map_failure_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct file *file; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x25); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + f->domains[1].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), -ENOMEM); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&f->iommu->dma_list)); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, f->domains[1].mapped, 0UL); + f->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); +} + +static void vfio_fragments_fine_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page = NULL; + unsigned char data[64]; + unsigned long length = PAGE_SIZE + SZ_4K; + size_t copied = 0; + + if (i && shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x64); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, SZ_4K, length), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, length); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* The physical base is offset inside a native page, not representable. */ + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), -EINVAL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + PAGE_SIZE, 1, IOMMU_READ, &page), -EINVAL); + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, + vfio_dirty_pgsize(f->iommu)), 0); + f->iommu->dirty_page_tracking = true; + memset(data, 0x97, sizeof(data)); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw_chunk(f->iommu, + VFIO_TEST_IOVA + SZ_4K - 17, data, sizeof(data), true, &copied), 0); + KUNIT_EXPECT_EQ(test, copied, sizeof(data)); + KUNIT_EXPECT_EQ(test, bitmap_weight(dma->bitmap, length / SZ_4K), 2U); + KUNIT_EXPECT_TRUE(test, test_bit(0, dma->bitmap)); + KUNIT_EXPECT_TRUE(test, test_bit(1, dma->bitmap)); + memset(data, 0, sizeof(data)); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, + VFIO_TEST_IOVA + SZ_4K - 17, data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x97, sizeof(data)), NULL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, + VFIO_TEST_IOVA + length, data, sizeof(data), false), -EINVAL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_info(test, "%uK userspace fine IOVA/physical offsets and dirty bytes checked", + 1U << (shifts[i] - 10)); + } +} + +static void vfio_fragments_replay_failure_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x58); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + f->domains[0].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), -ENOMEM); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_FALSE(test, dma->iommu_mapped); + f->domains[0].fail_after = 0; + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), 0); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* Detaching the last hardware domain keeps only the external page. */ + vfio_iommu_unmap_unpin_reaccount(f->iommu); + list_del_init(&f->domains[0].type1.next); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); +} + +static void vfio_fragments_anon_owner_test(struct kunit *test) +{ + struct vfio_account_mm *old = vfio_account_mm_new(test, 12); + struct vfio_account_mm *next = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + unsigned char *data = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL); + unsigned long addr, copied, charge; + unsigned char check[64]; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + KUNIT_ASSERT_NOT_NULL(test, data); + memset(data, 0x49, PAGE_SIZE); + kthread_use_mm(old->mm); + addr = vm_mmap(NULL, VFIO_TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + copied = addr == VFIO_TEST_VA ? + copy_to_user((void __user *)addr, data, PAGE_SIZE) : PAGE_SIZE; + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, copied, 0UL); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, old, 0, PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, vfio_fragment_find(dma, 0)->pin.subpage); + charge = dma->locked_vm; + KUNIT_ASSERT_GT(test, charge, 0UL); + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(old->mm), 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), charge); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + check, sizeof(check), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(check, 0x49, sizeof(check)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 0UL); + kunit_info(test, "anonymous 4K slot pins survive source exit and native-owner transfer"); +} + +static void vfio_fragments_domain_granule_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, PAGE_SIZE, 0x73); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, + vfio_dirty_pgsize(f->iommu)), 0); + f->iommu->dirty_page_tracking = true; + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 31, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, bitmap_weight(dma->bitmap, PAGE_SIZE / SZ_4K), + (unsigned int)(PAGE_SIZE / SZ_4K)); + f->domains[0].core.pgsize_bitmap = SZ_16K; + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), 0); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, 1UL << __ffs(f->iommu->pgsize_bitmap), SZ_16K); + KUNIT_EXPECT_EQ(test, vfio_dirty_pgsize(f->iommu), SZ_4K); + vfio_iommu_unmap_unpin_reaccount(f->iommu); + list_del_init(&f->domains[0].type1.next); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, 1UL << __ffs(f->iommu->pgsize_bitmap), SZ_4K); + KUNIT_EXPECT_EQ(test, vfio_dirty_pgsize(f->iommu), SZ_4K); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 31, 1); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); +} + +static void vfio_fragments_native_owner_test(struct kunit *test) +{ + for (unsigned int variant = 0; variant < 9; variant++) { + bool same_mm = variant == 8; + bool mapped = same_mm || (variant & 1); + bool alias = same_mm || (variant & 2), different = variant & 4; + struct vfio_account_mm *old = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_account_mm *next = same_mm ? old : vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_fragment_provider *prepared; + struct file *file, *replacement; + struct vfio_dma *dma; + struct page *pages[2]; + phys_addr_t external[2]; + unsigned long unique, addr; + unsigned char data[32]; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x63); + replacement = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x44); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, replacement); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(old, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + if (alias) { + kthread_use_mm(old->mm); + addr = vm_mmap(file, VFIO_TEST_VA + PAGE_SIZE, PAGE_SIZE, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, 0); + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA + PAGE_SIZE); + } + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, old, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NULL(test, dma->fragments); + if (different) { + kthread_use_mm(old->mm); + addr = vm_mmap(replacement, VFIO_TEST_VA, 2 * PAGE_SIZE, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, 0); + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA); + } + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 17, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + for (unsigned int i = 0; i < 2; i++) + external[i] = page_to_phys(pages[i]); + if (mapped && different) + KUNIT_EXPECT_NE(test, external[0], + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + KUNIT_ASSERT_EQ(test, dma->locked_vm, 2UL); + if (!same_mm) { + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&old->mm->mm_users), 0); + } + + prepared = vfio_fragments_prepare_native(f->iommu, dma); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, prepared); + unique = mapped ? (alias ? 1UL : 2UL) : 0; + if (!mapped || different) + unique += different || !alias ? 2UL : 1UL; + KUNIT_EXPECT_EQ(test, vfio_fragments_native_pages(prepared), unique); + /* Failed new-owner credit must not publish metadata or release pins. */ + if (!same_mm && task_rlimit(dma->task, RLIMIT_MEMLOCK) != RLIM_INFINITY) { + unsigned long budget = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> + mm_page_shift(next->mm); + + next->mm->locked_vm = budget; + KUNIT_EXPECT_EQ(test, vfio_change_dma_mm_pages(dma, dma->task, + next->mm, false, unique), -ENOMEM); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, old->mm); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 2UL); + KUNIT_EXPECT_EQ(test, next->mm->locked_vm, budget); + next->mm->locked_vm = 0; + } + vfio_fragments_abort_native(prepared); + KUNIT_EXPECT_PTR_EQ(test, dma->fragments, NULL); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(page_folio(pages[0]))); + kthread_use_mm(next->mm); + ret = vfio_change_dma_owner(f->iommu, dma, + VFIO_TEST_VA + (same_mm ? SZ_4K : 0)); + kthread_unuse_mm(next->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, unique); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), same_mm ? unique : 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), unique); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, + memchr_inv(data, !mapped && different ? 0x44 : 0x63, sizeof(data)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 31, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + for (unsigned int i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, page_to_phys(pages[i]), external[i]); + KUNIT_EXPECT_EQ(test, dma->locked_vm, unique); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 31, 2); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 17, 2); + KUNIT_EXPECT_EQ(test, dma->locked_vm, + mapped ? (alias ? 1UL : 2UL) : unique); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 0UL); + kunit_info(test, "native ownership variant=%u preserves PIN identity and unique charge", + variant); + } +} + +static int vfio_fragment_map_area(struct vfio_fragment_fixture *f, + struct vfio_account_mm *ctx, unsigned long iova_offset, + unsigned long va_offset, unsigned long length, unsigned int flags) +{ + struct vfio_iommu_type1_dma_map map = { + .argsz = sizeof(map), + .flags = flags, + .vaddr = VFIO_TEST_VA + va_offset, + .iova = VFIO_TEST_IOVA + iova_offset, + .size = length, + }; + int ret; + + kthread_use_mm(ctx->mm); + ret = vfio_dma_do_map(f->iommu, &map); + kthread_unuse_mm(ctx->mm); + return ret; +} + +static void vfio_fragments_external_span_test(struct kunit *test) +{ + unsigned int rw = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_account_mm *a = vfio_account_mm_new(test, 12); + struct vfio_account_mm *b = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x6b); + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + struct vfio_dma *first, *rest; + struct page *pages[3]; + unsigned long bits = 2 * PAGE_SIZE / SZ_4K - 1; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(a, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(b, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, a, 0, 0, SZ_4K, rw), 0); + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, b, SZ_4K, SZ_4K, + 2 * PAGE_SIZE - SZ_4K, rw), 0); + first = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + rest = vfio_find_dma(f->iommu, VFIO_TEST_IOVA + SZ_4K, 1); + KUNIT_ASSERT_NOT_NULL(test, first); + KUNIT_ASSERT_NOT_NULL(test, rest); + f->iommu->dirty_page_tracking = true; + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, SZ_4K), 0); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 37, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + KUNIT_EXPECT_EQ(test, first->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, rest->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, bitmap_weight(first->bitmap, 1), 1); + KUNIT_EXPECT_EQ(test, bitmap_weight(rest->bitmap, bits), (int)bits); + bitmap_clear(rest->bitmap, 0, bits); + vfio_dma_populate_bitmap(rest, SZ_4K); + KUNIT_EXPECT_EQ(test, bitmap_weight(rest->bitmap, bits), (int)bits); + if (mapped) + KUNIT_EXPECT_EQ(test, page_to_phys(pages[0]), + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + /* A failed larger request must leave prior caller references intact. */ + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 3, IOMMU_READ, pages), -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, pages[0], NULL); + KUNIT_EXPECT_PTR_EQ(test, pages[1], NULL); + KUNIT_EXPECT_PTR_EQ(test, pages[2], NULL); + a->live = b->live = false; + mmput(a->mm); + mmput(b->mm); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 2, IOMMU_READ, pages), 2); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 2); + /* Release the second page first: it shares the second DMA owner. */ + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + PAGE_SIZE + 7, 1); + KUNIT_EXPECT_EQ(test, rest->locked_vm, mapped ? 2UL : 1UL); + f->external_refs = 1; + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, f->notifications, 1U); + KUNIT_EXPECT_EQ(test, f->external_refs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(a->mm), 0UL); + KUNIT_EXPECT_EQ(test, rest->locked_vm, mapped ? 2UL : 0UL); + unmap.iova += SZ_4K; + unmap.size = 2 * PAGE_SIZE - SZ_4K; + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(b->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + } + for (unsigned int failure = 0; failure < 4; failure++) { + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x3a); + struct page *page = NULL; + struct rb_node *node; + unsigned long va = failure == 2 ? PAGE_SIZE + SZ_4K : SZ_4K; + int expected = failure == 1 ? -EPERM : -EINVAL; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, 0, 0, SZ_4K, rw), 0); + if (failure == 3) + va += SZ_4K; + if (failure != 0) + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, SZ_4K, va, + SZ_4K, failure == 1 ? VFIO_DMA_MAP_FLAG_READ : rw), 0); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 1, + IOMMU_READ | IOMMU_WRITE, &page), expected); + KUNIT_EXPECT_PTR_EQ(test, page, NULL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + for (node = rb_first(&f->iommu->dma_list); node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->pfn_list)); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->fragments->records)); + } + kunit_release_action(test, vfio_fragment_fixture_free, f); + } +} + +#define VFIO_STRESS_CYCLES 1024 + +struct vfio_pin_stress { + struct vfio_fragment_fixture *fixture; + struct vfio_device consumer; + struct vfio_device registrar; + struct vfio_device observer; + struct task_struct *competing_thread; + struct task_struct *pin_thread; + struct task_struct *register_thread; + struct mutex consumer_lock; + struct completion pin_go; + struct completion pin_ready; + struct completion notify_entered; + struct completion pin_done; + struct completion register_go; + struct completion register_entered; + struct completion register_done; + struct completion competing_go; + struct completion competing_done; + unsigned long competing_size; + int competing_ret; + bool competing; + unsigned int competing_unmaps; + unsigned int observations; + atomic_t error; + unsigned int pins; + unsigned int registrations; + bool stopping; +}; + +static void vfio_stress_error(struct vfio_pin_stress *s, int ret) +{ + atomic_cmpxchg(&s->error, 0, ret < 0 ? ret : -EIO); +} + +static void vfio_stress_notify(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_pin_stress *s = container_of(device, typeof(*s), consumer); + struct vfio_fragment_fixture *f = s->fixture; + + /* The callback holds the production device-list lock, not the DMA lock. */ + complete(&s->register_go); + if (!wait_for_completion_timeout(&s->register_entered, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + complete(&s->notify_entered); + mutex_lock(&s->consumer_lock); + f->notifications++; + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + mutex_unlock(&s->consumer_lock); + if (s->competing) { + /* A second unmap may free the DMA while this callback still runs. */ + complete(&s->competing_go); + if (!wait_for_completion_timeout(&s->competing_done, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + } +} + +static void vfio_stress_observe(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_pin_stress *s = container_of(device, typeof(*s), observer); + + s->observations++; + if (iova != VFIO_TEST_IOVA || length != SZ_4K) + vfio_stress_error(s, -ESTALE); +} + +static void vfio_stress_empty_notify(struct vfio_device *device, u64 iova, u64 length) +{ +} + +static const struct vfio_device_ops vfio_stress_consumer_ops = { + .dma_unmap = vfio_stress_notify, +}; + +static const struct vfio_device_ops vfio_stress_registrar_ops = { + .dma_unmap = vfio_stress_empty_notify, +}; + +static const struct vfio_device_ops vfio_stress_observer_ops = { + .dma_unmap = vfio_stress_observe, +}; + +static int vfio_stress_competing_unmaps(void *data) +{ + struct vfio_pin_stress *s = data; + + for (;;) { + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, + .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + + wait_for_completion(&s->competing_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + s->competing_ret = vfio_dma_do_unmap(s->fixture->iommu, &unmap, &bitmap); + s->competing_size = unmap.size; + s->competing_unmaps++; + complete(&s->competing_done); + } + return 0; +} + +static int vfio_stress_pinner(void *data) +{ + struct vfio_pin_stress *s = data; + struct vfio_fragment_fixture *f = s->fixture; + + for (;;) { + struct page *page = NULL, *original; + void *bytes; + int ret; + + wait_for_completion(&s->pin_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + mutex_lock(&s->consumer_lock); + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 37, 1, IOMMU_READ | IOMMU_WRITE, &page); + if (ret != 1) { + vfio_stress_error(s, ret); + mutex_unlock(&s->consumer_lock); + complete(&s->pin_ready); + complete(&s->pin_done); + continue; + } + f->external_refs++; + s->pins++; + original = page; + complete(&s->pin_ready); + /* Unmap cannot revoke a reference before the consumer publishes it. */ + if (!wait_for_completion_timeout(&s->notify_entered, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + if (!READ_ONCE(s->stopping)) { + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 17, 1, IOMMU_READ, &page); + if (ret == 1) { + f->external_refs++; + s->pins++; + if (page != original) + vfio_stress_error(s, -ESTALE); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 17, 1); + f->external_refs--; + } else { + vfio_stress_error(s, ret); + } + bytes = kmap_local_page(original); + if (memchr_inv(bytes, 0x6b, PAGE_SIZE)) + vfio_stress_error(s, -EIO); + kunmap_local(bytes); + } + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 37, 1); + f->external_refs--; + mutex_unlock(&s->consumer_lock); + complete(&s->pin_done); + } + return 0; +} + +static int vfio_stress_registrations(void *data) +{ + struct vfio_pin_stress *s = data; + + for (;;) { + wait_for_completion(&s->register_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + complete(&s->register_entered); + vfio_iommu_type1_register_device(s->fixture->iommu, &s->registrar); + vfio_iommu_type1_unregister_device(s->fixture->iommu, &s->registrar); + s->registrations++; + complete(&s->register_done); + } + return 0; +} + +static void vfio_stress_free(void *data) +{ + struct vfio_pin_stress *s = data; + struct vfio_fragment_fixture *f = s->fixture; + + WRITE_ONCE(s->stopping, true); + complete_all(&s->pin_go); + complete_all(&s->notify_entered); + complete_all(&s->register_go); + complete_all(&s->competing_go); + if (s->competing_thread) { + kthread_stop(s->competing_thread); + put_task_struct(s->competing_thread); + } + if (s->pin_thread) { + kthread_stop(s->pin_thread); + put_task_struct(s->pin_thread); + } + if (s->register_thread) { + kthread_stop(s->register_thread); + put_task_struct(s->register_thread); + } + mutex_lock(&s->consumer_lock); + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + mutex_unlock(&s->consumer_lock); + vfio_iommu_type1_unregister_device(f->iommu, &s->consumer); + vfio_iommu_type1_unregister_device(f->iommu, &s->observer); +} + +static struct vfio_pin_stress *vfio_stress_new(struct kunit *test, + struct vfio_fragment_fixture *f) +{ + struct vfio_pin_stress *s = kunit_kzalloc(test, sizeof(*s), GFP_KERNEL); + + if (!s) + return NULL; + s->fixture = f; + mutex_init(&s->consumer_lock); + init_completion(&s->pin_go); + init_completion(&s->pin_ready); + init_completion(&s->notify_entered); + init_completion(&s->pin_done); + init_completion(&s->register_go); + init_completion(&s->register_entered); + init_completion(&s->register_done); + init_completion(&s->competing_go); + init_completion(&s->competing_done); + atomic_set(&s->error, 0); + s->consumer.ops = &vfio_stress_consumer_ops; + s->registrar.ops = &vfio_stress_registrar_ops; + s->observer.ops = &vfio_stress_observer_ops; + vfio_iommu_type1_unregister_device(f->iommu, &f->device); + /* list_add puts the consumer before the observer in callback order. */ + vfio_iommu_type1_register_device(f->iommu, &s->observer); + vfio_iommu_type1_register_device(f->iommu, &s->consumer); + if (kunit_add_action_or_reset(test, vfio_stress_free, s)) + return NULL; + s->pin_thread = kthread_run(vfio_stress_pinner, s, "vfio-pin-stress"); + if (IS_ERR(s->pin_thread)) { + s->pin_thread = NULL; + return NULL; + } + get_task_struct(s->pin_thread); + s->register_thread = kthread_run(vfio_stress_registrations, s, "vfio-reg-stress"); + if (IS_ERR(s->register_thread)) { + s->register_thread = NULL; + return NULL; + } + get_task_struct(s->register_thread); + s->competing_thread = kthread_run(vfio_stress_competing_unmaps, s, "vfio-unmap-stress"); + if (IS_ERR(s->competing_thread)) { + s->competing_thread = NULL; + return NULL; + } + get_task_struct(s->competing_thread); + return s; +} + +static void vfio_fragments_concurrency_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14, 16 }; + unsigned int rw = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + + for (unsigned int shift = 0; shift < ARRAY_SIZE(shifts); shift++) { + if (shifts[shift] > PAGE_SHIFT) + continue; + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_account_mm *ctx = vfio_account_mm_new(test, shifts[shift]); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, PAGE_SIZE, 0x6b); + struct vfio_pin_stress *s; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, PAGE_SIZE), + VFIO_TEST_VA); + s = vfio_stress_new(test, f); + KUNIT_ASSERT_NOT_NULL(test, s); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + for (unsigned int cycle = 0; cycle < VFIO_STRESS_CYCLES; cycle++) { + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, + .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + bool fine = cycle & 1; + unsigned long remaining = PAGE_SIZE - SZ_4K; + + KUNIT_ASSERT_EQ(test, + vfio_fragment_map_area(f, ctx, 0, 0, SZ_4K, rw), 0); + for (unsigned long offset = SZ_4K; offset < PAGE_SIZE;) { + unsigned long length = fine ? SZ_4K : remaining; + + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, offset, + offset, length, rw), 0); + offset += length; + } + if (cycle == VFIO_STRESS_CYCLES - 1) { + struct page *page; + + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 1, + IOMMU_READ, &page), 1); + f->external_refs = 1; + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + } + s->competing = cycle & 2; + complete(&s->pin_go); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->pin_ready, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_EQ(test, atomic_read(&s->error), 0); + KUNIT_ASSERT_EQ(test, + vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, s->competing ? 0UL : SZ_4K); + if (s->competing) { + KUNIT_EXPECT_EQ(test, s->competing_ret, 0); + KUNIT_EXPECT_EQ(test, s->competing_size, SZ_4K); + } + unmap.iova += SZ_4K; + unmap.size = remaining; + KUNIT_ASSERT_EQ(test, + vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, remaining); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->pin_done, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->register_done, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_EQ(test, atomic_read(&s->error), 0); + KUNIT_EXPECT_EQ(test, f->external_refs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&f->iommu->dma_list)); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + reinit_completion(&s->pin_ready); + reinit_completion(&s->notify_entered); + reinit_completion(&s->pin_done); + reinit_completion(&s->register_entered); + reinit_completion(&s->register_done); + reinit_completion(&s->competing_done); + } + KUNIT_EXPECT_EQ(test, s->pins, 2U * VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->registrations, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, f->notifications, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->observations, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->competing_unmaps, VFIO_STRESS_CYCLES / 2U); + kunit_info(test, "%uK user/%luK native mapped=%u: %u forced pin/unmap/register cycles, half with competing unmap", + 1U << (shifts[shift] - 10), PAGE_SIZE >> 10, + mapped, VFIO_STRESS_CYCLES); + kunit_release_action(test, vfio_stress_free, s); + kunit_release_action(test, vfio_fragment_fixture_free, f); + } + } +} + +static struct kunit_case vfio_fragments_cases[] = { + KUNIT_CASE(vfio_fragments_lifetime_test), + KUNIT_CASE(vfio_fragments_lazy_test), + KUNIT_CASE(vfio_fragments_map_failure_test), + KUNIT_CASE(vfio_fragments_fine_test), + KUNIT_CASE(vfio_fragments_replay_failure_test), + KUNIT_CASE(vfio_fragments_anon_owner_test), + KUNIT_CASE(vfio_fragments_domain_granule_test), + KUNIT_CASE(vfio_fragments_native_owner_test), + KUNIT_CASE(vfio_fragments_external_span_test), + KUNIT_CASE(vfio_fragments_concurrency_test), + {} +}; + +static struct kunit_suite vfio_fragments_suite = { + .name = "vfio-type1-fragments", + .test_cases = vfio_fragments_cases, +}; + +kunit_test_suite(vfio_fragments_suite); diff --git a/drivers/vfio/vfio_iommu_type1_ioctl_test.c b/drivers/vfio/vfio_iommu_type1_ioctl_test.c new file mode 100644 index 0000000000000..33e981bcdf74d --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_ioctl_test.c @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included after the KUnit harness; real EL0 arguments, recorded domains only. */ +#include +#include +#include +#include "vfio_iommu_type1_test.h" + +struct vfio_ioctl_fixture { + struct vfio_fragment_fixture pins; + struct mutex request_lock; +}; + +struct vfio_ioctl_rw { + struct work_struct work; + struct vfio_iommu *iommu; + dma_addr_t iova; + void *data; + size_t size; + bool write; + int ret; +}; + +static void vfio_ioctl_rw_work(struct work_struct *work) +{ + struct vfio_ioctl_rw *rw = container_of(work, typeof(*rw), work); + + rw->ret = vfio_iommu_type1_dma_rw(rw->iommu, rw->iova, rw->data, + rw->size, rw->write); +} + +static int vfio_ioctl_test_open(struct inode *inode, struct file *file) +{ + struct vfio_ioctl_fixture *ctx; + struct vfio_fragment_fixture *f; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + f = &ctx->pins; + f->iommu = vfio_iommu_type1_open(VFIO_TYPE1v2_IOMMU); + if (IS_ERR(f->iommu)) { + int ret = PTR_ERR(f->iommu); + + kfree(ctx); + return ret; + } + mutex_init(&ctx->request_lock); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + struct vfio_test_domain *mock = &f->domains[i]; + + xa_init(&mock->translations); + mock->core.ops = &vfio_test_domain_ops; + mock->core.type = IOMMU_DOMAIN_UNMANAGED; + mock->core.pgsize_bitmap = SZ_4K | PAGE_SIZE; + mock->type1.domain = &mock->core; + INIT_LIST_HEAD(&mock->type1.next); + INIT_LIST_HEAD(&mock->type1.group_list); + } + f->group.iommu_group = (struct iommu_group *)ctx; + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + f->iommu->num_non_pinned_groups = 1; + f->device.ops = &vfio_test_device_ops; + vfio_iommu_type1_register_device(f->iommu, &f->device); + file->private_data = ctx; + return 0; +} + +static int vfio_ioctl_test_release(struct inode *inode, struct file *file) +{ + struct vfio_ioctl_fixture *ctx = file->private_data; + struct vfio_fragment_fixture *f = &ctx->pins; + + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + vfio_iommu_type1_unregister_device(f->iommu, &f->device); + vfio_iommu_unmap_unpin_all(f->iommu); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + list_del_init(&f->domains[i].type1.next); + xa_destroy(&f->domains[i].translations); + } + list_del(&f->group.next); + vfio_iommu_type1_release(f->iommu); + kfree(ctx); + return 0; +} + +static int vfio_ioctl_test_domain(struct vfio_fragment_fixture *f, + struct vfio_type1_test_cmd *cmd, bool add) +{ + struct vfio_test_domain *mock; + struct rb_node *node; + int ret = 0; + + if (cmd->domain >= ARRAY_SIZE(f->domains) || cmd->flags) + return -EINVAL; + mock = &f->domains[cmd->domain]; + mutex_lock(&f->iommu->lock); + if (add) { + if (!list_empty(&mock->type1.next)) { + ret = -EEXIST; + goto out; + } + if (cmd->size != SZ_4K && cmd->size != SZ_16K && cmd->size != SZ_64K) { + ret = -EINVAL; + goto out; + } + mock->core.pgsize_bitmap = cmd->size; + mock->fail_after = cmd->fail_after; + ret = vfio_iommu_replay(f->iommu, &mock->type1); + if (ret) + goto out; + list_add_tail(&mock->type1.next, &f->iommu->domain_list); + } else { + if (list_empty(&mock->type1.next)) { + ret = -ENOENT; + goto out; + } + if (list_is_singular(&f->iommu->domain_list)) { + vfio_iommu_unmap_unpin_reaccount(f->iommu); + } else { + for (node = rb_first(&f->iommu->dma_list); node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + iommu_unmap(&mock->core, dma->iova, dma->size); + } + } + list_del_init(&mock->type1.next); + } + vfio_update_pgsize_bitmap(f->iommu); +out: + mutex_unlock(&f->iommu->lock); + return ret; +} + +static int vfio_ioctl_test_rw(struct vfio_fragment_fixture *f, + struct vfio_type1_test_cmd *cmd) +{ + struct vfio_ioctl_rw rw = { + .iommu = f->iommu, .iova = cmd->iova, .size = cmd->size, + .write = cmd->flags & VFIO_TYPE1_TEST_WRITE, + }; + void __user *buffer = u64_to_user_ptr(cmd->buffer); + int ret; + + if (!cmd->size || cmd->size > 256 || + cmd->flags & ~(VFIO_TYPE1_TEST_WRITE | VFIO_TYPE1_TEST_WORKER)) + return -EINVAL; + rw.data = kzalloc(cmd->size, GFP_KERNEL); + if (!rw.data) + return -ENOMEM; + if (rw.write && copy_from_user(rw.data, buffer, cmd->size)) { + ret = -EFAULT; + goto out; + } + if (cmd->flags & VFIO_TYPE1_TEST_WORKER) { + INIT_WORK(&rw.work, vfio_ioctl_rw_work); + queue_work(system_unbound_wq, &rw.work); + flush_work(&rw.work); + ret = rw.ret; + } else { + ret = vfio_iommu_type1_dma_rw(f->iommu, rw.iova, rw.data, rw.size, rw.write); + } + if (!ret && !rw.write && copy_to_user(buffer, rw.data, cmd->size)) + ret = -EFAULT; +out: + kfree(rw.data); + return ret; +} + +static long vfio_ioctl_test_ioctl(struct file *file, unsigned int request, + unsigned long arg) +{ + struct vfio_ioctl_fixture *ctx = file->private_data; + struct vfio_fragment_fixture *f = &ctx->pins; + struct vfio_type1_test_cmd cmd; + void __user *user = (void __user *)arg; + struct page *page; + int ret = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + /* Request serialization belongs to this fixture, not production type1. */ + mutex_lock(&ctx->request_lock); + switch (request) { + case VFIO_CHECK_EXTENSION: + case VFIO_IOMMU_GET_INFO: + case VFIO_IOMMU_MAP_DMA: + case VFIO_IOMMU_UNMAP_DMA: + case VFIO_IOMMU_DIRTY_PAGES: + ret = vfio_iommu_type1_ioctl(f->iommu, request, arg); + goto out; + default: + break; + } + if (copy_from_user(&cmd, user, sizeof(cmd))) { + ret = -EFAULT; + goto out; + } + if (cmd.reserved) { + ret = -EINVAL; + goto out; + } + switch (request) { + case VFIO_TYPE1_TEST_DOMAIN_ADD: + case VFIO_TYPE1_TEST_DOMAIN_DEL: + ret = vfio_ioctl_test_domain(f, &cmd, request == VFIO_TYPE1_TEST_DOMAIN_ADD); + break; + case VFIO_TYPE1_TEST_RW: + ret = vfio_ioctl_test_rw(f, &cmd); + break; + case VFIO_TYPE1_TEST_PIN: + case VFIO_TYPE1_TEST_UNPIN: + if (ALIGN_DOWN(cmd.iova, PAGE_SIZE) != VFIO_TEST_IOVA || cmd.flags) { + ret = -EINVAL; + break; + } + if (request == VFIO_TYPE1_TEST_PIN) { + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + cmd.iova, 1, IOMMU_READ | IOMMU_WRITE, &page); + if (ret == 1) { + f->external_refs++; + ret = 0; + } + } else if (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, cmd.iova, 1); + f->external_refs--; + } else { + ret = -ENOENT; + } + break; + case VFIO_TYPE1_TEST_EMULATED: + if (cmd.flags > 1 || f->external_refs) { + ret = -EBUSY; + break; + } + mutex_lock(&f->iommu->lock); + if (cmd.flags && list_empty(&f->group.next)) { + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + if (!f->group.pinned_page_dirty_scope) + f->iommu->num_non_pinned_groups++; + } else if (!cmd.flags && !list_empty(&f->group.next)) { + list_del_init(&f->group.next); + if (!f->group.pinned_page_dirty_scope) + f->iommu->num_non_pinned_groups--; + } + mutex_unlock(&f->iommu->lock); + break; + case VFIO_TYPE1_TEST_STATE: + cmd.locked_bytes = 0; + cmd.owner_locked_bytes = mm_locked_vm_bytes(current->mm); + cmd.flags = 0; + cmd.owner_users = 0; + mutex_lock(&f->iommu->lock); + for (struct rb_node *node = rb_first(&f->iommu->dma_list); + node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + cmd.locked_bytes += dma->locked_vm * PAGE_SIZE; + cmd.owner_locked_bytes = mm_locked_vm_bytes(dma->mm); + cmd.owner_users = atomic_read(&dma->mm->mm_users); + if (dma->fragments) + cmd.flags |= VFIO_TYPE1_TEST_STATE_FRAGMENTS; + } + cmd.native_page_size = PAGE_SIZE; + cmd.minimum_page_size = 1UL << __ffs(f->iommu->pgsize_bitmap); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) + cmd.mapped_bytes[i] = f->domains[i].mapped; + mutex_unlock(&f->iommu->lock); + cmd.notifications = f->notifications; + cmd.external_refs = f->external_refs; + break; + default: + ret = -ENOTTY; + } + if (!ret && copy_to_user(user, &cmd, sizeof(cmd))) + ret = -EFAULT; +out: + mutex_unlock(&ctx->request_lock); + return ret; +} + +static const struct file_operations vfio_ioctl_test_fops = { + .owner = THIS_MODULE, + .open = vfio_ioctl_test_open, + .release = vfio_ioctl_test_release, + .unlocked_ioctl = vfio_ioctl_test_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static struct miscdevice vfio_ioctl_test_device = { + .minor = MISC_DYNAMIC_MINOR, + .name = "aurora-vfio-type1-test", + .fops = &vfio_ioctl_test_fops, + .mode = 0600, +}; + +static int __init vfio_ioctl_test_init(void) +{ + return misc_register(&vfio_ioctl_test_device); +} +late_initcall(vfio_ioctl_test_init); diff --git a/drivers/vfio/vfio_iommu_type1_test.h b/drivers/vfio/vfio_iommu_type1_test.h new file mode 100644 index 0000000000000..36c09030bcf0e --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_test.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* Private disposable-guest test protocol, not a supported VFIO ABI. */ +#ifndef VFIO_IOMMU_TYPE1_TEST_H +#define VFIO_IOMMU_TYPE1_TEST_H +#include +#include + +struct vfio_type1_test_cmd { + __u64 iova; + __u64 size; + __u64 buffer; + __u64 fail_after; + __u64 locked_bytes; + __u64 owner_locked_bytes; + __u64 mapped_bytes[2]; + __u64 native_page_size; + __u64 minimum_page_size; + __u32 domain; + __u32 flags; + __u32 owner_users; + __u32 notifications; + __u32 external_refs; + __u32 reserved; +}; + +#define VFIO_TYPE1_TEST_DOMAIN_ADD _IOWR('V', 1, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_DOMAIN_DEL _IOWR('V', 2, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_RW _IOWR('V', 3, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_PIN _IOWR('V', 4, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_UNPIN _IOWR('V', 5, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_STATE _IOWR('V', 6, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_EMULATED _IOWR('V', 7, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_STATE_FRAGMENTS 1U +#define VFIO_TYPE1_TEST_WRITE 1U +#define VFIO_TYPE1_TEST_WORKER 2U +#endif From 08ee694a5c9e5034188ced6ae5a46e67d9d331f0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:23 -0400 Subject: [PATCH 0580/1579] selftests: exercise the mixed userspace page-size ABI Cover exec policy, AArch32 fallback, mappings, sharing, reclaim, accounting and converted I/O interfaces. Keep extended PID1 fixtures separate from the ordinary page-contract executable. Signed-off-by: Diljit Singh (cherry picked from commit ad36aab5acc6e025db2a66192d259a1058d77d02) --- .../testing/kunit/configs/arm64_user4k.config | 33 + .../configs/arm64_user4k_data_kasan.config | 18 + .../configs/arm64_user4k_mm_kasan.config | 38 + tools/testing/selftests/arm64/Makefile | 2 +- .../testing/selftests/arm64/user4k/.gitignore | 3 + tools/testing/selftests/arm64/user4k/Makefile | 134 ++ tools/testing/selftests/arm64/user4k/advice.c | 199 +++ .../selftests/arm64/user4k/aio-ring-init.c | 178 +++ .../arm64/user4k/aslr-fixture-init.c | 175 +++ .../selftests/arm64/user4k/bpf-uptr-init.c | 583 ++++++++ .../arm64/user4k/coarse-shmem-init.c | 223 +++ .../arm64/user4k/compat-aslr-probe.c | 134 ++ .../arm64/user4k/compat-fixture-init.c | 149 ++ .../arm64/user4k/compat-mlock-init.c | 62 + .../arm64/user4k/compat-mlock-probe.c | 142 ++ .../user4k/compat-numa-balancing-probe.c | 223 +++ .../arm64/user4k/compat-numa-probe.c | 117 ++ .../arm64/user4k/compat-perf-probe.c | 107 ++ .../arm64/user4k/compat-policy-probe.c | 66 + .../selftests/arm64/user4k/compat-probe.c | 248 ++++ .../selftests/arm64/user4k/compat-script | 1 + .../selftests/arm64/user4k/compat-shm-probe.c | 111 ++ .../selftests/arm64/user4k/compat-uffd-init.c | 154 ++ .../arm64/user4k/compat-uffd-probe.c | 149 ++ .../arm64/user4k/compat-vfio-probe.c | 250 ++++ .../arm64/user4k/context-stress-init.c | 275 ++++ .../arm64/user4k/core-fixture-init.c | 447 ++++++ .../selftests/arm64/user4k/cpu-granule-init.c | 187 +++ .../arm64/user4k/cpu-granule16-init.c | 192 +++ .../selftests/arm64/user4k/dio-fixture-init.c | 266 ++++ .../arm64/user4k/exec-fixture-init.c | 167 +++ .../selftests/arm64/user4k/exec-policy-init.c | 214 +++ .../arm64/user4k/fex-contract-init.c | 52 + .../selftests/arm64/user4k/fex-native-abi.c | 20 + .../selftests/arm64/user4k/fex-x86-contract.c | 132 ++ .../selftests/arm64/user4k/gem-fixture-init.c | 325 ++++ .../arm64/user4k/granule-proc-init.c | 141 ++ .../arm64/user4k/granule-swap-init.c | 248 ++++ .../arm64/user4k/granule-threeway-init.c | 248 ++++ .../selftests/arm64/user4k/iommufd-init.c | 1314 +++++++++++++++++ .../arm64/user4k/lazyfree-fixture-init.c | 274 ++++ .../arm64/user4k/memcg-accounting-init.c | 153 ++ .../arm64/user4k/memcg-pressure-init.c | 417 ++++++ .../arm64/user4k/memory-interfaces.c | 165 +++ .../arm64/user4k/mglru-granule-init.c | 365 +++++ .../selftests/arm64/user4k/mixed-process.c | 209 +++ .../arm64/user4k/mlock-fixture-init.c | 249 ++++ .../arm64/user4k/numa-balancing-init.c | 307 ++++ .../selftests/arm64/user4k/numa-policy-init.c | 236 +++ .../selftests/arm64/user4k/page-contract.c | 232 +++ .../selftests/arm64/user4k/perf-ring-init.c | 323 ++++ .../selftests/arm64/user4k/proc-args.c | 164 ++ .../selftests/arm64/user4k/proc-memory.c | 340 +++++ .../selftests/arm64/user4k/proc-pages.c | 318 ++++ .../selftests/arm64/user4k/reclaim-advice.c | 366 +++++ .../selftests/arm64/user4k/runtime-app-init.c | 76 + .../selftests/arm64/user4k/runtime-app.py | 91 ++ .../arm64/user4k/shmem-collapse-init.c | 396 +++++ .../selftests/arm64/user4k/shmem-hole-init.c | 358 +++++ .../selftests/arm64/user4k/shmem-pin-init.c | 358 +++++ .../arm64/user4k/shmem-pressure-init.c | 225 +++ .../selftests/arm64/user4k/shmem-race-init.c | 308 ++++ .../selftests/arm64/user4k/shmem-thp-init.c | 271 ++++ .../arm64/user4k/shmem-uffd-swap-init.c | 160 ++ .../arm64/user4k/splice-fixture-init.c | 610 ++++++++ .../arm64/user4k/swap-fixture-init.c | 216 +++ .../selftests/arm64/user4k/swapfile-fixture.h | 50 + .../selftests/arm64/user4k/sysv-shm-init.c | 229 +++ .../arm64/user4k/thread-fixture-init.c | 424 ++++++ .../selftests/arm64/user4k/uffd-move-init.c | 451 ++++++ .../selftests/arm64/user4k/uffd-wp-init.c | 913 ++++++++++++ .../selftests/arm64/user4k/uprobes-init.c | 318 ++++ .../arm64/user4k/uring-fixture-init.c | 467 ++++++ .../selftests/arm64/user4k/user-events-init.c | 268 ++++ .../arm64/user4k/user16k-smoke-init.c | 229 +++ .../arm64/user4k/vfio-fixture-init.c | 542 +++++++ .../selftests/arm64/user4k/vhost-log-init.c | 392 +++++ .../selftests/arm64/user4k/wine-kernel32.def | 20 + .../arm64/user4k/wine-page-contract.c | 181 +++ .../selftests/arm64/user4k/x18-probe.S | 25 + tools/testing/vma/include/dup.h | 117 ++ tools/testing/vma/tests/merge.c | 2 +- tools/testing/vma/tests/mmap.c | 8 +- tools/testing/vma/tests/vma.c | 4 +- 84 files changed, 19546 insertions(+), 8 deletions(-) create mode 100644 tools/testing/kunit/configs/arm64_user4k.config create mode 100644 tools/testing/kunit/configs/arm64_user4k_data_kasan.config create mode 100644 tools/testing/kunit/configs/arm64_user4k_mm_kasan.config create mode 100644 tools/testing/selftests/arm64/user4k/.gitignore create mode 100644 tools/testing/selftests/arm64/user4k/Makefile create mode 100644 tools/testing/selftests/arm64/user4k/advice.c create mode 100644 tools/testing/selftests/arm64/user4k/aio-ring-init.c create mode 100644 tools/testing/selftests/arm64/user4k/aslr-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/bpf-uptr-init.c create mode 100644 tools/testing/selftests/arm64/user4k/coarse-shmem-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-aslr-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-mlock-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-mlock-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-numa-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-perf-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-policy-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-script create mode 100644 tools/testing/selftests/arm64/user4k/compat-shm-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-uffd-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-uffd-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-vfio-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/context-stress-init.c create mode 100644 tools/testing/selftests/arm64/user4k/core-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/cpu-granule-init.c create mode 100644 tools/testing/selftests/arm64/user4k/cpu-granule16-init.c create mode 100644 tools/testing/selftests/arm64/user4k/dio-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/exec-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/exec-policy-init.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-contract-init.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-native-abi.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-x86-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/gem-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-proc-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-swap-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-threeway-init.c create mode 100644 tools/testing/selftests/arm64/user4k/iommufd-init.c create mode 100644 tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memcg-accounting-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memcg-pressure-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memory-interfaces.c create mode 100644 tools/testing/selftests/arm64/user4k/mglru-granule-init.c create mode 100644 tools/testing/selftests/arm64/user4k/mixed-process.c create mode 100644 tools/testing/selftests/arm64/user4k/mlock-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/numa-balancing-init.c create mode 100644 tools/testing/selftests/arm64/user4k/numa-policy-init.c create mode 100644 tools/testing/selftests/arm64/user4k/page-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/perf-ring-init.c create mode 100644 tools/testing/selftests/arm64/user4k/proc-args.c create mode 100644 tools/testing/selftests/arm64/user4k/proc-memory.c create mode 100644 tools/testing/selftests/arm64/user4k/proc-pages.c create mode 100644 tools/testing/selftests/arm64/user4k/reclaim-advice.c create mode 100644 tools/testing/selftests/arm64/user4k/runtime-app-init.c create mode 100755 tools/testing/selftests/arm64/user4k/runtime-app.py create mode 100644 tools/testing/selftests/arm64/user4k/shmem-collapse-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-hole-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-pin-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-pressure-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-race-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-thp-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c create mode 100644 tools/testing/selftests/arm64/user4k/splice-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/swap-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/swapfile-fixture.h create mode 100644 tools/testing/selftests/arm64/user4k/sysv-shm-init.c create mode 100644 tools/testing/selftests/arm64/user4k/thread-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uffd-move-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uffd-wp-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uprobes-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uring-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/user-events-init.c create mode 100644 tools/testing/selftests/arm64/user4k/user16k-smoke-init.c create mode 100644 tools/testing/selftests/arm64/user4k/vfio-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/vhost-log-init.c create mode 100644 tools/testing/selftests/arm64/user4k/wine-kernel32.def create mode 100644 tools/testing/selftests/arm64/user4k/wine-page-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/x18-probe.S diff --git a/tools/testing/kunit/configs/arm64_user4k.config b/tools/testing/kunit/configs/arm64_user4k.config new file mode 100644 index 0000000000000..de6c8ec2fdff8 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k.config @@ -0,0 +1,33 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_KUNIT_TEST=y +CONFIG_ARM64_SW_TTBR0_PAN=y +# CONFIG_ARM64_CNP is not set +# CONFIG_ARM64_POE is not set +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_VM=y +CONFIG_DEBUG_VM_MAPLE_TREE=y +CONFIG_DEBUG_VM_RB=y +CONFIG_PROVE_LOCKING=y +CONFIG_PAGE_TABLE_CHECK=y +CONFIG_PAGE_TABLE_CHECK_ENFORCED=y +# Guest-only infrastructure for late swap/reclaim fixtures. +CONFIG_BLOCK=y +CONFIG_VIRTIO=y +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_BLK=y +CONFIG_DEVTMPFS=y +CONFIG_DEBUG_FS=y +CONFIG_KUNIT_DEBUGFS=y diff --git a/tools/testing/kunit/configs/arm64_user4k_data_kasan.config b/tools/testing/kunit/configs/arm64_user4k_data_kasan.config new file mode 100644 index 0000000000000..2e3e50bd67768 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k_data_kasan.config @@ -0,0 +1,18 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_DATA_KUNIT_TEST=y +# CONFIG_ARM64_USER4K_KUNIT_TEST is not set +CONFIG_KASAN=y +CONFIG_KASAN_GENERIC=y +CONFIG_KASAN_INLINE=y +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y diff --git a/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config b/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config new file mode 100644 index 0000000000000..11b9935770f53 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config @@ -0,0 +1,38 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_KUNIT_TEST=y +CONFIG_ARM64_SW_TTBR0_PAN=y +# CONFIG_ARM64_CNP is not set +# CONFIG_ARM64_POE is not set +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_VM=y +CONFIG_DEBUG_VM_MAPLE_TREE=y +CONFIG_DEBUG_VM_RB=y +CONFIG_PROVE_LOCKING=y +CONFIG_PAGE_TABLE_CHECK=y +CONFIG_PAGE_TABLE_CHECK_ENFORCED=y +# Guest-only infrastructure for late swap/reclaim fixtures. +CONFIG_BLOCK=y +CONFIG_VIRTIO=y +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_BLK=y +CONFIG_DEVTMPFS=y +CONFIG_DEBUG_FS=y +CONFIG_KUNIT_DEBUGFS=y + +# Software MM fixtures only; the hardware boot opt-in is rejected under KASAN. +CONFIG_KASAN=y +CONFIG_KASAN_GENERIC=y +CONFIG_KASAN_INLINE=y diff --git a/tools/testing/selftests/arm64/Makefile b/tools/testing/selftests/arm64/Makefile index e456f3b62fa13..d8948f5fb1ef8 100644 --- a/tools/testing/selftests/arm64/Makefile +++ b/tools/testing/selftests/arm64/Makefile @@ -4,7 +4,7 @@ ARCH ?= $(shell uname -m 2>/dev/null || echo not) ifneq (,$(filter $(ARCH),aarch64 arm64)) -ARM64_SUBTARGETS ?= tags signal pauth fp mte bti abi gcs +ARM64_SUBTARGETS ?= tags signal pauth fp mte bti abi gcs user4k else ARM64_SUBTARGETS := endif diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore new file mode 100644 index 0000000000000..2e6224b832f98 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -0,0 +1,3 @@ +page-contract + +swap-fixture-init diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile new file mode 100644 index 0000000000000..cc89cc93293fe --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: GPL-2.0 +TEST_GEN_PROGS := page-contract proc-args +TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init +TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init +TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init +CFLAGS += -Wall -Wextra -O2 -ffixed-x18 +TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe +TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe +TEST_GEN_FILES += compat-shm-probe compat-perf-probe compat-numa-probe compat-numa-balancing-probe +TEST_FILES += compat-script runtime-app.py +include ../../lib.mk + +$(OUTPUT)/page-contract: page-contract.c x18-probe.S + +$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init: LDLIBS += -pthread + +# Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. +AARCH32_CC ?= clang --target=arm-linux-gnueabihf +AARCH32_FLAGS := -march=armv8-a -marm -mfloat-abi=soft -ffreestanding \ + -fno-builtin -fno-stack-protector -O2 -nostdlib -static -fuse-ld=lld +$(OUTPUT)/compat-probe-4k: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=4096 $< -o $@ +$(OUTPUT)/compat-probe-64k: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ +$(OUTPUT)/compat-interp32: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536,--image-base=0x10000000 $< -o $@ +$(OUTPUT)/compat-probe-dynamic: compat-probe.c + $(AARCH32_CC) $(filter-out -static,$(AARCH32_FLAGS)) -no-pie \ + -Wl,-z,max-page-size=65536,--dynamic-linker=/interp32 $< -o $@ + +$(OUTPUT)/compat-aslr-probe: compat-aslr-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/compat-mlock-probe: compat-mlock-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# The disposable PID1 fixture also runs with the native 64K process ABI. +$(OUTPUT)/mglru-granule-init $(OUTPUT)/memcg-pressure-init $(OUTPUT)/memcg-accounting-init $(OUTPUT)/runtime-app-init $(OUTPUT)/fex-contract-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +# FEX fixture: the AArch64 emulator runs the freestanding x86-64 contract. +X86_64_CC ?= clang --target=x86_64-linux-gnu +$(OUTPUT)/fex-x86-contract: fex-x86-contract.c + $(X86_64_CC) -O2 -fno-builtin -fno-stack-protector -nostdlib -static \ + -fuse-ld=lld -Wl,-z,max-page-size=4096,-e,_start $< -o $@ + +$(OUTPUT)/libfex-abi.so: fex-native-abi.c + $(CC) $(CFLAGS) -shared -fPIC -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/exec-policy-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/exec-policy-init: LDLIBS += -pthread +$(OUTPUT)/compat-policy-probe: compat-policy-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# Optional Windows payload; requires Wine/FEX in a disposable guest to execute. +# Keep its toolchain out of the ordinary Linux selftest build. +WINDOWS_CC ?= clang --target=x86_64-pc-windows-msvc +WINDOWS_DLLTOOL ?= llvm-dlltool +WINDOWS_LD ?= lld-link +.PHONY: wine-contract +wine-contract: $(OUTPUT)/wine-page-contract.exe +$(OUTPUT)/wine-kernel32.lib: wine-kernel32.def + $(WINDOWS_DLLTOOL) -m i386:x86-64 -d $< -l $@ +$(OUTPUT)/wine-page-contract.obj: wine-page-contract.c + $(WINDOWS_CC) -O2 -Wall -Wextra -Werror -ffreestanding -fno-stack-protector -c $< -o $@ +$(OUTPUT)/wine-page-contract.exe: $(OUTPUT)/wine-page-contract.obj $(OUTPUT)/wine-kernel32.lib + $(WINDOWS_LD) /entry:mainCRTStartup /subsystem:console /nodefaultlib /out:$@ $^ +EXTRA_CLEAN += $(OUTPUT)/wine-page-contract.exe $(OUTPUT)/wine-page-contract.obj $(OUTPUT)/wine-kernel32.lib + +$(OUTPUT)/uffd-wp-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/uffd-wp-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-uffd-swap-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/compat-uffd-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-uffd-probe: compat-uffd-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/shmem-hole-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-hole-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-thp-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-thp-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-collapse-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/shmem-pin-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-pin-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-race-init $(OUTPUT)/shmem-pressure-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-race-init: LDLIBS += -pthread + +$(OUTPUT)/uffd-move-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/uffd-move-init: LDLIBS += -pthread + +$(OUTPUT)/user-events-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/uprobes-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/vhost-log-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/bpf-uptr-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/coarse-shmem-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/iommufd-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/iommufd-init: LDLIBS += -pthread +$(OUTPUT)/iommufd-init: CFLAGS += $(KHDR_INCLUDES) + +# Privileged recording-domain fixture; run only in a disposable test VM. +$(OUTPUT)/vfio-fixture-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-vfio-probe: compat-vfio-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/sysv-shm-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-shm-probe: compat-shm-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# Disposable guests exercise perf layout and software sampling across mm granules. +$(OUTPUT)/perf-ring-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-perf-probe: compat-perf-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/aio-ring-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +# Requires a disposable guest with two memory nodes and CONFIG_COMPAT. +$(OUTPUT)/numa-policy-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-numa-probe: compat-numa-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/numa-balancing-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/numa-balancing-init: LDLIBS += -pthread + +$(OUTPUT)/compat-numa-balancing-probe: compat-numa-balancing-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ diff --git a/tools/testing/selftests/arm64/user4k/advice.c b/tools/testing/selftests/arm64/user4k/advice.c new file mode 100644 index 0000000000000..4d4cb89350985 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/advice.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static size_t page; + +static bool wait_child(pid_t pid, int sig) +{ + int status = 0; + + if (pid <= 0 || waitpid(pid, &status, 0) != pid) + return false; + if (sig) + return WIFSIGNALED(status) && WTERMSIG(status) == sig; + return WIFEXITED(status) && !WEXITSTATUS(status); +} + +static void *anon(size_t length, int prot, int extra) +{ + void *p = mmap(NULL, length, prot, MAP_PRIVATE | MAP_ANONYMOUS | extra, + -1, 0); + + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap\n"); + return p; +} + +static void discard_and_guards(void) +{ + unsigned char *p = anon(3 * page, PROT_READ | PROT_WRITE, 0); + bool ok; + pid_t child; + + memset(p, 0x61, 3 * page); + ok = madvise(p + page, 1, MADV_DONTNEED) == 0; + ok &= p[0] == 0x61 && p[page] == 0 && p[2 * page] == 0x61; + ksft_test_result(ok, + "DONTNEED discards exactly one rounded user page\n"); + memset(p + page, 0x71, page); + child = fork(); + if (!child) { + ok = madvise(p + page, page, MADV_DONTNEED) == 0; + _exit(!(ok && p[page] == 0 && p[0] == 0x61)); + } + ksft_test_result( + wait_child(child, 0) && p[page] == 0x71, + "discard in a fork child leaves the parent quarter intact\n"); + ok = madvise(p + page, 1, MADV_GUARD_INSTALL) == 0; + ok &= madvise(p + page, page, MADV_DONTNEED) == 0; + child = fork(); + if (!child) { + volatile unsigned char value = p[page]; + + (void)value; + _exit(1); + } + ksft_test_result(ok && wait_child(child, SIGSEGV), + "a user-page guard survives discard and fork\n"); + ok = madvise(p + page, 1, MADV_GUARD_REMOVE) == 0; + ok &= p[page] == 0 && p[0] == 0x61 && p[2 * page] == 0x61; + ksft_test_result(ok, + "guard removal restores only the requested page\n"); + munmap(p, 3 * page); +} + +static void population(void) +{ + unsigned char *p = anon(5 * page, PROT_READ, 0), vec[5]; + bool ok; + unsigned int i; + + ok = mprotect(p + 2 * page, page, PROT_READ | PROT_WRITE) == 0; + ok &= madvise(p + 2 * page, 1, MADV_POPULATE_WRITE) == 0; + ok &= mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == (i == 2); + ksft_test_result( + ok, "POPULATE_WRITE faults only the selected user-page VMA\n"); + ok = madvise(p + page, 1, MADV_POPULATE_READ) == 0; + ok &= mincore(p + page, page, vec) == 0 && vec[0] == 1; + errno = 0; + ok &= madvise(p, page, MADV_POPULATE_WRITE) == -1 && errno == EINVAL; + ok &= mprotect(p + 3 * page, page, PROT_NONE) == 0; + errno = 0; + ok &= madvise(p + 3 * page, page, MADV_POPULATE_READ) == -1 && + errno == EINVAL; + ok &= munmap(p + 3 * page, page) == 0; + errno = 0; + ok &= madvise(p + 3 * page, page, MADV_POPULATE_READ) == -1 && + errno == ENOMEM; + ksft_test_result( + ok, "POPULATE_READ honors permissions and reports holes\n"); + munmap(p, 5 * page); + p = anon(5 * page, PROT_READ | PROT_WRITE, MAP_POPULATE); + ok = mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == 1; + ksft_test_result(ok, "MAP_POPULATE materializes every user page\n"); + munmap(p, 5 * page); +} + +static void fork_advice(void) +{ + unsigned char *p = anon(3 * page, PROT_READ | PROT_WRITE, 0); + bool ok; + pid_t child; + + memset(p, 0x58, 3 * page); + ok = madvise(p + page, 1, MADV_WIPEONFORK) == 0; + child = fork(); + if (!child) + _exit(!(p[page] == 0 && p[0] == 0x58 && p[2 * page] == 0x58)); + ok &= wait_child(child, 0) && p[page] == 0x58; + ksft_test_result( + ok, + "WIPEONFORK clears a single child page and preserves neighbors\n"); + ok = madvise(p + page, 1, MADV_KEEPONFORK) == 0; + ok &= madvise(p + page, 1, MADV_DONTFORK) == 0; + child = fork(); + if (!child) { + unsigned char vec; + + errno = 0; + _exit(!(mincore(p + page, page, &vec) == -1 && + errno == ENOMEM && p[0] == 0x58 && + p[2 * page] == 0x58)); + } + ok &= wait_child(child, 0); + ksft_test_result(ok, "DONTFORK omits only its selected user page\n"); + munmap(p, 3 * page); +} + +static void remove_file(void) +{ + int fd = memfd_create("advice-file", 0); + unsigned char *shared, *alias, *private; + bool ok; + size_t i; + + if (fd < 0 || ftruncate(fd, 5 * page)) + ksft_exit_fail_msg("memfd setup\n"); + shared = + mmap(NULL, 5 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + alias = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page); + private = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, + page); + if (shared == MAP_FAILED || alias == MAP_FAILED || + private == MAP_FAILED) + ksft_exit_fail_msg("file mmap\n"); + for (i = 0; i < 5 * page; i++) + shared[i] = 0x40 + i / page; + private[page] = 0x79; + ok = madvise(alias + page, 1, MADV_REMOVE) == 0; + for (i = 0; i < 5 * page; i++) + ok &= shared[i] == (i / page == 2 ? 0 : 0x40 + i / page); + ok &= private[page] == 0x79; + ksft_test_result( + ok, + "REMOVE uses the file byte offset and preserves private COW\n"); + ok = madvise(alias, 3 * page, MADV_WILLNEED) == 0; + ksft_test_result( + ok, "WILLNEED accepts a file range with the process granule\n"); + munmap(shared, 5 * page); + munmap(alias, 3 * page); + munmap(private, 3 * page); + close(fd); +} + +int main(int argc, char **argv) +{ + const struct rlimit no_core = { 0, 0 }; + + page = getauxval(AT_PAGESZ); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + ksft_exit_fail_msg("unexpected process page ABI: %zu\n", page); + ksft_print_msg("process page size: %zu\n", page); + if (setrlimit(RLIMIT_CORE, &no_core)) + return 1; + ksft_print_header(); + ksft_set_plan(11); + discard_and_guards(); + population(); + fork_advice(); + remove_file(); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/aio-ring-init.c b/tools/testing/selftests/arm64/user4k/aio-ring-init.c new file mode 100644 index 0000000000000..74f34d16cac29 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/aio-ring-init.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: legacy AIO ring relocation with per-process page sizes. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct aio_ring_header { + unsigned int id, nr, head, tail; + unsigned int magic, compat_features, incompat_features, header_length; +}; + +static void check(int good, const char *expression, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, + errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int wait_child(pid_t child) +{ + int status; + pid_t got; + + if (child < 0) + return 1; + do { + got = waitpid(child, &status, 0); + } while (got < 0 && errno == EINTR); + return got != child || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static void complete(aio_context_t context, struct iocb *request, + unsigned long bytes, int event_fd) +{ + struct io_event event; + struct timespec timeout = { .tv_sec = 5 }; + uint64_t notifications; + + CHECK(syscall(SYS_io_getevents, context, 1, 1, &event, &timeout) == 1); + CHECK(event.data == request->aio_data && + event.obj == (uintptr_t)request); + CHECK(event.res == (long long)bytes && event.res2 == 0); + CHECK(read(event_fd, ¬ifications, sizeof(notifications)) == + sizeof(notifications)); + CHECK(notifications == 1); +} + +static void exercise(unsigned long native, unsigned int requested) +{ + unsigned long page = getauxval(AT_PAGESZ), bytes = 2 * page + 37; + aio_context_t context = 0; + int file = memfd_create("aio-granule", 0); + int event_fd = eventfd(0, EFD_NONBLOCK); + unsigned char *source, *destination, *reserve, *target; + struct aio_ring_header *header; + unsigned long ring_size; + struct iocb request = { 0 }, *requests[] = { &request }; + unsigned char residency; + + CHECK(file >= 0 && event_fd >= 0); + CHECK(!ftruncate(file, 4 * page)); + source = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + destination = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(source != MAP_FAILED && destination != MAP_FAILED); + for (unsigned long i = 0; i < bytes; i++) + source[page - 19 + i] = i * 17 + i / page; + CHECK(!syscall(SYS_io_setup, requested, &context)); + header = (void *)(uintptr_t)context; + CHECK(header->magic == 0xa10a10a1 && + header->header_length == sizeof(*header)); + CHECK(header->nr >= requested); + ring_size = + header->header_length + header->nr * sizeof(struct io_event); + CHECK(ring_size && !(ring_size % native)); + reserve = mmap(NULL, ring_size + 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + target = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + target += native + (page < native ? page : 0); + target[-1] = 0x51; + target[ring_size] = 0x72; + CHECK(!munmap(target, ring_size)); + request.aio_data = 0x12345678; + request.aio_lio_opcode = IOCB_CMD_PWRITE; + request.aio_fildes = file; + request.aio_buf = (uintptr_t)(source + page - 19); + request.aio_nbytes = bytes; + request.aio_offset = 7; + request.aio_flags = IOCB_FLAG_RESFD; + request.aio_resfd = event_fd; + CHECK(syscall(SYS_io_submit, context, 1, requests) == 1); + /* Retain a queued completion while relocating the ring. */ + CHECK(mremap(header, ring_size, ring_size, + MREMAP_MAYMOVE | MREMAP_FIXED, target) == target); + context = (uintptr_t)target; + header = (void *)target; + CHECK(header->magic == 0xa10a10a1); + CHECK(target[-1] == 0x51 && target[ring_size] == 0x72); + complete(context, &request, bytes, event_fd); + request.aio_data = 0x87654321; + request.aio_lio_opcode = IOCB_CMD_PREAD; + request.aio_buf = (uintptr_t)(destination + page - 11); + CHECK(syscall(SYS_io_submit, context, 1, requests) == 1); + complete(context, &request, bytes, event_fd); + CHECK(!memcmp(source + page - 19, destination + page - 11, bytes)); + CHECK(!syscall(SYS_io_destroy, context)); + errno = 0; + CHECK(mincore(target, page, &residency) == -1 && errno == ENOMEM); + CHECK(target[-1] == 0x51 && target[ring_size] == 0x72); + CHECK(!munmap(reserve, ring_size + 4 * native)); + CHECK(!munmap(source, 4 * page) && !munmap(destination, 4 * page)); + CHECK(!close(file) && !close(event_fd)); + printf("ok - AIO page=%lu requested=%u ring=%lu relocation, I/O, eventfd and teardown\n", + page, requested, ring_size); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--case")) { + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + exercise(native, 16); + exercise(native, 4096); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + if (sizes[i] > native) + continue; + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], + 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", n, s, NULL); + _exit(2); + } + int bad = wait_child(child); + + printf("%s - AIO native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, sizes[i]); + failed |= bad; + } + printf("AIO GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c b/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c new file mode 100644 index 0000000000000..4ac03136f2ec0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c @@ -0,0 +1,175 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define SAMPLES 64 +struct sample { + unsigned long page_size, stack, mapping, brk; +}; + +static int get_sample(struct sample *s) +{ + char buf[4096], *pos, *end; + unsigned long low, high; + FILE *f; + int fd; + ssize_t size; + void *p; + + s->page_size = getauxval(AT_PAGESZ); + p = mmap(NULL, s->page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + s->mapping = (unsigned long)p; + f = fopen("/proc/self/maps", "r"); + if (!f) + return 1; + while (fgets(buf, sizeof(buf), f)) + if (strstr(buf, "[stack]") && + sscanf(buf, "%lx-%lx", &low, &high) == 2) + s->stack = high; + fclose(f); + fd = open("/proc/self/stat", O_RDONLY); + if (fd < 0) + return 1; + size = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (size <= 0) + return 1; + buf[size] = 0; + pos = strrchr(buf, ')'); + if (!pos) + return 1; + pos += 2; + /* Fields 1 and 2 precede the closing parenthesis; start_brk is 47. */ + for (unsigned int field = 3; field < 47; field++) { + pos = strchr(pos, ' '); + if (!pos) + return 1; + pos++; + } + errno = 0; + s->brk = strtoul(pos, &end, 10); + return errno || end == pos || !s->brk || !s->stack; +} + +static int collect(unsigned int mode, bool disabled, struct sample *s) +{ + int pipefd[2], status; + pid_t pid; + ssize_t size; + + if (pipe(pipefd)) + return 1; + pid = fork(); + if (!pid) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + if (prctl(SET_EXEC_PAGE_SIZE, mode == 1 ? 4096UL : 0UL, 0UL, + 0UL, 0UL)) + _exit(2); + if (disabled && personality(ADDR_NO_RANDOMIZE) < 0) + _exit(3); + execl(mode == 2 ? "/compat-aslr" : "/init", "/init", "--sample", + fd, NULL); + _exit(4); + } + close(pipefd[1]); + size = read(pipefd[0], s, sizeof(*s)); + close(pipefd[0]); + return pid < 0 || waitpid(pid, &status, 0) != pid || + !WIFEXITED(status) || WEXITSTATUS(status) || size != sizeof(*s); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + if (argc == 3 && !strcmp(argv[1], "--sample")) { + struct sample s = { 0 }; + if (getppid() != 1 || get_sample(&s)) + return 1; + return write(atoi(argv[2]), &s, sizeof(s)) != sizeof(s); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + if (mount("proc", "/proc", "proc", 0, NULL)) + failed++; + int control = open("/proc/sys/kernel/randomize_va_space", O_WRONLY); + if (control < 0 || write(control, "2", 1) != 1) + failed++; + if (control >= 0) + close(control); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + if (prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL)) + failed++; + for (unsigned int mode = 0; mode < 3; mode++) { + if (mode == 1 && !has4k) + continue; + unsigned long ps = mode && has4k ? 4096 : native; + struct sample first = { 0 }; + bool varied[3] = { 0 }, subnative[3] = { 0 }; + int ok = 1; + + for (unsigned int i = 0; i < SAMPLES; i++) { + struct sample s = { 0 }; + if (collect(mode, false, &s)) { + ok = 0; + break; + } + unsigned long address[] = { s.stack, s.mapping, s.brk }; + unsigned long original[] = { first.stack, first.mapping, + first.brk }; + ok &= s.page_size == ps; + for (unsigned int n = 0; n < 3; n++) { + ok &= !(address[n] & (ps - 1)); + varied[n] |= i && address[n] != original[n]; + subnative[n] |= i && + ((address[n] ^ original[n]) & + (native - 1)); + } + if (!i) + first = s; + } + for (unsigned int n = 0; n < 3; n++) + ok &= varied[n] && (ps == native || subnative[n]); + printf("%s - %s %luK ASLR: 64 execs, stack/mmap/brk varied=%d%d%d low-bits-varied=%d%d%d\n", + ok ? "ok" : "not ok", mode == 2 ? "AArch32" : "AArch64", + ps / 1024, varied[0], varied[1], varied[2], subnative[0], + subnative[1], subnative[2]); + failed += !ok; + struct sample a = { 0 }, b = { 0 }; + ok = !collect(mode, true, &a) && !collect(mode, true, &b) && + !memcmp(&a, &b, sizeof(a)) && a.page_size == ps; + printf("%s - %s %luK ADDR_NO_RANDOMIZE retains stable layout\n", + ok ? "ok" : "not ok", mode == 2 ? "AArch32" : "AArch64", + ps / 1024); + failed += !ok; + } + printf("GRANULE ASLR %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c b/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c new file mode 100644 index 0000000000000..33793cebc2b8c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c @@ -0,0 +1,583 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +struct user_data { + uint64_t a, b, result, marker; +}; +struct value { + struct user_data *p, *q; + uint64_t cookie; + uint8_t trailing; +} __attribute__((packed)); +static char verifier_log[65536]; +static int bpf(enum bpf_cmd cmd, union bpf_attr *a) +{ + size_t size; +#define END(field) (offsetof(union bpf_attr, field) + sizeof(a->field)) + switch (cmd) { + case BPF_BTF_LOAD: + size = END(btf_log_level); + break; + case BPF_MAP_CREATE: + size = END(btf_value_type_id); + break; + case BPF_PROG_LOAD: + size = END(log_buf); + break; + case BPF_RAW_TRACEPOINT_OPEN: + size = END(raw_tracepoint.prog_fd); + break; + case BPF_MAP_UPDATE_ELEM: + size = END(flags); + break; + case BPF_MAP_LOOKUP_ELEM: + size = END(flags); + break; + case BPF_MAP_DELETE_ELEM: + size = END(key); + break; + default: + errno = EINVAL; + return -1; + } +#undef END + return syscall(__NR_bpf, cmd, a, size); +} +struct btf_blob { + unsigned char types[1024]; + char strings[512]; + unsigned nt, ns; +}; +static uint32_t str(struct btf_blob *b, const char *s) +{ + unsigned at = b->ns; + size_t n = strlen(s) + 1; + memcpy(b->strings + b->ns, s, n); + b->ns += n; + return at; +} +static void push(struct btf_blob *b, const void *p, size_t n) +{ + memcpy(b->types + b->nt, p, n); + b->nt += n; +} +static void type(struct btf_blob *b, const char *name, unsigned kind, + unsigned n, unsigned size_type) +{ + struct btf_type t = { .name_off = name ? str(b, name) : 0, + .info = kind << 24 | n, + .size = size_type }; + push(b, &t, sizeof(t)); +} +static void member(struct btf_blob *b, const char *name, unsigned id, + unsigned offset) +{ + struct btf_member m = { .name_off = str(b, name), + .type = id, + .offset = offset * 8 }; + push(b, &m, sizeof(m)); +} +static int create_map(void) +{ + struct btf_blob b = { .ns = 1 }; + uint32_t encoding; + type(&b, "int", BTF_KIND_INT, 0, 4); + encoding = BTF_INT_SIGNED << 24 | 32; + push(&b, &encoding, 4); + type(&b, "u64", BTF_KIND_INT, 0, 8); + encoding = 64; + push(&b, &encoding, 4); + type(&b, "user_data", BTF_KIND_STRUCT, 4, sizeof(struct user_data)); + member(&b, "a", 2, 0); + member(&b, "b", 2, 8); + member(&b, "result", 2, 16); + member(&b, "marker", 2, 24); + type(&b, "uptr", BTF_KIND_TYPE_TAG, 0, 3); + type(&b, NULL, BTF_KIND_PTR, 0, 4); + type(&b, "value_type", BTF_KIND_STRUCT, 4, sizeof(struct value)); + member(&b, "p", 5, 0); + member(&b, "q", 5, 8); + member(&b, "cookie", 2, 16); + member(&b, "trailing", 7, 24); + type(&b, "u8", BTF_KIND_INT, 0, 1); + encoding = 8; + push(&b, &encoding, 4); + unsigned char data[2048]; + struct btf_header h = { .magic = BTF_MAGIC, + .version = BTF_VERSION, + .hdr_len = sizeof(h), + .type_len = b.nt, + .str_off = b.nt, + .str_len = b.ns }; + memcpy(data, &h, sizeof(h)); + memcpy(data + sizeof(h), b.types, b.nt); + memcpy(data + sizeof(h) + b.nt, b.strings, b.ns); + union bpf_attr a = { .btf = (uintptr_t)data, + .btf_size = sizeof(h) + b.nt + b.ns, + .btf_log_buf = (uintptr_t)verifier_log, + .btf_log_size = sizeof(verifier_log), + .btf_log_level = 1 }; + int btf = bpf(BPF_BTF_LOAD, &a); + if (btf < 0) { + printf("BTF load errno=%d: %s\n", errno, verifier_log); + return -1; + } + a = (union bpf_attr){ .map_type = BPF_MAP_TYPE_TASK_STORAGE, + .key_size = sizeof(int), + .value_size = sizeof(struct value), + .map_flags = BPF_F_NO_PREALLOC, + .btf_fd = btf, + .btf_key_type_id = 1, + .btf_value_type_id = 6 }; + int fd = bpf(BPF_MAP_CREATE, &a); + close(btf); + return fd; +} +#define I(c, d, s, o, i) \ + ((struct bpf_insn){ .code = (c), \ + .dst_reg = (d), \ + .src_reg = (s), \ + .off = (o), \ + .imm = (i) }) +static int create_prog(int map, int *link) +{ + struct bpf_insn insns[40]; + unsigned n = 0, jumps[4]; +#define EMIT(c, d, s, o, i) (insns[n++] = I(c, d, s, o, i)) + EMIT(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_current_task_btf); + EMIT(BPF_ALU64 | BPF_MOV | BPF_X, 2, 0, 0, 0); + EMIT(BPF_LD | BPF_DW | BPF_IMM, 1, BPF_PSEUDO_MAP_FD, 0, map); + EMIT(0, 0, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 3, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 4, 0, 0, 0); + EMIT(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_task_storage_get); + jumps[0] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 0, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_X, 6, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 7, 6, 16, 0); + jumps[1] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 7, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 1, 6, 0, 0); + jumps[2] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 1, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 2, 1, 0, 0); + EMIT(BPF_ALU64 | BPF_XOR | BPF_X, 2, 7, 0, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 2, 16, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 7, 24, 0); + insns[jumps[2]].off = n - jumps[2] - 1; + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 1, 6, 8, 0); + jumps[3] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 1, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 2, 1, 8, 0); + EMIT(BPF_ALU64 | BPF_ADD | BPF_X, 2, 7, 0, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 2, 16, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 7, 24, 0); + for (unsigned i = 0; i < 4; i++) + if (i != 2) + insns[jumps[i]].off = n - jumps[i] - 1; + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 0, 0, 0, 0); + EMIT(BPF_JMP | BPF_EXIT, 0, 0, 0, 0); +#undef EMIT + union bpf_attr a = { .prog_type = BPF_PROG_TYPE_RAW_TRACEPOINT, + .insn_cnt = n, + .insns = (uintptr_t)insns, + .license = (uintptr_t)"GPL", + .log_buf = (uintptr_t)verifier_log, + .log_size = sizeof(verifier_log), + .log_level = 1 }; + verifier_log[0] = 0; + int fd = bpf(BPF_PROG_LOAD, &a); + if (fd < 0) { + printf("prog load errno=%d: %s\n", errno, verifier_log); + return -1; + } + a = (union bpf_attr){}; + a.raw_tracepoint.name = (uintptr_t)"sys_enter"; + a.raw_tracepoint.prog_fd = fd; + *link = bpf(BPF_RAW_TRACEPOINT_OPEN, &a); + if (*link < 0) { + close(fd); + return -1; + } + return fd; +} +static int update(int map, int key, struct value *v, unsigned flags) +{ + union bpf_attr a = { .map_fd = map, + .key = (uintptr_t)&key, + .value = (uintptr_t)v, + .flags = flags }; + return bpf(BPF_MAP_UPDATE_ELEM, &a); +} +static int delete(int map, int key) +{ + union bpf_attr a = { .map_fd = map, .key = (uintptr_t)&key }; + return bpf(BPF_MAP_DELETE_ELEM, &a); +} +struct pins { + unsigned long long acquired, released; +}; +static int pin_stats(struct pins *p) +{ + int fd = open("/proc/sys/vm/stat_refresh", O_RDONLY); + char c; + if (fd < 0) + return -1; + int ret = read(fd, &c, 1); + close(fd); + if (ret < 0) + return -1; + FILE *f = fopen("/proc/vmstat", "r"); + if (!f) + return -1; + char name[128]; + unsigned long long value; + unsigned found = 0; + while (fscanf(f, "%127s %llu", name, &value) == 2) { + if (!strcmp(name, "nr_foll_pin_acquired")) { + p->acquired = value; + found |= 1; + } + if (!strcmp(name, "nr_foll_pin_released")) { + p->released = value; + found |= 2; + } + } + fclose(f); + return found == 3 ? 0 : -1; +} +static int wait_pins(struct pins base, unsigned active) +{ + struct pins p = {}; + for (unsigned i = 0; i < 500; i++) { + if (pin_stats(&p)) + return 0; + if (p.acquired - base.acquired >= active && + p.acquired - base.acquired - (p.released - base.released) == + active) + return 1; + usleep(10000); + } + printf("pin delta acquired=%llu released=%llu expected-active=%u\n", + p.acquired - base.acquired, p.released - base.released, active); + return 0; +} +static int filled(unsigned char *p, size_t len, unsigned char value) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value) + return 0; + return 1; +} +static int values(struct user_data *p, struct user_data *q, uint64_t cookie) +{ + return p->result == (p->a ^ cookie) && q->result == q->b + cookie && + p->marker == cookie && q->marker == cookie; +} +static int exercise_case(size_t ps, unsigned shared) +{ + struct pins base; + CHECK(!pin_stats(&base)); + int backing = -1; + if (shared) { + backing = memfd_create("bpf-uptr", 0); + CHECK(backing >= 0 && !ftruncate(backing, 4 * ps)); + } + unsigned char *src = mmap( + (void *)0x20000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), + backing, 0); + unsigned char *dst = + mmap((void *)0x30000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, 4 * ps); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps; + struct user_data *p = (void *)(leaf + ps - 64), *q = p + 1; + *p = (struct user_data){ .a = 0x12345678, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x87654321 }; + int map = create_map(); + CHECK(map >= 0); + int link = -1, prog = create_prog(map, &link); + CHECK(prog >= 0 && link >= 0); + int key = syscall(SYS_pidfd_open, getpid(), 0); + CHECK(key >= 0); + struct value probe = {}; + int probe_ret = update(map, key, &probe, BPF_NOEXIST); + CHECK(!probe_ret); + CHECK(!delete(map, key)); + struct value v = { + .p = p, .q = q, .cookie = 0xabc001, .trailing = 0x7d + }; + CHECK(!update(map, key, &v, BPF_NOEXIST)); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + for (unsigned i = 0; i < 64; i++) { + v.cookie++; + CHECK(!update(map, key, &v, BPF_EXIST)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + } + CHECK(wait_pins(base, 2)); + struct { + struct value v; + unsigned char tail[16]; + } lookup; + memset(&lookup, 0xa5, sizeof(lookup)); + union bpf_attr a = { .map_fd = map, + .key = (uintptr_t)&key, + .value = (uintptr_t)&lookup }; + CHECK(!bpf(BPF_MAP_LOOKUP_ELEM, &a)); + CHECK(!lookup.v.p && !lookup.v.q && lookup.v.cookie == v.cookie && + lookup.v.trailing == v.trailing && + filled(lookup.tail, sizeof(lookup.tail), 0xa5)); + errno = 0; + CHECK(update(map, key, &v, BPF_NOEXIST) == -1 && errno == EEXIST); + CHECK(wait_pins(base, 2)); + struct value half = v; + half.p = NULL; + CHECK(!update(map, key, &half, BPF_EXIST)); + CHECK(wait_pins(base, 1)); + p->result = p->marker = 0; + syscall(SYS_getpid); + CHECK(!p->result && !p->marker && q->result == q->b + v.cookie); + CHECK(!update(map, key, &v, BPF_EXIST)); + CHECK(wait_pins(base, 2)); + struct value bad = v; + bad.q = (void *)5; + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + bad.q = (void *)(leaf + ps - 16); + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EOPNOTSUPP); + CHECK(wait_pins(base, 2)); + bad.q = (void *)(UINTPTR_MAX - 15); + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(wait_pins(base, 2)); + CHECK(!mprotect(leaf, ps, PROT_READ)); + errno = 0; + CHECK(update(map, key, &v, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(!mprotect(leaf, ps, PROT_READ | PROT_WRITE)); + CHECK(wait_pins(base, 2)); + struct value empty = {}; + CHECK(!update(map, key, &empty, BPF_EXIST)); + CHECK(wait_pins(base, 0)); + p->result = q->result = p->marker = q->marker = 0; + syscall(SYS_getpid); + CHECK(!p->result && !q->result); + CHECK(!update(map, key, &v, BPF_EXIST)); + CHECK(wait_pins(base, 2)); + /* The child owns storage pointing into the parent's pinned memory. */ + int barrier[2]; + CHECK(!pipe(barrier)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + close(barrier[1]); + char token; + if (read(barrier[0], &token, 1) != 1) + _exit(3); + if (!shared) { + p->a = 1; + q->b = 2; + } + syscall(SYS_getpid); + _exit(0); + } + close(barrier[0]); + int childkey = syscall(SYS_pidfd_open, child, 0); + CHECK(childkey >= 0); + CHECK(!update(map, childkey, &v, BPF_NOEXIST)); + CHECK(!delete(map, key)); + CHECK(wait_pins(base, 2)); + p->result = q->result = p->marker = q->marker = 0; + CHECK(write(barrier[1], "x", 1) == 1); + close(barrier[1]); + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + close(childkey); + CHECK(values(p, q, v.cookie) && p->a == 0x12345678 && + q->b == 0x87654321); + CHECK(wait_pins(base, 0)); + /* The supplying mm exits while a different task retains its UPTRs. */ + int foreign_pipe[2]; + CHECK(!pipe(foreign_pipe)); + pid_t supplier = fork(); + CHECK(supplier >= 0); + if (!supplier) { + close(foreign_pipe[0]); + *p = (struct user_data){ .a = 0x1111, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x2222 }; + struct value foreign = { .p = p, .q = q, .cookie = 0x765432 }; + if (update(map, key, &foreign, BPF_NOEXIST)) + _exit(4); + struct iovec data = { .iov_base = p, .iov_len = 64 }; + if (vmsplice(foreign_pipe[1], &data, 1, 0) != 64) + _exit(5); + _exit(0); + } + close(foreign_pipe[1]); + CHECK(waitpid(supplier, &status, 0) == supplier && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + struct user_data retired[2]; + size_t done = 0; + while (done < sizeof(retired)) { + ssize_t got = read(foreign_pipe[0], (char *)retired + done, + sizeof(retired) - done); + CHECK(got > 0); + done += got; + } + close(foreign_pipe[0]); + CHECK(retired[0].a == 0x1111 && retired[1].b == 0x2222 && + values(retired, retired + 1, 0x765432)); + CHECK(!delete(map, key) && wait_pins(base, 0)); + *p = (struct user_data){ .a = 0x12345678, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x87654321 }; + CHECK(!update(map, key, &v, BPF_NOEXIST)); + CHECK(wait_pins(base, 2)); + CHECK(filled(leaf, ps - 64, 0x64)); + int pipefd[2]; + CHECK(!pipe(pipefd)); + struct iovec iov = { .iov_base = leaf, .iov_len = ps }; + CHECK(vmsplice(pipefd[1], &iov, 1, 0) == (ssize_t)ps); + CHECK(!munmap(leaf, ps)); + CHECK(mmap(leaf, ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, + 0) == leaf); + memset(leaf, 0xe7, ps); + unsigned char *old = malloc(ps); + CHECK(old); + size_t n = 0; + while (n < ps) { + ssize_t got = read(pipefd[0], old + n, ps - n); + CHECK(got > 0); + n += got; + } + close(pipefd[0]); + close(pipefd[1]); + CHECK(filled(old, ps - 64, 0x64) && + values((void *)(old + ps - 64), (void *)(old + ps - 32), + v.cookie)); + CHECK(filled(leaf, ps, 0xe7)); + free(old); + /* Closing the map alone is insufficient: the program retains its map ref. */ + close(map); + close(link); + close(prog); + close(key); + CHECK(wait_pins(base, 0)); + CHECK(filled(dst, ps, 0x91) && filled(dst + 2 * ps, 2 * ps, 0x91)); + for (unsigned i = 0; i < 3; i++) + CHECK(filled(src + i * ps, ps, 0x61 + i)); + CHECK(!munmap(src, 3 * ps) && !munmap(dst, 4 * ps)); + if (backing >= 0) + close(backing); + printf("ok - %zuK BPF UPTR %s: relocated writes, updates, rollback, storage/supplier exit, unmap and pin balance\n", + ps / 1024, shared ? "shared" : "private"); + return 0; +} +static int exercise(size_t ps) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(90); + for (unsigned shared = 0; shared < 2; shared++) + CHECK(!exercise_case(ps, shared)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char size[24]; + snprintf(size, sizeof(size), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", size, NULL); + _exit(5); + } + int ok = 0; + for (unsigned ticks = 0; c > 0 && ticks < 1000; ticks++) { + pid_t got = waitpid(c, &status, WNOHANG); + if (got == c) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 999) { + kill(c, SIGKILL); + waitpid(c, &status, 0); + break; + } + usleep(100000); + } + printf("%s - ABI %zuK BPF UPTR status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("BPF UPTR %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c b/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c new file mode 100644 index 0000000000000..2a00ca546751e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1 runner; fault injection is selected by the KUnit worker. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "swapfile-fixture.h" + +static int setting(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY | O_CLOEXEC); + ssize_t n; + + if (fd < 0) + return -1; + n = write(fd, value, strlen(value)); + close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} + +static int fault_setting(const char *kind, const char *name, const char *value) +{ + char path[192]; + + snprintf(path, sizeof(path), "/sys/kernel/debug/%s/%s", kind, name); + return setting(path, value); +} + +static int injection_off(void) +{ + int ret = fault_setting("failslab", "probability", "0"); + + ret |= fault_setting("fail_page_alloc", "probability", "0"); + ret |= fault_setting("failslab", "ignore-gfp-wait", "Y"); + ret |= fault_setting("fail_page_alloc", "ignore-gfp-wait", "Y"); + ret |= fault_setting("fail_page_alloc", "ignore-gfp-highmem", "Y"); + return ret; +} + +static int injection_on(bool page) +{ + const char *kind = page ? "fail_page_alloc" : "failslab"; + + if (injection_off() || fault_setting(kind, "task-filter", "Y") || + fault_setting(kind, "verbose", "0") || + fault_setting(kind, "ignore-gfp-wait", "N") || + fault_setting(kind, "times", "-1") || + fault_setting(kind, "interval", "1") || + fault_setting(kind, "space", "0")) + return -1; + if (!page) + return 0; /* Only current->fail_nth selects the slab allocation. */ + return fault_setting(kind, "ignore-gfp-highmem", "N") || + fault_setting(kind, "min-order", "2") || + fault_setting(kind, "probability", "100"); +} + +static int move_worker(const char *text, bool enter, bool swap_limit) +{ + char *end, path[192], pid[32], comm[64]; + long nr = strtol(text, &end, 10); + const char *group = swap_limit ? "coarse-swap-limit" : "coarse-zero"; + int fd; + ssize_t n; + + if (*end || nr <= 1 || nr == getpid()) + return -1; + snprintf(path, sizeof(path), "/proc/%ld/comm", nr); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = read(fd, comm, sizeof(comm) - 1); + close(fd); + if (n < 1) + return -1; + comm[n] = 0; + if (strncmp(comm, "kunit_", 6)) + return -1; + if (enter) { + if (mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) && + errno != EBUSY) + return -1; + if (setting("/sys/fs/cgroup/cgroup.subtree_control", "+memory")) + return -1; + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s", group); + if (mkdir(path, 0755) && errno != EEXIST) + return -1; + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s/memory.max", + group); + if (setting(path, swap_limit ? "max" : "0")) + return -1; + if (swap_limit && + setting("/sys/fs/cgroup/coarse-swap-limit/memory.swap.max", + "0")) + return -1; + } + snprintf(pid, sizeof(pid), "%ld", nr); + if (enter) + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s/cgroup.procs", + group); + else + snprintf(path, sizeof(path), "/sys/fs/cgroup/cgroup.procs"); + return setting(path, pid); +} + +static int helper(const char *op) +{ + struct stat init, self; + + if (getpid() == 1 || stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || init.st_dev != self.st_dev || + init.st_ino != self.st_ino) + return 1; + if (!strcmp(op, "--coarse-fixture-check")) + return getauxval(AT_PAGESZ) != 16384; + if (!strcmp(op, "--swapoff-helper")) + return swapoff(swap_path()) != 0; + if (!strcmp(op, "--swapon-helper")) + return swapon(swap_path(), 0) != 0; + if (!strncmp(op, "--coarse-swap-enter=", 20)) + return move_worker(op + 20, true, true) != 0; + if (!strncmp(op, "--coarse-swap-leave=", 20)) + return move_worker(op + 20, false, true) != 0; + if (!strcmp(op, "--coarse-swap-unlimit")) + return setting("/sys/fs/cgroup/coarse-swap-limit/memory.swap.max", + "max") != 0; + if (!strcmp(op, "--coarse-injection-off")) + return injection_off() != 0; + if (!strcmp(op, "--coarse-slab-on")) + return injection_on(false) != 0; + if (!strcmp(op, "--coarse-page-on")) + return injection_on(true) != 0; + if (!strncmp(op, "--coarse-memcg-enter=", 21)) + return move_worker(op + 21, true, false) != 0; + if (!strncmp(op, "--coarse-memcg-leave=", 21)) + return move_worker(op + 21, false, false) != 0; + return 1; +} + +static int run_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-coarse-shmem/"; + char path[192], *log = calloc(1, 1024 * 1024); + size_t used = 0, capacity = 1024 * 1024; + ssize_t n = 0; + int fd, ret = -1; + + if (!log) + return -1; + snprintf(path, sizeof(path), "%srun", base); + if (setting(path, "1")) + goto out; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + goto out; + while (used < capacity - 1 && + (n = read(fd, log + used, capacity - used - 1)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (!n && used < capacity - 1 && + strstr(log, "\nok 1 arm64-user4k-coarse-shmem\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; +out: + free(log); + return ret; +} + +int main(int argc, char **argv) +{ + struct stat st; + bool active = false; + int ret = 1; + unsigned int i; + + if (getpid() != 1 && argc == 2) + return helper(argv[1]); + if (getpid() != 1 || + (argc != 1 && (argc != 2 || strcmp(argv[1], "--swapfile")))) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + if (getauxval(AT_PAGESZ) != 16384) + goto out; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) + goto out; + for (i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode)) + goto out; + if (argc == 2 && prepare_swapfile()) + goto out; + printf("COARSE SWAP BACKING=%s\n", swap_path()); + if (swapon(swap_path(), 0)) + goto out; + active = true; + ret = run_suite() ? 1 : 0; + if (injection_off()) + ret = 1; +out: + if (active && swapoff(swap_path())) + ret = 1; + if (active && argc == 2 && umount("/swapdir")) + ret = 1; + printf("COARSE SHMEM SWAP PASS=%d errno=%d\n", !ret, errno); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c b/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c new file mode 100644 index 0000000000000..3a88db7511215 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding ELF32 address sampler for the disposable ASLR fixture. */ +typedef unsigned int u32; +typedef unsigned long long u64; +struct sample { + u64 page_size, stack, mapping, brk; +}; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static int error(long value) +{ + return (u32)value >= 0xfffff001U; +} +static unsigned number(const char *p) +{ + unsigned n = 0; + while (*p >= '0' && *p <= '9') + n = n * 10 + *p++ - '0'; + return n; +} +static char *find(char *p, char c) +{ + while (*p && *p != c) + p++; + return *p ? p : (char *)0; +} +static char *find_stack(char *p) +{ + const char name[] = "[stack]"; + while (*p) { + unsigned i = 0; + while (name[i] && p[i] == name[i]) + i++; + if (!name[i]) + return p; + p++; + } + return (char *)0; +} +static unsigned hex(char *p) +{ + unsigned n = 0, v; + for (;;) { + if (*p >= '0' && *p <= '9') + v = *p - '0'; + else if (*p >= 'a' && *p <= 'f') + v = *p - 'a' + 10; + else + return n; + n = (n << 4) | v; + p++; + } +} +static int load(const char *path, char *buf, unsigned size) +{ + long fd = call(5, (long)path, 0, 0, 0, 0, 0), n; + if (error(fd)) + return 1; + n = call(3, fd, (long)buf, size - 1, 0, 0, 0); + call(6, fd, 0, 0, 0, 0, 0); + if (n <= 0) + return 1; + buf[n] = 0; + return 0; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + unsigned argc = *sp++; + char **argv = (void *)sp, buf[4096], *p, *q; + struct sample sample = { 0 }; + long mapping; + int bad = 1; + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + sample.page_size = sp[1]; + sp += 2; + } + if (argc != 3 || call(64, 0, 0, 0, 0, 0, 0) != 1) + goto done; + mapping = call(192, 0, sample.page_size, 3, 0x22, -1, 0); + if (error(mapping)) + goto done; + sample.mapping = (u32)mapping; + if (load("/proc/self/maps", buf, sizeof(buf))) + goto done; + p = find_stack(buf); + if (!p) + goto done; + while (p > buf && p[-1] != '\n') + p--; + p = find(p, '-'); + if (!p) + goto done; + sample.stack = hex(p + 1); + if (load("/proc/self/stat", buf, sizeof(buf))) + goto done; + p = find(buf, ')'); + if (!p) + goto done; + while ((q = find(p + 1, ')'))) + p = q; + p += 2; + for (unsigned field = 3; field < 47; field++) { + p = find(p, ' '); + if (!p) + goto done; + p++; + } + sample.brk = number(p); + if (!sample.page_size || !sample.stack || !sample.brk) + goto done; + bad = call(4, number(argv[2]), (long)&sample, sizeof(sample), 0, 0, + 0) != sizeof(sample); +done: + call(1, bad, 0, 0, 0, 0, 0); + for (;;) { + } +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-fixture-init.c b/tools/testing/selftests/arm64/user4k/compat-fixture-init.c new file mode 100644 index 0000000000000..f576ed93623ed --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-fixture-init.c @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static unsigned long native_size; +static void result(int ok, const char *s) +{ + printf("%s - launcher %s\n", ok ? "ok" : "not ok", s); + if (!ok) + failed++; +} +static int launch(const char *path, char **args, int request, int expect_errno) +{ + pid_t child = fork(); + int status = 0; + if (!child) { + if (request && prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(95); + unsigned char *canary = + mmap(NULL, native_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (canary == MAP_FAILED) + _exit(94); + memset(canary, 0x69, native_size); + int before = prctl(0x41555002, 0UL, 0UL, 0UL, 0UL); + for (int i = 0; i < (expect_errno ? 64 : 1); i++) { + execv(path, args); + if (!expect_errno || errno != expect_errno) + _exit(96); + if (getauxval(AT_PAGESZ) != native_size || + prctl(0x41555002, 0UL, 0UL, 0UL, 0UL) != before) + _exit(93); + for (int j = 0; j < native_size; j++) + if (canary[j] != 0x69) + _exit(92); + } + _exit(0); + } + if (child < 0 || waitpid(child, &status, 0) != child) + return 0; + if (status) + printf("# child %s status=%#x\n", path, status); + return WIFEXITED(status) && !WEXITSTATUS(status); +} +int main(int argc, char **argv) +{ + if (argc > 1 && !strcmp(argv[1], "--returned")) + return getauxval(AT_PAGESZ) == + strtoul(getenv("AURORA_NATIVE_PAGE_SIZE"), + NULL, 10) ? + 0 : + 1; + if (argc > 1 && !strcmp(argv[1], "--returned4k")) + return getauxval(AT_PAGESZ) == 4096 ? 0 : 1; + if (getpid() != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + native_size = getauxval(AT_PAGESZ); + char native_text[24]; + snprintf(native_text, sizeof(native_text), "%lu", native_size); + setenv("AURORA_NATIVE_PAGE_SIZE", native_text, 1); + setvbuf(stdout, NULL, _IONBF, 0); + mount("proc", "/proc", "proc", 0, NULL); + int no4k = argc > 1 && !strcmp(argv[1], "--no4k"), + fd = memfd_create("compat-offsets", 0); + if (no4k) { + errno = 0; + result(prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL) == -1 && + errno == EOPNOTSUPP, + "4KB capability is actually masked"); + } + result(fd >= 0 && !ftruncate(fd, 0x100000000LL + 4 * native_size), + "sparse file fixture above 4GB"); + unsigned char page[4096]; + for (unsigned q = 0; q < 4 * native_size / 4096; q++) { + memset(page, 0x40 + q, sizeof(page)); + if (pwrite(fd, page, sizeof(page), q * 4096) != sizeof(page)) + failed++; + memset(page, 0x80 + q, sizeof(page)); + if (pwrite(fd, page, sizeof(page), 0x100000000LL + q * 4096) != + sizeof(page)) + failed++; + } + char fdstr[24]; + snprintf(fdstr, sizeof(fdstr), "%d", fd); + char *arg = malloc(90001); + for (unsigned i = 0; i < 90000; i++) + arg[i] = 'a' + i % 23; + arg[90000] = 0; + char *args[] = { "/probe32-64k", no4k ? native_text : "4096", fdstr, + arg, NULL }; + result(launch(args[0], args, 0, 0), + "64KB-aligned ELF32 automatically selects available ABI"); + if (!no4k) { + args[0] = "/probe32-4k"; + result(launch(args[0], args, 0, 0), + "4KB-aligned ELF32 executes with 4KB translation"); + args[0] = "/probe32-64k"; + result(launch(args[0], args, 1, 0), + "explicit pending 4KB request allows ELF32"); + } + args[0] = "/script32"; + result(launch(args[0], args, 0, 0), + "script final ELF class determines page granule"); + args[0] = "/probe32-dynamic"; + result(launch(args[0], args, 0, 0), + "ELF32 interpreter mapping uses selected page granule"); + char *chain[] = { + "/probe32-64k", "--exec32", no4k ? native_text : "4096", + fdstr, arg, NULL + }; + result(launch(chain[0], chain, 0, 0), + "ELF32 to ELF32 restages arguments and selects ABI"); + char *back[] = { "/probe32-64k", "--exec64", NULL }; + result(launch(back[0], back, 0, 0), + "ELF32 to ELF64 exec restores native default"); + if (!no4k) { + back[1] = "--exec64-4k"; + result(launch(back[0], back, 0, 0), + "ELF32 can request alternative ELF64 exec granule"); + } + char *big = malloc(200001); + memset(big, 'x', 200000); + big[200000] = 0; + args[0] = "/probe32-64k"; + args[3] = big; + if (!no4k) + result(launch(args[0], args, 0, E2BIG), + "64 oversized transfers return E2BIG and preserve old mm and request"); + printf("COMPAT FIXTURE %s mode=%s\n", failed ? "FAIL" : "PASS", + no4k ? "no4k-native-fallback" : "auto4k"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/compat-mlock-init.c b/tools/testing/selftests/arm64/user4k/compat-mlock-init.c new file mode 100644 index 0000000000000..679c39370cd6d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-mlock-init.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +struct result { + uint32_t pagesize, passed, failed, error; +}; +int main(int argc, char **argv) +{ + struct result out = { 0 }; + unsigned long expected; + int pipes[2], status = -1; + pid_t pid; + size_t done = 0; + int ok; + + if (getpid() != 1 || + (argc != 1 && (argc != 2 || strcmp(argv[1], "--no4k")))) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + expected = argc == 2 ? getauxval(AT_PAGESZ) : 4096; + if (mount("proc", "/proc", "proc", 0, NULL) || pipe(pipes)) + return 3; + pid = fork(); + if (!pid) { + char value[32]; + close(pipes[0]); + if (dup2(pipes[1], 1) < 0) + _exit(4); + close(pipes[1]); + snprintf(value, sizeof(value), "%lu", expected); + execl("/compat-mlock", "/compat-mlock", value, NULL); + _exit(5); + } + close(pipes[1]); + while (done < sizeof(out)) { + ssize_t n = + read(pipes[0], (char *)&out + done, sizeof(out) - done); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + break; + done += n; + } + close(pipes[0]); + ok = pid > 0 && waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status) && done == sizeof(out) && + out.pagesize == expected && out.passed == 6 && !out.failed; + printf("%s - AArch32 memlock expected=%lu actual=%u passed=%u stage=%u error=%d status=%d\n", + ok ? "ok" : "not ok", expected, out.pagesize, out.passed, + out.failed, (int32_t)out.error, status); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c b/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c new file mode 100644 index 0000000000000..97e76dd94bae9 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding ELF32 memlock probe, only for the disposable PID1 fixture. */ +typedef unsigned int u32; +struct result { + u32 pagesize, passed, failed, error; +}; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static int error(long n) +{ + return (u32)n >= 0xfffff001U; +} +static unsigned number(const char *p) +{ + unsigned n = 0; + while (*p >= '0' && *p <= '9') + n = n * 10 + *p++ - '0'; + return n; +} +static long locked_kb(void) +{ + char buf[8192]; + long fd = call(5, (long)"/proc/self/status", 0, 0, 0, 0, 0), n; + const char name[] = "VmLck:"; + + if (error(fd)) + return -1; + n = call(3, fd, (long)buf, sizeof(buf) - 1, 0, 0, 0); + call(6, fd, 0, 0, 0, 0, 0); + if (n <= 0) + return -1; + buf[n] = 0; + for (long i = 0; i < n; i++) { + unsigned j = 0; + while (name[j] && i + j < (unsigned)n && buf[i + j] == name[j]) + j++; + if (!name[j]) { + const char *p = buf + i + j; + while (*p == ' ' || *p == '\t') + p++; + return number(p); + } + } + return -1; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + unsigned argc = *sp++, ps = 0, kb, stage = 1; + char **argv = (void *)sp; + struct result out = { 0 }; + long p, q, pid, ret = -1; + int status = -1; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + ps = sp[1]; + sp += 2; + } + out.pagesize = ps; + if (argc != 2 || call(64, 0, 0, 0, 0, 0, 0) != 1 || + ps != number(argv[1])) + goto done; + kb = ps / 1024; + p = call(192, 0, 8 * ps, 3, 0x22, -1, 0); + if (error(p) || ((u32)p & (ps - 1)) || locked_kb() != 0) + goto done; + ret = call(150, p + ps + 17, ps - 32, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) + goto done; + out.passed++; + stage++; + ret = call(150, p + ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) + goto done; + ret = call(150, p + 2 * ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)(2 * kb)) + goto done; + ret = call(151, p + ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) + goto done; + out.passed++; + stage++; + *(volatile unsigned char *)(p + 2 * ps) = 0x63; + q = call(163, p + 2 * ps, ps, 2 * ps, 1, 0, 0); + ret = q; + if (error(q) || locked_kb() != (long)(2 * kb) || + *(volatile unsigned char *)q != 0x63) + goto done; + for (unsigned i = ps; i < 2 * ps; i++) + if (((volatile unsigned char *)q)[i]) + goto done; + out.passed++; + stage++; + pid = call(2, 0, 0, 0, 0, 0, 0); + if (!pid) { + call(1, locked_kb() != 0, 0, 0, 0, 0, 0); + for (;;) { + } + } + ret = call(114, pid, (long)&status, 0, 0, 0, 0); + if (error(pid) || ret != pid || status || locked_kb() != (long)(2 * kb)) + goto done; + out.passed++; + stage++; + ret = call(91, q, 2 * ps, 0, 0, 0, 0); + if (ret || locked_kb() != 0) + goto done; + out.passed++; + stage++; + ret = call(390, p + 4 * ps, 2 * ps, 1, 0, 0, 0); + if (ret || locked_kb() != (long)(2 * kb)) + goto done; + *(volatile unsigned char *)(p + 4 * ps) = 0x75; + ret = call(151, p + 4 * ps, 2 * ps, 0, 0, 0, 0); + if (ret || locked_kb() != 0) + goto done; + out.passed++; + stage = 0; +done: + out.failed = stage; + out.error = stage ? (u32)ret : 0; + call(4, 1, (long)&out, sizeof(out), 0, 0, 0); + call(1, stage != 0, 0, 0, 0, 0, 0); + for (;;) { + } +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c b/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c new file mode 100644 index 0000000000000..f2c155a888c15 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 automatic NUMA migration and fallback checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 NUMAB FAIL stage=00\n"; + + if (code) { + text[sizeof(text) - 4] = '0' + code / 10; + text[sizeof(text) - 3] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 NUMAB PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void value(const char *label, unsigned int length, u32 n) +{ + char text[16]; + unsigned int pos = sizeof(text); + + text[--pos] = '\n'; + do { + text[--pos] = '0' + n % 10; + n /= 10; + } while (n); + call(4, 1, (long)label, length, 0, 0, 0); + call(4, 1, (long)(text + pos), sizeof(text) - pos, 0, 0, 0); +} + +static u32 seconds(void) +{ + struct { + long long sec, nsec; + } ts; + + /* Use time64 even when CONFIG_COMPAT_32BIT_TIME is disabled. */ + if (call(403, 1, (long)&ts, 0, 0, 0, 0)) + finish(11); + return ts.sec; +} + +static void cpu(unsigned int id) +{ + u32 mask = 1U << id; + + if (call(241, 0, sizeof(mask), (long)&mask, 0, 0, 0)) + finish(12); +} + +static int node_at(unsigned char *address) +{ + int node = -1; + + if (call(320, (long)&node, 0, 0, (long)address, 3, 0)) + finish(13); + return node; +} + +static u32 sample(unsigned char *base, u32 length, u32 page) +{ + u32 moved = 0; + + for (u32 off = 0; off < length; off += page) + if (off != 2 * page) + moved += node_at(base + off + 16) == 1; + return moved; +} + +static void forbidden(unsigned char *address, int write_access) +{ + long child = call(2, 0, 0, 0, 0, 0, 0); + int status = 0; + + if (child < 0) + finish(40); + if (!child) { + /* Force the intended access; this is not thread synchronization. */ + if (write_access) + *(volatile unsigned char *)address = 0x27; + else + (void)*(volatile unsigned char *)address; + call(1, 2, 0, 0, 0, 0, 0); + for (;;) + ; + } + if (call(114, child, (long)&status, 0, 0, 0, 0) != child || + (status & 0x7f) != 11) + finish(41); +} + +static void mapping(u32 page) +{ + const u32 length = 8U << 20; + u32 mask[2] = { 1, 0 }, got[2] = { 0, 0 }; + u32 loops = 0, moved = 0, started, total = length / page - 1; + int mode = -1; + long raw; + unsigned char *base; + + if (call(320, (long)&mode, (long)got, 64, 0, 4, 0) || got[0] != 3 || + got[1]) + finish(20); + cpu(0); + raw = call(192, 0, length, 3, 0x22, -1, 0); + if ((u32)raw >= 0xfffff001U) + finish(21); + base = (void *)raw; + /* A missing 4K capability must retain native alignment restrictions. */ + if (call(125, raw + 4096, 4096, 1, 0, 0, 0) != (page == 4096 ? 0 : -22)) + finish(22); + if (call(125, raw, length, 3, 0, 0, 0)) + finish(23); + if (call(220, raw, length, 15, 0, 0, 0)) /* MADV_NOHUGEPAGE */ + finish(24); + if (call(319, raw, length, 2, (long)mask, 64, 0)) + finish(25); + for (u32 off = 0; off < length; off += page) { + u32 *p = (void *)(base + off); + + p[0] = 0x91720000 + off / page; + p[1] = 0; + p[page / 4 - 1] = 0x51320000 + off / page; + if (node_at(base + off + 16) != 0) + finish(26); + } + if (call(125, raw + page, page, 1, 0, 0, 0) || + call(125, raw + 2 * page, page, 0, 0, 0, 0)) + finish(27); + if (call(319, raw, length, 0, 0, 0, + 0)) /* MPOL_DEFAULT, no MOVE flags */ + finish(28); + cpu(1); + started = seconds(); + do { + for (u32 off = 0; off < length; off += page) { + volatile u32 *p = (void *)(base + off); + + if (off == 2 * page) + continue; + if (p[0] != 0x91720000 + off / page || + p[page / 4 - 1] != 0x51320000 + off / page) + finish(29); + if (off != page) + p[1]++; + } + loops++; + if (!(loops % 128)) { + moved = sample(base, length, page); + if (moved == total) + break; + } + } while (seconds() - started < 30); + moved = sample(base, length, page); + value("A32 NUMAB page=", sizeof("A32 NUMAB page=") - 1, page); + value("A32 NUMAB moved=", sizeof("A32 NUMAB moved=") - 1, moved); + value("A32 NUMAB expected=", sizeof("A32 NUMAB expected=") - 1, total); + value("A32 NUMAB loops=", sizeof("A32 NUMAB loops=") - 1, loops); + if (moved != total) + finish(30); + for (u32 off = 0; off < length; off += page) + if (off != page && off != 2 * page && + *(u32 *)(base + off + 4) != loops) + finish(31); + forbidden(base + page, 1); + forbidden(base + 2 * page, 0); + if (call(125, raw + 2 * page, page, 1, 0, 0, 0) || + *(u32 *)(base + 2 * page) != 0x91720002) + finish(32); + if (call(91, raw, length, 0, 0, 0, 0)) + finish(33); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, page = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + page = sp[1]; + sp += 2; + } + if (argc != 2 || page != number(argv[1])) + finish(10); + mapping(page); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-numa-probe.c b/tools/testing/selftests/arm64/user4k/compat-numa-probe.c new file mode 100644 index 0000000000000..b2652fe348b6e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-numa-probe.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 NUMA policy and native-granule fallback checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 NUMA FAIL stage=00\n"; + + if (code) { + text[sizeof(text) - 4] = '0' + code / 10; + text[sizeof(text) - 3] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 NUMA PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long native, unsigned long page) +{ + unsigned long length = 4 * native; + long raw = call(192, 0, length, 3, 0x22, -1, 0); + unsigned char *base; + u32 mask[2] = { 1, 0 }, got[2] = { 0, 0 }; + int mode = -1, node; + + if ((unsigned long)raw >= 0xfffff001UL) + finish(20); + base = (void *)(((unsigned long)raw + native - 1) & ~(native - 1)); + if (call(320, (long)&mode, (long)got, 64, 0, 4, 0) || got[0] != 3 || + got[1]) + finish(21); + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 0)) + finish(22); + mask[0] = 2; + if (call(319, (long)(base + page), 1, 2, (long)mask, 64, 0)) + finish(23); + if (call(450, (long)(base + page), 1, 1, 0, 0, 0)) + finish(24); + if (call(320, (long)&mode, (long)got, 64, (long)(base + page), 2, 0) || + mode != 2 || got[0] != 2 || got[1]) + finish(25); + for (unsigned long off = 0; off < 2 * native; off += page) { + base[off + 17] = 0x51 + off / page; + node = -1; + if (call(320, (long)&node, 0, 0, (long)(base + off + 17), 3, + 0) || + node != (off == page ? 1 : 0)) + finish(26); + } + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 1) != -5) + finish(27); + if (call(319, (long)(base + 4096), 4096, 2, (long)mask, 64, 0) != + (page == 4096 ? 0 : -22)) + finish(28); + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 5)) + finish(29); + for (unsigned long off = 0; off < 2 * native; off += page) { + node = -1; + if (base[off + 17] != (unsigned char)(0x51 + off / page) || + call(320, (long)&node, 0, 0, (long)(base + off + 17), 3, + 0) || + node != 1) + finish(30); + } + if (call(91, raw, length, 0, 0, 0, 0)) + finish(31); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + mapping(number(argv[1]), pages); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-perf-probe.c b/tools/testing/selftests/arm64/user4k/compat-perf-probe.c new file mode 100644 index 0000000000000..ee02fc34051ad --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-perf-probe.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 perf mmap layout, including native-page fallback. */ +typedef unsigned int u32; +typedef unsigned long long u64; + +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 PERF FAIL stage=00\n"; + + if (code) { + text[20] = '0' + code / 10; + text[21] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 PERF PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long pages, unsigned long native, + unsigned int count) +{ + static const struct { + u32 type, size; + u64 config, sample_period, sample_type, read_format, flags; + u32 wakeup, bp_type; + u64 config1; + } attr = { .type = 1, .size = 64, .flags = 1 }; + long fd = call(364, (long)&attr, 0, -1, -1, 0, 0); + unsigned long length = (1 + count) * pages; + long raw; + volatile u64 *metadata; + + if (fd < 0) + finish(20); + raw = call(192, 0, length, 3, 1, fd, 0); + if ((unsigned long)raw >= 0xfffff001UL) + finish(21); + metadata = (void *)(raw + 1024); + if (metadata[2] != pages || metadata[3] != count * pages) + finish(22); + metadata[1] = 0; + metadata[6] = length; + metadata[7] = native; + /* mmap2 always takes 4K units, including the no4K compat fallback. */ + if (call(192, 0, native, 1, 1, fd, length / 4096) != -95) + finish(23); + if (call(91, raw, length, 0, 0, 0, 0)) + finish(24); + if (call(6, fd, 0, 0, 0, 0, 0)) + finish(25); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + unsigned long native = number(argv[1]); + + mapping(pages, native, 0); + mapping(pages, native, 1); + mapping(pages, native, 2); + mapping(pages, native, 2 * native / pages); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-policy-probe.c b/tools/testing/selftests/arm64/user4k/compat-policy-probe.c new file mode 100644 index 0000000000000..dab5a79abe130 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-policy-probe.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 -> AArch64 exec with an inherited granule preference. */ +#include "../../../../../include/uapi/linux/prctl.h" +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static unsigned int number(const char *s) +{ + unsigned int n = 0; + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} +static void fail(int code) +{ + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + sp += argc + 1; + char **env = (void *)sp; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 4 || pages != number(argv[3])) + fail(10); + if (call(172, PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0, 0, 0, 0, 0) != + number(argv[2])) + fail(11); + if (call(172, PR_AURORA_GET_EXEC_PAGE_SIZE, 0, 0, 0, 0, 0)) + fail(12); + long mem = call(192, 0, 4 * pages, 3, 0x22, -1, 0); + if ((unsigned long)mem >= 0xfffff001UL) + fail(13); + long ret = call(125, mem + 4096, 4096, 1, 0, 0, 0); + if (ret != (pages == 4096 ? 0 : -22)) + fail(14); + if (call(91, mem, 4 * pages, 0, 0, 0, 0)) + fail(15); + char *args[] = { "/init", "--compat-return", argv[1], argv[2], 0 }; + call(11, (long)args[0], (long)args, (long)env, 0, 0, 0); + fail(16); + for (;;) + ; +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-probe.c b/tools/testing/selftests/arm64/user4k/compat-probe.c new file mode 100644 index 0000000000000..14a4deb716412 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-probe.c @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding AArch32 process; run only by compat-fixture-init in a guest. */ +typedef unsigned int u32; +typedef unsigned short u16; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f, r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static unsigned length(const char *s) +{ + unsigned n = 0; + while (s[n]) + n++; + return n; +} +static int equal(const char *a, const char *b) +{ + while (*a && *a == *b) { + a++; + b++; + } + return *a == *b; +} +static unsigned number(const char *s) +{ + unsigned n = 0; + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} +static void out(const char *s) +{ + call(4, 1, (long)s, length(s), 0, 0, 0); +} +static int failures, checks; +static void result(int pass, const char *s) +{ + checks++; + if (!pass) + failures++; + out(pass ? "ok - compat " : "not ok - compat "); + out(s); + out("\n"); +} +static int error(long v) +{ + return (unsigned long)v >= 0xfffff001UL; +} +static long map(unsigned size, int prot, int flags, int fd, unsigned off) +{ + return call(192, 0, size, prot, flags, fd, off); +} +static void unmap(long p, unsigned size) +{ + if (!error(p)) + call(91, p, size, 0, 0, 0, 0); +} +static volatile unsigned signals; +static void handler(int n) +{ + if (n == 10) + signals++; +} +struct action { + u32 handler, flags, restorer, mask[2]; +}; +struct ehdr { + unsigned char ident[16]; + u16 type, machine; + u32 version, entry, phoff, shoff, flags; + u16 ehsize, phentsize, phnum, shentsize, shnum, shstrndx; +}; +struct phdr { + u32 type, offset, vaddr, paddr, filesz, memsz, flags, align; +}; +struct dyn { + u32 tag, val; +}; +struct sym { + u32 name, value, size; + unsigned char info, other; + u16 shndx; +}; +static void *vdso_symbol(unsigned base, const char *name) +{ + struct ehdr *h = (void *)base; + struct phdr *p = (void *)(base + h->phoff); + struct dyn *d = 0; + char *strings = 0; + struct sym *symbols = 0; + u32 *hash = 0; + for (unsigned i = 0; i < h->phnum; i++) + if (p[i].type == 2) + d = (void *)(base + p[i].vaddr); + if (!d) + return 0; + for (; d->tag; d++) { + if (d->tag == 5) + strings = (void *)(base + d->val); + if (d->tag == 6) + symbols = (void *)(base + d->val); + if (d->tag == 4) + hash = (void *)(base + d->val); + } + if (!strings || !symbols || !hash) + return 0; + for (unsigned i = 0; i < hash[1]; i++) + if (equal(strings + symbols[i].name, name)) + return (void *)(base + symbols[i].value); + return 0; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + unsigned argc = *sp++, pagesz = 0, vdso = 0; + char **argv = (void *)sp; + sp += argc + 1; + char **env = (void *)sp; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pagesz = sp[1]; + if (sp[0] == 33) + vdso = sp[1]; + sp += 2; + } + if (argc > 1 && equal(argv[1], "--exec32")) { + char *a[] = { argv[0], argv[2], argv[3], argv[4], 0 }; + call(11, (long)a[0], (long)a, (long)env, 0, 0, 0); + call(1, 97, 0, 0, 0, 0, 0); + for (;;) + ; + } + if (argc > 1 && + (equal(argv[1], "--exec64") || equal(argv[1], "--exec64-4k"))) { + int alternative = equal(argv[1], "--exec64-4k"); + if (alternative && call(172, 0x41555001, 4096, 0, 0, 0, 0)) { + call(1, 99, 0, 0, 0, 0, 0); + for (;;) + ; + } + char *a[] = { "/init", + alternative ? "--returned4k" : "--returned", 0 }; + call(11, (long)a[0], (long)a, (long)env, 0, 0, 0); + call(1, 98, 0, 0, 0, 0, 0); + for (;;) + ; + } + /* Script interpreter contributes its pathname before the ordinary arguments. */ + unsigned start = argc > 1 && equal(argv[1], "/script32") ? 2 : 1; + unsigned expected = argc > start ? number(argv[start]) : 0; + int fd = argc > start + 1 ? (int)number(argv[start + 1]) : -1; + result(pagesz == expected, "AT_PAGESZ reflects final ELF32 selection"); + int args = argc > start + 2 && length(argv[start + 2]) == 90000; + if (args) + for (unsigned i = 0; i < 90000; i++) + if (argv[start + 2][i] != (char)('a' + i % 23)) { + args = 0; + break; + } + result(args, "90KB staged argument survives granule transfer"); + long mem = map(4 * pagesz, 3, 0x22, -1, 0); + int ok = !error(mem); + if (ok) { + unsigned char *p = (void *)mem; + for (unsigned i = 0; i < 4 * pagesz; i++) + p[i] = 0x30 + i / pagesz; + for (unsigned i = 0; i < 4 * pagesz; i++) + if (p[i] != (unsigned char)(0x30 + i / pagesz)) + ok = 0; + } + result(ok, "anonymous user leaves have independent bytes"); + result(!error(mem) && !call(125, mem + pagesz, pagesz, 1, 0, 0, 0) && + !call(125, mem + pagesz, pagesz, 3, 0, 0, 0), + "mprotect at selected page boundaries"); + result(!error(mem) && call(125, mem + 1, pagesz, 1, 0, 0, 0) == -22 && + call(91, mem + 1, pagesz, 0, 0, 0, 0) == -22, + "unaligned memory operations retain restrictions"); + long q = map(pagesz, 1, 1, fd, 3 * (pagesz >> 12)); + ok = !error(q); + if (ok) + for (unsigned i = 0; i < pagesz; i++) + if (((unsigned char *)q)[i] != + (unsigned char)(0x40 + (3 * pagesz + i) / 4096)) + ok = 0; + result(ok, "mmap2 native-aligned file offset"); + unmap(q, pagesz); + q = map(4096, 1, 1, fd, 1); + ok = pagesz == 4096 ? !error(q) && *(unsigned char *)q == 0x41 : + q == -22; + result(ok, "mmap2 4KB offset accepted only by matching ABI"); + unmap(q, pagesz); + q = map(pagesz, 1, 1, fd, 0x100000 + (pagesz >> 12)); + ok = !error(q); + if (ok) + for (unsigned i = 0; i < pagesz; i++) + if (((unsigned char *)q)[i] != + (unsigned char)(0x80 + (pagesz + i) / 4096)) + ok = 0; + result(ok, "mmap2 offset above 4GB remains full width"); + unmap(q, pagesz); + long child = call(2, 0, 0, 0, 0, 0, 0); + int status = 0; + if (!child) { + if (!error(mem)) + *(unsigned char *)mem = 0xa7; + call(1, 0, 0, 0, 0, 0, 0); + for (;;) + ; + } + ok = child > 0 && + call(114, child, (long)&status, 0, 0, 0, 0) == child && !status && + !error(mem) && *(unsigned char *)mem == 0x30; + result(ok, "fork isolates private COW data"); + unmap(mem, 4 * pagesz); + struct action act = { (u32)handler, 0, 0, { 0, 0 } }; + ok = !call(174, 10, (long)&act, 0, 8, 0, 0); + if (ok) + ok = !call(37, call(20, 0, 0, 0, 0, 0, 0), 10, 0, 0, 0, 0) && + signals == 1; + result(ok, "AArch32 signal frame and kernel sigreturn trampoline"); + volatile unsigned word = 17; + int (*cmpxchg)(unsigned, unsigned, volatile unsigned *) = + (void *)0xffff0fc0; + result(*(volatile unsigned *)0xffff0ffc >= 5 && + !cmpxchg(17, 29, &word) && word == 29, + "kuser helper page and atomic compare exchange"); + int (*gettime)(int, void *) = + vdso ? (void *)vdso_symbol(vdso, "__vdso_clock_gettime") : 0; + long time[2] = { 0, 0 }; + result(gettime && !gettime(1, time) && time[0] >= 0 && time[1] >= 0 && + time[1] < 1000000000, + "compat vDSO clock_gettime and vvar mapping"); + out(failures ? "COMPAT PROBE FAIL\n" : "COMPAT PROBE PASS\n"); + call(1, failures ? 1 : 0, 0, 0, 0, 0, 0); + for (;;) + ; +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-script b/tools/testing/selftests/arm64/user4k/compat-script new file mode 100644 index 0000000000000..9b56d0c7567f5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-script @@ -0,0 +1 @@ +#!/probe32-64k diff --git a/tools/testing/selftests/arm64/user4k/compat-shm-probe.c b/tools/testing/selftests/arm64/user4k/compat-shm-probe.c new file mode 100644 index 0000000000000..c4ee1da2c5a12 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-shm-probe.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 SysV attach/detach and fallback-alignment checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 SHM FAIL stage=00\n"; + + if (code) { + text[19] = '0' + code / 10; + text[20] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 SHM PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long pages, int rounded) +{ + unsigned long align = 4 * pages, length = 4 * pages; + long raw = call(192, 0, 12 * align, 3, 0x22, -1, 0); + unsigned char *target; + long id, result; + + if ((unsigned long)raw >= 0xfffff001UL) + finish(20); + result = call(125, raw + 4096, 4096, 1, 0, 0, 0); + if (result != (pages == 4096 ? 0 : -22)) + finish(21); + if (call(125, raw, 12 * align, 3, 0, 0, 0)) + finish(22); + target = (void *)(((unsigned long)raw + align - 1) & ~(align - 1)); + target += align + (rounded ? 0 : pages); + target[-1] = 0x51; + target[length] = 0x72; + if (call(91, (long)target, length, 0, 0, 0, 0)) + finish(23); + id = call(307, 0, length - 1, 0600, 0, 0, 0); + if (id < 0) + finish(24); + result = call(305, id, (long)(target + (rounded ? pages + 17 : 0)), + rounded ? 020000 : 0, 0, 0, 0); + if (result != (long)target) { + call(308, id, 0, 0, 0, 0, 0); + finish(rounded ? 26 : 25); + } + if (call(308, id, 0, 0, 0, 0, 0)) + finish(27); + target[pages + 13] = 0x67; + if (call(125, (long)(target + pages), pages, 1, 0, 0, 0)) + finish(28); + if (call(91, (long)target, pages, 0, 0, 0, 0)) + finish(29); + if (call(306, (long)target, 0, 0, 0, 0, 0)) + finish(30); + if (target[-1] != 0x51 || target[length] != 0x72) + finish(31); + if (call(91, raw, 12 * align, 0, 0, 0, 0)) + finish(32); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + mapping(pages, 0); + mapping(pages, 1); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-uffd-init.c b/tools/testing/selftests/arm64/user4k/compat-uffd-init.c new file mode 100644 index 0000000000000..5f1e90f3e484f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-uffd-init.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - compat uffd line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct wire { + uint32_t address, pagesize; +}; +static void expect_invalid(int fd, uint64_t dst, uint64_t src, uint64_t len) +{ + struct uffdio_copy c = { .dst = dst, .src = src, .len = len }; + errno = 0; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && errno == EINVAL); +} +static void run(unsigned long native, unsigned long expected, int shared) +{ + int sock[2], file = -1, status; + CHECK(!socketpair(AF_UNIX, SOCK_SEQPACKET, 0, sock)); + if (shared) { + file = memfd_create("compat-uffd", 0); + CHECK(file >= 0 && !ftruncate(file, 3 * expected)); + } + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char page_arg[24], sock_arg[24], file_arg[24]; + close(sock[0]); + snprintf(page_arg, sizeof(page_arg), "%lu", expected); + snprintf(sock_arg, sizeof(sock_arg), "%d", sock[1]); + snprintf(file_arg, sizeof(file_arg), "%d", file); + execl("/compat-uffd", "/compat-uffd", page_arg, sock_arg, + file_arg, NULL); + _exit(99); + } + close(sock[1]); + struct wire data = { 0 }; + struct iovec iov = { &data, sizeof(data) }; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct msghdr msg = { .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control.bytes) }; + CHECK(recvmsg(sock[0], &msg, 0) == sizeof(data)); + CHECK(!(msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) && + data.pagesize == expected); + struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg); + CHECK(cmsg && cmsg->cmsg_level == SOL_SOCKET && + cmsg->cmsg_type == SCM_RIGHTS && + cmsg->cmsg_len == CMSG_LEN(sizeof(int))); + int uffd; + memcpy(&uffd, CMSG_DATA(cmsg), sizeof(uffd)); + /* Force the source above every possible AArch32 task limit. */ + unsigned char *source = + mmap((void *)0x1000000000UL, 2 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(source != MAP_FAILED && (uintptr_t)source > UINT32_MAX); + /* Source alignment is independent of both processes' granules. */ + unsigned char *bytes = source + 17; + for (unsigned leaf = 0; leaf < 3; leaf++) { + struct pollfd pfd = { .fd = uffd, .events = POLLIN }; + struct uffd_msg event; + CHECK(poll(&pfd, 1, 10000) == 1 && (pfd.revents & POLLIN)); + CHECK(read(uffd, &event, sizeof(event)) == sizeof(event)); + uint64_t dst = (uint64_t)data.address + leaf * expected; + CHECK(event.event == UFFD_EVENT_PAGEFAULT && + !event.arg.pagefault.flags && + event.arg.pagefault.address == dst); + if (!leaf) { + expect_invalid(uffd, dst, (uintptr_t)bytes, 0); + expect_invalid(uffd, dst, (uintptr_t)bytes, + expected - 1); + expect_invalid(uffd, dst + 1, (uintptr_t)bytes, + expected); + expect_invalid(uffd, 0x100000000ULL, (uintptr_t)bytes, + expected); + expect_invalid(uffd, dst, UINT64_MAX - expected + 1, + expected); + if (expected > 4096) + expect_invalid(uffd, dst, (uintptr_t)bytes, + 4096); + } + if (leaf == 1) { + /* A valid but inaccessible pager source must leave the target missing. */ + CHECK(!mprotect(source, 2 * native, PROT_NONE)); + struct uffdio_copy bad = { .dst = dst, + .src = (uintptr_t)bytes, + .len = expected }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &bad) == -1 && + errno == EFAULT && bad.copy == -EFAULT); + CHECK(!mprotect(source, 2 * native, + PROT_READ | PROT_WRITE)); + } + memset(bytes, 0x51 + leaf, expected); + struct uffdio_copy c = { .dst = dst, + .src = (uintptr_t)bytes, + .len = expected }; + CHECK(!ioctl(uffd, UFFDIO_COPY, &c) && + c.copy == (long long)expected); + } + CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!munmap(source, 2 * native)); + CHECK(!close(uffd) && !close(sock[0])); + if (shared) + CHECK(!close(file)); + printf("ok - native %lu pager high unaligned COPY to AArch32 %lu %s, source failure and target alignment\n", + native, expected, shared ? "shared" : "anonymous"); +} +int main(void) +{ + if (getpid() != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + alarm(90); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + char cmdline[2048] = { 0 }; + int fd = open("/proc/cmdline", O_RDONLY); + CHECK(fd >= 0 && read(fd, cmdline, sizeof(cmdline) - 1) > 0 && + !close(fd)); + unsigned long native = getauxval(AT_PAGESZ); + unsigned long expected = + strstr(cmdline, "id_aa64mmfr0.tgran4=f") ? native : 4096; + run(native, expected, 0); + run(native, expected, 1); + puts("COMPAT UFFD PASS"); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c b/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c new file mode 100644 index 0000000000000..6abcdb86bb4d8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding ELF32 half of compat-uffd-init; no armhf sysroot required. */ +typedef unsigned int u32; +typedef unsigned long long u64; +struct api { + u64 api, features, ioctls; +}; +struct registration { + u64 start, len, mode, ioctls; +}; +struct movement { + u64 dst, src, len, mode; + long long moved; +}; +struct wire { + u32 address, pagesize; +}; +struct iovec32 { + void *base; + u32 len; +}; +struct message32 { + void *name; + u32 namelen; + struct iovec32 *iov; + u32 iovlen; + void *control; + u32 controllen, flags; +}; +struct rights32 { + u32 len, level, type; + int fd; +}; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static u32 number(const char *s) +{ + u32 n = 0; + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} +__attribute__((noreturn)) static void finish(int code) +{ + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 4 || pages != number(argv[1])) + finish(10); + int sock = number(argv[2]), shared = argv[3][0] != '-'; + int file = shared ? (int)number(argv[3]) : -1; + long map = call(192, 0, 3 * pages, 3, shared ? 1 : 0x22, file, 0); + if ((unsigned long)map >= 0xfffff001UL) + finish(11); + /* Fine operations must retain native alignment restrictions on fallback. */ + if (call(125, map + 4096, 4096, 1, 0, 0, 0) != + (pages == 4096 ? 0 : -22)) + finish(12); + if (call(125, map, 3 * pages, 3, 0, 0, 0)) + finish(13); + long fd = call(388, 0x80801, 0, 0, 0, 0, + 0); /* CLOEXEC|NONBLOCK|USER_MODE_ONLY */ + if (fd < 0) + finish(14); + /* Negotiate shared missing support under both fine and fallback ABIs. */ + struct api api = { 0xaa, (1ULL << 5) | (1ULL << 16), 0 }; + if (call(54, fd, 0xc018aa3fU, (long)&api, 0, 0, 0) || + !(api.features & (1ULL << 5))) + finish(15); + struct registration reg = { (u32)map, 3 * pages, 1, 0 }; + if (call(54, fd, 0xc020aa00U, (long)®, 0, 0, 0)) + finish(16); + if (!(reg.ioctls & (1ULL << 3)) || !(api.features & (1ULL << 16))) + finish(17); /* COPY and MOVE */ + struct wire data = { (u32)map, pages }; + struct iovec32 iov = { &data, sizeof(data) }; + struct rights32 rights = { sizeof(rights), 1, 1, fd }; + struct message32 msg = { 0, 0, &iov, 1, &rights, sizeof(rights), 0 }; + if (call(296, sock, (long)&msg, 0, 0, 0, 0) != sizeof(data)) + finish(18); + volatile unsigned char *p = (void *)map; + /* Each first access waits for the native pager's COPY. */ + for (u32 leaf = 0; leaf < 3; leaf++) + for (u32 i = 0; i < pages; i++) + if (p[leaf * pages + i] != (unsigned char)(0x51 + leaf)) + finish(19); + if (!shared) { + long target = call(192, 0, 3 * pages, 3, 0x22, -1, 0); + if ((unsigned long)target >= 0xfffff001UL) + finish(21); + struct registration dst = { (u32)target, 3 * pages, 1, 0 }; + if (call(54, fd, 0xc020aa00U, (long)&dst, 0, 0, 0) || + !(dst.ioctls & (1ULL << 5))) + finish(22); + struct movement m = { 1ULL << 32, (u32)map + 2 * pages, pages, + 0, 0 }; + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) != -22) + finish(23); + m.dst = (u32)target + pages; + if (pages > 4096) { + m.len = 4096; + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) != -22) + finish(24); + m.len = pages; + } + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) || + m.moved != pages) + finish(25); + volatile unsigned char *q = (void *)target; + for (u32 i = 0; i < pages; i++) + if (q[pages + i] != 0x53 || p[i] != 0x51 || + p[pages + i] != 0x52) + finish(26); + if (call(91, target, 3 * pages, 0, 0, 0, 0)) + finish(27); + static const char ok[] = + "ok - AArch32 MOVE, neighbours and native-fallback alignment\n"; + call(4, 1, (long)ok, sizeof(ok) - 1, 0, 0, 0); + } + if (call(91, map, 3 * pages, 0, 0, 0, 0)) + finish(20); + finish(0); +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c b/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c new file mode 100644 index 0000000000000..231e4b7996613 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 type1 probe; no armhf sysroot or physical DMA. */ +#define __user +#include "../../../../../include/uapi/linux/vfio.h" +#include "../../../../../drivers/vfio/vfio_iommu_type1_test.h" + +typedef unsigned int u32; +typedef unsigned long long u64; +#define IOVA (256UL * 1024 * 1024) +static unsigned int stage; + +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static u32 number(const char *s) +{ + u32 n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static void clear(void *data, u32 size) +{ + volatile unsigned char *p = data; + + for (u32 i = 0; i < size; i++) + p[i] = 0; +} + +/* Aggregate initialization may use these compiler-runtime entry points. */ +void __aeabi_memclr8(void *data, u32 size) +{ + clear(data, size); +} +void __aeabi_memclr4(void *data, u32 size) +{ + clear(data, size); +} +void __aeabi_memclr(void *data, u32 size) +{ + clear(data, size); +} + +_Noreturn static void finish(int code) +{ + if (code) { + char msg[] = "not ok - AArch32 type1 stage=00\n"; + + msg[29] = '0' + stage / 10; + msg[30] = '0' + stage % 10; + call(4, 1, (long)msg, sizeof(msg) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} +#define CHECK(x) \ + do { \ + stage++; \ + if (!(x)) \ + finish(stage); \ + } while (0) + +static long ioctl_test(int fd, u32 request, void *arg) +{ + return call(54, fd, request, (long)arg, 0, 0, 0); +} + +static int state(int fd, struct vfio_type1_test_cmd *cmd) +{ + clear(cmd, sizeof(*cmd)); + return ioctl_test(fd, VFIO_TYPE1_TEST_STATE, cmd); +} + +static long map(int fd, u32 address, u32 size) +{ + struct vfio_iommu_type1_dma_map cmd; + + clear(&cmd, sizeof(cmd)); + cmd.argsz = sizeof(cmd); + cmd.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + cmd.iova = IOVA; + cmd.vaddr = address; + cmd.size = size; + return ioctl_test(fd, VFIO_IOMMU_MAP_DMA, &cmd); +} + +static long unmap(int fd, u32 size) +{ + struct vfio_iommu_type1_dma_unmap cmd; + long ret; + + clear(&cmd, sizeof(cmd)); + cmd.argsz = sizeof(cmd); + cmd.iova = IOVA; + cmd.size = size; + ret = ioctl_test(fd, VFIO_IOMMU_UNMAP_DMA, &cmd); + return ret ? ret : cmd.size == size ? 0 : -1; +} + +static long rw(int fd, unsigned char *bytes, unsigned int flags) +{ + struct vfio_type1_test_cmd cmd; + + clear(&cmd, sizeof(cmd)); + cmd.iova = IOVA; + cmd.buffer = (u32)bytes; + cmd.size = 16; + cmd.flags = flags; + return ioctl_test(fd, VFIO_TYPE1_TEST_RW, &cmd); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_dirty_bitmap dirty; + struct vfio_iommu_type1_info info; + unsigned char bytes[16]; + long fd, file, data, pid; + int status = -1; + u32 native, length; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + CHECK(argc == 3 && pages == number(argv[2])); + native = number(argv[1]); + length = native + 4096; + fd = call(5, (long)"/dev/vfio-test", 2, 0, 0, 0, 0); + file = call(385, (long)"compat-vfio", 0, 0, 0, 0, 0); + CHECK(fd >= 0 && file >= 0 && !call(93, file, 2 * native, 0, 0, 0, 0)); + data = call(192, 0, 2 * native, 3, 1, file, 0); + CHECK((u32)data < 0xfffff001U); + CHECK(call(125, data + 4096, 4096, 1, 0, 0, 0) == + (pages == 4096 ? 0 : -22)); + CHECK(!call(125, data, 2 * native, 3, 0, 0, 0)); + for (u32 i = 0; i < 2 * native; i++) + ((volatile unsigned char *)data)[i] = 0x53; + clear(&info, sizeof(info)); + info.argsz = sizeof(info); + CHECK(!ioctl_test(fd, VFIO_IOMMU_GET_INFO, &info) && + (info.iova_pgsizes & 4096)); + CHECK(!map(fd, data, 2 * native)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes); + CHECK(!rw(fd, bytes, 0) && bytes[0] == 0x53 && bytes[15] == 0x53); + clear(&dirty, sizeof(dirty)); + dirty.argsz = sizeof(dirty); + dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_START; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, &dirty)); + clear(&cmd, sizeof(cmd)); + cmd.iova = IOVA + 19; + CHECK(!ioctl_test(fd, VFIO_TYPE1_TEST_PIN, &cmd)); + CHECK(!state(fd, &cmd) && cmd.locked_bytes == native && + cmd.external_refs == 1); + /* Compat pointer padding must not become part of the native pointer. */ + _Alignas(struct vfio_iommu_type1_dirty_bitmap_get) unsigned char + buffer[sizeof(dirty) + + sizeof(struct vfio_iommu_type1_dirty_bitmap_get)]; + struct vfio_iommu_type1_dirty_bitmap *header = (void *)buffer; + struct vfio_iommu_type1_dirty_bitmap_get *range = (void *)header->data; + u64 bits = 0; + + clear(buffer, sizeof(buffer)); + header->argsz = sizeof(buffer); + header->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP; + range->iova = IOVA; + range->size = 2 * native; + range->bitmap.pgsize = 4096; + range->bitmap.size = sizeof(bits); + range->bitmap.data = &bits; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, buffer) && + bits == ((1ULL << (native >> 12)) - 1)); + _Static_assert( + sizeof(range->bitmap.data) == 4 && sizeof(range->bitmap) == 24, + "AArch32 bitmap has a 32-bit pointer and trailing padding"); + ((unsigned char *)&range + ->bitmap)[__builtin_offsetof(struct vfio_bitmap, data) + + sizeof(range->bitmap.data)] = 0xa5; + bits = 0; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, buffer) && + bits == ((1ULL << (native >> 12)) - 1)); + _Alignas(struct vfio_bitmap) unsigned char + unmap_buffer[sizeof(struct vfio_iommu_type1_dma_unmap) + + sizeof(struct vfio_bitmap)]; + struct vfio_iommu_type1_dma_unmap *remove = (void *)unmap_buffer; + struct vfio_bitmap *remove_bitmap = (void *)remove->data; + + clear(unmap_buffer, sizeof(unmap_buffer)); + remove->argsz = sizeof(unmap_buffer); + remove->flags = VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP; + remove->iova = IOVA; + remove->size = 2 * native; + remove_bitmap->pgsize = 4096; + remove_bitmap->size = sizeof(bits); + remove_bitmap->data = &bits; + ((unsigned char *) + remove_bitmap)[__builtin_offsetof(struct vfio_bitmap, data) + + sizeof(remove_bitmap->data)] = 0x5a; + bits = 0; + CHECK(!ioctl_test(fd, VFIO_IOMMU_UNMAP_DMA, unmap_buffer) && + remove->size == 2 * native && + bits == ((1ULL << (native >> 12)) - 1)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.external_refs && + cmd.notifications == 1); + clear(&cmd, sizeof(cmd)); + cmd.size = 4096; + CHECK(!ioctl_test(fd, VFIO_TYPE1_TEST_DOMAIN_ADD, &cmd)); + pid = call(2, 0, 0, 0, 0, 0, 0); + if (!pid) + finish(map(fd, data + 4096, length) ? 99 : 0); + CHECK(pid > 0 && call(114, pid, (long)&status, 0, 0, 0, 0) == pid && + !status); + CHECK(!state(fd, &cmd) && !cmd.owner_users && + cmd.locked_bytes == 2 * native && cmd.mapped_bytes[0] == length); + CHECK(rw(fd, bytes, 0) == -14); + CHECK(!rw(fd, bytes, VFIO_TYPE1_TEST_WORKER) && bytes[0] == 0x53 && + bytes[15] == 0x53); + CHECK(!unmap(fd, length)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0]); + CHECK(!call(91, data, 2 * native, 0, 0, 0, 0)); + CHECK(!call(6, file, 0, 0, 0, 0, 0) && !call(6, fd, 0, 0, 0, 0, 0)); + static const char ok[] = + "ok - AArch32 type1 real ioctls, source exit and fallback alignment\n"; + + call(4, 1, (long)ok, sizeof(ok) - 1, 0, 0, 0); + finish(0); +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/context-stress-init.c b/tools/testing/selftests/arm64/user4k/context-stress-init.c new file mode 100644 index 0000000000000..869662502c15f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/context-stress-init.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* PID-1 mixed-granule EL0 stress runner for disposable VMs. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define DATA_ADDRESS 0x200000000UL +#define CODE_ADDRESS 0x202000000UL +#define CONTEXTS 70000 +#define SHORT_CONTEXTS 2048 + +static sigjmp_buf fault_return; +static volatile uintptr_t expected_fault; +static char generation_path[256]; + +static void fault(int sig, siginfo_t *info, void *context) +{ + (void)context; + if (sig != SIGSEGV || !expected_fault || + (uintptr_t)info->si_addr != expected_fault || + info->si_code != SEGV_ACCERR) + _exit(90); + expected_fault = 0; + siglongjmp(fault_return, 1); +} + +static bool rejects_write(uintptr_t address) +{ + expected_fault = address; + if (!sigsetjmp(fault_return, 1)) { + *(volatile unsigned char *)expected_fault = 1; + expected_fault = 0; + return false; + } + return true; +} + +static int migrate(unsigned int cpu) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(0, sizeof(set), &set); +} + +static unsigned char pattern(unsigned long serial, unsigned long offset) +{ + serial ^= (serial >> 8) ^ (serial >> 16); + return (serial * 37 + offset * 17 + (offset >> 8)) & 255; +} + +static bool validate(unsigned char *p, unsigned long serial, unsigned long len) +{ + for (unsigned long i = 0; i < len; i++) + if (p[i] != pattern(serial, i)) + return false; + return true; +} + +static int worker(unsigned long serial, unsigned long ps, int control) +{ + struct sigaction action = { .sa_sigaction = fault, + .sa_flags = SA_SIGINFO }; + unsigned char *data; + uint32_t *code; + unsigned int cpus = sysconf(_SC_NPROCESSORS_ONLN); + struct stat self, init; + unsigned long iterations = 0; + + if (getauxval(AT_PAGESZ) != ps || getppid() != 1 || + stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) != 0) + return 91; + if (cpus < 2 || migrate(serial % cpus) || + sigaction(SIGSEGV, &action, NULL)) + return 92; + data = mmap((void *)DATA_ADDRESS, 4 * ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + code = mmap((void *)CODE_ADDRESS, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (data == MAP_FAILED || code == MAP_FAILED || + mprotect(data, 3 * ps, PROT_READ | PROT_WRITE)) + return 93; + for (unsigned long i = 0; i < 3 * ps; i++) + data[i] = pattern(serial, i); + if (!rejects_write(DATA_ADDRESS + 3 * ps)) + return 94; + if (mprotect(data + ps, ps, PROT_READ)) + return 95; + if (!rejects_write(DATA_ADDRESS + ps)) + return 96; + do { + unsigned int expected = (serial + iterations * 13) & 65535; + int (*fn)(void) = (void *)code; + char stop; + ssize_t got; + + if (mprotect(code, ps, PROT_READ | PROT_WRITE)) + return 97; + code[0] = 0x52800000 | (expected << 5); /* movz w0, #expected */ + code[1] = 0xd65f03c0; /* ret */ + __builtin___clear_cache((char *)code, (char *)(code + 2)); + if (mprotect(code, ps, PROT_READ | PROT_EXEC) || + migrate((serial + iterations + 1) % cpus)) + return 98; + sched_yield(); + if (fn() != (int)expected || !validate(data, serial, 3 * ps)) + return 99; + if (!(iterations % 64) && + (!rejects_write(DATA_ADDRESS + 3 * ps) || + !rejects_write(DATA_ADDRESS + ps))) + return 101; + iterations++; + if (control < 0) + break; + got = read(control, &stop, 1); + if (got == 1) + break; + if (got != -1 || (errno != EAGAIN && errno != EINTR)) + return 100; + } while (true); + if (control >= 0) + printf("# persistent %luK worker: %lu migration/JIT/data cycles\n", + ps / 1024, iterations); + return 0; +} + +static pid_t spawn(unsigned long serial, unsigned long ps, int control, + int close_fd) +{ + pid_t pid = fork(); + + if (!pid) { + char seq[32], size[32], fd[32]; + + if (close_fd >= 0) + close(close_fd); + if (control >= 0 && fcntl(control, F_SETFD, 0)) + _exit(110); + snprintf(seq, sizeof(seq), "%lu", serial); + snprintf(size, sizeof(size), "%lu", ps); + snprintf(fd, sizeof(fd), "%d", control); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, ps, 0UL, 0UL, 0UL)) + _exit(111); + execl("/init", "/init", "--worker", seq, size, fd, NULL); + _exit(112); + } + return pid; +} + +static bool reap(pid_t pid) +{ + int status = 0; + pid_t got; + + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + if (got == pid && WIFEXITED(status) && !WEXITSTATUS(status)) + return true; + printf("not ok - worker pid=%d wait=%d status=%#x\n", pid, got, status); + return false; +} + +static unsigned long long generation(void) +{ + unsigned long long value = 0; + FILE *f = fopen(generation_path, "re"); + + if (!f || fscanf(f, "%llu", &value) != 1) { + perror("ASID generation"); + return 0; + } + fclose(f); + return value; +} + +int main(int argc, char **argv) +{ + unsigned long long before, after; + unsigned char *data; + pid_t persistent[3]; + int commands[3], pipes[2]; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + glob_t paths; + bool ok = true; + unsigned int contexts = CONTEXTS; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (argc == 2 && !strcmp(argv[1], "--short")) + contexts = SHORT_CONTEXTS; + if (getpid() != 1 || (argc != 1 && contexts != SHORT_CONTEXTS)) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + glob("/sys/module/*/parameters/user4k_asid_generation", 0, NULL, + &paths) || + paths.gl_pathc != 1) { + perror("fixture setup"); + return 2; + } + snprintf(generation_path, sizeof(generation_path), "%s", + paths.gl_pathv[0]); + globfree(&paths); + data = mmap((void *)DATA_ADDRESS, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (data == MAP_FAILED) + return 3; + for (unsigned long i = 0; i < 4 * native; i++) + data[i] = pattern(0, i); + before = generation(); + printf("# mixed context stress: %u fork/exec workers, initial ASID generation=%llu\n", + contexts, before); + for (unsigned int i = 0; i < nr; i++) { + if (pipe2(pipes, O_NONBLOCK | O_CLOEXEC)) + return 4; + persistent[i] = spawn(CONTEXTS + i + 1, granules[i], pipes[0], + pipes[1]); + if (persistent[i] < 0) + return 5; + close(pipes[0]); + commands[i] = pipes[1]; + } + for (unsigned long i = 1; i <= contexts; i++) { + pid_t pid = spawn(i, granules[i % nr], -1, -1); + + if (pid < 0 || !reap(pid) || !validate(data, 0, 4 * native)) { + printf("not ok - churn/parent isolation at worker %lu\n", + i); + ok = false; + break; + } + if (!(i % 128)) + printf("# workers=%lu ASID generation=%llu\n", i, + generation()); + } + after = generation(); + for (unsigned int i = 0; i < nr; i++) { + ok &= write(commands[i], "x", 1) == 1; + close(commands[i]); + ok &= reap(persistent[i]); + } + ok &= before && after >= before + 2 * (1ULL << 16); + printf("%s - mixed context stress: generation=%llu->%llu, measured rollovers=%llu\n", + ok ? "ok" : "not ok", before, after, (after - before) >> 16); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/core-fixture-init.c b/tools/testing/selftests/arm64/user4k/core-fixture-init.c new file mode 100644 index 0000000000000..4caadf98296a6 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/core-fixture-init.c @@ -0,0 +1,447 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Core-dump regression runner: PID 1 in a disposable VM only. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define ANON 0x100000000UL +#define FILEMAP 0x102000000UL +#define SPARSE 0x104000000UL +#define EXCLUDED 0x106000000UL +#define SWAPPED 0x108000000UL +#define SWAPPED_SIZE (1UL << 20) +#define SPARSE_SIZE (8UL << 20) + +static void die(const char *what) +{ + perror(what); + _exit(120); +} + +static void write_text(const char *path, const char *text) +{ + int fd = open(path, O_WRONLY); + size_t len = strlen(text); + + if (fd < 0 || write(fd, text, len) != (ssize_t)len) + die(path); + close(fd); +} + +static void *map_at(unsigned long address, size_t len, int prot, int fd, + off_t off) +{ + void *p = mmap((void *)address, len, prot, + MAP_PRIVATE | MAP_FIXED_NOREPLACE | + (fd < 0 ? MAP_ANONYMOUS : 0), + fd, off); + + if (p == MAP_FAILED) + die("mmap"); + return p; +} + +static void crash(bool headers, int bounded) +{ + unsigned long ps = getauxval(AT_PAGESZ); + struct rlimit limit = { 64UL << 20, 64UL << 20 }; + struct stat self, init; + unsigned char *p, *data; + int fd; + + if (getppid() != 1 || stat("/proc/self/exe", &self) || + stat("/proc/1/exe", &init) || self.st_dev != init.st_dev || + self.st_ino != init.st_ino) + _exit(121); + if (bounded) + limit.rlim_cur = limit.rlim_max = ps - (bounded < 0); + if (setrlimit(RLIMIT_CORE, &limit)) + die("rlimit"); + write_text("/proc/self/coredump_filter", headers ? "0x10" : "0x1f"); + p = map_at(ANON, 4 * ps, PROT_READ | PROT_WRITE, -1, 0); + for (unsigned int i = 0; i < 4; i++) + memset(p + i * ps, 0x31 + i, ps); + if (mprotect(p + ps, ps, PROT_NONE) || + mprotect(p + 2 * ps, ps, PROT_READ)) + die("mprotect"); + p = map_at(SPARSE, SPARSE_SIZE, PROT_READ | PROT_WRITE, -1, 0); + memset(p, 0x65, ps); + /* Also exercise an existing read-only zero PTE. */ + if (*(volatile unsigned char *)(p + ps) != 0) + _exit(122); + p = map_at(EXCLUDED, ps, PROT_READ | PROT_WRITE, -1, 0); + memset(p, 0xa7, ps); + if (madvise(p, ps, MADV_DONTDUMP)) + die("dontdump"); + fd = open("/core-input", O_CREAT | O_TRUNC | O_RDWR, 0700); + data = malloc(19 * ps); + if (fd < 0 || !data) + die("file setup"); + for (unsigned int i = 0; i < 19; i++) + memset(data + i * ps, 0x81 + i, ps); + memcpy(data, ELFMAG, SELFMAG); + if (write(fd, data, 19 * ps) != (ssize_t)(19 * ps)) + die("file data"); + /* Offset one user page: three mappings select non-native quarters. */ + map_at(FILEMAP, ps, PROT_READ, fd, headers ? 0 : ps); + map_at(FILEMAP + 2 * ps, 2 * ps, PROT_READ, fd, 2 * ps); + map_at(FILEMAP + 8 * ps, ps, PROT_READ, fd, 17 * ps); + close(fd); + fd = open("/core-library", O_CREAT | O_TRUNC | O_RDWR, 0600); + if (fd < 0 || write(fd, data, ps) != (ssize_t)ps) + die("non-executable ELF file"); + map_at(FILEMAP + 12 * ps, ps, PROT_READ, fd, 0); + close(fd); + free(data); + if (!headers && !bounded) { + unsigned int missing = 0; + uint64_t entry; + int pm = open("/proc/self/pagemap", O_RDONLY); + + p = map_at(SWAPPED, SWAPPED_SIZE, PROT_READ | PROT_WRITE, -1, + 0); + memset(p, 0xc4, SWAPPED_SIZE); + for (unsigned int tries = 0; tries < 200; tries++) { + if (madvise(p, SWAPPED_SIZE, MADV_COLD) || + madvise(p, SWAPPED_SIZE, MADV_PAGEOUT)) + die("pageout"); + missing = 0; + for (unsigned long i = 0; i < SWAPPED_SIZE / ps; i++) { + if (pread(pm, &entry, sizeof(entry), + (SWAPPED / ps + i) * 8) != 8) + die("pagemap"); + missing += !(entry & (1ULL << 62)); + } + if (!missing) + break; + usleep(10000); + } + close(pm); + if (missing) + _exit(124); + printf("# %luK dump precondition: all %lu leaves swapped\n", + ps / 1024, SWAPPED_SIZE / ps); + } + /* The file VMAs remain unfaulted, so dumping must retrieve file data. */ + raise(SIGABRT); + _exit(123); +} + +static Elf64_Phdr *find_load(unsigned char *data, Elf64_Ehdr *elf, + unsigned long addr) +{ + Elf64_Phdr *ph = (void *)(data + elf->e_phoff); + + for (unsigned int i = 0; i < elf->e_phnum; i++) + if (ph[i].p_type == PT_LOAD && addr >= ph[i].p_vaddr && + addr - ph[i].p_vaddr < ph[i].p_memsz) + return ph + i; + return NULL; +} + +static bool bytes_at(unsigned char *data, Elf64_Ehdr *elf, size_t len, + unsigned long addr, size_t count, unsigned char expected) +{ + Elf64_Phdr *p = find_load(data, elf, addr); + unsigned long off; + + if (!p || addr - p->p_vaddr + count > p->p_filesz) + return false; + off = p->p_offset + addr - p->p_vaddr; + if (off > len || count > len - off) + return false; + for (size_t i = 0; i < count; i++) + if (data[off + i] != expected) + return false; + return true; +} + +static void inspect(pid_t pid, unsigned long ps, bool headers) +{ + char path[64]; + struct stat st; + unsigned char *data; + Elf64_Ehdr *elf; + Elf64_Phdr *ph, *p; + bool geometry = true, aux = false; + unsigned int file = 0; + unsigned long vdso = 0; + int fd; + + snprintf(path, sizeof(path), "/core.%d", pid); + fd = open(path, O_RDONLY); + if (fd < 0 || fstat(fd, &st) || st.st_size < (off_t)sizeof(*elf)) + ksft_exit_fail_msg("missing/short core %s\n", path); + data = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (data == MAP_FAILED) + die("core read"); + elf = (void *)data; + if (memcmp(elf->e_ident, ELFMAG, SELFMAG) || elf->e_type != ET_CORE || + elf->e_phentsize != sizeof(*ph) || + elf->e_phoff > (uint64_t)st.st_size || + elf->e_phnum * sizeof(*ph) > (uint64_t)st.st_size - elf->e_phoff) + ksft_exit_fail_msg("invalid core ELF\n"); + ph = (void *)(data + elf->e_phoff); + for (unsigned int i = 0; i < elf->e_phnum; i++) { + if (ph[i].p_type == PT_LOAD) { + geometry &= ph[i].p_align == ps && + ph[i].p_filesz <= ph[i].p_memsz && + ph[i].p_offset % ps == ph[i].p_vaddr % ps; + } else if (ph[i].p_type == PT_NOTE) { + size_t at = ph[i].p_offset, end = at + ph[i].p_filesz; + + if (end > (size_t)st.st_size || end < at) + ksft_exit_fail_msg("invalid notes\n"); + while (at + sizeof(Elf64_Nhdr) <= end) { + Elf64_Nhdr *n = (void *)(data + at); + size_t desc = at + sizeof(*n) + + ((n->n_namesz + 3UL) & ~3UL); + uint64_t *words = (void *)(data + desc); + + if (desc > end || n->n_descsz > end - desc) + ksft_exit_fail_msg( + "invalid note length\n"); + if (n->n_type == NT_AUXV) { + for (size_t j = 0; + j + 1 < n->n_descsz / 8; j += 2) { + if (words[j] == AT_PAGESZ) + aux = words[j + 1] == + ps; + else if (words[j] == + AT_SYSINFO_EHDR) + vdso = words[j + 1]; + } + } + if (n->n_type == NT_FILE && n->n_descsz >= 16 && + words[0] <= (n->n_descsz / 8 - 2) / 3 && + words[1] == ps) { + for (size_t j = 0; j < words[0]; j++) { + unsigned long addr = + words[2 + 3 * j]; + unsigned long offset = + words[4 + 3 * j]; + + if (addr == FILEMAP && + offset == (headers ? 0UL : + 1UL)) + file |= 1; + if (addr == FILEMAP + 2 * ps && + offset == 2) + file |= 2; + if (addr == FILEMAP + 8 * ps && + offset == 17) + file |= 4; + } + } + at = desc + ((n->n_descsz + 3UL) & ~3UL); + } + } + } + ksft_test_result(geometry, "%luK core PT_LOAD granule/alignment%s\n", + ps / 1024, headers ? " headers-only" : ""); + ksft_test_result(aux && file == 7, + "%luK core AUXV and NT_FILE offsets%s\n", ps / 1024, + headers ? " headers-only" : ""); + if (headers) { + bool skipped; + + p = find_load(data, elf, FILEMAP + 2 * ps); + skipped = p && !p->p_filesz; + p = find_load(data, elf, FILEMAP + 8 * ps); + ksft_test_result( + skipped && p && !p->p_filesz, + "%luK header filter excludes nonzero quarter/native file offsets\n", + ps / 1024); + p = find_load(data, elf, FILEMAP + 12 * ps); + ksft_test_result( + p && p->p_filesz == ps && + p->p_offset + ps <= (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, SELFMAG), + "%luK header filter recognizes non-executable ELF file\n", + ps / 1024); + p = find_load(data, elf, FILEMAP); + ksft_test_result( + p && p->p_filesz == ps && + p->p_offset + ps <= (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, SELFMAG), + "%luK executable file header limited to one user page\n", + ps / 1024); + } else { + bool anon = true, files = true; + + p = find_load(data, elf, vdso); + ksft_test_result(p && p->p_filesz >= SELFMAG && + p->p_offset + p->p_filesz <= + (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, + SELFMAG), + "%luK core vDSO ELF bytes\n", ps / 1024); + ksft_test_result(bytes_at(data, elf, st.st_size, SWAPPED, + SWAPPED_SIZE, 0xc4), + "%luK core swapped anonymous bytes\n", + ps / 1024); + + for (unsigned int i = 0; i < 4; i++) + anon &= bytes_at(data, elf, st.st_size, ANON + i * ps, + ps, 0x31 + i); + for (unsigned int i = 1; i < 4; i++) + files &= bytes_at(data, elf, st.st_size, + FILEMAP + (i == 1 ? 0 : i * ps), ps, + 0x81 + i); + ksft_test_result( + anon, "%luK core anonymous and PROT_NONE/READ bytes\n", + ps / 1024); + files &= bytes_at(data, elf, st.st_size, FILEMAP + 8 * ps, ps, + 0x81 + 17); + ksft_test_result(files, "%luK core unfaulted file quarters\n", + ps / 1024); + ksft_test_result( + bytes_at(data, elf, st.st_size, SPARSE, ps, 0x65) && + bytes_at(data, elf, st.st_size, SPARSE + ps, + SPARSE_SIZE - ps, 0), + "%luK core sparse and zero-page bytes\n", ps / 1024); + ksft_test_result( + st.st_blocks * 512 < st.st_size / 2, + "%luK core keeps untouched anonymous range sparse\n", + ps / 1024); + p = find_load(data, elf, EXCLUDED); + ksft_test_result(p && !p->p_filesz, + "%luK core DONTDUMP excludes data\n", + ps / 1024); + } + munmap(data, st.st_size); + close(fd); + unlink(path); +} + +int main(int argc, char **argv) +{ + unsigned long native_size = getauxval(AT_PAGESZ); + unsigned long sizes[] = { native_size, 4096, 16384 }; + unsigned int modes = native_size == 65536 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && !strcmp(argv[1], "--crash")) + crash(false, 0); + if (argc == 2 && !strcmp(argv[1], "--headers")) + crash(true, 0); + if (argc == 2 && !strcmp(argv[1], "--limit")) + crash(false, 1); + if (argc == 2 && !strcmp(argv[1], "--below-limit")) + crash(false, -1); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Use only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) && errno != EBUSY) + die("proc"); + if (mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + swapon("/dev/vda", 0)) + die("VM swap setup"); + write_text("/proc/sys/kernel/core_pattern", "/core.%p"); + ksft_print_header(); + ksft_set_plan(16 * modes); + for (unsigned int mode = 0; mode < modes; mode++) { + unsigned long ps = sizes[mode]; + for (unsigned int headers = 0; headers < 4; headers++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + ps == native_size ? 0UL : ps, 0UL, + 0UL, 0UL)) + die("exec size"); + execl("/init", "/init", + headers == 3 ? "--below-limit" : + headers == 2 ? "--limit" : + headers ? "--headers" : + "--crash", + NULL); + die("exec"); + } + if (pid < 0 || waitpid(pid, &status, 0) != pid || + !WIFSIGNALED(status) || + WTERMSIG(status) != SIGABRT || + (headers < 3 && !WCOREDUMP(status))) + ksft_exit_fail_msg( + "child/core failure status=%#x\n", + status); + if (headers >= 2) { + char path[64]; + struct stat st; + bool exists, no_data = false; + + snprintf(path, sizeof(path), "/core.%d", pid); + exists = !stat(path, &st); + if (exists && headers == 2) { + Elf64_Ehdr elf; + Elf64_Phdr ph; + uint64_t data_start = UINT64_MAX; + int fd = open(path, O_RDONLY); + + if (fd < 0 || + pread(fd, &elf, sizeof(elf), 0) != + sizeof(elf)) + die("limited core header"); + for (unsigned int i = 0; + i < elf.e_phnum; i++) { + if (pread(fd, &ph, sizeof(ph), + elf.e_phoff + + i * sizeof(ph)) != + sizeof(ph)) + die("limited core phdr"); + if (ph.p_type == PT_LOAD && + ph.p_offset < data_start) + data_start = + ph.p_offset; + } + close(fd); + /* Sparse note padding is not charged to RLIMIT_CORE. */ + no_data = data_start != UINT64_MAX && + st.st_size <= + (off_t)data_start; + printf("# limited core: bytes=%lld first-data=%llu limit=%lu\n", + (long long)st.st_size, + (unsigned long long)data_start, + ps); + } + ksft_test_result( + headers == 2 ? + exists && st.st_size > 0 && + no_data : + !exists && errno == ENOENT, + "%luK core RLIMIT_CORE %s one user page\n", + ps / 1024, + headers == 2 ? "at" : "below"); + unlink(path); + } else { + inspect(pid, ps, headers); + } + } + } + ksft_print_cnts(); + printf("%s - core fixture complete\n", + ksft_get_fail_cnt() ? "not ok" : "ok"); + reboot(RB_POWER_OFF); + return ksft_get_fail_cnt() ? 1 : 0; +} diff --git a/tools/testing/selftests/arm64/user4k/cpu-granule-init.c b/tools/testing/selftests/arm64/user4k/cpu-granule-init.c new file mode 100644 index 0000000000000..1e983f91f74c7 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/cpu-granule-init.c @@ -0,0 +1,187 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static int pin_cpu(pid_t pid, unsigned int cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(pid, sizeof(set), &set); +} +static int online_cpu1(void) +{ + char c = '?'; + int fd = open("/sys/devices/system/cpu/cpu1/online", O_RDONLY); + if (fd >= 0) { + if (read(fd, &c, 1) != 1) + c = '?'; + close(fd); + } + return c == '0' ? 0 : c == '1' ? 1 : -1; +} +static int worker(int ready, int command, unsigned long expected) +{ + unsigned char *p; + unsigned long ps = getauxval(AT_PAGESZ); + char cpu; + int ok = getppid() == 1 && ps == expected; + + p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 2; + for (unsigned long i = 0; i < 4 * ps; i++) + p[i] = i ^ (i >> 12); + if (write(ready, "r", 1) != 1 || read(command, &cpu, 1) != 1) + return 3; + ok &= sched_getcpu() == cpu - '0'; + for (unsigned int round = 0; round < 128; round++) { + for (unsigned long i = 0; i < 4 * ps; i++) + if (p[i] != (unsigned char)(i ^ (i >> 12))) + ok = 0; + sched_yield(); + } + munmap(p, 4 * ps); + return ok ? 0 : 4; +} +static int compat_sample(unsigned long expected) +{ + uint64_t data[4] = { 0 }; + int pipefd[2], status; + pid_t child; + ssize_t size; + + if (pipe(pipefd)) + return 0; + child = fork(); + if (!child) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + execl("/compat-aslr", "/compat-aslr", "--sample", fd, NULL); + _exit(2); + } + close(pipefd[1]); + size = read(pipefd[0], data, sizeof(data)); + close(pipefd[0]); + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && + size == sizeof(data) && data[0] == expected; +} +int main(int argc, char **argv) +{ + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10)); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, + "Dedicated two-CPU disposable VM with maxcpus=1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) { + perror("mount"); + failed++; + goto done; + } + unsigned long native = getauxval(AT_PAGESZ); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + result(!prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL), + "native exec default restored"); + char cmdline[4096] = { 0 }; + int fd = open("/proc/cmdline", O_RDONLY); + if (fd < 0 || read(fd, cmdline, sizeof(cmdline) - 1) < 0) { + failed++; + goto done; + } + close(fd); + int reject = has4k && strstr(cmdline, "arm64.user4k_deny_late_cpu=1"); + result(online_cpu1() == 0, "CPU1 starts offline"); + if (online_cpu1() != 0) + goto done; + int ready[2], command[2]; + if (pipe(ready) || pipe(command)) { + failed++; + goto done; + } + pid_t child = fork(); + if (!child) { + char ready_fd[32], command_fd[32], ps[32]; + close(ready[0]); + close(command[1]); + snprintf(ready_fd, sizeof(ready_fd), "%d", ready[1]); + snprintf(command_fd, sizeof(command_fd), "%d", command[0]); + snprintf(ps, sizeof(ps), "%lu", has4k ? 4096UL : native); + if (prctl(SET_EXEC_PAGE_SIZE, has4k ? 4096UL : 0UL, 0UL, 0UL, + 0UL)) + _exit(5); + execl("/init", "/init", "--worker", ready_fd, command_fd, ps, + NULL); + _exit(6); + } + close(ready[1]); + close(command[0]); + if (child < 0) { + failed++; + goto done; + } + char token; + result(read(ready[0], &token, 1) == 1 && token == 'r', + "worker has live mappings before hotplug"); + close(ready[0]); + fd = open("/sys/devices/system/cpu/cpu1/online", O_WRONLY); + errno = 0; + ssize_t written = fd < 0 ? -1 : write(fd, "1", 1); + int saved_errno = errno; + if (fd >= 0) + close(fd); + int online = online_cpu1(); + printf("# has4k=%d expected-reject=%d write=%zd errno=%d online=%d\n", + has4k, reject, written, saved_errno, online); + result(reject ? written < 0 && online == 0 : + written == 1 && online == 1, + "late CPU admission matches advertised user granule"); + unsigned int cpu = online == 1 ? 1 : 0; + result(!pin_cpu(child, cpu), "worker affinity targets admitted CPU"); + token = '0' + cpu; + result(write(command[1], &token, 1) == 1, + "worker resumes after CPU admission"); + close(command[1]); + int status; + result(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status), + "live mapping bytes survive and worker runs on expected CPU"); + result(!pin_cpu(0, cpu), "launcher affinity targets admitted CPU"); + result(compat_sample(has4k ? 4096 : native), + "AArch32 exec retains selected or native-fallback granule"); +done: + printf("CPU GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c b/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c new file mode 100644 index 0000000000000..ecf682a22f732 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static int pin_cpu(pid_t pid, unsigned int cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(pid, sizeof(set), &set); +} +static int online_cpu1(void) +{ + char c = '?'; + int fd = open("/sys/devices/system/cpu/cpu1/online", O_RDONLY); + if (fd >= 0) { + if (read(fd, &c, 1) != 1) + c = '?'; + close(fd); + } + return c == '0' ? 0 : c == '1' ? 1 : -1; +} +static int worker(int ready, int command, unsigned long expected) +{ + unsigned char *p; + unsigned long ps = getauxval(AT_PAGESZ); + char cpu; + int ok = getppid() == 1 && ps == expected; + + p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 2; + for (unsigned long i = 0; i < 4 * ps; i++) + p[i] = i ^ (i >> 12); + if (write(ready, "r", 1) != 1 || read(command, &cpu, 1) != 1) + return 3; + ok &= sched_getcpu() == cpu - '0'; + for (unsigned int round = 0; round < 128; round++) { + for (unsigned long i = 0; i < 4 * ps; i++) + if (p[i] != (unsigned char)(i ^ (i >> 12))) + ok = 0; + sched_yield(); + } + munmap(p, 4 * ps); + return ok ? 0 : 4; +} +static int compat_sample(unsigned long expected) +{ + uint64_t data[4] = { 0 }; + int pipefd[2], status; + pid_t child; + ssize_t size; + + if (pipe(pipefd)) + return 0; + child = fork(); + if (!child) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + execl("/compat-aslr", "/compat-aslr", "--sample", fd, NULL); + _exit(2); + } + close(pipefd[1]); + size = read(pipefd[0], data, sizeof(data)); + close(pipefd[0]); + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && + size == sizeof(data) && data[0] == expected; +} +int main(int argc, char **argv) +{ + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10)); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, + "Dedicated two-CPU disposable VM with maxcpus=1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) { + perror("mount"); + failed++; + goto done; + } + unsigned long native = getauxval(AT_PAGESZ); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + int has16k = native > 16384 && + !prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL); + result(!prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL), + "native exec default restored"); + char cmdline[4096] = { 0 }; + int fd = open("/proc/cmdline", O_RDONLY); + if (fd < 0 || read(fd, cmdline, sizeof(cmdline) - 1) < 0) { + failed++; + goto done; + } + close(fd); + int reject = has16k && strstr(cmdline, "arm64.user16k_deny_late_cpu=1"); + result(online_cpu1() == 0, "CPU1 starts offline"); + if (online_cpu1() != 0) + goto done; + int ready[2], command[2]; + if (pipe(ready) || pipe(command)) { + failed++; + goto done; + } + pid_t child = fork(); + if (!child) { + char ready_fd[32], command_fd[32], ps[32]; + close(ready[0]); + close(command[1]); + snprintf(ready_fd, sizeof(ready_fd), "%d", ready[1]); + snprintf(command_fd, sizeof(command_fd), "%d", command[0]); + snprintf(ps, sizeof(ps), "%lu", has16k ? 16384UL : native); + if (prctl(SET_EXEC_PAGE_SIZE, has16k ? 16384UL : 0UL, 0UL, 0UL, + 0UL)) + _exit(5); + execl("/init", "/init", "--worker", ready_fd, command_fd, ps, + NULL); + _exit(6); + } + close(ready[1]); + close(command[0]); + if (child < 0) { + failed++; + goto done; + } + char token; + result(read(ready[0], &token, 1) == 1 && token == 'r', + "worker has live mappings before hotplug"); + close(ready[0]); + fd = open("/sys/devices/system/cpu/cpu1/online", O_WRONLY); + errno = 0; + ssize_t written = fd < 0 ? -1 : write(fd, "1", 1); + int saved_errno = errno; + if (fd >= 0) + close(fd); + int online = online_cpu1(); + printf("# has16k=%d expected-reject=%d write=%zd errno=%d online=%d\n", + has16k, reject, written, saved_errno, online); + result(reject ? written < 0 && online == 0 : + written == 1 && online == 1, + "late CPU admission matches advertised user granule"); + unsigned int cpu = online == 1 ? 1 : 0; + result(!pin_cpu(child, cpu), "worker affinity targets admitted CPU"); + token = '0' + cpu; + result(write(command[1], &token, 1) == 1, + "worker resumes after CPU admission"); + close(command[1]); + int status; + result(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status), + "live mapping bytes survive and worker runs on expected CPU"); + result(!pin_cpu(0, cpu), "launcher affinity targets admitted CPU"); + if (has16k) + result(!prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL), + "16K request staged before compat exec"); + result(compat_sample(has4k ? 4096 : native), + "AArch32 exec selects4K or native despite pending16K"); +done: + printf("CPU GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/dio-fixture-init.c b/tools/testing/selftests/arm64/user4k/dio-fixture-init.c new file mode 100644 index 0000000000000..36ebec4350c76 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/dio-fixture-init.c @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static bool use_file; +static unsigned char pattern(size_t i) +{ + return (i * 17 + (i >> 12) * 29 + 71) & 255; +} +static void fill(unsigned char *p, size_t len) +{ + for (size_t i = 0; i < len; i++) + p[i] = pattern(i); +} +static bool check(const unsigned char *p, size_t len, size_t off) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != pattern(i + off)) { + ksft_print_msg("mismatch at %zu: %u vs %u\n", i + off, + p[i], pattern(i + off)); + return false; + } + return true; +} +static void report(bool ok, size_t ps, const char *name) +{ + ksft_test_result(ok, "%zuK %s (errno=%d)\n", ps / 1024, name, errno); +} +static bool aio_transfer(int fd, void *p, size_t len, off_t off, bool write) +{ + aio_context_t ctx = 0; + struct iocb cb = { .aio_lio_opcode = write ? IOCB_CMD_PWRITE : + IOCB_CMD_PREAD, + .aio_fildes = fd, + .aio_buf = (uintptr_t)p, + .aio_nbytes = len, + .aio_offset = off }; + struct iocb *ptr = &cb; + struct io_event ev; + struct timespec timeout = { .tv_sec = 10 }; + bool ok; + + if (syscall(__NR_io_setup, 8, &ctx)) + return false; + ok = syscall(__NR_io_submit, ctx, 1, &ptr) == 1; + if (ok) + ok = syscall(__NR_io_getevents, ctx, 1, 1, &ev, &timeout) == + 1 && + ev.res == (long)len && !ev.res2; + ok &= !syscall(__NR_io_destroy, ctx); + return ok; +} +static int exercise(void) +{ + size_t ps = getauxval(AT_PAGESZ), len = 2 * 1024 * 1024; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *q = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + struct iovec vec[4]; + int fd = open(use_file ? "/fs/test" : "/dev/user4k-test", + O_RDWR | O_DIRECT | (use_file ? O_CREAT | O_TRUNC : 0), + 0600), + memfd; + bool ok; + ssize_t ret; + + ksft_print_header(); + ksft_set_plan(10); + ksft_print_msg("backing=%s\n", use_file ? "ext4" : "raw block"); + if ((ps != 4096 && ps != 16384) || p == MAP_FAILED || q == MAP_FAILED || + fd < 0) + ksft_exit_fail_msg("setup failed errno=%d\n", errno); + if (use_file && ftruncate(fd, 32 * 1024 * 1024)) + ksft_exit_fail_msg("ftruncate failed\n"); + fill(p, len); + memset(q, 0, len); + ok = pwrite(fd, p + ps, 3 * ps, 0) == (ssize_t)(3 * ps) && + pread(fd, q + ps, 3 * ps, 0) == (ssize_t)(3 * ps) && + check(q + ps, 3 * ps, ps); + report(ok, ps, "O_DIRECT exact user offsets and bytes"); + + vec[0] = (struct iovec){ p + 512, ps - 512 }; + vec[1] = (struct iovec){ NULL, 0 }; + vec[2] = (struct iovec){ p + 5 * ps, ps + 512 }; + vec[3] = (struct iovec){ p + 11 * ps + 512, 512 }; + ok = pwritev(fd, vec, 4, 65536) == (ssize_t)(2 * ps + 512); + memset(q, 0, 3 * ps); + ok &= pread(fd, q, 2 * ps + 512, 65536) == (ssize_t)(2 * ps + 512); + ok &= check(q, ps - 512, 512) && + check(q + ps - 512, ps + 512, 5 * ps) && + check(q + 2 * ps, 512, 11 * ps + 512); + report(ok, ps, "fragmented vectors and empty segment"); + + ok = pwrite(fd, p, len, 4 * 1024 * 1024) == (ssize_t)len; + memset(q, 0, len); + ok &= pread(fd, q, len, 4 * 1024 * 1024) == (ssize_t)len && + check(q, len, 0); + report(ok, ps, "multi-bio two-megabyte transfer"); + + ok = !mprotect(p + ps, ps, PROT_NONE); + ret = pwrite(fd, p, 2 * ps, 8 * 1024 * 1024); + ok &= use_file ? ret == -1 && errno == EFAULT : ret == (ssize_t)ps; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE); + if (use_file) + ok &= pwrite(fd, p, ps, 8 * 1024 * 1024) == (ssize_t)ps; + memset(q, 0, ps); + ok &= pread(fd, q, ps, 8 * 1024 * 1024) == (ssize_t)ps && + check(q, ps, 0); + report(ok, ps, "source fault and subsequent I/O recovery"); + + ok = !mprotect(q + ps, ps, PROT_NONE); + ret = pread(fd, q, 2 * ps, 4 * 1024 * 1024); + ok &= (use_file ? ret == -1 && errno == EFAULT : ret == (ssize_t)ps) && + check(q, ps, 0); + ok &= !mprotect(q + ps, ps, PROT_READ | PROT_WRITE); + report(ok, ps, "partial destination fault"); + + ok = !mprotect(p + ps, ps, PROT_NONE); + /* Only 256 accessible bytes: none can form a 512-byte device block. */ + ret = pwrite(fd, p + ps - 256, 512, 10 * 1024 * 1024); + ok &= ret == -1 && errno == EFAULT; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE); + report(ok, ps, "alignment trimming releases all short-fragment pins"); + + memfd = memfd_create("dio-shared", 0); + ok = memfd >= 0 && !ftruncate(memfd, 4 * ps); + unsigned char *shared = ok ? mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, 0) : + MAP_FAILED; + ok &= shared != MAP_FAILED; + if (ok) { + memset(shared, 0xa7, 4 * ps); + ok = pread(fd, shared + ps, 2 * ps, 4 * 1024 * 1024) == + (ssize_t)(2 * ps) && + !msync(shared, 4 * ps, MS_SYNC) && + pread(memfd, q, 4 * ps, 0) == (ssize_t)(4 * ps); + ok &= check(q + ps, 2 * ps, 0); + for (size_t i = 0; i < ps; i++) + ok &= q[i] == 0xa7 && q[3 * ps + i] == 0xa7; + munmap(shared, 4 * ps); + } + if (memfd >= 0) + close(memfd); + report(ok, ps, "shared-file destination dirtying and adjacent guards"); + + ok = aio_transfer(fd, p + ps, 7 * ps, 12 * 1024 * 1024, true); + memset(q, 0, 8 * ps); + ok &= aio_transfer(fd, q + ps, 7 * ps, 12 * 1024 * 1024, false) && + check(q + ps, 7 * ps, ps); + report(ok, ps, "asynchronous AIO read/write completion"); + + memset(q, 0x96, 8 * ps); + pid_t child = fork(); + int status = 0; + if (!child) { + _exit(aio_transfer(fd, q + ps, 7 * ps, 12 * 1024 * 1024, + false) && + check(q + ps, 7 * ps, ps) ? + 0 : + 1); + } + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + for (size_t i = 0; i < 8 * ps; i++) + ok &= q[i] == 0x96; + report(ok, ps, "writable-pin COW after fork"); + + ok = true; + for (unsigned i = 0; i < 128 && ok; i++) { + ok = !madvise(q + ps, ps, MADV_DONTNEED) && + pread(fd, q + ps, ps, 4 * 1024 * 1024) == (ssize_t)ps && + check(q + ps, ps, 0); + ok &= !mprotect(q + 2 * ps, ps, PROT_NONE); + ok &= pread(fd, q + 2 * ps, ps, 4 * 1024 * 1024) == -1 && + errno == EFAULT; + ok &= !mprotect(q + 2 * ps, ps, PROT_READ | PROT_WRITE); + } + report(ok, ps, "128 reuse and failed-pin cleanup cycles"); + close(fd); + munmap(p, len); + munmap(q, len); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +int main(int argc, char **argv) +{ + bool ok = true; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && + (!strcmp(argv[1], "--test") || !strcmp(argv[1], "--file")) && + getppid() == 1) { + use_file = !strcmp(argv[1], "--file"); + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + for (unsigned disk = 0; disk < 2; disk++) { + char path[80], serial[64]; + unsigned maj, min; + snprintf(path, sizeof(path), "/sys/class/block/vd%c/serial", + 'a' + disk); + FILE *dev = fopen(path, "r"); + if (!dev || fscanf(dev, "%63s", serial) != 1) + return 4; + fclose(dev); + snprintf(path, sizeof(path), "/sys/class/block/vd%c/dev", + 'a' + disk); + dev = fopen(path, "r"); + if (!dev || fscanf(dev, "%u:%u", &maj, &min) != 2) + return 4; + fclose(dev); + const char *node = + !strcmp(serial, "au4k-raw") ? "/dev/user4k-test" : + !strcmp(serial, "au4k-ext4") ? "/dev/user4k-fs" : + NULL; + if (!node || mknod(node, S_IFBLK | 0600, makedev(maj, min))) + return 5; + } + if (mount("/dev/user4k-fs", "/fs", "ext4", 0, NULL)) + return 7; + for (unsigned mode = 0; mode < 4; mode++) { + unsigned small = mode & 1; + pid_t pid = fork(); + int status = 0; + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + small ? 4096UL : 0UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", mode >= 2 ? "--file" : "--test", + NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + ok &= !umount("/fs"); + printf("%s - direct I/O fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/exec-fixture-init.c b/tools/testing/selftests/arm64/user4k/exec-fixture-init.c new file mode 100644 index 0000000000000..8a390d5df3ddc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/exec-fixture-init.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* PID-1 runner for the experimental one-shot exec ABI in a disposable VM. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include + +static int exec_size(unsigned long size) +{ + return prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL); +} + +static int pending_size(void) +{ + return prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); +} + +static int run(const char *path, const char *arg, unsigned long size) +{ + pid_t pid = fork(), got; + int status = 0; + + if (!pid) { + if (size && exec_size(size)) { + perror("set exec page size"); + _exit(125); + } + execl(path, path, arg, NULL); + perror(path); + _exit(126); + } + if (pid < 0) + return -1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + printf("# child %s %s: wait=%d status=%#x\n", path, arg ? arg : "", got, + status); + return got == pid && WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +static int argument_exec(unsigned long size) +{ + pid_t pid = fork(), got; + int status = 0; + + if (!pid) { + char *payload = malloc(140000), expected[16]; + unsigned int i; + + if (!payload || exec_size(size)) + _exit(120); + for (i = 0; i < 139999; i++) + payload[i] = 'a' + i % 23; + payload[139999] = 0; + if (size == 4096) { + execl("/init", "/init", "--too-long", payload, NULL); + if (errno != E2BIG || pending_size() != 4096) + _exit(121); + } + payload[120000] = 0; + if (setenv("ARG_PAYLOAD", payload, 1)) + _exit(122); + snprintf(expected, sizeof(expected), "%lu", + size ? size : 16384UL); + execl("/init", "/init", "--args-check", expected, payload, + NULL); + perror("argument exec"); + _exit(123); + } + if (pid < 0) + return -1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + printf("# large arguments from %lu to %lu: status=%#x\n", + getauxval(AT_PAGESZ), size ? size : 16384UL, status); + return got == pid && WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +int main(int argc, char **argv) +{ + bool ok; + int failures = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--args-check")) { + const char *env = getenv("ARG_PAYLOAD"); + unsigned int i; + + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10) || + pending_size() != 0 || strlen(argv[3]) != 120000 || !env || + strlen(env) != 120000) + return 1; + for (i = 0; i < 120000; i++) + if (argv[3][i] != 'a' + i % 23 || env[i] != argv[3][i]) + return 2; + return 0; + } + if (argc == 2 && !strcmp(argv[1], "--args-from-4k")) { + if (getauxval(AT_PAGESZ) != 4096) + return 1; + return argument_exec(4096) || argument_exec(0); + } + if (argc == 2 && !strcmp(argv[1], "--exec-cycle")) { + if (getauxval(AT_PAGESZ) != 4096 || pending_size() != 0) + return 1; + /* A successful exec consumes the request; ordinary exec is native. */ + execl("/init", "/init", "--native-after-4k", NULL); + return 2; + } + if (argc == 2 && !strcmp(argv[1], "--native-after-4k")) + return getauxval(AT_PAGESZ) != 16384 || pending_size() != 0; + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as /init in a disposable VM.\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) && errno != EBUSY) + return 3; + printf("# exec fixture: native AT_PAGESZ=%lu\n", getauxval(AT_PAGESZ)); + failures += run("/page-contract", NULL, 0) != 0; + ok = pending_size() == 0; + errno = 0; + ok &= exec_size(8192) == -1 && errno == EINVAL; + ok &= prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096UL, 1UL, 0UL, 0UL) == -1; + ok &= exec_size(4096) == 0 && pending_size() == 4096; + execl("/does-not-exist", "/does-not-exist", NULL); + ok &= errno == ENOENT && pending_size() == 4096; + ok &= getauxval(AT_PAGESZ) == 16384; + printf("%s - request validation and failed exec preserve live geometry\n", + ok ? "ok" : "not ok"); + failures += !ok; + /* fork inherits the pending request; successful child exec consumes it. */ + failures += run("/page-contract-4k", "--expect-4k", 0) != 0; + ok = pending_size() == 4096 && exec_size(0) == 0 && pending_size() == 0; + printf("%s - child exec leaves parent request intact; reset works\n", + ok ? "ok" : "not ok"); + failures += !ok; + failures += run("/page-contract-dynamic-4k", "--expect-4k", 4096) != 0; + failures += run("/init", "--exec-cycle", 4096) != 0; + failures += run("/proc-memory", NULL, 0) != 0; + failures += run("/proc-memory", "--expect-4k", 4096) != 0; + failures += run("/proc-pages", NULL, 0) != 0; + failures += run("/proc-pages", "--expect-4k", 4096) != 0; + failures += run("/advice", NULL, 0) != 0; + failures += run("/advice", "--expect-4k", 4096) != 0; + failures += run("/memory-interfaces", NULL, 0) != 0; + failures += run("/memory-interfaces", "--expect-4k", 4096) != 0; + failures += run("/mixed-process", NULL, 0) != 0; + failures += run("/mixed-process", NULL, 4096) != 0; + failures += argument_exec(4096) != 0; + failures += run("/init", "--args-from-4k", 4096) != 0; + failures += run("/page-contract", NULL, 0) != 0; + printf("%s - exec fixture complete: failures=%d\n", + failures ? "not ok" : "ok", failures); + reboot(RB_POWER_OFF); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/arm64/user4k/exec-policy-init.c b/tools/testing/selftests/arm64/user4k/exec-policy-init.c new file mode 100644 index 0000000000000..02fd3f9552237 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/exec-policy-init.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable guest PID1: per-thread one-shot and inherited exec preferences. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define REQUIRE(x) \ + do { \ + if (!(x)) { \ + fprintf(stderr, "FAIL line %d: %s errno=%d\n", \ + __LINE__, #x, errno); \ + exit(1); \ + } \ + } while (0) + +static int policy(int op, unsigned long value) +{ + return prctl(op, value, 0UL, 0UL, 0UL); +} + +static void state(unsigned long pages, unsigned long preferred, int pending) +{ + REQUIRE(getauxval(AT_PAGESZ) == pages); + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 0) == pending); +} + +static void chain(int stage, unsigned long native, unsigned long preferred) +{ + char s[32], n[32], p[32]; + + snprintf(s, sizeof(s), "%d", stage); + snprintf(n, sizeof(n), "%lu", native); + snprintf(p, sizeof(p), "%lu", preferred); + execl("/init", "/init", "--chain", s, n, p, NULL); + REQUIRE(0); +} + +static void *thread_policy(void *arg) +{ + unsigned long preferred = *(unsigned long *)arg; + + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 0)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 0)); + state(getauxval(AT_PAGESZ), 0, 0); + return NULL; +} + +static void *thread_exec(void *arg) +{ + unsigned long *sizes = arg; + + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, sizes[1])); + chain(0, sizes[0], sizes[1]); + return NULL; +} + +static int wait_child(pid_t pid) +{ + int status; + + REQUIRE(pid > 0); + REQUIRE(waitpid(pid, &status, 0) == pid); + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +static int run_preference(unsigned long native, unsigned long preferred) +{ + pid_t pid = fork(); + + if (!pid) { + pthread_t thread; + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, preferred)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, preferred)); + REQUIRE(!pthread_create(&thread, NULL, thread_policy, + &preferred)); + REQUIRE(!pthread_join(thread, NULL)); + state(native, preferred, preferred); + /* Rejected values and reserved arguments must preserve both policies. */ + REQUIRE(policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 8192) == -1 && + errno == EINVAL); + REQUIRE(policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 8192) == -1 && + errno == EINVAL); + int ops[] = { PR_AURORA_SET_DEFAULT_PAGE_SIZE, + PR_AURORA_GET_DEFAULT_PAGE_SIZE, + PR_AURORA_SET_EXEC_PAGE_SIZE, + PR_AURORA_GET_EXEC_PAGE_SIZE }; + for (unsigned int i = 0; i < sizeof(ops) / sizeof(ops[0]); + i++) { + REQUIRE(prctl(ops[i], 0UL, 1UL, 0UL, 0UL) == -1 && + errno == EINVAL); + REQUIRE(prctl(ops[i], 0UL, 0UL, 1UL, 0UL) == -1 && + errno == EINVAL); + REQUIRE(prctl(ops[i], 0UL, 0UL, 0UL, 1UL) == -1 && + errno == EINVAL); + } + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 1) == -1 && + errno == EINVAL); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 1) == -1 && + errno == EINVAL); + execl("/does-not-exist", "/does-not-exist", NULL); + REQUIRE(errno == ENOENT); + state(native, preferred, preferred); + chain(0, native, preferred); + } + return wait_child(pid); +} + +static int compat(unsigned long native, unsigned long preferred, int no4k) +{ + pid_t pid = fork(); + + if (!pid) { + char n[32], p[32], e[32]; + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, preferred)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, native)); + snprintf(n, sizeof(n), "%lu", native); + snprintf(p, sizeof(p), "%lu", preferred); + snprintf(e, sizeof(e), "%lu", no4k ? native : 4096UL); + execl("/compat-policy", "/compat-policy", n, p, e, NULL); + REQUIRE(0); + } + return wait_child(pid); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--chain")) { + int stage = atoi(argv[2]); + unsigned long n = strtoul(argv[3], NULL, 10); + unsigned long p = strtoul(argv[4], NULL, 10); + + state(stage == 2 || stage == 5 ? n : p, stage < 4 ? p : 0, 0); + if (stage == 5) + return 0; + if (stage == 1) { + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 0)); + state(p, p, n); + } + if (stage == 3) { + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 0)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, p)); + } + chain(stage + 1, n, p); + } + if (argc == 4 && !strcmp(argv[1], "--compat-return")) { + unsigned long n = strtoul(argv[2], NULL, 10); + unsigned long p = strtoul(argv[3], NULL, 10); + state(p ? p : n, p, 0); + return 0; + } + REQUIRE(getpid() == 1); + REQUIRE(!mount("proc", "/proc", "proc", 0, NULL)); + state(native, 0, 0); + int no4k = argc == 2 && !strcmp(argv[1], "--no4k"); + unsigned long preferences[] = { 4096, 16384 }; + for (unsigned int i = 0; i < 2; i++) { + unsigned long p = preferences[i]; + if (p >= native || (no4k && p == 4096)) + continue; + int ok = run_preference(native, p); + printf("%s - exec policy native=%lu alternative=%lu\n", + ok ? "ok" : "not ok", native, p); + failed |= !ok; + pid_t pid = fork(); + if (!pid) { + pthread_t thread; + unsigned long sizes[] = { native, p }; + REQUIRE(!pthread_create(&thread, NULL, thread_exec, + sizes)); + for (;;) + pause(); + } + ok = wait_child(pid); + printf("%s - non-leader exec alternative=%lu\n", + ok ? "ok" : "not ok", p); + failed |= !ok; + ok = compat(native, p, no4k); + printf("%s - compat policy alternative=%lu no4k=%d\n", + ok ? "ok" : "not ok", p, no4k); + failed |= !ok; + } + if (no4k) { + int ok = policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 4096) == -1 && + errno == EOPNOTSUPP; + ok &= policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096) == -1 && + errno == EOPNOTSUPP; + ok &= compat(native, 0, 1); + printf("%s - no4k preserves native compat restrictions\n", + ok ? "ok" : "not ok"); + failed |= !ok; + } + state(native, 0, 0); + printf("EXEC POLICY %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + return failed; +} diff --git a/tools/testing/selftests/arm64/user4k/fex-contract-init.c b/tools/testing/selftests/arm64/user4k/fex-contract-init.c new file mode 100644 index 0000000000000..3273af30eea7f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-contract-init.c @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +int main(void) +{ + if (getpid() != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + int failed = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + failed = 1; + goto done; + } + pid_t child = fork(); + if (!child) { + setenv("HOME", "/root", 1); + setenv("PATH", "/usr/bin:/bin", 1); + setenv("XDG_RUNTIME_DIR", "/run", 1); + setenv("FEX_ROOTFS", "/", 1); + setenv("LD_PRELOAD", "/usr/lib/libfex-abi.so", 1); + if (prctl(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 4096UL, 0UL, 0UL, + 0UL) || + prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/usr/bin/FEXInterpreter", "/usr/bin/FEXInterpreter", + "/x86-contract", NULL); + perror("FEX exec"); + _exit(4); + } + int status = 0; + int ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - FEX 4K native-process / x86 workload status=%#x\n", + ok ? "ok" : "not ok", status); + failed |= !ok; +done: + printf("GRANULE FEX RUNTIME %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/fex-native-abi.c b/tools/testing/selftests/arm64/user4k/fex-native-abi.c new file mode 100644 index 0000000000000..86ccb6e389d9d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-native-abi.c @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +__attribute__((constructor)) static void check_abi(void) +{ + char path[1024]; + ssize_t n = readlink("/proc/self/exe", path, sizeof(path) - 1); + if (n < 0) + return; + path[n] = 0; + if (!strstr(path, "FEXInterpreter")) + return; + unsigned long ps = getauxval(AT_PAGESZ); + dprintf(2, "FEX NATIVE HOST PROCESS PAGE SIZE %lu\n", ps); + if (ps != 4096) + _exit(90); +} diff --git a/tools/testing/selftests/arm64/user4k/fex-x86-contract.c b/tools/testing/selftests/arm64/user4k/fex-x86-contract.c new file mode 100644 index 0000000000000..047a3c9d28ddb --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-x86-contract.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0-only +typedef unsigned long size_t; +static long sc(long n, long a, long b, long c, long d, long e, long f) +{ + register long r10 __asm__("r10") = d, r8 __asm__("r8") = e, + r9 __asm__("r9") = f; + long r; + __asm__ volatile("syscall" + : "=a"(r) + : "a"(n), "D"(a), "S"(b), "d"(c), "r"(r10), "r"(r8), + "r"(r9) + : "rcx", "r11", "memory"); + return r; +} +static void say(const char *s, size_t n) +{ + sc(1, 1, (long)s, n, 0, 0, 0); +} +#define SAY(s) say(s, sizeof(s) - 1) +static void quit(int n) +{ + sc(231, n, 0, 0, 0, 0, 0); + for (;;) + ; +} +static int check(unsigned char *p, size_t size) +{ + for (size_t i = 0; i < size; i++) + if (p[i] != (unsigned char)(i ^ (i >> 12))) + return 0; + return 1; +} +long run(unsigned long *sp) +{ + unsigned long argc = *sp++, pagesz = 0; + sp += argc + 1; + unsigned long *env = sp; + while (*sp) + sp++; + sp++; + while (*sp) { + if (sp[0] == 6) + pagesz = sp[1]; + sp += 2; + } + if (pagesz != 4096) + return 10; + SAY("ok - FEX x86 guest 4K auxv\n"); + unsigned char *p = (void *)sc(9, 0, 12288, 3, 0x22, -1, 0); + if ((long)p < 0) + return 11; + for (size_t i = 0; i < 12288; i++) + p[i] = (unsigned char)(i ^ (i >> 12)); + if (sc(10, (long)p + 4096, 4096, 0, 0, 0, 0)) + return 12; + long child = sc(57, 0, 0, 0, 0, 0, 0); + int status = 0; + if (!child) { + *(volatile unsigned char *)(p + 4096) = 1; + quit(13); + } + if (child < 0 || sc(61, child, (long)&status, 0, 0, 0, 0) != child || + (status & 127) != 11) + return 14; + if (sc(10, (long)p + 4096, 4096, 3, 0, 0, 0) || !check(p, 12288)) + return 15; + SAY("ok - FEX x86 independent 4K protection fault\n"); + child = sc(57, 0, 0, 0, 0, 0, 0); + if (!child) { + p[4096] = 0x71; + quit(0); + } + if (child < 0 || sc(61, child, (long)&status, 0, 0, 0, 0) != child || + status || !check(p, 12288)) + return 16; + SAY("ok - FEX x86 fork COW\n"); + p = (void *)sc(25, (long)p, 12288, 16384, 1, 0, 0); + if ((long)p < 0 || !check(p, 12288)) + return 17; + for (size_t i = 12288; i < 16384; i++) + if (p[i]) + return 18; + SAY("ok - FEX x86 mremap preserve and zero extension\n"); + long fd = sc(319, (long)"fex-shared", 0, 0, 0, 0, 0); + if (fd < 0 || sc(77, fd, 12288, 0, 0, 0, 0)) + return 19; + unsigned char *a = (void *)sc(9, 0, 4096, 3, 1, fd, 4096), + *b = (void *)sc(9, 0, 4096, 3, 1, fd, 4096); + if ((long)a < 0 || (long)b < 0) + return 20; + a[37] = 0x65; + if (b[37] != 0x65) + return 21; + SAY("ok - FEX x86 memfd mapping at 4K file offset\n"); + unsigned char *code = (void *)sc(9, 0, 4096, 3, 0x22, -1, 0); + if ((long)code < 0) + return 22; + code[0] = 0xb8; + code[1] = 42; + code[2] = 0; + code[3] = 0; + code[4] = 0; + code[5] = 0xc3; + if (sc(10, (long)code, 4096, 5, 0, 0, 0) || + ((int (*)(void))code)() != 42) + return 23; + if (sc(10, (long)code, 4096, 3, 0, 0, 0)) + return 24; + code[1] = 99; + if (sc(10, (long)code, 4096, 5, 0, 0, 0) || + ((int (*)(void))code)() != 99) + return 25; + SAY("ok - FEX x86 JIT permission transition and code replacement\n"); + unsigned char random[32]; + if (sc(318, (long)random, 32, 0, 0, 0, 0) != 32) + return 26; + if (argc == 1) { + child = sc(57, 0, 0, 0, 0, 0, 0); + if (!child) { + char *args[] = { "/x86-contract", "--exec-child", 0 }; + sc(59, (long)args[0], (long)args, (long)env, 0, 0, 0); + quit(28); + } + if (child < 0 || + sc(61, child, (long)&status, 0, 0, 0, 0) != child || status) + return 27; + SAY("ok - FEX x86 fork and re-exec with 4K mappings\n"); + } + SAY("FEX X86 4K CONTRACT PASS\n"); + return 0; +} +__asm__(".global _start\n_start:\nmov %rsp,%rdi\nand $-16,%rsp\ncall run\nmov %rax,%rdi\nmov $231,%rax\nsyscall\n"); diff --git a/tools/testing/selftests/arm64/user4k/gem-fixture-init.c b/tools/testing/selftests/arm64/user4k/gem-fixture-init.c new file mode 100644 index 0000000000000..aadf343dc7e8d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/gem-fixture-init.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static unsigned char value(size_t off) +{ + return 0x31 + (off / 4096) % 128; +} +static bool check(unsigned char *p, size_t len, size_t off) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value(i + off)) { + ksft_print_msg("offset=%zu got=%u expected=%u\n", + i + off, p[i], value(i + off)); + return false; + } + return true; +} +static int exercise(size_t native) +{ + size_t ps = getauxval(AT_PAGESZ), + len = native < 8192 ? 65536 : 8 * native; + int fd = open("/dev/vgem", O_RDWR); + struct drm_mode_create_dumb create = { .width = 4096, + .height = len / (4096 * 4), + .bpp = 32 }; + struct drm_mode_map_dumb map = {}; + struct drm_prime_handle prime = { .flags = DRM_RDWR }; + unsigned char *p = MAP_FAILED, *q = MAP_FAILED, *alias = MAP_FAILED; + bool ok = fd >= 0 && !ioctl(fd, DRM_IOCTL_MODE_CREATE_DUMB, &create); + ksft_print_header(); + ksft_set_plan(12); + ksft_print_msg("ABI=%zuK\n", ps / 1024); + map.handle = create.handle; + ok &= !ioctl(fd, DRM_IOCTL_MODE_MAP_DUMB, &map) && create.size >= len; + if (ok) + p = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + ok &= p != MAP_FAILED; + ksft_test_result(ok, "full GEM shmem object mapping\n"); + q = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + bool separate = q != MAP_FAILED; + if (separate) { + memset(q, 0x39, ps); + memset(q + ps, 0xa2, ps); + for (size_t i = 0; i < ps; i++) + separate &= q[i] == 0x39 && q[ps + i] == 0xa2; + munmap(q, 2 * ps); + q = MAP_FAILED; + } + ksft_test_result(separate, + "adjacent user leaves map distinct backing bytes\n"); + if (ok) { + for (size_t i = 0; i < len; i++) + p[i] = value(i); + ok = check(p, len, 0); + } + ksft_test_result(ok, "full object has distinct quarter data\n"); + if (ok) { + q = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + ok = q != MAP_FAILED && !mprotect(q + ps, ps, PROT_READ); + if (ok) + ok = check(q, len, 0); + if (q != MAP_FAILED) + munmap(q, len); + q = MAP_FAILED; + } + ksft_test_result( + ok, + "cold faults after mprotect VMA splitting preserve object offsets\n"); + if (p != MAP_FAILED) { + q = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + ok = q != MAP_FAILED && !munmap(q + ps, ps); + if (ok) + ok = check(q + 2 * ps, len - 2 * ps, 2 * ps) && + check(q, ps, 0); + if (q != MAP_FAILED) + munmap(q, len); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result( + ok, + "cold faults after partial unmap preserve object offsets\n"); + prime.handle = create.handle; + ok = !ioctl(fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &prime); + if (ok) { + alias = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, + prime.fd, 0); + ok = alias != MAP_FAILED && check(alias, len, 0); + } + ksft_test_result(ok, "PRIME exported full mapping shares GEM data\n"); + if (alias != MAP_FAILED) { + q = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_SHARED, prime.fd, + 3 * ps); + ok = q != MAP_FAILED; + if (ok) + ok = check(q, ps, 3 * ps); + if (q != MAP_FAILED) + munmap(q, ps); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result( + ok, "PRIME partial mapping uses requested buffer offset\n"); + if (alias != MAP_FAILED) { + q = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, + prime.fd, 0); + void *target = mmap(NULL, len, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + ok = q != MAP_FAILED && target != MAP_FAILED; + if (ok) { + void *moved = mremap(q, len, len, + MREMAP_MAYMOVE | MREMAP_FIXED, + target); + ok = moved == target; + if (ok) { + q = target; + ok = check(q, len, 0); + } + } + if (q != MAP_FAILED) + munmap(q, len); + if (target != MAP_FAILED && target != q) + munmap(target, len); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result( + ok, + "cold GEM faults after fixed mremap retain object offsets\n"); + if (alias != MAP_FAILED) { + pid_t child = fork(); + int status = 0; + if (!child) { + unsigned char *cold = mmap(NULL, len, + PROT_READ | PROT_WRITE, + MAP_SHARED, prime.fd, 0); + bool pass = cold != MAP_FAILED && check(cold, len, 0); + if (pass) + cold[0] = 0xe1; + _exit(pass ? 0 : 1); + } + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && p[0] == 0xe1; + p[0] = value(0); + } else + ok = false; + ksft_test_result( + ok, "forked GEM mapping shares bytes and retains backing\n"); + if (alias != MAP_FAILED) { + q = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, + prime.fd, len - ps); + ok = q == MAP_FAILED && errno == EINVAL; + if (q != MAP_FAILED) + munmap(q, 2 * ps); + q = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset + ps); + ok &= q == MAP_FAILED && errno == EINVAL; + if (q != MAP_FAILED) + munmap(q, ps); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result(ok, "PRIME bounds and exact GEM offset rejection\n"); + if (alias != MAP_FAILED) { + size_t targets[] = { native, 4096, 16384 }; + + ok = true; + for (unsigned int t = 0; t < 3; t++) { + size_t target = targets[t]; + pid_t child; + int status = 0; + bool pair; + + if (target > native || target == ps || + (t && target == native)) + continue; + child = fork(); + if (!child) { + char number[24], length[24], pages[24]; + + snprintf(number, sizeof(number), "%d", + prime.fd); + snprintf(length, sizeof(length), "%zu", len); + snprintf(pages, sizeof(pages), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + target == native ? 0UL : target, 0UL, + 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--shared", number, + length, pages, NULL); + _exit(4); + } + pair = child > 0 && + waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + for (size_t i = 0; i < len; i += 4096) { + pair &= p[i] == + (unsigned char)(0x80 + i / 4096); + p[i] = value(i); + } + ksft_print_msg( + "PRIME sharing %zuK -> %zuK status=%d bytes=%zu\n", + ps / 1024, target / 1024, status, len); + ok &= pair; + } + } else { + ok = false; + } + ksft_test_result( + ok, + "PRIME backing shared between native and alternative processes\n"); + struct drm_gem_close close_arg = { .handle = create.handle }; + ok = !ioctl(fd, DRM_IOCTL_GEM_CLOSE, &close_arg); + close(fd); + if (alias != MAP_FAILED) { + close(prime.fd); + ok &= check(alias, len, 0); + munmap(alias, len); + } else + ok = false; + if (p != MAP_FAILED) { + ok &= check(p, len, 0); + munmap(p, len); + } else + ok = false; + ksft_test_result( + ok, + "mappings retain object after handles and descriptors close\n"); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +int main(int argc, char **argv) +{ + bool ok = true; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--shared")) { + struct stat init, self; + int fd = atoi(argv[2]); + size_t len = strtoul(argv[3], NULL, 10); + if (stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || + init.st_ino != self.st_ino || init.st_dev != self.st_dev || + len < 65536 || len > 8 * 65536 || + getauxval(AT_PAGESZ) != strtoul(argv[4], NULL, 10)) + return 2; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + bool pass = p != MAP_FAILED && check(p, len, 0); + if (pass) + for (size_t i = 0; i < len; i += 4096) + p[i] = 0x80 + i / 4096; + if (p != MAP_FAILED) + munmap(p, len); + return pass ? 0 : 1; + } + if (argc == 3 && !strcmp(argv[1], "--test") && getppid() == 1) + return exercise(strtoul(argv[2], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + FILE *dev = fopen("/sys/class/drm/card0/dev", "r"); + unsigned maj, min; + if (!dev || fscanf(dev, "%u:%u", &maj, &min) != 2) + return 4; + fclose(dev); + if (mknod("/dev/vgem", S_IFCHR | 0600, makedev(maj, min))) + return 5; + size_t native = getauxval(AT_PAGESZ), + targets[] = { native, 4096, 16384 }; + + for (unsigned int t = 0; t < 3; t++) { + size_t target = targets[t]; + pid_t pid; + int status = 0; + + if (target > native || (t && target == native)) + continue; + pid = fork(); + if (!pid) { + char number[24]; + + snprintf(number, sizeof(number), "%zu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + target == native ? 0UL : target, 0UL, 0UL, + 0UL)) + _exit(3); + execl("/init", "/init", "--test", number, NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("# GEM child ABI=%zuK native=%zuK status=%d\n", + target / 1024, native / 1024, status); + } + printf("%s - GEM fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/granule-proc-init.c b/tools/testing/selftests/arm64/user4k/granule-proc-init.c new file mode 100644 index 0000000000000..e70f8df18e57a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-proc-init.c @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok, const char *s) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", s); + failed += !ok; +} +static long pss(void *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end; + int active = 0; + long value = -1; + if (!f) + return -1; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + active = (unsigned long)p >= start && + (unsigned long)p < end; + else if (active && sscanf(line, "Pss: %ld kB", &value) == 1) + break; + } + free(line); + fclose(f); + return value; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--peer")) { + int fd = atoi(argv[2]), ready = atoi(argv[3]), + release = atoi(argv[4]); + size_t native = strtoul(argv[5], NULL, 10); + unsigned char *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, native - 4096); + char token = 'a'; + int ok = getauxval(AT_PAGESZ) == 4096 && p != MAP_FAILED; + if (ok) { + for (unsigned i = 0; i < 4096; i++) + ok &= p[i] == 0x7b; + p[0] = 0xc9; + ok &= pss(p) == 2; + } + token = ok ? 'a' : 'x'; + if (write(ready, &token, 1) != 1) + return 3; + if (read(release, &token, 1) != 1) + return 4; + return ok ? 0 : 2; + } + if (getpid() != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + mount("proc", "/proc", "proc", 0, NULL); + size_t native = getauxval(AT_PAGESZ); + int fd = memfd_create("high-slot-pss", 0); + result(fd >= 0 && !ftruncate(fd, native), + "native file backing created"); + unsigned char *p = + mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + result(0, "parent mapping"); + goto done; + } + memset(p, 0x7b, native); + long before = pss(p); + printf("# native=%zu initial PSS=%ldKB\n", native, before); + result(before == (long)native / 1024, + "native PSS accounts every physical slot"); + int ready[2], release[2]; + if (pipe(ready) || pipe(release)) { + result(0, "pipes"); + goto done; + } + pid_t child = fork(); + int status = 0; + if (!child) { + char f[24], r[24], w[24], n[24]; + snprintf(f, sizeof(f), "%d", fd); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(w, sizeof(w), "%d", release[0]); + snprintf(n, sizeof(n), "%zu", native); + close(ready[0]); + close(release[1]); + if (prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--peer", f, r, w, n, NULL); + _exit(5); + } + close(ready[1]); + close(release[0]); + char token = 0; + int ok = child > 0 && read(ready[0], &token, 1) == 1 && token == 'a'; + result(ok && p[native - 4096] == 0xc9, + "4K peer shares the final physical slot and observes 2KB PSS"); + long during = pss(p); + printf("# parent mixed PSS=%ldKB expected=%zuKB\n", during, + native / 1024 - 2); + result(during == (long)native / 1024 - 2, + "native PSS weights the high shared slot independently"); + token = 'b'; + if (write(release[1], &token, 1) != 1) + failed++; + ok = waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status); + result(ok && pss(p) == (long)native / 1024, + "PSS returns to native size after peer releases its mapping"); + FILE *f = fopen("/proc/self/smaps_rollup", "r"); + char buf[512]; + ok = f != NULL; + if (f) { + while (fgets(buf, sizeof(buf), f)) { + } + ok &= !ferror(f); + fclose(f); + } + result(ok, "smaps_rollup reads complete native slot accounting"); + munmap(p, native); +done: + printf("GRANULE PROC %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/granule-swap-init.c b/tools/testing/selftests/arm64/user4k/granule-swap-init.c new file mode 100644 index 0000000000000..5791f7e2c0ee9 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-swap-init.c @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "swapfile-fixture.h" +#define SET_EXEC_PAGE_SIZE 0x41555001 +static unsigned char pattern(size_t i, unsigned round) +{ + return (i ^ (i >> 12) ^ (i >> 20) ^ round) & 255; +} +static int check(unsigned char *p, size_t size, unsigned round) +{ + for (size_t i = 0; i < size; i++) + if (p[i] != pattern(i, round)) { + printf("# byte=%zu got=%u expected=%u\n", i, p[i], + pattern(i, round)); + return 0; + } + return 1; +} +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "r"); + char k[80]; + unsigned long n = 0, result = 0; + if (!f) + return 0; + while (fscanf(f, "%79s %lu", k, &n) == 2) + if (!strcmp(k, "pswpout")) + result = n; + fclose(f); + return result; +} +static unsigned swapped(void *p, size_t size, size_t ps) +{ + int fd = open("/proc/self/pagemap", O_RDONLY); + uint64_t bits; + unsigned n = 0; + if (fd < 0) + return 0; + for (size_t i = 0; i < size; i += ps) + if (pread(fd, &bits, 8, ((uintptr_t)p + i) / ps * 8) == 8) + n += (bits >> 62) & 1; + close(fd); + return n; +} +static int exercise(unsigned long expected) +{ + size_t ps = getauxval(AT_PAGESZ), size = 8 * 1024 * 1024; + unsigned char *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int failed = 0; + if (ps != expected || p == MAP_FAILED) { + printf("not ok - expected ABI=%lu actual=%zu\n", expected, ps); + return 1; + } + for (unsigned round = 0; round < 4; round++) { + for (size_t i = 0; i < size; i++) + p[i] = pattern(i, round); + unsigned long before = swapouts(); + int advice = 0; + unsigned n = 0, attempts = 0; + /* PAGEOUT is advisory. Require evidence, with a bounded retry window. */ + do { + advice = madvise(p, size, MADV_PAGEOUT); + attempts++; + if (advice) + break; + usleep(20000); + n = swapped(p, size, ps); + } while (!n && attempts < 100); + unsigned long written = swapouts() - before; + int ok = !advice && written > 0 && n > 0; + printf("%s - %zuK round%u real swapped PTEs=%u/%zu writes=%lu attempts=%u\n", + ok ? "ok" : "not ok", ps / 1024, round, n, size / ps, + written, attempts); + failed += !ok; + int release[2], ready[2]; + if (pipe(release) || pipe(ready)) + return 2; + pid_t child = fork(); + int status = 0; + if (!child) { + char token; + close(release[1]); + close(ready[0]); + int ok = read(release[0], &token, 1) == 1 && + check(p, size, round); + if (ok) { + for (size_t i = 0; i < size; i += ps) + p[i] ^= 0x6d; + } + token = ok ? 'a' : 'x'; + write(ready[1], &token, 1); + _exit(ok ? 0 : 3); + } + close(release[0]); + close(ready[1]); + ok = check(p, size, round); + printf("%s - %zuK round%u disk roundtrip bytes\n", + ok ? "ok" : "not ok", ps / 1024, round); + failed += !ok; + char token = 'b'; + write(release[1], &token, 1); + ok = read(ready[0], &token, 1) == 1 && token == 'a' && + waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status) && check(p, size, round); + printf("%s - %zuK round%u forked swap-in and COW retain parent bytes\n", + ok ? "ok" : "not ok", ps / 1024, round); + failed += !ok; + close(release[1]); + close(ready[0]); + } + munmap(p, size); + return failed ? 1 : 0; +} +static int run_suite(bool nonlinear) +{ + const char *base = + nonlinear ? + "/sys/kernel/debug/kunit/arm64-user4k-nonlinear-swap/" : + "/sys/kernel/debug/kunit/arm64-user4k-swap/"; + char path[160]; + snprintf(path, sizeof(path), "%srun", base); + int fd = open(path, O_WRONLY); + if (fd < 0) + return 1; + int ok = write(fd, "1", 1) == 1; + close(fd); + snprintf(path, sizeof(path), "%sresults", base); + FILE *f = fopen(path, "r"); + char *line = NULL; + size_t cap = 0; + int completed = 0; + if (!f) + return 1; + while (getline(&line, &cap, f) > 0) { + fputs(line, stdout); + if (strstr(line, "not ok") || strstr(line, "# SKIP")) + ok = 0; + if (strstr(line, nonlinear ? + "ok 1 arm64-user4k-nonlinear-swap" : + "ok 1 arm64-user4k-swap")) + completed = 1; + } + free(line); + fclose(f); + return !(ok && completed); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && (!strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--swapon-helper"))) { + struct stat a, b; + if (getpid() == 1 || stat("/proc/1/exe", &a) || + stat("/proc/self/exe", &b) || a.st_dev != b.st_dev || + a.st_ino != b.st_ino) + return 1; + return (!strcmp(argv[1], "--swapoff-helper") ? + swapoff(swap_path()) : + swapon(swap_path(), 0)) ? + 1 : + 0; + } + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 1; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || + (argc != 1 && (argc != 2 || (strcmp(argv[1], "--el0-only") && + strcmp(argv[1], "--nonlinear") && + strcmp(argv[1], "--swapfile"))))) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + int failed = 0, active = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) { + failed++; + goto done; + } + struct stat st; + for (int i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || + (argc == 2 && !strcmp(argv[1], "--swapfile") && + prepare_swapfile()) || + swapon(swap_path(), 0)) { + perror("swapon"); + failed++; + goto done; + } + active = 1; + printf("GRANULE SWAP BACKING=%s\n", swap_path()); + if (argc == 1 || (argc == 2 && !strcmp(argv[1], "--nonlinear"))) + failed += run_suite(argc == 2); + unsigned long sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned abi = 0; abi < (sizes[0] > 16384 ? 3U : 2U); abi++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[32]; + snprintf(expected, sizeof(expected), "%lu", sizes[abi]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[abi], 0UL, 0UL, + 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = c > 0 && waitpid(c, &status, 0) == c && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - swap ABI %luK child status=%#x\n", + ok ? "ok" : "not ok", sizes[abi] / 1024, status); + failed += !ok; + } +done: + if (active && swapoff(swap_path())) { + perror("swapoff"); + failed++; + } + if (active && argc == 2 && !strcmp(argv[1], "--swapfile") && + umount("/swapdir")) { + perror("umount"); + failed++; + } + printf("GRANULE SWAP %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/granule-threeway-init.c b/tools/testing/selftests/arm64/user4k/granule-threeway-init.c new file mode 100644 index 0000000000000..71a459c9b5221 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-threeway-init.c @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +struct snapshot { + long pss; + int first, last, status; +}; +struct peer { + pid_t pid; + int command, reply; + size_t size; +}; +static int failed; +static void result(int ok, const char *s) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", s); + failed += !ok; +} +static long pss(void *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end; + int active = 0; + long value = -1; + if (!f) + return -1; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + active = (unsigned long)p >= start && + (unsigned long)p < end; + else if (active && sscanf(line, "Pss: %ld kB", &value) == 1) + break; + } + free(line); + fclose(f); + return value; +} +static int transfer(int fd, void *data, size_t size, int writing) +{ + size_t done = 0; + while (done < size) { + ssize_t n = + writing ? write(fd, (char *)data + done, size - done) : + read(fd, (char *)data + done, size - done); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + return 0; + done += n; + } + return 1; +} +static int worker(int fd, int command, int reply, size_t size) +{ + const struct rlimit no_core = { 0, 0 }; + setrlimit(RLIMIT_CORE, &no_core); + if (getppid() != 1 || getauxval(AT_PAGESZ) != size) + return 2; + unsigned char *p = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, + fd, 65536 - size); + if (p == MAP_FAILED) + return 3; + for (;;) { + char op; + struct snapshot s = { .pss = -1, .first = -1, .last = -1 }; + if (!transfer(command, &op, 1, 0)) + return 4; + if (op == 'x') { + s.status = munmap(p, size); + transfer(reply, &s, sizeof(s), 1); + return s.status ? 5 : 0; + } + if (op == 'r') + s.status = mprotect(p, size, PROT_READ); + else if (op == 'w') + p[size - 1] = 0x99; + else if (op == 'd') + s.status = madvise(p, size, MADV_DONTNEED); + else if (op == 't') { + pid_t c = fork(); + int status = 0; + if (!c) { + unsigned char value = *( + volatile unsigned char *)(p + size - 1); + _exit(value); + } + if (c < 0 || waitpid(c, &status, 0) != c) + s.status = -1; + else + s.status = WIFSIGNALED(status) ? + 256 + WTERMSIG(status) : + WIFEXITED(status) ? + WEXITSTATUS(status) : + -2; + } + if (op != 'd' && op != 't') { + s.first = p[0]; + s.last = p[size - 1]; + } + s.pss = pss(p); + if (!transfer(reply, &s, sizeof(s), 1)) + return 6; + } +} +static struct peer spawn(int fd, size_t size) +{ + int command[2], reply[2]; + struct peer p = { .pid = -1, .size = size }; + if (pipe(command) || pipe(reply)) + return p; + p.pid = fork(); + if (!p.pid) { + char f[24], c[24], r[24], s[24]; + close(command[1]); + close(reply[0]); + snprintf(f, sizeof(f), "%d", fd); + snprintf(c, sizeof(c), "%d", command[0]); + snprintf(r, sizeof(r), "%d", reply[1]); + snprintf(s, sizeof(s), "%zu", size); + if (prctl(SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL)) + _exit(7); + execl("/init", "/init", "--worker", f, c, r, s, NULL); + _exit(8); + } + close(command[0]); + close(reply[1]); + p.command = command[1]; + p.reply = reply[0]; + return p; +} +static struct snapshot command(struct peer *p, char op) +{ + struct snapshot s = { .pss = -1, .first = -1, .last = -1, .status = -1 }; + if (p->pid <= 0 || !transfer(p->command, &op, 1, 1) || + !transfer(p->reply, &s, sizeof(s), 0)) + failed++; + return s; +} +static void finish(struct peer *p) +{ + struct snapshot s = command(p, 'x'); + int status = 0; + result(!s.status && waitpid(p->pid, &status, 0) == p->pid && + WIFEXITED(status) && !WEXITSTATUS(status), + "peer unmap and exit"); + close(p->command); + close(p->reply); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), + strtoul(argv[5], NULL, 10)); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable64K VM PID1 only\n"); + return 2; + } + mount("proc", "/proc", "proc", 0, NULL); + if (getauxval(AT_PAGESZ) != 65536) { + result(0, "native64K required"); + goto done; + } + int fd = memfd_create("three-granules", 0); + if (fd < 0 || ftruncate(fd, 65536)) { + result(0, "file setup"); + goto done; + } + unsigned char *p = + mmap(NULL, 65536, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + result(0, "native mapping"); + goto done; + } + memset(p, 0x42, 65536); + struct peer medium = spawn(fd, 16384); + struct snapshot s = command(&medium, 's'); + result(s.pss == 8 && s.first == 0x42 && s.last == 0x42 && pss(p) == 56, + "64K/16K initial sharing and PSS"); + struct peer small = spawn(fd, 4096); + s = command(&small, 's'); + result(s.pss == 1 && s.first == 0x42 && s.last == 0x42, + "4K peer shares the final slice with both larger ABIs"); + s = command(&medium, 's'); + result(s.pss == 7 && pss(p) == 55, + "three-way PSS weights partial overlaps independently"); + s = command(&small, 'r'); + result(!s.status, "4K alias becomes read-only"); + s = command(&medium, 'w'); + result(!s.status && p[65535] == 0x99, + "16K alias remains writable across local4K protection"); + s = command(&small, 's'); + result(s.last == 0x99, "read-only4K alias observes16K writes"); + s = command(&small, 'd'); + result(!s.status && s.pss == 0 && pss(p) == 56, + "4K discard removes only its mapping"); + s = command(&medium, 's'); + result(s.pss == 8 && s.last == 0x99, "16K mapping survives4K discard"); + s = command(&small, 's'); + result(s.pss == 1 && s.last == 0x99 && pss(p) == 55, + "4K refault restores coherent three-way sharing"); + result(!ftruncate(fd, 65536 - 4096), + "truncate at4K boundary inside16K leaf"); + s = command(&small, 't'); + result(s.status == 256 + SIGBUS, + "4K mapping wholly beyond EOF faults SIGBUS"); + s = command(&medium, 't'); + result(s.status == 0, "partial16K EOF leaf exposes zero tail"); + result(p[65535] == 0 && p[0] == 0x42, + "native64K sees zero tail and preserved file prefix"); + s = command(&medium, 's'); + result(s.first == 0x42 && s.last == 0 && s.pss == 8 && pss(p) == 56, + "16K EOF refault keeps correct mixed PSS"); + result(!ftruncate(fd, 65536), "restore file length"); + s = command(&small, 's'); + result(s.first == 0 && s.last == 0 && s.pss == 1 && pss(p) == 55, + "4K alias refaults zeroed extension"); + finish(&small); + s = command(&medium, 's'); + result(s.pss == 8 && pss(p) == 56, "16K alias survives4K exit"); + finish(&medium); + result(pss(p) == 64, + "native PSS restored after all alternative aliases exit"); + munmap(p, 65536); + close(fd); +done: + printf("THREE GRANULES %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/iommufd-init.c b/tools/testing/selftests/arm64/user4k/iommufd-init.c new file mode 100644 index 0000000000000..c678f35cf72a4 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/iommufd-init.c @@ -0,0 +1,1314 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable VM PID1: real IOMMUFD ioctls against mock AMDv1 page tables. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../drivers/iommu/iommufd/iommufd_test.h" + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define VA 0x200000000UL +#define IOVA 0x10000000UL +#define ALIAS 0x12000000UL +#define MAP_FLAGS \ + (IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_READABLE | \ + IOMMU_IOAS_MAP_WRITEABLE) +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) + +static size_t native_ps, user_ps; +struct context { + int fd, access; + uint32_t ioas, hwpt[2], dev[2]; +}; + +static int drop_pin_cap(void) +{ + struct __user_cap_header_struct h = { + .version = _LINUX_CAPABILITY_VERSION_3 + }; + struct __user_cap_data_struct d[2]; + unsigned int mask = 1U << (CAP_IPC_LOCK % 32); + + if (syscall(SYS_capget, &h, d)) + return -1; + d[CAP_IPC_LOCK / 32].effective &= ~mask; + d[CAP_IPC_LOCK / 32].permitted &= ~mask; + return syscall(SYS_capset, &h, d); +} + +static int limit_bytes(size_t bytes) +{ + struct rlimit r; + + if (getrlimit(RLIMIT_MEMLOCK, &r)) + return -1; + r.rlim_cur = bytes; + return setrlimit(RLIMIT_MEMLOCK, &r); +} + +static long stat_kb(pid_t pid, const char *key) +{ + char path[80], line[256]; + long result = -1; + FILE *f; + + snprintf(path, sizeof(path), "/proc/%d/status", pid); + f = fopen(path, "r"); + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) + if (!strncmp(line, key, strlen(key))) { + result = strtol(line + strlen(key), NULL, 10); + break; + } + fclose(f); + return result; +} + +static int new_domain(struct context *c, unsigned int index) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_MOCK_DOMAIN, + .id = c->ioas }; + int rc = ioctl(c->fd, IOMMU_TEST_CMD, &cmd); + + if (!rc) { + c->hwpt[index] = cmd.mock_domain.out_hwpt_id; + c->dev[index] = cmd.mock_domain.out_idev_id; + } + return rc; +} + +static int context_open_common(struct context *c, unsigned int mm_account, + bool domain, bool pins) +{ + struct iommu_ioas_alloc a = { .size = sizeof(a) }; + struct iommu_option o = { .size = sizeof(o), + .option_id = IOMMU_OPTION_RLIMIT_MODE, + .op = IOMMU_OPTION_OP_SET, + .val64 = mm_account }; + struct iommu_test_cmd access = { .size = sizeof(access), + .op = IOMMU_TEST_OP_CREATE_ACCESS }; + + memset(c, 0, sizeof(*c)); + c->fd = open("/dev/iommu", O_RDWR); + if (c->fd < 0 || ioctl(c->fd, IOMMU_OPTION, &o) || + ioctl(c->fd, IOMMU_IOAS_ALLOC, &a)) + return -1; + c->ioas = a.out_ioas_id; + if (domain && new_domain(c, 0)) + return -1; + access.id = c->ioas; + access.create_access.flags = + pins ? MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES : 0; + if (ioctl(c->fd, IOMMU_TEST_CMD, &access)) + return -1; + c->access = access.create_access.out_access_fd; + return 0; +} + +static int context_open(struct context *c, unsigned int mm_account) +{ + return context_open_common(c, mm_account, true, false); +} + +static void context_close(struct context *c) +{ + close(c->access); + close(c->fd); +} + +static int map_range(struct context *c, void *p, size_t len, unsigned long iova) +{ + struct iommu_ioas_map m = { .size = sizeof(m), + .flags = MAP_FLAGS, + .ioas_id = c->ioas, + .user_va = (uintptr_t)p, + .length = len, + .iova = iova }; + + return ioctl(c->fd, IOMMU_IOAS_MAP, &m); +} + +static int unmap_range(struct context *c, size_t len, unsigned long iova) +{ + struct iommu_ioas_unmap u = { .size = sizeof(u), + .ioas_id = c->ioas, + .iova = iova, + .length = len }; + int rc = ioctl(c->fd, IOMMU_IOAS_UNMAP, &u); + + return rc ?: (u.length == len ? 0 : -1); +} + +static int check_pa(struct context *c, unsigned int dom, void *p, size_t len, + unsigned long iova) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_MD_CHECK_MAP, + .id = c->hwpt[dom], + .check_map = { .iova = iova, + .length = len, + .uptr = (uintptr_t)p } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static int access_rw(struct context *c, void *p, size_t len, unsigned long iova, + unsigned int flags) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_ACCESS_RW, + .id = c->access, + .access_rw = { .iova = iova, + .length = len, + .uptr = (uintptr_t)p, + .flags = flags } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static bool filled(const void *ptr, unsigned char byte, size_t n) +{ + const unsigned char *p = ptr; + + for (size_t i = 0; i < n; i++) + if (p[i] != byte) + return false; + return true; +} + +static void *relocated(size_t len, unsigned char byte) +{ + void *old = mmap((void *)VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, + 0); + + if (old == MAP_FAILED) + return old; + memset(old, byte, len); + size_t distance = len > 32 * native_ps ? len + native_ps : + 32 * native_ps; + + return mremap(old, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, + (void *)(VA + distance + user_ps)); +} + +static int limits_and_lifetime(unsigned int mode) +{ + struct context c; + size_t len = 2 * native_ps; + unsigned char *p = relocated(3 * native_ps, 0x31), *buf = malloc(len); + struct iommu_ioas_copy copy; + + CHECK(p != MAP_FAILED && buf); + CHECK(!context_open(&c, mode)); + CHECK(!limit_bytes(len)); + CHECK(map_range(&c, p, 3 * native_ps, IOVA) == -1 && errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!map_range(&c, p, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(stat_kb(getpid(), "VmLck:") == (mode ? (long)len / 1024 : 0)); + CHECK(!check_pa(&c, 0, p, len, IOVA)); + CHECK(map_range(&c, p + len, native_ps, ALIAS) == -1 && + errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + copy = (struct iommu_ioas_copy){ .size = sizeof(copy), + .flags = MAP_FLAGS, + .src_ioas_id = c.ioas, + .dst_ioas_id = c.ioas, + .src_iova = IOVA + user_ps, + .dst_iova = ALIAS, + .length = len - user_ps }; + CHECK(!ioctl(c.fd, IOMMU_IOAS_COPY, ©)); + CHECK(!new_domain(&c, 1)); + CHECK(!check_pa(&c, 1, p + user_ps, len - user_ps, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(mmap(p, len, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) == p); + memset(p, 0x5a, len); + CHECK(check_pa(&c, 0, p, len, IOVA) == -1 && errno == EINVAL); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x31, len)); + CHECK(!access_rw(&c, buf, len, IOVA, 0)); + CHECK(filled(buf, 0x5a, len)); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == + (long)(user_ps < native_ps ? len : len - user_ps) / 1024); + CHECK(!unmap_range(&c, len - user_ps, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + /* Fresh anonymous backing follows the shifted VA's native linear index. + * A non-native-aligned replacement spans three physical backing pages. + */ + size_t replacement_pages = user_ps < native_ps ? 3 : 2; + + if (replacement_pages == 3) { + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + } + CHECK(!limit_bytes(replacement_pages * native_ps)); + CHECK(!map_range(&c, p, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == + (long)(replacement_pages * native_ps / 1024)); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!munmap(p, 3 * native_ps)); + context_close(&c); + free(buf); + return 0; +} + +static int holes_and_shared(unsigned int mode) +{ + struct context c; + size_t len = 3 * user_ps, file_len = 4 * native_ps; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0), + *buf = malloc(file_len); + int file = memfd_create("iommu-shared", 0); + + CHECK(p != MAP_FAILED && buf && file >= 0); + CHECK(!context_open(&c, mode) && !limit_bytes(8 * native_ps)); + memset(p, 0x31, len); + CHECK(!munmap(p + user_ps, user_ps)); + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == EFAULT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(mmap(p + user_ps, user_ps, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, + 0) == p + user_ps); + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == EFAULT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!mprotect(p + user_ps, user_ps, PROT_READ | PROT_WRITE)); + CHECK(!map_range(&c, p, len, IOVA) && !check_pa(&c, 0, p, len, IOVA)); + CHECK(!unmap_range(&c, len, IOVA) && !munmap(p, len)); + CHECK(!ftruncate(file, file_len)); + p = mmap(NULL, 2 * user_ps, PROT_READ | PROT_WRITE, MAP_SHARED, file, + user_ps); + CHECK(p != MAP_FAILED); + CHECK(!map_range(&c, p, 2 * user_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, 2 * user_ps, IOVA)); + CHECK(!munmap(p, 2 * user_ps)); + memset(buf, 0x93, 2 * user_ps); + CHECK(!access_rw(&c, buf, 2 * user_ps, IOVA, + MOCK_ACCESS_RW_WRITE | MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(!unmap_range(&c, 2 * user_ps, IOVA)); + CHECK(pread(file, buf, file_len, 0) == (ssize_t)file_len); + CHECK(filled(buf, 0, user_ps)); + CHECK(filled(buf + user_ps, 0x93, 2 * user_ps)); + CHECK(filled(buf + 3 * user_ps, 0, file_len - 3 * user_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + close(file); + context_close(&c); + free(buf); + return 0; +} + +static int producer(int argc, char **argv) +{ + struct context c = { .fd = atoi(argv[4]), + .ioas = strtoul(argv[5], NULL, 0) }; + int ready = atoi(argv[6]), go = atoi(argv[7]); + char byte = 'R'; + void *p; + + (void)argc; + CHECK(!prctl(PR_SET_PDEATHSIG, SIGKILL)); + CHECK(!drop_pin_cap()); + p = relocated(2 * native_ps, 0x6d); + CHECK(p != MAP_FAILED); + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(write(ready, &byte, 1) == 1); + CHECK(read(go, &byte, 1) == 1 && byte == 'G'); + return 0; +} + +static int destroy_id(struct context *c, uint32_t id) +{ + struct iommu_destroy cmd = { .size = sizeof(cmd), .id = id }; + + return ioctl(c->fd, IOMMU_DESTROY, &cmd); +} + +static int create_viommu(struct context *c, uint32_t *id, uint32_t *parent) +{ + struct iommu_hwpt_alloc hwpt = { .size = sizeof(hwpt), + .flags = IOMMU_HWPT_ALLOC_NEST_PARENT, + .dev_id = c->dev[0], + .pt_id = c->ioas }; + struct iommu_viommu_alloc cmd = { .size = sizeof(cmd), + .type = IOMMU_VIOMMU_TYPE_SELFTEST, + .dev_id = c->dev[0] }; + + if (ioctl(c->fd, IOMMU_HWPT_ALLOC, &hwpt)) + return -1; + *parent = hwpt.out_hwpt_id; + cmd.hwpt_id = *parent; + if (ioctl(c->fd, IOMMU_VIOMMU_ALLOC, &cmd)) + return -1; + *id = cmd.out_viommu_id; + return 0; +} + +static int queue_alloc(struct context *c, uint32_t viommu, unsigned int index, + unsigned long iova, size_t length, uint32_t *id) +{ + struct iommu_hw_queue_alloc cmd = { .size = sizeof(cmd), + .viommu_id = viommu, + .type = IOMMU_HW_QUEUE_TYPE_SELFTEST, + .index = index, + .nesting_parent_iova = iova, + .length = length }; + int rc = ioctl(c->fd, IOMMU_HW_QUEUE_ALLOC, &cmd); + + if (!rc) + *id = cmd.out_hw_queue_id; + return rc; +} + +static int queue_after_exit(struct context *c) +{ + uint32_t viommu, parent, id; + + CHECK(!create_viommu(c, &viommu, &parent)); + CHECK(!queue_alloc(c, viommu, 0, IOVA + native_ps / 4 + 3, + native_ps / 4, &id)); + CHECK(unmap_range(c, 2 * native_ps, IOVA) == -1 && errno == EBUSY); + CHECK(!destroy_id(c, id)); + CHECK(!destroy_id(c, viommu)); + CHECK(!destroy_id(c, parent)); + return 0; +} + +static int queue_ranges(unsigned int mode) +{ + struct context c; + unsigned char *p = relocated(2 * native_ps, 0x71); + uint32_t viommu, parent, q0, q1, unused; + size_t offset = user_ps / 4 + 3, length = user_ps / 2 + 17; + int file; + + CHECK(p != MAP_FAILED && !context_open(&c, mode)); + CHECK(!limit_bytes(8 * native_ps) && + !create_viommu(&c, &viommu, &parent)); + /* Map only one user leaf: queue pinning must not require surrounding IOVA. */ + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 1, IOVA + offset, length, &unused) == + -1 && + errno == EIO); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + offset, length, &q0)); + CHECK(queue_alloc(&c, viommu, 0, IOVA + offset, length, &unused) == + -1 && + errno == EEXIST); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(stat_kb(getpid(), "VmPin:") == (long)native_ps / 1024); + /* Pins keep the old physical bytes after replacing the source VMA. */ + CHECK(mmap(p, user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == p); + memset(p, 0x5a, user_ps); + CHECK(!queue_alloc(&c, viommu, 1, IOVA + offset + 7, length - 7, &q1)); + CHECK(destroy_id(&c, q0) == -1 && errno == EBUSY); + CHECK(!destroy_id(&c, q1)); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(!destroy_id(&c, q0)); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(queue_alloc(&c, viommu, 0, IOVA + offset, length, &unused) == + -1 && + errno == ENOENT); + CHECK(!munmap(p, 2 * native_ps)); + + /* Separate mappings of the same file byte range cannot form a physical run. + * The second area must fail after the first area's access was registered. + */ + file = memfd_create("queue-alias", 0); + CHECK(file >= 0 && !ftruncate(file, native_ps)); + p = mmap((void *)VA, 2 * user_ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(p != MAP_FAILED); + CHECK(mmap(p, user_ps, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, + file, 0) == p); + CHECK(mmap(p + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, 0) == p + user_ps); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && + errno == ENOENT); + CHECK(!map_range(&c, p + user_ps, user_ps, IOVA + user_ps)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && + errno == EFAULT); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(!unmap_range(&c, user_ps, IOVA + user_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + /* The same discontinuity inside one provider must also release its access. */ + CHECK(!map_range(&c, p, 2 * user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && + errno == EFAULT); + CHECK(!unmap_range(&c, 2 * user_ps, IOVA)); + if (user_ps < native_ps) { + /* Adjacent file quarters form a real physical run across two areas. */ + CHECK(mmap(p + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, + user_ps) == p + user_ps); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!map_range(&c, p + user_ps, user_ps, IOVA + user_ps)); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + 11, 2 * user_ps - 22, + &q0)); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(unmap_range(&c, user_ps, IOVA + user_ps) == -1 && + errno == EBUSY); + CHECK(!destroy_id(&c, q0)); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(!unmap_range(&c, user_ps, IOVA + user_ps)); + } + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, 2 * user_ps)); + close(file); + CHECK(!destroy_id(&c, viommu) && !destroy_id(&c, parent)); + context_close(&c); + printf("ok - hardware queue byte spans %zuK/%zuK %s, " + "relocated/overlap/rollback/unmap locks\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int cross_process(unsigned int mode) +{ + struct context c; + int ready[2], go[2], status; + pid_t child; + size_t child_ps = user_ps == native_ps ? 4096 : native_ps; + size_t len = 2 * native_ps; + unsigned char *p = mmap((void *)(VA + 128 * native_ps), len, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | + MAP_FIXED_NOREPLACE, + -1, 0), + *buf = malloc(len); + char byte; + + CHECK(p != MAP_FAILED && buf && !pipe(ready) && !pipe(go)); + CHECK(!context_open(&c, mode) && !limit_bytes(len)); + child = fork(); + CHECK(child >= 0); + if (!child) { + char ps[24], native[24], fd[24], ioas[24], r[24], g[24]; + + close(ready[0]); + close(go[1]); + snprintf(ps, sizeof(ps), "%zu", child_ps); + snprintf(native, sizeof(native), "%zu", native_ps); + snprintf(fd, sizeof(fd), "%d", c.fd); + snprintf(ioas, sizeof(ioas), "%u", c.ioas); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(g, sizeof(g), "%d", go[0]); + if (prctl(SET_EXEC_PAGE_SIZE, child_ps, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--producer", ps, native, fd, ioas, r, + g, NULL); + _exit(5); + } + close(ready[1]); + close(go[0]); + CHECK(read(ready[0], &byte, 1) == 1 && byte == 'R'); + CHECK(stat_kb(child, "VmPin:") == (long)len / 1024); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + if (!mode) + CHECK(map_range(&c, p, native_ps, ALIAS) == -1 && + errno == ENOMEM); + else { + CHECK(!map_range(&c, p, native_ps, ALIAS)); + CHECK(!unmap_range(&c, native_ps, ALIAS)); + } + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + byte = 'G'; + CHECK(write(go[1], &byte, 1) == 1); + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!new_domain(&c, 1)); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + CHECK(!queue_after_exit(&c)); + if (!mode) + CHECK(map_range(&c, p, native_ps, ALIAS) == -1 && + errno == ENOMEM); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!map_range(&c, p, len, ALIAS) && !unmap_range(&c, len, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + close(ready[0]); + close(go[1]); + CHECK(!munmap(p, len)); + context_close(&c); + free(buf); + return 0; +} + +struct stress_arg { + struct context *c; + void *p; + unsigned int index; + atomic_int *failed; +}; +static void *stress_thread(void *opaque) +{ + struct stress_arg *a = opaque; + unsigned long iova = IOVA + a->index * 0x100000UL; + size_t len = 2 * native_ps; + + for (unsigned int round = 0; round < 64; round++) { + if (map_range(a->c, a->p, len, iova) || + (!(round % 16) && check_pa(a->c, 1, a->p, len, iova)) || + unmap_range(a->c, len, iova)) { + printf("FAIL concurrent map thread=%u round=%u errno=%d\n", + a->index, round, errno); + atomic_store(a->failed, 1); + break; + } + } + return NULL; +} + +static int concurrent_maps(unsigned int mode) +{ + struct context c; + pthread_t threads[4]; + struct stress_arg args[4]; + atomic_int failed = 0; + void *p = relocated(2 * native_ps, 0x47); + + CHECK(p != MAP_FAILED && !context_open(&c, mode)); + CHECK(!limit_bytes(32 * native_ps) && !new_domain(&c, 1)); + for (unsigned int i = 0; i < 4; i++) { + args[i] = (struct stress_arg){ &c, p, i, &failed }; + CHECK(!pthread_create(&threads[i], NULL, stress_thread, + &args[i])); + } + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_join(threads[i], NULL)); + CHECK(!atomic_load(&failed)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, 2 * native_ps)); + context_close(&c); + return 0; +} + +static int write_attribute(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + ssize_t n; + + if (fd < 0) + return -1; + n = write(fd, value, strlen(value)); + close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} + +static int allocation_failures(unsigned int mode) +{ + struct context c; + void *p = relocated(2 * native_ps, 0x62); + int nth = open("/proc/self/fail-nth", O_RDWR); + unsigned int hit = 0, n; + + CHECK(nth >= 0 && p != MAP_FAILED); + CHECK(!context_open(&c, mode) && !limit_bytes(8 * native_ps)); + CHECK(!new_domain(&c, 1)); + CHECK(!write_attribute("/sys/kernel/debug/failslab/ignore-gfp-wait", + "N")); + CHECK(!write_attribute( + "/sys/kernel/debug/fail_page_alloc/ignore-gfp-wait", "N")); + CHECK(!write_attribute( + "/sys/kernel/debug/fail_page_alloc/ignore-gfp-highmem", "N")); + CHECK(!write_attribute("/sys/kernel/debug/failslab/verbose", "0")); + CHECK(!write_attribute("/sys/kernel/debug/fail_page_alloc/verbose", + "0")); + CHECK(!write_attribute("/sys/kernel/debug/fail_iommufd/verbose", "0")); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n); + ssize_t got, disabled; + int rc; + + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = map_range(&c, p, 2 * native_ps, IOVA); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(disabled == 1 && got > 0); + if (!rc) + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + /* An injected failure must not poison a successful retry or translation. */ + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, 2 * native_ps, IOVA)); + CHECK(!check_pa(&c, 1, p, 2 * native_ps, IOVA)); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + printf("ok - IOMMUFD %zuK %s accounting allocation-failure sweep: %u sites\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + CHECK(!munmap(p, 2 * native_ps)); + close(nth); + context_close(&c); + return 0; +} + +static int queue_failure_sweep(unsigned int mode) +{ + struct context c; + void *p = relocated(2 * native_ps, 0x4c); + int nth = open("/proc/self/fail-nth", O_RDWR); + uint32_t viommu, parent, id; + unsigned int n, hit = 0; + + CHECK(p != MAP_FAILED && nth >= 0 && !context_open(&c, mode)); + CHECK(!limit_bytes(8 * native_ps) && + !create_viommu(&c, &viommu, &parent)); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n), rc; + ssize_t got, disabled; + + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = queue_alloc(&c, viommu, 0, IOVA + user_ps / 4 + 3, + user_ps / 2, &id); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(got > 0 && disabled == 1); + if (!rc) + CHECK(!destroy_id(&c, id)); + /* A failed queue must not leave an internal access blocking unmap. */ + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + user_ps / 4 + 3, + user_ps / 2, &id)); + CHECK(!destroy_id(&c, id) && !unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + CHECK(!destroy_id(&c, viommu) && !destroy_id(&c, parent)); + context_close(&c); + close(nth); + CHECK(!munmap(p, 2 * native_ps)); + printf("ok - hardware queue %zuK %s failure sweep: %u sites, no stale locks/pins\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + return 0; +} + +static int page_access(struct context *c, void *p, size_t len, + unsigned long iova, uint32_t *id) +{ + struct iommu_test_cmd cmd = { + .size = sizeof(cmd), + .op = IOMMU_TEST_OP_ACCESS_PAGES, + .id = c->access, + .access_pages = { .iova = iova, + .length = len, + .uptr = (uintptr_t)p, + .flags = MOCK_FLAGS_ACCESS_WRITE } + }; + int rc = ioctl(c->fd, IOMMU_TEST_CMD, &cmd); + + if (!rc) + *id = cmd.access_pages.out_access_pages_id; + return rc; +} + +static int page_release(struct context *c, uint32_t id) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_DESTROY_ACCESS_PAGES, + .id = c->access, + .destroy_access_pages = { + .access_pages_id = id } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static int page_ranges(unsigned int mode) +{ + struct context c; + int file = memfd_create("native-page-array", 0); + unsigned char *p; + uint32_t a, b, d, e; + size_t len = 3 * native_ps, tail = native_ps + user_ps / 2 + 3; + + CHECK(file >= 0 && !ftruncate(file, 5 * native_ps)); + p = mmap((void *)VA, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED && !limit_bytes(4 * native_ps)); + memset(p, 0x67, len); + CHECK(!context_open_common(&c, mode, false, true)); + /* Separate areas exercise native output stride with alternative indices. */ + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, 2 * native_ps, IOVA + native_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, len, IOVA, &a)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(!page_access(&c, p, len, IOVA, &b)); + CHECK(!page_access(&c, p + native_ps, tail, IOVA + native_ps, &d)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + /* Source VA can move without changing its retained physical offset. */ + unsigned char *moved = mremap(p, len, len, + MREMAP_MAYMOVE | MREMAP_FIXED, + (void *)(VA + 32 * native_ps + user_ps)); + CHECK(moved != MAP_FAILED); + CHECK(!page_access(&c, moved, len, IOVA, &e) && !page_release(&c, e)); + CHECK(!page_release(&c, a) && !page_release(&c, b)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)(2 * native_ps) / 1024); + CHECK(!page_release(&c, d)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(mremap(moved, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, p) == p); + CHECK(!page_access(&c, p, len, IOVA, &a)); + /* Unmap callback releases an access covering both independent areas. */ + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(page_release(&c, a) == -1 && errno == ENOENT); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, len)); + + /* The second area is absent: first area's pin/accounting must roll back. */ + p = mmap((void *)VA, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED && !map_range(&c, p, native_ps, IOVA)); + CHECK(page_access(&c, NULL, 2 * native_ps, IOVA, &a) == -1 && + errno == ENOENT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, native_ps, IOVA, &a) && !page_release(&c, a)); + CHECK(!unmap_range(&c, native_ps, IOVA) && !munmap(p, native_ps)); + + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED); + CHECK(mmap(p + native_ps, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, native_ps) == p + native_ps); + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(!page_access(&c, p, 2 * native_ps, IOVA, &a)); + CHECK(stat_kb(getpid(), "VmPin:") == + (long)((user_ps == native_ps ? 2 : 1) * native_ps) / 1024); + CHECK(!page_release(&c, a) && !unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(!munmap(p, 2 * native_ps)); + + if (user_ps < native_ps) { + /* Native-sized logical range whose physical base has a fragment offset. */ + p = mmap((void *)VA, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, user_ps); + CHECK(p != MAP_FAILED && !map_range(&c, p, native_ps, IOVA)); + CHECK(page_access(&c, p, native_ps, IOVA, &a) == -1 && + errno == EINVAL); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!unmap_range(&c, native_ps, IOVA) && + !munmap(p, native_ps)); + + /* Aligned first fragment followed by an alias, not consecutive bytes. */ + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(p != MAP_FAILED); + CHECK(mmap(p + native_ps + user_ps, user_ps, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, file, + native_ps) == p + native_ps + user_ps); + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, native_ps, + IOVA + native_ps)); + CHECK(page_access(&c, p, 2 * native_ps, IOVA, &a) == -1 && + errno == EINVAL); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, native_ps, IOVA, &a)); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(page_release(&c, a) == -1 && errno == ENOENT); + CHECK(!munmap(p, 2 * native_ps)); + } + context_close(&c); + close(file); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + printf("ok - native page arrays %zuK/%zuK %s, lazy/partial/moved/rollback/callback\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int page_lifetime(unsigned int mode) +{ + struct context c; + int ready[2], go[2], status; + pid_t child; + size_t child_ps = user_ps == native_ps ? 4096 : native_ps; + size_t len = 2 * native_ps; + unsigned char *p = mmap((void *)(VA + 128 * native_ps), len, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | + MAP_FIXED_NOREPLACE, + -1, 0), + *buf = malloc(len); + uint32_t a, b; + char byte; + + CHECK(p != MAP_FAILED && buf && !pipe(ready) && !pipe(go)); + memset(p, 0x42, len); + CHECK(!context_open_common(&c, mode, false, true) && !limit_bytes(len)); + child = fork(); + CHECK(child >= 0); + if (!child) { + char ps[24], native[24], fd[24], ioas[24], r[24], g[24]; + + close(ready[0]); + close(go[1]); + snprintf(ps, sizeof(ps), "%zu", child_ps); + snprintf(native, sizeof(native), "%zu", native_ps); + snprintf(fd, sizeof(fd), "%d", c.fd); + snprintf(ioas, sizeof(ioas), "%u", c.ioas); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(g, sizeof(g), "%d", go[0]); + if (prctl(SET_EXEC_PAGE_SIZE, child_ps, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--producer", ps, native, fd, ioas, r, + g, NULL); + _exit(5); + } + close(ready[1]); + close(go[0]); + CHECK(read(ready[0], &byte, 1) == 1 && byte == 'R'); + CHECK(stat_kb(child, "VmPin:") == 0); + CHECK(!page_access(&c, NULL, len, IOVA, &a)); + CHECK(stat_kb(child, "VmPin:") == (long)len / 1024); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + byte = 'G'; + CHECK(write(go[1], &byte, 1) == 1); + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + /* Reuse retained anonymous fragments after source mm_users reaches zero. */ + CHECK(!page_access(&c, NULL, native_ps, IOVA, &b)); + CHECK(!page_release(&c, a)); + CHECK(!access_rw(&c, buf, native_ps, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, native_ps)); + CHECK(!map_range(&c, p, len, ALIAS)); + if (!mode) + CHECK(page_access(&c, p, len, ALIAS, &a) == -1 && + errno == ENOMEM); + else + CHECK(!page_access(&c, p, len, ALIAS, &a) && + !page_release(&c, a)); + /* Final access release refunds even a dead provider's account. */ + CHECK(!page_release(&c, b) && !unmap_range(&c, len, IOVA)); + CHECK(!page_access(&c, p, len, ALIAS, &a) && !page_release(&c, a)); + CHECK(!unmap_range(&c, len, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, len)); + context_close(&c); + close(ready[0]); + close(go[1]); + free(buf); + printf("ok - native page array anonymous source %zuK -> %zuK consumer, %s exit/refund\n", + child_ps / 1024, user_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int page_failure_sweep(unsigned int mode) +{ + struct context c; + int file = memfd_create("page-array-failures", 0); + int nth = open("/proc/self/fail-nth", O_RDWR); + unsigned char *p; + uint32_t id; + unsigned int n, hit = 0; + + CHECK(file >= 0 && nth >= 0 && !ftruncate(file, 2 * native_ps)); + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(p != MAP_FAILED); + memset(p, 0x53, 2 * native_ps); + CHECK(!context_open_common(&c, mode, false, true) && + !limit_bytes(2 * native_ps)); + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, native_ps, IOVA + native_ps)); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n), rc; + ssize_t got, disabled; + + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = page_access(&c, p, 2 * native_ps, IOVA, &id); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(got > 0 && disabled == 1); + if (!rc) + CHECK(!page_release(&c, id)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!page_access(&c, p, 2 * native_ps, IOVA, &id) && + !page_release(&c, id)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA) && + !munmap(p, 2 * native_ps)); + context_close(&c); + close(file); + close(nth); + printf("ok - native page arrays %zuK %s failure sweep: %u sites, refunds/retries\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + return 0; +} + +struct page_race { + struct context *c; + void *p; + pthread_barrier_t barrier; + atomic_int failed, pinned, absent; +}; + +static void *page_pin_thread(void *opaque) +{ + struct page_race *r = opaque; + + pthread_barrier_wait(&r->barrier); + for (unsigned int i = 0; i < 256; i++) { + uint32_t id; + + if (page_access(r->c, r->p, 2 * native_ps, IOVA, &id)) { + /* Unmap can mark the area inaccessible before notifying drivers. */ + if (errno != ENOENT && errno != EINVAL) + goto fail; + atomic_fetch_add(&r->absent, 1); + } else { + atomic_fetch_add(&r->pinned, 1); + sched_yield(); + if (page_release(r->c, id) && errno != ENOENT) + goto fail; + } + } + return NULL; +fail: + printf("FAIL page pin/unmap race errno=%d\n", errno); + atomic_store(&r->failed, 1); + return NULL; +} + +static void *page_unmap_thread(void *opaque) +{ + struct page_race *r = opaque; + + pthread_barrier_wait(&r->barrier); + for (unsigned int i = 0; i < 256; i++) { + if (unmap_range(r->c, 2 * native_ps, IOVA)) + goto fail; + sched_yield(); + if (map_range(r->c, r->p, 2 * native_ps, IOVA)) + goto fail; + } + return NULL; +fail: + printf("FAIL page unmap/remap race errno=%d\n", errno); + atomic_store(&r->failed, 1); + return NULL; +} + +static int page_concurrency(unsigned int mode) +{ + struct context c; + struct page_race r = { .c = &c }; + pthread_t threads[4]; + int file = memfd_create("page-array-race", 0); + + CHECK(file >= 0 && !ftruncate(file, 2 * native_ps)); + r.p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(r.p != MAP_FAILED); + memset(r.p, 0x49, 2 * native_ps); + CHECK(!context_open_common(&c, mode, false, true) && + !limit_bytes(8 * native_ps)); + CHECK(!map_range(&c, r.p, 2 * native_ps, IOVA)); + CHECK(!pthread_barrier_init(&r.barrier, NULL, 4)); + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_create( + &threads[i], NULL, + i == 3 ? page_unmap_thread : page_pin_thread, &r)); + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_join(threads[i], NULL)); + CHECK(!atomic_load(&r.failed) && atomic_load(&r.pinned) > 0); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(r.p, 2 * native_ps)); + context_close(&c); + close(file); + CHECK(!pthread_barrier_destroy(&r.barrier)); + printf("ok - native page arrays %zuK %s race: %d pins, %d unavailable, 256 remaps\n", + user_ps / 1024, mode ? "MM" : "USER", atomic_load(&r.pinned), + atomic_load(&r.absent)); + return 0; +} + +static int dirty_ioctl(struct context *c, uint32_t hwpt, void *bitmap, + unsigned long iova, size_t length, size_t granule, + unsigned int flags) +{ + struct iommu_hwpt_get_dirty_bitmap cmd = { .size = sizeof(cmd), + .hwpt_id = hwpt, + .flags = flags, + .iova = iova, + .length = length, + .page_size = granule, + .data = (uintptr_t)bitmap }; + + return ioctl(c->fd, IOMMU_HWPT_GET_DIRTY_BITMAP, &cmd); +} + +static int dirty_errors(unsigned int mode) +{ + struct context c; + size_t len = 66 * native_ps; + unsigned char *data = relocated(len, 0x34); + unsigned char *out = mmap(NULL, 2 * user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t *bitmap = (void *)(out + user_ps - 8), seed[2] = { 1, 2 }; + struct iommu_option option = { .size = sizeof(option), + .option_id = IOMMU_OPTION_HUGE_PAGES, + .op = IOMMU_OPTION_OP_SET, + .val64 = 0 }; + struct iommu_hwpt_alloc alloc = { + .size = sizeof(alloc), .flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING + }; + struct iommu_hwpt_set_dirty_tracking enable = { + .size = sizeof(enable), + .flags = IOMMU_HWPT_DIRTY_TRACKING_ENABLE + }; + struct iommu_test_cmd mark = { .size = sizeof(mark), + .op = IOMMU_TEST_OP_DIRTY, + .dirty = { .iova = IOVA, + .length = len, + .page_size = native_ps, + .uptr = (uintptr_t)seed } }; + unsigned int no_clear = IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR; + int failures = 0, rc, error; + + CHECK(data != MAP_FAILED && out != MAP_FAILED); + CHECK(!context_open(&c, mode) && !limit_bytes(128 * native_ps)); + option.object_id = c.ioas; + CHECK(!ioctl(c.fd, IOMMU_OPTION, &option)); + CHECK(!map_range(&c, data, len, IOVA)); + alloc.dev_id = c.dev[0]; + alloc.pt_id = c.ioas; + CHECK(!ioctl(c.fd, IOMMU_HWPT_ALLOC, &alloc)); + enable.hwpt_id = alloc.out_hwpt_id; + CHECK(!ioctl(c.fd, IOMMU_HWPT_SET_DIRTY_TRACKING, &enable)); + mark.id = alloc.out_hwpt_id; + CHECK(!ioctl(c.fd, IOMMU_TEST_CMD, &mark) && + mark.dirty.out_nr_dirty == 2); + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, + no_clear)); + CHECK(bitmap[0] == 1 && bitmap[1] == 2); + CHECK(filled(out, 0, user_ps - 8) && + filled(out + user_ps + 8, 0, user_ps - 8)); + +#define EXPECT_DIRTY_ERROR(call, expected, label) \ + do { \ + errno = 0; \ + rc = (call); \ + error = errno; \ + if (rc != -1 || error != (expected)) { \ + printf("FAIL dirty %s: rc=%d errno=%d expected=%d\n", \ + label, rc, error, expected); \ + failures++; \ + } \ + } while (0) + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, + IOVA + len, native_ps, native_ps, + no_clear), + EINVAL, "unmapped IOVA"); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, (void *)1, IOVA, + len, native_ps, no_clear), + EFAULT, "invalid output pointer"); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, + 4 * native_ps, 3 * native_ps, no_clear), + EINVAL, "non-power-of-two granule"); + CHECK(!mprotect(out, 2 * user_ps, PROT_READ)); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, + native_ps, no_clear), + EFAULT, "read-only output"); + CHECK(!mprotect(out, 2 * user_ps, PROT_READ | PROT_WRITE)); + memset(out, 0, 2 * user_ps); + CHECK(!munmap(out + user_ps, user_ps)); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, + native_ps, no_clear), + EFAULT, "output hole after valid prefix"); + CHECK(bitmap[0] == 1); + CHECK(mmap(out + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, + 0) == out + user_ps); + /* NO_CLEAR errors retain dirty state, so fixing the buffer permits retry. */ + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, + 0)); + CHECK(bitmap[0] == 1 && bitmap[1] == 2); + memset(out, 0, 2 * user_ps); + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, + no_clear)); + CHECK(bitmap[0] == 0 && bitmap[1] == 0); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!munmap(data, len) && !munmap(out, 2 * user_ps)); + context_close(&c); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!failures); +#undef EXPECT_DIRTY_ERROR + return 0; +} + +static int exercise(void) +{ + CHECK(!drop_pin_cap()); + for (unsigned int mode = 0; mode < 2; mode++) { + CHECK(!limits_and_lifetime(mode)); + CHECK(!holes_and_shared(mode)); + CHECK(!queue_ranges(mode)); + CHECK(!page_ranges(mode)); + CHECK(!page_lifetime(mode)); + CHECK(!cross_process(mode)); + CHECK(!concurrent_maps(mode)); + CHECK(!allocation_failures(mode)); + CHECK(!queue_failure_sweep(mode)); + CHECK(!page_failure_sweep(mode)); + CHECK(!page_concurrency(mode)); + CHECK(!dirty_errors(mode)); + printf("ok - IOMMUFD %zuK userspace/%zuK kernel %s accounting, limits/lifetime/shared/cross-process/256 concurrent cycles\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + } + return 0; +} + +int main(int argc, char **argv) +{ + int fail = 0; + + setbuf(stdout, NULL); + if (argc >= 4) { + user_ps = strtoul(argv[2], NULL, 0); + native_ps = strtoul(argv[3], NULL, 0); + CHECK(getauxval(AT_PAGESZ) == user_ps); + if (!strcmp(argv[1], "--exercise")) + return exercise(); + if (!strcmp(argv[1], "--producer") && argc == 8) + return producer(argc, argv); + return 2; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)); + CHECK(!mount("sysfs", "/sys", "sysfs", 0, NULL)); + CHECK(!mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)); + native_ps = getauxval(AT_PAGESZ); + size_t sizes[] = { native_ps, 4096, 16384 }; + + for (unsigned int i = 0; i < (native_ps > 16384 ? 3U : 2U); i++) { + pid_t child = fork(); + int status = 0, ok = 0; + + if (!child) { + char size[24], native[24]; + + snprintf(size, sizeof(size), "%zu", sizes[i]); + snprintf(native, sizeof(native), "%zu", native_ps); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", size, native, + NULL); + _exit(5); + } + for (unsigned int ticks = 0; child > 0 && ticks < 1800; + ticks++) { + if (waitpid(child, &status, WNOHANG) == child) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 1799) { + kill(child, SIGKILL); + waitpid(child, &status, 0); + } + usleep(100000); + } + printf("%s - IOMMUFD ABI %zuK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } + printf("IOMMUFD IOCTL CONTRACT %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c b/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c new file mode 100644 index 0000000000000..b652200c38b42 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static unsigned long stat_value(const char *path, const char *key) +{ + char line[256], name[100]; + unsigned long value = 0; + FILE *f = fopen(path, "re"); + + if (!f) + return 0; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "%99s %lu", name, &value) == 2 && + !strcmp(name, key)) + break; + fclose(f); + return value; +} + +static bool all_bytes(const unsigned char *p, size_t len, unsigned char value) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value) + return false; + return true; +} + +static unsigned long lazy_kb(void *address) +{ + FILE *f = fopen("/proc/self/smaps", "re"); + char line[256]; + unsigned long first, last, value = ~0UL; + bool found = false; + + if (!f) + return value; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &first, &last) == 2) { + if (found) + break; + found = (uintptr_t)address >= first && + (uintptr_t)address < last; + } else if (found && + sscanf(line, "LazyFree: %lu", &value) == 1) { + break; + } + } + fclose(f); + return value; +} + +static uint64_t pagemap(void *p, size_t ps) +{ + uint64_t entry = 0; + int fd = open("/proc/self/pagemap", O_RDONLY); + + if (fd < 0 || + pread(fd, &entry, sizeof(entry), (uintptr_t)p / ps * 8) != 8) + ksft_exit_fail_msg("pagemap: %s\n", strerror(errno)); + close(fd); + return entry; +} + +static bool pageout(void *p, size_t len) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(i, &one); + ok &= !sched_setaffinity(0, sizeof(one), &one); + ok &= !madvise(p, len, MADV_COLD); + ok &= !madvise(p, len, MADV_PAGEOUT); + } + return !sched_setaffinity(0, sizeof(allowed), &allowed) && ok; +} + +static bool mark_free(void *p, size_t len) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(i, &one); + ok &= !sched_setaffinity(0, sizeof(one), &one); + ok &= !madvise(p, len, MADV_FREE); + ok &= !madvise(p, len, MADV_COLD); + } + return !sched_setaffinity(0, sizeof(allowed), &allowed) && ok; +} + +static int exercise(unsigned long native) +{ + const size_t ps = getauxval(AT_PAGESZ), len = 4 * ps; + unsigned char *reservation = mmap(NULL, len + 2 * native, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *p = + (void *)(((uintptr_t)reservation + native - 1) & ~(native - 1)); + bool ok; + unsigned long writes; + uint64_t entry; + + if (reservation == MAP_FAILED || + mprotect(p, len, PROT_READ | PROT_WRITE)) + ksft_exit_fail_msg("mapping\n"); + ksft_print_header(); + ksft_set_plan(8); + for (unsigned int i = 0; i < 4; i++) + memset(p + i * ps, 0x40 + i, ps); + ok = mark_free(p + ps, ps); + if (!ok) + ksft_exit_fail_msg("%zuK MADV_FREE: %s\n", ps / 1024, + strerror(errno)); + ksft_test_result(all_bytes(p + ps, ps, 0x41), + "%zuK FREE retains bytes before reclaim\n", ps / 1024); + ksft_test_result(lazy_kb(p) == ps / 1024, + "%zuK partial LazyFree accounting\n", ps / 1024); + ok = pageout(p, len); + entry = pagemap(p + ps, ps); + ok &= !(entry & ((1ULL << 63) | (1ULL << 62))); + ok &= all_bytes(p + ps, ps, 0) && all_bytes(p, ps, 0x40) && + all_bytes(p + 2 * ps, ps, 0x42) && + all_bytes(p + 3 * ps, ps, 0x43); + ksft_test_result(ok, "%zuK partial FREE reclaims only advised bytes\n", + ps / 1024); + /* Drop old swap/cache state, then test cancellation by a real EL0 write. */ + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0x62, len); + ok &= mark_free(p + ps, ps); + memset(p + ps, 0xa5, ps); + ksft_test_result(lazy_kb(p) == 0, + "%zuK dirty leaf no longer reported LazyFree\n", + ps / 1024); + ok &= pageout(p, len); + ok &= all_bytes(p + ps, ps, 0xa5) && all_bytes(p, ps, 0x62) && + all_bytes(p + 2 * ps, 2 * ps, 0x62); + ksft_test_result(ok, "%zuK write after FREE cancels quarter discard\n", + ps / 1024); + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0x83, len); + writes = stat_value("/proc/vmstat", "pswpout"); + ok &= mark_free(p, len) && pageout(p, len); + for (unsigned int i = 0; i < 4; i++) + ok &= !(pagemap(p + i * ps, ps) & + ((1ULL << 63) | (1ULL << 62))); + printf("# full FREE pswpout=%lu->%lu, entries=%#llx %#llx %#llx %#llx\n", + writes, stat_value("/proc/vmstat", "pswpout"), + (unsigned long long)pagemap(p, ps), + (unsigned long long)pagemap(p + ps, ps), + (unsigned long long)pagemap(p + 2 * ps, ps), + (unsigned long long)pagemap(p + 3 * ps, ps)); + ok &= stat_value("/proc/vmstat", "pswpout") == writes && + all_bytes(p, len, 0); + ksft_test_result( + ok, "%zuK fully free backing reclaimed without swap writes\n", + ps / 1024); + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0xb7, len); + ok &= pageout(p, len); + ok &= !!(pagemap(p + ps, ps) & (1ULL << 62)); + ok &= mark_free(p + ps, ps); + ok &= !(pagemap(p + ps, ps) & ((1ULL << 63) | (1ULL << 62))); + ok &= all_bytes(p + ps, ps, 0) && all_bytes(p, ps, 0xb7) && + all_bytes(p + 2 * ps, 2 * ps, 0xb7); + ksft_test_result( + ok, + "%zuK FREE drops selected swap entry without neighbor loss\n", + ps / 1024); + { + int gate[2], status = 0; + pid_t pid; + char ready; + + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0xc8, len); + if (pipe(gate)) + ksft_exit_fail_msg("pipe\n"); + pid = fork(); + if (!pid) { + close(gate[1]); + if (read(gate[0], &ready, 1) != 1 || + !all_bytes(p, len, 0xc8)) + _exit(2); + _exit(0); + } + close(gate[0]); + ok &= mark_free(p + ps, ps) && pageout(p, len); + ok &= write(gate[1], "x", 1) == 1; + close(gate[1]); + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + ksft_test_result( + ok, + "%zuK FREE after fork preserves other process data\n", + ps / 1024); + } + munmap(reservation, len + 2 * native); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(strtoul(argv[3], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + swapon("/dev/vda", 0)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + char size[32], host_size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + snprintf(host_size, sizeof(host_size), "%lu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", size, host_size, + NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + printf("%s - lazyfree fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c b/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c new file mode 100644 index 0000000000000..f34602bb730e5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c @@ -0,0 +1,153 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/account" +#define LENGTH (32UL * 1024 * 1024) +static int put(const char *p, const char *v) +{ + int fd = open(p, O_WRONLY); + if (fd < 0) + return 1; + int bad = write(fd, v, strlen(v)) != (ssize_t)strlen(v); + close(fd); + return bad; +} +static unsigned long long stat_value(const char *key) +{ + FILE *f = fopen(CG "/memory.stat", "r"); + char name[128]; + unsigned long long value = 0; + if (!f) + exit(20); + while (fscanf(f, "%127s %llu", name, &value) == 2) + if (!strcmp(key, name)) { + fclose(f); + return value; + } + fclose(f); + exit(21); +} +static int worker(unsigned long expected, int ready, int release) +{ + if (getauxval(AT_PAGESZ) != expected) + return 2; + unsigned char *p = mmap(NULL, LENGTH, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || madvise(p, LENGTH, MADV_NOHUGEPAGE)) + return 3; + memset(p, 0x71, LENGTH); + char token = 'R'; + if (write(ready, &token, 1) != 1 || read(release, &token, 1) != 1) + return 4; + /* Keep the writes observable independently of the accounting check. */ + for (size_t i = 0; i < LENGTH; i++) + if (p[i] != 0x71) + return 5; + return munmap(p, LENGTH) ? 6 : 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (getppid() == 1 && argc == 5 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), atoi(argv[3]), + atoi(argv[4])); + if (getpid() != 1 || argc != 1) + return 2; + signal(SIGPIPE, SIG_IGN); + alarm(60); + int failed = 0; + unsigned long long slab[3] = {}; + unsigned long sizes[3] = { getauxval(AT_PAGESZ), 4096, 16384 }; + unsigned int modes = sizes[0] == 65536 ? 3 : 2; + if ((sizes[0] != 16384 && sizes[0] != 65536) || + mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) || + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory")) { + perror("setup"); + failed = 1; + goto done; + } + for (unsigned i = 0; i < modes; i++) { + int ready[2], release[2]; + if (mkdir(CG, 0755) || put(CG "/memory.max", "134217728") || + pipe(ready) || pipe(release)) { + failed = 1; + break; + } + pid_t child = fork(); + if (!child) { + char ps[32], rfd[16], cfd[16]; + close(ready[0]); + close(release[1]); + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(rfd, sizeof(rfd), "%d", ready[1]); + snprintf(cfd, sizeof(cfd), "%d", release[0]); + if (put(CG "/cgroup.procs", "0") || + prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(7); + execl("/init", "/init", "--worker", ps, rfd, cfd, NULL); + _exit(8); + } + close(ready[1]); + close(release[0]); + char token; + int ok = child > 0 && read(ready[0], &token, 1) == 1; + if (ok) { + /* Let per-CPU slab statistics become visible before the comparison. */ + usleep(1200000); + slab[i] = stat_value("slab_unreclaimable"); + printf("# ABI=%lu anon=%llu slab_unreclaimable=%llu pagetables=%llu\n", + sizes[i], stat_value("anon"), slab[i], + stat_value("pagetables")); + ok = stat_value("anon") >= LENGTH && + write(release[1], "G", 1) == 1; + } + int status = 0; + if (child > 0 && !ok) + kill(child, SIGKILL); + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && ok; + close(ready[0]); + close(release[1]); + printf("%s - memcg accounting ABI %luK worker status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + for (unsigned retry = 0; retry < 100 && stat_value("anon"); + retry++) + usleep(20000); + if (stat_value("anon") || rmdir(CG)) + failed = 1; + if (failed) + break; + } + /* Slot metadata must be charged for every alternative granule. The + * loose lower bound tolerates slab/KASAN layout differences. */ + for (unsigned i = 1; !failed && i < modes; i++) { + if (slab[i] < slab[0] + 512 * 1024) { + printf("not ok - slot metadata escaped cgroup accounting ABI=%lu native=%llu alternative=%llu\n", + sizes[i], slab[0], slab[i]); + failed = 1; + } + } + +done: + printf("GRANULE MEMCG ACCOUNTING %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c b/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c new file mode 100644 index 0000000000000..c2f50993ff805 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c @@ -0,0 +1,417 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/pressure" +#define MIB (1024UL * 1024) +#define LENGTH (32 * MIB) +#define ROUNDS 4 + +static int put(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + ssize_t ret = write(fd, value, strlen(value)); + int saved = errno; + close(fd); + errno = saved; + return ret == (ssize_t)strlen(value) ? 0 : -1; +} + +static unsigned long long get(const char *path, const char *key) +{ + FILE *f = fopen(path, "r"); + char name[128]; + unsigned long long value = 0, result = 0; + if (!f) { + perror(path); + exit(10); + } + if (!key) { + if (fscanf(f, "%llu", &result) != 1) + exit(11); + } else + while (fscanf(f, "%127s %llu", name, &value) == 2) + if (!strcmp(name, key)) { + result = value; + break; + } + fclose(f); + return result; +} + +static unsigned char pattern(size_t offset, unsigned seed, unsigned round) +{ + return (offset ^ (offset >> 12) ^ (offset >> 20) ^ (seed * 79) ^ + (round * 43)) & + 255; +} + +static int contents(unsigned char *p, size_t ps, unsigned seed, unsigned round, + int write_data, int child) +{ + size_t pages = LENGTH / ps; + /* Odd multiplication permutes every logical page of this power-of-two range. */ + for (size_t i = 0; i < pages; i++) { + size_t page = (i * 4051 + round * 31) & (pages - 1); + for (size_t j = 0; j < ps; j++) { + size_t offset = page * ps + j; + unsigned char want = pattern(offset, seed, round); + if (child && !(page & 3)) + want ^= 0x69; + if (write_data) + p[offset] = want; + else if (p[offset] != want) { + printf("not ok - ABI=%zu round=%u child=%d byte=%zu got=%u expected=%u\n", + ps, round, child, offset, p[offset], + want); + return 1; + } + } + } + return 0; +} + +static int barrier(int ready, int command) +{ + char token = 'R'; + return write(ready, &token, 1) != 1 || read(command, &token, 1) != 1 || + token != 'G'; +} + +static int worker(unsigned long expected, unsigned seed, int ready, int command) +{ + size_t ps = getauxval(AT_PAGESZ); + unsigned char *p; + if (ps != expected) + return 2; + p = mmap(NULL, LENGTH, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 3; + if (madvise(p, LENGTH, MADV_NOHUGEPAGE)) { + int error = errno; + struct stat st; + + /* Without THP this advice is unavailable and cannot change policy. */ + if (error != EINVAL || + !stat("/sys/kernel/mm/transparent_hugepage/enabled", &st) || + errno != ENOENT) + return 3; + } + if (contents(p, ps, seed, 0, 1, 0) || barrier(ready, command)) + return 4; + for (unsigned round = 0; round < ROUNDS; round++) { + if (contents(p, ps, seed, round, 0, 0)) + return 5; + pid_t child = fork(); + if (child < 0) + return 6; + if (!child) { + /* Dirty a quarter of the logical pages, retaining untouched siblings. */ + for (size_t page = 0; page < LENGTH / ps; page += 4) + for (size_t j = 0; j < ps; j++) + p[page * ps + j] ^= 0x69; + _exit(contents(p, ps, seed, round, 0, 1) ? 7 : 0); + } + int status; + if (waitpid(child, &status, 0) != child || !WIFEXITED(status) || + WEXITSTATUS(status) || contents(p, ps, seed, round, 0, 0)) + return 8; + printf("ok - memcg ABI %zuK round=%u swap-in bytes and fork COW isolation\n", + ps / 1024, round); + if (round + 1 < ROUNDS && + contents(p, ps, seed, round + 1, 1, 0)) + return 9; + if (barrier(ready, command)) + return 10; + } + return munmap(p, LENGTH) ? 11 : 0; +} + +static void report(unsigned phase) +{ + printf("# memcg phase=%u resident=%llu swap=%llu anon=%llu pgscan=%llu pgsteal=%llu max=%llu oom=%llu oom_kill=%llu\n", + phase, get(CG "/memory.current", NULL), + get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "anon"), get(CG "/memory.stat", "pgscan"), + get(CG "/memory.stat", "pgsteal"), + get(CG "/memory.events", "max"), get(CG "/memory.events", "oom"), + get(CG "/memory.events", "oom_kill")); +} + +static void pressure_stacks(void) +{ + FILE *tasks = fopen(CG "/cgroup.procs", "r"); + int pid; + + if (!tasks) + return; + while (fscanf(tasks, "%d", &pid) == 1) { + char path[64], stack[4096]; + int fd; + ssize_t n; + + snprintf(path, sizeof(path), "/proc/%d/stack", pid); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + continue; + n = read(fd, stack, sizeof(stack) - 1); + close(fd); + if (n > 0) { + stack[n] = 0; + printf("# pressure stack pid=%d\n%s", pid, stack); + } + } + fclose(tasks); +} + +static void observe_pressure(void) +{ + unsigned int samples = 0; + + for (;;) { + struct timespec now; + + clock_gettime(CLOCK_MONOTONIC, &now); + printf("# io-pressure ms=%lld resident=%llu swap=%llu writeback=%llu " + "swapcached=%llu scan=%llu steal=%llu max=%llu oom=%llu kill=%llu\n", + (long long)now.tv_sec * 1000 + now.tv_nsec / 1000000, + get(CG "/memory.current", NULL), + get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "file_writeback"), + get(CG "/memory.stat", "swapcached"), + get(CG "/memory.stat", "pgscan"), + get(CG "/memory.stat", "pgsteal"), + get(CG "/memory.events", "max"), + get(CG "/memory.events", "oom"), + get(CG "/memory.events", "oom_kill")); + if (!(++samples % 8) && + get(CG "/memory.stat", "file_writeback") > 8 * MIB) + pressure_stacks(); + usleep(250000); + } +} + +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker") && getppid() == 1) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4]), + atoi(argv[5])); + bool classic = false, native_control = false, user16k = false; + + if (getpid() != 1 || argc > 3) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--classic")) + classic = true; + else if (!strcmp(argv[i], "--native-control")) + native_control = true; + else if (!strcmp(argv[i], "--user16k")) + user16k = true; + else + return 2; + } + if (native_control && user16k) + return 2; + signal(SIGPIPE, SIG_IGN); + alarm(300); + int failed = 0, active = 0; + pid_t children[2] = { -1, -1 }, observer = -1; + int ready[2][2], command[2][2]; + unsigned long sizes[2] = { getauxval(AT_PAGESZ), 4096 }; + if (native_control) + sizes[1] = sizes[0]; + if (user16k) + sizes[1] = 16384; + if ((sizes[0] != 16384 && sizes[0] != 65536) || sizes[1] > sizes[0]) { + failed = 1; + goto done; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) || + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory") || + mkdir(CG, 0755) || put(CG "/memory.max", "50331648") || + put(CG "/memory.swap.max", "268435456")) { + perror("cgroup setup"); + failed = 1; + goto done; + } + /* Explicit diagnostic control, never change the production default. */ + if (classic && put("/sys/kernel/mm/lru_gen/enabled", "0")) { + perror("classic reclaim control"); + failed = 1; + goto done; + } + printf("# reclaim mode=%s\n", + classic ? "classic diagnostic" : "default"); + struct stat st; + for (int i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (swapon("/dev/vda", 0)) { + perror("swapon"); + failed = 1; + goto done; + } + active = 1; + for (unsigned i = 0; i < 2; i++) + if (pipe(ready[i]) || pipe(command[i])) { + failed = 1; + goto done; + } + observer = fork(); + if (!observer) { + for (unsigned int i = 0; i < 2; i++) { + close(ready[i][0]); + close(ready[i][1]); + close(command[i][0]); + close(command[i][1]); + } + observe_pressure(); + _exit(0); + } + if (observer < 0) { + failed = 1; + goto done; + } + for (unsigned i = 0; i < 2; i++) { + children[i] = fork(); + if (!children[i]) { + char ps[32], index[16], rfd[16], cfd[16]; + for (unsigned j = 0; j < 2; j++) { + close(ready[j][0]); + close(command[j][1]); + if (i != j) { + close(ready[j][1]); + close(command[j][0]); + } + } + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(index, sizeof(index), "%u", i); + snprintf(rfd, sizeof(rfd), "%d", ready[i][1]); + snprintf(cfd, sizeof(cfd), "%d", command[i][0]); + if (put(CG "/cgroup.procs", "0") || + prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(12); + execl("/init", "/init", "--worker", ps, index, rfd, cfd, + NULL); + _exit(13); + } + if (children[i] < 0) { + failed = 1; + goto done; + } + } + for (unsigned i = 0; i < 2; i++) { + close(ready[i][1]); + close(command[i][0]); + } + for (unsigned phase = 0; phase <= ROUNDS; phase++) { + char token; + for (unsigned i = 0; i < 2; i++) + if (read(ready[i][0], &token, 1) != 1 || token != 'R') { + failed = 1; + goto done; + } + report(phase); + if (!get(CG "/memory.swap.current", NULL) || + !get(CG "/memory.events", "max") || + !get(CG "/memory.stat", "pgscan") || + !get(CG "/memory.stat", "pgsteal") || + get(CG "/memory.events", "oom") || + get(CG "/memory.events", "oom_kill")) { + printf("not ok - missing pressure evidence or unexpected OOM\n"); + failed = 1; + goto done; + } + for (unsigned i = 0; i < 2; i++) + if (write(command[i][1], "G", 1) != 1) { + failed = 1; + goto done; + } + } + for (unsigned i = 0; i < 2; i++) { + int status = 0; + int ok = waitpid(children[i], &status, 0) == children[i] && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - memcg ABI %luK exit=%#x\n", ok ? "ok" : "not ok", + sizes[i] / 1024, status); + children[i] = -1; + failed |= !ok; + } + /* Pending swap writeback can outlive exit, especially on a slow device. */ + for (unsigned i = 0; i < 3000 && (get(CG "/memory.stat", "anon") || + get(CG "/memory.swap.current", NULL)); + i++) { + if (!(i % 250)) + printf("# exit drain ms=%u swap=%llu writeback=%llu\n", + i * 20, get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "file_writeback")); + /* Exit may leave clean, unmapped swap cache after writeback. Reclaim + * it explicitly; EAGAIN means fewer than the requested bytes existed. */ + if (!get(CG "/memory.stat", "file_writeback") && + put(CG "/memory.reclaim", "67108864 swappiness=200") && + errno != EAGAIN) { + perror("exit cache reclaim"); + failed = 1; + break; + } + usleep(20000); + } + report(ROUNDS + 1); + if (get(CG "/memory.stat", "anon") || + get(CG "/memory.swap.current", NULL)) { + printf("not ok - anonymous or swap charge remains after exit\n"); + failed = 1; + } +done: + if (observer > 0) { + kill(observer, SIGTERM); + waitpid(observer, NULL, 0); + } + if (active && failed) + report(ROUNDS + 2); + for (unsigned i = 0; i < 2; i++) + if (children[i] > 0) { + kill(children[i], SIGKILL); + waitpid(children[i], NULL, 0); + } + if (active && failed) { + put(CG "/cgroup.kill", "1"); + put(CG "/memory.max", "max"); + } + if (active && swapoff("/dev/vda")) { + perror("swapoff"); + failed = 1; + } + printf("GRANULE MEMCG PRESSURE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/memory-interfaces.c b/tools/testing/selftests/arm64/user4k/memory-interfaces.c new file mode 100644 index 0000000000000..330296ed44499 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memory-interfaces.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static size_t page; + +static void residency(void) +{ + unsigned char *p, vec[6]; + bool ok; + unsigned int i; + + p = mmap(NULL, 5 * page, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap\n"); + memset(vec, 0x55, sizeof(vec)); + ok = mincore(p, 5 * page - 1, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == 0; + ok &= vec[5] == 0x55; + ksft_test_result( + ok, "mincore emits exactly one byte per unfaulted user page\n"); + ok = mprotect(p + 2 * page, page, PROT_READ | PROT_WRITE) == 0; + p[2 * page] = 0x37; + ok &= mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == (i == 2); + ksft_test_result( + ok, + "mincore reports a separately faulted VMA among cold neighbors\n"); + ok = munmap(p + 3 * page, page) == 0; + errno = 0; + ok &= mincore(p, 5 * page, vec) == -1 && errno == ENOMEM; + errno = 0; + ok &= mincore(p + 1, page, vec) == -1 && errno == EINVAL; + ksft_test_result(ok, + "mincore rejects an unaligned start and a VMA hole\n"); + munmap(p, 5 * page); +} + +static void file_residency(void) +{ + const size_t native = 16384, count = 3 * native / page; + unsigned char *p, *guard, *vec, byte = 0x7b; + int fd = memfd_create("residency", 0); + bool ok; + size_t i; + + if (fd < 0 || ftruncate(fd, 4 * native) || + pwrite(fd, &byte, 1, native) != 1) + ksft_exit_fail_msg("memfd setup\n"); + p = mmap(NULL, 3 * native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page); + guard = mmap(NULL, 2 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || guard == MAP_FAILED || + mprotect(guard + page, page, PROT_NONE)) + ksft_exit_fail_msg("file/vector mmap\n"); + vec = guard + page - count; + memset(vec, 0x55, count); + ok = mincore(p, 3 * native, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == ((page + i * page) / native == 1); + ksft_test_result( + ok, + "unmapped file leaves share residency at native cache indices\n"); + ok = p[native - page] == byte && mincore(p, 3 * native, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == ((page + i * page) / native == 1); + ksft_test_result( + ok, + "present and cache-only file residency agree at a guarded vector boundary\n"); + ok = msync(p + page, 1, MS_SYNC) == 0; + errno = 0; + ok &= msync(p + page + 1, page, MS_SYNC) == -1 && errno == EINVAL; + ok &= munmap(p + page, page) == 0; + errno = 0; + ok &= msync(p, 3 * page, MS_SYNC) == -1 && errno == ENOMEM; + ksft_test_result(ok, + "msync uses process alignment and reports holes\n"); + munmap(p, 3 * native); + munmap(guard, 2 * page); + close(fd); +} + +static void large_vector(void) +{ + const size_t count = 16384 + 3; + unsigned char *p = mmap(NULL, count * page, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, + 0); + unsigned char *vec = malloc(count + 1); + bool ok; + size_t i; + + if (p == MAP_FAILED || !vec) + ksft_exit_fail_msg("large vector allocation\n"); + memset(vec, 0x55, count + 1); + ok = mincore(p, count * page - 7, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == 0; + ok &= vec[count] == 0x55; + ksft_test_result( + ok, "mincore advances across multiple native output buffers\n"); + free(vec); + munmap(p, count * page); +} + +static void seals(void) +{ + unsigned char *p = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + bool ok; + pid_t pid; + int status = 0; + + if (p == MAP_FAILED) + ksft_exit_fail_msg("seal mmap\n"); + ok = syscall(SYS_mseal, p + page, 1UL, 0UL) == 0; + errno = 0; + ok &= mprotect(p + page, page, PROT_READ) == -1 && errno == EPERM; + errno = 0; + ok &= munmap(p + page, page) == -1 && errno == EPERM; + ok &= mprotect(p, page, PROT_READ) == 0; + ok &= mprotect(p + 2 * page, page, PROT_READ) == 0; + ksft_test_result( + ok, + "a one-byte seal covers only its user page and protects it\n"); + pid = fork(); + if (!pid) + _exit(!(munmap(p + page, page) == -1 && errno == EPERM)); + ok = pid > 0 && waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status); + ksft_test_result(ok, "fork inherits the sealed user page\n"); + munmap(p, page); + munmap(p + 2 * page, page); + /* The sealed VMA is released by process exit. */ +} + +int main(int argc, char **argv) +{ + page = getauxval(AT_PAGESZ); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + ksft_exit_fail_msg("unexpected process page ABI: %zu\n", page); + ksft_print_msg("process page size: %zu\n", page); + ksft_print_header(); + ksft_set_plan(9); + residency(); + file_residency(); + large_vector(); + seals(); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/mglru-granule-init.c b/tools/testing/selftests/arm64/user4k/mglru-granule-init.c new file mode 100644 index 0000000000000..9bb9bb74e38e8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mglru-granule-init.c @@ -0,0 +1,365 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CONTROL "/sys/kernel/debug/lru_gen" +#define FULL "/sys/kernel/debug/lru_gen_full" +static int ready_fd = -1, command_fd = -1; +static unsigned worker_index; +static unsigned char pattern(unsigned region, size_t byte, unsigned round) +{ + return (byte ^ (byte >> 12) ^ (region * 37) ^ (round * 53)) & 255; +} +static int generation(unsigned long long *cg, unsigned *node, + unsigned long *max) +{ + FILE *f = fopen(CONTROL, "r"); + char line[256]; + int have_cg = 0, have_node = 0, have_seq = 0; + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) { + unsigned long long id; + unsigned n; + unsigned long seq, age, anon, file; + if (sscanf(line, "memcg %llu", &id) == 1) { + if (have_cg) + break; + *cg = id; + have_cg = 1; + } else if (sscanf(line, " node %u", &n) == 1) { + if (have_node) + break; + *node = n; + have_node = 1; + } else if (have_node && sscanf(line, "%lu %lu %lu %lu", &seq, + &age, &anon, &file) == 4) { + *max = seq; + have_seq = 1; + } + } + fclose(f); + return have_seq ? 0 : -1; +} +static unsigned long scan_evidence(void) +{ + FILE *f = fopen(FULL, "r"); + char line[512]; + unsigned long total = 0; + if (!f) + return 0; + while (fgets(line, sizeof(line), f)) { + unsigned long t, y, found, added; + if (sscanf(line, " %lut %luy %luf %lua", &t, &y, &found, + &added) == 4) { + printf("# MGLRU scan total=%lu young=%lu tables=%lu added=%lu\n", + t, y, found, added); + total += y; + } + } + fclose(f); + return total; +} +static int age(void) +{ + if (ready_fd >= 0) { + char token = 'A'; + return write(ready_fd, &token, 1) != 1 || + read(command_fd, &token, 1) != 1 || token != 'G'; + } + unsigned long long cg = 0, next_cg = 0; + unsigned node = 0, next_node = 0; + unsigned long seq = 0, next = 0; + char command[128]; + if (generation(&cg, &node, &seq)) + return 1; + /* This tree accepts swappiness here; 200 scans both anonymous and file pages. */ + int len = snprintf(command, sizeof(command), "+ %llu %u %lu 200 1\n", + cg, node, seq); + int fd = open(CONTROL, O_WRONLY); + if (fd < 0) + return 1; + ssize_t written = write(fd, command, len); + close(fd); + if (written != len || generation(&next_cg, &next_node, &next) || + next <= seq) { + printf("not ok - MGLRU generation command=%s result=%zd errno=%d next=%lu\n", + command, written, errno, next); + return 1; + } + printf("ok - MGLRU generation %lu -> %lu\n", seq, next); + return 0; +} +static int check_referenced(unsigned char **regions, unsigned nr, + size_t expected) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char line[512]; + int region = -1, failed = 0; + unsigned seen = 0; + unsigned long start, end, kb; + if (!f) + return 1; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + region = -1; + for (unsigned i = 0; i < nr; i++) + if (start == (uintptr_t)regions[i]) + region = i; + } else if (region >= 0 && + sscanf(line, "Referenced: %lu kB", &kb) == 1) { + printf("%s - region=%d start=%p referenced=%lu expected=%zu kB\n", + kb == expected ? "ok" : "not ok", region, + regions[region], kb, expected); + failed |= kb != expected; + seen++; + } + } + fclose(f); + return failed || seen != nr; +} +static int exercise(unsigned long expected) +{ + static const unsigned shifts[] = { 21, 25, 29, 30, 36, 39, 42, 47 }; + unsigned char *regions[sizeof(shifts) / sizeof(shifts[0])] = {}; + size_t ps = getauxval(AT_PAGESZ), len = 8 * ps; + unsigned nr = sizeof(shifts) / sizeof(shifts[0]); + int fail = 0, fd = memfd_create("granule-aging", 0); + cpu_set_t cpus; + void *scratch = MAP_FAILED; + CPU_ZERO(&cpus); + CPU_SET(ready_fd >= 0 ? worker_index % 2 : 0, &cpus); + if (sched_setaffinity(0, sizeof(cpus), &cpus)) + return 1; + if (ps != expected || fd < 0 || ftruncate(fd, nr * len)) + return 1; + for (unsigned i = 0; i < nr; i++) { + uintptr_t address = (1UL << shifts[i]) - 4 * ps; + regions[i] = mmap((void *)address, len, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | + (i % 2 ? MAP_SHARED : + MAP_PRIVATE | MAP_ANONYMOUS), + i % 2 ? fd : -1, i % 2 ? i * len : 0); + if (regions[i] != (void *)address) { + printf("not ok - %zuK boundary %u mmap errno=%d\n", + ps / 1024, shifts[i], errno); + fail = 1; + goto out; + } + } + for (unsigned i = 0; i < nr; i++) + memset(regions[i], 0x19, len); + /* Publish earlier folios from the bounded per-CPU LRU-add batch. */ + scratch = mmap(NULL, 8UL << 20, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (scratch == MAP_FAILED) { + fail = 1; + goto out; + } + memset(scratch, 0x7b, 8UL << 20); + /* Seed MGLRU's first-access referenced state before testing the next access. */ + if (age()) { + fail = 1; + goto out; + } + for (unsigned round = 0; round < 4; round++) { + if (ready_fd >= 0) { + unsigned cpu = (worker_index + round) % 2; + CPU_ZERO(&cpus); + CPU_SET(cpu, &cpus); + if (sched_setaffinity(0, sizeof(cpus), &cpus) || + sched_getcpu() != (int)cpu) { + fail = 1; + break; + } + printf("# concurrent ABI=%zuK round=%u cpu=%u\n", + ps / 1024, round, cpu); + } + for (unsigned i = 0; i < nr; i++) + for (size_t j = 0; j < len; j++) + regions[i][j] = pattern(i, j, round); + fail |= check_referenced(regions, nr, len / 1024); + if (age() || age()) { + fail = 1; + break; + } + fail |= check_referenced(regions, nr, 0); + int good = 1; + for (unsigned i = 0; i < nr; i++) + for (size_t j = 0; j < len; j++) + if (regions[i][j] != pattern(i, j, round)) { + good = 0; + break; + } + unsigned long young = scan_evidence(); + printf("%s - %zuK round%u eight table boundaries bytes and aging young=%lu\n", + good && young ? "ok" : "not ok", ps / 1024, round, + young); + fail |= !good || !young; + } +out: + if (scratch != MAP_FAILED) + munmap(scratch, 8UL << 20); + for (unsigned i = 0; i < nr; i++) + if (regions[i] && regions[i] != MAP_FAILED) + munmap(regions[i], len); + close(fd); + return fail; +} +static int concurrent(unsigned long *sizes, unsigned nr) +{ + int ready[3][2], commands[3][2], failed = 0; + pid_t workers[3]; + + for (unsigned i = 0; i < nr; i++) + if (pipe(ready[i]) || pipe(commands[i])) + return 1; + for (unsigned i = 0; i < nr; i++) { + workers[i] = fork(); + if (!workers[i]) { + char size[32], index[16], ready_arg[16], + command_arg[16]; + for (unsigned j = 0; j < nr; j++) { + close(ready[j][0]); + close(commands[j][1]); + if (j != i) { + close(ready[j][1]); + close(commands[j][0]); + } + } + snprintf(size, sizeof(size), "%lu", sizes[i]); + snprintf(index, sizeof(index), "%u", i); + snprintf(ready_arg, sizeof(ready_arg), "%d", + ready[i][1]); + snprintf(command_arg, sizeof(command_arg), "%d", + commands[i][0]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--worker", size, index, + ready_arg, command_arg, NULL); + _exit(4); + } + if (workers[i] < 0) + failed = 1; + } + for (unsigned i = 0; i < nr; i++) { + close(ready[i][1]); + close(commands[i][0]); + } + /* One warm-up barrier plus two aging barriers for each of four rounds. */ + for (unsigned phase = 0; phase < 9 && !failed; phase++) { + for (unsigned i = 0; i < nr; i++) { + char token; + if (read(ready[i][0], &token, 1) != 1 || token != 'A') + failed = 1; + } + if (failed || age()) { + failed = 1; + break; + } + for (unsigned i = 0; i < nr; i++) { + char token = 'G'; + if (write(commands[i][1], &token, 1) != 1) + failed = 1; + } + } + for (unsigned i = 0; i < nr; i++) { + close(ready[i][0]); + close(commands[i][1]); + } + for (unsigned i = 0; i < nr; i++) { + int status = 0; + int ok = workers[i] > 0 && + waitpid(workers[i], &status, 0) == workers[i] && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - concurrent MGLRU ABI %luK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + } + return failed; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (argc == 6 && !strcmp(argv[1], "--worker")) { + if (getppid() != 1) + return 2; + worker_index = strtoul(argv[3], NULL, 10); + ready_fd = atoi(argv[4]); + command_fd = atoi(argv[5]); + return exercise(strtoul(argv[2], NULL, 10)); + } + int concurrent_mode = argc == 2 && !strcmp(argv[1], "--concurrent"); + if (getpid() != 1 || (argc != 1 && !concurrent_mode)) + return 2; + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + FILE *f = fopen("/sys/kernel/mm/lru_gen/enabled", "r"); + unsigned enabled = 0; + if (f) { + fscanf(f, "%x", &enabled); + fclose(f); + } + if ((enabled & 3) != 3) { + printf("not ok - MGLRU/MM walk disabled %#x\n", enabled); + fail = 1; + goto done; + } + printf("ok - MGLRU enabled=%#x\n", enabled); + unsigned long sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + if (concurrent_mode) { + fail = concurrent(sizes, sizes[0] > 16384 ? 3 : 2); + goto done; + } + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t child = fork(); + int status = 0; + if (!child) { + char value[32]; + snprintf(value, sizeof(value), "%lu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--exercise", value, NULL); + _exit(4); + } + int good = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - MGLRU ABI %luK status=%#x\n", + good ? "ok" : "not ok", sizes[i] / 1024, status); + fail |= !good; + } +done: + printf("GRANULE MGLRU%s %s\n", concurrent_mode ? " CONCURRENT" : "", + fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/mixed-process.c b/tools/testing/selftests/arm64/user4k/mixed-process.c new file mode 100644 index 0000000000000..6d08ab55991b5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mixed-process.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real EL0 mixed-granule sharing, futex queues, and process_vm copies. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define NATIVE 16384U +#define QUARTER 4096U +#define ROUNDS 2000U +struct channel { + atomic_int turn; + unsigned int value; + unsigned int worker_wakes; +}; +struct ready { + uintptr_t base; + unsigned long page; +}; + +static int wait_turn(struct channel *channel, int wanted) +{ + const struct timespec timeout = { .tv_sec = 5 }; + int value; + + while ((value = atomic_load_explicit(&channel->turn, + memory_order_acquire)) != wanted) { + if (syscall(SYS_futex, &channel->turn, FUTEX_WAIT, value, + &timeout, NULL, 0) < 0 && + errno != EAGAIN && errno != EINTR) + return -1; + } + return 0; +} + +static int send_turn(struct channel *channel, int turn) +{ + atomic_store_explicit(&channel->turn, turn, memory_order_release); + return syscall(SYS_futex, &channel->turn, FUTEX_WAKE, 1, NULL, NULL, 0); +} + +static unsigned char pattern(size_t i) +{ + return (i ^ (i >> 8) ^ (i >> 16)) & 255; +} + +static int worker(int fd, int ready_fd, unsigned long expected) +{ + unsigned long page = getauxval(AT_PAGESZ); + unsigned char *mapping, *remote; + struct channel *channel; + struct ready ready; + size_t i, begin = page - 13, end = 3 * page + 16; + unsigned int round, wakes = 0; + int ret; + + if (page != expected) + return 10; + mapping = mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page == QUARTER ? QUARTER : 0); + remote = mmap(NULL, 5 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED || remote == MAP_FAILED) + return 11; + channel = (void *)(mapping + (page == QUARTER ? 0 : QUARTER)); + for (i = 0; i < 4 * page; i++) + remote[i] = pattern(i); + if (mprotect(remote + 4 * page, page, PROT_NONE)) + return 12; + ready = (struct ready){ (uintptr_t)remote, page }; + if (write(ready_fd, &ready, sizeof(ready)) != sizeof(ready)) + return 13; + close(ready_fd); + for (round = 0; round < ROUNDS; round++) { + if (wait_turn(channel, 1) || + channel->value != (round ^ 0xabcddcbaU)) + return 14; + if (!round) { + for (i = 0; i < 4 * page; i++) + if (remote[i] != + (pattern(i) ^ + (i >= begin && i < end ? 0x5a : 0))) + return 15; + } + channel->value = ~(round ^ 0xabcddcbaU); + ret = send_turn(channel, 0); + if (ret < 0) + return 16; + wakes += ret; + } + channel->worker_wakes = wakes; + return 0; +} + +static bool remote_copies(pid_t pid, const struct ready *ready) +{ + size_t begin = ready->page - 13, len = 2 * ready->page + 29, i; + unsigned char *data = malloc(len), edge[2] = {}; + struct iovec local = { data, len }, + remote = { (void *)(ready->base + begin), len }; + bool ok = data != NULL; + + if (!ok) + return false; + ok = process_vm_readv(pid, &local, 1, &remote, 1, 0) == (ssize_t)len; + for (i = 0; ok && i < len; i++) + ok = data[i] == pattern(i + begin); + for (i = 0; i < len; i++) + data[i] = pattern(i + begin) ^ 0x5a; + ok &= process_vm_writev(pid, &local, 1, &remote, 1, 0) == (ssize_t)len; + local = (struct iovec){ edge, sizeof(edge) }; + remote = (struct iovec){ (void *)(ready->base + 4 * ready->page - 1), + sizeof(edge) }; + ok &= process_vm_readv(pid, &local, 1, &remote, 1, 0) == 1; + ok &= edge[0] == pattern(4 * ready->page - 1); + free(data); + return ok; +} + +int main(int argc, char **argv) +{ + unsigned long page = getauxval(AT_PAGESZ), other; + unsigned char *mapping, *private; + struct channel *channel; + struct ready ready = {}; + int fd, pipefd[2], status = 0, wakes = 0, ret; + unsigned int round = 0; + pid_t pid; + bool ok; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10)); + if (argc != 1 || (page != QUARTER && page != NATIVE)) + return 2; + other = page == NATIVE ? QUARTER : NATIVE; + fd = memfd_create("mixed-process", 0); + if (fd < 0 || ftruncate(fd, 2 * NATIVE) || pipe(pipefd)) + return 3; + mapping = mmap(NULL, 2 * NATIVE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + private = + mmap(NULL, NATIVE, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + if (mapping == MAP_FAILED || private == MAP_FAILED) + return 4; + channel = (void *)(mapping + QUARTER); + private[QUARTER] = 0x7e; + pid = fork(); + if (!pid) { + char fdarg[32], pipearg[32], sizearg[32]; + + close(pipefd[0]); + snprintf(fdarg, sizeof(fdarg), "%d", fd); + snprintf(pipearg, sizeof(pipearg), "%d", pipefd[1]); + snprintf(sizearg, sizeof(sizearg), "%lu", other); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, other, 0UL, 0UL, 0UL)) + _exit(20); + execl("/mixed-process", "/mixed-process", "--worker", fdarg, + pipearg, sizearg, NULL); + _exit(21); + } + close(pipefd[1]); + if (pid < 0) + return 5; + ok = read(pipefd[0], &ready, sizeof(ready)) == sizeof(ready) && + ready.page == other; + close(pipefd[0]); + ok = ok && remote_copies(pid, &ready); + printf("%s - process_vm copies %lu -> %lu, including partial inaccessible boundary\n", + ok ? "ok" : "not ok", page, other); + for (round = 0; ok && round < ROUNDS; round++) { + channel->value = round ^ 0xabcddcbaU; + ret = send_turn(channel, 1); + ok = ret >= 0 && !wait_turn(channel, 0) && + channel->value == ~(round ^ 0xabcddcbaU); + if (ret > 0) + wakes += ret; + } + if (!ok) + kill(pid, SIGKILL); + ret = waitpid(pid, &status, 0); + ok &= ret == pid && WIFEXITED(status) && !WEXITSTATUS(status); + ok &= wakes > 0 && channel->worker_wakes > 0 && + private[QUARTER] == 0x7e; + ok &= mapping[0] == 0 && mapping[2 * QUARTER] == 0 && + mapping[NATIVE - 1] == 0; + printf("%s - mixed %lu/%lu shared futex/COW: rounds=%u wakes=%d/%u child=%#x\n", + ok ? "ok" : "not ok", page, other, round, wakes, + channel->worker_wakes, status); + munmap(private, NATIVE); + munmap(mapping, 2 * NATIVE); + close(fd); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c b/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c new file mode 100644 index 0000000000000..0dc94eabcc4fb --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static long locked_kb(void) +{ + char line[256]; + long value = -1; + FILE *f = fopen("/proc/self/status", "re"); + + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmLck: %ld", &value) == 1) + break; + fclose(f); + return value; +} + +static bool wait_ok(pid_t pid) +{ + int status = 0; + + return pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); +} + +static bool remap_locked(unsigned long ps) +{ + unsigned char *p, *q, *target; + unsigned long kb = ps / 1024; + bool ok; + + p = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || mlock(p, 2 * ps)) + ksft_exit_fail_msg("mlock remap setup\n"); + memset(p, 0x37, 2 * ps); + ok = locked_kb() == (long)(2 * kb); + q = mremap(p, 2 * ps, 4 * ps, MREMAP_MAYMOVE); + if (q == MAP_FAILED) + ksft_exit_fail_msg("locked remap grow: %s\n", strerror(errno)); + ok &= locked_kb() == (long)(4 * kb); + for (unsigned long i = 0; i < 4 * ps; i++) + ok &= q[i] == (i < 2 * ps ? 0x37 : 0); + p = mremap(q, 4 * ps, 3 * ps, 0); + if (p == MAP_FAILED) + ksft_exit_fail_msg("locked remap shrink: %s\n", + strerror(errno)); + ok &= locked_kb() == (long)(3 * kb); + p[2 * ps] = 0xb6; + ok &= !munmap(p + ps, ps) && locked_kb() == (long)(2 * kb); + target = mmap(NULL, ps, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (target == MAP_FAILED) + ksft_exit_fail_msg("locked remap target\n"); + q = mremap(p, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, target); + if (q == MAP_FAILED) + ksft_exit_fail_msg("locked remap move: %s\n", strerror(errno)); + ok &= q == target && locked_kb() == (long)(2 * kb); + for (unsigned long i = 0; i < ps; i++) + ok &= q[i] == 0x37; + ok &= p[2 * ps] == 0xb6; + ok &= !munmap(q, ps) && locked_kb() == (long)kb; + ok &= !munmap(p + 2 * ps, ps) && locked_kb() == 0; + return ok; +} + +static int exercise(void) +{ + unsigned long ps = getauxval(AT_PAGESZ), kb = ps / 1024; + unsigned char *p, *q, residency[4]; + int gate[2]; + pid_t pid; + bool ok; + + ksft_print_header(); + ksft_set_plan(9); + p = mmap(NULL, 32 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || locked_kb() != 0) + ksft_exit_fail_msg("initial mapping/accounting\n"); + ok = !mlock(p + ps + 17, ps - 32) && locked_kb() == (long)kb; + ok &= !mincore(p + ps, ps, residency) && residency[0] & 1; + p[ps] = 0x51; + ksft_test_result( + ok, "%luK unaligned mlock uses exactly one user leaf\n", kb); + ok = !mlock(p + ps, ps) && locked_kb() == (long)kb; + ok &= !mlock(p + 2 * ps, ps) && locked_kb() == (long)(2 * kb); + ok &= !munlock(p + ps, ps) && locked_kb() == (long)kb; + p[2 * ps] = 0x62; + errno = 0; + ok &= madvise(p + 2 * ps, ps, MADV_PAGEOUT) == -1 && errno == EINVAL && + p[2 * ps] == 0x62; + ok &= !munlock(p + 2 * ps, ps) && locked_kb() == 0; + ksft_test_result( + ok, "%luK repeat lock and independent adjacent unlock\n", kb); + ok = !mlock2(p + 4 * ps, 4 * ps, MLOCK_ONFAULT) && + locked_kb() == (long)(4 * kb); + ok &= !mincore(p + 4 * ps, 4 * ps, residency); + for (unsigned int i = 0; i < 4; i++) + ok &= !(residency[i] & 1); + p[4 * ps] = 0x74; + ok &= !mincore(p + 4 * ps, ps, residency) && residency[0] & 1; + ok &= !munlock(p + 4 * ps, 4 * ps) && locked_kb() == 0; + ksft_test_result( + ok, "%luK MLOCK_ONFAULT accounting and deferred faults\n", kb); + q = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_LOCKED, -1, 0); + ok = q != MAP_FAILED && locked_kb() == (long)(4 * kb); + if (q == MAP_FAILED || pipe(gate)) + ksft_exit_fail_msg("locked mmap/fork setup\n"); + memset(q, 0x8a, 4 * ps); + pid = fork(); + if (!pid) { + char token; + + close(gate[1]); + if (read(gate[0], &token, 1) != 1 || locked_kb() != 0) + _exit(2); + for (unsigned long i = 0; i < 4 * ps; i++) + if (q[i] != 0x8a) + _exit(3); + _exit(0); + } + close(gate[0]); + memset(q + ps, 0x9b, ps); + ok &= write(gate[1], "x", 1) == 1; + close(gate[1]); + ok &= wait_ok(pid) && locked_kb() == (long)(4 * kb); + ok &= !munmap(q, 4 * ps) && locked_kb() == 0; + ksft_test_result( + ok, "%luK MAP_LOCKED, fork noninheritance and parent COW\n", + kb); + ok = !mlockall(MCL_FUTURE | MCL_ONFAULT); + q = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + ok &= q != MAP_FAILED && locked_kb() == (long)kb; + if (q != MAP_FAILED) + *q = 0xaf; + ok &= !munlockall() && locked_kb() == 0; + if (q != MAP_FAILED) + munmap(q, ps); + ksft_test_result(ok, "%luK future on-fault locks and munlockall\n", kb); + pid = fork(); + if (!pid) { + struct rlimit limit = { ps, ps }; + + if (setrlimit(RLIMIT_MEMLOCK, &limit) || setuid(1000) || + mlock(p, ps)) + _exit(4); + errno = 0; + if (mlock(p + ps, ps) != -1 || errno != ENOMEM || + locked_kb() != (long)kb) + _exit(5); + _exit(0); + } + ksft_test_result( + wait_ok(pid), + "%luK unprivileged RLIMIT_MEMLOCK in user-page units\n", kb); + pid = fork(); + if (!pid) { + if (mlockall(MCL_CURRENT) || locked_kb() <= (long)kb || + munlockall() || locked_kb() != 0) + _exit(6); + _exit(0); + } + ksft_test_result(wait_ok(pid), "%luK lock/unlock current mappings\n", + kb); + { + unsigned long huge = ((1UL << 31) + 1) * ps; + + q = mmap(NULL, huge, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + ok = q != MAP_FAILED; + if (q != MAP_FAILED) { + ok &= !mlock2(q, huge, MLOCK_ONFAULT) && + locked_kb() == (long)(huge / 1024); + ok &= !munlock(q, huge) && locked_kb() == 0; + munmap(q, huge); + } + ksft_test_result( + ok, + "%luK deferred lock count beyond signed 32-bit leaves\n", + kb); + } + ksft_test_result( + remap_locked(ps), + "%luK locked grow/shrink/move and partial unmap accounting\n", + kb); + munmap(p, 32 * ps); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + + if (!pid) { + char size[32], host_size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + snprintf(host_size, sizeof(host_size), "%lu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(7); + execl("/init", "/init", "--test", size, host_size, + NULL); + _exit(8); + } + ok &= wait_ok(pid); + } + printf("%s - mlock fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/numa-balancing-init.c b/tools/testing/selftests/arm64/user4k/numa-balancing-init.c new file mode 100644 index 0000000000000..2ea31bda12968 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/numa-balancing-init.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Disposable two-node guest: real automatic NUMA hinting and migration. + * Volatile accesses intentionally trigger page faults and repeated accesses; + * thread coordination uses an atomic flag, not volatile storage. + */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int good, const char *expr, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expr, + errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static void setting(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + + CHECK(fd >= 0); + CHECK(write(fd, value, strlen(value)) == (ssize_t)strlen(value)); + CHECK(!close(fd)); +} + +static void cpu(int id) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(id, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} + +static unsigned long counter(const char *key) +{ + FILE *f = fopen("/proc/vmstat", "r"); + char name[128]; + unsigned long value; + + CHECK(f); + while (fscanf(f, "%127s %lu", name, &value) == 2) { + if (!strcmp(name, key)) { + CHECK(!fclose(f)); + return value; + } + } + CHECK(0); + return 0; +} + +static double now(void) +{ + struct timespec ts; + + CHECK(!clock_gettime(CLOCK_MONOTONIC, &ts)); + return ts.tv_sec + ts.tv_nsec / 1e9; +} + +static int node_at(void *address) +{ + int node = -1; + + CHECK(!syscall(SYS_get_mempolicy, &node, NULL, 0UL, address, + MPOL_F_ADDR | MPOL_F_NODE)); + return node; +} + +static unsigned long sample(unsigned char *map, unsigned long length, + unsigned long page) +{ + unsigned long moved = 0; + + for (unsigned long off = 0; off < length; off += page) + if (off != 2 * page) + moved += node_at(map + off + 16) == 1; + return moved; +} + +static void forbidden(unsigned char *address, int write_access) +{ + pid_t child = fork(); + int status; + + CHECK(child >= 0); + if (!child) { + if (write_access) + *(volatile unsigned char *)address = 0x73; + else + (void)*(volatile unsigned char *)address; + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + CHECK(WIFSIGNALED(status) && WTERMSIG(status) == SIGSEGV); +} + +struct local_reader { + unsigned char *map; + unsigned long length, page; + atomic_int stop; +}; + +static void *read_local(void *arg) +{ + struct local_reader *r = arg; + + cpu(1); + while (!atomic_load_explicit(&r->stop, memory_order_relaxed)) { + for (unsigned long off = 0; off < r->length; off += r->page) { + if (off == 2 * r->page) + continue; + CHECK(*(volatile uint64_t *)(r->map + off) == + 0x81720000 + off / r->page); + } + } + return NULL; +} + +static void local_restore(unsigned char *map, unsigned long length, + unsigned long page) +{ + struct local_reader r = { .map = map, .length = length, .page = page }; + unsigned long hints = counter("numa_hint_faults_local"); + pthread_t worker; + double deadline = now() + 2; + + /* Two threads permit hinting of already-local private mappings. */ + CHECK(!pthread_create(&worker, NULL, read_local, &r)); + do { + for (unsigned long off = 0; off < length; off += page) { + if (off == 2 * page) + continue; + CHECK(*(volatile uint64_t *)(map + off) == + 0x81720000 + off / page); + } + } while (now() < deadline); + atomic_store_explicit(&r.stop, 1, memory_order_relaxed); + CHECK(!pthread_join(worker, NULL)); + hints = counter("numa_hint_faults_local") - hints; + printf("NUMAB local page=%lu hints=%lu\n", page, hints); + CHECK(hints > 0); +} + +static void exercise(void) +{ + unsigned long page = getauxval(AT_PAGESZ), length = 8UL << 20; + unsigned long mask = 1, allowed = 0, hints, migrated, iterations = 0; + unsigned long moved = 0, total = length / page - 1; + unsigned char *map; + int mode; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &allowed, 64UL, NULL, + MPOL_F_MEMS_ALLOWED)); + CHECK(allowed == 3); + cpu(0); + map = mmap(NULL, length, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(map != MAP_FAILED); + CHECK(!madvise(map, length, MADV_NOHUGEPAGE)); + CHECK(!syscall(SYS_mbind, map, length, MPOL_BIND, &mask, 64UL, 0)); + for (unsigned long off = 0; off < length; off += page) { + *(uint64_t *)(map + off) = 0x81720000 + off / page; + *(uint64_t *)(map + off + 8) = 0; + *(uint64_t *)(map + off + page - 8) = 0x61320000 + off / page; + CHECK(node_at(map + off + 16) == 0); + } + CHECK(!mprotect(map + page, page, PROT_READ)); + CHECK(!mprotect(map + 2 * page, page, PROT_NONE)); + CHECK(!syscall(SYS_mbind, map, length, MPOL_DEFAULT, NULL, 0UL, 0)); + cpu(1); + hints = counter("numa_hint_faults"); + migrated = counter("numa_pages_migrated"); + double started = now(), deadline = started + 20; + + do { + for (unsigned long off = 0; off < length; off += page) { + volatile uint64_t *p = (void *)(map + off); + + if (off == 2 * page) + continue; + CHECK(p[0] == 0x81720000 + off / page); + CHECK(p[page / 8 - 1] == 0x61320000 + off / page); + if (off != page) + p[1]++; + } + iterations++; + if (!(iterations % 128)) { + moved = sample(map, length, page); + if (moved == total) + break; + } + } while (now() < deadline); + moved = sample(map, length, page); + hints = counter("numa_hint_faults") - hints; + migrated = counter("numa_pages_migrated") - migrated; + printf("NUMAB page=%lu moved=%lu/%lu hints=%lu migrated=%lu loops=%lu seconds=%.3f\n", + page, moved, total, hints, migrated, iterations, + now() - started); + CHECK(moved == total && hints > 0 && migrated > 0); + for (unsigned long off = 0; off < length; off += page) { + if (off != page && off != 2 * page) + CHECK(*(uint64_t *)(map + off + 8) == iterations); + } + local_restore(map, length, page); + forbidden(map + page, 1); + forbidden(map + 2 * page, 0); + CHECK(!mprotect(map + 2 * page, page, PROT_READ)); + CHECK(*(uint64_t *)(map + 2 * page) == 0x81720002); + CHECK(!munmap(map, length)); + printf("ok - NUMAB page=%lu automatic placement, bytes and permissions\n", + page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { 4096, 16384, 65536 }; + int failed = 0, compat_only = 0, no4k = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--case")) { + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[2], NULL, 10)); + exercise(); + return 0; + } + CHECK(getpid() == 1); + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--compat-only")) + compat_only = 1; + else if (!strcmp(argv[i], "--no4k")) + no4k = 1; + else + CHECK(0); + } + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("debugfs", "/debug", "debugfs", 0, NULL)); + setting("/proc/sys/kernel/numa_balancing", "1\n"); + setting("/debug/sched/numa_balancing/scan_delay_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_period_min_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_period_max_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_size_mb", "16\n"); + for (unsigned int i = 0; i < 3; i++) { + pid_t child; + int status; + + if (compat_only || sizes[i] > native || + (no4k && sizes[i] == 4096)) + continue; + child = fork(); + CHECK(child >= 0); + if (!child) { + char s[32]; + + snprintf(s, sizeof(s), "%lu", sizes[i]); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], + 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", s, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + } + CHECK(!access("/compat-numa-balancing-probe", X_OK)); + pid_t child = fork(); + int status; + + CHECK(child >= 0); + if (!child) { + char s[32]; + + snprintf(s, sizeof(s), "%lu", no4k ? native : 4096UL); + execl("/compat-numa-balancing-probe", + "/compat-numa-balancing-probe", s, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + printf("A32 NUMAB expected-page=%lu status=%d\n", + no4k ? native : 4096UL, status); + printf("NUMA BALANCING %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/numa-policy-init.c b/tools/testing/selftests/arm64/user4k/numa-policy-init.c new file mode 100644 index 0000000000000..fedcedb529709 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/numa-policy-init.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable two-node guest: policy ranges and native physical placement. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int good, const char *expr, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expr, + errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static long bind_to(void *addr, unsigned long bytes, int node, + unsigned int flags) +{ + unsigned long mask = 1UL << node; + + return syscall(SYS_mbind, addr, bytes, MPOL_BIND, &mask, 64UL, flags); +} + +static void policy_at(void *addr, int node) +{ + unsigned long mask = 0; + int mode = -1; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &mask, 64UL, addr, + MPOL_F_ADDR)); + CHECK(mode == MPOL_BIND && mask == (1UL << node)); +} + +static void node_at(void *addr, int expected) +{ + int node = -1; + + CHECK(!syscall(SYS_get_mempolicy, &node, NULL, 0UL, addr, + MPOL_F_NODE | MPOL_F_ADDR)); + if (node != expected) + fprintf(stderr, "node=%d expected=%d addr=%p\n", node, expected, + addr); + CHECK(node == expected); +} + +static void exercise(unsigned long native, int baseline) +{ + unsigned long page = getauxval(AT_PAGESZ), allowed = 0; + unsigned char *reserve, *base, *shared, *alias; + int mode, fd; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &allowed, 64UL, NULL, + MPOL_F_MEMS_ALLOWED)); + CHECK(allowed == + 3); /* Require two real memory nodes, not a silent skip. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + CHECK(!bind_to(base, 2 * native, 0, 0)); + if (baseline) { + errno = 0; + long ret = bind_to(base + page, 1, 1, 0); + + CHECK(page < native ? ret == -1 && errno == EINVAL : ret == 0); + printf("ok - baseline native=%lu page=%lu mbind=%ld errno=%d\n", + native, page, ret, errno); + CHECK(!munmap(reserve, 4 * native)); + return; + } + CHECK(!bind_to(base + page, 1, 1, 0)); + policy_at(base, 0); + policy_at(base + page, 1); + if (page < native) + policy_at(base + 2 * page, 0); + CHECK(!syscall(SYS_set_mempolicy_home_node, base + page, 1UL, 1UL, + 0UL)); + errno = 0; + CHECK(bind_to(base + 1, page, 1, 0) == -1 && errno == EINVAL); + for (unsigned long off = 0; off < 2 * native; off += page) { + base[off + 17] = 0x31 + off / page; + node_at(base + off + 17, off == page ? 1 : 0); + } + CHECK(!bind_to(base + page, page, 1, MPOL_MF_STRICT)); + errno = 0; + CHECK(bind_to(base, 2 * native, 1, MPOL_MF_STRICT) == -1 && + errno == EIO); + policy_at(base, 0); + CHECK(!munmap(reserve, 4 * native)); + + /* A split anonymous VMA must carry its native interleave index correctly. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + unsigned long mask = 3, inset = page < native ? page : 0; + + CHECK(!syscall(SYS_mbind, base + inset, 2 * native - 2 * inset, + MPOL_INTERLEAVE, &mask, 64UL, 0)); + for (unsigned long off = inset; off < 2 * native - inset; off += page) { + base[off + 17] = 0x43; + node_at(base + off + 17, ((uintptr_t)base + off) / native % 2); + } + CHECK(!munmap(reserve, 4 * native)); + + /* Moving a fragment moves its native physical owner, retaining bytes. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + CHECK(!bind_to(base, 2 * native, 0, 0)); + for (unsigned long off = 0; off < 2 * native; off += page) + base[off + 17] = 0x51 + off / page; + CHECK(!bind_to(base + native - page, page, 1, + MPOL_MF_MOVE_ALL | MPOL_MF_STRICT)); + for (unsigned long off = 0; off < 2 * native; off += page) { + CHECK(base[off + 17] == (unsigned char)(0x51 + off / page)); + node_at(base + off + 17, off < native ? 1 : 0); + } + CHECK(!bind_to(base, 2 * native, 0, MPOL_MF_MOVE_ALL | MPOL_MF_STRICT)); + for (unsigned long off = 0; off < 2 * native; off += page) + node_at(base + off + 17, 0); + CHECK(!munmap(reserve, 4 * native)); + + fd = memfd_create("numa-shared", 0); + CHECK(fd >= 0 && !ftruncate(fd, 2 * native)); + shared = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + CHECK(shared != MAP_FAILED); + CHECK(!bind_to(shared, native, 0, 0)); + CHECK(!bind_to(shared + native, native, 1, 0)); + if (page < native) { + alias = mmap(NULL, 2 * native - page, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, page); + CHECK(alias != MAP_FAILED); + policy_at(alias, 0); + policy_at(alias + native - page, 1); + errno = 0; + CHECK(bind_to(alias, page, 1, 0) == -1 && errno == EINVAL); + policy_at(alias, 0); + policy_at(shared, 0); + CHECK(!bind_to(alias + native - page, native, 1, 0)); + for (unsigned long off = 0; off < 2 * native - page; + off += page) { + alias[off + 17] = 0x72; + node_at(alias + off + 17, off + page < native ? 0 : 1); + CHECK(shared[off + page + 17] == 0x72); + } + CHECK(!munmap(alias, 2 * native - page)); + } + CHECK(!munmap(shared, 2 * native) && !close(fd)); + printf("ok - NUMA native=%lu page=%lu policy, placement, migration and shared offsets\n", + native, page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { 4096, 16384, 65536 }; + int failed = 0, baseline = 0, no4; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--case")) { + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + exercise(strtoul(argv[2], NULL, 10), atoi(argv[4])); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + FILE *cmdline = fopen("/proc/cmdline", "r"); + char line[4096]; + + CHECK(cmdline && fgets(line, sizeof(line), cmdline)); + baseline = strstr(line, "numa_fixture_baseline=1") != NULL; + no4 = strstr(line, "id_aa64mmfr0.tgran4=f") != NULL; + CHECK(!fclose(cmdline)); + for (unsigned int i = 0; i < 3; i++) { + int status; + pid_t child; + + if (sizes[i] > native || (no4 && sizes[i] == 4096)) + continue; + child = fork(); + CHECK(child >= 0); + if (!child) { + char n[32], s[32], b[4]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + snprintf(b, sizeof(b), "%d", baseline); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], + 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", n, s, b, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + } + if (!baseline) { + int status; + pid_t child = fork(); + + CHECK(child >= 0); + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", no4 ? native : 4096UL); + execl("/compat-numa-probe", "/compat-numa-probe", n, s, + NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + printf("A32 NUMA expected-page=%lu status=%d\n", + no4 ? native : 4096UL, status); + } + printf("NUMA GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/page-contract.c b/tools/testing/selftests/arm64/user4k/page-contract.c new file mode 100644 index 0000000000000..316ffe48d0a47 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/page-contract.c @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Run unchanged under the current process ABI, including native 4K/16K/64K. + * Explicit --expect-{4k,16k,64k} options assert the selected size; they never + * launch a different ABI or emulate it. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define X18_SENTINEL UINT64_C(0xfedcba9876543210) + +extern long x18_syscall(long nr, long arg0, long arg1, long arg2, + uint64_t *seen); + +static volatile sig_atomic_t signal_seen; +static uint64_t signal_x18; + +static void x18_signal(int sig, siginfo_t *info, void *context) +{ + ucontext_t *uc = context; + + (void)sig; + (void)info; + signal_x18 = uc->uc_mcontext.regs[18]; + signal_seen = 1; + /* rt_sigreturn must restore the interrupted value, not this value. */ + /* Built with -ffixed-x18: the compiler never allocates this register. */ + asm volatile("mov x18, xzr"); +} + +static bool wait_for(pid_t child, int expected_signal) +{ + int status; + pid_t ret; + + if (child < 0) + return false; + do { + ret = waitpid(child, &status, 0); + } while (ret < 0 && errno == EINTR); + if (ret != child) + return false; + if (expected_signal) + return WIFSIGNALED(status) && + WTERMSIG(status) == expected_signal; + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +static bool access_faults(volatile unsigned char *ptr, bool write) +{ + pid_t child = fork(); + + if (!child) { + if (write) + *ptr = 0x5a; + else + (void)*ptr; + _exit(1); + } + return wait_for(child, SIGSEGV); +} + +static void private_mappings(size_t page) +{ + unsigned char *p = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + bool ok; + pid_t child; + + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap: %s\n", strerror(errno)); + memset(p, 0x41, 4 * page); + if (page > 4096) { + errno = 0; + ok = mprotect(p + 4096, 4096, PROT_READ) == -1 && + errno == EINVAL; + } else { + ok = mprotect(p + page, page, PROT_READ) == 0; + } + ksft_test_result(ok, + "mapping alignment agrees with the selected ABI\n"); + ok = mprotect(p + page, page, PROT_READ) == 0 && + mprotect(p + 2 * page, page, PROT_NONE) == 0; + ksft_test_result( + ok && access_faults(p + page, true) && + access_faults(p + 2 * page, false), + "read-only and inaccessible neighbors fault independently\n"); + p[0] = 0x10; + p[3 * page] = 0x30; + ksft_test_result( + p[page] == 0x41 && p[0] == 0x10 && p[3 * page] == 0x30, + "neighbor permissions do not remove writable mappings\n"); + child = fork(); + if (!child) { + p[0] = 0x99; + _exit(p[3 * page] != 0x30); + } + ksft_test_result(wait_for(child, 0) && p[0] == 0x10, + "fork preserves private COW isolation\n"); + ok = munmap(p + page, page) == 0; + ksft_test_result(ok && access_faults(p + page, false) && p[0] == 0x10 && + p[3 * page] == 0x30, + "single-page unmap preserves surrounding mappings\n"); + munmap(p, 4 * page); +} + +static void shared_mappings(size_t page) +{ + int fd = memfd_create("page-contract", MFD_CLOEXEC); + unsigned char *a, *b; + bool ok = true; + pid_t child; + unsigned int i; + + if (fd < 0 || ftruncate(fd, 4 * page)) + ksft_exit_fail_msg("memfd: %s\n", strerror(errno)); + a = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + b = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (a == MAP_FAILED || b == MAP_FAILED) + ksft_exit_fail_msg("shared mmap: %s\n", strerror(errno)); + for (i = 0; i < 4; i++) { + a[i * page] = 0x50 + i; + ok &= b[i * page] == 0x50 + i; + } + ksft_test_result(ok, "shared aliases expose the same bytes\n"); + ok = mprotect(a + page, page, PROT_READ) == 0; + child = fork(); + if (!child) { + b[page] = 0x73; + _exit(a[page] != 0x73); + } + ok &= wait_for(child, 0); + ksft_test_result( + ok && a[page] == 0x73 && access_faults(a + page, true), + "shared writes cross fork while alias permissions stay local\n"); + munmap(a, 4 * page); + munmap(b, 4 * page); + close(fd); +} + +static void x18_contract(void) +{ + struct sigaction sa = { .sa_sigaction = x18_signal, + .sa_flags = SA_SIGINFO }; + struct timespec pause = { .tv_nsec = 1000000 }; + struct rusage before, after; + uint64_t seen; + long ret; + bool ok; + + ret = x18_syscall(SYS_getpid, 0, 0, 0, &seen); + ksft_test_result(ret == getpid() && seen == X18_SENTINEL, + "x18 survives a raw syscall\n"); + ok = getrusage(RUSAGE_SELF, &before) == 0; + ret = x18_syscall(SYS_nanosleep, (long)&pause, 0, 0, &seen); + ok &= getrusage(RUSAGE_SELF, &after) == 0; + ksft_test_result(ok && ret == 0 && seen == X18_SENTINEL && + after.ru_nvcsw > before.ru_nvcsw, + "x18 survives a measured voluntary context switch\n"); + sigemptyset(&sa.sa_mask); + ok = sigaction(SIGUSR1, &sa, NULL) == 0; + if (ok) + ret = x18_syscall(SYS_tgkill, getpid(), syscall(SYS_gettid), + SIGUSR1, &seen); + ksft_test_result( + ok && ret == 0 && signal_seen && signal_x18 == X18_SENTINEL && + seen == X18_SENTINEL, + "signal frame and sigreturn preserve x18 after handler clobber\n"); +} + +int main(int argc, char **argv) +{ + const struct rlimit no_core = { 0, 0 }; + size_t expected = getauxval(AT_PAGESZ); + int result; + + if (argc == 2 && !strcmp(argv[1], "--expect-4k")) { + expected = 4096; + } else if (argc == 2 && !strcmp(argv[1], "--expect-16k")) { + expected = 16384; + } else if (argc == 2 && !strcmp(argv[1], "--expect-64k")) { + expected = 65536; + } else if (argc != 1) { + fprintf(stderr, + "usage: %s [--expect-4k|--expect-16k|--expect-64k]\n", + argv[0]); + return KSFT_FAIL; + } + ksft_print_header(); + ksft_set_plan(11); + if (expected != 4096 && expected != 16384 && expected != 65536) + ksft_exit_skip("unsupported process page size: %zu\n", + expected); + if (sysconf(_SC_PAGESIZE) != (long)expected || + getauxval(AT_PAGESZ) != expected) + ksft_exit_fail_msg( + "expected %zu-byte ABI; sysconf=%ld AT_PAGESZ=%lu\n", + expected, sysconf(_SC_PAGESIZE), getauxval(AT_PAGESZ)); + ksft_test_result_pass( + "process reports the expected %zu-byte page ABI\n", expected); + if (setrlimit(RLIMIT_CORE, &no_core)) + ksft_exit_fail_msg("disable child core dumps: %s\n", + strerror(errno)); + private_mappings(expected); + shared_mappings(expected); + x18_contract(); + ksft_print_cnts(); + result = ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS; + /* This static executable can also be /init in a disposable test VM. */ + if (getpid() == 1) { + fflush(NULL); + reboot(RB_POWER_OFF); + } + return result; +} diff --git a/tools/testing/selftests/arm64/user4k/perf-ring-init.c b/tools/testing/selftests/arm64/user4k/perf-ring-init.c new file mode 100644 index 0000000000000..4252f017619ca --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/perf-ring-init.c @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: perf ring layout and sampling with per-mm page sizes. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int ok, const char *expression, int line) +{ + if (!ok) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, + errno); + exit(1); + } +} + +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int no4k; + +static int wait_child(pid_t child) +{ + int status; + pid_t got; + + if (child < 0) + return 1; + do { + got = waitpid(child, &status, 0); + } while (got < 0 && errno == EINTR); + return got != child || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static int event_open(int sampling) +{ + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .size = sizeof(attr), + .config = PERF_COUNT_SW_PAGE_FAULTS, + .disabled = 1, + .sample_period = sampling ? 1 : 0, + .sample_type = PERF_SAMPLE_TID | PERF_SAMPLE_TIME | + PERF_SAMPLE_ADDR | PERF_SAMPLE_PERIOD, + }; + int fd = syscall(SYS_perf_event_open, &attr, 0, -1, -1, 0); + + CHECK(fd >= 0); + return fd; +} + +static sigjmp_buf fault_return; +static volatile sig_atomic_t fault_signal; + +static void fault_handler(int signal) +{ + fault_signal = signal; + siglongjmp(fault_return, 1); +} + +static void readonly_data(void *address) +{ + struct sigaction action = { .sa_handler = fault_handler }, previous; + + CHECK(!sigemptyset(&action.sa_mask)); + CHECK(!sigaction(SIGBUS, &action, &previous)); + fault_signal = 0; + if (!sigsetjmp(fault_return, 1)) + *(volatile unsigned char *)address = 0; + CHECK(fault_signal == SIGBUS); + CHECK(!sigaction(SIGBUS, &previous, NULL)); +} + +static void copy_record(void *destination, const unsigned char *ring, + unsigned long capacity, uint64_t position, size_t size) +{ + unsigned long offset = position & (capacity - 1); + size_t first = capacity - offset; + + if (first > size) + first = size; + memcpy(destination, ring + offset, first); + memcpy((unsigned char *)destination + first, ring, size - first); +} + +static void sampling(unsigned long pages) +{ + unsigned long page = getauxval(AT_PAGESZ), capacity = pages * page; + unsigned long length = page + capacity; + int fd = event_open(1); + struct perf_event_mmap_page *map = + mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + unsigned char *region, *ring; + uint64_t tail = 0, samples = 0, wrapped = 0, region_samples = 0; + + CHECK(map != MAP_FAILED); + CHECK(map->data_offset == page && map->data_size == capacity); + map->data_tail = 0; + ring = (unsigned char *)map + map->data_offset; + for (unsigned long i = 0; i < capacity; i += page) + CHECK(ring[i] == 0); + readonly_data(ring); + region = mmap(NULL, 16 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(region != MAP_FAILED); + for (unsigned int batch = 0; batch < 10000 && tail < 3 * capacity; + batch++) { + CHECK(!madvise(region, 16 * page, MADV_DONTNEED)); + CHECK(!ioctl(fd, PERF_EVENT_IOC_ENABLE, 0)); + for (unsigned int i = 0; i < 16; i++) + *(volatile unsigned char *)(region + i * page) = 1; + CHECK(!ioctl(fd, PERF_EVENT_IOC_DISABLE, 0)); + uint64_t head = + __atomic_load_n(&map->data_head, __ATOMIC_ACQUIRE); + + CHECK(head >= tail && head - tail <= capacity); + while (tail < head) { + struct perf_event_header header; + struct { + struct perf_event_header header; + uint32_t pid, tid; + uint64_t time, address, period; + } record; + + copy_record(&header, ring, capacity, tail, + sizeof(header)); + CHECK(header.size >= sizeof(header) && + header.size <= head - tail); + if (header.type == PERF_RECORD_SAMPLE) { + CHECK(header.size == sizeof(record)); + copy_record(&record, ring, capacity, tail, + sizeof(record)); + CHECK(record.pid == (uint32_t)getpid()); + CHECK(record.tid == (uint32_t)getpid()); + if (record.address >= (uintptr_t)region && + record.address < + (uintptr_t)region + 16 * page) + region_samples++; + CHECK(record.period == 1); + samples++; + if ((tail & (capacity - 1)) + header.size > + capacity) + wrapped++; + } else { + CHECK(header.type == PERF_RECORD_THROTTLE || + header.type == PERF_RECORD_UNTHROTTLE); + } + tail += header.size; + } + __atomic_store_n(&map->data_tail, tail, __ATOMIC_RELEASE); + } + CHECK(tail >= 3 * capacity && samples && wrapped); + /* Library/stack faults may also occur while a batch is enabled. */ + CHECK(region_samples > samples * 9 / 10); + CHECK(!munmap(region, 16 * page)); + CHECK(!close(fd)); + CHECK(map->data_head == tail); + CHECK(!munmap(map, length)); + printf("ok - perf page=%lu ring=%lu samples=%llu region=%llu wrapped-records=%llu\n", + page, capacity, (unsigned long long)samples, + (unsigned long long)region_samples, (unsigned long long)wrapped); +} + +static void layout(unsigned long native) +{ + unsigned long page = getauxval(AT_PAGESZ); + int fd = event_open(0); + struct perf_event_mmap_page *map = + mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + + CHECK(map != MAP_FAILED); + CHECK(map->data_offset == page && map->data_size == 0); + map->data_tail = 0; + CHECK(!munmap(map, page)); + CHECK(!close(fd)); + fd = event_open(0); + errno = 0; + CHECK(mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0) == + MAP_FAILED); + CHECK(errno == EINVAL); + map = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + CHECK(map != MAP_FAILED); + struct perf_event_mmap_page *alias = + mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + CHECK(alias != MAP_FAILED); + map->data_tail = 7; + CHECK(alias->data_tail == 7); + map->data_tail = 0; + CHECK(!munmap(alias, 3 * page)); + /* A software event has no AUX provider. Check byte-exact offset routing. */ + map->aux_offset = 3 * page; + map->aux_size = native; + errno = 0; + CHECK(mmap(NULL, native, PROT_READ, MAP_SHARED, fd, map->aux_offset) == + MAP_FAILED); + CHECK(errno == EOPNOTSUPP); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + if (sizes[i] > native || sizes[i] == page || + (no4k && sizes[i] == 4096)) + continue; + pid_t child = fork(); + + if (!child) { + char f[32], s[32]; + + snprintf(f, sizeof(f), "%d", fd); + snprintf(s, sizeof(s), "%lu", sizes[i]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, + 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--alias", f, s, NULL); + _exit(3); + } + CHECK(!wait_child(child)); + } + CHECK(map->data_size == 2 * page && map->data_offset == page); + CHECK(!munmap(map, 3 * page)); + CHECK(!close(fd)); + printf("ok - perf page=%lu metadata, invalid size, aliases, AUX routing\n", + page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0, compat_only = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + no4k = getenv("PERF_TEST_NO4K") != NULL; + if (argc == 4 && !strcmp(argv[1], "--alias")) { + unsigned long page = strtoul(argv[3], NULL, 10); + + CHECK(getauxval(AT_PAGESZ) == page); + errno = 0; + CHECK(mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, + atoi(argv[2]), 0) == MAP_FAILED); + CHECK(errno == EINVAL); + return 0; + } + if (argc == 4 && !strcmp(argv[1], "--case")) { + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + layout(native); + sampling(1); + sampling(2); + sampling(2 * native / getauxval(AT_PAGESZ)); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--no4k")) { + no4k = 1; + CHECK(!setenv("PERF_TEST_NO4K", "1", 1)); + } + if (!strcmp(argv[i], "--compat-only")) + compat_only = 1; + } + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + if (compat_only || sizes[i] > native || + (no4k && sizes[i] == 4096)) + continue; + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, + 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--case", n, s, NULL); + _exit(3); + } + int bad = wait_child(child); + + printf("%s - perf native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, sizes[i]); + failed |= bad; + } + if (compat_only) + CHECK(access("/compat-perf", X_OK) == 0); + if (access("/compat-perf", X_OK) == 0) { + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", no4k ? native : 4096UL); + execl("/compat-perf", "/compat-perf", n, s, NULL); + _exit(3); + } + int bad = wait_child(child); + + printf("%s - perf AArch32 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, no4k ? native : 4096UL); + failed |= bad; + } + printf("PERF RING GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/proc-args.c b/tools/testing/selftests/arm64/user4k/proc-args.c new file mode 100644 index 0000000000000..300742a258863 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-args.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Compare argv/envp with /proc/PID/cmdline and /proc/PID/environ. + * + * Run it once natively and once under the 4K user granule (for example + * "userpages 4k ./proc-args"). The process reads its own files, and a forked + * child reads the parent's files through /proc/. "--hold SECONDS" + * keeps the process alive so a native process can read it from outside. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char **environ; + +static int failures; + +static ssize_t read_file(const char *path, char *buf, size_t size) +{ + ssize_t got, total = 0; + int fd = open(path, O_RDONLY); + + if (fd < 0) + return -errno; + while ((size_t)total < size) { + got = read(fd, buf + total, size - total); + if (got < 0) { + if (errno == EINTR) + continue; + total = -errno; + break; + } + if (!got) + break; + total += got; + } + close(fd); + return total; +} + +/* Join the complete vector; reserve one extra byte in the read buffer. */ +static char *join(char **vec, size_t *size) +{ + size_t len = 0, offset = 0; + char **entry, *buf; + + for (entry = vec; *entry; entry++) { + size_t n = strlen(*entry) + 1; + + if (n > SSIZE_MAX - 1 - len) { + errno = EOVERFLOW; + return NULL; + } + len += n; + } + buf = malloc(len ? len : 1); + if (!buf) + return NULL; + for (entry = vec; *entry; entry++) { + size_t n = strlen(*entry) + 1; + + memcpy(buf + offset, *entry, n); + offset += n; + } + *size = len; + return buf; +} + +static void check(const char *who, const char *name, pid_t pid, char **vec) +{ + char path[64], *want, *got; + size_t want_len; + ssize_t got_len; + + want = join(vec, &want_len); + got = want ? malloc(want_len + 1) : NULL; + if (!got) { + printf("not ok %s %s allocation: %s\n", who, name, + strerror(errno)); + free(want); + failures++; + return; + } + snprintf(path, sizeof(path), "/proc/%d/%s", pid, name); + /* A trailing byte rejects excess data; exact contents must reach EOF. */ + got_len = read_file(path, got, want_len + 1); + if (got_len == (ssize_t)want_len && !memcmp(want, got, want_len)) { + printf("ok %s %s %zd bytes\n", who, path, got_len); + } else { + printf("not ok %s %s want %zu got %zd", who, path, want_len, + got_len); + if (got_len < 0) + printf(" error %s", strerror(-got_len)); + else if (got_len > 0) + printf(" first [%.*s]", + (int)(got_len < 64 ? got_len : 64), got); + printf("\n"); + failures++; + } + free(got); + free(want); +} + +static void show_layout(pid_t pid, char **argv) +{ + char buf[4096], *p, *field; + ssize_t len; + int i; + + printf("pid %d AT_PAGESZ %lu argv[0] %p environ[0] %p\n", pid, + getauxval(AT_PAGESZ), (void *)argv[0], + environ[0] ? (void *)environ[0] : NULL); + len = read_file("/proc/self/stat", buf, sizeof(buf) - 1); + if (len <= 0) + return; + buf[len] = 0; + /* Fields after the parenthesised comm start at field 3. */ + p = strrchr(buf, ')'); + if (!p || p[1] != ' ') + return; + printf("stat arg_start arg_end env_start env_end:"); + for (i = 3, field = strtok(p + 2, " "); field; + field = strtok(NULL, " "), i++) + if (i >= 48 && i <= 51) + printf(" %#lx", strtoul(field, NULL, 10)); + printf("\n"); +} + +int main(int argc, char **argv) +{ + pid_t self = getpid(), child; + int status; + + setbuf(stdout, NULL); + show_layout(self, argv); + check("self", "cmdline", self, argv); + check("self", "environ", self, environ); + + child = fork(); + if (child < 0) + return 1; + if (!child) { + failures = 0; + check("child", "cmdline", self, argv); + check("child", "environ", self, environ); + _exit(failures ? 1 : 0); + } + if (waitpid(child, &status, 0) != child || !WIFEXITED(status) || + WEXITSTATUS(status)) + failures++; + + if (argc > 2 && !strcmp(argv[1], "--hold")) { + fflush(stdout); + sleep(atoi(argv[2])); + } + printf("%s: %d failure(s)\n", failures ? "FAIL" : "PASS", failures); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/arm64/user4k/proc-memory.c b/tools/testing/selftests/arm64/user4k/proc-memory.c new file mode 100644 index 0000000000000..81ca2d0553dd0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-memory.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +struct stats { + unsigned long size, rss, pss, private_clean, private_dirty, + shared_clean, shared_dirty; + unsigned long anonymous, kernel_page, mmu_page; +}; + +static bool get_stats(pid_t pid, uintptr_t address, struct stats *s) +{ + char path[64], line[512]; + unsigned long start, end, value; + unsigned int fields = 0; + char key[64]; + bool found = false; + FILE *f; + + memset(s, 0, sizeof(*s)); + snprintf(path, sizeof(path), "/proc/%d/smaps", pid); + f = fopen(path, "re"); + if (!f) + return false; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + if (found) + break; + found = start == address; + continue; + } + if (!found) + continue; + if (sscanf(line, "%63[^:]: %lu", key, &value) != 2) + continue; + if (!strcmp(key, "Size")) { + s->size = value; + fields |= 1U << 0; + } else if (!strcmp(key, "Rss")) { + s->rss = value; + fields |= 1U << 1; + } else if (!strcmp(key, "Pss")) { + s->pss = value; + fields |= 1U << 2; + } else if (!strcmp(key, "Private_Clean")) { + s->private_clean = value; + fields |= 1U << 3; + } else if (!strcmp(key, "Private_Dirty")) { + s->private_dirty = value; + fields |= 1U << 4; + } else if (!strcmp(key, "Shared_Clean")) { + s->shared_clean = value; + fields |= 1U << 5; + } else if (!strcmp(key, "Shared_Dirty")) { + s->shared_dirty = value; + fields |= 1U << 6; + } else if (!strcmp(key, "Anonymous")) { + s->anonymous = value; + fields |= 1U << 7; + } else if (!strcmp(key, "KernelPageSize")) { + s->kernel_page = value; + fields |= 1U << 8; + } else if (!strcmp(key, "MMUPageSize")) { + s->mmu_page = value; + fields |= 1U << 9; + } + } + fclose(f); + return found && fields == (1U << 10) - 1; +} + +static bool check(pid_t pid, void *address, size_t page, size_t length, + unsigned long pss, unsigned long private, + unsigned long shared, bool anon) +{ + struct stats s; + bool ok = get_stats(pid, (uintptr_t)address, &s); + + ok &= s.size == length / 1024 && s.rss == length / 1024 && + s.pss == pss && s.private_clean + s.private_dirty == private && + s.shared_clean + s.shared_dirty == shared && + s.anonymous == (anon ? length / 1024 : 0) && + s.kernel_page == page / 1024 && s.mmu_page == page / 1024; + if (!ok) + printf("# smaps size=%lu rss=%lu pss=%lu private=%lu shared=%lu anon=%lu pages=%lu/%lu\n", + s.size, s.rss, s.pss, s.private_clean + s.private_dirty, + s.shared_clean + s.shared_dirty, s.anonymous, + s.kernel_page, s.mmu_page); + return ok; +} + +static int check_numa(void *address, size_t page) +{ + char line[1024]; + unsigned long start, anonymous = 0, pages = 0, granule = 0; + bool found = false; + FILE *f = fopen("/proc/self/numa_maps", "re"); + + if (!f) + return errno == ENOENT ? -1 : 0; + while (fgets(line, sizeof(line), f)) { + char *save, *token; + + if (sscanf(line, "%lx", &start) != 1 || + start != (uintptr_t)address) + continue; + found = true; + for (token = strtok_r(line, "\n ", &save); token; + token = strtok_r(NULL, "\n ", &save)) { + unsigned int node; + unsigned long count; + + if (sscanf(token, "anon=%lu", &count) == 1) + anonymous = count; + else if (sscanf(token, "kernelpagesize_kB=%lu", + &count) == 1) + granule = count; + else if (sscanf(token, "N%u=%lu", &node, &count) == 2) + pages += count; + } + break; + } + fclose(f); + if (!found || anonymous != 4 || pages != 4 || granule != page / 1024) + printf("# numa found=%d anon=%lu pages=%lu granule=%lu\n", + found, anonymous, pages, granule); + return found && anonymous == 4 && pages == 4 && granule == page / 1024; +} + +static bool reap(pid_t child) +{ + int status; + + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); +} + +static bool resident_rewrite_reuses(unsigned char *p, size_t page) +{ + uint64_t before[4], after[4]; + int fd = open("/proc/self/pagemap", O_RDONLY); + unsigned int i; + bool ok = fd >= 0; + + if (ok) + ok = pread(fd, before, sizeof(before), + (uintptr_t)p / page * 8) == sizeof(before); + for (i = 0; i < 4; i++) + p[i * page] ^= 0x21; + if (ok) + ok = pread(fd, after, sizeof(after), (uintptr_t)p / page * 8) == + sizeof(after); + for (i = 0; ok && i < 4; i++) { + const uint64_t frame = (1ULL << 55) - 1; + + ok &= (before[i] & frame) && + (before[i] & frame) == (after[i] & frame) && + (before[i] & (1ULL << 63)) && (after[i] & (1ULL << 63)); + } + if (fd >= 0) + close(fd); + return ok; +} + +static void anonymous_sharing(size_t page) +{ + unsigned char *base, *p; + int ready[2], go[2]; + pid_t child; + char byte; + bool ok; + + base = mmap(NULL, 6 * page, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, + 0); + if (base == MAP_FAILED || + mprotect(base + page, 4 * page, PROT_READ | PROT_WRITE) || + pipe(ready) || pipe(go)) + ksft_exit_fail_msg("anonymous fixture setup\n"); + p = base + page; + memset(p, 0x57, 4 * page); + ksft_test_result( + check(getpid(), p, page, 4 * page, 4 * page / 1024, + 4 * page / 1024, 0, true), + "dense anonymous leaves have exact RSS/PSS/private bytes\n"); + { + int numa = check_numa(p, page); + + if (numa < 0) + ksft_test_result_skip( + "NUMA statistics require CONFIG_NUMA\n"); + else + ksft_test_result( + numa, + "NUMA statistics count target leaves and report their granule\n"); + } + child = fork(); + if (!child) { + close(ready[0]); + close(go[1]); + if (write(ready[1], "a", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(1); + p[page] = 0x96; + if (write(ready[1], "b", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(2); + _exit(0); + } + close(ready[1]); + close(go[0]); + ok = child > 0 && read(ready[0], &byte, 1) == 1 && + check(getpid(), p, page, 4 * page, 2 * page / 1024, 0, + 4 * page / 1024, true); + ksft_test_result( + ok, + "fork divides each anonymous leaf's PSS by its own mapcount\n"); + ok = write(go[1], "x", 1) == 1 && read(ready[0], &byte, 1) == 1 && + check(getpid(), p, page, 4 * page, (5 * page) / 2048, page / 1024, + 3 * page / 1024, true) && + p[page] == 0x57; + ksft_test_result( + ok, + "one child COW changes only that leaf's private/shared accounting\n"); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ksft_test_result(reap(child) && check(getpid(), p, page, 4 * page, + 4 * page / 1024, 4 * page / 1024, + 0, true), + "child exit restores full anonymous PSS\n"); + ksft_test_result( + resident_rewrite_reuses(p, page), + "resident writes reuse physical backing after the child exits\n"); + munmap(base, 6 * page); +} + +static int worker(int fd, int output, int input, size_t expected) +{ + size_t page = getauxval(AT_PAGESZ); + unsigned char *p; + uintptr_t addr; + char byte; + + if (page != expected) + return 1; + p = mmap(NULL, page, PROT_READ, MAP_SHARED, fd, + page == 4096 ? 4096 : 0); + if (p == MAP_FAILED || p[0] != 0x78) + return 2; + addr = (uintptr_t)p; + if (write(output, &addr, sizeof(addr)) != sizeof(addr) || + read(input, &byte, 1) != 1) + return 3; + return 0; +} + +static void mixed_file(size_t page) +{ + size_t target = page == 4096 ? 16384 : 4096; + int ready[2], go[2], fd = memfd_create("mixed-pss", 0); + unsigned char data[16384], *p; + uintptr_t remote; + pid_t child; + bool ok; + + memset(data, 0x78, sizeof(data)); + if (fd < 0 || write(fd, data, sizeof(data)) != sizeof(data) || + pipe(ready) || pipe(go)) + ksft_exit_fail_msg("mixed file fixture setup\n"); + p = mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page == 4096 ? 4096 : 0); + if (p == MAP_FAILED || p[0] != 0x78) + ksft_exit_fail_msg("mixed file mmap\n"); + child = fork(); + if (!child) { + char farg[16], out[16], in[16], expected[16]; + + close(ready[0]); + close(go[1]); + snprintf(farg, sizeof(farg), "%d", fd); + snprintf(out, sizeof(out), "%d", ready[1]); + snprintf(in, sizeof(in), "%d", go[0]); + snprintf(expected, sizeof(expected), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, 0UL, 0UL, 0UL)) + _exit(4); + execl("/proc/self/exe", "/proc/self/exe", "--worker", farg, out, + in, expected, NULL); + _exit(5); + } + close(ready[1]); + close(go[0]); + ok = child > 0 && + read(ready[0], &remote, sizeof(remote)) == sizeof(remote); + ok = ok && + check(getpid(), p, page, page, page == 4096 ? 2 : 14, + page == 4096 ? 0 : 12, 4, false) && + check(child, (void *)remote, target, target, + target == 4096 ? 2 : 14, target == 4096 ? 0 : 12, 4, false); + ksft_test_result( + ok, + "mixed file aliases divide only their overlapping physical quarter\n"); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ksft_test_result(reap(child) && + check(getpid(), p, page, page, page / 1024, + page / 1024, 0, false), + "mixed alias exit restores private file accounting\n"); + munmap(p, page); + close(fd); +} + +int main(int argc, char **argv) +{ + size_t page = getauxval(AT_PAGESZ); + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), + strtoul(argv[5], NULL, 10)); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + ksft_print_header(); + ksft_set_plan(8); + anonymous_sharing(page); + mixed_file(page); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/proc-pages.c b/tools/testing/selftests/arm64/user4k/proc-pages.c new file mode 100644 index 0000000000000..d3aaa11101cf2 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-pages.c @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define PRESENT (1ULL << 63) +#define EXCLUSIVE (1ULL << 56) +#define GUARD (1ULL << 58) +#define FRAME ((1ULL << 55) - 1) +#define NPAGE 5 + +struct ready { + uintptr_t address; + size_t page; + uint64_t entries[NPAGE]; + uint64_t shared_entry; +}; + +static int open_proc(pid_t pid, const char *name, int flags) +{ + char path[80]; + + snprintf(path, sizeof(path), "/proc/%d/%s", pid, name); + return open(path, flags); +} + +static bool entries(int fd, uintptr_t address, size_t page, uint64_t *values, + int nr) +{ + return pread(fd, values, nr * sizeof(*values), + address / page * sizeof(*values)) == + nr * (ssize_t)sizeof(*values); +} + +static unsigned char *mapping(size_t page) +{ + const size_t boundary = 32 * 1024 * 1024; + size_t reserve = boundary + 8 * page, head, tail; + unsigned char *base, *p; + + base = mmap(NULL, reserve, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, + 0); + if (base == MAP_FAILED) + return NULL; + /* Cross both native 32 MiB and alternative 2 MiB PTE-table boundaries. */ + p = (void *)(((uintptr_t)base + 2 * page + boundary - 1) & + ~(boundary - 1)); + p -= 2 * page; + head = p - base; + tail = reserve - head - NPAGE * page; + if ((head && munmap(base, head)) || + (tail && munmap(p + NPAGE * page, tail)) || + mprotect(p, NPAGE * page, PROT_READ | PROT_WRITE)) + return NULL; + memset(p, 0x67, NPAGE * page); + if (munmap(p + 2 * page, page) || + madvise(p + 3 * page, page, MADV_GUARD_INSTALL) || + mprotect(p + 4 * page, page, PROT_NONE)) + return NULL; + return p; +} + +static bool scan(int fd, uintptr_t addr, size_t page) +{ + struct page_region vec[8]; + struct pm_scan_arg arg = { + .size = sizeof(arg), + .start = addr, + .end = addr + NPAGE * page, + .vec = (uintptr_t)vec, + .vec_len = 8, + .return_mask = PAGE_IS_PRESENT | PAGE_IS_GUARD, + }; + int nr = ioctl(fd, PAGEMAP_SCAN, &arg); + bool ok = nr == 3 && arg.walk_end == arg.end; + if (!ok) + printf("# scan initial nr=%d errno=%d end_delta=%lld\n", nr, + errno, (long long)(arg.walk_end - arg.end)); + + /* The unmapped middle page produces no VMA result. */ + if (nr == 3) + ok &= vec[0].start == addr && vec[0].end == addr + 2 * page && + vec[0].categories == PAGE_IS_PRESENT && + vec[1].start == addr + 3 * page && + vec[1].end == addr + 4 * page && + vec[1].categories == PAGE_IS_GUARD && + vec[2].start == addr + 4 * page && + vec[2].end == arg.end && + vec[2].categories == PAGE_IS_PRESENT; + arg.max_pages = 1; + arg.walk_end = 0; + nr = ioctl(fd, PAGEMAP_SCAN, &arg); + ok &= nr == 1 && vec[0].start == addr && vec[0].end == addr + page && + arg.walk_end == addr + page; + arg.start = addr + 1; + errno = 0; + ok &= ioctl(fd, PAGEMAP_SCAN, &arg) == -1 && errno == EINVAL; + return ok; +} + +static int remote_worker(int output, int input, size_t expected, int shared_fd) +{ + struct ready ready = { .page = getauxval(AT_PAGESZ) }; + unsigned char *p; + int fd = open_proc(getpid(), "pagemap", O_RDONLY); + char byte; + unsigned char *shared = + mmap(NULL, 16384, PROT_READ, MAP_SHARED, shared_fd, 0); + size_t offset = ready.page == 4096 ? 4096 : 0; + + if (shared == MAP_FAILED || shared[offset] != 0x83 || + !entries(fd, (uintptr_t)shared + offset, ready.page, + &ready.shared_entry, 1)) + return 2; + /* Allocate aliases first so mmap cannot reuse our deliberate hole. */ + p = mapping(ready.page); + ready.address = (uintptr_t)p; + if (ready.page != expected || !p || fd < 0 || + !entries(fd, ready.address, ready.page, ready.entries, NPAGE) || + write(output, &ready, sizeof(ready)) != sizeof(ready) || + read(input, &byte, 1) != 1) + return 1; + return 0; +} + +static bool remote_read(size_t own_page) +{ + int ready_pipe[2], done_pipe[2], status, fd, shared_fd, self_fd; + unsigned char *shared; + uint64_t self_entry; + size_t offset = own_page == 4096 ? 4096 : 0; + size_t target = own_page == 4096 ? 16384 : 4096; + struct ready ready; + uint64_t actual[NPAGE]; + pid_t child; + bool ok; + + shared_fd = memfd_create("mixed-pagemap", 0); + if (shared_fd < 0 || ftruncate(shared_fd, 16384)) + return false; + shared = mmap(NULL, 16384, PROT_READ | PROT_WRITE, MAP_SHARED, + shared_fd, 0); + self_fd = open_proc(getpid(), "pagemap", O_RDONLY); + if (shared == MAP_FAILED || self_fd < 0) + return false; + memset(shared, 0x83, 16384); + if (!entries(self_fd, (uintptr_t)shared + offset, own_page, &self_entry, + 1)) + return false; + close(self_fd); + if (pipe(ready_pipe) || pipe(done_pipe)) + return false; + child = fork(); + if (!child) { + char output[16], input[16], expected[16], shared_arg[16]; + + close(ready_pipe[0]); + close(done_pipe[1]); + snprintf(output, sizeof(output), "%d", ready_pipe[1]); + snprintf(input, sizeof(input), "%d", done_pipe[0]); + snprintf(expected, sizeof(expected), "%zu", target); + snprintf(shared_arg, sizeof(shared_arg), "%d", shared_fd); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, 0UL, 0UL, 0UL)) + _exit(2); + execl("/proc/self/exe", "/proc/self/exe", "--remote", output, + input, expected, shared_arg, NULL); + _exit(3); + } + close(ready_pipe[1]); + close(done_pipe[0]); + ok = child > 0 && + read(ready_pipe[0], &ready, sizeof(ready)) == sizeof(ready); + fd = open_proc(child, "pagemap", O_RDONLY); + ok = ok && ready.page == target && fd >= 0 && + entries(fd, ready.address, target, actual, NPAGE) && + !memcmp(actual, ready.entries, sizeof(actual)) && + scan(fd, ready.address, target); + if (!ok) + printf("# remote geometry/read failure: target=%zu reported=%zu fd=%d errno=%d\n", + target, ready.page, fd, errno); + if (ok) { + size_t target_offset = target == 4096 ? 4096 : 0; + + /* Compare physical byte addresses, not differently sized PFNs. */ + ok &= (self_entry & FRAME) && (ready.shared_entry & FRAME) && + (self_entry & FRAME) * own_page - offset == + (ready.shared_entry & FRAME) * target - + target_offset; + if (!ok) + printf("# shared PFN mismatch: own_nonzero=%d target_nonzero=%d byte_delta=%lld\n", + !!(self_entry & FRAME), + !!(ready.shared_entry & FRAME), + (long long)((self_entry & FRAME) * own_page - + offset - + (ready.shared_entry & FRAME) * + target + + target_offset)); + } + if (fd >= 0) + close(fd); + write(done_pipe[1], "x", 1); + close(done_pipe[1]); + close(ready_pipe[0]); + ok &= child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + munmap(shared, 16384); + close(shared_fd); + return ok; +} + +int main(int argc, char **argv) +{ + size_t page = getauxval(AT_PAGESZ); + unsigned char *p; + uint64_t values[NPAGE], again[NPAGE]; + int fd, clear, i; + bool ok; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--remote")) + return remote_worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10), atoi(argv[5])); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + ksft_print_header(); + ksft_set_plan(6); + p = mapping(page); + fd = open_proc(getpid(), "pagemap", O_RDONLY); + if (!p || fd < 0) + ksft_exit_fail_msg("proc mapping setup: %s\n", strerror(errno)); + ok = entries(fd, (uintptr_t)p, page, values, NPAGE); + ok &= (values[0] & PRESENT) && (values[1] & PRESENT) && !values[2] && + (values[3] & GUARD) && !(values[3] & PRESENT) && + (values[4] & PRESENT); + ok &= (values[0] & EXCLUSIVE) && (values[1] & EXCLUSIVE) && + (values[4] & EXCLUSIVE); + ksft_test_result( + ok, + "pagemap reports each target leaf, hole, guard and PROT_NONE entry\n"); + ok = true; + for (i = 0; i < NPAGE; i++) { + uint64_t value; + + ok &= entries(fd, (uintptr_t)p + i * page, page, &value, 1) && + value == values[i]; + } + errno = 0; + ok &= pread(fd, again, 7, 0) == -1 && errno == EINVAL; + ksft_test_result( + ok, + "pagemap per-entry offsets agree with batch reads and reject short entries\n"); + ksft_test_result( + scan(fd, (uintptr_t)p, page), + "PAGEMAP_SCAN ranges, max_pages and alignment use target granules\n"); + clear = open_proc(getpid(), "clear_refs", O_WRONLY); + ok = clear >= 0; + for (i = 1; ok && i <= 3; i++) { + char operation = '0' + i; + + ok &= write(clear, &operation, 1) == 1; + } + if (clear >= 0) + close(clear); + ok &= entries(fd, (uintptr_t)p, page, again, NPAGE) && + !memcmp(values, again, sizeof(values)) && p[0] == 0x67 && + p[page] == 0x67; + ksft_test_result( + ok, + "clear_refs traverses short and split VMAs without changing mappings or bytes\n"); + ksft_test_result( + remote_read(page), + "opposite-ABI reader uses target pagemap and scan geometry\n"); + { + pid_t child = fork(); + int status; + + if (!child) { + uint64_t value = 0; + int unprivileged; + + if (setuid(1000) || prctl(PR_SET_DUMPABLE, 1)) + _exit(2); + unprivileged = open_proc(getpid(), "pagemap", O_RDONLY); + ok = unprivileged >= 0 && + entries(unprivileged, (uintptr_t)p, page, &value, + 1) && + (value & PRESENT) && !(value & FRAME); + if (!ok) + printf("# unprivileged fd=%d errno=%d flags=%#llx\n", + unprivileged, errno, + (unsigned long long)(value & ~FRAME)); + _exit(!ok); + } + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + ksft_test_result( + ok, + "unprivileged pagemap reads retain flags and hide physical addresses\n"); + } + close(fd); + munmap(p, NPAGE * page); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/reclaim-advice.c b/tools/testing/selftests/arm64/user4k/reclaim-advice.c new file mode 100644 index 0000000000000..92f28d49b830c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/reclaim-advice.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "re"); + char key[128]; + unsigned long value, result = 0; + + if (!f) + return 0; + while (fscanf(f, "%127s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) + result = value; + fclose(f); + return result; +} + +static unsigned long swapped_kb(void) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256]; + unsigned long result = 0; + + if (!f) + return 0; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmSwap: %lu kB", &result) == 1) + break; + fclose(f); + return result; +} + +static int advise(void *addr, size_t length, int operation, const char *name) +{ + int ret = madvise(addr, length, operation); + + if (ret) + printf("# %s failed: errno=%d (%s)\n", name, errno, + strerror(errno)); + return ret; +} + +static unsigned char pattern(size_t i, unsigned int round) +{ + return (i ^ (i >> 12) ^ (i >> 20) ^ (round * 0x55)) & 255; +} + +static size_t swapped_pages(unsigned char *p, size_t length, size_t page) +{ + uint64_t *values = calloc(length / page, sizeof(*values)); + int fd = open("/proc/self/pagemap", O_RDONLY); + size_t i, count = length / page, swapped = 0; + + if (!values || fd < 0 || + pread(fd, values, count * sizeof(*values), + (uintptr_t)p / page * sizeof(*values)) != + (ssize_t)(count * sizeof(*values))) { + swapped = SIZE_MAX; + } else { + for (i = 0; i < count; i++) + swapped += !!(values[i] & (1ULL << 62)) && + !(values[i] & (1ULL << 63)); + } + if (fd >= 0) + close(fd); + free(values); + return swapped; +} + +static void diagnose_resident(unsigned char *p, size_t length, size_t page) +{ + int pm = open("/proc/self/pagemap", O_RDONLY); + int flags = open("/proc/kpageflags", O_RDONLY); + int count = open("/proc/kpagecount", O_RDONLY); + + for (size_t i = 0; i < length / page; i++) { + uint64_t entry = 0, state = 0, maps = 0; + unsigned long native_pfn; + + if (pread(pm, &entry, 8, ((uintptr_t)p / page + i) * 8) != 8 || + !(entry & (1ULL << 63))) + continue; + native_pfn = (entry & ((1ULL << 55) - 1)) * page / 16384; + if (pread(flags, &state, 8, native_pfn * 8) != 8 || + pread(count, &maps, 8, native_pfn * 8) != 8) + printf("# resident metadata read failed: errno=%d\n", + errno); + printf("# resident leaf=%zu native_pfn=%#lx flags=%#llx maps=%llu\n", + i, native_pfn, (unsigned long long)state, + (unsigned long long)maps); + } + close(pm); + close(flags); + close(count); +} + +/* PAGEOUT drains only the calling CPU's pending LRU additions. */ +static bool drain_pageout_cpus(unsigned char *p, size_t length) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(cpu, &one); + if (sched_setaffinity(0, sizeof(one), &one)) { + ok = false; + break; + } + ok &= advise(p, length, MADV_COLD, "CPU drain COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, "CPU drain PAGEOUT") == 0; + } + ok &= sched_setaffinity(0, sizeof(allowed), &allowed) == 0; + return ok; +} + +static bool swapped_pagemap(unsigned char *p, size_t length, size_t page) +{ + uint64_t *values = calloc(length / page, sizeof(*values)); + int fd = open("/proc/self/pagemap", O_RDONLY); + size_t i, count = length / page; + bool ok = values && fd >= 0; + + if (ok) + ok = pread(fd, values, count * sizeof(*values), + (uintptr_t)p / page * sizeof(*values)) == + (ssize_t)(count * sizeof(*values)); + for (i = 0; ok && i < count; i++) { + uint64_t frame = values[i] & ((1ULL << 55) - 1); + + ok &= !!(values[i] & (1ULL << 62)) && + !(values[i] & (1ULL << 63)); + if (page == 4096) { + ok &= ((frame >> 5) & 3) == + (((uintptr_t)p / page + i) & 3); + if (i && (((uintptr_t)p / page + i) & 3)) + ok &= (values[i] & ((1ULL << 55) - 1)) == + (values[i - 1] & ((1ULL << 55) - 1)) + + (1U << 5); + } + } + if (fd >= 0) + close(fd); + free(values); + printf("%s - %zuK pagemap reports swapped leaves and exact quarter offsets\n", + ok ? "ok" : "not ok", page / 1024); + return ok; +} + +static bool check_swap_pss(pid_t pid, void *p, size_t length, + unsigned long expected_swap, + unsigned long expected_pss) +{ + char path[64], line[512]; + unsigned long start, end, swap = 0, pss = 0; + unsigned int fields = 0; + bool found = false, ok; + FILE *f; + + snprintf(path, sizeof(path), "/proc/%d/smaps", pid); + f = fopen(path, "re"); + if (!f) + return false; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + if (found) + break; + found = start == (uintptr_t)p && + end == (uintptr_t)p + length; + continue; + } + if (!found) + continue; + if (sscanf(line, "Swap: %lu", &swap) == 1) + fields |= 1; + else if (sscanf(line, "SwapPss: %lu", &pss) == 1) + fields |= 2; + } + fclose(f); + ok = found && fields == 3 && swap == expected_swap && + pss == expected_pss; + if (!ok) + printf("# swap stats found=%d fields=%u Swap=%lu/%lu SwapPss=%lu/%lu\n", + found, fields, swap, expected_swap, pss, expected_pss); + return ok; +} + +static bool swapped_accounting(unsigned char *p, size_t length, size_t page, + bool forked) +{ + int ready[2], go[2], status; + pid_t child; + char byte; + bool ok = check_swap_pss(getpid(), p, length, length / 1024, + length / 1024); + + if (!forked || !ok) + goto out; + if (pipe(ready) || pipe(go)) + return false; + child = fork(); + if (child < 0) + return false; + if (!child) { + close(ready[0]); + close(go[1]); + if (read(go[0], &byte, 1) != 1) + _exit(1); + p[page] ^= 0x37; + if (write(ready[1], "x", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(2); + _exit(0); + } + close(ready[1]); + close(go[0]); + ok &= check_swap_pss(getpid(), p, length, length / 1024, length / 2048); + ok &= check_swap_pss(child, p, length, length / 1024, length / 2048); + ok &= write(go[1], "x", 1) == 1 && read(ready[0], &byte, 1) == 1; + ok &= check_swap_pss(getpid(), p, length, length / 1024, + (length + page) / 2048); + ok &= check_swap_pss(child, p, length, (length - page) / 1024, + (length - page) / 2048); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ok &= waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status); + ok &= check_swap_pss(getpid(), p, length, length / 1024, length / 1024); +out: + printf("%s - %zuK SwapPss %s\n", ok ? "ok" : "not ok", page / 1024, + forked ? "fork, one-leaf swap-in/COW and child exit" : + "exclusive quarters"); + return ok; +} + +int main(int argc, char **argv) +{ + const size_t length = 4 * 1024 * 1024; + size_t page = getauxval(AT_PAGESZ), count = length / page, i, j, absent; + unsigned char *p, *vec, *reservation; + unsigned long before, after, swapped_before, swapped_after; + unsigned int round, tries; + size_t bad_bytes, bad_residency; + bool ok = true; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + reservation = mmap(NULL, length + 2 * page, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reservation == MAP_FAILED) + return 3; + p = reservation + page; + vec = malloc(count); + if (!vec || mprotect(p, length, PROT_READ | PROT_WRITE)) + return 3; + for (round = 0; round < 4 && ok; round++) { + for (i = 0; i < length; i++) + p[i] = pattern(i, round); + before = swapouts(); + swapped_before = swapped_kb(); + ok = advise(p, length, MADV_COLD, "COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, "PAGEOUT") == 0; + after = before; + absent = 0; + for (tries = 0; ok && tries < 200; tries++) { + if (mincore(p, length, vec)) { + ok = false; + break; + } + absent = 0; + for (i = 0; i < count; i++) + absent += !(vec[i] & 1); + after = swapouts(); + if (after > before && + swapped_pages(p, length, page) == count) + break; + if (tries == 32) { + printf("# checking pending remote LRU additions before CPU sweep\n"); + diagnose_resident(p, length, page); + ok &= drain_pageout_cpus(p, length); + printf("# after CPU sweep: swapped=%zu/%zu\n", + swapped_pages(p, length, page), count); + } + /* PAGEOUT may skip busy folios; establish the full-swap fixture. */ + usleep(10000); + ok &= advise(p, length, MADV_COLD, "retry COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, + "retry PAGEOUT") == 0; + } + printf("# pageout setup attempts=%u\n", tries + 1); + if (tries == 200) + diagnose_resident(p, length, page); + ok &= tries < 200; + swapped_after = swapped_kb(); + /* mincore also counts warm swap-cache data as resident. */ + ok &= after > before && swapped_after > swapped_before; + ok &= swapped_pagemap(p, length, page); + ok &= swapped_accounting(p, length, page, round == 1); + if (!round) + ok &= advise(p + page, page, MADV_WILLNEED, + "WILLNEED") == 0; + else { + if (round == 3) { + ok &= mlock(p, length) == 0; + ok &= swapped_pages(p, length, page) == 0; + } else { + ok &= advise(p, length, MADV_POPULATE_READ, + "POPULATE_READ") == 0; + } + ok &= mincore(p, length, vec) == 0; + bad_residency = 0; + for (i = 0; i < count; i++) + bad_residency += !(vec[i] & 1); + printf("# after %s: absent=%zu/%zu\n", + round == 3 ? "mlock" : "POPULATE_READ", + bad_residency, count); + ok &= bad_residency == 0; + } + bad_bytes = 0; + /* Nonsequential user-page order, every byte checked after swap. */ + for (i = 0; i < count; i++) { + size_t index = ((i * 61) & (count - 1)) * page; + + for (j = index; j < index + page; j++) + if (p[j] != pattern(j, round)) { + if (!bad_bytes) + printf("# first bad byte at %#zx: got=%#x expected=%#x\n", + j, p[j], + pattern(j, round)); + bad_bytes++; + ok = false; + } + } + if (round == 3) + ok &= munlock(p, length) == 0; + printf("# bad bytes=%zu\n", bad_bytes); + printf("%s - %zuK EL0 pageout round %u: pswpout=%lu->%lu VmSwap=%lu->%lu kB absent=%zu/%zu %s\n", + ok ? "ok" : "not ok", page / 1024, round, before, after, + swapped_before, swapped_after, absent, count, + round == 3 ? "mlock/readback" : + round ? "prefault/readback" : + "willneed/readback"); + } + munmap(reservation, length + 2 * page); + free(vec); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/runtime-app-init.c b/tools/testing/selftests/arm64/user4k/runtime-app-init.c new file mode 100644 index 0000000000000..9d1c7c2c49351 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/runtime-app-init.c @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +int main(int argc, char **argv) +{ + if (argc >= 4 && !strcmp(argv[1], "--exec")) { + if (prctl(SET_EXEC_PAGE_SIZE, strtoul(argv[2], NULL, 10), 0UL, + 0UL, 0UL)) + return 8; + execv(argv[3], argv + 3); + perror("exec"); + return 9; + } + if (getpid() != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + int failed = 0; + pid_t children[3]; + unsigned long sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + unsigned count = sizes[0] > 16384 ? 3 : 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + failed = 1; + goto done; + } + mkdir("/dev/shm", 0777); + mount("tmpfs", "/dev/shm", "tmpfs", 0, NULL); + for (unsigned i = 0; i < count; i++) { + children[i] = fork(); + if (!children[i]) { + char ps[32]; + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(3); + execl("/usr/bin/python3", "/usr/bin/python3", "-I", + "/runtime-app.py", ps, NULL); + perror("exec Python"); + _exit(4); + } + } + for (unsigned i = 0; i < count; i++) { + int status = 0; + int ok = children[i] > 0 && + waitpid(children[i], &status, 0) == children[i] && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - distro Python ABI %luK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + } +done: + if (failed) { + FILE *f = fopen("/trace4k", "r"); + if (f) { + char line[2048]; + while (fgets(line, sizeof(line), f)) + fputs(line, stdout); + fclose(f); + } + } + printf("GRANULE DISTRO RUNTIME %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/runtime-app.py b/tools/testing/selftests/arm64/user4k/runtime-app.py new file mode 100755 index 0000000000000..47fac03300efc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/runtime-app.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: GPL-2.0-only +import os,sys,mmap,ctypes,threading,concurrent.futures,tempfile,subprocess,signal,zlib,gzip,lzma,hashlib,socket,select,json,sqlite3 +ps=int(sys.argv[1]);assert os.sysconf('SC_PAGE_SIZE')==ps==mmap.PAGESIZE,(ps,mmap.PAGESIZE) +libc=ctypes.CDLL(None,use_errno=True);libc.prctl.argtypes=[ctypes.c_int,ctypes.c_ulong,ctypes.c_ulong,ctypes.c_ulong,ctypes.c_ulong] +def passed(name):print(f'ok - distro Python {ps//1024}K {name}',flush=True) +passed('dynamic loader and page-size contract') +with mmap.mmap(-1, 2*ps, flags=0x08|mmap.MAP_ANONYMOUS) as droppable: + assert droppable[:]==bytes(2*ps) + droppable[:]=bytes([0x45])*(2*ps) + child=os.fork() + if child==0:os._exit(0 if droppable[:]==bytes(2*ps) else 9) + assert os.waitpid(child,0)[1]==0 + droppable.madvise(mmap.MADV_DONTNEED) + assert droppable[:]==bytes(2*ps) +libc.arc4random_buf.argtypes=[ctypes.c_void_p,ctypes.c_size_t] +random_buffer=ctypes.create_string_buffer(32) +random_samples=set() +for _ in range(128): + libc.arc4random_buf(random_buffer,len(random_buffer));random_samples.add(random_buffer.raw) +assert len(random_samples)==128 +passed('droppable write, fork wipe, discard and glibc random state') + +blob=bytes(range(256))*4096 +with tempfile.TemporaryDirectory() as tmp: + path=tmp+'/mapped' + with open(path,'w+b') as f: + f.truncate(8*ps) + with mmap.mmap(f.fileno(),2*ps,offset=ps) as m: + m[:]=bytes([0x53])*(2*ps);m.flush() + assert os.pread(f.fileno(),2*ps,ps)==m[:] + m.resize(3*ps);m[2*ps:]=bytes([0x72])*ps;m.flush() + assert os.pread(f.fileno(),ps,3*ps)==bytes([0x72])*ps + passed('file mmap offset, flush and resize') + with mmap.mmap(-1,4*ps,flags=mmap.MAP_PRIVATE|mmap.MAP_ANONYMOUS) as m: + m[:]=bytes([0x39])*(4*ps) + c=os.fork() + if c==0: + m[ps:2*ps]=bytes([0x91])*ps + os._exit(0 if m[:ps]==bytes([0x39])*ps else 5) + assert os.waitpid(c,0)[1]==0 and m[:]==bytes([0x39])*(4*ps) + address=ctypes.addressof(ctypes.c_char.from_buffer(m)) + libc.mprotect.argtypes=[ctypes.c_void_p,ctypes.c_size_t,ctypes.c_int] + assert libc.mprotect(address+ps,ps,0)==0 + c=os.fork() + if c==0: + m[ps]=0 + os._exit(6) + status=os.waitpid(c,0)[1];assert os.WIFSIGNALED(status) and os.WTERMSIG(status)==signal.SIGSEGV + assert m[0]==0x39 and m[2*ps]==0x39 + assert libc.mprotect(address+ps,ps,mmap.PROT_READ|mmap.PROT_WRITE)==0 + passed('fork COW and independent protection fault') + def compressed(i): + d=blob+str(i).encode() + for compress,decompress in [(zlib.compress,zlib.decompress),(gzip.compress,gzip.decompress),(lambda d:lzma.compress(d,preset=1),lzma.decompress)]:assert decompress(compress(d))==d + return hashlib.sha256(d).hexdigest() + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + results=list(pool.map(compressed,range(8))) + assert len(set(results))==8 + passed('threaded native extension allocation, compression and hashing') + db=sqlite3.connect(tmp+'/db') + db.execute('pragma journal_mode=wal');db.execute('pragma mmap_size=1048576') + db.execute('create table entries (n integer primary key, data blob)') + db.executemany('insert into entries values (?,?)',[(i,blob[:4096]) for i in range(128)]);db.commit() + assert db.execute('select count(*) from entries').fetchone()[0]==128 + assert all(row[0]==blob[:4096] for row in db.execute('select data from entries'));db.close() + passed('SQLite WAL and mapped database reads') + left,right=socket.socketpair() + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + task=pool.submit(lambda:(right.sendall(blob),right.shutdown(socket.SHUT_WR))) + out=bytearray() + while True: + assert select.select([left],[],[],5)[0] + part=left.recv(997) + if not part:break + out.extend(part) + task.result();assert bytes(out)==blob + left.close();right.close();passed('socket polling and fragmented transfers') + fd=os.memfd_create('cross-abi',0);os.ftruncate(fd,131072);os.pwrite(fd,b'parent',ps) + # Execute the child in the other ABI, retaining the same shared file bytes. + target=16384 if ps==4096 else 4096 + def select_abi(): + if libc.prctl(0x41555001,target,0,0,0):os._exit(7) + code="import os,mmap,sys;fd=int(sys.argv[1]);off=int(sys.argv[2]);want=int(sys.argv[3]);assert os.sysconf('SC_PAGE_SIZE')==want;m=mmap.mmap(fd,131072);assert m[off:off+6]==b'parent';m[off:off+5]=b'child';m.close()" + subprocess.run([sys.executable,'-I','-c',code,str(fd),str(ps),str(target)],pass_fds=(fd,),preexec_fn=select_abi,check=True) + assert os.pread(fd,5,ps)==b'child';os.close(fd);passed('cross-ABI dynamic exec and memfd sharing') + command="set -e; a=(); for ((i=0;i<2048;i++)); do a+=(\"value-$i\"); done; [[ ${#a[@]} == 2048 ]]; [[ ${a[2047]} == value-2047 ]]; printf 'bash-ok\\n'" + def same_abi(): + if libc.prctl(0x41555001,ps,0,0,0):os._exit(8) + result=subprocess.run(['/bin/bash','--noprofile','--norc','-c',command],preexec_fn=same_abi,stdout=subprocess.PIPE,check=True,text=True) + assert result.stdout=='bash-ok\n';passed('distro Bash dynamic exec, allocation and pipes') +print(f'DISTRO PYTHON {ps} PASS',flush=True) diff --git a/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c b/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c new file mode 100644 index 0000000000000..b040d8fc43a4f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c @@ -0,0 +1,396 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#ifndef MADV_COLLAPSE +#define MADV_COLLAPSE 25 +#endif +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - collapse line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct command { + uint64_t op, offset, len, value; +}; +struct ready { + uint64_t base, head; +}; +static uint64_t entry(const char *path, uint64_t index) +{ + uint64_t value; + int fd = open(path, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &value, 8, index * 8) == 8 && !close(fd)); + return value; +} +static uint64_t mapped_pfn(pid_t pid, uintptr_t address, size_t ps) +{ + char path[80]; + snprintf(path, sizeof(path), "/proc/%d/pagemap", pid); + uint64_t e = entry(path, address / ps); + CHECK(e & (1ULL << 63)); + return e & ((1ULL << 55) - 1); +} +static unsigned char *map(int file, size_t len, size_t align, off_t offset) +{ + void *reserve = mmap(NULL, len + align, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + uintptr_t addr = ((uintptr_t)reserve + align - 1) & ~(align - 1); + CHECK(!munmap(reserve, len + align)); + unsigned char *p = mmap((void *)addr, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, offset); + CHECK(p != MAP_FAILED); + return p; +} +static int register_range(unsigned char *p, size_t len, int minor) +{ + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = minor ? UFFD_FEATURE_MINOR_SHMEM : + 0 }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register r = { + .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + (minor ? UFFDIO_REGISTER_MODE_MINOR : 0) + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, &r)); + return fd; +} +static void copy_leaf(int fd, uintptr_t dst, unsigned char value) +{ + unsigned char src[4096]; + memset(src, value, sizeof(src)); + struct uffdio_copy c = { .dst = dst, + .src = (uintptr_t)src, + .len = sizeof(src) }; + CHECK(!ioctl(fd, UFFDIO_COPY, &c) && c.copy == sizeof(src)); +} +static void cpu(unsigned id) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(id, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +static int worker(int file, int sock, size_t native, size_t len) +{ + CHECK(getauxval(AT_PAGESZ) == 4096); + alarm(90); + cpu(1); + unsigned char *p = map(file, len, len, 0); + unsigned char *q = map(file, len - 4096, len, 4096); + CHECK(p[0] == 0x61 && q[0] == 0x61); + uint64_t old_head = + mapped_pfn(getpid(), (uintptr_t)p, 4096) * 4096 / native; + CHECK(entry("/proc/kpageflags", old_head) & + (1ULL << KPF_COMPOUND_HEAD)); + int fd = register_range(p, len, 1); + for (size_t i = 4 * native; i < len; i += native) + copy_leaf(fd, (uintptr_t)p + i, 0x71); + int pipes[2]; + CHECK(!pipe(pipes)); + struct iovec iov = { p, 4096 }; + CHECK(vmsplice(pipes[1], &iov, 1, 0) == 4096); + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + native, native)); + CHECK(entry("/proc/kpageflags", old_head) & + (1ULL << KPF_COMPOUND_HEAD)); + CHECK(!close(pipes[0]) && !close(pipes[1])); + struct ready r = { (uintptr_t)p, old_head }; + struct iovec v = { &r, sizeof(r) }; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(fd))]; + } control = { 0 }; + struct msghdr msg = { .msg_iov = &v, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control.bytes) }; + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + c->cmsg_level = SOL_SOCKET; + c->cmsg_type = SCM_RIGHTS; + c->cmsg_len = CMSG_LEN(sizeof(fd)); + memcpy(CMSG_DATA(c), &fd, sizeof(fd)); + CHECK(sendmsg(sock, &msg, 0) == sizeof(r)); + struct command cmd; + while (recv(sock, &cmd, sizeof(cmd), 0) == sizeof(cmd)) { + CHECK(cmd.offset < len && cmd.len <= len - cmd.offset); + unsigned char value = 0; + if (!cmd.op) + break; + if (cmd.op == 1) + value = *(volatile unsigned char *)(p + cmd.offset); + else if (cmd.op == 2) + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_KEEP_SIZE, + cmd.offset, cmd.len)); + else if (cmd.op == 3) { + CHECK(cmd.offset >= 4096); + value = *(volatile unsigned char *)(q + cmd.offset - + 4096); + } else if (cmd.op == 4) { + CHECK(cmd.offset >= 4096); + memset(q + cmd.offset - 4096, cmd.value, cmd.len); + } else + CHECK(0); + CHECK(send(sock, &value, 1, 0) == 1); + } + CHECK(!munmap(q, len - 4096) && !munmap(p, len) && !close(fd) && + !close(file) && !close(sock)); + return 0; +} +static void send_command(int sock, uint64_t op, size_t offset, size_t len) +{ + struct command c = { .op = op, .offset = offset, .len = len }; + CHECK(send(sock, &c, sizeof(c), 0) == sizeof(c)); +} +static void reply(int sock, unsigned char expected) +{ + unsigned char value; + CHECK(recv(sock, &value, 1, 0) == 1 && value == expected); +} +static void fault(int sock, int fd, uintptr_t base, size_t offset, int minor, + unsigned char value) +{ + send_command(sock, 1, offset + 17, 0); + struct pollfd p = { .fd = fd, .events = POLLIN }; + struct uffd_msg m; + CHECK(poll(&p, 1, 10000) == 1 && (p.revents & POLLIN)); + CHECK(read(fd, &m, sizeof(m)) == sizeof(m)); + CHECK(m.event == UFFD_EVENT_PAGEFAULT && + m.arg.pagefault.address == base + offset && + m.arg.pagefault.flags == (minor ? UFFD_PAGEFAULT_FLAG_MINOR : 0)); + if (minor) { + struct uffdio_continue c = { .range = { base + offset, 4096 } }; + CHECK(!ioctl(fd, UFFDIO_CONTINUE, &c) && c.mapped == 4096); + } else + copy_leaf(fd, base + offset, value); + reply(sock, value); +} +static unsigned long pmd_mapped(unsigned char *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + CHECK(f); + char *line = NULL; + size_t cap = 0; + unsigned long a, b, kb, result = 0; + int selected = 0; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &a, &b) == 2) + selected = a == (uintptr_t)p; + else if (selected && + sscanf(line, "ShmemPmdMapped: %lu kB", &kb) == 1) + result = kb * 1024; + } + free(line); + fclose(f); + return result; +} +static int collapse(unsigned char *p, size_t len) +{ + int ret; + unsigned tries = 0; + do { + errno = 0; + ret = madvise(p, len, MADV_COLLAPSE); + if (!ret || errno != EAGAIN) + break; + usleep(20000); + } while (++tries < 10); + printf("# collapse ret=%d errno=%d PMD-mapped=%lu\n", ret, + ret ? errno : 0, pmd_mapped(p)); + return ret; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10), + strtoul(argv[5], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + alarm(120); + cpu(0); + CHECK(!mount("proc", "/proc", "proc", 0, NULL) && + !mount("sysfs", "/sys", "sysfs", 0, NULL)); + CHECK(!mount("tmpfs", "/tmp", "tmpfs", 0, + "huge=within_size,size=256m")); + size_t native = getauxval(AT_PAGESZ), len = 0; + FILE *f = fopen("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", + "r"); + CHECK(f && fscanf(f, "%zu", &len) == 1); + fclose(f); + CHECK(native == 16384 && len == 32 * 1024 * 1024); + int file = open("/tmp/collapse", O_CREAT | O_RDWR | O_TRUNC, 0600); + CHECK(file >= 0 && !ftruncate(file, 4 * native)); + unsigned char *initial = map(file, 4 * native, 4 * native, 0); + memset(initial, 0x61, 4 * native); + CHECK(entry("/proc/kpageflags", + mapped_pfn(getpid(), (uintptr_t)initial, native)) & + (1ULL << KPF_COMPOUND_HEAD)); + CHECK(!munmap(initial, 4 * native) && !unlink("/tmp/collapse") && + !ftruncate(file, len)); + int sock[2]; + CHECK(!socketpair(AF_UNIX, SOCK_SEQPACKET, 0, sock)); + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char fd[24], so[24], ns[24], ls[24]; + close(sock[0]); + snprintf(fd, sizeof(fd), "%d", file); + snprintf(so, sizeof(so), "%d", sock[1]); + snprintf(ns, sizeof(ns), "%zu", native); + snprintf(ls, sizeof(ls), "%zu", len); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, 4096, 0UL, 0UL, 0UL)); + execl("/init", "/init", "--worker", fd, so, ns, ls, NULL); + _exit(3); + } + close(sock[1]); + struct ready ready; + struct iovec v = { &ready, sizeof(ready) }; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct msghdr msg = { .msg_iov = &v, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control.bytes) }; + CHECK(recvmsg(sock[0], &msg, 0) == sizeof(ready) && + !(msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))); + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + CHECK(c && c->cmsg_level == SOL_SOCKET && c->cmsg_type == SCM_RIGHTS); + int uffd; + memcpy(&uffd, CMSG_DATA(c), sizeof(uffd)); + unsigned char *p = map(file, len, len, 0); + int native_uffd = register_range(p, len, 0); + CHECK(collapse(p, len) == -1 && errno == EINVAL && pmd_mapped(p) == 0); + fault(sock[0], uffd, ready.base, 0, 1, 0x61); + fault(sock[0], uffd, ready.base, len - native, 1, 0x71); + uint64_t head = mapped_pfn(pid, ready.base, 4096) * 4096 / native; + CHECK(head != ready.head && + (entry("/proc/kpageflags", head) & (1ULL << KPF_COMPOUND_HEAD))); + CHECK(mapped_pfn(pid, ready.base + len - native, 4096) * 4096 / + native == + head + len / native - 1); + for (size_t i = 1; i < len / native; i++) + CHECK(entry("/proc/kpageflags", head + i) & + (1ULL << KPF_COMPOUND_TAIL)); + /* Collapse retained the absent native unit and independent fine holes. */ + fault(sock[0], uffd, ready.base, native, 0, 0x92); + fault(sock[0], uffd, ready.base, 5 * native + 4096, 0, 0xa3); + puts("ok - PMD cache collapse preserves missing/minor state and neighbour data"); + /* Every native leaf now contains data: coarse exposure may populate zero subunits. */ + CHECK(!collapse(p, len) && pmd_mapped(p) == len); + fault(sock[0], uffd, ready.base, 6 * native + 4096, 1, 0); + struct uffdio_copy duplicate = { .dst = ready.base + 6 * native + 8192, + .src = (uintptr_t)p, + .len = 4096 }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &duplicate) == -1 && errno == EEXIST); + puts("ok - native PMD promotion exposes coarse leaf zeros without overwriting fine data"); + /* A new fine hole must invalidate the covering native PMD mapping. */ + send_command(sock[0], 2, 7 * native + 4096, 4096); + reply(sock[0], 0); + CHECK(pmd_mapped(p) == 0); + fault(sock[0], uffd, ready.base, 7 * native + 4096, 0, 0xb4); + CHECK(p[7 * native] == 0x71 && p[7 * native + 4096] == 0xb4 && + p[7 * native + 8192] == 0); + CHECK(p[native] == 0x92 && p[5 * native + 4096] == 0xa3); + puts("ok - fine hole splits PMD alias and preserves populated neighbouring leaves"); + for (unsigned round = 0; round < 8; round++) { + CHECK(!collapse(p, len) && pmd_mapped(p) == len); + size_t off = (16 + round) * native + 4096; + /* Prime CPU0's PMD translation before CPU1 punches the fine hole. */ + CHECK(*(volatile unsigned char *)(p + off) == 0); + send_command(sock[0], 3, off + 17, 0); + reply(sock[0], 0); + send_command(sock[0], 2, off, 4096); + reply(sock[0], 0); + CHECK(pmd_mapped(p) == 0); + if (round % 3) { + /* Stale native or shifted fine translations would miss promotion. */ + if (round % 3 == 1) + memset(p + off, 0xc0 + round, 4096); + else { + struct command write = { 4, off, 4096, + 0xc0 + round }; + CHECK(send(sock[0], &write, sizeof(write), 0) == + sizeof(write)); + reply(sock[0], 0); + } + struct uffdio_copy collision = { .dst = ready.base + + off, + .src = (uintptr_t)p, + .len = 4096 }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &collision) == -1 && + errno == EEXIST); + fault(sock[0], uffd, ready.base, off, 1, 0xc0 + round); + } else + fault(sock[0], uffd, ready.base, off, 0, 0xc0 + round); + CHECK(p[off - 4096] == 0x71 && p[off] == 0xc0 + round && + p[off + 4096] == 0); + send_command(sock[0], 3, off + 17, 0); + reply(sock[0], 0xc0 + round); + } + CHECK(!collapse(p, len) && pmd_mapped(p) == len); + for (size_t i = 0; i < len; i++) { + size_t page = i / native, low = i % native; + unsigned char expected = + page < 4 ? + (page == 1 ? (low < 4096 ? 0x92 : 0) : 0x61) : + (low < 4096 ? 0x71 : 0); + if (low >= 4096 && low < 8192) { + if (page == 5) + expected = 0xa3; + if (page == 7) + expected = 0xb4; + if (page >= 16 && page < 24) + expected = 0xc0 + page - 16; + } + CHECK(p[i] == expected); + } + puts("ok - eight cross-CPU PMD cycles, native/shifted-fine writes and complete 32MiB bytes"); + send_command(sock[0], 0, 0, 0); + int status; + CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!close(uffd) && !close(native_uffd) && !munmap(p, len) && + !close(file) && !close(sock[0])); + puts("SHMEM COLLAPSE CONTRACT PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-hole-init.c b/tools/testing/selftests/arm64/user4k/shmem-hole-init.c new file mode 100644 index 0000000000000..8831cc30a22c3 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-hole-init.c @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem hole line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct access { + volatile unsigned char *p; + unsigned char value; +}; +static void *read_byte(void *opaque) +{ + struct access *a = opaque; + a->value = *a->p; + return NULL; +} +static uint64_t pagemap(pid_t pid, uintptr_t address, size_t granule) +{ + char name[80]; + uint64_t entry; + snprintf(name, sizeof(name), "/proc/%d/pagemap", pid); + int fd = open(name, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &entry, 8, address / granule * 8) == 8); + CHECK(!close(fd)); + return entry; +} +static void fill(int fd, unsigned char *dst, unsigned char *src, size_t ps) +{ + struct uffdio_copy c = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = ps }; + CHECK(!ioctl(fd, UFFDIO_COPY, &c) && c.copy == (long long)ps); +} +static void resolve(int fd, unsigned char *p, size_t ps, int minor, + unsigned char value) +{ + struct access a = { .p = p + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &a)); + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + struct uffd_msg msg; + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == + (minor ? UFFD_PAGEFAULT_FLAG_MINOR : 0)); + if (minor) { + struct uffdio_continue c = { .range = { (uintptr_t)p, ps } }; + CHECK(!ioctl(fd, UFFDIO_CONTINUE, &c) && + c.mapped == (long long)ps); + } else { + unsigned char *source = malloc(ps); + CHECK(source); + memset(source, value, ps); + fill(fd, p, source, ps); + free(source); + } + CHECK(!pthread_join(thread, NULL) && a.value == value); +} +struct command { + uint64_t op, offset, len, value; +}; +static int alias_worker(int file, int input, int output, size_t len, + size_t native) +{ + CHECK(getauxval(AT_PAGESZ) == native); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + CHECK(p != MAP_FAILED); + /* Establish every coarse PTE before the fine process punches its hole. */ + for (size_t i = 0; i < len; i++) + CHECK(p[i] == 0x61); + uint64_t base = (uintptr_t)p; + CHECK(write(output, &base, sizeof(base)) == sizeof(base)); + struct command cmd; + while (read(input, &cmd, sizeof(cmd)) == sizeof(cmd)) { + CHECK(cmd.offset <= len && cmd.len <= len - cmd.offset); + if (!cmd.op) + break; + if (cmd.op == 2) + memset(p + cmd.offset, cmd.value, cmd.len); + for (size_t i = 0; i < cmd.len; i++) + CHECK(p[cmd.offset + i] == cmd.value); + CHECK(write(output, "y", 1) == 1); + } + CHECK(!munmap(p, len)); + return 0; +} +static void tell(int input, int output, uint64_t op, size_t offset, size_t len, + unsigned value) +{ + struct command cmd = { op, offset, len, value }; + char reply; + CHECK(write(input, &cmd, sizeof(cmd)) == sizeof(cmd)); + if (op) + CHECK(read(output, &reply, 1) == 1 && reply == 'y'); +} +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "r"); + char key[80]; + unsigned long value, result = 0; + CHECK(f); + while (fscanf(f, "%79s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) + result = value; + fclose(f); + return result; +} +static unsigned long mapping_swap(void *address) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end, value, result = 0; + int selected = 0; + CHECK(f); + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + selected = start == (uintptr_t)address; + else if (selected && sscanf(line, "Swap: %lu kB", &value) == 1) + result = value; + } + free(line); + fclose(f); + return result; +} +static void swapped_missing(size_t ps, size_t native) +{ + int file = memfd_create("partial-uffd-swap", 0); + CHECK(file >= 0 && !ftruncate(file, native)); + unsigned char *p = + mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *source = malloc(ps); + CHECK(p != MAP_FAILED && source); + memset(source, 0x71, ps); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MINOR_SHMEM }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, native }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_MINOR }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + fill(fd, p, source, ps); + unsigned long before = swapouts(), usage = 0, written; + unsigned tries; + for (tries = 0; tries < 100; tries++) { + CHECK(!madvise(p, native, MADV_PAGEOUT)); + usleep(20000); + usage = mapping_swap(p); + if (usage >= native / 1024 && swapouts() > before) + break; + } + written = swapouts() - before; + CHECK(usage >= native / 1024 && written > 0); + /* The selected cache index is swapped; its unfilled neighbour stays missing. */ + resolve(fd, p + ps, ps, 0, 0x92); + resolve(fd, p, ps, 1, 0x71); + for (size_t i = 0; i < ps; i++) + CHECK(p[i] == 0x71 && p[ps + i] == 0x92); + CHECK(mapping_swap(p) == 0); + printf("ok - shared uffd %zuK on %zuK swapped cache=%lukB writes=%lu attempts=%u missing COPY preserves neighbour\n", + ps / 1024, native / 1024, usage, written, tries + 1); + CHECK(!close(fd) && !munmap(p, native) && !close(file)); + free(source); +} +static int worker(size_t ps, size_t native) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(60); + if (getenv("SHMEM_HOLE_SWAP") && ps < native) + swapped_missing(ps, native); + size_t len = 4 * native; + int file = memfd_create("hole-contract", 0); + CHECK(file >= 0 && !ftruncate(file, len)); + unsigned char *source = malloc(len); + CHECK(source); + memset(source, 0x61, len); + CHECK(pwrite(file, source, len, 0) == (ssize_t)len); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *q = mmap(NULL, len - ps, PROT_READ | PROT_WRITE, + MAP_SHARED, file, ps); + unsigned char *cow = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, file, 0); + CHECK(p != MAP_FAILED && q != MAP_FAILED && cow != MAP_FAILED); + for (size_t i = 0; i < len; i += ps) + CHECK(p[i] == 0x61); + for (size_t i = 0; i < len - ps; i += ps) + CHECK(q[i] == 0x61); + memset(cow + ps, 0xe1, ps); + int commands[2], replies[2]; + CHECK(!pipe(commands) && !pipe(replies)); + pid_t alias = fork(); + CHECK(alias >= 0); + if (!alias) { + char f[24], in[24], out[24], size[24], ns[24]; + close(commands[1]); + close(replies[0]); + snprintf(f, sizeof(f), "%d", file); + snprintf(in, sizeof(in), "%d", commands[0]); + snprintf(out, sizeof(out), "%d", replies[1]); + snprintf(size, sizeof(size), "%zu", len); + snprintf(ns, sizeof(ns), "%zu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, native, 0UL, 0UL, 0UL)); + execl("/init", "/init", "--alias", f, in, out, size, ns, NULL); + _exit(3); + } + close(commands[0]); + close(replies[1]); + uint64_t alias_base; + CHECK(read(replies[0], &alias_base, sizeof(alias_base)) == + sizeof(alias_base)); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MINOR_SHMEM }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_MINOR }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(pagemap(alias, alias_base + ps, native) & (1ULL << 63)); + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, + ps)); + CHECK(!(pagemap(getpid(), (uintptr_t)p + ps, ps) & (1ULL << 63))); + CHECK(!(pagemap(getpid(), (uintptr_t)q, ps) & (1ULL << 63))); + CHECK(!(pagemap(alias, alias_base + ps, native) & (1ULL << 63))); + /* Fine neighbour PTEs survive, while COW retains its independent bytes. */ + CHECK(pagemap(getpid(), (uintptr_t)p, ps) & (1ULL << 63)); + CHECK(pagemap(getpid(), (uintptr_t)p + 2 * ps, ps) & (1ULL << 63)); + CHECK(p[0] == 0x61 && p[2 * ps] == 0x61 && cow[ps] == 0xe1); + resolve(fd, p + ps, ps, 0, 0xa2); + CHECK(q[0] == 0xa2 && cow[ps] == 0xe1); + tell(commands[1], replies[0], 1, ps, ps, 0xa2); + tell(commands[1], replies[0], 1, 0, ps, 0x61); + tell(commands[1], replies[0], 1, 2 * ps, ps, 0x61); + /* Refaulting a coarse writable alias makes its covered units populated. */ + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, + ps)); + tell(commands[1], replies[0], 2, ps, ps, 0xb7); + struct uffdio_copy collision = { .dst = (uintptr_t)p + ps, + .src = (uintptr_t)source, + .len = ps }; + errno = 0; + CHECK(ioctl(fd, UFFDIO_COPY, &collision) == -1 && errno == EEXIST); + resolve(fd, p + ps, ps, 1, 0xb7); + printf("ok - shmem hole %zuK on %zuK precise PTEs, shifted alias, COW, coarse revoke/refault and COPY collision\n", + ps / 1024, native / 1024); + /* Stop the alias before resize, so it cannot intentionally promote holes. */ + tell(commands[1], replies[0], 0, 0, 0, 0); + close(commands[1]); + close(replies[0]); + int status; + CHECK(waitpid(alias, &status, 0) == alias && WIFEXITED(status) && + !WEXITSTATUS(status)); + size_t cut = 2 * ps + ps / 2; + CHECK(!ftruncate(file, cut) && !ftruncate(file, len)); + /* Partial final user leaf remains valid, with only truncated bytes zero. */ + if (!(pagemap(getpid(), (uintptr_t)p + 2 * ps, ps) & (1ULL << 63))) + resolve(fd, p + 2 * ps, ps, 1, 0x61); + CHECK(p[2 * ps] == 0x61 && p[cut - 1] == 0x61 && p[cut] == 0 && + p[3 * ps - 1] == 0); + resolve(fd, p + 3 * ps, ps, 0, 0xc3); + CHECK(q[2 * ps] == 0xc3); + CHECK(cow[ps] == 0xe1); + printf("ok - shmem resize %zuK on %zuK partial tail retained, shrink/regrow missing leaf refilled\n", + ps / 1024, native / 1024); + CHECK(!close(fd) && !munmap(p, len) && !munmap(q, len - ps) && + !munmap(cow, len) && !close(file)); + free(source); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 7 && !strcmp(argv[1], "--alias")) + return alias_worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), + strtoul(argv[5], NULL, 10), + strtoul(argv[6], NULL, 10)); + if (argc == 4 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10)); + int swap_test = argc == 2 && !strcmp(argv[1], "--swap"); + if (getpid() != 1 || (argc != 1 && !swap_test)) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + if (swap_test) { + struct stat st; + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)); + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + CHECK(!stat("/dev/vda", &st) && S_ISBLK(st.st_mode) && + !swapon("/dev/vda", 0)); + CHECK(!setenv("SHMEM_HOLE_SWAP", "1", 1)); + } + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { native, 4096, 16384 }; + int failed = 0; + for (unsigned i = 0; i < (native == 65536 ? 3U : 2U); i++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char ps[24], ns[24]; + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(ns, sizeof(ns), "%lu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, + 0UL)); + execl("/init", "/init", "--worker", ps, ns, NULL); + _exit(3); + } + int status = 0; + int ok = waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status); + printf("%s - shmem hole ABI %luK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + } + if (swap_test) + CHECK(!swapoff("/dev/vda")); + printf("SHMEM HOLE CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-pin-init.c b/tools/testing/selftests/arm64/user4k/shmem-pin-init.c new file mode 100644 index 0000000000000..51deda0fde52c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-pin-init.c @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem pin line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct ring { + int fd; + struct io_uring_params p; + void *sq, *cq, *sqes; + size_t sqlen, cqlen, sqelen; +}; +static unsigned load(unsigned *p) +{ + return __atomic_load_n(p, __ATOMIC_ACQUIRE); +} +static void store(unsigned *p, unsigned v) +{ + __atomic_store_n(p, v, __ATOMIC_RELEASE); +} +static size_t align_size(size_t n) +{ + size_t ps = getauxval(AT_PAGESZ); + return (n + ps - 1) & ~(ps - 1); +} +static void destroy(struct ring *r) +{ + if (r->sq && r->sq != MAP_FAILED) + munmap(r->sq, r->sqlen); + if (r->cq && r->cq != MAP_FAILED && r->cq != r->sq) + munmap(r->cq, r->cqlen); + if (r->sqes && r->sqes != MAP_FAILED) + munmap(r->sqes, r->sqelen); + if (r->fd >= 0) + close(r->fd); + *r = (struct ring){ .fd = -1 }; +} +static bool setup(struct ring *r, unsigned entries, unsigned flags) +{ + *r = (struct ring){ .fd = -1 }; + r->p.flags = flags; + r->fd = syscall(__NR_io_uring_setup, entries, &r->p); + if (r->fd < 0) + return false; + r->sqlen = align_size(r->p.sq_off.array + + r->p.sq_entries * sizeof(unsigned)); + r->cqlen = align_size(r->p.cq_off.cqes + + r->p.cq_entries * sizeof(struct io_uring_cqe) * + (flags & IORING_SETUP_CQE32 ? 2 : 1)); + r->sqelen = align_size(r->p.sq_entries * sizeof(struct io_uring_sqe) * + (flags & IORING_SETUP_SQE128 ? 2 : 1)); + if (r->p.features & IORING_FEAT_SINGLE_MMAP) { + if (r->cqlen > r->sqlen) + r->sqlen = r->cqlen; + r->cqlen = r->sqlen; + } + r->sq = mmap(NULL, r->sqlen, PROT_READ | PROT_WRITE, MAP_SHARED, r->fd, + IORING_OFF_SQ_RING); + r->cq = (r->p.features & IORING_FEAT_SINGLE_MMAP) ? + r->sq : + mmap(NULL, r->cqlen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_CQ_RING); + r->sqes = mmap(NULL, r->sqelen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_SQES); + return r->sq != MAP_FAILED && r->cq != MAP_FAILED && + r->sqes != MAP_FAILED; +} +static bool request(struct ring *r, unsigned op, int fd, void *buf, + unsigned len, uint64_t off, bool async) +{ + unsigned *sqtail = r->sq + r->p.sq_off.tail, + *sqmask = r->sq + r->p.sq_off.ring_mask; + unsigned *cqhead = r->cq + r->p.cq_off.head, + *cqtail = r->cq + r->p.cq_off.tail, + *cqmask = r->cq + r->p.cq_off.ring_mask; + unsigned tail = load(sqtail), head = load(cqhead), + idx = tail & load(sqmask); + unsigned sqstride = sizeof(struct io_uring_sqe) * + (r->p.flags & IORING_SETUP_SQE128 ? 2 : 1); + unsigned cqstride = sizeof(struct io_uring_cqe) * + (r->p.flags & IORING_SETUP_CQE32 ? 2 : 1); + struct io_uring_sqe *sqe = r->sqes + idx * sqstride; + memset(sqe, 0, sqstride); + sqe->opcode = op; + sqe->fd = fd; + sqe->addr = (uintptr_t)buf; + sqe->len = len; + sqe->off = off; + sqe->user_data = (uint64_t)tail + 1234; + if (async) + sqe->flags = IOSQE_ASYNC; + ((unsigned *)(r->sq + r->p.sq_off.array))[idx] = idx; + store(sqtail, tail + 1); + int ret = syscall(__NR_io_uring_enter, r->fd, 1, 1, + IORING_ENTER_GETEVENTS, NULL, 0); + if (ret != 1 || load(cqtail) != head + 1) { + printf("enter=%d errno=%d\n", ret, errno); + return false; + } + struct io_uring_cqe *cqe = + r->cq + r->p.cq_off.cqes + (head & load(cqmask)) * cqstride; + unsigned expected = op == IORING_OP_NOP ? 0 : len; + if (op == IORING_OP_READV_FIXED || op == IORING_OP_WRITEV_FIXED) { + expected = 0; + for (unsigned i = 0; i < len; i++) + expected += ((struct iovec *)buf)[i].iov_len; + } + bool ok = cqe->user_data == (uint64_t)tail + 1234 && + cqe->res == (int)expected; + if (!ok) + printf("cqe res=%d expected=%u\n", cqe->res, expected); + store(cqhead, head + 1); + return ok; +} + +static void fragment_geometry(unsigned char *p, size_t ps, int input) +{ + struct ring r; + CHECK(setup(&r, 8, 0)); + struct iovec reg = { p + 123, 3 * ps - 246 }; + unsigned char *src = malloc(3 * ps), *expected = malloc(3 * ps); + CHECK(src && expected); + memset(expected, 0x61, 3 * ps); + memset(src, 0xb7, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_REGISTER_BUFFERS, + ®, 1)); + CHECK(request(&r, IORING_OP_READ_FIXED, input, p + ps + 71, ps + 31, 17, + true)); + memset(expected + ps + 71, 0xb7, ps + 31); + CHECK(!memcmp(p, expected, 3 * ps)); + memset(src, 0xc8, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_WRITE_FIXED, input, p + ps + 71, ps + 31, 9, + true)); + CHECK(pread(input, src, ps + 31, 9) == (ssize_t)(ps + 31)); + for (size_t i = 0; i < ps + 31; i++) + CHECK(src[i] == 0xb7); + struct iovec vec[2] = { { p + 2 * ps + 17, ps - 400 }, + { p + ps + 91, ps - 600 } }; + memset(src, 0xd5, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_READV_FIXED, input, vec, 2, 13, true)); + for (unsigned i = 0; i < 2; i++) + memset(expected + ((unsigned char *)vec[i].iov_base - p), 0xd5, + vec[i].iov_len); + CHECK(!memcmp(p, expected, 3 * ps)); + memset(src, 0xc8, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_WRITEV_FIXED, input, vec, 2, 23, true)); + size_t total = vec[0].iov_len + vec[1].iov_len; + CHECK(pread(input, src, total, 23) == (ssize_t)total); + for (size_t i = 0; i < total; i++) + CHECK(src[i] == 0xd5); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_UNREGISTER_BUFFERS, + NULL, 0)); + destroy(&r); + free(src); + free(expected); + memset(p, 0x61, 3 * ps); + printf("ok - %zuK fixed/vector IO uses user fragment geometry and preserves guards\n", + ps / 1024); +} + +struct punch_worker { + int file; + size_t off, ps; + pthread_barrier_t start; +}; +static void bind_cpu(unsigned cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +static void *punch_repeatedly(void *arg) +{ + struct punch_worker *w = arg; + bind_cpu(1); + pthread_barrier_wait(&w->start); + for (unsigned i = 0; i < 256; i++) { + CHECK(!fallocate(w->file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + w->off, w->ps)); + sched_yield(); + } + return NULL; +} +static void pin_hole_concurrency(struct ring *r, int file, int input, + unsigned char *p, size_t off, size_t ps) +{ + struct punch_worker w = { .file = file, .off = off, .ps = ps }; + pthread_t thread; + CHECK(!pthread_barrier_init(&w.start, NULL, 2)); + bind_cpu(0); + CHECK(!pthread_create(&thread, NULL, punch_repeatedly, &w)); + pthread_barrier_wait(&w.start); + for (unsigned i = 0; i < 256; i++) + CHECK(request(r, IORING_OP_READ_FIXED, input, p + off, ps, 0, + true)); + CHECK(!pthread_join(thread, NULL) && + !pthread_barrier_destroy(&w.start)); + printf("ok - %zuK concurrent fixed writes and 256 hole punches on separate CPUs\n", + ps / 1024); +} + +static int target(size_t native, size_t ps, int resize) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(30); + size_t len = 4 * native, off = native + ps; + bool retained = ps < native; + int file = memfd_create("pin-cache", 0), + input = memfd_create("pin-input", 0); + CHECK(file >= 0 && input >= 0 && !ftruncate(file, len) && + !ftruncate(input, ps)); + unsigned char *src = malloc(len), *back = malloc(len); + CHECK(src && back); + memset(src, 0x61, len); + CHECK(pwrite(file, src, len, 0) == (ssize_t)len); + memset(src, 0xb7, ps); + CHECK(pwrite(input, src, ps, 0) == (ssize_t)ps); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + CHECK(p != MAP_FAILED && !madvise(p, len, MADV_NOHUGEPAGE)); + CHECK(p[off] == 0x61); + fragment_geometry(p, ps, input); + memset(src, 0xb7, ps); + CHECK(pwrite(input, src, ps, 0) == (ssize_t)ps); + struct ring r; + CHECK(setup(&r, 8, 0)); + struct iovec reg = { p + off, ps }; + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_REGISTER_BUFFERS, + ®, 1)); + int uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register ur = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, &ur)); + if (resize) + CHECK(!ftruncate(file, off) && !ftruncate(file, len)); + else + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + off, ps)); + if (retained && !resize) + pin_hole_concurrency(&r, file, input, p, off, ps); + /* Fixed IO accesses its original FOLL_PIN reference after PTE removal. */ + CHECK(request(&r, IORING_OP_READ_FIXED, input, p + off, ps, 0, true)); + /* Also prove the original pinned object contains the completed write. */ + CHECK(request(&r, IORING_OP_WRITE_FIXED, input, p + off, ps, 0, true)); + CHECK(pread(input, back, ps, 0) == (ssize_t)ps); + for (size_t i = 0; i < ps; i++) + CHECK(back[i] == 0xb7); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_UNREGISTER_BUFFERS, + NULL, 0)); + memset(src, 0xc8, ps); + struct uffdio_copy c = { .dst = (uintptr_t)(p + off), + .src = (uintptr_t)src, + .len = ps }; + errno = 0; + int ret = ioctl(uffd, UFFDIO_COPY, &c); + printf("# pin %zuK/%zuK resize=%d retained=%d COPY=%d errno=%d bytes=%lld\n", + ps / 1024, native / 1024, resize, retained, ret, errno, + (long long)c.copy); + if (retained) + CHECK(ret == -1 && errno == EEXIST && c.copy == -EEXIST); + else + CHECK(!ret && c.copy == (long long)ps); + CHECK(pread(file, back, len, 0) == (ssize_t)len); + for (size_t i = 0; i < len; i++) { + unsigned char expected = i >= off && i < off + ps ? + (retained ? 0xb7 : 0xc8) : + resize && i >= off ? 0 : + 0x61; + CHECK(back[i] == expected); + } + for (size_t i = 0; i < ps; i++) + CHECK(p[off + i] == (retained ? 0xb7 : 0xc8)); + /* A new punch after unpin can once again be populated by the pager. */ + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, off, + ps)); + c.copy = 0; + CHECK(!ioctl(uffd, UFFDIO_COPY, &c) && c.copy == (long long)ps); + for (size_t i = 0; i < ps; i++) + CHECK(p[off + i] == 0xc8); + destroy(&r); + CHECK(!close(uffd) && !munmap(p, len) && !close(file) && !close(input)); + free(src); + free(back); + printf("ok - writable pin %zuK on %zuK resize=%d retention and detach\n", + ps / 1024, native / 1024, resize); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--target")) + return target(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) + for (int resize = 0; resize < 2; resize++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char n[24], g[24], r[12]; + snprintf(n, sizeof(n), "%zu", native); + snprintf(g, sizeof(g), "%zu", ps); + snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); + execl("/init", "init", "--target", n, g, r, + NULL); + _exit(127); + } + int status; + CHECK(waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status)); + } + puts("ok - shmem writable pin fixture complete"); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c b/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c new file mode 100644 index 0000000000000..f321878396f06 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/pressure" +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem pressure line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +static void put(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + CHECK(fd >= 0); + CHECK(write(fd, value, strlen(value)) == (ssize_t)strlen(value) && + !close(fd)); +} +static unsigned long long get(const char *path, const char *key) +{ + FILE *f = fopen(path, "r"); + CHECK(f); + char name[128]; + unsigned long long value, result = 0; + if (!key) + CHECK(fscanf(f, "%llu", &result) == 1); + else + while (fscanf(f, "%127s %llu", name, &value) == 2) + if (!strcmp(key, name)) { + result = value; + break; + } + fclose(f); + return result; +} +static void copy_leaf(int uffd, unsigned char *dst, unsigned char *src, + size_t ps, unsigned char value) +{ + memset(src, value, ps); + struct uffdio_copy c = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = ps }; + int ret = ioctl(uffd, UFFDIO_COPY, &c); + if (ret) + printf("# COPY destination=%p errno=%d transferred=%lld\n", dst, + errno, (long long)c.copy); + CHECK(!ret && c.copy == (long long)ps); +} +static void checkpoint(int out, int in) +{ + char token; + CHECK(write(out, "R", 1) == 1 && read(in, &token, 1) == 1 && + token == 'G'); +} +static int worker(size_t native, size_t ps, int out, int in) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(120); + size_t records = (16UL << 20) / native, + stride = ps < native ? native : 2 * native; + size_t len = records * stride; + int file = memfd_create("pressure-partial-cache", 0); + CHECK(file >= 0 && !ftruncate(file, len)); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *source = malloc(stride), *back = malloc(stride); + CHECK(p != MAP_FAILED && source && back); + CHECK(!madvise(p, len, MADV_NOHUGEPAGE)); + int uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, ®)); + for (size_t i = 0; i < records; i++) + copy_leaf(uffd, p + i * stride, source, ps, 0xa1); + checkpoint(out, in); + /* Revisit in permuted order, filling holes in reclaimed cache backing. */ + for (size_t i = 0; i < records; i++) { + size_t slot = (i * 4051) & (records - 1); + copy_leaf(uffd, p + slot * stride + ps, source, ps, 0xb2); + for (size_t j = 0; j < ps; j++) + CHECK(p[slot * stride + j] == 0xa1 && + p[slot * stride + ps + j] == 0xb2); + } + checkpoint(out, in); + for (size_t i = 0; i < records; i += 2) { + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + i * stride, ps)); + copy_leaf(uffd, p + i * stride, source, ps, 0xc3); + } + checkpoint(out, in); + for (size_t i = 0; i < records; i++) { + size_t slot = (i * 4051 + 31) & (records - 1); + CHECK(pread(file, back, stride, slot * stride) == + (ssize_t)stride); + for (size_t j = 0; j < stride; j++) { + unsigned char expected = + j < ps ? (slot & 1 ? 0xa1 : 0xc3) : + j < 2 * ps ? 0xb2 : + 0; + CHECK(back[j] == expected); + } + struct uffdio_copy duplicate = { + .dst = (uintptr_t)(p + slot * stride), + .src = (uintptr_t)source, + .len = ps + }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &duplicate) == -1 && + errno == EEXIST && duplicate.copy == -EEXIST); + } + checkpoint(out, in); + CHECK(!close(uffd) && !munmap(p, len) && !close(file)); + free(source); + free(back); + printf("ok - shmem pressure %zuK on %zuK partial-cache refill, holes, duplicate rejection and every byte\n", + ps / 1024, native / 1024); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4]), + atoi(argv[5])); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL) && + !mount("sysfs", "/sys", "sysfs", 0, NULL)); + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) && + !mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL)); + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory"); + CHECK(!mkdir(CG, 0755)); + put(CG "/memory.max", "8388608"); + put(CG "/memory.swap.max", "134217728"); + struct stat st; + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + CHECK(!swapon("/dev/vda", 0)); + size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) { + int ready[2], go[2]; + CHECK(!pipe(ready) && !pipe(go)); + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + close(ready[0]); + close(go[1]); + char n[24], p[24], r[24], g[24]; + snprintf(n, sizeof(n), "%zu", native); + snprintf(p, sizeof(p), "%zu", ps); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(g, sizeof(g), "%d", go[0]); + put(CG "/cgroup.procs", "0"); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); + execl("/init", "init", "--worker", n, p, r, g, NULL); + _exit(127); + } + close(ready[1]); + close(go[0]); + for (unsigned phase = 0; phase < 4; phase++) { + char token; + CHECK(read(ready[0], &token, 1) == 1 && token == 'R'); + unsigned long long swap = get(CG "/memory.swap.current", + NULL), + max = get(CG "/memory.events", + "max"); + unsigned long long scan = get(CG "/memory.stat", + "pgscan"), + steal = get(CG "/memory.stat", + "pgsteal"); + printf("# pressure %zuK phase=%u resident=%llu swap=%llu max=%llu scan=%llu steal=%llu\n", + ps / 1024, phase, + get(CG "/memory.current", NULL), swap, max, scan, + steal); + CHECK(swap && max && scan && steal && + !get(CG "/memory.events", "oom") && + !get(CG "/memory.events", "oom_kill")); + CHECK(write(go[1], "G", 1) == 1); + } + int status; + CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status)); + close(ready[0]); + close(go[1]); + /* Closed/unmapped file identity must release all of its cache/swap. */ + for (unsigned i = 0; + i < 3000 && (get(CG "/memory.swap.current", NULL) || + get(CG "/memory.stat", "shmem")); + i++) + usleep(20000); + CHECK(!get(CG "/memory.swap.current", NULL) && + !get(CG "/memory.stat", "shmem")); + } + CHECK(!swapoff("/dev/vda")); + puts("SHMEM PRESSURE CONTRACT PASS"); + sync(); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-race-init.c b/tools/testing/selftests/arm64/user4k/shmem-race-init.c new file mode 100644 index 0000000000000..5109654b6ec21 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-race-init.c @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define ROUNDS 512 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem race line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct control { + unsigned start, stop, ready; + unsigned long aliases; +}; +struct state { + struct control *control; + int file, uffd, resize; + size_t ps, native, len; + unsigned char *p, *q, *source; + unsigned copied, existed, eof, alloc_eof, retry; +}; +static unsigned load(unsigned *p) +{ + return __atomic_load_n(p, __ATOMIC_ACQUIRE); +} +static void store(unsigned *p, unsigned value) +{ + __atomic_store_n(p, value, __ATOMIC_RELEASE); +} +static void bind_cpu(unsigned cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +static void start_wait(struct control *c) +{ + __atomic_add_fetch(&c->ready, 1, __ATOMIC_RELEASE); + while (!load(&c->start)) + sched_yield(); +} +static void *mutate(void *arg) +{ + struct state *s = arg; + bind_cpu(0); + start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + if (s->resize) + CHECK(!ftruncate(s->file, s->ps) && + !ftruncate(s->file, s->len)); + else + CHECK(!fallocate(s->file, + FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_KEEP_SIZE, + s->ps, s->ps)); + sched_yield(); + } + return NULL; +} +static void *populate(void *arg) +{ + struct state *s = arg; + bind_cpu(1); + start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + struct uffdio_copy c = { .dst = (uintptr_t)(s->p + s->ps), + .src = (uintptr_t)s->source, + .len = s->ps }; + errno = 0; + int ret = ioctl(s->uffd, UFFDIO_COPY, &c); + if (!ret) { + CHECK(c.copy == (long long)s->ps); + s->copied++; + } else { + CHECK(ret == -1 && c.copy == -errno); + if (errno == EEXIST) + s->existed++; + else if (errno == EFAULT && s->resize) + s->eof++; + /* Native shmem allocator preserves its NULL/ENOMEM EOF ABI. */ + else if (errno == ENOMEM && s->resize && + s->ps == s->native) + s->alloc_eof++; + else if (errno == EAGAIN) + s->retry++; + else { + printf("# unexpected COPY error=%d bytes=%lld\n", + errno, (long long)c.copy); + CHECK(0); + } + } + sched_yield(); + } + return NULL; +} +static int valid_byte(unsigned char value) +{ + return value == 0 || value == 0x61 || value == 0xa2; +} +static void *fine_alias(void *arg) +{ + struct state *s = arg; + bind_cpu(0); + start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + if (!s->resize) { + CHECK(!madvise(s->q, s->ps, MADV_DONTNEED)); + CHECK(valid_byte(*(volatile unsigned char *)s->q)); + } else { + /* The retained prefix remains inside EOF throughout resize. */ + CHECK(*(volatile unsigned char *)s->p == 0x61); + } + sched_yield(); + } + return NULL; +} +static int native_alias(int file, int control, size_t native, size_t ps, + int resize) +{ + CHECK(getauxval(AT_PAGESZ) == native); + bind_cpu(1); + alarm(60); + struct control *c = mmap(NULL, native, PROT_READ | PROT_WRITE, + MAP_SHARED, control, 0); + unsigned char *p = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_SHARED, file, 0); + CHECK(c != MAP_FAILED && p != MAP_FAILED && + !madvise(p, 4 * native, MADV_NOHUGEPAGE)); + start_wait(c); + unsigned long cycles = 0; + while (!load(&c->stop)) { + CHECK(!madvise(p, 2 * native, MADV_DONTNEED)); + CHECK(*(volatile unsigned char *)p == 0x61); + if (!resize) + CHECK(valid_byte(*(volatile unsigned char *)(p + ps))); + cycles++; + sched_yield(); + } + c->aliases = cycles; + CHECK(!munmap(p, 4 * native) && !munmap(c, native)); + return 0; +} +static int target(size_t native, size_t ps, int resize) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(60); + struct state s = { + .native = native, .ps = ps, .len = 4 * native, .resize = resize + }; + s.file = memfd_create("race-cache", 0); + int control = memfd_create("race-control", 0); + CHECK(s.file >= 0 && control >= 0 && !ftruncate(s.file, s.len) && + !ftruncate(control, native)); + s.control = mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, + control, 0); + s.source = malloc(s.len); + CHECK(s.control != MAP_FAILED && s.source); + memset(s.source, 0x61, s.len); + CHECK(pwrite(s.file, s.source, s.len, 0) == (ssize_t)s.len); + s.p = mmap(NULL, s.len, PROT_READ | PROT_WRITE, MAP_SHARED, s.file, 0); + s.q = mmap(NULL, s.len - ps, PROT_READ | PROT_WRITE, MAP_SHARED, s.file, + ps); + CHECK(s.p != MAP_FAILED && s.q != MAP_FAILED && + !madvise(s.p, s.len, MADV_NOHUGEPAGE)); + CHECK(s.p[0] == + 0x61); /* Avoid blocking the resize prefix reader on UFFD. */ + unsigned char *cow = MAP_FAILED; + if (!resize) { + cow = mmap(NULL, s.len, PROT_READ | PROT_WRITE, MAP_PRIVATE, + s.file, 0); + CHECK(cow != MAP_FAILED); + memset(cow + ps, 0xe1, ps); + } + s.uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(s.uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(s.uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)s.p, s.len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(s.uffd, UFFDIO_REGISTER, ®)); + memset(s.source, 0xa2, ps); + pid_t alias = fork(); + CHECK(alias >= 0); + if (!alias) { + char f[24], c[24], n[24], p[24], r[12]; + snprintf(f, sizeof(f), "%d", s.file); + snprintf(c, sizeof(c), "%d", control); + snprintf(n, sizeof(n), "%zu", native); + snprintf(p, sizeof(p), "%zu", ps); + snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, native, 0, 0, 0)); + execl("/init", "init", "--alias", f, c, n, p, r, NULL); + _exit(127); + } + pthread_t mutation, copy, fine; + CHECK(!pthread_create(&mutation, NULL, mutate, &s) && + !pthread_create(©, NULL, populate, &s) && + !pthread_create(&fine, NULL, fine_alias, &s)); + while (load(&s.control->ready) != 4) + sched_yield(); + store(&s.control->start, 1); + CHECK(!pthread_join(mutation, NULL) && !pthread_join(copy, NULL) && + !pthread_join(fine, NULL)); + store(&s.control->stop, 1); + int status; + CHECK(waitpid(alias, &status, 0) == alias && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(s.control->aliases && + s.copied + s.existed + s.eof + s.alloc_eof + s.retry == ROUNDS); + printf("# race %zuK/%zuK resize=%d COPY=%u EEXIST=%u EOF=%u native-alloc-EOF=%u retry=%u native-refaults=%lu\n", + ps / 1024, native / 1024, resize, s.copied, s.existed, s.eof, + s.alloc_eof, s.retry, s.control->aliases); + /* Once concurrent actors stop, exact cache/PTE state must be recoverable. */ + CHECK(!ftruncate(s.file, s.len)); + CHECK(!fallocate(s.file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, + ps)); + memset(s.source, 0xd4, ps); + struct uffdio_copy fill = { .dst = (uintptr_t)(s.p + ps), + .src = (uintptr_t)s.source, + .len = ps }; + CHECK(!ioctl(s.uffd, UFFDIO_COPY, &fill) && fill.copy == (long long)ps); + unsigned char *back = malloc(s.len); + CHECK(back && pread(s.file, back, s.len, 0) == (ssize_t)s.len); + for (size_t i = 0; i < s.len; i++) { + unsigned char expected = i >= ps && i < 2 * ps ? 0xd4 : + resize && i >= ps ? 0 : + 0x61; + if (back[i] != expected) { + printf("# byte=%zu actual=%#x expected=%#x\n", i, + back[i], expected); + CHECK(0); + } + } + for (size_t i = 0; i < ps; i++) { + CHECK(s.p[ps + i] == 0xd4 && s.q[i] == 0xd4); + if (!resize) + CHECK(cow[ps + i] == 0xe1); + } + CHECK(!close(s.uffd) && !munmap(s.p, s.len) && + !munmap(s.q, s.len - ps)); + if (!resize) + CHECK(!munmap(cow, s.len)); + CHECK(!munmap(s.control, native) && !close(control) && !close(s.file)); + free(back); + free(s.source); + printf("ok - concurrent shmem %zuK on %zuK resize=%d cache bytes, aliases and private COW\n", + ps / 1024, native / 1024, resize); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 7 && !strcmp(argv[1], "--alias")) + return native_alias(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10), + strtoul(argv[5], NULL, 10), atoi(argv[6])); + if (argc == 5 && !strcmp(argv[1], "--target")) + return target(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) + for (int resize = 0; resize < 2; resize++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char n[24], p[24], r[12]; + snprintf(n, sizeof(n), "%zu", native); + snprintf(p, sizeof(p), "%zu", ps); + snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); + execl("/init", "init", "--target", n, p, r, + NULL); + _exit(127); + } + int status; + CHECK(waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status)); + } + puts("SHMEM RACE CONTRACT PASS"); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-thp-init.c b/tools/testing/selftests/arm64/user4k/shmem-thp-init.c new file mode 100644 index 0000000000000..e08ea92103fcc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-thp-init.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem THP line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +static uint64_t read_entry(const char *path, uint64_t index) +{ + uint64_t value; + int fd = open(path, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &value, 8, index * 8) == 8 && !close(fd)); + return value; +} +static uint64_t pfn(void *p, size_t ps) +{ + uint64_t entry = read_entry("/proc/self/pagemap", (uintptr_t)p / ps); + CHECK(entry & (1ULL << 63)); + return entry & ((1ULL << 55) - 1); +} +static unsigned char *map_aligned(int fd, size_t len) +{ + void *reserve = mmap(NULL, 2 * len, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + uintptr_t address = ((uintptr_t)reserve + len - 1) & ~(len - 1); + CHECK(!munmap(reserve, 2 * len)); + unsigned char *p = mmap((void *)address, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, fd, 0); + CHECK(p != MAP_FAILED); + return p; +} +struct reader { + volatile unsigned char *p; + unsigned char value; +}; +static void *read_byte(void *arg) +{ + struct reader *r = arg; + r->value = *r->p; + return NULL; +} +static int target(int file, size_t native, size_t ps, int hold, int zero, + int resize, uint64_t head) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(30); + size_t len = 4 * native, target_off = resize ? 2 * native : native; + size_t cut = native + ps / 2; + unsigned char *p = map_aligned(file, len); + for (size_t i = 0; i < len; i += ps) + CHECK(p[i] == 0x61); + CHECK(pfn(p, ps) * ps / native == head); + int pipes[2]; + CHECK(!pipe(pipes)); + if (hold) { + struct iovec iov = { p, ps }; + CHECK(vmsplice(pipes[1], &iov, 1, 0) == (ssize_t)ps); + } + int uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, ®)); + if (resize) + CHECK(!ftruncate(file, cut) && !ftruncate(file, len)); + else + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + native, ps)); + uint64_t flags = read_entry("/proc/kpageflags", head); + printf("# THP %zuK native=%zuK hold=%d zero=%d resize=%d head_flags=%#llx\n", + ps / 1024, native / 1024, hold, zero, resize, + (unsigned long long)flags); + if (hold) + CHECK((flags & + ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))) == + ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))); + struct reader r = { .p = p + target_off + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct pollfd pollfd = { .fd = uffd, .events = POLLIN }; + struct uffd_msg msg; + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(uffd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && !msg.arg.pagefault.flags && + msg.arg.pagefault.address == (uintptr_t)(p + target_off)); + unsigned char *source = malloc(ps); + CHECK(source); + memset(source, 0x92, ps); + int ret; + struct uffdio_copy invalid = { .dst = (uintptr_t)(p + target_off), + .src = 1, + .len = ps }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &invalid) == -1 && errno == EFAULT && + invalid.copy == -EFAULT); + if (zero) { + struct uffdio_zeropage z = { + .range = { (uintptr_t)(p + target_off), ps } + }; + ret = ioctl(uffd, UFFDIO_ZEROPAGE, &z); + printf("# ZEROPAGE returned=%d errno=%d bytes=%lld\n", ret, + errno, (long long)z.zeropage); + CHECK(!ret && z.zeropage == (long long)ps); + } else { + struct uffdio_copy c = { .dst = (uintptr_t)(p + target_off), + .src = (uintptr_t)source, + .len = ps }; + ret = ioctl(uffd, UFFDIO_COPY, &c); + printf("# COPY returned=%d errno=%d bytes=%lld\n", ret, errno, + (long long)c.copy); + CHECK(!ret && c.copy == (long long)ps); + } + CHECK(!pthread_join(thread, NULL) && r.value == (zero ? 0 : 0x92)); + memset(source, 0xd4, ps); + struct uffdio_copy duplicate = { .dst = (uintptr_t)(p + target_off), + .src = (uintptr_t)source, + .len = ps }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &duplicate) == -1 && errno == EEXIST && + duplicate.copy == -EEXIST); + /* File reads verify zeroed tail holes without faulting unfilled UFFD leaves. */ + unsigned char *back = malloc(len); + CHECK(back); + CHECK(pread(file, back, len, 0) == (ssize_t)len); + for (size_t i = 0; i < len; i++) { + unsigned char expected = i >= target_off && + i < target_off + ps ? + (zero ? 0 : 0x92) : + resize && i >= cut ? 0 : + 0x61; + CHECK(back[i] == expected); + if (!resize || (i >= target_off && i < target_off + ps)) + CHECK(p[i] == expected); + } + free(back); + if (hold) { + CHECK(read(pipes[0], source, ps) == (ssize_t)ps); + for (size_t i = 0; i < ps; i++) + CHECK(source[i] == 0x61); + } + CHECK(!close(pipes[0]) && !close(pipes[1]) && !close(uffd) && + !munmap(p, len) && !close(file)); + free(source); + printf("ok - THP %zuK on %zuK hold=%d zero=%d resize=%d missing refill and all neighbour bytes\n", + ps / 1024, native / 1024, hold, zero, resize); + return 0; +} +static int create_file(size_t native, uint64_t *head) +{ + int fd = open("/tmp/huge", O_CREAT | O_TRUNC | O_RDWR, 0600); + CHECK(fd >= 0); + CHECK(!ftruncate(fd, 4 * native)); + unsigned char *p = map_aligned(fd, 4 * native); + CHECK(!madvise(p, 4 * native, MADV_HUGEPAGE)); + memset(p, 0x61, 4 * native); + *head = pfn(p, native); + uint64_t f = read_entry("/proc/kpageflags", *head); + printf("# initial head=%llu flags=%#llx VA=%p native=%zu\n", + (unsigned long long)*head, (unsigned long long)f, p, native); + CHECK((f & ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))) == + ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))); + for (unsigned i = 1; i < 4; i++) { + CHECK(pfn(p + i * native, native) == *head + i); + CHECK(read_entry("/proc/kpageflags", *head + i) & + (1ULL << KPF_COMPOUND_TAIL)); + } + /* Keep the name until allocation: unlinked mappings use anon-shmem policy. */ + CHECK(!unlink("/tmp/huge") && !munmap(p, 4 * native)); + return fd; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 9 && !strcmp(argv[1], "--target")) + return target(atoi(argv[2]), strtoul(argv[3], NULL, 10), + strtoul(argv[4], NULL, 10), atoi(argv[5]), + atoi(argv[6]), atoi(argv[7]), + strtoull(argv[8], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("tmpfs", "/tmp", "tmpfs", 0, "huge=within_size,size=32m")); + size_t native = getauxval(AT_PAGESZ), sizes[] = { native, 4096, 16384 }; + int failed = 0; + for (unsigned abi = 0; abi < (native == 65536 ? 3U : 2U); abi++) + for (unsigned hold = 0; hold < 2; hold++) + for (unsigned zero = 0; zero < 2; zero++) + for (unsigned resize = 0; resize < 2; + resize++) { + uint64_t head; + int fd = create_file(native, &head); + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char fs[24], ns[24], ps[24], + hs[24], zs[24], rs[24], + pfs[24]; + snprintf(fs, sizeof(fs), "%d", + fd); + snprintf(ns, sizeof(ns), "%zu", + native); + snprintf(ps, sizeof(ps), "%zu", + sizes[abi]); + snprintf(hs, sizeof(hs), "%u", + hold); + snprintf(zs, sizeof(zs), "%u", + zero); + snprintf(rs, sizeof(rs), "%u", + resize); + snprintf(pfs, sizeof(pfs), + "%llu", + (unsigned long long) + head); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, + sizes[abi], 0UL, + 0UL, 0UL)); + execl("/init", "/init", + "--target", fs, ns, ps, + hs, zs, rs, pfs, NULL); + _exit(3); + } + int status = 0; + int ok = waitpid(pid, &status, 0) == + pid && + WIFEXITED(status) && + !WEXITSTATUS(status); + printf("%s - THP ABI %zuK hold=%u zero=%u resize=%u status=%#x\n", + ok ? "ok" : "not ok", + sizes[abi] / 1024, hold, zero, + resize, status); + failed |= !ok; + CHECK(!close(fd)); + } + printf("SHMEM THP CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c b/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c new file mode 100644 index 0000000000000..73ff33d5e0ba1 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PID-1 runner for a disposable VM with a dedicated virtio swap disk. + * Use kunit.autorun=0: tests skipped at boot stay skipped on a debugfs rerun. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int run_swap_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-shmem-swap/"; + char path[128], *log; + size_t used = 0, capacity = 1024 * 1024; + ssize_t n; + int fd, ret = -1; + + snprintf(path, sizeof(path), "%srun", base); + fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = write(fd, "1", 1); + close(fd); + if (n != 1) + return -1; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + log = calloc(1, capacity); + if (!log) { + close(fd); + return -1; + } + while (used < capacity - 1 && + (n = read(fd, log + used, capacity - 1 - used)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (n == 0 && used < capacity - 1 && + strstr(log, "\nok 1 arm64-user4k-shmem-swap\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; + free(log); + return ret; +} + +static int fault_setting(const char *name, const char *value) +{ + char path[160]; + snprintf(path, sizeof(path), "/sys/kernel/debug/failslab/%s", name); + int fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) + return -1; + ssize_t n = write(fd, value, strlen(value)); + close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} +static int configure_failslab(bool enable) +{ + if (!enable) + return fault_setting("probability", "0"); + /* The KUnit worker selects itself only around metadata writeout. */ + return fault_setting("task-filter", "Y") || + fault_setting("verbose", "0") || fault_setting("times", "-1") || + fault_setting("interval", "1") || fault_setting("space", "0") || + fault_setting("probability", "100"); +} + +static int swap_control_helper(const char *operation) +{ + struct stat init, self; + int ret; + + /* A helper is allowed only when this same executable is the VM's PID 1. */ + if (getpid() == 1 || stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || init.st_dev != self.st_dev || + init.st_ino != self.st_ino) { + fprintf(stderr, + "swap helper requires its fixture runner as PID 1\n"); + return 1; + } + if (!strcmp(operation, "--memcg-setup-helper")) { + if (mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) && + errno != EBUSY) + return 1; + int fd = open("/sys/fs/cgroup/cgroup.subtree_control", + O_WRONLY | O_CLOEXEC); + if (fd < 0) + return 1; + ssize_t n = write(fd, "+memory", 7); + close(fd); + if (n != 7) + return 1; + return mkdir("/sys/fs/cgroup/uffd-metadata", 0755) && + errno != EEXIST; + } + if (!strcmp(operation, "--failslab-on-helper")) + return configure_failslab(true); + if (!strcmp(operation, "--failslab-off-helper")) + return configure_failslab(false); + ret = !strcmp(operation, "--swapoff-helper") ? swapoff("/dev/vda") : + swapon("/dev/vda", 0); + if (ret) + perror(operation); + return ret ? 1 : 0; +} + +int main(int argc, char **argv) +{ + struct stat st; + bool active = false; + int ret = 1; + if (argc == 2 && (!strcmp(argv[1], "--swapon-helper") || + !strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--failslab-on-helper") || + !strcmp(argv[1], "--failslab-off-helper") || + !strcmp(argv[1], "--memcg-setup-helper"))) + return swap_control_helper(argv[1]); + if (getpid() != 1 || argc != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + unsigned long ps = getauxval(AT_PAGESZ); + if (ps != 16384 && ps != 65536) + goto out; + if (mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) + goto out; + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || + swapon("/dev/vda", 0)) + goto out; + active = true; + ret = run_swap_suite() ? 1 : 0; +out: + if (active && swapoff("/dev/vda")) + ret = 1; + printf("SHMEM UFFD STATE SWAP %s errno=%d\n", ret ? "FAIL" : "PASS", + errno); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/splice-fixture-init.c b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c new file mode 100644 index 0000000000000..282ba6268b25d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c @@ -0,0 +1,610 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static bool use_tcp, use_ipv6, use_udp; + +static int connected_pair(int sockets[2]) +{ + struct sockaddr_in a4 = { .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK) }; + struct sockaddr_in6 a6 = { .sin6_family = AF_INET6, + .sin6_addr = IN6ADDR_LOOPBACK_INIT }; + struct sockaddr_storage peer; + struct sockaddr *address = use_ipv6 ? (void *)&a6 : (void *)&a4; + socklen_t size = use_ipv6 ? sizeof(a6) : sizeof(a4), + peer_size = sizeof(peer); + int domain = use_ipv6 ? AF_INET6 : AF_INET; + int listener, saved; + + if (!use_tcp && !use_udp) + return socketpair(AF_UNIX, SOCK_STREAM, 0, sockets); + listener = socket(domain, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); + if (listener < 0) + return -1; + sockets[0] = -1; + if (bind(listener, address, size) || + getsockname(listener, address, &size)) + goto error; + if (!use_udp && listen(listener, 1)) + goto error; + sockets[0] = socket(domain, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); + if (sockets[0] < 0 || connect(sockets[0], address, size)) + goto error; + if (use_udp) { + if (getsockname(sockets[0], (void *)&peer, &peer_size) || + connect(listener, (void *)&peer, peer_size)) + goto error; + sockets[1] = listener; + return 0; + } + sockets[1] = accept(listener, NULL, NULL); + if (sockets[1] < 0) + goto error; + close(listener); + return 0; +error: + saved = errno; + if (sockets[0] >= 0) + close(sockets[0]); + close(listener); + errno = saved; + return -1; +} + +static unsigned char value(size_t off) +{ + return (off * 13 + (off >> 12) * 31) & 255; +} +static void fill(unsigned char *p, size_t n) +{ + for (size_t i = 0; i < n; i++) + p[i] = value(i); +} +static bool check(const unsigned char *p, size_t n, size_t off) +{ + for (size_t i = 0; i < n; i++) + if (p[i] != value(i + off)) { + ksft_print_msg( + "data mismatch at %zu: got %02x expected %02x\n", + i + off, p[i], value(i + off)); + return false; + } + return true; +} +static bool read_exact(int fd, unsigned char *p, size_t n) +{ + while (n) { + ssize_t got = read(fd, p, n); + + if (got <= 0) { + ksft_print_msg( + "read fd=%d got=%zd remaining=%zu errno=%d\n", + fd, got, n, errno); + return false; + } + p += got; + n -= got; + } + return true; +} +static bool replace(unsigned char *p, size_t n) +{ + if (munmap(p, n) || + mmap(p, n, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0) != p) + return false; + memset(p, 0xec, n); + return true; +} +static bool cow(unsigned char *p, size_t n) +{ + pid_t child = fork(); + int status = 0; + + if (!child) + _exit(0); + if (child < 0 || waitpid(child, &status, 0) != child || status) + return false; + memset(p, 0xab, n); + return true; +} +static void report(bool ok, size_t ps, const char *name) +{ + ksft_test_result(ok, "%zuK %s\n", ps / 1024, name); +} +static bool tls_pair(int sockets[2], unsigned int version) +{ + struct tls12_crypto_info_aes_gcm_128 crypto = { + .info.version = version, + .info.cipher_type = TLS_CIPHER_AES_GCM_128, + }; + struct timeval timeout = { .tv_sec = 5 }; + int no_padding = 1; + + if (connected_pair(sockets)) + return false; + return !setsockopt(sockets[0], IPPROTO_TCP, TCP_ULP, "tls", 4) && + !setsockopt(sockets[1], IPPROTO_TCP, TCP_ULP, "tls", 4) && + !setsockopt(sockets[0], SOL_TLS, TLS_TX, &crypto, + sizeof(crypto)) && + !setsockopt(sockets[1], SOL_TLS, TLS_RX, &crypto, + sizeof(crypto)) && + (version != TLS_1_3_VERSION || + !setsockopt(sockets[1], SOL_TLS, TLS_RX_EXPECT_NO_PAD, + &no_padding, sizeof(no_padding))) && + !setsockopt(sockets[1], SOL_SOCKET, SO_RCVTIMEO, &timeout, + sizeof(timeout)); +} + +static void tls_checks(size_t ps, unsigned char *p, size_t len, + unsigned char *back) +{ + int sockets[2], fds[2]; + char name[160]; + + if (pipe(fds)) + ksft_exit_fail_msg("TLS pipe errno=%d\n", errno); + for (unsigned int v = 0; v < 2; v++) { + unsigned int version = v ? TLS_1_3_VERSION : TLS_1_2_VERSION; + struct iovec vec; + bool ok; + + if (!tls_pair(sockets, version)) + ksft_exit_fail_msg("TLS %x setup errno=%d\n", version, + errno); + for (unsigned int mode = 0; mode < 2; mode++) { + const size_t count = mode ? 4096 : 4096 - 29; + const size_t offset = ps + (mode ? 0 : 29); + + fill(p, len); + vec = (struct iovec){ p + offset, count }; + if (vmsplice(fds[1], &vec, 1, 0) != (ssize_t)count || + splice(fds[0], NULL, sockets[0], NULL, count, + SPLICE_F_MORE) != (ssize_t)count) + ksft_exit_fail_msg("TLS MORE splice errno=%d\n", + errno); + errno = 0; + if (recv(sockets[1], back, len, MSG_DONTWAIT) != -1 || + errno != EAGAIN) + ksft_exit_fail_msg( + "TLS pending plaintext precondition errno=%d\n", + errno); + ok = mode ? cow(p + ps, 4096) : replace(p + ps, ps); + ok &= send(sockets[0], "!", 1, 0) == 1; + ok &= read_exact(sockets[1], back, count + 1) && + check(back, count, offset) && back[count] == '!'; + snprintf(name, sizeof(name), + "TLS 1.%u pending plaintext survives %s", + v + 2, + mode ? "fork/COW" : "unmap/replacement"); + report(ok, ps, name); + } + { + const size_t bytes = 3 * 16384 + 123, prefix = 137; + const size_t mapping = (bytes + ps - 1) & ~(ps - 1); + unsigned char *source = + mmap(NULL, mapping, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *received = malloc(bytes + prefix); + + if (source == MAP_FAILED || !received) + ksft_exit_fail_msg( + "TLS multi-record allocation\n"); + fill(source, mapping); + ok = send(sockets[0], source, prefix, 0) == + (ssize_t)prefix; + vec = (struct iovec){ source, bytes }; + if (vmsplice(fds[1], &vec, 1, 0) != (ssize_t)bytes || + splice(fds[0], NULL, sockets[0], NULL, bytes, 0) != + (ssize_t)bytes) + ksft_exit_fail_msg( + "TLS multi-record splice errno=%d\n", + errno); + ok &= replace(source, mapping); + ok &= read_exact(sockets[1], received, + bytes + prefix) && + check(received, prefix, 0) && + check(received + prefix, bytes, 0); + snprintf( + name, sizeof(name), + "TLS 1.%u ordinary send and multiple spliced records", + v + 2); + report(ok, ps, name); + munmap(source, mapping); + free(received); + } + { + const size_t bytes = 5003; + int file = memfd_create("tls-receive", 0); + unsigned char *mapped; + + if (file < 0 || ftruncate(file, 3 * ps)) + ksft_exit_fail_msg("TLS receive file setup\n"); + mapped = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, file, 0); + if (mapped == MAP_FAILED) + ksft_exit_fail_msg( + "TLS receive file mapping\n"); + memset(mapped, 0x5c, 3 * ps); + fill(p, len); + ok = send(sockets[0], p, bytes, 0) == (ssize_t)bytes && + read_exact(sockets[1], mapped + 17, bytes); + ok &= !msync(mapped, 3 * ps, MS_SYNC) && + pread(file, back, bytes + 34, 0) == + (ssize_t)(bytes + 34) && + check(back + 17, bytes, 0); + for (unsigned int i = 0; i < 17; i++) + ok &= back[i] == 0x5c && + back[bytes + 17 + i] == 0x5c; + snprintf( + name, sizeof(name), + "TLS 1.%u decrypts into shared file with neighboring bytes intact", + v + 2); + report(ok, ps, name); + munmap(mapped, 3 * ps); + close(file); + } + { + unsigned char *mapped = + mmap(NULL, 6 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + struct iovec vectors[4]; + struct msghdr message = { .msg_iov = vectors, + .msg_iovlen = 4 }; + + if (mapped == MAP_FAILED) + ksft_exit_fail_msg( + "TLS receive iovec mapping\n"); + memset(mapped, 0x81, 6 * ps); + vectors[0] = (struct iovec){ mapped + 13, 5003 }; + vectors[1] = (struct iovec){ NULL, 0 }; + vectors[2] = + (struct iovec){ mapped + 2 * ps + 7, 2222 }; + vectors[3] = + (struct iovec){ mapped + 4 * ps + 5, 1010 }; + fill(p, len); + ok = send(sockets[0], p, 8235, 0) == 8235 && + recvmsg(sockets[1], &message, MSG_WAITALL) == + 8235 && + check(mapped + 13, 5003, 0) && + check(mapped + 2 * ps + 7, 2222, 5003) && + check(mapped + 4 * ps + 5, 1010, 7225); + ok &= mapped[12] == 0x81 && mapped[5016] == 0x81 && + mapped[2 * ps + 6] == 0x81 && + mapped[2 * ps + 2229] == 0x81 && + mapped[4 * ps + 4] == 0x81 && + mapped[4 * ps + 1015] == 0x81; + snprintf( + name, sizeof(name), + "TLS 1.%u scattered receive, empty iovec and subpage boundaries", + v + 2); + report(ok, ps, name); + munmap(mapped, 6 * ps); + } + close(sockets[0]); + close(sockets[1]); + ok = true; + for (unsigned int round = 0; round < 64 && ok; round++) { + if (!tls_pair(sockets, version)) + ksft_exit_fail_msg( + "TLS teardown setup errno=%d\n", errno); + fill(p, len); + vec = (struct iovec){ p + ps, 4096 }; + ok = vmsplice(fds[1], &vec, 1, 0) == 4096 && + splice(fds[0], NULL, sockets[0], NULL, 4096, + SPLICE_F_MORE) == 4096; + close(sockets[1]); + close(sockets[0]); + ok &= !madvise(p + ps, ps, MADV_DONTNEED); + memset(p + ps, 0x75, ps); + } + snprintf(name, sizeof(name), + "TLS 1.%u repeated pending-record teardown and reuse", + v + 2); + report(ok, ps, name); + } + close(fds[0]); + close(fds[1]); +} + +static int exercise(void) +{ + const size_t ps = getauxval(AT_PAGESZ), len = 8 * ps; + unsigned char *p, *back = malloc(len); + int fds[2], dup[2], file, sockets[2]; + struct iovec v[3]; + struct stat self, init; + struct timeval timeout = { .tv_sec = 5 }; + bool ok; + ssize_t result; + + if (stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino) + return 2; + p = mmap((void *)0x300000000UL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (p == MAP_FAILED || !back || pipe(fds) || pipe(dup)) + ksft_exit_fail_msg("setup\n"); + ksft_print_header(); + ksft_set_plan(use_tcp ? 22 : 10); + ksft_print_msg("transport=%s family=%s\n", + use_tcp ? "TCP" : + use_udp ? "UDP" : + "Unix stream", + use_ipv6 ? "IPv6" : "IPv4/Unix"); + fill(p, len); + v[0] = (struct iovec){ NULL, 0 }; + v[1] = (struct iovec){ p + 17, ps - 17 }; + v[2] = (struct iovec){ p + 2 * ps + 31, 2 * ps - 31 }; + result = vmsplice(fds[1], v, 3, 0); + if (result != (ssize_t)(3 * ps - 48)) + ksft_exit_fail_msg("vmsplice returned %zd errno=%d\n", result, + errno); + ok = read_exact(fds[0], back, result) && check(back, ps - 17, 17) && + check(back + ps - 17, 2 * ps - 31, 2 * ps + 31); + report(ok, ps, "unaligned fragmented iovecs and empty segment"); + v[0] = (struct iovec){ p, 16384 }; + ok = fcntl(fds[1], F_SETPIPE_SZ, 16384) == 16384; + result = vmsplice(fds[1], v, 1, SPLICE_F_NONBLOCK); + { + int pending = 0; + + ok &= result == 16384 && !ioctl(fds[0], FIONREAD, &pending) && + pending == 16384; + } + if (result > 0) + ok &= read_exact(fds[0], back, result) && + check(back, result, 0); + report(ok, ps, + "contiguous quarters retain one-native-page pipe capacity"); + fcntl(fds[1], F_SETPIPE_SZ, 16 * 16384); + v[0] = (struct iovec){ p + 11, 3 * ps - 11 }; + result = vmsplice(fds[1], v, 1, 0); + if (result != (ssize_t)(3 * ps - 11) || + tee(fds[0], dup[1], result, 0) != result) + ksft_exit_fail_msg("tee setup\n"); + ok = replace(p, len); + ok &= read_exact(fds[0], back, 19) && check(back, 19, 11); + ok &= read_exact(fds[0], back + 19, result - 19) && + check(back, result, 11); + ok &= read_exact(dup[0], back, result) && check(back, result, 11); + report(ok, ps, + "tee and partial reads retain data across unmap/replacement"); + fill(p, len); + v[0] = (struct iovec){ p, ps }; + ok = vmsplice(fds[1], v, 1, 0) == (ssize_t)ps && cow(p, ps); + ok &= read_exact(fds[0], back, ps) && check(back, ps, 0); + report(ok, ps, "pipe GET retains pre-COW data after fork sharer exit"); + fill(p, len); + file = memfd_create("splice-target", 0); + v[0] = (struct iovec){ p + ps + 29, 2 * ps - 29 }; + result = vmsplice(fds[1], v, 1, SPLICE_F_GIFT); + ok = file >= 0 && result == (ssize_t)(2 * ps - 29) && + splice(fds[0], NULL, file, NULL, result, SPLICE_F_MOVE) == result; + ok &= pread(file, back, result, 0) == result && + check(back, result, ps + 29); + report(ok, ps, "gifted fragments splice to buffered file"); + close(file); + v[0] = (struct iovec){ p, ps }; + v[1] = (struct iovec){ (void *)0x1000, ps }; + result = vmsplice(fds[1], v, 2, 0); + ok = result == (ssize_t)ps && read_exact(fds[0], back, ps) && + check(back, ps, 0); + report(ok, ps, "partial vmsplice before inaccessible iovec"); + if (connected_pair(sockets)) + ksft_exit_fail_msg("socketpair errno=%d\n", errno); + setsockopt(sockets[1], SOL_SOCKET, SO_RCVTIMEO, &timeout, + sizeof(timeout)); + for (unsigned int mode = 0; mode < 3; mode++) { + const size_t bytes = mode == 1 ? ps : + (use_udp && ps == 4096 ? + 6 * ps - 256 : + 2 * ps - 256); + const size_t offset = mode == 1 ? 0 : ps + 29; + int cork = 1; + + if ((use_tcp || use_udp) && mode < 2 && + setsockopt(sockets[0], use_tcp ? IPPROTO_TCP : IPPROTO_UDP, + use_tcp ? TCP_CORK : UDP_CORK, &cork, + sizeof(cork))) + ksft_exit_fail_msg("TCP_CORK setup errno=%d\n", errno); + fill(p, len); + v[0] = (struct iovec){ p + offset, bytes }; + ok = vmsplice(fds[1], v, 1, 0) == (ssize_t)bytes && + splice(fds[0], NULL, sockets[0], NULL, bytes, + SPLICE_F_NONBLOCK) == (ssize_t)bytes; + if (!ok) + ksft_exit_fail_msg("socket splice setup errno=%d\n", + errno); + if (mode == 2) { + ok = splice(sockets[1], NULL, fds[1], NULL, bytes, 0) == + (ssize_t)bytes && + tee(fds[0], dup[1], bytes, 0) == (ssize_t)bytes; + if (!ok) + ksft_exit_fail_msg( + "socket-to-pipe setup errno=%d\n", + errno); + } + if (use_tcp && mode < 2) { + int unsent = 0; + + if (ioctl(sockets[0], SIOCOUTQNSD, &unsent) || + unsent != (int)bytes) + ksft_exit_fail_msg( + "TCP unsent precondition: %d expected %zu errno=%d\n", + unsent, bytes, errno); + } + if (use_udp && mode < 2) { + errno = 0; + if (recv(sockets[1], back, len, MSG_DONTWAIT) != -1 || + errno != EAGAIN) + ksft_exit_fail_msg( + "UDP corked precondition errno=%d\n", + errno); + } + ok &= mode == 1 ? cow(p, ps) : replace(p + ps, 2 * ps); + if ((use_tcp || use_udp) && mode < 2) { + cork = 0; + if (setsockopt(sockets[0], + use_tcp ? IPPROTO_TCP : IPPROTO_UDP, + use_tcp ? TCP_CORK : UDP_CORK, &cork, + sizeof(cork))) { + ksft_print_msg("uncork errno=%d\n", errno); + ok = false; + } + } + ok &= read_exact(mode == 2 ? fds[0] : sockets[1], back, + bytes) && + check(back, bytes, offset); + if (mode == 2) + ok &= read_exact(dup[0], back, bytes) && + check(back, bytes, offset); + report(ok, ps, + mode == 0 ? + "socket retains quarters across partial unmap/replacement" : + mode == 1 ? + "socket GET retains pre-COW data after fork sharer exit" : + "socket-to-pipe and tee retain independent quarter ownership"); + } + close(sockets[0]); + close(sockets[1]); + ok = true; + for (unsigned int round = 0; round < 128 && ok; round++) { + fill(p, len); + ok = !connected_pair(sockets); + v[0] = (struct iovec){ p + ps, ps }; + ok &= vmsplice(fds[1], v, 1, 0) == (ssize_t)ps && + splice(fds[0], NULL, sockets[0], NULL, ps, 0) == + (ssize_t)ps; + close(sockets[1]); + close(sockets[0]); + ok &= !madvise(p + ps, ps, MADV_DONTNEED); + memset(p + ps, 0x43, ps); + } + report(ok, ps, "repeated unread socket teardown and quarter reuse"); + if (use_tcp) + tls_checks(ps, p, len, back); + close(fds[0]); + close(fds[1]); + close(dup[0]); + close(dup[1]); + munmap(p, len); + free(back); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +static bool async_cleanup_balanced(void) +{ + const char *names[] = { "user4k_rx_deferred", "user4k_rx_released" }; + unsigned long counts[2]; + char path[128]; + FILE *file = fopen("/sys/module/tls/parameters/user4k_test_async", "r"); + int enabled; + + if (!file) + return false; + enabled = fgetc(file); + fclose(file); + for (unsigned int i = 0; i < 2; i++) { + snprintf(path, sizeof(path), "/sys/module/tls/parameters/%s", + names[i]); + file = fopen(path, "r"); + if (!file) + return false; + if (fscanf(file, "%lu", &counts[i]) != 1) { + fclose(file); + return false; + } + fclose(file); + } + printf("# TLS async=%c deferred=%lu released=%lu\n", enabled, counts[0], + counts[1]); + return counts[0] == counts[1] && (enabled != 'Y' || counts[0] > 0); +} + +int main(int argc, char **argv) +{ + bool ok = true; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && + (!strcmp(argv[1], "--test") || !strcmp(argv[1], "--tcp") || + !strcmp(argv[1], "--tcp6") || !strcmp(argv[1], "--udp") || + !strcmp(argv[1], "--udp6")) && + getppid() == 1) { + use_tcp = !strncmp(argv[1], "--tcp", 5); + use_udp = !strncmp(argv[1], "--udp", 5); + use_ipv6 = strchr(argv[1], '6') != NULL; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + { + struct ifreq request = { .ifr_name = "lo" }; + int fd = socket(AF_INET, SOCK_DGRAM, 0); + + if (fd < 0 || ioctl(fd, SIOCGIFFLAGS, &request)) + return 4; + request.ifr_flags |= IFF_UP; + if (ioctl(fd, SIOCSIFFLAGS, &request)) + return 4; + close(fd); + } + for (unsigned int mode = 0; mode < 10; mode++) { + unsigned int small = mode & 1; + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + small ? 4096UL : 0UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", + mode >= 8 ? "--udp6" : + mode >= 6 ? "--udp" : + mode >= 4 ? "--tcp6" : + mode >= 2 ? "--tcp" : + "--test", + NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + ok &= async_cleanup_balanced(); + printf("%s - splice fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/swap-fixture-init.c b/tools/testing/selftests/arm64/user4k/swap-fixture-init.c new file mode 100644 index 0000000000000..45f6aefdd094e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/swap-fixture-init.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PID-1 runner for a disposable VM with a dedicated virtio swap disk. + * Use kunit.autorun=0: tests skipped at boot stay skipped on a debugfs rerun. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "re"); + char key[128]; + unsigned long value, found = 0; + + if (!f) + return 0; + while (fscanf(f, "%127s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) + found = value; + fclose(f); + return found; +} + +static int native_swap_smoke(void) +{ + const size_t length = 4 * 1024 * 1024, page = 16384; + unsigned long before = swapouts(), after; + unsigned char *p = mmap(NULL, length, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + size_t i; + unsigned int tries; + int ret = 0; + + if (p == MAP_FAILED) + return -1; + for (i = 0; i < length; i++) + p[i] = (i ^ (i >> 12) ^ (i >> 20)) & 255; + if (madvise(p, length, MADV_PAGEOUT)) { + ret = -1; + goto out; + } + for (tries = 0; tries < 100 && swapouts() == before; tries++) + usleep(20000); + after = swapouts(); + for (i = 0; i < length; i++) + if (p[i] != ((i ^ (i >> 12) ^ (i >> 20)) & 255)) { + ret = -1; + break; + } + if (after <= before) + ret = -1; + printf("%s - native %zuK swap transport: pswpout %lu -> %lu, verified %zu bytes\n", + ret ? "not ok" : "ok", page / 1024, before, after, i); +out: + munmap(p, length); + return ret; +} + +static int run_swap_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-swap/"; + char path[128], *log; + size_t used = 0, capacity = 1024 * 1024; + ssize_t n; + int fd, ret = -1; + + snprintf(path, sizeof(path), "%srun", base); + fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = write(fd, "1", 1); + close(fd); + if (n != 1) + return -1; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + log = calloc(1, capacity); + if (!log) { + close(fd); + return -1; + } + while (used < capacity - 1 && + (n = read(fd, log + used, capacity - 1 - used)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (n == 0 && used < capacity - 1 && + strstr(log, "\nok 1 arm64-user4k-swap\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; + free(log); + return ret; +} + +static int swap_control_helper(const char *operation) +{ + struct stat init, self; + int ret; + + /* A helper is allowed only when this same executable is the VM's PID 1. */ + if (getpid() == 1 || stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || init.st_dev != self.st_dev || + init.st_ino != self.st_ino) { + fprintf(stderr, + "swap helper requires its fixture runner as PID 1\n"); + return 1; + } + ret = !strcmp(operation, "--swapoff-helper") ? swapoff("/dev/vda") : + swapon("/dev/vda", 0); + if (ret) + perror(operation); + return ret ? 1 : 0; +} + +static int pageout_exec(bool small) +{ + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (small && + prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(125); + execl("/reclaim-advice", "/reclaim-advice", + small ? "--expect-4k" : NULL, NULL); + _exit(126); + } + if (pid <= 0 || waitpid(pid, &status, 0) != pid) + return -1; + printf("# EL0 pageout %s child status=%#x\n", small ? "4K" : "native", + status); + return WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +int main(int argc, char **argv) +{ + bool smoke = argc == 2 && !strcmp(argv[1], "--smoke"); + bool pageout = argc == 2 && !strcmp(argv[1], "--pageout"); + bool native_only = argc == 2 && !strcmp(argv[1], "--pageout-native"); + bool active = false; + unsigned int tries; + struct stat st; + pid_t child; + int status, ret = 1; + + if (argc == 2 && (!strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--swapon-helper"))) + return swap_control_helper(argv[1]); + /* This runner must never mount filesystems or enable swap on the host. */ + if (getpid() != 1 || + (argc != 1 && !smoke && !pageout && !native_only)) { + fprintf(stderr, + "Use only as /init in the dedicated disposable test VM.\n"); + return 1; + } + setvbuf(stdout, NULL, _IONBF, 0); + if (getauxval(AT_PAGESZ) != 16384 || sysconf(_SC_PAGESIZE) != 16384) + goto out; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) + goto out; + child = fork(); + if (!child) { + execl("/page-contract", "page-contract", NULL); + _exit(127); + } + if (child < 0 || waitpid(child, &status, 0) != child || + !WIFEXITED(status) || WEXITSTATUS(status)) + goto out; + for (tries = 0; tries < 100 && stat("/dev/vda", &st); tries++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || + swapon("/dev/vda", 0)) + goto out; + active = true; + if (native_swap_smoke()) + goto out; + if (!smoke && !native_only && run_swap_suite()) + goto out; + if ((pageout || native_only) && pageout_exec(false)) + goto out; + if (pageout && pageout_exec(true)) + goto out; + ret = 0; +out: + if (ret) + printf("not ok - swap fixture runner: errno=%d (%s)\n", errno, + strerror(errno)); + if (active && swapoff("/dev/vda")) { + perror("swapoff"); + ret = 1; + } + printf("%s - swap fixture runner (%s)\n", ret ? "not ok" : "ok", + smoke ? "native transport smoke only" : "4K swap suite"); + fflush(NULL); + reboot(RB_POWER_OFF); + return ret; +} diff --git a/tools/testing/selftests/arm64/user4k/swapfile-fixture.h b/tools/testing/selftests/arm64/user4k/swapfile-fixture.h new file mode 100644 index 0000000000000..258b81fa05695 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/swapfile-fixture.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef USER4K_SWAPFILE_FIXTURE_H +#define USER4K_SWAPFILE_FIXTURE_H + +static const char *swap_path(void) +{ + return access("/swapdir/swapfile", F_OK) ? "/dev/vda" : + "/swapdir/swapfile"; +} + +/* Remove every fourth native page: every 64K group crosses file extents. */ +static int prepare_swapfile(void) +{ + const unsigned long page = getauxval(AT_PAGESZ), initial = 32UL << 20; + unsigned char *buf = calloc(1, page); + unsigned long offset; + int fd = -1, ret = -1; + + if (!buf || (mkdir("/swapdir", 0755) && errno != EEXIST) || + mount("/dev/vda", "/swapdir", "ext4", 0, NULL)) + goto out; + fd = open("/swapdir/swapfile", O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, + 0600); + if (fd < 0) + goto out; + memset(buf, 0xa5, page); + for (offset = 0; offset < initial; offset += page) + if (write(fd, buf, page) != (ssize_t)page) + goto out; + if (fsync(fd)) + goto out; + for (offset = initial; offset; offset -= 4 * page) + if (fallocate(fd, FALLOC_FL_COLLAPSE_RANGE, offset - 2 * page, + page)) + goto out; + memset(buf, 0, page); + ((unsigned int *)(buf + 1024))[0] = 1; + ((unsigned int *)(buf + 1024))[1] = (initial * 3 / 4) / page - 1; + memcpy(buf + page - 10, "SWAPSPACE2", 10); + if (pwrite(fd, buf, page, 0) != (ssize_t)page || fsync(fd)) + goto out; + ret = 0; +out: + if (fd >= 0) + close(fd); + free(buf); + return ret; +} + +#endif diff --git a/tools/testing/selftests/arm64/user4k/sysv-shm-init.c b/tools/testing/selftests/arm64/user4k/sysv-shm-init.c new file mode 100644 index 0000000000000..3b66d9c738fde --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/sysv-shm-init.c @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: SysV shared memory with per-process page granules. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int ok, const char *expression, int line) +{ + if (!ok) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, + errno); + exit(1); + } +} + +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int unmapped(void *address, size_t page) +{ + unsigned char resident; + + errno = 0; + return mincore(address, page, &resident) == -1 && errno == ENOMEM; +} + +static int mapping_case(unsigned long native, int rounded, int split) +{ + unsigned long page = getauxval(AT_PAGESZ), length = 4 * page; + unsigned char *reserve, *target, *mapping, *alias; + struct shmid_ds info; + int id; + + reserve = mmap(NULL, 16 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + target = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + target += native + (page < native ? page : 0); + target[-1] = 0x51; + target[length] = 0x72; + CHECK(!munmap(target, length)); + id = shmget(IPC_PRIVATE, length - 1, 0600); + CHECK(id >= 0); + mapping = shmat(id, target + (rounded ? page - 1 : 0), + rounded ? SHM_RND : 0); + if (mapping != target) { + int saved = errno; + + shmctl(id, IPC_RMID, NULL); + errno = saved; + CHECK(mapping == target); + } + alias = shmat(id, NULL, 0); + CHECK(alias != (void *)-1); + CHECK(!shmctl(id, IPC_RMID, NULL)); + mapping[0] = 0x31; + mapping[page + 7] = 0x49; + mapping[3 * page] = 0x63; + CHECK(alias[0] == 0x31 && alias[page + 7] == 0x49 && + alias[3 * page] == 0x63); + if (split) { + CHECK(!mprotect(mapping + page, page, PROT_READ)); + CHECK(!munmap(mapping + 2 * page, page)); + if (split == 2) + CHECK(!munmap(mapping, page)); + } + CHECK(!shmdt(mapping)); + for (unsigned long i = 0; i < length; i += page) + CHECK(unmapped(target + i, page)); + CHECK(target[-1] == 0x51 && target[length] == 0x72); + CHECK(alias[page + 7] == 0x49 && alias[3 * page] == 0x63); + CHECK(!shmdt(alias)); + errno = 0; + CHECK(shmctl(id, IPC_STAT, &info) == -1 && + (errno == EINVAL || errno == EIDRM)); + CHECK(!munmap(reserve, 16 * native)); + printf("ok - SysV page=%lu rounded=%d split=%d detach and alias lifetime\n", + page, rounded, split); + return 0; +} + +static int wait_child(pid_t pid) +{ + int status; + pid_t got; + + if (pid < 0) + return 1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + return got != pid || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static int sharing_child(int id, unsigned long expected) +{ + unsigned char *map; + + CHECK(getauxval(AT_PAGESZ) == expected); + map = shmat(id, NULL, 0); + CHECK(map != (void *)-1); + CHECK(map[37] == 0x35 && map[4096 + 9] == 0x56); + map[4096 + 9] = 0x78; + CHECK(!shmdt(map)); + return 0; +} + +static int sharing(unsigned long other, unsigned long native) +{ + unsigned long page = getauxval(AT_PAGESZ); + unsigned char *map; + pid_t pid; + int id; + + id = shmget(IPC_PRIVATE, 2 * native, 0600); + CHECK(id >= 0); + map = shmat(id, NULL, 0); + CHECK(map != (void *)-1); + CHECK(!shmctl(id, IPC_RMID, NULL)); + map[37] = 0x35; + map[4096 + 9] = 0x56; + pid = fork(); + if (!pid) { + char idarg[32], sizearg[32]; + + snprintf(idarg, sizeof(idarg), "%d", id); + snprintf(sizearg, sizeof(sizearg), "%lu", other); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, other, 0UL, 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--share", idarg, sizearg, NULL); + _exit(3); + } + CHECK(!wait_child(pid)); + CHECK(map[4096 + 9] == 0x78); + CHECK(!shmdt(map)); + printf("ok - SysV shared object %lu -> %lu after IPC_RMID\n", page, + other); + return 0; +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--share")) + return sharing_child(atoi(argv[2]), strtoul(argv[3], NULL, 10)); + if (argc == 5 && !strcmp(argv[1], "--case")) { + unsigned long expected = strtoul(argv[3], NULL, 10); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == expected); + failed |= mapping_case(native, 0, 0); + failed |= mapping_case(native, 1, 0); + failed |= mapping_case(native, 0, 1); + failed |= mapping_case(native, 0, 2); + for (unsigned int i = 0; i < 3; i++) + if (sizes[i] <= native && sizes[i] != expected && + !(atoi(argv[4]) && sizes[i] == 4096)) + failed |= sharing(sizes[i], native); + return failed; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + int no4k = argc == 2 && !strcmp(argv[1], "--no4k"); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + unsigned long page = sizes[i]; + pid_t pid; + + if (page > native || (no4k && page == 4096)) + continue; + pid = fork(); + if (!pid) { + char n[32], e[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(e, sizeof(e), "%lu", page); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, page, 0UL, 0UL, + 0UL)) + _exit(2); + execl("/init", "/init", "--case", n, e, + no4k ? "1" : "0", NULL); + _exit(3); + } + int bad = wait_child(pid); + + printf("%s - SysV AArch64 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, page); + failed |= bad; + } + if (access("/compat-shm", X_OK) == 0) { + pid_t pid = fork(); + + if (!pid) { + char n[32], e[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(e, sizeof(e), "%lu", no4k ? native : 4096UL); + execl("/compat-shm", "/compat-shm", n, e, NULL); + _exit(3); + } + int bad = wait_child(pid); + + printf("%s - SysV AArch32 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, no4k ? native : 4096UL); + failed |= bad; + } + printf("SYSV SHM GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/thread-fixture-init.c b/tools/testing/selftests/arm64/user4k/thread-fixture-init.c new file mode 100644 index 0000000000000..c472ea6697947 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/thread-fixture-init.c @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define THREADS 4 +#define ROUNDS 128 +#define PAGES 32 + +static pthread_barrier_t barrier; +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t condition = PTHREAD_COND_INITIALIZER; +static pthread_mutex_t robust; +static _Thread_local unsigned int tls_id; +static _Thread_local volatile sig_atomic_t signals; +static atomic_uint failures; +static unsigned long total; +static unsigned long ps; +static atomic_int waiter_tid; + +static void handler(int sig) +{ + if (sig == SIGUSR1 && tls_id) + signals++; + else + atomic_fetch_add(&failures, 1); +} + +static bool bytes_equal(unsigned char *p, size_t n, unsigned char value) +{ + for (size_t i = 0; i < n; i++) + if (p[i] != value) + return false; + return true; +} + +static void *worker(void *arg) +{ + unsigned int id = (uintptr_t)arg; + unsigned char *p, *heap; + unsigned long request = id & 1 ? 4096 : 0; + bool ok = true; + + tls_id = id; + p = mmap(NULL, PAGES * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (p == MAP_FAILED) + ok = false; + ok &= !prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, request, 0UL, 0UL, 0UL); + pthread_barrier_wait(&barrier); + for (unsigned int round = 0; round < ROUNDS; round++) { + unsigned char pattern = 1 + (id * 31 + round) % 254; + unsigned char residency[PAGES]; + cpu_set_t cpu; + + if (ok) { + CPU_ZERO(&cpu); + CPU_SET((id + round) % 2, &cpu); + ok &= !sched_setaffinity(0, sizeof(cpu), &cpu); + memset(p, pattern, PAGES * ps); + ok &= !mprotect(p + ps, ps, PROT_READ) && + p[ps] == pattern; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE); + ok &= !madvise(p + 2 * ps, ps, MADV_FREE); + memset(p + 2 * ps, pattern, ps); + ok &= !madvise(p + 3 * ps, ps, MADV_DONTNEED); + ok &= bytes_equal(p + 3 * ps, ps, 0); + memset(p + 3 * ps, pattern, ps); + heap = malloc(5000 + id * 777); + if (!heap) { + ok = false; + } else { + memset(heap, pattern, 5000 + id * 777); + ok &= bytes_equal(heap, 5000 + id * 777, + pattern); + free(heap); + } + ok &= !pthread_kill(pthread_self(), SIGUSR1); + ok &= tls_id == id && + signals == (sig_atomic_t)(round + 1); + ok &= prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, + 0UL) == (int)request; + ok &= !mincore(p, PAGES * ps, residency); + ok &= bytes_equal(p, PAGES * ps, pattern); + } + pthread_mutex_lock(&mutex); + total++; + pthread_cond_signal(&condition); + pthread_mutex_unlock(&mutex); + pthread_barrier_wait(&barrier); + } + if (p != MAP_FAILED) + munmap(p, PAGES * ps); + if (!ok) + atomic_fetch_add(&failures, 1); + return NULL; +} + +static void *owner_dies(void *arg) +{ + (void)arg; + if (pthread_mutex_lock(&robust)) + atomic_fetch_add(&failures, 1); + return NULL; +} + +static void *pi_waiter(void *arg) +{ + int error; + + (void)arg; + atomic_store(&waiter_tid, syscall(SYS_gettid)); + error = pthread_mutex_lock(&robust); + if (!error) + error = pthread_mutex_unlock(&robust); + return (void *)(uintptr_t)error; +} + +static bool wait_for_pi_block(void) +{ + for (unsigned int tries = 0; tries < 1000; tries++) { + int tid = atomic_load(&waiter_tid); + char path[80], line[256]; + unsigned long nr, address, operation; + FILE *f; + + if (tid) { + snprintf(path, sizeof(path), + "/proc/self/task/%d/syscall", tid); + f = fopen(path, "re"); + if (f) { + bool blocked = fgets(line, sizeof(line), f) && + sscanf(line, "%lu %lx %lx", &nr, + &address, + &operation) == 3 && + nr == SYS_futex && + ((operation & FUTEX_CMD_MASK) == + FUTEX_LOCK_PI || + (operation & FUTEX_CMD_MASK) == + FUTEX_LOCK_PI2); + + fclose(f); + if (blocked) + return true; + } + } + usleep(1000); + } + return false; +} + +static int exercise(void) +{ + pthread_t threads[THREADS], owner; + pthread_mutexattr_t attributes; + struct sigaction action = { .sa_handler = handler }; + struct stat self, init; + bool ok = true, forks = true; + int error; + + if (stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino) + return 2; + ps = getauxval(AT_PAGESZ); + ksft_print_header(); + ksft_set_plan(6); + if (sigaction(SIGUSR1, &action, NULL) || + pthread_barrier_init(&barrier, NULL, THREADS + 1)) + ksft_exit_fail_msg("thread setup\n"); + for (unsigned int i = 0; i < THREADS; i++) + if (pthread_create(&threads[i], NULL, worker, + (void *)(uintptr_t)(i + 1))) + ksft_exit_fail_msg("pthread_create\n"); + pthread_barrier_wait(&barrier); + for (unsigned int round = 0; round < ROUNDS; round++) { + pthread_mutex_lock(&mutex); + while (total < (round + 1) * THREADS) + pthread_cond_wait(&condition, &mutex); + pthread_mutex_unlock(&mutex); + ok &= prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) == + 0; + if (!(round % 16)) { + pid_t child = fork(); + int status = 0; + + if (!child) { + unsigned char *p = malloc(32768); + + if (!p || tls_id || + total != (round + 1) * THREADS || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, + 0UL, 0UL, 0UL)) + _exit(10); + memset(p, 0x75, 32768); + if (!bytes_equal(p, 32768, 0x75)) + _exit(11); + free(p); + _exit(0); + } + forks &= child > 0 && + waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + } + pthread_barrier_wait(&barrier); + } + for (unsigned int i = 0; i < THREADS; i++) + ok &= !pthread_join(threads[i], NULL); + ksft_test_result( + ok && !atomic_load(&failures) && total == THREADS * ROUNDS, + "%luK %u concurrent memory/TLS/signal/exec-request rounds across %u threads\n", + ps / 1024, ROUNDS, THREADS); + ksft_test_result( + forks, "%luK fork and allocator from a multithreaded process\n", + ps / 1024); + pthread_barrier_destroy(&barrier); + ok = !pthread_mutexattr_init(&attributes); + ok &= !pthread_mutexattr_setrobust(&attributes, PTHREAD_MUTEX_ROBUST); + ok &= !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + ksft_exit_fail_msg("robust mutex setup\n"); + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + ok &= !pthread_mutex_unlock(&robust); + ksft_test_result(ok, "%luK robust futex owner-death recovery\n", + ps / 1024); + pthread_mutex_destroy(&robust); + /* Exercise repeated TLS/stack allocation and set_tid_address clear/wake. */ + for (unsigned int i = 0; i < 128; i++) { + ok = !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + break; + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + pthread_mutex_unlock(&robust); + pthread_mutex_destroy(&robust); + if (!ok) + break; + } + ksft_test_result(ok && !atomic_load(&failures), + "%luK repeated thread exit/join and robust recovery\n", + ps / 1024); + ok = !pthread_mutexattr_setprotocol(&attributes, PTHREAD_PRIO_INHERIT); + ok &= !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_mutex_lock(&robust); + ok &= !pthread_create(&owner, NULL, pi_waiter, NULL); + if (!ok) + ksft_exit_fail_msg("PI mutex setup\n"); + ok &= wait_for_pi_block(); + ok &= !pthread_mutex_unlock(&robust); + { + void *result = (void *)1; + + ok &= !pthread_join(owner, &result) && !result; + } + ksft_test_result( + ok, + "%luK confirmed contended priority-inheritance futex handoff\n", + ps / 1024); + ok = !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + ksft_exit_fail_msg("PI owner setup\n"); + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + ok &= !pthread_mutex_unlock(&robust); + pthread_mutex_destroy(&robust); + pthread_mutexattr_destroy(&attributes); + ksft_test_result( + ok, "%luK priority-inheritance robust owner-death recovery\n", + ps / 1024); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +static pthread_barrier_t exec_barrier; + +static void *exec_sibling(void *arg) +{ + (void)arg; + pthread_barrier_wait(&exec_barrier); + for (;;) + pause(); + return NULL; +} + +static void *exec_nonleader(void *arg) +{ + unsigned long target = (uintptr_t)arg; + char size[32]; + + snprintf(size, sizeof(size), "%lu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, 0UL, 0UL, 0UL)) + _exit(30); + pthread_barrier_wait(&exec_barrier); + execl("/init", "/init", "--after-thread-exec", size, NULL); + _exit(31); +} + +static int thread_exec(unsigned long target) +{ + pthread_t siblings[2], executor; + + if (pthread_barrier_init(&exec_barrier, NULL, 4)) + return 32; + for (unsigned int i = 0; i < 2; i++) + if (pthread_create(&siblings[i], NULL, exec_sibling, NULL)) + return 33; + if (pthread_create(&executor, NULL, exec_nonleader, + (void *)(uintptr_t)target)) + return 34; + pthread_barrier_wait(&exec_barrier); + for (;;) + pause(); + return 35; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long granules[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + unsigned int nr = granules[0] > 16384 ? 3 : 2; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--thread-exec") && getppid() == 1) + return thread_exec(strtoul(argv[2], NULL, 10)); + if (argc == 3 && !strcmp(argv[1], "--after-thread-exec") && + getppid() == 1) { + unsigned long expected = strtoul(argv[2], NULL, 10); + + return getauxval(AT_PAGESZ) != expected || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, + 0UL) != 0; + } + if (argc == 3 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + char size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", size, NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + for (unsigned int source = 0; source < nr; source++) { + for (unsigned int target = 0; target < nr; target++) { + pid_t pid = fork(); + int status = 0; + bool passed; + + if (!pid) { + char size[32]; + + snprintf(size, sizeof(size), "%lu", + granules[target]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + granules[source], 0UL, 0UL, 0UL)) + _exit(36); + execl("/init", "/init", "--thread-exec", size, + NULL); + _exit(37); + } + passed = pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - non-leader exec %luK -> %luK status=%#x\n", + passed ? "ok" : "not ok", + granules[source] / 1024, granules[target] / 1024, + status); + ok &= passed; + } + } + printf("%s - thread fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/uffd-move-init.c b/tools/testing/selftests/arm64/user4k/uffd-move-init.c new file mode 100644 index 0000000000000..f2649698488b6 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uffd-move-init.c @@ -0,0 +1,451 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - MOVE line=%d errno=%d (%s)\n", \ + __LINE__, errno, strerror(errno)); \ + exit(1); \ + } \ + } while (0) +#define PRESENT (1ULL << 63) +#define SWAPPED (1ULL << 62) +#define VALUE_MASK ((1ULL << 55) - 1) +static uint64_t entry(void *p, size_t ps) +{ + uint64_t e; + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + CHECK(fd >= 0 && pread(fd, &e, 8, (uintptr_t)p / ps * 8) == 8); + CHECK(!close(fd)); + return e; +} +static uint64_t physical(void *p, size_t ps) +{ + uint64_t e = entry(p, ps); + CHECK(e & PRESENT); + return (e & VALUE_MASK) * ps; +} +static unsigned char pattern(size_t i) +{ + return 1 + ((i ^ (i >> 12) ^ (i >> 16)) % 251); +} +static void bytes(unsigned char *p, size_t original, size_t len) +{ + for (size_t i = 0; i < len; i++) + CHECK(p[i] == pattern(original + i)); +} +static long move(int fd, void *dst, void *src, size_t len, unsigned mode, + int expected_errno) +{ + struct uffdio_move m = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = len, + .mode = mode }; + errno = 0; + int r = ioctl(fd, UFFDIO_MOVE, &m); + if (expected_errno) + CHECK(r == -1 && errno == expected_errno); + else + CHECK(r == 0 && m.move == (long long)len); + return m.move; +} +static void on_cpu(unsigned cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +struct tlb_reader { + unsigned char *src, *dst; + atomic_int ready, done; +}; +static void *read_after_move(void *arg) +{ + struct tlb_reader *r = arg; + on_cpu(1); + unsigned char old = *(volatile unsigned char *)r->src; + atomic_store(&r->ready, 1); + while (!atomic_load(&r->done)) + sched_yield(); + CHECK(*(volatile unsigned char *)r->src == 0 && + *(volatile unsigned char *)r->dst == old); + return NULL; +} +struct pin_racer { + void *a, *b; + size_t ps; + atomic_int stop; + atomic_uint accepted, rejected; +}; +static void *race_pins(void *arg) +{ + struct pin_racer *r = arg; + struct io_uring_params p = { 0 }; + on_cpu(1); + int fd = syscall(SYS_io_uring_setup, 2, &p); + CHECK(fd >= 0); + unsigned round = 0; + while (!atomic_load(&r->stop)) { + struct iovec iov = { .iov_base = (round++ & 1) ? r->a : r->b, + .iov_len = r->ps }; + int ret = syscall(SYS_io_uring_register, fd, + IORING_REGISTER_BUFFERS, &iov, 1); + if (!ret) { + atomic_fetch_add(&r->accepted, 1); + usleep(50); + CHECK(!syscall(SYS_io_uring_register, fd, + IORING_UNREGISTER_BUFFERS, NULL, 0)); + } else { + CHECK(errno == EFAULT || errno == EAGAIN); + atomic_fetch_add(&r->rejected, 1); + } + } + CHECK(!close(fd)); + return NULL; +} +static void concurrent_move(size_t ps) +{ + unsigned char *a = + mmap((void *)0x50000000, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + unsigned char *b = + mmap((void *)0x60000000, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(a != MAP_FAILED && b != MAP_FAILED); + memset(a, 0x6d, ps); + uint64_t phys = physical(a, ps); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MOVE }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)a, ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + reg.range.start = (uintptr_t)b; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + struct pin_racer race = { .a = a, .b = b, .ps = ps }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, race_pins, &race)); + while (!atomic_load(&race.accepted)) + sched_yield(); + unsigned char *cur = a, *next = b; + unsigned moved = 0, busy = 0; + for (unsigned tries = 0; moved < 256 && tries < 100000; tries++) { + struct uffdio_move m = { .src = (uintptr_t)cur, + .dst = (uintptr_t)next, + .len = ps }; + int ret = ioctl(fd, UFFDIO_MOVE, &m); + if (ret) { + CHECK(errno == EBUSY && m.move == -EBUSY); + busy++; + sched_yield(); + continue; + } + CHECK(m.move == (long long)ps && physical(next, ps) == phys && + !(entry(cur, ps) & (PRESENT | SWAPPED))); + unsigned char *tmp = cur; + cur = next; + next = tmp; + moved++; + } + atomic_store(&race.stop, 1); + CHECK(!pthread_join(thread, NULL)); + CHECK(moved == 256 && atomic_load(&race.accepted) > 0); + for (size_t i = 0; i < ps; i++) + CHECK(cur[i] == 0x6d); + printf("ok - MOVE %zuK CPU0/CPU1 race moves=%u busy=%u pins=%u holes=%u unchanged PFN/data\n", + ps / 1024, moved, busy, atomic_load(&race.accepted), + atomic_load(&race.rejected)); + CHECK(!close(fd) && !munmap(a, ps) && !munmap(b, ps)); +} + +static void allocation_failure(size_t ps, size_t native) +{ + if (ps == native) + return; + unsigned char *a = + mmap((void *)0x70000000, 2 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + unsigned char *b = + mmap((void *)0x71000000, 2 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(a != MAP_FAILED && b != MAP_FAILED); + memset(a, 0x35, 2 * native); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MOVE }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)a, 2 * native }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + reg.range.start = (uintptr_t)b; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + /* Prepare both roots and target tables using a different native folio. */ + move(fd, b + native, a + native, ps, 0, 0); + uint64_t phys = physical(a, ps); + int inject = open("/proc/self/fail-nth", O_RDWR | O_CLOEXEC); + CHECK(inject >= 0); + unsigned char *src = a, *dst = b; + for (unsigned phase = 0; phase < 2; phase++) { + struct uffdio_move m = { .src = (uintptr_t)src, + .dst = (uintptr_t)dst, + .len = ps }; + char pending[16] = { 0 }; + CHECK(write(inject, "1", 1) == 1); + errno = 0; + int ret = ioctl(fd, UFFDIO_MOVE, &m), saved = errno; + CHECK(pread(inject, pending, sizeof(pending) - 1, 0) > 0); + CHECK(write(inject, "0", 1) == 1); + CHECK(ret == -1 && saved == ENOMEM && m.move == -ENOMEM && + strtoul(pending, NULL, 10) == 0); + CHECK(physical(src, ps) == phys && + !(entry(dst, ps) & (PRESENT | SWAPPED))); + for (size_t i = 0; i < ps; i++) + CHECK(src[i] == 0x35); + move(fd, dst, src, ps, 0, 0); + CHECK(physical(dst, ps) == phys); + unsigned char *tmp = src; + src = dst; + dst = tmp; + } + printf("ok - MOVE %zuK sidecar/retirement allocation failures preserve PTEs and retry\n", + ps / 1024); + CHECK(!close(inject) && !close(fd) && !munmap(a, 2 * native) && + !munmap(b, 2 * native)); +} + +static int exercise(size_t native) +{ + size_t ps = getauxval(AT_PAGESZ), len = 16 * native; + on_cpu(0); + unsigned char *src = + mmap((void *)0x20000000, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + unsigned char *dst = + mmap((void *)0x30000000, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (size_t i = 0; i < len; i++) + src[i] = pattern(i); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MOVE }; + CHECK(!ioctl(fd, UFFDIO_API, &api) && + (api.features & UFFD_FEATURE_MOVE)); + struct uffdio_register reg = { .range = { (uintptr_t)dst, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®) && + (reg.ioctls & (1ULL << _UFFDIO_MOVE))); + uint64_t pfns[256]; + CHECK(native / ps <= sizeof(pfns) / sizeof(pfns[0])); + for (size_t i = 0; i < native / ps; i++) + pfns[i] = physical(src + i * ps, ps); + uint64_t p = physical(src + 3 * ps, ps); + struct tlb_reader reader = { .src = src + 3 * ps, .dst = dst + ps }; + pthread_t tlb_thread; + CHECK(!pthread_create(&tlb_thread, NULL, read_after_move, &reader)); + while (!atomic_load(&reader.ready)) + sched_yield(); + move(fd, dst + ps, src + 3 * ps, ps, 0, 0); + CHECK(physical(dst + ps, ps) == p && + !(entry(src + 3 * ps, ps) & (PRESENT | SWAPPED))); + atomic_store(&reader.done, 1); + CHECK(!pthread_join(tlb_thread, NULL)); + bytes(dst + ps, 3 * ps, ps); + for (size_t i = 0; i < native / ps; i++) + if (i != 3) { + CHECK(physical(src + i * ps, ps) == pfns[i]); + bytes(src + i * ps, i * ps, ps); + } + CHECK(src[3 * ps] == + 0); /* Source has an independent zero-fault after MOVE. */ + printf("ok - MOVE %zuK exact PFN, divergent offsets, neighbour PFNs/data and source TLB\n", + ps / 1024); + + struct io_uring_params ur = { 0 }; + int ring = syscall(SYS_io_uring_setup, 2, &ur); + CHECK(ring >= 0); + struct iovec pin = { .iov_base = src + 2 * ps, .iov_len = ps }; + CHECK(!syscall(SYS_io_uring_register, ring, IORING_REGISTER_BUFFERS, + &pin, 1)); + p = physical(src + 2 * ps, ps); + CHECK(move(fd, dst + 2 * ps, src + 2 * ps, ps, 0, EBUSY) == -EBUSY); + CHECK(physical(src + 2 * ps, ps) == p && + !(entry(dst + 2 * ps, ps) & (PRESENT | SWAPPED))); + bytes(src + 2 * ps, 2 * ps, ps); + uint64_t neighbour = physical(src + ps, ps); + move(fd, dst + 3 * ps, src + ps, ps, 0, 0); + CHECK(physical(dst + 3 * ps, ps) == neighbour); + bytes(dst + 3 * ps, ps, ps); + CHECK(!syscall(SYS_io_uring_register, ring, IORING_UNREGISTER_BUFFERS, + NULL, 0)); + CHECK(!close(ring)); + move(fd, dst + 2 * ps, src + 2 * ps, ps, 0, 0); + CHECK(physical(dst + 2 * ps, ps) == p); + bytes(dst + 2 * ps, 2 * ps, ps); + printf("ok - MOVE %zuK source PIN rejected atomically, neighbour PIN allowed, unpin retry\n", + ps / 1024); + + CHECK(move(fd, dst + ps, src + 4 * ps, ps, 0, EEXIST) == -EEXIST); + bytes(src + 4 * ps, 4 * ps, ps); + bytes(dst + ps, 3 * ps, ps); + CHECK(!madvise(src + 5 * native, native, MADV_DONTNEED)); + CHECK(*(volatile unsigned char *)(src + 5 * native) == 0); + move(fd, dst + 5 * native, src + 5 * native, ps, 0, 0); + CHECK(dst[5 * native] == 0); + dst[5 * native] = 0x7d; + CHECK(src[5 * native] == 0); + CHECK(!madvise(src + 6 * native, native, MADV_DONTNEED)); + CHECK(move(fd, dst + 6 * native, src + 6 * native, ps, 0, ENOENT) == + -ENOENT); + move(fd, dst + 6 * native, src + 6 * native, ps, + UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES, 0); + CHECK(!(entry(dst + 6 * native, ps) & (PRESENT | SWAPPED))); + CHECK(!madvise(src + 10 * native + ps, ps, MADV_DONTNEED)); + p = physical(src + 10 * native, ps); + CHECK(move(fd, dst + 12 * native, src + 10 * native, 2 * ps, 0, + EAGAIN) == (long)ps); + CHECK(physical(dst + 12 * native, ps) == p); + bytes(dst + 12 * native, 10 * native, ps); + CHECK(!(entry(dst + 12 * native + ps, ps) & (PRESENT | SWAPPED))); + printf("ok - MOVE %zuK collision, zero, holes and partial progress\n", + ps / 1024); + + int pipefd[2]; + CHECK(!pipe(pipefd)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + close(pipefd[1]); + char c; + _exit(read(pipefd[0], &c, 1) == 1 ? 0 : 1); + } + close(pipefd[0]); + CHECK(move(fd, dst + 11 * native, src + 12 * native, ps, 0, EBUSY) == + -EBUSY); + CHECK(write(pipefd[1], "x", 1) == 1); + close(pipefd[1]); + int status; + CHECK(waitpid(child, &status, 0) == child && status == 0); + *(volatile unsigned char *)(src + 12 * native) = pattern(12 * native); + p = physical(src + 12 * native, ps); + move(fd, dst + 11 * native, src + 12 * native, ps, 0, 0); + CHECK(physical(dst + 11 * native, ps) == p); + bytes(dst + 11 * native, 12 * native, ps); + printf("ok - MOVE %zuK fork-shared source rejected, exclusive write/retry\n", + ps / 1024); + + /* Keep fault-in coverage as well as restoring an untouched PTE at swapoff. */ + for (unsigned int unuse = 0; unuse < 2; unuse++) { + size_t off = ps < native ? ps : 0; + size_t base = (unuse ? 13 : 8) * native; + unsigned char *s = src + base + off; + unsigned char *d = dst + base + 2 * ps; + + *(volatile unsigned char *)s = pattern(base + off); + for (unsigned int tries = 0; + !(entry(s, ps) & SWAPPED) && tries < 100; tries++) { + CHECK(!madvise(src + base, native, MADV_PAGEOUT)); + usleep(20000); + } + uint64_t old = entry(s, ps); + + CHECK(old & SWAPPED); + move(fd, d, s, ps, 0, 0); + uint64_t now = entry(d, ps); + + CHECK((now & SWAPPED) && ((now & VALUE_MASK) == (old & VALUE_MASK)) && + !(entry(s, ps) & (PRESENT | SWAPPED))); + if (unuse) { + /* Do not fault the moved slot in before swapoff restores it. */ + CHECK(!swapoff("/dev/vda")); + CHECK(entry(d, ps) & PRESENT); + } + bytes(d, base + off, ps); + if (unuse) + CHECK(!swapon("/dev/vda", 0)); + printf("ok - MOVE %zuK %s restores transferred swap PTE at new offset\n", + ps / 1024, unuse ? "swapoff" : "fault"); + } + CHECK(!close(fd)); + CHECK(!munmap(src, len)); + CHECK(!munmap(dst, len)); + concurrent_move(ps); + allocation_failure(ps, native); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)); + struct stat st; + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + CHECK(!stat("/dev/vda", &st) && S_ISBLK(st.st_mode) && + !swapon("/dev/vda", 0)); + size_t native = getauxval(AT_PAGESZ), sizes[] = { native, 4096, 16384 }; + int failed = 0; + for (unsigned i = 0; i < (native > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + CHECK(c >= 0); + if (!c) { + char n[32]; + snprintf(n, sizeof(n), "%zu", native); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", n, NULL); + _exit(5); + } + CHECK(waitpid(c, &status, 0) == c); + printf("%s - MOVE ABI %zuK status=%#x\n", + status ? "not ok" : "ok", sizes[i] / 1024, status); + failed += !!status; + } + if (swapoff("/dev/vda")) + failed++; + printf("UFFD MOVE CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c new file mode 100644 index 0000000000000..19747755c4c6c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c @@ -0,0 +1,913 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - uffd line=%d errno=%d\n", __LINE__, \ + errno); \ + exit(1); \ + } \ + } while (0) +struct writer { + volatile unsigned char *address; + unsigned char value; + atomic_int done, tid; +}; +static void *write_byte(void *opaque) +{ + struct writer *w = opaque; + atomic_store(&w->tid, syscall(SYS_gettid)); + *w->address = w->value; + atomic_store(&w->done, 1); + return NULL; +} +struct forker { + unsigned char *base; + size_t page_size; + pid_t pid; +}; +static void *fork_byte(void *opaque) +{ + struct forker *f = opaque; + f->pid = fork(); + if (!f->pid) { + alarm(10); + f->base[f->page_size + 17] = 0xa5; + _exit(f->base[f->page_size + 17] != 0xa5); + } + return NULL; +} +static struct uffd_msg read_event(int fd) +{ + struct pollfd p = { .fd = fd, .events = POLLIN }; + struct uffd_msg msg; + CHECK(poll(&p, 1, 10000) == 1 && (p.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + return msg; +} +static void protect(int fd, void *start, size_t len, unsigned long mode) +{ + struct uffdio_writeprotect wp = { .range = { (uintptr_t)start, len }, + .mode = mode }; + CHECK(!ioctl(fd, UFFDIO_WRITEPROTECT, &wp)); +} +static void resolve_write(int fd, unsigned char *base, size_t ps, unsigned page, + unsigned char value, int exact) +{ + struct writer w = { .address = base + page * ps + 17, .value = value }; + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + struct uffd_msg msg; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT); + CHECK(msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_WRITE)); + CHECK(msg.arg.pagefault.address == + (uintptr_t)(base + page * ps + (exact ? 17 : 0))); + CHECK(msg.arg.pagefault.feat.ptid == (unsigned int)atomic_load(&w.tid)); + CHECK(!atomic_load(&w.done)); + protect(fd, base + page * ps, ps, UFFDIO_WRITEPROTECT_MODE_DONTWAKE); + struct uffdio_range range = { (uintptr_t)(base + page * ps), ps }; + CHECK(!ioctl(fd, UFFDIO_WAKE, &range)); + CHECK(!pthread_join(thread, NULL) && atomic_load(&w.done)); + CHECK(base[page * ps + 17] == value); + CHECK(poll(&pollfd, 1, 0) == 0); +} +static void fill(int fd, void *dst, void *src, size_t len, unsigned long mode) +{ + struct uffdio_copy copy = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = len, + .mode = mode }; + CHECK(!ioctl(fd, UFFDIO_COPY, ©) && copy.copy == (long long)len); +} +static void zero(int fd, void *dst, size_t len) +{ + struct uffdio_zeropage z = { .range = { (uintptr_t)dst, len } }; + CHECK(!ioctl(fd, UFFDIO_ZEROPAGE, &z) && z.zeropage == (long long)len); +} +static uint64_t physical(void *address, size_t ps) +{ + uint64_t entry; + int fd = open("/proc/self/pagemap", O_RDONLY); + CHECK(fd >= 0 && + pread(fd, &entry, 8, (uintptr_t)address / ps * 8) == 8); + CHECK(!close(fd) && (entry & (1ULL << 63))); + return (entry & ((1ULL << 55) - 1)) * ps; +} +static void missing(size_t ps, size_t native) +{ + size_t len = 4 * native; + unsigned char *map = mmap(NULL, len + native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *src = mmap(NULL, len + native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(map != MAP_FAILED && src != MAP_FAILED); + unsigned char *p = map + ps; + CHECK(!mlock2(p, len, MLOCK_ONFAULT)); + for (size_t i = 0; i < len; i++) + src[i] = (i / ps) ^ (i * 7 + 3); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_WP }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK((reg.ioctls & + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE))) == + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE))); + /* A write blocks only on its missing leaf; COPY_MODE_WP causes a second event. */ + struct writer w = { .address = p + 17, .value = 0xd3 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == UFFD_PAGEFAULT_FLAG_WRITE && + !atomic_load(&w.done)); + fill(fd, p, src, ps, UFFDIO_COPY_MODE_WP); + msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WRITE | UFFD_PAGEFAULT_FLAG_WP)); + protect(fd, p, ps, 0); + CHECK(!pthread_join(thread, NULL) && atomic_load(&w.done)); + src[17] = w.value; + CHECK(!memcmp(p, src, ps)); + zero(fd, p + ps, ps); + for (size_t i = 0; i < ps; i++) + CHECK(p[ps + i] == 0); + p[ps + 17] = 0xe4; + CHECK(!memcmp(p, src, ps)); + /* A batch starts at a user-page boundary, not necessarily a native one. */ + fill(fd, p + 2 * ps, src + 2 * ps, len - 2 * ps, 0); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, len - 2 * ps)); + if (ps < native) { + unsigned char *a = + (void *)(((uintptr_t)(p + 2 * ps) + native - 1) & + ~(native - 1)); + uint64_t start = physical(a, ps); + CHECK(start != 0 && !(start & (native - 1))); + for (size_t i = ps; i < native; i += ps) + CHECK(physical(a + i, ps) == start + i); + } + struct uffdio_copy copy = { .dst = (uintptr_t)p, + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EEXIST && + copy.copy == -EEXIST); + CHECK(!munlock(p, len)); + /* Preserve the successfully copied prefix when the next PTE already exists. */ + CHECK(!madvise(p, ps, MADV_DONTNEED)); + copy = (struct uffdio_copy){ .dst = (uintptr_t)p, + .src = (uintptr_t)src, + .len = 2 * ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EAGAIN && + copy.copy == (long long)ps); + CHECK(!memcmp(p, src, ps) && p[ps + 17] == 0xe4); + /* An invalid source stops at its leaf, without installing it or touching neighbours. */ + CHECK(!madvise(p + 2 * ps, 2 * ps, MADV_DONTNEED)); + CHECK(!mprotect(src + 3 * ps, ps, PROT_NONE)); + copy = (struct uffdio_copy){ .dst = (uintptr_t)(p + 2 * ps), + .src = (uintptr_t)(src + 2 * ps), + .len = 2 * ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EAGAIN && + copy.copy == (long long)ps); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, ps)); + CHECK(!mprotect(src + 3 * ps, ps, PROT_READ | PROT_WRITE)); + /* A fresh source forces the unlocked copy retry. */ + CHECK(!madvise(src + 3 * ps, ps, MADV_DONTNEED)); + fill(fd, p + 3 * ps, src + 3 * ps, ps, 0); + for (size_t i = 0; i < ps; i++) + CHECK(p[3 * ps + i] == 0); + CHECK(!memcmp(p + 4 * ps, src + 4 * ps, len - 4 * ps)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + p[2 * ps + 9] ^= 0xff; + _exit(p[2 * ps + 9] == src[2 * ps + 9]); + } + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, ps)); + CHECK(!close(fd)); + CHECK(!munmap(map, len + native) && !munmap(src, len + native)); + printf("ok - uffd ABI %zuK missing copy/WP, zero, packed batch, partial EEXIST/EFAULT, source retry and COW\n", + ps / 1024); +} + +struct reader { + volatile unsigned char *address; + unsigned char value; + atomic_int done; +}; +static void *read_byte(void *opaque) +{ + struct reader *r = opaque; + r->value = *r->address; + atomic_store(&r->done, 1); + return NULL; +} +static void resume_minor(int fd, void *start, size_t len, unsigned long mode) +{ + struct uffdio_continue c = { .range = { (uintptr_t)start, len }, + .mode = mode }; + CHECK(!ioctl(fd, UFFDIO_CONTINUE, &c) && c.mapped == (long long)len); +} +static void file_minor(size_t ps, size_t native) +{ + for (int shared = 0; shared < 2; shared++) { + int memfd = memfd_create("uffd-minor-granule", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * native)); + unsigned char *alias = mmap(NULL, 4 * native, + PROT_READ | PROT_WRITE, MAP_SHARED, + memfd, 0); + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + shared ? MAP_SHARED : MAP_PRIVATE, + memfd, ps); + CHECK(alias != MAP_FAILED && p != MAP_FAILED); + CHECK(!mlock2(p, 3 * ps, MLOCK_ONFAULT)); + for (size_t i = 0; i < 4 * native; i++) + alias[i] = (i / ps) ^ (i * 3 + 7); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { + .api = UFFD_API, + .features = UFFD_FEATURE_MINOR_SHMEM | + UFFD_FEATURE_PAGEFAULT_FLAG_WP + }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { + .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_MINOR | + UFFDIO_REGISTER_MODE_WP + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(reg.ioctls & (1ULL << _UFFDIO_CONTINUE)); + + for (unsigned i = 0; i < 3; i++) { + struct reader r = { .address = p + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == + (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == + UFFD_PAGEFAULT_FLAG_MINOR && + !atomic_load(&r.done)); + alias[(i + 1) * ps + 17] ^= 0x3c; + resume_minor(fd, p + i * ps, ps, + UFFDIO_CONTINUE_MODE_WP | + UFFDIO_CONTINUE_MODE_DONTWAKE); + struct uffdio_range range = { (uintptr_t)(p + i * ps), + ps }; + CHECK(!ioctl(fd, UFFDIO_WAKE, &range)); + CHECK(!pthread_join(thread, NULL) && + atomic_load(&r.done)); + CHECK(r.value == alias[(i + 1) * ps + 17]); + CHECK(physical(p + i * ps, ps) == + physical(alias + (i + 1) * ps, ps)); + /* Independent write protection survives installing the file leaf. */ + struct writer w = { .address = p + i * ps + 17, + .value = 0xa1 + i }; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == + (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WP | + UFFD_PAGEFAULT_FLAG_WRITE)); + protect(fd, p + i * ps, ps, 0); + CHECK(!pthread_join(thread, NULL) && + atomic_load(&w.done)); + CHECK(p[i * ps + 17] == w.value); + CHECK(shared ? alias[(i + 1) * ps + 17] == w.value : + alias[(i + 1) * ps + 17] == r.value); + } + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + CHECK(poll(&pollfd, 1, 0) == 0); + CHECK(!munlock(p, 3 * ps)); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + resume_minor(fd, p, 3 * ps, 0); + CHECK(!memcmp(p, alias + ps, 3 * ps)); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + CHECK(!ftruncate(memfd, ps + 17)); + resume_minor(fd, p, ps, 0); + struct uffdio_continue c = { .range = { (uintptr_t)(p + ps), + ps } }; + CHECK(ioctl(fd, UFFDIO_CONTINUE, &c) == -1 && errno == EFAULT && + c.mapped == -EFAULT); + CHECK(!close(fd) && !munmap(p, 3 * ps) && + !munmap(alias, 4 * native) && !close(memfd)); + printf("ok - uffd ABI %zuK shmem %s minor/WP, byte offsets, alias coherence, COW, batch continue and EOF\n", + ps / 1024, shared ? "shared" : "private"); + } +} + +static uint64_t pagemap(void *address, size_t ps) +{ + uint64_t entry; + int fd = open("/proc/self/pagemap", O_RDONLY); + CHECK(fd >= 0 && + pread(fd, &entry, 8, (uintptr_t)address / ps * 8) == 8); + CHECK(!close(fd)); + return entry; +} +static sigjmp_buf bus_env; +static void *bus_address; +static void bus_handler(int sig, siginfo_t *info, void *context) +{ + (void)context; + if (sig != SIGBUS) + _exit(91); + bus_address = info->si_addr; + siglongjmp(bus_env, 1); +} +static void markers(size_t ps) +{ + for (int async = 0; async < 2; async++) { + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { + .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_THREAD_ID | + UFFD_FEATURE_WP_UNPOPULATED | + (async ? UFFD_FEATURE_WP_ASYNC : 0) + }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { + .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_WP + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + protect(fd, p, 3 * ps, UFFDIO_WRITEPROTECT_MODE_WP); + for (unsigned i = 0; i < 3; i++) + CHECK(pagemap(p + i * ps, ps) & (1ULL << 57)); + CHECK(*(volatile unsigned char *)(p + ps + 17) == 0); + if (async) + p[ps + 17] = 0x67; + else + resolve_write(fd, p, ps, 1, 0x67, 0); + CHECK(p[ps + 17] == 0x67); + CHECK(pagemap(p, ps) & (1ULL << 57)); + CHECK(!(pagemap(p + ps, ps) & (1ULL << 57))); + CHECK(pagemap(p + 2 * ps, ps) & (1ULL << 57)); + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + CHECK(poll(&pollfd, 1, 0) == 0); + CHECK(!close(fd)); + p[17] = 0x68; + p[2 * ps + 17] = 0x69; + CHECK(!munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK unpopulated WP async=%d per-leaf pagemap state and close\n", + ps / 1024, async); + } + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_POISON }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®) && + (reg.ioctls & (1ULL << _UFFDIO_POISON))); + zero(fd, p, ps); + zero(fd, p + 2 * ps, ps); + struct uffdio_poison poison = { .range = { (uintptr_t)(p + ps), ps } }; + CHECK(!ioctl(fd, UFFDIO_POISON, &poison) && + poison.updated == (long long)ps); + CHECK(ioctl(fd, UFFDIO_POISON, &poison) == -1 && errno == EEXIST && + poison.updated == -EEXIST); + struct sigaction sa = { .sa_sigaction = bus_handler, + .sa_flags = SA_SIGINFO }, + old; + CHECK(!sigemptyset(&sa.sa_mask) && !sigaction(SIGBUS, &sa, &old)); + if (!sigsetjmp(bus_env, 1)) { + (void)*(volatile unsigned char *)(p + ps + 17); + CHECK(0); + } + CHECK(bus_address == p + ps + 17); + CHECK(!sigaction(SIGBUS, &old, NULL)); + p[17] = 0x81; + p[2 * ps + 17] = 0x82; + CHECK(!close(fd) && !munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK poisoned leaf signals SIGBUS and preserves neighbours\n", + ps / 1024); +} +struct pager_request { + uintptr_t address; + size_t size; +}; +static void send_pager(int sock, int fd, struct pager_request *request) +{ + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct iovec iov = { .iov_base = request, .iov_len = sizeof(*request) }; + struct msghdr msg = { .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control) }; + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + c->cmsg_level = SOL_SOCKET; + c->cmsg_type = SCM_RIGHTS; + c->cmsg_len = CMSG_LEN(sizeof(int)); + memcpy(CMSG_DATA(c), &fd, sizeof(fd)); + CHECK(sendmsg(sock, &msg, 0) == sizeof(*request)); +} +static unsigned char pager_pattern(size_t i) +{ + return (i / 4096) ^ (i * 11 + 5); +} +static int pager_worker(int sock, size_t expected) +{ + CHECK(getauxval(AT_PAGESZ) == expected); + alarm(30); + struct pager_request request; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct iovec iov = { .iov_base = &request, .iov_len = sizeof(request) }; + struct msghdr msg = { .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control) }; + CHECK(recvmsg(sock, &msg, 0) == sizeof(request)); + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + CHECK(c && c->cmsg_level == SOL_SOCKET && c->cmsg_type == SCM_RIGHTS && + c->cmsg_len == CMSG_LEN(sizeof(int))); + int fd; + memcpy(&fd, CMSG_DATA(c), sizeof(fd)); + size_t ps = request.size; + unsigned char *source = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(source != MAP_FAILED); + for (size_t i = 0; i < 4 * ps; i++) + source[i] = pager_pattern(i); + for (unsigned i = 0; i < 3; i++) { + struct uffd_msg fault = read_event(fd); + CHECK(fault.event == UFFD_EVENT_PAGEFAULT && + fault.arg.pagefault.address == request.address + i * ps && + !fault.arg.pagefault.flags); + if (i == 1) + zero(fd, (void *)(request.address + ps), ps); + else + fill(fd, (void *)(request.address + i * ps), + source + i * ps, (i == 2 ? 2 : 1) * ps, 0); + } + CHECK(!close(fd) && !close(sock) && !munmap(source, 4 * ps)); + return 0; +} +static void cross_pager(size_t ps, size_t native, int shared) +{ + size_t pager_ps = ps == native ? 4096 : native; + int socks[2]; + CHECK(!socketpair(AF_UNIX, SOCK_SEQPACKET, 0, socks)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + char fd_arg[32], ps_arg[32]; + CHECK(!close(socks[0])); + snprintf(fd_arg, sizeof(fd_arg), "%d", socks[1]); + snprintf(ps_arg, sizeof(ps_arg), "%zu", pager_ps); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, pager_ps, 0UL, 0UL, 0UL)); + execl("/init", "/init", "--pager", fd_arg, ps_arg, NULL); + _exit(3); + } + CHECK(!close(socks[1])); + int memfd = shared ? memfd_create("remote-uffd-shmem", 0) : -1; + if (shared) + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap( + NULL, 4 * ps, PROT_READ | PROT_WRITE, + shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS, memfd, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, 4 * ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + struct pager_request request = { .address = (uintptr_t)p, .size = ps }; + send_pager(socks[0], fd, &request); + for (size_t i = 0; i < 4 * ps; i++) + CHECK(p[i] == (i >= ps && i < 2 * ps ? 0 : pager_pattern(i))); + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!close(fd) && !close(socks[0]) && !munmap(p, 4 * ps)); + if (shared) + CHECK(!close(memfd)); + printf("ok - uffd target %zuK remote pager %zuK shared=%d SCM_RIGHTS copy/zero/batch uses target granule\n", + ps / 1024, pager_ps / 1024, shared); +} + +static void private_missing(size_t ps) +{ + int memfd = memfd_create("uffd-private-hole", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE, memfd, ps); + unsigned char *src = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && src != MAP_FAILED); + for (size_t i = 0; i < 3 * ps; i++) + src[i] = pager_pattern(i); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®) && + (reg.ioctls & (1ULL << _UFFDIO_COPY))); + struct reader r = { .address = p + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + !msg.arg.pagefault.flags); + fill(fd, p, src, 2 * ps, 0); + CHECK(!pthread_join(thread, NULL) && r.value == src[17]); + zero(fd, p + 2 * ps, ps); + CHECK(!memcmp(p, src, 2 * ps)); + unsigned char bytes[256]; + for (size_t off = 0; off < 4 * ps; off += sizeof(bytes)) { + CHECK(pread(memfd, bytes, sizeof(bytes), off) == sizeof(bytes)); + for (unsigned i = 0; i < sizeof(bytes); i++) + CHECK(bytes[i] == 0); + } + CHECK(!madvise(p, ps, MADV_DONTNEED)); + CHECK(!madvise(src, 3 * ps, MADV_DONTNEED)); + fill(fd, p, src, ps, 0); + for (size_t i = 0; i < ps; i++) + CHECK(p[i] == 0); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + CHECK(!ftruncate(memfd, ps + 17)); + struct uffdio_copy copy = { .dst = (uintptr_t)(p + ps), + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EFAULT && + copy.copy == -EFAULT); + fill(fd, p, src, ps, 0); + CHECK(!close(fd) && !close(memfd) && !munmap(p, 3 * ps) && + !munmap(src, 3 * ps)); + printf("ok - uffd ABI %zuK private shmem missing COPY/ZERO, unchanged file bytes, source retry and EOF\n", + ps / 1024); +} + +static int register_shared(void *p, size_t len) +{ + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MISSING_SHMEM | + UFFD_FEATURE_MINOR_SHMEM | + UFFD_FEATURE_PAGEFAULT_FLAG_WP }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_MINOR | + UFFDIO_REGISTER_MODE_WP }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK((reg.ioctls & + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE) | + (1ULL << _UFFDIO_CONTINUE))) == + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE) | + (1ULL << _UFFDIO_CONTINUE))); + return fd; +} +static void shared_missing(size_t ps, size_t native) +{ + int memfd = memfd_create("uffd-shared-holes", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 8 * native)); + unsigned char *p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, ps); + unsigned char *q = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, ps); + unsigned char *src = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && q != MAP_FAILED && src != MAP_FAILED); + for (size_t i = 0; i < 4 * ps; i++) + src[i] = pager_pattern(i); + int fd = register_shared(p, 4 * ps), fd2 = register_shared(q, 4 * ps); + for (unsigned i = 0; i < 2; i++) { + struct reader r = { .address = p + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)(p + i * ps) && + !msg.arg.pagefault.flags); + if (!i) + fill(fd, p, src, ps, UFFDIO_COPY_MODE_WP); + else + zero(fd, p + ps, ps); + CHECK(!pthread_join(thread, NULL) && atomic_load(&r.done)); + CHECK(r.value == (i ? 0 : src[17])); + if (!i) { + struct uffdio_continue c = { + .range = { (uintptr_t)(p + ps), ps } + }; + CHECK(ioctl(fd, UFFDIO_CONTINUE, &c) == -1 && + errno == EFAULT && c.mapped == -EFAULT); + } + } + fill(fd, p + 2 * ps, src + 2 * ps, 2 * ps, 0); + CHECK(!memcmp(p, src, ps) && !memcmp(p + 2 * ps, src + 2 * ps, 2 * ps)); + for (size_t i = 0; i < ps; i++) + CHECK(p[ps + i] == 0); + for (unsigned i = 0; i < 4; i++) { + struct reader r = { .address = q + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd2); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)(q + i * ps) && + msg.arg.pagefault.flags == UFFD_PAGEFAULT_FLAG_MINOR); + resume_minor(fd2, q + i * ps, ps, 0); + CHECK(!pthread_join(thread, NULL) && r.value == p[i * ps + 17]); + CHECK(physical(p + i * ps, ps) == physical(q + i * ps, ps)); + } + CHECK(!memcmp(p, q, 4 * ps)); + if (ps < native) + CHECK(physical(p + ps, ps) == physical(p, ps) + ps); + struct writer w = { .address = p + 17, .value = 0x9a }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WRITE | UFFD_PAGEFAULT_FLAG_WP)); + protect(fd, p, ps, 0); + CHECK(!pthread_join(thread, NULL) && p[17] == w.value && + q[17] == w.value); + struct uffdio_copy c = { .dst = (uintptr_t)(p + ps), + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && errno == EEXIST && + c.copy == -EEXIST); + unsigned char bytes[256]; + CHECK(pread(memfd, bytes, sizeof(bytes), ps) == sizeof(bytes)); + CHECK(!memcmp(bytes, p, sizeof(bytes))); + /* Resolve an untouched source outside VMA locks before taking a cache folio. */ + unsigned char *r = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, 6 * native); + unsigned char *fresh = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(r != MAP_FAILED && fresh != MAP_FAILED); + int fd3 = register_shared(r, 2 * ps); + fill(fd3, r, fresh, ps, 0); + for (size_t i = 0; i < ps; i++) + CHECK(r[i] == 0); + CHECK(!mprotect(fresh, ps, PROT_NONE)); + c = (struct uffdio_copy){ .dst = (uintptr_t)(r + ps), + .src = (uintptr_t)fresh, + .len = ps }; + CHECK(ioctl(fd3, UFFDIO_COPY, &c) == -1 && errno == EFAULT && + c.copy == -EFAULT); + zero(fd3, r + ps, ps); + for (size_t i = 0; i < ps; i++) + CHECK(r[ps + i] == 0); + CHECK(!close(fd3) && !munmap(r, 2 * ps) && !munmap(fresh, ps)); + CHECK(!ftruncate(memfd, 2 * ps + 17)); + c = (struct uffdio_copy){ .dst = (uintptr_t)(p + 2 * ps), + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && + errno == (ps == native ? ENOMEM : EFAULT) && + c.copy == -(ps == native ? ENOMEM : EFAULT)); + CHECK(!close(fd) && !close(fd2) && !close(memfd)); + CHECK(!munmap(p, 4 * ps) && !munmap(q, 4 * ps) && !munmap(src, 4 * ps)); + printf("ok - uffd ABI %zuK shared COPY/ZERO, independent missing/minor pagers, CONTINUE holes, packing, WP/coherence and EOF\n", + ps / 1024); +} + +static void shared_buffered_write(size_t ps) +{ + int memfd = memfd_create("uffd-buffered-validity", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, 0); + unsigned char *src = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && src != MAP_FAILED); + memset(src, 0x57, ps); + int fd = register_shared(p, 4 * ps); + fill(fd, p, src, ps, 0); + CHECK(pwrite(memfd, src, 23, ps + 17) == 23); + for (unsigned i = 1; i < 3; i++) { + struct reader r = { .address = p + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == + (i == 1 ? UFFD_PAGEFAULT_FLAG_MINOR : 0)); + if (i == 1) + resume_minor(fd, p + ps, ps, 0); + else + zero(fd, p + i * ps, ps); + CHECK(!pthread_join(thread, NULL) && + r.value == (i == 1 ? 0x57 : 0)); + } + for (size_t i = 0; i < ps; i++) + CHECK(p[ps + i] == (i >= 17 && i < 40 ? 0x57 : 0)); + CHECK(!memcmp(p, src, ps)); + CHECK(!close(fd) && !close(memfd) && !munmap(p, 4 * ps) && + !munmap(src, ps)); + printf("ok - uffd ABI %zuK buffered short-range write populates its leaf and leaves neighbours missing\n", + ps / 1024); +} + +static int worker(unsigned long expected, unsigned long native) +{ + size_t ps = getauxval(AT_PAGESZ); + CHECK(ps == expected); + alarm(60); + missing(ps, native); + file_minor(ps, native); + markers(ps); + cross_pager(ps, native, 0); + cross_pager(ps, native, 1); + private_missing(ps); + shared_missing(ps, native); + shared_buffered_write(ps); + for (int exact = 0; exact < 2; exact++) { + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + for (unsigned i = 0; i < 3; i++) + memset(p + i * ps, 0x40 + i, ps); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { + .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_THREAD_ID | + (exact ? UFFD_FEATURE_EXACT_ADDRESS | + UFFD_FEATURE_EVENT_FORK : + 0) + }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + CHECK(api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP); + struct uffdio_register reg = { + .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_WP + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(reg.ioctls & (1ULL << _UFFDIO_WRITEPROTECT)); + + for (unsigned i = 0; i < 3; i++) { + protect(fd, p + i * ps, ps, + UFFDIO_WRITEPROTECT_MODE_WP); + /* Neighbours remain directly writable while this page faults. */ + p[((i + 1) % 3) * ps + 31] ^= 1; + resolve_write(fd, p, ps, i, 0x70 + i, exact); + } + protect(fd, p + ps, ps, UFFDIO_WRITEPROTECT_MODE_WP); + pid_t child; + int status; + if (!exact) { + child = fork(); + CHECK(child >= 0); + if (!child) { + alarm(10); + p[ps + 17] = 0xa5; + _exit(p[ps + 17] != 0xa5); + } + } else { + struct forker f = { .base = p, .page_size = ps }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, fork_byte, &f)); + struct uffd_msg event = read_event(fd); + CHECK(event.event == UFFD_EVENT_FORK); + int child_fd = event.arg.fork.ufd; + CHECK(child_fd >= 0); + CHECK(!pthread_join(thread, NULL)); + child = f.pid; + CHECK(child > 0); + event = read_event(child_fd); + CHECK(event.event == UFFD_EVENT_PAGEFAULT && + event.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WP | + UFFD_PAGEFAULT_FLAG_WRITE)); + CHECK(event.arg.pagefault.address == + (uintptr_t)(p + ps + 17)); + CHECK(event.arg.pagefault.feat.ptid == + (unsigned int)child); + protect(child_fd, p + ps, ps, 0); + CHECK(!close(child_fd)); + } + CHECK(waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(p[ps + 17] == 0x71); + resolve_write(fd, p, ps, 1, 0x91, exact); + protect(fd, p, 3 * ps, UFFDIO_WRITEPROTECT_MODE_WP); + struct uffdio_range middle = { (uintptr_t)(p + ps), ps }; + CHECK(!ioctl(fd, UFFDIO_UNREGISTER, &middle)); + p[ps + 17] = 0xb2; + resolve_write(fd, p, ps, 0, 0xb1, exact); + CHECK(!close(fd)); + /* Close removes the remaining registration and write protection. */ + p[2 * ps + 17] = 0xb3; + CHECK(p[17] == 0xb1 && p[ps + 17] == 0xb2 && + p[2 * ps + 17] == 0xb3); + CHECK(!munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK exact=%d events, neighbours, fork delivery/COW, partial unregister and close\n", + ps / 1024, exact); + } + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--pager")) + return pager_worker(atoi(argv[2]), strtoul(argv[3], NULL, 10)); + if (argc == 4 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { native, 4096, 16384 }; + unsigned modes = native == 65536 ? 3 : 2; + int failed = 0; + for (unsigned i = 0; i < modes; i++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char ps[32], ns[32]; + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(ns, sizeof(ns), "%lu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, + 0UL)); + execl("/init", "/init", "--worker", ps, ns, NULL); + _exit(3); + } + int status = 0; + int ok = waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status); + printf("%s - uffd ABI %luK status=%#x\n", ok ? "ok" : "not ok", + sizes[i] / 1024, status); + failed |= !ok; + } + printf("UFFD CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uprobes-init.c b/tools/testing/selftests/arm64/user4k/uprobes-init.c new file mode 100644 index 0000000000000..0c379e4309caf --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uprobes-init.c @@ -0,0 +1,318 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define TRACE "/sys/kernel/tracing/" +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +static int write_file(const char *path, const char *s) +{ + int fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + size_t len = strlen(s); + int ret = write(fd, s, len) == (ssize_t)len ? 0 : -1; + close(fd); + return ret; +} +static int enable(const char *event, int on) +{ + char path[160]; + snprintf(path, sizeof(path), TRACE "events/aurora_up/%s/enable", event); + return write_file(path, on ? "1" : "0"); +} +static uint64_t pagemap(void *p, size_t ps) +{ + uint64_t bits = 0; + int fd = open("/proc/self/pagemap", O_RDONLY); + if (fd >= 0) { + if (pread(fd, &bits, 8, (uintptr_t)p / ps * 8) != 8) + bits = 0; + close(fd); + } + return bits; +} +static unsigned records(const char *event) +{ + FILE *f = fopen(TRACE "trace", "r"); + char *line = NULL; + size_t len = 0; + unsigned count = 0; + char match[96]; + snprintf(match, sizeof(match), "%s:", event); + if (!f) + return 0; + while (getline(&line, &len, f) > 0) + if (strstr(line, match)) + count++; + free(line); + fclose(f); + return count; +} +static int xol_geometry(size_t native) +{ + FILE *f = fopen("/proc/self/maps", "r"); + char *line = NULL; + size_t len = 0; + int ok = 0; + if (!f) + return 0; + while (getline(&line, &len, f) > 0) + if (strstr(line, "[uprobes]")) { + unsigned long a, b; + if (sscanf(line, "%lx-%lx", &a, &b) == 2) { + printf("# XOL %#lx-%#lx native=%zu\n", a, b, + native); + ok = !(a % (unsigned long)native) && + b - a == native; + } + } + free(line); + fclose(f); + return ok; +} +static int exercise(size_t ps, size_t native, unsigned shifted) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(20); + printf("# start ps=%zu shifted=%u\n", ps, shifted); + int fd = memfd_create("aurora-uprobe-code", 0); + CHECK(fd >= 0); + size_t length = 8 * ps; + unsigned char *blob = malloc(length); + CHECK(blob); + memset(blob, 0x71, length); + size_t code_off = 3 * ps + 64, ctr_off = 5 * ps + 32; + size_t map_off = shifted ? 2 * ps : 0, + map_len = shifted ? 2 * ps : 4 * ps; + uint32_t insns[] = { 0x11001c00, 0xd65f03c0 }; + memcpy(blob + code_off, insns, sizeof(insns)); + memset(blob + ctr_off, 0, 2); + CHECK(write(fd, blob, length) == (ssize_t)length); + unsigned char *code = mmap((void *)(0x20000000 + (shifted ? ps : 0)), + map_len, PROT_READ | PROT_EXEC, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, + map_off); + unsigned char *counter = + mmap((void *)(0x30000000 + 2 * ps), ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, 5 * ps); + unsigned char *other = mmap((void *)0x40000000, ps, + PROT_READ | PROT_EXEC, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, 0); + CHECK(code != MAP_FAILED && counter != MAP_FAILED && + other != MAP_FAILED); + unsigned long (*target)(unsigned long) = + (void *)(code + code_off - map_off); + volatile unsigned short *ctr = (void *)(counter + 32); + CHECK(*ctr == 0); + CHECK(target(11) == 18); + /* Occupy the default top-of-address-space XOL hint, forcing a new placement. */ + void *guard = mmap((void *)((1UL << 48) - native), ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, + -1, 0); + CHECK(guard != MAP_FAILED); + char entry[64], retname[64], cmd[256]; + snprintf(entry, sizeof(entry), "entry_%zu_%u", ps, shifted); + snprintf(retname, sizeof(retname), "return_%zu_%u", ps, shifted); + snprintf(cmd, sizeof(cmd), + "p:aurora_up/%s /proc/self/fd/%d:0x%zx(0x%zx)\n", entry, fd, + code_off, ctr_off); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + snprintf(cmd, sizeof(cmd), + "r:aurora_up/%s /proc/self/fd/%d:0x%zx(0x%zx)\n", retname, fd, + code_off, ctr_off); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(*ctr == 1); + printf("# enabling return\n"); + CHECK(!enable(retname, 1)); + printf("# return enabled\n"); + CHECK(*ctr == 1); + uint32_t disk[2]; + CHECK(pread(fd, disk, sizeof(disk), code_off) == sizeof(disk)); + CHECK(!memcmp(disk, insns, sizeof(insns))); + CHECK(!memcmp(other, blob, ps)); + CHECK(!memcmp(counter, blob + 5 * ps, 32)); + CHECK(!memcmp(counter + 34, blob + 5 * ps + 34, ps - 34)); + int tracefd = open(TRACE "trace", O_WRONLY | O_TRUNC); + CHECK(tracefd >= 0); + close(tracefd); + printf("# invoking probes\n"); + for (unsigned i = 0; i < 16; i++) + CHECK(target(i) == i + 7); + printf("# invocations complete\n"); + CHECK(xol_geometry(native)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + int ok = *ctr == 1; + for (unsigned i = 0; i < 8; i++) + ok &= target(100 + i) == 107 + i; + _exit(ok ? 0 : 3); + } + int status; + CHECK(waitpid(child, &status, 0) == child); + CHECK(WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(*ctr == 1); + unsigned entries = records(entry), returns = records(retname); + printf("# ABI=%zu entry=%u return=%u\n", ps, entries, returns); + CHECK(entries == 24 && returns == 24); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 1); + CHECK(!enable(retname, 0)); + CHECK(*ctr == 0); + /* Identical restored text is dropped so it can be shared with the file again. */ + CHECK(!(pagemap((void *)target, ps) >> 63)); + CHECK(target(10) == 17); + CHECK(!memcmp(code, blob + map_off, map_len)); + CHECK(!memcmp(other, blob, ps)); + CHECK(!memcmp(counter, blob + 5 * ps, ps)); + /* A pipe GET must prevent dropping the mapped code slot on removal. */ + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(*ctr == 1); + int pipefd[2]; + CHECK(!pipe(pipefd)); + struct iovec iov = { .iov_base = + (void *)((uintptr_t)target & ~(ps - 1)), + .iov_len = ps }; + CHECK(vmsplice(pipefd[1], &iov, 1, 0) == (ssize_t)ps); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 0); + CHECK(pagemap((void *)target, ps) >> 63); + unsigned char *held = malloc(ps); + CHECK(held); + CHECK(read(pipefd[0], held, ps) == (ssize_t)ps); + CHECK(!memcmp(held, blob + (code_off & ~(ps - 1)), ps)); + free(held); + close(pipefd[0]); + close(pipefd[1]); + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 0); + /* Native folio-wide references may defer this optional optimization. */ + if (ps < native) + CHECK(!(pagemap((void *)target, ps) >> 63)); + CHECK(target(20) == 27); + snprintf(cmd, sizeof(cmd), "-:aurora_up/%s\n", entry); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + snprintf(cmd, sizeof(cmd), "-:aurora_up/%s\n", retname); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + CHECK(!munmap(code, map_len)); + CHECK(!munmap(guard, ps)); + CHECK(!munmap(counter, ps)); + CHECK(!munmap(other, ps)); + close(fd); + free(blob); + printf("ok - %zuK uprobe/uretprobe, byte offsets, counter, fork, restore and file sharing\n", + ps / 1024); + return 0; +} +static int wait_bounded(pid_t child, int *status) +{ + for (unsigned i = 0; i < 250; i++) { + pid_t got = waitpid(child, status, WNOHANG); + if (got == child) + return 1; + if (got < 0) + return 0; + if (i == 50 || i == 200) { + char path[64], buf[4096]; + snprintf(path, sizeof(path), "/proc/%d/stack", child); + int fd = open(path, O_RDONLY); + ssize_t n = fd < 0 ? -1 : + read(fd, buf, sizeof(buf) - 1); + if (fd >= 0) + close(fd); + if (n > 0) { + buf[n] = 0; + printf("# child %d stack:\n%s", child, buf); + } else + printf("# child %d stack unavailable errno=%d\n", + child, errno); + } + if (i == 200) { + printf("# killing stalled child %d\n", child); + kill(child, SIGKILL); + } + usleep(100000); + } + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), + strtoul(argv[4], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("tracefs", TRACE, "tracefs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned run = 0; run < (sizes[0] > 16384 ? 6U : 4U); run++) { + unsigned i = run / 2, shifted = run & 1; + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24], native[24], mode[8]; + snprintf(mode, sizeof(mode), "%u", shifted); + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + snprintf(native, sizeof(native), "%zu", sizes[0]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, native, + mode, NULL); + _exit(5); + } + int ok = c > 0 && wait_bounded(c, &status) && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - ABI %zuK uprobes shifted=%u status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, shifted, status); + fail += !ok; + } +done: + printf("UPROBES %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uring-fixture-init.c b/tools/testing/selftests/arm64/user4k/uring-fixture-init.c new file mode 100644 index 0000000000000..cafc11138d2e8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uring-fixture-init.c @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +struct ring { + int fd; + struct io_uring_params p; + void *sq, *cq, *sqes; + size_t sqlen, cqlen, sqelen; +}; +static unsigned load(unsigned *p) +{ + return __atomic_load_n(p, __ATOMIC_ACQUIRE); +} +static void store(unsigned *p, unsigned v) +{ + __atomic_store_n(p, v, __ATOMIC_RELEASE); +} +static size_t align_size(size_t n) +{ + size_t ps = getauxval(AT_PAGESZ); + return (n + ps - 1) & ~(ps - 1); +} +static void destroy(struct ring *r) +{ + if (r->sq && r->sq != MAP_FAILED) + munmap(r->sq, r->sqlen); + if (r->cq && r->cq != MAP_FAILED && r->cq != r->sq) + munmap(r->cq, r->cqlen); + if (r->sqes && r->sqes != MAP_FAILED) + munmap(r->sqes, r->sqelen); + if (r->fd >= 0) + close(r->fd); + *r = (struct ring){ .fd = -1 }; +} +static bool setup(struct ring *r, unsigned entries, unsigned flags) +{ + *r = (struct ring){ .fd = -1 }; + r->p.flags = flags; + r->fd = syscall(__NR_io_uring_setup, entries, &r->p); + if (r->fd < 0) + return false; + r->sqlen = align_size(r->p.sq_off.array + + r->p.sq_entries * sizeof(unsigned)); + r->cqlen = align_size(r->p.cq_off.cqes + + r->p.cq_entries * sizeof(struct io_uring_cqe) * + (flags & IORING_SETUP_CQE32 ? 2 : 1)); + r->sqelen = align_size(r->p.sq_entries * sizeof(struct io_uring_sqe) * + (flags & IORING_SETUP_SQE128 ? 2 : 1)); + if (r->p.features & IORING_FEAT_SINGLE_MMAP) { + if (r->cqlen > r->sqlen) + r->sqlen = r->cqlen; + r->cqlen = r->sqlen; + } + r->sq = mmap(NULL, r->sqlen, PROT_READ | PROT_WRITE, MAP_SHARED, r->fd, + IORING_OFF_SQ_RING); + r->cq = (r->p.features & IORING_FEAT_SINGLE_MMAP) ? + r->sq : + mmap(NULL, r->cqlen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_CQ_RING); + r->sqes = mmap(NULL, r->sqelen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_SQES); + return r->sq != MAP_FAILED && r->cq != MAP_FAILED && + r->sqes != MAP_FAILED; +} +static bool request(struct ring *r, unsigned op, int fd, void *buf, + unsigned len, uint64_t off, bool async) +{ + unsigned *sqtail = r->sq + r->p.sq_off.tail, + *sqmask = r->sq + r->p.sq_off.ring_mask; + unsigned *cqhead = r->cq + r->p.cq_off.head, + *cqtail = r->cq + r->p.cq_off.tail, + *cqmask = r->cq + r->p.cq_off.ring_mask; + unsigned tail = load(sqtail), head = load(cqhead), + idx = tail & load(sqmask); + unsigned sqstride = sizeof(struct io_uring_sqe) * + (r->p.flags & IORING_SETUP_SQE128 ? 2 : 1); + unsigned cqstride = sizeof(struct io_uring_cqe) * + (r->p.flags & IORING_SETUP_CQE32 ? 2 : 1); + struct io_uring_sqe *sqe = r->sqes + idx * sqstride; + memset(sqe, 0, sqstride); + sqe->opcode = op; + sqe->fd = fd; + sqe->addr = (uintptr_t)buf; + sqe->len = len; + sqe->off = off; + sqe->user_data = (uint64_t)tail + 1234; + if (async) + sqe->flags = IOSQE_ASYNC; + ((unsigned *)(r->sq + r->p.sq_off.array))[idx] = idx; + store(sqtail, tail + 1); + int ret = syscall(__NR_io_uring_enter, r->fd, 1, 1, + IORING_ENTER_GETEVENTS, NULL, 0); + if (ret != 1 || load(cqtail) != head + 1) { + ksft_print_msg("enter=%d errno=%d\n", ret, errno); + return false; + } + struct io_uring_cqe *cqe = + r->cq + r->p.cq_off.cqes + (head & load(cqmask)) * cqstride; + unsigned expected = op == IORING_OP_NOP ? 0 : len; + if (op == IORING_OP_READV_FIXED || op == IORING_OP_WRITEV_FIXED) { + expected = 0; + for (unsigned i = 0; i < len; i++) + expected += ((struct iovec *)buf)[i].iov_len; + } + bool ok = cqe->user_data == (uint64_t)tail + 1234 && + cqe->res == (int)expected; + if (!ok) + ksft_print_msg("cqe res=%d expected=%u\n", cqe->res, expected); + store(cqhead, head + 1); + return ok; +} +static unsigned long pinned_kb(void) +{ + FILE *f = fopen("/proc/self/status", "r"); + char line[160]; + unsigned long n = ~0UL; + if (!f) + return n; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmPin: %lu kB", &n) == 1) + break; + fclose(f); + return n; +} +static bool bytes_are(unsigned char *p, size_t len, unsigned char value) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value) + return false; + return true; +} +static void fixed_checks(int fd, unsigned char *p, size_t len, size_t ps) +{ + struct ring r = { .fd = -1 }; + unsigned char *q = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + struct iovec reg = { p + 512, len - 1024 }; + bool live = false, ok = setup(&r, 8, 0); + unsigned long baseline = pinned_kb(); + memset(p, 0xa6, len); + if (ok) + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + ksft_test_result(ok, "register unaligned long-term buffer\n"); + if (ok) { + ok = request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, 7 * 512, 0, + false); + memset(p + 512, 0, 7 * 512); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + 512, 7 * 512, 0, + false) && + bytes_are(p + 512, 7 * 512, 0xa6); + } + ksft_test_result(ok, "fixed first-quarter slice offsets\n"); + if (ok) { + size_t off = len - 3 * ps; + memset(p + off, 0x49, 2 * ps); + ok = request(&r, IORING_OP_WRITE_FIXED, fd, p + off, 2 * ps, + 65536, true); + memset(p + off, 0, 2 * ps); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + off, 2 * ps, + 65536, true) && + bytes_are(p + off, 2 * ps, 0x49); + } + ksft_test_result(ok, "fixed late-buffer fast indexing and async I/O\n"); + if (ok) { + struct iovec v[3] = { { p + 512, 512 }, + { p + 2 * ps + 512, ps - 512 }, + { p + 17 * ps + 512, 1024 } }; + memset(p, 0x78, len); + ok = request(&r, IORING_OP_WRITEV_FIXED, fd, v, 3, 131072, + true); + memset(p, 0x11, len); + ok &= request(&r, IORING_OP_READV_FIXED, fd, v, 3, 131072, + true); + for (unsigned i = 0; i < 3; i++) + ok &= bytes_are(v[i].iov_base, v[i].iov_len, 0x78); + ok &= p[0] == 0x11 && p[ps + 512] == 0x11 && p[17 * ps] == 0x11; + } + ksft_test_result(ok, "fixed vectored physical offsets and guards\n"); + if (ok) { + memset(p + 512, 0x3e, len - 1024); + ok = request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, len - 1024, + 4 * 1024 * 1024, true); + memset(p + 512, 0, len - 1024); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + 512, len - 1024, + 4 * 1024 * 1024, true) && + bytes_are(p + 512, len - 1024, 0x3e); + } + ksft_test_result(ok, "fixed multi-bio transfer\n"); + if (ok) { + ok = mmap(p, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == p; + memset(p, 0x55, len); + ok &= request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, 3 * ps, + 8 * 1024 * 1024, true); + if (q != MAP_FAILED) { + memset(q, 0, 4 * ps); + ok &= request(&r, IORING_OP_READ, fd, q, 3 * ps, + 8 * 1024 * 1024, false) && + bytes_are(q, 3 * ps, 0x3e) && + bytes_are(p, len, 0x55); + } else + ok = false; + } + ksft_test_result(ok, + "registered source survives unmap and replacement\n"); + if (ok) { + memset(q, 0xc1, 3 * ps); + ok = request(&r, IORING_OP_WRITE, fd, q, 3 * ps, + 10 * 1024 * 1024, false); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + 512, 3 * ps, + 10 * 1024 * 1024, true); + ok &= request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, 3 * ps, + 12 * 1024 * 1024, true); + memset(q, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ, fd, q, 3 * ps, + 12 * 1024 * 1024, false) && + bytes_are(q, 3 * ps, 0xc1) && bytes_are(p, len, 0x55); + } + ksft_test_result( + ok, + "registered destination survives unmap without touching replacement\n"); + if (live) { + ok = syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } else + ok = false; + ok &= pinned_kb() == baseline; + if (ok) { + ok = !mprotect(p + ps, ps, PROT_NONE); + struct iovec bad[2] = { { p, ps }, { p + ps, ps } }; + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, bad, 2) == -1 && + errno == EFAULT; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE) && + pinned_kb() == baseline; + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + } + ksft_test_result( + ok, + "failed multi-buffer registration rolls back pins and accounting\n"); + if (live) { + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } + for (unsigned i = 0; i < 64 && ok; i++) { + reg = (struct iovec){ p + ps + 512, 2 * ps }; + ok = !madvise(p + ps, 3 * ps, MADV_DONTNEED); + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + if (live) { + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } + ok &= pinned_kb() == baseline; + } + ksft_test_result(ok, + "64 register/reuse/unregister cycles balance VmPin\n"); + /* The file offset deliberately differs from the virtual quarter index. */ + int mem = memfd_create("uring-fixed-shared", 0); + ok &= mem >= 0 && !ftruncate(mem, len + 3 * ps) && q != MAP_FAILED; + if (ok) { + ok = mmap(p, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, mem, 3 * ps) == p; + memset(p, 0x6b, len); + reg = (struct iovec){ p + 512, len - 1024 }; + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + struct iovec v[3] = { { p + 512, 512 }, + { p + 3 * ps + 512, ps - 512 }, + { p + 23 * ps + 512, 1024 } }; + if (ok) { + ok = request(&r, IORING_OP_WRITEV_FIXED, fd, v, 3, + 14 * 1024 * 1024, true); + memset(p + 512, 0x18, len - 1024); + ok &= request(&r, IORING_OP_READV_FIXED, fd, v, 3, + 14 * 1024 * 1024, true); + for (unsigned i = 0; i < 3; i++) + ok &= bytes_are(v[i].iov_base, v[i].iov_len, + 0x6b); + ok &= !msync(p, len, MS_SYNC) && + pread(mem, q, ps, 3 * ps) == (ssize_t)ps; + ok &= bytes_are(q, 1024, 0x6b) && q[1024] == 0x18 && + p[len - 1] == 0x6b; + } + } + ksft_test_result( + ok, "registered shared-file quarters use physical offsets\n"); + if (ok) { + struct iovec replacement = { q + 512, 3 * ps }; + struct io_uring_rsrc_update2 update = { + .data = (uintptr_t)&replacement, .nr = 1 + }; + memset(q, 0x9d, 4 * ps); + ok = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS_UPDATE, &update, + sizeof(update)) == 1; + ok &= request(&r, IORING_OP_WRITE_FIXED, fd, q + 512, 3 * ps, + 16 * 1024 * 1024, true); + memset(q + 512, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ_FIXED, fd, q + 512, 3 * ps, + 16 * 1024 * 1024, true) && + bytes_are(q + 512, 3 * ps, 0x9d); + } + if (live) { + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } + ok &= pinned_kb() == baseline; + ksft_test_result( + ok, + "registered-buffer update transfers ownership and balances accounting\n"); + if (mem >= 0) + close(mem); + if (live) + syscall(__NR_io_uring_register, r.fd, IORING_UNREGISTER_BUFFERS, + NULL, 0); + destroy(&r); + if (q != MAP_FAILED) + munmap(q, len); +} + +static int exercise(void) +{ + struct ring r = { .fd = -1 }; + size_t ps = getauxval(AT_PAGESZ), len = 2 * 1024 * 1024; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fd = open("/dev/user4k-test", O_RDWR | O_DIRECT); + bool ok; + ksft_print_header(); + ksft_set_plan(19); + ksft_print_msg("ABI=%zuK\n", ps / 1024); + ok = setup(&r, 4, 0); + ksft_test_result(ok, "minimum rounded ring mappings\n"); + void *bad = ok ? mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_SHARED, + r.fd, 4096) : + MAP_FAILED; + bool rejected = ok && bad == MAP_FAILED && errno == EINVAL; + if (bad != MAP_FAILED) + munmap(bad, ps); + ksft_test_result(rejected, + "reject sub-native offset in ring selector\n"); + if (ok) { + ok = true; + for (unsigned i = 0; i < 64 && ok; i++) + ok = request(&r, IORING_OP_NOP, -1, NULL, 0, 0, false); + } + ksft_test_result(ok, "small submission/completion ring wrap\n"); + if (p == MAP_FAILED || fd < 0) + ksft_exit_fail_msg("buffer/device setup errno=%d\n", errno); + if (ok) { + memset(p + ps, 0x69, 3 * ps); + ok = request(&r, IORING_OP_WRITE, fd, p + ps, 3 * ps, 0, false); + memset(p + ps, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ, fd, p + ps, 3 * ps, 0, false); + for (size_t i = ps; i < 4 * ps; i++) + ok &= p[i] == 0x69; + } + ksft_test_result(ok, + "direct read/write from unregistered user buffers\n"); + if (ok) { + memset(p + ps, 0x37, 3 * ps); + ok = request(&r, IORING_OP_WRITE, fd, p + ps, 3 * ps, 65536, + true); + memset(p + ps, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ, fd, p + ps, 3 * ps, 65536, + true); + for (size_t i = ps; i < 4 * ps; i++) + ok &= p[i] == 0x37; + } + ksft_test_result(ok, "io-worker forced asynchronous read/write\n"); + if (ok) { + memset(p, 0xb6, len); + ok = request(&r, IORING_OP_WRITE, fd, p, len, 4 * 1024 * 1024, + true); + memset(p, 0, len); + ok &= request(&r, IORING_OP_READ, fd, p, len, 4 * 1024 * 1024, + true); + for (size_t i = 0; i < len; i++) + ok &= p[i] == 0xb6; + } + ksft_test_result(ok, "multi-bio io-worker transfer\n"); + destroy(&r); + ok = setup(&r, 4096, 0); + if (ok) + for (unsigned i = 0; i < 8193 && ok; i++) + ok = request(&r, IORING_OP_NOP, -1, NULL, 0, 0, false); + ksft_test_result(ok, "large ring crosses native pages and wraps\n"); + destroy(&r); + ok = setup(&r, 64, IORING_SETUP_SQE128 | IORING_SETUP_CQE32); + if (ok) + for (unsigned i = 0; i < 129 && ok; i++) + ok = request(&r, IORING_OP_NOP, -1, NULL, 0, 0, false); + ksft_test_result(ok, "128-byte SQEs and 32-byte CQEs\n"); + destroy(&r); + fixed_checks(fd, p, len, ps); + close(fd); + munmap(p, len); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +int main(int argc, char **argv) +{ + bool ok = true; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && !strcmp(argv[1], "--test") && getppid() == 1) + return exercise(); + if (getpid() != 1 || argc != 1) + return 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + FILE *dev = fopen("/sys/class/block/vda/dev", "r"); + unsigned maj, min; + if (!dev || fscanf(dev, "%u:%u", &maj, &min) != 2) + return 4; + fclose(dev); + if (mknod("/dev/user4k-test", S_IFBLK | 0600, makedev(maj, min))) + return 5; + for (unsigned small = 0; small < 2; small++) { + pid_t pid = fork(); + int status = 0; + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + small ? 4096UL : 0UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + printf("%s - io_uring fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/user-events-init.c b/tools/testing/selftests/arm64/user4k/user-events-init.c new file mode 100644 index 0000000000000..4b35954e2a210 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/user-events-init.c @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define TRACE "/sys/kernel/tracing/" +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) + +static int toggle(const char *name, int enabled) +{ + char path[192]; + snprintf(path, sizeof(path), TRACE "events/user_events/%s/enable", + name); + int fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + int ret = write(fd, enabled ? "1" : "0", 1) == 1 ? 0 : -1; + close(fd); + return ret; +} +static uint64_t value(void *p, unsigned bytes) +{ + return bytes == 4 ? __atomic_load_n((uint32_t *)p, __ATOMIC_ACQUIRE) : + __atomic_load_n((uint64_t *)p, __ATOMIC_ACQUIRE); +} +static int wait_value(void *p, unsigned bytes, uint64_t expected) +{ + for (unsigned i = 0; i < 3000; i++) { + if (value(p, bytes) == expected) + return 1; + usleep(1000); + } + return 0; +} +static int check_bytes(unsigned char *p, size_t size, unsigned char fill, + void *word, unsigned bytes) +{ + for (size_t i = 0; i < size; i++) + if ((!word || (uintptr_t)(p + i) < (uintptr_t)word || + (uintptr_t)(p + i) >= (uintptr_t)word + bytes) && + p[i] != fill) { + printf("byte mismatch at %zu got=%#x expected=%#x\n", i, + p[i], fill); + return 0; + } + return 1; +} +static int event_recorded(const char *name) +{ + char buf[16384]; + size_t n = 0; + int fd = open(TRACE "trace", O_RDONLY); + if (fd < 0) + return 0; + while (n < sizeof(buf) - 1) { + ssize_t got = read(fd, buf + n, sizeof(buf) - 1 - n); + if (got < 0) { + close(fd); + return 0; + } + if (!got) + break; + n += got; + } + close(fd); + buf[n] = 0; + char *record = strstr(buf, name), + *payload = record ? strstr(record, "payload=") : NULL; + int ok = payload && strtoul(payload + 8, NULL, 0) == 0x12345678; + if (!ok) + printf("trace mismatch:\n%s\n", buf); + return ok; +} +static int exercise_case(size_t ps, unsigned bytes, unsigned shared) +{ + const size_t len = 4 * ps; + unsigned char *src, *dst; + int memfd = -1; + if (shared) { + memfd = memfd_create("user-event-bitmap", 0); + CHECK(memfd >= 0); + CHECK(!ftruncate(memfd, len)); + } + src = mmap((void *)0x20000000, len, PROT_READ | PROT_WRITE, + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS) | + MAP_FIXED_NOREPLACE, + memfd, 0); + dst = mmap((void *)0x30000000, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, len); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps; + void *word = leaf + ps - bytes; + uint64_t mask = bytes == 4 ? 1ULL << 31 : 1ULL << 63; + memset(word, 0, bytes); + char name[80], spec[120]; + snprintf(name, sizeof(name), "aurora_ue_%zu_%u_%u", ps, bytes, shared); + snprintf(spec, sizeof(spec), "%s u32 payload", name); + int tracefd = open(TRACE "trace", O_WRONLY | O_TRUNC); + CHECK(tracefd >= 0); + close(tracefd); + int fd = open(TRACE "user_events_data", O_RDWR); + CHECK(fd >= 0); + struct user_reg reg = { .size = sizeof(reg), + .enable_bit = bytes * 8 - 1, + .enable_size = bytes, + .enable_addr = (uintptr_t)word, + .name_args = (uintptr_t)spec }; + CHECK(!ioctl(fd, DIAG_IOCSREG, ®)); + CHECK(wait_value(word, bytes, 0)); + CHECK(!toggle(name, 1)); + CHECK(wait_value(word, bytes, mask)); + struct { + uint32_t index, payload; + } event = { reg.write_index, 0x12345678 }; + CHECK(write(fd, &event, sizeof(event)) == sizeof(event)); + CHECK(event_recorded(name)); + CHECK(!toggle(name, 0)); + CHECK(wait_value(word, bytes, 0)); + /* Every other byte, including the adjacent half of the kernel word, is stable. */ + CHECK(check_bytes(leaf, ps, 0x64, word, bytes)); + CHECK(check_bytes(dst, ps, 0x91, NULL, 0)); + CHECK(check_bytes(dst + 2 * ps, 2 * ps, 0x91, NULL, 0)); + int commands[2], reply[2]; + CHECK(!pipe(commands) && !pipe(reply)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + close(commands[1]); + close(reply[0]); + char token; + int ok = 1; + for (unsigned round = 0; round < 32; round++) { + if (read(commands[0], &token, 1) != 1) { + ok = 0; + break; + } + ok = wait_value(word, bytes, token == '1' ? mask : 0) && + check_bytes(leaf, ps, 0x64, word, bytes); + token = ok ? 'y' : 'n'; + if (write(reply[1], &token, 1) != 1 || !ok) + break; + } + _exit(ok ? 0 : 3); + } + close(commands[0]); + close(reply[1]); + int ok = 1; + for (unsigned round = 0; round < 32; round++) { + int on = !(round & 1); + char token = on ? '1' : '0'; + if (toggle(name, on) || + !wait_value(word, bytes, on ? mask : 0) || + write(commands[1], &token, 1) != 1 || + read(reply[0], &token, 1) != 1 || token != 'y') { + ok = 0; + break; + } + } + close(commands[1]); + close(reply[0]); + int status; + CHECK(waitpid(child, &status, 0) == child); + CHECK(ok && WIFEXITED(status) && WEXITSTATUS(status) == 0); + CHECK(check_bytes(leaf, ps, 0x64, word, bytes)); + for (unsigned i = 0; i < 3; i++) + CHECK(check_bytes(src + i * ps, ps, 0x61 + i, NULL, 0)); + /* Force the asynchronous missing-PTE fixup without unregistering. */ + CHECK(!madvise(leaf, ps, MADV_DONTNEED)); + CHECK(!toggle(name, 1)); + CHECK(wait_value(word, bytes, mask)); + CHECK(check_bytes(leaf, ps, shared ? 0x64 : 0, word, bytes)); + CHECK(!toggle(name, 0)); + CHECK(wait_value(word, bytes, 0)); + struct user_unreg unreg = { .size = sizeof(unreg), + .disable_bit = reg.enable_bit, + .disable_addr = (uintptr_t)word }; + CHECK(!ioctl(fd, DIAG_IOCSUNREG, &unreg)); + close(fd); + CHECK(!munmap(src, 3 * ps)); + CHECK(!munmap(dst, len)); + if (memfd >= 0) + close(memfd); + printf("ok - %zuK %u-bit %s relocated bitmap, adjacent bytes, event write and fork toggles\n", + ps / 1024, bytes * 8, shared ? "shared" : "private"); + return 0; +} +static int exercise(size_t expected) +{ + size_t ps = getauxval(AT_PAGESZ); + CHECK(ps == expected); + for (unsigned shared = 0; shared < 2; shared++) + for (unsigned bytes = 4; bytes <= 8; bytes += 4) + CHECK(!exercise_case(ps, bytes, shared)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("tracefs", TRACE, "tracefs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24]; + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = c > 0 && waitpid(c, &status, 0) == c && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - ABI %zuK user events status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("USER EVENTS %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c b/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c new file mode 100644 index 0000000000000..b0a2f2bf385f0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c @@ -0,0 +1,229 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define GET_EXEC_PAGE_SIZE 0x41555002 +static int failed; +static _Thread_local int tls = 7; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static long pss(void *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end; + int active = 0; + long value = -1; + if (!f) + return -1; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + active = (unsigned long)p >= start && + (unsigned long)p < end; + else if (active && sscanf(line, "Pss: %ld kB", &value) == 1) + break; + } + free(line); + fclose(f); + return value; +} +static void *thread_worker(void *arg) +{ + unsigned char *p = arg; + if (tls != 7) + return (void *)1; + tls = 91; + for (unsigned i = 0; i < 16384; i++) + if (p[i] != 0x42) + return (void *)2; + return NULL; +} +static void memory_checks(void) +{ + const size_t ps = 16384; + int status = 0; + pid_t c; + unsigned char *p = mmap(NULL, 6 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + result(p != MAP_FAILED, "16K anonymous mmap"); + if (p == MAP_FAILED) + return; + for (unsigned i = 0; i < 6; i++) + memset(p + i * ps, 0x40 + i, ps); + c = fork(); + if (!c) { + memset(p + ps, 0xb7, ps); + _exit(p[0] == 0x40 && p[2 * ps] == 0x42 ? 0 : 1); + } + int ok = c > 0 && waitpid(c, &status, 0) == c && WIFEXITED(status) && + !WEXITSTATUS(status); + for (unsigned i = 0; i < 6 * ps; i++) + ok &= p[i] == 0x40 + i / ps; + result(ok, "16K fork COW preserves every parent byte"); + result(!mprotect(p + ps, ps, PROT_READ), "16K leaf write protection"); + c = fork(); + if (!c) { + *(volatile unsigned char *)(p + ps + 8192) = 3; + _exit(1); + } + result(c > 0 && waitpid(c, &status, 0) == c && WIFSIGNALED(status) && + WTERMSIG(status) == SIGSEGV, + "write protection covers the interior of a16K leaf"); + result(!mprotect(p + ps, ps, PROT_READ | PROT_WRITE), + "16K write permission restored"); + errno = 0; + result(mprotect(p + 4096, ps, PROT_READ) == -1 && errno == EINVAL, + "16K ABI rejects4K-only protection alignment"); + result(!madvise(p + ps, ps, MADV_DONTNEED), + "16K partial unmap via DONTNEED"); + ok = 1; + for (unsigned i = 0; i < ps; i++) + ok &= p[i] == 0x40 && p[ps + i] == 0 && p[2 * ps + i] == 0x42; + result(ok, "16K discard clears one leaf and preserves neighbours"); + pthread_t thread; + void *answer = (void *)3; + int err = pthread_create(&thread, NULL, thread_worker, p + 2 * ps); + if (!err) + err = pthread_join(thread, &answer); + result(!err && !answer && tls == 7, "16K pthread stack and TLS"); + struct timespec ts; + result(!clock_gettime(CLOCK_MONOTONIC, &ts), "16K vDSO clock access"); + munmap(p, 6 * ps); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (argc == 6 && !strcmp(argv[1], "--child")) { + int fd = atoi(argv[2]), ready = atoi(argv[3]), + release = atoi(argv[4]); + size_t native = strtoul(argv[5], NULL, 10); + const size_t ps = 16384; + result(getauxval(AT_PAGESZ) == ps && getpagesize() == ps, + "EL0 selects16K pages"); + result(prctl(GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) == 0, + "exec consumes alternative selection"); + if (!failed) + memory_checks(); + { + int status = 0; + pid_t c = fork(); + if (!c) { + if (prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, + 0UL)) + _exit(4); + execl("/page-contract", "/page-contract", NULL); + _exit(5); + } + result(c > 0 && waitpid(c, &status, 0) == c && + WIFEXITED(status) && + !WEXITSTATUS(status), + "16K-to16K exec and page/x18 contract"); + } + unsigned char *shared = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, native - ps); + unsigned char *private = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE, fd, native - ps); + int ok = shared != MAP_FAILED && private != MAP_FAILED; + if (ok) { + for (unsigned i = 0; i < ps; i++) + ok &= shared[i] == 0x7b && private[i] == 0x7b; + memset(private, 0xd8, ps); + for (unsigned i = 0; i < ps; i++) + ok &= shared[i] == 0x7b && private[i] == 0xd8; + } + result(ok, "16K file COW copies the whole leaf"); + if (private != MAP_FAILED) + munmap(private, ps); + result(shared != MAP_FAILED && pss(shared) == 8, + "16K file PSS shares all four4K slices"); + if (shared != MAP_FAILED) + shared[ps - 1] = 0xc9; + char token = failed ? 'x' : 'a'; + if (write(ready, &token, 1) != 1 || + read(release, &token, 1) != 1) + return 3; + if (shared != MAP_FAILED) + munmap(shared, ps); + return failed ? 1 : 0; + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + mount("proc", "/proc", "proc", 0, NULL); + size_t native = getauxval(AT_PAGESZ); + int fd = memfd_create("mixed16-smoke", 0); + if (native != 65536 || fd < 0 || ftruncate(fd, native)) { + result(0, "64K parent setup"); + goto done; + } + unsigned char *p = + mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + result(0, "parent shared mmap"); + goto done; + } + memset(p, 0x7b, native); + result(pss(p) == 64, "native64K initial PSS"); + int ready[2], release[2]; + if (pipe(ready) || pipe(release)) { + result(0, "pipes"); + goto done; + } + pid_t c = fork(); + int status = 0; + if (!c) { + char f[24], r[24], w[24], n[24]; + snprintf(f, sizeof(f), "%d", fd); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(w, sizeof(w), "%d", release[0]); + snprintf(n, sizeof(n), "%zu", native); + close(ready[0]); + close(release[1]); + if (prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--child", f, r, w, n, NULL); + _exit(5); + } + close(ready[1]); + close(release[0]); + char token = 0; + int ok = c > 0 && read(ready[0], &token, 1) == 1 && token == 'a'; + result(ok && p[native - 1] == 0xc9, + "native64K and16K shared bytes remain coherent"); + result(pss(p) == 56, + "native64K PSS accounts one16K alias as four shared slices"); + token = 'b'; + if (write(release[1], &token, 1) != 1) + failed++; + result(c > 0 && waitpid(c, &status, 0) == c && WIFEXITED(status) && + !WEXITSTATUS(status), + "16K child completes"); + printf("# child status=%#x\n", status); + result(pss(p) == 64, "native PSS returns after16K exit"); + munmap(p, native); +done: + printf("USER16K SMOKE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c b/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c new file mode 100644 index 0000000000000..c43e28ca8b21d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c @@ -0,0 +1,542 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#define __user +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/vfio.h" +#include "../../../../../drivers/vfio/vfio_iommu_type1_test.h" + +#define IOVA (256UL * 1024 * 1024) +static unsigned int number, failures; + +static void result(int ok, const char *name) +{ + printf("%s %u - %s (errno=%d)\n", ok ? "ok" : "not ok", ++number, name, + errno); + failures += !ok; +} + +static int state(int fd, struct vfio_type1_test_cmd *cmd) +{ + memset(cmd, 0, sizeof(*cmd)); + return ioctl(fd, VFIO_TYPE1_TEST_STATE, cmd); +} + +static int domain(int fd, unsigned int which, int add, size_t page, size_t fail) +{ + struct vfio_type1_test_cmd cmd = { + .domain = which, + .size = page, + .fail_after = fail, + }; + + return ioctl(fd, + add ? VFIO_TYPE1_TEST_DOMAIN_ADD : + VFIO_TYPE1_TEST_DOMAIN_DEL, + &cmd); +} + +static int rw(int fd, size_t offset, void *data, size_t size, + unsigned int flags) +{ + struct vfio_type1_test_cmd cmd = { + .iova = IOVA + offset, + .size = size, + .buffer = (uintptr_t)data, + .flags = flags, + }; + + return ioctl(fd, VFIO_TYPE1_TEST_RW, &cmd); +} + +static int map(int fd, void *data, size_t size) +{ + struct vfio_iommu_type1_dma_map cmd = { + .argsz = sizeof(cmd), + .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, + .iova = IOVA, + .vaddr = (uintptr_t)data, + .size = size, + }; + + return ioctl(fd, VFIO_IOMMU_MAP_DMA, &cmd); +} + +static int unmap(int fd, size_t size) +{ + struct vfio_iommu_type1_dma_unmap cmd = { + .argsz = sizeof(cmd), + .iova = IOVA, + .size = size, + }; + int ret = ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &cmd); + + return ret ? ret : cmd.size == size ? 0 : -2; +} + +static int dirty(int fd, unsigned int flags) +{ + struct vfio_iommu_type1_dirty_bitmap cmd = { .argsz = sizeof(cmd), + .flags = flags }; + + return ioctl(fd, VFIO_IOMMU_DIRTY_PAGES, &cmd); +} + +static int bitmap(int fd, size_t size, uint64_t *bits) +{ + unsigned char + buffer[sizeof(struct vfio_iommu_type1_dirty_bitmap) + + sizeof(struct vfio_iommu_type1_dirty_bitmap_get)] = {}; + struct vfio_iommu_type1_dirty_bitmap *cmd = (void *)buffer; + struct vfio_iommu_type1_dirty_bitmap_get *range = (void *)cmd->data; + + cmd->argsz = sizeof(buffer); + cmd->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP; + range->iova = IOVA; + range->size = size; + range->bitmap.pgsize = 4096; + range->bitmap.size = sizeof(*bits); + range->bitmap.data = (void *)bits; + *bits = 0; + return ioctl(fd, VFIO_IOMMU_DIRTY_PAGES, buffer); +} + +static int all_bytes(const unsigned char *data, size_t size, + unsigned char value) +{ + for (size_t i = 0; i < size; i++) + if (data[i] != value) + return 0; + return 1; +} + +static void external_span(int backing, size_t native) +{ + const size_t fine = 4096, count = native / fine; + uint64_t bits; + + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_dma_unmap remove = { + .argsz = sizeof(remove), + .iova = IOVA, + .size = fine, + }; + int fd = open("/dev/vfio-test", O_RDWR), status = 0; + pid_t child; + + result(fd >= 0 && (!mapped || !domain(fd, 0, 1, fine, 0)), + "span fixture opens lazy or mapped type1 container"); + if (fd < 0) + continue; + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), + "span dirty tracking starts before fine mappings"); + child = fork(); + if (!child) { + unsigned char *data = mmap(NULL, native, + PROT_READ | PROT_WRITE, + MAP_SHARED, backing, 0); + + if (data == MAP_FAILED) + _exit(1); + for (size_t i = 0; i < count; i++) { + struct vfio_iommu_type1_dma_map area = { + .argsz = sizeof(area), + .flags = VFIO_DMA_MAP_FLAG_READ | + VFIO_DMA_MAP_FLAG_WRITE, + .iova = IOVA + i * fine, + .vaddr = (uintptr_t)data + i * fine, + .size = fine, + }; + + if (ioctl(fd, VFIO_IOMMU_MAP_DMA, &area)) + _exit(2); + } + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 17 }; + _exit(ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) ? 3 : 0); + } + result(child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status), + "source pins native page across independent fine MAP ioctls"); + result(!state(fd, &cmd) && !cmd.owner_users && + cmd.external_refs == 1 && + cmd.locked_bytes == count * native && + cmd.owner_locked_bytes == count * native, + "every fine DMA area retains its own charge after source exit"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 31 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && + !state(fd, &cmd) && cmd.external_refs == 2 && + cmd.locked_bytes == count * native, + "cross-area duplicate pin reuses all cached constituent pins"); + result(!bitmap(fd, native, &bits) && + bits == ((1ULL << count) - 1), + "cross-area dirty bitmap marks only each mapping's fine extent"); + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &remove) && + remove.size == fine && !state(fd, &cmd) && + !cmd.external_refs && cmd.notifications == 1 && + cmd.locked_bytes == + (mapped ? (count - 1) * native : 0), + "unmapping one constituent drains references in all page constituents"); + remove.iova = IOVA + fine; + remove.size = native - fine; + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &remove) && + remove.size == native - fine && + !state(fd, &cmd) && !cmd.locked_bytes && + !cmd.mapped_bytes[0], + "remaining fine mappings unmap without pins or translations"); + close(fd); + } +} + +static void native_owner(int fd, int backing, unsigned char *data, + size_t native) +{ + struct vfio_type1_test_cmd cmd = {}; + struct vfio_iommu_type1_dma_unmap invalidate = { + .argsz = sizeof(invalidate), + .flags = VFIO_DMA_UNMAP_FLAG_VADDR, + .iova = IOVA, + .size = 2 * native, + }; + struct vfio_iommu_type1_dma_map update = { + .argsz = sizeof(update), + .flags = VFIO_DMA_MAP_FLAG_VADDR, + .iova = IOVA, + .size = 2 * native, + .vaddr = (uintptr_t)data, + }; + struct __user_cap_header_struct header = { + .version = _LINUX_CAPABILITY_VERSION_3 + }; + struct __user_cap_data_struct saved[2], reduced[2]; + struct rlimit limit, empty; + unsigned char bytes[64]; + int file_flags = fcntl(backing, F_GETFD), status = 0; + pid_t child; + int budget_ready, budget_captured, transferred; + + result(!ioctl(fd, VFIO_TYPE1_TEST_EMULATED, &cmd) && + !domain(fd, 0, 1, 4096, 0), + "VADDR fixture retains production group restrictions"); + result(file_flags >= 0 && + !fcntl(backing, F_SETFD, file_flags & ~FD_CLOEXEC), + "native source inherits exact backing descriptor"); + child = fork(); + if (!child) { + char fd_arg[24], file_arg[24], native_arg[24]; + + snprintf(fd_arg, sizeof(fd_arg), "%d", fd); + snprintf(file_arg, sizeof(file_arg), "%d", backing); + snprintf(native_arg, sizeof(native_arg), "%zu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL)) + _exit(98); + execl("/init", "/init", "--native-source", fd_arg, file_arg, + native_arg, NULL); + _exit(99); + } + if (file_flags >= 0) + fcntl(backing, F_SETFD, file_flags); + result(child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && + !state(fd, &cmd) && !cmd.owner_users && + !(cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && + cmd.mapped_bytes[0] == 2 * native, + "exited native source owns legacy GUP pins before transfer"); + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &invalidate), + "canonical VADDR invalidation leaves device translations alive"); + budget_captured = !getrlimit(RLIMIT_MEMLOCK, &limit) && + !syscall(SYS_capget, &header, saved); + budget_ready = budget_captured; + if (budget_ready) { + memcpy(reduced, saved, sizeof(saved)); + reduced[CAP_IPC_LOCK / 32].effective &= + ~(1U << (CAP_IPC_LOCK % 32)); + empty = (struct rlimit){ .rlim_cur = 0, + .rlim_max = limit.rlim_max }; + budget_ready = !setrlimit(RLIMIT_MEMLOCK, &empty) && + !syscall(SYS_capset, &header, reduced); + } + errno = 0; + result(budget_ready && ioctl(fd, VFIO_IOMMU_MAP_DMA, &update) < 0 && + errno == ENOMEM && !state(fd, &cmd) && + !cmd.owner_users && + !(cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && + cmd.mapped_bytes[0] == 2 * native, + "failed owner credit preserves native pins, metadata and charge"); + if (budget_captured) { + result(!syscall(SYS_capset, &header, saved) && + !setrlimit(RLIMIT_MEMLOCK, &limit), + "restore original memlock budget and effective capability"); + } else { + result(0, "owner budget setup failed"); + } + transferred = !ioctl(fd, VFIO_IOMMU_MAP_DMA, &update) && + !state(fd, &cmd) && cmd.owner_users && + (cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && + cmd.owner_locked_bytes == 2 * native; + result(transferred, + "canonical VADDR retry adopts native pins for alternative owner"); + /* Dependent CPU access is prohibited while VADDR remains invalid. */ + if (!transferred) + return; + result(!rw(fd, 0, bytes, sizeof(bytes), 0) && + all_bytes(bytes, sizeof(bytes), 0x64), + "new owner accesses retained native buffer with its own mm authorization"); + cmd = (struct vfio_type1_test_cmd){ .flags = 1 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_EMULATED, &cmd), + "external consumer rejoins after VADDR update"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 17 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == 2 * native && cmd.external_refs == 1, + "external page array reuses adopted pins without new charge"); + result(!unmap(fd, 2 * native) && !state(fd, &cmd) && + !cmd.locked_bytes && !cmd.owner_locked_bytes && + !cmd.external_refs && !cmd.mapped_bytes[0], + "owner transfer final cleanup refunds recipient and clears domain"); + result(!domain(fd, 0, 0, 0, 0), "owner transfer domain detaches"); +} + +static int exercise(size_t native) +{ + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_info info = { .argsz = sizeof(info) }; + struct vfio_iommu_type1_dma_map bad = { .argsz = 8 }; + unsigned char bytes[64], disk[64]; + uint64_t bits = 0; + int fd = open("/dev/vfio-test", O_RDWR), + backing = memfd_create("vfio", MFD_CLOEXEC); + unsigned char *data = MAP_FAILED; + int status = 0; + pid_t child; + size_t length = native + 4096; + + printf("TAP version 13\n# real type1 ioctl ABI=%luK native=%zuK\n", + getauxval(AT_PAGESZ) / 1024, native / 1024); + result(fd >= 0 && backing >= 0 && !ftruncate(backing, 2 * native), + "open privileged recording-domain fixture and backing"); + if (fd < 0 || backing < 0 || failures) + goto out; + data = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, MAP_SHARED, + backing, 0); + result(data != MAP_FAILED, + "map native-sized shmem backing in actual process mm"); + if (data == MAP_FAILED) + goto out; + memset(data, 0x64, 2 * native); + result(!ioctl(fd, VFIO_IOMMU_GET_INFO, &info) && + (info.iova_pgsizes & 4096), + "type1 GET_INFO reports fine IOVA capability"); + errno = 0; + result(ioctl(fd, VFIO_IOMMU_MAP_DMA, &bad) < 0 && errno == EINVAL, + "canonical MAP rejects truncated argument"); + errno = 0; + result(ioctl(fd, VFIO_IOMMU_MAP_DMA, (void *)1) < 0 && errno == EFAULT, + "canonical MAP rejects inaccessible userspace argument"); + result(!map(fd, data, 2 * native), "lazy full-range MAP from real EL0"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0], + "lazy MAP acquires no DMA pins or translations"); + result(!rw(fd, 0, bytes, sizeof(bytes), 0) && + all_bytes(bytes, sizeof(bytes), 0x64), + "CPU-only GET reads actual source bytes"); + result(!state(fd, &cmd) && !cmd.locked_bytes, + "CPU GET does not charge long-term pins"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), + "canonical dirty tracking starts"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 17 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == native && cmd.external_refs == 1, + "external page array preserves unaligned caller byte offset"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 31 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == native && cmd.external_refs == 2, + "duplicate external pin charges physical native page once"); + result(!bitmap(fd, 2 * native, &bits) && + bits == ((1ULL << (native / 4096)) - 1), + "dirty bitmap covers every fine unit in native external page"); + result(!unmap(fd, 2 * native), + "canonical UNMAP drains callback-held external refs"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.external_refs && + cmd.notifications == 1, + "external-unpin callback refunds all pins"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP), + "dirty tracking stops"); + result(!domain(fd, 0, 1, 4096, 0), + "recording4K domain attaches before child MAP"); + child = fork(); + if (!child) { + unsigned char *source = mmap(NULL, 2 * native, + PROT_READ | PROT_WRITE, MAP_SHARED, + backing, 0); + int ret = source == MAP_FAILED ? -1 : + map(fd, source + 4096, length); + + _exit(ret ? 1 : 0); + } + result(child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status), + "fine-offset MAP retains real child pins after exit"); + result(!state(fd, &cmd) && !cmd.owner_users && + cmd.locked_bytes == 2 * native && + cmd.owner_locked_bytes == 2 * native && + cmd.mapped_bytes[0] == length, + "exited source mm retains exact physical-page charge and translation length"); + errno = 0; + result(rw(fd, 0, bytes, sizeof(bytes), 0) < 0 && errno == EFAULT, + "foreign process CPU access preserves source-mm authorization"); + result(!rw(fd, 0, bytes, sizeof(bytes), VFIO_TYPE1_TEST_WORKER) && + all_bytes(bytes, sizeof(bytes), 0x64), + "kernel consumer reads exited source pins"); + errno = 0; + result(domain(fd, 1, 1, 4096, 4096) < 0 && errno == ENOMEM && + !state(fd, &cmd) && !cmd.mapped_bytes[1] && + cmd.mapped_bytes[0] == length && + cmd.locked_bytes == 2 * native, + "failed replay leaves existing domain and pins intact"); + result(!domain(fd, 1, 1, 4096, 0) && !state(fd, &cmd) && + cmd.mapped_bytes[0] == length && + cmd.mapped_bytes[1] == length && + cmd.locked_bytes == 2 * native, + "retry replays cached pins without resurrecting source mm"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), + "fine DMA dirty tracking starts"); + memset(bytes, 0x9a, sizeof(bytes)); + result(!rw(fd, 4096 - 32, bytes, sizeof(bytes), + VFIO_TYPE1_TEST_WRITE | VFIO_TYPE1_TEST_WORKER) && + pread(backing, disk, sizeof(disk), 8192 - 32) == + sizeof(disk) && + all_bytes(disk, sizeof(disk), 0x9a), + "cached worker write dirties original shmem bytes"); + result(!bitmap(fd, length, &bits) && bits == 3, + "canonical bitmap returns exact two4K units touched by worker write"); + result(!unmap(fd, length), + "fine UNMAP synchronizes both recording domains"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0] && + !cmd.mapped_bytes[1], + "source-exit cleanup refunds pins and clears all translations"); + result(!domain(fd, 1, 0, 0, 0) && !domain(fd, 0, 0, 0, 0), + "both domains detach after complete cleanup"); + external_span(backing, native); + if (getauxval(AT_PAGESZ) != native) + native_owner(fd, backing, data, native); +out: + if (data != MAP_FAILED) + munmap(data, 2 * native); + if (backing >= 0) + close(backing); + if (fd >= 0) + close(fd); + printf("1..%u\n# type1 ioctl failures=%u\n", number, failures); + return !!failures; +} + +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--native-source")) { + size_t native = strtoul(argv[4], NULL, 10); + int fd = atoi(argv[2]), backing = atoi(argv[3]); + void *data; + + if (getauxval(AT_PAGESZ) != native) + return 2; + data = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, + MAP_SHARED, backing, 0); + return data == MAP_FAILED || map(fd, data, 2 * native) ? 1 : 0; + } + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + size_t expected = strtoul(argv[3], NULL, 10); + + if (getauxval(AT_PAGESZ) != expected) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) + return 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + FILE *dev = fopen("/sys/class/misc/aurora-vfio-type1-test/dev", "r"); + unsigned int major, minor; + + if (!dev || fscanf(dev, "%u:%u", &major, &minor) != 2) + return 4; + fclose(dev); + if (mknod("/dev/vfio-test", S_IFCHR | 0600, makedev(major, minor))) + return 5; + size_t native = getauxval(AT_PAGESZ), + targets[] = { native, 4096, 16384 }; + int ok = 1; + + int compat_only = getenv("VFIO_COMPAT_ONLY") != NULL; + + for (unsigned int i = 0; i < 3; i++) { + if (compat_only) + break; + size_t target = targets[i]; + pid_t child; + int status = 0; + + if (target > native || (i && target == native)) + continue; + child = fork(); + if (!child) { + char native_arg[24], target_arg[24]; + + snprintf(native_arg, sizeof(native_arg), "%zu", native); + snprintf(target_arg, sizeof(target_arg), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + target == native ? 0UL : target, 0UL, 0UL, + 0UL)) + _exit(3); + execl("/init", "/init", "--test", native_arg, + target_arg, NULL); + _exit(4); + } + ok &= child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("# actual ABI=%zuK child_status=%d\n", target / 1024, + status); + } + const char *expected_arg = getenv("VFIO_COMPAT_PAGES"); + size_t expected = expected_arg ? strtoul(expected_arg, NULL, 10) : 4096; + pid_t compat = fork(); + int compat_status = 0; + + if (!compat) { + char native_arg[24], page_arg[24]; + + snprintf(native_arg, sizeof(native_arg), "%zu", native); + snprintf(page_arg, sizeof(page_arg), "%zu", expected); + execl("/compat-vfio", "/compat-vfio", native_arg, page_arg, + NULL); + _exit(98); + } + ok &= compat > 0 && waitpid(compat, &compat_status, 0) == compat && + WIFEXITED(compat_status) && !WEXITSTATUS(compat_status); + printf("# AArch32 expected=%zuK child_status=%d\n", expected / 1024, + compat_status); + printf("VFIO TYPE1 IOCTL %s\n", ok ? "PASS" : "FAIL"); + reboot(RB_POWER_OFF); + return !ok; +} diff --git a/tools/testing/selftests/arm64/user4k/vhost-log-init.c b/tools/testing/selftests/arm64/user4k/vhost-log-init.c new file mode 100644 index 0000000000000..07d6a8969bca0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/vhost-log-init.c @@ -0,0 +1,392 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#ifndef VHOST_SET_FORK_FROM_OWNER +#define VHOST_SET_FORK_FROM_OWNER _IOW(VHOST_VIRTIO, 0x84, __u8) +#define VHOST_GET_FORK_FROM_OWNER _IOR(VHOST_VIRTIO, 0x85, __u8) +#endif +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +#define N 8 +#define ROUNDS 32 +#define PACKET_SIZE 96 +#define PROTOCOL 0x88b5 +#define DATA_GPA (51ULL * 4096) +#define TX_USED_GPA (71ULL * 4096) +#define RX_USED_GPA (70ULL * 4096) + +struct queue { + struct vring_desc desc[N]; + struct { + uint16_t flags, idx, ring[N + 1]; + } avail; + struct { + uint16_t flags, idx; + struct vring_used_elem ring[N]; + uint16_t event; + } used; + int kick, call; +}; + +static int queue_setup(int fd, struct queue *q, unsigned index) +{ + struct vhost_vring_state state = { .index = index, .num = N }; + struct vhost_vring_addr addr = { + .index = index, + .flags = 1U << VHOST_VRING_F_LOG, + .desc_user_addr = (uintptr_t)q->desc, + .avail_user_addr = (uintptr_t)&q->avail, + .used_user_addr = (uintptr_t)&q->used, + .log_guest_addr = index ? TX_USED_GPA : RX_USED_GPA, + }; + q->kick = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + q->call = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + CHECK(q->kick >= 0 && q->call >= 0); + CHECK(!ioctl(fd, VHOST_SET_VRING_NUM, &state)); + state.num = 0; + CHECK(!ioctl(fd, VHOST_SET_VRING_BASE, &state)); + CHECK(!ioctl(fd, VHOST_SET_VRING_ADDR, &addr)); + struct vhost_vring_file file = { .index = index, .fd = q->kick }; + CHECK(!ioctl(fd, VHOST_SET_VRING_KICK, &file)); + file.fd = q->call; + CHECK(!ioctl(fd, VHOST_SET_VRING_CALL, &file)); + return 0; +} +static int submit(struct queue *q, unsigned round) +{ + uint64_t one = 1; + q->avail.ring[round % N] = 0; + __atomic_store_n(&q->avail.idx, round + 1, __ATOMIC_RELEASE); + return write(q->kick, &one, sizeof(one)) == sizeof(one); +} +static int complete(struct queue *q, unsigned round) +{ + for (unsigned i = 0; i < 2000; i++) { + if (__atomic_load_n(&q->used.idx, __ATOMIC_ACQUIRE) == + round + 1) + return 1; + usleep(1000); + } + printf("queue timeout round=%u used=%u avail=%u\n", round, q->used.idx, + q->avail.idx); + return 0; +} +static int bitmap_ok(unsigned char *leaf, size_t ps, int active, int missing) +{ + for (size_t i = 0; i < ps; i++) { + unsigned char expected = i < ps - 9 ? (missing ? 0 : 0x64) : 0; + if (active && i == ps - 3) + expected = 8; /* RX buffer, GPA page 51 */ + if (active && i == ps - 1) + expected = 0xc0; /* RX/TX rings, pages 70/71 */ + if (__atomic_load_n(leaf + i, __ATOMIC_ACQUIRE) != expected) { + printf("bitmap byte %zu got=%#x expected=%#x\n", i, + leaf[i], expected); + return 0; + } + } + return 1; +} +static int filled(unsigned char *p, size_t len, unsigned char byte) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != byte) + return 0; + return 1; +} +static int exercise_case(size_t ps, unsigned shared, unsigned forked, + uint8_t mode) +{ + int backing = -1; + if (shared) { + backing = memfd_create("vhost-log", 0); + CHECK(backing >= 0 && !ftruncate(backing, 4 * ps)); + } + unsigned char *src = mmap( + (void *)0x20000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), + backing, 0); + unsigned char *dst = + mmap((void *)0x30000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, 4 * ps); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps, *bitmap = leaf + ps - 9; + memset(bitmap, 0, 9); + int barrier[2]; + pid_t child = -1; + if (forked) { + CHECK(!pipe(barrier)); + child = fork(); + CHECK(child >= 0); + if (!child) { + close(barrier[1]); + char token; + int ok = read(barrier[0], &token, 1) == 1 && + bitmap_ok(leaf, ps, shared, 0); + _exit(ok ? 0 : 8); + } + close(barrier[0]); + } + int tap = open("/dev/net/tun", O_RDWR | O_CLOEXEC); + CHECK(tap >= 0); + struct ifreq ifr = { .ifr_flags = IFF_TAP | IFF_NO_PI }; + strcpy(ifr.ifr_name, "abi%d"); + CHECK(!ioctl(tap, TUNSETIFF, &ifr)); + int raw = socket(AF_PACKET, SOCK_RAW | SOCK_CLOEXEC, htons(PROTOCOL)); + CHECK(raw >= 0); + CHECK(!ioctl(raw, SIOCGIFINDEX, &ifr)); + int ifindex = ifr.ifr_ifindex; + CHECK(!ioctl(raw, SIOCGIFFLAGS, &ifr)); + ifr.ifr_flags |= IFF_UP; + CHECK(!ioctl(raw, SIOCSIFFLAGS, &ifr)); + int fd = open("/dev/vhost-net", O_RDWR | O_CLOEXEC); + CHECK(fd >= 0); + CHECK(!ioctl(fd, VHOST_SET_FORK_FROM_OWNER, &mode)); + uint8_t gotmode = 9; + CHECK(!ioctl(fd, VHOST_GET_FORK_FROM_OWNER, &gotmode) && + gotmode == mode); + CHECK(!ioctl(fd, VHOST_SET_OWNER, NULL)); + uint64_t log_base = (uintptr_t)bitmap; + CHECK(!ioctl(fd, VHOST_SET_LOG_BASE, &log_base)); + unsigned char *data = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(data != MAP_FAILED); + struct { + struct vhost_memory mem; + struct vhost_memory_region region; + } mem = { + .mem.nregions = 1, + .region = { .guest_phys_addr = DATA_GPA, + .memory_size = 4096, + .userspace_addr = (uintptr_t)data }, + }; + CHECK(!ioctl(fd, VHOST_SET_MEM_TABLE, &mem)); + uint64_t features = (1ULL << VHOST_F_LOG_ALL) | + (1ULL << VHOST_NET_F_VIRTIO_NET_HDR); + uint64_t available; + CHECK(!ioctl(fd, VHOST_GET_FEATURES, &available) && + (available & features) == features); + CHECK(!ioctl(fd, VHOST_SET_FEATURES, &features)); + struct queue *q = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(q != MAP_FAILED && sizeof(*q) * 2 < ps); + CHECK(!queue_setup(fd, q, 0) && !queue_setup(fd, q + 1, 1)); + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { .index = index, .fd = tap }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, &backend)); + } + struct sockaddr_ll dest = { .sll_family = AF_PACKET, + .sll_protocol = htons(PROTOCOL), + .sll_ifindex = ifindex, + .sll_halen = 6 }; + unsigned char packet[PACKET_SIZE]; + memset(packet, 0x5c, sizeof(packet)); + memset(packet, 0xff, 6); + packet[6] = 2; + packet[12] = PROTOCOL >> 8; + packet[13] = PROTOCOL & 0xff; + q[0].desc[0] = (struct vring_desc){ .addr = DATA_GPA, + .len = 512, + .flags = VRING_DESC_F_WRITE }; + q[1].desc[0] = + (struct vring_desc){ .addr = DATA_GPA + 1024, + .len = sizeof(struct virtio_net_hdr) + + PACKET_SIZE }; + memcpy(data + 1024 + sizeof(struct virtio_net_hdr), packet, + PACKET_SIZE); + for (unsigned phase = 0; phase < 2; phase++) { + if (phase) { + memset(bitmap, 0, 9); + CHECK(!madvise(leaf, ps, MADV_DONTNEED)); + int pagemap = open("/proc/self/pagemap", O_RDONLY); + uint64_t entry; + CHECK(pagemap >= 0 && + pread(pagemap, &entry, sizeof(entry), + (uintptr_t)leaf / ps * sizeof(entry)) == + sizeof(entry)); + CHECK(!(entry & (1ULL << 63))); + close(pagemap); + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { + .index = index, .fd = tap + }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, + &backend)); + } + } + for (unsigned round = phase * ROUNDS; + round < (phase + 1) * ROUNDS; round++) { + packet[20] = round; + memset(data, 0xaa, 512); + CHECK(submit(q, round)); + CHECK(sendto(raw, packet, sizeof(packet), 0, + (void *)&dest, + sizeof(dest)) == sizeof(packet)); + CHECK(submit(q + 1, round)); + CHECK(complete(q, round) && complete(q + 1, round)); + CHECK(q[0].used.ring[round % N].id == 0); + if (q[0].used.ring[round % N].len != + sizeof(struct virtio_net_hdr) + PACKET_SIZE) + printf("RX length=%u expected=%zu ethertype=%02x%02x\n", + q[0].used.ring[round % N].len, + sizeof(struct virtio_net_hdr) + + PACKET_SIZE, + data[sizeof(struct virtio_net_hdr) + 12], + data[sizeof(struct virtio_net_hdr) + 13]); + CHECK(q[0].used.ring[round % N].len == + sizeof(struct virtio_net_hdr) + PACKET_SIZE); + CHECK(!memcmp(data + sizeof(struct virtio_net_hdr), + packet, PACKET_SIZE)); + CHECK(filled(data + sizeof(struct virtio_net_hdr) + + PACKET_SIZE, + 512 - sizeof(struct virtio_net_hdr) - + PACKET_SIZE, + 0xaa)); + } + /* Stop and flush workers before checking the bitmap or discarding backing. */ + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { .index = index, + .fd = -1 }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, &backend)); + } + CHECK(bitmap_ok(leaf, ps, 1, phase && !shared)); + if (forked && !phase) { + CHECK(write(barrier[1], "x", 1) == 1); + close(barrier[1]); + int status; + CHECK(waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status)); + } + } + close(fd); + close(raw); + close(tap); + for (unsigned i = 0; i < 2; i++) { + close(q[i].kick); + close(q[i].call); + } + CHECK(filled(dst, ps, 0x91) && filled(dst + 2 * ps, 2 * ps, 0x91)); + for (unsigned i = 0; i < 3; i++) + CHECK(filled(src + i * ps, ps, 0x61 + i)); + CHECK(!munmap(src, 3 * ps) && !munmap(dst, 4 * ps)); + CHECK(!munmap(data, ps) && !munmap(q, ps)); + if (backing >= 0) + close(backing); + printf("ok - %zuK vhost %s fork=%u worker=%u: %u RX/TX packets, relocated dirty bits, missing-PTE repair and neighbors\n", + ps / 1024, shared ? "shared" : "private", forked, mode, + 2 * ROUNDS); + return 0; +} +static int exercise(size_t expected) +{ + CHECK(getauxval(AT_PAGESZ) == expected); + alarm(45); + for (uint8_t mode = 0; mode < 2; mode++) + for (unsigned shared = 0; shared < 2; shared++) + for (unsigned forked = 0; forked < 2; forked++) + CHECK(!exercise_case(expected, shared, forked, + mode)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + /* Keep unsolicited IPv6 DAD packets out of the isolated TAP queues. */ + int ipv6 = + open("/proc/sys/net/ipv6/conf/default/disable_ipv6", O_WRONLY); + if (ipv6 < 0 || write(ipv6, "1", 1) != 1) { + perror("disable IPv6"); + fail = 1; + goto done; + } + close(ipv6); + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24]; + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = 0; + for (unsigned ticks = 0; c > 0 && ticks < 600; ticks++) { + pid_t got = waitpid(c, &status, WNOHANG); + if (got == c) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 599) { + kill(c, SIGKILL); + waitpid(c, &status, 0); + break; + } + usleep(100000); + } + printf("%s - ABI %zuK vhost log status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("VHOST LOG %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/wine-kernel32.def b/tools/testing/selftests/arm64/user4k/wine-kernel32.def new file mode 100644 index 0000000000000..891e48c1fb83b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/wine-kernel32.def @@ -0,0 +1,20 @@ +LIBRARY KERNEL32.dll +EXPORTS +VirtualAlloc +VirtualFree +VirtualProtect +VirtualQuery +GetSystemInfo +GetStdHandle +WriteFile +ExitProcess +GetLastError +AddVectoredExceptionHandler +RemoveVectoredExceptionHandler +CreateThread +WaitForSingleObject +GetExitCodeThread +CloseHandle +CreateFileMappingA +MapViewOfFile +UnmapViewOfFile diff --git a/tools/testing/selftests/arm64/user4k/wine-page-contract.c b/tools/testing/selftests/arm64/user4k/wine-page-contract.c new file mode 100644 index 0000000000000..149f619de6651 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/wine-page-contract.c @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding Windows x64 PE; Win32 imports, no CRT or external SDK. */ +typedef unsigned long DWORD; +typedef int BOOL; +typedef unsigned long long SIZE_T; +typedef void *HANDLE; +#define API __declspec(dllimport) +API void *VirtualAlloc(void *, SIZE_T, DWORD, DWORD); +API BOOL VirtualFree(void *, SIZE_T, DWORD); +API BOOL VirtualProtect(void *, SIZE_T, DWORD, DWORD *); +API SIZE_T VirtualQuery(const void *, void *, SIZE_T); +API void GetSystemInfo(void *); +API HANDLE GetStdHandle(DWORD); +API BOOL WriteFile(HANDLE, const void *, DWORD, DWORD *, void *); +API void ExitProcess(DWORD); +API DWORD GetLastError(void); +API void *AddVectoredExceptionHandler(DWORD, void *); +API DWORD RemoveVectoredExceptionHandler(void *); +API HANDLE CreateThread(void *, SIZE_T, DWORD (*)(void *), void *, DWORD, + DWORD *); +API DWORD WaitForSingleObject(HANDLE, DWORD); +API BOOL GetExitCodeThread(HANDLE, DWORD *); +API BOOL CloseHandle(HANDLE); +API HANDLE CreateFileMappingA(HANDLE, void *, DWORD, DWORD, DWORD, + const char *); +API void *MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); +API BOOL UnmapViewOfFile(const void *); +struct system_info { + DWORD arch, page; + void *min, *max; + SIZE_T mask; + DWORD cpus, type, granule; + unsigned short level, revision; +}; +struct memory_info { + void *base, *allocation; + DWORD allocation_protect, pad; + SIZE_T size; + DWORD state, protect, type, pad2; +}; +struct exception_record { + DWORD code, flags; + void *record, *address; + DWORD count, pad; + SIZE_T info[15]; +}; +struct exception_pointers { + struct exception_record *record; + void *context; +}; +/* Keep the freestanding declarations consistent with the Windows x64 ABI. */ +_Static_assert(sizeof(DWORD) == 4, "DWORD must be 32 bits"); +_Static_assert(sizeof(SIZE_T) == 8, "SIZE_T must be 64 bits"); +_Static_assert(sizeof(struct system_info) == 48, "SYSTEM_INFO layout"); +_Static_assert(sizeof(struct memory_info) == 48, + "MEMORY_BASIC_INFORMATION layout"); +_Static_assert(sizeof(struct exception_record) == 152, + "EXCEPTION_RECORD layout"); +static volatile unsigned char *pages; +static volatile DWORD handled; +static void say(const char *s) +{ + DWORD n = 0, w = 0; + while (s[n]) + n++; + WriteFile(GetStdHandle((DWORD)-11), s, n, &w, 0); +} +static void number(DWORD n) +{ + char buf[11], *p = buf + 10; + *p = 0; + do { + *--p = '0' + n % 10; + n /= 10; + } while (n); + say(p); +} +static void fail(DWORD stage) +{ + DWORD e = GetLastError(); + say("WINE PAGE CONTRACT FAIL stage="); + number(stage); + say(" error="); + number(e); + say("\r\n"); + ExitProcess(1); +} +#define CHECK(x, n) \ + do { \ + if (!(x)) \ + fail(n); \ + } while (0) +static long handler(struct exception_pointers *e) +{ + DWORD old; + if (e->record->code != 0xc0000005 || e->record->count < 2 || + e->record->info[0] != 1 || + e->record->info[1] != (SIZE_T)(pages + 4103)) + return 0; + if (!VirtualProtect((void *)(pages + 4096), 4096, 4, &old)) + return 0; + handled++; + return -1; +} +static DWORD thread(void *arg) +{ + volatile unsigned char *p = arg; + for (unsigned int i = 0; i < 4096; i++) + p[i] = (unsigned char)(i ^ 0x5a); + return 37; +} +void mainCRTStartup(void) +{ + struct system_info si; + struct memory_info mi; + DWORD old, status; + GetSystemInfo(&si); + CHECK(si.page == 4096 && si.granule == 65536, 1); + say("ok - Wine reports 4K pages and 64K allocation granularity\r\n"); + pages = VirtualAlloc(0, 65536, 0x2000, 1); + CHECK(pages, 2); + CHECK(VirtualAlloc((void *)pages, 12288, 0x1000, 4) == pages, 3); + for (unsigned int i = 0; i < 12288; i++) + pages[i] = (unsigned char)(i ^ (i >> 12)); + CHECK(VirtualProtect((void *)(pages + 4096), 4096, 2, &old) && old == 4, + 4); + CHECK(VirtualQuery((void *)(pages + 4096), &mi, sizeof(mi)) == + sizeof(mi) && + mi.base == pages + 4096 && mi.size == 4096 && + mi.protect == 2, + 5); + CHECK(VirtualQuery((void *)(pages + 8192), &mi, sizeof(mi)) == + sizeof(mi) && + mi.protect == 4, + 6); + void *veh = AddVectoredExceptionHandler(1, handler); + CHECK(veh, 7); + pages[4103] = 0x6d; + CHECK(handled == 1 && pages[4103] == 0x6d, 8); + CHECK(RemoveVectoredExceptionHandler(veh), 9); + say("ok - Wine independent 4K protection and handled access violation\r\n"); + CHECK(VirtualFree((void *)(pages + 4096), 4096, 0x4000), 10); + CHECK(VirtualQuery((void *)(pages + 4096), &mi, sizeof(mi)) == + sizeof(mi) && + mi.state == 0x2000, + 11); + CHECK(VirtualAlloc((void *)(pages + 4096), 4096, 0x1000, 4) == + pages + 4096, + 12); + for (unsigned int i = 0; i < 4096; i++) + CHECK(!pages[4096 + i], 13); + for (unsigned int i = 0; i < 4096; i++) + CHECK(pages[i] == (unsigned char)i && + pages[8192 + i] == (unsigned char)(i ^ 2), + 14); + say("ok - Wine 4K decommit and zero-filled recommit preserve neighbours\r\n"); + HANDLE t = CreateThread(0, 0, thread, (void *)(pages + 4096), 0, 0); + CHECK(t, 15); + CHECK(WaitForSingleObject(t, 30000) == 0 && + GetExitCodeThread(t, &status) && status == 37, + 16); + CHECK(CloseHandle(t), 17); + for (unsigned int i = 0; i < 4096; i++) + CHECK(pages[4096 + i] == (unsigned char)(i ^ 0x5a), 18); + say("ok - Wine threads share committed 4K pages\r\n"); + HANDLE file = CreateFileMappingA((HANDLE)(SIZE_T)-1, 0, 4, 0, 16384, 0); + CHECK(file, 19); + unsigned char *a = MapViewOfFile(file, 2, 0, 0, 16384), + *b = MapViewOfFile(file, 2, 0, 0, 16384); + CHECK(a && b, 20); + a[4099] = 0xa7; + CHECK(b[4099] == 0xa7, 21); + CHECK(VirtualProtect(a + 4096, 4096, 2, &old), 22); + b[4099] = 0x53; + CHECK(a[4099] == 0x53, 23); + CHECK(UnmapViewOfFile(a) && UnmapViewOfFile(b) && CloseHandle(file), + 24); + CHECK(VirtualFree((void *)pages, 0, 0x8000), 25); + say("ok - Wine shared section aliases and 4K protection\r\nWINE PAGE CONTRACT PASS\r\n"); + ExitProcess(0); +} diff --git a/tools/testing/selftests/arm64/user4k/x18-probe.S b/tools/testing/selftests/arm64/user4k/x18-probe.S new file mode 100644 index 0000000000000..565f34e8323f5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/x18-probe.S @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include + + .text + .globl x18_syscall + .type x18_syscall, %function +/* (nr, arg0, arg1, arg2, observed_x18); no libc code inside the probe. */ +x18_syscall: + hint 34 /* bti c */ + stp x18, x19, [sp, #-16]! + mov x19, x4 + movz x18, #0x3210 + movk x18, #0x7654, lsl #16 + movk x18, #0xba98, lsl #32 + movk x18, #0xfedc, lsl #48 + mov x8, x0 + mov x0, x1 + mov x1, x2 + mov x2, x3 + svc #0 + str x18, [x19] + ldp x18, x19, [sp], #16 + ret + .size x18_syscall, .-x18_syscall + .section .note.GNU-stack,"",%progbits diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 9e0dfd3a85b0e..8d0377cda51f0 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -533,6 +533,9 @@ struct vm_area_desc { /* Mutable fields. Populated with initial state. */ pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif struct file *vm_file; vma_flags_t vma_flags; pgprot_t page_prot; @@ -1300,6 +1303,9 @@ static inline void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->end = vma->vm_end; desc->pgoff = vma->vm_pgoff; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = vma->vm_subpage_offset; +#endif desc->vm_file = vma->vm_file; desc->vma_flags = vma->flags; desc->page_prot = vma->vm_page_prot; @@ -1308,6 +1314,117 @@ static inline void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->action.type = MMAP_NOTHING; } +static inline unsigned int mm_page_shift(const struct mm_struct *mm) +{ + return PAGE_SHIFT; +} + +static inline unsigned long mm_page_size(const struct mm_struct *mm) +{ + return PAGE_SIZE; +} + +static inline unsigned long mm_page_mask(const struct mm_struct *mm) +{ + return PAGE_MASK; +} + +static inline unsigned long mm_page_align(const struct mm_struct *mm, unsigned long size) +{ + return PAGE_ALIGN(size); +} + +static inline unsigned long vma_user_pages(const struct vm_area_struct *vma) +{ + return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; +} + +#define mm_account_memory(mm, policy_mm, pages) security_vm_enough_memory_mm(policy_mm, pages) +#define mm_unacct_memory(mm, pages) vm_unacct_memory(pages) + +/* File-cache indices remain native-page units even for a 4K VMA. */ +struct vm_page_offset { + pgoff_t index; + unsigned int offset; +}; + +static inline struct vm_page_offset vm_page_offset_add(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = pos.offset + (bytes & ~PAGE_MASK); + + pos.index += (bytes >> PAGE_SHIFT) + (low >> PAGE_SHIFT); + pos.offset = low & ~PAGE_MASK; + return pos; +} + +static inline struct vm_page_offset vm_page_offset_sub(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = bytes & ~PAGE_MASK; + + pos.index -= (bytes >> PAGE_SHIFT) + (low > pos.offset); + pos.offset = (pos.offset - low) & ~PAGE_MASK; + return pos; +} + +static inline bool vm_page_offset_equal(struct vm_page_offset a, struct vm_page_offset b) +{ + return a.index == b.index && a.offset == b.offset; +} + +static inline unsigned int vma_subpage_offset(const struct vm_area_struct *vma) +{ +#ifdef CONFIG_MM_SUBPAGE + return vma->vm_subpage_offset; +#else + return 0; +#endif +} + +/* address must be >= vm_start; vm_end computes the next contiguous offset. */ +static inline struct vm_page_offset vma_page_offset_at(const struct vm_area_struct *vma, + unsigned long address) +{ + unsigned long delta = address - vma->vm_start; + unsigned long low = (delta & ~PAGE_MASK) + vma_subpage_offset(vma); + + return (struct vm_page_offset) { + .index = vma->vm_pgoff + (delta >> PAGE_SHIFT) + (low >> PAGE_SHIFT), + .offset = low & ~PAGE_MASK, + }; +} + +static inline void vma_set_page_offset(struct vm_area_struct *vma, struct vm_page_offset pos) +{ + vma->vm_pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = pos.offset; +#endif +} + +/* Inverse lookup of a single byte; an offset outside the VMA is not clamped. */ +static inline unsigned long vma_address_at_offset(const struct vm_area_struct *vma, + struct vm_page_offset pos) +{ + unsigned long len = vma->vm_end - vma->vm_start; + unsigned long index, delta, last; + unsigned int offset = vma_subpage_offset(vma); + + if (!len || pos.index < vma->vm_pgoff || pos.offset >= PAGE_SIZE) + return -EFAULT; + index = pos.index - vma->vm_pgoff; + last = ((len - 1) >> PAGE_SHIFT) + + ((((len - 1) & ~PAGE_MASK) + offset) >> PAGE_SHIFT); + if (index > last) + return -EFAULT; + delta = (index << PAGE_SHIFT) + pos.offset; + if (delta < offset || delta - offset >= len) + return -EFAULT; + return vma->vm_start + delta - offset; +} + + static inline unsigned long vma_pages(const struct vm_area_struct *vma) { return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; diff --git a/tools/testing/vma/tests/merge.c b/tools/testing/vma/tests/merge.c index 03b6f9820e0ac..0b0f4ae413973 100644 --- a/tools/testing/vma/tests/merge.c +++ b/tools/testing/vma/tests/merge.c @@ -227,7 +227,7 @@ static bool test_simple_shrink(void) ASSERT_FALSE(attach_vma(&mm, vma)); - ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, 0)); + ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, (struct vm_page_offset) { 0, 0 })); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x1000); diff --git a/tools/testing/vma/tests/mmap.c b/tools/testing/vma/tests/mmap.c index c85bc000d1cb7..c87baa48ea26e 100644 --- a/tools/testing/vma/tests/mmap.c +++ b/tools/testing/vma/tests/mmap.c @@ -12,19 +12,19 @@ static bool test_mmap_region_basic(void) current->mm = &mm; /* Map at 0x300000, length 0x3000. */ - addr = __mmap_region(NULL, 0x300000, 0x3000, vma_flags, 0x300, NULL); + addr = __mmap_region(NULL, 0x300000, 0x3000, vma_flags, (struct vm_page_offset) { 0x300, 0 }, NULL); ASSERT_EQ(addr, 0x300000); /* Map at 0x250000, length 0x3000. */ - addr = __mmap_region(NULL, 0x250000, 0x3000, vma_flags, 0x250, NULL); + addr = __mmap_region(NULL, 0x250000, 0x3000, vma_flags, (struct vm_page_offset) { 0x250, 0 }, NULL); ASSERT_EQ(addr, 0x250000); /* Map at 0x303000, merging to 0x300000 of length 0x6000. */ - addr = __mmap_region(NULL, 0x303000, 0x3000, vma_flags, 0x303, NULL); + addr = __mmap_region(NULL, 0x303000, 0x3000, vma_flags, (struct vm_page_offset) { 0x303, 0 }, NULL); ASSERT_EQ(addr, 0x303000); /* Map at 0x24d000, merging to 0x250000 of length 0x6000. */ - addr = __mmap_region(NULL, 0x24d000, 0x3000, vma_flags, 0x24d, NULL); + addr = __mmap_region(NULL, 0x24d000, 0x3000, vma_flags, (struct vm_page_offset) { 0x24d, 0 }, NULL); ASSERT_EQ(addr, 0x24d000); ASSERT_EQ(mm.map_count, 2); diff --git a/tools/testing/vma/tests/vma.c b/tools/testing/vma/tests/vma.c index 754a2da063215..38a7cc6702f23 100644 --- a/tools/testing/vma/tests/vma.c +++ b/tools/testing/vma/tests/vma.c @@ -38,7 +38,7 @@ static bool test_copy_vma(void) /* Move backwards and do not merge. */ vma = alloc_and_link_vma(&mm, 0x3000, 0x5000, 3, vma_flags); - vma_new = copy_vma(&vma, 0, 0x2000, 0, &need_locks); + vma_new = copy_vma(&vma, 0, 0x2000, (struct vm_page_offset) { 0, 0 }, &need_locks); ASSERT_NE(vma_new, vma); ASSERT_EQ(vma_new->vm_start, 0); ASSERT_EQ(vma_new->vm_end, 0x2000); @@ -51,7 +51,7 @@ static bool test_copy_vma(void) vma = alloc_and_link_vma(&mm, 0, 0x2000, 0, vma_flags); vma_next = alloc_and_link_vma(&mm, 0x6000, 0x8000, 6, vma_flags); - vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, &need_locks); + vma_new = copy_vma(&vma, 0x4000, 0x2000, (struct vm_page_offset) { 4, 0 }, &need_locks); vma_assert_attached(vma_new); ASSERT_EQ(vma_new, vma_next); From 4c8e272b03dfd773cffc5b3f3d86d9f36ea6dc20 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:23 -0400 Subject: [PATCH 0581/1579] docs: describe mixed userspace page sizes and the development launcher Document native backing versus per-process mappings, architecture responsibilities, supported combinations and remaining limitations. Provide a launcher for inherited and one-shot exec preferences. Signed-off-by: Diljit Singh (cherry picked from commit 6404f6a5b060c3b8730f7fcda19d4a72d59fab1e) --- Documentation/arch/arm64/index.rst | 1 + Documentation/arch/arm64/user4k.rst | 187 ++++++++++++++++++++++++++++ Documentation/mm/index.rst | 1 + Documentation/mm/user_page_size.rst | 84 +++++++++++++ tools/aurora-userpages/.gitignore | 1 + tools/aurora-userpages/Makefile | 14 +++ tools/aurora-userpages/README.rst | 34 +++++ tools/aurora-userpages/userpages.c | 83 ++++++++++++ 8 files changed, 405 insertions(+) create mode 100644 Documentation/arch/arm64/user4k.rst create mode 100644 Documentation/mm/user_page_size.rst create mode 100644 tools/aurora-userpages/.gitignore create mode 100644 tools/aurora-userpages/Makefile create mode 100644 tools/aurora-userpages/README.rst create mode 100644 tools/aurora-userpages/userpages.c diff --git a/Documentation/arch/arm64/index.rst b/Documentation/arch/arm64/index.rst index fc65c2d61a1de..23afe611f3039 100644 --- a/Documentation/arch/arm64/index.rst +++ b/Documentation/arch/arm64/index.rst @@ -33,5 +33,6 @@ ARM64 Architecture sve tagged-address-abi tagged-pointers + user4k features diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst new file mode 100644 index 0000000000000..d483c565d853b --- /dev/null +++ b/Documentation/arch/arm64/user4k.rst @@ -0,0 +1,187 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +=========================================== +Experimental per-process page sizes on arm64 +=========================================== + +The mixed-page implementation lets an address space use a different user +translation granule from the kernel's native physical page size. The kernel +page size remains fixed for the boot. An address space's user granule remains +fixed for its lifetime; exec can create a new address space with another +supported granule. See :doc:`../../mm/user_page_size` for the generic launch +policy and architecture integration contract. + +This is an experimental ABI. The launch commands use private prctl numbers, +not assigned upstream UAPI. The implementation and tests do not establish +complete application, device or hardware qualification. + +Configuration and supported combinations +======================================== + +Enable ``CONFIG_ARM64_USER4K_EXPERIMENTAL`` with 48-bit VA and PA and a native +16K or 64K page size. The option defaults off and selects the generic +``MM_SUBPAGE`` backing and ``ARCH_HAS_USER_PAGE_SIZE`` exec policy support. +LPA2 and ``NO_PAGE_MAPCOUNT`` configurations are excluded by Kconfig. + +The exposed combinations are: + +=================== ================== ============================ +Native kernel size AArch64 default AArch64 alternatives +=================== ================== ============================ +4K 4K None in this implementation +16K 16K 4K, when supported by the CPUs +64K 64K 4K and 16K, subject to support +=================== ================== ============================ + +Capability detection uses the sanitized system-wide granule intersection. +AArch32 executables select 4K when that capability is available and retain +the native granule otherwise, regardless of launch preferences. The latter +case still enforces native mapping/protection alignment. AArch32 mmap2 file +offsets retain their 4K ABI unit in either case. + +An AArch32 executable running under an arm64 kernel is distinct from an +``arch/arm`` kernel. The latter retains its existing native ABI; this series +does not yet provide its alternative-granule backend. Exploiting extra +translation capabilities available only within an asymmetric AArch32 CPU +domain also remains unimplemented. + +Larger-than-native process granules are not exposed. Internal 64K-on-16K +fixtures exercise allocation, mappings and argument staging, but those +fixtures do not enable a public 64K process ABI on a 16K kernel. + +Selecting an executable's page size +=================================== + +Build the development launcher with:: + + make -C tools/aurora-userpages + +For example:: + + tools/aurora-userpages/userpages 4k COMMAND [ARGUMENTS...] + tools/aurora-userpages/userpages --once native COMMAND [ARGUMENTS...] + tools/aurora-userpages/userpages --status + +The ordinary selection establishes an inherited per-thread exec preference. +``--once`` overrides it for one successful exec. A failed exec retains the +pending request; changing a preference never changes existing mappings. +Fork copies the preferences and mappings. Threads sharing an mm have the +same current granule but may establish different preferences for later exec. +``native`` clears an inherited alternative preference; a one-shot native +request can override that preference without clearing it. + +The final ELF loader resolves the granule before committing the new image. +If arguments were staged with another granule, their bytes are transferred +into a new private mm. Temporary argument mappings are bounded by both the +user leaf and native page size. Logical argument-size limits continue to +use the process granule. The interpreter and executable share one mm and +therefore one final granule. ELF segment alignment alone does not opt a +program into an alternative ABI. + +``AT_PAGESZ`` reports the process granule. User virtual address alignment, +protection boundaries and mapping lengths use that granule. Native physical +allocation, PFN arithmetic and file-cache indices continue to use the kernel +page size. Application execution does not require any of the KUnit options +or the ``arm64.user4k_test=1`` diagnostic boot parameter. + +Memory and table ownership +========================== + +Keep the following units separate: + +* ``PAGE_SIZE`` and physical PFNs describe native physical pages. +* ``mm_page_size(mm)`` describes the user mapping/protection unit. +* A folio owns one or more native pages and carries their physical charge. + +Smaller anonymous mappings retain explicit subpage ownership. A reference to +the containing folio alone does not retain an individual reusable slot. +The slot's byte offset must survive fault, COW, reverse mapping, migration, +swap and pinning operations. Newly exposed bytes are zeroed, and changing +one mapping's protection must not widen that change to adjacent slots. + +File offsets use a native cache index plus an explicit byte offset. VMA +split/merge, truncation and reverse mapping must preserve both components. +Physical allocation charges count the actual backing once, independently +of how many user mappings expose it. + +Lower page tables can share a native allocation. Table tokens preserve the +fragment address, while locking and lifetime use the native table owner. +Unissued cached fragments retain owner references. Issued fragments are not +reused within a live owner. Deferred reclamation must preserve the fragment +token through TLB invalidation and the required RCU grace period, including +allocation-failure fallback paths. + +Translation context and invalidation +=================================== + +The arm64 backend supplies per-mm page-table geometry. TTBR1 retains native +kernel tables, while TTBR0 uses the selected user geometry. A granule change +installs and synchronizes a reserved TTBR0 before changing TG0, then +synchronizes the new geometry before installing the user root. Software PAN +also selects the user TG0 even when restoration of TTBR0 is deferred until +uaccess or return to userspace. + +The existing mm ASID is retained. Selecting another granule does not add +an unconditional context-switch TLB flush. User invalidation operands and +range strides use the target mm's geometry; kernel and stage-2 operations +retain their own native geometry. Transition/idmap paths restore native +TG0 and then the active mm's user geometry when returning. + +The current configuration retains the existing 48-bit user address size. +Independent TG0/TG1 fields do not make all other translation controls +independent; the LPA2 exclusion must not be removed merely by changing TG0. + +Sharing, pinning and interface limits +==================================== + +Mixed-granule sharing is permitted. Shared mappings refer to one coherent +backing object, with permissions belonging to each virtual mapping. Shared +futex identity includes the object and byte offset. Private copying is not +an implementation of shared-memory semantics. + +Some interfaces have additional layout or ownership requirements. For +example, a perf ring's first mapper fixes its userspace granule; a mapper +using another granule is rejected. This does not prohibit mixed-granule +memfd or System V shared-memory aliases. + +A legacy page-array pin cannot express every smaller-granule byte range. +Converted consumers use typed fragment references carrying backing, offset, +length and ownership. Unconverted paths must not silently round a pin or DMA +mapping into neighboring slots. Mock IOMMU/VFIO tests validate the recorded +translations and ownership paths, not physical-device DMA on arbitrary +hardware. + +Alternative-granule THP/KSM and hugetlb support, remaining legacy pin +consumers, larger-than-native process activation and broader device +integration still need work. Userfaultfd feature bits and per-range ioctl +masks must be consulted; success for a shmem operation does not imply +support for the corresponding hugetlb operation. Sparse individual fills +can retain an entire native folio; packing behavior is not a throughput +or memory-efficiency guarantee for every workload. + +Test coverage +============= + +The arm64 selftests are in ``tools/testing/selftests/arm64/user4k``. The +``page-contract`` executable checks the exposed userspace contract. Extended +fixtures cover exec policy, AArch32, shared mappings, accounting, reclaim, +I/O and device-facing interfaces. Several are disposable-guest PID1 programs +with companion executables or storage requirements; they are not ordinary +host test commands. + +``CONFIG_ARM64_USER4K_KUNIT_TEST`` and the data/TLB suites exercise internal +geometry and ownership. Some fixtures require ``arm64.user4k_test=1`` and +some hardware translation tests are disabled under KASAN. Check individual +KTAP results and skips rather than treating boot success as a suite pass. +Private IOMMUFD/VFIO test-device options are separate and should remain off +in application kernels. + +The NUMA COW batching case runs only at boot, before userspace can change +its policy prerequisite. Its results remain visible in KUnit debugfs, but +it has no runtime rerun control. Other suites retain their rerun controls. + +Validation must distinguish native/default operation, each exposed +alternative, and AArch32 fallback with 4K support unavailable. Native ARM32 +short-descriptor and LPAE controls check that generic changes preserve the +existing ABI; they do not validate a mixed-page ARM32 backend. Internal +larger-leaf tests likewise do not qualify a public process mode. diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst index 7aa2a88869083..fb6128d42aa84 100644 --- a/Documentation/mm/index.rst +++ b/Documentation/mm/index.rst @@ -13,6 +13,7 @@ see the :doc:`admin guide <../admin-guide/mm/index>`. physical_memory page_tables process_addrs + user_page_size bootmem page_allocation vmalloc diff --git a/Documentation/mm/user_page_size.rst b/Documentation/mm/user_page_size.rst new file mode 100644 index 0000000000000..62ef6ae39e0ac --- /dev/null +++ b/Documentation/mm/user_page_size.rst @@ -0,0 +1,84 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +============================== +Per-process user mapping sizes +============================== + +The experimental mixed-page implementation separates the user mapping +size of an address space from the kernel's native allocation size. +``PAGE_SIZE`` still describes native physical pages. ``mm_page_size(mm)`` +describes the minimum user mapping and protection unit of that address +space. A process retains one mapping size for the lifetime of its ``mm``. +Exec may create a new ``mm`` with a different size; changing a launch +preference never changes an existing mapping. + +The generic interfaces described here are internal development interfaces. +The experimental launch commands still use private prctl numbers; these +are not assigned upstream UAPI. The arm64 backend's supported modes and +current restrictions are documented in +:doc:`../arch/arm64/user4k`. + +Exec policy and inheritance +=========================== + +``ARCH_HAS_USER_PAGE_SIZE`` enables two per-thread policy fields in +``task_struct``: + +``exec_page_shift`` + A one-shot request for the next exec. Zero means no one-shot request. + +``default_exec_page_shift`` + An inherited preference used when no one-shot request overrides it. + Zero means the architecture's ordinary executable policy. + +Fork and thread creation copy both fields. Subsequent changes affect only +the calling thread, including when it shares an ``mm`` with other threads. +An exec by a non-leader uses the execing thread's policy. The fields do not +belong in ``mm_struct``: threads sharing one address space may launch new +images with different preferences. + +Generic prctl handling validates reserved arguments before calling the +architecture validator. A rejected request leaves both fields unchanged. +Setting a native one-shot request while an inherited alternative is active +stores ``PAGE_SHIFT`` explicitly, so the next exec can override that +alternative. Clearing the inherited preference does not consume a pending +one-shot request. + +Exec stages arguments in a private ``mm``. Its initial allocation receives +the one-shot shift, if any; the final executable loader resolves the +inherited preference and any executable-specific policy. An ELF interpreter +must be loaded using the same final mapping size as its executable. +Failure before committing the new image retains both preferences. +``setup_new_exec()`` consumes the one-shot field after the new image is +committed, while retaining the inherited preference. + +Architecture responsibilities +============================= + +An architecture selecting ``ARCH_HAS_USER_PAGE_SIZE`` must implement: + +* ``arch_exec_page_size_shift(size)``: return zero for a native request, a + validated alternative shift, or a negative errno. Validation includes + the supported hardware and kernel configuration, not just whether the + requested byte size is a power of two. +* ``arch_mm_init_exec(mm, shift)``: initialize the private address space's + geometry before allocating its page-table root. Zero denotes native + geometry. The hook receives the request explicitly and does not read + a separate architecture thread-policy field. +* Binary-loader integration that resolves the final executable policy + before exposing the address space. ``bprm_set_page_shift()`` transfers + staged argument bytes when the final choice changes the initial granule. +* Page-table allocation, walking, invalidation, context switching and MM + operations for every size the architecture exposes. + +The architecture retains control over executable compatibility policy. +For example, the arm64 AArch32 loader prefers 4K where the backend supports +it and retains the native ABI when that capability is absent. This is +separate from running an AArch32 kernel: ``arch/arm`` needs its own backend +integration before it can advertise alternative mapping sizes. The generic +exec policy alone does not provide that backend or expose new sizes there. + +Architectures without this capability retain native mappings and reject +the experimental launch-policy commands. ``ARCH_HAS_MM_PGTABLE`` describes +per-mm page-table accessors; it does not by itself promise an executable +userspace ABI. diff --git a/tools/aurora-userpages/.gitignore b/tools/aurora-userpages/.gitignore new file mode 100644 index 0000000000000..a27b2bf8b140d --- /dev/null +++ b/tools/aurora-userpages/.gitignore @@ -0,0 +1 @@ +/userpages diff --git a/tools/aurora-userpages/Makefile b/tools/aurora-userpages/Makefile new file mode 100644 index 0000000000000..c6d42e2d15efa --- /dev/null +++ b/tools/aurora-userpages/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +CC ?= cc +CFLAGS ?= -O2 -g +CFLAGS += -Wall -Wextra +LDFLAGS += -Wl,-z,max-page-size=65536 + +.PHONY: all clean +all: userpages + +userpages: userpages.c ../../include/uapi/linux/prctl.h + $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $< $(LDLIBS) -o $@ + +clean: + rm -f userpages diff --git a/tools/aurora-userpages/README.rst b/tools/aurora-userpages/README.rst new file mode 100644 index 0000000000000..64f4fcb0dd12a --- /dev/null +++ b/tools/aurora-userpages/README.rst @@ -0,0 +1,34 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Per-process userspace page-size launcher +======================================= + +Build with ``make`` in this directory. The running arm64 kernel must provide +Aurora's experimental per-process granule API. + +Examples:: + + userpages 4k FEXInterpreter ./x86-program + userpages 4k python3 script.py + userpages native program + userpages --status + +The choice becomes the calling thread's inherited exec preference: it applies +to the launched program and subsequent execs in its descendants. Fork retains +the address space and both exec preferences. Other threads and processes are +unchanged. ``native`` clears the inherited alternative preference. Ordinary +processes that never opt in continue to use the kernel's native default. + +``--once`` changes only the next successful exec; an existing inherited +preference remains available for later execs. For example, +``userpages --once native program`` runs one program with native pages and +retains any inherited 4K preference for its subsequent execs. Failed exec does +not consume the pending request. No selection changes a live address space. +AArch32 programs always use 4K when supported, otherwise native pages, +regardless of either preference. + +The kernel checks supported sizes and CPU capabilities. The current branch +supports 4K userspace on 16K/64K kernels and 16K userspace on 64K kernels. +64K userspace on a 16K kernel is not enabled. This launcher uses the private +PR_AURORA_{SET,GET}_{EXEC,DEFAULT}_PAGE_SIZE API in this tree; +it is not an upstream Linux ABI. diff --git a/tools/aurora-userpages/userpages.c b/tools/aurora-userpages/userpages.c new file mode 100644 index 0000000000000..02d9e4357224d --- /dev/null +++ b/tools/aurora-userpages/userpages.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include "../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include + +static int usage(const char *name, int status) +{ + fprintf(status ? stderr : stdout, + "Usage: %s [--once] {native|4k|16k|64k} COMMAND [ARG ...]\n" + " %s --status\n" + "Select the userspace page size for this program and its descendants.\n", + name, name); + return status; +} + +int main(int argc, char **argv) +{ + unsigned long size; + int once = argc > 1 && !strcmp(argv[1], "--once"); + int first = once ? 2 : 1; + + if (argc == 2 && !strcmp(argv[1], "--help")) + return usage(argv[0], 0); + if (argc == 2 && !strcmp(argv[1], "--status")) { + int pending = prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); + int error = errno; + + printf("Current process page size: %lu bytes\n", getauxval(AT_PAGESZ)); + if (pending < 0) { + fprintf(stderr, "Page-size selection unavailable: %s\n", strerror(error)); + return 1; + } + int inherited = prctl(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); + + if (inherited < 0) { + perror("Cannot read inherited page-size preference"); + return 1; + } + if (inherited) + printf("Inherited exec preference: %d bytes\n", inherited); + else + puts("Inherited exec preference: native default"); + if (pending) + printf("Next exec page size: %d bytes\n", pending); + else + puts("Next exec page size: inherited preference"); + return 0; + } + if (argc < first + 2) + return usage(argv[0], 2); + if (!strcmp(argv[first], "native")) + size = 0; + else if (!strcmp(argv[first], "4k")) + size = 4096; + else if (!strcmp(argv[first], "16k")) + size = 16384; + else if (!strcmp(argv[first], "64k")) + size = 65536; + else + return usage(argv[0], 2); + + if (!once && prctl(PR_AURORA_SET_DEFAULT_PAGE_SIZE, size, 0UL, 0UL, 0UL)) { + fprintf(stderr, "Cannot set inherited %s pages: %s\n", + argv[first], strerror(errno)); + return 1; + } + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL)) { + fprintf(stderr, "Cannot select %s userspace pages: %s\n", + argv[first], strerror(errno)); + return 1; + } + execvp(argv[first + 1], &argv[first + 1]); + int error = errno; + + fprintf(stderr, "Cannot execute %s: %s\n", argv[first + 1], strerror(error)); + return error == ENOENT ? 127 : 126; +} From 7053bc1df5aa78fa0cdccc6dd444c437d9214f11 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:49:35 +1100 Subject: [PATCH 0582/1579] mm: defer linear subpage anon-vma destruction from atomic context Linear subpage owners retain an anon-vma root even without an rmap sidecar. Their last slot reference can be released by skb completion in softirq after the originating mm has exited. Route both kinds of root through the existing unbound cleanup work so the final anon-vma put and its sleeping synchronization run in process context. Preserve the physical unbinding and folio reference ordering. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 19594eb23573bb002d8f62087ecc1bd3cb38eab7) --- mm/user-subpage.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 133d75241485a..319d792ab5e9a 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -328,8 +328,8 @@ static void owner_free(struct mm_subpage_owner *owner, bool put_folio) /* Last slot reference, or wholly unissued supply. */ folio_put(owner->folio); } - if (owner->rmap) { - /* Historical roots can have no live VMA; their final put may sleep. */ + if (owner->anon_root || owner->rmap) { + /* Any final anon-vma put can sleep, including a linear owner's. */ INIT_WORK(&owner->free_work, owner_free_work); queue_work(system_unbound_wq, &owner->free_work); } else { From 904945425463c0450cb19582afeb4b935175ae51 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 2 Oct 2026 23:55:09 -0400 Subject: [PATCH 0583/1579] drm/asahi: use const float bit conversion supported by Rust minimum The kernel requires Rust 1.85, which supports const f32::to_bits. Replace the equivalent transmute and its obsolete lint allowances, including the unnecessary_transmutes lint unknown to Rust 1.85. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 201afd84ae50008095068d028af22ecf4c1356c5) --- drivers/gpu/drm/asahi/float.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index d58a3d284da12..46c05699d7722 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -54,11 +54,7 @@ impl F32 { // This must ONLY be used in const context. Use the `f32!{}` macro to do it safely. #[doc(hidden)] pub(crate) const fn from_f32(v: f32) -> F32 { - // Replace with to_bits() after kernel Rust minreq is >= 1.83.0 - #[allow(clippy::transmute_float_to_int)] - #[allow(unnecessary_transmutes)] - // SAFETY: Transmuting f32 to u32 is always safe - F32(unsafe { core::mem::transmute::(v) }) + F32(v.to_bits()) } // Convert an F32 into a `f32` value From 94b86f96bd9ef2275c99ddf5238374e615b2f510 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:47:26 -0400 Subject: [PATCH 0584/1579] rust: drm: sched: Fix job accounting when cancelling jobs drm_sched_fini() passes each job left on the pending list to cancel_job and frees it right after. Such a job is in one of three states: its hardware fence callback is still armed, the callback has already completed the job, or drm_sched_stop() detached the job from its hardware fence, returning its credits but not its score. cancel_job_cb() treated every job as unfinished and signalled its finished fence with -ECANCELED. For a completed job, that sets an error on a signalled fence. For a job whose callback is still armed, the callback stays armed on a job that is about to be freed, and the job's credits and score are never returned. For a detached job, the score is never returned. drm_sched_fini() stops the scheduler workers, but not the hardware fence, so remove the callback first. Fence callbacks run under the fence lock, so dma_fence_remove_callback() either disarms the callback or waits for it to complete. Then, unless the callback completed the job, return the job's score and any credits drm_sched_stop() has not already returned, as drm_sched_job_done() would have, and signal the finished fence with -ECANCELED. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 12f0a07a61666305a1efe1a1f4c4c88a013abafb) --- rust/kernel/drm/sched.rs | 42 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index e2f5cd96014f9..ccf5dfa307119 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -112,13 +112,49 @@ unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sched_job) { // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + // SAFETY: A job on the pending list has a valid scheduler fence. + let s_fence = unsafe { (*sched_job).s_fence }; + // SAFETY: `s_fence` is valid and holds a reference to its parent, if any. + let parent = unsafe { (*s_fence).parent }; + // SAFETY: `s_fence` is valid. + let finished = unsafe { addr_of_mut!((*s_fence).finished) }; + + // drm_sched_fini() has stopped the scheduler workers, but the hardware fence can still + // signal concurrently. Fence callbacks run under the fence lock, so removing ours either + // disarms it or waits until it has completed the job. + let callback_removed = !parent.is_null() + // SAFETY: run_job() armed `cb` on `parent` before the submission worker was stopped. + && unsafe { bindings::dma_fence_remove_callback(parent, addr_of_mut!((*sched_job).cb)) }; + + // A job without a parent either completed when run_job() returned, or had its parent + // detached by drm_sched_stop(), which also returned its credits. + let complete_here = callback_removed + // SAFETY: `finished` is a valid, initialized fence. + || (parent.is_null() && unsafe { bindings::dma_fence_get_status(finished) } == 0); // SAFETY: All of our jobs are Job. T::cancel(unsafe { &mut *p }); - let fence = unsafe { Fence::get_raw(&mut (*(*sched_job).s_fence).finished) }; - fence.set_error(ECANCELED); - let _ = fence.signal(); + if complete_here { + // Account for the job exactly as drm_sched_job_done() would have. + // + // SAFETY: The job belongs to the scheduler being torn down, which is still valid. + unsafe { + let sched = (*s_fence).sched; + if callback_removed { + bindings::atomic_sub( + (*sched_job).credits as i32, + addr_of_mut!((*sched).credit_count), + ); + } + bindings::atomic_dec((*sched).score); + } + + // SAFETY: `finished` is a valid fence. + let fence = unsafe { Fence::get_raw(finished) }; + fence.set_error(ECANCELED); + fence.signal(); + } } /// A DRM scheduler job. From 15a7eae7964f94a16fd273fc3b25a22ad2c27bab Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:48:27 -0400 Subject: [PATCH 0585/1579] rust: drm: sched: Keep the scheduler alive while jobs reference it drm_sched_entity_destroy() does not necessarily free the jobs still queued on the entity: drm_sched_entity_kill() hands each of them to a work item on the system workqueue, possibly only once the job's last dependency has signalled, and that work calls job->sched->ops->free_job(). The scheduler is embedded in SchedulerInner, which is freed as soon as the last Scheduler or Entity handle is dropped, so dropping a scheduler right after its last entity can leave that work dereferencing freed memory. Let every job hold a reference to the scheduler allocation. Finalizing the scheduler must not wait for those references, since drm_sched_fini() is what cancels and frees the pending jobs, so move it to a separate SchedulerOwner that only the Scheduler and Entity handles reference. The owner is allocated before drm_sched_init(), so that nothing can fail once the scheduler is initialized. The scheduler is now only reachable through shared references while C code keeps mutating it, so store it in an Opaque. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f8102be460b66b12af865d6250e73ffc6f99c73a) --- rust/kernel/drm/sched.rs | 66 ++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index ccf5dfa307119..b44a0afae9480 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -11,6 +11,7 @@ use crate::{ prelude::*, sync::{Arc, UniqueArc}, time::{self, msecs_to_jiffies}, + types::Opaque, }; use core::marker::PhantomData; use core::mem::MaybeUninit; @@ -161,6 +162,10 @@ unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sch pub struct Job { job: bindings::drm_sched_job, inner: T, + // Destroying an entity can leave its queued jobs to deferred work that still dereferences + // `job->sched` after the scheduler has been finalized. Keep the scheduler allocation alive + // for as long as any job references it; this does not delay finalization. + _scheduler: Arc>, } impl Deref for Job { @@ -274,14 +279,15 @@ impl<'a> JobFences<'a> { struct EntityInner { entity: bindings::drm_sched_entity, // TODO: Allow users to share guilty flag between entities - sched: Arc>, + sched: Arc>, guilty: bindings::atomic_t, _p: PhantomData, } impl Drop for EntityInner { fn drop(&mut self) { - // SAFETY: The EntityInner is initialized. This will cancel/free all jobs. + // SAFETY: The EntityInner is initialized. This will cancel/free all jobs, possibly from + // deferred work. Each job keeps the scheduler allocation alive until it is freed. unsafe { bindings::drm_sched_entity_destroy(&mut self.entity) }; } } @@ -300,7 +306,7 @@ impl Entity { let mut entity: KBox>> = KBox::new_uninit(GFP_KERNEL | __GFP_ZERO)?; - let mut sched_ptr = &sched.0.sched as *const _ as *mut _; + let mut sched_ptr = sched.0.allocation.sched.get(); // SAFETY: The Box is allocated above and valid. unsafe { @@ -344,33 +350,48 @@ impl Entity { // SAFETY: The Box pointer is valid, and this initializes the inner member. unsafe { addr_of_mut!((*job.as_mut_ptr()).inner).write(inner) }; + // SAFETY: The Box pointer is valid, and this initializes the scheduler reference. + unsafe { + addr_of_mut!((*job.as_mut_ptr())._scheduler).write(self.0.sched.allocation.clone()) + }; + // SAFETY: All fields of the Job are now initialized. Ok(PendingJob(unsafe { job.assume_init() }, PhantomData)) } } -/// DRM scheduler inner data +/// DRM scheduler allocation, kept alive by the scheduler owner and by every job. pub struct SchedulerInner { - sched: bindings::drm_gpu_scheduler, + sched: Opaque, _p: PhantomData, } -impl Drop for SchedulerInner { +// SAFETY: TODO +unsafe impl Sync for SchedulerInner {} +// SAFETY: TODO +unsafe impl Send for SchedulerInner {} + +/// Owner of an initialized DRM scheduler, shared by the scheduler handle and its entities. +/// +/// Jobs only reference the allocation, so that pending jobs cannot keep the scheduler from +/// being finalized. +struct SchedulerOwner { + allocation: Arc>, +} + +impl Drop for SchedulerOwner { fn drop(&mut self) { + let sched = self.allocation.sched.get(); + // SAFETY: The scheduler is valid. This assumes drm_sched_fini() will take care of // freeing all in-progress jobs. - unsafe { bindings::drm_sched_stop(&mut self.sched, core::ptr::null_mut()) }; - unsafe { bindings::drm_sched_fini(&mut self.sched) }; + unsafe { bindings::drm_sched_stop(sched, core::ptr::null_mut()) }; + unsafe { bindings::drm_sched_fini(sched) }; } } -// SAFETY: TODO -unsafe impl Sync for SchedulerInner {} -// SAFETY: TODO -unsafe impl Send for SchedulerInner {} - /// A DRM Scheduler -pub struct Scheduler(Arc>); +pub struct Scheduler(Arc>); impl Scheduler { const OPS: bindings::drm_sched_backend_ops = bindings::drm_sched_backend_ops { @@ -392,11 +413,16 @@ impl Scheduler { ) -> Result> { let mut sched: UniqueArc>> = UniqueArc::new_uninit(GFP_KERNEL)?; + // Allocate the owner up front: nothing may fail once the scheduler is initialized. + let owner: UniqueArc>> = UniqueArc::new_uninit(GFP_KERNEL)?; + + // SAFETY: `sched` was just allocated and is valid for writes. + let sched_ptr = Opaque::cast_into(unsafe { addr_of!((*sched.as_mut_ptr()).sched) }); // SAFETY: zero sched->sched_rq as drm_sched_init() uses it to exit early withoput initialisation // TODO: allocate sched zzeroed instead unsafe { - (*sched.as_mut_ptr()).sched.sched_rq = core::ptr::null_mut(); + (*sched_ptr).sched_rq = core::ptr::null_mut(); }; let init_ops = bindings::drm_sched_init_args { @@ -414,14 +440,10 @@ impl Scheduler { // SAFETY: The drm_sched pointer is valid and pinned as it was just allocated above. // `device` is valid by its type invarants - to_result(unsafe { - bindings::drm_sched_init( - addr_of_mut!((*sched.as_mut_ptr()).sched), - addr_of!(init_ops), - ) - })?; + to_result(unsafe { bindings::drm_sched_init(sched_ptr, addr_of!(init_ops)) })?; // SAFETY: All fields of SchedulerInner are now initialized. - Ok(Scheduler(unsafe { sched.assume_init() }.into())) + let allocation = unsafe { sched.assume_init() }.into(); + Ok(Scheduler(owner.write(SchedulerOwner { allocation }).into())) } } From 7d650853ea8e5de3f512a36b69f8653014de83b4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:49:13 -0400 Subject: [PATCH 0586/1579] rust: drm: sched: Do not stop the scheduler before tearing it down drm_sched_stop() belongs to the reset recovery protocol, and its documentation says not to call it before drm_sched_fini(). At teardown it detaches every in-flight job from its hardware fence and frees the jobs that have already completed. That is redundant: drm_sched_fini() stops the scheduler workers itself and hands every remaining job to cancel_job, which now handles jobs with an armed, completed or detached hardware fence callback. Call drm_sched_fini() alone, so that cancel_job sees each job's actual state and accounts for it exactly once. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f1f985d576810c7183ea9bcd1b701cb284927ac6) --- rust/kernel/drm/sched.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index b44a0afae9480..5fcad97f5727d 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -381,12 +381,9 @@ struct SchedulerOwner { impl Drop for SchedulerOwner { fn drop(&mut self) { - let sched = self.allocation.sched.get(); - - // SAFETY: The scheduler is valid. This assumes drm_sched_fini() will take care of - // freeing all in-progress jobs. - unsafe { bindings::drm_sched_stop(sched, core::ptr::null_mut()) }; - unsafe { bindings::drm_sched_fini(sched) }; + // SAFETY: The scheduler is valid. drm_sched_fini() stops the scheduler workers, then + // cancels and frees all remaining jobs through `cancel_job_cb()`. + unsafe { bindings::drm_sched_fini(self.allocation.sched.get()) }; } } From 9171e855e6d58a12059e265a7e3907f44ffa70a9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:22:17 -0400 Subject: [PATCH 0587/1579] rust: drm: sched: let drivers handle timeouts without a reset Drivers that recover individual jobs without the scheduler reset protocol need to keep unfinished jobs pending while retaining their original fences. Expose DRM_GPU_SCHED_STAT_NO_HANG as Status::NoHang: the scheduler reinserts the job, rearms its timeout, and frees it when completion is observed. Add Job::is_finished() to detect completion between timeout expiry and its handler. The finished fence is initialized only when armed, so an unarmed job reports false. Let a driver also reconsider a timeout before entering recovery, for work whose execution window has just been renewed. The false_timeout callback defaults to false and must not stop or restart the scheduler. Returning true follows the same NoHang reinsertion path. Existing drivers retain their timeout behavior unless they opt into the new callback or return Status::NoHang. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 14d7b3c379da4c764581d4c34aff495d7346af72) --- rust/kernel/drm/sched.rs | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index 5fcad97f5727d..add84da67356c 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -25,6 +25,11 @@ pub enum Status { Nominal = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_RESET, /// Device is no longer available NoDevice = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_ENODEV, + /// The job is not hung, or the driver dealt with the timeout without a scheduler reset. + /// + /// The job goes back to the pending list and the timeout is rearmed. The job is freed once + /// it has finished, as usual. + NoHang = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG, } /// Scheduler priorities @@ -42,6 +47,13 @@ pub enum Priority { /// Trait to be implemented by driver job objects. pub trait JobImpl: Sized { + /// Reconsider a timeout before the backend starts recovery. Returning true + /// reinserts this job and rearms its timeout without changing its fences. + /// The callback must not stop or restart the scheduler. + fn false_timeout(_job: &mut Job) -> bool { + false + } + /// Called when the scheduler is considering scheduling this job next, to get another Fence /// for this job to block on. Once it returns None, run() may be called. fn prepare(_job: &mut Job) -> Option { @@ -55,6 +67,9 @@ pub trait JobImpl: Sized { /// Called when a job has taken too long to execute, to trigger GPU recovery. /// + /// A driver that handles timeouts without a scheduler reset returns [`Status::NoHang`]. + /// [`Job::is_finished`] tells whether the job completed after the timeout fired. + /// /// This method is called in a workqueue context. fn timed_out(job: &mut Job) -> Status; @@ -97,7 +112,12 @@ unsafe extern "C" fn timedout_job_cb( // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: All of our jobs are Job. + // SAFETY: The timeout worker retains this detached Job until this + // callback returns, including when its parent fence signals concurrently. + if T::false_timeout(unsafe { &mut *p }) { + return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; + } + // SAFETY: The same retained job remains valid after the first callback. T::timed_out(unsafe { &mut *p }) as bindings::drm_gpu_sched_stat } @@ -168,6 +188,20 @@ pub struct Job { _scheduler: Arc>, } +impl Job { + /// Returns whether the job has finished, successfully or not. + pub fn is_finished(&self) -> bool { + // The finished fence is initialized when the job is armed, which also assigns its + // scheduler. + if self.job.sched.is_null() { + return false; + } + + // SAFETY: The job is armed, so its scheduler fence is valid and initialized. + unsafe { bindings::dma_fence_get_status(addr_of_mut!((*self.job.s_fence).finished)) != 0 } + } +} + impl Deref for Job { type Target = T; From f82e88f4049ed6ea278717be6d922f8e347aab13 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:52:57 -0400 Subject: [PATCH 0588/1579] soc: apple: rtkit: Support co-processors that place their own buffers Some co-processors place their shared buffers themselves, such as the GPU co-processors of the T8140 (A18 Pro). Their buffer requests carry the IOVA of memory the co-processor already uses, and the request must be acknowledged by sending it back unchanged, whereas RTKit sends nothing for such buffers today. These co-processors also have their crashlog endpoint started while the endpoint map is handled, before the map is acknowledged, instead of together with the other system endpoints after the last map message. Add a flags field to struct apple_rtkit_ops, with an APPLE_RTKIT_COPROC_PLACES_BUFFERS flag selecting this behavior. The buffer is still set up through shmem_setup, which lets the driver provide the memory behind the co-processor's IOVA, for instance to read crash logs from it. Without the flag, RTKit sends exactly the same messages as before. Expose the flag to Rust drivers as Operations::COPROC_PLACES_BUFFERS. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 928641d26be3083dc06c0de072ce99537fa059b7) --- drivers/soc/apple/rtkit.c | 20 ++++++++++++++++++++ include/linux/soc/apple/rtkit.h | 12 ++++++++++++ rust/bindings/bindings_helper.h | 1 + rust/kernel/soc/apple/rtkit.rs | 12 ++++++++++++ 4 files changed, 45 insertions(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 0797bfbd3a024..a2630df403f15 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -100,6 +100,11 @@ bool apple_rtkit_is_crashed(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_is_crashed); +static bool apple_rtkit_coproc_places_buffers(struct apple_rtkit *rtk) +{ + return rtk->ops->flags & APPLE_RTKIT_COPROC_PLACES_BUFFERS; +} + static int apple_rtkit_management_send(struct apple_rtkit *rtk, u8 type, u64 msg) { @@ -169,6 +174,14 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) set_bit(ep, rtk->endpoints); } + /* + * For co-processors that place their own buffers, the crashlog + * endpoint is started before the map is acknowledged. + */ + if (apple_rtkit_coproc_places_buffers(rtk) && base == 0 && + test_bit(APPLE_RTKIT_EP_CRASHLOG, &bitmap)) + apple_rtkit_start_ep(rtk, APPLE_RTKIT_EP_CRASHLOG); + reply = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, base); if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) reply |= APPLE_RTKIT_MGMT_EPMAP_LAST; @@ -193,6 +206,10 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) case APPLE_RTKIT_EP_IOREPORT: case APPLE_RTKIT_EP_OSLOG: case APPLE_RTKIT_EP_TRACEKIT: + /* already started while handling the map */ + if (ep == APPLE_RTKIT_EP_CRASHLOG && + apple_rtkit_coproc_places_buffers(rtk)) + break; dev_dbg(rtk->dev, "RTKit: Starting system endpoint 0x%02x\n", ep); apple_rtkit_start_ep(rtk, ep); @@ -338,6 +355,9 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, buffer->iova); } apple_rtkit_send_message(rtk, ep, reply, NULL, false); + } else if (apple_rtkit_coproc_places_buffers(rtk)) { + /* acknowledge the co-processor's own buffer with its request */ + apple_rtkit_send_message(rtk, ep, msg, NULL, false); } return 0; diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index d55551a975b54..e4b38a8f6e1ba 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -10,6 +10,7 @@ #ifndef _LINUX_APPLE_RTKIT_H_ #define _LINUX_APPLE_RTKIT_H_ +#include #include #include #include @@ -39,9 +40,19 @@ struct apple_rtkit_shmem { void *private; }; +/* + * The co-processor places its shared buffers itself. A buffer request that + * carries an IOVA is set up with shmem_setup and acknowledged by sending the + * request back unchanged. The crashlog endpoint is started while the endpoint + * map is handled, before the map is acknowledged. + */ +#define APPLE_RTKIT_COPROC_PLACES_BUFFERS BIT(0) + /* * Struct to represent implementation-specific RTKit operations. * + * @flags: Combination of APPLE_RTKIT_* flags describing the + * co-processor. * @crashed: Called when the co-processor has crashed. Runs in process * context. * @recv_message: Function called when a message from RTKit is received @@ -63,6 +74,7 @@ struct apple_rtkit_shmem { * context. */ struct apple_rtkit_ops { + unsigned int flags; void (*crashed)(void *cookie, const void *crashlog, size_t crashlog_size); void (*recv_message)(void *cookie, u8 endpoint, u64 message); bool (*recv_message_early)(void *cookie, u8 endpoint, u64 message); diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 6737592d7dbcc..3f8ebdaf0c35e 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -142,6 +142,7 @@ const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; const uint32_t RUST_CONST_HELPER_DRM_EXEC_INTERRUPTIBLE_WAIT = DRM_EXEC_INTERRUPTIBLE_WAIT; const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; +const u32 RUST_CONST_HELPER_APPLE_RTKIT_COPROC_PLACES_BUFFERS = APPLE_RTKIT_COPROC_PLACES_BUFFERS; const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; const u64 BINDINGS_SNDRV_PCM_FMTBIT_S32_LE = SNDRV_PCM_FMTBIT_S32_LE; diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 3381fc25c5052..ab3fc6c33fe5d 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -48,6 +48,13 @@ pub trait Operations { /// Type representing an allocated buffer for RTKit. type Buffer: Buffer; + /// Whether the coprocessor places its shared buffers itself. + /// + /// If set, buffer requests that carry an IOVA are served by [`Operations::shmem_map`] and + /// acknowledged by sending the request back unchanged, and the crashlog endpoint is started + /// before the endpoint map is acknowledged. + const COPROC_PLACES_BUFFERS: bool = false; + /// Called when RTKit crashes. fn crashed(_data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) {} @@ -200,6 +207,11 @@ unsafe extern "C" fn shmem_destroy_callback( impl RtKit { const VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { + flags: if T::COPROC_PLACES_BUFFERS { + bindings::APPLE_RTKIT_COPROC_PLACES_BUFFERS + } else { + 0 + }, crashed: Some(crashed_callback::), recv_message: Some(recv_message_callback::), recv_message_early: Some(recv_message_early_callback::), From fe5ade63f003c1901b5a65bd72a831f6bed8c621 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:53:05 -0400 Subject: [PATCH 0589/1579] dt-bindings: gpu: apple,agx: Add apple,agx-t8140 The T8140 (A18 Pro) GPU is run by two firmware co-processors, GFX and GFX1, each with its own control registers and mailbox. Its node therefore has three register ranges (asc, asc1, sgx) and two mailboxes. The firmware shares the GPU MMU translation tables, page tables and handoff structure through the same reserved memory regions as on earlier GPUs, but needs neither calibration blobs nor a global variable region, and it does not use apple,firmware-abi. The GPU power domain and the performance states are required. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c49ab96deb3c378015d1db27bf2122c837fd3bc0) --- .../devicetree/bindings/gpu/apple,agx.yaml | 67 ++++++++++++++++--- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index 59989d8bd1cbc..cc9c78a7624f8 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -17,6 +17,7 @@ properties: - apple,agx-g13s - apple,agx-g14g - apple,agx-g14s + - apple,agx-t8140 - items: - enum: - apple,agx-g13c @@ -29,22 +30,22 @@ properties: - const: apple,agx-g14s reg: - items: - - description: GPU coprocessor control registers - - description: GPU block MMIO registers + minItems: 2 + maxItems: 3 reg-names: - items: - - const: asc - - const: sgx + minItems: 2 + maxItems: 3 power-domains: maxItems: 1 mboxes: - maxItems: 1 + minItems: 1 + maxItems: 2 memory-region: + minItems: 3 items: - description: Region containing GPU MMU TTBs - description: Region containing GPU MMU page tables @@ -56,6 +57,7 @@ properties: - description: Shared global variables with GPU firmware memory-region-names: + minItems: 3 items: - const: ttbs - const: pagetables @@ -64,6 +66,8 @@ properties: - const: hw-cal-b - const: globals + operating-points-v2: true + apple,firmware-abi: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 3 @@ -77,7 +81,54 @@ required: - reg - mboxes - memory-region - - apple,firmware-abi + +allOf: + - if: + properties: + compatible: + contains: + const: apple,agx-t8140 + then: + properties: + reg: + items: + - description: GFX coprocessor control registers + - description: GFX1 coprocessor control registers + - description: GPU block MMIO registers + reg-names: + items: + - const: asc + - const: asc1 + - const: sgx + mboxes: + items: + - description: GFX coprocessor mailbox + - description: GFX1 coprocessor mailbox + memory-region: + maxItems: 3 + memory-region-names: + maxItems: 3 + required: + - power-domains + - operating-points-v2 + else: + properties: + reg: + items: + - description: GPU coprocessor control registers + - description: GPU block MMIO registers + reg-names: + items: + - const: asc + - const: sgx + mboxes: + maxItems: 1 + memory-region: + minItems: 6 + memory-region-names: + minItems: 6 + required: + - apple,firmware-abi additionalProperties: false From 4d0847d2323a9eab456204a31012213d923a7a0a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:53:05 -0400 Subject: [PATCH 0590/1579] dt-bindings: mailbox: apple,mailbox: Add the T8140 GPU mailboxes The GPU co-processors of the T8140 (A18 Pro) use the v4 ASC mailbox layout. Unlike the other T8140 ASC mailboxes, described by apple,t8140-asc-mailbox, they do not need the application processor to enable their receive FIFO. Add apple,t8140-gfx-asc-mailbox for them, with apple,asc-mailbox-v4 as the fallback. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 4c5aad430f6283d1d6b3b961a87d1629b397f22f) --- .../devicetree/bindings/mailbox/apple,mailbox.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index dc4cbb5168f20..21dc2231d88ba 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -37,6 +37,14 @@ properties: - apple,t6020-asc-mailbox - const: apple,asc-mailbox-v4 + - description: + ASC mailboxes of the T8140 GPU coprocessors. Unlike the other + T8140 ASC mailboxes, they do not need the application processor + to enable their receive FIFO. + items: + - const: apple,t8140-gfx-asc-mailbox + - const: apple,asc-mailbox-v4 + - description: AOP setup-port mailbox with a separate register layout. const: apple,t8140-aop-setup-mailbox From 6898f34ac7bd349ce29b60fb9785168950c5ed69 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:53:34 -0400 Subject: [PATCH 0591/1579] arm64: dts: apple: t8140-pmgr: Add the GPU power domains Add the power domain of the GPU and its co-processors, and keep the afggfxni0 domain always on. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit fb7b8edf4ee19a35c07c1a2c6aa6b8e65c71777b) --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 3fc3ea3ad8fb4..d1f53903c9a14 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -23,6 +23,23 @@ label = "gpio"; }; + ps_afggfxni0: power-controller@1b8 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "afggfxni0"; + apple,always-on; + }; + + ps_gfx: power-controller@1f0 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gfx"; + }; + ps_apcie_gp: power-controller@258 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x258 4>; From 52fac306b90783087af17af7ba3e2fc6570122e5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:53:45 -0400 Subject: [PATCH 0592/1579] arm64: dts: apple: t8140: Add the GPU Add the GPU, the mailboxes of its two co-processors, its performance states and the reserved memory regions it shares with its firmware. The mailbox nodes describe only the mailbox block of each co-processor wrapper, at offset 0x8000. The bootloader fills in the reserved memory regions and the device-specific performance states. It also reserves uat-l2-pagetables for GPU page tables; the GPU node does not reference that region. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e77d74a6be7645e64b9b50bb51713e72a08c9520) --- arch/arm64/boot/dts/apple/t8140.dtsi | 142 +++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 2dd921b23368c..29e8ad3a19ef0 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -251,6 +251,80 @@ }; }; + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The operating points are device-specific and must be + * filled in by the bootloader. Entries keep the order of the + * GPU performance states, which is not sorted by frequency. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <125000>; + }; + opp01 { + opp-hz = /bits/ 64 <338000000>; + opp-microvolt = <605000>; + }; + opp02 { + opp-hz = /bits/ 64 <492000000>; + opp-microvolt = <630000>; + }; + opp03 { + opp-hz = /bits/ 64 <618000000>; + opp-microvolt = <650000>; + }; + opp04 { + opp-hz = /bits/ 64 <796000000>; + opp-microvolt = <695000>; + }; + opp05 { + opp-hz = /bits/ 64 <928000000>; + opp-microvolt = <745000>; + }; + opp06 { + opp-hz = /bits/ 64 <952000000>; + opp-microvolt = <800000>; + }; + opp07 { + opp-hz = /bits/ 64 <1056000000>; + opp-microvolt = <800000>; + }; + opp08 { + opp-hz = /bits/ 64 <1053000000>; + opp-microvolt = <845000>; + }; + opp09 { + opp-hz = /bits/ 64 <1170000000>; + opp-microvolt = <845000>; + }; + opp10 { + opp-hz = /bits/ 64 <1152000000>; + opp-microvolt = <885000>; + }; + opp11 { + opp-hz = /bits/ 64 <1278000000>; + opp-microvolt = <885000>; + }; + opp12 { + opp-hz = /bits/ 64 <1204000000>; + opp-microvolt = <905000>; + }; + opp13 { + opp-hz = /bits/ 64 <1338000000>; + opp-microvolt = <905000>; + }; + opp14 { + opp-hz = /bits/ 64 <1326000000>; + opp-microvolt = <975000>; + }; + opp15 { + opp-hz = /bits/ 64 <1470000000>; + opp-microvolt = <975000>; + }; + }; + timer { compatible = "arm,armv8-timer"; interrupt-parent = <&aic>; @@ -268,6 +342,29 @@ clock-output-names = "clkref"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + }; + + /* Only reserved: the GPU node does not reference it */ + uat_l2_pagetables: uat-l2-pagetables { + reg = <0 0 0 0>; + }; + + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + }; + + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + }; + }; + soc: soc { compatible = "simple-bus"; #address-cells = <2>; @@ -703,6 +800,51 @@ power-domains = <&ps_atc0_usb>; status = "disabled"; }; + + gpu: gpu@482600000 { + compatible = "apple,agx-t8140"; + reg = <0x4 0x82600000 0x0 0x4000>, + <0x4 0x82e00000 0x0 0x4000>, + <0x4 0x80000000 0x0 0x4000000>; + reg-names = "asc", "asc1", "sgx"; + mboxes = <&gfx_mbox>, <&gfx1_mbox>; + power-domains = <&ps_gfx>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, + <&uat_handoff>; + memory-region-names = "ttbs", "pagetables", "handoff"; + operating-points-v2 = <&gpu_opp>; + status = "disabled"; + }; + + gfx_mbox: mailbox@482608000 { + compatible = "apple,t8140-gfx-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x82608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + status = "disabled"; + }; + + gfx1_mbox: mailbox@482e08000 { + compatible = "apple,t8140-gfx-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x82e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + status = "disabled"; + }; }; }; From 54124e3efacba7ba530e742cb2f297bbdeb719fa Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:54:00 -0400 Subject: [PATCH 0593/1579] arm64: dts: apple: t8140-j700: Enable the GPU Enable the GPU and its mailboxes, and add the gpu alias through which the bootloader finds the GPU node. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f4a46cd97b2a3e70875d76e3cf2b82f3959fd0f5) --- arch/arm64/boot/dts/apple/t8140-j700.dts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index cc9f115adb7ab..06f4d6385465a 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -17,6 +17,7 @@ aliases { atcphy0 = &atcphy0; + gpu = &gpu; keyboard = &keyboard; serial0 = &serial0; wifi0 = &wifi0; @@ -1348,6 +1349,18 @@ }; }; +&gfx_mbox { + status = "okay"; +}; + +&gfx1_mbox { + status = "okay"; +}; + +&gpu { + status = "okay"; +}; + #include "hwmon-common.dtsi" #include "hwmon-laptop.dtsi" From 630c43dac20e9621d3c2cd687c98ab85d51c8b9e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:55:29 -0400 Subject: [PATCH 0594/1579] drm/asahi: uapi: Add the G17 additions Extend the UAPI for the G17 generation of GPUs: - drm_asahi_params_global gains the shader instruction set generation and the command stream generation (enum drm_asahi_gpu_hal_generation), which userspace needs to select its code generation and the layout of its control streams. - DRM_ASAHI_FEATURE_VM_STATUS and DRM_ASAHI_PARAM_GROUP_VM_STATUS report the sticky error of a VM, so that userspace can tell whether work behind a signalled fence executed successfully. - DRM_ASAHI_FEATURE_VM_STATUS_MIRROR and DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR let userspace register a word that the kernel keeps equal to that error. - DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES and DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS report how many queues the device supports and how many jobs of a queue may run at once. - DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS promises that every command of a submission waits for all of its input syncs. - The background and end-of-tile resource specifiers of G17 are 64 bits wide. Their high halves are appended to drm_asahi_cmd_render and flagged with DRM_ASAHI_RENDER_RSRC_SPEC_HI, which older kernels reject. The existing driver reports the generations of the G13 and G14 GPUs it supports and none of the new features. It still accepts render commands of the previous size, zero-filling the new fields, and rejects the new flag. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e0a5c5ad84cbc6e48c0194882b8c1c67d72538a8) --- drivers/gpu/drm/asahi/file.rs | 5 + include/uapi/drm/asahi_drm.h | 203 +++++++++++++++++++++++++++++++++- 2 files changed, 206 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 061f289e40d93..63a092d1fdb9c 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -324,6 +324,11 @@ impl File { max_frequency_khz: gpu.get_dyncfg().pwr.max_frequency_khz(), command_timestamp_frequency_hz: 1_000_000_000, // User timestamps always in nanoseconds + + // G13 and G14 share the second shader ISA generation and predate the numbered + // command stream generations. + usc_generation: 2, + gpu_hal_generation: uapi::drm_asahi_gpu_hal_generation_DRM_ASAHI_GPU_HAL_LEGACY, }; for (i, mask) in gpu.get_dyncfg().id.core_masks.iter().enumerate() { diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index de67f1c603afd..8dae3950b4a02 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -191,6 +191,35 @@ struct drm_asahi_params_global { * seconds, rather than hardcoding a particular firmware's rate. */ __u64 command_timestamp_frequency_hz; + + /** + * @usc_generation: Instruction set generation of the unified shader + * cores: 2 for G13 and G14, 3 for G15 and newer. + */ + __u32 usc_generation; + + /** + * @gpu_hal_generation: Command stream generation, from + * enum drm_asahi_gpu_hal_generation. + */ + __u32 gpu_hal_generation; +}; + +/** + * enum drm_asahi_gpu_hal_generation - GPU command stream generation + * + * The command stream generation determines the layout of the control streams + * and descriptors that userspace builds for the GPU. + */ +enum drm_asahi_gpu_hal_generation { + /** @DRM_ASAHI_GPU_HAL_LEGACY: Unnumbered generation of G13 to G15. */ + DRM_ASAHI_GPU_HAL_LEGACY = 0, + + /** @DRM_ASAHI_GPU_HAL_200: Generation 200, used by G17P. */ + DRM_ASAHI_GPU_HAL_200 = 200, + + /** @DRM_ASAHI_GPU_HAL_300: Generation 300. */ + DRM_ASAHI_GPU_HAL_300 = 300, }; /** @@ -213,19 +242,168 @@ enum drm_asahi_feature { * userspace can speculate memory accesses more aggressively. */ DRM_ASAHI_FEATURE_SOFT_FAULTS = (1UL) << 0, + + /** + * @DRM_ASAHI_FEATURE_VM_STATUS: GET_PARAMS supports + * DRM_ASAHI_PARAM_GROUP_VM_STATUS. + */ + DRM_ASAHI_FEATURE_VM_STATUS = (1UL) << 1, + + /** + * @DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES: Each queue reserves its own + * hardware execution state when it is created, and GET_PARAMS + * supports DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS. + */ + DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES = (1UL) << 2, + + /** + * @DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS: Every hardware command of + * a submission waits for all of the submission's input syncs, including + * their errors, so independent commands need no extra dependency on the + * first command. Userspace that splits a batch across several + * submissions and relies on this must pass its input syncs to each of + * them. + */ + DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS = (1UL) << 3, + + /** + * @DRM_ASAHI_FEATURE_VM_STATUS_MIRROR: GET_PARAMS supports + * DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR, which registers a userspace + * word that always holds the VM status that + * DRM_ASAHI_PARAM_GROUP_VM_STATUS would return. The kernel stores a + * failure there before it signals any fence of the failed job, so after + * a successful wait a load of the word replaces the VM_STATUS query. + * Registration may still be refused. + */ + DRM_ASAHI_FEATURE_VM_STATUS_MIRROR = (1UL) << 4, +}; + +/** + * define DRM_ASAHI_PARAM_GROUP_VM_STATUS - GET_PARAMS group for + * struct drm_asahi_vm_status + */ +#define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 + +/** + * define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS - GET_PARAMS group for + * struct drm_asahi_queue_limits + */ +#define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS 2 + +/** + * define DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR - GET_PARAMS group for + * struct drm_asahi_vm_status_mirror + */ +#define DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR 3 + +/** + * struct drm_asahi_vm_status - VM status, GET_PARAMS group + * DRM_ASAHI_PARAM_GROUP_VM_STATUS + * + * This is an in/out query on a VM of the calling file. Userspace sets + * &drm_asahi_get_params.pointer to this structure and + * &drm_asahi_get_params.size to its size. + * + * The kernel records the failure of an accepted job before it signals the + * job's fences, so userspace must query the status after a successful syncobj + * wait before it treats the work as successfully executed. A signalled + * syncobj alone does not indicate successful execution. An execution failure + * also marks other VMs whose work can no longer make progress. A query never + * resets or recovers the GPU. + * + * Invalid requests fail with EINVAL (non-zero @flags or @pad, wrong size), + * ENOENT (@vm_id unknown to the calling file) or EFAULT, and write nothing. + */ +struct drm_asahi_vm_status { + /** @vm_id: VM to query */ + __u32 vm_id; + + /** @flags: MBZ */ + __u32 flags; + + /** + * @error: Output: 0 if no submission error was observed, otherwise the + * first error as a negative errno. The error is sticky for the lifetime + * of the VM: neither creating queues nor waiting on other fences clears + * it. + */ + __s32 error; + + /** @pad: MBZ */ + __u32 pad; +}; + +/** + * struct drm_asahi_queue_limits - Queue limits, GET_PARAMS group + * DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS + * + * The query neither allocates nor resets anything. + */ +struct drm_asahi_queue_limits { + /** + * @max_queues: Maximum number of queues on the device, shared by all + * files and VMs. This is a hardware bound, not a promise that as many + * are currently available: QUEUE_CREATE reserves a queue's hardware + * state and fails once it runs out. + */ + __u32 max_queues; + + /** + * @max_in_flight_per_queue: Number of jobs of a queue that the + * hardware may run at once. Further jobs wait behind the preceding work + * and their dependencies. + */ + __u32 max_in_flight_per_queue; + + /** @flags: Reserved, zero */ + __u32 flags; + + /** @pad: Reserved, zero */ + __u32 pad; +}; + +/** + * struct drm_asahi_vm_status_mirror - VM status mirror registration, GET_PARAMS + * group DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR + * + * Registers a 32-bit word in a GEM object of the calling file. The kernel + * keeps its own reference to the object for the lifetime of the VM status, + * zeroes the word, and from then on keeps it equal to the VM status error + * (see &drm_asahi_vm_status.error). Each VM accepts one registration. + * EOPNOTSUPP means that userspace must keep using + * DRM_ASAHI_PARAM_GROUP_VM_STATUS. + */ +struct drm_asahi_vm_status_mirror { + /** @vm_id: VM whose status is mirrored */ + __u32 vm_id; + + /** @handle: GEM object holding the word, at most 64 KiB in size */ + __u32 handle; + + /** @offset: Byte offset of the word in the object, 4-byte aligned */ + __u32 offset; + + /** @flags: MBZ */ + __u32 flags; }; /** * struct drm_asahi_get_params - Arguments passed to DRM_IOCTL_ASAHI_GET_PARAMS */ struct drm_asahi_get_params { - /** @param_group: Parameter group to fetch (MBZ) */ + /** + * @param_group: Parameter group: 0 for struct drm_asahi_params_global, + * otherwise one of DRM_ASAHI_PARAM_GROUP_*. + */ __u32 param_group; /** @pad: MBZ */ __u32 pad; - /** @pointer: User pointer to write parameter struct */ + /** + * @pointer: User pointer to the parameter struct. The VM status groups + * also read their input from it. + */ __u64 pointer; /** @@ -789,6 +967,15 @@ enum drm_asahi_render_flags { */ DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING = (1U << 2), + /** + * @DRM_ASAHI_RENDER_RSRC_SPEC_HI: The command carries the high halves + * of the background and end-of-tile resource specifiers + * (&drm_asahi_cmd_render.bg_rsrc_spec_hi and the following fields). + * Kernels that do not know the flag reject the command instead of + * silently truncating the specifiers. + */ + DRM_ASAHI_RENDER_RSRC_SPEC_HI = (1U << 4), + /** * @DRM_ASAHI_RENDER_DBIAS_IS_INT: Use integer depth bias formula. * @@ -1107,6 +1294,18 @@ struct drm_asahi_cmd_render { /** @ts_frag: Timestamps for the fragment portion of the render */ struct drm_asahi_timestamps ts_frag; + + /** @bg_eot_rsrc_spec_hi: High half of @eot.rsrc_spec */ + __u32 bg_eot_rsrc_spec_hi; + + /** @bg_eot_partial_rsrc_spec_hi: High half of @partial_eot.rsrc_spec */ + __u32 bg_eot_partial_rsrc_spec_hi; + + /** @bg_rsrc_spec_hi: High half of @bg.rsrc_spec */ + __u32 bg_rsrc_spec_hi; + + /** @bg_partial_rsrc_spec_hi: High half of @partial_bg.rsrc_spec */ + __u32 bg_partial_rsrc_spec_hi; }; /** From e46d8cfd391b00b302d5a9ff4608b82c3714c910 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:07:41 -0400 Subject: [PATCH 0595/1579] drm/asahi: separate DRM-facing GPU operations from the firmware manager The file and ioctl layer uses a manager trait that also exposes G13/G14 firmware allocators, VM slots, event handling and control channels. A GPU with a different firmware interface cannot implement those internals. Separate the DRM-facing operations into Gpu: VM and queue creation, timestamp mapping, sequence IDs, crash state, timer frequency and hardware parameters. Existing managers implement it directly and expose their firmware-specific interface through Gpu::manager(). Keep the legacy firmware-global update at ioctl entry after flag validation, before queue lookup and userspace copies, outside the queue mutex. Invalid queue submissions still refresh the firmware idle policy. Legacy operation ordering and reported hardware values are unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit aae9f7d3525f93a7a5b5ae0529f8e85219735d08) --- drivers/gpu/drm/asahi/driver.rs | 14 +- drivers/gpu/drm/asahi/file.rs | 26 +--- drivers/gpu/drm/asahi/gpu.rs | 198 ++++++++++++++++++----------- drivers/gpu/drm/asahi/mmu.rs | 6 +- drivers/gpu/drm/asahi/queue/mod.rs | 9 +- drivers/gpu/drm/asahi/workqueue.rs | 6 +- 6 files changed, 150 insertions(+), 109 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index bafcfc4edc1f6..7f07eb2429540 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -31,11 +31,11 @@ use crate::{ use kernel::macros::vtable; -/// Holds a reference to the top-level `GpuManager` object. +/// Holds a reference to the top-level GPU object. #[pin_data] pub(crate) struct AsahiData { #[pin] - pub(crate) gpu: Arc, + pub(crate) gpu: Arc, pub(crate) pdev: ARef, pub(crate) resources: regs::Resources, } @@ -191,19 +191,19 @@ impl platform::Driver for AsahiDriver { let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { - gpu::GpuManagerG13V12_3::new(&drm.clone(), &res, cfg)? as Arc + gpu::GpuManagerG13V12_3::new(&drm.clone(), &res, cfg)? as Arc } (hw::GpuGen::G14, hw::GpuVariant::G, &[12, 4, 0]) => { - gpu::GpuManagerG14V12_4::new(&drm.clone(), &res, cfg)? as Arc + gpu::GpuManagerG14V12_4::new(&drm.clone(), &res, cfg)? as Arc } (hw::GpuGen::G13, _, &[13, 5, 0]) => { - gpu::GpuManagerG13V13_5::new(&drm.clone(), &res, cfg)? as Arc + gpu::GpuManagerG13V13_5::new(&drm.clone(), &res, cfg)? as Arc } (hw::GpuGen::G14, hw::GpuVariant::G, &[13, 5, 0]) => { - gpu::GpuManagerG14V13_5::new(&drm.clone(), &res, cfg)? as Arc + gpu::GpuManagerG14V13_5::new(&drm.clone(), &res, cfg)? as Arc } (hw::GpuGen::G14, _, &[13, 5, 0]) => { - gpu::GpuManagerG14XV13_5::new(&drm.clone(), &res, cfg)? as Arc + gpu::GpuManagerG14XV13_5::new(&drm.clone(), &res, cfg)? as Arc } _ => { dev_info!( diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 63a092d1fdb9c..4b9b388c1b951 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -305,35 +305,19 @@ impl File { let mut params = uapi::drm_asahi_params_global { features: 0, - gpu_generation: gpu.get_dyncfg().id.gpu_gen as u32, - gpu_variant: gpu.get_dyncfg().id.gpu_variant as u32, - gpu_revision: gpu.get_dyncfg().id.gpu_rev as u32, - chip_id: gpu.get_cfg().chip_id, - - num_dies: gpu.get_cfg().num_dies, - num_clusters_total: gpu.get_dyncfg().id.num_clusters, - num_cores_per_cluster: gpu.get_dyncfg().id.num_cores, - core_masks: [0; uapi::DRM_ASAHI_MAX_CLUSTERS as usize], - vm_start: VM_USER_RANGE.start, vm_end: VM_USER_RANGE.end, vm_kernel_min_size: VM_KERNEL_MIN_SIZE, max_commands_per_submission: MAX_COMMANDS_PER_SUBMISSION, max_attachments: crate::microseq::MAX_ATTACHMENTS as u32, - max_frequency_khz: gpu.get_dyncfg().pwr.max_frequency_khz(), command_timestamp_frequency_hz: 1_000_000_000, // User timestamps always in nanoseconds - // G13 and G14 share the second shader ISA generation and predate the numbered - // command stream generations. - usc_generation: 2, - gpu_hal_generation: uapi::drm_asahi_gpu_hal_generation_DRM_ASAHI_GPU_HAL_LEGACY, + ..pin_init::zeroed() }; - for (i, mask) in gpu.get_dyncfg().id.core_masks.iter().enumerate() { - *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); - } + gpu.get_params(&mut params)?; if *module_parameters::fault_control.value() == 0xb { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; @@ -985,7 +969,9 @@ impl File { } let gpu = &device.gpu; - gpu.update_globals(); + if let Some(manager) = gpu.manager() { + manager.update_globals(); + } // Upgrade to Arc to drop the XArray lock early let queue: Arc>> = file @@ -1070,7 +1056,7 @@ impl File { // TODO: Do this on device-init for perf. let gpu = &device.gpu; - let frequency_hz = gpu.get_cfg().base_clock_hz as u64; + let frequency_hz = gpu.base_clock_hz() as u64; let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); let num = (NSEC_PER_SEC as u64) / ts_gcd; diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index b6c1c64c50367..d74de47117f68 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -47,7 +47,8 @@ use kernel::{ Instant, Monotonic, // }, - types::ForeignOwnable, // + types::ForeignOwnable, + uapi, // }; #[cfg(CONFIG_DEV_COREDUMP)] use kernel::{ @@ -262,24 +263,26 @@ pub(crate) struct GpuManager { garbage_contexts: Mutex>>>, } -/// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. -pub(crate) trait GpuManager: Send + Sync { +/// Operations the DRM driver performs on a GPU, independent of its firmware interface. +pub(crate) trait Gpu: Send + Sync { /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. fn arc_as_any(self: Arc) -> Arc; + /// Return the G13/G14 firmware manager, if this GPU uses that firmware interface. + fn manager(&self) -> Option<&dyn GpuManager>; /// Initialize the GPU. fn init(&self) -> Result; - /// Update the GPU globals from global info - /// - /// TODO: Unclear what can and cannot be updated like this. - fn update_globals(&self); - /// Get a reference to the KernelAllocators. - fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend>; - /// Create a new `Vm` given a unique `File` ID. + /// Return a reference to the global `SequenceIDs` instance. + fn ids(&self) -> &SequenceIDs; + /// Check whether the GPU is crashed + fn is_crashed(&self) -> bool; + /// Fill in the hardware description reported to userspace. + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; + /// Frequency of the CPU timer that GPU timestamps are derived from. + fn base_clock_hz(&self) -> u32; + /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; - /// Bind a `Vm` to an available slot and return the `VmBind`. - fn bind_vm(&self, vm: &mmu::Vm) -> Result; /// Create a new user command queue. fn new_queue( &self, @@ -289,8 +292,24 @@ pub(crate) trait GpuManager: Send + Sync { priority: u32, usc_exec_base: u64, ) -> Result>; - /// Return a reference to the global `SequenceIDs` instance. - fn ids(&self) -> &SequenceIDs; + /// Map a BO as a timestamp buffer + fn map_timestamp_buffer( + &self, + bo: gem::ObjectRef, + range: Range, + ) -> Result; +} + +/// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. +pub(crate) trait GpuManager: Send + Sync { + /// Update the GPU globals from global info + /// + /// TODO: Unclear what can and cannot be updated like this. + fn update_globals(&self); + /// Get a reference to the KernelAllocators. + fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend>; + /// Bind a `Vm` to an available slot and return the `VmBind`. + fn bind_vm(&self, vm: &mmu::Vm) -> Result; /// Kick the firmware (wake it up if asleep). /// /// This should be useful to reduce latency on work submission, so we can ask the firmware to @@ -322,14 +341,6 @@ pub(crate) trait GpuManager: Send + Sync { fn get_dyncfg(&self) -> &hw::DynConfig; /// Register an unused context as garbage fn free_context(&self, data: KBox>); - /// Check whether the GPU is crashed - fn is_crashed(&self) -> bool; - /// Map a BO as a timestamp buffer - fn map_timestamp_buffer( - &self, - bo: gem::ObjectRef, - range: Range, - ) -> Result; } /// Private generic trait for functions that don't need to escape this module. @@ -1236,7 +1247,7 @@ impl GpuManager::ver { } #[versions(AGX)] -impl GpuManager for GpuManager::ver { +impl Gpu for GpuManager::ver { fn as_any(&self) -> &dyn Any { self } @@ -1245,6 +1256,10 @@ impl GpuManager for GpuManager::ver { self as Arc } + fn manager(&self) -> Option<&dyn GpuManager> { + Some(self) + } + fn init(&self) -> Result { self.tx_channels.lock().device_control.send( &fw::channels::DeviceControlMsg::ver::Initialize(Default::default()), @@ -1267,6 +1282,89 @@ impl GpuManager for GpuManager::ver { Ok(()) } + fn ids(&self) -> &SequenceIDs { + &self.ids + } + + fn is_crashed(&self) -> bool { + self.crashed.load(Ordering::Relaxed) + } + + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + params.gpu_generation = self.dyncfg.id.gpu_gen as u32; + params.gpu_variant = self.dyncfg.id.gpu_variant as u32; + params.gpu_revision = self.dyncfg.id.gpu_rev as u32; + params.chip_id = self.cfg.chip_id; + params.num_dies = self.cfg.num_dies; + params.num_clusters_total = self.dyncfg.id.num_clusters; + params.num_cores_per_cluster = self.dyncfg.id.num_cores; + params.max_frequency_khz = self.dyncfg.pwr.max_frequency_khz(); + // G13 and G14 share the second shader ISA generation and predate the numbered command + // stream generations. + params.usc_generation = 2; + params.gpu_hal_generation = uapi::drm_asahi_gpu_hal_generation_DRM_ASAHI_GPU_HAL_LEGACY; + + for (i, mask) in self.dyncfg.id.core_masks.iter().enumerate() { + *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); + } + + Ok(()) + } + + fn base_clock_hz(&self) -> u32 { + self.cfg.base_clock_hz + } + + fn new_vm(&self, kernel_range: Range) -> Result { + self.uat.new_vm(self.ids.vm.next(), kernel_range) + } + + fn new_queue( + &self, + vm: mmu::Vm, + ualloc: Arc>, + ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result> { + let mut kalloc = self.alloc(); + let id = self.ids.queue.next(); + Ok(KBox::new( + queue::Queue::ver::new( + &self.dev, + self, + vm, + &mut kalloc, + ualloc, + ualloc_priv, + self.event_manager.clone(), + &self.buffer_mgr, + id, + priority, + usc_exec_base, + )?, + GFP_KERNEL, + )?) + } + + fn map_timestamp_buffer( + &self, + mut bo: gem::ObjectRef, + range: Range, + ) -> Result { + bo.map_range_into_range( + self.uat.kernel_vm(), + range, + IOVA_KERN_TIMESTAMP_RANGE, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + ) + } +} + +#[versions(AGX)] +impl GpuManager for GpuManager::ver { fn update_globals(&self) { let mut timeout: u32 = 2; if debug_enabled(DebugFlags::WaitForPowerOff) { @@ -1325,41 +1423,10 @@ impl GpuManager for GpuManager::ver { guard } - fn new_vm(&self, kernel_range: Range) -> Result { - self.uat.new_vm(self.ids.vm.next(), kernel_range) - } - fn bind_vm(&self, vm: &mmu::Vm) -> Result { self.uat.bind(vm) } - fn new_queue( - &self, - vm: mmu::Vm, - ualloc: Arc>, - ualloc_priv: Arc>, - priority: u32, - usc_exec_base: u64, - ) -> Result> { - let mut kalloc = self.alloc(); - let id = self.ids.queue.next(); - Ok(KBox::new( - queue::Queue::ver::new( - &self.dev, - vm, - &mut kalloc, - ualloc, - ualloc_priv, - self.event_manager.clone(), - &self.buffer_mgr, - id, - priority, - usc_exec_base, - )?, - GFP_KERNEL, - )?) - } - fn kick_firmware(&self) -> Result { if self.is_crashed() { return Err(ENODEV); @@ -1415,10 +1482,6 @@ impl GpuManager for GpuManager::ver { Ok(()) } - fn ids(&self) -> &SequenceIDs { - &self.ids - } - fn handle_timeout(&self, counter: u32, event_slot: i32, unk: u32) { dev_err!(self.dev.as_ref(), " (\\________/) \n"); dev_err!(self.dev.as_ref(), " | | \n"); @@ -1608,25 +1671,6 @@ impl GpuManager for GpuManager::ver { ); } } - - fn is_crashed(&self) -> bool { - self.crashed.load(Ordering::Relaxed) - } - - fn map_timestamp_buffer( - &self, - mut bo: gem::ObjectRef, - range: Range, - ) -> Result { - bo.map_range_into_range( - self.uat.kernel_vm(), - range, - IOVA_KERN_TIMESTAMP_RANGE, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - false, - ) - } } #[versions(AGX)] diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 144d6d933c427..4601eecedc0ed 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -752,7 +752,11 @@ impl KernelMapping { }; // Tell the firmware to do a cache flush - if let Err(e) = (*owner.dev).gpu.fwctl(cmd) { + let ret = match (*owner.dev).gpu.manager() { + Some(gpu) => gpu.fwctl(cmd), + None => Err(ENODEV), + }; + if let Err(e) = ret { dev_err!( owner.dev.as_ref(), "MMU: ASC cache flush {:#x}:{:#x} failed (err: {:?})\n", diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 85da1d539d1bc..a493f94fda583 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -26,7 +26,10 @@ use crate::debug::*; use crate::driver::{AsahiDevRef, AsahiDevice}; use crate::file::MAX_COMMANDS_PER_SUBMISSION; use crate::fw::types::*; -use crate::gpu::GpuManager; +use crate::gpu::{ + Gpu, + GpuManager, // +}; use crate::inner_weak_ptr; use crate::microseq; use crate::module_parameters; @@ -435,6 +438,7 @@ impl Queue::ver { #[allow(clippy::too_many_arguments)] pub(crate) fn new( dev: &AsahiDevice, + gpu: &dyn GpuManager, vm: mmu::Vm, alloc: &mut gpu::KernelAllocators, ualloc: Arc>, @@ -479,8 +483,7 @@ impl Queue::ver { sched::Scheduler::new(dev.as_ref(), 1, WQ_SIZE, 0, 100000, c_str!("asahi_sched"))?; let entity = sched::Entity::new(&sched, sched::Priority::Kernel)?; - let buffer = - buffer::Buffer::ver::new(&*(*dev).gpu, alloc, ualloc.clone(), ualloc_priv, mgr)?; + let buffer = buffer::Buffer::ver::new(gpu, alloc, ualloc.clone(), ualloc_priv, mgr)?; let mut ret = Queue::ver { dev: dev.into(), diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index 55b6058d50224..8e6c7e47de6aa 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -130,7 +130,11 @@ impl Drop for GpuContext { fn drop(&mut self) { mod_dev_dbg!(self.dev, "GpuContext: Freeing GPU context\n"); let data = self.data.take().unwrap(); - (*self.dev).gpu.free_context(data); + // A context only exists for the manager whose allocators created it, so the manager is + // always present. + if let Some(gpu) = (*self.dev).gpu.manager() { + gpu.free_context(data); + } } } From 54721b0f0bee7be5b3094112ffcfa36e76dec7ed Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 2 Oct 2026 02:57:46 -0400 Subject: [PATCH 0596/1579] drm/asahi: mmu: support per-SoC UAT geometry and user VA range G13 and G14 GPUs translate 39 input address bits per UAT root, and the driver hardcodes that width in the page table walker, the user and kernel VA ranges and the user range reported to userspace. Newer GPUs use the same 16 KiB granule and three-level walk, but translate 42 bits per root. Describe the address width of a UAT instance with a UatGeometry (input and output address bits) and derive the ranges from it: the user range spans the lower root, the dummy page sits two pages below its top, and the driver-managed kernel range is the third top-level entry of the upper root. The input width becomes per-SoC configuration data next to the output width, Uat::new takes the geometry instead of the firmware configuration, the page table walker takes the input width per table, and the file layer gets the user range from the device instead of a constant. All G13/G14 configurations describe a 39-bit input width, for which every derived value equals the constant it replaces: user top 1 << 39, dummy page 0x7f_ffff_8000, kernel range 0xffff_ffa0_0000_0000 to 0xffff_ffb0_0000_0000, upper range from 0xffff_ff80_0000_0000, and the walker's input mask. No functional change on those SoCs. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 7d3ddb6c35e1c478f68c78ac8bb8b3ee9d314e58) --- drivers/gpu/drm/asahi/file.rs | 36 ++++++---- drivers/gpu/drm/asahi/gpu.rs | 15 +++- drivers/gpu/drm/asahi/hw/mod.rs | 2 + drivers/gpu/drm/asahi/hw/t600x.rs | 1 + drivers/gpu/drm/asahi/hw/t602x.rs | 1 + drivers/gpu/drm/asahi/hw/t8103.rs | 1 + drivers/gpu/drm/asahi/hw/t8112.rs | 1 + drivers/gpu/drm/asahi/initdata.rs | 6 +- drivers/gpu/drm/asahi/mmu.rs | 116 ++++++++++++++++++++---------- drivers/gpu/drm/asahi/pgtable.rs | 30 +++++--- 10 files changed, 143 insertions(+), 66 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 4b9b388c1b951..cd41ff066e196 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -61,6 +61,7 @@ struct Vm { ualloc: Arc>, ualloc_priv: Arc>, vm: mmu::Vm, + user_range: Range, kernel_range: Range, _dummy_mapping: mmu::KernelMapping, } @@ -68,8 +69,8 @@ struct Vm { impl Drop for Vm { fn drop(&mut self) { // When the user Vm is dropped, unmap everything in the user range - let left_range = VM_USER_RANGE.start..self.kernel_range.start; - let right_range = self.kernel_range.end..VM_USER_RANGE.end; + let left_range = self.user_range.start..self.kernel_range.start; + let right_range = self.kernel_range.end..self.user_range.end; if !left_range.is_empty() && self @@ -201,6 +202,8 @@ pub(crate) enum Object { // #[pin_data] pub(crate) struct File { id: u64, + /// Available VM range for the user + user_range: Range, // #[pin] vms: xarray::XArray>, // #[pin] @@ -212,9 +215,6 @@ pub(crate) struct File { /// Convenience type alias for our DRM `File` type. pub(crate) type DrmFile = drm::File; -/// Available VM range for the user -const VM_USER_RANGE: Range = mmu::IOVA_USER_USABLE_RANGE; - /// Minimum reserved AS for kernel mappings const VM_KERNEL_MIN_SIZE: u64 = 0x20000000; @@ -227,9 +227,10 @@ impl drm::file::DriverFile for File { let gpu = &device.gpu; let id = gpu.ids().file.next(); + let user_range = gpu.uat_geometry().user_usable_range(); mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - Ok(KBox::pin_init(File::new(id), GFP_KERNEL)?) + Ok(KBox::pin_init(File::new(id, user_range), GFP_KERNEL)?) } fn as_raw(&self) -> *mut bindings::drm_file { @@ -241,7 +242,7 @@ impl drm::file::DriverFile for File { unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} impl File { - fn new(id: u64) -> impl PinInit { + fn new(id: u64, user_range: Range) -> impl PinInit { unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); @@ -259,6 +260,7 @@ impl File { .__pinned_init(raw_objects)?; (*slot).id = id; + (*slot).user_range = user_range; Ok(()) }) } @@ -305,8 +307,8 @@ impl File { let mut params = uapi::drm_asahi_params_global { features: 0, - vm_start: VM_USER_RANGE.start, - vm_end: VM_USER_RANGE.end, + vm_start: file.inner().user_range.start, + vm_end: file.inner().user_range.end, vm_kernel_min_size: VM_KERNEL_MIN_SIZE, max_commands_per_submission: MAX_COMMANDS_PER_SUBMISSION, @@ -344,9 +346,10 @@ impl File { file: &DrmFile, ) -> Result { let kernel_range = data.kernel_start..data.kernel_end; + let user_range = file.inner().user_range.clone(); // Validate requested kernel range - if !VM_USER_RANGE.is_superset(kernel_range.clone()) + if !user_range.is_superset(kernel_range.clone()) || kernel_range.range() < VM_KERNEL_MIN_SIZE || kernel_range.start & (mmu::UAT_PGMSK as u64) != 0 || kernel_range.end & (mmu::UAT_PGMSK as u64) != 0 @@ -415,8 +418,12 @@ impl File { ); let mut dummy_obj = gem::new_kernel_object(device, 0x4000)?; dummy_obj.vmap()?.memset(0); - let dummy_mapping = - dummy_obj.map_at(&vm, mmu::IOVA_UNK_PAGE, mmu::PROT_GPU_SHARED_RW, true)?; + let dummy_mapping = dummy_obj.map_at( + &vm, + gpu.uat_geometry().unk_page(), + mmu::PROT_GPU_SHARED_RW, + true, + )?; mod_dev_dbg!(device, "[File {} VM {}]: VM created\n", file_id, id); resv.fill(KBox::new( @@ -424,6 +431,7 @@ impl File { ualloc, ualloc_priv, vm, + user_range, kernel_range, _dummy_mapping: dummy_mapping, }, @@ -616,7 +624,7 @@ impl File { return Err(EINVAL); } - if !VM_USER_RANGE.is_superset(range.clone()) { + if !file.inner().user_range.is_superset(range.clone()) { cls_pr_debug!( Errors, "gem_bind: Invalid map range {:#x}..{:#x} (not contained in user range)\n", @@ -697,7 +705,7 @@ impl File { let end = data.addr.checked_add(data.range).ok_or(EINVAL)?; let range = start..end; - if !VM_USER_RANGE.is_superset(range.clone()) { + if !file.inner().user_range.is_superset(range.clone()) { cls_pr_debug!( Errors, "gem_bind: Invalid unmap range {:#x}..{:#x} (not contained in user range)\n", diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index d74de47117f68..0d966161f9e5d 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -281,6 +281,8 @@ pub(crate) trait Gpu: Send + Sync { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; /// Frequency of the CPU timer that GPU timestamps are derived from. fn base_clock_hz(&self) -> u32; + /// Return the GPU address space geometry. + fn uat_geometry(&self) -> mmu::UatGeometry; /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; /// Create a new user command queue. @@ -680,7 +682,14 @@ impl GpuManager::ver { let map_kernel_to_user = false; Ok(KBox::new( - mmu::Uat::new(dev, cfg, map_kernel_to_user)?, + mmu::Uat::new( + dev, + mmu::UatGeometry { + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }, + map_kernel_to_user, + )?, GFP_KERNEL, )?) } @@ -1315,6 +1324,10 @@ impl Gpu for GpuManager::ver { self.cfg.base_clock_hz } + fn uat_geometry(&self) -> mmu::UatGeometry { + self.uat.geometry() + } + fn new_vm(&self, kernel_range: Range) -> Result { self.uat.new_vm(self.ids.vm.next(), kernel_range) } diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 611764b5463b5..cf5407ecde438 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -209,6 +209,8 @@ pub(crate) struct HwConfig { /// Base clock used used for timekeeping. pub(crate) base_clock_hz: u32, + /// Input address bits translated by each UAT page table root on this SoC. + pub(crate) uat_ias: u32, /// Output address space for the UAT on this SoC. pub(crate) uat_oas: u32, /// Number of dies on this SoC. diff --git a/drivers/gpu/drm/asahi/hw/t600x.rs b/drivers/gpu/drm/asahi/hw/t600x.rs index 58665f985ec38..85cf88054e44c 100644 --- a/drivers/gpu/drm/asahi/hw/t600x.rs +++ b/drivers/gpu/drm/asahi/hw/t600x.rs @@ -50,6 +50,7 @@ pub(crate) const HWCONFIG_T6002: super::HwConfig = HwConfig { gpu_core: GpuCore::G13C, base_clock_hz: 24_000_000, + uat_ias: 39, uat_oas: 42, num_dies: 2, max_num_clusters: 8, diff --git a/drivers/gpu/drm/asahi/hw/t602x.rs b/drivers/gpu/drm/asahi/hw/t602x.rs index 98a7ac2b76e57..6e2033ecb50dd 100644 --- a/drivers/gpu/drm/asahi/hw/t602x.rs +++ b/drivers/gpu/drm/asahi/hw/t602x.rs @@ -53,6 +53,7 @@ pub(crate) const HWCONFIG_T6022: super::HwConfig = HwConfig { gpu_core: GpuCore::G14D, base_clock_hz: 24_000_000, + uat_ias: 39, uat_oas: 42, num_dies: 2, max_num_clusters: 8, diff --git a/drivers/gpu/drm/asahi/hw/t8103.rs b/drivers/gpu/drm/asahi/hw/t8103.rs index 484bf6c3414f2..a2a76600c0a3b 100644 --- a/drivers/gpu/drm/asahi/hw/t8103.rs +++ b/drivers/gpu/drm/asahi/hw/t8103.rs @@ -13,6 +13,7 @@ pub(crate) const HWCONFIG: super::HwConfig = HwConfig { gpu_core: GpuCore::G13G, base_clock_hz: 24_000_000, + uat_ias: 39, uat_oas: 40, num_dies: 1, max_num_clusters: 1, diff --git a/drivers/gpu/drm/asahi/hw/t8112.rs b/drivers/gpu/drm/asahi/hw/t8112.rs index 3eba0457d76ac..bac2757175f2c 100644 --- a/drivers/gpu/drm/asahi/hw/t8112.rs +++ b/drivers/gpu/drm/asahi/hw/t8112.rs @@ -13,6 +13,7 @@ pub(crate) const HWCONFIG: super::HwConfig = HwConfig { gpu_core: GpuCore::G14G, base_clock_hz: 24_000_000, + uat_ias: 39, uat_oas: 40, num_dies: 1, max_num_clusters: 1, diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index b190a7da6fe85..29488f1ba16d1 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -537,7 +537,11 @@ impl<'a> InitDataBuilder::ver<'a> { unk_28: U64(0), // U64(0x11_00000000), // Unknown page //unk_30: U64(0x6f_ffff8000), - unk_30: U64(mmu::IOVA_UNK_PAGE), + unk_30: U64(mmu::UatGeometry { + ias: cfg.uat_ias, + oas: cfg.uat_oas, + } + .unk_page()), timestamp_area_base: U64(gpu::IOVA_KERN_TIMESTAMP_RANGE.start), // TODO: yuv matrices chip_id: cfg.chip_id, diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 4601eecedc0ed..29cfe319c5138 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -63,7 +63,6 @@ use crate::{ driver, fw, gem, - hw, mem, pgtable, slotalloc, @@ -77,11 +76,10 @@ pub(crate) use pgtable::{ UatPageTable, UAT_PGBIT, UAT_PGMSK, - UAT_PGSZ, // + UAT_PGSZ, + UAT_TOP_LEVEL_SHIFT, // }; -use pgtable::UAT_IAS; - use pin_init; const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; @@ -98,32 +96,60 @@ const UAT_USER_CTX: usize = UAT_NUM_CTX - UAT_USER_CTX_START; /// Lower/user base VA pub(crate) const IOVA_USER_BASE: u64 = UAT_PGSZ as u64; -/// Lower/user top VA -pub(crate) const IOVA_USER_TOP: u64 = 1 << (UAT_IAS as u64); -/// Lower/user VA range -pub(crate) const IOVA_USER_RANGE: Range = IOVA_USER_BASE..IOVA_USER_TOP; - -/// Upper/kernel base VA -#[cfg(CONFIG_DEV_COREDUMP)] -const IOVA_TTBR1_BASE: u64 = 0xffffff8000000000; -/// Driver-managed kernel base VA -const IOVA_KERN_BASE: u64 = 0xffffffa000000000; -/// Driver-managed kernel top VA -const IOVA_KERN_TOP: u64 = 0xffffffb000000000; -/// Driver-managed kernel VA range -const IOVA_KERN_RANGE: Range = IOVA_KERN_BASE..IOVA_KERN_TOP; -/// Full kernel VA range -#[cfg(CONFIG_DEV_COREDUMP)] -const IOVA_KERN_FULL_RANGE: Range = IOVA_TTBR1_BASE..(!UAT_PGMSK as u64); + +/// Top-level page table entry of the upper half that holds the driver-managed kernel VA range. +const IOVA_KERN_TOP_LEVEL_INDEX: u64 = 2; const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; -/// Address of a special dummy page? -//const IOVA_UNK_PAGE: u64 = 0x6f_ffff8000; -pub(crate) const IOVA_UNK_PAGE: u64 = IOVA_USER_TOP - 2 * UAT_PGSZ as u64; -/// User VA range excluding the unk page -pub(crate) const IOVA_USER_USABLE_RANGE: Range = IOVA_USER_BASE..IOVA_UNK_PAGE; +/// UAT address space geometry of a given SoC. +#[derive(Debug, Copy, Clone)] +pub(crate) struct UatGeometry { + /// Input address bits translated by each page table root. + pub(crate) ias: u32, + /// Output (physical) address bits. + pub(crate) oas: u32, +} + +impl UatGeometry { + /// Lower/user top VA + const fn user_top(&self) -> u64 { + 1 << self.ias + } + + /// Lower/user VA range + pub(crate) const fn user_range(&self) -> Range { + IOVA_USER_BASE..self.user_top() + } + + /// Address of a special dummy page? + pub(crate) const fn unk_page(&self) -> u64 { + self.user_top() - 2 * UAT_PGSZ as u64 + } + + /// User VA range excluding the unk page + pub(crate) const fn user_usable_range(&self) -> Range { + IOVA_USER_BASE..self.unk_page() + } + + /// Upper/kernel base VA + const fn upper_base(&self) -> u64 { + !(self.user_top() - 1) + } + + /// Driver-managed kernel VA range + pub(crate) const fn kernel_range(&self) -> Range { + let start = self.upper_base() + (IOVA_KERN_TOP_LEVEL_INDEX << UAT_TOP_LEVEL_SHIFT); + start..(start + (1 << UAT_TOP_LEVEL_SHIFT)) + } + + /// Full kernel VA range + #[cfg(CONFIG_DEV_COREDUMP)] + const fn upper_range(&self) -> Range { + self.upper_base()..(!UAT_PGMSK as u64) + } +} /// A pre-allocated memory region for UAT management struct UatRegion { @@ -890,7 +916,7 @@ impl UatInner { /// Top-level UAT manager object pub(crate) struct Uat { dev: driver::AsahiDevRef, - cfg: &'static hw::HwConfig, + geometry: UatGeometry, inner: Arc, slots: slotalloc::SlotAllocator, @@ -1013,7 +1039,7 @@ impl Vm { dev: &driver::AsahiDevice, uat_inner: Arc, kernel_range: Range, - cfg: &'static hw::HwConfig, + geometry: UatGeometry, ttb: Option, id: u64, ) -> Result { @@ -1021,15 +1047,15 @@ impl Vm { let is_kernel = ttb.is_some(); let page_table = if let Some(ttb) = ttb { - UatPageTable::new_with_ttb(ttb, IOVA_KERN_RANGE, cfg.uat_oas)? + UatPageTable::new_with_ttb(ttb, geometry.kernel_range(), geometry.ias, geometry.oas)? } else { - UatPageTable::new(cfg.uat_oas)? + UatPageTable::new(geometry.ias, geometry.oas)? }; let (va_range, gpuvm_range) = if is_kernel { - (IOVA_KERN_RANGE, kernel_range.clone()) + (geometry.kernel_range(), kernel_range.clone()) } else { - (IOVA_USER_RANGE, IOVA_USER_USABLE_RANGE) + (geometry.user_range(), geometry.user_usable_range()) }; let mm = mm::Allocator::new(va_range.start, va_range.range(), ())?; @@ -1461,7 +1487,7 @@ impl Uat { #[cfg(CONFIG_DEV_COREDUMP)] pub(crate) fn dump_kernel_pages(&self) -> Result> { let mut inner = self.kernel_vm.inner.exec_lock(None, false)?; - inner.page_table.dump_pages(IOVA_KERN_FULL_RANGE) + inner.page_table.dump_pages(self.geometry.upper_range()) } /// Returns the base physical address of the TTBAT region. @@ -1535,12 +1561,17 @@ impl Uat { &self.dev, self.inner.clone(), kernel_range, - self.cfg, + self.geometry, None, id, ) } + /// Returns the address space geometry of this UAT. + pub(crate) fn geometry(&self) -> UatGeometry { + self.geometry + } + /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] fn make_inner(dev: &driver::AsahiDevice) -> Result> { @@ -1572,11 +1603,11 @@ impl Uat { ) } - /// Creates a new `Uat` instance given the relevant hardware config. + /// Creates a new `Uat` instance for the given address space geometry. #[inline(never)] pub(crate) fn new( dev: &driver::AsahiDevice, - cfg: &'static hw::HwConfig, + geometry: UatGeometry, map_kernel_to_user: bool, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); @@ -1594,8 +1625,15 @@ impl Uat { } dev_info!(dev.as_ref(), "MMU: Creating kernel page tables\n"); - let kernel_lower_vm = Vm::new(dev, inner.clone(), IOVA_USER_RANGE, cfg, None, 1)?; - let kernel_vm = Vm::new(dev, inner.clone(), IOVA_KERN_RANGE, cfg, Some(ttb1), 0)?; + let kernel_lower_vm = Vm::new(dev, inner.clone(), geometry.user_range(), geometry, None, 1)?; + let kernel_vm = Vm::new( + dev, + inner.clone(), + geometry.kernel_range(), + geometry, + Some(ttb1), + 0, + )?; dev_info!(dev.as_ref(), "MMU: Kernel page tables created\n"); @@ -1603,7 +1641,7 @@ impl Uat { let uat = Self { dev: dev.into(), - cfg, + geometry, kernel_vm, kernel_lower_vm, inner, diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 7c9b54c224251..ed3505aed36bd 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -59,9 +59,8 @@ const UAT_LVMSK: u64 = (UAT_LVSZ - 1) as u64; const UAT_LEVELS: usize = 3; -/// UAT input address space -pub(crate) const UAT_IAS: usize = 39; -const UAT_IASMSK: u64 = (1u64 << UAT_IAS) - 1; +/// Number of address bits below the index of a top-level page table entry. +pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; const PTE_TYPE_BITS: u64 = 3; const PTE_TYPE_LEAF_TABLE: u64 = 3; @@ -255,27 +254,35 @@ pub(crate) struct UatPageTable { ttb: PhysicalAddr, ttb_owned: bool, va_range: Range, + ias_mask: u64, oas_mask: u64, } impl UatPageTable { - pub(crate) fn new(oas: u32) -> Result { - mod_pr_debug!("UATPageTable::new: oas={}\n", oas); + pub(crate) fn new(ias: u32, oas: u32) -> Result { + mod_pr_debug!("UATPageTable::new: ias={} oas={}\n", ias, oas); let ttb_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; let ttb = Page::into_phys(ttb_page); Ok(UatPageTable { ttb, ttb_owned: true, - va_range: 0..(1u64 << UAT_IAS), + va_range: 0..(1u64 << ias), + ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, }) } - pub(crate) fn new_with_ttb(ttb: PhysicalAddr, va_range: Range, oas: u32) -> Result { + pub(crate) fn new_with_ttb( + ttb: PhysicalAddr, + va_range: Range, + ias: u32, + oas: u32, + ) -> Result { mod_pr_debug!( - "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} oas={}\n", + "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", ttb, va_range, + ias, oas ); if ttb & (UAT_PGMSK as PhysicalAddr) != 0 { @@ -297,6 +304,7 @@ impl UatPageTable { ttb, ttb_owned: false, va_range, + ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, }) } @@ -333,10 +341,10 @@ impl UatPageTable { return Ok(()); } - let mut iova = iova_range.start & UAT_IASMSK; + let mut iova = iova_range.start & self.ias_mask; let mut last_iova = iova; // Handle the case where iova_range.end is just at the top boundary of the IAS - let end = ((iova_range.end - 1) & UAT_IASMSK) + 1; + let end = ((iova_range.end - 1) & self.ias_mask) + 1; let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); pt_addr[UAT_LEVELS - 1] = Some(self.ttb); @@ -565,7 +573,7 @@ impl UatPageTable { pub(crate) fn dump_pages(&mut self, iova_range: Range) -> Result> { let mut pages = KVVec::new(); let oas_mask = self.oas_mask; - let iova_base = self.va_range.start & !UAT_IASMSK; + let iova_base = self.va_range.start & !self.ias_mask; self.with_pages(iova_range, false, false, |iova, ptes| { let iova = iova | iova_base; for (idx, ppte) in ptes.iter().enumerate() { From 49e3d35945c397f7af5a6477540e2ee254b3bd76 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:12:48 -0400 Subject: [PATCH 0597/1579] drm/asahi: pgtable: track ownership below firmware-shared roots A borrowed root does not determine ownership of its child tables. The handoff firmware interface leaves the driver owning all child tables in its managed range; another interface may publish its own child tables into that root. Make shared child-table ownership explicit when borrowing a root. In that mode, record each driver allocation before publishing its pointer and free only recorded tables. Keep the borrowed root itself unowned. The existing handoff caller selects non-shared children, preserving its allocation and teardown behavior without an extra ownership allocation. Driver-allocated roots also retain their existing ownership behavior. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5e21536704b56c38b478516d1f65bdcfef748a17) --- drivers/gpu/drm/asahi/mmu.rs | 8 +++++++- drivers/gpu/drm/asahi/pgtable.rs | 35 +++++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 29cfe319c5138..11484ec706498 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1047,7 +1047,13 @@ impl Vm { let is_kernel = ttb.is_some(); let page_table = if let Some(ttb) = ttb { - UatPageTable::new_with_ttb(ttb, geometry.kernel_range(), geometry.ias, geometry.oas)? + UatPageTable::new_with_ttb( + ttb, + geometry.kernel_range(), + geometry.ias, + geometry.oas, + false, + )? } else { UatPageTable::new(geometry.ias, geometry.oas)? }; diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index ed3505aed36bd..5a66406cc3dc1 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -253,6 +253,10 @@ pub(crate) struct DumpedPage { pub(crate) struct UatPageTable { ttb: PhysicalAddr, ttb_owned: bool, + /// Tables below a root the driver did not allocate may belong to the bootloader or the + /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the + /// tables below the root, as with the handoff firmware interface. + driver_tables: Option>, va_range: Range, ias_mask: u64, oas_mask: u64, @@ -266,17 +270,21 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: true, + driver_tables: None, va_range: 0..(1u64 << ias), ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, }) } + /// Borrow a root; track individual child ownership only when other agents + /// can install tables below it. Handoff roots have driver-owned children. pub(crate) fn new_with_ttb( ttb: PhysicalAddr, va_range: Range, ias: u32, oas: u32, + shared_tables: bool, ) -> Result { mod_pr_debug!( "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", @@ -291,7 +299,7 @@ impl UatPageTable { if (va_range.start | va_range.end) & (UAT_PGMSK as u64) != 0 { return Err(EINVAL); } - // SAFETY: The TTB is should remain valid (if properly mapped), as it is bootloader-managed. + // SAFETY: The bootloader-managed root stays valid while this page table exists. if unsafe { Page::borrow_phys(&ttb) }.is_none() { pr_err!( "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", @@ -303,6 +311,7 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: false, + driver_tables: shared_tables.then(KVec::new), va_range, ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, @@ -362,8 +371,7 @@ impl UatPageTable { level, phys ); - // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). - unsafe { Page::from_phys(phys) }; + self.free_table(phys); } mod_pr_debug!("UATPageTable::with_pages: invalidate level {}\n", level); } @@ -396,6 +404,9 @@ impl UatPageTable { if upte_val == 0 && alloc { let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + if let Some(tables) = self.driver_tables.as_mut() { + tables.push(pt_page.phys(), GFP_KERNEL)?; + } let pt_paddr = Page::into_phys(pt_page); upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; upte.store(upte_val, Ordering::Relaxed); @@ -467,8 +478,7 @@ impl UatPageTable { level, phys ); - // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). - unsafe { Page::from_phys(phys) }; + self.free_table(phys); } } } @@ -476,6 +486,21 @@ impl UatPageTable { Ok(()) } + /// Free a page table, unless it was not allocated by the driver. + fn free_table(&mut self, phys: PhysicalAddr) { + if let Some(tables) = self.driver_tables.as_mut() { + match tables.iter().position(|&table| table == phys) { + Some(index) => { + tables.swap_remove(index); + } + None => return, + } + } + // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without + // `driver_tables`, every child table belongs to the driver. + unsafe { Page::from_phys(phys) }; + } + pub(crate) fn alloc_pages(&mut self, iova_range: Range) -> Result { mod_pr_debug!("UATPageTable::alloc_pages: {:#x?}\n", iova_range); self.with_pages(iova_range, true, false, |_, _| Ok(())) From 87cab74a2498f5df62f7697786a2743e84125b99 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:47:16 -0400 Subject: [PATCH 0598/1579] drm/asahi: add the T8140 firmware interface Provide boot, address translation and retained firmware resources for T8140, with independent render and compute queues behind the existing GPU interface. Keep visible command storage and VM pins through exact retirement, firmware drain or processor stop, and finish fences outside the device lock. Implement the G17 command ABI, per-VM status, logical ordering, asynchronous resource growth and recovery isolation together so every exposed operation has its firmware and ownership consumers. Validate shared recovery views and use bounded halted-slot identity reads to resolve ambiguous render blame. Retain unexecuted compute work for bounded replay when no firmware consumer has already depended on it. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c0e36ff2a57f2a461a96b80cebb741f5b76de7ad) --- drivers/gpu/drm/asahi/asahi.rs | 1 + drivers/gpu/drm/asahi/driver.rs | 85 +- drivers/gpu/drm/asahi/file.rs | 250 +- drivers/gpu/drm/asahi/g17/buffer.rs | 889 +++++ drivers/gpu/drm/asahi/g17/buffer/target.rs | 297 ++ drivers/gpu/drm/asahi/g17/channel.rs | 220 ++ drivers/gpu/drm/asahi/g17/command.rs | 428 +++ drivers/gpu/drm/asahi/g17/completion.rs | 390 +++ drivers/gpu/drm/asahi/g17/context.rs | 361 ++ drivers/gpu/drm/asahi/g17/context/identity.rs | 71 + drivers/gpu/drm/asahi/g17/dependency.rs | 83 + drivers/gpu/drm/asahi/g17/event.rs | 121 + drivers/gpu/drm/asahi/g17/fence.rs | 197 ++ drivers/gpu/drm/asahi/g17/freelist.rs | 575 ++++ drivers/gpu/drm/asahi/g17/fw/buffer.rs | 248 ++ drivers/gpu/drm/asahi/g17/fw/channels.rs | 522 +++ drivers/gpu/drm/asahi/g17/fw/compute.rs | 379 ++ drivers/gpu/drm/asahi/g17/fw/context.rs | 72 + drivers/gpu/drm/asahi/g17/fw/freelist.rs | 124 + drivers/gpu/drm/asahi/g17/fw/initdata.rs | 3063 +++++++++++++++++ drivers/gpu/drm/asahi/g17/fw/kick.rs | 539 +++ drivers/gpu/drm/asahi/g17/fw/mod.rs | 109 + drivers/gpu/drm/asahi/g17/fw/qos.rs | 34 + drivers/gpu/drm/asahi/g17/fw/queue.rs | 540 +++ drivers/gpu/drm/asahi/g17/fw/render.rs | 1309 +++++++ .../gpu/drm/asahi/g17/fw/render/storage.rs | 178 + drivers/gpu/drm/asahi/g17/hazard.rs | 120 + drivers/gpu/drm/asahi/g17/ids.rs | 60 + drivers/gpu/drm/asahi/g17/initdata.rs | 1305 +++++++ drivers/gpu/drm/asahi/g17/job.rs | 334 ++ drivers/gpu/drm/asahi/g17/kick.rs | 335 ++ drivers/gpu/drm/asahi/g17/mod.rs | 1153 +++++++ drivers/gpu/drm/asahi/g17/object.rs | 579 ++++ drivers/gpu/drm/asahi/g17/power.rs | 125 + drivers/gpu/drm/asahi/g17/preparation.rs | 186 + drivers/gpu/drm/asahi/g17/qos.rs | 244 ++ drivers/gpu/drm/asahi/g17/queue/admission.rs | 541 +++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 1507 ++++++++ .../gpu/drm/asahi/g17/queue/compute/exit.rs | 207 ++ .../drm/asahi/g17/queue/compute/retirement.rs | 214 ++ drivers/gpu/drm/asahi/g17/queue/mod.rs | 318 ++ drivers/gpu/drm/asahi/g17/queue/render.rs | 1459 ++++++++ .../asahi/g17/queue/render/dependencies.rs | 109 + .../gpu/drm/asahi/g17/queue/render/memory.rs | 708 ++++ .../drm/asahi/g17/queue/render/observation.rs | 85 + .../gpu/drm/asahi/g17/queue/render/parked.rs | 243 ++ .../drm/asahi/g17/queue/render/publication.rs | 164 + .../drm/asahi/g17/queue/render/retirement.rs | 348 ++ .../gpu/drm/asahi/g17/queue/render/warm.rs | 49 + drivers/gpu/drm/asahi/g17/recovery.rs | 286 ++ .../gpu/drm/asahi/g17/recovery/attribution.rs | 100 + drivers/gpu/drm/asahi/g17/recovery/slot.rs | 74 + drivers/gpu/drm/asahi/g17/runtime.rs | 869 +++++ drivers/gpu/drm/asahi/g17/runtime/backend.rs | 186 + .../drm/asahi/g17/runtime/compute_control.rs | 56 + .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 166 + .../drm/asahi/g17/runtime/compute_replay.rs | 82 + drivers/gpu/drm/asahi/g17/runtime/events.rs | 419 +++ drivers/gpu/drm/asahi/g17/runtime/render.rs | 683 ++++ .../asahi/g17/runtime/render/dependencies.rs | 34 + .../gpu/drm/asahi/g17/runtime/render/exit.rs | 39 + .../gpu/drm/asahi/g17/runtime/render/grow.rs | 309 ++ .../drm/asahi/g17/runtime/render/observe.rs | 235 ++ .../gpu/drm/asahi/g17/runtime/render/pools.rs | 224 ++ .../drm/asahi/g17/runtime/render/prepare.rs | 597 ++++ .../drm/asahi/g17/runtime/render/recovery.rs | 27 + .../drm/asahi/g17/runtime/render/settle.rs | 235 ++ .../drm/asahi/g17/runtime/render/submit.rs | 306 ++ .../drm/asahi/g17/runtime/render/teardown.rs | 164 + drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 199 ++ drivers/gpu/drm/asahi/g17/status.rs | 102 + drivers/gpu/drm/asahi/g17/teardown.rs | 222 ++ drivers/gpu/drm/asahi/g17/validation.rs | 145 + drivers/gpu/drm/asahi/gem.rs | 14 +- drivers/gpu/drm/asahi/gpu.rs | 10 + drivers/gpu/drm/asahi/hw/mod.rs | 2 + drivers/gpu/drm/asahi/hw/t8140.rs | 392 +++ drivers/gpu/drm/asahi/mem.rs | 62 + drivers/gpu/drm/asahi/mmu.rs | 897 ++++- drivers/gpu/drm/asahi/mmu/bind.rs | 191 + drivers/gpu/drm/asahi/mmu/context.rs | 232 ++ drivers/gpu/drm/asahi/mmu/lifetime.rs | 605 ++++ drivers/gpu/drm/asahi/mmu/shared.rs | 134 + drivers/gpu/drm/asahi/pgtable.rs | 93 +- drivers/gpu/drm/asahi/regs.rs | 303 +- 85 files changed, 30073 insertions(+), 99 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/buffer.rs create mode 100644 drivers/gpu/drm/asahi/g17/buffer/target.rs create mode 100644 drivers/gpu/drm/asahi/g17/channel.rs create mode 100644 drivers/gpu/drm/asahi/g17/command.rs create mode 100644 drivers/gpu/drm/asahi/g17/completion.rs create mode 100644 drivers/gpu/drm/asahi/g17/context.rs create mode 100644 drivers/gpu/drm/asahi/g17/context/identity.rs create mode 100644 drivers/gpu/drm/asahi/g17/dependency.rs create mode 100644 drivers/gpu/drm/asahi/g17/event.rs create mode 100644 drivers/gpu/drm/asahi/g17/fence.rs create mode 100644 drivers/gpu/drm/asahi/g17/freelist.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/buffer.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/channels.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/compute.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/context.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/freelist.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/initdata.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/kick.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/mod.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/qos.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/queue.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/render.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/render/storage.rs create mode 100644 drivers/gpu/drm/asahi/g17/hazard.rs create mode 100644 drivers/gpu/drm/asahi/g17/ids.rs create mode 100644 drivers/gpu/drm/asahi/g17/initdata.rs create mode 100644 drivers/gpu/drm/asahi/g17/job.rs create mode 100644 drivers/gpu/drm/asahi/g17/kick.rs create mode 100644 drivers/gpu/drm/asahi/g17/mod.rs create mode 100644 drivers/gpu/drm/asahi/g17/object.rs create mode 100644 drivers/gpu/drm/asahi/g17/power.rs create mode 100644 drivers/gpu/drm/asahi/g17/preparation.rs create mode 100644 drivers/gpu/drm/asahi/g17/qos.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/admission.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/compute.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/compute/exit.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/mod.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/memory.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/observation.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/parked.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/publication.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/retirement.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/warm.rs create mode 100644 drivers/gpu/drm/asahi/g17/recovery.rs create mode 100644 drivers/gpu/drm/asahi/g17/recovery/attribution.rs create mode 100644 drivers/gpu/drm/asahi/g17/recovery/slot.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/backend.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/compute_control.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/events.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/exit.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/grow.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/observe.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/pools.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/settle.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/submit.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/teardown.rs create mode 100644 drivers/gpu/drm/asahi/g17/status.rs create mode 100644 drivers/gpu/drm/asahi/g17/teardown.rs create mode 100644 drivers/gpu/drm/asahi/g17/validation.rs create mode 100644 drivers/gpu/drm/asahi/hw/t8140.rs create mode 100644 drivers/gpu/drm/asahi/mmu/bind.rs create mode 100644 drivers/gpu/drm/asahi/mmu/context.rs create mode 100644 drivers/gpu/drm/asahi/mmu/lifetime.rs create mode 100644 drivers/gpu/drm/asahi/mmu/shared.rs diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index 296091dced554..4842e421a09ae 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -15,6 +15,7 @@ mod file; mod float; mod fw; mod gem; +mod g17; mod gpu; mod hw; mod initdata; diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 7f07eb2429540..a40d40fa3090c 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -23,6 +23,7 @@ use kernel::{ use crate::{ debug, file, + g17, gem::AsahiObject, gpu, hw, @@ -44,7 +45,6 @@ unsafe impl Send for AsahiData {} unsafe impl Sync for AsahiData {} pub(crate) struct AsahiDriver { - #[expect(unused)] drm: ARef>, } @@ -107,52 +107,69 @@ impl drm::driver::Driver for AsahiDriver { } } -// OF Device ID table.s +/// Firmware interface selected by the hardware compatible. +pub(crate) enum ProbeConfig { + Legacy(&'static hw::HwConfig), + G17(&'static g17::Config), +} + +// OF Device ID table. kernel::of_device_table!( OF_TABLE, MODULE_OF_TABLE, ::IdInfo, [ + ( + of::DeviceId::new(c_str!("apple,agx-t8140")), + ProbeConfig::G17(&hw::t8140::CONFIG) + ), ( of::DeviceId::new(c_str!("apple,agx-t8103")), - &hw::t8103::HWCONFIG + ProbeConfig::Legacy(&hw::t8103::HWCONFIG) ), ( of::DeviceId::new(c_str!("apple,agx-t8112")), - &hw::t8112::HWCONFIG + ProbeConfig::Legacy(&hw::t8112::HWCONFIG) ), ( of::DeviceId::new(c_str!("apple,agx-t6000")), - &hw::t600x::HWCONFIG_T6000 + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) ), ( of::DeviceId::new(c_str!("apple,agx-t6001")), - &hw::t600x::HWCONFIG_T6001 + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) ), ( of::DeviceId::new(c_str!("apple,agx-t6002")), - &hw::t600x::HWCONFIG_T6002 + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) ), ( of::DeviceId::new(c_str!("apple,agx-t6020")), - &hw::t602x::HWCONFIG_T6020 + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) ), ( of::DeviceId::new(c_str!("apple,agx-t6021")), - &hw::t602x::HWCONFIG_T6021 + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) ), ( of::DeviceId::new(c_str!("apple,agx-t6022")), - &hw::t602x::HWCONFIG_T6022 + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) ), ] ); /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiDriver { - type IdInfo = &'static hw::HwConfig; + type IdInfo = ProbeConfig; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn unbind(_pdev: &platform::Device, this: Pin<&Self>) { + if let Some(gpu) = (*this.drm).gpu.as_any().downcast_ref::() { + // Driver data is dropped after devres, so CPU control must happen in unbind. + gpu.shutdown(); + } + } + /// Device probe function. fn probe( pdev: &platform::Device, @@ -162,7 +179,10 @@ impl platform::Driver for AsahiDriver { dev_info!(pdev.as_ref(), "Probing...\n"); - let cfg = info.ok_or(ENODEV)?; + let cfg = match info.ok_or(ENODEV)? { + ProbeConfig::Legacy(cfg) => *cfg, + ProbeConfig::G17(cfg) => return Self::probe_g17(pdev, cfg), + }; unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; @@ -235,3 +255,44 @@ impl platform::Driver for AsahiDriver { Ok(Self { drm }) } } + +impl AsahiDriver { + fn probe_g17(pdev: &platform::Device, cfg: &'static g17::Config) -> Result { + // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; + let res = regs::Resources::new(pdev)?; + // SAFETY: As in the legacy probe, the data is inaccessible until the fully initialized + // manager is installed below and the DRM device is registered. + // If construction fails, drm::device::Device::release leaves the data + // untouched rather than dropping an uninitialized AsahiData. + let uninit = unsafe { pin_init::pin_init_from_closure::(|_slot| Ok(())) }; + let drm: ARef = drm::device::Device::new(pdev.as_ref(), uninit)?; + let gpu = g17::Gpu::new(pdev, &drm, cfg, res.clone())?; + let data_gpu = gpu.clone() as Arc; + let data = try_pin_init!(AsahiData { + gpu: data_gpu, + pdev: pdev.into(), + resources: res, + }); + let ptr = &raw const **drm; + // SAFETY: The allocation is pinned and its data slot has not been initialized yet. + if let Err(error) = unsafe { data.__pinned_init(ptr as *mut AsahiData) } { + gpu.shutdown(); + return Err(error); + } + if let Err(error) = drm::driver::Registration::new_foreign_owned(&drm, pdev.as_ref(), 0) { + gpu.shutdown(); + return Err(error); + } + Ok(Self { drm }) + } +} + +impl Drop for AsahiDriver { + fn drop(&mut self) { + if let Some(gpu) = (*self.drm).gpu.as_any().downcast_ref::() { + // Also covers failure to allocate platform driver data after the probe body returned. + gpu.shutdown(); + } + } +} diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index cd41ff066e196..874b13099c367 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -37,6 +37,7 @@ use kernel::error::code::*; use kernel::new_mutex; use kernel::prelude::*; use kernel::sync::{ + aref::ARef, Arc, Mutex, // }; @@ -56,6 +57,17 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::File; pub(crate) const MAX_COMMANDS_PER_SUBMISSION: u32 = 64; +/// Context-view registry updates applied after a prepared VM batch commits. +enum MappingUpdate { + Map { + object: ARef, + address: u64, + size: u64, + offset: u64, + }, + Unmap(Range), +} + /// A client instance of an `mmu::Vm` address space. struct Vm { ualloc: Arc>, @@ -68,6 +80,18 @@ struct Vm { impl Drop for Vm { fn drop(&mut self) { + if self.vm.status().is_some() { + // Tracked jobs can outlive their file's VM handle. Their last guard performs + // these unmaps once no accepted work can still dereference the mappings. + if let Err(error) = self + .vm + .unmap_user_ranges(self.user_range.clone(), self.kernel_range.clone()) + { + pr_err!("Vm::Drop: deferred user unmap failed: {:?}\n", error); + } + self.vm.bo_deferred_cleanup(); + return; + } // When the user Vm is dropped, unmap everything in the user range let left_range = self.user_range.start..self.kernel_range.start; let right_range = self.kernel_range.end..self.user_range.end; @@ -295,6 +319,32 @@ impl File { let gpu = &device.gpu; + match data.param_group { + uapi::DRM_ASAHI_PARAM_GROUP_VM_STATUS if gpu.supports_vm_status() => { + return Self::get_vm_status(data, file); + } + uapi::DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR if gpu.supports_vm_status() => { + return Self::register_vm_status_mirror(data, file); + } + uapi::DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS => { + if data.pad != 0 + || data.size < core::mem::size_of::() as u64 + { + return Err(EINVAL); + } + let limits = gpu.queue_limits().ok_or(EINVAL)?; + let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; + UserSlice::new( + UserPtr::from_addr(data.pointer as _), + core::mem::size_of_val(&words), + ) + .writer() + .write(&words)?; + return Ok(0); + } + _ => {} + } + if data.param_group != 0 || data.pad != 0 { cls_pr_debug!(Errors, "get_params: Invalid arguments\n"); return Err(EINVAL); @@ -321,6 +371,14 @@ impl File { gpu.get_params(&mut params)?; + if gpu.supports_vm_status() { + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; + } + if gpu.queue_limits().is_some() { + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; + } + if *module_parameters::fault_control.value() == 0xb { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; } @@ -339,6 +397,55 @@ impl File { Ok(0) } + /// Query only file-owned VM state; this does not start recovery or wait for work. + fn get_vm_status(data: &uapi::drm_asahi_get_params, file: &DrmFile) -> Result { + let size = core::mem::size_of::(); + if data.pad != 0 || data.size < size as u64 { + return Err(EINVAL); + } + let user = UserPtr::from_addr(data.pointer as _); + let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; + if flags != 0 || pad != 0 { + return Err(EINVAL); + } + let status = Self::vm_status(file, vm_id)?; + UserSlice::new(user, size) + .writer() + .write(&[vm_id, 0, status.get() as u32, 0])?; + Ok(0) + } + + /// Register a file-owned GEM word as the mirror of a file-owned VM's status. + fn register_vm_status_mirror(data: &uapi::drm_asahi_get_params, file: &DrmFile) -> Result { + let size = core::mem::size_of::(); + if data.pad != 0 || data.size < size as u64 { + return Err(EINVAL); + } + let [vm_id, handle, offset, flags] = + UserSlice::new(UserPtr::from_addr(data.pointer as _), size) + .reader() + .read::<[u32; 4]>()?; + if flags != 0 || handle == 0 || offset % core::mem::align_of::() as u32 != 0 { + return Err(EINVAL); + } + let status = Self::vm_status(file, vm_id)?; + let owner = gem::ObjectRef::new(gem::Object::lookup_handle(file, handle)?); + status.attach_mirror(owner, offset as usize)?; + Ok(0) + } + + fn vm_status(file: &DrmFile, vm_id: u32) -> Result> { + let vms = file.inner().vms(); + let guard = vms.lock(); + Ok(guard + .get(vm_id as usize) + .ok_or(ENOENT)? + .vm + .status() + .ok_or(EINVAL)? + .clone()) + } + /// IOCTL: vm_create: Create a new `Vm`. pub(crate) fn vm_create( device: &AsahiDevice, @@ -553,6 +660,10 @@ impl File { return Err(EINVAL); } + if device.gpu.as_any().is::() { + return Self::vm_bind_g17(data, file); + } + let vm_id = data.vm_id.try_into()?; let mut vec = KVec::new(); @@ -569,6 +680,118 @@ impl File { Ok(0) } + /// Validate all operations and retain their objects before changing a tracked VM. + fn vm_bind_g17(data: &uapi::drm_asahi_vm_bind, file: &DrmFile) -> Result { + let count = usize::try_from(data.num_binds)?; + let stride = usize::try_from(data.stride)?; + let size = stride.checked_mul(count).ok_or(EOVERFLOW)?; + let mut bytes = KVec::new(); + UserSlice::new(UserPtr::from_addr(data.userptr as _), size) + .reader() + .read_all(&mut bytes, GFP_KERNEL)?; + let (vm, user_range, kernel_range) = { + let vms = file.inner().vms(); + let guard = vms.lock(); + let entry = guard.get(data.vm_id as usize).ok_or(ENOENT)?; + ( + entry.vm.clone(), + entry.user_range.clone(), + entry.kernel_range.clone(), + ) + }; + let mut batch = mmu::PreparedUserBindBatch::new(count)?; + let mut updates = KVec::with_capacity(count, GFP_KERNEL)?; + let mut unmaps = 0; + // A fresh reader per declared stride also skips extension bytes in longer records. + for bytes in bytes.chunks_exact(stride) { + let op: uapi::drm_asahi_gem_bind_op = Reader::new(bytes).read_up_to(stride)?; + if op.range == 0 || (op.addr | op.range | op.offset) & mmu::UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + let range = op.addr..op.addr.checked_add(op.range).ok_or(EINVAL)?; + if !user_range.is_superset(range.clone()) + || kernel_range.overlaps(range.clone()) + || vm.driver_range_overlaps(range.clone()) + { + return Err(EINVAL); + } + if op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND != 0 { + if op.offset != 0 + || op.flags != uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND + || op.handle != 0 + { + return Err(EINVAL); + } + batch.push_unmap(vm.prepare_user_unmap(op.addr, op.range)?)?; + updates.push(MappingUpdate::Unmap(range), GFP_KERNEL)?; + unmaps += 1; + continue; + } + vm.wait_for_user_map_admission()?; + if op.flags + & !(uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE) + != 0 + { + return Err(EINVAL); + } + let read = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ != 0; + let write = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE != 0; + let prot = match (read, write) { + (true, true) => mmu::PROT_GPU_SHARED_RW, + (true, false) => mmu::PROT_GPU_SHARED_RO, + (false, true) => mmu::PROT_GPU_SHARED_WO, + (false, false) => return Err(EINVAL), + }; + let single_page = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE != 0; + let object = gem::Object::lookup_handle(file, op.handle)?; + let accessed = if single_page { + mmu::UAT_PGSZ as u64 + } else { + op.range + }; + if op.offset.checked_add(accessed).ok_or(EINVAL)? > u64::try_from(object.size())? { + return Err(EINVAL); + } + vm.validate_context_binding(op.addr, op.range, single_page)?; + batch.push_map(vm.prepare_bind_object( + &object, + op.addr, + op.range, + op.offset, + prot, + single_page, + )?)?; + updates.push( + MappingUpdate::Map { + object, + address: op.addr, + size: op.range, + offset: op.offset, + }, + GFP_KERNEL, + )?; + } + vm.reserve_deferred_user_unmaps(unmaps)?; + vm.commit_prepared_user_bind_batch(&mut batch)?; + for update in updates { + match update { + MappingUpdate::Map { + object, + address, + size, + offset, + } => { + vm.track_context_binding(object, address, size, offset)?; + } + MappingUpdate::Unmap(range) => vm.untrack_context_range(range), + } + } + vm.bo_deferred_cleanup(); + Ok(0) + } + pub(crate) fn do_gem_bind_unbind( vm_id: usize, data: &uapi::drm_asahi_gem_bind_op, @@ -755,6 +978,7 @@ impl File { let vm = file_vm.borrow().vm.clone(); core::mem::drop(file_vm); vm.drop_mappings(bo)?; + vm.untrack_context_object(bo); if idx == usize::MAX { break; } @@ -1006,6 +1230,21 @@ impl File { data.queue_id, id ); + let mut vec = KVec::new(); + let copy_commands = |bytes: &mut KVec| { + UserSlice::new( + UserPtr::from_addr(data.cmdbuf as _), + data.cmdbuf_size as usize, + ) + .reader() + .read_all(bytes, GFP_KERNEL) + }; + let guarded = gpu.as_any().is::(); + // Once G17 output syncobjs are resolved, its queue must install a fence + // on every failure. Take the fallible userspace snapshot before that point. + if guarded { + copy_commands(&mut vec)?; + } let syncs = SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; @@ -1017,17 +1256,12 @@ impl File { id ); - let mut vec = KVec::new(); - // Copy the command buffer into the kernel. Because we need to iterate // the command buffer twice, we do this in one big copy_from_user to // avoid TOCTOU issues. - let reader = UserSlice::new( - UserPtr::from_addr(data.cmdbuf as _), - data.cmdbuf_size as usize, - ) - .reader(); - reader.read_all(&mut vec, GFP_KERNEL)?; + if !guarded { + copy_commands(&mut vec)?; + } let objects = file.inner().objects(); let ret = queue diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs new file mode 100644 index 0000000000000..942d5047e5b44 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -0,0 +1,889 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Parameter-buffer IDs, scene ownership and sparse tiler heap growth. +//! +//! The device mutex serializes the shared ID allocator. A render graph owns +//! its buffer ID independently of the submission references using that graph; +//! a published command keeps its reference until firmware retirement is witnessed. + +use super::fw::buffer as fw; +use crate::hw::t8140::resources as cfg; +pub(crate) mod target; + +use kernel::prelude::*; + +const BUFFER_IDS: usize = 127; + +#[derive(Copy, Clone)] +struct BufferId { + references: u8, + owner: bool, + token: u64, +} +impl BufferId { + fn free(self) -> bool { + self.references == 0 && !self.owner + } +} + +/// Device-wide identities for retained render parameter buffers. +pub(crate) struct BufferIds { + entries: [BufferId; BUFFER_IDS], + returned: [u8; BUFFER_IDS], + returned_count: usize, + next_token: u64, +} +impl BufferIds { + pub(crate) fn new() -> Self { + let mut returned = [0; BUFFER_IDS]; + for (index, entry) in returned.iter_mut().enumerate() { + *entry = (BUFFER_IDS - index - 1) as u8; + } + Self { + entries: [BufferId { + references: 0, + owner: false, + token: 0, + }; BUFFER_IDS], + returned, + returned_count: BUFFER_IDS, + next_token: 0, + } + } + fn remove_returned(&mut self, id: u8) { + if let Some(at) = self.returned[..self.returned_count] + .iter() + .position(|entry| *entry == id) + { + self.returned.copy_within(at + 1..self.returned_count, at); + self.returned_count -= 1; + } + } + fn return_id(&mut self, id: u8) { + if !self.returned[..self.returned_count].contains(&id) && self.returned_count < BUFFER_IDS { + self.returned[self.returned_count] = id; + self.returned_count += 1; + } + } + fn fresh_token(&mut self) -> u64 { + let token = if self.next_token == u64::MAX { + 0 + } else { + self.next_token + }; + self.next_token = token + 1; + token + } + pub(crate) fn allocate_render(&mut self) -> Result<(u8, u64)> { + let id = if self.returned_count != 0 { + self.returned_count -= 1; + self.returned[self.returned_count] + } else { + self.entries + .iter() + .position(|entry| entry.free()) + .ok_or(ENOSPC)? as u8 + }; + if !self.entries[usize::from(id)].free() { + return Err(EIO); + } + let token = self.fresh_token(); + self.entries[usize::from(id)] = BufferId { + references: 1, + owner: false, + token, + }; + Ok((id, token)) + } + + /// Retains the existing identity for a graph independently of its jobs. + pub(crate) fn reserve_owner(&mut self, id: u8) -> Result { + let entry = self.entries.get(usize::from(id)).ok_or(EINVAL)?; + if entry.owner { + return Ok(()); + } + if entry.free() { + let token = self.fresh_token(); + self.entries[usize::from(id)].token = token; + } + self.entries[usize::from(id)].owner = true; + self.remove_returned(id); + Ok(()) + } + pub(crate) fn acquire_owner(&mut self, id: u8) -> Result<(u8, u64)> { + let entry = self.entries.get_mut(usize::from(id)).ok_or(EINVAL)?; + if !entry.owner { + return Err(EINVAL); + } + entry.references = entry.references.checked_add(1).ok_or(ENOSPC)?; + Ok((id, entry.token)) + } + /// Preflight shared completion bookkeeping before any of its counters change. + pub(crate) fn can_release(&self, id: u8) -> Result { + let entry = self.entries.get(usize::from(id)).ok_or(EINVAL)?; + if entry.references == 0 { + return Err(EINVAL); + } + Ok(()) + } + pub(crate) fn release(&mut self, id: u8) -> Result { + let entry = self.entries.get_mut(usize::from(id)).ok_or(EINVAL)?; + entry.references = entry.references.checked_sub(1).ok_or(EINVAL)?; + if entry.free() { + self.return_id(id); + } + Ok(()) + } + pub(crate) fn unreserve_owner(&mut self, id: u8) -> Result { + let entry = self.entries.get_mut(usize::from(id)).ok_or(EINVAL)?; + if !entry.owner { + return Ok(()); + } + entry.owner = false; + if entry.free() { + self.return_id(id); + } + Ok(()) + } +} + +/// Each scene and its shorter scratch slot are one lifetime. A completed +/// paired render releases both; merely publishing a later scene does not. +pub(crate) struct Scenes { + owners: [Option<(u64, u8)>; fw::SCRATCH_SLOTS], + next: usize, +} +impl Scenes { + pub(crate) const fn new() -> Self { + Self { + owners: [None; fw::SCRATCH_SLOTS], + next: fw::FIRST_SCENE, + } + } + pub(crate) fn select(&self, ordinal: u64) -> Result { + if let Some((_, scene)) = self + .owners + .iter() + .flatten() + .find(|(owner, _)| *owner == ordinal) + { + return Ok(usize::from(*scene)); + } + (0..fw::SCENES) + .map(|offset| (self.next + offset) % fw::SCENES) + .find(|scene| self.owners[scene % fw::SCRATCH_SLOTS].is_none()) + .ok_or(ENOSPC) + } + pub(crate) fn commit(&mut self, ordinal: u64, scene: usize) -> Result { + if self.select(ordinal)? != scene { + return Err(EINVAL); + } + let slot = &mut self.owners[scene % fw::SCRATCH_SLOTS]; + if slot.is_none() { + *slot = Some((ordinal, scene as u8)); + self.next = (scene + 1) % fw::SCENES; + } + Ok(()) + } + pub(crate) fn release(&mut self, ordinal: u64) -> Result { + let slot = self + .owners + .iter_mut() + .find(|entry| entry.is_some_and(|(owner, _)| owner == ordinal)) + .ok_or(EINVAL)?; + *slot = None; + Ok(()) + } +} + +/// A snapshot of the block ring; all values are modulo its fixed capacity. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Cursors { + pub(crate) total: u32, + pub(crate) committed: u32, + pub(crate) read: u32, +} +impl Cursors { + fn valid(self) -> bool { + [self.total, self.committed, self.read] + .into_iter() + .all(|v| (v as usize) < cfg::TVB_BLOCK_SLOTS) + } + /// Firmware-paused growth permits consumption of the already committed + /// prefix. Retired-host growth requires the complete snapshot unchanged. + pub(crate) fn permits(self, after: Self, firmware_request: bool) -> bool { + if !self.valid() + || !after.valid() + || self.total != after.total + || self.committed != after.committed + { + return false; + } + if !firmware_request { + return self == after; + } + let capacity = cfg::TVB_BLOCK_SLOTS as u32; + (after.read + capacity - self.read) % capacity + <= (self.committed + capacity - self.read) % capacity + } +} + +/// Checked block-table indices for one bulk allocation. One ring slot stays +/// empty even when all physical backing has been retained by the owner. +pub(crate) struct Growth { + before: Cursors, + added: usize, + retained: usize, + target: usize, +} +impl Growth { + pub(crate) fn new(before: Cursors, added: usize) -> Result { + if !before.valid() || before.total != before.committed || added == 0 { + return Err(EINVAL); + } + let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) + % cfg::TVB_BLOCK_SLOTS; + if retained.checked_add(added).ok_or(EOVERFLOW)? >= cfg::TVB_BLOCK_SLOTS { + return Err(ENOSPC); + } + let target = (before.total as usize + added) % cfg::TVB_BLOCK_SLOTS; + Ok(Self { + before, + added, + retained, + target, + }) + } + pub(crate) fn target(&self) -> u32 { + self.target as u32 + } + pub(crate) fn retained(&self) -> usize { + self.retained + } + pub(crate) fn available(&self) -> usize { + self.retained + self.added + } + pub(crate) fn added_slot(&self, index: usize) -> Result { + if index >= self.added { + return Err(EINVAL); + } + Ok((self.before.total as usize + index) % cfg::TVB_BLOCK_SLOTS) + } + pub(crate) fn available_slot(&self, index: usize) -> Result { + if index >= self.available() { + return Err(EINVAL); + } + Ok((self.before.read as usize + index) % cfg::TVB_BLOCK_SLOTS) + } +} + +pub(crate) fn firmware_growth_target(current: usize) -> usize { + current + .saturating_add(fw::GROW_BLOCKS) + .min(cfg::TVB_MAX_BLOCKS) +} + +/// The high-water sample estimates capacity, not current occupancy. A growth +/// reserves three times the demand only once twice that demand no longer fits. +pub(crate) fn automatic_growth_target(used_pages: usize, current: usize) -> Option { + let maximum = fw::NORMAL_MAX_BLOCKS; + if current >= maximum { + return None; + } + let need = used_pages + .saturating_mul(2) + .div_ceil(cfg::TVB_PAGES_PER_BLOCK) + .min(maximum); + if need <= current { + return None; + } + Some( + used_pages + .saturating_mul(3) + .div_ceil(cfg::TVB_PAGES_PER_BLOCK) + .min(maximum), + ) +} + +use super::object::{CpuMap, KernelObject}; +use crate::{driver::AsahiDevice, mmu}; +use core::ops::Range; + +pub(crate) fn compact_alias( + object: &KernelObject, + vm: &mmu::Vm, + source: Range, + align: u64, + prot: mmu::Prot, + guard: usize, +) -> Result { + for range in cfg::compact_ranges() { + match object.map_range(vm, source.clone(), range, align, prot, guard) { + Ok(mapping) => return Ok(mapping), + Err(error) if error == ENOSPC => {} + Err(error) => return Err(error), + } + } + Err(ENOSPC) +} + +/// Allocated aliases remain private until the owner installs them in its VM +/// cache. On rollback mappings are destroyed before backing memory. +pub(crate) struct TvbGrowth { + mappings: KVec, + addresses: KVec, + extension: KernelObject, +} + +/// Sparse 128-KiB data blocks at 160-KiB backing strides. Firmware owns their +/// contents; CPU zeroing occurs only on fresh backing, never on pool reuse. +pub(crate) struct Tvb { + addresses: KVec, + // Retain sparse-block backing until the pool and its aliases retire. + _primary: KernelObject, + extensions: KVec, +} +impl Tvb { + fn bytes(blocks: usize) -> Result { + blocks + .checked_sub(1) + .and_then(|v| v.checked_mul(cfg::TVB_BLOCK_STRIDE as usize)) + .and_then(|v| v.checked_add(cfg::TVB_BLOCK_SIZE)) + .ok_or(EOVERFLOW) + } + fn map_block( + object: &KernelObject, + vm: &mmu::Vm, + relative: usize, + ) -> Result { + let start = relative + .checked_mul(cfg::TVB_BLOCK_STRIDE as usize) + .ok_or(EOVERFLOW)?; + let source = start..start.checked_add(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; + let mapping = compact_alias( + object, + vm, + source, + cfg::TVB_PAGE, + mmu::PROT_GPU_SHARED_RW, + cfg::TVB_BLOCK_STRIDE as usize - cfg::TVB_BLOCK_SIZE, + )?; + fw::compact_page(mapping.iova())?; + Ok(mapping) + } + pub(crate) fn new( + dev: &AsahiDevice, + vm: &mmu::Vm, + blocks: usize, + ) -> Result<(Self, KVec)> { + if blocks == 0 || blocks > cfg::TVB_MAX_BLOCKS { + return Err(ERANGE); + } + let backing = KernelObject::backing(dev, Self::bytes(blocks)?, CpuMap::WriteCombined)?; + let mut mappings = KVec::with_capacity(blocks, GFP_KERNEL)?; + let mut addresses = KVec::with_capacity(blocks, GFP_KERNEL)?; + for index in 0..blocks { + let mapping = Self::map_block(&backing, vm, index)?; + addresses.push(mapping.iova(), GFP_KERNEL)?; + mappings.push(mapping, GFP_KERNEL)?; + } + Ok(( + Self { + addresses, + _primary: backing, + extensions: KVec::new(), + }, + mappings, + )) + } + pub(crate) fn blocks(&self) -> usize { + self.addresses.len() + } + pub(crate) fn addresses(&self) -> &[u64] { + &self.addresses + } + + pub(crate) fn prepare_growth( + &mut self, + dev: &AsahiDevice, + vm: &mmu::Vm, + target: usize, + ) -> Result> { + if target <= self.blocks() { + return Ok(None); + } + if target > cfg::TVB_MAX_BLOCKS { + return Err(ERANGE); + } + let added = target - self.blocks(); + self.extensions.reserve(1, GFP_KERNEL)?; + self.addresses.reserve(added, GFP_KERNEL)?; + let backing = KernelObject::backing(dev, Self::bytes(added)?, CpuMap::WriteCombined)?; + let mut mappings = KVec::with_capacity(added, GFP_KERNEL)?; + let mut addresses = KVec::with_capacity(added, GFP_KERNEL)?; + for relative in 0..added { + let mapping = Self::map_block(&backing, vm, relative)?; + addresses.push(mapping.iova(), GFP_KERNEL)?; + mappings.push(mapping, GFP_KERNEL)?; + } + Ok(Some(TvbGrowth { + mappings, + addresses, + extension: backing, + })) + } +} + +use super::object::Allocator; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use kernel::sync::Arc; + +/// First-free metrics entries are shared by all parameter-buffer graphs. +/// An entry returns only when the graph and its firmware references retire. +pub(crate) struct MetricsIds { + used: [AtomicU64; 2], +} +impl MetricsIds { + pub(crate) fn new() -> Self { + Self { + used: [ + AtomicU64::new(0), + AtomicU64::new(!((1u64 << (fw::METRICS_SLOTS - 64)) - 1)), + ], + } + } +} + +pub(crate) struct MetricsLease { + _backing: Arc, + allocator: Arc, + slot: usize, + addresses: [u64; 2], +} +impl MetricsLease { + /// Clears only the selected retired metrics entry before reusing it. + pub(crate) fn new( + ids: &Arc, + object: &Arc, + bases: [u64; 2], + ) -> Result { + let mut selected = None; + for (bank, used) in ids.used.iter().enumerate() { + let mut value = used.load(Ordering::Acquire); + loop { + let bit = value.trailing_ones(); + if bit == 64 { + break; + } + match used.compare_exchange_weak( + value, + value | (1u64 << bit), + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + selected = Some(bank * 64 + bit as usize); + break; + } + Err(current) => value = current, + } + } + if selected.is_some() { + break; + } + } + let slot = selected.ok_or(ENOSPC)?; + // Construct the lease before any fallible address/view validation so + // every constructor failure returns the selected bit. + let mut lease = Self { + _backing: object.clone(), + allocator: ids.clone(), + slot, + addresses: [0; 2], + }; + let offset = slot * fw::METRICS_STRIDE; + lease.addresses = [ + bases[0].checked_add(offset as u64).ok_or(EOVERFLOW)?, + bases[1].checked_add(offset as u64).ok_or(EOVERFLOW)?, + ]; + let words = object.pointer(offset, fw::METRICS_STRIDE)?.cast::(); + object.word(offset)?; + for index in 0..fw::SCENES { + // SAFETY: The complete naturally aligned metrics entry is inside + // the retained object; the exclusive lease excludes other owners. + unsafe { words.add(index).write_volatile(0) }; + } + Ok(lease) + } +} +impl Drop for MetricsLease { + fn drop(&mut self) { + self.allocator.used[self.slot / 64] + .fetch_and(!(1u64 << (self.slot % 64)), Ordering::Release); + } +} + +pub(crate) struct ManagerArgs { + pub(crate) scratch: u64, + pub(crate) discard: u64, + pub(crate) buffer_id: u8, + pub(crate) generation: u64, + pub(crate) first_ordinal: u64, +} + +/// A retained parameter-buffer graph and all of its sparse backing. Client +/// mappings live in the VM's owner cache; graph replacement never returns a +/// published backing early merely because an error fence was signalled. +pub(crate) struct Manager { + _page_list: mmu::KernelMapping, + graph: KernelObject, + tvb: Tvb, + metrics: MetricsLease, + scenes: Scenes, + page_list_client: u64, + scratch: u64, + buffer_id: u8, + generation: u64, +} + +pub(crate) struct Submitted<'a> { + word: &'a AtomicU32, + next: u32, +} +impl Submitted<'_> { + pub(crate) fn commit(self) { + self.word.store(self.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + } +} + +impl Manager { + pub(crate) fn new( + alloc: &Allocator<'_>, + vm: &mmu::Vm, + tvb: Tvb, + metrics: MetricsLease, + args: ManagerArgs, + ) -> Result { + let mut graph = alloc.kernel( + fw::SIZE, + cfg::TVB_PAGE, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let source = fw::PAGE_LIST..fw::PAGE_LIST + fw::PAGE_LIST_BYTES; + let page_list = compact_alias( + &graph, + vm, + source, + cfg::TVB_PAGE, + mmu::PROT_GPU_FW_SHARED_RW, + mmu::UAT_PGSZ, + )?; + let address = graph.gpu_va(); + graph.initialize::(0, |view| { + view.init(&fw::GraphArgs { + fw_va: address, + page_list_client: page_list.iova(), + metrics: metrics.addresses, + blocks: tvb.addresses(), + scratch: args.scratch, + discard: args.discard, + buffer_id: args.buffer_id.into(), + generation: args.generation, + }) + })?; + fence(Ordering::SeqCst); + let mut scenes = Scenes::new(); + scenes.commit(args.first_ordinal, fw::FIRST_SCENE)?; + let page_list_client = page_list.iova(); + Ok(Self { + _page_list: page_list, + graph, + tvb, + metrics, + scenes, + page_list_client, + scratch: args.scratch, + buffer_id: args.buffer_id, + generation: args.generation, + }) + } + pub(crate) fn buffer_id(&self) -> u8 { + self.buffer_id + } + pub(crate) fn blocks(&self) -> usize { + self.tvb.blocks() + } + pub(crate) fn state_va(&self) -> u64 { + self.graph.gpu_va() + } + pub(crate) fn page_list_va(&self) -> u64 { + self.page_list_client + } + pub(crate) fn trailer_va(&self) -> u64 { + self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (fw::SCENES * size_of::()) as u64 + } + pub(crate) fn scene_va(&self, index: usize) -> Result { + if index >= fw::SCENES { + return Err(EINVAL); + } + Ok(self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (index * size_of::()) as u64) + } + pub(crate) fn scene_registers(&self, index: usize) -> Result<(u64, u64)> { + fw::scene_registers(index, self.scratch, self.metrics.addresses[1]) + } + pub(crate) fn prepare_scene(&mut self, ordinal: u64) -> Result { + let scene = self.scenes.select(ordinal)?; + let offset = fw::SCENE_TABLE + scene * size_of::(); + let used = self + .graph + .word(offset + core::mem::offset_of!(fw::Scene, used))?; + let generation = self + .graph + .pointer(offset + core::mem::offset_of!(fw::Scene, generation), 8)?; + let id = self + .graph + .word(core::mem::offset_of!(fw::State, buffer_id))?; + used.store(0, Ordering::Relaxed); + // SAFETY: The complete packed field was checked before any write. + // This scene is exclusively retained or still unclaimed. + unsafe { generation.cast::().write_unaligned(self.generation) }; + id.store(u32::from(self.buffer_id), Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(scene) + } + pub(crate) fn claim_scene(&mut self, ordinal: u64, scene: usize) -> Result { + self.scenes.commit(ordinal, scene) + } + pub(crate) fn retire_scene(&mut self, ordinal: u64) -> Result { + self.scenes.release(ordinal) + } + pub(crate) fn prepare_submission(&mut self) -> Result> { + let word = self.graph.word(fw::COUNTER)?; + Ok(Submitted { + word, + next: word.load(Ordering::Relaxed).wrapping_add(1), + }) + } + pub(crate) fn automatic_target(&self) -> Result> { + let max_pages = self + .graph + .word(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages))?; + let reset = self + .graph + .word(fw::CONTROL + core::mem::offset_of!(fw::Control, reset))?; + fence(Ordering::Acquire); + let pages = max_pages.load(Ordering::Relaxed); + fence(Ordering::Release); + reset.store(1, Ordering::Relaxed); + Ok(automatic_growth_target(pages as usize, self.blocks())) + } + fn cursors(&self) -> Result { + Ok(Cursors { + total: self.graph.word(fw::CONTROL)?.load(Ordering::Relaxed), + committed: self.graph.word(fw::CONTROL + 4)?.load(Ordering::Relaxed), + read: self.graph.word(fw::CONTROL + 8)?.load(Ordering::Relaxed), + }) + } +} + +/// All allocations and inventory words prepared without publishing shared state. +/// The pair remains exclusively borrowed until commit or discard. +pub(crate) struct PreparedGrowth { + owner: u64, + previous: usize, + target: usize, + before: Cursors, + plan: Growth, + growth: TvbGrowth, + added: KVec<(usize, u32)>, + rebuilt: KVec, + pages: u32, + descriptor: Option<[u32; 4]>, +} +impl Manager { + /// Validate every retained sparse block and its reserved trailing guard in + /// one VM snapshot before constructing more aliases. + pub(crate) fn validate_vm(&self, vm: &mmu::Vm) -> Result { + if self.blocks() < cfg::TVB_INITIAL_PAGES.len() + || self.blocks() > cfg::TVB_MAX_BLOCKS + || !vm.covers_sparse_blocks_with_guards( + self.tvb.addresses(), + cfg::TVB_BLOCK_SIZE as u64, + cfg::TVB_BLOCK_STRIDE, + ) + { + return Err(EFAULT); + } + Ok(()) + } + + /// Allocate growth and precompute its words while the pair is exclusively + /// borrowed. A descriptor snapshot requests idle-host growth; None leaves + /// the firmware-owned page list and PB state untouched. + pub(crate) fn prepare_growth( + &mut self, + dev: &AsahiDevice, + vm: &mmu::Vm, + target: usize, + descriptor: Option<[u32; 4]>, + ) -> Result> { + let previous = self.blocks(); + if target <= previous { + return Ok(None); + } + self.validate_vm(vm)?; + let Some(growth) = self.tvb.prepare_growth(dev, vm, target)? else { + return Ok(None); + }; + let before = self.cursors()?; + let plan = Growth::new(before, target - previous)?; + let firmware_request = descriptor.is_none(); + let pages = target + .checked_mul(cfg::TVB_PAGES_PER_BLOCK) + .ok_or(EOVERFLOW)? as u32; + let mut added = KVec::with_capacity(growth.addresses.len(), GFP_KERNEL)?; + for (index, &address) in growth.addresses.iter().enumerate() { + let first = fw::compact_page(address)?; + first + .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .ok_or(EOVERFLOW)?; + added.push((plan.added_slot(index)?, first), GFP_KERNEL)?; + } + let rebuilt_count = if firmware_request { + 0 + } else { + plan.available() * cfg::TVB_PAGES_PER_BLOCK + }; + let mut rebuilt = KVec::with_capacity(rebuilt_count, GFP_KERNEL)?; + if !firmware_request { + if self.cursors()? != before { + return Err(EAGAIN); + } + for ordinal in 0..plan.available() { + let first = if ordinal < plan.retained() { + let slot = plan.available_slot(ordinal)?; + self.graph + .word(fw::BLOCK_TABLE + slot * size_of::())? + .load(Ordering::Relaxed) + } else { + added[ordinal - plan.retained()].1 + }; + first + .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .ok_or(EOVERFLOW)?; + for page in 0..cfg::TVB_PAGES_PER_BLOCK { + rebuilt.push(first + page as u32, GFP_KERNEL)?; + } + } + } + let descriptor = + descriptor.map(|old| fw::reload_descriptor(old, self.page_list_client, pages)); + Ok(Some(PreparedGrowth { + owner: self.graph.gpu_va(), + previous, + target, + before, + plan, + growth, + added, + rebuilt, + pages, + descriptor, + })) + } + + /// The caller holds the device mutex after revalidating its preparation + /// epoch. The final alias installation and all firmware writes occur under + /// that same exclusion. The returned PB row is committed there as well. + pub(crate) fn commit_growth( + &mut self, + prepared: &mut Option, + install: impl FnOnce(usize, usize, &mut KVec) -> Result, + ) -> Result> { + let candidate = prepared.as_mut().ok_or(EINVAL)?; + if self.graph.gpu_va() != candidate.owner || self.blocks() != candidate.previous { + return Err(EBUSY); + } + let firmware_request = candidate.descriptor.is_none(); + let base = self.graph.pointer(0, fw::SIZE)?.cast::(); + // Bounds, actual alignment and every reserved host entry are checked + // before transferring aliases or consuming the caller-owned plan. + self.graph.word(0)?; + if self.tvb.extensions.len() == self.tvb.extensions.capacity() + || candidate.growth.addresses.len() + > self.tvb.addresses.capacity() - self.tvb.addresses.len() + { + return Err(EIO); + } + if !candidate.before.permits(self.cursors()?, firmware_request) { + return Err(EAGAIN); + } + // The callback retains its input on error. The caller can therefore + // drop every rejected mapping/backing after releasing the device lock. + install( + candidate.previous, + candidate.target, + &mut candidate.growth.mappings, + )?; + if !candidate.growth.mappings.is_empty() { + return Err(EIO); + } + let PreparedGrowth { + before, + plan, + growth, + added, + rebuilt, + pages, + descriptor, + .. + } = prepared.take().ok_or(EIO)?; + let TvbGrowth { + addresses, + extension, + .. + } = growth; + + // From here all vector capacity, indices, addresses and CPU views have + // been checked. Retain backing before publishing its block entries. + for address in addresses.into_iter() { + append_reserved(&mut self.tvb.addresses, address); + } + let write = |offset: usize, value: u32| { + // SAFETY: The full graph and base alignment were checked above. + // Every offset is an aligned field or a checked array index of + // Graph. No mutable reference aliases firmware-owned fields. + unsafe { AtomicU32::from_ptr(base.add(offset / 4)).store(value, Ordering::Relaxed) }; + }; + for (slot, first) in added.into_iter() { + let at = fw::BLOCK_TABLE + slot * size_of::(); + write(at, first); + write(at + 4, 0); + } + if descriptor.is_some() { + for (entry, value) in rebuilt.into_iter().enumerate() { + write(fw::PAGE_LIST + entry * 4, value); + } + write(core::mem::offset_of!(fw::State, pages), pages); + write(core::mem::offset_of!(fw::State, producer), plan.target()); + write(core::mem::offset_of!(fw::State, consumer), before.read); + write(core::mem::offset_of!(fw::State, last_page), pages - 1); + } + append_reserved(&mut self.tvb.extensions, extension); + write(fw::CONTROL, plan.target()); + fence(Ordering::SeqCst); + write(fw::CONTROL + 4, plan.target()); + fence(Ordering::SeqCst); + Ok(descriptor) + } +} + +/// Caller reserves all needed entries before any firmware-visible mutation. +fn append_reserved(values: &mut KVec, value: T) { + let length = values.len(); + values.spare_capacity_mut()[0].write(value); + // SAFETY: All callers preflighted this spare entry and initialized it once. + unsafe { values.set_len(length + 1) }; +} diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs new file mode 100644 index 0000000000000..07ead6056329e --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Rotating tile-map/heap metadata and the retained tail-pointer cache. +//! Larger targets replace only the objects that need more capacity. Every old +//! backing keeps the exact commands that still reference it until retirement. + +use super::compact_alias; +use crate::g17::{ + fw::render::RenderPass, + object::{CpuMap, KernelObject}, +}; +use crate::{ + driver::AsahiDevice, + hw::t8140::{queues, resources as cfg}, + mmu, +}; +use kernel::prelude::*; + +const DEPTH: usize = queues::RENDER_DEPTH as usize; + +fn aligned(value: usize, alignment: usize) -> Result { + value + .checked_add(alignment - 1) + .map(|v| v & !(alignment - 1)) + .ok_or(EOVERFLOW) +} + +/// Per-command geometry, independent of the capacity already allocated. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Layout { + pub(crate) layer_offset: usize, + pub(crate) heap_offset: usize, + pub(crate) used: usize, + pub(crate) tpc_size: usize, + pub(crate) tvb_blocks: usize, +} +impl Layout { + pub(crate) fn new(pass: &RenderPass, clusters: u32) -> Result { + let width = pass.width as usize; + let height = pass.height as usize; + let layers = pass.layers as usize; + let (uw, uh) = (pass.utile_width as usize, pass.utile_height as usize); + if width == 0 + || height == 0 + || layers == 0 + || clusters == 0 + || uw == 0 + || uh == 0 + || 32 % uw != 0 + || 32 % uh != 0 + { + return Err(EINVAL); + } + let utiles = (32 / uw) * (32 / uh); + let tiles_x = width.div_ceil(32); + let tiles_y = height.div_ceil(32); + let macro_x = aligned(tiles_x.div_ceil(4), 4)?; + let macro_y = aligned(tiles_y.div_ceil(4), 4)?; + let macro_tiles = macro_x.checked_mul(macro_y).ok_or(EOVERFLOW)?; + let tilemap_size = aligned( + 5usize + .checked_mul(macro_tiles) + .and_then(|v| v.checked_mul(utiles)) + .ok_or(EOVERFLOW)?, + 4, + )? + .checked_mul(16) + .and_then(|v| v.checked_mul(layers)) + .ok_or(EOVERFLOW)?; + let tpc_size = 8usize + .checked_mul(utiles) + .and_then(|v| v.checked_mul(macro_tiles)) + .map(|v| v / 4 * 4) + .and_then(|v| v.checked_mul(16)) + .and_then(|v| v.checked_mul(layers)) + .and_then(|v| v.checked_mul(clusters as usize)) + .ok_or(EOVERFLOW)?; + let layer_offset = aligned(tilemap_size, 0x1000)?; + let heap_offset = layer_offset + .checked_add(if layers > 1 { 0x100 } else { 0 }) + .ok_or(EOVERFLOW)?; + let used = heap_offset.checked_add(0x200).ok_or(EOVERFLOW)?; + let tiles = tiles_x.checked_mul(tiles_y).ok_or(EOVERFLOW)?; + let mut tvb_blocks = aligned(tiles.div_ceil(128), 8)?.max(8); + if clusters > 1 { + tvb_blocks = tvb_blocks.max( + 7usize + .checked_add(2usize.checked_mul(layers).ok_or(EOVERFLOW)?) + .ok_or(EOVERFLOW)?, + ); + } + tvb_blocks = tvb_blocks.max(cfg::TVB_INITIAL_PAGES.len()); + Ok(Self { + layer_offset, + heap_offset, + used, + tpc_size, + tvb_blocks, + }) + } + pub(crate) fn tpc_capacity(&self) -> Result { + Ok(aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) + } + pub(crate) fn record_stride(&self) -> Result { + Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) + } +} + +struct Object { + address: u64, + _mapping: mmu::KernelMapping, + backing: KernelObject, +} +impl Object { + fn new(dev: &AsahiDevice, vm: &mmu::Vm, size: usize) -> Result { + let backing = KernelObject::backing(dev, size, CpuMap::WriteCombined)?; + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let mapping = compact_alias(&backing, vm, 0..size, cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)?; + Ok(Self { + address: mapping.iova(), + _mapping: mapping, + backing, + }) + } +} +struct Retired { + _object: Object, + owners: [Option; DEPTH], +} + +#[derive(Copy, Clone, Debug)] +pub(crate) struct Addresses { + pub(crate) tilemap: u64, + pub(crate) layer: u64, + pub(crate) heap: u64, +} + +pub(crate) struct Target { + records: Object, + tpc: Object, + stride: usize, + slots: [Option; DEPTH], + retired: KVec, +} +impl Target { + /// Keep this allocation frame separate from the rest of the render graph. + #[inline(never)] + pub(crate) fn new(dev: &AsahiDevice, vm: &mmu::Vm, layout: &Layout) -> Result> { + let tpc = layout.tpc_capacity()?; + let stride = layout.record_stride()?; + if tpc > cfg::TPC_MAX || stride > cfg::TARGET_RECORD_MAX { + return Err(ERANGE); + } + let records = Object::new(dev, vm, stride * DEPTH)?; + let tpc = Object::new(dev, vm, tpc)?; + KBox::new( + Self { + records, + tpc, + stride, + slots: [None; DEPTH], + retired: KVec::new(), + }, + GFP_KERNEL, + ) + .map_err(Into::into) + } + pub(crate) fn tpc_va(&self) -> u64 { + self.tpc.address + } + + /// Only the span consumed by this render is cleared. A small render does + /// not clear a previous large target's complete retained record capacity. + pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { + let slot = ordinal as usize % DEPTH; + if self.slots[slot].is_some_and(|owner| owner != ordinal) { + return Err(EBUSY); + } + if layout.used > self.stride { + return Err(ERANGE); + } + let offset = slot * self.stride; + let span = aligned(layout.used, mmu::UAT_PGSZ)?.min(self.stride); + self.records + .backing + .bytes_mut() + .get_mut(offset..offset + span) + .ok_or(ERANGE)? + .fill(0); + self.slots[slot] = Some(ordinal); + let tilemap = self + .records + .address + .checked_add(offset as u64) + .ok_or(EOVERFLOW)?; + Ok(Addresses { + tilemap, + layer: tilemap + .checked_add(layout.layer_offset as u64) + .ok_or(EOVERFLOW)?, + heap: tilemap + .checked_add(layout.heap_offset as u64) + .ok_or(EOVERFLOW)?, + }) + } + + /// Growth never shrinks a live allocation. A successfully replaced TPC + /// stays enlarged even if allocating the following record object fails. + pub(crate) fn ensure_capacity( + &mut self, + dev: &AsahiDevice, + vm: &mmu::Vm, + layout: &Layout, + owners: &[Option; DEPTH], + ) -> Result { + let tpc = layout.tpc_capacity()?; + let stride = layout.record_stride()?; + let mut grew = false; + if tpc > self.tpc.backing.size() { + if tpc > cfg::TPC_MAX { + return Err(ERANGE); + } + Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners)?; + grew = true; + } + if stride > self.stride { + if stride > cfg::TARGET_RECORD_MAX { + return Err(ERANGE); + } + Self::replace( + &mut self.retired, + &mut self.records, + dev, + vm, + stride * DEPTH, + owners, + )?; + self.stride = stride; + self.slots = [None; DEPTH]; + grew = true; + } + Ok(grew) + } + fn replace( + retired: &mut KVec, + old: &mut Object, + dev: &AsahiDevice, + vm: &mmu::Vm, + size: usize, + owners: &[Option; DEPTH], + ) -> Result { + let retain = owners.iter().any(Option::is_some); + if retain { + retired.reserve(1, GFP_KERNEL)?; + } + let new = Object::new(dev, vm, size)?; + let previous = core::mem::replace(old, new); + if retain { + super::append_reserved( + retired, + Retired { + _object: previous, + owners: *owners, + }, + ); + } + Ok(()) + } + pub(crate) fn retire(&mut self, ordinal: u64) -> Result { + let slot = &mut self.slots[ordinal as usize % DEPTH]; + if *slot == Some(ordinal) { + *slot = None; + } + let mut index = 0; + while index < self.retired.len() { + for owner in &mut self.retired[index].owners { + if *owner == Some(ordinal) { + *owner = None; + } + } + if self.retired[index].owners.iter().all(Option::is_none) { + drop(self.retired.remove(index)?); + } else { + index += 1; + } + } + Ok(()) + } + /// Cancels only an unpublished record; older referenced backing remains + /// owned until normal paired retirement or the firmware-stop boundary. + pub(crate) fn cancel(&mut self, ordinal: u64) { + let slot = &mut self.slots[ordinal as usize % DEPTH]; + if *slot == Some(ordinal) { + *slot = None; + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs new file mode 100644 index 0000000000000..c0609f148106e --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware-produced rings and the device-control producer. +//! +//! The device mutex serializes every caller. Entries are copied before a +//! single aligned consumer store returns their slots to the firmware. Trace +//! and statistics transport errors quarantine only that ring; event errors +//! propagate to the device owner. Device-control publication reserves one +//! empty slot and orders the record, power tally and producer together. + +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +use crate::driver::AsahiDevice; + +use super::fw::{channels::*, initdata::status}; +use super::initdata::InitData; +use super::Role; + +/// Firmware rings whose contents do not affect host scheduling. +#[derive(Copy, Clone, Debug)] +enum ReportRing { + Trace = 0, + Statistics = 1, +} + +impl ReportRing { + fn geometry(self) -> (usize, usize, u32, u32) { + match self { + Self::Trace => ( + status::TRACE_STATE as usize, + status::TRACE_RING as usize, + TRACE_SLOTS, + TRACE_TYPES, + ), + Self::Statistics => ( + status::STATS_STATE as usize, + status::STATS_RING as usize, + STATISTICS_SLOTS, + STATISTICS_TYPES, + ), + } + } +} + +/// Host-side receive-ring state. A quarantined ring stays stopped until the +/// device is destroyed; firmware restart does not clear it. +#[derive(Debug)] +pub(crate) struct Rings { + quarantined: [[bool; 2]; 2], +} + +impl Rings { + pub(crate) const fn new() -> Self { + Self { + quarantined: [[false; 2]; 2], + } + } + + pub(crate) fn drain_trace(&mut self, dev: &AsahiDevice, init: &InitData, role: Role) { + self.drain_report(dev, init, role, ReportRing::Trace); + } + + pub(crate) fn drain_statistics(&mut self, dev: &AsahiDevice, init: &InitData, role: Role) { + self.drain_report(dev, init, role, ReportRing::Statistics); + } + + fn drain_report(&mut self, dev: &AsahiDevice, init: &InitData, role: Role, ring: ReportRing) { + let quarantined = &mut self.quarantined[role as usize][ring as usize]; + if *quarantined { + return; + } + if let Err(error) = Self::consume_report(init, role, ring) { + *quarantined = true; + dev_err!( + dev.as_ref(), + "G17 {:?} {:?} ring stopped: {:?}\n", + role, + ring, + error + ); + } + } + + fn consume_report(init: &InitData, role: Role, ring: ReportRing) -> Result { + let (state, entries, capacity, allowed_types) = ring.geometry(); + let consumer_word = init.status_word(role, state)?; + let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; + fence(Ordering::Acquire); + let mut consumer = consumer_word.load(Ordering::Relaxed); + let producer = producer_word.load(Ordering::Relaxed); + // The producer makes the entries it covers available to the host. + fence(Ordering::Acquire); + if consumer >= capacity || producer >= capacity { + return Err(EIO); + } + while consumer != producer { + let offset = entries + consumer as usize * EVENT_RECORD_SIZE; + let tag = match ring { + // Copy the entire trace entry before returning its slot. + ReportRing::Trace => init.status_read_words::(role, offset)?[0], + ReportRing::Statistics => init.status_read_words::<1>(role, offset)?[0], + }; + if tag >= u32::BITS || allowed_types & (1 << tag) == 0 { + return Err(EIO); + } + consumer = (consumer + 1) & (capacity - 1); + fence(Ordering::SeqCst); + consumer_word.store(consumer, Ordering::Relaxed); + if matches!(ring, ReportRing::Trace) { + fence(Ordering::SeqCst); + } + } + if matches!(ring, ReportRing::Statistics) { + fence(Ordering::SeqCst); + } + Ok(()) + } + + /// Publishes one primary device-control record. On EAGAIN the caller + /// retains the request and retries before consuming more dependent events. + /// The returned cursor is the value the consumer must reach to retire it. + /// The caller announces the publication through the coprocessor mailbox. + pub(crate) fn publish_control(init: &InitData, record: &ControlRecord) -> Result { + fence(Ordering::Acquire); + let consumer = init.control_consumer()?.load(Ordering::Relaxed); + let producer = init.control_producer()?.load(Ordering::Relaxed); + if consumer >= DEVICE_CONTROL_SLOTS || producer >= DEVICE_CONTROL_SLOTS { + return Err(EIO); + } + let next = (producer + 1) & (DEVICE_CONTROL_SLOTS - 1); + if next == consumer { + return Err(EAGAIN); + } + // InitData validates every mapping before the first write, then writes + // record, packed tally, barrier, producer, barrier without failure. + init.publish_control( + producer, + record.bytes(), + record.opcode().asserts_gpu_power(), + next, + )?; + Ok(next) + } +} + +/// One bounded event-ring pass. No shared-memory borrow survives `next`, so +/// the caller can service the copied event using the complete device owner. +pub(crate) struct Events { + role: Role, + remaining: u32, + unknown_reported: bool, +} + +impl Events { + /// A bounded pass can leave records written while its earlier slots were + /// consumed. Check both cursors again before the worker goes idle. + pub(crate) fn pending(init: &InitData, role: Role) -> Result { + let state = status::EVENT_STATE as usize; + fence(Ordering::Acquire); + let consumer = init.status_word(role, state)?.load(Ordering::Relaxed); + let producer = init + .status_word(role, state + RX_PRODUCER_OFFSET)? + .load(Ordering::Relaxed); + if consumer >= EVENT_SLOTS || producer >= EVENT_SLOTS { + return Err(EIO); + } + Ok(consumer != producer) + } + + pub(crate) fn new(role: Role) -> Self { + Self { + role, + remaining: EVENT_SLOTS, + unknown_reported: false, + } + } + + pub(crate) fn next(&mut self, dev: &AsahiDevice, init: &InitData) -> Result> { + let role = self.role; + let state = status::EVENT_STATE as usize; + let consumer_word = init.status_word(role, state)?; + let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; + while self.remaining != 0 { + init.validate_event_ring(role)?; + self.remaining -= 1; + fence(Ordering::Acquire); + let consumer = consumer_word.load(Ordering::Relaxed); + let producer = producer_word.load(Ordering::Relaxed); + if consumer >= EVENT_SLOTS || producer >= EVENT_SLOTS { + return Err(EIO); + } + if consumer == producer { + self.remaining = 0; + break; + } + let offset = status::EVENT_RING as usize + consumer as usize * EVENT_RECORD_SIZE; + let record = EventRecord::from_words(init.status_read_words(role, offset)?); + let event = record.decode(); + fence(Ordering::SeqCst); + consumer_word.store((consumer + 1) & (EVENT_SLOTS - 1), Ordering::Relaxed); + fence(Ordering::SeqCst); + match event? { + Event::Unknown(tag) => { + if !self.unknown_reported { + dev_warn!( + dev.as_ref(), + "G17 {:?} unknown firmware event {}\n", + role, + tag + ); + self.unknown_reported = true; + } + } + event => return Ok(Some(event)), + } + } + Ok(None) + } +} diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs new file mode 100644 index 0000000000000..fef478310c23b --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Decode a submission and validate its user resources before publishing any work. + +use super::fw::{ + kick::WriteRange, + render::{RenderPass, UscProgram, ZlsSurface}, +}; +use crate::util::{AnyBitPattern, Reader}; +use core::ops::Range; +use kernel::{prelude::*, uapi}; + +const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; +const USC_WINDOW_SIZE: u64 = 1 << 32; +const MAX_SAMPLERS: u32 = 1024; +const SAMPLER_SIZE: u64 = 8; +const PAGE_SIZE: u64 = 0x4000; +const ZLS_METADATA_STRIDE: u64 = 0x80; +const MAX_DIMENSION: u16 = 16384; +const MAX_LAYERS: u16 = 2048; +const MAX_TILE_BUFFER: u64 = 32768; +const PROCESS_EMPTY: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES; +const RSRC_SPEC_HI: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI; +const DBIAS_INT: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT; +const RENDER_FLAGS: u32 = PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT; + +/// Attachment settings persist until replaced, including by an empty list. +#[derive(Clone, Copy)] +pub(crate) struct Attachments { + ranges: [WriteRange; MAX_ATTACHMENTS], + count: usize, +} + +impl Attachments { + const EMPTY: Self = Self { + ranges: [WriteRange { va: 0, size: 0 }; MAX_ATTACHMENTS], + count: 0, + }; + + pub(crate) fn as_slice(&self) -> &[WriteRange] { + &self.ranges[..self.count] + } + + fn parse(bytes: &[u8]) -> Result { + let size = size_of::(); + if bytes.len() % size != 0 || bytes.len() / size > MAX_ATTACHMENTS { + return Err(EINVAL); + } + let mut list = Self::EMPTY; + let mut reader = Reader::new(bytes); + for range in &mut list.ranges[..bytes.len() / size] { + let item: uapi::drm_asahi_attachment = reader.read()?; + if item.flags != 0 || item.pad != 0 { + return Err(EINVAL); + } + *range = WriteRange { + va: item.pointer, + size: item.size, + }; + list.count += 1; + } + Ok(list) + } + + fn validate(&self, space: &impl AddressSpace) -> Result { + for range in self.as_slice() { + require_range(space, range.va, range.size, Access::Write)?; + } + Ok(()) + } +} + +// Commands stay inline in the submission plan, avoiding an allocation per render pass. +#[allow(clippy::large_enum_variant)] +pub(crate) enum Payload { + Render(uapi::drm_asahi_cmd_render, [Attachments; 2]), + Compute(uapi::drm_asahi_cmd_compute, Attachments), +} + +pub(crate) struct Command { + /// Render and compute prefix indices, before this command. + pub(crate) barriers: [u16; 2], + pub(crate) payload: Payload, +} + +/// Parser over the single userspace snapshot taken by the submit ioctl. +/// Call `finish` after consuming it to reject submissions with no hardware work. +pub(crate) struct Parser<'a> { + bytes: &'a [u8], + counts: [u16; 2], + attachments: [Attachments; 3], +} + +impl<'a> Parser<'a> { + pub(crate) fn new(bytes: &'a [u8]) -> Self { + Self { + bytes, + counts: [0; 2], + attachments: [Attachments::EMPTY; 3], + } + } + + pub(crate) fn next(&mut self) -> Result> { + while !self.bytes.is_empty() { + let mut reader = Reader::new(self.bytes); + let header: uapi::drm_asahi_cmd_header = reader.read()?; + let start = size_of::(); + let end = start + header.size as usize; + let bytes = self.bytes.get(start..end).ok_or(EINVAL)?; + self.bytes = &self.bytes[end..]; + let barriers = [header.vdm_barrier, header.cdm_barrier]; + match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER + | uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + if barriers.iter().zip(self.counts).any(|(&barrier, count)| { + barrier != uapi::DRM_ASAHI_BARRIER_NONE as u16 && barrier > count + }) || u32::from(self.counts[0]) + u32::from(self.counts[1]) + >= crate::file::MAX_COMMANDS_PER_SUBMISSION + { + return Err(EINVAL); + } + let payload = if header.cmd_type as u32 + == uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER + { + self.counts[0] += 1; + Payload::Render(payload(bytes)?, [self.attachments[0], self.attachments[1]]) + } else { + self.counts[1] += 1; + Payload::Compute(payload(bytes)?, self.attachments[2]) + }; + return Ok(Some(Command { barriers, payload })); + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS + | uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_FRAGMENT_ATTACHMENTS + | uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_ATTACHMENTS => { + if barriers != [uapi::DRM_ASAHI_BARRIER_NONE as u16; 2] { + return Err(EINVAL); + } + let slot = header.cmd_type as usize + - uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS as usize; + self.attachments[slot] = Attachments::parse(bytes)?; + } + _ => return Err(EINVAL), + } + } + Ok(None) + } + + pub(crate) fn finish(&self) -> Result { + if !self.bytes.is_empty() || self.counts == [0; 2] { + return Err(EINVAL); + } + Ok(()) + } +} + +fn payload(bytes: &[u8]) -> Result { + if bytes + .get(size_of::()..) + .is_some_and(|tail| tail.iter().any(|&b| b != 0)) + { + return Err(EINVAL); + } + Reader::new(bytes).read_up_to(bytes.len()) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Access { + Read, + Write, + ReadWrite, +} + +/// The caller retains the VM mappings while validating and executing the command. +pub(crate) trait AddressSpace { + fn covers(&self, address: u64, size: u64, access: Access) -> bool; +} + +fn require_range(space: &impl AddressSpace, address: u64, size: u64, access: Access) -> Result { + if size == 0 || address.checked_add(size).is_none() || !space.covers(address, size, access) { + return Err(EINVAL); + } + Ok(()) +} + +/// A validated four-GiB USC window owned by one queue. +pub(crate) struct UscWindow { + base: u64, +} +impl UscWindow { + pub(crate) fn new(base: u64, range: Range) -> Result { + if base & (USC_WINDOW_SIZE - 1) != 0 + || base < range.start + || base + .checked_add(USC_WINDOW_SIZE) + .is_none_or(|end| end > range.end) + { + return Err(EINVAL); + } + Ok(Self { base }) + } + + fn program(&self, space: &impl AddressSpace, tagged_offset: u32) -> Result { + if tagged_offset == 0 { + return Err(EINVAL); + } + require_range( + space, + self.base + (tagged_offset & !7) as u64, + 4, + Access::Read, + ) + } +} + +fn sampler(space: &impl AddressSpace, heap: u64, count: u32) -> Result { + if count > MAX_SAMPLERS || (heap == 0) != (count == 0) || heap & (SAMPLER_SIZE - 1) != 0 { + return Err(EINVAL); + } + if count != 0 { + require_range(space, heap, u64::from(count) * SAMPLER_SIZE, Access::Read)?; + } + Ok(()) +} + +fn helper_is_empty(helper: &uapi::drm_asahi_helper_program) -> bool { + helper.binary == 0 && helper.cfg == 0 && helper.data == 0 +} + +fn surface( + space: &impl AddressSpace, + zls: &uapi::drm_asahi_zls_buffer, + layers: u16, +) -> Result { + if (zls.base == 0 && (zls.comp_base != 0 || zls.stride != 0 || zls.comp_stride != 0)) + || (zls.comp_base == 0 && zls.comp_stride != 0) + || (layers > 1 && zls.base != 0 && zls.stride == 0) + || (zls.stride != 0 && u64::from(zls.stride) & (PAGE_SIZE - 1) != 1) + || u64::from(zls.comp_stride) & (PAGE_SIZE - 1) != 0 + { + return Err(EINVAL); + } + if zls.base != 0 { + let stride = if zls.stride == 0 { + 0 + } else { + (u64::from(zls.stride) / PAGE_SIZE + 1) * PAGE_SIZE + }; + require_range( + space, + zls.base, + stride * u64::from(layers - 1) + 1, + Access::ReadWrite, + )?; + } + if zls.comp_base != 0 { + let stride = (u64::from(zls.comp_stride) / PAGE_SIZE + 1) * ZLS_METADATA_STRIDE; + require_range( + space, + zls.comp_base, + stride * u64::from(layers - 1) + 1, + Access::ReadWrite, + )?; + } + Ok(ZlsSurface { + va: zls.base, + compression_va: zls.comp_base, + stride: zls.stride.into(), + compression_stride: zls.comp_stride.into(), + }) +} + +// As with Payload, retain the bounded attachment lists inside the submission plan. +#[allow(clippy::large_enum_variant)] +pub(crate) enum Validated { + Render { + pass: RenderPass, + attachments: [Attachments; 2], + timestamps: [uapi::drm_asahi_timestamps; 2], + }, + Compute { + usc_base: u64, + cdm_va: u64, + cdm_end_va: u64, + sampler_heap_va: u64, + sampler_count: u32, + timestamps: uapi::drm_asahi_timestamps, + }, +} + +impl Payload { + pub(crate) fn validate( + &self, + window: &UscWindow, + space: &impl AddressSpace, + ) -> Result { + match self { + Self::Render(cmd, attachments) => { + if cmd.flags & !RENDER_FLAGS != 0 + || !(1..=MAX_DIMENSION).contains(&cmd.width_px) + || !(1..=MAX_DIMENSION).contains(&cmd.height_px) + || !(1..=MAX_LAYERS).contains(&cmd.layers) + || !matches!( + (cmd.utile_width_px, cmd.utile_height_px), + (16, 16) | (32, 16) | (32, 32) + ) + { + return Err(EINVAL); + } + let samples_log2 = match cmd.samples { + 1 => 0, + 2 => 1, + 4 => 2, + _ => return Err(EINVAL), + }; + if u64::from(cmd.sample_size_B) + * u64::from(cmd.utile_width_px) + * u64::from(cmd.utile_height_px) + * u64::from(cmd.samples) + > MAX_TILE_BUFFER + || cmd.vdm_ctrl_stream_base == 0 + || cmd.vdm_ctrl_stream_base & 3 != 0 + || cmd.isp_scissor_base == 0 + || cmd.isp_scissor_base & 7 != 0 + { + return Err(EINVAL); + } + require_range(space, cmd.vdm_ctrl_stream_base, 4, Access::Read)?; + require_range(space, cmd.isp_scissor_base, 8, Access::Read)?; + for (va, access) in [ + (cmd.isp_dbias_base, Access::Read), + (cmd.isp_oclqry_base, Access::Write), + ] { + if va != 0 { + if va & 7 != 0 { + return Err(EINVAL); + } + require_range(space, va, 8, access)?; + } + } + let depth = surface(space, &cmd.depth, cmd.layers)?; + let stencil = surface(space, &cmd.stencil, cmd.layers)?; + sampler(space, cmd.sampler_heap, cmd.sampler_count.into())?; + if !helper_is_empty(&cmd.vertex_helper) || !helper_is_empty(&cmd.fragment_helper) { + return Err(EINVAL); + } + for program in [&cmd.bg, &cmd.eot, &cmd.partial_bg, &cmd.partial_eot] { + window.program(space, program.usc)?; + } + for list in attachments { + list.validate(space)?; + } + let pass = RenderPass { + width: cmd.width_px.into(), + height: cmd.height_px.into(), + layers: cmd.layers, + utile_width: cmd.utile_width_px, + utile_height: cmd.utile_height_px, + samples_log2, + sample_size: cmd.sample_size_B, + process_empty_tiles: cmd.flags & PROCESS_EMPTY != 0, + msaa_control: cmd.ppp_multisamplectl, + ppp_control: cmd.ppp_ctrl.into(), + vdm_va: cmd.vdm_ctrl_stream_base, + scissor_va: cmd.isp_scissor_base, + depth_bias_va: cmd.isp_dbias_base, + depth_bias_is_int: cmd.flags & DBIAS_INT != 0, + occlusion_query_va: cmd.isp_oclqry_base, + sampler_heap_va: cmd.sampler_heap, + sampler_count: cmd.sampler_count.into(), + zls_control: cmd.zls_ctrl, + depth_dimensions: cmd.isp_zls_pixels.into(), + depth, + stencil, + depth_clear: cmd.isp_bgobjdepth, + stencil_clear: cmd.isp_bgobjvals, + merge_upper_x: cmd.isp_merge_upper_x, + merge_upper_y: cmd.isp_merge_upper_y, + usc_base: window.base, + bg: UscProgram { + offset: cmd.bg.usc, + rsrc_spec: u64::from(cmd.bg.rsrc_spec) + | (u64::from(cmd.bg_rsrc_spec_hi) << 32), + }, + partial_bg: UscProgram { + offset: cmd.partial_bg.usc, + rsrc_spec: u64::from(cmd.partial_bg.rsrc_spec) + | (u64::from(cmd.bg_partial_rsrc_spec_hi) << 32), + }, + eot_offset: cmd.eot.usc, + partial_eot_offset: cmd.partial_eot.usc, + rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, + }; + Ok(Validated::Render { + pass, + attachments: *attachments, + timestamps: [cmd.ts_vtx, cmd.ts_frag], + }) + } + Self::Compute(cmd, attachments) => { + if cmd.flags != 0 + || cmd.cdm_ctrl_stream_base & 3 != 0 + || cmd.cdm_ctrl_stream_end & 3 != 0 + || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base + || !helper_is_empty(&cmd.helper) + { + return Err(EINVAL); + } + require_range( + space, + cmd.cdm_ctrl_stream_base, + cmd.cdm_ctrl_stream_end - cmd.cdm_ctrl_stream_base, + Access::Read, + )?; + sampler(space, cmd.sampler_heap, cmd.sampler_count)?; + attachments.validate(space)?; + Ok(Validated::Compute { + usc_base: window.base, + cdm_va: cmd.cdm_ctrl_stream_base, + cdm_end_va: cmd.cdm_ctrl_stream_end - 4, + sampler_heap_va: cmd.sampler_heap, + sampler_count: cmd.sampler_count, + timestamps: cmd.ts, + }) + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs new file mode 100644 index 0000000000000..3ac8fe711ca61 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-command fences, timestamp destinations and VM pins retained through retirement. +//! +//! Runtime queues own completions before making work visible. They defer fence signalling +//! until after the device mutex is released, and retain failed visible work until a retirement +//! witness or processor stop. An error fence never substitutes for that ownership boundary. + +use super::{ + context::{Context, WorkStateLease}, + fence::{self, CompletionFence, Member}, + object::{CpuMap, KernelObject}, + recovery::Class, + status::VmStatus, +}; +use crate::{driver::AsahiDevice, file, hw::t8140::resources as cfg, mmu}; +use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; +use kernel::{ + dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, + new_mutex, + prelude::*, + sync::{Arc, Mutex}, + uapi, xarray, +}; + +struct Destination { + mapping: Arc, + offset: usize, +} + +impl Destination { + fn resolve( + objects: Pin<&xarray::XArray>>, + timestamp: uapi::drm_asahi_timestamp, + ) -> Result> { + if timestamp.handle == 0 { + return Ok(None); + } + let object = objects + .lock() + .get(timestamp.handle.try_into()?) + .ok_or(ENOENT)? + .clone(); + let file::Object::TimestampBuffer(mapping) = object; + let offset: usize = timestamp.offset.try_into()?; + if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { + return Err(ERANGE); + } + Ok(Some(Self { mapping, offset })) + } + + fn write(&self, ticks: u64) -> Result { + const NSEC_PER_SEC: u64 = 1_000_000_000; + let ns = (ticks / cfg::COMMAND_TIMESTAMP_HZ) + .wrapping_mul(NSEC_PER_SEC) + .wrapping_add( + ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, + ); + self.mapping.write_timestamp(self.offset, ns)?; + barrier(Ordering::Release); + Ok(()) + } +} + +/// Firmware writes two aligned words in a fresh write-combined page per compute command. +struct ComputeTimestamps { + // Remove the GPU alias before releasing its backing. + mapping: mmu::KernelMapping, + object: KernelObject, +} + +impl ComputeTimestamps { + fn new(dev: &AsahiDevice, vm: &mmu::Vm) -> Result { + let object = KernelObject::backing(dev, mmu::UAT_PGSZ, CpuMap::WriteCombined)?; + let mapping = object.alias_in( + vm, + cfg::TIMESTAMPS, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + )?; + Ok(Self { mapping, object }) + } +} + +#[pin_data] +pub(crate) struct Completion { + status: Arc, + fence: UserFence, + member: Member, + destinations: [Option; 4], + compute: Option, + #[pin] + vm_job: Mutex>, + #[pin] + render: Mutex>, + signalled: AtomicBool, + timeout_error: AtomicI32, + replays: AtomicU8, + replay_timeout: AtomicBool, + firmware_consumer: AtomicBool, + class: AtomicU8, +} + +impl Completion { + pub(crate) fn new( + dev: &AsahiDevice, + contexts: &FenceContexts, + context: &Arc, + member: Member, + vm_job: mmu::VmJobGuard, + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + compute: bool, + ) -> Result> { + if timestamps.len() != if compute { 1 } else { 2 } { + return Err(EINVAL); + } + let mut destinations = core::array::from_fn(|_| None); + for (index, timestamp) in timestamps.iter().enumerate() { + destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; + destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; + } + let fence = fence::independent(contexts)?; + let compute = if compute { + Some(ComputeTimestamps::new(dev, context.vm())?) + } else { + None + }; + Arc::pin_init( + pin_init!(Self { + status: context.status().clone(), + fence, + member, + destinations, + compute, + vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), + render <- new_mutex!(None, "G17 command render lease"), + signalled: AtomicBool::new(false), + timeout_error: AtomicI32::new(0), + replays: AtomicU8::new(0), + replay_timeout: AtomicBool::new(false), + firmware_consumer: AtomicBool::new(false), + class: AtomicU8::new(0), + }), + GFP_KERNEL, + ) + } + + pub(crate) fn fence(&self) -> Fence { + Fence::from_fence(&self.fence) + } + pub(crate) fn status(&self) -> &Arc { + &self.status + } + pub(crate) fn work_state(&self) -> Arc { + self.member.work_state() + } + pub(crate) fn has_timestamps(&self) -> bool { + self.destinations.iter().any(Option::is_some) + } + + pub(crate) fn compute_timestamp_va(&self) -> Result { + Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova()) + } + + /// The caller has proved that firmware retired this command, or has + /// received its scheduler kill acknowledgement with recovery closed. + /// These are the command's public timestamp words, not queue-private state. + pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { + let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let words = [ + object.dword(0)?.load(Ordering::Relaxed), + object.dword(8)?.load(Ordering::Relaxed), + ]; + barrier(Ordering::Acquire); + Ok(words) + } + + pub(crate) fn replay_witness_words(&self) -> Option<[u64; 2]> { + if self.signalled.load(Ordering::Acquire) { + return None; + } + self.compute_words().ok() + } + + pub(crate) fn begin_replay(&self) -> bool { + let accepted = self + .replays + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { + (count < 3).then_some(count + 1) + }) + .is_ok(); + if accepted { + self.replay_timeout.store(true, Ordering::Release); + } + accepted + } + + pub(crate) fn replay_count(&self) -> u8 { + self.replays.load(Ordering::Acquire) + } + + pub(crate) fn note_firmware_consumer(&self) { + self.firmware_consumer.store(true, Ordering::Release); + } + + pub(crate) fn has_firmware_consumer(&self) -> bool { + self.firmware_consumer.load(Ordering::Acquire) + } + + pub(crate) fn take_replay_timeout(&self) -> bool { + !self.signalled.load(Ordering::Acquire) && self.replay_timeout.swap(false, Ordering::AcqRel) + } + + /// Only a witnessed, retained command can be reset before republishing it. + pub(crate) fn reset_for_replay(&self) -> Result { + if self.signalled.load(Ordering::Acquire) { + return Err(ECANCELED); + } + let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let start = object.dword(0)?; + let end = object.dword(8)?; + start.store(0, Ordering::Relaxed); + end.store(0, Ordering::Relaxed); + barrier(Ordering::SeqCst); + Ok(()) + } + + /// The first timeout request retains its errno while the physical graph is + /// borrowed. Duplicate watchdog or cancellation callbacks cannot replace it. + pub(crate) fn defer_timeout(&self, error: Error) { + let _ = self.timeout_error.compare_exchange( + 0, + error.to_errno(), + Ordering::AcqRel, + Ordering::Acquire, + ); + } + + pub(crate) fn take_deferred_timeout(&self) -> Option { + let error = self.timeout_error.swap(0, Ordering::AcqRel); + (error != 0).then(|| Error::from_errno(error)) + } + + pub(crate) fn classify(&self, class: Class) { + self.class.store( + match class { + Class::Spared => 1, + Class::Guilty => 2, + }, + Ordering::Release, + ); + } + + pub(crate) fn spared(&self) -> bool { + self.class.load(Ordering::Acquire) == 1 + } + + /// Installed before the first firmware-visible prefix. Cancellation takes + /// the same lock after claiming the signal, so it cannot miss a late lease. + pub(crate) fn attach_render_lease( + &self, + lease: super::queue::admission::RenderLease, + ) -> Result { + let mut render = self.render.lock(); + if self.signalled.load(Ordering::Acquire) { + return Err(ECANCELED); + } + if render.is_some() { + return Err(EIO); + } + *render = Some(lease); + Ok(()) + } + + pub(crate) fn render_slot(&self) -> Option { + self.render + .lock() + .as_ref() + .map(super::queue::admission::RenderLease::slot) + } + + /// Called under the device mutex before deferring the error signal. + pub(crate) fn quarantine_render(&self) { + if let Some(lease) = self.render.lock().as_ref() { + lease.quarantine(); + } + } + + /// An exact whole-pair drain or processor stop allows rehabilitation. The + /// VM pin goes first; the still-held lease excludes reuse until both settle. + pub(crate) fn release_after_retirement(&self) { + drop(self.vm_job.lock().take()); + let lease = self.render.lock().take(); + if let Some(lease) = lease.as_ref() { + lease.rehabilitate_after_firmware_drain(); + } + drop(lease); + } + + fn release_healthy(&self) { + drop(self.render.lock().take()); + drop(self.vm_job.lock().take()); + } + + /// Exact retirement permits the physical lease and VM pin to be released + /// before waking userspace. A failure fence alone never establishes this. + pub(crate) fn complete(&self, result: Result<[u64; 4]>) { + if self.signalled.swap(true, Ordering::AcqRel) { + return; + } + let result = result.and_then(|ticks| { + for (destination, ticks) in self.destinations.iter().zip(ticks) { + if let Some(destination) = destination { + destination.write(ticks)?; + } + } + Ok(()) + }); + if let Err(error) = result { + if error != ECANCELED && !self.spared() { + self.status.record(error); + } + self.fence.set_error(error); + } + self.release_healthy(); + self.fence.signal(); + self.member.complete(result); + } + + /// A rejected, never-published command cannot poison the VM for contention. + pub(crate) fn fail_unpublished(&self, error: Error) { + if self.signalled.swap(true, Ordering::AcqRel) { + return; + } + self.status.record_if_device_loss(error); + self.fence.set_error(error); + self.release_healthy(); + self.fence.signal(); + self.member.complete(Err(error)); + } + + /// The physical queue must already be quarantined. Keep mappings pinned while + /// waking waiters, even if queue destruction requested cancellation. + pub(crate) fn fail_while_owned(&self, error: Error) { + if self.signalled.swap(true, Ordering::AcqRel) { + return; + } + if let Some(guard) = self.vm_job.lock().as_ref() { + guard.quarantine(); + } + if error != ECANCELED && !self.spared() { + self.status.record(error); + } + self.fence.set_error(error); + self.fence.signal(); + self.member.complete(Err(error)); + } +} + +/// Collected under the device mutex and executed only after releasing it. +/// Runtime queues establish the ownership witness before constructing `Retired` +/// or `Release`; `FailedOwned` leaves their physical and VM ownership intact. +pub(crate) enum Deferred { + Retired(Arc, Result<[u64; 4]>), + Drained(Arc, Result<[u64; 4]>), + FailedOwned(Arc, Error), + Unpublished(Arc, Error), + Release(Arc), +} + +impl Deferred { + pub(crate) fn finish(self) { + match self { + Self::Retired(completion, result) => { + completion.complete(result); + // An earlier failure may already have signalled this fence while + // retaining the pin. Retirement is a separate release witness. + completion.release_after_retirement(); + } + Self::Drained(completion, result) => { + completion.release_after_retirement(); + completion.complete(result); + } + Self::FailedOwned(completion, error) => completion.fail_while_owned(error), + Self::Unpublished(completion, error) => completion.fail_unpublished(error), + Self::Release(completion) => completion.release_after_retirement(), + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs new file mode 100644 index 0000000000000..c3a3c6e87a2f6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Logical queue ownership of scheduler, QoS and work-state identities. +//! +//! Installed bindings retain this context until replacement is acknowledged. +//! A job keeps its work-state lease through exact retirement, even if its error +//! fence has already been signalled. Context-local accounting never waits for GPU work. + +mod identity; +use super::{ + freelist::{RenderIds, RenderPool}, + fw::context as fw, + object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, + status::VmStatus, +}; +use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; +use core::sync::atomic::{fence, AtomicBool, AtomicU32, Ordering}; +use identity::{RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; +use kernel::{ + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; + +/// QoS IDs belong to one device; the two lowest values are reserved. +#[pin_data] +pub(crate) struct QosIds { + #[pin] + used: Mutex, +} +impl QosIds { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { used <- new_mutex!(3, "G17 QoS IDs") }), + GFP_KERNEL, + ) + } + fn acquire(self: &Arc) -> Result { + let mut used = self.used.lock(); + let id = used.trailing_ones(); + if id == 128 { + return Err(ENOSPC); + } + *used |= 1u128 << id; + Ok(QosLease { + ids: self.clone(), + id: id as u8, + }) + } +} +struct QosLease { + ids: Arc, + id: u8, +} +impl Drop for QosLease { + fn drop(&mut self) { + *self.ids.used.lock() &= !(1u128 << self.id); + } +} + +struct WorkStorage { + object: PooledObject, + words: [Option; STATE_COUNT], + counts: [u32; STATE_COUNT], + render: RenderSlots, +} +impl WorkStorage { + fn initialize(&mut self, word: u32) -> Result { + let slot = (word & 63) as usize; + if slot >= STATE_COUNT { + return Err(EINVAL); + } + let counter = UAT_PGSZ + slot * 4; + let node = fw::WorkNode::new(self.object.gpu_va() + counter as u64, word).ok_or(EINVAL)?; + // The exact work-state lease excludes a previous firmware owner. Only + // the node's 0xc8 initialized bytes are replaced; the stride tail is retained. + self.object.write(slot * NODE_STRIDE, node)?; + self.object.word(counter)?.store(0, Ordering::Relaxed); + self.words[slot] = Some(word); + self.counts[slot] = 0; + fence(Ordering::Release); + Ok(()) + } + fn change_count(&mut self, word: u32, kicks: u32, add: bool) -> Result { + let slot = (word & 63) as usize; + if self.words.get(slot) != Some(&Some(word)) { + return Err(EINVAL); + } + let value = if add { + self.counts[slot].checked_add(kicks) + } else { + self.counts[slot].checked_sub(kicks) + } + .ok_or(EOVERFLOW)?; + self.object + .word(UAT_PGSZ + slot * 4)? + .store(value, Ordering::Relaxed); + self.counts[slot] = value; + fence(Ordering::SeqCst); + Ok(value) + } +} + +pub(crate) struct Context { + execution: Arc, + status: Arc, + scheduler: Arc, + scheduler_generation: u32, + published: AtomicBool, + qos: QosLease, + owner_pid: u32, + states: WorkStates, + work: Arc>, + work_base: u64, + render: Arc>>>, +} +impl Context { + pub(crate) fn new( + alloc: &Allocator<'_>, + pool: &Arc, + qos: &Arc, + execution: Arc, + owner_pid: u32, + ) -> Result { + let status = execution.vm().status().ok_or(EINVAL)?.clone(); + let qos = qos.acquire()?; + let mut scheduler = alloc.kernel( + UAT_PGSZ, + UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + scheduler.initialize::(0, |value| { + value.init(); + Ok(()) + })?; + fence(Ordering::SeqCst); + let scheduler_generation = super::ids::OBJECTS.object()?; + let scheduler = Arc::new(scheduler, GFP_KERNEL)?; + let mut object = pool.allocate( + alloc, + Family::WorkState, + 2 * UAT_PGSZ, + UAT_PGSZ, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let work_base = object.gpu_va(); + object.write( + UAT_PGSZ + 0x400, + fw::WorkHead { + next: 0, + previous: work_base + (UAT_PGSZ + 0x400) as u64, + unk10: 0, + unk14: 0, + }, + )?; + fence(Ordering::Release); + let work = Arc::pin_init( + new_mutex!( + WorkStorage { + object, + words: [None; STATE_COUNT], + counts: [0; STATE_COUNT], + render: RenderSlots::new() + }, + "G17 work states" + ), + GFP_KERNEL, + )?; + let render = Arc::pin_init(new_mutex!(None, "G17 context private memory"), GFP_KERNEL)?; + Ok(Self { + execution, + status, + scheduler, + scheduler_generation, + published: AtomicBool::new(false), + qos, + owner_pid, + states: WorkStates::new(), + work, + work_base, + render, + }) + } + pub(crate) fn vm(&self) -> &Vm { + self.execution.vm() + } + pub(crate) fn status(&self) -> &Arc { + &self.status + } + pub(crate) fn id(&self) -> u32 { + self.execution.id() + } + + pub(crate) fn generation(&self) -> u8 { + self.execution.generation() + } + pub(crate) fn is_current(&self) -> bool { + self.execution.is_current() + } + pub(crate) fn owner_pid(&self) -> u32 { + self.owner_pid + } + pub(crate) fn qos_id(&self) -> u8 { + self.qos.id + } + pub(crate) fn scheduler_va(&self) -> u64 { + self.scheduler.gpu_va() + } + /// Installed physical channels retain this page after the logical execution + /// context and its VM lease have completed their scheduler release. + pub(crate) fn scheduler_owner(&self) -> Arc { + self.scheduler.clone() + } + pub(crate) fn scheduler_generation(&self) -> u32 { + self.scheduler_generation + } + pub(crate) fn work_head_va(&self) -> u64 { + self.work_base + (UAT_PGSZ + 0x400) as u64 + } + /// Shared scheduler list head, sampled under the context's storage lock. + pub(crate) fn work_head(&self) -> Result<[u64; 2]> { + let work = self.work.lock(); + let head = [ + work.object.dword(UAT_PGSZ + 0x400)?.load(Ordering::Relaxed), + work.object.dword(UAT_PGSZ + 0x408)?.load(Ordering::Relaxed), + ]; + fence(Ordering::Acquire); + Ok(head) + } + + pub(crate) fn is_published(&self) -> bool { + self.published.load(Ordering::Acquire) + } + pub(crate) fn mark_published(&self) { + self.published.store(true, Ordering::Release); + } + + pub(crate) fn release_identity(&self) -> Result<[u8; 4]> { + let mut result = [0; 4]; + fence(Ordering::Acquire); + for (value, offset) in result.iter_mut().zip([0, 1, 4, 0x26]) { + let pointer = self.scheduler.pointer(offset, 1)?; + // SAFETY: pointer checked against retained scheduler allocation; + // a byte read observes firmware state without borrowing its fields. + *value = unsafe { core::ptr::read_volatile(pointer) }; + } + Ok(result) + } + pub(crate) fn render_pool( + &self, + alloc: &Allocator<'_>, + ids: &Arc, + render_global: &Vm, + ) -> Result> { + if !self.is_current() { + return Err(EFAULT); + } + let mut render = self.render.lock(); + if let Some(pool) = &*render { + return Ok(pool.clone()); + } + let pool = Arc::new( + RenderPool::new(alloc, ids, render_global, self.vm())?, + GFP_KERNEL, + )?; + *render = Some(pool.clone()); + Ok(pool) + } + pub(crate) fn render_pools_released(&self, released: impl Fn(u32, u64) -> bool) -> bool { + self.render + .lock() + .as_ref() + .is_none_or(|pool| released(u32::from(pool.id()), pool.control_va())) + } +} + +/// Shared by all engines of one ioctl. Queued work initially consumes no node. +pub(crate) struct WorkStateLease { + context: Arc, + word: AtomicU32, +} +impl WorkStateLease { + pub(crate) fn deferred(context: Arc) -> Self { + Self { + context, + word: AtomicU32::new(FREE), + } + } + pub(crate) fn acquire(&self) -> Result { + let current = self.word(); + if current != FREE { + return Ok(current); + } + let Some(candidate) = self.context.states.allocate() else { + let current = self.word(); + return if current != FREE { + Ok(current) + } else { + Err(EBUSY) + }; + }; + if let Err(error) = self.context.work.lock().initialize(candidate) { + self.context.states.release(candidate); + return Err(error); + } + match self + .word + .compare_exchange(FREE, candidate, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => Ok(candidate), + Err(installed) => { + self.context.states.release(candidate); + Ok(installed) + } + } + } + pub(crate) fn word(&self) -> u32 { + self.word.load(Ordering::Acquire) + } + pub(crate) fn node_va(&self) -> Result { + let word = self.word(); + if word == FREE { + return Err(EINVAL); + } + Ok(self.context.work_base + u64::from(word & 63) * NODE_STRIDE as u64) + } + pub(crate) fn add_submitted_kicks(&self, kicks: u32) -> Result { + self.context + .work + .lock() + .change_count(self.word(), kicks, true) + } + pub(crate) fn cancel_unpublished_kicks(&self, kicks: u32) -> Result { + self.context + .work + .lock() + .change_count(self.word(), kicks, false) + } + + pub(crate) fn render_slot(&self, pair: u8) -> Result { + self.context.work.lock().render.get(pair).ok_or(ENOSPC) + } + pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { + self.context + .work + .lock() + .render + .payload(pair, ordinal) + .ok_or(EINVAL) + } +} +impl Drop for WorkStateLease { + fn drop(&mut self) { + let word = self.word(); + if word != FREE { + self.context.states.release(word); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs new file mode 100644 index 0000000000000..6c48367e2d686 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Context-local work-state identities. The low six bits address a node; +//! the complete sequence distinguishes its owner until exact retirement. + +use core::sync::atomic::{AtomicU32, Ordering}; + +pub(super) const FREE: u32 = u32::MAX; +pub(super) const STATE_COUNT: usize = 63; +pub(super) const NODE_STRIDE: usize = 0x100; + +pub(super) struct WorkStates { + next: AtomicU32, + live: [AtomicU32; 64], +} +impl WorkStates { + pub(super) const fn new() -> Self { + Self { + next: AtomicU32::new(0), + live: [const { AtomicU32::new(FREE) }; 64], + } + } + pub(super) fn allocate(&self) -> Option { + for _ in 0..64 { + let word = self.next.fetch_add(1, Ordering::Relaxed); + let index = (word & 63) as usize; + if index != STATE_COUNT + && self.live[index] + .compare_exchange(FREE, word, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + return Some(word); + } + } + None + } + pub(super) fn release(&self, word: u32) -> bool { + let index = (word & 63) as usize; + index != STATE_COUNT + && self.live[index] + .compare_exchange(word, FREE, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } +} + +/// Two stable logical ordinals, independent of physical pair numbering. +pub(super) struct RenderSlots { + pairs: [Option; 2], + origins: [Option; 2], +} +impl RenderSlots { + pub(super) const fn new() -> Self { + Self { + pairs: [None; 2], + origins: [None; 2], + } + } + pub(super) fn get(&mut self, pair: u8) -> Option { + if let Some(index) = self.pairs.iter().position(|value| *value == Some(pair)) { + return Some(index as u32); + } + let index = self.pairs.iter().position(Option::is_none)?; + self.pairs[index] = Some(pair); + Some(index as u32) + } + pub(super) fn payload(&mut self, pair: u8, ordinal: u64) -> Option { + let index = self.get(pair)? as usize; + let origin = *self.origins[index].get_or_insert(ordinal); + Some(ordinal.checked_sub(origin)?.wrapping_add(1) as u8) + } +} diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs new file mode 100644 index 0000000000000..5abd494eeb1f5 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Collapse a logical prefix to its newest still-live firmware kick per QID. + +use super::{ + fw::kick::{KickDependency, KICK_BARRIERS_MAX}, + job::Order, +}; +use kernel::prelude::*; + +/// A failed producer does not establish a fault in its consumer's VM. +pub(crate) fn inherited_error(error: Error) -> Error { + if error == EIO || error == ENODEV || error == ETIMEDOUT { + ECANCELED + } else { + error + } +} + +pub(crate) struct Frontier { + dependencies: [KickDependency; KICK_BARRIERS_MAX], + sequences: [u64; KICK_BARRIERS_MAX], + count: usize, +} + +impl Frontier { + pub(crate) const fn new() -> Self { + Self { + dependencies: [KickDependency::ZERO; KICK_BARRIERS_MAX], + sequences: [0; KICK_BARRIERS_MAX], + count: 0, + } + } + + pub(crate) fn as_slice(&self) -> &[KickDependency] { + &self.dependencies[..self.count] + } + + /// Each QID's implicit parent supplies transitive ordering. Choose its newest + /// logical sequence rather than comparing wrapped firmware timestamps. + pub(crate) fn insert(&mut self, sequence: u64, dependency: KickDependency) -> Result { + if let Some(index) = self + .as_slice() + .iter() + .position(|prior| prior.qid() == dependency.qid()) + { + if sequence > self.sequences[index] { + self.dependencies[index] = dependency; + self.sequences[index] = sequence; + } + return Ok(()); + } + if self.count == KICK_BARRIERS_MAX { + return Err(EIO); + } + self.dependencies[self.count] = dependency; + self.sequences[self.count] = sequence; + self.count += 1; + Ok(()) + } + + /// The caller has matched the logical owner and VM. Check every covered + /// predecessor for failure, including one a newer kick would subsume. + pub(crate) fn include( + &mut self, + prefix: Option, + sequence: u64, + status: i32, + word: u64, + ) -> Result { + if !Order::contains(prefix, sequence) { + return Ok(()); + } + if status < 0 { + return Err(ECANCELED); + } + if status > 0 { + return Ok(()); + } + let dependency = KickDependency::from_word(word).ok_or(EIO)?; + self.insert(sequence, dependency) + } +} diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs new file mode 100644 index 0000000000000..9e4e0253b7c81 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Event service ordering under the device mutex. A restart ends its role's batch; a worker +//! joins preparation leases and services it on a separate turn before continuing the rings. + +use super::{ + fw::channels::{Event, FreeListCompletion, TvbGrowRequest}, + recovery, Role, +}; +use kernel::prelude::*; + +/// The bounded shared-ring transport; recognized malformed entries return errors after consume. +pub(super) trait Transport { + fn trace(&mut self, role: Role); + fn statistics(&mut self, role: Role); + fn drain( + &mut self, + role: Role, + consume: impl FnMut(&mut Self, Event) -> Result, + ) -> Result; + fn recovery_state(&self) -> Result; +} + +/// Runtime callbacks with the device mutex held. None signals fences or allocates grow backing. +pub(super) trait Queues { + /// Flush the oldest deferred reply/release. False means ring backpressure; each pending + /// transaction retains its own bounded worker-attempt count across service turns. + fn flush_tvb_reply(&mut self) -> Result; + fn flush_freelist_release(&mut self) -> Result; + fn observe_render_releases(&mut self) -> Result; + fn control_backpressured(&self) -> bool; + fn queue_tvb_grow(&mut self, request: TvbGrowRequest) -> Result; + fn complete_freelist_grow(&mut self, completion: FreeListCompletion) -> Result; + fn retain_render_completions(&mut self, masks: [u64; 2]); + fn complete_compute(&mut self, masks: [u64; 2]) -> Result; + fn context_killed(&mut self, cookie: u64); +} + +/// Work requested after releasing the device mutex. +#[derive(Default)] +pub(super) struct Pending { + pub(super) recovery: bool, +} + +/// Access to the single recovery owner while event handlers borrow the device. +pub(super) trait Host: Transport + Queues { + fn recovery(&self) -> &recovery::State; + fn recovery_mut(&mut self) -> &mut recovery::State; +} + +fn batch(host: &mut H, role: Role, masks: &mut [u64; 2]) -> Result { + host.drain(role, |host, event| { + match event { + Event::Stamp(bits) => { + masks[0] |= bits[0]; + masks[1] |= bits[1]; + } + Event::Restart { generation, stamp } => { + if host.recovery_mut().request(generation, stamp)? { + return Ok(false); + } + } + Event::TvbGrow(request) => { + if request.halt_count == host.recovery().generation() { + host.queue_tvb_grow(request)?; + } + if host.control_backpressured() { + return Ok(false); + } + } + Event::FreeListGrow(completion) => host.complete_freelist_grow(completion)?, + Event::ContextKilled(cookie) => host.context_killed(cookie), + Event::Unknown(_) => {} + } + Ok(true) + }) +} + +pub(super) fn service(host: &mut H) -> Result { + if !host.flush_tvb_reply()? || !host.flush_freelist_release()? { + return Ok(Pending::default()); + } + host.observe_render_releases()?; + for role in [Role::Primary, Role::Secondary] { + host.trace(role); + } + for role in [Role::Primary, Role::Secondary] { + host.statistics(role); + } + let mut masks = [0; 2]; + if !host.recovery().pending() { + let mut settled = false; + // The firmware publishes restart before state 1, but its producer can race the snapshot. + for _ in 0..256 { + batch(host, Role::Primary, &mut masks)?; + if host.recovery().pending() || host.control_backpressured() { + settled = true; + break; + } + match host.recovery_state()? { + 0 => { + settled = true; + break; + } + 1 => {} + _ => return Err(EIO), + } + } + if !settled { + return Err(EIO); + } + } + if !host.control_backpressured() { + batch(host, Role::Secondary, &mut masks)?; + } + host.retain_render_completions(masks); + host.complete_compute(masks)?; + Ok(Pending { + recovery: host.recovery().pending(), + }) +} diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs new file mode 100644 index 0000000000000..95a13dbdd54ff --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware completion fences and the aggregate fence of one submit ioctl. +//! +//! These fences can complete out of order, so each has an independent timeline. +//! Signalling belongs outside the device lock, after timestamp and VM-status writes. + +use super::context::{Context, WorkStateLease}; +use crate::file::SyncItem; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; +use kernel::{ + bindings, c_str, + dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + prelude::*, + str::CStr, + sync::Arc, +}; + +pub(crate) struct CompletionFence; + +#[vtable] +impl FenceOps for CompletionFence { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("asahi") + } + + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("g17-submit") + } +} + +/// Give independently retired work a timeline that fence merging cannot subsume. +pub(crate) fn independent(contexts: &FenceContexts) -> Result> { + let fence = contexts.new_fence(0, CompletionFence)?; + // SAFETY: The fence is unique and unpublished: there are no waiters, callbacks or + // shared references. The DMA-fence allocator supplies a globally unique context. + unsafe { (*fence.raw()).context = bindings::dma_fence_context_alloc(1) }; + Ok(fence.into()) +} + +/// All command members of an ioctl share its work-state node and aggregate fence. +/// The initial member keeps the aggregate unsignalled until enqueue finishes. +pub(crate) struct Submission { + fence: UserFence, + work_state: Arc, + remaining: AtomicU32, + error: AtomicI32, +} + +impl Submission { + pub(crate) fn new(contexts: &FenceContexts, context: Arc) -> Result> { + Ok(Arc::new( + Self { + fence: independent(contexts)?, + work_state: Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?, + remaining: AtomicU32::new(1), + error: AtomicI32::new(0), + }, + GFP_KERNEL, + )?) + } + + pub(crate) fn fence(&self) -> Fence { + Fence::from_fence(&self.fence) + } + + /// Add one command before enqueue ends. The parser bounds the count to 64. + pub(crate) fn member(self: &Arc) -> Member { + self.remaining.fetch_add(1, Ordering::Relaxed); + Member { + submission: self.clone(), + complete: AtomicBool::new(false), + } + } + + fn member_done(&self, result: Result) { + if let Err(error) = result { + let _ = self.error.compare_exchange( + 0, + error.to_errno(), + Ordering::AcqRel, + Ordering::Acquire, + ); + } + if self.remaining.fetch_sub(1, Ordering::AcqRel) == 1 { + let error = self.error.load(Ordering::Acquire); + if error != 0 { + self.fence.set_error(Error::from_errno(error)); + } + self.fence.signal(); + } + } + + fn finish_enqueue(&self, result: Result) { + // A dropped, never-enqueued member reports cancellation. The ioctl's actual + // failure takes precedence, while published members still retain their work. + if let Err(error) = result { + self.error.store(error.to_errno(), Ordering::Release); + } + self.member_done(result); + } +} + +/// One command's contribution to its submission, including construction failure. +pub(crate) struct Member { + submission: Arc, + complete: AtomicBool, +} + +impl Member { + pub(crate) fn work_state(&self) -> Arc { + self.submission.work_state.clone() + } + + pub(crate) fn complete(&self, result: Result) { + if !self.complete.swap(true, Ordering::AcqRel) { + self.submission.member_done(result); + } + } +} + +impl Drop for Member { + fn drop(&mut self) { + self.complete(Err(ECANCELED)); + } +} + +fn install(outputs: impl Iterator, fence: &Fence) { + for mut sync in outputs { + if let Some(chain) = sync.chain_fence.take() { + sync.syncobj.add_point(chain, fence, sync.timeline_value); + } else { + sync.syncobj.replace_fence(Some(fence)); + } + } +} + +/// Keeps resolved output syncobjs reachable on every fallible submission path. +/// Input syncobjs stay in the same vector, avoiding a separate output allocation. +pub(crate) struct Outputs { + fallback: UserFence, + syncs: KVec, + input_count: usize, + submission: Option>, + result: Result, +} + +impl Outputs { + pub(crate) fn new( + contexts: &FenceContexts, + syncs: KVec, + input_count: usize, + ) -> Result { + if input_count > syncs.len() { + return Err(EINVAL); + } + Ok(Self { + fallback: independent(contexts)?, + syncs, + input_count, + submission: None, + result: Err(ECANCELED), + }) + } + + pub(crate) fn inputs(&self) -> &[SyncItem] { + &self.syncs[..self.input_count] + } + + /// Publish the actual aggregate before enqueueing any command. + pub(crate) fn publish(&mut self, submission: Arc) { + install(self.syncs.drain(self.input_count..), &submission.fence()); + self.submission = Some(submission); + } + + /// The caller first records terminal VM errors and settles prior failed work. + pub(crate) fn finish(mut self, result: Result) -> Result { + self.result = result; + result + } +} + +impl Drop for Outputs { + fn drop(&mut self) { + if let Some(submission) = self.submission.take() { + submission.finish_enqueue(self.result); + } + if let Err(error) = self.result { + self.fallback.set_error(error); + } + install( + self.syncs.drain(self.input_count..), + &Fence::from_fence(&self.fallback), + ); + self.fallback.signal(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs new file mode 100644 index 0000000000000..e1d42944a38fe --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -0,0 +1,575 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Retained USC private-memory pools and their submission counters. +//! +//! A logical context owns one render pool, shared by its render pairs. A +//! physical compute queue owns a separate pool. All mutation and publication +//! runs under the device mutex. Pool aliases must outlive every firmware +//! reference and are released only after the owner's teardown is acknowledged. + +use super::fw::freelist::*; +use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIST_BLOCK_SIZE}; +use super::object::{Allocator, CpuMap, KernelObject}; +use crate::mmu; +use core::sync::atomic::{fence, AtomicBool, AtomicU64, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +/// Render pools occupy 129..255; compute uses its QID plus one, in 1..128. +/// First-free allocation keeps the shared firmware namespace deterministic. +pub(crate) struct RenderIds { + used: [AtomicU64; 2], +} +impl RenderIds { + pub(crate) const fn new() -> Self { + Self { + used: [AtomicU64::new(0), AtomicU64::new(1 << 63)], + } + } +} +struct RenderId { + slot: u16, + allocator: Arc, +} +impl RenderId { + fn new(allocator: &Arc) -> Result { + for (bank, used) in allocator.used.iter().enumerate() { + let mut value = used.load(Ordering::Acquire); + loop { + let bit = value.trailing_ones(); + if bit == 64 { + break; + } + match used.compare_exchange_weak( + value, + value | (1u64 << bit), + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + return Ok(Self { + slot: 129 + bank as u16 * 64 + bit as u16, + allocator: allocator.clone(), + }) + } + Err(current) => value = current, + } + } + } + Err(ENOSPC) + } +} +impl Drop for RenderId { + fn drop(&mut self) { + let bit = self.slot as usize - 129; + self.allocator.used[bit / 64].fetch_and(!(1u64 << (bit % 64)), Ordering::Release); + } +} + +struct Pages { + backing: KernelObject, + blocks: usize, + layout: RunLayout, +} +impl Pages { + fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { + let mut backing = alloc.lower( + TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + )?; + let base = backing.gpu_va(); + backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; + Ok(Self { + backing, + blocks, + layout, + }) + } + fn page_list_va(&self) -> u64 { + self.backing.gpu_va() + } + fn run_list_va(&self) -> u64 { + self.page_list_va() + PAGE_LIST_SIZE as u64 + } + fn repopulate(&self) -> Result { + let table = self.backing.pointer(0, TABLE_SIZE)?.cast::(); + PoolTables::for_each_word( + self.page_list_va(), + self.blocks, + self.layout, + |index, value| { + // SAFETY: The page-aligned backing was validated for the complete + // tables. The enumerator bounds every index inside those tables; + // the owner has proven firmware release before this update. + unsafe { table.add(index).write_volatile(value) }; + }, + ) + } +} + +/// Pool-object identities are independent of recycled firmware IDs and VAs. +static RENDER_POOL_SERIAL: AtomicU64 = AtomicU64::new(0); + +/// A render pool retained by a logical context and every graph referencing it. +pub(crate) struct RenderPool { + // Remove all aliases before dropping the backing or returning the ID. + _client: mmu::KernelMapping, + _global: mmu::KernelMapping, + pages: Pages, + control: KernelObject, + state: KernelObject, + served: AtomicBool, + serial: u64, + id: RenderId, +} + +impl RenderPool { + pub(crate) fn new( + alloc: &Allocator<'_>, + ids: &Arc, + render_global: &mmu::Vm, + client: &mmu::Vm, + ) -> Result { + let id = RenderId::new(ids)?; + let pages = Pages::new(alloc, RENDER_BLOCKS, RunLayout::Render)?; + let address = pages.page_list_va(); + let global = pages + .backing + .map_alias(render_global, address, mmu::PROT_GPU_SHARED_RW)?; + let client = pages + .backing + .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; + let state = alloc.kernel( + CONTROL_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let mut control = alloc.kernel( + CONTROL_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + control.write( + 0, + FreeListControl::render( + &FreeListArgs { + buffer_slot: u32::from(id.slot), + class: FreeListClass::Two, + page_list_va: address, + run_list_va: pages.run_list_va(), + blocks: RENDER_BLOCKS as u32, + state_va: state.gpu_va(), + }, + 0, + ), + )?; + fence(Ordering::SeqCst); + let serial = RENDER_POOL_SERIAL + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value.checked_add(1)) + .map_err(|_| EOVERFLOW)? + 1; + Ok(Self { + _client: client, + _global: global, + pages, + control, + state, + served: AtomicBool::new(false), + serial, + id, + }) + } + + pub(crate) fn serial(&self) -> u64 { + self.serial + } + pub(crate) fn id(&self) -> u16 { + self.id.slot + } + pub(crate) fn control_va(&self) -> u64 { + self.control.gpu_va() + } + pub(crate) fn page_list_va(&self) -> u64 { + self.pages.page_list_va() + } + pub(crate) fn run_list_va(&self) -> u64 { + self.pages.run_list_va() + } + pub(crate) fn end_va(&self) -> u64 { + self.run_list_va() + + (RENDER_BLOCKS * size_of::()) as u64 + } + + /// Fresh pools are already populated. Reuse requires the previous release + /// to have been consumed; command completion alone does not establish it. + pub(crate) fn repopulate(&self) -> Result { + if !self.served.swap(true, Ordering::AcqRel) { + return Ok(()); + } + let submitted = self.state.dword(0)?.load(Ordering::Relaxed); + let header = FreeListControl::render( + &FreeListArgs { + buffer_slot: u32::from(self.id()), + class: FreeListClass::Two, + page_list_va: self.page_list_va(), + run_list_va: self.run_list_va(), + blocks: RENDER_BLOCKS as u32, + state_va: self.state.gpu_va(), + }, + submitted, + ); + let mut image = KVVec::from_elem(0u64, CONTROL_SIZE / 8, GFP_KERNEL)?; + // SAFETY: `image` owns a zeroed control page, is large enough for the + // header, and every header byte is initialized integer data. + unsafe { + image + .as_mut_ptr() + .cast::() + .write_unaligned(header) + }; + let control = self.control.pointer(0, CONTROL_SIZE)?.cast::(); + self.pages.repopulate()?; + for (index, value) in image.iter().enumerate() { + // SAFETY: The entire retained page was checked before any store; + // the allocation is page-aligned and release excludes the firmware. + unsafe { control.add(index).write_volatile(*value) }; + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Submitted is host-owned and aligned. Completed is the packed firmware + /// counter, so retain the byte accesses required by that interface. + pub(crate) fn counters(&self) -> Result<[u64; 2]> { + let submitted = self.state.dword(0)?.load(Ordering::Relaxed); + let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; + let mut bytes = [0; 8]; + for (index, byte) in bytes.iter_mut().enumerate() { + // SAFETY: The checked eight-byte range stays mapped; byte accesses + // permit the unaligned counter without widening shared accesses. + *byte = unsafe { completed.add(index).read_volatile() }; + } + Ok([submitted, u64::from_le_bytes(bytes)]) + } + + /// Snapshot the unaligned shared fields before loading their descriptor. + pub(crate) fn page_pool_state(&self) -> Result { + let raw = self.control.pointer(0x14, 0x1c)?; + fence(Ordering::Acquire); + let read = |offset: usize, count: usize| -> u64 { + let mut bytes = [0; 8]; + for (index, byte) in bytes.iter_mut().take(count).enumerate() { + // SAFETY: The complete 0x14..0x30 span was checked. These fixed + // field ranges lie inside it; byte loads permit unaligned words. + *byte = unsafe { raw.add(offset + index).read_volatile() }; + } + u64::from_le_bytes(bytes) + }; + Ok(PagePoolState { + page_list_va: read(0, 8), + unk_1c: read(8, 4) as u32, + unk_20: read(12, 4) as u32, + unk_24: read(16, 4) as u32, + unk_28: read(20, 4) as u32, + page_count: read(24, 4) as u32, + }) + } + + /// Save mutable descriptor state only after both engines and free-list + /// references retire, before the physical slot changes owners. + pub(crate) fn store_page_pool_state(&self, state: PagePoolState) -> Result { + let raw = self.control.pointer(0x20, 0x10)?; + for (word, value) in [state.unk_20, state.unk_24, state.unk_28, state.page_count] + .into_iter() + .enumerate() + { + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: The entire mutable range was checked, and each + // enumerated byte lies inside it. Caller owns the retired pool. + unsafe { raw.add(word * 4 + index).write_volatile(byte) }; + } + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Prepares the counter before any producer is written. Commit belongs to + /// the publication transaction; dropping this plan does not reserve work. + pub(crate) fn prepare_submission(&self, commands: u32) -> Result> { + if commands > 2 { + return Err(EINVAL); + } + let counter = self.state.dword(0)?; + Ok(Submitted { + counter, + next: counter + .load(Ordering::Relaxed) + .wrapping_add(u64::from(commands)), + }) + } +} + +/// One checked, infallible counter update in a submission transaction. +pub(crate) struct Submitted<'a> { + counter: &'a AtomicU64, + next: u64, +} +impl Submitted<'_> { + pub(crate) fn commit(self) { + self.counter.store(self.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + } +} + +/// A pool owned by one physical compute queue, with fixed QID-derived slot. +pub(crate) struct ComputePool { + pages: Pages, + control: KernelObject, + state: KernelObject, + id: u16, +} +impl ComputePool { + pub(crate) fn new(alloc: &Allocator<'_>, qid: u8) -> Result { + if qid > super::fw::kick::QID_MAX { + return Err(EINVAL); + } + let id = u16::from(qid) + 1; + let pages = Pages::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute)?; + let mut control = alloc.kernel( + CONTROL_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let state = alloc.kernel( + CONTROL_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + control.write( + 0, + FreeListControl::compute(&FreeListArgs { + buffer_slot: u32::from(id), + class: FreeListClass::One, + page_list_va: pages.page_list_va(), + run_list_va: pages.run_list_va(), + blocks: COMPUTE_BLOCKS as u32, + state_va: state.gpu_va(), + }), + )?; + fence(Ordering::SeqCst); + Ok(Self { + pages, + control, + state, + id, + }) + } + pub(crate) fn id(&self) -> u16 { + self.id + } + pub(crate) fn page_list_va(&self) -> u64 { + self.pages.page_list_va() + } + pub(crate) fn control_va(&self) -> u64 { + self.control.gpu_va() + } + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { + let address = self.pages.page_list_va(); + self.pages + .backing + .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) + } + pub(crate) fn reserve_submission(&self) -> Result { + let counter = self.state.dword(0)?; + let next = counter + .load(Ordering::Relaxed) + .checked_add(1) + .ok_or(EOVERFLOW)?; + counter.store(next, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(next) + } + /// The caller has proved no pointer from this submission was published. + pub(crate) fn cancel_unpublished(&self, target: u64) -> Result { + let counter = self.state.dword(0)?; + if target == 0 || counter.load(Ordering::Acquire) != target { + return Err(EINVAL); + } + counter.store(target - 1, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } +} + +/// Registration identity held by a free-list lease. The cookie is the end of +/// the run list; it remains stable while overlapping command pairs share it. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Owner { + /// Identifies the host pool object even when every firmware address recycles. + pub(crate) serial: u64, + pub(crate) buffer_slot: u32, + pub(crate) control_va: u64, + pub(crate) ring_va: u64, + pub(crate) cookie: u64, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum LeasePhase { + Preparing, + Ready, + ReleaseSent(u32), + Released, +} + +/// References to one prepopulated owner: preparation and two engine commands +/// per render. Published releases do not gate reacquisition; their consumption +/// witnesses are carried into the successor. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Lease { + owner: Owner, + generation: u64, + references: u32, + commands: u32, + phase: LeasePhase, + previous_release: Option, +} + +impl Lease { + /// Acquires an unchanged overlapping owner, or an owner whose previous + /// release was sent. A changed owner additionally requires caller-side + /// proof of old-pool release before its backing can be repopulated. + pub(crate) fn acquire(previous: Option<&Self>, owner: Owner) -> Result { + let Some(previous) = previous else { + return Ok(Self { + owner, + generation: 0, + references: 3, + commands: 2, + phase: LeasePhase::Preparing, + previous_release: None, + }); + }; + if previous.owner.buffer_slot != owner.buffer_slot { + return Err(EINVAL); + } + let generation = previous.generation.checked_add(1).ok_or(EOVERFLOW)?; + let same_owner = previous.owner == owner; + let overlapping = same_owner + && previous.phase == LeasePhase::Ready + && previous.references >= 2 + && previous.commands >= 2; + let reacquire = previous.reacquirable(); + if !overlapping && !reacquire { + return Err(EBUSY); + } + let references = previous.references.checked_add(3).ok_or(EOVERFLOW)?; + let commands = previous.commands.checked_add(2).ok_or(EOVERFLOW)?; + Ok(Self { + owner, + generation: if overlapping { + previous.generation + } else { + generation + }, + references, + commands, + phase: LeasePhase::Preparing, + previous_release: previous.release_witness(), + }) + } + + pub(crate) fn owner(&self) -> Owner { + self.owner + } + pub(crate) fn generation(&self) -> u64 { + self.generation + } + pub(crate) fn reacquirable(&self) -> bool { + matches!( + self.phase, + LeasePhase::ReleaseSent(_) | LeasePhase::Released + ) && self.references == 0 + && self.commands == 0 + } + pub(crate) fn released(&self) -> bool { + self.phase == LeasePhase::Released && self.previous_release.is_none() + } + pub(crate) fn release_witness(&self) -> Option { + match self.phase { + LeasePhase::ReleaseSent(cursor) => Some(cursor), + _ => self.previous_release, + } + } + + /// Publication retires preparation independently of the command references. + pub(crate) fn retire_prepare(&mut self) -> Result { + if self.phase != LeasePhase::Preparing || self.references < 3 { + return Err(EINVAL); + } + self.references -= 1; + self.phase = LeasePhase::Ready; + Ok(()) + } + /// Fixed prepopulated pools never submit an asynchronous growth request. + /// An incoming growth completion therefore cannot name a live request. + pub(crate) fn complete_grow( + &mut self, + _event: super::fw::channels::FreeListCompletion, + ) -> Result { + Err(EIO) + } + /// Both engines have retired this render's exact retained submission. + pub(crate) fn complete_render( + &mut self, + ) -> Result> { + if self.phase != LeasePhase::Ready || self.references < 2 || self.commands < 2 { + return Err(EINVAL); + } + self.references -= 2; + self.commands -= 2; + Ok(self.pending_release()) + } + pub(crate) fn pending_release(&self) -> Option { + (self.phase == LeasePhase::Ready && self.references == 0).then(|| { + super::fw::channels::FreeListRelease::new(self.generation, self.owner.buffer_slot) + }) + } + /// Records the producer-after cursor once the final release was published. + pub(crate) fn mark_release_sent(&mut self, cursor: u32) -> Result { + if self.pending_release().is_none() || cursor >= super::fw::channels::DEVICE_CONTROL_SLOTS { + return Err(EINVAL); + } + self.phase = LeasePhase::ReleaseSent(cursor); + Ok(()) + } + /// Observes all older release records passed by one consumer snapshot. + pub(crate) fn observe_releases(&mut self, consumer: u32, producer: u32) -> Result { + if let Some(cursor) = self.previous_release { + if control_consumed(consumer, producer, cursor)? { + self.previous_release = None; + } + } + if let LeasePhase::ReleaseSent(cursor) = self.phase { + if control_consumed(consumer, producer, cursor)? { + self.phase = LeasePhase::Released; + } + } + Ok(()) + } +} + +/// A publication is retired once its producer-after cursor leaves the +/// current `(consumer, producer]` interval, including wrapped intervals. +pub(crate) fn control_consumed(consumer: u32, producer: u32, cursor: u32) -> Result { + let slots = super::fw::channels::DEVICE_CONTROL_SLOTS; + if consumer >= slots || producer >= slots || cursor >= slots { + return Err(EIO); + } + let pending = producer.wrapping_sub(consumer) & (slots - 1); + let ahead = cursor.wrapping_sub(consumer) & (slots - 1); + Ok(ahead == 0 || ahead > pending) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/buffer.rs b/drivers/gpu/drm/asahi/g17/fw/buffer.rs new file mode 100644 index 0000000000000..3ed336dbadaad --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/buffer.rs @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Parameter-buffer inventory, scene and control records. These objects are +//! initialized before publication; live access uses the host buffer manager's +//! individual aligned words and packed byte fields. + +use super::{clear, offset_va}; +use crate::hw::t8140::resources as cfg; +use kernel::prelude::*; + +pub(crate) const SCENES: usize = 0x50; +pub(crate) const SCRATCH_SLOTS: usize = 0x24; +pub(crate) const SCRATCH_STRIDE: u64 = 0x20; +pub(crate) const FIRST_SCENE: usize = 1; +pub(crate) const PAGE_LIST: usize = 0x8000; +pub(crate) const PAGE_LIST_BYTES: usize = 0x10000; +pub(crate) const CONTROL: usize = 0x20000; +pub(crate) const COUNTER: usize = 0x28000; +pub(crate) const SCENE_TABLE: usize = 0x38000; +pub(crate) const BLOCK_TABLE: usize = 0x40000; +pub(crate) const SIZE: usize = 0x48000; +pub(crate) const METRICS_STRIDE: usize = SCENES * size_of::(); +pub(crate) const METRICS_SLOTS: usize = cfg::METRICS_SIZE as usize / METRICS_STRIDE; +pub(crate) const NORMAL_MAX_BLOCKS: usize = 0x45e; +pub(crate) const GROW_BLOCKS: usize = 10; + +/// Firmware's active parameter-buffer state. Pointers at +0x44, +0x4c, +/// +0x64 and +0x84 are intentionally unaligned. +#[repr(C, packed)] +pub(crate) struct State { + pub(crate) unk_0: [u8; 0xc], + pub(crate) buffer_id: u32, + pub(crate) unk_10: [u8; 0x10], + pub(crate) page_list_fw: u64, + pub(crate) page_list_client: u64, + pub(crate) page_list_size: u32, + pub(crate) pages: u32, + pub(crate) block_capacity: u32, + pub(crate) producer: u32, + pub(crate) consumer: u32, + pub(crate) block_table: u64, + pub(crate) control: u64, + pub(crate) last_page: u32, + pub(crate) block_size: u32, + pub(crate) unk_5c: [u8; 8], + pub(crate) counter: u64, + pub(crate) unk_6c: [u8; 0x10], + pub(crate) maximum_size: u32, + pub(crate) normal_maximum_size: u32, + pub(crate) discard: u64, + pub(crate) unk_8c: [u8; 0x34], +} + +/// One scene's accounting and scratch identity. +#[repr(C, packed)] +pub(crate) struct Scene { + pub(crate) metric_client: u64, + pub(crate) metric_fw: u64, + pub(crate) unk_10: [u8; 0x18], + pub(crate) scratch: u64, + pub(crate) unk_30: [u8; 0x10], + pub(crate) statistics: u64, + pub(crate) used: u32, + pub(crate) generation: u64, + pub(crate) unk_54: [u8; 0x2c], +} + +/// Circular inventory cursors followed by firmware statistics. Firmware owns +/// `read` and the statistics; the host requests reset by release-storing1. +#[repr(C)] +pub(crate) struct Control { + pub(crate) total: u32, + pub(crate) committed: u32, + pub(crate) read: u32, + pub(crate) unk_c: [u32; 13], + pub(crate) max_pages: u32, + pub(crate) unk_44: [u32; 7], + pub(crate) reset: u32, + pub(crate) unk_64: [u32; 7], +} + +#[repr(C)] +pub(crate) struct Block { + pub(crate) first_page: u32, + pub(crate) unk_4: u32, +} + +/// The complete parameter-buffer graph, including reserved spaces between +/// records. The page-list subrange also has a client mapping. +#[repr(C)] +pub(crate) struct Graph { + pub(crate) state: State, + pub(crate) pad_c0: [u8; PAGE_LIST - size_of::()], + pub(crate) pages: [u32; cfg::TVB_BLOCK_SLOTS * cfg::TVB_PAGES_PER_BLOCK], + pub(crate) pad_151a0: [u8; CONTROL - PAGE_LIST - cfg::TVB_PAGE_LIST_SIZE], + pub(crate) control: Control, + pub(crate) pad_20080: [u8; COUNTER - CONTROL - size_of::()], + pub(crate) counter: u32, + pub(crate) pad_28004: [u8; SCENE_TABLE - COUNTER - 4], + pub(crate) scenes: [Scene; SCENES], + pub(crate) trailer: [u8; 0x40], + pub(crate) pad_3a840: [u8; BLOCK_TABLE - SCENE_TABLE - SCENES * size_of::() - 0x40], + pub(crate) blocks: [Block; cfg::TVB_BLOCK_SLOTS], + pub(crate) pad_468d0: [u8; SIZE - BLOCK_TABLE - cfg::TVB_BLOCK_SLOTS * size_of::()], +} + +// SAFETY: Every field of the complete graph consists solely of integers. +unsafe impl Zeroable for Graph {} + +static_assert!(size_of::() == 0xc0); +static_assert!(size_of::() == 0x80); +static_assert!(size_of::() == 0x80); +static_assert!(size_of::() == SIZE); +static_assert!(core::mem::offset_of!(Graph, pages) == PAGE_LIST); +static_assert!(core::mem::offset_of!(Graph, control) == CONTROL); +static_assert!(core::mem::offset_of!(Graph, counter) == COUNTER); +static_assert!(core::mem::offset_of!(Graph, scenes) == SCENE_TABLE); +static_assert!(core::mem::offset_of!(Graph, blocks) == BLOCK_TABLE); +static_assert!(core::mem::offset_of!(State, block_table) == 0x44); +static_assert!(core::mem::offset_of!(State, discard) == 0x84); +static_assert!(core::mem::offset_of!(Scene, generation) == 0x4c); + +pub(crate) struct GraphArgs<'a> { + pub(crate) fw_va: u64, + pub(crate) page_list_client: u64, + /// Firmware and client views, respectively, of this graph's metrics lease. + pub(crate) metrics: [u64; 2], + pub(crate) blocks: &'a [u64], + pub(crate) scratch: u64, + pub(crate) discard: u64, + pub(crate) buffer_id: u32, + pub(crate) generation: u64, +} + +pub(crate) fn compact_page(address: u64) -> Result { + let relative = address.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; + if relative % cfg::TVB_PAGE != 0 + || address + .checked_add(cfg::TVB_BLOCK_SIZE as u64) + .ok_or(EOVERFLOW)? + > cfg::USER_END + { + return Err(EINVAL); + } + Ok((relative / cfg::TVB_PAGE).try_into()?) +} + +impl Graph { + pub(crate) fn init(&mut self, args: &GraphArgs<'_>) -> Result { + if args.generation == 0 + || args.blocks.len() < cfg::TVB_INITIAL_PAGES.len() + || args.blocks.len() > cfg::TVB_MAX_BLOCKS + { + return Err(EINVAL); + } + let scratch = args.scratch.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; + let discard = args.discard.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; + if scratch > u32::MAX as u64 || discard > u32::MAX as u64 { + return Err(EINVAL); + } + let stats = offset_va(args.fw_va, (CONTROL + 0x40) as u64)?; + clear(self); + for (index, &address) in args.blocks.iter().enumerate() { + let first = compact_page(address)?; + self.blocks[index].first_page = first; + for page in 0..cfg::TVB_PAGES_PER_BLOCK { + self.pages[index * cfg::TVB_PAGES_PER_BLOCK + page] = first + page as u32; + } + } + for (index, scene) in self.scenes.iter_mut().enumerate() { + scene.metric_client = offset_va(args.metrics[1], (index * 4) as u64)?; + scene.metric_fw = offset_va(args.metrics[0], (index * 4) as u64)?; + scene.scratch = scratch + (index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE; + scene.statistics = stats; + } + self.scenes[FIRST_SCENE].generation = args.generation; + self.control.total = args.blocks.len() as u32; + self.control.committed = self.control.total; + self.control.reset = 1; + let pages = (args.blocks.len() * cfg::TVB_PAGES_PER_BLOCK) as u32; + self.state.buffer_id = args.buffer_id; + self.state.page_list_fw = offset_va(args.fw_va, PAGE_LIST as u64)?; + self.state.page_list_client = args.page_list_client; + self.state.page_list_size = PAGE_LIST_BYTES as u32; + self.state.pages = pages; + self.state.block_capacity = cfg::TVB_BLOCK_SLOTS as u32; + self.state.producer = args.blocks.len() as u32; + self.state.block_table = offset_va(args.fw_va, BLOCK_TABLE as u64)?; + self.state.control = offset_va(args.fw_va, CONTROL as u64)?; + self.state.last_page = pages - 1; + self.state.block_size = cfg::TVB_BLOCK_SIZE as u32; + self.state.counter = offset_va(args.fw_va, COUNTER as u64)?; + self.state.maximum_size = (cfg::TVB_BLOCK_SLOTS * 4) as u32; + self.state.normal_maximum_size = (NORMAL_MAX_BLOCKS * 4) as u32; + self.state.discard = discard; + Ok(()) + } +} + +/// Rebuilds an idle parameter-buffer descriptor, preserving firmware-owned +/// bits and resetting both ring counters and flags for its rebuilt page list. +pub(crate) fn reload_descriptor(previous: [u32; 4], address: u64, pages: u32) -> [u32; 4] { + const COUNT: u32 = 0x003f_ffff; + let bias = if address & (1u64 << 42) == 0 { + 0x70_0000_0000 + } else { + 0 + }; + let segment = (address.wrapping_add(bias) & 0x70_0000_0000) | ((address >> 3) & 0x80_0000_0000); + [ + ((address >> 4) as u32 & !7) | (previous[0] & 6), + ((segment >> 8) as u32 & 0xf000_0000) | (previous[1] & 0x0fc0_0000) | (pages & COUNT), + previous[2] & !COUNT, + previous[3] & !(COUNT | 0x8000_0000), + ] +} + +/// Compact scratch and tagged metrics addresses carried by scene registers. +pub(crate) fn scene_registers(index: usize, scratch: u64, metrics: u64) -> Result<(u64, u64)> { + const SEGMENT_SELECT: u64 = 1 << 42; + const SEGMENT_BIAS: u64 = 0x70_0000_0000; + const ADDRESS_MASK: u64 = 0x7f_ffff_fffe; + const HIGH_BIT: u64 = 0x80_0000_0000; + if index >= SCENES || metrics & 3 != 0 { + return Err(EINVAL); + } + let scratch = scratch + .checked_sub(cfg::USER_BASE) + .ok_or(EINVAL)? + .checked_add((index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE) + .ok_or(EOVERFLOW)?; + if scratch > u32::MAX as u64 || scratch & 0xf != 0 { + return Err(EINVAL); + } + let metric = offset_va(metrics, (index * 4) as u64)?; + let adjusted = offset_va( + metric, + if metric & SEGMENT_SELECT == 0 { + SEGMENT_BIAS + } else { + 0 + }, + )?; + Ok(( + scratch, + (adjusted & ADDRESS_MASK) | ((metric >> 3) & HIGH_BIT) | 1, + )) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs new file mode 100644 index 0000000000000..c2c3b3eb7a80d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -0,0 +1,522 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device-control records and work-ring slots. +//! +//! The host sends out-of-band requests and replies through the device-control +//! ring, whose 0x40-byte records start with a [`DeviceControlOp`]. It hands +//! hardware queues to the firmware through work rings, one per priority and +//! engine, each holding [`WORK_RING_SLOTS`] [`WorkSlot`]s. +//! +//! Producers and consumers of both rings are live words in the primary state +//! grid; they are not part of these structures. Before publishing a +//! device-control record whose opcode [`DeviceControlOp::asserts_gpu_power`], +//! the host also increments its power-assert tally. + +use kernel::prelude::*; + +use super::kick::QID_MAX; +use super::queue::{DataMaster, QUEUE_PRIORITY}; + +/// Size of a device-control record. +pub(crate) const DEVICE_CONTROL_RECORD_SIZE: usize = 0x40; +/// Number of records in the device-control ring. +pub(crate) const DEVICE_CONTROL_SLOTS: u32 = 256; + +/// Entries in either coprocessor's event ring. +pub(crate) const EVENT_SLOTS: u32 = 256; +/// Entries in either coprocessor's trace ring. +pub(crate) const TRACE_SLOTS: u32 = 512; +/// Entries in either coprocessor's statistics ring. +pub(crate) const STATISTICS_SLOTS: u32 = 256; +/// Words in an event, trace or statistics entry. +pub(crate) const EVENT_WORDS: usize = 18; +/// Byte stride of an event, trace or statistics entry. +pub(crate) const EVENT_RECORD_SIZE: usize = EVENT_WORDS * size_of::(); +/// Producer offset within a firmware-produced ring's state block. +pub(crate) const RX_PRODUCER_OFFSET: usize = 0x20; +/// Trace records use type 5. +pub(crate) const TRACE_TYPES: u32 = 1 << 5; +/// Statistics record types accepted by the host. +pub(crate) const STATISTICS_TYPES: u32 = 0x5fff_0000; + +/// Completion stamp indices signalled by the firmware. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +struct StampEventRecord { + tag: u32, + masks: [u64; 2], + diagnostic_pairs: u16, + unk_16: [u8; 0x32], +} + +/// Restart generation and the affected stamp, or -1 without attribution. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +struct RestartEventRecord { + tag: u32, + generation: u64, + stamp: i32, + unk_10: [u8; 0x38], +} + +/// Parameter-buffer growth request. The reply moves `halt_count` to +0x18. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +struct TvbGrowEventRecord { + tag: u32, + vm_slot: i32, + buffer_slot: u32, + counter: u32, + subpipe: u32, + halt_count: u64, + unk_1c: [u8; 0x2c], +} + +/// Completed asynchronous free-list growth. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +struct FreeListEventRecord { + tag: u32, + stamp: i32, + buffer_slot: u32, + control_va: u64, + ring_va: u64, + cookie: u64, + result: u32, + unk_28: [u8; 0x20], +} + +/// A snapshot of one firmware event. Every variant occupies a complete slot; +/// callers copy the entry before publishing its consumer cursor. +#[repr(C)] +#[derive(Copy, Clone)] +pub(crate) union EventRecord { + words: [u32; EVENT_WORDS], + stamp: StampEventRecord, + restart: RestartEventRecord, + tvb: TvbGrowEventRecord, + free_list: FreeListEventRecord, +} + +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(core::mem::offset_of!(StampEventRecord, masks) == 4); +static_assert!(core::mem::offset_of!(TvbGrowEventRecord, halt_count) == 0x14); +static_assert!(core::mem::offset_of!(FreeListEventRecord, cookie) == 0x1c); + +impl core::fmt::Debug for EventRecord { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("EventRecord") + .field("tag", &self.tag()) + .finish() + } +} + +/// A decoded asynchronous free-list growth completion. Resource ownership and +/// the expected cookie/result are checked by the free-list owner. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct FreeListCompletion { + pub(crate) stamp: i32, + pub(crate) buffer_slot: u32, + pub(crate) control_va: u64, + pub(crate) ring_va: u64, + pub(crate) cookie: u64, + pub(crate) result: u32, +} + +/// Host actions requested by a firmware event. Unknown tags are consumed so +/// one unrecognized record cannot block later completions. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Event { + Stamp([u64; 2]), + Restart { generation: u64, stamp: i32 }, + TvbGrow(TvbGrowRequest), + FreeListGrow(FreeListCompletion), + ContextKilled(u64), + Unknown(u32), +} + +impl EventRecord { + const STAMP: u32 = 1; + const RESTART: u32 = 4; + const TVB_GROW: u32 = 6; + const FREE_LIST_GROW: u32 = 13; + const CONTEXT_KILLED: u32 = 12; + const DIAGNOSTIC_PAIRS_MAX: u16 = 5; + const TVB_SLOTS: u32 = 127; + const SUBPIPES: u32 = 64; + const FREE_LIST_SLOTS: u32 = 256; + + pub(crate) const fn from_words(words: [u32; EVENT_WORDS]) -> Self { + Self { words } + } + + pub(crate) fn tag(&self) -> u32 { + // SAFETY: All constructors initialize the entire union. Every bit + // pattern is valid for its u32 words, and word zero is the tag. + unsafe { self.words[0] } + } + + /// Validates the record's fields without interpreting resource ownership. + /// A malformed known event is an error, unlike an unknown tag. + pub(crate) fn decode(&self) -> Result { + match self.tag() { + Self::STAMP => { + // SAFETY: The entire slot is initialized, the tag selects + // this layout, and its integer fields have no invalid values. + let record = unsafe { self.stamp }; + if record.diagnostic_pairs > Self::DIAGNOSTIC_PAIRS_MAX { + return Err(EIO); + } + Ok(Event::Stamp(record.masks)) + } + Self::RESTART => { + // SAFETY: As above, for the restart layout. + let record = unsafe { self.restart }; + Ok(Event::Restart { + generation: record.generation, + stamp: record.stamp, + }) + } + Self::TVB_GROW => { + // SAFETY: As above, for the parameter-buffer growth layout. + let record = unsafe { self.tvb }; + if record.vm_slot < -1 + || record.buffer_slot >= Self::TVB_SLOTS + || record.subpipe >= Self::SUBPIPES + { + return Err(EIO); + } + Ok(Event::TvbGrow(TvbGrowRequest { + buffer_slot: record.buffer_slot, + vm_slot: record.vm_slot, + counter: record.counter, + subpipe: record.subpipe, + halt_count: record.halt_count, + })) + } + Self::CONTEXT_KILLED => { + // SAFETY: Every constructor initializes all integer words. + let words = unsafe { self.words }; + Ok(Event::ContextKilled( + u64::from(words[1]) | (u64::from(words[2]) << 32), + )) + } + Self::FREE_LIST_GROW => { + // SAFETY: As above, for the free-list growth layout. + let record = unsafe { self.free_list }; + if record.stamp < -1 + || record.buffer_slot >= Self::FREE_LIST_SLOTS + || record.control_va == 0 + || record.ring_va == 0 + { + return Err(EIO); + } + Ok(Event::FreeListGrow(FreeListCompletion { + stamp: record.stamp, + buffer_slot: record.buffer_slot, + control_va: record.control_va, + ring_va: record.ring_va, + cookie: record.cookie, + result: record.result, + })) + } + tag => Ok(Event::Unknown(tag)), + } + } +} + +/// A complete device-control request. Its variant supplies the power policy; +/// callers construct the contained record with its typed builder. +#[derive(Debug)] +pub(crate) enum ControlRecord { + TvbGrow(TvbGrowReply), + IdlePowerOff(IdlePowerOff), + SchedulerStateRelease(SchedulerStateRelease), + FreeListRelease(FreeListRelease), + ContextKill(ContextKill), +} + +impl ControlRecord { + pub(crate) const fn opcode(&self) -> DeviceControlOp { + match self { + Self::TvbGrow(_) => DeviceControlOp::TvbGrowReply, + Self::IdlePowerOff(_) => DeviceControlOp::IdlePowerOff, + Self::SchedulerStateRelease(_) => DeviceControlOp::SchedulerStateRelease, + Self::FreeListRelease(_) => DeviceControlOp::FreeListRelease, + Self::ContextKill(_) => DeviceControlOp::ContextKill, + } + } + + /// Byte view of the initialized record, excluding the host-only enum tag. + pub(crate) fn bytes(&self) -> &[u8; DEVICE_CONTROL_RECORD_SIZE] { + let record = match self { + Self::TvbGrow(record) => core::ptr::from_ref(record).cast(), + Self::IdlePowerOff(record) => core::ptr::from_ref(record).cast(), + Self::SchedulerStateRelease(record) => core::ptr::from_ref(record).cast(), + Self::FreeListRelease(record) => core::ptr::from_ref(record).cast(), + Self::ContextKill(record) => core::ptr::from_ref(record).cast(), + }; + // SAFETY: Each variant has a statically checked size of 0x40 bytes, + // contains only initialized integer fields with no implicit padding, + // and is borrowed for the lifetime of `self`. The byte view has align 1. + unsafe { &*record } + } +} + +/// Device-control opcodes the driver sends. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum DeviceControlOp { + /// Reply to a parameter-buffer grow request ([`TvbGrowReply`]). + TvbGrowReply = 0x08, + /// Set the idle power-off policy ([`IdlePowerOff`]). + IdlePowerOff = 0x0a, + /// Release a logical queue's scheduler state ([`SchedulerStateRelease`]). + SchedulerStateRelease = 0x14, + /// Release a free list's buffer slot ([`FreeListRelease`]). + FreeListRelease = 0x2e, + /// Stop the queues named by a closed logical scheduler. + ContextKill = 0x29, +} + +impl DeviceControlOp { + /// Tells whether records with this opcode count towards the host + /// power-assert tally. The firmware compares the tally with the counted + /// records it has drained and keeps the GPU cores powered while they + /// differ. + pub(crate) const fn asserts_gpu_power(self) -> bool { + !matches!(self, Self::FreeListRelease) + } +} + +/// The fields of a parameter-buffer grow request event that its reply +/// echoes. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct TvbGrowRequest { + /// Buffer slot of the parameter buffer. + pub(crate) buffer_slot: u32, + /// VM slot of the request, or -1. + pub(crate) vm_slot: i32, + /// Request counter. + pub(crate) counter: u32, + /// Sub-pipe of the request. + pub(crate) subpipe: u32, + /// Firmware halt count the request was made under. + pub(crate) halt_count: u64, +} + +/// Reply to a parameter-buffer grow request. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct TvbGrowReply { + /// [`DeviceControlOp::TvbGrowReply`]. + pub(crate) opcode: u32, + /// Non-zero when the parameter buffer grew. + pub(crate) grew: u32, + /// Buffer slot of the request. + pub(crate) buffer_slot: u32, + /// VM slot of the request. + pub(crate) vm_slot: i32, + /// Request counter. + pub(crate) counter: u32, + /// Sub-pipe of the request. + pub(crate) subpipe: u32, + /// Halt count of the request. + pub(crate) halt_count: u64, + pub(crate) unk_20: [u8; 0x20], +} + +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); + +impl TvbGrowReply { + /// Replies to `request`; `grew` tells whether the buffer was grown. + pub(crate) const fn new(request: &TvbGrowRequest, grew: bool) -> Self { + Self { + opcode: DeviceControlOp::TvbGrowReply as u32, + grew: grew as u32, + buffer_slot: request.buffer_slot, + vm_slot: request.vm_slot, + counter: request.counter, + subpipe: request.subpipe, + halt_count: request.halt_count, + unk_20: [0; 0x20], + } + } +} + +/// Idle power-off policy argument of [`IdlePowerOff`]. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum IdlePolicy { + /// Keep the GPU cores powered and power them up now. + KeepPowered = 0, + /// Let the firmware power the GPU cores off when idle. + AllowPowerOff = 1, +} + +/// Sets the firmware's idle power-off policy. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct IdlePowerOff { + /// [`DeviceControlOp::IdlePowerOff`]. + pub(crate) opcode: u32, + /// [`IdlePolicy`]. + pub(crate) policy: u32, + pub(crate) unk_08: [u8; 0x38], +} + +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); + +impl IdlePowerOff { + /// Requests `policy`. + pub(crate) const fn new(policy: IdlePolicy) -> Self { + Self { + opcode: DeviceControlOp::IdlePowerOff as u32, + policy: policy as u32, + unk_08: [0; 0x38], + } + } +} + +/// Releases the scheduler state of a logical queue whose work has drained. +/// +/// The page may be freed once the firmware has consumed the record. +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct SchedulerStateRelease { + /// [`DeviceControlOp::SchedulerStateRelease`]. + pub(crate) opcode: u32, + pub(crate) unk_04: u32, + /// The scheduler-page bytes at offsets 0x26, 0, 1 and 4, in record order. + /// Read when the release record is built. + pub(crate) id: [u8; 4], + /// GPU address of the scheduler-state page. + pub(crate) page_va: u64, + pub(crate) unk_14: [u8; 0x2c], +} + +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); +static_assert!(core::mem::offset_of!(SchedulerStateRelease, page_va) == 0x0c); + +impl SchedulerStateRelease { + /// Releases the scheduler page with its identity bytes in record order. + pub(crate) const fn new(page_va: u64, id: [u8; 4]) -> Self { + Self { + opcode: DeviceControlOp::SchedulerStateRelease as u32, + unk_04: 0, + id, + page_va, + unk_14: [0; 0x2c], + } + } +} + +/// Releases the buffer slot of a free list after its last user. +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct FreeListRelease { + /// [`DeviceControlOp::FreeListRelease`]. + pub(crate) opcode: u32, + /// Free-list generation of the registration being released. + pub(crate) generation: u64, + /// Buffer slot of the free list. + pub(crate) buffer_slot: u32, + pub(crate) unk_10: [u8; 0x30], +} + +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); +static_assert!(core::mem::offset_of!(FreeListRelease, buffer_slot) == 0x0c); + +impl FreeListRelease { + /// Releases `buffer_slot`, registered at `generation`. + pub(crate) const fn new(generation: u64, buffer_slot: u32) -> Self { + Self { + opcode: DeviceControlOp::FreeListRelease as u32, + generation, + buffer_slot, + unk_10: [0; 0x30], + } + } +} + +/// Number of slots in a work ring. +pub(crate) const WORK_RING_SLOTS: u32 = 256; +/// Number of engines with a work ring per priority. +const WORK_RINGS_PER_PRIORITY: usize = 3; + +/// Index in the channel table of the work ring for `data_master` at the +/// priority of every driver-created queue. +pub(crate) const fn work_ring_index(data_master: DataMaster) -> usize { + QUEUE_PRIORITY as usize * WORK_RINGS_PER_PRIORITY + data_master as usize +} + +/// One work-ring slot: a request to run a queue's newly published commands. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct WorkSlot { + pub(crate) unk_00: u64, + /// GPU address of the queue record. + pub(crate) queue_va: u64, + /// [`DataMaster`] of the queue. + pub(crate) data_master: u32, + /// Item-ring producer after the published commands. + pub(crate) items_end: u16, + /// Queue to run. + pub(crate) qid: u8, + /// One in the first slot naming this queue on this ring. + pub(crate) first: u8, +} + +static_assert!(size_of::() == 0x18); + +impl WorkSlot { + /// Builds the slot asking the firmware to run queue `qid` up to item-ring + /// position `items_end`. `first` tells whether this is the first slot + /// naming the queue on this ring. + pub(crate) fn new( + data_master: DataMaster, + queue_va: u64, + qid: u8, + items_end: u16, + first: bool, + ) -> Result { + if qid > QID_MAX { + return Err(EINVAL); + } + Ok(Self { + unk_00: 0, + queue_va, + data_master: data_master as u32, + items_end, + qid, + first: first as u8, + }) + } +} + +/// Stop a closed context's scheduler page and echo its cookie through event 12. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct ContextKill { + opcode: u32, + page_va: u64, + cookie: u64, + reserved: [u8; 0x2c], +} +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); +static_assert!(core::mem::offset_of!(ContextKill, page_va) == 4); +static_assert!(core::mem::offset_of!(ContextKill, cookie) == 0x0c); +impl ContextKill { + pub(crate) fn new(page_va: u64, cookie: u64) -> Self { + Self { + opcode: DeviceControlOp::ContextKill as u32, + page_va, + cookie, + reserved: [0; 0x2c], + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs new file mode 100644 index 0000000000000..22f8259508ff6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -0,0 +1,379 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Compute work descriptor. +//! +//! A compute kick launches one [`ComputeDescriptor`]. The descriptor embeds +//! the two register arrays its kick entry binds: the main array and a +//! four-entry array that writes the operand-state registers with their +//! regions exchanged. + +use kernel::prelude::*; + +use super::kick::{KickTimestamp, RegisterArray, RegisterArrayBinding, RegisterWrite}; +use super::queue::CommandTag; +use super::{ + clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, + UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, +}; + +/// Size of a compute descriptor. +pub(crate) const COMPUTE_DESCRIPTOR_SIZE: usize = 0x4000; +/// QoS class of compute kicks. +pub(crate) const COMPUTE_QOS_CLASS: u8 = 0x08; +/// Event mask of compute kicks. +pub(crate) const COMPUTE_KICK_EVENT_MASK: [u64; 4] = [0, 0, 4, 0]; + +const COMPUTE_REGISTERS: usize = 40; +const ALT_REGISTERS: usize = 4; + +/// Object IDs of every compute kick. +const OBJECT_IDS: ObjectIds = ObjectIds { + predecessor: 0x0100_01d7, + current: 0x0200_01dc, +}; +/// Values of the first and second write of register 0x1a440. +const UNK_1A440: u64 = 0x0000_0001_5402_4201; +const UNK_1A440_FINAL: u64 = 0x0000_0001_5402_4209; +/// Value of register 0x1a458: [`UNK_1C8F8`] below bits 31:20 = 0x10c. +const UNK_1A458: u64 = 0x10c0_0000 | UNK_1C8F8; + +/// Offsets in the preempt buffer of the four state words the program names. +const PREEMPT_STATE: [u64; 4] = [0x1480, 0x1488, 0x1490, 0x1498]; +/// Offsets in the operand-state buffer of the regions the program names. +const OPERAND_STATE_10229: u64 = 0x12800; +const OPERAND_STATE_140A8: u64 = 0x13000; +const OPERAND_STATE_10099: u64 = 0x9405; +const OPERAND_STATE_10091: u64 = 0x12400; +const OPERAND_STATE_0A5C1: u64 = 0x0005; +const OPERAND_STATE_0A5C9: u64 = 0x9000; + +/// Checkpoints into the main register array, as entry counts: after the +/// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full +/// count. +const CHECKPOINTS: [u16; 4] = [38, 39, 39, COMPUTE_REGISTERS as u16]; + +/// Inputs of one compute descriptor. +pub(crate) struct ComputeArgs { + /// Base of the queue's USC window. + pub(crate) usc_base: u64, + /// GPU address of the control stream. + pub(crate) cdm_va: u64, + /// GPU address of the control-stream terminator. + pub(crate) cdm_end_va: u64, + /// GPU address of the sampler heap, or zero. + pub(crate) sampler_heap_va: u64, + /// Number of samplers in the heap. + pub(crate) sampler_count: u32, + /// GPU address of this descriptor, as seen by the work's VM. + pub(crate) descriptor_va: u64, + /// UAT context the work runs in. + pub(crate) context_id: u16, + /// Allocation generation of that context. + pub(crate) context_generation: u8, + /// State word of the logical queue's work-state node. + pub(crate) work_state: u32, + /// GPU address of the logical queue's work-state node. + pub(crate) work_state_va: u64, + /// Queue the work is kicked on. + pub(crate) qid: u8, + /// Number of earlier kicks of the queue. + pub(crate) kick_count: u32, + /// Timestamp the work is kicked at. + pub(crate) kick: KickTimestamp, + /// GPU address of the preempt buffer, as seen by the work's VM. + pub(crate) preempt_va: u64, + /// GPU address of the operand-state buffer, as seen by the work's VM. + pub(crate) operand_state_va: u64, + /// GPU address of the private-memory usage record, as seen by the work's + /// VM. + pub(crate) usage_va: u64, + /// Firmware alias of the private-memory usage record. + pub(crate) usage_fw_va: u64, + /// Buffer slot of the USC private-memory free list. + pub(crate) free_list_slot: u32, + /// GPU address of the free list's shared control object. + pub(crate) free_list_control_va: u64, + /// GPU address the completion stamp value is stored to. + pub(crate) stamp_va: u64, + /// GPU address of the word following the completion stamp. + pub(crate) aux_stamp_va: u64, + /// GPU addresses of the two status words. + pub(crate) status_va: [u64; 2], + /// GPU address of the start/end timestamp pair. + pub(crate) timestamp_va: u64, +} + +fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS]> { + let preempt = args.preempt_va; + let operand = args.operand_state_va; + let key = work_key(args.context_id, args.work_state); + let ids = OBJECT_IDS.word(); + + Ok([ + RegisterWrite::new(0x1a510, preempt), + RegisterWrite::new(0x1a420, args.cdm_va), + RegisterWrite::new(0x1a4d0, offset_va(preempt, PREEMPT_STATE[0])?), + RegisterWrite::new(0x1a4d8, offset_va(preempt, PREEMPT_STATE[1])?), + RegisterWrite::new(0x1a4e0, offset_va(preempt, PREEMPT_STATE[2])?), + RegisterWrite::new(0x1a4e8, offset_va(preempt, PREEMPT_STATE[3])?), + RegisterWrite::new(0x10071, args.usc_base), + // Helper program binary, data and configuration: none. + RegisterWrite::new(0x11841, 0), + RegisterWrite::new(0x11849, 0), + RegisterWrite::new(0x11f81, 0), + RegisterWrite::new(0x1a440, UNK_1A440), + RegisterWrite::new(0x1a458, UNK_1A458), + RegisterWrite::new(0x101d9, UNK_101D9), + RegisterWrite::new(0x1a089, 0), + RegisterWrite::new(0x1a091, 0), + RegisterWrite::new(0x1a059, 0), + RegisterWrite::new(0x1a061, 0), + RegisterWrite::new(0x1a0b9, 0), + RegisterWrite::new(0x1a0c1, 0), + RegisterWrite::new(0x101d1, 0), + RegisterWrite::new(0x0d479, 0), + RegisterWrite::new(0x1a0e9, UNK_1A0E9), + RegisterWrite::new(0x107a1, UNK_10791), + RegisterWrite::new(0x0a599, UNK_0A599), + RegisterWrite::new(0x0d411, UNK_0D411), + RegisterWrite::new(0x1a540, ids), + RegisterWrite::new(0x014a9, ids), + RegisterWrite::new(0x0a351, ids), + RegisterWrite::new(0x10201, key), + RegisterWrite::new(0x10428, key), + RegisterWrite::new(0x14028, args.free_list_slot as u64), + RegisterWrite::new(0x14070, args.usage_va | 1), + RegisterWrite::new(0x10229, offset_va(operand, OPERAND_STATE_10229)?), + RegisterWrite::new(0x140a8, offset_va(operand, OPERAND_STATE_140A8)?), + RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_10099)?), + RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_10091)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_0A5C1)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_0A5C9)?), + RegisterWrite::new(0x1a440, UNK_1A440_FINAL), + RegisterWrite::new(0x0a599, UNK_0A599), + ]) +} + +/// The second array exchanges the operand-state regions of registers +/// 0x10099/0x0a5c1 and 0x10091/0x0a5c9. +fn alt_registers(args: &ComputeArgs) -> Result<[RegisterWrite; ALT_REGISTERS]> { + let operand = args.operand_state_va; + Ok([ + RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_0A5C1)?), + RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_0A5C9)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_10099)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_10091)?), + ]) +} + +/// Compute work descriptor (tag 3). +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct ComputeDescriptor { + /// [`CommandTag::Compute`]. + pub(crate) tag: u32, + pub(crate) unk_04: u64, + /// UAT context of the work. + pub(crate) context_id: u32, + /// GPU address of the logical queue's work-state node. + pub(crate) work_state_va: u64, + /// Checkpoints into the main register array. + pub(crate) checkpoints: [u16; 4], + pub(crate) unk_20: [u8; 0x20], + /// Main register array. + pub(crate) registers: RegisterArray, + /// Second register array. + pub(crate) alt_registers: RegisterArray, + pub(crate) unk_e80: [u8; 0x58], + /// GPU address of the preempt buffer. + pub(crate) preempt_va: u64, + /// GPU address of the control-stream terminator. + pub(crate) cdm_end_va: u64, + pub(crate) unk_ee8: [u8; 0x20], + /// First value of register 0x1a440. + pub(crate) unk_f08: u64, + pub(crate) unk_f10: [u8; 0x10], + /// Predecessor object ID. + pub(crate) predecessor_id: u32, + pub(crate) unk_f24: u32, + pub(crate) unk_f28: u32, + /// GPU address of the sampler heap. + pub(crate) sampler_heap_va: u64, + /// Number of samplers. + pub(crate) sampler_count: u32, + /// Number of samplers plus one, or zero without samplers. + pub(crate) sampler_max: u32, + pub(crate) unk_f3c: u32, + /// GPU address the firmware stores `stamp_value` to on completion. + pub(crate) stamp_va: u64, + /// GPU address of the word following the completion stamp. + pub(crate) aux_stamp_va: u64, + /// Completion stamp value of the kick timestamp. + pub(crate) stamp_value: u32, + /// QID of the compute queue. + pub(crate) qid: u32, + pub(crate) unk_f58: u32, + pub(crate) unk_f5c: u32, + /// Kick timestamp. + pub(crate) kick_timestamp: u64, + /// Current object ID. + pub(crate) current_id: u64, + /// Number of earlier kicks of the queue. + pub(crate) kick_count: u32, + pub(crate) unk_f74: [u8; 8], + /// GPU addresses of the two status words. + pub(crate) status_va: [u64; 2], + /// GPU address the start timestamp is written to. + pub(crate) timestamp_start_va: u64, + /// GPU address the end timestamp is written to. + pub(crate) timestamp_end_va: u64, + pub(crate) unk_f9c: [u8; 0x14], + pub(crate) unk_fb0: u16, + /// GPU address of the USC free list's shared control object. + pub(crate) free_list_control_va: u64, + pub(crate) unk_fba: u32, + pub(crate) unk_fbe: [u8; 7], + pub(crate) unk_fc5: u8, + pub(crate) unk_fc6: [u8; 5], + /// Firmware alias of the private-memory usage record. + pub(crate) usage_va: u64, + /// Allocation generation of the UAT context. + pub(crate) context_generation: u8, + pub(crate) unk_fd4: [u8; 0x302c], +} + +static_assert!(size_of::() == COMPUTE_DESCRIPTOR_SIZE); +static_assert!(core::mem::offset_of!(ComputeDescriptor, registers) == 0x40); +static_assert!(core::mem::offset_of!(ComputeDescriptor, alt_registers) == 0x760); +static_assert!(core::mem::offset_of!(ComputeDescriptor, preempt_va) == 0xed8); +static_assert!(core::mem::offset_of!(ComputeDescriptor, unk_f08) == 0xf08); +static_assert!(core::mem::offset_of!(ComputeDescriptor, predecessor_id) == 0xf20); +static_assert!(core::mem::offset_of!(ComputeDescriptor, sampler_heap_va) == 0xf2c); +static_assert!(core::mem::offset_of!(ComputeDescriptor, stamp_va) == 0xf40); +static_assert!(core::mem::offset_of!(ComputeDescriptor, kick_timestamp) == 0xf60); +static_assert!(core::mem::offset_of!(ComputeDescriptor, current_id) == 0xf68); +static_assert!(core::mem::offset_of!(ComputeDescriptor, status_va) == 0xf7c); +static_assert!(core::mem::offset_of!(ComputeDescriptor, free_list_control_va) == 0xfb2); +static_assert!(core::mem::offset_of!(ComputeDescriptor, usage_va) == 0xfcb); +static_assert!(core::mem::offset_of!(ComputeDescriptor, context_generation) == 0xfd3); + +// SAFETY: `ComputeDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for ComputeDescriptor {} + +impl ComputeDescriptor { + const REGISTERS_OFFSET: u64 = core::mem::offset_of!(Self, registers) as u64; + const ALT_REGISTERS_OFFSET: u64 = core::mem::offset_of!(Self, alt_registers) as u64; + const UNK_F28: u32 = u32::MAX; + const UNK_FB0: u16 = 0x1a; + const UNK_FBA: u32 = 0xe0a0_0001; + const UNK_FC5: u8 = 0x9f; + + /// Writes the complete descriptor. + pub(crate) fn write(&mut self, args: &ComputeArgs) -> Result { + if args.cdm_end_va < args.cdm_va + || (args.sampler_heap_va == 0) != (args.sampler_count == 0) + || args.sampler_heap_va & 7 != 0 + { + return Err(EINVAL); + } + let registers = compute_registers(args)?; + let alt_registers = alt_registers(args)?; + let registers_va = offset_va(args.descriptor_va, Self::REGISTERS_OFFSET)?; + let alt_registers_va = offset_va(args.descriptor_va, Self::ALT_REGISTERS_OFFSET)?; + let timestamp_end_va = timestamp_end_va(args.timestamp_va)?; + let sampler_max = sampler_max(args.sampler_count)?; + + clear(self); + self.tag = CommandTag::Compute as u32; + self.context_id = args.context_id.into(); + self.work_state_va = args.work_state_va; + self.checkpoints = CHECKPOINTS; + self.registers.set(registers_va, ®isters)?; + self.alt_registers.set(alt_registers_va, &alt_registers)?; + self.preempt_va = args.preempt_va; + self.cdm_end_va = args.cdm_end_va; + self.unk_f08 = UNK_1A440; + self.predecessor_id = OBJECT_IDS.predecessor; + self.unk_f28 = Self::UNK_F28; + self.sampler_heap_va = args.sampler_heap_va; + self.sampler_count = args.sampler_count; + self.sampler_max = sampler_max; + self.stamp_va = args.stamp_va; + self.aux_stamp_va = args.aux_stamp_va; + self.stamp_value = stamp_value(args.kick.get() as u32); + self.qid = args.qid.into(); + self.kick_timestamp = args.kick.get(); + self.current_id = OBJECT_IDS.current.into(); + self.kick_count = args.kick_count; + self.status_va = args.status_va; + self.timestamp_start_va = args.timestamp_va; + self.timestamp_end_va = timestamp_end_va; + self.unk_fb0 = Self::UNK_FB0; + self.free_list_control_va = args.free_list_control_va; + self.unk_fba = Self::UNK_FBA; + self.unk_fc5 = Self::UNK_FC5; + self.usage_va = args.usage_fw_va; + self.context_generation = args.context_generation; + Ok(()) + } + + /// Returns the register arrays a compute kick binds; `descriptor_va` is + /// the descriptor's address as seen by the work's VM. + pub(crate) fn register_bindings( + descriptor_va: u64, + ) -> Result<[Option; 4]> { + let registers = RegisterArrayBinding::new( + offset_va(descriptor_va, Self::REGISTERS_OFFSET)?, + CHECKPOINTS[0] as u8, + )?; + let alt_registers = RegisterArrayBinding::new( + offset_va(descriptor_va, Self::ALT_REGISTERS_OFFSET)?, + ALT_REGISTERS as u8, + )?; + Ok([Some(registers), Some(alt_registers), None, None]) + } +} + +/// One retained queue-context record. The descriptor pointer is refreshed for each compute +/// publication; the remaining fields are installed once with the queue. +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct QueueContextItem { + unk_00: u64, + unk_08: u64, + descriptor_va: u64, + queue_va: u64, + unk_20: u64, + qid: u64, + unk_30: [u8; 0x100], + unk_130: u64, + unk_138: u64, + unk_140: [u8; 0x10], + unk_150: u64, + unk_158: u64, + unk_160: [u8; 0x18], + unk_178: u64, + unk_180: [u8; 0x80], +} +impl QueueContextItem { + pub(crate) fn new(descriptor_va: u64, queue_va: u64, qid: u8) -> Self { + Self { + unk_00: 0x1000_1000_0000_0004, + unk_08: 0, + descriptor_va, + queue_va, + unk_20: 0xffff_0801_0000_0001, + qid: u64::from(qid) << 40, + unk_30: [0; 0x100], + unk_130: 0, + unk_138: 4, + unk_140: [0; 0x10], + unk_150: 0x0001_1003_8001_a002, + unk_158: 0x0000_2003_8001_a03b, + unk_160: [0; 0x18], + unk_178: 0x003f_ffff_ffff_ffff, + unk_180: [0; 0x80], + } + } +} +static_assert!(size_of::() == 0x200); diff --git a/drivers/gpu/drm/asahi/g17/fw/context.rs b/drivers/gpu/drm/asahi/g17/fw/context.rs new file mode 100644 index 0000000000000..51dbde231e238 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/context.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scheduler and timestamp-queue state shared with firmware. Only a freshly +//! leased node may be initialized. After publication, the host writes its +//! separate submitted counter and reads the node completion word individually. + +use kernel::prelude::*; + +#[repr(C)] +pub(crate) struct Scheduler { + unk0: u64, + unk8: [u64; 3], + unk20: u64, + unk28: u64, + unk30: u64, + unk38: [u8; 0x3fc8], +} +impl Scheduler { + /// Called on zeroed storage before its first publication. + pub(crate) fn init(&mut self) { + self.unk0 = 0x0000_0100_0000_ffff; + self.unk20 = 0x0002_0000_0000_0000; + self.unk30 = 0x0000_0000_ff00_0000; + } +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub(crate) struct WorkNode { + pub(crate) submitted: u64, + pub(crate) word: u32, + pub(crate) completed: u32, + pub(crate) threshold: u32, + unk14: [u32; 45], +} +impl WorkNode { + pub(crate) fn new(submitted: u64, word: u32) -> Option { + if submitted == 0 || submitted & 3 != 0 || word & 63 == 63 { + return None; + } + Some(Self { + submitted, + word, + completed: 0, + threshold: 0x50, + unk14: [0; 45], + }) + } +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub(crate) struct WorkHead { + pub(crate) next: u64, + pub(crate) previous: u64, + pub(crate) unk10: u32, + pub(crate) unk14: u32, +} + +const _: () = { + assert!(core::mem::size_of::() == 0x4000); + assert!(core::mem::size_of::() == 0xc8); + assert!(core::mem::offset_of!(WorkNode, completed) == 0xc); + assert!(core::mem::size_of::() == 0x18); +}; + +// SAFETY: These shared records contain only integer fields and arrays. +unsafe impl Zeroable for Scheduler {} +// SAFETY: WorkNode contains only integers; zero is valid for every field. +unsafe impl Zeroable for WorkNode {} +// SAFETY: WorkHead contains only integer addresses and words. +unsafe impl Zeroable for WorkHead {} diff --git a/drivers/gpu/drm/asahi/g17/fw/freelist.rs b/drivers/gpu/drm/asahi/g17/fw/freelist.rs new file mode 100644 index 0000000000000..2e5708dddd455 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/freelist.rs @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! USC free-list page addresses and block run tables. + +use super::initdata::{FreeListRunSlot, FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; +use kernel::prelude::*; + +pub(crate) const PAGE_LIST_SIZE: usize = 0x20_0000; +pub(crate) const RUN_LIST_SIZE: usize = 0x1_0000; +pub(crate) const TABLE_SIZE: usize = PAGE_LIST_SIZE + RUN_LIST_SIZE; +pub(crate) const CONTROL_SIZE: usize = 0x4000; +pub(crate) const RENDER_BLOCKS: usize = 22; +pub(crate) const COMPUTE_BLOCKS: usize = 21; + +/// Render reserves eight run words per block; compute packs runs consecutively. +#[derive(Copy, Clone, Debug)] +pub(crate) enum RunLayout { + Render, + Compute, +} + +impl RunLayout { + const fn words(self) -> usize { + match self { + Self::Render => size_of::() / 8, + Self::Compute => 1, + } + } +} + +#[repr(C)] +pub(crate) struct PoolTables { + pages: [u64; PAGE_LIST_SIZE / 8], + runs: [u64; RUN_LIST_SIZE / 8], +} + +// SAFETY: The table consists entirely of integer arrays. +unsafe impl Zeroable for PoolTables {} +static_assert!(size_of::() == TABLE_SIZE); +static_assert!(core::mem::offset_of!(PoolTables, runs) == PAGE_LIST_SIZE); + +impl PoolTables { + /// Enumerates only the initialized run words and page addresses, in their + /// publication order. Reserved run words and unused table space stay zero. + pub(crate) fn for_each_word( + base: u64, + blocks: usize, + layout: RunLayout, + mut put: impl FnMut(usize, u64), + ) -> Result { + if blocks > RUN_LIST_SIZE / (layout.words() * 8) + || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) + { + return Err(EINVAL); + } + let buffers = base.checked_add(TABLE_SIZE as u64).ok_or(EOVERFLOW)?; + buffers + .checked_add((blocks * FREE_LIST_BLOCK_SIZE) as u64) + .ok_or(EOVERFLOW)?; + for block in 0..blocks { + let address = buffers + (block * FREE_LIST_BLOCK_SIZE) as u64; + put( + PAGE_LIST_SIZE / 8 + block * layout.words(), + FreeListRunSlot::new(address).run, + ); + for page in 0..FREE_LIST_BLOCK_PAGES as usize { + put( + block * FREE_LIST_BLOCK_PAGES as usize + page, + address + + (page * (FREE_LIST_BLOCK_SIZE / FREE_LIST_BLOCK_PAGES as usize)) as u64, + ); + } + } + Ok(()) + } + + /// Initializes a zeroed table before its first publication. + pub(crate) fn init(&mut self, base: u64, blocks: usize, layout: RunLayout) -> Result { + Self::for_each_word(base, blocks, layout, |index, value| { + if index < self.pages.len() { + self.pages[index] = value; + } else { + self.runs[index - self.pages.len()] = value; + } + }) + } +} + +/// Mutable page-pool state carried between the retained control record and +/// its global descriptor. Offsets name the source control-record words. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct PagePoolState { + pub(crate) page_list_va: u64, + pub(crate) unk_1c: u32, + pub(crate) unk_20: u32, + pub(crate) unk_24: u32, + pub(crate) unk_28: u32, + pub(crate) page_count: u32, +} + +impl PagePoolState { + /// The fourth descriptor word belongs to firmware and survives every load. + pub(crate) const fn descriptor(self, unk_18: u64) -> super::initdata::PagePoolDescriptor { + super::initdata::PagePoolDescriptor { + page_list: ((self.unk_1c as u64) << 41) | (self.page_list_va >> 7), + counters: ((self.unk_24 as u64) << 33) + | ((self.unk_20 as u64) << 5) + | ((self.unk_28 as u64) << 61), + page_count: self.page_count as u64, + unk_18, + } + } + + /// Address and allocation unit remain properties of the retained pool. + pub(crate) const fn updated(self, descriptor: &super::initdata::PagePoolDescriptor) -> Self { + Self { + unk_20: ((descriptor.counters >> 5) & 0x0fff_ffff) as u32, + unk_24: ((descriptor.counters >> 33) & 0x0fff_ffff) as u32, + unk_28: (descriptor.counters >> 61) as u32, + page_count: descriptor.page_count as u32, + ..self + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs new file mode 100644 index 0000000000000..c25b843a913e7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -0,0 +1,3063 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 firmware initialization data. +//! +//! Each of the two firmware instances (primary and secondary) is handed one +//! root at boot. The roots name the objects the firmware needs before any +//! work is submitted: +//! +//! - region A and region C, shared by both instances; +//! - one main configuration per instance, holding its channel table; +//! - the hardware-data bundle, one allocation holding the hardware data, the +//! power configuration, the work rings and both main configurations with +//! their device-control rings; +//! - per-instance status blocks inside the private cluster, the fixed-address +//! arena the firmware also writes at fixed offsets. +//! +//! This module defines those layouts and the pure functions that fill them. +//! Allocation, mapping and publication belong to the builder. Initialization +//! requires exclusive access before either firmware instance can access the +//! object. These constructors must not be used to update a published object: +//! its live fields require individual accesses of the widths documented below. +//! +//! All objects start out zeroed. An `unk_` field documents the value +//! the firmware requires there; an `unk_` field without a doc comment is zero. + +use kernel::prelude::*; + +use super::clear; + +/// Interface version at root `+0x00`; the firmware rejects any other. +pub(crate) const INTERFACE_VERSION: [u16; 4] = [0x04c0, 0x0396, 0xa322, 0x0c8a]; + +/// UAT page size, as a shift. +pub(crate) const PAGE_BITS: u32 = 14; +/// UAT page size. +pub(crate) const PAGE_SIZE: usize = 1 << PAGE_BITS; +/// Page-table levels below the root. +const UAT_LEVELS: usize = 3; +/// Address bits resolved by one page-table level. +const BITS_PER_LEVEL: u32 = PAGE_BITS - 3; + +/// Firmware instance, encoded in its root. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum Role { + /// Runs the scheduler, the device-control ring and all work channels. + Primary = 0, + /// Control-only peer of the primary. + Secondary = 1, +} + +// --------------------------------------------------------------------------- +// Roots +// --------------------------------------------------------------------------- + +/// One UAT page-table level, as the firmware walks it. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct LevelDescriptor { + /// Required: 8. + pub(crate) unk_0: u8, + /// Page size shift of the tables at this level. + pub(crate) table_page_bits: u8, + /// Page size shift of the pages they map. + pub(crate) page_bits: u8, + /// Lowest input-address bit indexing this level. + pub(crate) index_shift: u8, + /// Entries per table. + pub(crate) entries: u16, + /// Table size in bytes. + pub(crate) table_size: u16, + /// Required: 1. + pub(crate) unk_8: u64, + /// Output-address bits of an entry. + pub(crate) phys_mask: u64, + /// Input-address bits indexing this level. + pub(crate) index_mask: u64, +} + +// SAFETY: `LevelDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for LevelDescriptor {} + +static_assert!(size_of::() == 0x20); + +impl LevelDescriptor { + /// Returns the descriptor of `level` (0 = top) for a UAT with `ias` + /// input-address and `oas` output-address bits. + const fn new(level: u32, ias: u32, oas: u32) -> Self { + let index_shift = PAGE_BITS + (UAT_LEVELS as u32 - 1 - level) * BITS_PER_LEVEL; + let index_bits = if level == 0 { + ias - index_shift + } else { + BITS_PER_LEVEL + }; + let entries = 1u64 << index_bits; + Self { + unk_0: 8, + table_page_bits: PAGE_BITS as u8, + page_bits: PAGE_BITS as u8, + index_shift: index_shift as u8, + entries: entries as u16, + table_size: PAGE_SIZE as u16, + unk_8: 1, + phys_mask: ((1u64 << oas) - 1) & !(PAGE_SIZE as u64 - 1), + index_mask: (entries - 1) << index_shift, + } + } +} + +/// Addresses and geometry common to both roots. +pub(crate) struct RootArgs { + /// Region A. + pub(crate) region_a_va: u64, + /// Region C. + pub(crate) region_c_va: u64, + /// This instance's main configuration. + pub(crate) main_config_va: u64, + /// This instance's status A block. + pub(crate) status_a_va: u64, + /// UAT input-address bits of the firmware's context. + pub(crate) ias: u32, + /// UAT output-address bits. + pub(crate) oas: u32, +} + +/// Root of the primary instance; the secondary extends it ([`SecondaryRoot`]). +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct Root { + /// [`INTERFACE_VERSION`]. + pub(crate) version: [u16; 4], + /// Region A. + pub(crate) region_a_va: u64, + pub(crate) unk_10: u64, + /// Main configuration of this instance. + pub(crate) main_config_va: u64, + /// Region C. + pub(crate) region_c_va: u64, + /// [`Role`] of this instance. + pub(crate) role: u32, + /// Required: 1. + pub(crate) unk_2c: u32, + /// UAT page size. + pub(crate) page_size: u16, + /// UAT page size shift. + pub(crate) page_bits: u8, + /// Number of page-table levels. + pub(crate) levels: u8, + /// Page-table levels, top first. + pub(crate) level: [LevelDescriptor; UAT_LEVELS], + pub(crate) unk_94: [u8; 0x14], + /// Status A block of this instance. + pub(crate) status_a_va: u64, + /// Primary status B ([`PrimaryStatusB`]); zero for the secondary. + pub(crate) status_b_va: u64, +} + +// SAFETY: `Root` consists of integers and arrays of integers only. +unsafe impl Zeroable for Root {} + +static_assert!(core::mem::offset_of!(Root, role) == 0x28); +static_assert!(core::mem::offset_of!(Root, level) == 0x34); +static_assert!(core::mem::offset_of!(Root, status_a_va) == 0xa8); +static_assert!(size_of::() == 0xb8); + +impl Root { + fn new(role: Role, args: &RootArgs, status_b_va: u64) -> Self { + Self { + version: INTERFACE_VERSION, + region_a_va: args.region_a_va, + unk_10: 0, + main_config_va: args.main_config_va, + region_c_va: args.region_c_va, + role: role as u32, + unk_2c: 1, + page_size: PAGE_SIZE as u16, + page_bits: PAGE_BITS as u8, + levels: UAT_LEVELS as u8, + level: [ + LevelDescriptor::new(0, args.ias, args.oas), + LevelDescriptor::new(1, args.ias, args.oas), + LevelDescriptor::new(2, args.ias, args.oas), + ], + unk_94: [0; 0x14], + status_a_va: args.status_a_va, + status_b_va, + } + } + + /// Returns the primary root. + pub(crate) fn primary(args: &RootArgs, status_b_va: u64) -> Self { + Self::new(Role::Primary, args, status_b_va) + } +} + +/// Root of the secondary instance. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct SecondaryRoot { + /// Common part; the secondary has no status B. + pub(crate) root: Root, + /// The primary's [`PowerState`]. + pub(crate) power_state_va: u64, + /// The secondary's [`SecondaryConfig`]. + pub(crate) config_va: u64, +} + +// SAFETY: `SecondaryRoot` consists of integers and arrays of integers only. +unsafe impl Zeroable for SecondaryRoot {} + +static_assert!(size_of::() == 0xc8); + +impl SecondaryRoot { + /// Returns the secondary root. + pub(crate) fn new(args: &RootArgs, power_state_va: u64, config_va: u64) -> Self { + Self { + root: Root::new(Role::Secondary, args, 0), + power_state_va, + config_va, + } + } +} + +/// Offset of the secondary root from the primary root. Both live in one +/// allocation, and the firmware relies on this distance. +pub(crate) const SECONDARY_ROOT_OFFSET: usize = 0x8000; +/// Size of the allocation holding both roots. +pub(crate) const ROOTS_SIZE: usize = 0xc000; + +// --------------------------------------------------------------------------- +// Channels +// --------------------------------------------------------------------------- + +/// Work channels: TA, 3D and compute at each of four priorities. +pub(crate) const WORK_CHANNELS: usize = 12; +/// Engines with a work channel at each priority (TA, 3D, compute). +const WORK_ENGINES: usize = 3; +/// Work priorities. +const WORK_PRIORITIES: usize = WORK_CHANNELS / WORK_ENGINES; +/// Channel-table index of the device-control channel. +pub(crate) const CONTROL_CHANNEL: usize = 12; +/// Channel-table index of the event and firmware-log rings. +pub(crate) const EVENT_CHANNEL: usize = 13; +/// Channel-table index of the trace and statistics rings. +pub(crate) const TRACE_CHANNEL: usize = 14; +/// Channel-table index of the channel with a single state address. +const CHANNEL_15: usize = 15; +/// Entries in the channel table. +pub(crate) const CHANNELS: usize = 17; + +/// One channel-table entry: four GPU addresses whose meaning depends on the +/// channel. Work and control channels name three counters of their +/// [`ChannelState`] and their ring; the event and trace channels name two +/// state/ring pairs. +#[repr(C)] +pub(crate) struct ChannelEntry(pub(crate) [u64; 4]); + +// SAFETY: `ChannelEntry` is an array of integers. +unsafe impl Zeroable for ChannelEntry {} + +static_assert!(size_of::() == 0x20); + +impl ChannelEntry { + /// Returns the entry of a work or control channel. + const fn ring(state_va: u64, ring_va: u64) -> Self { + Self([ + state_va + ChannelState::CONSUMER as u64, + state_va + ChannelState::UNK_10 as u64, + state_va + ChannelState::PRODUCER as u64, + ring_va, + ]) + } +} + +/// State of a work or control channel in an instance's state grid. +/// +/// All three counters are live words. The firmware advances `consumer` and +/// `unk_10`; the host writes `producer` with single aligned stores, after the +/// records it publishes are complete. At boot the host stores +/// zero to the control channel's consumer, then zero to `unk_10`, then +/// [`OPENING_PRODUCER`] to its producer, primary instance before secondary, +/// after the rest of the initialization data is written. A full barrier follows +/// both instances' stores. These creation-time writes occur before publication; the firmware moves +/// all three to that value once it has drained the opening record. +#[repr(C)] +pub(crate) struct ChannelState { + /// Records the firmware has consumed. + pub(crate) consumer: u32, + pub(crate) unk_4: [u8; 0xc], + /// Firmware-owned. + pub(crate) unk_10: u32, + pub(crate) unk_14: [u8; 0xc], + /// Records the host has published. + pub(crate) producer: u32, + pub(crate) unk_24: [u8; 0x1c], +} + +// SAFETY: `ChannelState` consists of integers and arrays of integers only. +unsafe impl Zeroable for ChannelState {} + +static_assert!(core::mem::offset_of!(ChannelState, consumer) == 0); +static_assert!(core::mem::offset_of!(ChannelState, unk_10) == 0x10); +static_assert!(core::mem::offset_of!(ChannelState, producer) == 0x20); +static_assert!(size_of::() == 0x40); + +impl ChannelState { + pub(crate) const CONSUMER: usize = core::mem::offset_of!(Self, consumer); + pub(crate) const UNK_10: usize = core::mem::offset_of!(Self, unk_10); + /// Offset of [`Self::producer`] in the block. + pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); +} + +/// Offset of the device-control channel's state in an instance's state grid. +pub(crate) const CONTROL_STATE_OFFSET: usize = WORK_CHANNELS * size_of::(); + +/// Producer value of the device-control ring of either instance at boot: one +/// opening record. +pub(crate) const OPENING_PRODUCER: u32 = 1; + +/// Position of work channel `channel` in the state grid and in the work-ring +/// array: grouped by engine, priorities in descending order. +const fn work_slot(channel: usize) -> usize { + let engine = channel % WORK_ENGINES; + let priority = channel / WORK_ENGINES; + engine * WORK_PRIORITIES + (WORK_PRIORITIES - 1 - priority) +} + +/// Offset of work channel `channel`'s state in the state grid. +pub(crate) const fn work_state_offset(channel: usize) -> usize { + work_slot(channel) * size_of::() +} + +/// Bundle offset of work channel `channel`'s ring. +pub(crate) const fn work_ring_offset(channel: usize) -> usize { + bundle::WORK_RINGS + work_slot(channel) * bundle::WORK_RING_SIZE +} + +/// Offsets in a status area, the region starting at an instance's status A +/// block, of the rings the firmware fills. +pub(crate) mod status { + /// Event ring state. + pub(crate) const EVENT_STATE: u64 = 0x40; + /// Firmware-log state. + pub(crate) const LOG_STATE: u64 = 0x80; + /// Trace ring state. + pub(crate) const TRACE_STATE: u64 = 0x240; + /// Statistics ring state. + pub(crate) const STATS_STATE: u64 = 0x280; + /// Event ring. + pub(crate) const EVENT_RING: u64 = 0x2c0; + /// Firmware-log ring. + pub(crate) const LOG_RING: u64 = 0x4ac0; + /// The single address in channel 15. + pub(crate) const UNK_2D2C0: u64 = 0x2d2c0; + /// Trace ring. + pub(crate) const TRACE_RING: u64 = 0xa6ac0; + /// Statistics ring. + pub(crate) const STATS_RING: u64 = 0xafac0; +} + +// --------------------------------------------------------------------------- +// Hardware-data bundle layout +// --------------------------------------------------------------------------- + +/// Offsets in the hardware-data bundle. +pub(crate) mod bundle { + /// Size of the bundle allocation. + pub(crate) const SIZE: usize = 0x28000; + /// The [`super::HwData`]. + pub(crate) const HW_DATA: usize = 0; + /// The five views named by the primary main configuration. + pub(crate) const VIEWS: [usize; 5] = [0x2740, 0x3380, 0x4400, 0xbc80, 0xbd00]; + /// The [`super::PowerConfig`], inside view 2. + pub(crate) const POWER_CONFIG: usize = 0x5240; + /// The [`super::HwDataAux`]. + pub(crate) const AUX: usize = 0x8000; + /// A second copy of the [`super::PowerConfigHead`], inside views 3 and 4. + pub(crate) const POWER_CONFIG_COPY: usize = 0xd240; + /// Region the firmware writes, named twice by each main configuration. + pub(crate) const FIRMWARE_AREA: usize = 0xc500; + /// The work rings, see [`super::work_ring_offset`]. + pub(crate) const WORK_RINGS: usize = 0xc5c0; + /// Size of one work ring (256 slots of 0x18 bytes). + pub(crate) const WORK_RING_SIZE: usize = 0x1800; + /// Main configuration of each instance, each followed by its + /// device-control ring. + pub(crate) const MAIN_CONFIG: [usize; 2] = [0x1e5c0, 0x22a80]; + /// Separately allocated pages the firmware finds at fixed offsets from + /// the bundle: a zeroed page named by the primary main configuration. + pub(crate) const ZERO_PAGE: usize = 0x40000; + /// The two completion rings ([`super::CompletionRing`]). + pub(crate) const COMPLETION_RINGS: [usize; 2] = [0x48000, 0x70000]; +} + +// --------------------------------------------------------------------------- +// Main configuration +// --------------------------------------------------------------------------- + +/// Record the host stages in slot 0 of each instance's device-control ring +/// before boot. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct OpeningRecord { + /// Opcode: [`Self::PRIMARY`] or [`Self::SECONDARY`]. + pub(crate) opcode: u32, + pub(crate) unk_4: [u8; 0x3c], +} + +// SAFETY: `OpeningRecord` consists of integers and arrays of integers only. +unsafe impl Zeroable for OpeningRecord {} + +static_assert!(size_of::() == 0x40); + +impl OpeningRecord { + const PRIMARY: u32 = 0x16; + const SECONDARY: u32 = 0x2a; + + /// Returns the opening record of `role`. + pub(crate) const fn new(role: Role) -> Self { + Self { + opcode: match role { + Role::Primary => Self::PRIMARY, + Role::Secondary => Self::SECONDARY, + }, + unk_4: [0; 0x3c], + } + } +} + +/// Addresses both main configurations need. +pub(crate) struct MainConfigArgs { + /// Hardware-data bundle. + pub(crate) bundle_va: u64, + /// This main configuration. + pub(crate) main_config_va: u64, + /// This instance's state grid. + pub(crate) state_grid_va: u64, + /// This instance's status A block. + pub(crate) status_a_va: u64, + /// Context-0 alias of the parameter-buffer descriptor table. + pub(crate) pb_descriptors_low_va: u64, + /// Context-0 alias of the page-pool descriptor table. + pub(crate) page_pool_descriptors_low_va: u64, +} + +/// Addresses only the primary main configuration names. +pub(crate) struct PrimaryViews { + /// The zeroed page at [`bundle::ZERO_PAGE`]. + pub(crate) zero_page_va: u64, + /// Parameter-buffer descriptor table. + pub(crate) pb_descriptors_va: u64, + /// page-pool descriptor table. + pub(crate) page_pool_descriptors_va: u64, +} + +/// Main configuration of one instance. Its device-control ring +/// (256 records) follows it directly. +#[repr(C, packed)] +pub(crate) struct MainConfig { + /// Hardware-data bundle ([`HwData`]). + pub(crate) hw_data_va: u64, + /// [`bundle::FIRMWARE_AREA`]. + pub(crate) firmware_area_va: [u64; 2], + pub(crate) unk_18: [u8; 8], + /// Channel table. + pub(crate) channels: [ChannelEntry; CHANNELS], + pub(crate) unk_240: [u8; 0x10], + /// Non-zero enables firmware log output; the driver leaves it off. + pub(crate) log_enable: u32, + /// The bundle views ([`bundle::VIEWS`]); primary only. + pub(crate) view_va: [u64; 5], + pub(crate) unk_27c: [u8; 0x54], + /// [`PrimaryViews::zero_page_va`]; primary only. + pub(crate) zero_page_va: u64, + /// Context-0 alias of the parameter-buffer descriptor table. + pub(crate) pb_descriptors_low_va: u64, + /// Parameter-buffer descriptor table; primary only. + pub(crate) pb_descriptors_va: u64, + /// Context-0 alias of the page-pool descriptor table. + pub(crate) page_pool_descriptors_low_va: u64, + /// page-pool descriptor table; primary only. + pub(crate) page_pool_descriptors_va: u64, + pub(crate) unk_2f8: [u8; 8], + /// Required: 4 for the secondary. + pub(crate) unk_300: u32, + pub(crate) unk_304: [u8; 0x28], + /// Required: 1 for the primary. + pub(crate) unk_32c: u32, + pub(crate) unk_330: [u8; 0x14], + /// Required: 0xabcd_abcd for the primary. + pub(crate) unk_344: u32, + pub(crate) unk_348: [u8; 0x98], + /// Required: 0xff for the primary. + pub(crate) unk_3e0: u32, + pub(crate) unk_3e4: [u8; 0x8d], + /// [`PowerConfig`]; secondary only. + pub(crate) power_config_va: u64, + pub(crate) unk_479: [u8; 0x47], +} + +static_assert!(core::mem::offset_of!(MainConfig, channels) == 0x20); +static_assert!(core::mem::offset_of!(MainConfig, log_enable) == 0x250); +static_assert!(core::mem::offset_of!(MainConfig, view_va) == 0x254); +static_assert!(core::mem::offset_of!(MainConfig, zero_page_va) == 0x2d0); +static_assert!(core::mem::offset_of!(MainConfig, unk_300) == 0x300); +static_assert!(core::mem::offset_of!(MainConfig, unk_3e0) == 0x3e0); +static_assert!(core::mem::offset_of!(MainConfig, power_config_va) == 0x471); +static_assert!(size_of::() == MainConfig::CONTROL_RING); +static_assert!( + bundle::MAIN_CONFIG[0] + MainConfig::CONTROL_RING + 256 * size_of::() + == bundle::MAIN_CONFIG[1] +); + +// SAFETY: `MainConfig` consists of integers and arrays of integers only. +unsafe impl Zeroable for MainConfig {} + +impl MainConfig { + /// Offset of the device-control ring from the main configuration. + pub(crate) const CONTROL_RING: usize = 0x4c0; + + fn init(&mut self, role: Role, args: &MainConfigArgs) { + clear(self); + let bundle = args.bundle_va; + let grid = args.state_grid_va; + let status_a = args.status_a_va; + self.hw_data_va = bundle + bundle::HW_DATA as u64; + self.firmware_area_va = [bundle + bundle::FIRMWARE_AREA as u64; 2]; + if role == Role::Primary { + for channel in 0..WORK_CHANNELS { + self.channels[channel] = ChannelEntry::ring( + grid + work_state_offset(channel) as u64, + bundle + work_ring_offset(channel) as u64, + ); + } + } + self.channels[CONTROL_CHANNEL] = ChannelEntry::ring( + grid + CONTROL_STATE_OFFSET as u64, + args.main_config_va + Self::CONTROL_RING as u64, + ); + self.channels[EVENT_CHANNEL] = ChannelEntry([ + status_a + status::EVENT_STATE, + status_a + status::EVENT_RING, + status_a + status::LOG_STATE, + status_a + status::LOG_RING, + ]); + self.channels[TRACE_CHANNEL] = ChannelEntry([ + status_a + status::TRACE_STATE, + status_a + status::TRACE_RING, + status_a + status::STATS_STATE, + status_a + status::STATS_RING, + ]); + self.channels[CHANNEL_15] = ChannelEntry([status_a + status::UNK_2D2C0, 0, 0, 0]); + self.pb_descriptors_low_va = args.pb_descriptors_low_va; + self.page_pool_descriptors_low_va = args.page_pool_descriptors_low_va; + } + + /// Initializes the primary main configuration. + pub(crate) fn init_primary(&mut self, args: &MainConfigArgs, views: &PrimaryViews) { + self.init(Role::Primary, args); + self.view_va = bundle::VIEWS.map(|offset| args.bundle_va + offset as u64); + self.zero_page_va = views.zero_page_va; + self.pb_descriptors_va = views.pb_descriptors_va; + self.page_pool_descriptors_va = views.page_pool_descriptors_va; + self.unk_32c = 1; + self.unk_344 = 0xabcd_abcd; + self.unk_3e0 = 0xff; + } + + /// Initializes the secondary main configuration. + pub(crate) fn init_secondary(&mut self, args: &MainConfigArgs) { + self.init(Role::Secondary, args); + self.unk_300 = 4; + self.power_config_va = args.bundle_va + bundle::POWER_CONFIG as u64; + } +} + +// --------------------------------------------------------------------------- +// Hardware data +// --------------------------------------------------------------------------- + +/// Slots in the hardware-data register map. +pub(crate) const REGISTER_MAP_SLOTS: usize = 53; + +/// One register window the firmware maps through the hardware data +/// (per-SoC configuration). +#[derive(Copy, Clone, Debug)] +pub(crate) struct RegisterWindow { + /// Register-map slot. + pub(crate) slot: usize, + /// Physical address of the window. + pub(crate) phys: u64, + /// GPU address of the window in the firmware MMIO aperture. + pub(crate) va: u64, + /// Size in bytes; need not be a multiple of the page size. + pub(crate) size: u32, + /// Value of [`RegisterMapEntry::unk_18`]. + pub(crate) unk_18: u64, + /// Value of [`RegisterMapEntry::unk_20`]. + pub(crate) unk_20: u32, +} + +impl RegisterWindow { + /// Returns the page-aligned physical address, GPU address and length the + /// window must be mapped with, or `None` if its physical and GPU + /// addresses differ within a page or the mapped span overflows. + pub(crate) fn mapping(&self) -> Option<(u64, u64, u64)> { + let mask = PAGE_SIZE as u64 - 1; + if self.phys & mask != self.va & mask { + return None; + } + let start = self.va & !mask; + let span = self + .va + .checked_add(self.size as u64)? + .checked_sub(start)? + .checked_add(mask)? + & !mask; + Some((self.phys & !mask, start, span)) + } +} + +/// Value of [`RegisterMapEntry::unk_20`] in a reserved register-map slot. +const RESERVED_SLOT_UNK_20: u32 = 2; + +/// One register-map slot of the hardware data. +#[repr(C)] +pub(crate) struct RegisterMapEntry { + /// Physical address of the window. + pub(crate) phys: u64, + /// GPU address of the window. + pub(crate) va: u64, + /// Window size in bytes. + pub(crate) total_size: u32, + /// Window size in bytes, again. + pub(crate) element_size: u32, + /// Per-window value, see [`RegisterWindow::unk_18`]. + pub(crate) unk_18: u64, + /// Per-window value; [`RESERVED_SLOT_UNK_20`] in a reserved slot. + pub(crate) unk_20: u32, + pub(crate) unk_24: u32, +} + +// SAFETY: `RegisterMapEntry` consists of integers and arrays of integers only. +unsafe impl Zeroable for RegisterMapEntry {} + +static_assert!(size_of::() == 0x28); + +impl RegisterMapEntry { + const fn window(window: &RegisterWindow) -> Self { + Self { + phys: window.phys, + va: window.va, + total_size: window.size, + element_size: window.size, + unk_18: window.unk_18, + unk_20: window.unk_20, + unk_24: 0, + } + } + + const fn reserved() -> Self { + Self { + phys: 0, + va: 0, + total_size: 0, + element_size: 0, + unk_18: 0, + unk_20: RESERVED_SLOT_UNK_20, + unk_24: 0, + } + } +} + +/// Performance states in each performance table. +pub(crate) const PERF_STATES: usize = 11; +/// Voltage lanes of a performance table. +const PERF_LANES: usize = 16; + +/// One GPU operating point, from the device tree, in performance-state +/// order. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Opp { + /// Frequency in Hz. + pub(crate) freq_hz: u64, + /// GPU core voltage in µV. + pub(crate) volt_uv: u32, +} + +/// Per-SoC parts of the performance tables that the operating points do not +/// give. +pub(crate) struct PerfConfig { + /// Operating point of each state of table A. + pub(crate) states_a: [u8; PERF_STATES], + /// Operating point of each state of table B. + pub(crate) states_b: [u8; PERF_STATES], + /// Relative performance of each state of table A, in percent. + pub(crate) relative_a: [u32; PERF_STATES], + /// Relative performance of each state of table B, in percent. + pub(crate) relative_b: [u32; PERF_STATES], + /// Lowest SRAM voltage in mV; a state's SRAM voltage is its core voltage + /// raised to this floor. + pub(crate) sram_min_mv: u32, +} + +/// One performance table of the hardware data. +#[repr(C, packed)] +pub(crate) struct PerfGroup { + /// Required: 10. + pub(crate) unk_0: u32, + /// Frequency of each state in MHz. + pub(crate) freq_mhz: [u32; PERF_STATES], + pub(crate) unk_30: [u8; 0x14], + /// GPU core voltage of each state in mV, per lane. + pub(crate) core_mv: [[u32; PERF_LANES]; PERF_STATES], + pub(crate) unk_304: [u8; 0x140], + /// SRAM voltage of each state in mV, per lane. + pub(crate) sram_mv: [[u32; PERF_LANES]; PERF_STATES], +} + +// SAFETY: `PerfGroup` consists of integers and arrays of integers only. +unsafe impl Zeroable for PerfGroup {} + +static_assert!(core::mem::offset_of!(PerfGroup, core_mv) == 0x44); +static_assert!(core::mem::offset_of!(PerfGroup, sram_mv) == 0x444); +static_assert!(size_of::() == 0x704); + +/// Frequency and voltages of one performance state. +#[derive(Copy, Clone)] +struct PerfState { + freq_mhz: u32, + core_mv: u32, + sram_mv: u32, +} + +impl PerfGroup { + /// Fills the table. `lanes` is the number of voltage lanes holding each + /// state's voltages; the remaining lanes stay zero. + fn fill(&mut self, states: &[PerfState; PERF_STATES], lanes: usize) { + self.unk_0 = 10; + for (index, state) in states.iter().enumerate() { + self.freq_mhz[index] = state.freq_mhz; + for lane in 0..lanes { + self.core_mv[index][lane] = state.core_mv; + self.sram_mv[index][lane] = state.sram_mv; + } + } + } +} + +/// Returns the performance states selected by `map` from `opps`. +fn perf_states( + opps: &[Opp], + map: &[u8; PERF_STATES], + sram_min_mv: u32, +) -> Result<[PerfState; PERF_STATES]> { + let mut states = [PerfState { + freq_mhz: 0, + core_mv: 0, + sram_mv: 0, + }; PERF_STATES]; + for (state, &index) in states.iter_mut().zip(map) { + let opp = opps.get(index as usize).ok_or(EINVAL)?; + let core_mv = opp.volt_uv / 1000; + *state = PerfState { + freq_mhz: u32::try_from(opp.freq_hz / 1_000_000)?, + core_mv, + sram_mv: core_mv.max(sram_min_mv), + }; + } + Ok(states) +} + +/// One completion ring the firmware fills. +#[repr(C)] +pub(crate) struct CompletionRingDescriptor { + /// Context-0 alias of the ring. + pub(crate) low_va: u64, + /// Firmware alias of the ring. + pub(crate) high_va: u64, + /// Entries in the ring. + pub(crate) entries: u32, + /// Size of one entry. + pub(crate) entry_size: u32, + /// Index of this descriptor in the table. + pub(crate) index: u32, + /// Firmware-owned. + pub(crate) unk_1c: u32, +} + +// SAFETY: `CompletionRingDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for CompletionRingDescriptor {} + +static_assert!(size_of::() == 0x20); + +/// Completion-ring descriptors in the hardware data; only +/// [`COMPLETION_RING_INDEX`] are used. +const COMPLETION_RING_DESCRIPTORS: usize = 4; +/// Descriptor index of each completion ring. +pub(crate) const COMPLETION_RING_INDEX: [u32; 2] = [0, 2]; +/// Entries in a completion ring. +const COMPLETION_RING_ENTRIES: u32 = 0x800; +/// Size of a completion-ring entry. +const COMPLETION_RING_ENTRY_SIZE: u32 = 0x40; +/// Size of a completion ring. +pub(crate) const COMPLETION_RING_SIZE: usize = + (COMPLETION_RING_ENTRIES * COMPLETION_RING_ENTRY_SIZE) as usize; + +/// The two aliases of one completion ring. +#[derive(Copy, Clone, Debug)] +pub(crate) struct CompletionRing { + /// Context-0 alias. + pub(crate) low_va: u64, + /// Firmware alias. + pub(crate) high_va: u64, +} + +/// Inputs of the hardware data. +pub(crate) struct HwDataArgs<'a> { + /// Chip identifier. + pub(crate) chip_id: u32, + /// Mapped register windows. + pub(crate) windows: &'a [RegisterWindow], + /// Register-map slots that are reserved but unmapped. + pub(crate) reserved_slots: &'a [usize], + /// GPU operating points, in performance-state order. + pub(crate) opps: &'a [Opp], + /// Per-SoC parts of the performance tables. + pub(crate) perf: &'a PerfConfig, + /// The completion rings, in [`COMPLETION_RING_INDEX`] order. + pub(crate) completion_rings: [CompletionRing; 2], + /// The zeroed QoS object ([`QOS_SIZE`] bytes). + pub(crate) qos_va: u64, +} + +/// Size of the QoS object named by the hardware data. +pub(crate) const QOS_SIZE: usize = 0xe40; + +/// The hardware data, at the base of the bundle and shared by both +/// instances. +#[repr(C, packed)] +pub(crate) struct HwData { + /// Required: 0x6f_0000_0000. + pub(crate) unk_0: u64, + /// Required: 0xffc0_0000. + pub(crate) unk_8: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_10: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_18: u64, + /// Required: 0x2ff_ffff_8000. + pub(crate) unk_20: u64, + /// Required: 0xffff_fc21_8140_0000, a GPU address in the firmware MMIO + /// aperture. + pub(crate) unk_28: u64, + pub(crate) unk_30: [u8; 0xb0], + /// Required: see [`HwData::UNK_E0`]. + pub(crate) unk_e0: [u16; 24], + pub(crate) unk_110: [u8; 0x1c8], + /// Required: see [`HwData::UNK_2D8`]. + pub(crate) unk_2d8: [u16; 12], + pub(crate) unk_2f0: [u8; 0xf0], + /// Required: see [`HwData::UNK_3E0`]. + pub(crate) unk_3e0: [u16; 24], + pub(crate) unk_410: [u8; 0x1c8], + /// Required: see [`HwData::UNK_5D8`]. + pub(crate) unk_5d8: [u16; 12], + pub(crate) unk_5f0: [u8; 0x50], + /// Register windows the firmware maps. + pub(crate) register_map: [RegisterMapEntry; REGISTER_MAP_SLOTS], + pub(crate) unk_e88: [u8; 8], + /// Chip identifier. + pub(crate) chip_id: u32, + /// Required: 1. + pub(crate) unk_e94: u32, + /// Required: 1. + pub(crate) unk_e98: u32, + pub(crate) unk_e9c: [u8; 4], + /// Required: 1. + pub(crate) unk_ea0: u32, + /// Required: 1. + pub(crate) unk_ea4: u32, + pub(crate) unk_ea8: [u8; 0x10], + /// Required: 1. + pub(crate) unk_eb8: u32, + pub(crate) unk_ebc: [u8; 4], + /// Required: 1. + pub(crate) unk_ec0: u32, + pub(crate) unk_ec4: [u8; 4], + /// Required: 1. + pub(crate) unk_ec8: u32, + pub(crate) unk_ecc: [u8; 4], + /// Required: 0x5dc0 (24000). + pub(crate) unk_ed0: u32, + /// Required: 1. + pub(crate) unk_ed4: u32, + /// Required: 0x10 (16). + pub(crate) unk_ed8: u32, + pub(crate) unk_edc: [u8; 4], + /// Required: 1 in every entry. + pub(crate) unk_ee0: [u32; 3], + pub(crate) unk_eec: [u8; 0x18], + /// Required: 0x1f (31). + pub(crate) unk_f04: u32, + pub(crate) unk_f08: [u8; 0x1c], + /// Required: 4. + pub(crate) unk_f24: u32, + pub(crate) unk_f28: [u8; 0xc], + /// Required: 1. + pub(crate) unk_f34: u32, + /// Required: 1. + pub(crate) unk_f38: u32, + pub(crate) unk_f3c: [u8; 0x10], + /// Required: 0x31 (49). + pub(crate) unk_f4c: u32, + pub(crate) unk_f50: [u8; 0x1c], + /// Required: 1. + pub(crate) unk_f6c: u32, + pub(crate) unk_f70: [u8; 0x18], + /// Required: 6. + pub(crate) unk_f88: u32, + /// Required: 1. + pub(crate) unk_f8c: u32, + pub(crate) unk_f90: [u8; 0x1c], + /// Required: 1. + pub(crate) unk_fac: u32, + pub(crate) unk_fb0: [u8; 8], + /// Required: 0x1e (30). + pub(crate) unk_fb8: u32, + /// Required: 4. + pub(crate) unk_fbc: u32, + /// Required: 6. + pub(crate) unk_fc0: u32, + /// Performance table A. + pub(crate) perf_a: PerfGroup, + pub(crate) unk_16c8: [u8; 0x140], + /// Frequency of each state of table B in MHz. + pub(crate) perf_b_freq_mhz: [u32; PERF_STATES], + pub(crate) unk_1834: [u8; 0x14], + /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. + pub(crate) unk_1848: [u32; 11], + pub(crate) unk_1874: [u8; 0x54], + /// Relative performance of each state of table A, in percent. + pub(crate) perf_a_relative: [u32; PERF_STATES], + pub(crate) unk_18f4: [u8; 0x14], + /// Relative performance of each state of table B, in percent. + pub(crate) perf_b_relative: [u32; PERF_STATES], + pub(crate) unk_1934: [u8; 0x94], + /// Operating point of each state of table A. + pub(crate) perf_a_states: [u32; PERF_STATES], + pub(crate) unk_19f4: [u8; 0x14], + /// Operating point of each state of table B. + pub(crate) perf_b_states: [u32; PERF_STATES], + pub(crate) unk_1a34: [u8; 0x2a4], + /// Performance table B. Each voltage is stored in lane 0 only. + pub(crate) perf_b: PerfGroup, + pub(crate) unk_23dc: [u8; 0x168], + /// Required: 4. + pub(crate) unk_2544: u32, + /// Required: 6. + pub(crate) unk_2548: u32, + /// Required: 3. + pub(crate) unk_254c: u32, + /// Required: 7. + pub(crate) unk_2550: u32, + /// Required: 7. + pub(crate) unk_2554: u32, + pub(crate) unk_2558: [u8; 0x10], + /// Required: 5. + pub(crate) unk_2568: u32, + pub(crate) unk_256c: [u8; 4], + /// Required: 1. + pub(crate) unk_2570: u32, + pub(crate) unk_2574: [u8; 0x1a], + /// Required: [`WORK_CHANNELS`]. + pub(crate) unk_258e: u16, + /// Required: 0, 1, 1, 1. + pub(crate) unk_2590: [u32; 4], + /// Required: 0, 1, 1, 1. + pub(crate) unk_25a0: [u32; 4], + pub(crate) unk_25b0: [u8; 4], + /// Required: 0xffff_ffff in every entry, one per work channel. + pub(crate) unk_25b4: [u32; WORK_CHANNELS], + pub(crate) unk_25e4: [u8; 0x10], + /// Required: 1. + pub(crate) unk_25f4: u32, + pub(crate) unk_25f8: [u8; 8], + /// Required: 1. + pub(crate) unk_2600: u32, + pub(crate) unk_2604: [u8; 4], + /// Required: 1. + pub(crate) unk_2608: u32, + pub(crate) unk_260c: [u8; 4], + /// Required: 0x100. + pub(crate) unk_2610: u32, + /// Completion-ring descriptors. + pub(crate) completion_rings: [CompletionRingDescriptor; COMPLETION_RING_DESCRIPTORS], + pub(crate) unk_2694: [u8; 0x18], + /// QoS object ([`QOS_SIZE`] bytes, zeroed). + pub(crate) qos_va: u64, + /// Required: 1. + pub(crate) unk_26b4: u32, + /// Required: 1. + pub(crate) unk_26b8: u32, + pub(crate) unk_26bc: [u8; 0x2c], + /// Required: 1. + pub(crate) unk_26e8: u32, + pub(crate) unk_26ec: [u8; 0x16ac], + /// Required: 0xffff_ffff. + pub(crate) unk_3d98: u32, + pub(crate) unk_3d9c: [u8; 0x14], + /// Required: 0xffff_ffff. + pub(crate) unk_3db0: u32, +} + +static_assert!(core::mem::offset_of!(HwData, register_map) == 0x640); +static_assert!(core::mem::offset_of!(HwData, chip_id) == 0xe90); +static_assert!(core::mem::offset_of!(HwData, perf_a) == 0xfc4); +static_assert!(core::mem::offset_of!(HwData, perf_b_freq_mhz) == 0x1808); +static_assert!(core::mem::offset_of!(HwData, perf_b) == 0x1cd8); +static_assert!(core::mem::offset_of!(HwData, unk_258e) == 0x258e); +static_assert!(core::mem::offset_of!(HwData, completion_rings) == 0x2614); +static_assert!(core::mem::offset_of!(HwData, qos_va) == 0x26ac); +static_assert!(size_of::() == 0x3db4); +static_assert!(bundle::HW_DATA + size_of::() <= bundle::VIEWS[2]); + +// SAFETY: `HwData` consists of integers and arrays of integers only. +unsafe impl Zeroable for HwData {} + +impl HwData { + const UNK_E0: [u16; 24] = [ + 0x2008, 0, 0, 0, 0, 0x2008, 0, 0, 0, 0, 0x2008, 0, 0x24cb, 0, 0x2cfa, 0xc98a, 0x24cb, + 0xf4f6, 0xe917, 0x1877, 0x24cb, 0x38d9, 0, 0xbdab, + ]; + const UNK_2D8: [u16; 12] = [ + 0x2000, 0xfff8, 0x2cdb, 0xd32d, 0x2000, 0xf500, 0xe926, 0x21da, 0x2000, 0x38b6, 8, 0xc742, + ]; + const UNK_3E0: [u16; 24] = [ + 0x7fe0, 0, 0, 0, 0, 0x7fe0, 0, 0, 0, 0, 0x7fe0, 0, 0x8000, 0, 0, 0, 0, 0x8000, 0, 0, 0, 0, + 0x8000, 0, + ]; + const UNK_5D8: [u16; 12] = [ + 0x2645, 0x4b23, 0xe98, 0, 0xea5f, 0xd5a2, 0x3fff, 0x4000, 0x4000, 0xca5e, 0xf5a2, 0x4000, + ]; + + /// Initializes the hardware data. + pub(crate) fn init(&mut self, args: &HwDataArgs<'_>) -> Result { + clear(self); + self.unk_0 = 0x6f_0000_0000; + self.unk_8 = 0xffc0_0000; + self.unk_10 = 0x10_0000_0000; + self.unk_18 = 0x10_0000_0000; + self.unk_20 = 0x2ff_ffff_8000; + self.unk_28 = 0xffff_fc21_8140_0000; + self.unk_e0 = Self::UNK_E0; + self.unk_2d8 = Self::UNK_2D8; + self.unk_3e0 = Self::UNK_3E0; + self.unk_5d8 = Self::UNK_5D8; + + let mut used = [false; REGISTER_MAP_SLOTS]; + let mut claim = |slot: usize| -> Result { + match used.get_mut(slot) { + Some(used) if !*used => { + *used = true; + Ok(()) + } + _ => Err(EINVAL), + } + }; + for window in args.windows { + claim(window.slot)?; + self.register_map[window.slot] = RegisterMapEntry::window(window); + } + for &slot in args.reserved_slots { + claim(slot)?; + self.register_map[slot] = RegisterMapEntry::reserved(); + } + + self.chip_id = args.chip_id; + self.unk_e94 = 1; + self.unk_e98 = 1; + self.unk_ea0 = 1; + self.unk_ea4 = 1; + self.unk_eb8 = 1; + self.unk_ec0 = 1; + self.unk_ec8 = 1; + self.unk_ed0 = 0x5dc0; + self.unk_ed4 = 1; + self.unk_ed8 = 0x10; + self.unk_ee0 = [1; 3]; + self.unk_f04 = 0x1f; + self.unk_f24 = 4; + self.unk_f34 = 1; + self.unk_f38 = 1; + self.unk_f4c = 0x31; + self.unk_f6c = 1; + self.unk_f88 = 6; + self.unk_f8c = 1; + self.unk_fac = 1; + self.unk_fb8 = 0x1e; + self.unk_fbc = 4; + self.unk_fc0 = 6; + + let perf = args.perf; + let states_a = perf_states(args.opps, &perf.states_a, perf.sram_min_mv)?; + let states_b = perf_states(args.opps, &perf.states_b, perf.sram_min_mv)?; + self.perf_a.fill(&states_a, PERF_LANES); + self.perf_b.fill(&states_b, 1); + for (index, state) in states_b.iter().enumerate() { + self.perf_b_freq_mhz[index] = state.freq_mhz; + self.perf_a_states[index] = perf.states_a[index].into(); + self.perf_b_states[index] = perf.states_b[index].into(); + } + self.perf_a_relative = perf.relative_a; + self.perf_b_relative = perf.relative_b; + self.unk_1848 = [0x3f82_8f5c; 11]; + self.unk_2544 = 4; + self.unk_2548 = 6; + self.unk_254c = 3; + self.unk_2550 = 7; + self.unk_2554 = 7; + self.unk_2568 = 5; + self.unk_2570 = 1; + + self.unk_258e = WORK_CHANNELS as u16; + self.unk_2590 = [0, 1, 1, 1]; + self.unk_25a0 = [0, 1, 1, 1]; + self.unk_25b4 = [u32::MAX; WORK_CHANNELS]; + self.unk_25f4 = 1; + self.unk_2600 = 1; + self.unk_2608 = 1; + self.unk_2610 = 0x100; + for (ring, index) in args.completion_rings.iter().zip(COMPLETION_RING_INDEX) { + self.completion_rings[index as usize] = CompletionRingDescriptor { + low_va: ring.low_va, + high_va: ring.high_va, + entries: COMPLETION_RING_ENTRIES, + entry_size: COMPLETION_RING_ENTRY_SIZE, + index, + unk_1c: 0, + }; + } + self.qos_va = args.qos_va; + self.unk_26b4 = 1; + self.unk_26b8 = 1; + self.unk_26e8 = 1; + self.unk_3d98 = 0xffff_ffff; + self.unk_3db0 = 0xffff_ffff; + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// Regions A and C +// --------------------------------------------------------------------------- + +/// Region A: a zeroed page both instances read. +#[repr(C)] +pub(crate) struct RegionA { + pub(crate) unk_0: [u8; 0x4000], +} + +// SAFETY: `RegionA` is an array of integers. +unsafe impl Zeroable for RegionA {} + +/// Engines whose work the firmware's context-switch timer may interrupt +/// (bit 0 TA, bit 1 3D, bit 2 compute). No engine is periodically resumed. +pub(crate) const CONTEXT_SWITCH_TIMER_MASK: u32 = 0; +/// Period of the context-switch timer, in units of the firmware's 1 ms +/// scheduling pass. +pub(crate) const CONTEXT_SWITCH_TIMER_MULTIPLIER: u8 = 1; + +/// Region C: firmware policy settings shared by both instances. +/// +/// Three fields are live. The host updates [`Self::effort`] while it submits +/// work: it stores zero to [`Self::unk_e8`], then the effort (0x10000, 1.0 +/// in 16.16 fixed point, while work is streaming; zero once the GPU has been +/// idle for 16 ms), each with one aligned 32-bit store, and then cleans the +/// cache line to the point of coherency. [`Self::power_assert_tally`] is +/// unaligned: for every device-control record whose opcode asserts GPU +/// power, the host adds one to it after writing the record and before +/// advancing the ring producer (with a barrier before the producer store). +/// The firmware powers the GPU cores up while the tally differs from its +/// own count of drained records. +#[repr(C, packed)] +pub(crate) struct RegionC { + /// Engines the context-switch timer applies to ([`CONTEXT_SWITCH_TIMER_MASK`]). + pub(crate) context_switch_timer_mask: u32, + pub(crate) unk_4: [u8; 0x20], + /// Required: 0xbb8 (3000). + pub(crate) unk_24: u32, + pub(crate) unk_28: [u8; 8], + /// Non-zero lets the firmware power the GPU cores off when idle. + pub(crate) idle_power_off: u32, + /// Required: 1. + pub(crate) unk_34: u32, + pub(crate) unk_38: [u8; 4], + /// Required: 0x78 (120). + pub(crate) unk_3c: u32, + pub(crate) unk_40: [u8; 0x14], + /// Required: 0xffff. + pub(crate) unk_54: u16, + /// Required: 0x28 (40). + pub(crate) unk_56: u16, + /// Required: 0xffff. + pub(crate) unk_58: u16, + pub(crate) unk_5a: [u8; 4], + /// Required: 1. + pub(crate) unk_5e: u32, + /// Required: 1. + pub(crate) unk_62: u32, + /// Required: 3. + pub(crate) unk_66: u32, + /// Period of the context-switch timer ([`CONTEXT_SWITCH_TIMER_MULTIPLIER`]). + pub(crate) context_switch_timer_multiplier: u8, + pub(crate) unk_6b: [u8; 0xd], + /// Required: 1 in every entry. + pub(crate) unk_78: [u32; 3], + /// Required: 0x254e (9550). + pub(crate) unk_84: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_88: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_8c: u32, + pub(crate) unk_90: [u8; 8], + /// Required: 0x3e8 (1000). + pub(crate) unk_98: u32, + /// Required: 0x3000_001d. + pub(crate) unk_9c: u32, + /// Required: 0x3100_0002. + pub(crate) unk_a0: u32, + pub(crate) unk_a4: [u8; 0x40], + /// Performance-floor request (live, see the type documentation). + pub(crate) effort: u32, + /// Live; the host stores zero here whenever it stores [`Self::effort`]. + pub(crate) unk_e8: u32, + pub(crate) unk_ec: [u8; 0x6d8], + /// Required: 0x7d0 (2000). + pub(crate) unk_7c4: u32, + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_7c8: u32, + /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). + pub(crate) unk_7cc: u32, + /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). + pub(crate) unk_7d0: u32, + /// Required: 0x3f66_6666 (0.9 as an IEEE-754 single). + pub(crate) unk_7d4: u32, + /// Required: 0x3dcc_cccd (0.1 as an IEEE-754 single). + pub(crate) unk_7d8: u32, + /// Required: 0x3e80_0000 (0.25 as an IEEE-754 single). + pub(crate) unk_7dc: u32, + /// Required: 0x3f19_999a (0.6 as an IEEE-754 single). + pub(crate) unk_7e0: u32, + /// Required: 0x3f66_6666 (0.9 as an IEEE-754 single). + pub(crate) unk_7e4: u32, + /// Required: 6. + pub(crate) unk_7e8: u32, + /// Required: 1. + pub(crate) unk_7ec: u32, + /// Required: 1. + pub(crate) unk_7f0: u32, + /// Required: 0x64 (100). + pub(crate) unk_7f4: u32, + /// Required: 0x64 (100). + pub(crate) unk_7f8: u32, + pub(crate) unk_7fc: [u8; 0x19c], + /// Required: 0x28 (40). + pub(crate) unk_998: u32, + /// Required: 0xa (10). + pub(crate) unk_99c: u32, + /// Required: 0xfa (250). + pub(crate) unk_9a0: u32, + /// Required: 1. + pub(crate) unk_9a4: u32, + pub(crate) unk_9a8: [u8; 0x10], + /// Required: 2. + pub(crate) unk_9b8: u32, + /// Idle time before the firmware powers the GPU cores off. + pub(crate) idle_power_off_delay: u32, + /// Required: 5. + pub(crate) unk_9c0: u32, + pub(crate) unk_9c4: [u8; 4], + /// Required: 0x28 (40). + pub(crate) unk_9c8: u32, + /// Compute progress watchdog threshold (zero disables the watchdog). + pub(crate) compute_watchdog: u32, + /// Required: 1. + pub(crate) unk_9d0: u32, + pub(crate) unk_9d4: [u8; 0x449], + /// Required: 1. + pub(crate) unk_e1d: u32, + pub(crate) unk_e21: [u8; 0x18], + /// Required: 3. + pub(crate) unk_e39: u32, + /// Required: 1. + pub(crate) unk_e3d: u32, + pub(crate) unk_e41: [u8; 0x10], + /// Host power-assert tally (live, unaligned; see the type documentation). + pub(crate) power_assert_tally: u32, + pub(crate) unk_e55: [u8; 0x1ab], +} + +static_assert!(core::mem::offset_of!(RegionC, idle_power_off) == 0x30); +static_assert!(core::mem::offset_of!(RegionC, context_switch_timer_multiplier) == 0x6a); +static_assert!(core::mem::offset_of!(RegionC, effort) == 0xe4); +static_assert!(core::mem::offset_of!(RegionC, idle_power_off_delay) == 0x9bc); +static_assert!(core::mem::offset_of!(RegionC, compute_watchdog) == 0x9cc); +static_assert!(core::mem::offset_of!(RegionC, power_assert_tally) == 0xe51); +static_assert!(size_of::() == 0x1000); + +// SAFETY: `RegionC` consists of integers and arrays of integers only. +unsafe impl Zeroable for RegionC {} + +impl RegionC { + /// Offset of [`Self::effort`]. + pub(crate) const EFFORT: usize = core::mem::offset_of!(Self, effort); + /// Offset of [`Self::unk_e8`]. + pub(crate) const UNK_E8: usize = core::mem::offset_of!(Self, unk_e8); + /// Offset of [`Self::power_assert_tally`]. + pub(crate) const POWER_ASSERT_TALLY: usize = core::mem::offset_of!(Self, power_assert_tally); + + /// Initializes region C. + pub(crate) fn init(&mut self) { + clear(self); + self.context_switch_timer_mask = CONTEXT_SWITCH_TIMER_MASK; + self.unk_24 = 0xbb8; + self.idle_power_off = 1; + self.unk_34 = 1; + self.unk_3c = 0x78; + self.unk_54 = 0xffff; + self.unk_56 = 0x28; + self.unk_58 = 0xffff; + self.unk_5e = 1; + self.unk_62 = 1; + self.unk_66 = 3; + self.context_switch_timer_multiplier = CONTEXT_SWITCH_TIMER_MULTIPLIER; + self.unk_78 = [1; 3]; + self.unk_84 = 0x254e; + self.unk_88 = 0x3e8; + self.unk_8c = 0x3e8; + self.unk_98 = 0x3e8; + self.unk_9c = 0x3000_001d; + self.unk_a0 = 0x3100_0002; + self.unk_7c4 = 0x7d0; + self.unk_7c8 = 0x3f80_0000; + self.unk_7cc = 0x3f4c_cccd; + self.unk_7d0 = 0x3e4c_cccd; + self.unk_7d4 = 0x3f66_6666; + self.unk_7d8 = 0x3dcc_cccd; + self.unk_7dc = 0x3e80_0000; + self.unk_7e0 = 0x3f19_999a; + self.unk_7e4 = 0x3f66_6666; + self.unk_7e8 = 6; + self.unk_7ec = 1; + self.unk_7f0 = 1; + self.unk_7f4 = 0x64; + self.unk_7f8 = 0x64; + self.unk_998 = 0x28; + self.unk_99c = 0xa; + self.unk_9a0 = 0xfa; + self.unk_9a4 = 1; + self.unk_9b8 = 2; + self.idle_power_off_delay = 40; + self.unk_9c0 = 5; + self.unk_9c8 = 0x28; + self.compute_watchdog = 50; + self.unk_9d0 = 1; + self.unk_e1d = 1; + self.unk_e39 = 3; + self.unk_e3d = 1; + } +} + +// --------------------------------------------------------------------------- +// Status objects +// --------------------------------------------------------------------------- + +/// Status block of an instance (its status A, and the head of the primary +/// status B). +/// +/// At creation only `+0x04` is one. Firmware later owns the status words at +/// `+0x10` and `+0x14`; the host observes each with an aligned 32-bit read. +/// The event-ring state begins at status A `+0x40`, inside this boot view: +/// clearing or copying the whole status block after publication is forbidden. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct StatusBlock { + pub(crate) unk_0: u32, + /// Required: 1. + pub(crate) unk_4: u32, + pub(crate) unk_8: [u8; 0x78], +} + +// SAFETY: `StatusBlock` consists of integers and arrays of integers only. +unsafe impl Zeroable for StatusBlock {} + +static_assert!(size_of::() == 0x80); + +impl StatusBlock { + /// Returns a status block in its boot state. + pub(crate) const fn new() -> Self { + Self { + unk_0: 0, + unk_4: 1, + unk_8: [0; 0x78], + } + } +} + +/// Power state the primary status B carries for the secondary instance, +/// named by [`SecondaryRoot::power_state_va`]. +#[repr(C, packed)] +pub(crate) struct PowerState { + pub(crate) unk_0: [u8; 0x10], + /// Required: 0xf (15). + pub(crate) unk_10: u32, + /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). + pub(crate) unk_14: u32, + /// Required: 0x4088_0000 (4.25 as an IEEE-754 single). + pub(crate) unk_18: u32, + /// Required: 0x28 (40). + pub(crate) unk_1c: u32, + /// Required: 1. + pub(crate) unk_20: u32, + pub(crate) unk_24: [u8; 0xa0], + /// Non-zero asks the secondary to reload the limiter settings that + /// follow into the power configuration; zero at boot. + pub(crate) temp_update: u32, + /// Target temperature of the limiter ([`TEMP_TARGET`]). + pub(crate) temp_target: u32, + /// Release temperature of the limiter ([`TEMP_RELEASE`]). + pub(crate) temp_release: u32, + pub(crate) unk_d0: [u8; 8], + /// Proportional gain of the limiter ([`TEMP_KP`]). + pub(crate) temp_kp: u32, + /// Integral gain of the limiter ([`TEMP_KI`]). + pub(crate) temp_ki: u32, + pub(crate) unk_e0: [u8; 0x10], + /// Required: 0x26ac (9900). + pub(crate) unk_f0: u32, + /// Required: 0xc8 (200). + pub(crate) unk_f4: u32, + pub(crate) unk_f8: [u8; 0x10], + /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). + pub(crate) unk_108: u32, + /// Required: 0x43c8_0000 (400.0 as an IEEE-754 single). + pub(crate) unk_10c: u32, + /// Required: 0xc8 (200). + pub(crate) unk_110: u32, + pub(crate) unk_114: [u8; 0x14], + /// Required: 1. + pub(crate) unk_128: u32, + /// Required: 0x2616 (9750). + pub(crate) unk_12c: u32, + /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). + pub(crate) unk_130: u32, + /// Required: 0x40cc_cccd (6.4 as an IEEE-754 single). + pub(crate) unk_134: u32, + pub(crate) unk_138: [u8; 0x14], + /// Required: 1. + pub(crate) unk_14c: u32, + pub(crate) unk_150: [u8; 0x70], + /// Required: 1. + pub(crate) unk_1c0: u32, + /// Required: 1. + pub(crate) unk_1c4: u32, + /// Required: 4. + pub(crate) unk_1c8: u32, + /// Required: 1 in every entry. + pub(crate) unk_1cc: [u32; 4], + pub(crate) unk_1dc: [u8; 0x308], + /// Required: 1. + pub(crate) unk_4e4: u32, + /// Required: 0x1f4 (500). + pub(crate) unk_4e8: u32, + /// Required: 6. + pub(crate) unk_4ec: u32, + /// Required: 0x30d4 (12500) in every entry. + pub(crate) unk_4f0: [u32; 6], + pub(crate) unk_508: [u8; 0x14], + /// Required: 0x30d4 (12500) in every entry. + pub(crate) unk_51c: [u32; 5], + /// Required: 1. + pub(crate) unk_530: u32, + pub(crate) unk_534: [u8; 0x58], + /// Required: 6. + pub(crate) unk_58c: u32, + /// Required: 0xdac (3500). + pub(crate) unk_590: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_594: u32, + /// Required: 0xbb8 (3000). + pub(crate) unk_598: u32, + /// Required: 0x64 (100). + pub(crate) unk_59c: u32, + pub(crate) unk_5a0: [u8; 0x2c], + /// Required: 1. + pub(crate) unk_5cc: u32, + /// Required: 4. + pub(crate) unk_5d0: u32, + pub(crate) unk_5d4: [u8; 8], + /// Required: 0x1f40 (8000). + pub(crate) unk_5dc: u32, + /// Required: 0xc8 (200). + pub(crate) unk_5e0: u32, + /// Required: 0xfa0 (4000). + pub(crate) unk_5e4: u32, + /// Required: 0xc8 (200). + pub(crate) unk_5e8: u32, + /// Required: 0x7d0 (2000). + pub(crate) unk_5ec: u32, + /// Required: 0xc8 (200). + pub(crate) unk_5f0: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_5f4: u32, + /// Required: 0xc8 (200). + pub(crate) unk_5f8: u32, + pub(crate) unk_5fc: [u8; 8], + /// Required: 0x4170_0000 (15.0 as an IEEE-754 single). + pub(crate) unk_604: u32, + /// Required: 0x40a0_0000 (5.0 as an IEEE-754 single). + pub(crate) unk_608: u32, + /// Required: 0x20 (32). + pub(crate) unk_60c: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_610: u32, + pub(crate) unk_614: [u8; 0xc], + /// Required: 0x384 (900). + pub(crate) unk_620: u32, + pub(crate) unk_624: [u8; 0x9c], +} + +// SAFETY: `PowerState` consists of integers and arrays of integers only. +unsafe impl Zeroable for PowerState {} + +static_assert!(size_of::() == 0x6c0); + +impl PowerState { + fn init(&mut self) { + self.unk_10 = 0xf; + self.unk_14 = 0x3f00_0000; + self.unk_18 = 0x4088_0000; + self.unk_1c = 0x28; + self.unk_20 = 1; + self.temp_target = TEMP_TARGET; + self.temp_release = TEMP_RELEASE; + self.temp_kp = TEMP_KP; + self.temp_ki = TEMP_KI; + self.unk_f0 = 0x26ac; + self.unk_f4 = 0xc8; + self.unk_108 = 0x42c8_0000; + self.unk_10c = 0x43c8_0000; + self.unk_110 = 0xc8; + self.unk_128 = 1; + self.unk_12c = 0x2616; + self.unk_130 = 0x3f00_0000; + self.unk_134 = 0x40cc_cccd; + self.unk_14c = 1; + self.unk_1c0 = 1; + self.unk_1c4 = 1; + self.unk_1c8 = 4; + self.unk_1cc = [1; 4]; + self.unk_4e4 = 1; + self.unk_4e8 = 0x1f4; + self.unk_4ec = 6; + self.unk_4f0 = [0x30d4; 6]; + self.unk_51c = [0x30d4; 5]; + self.unk_530 = 1; + self.unk_58c = 6; + self.unk_590 = 0xdac; + self.unk_594 = 0x3e8; + self.unk_598 = 0xbb8; + self.unk_59c = 0x64; + self.unk_5cc = 1; + self.unk_5d0 = 4; + self.unk_5dc = 0x1f40; + self.unk_5e0 = 0xc8; + self.unk_5e4 = 0xfa0; + self.unk_5e8 = 0xc8; + self.unk_5ec = 0x7d0; + self.unk_5f0 = 0xc8; + self.unk_5f4 = 0x3e8; + self.unk_5f8 = 0xc8; + self.unk_604 = 0x4170_0000; + self.unk_608 = 0x40a0_0000; + self.unk_60c = 0x20; + self.unk_610 = 0x3e8; + self.unk_620 = 0x384; + } +} + +/// Size of the firmware-control page named by the primary status B. +pub(crate) const FWCTL_SIZE: usize = 0x4000; +/// Offset of the firmware-control ring in its page. +const FWCTL_RING: u64 = 0x40; + +/// One firmware-written recovery-info entry. The host reads each word separately after the +/// recovery handshake reaches state 3; it never copies the live table as a structure. +#[repr(C)] +pub(crate) struct RecoveryInfoEntry { + pub(crate) flags: u32, + pub(crate) stamp: u32, +} +// SAFETY: Both fields are integers. +unsafe impl Zeroable for RecoveryInfoEntry {} +static_assert!(size_of::() == 8); + +/// Status B of the primary instance, directly after its state grid. +#[repr(C, packed)] +pub(crate) struct PrimaryStatusB { + /// Status block. + pub(crate) status: StatusBlock, + pub(crate) unk_80: [u8; 0x4038], + pub(crate) recovery_info: [RecoveryInfoEntry; 256], + pub(crate) unk_48b8: [u8; 0x20], + pub(crate) recovery_slot_mask: u64, + /// Firmware-control state, at the start of the firmware-control page. + pub(crate) fwctl_state_va: u64, + /// Firmware-control ring, in the firmware-control page. + pub(crate) fwctl_ring_va: u64, + /// Firmware-written epoch, read using an aligned 64-bit load. + pub(crate) recovery_epoch: u64, + pub(crate) unk_48f8: [u8; 8], + /// Firmware writes 1 and 3; host writes 2 and 0 with full barriers around each 32-bit store. + pub(crate) recovery_state: u32, + pub(crate) unk_4904: [u8; 0xc], + /// Must be zero for an event-backed restart; read using an aligned 32-bit load. + pub(crate) host_recovery: u32, + pub(crate) unk_4914: [u8; 0x9b20], + /// Required: 1. + pub(crate) unk_e434: u32, + pub(crate) unk_e438: [u8; 8], + /// Power state, see [`PowerState`]. + pub(crate) power: PowerState, +} + +static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_info) == 0x40b8); +static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_slot_mask) == 0x48d8); +static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_epoch) == 0x48f0); +static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_state) == 0x4900); +static_assert!(core::mem::offset_of!(PrimaryStatusB, host_recovery) == 0x4910); +static_assert!(core::mem::offset_of!(PrimaryStatusB, fwctl_state_va) == 0x48e0); +static_assert!(core::mem::offset_of!(PrimaryStatusB, unk_e434) == 0xe434); +static_assert!(core::mem::offset_of!(PrimaryStatusB, power) == PrimaryStatusB::POWER_STATE); +static_assert!(size_of::() == 0xeb00); + +// SAFETY: `PrimaryStatusB` consists of integers and arrays of integers only. +unsafe impl Zeroable for PrimaryStatusB {} + +impl PrimaryStatusB { + pub(crate) const RECOVERY_SLOT_MASK: usize = core::mem::offset_of!(Self, recovery_slot_mask); + pub(crate) const RECOVERY_INFO: usize = core::mem::offset_of!(Self, recovery_info); + pub(crate) const RECOVERY_STATE: usize = core::mem::offset_of!(Self, recovery_state); + pub(crate) const HOST_RECOVERY: usize = core::mem::offset_of!(Self, host_recovery); + + /// Offset of the [`PowerState`]. + pub(crate) const POWER_STATE: usize = 0xe440; + + /// Initializes the primary status B; `fwctl_va` is the zeroed + /// firmware-control page ([`FWCTL_SIZE`] bytes). + pub(crate) fn init(&mut self, fwctl_va: u64) { + clear(self); + self.status = StatusBlock::new(); + self.fwctl_state_va = fwctl_va; + self.fwctl_ring_va = fwctl_va + FWCTL_RING; + self.unk_e434 = 1; + self.power.init(); + } +} + +/// Configuration of the secondary instance, named by +/// [`SecondaryRoot::config_va`]. +#[repr(C)] +pub(crate) struct SecondaryConfig { + pub(crate) unk_0: [u8; 0x14], + /// Required: 1. + pub(crate) unk_14: u32, + pub(crate) unk_18: [u8; 0x14], + /// Required: 1. + pub(crate) unk_2c: u32, + /// Required: 1. + pub(crate) unk_30: u32, + pub(crate) unk_34: [u8; 0xc], + /// Required: 1. + pub(crate) unk_40: u32, + pub(crate) unk_44: [u8; 8], + /// Required: 1. + pub(crate) unk_4c: u32, + /// Required: 0x186a (6250). + pub(crate) unk_50: u32, + pub(crate) unk_54: [u8; 0x2c], +} + +static_assert!(size_of::() == 0x80); + +// SAFETY: `SecondaryConfig` consists of integers and arrays of integers only. +unsafe impl Zeroable for SecondaryConfig {} + +impl SecondaryConfig { + /// Initializes the secondary configuration. + pub(crate) fn init(&mut self) { + clear(self); + self.unk_14 = 1; + self.unk_2c = 1; + self.unk_30 = 1; + self.unk_40 = 1; + self.unk_4c = 1; + self.unk_50 = 0x186a; + } +} + +/// Offsets in the private cluster. +pub(crate) mod private { + /// Size of the private cluster. + pub(crate) const SIZE: usize = 0x178000; + /// The primary state grid. + pub(crate) const PRIMARY_STATE: usize = 0; + /// The primary status A block. + pub(crate) const PRIMARY_STATUS_A: usize = 0xee40; + /// The secondary status A block. + pub(crate) const SECONDARY_STATUS_A: usize = 0xc3100; + /// The secondary state grid. + pub(crate) const SECONDARY_STATE: usize = 0x1770c0; + /// [`super::PrimaryStatusB`] or [`super::SecondaryConfig`], from an + /// instance's state grid. + pub(crate) const STATUS_B: usize = 0x340; + /// Secondary state named by [`super::HwDataAux`], from the secondary + /// state grid. + pub(crate) const SECONDARY_STATUS: usize = 0x3c0; +} + +static_assert!(private::STATUS_B == CONTROL_STATE_OFFSET + size_of::()); +static_assert!( + private::PRIMARY_STATE + private::STATUS_B + size_of::() + == private::PRIMARY_STATUS_A +); +static_assert!(private::STATUS_B + size_of::() == private::SECONDARY_STATUS); + +// --------------------------------------------------------------------------- +// Power configuration and auxiliary hardware data (bundle) +// --------------------------------------------------------------------------- + +/// Die temperature sensors read by the die temperature limiter. +const TEMP_SENSOR_MASK: u64 = 0x82a; +/// Die temperature the limiter holds the GPU below, in 0.01 °C. +const TEMP_TARGET: u32 = 11_000; +/// Die temperature below which the limiter releases, in 0.01 °C. +const TEMP_RELEASE: u32 = 8_000; +/// Proportional gain of the limiter: 20.34 as an IEEE-754 single. +const TEMP_KP: u32 = 0x41a2_b852; +/// Integral gain of the limiter, per control period: 9.376 as an IEEE-754 +/// single. +const TEMP_KI: u32 = 0x4116_0419; + +/// Leading part of the [`PowerConfig`]. The host writes it twice: in the +/// power configuration and at [`bundle::POWER_CONFIG_COPY`], where the copy +/// ends with the views it lies in. +#[repr(C, packed)] +pub(crate) struct PowerConfigHead { + pub(crate) unk_0: [u8; 4], + /// Required: 0x5_dc00 (384000). + pub(crate) unk_4: u32, + /// Required: 0x5_dc00 (384000). + pub(crate) unk_8: u32, + pub(crate) unk_c: [u8; 4], + /// Required: 4. + pub(crate) unk_10: u32, + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_14: u32, + pub(crate) unk_18: [u8; 0x14], + /// Required: 1. + pub(crate) unk_2c: u32, + /// Required: 1. + pub(crate) unk_30: u32, + pub(crate) unk_34: [u8; 0xc], + /// Required: 0x64 (100). + pub(crate) unk_40: u32, + /// Required: 1. + pub(crate) unk_44: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_48: u32, + pub(crate) unk_4c: [u8; 4], + /// Required: 0x64 (100). + pub(crate) unk_50: u32, + pub(crate) unk_54: [u8; 0xc], + /// Required: 4. + pub(crate) unk_60: u32, + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_64: u32, + pub(crate) unk_68: [u8; 4], + /// Required: 1. + pub(crate) unk_6c: u32, + /// Required: 1. + pub(crate) unk_70: u32, + pub(crate) unk_74: [u8; 0xc], + /// Required: 0x64 (100). + pub(crate) unk_80: u32, + /// Required: 1. + pub(crate) unk_84: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_88: u32, + pub(crate) unk_8c: [u8; 4], + /// Required: 0x64 (100). + pub(crate) unk_90: u32, + pub(crate) unk_94: [u8; 0x2c], + /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. + pub(crate) unk_c0: [u32; 11], + pub(crate) unk_ec: [u8; 0x8cc], + /// Required: 0x271 (625). + pub(crate) unk_9b8: u32, + pub(crate) unk_9bc: [u8; 8], + /// Required: 0x3f7f_2e9f. + pub(crate) unk_9c4: u32, + pub(crate) unk_9c8: [u8; 4], + /// Required: 0x3b51_6154. + pub(crate) unk_9cc: u32, + pub(crate) unk_9d0: [u8; 4], + /// Required: 0x3ca5_9586 (0.020213 as an IEEE-754 single). + pub(crate) unk_9d4: u32, + pub(crate) unk_9d8: [u8; 8], + /// Required: 0x4615_3800 (9550.0 as an IEEE-754 single). + pub(crate) unk_9e0: u32, + /// Required: 0x40a9_0fdb. + pub(crate) unk_9e4: u32, + pub(crate) unk_9e8: [u8; 4], + /// Required: 0xbdd6_7344. + pub(crate) unk_9ec: u32, + /// Required: 0x28 (40). + pub(crate) unk_9f0: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_9f4: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_9f8: u32, + pub(crate) unk_9fc: [u8; 8], + /// Required: 0x254e (9550). + pub(crate) unk_a04: u32, + pub(crate) unk_a08: [u8; 0x30], + /// Required: 0x3e8 (1000). + pub(crate) unk_a38: u32, + pub(crate) unk_a3c: [u8; 0x34], + /// Required: 4. + pub(crate) unk_a70: u32, + pub(crate) unk_a74: [u8; 0x10], + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_a84: u32, + pub(crate) unk_a88: [u8; 4], + /// Required: 0x40cc_cccd (6.4 as an IEEE-754 single). + pub(crate) unk_a8c: u32, + pub(crate) unk_a90: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_a98: u32, + /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). + pub(crate) unk_a9c: u32, + pub(crate) unk_aa0: [u8; 8], + /// Required: 0x28 (40). + pub(crate) unk_aa8: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_aac: u32, + pub(crate) unk_ab0: [u8; 0xc], + /// Required: 0x254e (9550). + pub(crate) unk_abc: u32, + pub(crate) unk_ac0: [u8; 8], + /// Required: 0x10 (16). + pub(crate) unk_ac8: u32, + pub(crate) unk_acc: [u8; 4], + /// Required: 0x5_dc00 (384000). + pub(crate) unk_ad0: u32, + pub(crate) unk_ad4: [u8; 0x54], + /// Required: 0x5c (92). + pub(crate) unk_b28: u32, + pub(crate) unk_b2c: [u8; 4], + /// Required: 0x64 (100). + pub(crate) unk_b30: u32, + /// Required: 0x22 (34). + pub(crate) unk_b34: u32, + /// Required: 6. + pub(crate) unk_b38: u32, + pub(crate) unk_b3c: [u8; 4], + /// Required: 6. + pub(crate) unk_b40: u32, + /// Required: 1. + pub(crate) unk_b44: u32, + pub(crate) unk_b48: [u8; 4], + /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). + pub(crate) unk_b4c: u32, + /// Required: 0x3f7d_f3b6 (0.992 as an IEEE-754 single). + pub(crate) unk_b50: u32, + /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). + pub(crate) unk_b54: u32, + /// Required: 0x3c03_126f (0.008 as an IEEE-754 single). + pub(crate) unk_b58: u32, + /// Required: 0x3f69_d4d8 (0.913404 as an IEEE-754 single). + pub(crate) unk_b5c: u32, + /// Required: 0x3f69_d4d8 (0.913404 as an IEEE-754 single). + pub(crate) unk_b60: u32, + pub(crate) unk_b64: [u8; 4], + /// Required: 0x42be_0000 (95.0 as an IEEE-754 single). + pub(crate) unk_b68: u32, + /// Required: 0x4064_f5c3 (3.5775 as an IEEE-754 single). + pub(crate) unk_b6c: u32, + /// Required: 0x4064_f5c3 (3.5775 as an IEEE-754 single). + pub(crate) unk_b70: u32, + /// Required: 0x4117_9436. + pub(crate) unk_b74: u32, + /// Required: 0x64 (100). + pub(crate) unk_b78: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_b7c: u32, + /// Required: 0x64 (100). + pub(crate) unk_b80: u32, + pub(crate) unk_b84: [u8; 8], + /// Required: 0x5c (92). + pub(crate) unk_b8c: u32, + pub(crate) unk_b90: [u8; 0x30], + /// Required: 0x64 (100). + pub(crate) unk_bc0: u32, + pub(crate) unk_bc4: [u8; 0x1c], + /// Required: 1. + pub(crate) unk_be0: u8, + /// Required: 4. + pub(crate) unk_be1: u32, + /// Required: 0x64 (100). + pub(crate) unk_be5: u32, + pub(crate) unk_be9: [u8; 4], + /// Required: 0x1f40 (8000). + pub(crate) unk_bed: u32, + /// Required: 0xc8 (200). + pub(crate) unk_bf1: u32, + /// Required: 0xfa0 (4000). + pub(crate) unk_bf5: u32, + /// Required: 0xc8 (200). + pub(crate) unk_bf9: u32, + /// Required: 0x7d0 (2000). + pub(crate) unk_bfd: u32, + /// Required: 0xc8 (200). + pub(crate) unk_c01: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_c05: u32, + /// Required: 0xc8 (200). + pub(crate) unk_c09: u32, + /// Required: 1. + pub(crate) unk_c0d: u32, + pub(crate) unk_c11: [u8; 0x53], + /// Required: 1. + pub(crate) unk_c64: u32, + pub(crate) unk_c68: [u8; 0x28], + /// Required: 1. + pub(crate) unk_c90: u32, + pub(crate) unk_c94: [u8; 0x1003], + /// Required: 0x7fff in every entry; the table continues in + /// [`PowerConfigTail`]. + pub(crate) unk_1c97: [u16; 148], + /// Required: 0xff (low byte of the next table entry). + pub(crate) unk_1dbf: u8, +} + +static_assert!(bundle::POWER_CONFIG_COPY + size_of::() == 0xf000); + +// SAFETY: `PowerConfigHead` consists of integers and arrays of integers only. +unsafe impl Zeroable for PowerConfigHead {} + +impl PowerConfigHead { + /// Initializes the leading part of the power configuration. + pub(crate) fn init(&mut self) { + clear(self); + self.unk_4 = 0x5_dc00; + self.unk_8 = 0x5_dc00; + self.unk_10 = 4; + self.unk_14 = 0x3f80_0000; + self.unk_2c = 1; + self.unk_30 = 1; + self.unk_40 = 0x64; + self.unk_44 = 1; + self.unk_48 = 0x3e8; + self.unk_50 = 0x64; + self.unk_60 = 4; + self.unk_64 = 0x3f80_0000; + self.unk_6c = 1; + self.unk_70 = 1; + self.unk_80 = 0x64; + self.unk_84 = 1; + self.unk_88 = 0x3e8; + self.unk_90 = 0x64; + self.unk_c0 = [0x3f82_8f5c; 11]; + self.unk_9b8 = 0x271; + self.unk_9c4 = 0x3f7f_2e9f; + self.unk_9cc = 0x3b51_6154; + self.unk_9d4 = 0x3ca5_9586; + self.unk_9e0 = 0x4615_3800; + self.unk_9e4 = 0x40a9_0fdb; + self.unk_9ec = 0xbdd6_7344; + self.unk_9f0 = 0x28; + self.unk_9f4 = 0x3e8; + self.unk_9f8 = 0x3e8; + self.unk_a04 = 0x254e; + self.unk_a38 = 0x3e8; + self.unk_a70 = 4; + self.unk_a84 = 0x3f80_0000; + self.unk_a8c = 0x40cc_cccd; + self.unk_a98 = 0x4780_0000; + self.unk_a9c = 0x3f00_0000; + self.unk_aa8 = 0x28; + self.unk_aac = 0x3e8; + self.unk_abc = 0x254e; + self.unk_ac8 = 0x10; + self.unk_ad0 = 0x5_dc00; + self.unk_b28 = 0x5c; + self.unk_b30 = 0x64; + self.unk_b34 = 0x22; + self.unk_b38 = 6; + self.unk_b40 = 6; + self.unk_b44 = 1; + self.unk_b4c = 0x3f4c_cccd; + self.unk_b50 = 0x3f7d_f3b6; + self.unk_b54 = 0x3e4c_cccd; + self.unk_b58 = 0x3c03_126f; + self.unk_b5c = 0x3f69_d4d8; + self.unk_b60 = 0x3f69_d4d8; + self.unk_b68 = 0x42be_0000; + self.unk_b6c = 0x4064_f5c3; + self.unk_b70 = 0x4064_f5c3; + self.unk_b74 = 0x4117_9436; + self.unk_b78 = 0x64; + self.unk_b7c = 0x3e8; + self.unk_b80 = 0x64; + self.unk_b8c = 0x5c; + self.unk_bc0 = 0x64; + self.unk_be0 = 1; + self.unk_be1 = 4; + self.unk_be5 = 0x64; + self.unk_bed = 0x1f40; + self.unk_bf1 = 0xc8; + self.unk_bf5 = 0xfa0; + self.unk_bf9 = 0xc8; + self.unk_bfd = 0x7d0; + self.unk_c01 = 0xc8; + self.unk_c05 = 0x3e8; + self.unk_c09 = 0xc8; + self.unk_c0d = 1; + self.unk_c64 = 1; + self.unk_c90 = 1; + self.unk_1c97 = [0x7fff; 148]; + self.unk_1dbf = 0xff; + } +} + +/// Trailing part of the [`PowerConfig`]. Field names give the offset in the +/// whole power configuration. It carries the die temperature limiter: target +/// and release temperatures in 0.01 °C, proportional and integral gains as +/// IEEE-754 singles. +#[repr(C, packed)] +pub(crate) struct PowerConfigTail { + /// Required: 0x7f (high byte of the table entry started in the head). + pub(crate) unk_1dc0: u8, + /// Required: 0x7fff in every entry. + pub(crate) unk_1dc1: [u16; 363], + pub(crate) unk_2097: [u8; 0x201], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2298: u32, + /// Required: 0x4220_0000 (40.0 as an IEEE-754 single). + pub(crate) unk_229c: u32, + /// Required: 0x447a_0000 (1000.0 as an IEEE-754 single). + pub(crate) unk_22a0: u32, + /// Required: 0x5be (1470). + pub(crate) unk_22a4: u32, + pub(crate) unk_22a8: [u8; 0x10], + /// Required: 0x28 (40). + pub(crate) unk_22b8: u32, + pub(crate) unk_22bc: [u8; 4], + /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). + pub(crate) unk_22c0: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_22c4: u32, + pub(crate) unk_22c8: [u8; 4], + /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). + pub(crate) unk_22cc: u32, + /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). + pub(crate) unk_22d0: u32, + pub(crate) unk_22d4: [u8; 0x3c], + /// Die temperature sensors the limiter reads ([`TEMP_SENSOR_MASK`]). + pub(crate) temp_sensor_mask: u64, + /// Release temperature of the limiter ([`TEMP_RELEASE`]). + pub(crate) temp_release: u32, + pub(crate) unk_231c: [u8; 4], + /// Required: 4. + pub(crate) unk_2320: u32, + pub(crate) unk_2324: [u8; 0x10], + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_2334: u32, + pub(crate) unk_2338: [u8; 4], + /// Integral gain of the limiter ([`TEMP_KI`]). + pub(crate) temp_ki: u32, + pub(crate) unk_2340: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2348: u32, + /// Proportional gain of the limiter ([`TEMP_KP`]). + pub(crate) temp_kp: u32, + pub(crate) unk_2350: [u8; 8], + /// Required: 0x28 (40). + pub(crate) unk_2358: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_235c: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2360: u32, + /// Margin below the target at which the limiter acts, in 0.01 °C. + pub(crate) temp_margin: u32, + pub(crate) unk_2368: [u8; 4], + /// Target temperature of the limiter ([`TEMP_TARGET`]). + pub(crate) temp_target: u32, + pub(crate) unk_2370: [u8; 0x30], + /// Required: 0x3e8 (1000). + pub(crate) unk_23a0: u32, + pub(crate) unk_23a4: [u8; 0x218], + /// Required: 0x4200_0006. + pub(crate) unk_25bc: u32, + pub(crate) unk_25c0: [u8; 0x212], + /// Required: 0x3f7e_b852 (0.995 as an IEEE-754 single). + pub(crate) unk_27d2: u32, + pub(crate) unk_27d6: [u8; 4], + /// Required: 0x3ba3_d70a (0.005 as an IEEE-754 single). + pub(crate) unk_27da: u32, + pub(crate) unk_27de: [u8; 4], + /// Required: 0x43c8_0000 (400.0 as an IEEE-754 single). + pub(crate) unk_27e2: u32, + pub(crate) unk_27e6: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_27ee: u32, + /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). + pub(crate) unk_27f2: u32, + pub(crate) unk_27f6: [u8; 4], + /// Required: 0xbac8_0000. + pub(crate) unk_27fa: u32, + /// Required: 0x384 (900). + pub(crate) unk_27fe: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2802: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2806: u32, + pub(crate) unk_280a: [u8; 4], + /// Required: 0x453b_8000 (3000.0 as an IEEE-754 single). + pub(crate) unk_280e: u32, + /// Required: 0x26ac (9900). + pub(crate) unk_2812: u32, + pub(crate) unk_2816: [u8; 0x30], + /// Required: 0x3e8 (1000). + pub(crate) unk_2846: u32, + pub(crate) unk_284a: [u8; 0x18], + /// Required: 0xc8 (200). + pub(crate) unk_2862: u32, + pub(crate) unk_2866: [u8; 0xb2], + /// Required: 0x3e8 (1000). + pub(crate) unk_2918: u32, + pub(crate) unk_291c: [u8; 0x10], + /// Required: 1. + pub(crate) unk_292c: u32, + pub(crate) unk_2930: [u8; 0x30], + /// Required: 1. + pub(crate) unk_2960: u8, + /// Required: 1. + pub(crate) unk_2961: u32, + /// Required: 4. + pub(crate) unk_2965: u32, + pub(crate) unk_2969: [u8; 0x14], + /// Required: 0x23f (575). + pub(crate) unk_297d: u32, + /// Required: 1. + pub(crate) unk_2981: u32, + /// Required: 1. + pub(crate) unk_2985: u32, + pub(crate) unk_2989: [u8; 0x14], + /// Required: 0x240 (576). + pub(crate) unk_299d: u32, + /// Required: 1. + pub(crate) unk_29a1: u32, + /// Required: 1. + pub(crate) unk_29a5: u32, + pub(crate) unk_29a9: [u8; 0x14], + /// Required: 0x241 (577). + pub(crate) unk_29bd: u32, + pub(crate) unk_29c1: [u8; 0x47], + /// Required: 1. + pub(crate) unk_2a08: u32, + pub(crate) unk_2a0c: [u8; 4], + /// Required: 0x254e (9550) in every entry. + pub(crate) unk_2a10: [u32; 3], + pub(crate) unk_2a1c: [u8; 0xc], + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_2a28: u32, + /// Required: 4. + pub(crate) unk_2a2c: u32, + /// Required: 0x10 (16). + pub(crate) unk_2a30: u32, + /// Required: 0x5_dc00 (384000). + pub(crate) unk_2a34: u32, + pub(crate) unk_2a38: [u8; 0xcc], + /// Required: 0x3c (60). + pub(crate) unk_2b04: u32, + pub(crate) unk_2b08: [u8; 8], + /// Required: 0x3f6e_eeef. + pub(crate) unk_2b10: u32, + pub(crate) unk_2b14: [u8; 4], + /// Required: 0x3d88_8889. + pub(crate) unk_2b18: u32, + pub(crate) unk_2b1c: [u8; 4], + /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). + pub(crate) unk_2b20: u32, + pub(crate) unk_2b24: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2b2c: u32, + /// Required: 0x4088_0000 (4.25 as an IEEE-754 single). + pub(crate) unk_2b30: u32, + pub(crate) unk_2b34: [u8; 8], + /// Required: 0x28 (40). + pub(crate) unk_2b3c: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2b40: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2b44: u32, + pub(crate) unk_2b48: [u8; 4], + /// Required: 0x4615_3800 (9550.0 as an IEEE-754 single). + pub(crate) unk_2b4c: u32, + /// Required: 0x254e (9550). + pub(crate) unk_2b50: u32, + pub(crate) unk_2b54: [u8; 8], + /// Required: 0xf0 (240). + pub(crate) unk_2b5c: u32, + pub(crate) unk_2b60: [u8; 4], + /// Required: 0x57_e400. + pub(crate) unk_2b64: u32, + pub(crate) unk_2b68: [u8; 0x1c], + /// Required: 0x3e8 (1000). + pub(crate) unk_2b84: u32, + pub(crate) unk_2b88: [u8; 0x3c], + /// Required: 1. + pub(crate) unk_2bc4: u32, + pub(crate) unk_2bc8: [u8; 0xcc], + /// Required: 0x32 (50). + pub(crate) unk_2c94: u32, + /// Required: 1. + pub(crate) unk_2c98: u32, + pub(crate) unk_2c9c: [u8; 4], + /// Required: 0x3f63_8e39. + pub(crate) unk_2ca0: u32, + /// Required: 0x3f2a_aaab. + pub(crate) unk_2ca4: u32, + /// Required: 0x3de3_8e39. + pub(crate) unk_2ca8: u32, + /// Required: 0x3eaa_aaab. + pub(crate) unk_2cac: u32, + /// Required: 0xbf4c_cccd (-0.8 as an IEEE-754 single). + pub(crate) unk_2cb0: u32, + /// Required: 0xbf4c_cccd (-0.8 as an IEEE-754 single). + pub(crate) unk_2cb4: u32, + pub(crate) unk_2cb8: [u8; 4], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2cbc: u32, + /// Required: 0xc0a0_0000 (-5.0 as an IEEE-754 single). + pub(crate) unk_2cc0: u32, + /// Required: 0xc0a0_0000 (-5.0 as an IEEE-754 single). + pub(crate) unk_2cc4: u32, + pub(crate) unk_2cc8: [u8; 4], + /// Required: 0x64 (100). + pub(crate) unk_2ccc: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2cd0: u32, + /// Required: 0x64 (100). + pub(crate) unk_2cd4: u32, + pub(crate) unk_2cd8: [u8; 4], + /// Required: 0x467a_0000 (16000.0 as an IEEE-754 single). + pub(crate) unk_2cdc: u32, + /// Required: 0x578 (1400). + pub(crate) unk_2ce0: u32, + pub(crate) unk_2ce4: [u8; 8], + /// Required: 0x90 (144). + pub(crate) unk_2cec: u32, + /// Required: 0x30 (48). + pub(crate) unk_2cf0: u32, + /// Required: 0x34_bc00. + pub(crate) unk_2cf4: u32, + pub(crate) unk_2cf8: [u8; 4], + /// Required: 0x11_9400. + pub(crate) unk_2cfc: u32, + pub(crate) unk_2d00: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2d08: u32, + pub(crate) unk_2d0c: [u8; 0x24], + /// Required: 0x3e80 (16000). + pub(crate) unk_2d30: u32, + /// Required: 2. + pub(crate) unk_2d34: u32, + /// Required: 0x9c4 (2500). + pub(crate) unk_2d38: u32, + /// Required: 0x20d (525). + pub(crate) unk_2d3c: u32, + /// Required: 2. + pub(crate) unk_2d40: u32, + /// Required: 4. + pub(crate) unk_2d44: u32, + /// Required: 0x32 (50). + pub(crate) unk_2d48: u32, + pub(crate) unk_2d4c: [u8; 0x2c], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2d78: u32, + pub(crate) unk_2d7c: [u8; 0xc], + /// Required: 0x28 (40). + pub(crate) unk_2d88: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2d8c: u32, + pub(crate) unk_2d90: [u8; 0x30], +} + +static_assert!(core::mem::offset_of!(PowerConfigTail, temp_sensor_mask) == 0x2310 - 0x1dc0); +static_assert!(core::mem::offset_of!(PowerConfigTail, temp_target) == 0x236c - 0x1dc0); +static_assert!(size_of::() == 0x1000); + +// SAFETY: `PowerConfigTail` consists of integers and arrays of integers only. +unsafe impl Zeroable for PowerConfigTail {} + +impl PowerConfigTail { + fn init(&mut self) { + clear(self); + self.unk_1dc0 = 0x7f; + self.unk_1dc1 = [0x7fff; 363]; + self.unk_2298 = 0x4780_0000; + self.unk_229c = 0x4220_0000; + self.unk_22a0 = 0x447a_0000; + self.unk_22a4 = 0x5be; + self.unk_22b8 = 0x28; + self.unk_22c0 = 0x42c8_0000; + self.unk_22c4 = 0x3e8; + self.unk_22cc = 0x3f4c_cccd; + self.unk_22d0 = 0x3e4c_cccd; + self.temp_sensor_mask = TEMP_SENSOR_MASK; + self.temp_release = TEMP_RELEASE; + self.unk_2320 = 4; + self.unk_2334 = 0x3f80_0000; + self.temp_ki = TEMP_KI; + self.unk_2348 = 0x4780_0000; + self.temp_kp = TEMP_KP; + self.unk_2358 = 0x28; + self.unk_235c = 0x3e8; + self.unk_2360 = 0x3e8; + self.temp_target = TEMP_TARGET; + self.unk_23a0 = 0x3e8; + self.unk_25bc = 0x4200_0006; + self.unk_27d2 = 0x3f7e_b852; + self.unk_27da = 0x3ba3_d70a; + self.unk_27e2 = 0x43c8_0000; + self.unk_27ee = 0x4780_0000; + self.unk_27f2 = 0x42c8_0000; + self.unk_27fa = 0xbac8_0000; + self.unk_27fe = 0x384; + self.unk_2802 = 0x3e8; + self.unk_2806 = 0x3e8; + self.unk_280e = 0x453b_8000; + self.unk_2812 = 0x26ac; + self.unk_2846 = 0x3e8; + self.unk_2862 = 0xc8; + self.unk_2918 = 0x3e8; + self.unk_292c = 1; + self.unk_2960 = 1; + self.unk_2961 = 1; + self.unk_2965 = 4; + self.unk_297d = 0x23f; + self.unk_2981 = 1; + self.unk_2985 = 1; + self.unk_299d = 0x240; + self.unk_29a1 = 1; + self.unk_29a5 = 1; + self.unk_29bd = 0x241; + self.unk_2a08 = 1; + self.unk_2a10 = [0x254e; 3]; + self.unk_2a28 = 0x3f80_0000; + self.unk_2a2c = 4; + self.unk_2a30 = 0x10; + self.unk_2a34 = 0x5_dc00; + self.unk_2b04 = 0x3c; + self.unk_2b10 = 0x3f6e_eeef; + self.unk_2b18 = 0x3d88_8889; + self.unk_2b20 = 0x3f00_0000; + self.unk_2b2c = 0x4780_0000; + self.unk_2b30 = 0x4088_0000; + self.unk_2b3c = 0x28; + self.unk_2b40 = 0x3e8; + self.unk_2b44 = 0x3e8; + self.unk_2b4c = 0x4615_3800; + self.unk_2b50 = 0x254e; + self.unk_2b5c = 0xf0; + self.unk_2b64 = 0x57_e400; + self.unk_2b84 = 0x3e8; + self.unk_2bc4 = 1; + self.unk_2c94 = 0x32; + self.unk_2c98 = 1; + self.unk_2ca0 = 0x3f63_8e39; + self.unk_2ca4 = 0x3f2a_aaab; + self.unk_2ca8 = 0x3de3_8e39; + self.unk_2cac = 0x3eaa_aaab; + self.unk_2cb0 = 0xbf4c_cccd; + self.unk_2cb4 = 0xbf4c_cccd; + self.unk_2cbc = 0x4780_0000; + self.unk_2cc0 = 0xc0a0_0000; + self.unk_2cc4 = 0xc0a0_0000; + self.unk_2ccc = 0x64; + self.unk_2cd0 = 0x3e8; + self.unk_2cd4 = 0x64; + self.unk_2cdc = 0x467a_0000; + self.unk_2ce0 = 0x578; + self.unk_2cec = 0x90; + self.unk_2cf0 = 0x30; + self.unk_2cf4 = 0x34_bc00; + self.unk_2cfc = 0x11_9400; + self.unk_2d08 = 0x4780_0000; + self.unk_2d30 = 0x3e80; + self.unk_2d34 = 2; + self.unk_2d38 = 0x9c4; + self.unk_2d3c = 0x20d; + self.unk_2d40 = 2; + self.unk_2d44 = 4; + self.unk_2d48 = 0x32; + self.unk_2d78 = 0x4780_0000; + self.unk_2d88 = 0x28; + self.unk_2d8c = 0x3e8; + } +} + +/// Power configuration, in view 2 of the bundle. The secondary instance +/// finds it through [`MainConfig::power_config_va`]. +#[repr(C, packed)] +pub(crate) struct PowerConfig { + /// Leading part. + pub(crate) head: PowerConfigHead, + /// Trailing part. + pub(crate) tail: PowerConfigTail, +} + +static_assert!(bundle::POWER_CONFIG + size_of::() == bundle::AUX); +static_assert!(bundle::VIEWS[2] + 0xe40 == bundle::POWER_CONFIG); + +// SAFETY: `PowerConfig` consists of integers and arrays of integers only. +unsafe impl Zeroable for PowerConfig {} + +impl PowerConfig { + /// Initializes the power configuration. + pub(crate) fn init(&mut self) { + self.head.init(); + self.tail.init(); + } +} + +/// Auxiliary hardware data the firmware reads at [`bundle::AUX`], up to the +/// start of view 3. No pointer names it. +#[repr(C, packed)] +pub(crate) struct HwDataAux { + pub(crate) unk_0: [u8; 0xac], + /// Required: 6. + pub(crate) unk_ac: u32, + pub(crate) unk_b0: [u8; 8], + /// Required: 1. + pub(crate) unk_b8: u32, + pub(crate) unk_bc: [u8; 0xbb], + /// Required: 0xdac (3500). + pub(crate) unk_177: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_17b: u32, + /// Required: 0xbb8 (3000). + pub(crate) unk_17f: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_183: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_187: u32, + /// Required: 0x64 (100). + pub(crate) unk_18b: u32, + /// Required: 0x20 (32). + pub(crate) unk_18f: u32, + pub(crate) unk_193: [u8; 0x21], + /// Required: 4. + pub(crate) unk_1b4: u32, + pub(crate) unk_1b8: [u8; 8], + /// Required: 0x3f78_0000 (0.96875 as an IEEE-754 single). + pub(crate) unk_1c0: u32, + pub(crate) unk_1c4: [u8; 4], + /// Required: 0x3d00_0000 (0.03125 as an IEEE-754 single). + pub(crate) unk_1c8: u32, + pub(crate) unk_1cc: [u8; 4], + /// Required: 0x40a0_0000 (5.0 as an IEEE-754 single). + pub(crate) unk_1d0: u32, + pub(crate) unk_1d4: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_1dc: u32, + /// Required: 0x4170_0000 (15.0 as an IEEE-754 single). + pub(crate) unk_1e0: u32, + pub(crate) unk_1e4: [u8; 8], + /// Required: 0x384 (900). + pub(crate) unk_1ec: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_1f0: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_1f4: u32, + pub(crate) unk_1f8: [u8; 8], + /// Required: 0x3e8 (1000). + pub(crate) unk_200: u32, + pub(crate) unk_204: [u8; 0x30], + /// Required: 0x3e8 (1000). + pub(crate) unk_234: u32, + pub(crate) unk_238: [u8; 0x3c], + /// Required: 0x64 (100). + pub(crate) unk_274: u32, + pub(crate) unk_278: [u8; 0x18], + /// Required: 0x82a (2090). + pub(crate) unk_290: u32, + pub(crate) unk_294: [u8; 4], + /// Required: 0x7d (125). + pub(crate) unk_298: u32, + /// Required: 1. + pub(crate) unk_29c: u32, + pub(crate) unk_2a0: [u8; 0xc38], + /// GPU address of the secondary state, [`private::SECONDARY_STATUS`] bytes + /// into the secondary state grid. + pub(crate) secondary_status_va: u64, + /// Required: 0x18ae (6318). + pub(crate) unk_ee0: u32, + pub(crate) unk_ee4: [u8; 4], + /// Required: 1. + pub(crate) unk_ee8: u32, + pub(crate) unk_eec: [u8; 0x2000], + /// Required: 0x41c8_cccd (25.1 as an IEEE-754 single). + pub(crate) unk_2eec: u32, + /// Required: 0x41c8_cccd (25.1 as an IEEE-754 single). + pub(crate) unk_2ef0: u32, + pub(crate) unk_2ef4: [u8; 0x6f0], + /// Required: 1. + pub(crate) unk_35e4: u32, + /// Required: 0x1f4 (500). + pub(crate) unk_35e8: u32, + /// Required: 6. + pub(crate) unk_35ec: u32, + /// Required: 0x30d4 (12500) in every entry. + pub(crate) unk_35f0: [u32; 6], + /// Required: 0xfa0 (4000). + pub(crate) unk_3608: u32, + pub(crate) unk_360c: [u8; 0x10], + /// Required: 0x30d4 (12500) in every entry. + pub(crate) unk_361c: [u32; 5], + /// Required: 1. + pub(crate) unk_3630: u32, + pub(crate) unk_3634: [u8; 0x174], + /// Required: 1. + pub(crate) unk_37a8: u32, + pub(crate) unk_37ac: [u8; 8], + /// Required: 0x424c_cccd (51.2 as an IEEE-754 single). + pub(crate) unk_37b4: u32, + pub(crate) unk_37b8: [u8; 0x3c], + /// Required: 0x4406_8000 (538.0 as an IEEE-754 single). + pub(crate) unk_37f4: u32, + pub(crate) unk_37f8: [u8; 0x3c], + /// Required: 0x41c9_999a (25.2 as an IEEE-754 single). + pub(crate) unk_3834: u32, + pub(crate) unk_3838: [u8; 0xa8], + /// Required: 0x186a (6250). + pub(crate) unk_38e0: u32, + pub(crate) unk_38e4: [u8; 0x39c], +} + +static_assert!(core::mem::offset_of!(HwDataAux, secondary_status_va) == 0xed8); +static_assert!(bundle::AUX + size_of::() == bundle::VIEWS[3]); + +// SAFETY: `HwDataAux` consists of integers and arrays of integers only. +unsafe impl Zeroable for HwDataAux {} + +impl HwDataAux { + /// Initializes the auxiliary hardware data. `secondary_status_va` is + /// the secondary state grid plus [`private::SECONDARY_STATUS`]. + pub(crate) fn init(&mut self, secondary_status_va: u64) { + clear(self); + self.unk_ac = 6; + self.unk_b8 = 1; + self.unk_177 = 0xdac; + self.unk_17b = 0x3e8; + self.unk_17f = 0xbb8; + self.unk_183 = 0x3e8; + self.unk_187 = 0x3e8; + self.unk_18b = 0x64; + self.unk_18f = 0x20; + self.unk_1b4 = 4; + self.unk_1c0 = 0x3f78_0000; + self.unk_1c8 = 0x3d00_0000; + self.unk_1d0 = 0x40a0_0000; + self.unk_1dc = 0x4780_0000; + self.unk_1e0 = 0x4170_0000; + self.unk_1ec = 0x384; + self.unk_1f0 = 0x3e8; + self.unk_1f4 = 0x3e8; + self.unk_200 = 0x3e8; + self.unk_234 = 0x3e8; + self.unk_274 = 0x64; + self.unk_290 = 0x82a; + self.unk_298 = 0x7d; + self.unk_29c = 1; + self.unk_ee0 = 0x18ae; + self.unk_ee8 = 1; + self.unk_2eec = 0x41c8_cccd; + self.unk_2ef0 = 0x41c8_cccd; + self.unk_35e4 = 1; + self.unk_35e8 = 0x1f4; + self.unk_35ec = 6; + self.unk_35f0 = [0x30d4; 6]; + self.unk_3608 = 0xfa0; + self.unk_361c = [0x30d4; 5]; + self.unk_3630 = 1; + self.unk_37a8 = 1; + self.unk_37b4 = 0x424c_cccd; + self.unk_37f4 = 0x4406_8000; + self.unk_3834 = 0x41c9_999a; + self.unk_38e0 = 0x186a; + self.secondary_status_va = secondary_status_va; + } +} +// --------------------------------------------------------------------------- +// Free lists +// --------------------------------------------------------------------------- + +/// Size of a free-list page. +const FREE_LIST_PAGE_SIZE: u64 = 0x1000; +/// Pages in one free-list block. +pub(crate) const FREE_LIST_BLOCK_PAGES: u32 = 0x100; +/// Size of one free-list block. +pub(crate) const FREE_LIST_BLOCK_SIZE: usize = + FREE_LIST_BLOCK_PAGES as usize * FREE_LIST_PAGE_SIZE as usize; +/// Run-list qwords reserved for each block. +const RUN_SLOT_QWORDS: u32 = 8; +/// Bit position of the page count in a run descriptor. +const RUN_PAGES_SHIFT: u32 = 52; +/// Address bits of a run descriptor. +const RUN_ADDRESS_MASK: u64 = 0x0000_ffff_ffff_f000; + +/// Class of a free list, selecting [`FreeListControl::unk_10`] and +/// [`FreeListControl::unk_60`]. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum FreeListClass { + /// `unk_10` 1, `unk_60` 2: the boot render and compute free lists. + One = 1, + /// `unk_10` 2, `unk_60` 3: the auxiliary boot and runtime render pools. + Two = 2, +} + +impl FreeListClass { + const fn unk_60(self) -> u32 { + match self { + Self::One => 2, + Self::Two => 3, + } + } +} + +/// Description of a free list whose blocks are all populated. +pub(crate) struct FreeListArgs { + /// Buffer slot of the free list. + pub(crate) buffer_slot: u32, + /// Class of the free list. + pub(crate) class: FreeListClass, + /// Page list: one 64-bit address per page of the blocks. + pub(crate) page_list_va: u64, + /// Run list: one [`FreeListRunSlot`] per block. + pub(crate) run_list_va: u64, + /// Blocks in the free list. + pub(crate) blocks: u32, + /// [`FreeListState`]. + pub(crate) state_va: u64, +} + +/// Control object of a USC private-memory free list, at the start of its +/// page. +/// +/// This constructor describes the boot objects. Once a pool is published, its +/// page counters (`+0x24`, `+0x2c`) and run cursor (`+0x48`) are shared state. +/// The firmware-completed operation counter occupies the eight bytes at +/// `+0x54`; it is unaligned and the host reads its bytes individually. +/// A released render pool can be repopulated only after its release is +/// consumed: write page-list/run-list words first, then the complete control +/// page in ascending aligned 64-bit words, then a full barrier. The completed +/// count in that image must equal the retained submitted count, not zero. +/// Publication and reuse belong to the host free-list implementation. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FreeListControl { + /// Boot value 1, compute value 2, or the render submitted-counter address. + pub(crate) unk_0: u64, + /// Buffer slot of the free list. + pub(crate) buffer_slot: u32, + pub(crate) unk_c: u32, + /// Required: see [`FreeListClass`]. + pub(crate) unk_10: u32, + /// Page list. + pub(crate) page_list_va: u64, + /// Required: 0x40000. + pub(crate) unk_1c: u32, + pub(crate) unk_20: u32, + /// Required: [`Self::page_count`]. + pub(crate) unk_24: u32, + pub(crate) unk_28: u32, + /// Pages in the free list's blocks. + pub(crate) page_count: u32, + /// Run list. + pub(crate) run_list_va: u64, + pub(crate) unk_38: [u8; 8], + /// Required: 4. + pub(crate) unk_40: u32, + pub(crate) unk_44: u32, + /// Run-list qwords in use. + pub(crate) run_cursor: u32, + /// [`FreeListState`]. + pub(crate) state_va: u64, + pub(crate) unk_54: [u8; 0xc], + /// Required: see [`FreeListClass`]. + pub(crate) unk_60: u32, + pub(crate) unk_64: u32, +} + +// SAFETY: `FreeListControl` consists of integers and arrays of integers only. +unsafe impl Zeroable for FreeListControl {} + +static_assert!(core::mem::offset_of!(FreeListControl, page_list_va) == 0x14); +static_assert!(core::mem::offset_of!(FreeListControl, run_cursor) == 0x48); +static_assert!(core::mem::offset_of!(FreeListControl, state_va) == 0x4c); +static_assert!(size_of::() == 0x68); + +impl FreeListControl { + /// Runtime render control; submitted and completed work are retained + /// across release/repopulation of the same pool. + pub(crate) fn render(args: &FreeListArgs, completed: u64) -> Self { + let mut control = Self::new(args); + control.unk_0 = args.state_va; + control.unk_54[..8].copy_from_slice(&completed.to_le_bytes()); + control + } + + /// Runtime compute control, with a compact run list and shared counters. + pub(crate) fn compute(args: &FreeListArgs) -> Self { + let mut control = Self::new(args); + control.unk_0 = 2; + control.unk_10 = 2; + control.unk_54[8..12].copy_from_slice(&1u32.to_le_bytes()); + control + } + + /// Offset of the firmware-completed counter. It must be read bytewise. + pub(crate) const COMPLETED: usize = 0x54; + + /// Returns the control object of the free list `args` describes. + pub(crate) const fn new(args: &FreeListArgs) -> Self { + let pages = args.blocks * FREE_LIST_BLOCK_PAGES; + Self { + unk_0: 1, + buffer_slot: args.buffer_slot, + unk_c: 0, + unk_10: args.class as u32, + page_list_va: args.page_list_va, + unk_1c: 0x4_0000, + unk_20: 0, + unk_24: pages, + unk_28: 0, + page_count: pages, + run_list_va: args.run_list_va, + unk_38: [0; 8], + unk_40: 4, + unk_44: 0, + run_cursor: args.blocks * RUN_SLOT_QWORDS, + state_va: args.state_va, + unk_54: [0; 0xc], + unk_60: args.class.unk_60(), + unk_64: 0, + } + } +} + +/// Shared state of a free list, at the start of its page. +/// +/// Compute creation writes a 32-bit one at `+0x00` in a zeroed page. The +/// render boot object starts with a 64-bit two and the boot pool with zero. +/// In a live render pool this becomes the submitted-operation counter: the +/// host uses one aligned 64-bit store before making work visible. It is +/// retained across release/repopulation, never reset by a struct copy. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FreeListState { + /// [`Self::RENDER`], [`Self::BOOT`] or [`Self::COMPUTE`] at boot. + pub(crate) unk_0: u64, +} + +// SAFETY: `FreeListState` consists of integers and arrays of integers only. +unsafe impl Zeroable for FreeListState {} + +impl FreeListState { + /// Boot state of the render free list. + pub(crate) const RENDER: Self = Self { unk_0: 2 }; + /// Boot state of the boot free list. + pub(crate) const BOOT: Self = Self { unk_0: 0 }; + /// Boot state of a compute free list. + pub(crate) const COMPUTE: Self = Self { unk_0: 1 }; +} + +/// One run-list slot: the run descriptor of one block. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FreeListRunSlot { + /// Block address, with its page count in the top bits. + pub(crate) run: u64, + pub(crate) unk_8: [u8; 0x38], +} + +// SAFETY: `FreeListRunSlot` consists of integers and arrays of integers only. +unsafe impl Zeroable for FreeListRunSlot {} + +static_assert!(size_of::() == RUN_SLOT_QWORDS as usize * size_of::()); + +impl FreeListRunSlot { + /// Returns the slot describing the block at `block_va`. + pub(crate) const fn new(block_va: u64) -> Self { + Self { + run: (block_va & RUN_ADDRESS_MASK) + | ((FREE_LIST_BLOCK_PAGES as u64) << RUN_PAGES_SHIFT), + unk_8: [0; 0x38], + } + } +} + +/// Writes the run descriptors of `blocks` to the start of `slots`. +pub(crate) fn fill_run_list( + slots: &mut [FreeListRunSlot], + blocks: impl IntoIterator, +) -> Result { + let mut slots = slots.iter_mut(); + for block in blocks { + *slots.next().ok_or(EINVAL)? = FreeListRunSlot::new(block); + } + Ok(()) +} + +/// Writes the address of every page of `blocks` to the start of `list`. +pub(crate) fn fill_page_list(list: &mut [u64], blocks: impl IntoIterator) -> Result { + let mut entries = list.iter_mut(); + for block in blocks { + for page in 0..u64::from(FREE_LIST_BLOCK_PAGES) { + *entries.next().ok_or(EINVAL)? = block + .checked_add(page * FREE_LIST_PAGE_SIZE) + .ok_or(EOVERFLOW)?; + } + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Parameter-buffer and page-pool descriptor tables +// --------------------------------------------------------------------------- + +/// One parameter-buffer descriptor, indexed by buffer slot. +#[repr(C)] +pub(crate) struct PbDescriptor { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, +} + +// SAFETY: `PbDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for PbDescriptor {} + +static_assert!(size_of::() == 0x10); + +/// Parameter-buffer descriptor table, one page. +#[repr(C)] +pub(crate) struct PbDescriptorTable { + /// Descriptors. + pub(crate) entries: [PbDescriptor; PAGE_SIZE / size_of::()], +} + +// SAFETY: `PbDescriptorTable` consists of integers only. +unsafe impl Zeroable for PbDescriptorTable {} + +impl PbDescriptorTable { + /// Required boot contents of slots 0 and 1. + const BOOT: [PbDescriptor; 2] = [ + PbDescriptor { + unk_0: 0x1_9000, + unk_4: 0x20, + unk_8: 0, + unk_c: 0, + }, + PbDescriptor { + unk_0: 0x7_7000, + unk_4: 0x80, + unk_8: 0x17, + unk_c: 0x17, + }, + ]; + + /// Initializes the table. + pub(crate) fn init(&mut self) { + clear(self); + let [slot0, slot1] = Self::BOOT; + self.entries[0] = slot0; + self.entries[1] = slot1; + } +} + +/// Bit position of the unit size in [`PagePoolDescriptor::page_list`]. +const PAGE_POOL_UNIT_SHIFT: u32 = 41; +/// Required unit size of a page-pool descriptor. +const PAGE_POOL_UNIT: u64 = 0x4_0000; +/// Alignment shift of the page-list address in [`PagePoolDescriptor::page_list`]. +const PAGE_POOL_LIST_SHIFT: u32 = 7; +/// Bit position of the free-page count in [`PagePoolDescriptor::counters`]. +const PAGE_POOL_FREE_PAGES_SHIFT: u32 = 33; + +/// One Page-pool descriptor, indexed by buffer slot. +#[repr(C)] +pub(crate) struct PagePoolDescriptor { + /// Unit size and page-list address. + pub(crate) page_list: u64, + /// Pool counters. + pub(crate) counters: u64, + /// Pages in the pool. + pub(crate) page_count: u64, + /// Firmware-owned. + pub(crate) unk_18: u64, +} + +// SAFETY: `PagePoolDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for PagePoolDescriptor {} + +static_assert!(size_of::() == 0x20); + +impl PagePoolDescriptor { + /// Complete row for a physical compute queue's independent 21-block pool. + pub(crate) const fn compute(page_list_va: u64) -> Self { + let pages = 21 * FREE_LIST_BLOCK_PAGES as u64; + Self::new(page_list_va, pages << PAGE_POOL_FREE_PAGES_SHIFT, pages) + } + + const fn new(page_list_va: u64, counters: u64, page_count: u64) -> Self { + Self { + page_list: (PAGE_POOL_UNIT << PAGE_POOL_UNIT_SHIFT) + | (page_list_va >> PAGE_POOL_LIST_SHIFT), + counters, + page_count, + unk_18: 0, + } + } +} + +/// page-pool descriptor table, one page. +#[repr(C)] +pub(crate) struct PagePoolDescriptorTable { + /// Descriptors. + pub(crate) entries: [PagePoolDescriptor; PAGE_SIZE / size_of::()], +} + +// SAFETY: `PagePoolDescriptorTable` consists of integers only. +unsafe impl Zeroable for PagePoolDescriptorTable {} + +impl PagePoolDescriptorTable { + /// Required counters and page count of slot 0 at boot. + const RENDER_COUNTERS: u64 = 0x3d40_0000_3400; + const RENDER_PAGES: u64 = 0x1d00; + + /// Initializes the table: slot 0 describes the render free list's page + /// list, slot 1 the compute page list. + pub(crate) fn init(&mut self, render_page_list_va: u64, compute_page_list_va: u64) { + clear(self); + self.entries[0] = PagePoolDescriptor::new( + render_page_list_va, + Self::RENDER_COUNTERS, + Self::RENDER_PAGES, + ); + self.entries[1] = PagePoolDescriptor::compute(compute_page_list_va); + } +} + +// --------------------------------------------------------------------------- +// Index table +// --------------------------------------------------------------------------- + +/// Groups of the index table: first index and number of groups. +const INDEX_GROUPS: [(u32, usize); 2] = [(0x11, 6), (0x3c, 2)]; +/// Indices per group. +const INDEX_GROUP_SIZE: usize = 4; +/// Distance between the first indices of consecutive groups. +const INDEX_GROUP_STRIDE: u32 = 5; +/// Entries of the index table. +const INDEX_ENTRIES: usize = (INDEX_GROUPS[0].1 + INDEX_GROUPS[1].1) * INDEX_GROUP_SIZE; + +/// Index table at the start of the index object. +#[repr(C)] +pub(crate) struct IndexTable { + /// Groups of consecutive indices. + pub(crate) index: [u32; INDEX_ENTRIES], + pub(crate) unk_80: [u8; PAGE_SIZE - INDEX_ENTRIES * size_of::()], +} + +static_assert!(size_of::() == PAGE_SIZE); + +// SAFETY: `IndexTable` consists of integers only. +unsafe impl Zeroable for IndexTable {} + +impl IndexTable { + /// Initializes the table. + pub(crate) fn init(&mut self) { + clear(self); + let mut entries = self.index.iter_mut(); + for (first, groups) in INDEX_GROUPS { + for group in 0..groups as u32 { + for member in 0..INDEX_GROUP_SIZE as u32 { + if let Some(entry) = entries.next() { + *entry = first + group * INDEX_GROUP_STRIDE + member; + } + } + } + } + } +} + +/// Device-global compute state. Activation writes only the named fields, once, before the +/// first compute queue is exposed. The padding may contain firmware-owned state. +#[repr(C)] +pub(crate) struct ComputeGlobalState { + pub(crate) unk_0: u8, + unk_1: [u8; 7], + pub(crate) unk_8: u8, + unk_9: [u8; 7], + pub(crate) unk_10: u16, + unk_12: [u8; 6], + pub(crate) unk_18: u16, + unk_1a: [u8; 0x3ee], + pub(crate) unk_408: u32, + pub(crate) unk_40c: u32, + unk_410: [u8; 0x1f4], + pub(crate) unk_604: u32, + unk_608: [u8; 0x200], + pub(crate) scheduler_va: u64, + unk_810: [u8; 0x3f4], + pub(crate) unk_c04: u32, + unk_c08: [u8; 0x1f8], + pub(crate) unk_e00: u64, + pub(crate) unk_e08: u64, + unk_e10: [u8; 0x10], + pub(crate) unk_e20: u64, + pub(crate) unk_e28: u64, +} +static_assert!(core::mem::offset_of!(ComputeGlobalState, unk_408) == 0x408); +static_assert!(core::mem::offset_of!(ComputeGlobalState, scheduler_va) == 0x808); +static_assert!(core::mem::offset_of!(ComputeGlobalState, unk_c04) == 0xc04); +static_assert!(core::mem::offset_of!(ComputeGlobalState, unk_e00) == 0xe00); +static_assert!(size_of::() == 0xe30); + +/// Host-owned recovery report and progress records within the bundle views. +pub(crate) mod recovery { + pub(crate) const REPORT_VIEW: usize = 3; + pub(crate) const REASON: usize = 0x4c; + pub(crate) const PROGRESS_VIEW: usize = 1; + pub(crate) const PROGRESS_START: usize = 0xa40; + pub(crate) const PROGRESS_STRIDE: usize = 0x18; + pub(crate) const VIEW_EXTENTS: [usize; 5] = [0x18c0, 0x0c80, 0x3c00, 0x3380, 0x3300]; +} diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs new file mode 100644 index 0000000000000..c8b7d3f0dce3a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -0,0 +1,539 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Kick-queue entries and the objects they reference. +//! +//! Every hardware queue (QID) owns a kick ring of [`KICK_SLOTS`] entries spaced +//! [`KICK_ENTRY_STRIDE`] bytes apart. A [`KickTimestamp`] names one entry: its +//! low eight bits select the ring slot. To kick, the host writes the complete +//! slot selected by the queue's current timestamp, issues a barrier, advances +//! the timestamp and then announces the entry with a +//! [`super::queue::KickAnnounce`] record. +//! +//! An entry points at a work descriptor and its queue record, lists the kicks +//! it depends on, binds up to four [`RegisterArray`]s embedded in the +//! descriptor and may name a table of [`McacheRange`]s. +//! +//! Constructors of plain values ([`KickTimestamp::new`], +//! [`KickDependency::new`], [`KickDependency::from_word`]) return `None` for +//! values outside their domain; builders of firmware records and of encoded +//! words return `EINVAL`. + +use kernel::prelude::*; + +use super::queue::QUEUE_PRIORITY; + +/// Number of entries in a kick ring. +pub(crate) const KICK_SLOTS: usize = 256; +/// Distance between consecutive kick ring entries. +pub(crate) const KICK_ENTRY_STRIDE: usize = 0x200; +/// Size of a kick ring. +pub(crate) const KICK_RING_SIZE: usize = KICK_SLOTS * KICK_ENTRY_STRIDE; +/// Dependencies one entry can carry: explicit barriers plus the implicit parent. +pub(crate) const KICK_DEPENDENCIES: usize = 32; +/// Explicit barriers one entry can carry. +pub(crate) const KICK_BARRIERS_MAX: usize = KICK_DEPENDENCIES - 1; +/// Highest hardware queue ID. +pub(crate) const QID_MAX: u8 = 0x7f; +/// Highest UAT context ID. +pub(crate) const CONTEXT_MAX: u8 = 0x3f; + +const KICK_TIMESTAMP_BITS: u32 = 40; +const KICK_TIMESTAMP_MASK: u64 = (1 << KICK_TIMESTAMP_BITS) - 1; +const KICK_SLOT_MASK: u64 = KICK_SLOTS as u64 - 1; +/// GPU addresses packed into kick entries are 32-byte aligned and 43 bits wide. +const PACKED_VA_SHIFT: u32 = 5; +const PACKED_VA_LIMIT: u64 = 1 << (38 + PACKED_VA_SHIFT); + +/// Packs a 32-byte aligned GPU address into the low 38 bits of a word. +fn packed_va(va: u64) -> Result { + if va & ((1 << PACKED_VA_SHIFT) - 1) != 0 || va >= PACKED_VA_LIMIT { + return Err(EINVAL); + } + Ok(va >> PACKED_VA_SHIFT) +} + +/// A kick timestamp. +/// +/// The value is a 40-bit counter that advances by one per kick of a queue; +/// bits 7:0 select the kick ring slot. A queue's first kick uses +/// [`KickTimestamp::FIRST`]. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct KickTimestamp(u64); + +impl KickTimestamp { + /// The timestamp preceding a queue's first kick. + pub(crate) const ZERO: Self = Self(0); + /// The timestamp of a queue's first kick. + pub(crate) const FIRST: Self = Self(1); + + /// Returns the timestamp with raw value `value`, if it fits 40 bits. + pub(crate) const fn new(value: u64) -> Option { + if value & !KICK_TIMESTAMP_MASK != 0 { + None + } else { + Some(Self(value)) + } + } + + /// Returns the raw 40-bit value. + pub(crate) const fn get(self) -> u64 { + self.0 + } + + /// Returns the kick ring slot this timestamp selects. + pub(crate) const fn slot(self) -> usize { + (self.0 & KICK_SLOT_MASK) as usize + } + + /// Returns the byte offset of the selected entry within the kick ring. + pub(crate) const fn entry_offset(self) -> usize { + self.slot() * KICK_ENTRY_STRIDE + } + + /// Returns the timestamp of the following kick. + pub(crate) const fn next(self) -> Self { + Self(self.0.wrapping_add(1) & KICK_TIMESTAMP_MASK) + } + + /// Returns the timestamp of the preceding kick. + pub(crate) const fn prev(self) -> Self { + Self(self.0.wrapping_sub(1) & KICK_TIMESTAMP_MASK) + } + + /// Returns the completion value of this kick: the timestamp in bits 39:0 + /// and `pass_seq`, the low byte of the logical queue's render-pass count, + /// in bits 47:40. + pub(crate) const fn completion_value(self, pass_seq: u8) -> u64 { + self.0 | (pass_seq as u64) << KICK_TIMESTAMP_BITS + } +} + +/// A dependency on the completion of one kick. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct KickDependency { + qid: u8, + timestamp: KickTimestamp, +} + +impl KickDependency { + const QID_SHIFT: u32 = KICK_TIMESTAMP_BITS; + + /// Initial value for unused entries in a host-side dependency array. + pub(crate) const ZERO: Self = Self { + qid: 0, + timestamp: KickTimestamp::ZERO, + }; + + /// Returns the dependency on kick `timestamp` of queue `qid`. + pub(crate) const fn new(qid: u8, timestamp: KickTimestamp) -> Option { + if qid > QID_MAX { + None + } else { + Some(Self { qid, timestamp }) + } + } + + /// Returns the queue that was kicked. + pub(crate) const fn qid(self) -> u8 { + self.qid + } + + /// Returns the dependency word: the timestamp in bits 39:0 and the QID in + /// bits 46:40. + pub(crate) const fn word(self) -> u64 { + self.timestamp.0 | (self.qid as u64) << Self::QID_SHIFT + } + + /// Decodes a dependency word; bits 63:47 must be clear. + pub(crate) const fn from_word(word: u64) -> Option { + if word >> (Self::QID_SHIFT + 7) != 0 { + return None; + } + Some(Self { + qid: (word >> Self::QID_SHIFT) as u8, + timestamp: KickTimestamp(word & KICK_TIMESTAMP_MASK), + }) + } +} + +/// One register write of a [`RegisterArray`]. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct RegisterWrite { + /// Register number. + pub(crate) reg: u32, + /// Value written to the register. + pub(crate) value: u64, +} + +impl RegisterWrite { + /// Returns the write of `value` to register `reg`. + pub(crate) const fn new(reg: u32, value: u64) -> Self { + Self { reg, value } + } +} + +/// A buffer that work writes. +#[derive(Copy, Clone, Debug)] +pub(crate) struct WriteRange { + /// GPU address of the buffer. + pub(crate) va: u64, + /// Size of the buffer in bytes. + pub(crate) size: u64, +} + +/// Class of an [`McacheRange`]. +/// +/// The auxiliary framebuffer range takes class 2 and attachment ranges take +/// class 1. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum McacheClass { + /// A render-target attachment. + Attachment = 1, + /// The auxiliary framebuffer. + AuxFramebuffer = 2, +} + +/// One MCache range: a GPU address range that work writes, in 128-byte cache +/// lines. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct McacheRange { + /// First cache line in bits 40:0, UAT context in bits 46:41. + pub(crate) start: u64, + /// End cache line (exclusive) in bits 40:0 and the class above. + pub(crate) end: u64, +} + +impl McacheRange { + const LINE_SIZE: u64 = 0x80; + /// Ranges cover fewer than 2^32 cache lines (512 GiB). + const LINES_MAX: u64 = u32::MAX as u64; + const LINE_LIMIT: u64 = 1 << 41; + const CONTEXT_SHIFT: u32 = 41; + /// Bits 55:47 of the start word hold 0x118 in every range. + const UNK_START: u64 = 0x118 << 47; + /// Bit 41 of the end word is set in every range. + const UNK_END: u64 = 1 << 41; + const CLASS_SHIFT: u32 = 42; + + /// Encodes the cache lines `range` touches, translated in UAT context + /// `context`. + pub(crate) fn new(range: &WriteRange, class: McacheClass, context: u8) -> Result { + if range.size == 0 || context > CONTEXT_MAX { + return Err(EINVAL); + } + let start = range.va / Self::LINE_SIZE; + let end = range + .va + .checked_add(range.size) + .ok_or(EINVAL)? + .div_ceil(Self::LINE_SIZE); + if end >= Self::LINE_LIMIT || end - start > Self::LINES_MAX { + return Err(EINVAL); + } + Ok(Self { + start: start | (context as u64) << Self::CONTEXT_SHIFT | Self::UNK_START, + end: end | Self::UNK_END | 1 << (Self::CLASS_SHIFT + class as u32), + }) + } +} + +/// Register-write entries a [`RegisterArray`] holds. +pub(crate) const REGISTER_ARRAY_ENTRIES: usize = 128; +/// MCache ranges that fit in the tail of a [`RegisterArray`]. +pub(crate) const MCACHE_INLINE_RANGES: usize = 16; + +/// A register array embedded in a work descriptor and bound to its kick entry +/// by a [`RegisterArrayBinding`]. +/// +/// The array ends in a trailer that points back at the array and holds its +/// size. It is only written as part of a cleared descriptor: [`Self::set`] +/// leaves the unused entries and the MCache space untouched. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct RegisterArray { + /// Register writes, in order. + pub(crate) entries: [RegisterWrite; REGISTER_ARRAY_ENTRIES], + /// Space after the entries. The first fragment array keeps its MCache + /// table here when the table fits; it is zero everywhere else. + pub(crate) mcache: [McacheRange; MCACHE_INLINE_RANGES], + /// GPU address of this array, as seen by the work's VM. + pub(crate) self_va: u64, + /// Size of the used entries in bytes (bits 31:16) and their count (15:0). + pub(crate) size: u32, + pub(crate) unk_70c: [u8; 0x14], +} + +impl RegisterArray { + /// Stores `writes` and the trailer into a zeroed array. + pub(crate) fn set(&mut self, self_va: u64, writes: &[RegisterWrite]) -> Result { + let count = writes.len(); + if count > REGISTER_ARRAY_ENTRIES { + return Err(EINVAL); + } + self.entries[..count].copy_from_slice(writes); + self.self_va = self_va; + self.size = (size_of_val(writes) as u32) << 16 | count as u32; + Ok(()) + } +} + +/// Binding of one [`RegisterArray`] to a kick entry. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct RegisterArrayBinding { + word: u64, +} + +impl RegisterArrayBinding { + const CHECKPOINT_SHIFT: u32 = 43; + const CHECKPOINT_MAX: u8 = 0x7f; + + /// Binds the array at `array_va`, as seen by the work's VM, whose first + /// checkpoint is after `checkpoint` entries. + pub(crate) fn new(array_va: u64, checkpoint: u8) -> Result { + if checkpoint > Self::CHECKPOINT_MAX { + return Err(EINVAL); + } + Ok(Self { + word: packed_va(array_va)? | (checkpoint as u64) << Self::CHECKPOINT_SHIFT, + }) + } + + fn word(binding: Option) -> u64 { + binding.map_or(0, |binding| binding.word) + } +} + +/// Location of the MCache table a kick entry names. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct McacheTable { + va: u64, + count: u8, +} + +impl McacheTable { + const VA_SHIFT: u32 = 6; + const COUNT_SHIFT: u32 = 54; + const COUNT_MAX: usize = 64; + + /// Names the `count` ranges at `va`, as seen by the work's VM. + pub(crate) fn new(va: u64, count: usize) -> Result { + if count == 0 || count > Self::COUNT_MAX { + return Err(EINVAL); + } + packed_va(va)?; + Ok(Self { + va, + count: count as u8, + }) + } + + fn word(self) -> u64 { + (self.va >> PACKED_VA_SHIFT) << Self::VA_SHIFT + | ((self.count - 1) as u64) << Self::COUNT_SHIFT + } +} + +/// Quality-of-service identity of the kicked queue. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct KickQos { + /// QoS buffer slot of the logical queue. + pub(crate) slot: u8, + /// Scheduling class of the engine, at most 0x1f. + pub(crate) class: u8, +} + +/// Inputs of one kick entry. +pub(crate) struct KickArgs<'a> { + /// Kicked queue. + pub(crate) qid: u8, + /// Timestamp of this kick; it selects the ring slot. + pub(crate) timestamp: KickTimestamp, + /// Kick this kick implicitly depends on: the previous kick of the same + /// queue. A queue's first kick names `timestamp.prev()` on render queues + /// and [`KickTimestamp::ZERO`] on compute queues. + pub(crate) parent: KickTimestamp, + /// Kicks of other queues this kick must wait for. + pub(crate) barriers: &'a [KickDependency], + /// GPU address of the work descriptor (firmware alias). + pub(crate) descriptor_va: u64, + /// GPU address of the queue record. + pub(crate) queue_va: u64, + /// Queue QoS identity. + pub(crate) qos: KickQos, + /// MCache table of the work, if any. + pub(crate) mcache: Option, + /// Engine-specific event mask. + pub(crate) event_mask: [u64; 4], + /// Register arrays bound to the kick. + pub(crate) register_arrays: [Option; 4], +} + +/// One kick ring entry. +/// +/// Live state: none. The host writes the whole entry before announcing it and +/// does not touch it again until the slot is reused. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct KickEntry { + /// Timestamp in bits 41:2, QID in bits 48:42, MCache table present in bit + /// 58 and queue priority in bits 60:59. + pub(crate) header: u64, + /// MCache table: address >> 5 in bits 43:6, count - 1 in bits 59:54. + pub(crate) mcache: u64, + /// GPU address of the work descriptor. + pub(crate) descriptor_va: u64, + /// GPU address of the queue record. + pub(crate) queue_va: u64, + /// Valid-dependency mask in bits 31:0, QoS slot in bits 39:32, class in + /// bits 47:40; bits 63:48 are all ones. + pub(crate) dependency_control: u64, + /// Explicit barriers followed by the implicit parent. + pub(crate) dependencies: [u64; KICK_DEPENDENCIES], + /// Engine-specific event mask. + pub(crate) event_mask: [u64; 4], + pub(crate) unk_148: u64, + /// Register-array bindings: address >> 5 in bits 37:0, checkpoint in bits + /// 49:43. + pub(crate) register_arrays: [u64; 4], + pub(crate) unk_170: u64, + pub(crate) unk_178: u64, + pub(crate) pad_180: [u8; KICK_ENTRY_STRIDE - 0x180], +} + +static_assert!(size_of::() == KICK_ENTRY_STRIDE); +static_assert!(core::mem::offset_of!(KickEntry, dependency_control) == 0x20); +static_assert!(core::mem::offset_of!(KickEntry, event_mask) == 0x128); +static_assert!(core::mem::offset_of!(KickEntry, register_arrays) == 0x150); +static_assert!(core::mem::offset_of!(KickEntry, unk_178) == 0x178); +static_assert!(size_of::() == 0xc); +static_assert!(size_of::() == 0x10); +static_assert!(size_of::() == 0x720); +static_assert!(core::mem::offset_of!(RegisterArray, mcache) == 0x600); +static_assert!(core::mem::offset_of!(RegisterArray, self_va) == 0x700); + +impl KickEntry { + const TIMESTAMP_SHIFT: u32 = 2; + const QID_SHIFT: u32 = 42; + const MCACHE_PRESENT: u64 = 1 << 58; + const PRIORITY_SHIFT: u32 = 59; + const QOS_SLOT_SHIFT: u32 = 32; + const QOS_CLASS_SHIFT: u32 = 40; + const QOS_CLASS_MAX: u8 = 0x1f; + /// Bits 63:48 of the dependency control word. + const UNK_DEPENDENCY_CONTROL: u64 = 0xffff << 48; + /// Required value of the last word of the entry. + const UNK_178: u64 = 0x003f_ffff_ffff_ffff; + + /// Builds the entry described by `args`. + pub(crate) fn new(args: &KickArgs<'_>) -> Result { + let barriers = args.barriers.len(); + if args.qid > QID_MAX + || barriers > KICK_BARRIERS_MAX + || args.qos.class > Self::QOS_CLASS_MAX + { + return Err(EINVAL); + } + + let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT + | (args.qid as u64) << Self::QID_SHIFT + | (QUEUE_PRIORITY as u64) << Self::PRIORITY_SHIFT; + let mcache = match args.mcache { + Some(table) => { + header |= Self::MCACHE_PRESENT; + table.word() + } + None => 0, + }; + + let mut dependencies = [0; KICK_DEPENDENCIES]; + for (slot, barrier) in dependencies.iter_mut().zip(args.barriers) { + *slot = barrier.word(); + } + dependencies[barriers] = KickDependency { + qid: args.qid, + timestamp: args.parent, + } + .word(); + let count = barriers + 1; + let mask = if count == KICK_DEPENDENCIES { + u32::MAX + } else { + (1u32 << count) - 1 + }; + + Ok(Self { + header, + mcache, + descriptor_va: args.descriptor_va, + queue_va: args.queue_va, + dependency_control: mask as u64 + | (args.qos.slot as u64) << Self::QOS_SLOT_SHIFT + | (args.qos.class as u64) << Self::QOS_CLASS_SHIFT + | Self::UNK_DEPENDENCY_CONTROL, + dependencies, + event_mask: args.event_mask, + unk_148: 0, + register_arrays: args.register_arrays.map(RegisterArrayBinding::word), + unk_170: 0, + unk_178: Self::UNK_178, + pad_180: [0; KICK_ENTRY_STRIDE - 0x180], + }) + } +} + +/// The complete write-only installation of a fragment/tiling queue pair. +/// Firmware must see both queues configured before either outer command. +pub(crate) struct RenderRegistration { + writes: [(usize, u64); 8], +} + +impl RenderRegistration { + const TAG_PORT: usize = 0x4090; + const DATA_PORT: usize = 0x4098; + const VALID_LOW: usize = 0x21068; + const VALID_HIGH: usize = 0x21070; + const CONFIG_TAG: u64 = 0x0000_0400_0002_1008; + const ADDRESS_MASK: u64 = 0x0000_07ff_ffff_ffe0; + + /// Validates both queues without touching MMIO. The resulting transaction + /// writes fragment configuration and validity before tiling configuration + /// and validity. Each validity mask is stored high before low. + pub(crate) fn new(fragment: (u8, u64), tiling: (u8, u64)) -> Result { + if fragment.0 == tiling.0 { + return Err(EINVAL); + } + let mut writes = [(0, 0); 8]; + for (slot, ((qid, va), engine)) in [ + (fragment, super::queue::DataMaster::Fragment), + (tiling, super::queue::DataMaster::Tiling), + ] + .into_iter() + .enumerate() + { + if qid > QID_MAX || va & !Self::ADDRESS_MASK != 0 { + return Err(EINVAL); + } + let completion = (3 * (4 - u64::from(QUEUE_PRIORITY)) + 0x3fe).wrapping_shl(54); + let payload = ((engine as u64 + 1) << 60) | va | completion; + let (low, high) = if qid < 64 { + (1u64 << qid, 0) + } else { + (0, 1u64 << (qid - 64)) + }; + writes[slot * 4..slot * 4 + 4].copy_from_slice(&[ + (Self::TAG_PORT, Self::CONFIG_TAG | (u64::from(qid) << 43)), + (Self::DATA_PORT, payload), + (Self::VALID_HIGH, high), + (Self::VALID_LOW, low), + ]); + } + Ok(Self { writes }) + } + + pub(crate) fn writes(&self) -> &[(usize, u64); 8] { + &self.writes + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs new file mode 100644 index 0000000000000..4f0ae8aa253ed --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 firmware structures. +//! +//! Layouts of the records and descriptors the host shares with the G17 +//! firmware, and the pure builders that fill them. Nothing here allocates, +//! locks or publishes: the owners of the rings and queues do that. +//! +//! This module also holds the few rules shared by the render and compute +//! work descriptors. + +use kernel::prelude::*; + +pub(crate) mod buffer; +pub(crate) mod channels; +pub(crate) mod compute; +pub(crate) mod context; +pub(crate) mod freelist; +pub(crate) mod initdata; +pub(crate) mod kick; +pub(crate) mod queue; +pub(crate) mod render; +pub(crate) mod qos; + +/// Clears a firmware structure in place. +/// +/// Work descriptors are several KiB large and must not be built on the stack. +pub(crate) fn clear(object: &mut T) { + // SAFETY: `T: Zeroable` guarantees that all-zero bytes are a valid `T`, + // and `object` is a valid, exclusive reference to one `T`. + unsafe { core::ptr::write_bytes(object as *mut T, 0, 1) }; +} + +/// Returns `va + offset`, or `EINVAL` if the sum overflows. +pub(crate) fn offset_va(va: u64, offset: u64) -> Result { + va.checked_add(offset).ok_or(EINVAL) +} + +/// Size of one slot of a start/end timestamp pair. +const TIMESTAMP_SIZE: u64 = 8; + +/// Returns the address of the end slot of the timestamp pair at `start_va`. +/// +/// A work item's start and end timestamps are two consecutive 64-bit slots. +pub(crate) fn timestamp_end_va(start_va: u64) -> Result { + offset_va(start_va, TIMESTAMP_SIZE) +} + +/// Returns the work key registers take: the UAT context in bits 15:8 and the +/// low six bits of the work-state word below. +pub(crate) const fn work_key(context_id: u16, state_word: u32) -> u64 { + (context_id as u64) << 8 | (state_word & 0x3f) as u64 +} + +/// Returns the sampler bound descriptors store next to the sampler count: the +/// count plus one, or zero without samplers. +pub(crate) fn sampler_max(count: u32) -> Result { + match count { + 0 => Ok(0), + count => count.checked_add(1).ok_or(EINVAL), + } +} + +/// Returns the completion stamp value of sequence number `sequence`: the +/// value the firmware stores to a queue's completion stamp word when the work +/// completes. +pub(crate) const fn stamp_value(sequence: u32) -> u32 { + sequence.wrapping_shl(8) +} + +/// Object IDs of a work item, written to three registers of its engine and +/// to its descriptor. +/// +/// A render pass takes `predecessor` from its queue pair and a newly +/// allocated `current`; compute work uses fixed IDs. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct ObjectIds { + /// Predecessor object ID. + pub(crate) predecessor: u32, + /// Current object ID. + pub(crate) current: u32, +} + +impl ObjectIds { + /// Returns the register value: `predecessor` in bits 63:32 and `current` + /// in bits 31:0. + pub(crate) const fn word(self) -> u64 { + (self.predecessor as u64) << 32 | self.current as u64 + } +} + +/// Required value of the per-engine registers 0x0a599 (compute), 0x0a5a1 +/// (tiling) and 0x0a5a9 (fragment). +pub(crate) const UNK_0A599: u64 = 0x0000_0060_0040_0020; +/// Required value of the per-engine registers 0x0d411 (compute), 0x0d419 +/// (tiling) and 0x0d429 (fragment). +pub(crate) const UNK_0D411: u64 = 0x0000_0002_0000_0001; +/// Required value of the per-engine registers 0x101d9 (compute), 0x101e1 +/// (tiling) and 0x101e9 (fragment). +pub(crate) const UNK_101D9: u64 = 0x1c; +/// Required value of the per-engine registers 0x1a0e9 (compute), 0x1a0f1 +/// (tiling) and 0x1a0f9 (fragment). +pub(crate) const UNK_1A0E9: u64 = 8; +/// Required value of the per-engine registers 0x107a1 (compute) and 0x10799 +/// (tiling). The fragment register 0x10791 also sets bit 9. +pub(crate) const UNK_10791: u64 = 0xff_0000; +/// Required value of the tiling register 0x1c8f8 and the fragment register +/// 0x100b8; the compute register 0x1a458 holds it in its low bits. +pub(crate) const UNK_1C8F8: u64 = 0x8860; diff --git a/drivers/gpu/drm/asahi/g17/fw/qos.rs b/drivers/gpu/drm/asahi/g17/fw/qos.rs new file mode 100644 index 0000000000000..d939e012714ee --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/qos.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! QoS accounting object. Queue rows and submission counts use distinct queue +//! and logical-owner indices. Firmware may observe each live word separately. + +use kernel::prelude::*; + +pub(crate) const QUEUES: usize = 128; +pub(crate) const DATA_MASTERS: usize = 4; + +#[repr(C)] +pub(crate) struct Qos { + pub(crate) rows: [[u8; 8]; QUEUES], + pub(crate) queue_submitted: [u32; QUEUES], + pub(crate) owner_submitted: [u32; QUEUES], + pub(crate) scheduler: [u64; QUEUES], + pub(crate) share: [u32; QUEUES], + pub(crate) intervals: u64, + pub(crate) busy_start: [u64; 3], + pub(crate) dm_submitted: [u64; DATA_MASTERS], +} +// SAFETY: Qos consists entirely of integer fields and arrays. +unsafe impl Zeroable for Qos {} + +const _: () = { + assert!(core::mem::size_of::() == 0xe40); + assert!(core::mem::offset_of!(Qos, queue_submitted) == 0x400); + assert!(core::mem::offset_of!(Qos, owner_submitted) == 0x600); + assert!(core::mem::offset_of!(Qos, scheduler) == 0x800); + assert!(core::mem::offset_of!(Qos, share) == 0xc00); + assert!(core::mem::offset_of!(Qos, intervals) == 0xe00); + assert!(core::mem::offset_of!(Qos, busy_start) == 0xe08); + assert!(core::mem::offset_of!(Qos, dm_submitted) == 0xe20); +}; diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs new file mode 100644 index 0000000000000..db95ba99e0189 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -0,0 +1,540 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Queue records and the commands of a queue's item ring. +//! +//! A hardware queue is described by a [`QueueRecord`]. Its [`PointerBlock`] +//! holds the cursors of the queue's item ring, an array of [`ITEM_RING_ENTRIES`] +//! work pointers: 64-bit GPU addresses of commands. Every command starts with +//! a 32-bit [`CommandTag`]: work descriptors carry the tiling, fragment and +//! compute tags (see `render.rs` and `compute.rs`), and the per-kick records +//! defined here carry the others. +//! +//! One kick appends, in order, the work descriptor, a [`QueueConfig`], a +//! [`KickAnnounce`] record and, when a compute queue is first installed or +//! explicitly waits on another queue, a [`KickPredecessor`]. +//! Compute kicks always carry the [`QueueConfig`]. A render kick carries it +//! only when it installs the queue, updates the context or QoS identity, +//! carries a completion seed, or configures the queue differently from the +//! last [`QueueConfig`] published on it. + +use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::prelude::*; + +use super::kick::{KickTimestamp, QID_MAX}; + +/// Priority of every queue the driver creates. +/// +/// The same value appears in the queue record, [`QueueConfig`], +/// [`KickAnnounce`] and the kick entry header; the work rings it selects are +/// listed in `channels.rs`. +pub(crate) const QUEUE_PRIORITY: u8 = 2; +/// Scheduling policy of every queue the driver creates. +pub(crate) const QUEUE_POLICY: u16 = 2; +/// Number of work pointers in a queue's item ring. +pub(crate) const ITEM_RING_ENTRIES: u32 = 0x500; + +/// Tag in the first word of every command. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum CommandTag { + /// Tiling (TA) work descriptor. + Tiling = 0, + /// Fragment (3D) work descriptor. + Fragment = 1, + /// Compute work descriptor. + Compute = 3, + /// [`KickAnnounce`]. + KickAnnounce = 0x0e, + /// [`QueueConfig`]. + QueueConfig = 0x0f, + /// [`KickPredecessor`]. + KickPredecessor = 0x10, +} + +/// The engine that executes a queue's work. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum DataMaster { + /// Tiling (TA). + Tiling = 0, + /// Fragment (3D). + Fragment = 1, + /// Compute (CL). + Compute = 2, +} + +/// Cursors of a queue's item ring. +/// +/// The firmware writes `done` and `unk_30`. The host writes the block on +/// three paths, never with a structure copy once the queue was published: +/// +/// - creation: the whole block, [`PointerBlock::new`], before the queue is +/// first published; +/// - publication: `producer`, with one aligned 32-bit store per publication +/// step, after a barrier that orders the work pointers and commands before +/// it; +/// - reuse of a render queue for a new owner: [`PointerBlock::rebase`]. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct PointerBlock { + /// Item-ring position up to which the firmware has finished commands. + /// Free ring space is measured from it, so it bounds `producer`. + pub(crate) done: AtomicU32, + pub(crate) unk_04: [u32; 11], + /// Written by the firmware. The host treats the queue as idle only when + /// this cursor, `done` and `producer` are equal. + pub(crate) unk_30: AtomicU32, + pub(crate) unk_34: [u32; 3], + /// Item-ring position after the last work pointer the host published. + pub(crate) producer: AtomicU32, + pub(crate) unk_44: [u32; 3], + /// On recovery, the host copies producer here before acknowledging resume. + pub(crate) recovery_target: u32, + pub(crate) unk_54: [u32; 3], + /// Number of entries in the item ring. + pub(crate) entries: u32, + pub(crate) unk_64: [u32; 7], +} + +static_assert!(size_of::() == 0x80); +static_assert!(core::mem::offset_of!(PointerBlock, unk_30) == 0x30); +static_assert!(core::mem::offset_of!(PointerBlock, producer) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, entries) == 0x60); + +// SAFETY: PointerBlock contains only integers and integer atomics. +unsafe impl Zeroable for PointerBlock {} + +impl PointerBlock { + pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); + pub(crate) const RECOVERY_TARGET: usize = core::mem::offset_of!(Self, recovery_target); + + /// Returns the cursors of an empty item ring. + pub(crate) const fn new() -> Self { + Self { + done: AtomicU32::new(0), + unk_04: [0; 11], + unk_30: AtomicU32::new(0), + unk_34: [0; 3], + producer: AtomicU32::new(0), + unk_44: [0; 3], + recovery_target: u32::MAX, + unk_54: [0; 3], + entries: ITEM_RING_ENTRIES, + unk_64: [0; 7], + } + } + + /// Moves the cursors of an empty queue to item-ring position `base`. + /// + /// Used when a render queue pair that was kicked before is handed to a new + /// owner, with `base` the position the host tracks for the new owner's + /// queues. Only valid while the firmware does not run the queue, with all + /// three cursors zero. Writes `done`, `unk_30` and `producer`, in this + /// order, with single stores; the caller issues a full barrier afterwards. + pub(crate) fn rebase(&self, base: u32) { + self.done.store(base, Ordering::Relaxed); + self.unk_30.store(base, Ordering::Relaxed); + self.producer.store(base, Ordering::Relaxed); + } +} + +/// Head of a queue's job list. +/// +/// Live state: the firmware links and unlinks jobs; the host writes the head +/// only at queue creation. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct JobListHead { + /// GPU address of the first job, or zero. + pub(crate) first_va: u64, + /// GPU address of the last link: the head itself when the list is empty. + pub(crate) last_va: u64, +} + +impl JobListHead { + /// Returns an empty list whose head is at `self_va`. + pub(crate) const fn new(self_va: u64) -> Self { + Self { + first_va: 0, + last_va: self_va, + } + } +} + +/// Addresses and owner a [`QueueRecord`] names. +pub(crate) struct QueueRecordArgs { + /// GPU address of the queue's [`PointerBlock`]. + pub(crate) pointers_va: u64, + /// GPU address of the queue's item ring. + pub(crate) items_va: u64, + /// GPU address of the job list the queue's work is linked into. + pub(crate) job_list_va: u64, + /// GPU address of the owning logical queue's scheduler-state page. + pub(crate) scheduler_va: u64, + /// Thread-group ID of the owning process (0 while unowned). + pub(crate) owner_pid: u32, +} + +/// Description of one hardware queue, named by its work-ring slots. +/// +/// The host writes it whole at queue creation and changes the owner fields of +/// an idle queue with [`QueueRecord::set_owner`] or +/// [`QueueRecord::set_compute_owner`]. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct QueueRecord { + /// GPU address of the [`PointerBlock`]. + pub(crate) pointers_va: u64, + /// GPU address of the item ring. + pub(crate) items_va: u64, + /// GPU address of the job list. + pub(crate) job_list_va: u64, + pub(crate) unk_18: [u8; 0xc], + pub(crate) unk_24: u32, + /// Queue priority, stored twice. + pub(crate) priority: [u32; 2], + pub(crate) unk_30: [u8; 6], + pub(crate) unk_36: u16, + /// Zero at creation; cleared again when a compute queue changes owner. + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + /// Scheduling policy. + pub(crate) policy: u32, + pub(crate) unk_44: u32, + /// Thread-group ID of the owning process. + pub(crate) owner_pid: u32, + pub(crate) unk_4c: [u8; 0x50], + /// GPU address of the owner's scheduler-state page. + pub(crate) scheduler_va: u64, + pub(crate) unk_a4: [u8; 0x1c], +} + +static_assert!(size_of::() == 0xc0); +static_assert!(core::mem::offset_of!(QueueRecord, priority) == 0x28); +static_assert!(core::mem::offset_of!(QueueRecord, unk_38) == 0x38); +static_assert!(core::mem::offset_of!(QueueRecord, policy) == 0x40); +static_assert!(core::mem::offset_of!(QueueRecord, scheduler_va) == 0x9c); + +impl QueueRecord { + /// Builds the record of a newly created queue. + pub(crate) fn new(args: &QueueRecordArgs) -> Self { + Self { + pointers_va: args.pointers_va, + items_va: args.items_va, + job_list_va: args.job_list_va, + unk_18: [0; 0xc], + unk_24: u32::MAX, + priority: [QUEUE_PRIORITY as u32; 2], + unk_30: [0; 6], + unk_36: u16::MAX, + unk_38: 0, + unk_3c: 0, + policy: QUEUE_POLICY as u32, + unk_44: u32::MAX, + owner_pid: args.owner_pid, + unk_4c: [0; 0x50], + scheduler_va: args.scheduler_va, + unk_a4: [0; 0x1c], + } + } + + /// Hands an idle compute queue to a new owner. + /// + /// Replaces the job head, process identity and scheduler page, and clears `unk_38`. + pub(crate) fn set_compute_owner( + &mut self, + job_list_va: u64, + scheduler_va: u64, + owner_pid: u32, + ) { + self.job_list_va = job_list_va; + self.unk_38 = 0; + self.owner_pid = owner_pid; + self.scheduler_va = scheduler_va; + } +} + +/// Free list a queue's USC private memory is allocated from. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct FreeListBinding { + /// GPU address of the free list's shared control object. + pub(crate) va: u64, + /// Generation of the free list's registration. + pub(crate) generation: u64, + /// Buffer slot of the free list. + pub(crate) slot: u32, +} + +/// Parameter-buffer binding of a tiling queue. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct TilingBinding { + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// Allocation token of the parameter buffer's buffer slot. + pub(crate) pb_token: u64, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + /// QID of the fragment queue paired with this tiling queue. + pub(crate) fragment_qid: u8, +} + +/// Queue whose configuration a [`QueueConfig`] carries. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum QueueConfigTarget { + /// A tiling queue and its parameter-buffer binding. + Tiling(TilingBinding), + /// A fragment queue. + Fragment, + /// A compute queue. + Compute, +} + +/// Inputs of one [`QueueConfig`]. +pub(crate) struct QueueConfigArgs { + /// Kind of queue. + pub(crate) target: QueueConfigTarget, + /// GPU address of the queue's kick ring. + pub(crate) kick_ring_va: u64, + /// Firmware alias of the queue's kick ring. + pub(crate) kick_ring_fw_va: u64, + /// Configured queue. + pub(crate) qid: u8, + /// The firmware has not yet installed this QID. + pub(crate) install: bool, + /// Completion value the firmware loads for the queue, or `None` to keep + /// the value it holds. + pub(crate) completion_seed: Option, + /// UAT context the queue's work runs in. + pub(crate) context_id: u16, + /// USC private-memory free list. + pub(crate) free_list: FreeListBinding, + /// GPU address of the owner's scheduler-state page. + pub(crate) scheduler_va: u64, + /// QoS buffer slot of the logical queue. + pub(crate) qos_slot: u8, + /// The QoS identity or queue membership changed since the last update. + pub(crate) qos_update: bool, + /// Thread-group ID of the owning process. + pub(crate) owner_pid: u32, + /// The queue's UAT context changed since the last update. + pub(crate) context_update: bool, +} + +/// Configuration of one hardware queue (tag 15). +/// +/// The firmware applies it when it consumes the command, which may be after +/// later kicks were published, so a record must not be rewritten while a +/// work pointer names it. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct QueueConfig { + /// [`CommandTag::QueueConfig`]. + pub(crate) tag: u32, + pub(crate) unk_04: u32, + /// GPU address of the queue's kick ring. + pub(crate) kick_ring_va: u64, + /// Firmware alias of the queue's kick ring. + pub(crate) kick_ring_fw_va: u64, + /// Configured queue. + pub(crate) qid: u16, + /// Non-zero to install the QID. + pub(crate) install: u16, + pub(crate) unk_1c: u16, + /// Queue priority. + pub(crate) priority: u16, + pub(crate) unk_20: u16, + /// [`DataMaster`] of the queue. + pub(crate) data_master: u16, + pub(crate) unk_24: u16, + /// Non-zero when `completion_seed` is valid. + pub(crate) completion_seed_valid: u32, + /// Completion value: kick timestamp in bits 39:0, render-pass sequence in + /// bits 47:40 (see [`KickTimestamp::completion_value`]). + pub(crate) completion_seed: u64, + /// UAT context of the queue's work. + pub(crate) context_id: u16, + pub(crate) unk_34: u16, + /// GPU address of the USC free list's shared control object. + pub(crate) free_list_va: u64, + /// Generation of the USC free list. + pub(crate) free_list_generation: u64, + /// Buffer slot of the USC free list. + pub(crate) free_list_slot: u32, + /// GPU address of the owner's scheduler-state page. + pub(crate) scheduler_va: u64, + /// Non-zero when the QoS identity or membership changed. + pub(crate) qos_update: u32, + /// QoS buffer slot. + pub(crate) qos_slot: u16, + pub(crate) unk_58: u16, + /// Thread-group ID of the owning process. + pub(crate) owner_pid: u32, + /// Scheduling policy. + pub(crate) policy: u16, + pub(crate) unk_60: u16, + /// Non-zero when the UAT context changed. + pub(crate) context_update: u16, + pub(crate) unk_64: u16, + pub(crate) unk_66: u16, + pub(crate) unk_68: [u8; 6], + /// Tiling queues: GPU address of the parameter-buffer state; else zero. + pub(crate) pb_va: u64, + /// Tiling queues: parameter-buffer slot allocation token; else all ones. + pub(crate) pb_token: u64, + /// Tiling queues: parameter-buffer slot; else all ones. + pub(crate) pb_slot: u32, + /// Tiling queues: QID of the paired fragment queue; else all ones. + pub(crate) fragment_qid: u32, + pub(crate) unk_86: [u8; 0x3a], +} + +static_assert!(size_of::() == 0xc0); +static_assert!(core::mem::offset_of!(QueueConfig, completion_seed_valid) == 0x26); +static_assert!(core::mem::offset_of!(QueueConfig, free_list_va) == 0x36); +static_assert!(core::mem::offset_of!(QueueConfig, scheduler_va) == 0x4a); +static_assert!(core::mem::offset_of!(QueueConfig, qos_slot) == 0x56); +static_assert!(core::mem::offset_of!(QueueConfig, owner_pid) == 0x5a); +static_assert!(core::mem::offset_of!(QueueConfig, pb_va) == 0x6e); +static_assert!(core::mem::offset_of!(QueueConfig, fragment_qid) == 0x82); + +impl QueueConfig { + const UNK_66: u16 = 1; + + /// Builds the record described by `args`. + pub(crate) fn new(args: &QueueConfigArgs) -> Result { + if args.qid > QID_MAX { + return Err(EINVAL); + } + let (data_master, pb_va, pb_token, pb_slot, fragment_qid) = match args.target { + QueueConfigTarget::Tiling(binding) => { + if binding.fragment_qid > QID_MAX { + return Err(EINVAL); + } + ( + DataMaster::Tiling, + binding.pb_va, + binding.pb_token, + binding.pb_slot, + binding.fragment_qid as u32, + ) + } + QueueConfigTarget::Fragment => (DataMaster::Fragment, 0, u64::MAX, u32::MAX, u32::MAX), + QueueConfigTarget::Compute => (DataMaster::Compute, 0, u64::MAX, u32::MAX, u32::MAX), + }; + Ok(Self { + tag: CommandTag::QueueConfig as u32, + unk_04: 0, + kick_ring_va: args.kick_ring_va, + kick_ring_fw_va: args.kick_ring_fw_va, + qid: args.qid as u16, + install: args.install as u16, + unk_1c: 0, + priority: QUEUE_PRIORITY as u16, + unk_20: 0, + data_master: data_master as u16, + unk_24: 0, + completion_seed_valid: args.completion_seed.is_some() as u32, + completion_seed: args.completion_seed.unwrap_or(0), + context_id: args.context_id, + unk_34: 0, + free_list_va: args.free_list.va, + free_list_generation: args.free_list.generation, + free_list_slot: args.free_list.slot, + scheduler_va: args.scheduler_va, + qos_update: args.qos_update as u32, + qos_slot: args.qos_slot as u16, + unk_58: 0, + owner_pid: args.owner_pid, + policy: QUEUE_POLICY, + unk_60: 0, + context_update: args.context_update as u16, + unk_64: 0, + unk_66: Self::UNK_66, + unk_68: [0; 6], + pb_va, + pb_token, + pb_slot, + fragment_qid, + unk_86: [0; 0x3a], + }) + } +} + +/// Announcement of a newly written kick entry of a queue (tag 14). +/// +/// A record slot larger than this structure must be zero beyond it. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct KickAnnounce { + /// [`CommandTag::KickAnnounce`]. + pub(crate) tag: u32, + /// Kicked queue. + pub(crate) qid: u16, + /// Number of entries announced. + pub(crate) count: u16, + /// Queue priority. + pub(crate) priority: u8, + /// Timestamp of the announced entry. + pub(crate) timestamp: u64, + /// [`DataMaster`] of the queue. + pub(crate) data_master: u32, + pub(crate) unk_15: [u8; 3], +} + +static_assert!(size_of::() == 0x18); +static_assert!(core::mem::offset_of!(KickAnnounce, timestamp) == 0x09); +static_assert!(core::mem::offset_of!(KickAnnounce, data_master) == 0x11); + +impl KickAnnounce { + /// Announces the entry of queue `qid` written at `timestamp`. + pub(crate) fn new(qid: u8, data_master: DataMaster, timestamp: KickTimestamp) -> Result { + if qid > QID_MAX { + return Err(EINVAL); + } + Ok(Self { + tag: CommandTag::KickAnnounce as u32, + qid: qid as u16, + count: 1, + priority: QUEUE_PRIORITY, + timestamp: timestamp.get(), + data_master: data_master as u32, + unk_15: [0; 3], + }) + } +} + +/// Names the kick of a compute queue that precedes the announced one +/// (tag 16). +/// +/// A record slot larger than this structure must be zero beyond it. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct KickPredecessor { + /// [`CommandTag::KickPredecessor`]. + pub(crate) tag: u32, + /// Kicked queue. + pub(crate) qid: u16, + /// Timestamp of the queue's kick preceding the announced one. + pub(crate) previous: u64, + pub(crate) unk_0e: u32, +} + +static_assert!(size_of::() == 0x12); + +impl KickPredecessor { + const UNK_0E: u32 = 0xfe; + + /// Names `previous` as the kick of queue `qid` preceding the announced + /// one. + pub(crate) fn new(qid: u8, previous: KickTimestamp) -> Result { + if qid > QID_MAX { + return Err(EINVAL); + } + Ok(Self { + tag: CommandTag::KickPredecessor as u32, + qid: qid as u16, + previous: previous.get(), + unk_0e: Self::UNK_0E, + }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs new file mode 100644 index 0000000000000..349d03ed14d71 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -0,0 +1,1309 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Render work descriptors. +//! +//! A render pass is kicked as a tiling (TA) descriptor on one queue and a +//! fragment (3D) descriptor on the queue paired with it. Each descriptor +//! embeds the register arrays its kick entry binds: one for tiling, and for +//! fragment the main array followed by the partial-render store, resume and +//! load arrays. +//! +//! Descriptors are built in place with [`TaDescriptor::write`] and +//! [`FragmentSlot::write`]; the completion link and the fragment MCache table +//! are added just before publication. + +pub(crate) mod storage; + +use kernel::prelude::*; + +use super::kick::{ + KickTimestamp, McacheClass, McacheRange, McacheTable, RegisterArray, RegisterArrayBinding, + RegisterWrite, WriteRange, MCACHE_INLINE_RANGES, +}; +use super::queue::CommandTag; +use super::{ + clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, + UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, +}; + +/// Size of a tiling descriptor. +pub(crate) const TA_DESCRIPTOR_SIZE: usize = 0x9c0; +/// Size of a [`FragmentSlot`]: the descriptor and its MCache spill table. +pub(crate) const FRAGMENT_SLOT_SIZE: usize = 0x2380; +/// Write ranges of one fragment pass: its attachments and the auxiliary +/// framebuffer. +pub(crate) const MCACHE_RANGES_MAX: usize = 17; +/// Size of the auxiliary framebuffer of a pass. +pub(crate) const AUX_FB_SIZE: u64 = 0x8000; + +/// QoS class of tiling kicks. +pub(crate) const TA_QOS_CLASS: u8 = 0x0c; +/// QoS class of fragment kicks. +pub(crate) const FRAGMENT_QOS_CLASS: u8 = 0x18; +/// Event mask of render kicks. +pub(crate) const RENDER_KICK_EVENT_MASK: [u64; 4] = [0; 4]; + +/// Base of the user address window. Tiling resource registers hold offsets +/// from it rather than addresses. +const USER_BASE: u64 = 0x10_0000_0000; +/// The VDM control-stream register holds a 39-bit offset from [`USER_BASE`]. +const VDM_OFFSET_LIMIT: u64 = 1 << 39; +/// The deflake buffer holds three sub-buffers: at its start and at these +/// offsets. +const DEFLAKE_20: u64 = 0x20; +const DEFLAKE_2A0: u64 = 0x2a0; + +const TA_REGISTERS: usize = 73; +const FRAGMENT_REGISTERS: usize = 89; +const PARTIAL_STORE_REGISTERS: usize = 16; +const PARTIAL_RESUME_REGISTERS: usize = 23; +const PARTIAL_LOAD_REGISTERS: usize = 10; + +/// Checkpoints into the tiling array, as entry counts: before the final two +/// entries, then after the last one (twice). +const TA_CHECKPOINTS: [u16; 3] = [ + TA_REGISTERS as u16 - 2, + TA_REGISTERS as u16, + TA_REGISTERS as u16, +]; +/// Checkpoints into the main fragment array, as entry counts: after the first +/// of the four closing entries, then after the second (twice). +const FRAGMENT_CHECKPOINTS: [u16; 3] = [ + FRAGMENT_REGISTERS as u16 - 3, + FRAGMENT_REGISTERS as u16 - 2, + FRAGMENT_REGISTERS as u16 - 2, +]; + +const TILE_SIZE: u64 = 32; +const MACRO_TILES: u64 = 4; +const REGION_ENTRY_SIZE: u64 = 5; +/// Highest number of samples per pixel, as a power of two. +const SAMPLES_LOG2_MAX: u8 = 2; +/// Bytes of tile buffer per block. +const TIB_BLOCK_SIZE: u64 = 2048; +const TILE_CONFIG_BASE: u64 = 0x280; +const TILE_CONFIG_LAYERED: u64 = 1 << 0; +const TILE_CONFIG_PROCESS_EMPTY_TILES: u64 = 1 << 16; +const AUX_FB_FLAGS_BASE: u64 = 0xc000; +const AUX_FB_FLAGS_DBIAS_INT: u64 = 1 << 18; +/// Tiling layer mode: layer count less one, bit 15, and for layered targets +/// bit 14 plus the empty-tile policy. The second copy keeps only the count and +/// bit 14. +const LAYER_MODE_BASE: u64 = 1 << 15; +const LAYER_MODE_LAYERED: u64 = 1 << 14; +const LAYER_MODE_PROCESS_EMPTY: u64 = 1 << 13; +const LAYER_MODE_SKIP_EMPTY: u64 = 1 << 12; +const LAYER_MODE_PIPE_MASK: u64 = 0x47ff; +/// Fragment tile state: required bits 0x3717f and 37, layered targets in bit +/// 32, the utile size bits of the tile-buffer configuration in bits 43:40, +/// tiles in X less one from bit 44 and tiles in Y less one from bit 53. +const TILE_STATE_BASE: u64 = 0x3717f | 1 << 37; +const TILE_STATE_LAYERED: u64 = 1 << 32; +const TILE_STATE_UTILE_SHIFT: u32 = 28; +const TILE_STATE_TILES_X_SHIFT: u32 = 44; +const TILE_STATE_TILES_Y_SHIFT: u32 = 53; +/// Tile-buffer configuration: utile width / 16 in bits 13:12, utile height / +/// 16 in bits 15:14, samples (log2) in bits 1:0. +const UTILE_CONFIG_WIDTH_SHIFT: u32 = 12; +const UTILE_CONFIG_HEIGHT_SHIFT: u32 = 14; +const UTILE_CONFIG_SIZE_MASK: u64 = 0xf000; +/// The partial-render phases load and store every attached depth and stencil +/// surface. +const ZLS_PARTIAL_DEPTH: u64 = 1 << 15 | 1 << 19; +const ZLS_PARTIAL_STENCIL: u64 = 1 << 14 | 1 << 18; +const STENCIL_CLEAR_BASE: u32 = 0x300; +/// Value of the ZLS swizzle registers and descriptor field. +const ZLS_SWIZZLE: u32 = 0x0404_0404; +/// Value of register 0x15359 in the partial store and resume arrays; the +/// main array writes zero. +const UNK_15359_PARTIAL: u64 = 0x20; +/// Required value of register 0x15049 and of fragment descriptor field +/// +0x1f50. +const UNK_15049: u64 = 0x10_0000; +/// Value of the fragment register 0x10791. +const UNK_10791_FRAGMENT: u64 = UNK_10791 | 1 << 9; +/// USC program offsets carry tag bits that are not part of the address. +const USC_OFFSET_TAG_MASK: u32 = 7; + +/// A depth or stencil surface. +#[derive(Copy, Clone, Debug)] +pub(crate) struct ZlsSurface { + /// GPU address of the surface, or zero when it is not attached. + pub(crate) va: u64, + /// GPU address of its compression metadata. + pub(crate) compression_va: u64, + /// Stride of the surface. + pub(crate) stride: u64, + /// Stride of the compression metadata. + pub(crate) compression_stride: u64, +} + +/// A USC program and the resource specifier it is launched with. +#[derive(Copy, Clone, Debug)] +pub(crate) struct UscProgram { + /// Tagged offset of the program in the queue's USC window. + pub(crate) offset: u32, + /// Resource specifier. + pub(crate) rsrc_spec: u64, +} + +/// Render-pass state from the userspace command. +#[derive(Copy, Clone, Debug)] +pub(crate) struct RenderPass { + /// Width of the render target in pixels. + pub(crate) width: u32, + /// Height of the render target in pixels. + pub(crate) height: u32, + /// Number of layers, 1 to 2048. + pub(crate) layers: u16, + /// Utile width: 16 or 32. + pub(crate) utile_width: u8, + /// Utile height: 16 or 32, at most the width. + pub(crate) utile_height: u8, + /// Samples per pixel, as a power of two: 0 to 2. + pub(crate) samples_log2: u8, + /// Bytes of tile buffer per sample. + pub(crate) sample_size: u8, + /// Process tiles no primitive touches. + pub(crate) process_empty_tiles: bool, + /// Multisample control. + pub(crate) msaa_control: u64, + /// PPP control. + pub(crate) ppp_control: u64, + /// GPU address of the VDM control stream. + pub(crate) vdm_va: u64, + /// GPU address of the scissor array. + pub(crate) scissor_va: u64, + /// GPU address of the depth-bias array, or zero. + pub(crate) depth_bias_va: u64, + /// Depth-bias values are integers. + pub(crate) depth_bias_is_int: bool, + /// GPU address of the occlusion-query results, or zero. + pub(crate) occlusion_query_va: u64, + /// GPU address of the sampler heap, or zero. + pub(crate) sampler_heap_va: u64, + /// Number of samplers in the heap. + pub(crate) sampler_count: u32, + /// ZLS control at the pass boundaries. + pub(crate) zls_control: u64, + /// Depth surface dimensions. + pub(crate) depth_dimensions: u64, + /// Depth surface. + pub(crate) depth: ZlsSurface, + /// Stencil surface. + pub(crate) stencil: ZlsSurface, + /// Depth clear value (IEEE-754 bits). + pub(crate) depth_clear: u32, + /// Stencil clear value (background object register value). + pub(crate) stencil_clear: u32, + /// Merge upper bound in X (IEEE-754 bits). + pub(crate) merge_upper_x: u32, + /// Merge upper bound in Y (IEEE-754 bits). + pub(crate) merge_upper_y: u32, + /// Base of the queue's USC window. + pub(crate) usc_base: u64, + /// Background program. + pub(crate) bg: UscProgram, + /// Partial-render background program. + pub(crate) partial_bg: UscProgram, + /// Tagged USC offset of the end-of-tile program. Its resource specifier + /// is always written as zero. + pub(crate) eot_offset: u32, + /// Tagged USC offset of the partial-render end-of-tile program. + pub(crate) partial_eot_offset: u32, + /// The resource specifiers have their high 32 bits. + pub(crate) rsrc_spec_hi: bool, +} + +/// Kernel-owned objects and identities a render pass uses. +#[derive(Copy, Clone, Debug)] +pub(crate) struct RenderResources { + /// UAT context the pass runs in. + pub(crate) context_id: u16, + /// Allocation generation of that context. + pub(crate) context_generation: u8, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + /// Buffer slot of the USC private-memory free list. + pub(crate) free_list_slot: u32, + /// GPU address of the free list's shared control object. + pub(crate) free_list_control_va: u64, + /// GPU address of the pass's kick record, in which the firmware counts + /// the pass's completed kicks. + pub(crate) kick_record_va: u64, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// GPU address of the parameter-buffer scene entry of the pass. + pub(crate) scene_va: u64, + /// GPU address following the last scene entry. + pub(crate) scene_end_va: u64, + /// Parameter-buffer scratch offset register value. + pub(crate) pm_scratch: u64, + /// Parameter-buffer metric pointer register value. + pub(crate) pm_metric: u64, + /// GPU address of the tile map. + pub(crate) tilemap_va: u64, + /// GPU address of the layer metadata. + pub(crate) layermeta_va: u64, + /// GPU address of the heap metadata. + pub(crate) heapmeta_va: u64, + /// GPU address of the tail-pointer cache. + pub(crate) tpc_va: u64, + /// Size of the tail-pointer cache the target needs. + pub(crate) tpc_size: u64, + /// GPU address of the deflake buffer. + pub(crate) deflake_va: u64, + /// GPU address of the tiling status word. + pub(crate) ta_status_va: u64, + /// GPU address of the fragment status word, as seen by the pass's VM. + pub(crate) fragment_status_va: u64, + /// Firmware alias of the fragment status word. + pub(crate) fragment_status_fw_va: u64, + /// GPU address of the auxiliary framebuffer. + pub(crate) aux_fb_va: u64, + /// GPU address of the pass's start/end timestamp pair. + pub(crate) timestamp_va: u64, + /// Object IDs of the tiling stage. + pub(crate) ta_ids: ObjectIds, + /// Object IDs of the fragment stage. + pub(crate) fragment_ids: ObjectIds, + /// Value of fragment descriptor field +0x2104: bit 0 of GPU register + /// 0xe0141c, read at probe. + pub(crate) unk_2104: u32, + /// Value of fragment descriptor field +0x210c: bits 3:0 of GPU register + /// 0xe01480, read at probe. + pub(crate) unk_210c: u32, +} + +/// Per-stage inputs of a render pass. +#[derive(Copy, Clone, Debug)] +pub(crate) struct RenderStage { + /// Queue the stage is kicked on. + pub(crate) qid: u8, + /// GPU address of the stage's descriptor, as seen by the pass's VM. + pub(crate) descriptor_va: u64, + /// Command-buffer state word of the stage. + pub(crate) state_word: u32, + /// GPU address of the queue's completion stamp word. + pub(crate) stamp_va: u64, + /// GPU address of the queue's word in the auxiliary stamp page. + pub(crate) aux_stamp_va: u64, +} + +/// Inputs of both render descriptors. +pub(crate) struct RenderArgs<'a> { + /// Render-pass state. + pub(crate) pass: &'a RenderPass, + /// Kernel-owned objects. + pub(crate) resources: &'a RenderResources, + /// Submission ordinal of the render queue pair, from zero. + pub(crate) ordinal: u64, + /// Tiling stage. + pub(crate) ta: RenderStage, + /// Fragment stage. + pub(crate) fragment: RenderStage, +} + +impl RenderArgs<'_> { + /// Completion stamp value of this pass: the pass's sequence number is the + /// pair's submission ordinal plus one. + fn stamp_value(&self) -> u32 { + stamp_value((self.ordinal as u32).wrapping_add(1)) + } +} + +/// Tile-grid geometry of a render target, in the units its registers take. +struct Geometry { + tiles_x: u64, + tiles_y: u64, + /// Size of the region array. + region_size: u64, + /// Number of macro tiles. + macro_tiles: u64, + /// Two utile slots per utile of every macro tile. + utile_slots: u64, + /// Macro-tile columns times 3, 2 and 1, in 9-bit fields from bit 0. + x_blocks: u64, + /// Macro-tile rows times 3, 2 and 1, in 9-bit fields from bit 0. + y_blocks: u64, + /// Tiles in Y less one from bit 12, tiles in X less one below. + screen: u64, + /// Height less one from bit 16, width less one below. + pixels: u64, + /// Macro-tile width in utiles from bit 16, height in utiles below. + macro_size: u64, +} + +impl Geometry { + fn new(pass: &RenderPass) -> Result { + if pass.width == 0 || pass.height == 0 || pass.layers == 0 || pass.layers > 2048 { + return Err(EINVAL); + } + if !matches!( + (pass.utile_width, pass.utile_height), + (32, 32) | (32, 16) | (16, 16) + ) { + return Err(EINVAL); + } + let width = pass.width as u64; + let height = pass.height as u64; + let tiles_x = width.div_ceil(TILE_SIZE); + let tiles_y = height.div_ceil(TILE_SIZE); + let utiles_x = TILE_SIZE / pass.utile_width as u64; + let utiles_y = TILE_SIZE / pass.utile_height as u64; + let utiles = utiles_x * utiles_y; + let mtile_x = tiles_x.div_ceil(MACRO_TILES).next_multiple_of(4); + let mtile_y = tiles_y.div_ceil(MACRO_TILES).next_multiple_of(4); + let macro_tiles = mtile_x * mtile_y; + Ok(Self { + tiles_x, + tiles_y, + region_size: (REGION_ENTRY_SIZE * macro_tiles * utiles).div_ceil(4), + macro_tiles, + utile_slots: 2 * macro_tiles * utiles, + x_blocks: (3 * mtile_x) | ((2 * mtile_x) << 9) | (mtile_x << 18), + y_blocks: (3 * mtile_y) | ((2 * mtile_y) << 9) | (mtile_y << 18), + screen: (tiles_y - 1) << 12 | (tiles_x - 1), + pixels: (width - 1) | (height - 1) << 16, + macro_size: (mtile_y * utiles_y) | ((mtile_x * utiles_x) << 16), + }) + } +} + +/// Register and descriptor words derived from the [`RenderPass`]. +struct PassWords { + utile_config: u64, + tile_config: u64, + layer_mode: u64, + tib_blocks: u64, + partial_zls_control: u64, + aux_fb_flags: u64, + bg_rsrc_spec: u64, + bg_va: u64, + partial_bg_rsrc_spec: u64, + partial_bg_va: u64, + eot_va: u64, + partial_eot_va: u64, +} + +impl PassWords { + fn new(pass: &RenderPass) -> Result { + if pass.samples_log2 > SAMPLES_LOG2_MAX { + return Err(EINVAL); + } + let samples = 1u64 << pass.samples_log2; + let layered = pass.layers > 1; + + // The tile configuration carries the empty-tile request for every + // target; the tiling layer mode only for layered ones. + let mut tile_config = TILE_CONFIG_BASE; + let mut layer_mode = LAYER_MODE_BASE | (pass.layers as u64 - 1); + if layered { + tile_config |= TILE_CONFIG_LAYERED; + layer_mode |= LAYER_MODE_LAYERED; + layer_mode |= match pass.process_empty_tiles { + true => LAYER_MODE_PROCESS_EMPTY, + false => LAYER_MODE_SKIP_EMPTY, + }; + } + if pass.process_empty_tiles { + tile_config |= TILE_CONFIG_PROCESS_EMPTY_TILES; + } + + let mut partial_zls_control = pass.zls_control; + if pass.depth.va != 0 { + partial_zls_control |= ZLS_PARTIAL_DEPTH; + } + if pass.stencil.va != 0 { + partial_zls_control |= ZLS_PARTIAL_STENCIL; + } + + let aux_fb_flags = match pass.depth_bias_is_int { + true => AUX_FB_FLAGS_BASE | AUX_FB_FLAGS_DBIAS_INT, + false => AUX_FB_FLAGS_BASE, + }; + let rsrc_spec = |spec: u64| match pass.rsrc_spec_hi { + true => spec, + false => spec & u32::MAX as u64, + }; + let usc = |offset: u32| offset_va(pass.usc_base, (offset & !USC_OFFSET_TAG_MASK) as u64); + + Ok(Self { + utile_config: (pass.utile_width as u64 / 16) << UTILE_CONFIG_WIDTH_SHIFT + | (pass.utile_height as u64 / 16) << UTILE_CONFIG_HEIGHT_SHIFT + | pass.samples_log2 as u64, + tile_config, + layer_mode, + tib_blocks: (pass.sample_size as u64 + * pass.utile_width as u64 + * pass.utile_height as u64 + * samples) + .div_ceil(TIB_BLOCK_SIZE), + partial_zls_control, + aux_fb_flags, + bg_rsrc_spec: rsrc_spec(pass.bg.rsrc_spec), + bg_va: usc(pass.bg.offset)?, + partial_bg_rsrc_spec: rsrc_spec(pass.partial_bg.rsrc_spec), + partial_bg_va: usc(pass.partial_bg.offset)?, + eot_va: usc(pass.eot_offset)?, + partial_eot_va: usc(pass.partial_eot_offset)?, + }) + } +} + +/// Offset of an address from the user window base. +fn user_offset(va: u64) -> Result { + va.checked_sub(USER_BASE).ok_or(EINVAL) +} + +/// Offset of an address from the user window base that fits 32 bits. +fn user_offset32(va: u64) -> Result { + u32::try_from(user_offset(va)?).map_err(|_| EINVAL) +} + +fn ta_registers( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<[RegisterWrite; TA_REGISTERS]> { + let p = args.pass; + let r = args.resources; + let tilemap = user_offset(r.tilemap_va)?; + let layermeta = user_offset(r.layermeta_va)?; + let heapmeta = user_offset(r.heapmeta_va)? | 1 << 63; + let tpc = user_offset(r.tpc_va)?; + let deflake = user_offset32(r.deflake_va)? as u64; + let deflake_20 = user_offset32(offset_va(r.deflake_va, DEFLAKE_20)?)? as u64; + let deflake_2a0 = user_offset32(offset_va(r.deflake_va, DEFLAKE_2A0)?)? as u64; + let vdm = user_offset(p.vdm_va)?; + if vdm >= VDM_OFFSET_LIMIT { + return Err(EINVAL); + } + let key = work_key(r.context_id, args.ta.state_word); + let ids = r.ta_ids.word(); + + Ok([ + RegisterWrite::new(0x01748, 1), + RegisterWrite::new(0x10141, 0x200), + RegisterWrite::new(0x1c039, tilemap), + RegisterWrite::new(0x1c9c8, tilemap), + RegisterWrite::new(0x1c0a1, tpc), + RegisterWrite::new(0x1c031, heapmeta), + RegisterWrite::new(0x1c9c0, heapmeta), + RegisterWrite::new(0x1c051, 0x003a_0012_006b_0003), + RegisterWrite::new(0x1c061, 1), + RegisterWrite::new(0x10149, w.utile_config), + RegisterWrite::new(0x10139, p.msaa_control), + RegisterWrite::new(0x10111, deflake_2a0), + RegisterWrite::new(0x1c9b0, deflake_2a0), + RegisterWrite::new(0x10119, deflake_20), + RegisterWrite::new(0x1c9b8, deflake_20), + RegisterWrite::new(0x1c958, 1), + RegisterWrite::new(0x1c950, deflake | 1 << 50), + RegisterWrite::new(0x1c930, 0), + RegisterWrite::new(0x1c880, vdm), + RegisterWrite::new(0x1c079, layermeta), + RegisterWrite::new(0x1c9d8, layermeta), + RegisterWrite::new(0x10151, 0), + RegisterWrite::new(0x1c199, 0), + RegisterWrite::new(0x1c1a1, 0), + RegisterWrite::new(0x1c1a9, 0), + RegisterWrite::new(0x1c1b1, 0), + RegisterWrite::new(0x1c1b9, 0), + RegisterWrite::new(0x1c8f8, UNK_1C8F8), + RegisterWrite::new(0x1c0b1, g.region_size), + RegisterWrite::new(0x1c850, g.region_size), + RegisterWrite::new(0x10131, p.msaa_control), + RegisterWrite::new(0x10121, p.ppp_control), + RegisterWrite::new(0x10129, g.pixels), + RegisterWrite::new(0x101b9, g.screen), + RegisterWrite::new(0x1c069, g.x_blocks), + RegisterWrite::new(0x1c071, g.y_blocks), + RegisterWrite::new(0x1c081, g.macro_tiles), + RegisterWrite::new(0x1c0a9, g.utile_slots), + RegisterWrite::new(0x10171, 0x100), + RegisterWrite::new(0x10169, w.layer_mode), + RegisterWrite::new(0x0a309, 0), + RegisterWrite::new(0x1c8e0, u64::MAX), + RegisterWrite::new(0x1c8e8, u64::MAX), + RegisterWrite::new(0x1c898, 0), + RegisterWrite::new(0x101e1, UNK_101D9), + RegisterWrite::new(0x1c9e8, w.layer_mode & LAYER_MODE_PIPE_MASK), + RegisterWrite::new(0x1a099, 0), + RegisterWrite::new(0x1a0a1, 0), + RegisterWrite::new(0x1a069, 0), + RegisterWrite::new(0x1a071, 0), + RegisterWrite::new(0x1a0c9, 0), + RegisterWrite::new(0x1a0d1, 0), + RegisterWrite::new(0x101c9, 0), + RegisterWrite::new(0x0d471, 0), + RegisterWrite::new(0x1a0f1, UNK_1A0E9), + RegisterWrite::new(0x10799, UNK_10791), + RegisterWrite::new(0x1c830, r.pb_slot as u64), + RegisterWrite::new(0x1ca30, r.pm_scratch), + RegisterWrite::new(0x16c39, r.pm_scratch), + RegisterWrite::new(0x1c910, r.pm_metric), + RegisterWrite::new(0x0a5a1, UNK_0A599), + RegisterWrite::new(0x0d419, UNK_0D411), + RegisterWrite::new(0x1ca10, ids), + RegisterWrite::new(0x014a1, ids), + RegisterWrite::new(0x0a349, ids), + RegisterWrite::new(0x10209, key), + RegisterWrite::new(0x1c9f0, key), + RegisterWrite::new(0x14320, key), + RegisterWrite::new(0x14308, r.free_list_slot as u64), + RegisterWrite::new(0x14318, r.ta_status_va | 1), + RegisterWrite::new(0x01740, 1), + RegisterWrite::new(0x1c880, deflake_2a0), + RegisterWrite::new(0x1c898, 1), + ]) +} + +fn fragment_registers( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> [RegisterWrite; FRAGMENT_REGISTERS] { + let p = args.pass; + let r = args.resources; + let mut tile_state = TILE_STATE_BASE + | (w.utile_config & UTILE_CONFIG_SIZE_MASK) << TILE_STATE_UTILE_SHIFT + | (g.tiles_x - 1) << TILE_STATE_TILES_X_SHIFT + | (g.tiles_y - 1) << TILE_STATE_TILES_Y_SHIFT; + if p.layers > 1 { + tile_state |= TILE_STATE_LAYERED; + } + let key = work_key(r.context_id, args.fragment.state_word); + let ids = r.fragment_ids.word(); + + [ + RegisterWrite::new(0x01739, 1), + RegisterWrite::new(0x10009, w.utile_config), + // End-of-tile resource specifier. + RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15381, w.eot_va), + RegisterWrite::new(0x15369, w.bg_rsrc_spec), + RegisterWrite::new(0x15371, w.bg_va), + RegisterWrite::new(0x15131, p.merge_upper_x as u64), + RegisterWrite::new(0x15139, p.merge_upper_y as u64), + RegisterWrite::new(0x100a1, 0), + RegisterWrite::new(0x15069, 0), + RegisterWrite::new(0x15071, 0), + RegisterWrite::new(0x16058, 0), + RegisterWrite::new(0x10019, p.msaa_control), + RegisterWrite::new(0x100b1, g.macro_size), + RegisterWrite::new(0x16030, g.macro_size), + RegisterWrite::new(0x100d9, g.screen), + RegisterWrite::new(0x0a301, 0), + RegisterWrite::new(0x10791, UNK_10791_FRAGMENT), + RegisterWrite::new(0x16098, r.heapmeta_va), + RegisterWrite::new(0x15109, p.scissor_va), + RegisterWrite::new(0x15101, p.depth_bias_va), + RegisterWrite::new(0x15021, w.aux_fb_flags), + RegisterWrite::new(0x15211, (p.height as u64) << 32 | p.width as u64), + RegisterWrite::new(0x15049, UNK_15049), + RegisterWrite::new(0x10051, w.tib_blocks), + RegisterWrite::new(0x15321, p.depth_dimensions), + RegisterWrite::new(0x15301, p.depth_clear as u64), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15311, p.occlusion_query_va), + RegisterWrite::new(0x15319, p.zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15351, 0), + RegisterWrite::new(0x15329, p.depth.va), + RegisterWrite::new(0x15331, p.depth.va), + RegisterWrite::new(0x15339, p.stencil.va), + RegisterWrite::new(0x15341, p.stencil.va), + RegisterWrite::new(0x15231, 0), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15239, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15401, p.depth.stride), + RegisterWrite::new(0x15421, p.depth.stride), + RegisterWrite::new(0x15409, p.stencil.stride), + RegisterWrite::new(0x15429, p.stencil.stride), + RegisterWrite::new(0x153c1, p.depth.compression_va), + RegisterWrite::new(0x15411, p.depth.compression_stride), + RegisterWrite::new(0x153c9, p.depth.compression_va), + RegisterWrite::new(0x15431, p.depth.compression_stride), + RegisterWrite::new(0x153d1, p.stencil.compression_va), + RegisterWrite::new(0x15419, p.stencil.compression_stride), + RegisterWrite::new(0x153d9, p.stencil.compression_va), + RegisterWrite::new(0x15439, p.stencil.compression_stride), + RegisterWrite::new(0x16429, r.tilemap_va), + RegisterWrite::new(0x16060, r.layermeta_va), + RegisterWrite::new(0x16431, (4 * g.region_size) << 24), + RegisterWrite::new(0x10039, w.tile_config), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16451, 0), + RegisterWrite::new(0x15359, 0), + RegisterWrite::new(0x100b8, UNK_1C8F8), + RegisterWrite::new(0x16461, r.aux_fb_va), + RegisterWrite::new(0x16090, r.aux_fb_va), + RegisterWrite::new(0x101e9, UNK_101D9), + RegisterWrite::new(0x160a8, 0), + RegisterWrite::new(0x16068, tile_state), + RegisterWrite::new(0x1a0a9, 0), + RegisterWrite::new(0x1a0b1, 0), + RegisterWrite::new(0x1a079, 0), + RegisterWrite::new(0x1a081, 0), + RegisterWrite::new(0x1a0d9, 0), + RegisterWrite::new(0x1a0e1, 0), + RegisterWrite::new(0x101c1, 0), + RegisterWrite::new(0x0d469, 0), + RegisterWrite::new(0x1a0f9, UNK_1A0E9), + RegisterWrite::new(0x0a5a9, UNK_0A599), + RegisterWrite::new(0x0d429, UNK_0D411), + RegisterWrite::new(0x160e0, ids), + RegisterWrite::new(0x01499, ids), + RegisterWrite::new(0x0a341, ids), + RegisterWrite::new(0x1c838, r.pb_slot as u64), + RegisterWrite::new(0x1ca28, r.pm_scratch), + RegisterWrite::new(0x10211, key), + RegisterWrite::new(0x10420, key), + RegisterWrite::new(0x14048, r.free_list_slot as u64), + RegisterWrite::new(0x14080, r.fragment_status_va | 1), + RegisterWrite::new(0x01731, 1), + RegisterWrite::new(0x16020, 1), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16068, 0x4_0000), + ] +} + +fn partial_store_registers( + p: &RenderPass, + w: &PassWords, +) -> [RegisterWrite; PARTIAL_STORE_REGISTERS] { + [ + // End-of-tile resource specifier. + RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15381, w.partial_eot_va), + RegisterWrite::new(0x10039, w.tile_config), + RegisterWrite::new(0x15359, UNK_15359_PARTIAL), + RegisterWrite::new(0x15331, p.depth.va), + RegisterWrite::new(0x153c9, p.depth.compression_va), + RegisterWrite::new(0x15341, p.stencil.va), + RegisterWrite::new(0x153d9, p.stencil.compression_va), + RegisterWrite::new(0x15421, p.depth.stride), + RegisterWrite::new(0x15431, p.depth.compression_stride), + RegisterWrite::new(0x15429, p.stencil.stride), + RegisterWrite::new(0x15439, p.stencil.compression_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + ] +} + +fn partial_resume_registers( + p: &RenderPass, + w: &PassWords, +) -> [RegisterWrite; PARTIAL_RESUME_REGISTERS] { + [ + // End-of-tile resource specifier. + RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15381, w.partial_eot_va), + RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), + RegisterWrite::new(0x15371, w.partial_bg_va), + RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), + RegisterWrite::new(0x15359, UNK_15359_PARTIAL), + RegisterWrite::new(0x15331, p.depth.va), + RegisterWrite::new(0x153c9, p.depth.compression_va), + RegisterWrite::new(0x15341, p.stencil.va), + RegisterWrite::new(0x153d9, p.stencil.compression_va), + RegisterWrite::new(0x15421, p.depth.stride), + RegisterWrite::new(0x15431, p.depth.compression_stride), + RegisterWrite::new(0x15429, p.stencil.stride), + RegisterWrite::new(0x15439, p.stencil.compression_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15329, p.depth.va), + RegisterWrite::new(0x153c1, p.depth.compression_va), + RegisterWrite::new(0x15339, p.stencil.va), + RegisterWrite::new(0x153d1, p.stencil.compression_va), + RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + ] +} + +fn partial_load_registers( + p: &RenderPass, + w: &PassWords, +) -> [RegisterWrite; PARTIAL_LOAD_REGISTERS] { + [ + RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), + RegisterWrite::new(0x15371, w.partial_bg_va), + RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15329, p.depth.va), + RegisterWrite::new(0x153c1, p.depth.compression_va), + RegisterWrite::new(0x15339, p.stencil.va), + RegisterWrite::new(0x153d1, p.stencil.compression_va), + RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + ] +} + +/// Tiling (TA) work descriptor (tag 0). +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct TaDescriptor { + /// [`CommandTag::Tiling`]. + pub(crate) tag: u32, + /// Twice the pair's submission ordinal, plus one. + pub(crate) submit_sequence: u64, + /// UAT context of the pass. + pub(crate) context_id: u32, + /// GPU address of the pass's kick record. + pub(crate) kick_record_va: u64, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + pub(crate) unk_1c: u32, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// GPU address of the parameter-buffer scene entry. + pub(crate) scene_va: u64, + /// GPU address following the last scene entry. + pub(crate) scene_end_va: u64, + /// Checkpoints into the register array. + pub(crate) checkpoints: [u16; 3], + pub(crate) unk_3e: [u8; 0xa], + /// The pair's submission ordinal plus half of it. + pub(crate) unk_48: u32, + pub(crate) unk_4c: [u8; 0x14], + /// Register array. + pub(crate) registers: RegisterArray, + /// GPU address of the tail-pointer cache. + pub(crate) tpc_va: u64, + /// Size of the tail-pointer cache the target needs. + pub(crate) tpc_size: u64, + pub(crate) unk_790: [u8; 0xc], + /// QID of the fragment queue that consumes this pass. + pub(crate) fragment_qid: u32, + /// Completion stamp value of the pass. + pub(crate) stamp_value: u32, + /// Non-zero when the fragment queue had earlier kicks outstanding. + pub(crate) fragment_busy: u32, + /// Timestamp of the fragment kick of this pass. + pub(crate) fragment_kick: u64, + /// Render-pass sequence of the pass, stored twice. + pub(crate) pass_seq: [u8; 2], + pub(crate) unk_7b2: [u8; 0x24], + /// Offset from the user base of the deflake sub-buffer at +0x2a0. + pub(crate) deflake_2a0_offset: u32, + pub(crate) unk_7da: [u8; 0x94], + /// Predecessor object ID of the tiling stage. + pub(crate) predecessor_id: u32, + pub(crate) unk_872: u32, + pub(crate) unk_876: u32, + /// GPU address of the sampler heap. + pub(crate) sampler_heap_va: u64, + /// Number of samplers. + pub(crate) sampler_count: u32, + /// Number of samplers plus one, or zero without samplers. + pub(crate) sampler_max: u32, + pub(crate) unk_88a: [u8; 8], + pub(crate) unk_892: u32, + pub(crate) unk_896: [u8; 0x10], + /// GPU address of the tiling queue's completion stamp word. + pub(crate) stamp_va: u64, + /// GPU address of the tiling queue's auxiliary stamp word. + pub(crate) aux_stamp_va: u64, + /// Completion stamp value of the pass. + pub(crate) stamp_value_2: u32, + /// QID of the tiling queue. + pub(crate) qid: u32, + pub(crate) unk_8be: u32, + pub(crate) unk_8c2: u32, + /// Completion value of the tiling kick (see + /// [`KickTimestamp::completion_value`]). + pub(crate) completion: u64, + /// Current object ID of the tiling stage. + pub(crate) current_id: u64, + /// Submission ordinal of the pair. + pub(crate) ordinal: u32, + pub(crate) unk_8da: [u8; 0x24], + /// GPU address the pass start timestamp is written to. + pub(crate) timestamp_start_va: u64, + /// Zero: the tiling stage writes no end timestamp. + pub(crate) timestamp_end_va: u64, + pub(crate) unk_90e: [u8; 0x24], + pub(crate) unk_932: u8, + pub(crate) unk_933: u8, + /// GPU address of the USC free list's shared control object. + pub(crate) free_list_control_va: u64, + pub(crate) unk_93c: u8, + pub(crate) unk_93d: u8, + /// Set on the first kick of a queue pair. + pub(crate) unk_93e: [u8; 2], + pub(crate) unk_940: [u8; 5], + /// GPU address of the tiling status word. + pub(crate) status_va: u64, + /// Allocation generation of the UAT context. + pub(crate) context_generation: u8, + pub(crate) unk_94e: [u8; 0x72], +} + +static_assert!(size_of::() == TA_DESCRIPTOR_SIZE); +static_assert!(core::mem::offset_of!(TaDescriptor, unk_48) == 0x48); +static_assert!(core::mem::offset_of!(TaDescriptor, registers) == 0x60); +static_assert!(core::mem::offset_of!(TaDescriptor, tpc_va) == 0x780); +static_assert!(core::mem::offset_of!(TaDescriptor, fragment_qid) == 0x79c); +static_assert!(core::mem::offset_of!(TaDescriptor, deflake_2a0_offset) == 0x7d6); +static_assert!(core::mem::offset_of!(TaDescriptor, predecessor_id) == 0x86e); +static_assert!(core::mem::offset_of!(TaDescriptor, stamp_va) == 0x8a6); +static_assert!(core::mem::offset_of!(TaDescriptor, completion) == 0x8c6); +static_assert!(core::mem::offset_of!(TaDescriptor, timestamp_start_va) == 0x8fe); +static_assert!(core::mem::offset_of!(TaDescriptor, free_list_control_va) == 0x934); +static_assert!(core::mem::offset_of!(TaDescriptor, status_va) == 0x945); + +// SAFETY: `TaDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for TaDescriptor {} + +impl TaDescriptor { + const ARRAY_OFFSET: u64 = core::mem::offset_of!(Self, registers) as u64; + const UNK_876: u32 = u32::MAX; + const UNK_892: u32 = 1; + const UNK_8C2: u32 = 1; + const UNK_932: u8 = 0x44; + const UNK_93C: u8 = 1; + const UNK_93E_FIRST_KICK: [u8; 2] = [0xd0, 0x91]; + + /// Writes the complete descriptor except the completion link. + pub(crate) fn write(&mut self, args: &RenderArgs<'_>) -> Result { + let pass = args.pass; + let resources = args.resources; + let geometry = Geometry::new(pass)?; + let words = PassWords::new(pass)?; + let registers = ta_registers(args, &geometry, &words)?; + let deflake_2a0_offset = user_offset32(offset_va(resources.deflake_va, DEFLAKE_2A0)?)?; + let array_va = offset_va(args.ta.descriptor_va, Self::ARRAY_OFFSET)?; + let sampler_max = sampler_max(pass.sampler_count)?; + let stamp_value = args.stamp_value(); + + clear(self); + self.tag = CommandTag::Tiling as u32; + self.submit_sequence = args.ordinal.wrapping_mul(2).wrapping_add(1); + self.context_id = resources.context_id.into(); + self.kick_record_va = resources.kick_record_va; + self.pb_slot = resources.pb_slot; + self.pb_va = resources.pb_va; + self.scene_va = resources.scene_va; + self.scene_end_va = resources.scene_end_va; + self.checkpoints = TA_CHECKPOINTS; + self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); + self.registers.set(array_va, ®isters)?; + self.tpc_va = resources.tpc_va; + self.tpc_size = resources.tpc_size; + self.fragment_qid = args.fragment.qid.into(); + self.stamp_value = stamp_value; + self.deflake_2a0_offset = deflake_2a0_offset; + self.predecessor_id = resources.ta_ids.predecessor; + self.unk_876 = Self::UNK_876; + self.sampler_heap_va = pass.sampler_heap_va; + self.sampler_count = pass.sampler_count; + self.sampler_max = sampler_max; + self.unk_892 = Self::UNK_892; + self.stamp_va = args.ta.stamp_va; + self.aux_stamp_va = args.ta.aux_stamp_va; + self.stamp_value_2 = stamp_value; + self.qid = args.ta.qid.into(); + self.unk_8c2 = Self::UNK_8C2; + self.current_id = resources.ta_ids.current.into(); + self.ordinal = args.ordinal as u32; + self.timestamp_start_va = resources.timestamp_va; + self.unk_932 = Self::UNK_932; + self.free_list_control_va = resources.free_list_control_va; + self.unk_93c = Self::UNK_93C; + if args.ordinal == 0 { + self.unk_93e = Self::UNK_93E_FIRST_KICK; + } + self.status_va = resources.ta_status_va; + self.context_generation = resources.context_generation; + Ok(()) + } + + /// Links the pass to the timestamps its two kicks are published at. + /// + /// Must be called before the descriptor is published. `pass_seq` is the + /// low byte of the logical queue's render-pass count (one for its first + /// pass); `fragment_busy` tells whether the fragment queue had not yet + /// completed every earlier kick. + pub(crate) fn set_completion_link( + &mut self, + ta: KickTimestamp, + fragment: KickTimestamp, + pass_seq: u8, + fragment_busy: bool, + ) { + self.fragment_busy = fragment_busy as u32; + self.fragment_kick = fragment.get(); + self.pass_seq = [pass_seq; 2]; + self.completion = ta.completion_value(pass_seq); + } + + /// Returns the register arrays a tiling kick binds; `descriptor_va` is the + /// descriptor's address as seen by the pass's VM. + pub(crate) fn register_bindings( + descriptor_va: u64, + ) -> Result<[Option; 4]> { + let array = RegisterArrayBinding::new( + offset_va(descriptor_va, Self::ARRAY_OFFSET)?, + TA_CHECKPOINTS[0] as u8, + )?; + Ok([Some(array), None, None, None]) + } +} + +/// Fragment (3D) work descriptor (tag 1). +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct FragmentDescriptor { + /// [`CommandTag::Fragment`]. + pub(crate) tag: u32, + /// Twice the pair's submission ordinal. + pub(crate) submit_sequence: u64, + /// UAT context of the pass. + pub(crate) context_id: u32, + pub(crate) unk_10: [u8; 0x10], + /// GPU address of the pass's kick record. + pub(crate) kick_record_va: u64, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// GPU address of the parameter-buffer scene entry. + pub(crate) scene_va: u64, + /// GPU address following the last scene entry. + pub(crate) scene_end_va: u64, + /// GPU address of the tile map. + pub(crate) tilemap_va: u64, + /// Multisample control. + pub(crate) msaa_control: u64, + /// Samples per pixel. + pub(crate) samples: u32, + /// Macro-tile dimensions in utiles: height in bits 15:0, width above. + pub(crate) macro_size: u32, + pub(crate) unk_58: [u8; 0x10], + /// Merge upper bound in X. + pub(crate) merge_upper_x: u32, + /// Merge upper bound in Y. + pub(crate) merge_upper_y: u32, + pub(crate) unk_70: u64, + /// Number of tiles. + pub(crate) tiles: u64, + /// Checkpoints into the main register array. + pub(crate) checkpoints: [u16; 3], + pub(crate) unk_86: u16, + /// Entry count of the main register array. + pub(crate) entries: u16, + pub(crate) unk_8a: [u8; 6], + /// Render-pass sequence of the pass. + pub(crate) pass_seq: u8, + pub(crate) unk_91: [u8; 0xf], + /// Main, partial-store, partial-resume and partial-load register arrays. + pub(crate) arrays: [RegisterArray; 4], + /// GPU address of the depth-bias array. + pub(crate) depth_bias_va: u64, + pub(crate) unk_1d28: u64, + /// GPU address of the scissor array. + pub(crate) scissor_va: u64, + pub(crate) unk_1d38: u64, + /// GPU address of the occlusion-query results. + pub(crate) occlusion_query_va: u64, + pub(crate) unk_1d48: [u8; 0x130], + /// Background resource specifier. + pub(crate) bg_rsrc_spec: u64, + /// GPU address of the background program. + pub(crate) bg_va: u64, + pub(crate) unk_1e88: [u8; 0x20], + /// Partial-render background resource specifier. + pub(crate) partial_bg_rsrc_spec: u64, + /// GPU address of the partial-render background program. + pub(crate) partial_bg_va: u64, + pub(crate) unk_1eb8: u64, + /// ZLS swizzle. + pub(crate) zls_swizzle: u32, + pub(crate) unk_1ec4: [u8; 0x74], + /// Tile-buffer blocks per utile. + pub(crate) tib_blocks: u64, + /// Auxiliary framebuffer flags. + pub(crate) aux_fb_flags: u64, + /// Width of the render target. + pub(crate) width: u32, + /// Height of the render target. + pub(crate) height: u32, + /// [`UNK_15049`]. + pub(crate) unk_1f50: u64, + /// Tile configuration. + pub(crate) tile_config: u64, + /// Ends with the end-of-tile resource specifier, which is zero. + pub(crate) unk_1f60: [u8; 0x1c], + /// GPU address of the end-of-tile program. + pub(crate) eot_va: u64, + /// Ends with the partial end-of-tile resource specifier, which is zero. + pub(crate) unk_1f84: [u8; 0x18], + /// GPU address of the partial-render end-of-tile program. + pub(crate) partial_eot_va: u64, + pub(crate) unk_1fa4: u32, + /// Depth clear value. + pub(crate) depth_clear: u32, + /// Tile-buffer blocks in bits 63:33 above a constant. + pub(crate) unk_1fac: u64, + pub(crate) unk_1fb4: [u8; 0x150], + /// See [`RenderResources::unk_2104`]. + pub(crate) unk_2104: u32, + /// Predecessor object ID of the fragment stage. + pub(crate) predecessor_id: u32, + /// See [`RenderResources::unk_210c`]. + pub(crate) unk_210c: u32, + pub(crate) unk_2110: u32, + pub(crate) unk_2114: [u8; 0x10], + pub(crate) unk_2124: [u32; 2], + pub(crate) unk_212c: [u8; 0x14], + /// GPU address of the fragment queue's completion stamp word. + pub(crate) stamp_va: u64, + /// GPU address of the fragment queue's auxiliary stamp word. + pub(crate) aux_stamp_va: u64, + /// Completion stamp value of the pass. + pub(crate) stamp_value: u32, + /// QID of the fragment queue. + pub(crate) qid: u32, + pub(crate) unk_2158: u32, + /// Non-zero on the first kick of a queue pair. + pub(crate) first_kick: u32, + /// Completion value of the fragment kick (see + /// [`KickTimestamp::completion_value`]). + pub(crate) completion: u64, + /// Current object ID of the fragment stage. + pub(crate) current_id: u64, + /// Submission ordinal of the pair. + pub(crate) ordinal: u32, + pub(crate) unk_2174: [u8; 0x24], + /// GPU address the pass start timestamp is written to. + pub(crate) timestamp_start_va: u64, + /// GPU address the pass end timestamp is written to. + pub(crate) timestamp_end_va: u64, + pub(crate) unk_21a8: [u8; 0x24], + pub(crate) unk_21cc: u8, + pub(crate) unk_21cd: u8, + /// GPU address of the USC free list's shared control object. + pub(crate) free_list_control_va: u64, + pub(crate) unk_21d6: u8, + /// GPU address of the fragment status word, as seen by the pass's VM. + pub(crate) status_va: u64, + /// Firmware alias of the fragment status word. + pub(crate) status_fw_va: u64, + /// Allocation generation of the UAT context. + pub(crate) context_generation: u8, + pub(crate) unk_21e8: [u8; 0x20], + pub(crate) unk_2208: u8, + pub(crate) unk_2209: [u8; 3], + pub(crate) unk_220c: u8, + pub(crate) unk_220d: [u8; 0x20], + pub(crate) unk_222d: u8, + pub(crate) unk_222e: [u8; 0x12], +} + +static_assert!(size_of::() == 0x2240); +static_assert!(core::mem::offset_of!(FragmentDescriptor, checkpoints) == 0x80); +static_assert!(core::mem::offset_of!(FragmentDescriptor, pass_seq) == 0x90); +static_assert!(core::mem::offset_of!(FragmentDescriptor, arrays) == 0xa0); +static_assert!(core::mem::offset_of!(FragmentDescriptor, depth_bias_va) == 0x1d20); +static_assert!(core::mem::offset_of!(FragmentDescriptor, bg_rsrc_spec) == 0x1e78); +static_assert!(core::mem::offset_of!(FragmentDescriptor, tib_blocks) == 0x1f38); +static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_1f50) == 0x1f50); +static_assert!(core::mem::offset_of!(FragmentDescriptor, eot_va) == 0x1f7c); +static_assert!(core::mem::offset_of!(FragmentDescriptor, depth_clear) == 0x1fa8); +static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_2104) == 0x2104); +static_assert!(core::mem::offset_of!(FragmentDescriptor, stamp_va) == 0x2140); +static_assert!(core::mem::offset_of!(FragmentDescriptor, completion) == 0x2160); +static_assert!(core::mem::offset_of!(FragmentDescriptor, timestamp_start_va) == 0x2198); +static_assert!(core::mem::offset_of!(FragmentDescriptor, free_list_control_va) == 0x21ce); +static_assert!(core::mem::offset_of!(FragmentDescriptor, status_va) == 0x21d7); +static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_222d) == 0x222d); + +// SAFETY: `FragmentDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for FragmentDescriptor {} + +impl FragmentDescriptor { + /// Offset of register array `index` within the descriptor. + const fn array_offset(index: usize) -> u64 { + (core::mem::offset_of!(Self, arrays) + index * size_of::()) as u64 + } + + /// Returns the register arrays a fragment kick binds; `descriptor_va` is + /// the descriptor's address as seen by the pass's VM. + pub(crate) fn register_bindings( + descriptor_va: u64, + ) -> Result<[Option; 4]> { + let array = |index, checkpoint: usize| -> Result> { + let va = offset_va(descriptor_va, Self::array_offset(index))?; + Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) + }; + Ok([ + array(0, FRAGMENT_CHECKPOINTS[0] as usize)?, + array(1, PARTIAL_STORE_REGISTERS)?, + array(2, PARTIAL_RESUME_REGISTERS)?, + array(3, PARTIAL_LOAD_REGISTERS)?, + ]) + } +} + +/// A fragment descriptor followed by room for an MCache table that does not +/// fit its first register array. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct FragmentSlot { + /// The descriptor. + pub(crate) descriptor: FragmentDescriptor, + /// MCache table of more than [`MCACHE_INLINE_RANGES`] ranges. + pub(crate) mcache: [McacheRange; MCACHE_RANGES_MAX], + pub(crate) pad_2350: [u8; 0x30], +} + +static_assert!(size_of::() == FRAGMENT_SLOT_SIZE); + +impl FragmentSlot { + const INLINE_MCACHE_OFFSET: u64 = + FragmentDescriptor::array_offset(0) + core::mem::offset_of!(RegisterArray, mcache) as u64; + const SPILL_MCACHE_OFFSET: u64 = core::mem::offset_of!(Self, mcache) as u64; + const UNK_2110: u32 = u32::MAX; + const UNK_2124: [u32; 2] = [1, 1]; + const UNK_21CC: u8 = 0x53; + const UNK_21D6: u8 = 1; + const UNK_2208: u8 = 1; + const UNK_220C: u8 = 1; + const UNK_222D: u8 = 1; + const UNK_1FAC_BASE: u64 = 0x300; + const UNK_1FAC_TIB_SHIFT: u32 = 33; + + /// Writes the complete descriptor except the completion link and the + /// MCache table. + /// + /// The spill table is left as it is: a kick entry names it only after + /// [`Self::set_mcache`] rewrote it, so stale bytes there are never read. + pub(crate) fn write(&mut self, args: &RenderArgs<'_>) -> Result { + let pass = args.pass; + let resources = args.resources; + let geometry = Geometry::new(pass)?; + let words = PassWords::new(pass)?; + let base = args.fragment.descriptor_va; + let array_va = |index| offset_va(base, FragmentDescriptor::array_offset(index)); + let arrays = [array_va(0)?, array_va(1)?, array_va(2)?, array_va(3)?]; + let timestamp_end_va = timestamp_end_va(resources.timestamp_va)?; + let desc = &mut self.descriptor; + + clear(desc); + desc.tag = CommandTag::Fragment as u32; + desc.submit_sequence = args.ordinal.wrapping_mul(2); + desc.context_id = resources.context_id.into(); + desc.kick_record_va = resources.kick_record_va; + desc.pb_va = resources.pb_va; + desc.scene_va = resources.scene_va; + desc.scene_end_va = resources.scene_end_va; + desc.tilemap_va = resources.tilemap_va; + desc.msaa_control = pass.msaa_control; + desc.samples = 1 << pass.samples_log2; + desc.macro_size = geometry.macro_size as u32; + desc.merge_upper_x = pass.merge_upper_x; + desc.merge_upper_y = pass.merge_upper_y; + desc.tiles = geometry.tiles_x * geometry.tiles_y; + desc.checkpoints = FRAGMENT_CHECKPOINTS; + desc.entries = FRAGMENT_REGISTERS as u16; + desc.arrays[0].set(arrays[0], &fragment_registers(args, &geometry, &words))?; + desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; + desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; + desc.arrays[3].set(arrays[3], &partial_load_registers(pass, &words))?; + desc.depth_bias_va = pass.depth_bias_va; + desc.scissor_va = pass.scissor_va; + desc.occlusion_query_va = pass.occlusion_query_va; + desc.bg_rsrc_spec = words.bg_rsrc_spec; + desc.bg_va = words.bg_va; + desc.partial_bg_rsrc_spec = words.partial_bg_rsrc_spec; + desc.partial_bg_va = words.partial_bg_va; + desc.zls_swizzle = ZLS_SWIZZLE; + desc.tib_blocks = words.tib_blocks; + desc.aux_fb_flags = words.aux_fb_flags; + desc.width = pass.width; + desc.height = pass.height; + desc.unk_1f50 = UNK_15049; + desc.tile_config = words.tile_config; + desc.eot_va = words.eot_va; + desc.partial_eot_va = words.partial_eot_va; + desc.depth_clear = pass.depth_clear; + desc.unk_1fac = words.tib_blocks << Self::UNK_1FAC_TIB_SHIFT | Self::UNK_1FAC_BASE; + desc.unk_2104 = resources.unk_2104; + desc.predecessor_id = resources.fragment_ids.predecessor; + desc.unk_210c = resources.unk_210c; + desc.unk_2110 = Self::UNK_2110; + desc.unk_2124 = Self::UNK_2124; + desc.stamp_va = args.fragment.stamp_va; + desc.aux_stamp_va = args.fragment.aux_stamp_va; + desc.stamp_value = args.stamp_value(); + desc.qid = args.fragment.qid.into(); + desc.first_kick = (args.ordinal == 0) as u32; + desc.current_id = resources.fragment_ids.current.into(); + desc.ordinal = args.ordinal as u32; + desc.timestamp_start_va = resources.timestamp_va; + desc.timestamp_end_va = timestamp_end_va; + desc.unk_21cc = Self::UNK_21CC; + desc.free_list_control_va = resources.free_list_control_va; + desc.unk_21d6 = Self::UNK_21D6; + desc.status_va = resources.fragment_status_va; + desc.status_fw_va = resources.fragment_status_fw_va; + desc.context_generation = resources.context_generation; + desc.unk_2208 = Self::UNK_2208; + desc.unk_220c = Self::UNK_220C; + desc.unk_222d = Self::UNK_222D; + Ok(()) + } + + /// Links the pass to the timestamp of its fragment kick. + /// + /// Must be called before the descriptor is published; see + /// [`TaDescriptor::set_completion_link`]. + pub(crate) fn set_completion_link(&mut self, fragment: KickTimestamp, pass_seq: u8) { + self.descriptor.pass_seq = pass_seq; + self.descriptor.completion = fragment.completion_value(pass_seq); + } + + /// Writes the MCache table of the pass: the auxiliary framebuffer at + /// `aux_fb_va`, then `attachments`, translated in UAT context `context`. + /// `slot_va` is the GPU address of this slot as seen by the pass's VM. + /// + /// Returns the table for the fragment kick entry, or `None` when the pass + /// has no attachments. + pub(crate) fn set_mcache( + &mut self, + slot_va: u64, + aux_fb_va: u64, + attachments: &[WriteRange], + context: u8, + ) -> Result> { + if attachments.is_empty() { + return Ok(None); + } + let count = attachments.len() + 1; + if count > MCACHE_RANGES_MAX { + return Err(EINVAL); + } + let (offset, ranges) = if count <= MCACHE_INLINE_RANGES { + ( + Self::INLINE_MCACHE_OFFSET, + &mut self.descriptor.arrays[0].mcache[..count], + ) + } else { + (Self::SPILL_MCACHE_OFFSET, &mut self.mcache[..count]) + }; + let table = McacheTable::new(offset_va(slot_va, offset)?, count)?; + let aux_fb = WriteRange { + va: aux_fb_va, + size: AUX_FB_SIZE, + }; + ranges[0] = McacheRange::new(&aux_fb, McacheClass::AuxFramebuffer, context)?; + for (range, attachment) in ranges[1..].iter_mut().zip(attachments) { + *range = McacheRange::new(attachment, McacheClass::Attachment, context)?; + } + Ok(Some(table)) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/render/storage.rs b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs new file mode 100644 index 0000000000000..e1e6a10949fb4 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Auxiliary render records and their initial images. Live event counters +//! are accessed individually by the queue owner after publication. + +use super::super::{buffer, clear, context, initdata}; +use kernel::prelude::*; + +pub(crate) const SUPPORT_SIZE: usize = 0x40000; +pub(crate) const EVENT_SLOTS: usize = 0x4000; +pub(crate) const EVENT_COUNT: usize = 35; +pub(crate) const STATE_SIZE: usize = 0x18000; +pub(crate) const DEFLAKE: usize = 0x4000; +pub(crate) const TA_STATUS: usize = 0x8000; +pub(crate) const AUXILIARY: usize = 0x10000; +pub(crate) const AUXILIARY_SIZE: usize = 0x8000; + +#[repr(C)] +pub(crate) struct Event { + pub(crate) node: context::WorkNode, + unk_c8: [u8; 0x38], +} + +#[repr(C)] +pub(crate) struct Support { + secondary_index: [u64; 8], + unk_40: [u8; 0x3fc0], + pub(crate) event_slots: [u32; 0x1000], + scene_slots: [u32; 0x1000], + shared_slots: [u32; 0x1000], + flag: u32, + unk_10004: [u8; 0x3ffc], + pub(crate) events: [Event; EVENT_COUNT], + unk_16300: [u8; 0x1d00], + scenes: [buffer::Scene; 79], + unk_1a780: [u8; 0x1880], + shared: buffer::State, + unk_1c0c0: [u8; 0x7f40], + control: initdata::FreeListControl, + unk_24068: [u8; 0x3f98], + submitted: u32, + unk_28004: [u8; 0x3ffc], + scheduler: context::Scheduler, + unk_30000: [u8; 0x10000], +} +// SAFETY: The entire graph is integer storage; all nested types are Zeroable. +unsafe impl Zeroable for Support {} +impl Support { + pub(crate) fn init( + &mut self, + address: u64, + primary_index: u64, + operand_table: u64, + page_list: u64, + word: u32, + ) -> Result { + clear(self); + address.checked_add(SUPPORT_SIZE as u64).ok_or(EOVERFLOW)?; + self.secondary_index = [0x11, 0x16, 0x1b, 0x20, 0x25, 0x2a, 0x3c, 0x41]; + self.event_slots[1] = 1; + self.shared_slots[0] = 8; + self.shared_slots[1] = 8; + self.shared_slots[0x60 / 4] = 1; + self.flag = 1; + for (index, event) in self.events.iter_mut().enumerate() { + event.node.submitted = address + EVENT_SLOTS as u64 + index as u64 * 4; + if index == 1 { + event.node.threshold = 0x50; + event.node.word = word; + } + } + for (index, scene) in self.scenes.iter_mut().enumerate() { + scene.metric_client = 0x10_0008_0004 + index as u64 * 4; + scene.metric_fw = address + 0x8004 + index as u64 * 4; + let phase = index % buffer::SCRATCH_SLOTS; + scene.scratch = if phase == buffer::SCRATCH_SLOTS - 1 { + 0x178000 + } else { + 0x178020 + phase as u64 * buffer::SCRATCH_STRIDE + }; + scene.statistics = address + 0xc040; + if index == 0 { + scene.generation = 1; + } + } + let shared = &mut self.shared; + shared.page_list_fw = primary_index; + shared.page_list_client = 0x10_0019_0000; + shared.page_list_size = 0x10000; + shared.pages = 0x20; + shared.block_capacity = 0xc18; + shared.producer = 8; + shared.block_table = address; + shared.control = address + 0xc000; + shared.last_page = 0x1f; + shared.block_size = 0x20000; + shared.counter = address + 0x10000; + shared.maximum_size = 0x3060; + shared.normal_maximum_size = 0x1020; + shared.discard = 0x180000; + self.control = initdata::FreeListControl::new(&initdata::FreeListArgs { + buffer_slot: 0, + class: initdata::FreeListClass::Two, + page_list_va: page_list, + run_list_va: operand_table, + blocks: 25, + state_va: address + 0x28000, + }); + self.control.run_cursor = 0xe0; + self.submitted = 1; + self.scheduler.init(); + Ok(()) + } +} + +/// The viewport words consumed from the deflake page at +0x900. +#[repr(C)] +#[derive(Copy, Clone)] +pub(crate) struct Viewport { + unk_0: u32, + tiles_x: u32, + tiles_y: u32, + unk_c: u32, + scale_x: u32, + translate_x: u32, + scale_y: u32, + translate_y: u32, + unk_20: u32, + scale_z: u32, +} +impl Viewport { + /// Refresh only defined words; firmware-owned gaps retain their contents. + pub(crate) fn apply(&self, destination: &mut Self) { + destination.unk_0 = self.unk_0; + destination.tiles_x = self.tiles_x; + destination.tiles_y = self.tiles_y; + destination.scale_x = self.scale_x; + destination.translate_x = self.translate_x; + destination.scale_y = self.scale_y; + destination.translate_y = self.translate_y; + destination.scale_z = self.scale_z; + } + pub(crate) fn new(width: u32, height: u32) -> Result { + fn half(value: u32) -> Result { + if value == 0 { + return Err(EINVAL); + } + let highest = 31 - value.leading_zeros(); + let shift = 23u32.checked_sub(highest).ok_or(EINVAL)?; + Ok(((highest + 126) << 23) | ((value << shift) & 0x7f_ffff)) + } + let x = half(width)?; + let y = half(height)?; + Ok(Self { + unk_0: 0xc00, + tiles_x: 0x8000_0000 | (width.checked_add(31).ok_or(EOVERFLOW)? / 32 - 1), + tiles_y: height.checked_add(31).ok_or(EOVERFLOW)? / 32 - 1, + unk_c: 0, + scale_x: x, + translate_x: x, + scale_y: y, + translate_y: y | 0x8000_0000, + unk_20: 0, + scale_z: 0x3f80_0000, + }) + } +} + +const _: () = { + assert!(core::mem::size_of::() == 0x100); + assert!(core::mem::size_of::() == SUPPORT_SIZE); + assert!(core::mem::offset_of!(Support, events) == 0x14000); + assert!(core::mem::offset_of!(Support, scenes) == 0x18000); + assert!(core::mem::offset_of!(Support, shared) == 0x1c000); + assert!(core::mem::offset_of!(Support, control) == 0x24000); + assert!(core::mem::offset_of!(Support, scheduler) == 0x2c000); + assert!(core::mem::size_of::() == 0x28); +}; diff --git a/drivers/gpu/drm/asahi/g17/hazard.rs b/drivers/gpu/drm/asahi/g17/hazard.rs new file mode 100644 index 0000000000000..20daaab85b832 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/hazard.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Attachment overlap ordering across physical render pairs and shared images. +//! Tile stores can overlap even when the application's pixel regions do not. +//! Keep independent attachments concurrent; order only conflicting live spans. + +use super::job::Order; + +const MAX_SPANS: usize = 24; +pub(crate) const WAIT_MS: u32 = 1000; + +#[derive(Clone, Copy)] +struct Span { + key: u64, + start: u64, + end: u64, +} + +/// Up to sixteen attachment ranges and eight shared-object ranges. +pub(crate) struct Footprint { + vm: u64, + spans: [Span; MAX_SPANS], + count: usize, + shared: bool, +} + +impl Footprint { + pub(crate) fn new(vm: u64) -> Self { + Self { + vm, + spans: [Span { + key: 0, + start: 0, + end: 0, + }; MAX_SPANS], + count: 0, + shared: false, + } + } + + fn push(&mut self, key: u64, start: u64, end: u64) { + if start < end && self.count < MAX_SPANS { + self.spans[self.count] = Span { key, start, end }; + self.count += 1; + } + } + + pub(crate) fn push_vm(&mut self, start: u64, end: u64) { + self.push(self.vm << 1, start, end); + } + + /// An aligned GEM object address is an identity, never dereferenced here. + pub(crate) fn push_object(&mut self, object: usize, start: u64, end: u64) { + let before = self.count; + self.push(object as u64 | 1, start, end); + self.shared |= self.count != before; + } + + fn overlaps(&self, other: &Self) -> bool { + if self.vm != other.vm && !(self.shared && other.shared) { + return false; + } + self.spans[..self.count].iter().any(|a| { + other.spans[..other.count] + .iter() + .any(|b| a.key == b.key && a.start < b.end && b.start < a.end) + }) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Plan { + Any, + Pair(u8), + Wait, +} + +pub(crate) struct Live<'a> { + pub(crate) owner: u64, + pub(crate) sequence: u64, + pub(crate) slot: Option, + pub(crate) ticket: u64, + pub(crate) footprint: &'a Footprint, + pub(crate) quarantined: bool, +} + +/// A queue prefix already orders covered predecessors. Pending arrivals order +/// different owners fairly without imposing order on unrelated attachments. +pub(crate) fn plan<'a>( + owner: u64, + prefix: Option, + ticket: Option, + footprint: &Footprint, + live: impl IntoIterator>, +) -> Plan { + let mut selected = None; + for entry in live { + if entry.quarantined || !footprint.overlaps(entry.footprint) { + continue; + } + let Some(slot) = entry.slot else { + if entry.owner != owner && ticket.is_none_or(|mine| entry.ticket < mine) { + return Plan::Wait; + } + continue; + }; + if entry.owner != owner { + return Plan::Wait; + } + if Order::contains(prefix, entry.sequence) { + continue; + } + match selected { + None => selected = Some(slot), + Some(prior) if prior == slot => {} + Some(_) => return Plan::Wait, + } + } + selected.map_or(Plan::Any, Plan::Pair) +} diff --git a/drivers/gpu/drm/asahi/g17/ids.rs b/drivers/gpu/drm/asahi/g17/ids.rs new file mode 100644 index 0000000000000..6912c6501527a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/ids.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware-visible object generations shared by contexts, pools and render work. + +use super::fw::ObjectIds; +use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::prelude::*; + +/// Module-lifetime identities remain unique across retained graphs and device rebinds. +pub(crate) static OBJECTS: Counter = Counter::new(); + +pub(crate) struct Counter(AtomicU32); + +impl Counter { + const fn new() -> Self { + Self(AtomicU32::new(1)) + } + + fn reserve(&self, count: u32) -> Result { + let mut old = self.0.load(Ordering::Acquire); + loop { + let next = old.checked_add(count).ok_or(EOVERFLOW)?; + match self + .0 + .compare_exchange_weak(old, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return Ok(old), + Err(current) => old = current, + } + } + } + + pub(crate) fn object(&self) -> Result { + let id = self.reserve(1)?; + if id == 0 { + Err(EOVERFLOW) + } else { + Ok(id) + } + } + + /// Allocate consecutive current IDs atomically, fragment first and tiling + /// second. Both name the same retained predecessor of their render graph. + pub(crate) fn render_pair(&self, predecessor: u32) -> Result<(ObjectIds, ObjectIds)> { + let fragment = self.reserve(2)?; + if predecessor == 0 || fragment == 0 { + return Err(EOVERFLOW); + } + Ok(( + ObjectIds { + predecessor, + current: fragment, + }, + ObjectIds { + predecessor, + current: fragment + 1, + }, + )) + } +} diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs new file mode 100644 index 0000000000000..f8160f0faa36c --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -0,0 +1,1305 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Allocates, initializes and retains the graph shared by the two firmware instances. +//! +//! The owner is installed in the firmware session before construction starts. Even if an +//! allocation fails, the session stops both coprocessors before releasing any published memory. + +use super::{ + fw::initdata as abi, + object::{CpuMap, KernelObject, Placement}, + Config, Role, +}; +use crate::{driver::AsahiDevice, hw::t8140, mmu}; +use core::sync::atomic::{fence, AtomicU32, Ordering}; +use kernel::{c_str, prelude::*}; +use t8140::{kernel_window as high, low}; + +const EVENT_RING_RECHECK: u32 = 256; + +const INDEX_SIZE: usize = 0x10000; +const CONTEXT_SIZE: usize = 0x20000; +const REGION_A_SIZE: usize = core::mem::size_of::(); +const REGION_C_SIZE: usize = 0x1000; + +const COMPUTE_BLOCKS_ALLOCATED: usize = 23; +const COMPUTE_BLOCKS_MAPPED: usize = 22; +const COMPUTE_BLOCKS_POPULATED: usize = 21; +const COMPUTE_CONTROL_BLOCKS: u32 = 25; +const COMPUTE_CONTROL_CURSOR: u32 = 0x18; +const COMPUTE_RUN_ADDRESS_MASK: u64 = 0x0000_ffff_ffff_f000; + +/// Unpublished device-wide compute backing, prepared without the device mutex. The cache +/// identity belongs to this allocation, not to any one physical queue or client VM. +pub(super) struct ComputeShared { + id: u64, + pages: KernelObject, + runs: KernelObject, + blocks: KVec, +} + +impl ComputeShared { + pub(super) fn new(alloc: &super::object::Allocator<'_>) -> Result { + let pages = + KernelObject::backing(alloc.dev, low::RENDER_PAGE_LIST_SIZE, CpuMap::WriteCombined)?; + let runs = + KernelObject::backing(alloc.dev, low::RENDER_RUN_LIST_SIZE, CpuMap::WriteCombined)?; + let mut blocks = KVec::with_capacity(COMPUTE_BLOCKS_ALLOCATED, GFP_KERNEL)?; + for _ in 0..COMPUTE_BLOCKS_ALLOCATED { + blocks.push( + KernelObject::backing(alloc.dev, abi::FREE_LIST_BLOCK_SIZE, CpuMap::WriteCombined)?, + GFP_KERNEL, + )?; + } + let id = u64::from(super::ids::OBJECTS.object()?); + Ok(Self { + id, + pages, + runs, + blocks, + }) + } +} + +/// A device-global scheduler, retained independently of whichever physical queue first +/// needed it. Its generation reservation precedes later graph allocation failures. +pub(super) struct ComputeScheduler { + object: KernelObject, +} + +impl ComputeScheduler { + pub(super) fn new(alloc: &super::object::Allocator<'_>) -> Result { + let mut object = alloc.kernel( + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + object.initialize::(0, |state| { + state.init(); + Ok(()) + })?; + fence(Ordering::SeqCst); + // Scheduler creation reserves a generation even when only its address is used by work. + super::ids::OBJECTS.object()?; + Ok(Self { object }) + } +} + +/// Checked views used only within one publication call. Every fallible operation completes +/// before either member of a paired render publication is copied into shared memory. +struct WorkPublication<'a> { + record: &'a super::fw::channels::WorkSlot, + destination: *mut u8, + producer: &'a AtomicU32, + next: u32, +} + +impl WorkPublication<'_> { + fn copy_record(&self) { + // SAFETY: work_publication checked the complete destination range. The record is a + // fully initialized padding-free ABI value, and the device mutex owns publication. + unsafe { + core::ptr::copy_nonoverlapping( + self.record as *const _ as *const u8, + self.destination, + core::mem::size_of_val(self.record), + ); + } + } + + fn commit(self) -> u32 { + self.copy_record(); + fence(Ordering::SeqCst); + self.producer.store(self.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + self.next + } +} + +/// Session-long storage and mappings. Aliases are released before their backing objects. +pub(super) struct InitData { + // Live publication uses interior writes under the device mutex. This owner is Send but not + // Sync, so safe host code cannot issue those writes concurrently through shared references. + _single_publisher: core::marker::PhantomData>, + aliases: KVec, + objects: KVec, + roots: usize, + cluster: usize, + bundle: usize, + region_c: usize, + qos: usize, + pb: usize, + page_pool: usize, + compute_global: usize, + boot_control: usize, + boot_state: usize, + metrics_fw_va: u64, + compute_shared: Option, + compute_scheduler: Option, + compute_active: core::cell::Cell, + event_ring_checked: [core::cell::Cell<(u64, u64, u32)>; 2], + completion_rings: [usize; 2], + stamps: [usize; 2], + render_pages: usize, + render_runs: usize, + render_blocks: core::ops::Range, +} + +impl InitData { + /// Empty construction state, retained even if graph construction fails. + pub(super) fn new() -> Self { + Self { + _single_publisher: core::marker::PhantomData, + aliases: KVec::new(), + objects: KVec::new(), + roots: 0, + cluster: 0, + bundle: 0, + region_c: 0, + qos: 0, + pb: 0, + page_pool: 0, + compute_global: 0, + boot_control: 0, + boot_state: 0, + metrics_fw_va: 0, + compute_shared: None, + compute_scheduler: None, + compute_active: core::cell::Cell::new(false), + event_ring_checked: [const { core::cell::Cell::new((0, 0, 0)) }; 2], + completion_rings: [0; 2], + stamps: [0; 2], + render_pages: 0, + render_runs: 0, + render_blocks: 0..0, + } + } + + pub(super) fn metrics_fw_va(&self) -> Result { + if self.metrics_fw_va == 0 { + return Err(EIO); + } + Ok(self.metrics_fw_va) + } + + pub(super) fn pb_descriptor_table(&self) -> Result<&KernelObject> { + self.object(self.pb) + } + + pub(super) fn page_pool_descriptor_table(&self) -> Result<&KernelObject> { + self.object(self.page_pool) + } + + pub(super) fn stamp_addresses(&self, qid: u8) -> Result<[u64; 2]> { + if qid > super::fw::kick::QID_MAX { + return Err(EINVAL); + } + let offset = u64::from(qid) * core::mem::size_of::() as u64; + Ok([ + self.object(self.stamps[0])?.gpu_va() + offset, + self.object(self.stamps[1])?.gpu_va() + offset, + ]) + } + + /// The caller reads both engine lanes before its acquire fence. Page A is the completed + /// GPU-maintenance witness; page B stays under firmware ownership. + pub(super) fn stamp(&self, qid: u8) -> Result<&AtomicU32> { + if qid > super::fw::kick::QID_MAX { + return Err(EINVAL); + } + self.object(self.stamps[0])? + .word(usize::from(qid) * core::mem::size_of::()) + } + + /// Initialize a fresh physical compute QID's row before exposing its queue. An installed + /// or rebound queue keeps the row's firmware-owned counters and must never call this again. + pub(super) fn initialize_compute_pool(&self, slot: u16, page_list_va: u64) -> Result { + if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { + return Err(EINVAL); + } + let entry = abi::PagePoolDescriptor::compute(page_list_va); + let size = core::mem::size_of::(); + let offset = usize::from(slot) * size; + let object = self.object(self.page_pool)?; + let pointer = object.pointer(offset, size)?; + let page_list = object.dword(offset)?; + let counters = + object.dword(offset + core::mem::offset_of!(abi::PagePoolDescriptor, counters))?; + let count = + object.dword(offset + core::mem::offset_of!(abi::PagePoolDescriptor, page_count))?; + // SAFETY: the whole fresh row was bounds checked, the QID has never been published, + // and the device mutex excludes another host publisher. No other row is touched. + unsafe { + pointer.write_bytes(0, size); + } + page_list.store(entry.page_list, Ordering::Relaxed); + counters.store(entry.counters, Ordering::Relaxed); + count.store(entry.page_count, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } + + pub(super) fn qos(&self) -> Result<&KernelObject> { + self.object(self.qos) + } + + pub(super) fn has_compute_scheduler(&self) -> bool { + self.compute_scheduler.is_some() + } + + /// Only the reserved singleton builder may install a scheduler. Retain it before building + /// the first descriptor graph so later construction failure does not allocate it again. + pub(super) fn install_compute_scheduler(&mut self, scheduler: ComputeScheduler) -> Result { + if self.compute_scheduler.is_some() { + return Err(EBUSY); + } + self.compute_scheduler = Some(scheduler); + Ok(()) + } + + pub(super) fn has_compute_shared(&self) -> bool { + self.compute_shared.is_some() + } + + /// Install a prepared aperture once while holding the device mutex. A competing preparer + /// may have installed its allocation first; the redundant unpublished backing then drops. + pub(super) fn install_compute_shared(&mut self, mut shared: ComputeShared) -> Result { + if self.compute_shared.is_some() { + return Ok(()); + } + let state_va = self.object(self.boot_state)?.gpu_va(); + let mut control = abi::FreeListControl::new(&abi::FreeListArgs { + buffer_slot: u32::MAX, + class: abi::FreeListClass::Two, + page_list_va: low::COMPUTE_POOL_PAGE_LIST, + run_list_va: low::COMPUTE_POOL_RUN_LIST, + blocks: COMPUTE_CONTROL_BLOCKS, + state_va, + }); + control.run_cursor = COMPUTE_CONTROL_CURSOR; + self.object_mut(self.boot_control)?.bytes_mut().fill(0); + self.object_mut(self.boot_control)?.write(0, control)?; + self.object_mut(self.boot_state)?.bytes_mut().fill(0); + shared + .pages + .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |pages| { + abi::fill_page_list(pages, (0..COMPUTE_BLOCKS_POPULATED).map(low::compute_block)) + })?; + shared + .runs + .initialize::<[u64; low::RENDER_RUN_LIST_SIZE / 8]>(0, |runs| { + for (index, run) in runs.iter_mut().take(COMPUTE_BLOCKS_POPULATED).enumerate() { + *run = (low::compute_block(index) & COMPUTE_RUN_ADDRESS_MASK) + | ((abi::FREE_LIST_BLOCK_PAGES as u64) << 52); + } + Ok(()) + })?; + fence(Ordering::SeqCst); + self.compute_shared = Some(shared); + Ok(()) + } + + pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { + let shared = self.compute_shared.as_ref().ok_or(EIO)?; + if vm.compute_pool_mappings_match(shared.id) { + return Ok(()); + } + vm.clear_compute_pool_mappings(); + let mut mappings = KVec::with_capacity(26, GFP_KERNEL)?; + mappings.push( + shared + .pages + .map_alias(vm, low::COMPUTE_POOL_PAGE_LIST, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL, + )?; + mappings.push( + shared + .runs + .map_alias(vm, low::COMPUTE_POOL_RUN_LIST, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL, + )?; + for (index, block) in shared.blocks.iter().take(COMPUTE_BLOCKS_MAPPED).enumerate() { + mappings.push( + block.map_alias(vm, low::compute_block(index), mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL, + )?; + } + for slot in 0..2 { + mappings.push( + self.object(self.completion_rings[slot])?.map_alias( + vm, + low::COMPLETION_RINGS[slot], + mmu::PROT_GPU_SHARED_RW, + )?, + GFP_KERNEL, + )?; + } + vm.install_compute_pool_mappings(shared.id, mappings) + } + + /// The descriptor-table views remain immutable for the firmware session. + pub(super) fn validate_descriptor_views(&self) -> Result { + let values = [ + low::PB_DESCRIPTORS, + self.object(self.pb)?.gpu_va(), + low::PAGE_POOL_DESCRIPTORS, + self.object(self.page_pool)?.gpu_va(), + ]; + let base = abi::bundle::MAIN_CONFIG[Role::Primary as usize] + + core::mem::offset_of!(abi::MainConfig, pb_descriptors_low_va); + let bundle = self.object(self.bundle)?; + let mut changed = false; + for (index, value) in values.into_iter().enumerate() { + changed |= bundle.dword(base + index * 8)?.load(Ordering::Relaxed) != value; + } + fence(Ordering::SeqCst); + if changed { + Err(EIO) + } else { + Ok(()) + } + } + + pub(super) fn activate_compute(&self) -> Result { + self.validate_descriptor_views()?; + if self.compute_active.get() { + return Ok(()); + } + let scheduler = self.compute_scheduler.as_ref().ok_or(EIO)?.object.gpu_va(); + let object = self.object(self.compute_global)?; + let state = object + .pointer(0, core::mem::size_of::())? + .cast::(); + // SAFETY: The complete typed view is in the retained page-aligned object. InitData is + // not Sync and its caller holds the sole publication mutex. Each named field is + // naturally aligned and written individually; no reference aliases firmware state. + unsafe { + core::ptr::addr_of_mut!((*state).unk_0).write_volatile(0xff); + core::ptr::addr_of_mut!((*state).unk_8).write_volatile(0xff); + core::ptr::addr_of_mut!((*state).unk_10).write_volatile(0x0f01); + core::ptr::addr_of_mut!((*state).unk_18).write_volatile(0x0f01); + core::ptr::addr_of_mut!((*state).unk_408).write_volatile(2); + core::ptr::addr_of_mut!((*state).unk_40c).write_volatile(2); + core::ptr::addr_of_mut!((*state).unk_604).write_volatile(4); + core::ptr::addr_of_mut!((*state).scheduler_va).write_volatile(scheduler); + core::ptr::addr_of_mut!((*state).unk_c04).write_volatile(0x1000); + core::ptr::addr_of_mut!((*state).unk_e00).write_volatile(7); + core::ptr::addr_of_mut!((*state).unk_e08).write_volatile(0x0000_0001_0e22_618a); + core::ptr::addr_of_mut!((*state).unk_e20).write_volatile(4); + core::ptr::addr_of_mut!((*state).unk_e28).write_volatile(4); + } + self.compute_active.set(true); + Ok(()) + } + + fn work_publication<'a>( + &'a self, + record: &'a super::fw::channels::WorkSlot, + ) -> Result> { + use super::fw::{channels, queue::DataMaster}; + let engine = match record.data_master { + 0 => DataMaster::Tiling, + 1 => DataMaster::Fragment, + 2 => DataMaster::Compute, + _ => return Err(EINVAL), + }; + let channel = channels::work_ring_index(engine); + let [consumer, consumer_b, producer] = self.work_cursors(engine)?; + let next = (producer + 1) % channels::WORK_RING_SLOTS; + if next == consumer || next == consumer_b { + return Err(EINVAL); + } + let destination = self.object(self.bundle)?.pointer( + abi::work_ring_offset(channel) + producer as usize * core::mem::size_of_val(record), + core::mem::size_of_val(record), + )?; + let producer = self.object(self.cluster)?.word( + Self::state_offset(Role::Primary) + + abi::work_state_offset(channel) + + abi::ChannelState::PRODUCER, + )?; + Ok(WorkPublication { + record, + destination, + producer, + next, + }) + } + + /// Two firmware consumers and the host producer, sampled in that order. + pub(super) fn work_cursors(&self, engine: super::fw::queue::DataMaster) -> Result<[u32; 3]> { + use super::fw::channels; + let channel = channels::work_ring_index(engine); + let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); + let state = self.object(self.cluster)?; + let mut values = [0; 3]; + for (value, offset) in values.iter_mut().zip([ + abi::ChannelState::CONSUMER, + abi::ChannelState::UNK_10, + abi::ChannelState::PRODUCER, + ]) { + *value = state.word(base + offset)?.load(Ordering::Relaxed); + } + if values + .iter() + .any(|&value| value >= channels::WORK_RING_SLOTS) + { + return Err(EINVAL); + } + Ok(values) + } + + /// Inner queue records must be visible before this outer publication. The returned + /// producer is the retained consumer witness for this slot. + pub(super) fn publish_work(&self, record: &super::fw::channels::WorkSlot) -> Result { + Ok(self.work_publication(record)?.commit()) + } + + /// Validate both destinations before exposing fragment, then tiling. The accounting token + /// is committed after copying fragment; late tiling state is published before its slot. + /// Deferred pairs retain both slot bodies without exposing either producer. + pub(super) fn publish_work_pair( + &self, + fragment: &super::fw::channels::WorkSlot, + tiling: &super::fw::channels::WorkSlot, + deferred: bool, + commit_accounting: impl FnOnce() -> Result, + publish_late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]> { + use super::fw::queue::DataMaster; + if fragment.data_master != DataMaster::Fragment as u32 + || tiling.data_master != DataMaster::Tiling as u32 + { + return Err(EINVAL); + } + let fragment = self.work_publication(fragment)?; + let tiling = self.work_publication(tiling)?; + fragment.copy_record(); + commit_accounting()?; + fence(Ordering::SeqCst); + if !deferred { + fragment.producer.store(fragment.next, Ordering::Relaxed); + } + publish_late_tiling(); + tiling.copy_record(); + fence(Ordering::SeqCst); + if !deferred { + tiling.producer.store(tiling.next, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok([fragment.next, tiling.next]) + } + + fn retained_work_publication<'a>( + &'a self, + record: &'a super::fw::channels::WorkSlot, + next: u32, + ) -> Result> { + use super::fw::{channels, queue::DataMaster}; + let engine = match record.data_master { + 0 => DataMaster::Tiling, + 1 => DataMaster::Fragment, + _ => return Err(EINVAL), + }; + let channel = channels::work_ring_index(engine); + let producer = self.object(self.cluster)?.word( + Self::state_offset(Role::Primary) + + abi::work_state_offset(channel) + + abi::ChannelState::PRODUCER, + )?; + let current = producer.load(Ordering::Relaxed); + if current >= channels::WORK_RING_SLOTS || next != (current + 1) % channels::WORK_RING_SLOTS + { + return Err(EBUSY); + } + let destination = self.object(self.bundle)?.pointer( + abi::work_ring_offset(channel) + current as usize * core::mem::size_of_val(record), + core::mem::size_of_val(record), + )?; + Ok(WorkPublication { + record, + destination, + producer, + next, + }) + } + + /// Release a retained pair after its control transaction. Revalidate the original producer + /// witnesses before either copy; both bodies precede the fragment and tiling publications. + pub(super) fn publish_retained_work_pair( + &self, + fragment: &super::fw::channels::WorkSlot, + tiling: &super::fw::channels::WorkSlot, + next: [u32; 2], + ) -> Result { + use super::fw::queue::DataMaster; + if fragment.data_master != DataMaster::Fragment as u32 + || tiling.data_master != DataMaster::Tiling as u32 + { + return Err(EINVAL); + } + let fragment = self.retained_work_publication(fragment, next[0])?; + let tiling = self.retained_work_publication(tiling, next[1])?; + fragment.copy_record(); + tiling.copy_record(); + fence(Ordering::SeqCst); + fragment.producer.store(fragment.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + tiling.producer.store(tiling.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } + + /// Snapshot the shared compute completion ring once per service transaction. The queue + /// scanner latches exact identities; firmware retains sole ownership of these slots. + pub(super) fn scan_compute_completions( + &self, + mut record: impl FnMut([u64; 8]) -> Result, + ) -> Result { + let object = self.object(self.completion_rings[0])?; + fence(Ordering::Acquire); + for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { + let mut words = [0; 8]; + for (index, word) in words.iter_mut().enumerate() { + *word = object + .dword(slot * 0x40 + index * 8)? + .load(Ordering::Relaxed); + } + record(words)?; + } + Ok(()) + } + + fn object(&self, index: usize) -> Result<&KernelObject> { + self.objects.get(index).ok_or(EINVAL) + } + + fn object_mut(&mut self, index: usize) -> Result<&mut KernelObject> { + self.objects.get_mut(index).ok_or(EINVAL) + } + + fn add(&mut self, object: KernelObject) -> Result { + let index = self.objects.len(); + self.objects.push(object, GFP_KERNEL)?; + Ok(index) + } + + fn alias(&mut self, index: usize, vm: &mmu::Vm, va: u64, prot: mmu::Prot) -> Result { + let alias = self.object_mut(index)?.map_alias(vm, va, prot)?; + self.aliases.push(alias, GFP_KERNEL)?; + Ok(()) + } + + /// Constructs the graph after the firmware has published the high-root entries. + pub(super) fn build( + &mut self, + dev: &AsahiDevice, + cfg: &Config, + uat: &mmu::Uat, + render: &mmu::Vm, + metrics: &KernelObject, + ) -> Result { + let vm = uat.kernel_vm(); + let lower = uat.kernel_lower_vm(); + let range = uat.geometry().kernel_range(); + let base = range.start; + let dynamic = base + t8140::dynamic::KERNEL_OFFSET..range.end; + let fixed = |offset, size, prot, cpu| { + KernelObject::new(dev, vm, Placement::At(base + offset), size, prot, cpu) + }; + let dynamic_object = |size, align, prot, cpu| { + KernelObject::new( + dev, + vm, + Placement::In(dynamic.clone(), align), + size, + prot, + cpu, + ) + }; + use mmu::{ + PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_PRIV_RW as GFP, + PROT_GPU_FW_SHARED_RW as GFS, PROT_GPU_SHARED_RW as GS, + }; + use CpuMap::{WriteBack as WB, WriteCombined as WC}; + + for window in &t8140::REGISTER_WINDOWS { + let (phys, va, size) = window.mapping().ok_or(EINVAL)?; + self.aliases.push( + vm.map_io(va, phys.try_into()?, size.try_into()?, mmu::PROT_FW_MMIO_RW)?, + GFP_KERNEL, + )?; + } + self.cluster = self.add(fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; + self.compute_global = self.add(fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; + let pb = self.add(fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + self.object_mut(pb)? + .initialize::(0, |table| { + table.init(); + Ok(()) + })?; + let page_pool = self.add(fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + self.object_mut(page_pool)? + .initialize::(0, |table| { + table.init(low::RENDER_PAGE_LIST, low::COMPUTE_POOL_PAGE_LIST); + Ok(()) + })?; + self.pb = pb; + self.page_pool = page_pool; + self.alias(pb, lower, low::PB_DESCRIPTORS, GS)?; + self.alias(page_pool, lower, low::PAGE_POOL_DESCRIPTORS, GS)?; + let index = self.add(fixed(high::INDEX, INDEX_SIZE, GFS, WB)?)?; + self.object_mut(index)? + .initialize::(0, |table| { + table.init(); + Ok(()) + })?; + self.alias(index, lower, low::INDEX, GFS)?; + let fwctl = self.add(fixed(high::FWCTL, abi::FWCTL_SIZE, FS, WB)?)?; + + for (offset, state, class, prot, initial) in [ + ( + high::RENDER_FREE_LIST, + high::RENDER_FREE_LIST_STATE, + abi::FreeListClass::One, + FP, + abi::FreeListState::RENDER, + ), + ( + high::BOOT_FREE_LIST, + high::BOOT_FREE_LIST_STATE, + abi::FreeListClass::Two, + GFP, + abi::FreeListState::BOOT, + ), + ] { + let control = self.add(fixed(offset, mmu::UAT_PGSZ, prot, WC)?)?; + self.object_mut(control)?.write( + 0, + abi::FreeListControl::new(&abi::FreeListArgs { + buffer_slot: 0, + class, + page_list_va: low::RENDER_PAGE_LIST, + run_list_va: low::RENDER_RUN_LIST, + blocks: low::RENDER_BLOCKS_INITIAL, + state_va: base + state, + }), + )?; + let state_prot = if class == abi::FreeListClass::One { + FS + } else { + GFS + }; + let state = self.add(fixed(state, mmu::UAT_PGSZ, state_prot, WC)?)?; + self.object_mut(state)?.write(0, initial)?; + if class == abi::FreeListClass::Two { + self.boot_control = control; + self.boot_state = state; + } + } + self.stamps[0] = self.add(fixed(high::DESCRIPTOR_PAGE_A, mmu::UAT_PGSZ, GFS, WC)?)?; + self.stamps[1] = self.add(fixed(high::DESCRIPTOR_PAGE_B, mmu::UAT_PGSZ, GFP, WC)?)?; + self.add(fixed(high::TA_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + self.add(fixed(high::FRAGMENT_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + + let pages = self.add(KernelObject::backing(dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; + self.object_mut(pages)? + .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |list| { + abi::fill_page_list( + list, + (0..low::RENDER_BLOCKS_INITIAL as usize).map(low::render_block), + ) + })?; + let runs = self.add(KernelObject::backing(dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; + let first_block = self.objects.len(); + for _ in 0..low::RENDER_BLOCKS_ALLOCATED { + self.add(KernelObject::backing(dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + } + self.render_pages = pages; + self.render_runs = runs; + self.render_blocks = first_block..self.objects.len(); + self.alias(pages, render, low::RENDER_PAGE_LIST, GS)?; + self.alias(runs, render, low::RENDER_RUN_LIST, GS)?; + for block in 0..low::RENDER_BLOCKS_MAPPED { + self.alias(first_block + block, render, low::render_block(block), GS)?; + } + + let compute = self.add(fixed(high::COMPUTE_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; + self.object_mut(compute)?.write( + 0, + abi::FreeListControl::new(&abi::FreeListArgs { + buffer_slot: 0, + class: abi::FreeListClass::One, + page_list_va: low::COMPUTE_PAGE_LIST, + run_list_va: low::COMPUTE_RUN_LIST, + blocks: low::COMPUTE_BLOCKS, + state_va: base + high::COMPUTE_FREE_LIST_STATE, + }), + )?; + let compute_state = self.add(KernelObject::backing(dev, mmu::UAT_PGSZ, WB)?)?; + self.object_mut(compute_state)? + .write(0, abi::FreeListState::COMPUTE)?; + for _ in &low::COMPUTE_ZERO_BLOCKS { + self.add(KernelObject::backing(dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + } + let compute_runs = self.add(KernelObject::new( + dev, + lower, + Placement::At(low::COMPUTE_RUN_LIST), + mmu::UAT_PGSZ, + GFS, + WC, + )?)?; + self.object_mut(compute_runs)? + .initialize::<[abi::FreeListRunSlot; mmu::UAT_PGSZ / 0x40]>(0, |list| { + abi::fill_run_list(list, low::COMPUTE_FREE_LIST_BLOCKS) + })?; + let ready = self.add(fixed( + high::COMPUTE_READY_FREE_LIST, + mmu::UAT_PGSZ, + GFS, + WC, + )?)?; + self.object_mut(ready)?.write( + 0, + abi::FreeListControl::new(&abi::FreeListArgs { + buffer_slot: 1, + class: abi::FreeListClass::One, + page_list_va: low::COMPUTE_POOL_PAGE_LIST, + run_list_va: low::COMPUTE_POOL_RUN_LIST, + blocks: low::COMPUTE_READY_BLOCKS, + state_va: base + high::COMPUTE_READY_FREE_LIST_STATE, + }), + )?; + for state in [high::COMPUTE_READY_FREE_LIST_STATE, high::COMPUTE_STATE] { + let obj = self.add(fixed(state, mmu::UAT_PGSZ, GFS, WC)?)?; + self.object_mut(obj)? + .write(0, abi::FreeListState::COMPUTE)?; + } + self.object_mut(first_block + low::COMPUTE_PAGE_LIST_BLOCK)? + .initialize::<[u64; abi::FREE_LIST_BLOCK_SIZE / 8]>(0, |list| { + abi::fill_page_list(list, low::COMPUTE_FREE_LIST_BLOCKS) + })?; + fence(Ordering::SeqCst); + + self.bundle = self.add(fixed(high::BUNDLE, abi::bundle::SIZE, FP, WC)?)?; + let zero = self.add(fixed( + high::BUNDLE + abi::bundle::ZERO_PAGE as u64, + mmu::UAT_PGSZ, + GFS, + WC, + )?)?; + let mut completion_rings = [abi::CompletionRing { + low_va: 0, + high_va: 0, + }; 2]; + for slot in 0..2 { + let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; + let ring = self.add(fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; + self.completion_rings[slot] = ring; + self.alias(ring, lower, low::COMPLETION_RINGS[slot], GS)?; + completion_rings[slot] = abi::CompletionRing { + low_va: low::COMPLETION_RINGS[slot], + high_va: base + offset, + }; + } + let qos = self.add(dynamic_object(abi::QOS_SIZE, mmu::UAT_PGSZ as u64, FS, WC)?)?; + self.qos = qos; + fence(Ordering::SeqCst); + let metrics_alias = + metrics.alias_in(vm, dynamic.clone(), cfg.pm_metrics.size as u64, GFS)?; + self.metrics_fw_va = metrics_alias.iova(); + self.aliases.push(metrics_alias, GFP_KERNEL)?; + fence(Ordering::SeqCst); + self.roots = self.add(dynamic_object( + abi::ROOTS_SIZE, + abi::SECONDARY_ROOT_OFFSET as u64, + mmu::PROT_FW_SHARED_RO, + WB, + )?)?; + let region_a = self.add(dynamic_object( + REGION_A_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RO, + WB, + )?)?; + self.region_c = self.add(dynamic_object(REGION_C_SIZE, mmu::UAT_PGSZ as u64, FS, WB)?)?; + + let opps = Self::opps(dev)?; + let hw_args = abi::HwDataArgs { + chip_id: cfg.chip_id, + windows: &t8140::REGISTER_WINDOWS, + reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, + opps: &opps, + perf: &t8140::PERF, + completion_rings, + qos_va: self.object(qos)?.gpu_va(), + }; + self.object_mut(self.bundle)? + .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; + self.object_mut(self.region_c)? + .initialize::(0, |region| { + region.init(); + Ok(()) + })?; + self.object_mut(self.cluster)? + .write(abi::private::PRIMARY_STATUS_A, abi::StatusBlock::new())?; + self.object_mut(self.cluster)? + .write(abi::private::SECONDARY_STATUS_A, abi::StatusBlock::new())?; + let fwctl_va = self.object(fwctl)?.gpu_va(); + self.object_mut(self.cluster)? + .initialize::(abi::private::STATUS_B, |status| { + status.init(fwctl_va); + Ok(()) + })?; + self.object_mut(self.cluster)? + .initialize::( + abi::private::SECONDARY_STATE + abi::private::STATUS_B, + |config| { + config.init(); + Ok(()) + }, + )?; + self.object_mut(self.bundle)? + .initialize::(abi::bundle::POWER_CONFIG, |power| { + power.init(); + Ok(()) + })?; + let secondary_status = base + + high::PRIVATE_CLUSTER + + (abi::private::SECONDARY_STATE + abi::private::SECONDARY_STATUS) as u64; + self.object_mut(self.bundle)? + .initialize::(abi::bundle::AUX, |aux| { + aux.init(secondary_status); + Ok(()) + })?; + self.object_mut(self.bundle)? + .initialize::(abi::bundle::POWER_CONFIG_COPY, |power| { + power.init(); + Ok(()) + })?; + let views = abi::PrimaryViews { + zero_page_va: self.object(zero)?.gpu_va(), + pb_descriptors_va: self.object(pb)?.gpu_va(), + page_pool_descriptors_va: self.object(page_pool)?.gpu_va(), + }; + for role in [Role::Primary, Role::Secondary] { + let slot = role as usize; + let main_offset = abi::bundle::MAIN_CONFIG[slot]; + let main_va = base + high::BUNDLE + main_offset as u64; + let state_va = base + high::PRIVATE_CLUSTER + Self::state_offset(role) as u64; + let status_va = base + high::PRIVATE_CLUSTER + Self::status_offset(role) as u64; + let args = abi::MainConfigArgs { + bundle_va: base + high::BUNDLE, + main_config_va: main_va, + state_grid_va: state_va, + status_a_va: status_va, + pb_descriptors_low_va: low::PB_DESCRIPTORS, + page_pool_descriptors_low_va: low::PAGE_POOL_DESCRIPTORS, + }; + self.object_mut(self.bundle)? + .initialize::(main_offset, |main| { + match role { + Role::Primary => main.init_primary(&args, &views), + Role::Secondary => main.init_secondary(&args), + }; + Ok(()) + })?; + let abi_role = match role { + Role::Primary => abi::Role::Primary, + Role::Secondary => abi::Role::Secondary, + }; + self.object_mut(self.bundle)?.write( + main_offset + abi::MainConfig::CONTROL_RING, + abi::OpeningRecord::new(abi_role), + )?; + let root = abi::RootArgs { + region_a_va: self.object(region_a)?.gpu_va(), + region_c_va: self.object(self.region_c)?.gpu_va(), + main_config_va: main_va, + status_a_va: status_va, + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }; + match role { + Role::Primary => self.object_mut(self.roots)?.write( + 0, + abi::Root::primary(&root, state_va + abi::private::STATUS_B as u64), + )?, + Role::Secondary => self.object_mut(self.roots)?.write( + abi::SECONDARY_ROOT_OFFSET, + abi::SecondaryRoot::new( + &root, + base + high::PRIVATE_CLUSTER + + (abi::private::STATUS_B + abi::PrimaryStatusB::POWER_STATE) as u64, + state_va + abi::private::STATUS_B as u64, + ), + )?, + } + } + for role in [Role::Primary, Role::Secondary] { + for (offset, value) in [ + (0, 0), + (0x10, 0), + (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), + ] { + self.object(self.cluster)? + .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .store(value, Ordering::Relaxed); + } + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Retained fixed aliases for an ordinary client VM. The final block of the render-global + /// pool overlaps completion mappings in a client root and is deliberately not installed. + pub(super) fn user_aliases( + &mut self, + vm: &mmu::Vm, + metrics: &KernelObject, + ) -> Result> { + let mut mappings = KVec::with_capacity(24, GFP_KERNEL)?; + mappings.push( + self.object_mut(self.render_pages)?.map_alias( + vm, + low::RENDER_PAGE_LIST, + mmu::PROT_GPU_SHARED_RW, + )?, + GFP_KERNEL, + )?; + mappings.push( + self.object_mut(self.render_runs)?.map_alias( + vm, + low::RENDER_RUN_LIST, + mmu::PROT_GPU_SHARED_RW, + )?, + GFP_KERNEL, + )?; + for (block, index) in self.render_blocks.clone().take(21).enumerate() { + mappings.push( + self.object_mut(index)?.map_alias( + vm, + low::render_block(block), + mmu::PROT_GPU_SHARED_RW, + )?, + GFP_KERNEL, + )?; + } + mappings.push( + metrics.map_alias(vm, t8140::CONFIG.pm_metrics.va, mmu::PROT_GPU_FW_SHARED_RW)?, + GFP_KERNEL, + )?; + Ok(mappings) + } + + /// Writes the effort input in firmware order, then cleans its cache line to coherency. + pub(super) fn publish_effort(&self, effort: u32) -> Result { + let region = self.object(self.region_c)?; + let override_word = region.word(abi::RegionC::UNK_E8)?; + let effort_word = region.word(abi::RegionC::EFFORT)?; + let line = region.pointer(abi::RegionC::EFFORT, 4)?; + override_word.store(0, Ordering::Relaxed); + effort_word.store(effort, Ordering::Relaxed); + // SAFETY: line points into the retained kernel mapping; cleaning this host-owned cache + // line and the following system barrier publish the two naturally aligned stores. + unsafe { + core::arch::asm!("dc cvac, {line}", "dsb sy", line = in(reg) line, + options(nostack, preserves_flags)) + }; + Ok(()) + } + + pub(super) fn effort(&self) -> Result { + Ok(self + .object(self.region_c)? + .word(abi::RegionC::EFFORT)? + .load(Ordering::Relaxed)) + } + + fn opps(dev: &AsahiDevice) -> Result> { + let node = dev.as_ref().of_node().ok_or(EIO)?; + let table = node + .parse_phandle(c_str!("operating-points-v2"), 0) + .ok_or(EIO)?; + let mut opps = KVec::new(); + for opp in table.children() { + opps.push( + abi::Opp { + freq_hz: opp.get_property(c_str!("opp-hz"))?, + volt_uv: opp.get_property(c_str!("opp-microvolt"))?, + }, + GFP_KERNEL, + )?; + } + if opps.len() != t8140::OPPS { + return Err(EINVAL); + } + Ok(opps) + } + + fn state_offset(role: Role) -> usize { + match role { + Role::Primary => abi::private::PRIMARY_STATE, + Role::Secondary => abi::private::SECONDARY_STATE, + } + } + + fn status_offset(role: Role) -> usize { + match role { + Role::Primary => abi::private::PRIMARY_STATUS_A, + Role::Secondary => abi::private::SECONDARY_STATUS_A, + } + } + + /// Recheck the host-authored event pointer pair after a bounded number of drains. + /// The device mutex serializes the cache and all event consumption. + pub(super) fn validate_event_ring(&self, role: Role) -> Result { + let status = self.object(self.cluster)?.gpu_va() + Self::status_offset(role) as u64; + let expected = ( + status + abi::status::EVENT_STATE, + status + abi::status::EVENT_RING, + ); + let cache = &self.event_ring_checked[role as usize]; + let checked = cache.get(); + if checked.2 != 0 && (checked.0, checked.1) == expected { + cache.set((checked.0, checked.1, checked.2 - 1)); + return Ok(()); + } + let offset = abi::bundle::MAIN_CONFIG[role as usize] + + core::mem::offset_of!(abi::MainConfig, channels) + + abi::EVENT_CHANNEL * size_of::(); + let bundle = self.object(self.bundle)?; + let actual = ( + bundle.dword(offset)?.load(Ordering::Relaxed), + bundle + .dword(offset + size_of::())? + .load(Ordering::Relaxed), + ); + if actual != expected { + cache.set((0, 0, 0)); + return Err(EINVAL); + } + cache.set((actual.0, actual.1, EVENT_RING_RECHECK)); + Ok(()) + } + + pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { + self.object(self.cluster)?.word( + Self::status_offset(role) + .checked_add(offset) + .ok_or(EINVAL)?, + ) + } + + pub(super) fn status_read_words( + &self, + role: Role, + offset: usize, + ) -> Result<[u32; N]> { + self.object(self.cluster)?.read_words( + Self::status_offset(role) + .checked_add(offset) + .ok_or(EINVAL)?, + ) + } + + pub(super) fn control_counters(&self) -> Result<[u32; 3]> { + let state = self.object(self.cluster)?; + let mut counters = [0; 3]; + fence(Ordering::Acquire); + for (counter, offset) in counters.iter_mut().zip([ + abi::ChannelState::CONSUMER, + abi::ChannelState::UNK_10, + abi::ChannelState::PRODUCER, + ]) { + *counter = state + .word(abi::CONTROL_STATE_OFFSET + offset)? + .load(Ordering::Relaxed); + } + Ok(counters) + } + + pub(super) fn control_consumer(&self) -> Result<&AtomicU32> { + self.object(self.cluster)? + .word(abi::CONTROL_STATE_OFFSET + abi::ChannelState::CONSUMER) + } + + pub(super) fn control_producer(&self) -> Result<&AtomicU32> { + self.object(self.cluster)? + .word(abi::CONTROL_STATE_OFFSET + abi::ChannelState::PRODUCER) + } + + /// Publishes a control record under the device lock. All checked views are acquired before + /// the first write; record and packed power tally precede the aligned producer store. + pub(super) fn publish_control( + &self, + slot: u32, + record: &[u8; 64], + asserts_power: bool, + next: u32, + ) -> Result { + let offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize] + + abi::MainConfig::CONTROL_RING + + (slot as usize).checked_mul(record.len()).ok_or(EINVAL)?; + let ring = self.object(self.bundle)?.pointer(offset, record.len())?; + let power = self + .object(self.region_c)? + .pointer(abi::RegionC::POWER_ASSERT_TALLY, 4)?; + let producer = self.control_producer()?; + // SAFETY: The checked pointers remain valid for the lifetime of this non-Sync owner. + // Firmware only reads records after the producer store and the device mutex serializes + // host publishers. The unaligned power word uses the interface's four-byte copy access. + unsafe { + core::ptr::copy_nonoverlapping(record.as_ptr(), ring, record.len()); + if asserts_power { + let mut tally = [0u8; 4]; + core::ptr::copy_nonoverlapping(power, tally.as_mut_ptr(), 4); + let value = u32::from_le_bytes(tally).wrapping_add(1).to_le_bytes(); + core::ptr::copy_nonoverlapping(value.as_ptr(), power, 4); + } + } + fence(Ordering::SeqCst); + producer.store(next, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } + + pub(super) fn root_va(&self, role: Role) -> Result { + Ok(self.object(self.roots)?.gpu_va() + role as u64 * abi::SECONDARY_ROOT_OFFSET as u64) + } + + pub(super) fn opening_retired(&self, role: Role) -> Result { + for offset in [0, 0x10, abi::ChannelState::PRODUCER] { + if self + .object(self.cluster)? + .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .load(Ordering::SeqCst) + != abi::OPENING_PRODUCER + { + return Ok(false); + } + } + Ok(true) + } +} + +impl InitData { + /// Validate the serialized view before reading its retained host mapping. + fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { + let bundle = self.object(self.bundle)?; + let base = *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?; + let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; + let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) + .ok_or(EOVERFLOW)?; + if end > extent { + return Err(EINVAL); + } + let pointer_at = abi::bundle::MAIN_CONFIG[Role::Primary as usize] + + core::mem::offset_of!(abi::MainConfig, view_va) + view * size_of::(); + fence(Ordering::Acquire); + let pointer = bundle.read_words::<2>(pointer_at)?; + let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; + if actual != bundle.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { + return Err(EINVAL); + } + bundle.read_words(base + offset) + } +} + +impl super::recovery::Memory for InitData { + fn fault_sources(&self) -> Result { + let mask = self.object(self.cluster)? + .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? + .load(Ordering::Relaxed); + let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { + let key = super::recovery::slot_key(slot).unwrap_or(0); + let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, + abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) + .map(|words| words[0]).unwrap_or(0); + (key, monitor) + }); + sources.reason = self.recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) + .map(|words| words[0]).unwrap_or(0); + Ok(sources) + } + + fn recovery_state(&self) -> Result { + fence(Ordering::Acquire); + Ok(self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)? + .load(Ordering::Relaxed)) + } + + fn host_recovery(&self) -> Result { + Ok(self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::HOST_RECOVERY)? + .load(Ordering::Relaxed)) + } + + fn transition(&self, expected: u32, next: u32) -> Result { + let state = self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?; + fence(Ordering::Acquire); + if state.load(Ordering::Relaxed) != expected { + return Err(EBUSY); + } + fence(Ordering::SeqCst); + state.store(next, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } + + fn clear_timestamps(&self) -> Result { + let bundle = self.object(self.bundle)?; + // Validate the complete table before publishing its first live word. + const ENTRIES: usize = 128; + const STRIDE: usize = 0x10; + let offset = abi::bundle::VIEWS[4]; + bundle.pointer(offset, ENTRIES * STRIDE)?; + for qid in 0..ENTRIES { + bundle + .word(offset + qid * STRIDE)? + .store(0, Ordering::Relaxed); + bundle + .dword(offset + qid * STRIDE + 8)? + .store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } + + fn entries( + &self, + out: &mut [super::recovery::Entry; super::recovery::MAX_ENTRIES], + ) -> Result { + let cluster = self.object(self.cluster)?; + let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; + fence(Ordering::Acquire); + let mut used = 0; + for index in 0..256 { + if used == out.len() { + break; + } + let offset = base + index * core::mem::size_of::(); + let flags = cluster.word(offset)?.load(Ordering::Relaxed); + if flags & 1 == 0 { + continue; + } + let stamp = cluster.word(offset + 4)?.load(Ordering::Relaxed); + if let Some(entry) = super::recovery::Entry::decode(flags, stamp) { + out[used] = entry; + used += 1; + } + } + Ok(used) + } + + fn wait_tick(&self) { + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs new file mode 100644 index 0000000000000..ed2e79537d597 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Immutable submissions and the DRM scheduler boundary for independent engines. + +use super::{command::Validated, completion::Completion, context::Context}; +use crate::driver::AsahiDevice; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, Ordering}; +use kernel::{ + bindings, c_str, + dma_fence::{Fence, RawDmaFence}, + drm::sched, + prelude::*, + sync::Arc, +}; + +const TIMEOUT_MS: kernel::time::Msecs = 2000; +const RENDER_CREDITS: u32 = 32; +const COMPUTE_CREDITS: u32 = 16; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Engine { + Render, + Compute, +} + +impl Engine { + pub(crate) fn index(self) -> usize { + match self { + Self::Render => 0, + Self::Compute => 1, + } + } +} + +/// Logical sequence numbers do not wrap with firmware timestamps. A zero barrier +/// includes preceding ioctls; the sentinel means independent of that engine. +#[derive(Clone, Copy)] +pub(crate) struct Order { + pub(crate) sequence: u64, + pub(crate) wait_through: [Option; 2], +} + +impl Order { + pub(crate) fn new( + base: [u64; 2], + preceding: [u16; 2], + barriers: [u16; 2], + engine: Engine, + ) -> Result { + let mut sequences = [0; 2]; + let mut wait_through = [None; 2]; + for index in 0..2 { + sequences[index] = base[index] + .checked_add(u64::from(preceding[index])) + .and_then(|value| value.checked_add(1)) + .ok_or(EOVERFLOW)?; + if barriers[index] != u16::MAX { + if barriers[index] > preceding[index] { + return Err(EINVAL); + } + wait_through[index] = Some( + base[index] + .checked_add(u64::from(barriers[index])) + .ok_or(EOVERFLOW)?, + ); + } + } + Ok(Self { + sequence: sequences[engine.index()], + wait_through, + }) + } + + pub(crate) fn contains(prefix: Option, sequence: u64) -> bool { + sequence != 0 && prefix.is_some_and(|last| sequence <= last) + } +} + +pub(crate) struct Packet { + pub(crate) id: u64, + pub(crate) context: Arc, + pub(crate) command: Validated, + pub(crate) order: Order, + pub(crate) completion: Arc, + pub(crate) footprint: Option>, + inputs: KVec, + firmware: KVec, + publication: AtomicU8, + cancelled: AtomicBool, + early: AtomicI32, + backlog: AtomicBool, +} + +impl Packet { + pub(crate) fn new( + id: u64, + context: Arc, + command: Validated, + order: Order, + completion: Arc, + inputs: KVec, + firmware: KVec, + ) -> Result> { + let footprint = match &command { + Validated::Render { attachments, .. } => { + context.vm().render_footprint(&attachments[1])? + } + Validated::Compute { .. } => None, + }; + Ok(Arc::new( + Self { + id, + context, + command, + order, + completion, + footprint, + inputs, + firmware, + publication: AtomicU8::new(0), + cancelled: AtomicBool::new(false), + early: AtomicI32::new(0), + backlog: AtomicBool::new(false), + }, + GFP_KERNEL, + )?) + } + + pub(crate) fn engine(&self) -> Engine { + match &self.command { + Validated::Render { .. } => Engine::Render, + Validated::Compute { .. } => Engine::Compute, + } + } + + /// Scheduler dependency readiness includes failed fences. Consume the actual + /// status before publication, and retain the references through that check. + pub(crate) fn check_dependencies(&self) -> Result { + for fence in &self.inputs { + // SAFETY: The retained fence owns the pointer throughout the wait and read. + let waited = unsafe { + bindings::dma_fence_wait_timeout( + fence.raw(), + true, + kernel::task::MAX_SCHEDULE_TIMEOUT, + ) + }; + if waited < 0 { + return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); + } + if waited == 0 { + return Err(ECANCELED); + } + check_status(fence)?; + } + for fence in &self.firmware { + check_status(fence)?; + } + Ok(()) + } + + /// Dependency-bearing packets remain on the scheduler publication path. + pub(crate) fn has_dependencies(&self) -> bool { + !self.inputs.is_empty() || !self.firmware.is_empty() + } + + pub(crate) fn begin_publication(&self) -> bool { + self.publication + .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } + pub(crate) fn mark_published(&self) { + self.publication.store(2, Ordering::Release); + } + pub(crate) fn is_published(&self) -> bool { + self.publication.load(Ordering::Acquire) == 2 + } + pub(crate) fn cancel_requested(&self) -> bool { + self.cancelled.load(Ordering::Acquire) + } + pub(crate) fn request_cancel(&self) -> bool { + self.cancelled.store(true, Ordering::Release); + self.publication + .compare_exchange(0, 3, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } + pub(crate) fn set_early_result(&self, result: Result) { + self.early.store( + result.map_or_else(|error| error.to_errno(), |_| 1), + Ordering::Release, + ); + } + pub(crate) fn early_result(&self) -> Option { + match self.early.load(Ordering::Acquire) { + 0 => None, + 1 => Some(Ok(())), + error => Some(Err(Error::from_errno(error))), + } + } + pub(crate) fn count_backlog(&self) { + self.backlog.store(true, Ordering::Release); + } + pub(crate) fn take_backlog(&self) -> bool { + self.backlog.swap(false, Ordering::AcqRel) + } +} + +pub(crate) fn fence_status(fence: &Fence) -> i32 { + // SAFETY: The owned fence keeps the raw DMA fence alive for this read. + unsafe { bindings::dma_fence_get_status(fence.raw()) } +} + +fn check_status(fence: &Fence) -> Result { + let status = fence_status(fence); + if status < 0 { + Err(super::dependency::inherited_error(Error::from_errno(status))) + } else { + Ok(()) + } +} + +/// The backend retains visible packets through exact retirement or processor stop. +/// It performs all completion signalling after releasing the device mutex. +pub(crate) trait Backend: Send + Sync { + /// Lazily create or rebind this logical queue's physical compute graph. + fn ensure_compute(&self) -> Result; + /// Settle outstanding work of a terminal VM before an ioctl error is published. + fn fail_vm(&self, error: Error); + /// Called after both scheduler entities have cancelled their queued jobs. + fn release_owner(&self); + fn prepare(&self, packet: &Packet) -> Option; + fn publish(&self, packet: Arc) -> Result; + fn timed_out(&self, packet: Arc) -> sched::Status; + fn cancel(&self, packet: Arc); + /// Transfer a timeout to an outstanding off-lock graph transaction, if any. + fn defer_timeout(&self, packet: &Arc) -> bool; + /// Return true only after taking publication ownership and recording its result. + fn try_early(&self, packet: &Arc) -> bool; + fn count_backlog(&self, packet: &Packet); +} + +struct Job { + backend: Arc, + packet: Arc, +} + +impl sched::JobImpl for Job { + fn false_timeout(job: &mut sched::Job) -> bool { + job.is_finished() + || job.packet.completion.take_replay_timeout() + || job.backend.defer_timeout(&job.packet) + } + + fn prepare(job: &mut sched::Job) -> Option { + job.backend.prepare(&job.packet) + } + fn run(job: &mut sched::Job) -> Result> { + job.backend.publish(job.packet.clone()).map(Some) + } + fn timed_out(job: &mut sched::Job) -> sched::Status { + // The scheduler retains the detached job throughout this callback. Keep + // its parent callback intact while the driver settles the exact owner; + // reinsertion also handles completion racing with timeout observation. + if !job.is_finished() { + job.backend.timed_out(job.packet.clone()); + } + sched::Status::NoHang + } + fn cancel(job: &mut sched::Job) { + job.backend.cancel(job.packet.clone()); + } +} + +pub(crate) struct Scheduler { + // Cancel entity jobs before dropping the scheduler's owning reference. + entity: sched::Entity>, + _scheduler: sched::Scheduler>, + backend: Arc, +} + +#[derive(Clone)] +pub(crate) struct Fences { + pub(crate) ready: Fence, + pub(crate) completed: Fence, +} + +impl Scheduler { + pub(crate) fn new(dev: &AsahiDevice, backend: Arc, engine: Engine) -> Result { + let (credits, name) = match engine { + Engine::Render => (RENDER_CREDITS, c_str!("asahi_g17_render")), + Engine::Compute => (COMPUTE_CREDITS, c_str!("asahi_g17_compute")), + }; + let scheduler = sched::Scheduler::new(dev.as_ref(), 1, credits, 0, TIMEOUT_MS, name)?; + let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; + Ok(Self { + entity, + _scheduler: scheduler, + backend, + }) + } + + pub(crate) fn enqueue( + &mut self, + packet: Arc, + dependencies: KVec, + early: bool, + ) -> Result { + let completed = packet.completion.fence(); + let host_timestamps = packet.completion.has_timestamps(); + let mut job = self.entity.new_job( + 1, + Job { + backend: self.backend.clone(), + packet: packet.clone(), + }, + )?; + for dependency in dependencies { + job.add_dependency(dependency)?; + } + let mut job = job.arm(); + let ready = if host_timestamps { + completed.clone() + } else { + job.fences().scheduled() + }; + // All fallible scheduler setup precedes early hardware publication. The + // scheduler still owns watchdog, credits and cancellation after push. + if packet.engine() == Engine::Render && !(early && self.backend.try_early(&packet)) { + self.backend.count_backlog(&packet); + } + job.push(); + Ok(Fences { ready, completed }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs new file mode 100644 index 0000000000000..9d89262d8a3b6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Physical kick-ring backing, queue identities and entry publication. +//! +//! The device mutex serializes identities and publication. Installed identities +//! and their ring backing stay owned until both firmware processors stop; +//! command completion alone does not detach a hardware queue. + +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +use super::{ + fw::{kick::*, queue::DataMaster}, + object::{Allocator, CpuMap, KernelObject}, +}; +use crate::{hw::t8140, mem, mmu}; + +const QID_COUNT: usize = QID_MAX as usize + 1; + +/// Stable physical-channel identity. The owner must not be reused within a +/// device lifetime, including after an unpublished allocation is cancelled. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Id { + qid: u8, + owner: u64, + engine: DataMaster, +} + +impl Id { + pub(crate) fn qid(self) -> u8 { + self.qid + } + pub(crate) fn engine(self) -> DataMaster { + self.engine + } +} + +#[derive(Copy, Clone)] +struct Identity { + id: Id, + published: bool, +} + +/// One shared namespace for tiling, fragment and compute queues. +pub(crate) struct QueueIds { + entries: [Option; QID_COUNT], +} + +impl QueueIds { + pub(crate) const fn new() -> Self { + Self { + entries: [None; QID_COUNT], + } + } + + /// Reuses only an exact owner/engine match. Preferred placement is for + /// bootstrapping; it does not reserve a partition for an engine. + pub(crate) fn reserve( + &mut self, + owner: u64, + engine: DataMaster, + preferred: Option, + ) -> Result { + if owner == 0 { + return Err(EINVAL); + } + if let Some(entry) = self + .entries + .iter() + .flatten() + .find(|entry| entry.id.owner == owner) + { + return if entry.id.engine == engine { + Ok(entry.id) + } else { + Err(EINVAL) + }; + } + let index = preferred + .map(usize::from) + .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) + .or_else(|| self.entries.iter().position(Option::is_none)) + .ok_or(ENOSPC)?; + let id = Id { + qid: index as u8, + owner, + engine, + }; + self.entries[index] = Some(Identity { + id, + published: false, + }); + Ok(id) + } + + /// Call after preflighting the complete installation, before any address + /// becomes firmware-visible. A failed or partial installation retains it. + pub(crate) fn publish(&mut self, id: Id) -> Result { + let entry = self + .entries + .get_mut(usize::from(id.qid)) + .and_then(Option::as_mut) + .ok_or(EINVAL)?; + if entry.id != id { + return Err(EINVAL); + } + entry.published = true; + Ok(()) + } + + pub(crate) fn cancel_unpublished(&mut self, id: Id) -> Result { + let slot = self.entries.get_mut(usize::from(id.qid)).ok_or(EINVAL)?; + match slot { + Some(entry) if entry.id == id && !entry.published => { + *slot = None; + Ok(()) + } + _ => Err(EINVAL), + } + } +} + +// Aliases drop before their backing. A compute queue is allocated low first; +// a render queue is allocated high first. +enum Backing { + Compute { + high: mmu::KernelMapping, + low: KernelObject, + }, + Render { + low: mmu::KernelMapping, + high: KernelObject, + }, +} + +/// An installed queue's permanent ring. Client aliases are held by the bound +/// context graph and must be dropped before this object. +pub(crate) struct Queue { + backing: Backing, + id: Id, + timestamp: KickTimestamp, + previous: Option, +} + +/// A checked entry and destination held without changing the shared ring. +/// Discarding it leaves both the slot and the next timestamp unchanged. +pub(crate) struct PreparedKick<'a> { + queue: &'a mut Queue, + timestamp: KickTimestamp, + entry: KickEntry, + destination: *mut KickEntry, +} + +impl PreparedKick<'_> { + /// Publish translation updates after the queue configuration pointer and before its entry. + pub(crate) fn prepare_compute_install(&self) -> Result { + self.queue.prepare_compute_install() + } + + /// Writes and fences the complete entry, then advances host state. The + /// caller invokes this between the queue configuration and announcement. + pub(crate) fn commit(self) -> KickTimestamp { + // SAFETY: prepare_entry checked the complete destination range and + // this token exclusively borrows the queue and retains its mapping. + // The caller only prepares retired ring slots; no firmware access to + // the new entry starts before the outer command is published. + unsafe { self.destination.write_unaligned(self.entry) }; + fence(Ordering::SeqCst); + self.queue.previous = Some(self.timestamp); + self.queue.timestamp = self.timestamp.next(); + self.timestamp + } +} + +impl Queue { + pub(crate) fn compute(alloc: &Allocator<'_>, id: Id) -> Result { + if id.engine != DataMaster::Compute { + return Err(EINVAL); + } + let low = alloc.lower( + KICK_RING_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + )?; + let mut range = alloc.uat.geometry().kernel_range(); + range.start += t8140::dynamic::KERNEL_OFFSET; + let high = low.alias_in( + alloc.uat.kernel_vm(), + range, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + )?; + fence(Ordering::SeqCst); + Ok(Self { + backing: Backing::Compute { high, low }, + id, + timestamp: KickTimestamp::FIRST, + previous: None, + }) + } + + pub(crate) fn render( + alloc: &Allocator<'_>, + id: Id, + client: &mmu::Vm, + ) -> Result<(Self, mmu::KernelMapping)> { + if id.engine == DataMaster::Compute { + return Err(EINVAL); + } + let high = alloc.kernel( + KICK_RING_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let mut range = t8140::dynamic::LOWER; + let (low, client) = loop { + let low = high.alias_in( + alloc.uat.kernel_lower_vm(), + range.clone(), + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + )?; + let end = low + .iova() + .checked_add(KICK_RING_SIZE as u64) + .ok_or(EOVERFLOW)?; + match high.map_alias(client, low.iova(), mmu::PROT_GPU_SHARED_RW) { + Ok(client) => break (low, client), + Err(error) if error == ENOSPC && end < range.end => { + drop(low); + range.start = end; + } + Err(error) => return Err(error), + } + }; + Ok(( + Self { + backing: Backing::Render { low, high }, + id, + timestamp: KickTimestamp::FIRST, + previous: None, + }, + client, + )) + } + + pub(crate) fn id(&self) -> Id { + self.id + } + pub(crate) fn low_va(&self) -> u64 { + match &self.backing { + Backing::Compute { low, .. } => low.gpu_va(), + Backing::Render { low, .. } => low.iova(), + } + } + pub(crate) fn firmware_va(&self) -> u64 { + match &self.backing { + Backing::Compute { high, .. } => high.iova(), + Backing::Render { high, .. } => high.gpu_va(), + } + } + fn object(&mut self) -> &mut KernelObject { + match &mut self.backing { + Backing::Compute { low, .. } => low, + Backing::Render { high, .. } => high, + } + } + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { + let address = self.low_va(); + self.object() + .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) + } + + /// Reuse a permanent client view only when every page names this ring. + pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { + let object = match &self.backing { + Backing::Compute { low, .. } => low, + Backing::Render { high, .. } => high, + }; + object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) + } + + /// Both aliases are owned mappings of the same full ring. Publish their + /// page-table updates to each live translation regime before installation. + pub(crate) fn prepare_compute_install(&self) -> Result { + if self.id.engine != DataMaster::Compute { + return Err(EINVAL); + } + fence(Ordering::SeqCst); + for asid in 0..4 { + mem::tlbi_range_or_asid(asid, self.low_va() as usize, KICK_RING_SIZE); + mem::tlbi_range_or_asid(asid, self.firmware_va() as usize, KICK_RING_SIZE); + } + mem::sync(); + Ok(()) + } + + /// Detaches the implicit dependency after firmware recovery, retaining + /// the monotonically advancing 40-bit producer. + pub(crate) fn clear_parent_after_recovery(&mut self) { + self.previous = None; + } + + pub(crate) fn timestamp(&self) -> KickTimestamp { + self.timestamp + } + pub(crate) fn parent(&self) -> KickTimestamp { + self.previous.unwrap_or_else(|| match self.id.engine { + DataMaster::Compute => KickTimestamp::ZERO, + _ => self.timestamp.prev(), + }) + } + pub(crate) fn prepare_entry(&mut self, args: &KickArgs<'_>) -> Result> { + if args.qid != self.id.qid + || args.timestamp != self.timestamp + || args.parent != self.parent() + { + return Err(EINVAL); + } + let entry = KickEntry::new(args)?; + let destination = self + .object() + .pointer(args.timestamp.entry_offset(), size_of::())? + .cast(); + let timestamp = self.timestamp; + Ok(PreparedKick { + queue: self, + timestamp, + entry, + destination, + }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs new file mode 100644 index 0000000000000..fe354a52968cc --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -0,0 +1,1153 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 GPU support. +//! +//! G17 GPUs run their firmware on two coprocessor instances, the primary and the secondary +//! [`Role`]. Each has its own CPU control registers, RTKit mailbox, crash log buffer and kernel +//! page table root, and both share the UAT, the TTBAT and the firmware data structures that the +//! driver allocates at boot (see [`initdata`]). +//! +//! The firmware notifies the host of new records in its event and trace rings with a message on +//! either coprocessor; the event worker drains them. +//! +//! # Locking +//! +//! `Shared::state` is the device lock. It serializes all firmware-visible publication and all +//! firmware-shared state, including the bounded waits for the firmware that publication needs. +//! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state +//! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. + +mod buffer; +mod channel; +mod command; +pub(crate) use command::Attachments; +mod completion; +pub(crate) mod context; +mod dependency; +mod event; +mod fence; +mod freelist; +pub(crate) mod fw; +mod ids; +pub(crate) mod hazard; +mod initdata; +mod job; +mod kick; +mod object; +mod power; +mod preparation; +mod qos; +mod queue; +mod recovery; +mod runtime; +pub(crate) mod status; +mod teardown; +mod validation; + +use core::any::Any; +use core::ops::Range; +use core::sync::atomic::{ + AtomicBool, + Ordering, // +}; + +use kernel::{ + c_str, + device::Core, + drm::gem::shmem, + impl_has_delayed_work, impl_has_work, + iosys_map::IoSysMapRef, + new_condvar, new_delayed_work, new_mutex, new_work, platform, + prelude::*, + soc::apple::rtkit, + sync::{ + Arc, + CondVar, + Mutex, // + }, + time::{ + delay::fsleep, + msecs_to_jiffies, + Delta, // + }, + types::{ForeignOwnable, Opaque}, + uapi, + workqueue::{ + self, + DelayedWork, + Work, + WorkItem, // + }, +}; + +use crate::driver::{ + AsahiDevRef, + AsahiDevice, // +}; +use crate::gem::AsahiObject; +use crate::{ + alloc, + gem, + gpu, + hw, + mmu, + regs, // +}; + +use object::{ + CpuMap, + KernelObject, + Placement, // +}; + +/// Expected identification of a G17 GPU. +pub(crate) struct Identity { + /// GPU family field of the ID register. + pub(crate) family: u8, + /// GPU variant field of the ID register. + pub(crate) variant: u8, + /// Number of GPU dies. + pub(crate) num_dies: u32, + /// GPU variant reported to userspace. + pub(crate) gpu_variant: hw::GpuVariant, + /// Shader core generation reported to userspace. + pub(crate) usc_generation: u32, + /// Hardware abstraction generation reported to userspace. + pub(crate) hal_generation: u32, +} + +/// A buffer that the firmware expects at a fixed address of the kernel lower root. +pub(crate) struct FixedBuffer { + /// GPU address. + pub(crate) va: u64, + /// Size in bytes. + pub(crate) size: usize, +} + +/// Static configuration of a SoC with a G17 GPU. +pub(crate) struct Config { + /// SoC identifier. + pub(crate) chip_id: u32, + /// Expected GPU identification. + pub(crate) identity: Identity, + /// Input address bits translated by each UAT root. + pub(crate) uat_ias: u32, + /// Output (physical) address bits of the UAT. + pub(crate) uat_oas: u32, + /// Offset of the secondary coprocessor's kernel root in the `pagetables` region. + pub(crate) secondary_root_offset: u64, + /// Crash log buffer of each [`Role`]. + pub(crate) crash_buffers: [FixedBuffer; 2], + /// Parameter buffer page metrics shared with the firmware. + pub(crate) pm_metrics: FixedBuffer, +} + +/// A firmware coprocessor instance. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Role { + /// The coprocessor that owns the device-control ring. + Primary = 0, + /// The coprocessor that only runs work the primary one hands to it. + Secondary = 1, +} + +/// RTKit endpoint of firmware messages: initialization, its acknowledgement and event +/// notifications. +const EP_FIRMWARE: u8 = 0x20; +/// RTKit endpoint of host notifications to the firmware. +const EP_DOORBELL: u8 = 0x21; + +/// Bit position of the type field of firmware messages. +const MSG_TYPE_SHIFT: u32 = 48; +/// Host message handing a coprocessor the address of its initialization root. +const MSG_INITDATA: u64 = 0x81 << MSG_TYPE_SHIFT; +/// Address bits of an initialization root that [`MSG_INITDATA`] carries. +const MSG_INITDATA_ADDR_MASK: u64 = (1 << 44) - 1; +/// Firmware message acknowledging [`MSG_INITDATA`]. +const MSG_INITDATA_ACK: u64 = 0x09; +/// Firmware message announcing new records in the event and trace rings. +const MSG_EVENTS: u64 = 0x42 << MSG_TYPE_SHIFT; +/// Primary processor notification for newly published device-control records. +const MSG_CONTROL_NOTIFY: u64 = (0x84 << MSG_TYPE_SHIFT) | 0x11; +/// Host message, sent to both coprocessors once both are initialized, ending the power-on +/// transition. The firmware then retires the opening device-control records. +const MSG_POWER_ON_DONE: u64 = 0x89 << MSG_TYPE_SHIFT; + +/// Time the firmware takes at most to acknowledge its initialization. +const INITDATA_ACK_TIMEOUT_MS: u32 = 2000; +/// Bounded mailbox-provider readiness retries, before either coprocessor starts. +const RECEIVER_ATTEMPTS: usize = 3; +const RECEIVER_RETRY_MS: i64 = 20; +/// The firmware does not notify the host when it retires device-control records, so their +/// counters are polled at this interval, at most [`OPENING_RETIRE_POLLS`] times. +const OPENING_RETIRE_POLL_MS: i64 = 1; +const OPENING_RETIRE_POLLS: usize = 100; + +/// Address bits of a buffer the coprocessor places itself; the coprocessor sets higher bits in +/// its buffer requests. +const RTKIT_BUFFER_ADDR_MASK: u64 = (1 << 40) - 1; +/// Initial contents of crash log buffers, so that a crash log shows what the firmware wrote. +const CRASH_BUFFER_FILL: [u8; 2] = [0xa5, 0x5a]; + +/// A crash log buffer handed to RTKit. +pub(crate) struct CrashLog { + iova: usize, + map: shmem::VMap, +} + +impl rtkit::Buffer for CrashLog { + fn iova(&self) -> Result { + Ok(self.iova) + } + + fn buf(&mut self) -> Result> { + Ok(self.map.get()) + } +} + +/// Callback state; initialization ACKs can arrive in hard interrupt context. +#[pin_data] +struct RoleState { + role: Role, + shared: Arc, + crash_log: FixedBuffer, + crash_map: shmem::VMap, + crashed: AtomicBool, + #[pin] + ack: Opaque, +} + +// SAFETY: Completion access uses the kernel completion API; the remaining fields are immutable +// shared references or atomics. The completion is pinned before callbacks can observe it. +unsafe impl Send for RoleState {} +// SAFETY: As above; complete_all is safe concurrently with the single boot waiter. +unsafe impl Sync for RoleState {} + +impl RoleState { + fn signal(&self, crashed: bool) { + if crashed { + self.crashed.store(true, Ordering::Release); + } + // SAFETY: ack is initialized and pinned for the lifetime of the callback state. + unsafe { kernel::bindings::complete_all(self.ack.get()) }; + } + + fn wait_for_ack(&self) -> Result { + // SAFETY: ack is initialized and remains pinned while this reference exists. + let completed = unsafe { + kernel::bindings::wait_for_completion_timeout( + self.ack.get(), + msecs_to_jiffies(INITDATA_ACK_TIMEOUT_MS), + ) + }; + if self.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if completed == 0 { + return Err(ETIMEDOUT); + } + Ok(()) + } +} + +fn is_events_message(endpoint: u8, message: u64) -> bool { + endpoint == EP_FIRMWARE && message == MSG_EVENTS +} + +/// RTKit callbacks of a coprocessor. +struct RoleOps; + +#[vtable] +impl rtkit::Operations for RoleOps { + type Data = Arc; + type Buffer = CrashLog; + + const COPROC_PLACES_BUFFERS: bool = true; + + fn recv_message_early( + data: ::Borrowed<'_>, + ep: u8, + msg: u64, + ) -> bool { + if ep == EP_FIRMWARE && msg >> MSG_TYPE_SHIFT == MSG_INITDATA_ACK { + data.signal(false); + return true; + } + if !is_events_message(ep, msg) { + return false; + } + data.shared.queue_events(); + true + } + + fn crashed(data: ::Borrowed<'_>, crashlog: Option<&[u8]>) { + data.signal(true); + data.shared.lose_device(data.role, crashlog); + } + + fn shmem_map( + data: ::Borrowed<'_>, + iova: usize, + size: usize, + ) -> Result { + if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != data.crash_log.va || size > data.crash_log.size + { + return Err(EINVAL); + } + Ok(CrashLog { + iova, + map: data.crash_map.clone(), + }) + } +} + +/// A running firmware coprocessor. +/// +/// Dropping it stops the CPU before RTKit and the crash log buffer go away. +struct Coprocessor { + cpu: regs::CpuControl, + running: bool, + rtkit: rtkit::RtKit, + state: Arc, + _crash_buffer: KernelObject, +} + +impl Coprocessor { + /// Sets up a coprocessor's RTKit and crash log buffer, without starting it. + fn new( + pdev: &platform::Device, + dev: &AsahiDevice, + cfg: &Config, + uat: &mmu::Uat, + shared: &Arc, + role: Role, + ) -> Result { + let index = role as usize; + let buffer = &cfg.crash_buffers[index]; + let mut crash_buffer = KernelObject::new( + dev, + uat.kernel_lower_vm(), + Placement::At(buffer.va), + buffer.size, + mmu::PROT_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); + + let crash_map = crash_buffer.kernel_map()?; + let state = Arc::pin_init( + pin_init!(RoleState { + role, + shared: shared.clone(), + crash_log: FixedBuffer { + va: buffer.va, + size: buffer.size, + }, + crash_map, + crashed: AtomicBool::new(false), + ack <- Opaque::ffi_init(|ptr| { + // SAFETY: ptr is valid uninitialized completion storage that is pinned + // before any RTKit callback can use it. + unsafe { kernel::bindings::init_completion(ptr) }; + }), + }), + GFP_KERNEL, + )?; + + let name = match role { + Role::Primary => c_str!("asc"), + Role::Secondary => c_str!("asc1"), + }; + Ok(Coprocessor { + cpu: regs::CpuControl::new(pdev, name)?, + running: false, + rtkit: Self::receiver(dev, index, &state)?, + state, + _crash_buffer: crash_buffer, + }) + } + + fn receiver( + dev: &AsahiDevice, + index: usize, + state: &Arc, + ) -> Result> { + for attempt in 0..RECEIVER_ATTEMPTS { + match rtkit::RtKit::::new(dev.as_ref(), None, index, state.clone()) { + Ok(receiver) => return Ok(receiver), + Err(error) if error == EINVAL || error == ETIMEDOUT => { + if attempt + 1 < RECEIVER_ATTEMPTS { + fsleep(Delta::from_millis(RECEIVER_RETRY_MS)); + } + } + Err(error) => return Err(error), + } + } + Err(EPROBE_DEFER) + } + + /// Starts the CPU and waits for RTKit to come up. + fn boot(&mut self) -> Result { + self.cpu.start()?; + self.running = true; + Pin::new(&mut self.rtkit).boot() + } + + /// Starts the firmware endpoints and hands the firmware its initialization root. + fn init(&mut self, root_va: u64) -> Result { + let mut rtkit = Pin::new(&mut self.rtkit); + rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; + rtkit.as_mut().start_endpoint(EP_DOORBELL)?; + rtkit.send_message( + EP_FIRMWARE, + MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), + )?; + self.state.wait_for_ack() + } + + /// Sends a host notification to the firmware. + fn notify(&mut self, msg: u64) -> Result { + Pin::new(&mut self.rtkit).send_message(EP_DOORBELL, msg) + } + + /// Stops the CPU. The firmware keeps its state, but no longer runs. + fn stop(&mut self) { + if self.running { + self.cpu.stop(); + self.running = false; + } + } +} + +impl Drop for Coprocessor { + fn drop(&mut self) { + self.stop(); + } +} + +/// The GPU-to-fabric bridge, enabled for as long as this exists. +struct Bridge { + regs: regs::Resources, + enabled: bool, +} + +impl Bridge { + fn enable(regs: regs::Resources) -> Result { + let bridge = Bridge { + regs, + enabled: true, + }; + bridge.regs.enable_g17_bridge()?; + Ok(bridge) + } + + fn disable(&mut self) { + if self.enabled { + self.regs.disable_g17_bridge(); + self.enabled = false; + } + } +} + +impl Drop for Bridge { + fn drop(&mut self) { + self.disable(); + } +} + +/// Everything that exists while the firmware runs. +/// +/// Field order is teardown order: both coprocessors stop (in [`Drop`]) before their RTKit is +/// dropped, the bridge is disabled once they are stopped, and the firmware objects and their +/// mappings go before the UAT. Page tables the firmware or the bootloader own are never freed. +struct Firmware { + secondary: Coprocessor, + primary: Coprocessor, + bridge: Bridge, + rings: channel::Rings, + recovery: recovery::State, + effort: power::Effort, + queues: runtime::Registry, + pool: Arc, + render_ids: Arc, + init: initdata::InitData, + _render_global: mmu::Vm, + _pm_metrics: Arc, + uat: Arc, +} + +impl Drop for Firmware { + fn drop(&mut self) { + self.secondary.stop(); + self.primary.stop(); + self.pool.stop_after_firmware(); + // RTKit destruction releases the mailbox power references. Clear the bridge while + // those references still keep the register block powered. + self.bridge.disable(); + } +} + +impl Firmware { + /// Publishes controls only while the restart handshake is closed. The device mutex remains + /// held through publication and the following mailbox notification. + fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { + use recovery::Memory; + if self.recovery.pending() || self.init.recovery_state()? != 0 { + return Err(EAGAIN); + } + let target = channel::Rings::publish_control(&self.init, record)?; + core::sync::atomic::fence(Ordering::SeqCst); + self.primary.notify(MSG_CONTROL_NOTIFY)?; + Ok(target) + } + + /// Holds idle descent while a fresh physical render pair is registered. + fn acquire_registration_power(&mut self) -> Result { + power::acquire(self) + } + + fn release_registration_power(&mut self) -> Result { + power::release(self) + } + + /// Called at actual submission publication, not while a queued job is merely being built. + fn note_submission(&mut self) -> Result { + self.effort.submit(); + self.apply_effort(power::FULL_EFFORT) + } + + fn apply_effort(&mut self, effort: u32) -> Result { + Self::apply_shared_effort(&self.init, &mut self.effort, effort) + } + + fn apply_shared_effort( + init: &initdata::InitData, + state: &mut power::Effort, + effort: u32, + ) -> Result { + if state.applied() == effort && init.effort()? == effort { + return Ok(()); + } + init.publish_effort(effort)?; + state.did_apply(effort); + Ok(()) + } + + /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, + /// unretired work; quarantined work retaining mappings is excluded. + fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { + match self.effort.idle(now_ns, in_flight) { + power::IdleAction::None => Ok(None), + power::IdleAction::Clear => { + self.apply_effort(0)?; + Ok(None) + } + power::IdleAction::Recheck(delay) => Ok(Some(delay)), + } + } +} + +impl power::Control for Firmware { + fn crashed(&self) -> bool { + self.primary.state.shared.crashed.load(Ordering::Acquire) + } + fn powered(&self) -> bool { + self.bridge.regs.g17_configuration_powered() + } + fn counters(&self) -> Result<[u32; 3]> { + self.init.control_counters() + } + fn set_idle(&mut self, enabled: bool) -> Result { + use fw::channels::{ControlRecord, IdlePolicy, IdlePowerOff}; + self.publish_control(&ControlRecord::IdlePowerOff(IdlePowerOff::new( + if enabled { + IdlePolicy::AllowPowerOff + } else { + IdlePolicy::KeepPowered + }, + ))) + } + fn wait_tick(&self) { + fsleep(Delta::from_millis(1)); + } +} + +/// Event work item ID. +const EVENT_WORK: u64 = 0; +/// Short control/retirement polls do not wait behind the longer idle deadline. +const POLL_WORK: u64 = 1; +const IDLE_WORK: u64 = 2; +const GROW_WORK: u64 = 3; +const RECLAIM_WORK: u64 = 4; + +/// Device state shared with the coprocessor callbacks and the event worker. +#[pin_data] +struct Shared { + dev: AsahiDevRef, + clusters: u32, + descriptor_flags: [u32; 2], + crashed: AtomicBool, + preparations: Arc, + #[pin] + work: Work, + #[pin] + poll: DelayedWork, + #[pin] + idle: DelayedWork, + #[pin] + grow: Work, + #[pin] + reclaim_work: Work, + /// Nests inside the device mutex; detached mappings are dropped after both unlock. + #[pin] + reclaim: Mutex, + /// The device lock. `None` before boot completes and after teardown. + #[pin] + state: Mutex>>, + #[pin] + changed: CondVar, +} + +impl_has_work! { + impl HasWork for Shared { self.work } +} + +impl_has_delayed_work! { + impl HasDelayedWork for Shared { self.poll } +} +impl_has_delayed_work! { + impl HasDelayedWork for Shared { self.idle } +} +impl_has_work! { + impl HasWork for Shared { self.grow } +} +impl_has_work! { + impl HasWork for Shared { self.reclaim_work } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + if let Err(error) = this.grow_render() { + if this.state.lock().is_some() { + dev_err!( + this.dev.as_ref(), + "Render parameter-buffer growth failed: {:?}\n", + error + ); + this.lose_device_quietly(); + } + } + this.queue_events(); + } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + this.drain_reclaims(); + this.queue_events(); + } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + let result = { + let mut state = this.state.lock(); + match (*state).as_deref_mut() { + Some(firmware) if firmware.render_control_backpressured() => { + firmware.note_render_control_backpressure() + } + _ => Ok(()), + } + }; + // Only an elapsed one-millisecond timer consumes the retry budget. + // Concurrent event IRQs can request a pass without aging the control. + if result.is_err() { + this.lose_device_quietly(); + } else { + this.queue_events(); + } + } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + this.queue_events(); + } +} + +impl WorkItem for Shared { + type Pointer = Arc; + + fn run(this: Arc) { + this.run_runtime_events(); + } +} + +impl Shared { + /// Queues the event worker. Callable from any context. + fn queue_events(self: &Arc) { + // Already queued if this fails, which is just as good. + let _ = workqueue::system_highpri().enqueue::, EVENT_WORK>(self.clone()); + } + + fn queue_grow(self: &Arc) { + let _ = workqueue::system_unbound().enqueue::, GROW_WORK>(self.clone()); + } + + fn queue_reclaims(self: &Arc) { + let _ = workqueue::system_unbound().enqueue::, RECLAIM_WORK>(self.clone()); + } + + fn drain_reclaims(self: &Arc) { + loop { + let next = self.reclaim.lock().take_one(); + let Some(next) = next else { + break; + }; + next.finish(); + } + } + + fn disable_work(self: &Arc, work: &Work) { + // SAFETY: Callers pass this Shared's initialized pinned work field. + // Disabling joins a running callback and prevents subsequent enqueue. + let cancelled = unsafe { kernel::bindings::disable_work_sync(Work::raw_get(work)) }; + if cancelled { + // SAFETY: Cancelling the pending callback leaves its one enqueue- + // transferred Arc to reclaim; the callback can no longer consume it. + drop(unsafe { Arc::from_raw(Arc::as_ptr(self)) }); + } + } + + fn queue_poll(self: &Arc) { + let _ = workqueue::system_highpri() + .enqueue_delayed::, POLL_WORK>(self.clone(), msecs_to_jiffies(1)); + } + + fn queue_idle(self: &Arc, delay_ns: u64) { + // Effort::idle bounds this by the 16 ms idle interval. Round up so a + // timer never changes the firmware effort word before that deadline. + let delay_ms = delay_ns.div_ceil(1_000_000).min(u64::from(u32::MAX)) as u32; + let _ = workqueue::system_highpri() + .enqueue_delayed::, IDLE_WORK>(self.clone(), msecs_to_jiffies(delay_ms)); + } + + fn disable_timer(self: &Arc, timer: &DelayedWork) { + // SAFETY: Every caller passes one of this Shared's initialized pinned + // timer fields. raw_as_work points into its delayed_work; the C call + // disables future enqueues, cancels pending work and joins execution. + let cancelled = unsafe { + let work = Work::raw_get(DelayedWork::raw_as_work(timer)); + let delayed = kernel::container_of!(work, kernel::bindings::delayed_work, work); + kernel::bindings::disable_delayed_work_sync(delayed) + }; + if cancelled { + // SAFETY: The cancelled pending callback owned exactly one Arc + // transferred by enqueue_delayed. It cannot run to reclaim it. + drop(unsafe { Arc::from_raw(Arc::as_ptr(self)) }); + } + } + + /// Marks the device lost without a crash log. + fn lose_device_quietly(self: &Arc) { + self.preparations.fail(); + if !self.crashed.swap(true, Ordering::AcqRel) { + dev_err!(self.dev.as_ref(), "GPU runtime stopped, device lost\n"); + } + self.queue_events(); + } + + /// Marks the device lost after a coprocessor crash and publishes a crash dump. + /// + /// The firmware cannot be restarted: outstanding and future work fails. + fn lose_device(self: &Arc, role: Role, crashlog: Option<&[u8]>) { + self.preparations.fail(); + self.crashed.store(true, Ordering::Release); + dev_err!( + self.dev.as_ref(), + "GPU firmware ({:?}) crashed, device lost\n", + role + ); + + #[cfg(CONFIG_DEV_COREDUMP)] + if let Some(crashlog) = crashlog { + if let Err(e) = self.publish_crashlog(crashlog) { + dev_err!(self.dev.as_ref(), "Could not generate crashdump: {:?}\n", e); + } + } + #[cfg(not(CONFIG_DEV_COREDUMP))] + let _ = crashlog; + self.queue_events(); + } + + #[cfg(CONFIG_DEV_COREDUMP)] + fn publish_crashlog(&self, crashlog: &[u8]) -> Result { + use kernel::{devcoredump, time::msecs_to_jiffies}; + + // The dump does not include firmware memory: its kernel mappings are private to it. + let mut crashdump = crate::crashdump::CrashDumpBuilder::new(KVVec::new())?; + crashdump.add_crashlog(crashlog)?; + let crashdump = KBox::new(crashdump.finalize()?, GFP_KERNEL)?; + devcoredump::dev_coredump( + self.dev.as_ref(), + &crate::THIS_MODULE, + crashdump, + GFP_KERNEL, + msecs_to_jiffies(CRASHDUMP_TIMEOUT_MS), + ); + Ok(()) + } +} + +/// How long a crash dump stays available. +#[cfg(CONFIG_DEV_COREDUMP)] +const CRASHDUMP_TIMEOUT_MS: u32 = 60 * 60 * 1000; + +/// A G17 GPU. +pub(crate) struct Gpu { + cfg: &'static Config, + id: regs::G17Id, + max_frequency_khz: u32, + geometry: mmu::UatGeometry, + ids: gpu::SequenceIDs, + shared: Arc, +} + +impl Gpu { + /// Boots the firmware of a G17 GPU. + pub(crate) fn new( + pdev: &platform::Device, + dev: &AsahiDevice, + cfg: &'static Config, + regs: regs::Resources, + ) -> Result> { + let id = regs.get_g17_id()?; + if id.family != cfg.identity.family + || id.variant != cfg.identity.variant + || id.num_dies != cfg.identity.num_dies + { + dev_err!( + dev.as_ref(), + "Unexpected GPU family {:#x} variant {:#x} with {} dies\n", + id.family, + id.variant, + id.num_dies + ); + return Err(ENODEV); + } + + let geometry = mmu::UatGeometry { + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }; + let max_frequency_khz = Self::max_frequency_khz(dev)?; + let preparations = preparation::Gate::new()?; + let reclaim = runtime::teardown::ReclaimBatch::new()?; + let shared = Arc::pin_init( + pin_init!(Shared { + dev: dev.into(), + clusters: id.num_clusters, + descriptor_flags: [id.perf_control, id.perf_map[0]], + crashed: AtomicBool::new(false), + preparations, + work <- new_work!("g17::Shared::work"), + poll <- new_delayed_work!("g17::Shared::poll"), + idle <- new_delayed_work!("g17::Shared::idle"), + grow <- new_work!("g17::Shared::grow"), + reclaim_work <- new_work!("g17::Shared::reclaim_work"), + reclaim <- new_mutex!(reclaim, "g17::Shared::reclaim"), + state <- new_mutex!(None, "g17::Shared::state"), + changed <- new_condvar!("g17::Shared::changed"), + }), + GFP_KERNEL, + )?; + + let gpu = Arc::new( + Gpu { + cfg, + id, + max_frequency_khz, + geometry, + ids: Default::default(), + shared, + }, + GFP_KERNEL, + )?; + let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; + *gpu.shared.state.lock() = Some(firmware); + gpu.shared.queue_events(); + Ok(gpu) + } + + /// Returns the highest advertised operating-point frequency without changing table order. + fn max_frequency_khz(dev: &AsahiDevice) -> Result { + let node = dev.as_ref().of_node().ok_or(EIO)?; + let opps = node + .parse_phandle(c_str!("operating-points-v2"), 0) + .ok_or(EIO)?; + let mut freq_hz: Option = None; + for opp in opps.children() { + let frequency: u64 = opp.get_property(c_str!("opp-hz"))?; + freq_hz = Some(freq_hz.map_or(frequency, |highest| highest.max(frequency))); + } + Ok((freq_hz.ok_or(EINVAL)? / 1000).try_into()?) + } + + /// Starts both coprocessors and initializes their firmware. + fn boot( + pdev: &platform::Device, + dev: &AsahiDevice, + cfg: &'static Config, + regs: regs::Resources, + geometry: mmu::UatGeometry, + shared: &Arc, + ) -> Result> { + let uat = Arc::new( + mmu::Uat::new( + dev, + geometry, + true, + mmu::UatFirmware::PublishedRoots { + secondary_root_offset: cfg.secondary_root_offset, + }, + )?, + GFP_KERNEL, + )?; + + // The firmware only samples the kernel lower root's top-level entries when it starts, so + // everything it uses there must be mapped first. + let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary)?; + let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary)?; + let pm_metrics = KernelObject::new( + dev, + uat.kernel_lower_vm(), + Placement::At(cfg.pm_metrics.va), + cfg.pm_metrics.size, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let render_global = uat.new_vm(0, geometry.user_range())?; + + let bridge = Bridge::enable(regs)?; + + // Install all session ownership before either CPU starts. Error unwinding then runs + // Firmware::drop, stopping both CPUs before releasing shared objects or translations. + let mut firmware = KBox::new( + Firmware { + secondary, + primary, + bridge, + rings: channel::Rings::new(), + recovery: recovery::State::new(), + effort: power::Effort::new(), + queues: runtime::Registry::new(shared.clusters, shared.descriptor_flags)?, + pool: object::Pool::new()?, + render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, + init: initdata::InitData::new(), + _render_global: render_global, + _pm_metrics: Arc::new(pm_metrics, GFP_KERNEL)?, + uat, + }, + GFP_KERNEL, + )?; + let session = &mut *firmware; + session.primary.boot()?; + session.secondary.boot()?; + let root_entries = session.uat.adopt_firmware_root_entries()?; + session.init.build( + dev, + cfg, + &session.uat, + &session._render_global, + &session._pm_metrics, + )?; + session.uat.confirm_firmware_root_entries(root_entries)?; + session.uat.mirror_secondary_root()?; + session.bridge.regs.enable_g17_gating()?; + session.primary.init(session.init.root_va(Role::Primary)?)?; + session + .secondary + .init(session.init.root_va(Role::Secondary)?)?; + session.primary.notify(MSG_POWER_ON_DONE)?; + session.secondary.notify(MSG_POWER_ON_DONE)?; + Self::wait_for_opening(dev, &session.init, &session.primary, &session.secondary)?; + session + .uat + .activate_render_context(&session._render_global)?; + Ok(firmware) + } + + /// Waits for both coprocessors to retire their opening device-control records. + fn wait_for_opening( + dev: &AsahiDevice, + init: &initdata::InitData, + primary: &Coprocessor, + secondary: &Coprocessor, + ) -> Result { + for _ in 0..OPENING_RETIRE_POLLS { + if primary.state.crashed.load(Ordering::Acquire) + || secondary.state.crashed.load(Ordering::Acquire) + { + return Err(EIO); + } + if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { + return Ok(()); + } + fsleep(Delta::from_millis(OPENING_RETIRE_POLL_MS)); + } + dev_err!( + dev.as_ref(), + "Firmware did not retire its opening records\n" + ); + Err(ETIMEDOUT) + } + + /// Stops the firmware and releases everything it used. + pub(crate) fn shutdown(&self) { + self.shared.preparations.remove(); + self.shared.stop_queues(); + self.shared.disable_timer(&self.shared.poll); + self.shared.disable_timer(&self.shared.idle); + self.shared.disable_work(&self.shared.grow); + self.shared.disable_work(&self.shared.reclaim_work); + self.shared.disable_work(&self.shared.work); + self.shared.drain_reclaims(); + } +} + +impl gpu::Gpu for Gpu { + fn as_any(&self) -> &dyn Any { + self + } + + fn arc_as_any(self: Arc) -> Arc { + self as Arc + } + + fn manager(&self) -> Option<&dyn gpu::GpuManager> { + None + } + + fn init(&self) -> Result { + Ok(()) + } + + fn ids(&self) -> &gpu::SequenceIDs { + &self.ids + } + + fn is_crashed(&self) -> bool { + self.shared.crashed.load(Ordering::Acquire) + } + + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + params.gpu_generation = hw::GpuGen::G17 as u32; + params.gpu_variant = self.cfg.identity.gpu_variant as u32; + params.gpu_revision = self.id.gpu_rev as u32; + params.chip_id = self.cfg.chip_id; + params.num_dies = self.id.num_dies; + params.num_clusters_total = self.id.num_clusters; + params.num_cores_per_cluster = self.id.num_cores; + params.max_frequency_khz = self.max_frequency_khz; + params.usc_generation = self.cfg.identity.usc_generation; + params.gpu_hal_generation = self.cfg.identity.hal_generation; + + for (i, mask) in self.id.core_masks.iter().enumerate() { + *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); + } + + Ok(()) + } + + fn base_clock_hz(&self) -> u32 { + let frequency: u64; + // SAFETY: CNTFRQ_EL0 is a read-only register containing the architectural counter rate. + unsafe { core::arch::asm!("mrs {frequency}, CNTFRQ_EL0", frequency = out(reg) frequency) }; + frequency as u32 + } + + fn supports_vm_status(&self) -> bool { + true + } + + fn uat_geometry(&self) -> mmu::UatGeometry { + self.geometry + } + + fn new_vm(&self, kernel_range: Range) -> Result { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_mut().ok_or(ENODEV)?; + let firmware = &mut **firmware; + let vm = firmware + .uat + .new_vm(self.ids.vm.next(), kernel_range)? + .with_status()?; + let mappings = firmware.init.user_aliases(&vm, &firmware._pm_metrics)?; + let ranges = + hw::t8140::resources::client_reserved_ranges(self.cfg.pm_metrics.va).ok_or(EINVAL)?; + let mut reserved = KVec::with_capacity(ranges.len(), GFP_KERNEL)?; + for range in ranges { + reserved.push(range, GFP_KERNEL)?; + } + vm.install_driver_mappings(mappings, reserved)?; + Ok(vm) + } + + fn new_queue( + &self, + vm: mmu::Vm, + _ualloc: Arc>, + _ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result> { + let owner = self.ids.queue.next(); + let frontend = + queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; + let (backend, slot) = loop { + match runtime::Backend::new(&self.shared, &vm, owner) { + Err(error) if error == EAGAIN => continue, + result => break result?, + } + }; + let frontend = match queue::Queue::new( + &self.shared.dev, + backend.context.clone(), + backend.clone(), + frontend, + slot, + ) { + Ok(frontend) => frontend, + Err(error) => { + job::Backend::release_owner(&*backend); + return Err(error); + } + }; + Ok(KBox::new(frontend, GFP_KERNEL)?) + } + + fn map_timestamp_buffer( + &self, + mut bo: gem::ObjectRef, + range: Range, + ) -> Result { + let state = self.shared.state.lock(); + let firmware = (*state).as_ref().ok_or(ENODEV)?; + let mut addresses = firmware.uat.geometry().kernel_range(); + addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; + bo.map_range_into_range( + firmware.uat.kernel_vm(), + range, + addresses, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + ) + } +} + +impl Drop for Gpu { + fn drop(&mut self) { + self.shutdown(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs new file mode 100644 index 0000000000000..cf1d49fb03a6b --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -0,0 +1,579 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Kernel objects shared with the G17 firmware. +//! +//! Each firmware object is backed by its own GEM object, mapped into a kernel address space +//! either at an address the firmware interface fixes or anywhere in a given range, and kept +//! mapped into the kernel CPU address space for its whole lifetime. The host builds an object's +//! contents through [`KernelObject::bytes_mut`] and [`KernelObject::write`] before it publishes +//! its address; after that, words the firmware accesses concurrently are only touched through +//! the single-access helpers. + +use core::ops::Range; +use core::ptr::NonNull; +use core::sync::atomic::{ + AtomicU32, + AtomicU64, // +}; + +use kernel::{ + drm::gem::shmem, + prelude::*, // +}; + +use crate::driver::AsahiDevice; +use crate::{ + gem, + mmu, // +}; + +/// CPU caching of the kernel mapping of a [`KernelObject`]. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum CpuMap { + /// Write-back, for memory the firmware only reads through a cached mapping or that the host + /// only reads. + WriteBack, + /// Write-combined, for memory the firmware reads uncached as soon as the host published it. + WriteCombined, +} + +/// Where a [`KernelObject`] is mapped. +pub(crate) enum Placement { + /// At this address, which the firmware interface fixes. + At(u64), + /// Anywhere in this range, at this alignment, followed by an unmapped guard page. + In(Range, u64), +} + +/// A zero-initialized kernel object mapped into a GPU address space and the kernel. +/// +/// # Invariants +/// +/// `ptr` points to the start of the kernel mapping of `obj`, which is `size` bytes long and stays +/// valid as long as `obj` exists, since `obj` holds an owned kernel mapping. +pub(crate) struct KernelObject { + // Unmap from the GPU before dropping the backing object. + mapping: Option, + obj: gem::ObjectRef, + ptr: NonNull, + size: usize, +} + +// SAFETY: The object only refers to kernel memory; concurrent access goes through `&self` +// methods that use atomic or volatile accesses. +unsafe impl Send for KernelObject {} +// SAFETY: See `Send`. +unsafe impl Sync for KernelObject {} + +impl KernelObject { + /// Allocates a zeroed object of `size` bytes and maps it into `vm`. + pub(crate) fn new( + dev: &AsahiDevice, + vm: &mmu::Vm, + placement: Placement, + size: usize, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { + let mut this = Self::backing(dev, size, cpu)?; + let mapping = match placement { + Placement::At(addr) => this.obj.map_at(vm, addr, prot, false)?, + Placement::In(range, align) => this.obj.map_into_range(vm, range, align, prot, true)?, + }; + this.mapping = Some(mapping); + Ok(this) + } + + /// Allocates CPU-accessible backing whose GPU mappings will be installed later. + pub(crate) fn backing(dev: &AsahiDevice, size: usize, cpu: CpuMap) -> Result { + let mut obj = match cpu { + CpuMap::WriteBack => gem::new_kernel_object(dev, size)?, + CpuMap::WriteCombined => gem::new_kernel_object_wc(dev, size)?, + }; + let mut vmap = obj.vmap()?; + vmap.memset(0); + let ptr = NonNull::new(vmap.as_mut_ptr()).ok_or(ENOMEM)?; + core::mem::drop(vmap); + + // INVARIANT: `vmap()` created the owned kernel mapping of `obj` that `ptr` points into, + // and `obj` keeps it until it is dropped. + Ok(Self { + mapping: None, + obj, + ptr, + size, + }) + } + + /// Maps this object into another address space at `addr`, as an alias of the same memory. + pub(crate) fn map_alias( + &self, + vm: &mmu::Vm, + addr: u64, + prot: mmu::Prot, + ) -> Result { + gem::ObjectRef::new(self.obj.gem.clone()).map_at(vm, addr, prot, false) + } + + /// Reuses only a complete alias of this exact retained backing. Permanent + /// queue aliases stay owned by the client VM after a physical pair moves; + /// returning to that VM must not replace or duplicate those mappings. + pub(crate) fn prepare_alias( + &self, + vm: &mmu::Vm, + address: u64, + prot: mmu::Prot, + requires_write: bool, + ) -> Result> { + if self.size == 0 || self.size % mmu::UAT_PGSZ != 0 || address & mmu::UAT_PGMSK as u64 != 0 + { + return Err(EINVAL); + } + address.checked_add(self.size as u64).ok_or(EOVERFLOW)?; + let source = self.mapping.as_ref().ok_or(EINVAL)?; + let mut existing = 0; + for offset in (0..self.size).step_by(mmu::UAT_PGSZ) { + let client = address + offset as u64; + if vm.covers_range(client, mmu::UAT_PGSZ as u64, true, requires_write) { + if vm.translate_iova(client)? != source.translate_offset(offset)? { + return Err(EFAULT); + } + existing += 1; + } + } + if existing != 0 { + if existing != self.size / mmu::UAT_PGSZ { + return Err(EFAULT); + } + return Ok(None); + } + self.map_alias(vm, address, prot).map(Some) + } + + /// Maps this backing into a dynamically placed kernel alias. + pub(crate) fn alias_in( + &self, + vm: &mmu::Vm, + range: Range, + align: u64, + prot: mmu::Prot, + ) -> Result { + gem::ObjectRef::new(self.obj.gem.clone()).map_into_range(vm, range, align, prot, false) + } + + /// Maps an exact backing subrange, retaining an explicit unmapped guard. + pub(crate) fn map_range( + &self, + vm: &mmu::Vm, + source: Range, + range: Range, + align: u64, + prot: mmu::Prot, + guard_size: usize, + ) -> Result { + vm.map_in_range_with_guard_size(&self.obj.gem, source, align, range, prot, guard_size) + } + + /// Initializes a typed view before publishing the object to the firmware. + /// + /// The view is cleared first, making every `Zeroable` representation valid. + pub(crate) fn initialize( + &mut self, + offset: usize, + init: impl FnOnce(&mut T) -> Result, + ) -> Result { + let ptr = self.aligned::(offset)?; + // SAFETY: The checked view lies within the exclusively borrowed object. Zeroable makes + // the cleared bytes a valid T, and the closure cannot retain the reference. + unsafe { + ptr.write_bytes(0, 1); + init(&mut *ptr) + } + } + + /// Returns a reference-counted kernel mapping of the object. + pub(crate) fn kernel_map(&self) -> Result> { + self.obj.gem.owned_vmap() + } + + /// Transfers an unpublished object's GPU mapping to a longer-lived owner. + /// The CPU view remains valid; gpu_va returns zero after this transfer. + pub(crate) fn take_mapping(&mut self) -> Result { + self.mapping.take().ok_or(EINVAL) + } + + /// Returns the GPU address of the object. + pub(crate) fn gpu_va(&self) -> u64 { + self.mapping.as_ref().map_or(0, |mapping| mapping.iova()) + } + + /// Returns the size of the object in bytes. + pub(crate) fn size(&self) -> usize { + self.size + } + + /// Returns the contents of the object. + /// + /// Only for building an object the firmware does not access concurrently, before its address + /// is published or while the firmware is known not to use it. + pub(crate) fn bytes_mut(&mut self) -> &mut [u8] { + // SAFETY: Per the type invariant `ptr` is valid for `size` bytes, and the exclusive borrow + // excludes other host accesses. + unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.size) } + } + + /// Writes `value` at `offset`, with the same restrictions as [`Self::bytes_mut`]. + pub(crate) fn write(&mut self, offset: usize, value: T) -> Result { + let end = offset + .checked_add(core::mem::size_of::()) + .ok_or(EINVAL)?; + if end > self.size { + return Err(EINVAL); + } + // SAFETY: The range was checked to be within the object, and the exclusive borrow excludes + // other host accesses. The destination may be unaligned. + unsafe { + self.ptr + .as_ptr() + .add(offset) + .cast::() + .write_unaligned(value) + }; + Ok(()) + } + + /// Returns a pointer to the naturally aligned `T` at `offset`. + fn aligned(&self, offset: usize) -> Result<*mut T> { + let end = offset + .checked_add(core::mem::size_of::()) + .ok_or(EINVAL)?; + if end > self.size { + return Err(EINVAL); + } + // SAFETY: The checked offset lies within the retained CPU mapping. + let address = unsafe { self.ptr.as_ptr().add(offset) }; + if address as usize % core::mem::align_of::() != 0 { + return Err(EINVAL); + } + // The actual address, not only the offset, satisfies T's alignment. + Ok(address.cast()) + } + + /// Returns a checked raw view; dereferencing it requires external serialization. + pub(super) fn pointer(&self, offset: usize, length: usize) -> Result<*mut u8> { + if offset.checked_add(length).ok_or(EINVAL)? > self.size { + return Err(EINVAL); + } + // SAFETY: The offset was checked against the live object mapping. + Ok(unsafe { self.ptr.as_ptr().add(offset) }) + } + + /// Returns the live 32-bit word at `offset`, which must be naturally aligned. + pub(crate) fn word(&self, offset: usize) -> Result<&AtomicU32> { + // SAFETY: `aligned()` returns a valid, aligned pointer into the object, which lives as long + // as `self`. Other agents only access the word with single 32-bit accesses. + Ok(unsafe { AtomicU32::from_ptr(self.aligned(offset)?) }) + } + + /// Returns the live 64-bit word at `offset`, which must be naturally aligned. + pub(crate) fn dword(&self, offset: usize) -> Result<&AtomicU64> { + // SAFETY: As for `word()`, with 64-bit accesses. + Ok(unsafe { AtomicU64::from_ptr(self.aligned(offset)?) }) + } + + /// Copies `N` 32-bit words that start at `offset` out of the object, one volatile access each. + /// + /// Used for records another agent has published, which it no longer modifies. + pub(crate) fn read_words(&self, offset: usize) -> Result<[u32; N]> { + let base = self.aligned::<[u32; N]>(offset)?.cast::(); + let mut words = [0u32; N]; + for (i, word) in words.iter_mut().enumerate() { + // SAFETY: `aligned()` checked that all `N` words are within the object and aligned. + *word = unsafe { base.add(i).read_volatile() }; + } + Ok(words) + } +} + +/// Allocation domains shared by the G17 resource builders. Dynamic objects +/// retain a guard page; aliases keep the existing backing and its exact VA. +pub(crate) struct Allocator<'a> { + pub(crate) dev: &'a AsahiDevice, + pub(crate) uat: &'a mmu::Uat, +} + +impl Allocator<'_> { + /// Allocates in the dynamic part of the firmware's upper address space. + pub(crate) fn kernel( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { + let mut range = self.uat.geometry().kernel_range(); + range.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; + KernelObject::new( + self.dev, + self.uat.kernel_vm(), + Placement::In(range, align), + size, + prot, + cpu, + ) + } + + /// Allocates a canonical low mapping, which can be aliased at the same + /// address into an execution context without overlapping client objects. + pub(crate) fn lower( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { + KernelObject::new( + self.dev, + self.uat.kernel_lower_vm(), + Placement::In(crate::hw::t8140::dynamic::LOWER, align), + size, + prot, + cpu, + ) + } +} + +/// Families whose upper mappings remain cached and installed until firmware +/// stops, even after the logical owner releases the object. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Family { + Descriptor = 0, + WorkState = 1, + QueueState = 2, + QueueConfig = 3, +} + +const POOL_FAMILIES: usize = 4; +const POOL_SLOTS: usize = 64; +const POOL_BYTES: usize = 128 * 1024 * 1024; + +struct Retained { + object: KernelObject, + prefix: usize, + cpu: CpuMap, + poisoned: bool, +} + +struct PoolInner { + returned: KVec>, + reserved: [usize; POOL_FAMILIES], + bytes: usize, + stopped: bool, +} + +/// Per-device cache-safe object recycling. Every converted mapping reserves +/// one return slot before its page-table attributes change. Active leases and +/// poisoned mappings count against the same capacity as reusable objects. +#[pin_data] +pub(crate) struct Pool { + #[pin] + inner: kernel::sync::Mutex, +} + +struct CacheLease { + pool: kernel::sync::Arc, + family: Family, + prefix: usize, + cpu: CpuMap, + poisoned: bool, +} + +/// A firmware object whose cached mapping is returned to its device pool +/// instead of being unmapped while either firmware processor can reference it. +pub(crate) struct PooledObject { + object: core::mem::ManuallyDrop, + lease: Option, +} + +impl core::ops::Deref for PooledObject { + type Target = KernelObject; + fn deref(&self) -> &Self::Target { + &self.object + } +} +impl core::ops::DerefMut for PooledObject { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.object + } +} +impl Drop for PooledObject { + fn drop(&mut self) { + // SAFETY: This destructor is the only operation that moves `object` + // out; ManuallyDrop prevents a second implicit drop of the field. + let object = unsafe { core::mem::ManuallyDrop::take(&mut self.object) }; + match self.lease.take() { + Some(lease) => lease.pool.return_object(object, &lease), + None => drop(object), + } + } +} + +impl Pool { + pub(crate) fn new() -> Result> { + let mut returned = KVec::with_capacity(POOL_FAMILIES * POOL_SLOTS, GFP_KERNEL)?; + for _ in 0..POOL_FAMILIES * POOL_SLOTS { + returned.push(None, GFP_KERNEL)?; + } + kernel::sync::Arc::pin_init( + pin_init!(Self { + inner <- kernel::new_mutex!(PoolInner { returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false }, "g17::object::Pool"), + }), + GFP_KERNEL, + ) + } + + /// Allocates or reuses a matching object. Capacity pressure leaves a fresh + /// mapping uncached, preserving allocation and submission concurrency. + pub(crate) fn allocate( + self: &kernel::sync::Arc, + alloc: &Allocator<'_>, + family: Family, + size: usize, + prefix: usize, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { + let family_index = family as usize; + let begin = family_index * POOL_SLOTS; + let reused = { + let mut inner = self.inner.lock(); + if inner.stopped { + return Err(ENODEV); + } + let slot = inner.returned[begin..begin + POOL_SLOTS] + .iter_mut() + .find(|entry| { + entry.as_ref().is_some_and(|entry| { + !entry.poisoned + && entry.object.size() == size + && entry.prefix == prefix + && entry.cpu == cpu + }) + }); + slot.and_then(Option::take) + }; + let lease = || CacheLease { + pool: self.clone(), + family, + prefix, + cpu, + poisoned: false, + }; + if let Some(mut entry) = reused { + entry.object.bytes_mut().fill(0); + core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); + return Ok(PooledObject { + object: core::mem::ManuallyDrop::new(entry.object), + lease: Some(lease()), + }); + } + let object = alloc.kernel(size, mmu::UAT_PGSZ as u64, prot, cpu)?; + let reserved = { + let mut inner = self.inner.lock(); + if inner.stopped { + return Err(ENODEV); + } + if inner.reserved[family_index] == POOL_SLOTS + || inner + .bytes + .checked_add(size) + .is_none_or(|bytes| bytes > POOL_BYTES) + { + false + } else { + inner.reserved[family_index] += 1; + inner.bytes += size; + true + } + }; + let mut result = PooledObject { + object: core::mem::ManuallyDrop::new(object), + lease: reserved.then(lease), + }; + if !reserved { + return Ok(result); + } + let (proof, mutated) = match result.mapping.as_ref() { + // SAFETY: The allocator just created the private mapping. No + // pointer or alias has been published; its reserved lease retains + // it until processor stop if any attribute change was attempted. + Some(mapping) => unsafe { mapping.set_fw_cached_prefix(prefix) }, + None => (Err(EINVAL), false), + }; + match proof { + Ok(_) => Ok(result), + Err(error) if mutated => { + if let Some(lease) = &mut result.lease { + lease.poisoned = true; + } + // Drop retains the unproven mapping but never makes it reusable. + Err(error) + } + Err(_) => { + result.lease = None; + let mut inner = self.inner.lock(); + inner.reserved[family_index] -= 1; + inner.bytes -= size; + Ok(result) + } + } + } + + fn return_object(&self, object: KernelObject, lease: &CacheLease) { + let mut inner = self.inner.lock(); + let family = lease.family as usize; + if inner.stopped { + inner.reserved[family] -= 1; + inner.bytes -= object.size(); + drop(inner); + drop(object); + return; + } + let begin = family * POOL_SLOTS; + for slot in &mut inner.returned[begin..begin + POOL_SLOTS] { + if slot.is_none() { + *slot = Some(Retained { + object, + prefix: lease.prefix, + cpu: lease.cpu, + poisoned: lease.poisoned, + }); + return; + } + } + // A private CacheLease reserves capacity before conversion, so a + // missing slot means host accounting corruption. Never unmap a cached + // object the firmware may still hold in that failure case. + pr_err!("G17 cached-object return capacity was lost\n"); + core::mem::forget(object); + } + + /// Releases retained mappings after both firmware processors have stopped. + /// Active objects returned later observe the same stop witness. + pub(crate) fn stop_after_firmware(&self) { + self.inner.lock().stopped = true; + for index in 0..POOL_FAMILIES * POOL_SLOTS { + let entry = { + let mut inner = self.inner.lock(); + let entry = inner.returned[index].take(); + if let Some(entry) = &entry { + inner.reserved[index / POOL_SLOTS] -= 1; + inner.bytes -= entry.object.size(); + } + entry + }; + // Mapping teardown must not run while the pool mutex is held. + drop(entry); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs new file mode 100644 index 0000000000000..8806f438234e9 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Work-based firmware performance effort. Idle power gating is managed separately by the +//! device-control ring; this input never keeps an otherwise idle GPU powered. + +pub(super) const FULL_EFFORT: u32 = 0x10000; +const IDLE_HOLD_NS: u64 = 16_000_000; + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum IdleAction { + None, + Clear, + Recheck(u64), +} + +pub(super) struct Effort { + streaming: bool, + idle_since: Option, + applied: u32, +} + +impl Effort { + pub(super) const fn new() -> Self { + Self { + streaming: false, + idle_since: None, + applied: 0, + } + } + + pub(super) fn submit(&mut self) { + self.streaming = true; + self.idle_since = None; + } + + pub(super) fn applied(&self) -> u32 { + self.applied + } + pub(super) fn did_apply(&mut self, value: u32) { + self.applied = value; + } + + /// Quarantined jobs do not count as work in flight; they retain mappings but cannot justify + /// retaining the effort input. New submissions restart the continuous-idle interval. + pub(super) fn idle(&mut self, now_ns: u64, in_flight: bool) -> IdleAction { + if !self.streaming { + return IdleAction::None; + } + if in_flight { + self.idle_since = None; + return IdleAction::None; + } + let since = *self.idle_since.get_or_insert(now_ns.max(1)); + let idle = now_ns.saturating_sub(since); + if idle >= IDLE_HOLD_NS { + self.streaming = false; + self.idle_since = None; + IdleAction::Clear + } else { + IdleAction::Recheck(IDLE_HOLD_NS - idle) + } + } +} + +/// Single-producer control path under the device mutex. Event draining must not publish another +/// control between `set_idle` and the exact three-counter retirement witness. +pub(super) trait Control { + fn crashed(&self) -> bool; + fn powered(&self) -> bool; + fn counters(&self) -> kernel::error::Result<[u32; 3]>; + fn set_idle(&mut self, enabled: bool) -> kernel::error::Result; + fn wait_tick(&self); +} + +fn wait(control: &C, target: u32, require_powered: bool) -> kernel::error::Result { + use kernel::prelude::*; + const RETIRE_POLLS: usize = 20; + const POWER_POLLS: usize = 250; + let mut retired = false; + for elapsed in 0..RETIRE_POLLS + POWER_POLLS { + if control.crashed() { + return Err(EIO); + } + if !retired { + retired = control.counters()? == [target; 3]; + if !retired && elapsed + 1 >= RETIRE_POLLS { + break; + } + } + if retired && (!require_powered || control.powered()) { + return Ok(()); + } + control.wait_tick(); + } + Err(ETIMEDOUT) +} + +/// Disable idle descent through fresh pair registration, including a post-wake reassertion. +pub(super) fn acquire(control: &mut C) -> kernel::error::Result { + use kernel::prelude::*; + let result = (|| { + for _ in 0..3 { + let powered = control.powered(); + let target = control.set_idle(false)?; + wait(control, target, true)?; + if !powered { + let target = control.set_idle(false)?; + wait(control, target, true)?; + } + if control.powered() { + return Ok(()); + } + } + Err(EAGAIN) + })(); + if result.is_err() { + let _ = release(control); + } + result +} + +pub(super) fn release(control: &mut C) -> kernel::error::Result { + let target = control.set_idle(true)?; + wait(control, target, false) +} diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs new file mode 100644 index 0000000000000..8563beda979e6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Admission and lifetime of off-lock graph construction. +//! +//! Entering and leaving an open epoch use only atomics. Recovery closes the epoch while holding +//! the device mutex, then drops that mutex before waiting for builders. A builder's lease is +//! stored after its tentative mappings so those mappings are destroyed before the join completes. + +use core::sync::atomic::{AtomicU64, Ordering}; +use kernel::{ + new_condvar, new_mutex, + prelude::*, + sync::{Arc, CondVar, Mutex}, +}; + +const COUNT_BITS: u32 = 20; +const COUNT_MASK: u64 = (1 << COUNT_BITS) - 1; +const CLOSED: u64 = 1 << COUNT_BITS; +const REMOVED: u64 = CLOSED << 1; +const FAILED: u64 = CLOSED << 2; +const EPOCH: u64 = CLOSED << 3; + +#[pin_data] +pub(super) struct Gate { + state: AtomicU64, + #[pin] + wait: Mutex<()>, + #[pin] + changed: CondVar, +} + +impl Gate { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + state: AtomicU64::new(EPOCH), + wait <- new_mutex!((), "g17::Gate::wait"), + changed <- new_condvar!("g17::Gate::changed"), + }), + GFP_KERNEL, + ) + } + + pub(super) fn try_enter(self: &Arc) -> Result { + let mut previous = self.state.load(Ordering::Acquire); + loop { + if previous & (REMOVED | FAILED) != 0 { + return Err(ENODEV); + } + if previous & CLOSED != 0 { + return Err(EAGAIN); + } + if previous & COUNT_MASK == COUNT_MASK { + return Err(ENOSPC); + } + match self.state.compare_exchange_weak( + previous, + previous + 1, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + return Ok(Lease { + gate: self.clone(), + epoch: previous & !COUNT_MASK, + }) + } + Err(current) => previous = current, + } + } + } + + /// Wait only for a temporary recovery closure. Called with no device lock or existing lease. + pub(super) fn enter(self: &Arc, interruptible: bool) -> Result { + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + let mut wait = self.wait.lock(); + loop { + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + if interruptible { + if self.changed.wait_interruptible(&mut wait) { + return Err(ERESTARTSYS); + } + } else { + self.changed.wait(&mut wait); + } + } + } + + fn invalidate(&self, flags: u64) { + let mut previous = self.state.load(Ordering::Acquire); + loop { + let mut next = previous | CLOSED | flags; + if previous & CLOSED == 0 { + next = next + .checked_add(EPOCH) + .unwrap_or(previous | CLOSED | REMOVED | flags); + } + match self.state.compare_exchange_weak( + previous, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return, + Err(current) => previous = current, + } + } + } + + pub(super) fn close(&self) { + self.invalidate(0); + } + + pub(super) fn remove(&self) { + self.invalidate(REMOVED); + let _wait = self.wait.lock(); + self.changed.notify_all(); + } + + pub(super) fn fail(&self) { + self.invalidate(FAILED); + let _wait = self.wait.lock(); + self.changed.notify_all(); + } + + /// Join builders with no device, queue or mapping lock held. + pub(super) fn wait_drained(&self) { + let mut wait = self.wait.lock(); + while self.state.load(Ordering::Acquire) & COUNT_MASK != 0 { + self.changed.wait(&mut wait); + } + } + + /// Reopen only an empty, temporary closure. Failed or removed devices remain closed. + pub(super) fn reopen(&self) -> bool { + let _wait = self.wait.lock(); + let mut previous = self.state.load(Ordering::Acquire); + loop { + if previous & (REMOVED | FAILED | COUNT_MASK) != 0 { + return false; + } + match self.state.compare_exchange_weak( + previous, + previous & !CLOSED, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + self.changed.notify_all(); + return true; + } + Err(current) => previous = current, + } + } + } +} + +/// An off-lock builder's admission epoch. Keep after all of its tentative mappings. +pub(super) struct Lease { + gate: Arc, + epoch: u64, +} + +impl Lease { + /// Must be checked under the device mutex immediately before publishing the built graph. + pub(super) fn is_current(&self) -> bool { + self.gate.state.load(Ordering::Acquire) & !COUNT_MASK == self.epoch + } +} + +impl Drop for Lease { + fn drop(&mut self) { + let previous = self.gate.state.fetch_sub(1, Ordering::AcqRel); + if previous & COUNT_MASK == 1 && previous & CLOSED != 0 { + // Pair the final decrement with the joiner's check/sleep lock to avoid a lost wakeup. + let _wait = self.gate.wait.lock(); + self.gate.changed.notify_all(); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs new file mode 100644 index 0000000000000..8a925c23552a7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Shared QoS publication and exact-completion accounting for every engine. +//! +//! The device publication lock serializes this host state. Publishing identity +//! precedes submitted counters; retiring a final owner clears only its two row +//! bytes. Cancellation balances host references without rewinding shared counts. + +use super::{fw::qos as fw, object::KernelObject}; +use core::{ + mem::{offset_of, size_of}, + sync::atomic::{fence, AtomicU32, AtomicU64, AtomicU8, Ordering}, +}; +use kernel::prelude::*; +use crate::hw::t8140::qos as cfg; + +/// Atomic views retain the allocation borrow and never borrow firmware words +/// as ordinary mutable memory. The arrays are disjoint apart from the explicit +/// data-master time/count indexing in `times_and_counts`. +pub(crate) struct View<'a> { + rows: &'a [[AtomicU8; 8]; fw::QUEUES], + queue_submitted: &'a [AtomicU32; fw::QUEUES], + owner_submitted: &'a [AtomicU32; fw::QUEUES], + scheduler: &'a [AtomicU64; fw::QUEUES], + share: &'a [AtomicU32; fw::QUEUES], + intervals: &'a AtomicU64, + times_and_counts: &'a [AtomicU64; 7], +} +impl<'a> View<'a> { + pub(crate) fn new(object: &'a KernelObject) -> Result { + let ptr = object.pointer(0, size_of::())?; + if ptr as usize & 7 != 0 { + return Err(EINVAL); + } + // SAFETY: pointer spans the complete layout, whose asserted offsets + // and size bound every array below. Integer atomics have their integer + // representation and alignment. The retained object borrow keeps every + // shared reference alive; accesses use each field's fixed width. + Ok(unsafe { + Self { + rows: &*ptr.add(offset_of!(fw::Qos, rows)).cast(), + queue_submitted: &*ptr.add(offset_of!(fw::Qos, queue_submitted)).cast(), + owner_submitted: &*ptr.add(offset_of!(fw::Qos, owner_submitted)).cast(), + scheduler: &*ptr.add(offset_of!(fw::Qos, scheduler)).cast(), + share: &*ptr.add(offset_of!(fw::Qos, share)).cast(), + intervals: &*ptr.add(offset_of!(fw::Qos, intervals)).cast(), + times_and_counts: &*ptr.add(offset_of!(fw::Qos, busy_start)).cast(), + } + }) + } + fn row(&self, qid: usize) -> [u8; 8] { + core::array::from_fn(|index| self.rows[qid][index].load(Ordering::Relaxed)) + } +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Owner { + pub(crate) qid: u8, + pub(crate) qos: u8, + pub(crate) data_master: u8, +} +impl Owner { + fn indices(self) -> Result<(usize, usize, usize)> { + let (qid, qos, dm) = ( + usize::from(self.qid), + usize::from(self.qos), + usize::from(self.data_master), + ); + if qid >= fw::QUEUES || qos >= fw::QUEUES || dm >= fw::DATA_MASTERS { + return Err(EINVAL); + } + Ok((qid, qos, dm)) + } +} + +/// Retained until completion, or cancellation before the producer is exposed. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Publication { + pub(crate) record: [u8; 8], + owner: Owner, + dm_submitted: u64, +} + +pub(crate) struct Accounting { + in_flight: [u32; fw::QUEUES], + dm_in_flight: [u64; fw::DATA_MASTERS], + dm_by_qid: [u8; fw::QUEUES], +} +impl Accounting { + pub(crate) const fn new() -> Self { + Self { + in_flight: [0; fw::QUEUES], + dm_in_flight: [0; fw::DATA_MASTERS], + dm_by_qid: [0; fw::QUEUES], + } + } + + /// Called under the device publication lock immediately before exposing + /// the work record. The clock is sampled only when its engine becomes busy. + pub(crate) fn publish( + &mut self, + view: &View<'_>, + owner: Owner, + scheduler: u64, + clock: impl FnOnce() -> (u64, u64), + ) -> Result { + let (qid, qos, dm) = owner.indices()?; + if scheduler == 0 { + return Err(EINVAL); + } + let next_qid = self.in_flight[qid].checked_add(1).ok_or(EOVERFLOW)?; + let next_dm = self.dm_in_flight[dm].checked_add(1).ok_or(EOVERFLOW)?; + let starts_busy = self.dm_in_flight[dm] == 0; + let time = if starts_busy { + let (counter, frequency) = clock(); + time_from_host(counter, frequency)? + } else { + 0 + }; + if self.in_flight[qid] != 0 + && (view.rows[qid][0].load(Ordering::Relaxed) != owner.qos + || self.dm_by_qid[qid] != owner.data_master) + { + return Err(EBUSY); + } + view.rows[qid][0].store(owner.qos, Ordering::Relaxed); + view.rows[qid][1].store(cfg::CLASS, Ordering::Relaxed); + let record = view.row(qid); + view.scheduler[qos].store(scheduler, Ordering::Relaxed); + view.share[qos].store(cfg::SHARE, Ordering::Relaxed); + if self.in_flight[qid] == 0 { + fence(Ordering::SeqCst); + } + view.queue_submitted[qid].store( + view.queue_submitted[qid] + .load(Ordering::Relaxed) + .wrapping_add(1), + Ordering::Relaxed, + ); + view.owner_submitted[qos].store( + view.owner_submitted[qos] + .load(Ordering::Relaxed) + .wrapping_add(1), + Ordering::Relaxed, + ); + if starts_busy { + view.times_and_counts[dm].store(time, Ordering::Relaxed); + } + let submitted = &view.times_and_counts[3 + dm]; + let dm_submitted = submitted.load(Ordering::Relaxed).wrapping_add(1); + submitted.store(dm_submitted, Ordering::Relaxed); + if starts_busy { + view.intervals.store( + view.intervals.load(Ordering::Relaxed).wrapping_add(1), + Ordering::Relaxed, + ); + } + self.in_flight[qid] = next_qid; + self.dm_in_flight[dm] = next_dm; + self.dm_by_qid[qid] = owner.data_master; + fence(Ordering::SeqCst); + Ok(Publication { + record, + owner, + dm_submitted, + }) + } + + /// A possibly exposed producer must retain its reference until completion. + /// A newer submission on the same engine makes an old ticket uncancellable. + pub(crate) fn cancel(&mut self, view: &View<'_>, ticket: Publication) -> Result { + let (qid, _, dm) = ticket.owner.indices()?; + if self.in_flight[qid] == 0 + || self.dm_in_flight[dm] == 0 + || view.row(qid) != ticket.record + || view.times_and_counts[3 + dm].load(Ordering::Relaxed) != ticket.dm_submitted + { + return Err(EIO); + } + self.complete(view, &[ticket.owner]) + } + + /// Preflights the entire set before clearing any final row. QID membership, + /// scheduler mappings and monotonically increasing counts remain intact. + pub(crate) fn complete(&mut self, view: &View<'_>, owners: &[Owner]) -> Result { + let mut completed = [0u64; fw::DATA_MASTERS]; + for (index, owner) in owners.iter().copied().enumerate() { + let (qid, _, dm) = owner.indices()?; + if owners[..index].iter().any(|other| other.qid == owner.qid) { + return Err(EINVAL); + } + if self.in_flight[qid] == 0 + || view.rows[qid][0].load(Ordering::Relaxed) != owner.qos + || self.dm_by_qid[qid] != owner.data_master + { + return Err(EIO); + } + completed[dm] += 1; + } + for (dm, count) in completed.iter().enumerate() { + if *count > self.dm_in_flight[dm] { + return Err(EIO); + } + } + for owner in owners { + let qid = usize::from(owner.qid); + if self.in_flight[qid] == 1 { + view.rows[qid][0].store(0xff, Ordering::Relaxed); + view.rows[qid][1].store(0, Ordering::Relaxed); + } + } + for owner in owners { + self.in_flight[usize::from(owner.qid)] -= 1; + } + for (dm, count) in completed.iter().enumerate() { + self.dm_in_flight[dm] -= count; + } + fence(Ordering::SeqCst); + Ok(()) + } +} + +/// Firmware accounting uses a 24 MHz domain independent of host timer speed. +pub(crate) fn time_from_host(counter: u64, frequency: u64) -> Result { + if frequency == 0 || frequency > u64::from(u32::MAX) { + return Err(EINVAL); + } + let whole = (counter / frequency) + .checked_mul(cfg::CLOCK_HZ) + .ok_or(EOVERFLOW)?; + let fraction = (counter % frequency) * cfg::CLOCK_HZ / frequency; + whole.checked_add(fraction).ok_or(EOVERFLOW) +} + +pub(crate) fn clock() -> (u64, u64) { + let counter: u64; + let frequency: u64; + // SAFETY: Both timer registers are readable at EL1 and have no memory effects. + unsafe { + core::arch::asm!("mrs {counter}, CNTPCT_EL0", "mrs {frequency}, CNTFRQ_EL0", + counter=out(reg)counter, frequency=out(reg)frequency, options(nomem,nostack,preserves_flags)); + } + (counter, frequency) +} diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs new file mode 100644 index 0000000000000..1c0d6822c395f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -0,0 +1,541 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Logical queue and physical render-pair admission. A retained pair belongs +//! to one logical owner even while idle. Closing that owner makes it reusable +//! only after every active lease retires. Quarantine retains these leases until +//! a firmware drain witness or both processors stopping makes reuse safe. +//! +//! The pool lock protects host accounting only and is never held while building +//! a graph or waiting for GPU work. Capacity waiters sleep on the release event. + +use crate::g17::{ + hazard::{self, Footprint, Plan}, + job::Packet, +}; +use crate::{hw::t8140::queues as cfg, mmu::MAX_EXECUTION_CONTEXTS}; +use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::{ + new_condvar, new_mutex, + prelude::*, + sync::{Arc, CondVar, CondVarTimeoutResult, Mutex}, +}; + +#[derive(Clone, Copy)] +pub(crate) struct Request<'a> { + footprint: &'a Arc, + sequence: u64, + prefix: Option, +} + +impl<'a> Request<'a> { + pub(crate) fn of(packet: &'a Packet) -> Option { + packet.footprint.as_ref().map(|footprint| Self { + footprint, + sequence: packet.order.sequence, + prefix: packet.order.wait_through[0], + }) + } +} + +struct InFlight { + owner: u64, + sequence: u64, + slot: Option, + ticket: u64, + footprint: Arc, +} + +const QUARANTINE_WAIT_MS: u32 = 1000; + +struct State { + active: u64, + refs: [u8; cfg::RENDER_SLOTS], + owners: [u64; cfg::RENDER_SLOTS], + epochs: [u64; cfg::RENDER_SLOTS], + quarantined: u64, + kill_hold: u64, + released: u64, + epoch: u64, + footprints: KVec, + next_ticket: u64, +} +impl State { + const MASK: u64 = (u64::MAX >> (64 - cfg::RENDER_SLOTS)) & !1; + const fn new() -> Self { + Self { + active: 0, + refs: [0; cfg::RENDER_SLOTS], + owners: [0; cfg::RENDER_SLOTS], + epochs: [0; cfg::RENDER_SLOTS], + quarantined: 0, + kill_hold: 0, + released: 0, + epoch: 0, + footprints: KVec::new(), + next_ticket: 1, + } + } + fn exhausted(&self) -> bool { + self.quarantined & Self::MASK == Self::MASK + } + fn refusal(&self, owner: u64) -> Result { + if self.exhausted() { + return Err(EIO); + } + match self.owners.iter().position(|id| *id == owner) { + Some(slot) if self.quarantined & (1u64 << slot) != 0 => Err(EIO), + None if !self.owners[1..].contains(&0) + && self.released + & !self.active + & !self.quarantined + & !self.kill_hold + & Self::MASK + == 0 => + { + Err(ENOSPC) + } + _ => Ok(()), + } + } + fn advance(&mut self, slot: usize) { + self.epoch = self.epoch.wrapping_add(1); + self.epochs[slot] = self.epoch; + } + fn take_idle(&mut self, slot: usize, owner: u64) -> u8 { + self.refs[slot] += 1; + self.active |= 1u64 << slot; + self.released &= !(1u64 << slot); + self.owners[slot] = owner; + self.advance(slot); + slot as u8 + } + fn acquire(&mut self, owner: u64) -> Option { + if owner == 0 { + return None; + } + // Prefer the owner's idle pair. A missing sibling can be claimed + // only while this owner has no live kick, before constructing it. + for slot in 0..cfg::RENDER_SLOTS { + if self.owners[slot] == owner + && self.refs[slot] == 0 + && self.quarantined & (1u64 << slot) == 0 + { + if !(0..cfg::RENDER_SLOTS) + .any(|other| self.owners[other] == owner && self.refs[other] != 0) + { + self.claim_sibling(owner); + } + return Some(self.take_idle(slot, owner)); + } + } + if self.owners.contains(&owner) { + // Consecutive kicks are ordered by firmware dependencies. Stack + // onto the least-loaded retained pair without a retirement wait. + let slot = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + self.owners[*slot] == owner + && self.quarantined & (1u64 << slot) == 0 + && self.active & (1u64 << slot) != 0 + && self.refs[*slot] < cfg::RENDER_DEPTH + }) + .min_by_key(|slot| (self.refs[*slot], u64::MAX - self.epochs[*slot]))?; + self.refs[slot] += 1; + self.advance(slot); + return Some(slot as u8); + } + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + let eligible = |slot: usize| free & (1u64 << slot) != 0; + // Reuse a closed, exactly retired owner before consuming a new graph. + let slot = (0..cfg::RENDER_SLOTS) + .filter(|slot| eligible(*slot) && self.released & (1u64 << slot) != 0) + .min_by_key(|slot| self.epochs[*slot]) + .or_else(|| { + (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) + })?; + let first = self.take_idle(slot, owner); + self.claim_sibling(owner); + Some(first) + } + fn claim_sibling(&mut self, owner: u64) { + if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER { + return; + } + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + let eligible = |slot: usize| free & (1u64 << slot) != 0; + let sibling = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + eligible(*slot) + && self.owners[*slot] != owner + && self.released & (1u64 << slot) != 0 + }) + .min_by_key(|slot| self.epochs[*slot]) + .or_else(|| { + (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) + }); + if let Some(slot) = sibling { + self.released &= !(1u64 << slot); + self.owners[slot] = owner; + self.advance(slot); + } + } + fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { + if self.refs.get(usize::from(publishing)) != Some(&1) { + return None; + } + (0..cfg::RENDER_SLOTS) + .find(|slot| { + *slot != usize::from(publishing) + && self.owners[*slot] == owner + && self.refs[*slot] == 0 + && self.quarantined & (1u64 << slot) == 0 + }) + .map(|slot| slot as u8) + } + fn plan(&self, owner: u64, request: Option>, ticket: Option) -> Plan { + let Some(request) = request else { + return Plan::Any; + }; + hazard::plan( + owner, + request.prefix, + ticket, + request.footprint, + self.footprints.iter().map(|entry| hazard::Live { + owner: entry.owner, + sequence: entry.sequence, + slot: entry.slot, + ticket: entry.ticket, + footprint: &entry.footprint, + quarantined: entry + .slot + .is_some_and(|slot| self.quarantined & (1u64 << slot) != 0), + }), + ) + } + + fn pick(&mut self, owner: u64, plan: Plan) -> Option { + match plan { + Plan::Any => self.acquire(owner), + Plan::Wait => None, + Plan::Pair(slot) => { + let index = usize::from(slot); + if self.owners.get(index) != Some(&owner) + || (self.quarantined | self.kill_hold) & (1u64 << slot) != 0 + || self.refs[index] >= cfg::RENDER_DEPTH + { + return None; + } + if self.refs[index] == 0 { + Some(self.take_idle(index, owner)) + } else { + self.refs[index] += 1; + self.advance(index); + Some(slot) + } + } + } + } + + fn record( + &mut self, + owner: u64, + slot: Option, + ticket: u64, + request: Request<'_>, + ) -> Option<(u64, u64)> { + self.footprints + .push( + InFlight { + owner, + sequence: request.sequence, + slot, + ticket, + footprint: request.footprint.clone(), + }, + GFP_KERNEL, + ) + .ok()?; + Some((owner, request.sequence)) + } + + fn forget_waiter(&mut self, ticket: Option) { + if let Some(ticket) = ticket { + self.footprints + .retain(|entry| entry.slot.is_some() || entry.ticket != ticket); + } + } + + fn release(&mut self, slot: u8) { + let slot = usize::from(slot); + if self.refs[slot] != 0 { + self.refs[slot] -= 1; + } + if self.refs[slot] == 0 { + self.active &= !(1u64 << slot); + } + } +} + +#[pin_data] +pub(crate) struct Pool { + #[pin] + state: Mutex, + #[pin] + released: CondVar, + queues: AtomicU32, +} +impl Pool { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + state <- new_mutex!(State::new(), "G17 render admission"), + released <- new_condvar!("G17 render pair release"), + queues: AtomicU32::new(0), + }), + GFP_KERNEL, + ) + } + pub(crate) fn reserve_queue(self: &Arc) -> Result { + let mut count = self.queues.load(Ordering::Acquire); + loop { + if count >= MAX_EXECUTION_CONTEXTS { + return Err(ENOSPC); + } + match self.queues.compare_exchange( + count, + count + 1, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return Ok(QueueSlot { pool: self.clone() }), + Err(current) => count = current, + } + } + } + pub(crate) fn acquire_render( + self: &Arc, + owner: u64, + request: Option>, + ) -> Result { + if owner == 0 { + return Err(EINVAL); + } + let mut state = self.state.lock(); + let mut conflict_budget = kernel::time::msecs_to_jiffies(hazard::WAIT_MS); + let mut quarantine_budget = kernel::time::msecs_to_jiffies(QUARANTINE_WAIT_MS); + let mut expired = false; + let mut ticket = None; + let result = (|| loop { + if state.exhausted() { + return Err(EIO); + } + // The first retained pair's quarantine is not bypassed through its + // healthy sibling. Reclamation wakes this bounded admission wait. + if state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) + { + if quarantine_budget == 0 { + return Err(EIO); + } + quarantine_budget = match self + .released + .wait_interruptible_timeout(&mut state, quarantine_budget) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => 0, + CondVarTimeoutResult::Woken { jiffies } => jiffies, + }; + continue; + } + state.refusal(owner)?; + let plan = if expired { + Plan::Any + } else { + state.plan(owner, request, ticket) + }; + if let Some(slot) = state.pick(owner, plan) { + state.forget_waiter(ticket.take()); + let record = + request.and_then(|request| state.record(owner, Some(slot), 0, request)); + return Ok(RenderLease { + pool: self.clone(), + slot, + record, + }); + } + if plan != Plan::Any { + if ticket.is_none() { + if let Some(request) = request { + let next = state.next_ticket; + state.next_ticket = next.wrapping_add(1); + if state.record(owner, None, next, request).is_some() { + ticket = Some(next); + } + } + } + match self + .released + .wait_interruptible_timeout(&mut state, conflict_budget) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => { + conflict_budget = 0; + expired = true; + } + CondVarTimeoutResult::Woken { jiffies } => conflict_budget = jiffies, + } + } else if self.released.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + })(); + state.forget_waiter(ticket); + result + } + + /// Early publication never waits for overlap; the scheduler retries with + /// the same footprint after its dependencies have become ready. + pub(crate) fn try_acquire_render( + self: &Arc, + owner: u64, + request: Option>, + ) -> Result> { + if owner == 0 { + return Ok(None); + } + let mut state = self.state.lock(); + if state.refusal(owner).is_err() || !state.owners.contains(&owner) { + return Ok(None); + } + let plan = state.plan(owner, request, None); + let Some(slot) = state.pick(owner, plan) else { + return Ok(None); + }; + let record = request.and_then(|request| state.record(owner, Some(slot), 0, request)); + Ok(Some(RenderLease { + pool: self.clone(), + slot, + record, + })) + } + pub(crate) fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { + self.state.lock().warmable_sibling(owner, publishing) + } + pub(crate) fn release_owner(&self, owner: u64) { + let mut state = self.state.lock(); + for slot in 0..cfg::RENDER_SLOTS { + if state.owners[slot] == owner { + state.released |= 1u64 << slot; + } + } + drop(state); + self.released.notify_all(); + } + /// Active leases survive this transition. Their later drop is still the + /// event which releases capacity, so a stopped owner is never double-freed. + pub(crate) fn release_retained_owners_after_processor_stop(&self) { + let mut state = self.state.lock(); + let released = state.released & !state.active; + for slot in 1..cfg::RENDER_SLOTS { + if released & (1u64 << slot) != 0 { + state.owners[slot] = 0; + state.epochs[slot] = 0; + } + } + state.released &= !released; + state.quarantined &= !released; + state.kill_hold = 0; + drop(state); + self.released.notify_all(); + } +} + +/// Counted once per userspace logical queue, independently of installed graphs. +pub(crate) struct QueueSlot { + pool: Arc, +} +impl Drop for QueueSlot { + fn drop(&mut self) { + self.pool.queues.fetch_sub(1, Ordering::AcqRel); + } +} + +pub(crate) struct RenderLease { + pool: Arc, + slot: u8, + record: Option<(u64, u64)>, +} +impl RenderLease { + pub(crate) fn slot(&self) -> u8 { + self.slot + } + pub(crate) fn quarantine(&self) { + self.pool.state.lock().quarantined |= 1u64 << self.slot; + self.pool.released.notify_all(); + } + /// The VM pin must be retired first. Keep this lease until after clearing + /// quarantine; its drop is what permits waiting publications to proceed. + pub(crate) fn rehabilitate_after_processor_stop(&self) { + self.pool.state.lock().quarantined &= !(1u64 << self.slot); + } + /// Requires the runtime's closed-recovery, exact pair-drain witness. + pub(crate) fn rehabilitate_after_firmware_drain(&self) { + self.rehabilitate_after_processor_stop(); + } +} +impl Drop for RenderLease { + fn drop(&mut self) { + let mut state = self.pool.state.lock(); + if let Some((owner, sequence)) = self.record { + if let Some(index) = state.footprints.iter().position(|entry| { + entry.slot == Some(self.slot) && entry.owner == owner && entry.sequence == sequence + }) { + state.footprints.swap_remove(index); + } + } + state.release(self.slot); + drop(state); + self.pool.released.notify_all(); + } +} + +impl State { + fn hold_for_exit_kill(&mut self, owner: u64, slots: u64) -> bool { + if owner == 0 || slots & !Self::MASK != 0 { + return false; + } + for slot in 0..cfg::RENDER_SLOTS { + let bit = 1u64 << slot; + if slots & bit == 0 { + continue; + } + if self.owners[slot] != owner + || self.refs[slot] != 0 + || self.active & bit != 0 + || self.released & bit == 0 + { + return false; + } + } + self.kill_hold |= slots; + true + } +} +impl Pool { + /// Hold the departed owner's idle pairs before publishing its exit record. + /// Successful publication retains these bits until both processors stop. + pub(crate) fn hold_render_slots_for_exit_kill(&self, owner: u64, slots: u64) -> bool { + self.state.lock().hold_for_exit_kill(owner, slots) + } + + /// Undo only a hold whose control record was not published. + pub(crate) fn release_exit_kill_hold(&self, slots: u64) { + if slots == 0 { + return; + } + self.state.lock().kill_hold &= !slots; + self.released.notify_all(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs new file mode 100644 index 0000000000000..f8297f0462ff8 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -0,0 +1,1507 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Retained physical compute queues. Logical ownership and command retirement never free an +//! installed graph; the device retains it until both firmware processors have stopped. + +pub(crate) mod exit; +pub(crate) mod retirement; + +use crate::g17::{ + context::Context, + fw::{compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, + object::{Allocator, CpuMap, KernelObject}, +}; +use crate::{hw::t8140::dynamic, mmu}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{prelude::*, sync::Arc}; +use retirement::{Cursors, RECORD_SLOTS}; + +const DESCRIPTORS: usize = 256; +const GRAPH_SIZE: usize = 0x8000; +const CONTEXT_SIZE: usize = 0x20000; +const RECORD_STRIDE: usize = 0x1000; +const CONFIG: usize = 0; +const ANNOUNCE: usize = 0x800; +const PREDECESSOR: usize = 0x400; +const POINTERS: usize = 0x100; +const ITEMS: usize = 0x4000; +const STAMP: usize = 0x300; + +#[repr(C)] +struct GraphLayout { + queue: QueueRecord, + unk_c0: [u8; 0x40], + pointers: PointerBlock, + unk_180: [u8; 0x80], + jobs: JobListHead, + unk_210: [u8; 0x3df0], + items: [u64; ITEM_RING_ENTRIES as usize], + unk_6800: [u8; 0x1800], +} +// SAFETY: The layout contains only integer storage and integer atomics, all valid when zeroed. +unsafe impl Zeroable for GraphLayout {} +static_assert!(core::mem::size_of::() == GRAPH_SIZE); +static_assert!(core::mem::offset_of!(GraphLayout, pointers) == POINTERS); +static_assert!(core::mem::offset_of!(GraphLayout, items) == ITEMS); + +/// Aliases are destroyed before the logical root they name. Superseded bindings remain owned +/// until the first exact successful publication under their replacement has retired. +struct Binding { + _kick_alias: mmu::KernelMapping, + _pool_alias: mmu::KernelMapping, + _descriptors: mmu::KernelMapping, + preempt: mmu::KernelMapping, + usage: mmu::KernelMapping, + operand: mmu::KernelMapping, + _queue_context: mmu::KernelMapping, + context: Arc, +} + +/// Firmware-shared storage of one compute QID, separate from its common kick ring. +struct Graph { + descriptor_high: mmu::KernelMapping, + descriptors: KernelObject, + preempt: KernelObject, + usage: KernelObject, + operand: KernelObject, + support: KernelObject, + graph: KernelObject, + context_low: KernelObject, + context_high: KernelObject, + records: KernelObject, + record_ends: [Option; RECORD_SLOTS], + reclaimed: u32, +} + +impl Graph { + fn new(alloc: &Allocator<'_>, context: &Context) -> Result { + let page = mmu::UAT_PGSZ as u64; + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let cpu = CpuMap::WriteCombined; + let descriptors = alloc.lower(DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, page, prot, cpu)?; + let mut high = alloc.uat.geometry().kernel_range(); + high.start += dynamic::KERNEL_OFFSET; + let descriptor_high = descriptors.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; + let preempt = alloc.kernel(0x4000, page, prot, cpu)?; + let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; + let operand = alloc.lower(0x14000, page, prot, cpu)?; + let mut support = alloc.kernel(0xc000, page, prot, cpu)?; + let support_va = support.gpu_va(); + support.write(0, support_va + page)?; + support.write(0x100, support_va + page + 4)?; + support.write(0x110, 0x50u32)?; + support.write(mmu::UAT_PGSZ + 4, 1u32)?; + let mut graph = alloc.kernel(GRAPH_SIZE, page, prot, cpu)?; + let graph_va = graph.gpu_va(); + graph.initialize::(0, |view| { + view.queue = QueueRecord::new(&QueueRecordArgs { + pointers_va: graph_va + POINTERS as u64, + items_va: graph_va + ITEMS as u64, + job_list_va: context.work_head_va(), + scheduler_va: context.scheduler_va(), + owner_pid: context.owner_pid(), + }); + view.pointers = PointerBlock::new(); + view.jobs = JobListHead::new(graph_va + 0x200); + Ok(()) + })?; + let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; + let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; + let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; + Ok(Self { + descriptor_high, + descriptors, + preempt, + usage, + operand, + support, + graph, + context_low, + context_high, + records, + record_ends: [None; RECORD_SLOTS], + reclaimed: 0, + }) + } + + /// All fallible mapping work finishes before replacing a physical queue's logical owner. + fn bind( + &mut self, + context: Arc, + kick_alias: mmu::KernelMapping, + pool_alias: mmu::KernelMapping, + ) -> Result { + if !context.is_current() { + return Err(EFAULT); + } + let vm = context.vm(); + vm.install_job_context_aliases()?; + let descriptors = + self.descriptors + .map_alias(vm, self.descriptors.gpu_va(), mmu::PROT_GPU_SHARED_RO)?; + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let page = mmu::UAT_PGSZ as u64; + let preempt = self + .preempt + .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + let usage = self.usage.alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + let operand = self + .operand + .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + let queue_context = self + .context_low + .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + Ok(Binding { + _kick_alias: kick_alias, + _pool_alias: pool_alias, + _descriptors: descriptors, + preempt, + usage, + operand, + _queue_context: queue_context, + context, + }) + } + + /// Caller has excluded publication and proved the retained graph idle. + fn set_owner(&mut self, context: &Context) -> Result { + let pointer = self + .graph + .pointer(0, core::mem::size_of::())? + .cast::(); + // SAFETY: The checked record is in the retained mapping, has byte alignment and consists + // only of integer storage. The device mutex and idle witness exclude simultaneous access. + unsafe { + (*pointer).set_compute_owner( + context.work_head_va(), + context.scheduler_va(), + context.owner_pid(), + ) + }; + Ok(()) + } + + fn cursors(&self) -> Result { + Ok(Cursors { + producer: self.graph.word(POINTERS + 0x40)?.load(Ordering::Relaxed), + consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), + count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), + }) + } + fn stamp(&self) -> Result { + fence(Ordering::Acquire); + Ok(self.support.word(STAMP)?.load(Ordering::Relaxed)) + } + fn descriptor_vas(&self, slot: u8) -> (u64, u64) { + let offset = u64::from(slot) * COMPUTE_DESCRIPTOR_SIZE as u64; + ( + self.descriptor_high.iova() + offset, + self.descriptors.gpu_va() + offset, + ) + } + fn ensure_record_slot(&self, slot: usize) -> Result { + if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { + let cursors = self.cursors()?; + if !cursors.consumed(end, cursors.count)? { + return Err(EBUSY); + } + } + Ok(()) + } + + /// Reclaim only consumed item pointers, with individual aligned stores and a release fence. + /// This leaves records still named by firmware untouched across arbitrarily many wraps. + fn recycle(&mut self, cursors: Cursors) -> Result { + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } + if cursors.consumer >= cursors.count || self.reclaimed >= cursors.count { + return Err(ERANGE); + } + let done = cursors.consumer; + while self.reclaimed != done { + self.graph + .dword(ITEMS + self.reclaimed as usize * 8)? + .store(0, Ordering::Relaxed); + self.reclaimed = (self.reclaimed + 1) % cursors.count; + } + fence(Ordering::SeqCst); + Ok(()) + } +} + +use crate::g17::{ + completion::Deferred, context::WorkStateLease, freelist::ComputePool, job::Packet, kick, qos, + recovery, status::VmStatus, +}; +use retirement::{Batch, Observation, Recordless, Ticket, IN_FLIGHT}; + +struct Fifo { + entries: [Option; IN_FLIGHT], + head: usize, + len: usize, +} +impl Fifo { + fn new() -> Self { + Self { + entries: core::array::from_fn(|_| None), + head: 0, + len: 0, + } + } + fn front(&self) -> Option<&T> { + self.entries[self.head].as_ref() + } + fn back(&self) -> Option<&T> { + if self.len == 0 { + None + } else { + self.entries[(self.head + self.len - 1) % IN_FLIGHT].as_ref() + } + } + fn back_mut(&mut self) -> Option<&mut T> { + if self.len == 0 { + None + } else { + self.entries[(self.head + self.len - 1) % IN_FLIGHT].as_mut() + } + } + fn push(&mut self, value: T) -> core::result::Result<(), T> { + if self.len == IN_FLIGHT { + return Err(value); + } + self.entries[(self.head + self.len) % IN_FLIGHT] = Some(value); + self.len += 1; + Ok(()) + } + fn pop(&mut self) -> Option { + if self.len == 0 { + return None; + } + let value = self.entries[self.head].take(); + self.head = (self.head + 1) % IN_FLIGHT; + self.len -= 1; + value + } + fn pop_back(&mut self) -> Option { + if self.len == 0 { + return None; + } + self.len -= 1; + self.entries[(self.head + self.len) % IN_FLIGHT].take() + } + fn iter(&self) -> impl Iterator { + Iterator::chain( + self.entries[self.head..].iter(), + self.entries[..self.head].iter(), + ) + .take(self.len) + .filter_map(Option::as_ref) + } + fn iter_mut(&mut self) -> impl Iterator { + let (before, after) = self.entries.split_at_mut(self.head); + Iterator::chain(after.iter_mut(), before.iter_mut()) + .take(self.len) + .filter_map(Option::as_mut) + } +} + +struct Active { + packet: Arc, + work: Arc, + ticket: Option, + observed: Option, + failure_deferred: bool, + qos_completed: bool, +} +struct Previous { + binding: Option, + context: Arc, + publication: Option<(u64, u64)>, +} +#[derive(Copy, Clone, PartialEq, Eq)] +struct ConfigIdentity { + scheduler: u64, + qos: u8, + context: u32, +} + +/// A private physical pool whose descriptor row is installed before allocating its kick ring. +/// The QID remains unpublished throughout construction. +pub(crate) struct PoolBacking { + pool: ComputePool, + id: kick::Id, +} + +impl PoolBacking { + pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id) -> Result { + let pool = ComputePool::new(alloc, id.qid())?; + Ok(Self { pool, id }) + } + + /// Fresh page-pool row data, installed once while the QID is still unpublished. + pub(crate) fn pool_descriptor(&self) -> (u16, u64) { + (self.pool.id(), self.pool.page_list_va()) + } +} + +/// A physical pool and kick ring prepared before the device-global compute scheduler and +/// descriptor graph. No logical VM aliases or firmware publication exist at this stage. +pub(crate) struct Backing { + pool: ComputePool, + kick: kick::Queue, +} + +impl Backing { + pub(crate) fn new(alloc: &Allocator<'_>, backing: PoolBacking) -> Result { + let PoolBacking { pool, id } = backing; + let kick = kick::Queue::compute(alloc, id)?; + Ok(Self { pool, kick }) + } +} + +/// Installed graph and its exact logical owner. A released idle graph may change owners only +/// after a retirement witness. Failed work retains its pointers independently of fence state. +pub(crate) struct Queue { + owner: Option, + binding: Binding, + previous: Option, + kick: kick::Queue, + pool: ComputePool, + graph: Graph, + active: Fifo, + replays: Fifo>, + replay_retirement: bool, + replay_error: Option, + replay_prepared: bool, + exit: Option, + installed: bool, + outer_started: bool, + published_config: Option, + ordinal: u32, + submitted: u32, + last_end: u64, + released: bool, + retirement_ready: bool, + quarantined: bool, + retired_by_teardown: bool, + retire_pending: bool, + awaiting_witness: bool, + spared_quarantine: bool, + spared_deferred: bool, + terminal: Option<(Arc, Error)>, + quarantine_error: Option, + failure_status_pending: bool, + retirement_proved: bool, +} + +impl Queue { + /// The caller reserves a QID first, cancels it on construction failure, and retains every + /// successfully installed queue until processor stop. Shared VM apertures precede this call. + pub(crate) fn new( + alloc: &Allocator<'_>, + backing: Backing, + owner: u64, + context: Arc, + ) -> Result { + let Backing { mut pool, mut kick } = backing; + let mut graph = Graph::new(alloc, &context)?; + let kick_alias = kick.map_client(context.vm())?; + let pool_alias = pool.map_client(context.vm())?; + let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; + context.mark_published(); + Ok(Self { + owner: Some(owner), + binding, + previous: None, + kick, + pool, + graph, + active: Fifo::new(), + replays: Fifo::new(), + replay_retirement: false, + replay_error: None, + replay_prepared: false, + exit: None, + installed: false, + outer_started: false, + published_config: None, + ordinal: 0, + submitted: 0, + last_end: 0, + released: false, + retirement_ready: false, + quarantined: false, + retired_by_teardown: false, + retire_pending: false, + awaiting_witness: false, + spared_quarantine: false, + spared_deferred: false, + terminal: None, + quarantine_error: None, + failure_status_pending: false, + retirement_proved: false, + }) + } + pub(crate) fn qid(&self) -> u8 { + self.kick.id().qid() + } + pub(crate) fn owner(&self) -> Option { + self.owner + } + pub(crate) fn context(&self) -> &Arc { + &self.binding.context + } + /// Quarantined commands still retain the scheduler until their retirement witness. + pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { + !Arc::ptr_eq(self.context(), context) || (self.active.len == 0 && self.replays.len == 0) + } + pub(crate) fn in_flight(&self) -> bool { + self.active.len != 0 && !self.quarantined + } + pub(crate) fn room(&self) -> bool { + !self.quarantined && !self.released && self.replays.len == 0 && self.active.len < IN_FLIGHT + } + pub(crate) fn oldest_spared(&self) -> Option { + self.active + .front() + .map(|a| a.packet.completion.spared() && a.packet.completion.replay_count() == 0) + } + pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { + for active in self.active.iter() { + visit(active.packet.completion.status()); + } + } + pub(crate) fn classify(&self, class: &dyn Fn(&Arc) -> recovery::Class) { + for active in self.active.iter() { + active + .packet + .completion + .classify(class(active.packet.completion.status())); + } + } + fn qos_owner(&self) -> qos::Owner { + qos::Owner { + qid: self.qid(), + qos: self.binding.context.qos_id(), + data_master: DataMaster::Compute as u8, + } + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { + self.retired_by_teardown + || self.retire_pending + || masks.is_none_or(|m| m[self.qid() as usize / 64] & (1 << (self.qid() % 64)) != 0) + } + /// Stage every unlatched exact identity before the device's single shared-ring scan. + pub(crate) fn request_observations( + &self, + batch: &mut Batch, + masks: Option<[u64; 2]>, + ) -> Result { + if self.quarantined || !self.selected(masks) { + return Ok(false); + } + let mut requested = false; + for active in self.active.iter() { + let Some(ticket) = active.ticket else { + break; + }; + if active.observed.is_none() { + batch.request(&ticket)?; + requested = true; + } + } + Ok(requested) + } + pub(crate) fn latch_observations( + &mut self, + batch: &mut Batch, + masks: Option<[u64; 2]>, + ) -> Result { + if self.quarantined || !self.selected(masks) { + return Ok(()); + } + for active in self.active.iter_mut() { + let Some(ticket) = active.ticket else { + break; + }; + if active.observed.is_none() { + active.observed = batch.take(&ticket)?; + } + } + Ok(()) + } + fn front_finished(&self) -> bool { + self.active.front().is_some_and(|active| { + active.observed.is_some() + || active.ticket.is_some_and(|ticket| { + self.graph + .cursors() + .and_then(|c| c.consumed(ticket.item_producer, ticket.item_count)) + .unwrap_or(false) + }) + }) + } + fn valid_ticket(&self, active: &Active, ticket: &Ticket) -> Result { + if ticket.qid != self.qid() + || ticket.descriptor == 0 + || ticket.item_count == 0 + || ticket.work_node != active.work.node_va()? + || self.submitted == 0 + || self.owner != Some(ticket.owner) + || active.packet.order.sequence != ticket.submission + || active.packet.context.id() != ticket.context_id + || active.packet.context.id() != self.binding.context.id() + { + return Err(EIO); + } + Ok(()) + } + + /// Return VM identity for queue-local failure fanout. Deferred callbacks are collected under + /// the mutex and invoked afterwards; failed packets stay retained until their witness. + pub(crate) fn quarantine( + &mut self, + error: Error, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result>> { + if !self.quarantined { + self.quarantined = true; + self.quarantine_error = Some(error); + self.failure_status_pending = true; + self.retirement_proved = false; + self.retired_by_teardown = false; + let all_spared = + self.active.len != 0 && self.active.iter().all(|a| a.packet.completion.spared()); + self.spared_quarantine = all_spared && error == ENODATA; + self.spared_deferred = self.spared_quarantine; + } + let error = self.quarantine_error.ok_or(EIO)?; + let mut terminal = None; + for active in self.active.iter_mut() { + if !self.spared_deferred && !active.failure_deferred { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + error, + ))?; + active.failure_deferred = true; + } + if self.failure_status_pending + && !active.packet.completion.spared() + && terminal.is_none() + { + terminal = Some(active.packet.completion.status().clone()); + } + } + self.failure_status_pending = false; + self.awaiting_witness = + self.active.len != 0 && self.active.iter().all(|a| a.ticket.is_some()); + Ok(terminal) + } + + /// Poll only the ordered prefix; the next pass may recheck a finished front on another QID's + /// event. A final identity-less scan allows recordless retirement without the one-pass grace. + pub(crate) fn retire( + &mut self, + masks: Option<[u64; 2]>, + epoch: u64, + recovery_state: u32, + mut complete_qos: impl FnMut(qos::Owner) -> Result, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if self.quarantined || !self.selected(masks) { + return Ok(()); + } + let finished_before = self.retire_pending; + self.retire_pending = false; + let mut oldest = true; + while let Some(front) = self.active.front() { + let result = (|| { + let ticket = front.ticket.ok_or(EIO)?; + self.valid_ticket(front, &ticket)?; + if recovery_state != 0 { + return Err(EAGAIN); + } + let recordless = if epoch != ticket.epoch { + Recordless::CrossedRecovery + } else if masks.is_none() || (finished_before && oldest) { + Recordless::Allowed + } else { + Recordless::NotYet + }; + let next_end = self + .active + .iter() + .skip(1) + .find_map(|a| a.observed.map(|o| o.timestamps[1])); + let timestamps = retirement::poll( + &ticket, + front.observed, + self.graph.cursors()?, + self.graph.stamp()?, + recordless, + self.last_end, + next_end, + )?; + if timestamps.is_some() + && self.previous.as_ref().is_some_and(|p| { + p.publication != Some((front.packet.order.sequence, ticket.epoch)) + }) + { + return Err(EIO); + } + Ok(timestamps) + })(); + oldest = false; + match result { + Ok(Some(timestamps)) => { + let submitted = self.submitted.checked_sub(1).ok_or(EIO)?; + if !front.qos_completed { + complete_qos(self.qos_owner())?; + self.active.iter_mut().next().ok_or(EIO)?.qos_completed = true; + } + let front = self.active.front().ok_or(EIO)?; + self.retire_pending = true; + defer(Deferred::Retired( + front.packet.completion.clone(), + Ok([timestamps[0], timestamps[1], 0, 0]), + ))?; + self.retire_pending = false; + self.submitted = submitted; + let active = self.active.pop().ok_or(EIO)?; + if active.observed.is_some() { + self.last_end = self.last_end.max(timestamps[1]); + } + drop(self.previous.take()); + self.retirement_ready = self.active.len == 0; + if self.released && self.retirement_ready { + self.owner = None; + } + } + Ok(None) | Err(EAGAIN) => { + self.retire_pending = self.front_finished(); + break; + } + Err(error) => { + self.terminal = self.quarantine(error, defer)?.map(|status| (status, error)); + return Ok(()); + } + } + } + Ok(()) + } + + /// Exact terminal witness releases command pins, not installed graph ownership. Spared + /// zero-duration work can return its queue to service after settling its accounting. + pub(crate) fn release_witnessed( + &mut self, + recovery_state: u32, + mut complete_qos: impl FnMut(qos::Owner) -> Result, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.release_witnessed_inner(recovery_state, false, &mut complete_qos, defer) + } + + pub(crate) fn prepare_replays( + &mut self, + recovery_state: u32, + complete_qos: impl FnMut(qos::Owner) -> Result, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.release_witnessed_inner(recovery_state, true, complete_qos, defer) + } + + fn release_witnessed_inner( + &mut self, + recovery_state: u32, + replay: bool, + mut complete_qos: impl FnMut(qos::Owner) -> Result, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if !self.awaiting_witness || recovery_state != 0 { + return Ok(()); + } + if !self.retirement_proved { + for active in self.active.iter() { + self.valid_ticket(active, &active.ticket.ok_or(EIO)?)?; + } + let newest = self.active.back().and_then(|a| a.ticket).ok_or(EIO)?; + if !retirement::quarantined_retired( + &newest, + self.graph.cursors()?, + self.graph.stamp()?, + self.active.len as u32, + self.submitted, + )? { + return Ok(()); + } + let candidate = self.spared_deferred + && self.spared_quarantine + && !self.released + && self.owner == Some(newest.owner) + && !self + .active + .iter() + .any(|active| active.packet.completion.has_firmware_consumer()) + && self.active.iter().all(|active| { + active + .packet + .completion + .replay_witness_words() + .is_some_and(|words| words[0] == words[1]) + }); + if candidate && !replay { + return Ok(()); + } + self.replay_retirement = candidate; + self.retirement_proved = true; + } + while let Some(front) = self.active.front() { + if self.spared_quarantine && !front.qos_completed { + complete_qos(self.qos_owner())?; + self.active.iter_mut().next().ok_or(EIO)?.qos_completed = true; + } + let front = self.active.front().ok_or(EIO)?; + let completion = front.packet.completion.clone(); + if self.replay_retirement { + self.replays.push(front.packet.clone()).map_err(|_| EIO)?; + } else if self.spared_deferred { + defer(Deferred::Retired(completion, Err(ENODATA)))?; + } else { + defer(Deferred::Release(completion))?; + } + drop(self.active.pop()); + } + self.awaiting_witness = false; + self.retirement_proved = false; + self.replay_retirement = false; + self.spared_deferred = false; + if self.spared_quarantine { + self.spared_quarantine = false; + self.quarantine_error = None; + self.submitted = 0; + self.kick.clear_parent_after_recovery(); + self.quarantined = false; + self.retire_pending = false; + self.retirement_ready = true; + if self + .previous + .as_ref() + .is_some_and(|p| p.publication.is_some()) + { + drop(self.previous.take()); + } + if self.released { + self.owner = None; + } + } + Ok(()) + } + + pub(crate) fn replay_front(&self) -> Option> { + self.replays.front().cloned() + } + + /// Drop the witnessed owner only after either publication retained it or + /// the off-lock completion collector accepted its terminal result. + pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { + if !self + .replays + .front() + .is_some_and(|front| Arc::ptr_eq(front, packet)) + { + return Err(EIO); + } + drop(self.replays.pop()); + self.replay_prepared = false; + if self.replays.len == 0 { + self.replay_error = None; + } + Ok(()) + } + + pub(crate) fn prepare_replay(&mut self, packet: &Arc) -> Result { + if !self + .replays + .front() + .is_some_and(|front| Arc::ptr_eq(front, packet)) + { + return Err(EIO); + } + if let Some(error) = self.replay_error { + return Err(error); + } + if !self.replay_prepared { + if !packet.completion.begin_replay() { + return Err(EAGAIN); + } + packet.completion.reset_for_replay()?; + self.replay_prepared = true; + } + Ok(()) + } + + pub(crate) fn fail_replays(&mut self, error: Error) { + if self.replay_error.is_none() { + self.replay_error = Some(error); + } + } + + /// Caller already stopped both processors. No firmware owner can access retained mappings. + pub(crate) fn stopped(&mut self, defer: &mut impl FnMut(Deferred) -> Result) -> Result { + while let Some(packet) = self.replays.front() { + defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; + drop(self.replays.pop()); + } + while let Some(active) = self.active.front() { + defer(Deferred::Retired( + active.packet.completion.clone(), + Err(EIO), + ))?; + drop(self.active.pop()); + } + Ok(()) + } +} + +impl Queue { + pub(crate) fn matches(&self, owner: u64, qid: u8, context: &Arc) -> Result { + if self.owner != Some(owner) { + return Ok(false); + } + if self.released + || self.quarantined + || self.qid() != qid + || !Arc::ptr_eq(&self.binding.context, context) + || !context.is_current() + { + return Err(EIO); + } + Ok(true) + } + pub(crate) fn reusable(&self) -> bool { + self.installed + // An issued kill still names this graph even if its commands retire + // before the acknowledgement. It cannot acquire a successor owner. + && self.exit.is_none() + && self.owner.is_none() + && self.released + && self.retirement_ready + && !self.quarantined + && self.active.len == 0 + && self.replays.len == 0 + && self.previous.is_none() + } + pub(crate) fn bind_owner(&mut self, owner: u64, context: Arc) -> Result { + if !self.reusable() { + return Err(EBUSY); + } + if !context.is_current() { + return Err(EFAULT); + } + let old = self.binding.context.clone(); + if Arc::ptr_eq(old.status(), context.status()) && old.is_current() { + self.graph.set_owner(&context)?; + self.binding.context = context.clone(); + self.previous = Some(Previous { + binding: None, + context: old, + publication: None, + }); + } else { + let kick_alias = self.kick.map_client(context.vm())?; + let pool_alias = self.pool.map_client(context.vm())?; + let binding = self.graph.bind(context.clone(), kick_alias, pool_alias)?; + self.graph.set_owner(&context)?; + let old_binding = core::mem::replace(&mut self.binding, binding); + self.previous = Some(Previous { + binding: Some(old_binding), + context: old, + publication: None, + }); + } + context.mark_published(); + self.owner = Some(owner); + self.released = false; + self.retirement_ready = false; + self.retired_by_teardown = false; + Ok(()) + } + /// True allows the registry to cancel the never-published QID and drop this graph. An + /// installed graph remains retained even after the logical owner exits without work. + pub(crate) fn release_owner(&mut self) -> Result { + if !self.installed && self.active.len == 0 && !self.quarantined && self.previous.is_none() { + return Ok(true); + } + self.released = true; + if self.active.len == 0 && !self.quarantined { + if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { + if let Err(error) = self.graph.set_owner(&previous.context) { + self.quarantined = true; + self.retired_by_teardown = true; + return Err(error); + } + let mut previous = self.previous.take().ok_or(EIO)?; + if let Some(binding) = previous.binding.take() { + self.binding = binding; + } else { + self.binding.context = previous.context; + } + self.retirement_ready = true; + } + if self.retirement_ready { + self.owner = None; + } + } + Ok(false) + } + /// Exact host identity remains valid even when several commands share an ioctl ID. + pub(crate) fn owns_packet(&self, packet: &Arc) -> bool { + self.active + .iter() + .any(|active| Arc::ptr_eq(&active.packet, packet)) + } + + /// A watchdog settles spared, deferred waiters without changing their VM status. + /// Fully ticketed spared work can still return this queue after its drain witness. + pub(crate) fn timeout( + &mut self, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result>> { + let fresh = !self.quarantined; + let return_spared = self.active.len != 0 + && self + .active + .iter() + .all(|active| active.packet.completion.spared() && active.ticket.is_some()); + if self.spared_deferred { + for active in self.active.iter_mut() { + if !active.failure_deferred { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + ETIMEDOUT, + ))?; + active.failure_deferred = true; + } + } + self.spared_deferred = false; + } + let result = self.quarantine(ETIMEDOUT, defer); + if fresh { + self.spared_quarantine = return_spared; + } + result + } + + pub(crate) fn cancel( + &mut self, + packet: &Arc, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if !self.owns_packet(packet) { + return Ok(false); + } + if self.quarantined { + if self.spared_deferred { + for active in self.active.iter_mut() { + if !active.failure_deferred { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + ECANCELED, + ))?; + active.failure_deferred = true; + } + } + self.spared_deferred = false; + } + return Ok(true); + } + self.released = true; + self.retired_by_teardown = true; + if let Some(active) = self + .active + .iter_mut() + .find(|a| Arc::ptr_eq(&a.packet, packet)) + { + if !active.failure_deferred { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + ECANCELED, + ))?; + active.failure_deferred = true; + } + } + Ok(true) + } + /// Firmware is halted and publication is closed while its target is copied. + pub(crate) fn publish_recovery_target(&self) -> Result { + let producer = self + .graph + .graph + .word(POINTERS + PointerBlock::PRODUCER)? + .load(Ordering::Relaxed); + self.graph + .graph + .word(POINTERS + PointerBlock::RECOVERY_TARGET)? + .store(producer, Ordering::Relaxed); + Ok(()) + } + + /// The newest covered kick of a QID subsumes its earlier prefix; later independent work + /// must not become a dependency. Errors are observed before returning any firmware token. + pub(crate) fn dependency( + &self, + owner: u64, + status: &Arc, + through: Option, + recovery_state: u32, + ) -> Result> { + if through.is_none() || self.owner != Some(owner) { + return Ok(None); + } + let Some(active) = self + .active + .iter() + .filter(|a| crate::g17::job::Order::contains(through, a.packet.order.sequence)) + .last() + else { + return Ok(None); + }; + if self.quarantined && self.spared_quarantine { + return Err(ECANCELED); + } + if !Arc::ptr_eq(active.packet.completion.status(), status) + || self.quarantined + || recovery_state != 0 + { + return Err(EIO); + } + let ticket = active.ticket.ok_or(EIO)?; + let state = crate::g17::job::fence_status(&active.packet.completion.fence()); + if state < 0 { + return Err(crate::g17::dependency::inherited_error(Error::from_errno( + state, + ))); + } + if state > 0 { + return Ok(None); + } + active.packet.completion.note_firmware_consumer(); + let kick = crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?; + Ok(Some( + crate::g17::fw::kick::KickDependency::new(self.qid(), kick).ok_or(EIO)?, + )) + } +} + +impl Graph { + fn record( + &mut self, + slot: Option, + offset: usize, + window: usize, + value: T, + ) -> Result { + let (object, start) = if let Some(slot) = slot { + if slot >= RECORD_SLOTS { + return Err(EINVAL); + } + (&mut self.records, slot * RECORD_STRIDE + offset) + } else { + (&mut self.graph, 0x6800 + offset) + }; + if core::mem::size_of::() > window { + return Err(EINVAL); + } + let pointer = object.pointer(start, window)?; + // SAFETY: The checked record window is not named by any unconsumed item pointer. The + // device mutex excludes other host publishers, and no Rust reference to it is formed. + unsafe { pointer.write_bytes(0, window) }; + object.write(start, value)?; + fence(Ordering::SeqCst); + Ok(object.gpu_va() + start as u64) + } + fn append(&self, expected: u32, address: u64, before_publish: impl FnOnce()) -> Result { + let cursors = self.cursors()?; + if cursors.producer != expected + || cursors.count != ITEM_RING_ENTRIES + || expected >= ITEM_RING_ENTRIES + { + return Err(EBUSY); + } + let next = (expected + 1) % ITEM_RING_ENTRIES; + if next == cursors.consumer { + return Err(EBUSY); + } + let slot = self.graph.dword(ITEMS + expected as usize * 8)?; + let producer = self.graph.word(POINTERS + 0x40)?; + if slot.load(Ordering::Relaxed) != 0 { + return Err(EBUSY); + } + before_publish(); + slot.store(address, Ordering::Relaxed); + fence(Ordering::SeqCst); + producer.store(next, Ordering::Relaxed); + Ok(next) + } +} + +/// Borrowed device operations used during one publication. Implementations borrow the initdata, +/// coprocessor and accounting fields separately from the Registry that owns physical queues. +pub(crate) trait Host { + fn epoch(&self) -> Result<(u64, u32)>; + fn prepare_compute_shared(&mut self) -> Result; + fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; + fn qos_cancel(&mut self, publication: qos::Publication) -> Result; + fn publish_qid(&mut self, id: kick::Id) -> Result; + fn publish_outer(&mut self, slot: &crate::g17::fw::channels::WorkSlot) -> Result; + fn notify(&mut self, message: u64) -> Result; + fn note_submission(&mut self) -> Result; +} + +impl Queue { + pub(crate) fn spared_pending(&self) -> bool { + self.quarantined && self.spared_quarantine && self.awaiting_witness + } + + /// Retains the packet before any potentially visible store. An error before publication + /// rolls back its work count; every ambiguous error quarantines the graph and packet. + pub(crate) fn publish( + &mut self, + packet: Arc, + dependencies: &[crate::g17::fw::kick::KickDependency], + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + use crate::g17::fw::{ + channels::WorkSlot, + compute::{ComputeArgs, ComputeDescriptor, COMPUTE_KICK_EVENT_MASK, COMPUTE_QOS_CLASS}, + kick::{KickArgs, KickQos}, + }; + let (epoch, state) = host.epoch()?; + if state != 0 { + return Err(EBUSY); + } + let replay_front = self + .replays + .front() + .is_some_and(|front| Arc::ptr_eq(front, &packet)); + if !self.room() + && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) + { + return Err(EBUSY); + } + if !packet.context.is_current() || !Arc::ptr_eq(&packet.context, &self.binding.context) { + return Err(EFAULT); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let crate::g17::command::Validated::Compute { + usc_base, + cdm_va, + cdm_end_va, + sampler_heap_va, + sampler_count, + .. + } = &packet.command + else { + return Err(EINVAL); + }; + let work = packet.completion.work_state(); + let work_word = work.word(); + if work_word == u32::MAX { + return Err(EINVAL); + } + let work_node = work.node_va()?; + let timestamp_va = packet.completion.compute_timestamp_va()?; + let kick_timestamp = self.kick.timestamp(); + let kick_parent = self.kick.parent(); + let qid = self.qid(); + let ordinal = self.ordinal.checked_add(1).ok_or(EOVERFLOW)?; + let (descriptor_high, descriptor_low) = + self.graph.descriptor_vas(kick_timestamp.slot() as u8); + let first = !self.installed && self.ordinal == 0; + let entry_signal = first + || dependencies + .iter() + .any(|dependency| dependency.qid() != qid); + let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); + if let Some(slot) = record_slot { + self.graph.ensure_record_slot(slot)?; + } + let args = ComputeArgs { + usc_base: *usc_base, + cdm_va: *cdm_va, + cdm_end_va: *cdm_end_va, + sampler_heap_va: *sampler_heap_va, + sampler_count: *sampler_count, + descriptor_va: descriptor_low, + context_id: packet.context.id().try_into()?, + context_generation: packet.context.generation(), + work_state: work_word, + work_state_va: work_node, + qid, + kick_count: self.ordinal, + kick: kick_timestamp, + preempt_va: self.binding.preempt.iova(), + operand_state_va: self.binding.operand.iova(), + usage_va: self.binding.usage.iova(), + usage_fw_va: self.graph.usage.gpu_va(), + free_list_slot: u32::from(self.pool.id()), + free_list_control_va: self.pool.control_va(), + stamp_va: self.graph.support.gpu_va() + STAMP as u64, + aux_stamp_va: self.graph.support.gpu_va() + STAMP as u64 + 8, + status_va: [ + self.graph.support.gpu_va() + 0x310, + self.graph.support.gpu_va() + 0x318, + ], + timestamp_va, + }; + let identity = ConfigIdentity { + scheduler: packet.context.scheduler_va(), + qos: packet.context.qos_id(), + context: packet.context.id(), + }; + let config = QueueConfig::new(&QueueConfigArgs { + target: QueueConfigTarget::Compute, + kick_ring_va: self.kick.low_va(), + kick_ring_fw_va: self.kick.firmware_va(), + qid, + install: first, + completion_seed: None, + context_id: args.context_id, + free_list: FreeListBinding { + va: self.pool.control_va(), + generation: u64::from(packet.context.scheduler_generation()), + slot: u32::from(self.pool.id()), + }, + scheduler_va: identity.scheduler, + qos_slot: identity.qos, + qos_update: self + .published_config + .is_none_or(|p| p.scheduler != identity.scheduler || p.qos != identity.qos), + owner_pid: packet.context.owner_pid(), + context_update: self + .published_config + .is_none_or(|p| p.scheduler != identity.scheduler || p.context != identity.context), + })?; + let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp)?; + let predecessor = KickPredecessor::new(qid, kick_parent)?; + let kick_args = KickArgs { + qid, + timestamp: kick_timestamp, + parent: kick_parent, + barriers: dependencies, + descriptor_va: descriptor_high, + queue_va: self.graph.graph.gpu_va(), + qos: KickQos { + slot: identity.qos, + class: COMPUTE_QOS_CLASS, + }, + mcache: None, + event_mask: COMPUTE_KICK_EVENT_MASK, + register_arrays: ComputeDescriptor::register_bindings(descriptor_low)?, + }; + self.active + .push(Active { + packet: packet.clone(), + work: work.clone(), + ticket: None, + observed: None, + failure_deferred: false, + qos_completed: false, + }) + .map_err(|_| EBUSY)?; + self.retirement_ready = false; + let mut visible = false; + let mut accounted = false; + let result = (|| -> Result { + work.add_submitted_kicks(1)?; + accounted = true; + self.graph.descriptors.initialize::( + kick_timestamp.slot() * COMPUTE_DESCRIPTOR_SIZE, + |descriptor| descriptor.write(&args), + )?; + if first { + self.graph.context_high.write( + 0x200, + crate::g17::fw::compute::QueueContextItem::new( + descriptor_high, + self.graph.graph.gpu_va(), + qid, + ), + )?; + } else { + // The retained context item remains at index one; only its descriptor changes. + self.graph.context_high.write(0x210, descriptor_high)?; + let cursors = self.graph.cursors()?; + let _ = self.graph.recycle(cursors); + } + let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; + let cursors = self.graph.cursors()?; + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } + if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { + return Err(ERANGE); + } + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < if entry_signal { 4 } else { 3 } { + return Err(ENOSPC); + } + let prepared_kick = self.kick.prepare_entry(&kick_args)?; + let mut producer; + if first { + producer = self.graph.first_descriptor(descriptor_high)?; + host.prepare_compute_shared()?; + packet.mark_published(); + visible = true; + let qos = host.qos_publish( + qos::Owner { + qid, + qos: identity.qos, + data_master: 2, + }, + identity.scheduler, + )?; + let target = self.pool.reserve_submission().or_else(|error| { + host.qos_cancel(qos)?; + Err(error) + })?; + producer = self + .graph + .append(producer, config_va, || {}) + .or_else(|error| { + self.pool.cancel_unpublished(target)?; + host.qos_cancel(qos)?; + Err(error) + })?; + } else { + let target = self.pool.reserve_submission()?; + producer = self + .graph + .append(cursors.producer, descriptor_high, || { + packet.mark_published() + }) + .or_else(|error| { + self.pool.cancel_unpublished(target)?; + Err(error) + })?; + visible = true; + let qos = host.qos_publish( + qos::Owner { + qid, + qos: identity.qos, + data_master: 2, + }, + identity.scheduler, + )?; + producer = self + .graph + .append(producer, config_va, || {}) + .or_else(|error| { + host.qos_cancel(qos)?; + Err(error) + })?; + } + if first { + prepared_kick.prepare_compute_install()?; + } + prepared_kick.commit(); + self.submitted = self.submitted.checked_add(1).ok_or(EOVERFLOW)?; + let announce_va = self.graph.record(record_slot, ANNOUNCE, 0x40, announce)?; + producer = self.graph.append(producer, announce_va, || {})?; + fence(Ordering::SeqCst); + if entry_signal { + let predecessor_va = + self.graph + .record(record_slot, PREDECESSOR, 0x400, predecessor)?; + producer = self.graph.append(producer, predecessor_va, || {})?; + } + if let Some(slot) = record_slot { + self.graph.record_ends[slot] = Some(producer); + } + host.publish_qid(self.kick.id())?; + host.publish_outer(&WorkSlot::new( + DataMaster::Compute, + self.graph.graph.gpu_va(), + qid, + producer.try_into()?, + !self.outer_started, + )?)?; + let notify_activation = !self.outer_started; + self.outer_started = true; + self.installed = true; + self.published_config = Some(identity); + self.ordinal = ordinal; + let (accepted, state) = host.epoch()?; + if state != 0 || accepted != epoch { + return Err(EIO); + } + let ticket = Ticket { + owner: self.owner.ok_or(EIO)?, + submission: packet.order.sequence, + context_id: packet.context.id(), + qid, + descriptor: descriptor_high, + work_node, + kick: kick_timestamp.get(), + ordinal, + item_producer: producer, + item_count: ITEM_RING_ENTRIES, + epoch, + }; + self.active.back_mut().ok_or(EIO)?.ticket = Some(ticket); + if let Some(previous) = self.previous.as_mut() { + if previous.publication.is_none() { + previous.publication = Some((packet.order.sequence, epoch)); + } + } + if notify_activation { + fence(Ordering::SeqCst); + host.notify((0x83 << 48) | 0x0a)?; + } + fence(Ordering::SeqCst); + host.notify((0x83 << 48) | 0x10)?; + host.note_submission() + })(); + if let Err(error) = result { + if !visible { + if accounted { + if let Err(error) = work.cancel_unpublished_kicks(1) { + self.terminal = + self.quarantine(error, defer)?.map(|status| (status, error)); + return Err(error); + } + } + drop(self.active.pop_back()); + return Err(error); + } + self.terminal = self.quarantine(error, defer)?.map(|status| (status, error)); + return Err(error); + } + Ok(()) + } +} + +impl Graph { + /// The queue has never been installed. Retry staging replaces its initial three item slots; + /// no firmware reader can name this graph until its first complete outer publication. + fn first_descriptor(&self, address: u64) -> Result { + let first = self.graph.dword(ITEMS)?; + let second = self.graph.dword(ITEMS + 8)?; + let third = self.graph.dword(ITEMS + 16)?; + let producer = self.graph.word(POINTERS + 0x40)?; + first.store(address, Ordering::Relaxed); + second.store(0, Ordering::Relaxed); + third.store(0, Ordering::Relaxed); + producer.store(1, Ordering::Relaxed); + Ok(1) + } +} +impl Queue { + pub(crate) fn take_terminal(&mut self) -> Option<(Arc, Error)> { + self.terminal.take() + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs new file mode 100644 index 0000000000000..1ddadae2dba69 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Closed-context kill requests and the command-pin release witness. + +use super::{Context, Deferred, Queue}; +use core::sync::atomic::{fence, AtomicU32, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +pub(crate) const POLL_NS: u64 = 2_500_000_000; +const QUARANTINED_DELAY_NS: u64 = 250_000_000; +const CANCELLED_DELAY_NS: u64 = 2_000_000_000; +const ACK_SLOTS: usize = 16; + +// Firmware-visible cookie namespace persists for the loaded module's lifetime. +static NEXT_COOKIE: AtomicU32 = AtomicU32::new(1); + +pub(crate) fn next_cookie() -> u64 { + loop { + let cookie = NEXT_COOKIE.fetch_add(1, Ordering::Relaxed); + if cookie != 0 { + return cookie.into(); + } + } +} + +pub(crate) struct Acks([u64; ACK_SLOTS]); +impl Acks { + pub(crate) fn new() -> Self { + Self([0; ACK_SLOTS]) + } + pub(crate) fn note(&mut self, cookie: u64) { + if cookie == 0 || self.contains(cookie) { + return; + } + if let Some(slot) = self.0.iter_mut().find(|slot| **slot == 0) { + *slot = cookie; + } + } + pub(crate) fn contains(&self, cookie: u64) -> bool { + cookie != 0 && self.0.contains(&cookie) + } + pub(crate) fn forget(&mut self, cookie: u64) { + for slot in &mut self.0 { + if *slot == cookie { + *slot = 0; + } + } + } + pub(crate) fn snapshot(&self) -> [u64; ACK_SLOTS] { + self.0 + } +} + +pub(crate) struct State { + pub(crate) slot: u8, + pub(crate) qids: u128, + pub(crate) killed: u32, +} +impl State { + pub(crate) fn read(context: &Context) -> Result { + let words = context.scheduler_owner().read_words::<14>(0)?; + let mut bytes = [0u8; 0x38]; + for (destination, word) in bytes.chunks_exact_mut(4).zip(words) { + destination.copy_from_slice(&word.to_le_bytes()); + } + fence(Ordering::Acquire); + Ok(Self::decode(&bytes)) + } + fn decode(bytes: &[u8; 0x38]) -> Self { + let mut low = [0; 8]; + let mut high = [0; 8]; + let mut killed = [0; 4]; + low.copy_from_slice(&bytes[0x12..0x1a]); + high.copy_from_slice(&bytes[0x1a..0x22]); + killed.copy_from_slice(&bytes[0x34..0x38]); + Self { + slot: bytes[1], + qids: u128::from(u64::from_le_bytes(low)) + | (u128::from(u64::from_le_bytes(high)) << 64), + killed: u32::from_le_bytes(killed), + } + } + pub(crate) fn scope(&self, target: u8, own: u128, foreign: u128) -> Option { + if self.slot == 0xff || target >= 128 || self.qids & (1u128 << target) == 0 { + return None; + } + let allowed = (own | (1u128 << target)) & !foreign & !0b11; + (self.qids & !allowed == 0).then_some(self.qids) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Phase { + Published, + Acknowledged, + Unproven, + Released, +} + +pub(super) struct Transaction { + cookie: u64, + published: u64, + phase: Phase, +} + +impl Queue { + pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { + Arc::ptr_eq(self.context(), context) + && self.released + && self.active.len != 0 + && self.exit.is_none() + && (self.quarantined || self.retired_by_teardown) + && age + >= if self.quarantined { + QUARANTINED_DELAY_NS + } else { + CANCELLED_DELAY_NS + } + } + pub(crate) fn exit_owner(&self, context: &Arc) -> Option { + if !self + .active + .iter() + .all(|active| Arc::ptr_eq(&active.packet.context, context)) + { + return None; + } + self.owner.or_else(|| { + self.active + .back() + .and_then(|active| active.ticket.map(|ticket| ticket.owner)) + }) + } + pub(crate) fn record_exit(&mut self, cookie: u64, now: u64) { + self.exit = Some(Transaction { + cookie, + published: now, + phase: Phase::Published, + }); + } + pub(crate) fn exit_polling(&self, now: u64) -> bool { + self.exit.as_ref().is_some_and(|exit| { + exit.phase == Phase::Published && now.saturating_sub(exit.published) < POLL_NS + }) + } + pub(crate) fn exit_cookie(&self) -> Option { + self.exit + .as_ref() + .filter(|exit| matches!(exit.phase, Phase::Published | Phase::Acknowledged)) + .map(|exit| exit.cookie) + } + pub(crate) fn settle_exit( + &mut self, + acks: &Acks, + recovery_closed: bool, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + let Some(cookie) = self.exit_cookie() else { + return Ok(()); + }; + if !acks.contains(cookie) { + return Ok(()); + } + self.exit.as_mut().ok_or(EIO)?.phase = Phase::Acknowledged; + if self.active.len != 0 { + if !recovery_closed { + return Ok(()); + } + let state = match State::read(self.context()) { + Ok(state) => state, + Err(_) => return Ok(()), + }; + if state.killed != 1 { + self.exit.as_mut().ok_or(EIO)?.phase = Phase::Unproven; + return Ok(()); + } + if !self.active.iter().all(|active| { + active + .packet + .completion + .compute_words() + .is_ok_and(|words| words == [0, 0] || words[1] != 0) + }) { + return Ok(()); + } + } + while let Some(active) = self.active.front() { + defer(Deferred::Retired( + active.packet.completion.clone(), + Err(if self.spared_deferred { + ENODATA + } else { + ECANCELED + }), + ))?; + drop(self.active.pop()); + } + self.exit.as_mut().ok_or(EIO)?.phase = Phase::Released; + self.quarantined = true; + self.awaiting_witness = false; + self.spared_deferred = false; + self.spared_quarantine = false; + self.retire_pending = false; + self.retirement_ready = false; + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs new file mode 100644 index 0000000000000..7428f99ab1d7f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Exact identities and visibility witnesses for a physical compute queue. +//! A shared work-state counter can include other engines and cannot prove this queue retired. + +use kernel::prelude::*; + +pub(crate) const IN_FLIGHT: usize = 16; +pub(crate) const RECORD_SLOTS: usize = 32; +const QIDS: usize = 128; +const TIME_MASK: u64 = 0x003f_ffff_ffff_ffff; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Ticket { + pub(crate) owner: u64, + pub(crate) submission: u64, + pub(crate) context_id: u32, + pub(crate) qid: u8, + pub(crate) descriptor: u64, + pub(crate) work_node: u64, + pub(crate) kick: u64, + pub(crate) ordinal: u32, + pub(crate) item_producer: u32, + pub(crate) item_count: u32, + pub(crate) epoch: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Observation { + pub(crate) timestamps: [u64; 2], +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Cursors { + pub(crate) producer: u32, + pub(crate) consumer: u32, + pub(crate) count: u32, +} + +impl Cursors { + /// Whether firmware consumed every record of the given publication, allowing later work. + pub(crate) fn consumed(&self, end: u32, count: u32) -> Result { + if self.count == 0 || self.count != count { + return Err(EIO); + } + let producer = self.producer % count; + let consumer = self.consumer % count; + let unconsumed = (producer + count - consumer) % count; + let following = (producer + count - end % count) % count; + Ok(unconsumed <= following) + } +} + +/// Outstanding kicks span fewer than half the 24-bit stamp sequence. Zero is a valid wrap. +pub(crate) const fn stamp_covers(word: u32, kick: u64) -> bool { + let delta = word.wrapping_sub((kick as u32).wrapping_shl(8)); + delta & 0xff == 0 && delta < 0x8000_0000 +} + +#[derive(Clone, Copy)] +pub(crate) enum Recordless { + Allowed, + NotYet, + CrossedRecovery, +} + +/// A zero-duration record fails before visibility polling. It proves execution did not occur; +/// ownership still requires a separate retirement witness before mappings can be released. +pub(crate) fn poll( + ticket: &Ticket, + observed: Option, + cursors: Cursors, + stamp: u32, + recordless: Recordless, + floor: u64, + next_end: Option, +) -> Result> { + if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { + return Err(ENODATA); + } + if !stamp_covers(stamp, ticket.kick) + || !cursors.consumed(ticket.item_producer, ticket.item_count)? + { + return Ok(None); + } + Ok(Some(match observed { + Some(record) => record.timestamps, + None => match recordless { + Recordless::Allowed => [floor, next_end.unwrap_or(floor).max(floor)], + Recordless::NotYet => return Ok(None), + Recordless::CrossedRecovery => return Err(ENODATA), + }, + })) +} + +/// The newest exact publication covers all preceding kicks, but only a fully consumed ring +/// permits releasing quarantined command pins. The installed graph remains retained. +pub(crate) fn quarantined_retired( + ticket: &Ticket, + cursors: Cursors, + stamp: u32, + pending: u32, + submitted: u32, +) -> Result { + if ticket.item_count == 0 || ticket.descriptor == 0 || pending == 0 { + return Err(EINVAL); + } + if cursors.count == 0 || cursors.count != ticket.item_count { + return Err(EIO); + } + let end = ticket.item_producer % cursors.count; + Ok(stamp_covers(stamp, ticket.kick) + && cursors.producer % cursors.count == end + && cursors.consumer % cursors.count == end + && submitted == pending) +} + +#[derive(Clone, Copy)] +struct Request { + descriptor: u64, + kick: u64, + observed: Option, +} + +/// Reusable heap scratch for one shared-ring scan. QID requests are contiguous and bounded; +/// staging and folding never allocate under the device mutex. +pub(crate) struct Batch { + requests: KVVec, + slots: [(usize, usize); QIDS], + open: Option, +} + +impl Batch { + pub(crate) fn new() -> Result { + Ok(Self { + requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, + slots: [(0, 0); QIDS], + open: None, + }) + } + pub(crate) fn clear(&mut self) { + self.requests.clear(); + self.slots.fill((0, 0)); + self.open = None; + } + pub(crate) fn request(&mut self, ticket: &Ticket) -> Result { + if ticket.descriptor == 0 || ticket.kick == 0 { + return Err(EINVAL); + } + let slot = self.slots.get_mut(ticket.qid as usize).ok_or(EINVAL)?; + if slot.1 != 0 && self.open != Some(ticket.qid) { + return Err(EBUSY); + } + if slot.1 >= IN_FLIGHT { + return Err(ENOSPC); + } + let start = if slot.1 == 0 { + self.requests.len() + } else { + slot.0 + }; + self.requests + .push_within_capacity(Request { + descriptor: ticket.descriptor, + kick: ticket.kick, + observed: None, + }) + .map_err(|_| ENOSPC)?; + *slot = (start, slot.1 + 1); + self.open = Some(ticket.qid); + Ok(()) + } + /// The transport reads eight aligned words from a 0x40-byte firmware record. Records can be + /// overwritten during a scan: incomplete or older identities are skipped, never consumed. + pub(crate) fn fold(&mut self, words: [u64; 8]) -> Result { + let qid = ((words[0] >> 9) & 0x7f) as usize; + let (start, count) = self.slots[qid]; + if count == 0 || words[6] & !1 == 0 || words[7] == 0 { + return Ok(()); + } + let descriptor = words[6] & !1; + let kick = + (((words[0] >> 8) & 0xffff_ffff_00) | ((words[0] >> 48) & 0xff)) & 0xff_ffff_ffff; + let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; + let Some(request) = requests + .iter_mut() + .find(|r| r.descriptor == descriptor && r.kick == kick) + else { + return Ok(()); + }; + let timestamps = [words[2] & TIME_MASK, words[5] & TIME_MASK]; + if timestamps[0] == 0 + || timestamps[1] == 0 + || request + .observed + .is_some_and(|old| old.timestamps[1] >= timestamps[1]) + { + return Ok(()); + } + request.observed = Some(Observation { timestamps }); + Ok(()) + } + pub(crate) fn take(&mut self, ticket: &Ticket) -> Result> { + let &(start, count) = self.slots.get(ticket.qid as usize).ok_or(EINVAL)?; + let request = self + .requests + .get_mut(start..start + count) + .ok_or(EIO)? + .iter_mut() + .find(|r| r.descriptor == ticket.descriptor && r.kick == ticket.kick) + .ok_or(EIO)?; + Ok(request.observed.take()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs new file mode 100644 index 0000000000000..afc58009f4029 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Userspace submission queues: validate whole ioctls, retain mappings, and enqueue +//! independent engines behind user fences and explicit logical prefixes. + +pub(crate) mod compute; +pub(crate) mod render; + +use super::{ + command::{Parser, UscWindow, Validated}, + completion::Completion, + context::Context, + fence::{Outputs, Submission}, + job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, + validation::Proof, +}; +use crate::{ + driver::{AsahiDevRef, AsahiDevice}, + file, mmu, +}; +use core::ops::Range; +use kernel::{ + c_str, + dma_fence::FenceContexts, + prelude::*, + sync::{Arc, LockClassKey}, + xarray, +}; + +const EARLY_DEPTH: usize = 2; +static FENCE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); + +pub(crate) mod admission; + +/// Host identities for retained physical channels, separate from firmware object generations. +/// The device mutex protects allocation; cancelled preparations never reuse IDs. +pub(crate) struct Identities(u64); + +impl Identities { + pub(crate) const fn new() -> Self { + Self(3) + } + + /// The first two identities remain reserved with QIDs 0 and 1. + pub(crate) fn reserve(&mut self, count: u64) -> Result> { + let first = self.0; + let end = first.checked_add(count).ok_or(EOVERFLOW)?; + self.0 = end; + Ok(first..end) + } +} + +/// Validated frontend state prepared before allocating a logical execution context. +pub(crate) struct Frontend { + fences: FenceContexts, + range: Range, + window: UscWindow, +} + +impl Frontend { + pub(crate) fn new(priority: u32, usc_base: u64, range: Range) -> Result { + if priority > 3 { + return Err(EINVAL); + } + let window = UscWindow::new(usc_base, range.clone())?; + let fences = FenceContexts::new(1, c_str!("asahi_g17_queue"), FENCE_KEY)?; + Ok(Self { + fences, + range, + window, + }) + } +} + +pub(crate) struct Queue { + dev: AsahiDevRef, + context: Arc, + backend: Arc, + render: Option>, + compute: Option>, + fences: FenceContexts, + range: Range, + window: UscWindow, + proof: Proof, + sequence: [u64; 2], + frontiers: [KVec<(u64, Fences)>; 2], + // Closing the userspace queue releases admission even if firmware still + // retains one of its execution contexts or installed physical graphs. + _slot: admission::QueueSlot, +} + +impl Queue { + pub(crate) fn new( + dev: &AsahiDevice, + context: Arc, + backend: Arc, + frontend: Frontend, + slot: admission::QueueSlot, + ) -> Result { + let Frontend { + fences, + range, + window, + } = frontend; + let render = Some(Scheduler::new(dev, backend.clone(), Engine::Render)?); + Ok(Self { + dev: dev.into(), + context, + backend, + render, + compute: None, + fences, + range, + window, + proof: Proof::new(), + sequence: [0; 2], + frontiers: core::array::from_fn(|_| KVec::new()), + _slot: slot, + }) + } + + fn ensure_compute(&mut self) -> Result { + // Admission is checked even when this queue already owns a scheduler. + self.backend.ensure_compute()?; + if self.compute.is_none() { + self.compute = Some(Scheduler::new( + &self.dev, + self.backend.clone(), + Engine::Compute, + )?); + } + Ok(()) + } + + fn enqueue( + &mut self, + id: u64, + command: Validated, + order: Order, + guard: mmu::VmJobGuard, + aggregate: &Arc, + inputs: &[file::SyncItem], + objects: Pin<&xarray::XArray>>, + ) -> Result { + let engine = match &command { + Validated::Render { .. } => Engine::Render, + Validated::Compute { .. } => Engine::Compute, + }; + let index = engine.index(); + self.frontiers[index].retain(|(_, fences)| job::fence_status(&fences.completed) == 0); + self.frontiers[index].reserve(1, GFP_KERNEL)?; + if engine == Engine::Compute { + self.ensure_compute()?; + } + let completion = match &command { + Validated::Render { timestamps, .. } => Completion::new( + &self.dev, + &self.fences, + &self.context, + aggregate.member(), + guard, + objects, + timestamps, + false, + )?, + Validated::Compute { timestamps, .. } => Completion::new( + &self.dev, + &self.fences, + &self.context, + aggregate.member(), + guard, + objects, + core::slice::from_ref(timestamps), + true, + )?, + }; + if engine == Engine::Compute { + self.frontiers[Engine::Render.index()] + .retain(|(_, fences)| job::fence_status(&fences.completed) == 0); + } + let other = 1 - index; + let prefix = order.wait_through[other]; + let count = self.frontiers[other] + .iter() + .filter(|(sequence, _)| Order::contains(prefix, *sequence)) + .count(); + let mut dependencies = KVec::with_capacity(inputs.len() + count, GFP_KERNEL)?; + let mut checked_inputs = KVec::with_capacity(inputs.len(), GFP_KERNEL)?; + let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; + for (sequence, fences) in &self.frontiers[other] { + if Order::contains(prefix, *sequence) { + dependencies.push(fences.ready.clone(), GFP_KERNEL)?; + firmware.push(fences.completed.clone(), GFP_KERNEL)?; + } + } + // Each command owns every input fence; independent engines may publish in + // either order and must each consume an input's error status before running. + for sync in inputs { + if let Some(fence) = &sync.fence { + dependencies.push(fence.clone(), GFP_KERNEL)?; + checked_inputs.push(fence.clone(), GFP_KERNEL)?; + } + } + let early = engine == Engine::Render + && dependencies.is_empty() + && self.frontiers[index].len() < EARLY_DEPTH; + let packet = Packet::new( + id, + self.context.clone(), + command, + order, + completion, + checked_inputs, + firmware, + )?; + let scheduler = match engine { + Engine::Render => &mut self.render, + Engine::Compute => &mut self.compute, + }; + let fences = scheduler + .as_mut() + .ok_or(ENODEV)? + .enqueue(packet, dependencies, early)?; + self.sequence[index] = order.sequence; + self.frontiers[index] + .push_within_capacity((order.sequence, fences)) + .map_err(|_| ENOMEM)?; + Ok(()) + } + /// Finish parsing, validation and VM pinning before entering publication. + /// Keeping this frame separate bounds the scheduler/ioctl stack depth. + #[inline(never)] + fn prepare_submission( + &mut self, + bytes: &[u8], + ) -> Result<(KVec<(Validated, Order)>, KVec)> { + let parse_guard = self.context.vm().retain_first_job()?; + let mut parser = Parser::new(bytes); + let mut preceding = [0; 2]; + let mut pending = KVec::with_capacity(1, GFP_KERNEL)?; + while let Some(command) = parser.next()? { + let payload = self.proof.validate( + self.context.vm(), + self.range.clone(), + &self.window, + &command.payload, + )?; + let engine = match &payload { + Validated::Render { .. } => Engine::Render, + Validated::Compute { .. } => Engine::Compute, + }; + let order = Order::new(self.sequence, preceding, command.barriers, engine)?; + preceding[engine.index()] += 1; + pending.push((payload, order), GFP_KERNEL)?; + } + parser.finish()?; + // Every accepted command pins the VM before any command is enqueued. + let mut guards = KVec::with_capacity(pending.len(), GFP_KERNEL)?; + guards + .push_within_capacity(parse_guard) + .map_err(|_| ENOMEM)?; + while guards.len() < pending.len() { + guards + .push_within_capacity(self.context.vm().retain_job()?) + .map_err(|_| ENOMEM)?; + } + Ok((pending, guards)) + } +} + +impl crate::queue::Queue for Queue { + fn submit( + &mut self, + id: u64, + syncs: KVec, + input_count: usize, + bytes: &[u8], + objects: Pin<&xarray::XArray>>, + ) -> Result { + let mut outputs = Outputs::new(&self.fences, syncs, input_count)?; + let result = (|| { + if self.context.status().get() != 0 { + return Err(EIO); + } + let (pending, mut guards) = self.prepare_submission(bytes)?; + let aggregate = Submission::new(&self.fences, self.context.clone())?; + outputs.publish(aggregate.clone()); + for (command, order) in pending { + let guard = guards.remove(0).map_err(|_| EIO)?; + self.enqueue( + id, + command, + order, + guard, + &aggregate, + outputs.inputs(), + objects, + )?; + } + Ok(()) + })(); + if let Err(error) = result { + self.context.status().record_if_device_loss(error); + if [EIO, ETIMEDOUT, ENODEV].contains(&error) || self.context.status().get() != 0 { + self.backend.fail_vm(error); + } + } + outputs.finish(result) + } +} + +impl Drop for Queue { + fn drop(&mut self) { + drop(self.render.take()); + drop(self.compute.take()); + self.backend.release_owner(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs new file mode 100644 index 0000000000000..3a7852be1dab0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -0,0 +1,1459 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Physical paired tiling and fragment queues and their retained command storage. + +mod observation; +pub(crate) use observation::Observed; +mod dependencies; +mod memory; +pub(crate) mod parked; +mod publication; +pub(crate) mod retirement; +mod warm; + +use crate::g17::{ + buffer::{self, target}, + command::Validated, + completion::Deferred, + context::Context, + freelist::RenderPool, + fw::{ + channels::WorkSlot, + kick::{KickDependency, KickTimestamp, McacheTable, RenderRegistration}, + queue::DataMaster, + render as abi, + }, + job::Packet, + kick, + object::{Allocator, Pool}, + qos, recovery, + status::VmStatus, +}; +use crate::{hw::t8140::queues, mmu}; +pub(crate) use dependencies::Dependencies; +use kernel::{prelude::*, sync::Arc}; +use retirement::{Ticket, Tracker}; + +#[derive(Copy, Clone)] +pub(crate) struct RenderAddresses { + /// Tiling then fragment; each pair contains the primary and auxiliary stamp. + pub(crate) stamps: [[u64; 2]; 2], +} + +#[derive(Copy, Clone)] +pub(crate) struct PublicationLease { + pub(crate) free_list_generation: u64, + pub(crate) buffer_token: u64, +} + +/// Host services borrow the device-locked firmware state. Resource construction +/// and command geometry preparation do not borrow this interface. +pub(crate) trait Host { + fn epoch(&self) -> Result<(u64, u32)>; + fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; + fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; + fn qos_cancel(&mut self, publication: qos::Publication) -> Result; + fn install_pair(&mut self, registration: &RenderRegistration) -> Result; + fn publish_qid(&mut self, id: kick::Id) -> Result; + fn publish_outer_pair( + &mut self, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + commit: impl FnOnce() -> Result, + late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]>; + fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result; + fn publish_retained_pair( + &mut self, + fragment: &WorkSlot, + tiling: &WorkSlot, + next: [u32; 2], + ) -> Result; + fn notify_render(&mut self) -> Result; + fn note_submission(&mut self) -> Result; + fn complete_render( + &mut self, + owners: Option<[qos::Owner; 2]>, + pool_id: u16, + generation: u64, + buffer_id: u8, + ) -> Result; +} + +struct Prepared { + packet: Arc, + ordinal: u64, + mcache: Option, +} +struct Active { + qos_completed: bool, + flist_completed: bool, + outer_published: bool, + generation: u64, + packet: Arc, + ticket: Ticket, + kicks: [KickTimestamp; 2], + context_update: bool, + failed: bool, + drained_result: Option>, +} + +/// Installed kick backing remains owned even when this pair has no logical +/// owner. Commands retain their exact descriptor and allocation generations. +pub(crate) struct Pair { + kick_aliases: Option<[Option; 2]>, + memory: KBox, + kicks: [kick::Queue; 2], + free_list: Arc, + pending_aliases: Option>, + prepared: Option, + active: Tracker, + slot: u8, + owner: Option, + ordinal: u64, + initial_packet: u64, + clusters: u32, + descriptor_flags: [u32; 2], + installed: [bool; 2], + configs: [Option; 2], + cursors: [u32; 2], + recovery_generation: u64, + registered: bool, + outer_started: bool, + quarantined: bool, + terminal: Option<(Arc, Error)>, + previous: Option, + displaced: Option, + context_update: bool, +} +impl Pair { + #[inline(never)] + pub(crate) fn new( + alloc: &Allocator<'_>, + pool: &Arc, + ids: [kick::Id; 2], + slot: u8, + owner: u64, + context: Arc, + free_list: Arc, + metrics: buffer::MetricsLease, + buffer_id: (u8, u64), + pm_generation: u64, + first: &Arc, + clusters: u32, + descriptor_flags: [u32; 2], + ) -> Result> { + if slot == 0 + || usize::from(slot) >= queues::RENDER_SLOTS + || owner == 0 + || ids[0].engine() != DataMaster::Tiling + || ids[1].engine() != DataMaster::Fragment + || !Arc::ptr_eq(&context, &first.context) + || !context.is_current() + { + return Err(EINVAL); + } + if let Err(error) = alloc.uat.vm_context_mask(context.vm()) { + if error == ENOENT { + context.status().record(error); + } + return Err(error); + } + let Validated::Render { pass, .. } = &first.command else { + return Err(EINVAL); + }; + let word = first.completion.work_state().word(); + if word == u32::MAX { + return Err(EINVAL); + } + let layout = target::Layout::new(pass, clusters)?; + let storage = KBox::::new_uninit(GFP_KERNEL)?; + let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm())?; + let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm())?; + let (memory, aliases) = memory::Memory::new( + alloc, + pool, + context, + [ids[0].qid(), ids[1].qid()], + &layout, + metrics, + buffer_id, + pm_generation, + 0, + word, + )?; + storage.write_init(kernel::try_init!(Self { + kick_aliases: Some([Some(ta_alias), Some(fragment_alias)]), + memory, + kicks: [tiling, fragment], + free_list, + pending_aliases: Some(aliases), + prepared: None, + active: Tracker::new(), + slot, + owner: Some(owner), + ordinal: 0, + initial_packet: first.id, + clusters, + descriptor_flags, + installed: [false; 2], + configs: [None; 2], + cursors: [0; 2], + recovery_generation: 0, + registered: false, + outer_started: false, + quarantined: false, + terminal: None, + previous: None, + displaced: None, + context_update: true, + })) + } + pub(crate) fn qids(&self) -> [u8; 2] { + self.kicks.each_ref().map(|kick| kick.id().qid()) + } + pub(crate) fn slot(&self) -> u8 { + self.slot + } + pub(crate) fn owner(&self) -> Option { + self.owner + } + pub(crate) fn context(&self) -> &Arc { + &self.memory.context + } + pub(crate) fn pool(&self) -> &Arc { + &self.free_list + } + pub(crate) fn pool_id(&self) -> u64 { + u64::from(self.memory.lifecycle[0].predecessor) + } + pub(crate) fn buffer_id(&self) -> u8 { + self.memory.manager.buffer_id() + } + pub(crate) fn manager(&mut self) -> &mut buffer::Manager { + &mut self.memory.manager + } + pub(crate) fn in_flight(&self) -> bool { + self.active.len() != 0 + } + pub(crate) fn published_in_flight(&self) -> bool { + !self.quarantined && self.active.iter().any(|active| active.outer_published) + } + pub(crate) fn room(&self) -> bool { + !self.quarantined && self.owner.is_some() && self.prepared.is_none() && self.active.room() + } + pub(crate) fn take_terminal(&mut self) -> Option<(Arc, Error)> { + self.terminal.take() + } + + /// The registry owns this pair exclusively during potentially allocating + /// preparation. No queue producer or shared submission counter changes. + pub(crate) fn prepare( + &mut self, + alloc: &Allocator<'_>, + packet: &Arc, + addresses: RenderAddresses, + ) -> Result { + if !self.room() + || !Arc::ptr_eq(&self.memory.context, &packet.context) + || !packet.context.is_current() + { + return Err(EBUSY); + } + let Validated::Render { + pass, attachments, .. + } = &packet.command + else { + return Err(EINVAL); + }; + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let ordinal = self.ordinal; + ordinal.checked_add(1).ok_or(EOVERFLOW)?; + let work = packet.completion.work_state(); + let node = work.node_va()?; + let word = work.word(); + let layout = target::Layout::new(pass, self.clusters)?; + let owners = self.memory.owners(); + self.memory + .target + .ensure_capacity(alloc.dev, packet.context.vm(), &layout, &owners)?; + let target = self.memory.target.prepare(ordinal, &layout)?; + let scene = self.memory.manager.prepare_scene(ordinal)?; + self.memory.generation(ordinal)?.prepare(ordinal, pass)?; + let [deflake, ta_status, fragment_status, fragment_fw, auxiliary] = + self.memory.generation(ordinal)?.addresses(); + let lifecycle = if ordinal != 0 || packet.id != self.initial_packet { + let (fragment, tiling) = + crate::g17::ids::OBJECTS.render_pair(self.memory.lifecycle[0].predecessor)?; + [tiling, fragment] + } else { + self.memory.lifecycle + }; + let (scratch, metric) = self.memory.manager.scene_registers(scene)?; + let timestamp_offset = ordinal as usize % memory::RECORDS * 16; + let timestamp = self.memory.timestamps.gpu_va() + timestamp_offset as u64; + let resources = abi::RenderResources { + context_id: packet.context.id().try_into()?, + context_generation: packet.context.generation(), + pb_slot: self.buffer_id().into(), + free_list_slot: self.free_list.id().into(), + free_list_control_va: self.free_list.control_va(), + kick_record_va: node, + pb_va: self.memory.manager.state_va(), + scene_va: self.memory.manager.scene_va(scene)?, + scene_end_va: self.memory.manager.trailer_va(), + pm_scratch: scratch, + pm_metric: metric, + tilemap_va: target.tilemap, + layermeta_va: target.layer, + heapmeta_va: target.heap, + tpc_va: self.memory.target.tpc_va(), + tpc_size: layout.tpc_size as u64, + deflake_va: deflake, + ta_status_va: ta_status, + fragment_status_va: fragment_status, + fragment_status_fw_va: fragment_fw, + aux_fb_va: auxiliary, + timestamp_va: timestamp, + ta_ids: lifecycle[0], + fragment_ids: lifecycle[1], + unk_2104: self.descriptor_flags[0], + unk_210c: self.descriptor_flags[1], + }; + let qids = self.qids(); + let args = abi::RenderArgs { + pass, + resources: &resources, + ordinal, + ta: abi::RenderStage { + qid: qids[0], + descriptor_va: self.memory.graph.descriptor_client(0, ordinal), + state_word: word, + stamp_va: addresses.stamps[0][0], + aux_stamp_va: addresses.stamps[0][1], + }, + fragment: abi::RenderStage { + qid: qids[1], + descriptor_va: self.memory.graph.descriptor_client(1, ordinal), + state_word: word, + stamp_va: addresses.stamps[1][0], + aux_stamp_va: addresses.stamps[1][1], + }, + }; + self.memory.graph.claim(ordinal)?; + let result = self.memory.write_descriptors( + &args, + attachments[1].as_slice(), + packet.context.id().try_into()?, + ); + let mcache = match result { + Ok(mcache) => mcache, + Err(error) => { + self.memory.graph.release(ordinal)?; + self.memory.generation(ordinal)?.ordinal = None; + self.memory.target.cancel(ordinal); + return Err(error); + } + }; + self.memory.timestamps.write(timestamp_offset, [0u64; 2])?; + self.memory.manager.claim_scene(ordinal, scene)?; + self.prepared = Some(Prepared { + packet: packet.clone(), + ordinal, + mcache, + }); + Ok(()) + } +} + +impl Pair { + /// All descriptor bytes and retained allocations are ready before entry. + /// The first inner producer establishes firmware ownership, including on + /// later errors. Such failures retain the active packet for recovery. + pub(crate) fn publish( + &mut self, + packet: Arc, + dependencies: &[KickDependency], + lease: PublicationLease, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + use crate::g17::fw::{ + channels::WorkSlot, + kick::{KickArgs, KickQos}, + queue::{ + self, FreeListBinding, QueueConfig, QueueConfigArgs, QueueConfigTarget, + TilingBinding, + }, + }; + use core::sync::atomic::{fence, Ordering}; + if self.pending_aliases.is_some() || self.kick_aliases.is_some() { + return Err(EBUSY); + } + let (epoch, recovery_state) = host.epoch()?; + if recovery_state != 0 + || self.quarantined + || !self.active.room() + || packet.completion.render_slot() != Some(self.slot) + { + return Err(EBUSY); + } + let prepared = self.prepared.as_ref().ok_or(EINVAL)?; + if !Arc::ptr_eq(&prepared.packet, &packet) + || prepared.ordinal != self.ordinal + || !self.memory.owns(self.ordinal) + || !packet.context.is_current() + { + return Err(EIO); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let ordinal = self.ordinal; + let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; + let mcache = prepared.mcache; + let qids = self.qids(); + let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); + let parents = self.kicks.each_ref().map(kick::Queue::parent); + let work = packet.completion.work_state(); + let work_slot = work.render_slot(self.slot)?; + self.memory.bind_work_slot(ordinal, qids, work_slot)?; + let payload = work.channel_payload(self.slot, ordinal)?; + let stamps = host.stamps(qids)?; + fence(Ordering::Acquire); + let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); + self.memory + .completion_links(ordinal, kicks, payload, !covered[1])?; + let context = &packet.context; + let qos = context.qos_id(); + let qos_owners = [ + qos::Owner { + qid: qids[0], + qos, + data_master: DataMaster::Tiling as u8, + }, + qos::Owner { + qid: qids[1], + qos, + data_master: DataMaster::Fragment as u8, + }, + ]; + let mut configs = [None; 2]; + for stage in 0..2 { + configs[stage] = Some(QueueConfig::new(&QueueConfigArgs { + target: if stage == 0 { + QueueConfigTarget::Tiling(TilingBinding { + pb_va: self.memory.manager.state_va(), + pb_token: lease.buffer_token, + pb_slot: self.buffer_id().into(), + fragment_qid: qids[1], + }) + } else { + QueueConfigTarget::Fragment + }, + kick_ring_va: self.kicks[stage].low_va(), + kick_ring_fw_va: self.kicks[stage].firmware_va(), + qid: qids[stage], + install: !self.installed[stage], + completion_seed: covered[stage].then(|| { + kicks[stage] + .prev() + .completion_value(payload.wrapping_sub(1)) + }), + context_id: context.id().try_into()?, + free_list: FreeListBinding { + va: self.free_list.control_va(), + generation: lease.free_list_generation, + slot: self.free_list.id().into(), + }, + scheduler_va: context.scheduler_va(), + qos_slot: qos, + qos_update: self.context_update, + owner_pid: context.owner_pid(), + context_update: self.context_update, + })?); + } + let configs = [configs[0].ok_or(EIO)?, configs[1].ok_or(EIO)?]; + let cache = if self.recovery_generation == epoch { + self.configs + } else { + [None; 2] + }; + let present = [ + publication::required(cache[0], configs[0]), + publication::required(cache[1], configs[1]), + ]; + let cursors = self.memory.graph.cursors()?; + if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { + return Err(EBUSY); + } + let queues = qids.map(|qid| { + self.memory.graph.queues.gpu_va() + + u64::from(qid) * size_of::() as u64 + }); + let fragment_barriers = [KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?]; + let args = |stage: usize, barriers, arrays, mcache| KickArgs { + qid: qids[stage], + timestamp: kicks[stage], + parent: parents[stage], + barriers, + descriptor_va: self.memory.graph.descriptor_va(stage, ordinal), + queue_va: queues[stage], + qos: KickQos { + slot: qos, + class: if stage == 0 { + abi::TA_QOS_CLASS + } else { + abi::FRAGMENT_QOS_CLASS + }, + }, + mcache, + event_mask: abi::RENDER_KICK_EVENT_MASK, + register_arrays: arrays, + }; + let ta_args = args( + 0, + dependencies, + abi::TaDescriptor::register_bindings(self.memory.graph.descriptor_client(0, ordinal))?, + None, + ); + let fragment_args = args( + 1, + &fragment_barriers, + abi::FragmentDescriptor::register_bindings( + self.memory.graph.descriptor_client(1, ordinal), + )?, + mcache, + ); + let registration = RenderRegistration::new( + (qids[1], self.kicks[1].low_va()), + (qids[0], self.kicks[0].low_va()), + )?; + let [tiling_kick, fragment_kick] = &mut self.kicks; + let tiling_kick = tiling_kick.prepare_entry(&ta_args)?; + let fragment_kick = fragment_kick.prepare_entry(&fragment_args)?; + let memory = &mut *self.memory; + let lanes = [ + publication::Lane::new( + &memory.graph, + 0, + ordinal, + self.cursors[0], + configs[0], + present[0], + kicks[0], + )?, + publication::Lane::new( + &memory.graph, + 1, + ordinal, + self.cursors[1], + configs[1], + present[1], + kicks[1], + )?, + ]; + let windows = [lanes[0].window, lanes[1].window]; + let previous = self.active.front().map_or(cursors, |active| { + active.ticket.windows.map(|window| { + let target = window.target(); + retirement::Cursors { + done: target, + read: target, + write: target, + } + }) + }); + let fragment_slot = WorkSlot::new( + DataMaster::Fragment, + queues[1], + qids[1], + windows[1].target().try_into()?, + !self.outer_started, + )?; + let tiling_slot = WorkSlot::new( + DataMaster::Tiling, + queues[0], + qids[0], + windows[0].target().try_into()?, + !self.outer_started, + )?; + let event_slot = memory + .graph + .support + .word(abi::storage::EVENT_SLOTS + 4)?; + let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); + let inner = memory.graph.support.word(0x28000)?; + let next_inner = (next_ordinal as u32).wrapping_mul(2); + let flist_submission = self.free_list.prepare_submission(2)?; + let pm_submission = memory.manager.prepare_submission()?; + self.active + .push(Active { + qos_completed: false, + flist_completed: false, + outer_published: false, + generation: lease.free_list_generation, + packet: packet.clone(), + ticket: Ticket { + ordinal, + windows, + previous, + }, + kicks, + context_update: self.context_update, + failed: false, + drained_result: None, + }) + .map_err(|_| EBUSY)?; + self.prepared = None; + packet.mark_published(); + lanes[1].publish_prefix(); + fragment_kick.commit(); + lanes[1].publish_kick(); + tiling_kick.commit(); + let mut qos_publications = [None; 2]; + let result = (|| { + qos_publications[1] = Some(host.qos_publish(qos_owners[1], context.scheduler_va())?); + qos_publications[0] = Some(host.qos_publish(qos_owners[0], context.scheduler_va())?); + if !self.registered { + host.install_pair(®istration)?; + self.registered = true; + } + for kick in &self.kicks { + host.publish_qid(kick.id())?; + } + host.publish_outer_pair( + &fragment_slot, + &tiling_slot, + || { + work.add_submitted_kicks(2)?; + flist_submission.commit(); + pm_submission.commit(); + Ok(()) + }, + || lanes[0].publish_tiling(event_slot, next_event, inner, next_inner), + ) + })(); + let outer_next = match result { + Ok(next) => next, + Err(error) => { + self.quarantined = true; + if !packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), error)); + } + self.active + .back() + .ok_or(EIO)? + .packet + .completion + .quarantine_render(); + for publication in qos_publications.into_iter().flatten() { + host.qos_cancel(publication)?; + } + self.active.back_mut().ok_or(EIO)?.qos_completed = true; + self.quarantined = true; + if !packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), error)); + } + defer(Deferred::FailedOwned(packet.completion.clone(), error))?; + self.active.back_mut().ok_or(EIO)?.failed = true; + return Err(error); + } + }; + self.ordinal = next_ordinal; + self.cursors = windows.map(|window| window.target()); + for stage in 0..2 { + if present[stage] { + self.configs[stage] = Some(configs[stage]); + } + } + self.recovery_generation = epoch; + self.installed = [true; 2]; + self.outer_started = true; + if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { + previous.publication = Some(ordinal); + } + packet.context.mark_published(); + let publish = (|| { + host.activate_render_pool(self.free_list.id(), lease.free_list_generation)?; + host.publish_retained_pair(&fragment_slot, &tiling_slot, outer_next)?; + self.active.back_mut().ok_or(EIO)?.outer_published = true; + host.notify_render()?; + host.note_submission() + })(); + if let Err(error) = publish { + self.quarantined = true; + self.active + .back() + .ok_or(EIO)? + .packet + .completion + .quarantine_render(); + if !packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), error)); + } + defer(Deferred::FailedOwned(packet.completion.clone(), error))?; + self.active.back_mut().ok_or(EIO)?.failed = true; + return Err(error); + } + Ok(()) + } +} + +struct Observer<'a, H> { + memory: &'a memory::Memory, + qids: [u8; 2], + host: &'a mut H, +} +impl retirement::Memory for Observer<'_, H> { + fn snapshot(&mut self, ordinal: u64) -> Result { + observation::snapshot(self.memory, ordinal) + } + + fn stamps(&mut self) -> Result<[u32; 2]> { + self.host.stamps(self.qids) + } + fn pass(&mut self, ordinal: u64) -> Result<[u64; 2]> { + self.memory.pass(ordinal) + } +} + +impl Pair { + fn qos_owners(&self) -> [qos::Owner; 2] { + let qids = self.qids(); + let qos = self.context().qos_id(); + [ + qos::Owner { + qid: qids[0], + qos, + data_master: DataMaster::Tiling as u8, + }, + qos::Owner { + qid: qids[1], + qos, + data_master: DataMaster::Fragment as u8, + }, + ] + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { + masks.is_none_or(|masks| { + self.qids() + .into_iter() + .all(|qid| masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) + }) + } + pub(crate) fn contains_packet(&self, packet: &Arc) -> bool { + self.active + .iter() + .any(|active| Arc::ptr_eq(&active.packet, packet)) + || self + .prepared + .as_ref() + .is_some_and(|prepared| Arc::ptr_eq(&prepared.packet, packet)) + } + pub(crate) fn take_deferred_timeout(&self) -> Option<(Arc, Error)> { + self.active.iter().find_map(|active| { + active + .packet + .completion + .take_deferred_timeout() + .map(|error| (active.packet.clone(), error)) + }) + } + + pub(crate) fn oldest_packet(&self) -> Option> { + self.active.front().map(|active| active.packet.clone()) + } + /// Sample PM feedback only after this pair's exact retirement. Busy pairs + /// keep their current capacity and let firmware request growth as needed. + pub(crate) fn host_growth_target(&mut self, packet: &Packet) -> Result> { + let Validated::Render { pass, .. } = &packet.command else { + return Err(EINVAL); + }; + let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; + if self.in_flight() { + return Ok(None); + } + if !self.idle_for_context(self.context())? { + return Err(EBUSY); + } + let feedback = if self.ordinal != 0 { + self.memory.manager.automatic_target()?.unwrap_or(0) + } else { + 0 + }; + let required = required.max(feedback); + Ok((required > self.memory.manager.blocks()).then_some(required)) + } + + pub(crate) fn needs_drain(&self) -> bool { + self.quarantined && self.in_flight() + } + + pub(crate) fn oldest_spared(&self) -> Option { + self.active + .front() + .map(|active| active.packet.completion.spared()) + } + pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { + for active in self.active.iter() { + visit(active.packet.completion.status()); + } + } + pub(crate) fn classify(&self, classify: &dyn Fn(&Arc) -> recovery::Class) { + for active in self.active.iter() { + active + .packet + .completion + .classify(classify(active.packet.completion.status())); + } + } + /// A slot stamp must name retained work before its execution markers are meaningful. + pub(crate) fn pass_started(&self, stamp: u64) -> Option { + const MASK: u64 = (1 << 40) - 1; + let ordinal = self.active.iter().map(|active| active.ticket.ordinal) + .find(|ordinal| ordinal.wrapping_add(1) & MASK == stamp & MASK)?; + Some(observation::snapshot(&self.memory, ordinal).ok()?.pass[0] != 0) + } + + /// Firmware is halted and all graph borrowers have joined before this copy. + pub(crate) fn publish_recovery_targets(&self) -> Result { + self.memory.graph.publish_recovery_targets() + } + + pub(crate) fn resync(&mut self) { + self.configs = [None; 2]; + for kick in &mut self.kicks { + kick.clear_parent_after_recovery(); + } + } + pub(crate) fn matches(&self, owner: u64, context: &Arc) -> Result { + Ok(self.owner == Some(owner) + && Arc::ptr_eq(self.context(), context) + && context.is_current()) + } + /// Classification controls VM poisoning. Every failed render fence settles, + /// while command memory remains owned until paired retirement or stop. + pub(crate) fn quarantine( + &mut self, + error: Error, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.quarantined = true; + // Successors published after recovery can still fail behind its spared + // head. Classify the whole failing group before any fence can settle. + if self.active.iter().any(|active| active.packet.completion.spared()) { + for active in self.active.iter() { + active.packet.completion.classify(recovery::Class::Spared); + } + } + for active in self.active.iter_mut() { + active.packet.completion.quarantine_render(); + if !active.failed { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + error, + ))?; + active.failed = true; + if !active.packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((active.packet.completion.status().clone(), error)); + } + } + } + Ok(()) + } + pub(crate) fn retire( + &mut self, + masks: Option<[u64; 2]>, + _epoch: u64, + state: u32, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.retire_front(masks, state, None, host, defer) + } + + pub(crate) fn retire_observed( + &mut self, + observed: Observed, + state: u32, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.retire_front(None, state, Some(observed), host, defer) + } + + fn retire_front( + &mut self, + masks: Option<[u64; 2]>, + state: u32, + observed: Option, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if self.quarantined || state != 0 || !self.selected(masks) { + return Ok(false); + } + let qids = self.qids(); + let qos = self.qos_owners(); + let pool = self.free_list.id(); + let buffer = self.buffer_id(); + loop { + let Some(front) = self.active.front_mut() else { + return Ok(false); + }; + if !front.outer_published { + return Ok(false); + } + if !Arc::ptr_eq(&front.packet.context, &self.memory.context) + || !self.memory.owns(front.ticket.ordinal) + { + return Err(EIO); + } + let mut observer = Observer { + memory: &self.memory, + qids, + host, + }; + let result = if let Some(observed) = observed { + if observed.ticket != front.ticket { + return Err(EIO); + } + retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + } else { + retirement::poll(&mut front.ticket, &mut observer) + }; + let timestamps = match result { + Ok(retirement::Poll::Complete(timestamps)) => timestamps, + Ok(retirement::Poll::Retry | retirement::Poll::Pending) => return Ok(false), + Err(error) => { + self.quarantine(error, defer)?; + return Ok(false); + } + }; + let ordinal = front.ticket.ordinal; + let context_update = front.context_update; + self.note_retirement(ordinal, context_update); + let front = self.active.front_mut().ok_or(EIO)?; + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.qos_completed = true; + front.flist_completed = true; + } + defer(Deferred::Retired( + front.packet.completion.clone(), + Ok(timestamps), + ))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + return Ok(true); + } + } + + /// A closed preparation epoch invalidates only this unpublished descriptor. + /// Its completion remains available for a retry under the next epoch. + pub(crate) fn discard_prepared(&mut self, packet: &Arc) -> Result { + let Some(prepared) = self.prepared.as_ref() else { + return Ok(()); + }; + if !Arc::ptr_eq(&prepared.packet, packet) { + return Err(EBUSY); + } + self.memory.retire(prepared.ordinal)?; + self.prepared = None; + Ok(()) + } +} + +impl Pair { + /// Call before and again after servicing firmware events. FList completion + /// counters are full-width, including beyond the low-word wrap boundary. + pub(crate) fn firmware_drained(&self, host: &impl Host) -> Result { + if self.prepared.is_some() || self.active.iter().any(|a| !a.outer_published) { + return Ok(false); + } + let counters = self.free_list.counters()?; + let stamps = host.stamps(self.qids())?; + let (_, state) = host.epoch()?; + Ok(retirement::drained(self.ordinal, stamps, counters, state)) + } + + /// Account one witnessed tracker, preserving all completion ownership. + /// The caller services its pending FList release before accounting another + /// tracker and invokes release_witnessed only after that control stage. + pub(crate) fn account_drained(&mut self, host: &mut impl Host) -> Result { + use retirement::Memory; + if !self.firmware_drained(host)? { + return Ok(false); + } + let qids = self.qids(); + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let Some(active) = self.active.iter_mut().find(|a| !a.flist_completed) else { + return Ok(false); + }; + let mut observer = Observer { + memory: &self.memory, + qids, + host, + }; + let result = match retirement::poll(&mut active.ticket, &mut observer)? { + retirement::Poll::Complete(ticks) => Ok(ticks), + retirement::Poll::Pending | retirement::Poll::Retry => { + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + if !retirement::abandoned(&active.ticket, snapshot, stamps, state, true)? { + return Ok(false); + } + Err(EIO) + } + }; + let ordinal = active.ticket.ordinal; + let context_update = result.is_ok() && active.context_update; + self.note_retirement(ordinal, context_update); + let active = self + .active + .iter_mut() + .find(|active| active.ticket.ordinal == ordinal) + .ok_or(EIO)?; + host.complete_render( + (!active.qos_completed).then_some(qos), + self.free_list.id(), + active.generation, + buffer, + )?; + active.qos_completed = true; + active.flist_completed = true; + active.drained_result = Some(result); + Ok(true) + } + + /// Fresh proof after asynchronous control publication is the ownership + /// boundary. A refused deferred item leaves the precise owner available. + pub(crate) fn release_witnessed( + &mut self, + host: &impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if self.active.iter().any(|a| !a.flist_completed) || !self.firmware_drained(host)? { + return Ok(()); + } + while let Some(active) = self.active.front() { + let result = active.drained_result.ok_or(EIO)?; + defer(Deferred::Drained(active.packet.completion.clone(), result))?; + let ordinal = active.ticket.ordinal; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + } + self.quarantined = false; + self.resync(); + Ok(()) + } + + /// The registry first polls the completed prefix through this exact Arc. + /// With successors, source policy quarantines every remaining group. A + /// single unexecuted group may use the strict ownership witness instead. + pub(crate) fn fail_after_observation( + &mut self, + packet: &Arc, + error: Error, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + use retirement::Memory; + if !self.active.iter().any(|a| Arc::ptr_eq(&a.packet, packet)) { + return Ok(false); + } + if !self.quarantined && self.active.len() == 1 { + let active = self.active.front().ok_or(EIO)?; + let mut observer = Observer { + memory: &self.memory, + qids: self.qids(), + host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + if active.outer_published + && retirement::abandoned(&active.ticket, snapshot, stamps, state, false)? + { + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let active = self.active.front_mut().ok_or(EIO)?; + if !active.flist_completed { + host.complete_render( + (!active.qos_completed).then_some(qos), + self.free_list.id(), + active.generation, + buffer, + )?; + active.flist_completed = true; + active.qos_completed = true; + } + if !active.failed && !active.packet.completion.spared() { + active.packet.completion.status().record(EIO); + if self.terminal.is_none() { + self.terminal = Some((active.packet.completion.status().clone(), error)); + } + } + defer(Deferred::Retired( + active.packet.completion.clone(), + Err(EIO), + ))?; + let ordinal = active.ticket.ordinal; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + return Ok(true); + } + } + self.quarantine(error, defer)?; + Ok(true) + } + + /// Both processors have stopped before the caller reaches this method. + /// VM ownership settles before quarantined admission is rehabilitated. + pub(crate) fn stopped(&mut self, defer: &mut impl FnMut(Deferred) -> Result) -> Result { + if let Some(prepared) = self.prepared.as_ref() { + defer(Deferred::Unpublished( + prepared.packet.completion.clone(), + EIO, + ))?; + self.memory.retire(prepared.ordinal)?; + self.prepared = None; + } + while let Some(active) = self.active.front() { + defer(Deferred::Drained( + active.packet.completion.clone(), + Err(EIO), + ))?; + let ordinal = active.ticket.ordinal; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + } + if let Some(previous) = self.previous.as_mut() { + previous.ready = true; + } + Ok(()) + } +} + +impl Pair { + /// Commit a cold graph only after the registry revalidates its preparation + /// lease. The original PB row is restored on every later failure. + pub(crate) fn finish_install( + &mut self, + alloc: &Allocator<'_>, + descriptors: &crate::g17::object::KernelObject, + ) -> Result { + install_memory( + alloc, + descriptors, + self.slot, + &mut self.memory, + &mut self.pending_aliases, + &mut self.kick_aliases, + ) + } +} + +fn install_memory( + alloc: &Allocator<'_>, + descriptors: &crate::g17::object::KernelObject, + slot: u8, + memory: &mut memory::Memory, + pending_aliases: &mut Option>, + kick_aliases: &mut Option<[Option; 2]>, +) -> Result { + use core::sync::atomic::{fence, Ordering}; + if pending_aliases.is_none() && kick_aliases.is_none() { + return Ok(()); + } + if !memory.context.is_current() || kick_aliases.is_none() { + return Err(EIO); + } + let mut driver = KVec::new(); + driver.reserve(2, GFP_KERNEL)?; + let aliases = memory.take_persistent(pending_aliases.as_mut().ok_or(EIO)?)?; + let context = memory.context.clone(); + let vm = context.vm(); + let pool_id = u64::from(memory.lifecycle[0].predecessor); + let mut staged: Option<[u32; 4]> = None; + let offset = usize::from(memory.manager.buffer_id()) * 16; + let result: Result = (|| { + vm.install_render_pool_mappings(slot, pool_id, memory.manager.blocks(), aliases)?; + memory.installed_pool = Some(slot); + alloc.uat.flush_vm_contexts(vm)?; + let words = [ + descriptors.word(offset)?, + descriptors.word(offset + 4)?, + descriptors.word(offset + 8)?, + descriptors.word(offset + 12)?, + ]; + let previous = words.map(|word| word.load(Ordering::Relaxed)); + let pages: u32 = (memory.manager.blocks() + * crate::hw::t8140::resources::TVB_PAGES_PER_BLOCK) + .try_into()?; + let next = crate::g17::fw::buffer::reload_descriptor( + previous, + memory.manager.page_list_va(), + pages, + ); + for (word, value) in words.into_iter().zip(next) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + staged = Some(previous); + for mapping in kick_aliases.take().ok_or(EIO)?.into_iter().flatten() { + driver.push(mapping, GFP_KERNEL)?; + } + vm.append_driver_mappings(driver)?; + Ok(()) + })(); + if let Err(error) = result { + if error == ENOENT { + context.status().record(error); + } + if let Some(previous) = staged { + for (index, value) in previous.into_iter().enumerate() { + descriptors + .word(offset + index * 4)? + .store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + } + vm.clear_render_pool_mappings_if_owner(slot, pool_id); + memory.installed_pool = None; + return Err(error); + } + *pending_aliases = None; + Ok(()) +} + +/// A binding prepared while the registry exclusively lends the idle pair. +/// Its frontier prevents a stale off-lock result from replacing another owner. +pub(crate) struct PreparedBinding { + previous_context: Arc, + previous_owner: Option, + ordinal: u64, + owner: u64, + context: Arc, + pool: Arc, + memory: Option>, + aliases: Option>, + kicks: Option<[Option; 2]>, + initial_packet: u64, +} + +/// The registry retains displaced backing until this exact context's release +/// is acknowledged. A warm binding retains the old context without its graph. +pub(crate) struct RetiredBinding { + memory: Option>, + context: Arc, + pool: Arc, + buffer_id: u8, + publication: Option, + ready: bool, +} +impl RetiredBinding { + pub(crate) fn context(&self) -> &Arc { + &self.context + } + pub(crate) fn pool(&self) -> &Arc { + &self.pool + } + pub(crate) fn buffer_id(&self) -> Option { + self.memory.as_ref().map(|_| self.buffer_id) + } +} + +impl Pair { + fn note_retirement(&mut self, ordinal: u64, context_update: bool) { + if !context_update { + return; + } + self.context_update = false; + if let Some(previous) = self.previous.as_mut() { + if previous + .publication + .is_some_and(|published| published <= ordinal) + { + previous.ready = true; + } + } + } + pub(crate) fn take_retired_binding(&mut self) -> Option { + if self.displaced.is_some() { + return self.displaced.take(); + } + if self.previous.as_ref().is_some_and(|p| p.ready) { + self.previous.take() + } else { + None + } + } + pub(crate) fn idle_for_context(&self, context: &Arc) -> Result { + if !Arc::ptr_eq(context, self.context()) { + return Ok(true); + } + if self.prepared.is_some() || self.in_flight() || self.quarantined { + return Ok(false); + } + if self.ordinal == 0 { + return Ok(true); + } + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + Ok(self + .memory + .graph + .cursors()? + .into_iter() + .all(|c| c.done == c.read && c.read == c.write)) + } + pub(crate) fn reusable(&self) -> Result { + let previous_ready = self + .previous + .as_ref() + .is_none_or(|p| p.memory.is_none() && self.context_update && !p.ready); + Ok(self.owner.is_none() + && self.displaced.is_none() + && previous_ready + && self.idle_for_context(self.context())?) + } + /// Logical close leaves the physical channel identity and backing retained. + pub(crate) fn release_owner(&mut self) { + self.owner = None; + } + #[inline(never)] + pub(crate) fn prepare_bind( + &mut self, + alloc: &Allocator<'_>, + pool: &Arc, + owner: u64, + context: Arc, + free_list: Arc, + metrics: Option, + buffer_id: Option<(u8, u64)>, + pm_generation: Option, + first: &Arc, + ) -> Result { + if !self.reusable()? + || owner == 0 + || !Arc::ptr_eq(&context, &first.context) + || !context.is_current() + { + return Err(EBUSY); + } + alloc.uat.vm_context_mask(context.vm())?; + let same_vm = Arc::ptr_eq(self.context().status(), context.status()); + let (memory, aliases, kicks) = if same_vm { + if metrics.is_some() || buffer_id.is_some() || pm_generation.is_some() { + return Err(EINVAL); + } + (None, None, None) + } else { + let metrics = metrics.ok_or(EINVAL)?; + let buffer_id = buffer_id.ok_or(EINVAL)?; + let pm_generation = pm_generation.ok_or(EINVAL)?; + let Validated::Render { pass, .. } = &first.command else { + return Err(EINVAL); + }; + let layout = target::Layout::new(pass, self.clusters)?; + let aliases = [ + self.kicks[0].prepare_client_alias(context.vm())?, + self.kicks[1].prepare_client_alias(context.vm())?, + ]; + let (memory, persistent) = memory::Memory::new( + alloc, + pool, + context.clone(), + self.qids(), + &layout, + metrics, + buffer_id, + pm_generation, + self.ordinal, + first.completion.work_state().word(), + )?; + (Some(memory), Some(persistent), Some(aliases)) + }; + Ok(PreparedBinding { + previous_context: self.context().clone(), + previous_owner: self.owner, + ordinal: self.ordinal, + owner, + context, + pool: free_list, + memory, + aliases, + kicks, + initial_packet: first.id, + }) + } + pub(crate) fn bind_owner( + &mut self, + binding: &mut PreparedBinding, + alloc: &Allocator<'_>, + descriptors: &crate::g17::object::KernelObject, + ) -> Result { + if !self.reusable()? + || binding.ordinal != self.ordinal + || binding.previous_owner != self.owner + || !Arc::ptr_eq(&binding.previous_context, self.context()) + || !binding.context.is_current() + { + return Err(EBUSY); + } + let foreign = binding.memory.is_some(); + if let Some(memory) = binding.memory.as_mut() { + install_memory( + alloc, + descriptors, + self.slot, + memory, + &mut binding.aliases, + &mut binding.kicks, + )?; + } else { + self.memory.set_owner(self.qids(), &binding.context)?; + } + let old_context = self.context().clone(); + let old_buffer = self.buffer_id(); + let old_pool = core::mem::replace(&mut self.free_list, binding.pool.clone()); + let old_memory = binding + .memory + .take() + .map(|memory| core::mem::replace(&mut self.memory, memory)); + let retired = RetiredBinding { + memory: old_memory, + context: old_context, + pool: old_pool, + buffer_id: old_buffer, + publication: None, + ready: foreign, + }; + if foreign { + self.displaced = Some(retired); + } else if self.previous.is_none() { + self.previous = Some(retired); + } + self.memory.context = binding.context.clone(); + self.owner = Some(binding.owner); + self.initial_packet = binding.initial_packet; + self.context_update = true; + if foreign { + self.cursors = [0; 2]; + self.configs = [None; 2]; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs new file mode 100644 index 0000000000000..d1ec0eb76c7e1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Render predecessors retained while a physical pair is prepared off-lock. + +use super::Pair; +use crate::g17::{ + dependency::Frontier, + fw::kick::KickDependency, + job::{self, Packet}, + status::VmStatus, +}; +use crate::hw::t8140::queues::RENDER_DEPTH; +use kernel::{prelude::*, sync::Arc}; + +struct Predecessor { + packet: Arc, + word: u64, +} + +/// A loan of the graph cannot hide work already published on that pair. Fence +/// status remains live through the retained Packet, even while its storage is away. +pub(crate) struct Dependencies { + owner: Option, + entries: [Option; RENDER_DEPTH as usize], +} + +fn include( + frontier: &mut Frontier, + status: &Arc, + prefix: Option, + packet: &Arc, + word: u64, +) -> Result { + if !Arc::ptr_eq(packet.completion.status(), status) { + return Ok(()); + } + if !job::Order::contains(prefix, packet.order.sequence) { + return Ok(()); + } + frontier.include( + prefix, + packet.order.sequence, + job::fence_status(&packet.completion.fence()), + word, + ) +} + +impl Dependencies { + pub(crate) fn contains_packet(&self, packet: &Arc) -> bool { + self.entries + .iter() + .flatten() + .any(|entry| Arc::ptr_eq(&entry.packet, packet)) + } + + pub(crate) fn collect( + &self, + owner: u64, + status: &Arc, + prefix: Option, + frontier: &mut Frontier, + ) -> Result { + if self.owner != Some(owner) { + return Ok(()); + } + for entry in self.entries.iter().flatten() { + include(frontier, status, prefix, &entry.packet, entry.word)?; + } + Ok(()) + } +} + +impl Pair { + /// The fixed tracker bounds this snapshot; no allocation or packet can be + /// lost between capture and the registry moving the pair under the same lock. + pub(crate) fn dependency_snapshot(&self) -> Dependencies { + let qid = self.qids()[1]; + let mut active = self.active.iter(); + Dependencies { + owner: self.owner, + entries: core::array::from_fn(|_| { + active.next().map(|entry| Predecessor { + packet: entry.packet.clone(), + word: KickDependency::new(qid, entry.kicks[1]) + .map_or(u64::MAX, KickDependency::word), + }) + }), + } + } + + pub(crate) fn collect_dependencies( + &self, + owner: u64, + status: &Arc, + prefix: Option, + frontier: &mut Frontier, + ) -> Result { + if self.owner != Some(owner) { + return Ok(()); + } + let qid = self.qids()[1]; + for entry in self.active.iter() { + let word = + KickDependency::new(qid, entry.kicks[1]).map_or(u64::MAX, KickDependency::word); + include(frontier, status, prefix, &entry.packet, word)?; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs new file mode 100644 index 0000000000000..f614494e1216a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -0,0 +1,708 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Storage of one render pair and its per-command generations. Client aliases +//! precede their backing owners so teardown cannot leave a dangling GPU view. + +use super::retirement::Cursors; +use crate::g17::{ + buffer::{self, target}, + context::Context, + fw::{ + self, queue, + render::{self as abi, storage}, + }, + object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, +}; +use crate::{ + hw::t8140::{dynamic, kernel_window, low, queues, resources as cfg}, + mmu, +}; +use core::sync::atomic::Ordering; +use kernel::{prelude::*, sync::Arc}; + +pub(super) const RECORDS: usize = 32; +pub(super) const TA_BYTES: usize = 0x14000; +pub(super) const FRAGMENT_BYTES: usize = 0x48000; +pub(super) const GRAPH_SIZE: usize = 0x1c000; +pub(super) const POINTERS: [usize; 2] = [0x9000, 0x9080]; +pub(super) const ITEMS: [usize; 2] = [0xc000, 0x10000]; +pub(super) const ANNOUNCE_STRIDE: usize = 0x80; +pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; +pub(super) const JOB_LIST: usize = 0xa000; +pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; +pub(super) const CONFIG_STRIDE: usize = 0x180; +const DEPTH: usize = queues::RENDER_DEPTH as usize; + +struct Aliases { + client: mmu::KernelMapping, + _global: mmu::KernelMapping, +} +impl Aliases { + fn new( + object: &KernelObject, + alloc: &Allocator<'_>, + vm: &mmu::Vm, + source: core::ops::Range, + ) -> Result { + let mut range = dynamic::LOWER; + let size = (source.end - source.start) as u64; + loop { + let global = object.map_range( + alloc.uat.kernel_lower_vm(), + source.clone(), + range.clone(), + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + 0, + )?; + let address = global.iova(); + let end = address.checked_add(size).ok_or(EOVERFLOW)?; + match object.map_range( + vm, + source.clone(), + address..end, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + 0, + ) { + Ok(client) => { + return Ok(Self { + client, + _global: global, + }) + } + Err(error) if error == ENOSPC && end < range.end => { + drop(global); + range.start = end; + } + Err(error) => return Err(error), + } + } + } +} + +/// Stable queue records and rotating descriptors/configuration records. +pub(super) struct Graph { + aliases: [Aliases; 2], + pub(super) descriptors: PooledObject, + pub(super) queues: PooledObject, + pub(super) config: PooledObject, + pub(super) support: KernelObject, + pub(super) owners: [Option; RECORDS], +} +impl Graph { + pub(super) fn new( + alloc: &Allocator<'_>, + pool: &Arc, + context: &Context, + qids: [u8; 2], + ) -> Result { + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let descriptors = pool.allocate( + alloc, + Family::Descriptor, + TA_BYTES + FRAGMENT_BYTES, + TA_BYTES + FRAGMENT_BYTES, + prot, + CpuMap::WriteBack, + )?; + let ta = Aliases::new(&descriptors, alloc, context.vm(), 0..TA_BYTES)?; + let fragment = Aliases::new( + &descriptors, + alloc, + context.vm(), + TA_BYTES..TA_BYTES + FRAGMENT_BYTES, + )?; + let mut queues = pool.allocate( + alloc, + Family::QueueState, + GRAPH_SIZE, + 0x8000, + prot, + CpuMap::WriteCombined, + )?; + let config = pool.allocate( + alloc, + Family::QueueConfig, + 0x8000, + 0x8000, + prot, + CpuMap::WriteBack, + )?; + let support = alloc.kernel( + storage::SUPPORT_SIZE, + mmu::UAT_PGSZ as u64, + prot, + CpuMap::WriteCombined, + )?; + let address = queues.gpu_va(); + for stage in 0..2 { + queues.write( + qids[stage] as usize * size_of::(), + queue::QueueRecord::new(&queue::QueueRecordArgs { + pointers_va: address + POINTERS[stage] as u64, + items_va: address + ITEMS[stage] as u64, + job_list_va: context.work_head_va(), + scheduler_va: context.scheduler_va(), + owner_pid: context.owner_pid(), + }), + )?; + queues.initialize::(POINTERS[stage], |view| { + *view = queue::PointerBlock::new(); + Ok(()) + })?; + } + queues.write(JOB_LIST, queue::JobListHead::new(address + JOB_LIST as u64))?; + Ok(Self { + aliases: [ta, fragment], + descriptors, + queues, + config, + support, + owners: [None; RECORDS], + }) + } + pub(super) fn offset(stage: usize, ordinal: u64) -> usize { + let slot = ordinal as usize % RECORDS; + if stage == 0 { + slot * abi::TA_DESCRIPTOR_SIZE + } else { + TA_BYTES + slot * abi::FRAGMENT_SLOT_SIZE + } + } + pub(super) fn descriptor_va(&self, stage: usize, ordinal: u64) -> u64 { + self.descriptors.gpu_va() + Self::offset(stage, ordinal) as u64 + } + pub(super) fn descriptor_client(&self, stage: usize, ordinal: u64) -> u64 { + let stride = if stage == 0 { + abi::TA_DESCRIPTOR_SIZE + } else { + abi::FRAGMENT_SLOT_SIZE + }; + self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 + } + pub(super) fn publish_recovery_targets(&self) -> Result { + for at in POINTERS { + let producer = self.queues.word(at + queue::PointerBlock::PRODUCER)? + .load(Ordering::Relaxed); + self.queues.word(at + queue::PointerBlock::RECOVERY_TARGET)? + .store(producer, Ordering::Relaxed); + } + Ok(()) + } + + pub(super) fn cursors(&self) -> Result<[Cursors; 2]> { + let mut values = [Cursors { + done: 0, + read: 0, + write: 0, + }; 2]; + for stage in 0..2 { + let at = POINTERS[stage]; + values[stage] = Cursors { + done: self.queues.word(at)?.load(Ordering::Relaxed), + read: self.queues.word(at + 0x30)?.load(Ordering::Relaxed), + write: self.queues.word(at + 0x40)?.load(Ordering::Relaxed), + }; + } + Ok(values) + } + pub(super) fn claim(&mut self, ordinal: u64) -> Result { + let owner = &mut self.owners[ordinal as usize % RECORDS]; + if owner.is_some_and(|old| old != ordinal) { + return Err(EBUSY); + } + *owner = Some(ordinal); + Ok(()) + } + pub(super) fn owns(&self, ordinal: u64) -> bool { + self.owners[ordinal as usize % RECORDS] == Some(ordinal) + } + pub(super) fn release(&mut self, ordinal: u64) -> Result { + if !self.owns(ordinal) { + return Err(EIO); + } + self.owners[ordinal as usize % RECORDS] = None; + Ok(()) + } +} + +struct Compact { + mapping: Option, + address: u64, + _backing: KernelObject, +} +impl Compact { + fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, size: usize) -> Result { + let backing = KernelObject::backing(alloc.dev, size, CpuMap::WriteCombined)?; + let mapping = buffer::compact_alias( + &backing, + vm, + 0..size, + cfg::TVB_PAGE, + mmu::PROT_GPU_SHARED_RW, + mmu::UAT_PGSZ, + )?; + Ok(Self { + address: mapping.iova(), + mapping: Some(mapping), + _backing: backing, + }) + } +} + +pub(super) struct Generation { + deflake: mmu::KernelMapping, + auxiliary: mmu::KernelMapping, + fragment_client: mmu::KernelMapping, + state: KernelObject, + fragment: KernelObject, + pub(super) ordinal: Option, +} +impl Generation { + fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm) -> Result { + KernelObject::new( + alloc.dev, + vm, + crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, cfg::TVB_PAGE), + storage::STATE_SIZE, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + ) + } + fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject) -> Result> { + let deflake = buffer::compact_alias( + &state, + vm, + storage::DEFLAKE..storage::DEFLAKE + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + mmu::UAT_PGSZ, + )?; + let auxiliary_range = cfg::AUXILIARY; + let auxiliary = state.map_range( + vm, + storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE, + auxiliary_range, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + 0, + )?; + let fragment = alloc.kernel( + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let fragment_client = buffer::compact_alias( + &fragment, + vm, + 0..mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + mmu::UAT_PGSZ, + )?; + KBox::new( + Self { + deflake, + auxiliary, + fragment_client, + state, + fragment, + ordinal: None, + }, + GFP_KERNEL, + ) + .map_err(Into::into) + } + pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass) -> Result { + if self.ordinal.is_some_and(|old| old != ordinal) { + return Err(EBUSY); + } + let viewport = storage::Viewport::new(pass.width, pass.height)?; + self.state.bytes_mut()[storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE] + .fill(0); + let destination = self + .state + .pointer(storage::DEFLAKE + 0x900, size_of::())? + .cast::(); + if destination.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + // SAFETY: The complete typed integer view is in the retained CPU mapping + // and naturally aligned. This command generation is exclusively owned + // and unpublished; no prior command can access it until reassigned. + unsafe { viewport.apply(&mut *destination) }; + self.fragment.bytes_mut().fill(0); + self.ordinal = Some(ordinal); + Ok(()) + } + pub(super) fn addresses(&self) -> [u64; 5] { + [ + self.deflake.iova(), + self.state.gpu_va() + storage::TA_STATUS as u64, + self.fragment_client.iova(), + self.fragment.gpu_va(), + self.auxiliary.iova(), + ] + } +} + +pub(super) struct Memory { + _fragment_view: mmu::KernelMapping, + pub(super) graph: Graph, + scratch: Compact, + discard: Compact, + pub(super) target: KBox, + generations: [Option>; DEPTH], + generation_count: usize, + pub(super) timestamps: KernelObject, + pub(super) manager: buffer::Manager, + pub(super) lifecycle: [fw::ObjectIds; 2], + // The extra client views retain their allocation positions even though + // kick entries consume the register arrays embedded in the descriptors. + registers: KernelObject, + pub(super) context: Arc, + pub(super) installed_pool: Option, +} +impl Memory { + #[inline(never)] + pub(super) fn new( + alloc: &Allocator<'_>, + pool: &Arc, + context: Arc, + qids: [u8; 2], + layout: &target::Layout, + metrics: buffer::MetricsLease, + buffer_id: (u8, u64), + pm_generation: u64, + ordinal: u64, + word: u32, + ) -> Result<(KBox, KVec)> { + let storage = KBox::::new_uninit(GFP_KERNEL)?; + let vm = context.vm(); + let mut graph = Graph::new(alloc, pool, &context, qids)?; + let mut target = target::Target::new(alloc.dev, vm, layout)?; + target.prepare(ordinal, layout)?; + let state = Generation::allocate_state(alloc, vm)?; + let scratch = Compact::new(alloc, vm, cfg::SCENE_SIZE as usize)?; + let discard = Compact::new(alloc, vm, cfg::DISCARD_SIZE as usize)?; + let (tvb, mappings) = buffer::Tvb::new(alloc.dev, vm, layout.tvb_blocks)?; + let count = DEPTH; + let initial = if ordinal == 0 { + 0 + } else { + (ordinal ^ 1) as usize % count + }; + let mut generations = core::array::from_fn(|_| None); + generations[initial] = Some(Generation::new(alloc, vm, state)?); + for offset in 1..count { + let index = (initial + offset) % count; + let state = Generation::allocate_state(alloc, vm)?; + generations[index] = Some(Generation::new(alloc, vm, state)?); + } + let timestamps = alloc.kernel( + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let manager = buffer::Manager::new( + alloc, + vm, + tvb, + metrics, + buffer::ManagerArgs { + scratch: scratch.address, + discard: discard.address, + buffer_id: buffer_id.0, + generation: pm_generation, + first_ordinal: ordinal, + }, + )?; + let predecessor = crate::g17::ids::OBJECTS.object()?; + let (fragment, tiling) = crate::g17::ids::OBJECTS.render_pair(predecessor)?; + let support_va = graph.support.gpu_va(); + graph.support.initialize::(0, |view| { + view.init( + support_va, + alloc.uat.geometry().kernel_range().start + kernel_window::INDEX, + low::RENDER_RUN_LIST, + low::RENDER_PAGE_LIST, + word, + ) + })?; + let registers = KernelObject::new( + alloc.dev, + vm, + crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64), + mmu::UAT_PGSZ, + mmu::PROT_GPU_SHARED_RO, + CpuMap::WriteCombined, + )?; + let fragment_view = graph.descriptors.map_range( + vm, + TA_BYTES..TA_BYTES + mmu::UAT_PGSZ, + dynamic::CLIENT_LOWER, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RO, + 0, + )?; + Ok(( + storage.write_init(kernel::try_init!(Self { + graph, + scratch, + discard: discard, + target, + generations, + generation_count: count, + timestamps, + manager, + lifecycle: [tiling, fragment], + _fragment_view: fragment_view, + registers: registers, + context, + installed_pool: None, + }))?, + mappings, + )) + } + pub(super) fn generation(&mut self, ordinal: u64) -> Result<&mut Generation> { + self.generations[ordinal as usize % self.generation_count] + .as_deref_mut() + .ok_or(EIO) + } + pub(super) fn owns(&self, ordinal: u64) -> bool { + self.graph.owns(ordinal) + && self.generations[ordinal as usize % self.generation_count] + .as_ref() + .is_some_and(|generation| generation.ordinal == Some(ordinal)) + } + pub(super) fn pass(&self, ordinal: u64) -> Result<[u64; 2]> { + if !self.owns(ordinal) { + return Err(EIO); + } + let offset = ordinal as usize % RECORDS * 16; + Ok([ + self.timestamps.dword(offset)?.load(Ordering::Relaxed), + self.timestamps.dword(offset + 8)?.load(Ordering::Relaxed), + ]) + } + pub(super) fn owners(&self) -> [Option; DEPTH] { + core::array::from_fn(|index| self.generations[index].as_ref().and_then(|g| g.ordinal)) + } + pub(super) fn retire(&mut self, ordinal: u64) -> Result { + if !self.owns(ordinal) { + return Err(EIO); + } + self.target.retire(ordinal)?; + self.manager.retire_scene(ordinal)?; + self.graph.release(ordinal)?; + self.generation(ordinal)?.ordinal = None; + Ok(()) + } +} + +const _: () = { + assert!(abi::TA_DESCRIPTOR_SIZE * RECORDS <= TA_BYTES); + assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); + assert!(RECORDS * CONFIG_STRIDE <= 0x4000); +}; + +impl Memory { + pub(super) fn write_descriptors( + &mut self, + args: &abi::RenderArgs<'_>, + attachments: &[fw::kick::WriteRange], + context: u8, + ) -> Result> { + let tiling = self + .graph + .descriptors + .pointer( + Graph::offset(0, args.ordinal), + size_of::(), + )? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer( + Graph::offset(1, args.ordinal), + size_of::(), + )? + .cast::(); + if tiling.align_offset(align_of::()) != 0 + || fragment.align_offset(align_of::()) != 0 + { + return Err(EINVAL); + } + // SAFETY: Both complete integer-only views lie in the retained mapping, + // are aligned and disjoint. The full ordinal has an exclusive slot + // claim and no producer refers to this command yet. + unsafe { + (&mut *tiling).write(args)?; + let fragment = &mut *fragment; + fragment.write(args)?; + self.registers.bytes_mut().fill(0); + for (index, array) in fragment.descriptor.arrays.iter().enumerate() { + for (entry, value) in array + .entries + .get(..(array.size & 0xffff) as usize) + .ok_or(EINVAL)? + .iter() + .enumerate() + { + self.registers.write( + index * size_of::() + + entry * size_of::(), + *value, + )?; + } + } + fragment.set_mcache( + args.fragment.descriptor_va, + args.resources.aux_fb_va, + attachments, + context, + ) + } + } +} + +impl Memory { + pub(super) fn completion_links( + &mut self, + ordinal: u64, + kicks: [fw::kick::KickTimestamp; 2], + payload: u8, + fragment_busy: bool, + ) -> Result { + if !self.owns(ordinal) { + return Err(EIO); + } + let tiling = self + .graph + .descriptors + .pointer(Graph::offset(0, ordinal), size_of::())? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer(Graph::offset(1, ordinal), size_of::())? + .cast::(); + // SAFETY: The complete views were bounds/alignment checked during + // preparation. The unchanged full ordinal still exclusively owns both + // slots and neither descriptor has been published. + unsafe { + (&mut *tiling).set_completion_link(kicks[0], kicks[1], payload, fragment_busy); + (&mut *fragment).set_completion_link(kicks[1], payload); + } + Ok(()) + } +} + +impl Memory { + /// Preserve the initial mapping order while moving VM-lifetime mappings + /// out of the graph. Reserve all host storage before moving any mapping. + pub(super) fn take_persistent( + &mut self, + tvb: &mut KVec, + ) -> Result> { + if self.scratch.mapping.is_none() + || self.discard.mapping.is_none() + || self.registers.gpu_va() == 0 + { + return Err(EIO); + } + let mut aliases = KVec::new(); + aliases.reserve(tvb.len().checked_add(3).ok_or(EOVERFLOW)?, GFP_KERNEL)?; + aliases.push(self.scratch.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; + aliases.push(self.discard.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; + aliases.push(self.registers.take_mapping()?, GFP_KERNEL)?; + for mapping in core::mem::take(tvb) { + aliases.push(mapping, GFP_KERNEL)?; + } + Ok(aliases) + } +} + +impl Memory { + pub(super) fn bind_work_slot(&mut self, ordinal: u64, qids: [u8; 2], slot: u32) -> Result { + if slot >= 4 || !self.owns(ordinal) { + return Err(EINVAL); + } + let queues = [ + self.graph + .queues + .word(usize::from(qids[1]) * size_of::() + 0x38)?, + self.graph + .queues + .word(usize::from(qids[0]) * size_of::() + 0x38)?, + ]; + let ta = self + .graph + .descriptors + .pointer(Graph::offset(0, ordinal), size_of::())? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer(Graph::offset(1, ordinal), size_of::())? + .cast::(); + for word in queues { + word.store(slot, Ordering::Relaxed); + } + // SAFETY: The generation owns these prepared, unpublished POD views. + // Packed fields are assigned by value without an unaligned reference. + unsafe { + (*ta).unk_8be = slot; + (*fragment).descriptor.unk_2158 = slot; + } + core::sync::atomic::fence(Ordering::Release); + Ok(()) + } + + pub(super) fn set_owner(&self, qids: [u8; 2], context: &Context) -> Result { + let offsets = qids.map(|qid| usize::from(qid) * size_of::()); + let base = self.graph.queues.pointer(0, GRAPH_SIZE)?; + // Check every field before the first update, retaining the required + // job-head pair, PID pair, then packed scheduler-pointer pair order. + for at in offsets { + self.graph.queues.pointer(at + 0x10, 8)?; + self.graph.queues.pointer(at + 0x48, 4)?; + self.graph.queues.pointer(at + 0x9c, 8)?; + } + // SAFETY: Both queues are retired before retargeting. Complete byte + // views were checked above; packed scheduler words use unaligned stores. + unsafe { + for at in offsets { + base.add(at + 0x10) + .cast::() + .write_unaligned(context.work_head_va()); + } + for at in offsets { + base.add(at + 0x48) + .cast::() + .write_unaligned(context.owner_pid()); + } + for at in offsets { + base.add(at + 0x9c) + .cast::() + .write_unaligned(context.scheduler_va()); + } + } + Ok(()) + } +} + +impl Drop for Memory { + fn drop(&mut self) { + if let Some(slot) = self.installed_pool { + self.context.vm().clear_render_pool_mappings_if_owner( + slot, + u64::from(self.lifecycle[0].predecessor), + ); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/observation.rs b/drivers/gpu/drm/asahi/g17/queue/render/observation.rs new file mode 100644 index 0000000000000..3e853115e4f75 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/observation.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Read-only cursor and timestamp snapshots while a physical pair is borrowed. +//! Final stamps, timestamp refresh and ownership changes belong to commit. + +use super::{memory, retirement, Pair}; +use crate::g17::job::Packet; +use core::sync::atomic::{fence, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +pub(crate) struct Observed { + pub(super) ticket: retirement::Ticket, + pub(super) snapshot: retirement::Snapshot, +} + +impl Pair { + /// Reject incomplete event/stamp prefixes before lending the graph. This + /// is only a prefilter; commit repeats the final stamp and timestamp reads. + pub(crate) fn observation_packet( + &self, + masks: [u64; 2], + stamps: [u32; 2], + state: u32, + ) -> Result>> { + if self.quarantined || self.prepared.is_some() || state != 0 || !self.selected(Some(masks)) + { + return Ok(None); + } + let Some(front) = self.active.front() else { + return Ok(None); + }; + if !front.outer_published { + return Ok(None); + } + let published = front.ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; + if !stamps + .into_iter() + .all(|stamp| retirement::stamp_covers(stamp, published)) + { + return Ok(None); + } + Ok(Some(front.packet.clone())) + } + + pub(crate) fn observe_front(&self) -> Result { + let front = self.active.front().ok_or(EIO)?; + if !Arc::ptr_eq(&front.packet.context, &self.memory.context) { + return Err(EIO); + } + let ticket = front.ticket; + for _ in 0..2 { + fence(Ordering::Acquire); + let snapshot = snapshot(&self.memory, ticket.ordinal)?; + if retirement::observe(&ticket, snapshot).is_ok() { + return Ok(Observed { ticket, snapshot }); + } + } + Err(EAGAIN) + } +} + +pub(super) fn snapshot(memory: &memory::Memory, ordinal: u64) -> Result { + if !memory.owns(ordinal) { + return Err(EIO); + } + let cursors = memory.graph.cursors()?; + // Sample the private head before the logical shared head, whose membership + // may include other pairs. Individual completion does not require it empty. + let _first = memory + .graph + .queues + .dword(memory::JOB_LIST)? + .load(Ordering::Relaxed); + let _last = memory + .graph + .queues + .dword(memory::JOB_LIST + 8)? + .load(Ordering::Relaxed); + let head = memory.context.work_head()?; + Ok(retirement::Snapshot { + cursors, + job_list_empty: head == [0, memory.context.work_head_va()], + pass: memory.pass(ordinal)?, + }) +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs new file mode 100644 index 0000000000000..35955810a5af1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Installed kick storage retained after a logical scheduler release is acknowledged. + +use super::{install_memory, memory, target, Pair, RetiredBinding, Tracker}; +use crate::g17::{ + buffer, + command::Validated, + context::Context, + freelist::RenderPool, + job::Packet, + kick, + object::{Allocator, KernelObject, Pool}, +}; +use crate::mmu; +use kernel::{prelude::*, sync::Arc}; + +/// The empty physical pair retains installed addresses and the last scheduler page, +/// without retaining that scheduler's execution context or client root. A prior +/// context awaiting an executed replacement keeps its separate retirement owner. +pub(crate) struct ParkedPair { + kicks: Option<[kick::Queue; 2]>, + _scheduler: Arc, + ids: [kick::Id; 2], + slot: u8, + ordinal: u64, + clusters: u32, + descriptor_flags: [u32; 2], + installed: [bool; 2], + recovery_generation: u64, + registered: bool, + outer_started: bool, + previous: Option, +} + +/// All new graph ownership stays separate from the installed kick storage until commit. +pub(crate) struct Prepared { + memory: Option>, + pool: Arc, + aliases: Option>, + kicks: Option<[Option; 2]>, + ids: [kick::Id; 2], + slot: u8, + ordinal: u64, + owner: u64, + initial_packet: u64, + installed: bool, +} + +impl Pair { + /// The registry also checks the shared free-list release and scheduler-release witness. + /// Drain displaced owners before this preflight; they have their own logical releases. + pub(crate) fn can_park(&self, context: &Arc) -> Result { + Ok(Arc::ptr_eq(context, self.context()) + && self.owner.is_none() + && self + .previous + .as_ref() + .is_none_or(|previous| previous.memory.is_none() && !previous.ready) + && self.displaced.is_none() + && self.pending_aliases.is_none() + && self.kick_aliases.is_none() + && self.idle_for_context(context)?) + } + + /// Consumes the pair only after can_park and all collector capacity checks succeeded + /// under the same device mutex. Moving these owners performs no firmware operation. + pub(crate) fn park(self) -> (ParkedPair, RetiredBinding) { + let context = self.memory.context.clone(); + let scheduler = context.scheduler_owner(); + let buffer_id = self.buffer_id(); + let parked = ParkedPair { + ids: self.kicks.each_ref().map(|queue| queue.id()), + kicks: Some(self.kicks), + _scheduler: scheduler, + slot: self.slot, + ordinal: self.ordinal, + clusters: self.clusters, + descriptor_flags: self.descriptor_flags, + installed: self.installed, + recovery_generation: self.recovery_generation, + registered: self.registered, + outer_started: self.outer_started, + previous: self.previous, + }; + let retired = RetiredBinding { + memory: Some(self.memory), + context, + pool: self.free_list, + buffer_id, + publication: None, + ready: true, + }; + (parked, retired) + } +} + +impl ParkedPair { + pub(crate) fn qids(&self) -> [u8; 2] { + self.ids.map(kick::Id::qid) + } + + #[inline(never)] + pub(crate) fn prepare( + &self, + alloc: &Allocator<'_>, + pool: &Arc, + owner: u64, + context: Arc, + free_list: Arc, + metrics: buffer::MetricsLease, + buffer_id: (u8, u64), + pm_generation: u64, + first: &Arc, + ) -> Result { + if owner == 0 || !Arc::ptr_eq(&context, &first.context) || !context.is_current() { + return Err(EINVAL); + } + let kicks = self.kicks.as_ref().ok_or(EBUSY)?; + let Validated::Render { pass, .. } = &first.command else { + return Err(EINVAL); + }; + let word = first.completion.work_state().word(); + if word == u32::MAX { + return Err(EINVAL); + } + alloc.uat.vm_context_mask(context.vm())?; + let layout = target::Layout::new(pass, self.clusters)?; + let aliases = [ + kicks[0].prepare_client_alias(context.vm())?, + kicks[1].prepare_client_alias(context.vm())?, + ]; + let (memory, persistent) = memory::Memory::new( + alloc, + pool, + context, + self.qids(), + &layout, + metrics, + buffer_id, + pm_generation, + self.ordinal, + word, + )?; + Ok(Prepared { + memory: Some(memory), + pool: free_list, + aliases: Some(persistent), + kicks: Some(aliases), + ids: self.ids, + slot: self.slot, + ordinal: self.ordinal, + owner, + initial_packet: first.id, + installed: false, + }) + } + + fn matches(&self, prepared: &Prepared) -> bool { + self.kicks.is_some() + && self.ids == prepared.ids + && self.slot == prepared.slot + && self.ordinal == prepared.ordinal + && prepared + .memory + .as_ref() + .is_some_and(|memory| memory.context.is_current()) + } + + /// All fallible graph installation precedes taking the installed kicks. Keep the empty + /// parked owner until the caller drops it off-lock, including its old scheduler page. + pub(crate) fn commit( + &mut self, + prepared: &mut Option, + storage: KBox>, + ) -> Result> { + let candidate = prepared.as_ref().ok_or(EINVAL)?; + if !candidate.installed || !self.matches(candidate) { + return Err(EBUSY); + } + let kicks = self.kicks.take().ok_or(EIO)?; + // The candidate was validated above and cannot change through the exclusive borrow. + let Some(prepared) = prepared.take() else { + self.kicks = Some(kicks); + return Err(EIO); + }; + let mut next = prepared; + let Some(memory) = next.memory.take() else { + self.kicks = Some(kicks); + return Err(EIO); + }; + storage.write_init(kernel::try_init!(Pair { + kick_aliases: None, + memory, + kicks, + free_list: next.pool.clone(), + pending_aliases: None, + prepared: None, + active: Tracker::new(), + slot: self.slot, + owner: Some(next.owner), + ordinal: self.ordinal, + initial_packet: next.initial_packet, + clusters: self.clusters, + descriptor_flags: self.descriptor_flags, + installed: self.installed, + configs: [None; 2], + cursors: [0; 2], + recovery_generation: self.recovery_generation, + registered: self.registered, + outer_started: self.outer_started, + quarantined: false, + terminal: None, + previous: self.previous.take(), + displaced: None, + context_update: true, + })) + } +} + +impl Prepared { + /// Called after the registry revalidates its preparation epoch and parked pair identity. + pub(crate) fn finish_install( + &mut self, + parked: &ParkedPair, + alloc: &Allocator<'_>, + descriptors: &KernelObject, + ) -> Result { + if !parked.matches(self) { + return Err(EBUSY); + } + install_memory( + alloc, + descriptors, + self.slot, + self.memory.as_mut().ok_or(EIO)?, + &mut self.aliases, + &mut self.kicks, + )?; + self.installed = true; + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs new file mode 100644 index 0000000000000..3df92062735e3 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Preflighted item-ring publications. Each lane retains an independent cursor +//! and omits its configuration record only when its durable identity matches. + +use super::{memory, retirement::Window}; +use crate::g17::fw::{ + kick::KickTimestamp, + queue::{DataMaster, KickAnnounce, QueueConfig, ITEM_RING_ENTRIES}, +}; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use kernel::prelude::*; + +fn identity(mut config: QueueConfig) -> QueueConfig { + config.install = 0; + config.context_update = 0; + config.completion_seed_valid = 0; + config.completion_seed = 0; + config.qos_update = 0; + config +} +pub(super) fn required(previous: Option, config: QueueConfig) -> bool { + config.install != 0 + || config.context_update != 0 + || config.completion_seed_valid != 0 + || config.qos_update != 0 + || previous.map(identity) != Some(identity(config)) +} + +pub(super) struct Lane<'a> { + items: [&'a AtomicU64; 3], + producer: &'a AtomicU32, + values: [u64; 3], + config: QueueConfig, + config_destination: *mut u64, + config_present: bool, + announce: KickAnnounce, + announce_destination: *mut KickAnnounce, + prefix: usize, + prefix_end: u32, + pub(super) window: Window, +} +impl<'a> Lane<'a> { + pub(super) fn new( + graph: &'a memory::Graph, + stage: usize, + ordinal: u64, + base: u32, + config: QueueConfig, + present: bool, + timestamp: KickTimestamp, + ) -> Result { + let count = 2 + u32::from(present); + let window = Window::new(base, count, ITEM_RING_ENTRIES)?; + let offset = ordinal as usize % memory::RECORDS; + let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; + let announce_offset = memory::ANNOUNCE[stage] + offset * memory::ANNOUNCE_STRIDE; + let config_destination = graph + .config + .pointer(config_offset, size_of::())? + .cast::(); + if config_destination.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + let announce_destination = graph + .queues + .pointer(announce_offset, size_of::())? + .cast::(); + let index = + |delta: u32| memory::ITEMS[stage] + ((base + delta) % ITEM_RING_ENTRIES) as usize * 8; + let items = [ + graph.queues.dword(index(0))?, + graph.queues.dword(index(1))?, + graph.queues.dword(index(2))?, + ]; + let announce_va = graph.queues.gpu_va() + announce_offset as u64; + let descriptor = graph.descriptor_va(stage, ordinal); + let values = if present { + [ + descriptor, + graph.config.gpu_va() + config_offset as u64, + announce_va, + ] + } else { + [descriptor, announce_va, 0] + }; + Ok(Self { + items, + producer: graph.queues.word(memory::POINTERS[stage] + 0x40)?, + values, + config, + config_destination, + config_present: present, + announce: KickAnnounce::new( + config.qid.try_into()?, + if stage == 0 { + DataMaster::Tiling + } else { + DataMaster::Fragment + }, + timestamp, + )?, + announce_destination, + prefix: count as usize - 1, + prefix_end: (base + count - 1) % ITEM_RING_ENTRIES, + window, + }) + } + fn items(&self, range: core::ops::Range) { + for index in range { + self.items[index].store(self.values[index], Ordering::Relaxed); + } + } + fn config(&self) { + if !self.config_present { + return; + } + let source = (&self.config as *const QueueConfig).cast::(); + for index in 0..size_of::() / 8 { + // SAFETY: QueueConfig is packed integer storage with no padding; + // every source qword is read unaligned. The destination is a + // checked, aligned record owned by this unpublished ordinal. + unsafe { + self.config_destination + .add(index) + .write(source.add(index).read_unaligned()); + } + } + } + fn announce(&self) { + // SAFETY: The full record is inside the retained queue graph and is + // exclusively owned by this ordinal. Publication follows its copy. + unsafe { + self.announce_destination.write_unaligned(self.announce); + } + } + pub(super) fn publish_prefix(&self) { + self.items(0..self.prefix); + self.config(); + fence(Ordering::SeqCst); + self.producer.store(self.prefix_end, Ordering::Relaxed); + } + pub(super) fn publish_kick(&self) { + self.announce(); + self.items(self.prefix..self.prefix + 1); + fence(Ordering::SeqCst); + self.producer.store(self.window.target(), Ordering::Relaxed); + } + pub(super) fn publish_tiling( + &self, + event_slot: &AtomicU32, + next_slot: u32, + inner: &AtomicU32, + next_inner: u32, + ) { + self.items(0..self.prefix + 1); + self.config(); + self.announce(); + fence(Ordering::SeqCst); + self.producer.store(self.window.target(), Ordering::Relaxed); + event_slot.store(next_slot, Ordering::Relaxed); + inner.store(next_inner, Ordering::Relaxed); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs new file mode 100644 index 0000000000000..d0a2d566bdc8d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Paired render retirement evidence and bounded ownership of in-flight work. +//! Queue progress and execution timestamps precede the completion stamps which +//! authorize releasing command memory. A signalled error is not that witness. + +use crate::hw::t8140::queues; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +const DEPTH: usize = queues::RENDER_DEPTH as usize; +const MAX_GROUP_ITEMS: u32 = 4; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Cursors { + pub(crate) done: u32, + pub(crate) read: u32, + pub(crate) write: u32, +} + +impl Cursors { + fn valid(self) -> bool { + self.done <= self.write && self.read <= self.write + } + + fn idle(self) -> bool { + self.done == self.read && self.read == self.write + } +} + +/// Each engine may omit its configuration update independently. Preserve both +/// exact windows; neither target can be reconstructed from a render ordinal. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Window { + base: u32, + count: u32, + capacity: u32, +} + +impl Window { + pub(crate) fn new(base: u32, count: u32, capacity: u32) -> Result { + if base >= capacity || !(2..=MAX_GROUP_ITEMS).contains(&count) || count >= capacity { + return Err(EINVAL); + } + Ok(Self { + base, + count, + capacity, + }) + } + + pub(crate) fn target(self) -> u32 { + ((u64::from(self.base) + u64::from(self.count)) % u64::from(self.capacity)) as u32 + } + + fn normalize(self, cursors: Cursors) -> Result { + // Earlier and later groups can coexist in a snapshot. Anchor behind + // this group's base so a backwards observation cannot look like wrap. + let back = MAX_GROUP_ITEMS * (DEPTH as u32 - 1); + let limit = MAX_GROUP_ITEMS * DEPTH as u32 + back; + if limit >= self.capacity { + return Err(EINVAL); + } + let base = (u64::from(self.base) + u64::from(self.capacity) - u64::from(back)) + % u64::from(self.capacity); + let distance = |value: u32| -> Result { + if value >= self.capacity { + return Err(EIO); + } + let distance = + (u64::from(value) + u64::from(self.capacity) - base) % u64::from(self.capacity); + if distance > u64::from(limit) { + return Err(EIO); + } + Ok(distance as u32) + }; + Ok(Cursors { + done: distance(cursors.done)?, + read: distance(cursors.read)?, + write: distance(cursors.write)?, + }) + } + + fn progress(self, previous: Cursors, current: Cursors) -> Result<(bool, bool, bool)> { + let previous = self.normalize(previous)?; + let current = self.normalize(current)?; + let prefix = MAX_GROUP_ITEMS * (DEPTH as u32 - 1) + self.count; + if !previous.valid() + || !current.valid() + || current.done < previous.done + || current.read < previous.read + || current.write < previous.write + || prefix > current.write + { + return Err(EIO); + } + Ok(( + current.read >= prefix, + current.done >= prefix, + current.idle(), + )) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Ticket { + /// Full physical-pair ordinal, retained through owner switches and slot wrap. + pub(crate) ordinal: u64, + pub(crate) windows: [Window; 2], + pub(crate) previous: [Cursors; 2], +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Snapshot { + pub(crate) cursors: [Cursors; 2], + pub(crate) job_list_empty: bool, + /// The tiler writes the start and fragment processing writes the end. + pub(crate) pass: [u64; 2], +} + +impl Snapshot { + fn executed(self) -> bool { + self.pass[0] != 0 && self.pass[1] != 0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Observation { + pub(crate) accepted: bool, + pub(crate) retired: bool, + pub(crate) complete: bool, + pub(crate) idle: bool, +} + +pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result { + let ta = ticket.windows[0].progress(ticket.previous[0], snapshot.cursors[0])?; + let fragment = ticket.windows[1].progress(ticket.previous[1], snapshot.cursors[1])?; + let retired = ta.1 && fragment.1; + Ok(Observation { + accepted: ta.0 && fragment.0, + retired, + complete: retired && snapshot.executed(), + idle: ta.2 && fragment.2 && snapshot.job_list_empty, + }) +} + +/// Outstanding publications span less than half the 24-bit stamp sequence. +/// A zero stamp is valid when that sequence wraps. +pub(crate) const fn stamp_covers(stamp: u32, published: u64) -> bool { + let delta = stamp.wrapping_sub((published as u32).wrapping_shl(8)); + delta & 0xff == 0 && delta < 0x8000_0000 +} + +fn visible(ticket: &Ticket, stamps: [u32; 2]) -> Result { + let published = ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; + Ok(stamps + .into_iter() + .all(|stamp| stamp_covers(stamp, published))) +} + +/// Reads the graph owned by one physical pair. The owner validates the full +/// ordinal before reading reused descriptor/timestamp slots. Word accessors +/// perform aligned atomic loads; the observer supplies the acquire barriers. +pub(crate) trait Memory { + fn snapshot(&mut self, ordinal: u64) -> Result; + fn stamps(&mut self) -> Result<[u32; 2]>; + fn pass(&mut self, ordinal: u64) -> Result<[u64; 2]>; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Poll { + Retry, + Pending, + Complete([u64; 4]), +} + +/// Observe the ordered front. Individually sampled cursor words can straddle +/// a firmware update, so an invalid snapshot receives one immediate reread. +/// Shared work-state membership can include other engines and cannot gate a +/// successful individual member's completion. +pub(crate) fn poll(ticket: &mut Ticket, memory: &mut impl Memory) -> Result { + fence(Ordering::Acquire); + let mut snapshot = memory.snapshot(ticket.ordinal)?; + let observation = match observe(ticket, snapshot) { + Ok(observation) => observation, + Err(_) => { + fence(Ordering::Acquire); + snapshot = memory.snapshot(ticket.ordinal)?; + match observe(ticket, snapshot) { + Ok(observation) => observation, + Err(_) => return Ok(Poll::Retry), + } + } + }; + complete_poll(ticket, memory, snapshot, observation) +} + +/// Finish an observation sampled while its exact pair was exclusively borrowed. +/// The caller restores that pair and validates its epoch and front ticket first. +/// Completion stamps and the final timestamp refresh remain commit-time reads. +pub(crate) fn poll_observed( + ticket: &mut Ticket, + memory: &mut impl Memory, + snapshot: Snapshot, +) -> Result { + let observation = observe(ticket, snapshot)?; + complete_poll(ticket, memory, snapshot, observation) +} + +fn complete_poll( + ticket: &mut Ticket, + memory: &mut impl Memory, + snapshot: Snapshot, + observation: Observation, +) -> Result { + if observation.complete { + if !visible(ticket, memory.stamps()?)? { + return Ok(Poll::Pending); + } + // Refresh this generation's pass after observing the post-maintenance + // stamps. The pre-stamp snapshot alone cannot authorize export. + fence(Ordering::Acquire); + let [start, end] = memory.pass(ticket.ordinal)?; + return Ok(Poll::Complete([start, start, start, end])); + } + ticket.previous = snapshot.cursors; + Ok(Poll::Pending) +} + +/// Strict timeout settlement accepts only a nonexecuted, retired and unlinked +/// group. Drain reclamation may also settle executed groups, but retains both +/// ownership proofs and the exact post-maintenance stamps in either case. +pub(crate) fn abandoned( + ticket: &Ticket, + snapshot: Snapshot, + stamps: [u32; 2], + recovery_state: u32, + accept_executed: bool, +) -> Result { + let completion = observe(ticket, snapshot)?; + if !completion.retired || !snapshot.job_list_empty || (snapshot.executed() && !accept_executed) + { + return Ok(false); + } + Ok(visible(ticket, stamps)? && recovery_state == 0) +} + +/// Pair-wide proof, checked before and after servicing free-list events and +/// again after retiring all trackers. A proof from before event service must +/// not authorize releasing guards after a new recovery began. +pub(crate) fn drained( + published: u64, + stamps: [u32; 2], + free_list: [u64; 2], + recovery_state: u32, +) -> bool { + published == 0 + || (stamps + .into_iter() + .all(|stamp| stamp_covers(stamp, published)) + && free_list[0] == free_list[1] + && recovery_state == 0) +} + +/// Fixed storage keeps admission, rollback and retirement allocation-free. +/// Each element retains its packet and resource leases independently of fences. +pub(crate) struct Tracker { + entries: [Option; DEPTH], + head: usize, + len: usize, +} + +impl Tracker { + pub(crate) const fn new() -> Self { + Self { + entries: [const { None }; DEPTH], + head: 0, + len: 0, + } + } + + pub(crate) fn len(&self) -> usize { + self.len + } + pub(crate) fn room(&self) -> bool { + self.len < DEPTH + } + pub(crate) fn front(&self) -> Option<&T> { + if self.len == 0 { + None + } else { + self.entries[self.head].as_ref() + } + } + pub(crate) fn front_mut(&mut self) -> Option<&mut T> { + if self.len == 0 { + None + } else { + self.entries[self.head].as_mut() + } + } + pub(crate) fn back(&self) -> Option<&T> { + if self.len == 0 { + None + } else { + self.entries[(self.head + self.len - 1) % DEPTH].as_ref() + } + } + pub(crate) fn back_mut(&mut self) -> Option<&mut T> { + if self.len == 0 { + None + } else { + self.entries[(self.head + self.len - 1) % DEPTH].as_mut() + } + } + pub(crate) fn push(&mut self, value: T) -> core::result::Result<(), T> { + if !self.room() { + return Err(value); + } + self.entries[(self.head + self.len) % DEPTH] = Some(value); + self.len += 1; + Ok(()) + } + pub(crate) fn pop(&mut self) -> Option { + if self.len == 0 { + return None; + } + let value = self.entries[self.head].take(); + self.head = (self.head + 1) % DEPTH; + self.len -= 1; + value + } + + pub(crate) fn iter(&self) -> impl Iterator { + Iterator::chain( + self.entries[self.head..].iter(), + self.entries[..self.head].iter(), + ) + .take(self.len) + .filter_map(Option::as_ref) + } + pub(crate) fn iter_mut(&mut self) -> impl Iterator { + let (before, after) = self.entries.split_at_mut(self.head); + Iterator::chain(after.iter_mut(), before.iter_mut()) + .take(self.len) + .filter_map(Option::as_mut) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs new file mode 100644 index 0000000000000..4db471f254ec6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Fresh render graph cursor initialization at a retained physical ordinal. + +use super::{memory, Pair}; +use crate::g17::fw::queue::PointerBlock; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +impl Pair { + /// A new graph at a retained ordinal starts its variable item rings at zero. + /// The old graph has retired; firmware has not received this graph's queues. + pub(crate) fn rebase_fresh_graph(&mut self) -> Result { + if self.ordinal == 0 { + return Ok(()); + } + fence(Ordering::Acquire); + if self + .memory + .graph + .cursors()? + .into_iter() + .any(|cursor| cursor.done != 0 || cursor.read != 0 || cursor.write != 0) + { + return Err(EBUSY); + } + let mut blocks = [core::ptr::null::(); 2]; + for (block, offset) in blocks.iter_mut().zip(memory::POINTERS) { + let pointer = self + .memory + .graph + .queues + .pointer(offset, size_of::())?; + if pointer.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + *block = pointer.cast(); + } + for block in blocks { + // SAFETY: Both complete pointer blocks were checked above. This fresh + // graph is unpublished; only the live atomic cursors are accessed. + unsafe { &*block }.rebase(0); + } + fence(Ordering::SeqCst); + self.cursors = [0; 2]; + Ok(()) + } + +} diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs new file mode 100644 index 0000000000000..7e3db2f6dae78 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware restart handshake and per-VM failure classification. +//! +//! The device mutex serializes this state with publication. Preparations are closed and joined +//! without that mutex before `service`; no queue callback may signal a fence here. Queue +//! owners retain failed work until its retirement witness, a complete pair-drain witness, or +//! processor stop. A failed fence alone never releases mappings. + +mod attribution; +mod slot; +pub(crate) use attribution::Sources; +use attribution::Attribution; + +pub(super) fn slot_key(slot: usize) -> Option { + slot::key(slot).ok() +} + +use super::status::VmStatus; +use kernel::{prelude::*, sync::Arc}; + +const QIDS: u8 = 128; +const GUILTY: u8 = 1 << 1; +const CANCELLED: u8 = 1 << 2; +/// Retained valid entries, in firmware table order. +pub(super) const MAX_ENTRIES: usize = 64; +const MAX_GUILTY_VMS: usize = 8; +const RECOVERY_POLLS: usize = 500; + +/// One valid entry from the firmware's 256-entry recovery table. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(super) struct Entry { + pub(super) qid: u8, + pub(super) flags: u8, + pub(super) stamp: u32, +} + +impl Entry { + pub(super) const fn decode(flags: u32, stamp: u32) -> Option { + if flags & 1 == 0 { + return None; + } + Some(Self { + qid: ((flags >> 4) & 0x7f) as u8, + flags: (flags & 0xf) as u8, + stamp: stamp >> 8, + }) + } +} + +/// How an error completion affects its VM's sticky status and other jobs. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Class { + /// Only this work's fence fails; VM status and unrelated work remain usable. + Spared, + /// This VM owns guilty work. + Guilty, +} + +/// Queue operations under the device mutex. Completion signalling belongs to the caller after +/// that mutex is dropped. `oldest_spared` returns None if the QID has no in-flight owner. +pub(super) trait Queues { + fn render_scheduler_active(&self) -> bool; + fn render_pass_started(&self, qid: u8, stamp: u64) -> Option; + /// Copy every retained queue's live producer to its recovery target, then + /// issue a full barrier. A failed graph must not skip subsequent queues. + fn publish_recovery_targets(&self) -> Option; + fn oldest_spared(&self, qid: u8) -> Option; + fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)); + /// Classify every in-flight completion; return QIDs of render slots with spared work. + fn classify_work(&mut self, class: &dyn Fn(&Arc) -> Class) -> u128; + fn settle_blamed_render(&mut self, qid: u8, error: Error); + fn settle_cancelled_renders(&mut self, qids: u128); + fn rescan_compute(&mut self) -> Result; + fn release_retired_compute(&mut self); +} + +/// Live shared-memory operations. The handshake word uses aligned 32-bit accesses; transitions +/// check the expected value, issue a full barrier, store the next value, then another barrier. +pub(super) trait Memory { + fn recovery_state(&self) -> Result; + fn host_recovery(&self) -> Result; + fn fault_sources(&self) -> Result; + fn transition(&self, expected: u32, next: u32) -> Result; + fn clear_timestamps(&self) -> Result; + fn entries(&self, out: &mut [Entry; MAX_ENTRIES]) -> Result; + /// One millisecond wait; this is polled because the firmware sends no state-3 notification. + fn wait_tick(&self); +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +struct Request { + generation: u64, + blamed: Option, +} + +/// Generation accepted by the host and the coalesced request from either coprocessor. +pub(super) struct State { + accepted: u64, + pending: Option, +} + +/// Non-fatal post-recovery failures. The firmware handshake has already closed successfully. +pub(super) struct ServiceResult { + pub(super) scan_error: Option, + pub(super) target_error: Option, +} + +impl State { + pub(super) const fn new() -> Self { + Self { + accepted: 0, + pending: None, + } + } + pub(super) fn generation(&self) -> u64 { + self.accepted + } + pub(super) fn pending(&self) -> bool { + self.pending.is_some() + } + + /// Returns false for a consumed duplicate from an older generation, true when the event + /// drain must stop until the current request is serviced. The first blamed QID wins. + pub(super) fn request(&mut self, generation: u64, stamp: i32) -> Result { + if generation < self.accepted { + return Ok(false); + } + if generation != self.accepted { + return Err(EIO); + } + if stamp != -1 && !(0..i32::from(QIDS)).contains(&stamp) { + return Err(EIO); + } + let blamed = if (0..i32::from(QIDS)).contains(&stamp) { + Some(stamp as u8) + } else { + None + }; + if let Some(pending) = self.pending.as_mut() { + if pending.generation != generation { + return Err(EBUSY); + } + pending.blamed = pending.blamed.or(blamed); + } else { + self.pending = Some(Request { generation, blamed }); + } + Ok(true) + } +} + +/// The firmware owner implements memory and queue access without splitting its +/// mutable borrow or duplicating the authoritative restart generation. +pub(super) trait Host: Memory + Queues { + fn recovery(&self) -> &State; + fn recovery_mut(&mut self) -> &mut State; +} + +/// Performs one pending restart. The caller has closed preparation admission and joined all +/// builders off-lock. Work remains mapped throughout every callback, including on error. +pub(super) fn service( + host: &mut H, + crashed: impl Fn() -> bool, +) -> Result { + let request = host.recovery().pending.ok_or(EINVAL)?; + if host.recovery_state()? != 1 + || request.generation != host.recovery().accepted + || host.host_recovery()? != 0 + { + return Err(EIO); + } + host.recovery_mut().accepted = host.recovery().accepted.wrapping_add(1); + if request.generation == 0 && !host.render_scheduler_active() { + host.clear_timestamps()?; + } + let mut sources = host.fault_sources()?; + let target_error = host.publish_recovery_targets(); + host.transition(1, 2)?; + let mut recovered = false; + for _ in 0..RECOVERY_POLLS { + if host.recovery_state()? == 3 { + recovered = true; + break; + } + if crashed() { + return Err(EIO); + } + host.wait_tick(); + } + if !recovered { + return Err(ETIMEDOUT); + } + let mut entries = [Entry::default(); MAX_ENTRIES]; + // A failed table read leaves only the explicit blamed QID as evidence of guilt. + let count = host.entries(&mut entries).unwrap_or(0).min(MAX_ENTRIES); + host.transition(3, 0)?; + if host.recovery_state()? != 0 { + return Err(EIO); + } + host.recovery_mut().pending = None; + + let cancelled = classify(host, &entries[..count], request.blamed, &mut sources); + if let Some(qid) = request.blamed { + host.settle_blamed_render(qid, EIO); + } + if cancelled != 0 { + host.settle_cancelled_renders(cancelled); + } + let scan_error = host.rescan_compute().err(); + host.release_retired_compute(); + Ok(ServiceResult { scan_error, target_error }) +} + +/// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or +/// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work +/// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. +fn classify(queues: &mut Q, entries: &[Entry], blamed: Option, sources: &mut Sources) -> u128 { + let mut guilty_qids = blamed.map_or(0, |qid| 1u128 << qid); + let mut cancelled = 0u128; + for entry in entries { + if entry.flags & GUILTY != 0 { + guilty_qids |= 1u128 << entry.qid; + } + if entry.flags & CANCELLED != 0 { + cancelled |= 1u128 << entry.qid; + } + } + let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); + match attribution { + Attribution::Firmware => {} + Attribution::Queues(qids) => guilty_qids = qids, + Attribution::NoCulprit => guilty_qids = 0, + } + let no_culprit = attribution == Attribution::NoCulprit; + if guilty_qids == 0 && !no_culprit { + return 0; + } + let mut guilty: [Option>; MAX_GUILTY_VMS] = Default::default(); + let mut count = 0; + let mut collateral = 0u128; + let mut unresolved = false; + for qid in 0..QIDS { + if guilty_qids & (1u128 << qid) == 0 { + continue; + } + match queues.oldest_spared(qid) { + None => unresolved = true, + Some(true) => collateral |= 1u128 << qid, + Some(false) => { + let mut found = false; + queues.visit_owners(qid, &mut |status| { + found = true; + if guilty[..count] + .iter() + .flatten() + .any(|known| Arc::ptr_eq(known, status)) + { + return; + } + if let Some(slot) = guilty.get_mut(count) { + *slot = Some(status.clone()); + count += 1; + } else { + unresolved = true; + } + }); + unresolved |= !found; + } + } + } + if unresolved || (count == 0 && collateral == 0 && !no_culprit) { + return 0; + } + let spared = queues.classify_work(&|status| { + guilty[..count] + .iter() + .find_map(|known| { + known + .as_ref() + .filter(|known| Arc::ptr_eq(known, status)) + .map(|_| Class::Guilty) + }) + .unwrap_or(Class::Spared) + }); + cancelled & !(guilty_qids & !collateral) & spared +} diff --git a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs new file mode 100644 index 0000000000000..b0c42d4e3f7c1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Resolve ambiguous recovery blame using slot identity and render execution evidence. + +pub(super) const SLOTS: usize = 16; +const STAMP_MASK: u64 = (1 << 40) - 1; +const PAGE_FAULT: u32 = 3; + +#[derive(Clone, Copy, Default)] +pub(super) struct Source { + slot: u8, + qid: Option, + stamp: u64, + monitor: u32, + started: Option, +} + +pub(crate) struct Sources { + pub(crate) reason: u32, + mask: u32, + records: [Source; SLOTS], + count: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Attribution { + Firmware, + Queues(u128), + NoCulprit, +} + +impl Sources { + /// Firmware is halted for every selected key and progress-record read. + /// Missing keys stay unknown; they cannot establish execution by an owner. + pub(crate) fn sample(mask: u32, mut read: impl FnMut(usize) -> (u64, u32)) -> Self { + let mut sources = Self { + reason: 0, + mask, + records: [Source::default(); SLOTS], + count: 0, + }; + for slot in 0..SLOTS { + if mask & (1 << slot) == 0 { + continue; + } + let (key, monitor) = read(slot); + let qid = (key >> 40) & 0xff; + sources.records[sources.count] = Source { + slot: slot as u8, + qid: (key != 0 && qid < 128).then_some(qid as u8), + stamp: key & STAMP_MASK, + monitor, + started: None, + }; + sources.count += 1; + } + sources + } + + pub(super) fn attribute( + &mut self, + blamed: Option, + mut started: impl FnMut(u8, u64) -> Option, + ) -> Attribution { + for source in &mut self.records[..self.count] { + if let Some(qid) = source.qid { + source.started = started(qid, source.stamp); + } + } + if self.reason == PAGE_FAULT || self.mask.count_ones() < 2 { + return Attribution::Firmware; + } + let Some(blamed) = blamed else { + return Attribution::Firmware; + }; + let Some(source) = self.records[..self.count] + .iter() + .find(|source| source.qid == Some(blamed)) + else { + return Attribution::Firmware; + }; + if source.started != Some(false) || source.monitor != 0 { + return Attribution::Firmware; + } + let slot = source.slot; + let mut guilty = 0; + for source in &self.records[..self.count] { + if source.slot != slot && source.started == Some(true) { + if let Some(qid) = source.qid { + guilty |= 1u128 << qid; + } + } + } + if guilty == 0 { + Attribution::NoCulprit + } else { + Attribution::Queues(guilty) + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/recovery/slot.rs b/drivers/gpu/drm/asahi/g17/recovery/slot.rs new file mode 100644 index 0000000000000..307534708e922 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/recovery/slot.rs @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded read-only slot identity sampling while firmware is halted. + +use crate::{hw::t8140, mmu}; +use core::{ + ffi::c_void, + ptr::NonNull, + sync::atomic::{AtomicU64, Ordering}, +}; +use kernel::{bindings, prelude::*}; + +struct Mapping(NonNull); + +impl Drop for Mapping { + fn drop(&mut self) { + // SAFETY: This is the successful memremap result owned exclusively by this guard. + unsafe { bindings::memunmap(self.0.as_ptr()) }; + } +} + +pub(super) fn key(slot: usize) -> Result { + if slot >= super::attribution::SLOTS { + return Err(EINVAL); + } + let physical = t8140::recovery::SLOT_KEYS + slot as u64 * size_of::() as u64; + let offset = physical as usize & (mmu::UAT_PGSZ - 1); + // SAFETY: The fixed recovery-key range is firmware RAM, page aligned here + // for mapping. Only the selected aligned eight-byte word is read below. + let raw = unsafe { + bindings::memremap( + physical - offset as u64, + mmu::UAT_PGSZ, + bindings::MEMREMAP_WB as _, + ) + }; + let mapping = Mapping(NonNull::new(raw).ok_or(ENOMEM)?); + let ctr: u64; + // SAFETY: CTR_EL0 is readable at EL1. DminLine specifies the cache maintenance granule. + unsafe { + core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags)); + } + let line = 4usize << ((ctr >> 16) & 0xf); + if line > mmu::UAT_PGSZ { + return Err(EINVAL); + } + // SAFETY: The range and offset are eight-byte aligned and lie within the mapped page. + let pointer = unsafe { + mapping + .0 + .as_ptr() + .cast::() + .add(offset) + .cast::() + }; + let mut address = pointer as usize & !(line - 1); + let end = (pointer as usize) + .checked_add(size_of::()) + .ok_or(EOVERFLOW)?; + // SAFETY: The mapping stays live and is never written by the host. The + // invalidated cache line lies within its page; barriers complete the + // invalidation before the aligned load of the selected firmware-owned word. + unsafe { + core::arch::asm!("dsb osh", options(nostack, preserves_flags)); + while address < end { + core::arch::asm!("dc ivac, {address}", address = in(reg) address, + options(nostack, preserves_flags)); + address = address.checked_add(line).ok_or(EOVERFLOW)?; + } + core::arch::asm!("dsb osh", options(nostack, preserves_flags)); + Ok((*pointer).load(Ordering::Acquire)) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs new file mode 100644 index 0000000000000..e908ca3e08e23 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -0,0 +1,869 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device ownership of physical queues and off-lock construction reservations. +//! +//! The device mutex protects this registry. Logical queues may disappear while +//! their published graphs remain here; only exact retirement permits rebinding, +//! and only processor stop permits dropping installed firmware addresses. +//! A preparation lease keeps Shared::state populated: shutdown and loss join +//! every lease before taking the session, including builders asleep on changed. + +mod backend; +mod compute_control; +mod compute_replay; +mod compute_exit; +mod events; +mod render; +pub(super) mod teardown; + +use super::{ + buffer::{BufferIds, MetricsIds}, + context::{Context, QosIds}, + fw::queue::DataMaster, + kick, qos, + queue::{self, admission, compute}, +}; +use kernel::{prelude::*, sync::Arc}; + +const QIDS: usize = 128; +const FIRST_COMPUTE_QID: u8 = 4; +const RECOVERY_SETTLE_POLLS: usize = 200; + +/// A reservation remains identifiable while its backing is built without the +/// device mutex. Failed construction returns the slot, never its host identity. +#[derive(Clone, Copy)] +pub(super) struct ComputeReservation { + slot: usize, + pub(super) id: kick::Id, + owner: u64, +} + +struct ComputeEntry { + reservation: ComputeReservation, + queue: Option>, +} + +pub(super) struct Registry { + pub(super) qids: kick::QueueIds, + pub(super) buffers: BufferIds, + pub(super) accounting: qos::Accounting, + pub(super) metrics: Arc, + pub(super) qos_ids: Arc, + pub(super) admission: Arc, + render: render::RenderState, + identities: queue::Identities, + compute: KVec>, + pub(super) compute_building: bool, + compute_scheduler_building: bool, + pub(super) observations: compute::retirement::Batch, + worker_deferred: Option, + teardown: super::teardown::Pending, + exit_acks: compute::exit::Acks, +} + +impl Registry { + pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { + let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; + for _ in 0..QIDS { + compute.push(None, GFP_KERNEL)?; + } + let mut qids = kick::QueueIds::new(); + qids.reserve(1, DataMaster::Tiling, Some(0))?; + qids.reserve(2, DataMaster::Fragment, Some(1))?; + Ok(Self { + qids, + buffers: BufferIds::new(), + accounting: qos::Accounting::new(), + metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, + qos_ids: QosIds::new()?, + admission: admission::Pool::new()?, + render: render::RenderState::new(clusters, descriptor_flags)?, + identities: queue::Identities::new(), + compute, + compute_building: false, + compute_scheduler_building: false, + observations: compute::retirement::Batch::new()?, + worker_deferred: Some(DeferredBatch::worker()?), + teardown: super::teardown::Pending::new()?, + exit_acks: compute::exit::Acks::new(), + }) + } + + pub(super) fn compute(&self, owner: u64) -> Option<&compute::Queue> { + self.compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .find(|queue| queue.owner() == Some(owner)) + } + + pub(super) fn compute_mut(&mut self, owner: u64) -> Option<&mut compute::Queue> { + self.compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.owner() == Some(owner)) + } + + /// An installed idle queue keeps its original QID and backing. Mapping a + /// replacement context must succeed before its old binding is displaced. + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc) -> Result { + if let Some(queue) = self.compute(owner) { + return queue.matches(owner, queue.qid(), context); + } + let Some(queue) = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.reusable()) + else { + return Ok(false); + }; + queue.bind_owner(owner, context.clone())?; + Ok(true) + } + + pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { + if owner == 0 + || self.compute.iter().flatten().any(|entry| { + entry + .queue + .as_ref() + .map_or(entry.reservation.owner == owner, |queue| { + queue.owner() == Some(owner) + }) + }) + { + return Err(EINVAL); + } + let slot = self + .compute + .iter() + .position(Option::is_none) + .ok_or(ENOSPC)?; + let identity = self.identities.reserve(1)?.start; + let id = self + .qids + .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID))?; + let reservation = ComputeReservation { slot, id, owner }; + self.compute[slot] = Some(ComputeEntry { + reservation, + queue: None, + }); + Ok(reservation) + } + + fn reserved(&mut self, reservation: ComputeReservation) -> Result<&mut ComputeEntry> { + let entry = self + .compute + .get_mut(reservation.slot) + .and_then(Option::as_mut) + .ok_or(EIO)?; + if entry.reservation.id != reservation.id + || entry.reservation.owner != reservation.owner + || entry.queue.is_some() + { + return Err(EIO); + } + Ok(entry) + } + + /// The caller checked its preparation epoch and retained the same device + /// session before placing this graph where publication can find it. + pub(super) fn install_compute( + &mut self, + reservation: ComputeReservation, + queue: &mut Option>, + ) -> Result { + let built = queue.as_ref().ok_or(EINVAL)?; + if built.qid() != reservation.id.qid() || built.owner() != Some(reservation.owner) { + return Err(EIO); + } + let entry = self.reserved(reservation)?; + entry.queue = queue.take(); + Ok(()) + } + + pub(super) fn cancel_compute(&mut self, reservation: ComputeReservation) -> Result { + self.reserved(reservation)?; + self.qids.cancel_unpublished(reservation.id)?; + self.compute[reservation.slot] = None; + Ok(()) + } + + pub(super) fn release_compute(&mut self, owner: u64) -> Result { + let Some(slot) = self.compute.iter().position(|entry| { + entry + .as_ref() + .and_then(|entry| entry.queue.as_ref()) + .is_some_and(|queue| queue.owner() == Some(owner)) + }) else { + return Ok(()); + }; + let entry = self.compute[slot].as_mut().ok_or(EIO)?; + if entry.queue.as_mut().ok_or(EIO)?.release_owner()? { + self.qids.cancel_unpublished(entry.reservation.id)?; + self.compute[slot] = None; + } + Ok(()) + } +} + +/// An immutable logical owner shared by its two scheduler entities. Installed +/// physical graphs retain the context independently of this frontend lifetime. +pub(super) struct Backend { + shared: Arc, + pub(super) context: Arc, + owner: u64, + backlog: core::sync::atomic::AtomicU32, +} + +impl Backend { + pub(super) fn new( + shared: &Arc, + vm: &crate::mmu::Vm, + owner: u64, + ) -> Result<(Arc, admission::QueueSlot)> { + let lease = shared.preparations.enter(true)?; + let (uat, pool, qos, execution, slot) = { + let mut state = shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if shared.crashed.load(core::sync::atomic::Ordering::Acquire) { + return Err(EIO); + } + let slot = firmware.queues.admission.reserve_queue()?; + let execution = firmware.uat.new_execution_context(vm)?; + ( + firmware.uat.clone(), + firmware.pool.clone(), + firmware.queues.qos_ids.clone(), + execution, + slot, + ) + }; + let alloc = super::object::Allocator { + dev: &shared.dev, + uat: &uat, + }; + let owner_pid = + u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; + let context = Arc::new( + Context::new(&alloc, &pool, &qos, execution, owner_pid)?, + GFP_KERNEL, + )?; + let state = shared.state.lock(); + if !lease.is_current() || state.is_none() { + return Err(EAGAIN); + } + drop(state); + Ok(( + Arc::new( + Self { + shared: shared.clone(), + context, + owner, + backlog: core::sync::atomic::AtomicU32::new(0), + }, + GFP_KERNEL, + )?, + slot, + )) + } + + /// The shared aperture has exactly one builder. Other queues sleep without + /// holding the device mutex; recovery joins its lease before closing roots. + fn shared_compute(&self) -> Result { + loop { + let lease = self.shared.preparations.enter(false)?; + let uat = { + let mut state = self.shared.state.lock(); + loop { + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if self + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(EIO); + } + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.init.has_compute_shared() { + return firmware.init.cache_compute_vm(self.context.vm()); + } + if !firmware.queues.compute_building { + firmware.queues.compute_building = true; + break firmware.uat.clone(); + } + self.shared.changed.wait(&mut state); + } + }; + let alloc = super::object::Allocator { + dev: &self.shared.dev, + uat: &uat, + }; + let mut built = Some(super::initdata::ComputeShared::new(&alloc)); + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + Err(EAGAIN) + } else { + built + .take() + .ok_or(EIO)? + .and_then(|built| firmware.init.install_compute_shared(built)) + .and_then(|()| firmware.init.cache_compute_vm(self.context.vm())) + } + }; + drop(built); + { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .compute_building = false; + } + self.shared.changed.notify_all(); + drop(lease); + if result != Err(EAGAIN) { + return result; + } + } + } + + /// Uses the queue builder's existing lease. Entering another closed epoch + /// here would make recovery wait for the very builder blocked on reopening. + fn compute_scheduler( + &self, + alloc: &super::object::Allocator<'_>, + lease: &super::preparation::Lease, + ) -> Result { + { + let mut state = self.shared.state.lock(); + loop { + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.init.has_compute_scheduler() { + return Ok(()); + } + if !firmware.queues.compute_scheduler_building { + firmware.queues.compute_scheduler_building = true; + break; + } + self.shared.changed.wait(&mut state); + } + } + let mut built = Some(super::initdata::ComputeScheduler::new(alloc)); + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + Err(EAGAIN) + } else { + built + .take() + .ok_or(EIO)? + .and_then(|built| firmware.init.install_compute_scheduler(built)) + } + }; + drop(built); + { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .compute_scheduler_building = false; + } + self.shared.changed.notify_all(); + result + } + + fn build_compute(&self) -> Result { + loop { + match self.shared_compute() { + Err(error) if error == EAGAIN => continue, + result => { + result?; + break; + } + } + } + loop { + let lease = self.shared.preparations.enter(false)?; + let (uat, reservation) = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if self + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(EIO); + } + if firmware.queues.rebind_compute(self.owner, &self.context)? { + return Ok(()); + } + let reservation = firmware.queues.reserve_compute(self.owner)?; + (firmware.uat.clone(), reservation) + }; + let alloc = super::object::Allocator { + dev: &self.shared.dev, + uat: &uat, + }; + let built = (|| { + let pool = compute::PoolBacking::new(&alloc, reservation.id)?; + { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + firmware.queues.reserved(reservation)?; + let (slot, page_list) = pool.pool_descriptor(); + firmware.init.initialize_compute_pool(slot, page_list)?; + } + let backing = compute::Backing::new(&alloc, pool)?; + // The global scheduler is created after pool/kick backing and + // retained even if this queue's later descriptor graph fails. + self.compute_scheduler(&alloc, &lease)?; + let queue = compute::Queue::new(&alloc, backing, self.owner, self.context.clone())?; + Ok(KBox::new(queue, GFP_KERNEL)?) + })(); + let (mut built, build_error) = match built { + Ok(queue) => (Some(queue), None), + Err(error) => (None, Some(error)), + }; + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + Err(EAGAIN) + } else if let Some(error) = build_error { + Err(error) + } else { + firmware.queues.install_compute(reservation, &mut built) + } + }; + // The old aliases must disappear before a waiting constructor can + // reuse the QID and its queue-local free-list identity. Keep the + // preparation lease until both mappings and reservation are gone. + drop(built); + if result.is_err() { + let mut state = self.shared.state.lock(); + (*state) + .as_mut() + .ok_or(ENODEV)? + .queues + .cancel_compute(reservation)?; + } + drop(lease); + if result != Err(EAGAIN) { + return result; + } + } + } +} + +/// One device transaction can fail each retained kick once and then retire it. +/// The physical namespace bounds the batch independently of open logical queues. +/// Collection failure leaves the engine's Active owner intact for a stopped retry. +pub(super) struct DeferredBatch { + values: KVVec, +} +impl DeferredBatch { + const DEPTH: usize = if crate::hw::t8140::queues::RENDER_DEPTH as usize + > crate::hw::t8140::queues::COMPUTE_DEPTH + { + crate::hw::t8140::queues::RENDER_DEPTH as usize + } else { + crate::hw::t8140::queues::COMPUTE_DEPTH + }; + + /// Reused by the sole event work item; no allocation on a completion pass. + fn worker() -> Result { + Self::with_capacity(2 * QIDS * Self::DEPTH) + } + /// One publication can quarantine only its selected physical queue/pair. + /// VM-wide settlement is a separate transaction before publishing its error. + fn publication() -> Result { + Self::with_capacity(2 * Self::DEPTH) + } + fn with_capacity(capacity: usize) -> Result { + Ok(Self { + values: KVVec::with_capacity(capacity, GFP_KERNEL)?, + }) + } + pub(super) fn push(&mut self, value: super::completion::Deferred) -> Result { + use super::completion::Deferred; + // Publishers may acquire the device mutex as soon as this pass drops + // it. Expose terminal VM status now, before off-lock fence callbacks. + // Recovery-spared and cancelled commands never poison their VM. + if let Deferred::FailedOwned(completion, error) = &value { + if *error != ECANCELED && !completion.spared() { + completion.status().record(*error); + } + } + self.values.push_within_capacity(value).map_err(|_| EIO) + } + /// Called only after releasing the device mutex. + pub(super) fn finish(&mut self) { + for value in self.values.drain_all() { + value.finish(); + } + } +} + +struct ComputeHost<'a> { + init: &'a super::initdata::InitData, + primary: &'a mut super::Coprocessor, + effort: &'a mut super::power::Effort, + qids: &'a mut kick::QueueIds, + accounting: &'a mut qos::Accounting, + generation: u64, +} +impl compute::Host for ComputeHost<'_> { + fn epoch(&self) -> Result<(u64, u32)> { + Ok(( + self.generation, + super::recovery::Memory::recovery_state(self.init)?, + )) + } + fn prepare_compute_shared(&mut self) -> Result { + self.init.activate_compute() + } + fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result { + self.accounting.publish( + &qos::View::new(self.init.qos()?)?, + owner, + scheduler, + qos::clock, + ) + } + fn qos_cancel(&mut self, publication: qos::Publication) -> Result { + self.accounting + .cancel(&qos::View::new(self.init.qos()?)?, publication) + } + fn publish_qid(&mut self, id: kick::Id) -> Result { + self.qids.publish(id) + } + fn publish_outer(&mut self, slot: &super::fw::channels::WorkSlot) -> Result { + self.init.publish_work(slot).map(|_| ()) + } + fn notify(&mut self, message: u64) -> Result { + self.primary.notify(message) + } + fn note_submission(&mut self) -> Result { + self.effort.submit(); + super::Firmware::apply_shared_effort(self.init, self.effort, super::power::FULL_EFFORT) + } +} + +impl super::Firmware { + fn publish_compute( + &mut self, + owner: u64, + packet: Arc, + dependencies: &[super::fw::kick::KickDependency], + deferred: &mut DeferredBatch, + ) -> Result { + if self.recovery.pending() { + return Err(EAGAIN); + } + let queues = &mut self.queues; + let queue = queues + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.owner() == Some(owner)) + .ok_or(EIO)?; + let mut host = ComputeHost { + init: &self.init, + primary: &mut self.primary, + effort: &mut self.effort, + qids: &mut queues.qids, + accounting: &mut queues.accounting, + generation: self.recovery.generation(), + }; + queue.publish(packet, dependencies, &mut host, &mut |event| { + deferred.push(event) + }) + } +} + +impl Registry { + fn release_compute_witness( + &mut self, + init: &super::initdata::InitData, + owner: u64, + deferred: &mut DeferredBatch, + ) -> Result { + let Some(queue) = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.owner() == Some(owner)) + else { + return Ok(false); + }; + if !queue.spared_pending() { + return Ok(false); + } + let view = qos::View::new(init.qos()?)?; + queue.release_witnessed( + super::recovery::Memory::recovery_state(init)?, + |owner| self.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + Ok(queue.spared_pending()) + } + + fn fail_compute_vm( + &mut self, + status: &Arc, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + for queue in self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + if queue.in_flight() && Arc::ptr_eq(queue.context().status(), status) { + queue.quarantine(error, &mut |event| deferred.push(event))?; + } + } + Ok(()) + } +} + +impl super::Firmware { + /// Latch one shared snapshot for all selected queues before settling any VM. + /// A later queue's exact success in this batch wins over a peer's failure. + fn scan_compute(&mut self, masks: Option<[u64; 2]>, deferred: &mut DeferredBatch) -> Result { + let state = super::recovery::Memory::recovery_state(&self.init)?; + let generation = self.recovery.generation(); + let registry = &mut self.queues; + registry.observations.clear(); + let mut requested = false; + for queue in registry + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + { + requested |= queue.request_observations(&mut registry.observations, masks)?; + } + if requested { + self.init + .scan_compute_completions(|record| registry.observations.fold(record))?; + } + let view = qos::View::new(self.init.qos()?)?; + for queue in registry + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + queue.latch_observations(&mut registry.observations, masks)?; + queue.retire( + masks, + generation, + state, + |owner| registry.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + } + // All compute observations in this batch are latched before a VM + // failure can settle its render peers. Exact successes win first. + for index in 0..self.queues.compute.len() { + let terminal = self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .and_then(compute::Queue::take_terminal); + if let Some((status, error)) = terminal { + status.record(error); + self.settle_render_vm(&status, error, deferred)?; + self.queues.fail_compute_vm(&status, error, deferred)?; + } + } + let view = qos::View::new(self.init.qos()?)?; + let registry = &mut self.queues; + for queue in registry + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + queue.release_witnessed( + state, + |owner| registry.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + } + Ok(()) + } +} + +impl Backend { + fn ensure_compute(&self) -> Result { + use kernel::time::{delay::fsleep, Delta}; + const SPARED_WAIT_MS: usize = 200; + let mut deferred = DeferredBatch::publication()?; + for _ in 0..SPARED_WAIT_MS { + let pending = (|| { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + firmware.replay_compute(&mut deferred)?; + firmware + .queues + .release_compute_witness(&firmware.init, self.owner, &mut deferred) + })(); + deferred.finish(); + let pending = pending?; + if !pending { + break; + } + // Other owners continue submitting while this owner gives its + // queue-local drain witness time to arrive. + fsleep(Delta::from_millis(1)); + } + self.build_compute() + } + + /// Called after taking this packet's publication claim. All waits and work + /// state allocation precede the device mutex; only the checked epoch commits. + fn compute_packet( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + packet.check_dependencies()?; + let mut recovery_polls = 0; + loop { + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let lease = self.shared.preparations.enter(false)?; + packet.completion.work_state().acquire()?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + drop(state); + continue; + } + if packet.cancel_requested() { + return Err(ECANCELED); + } + if self + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(EIO); + } + let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; + if firmware.recovery.pending() || recovery_state != 0 { + if recovery_polls == RECOVERY_SETTLE_POLLS { + return Err(EBUSY); + } + recovery_polls += 1; + drop(state); + drop(lease); + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; + } + firmware.replay_compute(deferred)?; + let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + let result = firmware.publish_compute( + self.owner, + packet.clone(), + dependencies.as_slice(), + deferred, + ); + if matches!(result, Err(EBUSY) | Err(EAGAIN)) + && !packet.is_published() + && (firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0) + { + if recovery_polls == RECOVERY_SETTLE_POLLS { + return Err(EBUSY); + } + recovery_polls += 1; + drop(state); + drop(lease); + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; + } + return result; + } + } +} + +impl super::Firmware { + /// Both processors are stopped and their owner detached before this call. + /// Error fences release their command mappings without the device mutex + /// or an allocation-dependent collector. + fn finish_stopped_queues(&mut self) { + self.stop_render(); + for queue in self + .queues + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + let result = queue.stopped(&mut |event| { + event.finish(); + Ok(()) + }); + if let Err(error) = result { + dev_err!( + self.primary.state.shared.dev.as_ref(), + "Could not settle stopped compute queue: {:?}\n", + error + ); + } + } + self.queues + .admission + .release_retained_owners_after_processor_stop(); + } +} + +impl super::Shared { + /// Joins construction and serializes the stop writes with detachment. + /// After installation, seeing no owner proves that an earlier stop caller + /// has already stopped both processors. Callbacks and destruction follow + /// outside the device mutex. + pub(super) fn stop_queues(&self) { + self.preparations.wait_drained(); + let firmware = { + let mut state = self.state.lock(); + if let Some(firmware) = (*state).as_deref_mut() { + firmware.secondary.stop(); + firmware.primary.stop(); + } + state.take() + }; + if let Some(mut firmware) = firmware { + firmware.finish_stopped_queues(); + } + } +} + +/// One monotonic domain for recovery grace, idle effort and control retirement. +fn now_ns() -> u64 { + ::ktime_get() as u64 +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs new file mode 100644 index 0000000000000..f1ae1ffe80783 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scheduler callbacks and the off-lock fence boundary of a logical queue. + +use super::{Backend, DeferredBatch}; +use crate::g17::{ + completion::Deferred, + job::{self, Engine, Packet}, +}; +use core::sync::atomic::Ordering; +use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; + +impl Backend { + pub(super) fn release_backlog(&self, packet: &Packet) { + if packet.take_backlog() { + self.backlog.fetch_sub(1, Ordering::AcqRel); + } + } + + fn stop_after_error(&self) { + self.shared.lose_device_quietly(); + self.shared.stop_queues(); + } + + /// Complete all failure fanout before signalling this packet's unpublished + /// error. Visible failures keep the engine's retained owner through a witness. + pub(super) fn finish_publication( + &self, + packet: &Arc, + result: Result, + deferred: &mut DeferredBatch, + ) -> Result { + if let Err(error) = result { + if [EIO, ETIMEDOUT, ENODEV].contains(&error) || packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + deferred.finish(); + if !packet.is_published() { + packet.completion.fail_unpublished(error); + } + Err(error) + } else { + deferred.finish(); + Ok(packet.completion.fence()) + } + } + + fn release_queue(&self) -> Result { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + // Retain a published scheduler before a never-kicked physical graph is + // removed. The teardown entry owns its root until control consumption. + firmware + .queues + .teardown + .close(&self.context, super::now_ns())?; + firmware.queues.release_compute(self.owner)?; + firmware.queues.release_render(self.owner)?; + firmware.queues.admission.release_owner(self.owner); + drop(state); + self.shared.queue_events(); + Ok(()) + } +} + +impl job::Backend for Backend { + fn ensure_compute(&self) -> Result { + Backend::ensure_compute(self) + } + + fn fail_vm(&self, error: Error) { + self.context.status().record(error); + let mut deferred = match DeferredBatch::worker() { + Ok(batch) => batch, + Err(_) => { + self.stop_after_error(); + return; + } + }; + let result = (|| { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + // Final observations win over this VM's terminal fanout. + firmware.scan_compute(None, &mut deferred)?; + firmware.settle_render_vm(self.context.status(), error, &mut deferred)?; + firmware + .queues + .fail_compute_vm(self.context.status(), error, &mut deferred) + })(); + deferred.finish(); + if result.is_err() { + self.stop_after_error(); + } + } + + fn release_owner(&self) { + if self.release_queue().is_err() { + self.stop_after_error(); + } + } + + /// Firmware kick parents order the same engine. Explicit cross-engine and + /// input dependencies were already attached to the DRM scheduled job. + fn prepare(&self, _packet: &Packet) -> Option { + None + } + + fn publish(&self, packet: Arc) -> Result { + if let Some(result) = packet.early_result() { + return result.map(|()| packet.completion.fence()); + } + let mut deferred = match DeferredBatch::publication() { + Ok(batch) => batch, + Err(error) => { + self.release_backlog(&packet); + if packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + packet.completion.fail_unpublished(error); + return Err(error); + } + }; + let result = if !packet.begin_publication() { + Err(ECANCELED) + } else if packet.completion.status().get() != 0 { + Err(EIO) + } else { + match packet.engine() { + Engine::Compute => self.compute_packet(&packet, &mut deferred), + Engine::Render => self.render_packet(&packet, &mut deferred), + } + }; + self.release_backlog(&packet); + self.finish_publication(&packet, result, &mut deferred) + } + + fn timed_out(&self, packet: Arc) -> sched::Status { + self.release_backlog(&packet); + let result = match packet.engine() { + Engine::Compute => self.timeout_compute(&packet), + Engine::Render => self.timeout_render(&packet), + }; + if result.is_err() { + self.stop_after_error(); + } + sched::Status::NoHang + } + + fn cancel(&self, packet: Arc) { + self.release_backlog(&packet); + if packet.request_cancel() { + Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); + return; + } + let result = match packet.engine() { + Engine::Compute => self.cancel_compute(&packet), + Engine::Render => self.cancel_render(&packet), + }; + if result.is_err() { + self.stop_after_error(); + } + } + + fn defer_timeout(&self, packet: &Arc) -> bool { + if packet.engine() != Engine::Render { + return false; + } + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .is_some_and(|firmware| firmware.queues.defer_render_timeout(packet, ETIMEDOUT)) + } + + fn try_early(&self, packet: &Arc) -> bool { + self.try_render_early(packet) + } + + fn count_backlog(&self, packet: &Packet) { + packet.count_backlog(); + self.backlog.fetch_add(1, Ordering::AcqRel); + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs new file mode 100644 index 0000000000000..610643ef6e379 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scheduler cancellation and final retirement checks for retained compute work. + +use super::{Backend, DeferredBatch}; +use crate::g17::job::{self, Packet}; +use kernel::{prelude::*, sync::Arc}; + +impl Backend { + /// The caller owns cancellation of a packet that never claimed publication. + /// A missing retained packet may already have a healthy callback waiting off-lock. + pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { + let mut deferred = DeferredBatch::worker()?; + let result = (|| { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + // Teardown owns the detached queues and will settle their callbacks. + return Ok(None); + }; + firmware.scan_compute(None, &mut deferred)?; + let Some(queue) = firmware.queues.compute_mut(self.owner) else { + return Ok(None); + }; + if !queue.owns_packet(packet) { + return Ok(None); + } + queue.timeout(&mut |event| deferred.push(event)) + })(); + if result.as_ref().is_ok_and(|status| status.is_some()) { + // VM status and peer settlement precede signalling this timeout. The backend + // owns the same logical VM as the retained packet matched under the mutex. + ::fail_vm(self, ETIMEDOUT); + } + deferred.finish(); + result.map(|_| ()) + } + + /// Cancel only a scheduler waiter. Visible mappings stay pinned until their witness; + /// cancellation neither quarantines hardware admission nor fans out a VM failure. + pub(super) fn cancel_compute(&self, packet: &Arc) -> Result { + let mut deferred = DeferredBatch::worker()?; + let result = (|| { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + firmware.scan_compute(None, &mut deferred)?; + if let Some(queue) = firmware.queues.compute_mut(self.owner) { + queue.cancel(packet, &mut |event| deferred.push(event))?; + } + Ok(()) + })(); + deferred.finish(); + result + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs new file mode 100644 index 0000000000000..0643da70d72ca --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Process-exit controls for closed contexts whose compute commands remain owned. + +use super::{now_ns, DeferredBatch}; +use crate::g17::{ + channel::Rings, + context::Context, + fw::channels::{ContextKill, ControlRecord}, + queue::compute::exit, + recovery::Memory as _, + Firmware, MSG_CONTROL_NOTIFY, +}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +impl Firmware { + pub(in crate::g17) fn service_compute_exit(&mut self, deferred: &mut DeferredBatch) -> Result { + let recovery_closed = self.init.recovery_state()? == 0; + for entry in self.queues.compute.iter_mut().flatten() { + if let Some(queue) = entry.queue.as_deref_mut() { + queue.settle_exit(&self.queues.exit_acks, recovery_closed, &mut |event| { + deferred.push(event) + })?; + } + } + for cookie in self.queues.exit_acks.snapshot() { + if cookie != 0 + && !self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.exit_cookie() == Some(cookie)) + { + self.queues.exit_acks.forget(cookie); + } + } + if self.recovery.pending() || !recovery_closed { + return Ok(()); + } + let now = now_ns(); + for index in 0..self.queues.teardown.len() { + let Some((context, age)) = self.queues.teardown.closed(index, now) else { + continue; + }; + if !self.queues.render.teardown_idle(&context)? + || !self.queues.render.teardown_pools_released(&context) + { + continue; + } + if let Err(error) = self.publish_compute_exit(&context, age, now) { + dev_err!( + self.primary.state.shared.dev.as_ref(), + "Scheduler {} exit request failed: {:?}\n", + context.id(), + error + ); + } + } + Ok(()) + } + + fn publish_compute_exit(&mut self, context: &Arc, age: u64, now: u64) -> Result { + let Some(index) = self.queues.compute.iter().position(|entry| { + entry + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + .is_some_and(|queue| queue.exit_candidate(context, age)) + }) else { + return Ok(()); + }; + if self.render_control_backpressured() { + return Ok(()); + } + let queue = self.queues.compute[index] + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + .ok_or(EIO)?; + let Some(owner) = queue.exit_owner(context) else { + return Ok(()); + }; + let target = queue.qid(); + let (mut own, mut foreign, slots) = self.queues.render.exit_scope(context); + for entry in self.queues.compute.iter().flatten() { + if let Some(queue) = entry.queue.as_deref() { + let bit = 1u128 << queue.qid(); + if Arc::ptr_eq(queue.context(), context) { + own |= bit; + } else { + foreign |= bit; + } + } + } + let state = exit::State::read(context)?; + let Some(qids) = state.scope(target, own, foreign) else { + return Ok(()); + }; + let mut hold = 0u64; + for slot in 0..crate::hw::t8140::queues::RENDER_SLOTS as u8 { + if slots & (1u64 << slot) != 0 && self.queues.render.qids_for_slot(slot) & qids != 0 { + hold |= 1u64 << slot; + } + } + if hold != 0 + && !self + .queues + .admission + .hold_render_slots_for_exit_kill(owner, hold) + { + return Ok(()); + } + let cookie = exit::next_cookie(); + let record = ControlRecord::ContextKill(ContextKill::new(context.scheduler_va(), cookie)); + if let Err(error) = Rings::publish_control(&self.init, &record) { + self.queues.admission.release_exit_kill_hold(hold); + return if error == EAGAIN { Ok(()) } else { Err(error) }; + } + // Publication is irrevocable even if its doorbell fails. Keep both + // the killed graph and the held render identities until processor stop. + self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)? + .record_exit(cookie, now); + fence(Ordering::SeqCst); + if let Err(error) = self.primary.notify(MSG_CONTROL_NOTIFY) { + dev_warn!( + self.primary.state.shared.dev.as_ref(), + "Scheduler {} exit announcement failed: {:?}\n", + context.id(), + error + ); + } + Ok(()) + } + + pub(in crate::g17) fn compute_exit_polling(&self, now: u64) -> bool { + if self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.exit_polling(now)) + { + return true; + } + (0..self.queues.teardown.len()).any(|index| { + self.queues + .teardown + .closed(index, now) + .is_some_and(|(context, age)| { + age < exit::POLL_NS + && self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.exit_candidate(&context, u64::MAX)) + }) + }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs new file mode 100644 index 0000000000000..17f228b9d1f04 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Ordered republication of witnessed, unexecuted compute commands. + +use super::DeferredBatch; +use crate::g17::{completion::Deferred, qos, recovery, Firmware}; +use kernel::prelude::*; + +impl Firmware { + /// Called only from an outer worker or client submission, never from event + /// dispatch or a recovery callback nested within publication. All candidate + /// queues are observed before any retained packet is published again. + pub(in crate::g17) fn replay_compute(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.pending() { + return Ok(()); + } + let state = recovery::Memory::recovery_state(&self.init)?; + if state != 0 { + return Ok(()); + } + let view = qos::View::new(self.init.qos()?)?; + for entry in self.queues.compute.iter_mut().flatten() { + if let Some(queue) = entry.queue.as_deref_mut() { + queue.prepare_replays( + state, + |owner| self.queues.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + } + } + for index in 0..self.queues.compute.len() { + loop { + if self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0 { + return Ok(()); + } + let Some(queue) = self.queues.compute[index] + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + else { + break; + }; + let Some(packet) = queue.replay_front() else { + break; + }; + let owner = queue.owner(); + + let result = (|| { + self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)? + .prepare_replay(&packet)?; + let owner = owner.ok_or(EIO)?; + packet.check_dependencies()?; + let dependencies = self.queues.render_prefix(owner, &packet)?; + self.publish_compute(owner, packet.clone(), dependencies.as_slice(), deferred) + })(); + let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) + && (self.recovery.pending() + || recovery::Memory::recovery_state(&self.init)? != 0); + let queue = self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)?; + if let Err(error) = result { + if recovery_wait && !queue.owns_packet(&packet) { + // Keep this prepared attempt and its one timeout credit + // while the outer worker closes the new recovery. + return Ok(()); + } + queue.fail_replays(error); + if !queue.owns_packet(&packet) { + deferred + .push(Deferred::Retired(packet.completion.clone(), Err(ENODATA)))?; + } + } + queue.finish_replay(&packet)?; + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs new file mode 100644 index 0000000000000..e9fc977d623b8 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Event and restart callbacks borrow the complete firmware owner. The reusable +//! completion collector belongs to the worker and is drained outside its mutex. + +use super::DeferredBatch; +use crate::g17::{ + channel, event, + fw::channels::{Event, FreeListCompletion, TvbGrowRequest}, + recovery, + status::VmStatus, + Firmware, Role, +}; +use kernel::{prelude::*, sync::Arc}; + +struct Service<'a> { + firmware: &'a mut Firmware, + deferred: &'a mut DeferredBatch, + render_masks: [u64; 2], + settlement_error: Option, +} + +impl event::Transport for Service<'_> { + fn trace(&mut self, role: Role) { + let firmware = &mut self.firmware; + firmware + .rings + .drain_trace(&firmware.primary.state.shared.dev, &firmware.init, role); + } + fn statistics(&mut self, role: Role) { + let firmware = &mut self.firmware; + firmware + .rings + .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); + } + fn drain( + &mut self, + role: Role, + mut consume: impl FnMut(&mut Self, Event) -> Result, + ) -> Result { + let mut events = channel::Events::new(role); + while let Some(event) = + events.next(&self.firmware.primary.state.shared.dev, &self.firmware.init)? + { + if !consume(self, event)? { + break; + } + } + Ok(()) + } + fn recovery_state(&self) -> Result { + recovery::Memory::recovery_state(&self.firmware.init) + } +} + +impl event::Host for Service<'_> { + fn recovery(&self) -> &recovery::State { + &self.firmware.recovery + } + fn recovery_mut(&mut self) -> &mut recovery::State { + &mut self.firmware.recovery + } +} + +impl event::Queues for Service<'_> { + fn context_killed(&mut self, cookie: u64) { + self.firmware.queues.exit_acks.note(cookie); + } + fn flush_tvb_reply(&mut self) -> Result { + self.firmware.flush_tvb_reply() + } + fn flush_freelist_release(&mut self) -> Result { + self.firmware.flush_freelist_release() + } + fn observe_render_releases(&mut self) -> Result { + self.firmware.observe_render_releases() + } + fn control_backpressured(&self) -> bool { + self.firmware.render_control_backpressured() + } + fn queue_tvb_grow(&mut self, request: TvbGrowRequest) -> Result { + self.firmware.queue_tvb_grow(request) + } + fn complete_freelist_grow(&mut self, completion: FreeListCompletion) -> Result { + self.firmware.complete_freelist_grow(completion) + } + fn retain_render_completions(&mut self, masks: [u64; 2]) { + // Inline timeout and drain callers may discard the worker hints. The + // queue owner retains stamps until its borrowed pair returns to scan. + self.firmware.queues.retain_render_completions(masks); + self.render_masks[0] |= masks[0]; + self.render_masks[1] |= masks[1]; + } + fn complete_compute(&mut self, masks: [u64; 2]) -> Result { + self.firmware.scan_compute(Some(masks), self.deferred) + } +} + +impl recovery::Host for Service<'_> { + fn recovery(&self) -> &recovery::State { + &self.firmware.recovery + } + fn recovery_mut(&mut self) -> &mut recovery::State { + &mut self.firmware.recovery + } +} + +impl recovery::Memory for Service<'_> { + fn recovery_state(&self) -> Result { + self.firmware.init.recovery_state() + } + fn host_recovery(&self) -> Result { + self.firmware.init.host_recovery() + } + fn fault_sources(&self) -> Result { + self.firmware.init.fault_sources() + } + + fn transition(&self, expected: u32, next: u32) -> Result { + self.firmware.init.transition(expected, next) + } + fn clear_timestamps(&self) -> Result { + self.firmware.init.clear_timestamps() + } + fn entries(&self, out: &mut [recovery::Entry; recovery::MAX_ENTRIES]) -> Result { + self.firmware.init.entries(out) + } + fn wait_tick(&self) { + self.firmware.init.wait_tick(); + } +} + +impl recovery::Queues for Service<'_> { + fn render_pass_started(&self, qid: u8, stamp: u64) -> Option { + self.firmware.queues.render_pass_started(qid, stamp) + } + fn publish_recovery_targets(&self) -> Option { + let mut failure = self.firmware.queues.publish_render_recovery_targets(); + for queue in self + .firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + { + if let Err(error) = queue.publish_recovery_target() { + failure = failure.or(Some(error)); + } + } + core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); + failure + } + fn render_scheduler_active(&self) -> bool { + self.firmware.queues.render_scheduler_active() || !self.firmware.queues.teardown.is_empty() + } + fn oldest_spared(&self, qid: u8) -> Option { + self.firmware.queues.render_oldest_spared(qid).or_else(|| { + self.firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .find(|queue| queue.qid() == qid) + .and_then(|queue| queue.oldest_spared()) + }) + } + fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)) { + self.firmware.queues.visit_render_owners(qid, visit); + for queue in self + .firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + { + if queue.qid() == qid { + queue.visit_owners(visit); + } + } + } + fn classify_work(&mut self, class: &dyn Fn(&Arc) -> recovery::Class) -> u128 { + for queue in self + .firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + { + queue.classify(class); + } + self.firmware.queues.classify_render(class) + } + fn settle_blamed_render(&mut self, qid: u8, error: Error) { + let result = self + .firmware + .settle_blamed_render(qid, error, self.deferred); + self.settlement_error = self.settlement_error.or(result.err()); + } + fn settle_cancelled_renders(&mut self, qids: u128) { + let result = self.firmware.settle_cancelled_renders(qids, self.deferred); + self.settlement_error = self.settlement_error.or(result.err()); + } + fn rescan_compute(&mut self) -> Result { + self.firmware.scan_compute(None, self.deferred) + } + fn release_retired_compute(&mut self) { + let result = self.firmware.release_compute_witnesses(self.deferred); + self.settlement_error = self.settlement_error.or(result.err()); + } +} + +impl Firmware { + fn release_compute_witnesses(&mut self, deferred: &mut DeferredBatch) -> Result { + let state = recovery::Memory::recovery_state(&self.init)?; + let view = crate::g17::qos::View::new(self.init.qos()?)?; + let registry = &mut self.queues; + for queue in registry + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + queue.release_witnessed( + state, + |owner| registry.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + } + Ok(()) + } + + pub(super) fn service_runtime_events( + &mut self, + deferred: &mut DeferredBatch, + ) -> Result<(event::Pending, [u64; 2])> { + let mut service = Service { + firmware: self, + deferred, + render_masks: [0; 2], + settlement_error: None, + }; + let pending = event::service(&mut service)?; + if pending.recovery { + // Inline timeout/drain service can consume the restart record too. + // Close publication in this same critical section and ensure the + // event worker will join builders before the restart handshake. + let shared = &service.firmware.primary.state.shared; + shared.preparations.close(); + shared.queue_events(); + } + Ok((pending, service.render_masks)) + } + + /// Preparation admission is already closed and all builders have joined. + pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { + let shared = self.primary.state.shared.clone(); + let mut service = Service { + firmware: self, + deferred, + render_masks: [0; 2], + settlement_error: None, + }; + let result = recovery::service(&mut service, || { + shared.crashed.load(core::sync::atomic::Ordering::Acquire) + })?; + if let Some(error) = result.target_error { + dev_warn!( + shared.dev.as_ref(), + "Recovery target copy failed: {:?}\n", + error + ); + } + if let Some(error) = result.scan_error { + dev_warn!( + shared.dev.as_ref(), + "Post-recovery compute scan failed: {:?}\n", + error + ); + } + if let Some(error) = service.settlement_error { + return Err(error); + } + Ok(()) + } +} + +impl crate::g17::Shared { + /// The single event work item owns this preallocated collector until all + /// callbacks are finished. Shutdown can detach Firmware during an off-lock + /// boundary; the collector itself still owns every deferred completion. + pub(in crate::g17) fn run_runtime_events(self: &Arc) { + if self.crashed.load(core::sync::atomic::Ordering::Acquire) { + self.stop_queues(); + return; + } + let mut deferred = { + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return; + }; + let Some(deferred) = firmware.queues.worker_deferred.take() else { + drop(state); + self.lose_device_quietly(); + return; + }; + deferred + }; + let result = self.runtime_event_turn(&mut deferred); + deferred.finish(); + if let Some(firmware) = (*self.state.lock()).as_deref_mut() { + firmware.queues.worker_deferred = Some(deferred); + } + if let Err(error) = result { + dev_err!(self.dev.as_ref(), "GPU event service failed: {:?}\n", error); + self.lose_device_quietly(); + self.stop_queues(); + } + } + + fn runtime_event_turn(self: &Arc, deferred: &mut DeferredBatch) -> Result { + let (recover, masks) = { + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + let (pending, masks) = firmware.service_runtime_events(deferred)?; + let recover = pending.recovery || firmware.recovery.pending(); + if recover { + // Epoch closure and event consumption are serialized with every + // publisher. Joining builders must happen after unlocking. + self.preparations.close(); + } + (recover, masks) + }; + deferred.finish(); + if recover { + self.preparations.wait_drained(); + } + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + if self.crashed.load(core::sync::atomic::Ordering::Acquire) { + return Err(EIO); + } + if recover { + firmware.service_runtime_recovery(deferred)?; + if !firmware.recovery.pending() { + if !self.preparations.reopen() { + return Ok(()); + } + // Restart stopped consumption before its following record. + // Resume builders before observing pairs outside this lock. + self.queue_events(); + } + } + // Replay only at this outer boundary, after recovery has settled. + // Publication may service events but must never recurse into replay. + firmware.replay_compute(deferred)?; + drop(state); + self.observe_renders(masks, deferred)?; + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + if self.crashed.load(core::sync::atomic::Ordering::Acquire) { + return Err(EIO); + } + let now = super::now_ns(); + let compute = firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.in_flight()); + let in_flight = compute || firmware.queues.render_in_flight(); + let idle = firmware.idle_check(now, in_flight)?; + firmware.reclaim_drained_renders(deferred)?; + firmware.service_compute_exit(deferred)?; + { + let mut reclaim = self.reclaim.lock(); + firmware.service_teardown(&mut reclaim)?; + if !reclaim.is_empty() { + self.queue_reclaims(); + } + } + let backpressured = firmware.render_control_backpressured(); + let polling = backpressured + || firmware.queues.teardown.polling(now) + || firmware.compute_exit_polling(now); + if polling { + self.queue_poll(); + } + if let Some(delay) = idle { + self.queue_idle(delay); + } + if firmware.recovery.pending() { + self.preparations.close(); + self.queue_events(); + } else if !recover + && !polling + && (channel::Events::pending(&firmware.init, Role::Primary)? + || channel::Events::pending(&firmware.init, Role::Secondary)?) + { + // A full bounded pass must not depend on a fresh notification for + // records that arrived while it was consuming the same ring. + self.queue_events(); + } + if firmware.render_grow_pending() && !firmware.recovery.pending() { + self.queue_grow(); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs new file mode 100644 index 0000000000000..a6aa740ecf4e3 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -0,0 +1,683 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device ownership of retained render pairs and their shared free-list leases. +//! The device mutex protects identities and publication. A preparation gate +//! lease permits graph construction outside that mutex without session teardown. + +mod exit; +mod dependencies; +mod grow; +mod observe; +mod pools; +mod prepare; +mod recovery; +mod settle; +mod submit; +pub(super) mod teardown; + +use super::{DeferredBatch, Registry}; +use crate::g17::{context::Context, fw::queue::DataMaster, job::Packet, kick, queue::render::Pair}; +use crate::hw::t8140::queues::RENDER_SLOTS; +use kernel::{prelude::*, sync::Arc}; + +#[derive(Copy, Clone)] +struct Config { + clusters: u32, + descriptor_flags: [u32; 2], +} + +#[derive(Copy, Clone)] +struct Reservation { + slot: u8, + ids: [kick::Id; 2], + owner: u64, + buffer: (u8, u64), +} + +struct Entry { + reservation: Reservation, + pair: Option>, + parked: Option, + preparing: Option>, + context: Option>, + pool: Option>, + pending_owner: Option, + pending_owner_closed: bool, + borrowed_active: bool, + borrowed_dependencies: Option, + event_masks: [u64; 2], + closed: bool, + pending_error: Option<(Arc, Error)>, + pending_timeout: bool, + constructing: bool, + growing: bool, + published: bool, +} + +pub(super) struct RenderState { + config: Config, + entries: KVec>, + pools: pools::Pools, + grow: grow::Grow, + retired: KVec, + next_pm_generation: u64, +} + +impl RenderState { + pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { + if clusters == 0 { + return Err(EINVAL); + } + let mut entries = KVec::with_capacity(RENDER_SLOTS, GFP_KERNEL)?; + for _ in 0..RENDER_SLOTS { + entries.push(None, GFP_KERNEL)?; + } + Ok(Self { + config: Config { + clusters, + descriptor_flags, + }, + entries, + pools: pools::Pools::new()?, + grow: grow::Grow::new()?, + retired: KVec::with_capacity(128, GFP_KERNEL)?, + next_pm_generation: 1, + }) + } + + fn reserve_pm_generation(&mut self) -> Result { + let generation = self.next_pm_generation; + self.next_pm_generation = generation.checked_add(1).ok_or(EOVERFLOW)?; + Ok(generation) + } + + fn collect_retired(&mut self, slot: usize) -> Result { + loop { + if self.retired.len() == self.retired.capacity() { + return Ok(()); + } + let retired = self.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .and_then(|pair| pair.take_retired_binding()); + let Some(retired) = retired else { + return Ok(()); + }; + self.retired + .push_within_capacity(retired) + .map_err(|_| EIO)?; + } + } + + fn entry(&self, slot: u8) -> Result<&Entry> { + self.entries + .get(usize::from(slot)) + .and_then(Option::as_ref) + .ok_or(EIO) + } + + fn entry_mut(&mut self, slot: u8) -> Result<&mut Entry> { + self.entries + .get_mut(usize::from(slot)) + .and_then(Option::as_mut) + .ok_or(EIO) + } + + fn reserved(&mut self, reservation: Reservation) -> Result<&mut Entry> { + let entry = self.entry_mut(reservation.slot)?; + if entry.reservation.ids != reservation.ids + || entry.reservation.owner != reservation.owner + || entry.reservation.buffer != reservation.buffer + || entry.pair.is_some() + || entry.preparing.is_some() + || entry.context.is_some() + || !entry.constructing + { + return Err(EIO); + } + Ok(entry) + } + + /// Keep metadata while graph preparation owns the pair outside the mutex. + /// Completion scans skip it; cancellation is recorded and applied on return. + fn take_pair(&mut self, slot: u8, owner: u64, packet: &Arc) -> Result> { + let entry = self.entry_mut(slot)?; + if entry.closed + || entry.reservation.owner != owner + || entry.preparing.is_some() + || entry.pending_error.is_some() + || entry.pending_timeout + { + return Err(EBUSY); + } + let pair = entry.pair.as_ref().ok_or(EIO)?; + if !pair.matches(owner, &packet.context)? { + return Err(EIO); + } + entry.borrowed_active = pair.published_in_flight(); + entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.preparing = Some(packet.clone()); + entry.pair.take().ok_or(EIO) + } + + /// Validate before taking the owner from the caller. Even rejected returns + /// keep their backing alive until the caller resolves the reservation. + fn return_pair(&mut self, slot: u8, pair: &mut Option>) -> Result { + let returning = pair.as_ref().ok_or(EINVAL)?; + let entry = self.entry_mut(slot)?; + if entry.pair.is_some() + || entry.preparing.is_none() + || returning.slot() != slot + || returning.qids() != entry.reservation.ids.map(kick::Id::qid) + || returning.owner() != Some(entry.reservation.owner) + { + return Err(EIO); + } + entry.pair = pair.take(); + entry.borrowed_dependencies = None; + entry.preparing = None; + Ok(()) + } + + fn save_detaching_pool(&self, init: &crate::g17::initdata::InitData, slot: u8) -> Result { + let pool = self.entry(slot)?.pool.as_ref().ok_or(EIO)?; + let shared = self.entries.iter().enumerate().any(|(index, entry)| { + index != usize::from(slot) + && entry.as_ref().is_some_and(|entry| { + entry.pool.as_ref().is_some_and(|other| { + other.id() == pool.id() && other.control_va() == pool.control_va() + }) + }) + }); + if !shared { + self.pools.save(init, pool)?; + } + Ok(()) + } +} + +impl Registry { + fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { + if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { + return Err(EINVAL); + } + if self.render.entries[usize::from(slot)].is_none() { + let identity = self.identities.reserve(2)?.start; + let tiling = self.qids.reserve(identity, DataMaster::Tiling, None)?; + let fragment = match self.qids.reserve(identity + 1, DataMaster::Fragment, None) { + Ok(id) => id, + Err(error) => { + self.qids.cancel_unpublished(tiling)?; + return Err(error); + } + }; + self.render.entries[usize::from(slot)] = Some(Entry { + reservation: Reservation { + slot, + ids: [tiling, fragment], + owner, + buffer: (0, 0), + }, + pair: None, + parked: None, + preparing: None, + context: None, + pool: None, + pending_owner: None, + pending_owner_closed: false, + borrowed_active: false, + borrowed_dependencies: None, + event_masks: [0; 2], + closed: false, + pending_error: None, + pending_timeout: false, + constructing: false, + growing: false, + published: false, + }); + } + Ok(()) + } + + fn reserve_render(&mut self, slot: u8, owner: u64) -> Result { + self.ensure_render_ids(slot, owner)?; + let entry = self.render.entry(slot)?; + if entry.constructing + || entry.pair.is_some() + || entry.parked.is_some() + || entry.preparing.is_some() + || entry.growing + { + return Err(EBUSY); + } + let ids = self.render.entry(slot)?.reservation.ids; + // Physical queue identities survive failed graph construction. Only + // failure to acquire the second identity can return the first one. + for id in ids { + self.qids.publish(id)?; + } + let buffer = self.buffers.allocate_render()?; + let entry = self.render.entry_mut(slot)?; + entry.reservation.owner = owner; + entry.reservation.buffer = buffer; + entry.constructing = true; + entry.closed = false; + Ok(entry.reservation) + } + + fn install_render( + &mut self, + reservation: Reservation, + pair: &mut Option>, + ) -> Result { + let built = pair.as_ref().ok_or(EINVAL)?; + if built.slot() != reservation.slot + || built.qids() != reservation.ids.map(kick::Id::qid) + || built.owner() != Some(reservation.owner) + || built.buffer_id() != reservation.buffer.0 + { + return Err(EIO); + } + if self.render.reserved(reservation)?.closed { + return Err(ECANCELED); + } + // Construction holds one PB reference; retain the owner independently + // and return that reference before the first command acquires its own. + self.buffers.reserve_owner(reservation.buffer.0)?; + self.buffers.release(reservation.buffer.0)?; + let entry = self.render.reserved(reservation)?; + entry.context = Some(built.context().clone()); + entry.pool = Some(built.pool().clone()); + entry.pair = pair.take(); + entry.constructing = false; + Ok(()) + } + + /// Drop the rejected graph outside the device mutex before returning its + /// temporary PB reference. The physical queue identities remain reserved. + fn cancel_render_reservation(&mut self, reservation: Reservation) -> Result { + self.render.reserved(reservation)?; + self.buffers.release(reservation.buffer.0)?; + self.render.entry_mut(reservation.slot)?.constructing = false; + Ok(()) + } + + pub(super) fn fail_render_vm( + &mut self, + status: &Arc, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + for entry in self.render.entries.iter_mut().flatten() { + if !entry + .context + .as_ref() + .is_some_and(|context| Arc::ptr_eq(context.status(), status)) + { + continue; + } + if let Some(pair) = entry.pair.as_mut() { + pair.quarantine(error, &mut |event| deferred.push(event))?; + } else if entry.preparing.is_some() { + entry.pending_error = Some((status.clone(), error)); + } + } + Ok(()) + } +} + +struct Host<'a> { + firmware: &'a mut crate::g17::Firmware, + registration_powered: bool, +} +impl Host<'_> { + fn release_registration(&mut self) { + if !core::mem::take(&mut self.registration_powered) { + return; + } + if let Err(error) = self.firmware.release_registration_power() { + dev_warn!( + self.firmware.primary.state.shared.dev.as_ref(), + "Could not restore render idle policy: {:?}\n", + error + ); + } + } +} +impl crate::g17::queue::render::Host for Host<'_> { + fn epoch(&self) -> Result<(u64, u32)> { + Ok(( + self.firmware.recovery.generation(), + crate::g17::recovery::Memory::recovery_state(&self.firmware.init)?, + )) + } + fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]> { + use core::sync::atomic::Ordering; + let stamps = [ + self.firmware.init.stamp(qids[0])?.load(Ordering::Relaxed), + self.firmware.init.stamp(qids[1])?.load(Ordering::Relaxed), + ]; + core::sync::atomic::fence(Ordering::Acquire); + Ok(stamps) + } + fn qos_publish( + &mut self, + owner: crate::g17::qos::Owner, + scheduler: u64, + ) -> Result { + use crate::g17::qos; + self.firmware.queues.accounting.publish( + &qos::View::new(self.firmware.init.qos()?)?, + owner, + scheduler, + qos::clock, + ) + } + fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { + use crate::g17::qos; + self.firmware + .queues + .accounting + .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) + } + fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + for _ in 0..3 { + if !self.firmware.bridge.regs.g17_configuration_powered() { + self.firmware.acquire_registration_power()?; + self.registration_powered = true; + } + if self + .firmware + .bridge + .regs + .install_render_pair(registration)? + { + return Ok(()); + } + } + Err(EAGAIN) + } + fn publish_qid(&mut self, id: kick::Id) -> Result { + self.firmware.queues.qids.publish(id) + } + fn publish_outer_pair( + &mut self, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + commit: impl FnOnce() -> Result, + late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]> { + self.firmware + .init + .publish_work_pair(fragment, tiling, true, commit, late_tiling) + } + fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { + self.release_registration(); + let result = self.firmware.queues.render.pools.activate(id, generation); + if result.is_err() { + self.firmware.primary.state.shared.lose_device_quietly(); + } + result + } + fn publish_retained_pair( + &mut self, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + next: [u32; 2], + ) -> Result { + let result = self + .firmware + .init + .publish_retained_work_pair(fragment, tiling, next); + if result.is_err() { + self.firmware.primary.state.shared.lose_device_quietly(); + } + result + } + fn notify_render(&mut self) -> Result { + self.release_registration(); + let result = self + .firmware + .primary + .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | 0x08); + if result.is_err() { + self.firmware.primary.state.shared.lose_device_quietly(); + } + result + } + fn note_submission(&mut self) -> Result { + self.firmware.note_submission() + } + fn complete_render( + &mut self, + owners: Option<[crate::g17::qos::Owner; 2]>, + pool_id: u16, + generation: u64, + buffer_id: u8, + ) -> Result { + use crate::g17::qos; + let queues = &mut self.firmware.queues; + let next = queues.render.pools.completed(pool_id, generation)?; + queues.buffers.can_release(buffer_id)?; + let view = qos::View::new(self.firmware.init.qos()?)?; + if let Some(owners) = owners { + queues.accounting.complete(&view, &owners)?; + } + queues.render.pools.commit_completed(pool_id, next); + queues.buffers.release(buffer_id) + } +} + +impl crate::g17::Firmware { + /// The pair is returned even when transport, accounting or collection fails. + /// No off-lock operation or callback which can re-enter the device runs here. + fn with_render( + &mut self, + slot: usize, + operation: impl FnOnce(&mut Pair, &mut Host<'_>) -> Result, + ) -> Result { + let mut pair = self + .queues + .render + .entries + .get_mut(slot) + .and_then(Option::as_mut) + .and_then(|entry| entry.pair.take()) + .ok_or(EBUSY)?; + let mut host = Host { + firmware: self, + registration_powered: false, + }; + let result = operation(&mut pair, &mut host); + host.release_registration(); + self.queues.render.entries[slot].as_mut().ok_or(EIO)?.pair = Some(pair); + result + } + + fn release_render_lists(&mut self) -> Result { + if self.queues.render.grow.reply.is_some() { + return Err(EAGAIN); + } + let primary = &mut self.primary; + self.queues.render.pools.publish_releases(&self.init, || { + primary.notify(crate::g17::MSG_CONTROL_NOTIFY) + }) + } + + /// Events have already been consumed by the caller. Each healthy completion + /// services its final pool release before examining the next retained job. + fn retire_render_slot( + &mut self, + slot: usize, + masks: Option<[u64; 2]>, + deferred: &mut DeferredBatch, + ) -> Result { + let generation = self.recovery.generation(); + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + loop { + let progressed = self.with_render(slot, |pair, host| { + pair.retire(masks, generation, state, host, &mut |event| { + deferred.push(event) + }) + })?; + if !progressed { + break; + } + match self.release_render_lists() { + Err(error) if error == EAGAIN => break, + result => result?, + } + } + Ok(()) + } + + pub(in crate::g17) fn settle_render_vm( + &mut self, + status: &Arc, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + for slot in 1..self.queues.render.entries.len() { + if self.queues.render.entries[slot] + .as_ref() + .is_some_and(|entry| { + entry.pair.is_some() + && entry + .context + .as_ref() + .is_some_and(|context| Arc::ptr_eq(context.status(), status)) + }) + { + self.retire_render_slot(slot, None, deferred)?; + } + } + self.queues.fail_render_vm(status, error, deferred) + } + + pub(in crate::g17) fn stop_render(&mut self) { + for entry in self.queues.render.entries.iter_mut().flatten() { + if let Some(pair) = entry.pair.as_mut() { + if let Err(error) = pair.stopped(&mut |event| { + event.finish(); + Ok(()) + }) { + dev_err!( + self.primary.state.shared.dev.as_ref(), + "Could not settle stopped render pair: {:?}\n", + error + ); + } + } + } + } +} + +impl Registry { + pub(in crate::g17) fn render_oldest_spared(&self, qid: u8) -> Option { + self.render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid)) + .and_then(|pair| pair.oldest_spared()) + } + pub(in crate::g17) fn visit_render_owners( + &self, + qid: u8, + visit: &mut dyn FnMut(&Arc), + ) { + for pair in self + .render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + { + if pair.qids().contains(&qid) { + pair.visit_owners(visit); + } + } + } + pub(in crate::g17) fn classify_render( + &self, + classify: &dyn Fn(&Arc) -> crate::g17::recovery::Class, + ) -> u128 { + let mut spared = 0; + for pair in self + .render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + { + pair.classify(classify); + if pair.oldest_spared() == Some(true) { + for qid in pair.qids() { + spared |= 1u128 << qid; + } + } + } + spared + } + + pub(in crate::g17) fn render_scheduler_active(&self) -> bool { + self.render.entries.iter().any(Option::is_some) + || !self.render.retired.is_empty() + || !self.teardown.is_empty() + } + pub(in crate::g17) fn render_in_flight(&self) -> bool { + self.render.entries.iter().flatten().any(|entry| { + entry.pair.as_ref().map_or( + (entry.preparing.is_some() || entry.growing) && entry.borrowed_active, + |pair| pair.published_in_flight(), + ) + }) + } + pub(in crate::g17) fn retain_render_completions(&mut self, masks: [u64; 2]) { + for entry in self.render.entries.iter_mut().flatten() { + for id in entry.reservation.ids { + let qid = usize::from(id.qid()); + entry.event_masks[qid / 64] |= masks[qid / 64] & (1u64 << (qid % 64)); + } + } + } + pub(in crate::g17) fn render_needs_rescan(&self) -> bool { + self.render.entries.iter().flatten().any(|entry| { + let active = entry + .pair + .as_ref() + .map_or(entry.borrowed_active, |pair| pair.published_in_flight()); + active + && entry.reservation.ids.into_iter().all(|id| { + let qid = usize::from(id.qid()); + entry.event_masks[qid / 64] & (1u64 << (qid % 64)) != 0 + }) + }) + } + pub(in crate::g17) fn defer_render_timeout( + &mut self, + packet: &Arc, + error: Error, + ) -> bool { + let Some(slot) = packet.completion.render_slot() else { + return false; + }; + let Ok(entry) = self.render.entry_mut(slot) else { + return false; + }; + if entry.preparing.is_none() && !entry.growing { + return false; + } + if !entry + .borrowed_dependencies + .as_ref() + .is_some_and(|dependencies| dependencies.contains_packet(packet)) + { + return false; + } + packet.completion.defer_timeout(error); + entry.pending_timeout = true; + true + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs new file mode 100644 index 0000000000000..09e4322bf42ce --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Both engines preserve the covered render prefix while physical graphs are borrowed. + +use crate::g17::{dependency::Frontier, job::Packet, runtime::Registry}; +use kernel::{prelude::*, sync::Arc}; + +impl Registry { + pub(in crate::g17::runtime) fn render_prefix( + &self, + owner: u64, + packet: &Arc, + ) -> Result { + let mut frontier = Frontier::new(); + for entry in self.render.entries.iter().flatten() { + if let Some(pair) = entry.pair.as_ref() { + pair.collect_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + )?; + } else if let Some(dependencies) = entry.borrowed_dependencies.as_ref() { + dependencies.collect( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + )?; + } + } + Ok(frontier) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/exit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/exit.rs new file mode 100644 index 0000000000000..c47110cb76fad --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/exit.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Physical render queue scope for an exiting logical context. A graph loan +//! remains foreign to the transaction until its exact owner is restored. + +use super::RenderState; +use crate::g17::context::Context; +use kernel::sync::Arc; + +impl RenderState { + pub(in crate::g17::runtime) fn exit_scope(&self, context: &Arc) -> (u128, u128, u64) { + let (mut own, mut foreign, mut slots) = (0, 0, 0); + for (slot, entry) in self.entries.iter().enumerate() { + let Some(entry) = entry else { + continue; + }; + let qids = self.qids_for_slot(slot as u8); + let preparing = entry.preparing.is_some() || entry.constructing || entry.growing; + if !preparing && entry.context.as_ref().is_some_and(|owner| Arc::ptr_eq(owner, context)) { + own |= qids; + slots |= 1u64 << slot; + } else if preparing || entry.context.is_some() { + foreign |= qids; + } + } + (own, foreign, slots) + } + + pub(in crate::g17::runtime) fn qids_for_slot(&self, slot: u8) -> u128 { + self.entries + .get(usize::from(slot)) + .and_then(Option::as_ref) + .map_or(0, |entry| { + entry.reservation.ids.into_iter().fold(0, |mask, id| { + mask | (1u128 << id.qid()) + }) + }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs new file mode 100644 index 0000000000000..a9636cb7ceda0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Deferred parameter-buffer growth. Event ingress retains the entire request; +//! allocation borrows one pair under a preparation lease and runs off-lock. + +use super::RenderState; +use crate::g17::{ + channel::Rings, + fw::channels::{ControlRecord, FreeListCompletion, TvbGrowReply, TvbGrowRequest, EVENT_SLOTS}, +}; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +const RETRY_LIMIT: u16 = 1000; + +pub(super) struct Grow { + requests: KVec, + active: Option, + pub(super) reply: Option<(TvbGrowRequest, bool, u16)>, + release_attempts: u16, +} +impl Grow { + pub(super) fn new() -> Result { + Ok(Self { + requests: KVec::with_capacity(2 * EVENT_SLOTS as usize, GFP_KERNEL)?, + active: None, + reply: None, + release_attempts: 0, + }) + } +} + +impl RenderState { + pub(in crate::g17::runtime) fn control_backpressured(&self) -> bool { + self.grow.reply.is_some() || self.pools.pending() + } + fn grow_owner(&self, request: TvbGrowRequest) -> Option { + self.entries.iter().position(|entry| { + entry.as_ref().is_some_and(|entry| { + entry.published + && u32::from(entry.reservation.buffer.0) == request.buffer_slot + && entry.context.as_ref().is_some_and(|context| { + request.vm_slot == -1 || request.vm_slot as u32 == u32::from(context.id()) + }) + }) + }) + } + fn grow_owner_busy(&self, request: TvbGrowRequest) -> bool { + self.grow_owner(request).is_some_and(|slot| { + self.entries[slot] + .as_ref() + .is_some_and(|entry| entry.preparing.is_some() || entry.growing) + }) + } +} + +impl crate::g17::Firmware { + pub(in crate::g17) fn queue_tvb_grow(&mut self, request: TvbGrowRequest) -> Result { + let render = &mut self.queues.render; + if request.halt_count != self.recovery.generation() + || render.grow.active == Some(request) + || render.grow.requests.contains(&request) + { + return Ok(()); + } + render + .grow + .requests + .push_within_capacity(request) + .map_err(|_| EOVERFLOW) + } + pub(in crate::g17) fn render_grow_pending(&self) -> bool { + !self.queues.render.grow.requests.is_empty() + } + pub(in crate::g17) fn render_control_backpressured(&self) -> bool { + self.queues.render.grow.reply.is_some() || self.queues.render.pools.pending() + } + pub(in crate::g17) fn complete_freelist_grow(&mut self, event: FreeListCompletion) -> Result { + self.queues.render.pools.complete_grow(event) + } + pub(in crate::g17) fn observe_render_releases(&mut self) -> Result { + self.queues.render.pools.observe(&self.init) + } + pub(in crate::g17) fn flush_tvb_reply(&mut self) -> Result { + let Some((request, grew, _)) = self.queues.render.grow.reply else { + return Ok(true); + }; + if request.halt_count != self.recovery.generation() { + return Err(EIO); + } + let record = ControlRecord::TvbGrow(TvbGrowReply::new(&request, grew)); + match Rings::publish_control(&self.init, &record) { + Err(error) if error == EAGAIN => return Ok(false), + result => { + result?; + } + } + // Producer ownership is irreversible, even if announcement fails. + self.queues.render.grow.reply = None; + fence(Ordering::SeqCst); + self.primary + .notify((0x84 << crate::g17::MSG_TYPE_SHIFT) | 0x11)?; + Ok(true) + } + pub(in crate::g17) fn flush_freelist_release(&mut self) -> Result { + if self.queues.render.grow.reply.is_some() { + return Ok(false); + } + match self.release_render_lists() { + Err(error) if error == EAGAIN => Ok(false), + result => { + result?; + self.queues.render.grow.release_attempts = 0; + Ok(true) + } + } + } + /// Only the sleeping event worker consumes this budget. Inline submission + /// probes cannot turn a transient full control ring into a rapid timeout. + pub(in crate::g17) fn note_render_control_backpressure(&mut self) -> Result { + let render = &mut self.queues.render; + let attempts = if let Some((_, _, attempts)) = render.grow.reply.as_mut() { + attempts + } else if render.pools.pending() { + &mut render.grow.release_attempts + } else { + return Err(EINVAL); + }; + *attempts = attempts.checked_add(1).ok_or(ETIMEDOUT)?; + if *attempts > RETRY_LIMIT { + return Err(ETIMEDOUT); + } + Ok(()) + } +} + +impl crate::g17::Firmware { + pub(super) fn commit_render_growth( + &mut self, + slot: u8, + prepared: &mut Option, + ) -> Result { + if prepared.is_none() { + return Ok(()); + } + let pair = self + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)?; + let vm = pair.context().vm().clone(); + let pool = pair.pool_id(); + let buffer_id = pair.buffer_id(); + // Preflight every destination before the graph starts publishing. + let table = self.init.pb_descriptor_table()?; + let offset = usize::from(buffer_id) * 16; + let words = [ + table.word(offset)?, + table.word(offset + 4)?, + table.word(offset + 8)?, + table.word(offset + 12)?, + ]; + let descriptor = pair + .manager() + .commit_growth(prepared, |previous, blocks, mappings| { + vm.append_render_pool_mappings_prepared(slot, pool, previous, blocks, mappings) + })?; + if let Some(descriptor) = descriptor { + for (word, value) in words.into_iter().zip(descriptor) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + } + Ok(()) + } +} + +impl crate::g17::Shared { + pub(in crate::g17) fn grow_render(&self) -> Result { + loop { + let lease = self.preparations.enter(false)?; + let (request, slot, mut pair) = { + let mut state = self.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if firmware.render_control_backpressured() { + return Ok(()); + } + let generation = firmware.recovery.generation(); + let render = &mut firmware.queues.render; + if render.grow.active.is_some() { + return Ok(()); + } + let Some(index) = render.grow.requests.iter().position(|request| { + request.halt_count != generation || !render.grow_owner_busy(*request) + }) else { + return Ok(()); + }; + let request = render.grow.requests.remove(index).map_err(|_| EIO)?; + if request.halt_count != generation { + continue; + } + let slot = render.grow_owner(request); + let pair = match slot { + Some(slot) => { + let entry = render.entries[slot].as_mut().ok_or(EIO)?; + if entry.preparing.is_some() || entry.growing { + return Err(EIO); + } + entry.borrowed_active = entry + .pair + .as_ref() + .is_some_and(|pair| pair.published_in_flight()); + entry.borrowed_dependencies = + entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.growing = true; + Some(entry.pair.take().ok_or(EIO)?) + } + None => None, + }; + render.grow.active = Some(request); + (request, slot, pair) + }; + let prepared = match pair.as_mut() { + Some(pair) => { + let vm = pair.context().vm().clone(); + let pool = pair.pool_id(); + let slot = pair.slot(); + let manager = pair.manager(); + let before = manager.blocks(); + if vm.render_pool_tvb_blocks(slot, pool) != Some(before) { + Err(EFAULT) + } else { + manager.prepare_growth( + &self.dev, + &vm, + crate::g17::buffer::firmware_growth_target(before), + None, + ) + } + } + None => Err(ENOENT), + }; + let (mut prepared, prepare_error) = match prepared { + Ok(plan) => (plan, None), + Err(error) => (None, Some(error)), + }; + let mut state = self.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if let Some(slot) = slot { + let entry = firmware.queues.render.entries[slot].as_mut().ok_or(EIO)?; + if entry.pair.is_some() || !entry.growing { + return Err(EIO); + } + entry.pair = pair.take(); + entry.borrowed_dependencies = None; + entry.growing = false; + } + if firmware.queues.render.grow.active != Some(request) { + return Err(EIO); + } + firmware.queues.render.grow.active = None; + self.changed.notify_all(); + if !lease.is_current() || request.halt_count != firmware.recovery.generation() { + continue; + } + let result = if let Some(error) = prepare_error { + Err(error) + } else if prepared.is_none() { + Ok(false) + } else if let Some(slot) = slot { + let pair = firmware.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .ok_or(EIO)?; + let vm = pair.context().vm().clone(); + let pool = pair.pool_id(); + let slot = pair.slot(); + let grew = prepared.is_some(); + pair.manager() + .commit_growth(&mut prepared, |previous, blocks, mappings| { + vm.append_render_pool_mappings_prepared( + slot, pool, previous, blocks, mappings, + ) + }) + .map(|_| grew) + } else { + Err(ENOENT) + }; + let grew = match result { + Ok(grew) => grew, + Err(error) => { + dev_warn!( + self.dev.as_ref(), + "Could not grow render parameter buffer {}: {:?}\n", + request.buffer_slot, + error + ); + false + } + }; + firmware.queues.render.grow.reply = Some((request, grew, 0)); + if !firmware.flush_tvb_reply()? { + return Ok(()); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs new file mode 100644 index 0000000000000..31bcb688c8001 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Event-driven render snapshots outside the device mutex. The preparation +//! epoch and exact pair owner survive every read, cancellation and retry. + +use super::super::DeferredBatch; +use crate::g17::queue::render::Pair; +use core::sync::atomic::Ordering; +use kernel::{prelude::*, sync::Arc}; + +impl crate::g17::Firmware { + fn take_render_observation(&mut self, start: usize) -> Result)>> { + if self.recovery.pending() { + return Ok(None); + } + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + for slot in start..self.queues.render.entries.len() { + let Some(entry) = self.queues.render.entries[slot].as_mut() else { + continue; + }; + if entry.preparing.is_some() || entry.growing { + continue; + } + let Some(pair) = entry.pair.as_ref() else { + continue; + }; + let qids = pair.qids(); + if !pair.published_in_flight() + || !qids + .into_iter() + .all(|qid| entry.event_masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) + { + continue; + } + let stamps = [ + self.init.stamp(qids[0])?.load(Ordering::Relaxed), + self.init.stamp(qids[1])?.load(Ordering::Relaxed), + ]; + core::sync::atomic::fence(Ordering::Acquire); + let Some(packet) = pair.observation_packet(entry.event_masks, stamps, state)? else { + continue; + }; + entry.borrowed_active = pair.published_in_flight(); + entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.preparing = Some(packet); + return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); + } + Ok(None) + } + + fn return_render_observation(&mut self, slot: usize, pair: &mut Option>) -> Result { + let returning = pair.as_ref().ok_or(EINVAL)?; + let entry = self.queues.render.entry_mut(slot as u8)?; + if entry.pair.is_some() + || entry.preparing.is_none() + || returning.slot() as usize != slot + || returning.qids() != entry.reservation.ids.map(crate::g17::kick::Id::qid) + { + return Err(EIO); + } + entry.pair = pair.take(); + entry.borrowed_dependencies = None; + entry.preparing = None; + Ok(()) + } + + pub(super) fn settle_render_terminal( + &mut self, + slot: usize, + deferred: &mut DeferredBatch, + ) -> Result { + let terminal = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .and_then(|pair| pair.take_terminal()); + if let Some((status, error)) = terminal { + status.record(error); + self.scan_compute(None, deferred)?; + self.settle_render_vm(&status, error, deferred)?; + self.queues.fail_compute_vm(&status, error, deferred)?; + } + Ok(()) + } +} + +impl crate::g17::Shared { + pub(in crate::g17) fn observe_renders( + self: &Arc, + masks: [u64; 2], + deferred: &mut DeferredBatch, + ) -> Result { + { + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + firmware.queues.retain_render_completions(masks); + // Returned preparers can have deferred a VM failure or timeout. + // Only this full worker collector runs their final observations. + for slot in 1..firmware.queues.render.entries.len() { + if firmware.queues.render.entries[slot] + .as_ref() + .is_some_and(|entry| { + entry.pair.is_some() + && (entry.pending_error.is_some() + || entry.pending_timeout + || entry.closed) + }) + { + firmware.apply_render_deferred(slot as u8, deferred)?; + } + } + } + deferred.finish(); + let mut cursor = 1; + let mut remaining = crate::hw::t8140::queues::RENDER_DEPTH; + let mut retries = 0; + loop { + let epoch = match self.preparations.try_enter() { + Err(error) if error == EAGAIN || error == ENODEV => break, + result => result?, + }; + let loan = { + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + if !epoch.is_current() || self.crashed.load(Ordering::Acquire) { + break; + } + firmware.take_render_observation(cursor)? + }; + let Some((slot, pair)) = loan else { + break; + }; + if slot != cursor { + remaining = crate::hw::t8140::queues::RENDER_DEPTH; + retries = 0; + } + let observed = pair.observe_front(); + let mut pair = Some(pair); + let mut state = self.state.lock(); + // Return installed ownership before examining read errors or epoch. + let restored = (*state) + .as_deref_mut() + .ok_or(ENODEV) + .and_then(|firmware| firmware.return_render_observation(slot, &mut pair)); + if let Err(error) = restored { + // A failed invariant must not drop firmware-owned storage. Join + // the other preparers only after releasing our own epoch, and + // keep this detached owner until both processors have stopped. + drop(state); + drop(epoch); + self.lose_device_quietly(); + self.stop_queues(); + if let Some(mut pair) = pair { + pair.stopped(&mut |event| { + event.finish(); + Ok(()) + })?; + } + return Err(error); + } + let firmware = (*state).as_deref_mut().ok_or(EIO)?; + self.changed.notify_all(); + if firmware.render_grow_pending() { + self.queue_events(); + } + if !epoch.is_current() + || firmware.recovery.pending() + || self.crashed.load(Ordering::Acquire) + { + break; + } + let result = (|| { + let state = crate::g17::recovery::Memory::recovery_state(&firmware.init)?; + let progressed = match observed { + Ok(observed) => firmware.with_render(slot, |pair, host| { + pair.retire_observed(observed, state, host, &mut |event| { + deferred.push(event) + }) + })?, + Err(error) if error == EAGAIN => false, + Err(error) => return Err(error), + }; + if progressed { + match firmware.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + firmware.queues.render.collect_retired(slot)?; + // An observation of the old front is committed before a queued + // exact timeout is permitted to advance that same tracker. + firmware.apply_render_deferred(slot as u8, deferred)?; + firmware.settle_render_terminal(slot, deferred)?; + if firmware + .queues + .render + .entry(slot as u8)? + .pair + .as_ref() + .is_some_and(|pair| !pair.in_flight()) + { + firmware.queues.render.entry_mut(slot as u8)?.event_masks = [0; 2]; + } + Ok(progressed) + })(); + drop(state); + drop(epoch); + deferred.finish(); + if result? { + remaining -= 1; + retries = 0; + cursor = if remaining == 0 { slot + 1 } else { slot }; + } else { + retries += 1; + cursor = if retries >= 2 { slot + 1 } else { slot }; + } + if cursor != slot { + remaining = crate::hw::t8140::queues::RENDER_DEPTH; + retries = 0; + } + } + { + let state = self.state.lock(); + if (*state).as_deref().is_some_and(|firmware| { + !firmware.recovery.pending() && firmware.queues.render_needs_rescan() + }) { + self.queue_poll(); + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs new file mode 100644 index 0000000000000..7047992754b37 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device-wide references to render free lists. Two physical pairs bound to +//! one logical context share one registration and one final release witness. +//! The device mutex serializes every transition and descriptor-table access. + +use crate::g17::{ + channel::Rings, + freelist::{Lease, Owner, RenderPool}, + fw::{ + channels::{ControlRecord, FreeListCompletion}, + initdata::PagePoolDescriptor, + }, + initdata::InitData, + object::KernelObject, +}; +use core::sync::atomic::{fence, AtomicU64, Ordering}; +use kernel::prelude::*; + +const POOLS: usize = 256; +const FIRST_RENDER_POOL: usize = 129; + +pub(super) struct Pools { + leases: KVec>, +} + +/// Undo state cannot leave the device-locked publication transaction. Rollback +/// is permitted only before the packet exposes its first firmware reference. +pub(super) struct Preparation { + previous: Option, + slot: usize, + pub(super) generation: u64, +} + +impl Pools { + pub(super) fn new() -> Result { + let mut leases = KVec::with_capacity(POOLS, GFP_KERNEL)?; + for _ in 0..POOLS { + leases.push(None, GFP_KERNEL)?; + } + Ok(Self { leases }) + } + + fn slot(id: u16) -> Result { + let slot = usize::from(id); + if !(FIRST_RENDER_POOL..POOLS).contains(&slot) { + return Err(EINVAL); + } + Ok(slot) + } + + fn owner(pool: &RenderPool) -> Owner { + Owner { + serial: pool.serial(), + buffer_slot: u32::from(pool.id()), + control_va: pool.control_va(), + ring_va: pool.run_list_va(), + cookie: pool.end_va(), + } + } + + pub(super) fn released(&self, id: u32, _control: u64) -> bool { + self.leases + .get(id as usize) + .is_none_or(|lease| lease.as_ref().is_none_or(|lease| lease.released())) + } + + /// Install both engine references before either inner queue becomes + /// visible. Warm overlap keeps the existing generation and pool contents. + pub(super) fn prepare(&mut self, init: &InitData, pool: &RenderPool) -> Result { + self.observe(init)?; + let slot = Self::slot(pool.id())?; + let owner = Self::owner(pool); + let previous = self.leases[slot]; + if previous.is_some_and(|lease| lease.owner() != owner && !lease.released()) { + return Err(EBUSY); + } + // Pools arrive fully populated. There is no asynchronous initial grow. + let next = Lease::acquire(previous.as_ref(), owner)?; + if previous.is_none_or(|lease| lease.owner() != owner) { + self.stage(init, pool)?; + } + let generation = next.generation(); + self.leases[slot] = Some(next); + Ok(Preparation { + previous, + slot, + generation, + }) + } + + /// Warming validates the incoming owner and stages its descriptor without + /// acquiring command references or publishing an asynchronous request. + pub(super) fn warm(&self, init: &InitData, pool: &RenderPool) -> Result { + let previous = self.leases[Self::slot(pool.id())?]; + let owner = Self::owner(pool); + if previous.is_some_and(|lease| lease.owner() == owner) { + return Ok(()); + } + if previous.is_some_and(|lease| !lease.released()) { + return Err(EBUSY); + } + Lease::acquire(previous.as_ref(), owner)?; + self.stage(init, pool) + } + + fn stage(&self, init: &InitData, pool: &RenderPool) -> Result { + let words = descriptor_words(init.page_pool_descriptor_table()?, pool.id())?; + pool.repopulate()?; + let descriptor = pool + .page_pool_state()? + .descriptor(words[3].load(Ordering::Relaxed)); + for (word, value) in words.into_iter().zip([ + descriptor.page_list, + descriptor.counters, + descriptor.page_count, + descriptor.unk_18, + ]) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Retire preparation only after both deferred outer bodies and their + /// inner commands exist, before publishing either outer producer. + pub(super) fn activate(&mut self, id: u16, generation: u64) -> Result { + let lease = self.leases[Self::slot(id)?].as_mut().ok_or(EIO)?; + if lease.generation() != generation { + return Err(EIO); + } + lease.retire_prepare() + } + + pub(super) fn rollback(&mut self, preparation: Preparation) { + self.leases[preparation.slot] = preparation.previous; + } + + /// Preflight a paired retirement before QoS or PB reference accounting. + pub(super) fn completed(&self, id: u16, generation: u64) -> Result { + let mut next = self.leases[Self::slot(id)?].ok_or(EIO)?; + if next.generation() != generation { + return Err(EIO); + } + next.complete_render()?; + Ok(next) + } + + /// The matching completed() result and other accounting preflights were + /// accepted under the same device mutex, so this assignment cannot fail. + pub(super) fn commit_completed(&mut self, id: u16, next: Lease) { + self.leases[usize::from(id)] = Some(next); + } + + pub(super) fn pending(&self) -> bool { + self.leases + .iter() + .flatten() + .any(|lease| lease.pending_release().is_some()) + } + + pub(super) fn complete_grow(&mut self, event: FreeListCompletion) -> Result { + let id = u16::try_from(event.buffer_slot).map_err(|_| EINVAL)?; + let lease = self.leases[Self::slot(id)?].as_mut().ok_or(EIO)?; + lease.complete_grow(event) + } + + pub(super) fn observe(&mut self, init: &InitData) -> Result { + fence(Ordering::Acquire); + let consumer = init.control_consumer()?.load(Ordering::Relaxed); + let producer = init.control_producer()?.load(Ordering::Relaxed); + for lease in self.leases.iter_mut().flatten() { + lease.observe_releases(consumer, producer)?; + } + Ok(()) + } + + /// Store the release witness immediately after the ring producer. A failed + /// mailbox announcement must never cause the same release to be replayed. + pub(super) fn publish_releases( + &mut self, + init: &InitData, + mut announce: impl FnMut() -> Result, + ) -> Result { + for lease in self.leases.iter_mut().flatten() { + let Some(release) = lease.pending_release() else { + continue; + }; + let cursor = Rings::publish_control(init, &ControlRecord::FreeListRelease(release))?; + lease.mark_release_sent(cursor)?; + fence(Ordering::SeqCst); + announce()?; + } + self.observe(init) + } + + /// A detached pool may capture firmware counters only after its last pair + /// stopped using this registration. Callers check all retained pair owners. + pub(super) fn save(&self, init: &InitData, pool: &RenderPool) -> Result { + let state = pool.page_pool_state()?; + let words = descriptor_words(init.page_pool_descriptor_table()?, pool.id())?; + fence(Ordering::Acquire); + let descriptor = PagePoolDescriptor { + page_list: words[0].load(Ordering::Relaxed), + counters: words[1].load(Ordering::Relaxed), + page_count: words[2].load(Ordering::Relaxed), + unk_18: words[3].load(Ordering::Relaxed), + }; + if descriptor.page_list != state.descriptor(descriptor.unk_18).page_list { + return Err(EIO); + } + pool.store_page_pool_state(state.updated(&descriptor)) + } +} + +fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; 4]> { + let offset = Pools::slot(id)? * size_of::(); + Ok([ + table.dword(offset)?, + table.dword(offset + 8)?, + table.dword(offset + 16)?, + table.dword(offset + 24)?, + ]) +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs new file mode 100644 index 0000000000000..30ea174f611b1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -0,0 +1,597 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Off-lock render graph construction and command preparation. Every borrowed +//! installed pair returns to its device owner before its preparation lease ends. + +use super::{Config, Reservation}; +use crate::g17::{ + buffer::MetricsLease, + job::Packet, + object::{Allocator, KernelObject, Pool}, + queue::render::{Pair, RenderAddresses}, +}; +use kernel::{prelude::*, sync::Arc}; + +struct BuildResources { + uat: Arc, + objects: Arc, + ids: Arc, + global: crate::mmu::Vm, + metrics: Arc, + metrics_ids: Arc, + metrics_bases: [u64; 2], + config: Config, +} +impl BuildResources { + fn new(firmware: &crate::g17::Firmware) -> Result { + Ok(Self { + uat: firmware.uat.clone(), + objects: firmware.pool.clone(), + ids: firmware.render_ids.clone(), + global: firmware._render_global.clone(), + metrics: firmware._pm_metrics.clone(), + metrics_ids: firmware.queues.metrics.clone(), + metrics_bases: [ + crate::hw::t8140::CONFIG.pm_metrics.va, + firmware.init.metrics_fw_va()?, + ], + config: firmware.queues.render.config, + }) + } + fn metrics(&self) -> Result { + MetricsLease::new(&self.metrics_ids, &self.metrics, self.metrics_bases) + } +} + +enum Build { + Fresh(Reservation, u64, MetricsLease), + Parked { + parked: crate::g17::queue::render::parked::ParkedPair, + buffer: (u8, u64), + pm_generation: u64, + metrics: MetricsLease, + }, + Rebind { + pair: KBox, + buffer: Option<(u8, u64)>, + pm_generation: Option, + metrics: Option, + }, +} + +impl super::super::Backend { + fn build_render_pair(&self, slot: u8, first: &Arc) -> Result { + let lease = self.shared.preparations.enter(false)?; + let (resources, build) = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + firmware.queues.ensure_render_ids(slot, self.owner)?; + let resources = BuildResources::new(firmware)?; + let build = + if let Some(entry) = firmware.queues.render.entries[usize::from(slot)].as_ref() { + if entry.pending_error.is_some() || entry.pending_timeout { + self.shared.queue_events(); + return Err(EBUSY); + } + if entry.constructing || entry.preparing.is_some() || entry.growing { + return Err(EBUSY); + } + if entry.parked.is_some() { + let metrics = resources.metrics()?; + let pm_generation = firmware.queues.render.reserve_pm_generation()?; + let buffer = firmware.queues.buffers.allocate_render()?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = Some(first.clone()); + entry.pending_owner = Some(self.owner); + entry.borrowed_active = false; + Build::Parked { + parked: entry.parked.take().ok_or(EIO)?, + buffer, + pm_generation, + metrics, + } + } else if entry.pair.is_none() { + let metrics = resources.metrics()?; + let pm_generation = firmware.queues.render.reserve_pm_generation()?; + Build::Fresh( + firmware.queues.reserve_render(slot, self.owner)?, + pm_generation, + metrics, + ) + } else { + let pair = entry.pair.as_ref().ok_or(EBUSY)?; + if pair.matches(self.owner, &self.context)? { + return Ok(()); + } + if !pair.reusable()? { + return Err(EBUSY); + } + let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); + if foreign { + firmware + .queues + .render + .save_detaching_pool(&firmware.init, slot)?; + } + let metrics = if foreign { + Some(resources.metrics()?) + } else { + None + }; + let pm_generation = if foreign { + Some(firmware.queues.render.reserve_pm_generation()?) + } else { + None + }; + let buffer = if foreign { + Some(firmware.queues.buffers.allocate_render()?) + } else { + None + }; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = Some(first.clone()); + entry.pending_owner = Some(self.owner); + entry.borrowed_active = entry + .pair + .as_ref() + .is_some_and(|pair| pair.published_in_flight()); + entry.borrowed_dependencies = + entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + Build::Rebind { + pair: entry.pair.take().ok_or(EIO)?, + buffer, + pm_generation, + metrics, + } + } + } else { + let metrics = resources.metrics()?; + let pm_generation = firmware.queues.render.reserve_pm_generation()?; + Build::Fresh( + firmware.queues.reserve_render(slot, self.owner)?, + pm_generation, + metrics, + ) + }; + (resources, build) + }; + let alloc = Allocator { + dev: &self.shared.dev, + uat: &resources.uat, + }; + let fresh_graph = match &build { + Build::Fresh(..) | Build::Parked { .. } => true, + Build::Rebind { buffer, .. } => buffer.is_some(), + }; + let result = match build { + Build::Fresh(reservation, pm_generation, metrics) => { + let built = (|| -> Result> { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + Pair::new( + &alloc, + &resources.objects, + reservation.ids, + slot, + self.owner, + self.context.clone(), + free_list, + metrics, + reservation.buffer, + pm_generation, + first, + resources.config.clusters, + resources.config.descriptor_flags, + ) + })(); + let (mut built, error) = match built { + Ok(pair) => (Some(pair), None), + Err(error) => (None, Some(error)), + }; + let result = (|| { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + if firmware.queues.render.reserved(reservation)?.closed { + return Err(ECANCELED); + } + built + .as_mut() + .ok_or(EIO)? + .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; + firmware.queues.install_render(reservation, &mut built) + })(); + // Fresh backing disappears before its temporary PB reference returns. + drop(built); + if result.is_err() { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .cancel_render_reservation(reservation)?; + } + result + } + Build::Parked { + mut parked, + buffer, + pm_generation, + metrics, + } => { + let built = (|| -> Result<_> { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + let prepared = parked.prepare( + &alloc, + &resources.objects, + self.owner, + self.context.clone(), + free_list, + metrics, + buffer, + pm_generation, + first, + )?; + let storage = KBox::::new_uninit(GFP_KERNEL)?; + Ok((prepared, storage)) + })(); + let (mut prepared, mut storage, error) = match built { + Ok((prepared, storage)) => (Some(prepared), Some(storage), None), + Err(error) => (None, None, Some(error)), + }; + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + (|| { + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_owner != Some(self.owner) + || entry.pending_owner_closed + || entry.pair.is_some() + || entry.parked.is_some() + { + return Err(ECANCELED); + } + let storage = storage.take().ok_or(EIO)?; + firmware.queues.buffers.can_release(buffer.0)?; + prepared.as_mut().ok_or(EIO)?.finish_install( + &parked, + &alloc, + firmware.init.pb_descriptor_table()?, + )?; + let pair = parked.commit(&mut prepared, storage)?; + firmware.queues.buffers.reserve_owner(buffer.0)?; + firmware.queues.buffers.release(buffer.0)?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.reservation.owner = self.owner; + entry.reservation.buffer = buffer; + entry.context = Some(self.context.clone()); + entry.pool = Some(pair.pool().clone()); + entry.pair = Some(pair); + entry.closed = false; + entry.published = false; + Ok(()) + })() + }; + // Rejected candidate mappings disappear before the PB identity + // returns, while the original installed kick owner stays live. + drop(prepared); + drop(storage); + { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if result.is_err() { + firmware.queues.render.entry_mut(slot)?.parked = Some(parked); + firmware.queues.buffers.release(buffer.0)?; + } else { + drop(state); + drop(parked); + state = self.shared.state.lock(); + } + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = None; + entry.pending_owner = None; + entry.pending_owner_closed = false; + } + self.shared.changed.notify_all(); + result + } + Build::Rebind { + mut pair, + buffer, + pm_generation, + metrics, + } => { + let prepared = (|| { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + pair.prepare_bind( + &alloc, + &resources.objects, + self.owner, + self.context.clone(), + free_list, + metrics, + buffer, + pm_generation, + first, + ) + })(); + let (mut prepared, error) = match prepared { + Ok(binding) => (Some(binding), None), + Err(error) => (None, Some(error)), + }; + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + let result = (|| { + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { + return Err(ECANCELED); + } + if firmware.queues.render.retired.len() + == firmware.queues.render.retired.capacity() + { + return Err(ENOSPC); + } + firmware.queues.teardown.displace( + pair.context(), + true, + super::super::now_ns(), + )?; + pair.bind_owner( + prepared.as_mut().ok_or(EIO)?, + &alloc, + firmware.init.pb_descriptor_table()?, + )?; + if let Some((id, _)) = buffer { + firmware.queues.buffers.reserve_owner(id)?; + firmware.queues.buffers.release(id)?; + } + let entry = firmware.queues.render.entry_mut(slot)?; + entry.reservation.owner = self.owner; + if let Some(buffer) = buffer { + entry.reservation.buffer = buffer; + } + entry.context = Some(self.context.clone()); + entry.pool = Some(pair.pool().clone()); + entry.closed = false; + Ok(()) + })(); + // The original installed pair remains owned on every error. + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = None; + entry.pending_owner = None; + entry.pending_owner_closed = false; + entry.pair = Some(pair); + entry.borrowed_dependencies = None; + result + }; + drop(prepared); + if result.is_err() { + if let Some((id, _)) = buffer { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .buffers + .release(id)?; + } + } + self.shared.changed.notify_all(); + result + } + }; + result?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.queues.render.entry(slot)?.closed { + return Err(ECANCELED); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + // From this point every error leaves the complete graph retained for + // retry. Physical queue identities and installed mappings stay owned. + if fresh_graph { + firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .rebase_fresh_graph()?; + } + let pool = firmware + .queues + .render + .entry(slot)? + .pool + .clone() + .ok_or(EIO)?; + firmware.queues.render.pools.warm(&firmware.init, &pool) + } + + pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { + self.build_render_pair(slot, packet)?; + let sibling = { + let state = self.shared.state.lock(); + (*state) + .as_deref() + .ok_or(ENODEV)? + .queues + .admission + .warmable_sibling(self.owner, slot) + }; + if let Some(sibling) = sibling { + self.build_render_pair(sibling, packet)?; + } + Ok(()) + } + + pub(super) fn prepare_render( + &self, + slot: u8, + packet: &Arc, + deferred: &mut super::super::DeferredBatch, + ) -> Result { + let lease = self.shared.preparations.enter(false)?; + let (uat, mut pair, addresses, growth_target) = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + let qids = firmware + .queues + .render + .entry(slot)? + .pair + .as_ref() + .ok_or(EBUSY)? + .qids(); + let addresses = RenderAddresses { + stamps: [ + firmware.init.stamp_addresses(qids[0])?, + firmware.init.stamp_addresses(qids[1])?, + ], + }; + let pair = firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)?; + if !pair.matches(self.owner, &packet.context)? { + return Err(EIO); + } + // The selected scheduler is retained through release acknowledgement + // even if later private command allocation fails. Sibling warming + // does not cross this publication boundary. + packet.context.mark_published(); + let target = pair.host_growth_target(packet)?; + let buffer_id = pair.buffer_id(); + let growth_target = if let Some(target) = target { + let table = firmware.init.pb_descriptor_table()?; + let offset = usize::from(buffer_id) * 16; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let mut descriptor = [0; 4]; + for (index, word) in descriptor.iter_mut().enumerate() { + *word = table + .word(offset + index * 4)? + .load(core::sync::atomic::Ordering::Relaxed); + } + Some((target, descriptor)) + } else { + None + }; + let pair = firmware.queues.render.take_pair(slot, self.owner, packet)?; + (firmware.uat.clone(), pair, addresses, growth_target) + }; + let alloc = Allocator { + dev: &self.shared.dev, + uat: &uat, + }; + let mut growth = None; + let result = (|| { + if let Some((target, descriptor)) = growth_target { + let vm = pair.context().vm().clone(); + growth = pair.manager().prepare_growth( + &self.shared.dev, + &vm, + target, + Some(descriptor), + )?; + } + pair.prepare(&alloc, packet, addresses) + })(); + let mut returned = Some(pair); + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + firmware.queues.render.return_pair(slot, &mut returned)?; + if firmware.queues.render.entry(slot)?.closed { + firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .release_owner(); + } + self.shared.changed.notify_all(); + let published = (|| { + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_error.is_some() || entry.pending_timeout { + // Final timeout observations can settle unrelated queues. The + // event worker owns the full collector needed for that sweep. + self.shared.queue_events(); + return Err(EBUSY); + } + if !lease.is_current() + || firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(EAGAIN); + } + // Private restaging can fail after preparing a larger inventory. + // Finish its descriptor before discarding the command preparation. + firmware.commit_render_growth(slot, &mut growth)?; + result?; + if packet.cancel_requested() || firmware.queues.render.entry(slot)?.closed { + return Err(ECANCELED); + } + firmware.publish_render(self.owner, slot, packet, deferred) + })(); + if published.is_err() && !packet.is_published() { + firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .discard_prepared(packet)?; + } + published + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs b/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs new file mode 100644 index 0000000000000..3a5b042fff683 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Recovery targets for retained render graphs while firmware is halted. + +use crate::g17::runtime::Registry; +use kernel::prelude::*; + +impl Registry { + pub(in crate::g17::runtime) fn render_pass_started(&self, qid: u8, stamp: u64) -> Option { + self.render.entries.iter().flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid))? + .pass_started(stamp) + } + + pub(in crate::g17::runtime) fn publish_render_recovery_targets(&self) -> Option { + let mut failure = None; + for pair in self.render.entries.iter().flatten() + .filter_map(|entry| entry.pair.as_ref()) + { + if let Err(error) = pair.publish_recovery_targets() { + failure = failure.or(Some(error)); + } + } + failure + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs new file mode 100644 index 0000000000000..a6d5821561d24 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Exact packet timeout and paired firmware-drain settlement. Every fence and +//! admission release is collected under the device lock and finished outside it. + +use crate::g17::{ + job::Packet, + runtime::{Backend, DeferredBatch}, +}; +use kernel::{dma_fence::RawDmaFence, prelude::*, sync::Arc}; + +impl Backend { + pub(in crate::g17) fn timeout_render(&self, packet: &Arc) -> Result { + let mut deferred = DeferredBatch::worker()?; + let result = { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + if firmware.queues.defer_render_timeout(packet, ETIMEDOUT) { + return Ok(()); + } + firmware.fail_render_packet(packet, ETIMEDOUT, &mut deferred) + }; + deferred.finish(); + result + } + pub(in crate::g17) fn cancel_render(&self, packet: &Arc) -> Result { + let fence = packet.completion.fence(); + // SAFETY: The retained fence remains alive across this bounded, + // uninterruptible queue-teardown wait, without holding a device lock. + let completed = unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + false, + kernel::time::msecs_to_jiffies(2000) as _, + ) + }; + if completed > 0 { + return Ok(()); + } + if packet.is_published() { + self.timeout_render(packet) + } else { + // The outer callback already completed a successful 0 -> 3 + // cancellation. A claimed publisher still owns its VM pin and + // observes cancel_requested before committing or finishing. + Ok(()) + } + } +} + +impl crate::g17::Firmware { + pub(super) fn fail_render_packet( + &mut self, + packet: &Arc, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + let Some(slot) = packet.completion.render_slot() else { + return Ok(()); + }; + let slot = usize::from(slot); + loop { + let pair = self + .queues + .render + .entries + .get(slot) + .and_then(Option::as_ref) + .and_then(|entry| entry.pair.as_ref()) + .ok_or(EIO)?; + if !pair.contains_packet(packet) { + return Ok(()); + } + let generation = self.recovery.generation(); + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + let progressed = self.with_render(slot, |pair, host| { + pair.retire(None, generation, state, host, &mut |event| { + deferred.push(event) + }) + })?; + if !progressed { + break; + } + match self.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + self.with_render(slot, |pair, host| { + pair.fail_after_observation(packet, error, host, &mut |event| deferred.push(event)) + })?; + self.queues.render.collect_retired(slot)?; + let terminal = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .and_then(|pair| pair.take_terminal()); + if let Some((status, error)) = terminal { + status.record(error); + self.scan_compute(None, deferred)?; + self.settle_render_vm(&status, error, deferred)?; + self.queues.fail_compute_vm(&status, error, deferred)?; + } + Ok(()) + } + + pub(in crate::g17) fn settle_blamed_render( + &mut self, + qid: u8, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + let packet = self + .queues + .render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid)) + .and_then(|pair| pair.oldest_packet()); + if let Some(packet) = packet { + self.fail_render_packet(&packet, error, deferred)?; + } + Ok(()) + } + + pub(in crate::g17) fn settle_cancelled_renders( + &mut self, + mask: u128, + deferred: &mut DeferredBatch, + ) -> Result { + for slot in 1..self.queues.render.entries.len() { + let packet = self.queues.render.entries[slot] + .as_ref() + .and_then(|entry| entry.pair.as_ref()) + .filter(|pair| { + pair.qids() + .into_iter() + .any(|qid| mask & (1u128 << qid) != 0) + && pair.oldest_spared() == Some(true) + }) + .and_then(|pair| pair.oldest_packet()); + if let Some(packet) = packet { + self.fail_render_packet(&packet, ECANCELED, deferred)?; + } + } + Ok(()) + } + + pub(in crate::g17) fn reclaim_drained_renders( + &mut self, + deferred: &mut DeferredBatch, + ) -> Result { + if self.recovery.pending() + || self + .primary + .state + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Ok(()); + } + let mut serviced = false; + for slot in 1..self.queues.render.entries.len() { + if !self.queues.render.entries[slot] + .as_ref() + .and_then(|entry| entry.pair.as_ref()) + .is_some_and(|pair| pair.needs_drain()) + { + continue; + } + if let Err(error) = self.drain_render_slot(slot, &mut serviced, deferred) { + dev_warn!( + self.primary.state.shared.dev.as_ref(), + "Could not settle drained render pair {}: {:?}\n", + slot, + error + ); + } + } + Ok(()) + } + + fn drain_render_slot( + &mut self, + slot: usize, + serviced: &mut bool, + deferred: &mut DeferredBatch, + ) -> Result { + if !self.with_render(slot, |pair, host| pair.firmware_drained(host))? { + return Ok(()); + } + if !*serviced { + self.service_runtime_events(deferred)?; + *serviced = true; + } + if self.recovery.pending() + || self + .primary + .state + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Ok(()); + } + loop { + let accounted = self.with_render(slot, |pair, host| pair.account_drained(host))?; + if !accounted { + break; + } + match self.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + if self.recovery.pending() + || self + .primary + .state + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Ok(()); + } + self.with_render(slot, |pair, host| { + pair.release_witnessed(host, &mut |event| deferred.push(event)) + })?; + self.queues.render.collect_retired(slot) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs new file mode 100644 index 0000000000000..86a00e7bd741a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Submission admission, preparation and the firmware visibility boundary. + +use super::super::{Backend, DeferredBatch, Registry}; +use crate::g17::{ + job::Packet, + queue::admission::{RenderLease, Request}, +}; +use core::sync::atomic::Ordering; +use kernel::{ + prelude::*, + sync::Arc, + time::{delay::fsleep, Delta}, +}; + +impl Backend { + pub(in crate::g17) fn render_packet( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + self.render_with_lease(packet, None, deferred) + } + + fn render_with_lease( + &self, + packet: &Arc, + early: Option, + deferred: &mut DeferredBatch, + ) -> Result { + packet.check_dependencies()?; + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let interruptible = early.is_none(); + let admission = match early { + Some(lease) => lease, + None => { + let pool = { + let state = self.shared.state.lock(); + (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() + }; + pool.acquire_render(self.owner, Request::of(packet))? + } + }; + let slot = admission.slot(); + packet.completion.attach_render_lease(admission)?; + let mut state_polls = 0; + let mut state_delay = 20i64; + let mut takeover_polls = 0; + let mut recovery_polls = 0; + loop { + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) + { + return Err(EIO); + } + if !self.context.is_current() { + return Err(EFAULT); + } + let work_state = { + let preparation = self.shared.preparations.enter(interruptible)?; + let result = packet.completion.work_state().acquire(); + if preparation.is_current() { + result + } else { + Err(EAGAIN) + } + }; + match work_state { + Err(error) if error == EAGAIN => continue, + Err(error) if error == EBUSY => { + if state_polls == 1000 { + return Err(ETIMEDOUT); + } + state_polls += 1; + fsleep(Delta::from_micros(state_delay)); + state_delay = (state_delay * 2).min(1000); + continue; + } + result => { + result?; + } + } + match self.ensure_render(packet, slot) { + Err(error) if error == EAGAIN => continue, + Err(error) if error == EBUSY => { + if takeover_polls == 5000 { + return Err(ETIMEDOUT); + } + takeover_polls += 1; + fsleep(Delta::from_micros(200)); + continue; + } + result => result?, + } + match self.prepare_render(slot, packet, deferred) { + Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { + let recovering = { + let state = self.shared.state.lock(); + let firmware = (*state).as_deref().ok_or(ENODEV)?; + firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + }; + if recovering { + if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { + return Err(ETIMEDOUT); + } + recovery_polls += 1; + self.shared.queue_events(); + fsleep(Delta::from_millis(1)); + } else if error == EBUSY { + if takeover_polls == 5000 { + return Err(ETIMEDOUT); + } + takeover_polls += 1; + fsleep(Delta::from_micros(200)); + } + } + result => return result, + } + } + } + + pub(in crate::g17) fn try_render_early(&self, packet: &Arc) -> bool { + if self.backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { + return false; + } + let admission = { + let state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref() else { + return false; + }; + match firmware + .queues + .admission + .try_acquire_render(self.owner, Request::of(packet)) + { + Ok(Some(lease)) => lease, + _ => return false, + } + }; + if !packet.begin_publication() { + packet.completion.fail_unpublished(ECANCELED); + packet.set_early_result(Err(ECANCELED)); + return true; + } + let mut deferred = match DeferredBatch::publication() { + Ok(deferred) => deferred, + Err(error) => { + if packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + packet.completion.fail_unpublished(error); + packet.set_early_result(Err(error)); + return true; + } + }; + let result = self.render_with_lease(packet, Some(admission), &mut deferred); + let result = self + .finish_publication(packet, result, &mut deferred) + .map(|_| ()); + packet.set_early_result(result); + true + } +} + +impl Registry { + fn render_dependencies( + &self, + owner: u64, + packet: &Arc, + state: u32, + dependencies: &mut [crate::g17::fw::kick::KickDependency; 3], + ) -> Result { + let mut count = 0; + let frontier = self.render_prefix(owner, packet)?; + for dependency in frontier.as_slice() { + *dependencies.get_mut(count).ok_or(EOVERFLOW)? = *dependency; + count += 1; + } + if let Some(queue) = self.compute(owner) { + if let Some(dependency) = queue.dependency( + owner, + packet.completion.status(), + packet.order.wait_through[1], + state, + )? { + *dependencies.get_mut(count).ok_or(EOVERFLOW)? = dependency; + count += 1; + } + } + Ok(count) + } +} + +impl crate::g17::Firmware { + pub(super) fn apply_render_deferred( + &mut self, + slot: u8, + deferred: &mut DeferredBatch, + ) -> Result { + let entry = self.queues.render.entry_mut(slot)?; + let pair = entry.pair.as_mut().ok_or(EIO)?; + if let Some((status, error)) = entry.pending_error.take() { + if Arc::ptr_eq(pair.context().status(), &status) { + pair.quarantine(error, &mut |event| deferred.push(event))?; + } + } + if entry.closed { + pair.release_owner(); + } + if !entry.pending_timeout { + return Ok(()); + } + for _ in 0..crate::hw::t8140::queues::RENDER_DEPTH { + if self.recovery.pending() || self.primary.state.shared.crashed.load(Ordering::Acquire) + { + return Ok(()); + } + let Some((packet, error)) = self + .queues + .render + .entry(slot)? + .pair + .as_ref() + .ok_or(EIO)? + .take_deferred_timeout() + else { + self.queues.render.entry_mut(slot)?.pending_timeout = false; + return Ok(()); + }; + if let Err(failure) = self.fail_render_packet(&packet, error, deferred) { + packet.completion.defer_timeout(error); + return Err(failure); + } + } + self.queues.render.entry_mut(slot)?.pending_timeout = false; + Ok(()) + } + + pub(super) fn publish_render( + &mut self, + owner: u64, + slot: u8, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + use crate::g17::{fw::kick::KickDependency, queue::render::PublicationLease}; + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + if self.recovery.pending() || state != 0 { + return Err(EAGAIN); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let mut dependencies = [KickDependency::ZERO; 3]; + let count = self + .queues + .render_dependencies(owner, packet, state, &mut dependencies)?; + let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; + let pool = pair.pool().clone(); + let buffer_id = pair.buffer_id(); + let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; + let preparation = match self.queues.render.pools.prepare(&self.init, &pool) { + Ok(preparation) => preparation, + Err(error) => { + self.queues.buffers.release(buffer_id)?; + return Err(error); + } + }; + let lease = PublicationLease { + free_list_generation: preparation.generation, + buffer_token, + }; + let result = self.with_render(usize::from(slot), |pair, host| { + pair.publish( + packet.clone(), + &dependencies[..count], + lease, + host, + &mut |event| deferred.push(event), + ) + }); + if packet.is_published() { + self.queues.render.entry_mut(slot)?.published = true; + } else if result.is_err() { + self.queues.render.pools.rollback(preparation); + self.queues.buffers.release(buffer_id)?; + self.queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .discard_prepared(packet)?; + } + result + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs new file mode 100644 index 0000000000000..759c0c2e43dc0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Exact logical ownership gates for scheduler release and physical-pair parking. + +use super::RenderState; +use crate::g17::{ + buffer::BufferIds, + context::Context, + initdata::InitData, + runtime::{teardown::ReclaimBatch, Registry}, +}; +use kernel::{prelude::*, sync::Arc}; + +impl RenderState { + pub(in crate::g17::runtime) fn observe_teardown_releases(&mut self, init: &InitData) -> Result { + self.pools.observe(init) + } + + pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { + for entry in self.entries.iter().flatten() { + if !entry + .context + .as_ref() + .is_some_and(|owner| Arc::ptr_eq(owner, context)) + { + continue; + } + if entry.preparing.is_some() || entry.growing { + return Ok(false); + } + if let Some(pair) = entry.pair.as_ref() { + if !pair.idle_for_context(context)? { + return Ok(false); + } + } + } + Ok(true) + } + + pub(in crate::g17::runtime) fn teardown_pools_released(&self, context: &Arc) -> bool { + context.render_pools_released(|id, control| self.pools.released(id, control)) + && self + .retired + .iter() + .filter(|old| Arc::ptr_eq(old.context(), context)) + .all(|old| { + self.pools + .released(u32::from(old.pool().id()), old.pool().control_va()) + }) + } + + /// Warm superseded roots need only their exact executed replacement witness. + /// Graphs with their own memory remain until that Context's scheduler ack. + pub(in crate::g17::runtime) fn collect_reclaimable( + &mut self, + out: &mut ReclaimBatch, + ) -> Result { + for slot in 0..self.entries.len() { + self.collect_retired(slot)?; + } + let mut index = 0; + while index < self.retired.len() { + if self.retired[index].buffer_id().is_some() { + index += 1; + continue; + } + let Ok(target) = out.binding_slot() else { + break; + }; + *target = Some(self.retired.remove(index).map_err(|_| EIO)?); + } + Ok(()) + } + + /// Preflight the entire Context before detaching its first graph. Successor + /// graphs on the same VM have a different Arc and remain installed. + pub(in crate::g17::runtime) fn detach_context( + &mut self, + context: &Arc, + buffers: &mut BufferIds, + out: &mut ReclaimBatch, + ) -> Result { + let mut count = self + .retired + .iter() + .filter(|old| Arc::ptr_eq(old.context(), context)) + .count(); + for entry in self.entries.iter().flatten() { + if !entry + .context + .as_ref() + .is_some_and(|owner| Arc::ptr_eq(owner, context)) + { + continue; + } + if entry.preparing.is_some() || entry.growing || entry.parked.is_some() { + return Err(EBUSY); + } + if let Some(pair) = entry.pair.as_ref() { + if !pair.can_park(context)? { + return Err(EBUSY); + } + count += 1; + } + } + if !out.has_room(count, 1) { + return Err(EBUSY); + } + for entry in self.entries.iter_mut().flatten() { + if !entry + .context + .as_ref() + .is_some_and(|owner| Arc::ptr_eq(owner, context)) + { + continue; + } + let Some(pair) = entry.pair.as_ref() else { + continue; + }; + buffers.unreserve_owner(pair.buffer_id())?; + let target = out.binding_slot()?; + let pair = entry.pair.take().ok_or(EIO)?; + let (parked, retired) = KBox::into_inner(pair).park(); + *target = Some(retired); + entry.parked = Some(parked); + // Detached backing retains these references until the worker runs. + entry.context = None; + entry.pool = None; + } + let mut index = 0; + while index < self.retired.len() { + if !Arc::ptr_eq(self.retired[index].context(), context) { + index += 1; + continue; + } + if let Some(id) = self.retired[index].buffer_id() { + buffers.unreserve_owner(id)?; + } + let target = out.binding_slot()?; + *target = Some(self.retired.remove(index).map_err(|_| EIO)?); + } + Ok(()) + } +} + +impl Registry { + /// Close while a builder owns a pair records intent. Its return path applies + /// the release before admitting further commands from either logical owner. + pub(in crate::g17::runtime) fn release_render(&mut self, owner: u64) -> Result { + for entry in self.render.entries.iter_mut().flatten() { + if entry.pending_owner == Some(owner) { + entry.pending_owner_closed = true; + } + if entry.reservation.owner != owner { + continue; + } + entry.closed = true; + if let Some(pair) = entry.pair.as_mut() { + pair.release_owner(); + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs new file mode 100644 index 0000000000000..10f334e4fe561 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scheduler release and off-lock destruction of acknowledged logical queue resources. + +use super::{ComputeEntry, Registry}; +use crate::g17::{ + buffer::BufferIds, + channel::Rings, + context::Context, + fw::channels::{ControlRecord, SchedulerStateRelease}, + initdata::InitData, + queue::render::RetiredBinding, + recovery::Memory as _, + teardown::{Failure, Host}, + Coprocessor, Firmware, MSG_CONTROL_NOTIFY, +}; +use crate::hw::t8140::queues::RENDER_SLOTS; +use core::sync::atomic::Ordering; +use kernel::{prelude::*, sync::Arc}; + +const CONTEXTS: usize = 128; + +/// Pop under the reclaim mutex, then finish after dropping every device lock. +pub(in crate::g17) enum Reclaim { + Binding(RetiredBinding), + Context(Arc), +} + +impl Reclaim { + pub(in crate::g17) fn finish(self) { + match self { + Self::Binding(binding) => drop(binding), + Self::Context(context) => drop(context), + } + } +} + +/// All slots are allocated at probe. A full collector postpones detach. Empty +/// slots let a validated transfer move ownership without a fallible vector push. +pub(in crate::g17) struct ReclaimBatch { + bindings: KVVec>, + contexts: KVec>>, +} + +impl ReclaimBatch { + pub(in crate::g17) fn new() -> Result { + let mut bindings = KVVec::with_capacity(CONTEXTS + RENDER_SLOTS, GFP_KERNEL)?; + for _ in 0..CONTEXTS + RENDER_SLOTS { + bindings.push(None, GFP_KERNEL)?; + } + let mut contexts = KVec::with_capacity(CONTEXTS, GFP_KERNEL)?; + for _ in 0..CONTEXTS { + contexts.push(None, GFP_KERNEL)?; + } + Ok(Self { bindings, contexts }) + } + + pub(in crate::g17) fn is_empty(&self) -> bool { + self.bindings.iter().all(Option::is_none) && self.contexts.iter().all(Option::is_none) + } + + /// Graphs go before the Context owners that armed their scheduler releases. + pub(in crate::g17) fn take_one(&mut self) -> Option { + if let Some(slot) = self.bindings.iter_mut().find(|slot| slot.is_some()) { + return slot.take().map(Reclaim::Binding); + } + self.contexts + .iter_mut() + .find(|slot| slot.is_some()) + .and_then(Option::take) + .map(Reclaim::Context) + } + + pub(in crate::g17::runtime) fn has_room(&self, bindings: usize, contexts: usize) -> bool { + self.bindings.iter().filter(|slot| slot.is_none()).count() >= bindings + && self.contexts.iter().filter(|slot| slot.is_none()).count() >= contexts + } + + pub(in crate::g17::runtime) fn binding_slot(&mut self) -> Result<&mut Option> { + self.bindings + .iter_mut() + .find(|slot| slot.is_none()) + .ok_or(EBUSY) + } + + fn context(&mut self, context: &Arc) -> Result { + let slot = self + .contexts + .iter_mut() + .find(|slot| slot.is_none()) + .ok_or(EBUSY)?; + *slot = Some(context.clone()); + Ok(()) + } +} + +struct Service<'a> { + render: &'a mut super::render::RenderState, + compute: &'a [Option], + buffers: &'a mut BufferIds, + init: &'a InitData, + primary: &'a mut Coprocessor, + reclaim: &'a mut ReclaimBatch, +} + +impl Host for Service<'_> { + fn render_idle(&mut self, context: &Arc) -> Result { + self.render.teardown_idle(context) + } + fn compute_idle(&self, context: &Arc) -> bool { + self.compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_ref()) + .all(|queue| queue.idle_for_context(context)) + } + fn flists_released(&self, context: &Arc) -> bool { + self.render.teardown_pools_released(context) + } + fn control_backpressured(&self) -> bool { + self.render.control_backpressured() + } + fn publish(&mut self, record: &SchedulerStateRelease) -> Result { + Rings::publish_control( + self.init, + &ControlRecord::SchedulerStateRelease(SchedulerStateRelease::new( + record.page_va, + record.id, + )), + ) + } + fn notify(&mut self) -> Result { + self.primary.notify(MSG_CONTROL_NOTIFY) + } + fn detach(&mut self, context: &Arc) -> Result { + self.render + .detach_context(context, self.buffers, self.reclaim)?; + // detach_context preflights this slot together with every graph slot. + // Retain it before Pending removes its reference under the device lock. + self.reclaim.context(context) + } + fn report(&mut self, context: &Arc, failure: Failure) { + let dev = self.primary.state.shared.dev.as_ref(); + match failure { + Failure::Publication(error) => dev_err!( + dev, + "Scheduler {} release failed: {:?}\n", + context.id(), + error + ), + Failure::Announcement(error) => dev_warn!( + dev, + "Scheduler {} release announcement failed: {:?}\n", + context.id(), + error + ), + Failure::Stalled => { + dev_warn!(dev, "Scheduler {} release is still pending\n", context.id()) + } + } + } +} + +impl Firmware { + /// The event worker has already reclaimed drained pairs and reannounced + /// pending controls. Recovery blocks new releases, never an observed ack. + pub(in crate::g17) fn service_teardown(&mut self, reclaim: &mut ReclaimBatch) -> Result { + self.queues.render.collect_reclaimable(reclaim)?; + if self.queues.teardown.is_empty() { + return Ok(()); + } + self.queues.render.observe_teardown_releases(&self.init)?; + let recovery = self.recovery.pending() || self.init.recovery_state()? != 0; + let Registry { + render, + compute, + buffers, + teardown, + .. + } = &mut self.queues; + let mut service = Service { + render, + compute, + buffers, + init: &self.init, + primary: &mut self.primary, + reclaim, + }; + teardown.publish_ready(&mut service, recovery)?; + let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); + let producer = self.init.control_producer()?.load(Ordering::Relaxed); + while teardown + .take_consumed(consumer, producer, &mut service)? + .is_some() + {} + teardown.report_stalled(super::now_ns(), &mut service); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs new file mode 100644 index 0000000000000..178d275da9cf4 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Sticky VM errors and their optional userspace mirror. No firmware state is accessed here. + +use crate::gem; +use core::cell::UnsafeCell; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicPtr, Ordering}; +use kernel::prelude::*; + +const MIRROR_MAX_SIZE: usize = 64 * 1024; + +/// A VM and its accepted jobs share this status, including after queue destruction. +/// The first error is permanent. Callers record it before signalling the job's fence. +pub(crate) struct VmStatus { + error: AtomicI32, + mirror: AtomicPtr, + mirror_claimed: AtomicBool, + mirror_owner: UnsafeCell>, +} + +// SAFETY: Only the winner of mirror_claimed writes mirror_owner. The owner is never +// read or replaced while the status lives, and its mapping is retained until drop. +// Other shared accesses use atomics, including accesses through the mirror pointer. +unsafe impl Sync for VmStatus {} +// SAFETY: The retained GEM reference and mapping can move between threads. Moving +// the status does not move that mapping, and its remaining fields are atomic. +unsafe impl Send for VmStatus {} + +impl VmStatus { + pub(crate) fn new() -> Self { + Self { + error: AtomicI32::new(0), + mirror: AtomicPtr::new(core::ptr::null_mut()), + mirror_claimed: AtomicBool::new(false), + mirror_owner: UnsafeCell::new(None), + } + } + + pub(crate) fn get(&self) -> i32 { + self.error.load(Ordering::Acquire) + } + + /// Store the first error in both the status and any registered mirror. + pub(crate) fn record(&self, error: Error) { + let error = match self.error.compare_exchange( + 0, + error.to_errno(), + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => error.to_errno(), + Err(first) => first, + }; + let word = self.mirror.load(Ordering::SeqCst); + if !word.is_null() { + // SAFETY: attach_mirror publishes only an aligned word within a mapping + // owned by this status. Every kernel access to this word is atomic. + unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); + } + } + + /// Rejected submissions and contention leave the VM usable. + pub(crate) fn record_if_device_loss(&self, error: Error) { + if error == EIO || error == ETIMEDOUT || error == ENODEV { + self.record(error); + } + } + + /// Retain one mapped GEM word until the last reference to this status is dropped. + pub(crate) fn attach_mirror(&self, mut owner: gem::ObjectRef, offset: usize) -> Result { + if owner.size() > MIRROR_MAX_SIZE + || offset % core::mem::align_of::() != 0 + || offset + .checked_add(core::mem::size_of::()) + .is_none_or(|end| end > owner.size()) + { + return Err(EINVAL); + } + let base = owner.vmap()?.as_mut_ptr(); + // SAFETY: The checked range is inside the retained, page-aligned mapping. + let word = unsafe { base.add(offset) }.cast::(); + if self.mirror_claimed.swap(true, Ordering::AcqRel) { + return Err(EBUSY); + } + // SAFETY: This is the only thread that may write mirror_owner. It is not + // read again until the status is dropped, after all shared references end. + unsafe { *self.mirror_owner.get() = Some(owner) }; + // SAFETY: The retained owner keeps this checked, aligned word mapped. + unsafe { AtomicI32::from_ptr(word) }.store(0, Ordering::SeqCst); + + // The pointer publication and error update form a store/load handshake: + // either this load sees record's error or record sees our pointer. Stores + // after publishing the pointer write only the same first error, never zero. + self.mirror.store(word, Ordering::SeqCst); + let error = self.error.load(Ordering::SeqCst); + if error != 0 { + // SAFETY: The mapping remains owned by this status as above. + unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs new file mode 100644 index 0000000000000..9acc01fbc2125 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Retained scheduler-state releases. Logical close arms a release; render and compute drain, +//! private-memory release and control-ring consumption are separate ownership witnesses. +//! The device mutex serializes this state with the Registry's displaced graph ownership. + +use super::{context::Context, freelist::control_consumed, fw::channels::SchedulerStateRelease}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +const CAPACITY: usize = 128; +const POLL_NS: u64 = 2_000_000_000; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Phase { + Deferred, + Draining, + Released(u32), +} + +struct Entry { + context: Arc, + phase: Phase, + armed: u64, + warned: bool, +} + +/// Publication and announcement errors retain the entry. A stalled release is reported once; +/// its timeout never authorizes freeing memory still owned by firmware. +#[derive(Clone, Copy)] +pub(crate) enum Failure { + Publication(Error), + Announcement(Error), + Stalled, +} + +/// Borrowed device operations. Readiness covers every matching installed and displaced graph, +/// including pools kept by the Context after a same-VM successor took over its physical pair. +/// `detach` must retain all detached backing for off-lock destruction before returning success. +/// It compares logical Arc identity so a successor sharing an execution ID remains installed. +pub(crate) trait Host { + fn render_idle(&mut self, context: &Arc) -> Result; + fn compute_idle(&self, context: &Arc) -> bool; + fn flists_released(&self, context: &Arc) -> bool; + fn control_backpressured(&self) -> bool; + fn publish(&mut self, record: &SchedulerStateRelease) -> Result; + fn notify(&mut self) -> Result; + fn detach(&mut self, context: &Arc) -> Result; + fn report(&mut self, context: &Arc, failure: Failure); +} + +pub(crate) struct Pending { + entries: KVec, +} + +impl Pending { + pub(crate) fn new() -> Result { + Ok(Self { + entries: KVec::with_capacity(CAPACITY, GFP_KERNEL)?, + }) + } + + pub(crate) fn closed(&self, index: usize, now: u64) -> Option<(Arc, u64)> { + self.entries + .get(index) + .filter(|entry| entry.phase == Phase::Draining) + .map(|entry| (entry.context.clone(), now.saturating_sub(entry.armed))) + } + + pub(crate) fn len(&self) -> usize { + self.entries.len() + } + + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn find(&self, context: &Arc) -> Option { + self.entries + .iter() + .position(|entry| Arc::ptr_eq(&entry.context, context)) + } + + fn insert(&mut self, context: &Arc, phase: Phase, now: u64) -> Result { + self.entries + .push_within_capacity(Entry { + context: context.clone(), + phase, + armed: now, + warned: false, + }) + .map_err(|_| ENOSPC) + } + + /// Call before replacing a graph's logical owner. Failure leaves that graph with its + /// original owner. The Registry, rather than this state machine, retains displaced backing. + pub(crate) fn displace( + &mut self, + context: &Arc, + owner_closed: bool, + now: u64, + ) -> Result { + if self.find(context).is_some() { + return Ok(()); + } + let phase = if owner_closed { + Phase::Draining + } else { + Phase::Deferred + }; + self.insert(context, phase, now) + } + + /// No allocation is needed at close. A scheduler page never exposed to firmware requires + /// no release, unless the Registry already retained one of its displaced graphs. + pub(crate) fn close(&mut self, context: &Arc, now: u64) -> Result { + if let Some(index) = self.find(context) { + let entry = &mut self.entries[index]; + if entry.phase == Phase::Deferred { + entry.phase = Phase::Draining; + entry.armed = now; + } + return Ok(()); + } + if context.is_published() { + self.insert(context, Phase::Draining, now)?; + } + Ok(()) + } + + /// Publish in insertion order. Mandatory control replies and a full control ring stop this + /// pass; an error for one page leaves it retained while other ready releases may progress. + pub(crate) fn publish_ready(&mut self, host: &mut impl Host, recovery: bool) -> Result { + if !recovery { + for entry in self.entries.iter_mut() { + if entry.phase != Phase::Draining + || !host.render_idle(&entry.context)? + || !host.flists_released(&entry.context) + || !host.compute_idle(&entry.context) + { + continue; + } + if host.control_backpressured() { + break; + } + let result = (|| { + let id = entry.context.release_identity()?; + let record = SchedulerStateRelease::new( + entry.context.scheduler_va(), + [id[3], id[0], id[1], id[2]], + ); + host.publish(&record) + })(); + let next = match result { + Ok(next) => next, + Err(EAGAIN) => break, + Err(error) => { + host.report(&entry.context, Failure::Publication(error)); + continue; + } + }; + entry.phase = Phase::Released(next); + fence(Ordering::SeqCst); + if let Err(error) = host.notify() { + host.report(&entry.context, Failure::Announcement(error)); + } + } + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// The caller snapshots both control cursors once for this settlement pass and repeatedly + /// takes consumed entries. It retains each returned Context until its detached graphs and + /// VM aliases can be destroyed outside the device mutex. + pub(crate) fn take_consumed( + &mut self, + consumer: u32, + producer: u32, + host: &mut impl Host, + ) -> Result>> { + for index in 0..self.entries.len() { + let entry = &self.entries[index]; + let Phase::Released(target) = entry.phase else { + continue; + }; + if !control_consumed(consumer, producer, target)? + || !host.flists_released(&entry.context) + { + continue; + } + match host.detach(&entry.context) { + Ok(()) => {} + Err(EBUSY) => continue, + Err(error) => return Err(error), + } + let entry = self.entries.remove(index).map_err(|_| EIO)?; + return Ok(Some(entry.context)); + } + Ok(None) + } + + /// Poll quickly only during the initial release window; late arrivals still settle when + /// serviced. Time passage never substitutes for a firmware-consumption witness. + pub(crate) fn polling(&self, now: u64) -> bool { + self.entries.iter().any(|entry| { + entry.phase != Phase::Deferred && now.saturating_sub(entry.armed) < POLL_NS + }) + } + + pub(crate) fn report_stalled(&mut self, now: u64, host: &mut impl Host) { + for entry in self.entries.iter_mut() { + if entry.phase != Phase::Deferred + && !entry.warned + && now.saturating_sub(entry.armed) >= POLL_NS + { + entry.warned = true; + host.report(&entry.context, Failure::Stalled); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs new file mode 100644 index 0000000000000..da43c10cd3bef --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Reuse address-coverage proofs only while the VM's page tables are unchanged. + +use super::command::{Access, AddressSpace, Payload, UscWindow, Validated}; +use crate::mmu; +use core::{cell::RefCell, ops::Range}; +use kernel::prelude::*; + +const MAX_RANGES: usize = 48; +type Request = (u64, u64, u8); + +#[derive(Clone, Copy)] +struct Requests { + ranges: [Request; MAX_RANGES], + count: usize, + overflow: bool, +} + +impl Requests { + const fn new() -> Self { + Self { + ranges: [(0, 0, 0); MAX_RANGES], + count: 0, + overflow: false, + } + } + + fn as_slice(&self) -> &[Request] { + &self.ranges[..self.count] + } + + fn insert(&mut self, request: Request) { + if self.as_slice().contains(&request) { + return; + } + if self.count == MAX_RANGES { + self.overflow = true; + return; + } + self.ranges[self.count] = request; + self.count += 1; + } +} + +/// One logical queue's bounded cache of exact address, size and permission tuples. +/// Mapping mutations make the epoch unavailable until their final page-table write. +pub(crate) struct Proof { + epoch: Option, + requests: Requests, +} + +impl Proof { + pub(crate) const fn new() -> Self { + Self { + epoch: None, + requests: Requests::new(), + } + } + + fn covers(&self, epoch: Option, requests: &Requests) -> bool { + self.epoch.is_some() + && self.epoch == epoch + && requests + .as_slice() + .iter() + .all(|request| self.requests.as_slice().contains(request)) + } + + fn record(&mut self, epoch: Option, requests: Requests) { + // Alternating in-flight buffers must not evict each other's proofs. + if epoch.is_some() && self.epoch == epoch && !requests.overflow { + for &request in requests.as_slice() { + self.requests.insert(request); + } + if !self.requests.overflow { + return; + } + } + // An incomplete union cannot be reused; retain the latest complete set. + self.epoch = if requests.overflow { None } else { epoch }; + self.requests = requests; + } + + /// The VM-job guard must already pin the mappings for this command. Keep the + /// range arrays off the caller's frame before descending into publication. + #[inline(never)] + pub(crate) fn validate( + &mut self, + vm: &mmu::Vm, + range: Range, + window: &UscWindow, + payload: &Payload, + ) -> Result { + let requests = RefCell::new(Requests::new()); + let optimistic = payload.validate(window, &Space::Record(&range, &requests)); + let requests = requests.into_inner(); + if optimistic.is_ok() && !requests.overflow { + if self.covers(vm.mapping_validation_epoch(), &requests) { + return optimistic; + } + let (covered, epoch) = vm.covers_ranges_batch(requests.as_slice()); + if covered { + self.record(epoch, requests); + return optimistic; + } + self.epoch = None; + } + // Preserve per-range validation's failure and concurrent-mutation behavior. + payload.validate(window, &Space::Checked(&range, vm)) + } +} + +enum Space<'a> { + Record(&'a Range, &'a RefCell), + Checked(&'a Range, &'a mmu::Vm), +} + +impl AddressSpace for Space<'_> { + fn covers(&self, address: u64, size: u64, access: Access) -> bool { + let range = match self { + Self::Record(range, _) | Self::Checked(range, _) => range, + }; + let Some(end) = address.checked_add(size) else { + return false; + }; + if size == 0 || address < range.start || end > range.end { + return false; + } + let permissions = match access { + Access::Read => 1, + Access::Write => 2, + Access::ReadWrite => 3, + }; + match self { + Self::Record(_, requests) => { + requests.borrow_mut().insert((address, size, permissions)); + true + } + Self::Checked(_, vm) => { + vm.covers_range(address, size, permissions & 1 != 0, permissions & 2 != 0) + } + } + } +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 12966cb98ba3d..1f766982074e3 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -158,11 +158,23 @@ pub(crate) struct AsahiObjConfig { /// Create a new kernel-owned GEM object. pub(crate) fn new_kernel_object(dev: &AsahiDevice, size: usize) -> Result { + new_kernel_object_mapped(dev, size, false) +} + +/// Create a new kernel-owned GEM object whose kernel mapping is write-combined. +/// +/// Used for memory that a non-coherent agent reads as soon as the host has stored to it and +/// issued a barrier, without cache maintenance. +pub(crate) fn new_kernel_object_wc(dev: &AsahiDevice, size: usize) -> Result { + new_kernel_object_mapped(dev, size, true) +} + +fn new_kernel_object_mapped(dev: &AsahiDevice, size: usize, map_wc: bool) -> Result { let gem = shmem::Object::::new( dev, align(size, mmu::UAT_PGSZ), shmem::ObjectConfig:: { - map_wc: false, + map_wc, parent_resv_obj: None, }, AsahiObjConfig { diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 0d966161f9e5d..34823251fb755 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -279,6 +279,15 @@ pub(crate) trait Gpu: Send + Sync { fn is_crashed(&self) -> bool; /// Fill in the hardware description reported to userspace. fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; + /// Whether user VMs retain a sticky error and support a userspace mirror. + fn supports_vm_status(&self) -> bool { + false + } + /// Limits for independently executing queues, if supported by this GPU. + fn queue_limits(&self) -> Option { + None + } + /// Frequency of the CPU timer that GPU timestamps are derived from. fn base_clock_hz(&self) -> u32; /// Return the GPU address space geometry. @@ -689,6 +698,7 @@ impl GpuManager::ver { oas: cfg.uat_oas, }, map_kernel_to_user, + mmu::UatFirmware::Handoff, )?, GFP_KERNEL, )?) diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index cf5407ecde438..9fbf1236bdd84 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -15,6 +15,7 @@ pub(crate) mod t600x; pub(crate) mod t602x; pub(crate) mod t8103; pub(crate) mod t8112; +pub(crate) mod t8140; /// GPU generation enumeration. Note: Part of the UABI. #[derive(Debug, PartialEq, Copy, Clone)] @@ -22,6 +23,7 @@ pub(crate) mod t8112; pub(crate) enum GpuGen { G13 = 13, G14 = 14, + G17 = 17, } /// GPU variant enumeration. Note: Part of the UABI. diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs new file mode 100644 index 0000000000000..4a97217a00705 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t8140 platforms (A18 Pro). + +use crate::g17::{ + Config, + FixedBuffer, + Identity, // +}; +use crate::hw::GpuVariant; + +/// Static configuration of the T8140 GPU. +pub(crate) const CONFIG: Config = Config { + chip_id: CHIP_ID, + identity: Identity { + family: 0xa, + variant: 0, + num_dies: 1, + gpu_variant: GpuVariant::P, + usc_generation: 3, + hal_generation: 200, + }, + uat_ias: 42, + uat_oas: 42, + // The secondary coprocessor's kernel root follows the primary one's 256 KiB of page tables. + secondary_root_offset: 0x40000, + // The coprocessors request their crash log buffers at these addresses, in this order, and + // expect them to be mapped before they start. + crash_buffers: [ + FixedBuffer { + va: 0x01e1_c000, + size: 0x8000, + }, + FixedBuffer { + va: 0x01f5_0000, + size: 0x8000, + }, + ], + // Read by the firmware through the kernel lower root, whose top-level entries it only + // samples when it starts. + pm_metrics: FixedBuffer { + va: 0x10_0008_0000, + size: 0x8000, + }, +}; + +use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; + +/// Chip identifier written to the hardware data. +pub(crate) const CHIP_ID: u32 = 0x8140; + +/// Base of the firmware MMIO aperture in the kernel window. +const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; + +const fn window( + slot: usize, + phys: u64, + offset: u64, + size: u32, + unk_18: u64, + unk_20: u32, +) -> RegisterWindow { + RegisterWindow { + slot, + phys, + va: MMIO_APERTURE + offset, + size, + unk_18, + unk_20, + } +} + +/// Register windows the firmware maps, by register-map slot. +pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 18] = [ + window(0, 0x3_0101_4000, 0x0_0000, 0x4000, 0, 2), + window(3, 0x2_2010_4000, 0x0_8000, 0x1_8000, 0, 2), + window(9, 0x3_003d_0000, 0x2_8000, 0x1000, 0, 2), + window(10, 0x3_003c_0000, 0x3_0000, 0x2000, 0, 0), + window(12, 0x4_0165_c000, 0x3_8000, 0x4000, 0, 2), + window(14, 0x3_0028_0000, 0x4_0000, 0x8000, 0, 0), + window(17, 0x4_8000_0000, 0x5_0000, 0x2_1400, 0, 2), + window(22, 0x4_8100_0000, 0x7_8000, 0x8000, 0, 2), + window(26, 0x4_80d0_4000, 0x8_8000, 0x8000, 0xd0_4000, 2), + window(27, 0x4_80d0_d000, 0x9_9000, 0x1000, 0xd0_d000, 2), + window(28, 0x4_80d5_8000, 0xa_0000, 0x8000, 0xd5_8000, 2), + window(29, 0x4_80d1_0000, 0xb_0000, 0x4000, 0xd1_0000, 2), + window(31, 0x4_80d4_0000, 0xb_8000, 0x4000, 0xd4_0000, 2), + window(32, 0x4_80d6_0000, 0xc_0000, 0x4000, 0xd6_0000, 2), + window(35, 0x4_80e0_0000, 0xc_8000, 0x4000, 0xe0_0000, 2), + window(39, 0x4_80e0_8000, 0xd_0000, 0x8000, 0, 2), + window(40, 0x4_80e1_c000, 0xe_0000, 0x4000, 0xe1_c000, 2), + window(41, 0x4_80e1_f800, 0xe_b800, 0x4000, 0, 2), +]; + +/// Register-map slots that are reserved but have no window. +pub(crate) const RESERVED_REGISTER_SLOTS: [usize; 17] = [ + 2, 5, 6, 7, 8, 30, 33, 34, 37, 38, 42, 43, 46, 47, 48, 49, 52, +]; + +/// Performance-table configuration. The device tree lists the operating +/// points in performance-state order; table A uses the odd states above +/// state 5, table B the even ones. +pub(crate) const PERF: PerfConfig = PerfConfig { + states_a: [0, 1, 2, 3, 4, 5, 7, 9, 11, 13, 15], + states_b: [0, 1, 2, 3, 4, 5, 6, 8, 10, 12, 14], + relative_a: [0, 15, 22, 29, 42, 56, 74, 86, 90, 92, 100], + relative_b: [0, 0, 13, 24, 40, 52, 63, 73, 83, 88, 100], + sram_min_mv: 765, +}; + +/// Operating points the performance tables reference. +pub(crate) const OPPS: usize = 16; + +const _: () = { + let mut index = 0; + while index < PERF_STATES { + assert!((PERF.states_a[index] as usize) < OPPS); + assert!((PERF.states_b[index] as usize) < OPPS); + index += 1; + } +}; + +/// Offsets of the fixed objects in the kernel window, in the order the +/// builder creates them. +pub(crate) mod kernel_window { + /// Private cluster (`initdata::private`). + pub(crate) const PRIVATE_CLUSTER: u64 = 0x2_0000; + /// Zeroed firmware-private page. + pub(crate) const ZERO_PAGE: u64 = 0x15d_8000; + /// Parameter-buffer descriptor table. + pub(crate) const PB_DESCRIPTORS: u64 = 0x15e_0000; + /// page-pool descriptor table. + pub(crate) const PAGE_POOL_DESCRIPTORS: u64 = 0x15e_8000; + /// Index object (64 KiB, `initdata::IndexTable` in its first page). + pub(crate) const INDEX: u64 = 0xc084_8000; + /// Firmware-control page. + pub(crate) const FWCTL: u64 = 0x1c_0000; + /// Render free-list control and state pages. + pub(crate) const RENDER_FREE_LIST: u64 = 0xc083_0000; + pub(crate) const RENDER_FREE_LIST_STATE: u64 = 0x160_8000; + /// Boot free-list control and state pages. + pub(crate) const BOOT_FREE_LIST: u64 = 0xc082_8000; + pub(crate) const BOOT_FREE_LIST_STATE: u64 = 0x160_0000; + /// Zeroed descriptor pages. + pub(crate) const DESCRIPTOR_PAGE_A: u64 = 0x1c_8000; + pub(crate) const DESCRIPTOR_PAGE_B: u64 = 0xc07c_0000; + /// Zeroed TA and 3D context pages (128 KiB each). + pub(crate) const TA_CONTEXT: u64 = 0x1d_8000; + pub(crate) const FRAGMENT_CONTEXT: u64 = 0x20_0000; + /// Compute free-list control page and the state page it names (the + /// state page is not mapped at boot). + pub(crate) const COMPUTE_FREE_LIST: u64 = 0xc086_8000; + pub(crate) const COMPUTE_FREE_LIST_STATE: u64 = 0x163_0000; + /// Second compute free-list control page and its state page. + pub(crate) const COMPUTE_READY_FREE_LIST: u64 = 0xc087_8000; + pub(crate) const COMPUTE_READY_FREE_LIST_STATE: u64 = 0x164_8000; + /// Further compute free-list state page. + pub(crate) const COMPUTE_STATE: u64 = 0x165_0000; + /// Hardware-data bundle (`initdata::bundle`). + pub(crate) const BUNDLE: u64 = 0xc078_8000; +} + +/// Arenas used by dynamically allocated firmware objects. +pub(crate) mod dynamic { + /// First dynamic byte, relative to the upper kernel window. + pub(crate) const KERNEL_OFFSET: u64 = 0x168_4000; + /// Client-private compute and render backing after the shared queue arena. + pub(crate) const CLIENT_LOWER: core::ops::Range = 0x70_8000_0000..0x71_0000_0000; + /// Canonical low mappings shared by hardware queues and their clients. + /// Client-private allocations start after this arena. + pub(crate) const LOWER: core::ops::Range = 0x70_0400_0000..0x70_8000_0000; +} + +/// Fixed addresses in the firmware's context-0 and context-1 address spaces. +pub(crate) mod low { + /// Context-0 alias of the parameter-buffer descriptor table. + pub(crate) const PB_DESCRIPTORS: u64 = 0x70_0183_8000; + /// Context-0 alias of the page-pool descriptor table. + pub(crate) const PAGE_POOL_DESCRIPTORS: u64 = 0x70_0184_0000; + /// Context-0 alias of the index object. + pub(crate) const INDEX: u64 = 0x10_0019_0000; + /// Context-0 aliases of the completion rings. + pub(crate) const COMPLETION_RINGS: [u64; 2] = [0x70_03f0_0000, 0x70_03f2_8000]; + /// Context-0 run list of the compute free list. + pub(crate) const COMPUTE_RUN_LIST: u64 = 0x70_03ad_8000; + + /// Render free list (context 1): page list. + pub(crate) const RENDER_PAGE_LIST: u64 = 0x70_0000_0000; + /// Size of the render page list. + pub(crate) const RENDER_PAGE_LIST_SIZE: usize = 0x20_0000; + /// Render free list: run list (64 KiB, zero at boot). + pub(crate) const RENDER_RUN_LIST: u64 = 0x70_0020_8000; + /// Size of the render run list. + pub(crate) const RENDER_RUN_LIST_SIZE: usize = 0x1_0000; + /// Render free list: first block. + pub(crate) const RENDER_BLOCKS: u64 = 0x70_0022_0000; + /// Distance between consecutive blocks. + pub(crate) const BLOCK_STRIDE: u64 = 0x10_8000; + /// Render blocks allocated at boot. + pub(crate) const RENDER_BLOCKS_ALLOCATED: usize = 28; + /// Render blocks mapped at boot. + pub(crate) const RENDER_BLOCKS_MAPPED: usize = 22; + /// Render blocks in the free list at boot. + pub(crate) const RENDER_BLOCKS_INITIAL: u32 = 17; + /// Render block holding the compute page list. + pub(crate) const COMPUTE_PAGE_LIST_BLOCK: usize = 22; + + /// Distance of the compute pool from the render pool. + const COMPUTE: u64 = 0x200_0000; + /// Compute pool: page list. + pub(crate) const COMPUTE_POOL_PAGE_LIST: u64 = RENDER_PAGE_LIST + COMPUTE; + /// Compute pool: run list. + pub(crate) const COMPUTE_POOL_RUN_LIST: u64 = RENDER_RUN_LIST + COMPUTE; + /// Compute pool: first block. + pub(crate) const COMPUTE_POOL_BLOCKS: u64 = RENDER_BLOCKS + COMPUTE; + /// Blocks the compute free list declares. + pub(crate) const COMPUTE_BLOCKS: u32 = 18; + /// Compute pool blocks in the second compute free list. + pub(crate) const COMPUTE_READY_BLOCKS: u32 = 17; + /// Page list of the compute free list (inside the block + /// [`COMPUTE_PAGE_LIST_BLOCK`] holds, relocated to the compute pool). + pub(crate) const COMPUTE_PAGE_LIST: u64 = + RENDER_BLOCKS + COMPUTE_PAGE_LIST_BLOCK as u64 * BLOCK_STRIDE + COMPUTE; + /// Two zeroed compute blocks. + pub(crate) const COMPUTE_ZERO_BLOCKS: [u64; 2] = [ + RENDER_PAGE_LIST + COMPUTE, + RENDER_PAGE_LIST + BLOCK_STRIDE + COMPUTE, + ]; + + /// Returns block `index` of the render pool. + pub(crate) const fn render_block(index: usize) -> u64 { + RENDER_BLOCKS + index as u64 * BLOCK_STRIDE + } + + /// Returns block `index` of the compute pool. + pub(crate) const fn compute_block(index: usize) -> u64 { + COMPUTE_POOL_BLOCKS + index as u64 * BLOCK_STRIDE + } + + /// Blocks of the compute free list, in run-list order: pool blocks 0 to + /// 17, the two zeroed blocks, then pool block 18. + pub(crate) const COMPUTE_FREE_LIST_BLOCKS: [u64; 21] = { + let mut blocks = [0; 21]; + let mut index = 0; + while index < 18 { + blocks[index] = compute_block(index); + index += 1; + } + blocks[18] = COMPUTE_ZERO_BLOCKS[0]; + blocks[19] = COMPUTE_ZERO_BLOCKS[1]; + blocks[20] = compute_block(18); + blocks + }; +} + +/// Client-root reservations and parameter-buffer geometry. +pub(crate) mod resources { + use core::ops::Range; + + pub(crate) const UAT_PAGE: u64 = 0x4000; + pub(crate) const USER_BASE: u64 = 0x10_0000_0000; + pub(crate) const TIMESTAMPS: Range = 0x70_0000_0000..0x71_0000_0000; + pub(crate) const COMMAND_TIMESTAMP_HZ: u64 = 24_000_000; + pub(crate) const USER_END: u64 = USER_BASE + (1 << 32); + pub(crate) const FIXED_PAIRS: usize = 2; + pub(crate) const FIXED_PAIR_STRIDE: u64 = 0x8_0000; + pub(crate) const SCENE: u64 = USER_BASE + 0x17_8000; + pub(crate) const SCENE_SIZE: u64 = 0x8000; + pub(crate) const DISCARD: u64 = USER_BASE + 0x18_0000; + pub(crate) const DISCARD_SIZE: u64 = 0x8000; + pub(crate) const PAGE_LIST: u64 = USER_BASE + 0x19_0000; + pub(crate) const FRAGMENT_STATUS: u64 = USER_BASE + 0x1a_8000; + pub(crate) const RENDER_ALIASES: Range = 0x10_0200_0000..0x10_0220_0000; + pub(crate) const TARGETS: Range = 0x10_5000_0000..0x10_6400_0000; + pub(crate) const TARGET_RECORD_MIN: usize = 0x4_0000; + pub(crate) const TARGET_RECORD_MAX: usize = 0x80_0000; + pub(crate) const TPC_MIN: usize = 0x3_0000; + pub(crate) const TPC_MAX: usize = 0x200_0000; + pub(crate) const AUXILIARY: Range = 0x100_0200_0000..0x100_0240_0000; + pub(crate) const METRICS_SIZE: u64 = 0x8000; + pub(crate) const TVB_PAGE: u64 = 0x8000; + pub(crate) const TVB_BLOCK_SIZE: usize = 0x2_0000; + pub(crate) const TVB_BLOCK_STRIDE: u64 = 0x2_8000; + pub(crate) const TVB_BLOCK_SLOTS: usize = 0xd1a; + pub(crate) const TVB_MAX_BLOCKS: usize = TVB_BLOCK_SLOTS - 1; + pub(crate) const TVB_PAGES_PER_BLOCK: usize = 4; + pub(crate) const TVB_PAGE_LIST_SIZE: usize = TVB_BLOCK_SLOTS * TVB_PAGES_PER_BLOCK * 4; + pub(crate) const TVB_INITIAL_PAGES: [u32; 32] = [ + 0x11, 0x16, 0x1b, 0x20, 0x25, 0x2a, 0x4a, 0x4f, 0x54, 0x59, 0x5e, 0x63, 0x68, 0x6d, 0x72, + 0x77, 0x7c, 0x81, 0x86, 0x8b, 0x90, 0x95, 0x9a, 0x9f, 0xa4, 0xa9, 0xae, 0xb3, 0xb8, 0xbd, + 0xc2, 0xc7, + ]; + pub(crate) const TVB_GROWTH: u64 = USER_BASE + 0x0400_0000; + pub(crate) const TVB_PAIR_STRIDE: u64 = 0x2800_0000; + pub(crate) const RESERVED_RANGES: usize = 48; + + /// Compact allocations search these shared client arenas in this order. + pub(crate) fn compact_ranges() -> [Range; 4] { + let second = TVB_GROWTH + TVB_PAIR_STRIDE; + [ + RENDER_ALIASES, + TVB_GROWTH + ..TVB_GROWTH + (TVB_MAX_BLOCKS - TVB_INITIAL_PAGES.len()) as u64 * TVB_BLOCK_STRIDE, + second..second + TVB_MAX_BLOCKS as u64 * TVB_BLOCK_STRIDE, + TARGETS, + ] + } + + /// All driver-owned windows reserved before the first userspace bind. + /// Reserving them does not install pages or allocate backing memory. + pub(crate) fn client_reserved_ranges(metrics_va: u64) -> Option<[Range; RESERVED_RANGES]> { + let mut ranges = core::array::from_fn(|_| 0..0); + let mut index = 0; + let mut add = |range| { + ranges[index] = range; + index += 1; + }; + add(0..0x1_0000); + add(TIMESTAMPS); + let page_list_size = (TVB_PAGE_LIST_SIZE as u64 + UAT_PAGE - 1) & !(UAT_PAGE - 1); + for pair in 0..FIXED_PAIRS { + let offset = pair as u64 * FIXED_PAIR_STRIDE; + add(SCENE + offset..SCENE + offset + SCENE_SIZE); + add(DISCARD + offset..DISCARD + offset + DISCARD_SIZE); + add(PAGE_LIST + offset..PAGE_LIST + offset + page_list_size); + add(FRAGMENT_STATUS + offset..FRAGMENT_STATUS + offset + UAT_PAGE); + } + add(RENDER_ALIASES); + add(TARGETS); + for page in TVB_INITIAL_PAGES { + let start = USER_BASE + u64::from(page) * TVB_PAGE; + add(start..start + TVB_BLOCK_STRIDE); + } + add(TVB_GROWTH + ..TVB_GROWTH + (TVB_MAX_BLOCKS - TVB_INITIAL_PAGES.len()) as u64 * TVB_BLOCK_STRIDE); + for pair in 1..FIXED_PAIRS { + let start = TVB_GROWTH + pair as u64 * TVB_PAIR_STRIDE; + add(start..start + TVB_MAX_BLOCKS as u64 * TVB_BLOCK_STRIDE); + } + add(AUXILIARY); + add(metrics_va..metrics_va.checked_add(METRICS_SIZE)?); + Some(ranges) + } +} + +/// Canonical userspace resource and its presentation in an execution context. +#[derive(Clone, Copy)] +pub(crate) struct ContextView { + pub(crate) source: u64, + pub(crate) low: u64, + pub(crate) size: usize, + pub(crate) write: bool, +} + +const fn context_view(low: u64, size: usize, write: bool) -> ContextView { + ContextView { + source: 0x10_0000_0000 + low, + low, + size, + write, + } +} + +pub(crate) const CONTEXT_VIEWS: [ContextView; 7] = [ + context_view(0, 0x10000, false), + context_view(0x18000, 0x8000, false), + context_view(0x28000, 0x8000, false), + context_view(0x38000, 0x8000, false), + context_view(0x48000, 0x8000, false), + context_view(0x58000, 0x8000, true), + context_view(0x68000, 0xc000, true), +]; + +/// Scheduler accounting policy and its firmware clock domain. +pub(crate) mod qos { + pub(crate) const CLASS: u8 = 0x0f; + pub(crate) const SHARE: u32 = 0x2000; + pub(crate) const CLOCK_HZ: u64 = 24_000_000; +} + +/// Physical workqueue capacity and logical-owner admission limits. +pub(crate) mod queues { + pub(crate) const RENDER_SLOTS: usize = 64; + pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 2; + pub(crate) const RENDER_DEPTH: u8 = 16; + pub(crate) const COMPUTE_DEPTH: usize = 16; +} + +/// Read-only recovery evidence, sampled only for selected halted slots. +pub(crate) mod recovery { + pub(crate) const SLOT_KEYS: u64 = 0x0000_0100_01ea_74b8; +} diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 60a64e23a161c..f354e4bb1f71a 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -11,6 +11,8 @@ use core::arch::asm; use core::cmp::min; +use kernel::page::PAGE_SIZE; + use crate::debug::*; use crate::mmu; @@ -134,6 +136,66 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { } } +/// Range length from which [`tlbi_range_or_asid`] invalidates the whole ASID. +const TLBI_RANGE_MAX: usize = 0x200_0000; +/// Range length from which [`tlbi_range_or_asid`] counts in 1 MiB units (scale 1) instead of +/// 32 KiB units (scale 0). +const TLBI_RANGE_SCALE1_MIN: usize = 0x10_0000; +/// Translation granule field value for 16 KiB pages. +const TLBI_TG_16K: u64 = 2; +/// Width of the base address field of a range TLBI operand. +const TLBI_RANGE_BADDR_MASK: u64 = (1 << 38) - 1; + +/// Invalidate a range of pages for a given ASID with a single range operation. +/// +/// Unlike [`tlbi_range`], which issues as many range operations as needed, this invalidates +/// ranges that do not fit one operation (and ranges starting at VA 0) by ASID. Single pages use a +/// page operation, at the CPU page size. +#[inline(always)] +pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + if len == 0 { + return; + } + + if va == 0 || len >= TLBI_RANGE_MAX { + tlbi_asid(asid); + return; + } + if len <= PAGE_SIZE { + tlbi_page(asid, va); + return; + } + + // The range covers (NUM + 1) << (5 * SCALE + 1) granules of 16 KiB. + let (scale, unit_shift) = if len >= TLBI_RANGE_SCALE1_MIN { + (1u64, 20) + } else { + (0u64, 15) + }; + let num = ((len - 1) >> unit_shift) as u64; + + let val = ((asid as u64) << 48) + | (TLBI_TG_16K << 46) + | (scale << 44) + | (num << 39) + | (((va >> mmu::UAT_PGBIT) as u64) & TLBI_RANGE_BADDR_MASK); + + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi rvae1os, {x}", + x = in(reg) val + ); + } +} + /// Issue a memory barrier (`dsb sy`). #[inline(always)] pub(crate) fn sync() { diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 11484ec706498..3f05335407b73 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -9,6 +9,10 @@ //! unmap buffer objects into a single user or kernel address space. //! //! The actual page table management is in the `pt` module. +//! +//! G17 firmware does not take part in the handoff protocol. Instead, each of its two coprocessor +//! instances publishes the top-level entries it owns into its own kernel root while it boots, and +//! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. use core::fmt::Debug; use core::mem::size_of; @@ -84,11 +88,24 @@ use pin_init; const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; +mod bind; +mod context; +mod shared; +mod lifetime; +pub(crate) use bind::PreparedUserBindBatch; +pub(crate) use lifetime::VmJobGuard; + /// PPL magic number for the handoff region const PPL_MAGIC: u64 = 0x4b1d000000000002; /// Number of supported context entries in the TTBAT const UAT_NUM_CTX: usize = 64; + +/// Application contexts owned by independent logical queues. Contexts 0..4 +/// and 63 remain reserved for the firmware and device-global resources. +const EXECUTION_CONTEXT_MASK: u64 = ((1u64 << 63) - 1) & !((1u64 << 5) - 1); +/// Maximum simultaneously retained logical queue contexts. +pub(crate) const MAX_EXECUTION_CONTEXTS: u32 = EXECUTION_CONTEXT_MASK.count_ones(); /// First context available for users const UAT_USER_CTX_START: usize = 1; /// Number of available user contexts @@ -100,6 +117,30 @@ pub(crate) const IOVA_USER_BASE: u64 = UAT_PGSZ as u64; /// Top-level page table entry of the upper half that holds the driver-managed kernel VA range. const IOVA_KERN_TOP_LEVEL_INDEX: u64 = 2; +/// TTBAT context that starts out as an alias of the kernel context (with its own ASID) with +/// [`UatFirmware::PublishedRoots`]. +const KERNEL_ALIAS_CTX: usize = 1; + +/// How the GPU firmware shares UAT management with the driver. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum UatFirmware { + /// One coprocessor that publishes its magic number in the handoff region, arbitrates the + /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). + Handoff, + /// Two coprocessor instances that never touch the handoff region (G17). + /// + /// Each instance has its own kernel root and writes the top-level entries below and including + /// the driver-managed kernel window into it while it boots: the lower ones are private to the + /// instance, and the kernel window entry points at a table that the bootloader reserved. The + /// primary instance uses the `pagetables` root, the secondary one the root at + /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel + /// context. + PublishedRoots { + /// Offset of the secondary instance's kernel root in the `pagetables` region. + secondary_root_offset: u64, + }, +} + const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; @@ -203,12 +244,52 @@ const SLOTS_SIZE: usize = UAT_NUM_CTX * size_of::(); // We need at least page 0 (ttb0) const PAGETABLES_SIZE: usize = UAT_PGSZ; +/// Lock-free visibility of the mapping epoch. Writers run under the GPUVM +/// exec lock; nested mutations keep cached validation unavailable until the +/// outermost mutation has finished its leaf writes and invalidations. +struct MappingEpoch { + visible: AtomicU64, + pending: AtomicU64, + writers: AtomicU32, +} + +impl MappingEpoch { + fn new() -> Self { + Self { + visible: AtomicU64::new(0), + pending: AtomicU64::new(0), + writers: AtomicU32::new(0), + } + } + fn snapshot(&self) -> Option { + let value = self.visible.load(Ordering::Acquire); + (value != u64::MAX).then_some(value) + } +} + +#[must_use = "retain until the mapping mutation and its invalidations finish"] +struct MappingMutation(Option>); + +impl Drop for MappingMutation { + fn drop(&mut self) { + if let Some(epoch) = self.0.as_ref() { + if epoch.writers.fetch_sub(1, Ordering::Relaxed) == 1 { + epoch + .visible + .store(epoch.pending.load(Ordering::Relaxed), Ordering::Release); + } + } + } +} + /// Inner data for a Vm instance. This is reference-counted by the outer Vm object. struct VmInner { dev: driver::AsahiDevRef, is_kernel: bool, va_range: Range, page_table: UatPageTable, + mapping_epoch: Option>, + epoch: Option, mm: mm::Allocator<(), KernelMappingInner>, uat_inner: Arc, binding: Arc>, @@ -265,6 +346,7 @@ impl gpuvm::DriverGpuVm for VmInner { op: &mut gpuvm::OpMap, ctx: &mut Self::StepContext, ) -> Result { + let _mutation = self.mapping_mutation(); let mut iova = op.addr(); let mut left = op.range() as usize; let mut offset = op.offset() as usize; @@ -369,6 +451,7 @@ impl gpuvm::DriverGpuVm for VmInner { ); return Err(EINVAL); } + self.tlbi_contexts_naming_root(op.addr(), op.range() as usize); Ok(()) } fn step_unmap( @@ -380,12 +463,13 @@ impl gpuvm::DriverGpuVm for VmInner { mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); + let _mutation = self.mapping_mutation(); self.page_table .unmap_pages(va.addr()..(va.addr() + va.range()))?; if let Some(asid) = self.slot() { fence(Ordering::SeqCst); - mem::tlbi_range(asid as u8, va.addr() as usize, va.range() as usize); + self.tlbi_range(asid as u8, va.addr(), va.range() as usize); mod_dev_dbg!( self.dev, "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", @@ -396,6 +480,8 @@ impl gpuvm::DriverGpuVm for VmInner { mem::sync(); } + self.tlbi_contexts_naming_root(va.addr(), va.range() as usize); + if op.unmap_and_unlink_va_defer().is_none() { dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); } @@ -435,11 +521,12 @@ impl gpuvm::DriverGpuVm for VmInner { let unmap_range = unmap_end - unmap_start; + let _mutation = self.mapping_mutation(); self.page_table.unmap_pages(unmap_start..unmap_end)?; if let Some(asid) = self.slot() { fence(Ordering::SeqCst); - mem::tlbi_range(asid as u8, unmap_start as usize, unmap_range as usize); + self.tlbi_range(asid as u8, unmap_start, unmap_range as usize); mod_dev_dbg!( self.dev, "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", @@ -450,6 +537,8 @@ impl gpuvm::DriverGpuVm for VmInner { mem::sync(); } + self.tlbi_contexts_naming_root(unmap_start, unmap_range as usize); + if op.unmap().unmap_and_unlink_va_defer().is_none() { dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); } @@ -481,6 +570,19 @@ impl gpuvm::DriverGpuVm for VmInner { } impl VmInner { + fn mapping_mutation(&mut self) -> MappingMutation { + let state = self.mapping_epoch.clone(); + if let Some(state) = state.as_ref() { + state.writers.fetch_add(1, Ordering::Relaxed); + state.visible.swap(u64::MAX, Ordering::AcqRel); + self.epoch = self.epoch.and_then(|epoch| epoch.checked_add(1)); + state + .pending + .store(self.epoch.unwrap_or(u64::MAX), Ordering::Relaxed); + } + MappingMutation(state) + } + /// Returns the slot index, if this VM is bound. fn slot(&self) -> Option { if self.is_kernel { @@ -508,11 +610,47 @@ impl VmInner { self.page_table.ttb() } + /// Invalidate a VA range of this Vm's address space in the TLB of one ASID. + fn tlbi_range(&self, asid: u8, iova: u64, size: usize) { + match self.uat_inner.firmware { + UatFirmware::Handoff => mem::tlbi_range(asid, iova as usize, size), + UatFirmware::PublishedRoots { .. } => { + mem::tlbi_range_or_asid(asid, iova as usize, size) + } + } + } + + /// Invalidate a VA range in every TTBAT context that names this Vm's root as its lower half. + /// + /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root + /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping + /// change must reach all of them. Other firmware only uses the bind slot, which the callers + /// already invalidate. + fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { + if self.uat_inner.firmware == UatFirmware::Handoff { + return; + } + + let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); + if contexts == 0 { + return; + } + + fence(Ordering::SeqCst); + for ctx in 0..UAT_NUM_CTX { + if contexts & (1 << ctx) != 0 { + mem::tlbi_range_or_asid(ctx as u8, iova as usize, size); + } + } + mem::sync(); + } + /// Map an `mm::Node` representing an mapping in VA space. fn map_node(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); - let guard = node.bo.as_ref().ok_or(EINVAL)?.inner().inner.lock(); + let guard = node.bo.as_ref().ok_or(EINVAL)?.get()?.inner().inner.lock(); let sgt = guard.sgt.as_ref().ok_or(EINVAL)?; + let _mutation = self.mapping_mutation(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -567,6 +705,9 @@ impl VmInner { iova += len as u64; left -= len; } + + // A context naming this root may be live and hold a negative translation for the range. + self.tlbi_contexts_naming_root(node.start(), node.mapped_size); Ok(()) } } @@ -575,12 +716,84 @@ impl VmInner { #[derive(Clone)] pub(crate) struct Vm { id: u64, + mapping_epoch: Option>, + lifetime: Option>, + context_bindings: Option>, + shared_bindings: Option>, + status: Option>, inner: ARef>, dummy_obj: ARef, binding: Arc>, } no_debug!(Vm); +/// An application TTBAT identity retained by a logical queue and its jobs. +/// The root lease is dropped before the VM, so no page table can be freed +/// while a context still names it. Installed firmware queues must retain this +/// object through replacement retirement or processor stop. +pub(crate) struct ExecutionContext { + root: ExecutionRoot, + vm: Vm, +} + +struct ExecutionRoot { + inner: Arc, + id: u8, + generation: u8, + low: u64, + high: u64, +} + +impl ExecutionContext { + pub(crate) fn id(&self) -> u32 { + self.root.id.into() + } + pub(crate) fn generation(&self) -> u8 { + self.root.generation + } + pub(crate) fn vm(&self) -> &Vm { + &self.vm + } + + pub(crate) fn is_current(&self) -> bool { + let inner = self.root.inner.lock(); + let roots = &inner.ttbs()[usize::from(self.root.id)]; + roots.ttb0.load(Ordering::Acquire) == self.root.low + && roots.ttb1.load(Ordering::Acquire) == self.root.high + } +} + +impl Drop for ExecutionRoot { + fn drop(&mut self) { + let id = usize::from(self.id); + let release = { + let inner = self.inner.lock(); + inner.handoff().lock(); + let roots = &inner.ttbs()[id]; + let low = roots.ttb0.load(Ordering::Acquire); + let high = roots.ttb1.load(Ordering::Acquire); + let owned = low == self.low && high == self.high; + if owned { + roots.ttb0.store(0, Ordering::Release); + roots.ttb1.store(0, Ordering::Release); + } + inner.handoff().unlock(); + owned || (low == 0 && high == 0) + }; + if release { + fence(Ordering::SeqCst); + mem::tlbi_asid(self.id); + mem::sync(); + self.inner.lock().execution_contexts &= !(1u64 << id); + } else { + pr_err!( + "MMU: application context {} changed owners while leased\n", + id + ); + } + } +} + /// Slot data for a [`Vm`] slot (nothing, we only care about the indices). pub(crate) struct SlotInner(); @@ -632,13 +845,60 @@ impl Clone for VmBind { } } +/// A mapping BO reference selecting the teardown protocol of its firmware. +/// Retain the owner until deferred cleanup completes, including its final BO. +struct MappingBo { + deferred: bool, + bo: Option>>, + owner: ARef>, +} +impl MappingBo { + fn new( + bo: ARef>, + owner: ARef>, + deferred: bool, + ) -> Self { + Self { + deferred, + bo: Some(bo), + owner, + } + } + fn get(&self) -> Result<&gpuvm::GpuVmBo> { + self.bo.as_deref().ok_or(EINVAL) + } +} +impl Drop for MappingBo { + fn drop(&mut self) { + let Some(bo) = self.bo.take() else { + return; + }; + if !self.deferred { + // Handoff VMs retain the legacy ARef decrement and reservation + // locking behavior. Only published-root VMs use deferred cleanup. + drop(bo); + return; + } + let raw = ARef::into_raw(bo); + // SAFETY: GpuVmBo is repr(C) with drm_gpuvm_bo first. into_raw + // transfers one live reference. Every Vm here uses IMMEDIATE_MODE; + // deferred put selects its GPUVA lock and retains cleanup ownership. + unsafe { + kernel::bindings::drm_gpuvm_bo_put_deferred( + raw.as_ptr().cast::(), + ) + }; + self.owner.bo_deferred_cleanup(); + } +} + /// Inner data required for an object mapping into a [`Vm`]. pub(crate) struct KernelMappingInner { // Drop order matters: - // - Drop the GpuVmBo first, which resv locks its BO and drops a GpuVm reference + // - Deferred-put the GpuVmBo under its immediate-mode GPUVA lock // - Drop the GEM BO next, since BO free can take the resv lock itself // - Drop the owner GpuVm last, since that again can take resv locks when the refcount drops to 0 - bo: Option>>, + bo: Option, _gem: Option>, owner: ARef>, uat_inner: Arc, @@ -666,6 +926,108 @@ impl KernelMapping { self.0.start()..(self.0.start() + self.0.mapped_size as u64) } + /// Resolves a byte in this retained mapping without creating page tables. + pub(crate) fn translate_offset(&self, offset: usize) -> Result { + if offset >= self.size() { + return Err(EINVAL); + } + let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; + self.0.owner.exec_lock(None, false)?.page_table.translate_iova(address) + } + + /// Writes a userspace timestamp through the retained GEM mapping. Byte + /// stores support unaligned destinations and concurrent completion writes + /// without creating mutable aliases. The complete timestamp is not atomic. + pub(crate) fn write_timestamp(&self, offset: usize, value: u64) -> Result { + use core::sync::atomic::AtomicU8; + use kernel::drm::gem::BaseObject; + let gem = self.0._gem.as_ref().ok_or(EINVAL)?; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { + return Err(ERANGE); + } + let start = self.0.offset.checked_add(offset).ok_or(EOVERFLOW)?; + let end = start.checked_add(size_of::()).ok_or(EOVERFLOW)?; + if end > gem.size() { + return Err(ERANGE); + } + let vmap = gem.vmap::()?; + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: The retained GEM/vmap covers each checked byte. AtomicU8 + // has byte alignment and all concurrent completion writers use + // the same access width, without borrowing a mutable byte slice. + let destination = unsafe { &*vmap.as_mut_ptr().add(start + index).cast::() }; + destination.store(byte, Ordering::Relaxed); + } + Ok(()) + } + + /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. + /// The result includes whether any page-table entry may have changed; after + /// mutation the caller must retain the mapping until processor stop even if + /// the readback fails. The original protection is retained for teardown. + /// + /// # Safety + /// The mapping must be unpublished. After mutation it must not be unmapped + /// while a firmware instance can retain a cached translation or contents. + pub(crate) unsafe fn set_fw_cached_prefix(&self, prefix: usize) -> (Result<[u64; 2]>, bool) { + const MAX_PAGES: usize = 32; + const ATTRIBUTE_SHIFT: u32 = 2; + const ATTRIBUTE_MASK: u64 = 7; + const UNCACHED: u64 = 2; + const CACHE_ATTRIBUTE_CHANGE: u64 = UNCACHED << ATTRIBUTE_SHIFT; + let mut mutated = false; + let result = (|| { + if self.0.uat_inner.firmware == UatFirmware::Handoff + || prefix == 0 + || prefix > self.size() + || (prefix | self.size()) & UAT_PGMSK != 0 + || self.size() > MAX_PAGES * UAT_PGSZ + { + return Err(EINVAL); + } + let mut owner = self.0.owner.exec_lock(None, false)?; + if !owner.is_kernel { + return Err(EINVAL); + } + let pages = self.size() / UAT_PGSZ; + let cached_pages = prefix / UAT_PGSZ; + let mut before = [0; MAX_PAGES]; + for (index, value) in before[..pages].iter_mut().enumerate() { + *value = owner + .page_table + .leaf_entry(self.iova() + (index * UAT_PGSZ) as u64)?; + if (*value >> ATTRIBUTE_SHIFT) & ATTRIBUTE_MASK != UNCACHED { + return Err(EINVAL); + } + } + let _mutation = owner.mapping_mutation(); + mutated = true; + owner.page_table.reprot_pages( + self.iova()..self.iova() + prefix as u64, + PROT_GPU_FW_PRIV_RW, + )?; + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + let mut after = [0; MAX_PAGES]; + for (index, value) in after[..pages].iter_mut().enumerate() { + *value = owner + .page_table + .leaf_entry(self.iova() + (index * UAT_PGSZ) as u64)?; + let change = if index < cached_pages { + CACHE_ATTRIBUTE_CHANGE + } else { + 0 + }; + if before[index] ^ *value != change { + return Err(EFAULT); + } + } + Ok([after[0], after[cached_pages.min(pages - 1)]]) + })(); + (result, mutated) + } + /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the /// coprocessor cache. This is required to safely unmap cached/private mappings. fn remap_uncached_and_flush(&mut self) { @@ -685,6 +1047,7 @@ impl KernelMapping { // Remap in-place as uncached. // Do not try to unmap the guard page (-1) let prot = self.0.prot.as_uncached(); + let _mutation = owner.mapping_mutation(); if owner .page_table .reprot_pages(self.iova_range(), prot) @@ -822,7 +1185,10 @@ impl Drop for KernelMapping { // 4. Unmap // 5. Flush the TLB range again - if self.0.prot.is_cached_noncoherent() { + // Firmware that does not use the handoff protocol cannot be asked to flush its cache. + // Such firmware only caches kernel mappings that live until its coprocessors stop. + if self.0.prot.is_cached_noncoherent() && self.0.uat_inner.firmware == UatFirmware::Handoff + { mod_pr_debug!( "MMU: remap as uncached {:#x}:{:#x}\n", self.iova(), @@ -843,6 +1209,7 @@ impl Drop for KernelMapping { self.size() ); + let _mutation = owner.mapping_mutation(); if owner.page_table.unmap_pages(self.iova_range()).is_err() { dev_err!( owner.dev.as_ref(), @@ -854,7 +1221,7 @@ impl Drop for KernelMapping { if let Some(asid) = owner.slot() { fence(Ordering::SeqCst); - mem::tlbi_range(asid as u8, self.iova() as usize, self.size()); + owner.tlbi_range(asid as u8, self.iova(), self.size()); mod_dev_dbg!( owner.dev, "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", @@ -864,12 +1231,20 @@ impl Drop for KernelMapping { ); mem::sync(); } + + // The range returns to the VA allocator once this mapping is gone, so no context that + // names this root may keep a stale translation for it. + owner.tlbi_contexts_naming_root(self.iova(), self.size()); } } /// Shared UAT global data structures struct UatShared { kernel_ttb1: u64, + execution_contexts: u64, + context_generations: [u8; UAT_NUM_CTX], + /// Kernel root of the secondary firmware instance, if there is one. + secondary_ttb1: Option, map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, @@ -887,6 +1262,20 @@ impl UatShared { // SAFETY: pointer is non-null per the type invariant unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() } + + /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. + fn contexts_naming_root(&self, ttb: u64) -> u64 { + self.ttbs() + .iter() + .enumerate() + .filter(|(ctx, slot)| slot.ttb0.load(Ordering::Acquire) == tagged_root(ttb, *ctx)) + .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) + } +} + +/// Returns the TTBAT value naming root `ttb` in context `ctx`. +const fn tagged_root(ttb: u64, ctx: usize) -> u64 { + ttb | TTBR_VALID | (ctx as u64) << TTBR_ASID_SHIFT } // SAFETY: Nothing here is unsafe to send across threads. @@ -895,6 +1284,7 @@ unsafe impl Send for UatShared {} /// Inner data for the top-level UAT instance. #[pin_data] struct UatInner { + firmware: UatFirmware, #[pin] shared: Mutex, #[pin] @@ -959,12 +1349,27 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self) -> Result { + fn init(&self, firmware: UatFirmware) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); self.cur_slot.store(0, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); + if firmware == UatFirmware::Handoff { + self.wait_for_firmware()?; + } + + for i in 0..=UAT_NUM_CTX { + self.flush[i].state.store(0, Ordering::Relaxed); + self.flush[i].addr.store(0, Ordering::Relaxed); + self.flush[i].size.store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Wait for the firmware to publish its half of the handoff region. + fn wait_for_firmware(&self) -> Result { let start = Instant::::now(); const TIMEOUT: Delta = Delta::from_millis(1000); @@ -986,13 +1391,6 @@ impl Handoff { } self.unlock(); - - for i in 0..=UAT_NUM_CTX { - self.flush[i].state.store(0, Ordering::Relaxed); - self.flush[i].addr.store(0, Ordering::Relaxed); - self.flush[i].size.store(0, Ordering::Relaxed); - } - fence(Ordering::SeqCst); Ok(()) } } @@ -1052,7 +1450,7 @@ impl Vm { geometry.kernel_range(), geometry.ias, geometry.oas, - false, + uat_inner.firmware != UatFirmware::Handoff, )? } else { UatPageTable::new(geometry.ias, geometry.oas)? @@ -1061,7 +1459,11 @@ impl Vm { let (va_range, gpuvm_range) = if is_kernel { (geometry.kernel_range(), kernel_range.clone()) } else { - (geometry.user_range(), geometry.user_usable_range()) + let mut range = geometry.user_range(); + if uat_inner.firmware != UatFirmware::Handoff { + range.start = 0; + } + (range, geometry.user_usable_range()) }; let mm = mm::Allocator::new(va_range.start, va_range.range(), ())?; @@ -1079,9 +1481,19 @@ impl Vm { GFP_KERNEL, )?; + let mapping_epoch = if uat_inner.firmware != UatFirmware::Handoff { + Some(Arc::new(MappingEpoch::new(), GFP_KERNEL)?) + } else { + None + }; let binding_clone = binding.clone(); Ok(Vm { id, + mapping_epoch: mapping_epoch.clone(), + lifetime: None, + context_bindings: None, + shared_bindings: None, + status: None, dummy_obj: dummy_obj.gem.clone(), inner: gpuvm::GpuVm::new( c_str!("Asahi::GpuVm"), @@ -1096,6 +1508,8 @@ impl Vm { va_range, is_kernel, page_table, + mapping_epoch, + epoch: Some(0), mm, uat_inner, binding: binding_clone, @@ -1106,6 +1520,102 @@ impl Vm { }) } + /// Attach a status before sharing a newly created user VM with queues. + pub(crate) fn with_status(mut self) -> Result { + self.status = Some(Arc::new(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.lifetime = Some(lifetime::VmLifetime::new()?); + self.context_bindings = Some(context::ContextBindings::new()?); + self.shared_bindings = Some(shared::SharedBindings::new()?); + Ok(self) + } + + /// Status shared by every queue using this VM, if enabled by its GPU. + pub(crate) fn status(&self) -> Option<&Arc> { + self.status.as_ref() + } + + fn covers_range_locked( + inner: &mut VmInner, + address: u64, + size: u64, + read: bool, + write: bool, + ) -> bool { + if size == 0 { + return false; + } + let Some(end) = address.checked_add(size) else { + return false; + }; + let mask = UAT_PGMSK as u64; + let start = address & !mask; + let end = if end & mask == 0 { + end + } else { + let Some(end) = (end | mask).checked_add(1) else { + return false; + }; + end + }; + if start < inner.va_range.start || end > inner.va_range.end { + return false; + } + inner + .page_table + .covers_range(start..end, read, write) + .unwrap_or(false) + } + + pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool { + let Ok(mut inner) = self.inner.exec_lock(None, false) else { + return false; + }; + Self::covers_range_locked(&mut inner, address, size, read, write) + } + + /// Checks all writable sparse blocks and their guard coverage under one + /// GPUVM lock. A fully mapped guard is not a valid sparse allocation. + pub(crate) fn covers_sparse_blocks_with_guards( + &self, + addresses: &[u64], + block_size: u64, + stride: u64, + ) -> bool { + if stride <= block_size { + return false; + } + let Ok(mut inner) = self.inner.exec_lock(None, false) else { + return false; + }; + addresses.iter().all(|&address| { + let Some(guard) = address.checked_add(block_size) else { + return false; + }; + Self::covers_range_locked(&mut inner, address, block_size, true, true) + && !Self::covers_range_locked( + &mut inner, guard, stride - block_size, false, false, + ) + }) + } + + /// Checks all exact request tuples under one GPUVM exec lock. Access bit0 + /// requests reading and bit1 writing. The epoch names this same snapshot. + pub(crate) fn covers_ranges_batch(&self, ranges: &[(u64, u64, u8)]) -> (bool, Option) { + let Ok(mut inner) = self.inner.exec_lock(None, false) else { + return (false, None); + }; + let epoch = inner.epoch; + let covered = ranges.iter().all(|&(address, size, access)| { + Self::covers_range_locked(&mut inner, address, size, access & 1 != 0, access & 2 != 0) + }); + (covered, epoch) + } + + /// Returns None while a leaf mutation is in progress or after epoch overflow. + pub(crate) fn mapping_validation_epoch(&self) -> Option { + self.mapping_epoch.as_ref()?.snapshot() + } + /// Get the translation table base for this Vm fn ttb(&self) -> u64 { self.binding.lock().ttb @@ -1122,7 +1632,39 @@ impl Vm { prot: Prot, guard: bool, ) -> Result { - let size = object_range.range(); + self.map_in_range_with_guard_size( + gem, + object_range, + alignment, + range, + prot, + if guard { UAT_PGSZ } else { 0 }, + ) + } + + /// Reserves a whole-page unmapped guard without adding it to the mapped backing. + #[allow(clippy::too_many_arguments)] + pub(crate) fn map_in_range_with_guard_size( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + ) -> Result { + use kernel::drm::gem::BaseObject; + if guard_size % UAT_PGSZ != 0 { + return Err(EINVAL); + } + let size = object_range + .end + .checked_sub(object_range.start) + .ok_or(EINVAL)?; + if size == 0 || object_range.end > gem.size() { + return Err(EINVAL); + } + let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let sgt = gem.owned_sg_table()?; let mut inner = self.inner.exec_lock(Some(gem), false)?; let vm_bo = self.inner.obtain_bo(gem)?; @@ -1133,24 +1675,41 @@ impl Vm { } core::mem::drop(vm_bo_guard); + let deferred = inner.uat_inner.firmware != UatFirmware::Handoff; + let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone(), deferred)); let uat_inner = inner.uat_inner.clone(); - let node = inner.mm.insert_node_in_range( + // A failed reservation drops its payload while exec still owns GEM + // reservations. Published-root VMs attach BO/GEM only after success; + // handoff VMs retain their existing payload and decrement behavior. + let result = inner.mm.insert_node_in_range( KernelMappingInner { owner: self.inner.clone(), uat_inner, prot, - bo: Some(vm_bo), - _gem: Some(gem.into()), + bo: if deferred { None } else { vm_bo.take() }, + _gem: if deferred { None } else { Some(gem.into()) }, offset: object_range.start, mapped_size: size, }, - (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + reserved_size as u64, alignment, 0, range.start, range.end, mm::InsertMode::Best, - )?; + ); + let mut node = match result { + Ok(node) => node, + Err(error) => { + drop(inner); + return Err(error); + } + }; + if deferred { + let payload = node.as_mut().inner_mut(); + payload.bo = vm_bo.take(); + payload._gem = Some(gem.into()); + } let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the @@ -1170,6 +1729,9 @@ impl Vm { prot: Prot, guard: bool, ) -> Result { + let reserved_size = size + .checked_add(if guard { UAT_PGSZ } else { 0 }) + .ok_or(EOVERFLOW)?; let sgt = gem.owned_sg_table()?; let mut inner = self.inner.exec_lock(Some(&gem), false)?; @@ -1181,21 +1743,36 @@ impl Vm { } core::mem::drop(vm_bo_guard); + let deferred = inner.uat_inner.firmware != UatFirmware::Handoff; + let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone(), deferred)); let uat_inner = inner.uat_inner.clone(); - let node = inner.mm.reserve_node( + // Published-root VMs defer BO/GEM attachment until reservation succeeds. + let result = inner.mm.reserve_node( KernelMappingInner { owner: self.inner.clone(), uat_inner, prot, - bo: Some(vm_bo), - _gem: Some(gem.clone()), + bo: if deferred { None } else { vm_bo.take() }, + _gem: if deferred { None } else { Some(gem.clone()) }, offset: 0, mapped_size: size, }, addr, - (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + reserved_size as u64, 0, - )?; + ); + let mut node = match result { + Ok(node) => node, + Err(error) => { + drop(inner); + return Err(error); + } + }; + if deferred { + let payload = node.as_mut().inner_mut(); + payload.bo = vm_bo.take(); + payload._gem = Some(gem.clone()); + } let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the @@ -1216,6 +1793,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + self.wait_for_user_map_admission()?; // Mapping needs a complete context let mut ctx = StepContext { new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), @@ -1278,6 +1856,7 @@ impl Vm { size, addr ); + let _commit = self.mapping_commit()?; inner.sm_map(&mut ctx, addr, size, offset, gem_range, flags) } @@ -1326,6 +1905,7 @@ impl Vm { 0, )?; + let mutation = inner.mapping_mutation(); let ret = inner.page_table.map_pages( iova..(iova + size as u64), phys as PhysicalAddr, @@ -1334,6 +1914,7 @@ impl Vm { ); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + drop(mutation); core::mem::drop(inner); ret?; Ok(KernelMapping(node)) @@ -1341,6 +1922,17 @@ impl Vm { /// Unmap everything in an address range. pub(crate) fn unmap_range(&self, iova: u64, size: u64) -> Result { + if self.defer_unmap(iova, size)? { + return Ok(()); + } + self.unmap_range_commit(iova, size, true) + } + + fn unmap_range_now(&self, iova: u64, size: u64) -> Result { + self.unmap_range_commit(iova, size, false) + } + + fn unmap_range_commit(&self, iova: u64, size: u64, user: bool) -> Result { // Unmapping a range can only do a single split, so just preallocate // the prev and next GpuVas let mut ctx = StepContext { @@ -1352,11 +1944,25 @@ impl Vm { let mut inner = self.inner.exec_lock(None, false)?; mod_dev_dbg!(inner.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); - inner.sm_unmap(&mut ctx, iova, size) + let _commit = if user { + match self.mapping_commit() { + Ok(guard) => guard, + Err(error) => { + if self.defer_unmap(iova, size)? { + return Ok(()); + } + return Err(error); + } + } + } else { + None + }; + inner.sm_unmap(&mut ctx, iova, size)?; + self.untrack_shared_range(iova, size); + Ok(()) } - /// Drop mappings for a given bo. - pub(crate) fn drop_mappings(&self, gem: &gem::Object) -> Result { + fn drop_mappings_now(&self, gem: &gem::Object) -> Result { // Removing whole mappings only does unmaps, so no preallocated VAs let mut ctx = Default::default(); @@ -1365,6 +1971,7 @@ impl Vm { if let Some(bo) = self.inner.find_bo(gem) { mod_dev_dbg!(inner.dev, "MMU: bo_unmap\n"); self.inner.bo_unmap(&mut ctx, &bo)?; + self.untrack_shared_object(gem); mod_dev_dbg!(inner.dev, "MMU: bo_unmap done\n"); // We need to drop the exec_lock first, then the GpuVmBo since that will take the lock itself. core::mem::drop(inner); @@ -1490,6 +2097,39 @@ impl Uat { &self.kernel_lower_vm } + /// Installed descriptor and bind contexts rooted in this user VM. A graph + /// constructor checks this before changing a retained pool's mappings. + pub(crate) fn vm_context_mask(&self, vm: &Vm) -> Result { + if !matches!(self.inner.firmware, UatFirmware::PublishedRoots { .. }) { + return Err(EINVAL); + } + let root = { + let owner = vm.inner.exec_lock(None, false)?; + if owner.is_kernel || !Arc::ptr_eq(&owner.uat_inner, &self.inner) { + return Err(EINVAL); + } + owner.ttb() + }; + Ok(self.inner.lock().contexts_naming_root(root)) + } + + /// Publish a completed alias set before its descriptors become visible. + /// Match actual roots, including independently allocated execution contexts. + pub(crate) fn flush_vm_contexts(&self, vm: &Vm) -> Result { + let contexts = self.vm_context_mask(vm)?; + if contexts == 0 { + return Err(ENOENT); + } + fence(Ordering::SeqCst); + for context in 0..UAT_NUM_CTX { + if contexts & (1u64 << context) != 0 { + mem::tlbi_asid(context as u8); + } + } + mem::sync(); + Ok(contexts) + } + #[cfg(CONFIG_DEV_COREDUMP)] pub(crate) fn dump_kernel_pages(&self) -> Result> { let mut inner = self.kernel_vm.inner.exec_lock(None, false)?; @@ -1561,6 +2201,63 @@ impl Uat { Ok(VmBind(vm.clone(), slot)) } + /// Allocates a stable application context without changing the VM bind slot. + /// Both roots and the generation remain unchanged for the lease's lifetime. + pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { + if self.inner.firmware == UatFirmware::Handoff { + return Err(EINVAL); + } + { + let owner = vm.inner.exec_lock(None, false)?; + if owner.is_kernel + || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) + { + return Err(EINVAL); + } + } + let (id, generation) = { + let mut inner = self.inner.lock(); + let free = !inner.execution_contexts & EXECUTION_CONTEXT_MASK; + if free == 0 { + return Err(ENOSPC); + } + let id = free.trailing_zeros() as usize; + inner.execution_contexts |= 1u64 << id; + inner.context_generations[id] = inner.context_generations[id].wrapping_add(1); + (id, inner.context_generations[id]) + }; + let low = tagged_root(vm.ttb(), id); + let high = tagged_root(self.kernel_vm.ttb(), id); + // Allocate the complete host owner before either TTBAT half is published. + // ExecutionRoot releases an empty claimed slot if allocation fails. + let context = Arc::new( + ExecutionContext { + root: ExecutionRoot { + inner: self.inner.clone(), + id: id as u8, + generation, + low, + high, + }, + vm: vm.clone(), + }, + GFP_KERNEL, + )?; + { + let inner = self.inner.lock(); + let roots = &inner.ttbs()[id]; + if roots.ttb0.load(Ordering::Acquire) != 0 || roots.ttb1.load(Ordering::Acquire) != 0 { + return Err(EBUSY); + } + roots.ttb0.store(low, Ordering::Release); + roots.ttb1.store(high, Ordering::Release); + } + fence(Ordering::SeqCst); + mem::tlbi_asid(id as u8); + mem::sync(); + Ok(context) + } + /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { Vm::new( @@ -1578,9 +2275,109 @@ impl Uat { self.geometry } + /// Adopts the top-level entries that the primary firmware instance published in the kernel + /// root while booting ([`UatFirmware::PublishedRoots`]). + /// + /// The entries below the kernel window belong to the firmware, and the kernel window entry + /// points at a table that the bootloader reserved, so the driver maps kernel objects beneath + /// it and never frees it. Entries above the kernel window do not belong to anyone and are + /// cleared. Must run after the primary instance booted and before anything is mapped into the + /// kernel `Vm`. + pub(crate) fn adopt_firmware_root_entries(&self) -> Result<[u64; 3]> { + let owned = IOVA_KERN_TOP_LEVEL_INDEX as usize + 1; + let mut snapshot = [0; 3]; + // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives + // this `Uat`. + unsafe { + pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { + for (value, pte) in snapshot.iter_mut().zip(ptes) { + *value = pte.load(Ordering::Relaxed); + } + for pte in &ptes[owned..] { + pte.store(0, Ordering::Relaxed); + } + }) + }?; + + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + Ok(snapshot) + } + + /// Checks that allocation under the shared subtree did not replace a firmware-owned entry. + pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { + // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. + let matches = unsafe { + pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { + ptes.iter() + .zip(snapshot) + .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) + }) + }?; + if !matches { + return Err(EIO); + } + Ok(()) + } + + /// Copies the kernel root's top-level entries from the kernel window up into the root of the + /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are + /// private to that instance. + pub(crate) fn mirror_secondary_root(&self) -> Result { + let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; + let count = UAT_PGSZ / size_of::(); + let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; + + // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives + // this `Uat`. + unsafe { + pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(secondary, count, |secondary| { + for (src, dst) in primary.iter().zip(secondary).skip(first) { + let pte = src.load(Ordering::Relaxed); + if pte != 0 && dst.load(Ordering::Relaxed) != pte { + dst.store(pte, Ordering::Relaxed); + } + } + }) + }) + }??; + + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + Ok(()) + } + + /// Activates the device-global render root in context 1 after the boot records retire. + /// Both contexts must still contain their original kernel roots. + pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { + let inner = self.inner.lock(); + inner.handoff().lock(); + let ttbs = inner.ttbs(); + let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { + ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) + && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) + }); + if !ready { + inner.handoff().unlock(); + return Err(EIO); + } + ttbs[KERNEL_ALIAS_CTX] + .ttb0 + .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); + inner.handoff().unlock(); + core::mem::drop(inner); + fence(Ordering::SeqCst); + mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); + mem::sync(); + Ok(()) + } + /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner(dev: &driver::AsahiDevice) -> Result> { + fn make_inner(dev: &driver::AsahiDevice, firmware: UatFirmware) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref(), c_str!("handoff"), HANDOFF_SIZE, true)?; let ttbs_rgn = Self::map_region(dev.as_ref(), c_str!("ttbs"), SLOTS_SIZE, true)?; @@ -1592,12 +2389,16 @@ impl Uat { Arc::pin_init( try_pin_init!(UatInner { + firmware, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), shared <- new_mutex!( UatShared { kernel_ttb1: 0, + execution_contexts: !EXECUTION_CONTEXT_MASK, + context_generations: [0; UAT_NUM_CTX], + secondary_ttb1: None, map_kernel_to_user: false, handoff_rgn, ttbs_rgn, @@ -1609,29 +2410,40 @@ impl Uat { ) } - /// Creates a new `Uat` instance for the given address space geometry. + /// Creates a new `Uat` instance for the given address space geometry and firmware. #[inline(never)] pub(crate) fn new( dev: &driver::AsahiDevice, geometry: UatGeometry, map_kernel_to_user: bool, + firmware: UatFirmware, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - let inner = Self::make_inner(dev)?; + let inner = Self::make_inner(dev, firmware)?; let of_node = dev.as_ref().of_node().ok_or(EINVAL)?; let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); let ttb1size: usize = res.size().try_into()?; - if ttb1size < PAGETABLES_SIZE { + let (pagetables_size, secondary_ttb1) = match firmware { + UatFirmware::Handoff => (PAGETABLES_SIZE, None), + UatFirmware::PublishedRoots { + secondary_root_offset, + } => ( + secondary_root_offset as usize + UAT_PGSZ, + Some(ttb1 + secondary_root_offset), + ), + }; + if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); return Err(ENOMEM); } dev_info!(dev.as_ref(), "MMU: Creating kernel page tables\n"); - let kernel_lower_vm = Vm::new(dev, inner.clone(), geometry.user_range(), geometry, None, 1)?; + let kernel_lower_vm = + Vm::new(dev, inner.clone(), geometry.user_range(), geometry, None, 1)?; let kernel_vm = Vm::new( dev, inner.clone(), @@ -1665,8 +2477,9 @@ impl Uat { inner.map_kernel_to_user = map_kernel_to_user; inner.kernel_ttb1 = ttb1; + inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init()?; + inner.handoff().init(firmware)?; dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); @@ -1682,6 +2495,16 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } + if secondary_ttb1.is_some() { + let alias = &ttbs[KERNEL_ALIAS_CTX]; + alias + .ttb0 + .store(tagged_root(ttb0, KERNEL_ALIAS_CTX), Ordering::SeqCst); + alias + .ttb1 + .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); + } + inner.handoff().unlock(); core::mem::drop(inner); diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs new file mode 100644 index 0000000000000..0d5abcf9b84f1 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Prepared userspace mapping transactions. Host callback resources are reserved for the +//! complete ioctl before its first visible mapping changes; leaf commits retain their +//! individual interruptible GPUVM reservation acquisition and quarantine admission check. + +use super::*; + +pub(crate) struct PreparedUserMap { + gem: ARef, + ctx: StepContext, + addr: u64, + size: u64, + offset: u64, + single_page: bool, +} + +pub(crate) struct PreparedUserUnmap { + pub(super) iova: u64, + pub(super) size: u64, + ctx: StepContext, +} + +pub(super) enum PreparedUserBindOp { + Map(PreparedUserMap), + Unmap(PreparedUserUnmap), +} + +pub(crate) struct PreparedUserBindBatch { + pub(super) ops: KVec, +} + +impl PreparedUserBindBatch { + pub(crate) fn new(capacity: usize) -> Result { + Ok(Self { + ops: KVec::with_capacity(capacity, GFP_KERNEL)?, + }) + } + pub(crate) fn push_map(&mut self, map: PreparedUserMap) -> Result { + Ok(self.ops.push(PreparedUserBindOp::Map(map), GFP_KERNEL)?) + } + pub(crate) fn push_unmap(&mut self, unmap: PreparedUserUnmap) -> Result { + Ok(self + .ops + .push(PreparedUserBindOp::Unmap(unmap), GFP_KERNEL)?) + } +} + +impl Vm { + /// Prepares host state without taking DMA reservations or changing any leaf. + #[allow(clippy::too_many_arguments)] + pub(crate) fn prepare_bind_object( + &self, + gem: &gem::Object, + addr: u64, + size: u64, + offset: u64, + prot: Prot, + single_page: bool, + ) -> Result { + let mut ctx = StepContext { + new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + prot, + ..Default::default() + }; + let vm_bo = self.inner.obtain_bo(gem)?; + { + let mut bo = vm_bo.inner().inner.lock(); + if bo.sgt.is_none() { + let sgt = gem.owned_sg_table()?; + if bo.sg_vec.is_none() { + let mut segments = KVVec::new(); + let mut offset = 0; + for range in sgt.iter() { + let address = range.dma_address() as usize; + let length = range.dma_len() as usize; + segments.push((offset, address..address + length), GFP_KERNEL)?; + offset += length; + } + bo.sg_vec = Some(segments); + } + bo.sgt = Some(sgt); + } + } + ctx.vm_bo = Some(vm_bo); + if (addr | size | offset) & UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + Ok(PreparedUserMap { + gem: gem.into(), + ctx, + addr, + size, + offset, + single_page, + }) + } + + pub(crate) fn prepare_user_unmap(&self, iova: u64, size: u64) -> Result { + Ok(PreparedUserUnmap { + iova, + size, + ctx: StepContext { + prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + ..Default::default() + }, + }) + } + + /// Commits in ioctl order. Quarantine may defer an all-unmap batch, but can + /// never allow a map to replace a mapping still owned by failed work. + pub(crate) fn commit_prepared_user_bind_batch( + &self, + batch: &mut PreparedUserBindBatch, + ) -> Result { + if self.defer_bind_batch(batch)? { + return Ok(()); + } + let mut index = 0; + while index < batch.ops.len() { + let result = match &mut batch.ops[index] { + PreparedUserBindOp::Map(map) => { + let mut inner = self.inner.exec_lock(Some(&map.gem), true)?; + match self.mapping_commit() { + Ok(commit) => { + let (flags, repeat) = if map.single_page { + (gpuvm::GpuVaFlags::REPEAT, UAT_PGSZ as u32) + } else { + (gpuvm::GpuVaFlags::NONE, 0) + }; + let result = inner.sm_map( + &mut map.ctx, + map.addr, + map.size, + map.offset, + repeat, + flags, + ); + if result.is_ok() { + self.track_shared_binding( + &map.gem, + map.addr, + map.size, + map.offset, + map.single_page, + ); + } + drop(inner); + drop(commit); + Some(result) + } + Err(error) if error == EIO => { + drop(inner); + None + } + Err(error) => return Err(error), + } + } + PreparedUserBindOp::Unmap(unmap) => { + let mut inner = self.inner.exec_lock(None, true)?; + match self.mapping_commit() { + Ok(commit) => { + let result = inner.sm_unmap(&mut unmap.ctx, unmap.iova, unmap.size); + if result.is_ok() { + self.untrack_shared_range(unmap.iova, unmap.size); + } + drop(inner); + drop(commit); + Some(result) + } + Err(error) if error == EIO => { + drop(inner); + None + } + Err(error) => return Err(error), + } + } + }; + if let Some(result) = result { + result?; + index += 1; + } else if self.defer_bind_batch(batch)? { + return Ok(()); + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/mmu/context.rs b/drivers/gpu/drm/asahi/mmu/context.rs new file mode 100644 index 0000000000000..c47b7ae1fd782 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/context.rs @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Tracks canonical context resources and retains their fixed low presentation. +//! +//! Userspace owns low views 1 through 6. Only the reserved null view is created +//! here. Its installation is serialized per VM and retained through VM teardown. + +use super::*; +use crate::hw::t8140::CONTEXT_VIEWS; +use kernel::drm::gem::BaseObject; + +struct Binding { + gem: ARef, + offset: usize, +} +struct Bindings { + generation: u64, + slots: [Option; CONTEXT_VIEWS.len()], +} +#[pin_data] +pub(super) struct ContextBindings { + #[pin] + bindings: Mutex, + #[pin] + install: Mutex<()>, +} +impl ContextBindings { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + bindings <- new_mutex!(Bindings { generation: 0, + slots: core::array::from_fn(|_| None) }, "VM context resources"), + install <- new_mutex!((), "VM context alias installation"), + }), + GFP_KERNEL, + ) + } +} + +impl Vm { + pub(crate) fn validate_context_binding( + &self, + address: u64, + size: u64, + single_page: bool, + ) -> Result { + if self.context_bindings.is_none() || !single_page { + return Ok(()); + } + let end = address.checked_add(size).ok_or(EOVERFLOW)?; + if CONTEXT_VIEWS + .iter() + .any(|view| address < view.source + view.size as u64 && view.source < end) + { + return Err(EINVAL); + } + Ok(()) + } + + /// Arithmetic is preflighted before invalidating any retained candidate. + pub(crate) fn track_context_binding( + &self, + gem: ARef, + address: u64, + size: u64, + object_offset: u64, + ) -> Result { + let Some(context) = self.context_bindings.as_ref() else { + return Ok(()); + }; + let end = address.checked_add(size).ok_or(EOVERFLOW)?; + let mut offsets = [None; CONTEXT_VIEWS.len()]; + for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if address <= view.source && view.source + view.size as u64 <= end { + offsets[index] = Some(usize::try_from( + object_offset + .checked_add(view.source - address) + .ok_or(EOVERFLOW)?, + )?); + } + } + let mut bindings = context.bindings.lock(); + let mut touched = false; + for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if address < view.source + view.size as u64 && view.source < end { + bindings.slots[index] = None; + touched = true; + } + if let Some(offset) = offsets[index] { + bindings.slots[index] = Some(Binding { + gem: gem.clone(), + offset, + }); + touched = true; + } + } + if touched { + bindings.generation = bindings.generation.wrapping_add(1); + } + Ok(()) + } + + pub(crate) fn untrack_context_range(&self, range: Range) { + let Some(context) = self.context_bindings.as_ref() else { + return; + }; + let mut bindings = context.bindings.lock(); + let mut touched = false; + for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if range.start < view.source + view.size as u64 && view.source < range.end { + bindings.slots[index] = None; + touched = true; + } + } + if touched { + bindings.generation = bindings.generation.wrapping_add(1); + } + } + + pub(crate) fn untrack_context_object(&self, gem: &gem::Object) { + let Some(context) = self.context_bindings.as_ref() else { + return; + }; + let mut bindings = context.bindings.lock(); + let mut touched = false; + for slot in &mut bindings.slots { + if slot + .as_ref() + .is_some_and(|binding| core::ptr::eq(&*binding.gem, gem)) + { + *slot = None; + touched = true; + } + } + if touched { + bindings.generation = bindings.generation.wrapping_add(1); + } + } + + pub(crate) fn translate_iova(&self, address: u64) -> Result { + self.inner + .exec_lock(None, false)? + .page_table + .translate_iova(address) + } + + /// Requires a job mapping guard before validating or installing these views. + pub(crate) fn install_job_context_aliases(&self) -> Result { + let context = self.context_bindings.as_ref().ok_or(EINVAL)?; + let (generation, snapshot) = { + let bindings = context.bindings.lock(); + let mut snapshot = KVec::with_capacity(CONTEXT_VIEWS.len(), GFP_KERNEL)?; + for slot in &bindings.slots { + let binding = slot.as_ref().ok_or(ENOENT)?; + snapshot.push((binding.gem.clone(), binding.offset), GFP_KERNEL)?; + } + (bindings.generation, snapshot) + }; + for (index, (gem, offset)) in snapshot.iter().enumerate().skip(1) { + let view = CONTEXT_VIEWS[index]; + if !self.covers_range(view.source, view.size as u64, true, view.write) + || !self.covers_range(view.low, view.size as u64, true, view.write) + { + return Err(EFAULT); + } + if offset.checked_add(view.size).ok_or(EOVERFLOW)? > gem.size() { + return Err(ERANGE); + } + for page in (0..view.size).step_by(UAT_PGSZ) { + if self.translate_iova(view.source + page as u64)? + != self.translate_iova(view.low + page as u64)? + { + return Err(EFAULT); + } + } + } + let _install = context.install.lock(); + let view = CONTEXT_VIEWS[0]; + let (gem, offset) = &snapshot[0]; + // Null presentation requires writes in addition to the canonical view's + // ordinary read requirement. + if !self.covers_range(view.source, view.size as u64, true, true) { + return Err(EFAULT); + } + let end = offset.checked_add(view.size).ok_or(EOVERFLOW)?; + if end > gem.size() { + return Err(ERANGE); + } + let mut existing = 0; + for page in (0..view.size).step_by(UAT_PGSZ) { + if self.covers_range(view.low + page as u64, UAT_PGSZ as u64, true, true) { + existing += 1; + if self.translate_iova(view.source + page as u64)? + != self.translate_iova(view.low + page as u64)? + { + return Err(EFAULT); + } + } + } + if existing != 0 { + if existing != view.size / UAT_PGSZ { + return Err(EFAULT); + } + return if context.bindings.lock().generation == generation { + Ok(()) + } else { + Err(EAGAIN) + }; + } + let mut mappings = KVec::with_capacity(1, GFP_KERNEL)?; + let mapping = self.map_in_range( + gem, + *offset..end, + UAT_PGSZ as u64, + view.low..view.low + view.size as u64, + PROT_GPU_SHARED_RW, + false, + )?; + for page in (0..view.size).step_by(UAT_PGSZ) { + if self.translate_iova(view.source + page as u64)? + != self.translate_iova(view.low + page as u64)? + { + return Err(EFAULT); + } + } + mappings.push(mapping, GFP_KERNEL)?; + if context.bindings.lock().generation != generation { + return Err(EAGAIN); + } + self.append_driver_mappings(mappings) + } +} diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs new file mode 100644 index 0000000000000..3b8a5e5a40aeb --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -0,0 +1,605 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Keeps user mappings alive until accepted work has relinquished firmware ownership. +//! +//! The state lock is taken after GPUVM's exec lock when committing user mappings. Waiting for +//! that lock's commits never holds a GPUVM reservation. Draining takes no state lock while it +//! unmaps memory. Driver aliases retain only the inner GPUVM, so they create no cycle with Vm. + +use super::*; +use kernel::{ + new_condvar, + sync::{CondVar, CondVarTimeoutResult}, + time::msecs_to_jiffies, +}; + +const ADMISSION_WAIT_MS: u32 = 1000; + +pub(super) struct DriverMappings { + mappings: KVec, + reserved: KVec>, + compute: Option, + render: [Option; crate::hw::t8140::queues::RENDER_SLOTS], +} + +struct PoolMappings { + id: u64, + blocks: usize, + mappings: KVec, +} + +impl DriverMappings { + fn ranges(&self) -> impl Iterator> + '_ { + Iterator::chain( + Iterator::chain( + self.mappings.iter(), + Iterator::chain( + self.compute.iter().flat_map(|pool| pool.mappings.iter()), + self.render + .iter() + .flatten() + .flat_map(|pool| pool.mappings.iter()), + ), + ) + .map(KernelMapping::iova_range), + self.reserved.iter().cloned(), + ) + } +} + +struct State { + active: usize, + commits: usize, + blocked: bool, + quarantined: bool, + draining: bool, + closed: bool, + close_ranges: Option<(Range, Range)>, + unmaps: KVec<(u64, u64)>, + objects: KVec>, +} + +#[pin_data] +pub(super) struct VmLifetime { + #[pin] + state: Mutex, + #[pin] + changed: CondVar, + #[pin] + driver: Mutex>, +} + +impl VmLifetime { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + state <- new_mutex!(State { active:0, commits:0, blocked:false, + quarantined:false, draining:false, closed:false, close_ranges:None, + unmaps:KVec::new(), objects:KVec::new() }, "VM job lifetime"), + changed <- new_condvar!("VM mapping admission"), + driver <- new_mutex!(None, "VM driver mappings"), + }), + GFP_KERNEL, + ) + } +} + +/// Retains a VM's mappings through a command's exact retirement. An error fence is not +/// retirement: potentially visible work must retain its guard through quarantine or stop. +pub(crate) struct VmJobGuard { + vm: Vm, +} + +impl VmJobGuard { + /// Blocks mapping commits before the error fence can become visible to userspace. + pub(crate) fn quarantine(&self) { + if let Some(lifetime) = self.vm.lifetime.as_ref() { + let mut state = lifetime.state.lock(); + state.blocked = true; + while state.commits != 0 { + lifetime.changed.wait(&mut state); + } + state.quarantined = true; + } + } +} + +pub(super) struct MappingCommit { + lifetime: Arc, +} +impl Drop for MappingCommit { + fn drop(&mut self) { + let mut state = self.lifetime.state.lock(); + state.commits -= 1; + if state.commits == 0 { + self.lifetime.changed.notify_all(); + } + } +} + +impl Vm { + /// Attaches aliases while constructing this VM, before exposing it to userspace. + pub(crate) fn install_driver_mappings( + &self, + mappings: KVec, + reserved: KVec>, + ) -> Result { + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + if driver.is_some() { + return Err(EBUSY); + } + *driver = Some(DriverMappings { + mappings, + reserved, + compute: None, + render: core::array::from_fn(|_| None), + }); + Ok(()) + } + + /// Reserves the complete append before moving aliases into VM ownership. + pub(crate) fn append_driver_mappings(&self, mappings: KVec) -> Result { + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; + driver.mappings.reserve(mappings.len(), GFP_KERNEL)?; + for mapping in mappings { + driver.mappings.push(mapping, GFP_KERNEL)?; + } + Ok(()) + } + + pub(crate) fn render_pool_tvb_blocks(&self, slot: u8, id: u64) -> Option { + if id == 0 { + return None; + } + self.lifetime + .as_ref()? + .driver + .lock() + .as_ref()? + .render + .get(usize::from(slot))? + .as_ref() + .filter(|pool| pool.id == id) + .map(|pool| pool.blocks) + } + + /// Retains a new generation after all references to a replaced pool retire. + pub(crate) fn install_render_pool_mappings( + &self, + slot: u8, + id: u64, + blocks: usize, + mappings: KVec, + ) -> Result { + if id == 0 { + return Err(EINVAL); + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; + let pool = driver.render.get_mut(usize::from(slot)).ok_or(EINVAL)?; + if pool.is_some() { + return Err(EBUSY); + } + *pool = Some(PoolMappings { + id, + blocks, + mappings, + }); + Ok(()) + } + + /// Consumes aliases only after the exact cached owner and complete append + /// capacity have been checked. Failed preparation retains its mappings for + /// destruction after the caller releases the device mutex. + pub(crate) fn append_render_pool_mappings_prepared( + &self, + slot: u8, + id: u64, + previous: usize, + blocks: usize, + mappings: &mut KVec, + ) -> Result { + if id == 0 || blocks <= previous || mappings.len() != blocks - previous { + return Err(EINVAL); + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let pool = Option::as_mut(&mut *driver) + .and_then(|driver| driver.render.get_mut(usize::from(slot))) + .and_then(Option::as_mut) + .filter(|pool| pool.id == id) + .ok_or(EINVAL)?; + if pool.blocks != previous { + return Err(EBUSY); + } + pool.mappings + .len() + .checked_add(mappings.len()) + .ok_or(EOVERFLOW)?; + pool.mappings.reserve(mappings.len(), GFP_KERNEL)?; + for mapping in mappings.drain_all() { + pool.mappings.push(mapping, GFP_KERNEL)?; + } + pool.blocks = blocks; + Ok(()) + } + + /// A delayed teardown must not remove a successor generation. Remove + /// under the cache lock, then unmap after dropping that lock. + pub(crate) fn clear_render_pool_mappings_if_owner(&self, slot: u8, id: u64) { + if id == 0 { + return; + } + let previous = self.lifetime.as_ref().and_then(|lifetime| { + let mut driver = lifetime.driver.lock(); + Option::as_mut(&mut *driver) + .and_then(|driver| driver.render.get_mut(usize::from(slot))) + .filter(|pool| pool.as_ref().is_some_and(|pool| pool.id == id)) + .and_then(Option::take) + }); + drop(previous); + } + + pub(crate) fn compute_pool_mappings_match(&self, id: u64) -> bool { + id != 0 + && self.lifetime.as_ref().is_some_and(|lifetime| { + lifetime + .driver + .lock() + .as_ref() + .is_some_and(|driver| driver.compute.as_ref().is_some_and(|pool| pool.id == id)) + }) + } + + /// The device publication lock serializes first-use alias construction. + /// Never replace a live generation; its jobs must retire before clearing. + pub(crate) fn install_compute_pool_mappings( + &self, + id: u64, + mappings: KVec, + ) -> Result { + if id == 0 { + return Err(EINVAL); + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; + if driver.compute.is_some() { + return Err(EBUSY); + } + driver.compute = Some(PoolMappings { + id, + blocks: 0, + mappings, + }); + Ok(()) + } + + /// Called under the publication lock after all obsolete pool work retires. + /// Drop outside the driver lock because mapping teardown acquires GPUVM locks. + pub(crate) fn clear_compute_pool_mappings(&self) { + let previous = self.lifetime.as_ref().and_then(|lifetime| { + let mut driver = lifetime.driver.lock(); + Option::as_mut(&mut *driver).and_then(|driver| driver.compute.take()) + }); + drop(previous); + } + + /// Tests mappings and reservations which userspace must never replace. + pub(crate) fn driver_range_overlaps(&self, range: Range) -> bool { + let Some(lifetime) = self.lifetime.as_ref() else { + return false; + }; + let driver = lifetime.driver.lock(); + driver.as_ref().is_some_and(|driver| { + driver + .ranges() + .any(|reserved| range.start < reserved.end && reserved.start < range.end) + }) + } + + /// Waits before reservation acquisition, allowing healthy retirement to drain. + pub(crate) fn wait_for_user_map_admission(&self) -> Result { + let Some(lifetime) = self.lifetime.as_ref() else { + return Ok(()); + }; + let mut state = lifetime.state.lock(); + let mut remaining = msecs_to_jiffies(ADMISSION_WAIT_MS); + loop { + let failed = self.status.as_ref().is_some_and(|status| status.get() != 0); + if !failed + && !state.closed + && (state.blocked || (state.active != 0 && state.quarantined)) + && remaining != 0 + { + remaining = match lifetime + .changed + .wait_interruptible_timeout(&mut state, remaining) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => 0, + CondVarTimeoutResult::Woken { jiffies } => jiffies, + }; + continue; + } + if state.closed { + return Err(ENOENT); + } + if failed || state.blocked || (state.active != 0 && state.quarantined) { + return Err(EIO); + } + if !state.draining { + return Ok(()); + } + if lifetime.changed.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + } + } + + /// Acquires a job reference without imposing a VM-wide limit on healthy work. + pub(crate) fn retain_job(&self) -> Result { + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut state = lifetime.state.lock(); + loop { + if state.closed { + return Err(ENOENT); + } + if state.blocked { + return Err(EIO); + } + if !state.draining { + state.active = state.active.checked_add(1).ok_or(EOVERFLOW)?; + return Ok(VmJobGuard { vm: self.clone() }); + } + lifetime.changed.wait(&mut state); + } + } + + /// The first job of a submission may wait for a clean VM's quarantine drain. + pub(crate) fn retain_first_job(&self) -> Result { + match self.retain_job() { + Err(error) + if error == EIO && self.status.as_ref().is_some_and(|status| status.get() == 0) => { + } + result => return result, + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut remaining = msecs_to_jiffies(ADMISSION_WAIT_MS); + let mut state = lifetime.state.lock(); + while state.blocked + && remaining != 0 + && self.status.as_ref().is_some_and(|status| status.get() == 0) + { + remaining = match lifetime + .changed + .wait_interruptible_timeout(&mut state, remaining) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => 0, + CondVarTimeoutResult::Woken { jiffies } => jiffies, + }; + } + drop(state); + self.retain_job() + } + + /// Called only after the GPUVM exec/reservation locks have been acquired. + pub(super) fn mapping_commit(&self) -> Result> { + let Some(lifetime) = self.lifetime.as_ref() else { + return Ok(None); + }; + let mut state = lifetime.state.lock(); + if state.closed { + return Err(ENOENT); + } + if state.blocked { + return Err(EIO); + } + state.commits = state.commits.checked_add(1).ok_or(EOVERFLOW)?; + Ok(Some(MappingCommit { + lifetime: lifetime.clone(), + })) + } + + /// Reserves cleanup bookkeeping before a batch starts committing leaves. + pub(crate) fn reserve_deferred_user_unmaps(&self, count: usize) -> Result { + if let Some(lifetime) = self.lifetime.as_ref() { + lifetime.state.lock().unmaps.reserve(count, GFP_KERNEL)?; + } + Ok(()) + } + + pub(super) fn defer_bind_batch(&self, batch: &PreparedUserBindBatch) -> Result { + use super::bind::PreparedUserBindOp; + let Some(lifetime) = self.lifetime.as_ref() else { + return Ok(false); + }; + let mut state = lifetime.state.lock(); + if !state.blocked { + return Ok(false); + } + if state.closed { + return Err(ENOENT); + } + if batch + .ops + .iter() + .any(|op| matches!(op, PreparedUserBindOp::Map(_))) + { + return Err(EIO); + } + if !state.quarantined { + return Err(EBUSY); + } + for op in &batch.ops { + if let PreparedUserBindOp::Unmap(unmap) = op { + let range = (unmap.iova, unmap.size); + if !state.unmaps.contains(&range) { + state.unmaps.push(range, GFP_KERNEL)?; + } + } + } + Ok(true) + } + + /// Returns true if a quarantined owner requires this unmap to be deferred. + pub(super) fn defer_unmap(&self, iova: u64, size: u64) -> Result { + let Some(lifetime) = self.lifetime.as_ref() else { + return Ok(false); + }; + let mut state = lifetime.state.lock(); + if state.closed { + return Err(ENOENT); + } + if !state.blocked { + return Ok(false); + } + if !state.quarantined { + return Err(EBUSY); + } + if !state.unmaps.contains(&(iova, size)) { + state.unmaps.push((iova, size), GFP_KERNEL)?; + } + Ok(true) + } + + /// Defers handle-close cleanup while any accepted job can still use the BO. + pub(crate) fn drop_mappings(&self, gem: &gem::Object) -> Result { + let Some(lifetime) = self.lifetime.as_ref() else { + return self.drop_mappings_now(gem); + }; + let mut state = lifetime.state.lock(); + if !state + .objects + .iter() + .any(|object| core::ptr::eq(&**object, gem)) + { + state.objects.push(gem.into(), GFP_KERNEL)?; + } + if state.active != 0 || state.draining { + return Ok(()); + } + state.draining = true; + drop(state); + self.drain_mappings(lifetime); + Ok(()) + } + + /// Closes admission permanently and preserves mappings referenced by accepted jobs. + pub(crate) fn unmap_user_ranges(&self, user: Range, kernel: Range) -> Result { + if let Some(lifetime) = self.lifetime.as_ref() { + let mut state = lifetime.state.lock(); + state.blocked = true; + while state.commits != 0 { + lifetime.changed.wait(&mut state); + } + state.closed = true; + if state.active != 0 { + state.close_ranges = Some((user, kernel)); + return Ok(()); + } + } + self.unmap_user_ranges_now(user, kernel) + } + + fn unmap_user_ranges_now(&self, user: Range, kernel: Range) -> Result { + let driver = self + .lifetime + .as_ref() + .map(|lifetime| lifetime.driver.lock()); + let mut cursor = user.start; + while cursor < user.end { + let mut next = user.end; + let mut covered = cursor; + for range in Iterator::chain( + core::iter::once(kernel.clone()), + driver + .as_ref() + .and_then(|guard| guard.as_ref()) + .into_iter() + .flat_map(DriverMappings::ranges), + ) { + if range.contains(&cursor) { + covered = covered.max(range.end); + } else if cursor < range.start { + next = next.min(range.start); + } + } + if covered != cursor { + cursor = covered.min(user.end); + continue; + } + if next <= cursor { + return Err(EIO); + } + self.unmap_range_now(cursor, next - cursor)?; + cursor = next; + } + Ok(()) + } +} + +impl Drop for VmJobGuard { + fn drop(&mut self) { + let Some(lifetime) = self.vm.lifetime.as_ref() else { + return; + }; + let mut state = lifetime.state.lock(); + state.active -= 1; + if state.active != 0 { + return; + } + state.draining = true; + drop(state); + self.vm.drain_mappings(lifetime); + } +} + +impl Vm { + /// Admission stays closed to new jobs while the exec lock is acquired for cleanup. + /// Recheck queued cleanup and reopen under the same lock used to append it. + fn drain_mappings(&self, lifetime: &VmLifetime) { + let mut state = lifetime.state.lock(); + loop { + let close = state.close_ranges.take(); + let unmaps = core::mem::replace(&mut state.unmaps, KVec::new()); + let objects = core::mem::replace(&mut state.objects, KVec::new()); + drop(state); + if let Some((user, kernel)) = close { + if self.unmap_user_ranges_now(user, kernel).is_err() { + pr_err!("MMU: deferred VM close failed\n"); + } + } else { + for (address, size) in unmaps { + if self.unmap_range_now(address, size).is_err() { + pr_err!("MMU: deferred user unmap failed\n"); + } + } + } + for object in objects { + if self.drop_mappings_now(&object).is_err() { + pr_err!("MMU: deferred object unmap failed\n"); + } + } + state = lifetime.state.lock(); + if state.close_ranges.is_some() || !state.unmaps.is_empty() || !state.objects.is_empty() + { + continue; + } + state.quarantined = false; + state.draining = false; + if !state.closed { + state.blocked = false; + } + drop(state); + lifetime.changed.notify_all(); + break; + } + self.bo_deferred_cleanup(); + } +} diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs new file mode 100644 index 0000000000000..cf29e53fc9cc9 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Shared-object identities for attachment ordering across user address spaces. +//! These records do not own mappings. The VM's job guard protects the mappings +//! while submission captures the intersecting object offsets. Mapping commits +//! hold the GPUVM exec lock through metadata updates, then release it. The lock +//! order is GPUVM exec followed by bindings; footprint readers take only bindings. + +use super::*; +use crate::g17::{hazard::Footprint, Attachments}; + +struct Binding { + start: u64, + end: u64, + object: usize, + offset: u64, +} + +#[pin_data] +pub(super) struct SharedBindings { + #[pin] + bindings: Mutex>, + count: AtomicU32, +} + +impl SharedBindings { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + bindings <- new_mutex!(KVec::new(), "Shared attachment bindings"), + count: AtomicU32::new(0), + }), + GFP_KERNEL, + ) + } +} + +impl Vm { + pub(super) fn track_shared_binding( + &self, + gem: &gem::Object, + address: u64, + size: u64, + offset: u64, + single_page: bool, + ) { + let Some(shared) = &self.shared_bindings else { + return; + }; + let Some(end) = address.checked_add(size) else { + return; + }; + let external = !single_page && self.is_extobj(gem); + if !external && shared.count.load(Ordering::Relaxed) == 0 { + return; + } + let mut bindings = shared.bindings.lock(); + bindings.retain(|entry| entry.end <= address || end <= entry.start); + if external { + let _ = bindings.push( + Binding { + start: address, + end, + object: gem as *const gem::Object as usize, + offset, + }, + GFP_KERNEL, + ); + } + shared.count.store(bindings.len() as u32, Ordering::Relaxed); + } + + pub(super) fn untrack_shared_range(&self, address: u64, size: u64) { + let Some(shared) = &self.shared_bindings else { + return; + }; + if shared.count.load(Ordering::Relaxed) == 0 { + return; + } + let Some(end) = address.checked_add(size) else { + return; + }; + let mut bindings = shared.bindings.lock(); + bindings.retain(|entry| entry.end <= address || end <= entry.start); + shared.count.store(bindings.len() as u32, Ordering::Relaxed); + } + + pub(crate) fn untrack_shared_object(&self, gem: &gem::Object) { + let Some(shared) = &self.shared_bindings else { + return; + }; + if shared.count.load(Ordering::Relaxed) == 0 { + return; + } + let object = gem as *const gem::Object as usize; + let mut bindings = shared.bindings.lock(); + bindings.retain(|entry| entry.object != object); + shared.count.store(bindings.len() as u32, Ordering::Relaxed); + } + + pub(crate) fn render_footprint( + &self, + attachments: &Attachments, + ) -> Result>> { + if attachments.as_slice().is_empty() { + return Ok(None); + } + let mut footprint = Footprint::new(self.id); + for attachment in attachments.as_slice() { + let end = attachment.va.saturating_add(attachment.size); + footprint.push_vm(attachment.va, end); + let Some(shared) = &self.shared_bindings else { + continue; + }; + if shared.count.load(Ordering::Relaxed) == 0 { + continue; + } + let bindings = shared.bindings.lock(); + for entry in bindings.iter() { + let first = entry.start.max(attachment.va); + let last = entry.end.min(end); + if first < last { + let offset = entry.offset.saturating_add(first - entry.start); + footprint.push_object( + entry.object, + offset, + offset.saturating_add(last - first), + ); + } + } + } + Ok(Some(Arc::new(footprint, GFP_KERNEL)?)) + } +} diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 5a66406cc3dc1..a6b5f13574b3f 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -62,6 +62,32 @@ const UAT_LEVELS: usize = 3; /// Number of address bits below the index of a top-level page table entry. pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; +/// Runs `f` on the first `count` top-level entries of the root page table at `ttb`. +/// +/// # Safety +/// +/// `ttb` must be the physical address of a page table root in RAM that stays allocated while `f` +/// runs. Other agents (the GPU firmware) may access the entries concurrently, which is why they +/// are only exposed as atomics. +pub(crate) unsafe fn with_root_entries( + ttb: PhysicalAddr, + count: usize, + f: impl FnOnce(&[AtomicU64]) -> R, +) -> Result { + if count > UAT_NPTE { + return Err(EINVAL); + } + // SAFETY: The caller guarantees that `ttb` is an allocated page in RAM. + let page = unsafe { Page::borrow_phys(&ttb) }.ok_or(EIO)?; + page.with_pointer_into_page(0, count * PTE_SIZE, |p| { + // SAFETY: with_pointer_into_page() ensures the pointer is valid for `count` PTEs, and page + // table entries are naturally aligned. + Ok(f(unsafe { + core::slice::from_raw_parts(p as *const AtomicU64, count) + })) + }) +} + const PTE_TYPE_BITS: u64 = 3; const PTE_TYPE_LEAF_TABLE: u64 = 3; @@ -179,7 +205,6 @@ impl Prot { } } - #[cfg(CONFIG_DEV_COREDUMP)] pub(crate) const fn from_pte(pte: u64) -> Self { Prot { high_bits: (pte >> UAT_HIGH_BITS_SHIFT) as u16, @@ -228,6 +253,19 @@ impl Prot { | UAT_AF } + /// Whether this leaf allows the requested GPU accesses. + pub(crate) const fn allows_gpu(&self, need_read: bool, need_write: bool) -> bool { + let uxn = self.high_bits & HIGH_BITS_UXN != 0; + let pxn = self.high_bits & HIGH_BITS_PXN != 0; + let (readable, writable) = match self.ap { + AP_FW_GPU => (pxn, uxn), + AP_FW => (uxn && pxn, false), + AP_GPU => (!pxn, uxn != pxn), + _ => (false, false), + }; + (!need_read || readable) && (!need_write || writable) + } + pub(crate) const fn is_cached_noncoherent(&self) -> bool { self.ap != AP_GPU && self.memattr == MEMATTR_CACHED } @@ -501,6 +539,59 @@ impl UatPageTable { unsafe { Page::from_phys(phys) }; } + /// Checks every page, including holes skipped by the page-table walker. + pub(crate) fn covers_range( + &mut self, + range: Range, + read: bool, + write: bool, + ) -> Result { + if range.is_empty() || (range.start | range.end) & UAT_PGMSK as u64 != 0 { + return Ok(false); + } + let expected = (range.end - range.start) >> UAT_PGBIT; + let mut visited = 0u64; + let mut permitted = true; + self.with_pages(range, false, false, |_, ptes| { + visited = visited.checked_add(ptes.len() as u64).ok_or(EOVERFLOW)?; + for pte in ptes { + let value = pte.load(Ordering::Acquire); + if value & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE + || !Prot::from_pte(value).allows_gpu(read, write) + { + permitted = false; + } + } + Ok(()) + })?; + Ok(expected == visited && permitted) + } + + /// Resolves an existing leaf without creating page tables. + pub(crate) fn translate_iova(&mut self, iova: u64) -> Result { + let entry = self.leaf_entry(iova & !(UAT_PGMSK as u64))?; + Ok((entry & self.oas_mask & !(UAT_PGMSK as u64)) | (iova & UAT_PGMSK as u64)) + } + + /// Reads one mapped leaf entry without allocating missing tables. + pub(crate) fn leaf_entry(&mut self, iova: u64) -> Result { + if iova & UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + let end = iova.checked_add(UAT_PGSZ as u64).ok_or(EINVAL)?; + let mut value = 0; + self.with_pages(iova..end, false, false, |_, ptes| { + if let Some(pte) = ptes.first() { + value = pte.load(Ordering::Relaxed); + } + Ok(()) + })?; + if value & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE { + return Err(EFAULT); + } + Ok(value) + } + pub(crate) fn alloc_pages(&mut self, iova_range: Range) -> Result { mod_pr_debug!("UATPageTable::alloc_pages: {:#x?}\n", iova_range); self.with_pages(iova_range, true, false, |_, _| Ok(())) diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index e40adb7b106e2..dacebd97fc482 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -6,6 +6,9 @@ //! there is very little need to interact directly with GPU MMIO register. This module abstracts //! the few operations that require that, mainly reading the MMU fault status, reading GPU ID //! information, and starting the GPU firmware coprocessor. +//! +//! On G17, the driver also enables the bridge between the GPU and the fabric before the +//! coprocessors start, and enables dynamic power gating before the firmware is initialized. use crate::hw; use kernel::{ @@ -18,7 +21,8 @@ use kernel::{ }, platform, prelude::*, - sync::aref::ARef, // + str::CStr, + sync::{aref::ARef, Arc}, // }; /// Size of the ASC control MMIO region. @@ -32,6 +36,10 @@ const CPU_RUN: u32 = 0x1 << 4; // BIT(4) const FAULT_INFO: usize = 0x17030; +/// A nonzero low nibble permits writes through the G17 queue configuration port. +const G17_HOST_IRQ_SUMMARY: usize = 0xe01000; +const G17_HOST_POWERED: u32 = 0xf; + const ID_VERSION: usize = 0xd04000; const ID_UNK08: usize = 0xd04008; const ID_COUNTS_1: usize = 0xd04010; @@ -46,6 +54,22 @@ const CORE_MASKS_G14X: usize = 0xe01500; const FAULT_INFO_G14X: usize = 0xd8c0; const FAULT_ADDR_G14X: usize = 0xd8c8; +/// Cleared after `CPU_RUN` when stopping a G17 coprocessor. +const CPU_STOP_ACK: u32 = 0x1 << 5; // BIT(5) + +/// G17 GPU-to-fabric bridge control words. Bit 0 enables the bridge; the other bits are status. +const G17_BRIDGE_CONTROL: [usize; 2] = [0x1000104, 0x1000108]; +const G17_BRIDGE_ENABLE: u32 = 0x1; // BIT(0) + +/// G17 dynamic power gating control. Clearing these bits enables gating of idle GPU cores. +const G17_GATING_CONTROL: usize = 0xd06030; +const G17_GATING_DISABLE: u32 = 0x6; // BIT(1) | BIT(2) + +/// These performance selectors must be sampled before firmware can gate the core registers. +const G17_PERF_CONTROL: usize = 0xe0141c; +const G17_PERF_MAP: usize = 0xe01480; +const G17_CORE_POWER: usize = 0xe01000; + /// Enum representing the unit that caused an MMU fault. #[allow(non_camel_case_types)] #[allow(clippy::upper_case_acronyms)] @@ -151,17 +175,73 @@ pub(crate) struct FaultInfo { pub(crate) reason: FaultReason, } +/// Identification of a G17 GPU. +pub(crate) struct G17Id { + /// GPU family field of the ID register. + pub(crate) family: u8, + /// GPU variant field of the ID register. + pub(crate) variant: u8, + /// Silicon revision. + pub(crate) gpu_rev: hw::GpuRevision, + /// Number of dies. + pub(crate) num_dies: u32, + /// Number of clusters, over all dies. + pub(crate) num_clusters: u32, + /// Number of cores per cluster. + pub(crate) num_cores: u32, + /// Enabled cores of each cluster. + pub(crate) core_masks: KVec, + /// Performance selector control, sampled before core power gating starts. + pub(crate) perf_control: u32, + /// Performance selector table indices, low and high nibbles. + pub(crate) perf_map: [u32; 2], +} + +/// CPU control registers of one GPU coprocessor. +pub(crate) struct CpuControl { + asc: Pin>>>, +} + +impl CpuControl { + /// Map the CPU control registers in the `name` register region. + pub(crate) fn new(pdev: &platform::Device, name: &CStr) -> Result { + let req = pdev.io_request_by_name(name).ok_or(EINVAL)?; + Ok(CpuControl { + asc: KBox::pin_init(req.iomap_sized::(), GFP_KERNEL)?, + }) + } + + /// Start the coprocessor CPU. + pub(crate) fn start(&self) -> Result { + let asc = self.asc.try_access().ok_or(ENODEV)?; + let val = asc.read32(CPU_CONTROL); + asc.write32(val | CPU_RUN, CPU_CONTROL); + Ok(()) + } + + /// Stop the coprocessor CPU. This only gates it: its firmware keeps its state. + pub(crate) fn stop(&self) { + if let Some(asc) = self.asc.try_access() { + let val = asc.read32(CPU_CONTROL); + asc.write32(val & !CPU_RUN, CPU_CONTROL); + let val = asc.read32(CPU_CONTROL); + asc.write32(val & !CPU_STOP_ACK, CPU_CONTROL); + } + } +} + /// Device resources for this GPU instance. +#[derive(Clone)] pub(crate) struct Resources { dev: ARef, - sgx: Pin>>>, + sgx: Arc>>, } impl Resources { /// Map the required resources given our platform device. pub(crate) fn new(pdev: &platform::Device) -> Result { let sgx_req = pdev.io_request_by_name(c_str!("sgx")).ok_or(EINVAL)?; - let sgx_iomem = KBox::pin_init(sgx_req.iomap_sized::(), GFP_KERNEL)?; + let sgx_iomem = Arc::pin_init(sgx_req.iomap_sized::(), GFP_KERNEL)?; Ok(Resources { // SAFETY: This device does DMA via the UAT IOMMU. @@ -170,6 +250,23 @@ impl Resources { }) } + /// Installs both render queues while the caller holds the device mutex. + /// No other port publisher may interleave between this power witness and + /// the final validity strobe. False means no write was attempted. + pub(crate) fn install_render_pair( + &self, + registration: &crate::g17::fw::kick::RenderRegistration, + ) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + if sgx.relaxed().read32(G17_HOST_IRQ_SUMMARY) & G17_HOST_POWERED == 0 { + return Ok(false); + } + for &(offset, value) in registration.writes() { + sgx.try_write64(value, offset)?; + } + Ok(true) + } + fn sgx_read32(&self) -> u32 { if let Some(sgx) = self.sgx.try_access() { sgx.relaxed().read32(OFF) @@ -294,43 +391,9 @@ impl Resources { return Err(ENODEV); } - let mut core_masks = KVec::new(); - let mut total_active_cores: u32 = 0; - - let max_core_mask = ((1u64 << num_cores) - 1) as u32; - for _ in 0..num_clusters { - let mask = core_mask_regs[0] & max_core_mask; - core_masks.push(mask, GFP_KERNEL)?; - for i in 0..core_mask_regs.len() { - core_mask_regs[i] >>= num_cores; - if i < (core_mask_regs.len() - 1) { - core_mask_regs[i] |= core_mask_regs[i + 1] << (32 - num_cores); - } - } - total_active_cores += mask.count_ones(); - } - - if core_mask_regs.iter().any(|a| *a != 0) { - dev_err!( - self.dev.as_ref(), - "Leftover core mask: {:#x?}\n", - core_mask_regs - ); - return Err(EIO); - } - - let (gpu_rev, gpu_rev_id) = match (id_version >> 8) & 0xff { - 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), - 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), - 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), - 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), - 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), - 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), - a => { - dev_err!(self.dev.as_ref(), "Unknown GPU revision {}\n", a); - return Err(ENODEV); - } - }; + let (core_masks, total_active_cores) = + self.split_core_masks(&mut core_mask_regs, num_clusters, num_cores)?; + let (gpu_rev, gpu_rev_id) = self.gpu_revision(id_version)?; Ok(hw::GpuIdConfig { gpu_gen: match (id_version >> 24) & 0xff { @@ -370,6 +433,166 @@ impl Resources { }) } + /// Get the identification of a G17 GPU from its registers. + pub(crate) fn get_g17_id(&self) -> Result { + let id_version = self.sgx_read32::(); + let id_unk08 = self.sgx_read32::(); + let id_counts_1 = self.sgx_read32::(); + let id_counts_2 = self.sgx_read32::(); + let id_unk18 = self.sgx_read32::(); + let id_clusters = self.sgx_read32::(); + + dev_info!( + self.dev.as_ref(), + "GPU ID registers: {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}\n", + id_version, + id_unk08, + id_counts_1, + id_counts_2, + id_unk18, + id_clusters + ); + + // Later reads can fault while the firmware has gated the cores. Submission uses only + // these saved values and never re-reads the registers. + let perf_control = self.sgx_read32::() & 1; + let perf_map = self.sgx_read32::(); + let mut core_mask_regs = [ + self.sgx_read32::(), + self.sgx_read32::<{ CORE_MASKS_G14X + 4 }>(), + self.sgx_read32::<{ CORE_MASKS_G14X + 8 }>(), + ]; + dev_info!(self.dev.as_ref(), "Core masks: {:#x?}\n", core_mask_regs); + + let num_dies = (id_counts_1 >> 16) & 0xf; + let num_clusters = ((id_counts_1 >> 8) & 0xff) * num_dies; + let num_cores = id_counts_1 & 0xff; + if num_cores == 0 || num_cores > 32 || num_cores * num_clusters > 3 * 32 { + dev_err!( + self.dev.as_ref(), + "Invalid core counts ({} x {})\n", + num_clusters, + num_cores + ); + return Err(ENODEV); + } + let (core_masks, _) = + self.split_core_masks(&mut core_mask_regs, num_clusters, num_cores)?; + let (gpu_rev, _) = self.gpu_revision(id_version)?; + + Ok(G17Id { + family: ((id_version >> 24) & 0xff) as u8, + variant: ((id_version >> 16) & 0xff) as u8, + gpu_rev, + num_dies, + num_clusters, + num_cores, + core_masks, + perf_control, + perf_map: [perf_map & 0xf, (perf_map >> 16) & 0xf], + }) + } + + /// Enable the G17 GPU-to-fabric bridge and check that it took effect. + /// + /// The GPU power domain must be on. The coprocessors cannot reach memory until this is done. + pub(crate) fn enable_g17_bridge(&self) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + for offset in G17_BRIDGE_CONTROL { + let val = sgx.try_read32(offset)?; + sgx.try_write32(val | G17_BRIDGE_ENABLE, offset)?; + if sgx.try_read32(offset)? & G17_BRIDGE_ENABLE == 0 { + dev_err!( + self.dev.as_ref(), + "GPU bridge {:#x} did not enable\n", + offset + ); + return Err(EIO); + } + } + Ok(()) + } + + /// Disable the G17 GPU-to-fabric bridge. Both coprocessors must be stopped. + pub(crate) fn disable_g17_bridge(&self) { + if let Some(sgx) = self.sgx.try_access() { + for offset in G17_BRIDGE_CONTROL { + if let Ok(val) = sgx.try_read32(offset) { + // Cannot fail: the offset was read just before. + let _ = sgx.try_write32(val & !G17_BRIDGE_ENABLE, offset); + } + } + } + } + + /// Power witness for the queue-configuration port, accessible while the cores are gated. + pub(crate) fn g17_configuration_powered(&self) -> bool { + self.sgx_read32::() & 0xf != 0 + } + + /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is + /// initialized. + pub(crate) fn enable_g17_gating(&self) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let val = sgx.try_read32(G17_GATING_CONTROL)?; + sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) + } + + /// Split the packed per-cluster core masks into one mask per cluster. + /// + /// Returns the masks and the total number of enabled cores. + fn split_core_masks( + &self, + core_mask_regs: &mut [u32], + num_clusters: u32, + num_cores: u32, + ) -> Result<(KVec, u32)> { + let mut core_masks = KVec::new(); + let mut total_active_cores: u32 = 0; + + let max_core_mask = ((1u64 << num_cores) - 1) as u32; + for _ in 0..num_clusters { + let mask = core_mask_regs[0] & max_core_mask; + core_masks.push(mask, GFP_KERNEL)?; + for i in 0..core_mask_regs.len() { + core_mask_regs[i] = core_mask_regs[i].checked_shr(num_cores).unwrap_or(0); + if i < (core_mask_regs.len() - 1) { + core_mask_regs[i] |= core_mask_regs[i + 1] + .checked_shl(32 - num_cores) + .unwrap_or(0); + } + } + total_active_cores += mask.count_ones(); + } + + if core_mask_regs.iter().any(|a| *a != 0) { + dev_err!( + self.dev.as_ref(), + "Leftover core mask: {:#x?}\n", + core_mask_regs + ); + return Err(EIO); + } + + Ok((core_masks, total_active_cores)) + } + + /// Decode the silicon revision from the ID version register. + fn gpu_revision(&self, id_version: u32) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { + Ok(match (id_version >> 8) & 0xff { + 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), + 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), + 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), + 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), + 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), + 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU revision {}\n", a); + return Err(ENODEV); + } + }) + } + /// Get the fault information from the MMU status register, if one occurred. pub(crate) fn get_fault_info(&self, cfg: &'static hw::HwConfig) -> Option { let g14x = cfg.gpu_core as u32 >= hw::GpuCore::G14S as u32; From 68a0993d345b2de35bb288434a25d938207578ba Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:47:16 -0400 Subject: [PATCH 0599/1579] sched/core: add scoped utilization floors for kernel waiters Allow a kernel waiter to request a temporary minimum utilization while preserving the task's requested clamp and aggregate runqueue accounting. Balance nested scopes in LIFO order and restore the previous floor on exit. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 29a2e2acec42f8361d72bfd9506e3a0527afaf40) --- include/linux/sched.h | 10 +++++++ kernel/sched/core.c | 68 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/include/linux/sched.h b/include/linux/sched.h index 31b894db80f11..56073906c770e 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -904,6 +904,8 @@ struct task_struct { * Must be updated with task_rq_lock() held. */ struct uclamp_se uclamp[UCLAMP_CNT]; + /* Current-task kernel scope; separate from userspace clamp requests. */ + unsigned int uclamp_scope_min; #endif struct sched_statistics stats; @@ -1946,6 +1948,14 @@ extern void sched_set_fifo_low(struct task_struct *p); extern void sched_set_fifo_secondary(struct task_struct *p); extern void sched_set_normal(struct task_struct *p, int nice); extern int sched_setattr(struct task_struct *, const struct sched_attr *); +#ifdef CONFIG_UCLAMP_TASK +unsigned int sched_util_min_scope_enter(unsigned int value); +void sched_util_min_scope_exit(unsigned int previous); +#else +static inline unsigned int sched_util_min_scope_enter(unsigned int value) { return 0; } +static inline void sched_util_min_scope_exit(unsigned int previous) { } +#endif + extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *); extern struct task_struct *idle_task(int cpu); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 69c2aa8c62466..086908f82ea20 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1666,7 +1666,20 @@ uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id) struct uclamp_se uc_req = p->uclamp_req[clamp_id]; #ifdef CONFIG_UCLAMP_TASK_GROUP unsigned int tg_min, tg_max, value; +#endif + + /* + * Kernel scopes never replace user attributes or exceed their cap. + * The existing group and system restrictions are applied below. + */ + if (clamp_id == UCLAMP_MIN && p->uclamp_scope_min) { + unsigned int hint = min(p->uclamp_scope_min, + (unsigned int)p->uclamp_req[UCLAMP_MAX].value); + if (hint > uc_req.value) + uclamp_se_set(&uc_req, hint, false); + } +#ifdef CONFIG_UCLAMP_TASK_GROUP /* * Tasks in autogroups or root task group will be * restricted by system defaults. @@ -1902,6 +1915,58 @@ static inline void uclamp_rq_reinc_id(struct rq *rq, struct task_struct *p, rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; } +/* + * Only the current task may enter/leave its own balanced, nestable scope. + * pi/rq locking serializes the effective clamp with wakeup, migration and + * concurrent userspace attribute changes; user requests remain untouched. + */ +static unsigned int uclamp_scope_set(unsigned int value, bool raise) +{ + struct task_struct *p = current; + struct rq_flags rf; + struct rq *rq; + unsigned int previous; + + might_sleep(); + value = min(value, (unsigned int)SCHED_CAPACITY_SCALE); + if (value) + sched_uclamp_enable(); + rq = task_rq_lock(p, &rf); + previous = p->uclamp_scope_min; + if (raise) + value = max(value, previous); + if (value != previous) { + p->uclamp_scope_min = value; + uclamp_rq_reinc_id(rq, p, UCLAMP_MIN); + } + task_rq_unlock(rq, p, &rf); + return previous; +} + +/** + * sched_util_min_scope_enter - request a temporary current-task capacity floor + * @value: capacity hint, 0..1024, subject to user, cgroup and system caps + * + * May sleep. The caller must balance this scope in LIFO order on the same + * task with sched_util_min_scope_exit(), including error and signal paths. + * Return: opaque previous scope value for the matching exit. + */ +unsigned int sched_util_min_scope_enter(unsigned int value) +{ + return uclamp_scope_set(value, true); +} +EXPORT_SYMBOL_GPL(sched_util_min_scope_enter); + +/** + * sched_util_min_scope_exit - restore a balanced current-task capacity scope + * @previous: value returned by the matching sched_util_min_scope_enter() + */ +void sched_util_min_scope_exit(unsigned int previous) +{ + uclamp_scope_set(previous, false); +} +EXPORT_SYMBOL_GPL(sched_util_min_scope_exit); + static inline void uclamp_update_active(struct task_struct *p) { @@ -2058,6 +2123,9 @@ static void uclamp_fork(struct task_struct *p) { enum uclamp_id clamp_id; + /* A child never inherits its parent's temporary kernel scope. */ + p->uclamp_scope_min = 0; + /* * We don't need to hold task_rq_lock() when updating p->uclamp_* here * as the task is still at its early fork stages. From b0cc4eeb554a74c2db220e35ceb14955142a79ed Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:47:16 -0400 Subject: [PATCH 0600/1579] drm/syncobj: apply expiring capacity hints during blocking waits Associate a bounded utilization hint with a DRM file after successful submission. Apply it only around a blocking syncobj wait while it remains valid, and release the scheduler scope on every wait exit. Expose the hint setter to Rust DRM drivers. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 7c99d8debcc54ebf3f76ede14ced9b845db18215) --- drivers/gpu/drm/drm_syncobj.c | 55 +++++++++++++++++++++++++++++++++-- include/drm/drm_file.h | 4 +++ include/drm/drm_syncobj.h | 3 ++ rust/kernel/drm/file.rs | 8 +++++ 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c index c9dbf64c0c9f2..2ac67e51a5b15 100644 --- a/drivers/gpu/drm/drm_syncobj.c +++ b/drivers/gpu/drm/drm_syncobj.c @@ -1032,14 +1032,52 @@ static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj, list_del_init(&wait->node); } +/** + * drm_syncobj_set_wait_hint - tag recent GPU-feeder work on an open DRM file + * @file: submitting client + * @util_min: scheduler capacity hint, bounded to 1024 + * @duration_ms: validity for subsequent waits, bounded to one second + * + * This changes no scheduling attribute or affinity. A blocking syncobj wait + * may use the hint in a balanced kernel scope while this recent-work tag is + * valid. Sleeping tasks consume no CPU; the scope is removed before return. + */ +void drm_syncobj_set_wait_hint(struct drm_file *file, unsigned int util_min, + unsigned int duration_ms) +{ + unsigned long flags; + + spin_lock_irqsave(&file->table_lock, flags); + file->syncobj_wait_util_min = min(util_min, 1024u); + file->syncobj_wait_hint_until = jiffies + + msecs_to_jiffies(min(duration_ms, 1000u)); + spin_unlock_irqrestore(&file->table_lock, flags); +} +EXPORT_SYMBOL_GPL(drm_syncobj_set_wait_hint); + +static unsigned int drm_syncobj_wait_hint(struct drm_file *file) +{ + unsigned long flags; + unsigned int value = 0; + + spin_lock_irqsave(&file->table_lock, flags); + if (time_before(jiffies, file->syncobj_wait_hint_until)) + value = file->syncobj_wait_util_min; + spin_unlock_irqrestore(&file->table_lock, flags); + return value; +} + static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, void __user *user_points, uint32_t count, uint32_t flags, signed long timeout, uint32_t *idx, - ktime_t *deadline) + ktime_t *deadline, + struct drm_file *file_private) { + unsigned int scope_previous = 0, util_min; + bool scope_active = false; struct syncobj_wait_entry *entries; struct dma_fence *fence; uint64_t *points; @@ -1132,6 +1170,15 @@ static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, } } + /* Recheck expiry after allocations, user copies and deadline callbacks. */ + util_min = drm_syncobj_wait_hint(file_private); + /* Polls, availability waits and already-complete fences need no boost. */ + if (util_min && timeout > 0 && + !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { + scope_previous = sched_util_min_scope_enter(util_min); + scope_active = true; + } + do { set_current_state(TASK_INTERRUPTIBLE); @@ -1176,6 +1223,8 @@ static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, done_waiting: __set_current_state(TASK_RUNNING); + if (scope_active) + sched_util_min_scope_exit(scope_previous); cleanup_entries: for (i = 0; i < count; ++i) { @@ -1243,7 +1292,7 @@ static int drm_syncobj_array_wait(struct drm_device *dev, wait->count_handles, wait->flags, timeout, &first, - deadline); + deadline, file_private); if (timeout < 0) return timeout; wait->first_signaled = first; @@ -1254,7 +1303,7 @@ static int drm_syncobj_array_wait(struct drm_device *dev, timeline_wait->count_handles, timeline_wait->flags, timeout, &first, - deadline); + deadline, file_private); if (timeout < 0) return timeout; timeline_wait->first_signaled = first; diff --git a/include/drm/drm_file.h b/include/drm/drm_file.h index 6ee70ad65e1fd..b766e194561d9 100644 --- a/include/drm/drm_file.h +++ b/include/drm/drm_file.h @@ -320,6 +320,10 @@ struct drm_file { /** @syncobj_xa: Mapping of sync object handles to object pointers. */ struct xarray syncobj_xa; + /* Driver-provided recent-feeder hint, protected by table_lock. */ + unsigned int syncobj_wait_util_min; + unsigned long syncobj_wait_hint_until; + /** @filp: Pointer to the core file structure. */ struct file *filp; diff --git a/include/drm/drm_syncobj.h b/include/drm/drm_syncobj.h index b40052132e523..ef53cc2557b1b 100644 --- a/include/drm/drm_syncobj.h +++ b/include/drm/drm_syncobj.h @@ -133,4 +133,7 @@ int drm_syncobj_get_handle(struct drm_file *file_private, struct drm_syncobj *syncobj, u32 *handle); int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd); +void drm_syncobj_set_wait_hint(struct drm_file *file, unsigned int util_min, + unsigned int duration_ms); + #endif diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 3603e473c4e0f..c326301888028 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -48,6 +48,14 @@ impl File { self.0.get() } + /// Tag recent feeder work for temporary capacity hints during syncobj waits. + /// User scheduling attributes and CPU affinity remain unchanged. + pub fn set_syncobj_wait_hint(&self, util_min: u32, duration_ms: u32) { + // SAFETY: this File keeps the opened drm_file alive; the C helper + // bounds both arguments and serializes the per-file hint internally. + unsafe { bindings::drm_syncobj_set_wait_hint(self.as_raw(), util_min, duration_ms) }; + } + fn driver_priv(&self) -> *mut T { // SAFETY: By the type invariants of `Self`, `self.as_raw()` is always valid. unsafe { (*self.as_raw()).driver_priv }.cast() From d7d77474e4590fcd9c1d0e30ab16dc9ca0c73346 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:47:17 -0400 Subject: [PATCH 0601/1579] drm/asahi: request temporary CPU capacity for sustained GPU feeds Detect sustained render completion intervals and request temporary CPU capacity while submissions remain latency-sensitive. Retain detector ownership through completion, service the actuator on one work item, and remove its frequency requests after joining producers during shutdown. Provide an expiring syncobj wait hint only while the detector and applied capacity request agree. Keep this policy separate from firmware execution, queue depth and resource retirement. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit bc4c50abf8ede16fec7f2a79546a310134b208df) --- drivers/gpu/drm/asahi/file.rs | 6 + drivers/gpu/drm/asahi/g17/completion.rs | 6 + drivers/gpu/drm/asahi/g17/feed.rs | 330 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/job.rs | 5 + drivers/gpu/drm/asahi/g17/mod.rs | 15 + drivers/gpu/drm/asahi/g17/queue/mod.rs | 2 + drivers/gpu/drm/asahi/g17/runtime.rs | 1 + drivers/gpu/drm/asahi/g17/runtime/backend.rs | 4 + drivers/gpu/drm/asahi/g17/runtime/events.rs | 1 + drivers/gpu/drm/asahi/g17/runtime/feed.rs | 33 ++ drivers/gpu/drm/asahi/gpu.rs | 5 +- 11 files changed, 407 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/asahi/g17/feed.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/feed.rs diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 874b13099c367..2f237c4b06a1d 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -1268,6 +1268,12 @@ impl File { .lock() .submit(id, syncs, data.in_sync_count as usize, &vec, objects); + if ret.is_ok() { + if let Some((util_min, duration_ms)) = gpu.syncobj_wait_hint() { + file.set_syncobj_wait_hint(util_min, duration_ms); + } + } + match ret { Err(ERESTARTSYS) => Err(ERESTARTSYS), Err(e) => { diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 3ac8fe711ca61..1151a82084cee 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -89,6 +89,7 @@ pub(crate) struct Completion { member: Member, destinations: [Option; 4], compute: Option, + feed: Option>, #[pin] vm_job: Mutex>, #[pin] @@ -111,6 +112,7 @@ impl Completion { objects: Pin<&xarray::XArray>>, timestamps: &[uapi::drm_asahi_timestamps], compute: bool, + feed: Option>, ) -> Result> { if timestamps.len() != if compute { 1 } else { 2 } { return Err(EINVAL); @@ -133,6 +135,7 @@ impl Completion { member, destinations, compute, + feed, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), signalled: AtomicBool::new(false), @@ -309,6 +312,9 @@ impl Completion { if self.signalled.swap(true, Ordering::AcqRel) { return; } + if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { + feed.note_render_pass(ticks[2], ticks[3]); + } let result = result.and_then(|ticks| { for (destination, ticks) in self.destinations.iter().zip(ticks) { if let Some(destination) = destination { diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs new file mode 100644 index 0000000000000..ca3ecc85ee25a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Detects sustained host refill gaps and temporarily requests CPU capacity. +//! Detector updates are lock-free. The device's single feed worker owns the +//! frequency requests; its lock never nests inside a driver publication lock. +//! Shutdown first closes the detector, joins its producers and the worker, +//! then removes requests. All request storage and policy references are owned +//! by this device, independently of other GPU instances. + +use crate::hw::t8140::qos::CLOCK_HZ; +use core::{ + pin::Pin, + ptr, + sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, +}; +use kernel::{ + bindings, new_mutex, + prelude::*, + sync::{Arc, Mutex}, + time::{delay::fsleep, ClockSource, Delta, Monotonic}, + types::Opaque, +}; + +const MAX_REFILL_TICKS: u64 = CLOCK_HZ / 500; +const STREAK_ON: u32 = 8; +const STREAK_MAX: u32 = 64; +const LINGER_NS: u64 = 8_000_000; +const POLL_MS: i64 = 2; +const FLOOR_PERCENT: u64 = 50; +const WAIT_UTIL_MIN: u32 = 512; +const WAIT_HINT_MS: u32 = 8; +const MAX_POLICIES: usize = 8; + +struct Detector { + last_end: AtomicU64, + idle: AtomicU64, + busy: AtomicU64, + streak: AtomicU32, + until: AtomicU64, + active: AtomicBool, + pending: AtomicBool, + stopping: AtomicBool, +} + +impl Detector { + const fn new() -> Self { + Self { + last_end: AtomicU64::new(0), + idle: AtomicU64::new(0), + busy: AtomicU64::new(0), + streak: AtomicU32::new(0), + until: AtomicU64::new(0), + active: AtomicBool::new(false), + pending: AtomicBool::new(false), + stopping: AtomicBool::new(false), + } + } + + fn decay_add(sum: &AtomicU64, add: u64) -> u64 { + let mut next = 0; + let _ = sum.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| { + next = value - value / 8 + add; + Some(next) + }); + next + } + + fn note(&self, start: u64, end: u64, now: u64) { + if end <= start { + return; + } + let previous = self.last_end.fetch_max(end, Ordering::AcqRel); + let (idle, busy, paced) = if previous == 0 { + (0, end - start, true) + } else if start >= previous { + let gap = start - previous; + if gap < MAX_REFILL_TICKS { + (gap, end - start, false) + } else { + (0, end - start, true) + } + } else if end > previous { + (0, end - previous, false) + } else { + (0, 0, false) + }; + let idle_sum = Self::decay_add(&self.idle, idle); + let busy_sum = Self::decay_add(&self.busy, busy); + let streak = if paced { + self.streak.store(0, Ordering::Release); + 0 + } else { + let previous = self + .streak + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| { + Some((value + 1).min(STREAK_MAX)) + }); + // The closure always returns Some; fetch_update retries contention. + match previous { + Ok(value) => value + 1, + Err(_) => 0, + } + }; + if streak >= STREAK_ON && idle_sum.saturating_mul(4) >= busy_sum { + self.until.store(now + LINGER_NS, Ordering::Release); + if !self.active.swap(true, Ordering::AcqRel) { + self.pending.store(true, Ordering::Release); + } + } + } + + fn wanted(&self, now: u64) -> bool { + !self.stopping.load(Ordering::Acquire) && now < self.until.load(Ordering::Acquire) + } +} + +#[pin_data] +struct Requests { + #[pin] + requests: Opaque<[bindings::freq_qos_request; MAX_POLICIES]>, + policies: [*mut bindings::cpufreq_policy; MAX_POLICIES], + maxima: [u32; MAX_POLICIES], + applied: [u32; MAX_POLICIES], + count: usize, +} + +// SAFETY: Each nonnull policy retains cpufreq_cpu_get's reference. Access to +// the pinned request array and metadata is exclusive under Feed::requests. +unsafe impl Send for Requests {} + +impl Requests { + fn new() -> Self { + Self { + requests: Opaque::zeroed(), + policies: [ptr::null_mut(); MAX_POLICIES], + maxima: [0; MAX_POLICIES], + applied: [0; MAX_POLICIES], + count: 0, + } + } + + fn register(self: Pin<&mut Self>) { + let this = self.project(); + if *this.count != 0 { + return; + } + // SAFETY: nr_cpu_ids is initialized before driver probe. + let cpus = unsafe { bindings::nr_cpu_ids }; + for cpu in 0..cpus { + // SAFETY: Accepts any CPU number; a nonnull result owns one reference. + let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; + if policy.is_null() { + continue; + } + let count = *this.count; + if count < MAX_POLICIES && !this.policies[..count].contains(&policy) { + // SAFETY: The policy is referenced. This zeroed, unregistered + // slot is pinned and remains alive until remove_request. + let result = unsafe { + bindings::freq_qos_add_request( + ptr::addr_of_mut!((*policy).constraints), + this.requests + .get() + .cast::() + .add(count), + bindings::freq_qos_req_type_FREQ_QOS_MIN, + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as i32, + ) + }; + if result >= 0 { + this.policies[count] = policy; + // SAFETY: cpuinfo is immutable while this policy is referenced. + this.maxima[count] = unsafe { (*policy).cpuinfo.max_freq }; + this.applied[count] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + *this.count += 1; + continue; + } + } + // SAFETY: Balances the reference acquired above for a duplicate, + // unavailable slot or failed registration. Successful slots retain it. + unsafe { bindings::cpufreq_cpu_put(policy) }; + } + } + + fn apply(mut self: Pin<&mut Self>, enabled: bool) -> bool { + if enabled { + self.as_mut().register(); + } + let this = self.project(); + for index in 0..*this.count { + let floor = if enabled { + (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 + } else { + 0 + }; + let value = if floor == 0 { + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 + } else { + floor + }; + if this.applied[index] == value { + continue; + } + this.applied[index] = value; + // SAFETY: This slot is registered, pinned and exclusively locked; + // its referenced policy keeps the constraints alive. + unsafe { + bindings::freq_qos_update_request( + this.requests + .get() + .cast::() + .add(index), + value as i32, + ) + }; + } + *this.count != 0 && enabled + } + + fn clear(self: Pin<&mut Self>) { + let this = self.project(); + for index in 0..*this.count { + // SAFETY: Registered pinned slot, exclusively locked. Remove the + // constraint link before releasing its policy reference. + unsafe { + bindings::freq_qos_remove_request( + this.requests + .get() + .cast::() + .add(index), + ); + bindings::cpufreq_cpu_put(this.policies[index]); + } + this.policies[index] = ptr::null_mut(); + } + *this.count = 0; + } +} + +#[pin_data(PinnedDrop)] +pub(crate) struct Feed { + detector: Detector, + #[pin] + requests: Mutex, + applied: AtomicBool, +} + +impl Feed { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + detector: Detector::new(), + requests <- new_mutex!(Requests::new(), "G17 CPU feed"), + applied: AtomicBool::new(false), + }), + GFP_KERNEL, + ) + } + + fn now() -> u64 { + Monotonic::ktime_get() as u64 + } + + pub(crate) fn note_render_pass(&self, start: u64, end: u64) { + self.detector.note(start, end, Self::now()); + } + + /// Called after a firmware event pass, before queuing the single feed work item. + pub(crate) fn take_pending(&self) -> bool { + self.detector.pending.swap(false, Ordering::AcqRel) + } + + fn apply(&self, enabled: bool) { + let mut requests = self.requests.lock(); + let enabled = enabled && !self.detector.stopping.load(Ordering::Acquire); + let applied = requests.as_mut().apply(enabled); + self.applied.store(applied, Ordering::Relaxed); + } + + /// Worker context; the linger deadline has no firmware event of its own. + pub(crate) fn run(&self) { + loop { + if self.detector.wanted(Self::now()) { + self.apply(true); + fsleep(Delta::from_millis(POLL_MS)); + continue; + } + self.apply(false); + self.detector.active.store(false, Ordering::SeqCst); + // A detector can extend the deadline after observing the old active + // bit. Reclaim that work unless it already queued another owner. + if self.detector.wanted(Self::now()) + && !self.detector.active.swap(true, Ordering::AcqRel) + { + continue; + } + return; + } + } + + pub(crate) fn wait_hint(&self) -> Option<(u32, u32)> { + if cfg!(CONFIG_UCLAMP_TASK) + && self.detector.wanted(Self::now()) + && self.applied.load(Ordering::Relaxed) + { + Some((WAIT_UTIL_MIN, WAIT_HINT_MS)) + } else { + None + } + } + + /// Call before disabling/joining the event producer and actuator work items. + pub(crate) fn begin_shutdown(&self) { + self.detector.stopping.store(true, Ordering::Release); + } + + /// Call after work items are joined. Removing requests withdraws their floor. + pub(crate) fn shutdown(&self) { + self.begin_shutdown(); + self.requests.lock().as_mut().clear(); + self.applied.store(false, Ordering::Relaxed); + } +} + +#[pinned_drop] +impl PinnedDrop for Feed { + fn drop(self: Pin<&mut Self>) { + self.shutdown(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index ed2e79537d597..0d28b80baa633 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -222,6 +222,11 @@ fn check_status(fence: &Fence) -> Result { /// The backend retains visible packets through exact retirement or processor stop. /// It performs all completion signalling after releasing the device mutex. pub(crate) trait Backend: Send + Sync { + /// Device detector retained by render completions after queue teardown. + fn feed(&self) -> Option> { + None + } + /// Lazily create or rebind this logical queue's physical compute graph. fn ensure_compute(&self) -> Result; /// Settle outstanding work of a terminal VM before an ioctl error is published. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index fe354a52968cc..a574d54040479 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -25,6 +25,7 @@ mod completion; pub(crate) mod context; mod dependency; mod event; +mod feed; mod fence; mod freelist; pub(crate) mod fw; @@ -579,6 +580,7 @@ const POLL_WORK: u64 = 1; const IDLE_WORK: u64 = 2; const GROW_WORK: u64 = 3; const RECLAIM_WORK: u64 = 4; +const FEED_WORK: u64 = 5; /// Device state shared with the coprocessor callbacks and the event worker. #[pin_data] @@ -587,6 +589,9 @@ struct Shared { clusters: u32, descriptor_flags: [u32; 2], crashed: AtomicBool, + feed: Arc, + #[pin] + feed_work: Work, preparations: Arc, #[pin] work: Work, @@ -843,12 +848,15 @@ impl Gpu { let max_frequency_khz = Self::max_frequency_khz(dev)?; let preparations = preparation::Gate::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; + let feed = feed::Feed::new()?; let shared = Arc::pin_init( pin_init!(Shared { dev: dev.into(), clusters: id.num_clusters, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), + feed, + feed_work <- new_work!("g17::Shared::feed_work"), preparations, work <- new_work!("g17::Shared::work"), poll <- new_delayed_work!("g17::Shared::poll"), @@ -1004,6 +1012,7 @@ impl Gpu { /// Stops the firmware and releases everything it used. pub(crate) fn shutdown(&self) { + self.shared.feed.begin_shutdown(); self.shared.preparations.remove(); self.shared.stop_queues(); self.shared.disable_timer(&self.shared.poll); @@ -1011,11 +1020,17 @@ impl Gpu { self.shared.disable_work(&self.shared.grow); self.shared.disable_work(&self.shared.reclaim_work); self.shared.disable_work(&self.shared.work); + self.shared.disable_work(&self.shared.feed_work); + self.shared.feed.shutdown(); self.shared.drain_reclaims(); } } impl gpu::Gpu for Gpu { + fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { + self.shared.feed.wait_hint() + } + fn as_any(&self) -> &dyn Any { self } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index afc58009f4029..1b349abbee86b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -162,6 +162,7 @@ impl Queue { objects, timestamps, false, + self.backend.feed(), )?, Validated::Compute { timestamps, .. } => Completion::new( &self.dev, @@ -172,6 +173,7 @@ impl Queue { objects, core::slice::from_ref(timestamps), true, + None, )?, }; if engine == Engine::Compute { diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index e908ca3e08e23..d7a71a2c4c899 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -13,6 +13,7 @@ mod compute_control; mod compute_replay; mod compute_exit; mod events; +mod feed; mod render; pub(super) mod teardown; diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index f1ae1ffe80783..085655beb8194 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -66,6 +66,10 @@ impl Backend { } impl job::Backend for Backend { + fn feed(&self) -> Option> { + Some(self.shared.feed.clone()) + } + fn ensure_compute(&self) -> Result { Backend::ensure_compute(self) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index e9fc977d623b8..43a27b272ac88 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -311,6 +311,7 @@ impl crate::g17::Shared { }; let result = self.runtime_event_turn(&mut deferred); deferred.finish(); + self.queue_feed(); if let Some(firmware) = (*self.state.lock()).as_deref_mut() { firmware.queues.worker_deferred = Some(deferred); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/feed.rs b/drivers/gpu/drm/asahi/g17/runtime/feed.rs new file mode 100644 index 0000000000000..f3faffb89df76 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/feed.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Schedules CPU capacity updates after completed firmware event passes. The +//! worker owns frequency requests independently of the device publication lock. + +use crate::g17::{Shared, FEED_WORK}; +use kernel::{ + impl_has_work, + sync::Arc, + workqueue::{self, WorkItem}, +}; + +impl_has_work! { + impl HasWork for Shared { self.feed_work } +} + +impl WorkItem for Shared { + type Pointer = Arc; + + fn run(this: Arc) { + this.feed.run(); + } +} + +impl Shared { + /// The completion callbacks have finished outside the device mutex before + /// the event worker hands the detector's request to the actuator. + pub(in crate::g17) fn queue_feed(self: &Arc) { + if self.feed.take_pending() { + let _ = workqueue::system_unbound().enqueue::, FEED_WORK>(self.clone()); + } + } +} diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 34823251fb755..5f30660ed2fc4 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -287,7 +287,10 @@ pub(crate) trait Gpu: Send + Sync { fn queue_limits(&self) -> Option { None } - + /// Temporary capacity hint and validity in milliseconds for subsequent syncobj waits. + fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { + None + } /// Frequency of the CPU timer that GPU timestamps are derived from. fn base_clock_hz(&self) -> u32; /// Return the GPU address space geometry. From 15d553b170f157558da0343d477c39161f9870f1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:34:52 -0400 Subject: [PATCH 0602/1579] drm/asahi: observe compute completions without a queue mask An event batch without a usable queue identity must still observe completed compute commands. Use the full scan for zero masks while retaining QID filtering for identified batches, so coalesced notifications cannot strand an otherwise retired command. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e1e0c5c90faf0307f95374c83efe3009d58f0bdb) --- drivers/gpu/drm/asahi/g17/runtime/events.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 43a27b272ac88..6776707fd944c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -92,7 +92,10 @@ impl event::Queues for Service<'_> { self.render_masks[1] |= masks[1]; } fn complete_compute(&mut self, masks: [u64; 2]) -> Result { - self.firmware.scan_compute(Some(masks), self.deferred) + // A batch without a queue identity still observes completions whose + // notification was coalesced; identified batches retain QID filtering. + let selected = (masks != [0; 2]).then_some(masks); + self.firmware.scan_compute(selected, self.deferred) } } From 7dcd8d1467e2bc1e90eeb04489d4ab7b392351e8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:34:52 -0400 Subject: [PATCH 0603/1579] drm/asahi: avoid repeated submission bookkeeping Consume prepared VM guards in order without shifting the remaining vector for every command. Iterator destruction still releases unused pins on partial enqueue failure. Capture shared attachment metadata under one binding-lock hold per footprint, preserving range insertion order and releasing the lock before allocation. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 0c69741cdc26f2dbb26754e87f1b53db1f4bed38) --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 5 +++-- drivers/gpu/drm/asahi/mmu/shared.rs | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 1b349abbee86b..5495f3895abd3 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -284,11 +284,12 @@ impl crate::queue::Queue for Queue { if self.context.status().get() != 0 { return Err(EIO); } - let (pending, mut guards) = self.prepare_submission(bytes)?; + let (pending, guards) = self.prepare_submission(bytes)?; + let mut guards = guards.into_iter(); let aggregate = Submission::new(&self.fences, self.context.clone())?; outputs.publish(aggregate.clone()); for (command, order) in pending { - let guard = guards.remove(0).map_err(|_| EIO)?; + let guard = guards.next().ok_or(EIO)?; self.enqueue( id, command, diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs index cf29e53fc9cc9..5b87c55a06b72 100644 --- a/drivers/gpu/drm/asahi/mmu/shared.rs +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -106,16 +106,15 @@ impl Vm { return Ok(None); } let mut footprint = Footprint::new(self.id); + let bindings = self.shared_bindings.as_ref().and_then(|shared| { + (shared.count.load(Ordering::Relaxed) != 0).then(|| shared.bindings.lock()) + }); for attachment in attachments.as_slice() { let end = attachment.va.saturating_add(attachment.size); footprint.push_vm(attachment.va, end); - let Some(shared) = &self.shared_bindings else { + let Some(bindings) = &bindings else { continue; }; - if shared.count.load(Ordering::Relaxed) == 0 { - continue; - } - let bindings = shared.bindings.lock(); for entry in bindings.iter() { let first = entry.start.max(attachment.va); let last = entry.end.min(end); @@ -129,6 +128,7 @@ impl Vm { } } } + drop(bindings); Ok(Some(Arc::new(footprint, GFP_KERNEL)?)) } } From ffa3234a92d789514f3e6b05213f417454b73639 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:25:07 -0400 Subject: [PATCH 0604/1579] drm/asahi: settle unpublished jobs discarded by entity teardown Entity teardown can free a queued job without invoking its cancellation callback. Signal its independent completion fence through the existing unpublished cancellation path so cross-engine timestamp dependencies can retire. Leave jobs whose publication has already been claimed with their existing owner. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e0d2761bd854136076d642b614c9b6078699243e) --- drivers/gpu/drm/asahi/g17/job.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 0d28b80baa633..f16897efa00b8 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -249,6 +249,17 @@ struct Job { packet: Arc, } +impl Drop for Job { + fn drop(&mut self) { + // Entity teardown can free a queued job without invoking cancel. + // Its independent fence may still order another engine. A claimed + // publisher keeps responsibility for visible work and its VM pin. + if self.packet.publication.load(Ordering::Acquire) == 0 { + self.backend.cancel(self.packet.clone()); + } + } +} + impl sched::JobImpl for Job { fn false_timeout(job: &mut sched::Job) -> bool { job.is_finished() From 0a72b08543fdec99aa8117c8b4983eeee9b8c856 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:25:08 -0400 Subject: [PATCH 0605/1579] drm/asahi: reserve completion capacity for global compute replay Submission collectors are sized for one physical queue, but recovery can settle retained commands from every compute queue. Reserve for the complete replay pass before changing any queue, including the publication that follows it. Skip reservation when no queue needs recovery work, preserving the small collector used by healthy submissions. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d41b760af0bcc10744b4145cfdbc19dab7625e8e) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 10 ++++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 8 ++++++++ drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs | 12 ++++++++++++ 3 files changed, 30 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index f8297f0462ff8..832d5073ecc7b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -798,6 +798,16 @@ impl Queue { Ok(()) } + /// Both FIFOs are bounded by IN_FLIGHT. Count all retained commands of + /// queues which this pass can release, replay or quarantine on failure. + pub(crate) fn replay_commands(&self) -> usize { + if self.awaiting_witness || self.replays.len != 0 { + self.active.len + self.replays.len + } else { + 0 + } + } + pub(crate) fn replay_front(&self) -> Option> { self.replays.front().cloned() } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index d7a71a2c4c899..6fe67478a777f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -501,6 +501,14 @@ impl DeferredBatch { values: KVVec::with_capacity(capacity, GFP_KERNEL)?, }) } + /// A recovery pass can fail and retire each retained command at most once. + /// Reserve before changing queues, plus a full publication for its caller. + /// Healthy submissions keep the small publication collector unchanged. + fn reserve_replays(&mut self, commands: usize) -> Result { + self.values + .reserve(2 * commands + 2 * Self::DEPTH, GFP_KERNEL)?; + Ok(()) + } pub(super) fn push(&mut self, value: super::completion::Deferred) -> Result { use super::completion::Deferred; // Publishers may acquire the device mutex as soon as this pass drops diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 17f228b9d1f04..c872a7c8dacc7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -18,6 +18,18 @@ impl Firmware { if state != 0 { return Ok(()); } + let commands = self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .map(|queue| queue.replay_commands()) + .sum::(); + if commands == 0 { + return Ok(()); + } + deferred.reserve_replays(commands)?; let view = qos::View::new(self.init.qos()?)?; for entry in self.queues.compute.iter_mut().flatten() { if let Some(queue) = entry.queue.as_deref_mut() { From 945285b800d30fcc9a60461b70249b96b4c053ad Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:30:36 -0400 Subject: [PATCH 0606/1579] drm/asahi: filter compute completion identities before payload reads A shared completion scan needs only the control word of unrelated QIDs. Retain a bounded record view, capture control once, and read descriptor and queue identity before the two timestamps used for retirement. This avoids unused payload loads while preserving the single Acquire-ordered scan and exact kick matching. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 868c80c921527154df5e73a271583a4b0b63fdeb) --- drivers/gpu/drm/asahi/g17/initdata.rs | 33 ++++++++++++++----- .../drm/asahi/g17/queue/compute/retirement.rs | 22 ++++++++----- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index f8160f0faa36c..22b67575c133c 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -29,6 +29,26 @@ const COMPUTE_CONTROL_BLOCKS: u32 = 25; const COMPUTE_CONTROL_CURSOR: u32 = 0x18; const COMPUTE_RUN_ADDRESS_MASK: u64 = 0x0000_ffff_ffff_f000; +/// A borrowed completion slot. Read the identity before requesting its timestamps; +/// unrelated QIDs need only the control word. Each access is an aligned live load. +pub(crate) struct ComputeCompletion<'a> { + object: &'a KernelObject, + offset: usize, +} + +impl ComputeCompletion<'_> { + pub(crate) fn read(&self, word: usize) -> Result { + if word >= 8 { + return Err(EINVAL); + } + // The scanner bounds offset to a 0x40-byte slot in the completion ring. + Ok(self + .object + .dword(self.offset + word * 8)? + .load(Ordering::Relaxed)) + } +} + /// Unpublished device-wide compute backing, prepared without the device mutex. The cache /// identity belongs to this allocation, not to any one physical queue or client VM. pub(super) struct ComputeShared { @@ -555,18 +575,15 @@ impl InitData { /// scanner latches exact identities; firmware retains sole ownership of these slots. pub(super) fn scan_compute_completions( &self, - mut record: impl FnMut([u64; 8]) -> Result, + mut record: impl FnMut(ComputeCompletion<'_>) -> Result, ) -> Result { let object = self.object(self.completion_rings[0])?; fence(Ordering::Acquire); for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { - let mut words = [0; 8]; - for (index, word) in words.iter_mut().enumerate() { - *word = object - .dword(slot * 0x40 + index * 8)? - .load(Ordering::Relaxed); - } - record(words)?; + record(ComputeCompletion { + object, + offset: slot * 0x40, + })?; } Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 7428f99ab1d7f..766177ebc32c9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -3,6 +3,7 @@ //! Exact identities and visibility witnesses for a physical compute queue. //! A shared work-state counter can include other engines and cannot prove this queue retired. +use crate::g17::initdata::ComputeCompletion; use kernel::prelude::*; pub(crate) const IN_FLIGHT: usize = 16; @@ -170,17 +171,20 @@ impl Batch { self.open = Some(ticket.qid); Ok(()) } - /// The transport reads eight aligned words from a 0x40-byte firmware record. Records can be - /// overwritten during a scan: incomplete or older identities are skipped, never consumed. - pub(crate) fn fold(&mut self, words: [u64; 8]) -> Result { - let qid = ((words[0] >> 9) & 0x7f) as usize; + /// Capture the control word once, then read only a requested identity's payload. + /// Records can be overwritten during a scan: incomplete or older identities are skipped. + pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { + let control = record.read(0)?; + let qid = ((control >> 9) & 0x7f) as usize; let (start, count) = self.slots[qid]; - if count == 0 || words[6] & !1 == 0 || words[7] == 0 { + if count == 0 { return Ok(()); } - let descriptor = words[6] & !1; - let kick = - (((words[0] >> 8) & 0xffff_ffff_00) | ((words[0] >> 48) & 0xff)) & 0xff_ffff_ffff; + let descriptor = record.read(6)? & !1; + if descriptor == 0 || record.read(7)? == 0 { + return Ok(()); + } + let kick = (((control >> 8) & 0xffff_ffff_00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; let Some(request) = requests .iter_mut() @@ -188,7 +192,7 @@ impl Batch { else { return Ok(()); }; - let timestamps = [words[2] & TIME_MASK, words[5] & TIME_MASK]; + let timestamps = [record.read(2)? & TIME_MASK, record.read(5)? & TIME_MASK]; if timestamps[0] == 0 || timestamps[1] == 0 || request From 513f7568ad7eb30b97ac1309f37bbb3a11229409 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:31:21 -0400 Subject: [PATCH 0607/1579] drm/asahi: sample opening counters at the polling deadline Read the opening control counters after the final one-millisecond wait, matching the existing firmware initialization polling budget. Firmware that retires the records during that last interval now completes initialization instead of being torn down as timed out. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f61c86ab28302e46cba8fbdbaa8339aaa4e17b40) --- drivers/gpu/drm/asahi/g17/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index a574d54040479..35916dd41bfa4 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -992,7 +992,7 @@ impl Gpu { primary: &Coprocessor, secondary: &Coprocessor, ) -> Result { - for _ in 0..OPENING_RETIRE_POLLS { + for poll in 0..=OPENING_RETIRE_POLLS { if primary.state.crashed.load(Ordering::Acquire) || secondary.state.crashed.load(Ordering::Acquire) { @@ -1001,7 +1001,9 @@ impl Gpu { if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { return Ok(()); } - fsleep(Delta::from_millis(OPENING_RETIRE_POLL_MS)); + if poll < OPENING_RETIRE_POLLS { + fsleep(Delta::from_millis(OPENING_RETIRE_POLL_MS)); + } } dev_err!( dev.as_ref(), From c0bd8b9fec163274b9e6056ae94fe54d53b565fb Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:44:57 -0400 Subject: [PATCH 0608/1579] drm/asahi: clarify the opening polling budget The initial sample precedes the bounded wait intervals; the final interval also ends with a counter sample. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit b4ee9bd8d923949b19cc22484262670948cafc57) --- drivers/gpu/drm/asahi/g17/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 35916dd41bfa4..a62f1419d6122 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -180,7 +180,7 @@ const INITDATA_ACK_TIMEOUT_MS: u32 = 2000; const RECEIVER_ATTEMPTS: usize = 3; const RECEIVER_RETRY_MS: i64 = 20; /// The firmware does not notify the host when it retires device-control records, so their -/// counters are polled at this interval, at most [`OPENING_RETIRE_POLLS`] times. +/// counters are sampled initially and after at most [`OPENING_RETIRE_POLLS`] waits. const OPENING_RETIRE_POLL_MS: i64 = 1; const OPENING_RETIRE_POLLS: usize = 100; From 26ba1aa06add76aa7e97a843d752fe0d08afd617 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:49:09 -0400 Subject: [PATCH 0609/1579] drm/asahi: commit canonical binding metadata with each user mapping Prepared mapping batches can fail after earlier leaves have committed. Update canonical context metadata with each successful leaf so context setup observes mappings already visible in the VM. Preflight arithmetic in a compact map plan and retain displaced GEM references until reservation locks are released. Preserve canonical invalidation when quarantined unmaps enter deferred cleanup. Remove the duplicate ioctl update vector, extra object references and post-batch metadata pass. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 99dd8deec12fe28abb844854059bce6a87764199) --- drivers/gpu/drm/asahi/file.rs | 36 -------- drivers/gpu/drm/asahi/mmu/bind.rs | 19 ++++- drivers/gpu/drm/asahi/mmu/context.rs | 117 +++++++++++++++++++------- drivers/gpu/drm/asahi/mmu/lifetime.rs | 9 ++ 4 files changed, 110 insertions(+), 71 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 2f237c4b06a1d..b5da3018d96bd 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -37,7 +37,6 @@ use kernel::error::code::*; use kernel::new_mutex; use kernel::prelude::*; use kernel::sync::{ - aref::ARef, Arc, Mutex, // }; @@ -57,17 +56,6 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::File; pub(crate) const MAX_COMMANDS_PER_SUBMISSION: u32 = 64; -/// Context-view registry updates applied after a prepared VM batch commits. -enum MappingUpdate { - Map { - object: ARef, - address: u64, - size: u64, - offset: u64, - }, - Unmap(Range), -} - /// A client instance of an `mmu::Vm` address space. struct Vm { ualloc: Arc>, @@ -700,7 +688,6 @@ impl File { ) }; let mut batch = mmu::PreparedUserBindBatch::new(count)?; - let mut updates = KVec::with_capacity(count, GFP_KERNEL)?; let mut unmaps = 0; // A fresh reader per declared stride also skips extension bytes in longer records. for bytes in bytes.chunks_exact(stride) { @@ -723,7 +710,6 @@ impl File { return Err(EINVAL); } batch.push_unmap(vm.prepare_user_unmap(op.addr, op.range)?)?; - updates.push(MappingUpdate::Unmap(range), GFP_KERNEL)?; unmaps += 1; continue; } @@ -763,31 +749,9 @@ impl File { prot, single_page, )?)?; - updates.push( - MappingUpdate::Map { - object, - address: op.addr, - size: op.range, - offset: op.offset, - }, - GFP_KERNEL, - )?; } vm.reserve_deferred_user_unmaps(unmaps)?; vm.commit_prepared_user_bind_batch(&mut batch)?; - for update in updates { - match update { - MappingUpdate::Map { - object, - address, - size, - offset, - } => { - vm.track_context_binding(object, address, size, offset)?; - } - MappingUpdate::Unmap(range) => vm.untrack_context_range(range), - } - } vm.bo_deferred_cleanup(); Ok(0) } diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 0d5abcf9b84f1..89c7db4f3d2e7 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -13,6 +13,7 @@ pub(crate) struct PreparedUserMap { size: u64, offset: u64, single_page: bool, + context: context::PreparedBinding, } pub(crate) struct PreparedUserUnmap { @@ -58,6 +59,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + let context = self.prepare_context_binding(addr, size, offset)?; let mut ctx = StepContext { new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), @@ -95,6 +97,7 @@ impl Vm { size, offset, single_page, + context, }) } @@ -139,7 +142,8 @@ impl Vm { repeat, flags, ); - if result.is_ok() { + let retired = result.is_ok().then(|| { + let retired = self.commit_context_binding(&map.gem, &map.context); self.track_shared_binding( &map.gem, map.addr, @@ -147,9 +151,11 @@ impl Vm { map.offset, map.single_page, ); - } + retired + }); drop(inner); drop(commit); + drop(retired); Some(result) } Err(error) if error == EIO => { @@ -164,11 +170,16 @@ impl Vm { match self.mapping_commit() { Ok(commit) => { let result = inner.sm_unmap(&mut unmap.ctx, unmap.iova, unmap.size); - if result.is_ok() { + let retired = result.is_ok().then(|| { + let retired = self.untrack_context_ranges(core::iter::once( + unmap.iova..unmap.iova + unmap.size, + )); self.untrack_shared_range(unmap.iova, unmap.size); - } + retired + }); drop(inner); drop(commit); + drop(retired); Some(result) } Err(error) if error == EIO => { diff --git a/drivers/gpu/drm/asahi/mmu/context.rs b/drivers/gpu/drm/asahi/mmu/context.rs index c47b7ae1fd782..efebcc6a5a84f 100644 --- a/drivers/gpu/drm/asahi/mmu/context.rs +++ b/drivers/gpu/drm/asahi/mmu/context.rs @@ -4,6 +4,8 @@ //! //! Userspace owns low views 1 through 6. Only the reserved null view is created //! here. Its installation is serialized per VM and retained through VM teardown. +//! Commits take bindings after the GPUVM exec lock; deferred unmaps take it +//! after the VM lifetime lock. Readers release bindings before taking either. use super::*; use crate::hw::t8140::CONTEXT_VIEWS; @@ -17,6 +19,24 @@ struct Bindings { generation: u64, slots: [Option; CONTEXT_VIEWS.len()], } +/// All arithmetic for a canonical map is checked before its page-table commit. +pub(super) struct PreparedBinding { + address: u64, + object_offset: u64, + touched: u8, + covered: u8, +} + +/// Final GEM release can acquire DMA reservations. Keep replaced references +/// until the caller has released its GPUVM and mapping-admission locks. +pub(super) struct RetiredBindings([Option; CONTEXT_VIEWS.len()]); + +impl RetiredBindings { + fn new() -> Self { + Self(core::array::from_fn(|_| None)) + } +} + #[pin_data] pub(super) struct ContextBindings { #[pin] @@ -57,64 +77,99 @@ impl Vm { Ok(()) } - /// Arithmetic is preflighted before invalidating any retained candidate. - pub(crate) fn track_context_binding( + pub(super) fn prepare_context_binding( &self, - gem: ARef, address: u64, size: u64, object_offset: u64, - ) -> Result { - let Some(context) = self.context_bindings.as_ref() else { - return Ok(()); + ) -> Result { + let mut update = PreparedBinding { + address, + object_offset, + touched: 0, + covered: 0, }; + if self.context_bindings.is_none() { + return Ok(update); + } let end = address.checked_add(size).ok_or(EOVERFLOW)?; - let mut offsets = [None; CONTEXT_VIEWS.len()]; for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if address < view.source + view.size as u64 && view.source < end { + update.touched |= 1 << index; + } if address <= view.source && view.source + view.size as u64 <= end { - offsets[index] = Some(usize::try_from( + update.covered |= 1 << index; + usize::try_from( object_offset .checked_add(view.source - address) .ok_or(EOVERFLOW)?, - )?); + )?; } } + Ok(update) + } + + /// Called after a successful leaf commit, while its exec lock still orders + /// mapping changes. The prepared update cannot fail after visibility. + pub(super) fn commit_context_binding( + &self, + gem: &ARef, + update: &PreparedBinding, + ) -> RetiredBindings { + let mut retired = RetiredBindings::new(); + if update.touched == 0 { + return retired; + } + let Some(context) = self.context_bindings.as_ref() else { + return retired; + }; let mut bindings = context.bindings.lock(); - let mut touched = false; - for (index, view) in CONTEXT_VIEWS.iter().enumerate() { - if address < view.source + view.size as u64 && view.source < end { - bindings.slots[index] = None; - touched = true; + for (index, slot) in bindings.slots.iter_mut().enumerate() { + if update.touched & (1 << index) != 0 { + retired.0[index] = slot.take(); } - if let Some(offset) = offsets[index] { - bindings.slots[index] = Some(Binding { + if update.covered & (1 << index) != 0 { + // Coverage proves source >= address. Preparation checked this + // exact sum and its usize conversion against the same view. + let offset = (update.object_offset + + (CONTEXT_VIEWS[index].source - update.address)) as usize; + *slot = Some(Binding { gem: gem.clone(), offset, }); - touched = true; } } - if touched { - bindings.generation = bindings.generation.wrapping_add(1); - } - Ok(()) + bindings.generation = bindings.generation.wrapping_add(1); + retired } - pub(crate) fn untrack_context_range(&self, range: Range) { + /// Successful user unmaps and accepted deferred unmaps invalidate the + /// matching candidates. Returning their owners avoids destruction under + /// the caller's exec or lifetime lock. + pub(super) fn untrack_context_ranges( + &self, + ranges: impl Iterator>, + ) -> RetiredBindings { + let mut retired = RetiredBindings::new(); let Some(context) = self.context_bindings.as_ref() else { - return; + return retired; }; let mut bindings = context.bindings.lock(); - let mut touched = false; - for (index, view) in CONTEXT_VIEWS.iter().enumerate() { - if range.start < view.source + view.size as u64 && view.source < range.end { - bindings.slots[index] = None; - touched = true; + for range in ranges { + let mut touched = false; + for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if range.start < view.source + view.size as u64 && view.source < range.end { + if let Some(binding) = bindings.slots[index].take() { + retired.0[index] = Some(binding); + } + touched = true; + } + } + if touched { + bindings.generation = bindings.generation.wrapping_add(1); } } - if touched { - bindings.generation = bindings.generation.wrapping_add(1); - } + retired } pub(crate) fn untrack_context_object(&self, gem: &gem::Object) { diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 3b8a5e5a40aeb..bff43fbb3795e 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -436,6 +436,9 @@ impl Vm { if !state.quarantined { return Err(EBUSY); } + // Reserve before accepting any range; a partial deferred batch must + // not leave canonical metadata ahead of the accepted cleanup. + state.unmaps.reserve(batch.ops.len(), GFP_KERNEL)?; for op in &batch.ops { if let PreparedUserBindOp::Unmap(unmap) = op { let range = (unmap.iova, unmap.size); @@ -444,6 +447,12 @@ impl Vm { } } } + let retired = self.untrack_context_ranges(batch.ops.iter().filter_map(|op| match op { + PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), + PreparedUserBindOp::Map(_) => None, + })); + drop(state); + drop(retired); Ok(true) } From 906cacc20dfcccb95c6ba7809f8d3ccac4624791 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 03:04:01 -0400 Subject: [PATCH 0610/1579] drm/asahi: order canonical object invalidation with mapping removal Remove canonical candidates under the existing GPUVM exec guard after successful object unmap. Keep deferred acceptance invalidation under the lifetime lock, with the queued GEM reference retaining the object. This prevents a trailing close callback from erasing a newer committed binding without adding locking or allocation. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 11b50b74597c2c3e1f5687d5db50ce0eaa103c21) --- drivers/gpu/drm/asahi/file.rs | 1 - drivers/gpu/drm/asahi/mmu.rs | 3 +++ drivers/gpu/drm/asahi/mmu/lifetime.rs | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index b5da3018d96bd..e626b77419e53 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -942,7 +942,6 @@ impl File { let vm = file_vm.borrow().vm.clone(); core::mem::drop(file_vm); vm.drop_mappings(bo)?; - vm.untrack_context_object(bo); if idx == usize::MAX { break; } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 3f05335407b73..4daa6c79d502a 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1971,6 +1971,9 @@ impl Vm { if let Some(bo) = self.inner.find_bo(gem) { mod_dev_dbg!(inner.dev, "MMU: bo_unmap\n"); self.inner.bo_unmap(&mut ctx, &bo)?; + // The close callback or deferred object list retains this GEM, so + // clearing its candidate cannot release the final reference here. + self.untrack_context_object(gem); self.untrack_shared_object(gem); mod_dev_dbg!(inner.dev, "MMU: bo_unmap done\n"); // We need to drop the exec_lock first, then the GpuVmBo since that will take the lock itself. diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index bff43fbb3795e..78f906ff9c46c 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -491,6 +491,10 @@ impl Vm { state.objects.push(gem.into(), GFP_KERNEL)?; } if state.active != 0 || state.draining { + // Stop new jobs from selecting this accepted cleanup candidate. + // The queued object retains its GEM while the candidate is removed. + // Actual unmap repeats this under exec to order any intervening bind. + self.untrack_context_object(gem); return Ok(()); } state.draining = true; From eb1d380fc62d3f8febbcf3a19d173eca44e49f19 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:22:22 -0400 Subject: [PATCH 0611/1579] drm/asahi: advertise G17 queue and input-sync capabilities Report the independently reserved logical queue capacity and per-command input synchronization already provided by G17. Userspace can then expose multiple queues and preserve independent command roots while forwarding input syncs to each submission. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f594e2be7f492d0cb4fdbdf5197122418f330036) --- drivers/gpu/drm/asahi/g17/mod.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index a62f1419d6122..1ccbb9b708ce0 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1058,6 +1058,8 @@ impl gpu::Gpu for Gpu { } fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + params.features |= + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; @@ -1087,6 +1089,14 @@ impl gpu::Gpu for Gpu { true } + fn queue_limits(&self) -> Option { + Some(uapi::drm_asahi_queue_limits { + max_queues: mmu::MAX_EXECUTION_CONTEXTS, + max_in_flight_per_queue: hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u32, + ..Default::default() + }) + } + fn uat_geometry(&self) -> mmu::UatGeometry { self.geometry } From 4810fcc3be491f1eaa0c78a183c800d08fecbc1d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:27:10 -0400 Subject: [PATCH 0612/1579] drm/asahi: acquire event records after observing their producer Order firmware event payload reads after the producer cursor establishes that the record is available. Move the existing acquire fence to this boundary, preserving the record-copy and consumer-store ordering without adding a barrier to nonempty drains. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 98783a0fc38dbac9eecb4164ed6c090e3adf591d) --- drivers/gpu/drm/asahi/g17/channel.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index c0609f148106e..a90693e824b0e 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -184,7 +184,6 @@ impl Events { while self.remaining != 0 { init.validate_event_ring(role)?; self.remaining -= 1; - fence(Ordering::Acquire); let consumer = consumer_word.load(Ordering::Relaxed); let producer = producer_word.load(Ordering::Relaxed); if consumer >= EVENT_SLOTS || producer >= EVENT_SLOTS { @@ -194,6 +193,8 @@ impl Events { self.remaining = 0; break; } + // Observe the producer before reading the record it publishes. + fence(Ordering::Acquire); let offset = status::EVENT_RING as usize + consumer as usize * EVENT_RECORD_SIZE; let record = EventRecord::from_words(init.status_read_words(role, offset)?); let event = record.decode(); From 7703d07ccd6dbb8c50cbd993a325e0d6a61cad33 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:27:47 -0400 Subject: [PATCH 0613/1579] drm/asahi: discard partial G17 mappings when binding fails A G17 map can fail allocating a later page table after earlier leaves have been written, before a GPUVA retains the backing. Clear the full failed operation range and invalidate every context naming its root before returning the mapping error. Apply the same cleanup if the final GPUVA insertion fails. Use a nonallocating partial-map discard that tolerates absent leaves without normal unmap diagnostics. Preserve Handoff error behavior and avoid additional work on successful mappings. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 267dc9ba9ea8e068a3d9d0683a58b72a51291ece) --- drivers/gpu/drm/asahi/mmu.rs | 27 +++++++++++++++++++++++++-- drivers/gpu/drm/asahi/pgtable.rs | 11 +++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 4daa6c79d502a..36dbf1da2ad89 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -386,12 +386,15 @@ impl gpuvm::DriverGpuVm for VmInner { one_page ); - self.page_table.map_pages( + if let Err(error) = self.page_table.map_pages( iova..(iova + len as u64), addr as PhysicalAddr, ctx.prot, one_page, - )?; + ) { + self.rollback_failed_map(op.addr(), op.range()); + return Err(error); + } left -= len; iova += len as u64; @@ -449,6 +452,7 @@ impl gpuvm::DriverGpuVm for VmInner { op.range(), op.addr() ); + self.rollback_failed_map(op.addr(), op.range()); return Err(EINVAL); } self.tlbi_contexts_naming_root(op.addr(), op.range() as usize); @@ -570,6 +574,25 @@ impl gpuvm::DriverGpuVm for VmInner { } impl VmInner { + /// A failed map has no GPUVA owner to retain its backing or remove its leaves. + /// GPUVM has already split or removed overlaps throughout this operation's range. + fn rollback_failed_map(&mut self, addr: u64, size: u64) { + if self.uat_inner.firmware == UatFirmware::Handoff { + return; + } + // Include the current segment: map_pages can fail after writing some of its leaves. + // The checked GPUVM operation supplies an aligned range without overflow. + let result = self.page_table.discard_partial_map(addr..addr + size); + self.tlbi_contexts_naming_root(addr, size as usize); + if let Err(error) = result { + dev_err!( + self.dev.as_ref(), + "Could not discard failed mapping: {:?}\n", + error + ); + } + } + fn mapping_mutation(&mut self) -> MappingMutation { let state = self.mapping_epoch.clone(); if let Some(state) = state.as_ref() { diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index a6b5f13574b3f..35ac52f8fd2bf 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -685,6 +685,17 @@ impl UatPageTable { }) } + /// Discards an aligned map operation that failed before acquiring a GPUVA owner. + /// Missing tables and leaves are expected; existing table storage stays allocated. + pub(crate) fn discard_partial_map(&mut self, iova_range: Range) -> Result { + self.with_pages(iova_range, false, false, |_, ptes| { + for pte in ptes { + pte.store(0, Ordering::Relaxed); + } + Ok(()) + }) + } + #[cfg(CONFIG_DEV_COREDUMP)] pub(crate) fn dump_pages(&mut self, iova_range: Range) -> Result> { let mut pages = KVVec::new(); From bdfbf517e1fb478c202e1531a8a084dc7557d4c5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:34:59 -0400 Subject: [PATCH 0614/1579] drm/asahi: reuse the reserved output fence for submitted work Reserve one fence before submission validation and reuse it as the aggregate after preparation succeeds. Keep the same fence available for every pre-publication failure, including aggregate allocation failure, without allocating a second unused fence on successful submits. Once published, only the aggregate enqueue sentinel and command members may signal it. Leave output installation and command completion ordering unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 9f204371cd6118ecaa4635b76a763e6c4d53b728) --- drivers/gpu/drm/asahi/g17/fence.rs | 11 +++++++---- drivers/gpu/drm/asahi/g17/queue/mod.rs | 3 +-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 95a13dbdd54ff..32d25b3b608cb 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -48,10 +48,10 @@ pub(crate) struct Submission { } impl Submission { - pub(crate) fn new(contexts: &FenceContexts, context: Arc) -> Result> { + fn new(fence: UserFence, context: Arc) -> Result> { Ok(Arc::new( Self { - fence: independent(contexts)?, + fence, work_state: Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?, remaining: AtomicU32::new(1), error: AtomicI32::new(0), @@ -168,9 +168,11 @@ impl Outputs { } /// Publish the actual aggregate before enqueueing any command. - pub(crate) fn publish(&mut self, submission: Arc) { + pub(crate) fn publish(&mut self, context: Arc) -> Result> { + let submission = Submission::new(self.fallback.clone(), context)?; install(self.syncs.drain(self.input_count..), &submission.fence()); - self.submission = Some(submission); + self.submission = Some(submission.clone()); + Ok(submission) } /// The caller first records terminal VM errors and settles prior failed work. @@ -184,6 +186,7 @@ impl Drop for Outputs { fn drop(&mut self) { if let Some(submission) = self.submission.take() { submission.finish_enqueue(self.result); + return; } if let Err(error) = self.result { self.fallback.set_error(error); diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 5495f3895abd3..0bcff40a1bf0d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -286,8 +286,7 @@ impl crate::queue::Queue for Queue { } let (pending, guards) = self.prepare_submission(bytes)?; let mut guards = guards.into_iter(); - let aggregate = Submission::new(&self.fences, self.context.clone())?; - outputs.publish(aggregate.clone()); + let aggregate = outputs.publish(self.context.clone())?; for (command, order) in pending { let guard = guards.next().ok_or(EIO)?; self.enqueue( From b92470c0d40d9a1f6ef82abb0da109ba2a064471 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:34:59 -0400 Subject: [PATCH 0615/1579] drm/syncobj: skip wait hint lookup for ineligible waits Check timeout and availability-wait eligibility before reading the optional utilization hint under the file spinlock. Preserve expiry checking immediately before entering the wait scope and avoid the extra lock on polls and availability waits. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 7dce231ad73d98e102b4b9e0b362cbab21750854) --- drivers/gpu/drm/drm_syncobj.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c index 2ac67e51a5b15..d9a6445c63353 100644 --- a/drivers/gpu/drm/drm_syncobj.c +++ b/drivers/gpu/drm/drm_syncobj.c @@ -1170,13 +1170,14 @@ static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, } } - /* Recheck expiry after allocations, user copies and deadline callbacks. */ - util_min = drm_syncobj_wait_hint(file_private); /* Polls, availability waits and already-complete fences need no boost. */ - if (util_min && timeout > 0 && - !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { - scope_previous = sched_util_min_scope_enter(util_min); - scope_active = true; + if (timeout > 0 && !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { + /* Recheck expiry after allocations, user copies and deadline callbacks. */ + util_min = drm_syncobj_wait_hint(file_private); + if (util_min) { + scope_previous = sched_util_min_scope_enter(util_min); + scope_active = true; + } } do { From c97c4be0ce4e1a265fa915b9ec4e634a7db2c3af Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:29:06 -0400 Subject: [PATCH 0616/1579] drm/asahi: wake render growth when its prerequisites become ready A retained TVB request can wait for a borrowed render pair or control-ring publication. Requeuing the growth worker after every event pass makes it bounce between growth and event work while those prerequisites are unchanged. Schedule growth only when a request can run, including stale requests that need discarding. Wake parked requests after render preparation restores its pair, including failed and cancelled preparation. Preserve the independent growth worker and existing control and recovery wakeups. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit fa771fbd7629800795268b560e9e2ff1b0cb806c) --- drivers/gpu/drm/asahi/g17/runtime/events.rs | 4 +- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 13 ++++++ .../drm/asahi/g17/runtime/render/prepare.rs | 43 +++++++++++++------ 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 6776707fd944c..f249e3520e0dd 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -415,7 +415,9 @@ impl crate::g17::Shared { // records that arrived while it was consuming the same ring. self.queue_events(); } - if firmware.render_grow_pending() && !firmware.recovery.pending() { + let grow_ready = firmware.render_grow_ready(); + drop(state); + if grow_ready { self.queue_grow(); } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index a9636cb7ceda0..db5820cfb6663 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -72,6 +72,19 @@ impl crate::g17::Firmware { pub(in crate::g17) fn render_grow_pending(&self) -> bool { !self.queues.render.grow.requests.is_empty() } + /// Borrowed owners and control publication each have their own retry wake. + /// A stale request is runnable so the worker can discard it immediately. + pub(in crate::g17) fn render_grow_ready(&self) -> bool { + let render = &self.queues.render; + !render.grow.requests.is_empty() + && !self.recovery.pending() + && !self.render_control_backpressured() + && render.grow.active.is_none() + && render.grow.requests.iter().any(|request| { + request.halt_count != self.recovery.generation() + || !render.grow_owner_busy(*request) + }) + } pub(in crate::g17) fn render_control_backpressured(&self) -> bool { self.queues.render.grow.reply.is_some() || self.queues.render.pools.pending() } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 30ea174f611b1..1aa6cb67a90eb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -299,7 +299,7 @@ impl super::super::Backend { // returns, while the original installed kick owner stays live. drop(prepared); drop(storage); - { + let grow_ready = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if result.is_err() { @@ -315,8 +315,12 @@ impl super::super::Backend { entry.preparing = None; entry.pending_owner = None; entry.pending_owner_closed = false; - } + firmware.render_grow_ready() + }; self.shared.changed.notify_all(); + if grow_ready { + self.shared.queue_grow(); + } result } Build::Rebind { @@ -345,7 +349,7 @@ impl super::super::Backend { Ok(binding) => (Some(binding), None), Err(error) => (None, Some(error)), }; - let result = { + let (result, grow_ready) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; let result = (|| { @@ -398,7 +402,7 @@ impl super::super::Backend { entry.pending_owner_closed = false; entry.pair = Some(pair); entry.borrowed_dependencies = None; - result + (result, firmware.render_grow_ready()) }; drop(prepared); if result.is_err() { @@ -413,6 +417,9 @@ impl super::super::Backend { } } self.shared.changed.notify_all(); + if grow_ready { + self.shared.queue_grow(); + } result } }; @@ -582,16 +589,24 @@ impl super::super::Backend { } firmware.publish_render(self.owner, slot, packet, deferred) })(); - if published.is_err() && !packet.is_published() { - firmware - .queues - .render - .entry_mut(slot)? - .pair - .as_mut() - .ok_or(EIO)? - .discard_prepared(packet)?; + let result = (|| { + if published.is_err() && !packet.is_published() { + firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .discard_prepared(packet)?; + } + published + })(); + let grow_ready = firmware.render_grow_ready(); + drop(state); + if grow_ready { + self.shared.queue_grow(); } - published + result } } From 4fe42ca7a85341ec777ca2f19e5ed0eb8fdafa82 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:36:44 -0400 Subject: [PATCH 0617/1579] drm/asahi: skip frontier scans without a cross-engine prefix Select an empty borrowed frontier when a command has no opposite-engine prefix. The existing count and fill paths then do no predecessor work for that command, while explicit prefixes retain every covered fence in its original order. Keep both same-engine and compute-triggered render pruning unchanged, along with per-command input fences and early-publication eligibility. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 956a8e2fd2ddcb7e9dc596796b79f62b4e017c55) --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 0bcff40a1bf0d..9af2cd4b7643f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -182,14 +182,19 @@ impl Queue { } let other = 1 - index; let prefix = order.wait_through[other]; - let count = self.frontiers[other] + let frontier = if prefix.is_some() { + self.frontiers[other].as_slice() + } else { + &[] + }; + let count = frontier .iter() .filter(|(sequence, _)| Order::contains(prefix, *sequence)) .count(); let mut dependencies = KVec::with_capacity(inputs.len() + count, GFP_KERNEL)?; let mut checked_inputs = KVec::with_capacity(inputs.len(), GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; - for (sequence, fences) in &self.frontiers[other] { + for (sequence, fences) in frontier { if Order::contains(prefix, *sequence) { dependencies.push(fences.ready.clone(), GFP_KERNEL)?; firmware.push(fences.completed.clone(), GFP_KERNEL)?; From 75ea155f8b2ed4da38c39179a9a653fec0eb1c37 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:40:54 -0400 Subject: [PATCH 0618/1579] drm/asahi: track CPU feed requests through policy lifetime Register a device-owned policy notifier before a single existing-policy snapshot. Match creation with removal of the exact pinned frequency request and policy reference, so CPUFreq teardown does not wait for GPU removal. Revalidate snapshot policy identity under the request lock after policy initialization, and keep notifier unregistration outside that lock. Preserve feed thresholds, the frequency floor and worker cadence. Handle later policy creation without periodic CPU scans, and leave the detector inactive when CPUFreq is disabled. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5026a187d811c441374d11c71a3fa0df58e118fd) --- drivers/gpu/drm/asahi/g17/feed.rs | 331 ++++++++++++++++++++++-------- 1 file changed, 245 insertions(+), 86 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index ca3ecc85ee25a..4cd05fde16581 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -1,34 +1,41 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Detects sustained host refill gaps and temporarily requests CPU capacity. -//! Detector updates are lock-free. The device's single feed worker owns the -//! frequency requests; its lock never nests inside a driver publication lock. +//! Detector updates are lock-free. The worker and policy callbacks serialize +//! frequency requests without nesting their mutex inside a publication lock. //! Shutdown first closes the detector, joins its producers and the worker, //! then removes requests. All request storage and policy references are owned //! by this device, independently of other GPU instances. +//! Policy creation and the initial snapshot take the policy rwsem before the +//! requests mutex. Removal callbacks and the worker take only requests; +//! notifier unregistration joins callbacks without holding that mutex. use crate::hw::t8140::qos::CLOCK_HZ; use core::{ pin::Pin, - ptr, sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, }; use kernel::{ - bindings, new_mutex, prelude::*, - sync::{Arc, Mutex}, + sync::Arc, time::{delay::fsleep, ClockSource, Delta, Monotonic}, - types::Opaque, }; +#[cfg(CONFIG_CPU_FREQ)] +use core::ptr; +#[cfg(CONFIG_CPU_FREQ)] +use kernel::{bindings, new_mutex, sync::Mutex, types::Opaque}; + const MAX_REFILL_TICKS: u64 = CLOCK_HZ / 500; const STREAK_ON: u32 = 8; const STREAK_MAX: u32 = 64; const LINGER_NS: u64 = 8_000_000; const POLL_MS: i64 = 2; +#[cfg(CONFIG_CPU_FREQ)] const FLOOR_PERCENT: u64 = 50; const WAIT_UTIL_MIN: u32 = 512; const WAIT_HINT_MS: u32 = 8; +#[cfg(CONFIG_CPU_FREQ)] const MAX_POLICIES: usize = 8; struct Detector { @@ -114,6 +121,7 @@ impl Detector { } } +#[cfg(CONFIG_CPU_FREQ)] #[pin_data] struct Requests { #[pin] @@ -121,13 +129,14 @@ struct Requests { policies: [*mut bindings::cpufreq_policy; MAX_POLICIES], maxima: [u32; MAX_POLICIES], applied: [u32; MAX_POLICIES], - count: usize, } -// SAFETY: Each nonnull policy retains cpufreq_cpu_get's reference. Access to -// the pinned request array and metadata is exclusive under Feed::requests. +// SAFETY: Each nonnull policy owns a kobject reference. The device requests +// mutex serializes the pinned slots with the worker and policy callbacks. +#[cfg(CONFIG_CPU_FREQ)] unsafe impl Send for Requests {} +#[cfg(CONFIG_CPU_FREQ)] impl Requests { fn new() -> Self { Self { @@ -135,126 +144,255 @@ impl Requests { policies: [ptr::null_mut(); MAX_POLICIES], maxima: [0; MAX_POLICIES], applied: [0; MAX_POLICIES], - count: 0, } } - fn register(self: Pin<&mut Self>) { + /// The caller holds the policy's read or write lock and keeps it alive. + /// Policy removal must subsequently pass through this device's notifier. + unsafe fn add(self: Pin<&mut Self>, policy: *mut bindings::cpufreq_policy) { let this = self.project(); - if *this.count != 0 { + if this.policies.contains(&policy) { return; } - // SAFETY: nr_cpu_ids is initialized before driver probe. - let cpus = unsafe { bindings::nr_cpu_ids }; - for cpu in 0..cpus { - // SAFETY: Accepts any CPU number; a nonnull result owns one reference. - let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; - if policy.is_null() { + let Some(index) = this.policies.iter().position(|entry| entry.is_null()) else { + return; + }; + // SAFETY: The caller keeps the initialized policy alive. The vacant + // request slot is pinned and exclusively owned under the requests mutex. + let result = unsafe { + bindings::freq_qos_add_request( + ptr::addr_of_mut!((*policy).constraints), + this.requests + .get() + .cast::() + .add(index), + bindings::freq_qos_req_type_FREQ_QOS_MIN, + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as i32, + ) + }; + if result >= 0 { + // SAFETY: The caller's policy reference or CREATE callback keeps the + // kobject alive. Retain it until REMOVE or device shutdown removes + // this request; cpuinfo is protected by the caller's policy lock. + unsafe { + bindings::kobject_get(ptr::addr_of_mut!((*policy).kobj)); + this.maxima[index] = (*policy).cpuinfo.max_freq; + } + this.policies[index] = policy; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + } + } + + fn apply(self: Pin<&mut Self>, enabled: bool) -> bool { + let this = self.project(); + let mut applied = false; + for index in 0..MAX_POLICIES { + if this.policies[index].is_null() { continue; } - let count = *this.count; - if count < MAX_POLICIES && !this.policies[..count].contains(&policy) { - // SAFETY: The policy is referenced. This zeroed, unregistered - // slot is pinned and remains alive until remove_request. + let value = if enabled { + (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 + } else { + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 + }; + if this.applied[index] != value { + // SAFETY: This slot is registered, pinned and exclusively locked; + // the policy reference keeps its constraints alive through REMOVE. let result = unsafe { - bindings::freq_qos_add_request( - ptr::addr_of_mut!((*policy).constraints), + bindings::freq_qos_update_request( this.requests .get() .cast::() - .add(count), - bindings::freq_qos_req_type_FREQ_QOS_MIN, - bindings::FREQ_QOS_MIN_DEFAULT_VALUE as i32, + .add(index), + value as i32, ) }; if result >= 0 { - this.policies[count] = policy; - // SAFETY: cpuinfo is immutable while this policy is referenced. - this.maxima[count] = unsafe { (*policy).cpuinfo.max_freq }; - this.applied[count] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; - *this.count += 1; - continue; + this.applied[index] = value; } } - // SAFETY: Balances the reference acquired above for a duplicate, - // unavailable slot or failed registration. Successful slots retain it. - unsafe { bindings::cpufreq_cpu_put(policy) }; + applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; } + applied } - fn apply(mut self: Pin<&mut Self>, enabled: bool) -> bool { - if enabled { - self.as_mut().register(); - } + fn remove(self: Pin<&mut Self>, policy: *mut bindings::cpufreq_policy) { let this = self.project(); - for index in 0..*this.count { - let floor = if enabled { - (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 - } else { - 0 - }; - let value = if floor == 0 { - bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 - } else { - floor - }; - if this.applied[index] == value { - continue; - } - this.applied[index] = value; - // SAFETY: This slot is registered, pinned and exclusively locked; - // its referenced policy keeps the constraints alive. - unsafe { - bindings::freq_qos_update_request( - this.requests - .get() - .cast::() - .add(index), - value as i32, - ) - }; + let Some(index) = this.policies.iter().position(|entry| *entry == policy) else { + return; + }; + // SAFETY: This registered pinned slot is exclusively locked. The policy + // cannot finish removal until this reference is dropped. Keep all other + // request slots at their original addresses when releasing this slot. + unsafe { + bindings::freq_qos_remove_request( + this.requests + .get() + .cast::() + .add(index), + ); + bindings::cpufreq_cpu_put(policy); } - *this.count != 0 && enabled + this.policies[index] = ptr::null_mut(); + this.maxima[index] = 0; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; } - fn clear(self: Pin<&mut Self>) { - let this = self.project(); - for index in 0..*this.count { - // SAFETY: Registered pinned slot, exclusively locked. Remove the - // constraint link before releasing its policy reference. - unsafe { - bindings::freq_qos_remove_request( - this.requests - .get() - .cast::() - .add(index), - ); - bindings::cpufreq_cpu_put(this.policies[index]); + fn clear(mut self: Pin<&mut Self>) { + for index in 0..MAX_POLICIES { + let policy = self.policies[index]; + if !policy.is_null() { + self.as_mut().remove(policy); } - this.policies[index] = ptr::null_mut(); } - *this.count = 0; } } +/// Only the blocking notifier core accesses this block after registration. +#[cfg(CONFIG_CPU_FREQ)] +#[pin_data] +struct PolicyNotifier { + #[pin] + block: Opaque, +} + +// SAFETY: The pinned notifier is registered once before publishing Feed and +// unregistered at shutdown. The blocking notifier core serializes its links; +// callbacks use only Feed's atomic state and request mutex. +#[cfg(CONFIG_CPU_FREQ)] +unsafe impl Send for PolicyNotifier {} +// SAFETY: As above; no Rust access mutates the registered block. +#[cfg(CONFIG_CPU_FREQ)] +unsafe impl Sync for PolicyNotifier {} + #[pin_data(PinnedDrop)] pub(crate) struct Feed { detector: Detector, + #[cfg(CONFIG_CPU_FREQ)] #[pin] requests: Mutex, + #[cfg(CONFIG_CPU_FREQ)] + #[pin] + notifier: PolicyNotifier, + #[cfg(CONFIG_CPU_FREQ)] + registered: AtomicBool, applied: AtomicBool, } impl Feed { pub(crate) fn new() -> Result> { - Arc::pin_init( + let feed = Arc::pin_init( pin_init!(Self { detector: Detector::new(), + #[cfg(CONFIG_CPU_FREQ)] requests <- new_mutex!(Requests::new(), "G17 CPU feed"), + #[cfg(CONFIG_CPU_FREQ)] + notifier <- pin_init!(PolicyNotifier { + block: Opaque::new(bindings::notifier_block { + notifier_call: Some(Self::policy_event), + next: ptr::null_mut(), + priority: 0, + }), + }), + #[cfg(CONFIG_CPU_FREQ)] + registered: AtomicBool::new(false), applied: AtomicBool::new(false), }), GFP_KERNEL, - ) + )?; + #[cfg(CONFIG_CPU_FREQ)] + feed.register_policies(); + Ok(feed) + } + + /// Seed existing policies once; later creation/removal is event driven. + #[cfg(CONFIG_CPU_FREQ)] + fn register_policies(&self) { + // SAFETY: Feed is pinned in its Arc and has not been published. The + // notifier block lives until shutdown unregisters and joins callbacks. + let result = unsafe { + bindings::cpufreq_register_notifier( + self.notifier.block.get(), + bindings::CPUFREQ_POLICY_NOTIFIER, + ) + }; + if result < 0 { + // CPUFreq can be disabled at boot independently of this driver. + return; + } + self.registered.store(true, Ordering::Release); + // SAFETY: nr_cpu_ids is initialized before driver probe. + let cpus = unsafe { bindings::nr_cpu_ids }; + for cpu in 0..cpus { + // SAFETY: The bounded CPU index is valid; a nonnull result owns a + // reference even if removal begins before acquiring the policy lock. + let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; + if policy.is_null() { + continue; + } + // CREATE holds the policy write lock before taking requests. Use + // the same lock order and wait out partial policy initialization. + // SAFETY: The retained reference keeps the policy and rwsem alive. + unsafe { bindings::down_read(ptr::addr_of_mut!((*policy).rwsem)) }; + { + let mut requests = self.requests.lock(); + // Removal unpublishes the CPU pointer before its notifier takes + // requests. Rechecking under this mutex makes either the seed + // skip the policy or the subsequent REMOVE release its slot. + // SAFETY: The CPU index is valid; balance this extra reference below. + let current = unsafe { bindings::cpufreq_cpu_get(cpu) }; + // SAFETY: The policy read lock excludes unfinished CREATE. If + // this policy is still published, REMOVE cannot yet have removed + // the core max request without passing our held requests mutex. + let initialized = + current == policy && unsafe { !(*policy).max_freq_req.qos.is_null() }; + if initialized { + // SAFETY: The reference and read lock keep the initialized + // policy alive; registered REMOVE callbacks cover its lifetime. + unsafe { requests.as_mut().add(policy) }; + } + if !current.is_null() { + // SAFETY: Balances the second lookup, independently of its identity. + unsafe { bindings::cpufreq_cpu_put(current) }; + } + } + // SAFETY: Balance the read lock and first lookup after releasing requests. + unsafe { + bindings::up_read(ptr::addr_of_mut!((*policy).rwsem)); + bindings::cpufreq_cpu_put(policy); + } + } + } + + #[cfg(CONFIG_CPU_FREQ)] + unsafe extern "C" fn policy_event( + block: *mut bindings::notifier_block, + event: c_ulong, + data: *mut core::ffi::c_void, + ) -> core::ffi::c_int { + // SAFETY: This callback is installed only on Feed::notifier.block. Its + // owner remains pinned until unregister joins every callback. + let notifier = + unsafe { kernel::container_of!(Opaque::cast_from(block), PolicyNotifier, block) }; + // SAFETY: The notifier is the embedded, pinned field of this live Feed. + let feed = unsafe { &*kernel::container_of!(notifier, Feed, notifier) }; + let policy = data.cast::(); + let mut requests = feed.requests.lock(); + if event == bindings::CPUFREQ_REMOVE_POLICY as c_ulong { + requests.as_mut().remove(policy); + } else if event == bindings::CPUFREQ_CREATE_POLICY as c_ulong + && !feed.detector.stopping.load(Ordering::Acquire) + { + // SAFETY: CREATE supplies an initialized policy with its write lock + // held and guarantees REMOVE before the core releases its storage. + unsafe { requests.as_mut().add(policy) }; + } else { + return bindings::NOTIFY_DONE as _; + } + let enabled = feed.detector.wanted(Self::now()); + let applied = requests.as_mut().apply(enabled); + feed.applied.store(applied, Ordering::Relaxed); + bindings::NOTIFY_OK as _ } fn now() -> u64 { @@ -262,7 +400,9 @@ impl Feed { } pub(crate) fn note_render_pass(&self, start: u64, end: u64) { - self.detector.note(start, end, Self::now()); + if cfg!(CONFIG_CPU_FREQ) { + self.detector.note(start, end, Self::now()); + } } /// Called after a firmware event pass, before queuing the single feed work item. @@ -270,6 +410,7 @@ impl Feed { self.detector.pending.swap(false, Ordering::AcqRel) } + #[cfg(CONFIG_CPU_FREQ)] fn apply(&self, enabled: bool) { let mut requests = self.requests.lock(); let enabled = enabled && !self.detector.stopping.load(Ordering::Acquire); @@ -277,6 +418,9 @@ impl Feed { self.applied.store(applied, Ordering::Relaxed); } + #[cfg(not(CONFIG_CPU_FREQ))] + fn apply(&self, _enabled: bool) {} + /// Worker context; the linger deadline has no firmware event of its own. pub(crate) fn run(&self) { loop { @@ -317,7 +461,22 @@ impl Feed { /// Call after work items are joined. Removing requests withdraws their floor. pub(crate) fn shutdown(&self) { self.begin_shutdown(); - self.requests.lock().as_mut().clear(); + #[cfg(CONFIG_CPU_FREQ)] + { + // Unregister joins callbacks, which take requests: never hold that + // mutex here. Device shutdown has already joined the feed worker. + if self.registered.swap(false, Ordering::AcqRel) { + // SAFETY: The block was registered successfully and remains + // pinned and alive until this synchronous removal returns. + unsafe { + bindings::cpufreq_unregister_notifier( + self.notifier.block.get(), + bindings::CPUFREQ_POLICY_NOTIFIER, + ); + } + } + self.requests.lock().as_mut().clear(); + } self.applied.store(false, Ordering::Relaxed); } } From a23fed471fe1d97b7d75c738cb63d045ad434754 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 05:15:19 -0400 Subject: [PATCH 0619/1579] drm/asahi: bound dependency scans to the ordered command prefix Accepted queue sequences are nonzero and increasing, and frontier pruning preserves their order. Locate the end of an explicit cross-engine prefix with partition_point and clone that slice directly. Keep the empty slice for absent prefixes and leave completion pruning, input synchronization and early-publication admission unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c42fd8c45c3a20ed2ec94c40bf5fbcf782220e7c) --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 9af2cd4b7643f..65bfdf10ab257 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -183,22 +183,21 @@ impl Queue { let other = 1 - index; let prefix = order.wait_through[other]; let frontier = if prefix.is_some() { - self.frontiers[other].as_slice() + let frontier = self.frontiers[other].as_slice(); + // Accepted sequences increase from one, and retain preserves their + // order, so all members of a logical prefix form one slice. + let end = frontier.partition_point(|(sequence, _)| Order::contains(prefix, *sequence)); + &frontier[..end] } else { &[] }; - let count = frontier - .iter() - .filter(|(sequence, _)| Order::contains(prefix, *sequence)) - .count(); + let count = frontier.len(); let mut dependencies = KVec::with_capacity(inputs.len() + count, GFP_KERNEL)?; let mut checked_inputs = KVec::with_capacity(inputs.len(), GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; - for (sequence, fences) in frontier { - if Order::contains(prefix, *sequence) { - dependencies.push(fences.ready.clone(), GFP_KERNEL)?; - firmware.push(fences.completed.clone(), GFP_KERNEL)?; - } + for (_, fences) in frontier { + dependencies.push(fences.ready.clone(), GFP_KERNEL)?; + firmware.push(fences.completed.clone(), GFP_KERNEL)?; } // Each command owns every input fence; independent engines may publish in // either order and must each consume an input's error status before running. From b47f66634c46afbc19bea8f135fb77f5957d57ab Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 4 Oct 2026 02:13:13 -0400 Subject: [PATCH 0620/1579] rust: check generated layouts and provide corrected private bindgen Keep compile-time size, alignment and field-offset assertions in generated bindings so incompatible layouts fail during the ordinary bindings build. Provide a pinned private generator recipe that preserves alignment on defined zero-sized structs and actual anonymous record members. Include explicit-target C and Rust compiler checks and generator provenance without changing C storage, scheduler configuration or the system toolchain. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 1f7383d5a2811b99a9f2950581ec392837bfc469) --- Documentation/rust/bindgen-layout-checks.rst | 49 +++++++++++ Documentation/rust/index.rst | 1 + rust/Makefile | 5 +- tools/aurora-bindgen/.gitattributes | 1 + tools/aurora-bindgen/README.rst | 72 ++++++++++++++++ tools/aurora-bindgen/build.py | 22 +++++ .../patches/0001-empty-alignment.patch | 20 +++++ .../patches/0002-anonymous-tag-members.patch | 85 +++++++++++++++++++ tools/aurora-bindgen/sources.json | 19 +++++ tools/aurora-bindgen/test.py | 44 ++++++++++ tools/aurora-bindgen/tests/aligned-empty.h | 13 +++ tools/aurora-bindgen/tests/aligned-empty.hpp | 13 +++ .../aurora-bindgen/tests/anonymous-members.h | 9 ++ tools/aurora-bindgen/tests/tag-only.h | 4 + 14 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 Documentation/rust/bindgen-layout-checks.rst create mode 100644 tools/aurora-bindgen/.gitattributes create mode 100644 tools/aurora-bindgen/README.rst create mode 100644 tools/aurora-bindgen/build.py create mode 100644 tools/aurora-bindgen/patches/0001-empty-alignment.patch create mode 100644 tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch create mode 100644 tools/aurora-bindgen/sources.json create mode 100644 tools/aurora-bindgen/test.py create mode 100644 tools/aurora-bindgen/tests/aligned-empty.h create mode 100644 tools/aurora-bindgen/tests/aligned-empty.hpp create mode 100644 tools/aurora-bindgen/tests/anonymous-members.h create mode 100644 tools/aurora-bindgen/tests/tag-only.h diff --git a/Documentation/rust/bindgen-layout-checks.rst b/Documentation/rust/bindgen-layout-checks.rst new file mode 100644 index 0000000000000..8ad8c24526b67 --- /dev/null +++ b/Documentation/rust/bindgen-layout-checks.rst @@ -0,0 +1,49 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Checking generated binding layouts +================================== + +Kernel bindings retain bindgen's layout checks. With the configured Rust target, +these are unconditional constant assertions, checked while the bindings crate +is compiled. They compare Rust sizes, alignments and field offsets against the +values obtained from Clang, without executing kernel code. + +Do not disable these checks to make an incompatible binding generator compile. +A type can have the right size and still have incorrect member offsets. For +example, an empty GNU C struct may have size zero and alignment 64; losing that +alignment moves later fields in an enclosing struct without necessarily +changing the enclosing size. + +Known generator requirements +---------------------------- + +Unmodified bindgen 0.73.2 loses explicit alignment on defined zero-sized C +structs. It also omits the named anonymous record member accepted by Clang's +``-fms-extensions`` in a declaration such as:: + + struct payload { unsigned long words[20]; }; + union container { + struct payload; + unsigned long other; + }; + +The generator used for this tree must preserve these layouts. Use a corrected +bindgen build through the normal ``BINDGEN=/path/to/bindgen`` make variable. +``tools/aurora-bindgen/`` contains pinned patches, a private build recipe and +compile-only valid-input tests. It never installs a system-wide tool. The +associated build provenance must retain the generator source revision, +patches, binary digest, and valid-input C/Rust layout checks. A version string +alone does not identify a locally corrected tool. Do not add dummy kernel +storage, hardcode task offsets, or change scheduler configuration to conceal a +binding generator error. + +After a binding layout correction, rebuild the kernel Image and all matching +modules. The generated bindings are consumed by the Rust kernel crate and by +multiple drivers; rebuilding just one module can leave incompatible layouts in +the same kernel. Keep the old Image and its own modules together as a fallback. + +The minimum accepted bindgen version is 0.71.1. Its Rust 1.77 ``offset_of`` +feature enables unconditional const layout assertions; the configured Rust +target 1.85 selects that path rather than ``#[test]``-only functions. The pinned +0.73.2 tool has the same behavior. See the source links and explicit target +accounting in ``tools/aurora-bindgen/README.rst``. diff --git a/Documentation/rust/index.rst b/Documentation/rust/index.rst index b78ed0efa7846..c03031b241086 100644 --- a/Documentation/rust/index.rst +++ b/Documentation/rust/index.rst @@ -37,6 +37,7 @@ more details. coding-guidelines arch-support testing + bindgen-layout-checks You can also find learning materials for Rust in its section in :doc:`../process/kernel-docs`. diff --git a/rust/Makefile b/rust/Makefile index bec9726f256ce..2fee22bdb0e14 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -444,10 +444,13 @@ endif bindgen_c_flags_final = $(bindgen_c_flags_lto) -fno-builtin -D__BINDGEN__ # `--rust-target` points to our minimum supported Rust version. +# Keep bindgen layout assertions enabled: a lost alignment can change field +# offsets without changing the containing type size. See Documentation/rust/ +# bindgen-layout-checks.rst for generator requirements. quiet_cmd_bindgen = BINDGEN $@ cmd_bindgen = \ $(BINDGEN) $< $(bindgen_target_flags) --rust-target 1.85 \ - --use-core --with-derive-default --ctypes-prefix ffi --no-layout-tests \ + --use-core --with-derive-default --ctypes-prefix ffi \ --no-debug '.*' --enable-function-attribute-detection \ -o $@ -- $(bindgen_c_flags_final) -DMODULE \ $(bindgen_target_cflags) $(bindgen_target_extra) diff --git a/tools/aurora-bindgen/.gitattributes b/tools/aurora-bindgen/.gitattributes new file mode 100644 index 0000000000000..813c74962d8c7 --- /dev/null +++ b/tools/aurora-bindgen/.gitattributes @@ -0,0 +1 @@ +patches/*.patch whitespace=-blank-at-eol diff --git a/tools/aurora-bindgen/README.rst b/tools/aurora-bindgen/README.rst new file mode 100644 index 0000000000000..bd05375d92ddd --- /dev/null +++ b/tools/aurora-bindgen/README.rst @@ -0,0 +1,72 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Private bindgen with checked C/Rust layouts +========================================== + +This package builds bindgen 0.73.2 with two generic binding-generation repairs: +preserve explicit alignment on defined zero-sized structs, and retain actual +unnamed named-tag fields exposed by Clang's type-field visitor. The latter is +needed for the Microsoft anonymous-member extension used by kernel namespaces. +Neither repair changes C structure storage or kernel configuration. + +``sources.json`` pins the official source archive, Cargo.lock, both patches and +the patched source files. It also records the qualified private tool's digest; +that digest is evidence for that build, not a promise that a different host, +compiler or build path produces identical bytes. + +Build in a fresh private directory:: + + python3 tools/aurora-bindgen/build.py --output-dir /absolute/new/tool-build + +Python 3.12 or newer, Cargo/Rust, Clang/libclang and patch are required. Cargo's +locked dependency graph is used, with at most two build jobs. ``--archive`` can +supply an already downloaded source archive; its digest is still checked. The +script builds and runs compile-only valid-input checks, then prints the exact +``BINDGEN=/absolute/path/to/bindgen`` make argument. It installs nothing globally. +Use that argument for the complete kernel Image and matching modules build. + +Tests and target accounting +--------------------------- + +``test.py`` independently compiles the C/C++ fixtures and generated Rust. It +covers empty alignments 1/2/8/64/128, nesting, packed empty records, anonymous +struct/union members, field access and ordinary tag-only declarations. It +requires unconditional const layout assertions in the generated output. +Nothing is executed on a GPU, and the generated object files are not run. + +Example using an existing private tool and the installed unpatched baseline:: + + python3 tools/aurora-bindgen/test.py \ + --bindgen /absolute/fixed/bindgen --baseline-bindgen /usr/bin/bindgen \ + --output-dir /absolute/new/check-results + +The optional baseline must be an affected unpatched tool: alignment/member +fixtures must fail at compile time, while tag-only controls must pass unchanged. +Without that option only the repaired generator is checked. The Rust target is +always passed explicitly, defaulting to the Rust compiler's reported host; the +Clang target defaults to the same triple. ``--rust-target`` and ``--clang-target`` +allow explicit kernel/cross targets, with matching architecture required. +``--rust-sysroot`` and ``--rust-libdir`` support a separately prepared Rust core. +An unavailable Rust target/core emits a machine-readable SKIP and exit 77; +that target has not passed. Other compiler or layout errors fail normally. + +Repository guard and supported tool versions +-------------------------------------------- + +The kernel keeps bindgen layout checks enabled. Its minimum bindgen is 0.71.1, +and its fixed ``--rust-target 1.85`` enables the ``offset_of`` feature (available +from Rust 1.77). In both official 0.71.1 and the pinned 0.73.2 generator, that +feature selects unconditional const size/alignment/offset assertions rather +than ``#[test]`` functions. A normal bindings crate compile therefore checks +layouts without requiring a Rust unit-test configuration. This is source-backed +for the accepted minimum and pinned tool; future generator changes must retain +that capability. Never remove the assertions to accept an incompatible tool. + +Primary sources: + +* https://github.com/rust-lang/rust-bindgen/blob/v0.71.1/bindgen/features.rs +* https://github.com/rust-lang/rust-bindgen/blob/v0.71.1/bindgen/codegen/mod.rs +* https://github.com/rust-lang/rust-bindgen/tree/v0.73.2 + +The complete kernel build remains the configuration-specific layout gate; these +small generator fixtures do not replace it or establish runtime GPU stability. diff --git a/tools/aurora-bindgen/build.py b/tools/aurora-bindgen/build.py new file mode 100644 index 0000000000000..cf3afbc4a53a5 --- /dev/null +++ b/tools/aurora-bindgen/build.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Fetch pinned source, apply generic fixes, and build/test a private bindgen.""" +import argparse,hashlib,json,subprocess,tarfile,urllib.request,shutil,os,sys +from pathlib import Path +P=Path(__file__).resolve().parent;ap=argparse.ArgumentParser();ap.add_argument('--output-dir',required=True,type=Path);ap.add_argument('--archive',type=Path);ap.add_argument('--jobs',type=int,default=2);a=ap.parse_args();assert 1<=a.jobs<=2 +O=a.output_dir.resolve();assert not O.exists(),'refusing to overwrite an existing source or tool';O.mkdir(parents=True) +m=json.loads((P/'sources.json').read_text());sha=lambda p:hashlib.sha256(p.read_bytes()).hexdigest();archive=O/'source.tar.gz' +if a.archive:shutil.copyfile(a.archive,archive) +else: + with urllib.request.urlopen(m['source_url'],timeout=60) as response, archive.open('wb') as f:shutil.copyfileobj(response,f) +assert sha(archive)==m['source_sha256'],'source digest mismatch' +with tarfile.open(archive) as t:t.extractall(O,filter='data') +S=O/m['source_directory'] +for name,h in m['patches'].items(): + p=P/'patches'/name;assert sha(p)==h,name;subprocess.run(['patch','--batch','--fuzz=0','-p1','-i',str(p)],cwd=S,check=True) +assert sha(S/'Cargo.lock')==m['cargo_lock_sha256'] +for f,h in m['patched_files'].items():assert sha(S/f)==h,f +cmd=['cargo','build','--locked','--release','-p','bindgen-cli','-j'+str(a.jobs)] +with (O/'build.log').open('w') as log:subprocess.run(cmd,cwd=S,stdout=log,stderr=subprocess.STDOUT,check=True) +binary=S/'target/release/bindgen';subprocess.run([sys.executable,str(P/'test.py'),'--bindgen',str(binary),'--output-dir',str(O/'tests')],check=True) +r={'status':'BUILT_AND_COMPILE_TESTED','binary':str(binary),'binary_sha256':sha(binary),'source_manifest_sha256':sha(P/'sources.json'),'build_command':cmd,'rustc_version':subprocess.check_output(['rustc','-vV'],text=True),'cargo_version':subprocess.check_output(['cargo','-vV'],text=True),'tests_manifest_sha256':sha(O/'tests/result.json'),'installed_globally':False};(O/'result.json').write_text(json.dumps(r,indent=2)+'\n');print(json.dumps(r,indent=2));print('Use make BINDGEN='+str(binary)+' ...; keep full kernel layout assertions enabled.') diff --git a/tools/aurora-bindgen/patches/0001-empty-alignment.patch b/tools/aurora-bindgen/patches/0001-empty-alignment.patch new file mode 100644 index 0000000000000..f595b480c6b8d --- /dev/null +++ b/tools/aurora-bindgen/patches/0001-empty-alignment.patch @@ -0,0 +1,20 @@ +--- a/bindgen/codegen/mod.rs ++++ b/bindgen/codegen/mod.rs +@@ -2468,6 +2468,17 @@ + } + } + } ++ } else if !is_union && !forward_decl { ++ // A zero-sized C struct can still require explicit alignment. ++ // Its alignment also determines the offsets of subsequent fields ++ // when it is embedded in another type. Do not add storage here. ++ if let Some(layout) = layout { ++ if struct_layout.requires_explicit_align(layout) && ++ layout.align > 1 ++ { ++ explicit_align = Some(layout.align); ++ } ++ } + } else if is_union && !forward_decl { + if let Some(layout) = layout { + // TODO(emilio): It'd be nice to unify this with the struct path above somehow. diff --git a/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch b/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch new file mode 100644 index 0000000000000..395b537796a45 --- /dev/null +++ b/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch @@ -0,0 +1,85 @@ +--- a/bindgen/clang.rs ++++ b/bindgen/clang.rs +@@ -1204,6 +1204,29 @@ + } + + impl Type { ++ /// Get actual record fields, including implicit anonymous fields omitted ++ /// from `clang_visitChildren` for Microsoft anonymous tag members. ++ pub(crate) fn fields(&self) -> Vec { ++ extern "C" fn collect_field( ++ cur: CXCursor, ++ data: CXClientData, ++ ) -> CXVisitorResult { ++ let fields = unsafe { &mut *data.cast::>() }; ++ fields.push(Cursor { x: cur }); ++ CXVisit_Continue ++ } ++ ++ let mut fields = Vec::::new(); ++ unsafe { ++ clang_Type_visitFields( ++ self.x, ++ collect_field, ++ (&mut fields as *mut Vec).cast(), ++ ); ++ } ++ fields ++ } ++ + /// Get this type's kind. + pub(crate) fn kind(&self) -> CXTypeKind { + self.x.kind +--- a/bindgen/ir/comp.rs ++++ b/bindgen/ir/comp.rs +@@ -1340,6 +1340,16 @@ + _ => false, + }); + ++ // Microsoft anonymous named-tag members have real implicit FieldDecls ++ // in the type's field list, but only a non-definition record cursor in ++ // child traversal. Keep positive field evidence: a tag declaration by ++ // itself is not necessarily storage in the containing record. ++ let mut implicit_fields: Vec<_> = ty ++ .fields() ++ .into_iter() ++ .filter(|field| field.spelling().is_empty() && !field.is_bit_field()) ++ .collect(); ++ + let mut maybe_anonymous_struct_field = None; + cursor.visit(|cur| { + if cur.kind() != CXCursor_FieldDecl { +@@ -1463,6 +1473,33 @@ + // definition, it's a valid inner type. + // + // [1]: https://github.com/rust-lang/rust-bindgen/issues/482 ++ if matches!(cur.kind(), CXCursor_StructDecl | CXCursor_UnionDecl) && ++ !cur.is_definition() ++ { ++ if let Some(index) = implicit_fields.iter().position(|field| { ++ field.cur_type().canonical_type() == ++ cur.cur_type().canonical_type() ++ }) { ++ let field = implicit_fields.remove(index); ++ let field_type = Item::from_ty_or_ref( ++ field.cur_type(), ++ field, ++ Some(potential_id), ++ ctx, ++ ); ++ ci.fields.append_raw_field(RawField::new( ++ None, ++ field_type, ++ field.raw_comment(), ++ Annotations::new(&field), ++ None, ++ field.public_accessible(), ++ field.offset_of_field().ok(), ++ )); ++ return CXChildVisit_Continue; ++ } ++ } ++ + let is_inner_struct = + cur.semantic_parent() == cursor || cur.is_definition(); + if !is_inner_struct { diff --git a/tools/aurora-bindgen/sources.json b/tools/aurora-bindgen/sources.json new file mode 100644 index 0000000000000..ca75ddc018d28 --- /dev/null +++ b/tools/aurora-bindgen/sources.json @@ -0,0 +1,19 @@ +{ + "version": "0.73.2", + "source_url": "https://codeload.github.com/rust-lang/rust-bindgen/tar.gz/refs/tags/v0.73.2", + "source_sha256": "5fc3277dd334e4bbbd7bdb8cad5d4fb87a667b2e95a2bae350a8d5424c280629", + "source_directory": "rust-bindgen-0.73.2", + "cargo_lock_sha256": "85edb9aab2d5649f39b805dc52e3f27ebf8af4d94b049601aee1c55fcf450393", + "patches": { + "0001-empty-alignment.patch": "eb71efca5c60adc9db6837c933ff377e8aefad53655057406ff9f1464dd3147f", + "0002-anonymous-tag-members.patch": "9072ec2fd9d0f0160615cbfd0d9cfe16d23cd0ae90747a2aa984b8694bdee9ec" + }, + "patched_files": { + "bindgen/codegen/mod.rs": "4c100fa631bdd655b509e1797411e341f9a954207d64f8f2efc285201f5f14d9", + "bindgen/clang.rs": "bd65a80b028045dec2ff1ca9ee347301726b30f94a0ee743a3041d4f598e45e4", + "bindgen/ir/comp.rs": "7d53770db84376699421c8057fbecbfc00e8a6f62ca38919bd4c718e56e40438" + }, + "qualified_private_tool_sha256": "dc794acc7b0d7e3cbfa1533b4b59f2518cd0cc1bff1dbb7fd104229c23c0d146", + "qualified_host_toolchain": "rustc/cargo1.98.1, LLVM/libclang22.1.8, nativeAArch64; kernel Rust1.85.0", + "runtime_tests_performed": false +} diff --git a/tools/aurora-bindgen/test.py b/tools/aurora-bindgen/test.py new file mode 100644 index 0000000000000..91b6d18d7fd4a --- /dev/null +++ b/tools/aurora-bindgen/test.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Compile valid generated bindings for explicitly matching C/Rust targets.""" +import argparse,hashlib,json,re,subprocess,sys +from pathlib import Path +P=Path(__file__).resolve().parent +ap=argparse.ArgumentParser();ap.add_argument('--bindgen',required=True);ap.add_argument('--output-dir',required=True,type=Path);ap.add_argument('--rustc',default='rustc');ap.add_argument('--clang',default='clang');ap.add_argument('--rust-target');ap.add_argument('--clang-target');ap.add_argument('--rust-sysroot');ap.add_argument('--rust-libdir');ap.add_argument('--baseline-bindgen');a=ap.parse_args() +O=a.output_dir.resolve();assert not O.exists(),'use a fresh evidence directory';O.mkdir(parents=True) +def run(cmd):return subprocess.run(cmd,capture_output=True,text=True) +host=next(x.split(': ',1)[1] for x in subprocess.check_output([a.rustc,'-vV'],text=True).splitlines() if x.startswith('host: '));rt=a.rust_target or host;ct=a.clang_target or rt +arch=lambda t:{'arm64':'aarch64','amd64':'x86_64'}.get(t.split('-')[0],t.split('-')[0]) +assert arch(rt)==arch(ct),('C/Rust architecture mismatch',ct,rt) +rust=[a.rustc,'--edition=2021','--crate-type=rlib','--crate-name','layout_check','--emit=obj','-Cpanic=abort','--target='+rt] +if a.rust_sysroot:rust+=['--sysroot='+a.rust_sysroot] +if a.rust_libdir:rust+=['-L',str(Path(a.rust_libdir).resolve())] +probe=O/'target.rs';probe.write_text('#![no_std]\npub fn available() -> usize { core::mem::size_of::() }\n');r=run(rust+[str(probe),'-o',str(O/'target.o')]);(O/'target.log').write_text(r.stdout+r.stderr) +if r.returncode: + status='SKIP' if 'E0463' in r.stderr and 'core' in r.stderr else 'FAIL' + result={'status':status,'reason':'matching Rust target/core unavailable' if status=='SKIP' else 'target preflight compile failed','clang_target':ct,'rust_target':rt,'rust_argv':rust,'log':'target.log'};(O/'result.json').write_text(json.dumps(result,indent=2)+'\n');print(json.dumps(result));sys.exit(77 if status=='SKIP' else 1) +access=''' +unsafe fn check_fields(a: &mut outer, b: &mut wrapper, c: &mut union_member, d: &mut trailing_member) { + a.__bindgen_anon_1.__bindgen_anon_1.words[19] = 1; + b.__bindgen_anon_1.words[19] = 2; + c.__bindgen_anon_1.bytes[23] = 3; + d.__bindgen_anon_1.words[0] = 4; +} +''' +cases=[];equal={};toolset=[('fixed',a.bindgen)]+([('baseline',a.baseline_bindgen)] if a.baseline_bindgen else []) +for fixture,language,extra in [('aligned-empty.h','c',[]),('aligned-empty.hpp','c++',[]),('anonymous-members.h','c',['-fms-extensions']),('tag-only.h','c',[])]: + header=P/'tests'/fixture + # Also ask the C compiler to accept the ordinary fixture independently. + ccmd=[a.clang,'--target='+ct,'-x',language,*extra,'-Wno-microsoft-anon-tag','-c',str(header),'-o',str(O/(fixture+'.c.o'))];cr=run(ccmd);(O/(fixture+'.c.log')).write_text(cr.stdout+cr.stderr);assert cr.returncode==0,(fixture,cr.stderr) + for label,tool in toolset: + cmd=[tool,str(header),'--rust-target','1.85','--no-doc-comments','--use-core','--no-derive-debug','--ctypes-prefix','core::ffi','--','-target',ct,'-x',language,*extra] + g=run(cmd);assert g.returncode==0,(fixture,g.stderr) + assert 'const _: ()' in g.stdout and '#[test]' not in g.stdout,'generator did not emit unconditional const layout checks' + rs=O/(fixture+'.'+label+'.rs');rs.write_text('#![no_std]\n'+g.stdout+(access if fixture=='anonymous-members.h' and label=='fixed' else '')) + rcmd=rust+[str(rs),'-o',str(O/(fixture+'.'+label+'.o'))];r=run(rcmd);(O/(fixture+'.'+label+'.log')).write_text(r.stdout+r.stderr) + expected=label=='fixed' or fixture=='tag-only.h';assert (r.returncode==0)==expected,(fixture,label,r.stderr) + if not expected:assert 'E0080' in r.stderr,'baseline failure must be a const layout mismatch' + cases.append({'fixture':fixture,'tool':label,'clang_argv':ccmd,'generator_argv':cmd,'rust_argv':rcmd,'compile_exit':r.returncode,'expected':'layout compile success' if expected else 'const layout rejection','generated_sha256':hashlib.sha256(rs.read_bytes()).hexdigest()}) + if fixture=='tag-only.h':equal[label]=g.stdout +if a.baseline_bindgen:assert equal['baseline']==equal['fixed'],'ordinary tag-only binding changed' +result={'status':'PASS','clang_target':ct,'rust_target':rt,'rust_host':host,'explicit_rust_target':True,'cases':cases,'baseline_checked':bool(a.baseline_bindgen),'hardware_executed':False};(O/'result.json').write_text(json.dumps(result,indent=2)+'\n');print('PASS matching C/Rust targets:',ct,rt) diff --git a/tools/aurora-bindgen/tests/aligned-empty.h b/tools/aurora-bindgen/tests/aligned-empty.h new file mode 100644 index 0000000000000..9ac1f0546a623 --- /dev/null +++ b/tools/aurora-bindgen/tests/aligned-empty.h @@ -0,0 +1,13 @@ +struct incomplete; +struct empty1 {} __attribute__((aligned(1))); +struct outer1 { unsigned char prefix[12]; struct empty1 value; unsigned long after; }; +struct empty2 {} __attribute__((aligned(2))); +struct outer2 { unsigned char prefix[12]; struct empty2 value; unsigned long after; }; +struct empty8 {} __attribute__((aligned(8))); +struct outer8 { unsigned char prefix[12]; struct empty8 value; unsigned long after; }; +struct empty64 {} __attribute__((aligned(64))); +struct outer64 { unsigned char prefix[12]; struct empty64 value; unsigned long after; }; +struct empty128 {} __attribute__((aligned(128))); +struct outer128 { unsigned char prefix[12]; struct empty128 value; unsigned long after; }; +struct packed_empty {} __attribute__((packed,aligned(64))); +struct packed_container { char prefix; struct packed_empty value; unsigned long after; }; diff --git a/tools/aurora-bindgen/tests/aligned-empty.hpp b/tools/aurora-bindgen/tests/aligned-empty.hpp new file mode 100644 index 0000000000000..9ac1f0546a623 --- /dev/null +++ b/tools/aurora-bindgen/tests/aligned-empty.hpp @@ -0,0 +1,13 @@ +struct incomplete; +struct empty1 {} __attribute__((aligned(1))); +struct outer1 { unsigned char prefix[12]; struct empty1 value; unsigned long after; }; +struct empty2 {} __attribute__((aligned(2))); +struct outer2 { unsigned char prefix[12]; struct empty2 value; unsigned long after; }; +struct empty8 {} __attribute__((aligned(8))); +struct outer8 { unsigned char prefix[12]; struct empty8 value; unsigned long after; }; +struct empty64 {} __attribute__((aligned(64))); +struct outer64 { unsigned char prefix[12]; struct empty64 value; unsigned long after; }; +struct empty128 {} __attribute__((aligned(128))); +struct outer128 { unsigned char prefix[12]; struct empty128 value; unsigned long after; }; +struct packed_empty {} __attribute__((packed,aligned(64))); +struct packed_container { char prefix; struct packed_empty value; unsigned long after; }; diff --git a/tools/aurora-bindgen/tests/anonymous-members.h b/tools/aurora-bindgen/tests/anonymous-members.h new file mode 100644 index 0000000000000..56b129a6aab14 --- /dev/null +++ b/tools/aurora-bindgen/tests/anonymous-members.h @@ -0,0 +1,9 @@ +struct payload { unsigned long words[20]; }; +union tag_union { unsigned long word; unsigned char bytes[24]; }; +struct outer { unsigned before; union { struct payload; void *ptr; }; unsigned after; }; +struct wrapper { unsigned prefix; struct payload; unsigned suffix; }; +struct union_member { char prefix; union tag_union; unsigned suffix; }; +struct normal_named { char prefix; struct payload named; unsigned suffix; }; +struct anonymous_definition { char prefix; struct { unsigned long inner; }; unsigned suffix; }; +struct forward_only { struct never_defined *ptr; unsigned suffix; }; +struct trailing_member { unsigned prefix; struct payload; }; diff --git a/tools/aurora-bindgen/tests/tag-only.h b/tools/aurora-bindgen/tests/tag-only.h new file mode 100644 index 0000000000000..525bc24774be0 --- /dev/null +++ b/tools/aurora-bindgen/tests/tag-only.h @@ -0,0 +1,4 @@ +struct payload { unsigned long words[20]; }; +struct harmless_tag { unsigned before; struct payload; unsigned after; }; +struct ordinary_forward { struct forward_tag; void *ptr; unsigned after; }; +struct normal_named { unsigned before; struct payload named; unsigned after; }; From b42e20b3a8d3e03bdf184be067a0149ed6fbee79 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 4 Oct 2026 02:19:36 -0400 Subject: [PATCH 0621/1579] drm/asahi: wait for host timestamp writes in compute prefixes Explicit compute prefixes must cover timestamp copies performed by the host after GPU retirement. Add completion dependencies and retained status checks for timestamp publishers while preserving firmware ordering for ordinary compute work. Keep the earliest failed compute timestamp as a bounded error witness so cancellation or recovery-spared failures cannot become valid query results after pruning. Reuse ordered prefix lookup and retain only independent fences, without extending packet or mapping lifetimes. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e4d41aaa9c0cdd145d5d07fe0b3d0eb209bf0879) --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 48 +++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 65bfdf10ab257..676180319ec63 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -21,7 +21,7 @@ use crate::{ use core::ops::Range; use kernel::{ c_str, - dma_fence::FenceContexts, + dma_fence::{FenceContexts, RawDmaFence}, prelude::*, sync::{Arc, LockClassKey}, xarray, @@ -147,7 +147,24 @@ impl Queue { Validated::Compute { .. } => Engine::Compute, }; let index = engine.index(); - self.frontiers[index].retain(|(_, fences)| job::fence_status(&fences.completed) == 0); + // One earliest failed timestamp covers every prefix containing a later + // failure, without retaining the producer's packet or mappings. + let mut retained_timestamp_failure = false; + self.frontiers[index].retain(|(_, fences)| { + let status = job::fence_status(&fences.completed); + if status == 0 { + return true; + } + if engine == Engine::Compute + && status < 0 + && fences.ready.raw() == fences.completed.raw() + && !retained_timestamp_failure + { + retained_timestamp_failure = true; + return true; + } + false + }); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { self.ensure_compute()?; @@ -192,9 +209,32 @@ impl Queue { &[] }; let count = frontier.len(); - let mut dependencies = KVec::with_capacity(inputs.len() + count, GFP_KERNEL)?; - let mut checked_inputs = KVec::with_capacity(inputs.len(), GFP_KERNEL)?; + let timestamp_prefix = order.wait_through[index]; + let timestamps = if engine == Engine::Compute && timestamp_prefix.is_some() { + let frontier = self.frontiers[index].as_slice(); + let end = frontier + .partition_point(|(sequence, _)| Order::contains(timestamp_prefix, *sequence)); + &frontier[..end] + } else { + &[] + }; + let timestamp_count = timestamps + .iter() + .filter(|(_, fences)| fences.ready.raw() == fences.completed.raw()) + .count(); + let mut dependencies = + KVec::with_capacity(inputs.len() + count + timestamp_count, GFP_KERNEL)?; + let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; + // Same-engine parents order GPU execution, not the host's timestamp + // writes. Only timestamp publishers need these completion waits and + // status checks; ordinary compute keeps its existing queue ordering. + for (_, fences) in timestamps { + if fences.ready.raw() == fences.completed.raw() { + checked_inputs.push(fences.completed.clone(), GFP_KERNEL)?; + dependencies.push(fences.ready.clone(), GFP_KERNEL)?; + } + } for (_, fences) in frontier { dependencies.push(fences.ready.clone(), GFP_KERNEL)?; firmware.push(fences.completed.clone(), GFP_KERNEL)?; From 2101861ac61164d0cc6e52a807b661078a957a9a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 4 Oct 2026 02:20:09 -0400 Subject: [PATCH 0622/1579] tools: preserve bindgen recipe validation across Python modes Reject optimized Python before creating output so archive and layout assertions cannot be disabled. Exercise bitfield generation with unknown Rust lints denied, and document explicit compiler selection separately from the generator language target. Document the synchronous Clang visitor safety requirements and refresh the affected pinned patch and source digests. Retain the previously qualified binary digest as historical build evidence without claiming a rebuilt binary is identical. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit bc7394673cbed4ef22971a49926be6070a7312be) --- tools/aurora-bindgen/README.rst | 14 ++++++++++++++ tools/aurora-bindgen/build.py | 3 +++ .../patches/0002-anonymous-tag-members.patch | 7 ++++++- tools/aurora-bindgen/sources.json | 7 ++++--- tools/aurora-bindgen/test.py | 10 ++++++++-- tools/aurora-bindgen/tests/bitfields.h | 11 +++++++++++ 6 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 tools/aurora-bindgen/tests/bitfields.h diff --git a/tools/aurora-bindgen/README.rst b/tools/aurora-bindgen/README.rst index bd05375d92ddd..2c560aa13f96e 100644 --- a/tools/aurora-bindgen/README.rst +++ b/tools/aurora-bindgen/README.rst @@ -13,6 +13,8 @@ Neither repair changes C structure storage or kernel configuration. the patched source files. It also records the qualified private tool's digest; that digest is evidence for that build, not a promise that a different host, compiler or build path produces identical bytes. +The manifest identifies when that digest predates comment-only patch changes; +each rebuild must record its own binary digest. Build in a fresh private directory:: @@ -24,6 +26,8 @@ supply an already downloaded source archive; its digest is still checked. The script builds and runs compile-only valid-input checks, then prints the exact ``BINDGEN=/absolute/path/to/bindgen`` make argument. It installs nothing globally. Use that argument for the complete kernel Image and matching modules build. +Both scripts reject optimized Python (``-O``, ``-OO`` or ``PYTHONOPTIMIZE``) +before creating output, since their validation assertions must remain enabled. Tests and target accounting --------------------------- @@ -32,6 +36,9 @@ Tests and target accounting covers empty alignments 1/2/8/64/128, nesting, packed empty records, anonymous struct/union members, field access and ordinary tag-only declarations. It requires unconditional const layout assertions in the generated output. +An additional bitfield fixture rejects unknown Rust lints in the fixed tool's +output. This fixture checks compiler compatibility, not the layout repairs, +and is excluded from the optional unpatched-baseline comparison. Nothing is executed on a GPU, and the generated object files are not run. Example using an existing private tool and the installed unpatched baseline:: @@ -50,6 +57,13 @@ allow explicit kernel/cross targets, with matching architecture required. An unavailable Rust target/core emits a machine-readable SKIP and exit 77; that target has not passed. Other compiler or layout errors fail normally. +Select ``--rustc`` explicitly to check the compiler used for the kernel. The +generator's ``--rust-target 1.85`` selects emitted language features; it does +not select the Rust compiler used by the tests. A compatibility adapter may +remove an exact generated lint attribute unsupported by that compiler, while +retaining all layout assertions. Its underlying binary must be the corrected +generator, and its changes and both tool digests belong in the build record. + Repository guard and supported tool versions -------------------------------------------- diff --git a/tools/aurora-bindgen/build.py b/tools/aurora-bindgen/build.py index cf3afbc4a53a5..5895a84a7b5c4 100644 --- a/tools/aurora-bindgen/build.py +++ b/tools/aurora-bindgen/build.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only """Fetch pinned source, apply generic fixes, and build/test a private bindgen.""" +if not __debug__: + raise SystemExit("optimized Python is unsupported: validation assertions must remain enabled") + import argparse,hashlib,json,subprocess,tarfile,urllib.request,shutil,os,sys from pathlib import Path P=Path(__file__).resolve().parent;ap=argparse.ArgumentParser();ap.add_argument('--output-dir',required=True,type=Path);ap.add_argument('--archive',type=Path);ap.add_argument('--jobs',type=int,default=2);a=ap.parse_args();assert 1<=a.jobs<=2 diff --git a/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch b/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch index 395b537796a45..7881d88237235 100644 --- a/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch +++ b/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch @@ -1,6 +1,6 @@ --- a/bindgen/clang.rs +++ b/bindgen/clang.rs -@@ -1204,6 +1204,29 @@ +@@ -1204,6 +1204,34 @@ } impl Type { @@ -11,12 +11,17 @@ + cur: CXCursor, + data: CXClientData, + ) -> CXVisitorResult { ++ // SAFETY: clang_Type_visitFields invokes this callback synchronously ++ // with the live Vec pointer supplied below; callbacks do not overlap. + let fields = unsafe { &mut *data.cast::>() }; + fields.push(Cursor { x: cur }); + CXVisit_Continue + } + + let mut fields = Vec::::new(); ++ // SAFETY: self.x is the live Clang type owned by the parsing context. ++ // The visitor only borrows fields during this call, and Clang does not ++ // retain the callback data after returning. + unsafe { + clang_Type_visitFields( + self.x, diff --git a/tools/aurora-bindgen/sources.json b/tools/aurora-bindgen/sources.json index ca75ddc018d28..cf97c6753218e 100644 --- a/tools/aurora-bindgen/sources.json +++ b/tools/aurora-bindgen/sources.json @@ -6,14 +6,15 @@ "cargo_lock_sha256": "85edb9aab2d5649f39b805dc52e3f27ebf8af4d94b049601aee1c55fcf450393", "patches": { "0001-empty-alignment.patch": "eb71efca5c60adc9db6837c933ff377e8aefad53655057406ff9f1464dd3147f", - "0002-anonymous-tag-members.patch": "9072ec2fd9d0f0160615cbfd0d9cfe16d23cd0ae90747a2aa984b8694bdee9ec" + "0002-anonymous-tag-members.patch": "3c1f6a9fdf02b7cd6f11b1d0787eec7d8a14cf6f140f414cc3069eeb75524562" }, "patched_files": { "bindgen/codegen/mod.rs": "4c100fa631bdd655b509e1797411e341f9a954207d64f8f2efc285201f5f14d9", - "bindgen/clang.rs": "bd65a80b028045dec2ff1ca9ee347301726b30f94a0ee743a3041d4f598e45e4", + "bindgen/clang.rs": "4271adf1fd24f9a8e58e2a69c48f3de6574318461203b513d34308e38852dcd1", "bindgen/ir/comp.rs": "7d53770db84376699421c8057fbecbfc00e8a6f62ca38919bd4c718e56e40438" }, "qualified_private_tool_sha256": "dc794acc7b0d7e3cbfa1533b4b59f2518cd0cc1bff1dbb7fd104229c23c0d146", "qualified_host_toolchain": "rustc/cargo1.98.1, LLVM/libclang22.1.8, nativeAArch64; kernel Rust1.85.0", - "runtime_tests_performed": false + "runtime_tests_performed": false, + "qualified_private_tool_note": "Digest identifies the previously qualified binary before safety-comment-only patch changes; a rebuild from this manifest must record its own binary digest." } diff --git a/tools/aurora-bindgen/test.py b/tools/aurora-bindgen/test.py index 91b6d18d7fd4a..6d93f8b35b9d8 100644 --- a/tools/aurora-bindgen/test.py +++ b/tools/aurora-bindgen/test.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only """Compile valid generated bindings for explicitly matching C/Rust targets.""" +if not __debug__: + raise SystemExit("optimized Python is unsupported: validation assertions must remain enabled") + import argparse,hashlib,json,re,subprocess,sys from pathlib import Path P=Path(__file__).resolve().parent @@ -10,7 +13,7 @@ def run(cmd):return subprocess.run(cmd,capture_output=True,text=True) host=next(x.split(': ',1)[1] for x in subprocess.check_output([a.rustc,'-vV'],text=True).splitlines() if x.startswith('host: '));rt=a.rust_target or host;ct=a.clang_target or rt arch=lambda t:{'arm64':'aarch64','amd64':'x86_64'}.get(t.split('-')[0],t.split('-')[0]) assert arch(rt)==arch(ct),('C/Rust architecture mismatch',ct,rt) -rust=[a.rustc,'--edition=2021','--crate-type=rlib','--crate-name','layout_check','--emit=obj','-Cpanic=abort','--target='+rt] +rust=[a.rustc,'--edition=2021','--crate-type=rlib','--crate-name','layout_check','--deny=unknown-lints','--emit=obj','-Cpanic=abort','--target='+rt] if a.rust_sysroot:rust+=['--sysroot='+a.rust_sysroot] if a.rust_libdir:rust+=['-L',str(Path(a.rust_libdir).resolve())] probe=O/'target.rs';probe.write_text('#![no_std]\npub fn available() -> usize { core::mem::size_of::() }\n');r=run(rust+[str(probe),'-o',str(O/'target.o')]);(O/'target.log').write_text(r.stdout+r.stderr) @@ -26,11 +29,14 @@ def run(cmd):return subprocess.run(cmd,capture_output=True,text=True) } ''' cases=[];equal={};toolset=[('fixed',a.bindgen)]+([('baseline',a.baseline_bindgen)] if a.baseline_bindgen else []) -for fixture,language,extra in [('aligned-empty.h','c',[]),('aligned-empty.hpp','c++',[]),('anonymous-members.h','c',['-fms-extensions']),('tag-only.h','c',[])]: +for fixture,language,extra in [('aligned-empty.h','c',[]),('aligned-empty.hpp','c++',[]),('anonymous-members.h','c',['-fms-extensions']),('tag-only.h','c',[]),('bitfields.h','c',[])]: header=P/'tests'/fixture # Also ask the C compiler to accept the ordinary fixture independently. ccmd=[a.clang,'--target='+ct,'-x',language,*extra,'-Wno-microsoft-anon-tag','-c',str(header),'-o',str(O/(fixture+'.c.o'))];cr=run(ccmd);(O/(fixture+'.c.log')).write_text(cr.stdout+cr.stderr);assert cr.returncode==0,(fixture,cr.stderr) for label,tool in toolset: + # The baseline comparison covers layout repairs; this fixture checks the fixed tool + # against the selected compiler, including compatibility adapters when needed. + if fixture == 'bitfields.h' and label == 'baseline':continue cmd=[tool,str(header),'--rust-target','1.85','--no-doc-comments','--use-core','--no-derive-debug','--ctypes-prefix','core::ffi','--','-target',ct,'-x',language,*extra] g=run(cmd);assert g.returncode==0,(fixture,g.stderr) assert 'const _: ()' in g.stdout and '#[test]' not in g.stdout,'generator did not emit unconditional const layout checks' diff --git a/tools/aurora-bindgen/tests/bitfields.h b/tools/aurora-bindgen/tests/bitfields.h new file mode 100644 index 0000000000000..47a4332ccf4cf --- /dev/null +++ b/tools/aurora-bindgen/tests/bitfields.h @@ -0,0 +1,11 @@ +struct bitfields { + unsigned int enabled : 1; + unsigned int count : 7; + signed int delta : 8; + unsigned int reserved : 16; +}; +struct bitfield_container { + unsigned long before; + struct bitfields value; + unsigned long after; +}; From 55df4049246b6da9482b15411e173a11cc7f04e8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 01:53:22 -0400 Subject: [PATCH 0623/1579] drm/asahi: release successful submission work-state ownership Drop the aggregate bookkeeping reference once every command and the enqueue gate have succeeded, before signalling its fence. Scheduler packets can otherwise retain all work-state slots while their cleanup waits behind another submission on the same worker. Preserve failed submissions and independent runtime owners for retirement and replay. Reject work-state access after successful aggregate completion instead of creating a replacement allocation. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 81d324022d0f97346639b06a6abaa8b86d480ead) --- drivers/gpu/drm/asahi/g17/completion.rs | 2 +- drivers/gpu/drm/asahi/g17/fence.rs | 35 ++++++++++++++----- drivers/gpu/drm/asahi/g17/queue/compute.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 8 ++--- .../gpu/drm/asahi/g17/queue/render/parked.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime.rs | 2 +- .../drm/asahi/g17/runtime/render/submit.rs | 2 +- 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 1151a82084cee..d9c3896fe9368 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -155,7 +155,7 @@ impl Completion { pub(crate) fn status(&self) -> &Arc { &self.status } - pub(crate) fn work_state(&self) -> Arc { + pub(crate) fn work_state(&self) -> Result> { self.member.work_state() } pub(crate) fn has_timestamps(&self) -> bool { diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 32d25b3b608cb..1217d7e7b3f4e 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -11,9 +11,10 @@ use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; use kernel::{ bindings, c_str, dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + new_mutex, prelude::*, str::CStr, - sync::Arc, + sync::{Arc, Mutex}, }; pub(crate) struct CompletionFence; @@ -40,24 +41,29 @@ pub(crate) fn independent(contexts: &FenceContexts) -> Result, - work_state: Arc, + /// Successful retirement drops only this bookkeeping owner; runtime owners + /// and failed submissions keep their work-state allocation. + #[pin] + work_state: Mutex>>, remaining: AtomicU32, error: AtomicI32, } impl Submission { fn new(fence: UserFence, context: Arc) -> Result> { - Ok(Arc::new( - Self { + let work_state = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; + Arc::pin_init( + pin_init!(Self { fence, - work_state: Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?, + work_state <- new_mutex!(Some(work_state), "G17 submission work state"), remaining: AtomicU32::new(1), error: AtomicI32::new(0), - }, + }), GFP_KERNEL, - )?) + ) } pub(crate) fn fence(&self) -> Fence { @@ -86,6 +92,12 @@ impl Submission { let error = self.error.load(Ordering::Acquire); if error != 0 { self.fence.set_error(Error::from_errno(error)); + } else { + // Scheduler cleanup can wait behind submission on the same worker. + // Once every member and enqueue have succeeded, retained packets + // must not prevent new submissions from acquiring work-state nodes. + let retired = self.work_state.lock().take(); + drop(retired); } self.fence.signal(); } @@ -108,8 +120,13 @@ pub(crate) struct Member { } impl Member { - pub(crate) fn work_state(&self) -> Arc { - self.submission.work_state.clone() + pub(crate) fn work_state(&self) -> Result> { + self.submission + .work_state + .lock() + .as_ref() + .cloned() + .ok_or(ECANCELED) } pub(crate) fn complete(&self, result: Result) { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 832d5073ecc7b..2a30cd54cecd1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1218,7 +1218,7 @@ impl Queue { else { return Err(EINVAL); }; - let work = packet.completion.work_state(); + let work = packet.completion.work_state()?; let work_word = work.word(); if work_word == u32::MAX { return Err(EINVAL); diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 3a7852be1dab0..c3ac091333c27 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -162,7 +162,7 @@ impl Pair { let Validated::Render { pass, .. } = &first.command else { return Err(EINVAL); }; - let word = first.completion.work_state().word(); + let word = first.completion.work_state()?.word(); if word == u32::MAX { return Err(EINVAL); } @@ -271,7 +271,7 @@ impl Pair { } let ordinal = self.ordinal; ordinal.checked_add(1).ok_or(EOVERFLOW)?; - let work = packet.completion.work_state(); + let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); let layout = target::Layout::new(pass, self.clusters)?; @@ -417,7 +417,7 @@ impl Pair { let qids = self.qids(); let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); - let work = packet.completion.work_state(); + let work = packet.completion.work_state()?; let work_slot = work.render_slot(self.slot)?; self.memory.bind_work_slot(ordinal, qids, work_slot)?; let payload = work.channel_payload(self.slot, ordinal)?; @@ -1382,7 +1382,7 @@ impl Pair { buffer_id, pm_generation, self.ordinal, - first.completion.work_state().word(), + first.completion.work_state()?.word(), )?; (Some(memory), Some(persistent), Some(aliases)) }; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 35955810a5af1..3d67578e2aaf9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -120,7 +120,7 @@ impl ParkedPair { let Validated::Render { pass, .. } = &first.command else { return Err(EINVAL); }; - let word = first.completion.work_state().word(); + let word = first.completion.work_state()?.word(); if word == u32::MAX { return Err(EINVAL); } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 6fe67478a777f..8978693a789ea 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -763,7 +763,7 @@ impl Backend { return Err(EIO); } let lease = self.shared.preparations.enter(false)?; - packet.completion.work_state().acquire()?; + packet.completion.work_state()?.acquire()?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 86a00e7bd741a..0586dfdc0944f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -66,7 +66,7 @@ impl Backend { } let work_state = { let preparation = self.shared.preparations.enter(interruptible)?; - let result = packet.completion.work_state().acquire(); + let result = packet.completion.work_state().and_then(|state| state.acquire()); if preparation.is_current() { result } else { From 1660e75c7e3ccb4b906662db275d6c5ccaebd452 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 01:54:32 -0400 Subject: [PATCH 0624/1579] drm/asahi: preserve exact historical command prefixes Accept queue-local render and compute ordinals from extensible payloads. Validate them against frozen ioctl-entry counters before publication and merge them with relative barriers without including unrelated newer work. Keep the reserved compute layout slot before the new fields and require it to remain zero. Existing short payloads retain their zero-filled extension behavior. Advertise the protocol only for G17 queues. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e0f7410fd5b368d06ab17d802720f9d76e12880c) --- drivers/gpu/drm/asahi/g17/command.rs | 8 +++++ drivers/gpu/drm/asahi/g17/job.rs | 12 ++++++- drivers/gpu/drm/asahi/g17/mod.rs | 3 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 11 +++++- include/uapi/drm/asahi_drm.h | 46 ++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index fef478310c23b..47f9c0048e673 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -289,6 +289,13 @@ pub(crate) enum Validated { } impl Payload { + pub(crate) fn prior(&self) -> [u64; 2] { + match self { + Self::Render(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Compute(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], + } + } + pub(crate) fn validate( &self, window: &UscWindow, @@ -399,6 +406,7 @@ impl Payload { } Self::Compute(cmd, attachments) => { if cmd.flags != 0 + || cmd.scs_layout != 0 || cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index f16897efa00b8..c05b5119790fc 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -33,7 +33,7 @@ impl Engine { } /// Logical sequence numbers do not wrap with firmware timestamps. A zero barrier -/// includes preceding ioctls; the sentinel means independent of that engine. +/// includes preceding ioctls; the sentinel leaves only an explicit historical prefix. #[derive(Clone, Copy)] pub(crate) struct Order { pub(crate) sequence: u64, @@ -45,11 +45,16 @@ impl Order { base: [u64; 2], preceding: [u16; 2], barriers: [u16; 2], + prior: [u64; 2], engine: Engine, ) -> Result { let mut sequences = [0; 2]; let mut wait_through = [None; 2]; for index in 0..2 { + // Earlier commands in this ioctl are not historical commands. + if prior[index] > base[index] { + return Err(EINVAL); + } sequences[index] = base[index] .checked_add(u64::from(preceding[index])) .and_then(|value| value.checked_add(1)) @@ -64,6 +69,11 @@ impl Order { .ok_or(EOVERFLOW)?, ); } + if prior[index] != 0 { + wait_through[index] = Some( + wait_through[index].map_or(prior[index], |last| last.max(prior[index])), + ); + } } Ok(Self { sequence: sequences[engine.index()], diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 1ccbb9b708ce0..cba7b6931c04e 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1059,7 +1059,8 @@ impl gpu::Gpu for Gpu { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { params.features |= - uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64; + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 676180319ec63..441b64602ae23 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -282,6 +282,9 @@ impl Queue { ) -> Result<(KVec<(Validated, Order)>, KVec)> { let parse_guard = self.context.vm().retain_first_job()?; let mut parser = Parser::new(bytes); + // All historical ordinals use the same entry counters; publication + // starts only after the entire ioctl has been validated and pinned. + let base = self.sequence; let mut preceding = [0; 2]; let mut pending = KVec::with_capacity(1, GFP_KERNEL)?; while let Some(command) = parser.next()? { @@ -295,7 +298,13 @@ impl Queue { Validated::Render { .. } => Engine::Render, Validated::Compute { .. } => Engine::Compute, }; - let order = Order::new(self.sequence, preceding, command.barriers, engine)?; + let order = Order::new( + base, + preceding, + command.barriers, + command.payload.prior(), + engine, + )?; preceding[engine.index()] += 1; pending.push((payload, order), GFP_KERNEL)?; } diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 8dae3950b4a02..cba36bd5db82e 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -276,6 +276,13 @@ enum drm_asahi_feature { * Registration may still be refused. */ DRM_ASAHI_FEATURE_VM_STATUS_MIRROR = (1UL) << 4, + + /** + * @DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS: Render and compute payloads + * support exact queue-local historical prefixes in prior_vdm/prior_cdm. + * These fields supplement the header barriers. + */ + DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS = (1UL) << 5, }; /** @@ -1306,6 +1313,24 @@ struct drm_asahi_cmd_render { /** @bg_partial_rsrc_spec_hi: High half of @partial_bg.rsrc_spec */ __u32 bg_partial_rsrc_spec_hi; + + /** + * @prior_vdm: Historical render prefix, or zero for no added dependency. + * + * Requires DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS. Counts hardware render + * commands accepted on this queue since creation, starting at one. A + * nonzero value must not exceed the render count at this ioctl's entry; + * it waits for every render command through that ordinal. The effective + * dependency is the maximum of this prefix and the header's resolved + * vdm_barrier. Header NONE does not suppress this field. + */ + __u64 prior_vdm; + + /** + * @prior_cdm: Historical compute prefix; see @prior_vdm, using the + * queue's compute command count and the header's cdm_barrier instead. + */ + __u64 prior_cdm; }; /** @@ -1343,6 +1368,27 @@ struct drm_asahi_cmd_compute { /** @ts: Timestamps for the compute command */ struct drm_asahi_timestamps ts; + + /** @scs_layout: Reserved; must be zero. */ + __u64 scs_layout; + + /** + * @prior_vdm: Historical render prefix, or zero for no added dependency. + * + * Requires DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS. Counts hardware render + * commands accepted on this queue since creation, starting at one. A + * nonzero value must not exceed the render count at this ioctl's entry; + * it waits for every render command through that ordinal. The effective + * dependency is the maximum of this prefix and the header's resolved + * vdm_barrier. Header NONE does not suppress this field. + */ + __u64 prior_vdm; + + /** + * @prior_cdm: Historical compute prefix; see @prior_vdm, using the + * queue's compute command count and the header's cdm_barrier instead. + */ + __u64 prior_cdm; }; /** From 437f51c11728f3183b0c17af6264547e44a60c26 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 28 Sep 2026 16:36:16 +1000 Subject: [PATCH 0625/1579] soc: apple: synchronize the ASCWrap mailbox provider with j700 Carry the provider-owned T8140 CPU lifecycle, removal serialization and runtime-PM retry fixes into the cleanroom integration. Both provider files are byte-identical to j700 at 43bb4bbb36ec. This carries the mailbox portions of upstream commits: 10636bd526b981ea233d30d8ad6e7e09b2f9d733 2230686542a49f6d0ebbaf11a9e75fb0981abc20 41afc106aece4a6e319f3b20327d1842174aa0a8 Validation: matching Image/modules build; both GPU firmware processors initialize on local Neo with the rewritten consumer. Signed-off-by: Ryan Murray Assisted-by: LLM (cherry picked from commit 29db1d8ab517933c2e0114fb3c6bf9beda5417aa) --- drivers/soc/apple/mailbox.c | 394 ++++++++++++++++++++++++++++-- include/linux/soc/apple/mailbox.h | 57 ++++- 2 files changed, 430 insertions(+), 21 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 16631fc352f44..ba8d124e853b1 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -47,7 +47,33 @@ #define APPLE_ASC_MBOX_I2A_RECV0 0x830 #define APPLE_ASC_MBOX_I2A_RECV1 0x838 -/* T8140 AOP setup-port FIFO/control offsets. */ +/* ASCWrap v6 embeds the unchanged ASC mailbox block at this offset. */ +#define APPLE_ASCWRAP_V6_MBOX_OFFSET 0x8000 +#define APPLE_ASCWRAP_V6_WRAPPER_SIZE 0x88000 +#define APPLE_ASCWRAP_V6_IOP_VBAR_SIZE 0x8 +#define APPLE_ASCWRAP_V6_CPU_CONTROL 0x44 +#define APPLE_ASCWRAP_V6_CPU_RUN BIT(4) +#define APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR BIT(5) + +#define APPLE_T8140_GFX_WRAPPER_BASE 0x482600000ULL +#define APPLE_T8140_GFX_IOP_VBAR_BASE 0x482050000ULL +#define APPLE_T8140_GFX1_WRAPPER_BASE 0x482e00000ULL +#define APPLE_T8140_GFX1_IOP_VBAR_BASE 0x482850000ULL + +static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == + 0x8110); +static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_CONTROL == + 0x8114); +static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_SEND0 == + 0x8800); +static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == + 0x8830); + +/* + * The T8140 AOP "setup port" (ADT aop-exclave-mailbox): the second mailbox + * the J700 AOP firmware boots through. Layout measured on hardware by the + * native m1n1 microphone host (artifacts/j700-native-lpmic-20260906). + */ #define APPLE_T8140_AOP_SETUP_A2I_CONTROL 0x000 #define APPLE_T8140_AOP_SETUP_I2A_CONTROL 0x004 #define APPLE_T8140_AOP_SETUP_A2I_SEND0 0x180 @@ -89,6 +115,8 @@ #define APPLE_MBOX_TX_TIMEOUT 500 struct apple_mbox_hw { + unsigned int reg_offset; + bool is_ascwrap_v6; bool ap_initializes_mailboxes; unsigned int control_full; @@ -109,6 +137,140 @@ struct apple_mbox_hw { unsigned int irq_bit_send_empty; }; +int apple_mbox_ascwrap_v6_get_lifecycle( + struct apple_mbox *mbox, + struct apple_mbox_ascwrap_v6_lifecycle *lifecycle) +{ + if (!mbox || !lifecycle) + return -EINVAL; + if (READ_ONCE(mbox->removing)) + return -ENODEV; + if (!mbox->hw->is_ascwrap_v6) + return -EOPNOTSUPP; + + *lifecycle = mbox->ascwrap_v6; + return 0; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_get_lifecycle); + +int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox) +{ + if (!mbox || !mbox->hw->is_ascwrap_v6) + return -EOPNOTSUPP; + if (READ_ONCE(mbox->removing)) + return -ENODEV; + + /* Future provider revisions may report additional lifecycle work. */ + if (mbox->ascwrap_v6.missing) + return -EOPNOTSUPP; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle); + +static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) +{ + void __iomem *cpu_control; + u32 val; + + if (!mbox->ascwrap_v6_cpu_running) + return; + + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V6_CPU_CONTROL; + + /* AppleASCWrapV6::_runCPU(false): clear RUN, reread, clear bit 5. */ + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR, cpu_control); + + mbox->ascwrap_v6_cpu_running = false; + pm_runtime_mark_last_busy(mbox->dev); + pm_runtime_put_autosuspend(mbox->dev); +} + +/* + * A failed resume leaves the runtime PM error set, and runtime PM then fails + * every later resume with -EINVAL without retrying the power domain. The + * provider keeps no state across a failed resume, so mark it suspended again + * and let the next start retry the domain. + */ +static void apple_mbox_clear_resume_error(struct apple_mbox *mbox) +{ + if (mbox->dev->power.runtime_error) + pm_runtime_set_suspended(mbox->dev); +} + +int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) +{ + void __iomem *cpu_control; + u32 val; + int ret = 0; + + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + return -EOPNOTSUPP; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + ret = -ENODEV; + goto out_unlock; + } + if (mbox->ascwrap_v6.missing) { + ret = -EOPNOTSUPP; + goto out_unlock; + } + if (mbox->ascwrap_v6_cpu_running) { + ret = -EBUSY; + goto out_unlock; + } + + ret = pm_runtime_resume_and_get(mbox->dev); + if (ret) { + dev_err(mbox->dev, + "ASC CPU start: runtime resume failed: %d (active=%d status=%d disable_depth=%d runtime_error=%d)\n", + ret, mbox->active, mbox->dev->power.runtime_status, + mbox->dev->power.disable_depth, + mbox->dev->power.runtime_error); + apple_mbox_clear_resume_error(mbox); + goto out_unlock; + } + + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V6_CPU_CONTROL; + + /* AppleASCWrapV6::_runCPU(true): read, set RUN, write. */ + val = readl(cpu_control); + writel(val | APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + mbox->ascwrap_v6_cpu_running = true; + +out_unlock: + mutex_unlock(&mbox->lifecycle_lock); + return ret; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_start_cpu); + +int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) +{ + int ret = 0; + + if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + return -EOPNOTSUPP; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + ret = -ENODEV; + goto out_unlock; + } + + apple_mbox_ascwrap_v6_stop_cpu_locked(mbox); + +out_unlock: + mutex_unlock(&mbox->lifecycle_lock); + return ret; +} +EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_stop_cpu); + int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, bool atomic) { @@ -118,6 +280,10 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, long t; spin_lock_irqsave(&mbox->tx_lock, flags); + if (unlikely(READ_ONCE(mbox->removing))) { + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return -ENODEV; + } mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); while (mbox_ctrl & mbox->hw->control_full) { @@ -169,6 +335,10 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, } spin_lock_irqsave(&mbox->tx_lock, flags); + if (unlikely(READ_ONCE(mbox->removing))) { + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return -ENODEV; + } mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); } @@ -210,10 +380,7 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) int ret = 0; u32 mbox_ctrl; - /* - * A detached receiver (RTKit teardown in progress) leaves firmware - * traffic in the FIFO; it is drained again once a receiver is attached. - */ + /* A stopped receiver may still have firmware traffic in its FIFO. */ if (!mbox->rx) return 0; mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); @@ -250,7 +417,8 @@ static irqreturn_t apple_mbox_recv_irq(int irq, void *data) struct apple_mbox *mbox = data; spin_lock(&mbox->rx_lock); - apple_mbox_poll_locked(mbox); + if (likely(!READ_ONCE(mbox->removing))) + apple_mbox_poll_locked(mbox); spin_unlock(&mbox->rx_lock); return IRQ_HANDLED; @@ -262,7 +430,10 @@ int apple_mbox_poll(struct apple_mbox *mbox) int ret; spin_lock_irqsave(&mbox->rx_lock, flags); - ret = apple_mbox_poll_locked(mbox); + if (unlikely(READ_ONCE(mbox->removing))) + ret = -ENODEV; + else + ret = apple_mbox_poll_locked(mbox); spin_unlock_irqrestore(&mbox->rx_lock, flags); return ret; @@ -272,14 +443,26 @@ EXPORT_SYMBOL(apple_mbox_poll); int apple_mbox_start(struct apple_mbox *mbox) { u32 control; - int ret; + int ret = 0; + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + ret = -ENODEV; + goto out_unlock; + } if (mbox->active) - return 0; + goto out_unlock; ret = pm_runtime_resume_and_get(mbox->dev); - if (ret) - return ret; + if (ret) { + dev_err(mbox->dev, + "mailbox start: runtime resume failed: %d (active=%d status=%d disable_depth=%d runtime_error=%d)\n", + ret, mbox->active, mbox->dev->power.runtime_status, + mbox->dev->power.disable_depth, + mbox->dev->power.runtime_error); + apple_mbox_clear_resume_error(mbox); + goto out_unlock; + } if (mbox->hw->ap_initializes_mailboxes) { /* The primary T8140 outbox requires its enable bit before use. */ @@ -303,11 +486,14 @@ int apple_mbox_start(struct apple_mbox *mbox) enable_irq(mbox->irq_recv_not_empty); mbox->active = true; - return 0; + +out_unlock: + mutex_unlock(&mbox->lifecycle_lock); + return ret; } EXPORT_SYMBOL(apple_mbox_start); -void apple_mbox_stop(struct apple_mbox *mbox) +static void apple_mbox_stop_locked(struct apple_mbox *mbox) { if (!mbox->active) return; @@ -317,8 +503,80 @@ void apple_mbox_stop(struct apple_mbox *mbox) pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); } + +void apple_mbox_stop(struct apple_mbox *mbox) +{ + mutex_lock(&mbox->lifecycle_lock); + apple_mbox_stop_locked(mbox); + mutex_unlock(&mbox->lifecycle_lock); +} EXPORT_SYMBOL(apple_mbox_stop); +/* + * Stop the provider-owned CPU, then quiesce every transport edge before + * devres releases IRQs or MMIO mappings. Device links order consumer removal + * first; the callback clear and waiter wake also cover future consumers that + * violate that ownership rule. + */ +static void apple_mbox_quiesce(void *data) +{ + struct apple_mbox *mbox = data; + unsigned long flags; + bool had_rx; + bool was_active; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + mutex_unlock(&mbox->lifecycle_lock); + return; + } + apple_mbox_ascwrap_v6_stop_cpu_locked(mbox); + + /* Serialize the removal fence with every MMIO send critical section. */ + spin_lock_irqsave(&mbox->tx_lock, flags); + WRITE_ONCE(mbox->removing, true); + spin_unlock_irqrestore(&mbox->tx_lock, flags); + + /* + * Publish the inactive state under the lifecycle lock, then drop it + * before disable_irq() waits for a callback. A callback is allowed to + * finish a send or stop request without an rx_lock -> lifecycle_lock + * inversion. The removal fence prevents any matching restart. + */ + was_active = mbox->active; + mbox->active = false; + mutex_unlock(&mbox->lifecycle_lock); + if (was_active) { + disable_irq(mbox->irq_recv_not_empty); + pm_runtime_mark_last_busy(mbox->dev); + pm_runtime_put_autosuspend(mbox->dev); + } + + /* Serialize once-only masking with the IRQ and send-error paths. */ + spin_lock_irqsave(&mbox->tx_lock, flags); + if (mbox->tx_irq_unmasked) { + disable_irq_nosync(mbox->irq_send_empty); + mbox->tx_irq_unmasked = false; + } + spin_unlock_irqrestore(&mbox->tx_lock, flags); + /* The handler takes tx_lock, so drain it after releasing the lock. */ + synchronize_irq(mbox->irq_send_empty); + complete_all(&mbox->tx_empty); + synchronize_irq(mbox->irq_recv_not_empty); + + spin_lock_irqsave(&mbox->rx_lock, flags); + had_rx = mbox->rx; + mbox->rx = NULL; + mbox->cookie = NULL; + spin_unlock_irqrestore(&mbox->rx_lock, flags); + + if (WARN_ON_ONCE(had_rx)) + dev_warn(mbox->dev, + "consumer callback survived provider remove ordering\n"); + + pm_runtime_barrier(mbox->dev); +} + struct apple_mbox *apple_mbox_get(struct device *dev, int index) { struct of_phandle_args args; @@ -371,9 +629,10 @@ static int apple_mbox_probe(struct platform_device *pdev) { int ret; char *irqname; + const char *firmware_role; struct apple_mbox *mbox; struct device *dev = &pdev->dev; - struct resource *res; + struct resource *res, *iop_vbar_res; mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); if (!mbox) @@ -384,13 +643,75 @@ static int apple_mbox_probe(struct platform_device *pdev) if (!mbox->hw) return -EINVAL; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res || resource_size(res) < mbox->hw->i2a_recv1 + sizeof(u64)) - return dev_err_probe(dev, -EINVAL, "mailbox register window is too small\n"); + if (mbox->hw->is_ascwrap_v6) { + /* + * Validate the entire provider identity before mapping either + * resource. These are fixed T8140 CPU-physical windows and role + * assignments, not generic consumer-provided aliases. + */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "wrapper"); + iop_vbar_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "iop-vbar"); + if (!res || !iop_vbar_res || + resource_size(res) != APPLE_ASCWRAP_V6_WRAPPER_SIZE || + resource_size(iop_vbar_res) != APPLE_ASCWRAP_V6_IOP_VBAR_SIZE) + return dev_err_probe(dev, -EINVAL, + "invalid ASCWrap v6 provider resources\n"); + + ret = of_property_read_string(dev->of_node, "apple,firmware-role", + &firmware_role); + if (ret) + return dev_err_probe(dev, ret, + "missing ASCWrap v6 firmware role\n"); + + if (!strcmp(firmware_role, "GFX")) { + mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; + if (res->start != APPLE_T8140_GFX_WRAPPER_BASE || + iop_vbar_res->start != APPLE_T8140_GFX_IOP_VBAR_BASE) + return dev_err_probe(dev, -EINVAL, + "GFX ASCWrap v6 resources do not match T8140\n"); + } else if (!strcmp(firmware_role, "GFX1")) { + mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1; + if (res->start != APPLE_T8140_GFX1_WRAPPER_BASE || + iop_vbar_res->start != APPLE_T8140_GFX1_IOP_VBAR_BASE) + return dev_err_probe(dev, -EINVAL, + "GFX1 ASCWrap v6 resources do not match T8140\n"); + } else { + return dev_err_probe(dev, -EINVAL, + "invalid ASCWrap v6 firmware role\n"); + } - mbox->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(mbox->regs)) - return PTR_ERR(mbox->regs); + mbox->ascwrap_v6.wrapper_start = res->start; + mbox->ascwrap_v6.wrapper_size = resource_size(res); + mbox->ascwrap_v6.iop_vbar_start = iop_vbar_res->start; + mbox->ascwrap_v6.iop_vbar_size = resource_size(iop_vbar_res); + /* resetState() is software-only; the matching stop lives above. */ + mbox->ascwrap_v6.missing = 0; + + mbox->wrapper_regs = + devm_platform_ioremap_resource_byname(pdev, "wrapper"); + if (IS_ERR(mbox->wrapper_regs)) + return PTR_ERR(mbox->wrapper_regs); + + mbox->iop_vbar_regs = + devm_platform_ioremap_resource_byname(pdev, "iop-vbar"); + if (IS_ERR(mbox->iop_vbar_regs)) + return PTR_ERR(mbox->iop_vbar_regs); + + mbox->regs = (u8 __iomem *)mbox->wrapper_regs + + mbox->hw->reg_offset; + } else { + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res || resource_size(res) < mbox->hw->reg_offset + + mbox->hw->i2a_recv1 + sizeof(u64)) + return dev_err_probe(dev, -EINVAL, + "mailbox register window is too small\n"); + + mbox->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(mbox->regs)) + return PTR_ERR(mbox->regs); + mbox->regs = (u8 __iomem *)mbox->regs + mbox->hw->reg_offset; + } mbox->irq_recv_not_empty = platform_get_irq_byname(pdev, "recv-not-empty"); @@ -403,6 +724,7 @@ static int apple_mbox_probe(struct platform_device *pdev) spin_lock_init(&mbox->rx_lock); spin_lock_init(&mbox->tx_lock); + mutex_init(&mbox->lifecycle_lock); init_completion(&mbox->tx_empty); irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); @@ -426,6 +748,9 @@ static int apple_mbox_probe(struct platform_device *pdev) return ret; ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, apple_mbox_quiesce, mbox); if (ret) return ret; @@ -433,6 +758,15 @@ static int apple_mbox_probe(struct platform_device *pdev) return 0; } +static void apple_mbox_remove(struct platform_device *pdev) +{ + struct apple_mbox *mbox = platform_get_drvdata(pdev); + + /* Block new phandle lookups before quiescing the provider. */ + platform_set_drvdata(pdev, NULL); + apple_mbox_quiesce(mbox); +} + static const struct apple_mbox_hw apple_mbox_t8015_hw = { .control_full = APPLE_ASC_MBOX_CONTROL_FULL, .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, @@ -480,6 +814,24 @@ static const struct apple_mbox_hw apple_mbox_t8140_asc_hw = { .has_irq_controls = false, }; +static const struct apple_mbox_hw apple_mbox_ascwrap_v6_hw = { + .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, + .is_ascwrap_v6 = true, + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + + .has_irq_controls = false, +}; + static const struct apple_mbox_hw apple_mbox_t8140_aop_setup_hw = { .control_full = APPLE_ASC_MBOX_CONTROL_FULL, .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, @@ -515,6 +867,7 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { }; static const struct of_device_id apple_mbox_of_match[] = { + { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, { .compatible = "apple,t8140-aop-setup-mailbox", .data = &apple_mbox_t8140_aop_setup_hw }, { .compatible = "apple,asc-mailbox-v4", .data = &apple_mbox_asc_hw }, @@ -530,6 +883,7 @@ static struct platform_driver apple_mbox_driver = { .of_match_table = apple_mbox_of_match, }, .probe = apple_mbox_probe, + .remove = apple_mbox_remove, }; module_platform_driver(apple_mbox_driver); diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index 9cfbbac411b8e..0d6ad8b16ae4c 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -9,6 +9,7 @@ #define _APPLE_MAILBOX_H_ #include +#include #include /* encodes a single 96bit message sent over the single channel */ @@ -17,11 +18,54 @@ struct apple_mbox_msg { u32 msg1; }; +/** + * enum apple_mbox_ascwrap_v6_role - firmware role owned by an ASCWrap v6 + * mailbox provider + * @APPLE_MBOX_ASCWRAP_V6_ROLE_NONE: not an ASCWrap v6 provider + * @APPLE_MBOX_ASCWRAP_V6_ROLE_GFX: primary GPU firmware role + * @APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1: secondary GPU firmware role + */ +enum apple_mbox_ascwrap_v6_role { + APPLE_MBOX_ASCWRAP_V6_ROLE_NONE, + APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, +}; + +/** + * struct apple_mbox_ascwrap_v6_lifecycle - provider-owned lifecycle facts + * @role: firmware role owned by this provider + * @wrapper_start: CPU physical base of the provider's ASCWrap window + * @wrapper_size: size of the provider's ASCWrap window + * @iop_vbar_start: CPU physical base of the provider's IOP VBAR window + * @iop_vbar_size: size of the provider's IOP VBAR window + * @missing: remaining provider lifecycle work; zero for T8140 + * + * The apple-mailbox provider owns both mappings. Consumers request CPU + * transitions through the provider and never map either resource themselves. + * T8140 has a complete matching start and stop transform. AppleA7IOP's + * resetState() only updates software state and performs no reset MMIO. + */ +struct apple_mbox_ascwrap_v6_lifecycle { + enum apple_mbox_ascwrap_v6_role role; + u64 wrapper_start; + u64 wrapper_size; + u64 iop_vbar_start; + u64 iop_vbar_size; + u32 missing; +}; + struct apple_mbox { struct device *dev; void __iomem *regs; + void __iomem *wrapper_regs; + void __iomem *iop_vbar_regs; const struct apple_mbox_hw *hw; + struct apple_mbox_ascwrap_v6_lifecycle ascwrap_v6; + /* Serializes transport start/stop against provider removal. */ + struct mutex lifecycle_lock; bool active; + bool ascwrap_v6_cpu_running; + bool removing; int irq_recv_not_empty; int irq_send_empty; @@ -35,7 +79,11 @@ struct apple_mbox { void (*rx)(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie); void *cookie; - /* Protected by tx_lock; send-error and IRQ paths mask this only once. */ + /* + * Protected by tx_lock. IRQ, send-error and provider-removal paths + * mask an outstanding unmask only once; rebuild every consumer when + * changing this internal structure's layout. + */ bool tx_irq_unmasked; }; @@ -48,4 +96,11 @@ int apple_mbox_poll(struct apple_mbox *mbox); int apple_mbox_send(struct apple_mbox *mbox, struct apple_mbox_msg msg, bool atomic); +int apple_mbox_ascwrap_v6_get_lifecycle( + struct apple_mbox *mbox, + struct apple_mbox_ascwrap_v6_lifecycle *lifecycle); +int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox); +int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox); +int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox); + #endif From c7986b27e066b3d3b4fd81e95b6e174cd6c21807 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 05:17:22 -0400 Subject: [PATCH 0626/1579] drm/asahi: bring the cleanroom driver up on Neo hardware Acquire both GPU ASC lifecycle handles through the mailbox provider rather than looking for a nonexistent secondary platform resource. Initialize the firmware registry and queue/buffer arrays in place to bound the probe stack. Keep firmware and client metrics addresses in named fields. Their reversed order caused the first render to fault in firmware at the client address; the corrected addresses pass both light and heavy rendering on hardware. Report failed spared work to userspace independently of permanent VM admission failure, and skip already successful internal prefix fences. Validation on local Neo with unchanged Mesa: desktop at 2408x1506, four exact-image game replays, native/4K userspace checks, render/compute depth 1/2/4/8/16, four concurrent clients, and 45 matched benchmark windows. Static probe chain drops from 24784 to 13584 bytes (16384-byte audit limit). Latest named-address form also passes the 15-window hardware matrix. This is hardware bring-up, not a full-parity or leak-free qualification. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit fbfb5d96f25efbf2b1571ca635b195e0ad4a54be) --- drivers/gpu/drm/asahi/file.rs | 2 +- drivers/gpu/drm/asahi/g17/buffer.rs | 32 ++++---- drivers/gpu/drm/asahi/g17/completion.rs | 8 +- drivers/gpu/drm/asahi/g17/fw/buffer.rs | 14 +++- drivers/gpu/drm/asahi/g17/kick.rs | 8 +- drivers/gpu/drm/asahi/g17/mod.rs | 20 +++-- drivers/gpu/drm/asahi/g17/queue/mod.rs | 6 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 32 +++++--- .../drm/asahi/g17/runtime/render/prepare.rs | 10 +-- drivers/gpu/drm/asahi/g17/status.rs | 34 +++++++-- drivers/gpu/drm/asahi/regs.rs | 73 +++++++++++++------ 11 files changed, 151 insertions(+), 88 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index e626b77419e53..a8d7a2db73e49 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -399,7 +399,7 @@ impl File { let status = Self::vm_status(file, vm_id)?; UserSlice::new(user, size) .writer() - .write(&[vm_id, 0, status.get() as u32, 0])?; + .write(&[vm_id, 0, status.reported_error() as u32, 0])?; Ok(0) } diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 942d5047e5b44..90ab8ae21aa14 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -34,21 +34,17 @@ pub(crate) struct BufferIds { next_token: u64, } impl BufferIds { - pub(crate) fn new() -> Self { - let mut returned = [0; BUFFER_IDS]; - for (index, entry) in returned.iter_mut().enumerate() { - *entry = (BUFFER_IDS - index - 1) as u8; - } - Self { - entries: [BufferId { + pub(crate) fn new() -> impl Init { + kernel::try_init!(Self { + entries <- pin_init::init_array_from_fn(|_| BufferId { references: 0, owner: false, token: 0, - }; BUFFER_IDS], - returned, + }), + returned <- pin_init::init_array_from_fn(|index| (BUFFER_IDS - index - 1) as u8), returned_count: BUFFER_IDS, next_token: 0, - } + }) } fn remove_returned(&mut self, id: u8) { if let Some(at) = self.returned[..self.returned_count] @@ -458,14 +454,14 @@ pub(crate) struct MetricsLease { _backing: Arc, allocator: Arc, slot: usize, - addresses: [u64; 2], + addresses: fw::MetricsAddresses, } impl MetricsLease { /// Clears only the selected retired metrics entry before reusing it. pub(crate) fn new( ids: &Arc, object: &Arc, - bases: [u64; 2], + bases: fw::MetricsAddresses, ) -> Result { let mut selected = None; for (bank, used) in ids.used.iter().enumerate() { @@ -499,13 +495,13 @@ impl MetricsLease { _backing: object.clone(), allocator: ids.clone(), slot, - addresses: [0; 2], + addresses: fw::MetricsAddresses::default(), }; let offset = slot * fw::METRICS_STRIDE; - lease.addresses = [ - bases[0].checked_add(offset as u64).ok_or(EOVERFLOW)?, - bases[1].checked_add(offset as u64).ok_or(EOVERFLOW)?, - ]; + lease.addresses = fw::MetricsAddresses { + firmware: bases.firmware.checked_add(offset as u64).ok_or(EOVERFLOW)?, + client: bases.client.checked_add(offset as u64).ok_or(EOVERFLOW)?, + }; let words = object.pointer(offset, fw::METRICS_STRIDE)?.cast::(); object.word(offset)?; for index in 0..fw::SCENES { @@ -631,7 +627,7 @@ impl Manager { Ok(self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (index * size_of::()) as u64) } pub(crate) fn scene_registers(&self, index: usize) -> Result<(u64, u64)> { - fw::scene_registers(index, self.scratch, self.metrics.addresses[1]) + fw::scene_registers(index, self.scratch, self.metrics.addresses.client) } pub(crate) fn prepare_scene(&mut self, ordinal: u64) -> Result { let scene = self.scenes.select(ordinal)?; diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index d9c3896fe9368..777d54222c0ea 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -324,7 +324,9 @@ impl Completion { Ok(()) }); if let Err(error) = result { - if error != ECANCELED && !self.spared() { + if self.spared() { + self.status.report_failure(error); + } else if error != ECANCELED { self.status.record(error); } self.fence.set_error(error); @@ -355,7 +357,9 @@ impl Completion { if let Some(guard) = self.vm_job.lock().as_ref() { guard.quarantine(); } - if error != ECANCELED && !self.spared() { + if self.spared() { + self.status.report_failure(error); + } else if error != ECANCELED { self.status.record(error); } self.fence.set_error(error); diff --git a/drivers/gpu/drm/asahi/g17/fw/buffer.rs b/drivers/gpu/drm/asahi/g17/fw/buffer.rs index 3ed336dbadaad..ec6797637bdbe 100644 --- a/drivers/gpu/drm/asahi/g17/fw/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/fw/buffer.rs @@ -120,11 +120,17 @@ static_assert!(core::mem::offset_of!(State, block_table) == 0x44); static_assert!(core::mem::offset_of!(State, discard) == 0x84); static_assert!(core::mem::offset_of!(Scene, generation) == 0x4c); +/// The same metrics storage in the firmware and application address spaces. +#[derive(Clone, Copy, Default)] +pub(crate) struct MetricsAddresses { + pub(crate) firmware: u64, + pub(crate) client: u64, +} + pub(crate) struct GraphArgs<'a> { pub(crate) fw_va: u64, pub(crate) page_list_client: u64, - /// Firmware and client views, respectively, of this graph's metrics lease. - pub(crate) metrics: [u64; 2], + pub(crate) metrics: MetricsAddresses, pub(crate) blocks: &'a [u64], pub(crate) scratch: u64, pub(crate) discard: u64, @@ -168,8 +174,8 @@ impl Graph { } } for (index, scene) in self.scenes.iter_mut().enumerate() { - scene.metric_client = offset_va(args.metrics[1], (index * 4) as u64)?; - scene.metric_fw = offset_va(args.metrics[0], (index * 4) as u64)?; + scene.metric_client = offset_va(args.metrics.client, (index * 4) as u64)?; + scene.metric_fw = offset_va(args.metrics.firmware, (index * 4) as u64)?; scene.scratch = scratch + (index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE; scene.statistics = stats; } diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 9d89262d8a3b6..f36ab9798996a 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -47,10 +47,10 @@ pub(crate) struct QueueIds { } impl QueueIds { - pub(crate) const fn new() -> Self { - Self { - entries: [None; QID_COUNT], - } + pub(crate) fn new() -> impl Init { + kernel::try_init!(Self { + entries <- pin_init::init_array_from_fn(|_| None), + }) } /// Reuses only an exact owner/engine match. Preferred placement is for diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index cba7b6931c04e..f379c1b69d256 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -355,12 +355,8 @@ impl Coprocessor { GFP_KERNEL, )?; - let name = match role { - Role::Primary => c_str!("asc"), - Role::Secondary => c_str!("asc1"), - }; Ok(Coprocessor { - cpu: regs::CpuControl::new(pdev, name)?, + cpu: regs::CpuControl::new(pdev, index)?, running: false, rtkit: Self::receiver(dev, index, &state)?, state, @@ -924,8 +920,6 @@ impl Gpu { // The firmware only samples the kernel lower root's top-level entries when it starts, so // everything it uses there must be mapped first. - let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary)?; - let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary)?; let pm_metrics = KernelObject::new( dev, uat.kernel_lower_vm(), @@ -934,28 +928,32 @@ impl Gpu { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; + let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) + .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; + let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) + .inspect_err(|e| dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e))?; let render_global = uat.new_vm(0, geometry.user_range())?; let bridge = Bridge::enable(regs)?; // Install all session ownership before either CPU starts. Error unwinding then runs // Firmware::drop, stopping both CPUs before releasing shared objects or translations. - let mut firmware = KBox::new( - Firmware { + let mut firmware = KBox::try_init( + kernel::try_init!(Firmware { secondary, primary, bridge, rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), - queues: runtime::Registry::new(shared.clusters, shared.descriptor_flags)?, + queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), pool: object::Pool::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, init: initdata::InitData::new(), _render_global: render_global, _pm_metrics: Arc::new(pm_metrics, GFP_KERNEL)?, uat, - }, + }), GFP_KERNEL, )?; let session = &mut *firmware; diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 441b64602ae23..9d2bc056cce65 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -236,6 +236,12 @@ impl Queue { } } for (_, fences) in frontier { + // Successful completion includes host timestamp publication. Its + // ready fence adds no ordering and would prevent shallow inline + // publication. Failed and pending producers still need checking. + if job::fence_status(&fences.completed) > 0 { + continue; + } dependencies.push(fences.ready.clone(), GFP_KERNEL)?; firmware.push(fences.completed.clone(), GFP_KERNEL)?; } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 8978693a789ea..d100fef631630 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -63,24 +63,32 @@ pub(super) struct Registry { } impl Registry { - pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { - let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; - for _ in 0..QIDS { - compute.push(None, GFP_KERNEL)?; - } - let mut qids = kick::QueueIds::new(); - qids.reserve(1, DataMaster::Tiling, Some(0))?; - qids.reserve(2, DataMaster::Fragment, Some(1))?; - Ok(Self { - qids, - buffers: BufferIds::new(), + /// Initialize inside the firmware allocation instead of staging the whole + /// registry on the probe stack. No additional allocation is required. + pub(super) fn new( + clusters: u32, + descriptor_flags: [u32; 2], + ) -> impl Init { + kernel::try_init!(Self { + qids <- kick::QueueIds::new().chain(|qids| { + qids.reserve(1, DataMaster::Tiling, Some(0))?; + qids.reserve(2, DataMaster::Fragment, Some(1))?; + Ok(()) + }), + buffers <- BufferIds::new(), accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, admission: admission::Pool::new()?, render: render::RenderState::new(clusters, descriptor_flags)?, identities: queue::Identities::new(), - compute, + compute: { + let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; + for _ in 0..QIDS { + compute.push(None, GFP_KERNEL)?; + } + compute + }, compute_building: false, compute_scheduler_building: false, observations: compute::retirement::Batch::new()?, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 1aa6cb67a90eb..e8fc30560a890 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -19,7 +19,7 @@ struct BuildResources { global: crate::mmu::Vm, metrics: Arc, metrics_ids: Arc, - metrics_bases: [u64; 2], + metrics_bases: crate::g17::fw::buffer::MetricsAddresses, config: Config, } impl BuildResources { @@ -31,10 +31,10 @@ impl BuildResources { global: firmware._render_global.clone(), metrics: firmware._pm_metrics.clone(), metrics_ids: firmware.queues.metrics.clone(), - metrics_bases: [ - crate::hw::t8140::CONFIG.pm_metrics.va, - firmware.init.metrics_fw_va()?, - ], + metrics_bases: crate::g17::fw::buffer::MetricsAddresses { + firmware: firmware.init.metrics_fw_va()?, + client: crate::hw::t8140::CONFIG.pm_metrics.va, + }, config: firmware.queues.render.config, }) } diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs index 178d275da9cf4..e5ac662a4d1aa 100644 --- a/drivers/gpu/drm/asahi/g17/status.rs +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Sticky VM errors and their optional userspace mirror. No firmware state is accessed here. +//! VM admission errors and client failure reporting. No firmware state is accessed here. use crate::gem; use core::cell::UnsafeCell; @@ -10,9 +10,11 @@ use kernel::prelude::*; const MIRROR_MAX_SIZE: usize = 64 * 1024; /// A VM and its accepted jobs share this status, including after queue destruction. -/// The first error is permanent. Callers record it before signalling the job's fence. +/// Client failure is separate from hardware admission: recovery may preserve the +/// VM while failed accepted work must still be reported before signalling fences. pub(crate) struct VmStatus { error: AtomicI32, + reported_error: AtomicI32, mirror: AtomicPtr, mirror_claimed: AtomicBool, mirror_owner: UnsafeCell>, @@ -30,6 +32,7 @@ impl VmStatus { pub(crate) fn new() -> Self { Self { error: AtomicI32::new(0), + reported_error: AtomicI32::new(0), mirror: AtomicPtr::new(core::ptr::null_mut()), mirror_claimed: AtomicBool::new(false), mirror_owner: UnsafeCell::new(None), @@ -40,9 +43,26 @@ impl VmStatus { self.error.load(Ordering::Acquire) } - /// Store the first error in both the status and any registered mirror. + /// Record permanent admission failure and report it to the client. pub(crate) fn record(&self, error: Error) { - let error = match self.error.compare_exchange( + self.report_failure(error); + let _ = self.error.compare_exchange( + 0, + error.to_errno(), + Ordering::SeqCst, + Ordering::SeqCst, + ); + } + + pub(crate) fn reported_error(&self) -> i32 { + self.reported_error.load(Ordering::Acquire) + } + + /// Report failed accepted work without preventing recovery of the VM. + /// Every reporter publishes the first error before returning: a losing + /// reporter may signal its fence before the winning reporter resumes. + pub(crate) fn report_failure(&self, error: Error) { + let error = match self.reported_error.compare_exchange( 0, error.to_errno(), Ordering::SeqCst, @@ -88,11 +108,11 @@ impl VmStatus { // SAFETY: The retained owner keeps this checked, aligned word mapped. unsafe { AtomicI32::from_ptr(word) }.store(0, Ordering::SeqCst); - // The pointer publication and error update form a store/load handshake: - // either this load sees record's error or record sees our pointer. Stores + // The pointer publication and reported error form a store/load handshake: + // either this load sees the error or the reporter sees our pointer. Stores // after publishing the pointer write only the same first error, never zero. self.mirror.store(word, Ordering::SeqCst); - let error = self.error.load(Ordering::SeqCst); + let error = self.reported_error.load(Ordering::SeqCst); if error != 0 { // SAFETY: The mapping remains owned by this status as above. unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index dacebd97fc482..2bfd754864358 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -11,17 +11,18 @@ //! coprocessors start, and enables dynamic power gating before the firmware is initialized. use crate::hw; +use core::mem::MaybeUninit; use kernel::{ - c_str, + bindings, c_str, device::Core, devres::Devres, + error::{from_err_ptr, to_result}, io::{ mem::IoMem, // Io, }, platform, prelude::*, - str::CStr, sync::{aref::ARef, Arc}, // }; @@ -54,9 +55,6 @@ const CORE_MASKS_G14X: usize = 0xe01500; const FAULT_INFO_G14X: usize = 0xd8c0; const FAULT_ADDR_G14X: usize = 0xd8c8; -/// Cleared after `CPU_RUN` when stopping a G17 coprocessor. -const CPU_STOP_ACK: u32 = 0x1 << 5; // BIT(5) - /// G17 GPU-to-fabric bridge control words. Bit 0 enables the bridge; the other bits are status. const G17_BRIDGE_CONTROL: [usize; 2] = [0x1000104, 0x1000108]; const G17_BRIDGE_ENABLE: u32 = 0x1; // BIT(0) @@ -197,35 +195,62 @@ pub(crate) struct G17Id { pub(crate) perf_map: [u32; 2], } -/// CPU control registers of one GPU coprocessor. +/// Provider-owned lifecycle of one G17 firmware coprocessor. +/// +/// The mailbox provider owns the wrapper mappings and runtime-PM references. +/// The consumer device link keeps it alive until this GPU has stopped using it. pub(crate) struct CpuControl { - asc: Pin>>>, + dev: ARef, + mbox: *mut bindings::apple_mbox, } +// SAFETY: The live provider serializes CPU transitions with its lifecycle mutex; +// the device link orders provider removal after this consumer's teardown. +unsafe impl Send for CpuControl {} +// SAFETY: All accesses use the same provider-serialized lifecycle API. +unsafe impl Sync for CpuControl {} + impl CpuControl { - /// Map the CPU control registers in the `name` register region. - pub(crate) fn new(pdev: &platform::Device, name: &CStr) -> Result { - let req = pdev.io_request_by_name(name).ok_or(EINVAL)?; - Ok(CpuControl { - asc: KBox::pin_init(req.iomap_sized::(), GFP_KERNEL)?, - }) + /// Resolve the mailbox phandle and validate its firmware role before use. + pub(crate) fn new(pdev: &platform::Device, index: usize) -> Result { + let expected = match index { + 0 => bindings::apple_mbox_ascwrap_v6_role_APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + 1 => bindings::apple_mbox_ascwrap_v6_role_APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, + _ => return Err(EINVAL), + }; + // SAFETY: pdev is live; the provider establishes the consumer device link. + let mbox = unsafe { + from_err_ptr(bindings::apple_mbox_get(pdev.as_ref().as_raw(), index.try_into()?))? + }; + let mut lifecycle = MaybeUninit::uninit(); + // SAFETY: mbox is live, and lifecycle is writable storage for the C result. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_get_lifecycle(mbox, lifecycle.as_mut_ptr()) + })?; + // SAFETY: A successful provider query initialized every field. + if unsafe { lifecycle.assume_init() }.role != expected { + return Err(EINVAL); + } + // SAFETY: This only checks the provider's matching start/stop capability. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(mbox) })?; + Ok(Self { dev: pdev.into(), mbox }) } - /// Start the coprocessor CPU. + /// Start through the owner of the CPU registers and runtime-PM reference. pub(crate) fn start(&self) -> Result { - let asc = self.asc.try_access().ok_or(ENODEV)?; - let val = asc.read32(CPU_CONTROL); - asc.write32(val | CPU_RUN, CPU_CONTROL); - Ok(()) + // SAFETY: The constructor established the live, role-checked provider. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_start_cpu(self.mbox) }) } - /// Stop the coprocessor CPU. This only gates it: its firmware keeps its state. + /// Stop before releasing the transport or any firmware-visible memory. pub(crate) fn stop(&self) { - if let Some(asc) = self.asc.try_access() { - let val = asc.read32(CPU_CONTROL); - asc.write32(val & !CPU_RUN, CPU_CONTROL); - let val = asc.read32(CPU_CONTROL); - asc.write32(val & !CPU_STOP_ACK, CPU_CONTROL); + // SAFETY: The provider remains live during consumer teardown. Removal + // stops the CPU before setting the fence that can return ENODEV here. + let result = to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }); + if let Err(error) = result { + if error != ENODEV { + dev_err!(self.dev.as_ref(), "Could not stop GPU coprocessor: {:?}\n", error); + } } } } From 01d0b0bed7a763fa775654a0a115c01f38b34584 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 6 Oct 2026 00:26:31 +1100 Subject: [PATCH 0627/1579] drm/asahi: report live GEM object accounting Nothing in the driver says how much memory its GEM objects hold. fdinfo has no drm-* keys, and debugfs only lists flink names. A G17P GEM object is pinned once it is mapped into a GPU VM and stays pinned until its last reference drops, and drm_gem_get_pages() marks the shmem mapping unevictable. So when /proc/meminfo Unevictable grows, nothing shows whether that is live client memory, kernel firmware objects or references the driver kept after the client went away. Count every object from creation until it is freed, split by user and kernel ownership, and expose the totals read-only as /sys/module/asahi/parameters/gem_stats: user_objects user_bytes user_created user_freed kernel_objects kernel_bytes kernel_created kernel_freed The counters are relaxed atomics, updated in the DriverObject constructor and in a new PinnedDrop of the per-object data, which records the object size. If user_bytes stays flat across a client-exit loop, clients are not leaking. If it keeps growing while the per-process mappings do not, the driver is holding objects for clients that have gone. Assisted-by: LLM Signed-off-by: Ryan Murray Adapted to the rewrite GEM constructors and pinned drop. The counter balances construction and destruction; its sysfs output is verified on local Neo. Upstream-commit: 3361051608523c5c31f5dba92f48ff4258126f2d (cherry picked from commit 66af386568f748d2499be3e3587ce1e7ca0ae9cd) --- drivers/gpu/drm/asahi/asahi.rs | 1 + drivers/gpu/drm/asahi/gem.rs | 19 ++++++- drivers/gpu/drm/asahi/gem_stats.rs | 91 ++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/asahi/gem_stats.rs diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index 4842e421a09ae..513bb6d103f65 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -15,6 +15,7 @@ mod file; mod float; mod fw; mod gem; +mod gem_stats; mod g17; mod gpu; mod hw; diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 1f766982074e3..479e3cf1c15ad 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -41,10 +41,12 @@ use crate::{ const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; /// Represents the inner data of a GEM object for this driver. -#[pin_data] +#[pin_data(PinnedDrop)] pub(crate) struct AsahiObject { /// ID for debug id: u64, + /// Object size retained for live-object accounting. + size: usize, /// Object creation flags. flags: u32, /// Whether this object can be exported. @@ -150,6 +152,13 @@ impl ObjectRef { } } +#[pinned_drop] +impl PinnedDrop for AsahiObject { + fn drop(self: Pin<&mut Self>) { + crate::gem_stats::note_free(self.kernel, self.size); + } +} + pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, @@ -226,11 +235,17 @@ impl DriverObject for AsahiObject { const HAS_EXPORT: bool = true; /// Callback to create the inner data of a GEM object - fn new(_dev: &AsahiDevice, _size: usize, args: Self::Args) -> impl PinInit { + fn new(_dev: &AsahiDevice, size: usize, args: Self::Args) -> impl PinInit { let id = GEM_ID.fetch_add(1, Ordering::Relaxed); mod_pr_debug!("AsahiObject::new id={}\n", id); try_pin_init!(AsahiObject { id, + // All remaining fields are infallible. PinnedDrop balances the + // accounting once this initializer completes. + size: { + crate::gem_stats::note_new(args.kernel, size); + size + }, flags: args.flags, exportable: args.exportable, kernel: args.kernel, diff --git a/drivers/gpu/drm/asahi/gem_stats.rs b/drivers/gpu/drm/asahi/gem_stats.rs new file mode 100644 index 0000000000000..abd01922273bd --- /dev/null +++ b/drivers/gpu/drm/asahi/gem_stats.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Live GEM object accounting (`gem_stats`, root-only, read-only sysfs). +//! +//! Every shmem object this driver creates is counted from creation until its +//! last reference drops, split by user and kernel ownership. Once a GEM object +//! is mapped into a GPU VM its pages stay pinned (and unevictable) until the +//! object is freed, so `user_bytes + kernel_bytes` is an upper bound on the +//! GPU share of /proc/meminfo `Unevictable`, and a value that keeps growing +//! after clients exit identifies a reference leak in the driver. The +//! `*_created` / `*_freed` totals show allocation churn. +use core::fmt::Write; +use core::sync::atomic::{AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +static USER_OBJECTS: AtomicU64 = AtomicU64::new(0); +static USER_BYTES: AtomicU64 = AtomicU64::new(0); +static USER_CREATED: AtomicU64 = AtomicU64::new(0); +static KERNEL_OBJECTS: AtomicU64 = AtomicU64::new(0); +static KERNEL_BYTES: AtomicU64 = AtomicU64::new(0); +static KERNEL_CREATED: AtomicU64 = AtomicU64::new(0); + +fn counters(kernel: bool) -> (&'static AtomicU64, &'static AtomicU64, &'static AtomicU64) { + if kernel { + (&KERNEL_OBJECTS, &KERNEL_BYTES, &KERNEL_CREATED) + } else { + (&USER_OBJECTS, &USER_BYTES, &USER_CREATED) + } +} + +/// Count one new GEM object. +pub(crate) fn note_new(kernel: bool, size: usize) { + let (objects, bytes, created) = counters(kernel); + objects.fetch_add(1, Ordering::Relaxed); + bytes.fetch_add(size as u64, Ordering::Relaxed); + created.fetch_add(1, Ordering::Relaxed); +} + +/// Count one freed GEM object. +pub(crate) fn note_free(kernel: bool, size: usize) { + let (objects, bytes, _) = counters(kernel); + objects.fetch_sub(1, Ordering::Relaxed); + bytes.fetch_sub(size as u64, Ordering::Relaxed); +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); + Ok(()) + } +} + +unsafe extern "C" fn stats_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; + let mut out = Output { bytes, used: 0 }; + let r = |c: &AtomicU64| c.load(Ordering::Relaxed); + let uc = r(&USER_CREATED); + let kc = r(&KERNEL_CREATED); + let (uo, ko) = (r(&USER_OBJECTS), r(&KERNEL_OBJECTS)); + let _ = writeln!(out, + "user_objects={} user_bytes={} user_created={} user_freed={} \ + kernel_objects={} kernel_bytes={} kernel_created={} kernel_freed={}", + uo, r(&USER_BYTES), uc, uc.wrapping_sub(uo), + ko, r(&KERNEL_BYTES), kc, kc.wrapping_sub(ko)); + out.used as core::ffi::c_int +} + +unsafe extern "C" fn stats_set(_: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + -1 // EPERM: read-only counters +} + +static STATS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(stats_set), get: Some(stats_get), free: None, +}; +#[link_section = "__param"] +#[used] +static STATS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"gem_stats" + } else { c"asahi.gem_stats" }), + #[cfg(MODULE)] + mod_: core::ptr::addr_of_mut!(bindings::__this_module), + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&STATS_OPS), perm: 0o400, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); From 2c076e0de6fe0a0a38a2041b02d5b9e75347375a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 6 Oct 2026 09:40:35 +1100 Subject: [PATCH 0628/1579] drm/asahi: port j700 queue accounting and compute recovery isolation Keep sixteen physical QIDs available for render while allowing existing compute channels to continue unchanged. Account for queue ownership and exhaustion at structural transitions, without adding full-table scans to normal retirement. Classify bounded compute watchdog deferrals using older unfinished work from another VM; do not blame a potential victim VM when that grace period expires. Adapted from Ryan Murray's upstream commits: dc66a43fdbd8 (firmware queue-slot accounting) dd27c0e2de7b (render QID reserve and exhausted-pool diagnostics) ccd546c5cb23 (compute watchdog isolation) The rewrite already reclaims drained quarantined render pairs after a fresh retirement witness; f055aaf00ec2 requires no default toggle here. Validation: matching module build and stack audit; production allocator, accounting and timeout-policy tests; local Neo desktop and the full light/heavy/compute depth 1/2/4/8/16 matrix; four concurrent clients for 60 seconds with sampled readback checks. Fault-injected recovery remains unqualified by these healthy-path hardware tests. Signed-off-by: Ryan Murray (cherry picked from commit f7fcc50568b9467f241a425b20b92ff1de10f641) --- drivers/gpu/drm/asahi/g17/context.rs | 2 +- drivers/gpu/drm/asahi/g17/job.rs | 15 +- drivers/gpu/drm/asahi/g17/kick.rs | 65 ++++++- drivers/gpu/drm/asahi/g17/mod.rs | 2 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 41 ++++ drivers/gpu/drm/asahi/g17/queue_stats.rs | 177 ++++++++++++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 3 + drivers/gpu/drm/asahi/g17/runtime/backend.rs | 12 +- .../drm/asahi/g17/runtime/compute_control.rs | 40 ++++ .../drm/asahi/g17/runtime/render/settle.rs | 3 + drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 2 + drivers/gpu/drm/asahi/g17/timeout.rs | 60 ++++++ 12 files changed, 410 insertions(+), 12 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/queue_stats.rs create mode 100644 drivers/gpu/drm/asahi/g17/timeout.rs diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index c3a3c6e87a2f6..1a28386d788d0 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -39,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(ENOSPC); + return Err(crate::g17::queue_stats::enospc(crate::g17::queue_stats::Pool::QueueContextQos)); } *used |= 1u128 << id; Ok(QosLease { diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index c05b5119790fc..c80955527cc06 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -248,7 +248,7 @@ pub(crate) trait Backend: Send + Sync { fn timed_out(&self, packet: Arc) -> sched::Status; fn cancel(&self, packet: Arc); /// Transfer a timeout to an outstanding off-lock graph transaction, if any. - fn defer_timeout(&self, packet: &Arc) -> bool; + fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool; /// Return true only after taking publication ownership and recording its result. fn try_early(&self, packet: &Arc) -> bool; fn count_backlog(&self, packet: &Packet); @@ -256,6 +256,7 @@ pub(crate) trait Backend: Send + Sync { struct Job { backend: Arc, + victim_renewals: u8, packet: Arc, } @@ -272,9 +273,14 @@ impl Drop for Job { impl sched::JobImpl for Job { fn false_timeout(job: &mut sched::Job) -> bool { - job.is_finished() - || job.packet.completion.take_replay_timeout() - || job.backend.defer_timeout(&job.packet) + if job.is_finished() || job.packet.completion.take_replay_timeout() { + return true; + } + let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); + if deferred && job.packet.engine() == Engine::Compute { + job.victim_renewals += 1; + } + deferred } fn prepare(job: &mut sched::Job) -> Option { @@ -337,6 +343,7 @@ impl Scheduler { 1, Job { backend: self.backend.clone(), + victim_renewals: 0, packet: packet.clone(), }, )?; diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index f36ab9798996a..aa2c054a9bbc7 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,7 +6,7 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::sync::atomic::{fence, Ordering}; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; use kernel::prelude::*; use super::{ @@ -15,7 +15,30 @@ use super::{ }; use crate::{hw::t8140, mem, mmu}; -const QID_COUNT: usize = QID_MAX as usize + 1; +pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; + +pub(super) const COMPUTE_RENDER_RESERVE: usize = 16; +pub(super) static LEASED: [AtomicU32; 3] = [const { AtomicU32::new(0) }; 3]; +pub(super) static PUBLISHED: [AtomicU32; 3] = [const { AtomicU32::new(0) }; 3]; +pub(super) static HIGH_WATER: AtomicU32 = AtomicU32::new(0); +pub(super) static EXHAUSTED: AtomicU64 = AtomicU64::new(0); +pub(super) static RESERVE_REFUSED: AtomicU64 = AtomicU64::new(0); + +pub(super) fn leased_total() -> u32 { + LEASED.iter().map(|count| count.load(Ordering::Relaxed)).sum() +} + +fn exhausted(engine: DataMaster, reserve: bool) -> Error { + if reserve { + RESERVE_REFUSED.fetch_add(1, Ordering::Relaxed); + } else { + EXHAUSTED.fetch_add(1, Ordering::Relaxed); + } + super::queue_stats::enospc(match engine { + DataMaster::Compute => super::queue_stats::Pool::QidCompute, + _ => super::queue_stats::Pool::QidRenderPair, + }) +} /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -44,12 +67,16 @@ struct Identity { /// One shared namespace for tiling, fragment and compute queues. pub(crate) struct QueueIds { entries: [Option; QID_COUNT], + compute_publication: u64, + leased: usize, } impl QueueIds { pub(crate) fn new() -> impl Init { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), + compute_publication: 0, + leased: 0, }) } @@ -76,11 +103,16 @@ impl QueueIds { Err(EINVAL) }; } + // Healthy existing channels keep their QID. Never recycle a published + // identity to recover capacity after a fault; leave eight render pairs. + if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { + return Err(exhausted(engine, true)); + } let index = preferred .map(usize::from) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) .or_else(|| self.entries.iter().position(Option::is_none)) - .ok_or(ENOSPC)?; + .ok_or_else(|| exhausted(engine, false))?; let id = Id { qid: index as u8, owner, @@ -90,9 +122,18 @@ impl QueueIds { id, published: false, }); + self.leased += 1; + LEASED[engine as usize].fetch_add(1, Ordering::Relaxed); + HIGH_WATER.fetch_max(leased_total(), Ordering::Relaxed); Ok(id) } + /// Ordered under the device mutex; no atomic operation on the submission path. + pub(crate) fn next_compute_publication(&mut self) -> Result { + self.compute_publication = self.compute_publication.checked_add(1).ok_or(EOVERFLOW)?; + Ok(self.compute_publication) + } + /// Call after preflighting the complete installation, before any address /// becomes firmware-visible. A failed or partial installation retains it. pub(crate) fn publish(&mut self, id: Id) -> Result { @@ -104,7 +145,10 @@ impl QueueIds { if entry.id != id { return Err(EINVAL); } - entry.published = true; + if !entry.published { + entry.published = true; + PUBLISHED[id.engine as usize].fetch_add(1, Ordering::Relaxed); + } Ok(()) } @@ -113,6 +157,8 @@ impl QueueIds { match slot { Some(entry) if entry.id == id && !entry.published => { *slot = None; + self.leased -= 1; + LEASED[id.engine as usize].fetch_sub(1, Ordering::Relaxed); Ok(()) } _ => Err(EINVAL), @@ -120,6 +166,17 @@ impl QueueIds { } } +impl Drop for QueueIds { + fn drop(&mut self) { + for entry in self.entries.iter().flatten() { + LEASED[entry.id.engine as usize].fetch_sub(1, Ordering::Relaxed); + if entry.published { + PUBLISHED[entry.id.engine as usize].fetch_sub(1, Ordering::Relaxed); + } + } + } +} + // Aliases drop before their backing. A compute queue is allocated low first; // a render queue is allocated high first. enum Backing { diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index f379c1b69d256..a5c867409d379 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -43,6 +43,8 @@ mod recovery; mod runtime; pub(crate) mod status; mod teardown; +mod timeout; +mod queue_stats; mod validation; use core::any::Any; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 2a30cd54cecd1..e0cdc05044362 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -308,6 +308,7 @@ impl Fifo { struct Active { packet: Arc, + published: u64, work: Arc, ticket: Option, observed: Option, @@ -363,6 +364,7 @@ impl Backing { /// Installed graph and its exact logical owner. A released idle graph may change owners only /// after a retirement witness. Failed work retains its pointers independently of fence state. pub(crate) struct Queue { + slot_accounting: crate::g17::queue_stats::ComputeAccount, owner: Option, binding: Binding, previous: Option, @@ -412,6 +414,7 @@ impl Queue { context.mark_published(); Ok(Self { owner: Some(owner), + slot_accounting: crate::g17::queue_stats::ComputeAccount::new(), binding, previous: None, kick, @@ -446,6 +449,11 @@ impl Queue { pub(crate) fn qid(&self) -> u8 { self.kick.id().qid() } + fn update_slot_accounting(&mut self) { + self.slot_accounting.update(self.owner.is_some(), self.released, + self.quarantined, self.retired_by_teardown); + } + pub(crate) fn owner(&self) -> Option { self.owner } @@ -462,6 +470,22 @@ impl Queue { pub(crate) fn room(&self) -> bool { !self.quarantined && !self.released && self.replays.len == 0 && self.active.len < IN_FLIGHT } + /// Quarantines without a pending witness no longer describe live pipe order. + pub(crate) fn timeout_head(&self) -> Option { + if self.quarantined && !self.awaiting_witness { + return None; + } + self.active.front().map(|active| crate::g17::timeout::Head { + vm: Arc::as_ptr(active.packet.completion.status()) as usize, + published: active.published, + finished: active.observed.is_some(), + }) + } + + pub(crate) fn can_classify_timeout(&self, packet: &Arc) -> bool { + !self.quarantined && self.owns_packet(packet) + } + pub(crate) fn oldest_spared(&self) -> Option { self.active .front() @@ -567,10 +591,12 @@ impl Queue { ) -> Result>> { if !self.quarantined { self.quarantined = true; + self.update_slot_accounting(); self.quarantine_error = Some(error); self.failure_status_pending = true; self.retirement_proved = false; self.retired_by_teardown = false; + self.update_slot_accounting(); let all_spared = self.active.len != 0 && self.active.iter().all(|a| a.packet.completion.spared()); self.spared_quarantine = all_spared && error == ENODATA; @@ -676,6 +702,7 @@ impl Queue { self.retirement_ready = self.active.len == 0; if self.released && self.retirement_ready { self.owner = None; + self.update_slot_accounting(); } } Ok(None) | Err(EAGAIN) => { @@ -782,6 +809,7 @@ impl Queue { self.submitted = 0; self.kick.clear_parent_after_recovery(); self.quarantined = false; + self.update_slot_accounting(); self.retire_pending = false; self.retirement_ready = true; if self @@ -793,6 +821,7 @@ impl Queue { } if self.released { self.owner = None; + self.update_slot_accounting(); } } Ok(()) @@ -932,9 +961,12 @@ impl Queue { } context.mark_published(); self.owner = Some(owner); + self.update_slot_accounting(); self.released = false; + self.update_slot_accounting(); self.retirement_ready = false; self.retired_by_teardown = false; + self.update_slot_accounting(); Ok(()) } /// True allows the registry to cancel the never-published QID and drop this graph. An @@ -944,11 +976,14 @@ impl Queue { return Ok(true); } self.released = true; + self.update_slot_accounting(); if self.active.len == 0 && !self.quarantined { if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { if let Err(error) = self.graph.set_owner(&previous.context) { self.quarantined = true; + self.update_slot_accounting(); self.retired_by_teardown = true; + self.update_slot_accounting(); return Err(error); } let mut previous = self.previous.take().ok_or(EIO)?; @@ -961,6 +996,7 @@ impl Queue { } if self.retirement_ready { self.owner = None; + self.update_slot_accounting(); } } Ok(false) @@ -1027,7 +1063,9 @@ impl Queue { return Ok(true); } self.released = true; + self.update_slot_accounting(); self.retired_by_teardown = true; + self.update_slot_accounting(); if let Some(active) = self .active .iter_mut() @@ -1159,6 +1197,7 @@ impl Graph { /// Borrowed device operations used during one publication. Implementations borrow the initdata, /// coprocessor and accounting fields separately from the Registry that owns physical queues. pub(crate) trait Host { + fn next_compute_publication(&mut self) -> Result; fn epoch(&self) -> Result<(u64, u32)>; fn prepare_compute_shared(&mut self) -> Result; fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; @@ -1313,9 +1352,11 @@ impl Queue { event_mask: COMPUTE_KICK_EVENT_MASK, register_arrays: ComputeDescriptor::register_bindings(descriptor_low)?, }; + let published = host.next_compute_publication()?; self.active .push(Active { packet: packet.clone(), + published, work: work.clone(), ticket: None, observed: None, diff --git a/drivers/gpu/drm/asahi/g17/queue_stats.rs b/drivers/gpu/drm/asahi/g17/queue_stats.rs new file mode 100644 index 0000000000000..5a5605e1d19a2 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue_stats.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Firmware queue-slot accounting (`g17p_queue_slots`, root-only, read-only +//! sysfs). Relaxed counters; exhaustion logging is rate-limited. +//! +//! Every render pair and compute queue leases QIDs from one 128-entry table +//! (`kick::QueueIds`). A published QID is kept until processor stop, and a +//! quarantined or exit-killed compute graph is never handed out again, so a +//! boot can run out of QIDs while memory is fine. Queue creation then fails +//! with ENOSPC, and so does a new render pair (`render-warm-pairs`). The +//! `qids` fields are exact (kept by the table itself); compute +//! graph ownership gauges track structural changes. Pending teardown and +//! draining-pair counts are sampled in their existing service passes. +//! +//! Every ENOSPC from one of these pools is counted by pool and named in +//! dmesg (the first one, then each power of two), so a failed QUEUE_CREATE +//! or submit says which pool ran out. +use core::fmt::Write; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam, prelude::*}; + +use super::kick as g17_channel; +use g17_channel::QID_COUNT; + +static COMPUTE_GRAPHS: AtomicU32 = AtomicU32::new(0); +static COMPUTE_OWNED: AtomicU32 = AtomicU32::new(0); +static COMPUTE_RELEASED: AtomicU32 = AtomicU32::new(0); +static COMPUTE_QUARANTINED: AtomicU32 = AtomicU32::new(0); +static COMPUTE_EXIT_KILLED: AtomicU32 = AtomicU32::new(0); +static RETIRED_GRAPHS: AtomicU32 = AtomicU32::new(0); +static TEARDOWNS: AtomicU32 = AtomicU32::new(0); +static RENDER_QUARANTINED: AtomicU32 = AtomicU32::new(0); + +/// Pools whose exhaustion returns ENOSPC to a client. +#[derive(Clone, Copy)] +pub(crate) enum Pool { + /// A render slot's first use leases two QIDs (`render-warm-pairs`). + QidRenderPair, + /// A new compute graph leases one QID (queue creation / first compute). + QidCompute, + /// A new logical queue context (QUEUE_CREATE): its only 128-wide pool + /// is the QoS hardware-buffer IDs, held by logical queues and installed + /// graph bindings. + QueueContextQos, +} + +const POOL_COUNT: usize = 3; +const POOL_NAMES: [&str; POOL_COUNT] = ["qid-render-pair", "qid-compute", "queue-context-qos"]; +static POOL_ENOSPC: [AtomicU64; POOL_COUNT] = + [AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0)]; +/// Submits that failed with ENOSPC, whichever pool. +static SUBMIT_ENOSPC: AtomicU64 = AtomicU64::new(0); + +fn log_enospc(pool: &str, stage: &str, count: u64) { + // First one, then each power of two: a stuck pool cannot flood dmesg. + if count.is_power_of_two() { + pr_warn!( + "G17P ENOSPC pool={} stage={} (#{}): qids leased={}/{} compute={} compute-quarantined={} \ + reserve-refused={}\n", + pool, stage, count, g17_channel::leased_total(), QID_COUNT, + g17_channel::LEASED[2].load(Ordering::Relaxed), + COMPUTE_QUARANTINED.load(Ordering::Relaxed), + g17_channel::RESERVE_REFUSED.load(Ordering::Relaxed), + ); + } +} + +/// Count and name one ENOSPC from `pool`, and return it. +#[cold] +pub(crate) fn enospc(pool: Pool) -> Error { + let count = POOL_ENOSPC[pool as usize].fetch_add(1, Ordering::Relaxed) + 1; + log_enospc(POOL_NAMES[pool as usize], "-", count); + ENOSPC +} + +/// A submit failed with ENOSPC at `stage`. Pools tagged at their source +/// were already counted and named; this names the rest by stage. +#[cold] +pub(crate) fn note_submit_enospc(stage: &str) { + let count = SUBMIT_ENOSPC.fetch_add(1, Ordering::Relaxed) + 1; + // Source-specific QID/QoS failures already counted their actual pool. + // This boundary knows the engine stage, not which nested allocator failed. + log_enospc("submit", stage, count); +} + +/// Incremental graph ownership accounting. Normal submissions do not touch +/// these counters; only graph creation, ownership transitions and drop do. +pub(crate) struct ComputeAccount { flags: u8 } +impl ComputeAccount { + pub(crate) fn new() -> Self { + COMPUTE_GRAPHS.fetch_add(1, Ordering::Relaxed); + COMPUTE_OWNED.fetch_add(1, Ordering::Relaxed); + Self { flags: 1 } + } + pub(crate) fn update(&mut self, owned: bool, released: bool, quarantined: bool, exit_killed: bool) { + let next = u8::from(owned) | (u8::from(released) << 1) + | (u8::from(quarantined) << 2) | (u8::from(exit_killed) << 3); + let changed = next ^ self.flags; + for (bit, counter) in [&COMPUTE_OWNED, &COMPUTE_RELEASED, + &COMPUTE_QUARANTINED, &COMPUTE_EXIT_KILLED].into_iter().enumerate() + { + if changed & (1 << bit) != 0 { + if next & (1 << bit) != 0 { counter.fetch_add(1, Ordering::Relaxed); } + else { counter.fetch_sub(1, Ordering::Relaxed); } + } + } + self.flags = next; + } +} +impl Drop for ComputeAccount { + fn drop(&mut self) { + self.update(false, false, false, false); + COMPUTE_GRAPHS.fetch_sub(1, Ordering::Relaxed); + } +} + +pub(crate) fn note_teardowns(count: usize) { + TEARDOWNS.store(count as u32, Ordering::Relaxed); +} +pub(crate) fn note_draining_pairs(count: usize) { + RENDER_QUARANTINED.store(count as u32, Ordering::Relaxed); +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); + Ok(()) + } +} + +unsafe extern "C" fn slots_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; + let mut out = Output { bytes, used: 0 }; + let r = |c: &AtomicU32| c.load(Ordering::Relaxed); + let e = |pool: usize| POOL_ENOSPC[pool].load(Ordering::Relaxed); + let leased = g17_channel::leased_total(); + let published: u32 = g17_channel::PUBLISHED.iter().map(r).sum(); + let _ = writeln!(out, + "qids leased={} free={} published={} tiling={} fragment={} compute={} high_water={} exhausted={} \ + reserve={} reserve_refused={} \ + compute graphs={} owned={} released={} quarantined={} exit_killed={} retired_graphs={} \ + teardowns={} render_draining_pairs={} \ + enospc qid-render-pair={} qid-compute={} queue-context-qos={} submits={}", + leased, (QID_COUNT as u32).saturating_sub(leased), published, + r(&g17_channel::LEASED[0]), r(&g17_channel::LEASED[1]), r(&g17_channel::LEASED[2]), + r(&g17_channel::HIGH_WATER), g17_channel::EXHAUSTED.load(Ordering::Relaxed), + g17_channel::COMPUTE_RENDER_RESERVE, g17_channel::RESERVE_REFUSED.load(Ordering::Relaxed), + r(&COMPUTE_GRAPHS), r(&COMPUTE_OWNED), r(&COMPUTE_RELEASED), r(&COMPUTE_QUARANTINED), + r(&COMPUTE_EXIT_KILLED), r(&RETIRED_GRAPHS), r(&TEARDOWNS), r(&RENDER_QUARANTINED), + e(0), e(1), e(2), SUBMIT_ENOSPC.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} + +unsafe extern "C" fn slots_set(_: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + -1 // EPERM: read-only counters +} + +static SLOTS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(slots_set), get: Some(slots_get), free: None, +}; +#[link_section = "__param"] +#[used] +static SLOTS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_queue_slots" + } else { c"asahi.g17p_queue_slots" }), + #[cfg(MODULE)] + mod_: core::ptr::addr_of_mut!(bindings::__this_module), + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&SLOTS_OPS), perm: 0o400, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index d100fef631630..294d66b91f8dc 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -546,6 +546,9 @@ struct ComputeHost<'a> { generation: u64, } impl compute::Host for ComputeHost<'_> { + fn next_compute_publication(&mut self) -> Result { + self.qids.next_compute_publication() + } fn epoch(&self) -> Result<(u64, u32)> { Ok(( self.generation, diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 085655beb8194..3fb40779021e5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -138,6 +138,12 @@ impl job::Backend for Backend { Engine::Render => self.render_packet(&packet, &mut deferred), } }; + if result.as_ref().is_err_and(|error| *error == ENOSPC) { + crate::g17::queue_stats::note_submit_enospc(match packet.engine() { + Engine::Compute => "compute-publish", + Engine::Render => "render-publish", + }); + } self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } @@ -169,9 +175,9 @@ impl job::Backend for Backend { } } - fn defer_timeout(&self, packet: &Arc) -> bool { - if packet.engine() != Engine::Render { - return false; + fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool { + if packet.engine() == Engine::Compute { + return self.defer_compute_timeout(packet, renewals); } let mut state = self.shared.state.lock(); (*state) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index 610643ef6e379..dac4ca48a34ae 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -7,6 +7,46 @@ use crate::g17::job::{self, Packet}; use kernel::{prelude::*, sync::Arc}; impl Backend { + /// A final retirement pass precedes bounded cross-VM timeout attribution. + /// All completion callbacks run after releasing the device mutex. + pub(super) fn defer_compute_timeout(&self, packet: &Arc, renewals: u8) -> bool { + use crate::g17::{recovery::Class, timeout::{self, Decision}}; + let Ok(mut deferred) = DeferredBatch::worker() else { + return false; + }; + let result = (|| -> Result { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(Decision::Own); + }; + firmware.scan_compute(None, &mut deferred)?; + let Some(queue) = firmware.queues.compute(self.owner) + .filter(|queue| queue.can_classify_timeout(packet)) else { + return Ok(Decision::Own); + }; + let Some(own) = queue.timeout_head() else { + return Ok(Decision::Own); + }; + let others = firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|other| other.owner() != Some(self.owner)) + .filter_map(|other| other.timeout_head()); + let decision = timeout::classify(own, renewals, others); + if decision == Decision::Spare { + firmware.queues.compute(self.owner).ok_or(EIO)? + .classify(&|_| Class::Spared); + } + if decision != Decision::Own && (renewals == 0 || decision == Decision::Spare) { + dev_warn!(self.shared.dev.as_ref(), + "Compute watchdog owner={} publication={} renewals={} decision={:?} behind earlier work of another VM\n", + self.owner, own.published, renewals, decision); + } + Ok(decision) + })(); + deferred.finish(); + matches!(result, Ok(Decision::Defer)) + } + /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index a6d5821561d24..2cf7a5d77ee5a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -164,6 +164,7 @@ impl crate::g17::Firmware { return Ok(()); } let mut serviced = false; + let mut draining = 0; for slot in 1..self.queues.render.entries.len() { if !self.queues.render.entries[slot] .as_ref() @@ -172,6 +173,7 @@ impl crate::g17::Firmware { { continue; } + draining += 1; if let Err(error) = self.drain_render_slot(slot, &mut serviced, deferred) { dev_warn!( self.primary.state.shared.dev.as_ref(), @@ -181,6 +183,7 @@ impl crate::g17::Firmware { ); } } + crate::g17::queue_stats::note_draining_pairs(draining); Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 10f334e4fe561..06465629eb2be 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -165,6 +165,7 @@ impl Firmware { /// The event worker has already reclaimed drained pairs and reannounced /// pending controls. Recovery blocks new releases, never an observed ack. pub(in crate::g17) fn service_teardown(&mut self, reclaim: &mut ReclaimBatch) -> Result { + crate::g17::queue_stats::note_teardowns(self.queues.teardown.len()); self.queues.render.collect_reclaimable(reclaim)?; if self.queues.teardown.is_empty() { return Ok(()); @@ -194,6 +195,7 @@ impl Firmware { .is_some() {} teardown.report_stalled(super::now_ns(), &mut service); + crate::g17::queue_stats::note_teardowns(teardown.len()); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs new file mode 100644 index 0000000000000..e76dab07f56db --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded isolation for a compute watchdog blocked behind another VM's older kick. +//! Publication order is evidence of a possible blocker, not proof of a firmware +//! dependency. Renewals are bounded; the oldest unfinished queue is never excused. + +pub(crate) const MAX_RENEWALS: u8 = 8; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Head { + pub(crate) vm: usize, + pub(crate) published: u64, + pub(crate) finished: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Decision { + Own, + Defer, + Spare, +} + +pub(crate) fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { + if own.finished || !others.into_iter().any(|other| { + other.vm != own.vm && !other.finished && other.published < own.published + }) { + Decision::Own + } else if renewals < MAX_RENEWALS { + Decision::Defer + } else { + Decision::Spare + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn head(vm: usize, published: u64, finished: bool) -> Head { + Head { vm, published, finished } + } + #[test] + fn earlier_other_vm_gets_bounded_grace() { + let own = head(5, 101, false); + for count in 0..MAX_RENEWALS { + assert_eq!(classify(own, count, [head(7, 100, false)].into_iter()), Decision::Defer); + } + assert_eq!(classify(own, MAX_RENEWALS, [head(7, 100, false)].into_iter()), Decision::Spare); + assert_eq!(classify(own, 2, [].into_iter()), Decision::Own); + } + #[test] + fn oldest_unfinished_queue_is_never_excused() { + assert_eq!(classify(head(7, 100, false), 0, [head(5, 101, false)].into_iter()), Decision::Own); + } + #[test] + fn same_vm_finished_and_later_work_do_not_block() { + let own = head(5, 101, false); + assert_eq!(classify(own, 0, [head(5, 100, false), head(7, 100, true), head(7, 101, false), head(7, 102, false)].into_iter()), Decision::Own); + assert_eq!(classify(head(5, 101, true), 0, [head(7, 100, false)].into_iter()), Decision::Own); + } +} From 2d84d7470c0f0d60c11c9df9e9665ee6a6a2bf71 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 06:59:34 -0400 Subject: [PATCH 0629/1579] drm/asahi: account once per compute queue transition Queue quarantine, owner rebinding, release failure and cancellation each updated the accounting state between consecutive field assignments. Keep one update after each complete transition, removing five redundant calls without changing the final gauges or queue policy. Exercise the production state assignments and counter implementation over all 64 transition/flag combinations. The work-count assertion fails on the previous implementation and passes here. The native GPU module builds without warnings against the fully built integration kernel; the static module-frame stack gate passes. No hardware performance gain is claimed. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 3c65418ee698735d33f299fe208aa8c22ff310a4) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index e0cdc05044362..3b259b37b38f4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -591,7 +591,6 @@ impl Queue { ) -> Result>> { if !self.quarantined { self.quarantined = true; - self.update_slot_accounting(); self.quarantine_error = Some(error); self.failure_status_pending = true; self.retirement_proved = false; @@ -961,9 +960,7 @@ impl Queue { } context.mark_published(); self.owner = Some(owner); - self.update_slot_accounting(); self.released = false; - self.update_slot_accounting(); self.retirement_ready = false; self.retired_by_teardown = false; self.update_slot_accounting(); @@ -981,7 +978,6 @@ impl Queue { if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { if let Err(error) = self.graph.set_owner(&previous.context) { self.quarantined = true; - self.update_slot_accounting(); self.retired_by_teardown = true; self.update_slot_accounting(); return Err(error); @@ -1063,7 +1059,6 @@ impl Queue { return Ok(true); } self.released = true; - self.update_slot_accounting(); self.retired_by_teardown = true; self.update_slot_accounting(); if let Some(active) = self From 2ebb1eb927960a96456df1369294d3d6e392a591 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 22:55:07 -0400 Subject: [PATCH 0630/1579] drm: carry shared GPUVM and scheduler interfaces used by the G17 review Carry the GPUVM array reservation, allocator interval query and scheduler lifetime interfaces from the cleanroom current-kernel integration. Preserve the zero-fence GPUVM locking contract and initialize zero-valid C storage without importing unexported bindgen Default methods into loadable modules. This isolates the four DRM prerequisite files from the wider j700 merge; the public series retains its own platform and RTKit base. The following review fixes are applied against these shared interfaces. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 982df2dcc66e2cb2012df476c4325c8fce016dda) --- drivers/gpu/drm/drm_gpuvm.c | 18 ++- rust/kernel/drm/gpuvm.rs | 155 ++++++++++++++++++++++++-- rust/kernel/drm/mm.rs | 38 ++++++- rust/kernel/drm/sched.rs | 216 +++++++++++++++++++++--------------- 4 files changed, 325 insertions(+), 102 deletions(-) diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c index 91f7861e36b84..b123f6164206b 100644 --- a/drivers/gpu/drm/drm_gpuvm.c +++ b/drivers/gpu/drm/drm_gpuvm.c @@ -1396,9 +1396,23 @@ fn_lock_array(struct drm_gpuvm_exec *vm_exec) struct drm_gem_object **objs; unsigned int num_objs; } *args = vm_exec->extra.priv; + unsigned int i; + int ret; + + /* + * Keep the zero-fence contract of every other drm_gpuvm_exec helper. + * drm_exec_prepare_array() unconditionally calls + * dma_resv_reserve_fences(), where zero is invalid; exec_prepare_obj() + * deliberately selects drm_exec_lock_obj() for that case instead. + */ + for (i = 0; i < args->num_objs; i++) { + ret = exec_prepare_obj(&vm_exec->exec, args->objs[i], + vm_exec->num_fences); + if (ret) + return ret; + } - return drm_exec_prepare_array(&vm_exec->exec, args->objs, - args->num_objs, vm_exec->num_fences); + return 0; } /** diff --git a/rust/kernel/drm/gpuvm.rs b/rust/kernel/drm/gpuvm.rs index b0d4b12f52978..3f7176a9e32cb 100644 --- a/rust/kernel/drm/gpuvm.rs +++ b/rust/kernel/drm/gpuvm.rs @@ -36,7 +36,7 @@ use crate::{ use core::cell::UnsafeCell; use core::marker::{PhantomData, PhantomPinned}; -use core::mem::ManuallyDrop; +use core::mem::{ManuallyDrop, MaybeUninit}; use core::ops::{Deref, DerefMut, Range}; use core::ptr::NonNull; use pin_init; @@ -384,7 +384,10 @@ pub(super) unsafe extern "C" fn vm_bo_alloc_callback() -> *mut b { let obj: Result>>> = KBox::try_pin_init( try_pin_init!(GpuVmBo:: { - bo <- pin_init::default(), + // The bindgen `Default` implementation is exactly a zero fill, but its + // out-of-line symbol is not exported to loadable modules. `drm_gpuvm_bo` + // is `Zeroable`, so initialize it directly without a cross-crate call. + bo <- pin_init::zeroed::(), inner <- T::GpuVmBo::new(), _p: PhantomPinned }), @@ -576,19 +579,26 @@ impl GpuVm { } else { 0 }, - exec: Default::default(), + // SAFETY: bindgen's `Default` for this C structure is exactly + // `MaybeUninit::zeroed().assume_init()`. Spell that out here so + // external modules do not import its unexported trait method. + exec: unsafe { MaybeUninit::::zeroed().assume_init() }, extra: match (is_ext, obj) { (true, Some(obj)) => bindings::drm_gpuvm_exec__bindgen_ty_1 { fn_: Some(exec_lock_gem_object), priv_: obj.as_raw() as *const _ as *mut _, }, - _ => Default::default(), + // SAFETY: as above, this bindgen C structure is zero-valid. + _ => unsafe { + MaybeUninit::::zeroed() + .assume_init() + }, }, num_fences: 0, }), GFP_KERNEL, )?, - obj, + objects: LockedObjects::Single(obj), }); // SAFETY: The object is valid and was initialized above @@ -597,6 +607,59 @@ impl GpuVm { Ok(ManuallyDrop::into_inner(guard)) } + /// Lock the GPUVM and an array of additional GEM objects in one + /// wound/wait transaction. + pub fn exec_lock_array<'a, 'b>( + &'a self, + objects: &'b [ARef>], + interruptible: bool, + ) -> Result> { + let count = u32::try_from(objects.len()).map_err(|_| EINVAL)?; + let mut raw_objects = KVec::with_capacity(objects.len(), GFP_KERNEL)?; + for object in objects { + raw_objects.push(object.as_raw(), GFP_KERNEL)?; + } + + let mut guard = ManuallyDrop::new(LockedGpuVm { + gpuvm: self, + // vm_exec needs to be pinned, so stick it in a Box. + vm_exec: KBox::init( + init!(bindings::drm_gpuvm_exec { + vm: self.gpuvm() as *mut _, + flags: (if interruptible { + bindings::DRM_EXEC_INTERRUPTIBLE_WAIT + } else { + 0 + }) | bindings::DRM_EXEC_IGNORE_DUPLICATES, + // SAFETY: both bindgen C structures are zero-valid; their generated + // `Default` methods perform the same zero initialization but are not + // exported from the kernel crate to loadable modules. + exec: unsafe { MaybeUninit::::zeroed().assume_init() }, + extra: unsafe { + MaybeUninit::::zeroed() + .assume_init() + }, + num_fences: 0, + }), + GFP_KERNEL, + )?, + objects: LockedObjects::Array(objects), + }); + + // SAFETY: Every raw pointer is backed by an ARef in `objects`, which + // outlives the returned guard. The helper consumes the temporary + // pointer array before returning and keeps its own object references. + to_result(unsafe { + bindings::drm_gpuvm_exec_lock_array( + &mut *guard.vm_exec, + raw_objects.as_mut_ptr(), + count, + ) + })?; + + Ok(ManuallyDrop::into_inner(guard)) + } + /// Returns true if the given object is external to the GPUVM /// (that is, if it does not share the DMA reservation object of the GPUVM). pub fn is_extobj(&self, obj: &impl IntoGEMObject) -> bool { @@ -671,15 +734,36 @@ unsafe impl AlwaysRefCounted for GpuVm { } } +enum LockedObjects<'a, T: DriverGpuVm> { + Single(Option<&'a Object>), + Array(&'a [ARef>]), +} + +impl LockedObjects<'_, T> { + fn single(&self) -> Option<&Object> { + match self { + Self::Single(object) => *object, + Self::Array(_) => None, + } + } + + fn indexed(&self, index: usize) -> Option<&Object> { + match self { + Self::Array(objects) => objects.get(index).map(|object| &**object), + Self::Single(_) => None, + } + } +} + pub struct LockedGpuVm<'a, 'b, T: DriverGpuVm> { gpuvm: &'a GpuVm, vm_exec: KBox, - obj: Option<&'b Object>, + objects: LockedObjects<'b, T>, } impl LockedGpuVm<'_, '_, T> { pub fn find_bo(&mut self) -> Option>> { - let obj = self.obj?; + let obj = self.objects.single()?; // SAFETY: LockedGpuVm implies the right locks are held. let p = unsafe { bindings::drm_gpuvm_bo_find( @@ -699,7 +783,7 @@ impl LockedGpuVm<'_, '_, T> { } pub fn obtain_bo(&mut self) -> Result>> { - let obj = self.obj.ok_or(EINVAL)?; + let obj = self.objects.single().ok_or(EINVAL)?; // SAFETY: LockedGpuVm implies the right locks are held. let p = unsafe { bindings::drm_gpuvm_bo_obtain_locked( @@ -727,7 +811,55 @@ impl LockedGpuVm<'_, '_, T> { req_gem_range: u32, flags: GpuVaFlags, ) -> Result { - let obj = self.obj.ok_or(EINVAL)?; + let obj = self.objects.single().ok_or(EINVAL)?.as_raw(); + self.sm_map_raw( + obj, + ctx, + req_addr, + req_range, + req_offset, + req_gem_range, + flags, + ) + } + + /// Map with an object acquired by [`GpuVm::exec_lock_array`]. + pub fn sm_map_indexed( + &mut self, + object_index: usize, + ctx: &mut T::StepContext, + req_addr: u64, + req_range: u64, + req_offset: u64, + req_gem_range: u32, + flags: GpuVaFlags, + ) -> Result { + let obj = self + .objects + .indexed(object_index) + .ok_or(EINVAL)? + .as_raw(); + self.sm_map_raw( + obj, + ctx, + req_addr, + req_range, + req_offset, + req_gem_range, + flags, + ) + } + + fn sm_map_raw( + &mut self, + obj: *mut bindings::drm_gem_object, + ctx: &mut T::StepContext, + req_addr: u64, + req_range: u64, + req_offset: u64, + req_gem_range: u32, + flags: GpuVaFlags, + ) -> Result { let mut ctx = StepContext { ctx, gpuvm: self.gpuvm, @@ -742,13 +874,14 @@ impl LockedGpuVm<'_, '_, T> { gem: bindings::drm_gpuva_op_map__bindgen_ty_2 { offset: req_offset, range: req_gem_range, - obj: obj.as_raw(), + obj, }, flags: flags.as_raw(), }, }; - // SAFETY: LockedGpuVm implies the right locks are held. + // SAFETY: The object pointer came from the reference set retained by + // this guard, and LockedGpuVm implies the required locks are held. to_result(unsafe { bindings::drm_gpuvm_sm_map( self.gpuvm.gpuvm() as *mut _, diff --git a/rust/kernel/drm/mm.rs b/rust/kernel/drm/mm.rs index 7b13cfd7d5309..e1ec944d8424f 100644 --- a/rust/kernel/drm/mm.rs +++ b/rust/kernel/drm/mm.rs @@ -7,7 +7,7 @@ use crate::{ alloc::flags::*, bindings, - error::{to_result, Result}, + error::{code::EINVAL, to_result, Result}, sync::{new_mutex, Arc, Mutex, UniqueArc}, types::Opaque, }; @@ -17,7 +17,7 @@ use crate::prelude::KBox; use core::{ marker::{PhantomData, PhantomPinned}, - ops::Deref, + ops::{Deref, Range}, pin::Pin, }; @@ -161,6 +161,40 @@ pub struct Allocator, T> { } impl, T> Allocator { + /// Return the first allocator reservation intersecting an address range. + /// Unlike a page-table walk, this includes unmapped guard space retained + /// inside a node. The interval query and its result are protected by the + /// allocator mutex; no probe node is inserted into the live VA space. + pub fn first_overlap(&self, range: Range) -> Result>> { + if range.start >= range.end { + return Err(EINVAL); + } + let guard = self.mm.lock(); + // The DRM interval query requires both endpoints to lie inside its + // allocator. head_node is the end sentinel; its wrapping size is the + // negated allocator span. + let head = unsafe { &(*guard.0.get()).head_node }; + let allocator_start = head.start.wrapping_add(head.size); + if range.start < allocator_start || range.end > head.start { + return Err(EINVAL); + } + // SAFETY: The allocator is initialized for the lifetime of `self`. + // Its mutex prevents node removal while the returned pointer is read. + // drm_mm returns its end sentinel when no interval intersects. + let node = unsafe { + bindings::__drm_mm_interval_first( + guard.0.get(), range.start, range.end - 1, + ) + }; + // SAFETY: drm_mm_interval_first returns a valid node or the allocator + // end sentinel, both of which remain live while `guard` is held. + let node = unsafe { node.as_ref().ok_or(EINVAL)? }; + if node.start >= range.end { + return Ok(None); + } + Ok(Some(node.start..node.start.saturating_add(node.size))) + } + /// Create a new range allocator for the given start and size range of addresses. /// /// The user may optionally provide an inner object representing allocator state, which will diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index add84da67356c..90bd7142fe192 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -11,7 +11,6 @@ use crate::{ prelude::*, sync::{Arc, UniqueArc}, time::{self, msecs_to_jiffies}, - types::Opaque, }; use core::marker::PhantomData; use core::mem::MaybeUninit; @@ -25,10 +24,7 @@ pub enum Status { Nominal = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_RESET, /// Device is no longer available NoDevice = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_ENODEV, - /// The job is not hung, or the driver dealt with the timeout without a scheduler reset. - /// - /// The job goes back to the pending list and the timeout is rearmed. The job is freed once - /// it has finished, as usual. + /// Reinsert the job and rearm its timeout without resetting the scheduler. NoHang = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG, } @@ -47,12 +43,20 @@ pub enum Priority { /// Trait to be implemented by driver job objects. pub trait JobImpl: Sized { - /// Reconsider a timeout before the backend starts recovery. Returning true - /// reinserts this job and rearms its timeout without changing its fences. - /// The callback must not stop or restart the scheduler. - fn false_timeout(_job: &mut Job) -> bool { - false - } + /// Handle a scoped timeout without resetting the C scheduler. + /// + /// The backend must settle/quarantine its exact hardware work, or retain + /// that owner until an in-progress host transaction can classify it. It + /// must not invoke drm_sched_stop/start or replay scheduler jobs. The core + /// reinserts the detached guilty job through NO_HANG, preserving every + /// parent callback and the backend's actual success/error fence result. + const MANAGED_TIMEOUT_RECOVERY: bool = false; + + /// Reconsider a timeout before stopping the scheduler or settling fences. + /// Returning true asks the core to reinsert the retained job and rearm. + /// The backend must not stop/start the scheduler from this hook. + fn false_timeout(_job: &mut Job) -> bool { false } + /// Called when the scheduler is considering scheduling this job next, to get another Fence /// for this job to block on. Once it returns None, run() may be called. @@ -67,9 +71,6 @@ pub trait JobImpl: Sized { /// Called when a job has taken too long to execute, to trigger GPU recovery. /// - /// A driver that handles timeouts without a scheduler reset returns [`Status::NoHang`]. - /// [`Job::is_finished`] tells whether the job completed after the timeout fired. - /// /// This method is called in a workqueue context. fn timed_out(job: &mut Job) -> Status; @@ -112,19 +113,49 @@ unsafe extern "C" fn timedout_job_cb( // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: The timeout worker retains this detached Job until this - // callback returns, including when its parent fence signals concurrently. + // The C timeout worker owns this job outside pending_list until return. + // Free work cannot release it even if its parent signals concurrently. + if T::MANAGED_TIMEOUT_RECOVERY { + let finished = unsafe { addr_of_mut!((*(*sched_job).s_fence).finished) }; + if unsafe { bindings::dma_fence_get_status(finished) } != 0 { + return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; + } + } + // SAFETY: the C core retains the detached job across both callbacks. if T::false_timeout(unsafe { &mut *p }) { return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; } - // SAFETY: The same retained job remains valid after the first callback. - T::timed_out(unsafe { &mut *p }) as bindings::drm_gpu_sched_stat + let status = T::timed_out(unsafe { &mut *p }); + if T::MANAGED_TIMEOUT_RECOVERY { + // This is scoped fence handling, not C reset recovery. stop would + // detach and discard healthy parents; start would cancel those jobs. + // NO_HANG reinserts the exact job and queues free work if it completed + // during the callback. It does not change the parent's error or replay + // work. A deferred exact owner remains eligible for its real callback. + bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG + } else { + status as bindings::drm_gpu_sched_stat + } } unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched_job) { // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + // Fault-only ownership witness: this callback is where the core returns + // ownership of an errored job. A signalled fence alone is not that proof. + // SAFETY: A job reaching free_job has a live scheduler fence; its error + // was fixed before signal and the core's completion callback has finished. + if T::MANAGED_TIMEOUT_RECOVERY + && unsafe { (*(*sched_job).s_fence).finished.error } == ETIMEDOUT.to_errno() + { + // Use numeric identity: the fmt adapter borrows arguments, and + // Pointer formatting of that reference can print a stack address. + pr_info!("DRM Rust scheduler: released timed-out job={:#x} context={} seqno={}\n", + sched_job as usize, + unsafe { (*(*sched_job).s_fence).finished.context }, + unsafe { (*(*sched_job).s_fence).finished.seqno }); + } // Convert the job back to a Box and drop it // SAFETY: All of our Jobs are created inside a box. unsafe { drop(KBox::from_raw(p)) }; @@ -133,71 +164,70 @@ unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sched_job) { // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: A job on the pending list has a valid scheduler fence. + + // fini has stopped/synchronized submission and timeout workers. A parent + // callback can still run on the device completion thread. The scheduler + // fence retains the parent reference while we remove its callback under + // the parent's lock; that also waits for an already-running callback. let s_fence = unsafe { (*sched_job).s_fence }; - // SAFETY: `s_fence` is valid and holds a reference to its parent, if any. let parent = unsafe { (*s_fence).parent }; - // SAFETY: `s_fence` is valid. - let finished = unsafe { addr_of_mut!((*s_fence).finished) }; - - // drm_sched_fini() has stopped the scheduler workers, but the hardware fence can still - // signal concurrently. Fence callbacks run under the fence lock, so removing ours either - // disarms it or waits until it has completed the job. - let callback_removed = !parent.is_null() - // SAFETY: run_job() armed `cb` on `parent` before the submission worker was stopped. - && unsafe { bindings::dma_fence_remove_callback(parent, addr_of_mut!((*sched_job).cb)) }; - - // A job without a parent either completed when run_job() returned, or had its parent - // detached by drm_sched_stop(), which also returned its credits. - let complete_here = callback_removed - // SAFETY: `finished` is a valid, initialized fence. - || (parent.is_null() && unsafe { bindings::dma_fence_get_status(finished) } == 0); - - // SAFETY: All of our jobs are Job. + let removed_parent_callback = if !parent.is_null() { + // SAFETY: run_job installed (or attempted to install on an already + // signaled parent) this callback before fini synchronized submit work. + unsafe { bindings::dma_fence_remove_callback(parent, addr_of_mut!((*sched_job).cb)) } + } else { false }; + let finish_here = removed_parent_callback || (parent.is_null() + && unsafe { bindings::dma_fence_get_status(addr_of_mut!((*s_fence).finished)) == 0 }); + // Null/error run_job calls job_done before submit work returns, so its + // finished fence is already signaled. NULL+unfinished can instead survive + // a legacy backend's prior reset stop: C stop dropped the parent and its + // credits, but not its score. Do not subtract those credits twice. + + // SAFETY: cancellation cannot race a scheduler callback using this job + // after the removal/parent-lock synchronization above. The backend must + // retain any still-hardware-owned resources independently of Job storage. T::cancel(unsafe { &mut *p }); - if complete_here { - // Account for the job exactly as drm_sched_job_done() would have. - // - // SAFETY: The job belongs to the scheduler being torn down, which is still valid. + if finish_here { + // Match drm_sched_job_done's accounting exactly once, only when its + // callback did not do it. Submission/free work is stopped, so fini + // itself unlinks and frees this job; no workqueue rearm is appropriate. + let sched = unsafe { (*sched_job).sched }; unsafe { - let sched = (*s_fence).sched; - if callback_removed { - bindings::atomic_sub( - (*sched_job).credits as i32, - addr_of_mut!((*sched).credit_count), - ); + if removed_parent_callback { + bindings::atomic_sub((*sched_job).credits as i32, + addr_of_mut!((*sched).credit_count)); } - bindings::atomic_dec((*sched).score); + bindings::atomic_sub(1, (*sched).score); } - - // SAFETY: `finished` is a valid fence. - let fence = unsafe { Fence::get_raw(finished) }; + let fence = unsafe { Fence::get_raw(addr_of_mut!((*s_fence).finished)) }; fence.set_error(ECANCELED); - fence.signal(); + let _ = fence.signal(); } + // Do not clear/put parent here: s_fence still owns that reference and its + // normal release path drops it. A completed callback's result is retained. } /// A DRM scheduler job. pub struct Job { job: bindings::drm_sched_job, inner: T, - // Destroying an entity can leave its queued jobs to deferred work that still dereferences - // `job->sched` after the scheduler has been finalized. Keep the scheduler allocation alive - // for as long as any job references it; this does not delay finalization. + // Entity destruction can leave kill_jobs_work queued on the system + // workqueue or waiting for a dependency. That work still dereferences + // job->sched->ops after drm_sched_fini has stopped the scheduler workers. + // Retain only the allocation: retaining SchedulerOwner here would keep + // fini from running while pending jobs still need its cancellation. _scheduler: Arc>, } impl Job { - /// Returns whether the job has finished, successfully or not. + /// Returns whether an armed job has finished, successfully or not. pub fn is_finished(&self) -> bool { - // The finished fence is initialized when the job is armed, which also assigns its - // scheduler. if self.job.sched.is_null() { return false; } - - // SAFETY: The job is armed, so its scheduler fence is valid and initialized. + // SAFETY: An armed job has an initialized scheduler fence, retained + // for its lifetime, including while the timeout worker owns the job. unsafe { bindings::dma_fence_get_status(addr_of_mut!((*self.job.s_fence).finished)) != 0 } } } @@ -320,8 +350,8 @@ struct EntityInner { impl Drop for EntityInner { fn drop(&mut self) { - // SAFETY: The EntityInner is initialized. This will cancel/free all jobs, possibly from - // deferred work. Each job keeps the scheduler allocation alive until it is freed. + // SAFETY: The EntityInner is initialized. Jobs removed from the entity + // may still await kill_jobs_work; each retains the scheduler allocation. unsafe { bindings::drm_sched_entity_destroy(&mut self.entity) }; } } @@ -340,7 +370,7 @@ impl Entity { let mut entity: KBox>> = KBox::new_uninit(GFP_KERNEL | __GFP_ZERO)?; - let mut sched_ptr = sched.0.allocation.sched.get(); + let mut sched_ptr = &sched.0.allocation.sched as *const _ as *mut _; // SAFETY: The Box is allocated above and valid. unsafe { @@ -384,9 +414,13 @@ impl Entity { // SAFETY: The Box pointer is valid, and this initializes the inner member. unsafe { addr_of_mut!((*job.as_mut_ptr()).inner).write(inner) }; - // SAFETY: The Box pointer is valid, and this initializes the scheduler reference. + // SAFETY: The job allocation is valid and this initializes its last + // field. Arc::clone is infallible and the entity still owns the live + // scheduler, so every successfully initialized job retains its C + // callback target before it can be armed or submitted. unsafe { - addr_of_mut!((*job.as_mut_ptr())._scheduler).write(self.0.sched.allocation.clone()) + addr_of_mut!((*job.as_mut_ptr())._scheduler) + .write(self.0.as_ref().get_ref().sched.allocation.clone()) }; // SAFETY: All fields of the Job are now initialized. @@ -394,33 +428,36 @@ impl Entity { } } -/// DRM scheduler allocation, kept alive by the scheduler owner and by every job. +/// DRM scheduler allocation, retained by live owners and deferred job callbacks. pub struct SchedulerInner { - sched: Opaque, + sched: bindings::drm_gpu_scheduler, _p: PhantomData, } -// SAFETY: TODO -unsafe impl Sync for SchedulerInner {} -// SAFETY: TODO -unsafe impl Send for SchedulerInner {} - -/// Owner of an initialized DRM scheduler, shared by the scheduler handle and its entities. -/// -/// Jobs only reference the allocation, so that pending jobs cannot keep the scheduler from -/// being finalized. +// Only public scheduler/entity handles keep this owner alive. Jobs retain +// SchedulerInner instead, so outstanding work cannot prevent finalization. struct SchedulerOwner { allocation: Arc>, } impl Drop for SchedulerOwner { fn drop(&mut self) { - // SAFETY: The scheduler is valid. drm_sched_fini() stops the scheduler workers, then - // cancels and frees all remaining jobs through `cancel_job_cb()`. - unsafe { bindings::drm_sched_fini(self.allocation.sched.get()) }; + // SAFETY: fini stops submission, synchronizes the timeout worker and + // cancels/frees remaining jobs through our synchronized cancel callback. + // The C contract explicitly prohibits drm_sched_stop before fini. + // Deferred entity-kill callbacks are on a different workqueue and may + // survive this call. Their job guards retain allocation through cleanup. + unsafe { + bindings::drm_sched_fini(addr_of!(self.allocation.sched).cast_mut()) + }; } } +// SAFETY: TODO +unsafe impl Sync for SchedulerInner {} +// SAFETY: TODO +unsafe impl Send for SchedulerInner {} + /// A DRM Scheduler pub struct Scheduler(Arc>); @@ -444,16 +481,15 @@ impl Scheduler { ) -> Result> { let mut sched: UniqueArc>> = UniqueArc::new_uninit(GFP_KERNEL)?; - // Allocate the owner up front: nothing may fail once the scheduler is initialized. - let owner: UniqueArc>> = UniqueArc::new_uninit(GFP_KERNEL)?; - - // SAFETY: `sched` was just allocated and is valid for writes. - let sched_ptr = Opaque::cast_into(unsafe { addr_of!((*sched.as_mut_ptr()).sched) }); + // Allocate the owner before starting C workers: no fallible allocation + // may abandon an initialized scheduler without running drm_sched_fini. + let owner: UniqueArc>> = + UniqueArc::new_uninit(GFP_KERNEL)?; // SAFETY: zero sched->sched_rq as drm_sched_init() uses it to exit early withoput initialisation // TODO: allocate sched zzeroed instead unsafe { - (*sched_ptr).sched_rq = core::ptr::null_mut(); + (*sched.as_mut_ptr()).sched.sched_rq = core::ptr::null_mut(); }; let init_ops = bindings::drm_sched_init_args { @@ -471,9 +507,15 @@ impl Scheduler { // SAFETY: The drm_sched pointer is valid and pinned as it was just allocated above. // `device` is valid by its type invarants - to_result(unsafe { bindings::drm_sched_init(sched_ptr, addr_of!(init_ops)) })?; + to_result(unsafe { + bindings::drm_sched_init( + addr_of_mut!((*sched.as_mut_ptr()).sched), + addr_of!(init_ops), + ) + })?; - // SAFETY: All fields of SchedulerInner are now initialized. + // SAFETY: C initialized sched and PhantomData has no stored state. + // No fallible operation follows successful initialization. let allocation = unsafe { sched.assume_init() }.into(); Ok(Scheduler(owner.write(SchedulerOwner { allocation }).into())) } From 84f4e1fd0eaf13c985e6535d2906b68f1c519947 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 22:55:27 -0400 Subject: [PATCH 0631/1579] drm/asahi: correct reviewed lifetime, failure and submission paths Fix G17 descriptor fields, scratch state, prefix failure history, recovery evidence, retirement wakeups, queue admission and output-fence publication. Validate all timestamp destinations before enqueue and defer their cleanup onto a preallocated module-owned workqueue so fence producers do not take buffer reservations while destroying timestamp mappings. Correct shared GPUVM ownership and lock ordering, fixed-map rollback, page-table child unlinking, repeated-page metadata and legacy TLBI/bind validation. Balance legacy power/kick bookkeeping and propagate RTKit endpoint-start errors. Retain module code through DRM files, PRIME exports and detached scheduler callbacks, disable scheduler work at finalization, release the G17 shutdown DRM backedge and destroy initialized device data. The source-hashed integration snapshot passed all 64 local host validation commands and the shared Rust, C DRM/RTKit and Asahi module build. These checks use explicit host doubles and original-failure controls; they are not hardware, lockdep, module-unload or performance qualification. The public-base RTKit change is limited to the same STARTEP error propagation. Remaining timestamp publication ordering, deferred cleanup, legacy reference cycles and firmware-contract questions are tracked for follow-up. No queue depth reduction or serialization workaround is introduced. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit ca389fba9b6c7c9354c2f765fe81c5ef5f069f3b) --- drivers/gpu/drm/asahi/asahi.rs | 28 +- drivers/gpu/drm/asahi/cleanup.rs | 118 +++++++++ drivers/gpu/drm/asahi/crashdump.rs | 2 + drivers/gpu/drm/asahi/driver.rs | 1 + drivers/gpu/drm/asahi/file.rs | 18 +- drivers/gpu/drm/asahi/g17/channel.rs | 12 +- drivers/gpu/drm/asahi/g17/command.rs | 8 +- drivers/gpu/drm/asahi/g17/completion.rs | 78 ++++-- drivers/gpu/drm/asahi/g17/context.rs | 5 + drivers/gpu/drm/asahi/g17/fence.rs | 24 +- drivers/gpu/drm/asahi/g17/fw/compute.rs | 75 +++++- drivers/gpu/drm/asahi/g17/fw/kick.rs | 8 +- drivers/gpu/drm/asahi/g17/fw/render.rs | 29 +- drivers/gpu/drm/asahi/g17/initdata.rs | 38 ++- drivers/gpu/drm/asahi/g17/job.rs | 5 +- drivers/gpu/drm/asahi/g17/mod.rs | 43 ++- drivers/gpu/drm/asahi/g17/power.rs | 22 +- drivers/gpu/drm/asahi/g17/queue/compute.rs | 28 +- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 92 +++---- drivers/gpu/drm/asahi/g17/queue/mod.rs | 125 +++++---- drivers/gpu/drm/asahi/g17/queue/render.rs | 3 +- .../gpu/drm/asahi/g17/queue/render/memory.rs | 6 +- drivers/gpu/drm/asahi/g17/recovery.rs | 83 +++--- .../gpu/drm/asahi/g17/recovery/attribution.rs | 17 +- drivers/gpu/drm/asahi/g17/runtime.rs | 67 ++++- .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 53 ++-- .../drm/asahi/g17/runtime/compute_replay.rs | 22 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 16 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 28 +- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 3 +- .../drm/asahi/g17/runtime/render/prepare.rs | 23 +- .../drm/asahi/g17/runtime/render/submit.rs | 7 + drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 7 +- drivers/gpu/drm/asahi/g17/teardown.rs | 31 ++- drivers/gpu/drm/asahi/gem.rs | 23 +- drivers/gpu/drm/asahi/gpu.rs | 3 +- drivers/gpu/drm/asahi/hw/mod.rs | 9 +- drivers/gpu/drm/asahi/mem.rs | 7 +- drivers/gpu/drm/asahi/mmu.rs | 246 ++++++++++------- drivers/gpu/drm/asahi/mmu/bind.rs | 33 ++- drivers/gpu/drm/asahi/mmu/lifetime.rs | 47 +--- drivers/gpu/drm/asahi/pgtable.rs | 82 +++++- drivers/gpu/drm/asahi/queue/mod.rs | 17 +- drivers/gpu/drm/asahi/util.rs | 13 +- drivers/gpu/drm/drm_gpuvm.c | 8 +- drivers/gpu/drm/scheduler/sched_entity.c | 9 + drivers/gpu/drm/scheduler/sched_main.c | 9 + drivers/soc/apple/rtkit.c | 4 +- include/drm/drm_gpuvm.h | 1 + include/drm/gpu_scheduler.h | 8 + rust/kernel/drm/device.rs | 15 +- rust/kernel/drm/driver.rs | 4 + rust/kernel/drm/gem/mod.rs | 9 +- rust/kernel/drm/gpuvm.rs | 250 +++++++++++++----- rust/kernel/drm/sched.rs | 8 + 55 files changed, 1379 insertions(+), 551 deletions(-) create mode 100644 drivers/gpu/drm/asahi/cleanup.rs diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index 513bb6d103f65..d835c8e283666 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -6,6 +6,7 @@ mod alloc; mod buffer; mod channel; +mod cleanup; #[cfg(CONFIG_DEV_COREDUMP)] mod crashdump; mod debug; @@ -33,8 +34,31 @@ mod util; mod vm; mod workqueue; -kernel::module_platform_driver! { - type: driver::AsahiDriver, +use kernel::prelude::*; + +#[pin_data] +struct DriverModule { + // Field drop order unregisters/joins driver producers before queue teardown. + #[pin] + _driver: kernel::driver::Registration>, + _cleanup: cleanup::QueueOwner, +} + +impl kernel::InPlaceModule for DriverModule { + fn init(module: &'static kernel::ThisModule) -> impl PinInit { + try_pin_init!(Self { + // Initialize the queue first, before registration can invoke probe. + _cleanup: cleanup::QueueOwner::new()?, + _driver <- kernel::driver::Registration::new( + ::NAME, + module, + ), + }) + } +} + +module! { + type: DriverModule, name: "asahi", description: "AGX GPU driver for Apple silicon SoCs", license: "Dual MIT/GPL", diff --git a/drivers/gpu/drm/asahi/cleanup.rs b/drivers/gpu/drm/asahi/cleanup.rs new file mode 100644 index 0000000000000..e3dd5ad95dc0c --- /dev/null +++ b/drivers/gpu/drm/asahi/cleanup.rs @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Reservation-taking timestamp destruction must not block fence producers. +//! +//! The module owns this queue outside its platform registration. Files and +//! exported buffers retain the module; detached scheduler cleanup retains it +//! until its driver callback returns. Unregister joins the firmware/event work. +//! Only after those producer boundaries close may the module destroy this queue. + +use core::{ + ops::Deref, + ptr::{self, NonNull}, + sync::atomic::{AtomicPtr, Ordering}, +}; +use kernel::{ + bindings, c_str, impl_has_work, new_work, + prelude::*, + workqueue::{Queue, Work, WorkItem}, +}; + +static QUEUE: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + +/// Destroyed by module exit after platform-driver unregister, never by a worker. +pub(crate) struct QueueOwner(NonNull); + +// SAFETY: QueueOwner only transfers ownership of a thread-safe kernel queue. +unsafe impl Send for QueueOwner {} +// SAFETY: The queue API synchronizes all accesses; destruction requires ownership. +unsafe impl Sync for QueueOwner {} + +impl QueueOwner { + pub(crate) fn new() -> Result { + // SAFETY: The name is NUL terminated and contains no format specifiers. + let raw = unsafe { + bindings::alloc_workqueue_noprof( + c_str!("asahi-ts-cleanup").as_char_ptr(), + bindings::wq_flags_WQ_UNBOUND | bindings::wq_flags_WQ_MEM_RECLAIM, + 0, + ) + }; + let raw = NonNull::new(raw).ok_or(ENOMEM)?; + if QUEUE + .compare_exchange(ptr::null_mut(), raw.as_ptr(), Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + // SAFETY: This fresh queue has never been published or used. + unsafe { bindings::destroy_workqueue(raw.as_ptr()) }; + return Err(EBUSY); + } + Ok(Self(raw)) + } +} + +impl Drop for QueueOwner { + fn drop(&mut self) { + // No external producers remain: module registration was dropped first. + // Pending cleanup owns no queue reference and cannot enqueue more work. + // SAFETY: The module's unique owner retains the queue through its drain. + unsafe { bindings::destroy_workqueue(self.0.as_ptr()) }; + QUEUE.store(ptr::null_mut(), Ordering::Release); + } +} + +#[pin_data] +struct Item { + value: T, + #[pin] + work: Work, +} + +impl_has_work! { + impl{T: Send + 'static} HasWork> for Item { self.work } +} + +impl WorkItem for Item { + type Pointer = Pin>; + + fn run(this: Self::Pointer) { + drop(this); + } +} + +/// Allocate the destruction work before publication; final handoff cannot fail. +pub(crate) struct Deferred(Option>>>); + +impl Deferred { + pub(crate) fn new(value: T) -> Result { + let item = KBox::pin_init( + pin_init!(Item { + value, + work <- new_work!("Asahi timestamp cleanup"), + }), + GFP_KERNEL, + )?; + Ok(Self(Some(item))) + } +} + +impl Deref for Deferred { + type Target = T; + + fn deref(&self) -> &T { + &self.0.as_ref().unwrap().value + } +} + +impl Drop for Deferred { + fn drop(&mut self) { + if let Some(item) = self.0.take() { + let raw = QUEUE.load(Ordering::Acquire); + assert!(!raw.is_null()); + // SAFETY: Module ownership and producer joins retain the queue until + // this handoff has returned. The unique pinned box is never queued + // twice; enqueue transfers it without allocation or a failure path. + unsafe { Queue::from_raw(raw) }.enqueue(item); + } + } +} diff --git a/drivers/gpu/drm/asahi/crashdump.rs b/drivers/gpu/drm/asahi/crashdump.rs index bd9f2f1649584..148cae7aa9b46 100644 --- a/drivers/gpu/drm/asahi/crashdump.rs +++ b/drivers/gpu/drm/asahi/crashdump.rs @@ -40,6 +40,7 @@ pub(crate) struct AGXDumpInfo { gpu_rev: hw::GpuRevision, total_active_cores: u32, firmware_version: [u32; 6], + reserved: u32, } struct ELFNote { @@ -104,6 +105,7 @@ impl CrashDumpBuilder { gpu_rev: dyncfg.id.gpu_rev, total_active_cores: dyncfg.id.total_active_cores, firmware_version: [0; 6], + reserved: 0, initdata_address, }; info.firmware_version[..dyncfg.firmware_version.len().min(6)] diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index a40d40fa3090c..36c223b1d372d 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -75,6 +75,7 @@ impl drm::driver::Driver for AsahiDriver { type Object = drm::gem::shmem::Object; const INFO: drm::driver::DriverInfo = INFO; + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); const FEATURES: u32 = drm::driver::FEAT_GEM | drm::driver::FEAT_RENDER | drm::driver::FEAT_SYNCOBJ diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index a8d7a2db73e49..37eb832ceee37 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -178,11 +178,11 @@ impl SyncItem { in_count: u32, out_count: u32, ) -> Result> { - let count = in_count + out_count; - let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; + let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); - let size = STRIDE * count as usize; + let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; + let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; // SAFETY: We only read this once, so there are no TOCTOU issues. let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); @@ -655,7 +655,9 @@ impl File { let vm_id = data.vm_id.try_into()?; let mut vec = KVec::new(); - let size = (data.stride * data.num_binds) as usize; + let size = usize::try_from(data.stride)? + .checked_mul(usize::try_from(data.num_binds)?) + .ok_or(EOVERFLOW)?; let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); reader.read_all(&mut vec, GFP_KERNEL)?; let mut reader = Reader::new(&vec); @@ -761,6 +763,9 @@ impl File { data: &uapi::drm_asahi_gem_bind_op, file: &DrmFile, ) -> Result { + if data.range == 0 { + return Err(EINVAL); + } if (data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND) != 0 { Self::do_gem_unbind(vm_id, data, file) } else { @@ -1033,7 +1038,10 @@ impl File { .inner() .objects() .lock() - .insert_limit(1..=u32::MAX, obj, GFP_KERNEL)? as u64; + .insert_limit(1..=u32::MAX, obj, GFP_KERNEL); + // StoreError owns the rejected object. Convert it only after the + // temporary XArray guard is gone: dropping a mapping may sleep. + let handle = handle? as u64; data.object_handle = handle as u32; Ok(0) diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index a90693e824b0e..8e4740b76a166 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -9,9 +9,7 @@ //! empty slot and orders the record, power tally and producer together. use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; - -use crate::driver::AsahiDevice; +use kernel::{device::Device, prelude::*}; use super::fw::{channels::*, initdata::status}; use super::initdata::InitData; @@ -57,15 +55,15 @@ impl Rings { } } - pub(crate) fn drain_trace(&mut self, dev: &AsahiDevice, init: &InitData, role: Role) { + pub(crate) fn drain_trace(&mut self, dev: &Device, init: &InitData, role: Role) { self.drain_report(dev, init, role, ReportRing::Trace); } - pub(crate) fn drain_statistics(&mut self, dev: &AsahiDevice, init: &InitData, role: Role) { + pub(crate) fn drain_statistics(&mut self, dev: &Device, init: &InitData, role: Role) { self.drain_report(dev, init, role, ReportRing::Statistics); } - fn drain_report(&mut self, dev: &AsahiDevice, init: &InitData, role: Role, ring: ReportRing) { + fn drain_report(&mut self, dev: &Device, init: &InitData, role: Role, ring: ReportRing) { let quarantined = &mut self.quarantined[role as usize][ring as usize]; if *quarantined { return; @@ -176,7 +174,7 @@ impl Events { } } - pub(crate) fn next(&mut self, dev: &AsahiDevice, init: &InitData) -> Result> { + pub(crate) fn next(&mut self, dev: &Device, init: &InitData) -> Result> { let role = self.role; let state = status::EVENT_STATE as usize; let consumer_word = init.status_word(role, state)?; diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 47f9c0048e673..5929c14b264fd 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -3,6 +3,7 @@ //! Decode a submission and validate its user resources before publishing any work. use super::fw::{ + compute::ScratchRequest, kick::WriteRange, render::{RenderPass, UscProgram, ZlsSurface}, }; @@ -279,6 +280,7 @@ pub(crate) enum Validated { timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { + scratch: ScratchRequest, usc_base: u64, cdm_va: u64, cdm_end_va: u64, @@ -395,7 +397,11 @@ impl Payload { | (u64::from(cmd.bg_partial_rsrc_spec_hi) << 32), }, eot_offset: cmd.eot.usc, + eot_rsrc_spec: u64::from(cmd.eot.rsrc_spec) + | (u64::from(cmd.bg_eot_rsrc_spec_hi) << 32), partial_eot_offset: cmd.partial_eot.usc, + partial_eot_rsrc_spec: u64::from(cmd.partial_eot.rsrc_spec) + | (u64::from(cmd.bg_eot_partial_rsrc_spec_hi) << 32), rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, }; Ok(Validated::Render { @@ -406,7 +412,6 @@ impl Payload { } Self::Compute(cmd, attachments) => { if cmd.flags != 0 - || cmd.scs_layout != 0 || cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base @@ -423,6 +428,7 @@ impl Payload { sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; Ok(Validated::Compute { + scratch: ScratchRequest::new(cmd.scs_layout)?, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, cdm_end_va: cmd.cdm_ctrl_stream_end - 4, diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 777d54222c0ea..f137a7e3434de 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -24,8 +24,9 @@ use kernel::{ }; struct Destination { - mapping: Arc, - offset: usize, + // Keep the existing GPU alias alive along with its prepared CPU view. + _mapping: Arc, + cpu: mmu::TimestampMapping, } impl Destination { @@ -46,7 +47,8 @@ impl Destination { if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { return Err(ERANGE); } - Ok(Some(Self { mapping, offset })) + let cpu = mapping.timestamp_mapping(offset)?; + Ok(Some(Self { _mapping: mapping, cpu })) } fn write(&self, ticks: u64) -> Result { @@ -56,12 +58,45 @@ impl Destination { .wrapping_add( ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, ); - self.mapping.write_timestamp(self.offset, ns)?; + self.cpu.write(ns); barrier(Ordering::Release); Ok(()) } } +/// Timestamp mappings resolved and retained before any command is enqueued. +pub(crate) struct Destinations(Option; 4]>>); + +impl Destinations { + pub(crate) fn resolve( + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + ) -> Result { + if timestamps.is_empty() || timestamps.len() > 2 { + return Err(EINVAL); + } + let mut destinations = core::array::from_fn(|_| None); + for (index, timestamp) in timestamps.iter().enumerate() { + destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; + destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; + } + // Empty submissions need no cleanup allocation. Timestamped commands + // prepare the unique work item before any output fence is published. + let retained = if destinations.iter().any(Option::is_some) { + Some(crate::cleanup::Deferred::new(destinations)?) + } else { + None + }; + Ok(Self(retained)) + } + + fn iter(&self) -> core::slice::Iter<'_, Option> { + let destinations: &[Option] = + self.0.as_deref().map_or(&[], |destinations| destinations.as_slice()); + destinations.iter() + } +} + /// Firmware writes two aligned words in a fresh write-combined page per compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. @@ -87,7 +122,7 @@ pub(crate) struct Completion { status: Arc, fence: UserFence, member: Member, - destinations: [Option; 4], + destinations: Destinations, compute: Option, feed: Option>, #[pin] @@ -95,6 +130,8 @@ pub(crate) struct Completion { #[pin] render: Mutex>, signalled: AtomicBool, + retirement_queued: AtomicBool, + accepted: AtomicBool, timeout_error: AtomicI32, replays: AtomicU8, replay_timeout: AtomicBool, @@ -109,19 +146,10 @@ impl Completion { context: &Arc, member: Member, vm_job: mmu::VmJobGuard, - objects: Pin<&xarray::XArray>>, - timestamps: &[uapi::drm_asahi_timestamps], + destinations: Destinations, compute: bool, feed: Option>, ) -> Result> { - if timestamps.len() != if compute { 1 } else { 2 } { - return Err(EINVAL); - } - let mut destinations = core::array::from_fn(|_| None); - for (index, timestamp) in timestamps.iter().enumerate() { - destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; - destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; - } let fence = fence::independent(contexts)?; let compute = if compute { Some(ComputeTimestamps::new(dev, context.vm())?) @@ -139,6 +167,8 @@ impl Completion { vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), signalled: AtomicBool::new(false), + retirement_queued: AtomicBool::new(false), + accepted: AtomicBool::new(false), timeout_error: AtomicI32::new(0), replays: AtomicU8::new(0), replay_timeout: AtomicBool::new(false), @@ -152,6 +182,11 @@ impl Completion { pub(crate) fn fence(&self) -> Fence { Fence::from_fence(&self.fence) } + /// All fallible scheduler setup has succeeded. Failures from this point + /// must be visible through VM_STATUS before the accepted job's fence. + pub(crate) fn mark_accepted(&self) { + self.accepted.store(true, Ordering::Release); + } pub(crate) fn status(&self) -> &Arc { &self.status } @@ -229,6 +264,16 @@ impl Completion { Ok(()) } + /// A retirement witness has been accepted by an off-lock completion batch. + /// This is separate from fence signalling and from a failure still owned by firmware. + pub(crate) fn note_retirement_queued(&self) { + self.retirement_queued.store(true, Ordering::Release); + } + + pub(crate) fn retirement_queued(&self) -> bool { + self.retirement_queued.load(Ordering::Acquire) + } + /// The first timeout request retains its errno while the physical graph is /// borrowed. Duplicate watchdog or cancellation callbacks cannot replace it. pub(crate) fn defer_timeout(&self, error: Error) { @@ -342,6 +387,9 @@ impl Completion { return; } self.status.record_if_device_loss(error); + if self.accepted.load(Ordering::Acquire) { + self.status.report_failure(error); + } self.fence.set_error(error); self.release_healthy(); self.fence.signal(); diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 1a28386d788d0..ee2d8133ea8cd 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -213,6 +213,11 @@ impl Context { pub(crate) fn scheduler_owner(&self) -> Arc { self.scheduler.clone() } + /// Scheduler release and command retirement have both been witnessed. + /// Keep installed storage alive while returning the finite TTBAT identity. + pub(crate) fn release_execution(&self) { + self.execution.release(); + } pub(crate) fn scheduler_generation(&self) -> u32 { self.scheduler_generation } diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 1217d7e7b3f4e..3414e36db9d36 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -39,6 +39,15 @@ pub(crate) fn independent(contexts: &FenceContexts) -> Result Result { + let fence = independent(contexts)?; + fence.set_error(ENOMEM); + fence.signal(); + Ok(Fence::from_fence(&fence)) +} + /// All command members of an ioctl share its work-state node and aggregate fence. /// The initial member keeps the aggregate unsignalled until enqueue finishes. #[pin_data] @@ -165,14 +174,25 @@ pub(crate) struct Outputs { impl Outputs { pub(crate) fn new( contexts: &FenceContexts, - syncs: KVec, + mut syncs: KVec, input_count: usize, + allocation_failure: &Fence, ) -> Result { if input_count > syncs.len() { return Err(EINVAL); } + let fallback = match independent(contexts) { + Ok(fence) => fence, + Err(error) => { + // Syncobjs and timeline-chain storage are already resolved. + // No work has been accepted; publish the pre-signaled ENOMEM + // fence without another allocation before returning the error. + install(syncs.drain(input_count..), allocation_failure); + return Err(error); + } + }; Ok(Self { - fallback: independent(contexts)?, + fallback, syncs, input_count, submission: None, diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 22f8259508ff6..5b7489cfe93f4 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -24,6 +24,62 @@ pub(crate) const COMPUTE_QOS_CLASS: u8 = 0x08; pub(crate) const COMPUTE_KICK_EVENT_MASK: [u64; 4] = [0, 0, 4, 0]; const COMPUTE_REGISTERS: usize = 40; +const COMPUTE_REGISTERS_MAX: usize = COMPUTE_REGISTERS + 1; + +/// Validated per-command shader-context scratch request. +#[derive(Clone, Copy)] +pub(crate) struct ScratchRequest { + pages: u16, +} + +impl ScratchRequest { + const FIXED_BYTES: u64 = 0x58000; + const THREADS_PER_CLIQUE: u64 = 32; + const BYTES_PER_STACK_ENTRY: u64 = 8; + const PAGE_SHIFT: u32 = 12; + const PAGE_SIZE: u64 = 1 << Self::PAGE_SHIFT; + const PAGES_PER_MIB: u64 = 1 << (20 - Self::PAGE_SHIFT); + const WINDOW_ENABLED: u64 = 1 << 24; + + /// Payload-free layouts request no scratch even if depth/cliques are set. + /// Otherwise round the fixed and per-thread sections and add a guard page. + pub(crate) fn new(layout: u64) -> Result { + let depth = layout & u16::MAX as u64; + let cliques = (layout >> 24) & u8::MAX as u64; + let user = (layout >> 32) & u16::MAX as u64; + let driver = (layout >> 48) & u16::MAX as u64; + if user == 0 && driver == 0 { + return Ok(Self { pages: 0 }); + } + let threads = cliques.checked_mul(Self::THREADS_PER_CLIQUE).ok_or(EINVAL)?; + let per_thread = depth.checked_mul(Self::BYTES_PER_STACK_ENTRY) + .and_then(|bytes| bytes.checked_add(user)) + .and_then(|bytes| bytes.checked_add(driver)).ok_or(EINVAL)?; + let bytes = threads.checked_mul(per_thread) + .and_then(|bytes| bytes.checked_add(Self::FIXED_BYTES)) + .and_then(|bytes| bytes.checked_add(2 * Self::PAGE_SIZE - 1)) + .ok_or(EINVAL)?; + let pages = bytes >> Self::PAGE_SHIFT; + Ok(Self { pages: u16::try_from(pages).map_err(|_| EINVAL)? }) + } + + pub(crate) fn enabled(self) -> bool { + self.pages != 0 + } + + fn size_word(self) -> u64 { + let pages = u64::from(self.pages); + pages | (pages.div_ceil(Self::PAGES_PER_MIB) << 16) + } + + fn window_word(self) -> u64 { + if self.enabled() { Self::WINDOW_ENABLED } else { UNK_10791 } + } + + fn checkpoints(self) -> [u16; 4] { + CHECKPOINTS.map(|count| count + u16::from(self.enabled())) + } +} const ALT_REGISTERS: usize = 4; /// Object IDs of every compute kick. @@ -54,6 +110,8 @@ const CHECKPOINTS: [u16; 4] = [38, 39, 39, COMPUTE_REGISTERS as u16]; /// Inputs of one compute descriptor. pub(crate) struct ComputeArgs { + /// Scratch request validated with the userspace command. + pub(crate) scratch: ScratchRequest, /// Base of the queue's USC window. pub(crate) usc_base: u64, /// GPU address of the control stream. @@ -103,13 +161,14 @@ pub(crate) struct ComputeArgs { pub(crate) timestamp_va: u64, } -fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS]> { +fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS_MAX]> { let preempt = args.preempt_va; let operand = args.operand_state_va; let key = work_key(args.context_id, args.work_state); let ids = OBJECT_IDS.word(); Ok([ + RegisterWrite::new(0x017d9, 1), RegisterWrite::new(0x1a510, preempt), RegisterWrite::new(0x1a420, args.cdm_va), RegisterWrite::new(0x1a4d0, offset_va(preempt, PREEMPT_STATE[0])?), @@ -130,10 +189,10 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS RegisterWrite::new(0x1a061, 0), RegisterWrite::new(0x1a0b9, 0), RegisterWrite::new(0x1a0c1, 0), - RegisterWrite::new(0x101d1, 0), + RegisterWrite::new(0x101d1, args.scratch.size_word()), RegisterWrite::new(0x0d479, 0), RegisterWrite::new(0x1a0e9, UNK_1A0E9), - RegisterWrite::new(0x107a1, UNK_10791), + RegisterWrite::new(0x107a1, args.scratch.window_word()), RegisterWrite::new(0x0a599, UNK_0A599), RegisterWrite::new(0x0d411, UNK_0D411), RegisterWrite::new(0x1a540, ids), @@ -287,8 +346,11 @@ impl ComputeDescriptor { self.tag = CommandTag::Compute as u32; self.context_id = args.context_id.into(); self.work_state_va = args.work_state_va; - self.checkpoints = CHECKPOINTS; - self.registers.set(registers_va, ®isters)?; + self.checkpoints = args.scratch.checkpoints(); + // The enabled program prepends one register; disabled descriptors keep + // the same forty writes, trailer count and resume checkpoints. + let first = usize::from(!args.scratch.enabled()); + self.registers.set(registers_va, ®isters[first..])?; self.alt_registers.set(alt_registers_va, &alt_registers)?; self.preempt_va = args.preempt_va; self.cdm_end_va = args.cdm_end_va; @@ -321,10 +383,11 @@ impl ComputeDescriptor { /// the descriptor's address as seen by the work's VM. pub(crate) fn register_bindings( descriptor_va: u64, + scratch: ScratchRequest, ) -> Result<[Option; 4]> { let registers = RegisterArrayBinding::new( offset_va(descriptor_va, Self::REGISTERS_OFFSET)?, - CHECKPOINTS[0] as u8, + scratch.checkpoints()[0] as u8, )?; let alt_registers = RegisterArrayBinding::new( offset_va(descriptor_va, Self::ALT_REGISTERS_OFFSET)?, diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index c8b7d3f0dce3a..2e6133e882b2d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -369,6 +369,8 @@ pub(crate) struct KickArgs<'a> { pub(crate) event_mask: [u64; 4], /// Register arrays bound to the kick. pub(crate) register_arrays: [Option; 4], + /// The compute register program includes a shader-context scratch request. + pub(crate) compute_scratch: bool, } /// One kick ring entry. @@ -379,7 +381,7 @@ pub(crate) struct KickArgs<'a> { #[derive(Copy, Clone, Debug)] pub(crate) struct KickEntry { /// Timestamp in bits 41:2, QID in bits 48:42, MCache table present in bit - /// 58 and queue priority in bits 60:59. + /// 58, queue priority in bits 60:59 and compute scratch enabled in bit 61. pub(crate) header: u64, /// MCache table: address >> 5 in bits 43:6, count - 1 in bits 59:54. pub(crate) mcache: u64, @@ -419,6 +421,7 @@ impl KickEntry { const QID_SHIFT: u32 = 42; const MCACHE_PRESENT: u64 = 1 << 58; const PRIORITY_SHIFT: u32 = 59; + const COMPUTE_SCRATCH: u64 = 1 << 61; const QOS_SLOT_SHIFT: u32 = 32; const QOS_CLASS_SHIFT: u32 = 40; const QOS_CLASS_MAX: u8 = 0x1f; @@ -440,6 +443,9 @@ impl KickEntry { let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT | (args.qid as u64) << Self::QID_SHIFT | (QUEUE_PRIORITY as u64) << Self::PRIORITY_SHIFT; + if args.compute_scratch { + header |= Self::COMPUTE_SCRATCH; + } let mcache = match args.mcache { Some(table) => { header |= Self::MCACHE_PRESENT; diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 349d03ed14d71..e3da1b0e6d954 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -206,11 +206,14 @@ pub(crate) struct RenderPass { pub(crate) bg: UscProgram, /// Partial-render background program. pub(crate) partial_bg: UscProgram, - /// Tagged USC offset of the end-of-tile program. Its resource specifier - /// is always written as zero. + /// Tagged USC offset of the end-of-tile program. pub(crate) eot_offset: u32, + /// Resource specifier of the end-of-tile program. + pub(crate) eot_rsrc_spec: u64, /// Tagged USC offset of the partial-render end-of-tile program. pub(crate) partial_eot_offset: u32, + /// Resource specifier of the partial-render end-of-tile program. + pub(crate) partial_eot_rsrc_spec: u64, /// The resource specifiers have their high 32 bits. pub(crate) rsrc_spec_hi: bool, } @@ -383,7 +386,9 @@ struct PassWords { partial_bg_rsrc_spec: u64, partial_bg_va: u64, eot_va: u64, + eot_rsrc_spec: u64, partial_eot_va: u64, + partial_eot_rsrc_spec: u64, } impl PassWords { @@ -446,7 +451,9 @@ impl PassWords { partial_bg_rsrc_spec: rsrc_spec(pass.partial_bg.rsrc_spec), partial_bg_va: usc(pass.partial_bg.offset)?, eot_va: usc(pass.eot_offset)?, + eot_rsrc_spec: rsrc_spec(pass.eot_rsrc_spec), partial_eot_va: usc(pass.partial_eot_offset)?, + partial_eot_rsrc_spec: rsrc_spec(pass.partial_eot_rsrc_spec), }) } } @@ -580,7 +587,7 @@ fn fragment_registers( RegisterWrite::new(0x01739, 1), RegisterWrite::new(0x10009, w.utile_config), // End-of-tile resource specifier. - RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15379, w.eot_rsrc_spec), RegisterWrite::new(0x15381, w.eot_va), RegisterWrite::new(0x15369, w.bg_rsrc_spec), RegisterWrite::new(0x15371, w.bg_va), @@ -676,7 +683,7 @@ fn partial_store_registers( ) -> [RegisterWrite; PARTIAL_STORE_REGISTERS] { [ // End-of-tile resource specifier. - RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15379, w.partial_eot_rsrc_spec), RegisterWrite::new(0x15381, w.partial_eot_va), RegisterWrite::new(0x10039, w.tile_config), RegisterWrite::new(0x15359, UNK_15359_PARTIAL), @@ -701,7 +708,7 @@ fn partial_resume_registers( ) -> [RegisterWrite; PARTIAL_RESUME_REGISTERS] { [ // End-of-tile resource specifier. - RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15379, w.partial_eot_rsrc_spec), RegisterWrite::new(0x15381, w.partial_eot_va), RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), RegisterWrite::new(0x15371, w.partial_bg_va), @@ -1036,12 +1043,14 @@ pub(crate) struct FragmentDescriptor { pub(crate) unk_1f50: u64, /// Tile configuration. pub(crate) tile_config: u64, - /// Ends with the end-of-tile resource specifier, which is zero. - pub(crate) unk_1f60: [u8; 0x1c], + pub(crate) unk_1f60: [u8; 0x18], + /// Low word of the end-of-tile resource specifier. + pub(crate) eot_rsrc_spec: u32, /// GPU address of the end-of-tile program. pub(crate) eot_va: u64, - /// Ends with the partial end-of-tile resource specifier, which is zero. - pub(crate) unk_1f84: [u8; 0x18], + pub(crate) unk_1f84: [u8; 0x14], + /// Low word of the partial-render end-of-tile resource specifier. + pub(crate) partial_eot_rsrc_spec: u32, /// GPU address of the partial-render end-of-tile program. pub(crate) partial_eot_va: u64, pub(crate) unk_1fa4: u32, @@ -1228,7 +1237,9 @@ impl FragmentSlot { desc.height = pass.height; desc.unk_1f50 = UNK_15049; desc.tile_config = words.tile_config; + desc.eot_rsrc_spec = words.eot_rsrc_spec as u32; desc.eot_va = words.eot_va; + desc.partial_eot_rsrc_spec = words.partial_eot_rsrc_spec as u32; desc.partial_eot_va = words.partial_eot_va; desc.depth_clear = pass.depth_clear; desc.unk_1fac = words.tib_blocks << Self::UNK_1FAC_TIB_SHIFT | Self::UNK_1FAC_BASE; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 22b67575c133c..aea894ba2c2d8 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -413,6 +413,15 @@ impl InitData { Ok(()) } + /// The sole host publisher holds the device mutex through the following + /// inner and outer publication. Firmware consumers can only free capacity; + /// neither this check nor waiting for room reserves or limits other queues. + pub(super) fn work_ready(&self, engine: super::fw::queue::DataMaster) -> Result { + let [consumer, consumer_b, producer] = self.work_cursors(engine)?; + let next = (producer + 1) % super::fw::channels::WORK_RING_SLOTS; + Ok(next != consumer && next != consumer_b) + } + fn work_publication<'a>( &'a self, record: &'a super::fw::channels::WorkSlot, @@ -1231,14 +1240,14 @@ impl super::recovery::Memory for InitData { .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - let key = super::recovery::slot_key(slot).unwrap_or(0); + let key = super::recovery::slot_key(slot)?; let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) - .map(|words| words[0]).unwrap_or(0); - (key, monitor) + .ok().map(|words| words[0])?; + Some((key, monitor)) }); sources.reason = self.recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) - .map(|words| words[0]).unwrap_or(0); + .ok().map(|words| words[0]); Ok(sources) } @@ -1290,30 +1299,17 @@ impl super::recovery::Memory for InitData { Ok(()) } - fn entries( - &self, - out: &mut [super::recovery::Entry; super::recovery::MAX_ENTRIES], - ) -> Result { + fn entries(&self) -> Result { let cluster = self.object(self.cluster)?; let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; fence(Ordering::Acquire); - let mut used = 0; + let mut entries = super::recovery::Entries::default(); for index in 0..256 { - if used == out.len() { - break; - } let offset = base + index * core::mem::size_of::(); let flags = cluster.word(offset)?.load(Ordering::Relaxed); - if flags & 1 == 0 { - continue; - } - let stamp = cluster.word(offset + 4)?.load(Ordering::Relaxed); - if let Some(entry) = super::recovery::Entry::decode(flags, stamp) { - out[used] = entry; - used += 1; - } + entries.include(flags); } - Ok(used) + Ok(entries) } fn wait_tick(&self) { diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index c80955527cc06..b25db8e00f167 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -13,7 +13,7 @@ use kernel::{ sync::Arc, }; -const TIMEOUT_MS: kernel::time::Msecs = 2000; +pub(super) const TIMEOUT_MS: kernel::time::Msecs = 2000; const RENDER_CREDITS: u32 = 32; const COMPUTE_CREDITS: u32 = 16; @@ -272,6 +272,8 @@ impl Drop for Job { } impl sched::JobImpl for Job { + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + fn false_timeout(job: &mut sched::Job) -> bool { if job.is_finished() || job.packet.completion.take_replay_timeout() { return true; @@ -351,6 +353,7 @@ impl Scheduler { job.add_dependency(dependency)?; } let mut job = job.arm(); + packet.completion.mark_accepted(); let ready = if host_timestamps { completed.clone() } else { diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index a5c867409d379..6f75dfb5822c2 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -56,7 +56,7 @@ use core::sync::atomic::{ use kernel::{ c_str, - device::Core, + device::{self, Core}, drm::gem::shmem, impl_has_delayed_work, impl_has_work, iosys_map::IoSysMapRef, @@ -64,6 +64,7 @@ use kernel::{ prelude::*, soc::apple::rtkit, sync::{ + aref::ARef, Arc, CondVar, Mutex, // @@ -466,6 +467,7 @@ struct Firmware { rings: channel::Rings, recovery: recovery::State, effort: power::Effort, + registration_power_release_pending: bool, queues: runtime::Registry, pool: Arc, render_ids: Arc, @@ -569,6 +571,12 @@ impl power::Control for Firmware { fn wait_tick(&self) { fsleep(Delta::from_millis(1)); } + fn registration_release_failed(&mut self) { + self.registration_power_release_pending = true; + } + fn registration_release_succeeded(&mut self) { + self.registration_power_release_pending = false; + } } /// Event work item ID. @@ -583,7 +591,12 @@ const FEED_WORK: u64 = 5; /// Device state shared with the coprocessor callbacks and the event worker. #[pin_data] struct Shared { - dev: AsahiDevRef, + /// Diagnostics may outlive shutdown without retaining the DRM data graph. + dev: ARef, + /// Allocation users take independent references before reserving resources. + /// Shutdown removes this owning backedge after joining device work. + #[pin] + drm: Mutex>, clusters: u32, descriptor_flags: [u32; 2], crashed: AtomicBool, @@ -604,7 +617,11 @@ struct Shared { /// Nests inside the device mutex; detached mappings are dropped after both unlock. #[pin] reclaim: Mutex, - /// The device lock. `None` before boot completes and after teardown. + /// Serializes detached firmware destruction with unbind. Stop paths only; + /// normal publication and RTKit/fence handlers never acquire this lock. + #[pin] + stop: Mutex<()>, + /// The device lock. `None` before boot completes and after detachment. #[pin] state: Mutex>>, #[pin] @@ -687,6 +704,17 @@ impl WorkItem for Shared { } impl Shared { + fn drm(&self) -> Result { + self.drm.lock().as_ref().cloned().ok_or(ENODEV) + } + + fn release_drm(&self) { + // The final put may call device-release callbacks. Never run those with + // the reference-slot mutex held. + let retired = self.drm.lock().take(); + drop(retired); + } + /// Queues the event worker. Callable from any context. fn queue_events(self: &Arc) { // Already queued if this fails, which is just as good. @@ -849,7 +877,8 @@ impl Gpu { let feed = feed::Feed::new()?; let shared = Arc::pin_init( pin_init!(Shared { - dev: dev.into(), + drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), + dev: dev.as_ref().into(), clusters: id.num_clusters, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), @@ -862,6 +891,7 @@ impl Gpu { grow <- new_work!("g17::Shared::grow"), reclaim_work <- new_work!("g17::Shared::reclaim_work"), reclaim <- new_mutex!(reclaim, "g17::Shared::reclaim"), + stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), }), @@ -948,6 +978,7 @@ impl Gpu { rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), + registration_power_release_pending: false, queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), pool: object::Pool::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, @@ -1025,6 +1056,7 @@ impl Gpu { self.shared.disable_work(&self.shared.feed_work); self.shared.feed.shutdown(); self.shared.drain_reclaims(); + self.shared.release_drm(); } } @@ -1129,6 +1161,7 @@ impl gpu::Gpu for Gpu { priority: u32, usc_exec_base: u64, ) -> Result> { + let dev = self.shared.drm()?; let owner = self.ids.queue.next(); let frontend = queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; @@ -1139,7 +1172,7 @@ impl gpu::Gpu for Gpu { } }; let frontend = match queue::Queue::new( - &self.shared.dev, + &dev, backend.context.clone(), backend.clone(), frontend, diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 8806f438234e9..96b363bc27447 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -70,6 +70,8 @@ pub(super) trait Control { fn counters(&self) -> kernel::error::Result<[u32; 3]>; fn set_idle(&mut self, enabled: bool) -> kernel::error::Result; fn wait_tick(&self); + fn registration_release_failed(&mut self) {} + fn registration_release_succeeded(&mut self) {} } fn wait(control: &C, target: u32, require_powered: bool) -> kernel::error::Result { @@ -113,13 +115,25 @@ pub(super) fn acquire(control: &mut C) -> kernel::error::Result { } Err(EAGAIN) })(); - if result.is_err() { - let _ = release(control); + if result.is_err() && release(control).is_err() { + control.registration_release_failed(); } result } pub(super) fn release(control: &mut C) -> kernel::error::Result { - let target = control.set_idle(true)?; - wait(control, target, false) + let target = match control.set_idle(true) { + Ok(target) => target, + Err(error) => { + control.registration_release_failed(); + return Err(error); + } + }; + let result = wait(control, target, false); + if result.is_ok() { + control.registration_release_succeeded(); + } else { + control.registration_release_failed(); + } + result } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 3b259b37b38f4..5ca9b85f8aeb6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -313,7 +313,7 @@ struct Active { ticket: Option, observed: Option, failure_deferred: bool, - qos_completed: bool, + qos_pending: bool, } struct Previous { binding: Option, @@ -511,6 +511,12 @@ impl Queue { data_master: DataMaster::Compute as u8, } } + /// A completion hint can require a second visibility/recordless pass even + /// when firmware has no more events to send. Do not poll idle or failed queues. + pub(crate) fn retirement_polling(&self) -> bool { + !self.quarantined && self.retire_pending + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { self.retired_by_teardown || self.retire_pending @@ -681,9 +687,9 @@ impl Queue { match result { Ok(Some(timestamps)) => { let submitted = self.submitted.checked_sub(1).ok_or(EIO)?; - if !front.qos_completed { + if front.qos_pending { complete_qos(self.qos_owner())?; - self.active.iter_mut().next().ok_or(EIO)?.qos_completed = true; + self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; } let front = self.active.front().ok_or(EIO)?; self.retire_pending = true; @@ -783,9 +789,9 @@ impl Queue { self.retirement_proved = true; } while let Some(front) = self.active.front() { - if self.spared_quarantine && !front.qos_completed { + if front.qos_pending { complete_qos(self.qos_owner())?; - self.active.iter_mut().next().ok_or(EIO)?.qos_completed = true; + self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; } let front = self.active.front().ok_or(EIO)?; let completion = front.packet.completion.clone(); @@ -1242,6 +1248,7 @@ impl Queue { return Err(EIO); } let crate::g17::command::Validated::Compute { + scratch, usc_base, cdm_va, cdm_end_va, @@ -1275,6 +1282,7 @@ impl Queue { self.graph.ensure_record_slot(slot)?; } let args = ComputeArgs { + scratch: *scratch, usc_base: *usc_base, cdm_va: *cdm_va, cdm_end_va: *cdm_end_va, @@ -1345,7 +1353,8 @@ impl Queue { }, mcache: None, event_mask: COMPUTE_KICK_EVENT_MASK, - register_arrays: ComputeDescriptor::register_bindings(descriptor_low)?, + register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch)?, + compute_scratch: scratch.enabled(), }; let published = host.next_compute_publication()?; self.active @@ -1356,7 +1365,7 @@ impl Queue { ticket: None, observed: None, failure_deferred: false, - qos_completed: false, + qos_pending: false, }) .map_err(|_| EBUSY)?; self.retirement_ready = false; @@ -1411,8 +1420,10 @@ impl Queue { }, identity.scheduler, )?; + self.active.back_mut().ok_or(EIO)?.qos_pending = true; let target = self.pool.reserve_submission().or_else(|error| { host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; Err(error) })?; producer = self @@ -1421,6 +1432,7 @@ impl Queue { .or_else(|error| { self.pool.cancel_unpublished(target)?; host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; Err(error) })?; } else { @@ -1443,11 +1455,13 @@ impl Queue { }, identity.scheduler, )?; + self.active.back_mut().ok_or(EIO)?.qos_pending = true; producer = self .graph .append(producer, config_va, || {}) .or_else(|error| { host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; Err(error) })?; } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 1ddadae2dba69..e420427d594bf 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -9,46 +9,17 @@ use kernel::{prelude::*, sync::Arc}; pub(crate) const POLL_NS: u64 = 2_500_000_000; const QUARANTINED_DELAY_NS: u64 = 250_000_000; const CANCELLED_DELAY_NS: u64 = 2_000_000_000; -const ACK_SLOTS: usize = 16; // Firmware-visible cookie namespace persists for the loaded module's lifetime. static NEXT_COOKIE: AtomicU32 = AtomicU32::new(1); -pub(crate) fn next_cookie() -> u64 { - loop { - let cookie = NEXT_COOKIE.fetch_add(1, Ordering::Relaxed); - if cookie != 0 { - return cookie.into(); - } - } -} - -pub(crate) struct Acks([u64; ACK_SLOTS]); -impl Acks { - pub(crate) fn new() -> Self { - Self([0; ACK_SLOTS]) - } - pub(crate) fn note(&mut self, cookie: u64) { - if cookie == 0 || self.contains(cookie) { - return; - } - if let Some(slot) = self.0.iter_mut().find(|slot| **slot == 0) { - *slot = cookie; - } - } - pub(crate) fn contains(&self, cookie: u64) -> bool { - cookie != 0 && self.0.contains(&cookie) - } - pub(crate) fn forget(&mut self, cookie: u64) { - for slot in &mut self.0 { - if *slot == cookie { - *slot = 0; - } - } - } - pub(crate) fn snapshot(&self) -> [u64; ACK_SLOTS] { - self.0 - } +pub(crate) fn next_cookie() -> Result { + // Never reuse an identity while a delayed firmware ACK can still exist. + // The final counter value is an exhaustion marker, not a wrapping cookie. + NEXT_COOKIE + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |next| next.checked_add(1)) + .map(u64::from) + .map_err(|_| EOVERFLOW) } pub(crate) struct State { @@ -101,6 +72,7 @@ pub(super) struct Transaction { cookie: u64, published: u64, phase: Phase, + announced: Option, } impl Queue { @@ -136,32 +108,50 @@ impl Queue { cookie, published: now, phase: Phase::Published, + announced: None, }); } + /// A failed doorbell does not undo the published kill or consume its retry. + /// The caller retries under the device mutex; only the original record exists. + pub(crate) fn announce_exit(&mut self, now: u64, notify: impl FnOnce() -> Result) -> Result { + let Some(exit) = self.exit.as_mut() else { + return Ok(()); + }; + if exit.phase != Phase::Published || exit.announced.is_some() { + return Ok(()); + } + notify()?; + exit.announced = Some(now); + Ok(()) + } pub(crate) fn exit_polling(&self, now: u64) -> bool { self.exit.as_ref().is_some_and(|exit| { - exit.phase == Phase::Published && now.saturating_sub(exit.published) < POLL_NS + exit.phase == Phase::Published + && now.saturating_sub(exit.announced.unwrap_or(exit.published)) < POLL_NS }) } - pub(crate) fn exit_cookie(&self) -> Option { - self.exit - .as_ref() - .filter(|exit| matches!(exit.phase, Phase::Published | Phase::Acknowledged)) - .map(|exit| exit.cookie) + /// Event consumption and record_exit are serialized by the device mutex. + /// A kill pins this exact queue transaction through its final witness; + /// duplicate, unknown and terminal cookies cannot acknowledge another owner. + pub(crate) fn acknowledge_exit(&mut self, cookie: u64) -> bool { + let Some(exit) = self.exit.as_mut() else { + return false; + }; + if cookie == 0 || exit.cookie != cookie || exit.phase != Phase::Published { + return false; + } + exit.phase = Phase::Acknowledged; + true } pub(crate) fn settle_exit( &mut self, - acks: &Acks, recovery_closed: bool, + complete_qos: &mut impl FnMut(crate::g17::qos::Owner) -> Result, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { - let Some(cookie) = self.exit_cookie() else { - return Ok(()); - }; - if !acks.contains(cookie) { + if !self.exit.as_ref().is_some_and(|exit| exit.phase == Phase::Acknowledged) { return Ok(()); } - self.exit.as_mut().ok_or(EIO)?.phase = Phase::Acknowledged; if self.active.len != 0 { if !recovery_closed { return Ok(()); @@ -185,6 +175,11 @@ impl Queue { } } while let Some(active) = self.active.front() { + if active.qos_pending { + complete_qos(self.qos_owner())?; + self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; + } + let active = self.active.front().ok_or(EIO)?; defer(Deferred::Retired( active.packet.completion.clone(), Err(if self.spared_deferred { @@ -202,6 +197,7 @@ impl Queue { self.spared_quarantine = false; self.retire_pending = false; self.retirement_ready = false; + self.update_slot_accounting(); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 9d2bc056cce65..78765523254b6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -8,7 +8,7 @@ pub(crate) mod render; use super::{ command::{Parser, UscWindow, Validated}, - completion::Completion, + completion::{Completion, Destinations}, context::Context, fence::{Outputs, Submission}, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, @@ -21,7 +21,7 @@ use crate::{ use core::ops::Range; use kernel::{ c_str, - dma_fence::{FenceContexts, RawDmaFence}, + dma_fence::{Fence, FenceContexts, RawDmaFence}, prelude::*, sync::{Arc, LockClassKey}, xarray, @@ -53,6 +53,7 @@ impl Identities { /// Validated frontend state prepared before allocating a logical execution context. pub(crate) struct Frontend { fences: FenceContexts, + allocation_failure: Fence, range: Range, window: UscWindow, } @@ -64,8 +65,10 @@ impl Frontend { } let window = UscWindow::new(usc_base, range.clone())?; let fences = FenceContexts::new(1, c_str!("asahi_g17_queue"), FENCE_KEY)?; + let allocation_failure = super::fence::allocation_failure(&fences)?; Ok(Self { fences, + allocation_failure, range, window, }) @@ -79,11 +82,13 @@ pub(crate) struct Queue { render: Option>, compute: Option>, fences: FenceContexts, + allocation_failure: Fence, range: Range, window: UscWindow, proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], + failures: [Option<(u64, Fence)>; 2], // Closing the userspace queue releases admission even if firmware still // retains one of its execution contexts or installed physical graphs. _slot: admission::QueueSlot, @@ -99,6 +104,7 @@ impl Queue { ) -> Result { let Frontend { fences, + allocation_failure, range, window, } = frontend; @@ -110,11 +116,13 @@ impl Queue { render, compute: None, fences, + allocation_failure, range, window, proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), + failures: [None, None], _slot: slot, }) } @@ -132,6 +140,22 @@ impl Queue { Ok(()) } + fn prune_frontier(&mut self, index: usize) { + let failure = &mut self.failures[index]; + self.frontiers[index].retain(|(sequence, fences)| { + let status = job::fence_status(&fences.completed); + if status == 0 { + return true; + } + // One earliest failed ordinal covers every later prefix without + // retaining its packet, mappings, or every failed fence. + if status < 0 && failure.as_ref().is_none_or(|(first, _)| *sequence < *first) { + *failure = Some((*sequence, fences.completed.clone())); + } + false + }); + } + fn enqueue( &mut self, id: u64, @@ -140,62 +164,30 @@ impl Queue { guard: mmu::VmJobGuard, aggregate: &Arc, inputs: &[file::SyncItem], - objects: Pin<&xarray::XArray>>, + destinations: Destinations, ) -> Result { let engine = match &command { Validated::Render { .. } => Engine::Render, Validated::Compute { .. } => Engine::Compute, }; let index = engine.index(); - // One earliest failed timestamp covers every prefix containing a later - // failure, without retaining the producer's packet or mappings. - let mut retained_timestamp_failure = false; - self.frontiers[index].retain(|(_, fences)| { - let status = job::fence_status(&fences.completed); - if status == 0 { - return true; - } - if engine == Engine::Compute - && status < 0 - && fences.ready.raw() == fences.completed.raw() - && !retained_timestamp_failure - { - retained_timestamp_failure = true; - return true; - } - false - }); + self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { self.ensure_compute()?; } - let completion = match &command { - Validated::Render { timestamps, .. } => Completion::new( - &self.dev, - &self.fences, - &self.context, - aggregate.member(), - guard, - objects, - timestamps, - false, - self.backend.feed(), - )?, - Validated::Compute { timestamps, .. } => Completion::new( - &self.dev, - &self.fences, - &self.context, - aggregate.member(), - guard, - objects, - core::slice::from_ref(timestamps), - true, - None, - )?, - }; + let completion = Completion::new( + &self.dev, + &self.fences, + &self.context, + aggregate.member(), + guard, + destinations, + engine == Engine::Compute, + if engine == Engine::Render { self.backend.feed() } else { None }, + )?; if engine == Engine::Compute { - self.frontiers[Engine::Render.index()] - .retain(|(_, fences)| job::fence_status(&fences.completed) == 0); + self.prune_frontier(Engine::Render.index()); } let other = 1 - index; let prefix = order.wait_through[other]; @@ -222,10 +214,21 @@ impl Queue { .iter() .filter(|(_, fences)| fences.ready.raw() == fences.completed.raw()) .count(); + let failures: [Option<&Fence>; 2] = core::array::from_fn(|index| { + self.failures[index].as_ref().and_then(|(sequence, fence)| { + Order::contains(order.wait_through[index], *sequence).then_some(fence) + }) + }); + let failure_count = failures.iter().flatten().count(); let mut dependencies = - KVec::with_capacity(inputs.len() + count + timestamp_count, GFP_KERNEL)?; - let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count, GFP_KERNEL)?; + KVec::with_capacity(inputs.len() + count + timestamp_count + failure_count, GFP_KERNEL)?; + let mut checked_inputs = + KVec::with_capacity(inputs.len() + timestamp_count + failure_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; + for fence in failures.into_iter().flatten() { + checked_inputs.push(fence.clone(), GFP_KERNEL)?; + dependencies.push(fence.clone(), GFP_KERNEL)?; + } // Same-engine parents order GPU execution, not the host's timestamp // writes. Only timestamp publishers need these completion waits and // status checks; ordinary compute keeps its existing queue ordering. @@ -285,7 +288,8 @@ impl Queue { fn prepare_submission( &mut self, bytes: &[u8], - ) -> Result<(KVec<(Validated, Order)>, KVec)> { + objects: Pin<&xarray::XArray>>, + ) -> Result<(KVec<(Validated, Order, Destinations)>, KVec)> { let parse_guard = self.context.vm().retain_first_job()?; let mut parser = Parser::new(bytes); // All historical ordinals use the same entry counters; publication @@ -312,7 +316,13 @@ impl Queue { engine, )?; preceding[engine.index()] += 1; - pending.push((payload, order), GFP_KERNEL)?; + let destinations = match &payload { + Validated::Render { timestamps, .. } => Destinations::resolve(objects, timestamps)?, + Validated::Compute { timestamps, .. } => { + Destinations::resolve(objects, core::slice::from_ref(timestamps))? + } + }; + pending.push((payload, order, destinations), GFP_KERNEL)?; } parser.finish()?; // Every accepted command pins the VM before any command is enqueued. @@ -338,15 +348,20 @@ impl crate::queue::Queue for Queue { bytes: &[u8], objects: Pin<&xarray::XArray>>, ) -> Result { - let mut outputs = Outputs::new(&self.fences, syncs, input_count)?; + let mut outputs = Outputs::new( + &self.fences, + syncs, + input_count, + &self.allocation_failure, + )?; let result = (|| { if self.context.status().get() != 0 { return Err(EIO); } - let (pending, guards) = self.prepare_submission(bytes)?; + let (pending, guards) = self.prepare_submission(bytes, objects)?; let mut guards = guards.into_iter(); let aggregate = outputs.publish(self.context.clone())?; - for (command, order) in pending { + for (command, order, destinations) in pending { let guard = guards.next().ok_or(EIO)?; self.enqueue( id, @@ -355,7 +370,7 @@ impl crate::queue::Queue for Queue { guard, &aggregate, outputs.inputs(), - objects, + destinations, )?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index c3ac091333c27..fd67523ab91c3 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -512,6 +512,7 @@ impl Pair { mcache, event_mask: abi::RENDER_KICK_EVENT_MASK, register_arrays: arrays, + compute_scratch: false, }; let ta_args = args( 0, @@ -789,7 +790,7 @@ impl Pair { } pub(crate) fn needs_drain(&self) -> bool { - self.quarantined && self.in_flight() + self.quarantined } pub(crate) fn oldest_spared(&self) -> Option { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index f614494e1216a..859ff61901570 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -47,12 +47,14 @@ impl Aliases { let mut range = dynamic::LOWER; let size = (source.end - source.start) as u64; loop { + // Descriptor aliases are read-only to GPU consumers; the canonical + // firmware mapping remains writable for command state updates. let global = object.map_range( alloc.uat.kernel_lower_vm(), source.clone(), range.clone(), mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, + mmu::PROT_GPU_SHARED_RO, 0, )?; let address = global.iova(); @@ -62,7 +64,7 @@ impl Aliases { source.clone(), address..end, mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, + mmu::PROT_GPU_SHARED_RO, 0, ) { Ok(client) => { diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 7e3db2f6dae78..ba368341d84d2 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -22,29 +22,30 @@ use kernel::{prelude::*, sync::Arc}; const QIDS: u8 = 128; const GUILTY: u8 = 1 << 1; const CANCELLED: u8 = 1 << 2; -/// Retained valid entries, in firmware table order. -pub(super) const MAX_ENTRIES: usize = 64; const MAX_GUILTY_VMS: usize = 8; const RECOVERY_POLLS: usize = 500; -/// One valid entry from the firmware's 256-entry recovery table. -#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] -pub(super) struct Entry { - pub(super) qid: u8, - pub(super) flags: u8, - pub(super) stamp: u32, +/// Per-QID classifications from the complete firmware recovery table. +/// Multiple records for a queue contribute all of their classification bits. +#[derive(Default)] +pub(super) struct Entries { + guilty: u128, + cancelled: u128, } -impl Entry { - pub(super) const fn decode(flags: u32, stamp: u32) -> Option { +impl Entries { + pub(super) fn include(&mut self, flags: u32) { if flags & 1 == 0 { - return None; + return; + } + let qid = (flags >> 4) & 0x7f; + let bit = 1u128 << qid; + if flags & u32::from(GUILTY) != 0 { + self.guilty |= bit; + } + if flags & u32::from(CANCELLED) != 0 { + self.cancelled |= bit; } - Some(Self { - qid: ((flags >> 4) & 0x7f) as u8, - flags: (flags & 0xf) as u8, - stamp: stamp >> 8, - }) } } @@ -83,7 +84,7 @@ pub(super) trait Memory { fn fault_sources(&self) -> Result; fn transition(&self, expected: u32, next: u32) -> Result; fn clear_timestamps(&self) -> Result; - fn entries(&self, out: &mut [Entry; MAX_ENTRIES]) -> Result; + fn entries(&self) -> Result; /// One millisecond wait; this is polled because the firmware sends no state-3 notification. fn wait_tick(&self); } @@ -163,10 +164,31 @@ pub(super) fn service( crashed: impl Fn() -> bool, ) -> Result { let request = host.recovery().pending.ok_or(EINVAL)?; - if host.recovery_state()? != 1 - || request.generation != host.recovery().accepted - || host.host_recovery()? != 0 - { + if request.generation != host.recovery().accepted || host.host_recovery()? != 0 { + return Err(EIO); + } + // The restart record is published before firmware announces state 1. + // Its ring cursor is already consumed; wait only for that readiness word, + // without advancing the host generation or acknowledging an early event. + let mut ready = false; + for _ in 0..RECOVERY_POLLS { + if crashed() { + return Err(EIO); + } + match host.recovery_state()? { + 1 => { + ready = true; + break; + } + 0 => host.wait_tick(), + _ => return Err(EIO), + } + } + if !ready { + return Err(ETIMEDOUT); + } + // Recheck ownership after the off-CPU waits, before any handshake mutation. + if host.host_recovery()? != 0 { return Err(EIO); } host.recovery_mut().accepted = host.recovery().accepted.wrapping_add(1); @@ -190,16 +212,15 @@ pub(super) fn service( if !recovered { return Err(ETIMEDOUT); } - let mut entries = [Entry::default(); MAX_ENTRIES]; // A failed table read leaves only the explicit blamed QID as evidence of guilt. - let count = host.entries(&mut entries).unwrap_or(0).min(MAX_ENTRIES); + let entries = host.entries().unwrap_or_default(); host.transition(3, 0)?; if host.recovery_state()? != 0 { return Err(EIO); } host.recovery_mut().pending = None; - let cancelled = classify(host, &entries[..count], request.blamed, &mut sources); + let cancelled = classify(host, entries, request.blamed, &mut sources); if let Some(qid) = request.blamed { host.settle_blamed_render(qid, EIO); } @@ -214,17 +235,9 @@ pub(super) fn service( /// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or /// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work /// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. -fn classify(queues: &mut Q, entries: &[Entry], blamed: Option, sources: &mut Sources) -> u128 { - let mut guilty_qids = blamed.map_or(0, |qid| 1u128 << qid); - let mut cancelled = 0u128; - for entry in entries { - if entry.flags & GUILTY != 0 { - guilty_qids |= 1u128 << entry.qid; - } - if entry.flags & CANCELLED != 0 { - cancelled |= 1u128 << entry.qid; - } - } +fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> u128 { + let mut guilty_qids = entries.guilty | blamed.map_or(0, |qid| 1u128 << qid); + let cancelled = entries.cancelled; let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); match attribution { Attribution::Firmware => {} diff --git a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs index b0c42d4e3f7c1..1755484219187 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs @@ -16,7 +16,7 @@ pub(super) struct Source { } pub(crate) struct Sources { - pub(crate) reason: u32, + pub(crate) reason: Option, mask: u32, records: [Source; SLOTS], count: usize, @@ -31,10 +31,10 @@ pub(super) enum Attribution { impl Sources { /// Firmware is halted for every selected key and progress-record read. - /// Missing keys stay unknown; they cannot establish execution by an owner. - pub(crate) fn sample(mask: u32, mut read: impl FnMut(usize) -> (u64, u32)) -> Self { + /// Failed reads stay unknown; incomplete evidence cannot override firmware blame. + pub(crate) fn sample(mask: u32, mut read: impl FnMut(usize) -> Option<(u64, u32)>) -> Self { let mut sources = Self { - reason: 0, + reason: None, mask, records: [Source::default(); SLOTS], count: 0, @@ -43,7 +43,9 @@ impl Sources { if mask & (1 << slot) == 0 { continue; } - let (key, monitor) = read(slot); + let Some((key, monitor)) = read(slot) else { + continue; + }; let qid = (key >> 40) & 0xff; sources.records[sources.count] = Source { slot: slot as u8, @@ -62,12 +64,15 @@ impl Sources { blamed: Option, mut started: impl FnMut(u8, u64) -> Option, ) -> Attribution { + if self.reason.is_none() || self.count as u32 != self.mask.count_ones() { + return Attribution::Firmware; + } for source in &mut self.records[..self.count] { if let Some(qid) = source.qid { source.started = started(qid, source.stamp); } } - if self.reason == PAGE_FAULT || self.mask.count_ones() < 2 { + if self.reason == Some(PAGE_FAULT) || self.mask.count_ones() < 2 { return Attribution::Firmware; } let Some(blamed) = blamed else { diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 294d66b91f8dc..805087d7082ab 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -29,6 +29,7 @@ use kernel::{prelude::*, sync::Arc}; const QIDS: usize = 128; const FIRST_COMPUTE_QID: u8 = 4; const RECOVERY_SETTLE_POLLS: usize = 200; +const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; /// A reservation remains identifiable while its backing is built without the /// device mutex. Failed construction returns the slot, never its host identity. @@ -56,10 +57,10 @@ pub(super) struct Registry { compute: KVec>, pub(super) compute_building: bool, compute_scheduler_building: bool, + replay_outer_wait: Option, pub(super) observations: compute::retirement::Batch, worker_deferred: Option, teardown: super::teardown::Pending, - exit_acks: compute::exit::Acks, } impl Registry { @@ -91,10 +92,10 @@ impl Registry { }, compute_building: false, compute_scheduler_building: false, + replay_outer_wait: None, observations: compute::retirement::Batch::new()?, worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, - exit_acks: compute::exit::Acks::new(), }) } @@ -235,6 +236,7 @@ impl Backend { owner: u64, ) -> Result<(Arc, admission::QueueSlot)> { let lease = shared.preparations.enter(true)?; + let dev = shared.drm()?; let (uat, pool, qos, execution, slot) = { let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -252,7 +254,7 @@ impl Backend { ) }; let alloc = super::object::Allocator { - dev: &shared.dev, + dev: &dev, uat: &uat, }; let owner_pid = @@ -285,6 +287,7 @@ impl Backend { fn shared_compute(&self) -> Result { loop { let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; let uat = { let mut state = self.shared.state.lock(); loop { @@ -310,7 +313,7 @@ impl Backend { } }; let alloc = super::object::Allocator { - dev: &self.shared.dev, + dev: &dev, uat: &uat, }; let mut built = Some(super::initdata::ComputeShared::new(&alloc)); @@ -406,6 +409,7 @@ impl Backend { } loop { let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; let (uat, reservation) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -423,7 +427,7 @@ impl Backend { (firmware.uat.clone(), reservation) }; let alloc = super::object::Allocator { - dev: &self.shared.dev, + dev: &dev, uat: &uat, }; let built = (|| { @@ -527,7 +531,19 @@ impl DeferredBatch { completion.status().record(*error); } } - self.values.push_within_capacity(value).map_err(|_| EIO) + self.values.push_within_capacity(value).map_err(|_| EIO)?; + // A later publisher can borrow this pair before finish signals its old + // fence. Publish the exact completion's witness only after collection + // succeeds; failure fences and rejected events are not retirement. + if let Some( + Deferred::Retired(completion, _) + | Deferred::Drained(completion, _) + | Deferred::Release(completion), + ) = self.values.last() + { + completion.note_retirement_queued(); + } + Ok(()) } /// Called only after releasing the device mutex. pub(super) fn finish(&mut self) { @@ -596,6 +612,11 @@ impl super::Firmware { if self.recovery.pending() { return Err(EAGAIN); } + // No descriptor, accounting or inner producer changes precede this + // capacity check. The same device lock covers queue.publish below. + if !self.init.work_ready(DataMaster::Compute)? { + return Err(EAGAIN); + } let queues = &mut self.queues; let queue = queues .compute @@ -751,13 +772,15 @@ impl Backend { deferred.finish(); let pending = pending?; if !pending { - break; + return self.build_compute(); } // Other owners continue submitting while this owner gives its // queue-local drain witness time to arrive. fsleep(Delta::from_millis(1)); } - self.build_compute() + // No new command has been accepted. Keep the spared queue and its + // retained work intact until a later admission observes its witness. + Err(EBUSY) } /// Called after taking this packet's publication claim. All waits and work @@ -769,6 +792,7 @@ impl Backend { ) -> Result { packet.check_dependencies()?; let mut recovery_polls = 0; + let mut outer_wait = None; loop { if packet.completion.status().get() != 0 { return Err(EIO); @@ -826,6 +850,20 @@ impl Backend { kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); continue; } + if matches!(result, Err(EAGAIN)) && !packet.is_published() { + let now = now_ns(); + let since = *outer_wait.get_or_insert(now); + if now.saturating_sub(since) >= WORK_RING_WAIT_NS { + return Err(EBUSY); + } + // Backpressure owns no firmware-visible attempt. Other queues + // can submit and the event worker can retire while we sleep. + drop(state); + drop(lease); + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; + } return result; } } @@ -863,12 +901,15 @@ impl super::Firmware { } impl super::Shared { - /// Joins construction and serializes the stop writes with detachment. - /// After installation, seeing no owner proves that an earlier stop caller - /// has already stopped both processors. Callbacks and destruction follow - /// outside the device mutex. + /// Joins construction and complete host teardown, including RTKit drops. + /// The stop-only mutex outlives detached firmware destruction, so unbind + /// cannot return while another stop caller still owns mailbox clients. + /// Completion callbacks remain outside the device mutex. Direct handlers + /// do not synchronously request or join stop; this path does not join + /// scheduler workers. pub(super) fn stop_queues(&self) { self.preparations.wait_drained(); + let _stopping = self.stop.lock(); let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { @@ -879,6 +920,8 @@ impl super::Shared { }; if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); + // RTKit destruction still dereferences the mailbox provider. + drop(firmware); } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 0643da70d72ca..439d5c9831cbb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -14,33 +14,44 @@ use crate::g17::{ use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; +impl super::Registry { + /// ACK storage belongs to the already-retained transaction, so a full event + /// batch cannot overflow a separate cache. No allocation or wait is needed. + pub(super) fn acknowledge_compute_exit(&mut self, cookie: u64) { + for entry in self.compute.iter_mut().flatten() { + if let Some(queue) = entry.queue.as_deref_mut() { + if queue.acknowledge_exit(cookie) { + return; + } + } + } + } +} + impl Firmware { pub(in crate::g17) fn service_compute_exit(&mut self, deferred: &mut DeferredBatch) -> Result { let recovery_closed = self.init.recovery_state()? == 0; + let view = crate::g17::qos::View::new(self.init.qos()?)?; for entry in self.queues.compute.iter_mut().flatten() { if let Some(queue) = entry.queue.as_deref_mut() { - queue.settle_exit(&self.queues.exit_acks, recovery_closed, &mut |event| { - deferred.push(event) - })?; - } - } - for cookie in self.queues.exit_acks.snapshot() { - if cookie != 0 - && !self - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.exit_cookie() == Some(cookie)) - { - self.queues.exit_acks.forget(cookie); + queue.settle_exit( + recovery_closed, + &mut |owner| self.queues.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; } } if self.recovery.pending() || !recovery_closed { return Ok(()); } let now = now_ns(); + // Publication has already retained each kill. Retry only the doorbell; + // failed attempts do not extend its bounded polling window. + for entry in self.queues.compute.iter_mut().flatten() { + if let Some(queue) = entry.queue.as_deref_mut() { + let _ = queue.announce_exit(now, || self.primary.notify(MSG_CONTROL_NOTIFY)); + } + } for index in 0..self.queues.teardown.len() { let Some((context, age)) = self.queues.teardown.closed(index, now) else { continue; @@ -103,6 +114,7 @@ impl Firmware { hold |= 1u64 << slot; } } + let cookie = exit::next_cookie()?; if hold != 0 && !self .queues @@ -111,7 +123,6 @@ impl Firmware { { return Ok(()); } - let cookie = exit::next_cookie(); let record = ControlRecord::ContextKill(ContextKill::new(context.scheduler_va(), cookie)); if let Err(error) = Rings::publish_control(&self.init, &record) { self.queues.admission.release_exit_kill_hold(hold); @@ -119,13 +130,13 @@ impl Firmware { } // Publication is irrevocable even if its doorbell fails. Keep both // the killed graph and the held render identities until processor stop. - self.queues.compute[index] + let queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) - .ok_or(EIO)? - .record_exit(cookie, now); + .ok_or(EIO)?; + queue.record_exit(cookie, now); fence(Ordering::SeqCst); - if let Err(error) = self.primary.notify(MSG_CONTROL_NOTIFY) { + if let Err(error) = queue.announce_exit(now, || self.primary.notify(MSG_CONTROL_NOTIFY)) { dev_warn!( self.primary.state.shared.dev.as_ref(), "Scheduler {} exit announcement failed: {:?}\n", diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index c872a7c8dacc7..65b00d36714bb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -27,6 +27,7 @@ impl Firmware { .map(|queue| queue.replay_commands()) .sum::(); if commands == 0 { + self.queues.replay_outer_wait = None; return Ok(()); } deferred.reserve_replays(commands)?; @@ -55,8 +56,24 @@ impl Firmware { break; }; let owner = queue.owner(); + let outer_ready = self.init.work_ready(crate::g17::fw::queue::DataMaster::Compute)?; + if !outer_ready { + let now = super::now_ns(); + let since = *self.queues.replay_outer_wait.get_or_insert(now); + if now.saturating_sub(since) < super::WORK_RING_WAIT_NS { + // Keep the FIFO and its replay credit untouched. The + // event worker schedules the existing delayed poll. + return Ok(()); + } + } else { + self.queues.replay_outer_wait = None; + } - let result = (|| { + let result = if !outer_ready { + // This replay has no live firmware ownership. Persistent + // backpressure uses the existing failed-replay retirement. + Err(EBUSY) + } else { (|| { self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) @@ -66,7 +83,7 @@ impl Firmware { packet.check_dependencies()?; let dependencies = self.queues.render_prefix(owner, &packet)?; self.publish_compute(owner, packet.clone(), dependencies.as_slice(), deferred) - })(); + })() }; let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) && (self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0); @@ -89,6 +106,7 @@ impl Firmware { queue.finish_replay(&packet)?; } } + self.queues.replay_outer_wait = None; Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index f249e3520e0dd..08c11eb98790e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -64,7 +64,7 @@ impl event::Host for Service<'_> { impl event::Queues for Service<'_> { fn context_killed(&mut self, cookie: u64) { - self.firmware.queues.exit_acks.note(cookie); + self.firmware.queues.acknowledge_compute_exit(cookie); } fn flush_tvb_reply(&mut self) -> Result { self.firmware.flush_tvb_reply() @@ -125,8 +125,8 @@ impl recovery::Memory for Service<'_> { fn clear_timestamps(&self) -> Result { self.firmware.init.clear_timestamps() } - fn entries(&self, out: &mut [recovery::Entry; recovery::MAX_ENTRIES]) -> Result { - self.firmware.init.entries(out) + fn entries(&self) -> Result { + self.firmware.init.entries() } fn wait_tick(&self) { self.firmware.init.wait_tick(); @@ -395,8 +395,16 @@ impl crate::g17::Shared { } let backpressured = firmware.render_control_backpressured(); let polling = backpressured + || firmware.queues.replay_outer_wait.is_some() || firmware.queues.teardown.polling(now) - || firmware.compute_exit_polling(now); + || firmware.compute_exit_polling(now) + || firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.retirement_polling()); if polling { self.queue_poll(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index a6aa740ecf4e3..e466571bc88c7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -239,6 +239,17 @@ impl Registry { Ok(()) } + /// Only an optional sibling's unpublished QID-capacity refusal is soft. + /// ensure_render_ids cancels a first ID if the second cannot be reserved; + /// validation, identity overflow, and cancellation errors still propagate. + fn prepare_render_ids(&mut self, slot: u8, owner: u64, optional: bool) -> Result { + match self.ensure_render_ids(slot, owner) { + Ok(()) => Ok(true), + Err(error) if optional && error == ENOSPC => Ok(false), + Err(error) => Err(error), + } + } + fn reserve_render(&mut self, slot: u8, owner: u64) -> Result { self.ensure_render_ids(slot, owner)?; let entry = self.render.entry(slot)?; @@ -318,7 +329,7 @@ impl Registry { } if let Some(pair) = entry.pair.as_mut() { pair.quarantine(error, &mut |event| deferred.push(event))?; - } else if entry.preparing.is_some() { + } else if entry.preparing.is_some() || entry.growing { entry.pending_error = Some((status.clone(), error)); } } @@ -332,7 +343,7 @@ struct Host<'a> { } impl Host<'_> { fn release_registration(&mut self) { - if !core::mem::take(&mut self.registration_powered) { + if !self.registration_powered && !self.firmware.registration_power_release_pending { return; } if let Err(error) = self.firmware.release_registration_power() { @@ -341,6 +352,8 @@ impl Host<'_> { "Could not restore render idle policy: {:?}\n", error ); + } else { + self.registration_powered = false; } } } @@ -647,7 +660,10 @@ impl Registry { let active = entry .pair .as_ref() - .map_or(entry.borrowed_active, |pair| pair.published_in_flight()); + .map_or( + (entry.preparing.is_some() || entry.growing) && entry.borrowed_active, + |pair| pair.published_in_flight(), + ); active && entry.reservation.ids.into_iter().all(|id| { let qid = usize::from(id.qid()); @@ -660,6 +676,12 @@ impl Registry { packet: &Arc, error: Error, ) -> bool { + // Its tracker may already be gone while off-lock completion still + // owns the fence and admission lease. A loan for a subsequent packet + // cannot turn this exact accepted retirement into a missing-owner error. + if packet.completion.retirement_queued() { + return true; + } let Some(slot) = packet.completion.render_slot() else { return false; }; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index db5820cfb6663..b283fb8e7d55c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -194,6 +194,7 @@ impl crate::g17::Shared { pub(in crate::g17) fn grow_render(&self) -> Result { loop { let lease = self.preparations.enter(false)?; + let dev = self.drm()?; let (request, slot, mut pair) = { let mut state = self.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -246,7 +247,7 @@ impl crate::g17::Shared { Err(EFAULT) } else { manager.prepare_growth( - &self.dev, + &dev, &vm, crate::g17::buffer::firmware_growth_target(before), None, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index e8fc30560a890..69f8249e0818d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -60,15 +60,18 @@ enum Build { } impl super::super::Backend { - fn build_render_pair(&self, slot: u8, first: &Arc) -> Result { + fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; let (resources, build) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } - firmware.queues.ensure_render_ids(slot, self.owner)?; + if !firmware.queues.prepare_render_ids(slot, self.owner, optional)? { + return Ok(()); + } let resources = BuildResources::new(firmware)?; let build = if let Some(entry) = firmware.queues.render.entries[usize::from(slot)].as_ref() { @@ -159,7 +162,7 @@ impl super::super::Backend { (resources, build) }; let alloc = Allocator { - dev: &self.shared.dev, + dev: &dev, uat: &resources.uat, }; let fresh_graph = match &build { @@ -458,7 +461,7 @@ impl super::super::Backend { } pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { - self.build_render_pair(slot, packet)?; + self.build_render_pair(slot, packet, false)?; let sibling = { let state = self.shared.state.lock(); (*state) @@ -469,7 +472,12 @@ impl super::super::Backend { .warmable_sibling(self.owner, slot) }; if let Some(sibling) = sibling { - self.build_render_pair(sibling, packet)?; + // An optional sibling may be borrowed or waiting for recovery. + // The selected pair is revalidated by prepare_render before publication. + match self.build_render_pair(sibling, packet, true) { + Err(error) if error == EBUSY || error == EAGAIN => {} + result => result?, + } } Ok(()) } @@ -481,6 +489,7 @@ impl super::super::Backend { deferred: &mut super::super::DeferredBatch, ) -> Result { let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -535,7 +544,7 @@ impl super::super::Backend { (firmware.uat.clone(), pair, addresses, growth_target) }; let alloc = Allocator { - dev: &self.shared.dev, + dev: &dev, uat: &uat, }; let mut growth = None; @@ -543,7 +552,7 @@ impl super::super::Backend { if let Some((target, descriptor)) = growth_target { let vm = pair.context().vm().clone(); growth = pair.manager().prepare_growth( - &self.shared.dev, + &dev, &vm, target, Some(descriptor), diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 0586dfdc0944f..59a54b1926375 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -260,6 +260,13 @@ impl crate::g17::Firmware { if packet.completion.status().get() != 0 { return Err(EIO); } + // Both outer destinations must have room before pool accounting or + // either inner queue becomes visible. All host producers share this lock. + if !self.init.work_ready(crate::g17::fw::queue::DataMaster::Fragment)? + || !self.init.work_ready(crate::g17::fw::queue::DataMaster::Tiling)? + { + return Err(EBUSY); + } let mut dependencies = [KickDependency::ZERO; 3]; let count = self .queues diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 06465629eb2be..b1495dbf3e23e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -30,7 +30,10 @@ impl Reclaim { pub(in crate::g17) fn finish(self) { match self { Self::Binding(binding) => drop(binding), - Self::Context(context) => drop(context), + Self::Context(context) => { + context.release_execution(); + drop(context); + } } } } @@ -187,7 +190,7 @@ impl Firmware { primary: &mut self.primary, reclaim, }; - teardown.publish_ready(&mut service, recovery)?; + teardown.publish_ready(&mut service, recovery, super::now_ns())?; let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); let producer = self.init.control_producer()?.load(Ordering::Relaxed); while teardown diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 9acc01fbc2125..f53c2d45c129a 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -23,6 +23,7 @@ struct Entry { phase: Phase, armed: u64, warned: bool, + announce_pending: bool, } /// Publication and announcement errors retain the entry. A stalled release is reported once; @@ -88,6 +89,7 @@ impl Pending { phase, armed: now, warned: false, + announce_pending: false, }) .map_err(|_| ENOSPC) } @@ -130,9 +132,21 @@ impl Pending { /// Publish in insertion order. Mandatory control replies and a full control ring stop this /// pass; an error for one page leaves it retained while other ready releases may progress. - pub(crate) fn publish_ready(&mut self, host: &mut impl Host, recovery: bool) -> Result { + pub(crate) fn publish_ready(&mut self, host: &mut impl Host, recovery: bool, now: u64) -> Result { if !recovery { for entry in self.entries.iter_mut() { + // The ring record is already published. Retry only its doorbell; + // never duplicate a scheduler release after a transport error. + if matches!(entry.phase, Phase::Released(_)) { + if entry.announce_pending && host.notify().is_ok() { + entry.announce_pending = false; + // One successful retry starts its consumption window. + // Failed retries never slide the polling deadline. + entry.armed = now; + entry.warned = false; + } + continue; + } if entry.phase != Phase::Draining || !host.render_idle(&entry.context)? || !host.flists_released(&entry.context) @@ -160,9 +174,15 @@ impl Pending { } }; entry.phase = Phase::Released(next); + // A long command drain must not consume the acknowledgement + // polling window before this release exists on the control ring. + entry.armed = now; + entry.warned = false; + entry.announce_pending = true; fence(Ordering::SeqCst); - if let Err(error) = host.notify() { - host.report(&entry.context, Failure::Announcement(error)); + match host.notify() { + Ok(()) => entry.announce_pending = false, + Err(error) => host.report(&entry.context, Failure::Announcement(error)), } } } @@ -200,8 +220,9 @@ impl Pending { Ok(None) } - /// Poll quickly only during the initial release window; late arrivals still settle when - /// serviced. Time passage never substitutes for a firmware-consumption witness. + /// Poll for a bounded window at close and again at actual release publication. + /// Failed announcements remain retryable on later service even after polling + /// expires. Time passage never substitutes for a firmware-consumption witness. pub(crate) fn polling(&self, now: u64) -> bool { self.entries.iter().any(|entry| { entry.phase != Phase::Deferred && now.saturating_sub(entry.armed) < POLL_NS diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 479e3cf1c15ad..47550f81464de 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -58,6 +58,17 @@ pub(crate) struct AsahiObject { /// Type alias for the shmem GEM object type for this driver. pub(crate) type Object = shmem::Object; +/// VM-private BOs share their assigned VM's reservation object. Validate this +/// before acquiring mapping state; generic GPUVM accepts external shareable BOs. +pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { + if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 + && vm.is_extobj(gem) + { + return Err(EINVAL); + } + Ok(()) +} + unsafe impl Send for AsahiObject {} unsafe impl Sync for AsahiObject {} @@ -124,11 +135,7 @@ impl ObjectRef { if obj_range.end > self.gem.size() { return Err(EINVAL); } - if self.gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 - && vm.is_extobj(&*self.gem) - { - return Err(EINVAL); - } + validate_vm_binding(&self.gem, vm)?; vm.map_in_range(&self.gem, obj_range, alignment, range, prot, guard) } @@ -142,11 +149,7 @@ impl ObjectRef { prot: mmu::Prot, guard: bool, ) -> Result { - if self.gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 - && vm.is_extobj(&*self.gem) - { - return Err(EINVAL); - } + validate_vm_binding(&self.gem, vm)?; vm.map_at(addr, self.gem.size(), self.gem.clone(), prot, guard) } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 5f30660ed2fc4..6b057e4a84f2f 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -1172,8 +1172,9 @@ impl GpuManager::ver { .with(|raw, _inner| raw.pending_submissions.fetch_add(1, Ordering::Acquire)); mod_dev_dbg!(self.dev, "OP start (pending: {})\n", val + 1); + let guard = OpGuard(self.clone()); self.kick_firmware()?; - Ok(OpGuard(self.clone())) + Ok(guard) } fn invalidate_context( diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 9fbf1236bdd84..8ed9c8625243a 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -548,14 +548,13 @@ impl PwrConfig { return Err(EINVAL); } - let pz_count = pz_data.len() / 3; let mut power_zones = KVec::new(); - for i in (0..pz_count).step_by(3) { + for zone in pz_data.chunks_exact(3) { power_zones.push( PowerZone { - target: pz_data[i], - target_offset: pz_data[i + 1], - filter_tc: pz_data[i + 2], + target: zone[0], + target_offset: zone[1], + filter_tc: zone[2], }, GFP_KERNEL, )?; diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index f354e4bb1f71a..0e99ab33fd3fc 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -82,7 +82,8 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { let start_pg = va >> mmu::UAT_PGBIT; let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; - let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & 0x1fffffffff); + const BADDR_MASK: u64 = (1 << 37) - 1; + let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); let pages = end_pg - start_pg; // Guess? It's possible that the page count is in terms of 4K pages @@ -124,6 +125,10 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { ); } base -= 32; + // Each full operation covers 32 units of 2 << bits GPU granules. + // Advance only the base-address field, preserving the ASID and scale. + let next = (val & BADDR_MASK) + (32u64 << (bits + 1)); + val = (val & !BADDR_MASK) | (next & BADDR_MASK); } // SAFETY: tlbi is always safe by definition diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 36dbf1da2ad89..84eff788e0cf9 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -20,6 +20,7 @@ use core::num::NonZeroUsize; use core::ops::Range; use core::sync::atomic::{ fence, + AtomicBool, AtomicU32, AtomicU64, AtomicU8, @@ -465,26 +466,7 @@ impl gpuvm::DriverGpuVm for VmInner { ) -> Result { let va = op.va().expect("step_unmap: missing VA"); - mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); - - let _mutation = self.mapping_mutation(); - self.page_table - .unmap_pages(va.addr()..(va.addr() + va.range()))?; - - if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); - self.tlbi_range(asid as u8, va.addr(), va.range() as usize); - mod_dev_dbg!( - self.dev, - "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", - asid, - va.addr(), - va.range(), - ); - mem::sync(); - } - - self.tlbi_contexts_naming_root(va.addr(), va.range() as usize); + self.unmap_gpuva(va)?; if op.unmap_and_unlink_va_defer().is_none() { dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); @@ -574,15 +556,45 @@ impl gpuvm::DriverGpuVm for VmInner { } impl VmInner { + /// Remove hardware leaves for one complete user GPUVA without allocating. + fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { + mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); + + let _mutation = self.mapping_mutation(); + self.page_table + .unmap_pages(va.addr()..(va.addr() + va.range()))?; + + if let Some(asid) = self.slot() { + fence(Ordering::SeqCst); + self.tlbi_range(asid as u8, va.addr(), va.range() as usize); + mod_dev_dbg!( + self.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + va.addr(), + va.range(), + ); + mem::sync(); + } + + self.tlbi_contexts_naming_root(va.addr(), va.range() as usize); + + Ok(()) + } + /// A failed map has no GPUVA owner to retain its backing or remove its leaves. /// GPUVM has already split or removed overlaps throughout this operation's range. fn rollback_failed_map(&mut self, addr: u64, size: u64) { - if self.uat_inner.firmware == UatFirmware::Handoff { - return; - } // Include the current segment: map_pages can fail after writing some of its leaves. // The checked GPUVM operation supplies an aligned range without overflow. let result = self.page_table.discard_partial_map(addr..addr + size); + if self.uat_inner.firmware == UatFirmware::Handoff { + if let Some(asid) = self.slot() { + fence(Ordering::SeqCst); + self.tlbi_range(asid as u8, addr, size as usize); + mem::sync(); + } + } self.tlbi_contexts_naming_root(addr, size as usize); if let Err(error) = result { dev_err!( @@ -670,10 +682,24 @@ impl VmInner { /// Map an `mm::Node` representing an mapping in VA space. fn map_node(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { + let end = node + .start() + .checked_add(node.mapped_size as u64) + .ok_or(EOVERFLOW)?; + let _mutation = self.mapping_mutation(); + self.page_table.prepare_map(node.start()..end)?; + let result = self.map_node_prepared(node, prot); + if result.is_err() { + self.rollback_failed_map(node.start(), node.mapped_size as u64); + } + result + } + + /// Install only into the empty range reserved by `map_node`. + fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); let guard = node.bo.as_ref().ok_or(EINVAL)?.get()?.inner().inner.lock(); let sgt = guard.sgt.as_ref().ok_or(EINVAL)?; - let _mutation = self.mapping_mutation(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -752,14 +778,16 @@ no_debug!(Vm); /// An application TTBAT identity retained by a logical queue and its jobs. /// The root lease is dropped before the VM, so no page table can be freed -/// while a context still names it. Installed firmware queues must retain this -/// object through replacement retirement or processor stop. +/// while an active context still names it. Installed firmware queues retain +/// this object through replacement retirement or processor stop; a consumed +/// closed-scheduler release may return its root identity while keeping storage. pub(crate) struct ExecutionContext { root: ExecutionRoot, vm: Vm, } struct ExecutionRoot { + released: AtomicBool, inner: Arc, id: u8, generation: u8, @@ -778,16 +806,28 @@ impl ExecutionContext { &self.vm } + /// Called only after the closed scheduler's release was consumed and all + /// its commands retired. Installed storage may outlive this root lease. + pub(crate) fn release(&self) { + self.root.release(); + } + pub(crate) fn is_current(&self) -> bool { let inner = self.root.inner.lock(); + if self.root.released.load(Ordering::Acquire) { + return false; + } let roots = &inner.ttbs()[usize::from(self.root.id)]; roots.ttb0.load(Ordering::Acquire) == self.root.low && roots.ttb1.load(Ordering::Acquire) == self.root.high } } -impl Drop for ExecutionRoot { - fn drop(&mut self) { +impl ExecutionRoot { + fn release(&self) { + if self.released.swap(true, Ordering::AcqRel) { + return; + } let id = usize::from(self.id); let release = { let inner = self.inner.lock(); @@ -817,6 +857,12 @@ impl Drop for ExecutionRoot { } } +impl Drop for ExecutionRoot { + fn drop(&mut self) { + self.release(); + } +} + /// Slot data for a [`Vm`] slot (nothing, we only care about the indices). pub(crate) struct SlotInner(); @@ -868,10 +914,9 @@ impl Clone for VmBind { } } -/// A mapping BO reference selecting the teardown protocol of its firmware. +/// A mapping BO reference released under the immediate-mode GPUVA lock. /// Retain the owner until deferred cleanup completes, including its final BO. struct MappingBo { - deferred: bool, bo: Option>>, owner: ARef>, } @@ -879,10 +924,8 @@ impl MappingBo { fn new( bo: ARef>, owner: ARef>, - deferred: bool, ) -> Self { Self { - deferred, bo: Some(bo), owner, } @@ -896,12 +939,6 @@ impl Drop for MappingBo { let Some(bo) = self.bo.take() else { return; }; - if !self.deferred { - // Handoff VMs retain the legacy ARef decrement and reservation - // locking behavior. Only published-root VMs use deferred cleanup. - drop(bo); - return; - } let raw = ARef::into_raw(bo); // SAFETY: GpuVmBo is repr(C) with drm_gpuvm_bo first. into_raw // transfers one live reference. Every Vm here uses IMMEDIATE_MODE; @@ -930,6 +967,27 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } +/// Prepared CPU timestamp access; destruction happens after completion signalling. +pub(crate) struct TimestampMapping { + mapping: shmem::VMap, + offset: usize, +} + +impl TimestampMapping { + pub(crate) fn write(&self, value: u64) { + let map = self.mapping.get(); + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: timestamp_mapping checked the complete eight-byte extent. + // The owned vmap pins its backing; byte atomics permit unaligned and + // overlapping timestamp destinations without mutable slice aliases. + let destination = unsafe { + &*map.as_mut_ptr().add(self.offset + index).cast::() + }; + destination.store(byte, Ordering::Relaxed); + } + } +} + /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); @@ -961,8 +1019,8 @@ impl KernelMapping { /// Writes a userspace timestamp through the retained GEM mapping. Byte /// stores support unaligned destinations and concurrent completion writes /// without creating mutable aliases. The complete timestamp is not atomic. - pub(crate) fn write_timestamp(&self, offset: usize, value: u64) -> Result { - use core::sync::atomic::AtomicU8; + /// Retain CPU access before this destination's completion fence is published. + pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -973,15 +1031,7 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - let vmap = gem.vmap::()?; - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: The retained GEM/vmap covers each checked byte. AtomicU8 - // has byte alignment and all concurrent completion writers use - // the same access width, without borrowing a mutable byte slice. - let destination = unsafe { &*vmap.as_mut_ptr().add(start + index).cast::() }; - destination.store(byte, Ordering::Relaxed); - } - Ok(()) + Ok(TimestampMapping { mapping: gem.owned_vmap::()?, offset: start }) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1054,11 +1104,7 @@ impl KernelMapping { /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the /// coprocessor cache. This is required to safely unmap cached/private mappings. fn remap_uncached_and_flush(&mut self) { - let mut owner = self - .0 - .owner - .exec_lock(None, false) - .expect("Failed to exec_lock in remap_uncached_and_flush"); + let mut owner = self.0.owner.lock_inner(); mod_dev_dbg!( owner.dev, @@ -1220,11 +1266,9 @@ impl Drop for KernelMapping { self.remap_uncached_and_flush(); } - let mut owner = self - .0 - .owner - .exec_lock(None, false) - .expect("exec_lock failed in KernelMapping::drop"); + // Only VM page tables/binding state are touched here. The shared + // reservation is sufficient and avoids fallible exec allocation in Drop. + let mut owner = self.0.owner.lock_inner(); mod_dev_dbg!( owner.dev, "MMU: unmap {:#x}:{:#x}\n", @@ -1698,19 +1742,18 @@ impl Vm { } core::mem::drop(vm_bo_guard); - let deferred = inner.uat_inner.firmware != UatFirmware::Handoff; - let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone(), deferred)); + let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone())); let uat_inner = inner.uat_inner.clone(); // A failed reservation drops its payload while exec still owns GEM - // reservations. Published-root VMs attach BO/GEM only after success; - // handoff VMs retain their existing payload and decrement behavior. + // reservations. Attach BO/GEM references only after success, since + // their destructors may acquire the same reservation locks. let result = inner.mm.insert_node_in_range( KernelMappingInner { owner: self.inner.clone(), uat_inner, prot, - bo: if deferred { None } else { vm_bo.take() }, - _gem: if deferred { None } else { Some(gem.into()) }, + bo: None, + _gem: None, offset: object_range.start, mapped_size: size, }, @@ -1728,7 +1771,7 @@ impl Vm { return Err(error); } }; - if deferred { + { let payload = node.as_mut().inner_mut(); payload.bo = vm_bo.take(); payload._gem = Some(gem.into()); @@ -1766,17 +1809,17 @@ impl Vm { } core::mem::drop(vm_bo_guard); - let deferred = inner.uat_inner.firmware != UatFirmware::Handoff; - let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone(), deferred)); + let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone())); let uat_inner = inner.uat_inner.clone(); - // Published-root VMs defer BO/GEM attachment until reservation succeeds. + // Keep reservation-locking destructors out of the allocator payload + // until reservation succeeds. let result = inner.mm.reserve_node( KernelMappingInner { owner: self.inner.clone(), uat_inner, prot, - bo: if deferred { None } else { vm_bo.take() }, - _gem: if deferred { None } else { Some(gem.clone()) }, + bo: None, + _gem: None, offset: 0, mapped_size: size, }, @@ -1791,7 +1834,7 @@ impl Vm { return Err(error); } }; - if deferred { + { let payload = node.as_mut().inner_mut(); payload.bo = vm_bo.take(); payload._gem = Some(gem.clone()); @@ -1816,6 +1859,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; // Mapping needs a complete context let mut ctx = StepContext { @@ -1828,24 +1872,30 @@ impl Vm { let vm_bo = self.inner.obtain_bo(gem)?; { - let mut vm_bo_guard = vm_bo.inner().inner.lock(); - if vm_bo_guard.sgt.is_none() { + let needs_sgt = vm_bo.inner().inner.lock().sgt.is_none(); + if needs_sgt { + // Pin without the BO mutex: alias construction can hold the GEM + // reservation while acquiring this mutex. let sgt = gem.owned_sg_table()?; - - if vm_bo_guard.sg_vec.is_none() { - let mut sg_vec = KVVec::new(); - let mut offset = 0; - for range in sgt.iter() { - let addr = range.dma_address() as usize; - let len = range.dma_len() as usize; - sg_vec.push((offset, addr..(addr + len)), GFP_KERNEL)?; - offset += len; + let mut vm_bo_guard = vm_bo.inner().inner.lock(); + if vm_bo_guard.sgt.is_none() { + if vm_bo_guard.sg_vec.is_none() { + let mut sg_vec = KVVec::new(); + let mut offset = 0; + for range in sgt.iter() { + let addr = range.dma_address() as usize; + let len = range.dma_len() as usize; + sg_vec.push((offset, addr..(addr + len)), GFP_KERNEL)?; + offset += len; + } + vm_bo_guard.sg_vec.replace(sg_vec); } - vm_bo_guard.sg_vec.replace(sg_vec); + vm_bo_guard.sgt.replace(sgt); } - vm_bo_guard.sgt.replace(sgt); + // Release the mutex before a losing initializer drops its SG + // owner. The declaration order also preserves this on errors. + drop(vm_bo_guard); } - core::mem::drop(vm_bo_guard); } let mut inner = self.inner.exec_lock(Some(gem), true)?; @@ -1928,13 +1978,18 @@ impl Vm { 0, )?; + let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; + inner.page_table.prepare_map(iova..end)?; let mutation = inner.mapping_mutation(); let ret = inner.page_table.map_pages( - iova..(iova + size as u64), + iova..end, phys as PhysicalAddr, prot, false, ); + if ret.is_err() { + inner.rollback_failed_map(iova, size as u64); + } // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. drop(mutation); @@ -1993,15 +2048,19 @@ impl Vm { if let Some(bo) = self.inner.find_bo(gem) { mod_dev_dbg!(inner.dev, "MMU: bo_unmap\n"); - self.inner.bo_unmap(&mut ctx, &bo)?; - // The close callback or deferred object list retains this GEM, so - // clearing its candidate cannot release the final reference here. - self.untrack_context_object(gem); - self.untrack_shared_object(gem); + let result = self.inner.bo_unmap(&mut ctx, &bo); + if result.is_ok() { + // The close callback or deferred object list retains this GEM, + // so clearing its candidate cannot release its final reference. + self.untrack_context_object(gem); + self.untrack_shared_object(gem); + } mod_dev_dbg!(inner.dev, "MMU: bo_unmap done\n"); - // We need to drop the exec_lock first, then the GpuVmBo since that will take the lock itself. + // BO release can drain unrelated deferred GEMs, including on an + // unmap error. Always release reservations before dropping it. core::mem::drop(inner); core::mem::drop(bo); + result?; } Ok(()) @@ -2259,6 +2318,7 @@ impl Uat { let context = Arc::new( ExecutionContext { root: ExecutionRoot { + released: AtomicBool::new(false), inner: self.inner.clone(), id: id as u8, generation, diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 89c7db4f3d2e7..67d18a275ca32 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -59,6 +59,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + gem::validate_vm_binding(gem, self)?; let context = self.prepare_context_binding(addr, size, offset)?; let mut ctx = StepContext { new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), @@ -69,21 +70,29 @@ impl Vm { }; let vm_bo = self.inner.obtain_bo(gem)?; { - let mut bo = vm_bo.inner().inner.lock(); - if bo.sgt.is_none() { + let needs_sgt = vm_bo.inner().inner.lock().sgt.is_none(); + if needs_sgt { + // Pin without the BO mutex: alias construction can hold the GEM + // reservation while acquiring this mutex. let sgt = gem.owned_sg_table()?; - if bo.sg_vec.is_none() { - let mut segments = KVVec::new(); - let mut offset = 0; - for range in sgt.iter() { - let address = range.dma_address() as usize; - let length = range.dma_len() as usize; - segments.push((offset, address..address + length), GFP_KERNEL)?; - offset += length; + let mut bo = vm_bo.inner().inner.lock(); + if bo.sgt.is_none() { + if bo.sg_vec.is_none() { + let mut segments = KVVec::new(); + let mut offset = 0; + for range in sgt.iter() { + let address = range.dma_address() as usize; + let length = range.dma_len() as usize; + segments.push((offset, address..address + length), GFP_KERNEL)?; + offset += length; + } + bo.sg_vec = Some(segments); } - bo.sg_vec = Some(segments); + bo.sgt = Some(sgt); } - bo.sgt = Some(sgt); + // Release the mutex before a losing initializer drops its SG + // owner. The declaration order also preserves this on errors. + drop(bo); } } ctx.vm_bo = Some(vm_bo); diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 78f906ff9c46c..e33b578bb13a7 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -520,40 +520,19 @@ impl Vm { self.unmap_user_ranges_now(user, kernel) } - fn unmap_user_ranges_now(&self, user: Range, kernel: Range) -> Result { - let driver = self - .lifetime - .as_ref() - .map(|lifetime| lifetime.driver.lock()); - let mut cursor = user.start; - while cursor < user.end { - let mut next = user.end; - let mut covered = cursor; - for range in Iterator::chain( - core::iter::once(kernel.clone()), - driver - .as_ref() - .and_then(|guard| guard.as_ref()) - .into_iter() - .flat_map(DriverMappings::ranges), - ) { - if range.contains(&cursor) { - covered = covered.max(range.end); - } else if cursor < range.start { - next = next.min(range.start); - } - } - if covered != cursor { - cursor = covered.min(user.end); - continue; - } - if next <= cursor { - return Err(EIO); - } - self.unmap_range_now(cursor, next - cursor)?; - cursor = next; - } - Ok(()) + fn unmap_user_ranges_now(&self, user: Range, _kernel: Range) -> Result { + // User mappings alone have GPUVA nodes. Kernel and driver aliases use + // mm::Node and are untouched; GPUVM's embedded kernel cutout is skipped. + // Closed admission and zero accepted jobs make whole-node teardown safe. + let mut inner = self.inner.lock_inner(); + let result = inner.unmap_all(VmInner::unmap_gpuva); + if result.is_ok() { + self.untrack_shared_range(user.start, user.end - user.start); + } + drop(inner); + // Final GEM/SG-table destruction may acquire reservations itself. + self.bo_deferred_cleanup(); + result } } diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 35ac52f8fd2bf..5240cba7b3fc1 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -394,12 +394,18 @@ impl UatPageTable { let end = ((iova_range.end - 1) & self.ias_mask) + 1; let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); + // For every child table, retain the parent page and entry that names it. + // Children are detached before their parents are freed, so these borrowed + // parent pages remain valid while the descriptor is cleared. + let mut pt_parent: [Option<(PhysicalAddr, usize)>; UAT_LEVELS - 1] = Default::default(); pt_addr[UAT_LEVELS - 1] = Some(self.ttb); 'outer: while iova < end { mod_pr_debug!("UATPageTable::with_pages: iova={:#x}\n", iova); let addr_diff = last_iova ^ iova; - for level in (0..UAT_LEVELS - 1).rev() { + // Detach leaves before intermediate tables. The old order freed a + // parent first and could leave its descriptor naming a freed child. + for level in 0..UAT_LEVELS - 1 { // If the iova has changed at this level or above, invalidate the physaddr if addr_diff & !((1 << (UAT_PGBIT + (level + 1) * UAT_LVBIT)) - 1) != 0 { if let Some(phys) = pt_addr[level].take() { @@ -409,7 +415,10 @@ impl UatPageTable { level, phys ); - self.free_table(phys); + let parent = pt_parent[level].take(); + self.free_table(phys, parent)?; + } else { + pt_parent[level] = None; } mod_pr_debug!("UATPageTable::with_pages: invalidate level {}\n", level); } @@ -460,6 +469,9 @@ impl UatPageTable { Ok(None) } })?; + if pt_addr[level].is_some() { + pt_parent[level] = Some((phys, upidx)); + } mod_pr_debug!( "UATPageTable::with_pages: level {} PT {:#x?}\n", level, @@ -509,14 +521,16 @@ impl UatPageTable { } if free { - for level in (0..UAT_LEVELS - 1).rev() { + // The final path is detached bottom-up for the same reason as the + // boundary transitions above. + for level in 0..UAT_LEVELS - 1 { if let Some(phys) = pt_addr[level] { mod_pr_debug!( "UATPageTable::with_pages: free level {} {:#x?}\n", level, phys ); - self.free_table(phys); + self.free_table(phys, pt_parent[level].take())?; } } } @@ -525,18 +539,50 @@ impl UatPageTable { } /// Free a page table, unless it was not allocated by the driver. - fn free_table(&mut self, phys: PhysicalAddr) { - if let Some(tables) = self.driver_tables.as_mut() { + fn free_table( + &mut self, + phys: PhysicalAddr, + parent: Option<(PhysicalAddr, usize)>, + ) -> Result { + let owned = if let Some(tables) = self.driver_tables.as_ref() { match tables.iter().position(|&table| table == phys) { - Some(index) => { - tables.swap_remove(index); - } - None => return, + Some(_) => true, + None => false, } + } else { + true + }; + if !owned { + return Ok(()); + } + if let Some((parent_phys, parent_index)) = parent { + // SAFETY: The bottom-up traversal keeps the parent allocated while + // its child descriptor is detached. A changed descriptor indicates + // an ownership violation; fail closed instead of freeing a table + // that may still be reachable. + let page = unsafe { Page::borrow_phys(&parent_phys) }.ok_or(EIO)?; + page.with_pointer_into_page(parent_index * PTE_SIZE, PTE_SIZE, |p| { + let pte = unsafe { &*(p as *const AtomicU64) }; + let current = pte.load(Ordering::Acquire); + let expected = phys & self.oas_mask & (!UAT_PGMSK as u64); + if current & self.oas_mask & (!UAT_PGMSK as u64) != expected { + return Err(EIO); + } + pte.store(0, Ordering::Release); + Ok(()) + })?; + } + if let Some(tables) = self.driver_tables.as_mut() { + let index = tables + .iter() + .position(|&table| table == phys) + .ok_or(EIO)?; + tables.swap_remove(index); } // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without // `driver_tables`, every child table belongs to the driver. unsafe { Page::from_phys(phys) }; + Ok(()) } /// Checks every page, including holes skipped by the page-table walker. @@ -597,6 +643,22 @@ impl UatPageTable { self.with_pages(iova_range, true, false, |_, _| Ok(())) } + /// Reserve all tables for a fixed mapping before publishing any leaves. + /// GPUVM and the fixed-address allocator may share a page table, so check + /// that neither owns a destination leaf before allowing failure rollback. + pub(crate) fn prepare_map(&mut self, range: Range) -> Result { + if range.is_empty() || (range.start | range.end) & UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + self.with_pages(range.clone(), false, false, |_, ptes| { + if ptes.iter().any(|pte| pte.load(Ordering::Acquire) != 0) { + return Err(EBUSY); + } + Ok(()) + })?; + self.alloc_pages(range) + } + fn pte_bits(&self) -> u64 { if self.ttb_owned { // Owned page tables are userspace, so non-global diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index a493f94fda583..6fcb901aa4d2c 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -244,6 +244,8 @@ impl QueueJob::ver { #[versions(AGX)] impl sched::JobImpl for QueueJob::ver { + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + fn prepare(job: &mut sched::Job) -> Option { mod_dev_dbg!(job.dev, "QueueJob {}: Checking runnability\n", job.id); @@ -328,10 +330,6 @@ impl sched::JobImpl for QueueJob::ver { // are "invisible" to the firmware. Incrementing the notifier threshold // earlier than this violates that which leads to circles like the // above. Deferring the increment to submit solves the race. - job.notifier.threshold.with(|raw, _inner| { - raw.increase(job.notification_count); - }); - let gpu = match (*job.dev) .gpu .clone() @@ -381,6 +379,13 @@ impl sched::JobImpl for QueueJob::ver { } } + // Prepared submissions still hold their workqueue locks and have not + // published cpu_wptr. Only count them after every fallible submit() + // succeeded, before the first run_job exposes commands to firmware. + job.notifier.threshold.with(|raw, _inner| { + raw.increase(job.notification_count); + }); + // Now we fully commit to running the job mod_dev_dbg!(job.dev, "QueueJob {}: Run fragment\n", job.id); frag_sub.map(|a| gpu.run_job(a)).transpose()?; @@ -705,11 +710,11 @@ impl Queue for Queue::ver { match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { - last_compute = nr_commands; + last_render = nr_commands; nr_render += 1; } uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { - last_render = nr_commands; + last_compute = nr_commands; nr_compute += 1; } _ => {} diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs index 1a41d8f16d443..00246e1f64ab7 100644 --- a/drivers/gpu/drm/asahi/util.rs +++ b/drivers/gpu/drm/asahi/util.rs @@ -108,23 +108,24 @@ impl<'a> Reader<'a> { Reader { buffer, offset: 0 } } - pub(crate) fn read_up_to(&mut self, max_size: usize) -> Result { + pub(crate) fn read_up_to(&mut self, declared_size: usize) -> Result { let mut obj: T = Default::default(); - let size: usize = core::mem::size_of::().min(max_size); - let range = self.offset..self.offset + size; + let size: usize = core::mem::size_of::().min(declared_size); + let end = self.offset.checked_add(declared_size).ok_or(EINVAL)?; + let range = self.offset..end; let src = self.buffer.get(range).ok_or(EINVAL)?; // SAFETY: The output pointer is valid, and the size does not exceed // the type size, and all bit patterns are valid. let dst = unsafe { core::slice::from_raw_parts_mut(&mut obj as *mut _ as *mut u8, size) }; - dst.copy_from_slice(src); - self.offset += size; + dst.copy_from_slice(&src[..size]); + self.offset = end; Ok(obj) } pub(crate) fn read(&mut self) -> Result { - self.read_up_to(!0) + self.read_up_to(core::mem::size_of::()) } pub(crate) fn is_empty(&self) -> bool { diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c index b123f6164206b..92ab5271e523e 100644 --- a/drivers/gpu/drm/drm_gpuvm.c +++ b/drivers/gpu/drm/drm_gpuvm.c @@ -2382,6 +2382,7 @@ op_map_cb(const struct drm_gpuvm_ops *fn, void *priv, op.map.va.range = req->map.va.range; op.map.gem.obj = req->map.gem.obj; op.map.gem.offset = req->map.gem.offset; + op.map.gem.range = req->map.gem.range; op.map.flags = req->map.flags; return fn->sm_step_map(&op, priv); @@ -2438,7 +2439,7 @@ static bool __can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva_op_map * if (a->gem.obj != b->gem.obj || !a->gem.obj) return false; - if (can_merge_flags(gpuvm, a->flags, b->flags)) + if (!can_merge_flags(gpuvm, a->flags, b->flags)) return false; /* Order VAs for the rest of the checks. */ @@ -2457,7 +2458,7 @@ static bool __can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva_op_map * /* If this is a repeated mapping, both the GEM range * and offset must match. */ - if (a->gem.range != b->gem.range || + if (!a->gem.range || a->gem.range != b->gem.range || a->gem.offset != b->gem.offset) return false; @@ -2485,6 +2486,7 @@ static bool can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva *a, .va.range = a->va.range, .gem.offset = a->gem.offset, .gem.obj = a->gem.obj, + .gem.range = a->gem.range, .flags = a->flags, }; @@ -2738,7 +2740,7 @@ __drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, next.va.addr = req_end; next.va.range = end - req_end; next.gem.obj = obj; - prev.gem.range = va->gem.range; + next.gem.range = va->gem.range; next.gem.offset = offset + (va->flags & DRM_GPUVA_REPEAT ? 0 : req_end - addr); next.flags = va->flags; diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index e1920f3f920ae..ff068830f7fcb 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -179,6 +180,7 @@ static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, static void drm_sched_entity_kill_jobs_work(struct work_struct *wrk) { struct drm_sched_job *job = container_of(wrk, typeof(*job), work); + struct module *owner = job->sched->ops->owner; struct dma_fence *f; unsigned long index; @@ -213,6 +215,8 @@ static void drm_sched_entity_kill_jobs_work(struct work_struct *wrk) drm_sched_fence_finished(job->s_fence, -ESRCH); WARN_ON(job->s_fence->parent); job->sched->ops->free_job(job); + /* free_job may have freed both job and its scheduler allocation. */ + module_put(owner); } /* Signal the scheduler finished fence when the entity in question is killed. */ @@ -251,6 +255,11 @@ static void drm_sched_entity_kill(struct drm_sched_entity *entity) while ((job = drm_sched_entity_queue_pop(entity))) { struct drm_sched_fence *s_fence = job->s_fence; + /* The caller still owns the backend module here. Keep it alive + * across every dependency callback and through the driver callback's + * return, even after the entity and scheduler have been finalized. + */ + __module_get(job->sched->ops->owner); dma_fence_get(&s_fence->finished); if (!prev || dma_fence_add_callback(prev, &job->finish_cb, diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index 21dc82c75c9ef..2f0c2ac4a0ce0 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -1422,6 +1422,15 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) int i; drm_sched_wqueue_stop(sched); + /* + * wqueue_stop() cancels work that is already queued, but the completion + * callback can pass its pause check concurrently and queue either item + * after cancel_work_sync() returns. Finalization never restarts this + * scheduler, so permanently disable and drain both work items before + * touching scheduler-owned queues or destroying the workqueue. + */ + disable_work_sync(&sched->work_run_job); + disable_work_sync(&sched->work_free_job); for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) kfree(sched->sched_rq[i]); diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index a2630df403f15..fcbaf0addff22 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -750,9 +750,7 @@ int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) msg = FIELD_PREP(APPLE_RTKIT_MGMT_STARTEP_EP, endpoint); msg |= APPLE_RTKIT_MGMT_STARTEP_FLAG; - apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_STARTEP, msg); - - return 0; + return apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_STARTEP, msg); } EXPORT_SYMBOL_GPL(apple_rtkit_start_ep); diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h index aca218734dcde..367444b74e8ba 100644 --- a/include/drm/drm_gpuvm.h +++ b/include/drm/drm_gpuvm.h @@ -1165,6 +1165,7 @@ static inline void drm_gpuva_init_from_op(struct drm_gpuva *va, va->va.range = op->va.range; va->gem.obj = op->gem.obj; va->gem.offset = op->gem.offset; + va->gem.range = op->gem.range; } /** diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index e29fb50e9f4cc..8c2b0dfe0b8e6 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -538,6 +538,14 @@ struct drm_sched_backend_ops { * drm_sched_backend_ops.free_job. */ void (*cancel_job)(struct drm_sched_job *sched_job); + + /** + * @owner: Optional module implementing these callbacks. The caller must + * retain this module through entity destruction. Detached entity-kill + * work takes its own reference before leaving that boundary and releases + * it only after free_job returns to the scheduler core. + */ + struct module *owner; }; /** diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index fb6baa0a6bbef..6c2cd7459f2b0 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -111,7 +111,7 @@ impl Device { fops: &Self::GEM_FOPS, }; - const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(); + const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(T::MODULE); /// Create a new `drm::Device` for a `drm::Driver`. pub fn new(dev: &device::Device, data: impl PinInit) -> Result> { @@ -210,13 +210,12 @@ impl Device { // SAFETY: `ptr` is a valid pointer to a `struct drm_device` and embedded in `Self`. let this = unsafe { Self::from_drm_device(ptr) }; - // SAFETY: - // - When `release` runs it is guaranteed that there is no further access to `this`. - // - `this` is valid for dropping. - // unsafe { core::ptr::drop_in_place(this) }; - // HACK: data might be uninitialized so leak the DRM device instead. The expected number - // of times the asahi device gets released is once at poweroff or reboot. - let _ = core::mem::ManuallyDrop::new(this); + // SAFETY: new() installs this callback only after data.__pinned_init + // succeeds. Initialization failures use the temporary release=None + // table instead. The final DRM reference excludes further data users. + // Drop only the Rust data: drm_dev_release still owns the embedded C + // device and subsequently runs its managed cleanup and final kfree. + unsafe { ptr::drop_in_place(ptr::addr_of_mut!((*this).data)) }; } } diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 18a8a90cf5264..e02007e3bd75c 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -122,6 +122,10 @@ pub trait Driver { /// Driver metadata const INFO: DriverInfo; + /// Module implementing the driver callbacks. Loadable drivers should provide + /// their `THIS_MODULE` so open files and exported DMA buffers retain the code. + const MODULE: Option<&'static crate::ThisModule> = None; + /// Feature flags const FEATURES: u32; diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index f766e90d2dd7f..bea304395d67c 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -469,12 +469,17 @@ impl DmaBuf { } } -pub(super) const fn create_fops() -> bindings::file_operations { +pub(super) const fn create_fops( + module: Option<&'static crate::ThisModule>, +) -> bindings::file_operations { // SAFETY: As by the type invariant, it is safe to initialize `bindings::file_operations` // zeroed. let mut fops: bindings::file_operations = unsafe { core::mem::zeroed() }; - fops.owner = core::ptr::null_mut(); + fops.owner = match module { + Some(module) => module.as_ptr(), + None => core::ptr::null_mut(), + }; fops.open = Some(bindings::drm_open); fops.release = Some(bindings::drm_release); fops.unlocked_ioctl = Some(bindings::drm_ioctl); diff --git a/rust/kernel/drm/gpuvm.rs b/rust/kernel/drm/gpuvm.rs index 3f7176a9e32cb..970109249d73c 100644 --- a/rust/kernel/drm/gpuvm.rs +++ b/rust/kernel/drm/gpuvm.rs @@ -21,6 +21,7 @@ use crate::{ EINVAL, ENOMEM, // }, + from_err_ptr, from_result, to_result, Error, @@ -31,12 +32,12 @@ use crate::{ ARef, AlwaysRefCounted, // }, - types::Opaque, // + types::{NotThreadSafe, Opaque}, // }; use core::cell::UnsafeCell; use core::marker::{PhantomData, PhantomPinned}; -use core::mem::{ManuallyDrop, MaybeUninit}; +use core::mem::MaybeUninit; use core::ops::{Deref, DerefMut, Range}; use core::ptr::NonNull; use pin_init; @@ -339,15 +340,31 @@ unsafe impl AlwaysRefCounted for GpuVmBo { unsafe { bindings::drm_gpuvm_bo_get(&self.bo as *const _ as *mut _) }; } - unsafe fn dec_ref(mut obj: NonNull) { - // SAFETY: drm_gpuvm_bo_put() requires holding the gpuva lock, which is the dma_resv lock by default. - // The drm_gpuvm_put function satisfies the requirements for dec_ref(). - // (We do not support custom locks yet.) + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: This owns a live BO reference. Immediate-mode GPUVA lists + // use the GEM's gpuva mutex, not its reservation. The deferred put + // releases that mutex before final GEM destruction. Keep the GPUVM + // alive across draining, which can drop its last BO-owned reference. unsafe { - let resv = (*obj.as_mut().bo.obj).resv; + let bo = &raw mut (*obj.as_ptr()).bo; + let vm = (*bo).vm; + if (*vm).flags & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE != 0 { + bindings::drm_gpuvm_get(vm); + bindings::drm_gpuvm_bo_put_deferred(bo); + bindings::drm_gpuvm_bo_deferred_cleanup(vm); + bindings::drm_gpuvm_put(vm); + return; + } + + // A final BO put also puts its GEM. Retain the GEM until after + // unlocking its reservation, which may be embedded in that GEM. + let gem = (*bo).obj; + bindings::drm_gem_object_get(gem); + let resv = (*gem).resv; bindings::dma_resv_lock(resv, core::ptr::null_mut()); - bindings::drm_gpuvm_bo_put(&mut obj.as_mut().bo); + bindings::drm_gpuvm_bo_put(bo); bindings::dma_resv_unlock(resv); + bindings::drm_gem_object_put(gem); } } } @@ -560,6 +577,25 @@ impl GpuVm { Ok(vm_ref) } + /// Lock only the VM's driver data, without allocating an exec context. + /// + /// This takes the same shared reservation as `exec_lock`, but does not lock + /// external BO reservations. The returned guard exposes only the inner + /// driver data, not GPUVA operations requiring those other reservations. + /// Callers must not already hold a reservation lock. + pub fn lock_inner(&self) -> GpuVmInnerGuard<'_, T> { + // SAFETY: The borrowed GPUVM retains r_obj and its reservation for the + // guard's lifetime. A non-interruptible single-reservation lock with no + // ww context cannot fail or allocate. No other reservation is acquired. + unsafe { + bindings::dma_resv_lock((*(*self.gpuvm()).r_obj).resv, core::ptr::null_mut()); + } + GpuVmInnerGuard { + gpuvm: self, + _not_send: NotThreadSafe, + } + } + pub fn exec_lock<'a, 'b>( &'a self, obj: Option<&'b Object>, @@ -568,43 +604,44 @@ impl GpuVm { // Do not try to lock the object if it is internal (since it is already locked). let is_ext = obj.map(|a| self.is_extobj(a)).unwrap_or(false); - let mut guard = ManuallyDrop::new(LockedGpuVm { - gpuvm: self, - // vm_exec needs to be pinned, so stick it in a Box. - vm_exec: KBox::init( - init!(bindings::drm_gpuvm_exec { - vm: self.gpuvm() as *mut _, - flags: if interruptible { - bindings::DRM_EXEC_INTERRUPTIBLE_WAIT - } else { - 0 + // vm_exec needs a stable address while locking. Construct the unlock + // guard only after success; C already finalizes the exec on error. + let mut vm_exec = KBox::init( + init!(bindings::drm_gpuvm_exec { + vm: self.gpuvm() as *mut _, + flags: if interruptible { + bindings::DRM_EXEC_INTERRUPTIBLE_WAIT + } else { + 0 + }, + // SAFETY: bindgen's `Default` for this C structure is exactly + // `MaybeUninit::zeroed().assume_init()`. Spell that out here so + // external modules do not import its unexported trait method. + exec: unsafe { MaybeUninit::::zeroed().assume_init() }, + extra: match (is_ext, obj) { + (true, Some(obj)) => bindings::drm_gpuvm_exec__bindgen_ty_1 { + fn_: Some(exec_lock_gem_object), + priv_: obj.as_raw() as *const _ as *mut _, }, - // SAFETY: bindgen's `Default` for this C structure is exactly - // `MaybeUninit::zeroed().assume_init()`. Spell that out here so - // external modules do not import its unexported trait method. - exec: unsafe { MaybeUninit::::zeroed().assume_init() }, - extra: match (is_ext, obj) { - (true, Some(obj)) => bindings::drm_gpuvm_exec__bindgen_ty_1 { - fn_: Some(exec_lock_gem_object), - priv_: obj.as_raw() as *const _ as *mut _, - }, - // SAFETY: as above, this bindgen C structure is zero-valid. - _ => unsafe { - MaybeUninit::::zeroed() - .assume_init() - }, + // SAFETY: as above, this bindgen C structure is zero-valid. + _ => unsafe { + MaybeUninit::::zeroed() + .assume_init() }, - num_fences: 0, - }), - GFP_KERNEL, - )?, - objects: LockedObjects::Single(obj), - }); + }, + num_fences: 0, + }), + GFP_KERNEL, + )?; // SAFETY: The object is valid and was initialized above - to_result(unsafe { bindings::drm_gpuvm_exec_lock(&mut *guard.vm_exec) })?; + to_result(unsafe { bindings::drm_gpuvm_exec_lock(&mut *vm_exec) })?; - Ok(ManuallyDrop::into_inner(guard)) + Ok(LockedGpuVm { + gpuvm: self, + vm_exec, + objects: LockedObjects::Single(obj), + }) } /// Lock the GPUVM and an array of additional GEM objects in one @@ -620,44 +657,45 @@ impl GpuVm { raw_objects.push(object.as_raw(), GFP_KERNEL)?; } - let mut guard = ManuallyDrop::new(LockedGpuVm { - gpuvm: self, - // vm_exec needs to be pinned, so stick it in a Box. - vm_exec: KBox::init( - init!(bindings::drm_gpuvm_exec { - vm: self.gpuvm() as *mut _, - flags: (if interruptible { - bindings::DRM_EXEC_INTERRUPTIBLE_WAIT - } else { - 0 - }) | bindings::DRM_EXEC_IGNORE_DUPLICATES, - // SAFETY: both bindgen C structures are zero-valid; their generated - // `Default` methods perform the same zero initialization but are not - // exported from the kernel crate to loadable modules. - exec: unsafe { MaybeUninit::::zeroed().assume_init() }, - extra: unsafe { - MaybeUninit::::zeroed() - .assume_init() - }, - num_fences: 0, - }), - GFP_KERNEL, - )?, - objects: LockedObjects::Array(objects), - }); + // vm_exec needs a stable address while locking. Construct the unlock + // guard only after success; C already finalizes the exec on error. + let mut vm_exec = KBox::init( + init!(bindings::drm_gpuvm_exec { + vm: self.gpuvm() as *mut _, + flags: (if interruptible { + bindings::DRM_EXEC_INTERRUPTIBLE_WAIT + } else { + 0 + }) | bindings::DRM_EXEC_IGNORE_DUPLICATES, + // SAFETY: both bindgen C structures are zero-valid; their generated + // `Default` methods perform the same zero initialization but are not + // exported from the kernel crate to loadable modules. + exec: unsafe { MaybeUninit::::zeroed().assume_init() }, + extra: unsafe { + MaybeUninit::::zeroed() + .assume_init() + }, + num_fences: 0, + }), + GFP_KERNEL, + )?; // SAFETY: Every raw pointer is backed by an ARef in `objects`, which // outlives the returned guard. The helper consumes the temporary // pointer array before returning and keeps its own object references. to_result(unsafe { bindings::drm_gpuvm_exec_lock_array( - &mut *guard.vm_exec, + &mut *vm_exec, raw_objects.as_mut_ptr(), count, ) })?; - Ok(ManuallyDrop::into_inner(guard)) + Ok(LockedGpuVm { + gpuvm: self, + vm_exec, + objects: LockedObjects::Array(objects), + }) } /// Returns true if the given object is external to the GPUVM @@ -700,6 +738,7 @@ impl GpuVm { ) }; obj.unlock_gpuva(); + let p = from_err_ptr(p)?; if p.is_null() { Err(ENOMEM) } else { @@ -755,6 +794,82 @@ impl LockedObjects<'_, T> { } } +/// Access to VM driver data under its shared reservation, without external BO locks. +pub struct GpuVmInnerGuard<'a, T: DriverGpuVm> { + gpuvm: &'a GpuVm, + // Reservation locks must be released by the task that acquired them. + _not_send: NotThreadSafe, +} + +impl GpuVmInnerGuard<'_, T> { + /// Remove every driver GPUVA in an immediate-mode VM, preserving its kernel cutout. + /// + /// `unmap` must remove the GPU translation for each complete GPUVA before + /// returning success. Only the VM's shared reservation is held. No external + /// BO reservations are acquired and no split nodes or operation lists are + /// allocated. The callback cannot retain the borrowed GPUVA. + /// + /// Call `GpuVm::bo_deferred_cleanup` after releasing this guard, including + /// when a callback fails after earlier mappings have already been removed. + pub fn unmap_all( + &mut self, + mut unmap: impl FnMut(&mut T, &GpuVa) -> Result, + ) -> Result { + let vm = self.gpuvm.gpuvm() as *mut bindings::drm_gpuvm; + // SAFETY: The guard owns this live VM's shared reservation. Immediate + // mode lets unlink take the GEM GPUVA mutex independently of its resv. + unsafe { + if (*vm).flags & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 { + return Err(EINVAL); + } + let head = &raw mut (*vm).rb.list; + let mut entry = (*head).next; + while entry != head { + let va = crate::container_of!(entry, bindings::drm_gpuva, rb.entry) as *mut _; + // Save the successor before removing and freeing this GPUVA. + entry = (*entry).next; + // This node is embedded in the C GPUVM, not a driver GpuVa. + if core::ptr::eq(va, &raw const (*vm).kernel_alloc_node) { + continue; + } + let driver_va = crate::container_of!(va, GpuVa, gpuva); + unmap(&mut *self.gpuvm.inner.get(), &*driver_va)?; + let mut op = OpUnMap( + bindings::drm_gpuva_op_unmap { va, keep: false }, + PhantomData::, + ); + drop(op.unmap_and_unlink_va_defer()); + } + } + Ok(()) + } +} + +impl Deref for GpuVmInnerGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: The shared reservation excludes every other inner-data guard, + // including LockedGpuVm and its UpdatingGpuVm callbacks. + unsafe { &*self.gpuvm.inner.get() } + } +} + +impl DerefMut for GpuVmInnerGuard<'_, T> { + fn deref_mut(&mut self) -> &mut T { + // SAFETY: As above, with exclusive access to this guard. + unsafe { &mut *self.gpuvm.inner.get() } + } +} + +impl Drop for GpuVmInnerGuard<'_, T> { + fn drop(&mut self) { + // SAFETY: This non-Send guard owns the reservation on this task, and + // its GPUVM borrow keeps r_obj alive until after the unlock. + unsafe { bindings::dma_resv_unlock((*(*self.gpuvm.gpuvm()).r_obj).resv) }; + } +} + pub struct LockedGpuVm<'a, 'b, T: DriverGpuVm> { gpuvm: &'a GpuVm, vm_exec: KBox, @@ -791,6 +906,7 @@ impl LockedGpuVm<'_, '_, T> { obj.as_raw() as *const _ as *mut _, ) }; + let p = from_err_ptr(p)?; if p.is_null() { Err(ENOMEM) } else { diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index 90bd7142fe192..b27ab32602698 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -43,6 +43,10 @@ pub enum Priority { /// Trait to be implemented by driver job objects. pub trait JobImpl: Sized { + /// Module implementing the callbacks. Its caller must retain the module + /// through entity destruction; the core pins detached kill-job callbacks. + const MODULE: Option<&'static crate::ThisModule> = None; + /// Handle a scoped timeout without resetting the C scheduler. /// /// The backend must settle/quarantine its exact hardware work, or retain @@ -468,6 +472,10 @@ impl Scheduler { timedout_job: Some(timedout_job_cb::), free_job: Some(free_job_cb::), cancel_job: Some(cancel_job_cb::), + owner: match T::MODULE { + Some(module) => module.as_ptr(), + None => core::ptr::null_mut(), + }, }; /// Creates a new DRM Scheduler object // TODO: Shared timeout workqueues & scores From 9bd39aa56b769ebdb64da019b674c16893a1da13 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 22:58:19 -0400 Subject: [PATCH 0632/1579] rust: expose the DRM duplicate-reservation flag used by GPUVM Expose DRM_EXEC_IGNORE_DUPLICATES through the constant helper so the GPUVM array-lock wrapper builds against the public kernel bindings. The current-kernel integration already provided this helper; the public aurora-wip base did not. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 4a577fcddddbefec7d13bfd4ae3503893602d29b) --- rust/bindings/bindings_helper.h | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 3f8ebdaf0c35e..d819681a1e9c6 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -141,6 +141,7 @@ const gfp_t RUST_CONST_HELPER___GFP_HIGHMEM = ___GFP_HIGHMEM; const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; const uint32_t RUST_CONST_HELPER_DRM_EXEC_INTERRUPTIBLE_WAIT = DRM_EXEC_INTERRUPTIBLE_WAIT; +const uint32_t RUST_CONST_HELPER_DRM_EXEC_IGNORE_DUPLICATES = DRM_EXEC_IGNORE_DUPLICATES; const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; const u32 RUST_CONST_HELPER_APPLE_RTKIT_COPROC_PLACES_BUFFERS = APPLE_RTKIT_COPROC_PLACES_BUFFERS; From ceb8f96ba8232067893d3956bcc4c335bda7e268 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 23:36:52 -0400 Subject: [PATCH 0633/1579] drm/asahi: order render timestamp publication independently of GPU retirement Physical render pairs can be observed out of logical order. When a command waits through an earlier render timestamp producer, defer its host writes and public completion until that producer's host publication has finished. Preallocate forward-only publication links before accepting the job and install them before early publication or scheduler push. Process ready successors iteratively without allocating or waiting in completion. Keep scheduler credits independent: commands with host predecessors use a separate terminal fence as their scheduler parent. Healthy retirement still releases physical resources immediately, while failed-owned commands keep their pins until a retirement witness. Cancellation and physical dependency snapshots use that terminal fence. Ordinary scheduled-fence readiness and compute timestamp dependencies retain their existing behavior. Fifteen extracted-production host tests pass, including the original reverse observation failure, allocation/cancellation paths, racing registration and completion, and a 12000-node chain on a 64 KiB host stack. Full AArch64 16K Image/modules/dtbs builds with 1591 modules and 1805 DTBs. Test harnesses and receipts remain outside the source tree. Hardware and performance validation are still outstanding; no queue-depth reduction or GPU serialization is added. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d9b7b82975e2dad4b9a5941ade2f4debadd61c6c) --- drivers/gpu/drm/asahi/g17/completion.rs | 123 ++++++++--- .../drm/asahi/g17/completion/publication.rs | 209 ++++++++++++++++++ drivers/gpu/drm/asahi/g17/job.rs | 15 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 17 +- .../asahi/g17/queue/render/dependencies.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 4 +- .../drm/asahi/g17/runtime/render/settle.rs | 2 +- 7 files changed, 337 insertions(+), 35 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/completion/publication.rs diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index f137a7e3434de..0c877743873d1 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -23,6 +23,9 @@ use kernel::{ uapi, xarray, }; +mod publication; +pub(crate) use publication::{Prepared as PreparedPublication, Publication}; + struct Destination { // Keep the existing GPU alias alive along with its prepared CPU view. _mapping: Arc, @@ -51,7 +54,7 @@ impl Destination { Ok(Some(Self { _mapping: mapping, cpu })) } - fn write(&self, ticks: u64) -> Result { + fn write(&self, ticks: u64) { const NSEC_PER_SEC: u64 = 1_000_000_000; let ns = (ticks / cfg::COMMAND_TIMESTAMP_HZ) .wrapping_mul(NSEC_PER_SEC) @@ -60,7 +63,6 @@ impl Destination { ); self.cpu.write(ns); barrier(Ordering::Release); - Ok(()) } } @@ -97,6 +99,51 @@ impl Destinations { } } +/// Public outputs do not own physical command resources or a scheduler job. +struct Output { + fence: UserFence, + member: Member, + destinations: Destinations, +} + +impl Output { + fn publish(&self, result: Result<[u64; 4]>) { + match result { + Ok(ticks) => { + for (destination, ticks) in self.destinations.iter().zip(ticks) { + if let Some(destination) = destination { + destination.write(ticks); + } + } + } + Err(error) => self.fence.set_error(error), + } + self.fence.signal(); + self.member.complete(result.map(|_| ())); + } +} + +enum HostOutput { + Immediate(Output), + Ordered(Arc), +} + +impl HostOutput { + fn output(&self) -> &Output { + match self { + Self::Immediate(output) => output, + Self::Ordered(publication) => publication.output(), + } + } + + fn finish(&self, result: Result<[u64; 4]>) { + match self { + Self::Immediate(output) => output.publish(result), + Self::Ordered(publication) => publication.finish(result), + } + } +} + /// Firmware writes two aligned words in a fresh write-combined page per compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. @@ -120,9 +167,10 @@ impl ComputeTimestamps { #[pin_data] pub(crate) struct Completion { status: Arc, - fence: UserFence, - member: Member, - destinations: Destinations, + output: HostOutput, + // Present only when this command's public result may wait for a render + // predecessor's host writes. DRM credits must follow the terminal result. + terminal: Option>, compute: Option, feed: Option>, #[pin] @@ -148,9 +196,23 @@ impl Completion { vm_job: mmu::VmJobGuard, destinations: Destinations, compute: bool, + host_predecessors: bool, feed: Option>, ) -> Result> { let fence = fence::independent(contexts)?; + let ordered = !compute + && (host_predecessors || destinations.iter().any(Option::is_some)); + let terminal = if host_predecessors { + Some(fence::independent(contexts)?) + } else { + None + }; + let output = Output { fence, member, destinations }; + let output = if ordered { + HostOutput::Ordered(Publication::new(output, context.status().clone())?) + } else { + HostOutput::Immediate(output) + }; let compute = if compute { Some(ComputeTimestamps::new(dev, context.vm())?) } else { @@ -159,9 +221,8 @@ impl Completion { Arc::pin_init( pin_init!(Self { status: context.status().clone(), - fence, - member, - destinations, + output, + terminal, compute, feed, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), @@ -180,7 +241,24 @@ impl Completion { } pub(crate) fn fence(&self) -> Fence { - Fence::from_fence(&self.fence) + Fence::from_fence(&self.output.output().fence) + } + pub(crate) fn scheduler_fence(&self) -> Fence { + self.terminal.as_ref().map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) + } + pub(crate) fn publication(&self) -> Option> { + match &self.output { + HostOutput::Ordered(publication) => Some(publication.clone()), + HostOutput::Immediate(_) => None, + } + } + fn signal_terminal(&self, result: Result<[u64; 4]>) { + if let Some(fence) = &self.terminal { + if let Err(error) = result { + fence.set_error(error); + } + fence.signal(); + } } /// All fallible scheduler setup has succeeded. Failures from this point /// must be visible through VM_STATUS before the accepted job's fence. @@ -191,10 +269,10 @@ impl Completion { &self.status } pub(crate) fn work_state(&self) -> Result> { - self.member.work_state() + self.output.output().member.work_state() } pub(crate) fn has_timestamps(&self) -> bool { - self.destinations.iter().any(Option::is_some) + self.output.output().destinations.iter().any(Option::is_some) } pub(crate) fn compute_timestamp_va(&self) -> Result { @@ -360,25 +438,16 @@ impl Completion { if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { feed.note_render_pass(ticks[2], ticks[3]); } - let result = result.and_then(|ticks| { - for (destination, ticks) in self.destinations.iter().zip(ticks) { - if let Some(destination) = destination { - destination.write(ticks)?; - } - } - Ok(()) - }); if let Err(error) = result { if self.spared() { self.status.report_failure(error); } else if error != ECANCELED { self.status.record(error); } - self.fence.set_error(error); } self.release_healthy(); - self.fence.signal(); - self.member.complete(result); + self.signal_terminal(result); + self.output.finish(result); } /// A rejected, never-published command cannot poison the VM for contention. @@ -390,10 +459,9 @@ impl Completion { if self.accepted.load(Ordering::Acquire) { self.status.report_failure(error); } - self.fence.set_error(error); self.release_healthy(); - self.fence.signal(); - self.member.complete(Err(error)); + self.signal_terminal(Err(error)); + self.output.finish(Err(error)); } /// The physical queue must already be quarantined. Keep mappings pinned while @@ -410,9 +478,8 @@ impl Completion { } else if error != ECANCELED { self.status.record(error); } - self.fence.set_error(error); - self.fence.signal(); - self.member.complete(Err(error)); + self.signal_terminal(Err(error)); + self.output.finish(Err(error)); } } diff --git a/drivers/gpu/drm/asahi/g17/completion/publication.rs b/drivers/gpu/drm/asahi/g17/completion/publication.rs new file mode 100644 index 0000000000000..a1e05a9408552 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/completion/publication.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Host-only ordering of timestamp writes, independent of scheduler credits. +//! +//! Links point from earlier logical render ordinals to later ones. They retain +//! only public output state, never a packet, physical render lease or VM job pin. +//! All link storage is allocated before acceptance. Completion neither allocates +//! nor waits for another command; the last prerequisite publishes a ready result. + +use super::Output; +use crate::g17::{dependency::inherited_error, status::VmStatus}; +use kernel::{new_mutex, prelude::*, sync::{Arc, Mutex}}; + +type Links = Option>; + +struct Link { + target: Arc, + result: Result, + next: Links, + // Reuse an already-consumed link as a traversal frame instead of recursing + // or allocating when publication unlocks a long chain of ready successors. + resume: Links, +} + +struct State { + remaining: usize, + result: Option>, + predecessor_error: Option, + claimed: bool, + published: Option, + successors: Links, +} + +impl State { + fn claim(&mut self) -> Option> { + if self.remaining != 0 || self.claimed { + return None; + } + let result = self.result.take()?; + self.claimed = true; + Some(match (result, self.predecessor_error) { + (Ok(_), Some(error)) => Err(error), + (result, _) => result, + }) + } +} + +#[pin_data] +pub(crate) struct Publication { + output: Output, + status: Arc, + #[pin] + state: Mutex, +} + +impl Publication { + pub(super) fn new(output: Output, status: Arc) -> Result> { + Arc::pin_init( + pin_init!(Self { + output, + status, + state <- new_mutex!(State { + // Acceptance installs all links before releasing this sentinel. + remaining: 1, + result: None, + predecessor_error: None, + claimed: false, + published: None, + successors: None, + }, "G17 host timestamp publication"), + }), + GFP_KERNEL, + ) + } + + pub(super) fn output(&self) -> &Output { + &self.output + } + + /// One terminal result, offered separately from its physical-retirement proof. + pub(super) fn finish(&self, result: Result<[u64; 4]>) { + let ready = { + let mut state = self.state.lock(); + if state.claimed || state.result.is_some() { + return; + } + state.result = Some(result); + state.claim() + }; + if let Some(result) = ready { + Self::drain(self.publish(result)); + } + } + + fn publish(&self, result: Result<[u64; 4]>) -> Links { + if let Err(error) = result { + // Inherited failure is visible to the client without inventing a + // permanent admission failure in this command's VM. + self.status.report_failure(error); + } + self.output.publish(result); + let result = result.map(|_| ()); + let mut links = { + let mut state = self.state.lock(); + // A late link is acknowledged only after timestamp writes, the public + // fence and aggregate membership have all been published off this lock. + state.published = Some(result); + state.successors.take() + }; + let mut next = links.as_mut(); + while let Some(link) = next { + link.result = result; + next = link.next.as_mut(); + } + links + } + + fn predecessor_done(&self, result: Result) -> Links { + let ready = { + let mut state = self.state.lock(); + debug_assert!(state.remaining != 0); + if let Err(error) = result { + if state.predecessor_error.is_none() { + state.predecessor_error = Some(inherited_error(error)); + } + } + state.remaining -= 1; + state.claim() + }; + ready.and_then(|result| self.publish(result)) + } + + fn add_successor(&self, mut link: KBox) { + let published = { + let mut state = self.state.lock(); + match state.published { + Some(result) => result, + None => { + link.next = state.successors.take(); + state.successors = Some(link); + return; + } + } + }; + Self::drain(link.target.predecessor_done(published)); + } + + /// Iterative depth-first notification, O(nodes + links), with no stack growth. + fn drain(mut pending: Links) { + let mut frames: Links = None; + loop { + if let Some(mut link) = pending.take() { + pending = link.next.take(); + let children = link.target.predecessor_done(link.result); + if children.is_some() { + link.resume = pending.take(); + link.next = frames.take(); + frames = Some(link); + pending = children; + } + } else if let Some(mut frame) = frames.take() { + frames = frame.next.take(); + pending = frame.resume.take(); + } else { + break; + } + } + } +} + +/// Temporary backwards references exist only during fallible setup. Installed +/// links retain the successor only, so closing a queue cannot form an Arc cycle. +pub(crate) struct Prepared { + target: Arc, + links: KVec<(Arc, KBox)>, +} + +impl Prepared { + pub(crate) fn new<'a>( + target: Arc, + predecessors: impl Iterator> + Clone, + ) -> Result { + let mut links = KVec::with_capacity(predecessors.clone().count(), GFP_KERNEL)?; + for predecessor in predecessors { + let link = KBox::new(Link { + target: target.clone(), + result: Ok(()), + next: None, + resume: None, + }, GFP_KERNEL)?; + links.push((predecessor.clone(), link), GFP_KERNEL)?; + } + Ok(Self { target, links }) + } + + /// Called after all fallible scheduler setup and before any GPU publication. + pub(crate) fn activate(self) { + { + let mut state = self.target.state.lock(); + debug_assert!(state.remaining == 1 && !state.claimed); + // The fully allocated link vector bounds this count below usize::MAX. + state.remaining += self.links.len(); + } + for (predecessor, link) in self.links { + predecessor.add_successor(link); + } + Publication::drain(self.target.predecessor_done(Ok(()))); + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index b25db8e00f167..92123312bc932 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -316,6 +316,7 @@ pub(crate) struct Scheduler { pub(crate) struct Fences { pub(crate) ready: Fence, pub(crate) completed: Fence, + pub(crate) publication: Option>, } impl Scheduler { @@ -338,9 +339,18 @@ impl Scheduler { packet: Arc, dependencies: KVec, early: bool, + publication: Option, ) -> Result { let completed = packet.completion.fence(); let host_timestamps = packet.completion.has_timestamps(); + // Only commands which actually write timestamps become host-prefix + // producers. Ordinary consumers retain the scheduled ready fence and + // their existing firmware dependency path for cross-engine overlap. + let host_publication = if host_timestamps { + packet.completion.publication() + } else { + None + }; let mut job = self.entity.new_job( 1, Job { @@ -354,6 +364,9 @@ impl Scheduler { } let mut job = job.arm(); packet.completion.mark_accepted(); + if let Some(publication) = publication { + publication.activate(); + } let ready = if host_timestamps { completed.clone() } else { @@ -365,6 +378,6 @@ impl Scheduler { self.backend.count_backlog(&packet); } job.push(); - Ok(Fences { ready, completed }) + Ok(Fences { ready, completed, publication: host_publication }) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 78765523254b6..5f535a8321cf4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -8,7 +8,7 @@ pub(crate) mod render; use super::{ command::{Parser, UscWindow, Validated}, - completion::{Completion, Destinations}, + completion::{Completion, Destinations, PreparedPublication}, context::Context, fence::{Outputs, Submission}, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, @@ -176,6 +176,15 @@ impl Queue { if engine == Engine::Compute { self.ensure_compute()?; } + // Only earlier render ordinals covered by the explicit logical prefix + // can delay host publication. These links never become GPU dependencies. + let host_predecessors = self.frontiers[index] + .iter() + .take_while(|(sequence, _)| { + engine == Engine::Render + && Order::contains(order.wait_through[index], *sequence) + }) + .filter_map(|(_, fences)| fences.publication.as_ref()); let completion = Completion::new( &self.dev, &self.fences, @@ -184,8 +193,12 @@ impl Queue { guard, destinations, engine == Engine::Compute, + host_predecessors.clone().next().is_some(), if engine == Engine::Render { self.backend.feed() } else { None }, )?; + let publication = completion.publication() + .map(|target| PreparedPublication::new(target, host_predecessors)) + .transpose()?; if engine == Engine::Compute { self.prune_frontier(Engine::Render.index()); } @@ -275,7 +288,7 @@ impl Queue { let fences = scheduler .as_mut() .ok_or(ENODEV)? - .enqueue(packet, dependencies, early)?; + .enqueue(packet, dependencies, early, publication)?; self.sequence[index] = order.sequence; self.frontiers[index] .push_within_capacity((order.sequence, fences)) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs index d1ec0eb76c7e1..be1d9fc9c902c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs @@ -40,7 +40,7 @@ fn include( frontier.include( prefix, packet.order.sequence, - job::fence_status(&packet.completion.fence()), + job::fence_status(&packet.completion.scheduler_fence()), word, ) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 3fb40779021e5..7f204b9328ffb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -41,7 +41,7 @@ impl Backend { Err(error) } else { deferred.finish(); - Ok(packet.completion.fence()) + Ok(packet.completion.scheduler_fence()) } } @@ -115,7 +115,7 @@ impl job::Backend for Backend { fn publish(&self, packet: Arc) -> Result { if let Some(result) = packet.early_result() { - return result.map(|()| packet.completion.fence()); + return result.map(|()| packet.completion.scheduler_fence()); } let mut deferred = match DeferredBatch::publication() { Ok(batch) => batch, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 2cf7a5d77ee5a..0a4634bccaea5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -26,7 +26,7 @@ impl Backend { result } pub(in crate::g17) fn cancel_render(&self, packet: &Arc) -> Result { - let fence = packet.completion.fence(); + let fence = packet.completion.scheduler_fence(); // SAFETY: The retained fence remains alive across this bounded, // uninterruptible queue-teardown wait, without holding a device lock. let completed = unsafe { From 3bf578fd8958a40af5ab8b3ea892749a1dc5b4bf Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:07 -0400 Subject: [PATCH 0634/1579] drm/sched: separate Rust job timeout state from submission state drm_sched_job_begin() arms the timeout before run_job() returns, so the timeout worker can run while the submit worker still owns the job. Both Rust JobImpl callbacks exclusively borrowed the whole Job. Give the timeout worker its own TimeoutData field, pass the finished status explicitly, and keep run/prepare/cancel on the submission half only. Let a backend that returns ENODEV without reset recovery keep the detached job owned by the scheduler (retain_job_on_enodev) so it reaches normal free work after its parent completes instead of being orphaned. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e22376a0ca679b5359fdef009b2479616619677b) --- drivers/gpu/drm/asahi/g17/job.rs | 29 ++++++++++----- drivers/gpu/drm/asahi/queue/mod.rs | 26 ++++++++------ drivers/gpu/drm/scheduler/sched_main.c | 16 +++++++-- include/drm/gpu_scheduler.h | 13 +++++++ rust/kernel/drm/sched.rs | 49 ++++++++++++++++++-------- 5 files changed, 97 insertions(+), 36 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 92123312bc932..34b6308620644 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -255,6 +255,11 @@ pub(crate) trait Backend: Send + Sync { } struct Job { + backend: Arc, + packet: Arc, +} + +struct Timeout { backend: Arc, victim_renewals: u8, packet: Arc, @@ -272,10 +277,19 @@ impl Drop for Job { } impl sched::JobImpl for Job { + type TimeoutData = Timeout; const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); - fn false_timeout(job: &mut sched::Job) -> bool { - if job.is_finished() || job.packet.completion.take_replay_timeout() { + fn timeout_data(&self) -> Timeout { + Timeout { + backend: self.backend.clone(), + victim_renewals: 0, + packet: self.packet.clone(), + } + } + + fn false_timeout(job: &mut Timeout, finished: bool) -> bool { + if finished || job.packet.completion.take_replay_timeout() { return true; } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); @@ -285,22 +299,22 @@ impl sched::JobImpl for Job { deferred } - fn prepare(job: &mut sched::Job) -> Option { + fn prepare(job: &mut Self) -> Option { job.backend.prepare(&job.packet) } - fn run(job: &mut sched::Job) -> Result> { + fn run(job: &mut Self) -> Result> { job.backend.publish(job.packet.clone()).map(Some) } - fn timed_out(job: &mut sched::Job) -> sched::Status { + fn timed_out(job: &mut Timeout, finished: bool) -> sched::Status { // The scheduler retains the detached job throughout this callback. Keep // its parent callback intact while the driver settles the exact owner; // reinsertion also handles completion racing with timeout observation. - if !job.is_finished() { + if !finished { job.backend.timed_out(job.packet.clone()); } sched::Status::NoHang } - fn cancel(job: &mut sched::Job) { + fn cancel(job: &mut Self) { job.backend.cancel(job.packet.clone()); } } @@ -355,7 +369,6 @@ impl Scheduler { 1, Job { backend: self.backend.clone(), - victim_renewals: 0, packet: packet.clone(), }, )?; diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 6fcb901aa4d2c..7bdf3c50ebbc0 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -196,7 +196,11 @@ pub(crate) struct QueueJob { fence: UserFence, notifier: Arc>, notification_count: u32, - did_run: bool, + id: u64, +} + +pub(crate) struct QueueTimeout { + dev: AsahiDevRef, id: u64, } @@ -244,9 +248,14 @@ impl QueueJob::ver { #[versions(AGX)] impl sched::JobImpl for QueueJob::ver { + type TimeoutData = QueueTimeout; const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); - fn prepare(job: &mut sched::Job) -> Option { + fn timeout_data(&self) -> QueueTimeout { + QueueTimeout { dev: self.dev.clone(), id: self.id } + } + + fn prepare(job: &mut Self) -> Option { mod_dev_dbg!(job.dev, "QueueJob {}: Checking runnability\n", job.id); if let Some(sj) = job.sj_vtx.as_ref() { @@ -283,7 +292,7 @@ impl sched::JobImpl for QueueJob::ver { } #[allow(unused_assignments)] - fn run(job: &mut sched::Job) -> Result> { + fn run(job: &mut Self) -> Result> { mod_dev_dbg!(job.dev, "QueueJob {}: Running Job\n", job.id); // We can only increase the notifier threshold here, now that we are @@ -403,22 +412,20 @@ impl sched::JobImpl for QueueJob::ver { mod_dev_dbg!(job.dev, "QueueJob {}: Drop fragment job\n", job.id); core::mem::drop(frag_job); - job.did_run = true; - Ok(Some(Fence::from_fence(&job.fence))) } - fn timed_out(job: &mut sched::Job) -> sched::Status { + fn timed_out(job: &mut QueueTimeout, _finished: bool) -> sched::Status { // FIXME: Handle timeouts properly dev_err!( job.dev.as_ref(), - "QueueJob {}: Job timed out on the DRM scheduler, things will probably break (ran: {})\n", - job.id, job.did_run + "QueueJob {}: Job timed out on the DRM scheduler, things will probably break\n", + job.id ); sched::Status::NoDevice } - fn cancel(job: &mut sched::Job) { + fn cancel(job: &mut Self) { dev_info!( job.dev.as_ref(), "QueueJob {}: Job canceled on DRM scheduler teardown\n", @@ -748,7 +755,6 @@ impl Queue for Queue::ver { // notification count for the job. notification_count: (2 * nr_render) + nr_compute, - did_run: false, id, }, )?; diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index 2f0c2ac4a0ce0..c962265b79b02 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -566,10 +566,20 @@ static void drm_sched_job_timedout(struct work_struct *work) if (sched->free_guilty) { job->sched->ops->free_job(job); sched->free_guilty = false; - } - - if (status == DRM_GPU_SCHED_STAT_NO_HANG) + } else if (status == DRM_GPU_SCHED_STAT_NO_HANG) { drm_sched_job_reinsert_on_false_timeout(sched, job); + } else if (status == DRM_GPU_SCHED_STAT_ENODEV && + sched->ops->retain_job_on_enodev) { + /* Preserve ownership without changing the backend's terminal + * result into NO_HANG or restarting its watchdog. A parent + * signaled during the callback must also reach free work. + */ + spin_lock(&sched->job_list_lock); + if (list_empty(&job->list)) + list_add(&job->list, &sched->pending_list); + drm_sched_run_free_queue(sched); + spin_unlock(&sched->job_list_lock); + } } else { spin_unlock(&sched->job_list_lock); } diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 8c2b0dfe0b8e6..7af53c62078bb 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -546,6 +546,19 @@ struct drm_sched_backend_ops { * it only after free_job returns to the scheduler core. */ struct module *owner; + + /** + * @retain_job_on_enodev: Keep the detached timeout job owned by the + * scheduler when timedout_job returns ENODEV without reset recovery. + * The backend must not free that job itself. Reinsertion happens only + * after the callback returns, so its parent may complete concurrently + * without freeing storage still borrowed by the callback. This does not + * rearm the timeout, signal a hardware fence or establish GPU quiescence. + * Completed jobs use normal free work; unfinished jobs remain reachable + * by final cancellation. Backends using their own reset ownership leave + * this false. + */ + bool retain_job_on_enodev; }; /** diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index b27ab32602698..c5a4992ba3006 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -43,6 +43,13 @@ pub enum Priority { /// Trait to be implemented by driver job objects. pub trait JobImpl: Sized { + /// State owned exclusively by the timeout worker. It must not borrow the + /// mutable submission state: timeout can run before `run` returns. + type TimeoutData; + + /// Snapshot or independently retain timeout state before scheduler ownership. + fn timeout_data(&self) -> Self::TimeoutData; + /// Module implementing the callbacks. Its caller must retain the module /// through entity destruction; the core pins detached kill-job callbacks. const MODULE: Option<&'static crate::ThisModule> = None; @@ -59,28 +66,28 @@ pub trait JobImpl: Sized { /// Reconsider a timeout before stopping the scheduler or settling fences. /// Returning true asks the core to reinsert the retained job and rearm. /// The backend must not stop/start the scheduler from this hook. - fn false_timeout(_job: &mut Job) -> bool { false } + fn false_timeout(_data: &mut Self::TimeoutData, _finished: bool) -> bool { false } /// Called when the scheduler is considering scheduling this job next, to get another Fence /// for this job to block on. Once it returns None, run() may be called. - fn prepare(_job: &mut Job) -> Option { + fn prepare(_job: &mut Self) -> Option { None // Equivalent to NULL function pointer } /// Called to execute the job once all of the dependencies have been resolved. This may be /// called multiple times, if timed_out() has happened and drm_sched_job_recovery() decides /// to try it again. - fn run(job: &mut Job) -> Result>; + fn run(job: &mut Self) -> Result>; /// Called when a job has taken too long to execute, to trigger GPU recovery. /// /// This method is called in a workqueue context. - fn timed_out(job: &mut Job) -> Status; + fn timed_out(data: &mut Self::TimeoutData, finished: bool) -> Status; /// Called for remaining jobs in drm_sched_fini() to ensure the job's fences /// get signalled before the scheduler is torn down. - fn cancel(job: &mut Job); + fn cancel(job: &mut Self); } unsafe extern "C" fn prepare_job_cb( @@ -90,8 +97,9 @@ unsafe extern "C" fn prepare_job_cb( // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: All of our jobs are Job. - match T::prepare(unsafe { &mut *p }) { + // SAFETY: prepare/run are serialized by submit work. Only borrow inner; + // the timeout worker independently owns timeout_data and C owns job. + match T::prepare(unsafe { &mut *addr_of_mut!((*p).inner) }) { None => core::ptr::null_mut(), Some(fence) => fence.into_raw(), } @@ -103,8 +111,9 @@ unsafe extern "C" fn run_job_cb( // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: All of our jobs are Job. - match T::run(unsafe { &mut *p }) { + // SAFETY: submit work owns inner until return, even while timeout work + // accesses the disjoint timeout_data field. Do not borrow the whole Job. + match T::run(unsafe { &mut *addr_of_mut!((*p).inner) }) { Err(e) => e.to_ptr(), Ok(None) => core::ptr::null_mut(), Ok(Some(fence)) => fence.into_raw(), @@ -119,17 +128,20 @@ unsafe extern "C" fn timedout_job_cb( // The C timeout worker owns this job outside pending_list until return. // Free work cannot release it even if its parent signals concurrently. + let finished = unsafe { addr_of_mut!((*(*sched_job).s_fence).finished) }; + let is_finished = || unsafe { bindings::dma_fence_get_status(finished) != 0 }; if T::MANAGED_TIMEOUT_RECOVERY { - let finished = unsafe { addr_of_mut!((*(*sched_job).s_fence).finished) }; - if unsafe { bindings::dma_fence_get_status(finished) } != 0 { + if is_finished() { return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; } } - // SAFETY: the C core retains the detached job across both callbacks. - if T::false_timeout(unsafe { &mut *p }) { + // SAFETY: the C core retains the detached job and serializes its timeout + // worker. This field is disjoint from inner, which run may still mutate. + let data = unsafe { &mut *addr_of_mut!((*p).timeout_data) }; + if T::false_timeout(data, is_finished()) { return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; } - let status = T::timed_out(unsafe { &mut *p }); + let status = T::timed_out(data, is_finished()); if T::MANAGED_TIMEOUT_RECOVERY { // This is scoped fence handling, not C reset recovery. stop would // detach and discard healthy parents; start would cancel those jobs. @@ -190,7 +202,7 @@ unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sch // SAFETY: cancellation cannot race a scheduler callback using this job // after the removal/parent-lock synchronization above. The backend must // retain any still-hardware-owned resources independently of Job storage. - T::cancel(unsafe { &mut *p }); + T::cancel(unsafe { &mut *addr_of_mut!((*p).inner) }); if finish_here { // Match drm_sched_job_done's accounting exactly once, only when its @@ -216,6 +228,7 @@ unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sch pub struct Job { job: bindings::drm_sched_job, inner: T, + timeout_data: T::TimeoutData, // Entity destruction can leave kill_jobs_work queued on the system // workqueue or waiting for a dependency. That work still dereferences // job->sched->ops after drm_sched_fini has stopped the scheduler workers. @@ -402,6 +415,7 @@ impl Entity { /// in flight at once. pub fn new_job(&mut self, credits: u32, inner: T) -> Result> { let mut job: KBox>> = Box::new_uninit(GFP_KERNEL | __GFP_ZERO)?; + let timeout_data = inner.timeout_data(); // SAFETY: We hold a reference to the entity (which is a valid pointer), // and the job object was just allocated above. @@ -417,6 +431,8 @@ impl Entity { // SAFETY: The Box pointer is valid, and this initializes the inner member. unsafe { addr_of_mut!((*job.as_mut_ptr()).inner).write(inner) }; + // SAFETY: initialize the independent timeout worker's owned state. + unsafe { addr_of_mut!((*job.as_mut_ptr()).timeout_data).write(timeout_data) }; // SAFETY: The job allocation is valid and this initializes its last // field. Arc::clone is infallible and the entity still owns the live @@ -472,6 +488,9 @@ impl Scheduler { timedout_job: Some(timedout_job_cb::), free_job: Some(free_job_cb::), cancel_job: Some(cancel_job_cb::), + // A terminal backend result must not orphan the detached Job. The + // core restores ownership after all Rust callback borrows end. + retain_job_on_enodev: true, owner: match T::MODULE { Some(module) => module.as_ptr(), None => core::ptr::null_mut(), From 20560fe15a23c2dd8db188a074e245c20bcf9858 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:08 -0400 Subject: [PATCH 0635/1579] drm/asahi: observe completed render work before a deferred VM failure A VM failure that arrives while preparation owns a render pair is applied later from the deferred path, which quarantined the pair at once. Retire its completed commands first, as settle_render_vm() does, before quarantining what is still owned. Keep the pending error if the observation itself fails. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 3e464649d37234b0f47dfe3c22d3f73cbb6ef45b) --- drivers/gpu/drm/asahi/g17/runtime/render/submit.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 59a54b1926375..8b6c647668af4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -206,6 +206,20 @@ impl crate::g17::Firmware { slot: u8, deferred: &mut DeferredBatch, ) -> Result { + let observe_before_failure = { + let entry = self.queues.render.entry_mut(slot)?; + let pair = entry.pair.as_ref().ok_or(EIO)?; + entry.pending_error.as_ref().is_some_and(|(status, _)| { + Arc::ptr_eq(pair.context().status(), status) + }) + }; + if observe_before_failure { + // VM failure could not observe this pair while preparation owned + // it. Restore the same final-observation boundary used by + // settle_render_vm before quarantining any still-owned commands. + // Keep pending_error intact if observation itself fails. + self.retire_render_slot(usize::from(slot), None, deferred)?; + } let entry = self.queues.render.entry_mut(slot)?; let pair = entry.pair.as_mut().ok_or(EIO)?; if let Some((status, error)) = entry.pending_error.take() { From d20cdd0dba5a51899ea79cb8ac2df9e14cd3cd8a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:08 -0400 Subject: [PATCH 0636/1579] drm/asahi: publish child page tables before linking them Complete the child table initialization before storing the parent entry, so a concurrent device walk cannot follow a visible parent to a table whose clears have not reached memory yet. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 83cf9f5ad0baf71001cc0aa9c25c83f27f4a219a) --- drivers/gpu/drm/asahi/pgtable.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 5240cba7b3fc1..03001892d98bb 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -456,6 +456,11 @@ impl UatPageTable { } let pt_paddr = Page::into_phys(pt_page); upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + // Complete child-table initialization before a + // concurrent device walk can follow its parent. + // A later mapping/TLBI barrier cannot order an + // already-visible parent against earlier clears. + crate::mem::sync(); upte.store(upte_val, Ordering::Relaxed); } if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { From da35a7e8b9502f0afb30feac9aa41b03b0b7d080 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:09 -0400 Subject: [PATCH 0637/1579] drm/asahi: retire deferred mapping cleanup without allocating Deferred user unmaps keep their preallocated split nodes and retire through immediate-mode drm_gpuvm_sm_unmap() without an exec context. Whole-object unmaps that hit ENOMEM fall back to an allocation-free removal of that GEM's nodes. A partially committed bind batch defers only its uncommitted operations, so committed prefixes are never replayed over a range another binding may now occupy. Validate unmap ranges against the VM geometry at preparation. A cleanup failure keeps admission blocked and is recorded on the VM status rather than silently reopening the VM over stale leaves. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit b321f0e7053949d9144028d0a465a4f03eb6cc83) --- drivers/gpu/drm/asahi/mmu.rs | 37 +++++++++++++-- drivers/gpu/drm/asahi/mmu/bind.rs | 15 ++++-- drivers/gpu/drm/asahi/mmu/lifetime.rs | 67 ++++++++++++++++++--------- rust/kernel/drm/gpuvm.rs | 51 ++++++++++++++++++++ 4 files changed, 143 insertions(+), 27 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 84eff788e0cf9..d47254243ce00 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2006,8 +2006,18 @@ impl Vm { self.unmap_range_commit(iova, size, true) } - fn unmap_range_now(&self, iova: u64, size: u64) -> Result { - self.unmap_range_commit(iova, size, false) + fn unmap_prepared_cleanup(&self, unmap: &mut bind::PreparedUserUnmap) -> Result { + let mut inner = self.inner.lock_inner(); + // SAFETY: step_unmap/step_remap modify this VM's translations and + // immediate GPUVA metadata only, using the retained split nodes. They + // do not access external BO backing and defer all BO reference puts. + let result = unsafe { inner.sm_unmap_inner(&mut unmap.ctx, unmap.iova, unmap.size) }; + if result.is_ok() { + self.untrack_shared_range(unmap.iova, unmap.size); + } + drop(inner); + self.bo_deferred_cleanup(); + result } fn unmap_range_commit(&self, iova: u64, size: u64, user: bool) -> Result { @@ -2026,6 +2036,7 @@ impl Vm { match self.mapping_commit() { Ok(guard) => guard, Err(error) => { + drop(inner); if self.defer_unmap(iova, size)? { return Ok(()); } @@ -2044,7 +2055,10 @@ impl Vm { // Removing whole mappings only does unmaps, so no preallocated VAs let mut ctx = Default::default(); - let inner = self.inner.exec_lock(Some(gem), false)?; + let inner = match self.inner.exec_lock(Some(gem), false) { + Err(error) if error == ENOMEM => return self.drop_mappings_low_memory(gem), + result => result?, + }; if let Some(bo) = self.inner.find_bo(gem) { mod_dev_dbg!(inner.dev, "MMU: bo_unmap\n"); @@ -2060,12 +2074,29 @@ impl Vm { // unmap error. Always release reservations before dropping it. core::mem::drop(inner); core::mem::drop(bo); + if result == Err(ENOMEM) { + return self.drop_mappings_low_memory(gem); + } result?; } Ok(()) } + /// Preserve the indexed fast path normally; memory pressure must not lose + /// accepted object cleanup just because an exec or operation list failed. + fn drop_mappings_low_memory(&self, gem: &gem::Object) -> Result { + let mut inner = self.inner.lock_inner(); + let result = inner.unmap_object(gem, VmInner::unmap_gpuva); + if result.is_ok() { + self.untrack_context_object(gem); + self.untrack_shared_object(gem); + } + drop(inner); + self.bo_deferred_cleanup(); + result + } + /// Returns the dummy GEM object used to hold the shared DMA reservation locks pub(crate) fn get_resv_obj(&self) -> ARef { self.dummy_obj.clone() diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 67d18a275ca32..d2e429c43d311 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -19,9 +19,15 @@ pub(crate) struct PreparedUserMap { pub(crate) struct PreparedUserUnmap { pub(super) iova: u64, pub(super) size: u64, - ctx: StepContext, + pub(super) ctx: StepContext, } +// SAFETY: This owns only its numeric range and unused, unlinked split-node +// allocations. step_remap moves each node out of ctx before linking it into +// the VM, so a retained ctx never owns a linked GPUVA. The unmap context has +// no borrowed object/lock state; lifetime.state serializes deferred ownership. +unsafe impl Send for PreparedUserUnmap {} + pub(super) enum PreparedUserBindOp { Map(PreparedUserMap), Unmap(PreparedUserUnmap), @@ -111,6 +117,9 @@ impl Vm { } pub(crate) fn prepare_user_unmap(&self, iova: u64, size: u64) -> Result { + if (iova | size) & UAT_PGMSK as u64 != 0 || !self.inner.range_valid(iova, size) { + return Err(EINVAL); + } Ok(PreparedUserUnmap { iova, size, @@ -128,7 +137,7 @@ impl Vm { &self, batch: &mut PreparedUserBindBatch, ) -> Result { - if self.defer_bind_batch(batch)? { + if self.defer_bind_batch(batch, 0)? { return Ok(()); } let mut index = 0; @@ -202,7 +211,7 @@ impl Vm { if let Some(result) = result { result?; index += 1; - } else if self.defer_bind_batch(batch)? { + } else if self.defer_bind_batch(batch, index)? { return Ok(()); } } diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index e33b578bb13a7..20a79037593df 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -54,8 +54,9 @@ struct State { quarantined: bool, draining: bool, closed: bool, + cleanup_failed: bool, close_ranges: Option<(Range, Range)>, - unmaps: KVec<(u64, u64)>, + unmaps: KVec, objects: KVec>, } @@ -74,7 +75,7 @@ impl VmLifetime { Arc::pin_init( pin_init!(Self { state <- new_mutex!(State { active:0, commits:0, blocked:false, - quarantined:false, draining:false, closed:false, close_ranges:None, + quarantined:false, draining:false, closed:false, cleanup_failed:false, close_ranges:None, unmaps:KVec::new(), objects:KVec::new() }, "VM job lifetime"), changed <- new_condvar!("VM mapping admission"), driver <- new_mutex!(None, "VM driver mappings"), @@ -414,7 +415,7 @@ impl Vm { Ok(()) } - pub(super) fn defer_bind_batch(&self, batch: &PreparedUserBindBatch) -> Result { + pub(super) fn defer_bind_batch(&self, batch: &mut PreparedUserBindBatch, start: usize) -> Result { use super::bind::PreparedUserBindOp; let Some(lifetime) = self.lifetime.as_ref() else { return Ok(false); @@ -428,7 +429,7 @@ impl Vm { } if batch .ops - .iter() + .iter().skip(start) .any(|op| matches!(op, PreparedUserBindOp::Map(_))) { return Err(EIO); @@ -438,19 +439,21 @@ impl Vm { } // Reserve before accepting any range; a partial deferred batch must // not leave canonical metadata ahead of the accepted cleanup. - state.unmaps.reserve(batch.ops.len(), GFP_KERNEL)?; - for op in &batch.ops { + state.unmaps.reserve(batch.ops.len() - start, GFP_KERNEL)?; + let retired = self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { + PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), + PreparedUserBindOp::Map(_) => None, + })); + // Transfer the preallocated split nodes along with each accepted range. + // Already-committed prefix operations must not be replayed: their nodes + // may have been consumed and another binding may now occupy the range. + for op in batch.ops.drain(start..) { if let PreparedUserBindOp::Unmap(unmap) = op { - let range = (unmap.iova, unmap.size); - if !state.unmaps.contains(&range) { - state.unmaps.push(range, GFP_KERNEL)?; + if !state.unmaps.iter().any(|old| old.iova == unmap.iova && old.size == unmap.size) { + state.unmaps.push(unmap, GFP_KERNEL)?; } } } - let retired = self.untrack_context_ranges(batch.ops.iter().filter_map(|op| match op { - PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), - PreparedUserBindOp::Map(_) => None, - })); drop(state); drop(retired); Ok(true) @@ -461,7 +464,7 @@ impl Vm { let Some(lifetime) = self.lifetime.as_ref() else { return Ok(false); }; - let mut state = lifetime.state.lock(); + let state = lifetime.state.lock(); if state.closed { return Err(ENOENT); } @@ -471,8 +474,17 @@ impl Vm { if !state.quarantined { return Err(EBUSY); } - if !state.unmaps.contains(&(iova, size)) { - state.unmaps.push((iova, size), GFP_KERNEL)?; + if state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { + return Ok(true); + } + drop(state); + let unmap = self.prepare_user_unmap(iova, size)?; + let mut state = lifetime.state.lock(); + if state.closed { return Err(ENOENT); } + if !state.blocked { return Ok(false); } + if !state.quarantined { return Err(EBUSY); } + if !state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { + state.unmaps.push(unmap, GFP_KERNEL)?; } Ok(true) } @@ -557,24 +569,28 @@ impl Vm { /// Recheck queued cleanup and reopen under the same lock used to append it. fn drain_mappings(&self, lifetime: &VmLifetime) { let mut state = lifetime.state.lock(); + let mut failure = None; loop { let close = state.close_ranges.take(); let unmaps = core::mem::replace(&mut state.unmaps, KVec::new()); let objects = core::mem::replace(&mut state.objects, KVec::new()); drop(state); if let Some((user, kernel)) = close { - if self.unmap_user_ranges_now(user, kernel).is_err() { + if let Err(error) = self.unmap_user_ranges_now(user, kernel) { + failure.get_or_insert(error); pr_err!("MMU: deferred VM close failed\n"); } } else { - for (address, size) in unmaps { - if self.unmap_range_now(address, size).is_err() { + for mut unmap in unmaps { + if let Err(error) = self.unmap_prepared_cleanup(&mut unmap) { + failure.get_or_insert(error); pr_err!("MMU: deferred user unmap failed\n"); } } } for object in objects { - if self.drop_mappings_now(&object).is_err() { + if let Err(error) = self.drop_mappings_now(&object) { + failure.get_or_insert(error); pr_err!("MMU: deferred object unmap failed\n"); } } @@ -585,7 +601,16 @@ impl Vm { } state.quarantined = false; state.draining = false; - if !state.closed { + if let Some(error) = failure { + // Cleanup is allocation-free for accepted ranges and has a + // whole-node low-memory fallback for objects. An unexpected + // invariant error must not reopen admission over stale leaves. + // GPUVA owners retain backing until final VM close; report the + // terminal failure instead of silently discarding its outcome. + state.blocked = true; + state.cleanup_failed = true; + if let Some(status) = &self.status { status.record(error); } + } else if !state.closed && !state.cleanup_failed { state.blocked = false; } drop(state); diff --git a/rust/kernel/drm/gpuvm.rs b/rust/kernel/drm/gpuvm.rs index 970109249d73c..5cfa57fd4fe9a 100644 --- a/rust/kernel/drm/gpuvm.rs +++ b/rust/kernel/drm/gpuvm.rs @@ -706,6 +706,12 @@ impl GpuVm { unsafe { bindings::drm_gpuvm_is_extobj(self.gpuvm() as *mut _, gem) } } + /// Validate against the immutable address-space bounds and kernel cutout. + pub fn range_valid(&self, addr: u64, range: u64) -> bool { + // SAFETY: the borrowed VM retains its immutable geometry. + unsafe { bindings::drm_gpuvm_range_valid(self.gpuvm() as *mut _, addr, range) } + } + pub fn bo_deferred_cleanup(&self) { unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.gpuvm() as *mut _) } } @@ -802,6 +808,29 @@ pub struct GpuVmInnerGuard<'a, T: DriverGpuVm> { } impl GpuVmInnerGuard<'_, T> { + /// Run immediate-mode unmap steps without allocating an exec context. + /// + /// # Safety + /// The driver's unmap/remap callbacks must touch only VM metadata and + /// translations protected by this reservation. They must not access + /// external BO backing that requires its own reservation. Any split-node + /// storage must already be owned by `ctx`; callbacks must defer BO puts. + pub unsafe fn sm_unmap_inner( + &mut self, ctx: &mut T::StepContext, addr: u64, range: u64, + ) -> Result { + let vm = self.gpuvm.gpuvm() as *mut bindings::drm_gpuvm; + // SAFETY: this guard retains the VM and its shared reservation. + if unsafe { (*vm).flags } & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 { + return Err(EINVAL); + } + let mut ctx = StepContext { gpuvm: self.gpuvm, ctx }; + // SAFETY: caller supplies the callback contract above. Immediate-mode + // link/unlink acquire the separate GEM GPUVA-list mutex themselves. + to_result(unsafe { + bindings::drm_gpuvm_sm_unmap(vm, &mut ctx as *mut _ as *mut _, addr, range) + }) + } + /// Remove every driver GPUVA in an immediate-mode VM, preserving its kernel cutout. /// /// `unmap` must remove the GPU translation for each complete GPUVA before @@ -813,6 +842,25 @@ impl GpuVmInnerGuard<'_, T> { /// when a callback fails after earlier mappings have already been removed. pub fn unmap_all( &mut self, + unmap: impl FnMut(&mut T, &GpuVa) -> Result, + ) -> Result { + self.unmap_matching(None, unmap) + } + + /// Allocation-free whole-node removal for one GEM in an immediate-mode VM. + /// Uses the same translation-removal and deferred-put contract as unmap_all. + /// This scans the VM list and is intended for low-memory cleanup fallback. + pub fn unmap_object( + &mut self, + object: &Object, + unmap: impl FnMut(&mut T, &GpuVa) -> Result, + ) -> Result { + self.unmap_matching(Some(object), unmap) + } + + fn unmap_matching( + &mut self, + object: Option<&Object>, mut unmap: impl FnMut(&mut T, &GpuVa) -> Result, ) -> Result { let vm = self.gpuvm.gpuvm() as *mut bindings::drm_gpuvm; @@ -832,6 +880,9 @@ impl GpuVmInnerGuard<'_, T> { if core::ptr::eq(va, &raw const (*vm).kernel_alloc_node) { continue; } + if object.is_some_and(|object| (*va).gem.obj != object.as_raw()) { + continue; + } let driver_va = crate::container_of!(va, GpuVa, gpuva); unmap(&mut *self.gpuvm.inner.get(), &*driver_va)?; let mut op = OpUnMap( From 982da05e5f831a54800a778df91464ab6d6372b2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:09 -0400 Subject: [PATCH 0638/1579] drm/asahi: keep compute timestamps outside shared queue aliases Private per-command compute timestamp pages were allocated from the range later used for canonical shared queue aliases, so a late shared alias could collide with a live timestamp page. Allocate them from the client-private lower arena instead. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 7d57556a8962dd49aadbb24af772ee5240b8c1c1) --- drivers/gpu/drm/asahi/g17/completion.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 0c877743873d1..7308fde83d568 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -13,7 +13,7 @@ use super::{ recovery::Class, status::VmStatus, }; -use crate::{driver::AsahiDevice, file, hw::t8140::resources as cfg, mmu}; +use crate::{driver::AsahiDevice, file, hw::t8140::{dynamic, resources as cfg}, mmu}; use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; use kernel::{ dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, @@ -156,7 +156,9 @@ impl ComputeTimestamps { let object = KernelObject::backing(dev, mmu::UAT_PGSZ, CpuMap::WriteCombined)?; let mapping = object.alias_in( vm, - cfg::TIMESTAMPS, + // Canonical shared queue aliases may be installed in this VM later. + // Private timestamp pages must never occupy their retained addresses. + dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64, mmu::PROT_GPU_FW_SHARED_RW, )?; From e3cfe66748744b6aadf6b863b57818adb0721cdd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:11:49 -0400 Subject: [PATCH 0639/1579] drm/shmem-helper: add revocable CPU mappings and idle backing release Let a driver release the backing of an idle shmem object without purging it: drm_gem_shmem_release_idle_pages_locked() drops the cached SG table and page array once no pin, vmap, SG borrower or legacy CPU mapping remains, so ordinary shmem reclaim can swap the contents. The next get_pages_sgt() restores them, possibly at new DMA addresses. Objects created with reclaimable_cpu_mappings keep their CPU VMAs revocable: mmap no longer pins pages, faults take an object-scoped page reference under the reservation, release zaps the device anon mapping and the PTEs refault afterwards. Non-opted objects keep the existing behavior. The Rust shmem wrapper counts owned SG-table leases and marks the borrowed sg_table() accessor permanent, so release is refused while any Rust user still holds the table. Add the dma_resv_trylock helper used by non-blocking reclaim. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 733634bcb0c4de6e9885878dc8b1aad744353c83) --- drivers/gpu/drm/drm_gem_shmem_helper.c | 255 +++++++++++++++++++++++-- include/drm/drm_gem_shmem_helper.h | 11 ++ rust/helpers/dma-resv.c | 6 + rust/kernel/drm/gem/shmem.rs | 92 ++++++++- 4 files changed, 337 insertions(+), 27 deletions(-) diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 7d019582c0cc2..d42f9d6025de8 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -7,7 +7,9 @@ #include #include #include +#include #include +#include #include #include @@ -165,10 +167,18 @@ EXPORT_SYMBOL_GPL(drm_gem_shmem_create); * This function cleans up the GEM object state, but does not free the memory used to store the * object itself. This function is meant to be a dedicated helper for the Rust GEM bindings. */ +static void drm_gem_shmem_put_cpu_pages_locked(struct drm_gem_shmem_object *shmem); + void drm_gem_shmem_release(struct drm_gem_shmem_object *shmem) { struct drm_gem_object *obj = &shmem->base; + if (shmem->cpu_pages) { + dma_resv_lock(obj->resv, NULL); + drm_gem_shmem_put_cpu_pages_locked(shmem); + dma_resv_unlock(obj->resv); + } + if (drm_gem_is_imported(obj)) { drm_prime_gem_destroy(obj, shmem->sgt); } else { @@ -492,25 +502,176 @@ int drm_gem_shmem_madvise_locked(struct drm_gem_shmem_object *shmem, int madv) } EXPORT_SYMBOL_GPL(drm_gem_shmem_madvise_locked); -void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem) +/* + * Revocable CPU mappings populate pages one at a time, so touching a few + * bytes of a large object neither materializes nor pins the whole object. + * The device path keeps using the full @pages array; both may coexist and + * reference the same page-cache pages. + */ +static struct page *drm_gem_shmem_cpu_page_locked(struct drm_gem_shmem_object *shmem, + pgoff_t index) { struct drm_gem_object *obj = &shmem->base; - struct drm_device *dev = obj->dev; + struct address_space *mapping = file_inode(obj->filp)->i_mapping; + struct page *page; - dma_resv_assert_held(shmem->base.resv); + dma_resv_assert_held(obj->resv); - drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem)); + if (!shmem->cpu_pages) { + shmem->cpu_pages = kvcalloc(obj->size >> PAGE_SHIFT, + sizeof(*shmem->cpu_pages), GFP_KERNEL); + if (!shmem->cpu_pages) + return ERR_PTR(-ENOMEM); + } + page = shmem->cpu_pages[index]; + if (page) + return page; + /* + * PFN mappings have no rmap, so reclaim could write the page to swap + * and later drop it while the PTE still names it. Keep the mapping + * unevictable while sparse pages exist, as drm_gem_get_pages() does + * for the device array; the flag is cleared when the last holder goes. + */ + mapping_set_unevictable(mapping); + page = shmem_read_mapping_page(mapping, index); + if (IS_ERR(page)) + return page; + shmem->cpu_pages[index] = page; + shmem->cpu_pages_count++; + return page; +} + +static void drm_gem_shmem_put_cpu_pages_locked(struct drm_gem_shmem_object *shmem) +{ + struct drm_gem_object *obj = &shmem->base; + unsigned long i, npages = obj->size >> PAGE_SHIFT; + struct folio_batch fbatch; + + dma_resv_assert_held(obj->resv); + + if (!shmem->cpu_pages) + return; + /* A device array still holding these pages keeps the mapping unevictable. */ + if (!shmem->pages) + mapping_clear_unevictable(file_inode(obj->filp)->i_mapping); + folio_batch_init(&fbatch); + for (i = 0; i < npages && shmem->cpu_pages_count; i++) { + struct folio *folio; + + if (!shmem->cpu_pages[i]) + continue; + folio = page_folio(shmem->cpu_pages[i]); + folio_mark_dirty(folio); + folio_mark_accessed(folio); + shmem->cpu_pages_count--; + if (!folio_batch_add(&fbatch, folio)) { + check_move_unevictable_folios(&fbatch); + __folio_batch_release(&fbatch); + } + } + if (folio_batch_count(&fbatch)) { + check_move_unevictable_folios(&fbatch); + __folio_batch_release(&fbatch); + } + kvfree(shmem->cpu_pages); + shmem->cpu_pages = NULL; +} - dma_unmap_sgtable(dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0); +static struct page *drm_gem_shmem_present_page(struct drm_gem_shmem_object *shmem, + pgoff_t index) +{ + if (shmem->pages && shmem->pages[index]) + return shmem->pages[index]; + if (shmem->cpu_pages) + return shmem->cpu_pages[index]; + return NULL; +} + +bool drm_gem_shmem_idle_pages_releasable_locked(struct drm_gem_shmem_object *shmem) +{ + struct drm_gem_object *obj = &shmem->base; + + dma_resv_assert_held(obj->resv); + return shmem->sgt && !drm_gem_is_imported(obj) && !obj->dma_buf && + shmem->madv >= 0 && !refcount_read(&shmem->pages_pin_count) && + !refcount_read(&shmem->vmap_use_count) && + refcount_read(&shmem->pages_use_count) == 1 && + dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_BOOKKEEP); +} +EXPORT_SYMBOL_GPL(drm_gem_shmem_idle_pages_releasable_locked); + +/** + * drm_gem_shmem_release_idle_pages_locked - Release idle backing without purging + * @shmem: shmem GEM object whose device mappings have been removed + * + * Drop the cached SG table and the page array of an object that no device + * mapping, pin, vmap or SG-table borrower uses, so ordinary shmem reclaim can + * swap its contents. Revocable CPU VMAs are zapped first and refault later. + * The caller holds the reservation lock, has removed every device mapping + * including cached translations, and excludes new device access. A later + * drm_gem_shmem_get_pages_sgt() may return different DMA addresses. + * + * Returns: 0 on success, -EBUSY if backing is still in use, -EINVAL for an + * imported, exported or purged object. + */ +int drm_gem_shmem_release_idle_pages_locked(struct drm_gem_shmem_object *shmem) +{ + struct drm_gem_object *obj = &shmem->base; + + dma_resv_assert_held(obj->resv); + + if (drm_gem_is_imported(obj) || obj->dma_buf || shmem->madv < 0) + return -EINVAL; + + if (refcount_read(&shmem->pages_pin_count) || + refcount_read(&shmem->vmap_use_count) || + refcount_read(&shmem->pages_use_count) != !!shmem->sgt || + !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_BOOKKEEP)) + return -EBUSY; + + /* Zap revocable PFNs under resv so no fault can reinsert them. */ + if (shmem->reclaimable_cpu_mappings && refcount_read(&shmem->cpu_vma_count)) { + drm_vma_node_unmap(&obj->vma_node, obj->dev->anon_inode->i_mapping); + drm_gem_shmem_put_cpu_pages_locked(shmem); + } + + if (!shmem->sgt) + return 0; + + dma_unmap_sgtable(obj->dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0); sg_free_table(shmem->sgt); kfree(shmem->sgt); shmem->sgt = NULL; + shmem->pages_mark_dirty_on_put = true; drm_gem_shmem_put_pages_locked(shmem); - shmem->madv = -1; + return 0; +} +EXPORT_SYMBOL_GPL(drm_gem_shmem_release_idle_pages_locked); + +void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem) +{ + struct drm_gem_object *obj = &shmem->base; + struct drm_device *dev = obj->dev; + + dma_resv_assert_held(shmem->base.resv); + drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem)); + + /* Revoke PFNs and complete the TLB flush while backing is still pinned. */ + shmem->madv = -1; drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping); + + if (shmem->sgt) { + dma_unmap_sgtable(dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0); + sg_free_table(shmem->sgt); + kfree(shmem->sgt); + shmem->sgt = NULL; + drm_gem_shmem_put_pages_locked(shmem); + } + drm_gem_shmem_put_cpu_pages_locked(shmem); + drm_gem_free_mmap_offset(obj); /* Our goal here is to return as much of the memory as @@ -572,11 +733,16 @@ static void drm_gem_shmem_record_mkwrite(struct vm_fault *vmf) loff_t num_pages = obj->size >> PAGE_SHIFT; pgoff_t page_offset = drm_gem_shmem_object_offset(obj, vmf->pgoff); - if (drm_WARN_ON(obj->dev, !shmem->pages || page_offset >= num_pages)) + struct page *page; + + if (drm_WARN_ON(obj->dev, page_offset >= num_pages)) + return; + page = drm_gem_shmem_present_page(shmem, page_offset); + if (drm_WARN_ON(obj->dev, !page)) return; file_update_time(vma->vm_file); - folio_mark_dirty(page_folio(shmem->pages[page_offset])); + folio_mark_dirty(page_folio(page)); } /* @@ -631,7 +797,7 @@ static vm_fault_t drm_gem_shmem_any_fault(struct vm_fault *vmf, unsigned int ord struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); loff_t num_pages = obj->size >> PAGE_SHIFT; vm_fault_t ret = VM_FAULT_SIGBUS; - struct page **pages = shmem->pages; + struct page **pages; pgoff_t page_offset = drm_gem_shmem_object_offset(obj, vmf->pgoff); struct page *page; struct folio *folio; @@ -642,13 +808,28 @@ static vm_fault_t drm_gem_shmem_any_fault(struct vm_fault *vmf, unsigned int ord dma_resv_lock(obj->resv, NULL); - if (page_offset >= num_pages || drm_WARN_ON_ONCE(dev, !shmem->pages) || - shmem->madv < 0) + if (page_offset >= num_pages || shmem->madv < 0) goto out; - page = pages[page_offset]; - if (drm_WARN_ON_ONCE(dev, !page)) - goto out; + if (shmem->reclaimable_cpu_mappings && !shmem->pages) { + /* Per-page population; huge mappings need the full array. */ + if (order) { + ret = VM_FAULT_FALLBACK; + goto out; + } + page = drm_gem_shmem_cpu_page_locked(shmem, page_offset); + if (IS_ERR(page)) { + ret = vmf_error(PTR_ERR(page)); + goto out; + } + } else { + pages = shmem->pages; + if (drm_WARN_ON_ONCE(dev, !pages)) + goto out; + page = pages[page_offset]; + if (drm_WARN_ON_ONCE(dev, !page)) + goto out; + } folio = page_folio(page); pfn = page_to_pfn(page); @@ -682,8 +863,11 @@ static void drm_gem_shmem_vm_open(struct vm_area_struct *vma) * mmap'd, vm_open() just grabs an additional reference for the new * mm the vma is getting copied into (ie. on fork()). */ - drm_WARN_ON_ONCE(obj->dev, - !refcount_inc_not_zero(&shmem->pages_use_count)); + if (shmem->reclaimable_cpu_mappings) + refcount_inc(&shmem->cpu_vma_count); + else + drm_WARN_ON_ONCE(obj->dev, + !refcount_inc_not_zero(&shmem->pages_use_count)); dma_resv_unlock(shmem->base.resv); @@ -696,7 +880,12 @@ static void drm_gem_shmem_vm_close(struct vm_area_struct *vma) struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); dma_resv_lock(shmem->base.resv, NULL); - drm_gem_shmem_put_pages_locked(shmem); + if (shmem->reclaimable_cpu_mappings) { + if (refcount_dec_and_test(&shmem->cpu_vma_count)) + drm_gem_shmem_put_cpu_pages_locked(shmem); + } else { + drm_gem_shmem_put_pages_locked(shmem); + } dma_resv_unlock(shmem->base.resv); drm_gem_vm_close(vma); @@ -704,8 +893,23 @@ static void drm_gem_shmem_vm_close(struct vm_area_struct *vma) static vm_fault_t drm_gem_shmem_pfn_mkwrite(struct vm_fault *vmf) { - drm_gem_shmem_record_mkwrite(vmf); - return 0; + struct drm_gem_object *obj = vmf->vma->vm_private_data; + struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); + vm_fault_t ret = 0; + + if (!shmem->reclaimable_cpu_mappings) { + drm_gem_shmem_record_mkwrite(vmf); + return 0; + } + + /* A revoked PFN must refault rather than be marked writable. */ + dma_resv_lock(obj->resv, NULL); + if (!drm_gem_shmem_present_page(shmem, drm_gem_shmem_object_offset(obj, vmf->pgoff))) + ret = VM_FAULT_NOPAGE; + else + drm_gem_shmem_record_mkwrite(vmf); + dma_resv_unlock(obj->resv); + return ret; } const struct vm_operations_struct drm_gem_shmem_vm_ops = { @@ -766,8 +970,19 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct return -EINVAL; } + /* Revocation zaps the device anon mapping only. */ + if (shmem->reclaimable_cpu_mappings && + (!vma->vm_file || vma->vm_file->f_mapping != obj->dev->anon_inode->i_mapping)) + return -EINVAL; + dma_resv_lock(shmem->base.resv, NULL); - ret = drm_gem_shmem_get_pages_locked(shmem); + if (shmem->reclaimable_cpu_mappings) { + if (!refcount_inc_not_zero(&shmem->cpu_vma_count)) + refcount_set(&shmem->cpu_vma_count, 1); + ret = 0; + } else { + ret = drm_gem_shmem_get_pages_locked(shmem); + } dma_resv_unlock(shmem->base.resv); if (ret) diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index 5ccdae21b94a9..a0f2b3ebcf59f 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -31,6 +31,15 @@ struct drm_gem_shmem_object { */ struct page **pages; + /** @cpu_vma_count: Revocable CPU VMAs, protected by base.resv. */ + refcount_t cpu_vma_count; + + /** @reclaimable_cpu_mappings: CPU VMAs may be revoked by the driver. */ + bool reclaimable_cpu_mappings; + + /** @cpu_pages_owned: One object-scoped CPU page-array ref, under resv. */ + bool cpu_pages_owned; + /** * @pages_use_count: * @@ -133,6 +142,8 @@ static inline bool drm_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem !shmem->base.dma_buf && !drm_gem_is_imported(&shmem->base); } +bool drm_gem_shmem_idle_pages_releasable_locked(struct drm_gem_shmem_object *shmem); +int drm_gem_shmem_release_idle_pages_locked(struct drm_gem_shmem_object *shmem); void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem); struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_shmem_object *shmem); diff --git a/rust/helpers/dma-resv.c b/rust/helpers/dma-resv.c index 71914d8241e29..957d80992fef3 100644 --- a/rust/helpers/dma-resv.c +++ b/rust/helpers/dma-resv.c @@ -12,3 +12,9 @@ __rust_helper void rust_helper_dma_resv_unlock(struct dma_resv *obj) { dma_resv_unlock(obj); } + +__rust_helper +bool rust_helper_dma_resv_trylock(struct dma_resv *obj) +{ + return dma_resv_trylock(obj); +} diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index b3e79f83c6002..51f25fe9c1f0a 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -38,6 +38,7 @@ use core::{ DerefMut, // }, ptr::NonNull, + sync::atomic::{AtomicUsize, Ordering}, }; use gem::{ BaseObject, @@ -55,12 +56,19 @@ pub struct ObjectConfig<'a, T: DriverObject> { /// Whether to set the write-combine map flag. pub map_wc: bool, + /// Opt in to revocable CPU VMAs, with driver-controlled idle eviction. + pub reclaimable_cpu_mappings: bool, + /// Reuse the DMA reservation from another GEM object. /// /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. pub parent_resv_obj: Option<&'a Object>, } +const SGT_PERMANENT: usize = 1 << (usize::BITS - 1); +const SGT_RECLAIMING: usize = 1 << (usize::BITS - 2); +const SGT_USERS: usize = SGT_RECLAIMING - 1; + /// A shmem-backed GEM object. /// /// # Invariants @@ -74,6 +82,8 @@ pub struct Object { obj: Opaque, /// Parent object that owns this object's DMA reservation object. parent_resv_obj: Option>>, + /// Owned SGT readers, or permanent borrowed access, exclude backing eviction. + sgt_access: AtomicUsize, #[pin] inner: T, } @@ -130,6 +140,7 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + sgt_access: AtomicUsize::new(0), inner <- T::new(dev, size, args), }), GFP_KERNEL, @@ -158,6 +169,7 @@ impl Object { // to have exclusive access - thus making this safe to hold a mutable reference to. let shmem = unsafe { &mut *obj.as_raw_shmem() }; shmem.set_map_wc(config.map_wc); + shmem.reclaimable_cpu_mappings = config.reclaimable_cpu_mappings; Ok(obj) } @@ -196,6 +208,33 @@ impl Object { /// This will pin the object in memory. #[inline] pub fn sg_table(&self) -> Result<&scatterlist::SGTable> { + // The borrowed reference can outlive any lock, so pin permanently. + self.acquire_sgt_access(true)?; + self.sg_table_resident() + } + + fn acquire_sgt_access(&self, permanent: bool) -> Result { + let mut state = self.sgt_access.load(Ordering::Acquire); + loop { + if state & SGT_RECLAIMING != 0 { + return Err(EBUSY); + } + let next = if permanent { + state | SGT_PERMANENT + } else { + if state & SGT_USERS == SGT_USERS { return Err(EOVERFLOW); } + state + 1 + }; + match self.sgt_access.compare_exchange_weak( + state, next, Ordering::AcqRel, Ordering::Acquire, + ) { + Ok(_) => return Ok(()), + Err(current) => state = current, + } + } + } + + fn sg_table_resident(&self) -> Result<&scatterlist::SGTable> { // SAFETY: // - drm_gem_shmem_get_pages_sgt is thread-safe. // - drm_gem_shmem_get_pages_sgt returns either a valid pointer to a scatterlist, or an @@ -218,12 +257,44 @@ impl Object { /// /// [`shmem::SGTable`]: SGTable pub fn owned_sg_table(&self) -> Result> { - Ok(SGTable { - sgt: self.sg_table()?.into(), - // INVARIANT: We take an owned refcount to `self` here, ensuring that `sgt` remains - // valid for as long as this `SGTable`. - _owner: self.into(), - }) + self.acquire_sgt_access(false)?; + let sgt = match self.sg_table_resident() { + Ok(sgt) => sgt.into(), + Err(error) => { + self.sgt_access.fetch_sub(1, Ordering::Release); + return Err(error); + } + }; + Ok(SGTable { sgt, _owner: self.into() }) + } + + /// Whether the caller's sole SG lease is the last user of the backing. + /// + /// # Safety + /// The caller holds the object's DMA reservation lock. + pub unsafe fn sole_sg_lease_releasable_locked(&self) -> bool { + self.sgt_access.load(Ordering::Acquire) == 1 && + // SAFETY: The caller establishes the reservation-lock contract. + unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } + } + + /// Release idle backing without discarding contents. A later + /// `owned_sg_table()` may return different DMA addresses. + /// + /// # Safety + /// + /// The caller holds this object's DMA reservation lock, has removed and + /// invalidated every device mapping, and excludes new device access. + pub unsafe fn release_idle_pages_locked(&self) -> Result { + self.sgt_access.compare_exchange( + 0, SGT_RECLAIMING, Ordering::AcqRel, Ordering::Acquire, + ).map_err(|_| EBUSY)?; + // SAFETY: The caller's contract plus the exclusive state above. + let result = to_result(unsafe { + bindings::drm_gem_shmem_release_idle_pages_locked(self.as_raw_shmem()) + }); + self.sgt_access.store(0, Ordering::Release); + result } /// Attempt to create a [`RawIoSysMap`] from the gem object. @@ -438,7 +509,8 @@ unsafe impl Sync for VMap {} /// /// # Invariants /// -/// - `sgt` is kept alive by `_owner`, ensuring it remains valid for as long as `Self`. +/// - `_owner` retains both the GEM object and one counted SG-table lease; eviction +/// cannot begin until every lease has been released. /// - `sgt` corresponds to the owned object in `_owner`. /// - This object is only exposed in situations where we know the underlying `SGTable` will not be /// modified for the lifetime of this object. Thus, it is safe to send/access this type across @@ -453,6 +525,12 @@ unsafe impl Send for SGTable {} // SAFETY: This object is thread-safe via our type invariants. unsafe impl Sync for SGTable {} +impl Drop for SGTable { + fn drop(&mut self) { + self._owner.sgt_access.fetch_sub(1, Ordering::Release); + } +} + impl Deref for SGTable { type Target = scatterlist::SGTable; From 03329eff23bd3a72bdcfa09a3b98d0ac15107e6e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:11:49 -0400 Subject: [PATCH 0640/1579] rust: drm: gpuvm: add non-blocking private mapping traversal Add the interfaces a driver needs to evict and restore the backing of idle VM-private objects: expose per-GPUVA driver metadata, borrow the GEM object of a VM/BO association, take the VM reservation with dma_resv_trylock, walk private associations and mark every alias INVALIDATED without allocating, iterate current private mappings by cursor, and clear the flag after the page tables are rebuilt. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 35b057e572224c39f79367df9e7c464f1822d15a) --- rust/kernel/drm/gpuvm.rs | 229 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) diff --git a/rust/kernel/drm/gpuvm.rs b/rust/kernel/drm/gpuvm.rs index 5cfa57fd4fe9a..467222adb36b9 100644 --- a/rust/kernel/drm/gpuvm.rs +++ b/rust/kernel/drm/gpuvm.rs @@ -285,6 +285,15 @@ impl GpuVa { ) } + /// Driver metadata retained independently of the resident page tables. + pub fn inner(&self) -> &T::GpuVa { &self.inner } + + /// Update metadata before linking a newly allocated GPUVA. + pub fn inner_mut(self: Pin<&mut Self>) -> Pin<&mut T::GpuVa> { + // SAFETY: inner is structurally pinned with its enclosing GPUVA. + unsafe { self.map_unchecked_mut(|va| &mut va.inner) } + } + pub fn addr(&self) -> u64 { self.gpuva.va.addr } @@ -312,6 +321,13 @@ pub struct GpuVmBo { } impl GpuVmBo { + /// The GEM object retained by this VM/BO association. + pub fn object(&self) -> &Object { + // SAFETY: drm_gpuvm_bo owns this GEM reference until its destructor; + // DriverGpuVm fixes the GEM type for every association in this VM. + unsafe { &* as IntoGEMObject>::from_raw(self.bo.obj) } + } + /// Return a reference to the inner driver data for this GpuVmBo pub fn inner(&self) -> &T::GpuVmBo { &self.inner @@ -596,6 +612,32 @@ impl GpuVm { } } + // The caller owns the VM reservation and keeps the exact logical VA linked. + unsafe fn set_invalidated_locked(&self, addr: u64, range: u64, invalidated: bool) -> Result { + if range == 0 || addr.checked_add(range).is_none() { return Err(EINVAL); } + // SAFETY: The caller owns the VM reservation and keeps the VA linked. + let raw = unsafe { bindings::drm_gpuva_find(self.gpuvm() as *mut _, addr, range) }; + if raw.is_null() { return Err(EINVAL); } + // SAFETY: Reservation ownership excludes all concurrent VA mutation. + unsafe { + if (*raw).va.addr != addr || (*raw).va.range != range { return Err(EINVAL); } + if invalidated { (*raw).flags |= GpuVaFlags::INVALIDATED.as_raw(); } + else { (*raw).flags &= !GpuVaFlags::INVALIDATED.as_raw(); } + } + Ok(()) + } + + /// Try to lock only the VM reservation, without allocation or waiting. + /// The guard protects VM metadata and private BOs, not external backing. + /// Callers must not already hold a reservation lock. + pub fn try_lock_private(&self) -> Option> { + // SAFETY: r_obj and its reservation are retained by this GPUVM. + let resv = unsafe { (*(*self.gpuvm()).r_obj).resv }; + // SAFETY: The reservation outlives the returned borrow and guard. + if !unsafe { bindings::dma_resv_trylock(resv) } { return None; } + Some(PrivateGpuVmGuard { gpuvm: self, resv, _not_send: NotThreadSafe }) + } + pub fn exec_lock<'a, 'b>( &'a self, obj: Option<&'b Object>, @@ -921,6 +963,187 @@ impl Drop for GpuVmInnerGuard<'_, T> { } } +/// Logical binding retained independently of hardware page-table residency. +/// Owns a GEM reference and copied metadata, never a GpuVmBo reference. +/// Driver metadata cloning must not introduce a reservation-taking destructor. +pub struct MappingSnapshot { + pub object: ARef>, + pub addr: u64, + pub range: u64, + pub offset: u64, + pub flags: GpuVaFlags, + pub inner: T::GpuVa, +} + +/// Nonblocking reservation ownership for pressure scans of private objects. +pub struct PrivateGpuVmGuard<'a, T: DriverGpuVm> { + gpuvm: &'a GpuVm, + resv: *mut bindings::dma_resv, + // DMA reservation ownership must remain on the acquiring task. + _not_send: NotThreadSafe, +} + +impl PrivateGpuVmGuard<'_, T> { + /// Visit each private BO with linked logical mappings once, without allocating or retaining + /// references. Returning false stops the walk. Contended GEM lists are skipped. + /// + /// # Safety + /// The callback must not insert/remove logical mappings or modify either GPUVA list. This + /// reservation keeps their linked BO/GEM references alive for the complete callback. + /// The callback must not retain a borrowed BO or acquire its GEM GPUVA mutex recursively. + pub unsafe fn for_each_private_bo( + &mut self, + mut visit: impl FnMut(&mut Self, &GpuVmBo) -> Result, + ) -> Result { + let vm = self.gpuvm.gpuvm() as *mut bindings::drm_gpuvm; + if unsafe { (*vm).flags } & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 { + return Err(EINVAL); + } + let head = unsafe { &raw const (*vm).rb.list }; + // SAFETY: The VM reservation protects this ordered mapping list. The callback does not + // mutate its links, and no borrowed or owned reference outlives the guard. + let mut node = unsafe { (*head).next }; + while node != head.cast_mut() { + let (next, raw) = unsafe { + ((*node).next, crate::container_of!(node, bindings::drm_gpuva, rb.entry)) + }; + let va = unsafe { &*raw }; + if !va.gem.obj.is_null() && !va.vm_bo.is_null() { + let object = unsafe { &*Object::::from_raw(va.gem.obj) }; + if !self.gpuvm.is_extobj(object) { + let lock = unsafe { &raw mut (*va.gem.obj).gpuva.lock }; + // Never block direct reclaim on a GEM-list owner. The canonical alias is + // the list's first-linked node, not its lowest address or current residency. + if unsafe { bindings::mutex_trylock(lock) } != 0 { + let canonical = unsafe { + (*va.vm_bo).list.gpuva.next == (&raw const va.gem.entry).cast_mut() + }; + unsafe { bindings::mutex_unlock(lock) }; + if canonical { + let bo = unsafe { + &*crate::container_of!(va.vm_bo, GpuVmBo, bo) + }; + if !visit(self, bo)? { break; } + } + } + } + } + node = next; + } + Ok(()) + } + + /// Mark and visit every logical alias of this private BO without a snapshot allocation. + /// GEM-list locking is nonblocking and held only while reading a cursor, never during the + /// callback. Failure (including contention) can leave a prefix marked INVALIDATED; callers + /// must retain backing unless this method succeeds for the entire association. + /// + /// # Safety + /// The callback must not insert/remove logical mappings, alter list links, or release the + /// BO's SG/backing lease. The BO must remain linked throughout the walk. Exclude new jobs and + /// binding edits for the whole invalidation and backing-release transaction. + pub unsafe fn invalidate_private_bo_mappings( + &mut self, + bo: &GpuVmBo, + mut visit: impl FnMut(&mut Self, u64, u64) -> Result, + ) -> Result { + let vm = self.gpuvm.gpuvm() as *mut bindings::drm_gpuvm; + if bo.bo.vm != vm || self.gpuvm.is_extobj(bo.object()) + || unsafe { (*vm).flags } & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 + { return Err(EINVAL); } + let head = &raw const bo.bo.list.gpuva; + let lock = unsafe { &raw mut (*bo.bo.obj).gpuva.lock }; + if unsafe { bindings::mutex_trylock(lock) } == 0 { return Err(EAGAIN); } + let mut node = unsafe { (*head).next }; + unsafe { bindings::mutex_unlock(lock) }; + while node != head.cast_mut() { + if unsafe { bindings::mutex_trylock(lock) } == 0 { return Err(EAGAIN); } + // SAFETY: The list mutex protects link reads; this reservation and the caller's + // no-unlink contract keep current/next and their association alive after unlocking. + let (next, raw) = unsafe { + ((*node).next, crate::container_of!(node, bindings::drm_gpuva, gem.entry)) + }; + unsafe { bindings::mutex_unlock(lock) }; + let va = unsafe { &mut *raw }; + if va.vm != vm || va.vm_bo != (&raw const bo.bo).cast_mut() + || va.va.range == 0 || va.va.addr.checked_add(va.va.range).is_none() + { return Err(EINVAL); } + // Flags are serialized by the VM reservation, independently of GEM's list mutex. + va.flags |= GpuVaFlags::INVALIDATED.as_raw(); + visit(self, va.va.addr, va.va.range)?; + node = next; + } + Ok(()) + } + + /// Retain the next private mapping containing all `required_flags`. + /// + /// `cursor` is an exclusive end address and advances past skipped mappings + /// as well. No collection is allocated; only the returned object is retained. + /// Callers must drop this reservation guard before releasing that reference + /// or acquiring backing. To cover a stable set across calls, separately + /// exclude logical binding edits (including retirement) for the whole walk. + pub fn next_private_mapping( + &self, + cursor: &mut u64, + required_flags: GpuVaFlags, + ) -> Result>> + where T::GpuVa: Clone { + while *cursor != u64::MAX { + // SAFETY: The VM reservation serializes mutation of its VA tree. + let raw = unsafe { + bindings::drm_gpuva_find_first( + self.gpuvm.gpuvm() as *mut _, *cursor, u64::MAX - *cursor, + ) + }; + if raw.is_null() { return Ok(None); } + // SAFETY: raw stays linked and alive while this guard is held. + let va = unsafe { &*raw }; + let end = va.va.addr.checked_add(va.va.range).ok_or(EINVAL)?; + if end <= *cursor { return Err(EINVAL); } + *cursor = end; + if !GpuVaFlags(va.flags).contains(required_flags) + || va.gem.obj.is_null() || va.vm_bo.is_null() { + continue; + } + // SAFETY: The linked VA retains this driver's object. + let obj = unsafe { &*Object::::from_raw(va.gem.obj) }; + if self.gpuvm.is_extobj(obj) { continue; } + // SAFETY: Driver-owned mappings have this allocation layout. The + // special kernel cutout has no GEM object and was skipped above. + let typed = unsafe { &*crate::container_of!(raw, GpuVa, gpuva) }; + return Ok(Some(MappingSnapshot { + object: obj.into(), addr: va.va.addr, + range: va.va.range, offset: va.gem.offset, + flags: GpuVaFlags(va.flags), inner: typed.inner.clone(), + })); + } + Ok(None) + } + + +} + +impl Deref for PrivateGpuVmGuard<'_, T> { + type Target = T; + fn deref(&self) -> &T { + // SAFETY: The reservation serializes all access to the VM's inner data. + unsafe { &*self.gpuvm.inner.get() } + } +} +impl DerefMut for PrivateGpuVmGuard<'_, T> { + fn deref_mut(&mut self) -> &mut T { + // SAFETY: This guard has exclusive reservation ownership. + unsafe { &mut *self.gpuvm.inner.get() } + } +} +impl Drop for PrivateGpuVmGuard<'_, T> { + fn drop(&mut self) { + // SAFETY: try_lock_private acquired exactly this reservation. + unsafe { bindings::dma_resv_unlock(self.resv) }; + } +} + pub struct LockedGpuVm<'a, 'b, T: DriverGpuVm> { gpuvm: &'a GpuVm, vm_exec: KBox, @@ -928,6 +1151,12 @@ pub struct LockedGpuVm<'a, 'b, T: DriverGpuVm> { } impl LockedGpuVm<'_, '_, T> { + /// Mark the exact retained logical mapping after page-table restoration. + pub fn set_invalidated(&mut self, addr: u64, range: u64, invalidated: bool) -> Result { + // SAFETY: Both guard types own the VM reservation. + unsafe { self.gpuvm.set_invalidated_locked(addr, range, invalidated) } + } + pub fn find_bo(&mut self) -> Option>> { let obj = self.objects.single()?; // SAFETY: LockedGpuVm implies the right locks are held. From 449bb6888c6feeb79e6390cfe00dd7d4d58b6261 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:13:08 -0400 Subject: [PATCH 0641/1579] drm/asahi: reclaim idle private VM backing under memory pressure Register a shrinker per client VM. When the VM has no accepted jobs, no mapping commits and a healthy status, the scan invalidates every alias of an idle VM-private object, clears its page tables and TLB entries, drops the VM's SG lease and releases the shmem backing so ordinary reclaim can swap it. Objects with kernel aliases, exports/imports or other SG users are skipped. Private objects also get revocable CPU mappings, so an mmap no longer pins the whole object before it is touched. Jobs, kernel aliases and bind preparation take a residency lease; the first lease after an eviction rebuilds the invalidated mappings from the logical GPUVAs before any work is admitted, and retirement serializes with that restore. Mapping permissions are kept as GPUVA metadata so restored and split mappings keep their protection. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e2b08ebbcadc3a9ebe2733140e1a88c62af87ba1) --- drivers/gpu/drm/asahi/file.rs | 24 +- drivers/gpu/drm/asahi/gem.rs | 40 ++ drivers/gpu/drm/asahi/mmu.rs | 52 ++- drivers/gpu/drm/asahi/mmu/bind.rs | 13 +- drivers/gpu/drm/asahi/mmu/lifetime.rs | 14 +- drivers/gpu/drm/asahi/mmu/residency.rs | 484 +++++++++++++++++++++++++ 6 files changed, 604 insertions(+), 23 deletions(-) create mode 100644 drivers/gpu/drm/asahi/mmu/residency.rs diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 37eb832ceee37..d901fd5b4ff42 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -58,6 +58,7 @@ pub(crate) const MAX_COMMANDS_PER_SUBMISSION: u32 = 64; /// A client instance of an `mmu::Vm` address space. struct Vm { + shrinker: Option>, ualloc: Arc>, ualloc_priv: Arc>, vm: mmu::Vm, @@ -68,6 +69,8 @@ struct Vm { impl Drop for Vm { fn drop(&mut self) { + self.shrinker = None; + self.vm.close_idle_reclaim(); if self.vm.status().is_some() { // Tracked jobs can outlive their file's VM handle. Their last guard performs // these unmaps once no accepted work can still dereference the mappings. @@ -523,6 +526,11 @@ impl File { mod_dev_dbg!(device, "[File {} VM {}]: VM created\n", file_id, id); resv.fill(KBox::new( Vm { + shrinker: if vm.residency_reclaim_enabled() { + Some(mmu::VmShrinker::new(&vm)?) + } else { + None + }, ualloc, ualloc_priv, vm, @@ -577,21 +585,19 @@ impl File { } let resv_gem; + let mut reclaim_cpu = false; let resv_obj = if data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 { - resv_gem = file - .inner() - .vms() - .lock() - .get(data.vm_id.try_into()?) - .ok_or(ENOENT)? - .vm - .get_resv_obj(); + let vms = file.inner().vms(); + let vms = vms.lock(); + let vm = &vms.get(data.vm_id.try_into()?).ok_or(ENOENT)?.vm; + reclaim_cpu = vm.residency_reclaim_enabled(); + resv_gem = vm.get_resv_obj(); Some(resv_gem.deref()) } else { None }; - let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj)?; + let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; let handle = gem.create_handle(file)?; data.handle = handle; diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 47550f81464de..a169b8594de83 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -53,6 +53,8 @@ pub(crate) struct AsahiObject { exportable: bool, /// Whether this is a kernel-created object. kernel: bool, + /// Driver aliases outside the user GPUVA tree; they pin the backing. + kernel_mappings: AtomicU64, } /// Type alias for the shmem GEM object type for this driver. @@ -186,6 +188,7 @@ fn new_kernel_object_mapped(dev: &AsahiDevice, size: usize, map_wc: bool) -> Res dev, align(size, mmu::UAT_PGSZ), shmem::ObjectConfig:: { + reclaimable_cpu_mappings: false, map_wc, parent_resv_obj: None, }, @@ -206,6 +209,7 @@ pub(crate) fn new_object( size: usize, flags: u32, parent_object: Option<&shmem::Object>, + reclaim_cpu: bool, ) -> Result> { if (flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0) != parent_object.is_some() { @@ -216,6 +220,7 @@ pub(crate) fn new_object( dev, align(size, mmu::UAT_PGSZ), shmem::ObjectConfig:: { + reclaimable_cpu_mappings: parent_object.is_some() && reclaim_cpu, map_wc: flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK == 0, parent_resv_obj: parent_object, }, @@ -252,6 +257,7 @@ impl DriverObject for AsahiObject { flags: args.flags, exportable: args.exportable, kernel: args.kernel, + kernel_mappings: AtomicU64::new(0), }) } @@ -276,3 +282,37 @@ impl DriverObject for AsahiObject { obj.prime_export(flags) } } + +/// A driver-created alias of a GEM object, possibly in a root the user GPUVA +/// tree does not describe. It keeps the object out of idle reclaim. +pub(crate) struct KernelMappingPin { + object: ARef, +} + +impl KernelMappingPin { + pub(crate) fn new(object: &Object) -> Self { + object.kernel_mappings.fetch_add(1, Ordering::AcqRel); + Self { + object: object.into(), + } + } +} + +impl core::ops::Deref for KernelMappingPin { + type Target = Object; + fn deref(&self) -> &Object { + &self.object + } +} + +impl Drop for KernelMappingPin { + fn drop(&mut self) { + self.object.kernel_mappings.fetch_sub(1, Ordering::Release); + } +} + +impl AsahiObject { + pub(crate) fn idle_reclaim_candidate(&self) -> bool { + !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 + } +} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index d47254243ce00..7db6c5fb1c0bf 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -93,6 +93,8 @@ mod bind; mod context; mod shared; mod lifetime; +mod residency; +pub(crate) use residency::VmShrinker; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; @@ -328,6 +330,13 @@ impl gpuvm::DriverGpuVmBo for VmBo { } } +/// Per-mapping metadata that survives backing eviction and GPUVA splits. +#[derive(Default, Clone)] +struct VmGpuVa { + prot: Option, +} +impl gpuvm::DriverGpuVa for VmGpuVa {} + #[derive(Default)] struct StepContext { new_va: Option>>>, @@ -340,6 +349,7 @@ struct StepContext { impl gpuvm::DriverGpuVm for VmInner { type Driver = driver::AsahiDriver; type GpuVmBo = VmBo; + type GpuVa = VmGpuVa; type StepContext = StepContext; fn step_map( @@ -436,7 +446,8 @@ impl gpuvm::DriverGpuVm for VmInner { } } - let gpuva = ctx.new_va.take().expect("Multiple step_map calls"); + let mut gpuva = ctx.new_va.take().expect("Multiple step_map calls"); + gpuva.as_mut().inner_mut().get_mut().prot = Some(ctx.prot); if op .map_and_link_va( @@ -482,6 +493,8 @@ impl gpuvm::DriverGpuVm for VmInner { let va = op.unmap().va().expect("No previous VA"); let orig_addr = va.addr(); let orig_range = va.range(); + let orig_prot = va.inner().prot.ok_or(EINVAL)?; + let orig_invalidated = va.flags().contains(gpuvm::GpuVaFlags::INVALIDATED); // Only unmap the hole between prev/next, if they exist let unmap_start = if let Some(op) = op.prev_map() { @@ -508,7 +521,11 @@ impl gpuvm::DriverGpuVm for VmInner { let unmap_range = unmap_end - unmap_start; let _mutation = self.mapping_mutation(); - self.page_table.unmap_pages(unmap_start..unmap_end)?; + if orig_invalidated { + self.page_table.discard_partial_map(unmap_start..unmap_end)?; + } else { + self.page_table.unmap_pages(unmap_start..unmap_end)?; + } if let Some(asid) = self.slot() { fence(Ordering::SeqCst); @@ -530,10 +547,11 @@ impl gpuvm::DriverGpuVm for VmInner { } if let Some(prev_op) = op.prev_map() { - let prev_gpuva = ctx + let mut prev_gpuva = ctx .prev_va .take() .expect("Multiple step_remap calls with prev_op"); + prev_gpuva.as_mut().inner_mut().get_mut().prot = Some(orig_prot); if prev_op.map_and_link_va(self, prev_gpuva, vm_bo).is_err() { dev_err!(self.dev.as_ref(), "step_remap: could not relink prev gpuva"); return Err(EINVAL); @@ -541,10 +559,11 @@ impl gpuvm::DriverGpuVm for VmInner { } if let Some(next_op) = op.next_map() { - let next_gpuva = ctx + let mut next_gpuva = ctx .next_va .take() .expect("Multiple step_remap calls with next_op"); + next_gpuva.as_mut().inner_mut().get_mut().prot = Some(orig_prot); if next_op.map_and_link_va(self, next_gpuva, vm_bo).is_err() { dev_err!(self.dev.as_ref(), "step_remap: could not relink next gpuva"); return Err(EINVAL); @@ -561,8 +580,11 @@ impl VmInner { mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); let _mutation = self.mapping_mutation(); - self.page_table - .unmap_pages(va.addr()..(va.addr() + va.range()))?; + if va.flags().contains(gpuvm::GpuVaFlags::INVALIDATED) { + self.page_table.discard_partial_map(va.addr()..(va.addr() + va.range()))?; + } else { + self.page_table.unmap_pages(va.addr()..(va.addr() + va.range()))?; + } if let Some(asid) = self.slot() { fence(Ordering::SeqCst); @@ -667,6 +689,10 @@ impl VmInner { } let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); + self.tlbi_context_mask(iova, size, contexts); + } + + fn tlbi_context_mask(&self, iova: u64, size: usize, contexts: u64) { if contexts == 0 { return; } @@ -767,6 +793,7 @@ pub(crate) struct Vm { id: u64, mapping_epoch: Option>, lifetime: Option>, + residency: Option>, context_bindings: Option>, shared_bindings: Option>, status: Option>, @@ -959,7 +986,7 @@ pub(crate) struct KernelMappingInner { // - Drop the GEM BO next, since BO free can take the resv lock itself // - Drop the owner GpuVm last, since that again can take resv locks when the refcount drops to 0 bo: Option, - _gem: Option>, + _gem: Option, owner: ARef>, uat_inner: Arc, prot: Prot, @@ -1558,6 +1585,7 @@ impl Vm { id, mapping_epoch: mapping_epoch.clone(), lifetime: None, + residency: None, context_bindings: None, shared_bindings: None, status: None, @@ -1591,6 +1619,7 @@ impl Vm { pub(crate) fn with_status(mut self) -> Result { self.status = Some(Arc::new(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); self.lifetime = Some(lifetime::VmLifetime::new()?); + self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); self.shared_bindings = Some(shared::SharedBindings::new()?); Ok(self) @@ -1732,6 +1761,7 @@ impl Vm { return Err(EINVAL); } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; + let _residency = self.enter_residency()?; let sgt = gem.owned_sg_table()?; let mut inner = self.inner.exec_lock(Some(gem), false)?; let vm_bo = self.inner.obtain_bo(gem)?; @@ -1774,7 +1804,7 @@ impl Vm { { let payload = node.as_mut().inner_mut(); payload.bo = vm_bo.take(); - payload._gem = Some(gem.into()); + payload._gem = Some(gem::KernelMappingPin::new(gem)); } let ret = inner.map_node(&node, prot); @@ -1798,6 +1828,7 @@ impl Vm { let reserved_size = size .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; + let _residency = self.enter_residency()?; let sgt = gem.owned_sg_table()?; let mut inner = self.inner.exec_lock(Some(&gem), false)?; @@ -1837,7 +1868,7 @@ impl Vm { { let payload = node.as_mut().inner_mut(); payload.bo = vm_bo.take(); - payload._gem = Some(gem.clone()); + payload._gem = Some(gem::KernelMappingPin::new(&gem)); } let ret = inner.map_node(&node, prot); @@ -1859,6 +1890,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + let _residency = self.enter_residency_metadata()?; gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; // Mapping needs a complete context @@ -2021,6 +2053,7 @@ impl Vm { } fn unmap_range_commit(&self, iova: u64, size: u64, user: bool) -> Result { + let _residency = self.enter_retirement_residency(); // Unmapping a range can only do a single split, so just preallocate // the prev and next GpuVas let mut ctx = StepContext { @@ -2052,6 +2085,7 @@ impl Vm { } fn drop_mappings_now(&self, gem: &gem::Object) -> Result { + let _residency = self.enter_retirement_residency(); // Removing whole mappings only does unmaps, so no preallocated VAs let mut ctx = Default::default(); diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index d2e429c43d311..df189e9bc0f5f 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -14,18 +14,19 @@ pub(crate) struct PreparedUserMap { offset: u64, single_page: bool, context: context::PreparedBinding, + _residency: Option, } pub(crate) struct PreparedUserUnmap { + _residency: Option, pub(super) iova: u64, pub(super) size: u64, pub(super) ctx: StepContext, } -// SAFETY: This owns only its numeric range and unused, unlinked split-node -// allocations. step_remap moves each node out of ctx before linking it into -// the VM, so a retained ctx never owns a linked GPUVA. The unmap context has -// no borrowed object/lock state; lifetime.state serializes deferred ownership. +// SAFETY: This owns a thread-safe residency lease, a numeric range and unused, +// unlinked split-node allocations; step_remap moves each node out of ctx before +// linking it, so a retained ctx never owns a linked GPUVA. unsafe impl Send for PreparedUserUnmap {} pub(super) enum PreparedUserBindOp { @@ -66,6 +67,7 @@ impl Vm { single_page: bool, ) -> Result { gem::validate_vm_binding(gem, self)?; + let residency = self.enter_residency_metadata()?; let context = self.prepare_context_binding(addr, size, offset)?; let mut ctx = StepContext { new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), @@ -106,6 +108,7 @@ impl Vm { return Err(EINVAL); } Ok(PreparedUserMap { + _residency: residency, gem: gem.into(), ctx, addr, @@ -120,7 +123,9 @@ impl Vm { if (iova | size) & UAT_PGMSK as u64 != 0 || !self.inner.range_valid(iova, size) { return Err(EINVAL); } + let residency = self.enter_residency_metadata()?; Ok(PreparedUserUnmap { + _residency: residency, iova, size, ctx: StepContext { diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 20a79037593df..d53c19b93bfdc 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -71,6 +71,14 @@ pub(super) struct VmLifetime { } impl VmLifetime { + pub(super) fn can_reclaim(&self) -> bool { + self.state.try_lock().is_some_and(|state| { + state.active == 0 && state.commits == 0 + && !state.blocked && !state.quarantined && !state.draining + && !state.closed && !state.cleanup_failed + }) + } + pub(super) fn new() -> Result> { Arc::pin_init( pin_init!(Self { @@ -89,6 +97,7 @@ impl VmLifetime { /// retirement: potentially visible work must retain its guard through quarantine or stop. pub(crate) struct VmJobGuard { vm: Vm, + _residency: Option, } impl VmJobGuard { @@ -344,6 +353,7 @@ impl Vm { /// Acquires a job reference without imposing a VM-wide limit on healthy work. pub(crate) fn retain_job(&self) -> Result { + let residency = self.enter_residency()?; let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; let mut state = lifetime.state.lock(); loop { @@ -355,7 +365,7 @@ impl Vm { } if !state.draining { state.active = state.active.checked_add(1).ok_or(EOVERFLOW)?; - return Ok(VmJobGuard { vm: self.clone() }); + return Ok(VmJobGuard { vm: self.clone(), _residency: residency }); } lifetime.changed.wait(&mut state); } @@ -533,6 +543,7 @@ impl Vm { } fn unmap_user_ranges_now(&self, user: Range, _kernel: Range) -> Result { + let _residency = self.enter_retirement_residency(); // User mappings alone have GPUVA nodes. Kernel and driver aliases use // mm::Node and are untouched; GPUVM's embedded kernel cutout is skipped. // Closed admission and zero accepted jobs make whole-node teardown safe. @@ -568,6 +579,7 @@ impl Vm { /// Admission stays closed to new jobs while the exec lock is acquired for cleanup. /// Recheck queued cleanup and reopen under the same lock used to append it. fn drain_mappings(&self, lifetime: &VmLifetime) { + let _residency = self.enter_retirement_residency(); let mut state = lifetime.state.lock(); let mut failure = None; loop { diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs new file mode 100644 index 0000000000000..a66d9cff73e2d --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -0,0 +1,484 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Pressure-only backing residency for idle G17 user VMs. +//! +//! This gate protects backing/address preparation as well as accepted jobs. +//! Its writer never waits for active readers: reclaim skips the VM instead. + +use super::*; +use kernel::drm::gem::BaseObject; +use kernel::{new_condvar, sync::CondVar}; + +#[derive(Default)] +struct State { + users: usize, + busy: bool, + evicted: bool, + closed: bool, +} + +#[pin_data] +pub(crate) struct Gate { + #[pin] + state: Mutex, + #[pin] + ready: CondVar, +} + +pub(crate) struct Lease { + gate: Arc, +} + +pub(crate) struct Reclaim { + gate: Arc, + changed: bool, +} + +impl Gate { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + state <- new_mutex!(State::default(), "Asahi residency"), + ready <- new_condvar!("Asahi residency ready"), + }), + GFP_KERNEL, + ) + } + + /// Only a previous pressure eviction can cause restoration or waiting. + pub(crate) fn enter(gate: &Arc, mut restore: impl FnMut() -> Result) -> Result { + loop { + let mut state = gate.state.lock(); + if state.closed { + return Err(ENOENT); + } + while state.busy { + if gate.ready.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + } + if state.closed { + return Err(ENOENT); + } + if !state.evicted { + state.users = state.users.checked_add(1).ok_or(EOVERFLOW)?; + return Ok(Lease { gate: gate.clone() }); + } + // Binding edits can add mappings or retire invalidated GPUVAs + // without restoring the rest of the VM. Wait for those mutations + // before walking the authoritative logical bindings for restoration. + if state.users != 0 { + if gate.ready.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + continue; + } + // No new reader can enter until every retained mapping is restored. + state.busy = true; + drop(state); + let result = restore(); + let mut state = gate.state.lock(); + state.busy = false; + if result.is_ok() { + state.evicted = false; + } + drop(state); + gate.ready.notify_all(); + result?; + } + } + + pub(crate) fn enter_metadata(gate: &Arc) -> Result { + let mut state = gate.state.lock(); + while state.busy { + if gate.ready.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + } + if state.closed { + return Err(ENOENT); + } + state.users = state.users.checked_add(1).ok_or(EOVERFLOW)?; + Ok(Lease { gate: gate.clone() }) + } + + /// Retirement of an already closed handle may outlive the file's VM. + /// A closed gate excludes every new reclaimer and restorer, so no lease + /// is necessary then. Otherwise serialize metadata removal with restoration. + pub(super) fn enter_retirement(gate: &Arc) -> Option { + let mut state = gate.state.lock(); + while state.busy { + gate.ready.wait(&mut state); + } + if state.closed { + return None; + } + state.users += 1; + Some(Lease { gate: gate.clone() }) + } + + /// Direct reclaim cannot block on a job, another reclaimer or restoration. + /// A prior partial eviction does not exclude another scan: the VM may + /// still retain eligible BOs after satisfying a previous scan target. + /// The busy gate continues to exclude restoration and metadata changes. + pub(crate) fn try_reclaim(gate: &Arc) -> Option { + let mut state = gate.state.try_lock()?; + if state.users != 0 || state.busy || state.closed { + return None; + } + state.busy = true; + Some(Reclaim { + gate: gate.clone(), + changed: false, + }) + } + + pub(crate) fn close(&self) { + let mut state = self.state.lock(); + state.closed = true; + while state.busy { + self.ready.wait(&mut state); + } + } + + pub(crate) fn idle(&self) -> bool { + self.state + .try_lock() + .is_some_and(|state| state.users == 0 && !state.busy && !state.closed) + } +} + +impl Reclaim { + /// Call before the first page-table mutation, including fallible unmaps. + pub(crate) fn mark_changed(&mut self) { + self.changed = true; + } +} + +impl Drop for Reclaim { + fn drop(&mut self) { + let mut state = self.gate.state.lock(); + state.evicted |= self.changed; + state.busy = false; + drop(state); + self.gate.ready.notify_all(); + } +} + +impl Drop for Lease { + fn drop(&mut self) { + let mut state = self.gate.state.lock(); + debug_assert!(state.users != 0); + state.users -= 1; + if state.users == 0 { + drop(state); + self.gate.ready.notify_all(); + } + } +} + +/// Registration lifetime belongs to the file's VM handle, not the VM clone +/// retained by the callback. Unregister synchronously before dropping the clone. +pub(crate) struct VmShrinker { + vm: crate::mmu::Vm, + raw: *mut kernel::bindings::shrinker, +} + +// SAFETY: shrinker core serializes registration lifetime; callback VM operations +// use its residency/reservation locks. raw is never dereferenced after free. +unsafe impl Send for VmShrinker {} +unsafe impl Sync for VmShrinker {} + +impl VmShrinker { + pub(crate) fn new(vm: &crate::mmu::Vm) -> Result> { + let mut owner = KBox::new( + Self { + vm: vm.clone(), + raw: core::ptr::null_mut(), + }, + GFP_KERNEL, + )?; + // SAFETY: The format is static and has no format arguments. + let raw = + unsafe { kernel::bindings::shrinker_alloc(0, c_str!("asahi-idle-vm").as_char_ptr()) }; + if raw.is_null() { + return Err(ENOMEM); + } + owner.raw = raw; + // SAFETY: Newly allocated/unregistered shrinker is exclusively owned. + unsafe { + (*raw).count_objects = Some(Self::count); + (*raw).scan_objects = Some(Self::scan); + (*raw).private_data = (&*owner as *const Self).cast_mut().cast(); + (*raw).seeks = 2; + (*raw).batch = 256; + kernel::bindings::shrinker_register(raw); + } + Ok(owner) + } + + unsafe extern "C" fn count( + raw: *mut kernel::bindings::shrinker, + _sc: *mut kernel::bindings::shrink_control, + ) -> kernel::ffi::c_ulong { + // SAFETY: The registration owns this stable allocation until callbacks drain. + let owner = unsafe { &*((*raw).private_data as *const Self) }; + owner.vm.idle_reclaim_count() as _ + } + + unsafe extern "C" fn scan( + raw: *mut kernel::bindings::shrinker, + sc: *mut kernel::bindings::shrink_control, + ) -> kernel::ffi::c_ulong { + // SAFETY: Both pointers are supplied by the registered shrinker core. + let owner = unsafe { &*((*raw).private_data as *const Self) }; + let target = unsafe { (*sc).nr_to_scan } as usize; + owner.vm.reclaim_idle_pages(target) as _ + } +} + +impl Drop for VmShrinker { + fn drop(&mut self) { + if !self.raw.is_null() { + // SAFETY: This owns the sole registration; free drains callbacks + // before the embedded VM and callback private_data can disappear. + unsafe { kernel::bindings::shrinker_free(self.raw) }; + } + } +} + +impl Vm { + pub(crate) fn close_idle_reclaim(&self) { + if let Some(gate) = self.residency.as_ref() { + gate.close(); + } + } + + pub(crate) fn residency_reclaim_enabled(&self) -> bool { + self.residency.is_some() + } + + pub(super) fn enter_residency_metadata(&self) -> Result> { + self.residency + .as_ref() + .map(Gate::enter_metadata) + .transpose() + } + + pub(super) fn enter_residency(&self) -> Result> { + self.residency + .as_ref() + .map(|gate| Gate::enter(gate, || self.restore_idle_pages())) + .transpose() + } + + pub(crate) fn idle_reclaim_count(&self) -> usize { + let Some(gate) = self.residency.as_ref() else { + return 0; + }; + if !gate.idle() + || self + .status + .as_ref() + .map_or(true, |status| status.get() != 0) + { + return 0; + } + let Some(mut inner) = self.inner.try_lock_private() else { + return 0; + }; + let mut pages = 0usize; + // SAFETY: Counting changes no logical mapping/list. Private BOs share + // this reservation; linked aliases retain their borrowed BO and GEM. + let _ = unsafe { inner.for_each_private_bo(|_, bo| { + let object = bo.object(); + if object.idle_reclaim_candidate() && object.sole_sg_lease_releasable_locked() { + pages = pages.saturating_add(object.size() / kernel::page::PAGE_SIZE); + } + Ok(true) + }) }; + + pages + } + + pub(crate) fn reclaim_idle_pages(&self, target: usize) -> usize { + let Some(gate) = self.residency.as_ref() else { + return 0; + }; + let Some(mut reclaim) = Gate::try_reclaim(gate) else { + return 0; + }; + // Error fences cannot prove firmware ownership ended. + if self + .status + .as_ref() + .map_or(true, |status| status.get() != 0) + || !self + .lifetime + .as_ref() + .is_some_and(|lifetime| lifetime.can_reclaim()) + { + return 0; + } + let Some(mut inner) = self.inner.try_lock_private() else { + return 0; + }; + let uat = inner.uat_inner.clone(); + let Some(shared) = uat.shared.try_lock() else { + return 0; + }; + let contexts = shared.contexts_naming_root(inner.ttb()); + let mut unpinned = 0usize; + // SAFETY: Exclusive residency and the private reservation exclude jobs, + // binding edits and retirement. Callbacks change PTEs/flags only; all + // logical aliases stay linked and retain their borrowed BO/GEM. + let _ = unsafe { inner.for_each_private_bo(|inner, bo| { + if unpinned >= target { return Ok(false); } + let object = bo.object(); + if !object.idle_reclaim_candidate() { return Ok(true); } + { + // Avoid inversion against preparation (VmBoInner -> resv). + let Some(bo_inner) = bo.inner().inner.try_lock() else { return Ok(true); }; + if bo_inner.sgt.is_none() || !object.sole_sg_lease_releasable_locked() { + return Ok(true); + } + } + // The association retains SG backing until *all* aliases have + // been invalidated and their device translations flushed. A + // contended cursor or PTE failure retains backing and the touched + // INVALIDATED prefix for restoration or a later reclaim attempt. + inner.invalidate_private_bo_mappings(bo, |inner, start, range| { + reclaim.mark_changed(); + let end = start.checked_add(range).ok_or(EOVERFLOW)?; + let _change = inner.mapping_mutation(); + inner.page_table.discard_partial_map(start..end)?; + inner.tlbi_context_mask(start, range as usize, contexts); + Ok(()) + })?; + { + let Some(mut bo_inner) = bo.inner().inner.try_lock() else { return Ok(true); }; + bo_inner.sg_vec = None; + bo_inner.sgt = None; + } + // Every private GPUVA alias was removed from the device page + // tables above. Kernel mappings and other SG owners are excluded; + // the C/Rust backing helper rechecks exclusion before release. + if object.release_idle_pages_locked().is_ok() { + unpinned = unpinned.saturating_add(object.size() / kernel::page::PAGE_SIZE); + } + Ok(unpinned < target) + }) }; + drop(shared); + drop(inner); + + unpinned + } + + fn restore_idle_pages(&self) -> Result { + // Unbind/free can remove or split invalidated mappings while the VM + // stays evicted. Rebuild solely from the authoritative logical GPUVAs, + // never from stale pre-eviction records that could resurrect a BO. + // Gate::enter holds exclusive residency ownership for this entire + // walk: metadata edits, retirement and reclaim cannot change the tree. + // Retain one invalidated mapping at a time, avoiding a VM-sized KVec + // allocation precisely when memory pressure requires restoration. + let mut cursor = 0u64; + loop { + let map = loop { + if let Some(inner) = self.inner.try_lock_private() { + break inner.next_private_mapping( + &mut cursor, gpuvm::GpuVaFlags::INVALIDATED, + )?; + } + // Wait for reservation ownership without retaining a mapping. + drop(self.inner.exec_lock(None, true)?); + }; + // The reservation guard has dropped before pinning backing, and + // before any returned object reference can be released on failure. + let Some(map) = map else { break; }; + let mut restored = 0u64; + let bo_owner = self.inner.find_bo(&map.object).ok_or(EFAULT)?; + { + let needs_sgt = bo_owner.inner().inner.lock().sgt.is_none(); + if needs_sgt { + // Preserve review1's reservation/BO lock order: the getter + // may acquire the object reservation; never hold BO here. + let sgt = map.object.owned_sg_table()?; + let mut bo = bo_owner.inner().inner.lock(); + if bo.sgt.is_none() { bo.sgt = Some(sgt); } + drop(bo); + } + } + let mut inner = self.inner.exec_lock(None, true)?; + let bo = bo_owner.inner().inner.lock(); + let sgt = bo.sgt.as_ref().ok_or(EFAULT)?; + let result = (|| -> Result { + let end = map.addr.checked_add(map.range).ok_or(EOVERFLOW)?; + let _mutation = inner.mapping_mutation(); + inner.page_table.discard_partial_map(map.addr..end)?; + inner.tlbi_contexts_naming_root(map.addr, map.range as usize); + let prot = map.inner.prot.ok_or(EINVAL)?; + let repeat = map.flags.contains(gpuvm::GpuVaFlags::REPEAT); + let mut skip = map + .offset + .checked_add(if repeat { 0 } else { restored }) + .ok_or(EOVERFLOW)? as usize; + let mut left = map.range.checked_sub(restored).ok_or(EOVERFLOW)? as usize; + for entry in sgt.iter() { + if left == 0 { + break; + } + let len = entry.dma_len() as usize; + if skip >= len { + skip -= len; + continue; + } + let physical = entry.dma_address() as usize + skip; + let bytes = if repeat { left } else { left.min(len - skip) }; + let start = map.addr.checked_add(restored).ok_or(EOVERFLOW)?; + let end = start.checked_add(bytes as u64).ok_or(EOVERFLOW)?; + let _change = inner.mapping_mutation(); + inner.page_table.map_pages( + start..end, + physical as PhysicalAddr, + prot, + repeat, + )?; + restored += bytes as u64; + left -= bytes; + skip = 0; + } + if left != 0 { + return Err(EFAULT); + } + // The exclusive residency gate excludes submissions until + // every invalidated mapping is restored. Publish this logical + // range once, instead of issuing a synchronized TLBI for each + // physical SG segment. Keep the preceding break-before-make + // invalidation; the error path clears and flushes the entire + // range if any map operation (including a partial one) fails. + inner.tlbi_contexts_naming_root(map.addr, map.range as usize); + inner.set_invalidated(map.addr, map.range, false)?; + + Ok(()) + })(); + if let Err(error) = result { + // Keep an invalidated GPUVA wholly unmapped even on partial + // restoration failure, so later unbind/close cannot skip a + // prefix that was already reinstalled. Backing stays retained. + // map_pages may fail inside its current SG segment. Clear the + // complete logical range, including that partially written segment. + let end = map.addr.checked_add(map.range).ok_or(EOVERFLOW)?; + let _mutation = inner.mapping_mutation(); + inner.page_table.discard_partial_map(map.addr..end)?; + inner.tlbi_contexts_naming_root(map.addr, map.range as usize); + return Err(error); + } + } + Ok(()) + } + + pub(super) fn enter_retirement_residency(&self) -> Option { + self.residency.as_ref().and_then(Gate::enter_retirement) + } +} From 717d6fe9e152af9bdda4a73a81151145cb2681ca Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:01:06 -0400 Subject: [PATCH 0642/1579] drm/asahi: free a closed context's render pool once its release is consumed A logical context owns one render free-list pool (about 24 MiB of page and run tables plus 22 USC blocks). The pool was dropped only with the Context, but an installed compute graph keeps its last Context until a successor owner replaces it, so every idle compute QID pinned the exited client's render pool as well. On an 8 GB machine this grew kernel GEM from 252 MB (fresh desktop) to 752 MB after bursts of parallel clients and never shrank. The teardown service only publishes a scheduler release after every pool the context registered has had its free-list release consumed, and the context reaches the reclaim batch after that scheduler release is consumed too. Drop the pool there: nothing on the firmware side names it any more, and the retained compute binding never uses it. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e82b27902f91d8c833395a966366d0ffc30769c3) --- drivers/gpu/drm/asahi/g17/context.rs | 6 ++++++ drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 1 + 2 files changed, 7 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index ee2d8133ea8cd..eb4ddcac7b808 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -273,6 +273,12 @@ impl Context { *render = Some(pool.clone()); Ok(pool) } + /// Drops the pool once its free-list release is consumed and the closed + /// scheduler is detached. Installed compute bindings may retain the + /// context itself until replacement; they never use the render pool. + pub(crate) fn take_render_pool(&self) -> Option> { + self.render.lock().take() + } pub(crate) fn render_pools_released(&self, released: impl Fn(u32, u64) -> bool) -> bool { self.render .lock() diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index b1495dbf3e23e..f310f8154aae7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -32,6 +32,7 @@ impl Reclaim { Self::Binding(binding) => drop(binding), Self::Context(context) => { context.release_execution(); + drop(context.take_render_pool()); drop(context); } } From 20a55bacd2fb91e52b6bb5cbb3e88e5218b2f726 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:13:49 -0400 Subject: [PATCH 0643/1579] drm/asahi: hold idle descent for every first render pair install The first install of a physical render pair acquired the registration power hold only when the core power register read unpowered at that instant. A powered reading does not keep the firmware from starting idle descent before the host's configuration writes land. An earlier version of this driver with the same conditional hold hard-hung (watchdog reset, no log) when several clients installed their first pairs at once; with the hold taken unconditionally the same workloads ran clean. Acquire the hold before every first install and release it where the pair's pool is activated, as before. This costs a few tens of microseconds on a client's first render submission only. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit fa210660c43c1e8ffc4c0705792febb70c29e779) --- drivers/gpu/drm/asahi/g17/runtime/render.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index e466571bc88c7..bf86d46e7ca6b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -393,12 +393,15 @@ impl crate::g17::queue::render::Host for Host<'_> { .accounting .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) } + /// Idle descent is held for the whole first install. A point-in-time power + /// check let the firmware gate the cores between the host's configuration + /// writes while several clients installed their first pairs at once. fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + if !self.registration_powered { + self.firmware.acquire_registration_power()?; + self.registration_powered = true; + } for _ in 0..3 { - if !self.firmware.bridge.regs.g17_configuration_powered() { - self.firmware.acquire_registration_power()?; - self.registration_powered = true; - } if self .firmware .bridge From bb6c54f04f3245e7e39a22f7a4103f913b544f77 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:10:32 -0400 Subject: [PATCH 0644/1579] drm/shmem-helper: index fault pages by object offset The fault, huge-fault and mkwrite paths computed the page index as vmf->pgoff - vma->vm_pgoff, the offset within the VMA. vm_pgoff keeps the DRM fake offset (the object's vma_node start, plus the mmap offset of a dma-buf mapping), so after a VMA split or an offset mmap that index names the wrong page. Subtract the object's vma_node start instead. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6db05cf4b231f020020f31a479bd45ccff83520f) # Conflicts: # drivers/gpu/drm/drm_gem_shmem_helper.c From 4a98d86960c13bfe5f688a47562f7f02ece7412d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:41:38 -0400 Subject: [PATCH 0645/1579] drm/shmem-helper: populate revocable CPU mappings one page at a time A fault on a revocable CPU mapping took the whole page array, so touching two bytes of a 64 MiB object materialized and pinned all of it. Keep a sparse per-object array for CPU-faulted pages while no device mapping holds the full array: each fault reads only its own page from the shmem mapping. The device path is unchanged and may coexist with the sparse array (both reference the same page-cache pages). The sparse pages are released when the last VMA closes, when the driver releases idle backing (after zapping the VMAs) or on purge/free. The sparse pages are inserted as PFN mappings, which have no rmap, so keep the mapping unevictable while sparse pages exist and move the folios back to the evictable lists when the last holder releases them, as drm_gem_get_pages() and drm_gem_put_pages() do for the device array. Otherwise reclaim would see the pages as unmapped and could swap them out behind the live PTEs. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 4f7a3272df9627d98bc11a7da0c8189ca4aba196) --- include/drm/drm_gem_shmem_helper.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index a0f2b3ebcf59f..736e49baf803f 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -37,8 +37,14 @@ struct drm_gem_shmem_object { /** @reclaimable_cpu_mappings: CPU VMAs may be revoked by the driver. */ bool reclaimable_cpu_mappings; - /** @cpu_pages_owned: One object-scoped CPU page-array ref, under resv. */ - bool cpu_pages_owned; + /** + * @cpu_pages: Pages populated one at a time by CPU faults on revocable + * VMAs while no device mapping holds @pages. Sparse; under base.resv. + */ + struct page **cpu_pages; + + /** @cpu_pages_count: Populated entries of @cpu_pages. */ + unsigned long cpu_pages_count; /** * @pages_use_count: From e94491e80ac12de2fb9b741739b560ab17ecbb34 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:56:27 -0400 Subject: [PATCH 0646/1579] drm/asahi: add GEM_MADVISE and purge advised objects under memory pressure The kernel cannot tell which VM-private objects an in-flight job may reference, so pressure reclaim only touches idle VMs. Userspace knows: Mesa's BO cache holds freed objects that no pending or future work uses until they are handed out again. Add DRM_IOCTL_ASAHI_GEM_MADVISE with WILLNEED/DONTNEED advice for VM-private, non-exportable objects. The VM shrinker now also scans DONTNEED objects of busy VMs: it invalidates their aliases, clears the page tables and TLB, drops the VM's SG lease and purges the shmem backing. Restoration skips purged objects (their mappings stay INVALIDATED until unbound) and WILLNEED reports retained=0 for them so userspace reallocates. A purge that invalidates aliases but does not complete (contended GEM list, page-table error, or a backing lease that appeared meanwhile) records the eviction on the residency gate, so the next submission restores those aliases instead of leaving them unmapped behind a WILLNEED that reports the contents retained. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d7715ebbd2d3397958f08442d096a2476d32cb6a) --- drivers/gpu/drm/asahi/driver.rs | 2 + drivers/gpu/drm/asahi/file.rs | 17 ++++ drivers/gpu/drm/asahi/gem.rs | 9 ++ drivers/gpu/drm/asahi/mmu/residency.rs | 126 ++++++++++++++++++++++--- include/uapi/drm/asahi_drm.h | 39 ++++++++ rust/helpers/drm.c | 6 ++ rust/kernel/drm/gem/shmem.rs | 43 +++++++++ 7 files changed, 229 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 36c223b1d372d..abdef98673f96 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -105,6 +105,8 @@ impl drm::driver::Driver for AsahiDriver { ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::queue_destroy), (ASAHI_SUBMIT, drm_asahi_submit, ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::submit), + (ASAHI_GEM_MADVISE, drm_asahi_gem_madvise, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_madvise), } } diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index d901fd5b4ff42..aaf289c411c20 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -636,6 +636,23 @@ impl File { Ok(0) } + /// IOCTL: gem_madvise: Mark a VM-private object purgeable or needed. + pub(crate) fn gem_madvise( + _device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_madvise, + file: &DrmFile, + ) -> Result { + if data.pad != 0 || data.madv > uapi::drm_asahi_gem_madv_DRM_ASAHI_GEM_MADV_DONTNEED { + return Err(EINVAL); + } + let gem = gem::Object::lookup_handle(file, data.handle)?; + data.retained = gem::madvise( + &gem, + data.madv == uapi::drm_asahi_gem_madv_DRM_ASAHI_GEM_MADV_DONTNEED, + )?; + Ok(0) + } + /// IOCTL: vm_bind: Map or unmap memory into a Vm. pub(crate) fn vm_bind( device: &AsahiDevice, diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index a169b8594de83..cb3fe6e038eae 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -316,3 +316,12 @@ impl AsahiObject { !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 } } + +/// Purge advice for a VM-private, non-exportable object. Returns whether the +/// contents are retained (0 once purged). +pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { + if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE == 0 || gem.exportable { + return Err(EINVAL); + } + Ok(u32::from(gem.madvise(i32::from(dontneed)))) +} diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index a66d9cff73e2d..343da90957602 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -146,6 +146,12 @@ impl Gate { .try_lock() .is_some_and(|state| state.users == 0 && !state.busy && !state.closed) } + + /// A page-table mutation happened outside a `Reclaim` (the purge path runs + /// on busy VMs). The next `enter` restores every INVALIDATED mapping. + pub(crate) fn note_evicted(&self) { + self.state.lock().evicted = true; + } } impl Reclaim { @@ -276,14 +282,14 @@ impl Vm { let Some(gate) = self.residency.as_ref() else { return 0; }; - if !gate.idle() - || self - .status - .as_ref() - .map_or(true, |status| status.get() != 0) + if self + .status + .as_ref() + .map_or(true, |status| status.get() != 0) { return 0; } + let idle = gate.idle(); let Some(mut inner) = self.inner.try_lock_private() else { return 0; }; @@ -292,7 +298,11 @@ impl Vm { // this reservation; linked aliases retain their borrowed BO and GEM. let _ = unsafe { inner.for_each_private_bo(|_, bo| { let object = bo.object(); - if object.idle_reclaim_candidate() && object.sole_sg_lease_releasable_locked() { + // Purgeable objects count on busy VMs too; idle backing only on idle ones. + if object.idle_reclaim_candidate() + && (object.madv_locked() > 0 || idle) + && object.sole_sg_lease_releasable_locked() + { pages = pages.saturating_add(object.size() / kernel::page::PAGE_SIZE); } Ok(true) @@ -301,12 +311,94 @@ impl Vm { pages } + /// Discard the backing of objects userspace marked DONTNEED. This needs + /// only the VM reservation: binding edits and restoration hold it too, + /// and the advice guarantees no pending or future GPU reference until + /// the next WILLNEED. Purged mappings stay INVALIDATED until unbound. + fn purge_advised_pages(&self, target: usize) -> usize { + let Some(gate) = self.residency.as_ref() else { + return 0; + }; + if self + .status + .as_ref() + .map_or(true, |status| status.get() != 0) + { + return 0; + } + let Some(mut inner) = self.inner.try_lock_private() else { + return 0; + }; + let uat = inner.uat_inner.clone(); + let Some(shared) = uat.shared.try_lock() else { + return 0; + }; + let contexts = shared.contexts_naming_root(inner.ttb()); + let mut purged = 0usize; + // SAFETY: The private reservation excludes binding edits and restoration. + // Callbacks change PTEs/flags only; all aliases stay linked. + let _ = unsafe { inner.for_each_private_bo(|inner, bo| { + if purged >= target { return Ok(false); } + let object = bo.object(); + if !object.idle_reclaim_candidate() || object.madv_locked() <= 0 { + return Ok(true); + } + { + let Some(bo_inner) = bo.inner().inner.try_lock() else { return Ok(true); }; + if bo_inner.sgt.is_some() && !object.sole_sg_lease_releasable_locked() { + return Ok(true); + } + } + // Any mutation that does not end in a completed purge leaves + // INVALIDATED aliases with cleared PTEs while madv stays >= 0, so + // the next WILLNEED would report the contents retained. Record the + // eviction so the gate restores those aliases before the next job. + let mut mutated = false; + let invalidated = inner.invalidate_private_bo_mappings(bo, |inner, start, range| { + mutated = true; + let end = start.checked_add(range).ok_or(EOVERFLOW)?; + let _change = inner.mapping_mutation(); + inner.page_table.discard_partial_map(start..end)?; + inner.tlbi_context_mask(start, range as usize, contexts); + Ok(()) + }); + if let Err(error) = invalidated { + if mutated { + gate.note_evicted(); + } + return Err(error); + } + { + let Some(mut bo_inner) = bo.inner().inner.try_lock() else { + gate.note_evicted(); + return Ok(true); + }; + bo_inner.sg_vec = None; + bo_inner.sgt = None; + } + if object.purge_locked().is_ok() { + purged = purged.saturating_add(object.size() / kernel::page::PAGE_SIZE); + } else { + gate.note_evicted(); + } + Ok(purged < target) + }) }; + drop(shared); + drop(inner); + purged + } + pub(crate) fn reclaim_idle_pages(&self, target: usize) -> usize { let Some(gate) = self.residency.as_ref() else { return 0; }; + let purged = self.purge_advised_pages(target); + if purged >= target { + return purged; + } + let target = target - purged; let Some(mut reclaim) = Gate::try_reclaim(gate) else { - return 0; + return purged; }; // Error fences cannot prove firmware ownership ended. if self @@ -318,14 +410,14 @@ impl Vm { .as_ref() .is_some_and(|lifetime| lifetime.can_reclaim()) { - return 0; + return purged; } let Some(mut inner) = self.inner.try_lock_private() else { - return 0; + return purged; }; let uat = inner.uat_inner.clone(); let Some(shared) = uat.shared.try_lock() else { - return 0; + return purged; }; let contexts = shared.contexts_naming_root(inner.ttb()); let mut unpinned = 0usize; @@ -371,7 +463,7 @@ impl Vm { drop(shared); drop(inner); - unpinned + purged + unpinned } fn restore_idle_pages(&self) -> Result { @@ -386,16 +478,24 @@ impl Vm { loop { let map = loop { if let Some(inner) = self.inner.try_lock_private() { - break inner.next_private_mapping( + let map = inner.next_private_mapping( &mut cursor, gpuvm::GpuVaFlags::INVALIDATED, )?; + // Purged (DONTNEED) objects have no contents to restore; + // their mappings stay INVALIDATED until unbound. + // SAFETY: The private guard holds the VM reservation. + let purged = map.as_ref().is_some_and(|map| unsafe { map.object.madv_locked() < 0 }); + break (map, purged); } // Wait for reservation ownership without retaining a mapping. drop(self.inner.exec_lock(None, true)?); }; // The reservation guard has dropped before pinning backing, and // before any returned object reference can be released on failure. - let Some(map) = map else { break; }; + let (Some(map), purged) = map else { break; }; + if purged { + continue; + } let mut restored = 0u64; let bo_owner = self.inner.find_bo(&map.object).ok_or(EFAULT)?; { diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index cba36bd5db82e..cdb177bf4df05 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -93,6 +93,9 @@ enum drm_asahi_ioctl_id { /** @DRM_ASAHI_SUBMIT: Submit commands to a queue. */ DRM_ASAHI_SUBMIT, + + /** @DRM_ASAHI_GEM_MADVISE: Mark a VM-private object purgeable or needed. */ + DRM_ASAHI_GEM_MADVISE, }; #define DRM_ASAHI_MAX_CLUSTERS 64 @@ -519,6 +522,41 @@ struct drm_asahi_gem_mmap_offset { __u64 offset; }; +/** + * enum drm_asahi_gem_madv - Advice for DRM_IOCTL_ASAHI_GEM_MADVISE + */ +enum drm_asahi_gem_madv { + /** @DRM_ASAHI_GEM_MADV_WILLNEED: The object's contents are needed. */ + DRM_ASAHI_GEM_MADV_WILLNEED = 0, + /** + * @DRM_ASAHI_GEM_MADV_DONTNEED: The object is not referenced by any + * pending or future GPU work until the next WILLNEED. The kernel may + * discard its contents under memory pressure. + */ + DRM_ASAHI_GEM_MADV_DONTNEED = 1, +}; + +/** + * struct drm_asahi_gem_madvise - Arguments passed to + * DRM_IOCTL_ASAHI_GEM_MADVISE + * + * Only VM-private objects accept advice. A purged object keeps its handle + * and mappings but has no contents; userspace must not use it again. + */ +struct drm_asahi_gem_madvise { + /** @handle: Handle of the object. */ + __u32 handle; + + /** @madv: One of enum drm_asahi_gem_madv. */ + __u32 madv; + + /** @retained: Set by the kernel: 1 if the contents are retained, 0 if purged. */ + __u32 retained; + + /** @pad: Must be zero. */ + __u32 pad; +}; + /** * enum drm_asahi_bind_flags - Flags for GEM binding */ @@ -1430,6 +1468,7 @@ enum { DRM_IOCTL_ASAHI_QUEUE_CREATE = DRM_IOCTL_ASAHI(WR, QUEUE_CREATE, queue_create), DRM_IOCTL_ASAHI_QUEUE_DESTROY = DRM_IOCTL_ASAHI(W, QUEUE_DESTROY, queue_destroy), DRM_IOCTL_ASAHI_SUBMIT = DRM_IOCTL_ASAHI(W, SUBMIT, submit), + DRM_IOCTL_ASAHI_GEM_MADVISE = DRM_IOCTL_ASAHI(WR, GEM_MADVISE, gem_madvise), }; #if defined(__cplusplus) diff --git a/rust/helpers/drm.c b/rust/helpers/drm.c index 65f3f22b0e1d2..0975c35426f81 100644 --- a/rust/helpers/drm.c +++ b/rust/helpers/drm.c @@ -29,6 +29,12 @@ rust_helper_drm_gem_shmem_object_free(struct drm_gem_object *obj) return drm_gem_shmem_object_free(obj); } +__rust_helper bool +rust_helper_drm_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem) +{ + return drm_gem_shmem_is_purgeable(shmem); +} + __rust_helper void rust_helper_drm_gem_shmem_object_print_info(struct drm_printer *p, unsigned int indent, const struct drm_gem_object *obj) diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 51f25fe9c1f0a..f0e49537e0c3a 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -297,6 +297,49 @@ impl Object { result } + /// Apply purge advice (0 = needed, >0 = may be purged) under the + /// object's reservation lock. Returns false if the object was already + /// purged; the advice is then ignored. + pub fn madvise(&self, madv: i32) -> bool { + // SAFETY: The object retains its reservation; the C helper requires it held. + unsafe { + bindings::dma_resv_lock(self.raw_dma_resv(), core::ptr::null_mut()); + let retained = bindings::drm_gem_shmem_madvise_locked(self.as_raw_shmem(), madv) != 0; + bindings::dma_resv_unlock(self.raw_dma_resv()); + retained + } + } + + /// Current purge advice: negative once purged. + /// + /// # Safety + /// The caller holds the object's DMA reservation lock. + pub unsafe fn madv_locked(&self) -> i32 { + // SAFETY: Reservation ownership serializes the field. + unsafe { (*self.as_raw_shmem()).madv } + } + + /// Discard the backing of a purgeable object. The handle and logical + /// mappings remain; the contents are gone and `madv_locked()` turns negative. + /// + /// # Safety + /// + /// The caller holds this object's DMA reservation lock, has removed and + /// invalidated every device mapping, and no Rust SG lease remains. + pub unsafe fn purge_locked(&self) -> Result { + // SAFETY: The caller's contract; the exclusive state below excludes new leases. + if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } { + return Err(EBUSY); + } + self.sgt_access.compare_exchange( + 0, SGT_RECLAIMING, Ordering::AcqRel, Ordering::Acquire, + ).map_err(|_| EBUSY)?; + // SAFETY: See above. + unsafe { bindings::drm_gem_shmem_purge_locked(self.as_raw_shmem()) }; + self.sgt_access.store(0, Ordering::Release); + Ok(()) + } + /// Attempt to create a [`RawIoSysMap`] from the gem object. fn raw_vmap(&self) -> Result> { build_assert!( From 85271d321506ec35cc7e6a82665649e7fd2e4f8c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:57:01 -0400 Subject: [PATCH 0647/1579] drm/asahi: drop the sysfs object and queue-slot counters gem_stats and g17p_queue_slots exported driver-internal counters as read-only module parameters, backed by global atomics on the GEM constructor/destructor, the QID lease table, compute queue state transitions, the teardown and settle passes and every ENOSPC return. Nothing in the driver reads them, and DRM exposes memory accounting through fdinfo. Remove the parameters, the atomics and the hooks; an exhausted pool returns ENOSPC without a counted log line. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c8c442ae6a83502a90fca56d1b8e08e669ce03cb) --- drivers/gpu/drm/asahi/asahi.rs | 1 - drivers/gpu/drm/asahi/g17/context.rs | 2 +- drivers/gpu/drm/asahi/g17/kick.rs | 49 +---- drivers/gpu/drm/asahi/g17/mod.rs | 1 - drivers/gpu/drm/asahi/g17/queue/compute.rs | 16 -- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 1 - drivers/gpu/drm/asahi/g17/queue_stats.rs | 177 ------------------ drivers/gpu/drm/asahi/g17/runtime/backend.rs | 6 - .../drm/asahi/g17/runtime/render/settle.rs | 3 - drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 2 - drivers/gpu/drm/asahi/gem.rs | 16 +- drivers/gpu/drm/asahi/gem_stats.rs | 91 --------- 12 files changed, 9 insertions(+), 356 deletions(-) delete mode 100644 drivers/gpu/drm/asahi/g17/queue_stats.rs delete mode 100644 drivers/gpu/drm/asahi/gem_stats.rs diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index d835c8e283666..36fd8273c81ea 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -16,7 +16,6 @@ mod file; mod float; mod fw; mod gem; -mod gem_stats; mod g17; mod gpu; mod hw; diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index eb4ddcac7b808..b7d0a351f234d 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -39,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(crate::g17::queue_stats::enospc(crate::g17::queue_stats::Pool::QueueContextQos)); + return Err(ENOSPC); } *used |= 1u128 << id; Ok(QosLease { diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index aa2c054a9bbc7..f109c8261476e 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,7 +6,7 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use core::sync::atomic::{fence, Ordering}; use kernel::prelude::*; use super::{ @@ -17,28 +17,8 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -pub(super) const COMPUTE_RENDER_RESERVE: usize = 16; -pub(super) static LEASED: [AtomicU32; 3] = [const { AtomicU32::new(0) }; 3]; -pub(super) static PUBLISHED: [AtomicU32; 3] = [const { AtomicU32::new(0) }; 3]; -pub(super) static HIGH_WATER: AtomicU32 = AtomicU32::new(0); -pub(super) static EXHAUSTED: AtomicU64 = AtomicU64::new(0); -pub(super) static RESERVE_REFUSED: AtomicU64 = AtomicU64::new(0); - -pub(super) fn leased_total() -> u32 { - LEASED.iter().map(|count| count.load(Ordering::Relaxed)).sum() -} - -fn exhausted(engine: DataMaster, reserve: bool) -> Error { - if reserve { - RESERVE_REFUSED.fetch_add(1, Ordering::Relaxed); - } else { - EXHAUSTED.fetch_add(1, Ordering::Relaxed); - } - super::queue_stats::enospc(match engine { - DataMaster::Compute => super::queue_stats::Pool::QidCompute, - _ => super::queue_stats::Pool::QidRenderPair, - }) -} +/// QIDs kept free for render pairs when compute queues fill the table. +const COMPUTE_RENDER_RESERVE: usize = 16; /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -106,13 +86,13 @@ impl QueueIds { // Healthy existing channels keep their QID. Never recycle a published // identity to recover capacity after a fault; leave eight render pairs. if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { - return Err(exhausted(engine, true)); + return Err(ENOSPC); } let index = preferred .map(usize::from) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) .or_else(|| self.entries.iter().position(Option::is_none)) - .ok_or_else(|| exhausted(engine, false))?; + .ok_or(ENOSPC)?; let id = Id { qid: index as u8, owner, @@ -123,8 +103,6 @@ impl QueueIds { published: false, }); self.leased += 1; - LEASED[engine as usize].fetch_add(1, Ordering::Relaxed); - HIGH_WATER.fetch_max(leased_total(), Ordering::Relaxed); Ok(id) } @@ -145,10 +123,7 @@ impl QueueIds { if entry.id != id { return Err(EINVAL); } - if !entry.published { - entry.published = true; - PUBLISHED[id.engine as usize].fetch_add(1, Ordering::Relaxed); - } + entry.published = true; Ok(()) } @@ -158,7 +133,6 @@ impl QueueIds { Some(entry) if entry.id == id && !entry.published => { *slot = None; self.leased -= 1; - LEASED[id.engine as usize].fetch_sub(1, Ordering::Relaxed); Ok(()) } _ => Err(EINVAL), @@ -166,17 +140,6 @@ impl QueueIds { } } -impl Drop for QueueIds { - fn drop(&mut self) { - for entry in self.entries.iter().flatten() { - LEASED[entry.id.engine as usize].fetch_sub(1, Ordering::Relaxed); - if entry.published { - PUBLISHED[entry.id.engine as usize].fetch_sub(1, Ordering::Relaxed); - } - } - } -} - // Aliases drop before their backing. A compute queue is allocated low first; // a render queue is allocated high first. enum Backing { diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 6f75dfb5822c2..cb2af33c60f90 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -44,7 +44,6 @@ mod runtime; pub(crate) mod status; mod teardown; mod timeout; -mod queue_stats; mod validation; use core::any::Any; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 5ca9b85f8aeb6..fcd0a1d6b40e6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -364,7 +364,6 @@ impl Backing { /// Installed graph and its exact logical owner. A released idle graph may change owners only /// after a retirement witness. Failed work retains its pointers independently of fence state. pub(crate) struct Queue { - slot_accounting: crate::g17::queue_stats::ComputeAccount, owner: Option, binding: Binding, previous: Option, @@ -414,7 +413,6 @@ impl Queue { context.mark_published(); Ok(Self { owner: Some(owner), - slot_accounting: crate::g17::queue_stats::ComputeAccount::new(), binding, previous: None, kick, @@ -449,11 +447,6 @@ impl Queue { pub(crate) fn qid(&self) -> u8 { self.kick.id().qid() } - fn update_slot_accounting(&mut self) { - self.slot_accounting.update(self.owner.is_some(), self.released, - self.quarantined, self.retired_by_teardown); - } - pub(crate) fn owner(&self) -> Option { self.owner } @@ -601,7 +594,6 @@ impl Queue { self.failure_status_pending = true; self.retirement_proved = false; self.retired_by_teardown = false; - self.update_slot_accounting(); let all_spared = self.active.len != 0 && self.active.iter().all(|a| a.packet.completion.spared()); self.spared_quarantine = all_spared && error == ENODATA; @@ -707,7 +699,6 @@ impl Queue { self.retirement_ready = self.active.len == 0; if self.released && self.retirement_ready { self.owner = None; - self.update_slot_accounting(); } } Ok(None) | Err(EAGAIN) => { @@ -814,7 +805,6 @@ impl Queue { self.submitted = 0; self.kick.clear_parent_after_recovery(); self.quarantined = false; - self.update_slot_accounting(); self.retire_pending = false; self.retirement_ready = true; if self @@ -826,7 +816,6 @@ impl Queue { } if self.released { self.owner = None; - self.update_slot_accounting(); } } Ok(()) @@ -969,7 +958,6 @@ impl Queue { self.released = false; self.retirement_ready = false; self.retired_by_teardown = false; - self.update_slot_accounting(); Ok(()) } /// True allows the registry to cancel the never-published QID and drop this graph. An @@ -979,13 +967,11 @@ impl Queue { return Ok(true); } self.released = true; - self.update_slot_accounting(); if self.active.len == 0 && !self.quarantined { if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { if let Err(error) = self.graph.set_owner(&previous.context) { self.quarantined = true; self.retired_by_teardown = true; - self.update_slot_accounting(); return Err(error); } let mut previous = self.previous.take().ok_or(EIO)?; @@ -998,7 +984,6 @@ impl Queue { } if self.retirement_ready { self.owner = None; - self.update_slot_accounting(); } } Ok(false) @@ -1066,7 +1051,6 @@ impl Queue { } self.released = true; self.retired_by_teardown = true; - self.update_slot_accounting(); if let Some(active) = self .active .iter_mut() diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index e420427d594bf..d1048fd04607f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -197,7 +197,6 @@ impl Queue { self.spared_quarantine = false; self.retire_pending = false; self.retirement_ready = false; - self.update_slot_accounting(); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/queue_stats.rs b/drivers/gpu/drm/asahi/g17/queue_stats.rs deleted file mode 100644 index 5a5605e1d19a2..0000000000000 --- a/drivers/gpu/drm/asahi/g17/queue_stats.rs +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Firmware queue-slot accounting (`g17p_queue_slots`, root-only, read-only -//! sysfs). Relaxed counters; exhaustion logging is rate-limited. -//! -//! Every render pair and compute queue leases QIDs from one 128-entry table -//! (`kick::QueueIds`). A published QID is kept until processor stop, and a -//! quarantined or exit-killed compute graph is never handed out again, so a -//! boot can run out of QIDs while memory is fine. Queue creation then fails -//! with ENOSPC, and so does a new render pair (`render-warm-pairs`). The -//! `qids` fields are exact (kept by the table itself); compute -//! graph ownership gauges track structural changes. Pending teardown and -//! draining-pair counts are sampled in their existing service passes. -//! -//! Every ENOSPC from one of these pools is counted by pool and named in -//! dmesg (the first one, then each power of two), so a failed QUEUE_CREATE -//! or submit says which pool ran out. -use core::fmt::Write; -use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; -use kernel::{bindings, module_param::KernelParam, prelude::*}; - -use super::kick as g17_channel; -use g17_channel::QID_COUNT; - -static COMPUTE_GRAPHS: AtomicU32 = AtomicU32::new(0); -static COMPUTE_OWNED: AtomicU32 = AtomicU32::new(0); -static COMPUTE_RELEASED: AtomicU32 = AtomicU32::new(0); -static COMPUTE_QUARANTINED: AtomicU32 = AtomicU32::new(0); -static COMPUTE_EXIT_KILLED: AtomicU32 = AtomicU32::new(0); -static RETIRED_GRAPHS: AtomicU32 = AtomicU32::new(0); -static TEARDOWNS: AtomicU32 = AtomicU32::new(0); -static RENDER_QUARANTINED: AtomicU32 = AtomicU32::new(0); - -/// Pools whose exhaustion returns ENOSPC to a client. -#[derive(Clone, Copy)] -pub(crate) enum Pool { - /// A render slot's first use leases two QIDs (`render-warm-pairs`). - QidRenderPair, - /// A new compute graph leases one QID (queue creation / first compute). - QidCompute, - /// A new logical queue context (QUEUE_CREATE): its only 128-wide pool - /// is the QoS hardware-buffer IDs, held by logical queues and installed - /// graph bindings. - QueueContextQos, -} - -const POOL_COUNT: usize = 3; -const POOL_NAMES: [&str; POOL_COUNT] = ["qid-render-pair", "qid-compute", "queue-context-qos"]; -static POOL_ENOSPC: [AtomicU64; POOL_COUNT] = - [AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0)]; -/// Submits that failed with ENOSPC, whichever pool. -static SUBMIT_ENOSPC: AtomicU64 = AtomicU64::new(0); - -fn log_enospc(pool: &str, stage: &str, count: u64) { - // First one, then each power of two: a stuck pool cannot flood dmesg. - if count.is_power_of_two() { - pr_warn!( - "G17P ENOSPC pool={} stage={} (#{}): qids leased={}/{} compute={} compute-quarantined={} \ - reserve-refused={}\n", - pool, stage, count, g17_channel::leased_total(), QID_COUNT, - g17_channel::LEASED[2].load(Ordering::Relaxed), - COMPUTE_QUARANTINED.load(Ordering::Relaxed), - g17_channel::RESERVE_REFUSED.load(Ordering::Relaxed), - ); - } -} - -/// Count and name one ENOSPC from `pool`, and return it. -#[cold] -pub(crate) fn enospc(pool: Pool) -> Error { - let count = POOL_ENOSPC[pool as usize].fetch_add(1, Ordering::Relaxed) + 1; - log_enospc(POOL_NAMES[pool as usize], "-", count); - ENOSPC -} - -/// A submit failed with ENOSPC at `stage`. Pools tagged at their source -/// were already counted and named; this names the rest by stage. -#[cold] -pub(crate) fn note_submit_enospc(stage: &str) { - let count = SUBMIT_ENOSPC.fetch_add(1, Ordering::Relaxed) + 1; - // Source-specific QID/QoS failures already counted their actual pool. - // This boundary knows the engine stage, not which nested allocator failed. - log_enospc("submit", stage, count); -} - -/// Incremental graph ownership accounting. Normal submissions do not touch -/// these counters; only graph creation, ownership transitions and drop do. -pub(crate) struct ComputeAccount { flags: u8 } -impl ComputeAccount { - pub(crate) fn new() -> Self { - COMPUTE_GRAPHS.fetch_add(1, Ordering::Relaxed); - COMPUTE_OWNED.fetch_add(1, Ordering::Relaxed); - Self { flags: 1 } - } - pub(crate) fn update(&mut self, owned: bool, released: bool, quarantined: bool, exit_killed: bool) { - let next = u8::from(owned) | (u8::from(released) << 1) - | (u8::from(quarantined) << 2) | (u8::from(exit_killed) << 3); - let changed = next ^ self.flags; - for (bit, counter) in [&COMPUTE_OWNED, &COMPUTE_RELEASED, - &COMPUTE_QUARANTINED, &COMPUTE_EXIT_KILLED].into_iter().enumerate() - { - if changed & (1 << bit) != 0 { - if next & (1 << bit) != 0 { counter.fetch_add(1, Ordering::Relaxed); } - else { counter.fetch_sub(1, Ordering::Relaxed); } - } - } - self.flags = next; - } -} -impl Drop for ComputeAccount { - fn drop(&mut self) { - self.update(false, false, false, false); - COMPUTE_GRAPHS.fetch_sub(1, Ordering::Relaxed); - } -} - -pub(crate) fn note_teardowns(count: usize) { - TEARDOWNS.store(count as u32, Ordering::Relaxed); -} -pub(crate) fn note_draining_pairs(count: usize) { - RENDER_QUARANTINED.store(count as u32, Ordering::Relaxed); -} - -struct Output<'a> { bytes: &'a mut [u8], used: usize } -impl Write for Output<'_> { - fn write_str(&mut self, value: &str) -> core::fmt::Result { - if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } - self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); - self.used += value.len(); - Ok(()) - } -} - -unsafe extern "C" fn slots_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { - if buf.is_null() { return -22; } - // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. - let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; - let mut out = Output { bytes, used: 0 }; - let r = |c: &AtomicU32| c.load(Ordering::Relaxed); - let e = |pool: usize| POOL_ENOSPC[pool].load(Ordering::Relaxed); - let leased = g17_channel::leased_total(); - let published: u32 = g17_channel::PUBLISHED.iter().map(r).sum(); - let _ = writeln!(out, - "qids leased={} free={} published={} tiling={} fragment={} compute={} high_water={} exhausted={} \ - reserve={} reserve_refused={} \ - compute graphs={} owned={} released={} quarantined={} exit_killed={} retired_graphs={} \ - teardowns={} render_draining_pairs={} \ - enospc qid-render-pair={} qid-compute={} queue-context-qos={} submits={}", - leased, (QID_COUNT as u32).saturating_sub(leased), published, - r(&g17_channel::LEASED[0]), r(&g17_channel::LEASED[1]), r(&g17_channel::LEASED[2]), - r(&g17_channel::HIGH_WATER), g17_channel::EXHAUSTED.load(Ordering::Relaxed), - g17_channel::COMPUTE_RENDER_RESERVE, g17_channel::RESERVE_REFUSED.load(Ordering::Relaxed), - r(&COMPUTE_GRAPHS), r(&COMPUTE_OWNED), r(&COMPUTE_RELEASED), r(&COMPUTE_QUARANTINED), - r(&COMPUTE_EXIT_KILLED), r(&RETIRED_GRAPHS), r(&TEARDOWNS), r(&RENDER_QUARANTINED), - e(0), e(1), e(2), SUBMIT_ENOSPC.load(Ordering::Relaxed)); - out.used as core::ffi::c_int -} - -unsafe extern "C" fn slots_set(_: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { - -1 // EPERM: read-only counters -} - -static SLOTS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { - flags: 0, set: Some(slots_set), get: Some(slots_get), free: None, -}; -#[link_section = "__param"] -#[used] -static SLOTS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { - name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { - c"g17p_queue_slots" - } else { c"asahi.g17p_queue_slots" }), - #[cfg(MODULE)] - mod_: core::ptr::addr_of_mut!(bindings::__this_module), - #[cfg(not(MODULE))] - mod_: core::ptr::null_mut(), - ops: core::ptr::from_ref(&SLOTS_OPS), perm: 0o400, level: -1, flags: 0, - __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, -}); diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 7f204b9328ffb..ef72699e8e7ce 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -138,12 +138,6 @@ impl job::Backend for Backend { Engine::Render => self.render_packet(&packet, &mut deferred), } }; - if result.as_ref().is_err_and(|error| *error == ENOSPC) { - crate::g17::queue_stats::note_submit_enospc(match packet.engine() { - Engine::Compute => "compute-publish", - Engine::Render => "render-publish", - }); - } self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 0a4634bccaea5..415d1f391f601 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -164,7 +164,6 @@ impl crate::g17::Firmware { return Ok(()); } let mut serviced = false; - let mut draining = 0; for slot in 1..self.queues.render.entries.len() { if !self.queues.render.entries[slot] .as_ref() @@ -173,7 +172,6 @@ impl crate::g17::Firmware { { continue; } - draining += 1; if let Err(error) = self.drain_render_slot(slot, &mut serviced, deferred) { dev_warn!( self.primary.state.shared.dev.as_ref(), @@ -183,7 +181,6 @@ impl crate::g17::Firmware { ); } } - crate::g17::queue_stats::note_draining_pairs(draining); Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index f310f8154aae7..6eb8da6377533 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -169,7 +169,6 @@ impl Firmware { /// The event worker has already reclaimed drained pairs and reannounced /// pending controls. Recovery blocks new releases, never an observed ack. pub(in crate::g17) fn service_teardown(&mut self, reclaim: &mut ReclaimBatch) -> Result { - crate::g17::queue_stats::note_teardowns(self.queues.teardown.len()); self.queues.render.collect_reclaimable(reclaim)?; if self.queues.teardown.is_empty() { return Ok(()); @@ -199,7 +198,6 @@ impl Firmware { .is_some() {} teardown.report_stalled(super::now_ns(), &mut service); - crate::g17::queue_stats::note_teardowns(teardown.len()); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index cb3fe6e038eae..6c05da6ff096b 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -41,7 +41,7 @@ use crate::{ const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; /// Represents the inner data of a GEM object for this driver. -#[pin_data(PinnedDrop)] +#[pin_data] pub(crate) struct AsahiObject { /// ID for debug id: u64, @@ -157,13 +157,6 @@ impl ObjectRef { } } -#[pinned_drop] -impl PinnedDrop for AsahiObject { - fn drop(self: Pin<&mut Self>) { - crate::gem_stats::note_free(self.kernel, self.size); - } -} - pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, @@ -248,12 +241,7 @@ impl DriverObject for AsahiObject { mod_pr_debug!("AsahiObject::new id={}\n", id); try_pin_init!(AsahiObject { id, - // All remaining fields are infallible. PinnedDrop balances the - // accounting once this initializer completes. - size: { - crate::gem_stats::note_new(args.kernel, size); - size - }, + size, flags: args.flags, exportable: args.exportable, kernel: args.kernel, diff --git a/drivers/gpu/drm/asahi/gem_stats.rs b/drivers/gpu/drm/asahi/gem_stats.rs deleted file mode 100644 index abd01922273bd..0000000000000 --- a/drivers/gpu/drm/asahi/gem_stats.rs +++ /dev/null @@ -1,91 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Live GEM object accounting (`gem_stats`, root-only, read-only sysfs). -//! -//! Every shmem object this driver creates is counted from creation until its -//! last reference drops, split by user and kernel ownership. Once a GEM object -//! is mapped into a GPU VM its pages stay pinned (and unevictable) until the -//! object is freed, so `user_bytes + kernel_bytes` is an upper bound on the -//! GPU share of /proc/meminfo `Unevictable`, and a value that keeps growing -//! after clients exit identifies a reference leak in the driver. The -//! `*_created` / `*_freed` totals show allocation churn. -use core::fmt::Write; -use core::sync::atomic::{AtomicU64, Ordering}; -use kernel::{bindings, module_param::KernelParam}; - -static USER_OBJECTS: AtomicU64 = AtomicU64::new(0); -static USER_BYTES: AtomicU64 = AtomicU64::new(0); -static USER_CREATED: AtomicU64 = AtomicU64::new(0); -static KERNEL_OBJECTS: AtomicU64 = AtomicU64::new(0); -static KERNEL_BYTES: AtomicU64 = AtomicU64::new(0); -static KERNEL_CREATED: AtomicU64 = AtomicU64::new(0); - -fn counters(kernel: bool) -> (&'static AtomicU64, &'static AtomicU64, &'static AtomicU64) { - if kernel { - (&KERNEL_OBJECTS, &KERNEL_BYTES, &KERNEL_CREATED) - } else { - (&USER_OBJECTS, &USER_BYTES, &USER_CREATED) - } -} - -/// Count one new GEM object. -pub(crate) fn note_new(kernel: bool, size: usize) { - let (objects, bytes, created) = counters(kernel); - objects.fetch_add(1, Ordering::Relaxed); - bytes.fetch_add(size as u64, Ordering::Relaxed); - created.fetch_add(1, Ordering::Relaxed); -} - -/// Count one freed GEM object. -pub(crate) fn note_free(kernel: bool, size: usize) { - let (objects, bytes, _) = counters(kernel); - objects.fetch_sub(1, Ordering::Relaxed); - bytes.fetch_sub(size as u64, Ordering::Relaxed); -} - -struct Output<'a> { bytes: &'a mut [u8], used: usize } -impl Write for Output<'_> { - fn write_str(&mut self, value: &str) -> core::fmt::Result { - if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } - self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); - self.used += value.len(); - Ok(()) - } -} - -unsafe extern "C" fn stats_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { - if buf.is_null() { return -22; } - // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. - let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; - let mut out = Output { bytes, used: 0 }; - let r = |c: &AtomicU64| c.load(Ordering::Relaxed); - let uc = r(&USER_CREATED); - let kc = r(&KERNEL_CREATED); - let (uo, ko) = (r(&USER_OBJECTS), r(&KERNEL_OBJECTS)); - let _ = writeln!(out, - "user_objects={} user_bytes={} user_created={} user_freed={} \ - kernel_objects={} kernel_bytes={} kernel_created={} kernel_freed={}", - uo, r(&USER_BYTES), uc, uc.wrapping_sub(uo), - ko, r(&KERNEL_BYTES), kc, kc.wrapping_sub(ko)); - out.used as core::ffi::c_int -} - -unsafe extern "C" fn stats_set(_: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { - -1 // EPERM: read-only counters -} - -static STATS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { - flags: 0, set: Some(stats_set), get: Some(stats_get), free: None, -}; -#[link_section = "__param"] -#[used] -static STATS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { - name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { - c"gem_stats" - } else { c"asahi.gem_stats" }), - #[cfg(MODULE)] - mod_: core::ptr::addr_of_mut!(bindings::__this_module), - #[cfg(not(MODULE))] - mod_: core::ptr::null_mut(), - ops: core::ptr::from_ref(&STATS_OPS), perm: 0o400, level: -1, flags: 0, - __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, -}); From ed235501b6531eb218d3b058bab11a807ae17713 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:57:43 -0400 Subject: [PATCH 0648/1579] drm/asahi: drop the stalled scheduler-release warning A release whose firmware acknowledgement took longer than two seconds logged "release is still pending" once and otherwise changed nothing: the entry stayed retained until the acknowledgement arrived. Hardware runs never produced the line. Keep the acknowledgement state machine and its polling window; remove the warning bookkeeping. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 152092e32a7fd7a7b3a96b8e45be7db78934b7d8) --- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 4 ---- drivers/gpu/drm/asahi/g17/teardown.rs | 21 ++----------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 6eb8da6377533..8a637e0b3ed01 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -158,9 +158,6 @@ impl Host for Service<'_> { context.id(), error ), - Failure::Stalled => { - dev_warn!(dev, "Scheduler {} release is still pending\n", context.id()) - } } } } @@ -197,7 +194,6 @@ impl Firmware { .take_consumed(consumer, producer, &mut service)? .is_some() {} - teardown.report_stalled(super::now_ns(), &mut service); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index f53c2d45c129a..791ac690c0f50 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -22,17 +22,15 @@ struct Entry { context: Arc, phase: Phase, armed: u64, - warned: bool, announce_pending: bool, } -/// Publication and announcement errors retain the entry. A stalled release is reported once; -/// its timeout never authorizes freeing memory still owned by firmware. +/// Publication and announcement errors retain the entry; no timeout frees +/// memory the firmware may still own. #[derive(Clone, Copy)] pub(crate) enum Failure { Publication(Error), Announcement(Error), - Stalled, } /// Borrowed device operations. Readiness covers every matching installed and displaced graph, @@ -88,7 +86,6 @@ impl Pending { context: context.clone(), phase, armed: now, - warned: false, announce_pending: false, }) .map_err(|_| ENOSPC) @@ -143,7 +140,6 @@ impl Pending { // One successful retry starts its consumption window. // Failed retries never slide the polling deadline. entry.armed = now; - entry.warned = false; } continue; } @@ -177,7 +173,6 @@ impl Pending { // A long command drain must not consume the acknowledgement // polling window before this release exists on the control ring. entry.armed = now; - entry.warned = false; entry.announce_pending = true; fence(Ordering::SeqCst); match host.notify() { @@ -228,16 +223,4 @@ impl Pending { entry.phase != Phase::Deferred && now.saturating_sub(entry.armed) < POLL_NS }) } - - pub(crate) fn report_stalled(&mut self, now: u64, host: &mut impl Host) { - for entry in self.entries.iter_mut() { - if entry.phase != Phase::Deferred - && !entry.warned - && now.saturating_sub(entry.armed) >= POLL_NS - { - entry.warned = true; - host.report(&entry.context, Failure::Stalled); - } - } - } } From 9d8357e275907651adbb7022b812f789ece7d040 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 11:15:54 -0400 Subject: [PATCH 0649/1579] drm/asahi: build constant firmware objects from word tables PowerState, SecondaryConfig, PowerConfigHead, PowerConfigTail and HwDataAux are written once from constants and never read by the host. Their 600-odd unnamed fields, offset assertions and field-by-field builders are replaced by byte arrays filled from (offset, value, repeat) word tables, plus the one address HwDataAux carries. The five objects are byte-identical to the previous builders for every input (verified with a host-side dump of each object). Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 24ce1bcdd8ebf0f8664c043a639c0201286f94e5) --- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 1375 ++++++---------------- 1 file changed, 341 insertions(+), 1034 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index c25b843a913e7..8dbb6a81bcd47 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -1406,187 +1406,7 @@ impl StatusBlock { } } -/// Power state the primary status B carries for the secondary instance, -/// named by [`SecondaryRoot::power_state_va`]. -#[repr(C, packed)] -pub(crate) struct PowerState { - pub(crate) unk_0: [u8; 0x10], - /// Required: 0xf (15). - pub(crate) unk_10: u32, - /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). - pub(crate) unk_14: u32, - /// Required: 0x4088_0000 (4.25 as an IEEE-754 single). - pub(crate) unk_18: u32, - /// Required: 0x28 (40). - pub(crate) unk_1c: u32, - /// Required: 1. - pub(crate) unk_20: u32, - pub(crate) unk_24: [u8; 0xa0], - /// Non-zero asks the secondary to reload the limiter settings that - /// follow into the power configuration; zero at boot. - pub(crate) temp_update: u32, - /// Target temperature of the limiter ([`TEMP_TARGET`]). - pub(crate) temp_target: u32, - /// Release temperature of the limiter ([`TEMP_RELEASE`]). - pub(crate) temp_release: u32, - pub(crate) unk_d0: [u8; 8], - /// Proportional gain of the limiter ([`TEMP_KP`]). - pub(crate) temp_kp: u32, - /// Integral gain of the limiter ([`TEMP_KI`]). - pub(crate) temp_ki: u32, - pub(crate) unk_e0: [u8; 0x10], - /// Required: 0x26ac (9900). - pub(crate) unk_f0: u32, - /// Required: 0xc8 (200). - pub(crate) unk_f4: u32, - pub(crate) unk_f8: [u8; 0x10], - /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). - pub(crate) unk_108: u32, - /// Required: 0x43c8_0000 (400.0 as an IEEE-754 single). - pub(crate) unk_10c: u32, - /// Required: 0xc8 (200). - pub(crate) unk_110: u32, - pub(crate) unk_114: [u8; 0x14], - /// Required: 1. - pub(crate) unk_128: u32, - /// Required: 0x2616 (9750). - pub(crate) unk_12c: u32, - /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). - pub(crate) unk_130: u32, - /// Required: 0x40cc_cccd (6.4 as an IEEE-754 single). - pub(crate) unk_134: u32, - pub(crate) unk_138: [u8; 0x14], - /// Required: 1. - pub(crate) unk_14c: u32, - pub(crate) unk_150: [u8; 0x70], - /// Required: 1. - pub(crate) unk_1c0: u32, - /// Required: 1. - pub(crate) unk_1c4: u32, - /// Required: 4. - pub(crate) unk_1c8: u32, - /// Required: 1 in every entry. - pub(crate) unk_1cc: [u32; 4], - pub(crate) unk_1dc: [u8; 0x308], - /// Required: 1. - pub(crate) unk_4e4: u32, - /// Required: 0x1f4 (500). - pub(crate) unk_4e8: u32, - /// Required: 6. - pub(crate) unk_4ec: u32, - /// Required: 0x30d4 (12500) in every entry. - pub(crate) unk_4f0: [u32; 6], - pub(crate) unk_508: [u8; 0x14], - /// Required: 0x30d4 (12500) in every entry. - pub(crate) unk_51c: [u32; 5], - /// Required: 1. - pub(crate) unk_530: u32, - pub(crate) unk_534: [u8; 0x58], - /// Required: 6. - pub(crate) unk_58c: u32, - /// Required: 0xdac (3500). - pub(crate) unk_590: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_594: u32, - /// Required: 0xbb8 (3000). - pub(crate) unk_598: u32, - /// Required: 0x64 (100). - pub(crate) unk_59c: u32, - pub(crate) unk_5a0: [u8; 0x2c], - /// Required: 1. - pub(crate) unk_5cc: u32, - /// Required: 4. - pub(crate) unk_5d0: u32, - pub(crate) unk_5d4: [u8; 8], - /// Required: 0x1f40 (8000). - pub(crate) unk_5dc: u32, - /// Required: 0xc8 (200). - pub(crate) unk_5e0: u32, - /// Required: 0xfa0 (4000). - pub(crate) unk_5e4: u32, - /// Required: 0xc8 (200). - pub(crate) unk_5e8: u32, - /// Required: 0x7d0 (2000). - pub(crate) unk_5ec: u32, - /// Required: 0xc8 (200). - pub(crate) unk_5f0: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_5f4: u32, - /// Required: 0xc8 (200). - pub(crate) unk_5f8: u32, - pub(crate) unk_5fc: [u8; 8], - /// Required: 0x4170_0000 (15.0 as an IEEE-754 single). - pub(crate) unk_604: u32, - /// Required: 0x40a0_0000 (5.0 as an IEEE-754 single). - pub(crate) unk_608: u32, - /// Required: 0x20 (32). - pub(crate) unk_60c: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_610: u32, - pub(crate) unk_614: [u8; 0xc], - /// Required: 0x384 (900). - pub(crate) unk_620: u32, - pub(crate) unk_624: [u8; 0x9c], -} - -// SAFETY: `PowerState` consists of integers and arrays of integers only. -unsafe impl Zeroable for PowerState {} -static_assert!(size_of::() == 0x6c0); - -impl PowerState { - fn init(&mut self) { - self.unk_10 = 0xf; - self.unk_14 = 0x3f00_0000; - self.unk_18 = 0x4088_0000; - self.unk_1c = 0x28; - self.unk_20 = 1; - self.temp_target = TEMP_TARGET; - self.temp_release = TEMP_RELEASE; - self.temp_kp = TEMP_KP; - self.temp_ki = TEMP_KI; - self.unk_f0 = 0x26ac; - self.unk_f4 = 0xc8; - self.unk_108 = 0x42c8_0000; - self.unk_10c = 0x43c8_0000; - self.unk_110 = 0xc8; - self.unk_128 = 1; - self.unk_12c = 0x2616; - self.unk_130 = 0x3f00_0000; - self.unk_134 = 0x40cc_cccd; - self.unk_14c = 1; - self.unk_1c0 = 1; - self.unk_1c4 = 1; - self.unk_1c8 = 4; - self.unk_1cc = [1; 4]; - self.unk_4e4 = 1; - self.unk_4e8 = 0x1f4; - self.unk_4ec = 6; - self.unk_4f0 = [0x30d4; 6]; - self.unk_51c = [0x30d4; 5]; - self.unk_530 = 1; - self.unk_58c = 6; - self.unk_590 = 0xdac; - self.unk_594 = 0x3e8; - self.unk_598 = 0xbb8; - self.unk_59c = 0x64; - self.unk_5cc = 1; - self.unk_5d0 = 4; - self.unk_5dc = 0x1f40; - self.unk_5e0 = 0xc8; - self.unk_5e4 = 0xfa0; - self.unk_5e8 = 0xc8; - self.unk_5ec = 0x7d0; - self.unk_5f0 = 0xc8; - self.unk_5f4 = 0x3e8; - self.unk_5f8 = 0xc8; - self.unk_604 = 0x4170_0000; - self.unk_608 = 0x40a0_0000; - self.unk_60c = 0x20; - self.unk_610 = 0x3e8; - self.unk_620 = 0x384; - } -} /// Size of the firmware-control page named by the primary status B. pub(crate) const FWCTL_SIZE: usize = 0x4000; @@ -1667,46 +1487,7 @@ impl PrimaryStatusB { } } -/// Configuration of the secondary instance, named by -/// [`SecondaryRoot::config_va`]. -#[repr(C)] -pub(crate) struct SecondaryConfig { - pub(crate) unk_0: [u8; 0x14], - /// Required: 1. - pub(crate) unk_14: u32, - pub(crate) unk_18: [u8; 0x14], - /// Required: 1. - pub(crate) unk_2c: u32, - /// Required: 1. - pub(crate) unk_30: u32, - pub(crate) unk_34: [u8; 0xc], - /// Required: 1. - pub(crate) unk_40: u32, - pub(crate) unk_44: [u8; 8], - /// Required: 1. - pub(crate) unk_4c: u32, - /// Required: 0x186a (6250). - pub(crate) unk_50: u32, - pub(crate) unk_54: [u8; 0x2c], -} - -static_assert!(size_of::() == 0x80); - -// SAFETY: `SecondaryConfig` consists of integers and arrays of integers only. -unsafe impl Zeroable for SecondaryConfig {} -impl SecondaryConfig { - /// Initializes the secondary configuration. - pub(crate) fn init(&mut self) { - clear(self); - self.unk_14 = 1; - self.unk_2c = 1; - self.unk_30 = 1; - self.unk_40 = 1; - self.unk_4c = 1; - self.unk_50 = 0x186a; - } -} /// Offsets in the private cluster. pub(crate) mod private { @@ -1751,295 +1532,332 @@ const TEMP_KP: u32 = 0x41a2_b852; /// single. const TEMP_KI: u32 = 0x4116_0419; +/// Nonzero 32-bit words of a firmware object the host builds from constants: (byte offset, value, repeat). +/// Every other word is zero. Generated from the previously field-by-field builders; the fwdump harness +/// (`neo-smooth-20261007/kernel/fwdump`) verifies the objects byte for byte. +type Words = &'static [(usize, u32, usize)]; + +fn fill_words(bytes: &mut [u8], words: Words) { + bytes.fill(0); + for &(offset, value, repeat) in words { + for index in 0..repeat { + let at = offset + 4 * index; + bytes[at..at + 4].copy_from_slice(&value.to_le_bytes()); + } + } +} + +const POWER_CONFIG_HEAD_WORDS: Words = &[ + (0x4, 0x5dc00, 2), + (0x10, 4, 1), + (0x14, 0x3f800000, 1), + (0x2c, 1, 2), + (0x40, 0x64, 1), + (0x44, 1, 1), + (0x48, 0x3e8, 1), + (0x50, 0x64, 1), + (0x60, 4, 1), + (0x64, 0x3f800000, 1), + (0x6c, 1, 2), + (0x80, 0x64, 1), + (0x84, 1, 1), + (0x88, 0x3e8, 1), + (0x90, 0x64, 1), + (0xc0, 0x3f828f5c, 11), + (0x9b8, 0x271, 1), + (0x9c4, 0x3f7f2e9f, 1), + (0x9cc, 0x3b516154, 1), + (0x9d4, 0x3ca59586, 1), + (0x9e0, 0x46153800, 1), + (0x9e4, 0x40a90fdb, 1), + (0x9ec, 0xbdd67344, 1), + (0x9f0, 0x28, 1), + (0x9f4, 0x3e8, 2), + (0xa04, 0x254e, 1), + (0xa38, 0x3e8, 1), + (0xa70, 4, 1), + (0xa84, 0x3f800000, 1), + (0xa8c, 0x40cccccd, 1), + (0xa98, 0x47800000, 1), + (0xa9c, 0x3f000000, 1), + (0xaa8, 0x28, 1), + (0xaac, 0x3e8, 1), + (0xabc, 0x254e, 1), + (0xac8, 0x10, 1), + (0xad0, 0x5dc00, 1), + (0xb28, 0x5c, 1), + (0xb30, 0x64, 1), + (0xb34, 0x22, 1), + (0xb38, 6, 1), + (0xb40, 6, 1), + (0xb44, 1, 1), + (0xb4c, 0x3f4ccccd, 1), + (0xb50, 0x3f7df3b6, 1), + (0xb54, 0x3e4ccccd, 1), + (0xb58, 0x3c03126f, 1), + (0xb5c, 0x3f69d4d8, 2), + (0xb68, 0x42be0000, 1), + (0xb6c, 0x4064f5c3, 2), + (0xb74, 0x41179436, 1), + (0xb78, 0x64, 1), + (0xb7c, 0x3e8, 1), + (0xb80, 0x64, 1), + (0xb8c, 0x5c, 1), + (0xbc0, 0x64, 1), + (0xbe0, 0x401, 1), + (0xbe4, 0x6400, 1), + (0xbec, 0x1f4000, 1), + (0xbf0, 0xc800, 1), + (0xbf4, 0xfa000, 1), + (0xbf8, 0xc800, 1), + (0xbfc, 0x7d000, 1), + (0xc00, 0xc800, 1), + (0xc04, 0x3e800, 1), + (0xc08, 0xc800, 1), + (0xc0c, 0x100, 1), + (0xc64, 1, 1), + (0xc90, 1, 1), + (0x1c94, 0xff000000, 1), + (0x1c98, 0xff7fff7f, 74), +]; + +const POWER_CONFIG_TAIL_WORDS: Words = &[ + (0x0, 0xff7fff7f, 181), + (0x2d4, 0x7fff7f, 1), + (0x4d8, 0x47800000, 1), + (0x4dc, 0x42200000, 1), + (0x4e0, 0x447a0000, 1), + (0x4e4, 0x5be, 1), + (0x4f8, 0x28, 1), + (0x500, 0x42c80000, 1), + (0x504, 0x3e8, 1), + (0x50c, 0x3f4ccccd, 1), + (0x510, 0x3e4ccccd, 1), + (0x550, TEMP_SENSOR_MASK as u32, 1), + (0x558, TEMP_RELEASE, 1), + (0x560, 4, 1), + (0x574, 0x3f800000, 1), + (0x57c, TEMP_KI, 1), + (0x588, 0x47800000, 1), + (0x58c, TEMP_KP, 1), + (0x598, 0x28, 1), + (0x59c, 0x3e8, 2), + (0x5ac, TEMP_TARGET, 1), + (0x5e0, 0x3e8, 1), + (0x7fc, 0x42000006, 1), + (0xa10, 0xb8520000, 1), + (0xa14, 0x3f7e, 1), + (0xa18, 0xd70a0000, 1), + (0xa1c, 0x3ba3, 1), + (0xa24, 0x43c8, 1), + (0xa30, 0x4780, 1), + (0xa34, 0x42c8, 1), + (0xa3c, 0x384bac8, 1), + (0xa40, 0x3e80000, 2), + (0xa4c, 0x80000000, 1), + (0xa50, 0x26ac453b, 1), + (0xa84, 0x3e80000, 1), + (0xaa0, 0xc80000, 1), + (0xb58, 0x3e8, 1), + (0xb6c, 1, 1), + (0xba0, 0x101, 1), + (0xba4, 0x400, 1), + (0xbbc, 0x23f00, 1), + (0xbc0, 0x100, 2), + (0xbdc, 0x24000, 1), + (0xbe0, 0x100, 2), + (0xbfc, 0x24100, 1), + (0xc48, 1, 1), + (0xc50, 0x254e, 3), + (0xc68, 0x3f800000, 1), + (0xc6c, 4, 1), + (0xc70, 0x10, 1), + (0xc74, 0x5dc00, 1), + (0xd44, 0x3c, 1), + (0xd50, 0x3f6eeeef, 1), + (0xd58, 0x3d888889, 1), + (0xd60, 0x3f000000, 1), + (0xd6c, 0x47800000, 1), + (0xd70, 0x40880000, 1), + (0xd7c, 0x28, 1), + (0xd80, 0x3e8, 2), + (0xd8c, 0x46153800, 1), + (0xd90, 0x254e, 1), + (0xd9c, 0xf0, 1), + (0xda4, 0x57e400, 1), + (0xdc4, 0x3e8, 1), + (0xe04, 1, 1), + (0xed4, 0x32, 1), + (0xed8, 1, 1), + (0xee0, 0x3f638e39, 1), + (0xee4, 0x3f2aaaab, 1), + (0xee8, 0x3de38e39, 1), + (0xeec, 0x3eaaaaab, 1), + (0xef0, 0xbf4ccccd, 2), + (0xefc, 0x47800000, 1), + (0xf00, 0xc0a00000, 2), + (0xf0c, 0x64, 1), + (0xf10, 0x3e8, 1), + (0xf14, 0x64, 1), + (0xf1c, 0x467a0000, 1), + (0xf20, 0x578, 1), + (0xf2c, 0x90, 1), + (0xf30, 0x30, 1), + (0xf34, 0x34bc00, 1), + (0xf3c, 0x119400, 1), + (0xf48, 0x47800000, 1), + (0xf70, 0x3e80, 1), + (0xf74, 2, 1), + (0xf78, 0x9c4, 1), + (0xf7c, 0x20d, 1), + (0xf80, 2, 1), + (0xf84, 4, 1), + (0xf88, 0x32, 1), + (0xfb8, 0x47800000, 1), + (0xfc8, 0x28, 1), + (0xfcc, 0x3e8, 1), +]; + +const POWER_STATE_WORDS: Words = &[ + (0x10, 0xf, 1), + (0x14, 0x3f000000, 1), + (0x18, 0x40880000, 1), + (0x1c, 0x28, 1), + (0x20, 1, 1), + (0xc8, TEMP_TARGET, 1), + (0xcc, TEMP_RELEASE, 1), + (0xd8, TEMP_KP, 1), + (0xdc, TEMP_KI, 1), + (0xf0, 0x26ac, 1), + (0xf4, 0xc8, 1), + (0x108, 0x42c80000, 1), + (0x10c, 0x43c80000, 1), + (0x110, 0xc8, 1), + (0x128, 1, 1), + (0x12c, 0x2616, 1), + (0x130, 0x3f000000, 1), + (0x134, 0x40cccccd, 1), + (0x14c, 1, 1), + (0x1c0, 1, 2), + (0x1c8, 4, 1), + (0x1cc, 1, 4), + (0x4e4, 1, 1), + (0x4e8, 0x1f4, 1), + (0x4ec, 6, 1), + (0x4f0, 0x30d4, 6), + (0x51c, 0x30d4, 5), + (0x530, 1, 1), + (0x58c, 6, 1), + (0x590, 0xdac, 1), + (0x594, 0x3e8, 1), + (0x598, 0xbb8, 1), + (0x59c, 0x64, 1), + (0x5cc, 1, 1), + (0x5d0, 4, 1), + (0x5dc, 0x1f40, 1), + (0x5e0, 0xc8, 1), + (0x5e4, 0xfa0, 1), + (0x5e8, 0xc8, 1), + (0x5ec, 0x7d0, 1), + (0x5f0, 0xc8, 1), + (0x5f4, 0x3e8, 1), + (0x5f8, 0xc8, 1), + (0x604, 0x41700000, 1), + (0x608, 0x40a00000, 1), + (0x60c, 0x20, 1), + (0x610, 0x3e8, 1), + (0x620, 0x384, 1), +]; + +const HW_DATA_AUX_WORDS: Words = &[ + (0xac, 6, 1), + (0xb8, 1, 1), + (0x174, 0xac000000, 1), + (0x178, 0xe800000d, 1), + (0x17c, 0xb8000003, 1), + (0x180, 0xe800000b, 1), + (0x184, 0xe8000003, 1), + (0x188, 0x64000003, 1), + (0x18c, 0x20000000, 1), + (0x1b4, 4, 1), + (0x1c0, 0x3f780000, 1), + (0x1c8, 0x3d000000, 1), + (0x1d0, 0x40a00000, 1), + (0x1dc, 0x47800000, 1), + (0x1e0, 0x41700000, 1), + (0x1ec, 0x384, 1), + (0x1f0, 0x3e8, 2), + (0x200, 0x3e8, 1), + (0x234, 0x3e8, 1), + (0x274, 0x64, 1), + (0x290, 0x82a, 1), + (0x298, 0x7d, 1), + (0x29c, 1, 1), + (0xee0, 0x18ae, 1), + (0xee8, 1, 1), + (0x2eec, 0x41c8cccd, 2), + (0x35e4, 1, 1), + (0x35e8, 0x1f4, 1), + (0x35ec, 6, 1), + (0x35f0, 0x30d4, 6), + (0x3608, 0xfa0, 1), + (0x361c, 0x30d4, 5), + (0x3630, 1, 1), + (0x37a8, 1, 1), + (0x37b4, 0x424ccccd, 1), + (0x37f4, 0x44068000, 1), + (0x3834, 0x41c9999a, 1), + (0x38e0, 0x186a, 1), +]; + +const SECONDARY_CONFIG_WORDS: Words = &[ + (0x14, 1, 1), + (0x2c, 1, 2), + (0x40, 1, 1), + (0x4c, 1, 1), + (0x50, 0x186a, 1), +]; + +/// Power state the primary status B carries for the secondary instance, +/// named by [`SecondaryRoot::power_state_va`]. +#[repr(C)] +pub(crate) struct PowerState(pub(crate) [u8; 0x6c0]); + +// SAFETY: `PowerState` is a byte array. +unsafe impl Zeroable for PowerState {} + +impl PowerState { + fn init(&mut self) { + fill_words(&mut self.0, POWER_STATE_WORDS); + } +} + +/// Configuration of the secondary instance, named by +/// [`SecondaryRoot::config_va`]. +#[repr(C)] +pub(crate) struct SecondaryConfig(pub(crate) [u8; 0x80]); + +// SAFETY: `SecondaryConfig` is a byte array. +unsafe impl Zeroable for SecondaryConfig {} + +impl SecondaryConfig { + pub(crate) fn init(&mut self) { + fill_words(&mut self.0, SECONDARY_CONFIG_WORDS); + } +} + /// Leading part of the [`PowerConfig`]. The host writes it twice: in the /// power configuration and at [`bundle::POWER_CONFIG_COPY`], where the copy /// ends with the views it lies in. -#[repr(C, packed)] -pub(crate) struct PowerConfigHead { - pub(crate) unk_0: [u8; 4], - /// Required: 0x5_dc00 (384000). - pub(crate) unk_4: u32, - /// Required: 0x5_dc00 (384000). - pub(crate) unk_8: u32, - pub(crate) unk_c: [u8; 4], - /// Required: 4. - pub(crate) unk_10: u32, - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_14: u32, - pub(crate) unk_18: [u8; 0x14], - /// Required: 1. - pub(crate) unk_2c: u32, - /// Required: 1. - pub(crate) unk_30: u32, - pub(crate) unk_34: [u8; 0xc], - /// Required: 0x64 (100). - pub(crate) unk_40: u32, - /// Required: 1. - pub(crate) unk_44: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_48: u32, - pub(crate) unk_4c: [u8; 4], - /// Required: 0x64 (100). - pub(crate) unk_50: u32, - pub(crate) unk_54: [u8; 0xc], - /// Required: 4. - pub(crate) unk_60: u32, - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_64: u32, - pub(crate) unk_68: [u8; 4], - /// Required: 1. - pub(crate) unk_6c: u32, - /// Required: 1. - pub(crate) unk_70: u32, - pub(crate) unk_74: [u8; 0xc], - /// Required: 0x64 (100). - pub(crate) unk_80: u32, - /// Required: 1. - pub(crate) unk_84: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_88: u32, - pub(crate) unk_8c: [u8; 4], - /// Required: 0x64 (100). - pub(crate) unk_90: u32, - pub(crate) unk_94: [u8; 0x2c], - /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. - pub(crate) unk_c0: [u32; 11], - pub(crate) unk_ec: [u8; 0x8cc], - /// Required: 0x271 (625). - pub(crate) unk_9b8: u32, - pub(crate) unk_9bc: [u8; 8], - /// Required: 0x3f7f_2e9f. - pub(crate) unk_9c4: u32, - pub(crate) unk_9c8: [u8; 4], - /// Required: 0x3b51_6154. - pub(crate) unk_9cc: u32, - pub(crate) unk_9d0: [u8; 4], - /// Required: 0x3ca5_9586 (0.020213 as an IEEE-754 single). - pub(crate) unk_9d4: u32, - pub(crate) unk_9d8: [u8; 8], - /// Required: 0x4615_3800 (9550.0 as an IEEE-754 single). - pub(crate) unk_9e0: u32, - /// Required: 0x40a9_0fdb. - pub(crate) unk_9e4: u32, - pub(crate) unk_9e8: [u8; 4], - /// Required: 0xbdd6_7344. - pub(crate) unk_9ec: u32, - /// Required: 0x28 (40). - pub(crate) unk_9f0: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_9f4: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_9f8: u32, - pub(crate) unk_9fc: [u8; 8], - /// Required: 0x254e (9550). - pub(crate) unk_a04: u32, - pub(crate) unk_a08: [u8; 0x30], - /// Required: 0x3e8 (1000). - pub(crate) unk_a38: u32, - pub(crate) unk_a3c: [u8; 0x34], - /// Required: 4. - pub(crate) unk_a70: u32, - pub(crate) unk_a74: [u8; 0x10], - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_a84: u32, - pub(crate) unk_a88: [u8; 4], - /// Required: 0x40cc_cccd (6.4 as an IEEE-754 single). - pub(crate) unk_a8c: u32, - pub(crate) unk_a90: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_a98: u32, - /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). - pub(crate) unk_a9c: u32, - pub(crate) unk_aa0: [u8; 8], - /// Required: 0x28 (40). - pub(crate) unk_aa8: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_aac: u32, - pub(crate) unk_ab0: [u8; 0xc], - /// Required: 0x254e (9550). - pub(crate) unk_abc: u32, - pub(crate) unk_ac0: [u8; 8], - /// Required: 0x10 (16). - pub(crate) unk_ac8: u32, - pub(crate) unk_acc: [u8; 4], - /// Required: 0x5_dc00 (384000). - pub(crate) unk_ad0: u32, - pub(crate) unk_ad4: [u8; 0x54], - /// Required: 0x5c (92). - pub(crate) unk_b28: u32, - pub(crate) unk_b2c: [u8; 4], - /// Required: 0x64 (100). - pub(crate) unk_b30: u32, - /// Required: 0x22 (34). - pub(crate) unk_b34: u32, - /// Required: 6. - pub(crate) unk_b38: u32, - pub(crate) unk_b3c: [u8; 4], - /// Required: 6. - pub(crate) unk_b40: u32, - /// Required: 1. - pub(crate) unk_b44: u32, - pub(crate) unk_b48: [u8; 4], - /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). - pub(crate) unk_b4c: u32, - /// Required: 0x3f7d_f3b6 (0.992 as an IEEE-754 single). - pub(crate) unk_b50: u32, - /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). - pub(crate) unk_b54: u32, - /// Required: 0x3c03_126f (0.008 as an IEEE-754 single). - pub(crate) unk_b58: u32, - /// Required: 0x3f69_d4d8 (0.913404 as an IEEE-754 single). - pub(crate) unk_b5c: u32, - /// Required: 0x3f69_d4d8 (0.913404 as an IEEE-754 single). - pub(crate) unk_b60: u32, - pub(crate) unk_b64: [u8; 4], - /// Required: 0x42be_0000 (95.0 as an IEEE-754 single). - pub(crate) unk_b68: u32, - /// Required: 0x4064_f5c3 (3.5775 as an IEEE-754 single). - pub(crate) unk_b6c: u32, - /// Required: 0x4064_f5c3 (3.5775 as an IEEE-754 single). - pub(crate) unk_b70: u32, - /// Required: 0x4117_9436. - pub(crate) unk_b74: u32, - /// Required: 0x64 (100). - pub(crate) unk_b78: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_b7c: u32, - /// Required: 0x64 (100). - pub(crate) unk_b80: u32, - pub(crate) unk_b84: [u8; 8], - /// Required: 0x5c (92). - pub(crate) unk_b8c: u32, - pub(crate) unk_b90: [u8; 0x30], - /// Required: 0x64 (100). - pub(crate) unk_bc0: u32, - pub(crate) unk_bc4: [u8; 0x1c], - /// Required: 1. - pub(crate) unk_be0: u8, - /// Required: 4. - pub(crate) unk_be1: u32, - /// Required: 0x64 (100). - pub(crate) unk_be5: u32, - pub(crate) unk_be9: [u8; 4], - /// Required: 0x1f40 (8000). - pub(crate) unk_bed: u32, - /// Required: 0xc8 (200). - pub(crate) unk_bf1: u32, - /// Required: 0xfa0 (4000). - pub(crate) unk_bf5: u32, - /// Required: 0xc8 (200). - pub(crate) unk_bf9: u32, - /// Required: 0x7d0 (2000). - pub(crate) unk_bfd: u32, - /// Required: 0xc8 (200). - pub(crate) unk_c01: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_c05: u32, - /// Required: 0xc8 (200). - pub(crate) unk_c09: u32, - /// Required: 1. - pub(crate) unk_c0d: u32, - pub(crate) unk_c11: [u8; 0x53], - /// Required: 1. - pub(crate) unk_c64: u32, - pub(crate) unk_c68: [u8; 0x28], - /// Required: 1. - pub(crate) unk_c90: u32, - pub(crate) unk_c94: [u8; 0x1003], - /// Required: 0x7fff in every entry; the table continues in - /// [`PowerConfigTail`]. - pub(crate) unk_1c97: [u16; 148], - /// Required: 0xff (low byte of the next table entry). - pub(crate) unk_1dbf: u8, -} - -static_assert!(bundle::POWER_CONFIG_COPY + size_of::() == 0xf000); +#[repr(C)] +pub(crate) struct PowerConfigHead(pub(crate) [u8; 0x1dc0]); -// SAFETY: `PowerConfigHead` consists of integers and arrays of integers only. +// SAFETY: `PowerConfigHead` is a byte array. unsafe impl Zeroable for PowerConfigHead {} impl PowerConfigHead { - /// Initializes the leading part of the power configuration. pub(crate) fn init(&mut self) { - clear(self); - self.unk_4 = 0x5_dc00; - self.unk_8 = 0x5_dc00; - self.unk_10 = 4; - self.unk_14 = 0x3f80_0000; - self.unk_2c = 1; - self.unk_30 = 1; - self.unk_40 = 0x64; - self.unk_44 = 1; - self.unk_48 = 0x3e8; - self.unk_50 = 0x64; - self.unk_60 = 4; - self.unk_64 = 0x3f80_0000; - self.unk_6c = 1; - self.unk_70 = 1; - self.unk_80 = 0x64; - self.unk_84 = 1; - self.unk_88 = 0x3e8; - self.unk_90 = 0x64; - self.unk_c0 = [0x3f82_8f5c; 11]; - self.unk_9b8 = 0x271; - self.unk_9c4 = 0x3f7f_2e9f; - self.unk_9cc = 0x3b51_6154; - self.unk_9d4 = 0x3ca5_9586; - self.unk_9e0 = 0x4615_3800; - self.unk_9e4 = 0x40a9_0fdb; - self.unk_9ec = 0xbdd6_7344; - self.unk_9f0 = 0x28; - self.unk_9f4 = 0x3e8; - self.unk_9f8 = 0x3e8; - self.unk_a04 = 0x254e; - self.unk_a38 = 0x3e8; - self.unk_a70 = 4; - self.unk_a84 = 0x3f80_0000; - self.unk_a8c = 0x40cc_cccd; - self.unk_a98 = 0x4780_0000; - self.unk_a9c = 0x3f00_0000; - self.unk_aa8 = 0x28; - self.unk_aac = 0x3e8; - self.unk_abc = 0x254e; - self.unk_ac8 = 0x10; - self.unk_ad0 = 0x5_dc00; - self.unk_b28 = 0x5c; - self.unk_b30 = 0x64; - self.unk_b34 = 0x22; - self.unk_b38 = 6; - self.unk_b40 = 6; - self.unk_b44 = 1; - self.unk_b4c = 0x3f4c_cccd; - self.unk_b50 = 0x3f7d_f3b6; - self.unk_b54 = 0x3e4c_cccd; - self.unk_b58 = 0x3c03_126f; - self.unk_b5c = 0x3f69_d4d8; - self.unk_b60 = 0x3f69_d4d8; - self.unk_b68 = 0x42be_0000; - self.unk_b6c = 0x4064_f5c3; - self.unk_b70 = 0x4064_f5c3; - self.unk_b74 = 0x4117_9436; - self.unk_b78 = 0x64; - self.unk_b7c = 0x3e8; - self.unk_b80 = 0x64; - self.unk_b8c = 0x5c; - self.unk_bc0 = 0x64; - self.unk_be0 = 1; - self.unk_be1 = 4; - self.unk_be5 = 0x64; - self.unk_bed = 0x1f40; - self.unk_bf1 = 0xc8; - self.unk_bf5 = 0xfa0; - self.unk_bf9 = 0xc8; - self.unk_bfd = 0x7d0; - self.unk_c01 = 0xc8; - self.unk_c05 = 0x3e8; - self.unk_c09 = 0xc8; - self.unk_c0d = 1; - self.unk_c64 = 1; - self.unk_c90 = 1; - self.unk_1c97 = [0x7fff; 148]; - self.unk_1dbf = 0xff; + fill_words(&mut self.0, POWER_CONFIG_HEAD_WORDS); } } @@ -2047,381 +1865,37 @@ impl PowerConfigHead { /// whole power configuration. It carries the die temperature limiter: target /// and release temperatures in 0.01 °C, proportional and integral gains as /// IEEE-754 singles. -#[repr(C, packed)] -pub(crate) struct PowerConfigTail { - /// Required: 0x7f (high byte of the table entry started in the head). - pub(crate) unk_1dc0: u8, - /// Required: 0x7fff in every entry. - pub(crate) unk_1dc1: [u16; 363], - pub(crate) unk_2097: [u8; 0x201], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2298: u32, - /// Required: 0x4220_0000 (40.0 as an IEEE-754 single). - pub(crate) unk_229c: u32, - /// Required: 0x447a_0000 (1000.0 as an IEEE-754 single). - pub(crate) unk_22a0: u32, - /// Required: 0x5be (1470). - pub(crate) unk_22a4: u32, - pub(crate) unk_22a8: [u8; 0x10], - /// Required: 0x28 (40). - pub(crate) unk_22b8: u32, - pub(crate) unk_22bc: [u8; 4], - /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). - pub(crate) unk_22c0: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_22c4: u32, - pub(crate) unk_22c8: [u8; 4], - /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). - pub(crate) unk_22cc: u32, - /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). - pub(crate) unk_22d0: u32, - pub(crate) unk_22d4: [u8; 0x3c], - /// Die temperature sensors the limiter reads ([`TEMP_SENSOR_MASK`]). - pub(crate) temp_sensor_mask: u64, - /// Release temperature of the limiter ([`TEMP_RELEASE`]). - pub(crate) temp_release: u32, - pub(crate) unk_231c: [u8; 4], - /// Required: 4. - pub(crate) unk_2320: u32, - pub(crate) unk_2324: [u8; 0x10], - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_2334: u32, - pub(crate) unk_2338: [u8; 4], - /// Integral gain of the limiter ([`TEMP_KI`]). - pub(crate) temp_ki: u32, - pub(crate) unk_2340: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2348: u32, - /// Proportional gain of the limiter ([`TEMP_KP`]). - pub(crate) temp_kp: u32, - pub(crate) unk_2350: [u8; 8], - /// Required: 0x28 (40). - pub(crate) unk_2358: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_235c: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2360: u32, - /// Margin below the target at which the limiter acts, in 0.01 °C. - pub(crate) temp_margin: u32, - pub(crate) unk_2368: [u8; 4], - /// Target temperature of the limiter ([`TEMP_TARGET`]). - pub(crate) temp_target: u32, - pub(crate) unk_2370: [u8; 0x30], - /// Required: 0x3e8 (1000). - pub(crate) unk_23a0: u32, - pub(crate) unk_23a4: [u8; 0x218], - /// Required: 0x4200_0006. - pub(crate) unk_25bc: u32, - pub(crate) unk_25c0: [u8; 0x212], - /// Required: 0x3f7e_b852 (0.995 as an IEEE-754 single). - pub(crate) unk_27d2: u32, - pub(crate) unk_27d6: [u8; 4], - /// Required: 0x3ba3_d70a (0.005 as an IEEE-754 single). - pub(crate) unk_27da: u32, - pub(crate) unk_27de: [u8; 4], - /// Required: 0x43c8_0000 (400.0 as an IEEE-754 single). - pub(crate) unk_27e2: u32, - pub(crate) unk_27e6: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_27ee: u32, - /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). - pub(crate) unk_27f2: u32, - pub(crate) unk_27f6: [u8; 4], - /// Required: 0xbac8_0000. - pub(crate) unk_27fa: u32, - /// Required: 0x384 (900). - pub(crate) unk_27fe: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2802: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2806: u32, - pub(crate) unk_280a: [u8; 4], - /// Required: 0x453b_8000 (3000.0 as an IEEE-754 single). - pub(crate) unk_280e: u32, - /// Required: 0x26ac (9900). - pub(crate) unk_2812: u32, - pub(crate) unk_2816: [u8; 0x30], - /// Required: 0x3e8 (1000). - pub(crate) unk_2846: u32, - pub(crate) unk_284a: [u8; 0x18], - /// Required: 0xc8 (200). - pub(crate) unk_2862: u32, - pub(crate) unk_2866: [u8; 0xb2], - /// Required: 0x3e8 (1000). - pub(crate) unk_2918: u32, - pub(crate) unk_291c: [u8; 0x10], - /// Required: 1. - pub(crate) unk_292c: u32, - pub(crate) unk_2930: [u8; 0x30], - /// Required: 1. - pub(crate) unk_2960: u8, - /// Required: 1. - pub(crate) unk_2961: u32, - /// Required: 4. - pub(crate) unk_2965: u32, - pub(crate) unk_2969: [u8; 0x14], - /// Required: 0x23f (575). - pub(crate) unk_297d: u32, - /// Required: 1. - pub(crate) unk_2981: u32, - /// Required: 1. - pub(crate) unk_2985: u32, - pub(crate) unk_2989: [u8; 0x14], - /// Required: 0x240 (576). - pub(crate) unk_299d: u32, - /// Required: 1. - pub(crate) unk_29a1: u32, - /// Required: 1. - pub(crate) unk_29a5: u32, - pub(crate) unk_29a9: [u8; 0x14], - /// Required: 0x241 (577). - pub(crate) unk_29bd: u32, - pub(crate) unk_29c1: [u8; 0x47], - /// Required: 1. - pub(crate) unk_2a08: u32, - pub(crate) unk_2a0c: [u8; 4], - /// Required: 0x254e (9550) in every entry. - pub(crate) unk_2a10: [u32; 3], - pub(crate) unk_2a1c: [u8; 0xc], - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_2a28: u32, - /// Required: 4. - pub(crate) unk_2a2c: u32, - /// Required: 0x10 (16). - pub(crate) unk_2a30: u32, - /// Required: 0x5_dc00 (384000). - pub(crate) unk_2a34: u32, - pub(crate) unk_2a38: [u8; 0xcc], - /// Required: 0x3c (60). - pub(crate) unk_2b04: u32, - pub(crate) unk_2b08: [u8; 8], - /// Required: 0x3f6e_eeef. - pub(crate) unk_2b10: u32, - pub(crate) unk_2b14: [u8; 4], - /// Required: 0x3d88_8889. - pub(crate) unk_2b18: u32, - pub(crate) unk_2b1c: [u8; 4], - /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). - pub(crate) unk_2b20: u32, - pub(crate) unk_2b24: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2b2c: u32, - /// Required: 0x4088_0000 (4.25 as an IEEE-754 single). - pub(crate) unk_2b30: u32, - pub(crate) unk_2b34: [u8; 8], - /// Required: 0x28 (40). - pub(crate) unk_2b3c: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2b40: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2b44: u32, - pub(crate) unk_2b48: [u8; 4], - /// Required: 0x4615_3800 (9550.0 as an IEEE-754 single). - pub(crate) unk_2b4c: u32, - /// Required: 0x254e (9550). - pub(crate) unk_2b50: u32, - pub(crate) unk_2b54: [u8; 8], - /// Required: 0xf0 (240). - pub(crate) unk_2b5c: u32, - pub(crate) unk_2b60: [u8; 4], - /// Required: 0x57_e400. - pub(crate) unk_2b64: u32, - pub(crate) unk_2b68: [u8; 0x1c], - /// Required: 0x3e8 (1000). - pub(crate) unk_2b84: u32, - pub(crate) unk_2b88: [u8; 0x3c], - /// Required: 1. - pub(crate) unk_2bc4: u32, - pub(crate) unk_2bc8: [u8; 0xcc], - /// Required: 0x32 (50). - pub(crate) unk_2c94: u32, - /// Required: 1. - pub(crate) unk_2c98: u32, - pub(crate) unk_2c9c: [u8; 4], - /// Required: 0x3f63_8e39. - pub(crate) unk_2ca0: u32, - /// Required: 0x3f2a_aaab. - pub(crate) unk_2ca4: u32, - /// Required: 0x3de3_8e39. - pub(crate) unk_2ca8: u32, - /// Required: 0x3eaa_aaab. - pub(crate) unk_2cac: u32, - /// Required: 0xbf4c_cccd (-0.8 as an IEEE-754 single). - pub(crate) unk_2cb0: u32, - /// Required: 0xbf4c_cccd (-0.8 as an IEEE-754 single). - pub(crate) unk_2cb4: u32, - pub(crate) unk_2cb8: [u8; 4], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2cbc: u32, - /// Required: 0xc0a0_0000 (-5.0 as an IEEE-754 single). - pub(crate) unk_2cc0: u32, - /// Required: 0xc0a0_0000 (-5.0 as an IEEE-754 single). - pub(crate) unk_2cc4: u32, - pub(crate) unk_2cc8: [u8; 4], - /// Required: 0x64 (100). - pub(crate) unk_2ccc: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2cd0: u32, - /// Required: 0x64 (100). - pub(crate) unk_2cd4: u32, - pub(crate) unk_2cd8: [u8; 4], - /// Required: 0x467a_0000 (16000.0 as an IEEE-754 single). - pub(crate) unk_2cdc: u32, - /// Required: 0x578 (1400). - pub(crate) unk_2ce0: u32, - pub(crate) unk_2ce4: [u8; 8], - /// Required: 0x90 (144). - pub(crate) unk_2cec: u32, - /// Required: 0x30 (48). - pub(crate) unk_2cf0: u32, - /// Required: 0x34_bc00. - pub(crate) unk_2cf4: u32, - pub(crate) unk_2cf8: [u8; 4], - /// Required: 0x11_9400. - pub(crate) unk_2cfc: u32, - pub(crate) unk_2d00: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2d08: u32, - pub(crate) unk_2d0c: [u8; 0x24], - /// Required: 0x3e80 (16000). - pub(crate) unk_2d30: u32, - /// Required: 2. - pub(crate) unk_2d34: u32, - /// Required: 0x9c4 (2500). - pub(crate) unk_2d38: u32, - /// Required: 0x20d (525). - pub(crate) unk_2d3c: u32, - /// Required: 2. - pub(crate) unk_2d40: u32, - /// Required: 4. - pub(crate) unk_2d44: u32, - /// Required: 0x32 (50). - pub(crate) unk_2d48: u32, - pub(crate) unk_2d4c: [u8; 0x2c], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2d78: u32, - pub(crate) unk_2d7c: [u8; 0xc], - /// Required: 0x28 (40). - pub(crate) unk_2d88: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2d8c: u32, - pub(crate) unk_2d90: [u8; 0x30], -} - -static_assert!(core::mem::offset_of!(PowerConfigTail, temp_sensor_mask) == 0x2310 - 0x1dc0); -static_assert!(core::mem::offset_of!(PowerConfigTail, temp_target) == 0x236c - 0x1dc0); -static_assert!(size_of::() == 0x1000); +#[repr(C)] +pub(crate) struct PowerConfigTail(pub(crate) [u8; 0x1000]); -// SAFETY: `PowerConfigTail` consists of integers and arrays of integers only. +// SAFETY: `PowerConfigTail` is a byte array. unsafe impl Zeroable for PowerConfigTail {} impl PowerConfigTail { fn init(&mut self) { - clear(self); - self.unk_1dc0 = 0x7f; - self.unk_1dc1 = [0x7fff; 363]; - self.unk_2298 = 0x4780_0000; - self.unk_229c = 0x4220_0000; - self.unk_22a0 = 0x447a_0000; - self.unk_22a4 = 0x5be; - self.unk_22b8 = 0x28; - self.unk_22c0 = 0x42c8_0000; - self.unk_22c4 = 0x3e8; - self.unk_22cc = 0x3f4c_cccd; - self.unk_22d0 = 0x3e4c_cccd; - self.temp_sensor_mask = TEMP_SENSOR_MASK; - self.temp_release = TEMP_RELEASE; - self.unk_2320 = 4; - self.unk_2334 = 0x3f80_0000; - self.temp_ki = TEMP_KI; - self.unk_2348 = 0x4780_0000; - self.temp_kp = TEMP_KP; - self.unk_2358 = 0x28; - self.unk_235c = 0x3e8; - self.unk_2360 = 0x3e8; - self.temp_target = TEMP_TARGET; - self.unk_23a0 = 0x3e8; - self.unk_25bc = 0x4200_0006; - self.unk_27d2 = 0x3f7e_b852; - self.unk_27da = 0x3ba3_d70a; - self.unk_27e2 = 0x43c8_0000; - self.unk_27ee = 0x4780_0000; - self.unk_27f2 = 0x42c8_0000; - self.unk_27fa = 0xbac8_0000; - self.unk_27fe = 0x384; - self.unk_2802 = 0x3e8; - self.unk_2806 = 0x3e8; - self.unk_280e = 0x453b_8000; - self.unk_2812 = 0x26ac; - self.unk_2846 = 0x3e8; - self.unk_2862 = 0xc8; - self.unk_2918 = 0x3e8; - self.unk_292c = 1; - self.unk_2960 = 1; - self.unk_2961 = 1; - self.unk_2965 = 4; - self.unk_297d = 0x23f; - self.unk_2981 = 1; - self.unk_2985 = 1; - self.unk_299d = 0x240; - self.unk_29a1 = 1; - self.unk_29a5 = 1; - self.unk_29bd = 0x241; - self.unk_2a08 = 1; - self.unk_2a10 = [0x254e; 3]; - self.unk_2a28 = 0x3f80_0000; - self.unk_2a2c = 4; - self.unk_2a30 = 0x10; - self.unk_2a34 = 0x5_dc00; - self.unk_2b04 = 0x3c; - self.unk_2b10 = 0x3f6e_eeef; - self.unk_2b18 = 0x3d88_8889; - self.unk_2b20 = 0x3f00_0000; - self.unk_2b2c = 0x4780_0000; - self.unk_2b30 = 0x4088_0000; - self.unk_2b3c = 0x28; - self.unk_2b40 = 0x3e8; - self.unk_2b44 = 0x3e8; - self.unk_2b4c = 0x4615_3800; - self.unk_2b50 = 0x254e; - self.unk_2b5c = 0xf0; - self.unk_2b64 = 0x57_e400; - self.unk_2b84 = 0x3e8; - self.unk_2bc4 = 1; - self.unk_2c94 = 0x32; - self.unk_2c98 = 1; - self.unk_2ca0 = 0x3f63_8e39; - self.unk_2ca4 = 0x3f2a_aaab; - self.unk_2ca8 = 0x3de3_8e39; - self.unk_2cac = 0x3eaa_aaab; - self.unk_2cb0 = 0xbf4c_cccd; - self.unk_2cb4 = 0xbf4c_cccd; - self.unk_2cbc = 0x4780_0000; - self.unk_2cc0 = 0xc0a0_0000; - self.unk_2cc4 = 0xc0a0_0000; - self.unk_2ccc = 0x64; - self.unk_2cd0 = 0x3e8; - self.unk_2cd4 = 0x64; - self.unk_2cdc = 0x467a_0000; - self.unk_2ce0 = 0x578; - self.unk_2cec = 0x90; - self.unk_2cf0 = 0x30; - self.unk_2cf4 = 0x34_bc00; - self.unk_2cfc = 0x11_9400; - self.unk_2d08 = 0x4780_0000; - self.unk_2d30 = 0x3e80; - self.unk_2d34 = 2; - self.unk_2d38 = 0x9c4; - self.unk_2d3c = 0x20d; - self.unk_2d40 = 2; - self.unk_2d44 = 4; - self.unk_2d48 = 0x32; - self.unk_2d78 = 0x4780_0000; - self.unk_2d88 = 0x28; - self.unk_2d8c = 0x3e8; + fill_words(&mut self.0, POWER_CONFIG_TAIL_WORDS); } } +/// Auxiliary hardware data the firmware reads at [`bundle::AUX`], up to the +/// start of view 3. No pointer names it. +#[repr(C)] +pub(crate) struct HwDataAux(pub(crate) [u8; 0x3c80]); + +// SAFETY: `HwDataAux` is a byte array. +unsafe impl Zeroable for HwDataAux {} + +impl HwDataAux { + pub(crate) fn init(&mut self, secondary_status_va: u64) { + fill_words(&mut self.0, HW_DATA_AUX_WORDS); + self.0[0xed8..0xee0].copy_from_slice(&secondary_status_va.to_le_bytes()); + } +} + + + + + /// Power configuration, in view 2 of the bundle. The secondary instance /// finds it through [`MainConfig::power_config_va`]. #[repr(C, packed)] @@ -2446,174 +1920,7 @@ impl PowerConfig { } } -/// Auxiliary hardware data the firmware reads at [`bundle::AUX`], up to the -/// start of view 3. No pointer names it. -#[repr(C, packed)] -pub(crate) struct HwDataAux { - pub(crate) unk_0: [u8; 0xac], - /// Required: 6. - pub(crate) unk_ac: u32, - pub(crate) unk_b0: [u8; 8], - /// Required: 1. - pub(crate) unk_b8: u32, - pub(crate) unk_bc: [u8; 0xbb], - /// Required: 0xdac (3500). - pub(crate) unk_177: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_17b: u32, - /// Required: 0xbb8 (3000). - pub(crate) unk_17f: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_183: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_187: u32, - /// Required: 0x64 (100). - pub(crate) unk_18b: u32, - /// Required: 0x20 (32). - pub(crate) unk_18f: u32, - pub(crate) unk_193: [u8; 0x21], - /// Required: 4. - pub(crate) unk_1b4: u32, - pub(crate) unk_1b8: [u8; 8], - /// Required: 0x3f78_0000 (0.96875 as an IEEE-754 single). - pub(crate) unk_1c0: u32, - pub(crate) unk_1c4: [u8; 4], - /// Required: 0x3d00_0000 (0.03125 as an IEEE-754 single). - pub(crate) unk_1c8: u32, - pub(crate) unk_1cc: [u8; 4], - /// Required: 0x40a0_0000 (5.0 as an IEEE-754 single). - pub(crate) unk_1d0: u32, - pub(crate) unk_1d4: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_1dc: u32, - /// Required: 0x4170_0000 (15.0 as an IEEE-754 single). - pub(crate) unk_1e0: u32, - pub(crate) unk_1e4: [u8; 8], - /// Required: 0x384 (900). - pub(crate) unk_1ec: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_1f0: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_1f4: u32, - pub(crate) unk_1f8: [u8; 8], - /// Required: 0x3e8 (1000). - pub(crate) unk_200: u32, - pub(crate) unk_204: [u8; 0x30], - /// Required: 0x3e8 (1000). - pub(crate) unk_234: u32, - pub(crate) unk_238: [u8; 0x3c], - /// Required: 0x64 (100). - pub(crate) unk_274: u32, - pub(crate) unk_278: [u8; 0x18], - /// Required: 0x82a (2090). - pub(crate) unk_290: u32, - pub(crate) unk_294: [u8; 4], - /// Required: 0x7d (125). - pub(crate) unk_298: u32, - /// Required: 1. - pub(crate) unk_29c: u32, - pub(crate) unk_2a0: [u8; 0xc38], - /// GPU address of the secondary state, [`private::SECONDARY_STATUS`] bytes - /// into the secondary state grid. - pub(crate) secondary_status_va: u64, - /// Required: 0x18ae (6318). - pub(crate) unk_ee0: u32, - pub(crate) unk_ee4: [u8; 4], - /// Required: 1. - pub(crate) unk_ee8: u32, - pub(crate) unk_eec: [u8; 0x2000], - /// Required: 0x41c8_cccd (25.1 as an IEEE-754 single). - pub(crate) unk_2eec: u32, - /// Required: 0x41c8_cccd (25.1 as an IEEE-754 single). - pub(crate) unk_2ef0: u32, - pub(crate) unk_2ef4: [u8; 0x6f0], - /// Required: 1. - pub(crate) unk_35e4: u32, - /// Required: 0x1f4 (500). - pub(crate) unk_35e8: u32, - /// Required: 6. - pub(crate) unk_35ec: u32, - /// Required: 0x30d4 (12500) in every entry. - pub(crate) unk_35f0: [u32; 6], - /// Required: 0xfa0 (4000). - pub(crate) unk_3608: u32, - pub(crate) unk_360c: [u8; 0x10], - /// Required: 0x30d4 (12500) in every entry. - pub(crate) unk_361c: [u32; 5], - /// Required: 1. - pub(crate) unk_3630: u32, - pub(crate) unk_3634: [u8; 0x174], - /// Required: 1. - pub(crate) unk_37a8: u32, - pub(crate) unk_37ac: [u8; 8], - /// Required: 0x424c_cccd (51.2 as an IEEE-754 single). - pub(crate) unk_37b4: u32, - pub(crate) unk_37b8: [u8; 0x3c], - /// Required: 0x4406_8000 (538.0 as an IEEE-754 single). - pub(crate) unk_37f4: u32, - pub(crate) unk_37f8: [u8; 0x3c], - /// Required: 0x41c9_999a (25.2 as an IEEE-754 single). - pub(crate) unk_3834: u32, - pub(crate) unk_3838: [u8; 0xa8], - /// Required: 0x186a (6250). - pub(crate) unk_38e0: u32, - pub(crate) unk_38e4: [u8; 0x39c], -} - -static_assert!(core::mem::offset_of!(HwDataAux, secondary_status_va) == 0xed8); -static_assert!(bundle::AUX + size_of::() == bundle::VIEWS[3]); - -// SAFETY: `HwDataAux` consists of integers and arrays of integers only. -unsafe impl Zeroable for HwDataAux {} -impl HwDataAux { - /// Initializes the auxiliary hardware data. `secondary_status_va` is - /// the secondary state grid plus [`private::SECONDARY_STATUS`]. - pub(crate) fn init(&mut self, secondary_status_va: u64) { - clear(self); - self.unk_ac = 6; - self.unk_b8 = 1; - self.unk_177 = 0xdac; - self.unk_17b = 0x3e8; - self.unk_17f = 0xbb8; - self.unk_183 = 0x3e8; - self.unk_187 = 0x3e8; - self.unk_18b = 0x64; - self.unk_18f = 0x20; - self.unk_1b4 = 4; - self.unk_1c0 = 0x3f78_0000; - self.unk_1c8 = 0x3d00_0000; - self.unk_1d0 = 0x40a0_0000; - self.unk_1dc = 0x4780_0000; - self.unk_1e0 = 0x4170_0000; - self.unk_1ec = 0x384; - self.unk_1f0 = 0x3e8; - self.unk_1f4 = 0x3e8; - self.unk_200 = 0x3e8; - self.unk_234 = 0x3e8; - self.unk_274 = 0x64; - self.unk_290 = 0x82a; - self.unk_298 = 0x7d; - self.unk_29c = 1; - self.unk_ee0 = 0x18ae; - self.unk_ee8 = 1; - self.unk_2eec = 0x41c8_cccd; - self.unk_2ef0 = 0x41c8_cccd; - self.unk_35e4 = 1; - self.unk_35e8 = 0x1f4; - self.unk_35ec = 6; - self.unk_35f0 = [0x30d4; 6]; - self.unk_3608 = 0xfa0; - self.unk_361c = [0x30d4; 5]; - self.unk_3630 = 1; - self.unk_37a8 = 1; - self.unk_37b4 = 0x424c_cccd; - self.unk_37f4 = 0x4406_8000; - self.unk_3834 = 0x41c9_999a; - self.unk_38e0 = 0x186a; - self.secondary_status_va = secondary_status_va; - } -} // --------------------------------------------------------------------------- // Free lists // --------------------------------------------------------------------------- From 8e98d0432927d780061fcc55516769f3d830291f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 12:32:59 -0400 Subject: [PATCH 0650/1579] drm/asahi: grow the scheduler-release table instead of failing a close The table of contexts awaiting their firmware scheduler-state release was fixed at 128 entries and a close that found it full returned ENOSPC, which the queue release path turns into a device stop for every client. A burst of closes that outpaces the firmware's consumption of release records is a legitimate workload, not a fault. Grow the table on demand (the close path runs under a sleepable mutex) and log once when the preallocated capacity is exceeded. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 7e317cd1218e5461124fbfc8db7dd72e71253f8e) --- drivers/gpu/drm/asahi/g17/teardown.rs | 28 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 791ac690c0f50..5d56931879f41 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -5,10 +5,12 @@ //! The device mutex serializes this state with the Registry's displaced graph ownership. use super::{context::Context, freelist::control_consumed, fw::channels::SchedulerStateRelease}; -use core::sync::atomic::{fence, Ordering}; +use core::sync::atomic::{fence, AtomicBool, Ordering}; use kernel::{prelude::*, sync::Arc}; +/// Entries preallocated at device start; a burst of closes beyond it grows the table. const CAPACITY: usize = 128; +static GREW: AtomicBool = AtomicBool::new(false); const POLL_NS: u64 = 2_000_000_000; #[derive(Clone, Copy, PartialEq, Eq)] @@ -80,15 +82,23 @@ impl Pending { .position(|entry| Arc::ptr_eq(&entry.context, context)) } + /// Closes outpacing firmware release consumption grow the table instead of + /// failing the close: a failed close stops the device for every client. fn insert(&mut self, context: &Arc, phase: Phase, now: u64) -> Result { - self.entries - .push_within_capacity(Entry { - context: context.clone(), - phase, - armed: now, - announce_pending: false, - }) - .map_err(|_| ENOSPC) + let entry = Entry { + context: context.clone(), + phase, + armed: now, + announce_pending: false, + }; + if self.entries.len() >= CAPACITY && !GREW.swap(true, Ordering::Relaxed) { + pr_info!( + "asahi: G17 scheduler-release table grew past {} pending contexts\n", + CAPACITY + ); + } + self.entries.push(entry, GFP_KERNEL)?; + Ok(()) } /// Call before replacing a graph's logical owner. Failure leaves that graph with its From be6a3b5d938f2c51ef5b6d60990dcefd64cb677f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:33:29 -0400 Subject: [PATCH 0651/1579] drm/asahi: add a render flag for passes with at most four known primitives Userspace knows when a render pass rasterizes at most four primitives in total (every draw's topology-derived count times instances, plus internal work) and that no indirect, mesh or otherwise uncounted work is part of it. Let it say so: DRM_ASAHI_RENDER_FEW_PRIMITIVES sets bit 0 of the fragment stage's auxiliary framebuffer control word, which the driver left clear so far. The flag is advertised by DRM_ASAHI_FEATURE_FEW_PRIMITIVES and rejected by kernels without it, like every other render flag. The producer of that bit in the native stack was recovered from the saved userland driver disassembly (end of render pass: bit 0 set iff the pass's known-primitive counter is at most four and its known-work byte is set; cleared for indirect and mesh work), and the saved native capture of a four-draw workload carries the bit on all 160 fragment commands. The hardware effect of the bit is not established; omitting it is the safe default and remains the behaviour for every pass that does not set the flag. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 18dbc2de6f21c92c49ffafce39a38c8d6439a5b5) --- drivers/gpu/drm/asahi/g17/command.rs | 4 +++- drivers/gpu/drm/asahi/g17/fw/render.rs | 15 +++++++++++---- drivers/gpu/drm/asahi/g17/mod.rs | 3 ++- include/uapi/drm/asahi_drm.h | 21 +++++++++++++++++++++ 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 5929c14b264fd..16727b8559b2b 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -23,7 +23,8 @@ const MAX_TILE_BUFFER: u64 = 32768; const PROCESS_EMPTY: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES; const RSRC_SPEC_HI: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI; const DBIAS_INT: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT; -const RENDER_FLAGS: u32 = PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT; +const FEW_PRIMITIVES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FEW_PRIMITIVES; +const RENDER_FLAGS: u32 = PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES; /// Attachment settings persist until replaced, including by an empty list. #[derive(Clone, Copy)] @@ -374,6 +375,7 @@ impl Payload { scissor_va: cmd.isp_scissor_base, depth_bias_va: cmd.isp_dbias_base, depth_bias_is_int: cmd.flags & DBIAS_INT != 0, + few_primitives: cmd.flags & FEW_PRIMITIVES != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, sampler_count: cmd.sampler_count.into(), diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index e3da1b0e6d954..657efe508bb3d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -85,6 +85,8 @@ const TILE_CONFIG_BASE: u64 = 0x280; const TILE_CONFIG_LAYERED: u64 = 1 << 0; const TILE_CONFIG_PROCESS_EMPTY_TILES: u64 = 1 << 16; const AUX_FB_FLAGS_BASE: u64 = 0xc000; +/// The pass rasterizes at most four known primitives (userspace's promise). +const AUX_FB_FLAGS_FEW_PRIMITIVES: u64 = 1 << 0; const AUX_FB_FLAGS_DBIAS_INT: u64 = 1 << 18; /// Tiling layer mode: layer count less one, bit 15, and for layered targets /// bit 14 plus the empty-tile policy. The second copy keeps only the count and @@ -178,6 +180,8 @@ pub(crate) struct RenderPass { pub(crate) depth_bias_va: u64, /// Depth-bias values are integers. pub(crate) depth_bias_is_int: bool, + /// At most four known primitives; sets the auxiliary framebuffer bit 0. + pub(crate) few_primitives: bool, /// GPU address of the occlusion-query results, or zero. pub(crate) occlusion_query_va: u64, /// GPU address of the sampler heap, or zero. @@ -423,10 +427,13 @@ impl PassWords { partial_zls_control |= ZLS_PARTIAL_STENCIL; } - let aux_fb_flags = match pass.depth_bias_is_int { - true => AUX_FB_FLAGS_BASE | AUX_FB_FLAGS_DBIAS_INT, - false => AUX_FB_FLAGS_BASE, - }; + let mut aux_fb_flags = AUX_FB_FLAGS_BASE; + if pass.depth_bias_is_int { + aux_fb_flags |= AUX_FB_FLAGS_DBIAS_INT; + } + if pass.few_primitives { + aux_fb_flags |= AUX_FB_FLAGS_FEW_PRIMITIVES; + } let rsrc_spec = |spec: u64| match pass.rsrc_spec_hi { true => spec, false => spec & u32::MAX as u64, diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index cb2af33c60f90..aaad02a68d978 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1091,7 +1091,8 @@ impl gpu::Gpu for Gpu { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64; + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index cdb177bf4df05..f4c36d1c2fc1b 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -286,6 +286,12 @@ enum drm_asahi_feature { * These fields supplement the header barriers. */ DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS = (1UL) << 5, + + /** + * @DRM_ASAHI_FEATURE_FEW_PRIMITIVES: Render commands accept + * DRM_ASAHI_RENDER_FEW_PRIMITIVES. + */ + DRM_ASAHI_FEATURE_FEW_PRIMITIVES = (1UL) << 6, }; /** @@ -1034,6 +1040,21 @@ enum drm_asahi_render_flags { * so we match that here for efficiency. */ DRM_ASAHI_RENDER_DBIAS_IS_INT = (1U << 18), + + /** + * @DRM_ASAHI_RENDER_FEW_PRIMITIVES: The render pass is known to rasterize + * at most four primitives in total. + * + * The count includes every primitive the pass generates: topology-derived + * primitive counts of all draws multiplied by their instance counts, plus + * internal work (clears, visibility, scissor/state generated work). Passes + * with indirect draws, mesh work or any work whose primitive count is not + * known at record time must not set this flag. The flag sets bit 0 of the + * auxiliary framebuffer control word for the fragment stage; the hardware + * effect is an optimization for small known passes and its omission is + * always safe. Requires DRM_ASAHI_FEATURE_FEW_PRIMITIVES. + */ + DRM_ASAHI_RENDER_FEW_PRIMITIVES = (1U << 19), }; /** From 3318d3e23f8e49a5721f9e2da0eea674eece1d2e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:43:08 -0400 Subject: [PATCH 0652/1579] drm/asahi: add a render flag that orders only the fragment stage on barriers The render prefix resolved from vdm_barrier/prior_vdm names the covered renders' fragment completions and was attached to the tiling kick, so a render's vertex processing waited for its predecessors' fragment work even when only its fragment stage consumes them. Keep the resolved dependencies apart by origin until the kick entries are built, and let userspace choose with DRM_ASAHI_RENDER_FRAGMENT_BARRIERS (advertised by DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS): with the flag the render prefix orders the fragment kick, which still waits for its own tiling kick and for the previous fragment work of its physical queue, so fragment order across a client's physical pairs is preserved; the tiling kick keeps its implicit parent. Compute dependencies keep ordering the tiling kick in both modes. Without the flag nothing changes. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c00f3a06ed2d0400a5ec210c856f3a382971f033) --- drivers/gpu/drm/asahi/g17/command.rs | 5 +- drivers/gpu/drm/asahi/g17/fw/render.rs | 2 + drivers/gpu/drm/asahi/g17/mod.rs | 3 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 56 +++++++++++++++++-- .../drm/asahi/g17/runtime/render/submit.rs | 30 +++++----- include/uapi/drm/asahi_drm.h | 25 +++++++++ 6 files changed, 98 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 16727b8559b2b..83191ddbcb1c3 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -24,7 +24,9 @@ const PROCESS_EMPTY: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS const RSRC_SPEC_HI: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI; const DBIAS_INT: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT; const FEW_PRIMITIVES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FEW_PRIMITIVES; -const RENDER_FLAGS: u32 = PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES; +const FRAGMENT_BARRIERS: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FRAGMENT_BARRIERS; +const RENDER_FLAGS: u32 = + PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES | FRAGMENT_BARRIERS; /// Attachment settings persist until replaced, including by an empty list. #[derive(Clone, Copy)] @@ -376,6 +378,7 @@ impl Payload { depth_bias_va: cmd.isp_dbias_base, depth_bias_is_int: cmd.flags & DBIAS_INT != 0, few_primitives: cmd.flags & FEW_PRIMITIVES != 0, + fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, sampler_count: cmd.sampler_count.into(), diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 657efe508bb3d..23eaa7e7dfba9 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -182,6 +182,8 @@ pub(crate) struct RenderPass { pub(crate) depth_bias_is_int: bool, /// At most four known primitives; sets the auxiliary framebuffer bit 0. pub(crate) few_primitives: bool, + /// Host-only: the resolved barriers order the fragment kick, not the tiling kick. + pub(crate) fragment_barriers: bool, /// GPU address of the occlusion-query results, or zero. pub(crate) occlusion_query_va: u64, /// GPU address of the sampler heap, or zero. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index aaad02a68d978..60c963b94a5a4 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1092,7 +1092,8 @@ impl gpu::Gpu for Gpu { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64; + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index fd67523ab91c3..720f2d8a0897e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -34,6 +34,29 @@ pub(crate) use dependencies::Dependencies; use kernel::{prelude::*, sync::Arc}; use retirement::{Ticket, Tracker}; +/// Firmware dependencies of one render, kept by origin until the kick entries +/// are built: the covered earlier renders (their fragment completions) and the +/// covered compute completion. Which kick waits for the render prefix is the +/// command's choice (`RenderPass::fragment_barriers`); compute always orders +/// the tiling kick, since compute output may feed vertex work. +#[derive(Copy, Clone)] +pub(crate) struct RenderDependencies { + pub(crate) render: [KickDependency; 2], + pub(crate) render_count: usize, + pub(crate) compute: Option, +} + +impl RenderDependencies { + pub(crate) const EMPTY: Self = Self { + render: [KickDependency::ZERO; 2], + render_count: 0, + compute: None, + }; + pub(crate) fn render(&self) -> &[KickDependency] { + &self.render[..self.render_count] + } +} + #[derive(Copy, Clone)] pub(crate) struct RenderAddresses { /// Tiling then fragment; each pair contains the primary and auxiliary stamp. @@ -375,7 +398,7 @@ impl Pair { pub(crate) fn publish( &mut self, packet: Arc, - dependencies: &[KickDependency], + dependencies: &RenderDependencies, lease: PublicationLease, host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, @@ -493,7 +516,32 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - let fragment_barriers = [KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?]; + // The fragment kick always waits for its own tiling kick. With the + // fragment-barriers flag the resolved prefix orders the fragment kick + // too, and the tiling kick keeps only its implicit parent, so vertex + // work overlaps the predecessors' fragment work as on the native stack. + let Validated::Render { pass, .. } = &packet.command else { + return Err(EINVAL); + }; + let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; + let mut tiling_barriers = [KickDependency::ZERO; 3]; + let mut tiling_count = 0; + let mut fragment_barriers = [own_tiling; 3]; + let mut fragment_count = 1; + for dependency in dependencies.render() { + if pass.fragment_barriers { + fragment_barriers[fragment_count] = *dependency; + fragment_count += 1; + } else { + tiling_barriers[tiling_count] = *dependency; + tiling_count += 1; + } + } + if let Some(compute) = dependencies.compute { + tiling_barriers[tiling_count] = compute; + tiling_count += 1; + } + let tiling_barriers = &tiling_barriers[..tiling_count]; let args = |stage: usize, barriers, arrays, mcache| KickArgs { qid: qids[stage], timestamp: kicks[stage], @@ -516,13 +564,13 @@ impl Pair { }; let ta_args = args( 0, - dependencies, + tiling_barriers, abi::TaDescriptor::register_bindings(self.memory.graph.descriptor_client(0, ordinal))?, None, ); let fragment_args = args( 1, - &fragment_barriers, + &fragment_barriers[..fragment_count], abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), )?, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 8b6c647668af4..39b8c064d0106 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -177,26 +177,25 @@ impl Registry { owner: u64, packet: &Arc, state: u32, - dependencies: &mut [crate::g17::fw::kick::KickDependency; 3], - ) -> Result { - let mut count = 0; + ) -> Result { + let mut dependencies = crate::g17::queue::render::RenderDependencies::EMPTY; let frontier = self.render_prefix(owner, packet)?; for dependency in frontier.as_slice() { - *dependencies.get_mut(count).ok_or(EOVERFLOW)? = *dependency; - count += 1; + *dependencies + .render + .get_mut(dependencies.render_count) + .ok_or(EOVERFLOW)? = *dependency; + dependencies.render_count += 1; } if let Some(queue) = self.compute(owner) { - if let Some(dependency) = queue.dependency( + dependencies.compute = queue.dependency( owner, packet.completion.status(), packet.order.wait_through[1], state, - )? { - *dependencies.get_mut(count).ok_or(EOVERFLOW)? = dependency; - count += 1; - } + )?; } - Ok(count) + Ok(dependencies) } } @@ -266,7 +265,7 @@ impl crate::g17::Firmware { packet: &Arc, deferred: &mut DeferredBatch, ) -> Result { - use crate::g17::{fw::kick::KickDependency, queue::render::PublicationLease}; + use crate::g17::queue::render::PublicationLease; let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; if self.recovery.pending() || state != 0 { return Err(EAGAIN); @@ -281,10 +280,7 @@ impl crate::g17::Firmware { { return Err(EBUSY); } - let mut dependencies = [KickDependency::ZERO; 3]; - let count = self - .queues - .render_dependencies(owner, packet, state, &mut dependencies)?; + let dependencies = self.queues.render_dependencies(owner, packet, state)?; let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); @@ -303,7 +299,7 @@ impl crate::g17::Firmware { let result = self.with_render(usize::from(slot), |pair, host| { pair.publish( packet.clone(), - &dependencies[..count], + &dependencies, lease, host, &mut |event| deferred.push(event), diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index f4c36d1c2fc1b..437ad42aeafda 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -292,6 +292,12 @@ enum drm_asahi_feature { * DRM_ASAHI_RENDER_FEW_PRIMITIVES. */ DRM_ASAHI_FEATURE_FEW_PRIMITIVES = (1UL) << 6, + + /** + * @DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS: Render commands accept + * DRM_ASAHI_RENDER_FRAGMENT_BARRIERS. + */ + DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS = (1UL) << 7, }; /** @@ -1055,6 +1061,25 @@ enum drm_asahi_render_flags { * always safe. Requires DRM_ASAHI_FEATURE_FEW_PRIMITIVES. */ DRM_ASAHI_RENDER_FEW_PRIMITIVES = (1U << 19), + + /** + * @DRM_ASAHI_RENDER_FRAGMENT_BARRIERS: Only the fragment stage waits for + * this command's barriers. + * + * The render dependencies resolved from @vdm_barrier and @prior_vdm are + * attached to the fragment half of the render instead of its vertex + * (tiling) half, so vertex processing may start while the renders this + * command depends on are still executing their fragment work. The + * fragment half still waits for its own vertex half and for the previous + * fragment work of its physical queue; the compute dependencies from + * @cdm_barrier and @prior_cdm keep ordering the vertex half. Set this + * only when every render dependency into the command is consumed by + * fragment-stage or attachment accesses alone: nothing the vertex stage + * reads (vertex/index/indirect buffers, vertex, tessellation or geometry + * shader resources, transform feedback, conditional-rendering predicates) + * may depend on it. Requires DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS. + */ + DRM_ASAHI_RENDER_FRAGMENT_BARRIERS = (1U << 20), }; /** From 0751d1c543c933f65ec2ce2606d5effd25206721 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 20:15:59 -0400 Subject: [PATCH 0653/1579] drm/asahi: give back the USC backing of idle retained compute queues An installed compute queue is retained for the device's lifetime, and each one owns a private USC free-list pool of about 23 MiB. After a burst of clients the retained pools pinned 1.6 GB on an 8 GB machine for the rest of the boot. Keep only the four newest idle retained queues populated: older idle queues publish the release of their free-list buffer slot, and once the control consumer has passed the record their pool backing is dropped. The queue, its ring and its control objects stay installed; an owner that later takes such a vacant queue first allocates fresh backing, rewrites the pool's control header and descriptor row, and only then binds and kicks. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 64820e0cb952592fff5708712a4b4766a8a12f4c) --- drivers/gpu/drm/asahi/g17/freelist.rs | 64 ++++++- drivers/gpu/drm/asahi/g17/queue/compute.rs | 92 +++++++++- drivers/gpu/drm/asahi/g17/runtime.rs | 192 ++++++++++++++++++-- drivers/gpu/drm/asahi/g17/runtime/events.rs | 1 + 4 files changed, 323 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index e1d42944a38fe..e40dfec7fc5fd 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -65,7 +65,7 @@ impl Drop for RenderId { } } -struct Pages { +pub(crate) struct Pages { backing: KernelObject, blocks: usize, layout: RunLayout, @@ -86,6 +86,10 @@ impl Pages { layout, }) } + /// Fresh zeroed compute backing, built without the device mutex. + pub(crate) fn compute(alloc: &Allocator<'_>) -> Result { + Self::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute) + } fn page_list_va(&self) -> u64 { self.backing.gpu_va() } @@ -324,8 +328,11 @@ impl Submitted<'_> { } /// A pool owned by one physical compute queue, with fixed QID-derived slot. +/// Idle retained queues may give their USC backing back (`pages` is `None`) +/// once the firmware consumed the pool's release; the control and state +/// objects the queue record names stay until processor stop. pub(crate) struct ComputePool { - pages: Pages, + pages: Option, control: KernelObject, state: KernelObject, id: u16, @@ -362,7 +369,7 @@ impl ComputePool { )?; fence(Ordering::SeqCst); Ok(Self { - pages, + pages: Some(pages), control, state, id, @@ -371,15 +378,58 @@ impl ComputePool { pub(crate) fn id(&self) -> u16 { self.id } - pub(crate) fn page_list_va(&self) -> u64 { - self.pages.page_list_va() + /// EAGAIN while the pool is vacant: the owner must re-arm it first. + pub(crate) fn page_list_va(&self) -> Result { + self.pages.as_ref().map(Pages::page_list_va).ok_or(EAGAIN) } pub(crate) fn control_va(&self) -> u64 { self.control.gpu_va() } + pub(crate) fn populated(&self) -> bool { + self.pages.is_some() + } + /// The release record for this pool's buffer slot under the free-list + /// generation the queue last published. + pub(crate) fn release_record(&self, generation: u64) -> super::fw::channels::FreeListRelease { + super::fw::channels::FreeListRelease::new(generation, u32::from(self.id)) + } + /// The caller has observed the firmware consume this pool's release. + pub(crate) fn drop_pages(&mut self) { + self.pages = None; + } + /// Rewrites the shared control header for fresh backing. The caller then + /// rewrites the page-pool descriptor row before the queue is kicked again. + /// As render repopulation does, the firmware-completed counter is carried + /// over so it stays consistent with the host submitted counter in `state`. + pub(crate) fn rearm(&mut self, pages: Pages) -> Result { + if self.pages.is_some() { + return Err(EBUSY); + } + let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; + let mut bytes = [0; 8]; + for (index, byte) in bytes.iter_mut().enumerate() { + // SAFETY: The checked eight-byte range stays mapped; byte accesses + // match the unaligned packed counter the firmware writes. + *byte = unsafe { completed.add(index).read_volatile() }; + } + let mut header = FreeListControl::compute(&FreeListArgs { + buffer_slot: u32::from(self.id), + class: FreeListClass::One, + page_list_va: pages.page_list_va(), + run_list_va: pages.run_list_va(), + blocks: COMPUTE_BLOCKS as u32, + state_va: self.state.gpu_va(), + }); + header.unk_54[..8].copy_from_slice(&bytes); + self.control.write(0, header)?; + fence(Ordering::SeqCst); + self.pages = Some(pages); + Ok(()) + } pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { - let address = self.pages.page_list_va(); - self.pages + let pages = self.pages.as_mut().ok_or(EAGAIN)?; + let address = pages.page_list_va(); + pages .backing .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index fcd0a1d6b40e6..cbca61493611e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -7,8 +7,9 @@ pub(crate) mod exit; pub(crate) mod retirement; use crate::g17::{ + channel::Rings, context::Context, - fw::{compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, + fw::{channels::ControlRecord, compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, object::{Allocator, CpuMap, KernelObject}, }; use crate::{hw::t8140::dynamic, mmu}; @@ -16,6 +17,10 @@ use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; use retirement::{Cursors, RECORD_SLOTS}; +fn now_ns() -> u64 { + ::ktime_get() as u64 +} + const DESCRIPTORS: usize = 256; const GRAPH_SIZE: usize = 0x8000; const CONTEXT_SIZE: usize = 0x20000; @@ -48,7 +53,9 @@ static_assert!(core::mem::offset_of!(GraphLayout, items) == ITEMS); /// until the first exact successful publication under their replacement has retired. struct Binding { _kick_alias: mmu::KernelMapping, - _pool_alias: mmu::KernelMapping, + /// Client alias of the USC pool pages; dropped when the pool is vacated and + /// recreated when an owner binds a re-armed pool (it pins the backing). + pool_alias: Option, _descriptors: mmu::KernelMapping, preempt: mmu::KernelMapping, usage: mmu::KernelMapping, @@ -153,7 +160,7 @@ impl Graph { .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; Ok(Binding { _kick_alias: kick_alias, - _pool_alias: pool_alias, + pool_alias: Some(pool_alias), _descriptors: descriptors, preempt, usage, @@ -341,8 +348,8 @@ impl PoolBacking { } /// Fresh page-pool row data, installed once while the QID is still unpublished. - pub(crate) fn pool_descriptor(&self) -> (u16, u64) { - (self.pool.id(), self.pool.page_list_va()) + pub(crate) fn pool_descriptor(&self) -> Result<(u16, u64)> { + Ok((self.pool.id(), self.pool.page_list_va()?)) } } @@ -363,8 +370,22 @@ impl Backing { /// Installed graph and its exact logical owner. A released idle graph may change owners only /// after a retirement witness. Failed work retains its pointers independently of fence state. +/// USC pool backing of a retained queue. A release is published only for an idle +/// queue without an owner; the backing is dropped once the firmware consumed it. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum PoolPhase { + Populated, + ReleaseSent(u32), + Vacant, +} + pub(crate) struct Queue { owner: Option, + pool_phase: PoolPhase, + /// Monotonic time the last owner released this queue (vacate order). + released_at: u64, + /// Free-list generation of the last published configuration. + free_list_generation: u64, binding: Binding, previous: Option, kick: kick::Queue, @@ -413,6 +434,9 @@ impl Queue { context.mark_published(); Ok(Self { owner: Some(owner), + pool_phase: PoolPhase::Populated, + released_at: 0, + free_list_generation: 0, binding, previous: None, kick, @@ -914,6 +938,8 @@ impl Queue { } pub(crate) fn reusable(&self) -> bool { self.installed + // A pool release in flight must be witnessed before any successor. + && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) // An issued kill still names this graph even if its commands retire // before the acknowledgement. It cannot acquire a successor owner. && self.exit.is_none() @@ -934,6 +960,10 @@ impl Queue { } let old = self.binding.context.clone(); if Arc::ptr_eq(old.status(), context.status()) && old.is_current() { + if self.binding.pool_alias.is_none() { + // The same VM takes back a re-armed pool: alias the new backing. + self.binding.pool_alias = Some(self.pool.map_client(context.vm())?); + } self.graph.set_owner(&context)?; self.binding.context = context.clone(); self.previous = Some(Previous { @@ -984,10 +1014,61 @@ impl Queue { } if self.retirement_ready { self.owner = None; + self.released_at = now_ns(); } } Ok(false) } + /// An idle retained queue whose USC backing may be given back. + pub(crate) fn vacate_candidate(&self) -> bool { + self.reusable() && self.pool_phase == PoolPhase::Populated && self.pool.populated() + } + pub(crate) fn released_at(&self) -> u64 { + self.released_at + } + pub(crate) fn pool_vacant(&self) -> bool { + self.pool_phase == PoolPhase::Vacant + } + /// Publishes the release of this idle queue's free list. The caller holds the + /// device mutex and rings the control doorbell afterwards. + pub(crate) fn vacate(&mut self, init: &crate::g17::initdata::InitData) -> Result { + if !self.vacate_candidate() { + return Err(EBUSY); + } + let record = ControlRecord::FreeListRelease( + self.pool.release_record(self.free_list_generation), + ); + let cursor = Rings::publish_control(init, &record)?; + self.pool_phase = PoolPhase::ReleaseSent(cursor); + fence(Ordering::SeqCst); + Ok(()) + } + /// Drops the USC backing once the firmware's control consumer passed the + /// release. Returns true when the backing was freed by this call. + pub(crate) fn observe_vacancy(&mut self, consumer: u32, producer: u32) -> Result { + let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { + return Ok(false); + }; + if !crate::g17::freelist::control_consumed(consumer, producer, cursor)? { + return Ok(false); + } + // The retained owner binding's client alias pins the backing: drop it + // first (the owner has exited; `reusable()` proved no previous binding). + self.binding.pool_alias = None; + self.pool.drop_pages(); + self.pool_phase = PoolPhase::Vacant; + Ok(true) + } + /// Installs fresh backing into a vacant pool; returns the descriptor row data + /// the caller writes before binding an owner. + pub(crate) fn rearm(&mut self, pages: crate::g17::freelist::Pages) -> Result<(u16, u64)> { + if self.pool_phase != PoolPhase::Vacant || !self.reusable() { + return Err(EAGAIN); + } + self.pool.rearm(pages)?; + self.pool_phase = PoolPhase::Populated; + Ok((self.pool.id(), self.pool.page_list_va()?)) + } /// Exact host identity remains valid even when several commands share an ioctl ID. pub(crate) fn owns_packet(&self, packet: &Arc) -> bool { self.active @@ -1299,6 +1380,7 @@ impl Queue { qos: packet.context.qos_id(), context: packet.context.id(), }; + self.free_list_generation = u64::from(packet.context.scheduler_generation()); let config = QueueConfig::new(&QueueConfigArgs { target: QueueConfigTarget::Compute, kick_ring_va: self.kick.low_va(), diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 805087d7082ab..0de6f62760248 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -24,6 +24,7 @@ use super::{ kick, qos, queue::{self, admission, compute}, }; +use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; const QIDS: usize = 128; @@ -117,21 +118,127 @@ impl Registry { /// An installed idle queue keeps its original QID and backing. Mapping a /// replacement context must succeed before its old binding is displaced. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc) -> Result { + /// A vacant queue (USC backing given back) is offered for re-arming only + /// when no populated idle queue exists. + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc) -> Result { if let Some(queue) = self.compute(owner) { - return queue.matches(owner, queue.qid(), context); + return Ok(if queue.matches(owner, queue.qid(), context)? { + Rebind::Done + } else { + Rebind::Fresh + }); } - let Some(queue) = self + let mut vacant = None; + for queue in self .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.reusable()) - else { - return Ok(false); - }; - queue.bind_owner(owner, context.clone())?; - Ok(true) + .filter(|queue| queue.reusable()) + { + if queue.pool_vacant() { + vacant.get_or_insert(queue.qid()); + continue; + } + queue.bind_owner(owner, context.clone())?; + return Ok(Rebind::Done); + } + Ok(vacant.map_or(Rebind::Fresh, Rebind::Rearm)) + } + + /// Installs fresh USC backing into the vacant queue `qid`; the caller writes + /// the returned descriptor row before binding. EAGAIN if the queue was taken. + pub(super) fn rearm_compute( + &mut self, + qid: u8, + pages: crate::g17::freelist::Pages, + ) -> Result<(u16, u64)> { + let queue = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EAGAIN)?; + queue.rearm(pages) + } + + pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { + let queue = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid && queue.reusable()) + .ok_or(EAGAIN)?; + queue.bind_owner(owner, context.clone()) + } + + /// Publishes releases for the oldest idle retained pools beyond `cap`, at + /// most four per pass. Returns the number published. + pub(super) fn vacate_idle_pools( + &mut self, + init: &super::initdata::InitData, + cap: usize, + ) -> Result { + // Count first: event turns are frequent and the steady state has no excess. + let count = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.vacate_candidate()) + .count(); + if count <= cap { + return Ok(0); + } + let mut candidates: KVec<(u64, u8)> = KVec::with_capacity(count, GFP_KERNEL)?; + for queue in self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.vacate_candidate()) + { + candidates.push((queue.released_at(), queue.qid()), GFP_KERNEL)?; + } + candidates.sort_unstable(); + let excess = candidates.len() - cap; + let mut published = 0; + for (_, qid) in candidates.iter().take(excess.min(4)) { + let queue = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == *qid) + .ok_or(EIO)?; + match queue.vacate(init) { + Err(error) if error == EAGAIN => break, + result => result?, + } + published += 1; + } + Ok(published) + } + + /// Frees the backing of every pool whose release the firmware consumed. + pub(super) fn observe_pool_releases(&mut self, init: &super::initdata::InitData) -> Result { + fence(Ordering::Acquire); + let consumer = init.control_consumer()?.load(Ordering::Relaxed); + let producer = init.control_producer()?.load(Ordering::Relaxed); + let mut freed = 0; + for queue in self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + if queue.observe_vacancy(consumer, producer)? { + freed += 1; + } + } + Ok(freed) } pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { @@ -220,6 +327,17 @@ impl Registry { } } +/// Outcome of offering an owner a retained compute queue. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Rebind { + /// The owner now holds an installed, populated queue. + Done, + /// Only a vacant queue is available; re-arm `qid` with fresh backing. + Rearm(u8), + /// No retained queue suits the owner: reserve a fresh QID. + Fresh, +} + /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. pub(super) struct Backend { @@ -420,16 +538,41 @@ impl Backend { { return Err(EIO); } - if firmware.queues.rebind_compute(self.owner, &self.context)? { - return Ok(()); + match firmware.queues.rebind_compute(self.owner, &self.context)? { + Rebind::Done => return Ok(()), + Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), + Rebind::Fresh => { + let reservation = firmware.queues.reserve_compute(self.owner)?; + (firmware.uat.clone(), Ok(reservation)) + } } - let reservation = firmware.queues.reserve_compute(self.owner)?; - (firmware.uat.clone(), reservation) }; let alloc = super::object::Allocator { dev: &dev, uat: &uat, }; + let reservation = match reservation { + Ok(reservation) => reservation, + Err(qid) => { + // Fresh zeroed backing is built without the device mutex; the + // queue may have been taken or re-armed by another owner since. + let pages = crate::g17::freelist::Pages::compute(&alloc)?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + continue; + } + let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages) { + Err(error) if error == EAGAIN => continue, + result => result?, + }; + // The firmware consumed this slot's release before the backing + // was dropped; the row is rewritten before the queue is kicked. + firmware.init.initialize_compute_pool(slot, page_list)?; + firmware.queues.bind_rearmed(qid, self.owner, &self.context)?; + return Ok(()); + } + }; let built = (|| { let pool = compute::PoolBacking::new(&alloc, reservation.id)?; { @@ -439,7 +582,7 @@ impl Backend { return Err(EAGAIN); } firmware.queues.reserved(reservation)?; - let (slot, page_list) = pool.pool_descriptor(); + let (slot, page_list) = pool.pool_descriptor()?; firmware.init.initialize_compute_pool(slot, page_list)?; } let backing = compute::Backing::new(&alloc, pool)?; @@ -926,6 +1069,27 @@ impl super::Shared { } } +/// Idle retained compute queues that keep their USC pool populated; older idle +/// pools give their backing back. +const COMPUTE_WARM_CAP: usize = 4; + +impl super::Firmware { + /// Event-worker service step: witness consumed pool releases, then release the + /// oldest idle retained pools beyond the warm cap. + pub(in crate::g17) fn service_compute_pools(&mut self) -> Result { + self.queues.observe_pool_releases(&self.init)?; + if self.render_control_backpressured() { + return Ok(()); + } + let published = self.queues.vacate_idle_pools(&self.init, COMPUTE_WARM_CAP)?; + if published != 0 { + fence(Ordering::SeqCst); + self.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } + Ok(()) + } +} + /// One monotonic domain for recovery grace, idle effort and control retirement. fn now_ns() -> u64 { ::ktime_get() as u64 diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 08c11eb98790e..f92c493f4813d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -386,6 +386,7 @@ impl crate::g17::Shared { let idle = firmware.idle_check(now, in_flight)?; firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; + firmware.service_compute_pools()?; { let mut reclaim = self.reclaim.lock(); firmware.service_teardown(&mut reclaim)?; From ffbf4b465de966c4a5bf3b2e2881fd5678f98bfa Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 20:16:11 -0400 Subject: [PATCH 0654/1579] drm/asahi: drop an exited owner's context from idle retained compute queues A retained compute queue kept its last owner's Context, and through it the exited client's whole VM: page tables, every alias the binding held into that VM, the status and the QoS lease, until another owner took the queue. Once the owner's execution root has been released (after the scheduler-state release and command retirement were witnessed) and the queue is idle without an owner, drop the binding and the context and keep only the scheduler page and work storage the installed queue record still names. A later owner binds afresh, as a first owner does. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 3db48c0ac600b5e919c4d2713dbd84e9b12b4ec4) --- drivers/gpu/drm/asahi/g17/context.rs | 16 +++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 97 ++++++++++++++----- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 7 +- drivers/gpu/drm/asahi/g17/runtime.rs | 15 ++- .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 2 +- 5 files changed, 109 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index b7d0a351f234d..7076708be9f2f 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -101,6 +101,13 @@ impl WorkStorage { } } +/// Firmware-visible pages of a context, retained by an installed queue whose +/// last owner has exited (see [`Context::retain_firmware_pages`]). +pub(crate) struct FirmwarePages { + _scheduler: Arc, + _work: Arc>, +} + pub(crate) struct Context { execution: Arc, status: Arc, @@ -221,6 +228,15 @@ impl Context { pub(crate) fn scheduler_generation(&self) -> u32 { self.scheduler_generation } + /// The pages an installed queue record of this context still names: the + /// scheduler page and the work storage holding the job-list head. A retained + /// physical queue keeps these after dropping the context itself. + pub(crate) fn retain_firmware_pages(&self) -> FirmwarePages { + FirmwarePages { + _scheduler: self.scheduler.clone(), + _work: self.work.clone(), + } + } pub(crate) fn work_head_va(&self) -> u64 { self.work_base + (UAT_PGSZ + 0x400) as u64 } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index cbca61493611e..df38a727c8d60 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -381,12 +381,16 @@ pub(crate) enum PoolPhase { pub(crate) struct Queue { owner: Option, + /// Scheduler/work pages of an exited last owner whose context was dropped; + /// the installed queue record still names them until the next owner binds. + retained_owner: Option, pool_phase: PoolPhase, /// Monotonic time the last owner released this queue (vacate order). released_at: u64, /// Free-list generation of the last published configuration. free_list_generation: u64, - binding: Binding, + /// `None` after an exited owner was detached; a new owner binds afresh. + binding: Option, previous: Option, kick: kick::Queue, pool: ComputePool, @@ -434,10 +438,11 @@ impl Queue { context.mark_published(); Ok(Self { owner: Some(owner), + retained_owner: None, pool_phase: PoolPhase::Populated, released_at: 0, free_list_generation: 0, - binding, + binding: Some(binding), previous: None, kick, pool, @@ -474,12 +479,14 @@ impl Queue { pub(crate) fn owner(&self) -> Option { self.owner } - pub(crate) fn context(&self) -> &Arc { - &self.binding.context + /// The bound owner's context; `None` once an exited owner was detached. + pub(crate) fn context(&self) -> Option<&Arc> { + self.binding.as_ref().map(|binding| &binding.context) } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { - !Arc::ptr_eq(self.context(), context) || (self.active.len == 0 && self.replays.len == 0) + !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + || (self.active.len == 0 && self.replays.len == 0) } pub(crate) fn in_flight(&self) -> bool { self.active.len != 0 && !self.quarantined @@ -524,7 +531,9 @@ impl Queue { fn qos_owner(&self) -> qos::Owner { qos::Owner { qid: self.qid(), - qos: self.binding.context.qos_id(), + // Accounting completes only for bound owners; a detached queue has no + // pending work and never reaches this. + qos: self.binding.as_ref().map_or(0, |binding| binding.context.qos_id()), data_master: DataMaster::Compute as u8, } } @@ -598,7 +607,10 @@ impl Queue { || self.owner != Some(ticket.owner) || active.packet.order.sequence != ticket.submission || active.packet.context.id() != ticket.context_id - || active.packet.context.id() != self.binding.context.id() + || self + .binding + .as_ref() + .is_none_or(|binding| active.packet.context.id() != binding.context.id()) { return Err(EIO); } @@ -929,7 +941,10 @@ impl Queue { if self.released || self.quarantined || self.qid() != qid - || !Arc::ptr_eq(&self.binding.context, context) + || !self + .binding + .as_ref() + .is_some_and(|binding| Arc::ptr_eq(&binding.context, context)) || !context.is_current() { return Err(EIO); @@ -958,17 +973,21 @@ impl Queue { if !context.is_current() { return Err(EFAULT); } - let old = self.binding.context.clone(); - if Arc::ptr_eq(old.status(), context.status()) && old.is_current() { - if self.binding.pool_alias.is_none() { + let old = self.binding.as_ref().map(|binding| binding.context.clone()); + let same_vm = old + .as_ref() + .is_some_and(|old| Arc::ptr_eq(old.status(), context.status()) && old.is_current()); + if same_vm { + let binding = self.binding.as_mut().ok_or(EIO)?; + if binding.pool_alias.is_none() { // The same VM takes back a re-armed pool: alias the new backing. - self.binding.pool_alias = Some(self.pool.map_client(context.vm())?); + binding.pool_alias = Some(self.pool.map_client(context.vm())?); } self.graph.set_owner(&context)?; - self.binding.context = context.clone(); + binding.context = context.clone(); self.previous = Some(Previous { binding: None, - context: old, + context: old.ok_or(EIO)?, publication: None, }); } else { @@ -976,12 +995,15 @@ impl Queue { let pool_alias = self.pool.map_client(context.vm())?; let binding = self.graph.bind(context.clone(), kick_alias, pool_alias)?; self.graph.set_owner(&context)?; - let old_binding = core::mem::replace(&mut self.binding, binding); - self.previous = Some(Previous { - binding: Some(old_binding), + let old_binding = self.binding.replace(binding); + // A detached queue (exited owner already dropped) has nothing to supersede; + // the new owner's record fields replace the retained pages' references. + self.previous = old.map(|old| Previous { + binding: old_binding, context: old, publication: None, }); + self.retained_owner = None; } context.mark_published(); self.owner = Some(owner); @@ -1006,9 +1028,9 @@ impl Queue { } let mut previous = self.previous.take().ok_or(EIO)?; if let Some(binding) = previous.binding.take() { - self.binding = binding; + self.binding = Some(binding); } else { - self.binding.context = previous.context; + self.binding.as_mut().ok_or(EIO)?.context = previous.context; } self.retirement_ready = true; } @@ -1019,6 +1041,25 @@ impl Queue { } Ok(false) } + /// The last owner's file and VM are gone: its execution root was released + /// after the scheduler-state release and command retirement were witnessed + /// (`Context::release_execution`), and this queue is idle without an owner. + /// Drop the binding (every alias into that VM) and the context, keeping only + /// the scheduler page and work storage the installed queue record names. + pub(crate) fn detach_exited_owner(&mut self) -> bool { + if !self.reusable() { + return false; + } + let Some(binding) = self.binding.as_ref() else { + return false; + }; + if binding.context.is_current() { + return false; + } + self.retained_owner = Some(binding.context.retain_firmware_pages()); + self.binding = None; + true + } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { self.reusable() && self.pool_phase == PoolPhase::Populated && self.pool.populated() @@ -1054,7 +1095,9 @@ impl Queue { } // The retained owner binding's client alias pins the backing: drop it // first (the owner has exited; `reusable()` proved no previous binding). - self.binding.pool_alias = None; + if let Some(binding) = self.binding.as_mut() { + binding.pool_alias = None; + } self.pool.drop_pages(); self.pool_phase = PoolPhase::Vacant; Ok(true) @@ -1306,9 +1349,15 @@ impl Queue { { return Err(EBUSY); } - if !packet.context.is_current() || !Arc::ptr_eq(&packet.context, &self.binding.context) { + if !packet.context.is_current() + || !self + .binding + .as_ref() + .is_some_and(|binding| Arc::ptr_eq(&packet.context, &binding.context)) + { return Err(EFAULT); } + let binding = self.binding.as_ref().ok_or(EFAULT)?; if packet.completion.status().get() != 0 { return Err(EIO); } @@ -1361,9 +1410,9 @@ impl Queue { qid, kick_count: self.ordinal, kick: kick_timestamp, - preempt_va: self.binding.preempt.iova(), - operand_state_va: self.binding.operand.iova(), - usage_va: self.binding.usage.iova(), + preempt_va: binding.preempt.iova(), + operand_state_va: binding.operand.iova(), + usage_va: binding.usage.iova(), usage_fw_va: self.graph.usage.gpu_va(), free_list_slot: u32::from(self.pool.id()), free_list_control_va: self.pool.control_va(), diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index d1048fd04607f..08d1013e4762b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -77,7 +77,7 @@ pub(super) struct Transaction { impl Queue { pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { - Arc::ptr_eq(self.context(), context) + self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) && self.released && self.active.len != 0 && self.exit.is_none() @@ -156,7 +156,10 @@ impl Queue { if !recovery_closed { return Ok(()); } - let state = match State::read(self.context()) { + let Some(context) = self.context() else { + return Ok(()); + }; + let state = match State::read(context) { Ok(state) => state, Err(_) => return Ok(()), }; diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 0de6f62760248..204a3f97b3c52 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -822,7 +822,11 @@ impl Registry { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - if queue.in_flight() && Arc::ptr_eq(queue.context().status(), status) { + if queue.in_flight() + && queue + .context() + .is_some_and(|context| Arc::ptr_eq(context.status(), status)) + { queue.quarantine(error, &mut |event| deferred.push(event))?; } } @@ -1077,6 +1081,15 @@ impl super::Firmware { /// Event-worker service step: witness consumed pool releases, then release the /// oldest idle retained pools beyond the warm cap. pub(in crate::g17) fn service_compute_pools(&mut self) -> Result { + for queue in self + .queues + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + queue.detach_exited_owner(); + } self.queues.observe_pool_releases(&self.init)?; if self.render_control_backpressured() { return Ok(()); diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 439d5c9831cbb..984b6c1f643eb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -97,7 +97,7 @@ impl Firmware { for entry in self.queues.compute.iter().flatten() { if let Some(queue) = entry.queue.as_deref() { let bit = 1u128 << queue.qid(); - if Arc::ptr_eq(queue.context(), context) { + if queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) { own |= bit; } else { foreign |= bit; From 9964d01e50add98a41985adc69ace3bc5dc867f1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 16:02:55 -0400 Subject: [PATCH 0655/1579] drm/asahi: keep reclaim away from VMs that are in use or were just restored Idle-VM reclaim judged a VM idle whenever no job was accepted at that instant. A loading game submits every few tens of milliseconds, so its VM qualified between submissions: under memory pressure the shrinker released its GPU backing, the next SUBMIT restored every invalidated mapping while holding the residency gate (observed as 260-280 ms SUBMIT and VM_BIND stalls ending together), and the restored pages were evicted again seconds later. Two rules: a VM is eligible only once its last accepted job ended at least 500 ms ago, and after a restoration the VM stays ineligible for sixteen times the restoration's duration, bounded to 1-30 s. Truly idle VMs are reclaimed as before. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5427403e7135dd886ee52a78d73639bd5a2ccb17) --- drivers/gpu/drm/asahi/mmu/lifetime.rs | 14 ++++++++- drivers/gpu/drm/asahi/mmu/residency.rs | 42 +++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index d53c19b93bfdc..79b4dca112ce9 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -14,6 +14,13 @@ use kernel::{ }; const ADMISSION_WAIT_MS: u32 = 1000; +/// A VM whose last accepted job ended more recently than this is still in +/// use: evicting its backing would be restored by its next job. +const RECLAIM_QUIET_NS: u64 = 500_000_000; + +fn now_ns() -> u64 { + ::ktime_get() as u64 +} pub(super) struct DriverMappings { mappings: KVec, @@ -49,6 +56,8 @@ impl DriverMappings { struct State { active: usize, + /// Monotonic time the VM last had no accepted job (0 = never had one). + quiet_since: u64, commits: usize, blocked: bool, quarantined: bool, @@ -76,13 +85,15 @@ impl VmLifetime { state.active == 0 && state.commits == 0 && !state.blocked && !state.quarantined && !state.draining && !state.closed && !state.cleanup_failed + && (state.quiet_since == 0 + || now_ns().saturating_sub(state.quiet_since) >= RECLAIM_QUIET_NS) }) } pub(super) fn new() -> Result> { Arc::pin_init( pin_init!(Self { - state <- new_mutex!(State { active:0, commits:0, blocked:false, + state <- new_mutex!(State { active:0, quiet_since:0, commits:0, blocked:false, quarantined:false, draining:false, closed:false, cleanup_failed:false, close_ranges:None, unmaps:KVec::new(), objects:KVec::new() }, "VM job lifetime"), changed <- new_condvar!("VM mapping admission"), @@ -569,6 +580,7 @@ impl Drop for VmJobGuard { if state.active != 0 { return; } + state.quiet_since = now_ns(); state.draining = true; drop(state); self.vm.drain_mappings(lifetime); diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 343da90957602..d944d168a14ee 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -9,12 +9,39 @@ use super::*; use kernel::drm::gem::BaseObject; use kernel::{new_condvar, sync::CondVar}; +/// Re-evicting backing that a job just had to restore only repeats the +/// restoration. After a restore, reclaim of this VM pauses for a backoff +/// proportional to what the restore cost, within these bounds. +const RESTORE_BACKOFF_MIN_NS: u64 = 1_000_000_000; +const RESTORE_BACKOFF_MAX_NS: u64 = 30_000_000_000; +const RESTORE_BACKOFF_FACTOR: u64 = 16; + +fn now_ns() -> u64 { + ::ktime_get() as u64 +} + #[derive(Default)] struct State { users: usize, busy: bool, evicted: bool, closed: bool, + /// Monotonic time the last restoration finished and its duration. + restored_at: u64, + restore_cost: u64, +} + +impl State { + fn in_restore_backoff(&self, now: u64) -> bool { + if self.restored_at == 0 { + return false; + } + let backoff = self + .restore_cost + .saturating_mul(RESTORE_BACKOFF_FACTOR) + .clamp(RESTORE_BACKOFF_MIN_NS, RESTORE_BACKOFF_MAX_NS); + now.saturating_sub(self.restored_at) < backoff + } } #[pin_data] @@ -76,11 +103,15 @@ impl Gate { // No new reader can enter until every retained mapping is restored. state.busy = true; drop(state); + let started = now_ns(); let result = restore(); let mut state = gate.state.lock(); state.busy = false; if result.is_ok() { state.evicted = false; + let now = now_ns(); + state.restored_at = now; + state.restore_cost = now.saturating_sub(started); } drop(state); gate.ready.notify_all(); @@ -123,7 +154,7 @@ impl Gate { /// The busy gate continues to exclude restoration and metadata changes. pub(crate) fn try_reclaim(gate: &Arc) -> Option { let mut state = gate.state.try_lock()?; - if state.users != 0 || state.busy || state.closed { + if state.users != 0 || state.busy || state.closed || state.in_restore_backoff(now_ns()) { return None; } state.busy = true; @@ -142,9 +173,12 @@ impl Gate { } pub(crate) fn idle(&self) -> bool { - self.state - .try_lock() - .is_some_and(|state| state.users == 0 && !state.busy && !state.closed) + self.state.try_lock().is_some_and(|state| { + state.users == 0 + && !state.busy + && !state.closed + && !state.in_restore_backoff(now_ns()) + }) } /// A page-table mutation happened outside a `Reclaim` (the purge path runs From 5eb07ab2539bf7628366644c96a4742c934b9a78 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 16:26:15 -0400 Subject: [PATCH 0656/1579] drm/asahi: count bindings and VM creation as reclaim activity Idle-VM reclaim deferred a VM for 500 ms after its last accepted job ended, tracked in the job lifetime. A VM that binds memory without submitting, as a client does while loading or before its first job, was eligible at once, and the shrinker's count did not apply the deferral that its scan applied. Track the time the last residency lease (job, binding or retirement) was released, starting at the VM's creation, in the residency gate, and apply the same 500 ms deferral and restore backoff to both the count and the scan. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 1c62cd0853b670104424bcfddeee7f7d9279304e) --- drivers/gpu/drm/asahi/mmu/lifetime.rs | 14 +------------- drivers/gpu/drm/asahi/mmu/residency.rs | 23 +++++++++++++++++++---- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 79b4dca112ce9..d53c19b93bfdc 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -14,13 +14,6 @@ use kernel::{ }; const ADMISSION_WAIT_MS: u32 = 1000; -/// A VM whose last accepted job ended more recently than this is still in -/// use: evicting its backing would be restored by its next job. -const RECLAIM_QUIET_NS: u64 = 500_000_000; - -fn now_ns() -> u64 { - ::ktime_get() as u64 -} pub(super) struct DriverMappings { mappings: KVec, @@ -56,8 +49,6 @@ impl DriverMappings { struct State { active: usize, - /// Monotonic time the VM last had no accepted job (0 = never had one). - quiet_since: u64, commits: usize, blocked: bool, quarantined: bool, @@ -85,15 +76,13 @@ impl VmLifetime { state.active == 0 && state.commits == 0 && !state.blocked && !state.quarantined && !state.draining && !state.closed && !state.cleanup_failed - && (state.quiet_since == 0 - || now_ns().saturating_sub(state.quiet_since) >= RECLAIM_QUIET_NS) }) } pub(super) fn new() -> Result> { Arc::pin_init( pin_init!(Self { - state <- new_mutex!(State { active:0, quiet_since:0, commits:0, blocked:false, + state <- new_mutex!(State { active:0, commits:0, blocked:false, quarantined:false, draining:false, closed:false, cleanup_failed:false, close_ranges:None, unmaps:KVec::new(), objects:KVec::new() }, "VM job lifetime"), changed <- new_condvar!("VM mapping admission"), @@ -580,7 +569,6 @@ impl Drop for VmJobGuard { if state.active != 0 { return; } - state.quiet_since = now_ns(); state.draining = true; drop(state); self.vm.drain_mappings(lifetime); diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index d944d168a14ee..6643b27cb0c31 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -15,6 +15,10 @@ use kernel::{new_condvar, sync::CondVar}; const RESTORE_BACKOFF_MIN_NS: u64 = 1_000_000_000; const RESTORE_BACKOFF_MAX_NS: u64 = 30_000_000_000; const RESTORE_BACKOFF_FACTOR: u64 = 16; +/// A VM whose last job, binding or other residency user ended (or which was +/// created) more recently than this is in use: a submitting or loading client +/// would restore whatever reclaim released, synchronously in its next job. +const RECLAIM_QUIET_NS: u64 = 500_000_000; fn now_ns() -> u64 { ::ktime_get() as u64 @@ -29,10 +33,17 @@ struct State { /// Monotonic time the last restoration finished and its duration. restored_at: u64, restore_cost: u64, + /// Monotonic time the last residency user (job, binding, retirement) + /// released its lease; the gate's creation time before the first one. + last_active: u64, } impl State { - fn in_restore_backoff(&self, now: u64) -> bool { + /// Reclaim skips a VM that is in use, was recently used or recently restored. + fn reclaim_deferred(&self, now: u64) -> bool { + if now.saturating_sub(self.last_active) < RECLAIM_QUIET_NS { + return true; + } if self.restored_at == 0 { return false; } @@ -65,7 +76,10 @@ impl Gate { pub(crate) fn new() -> Result> { Arc::pin_init( pin_init!(Self { - state <- new_mutex!(State::default(), "Asahi residency"), + state <- new_mutex!( + State { last_active: now_ns(), ..State::default() }, + "Asahi residency" + ), ready <- new_condvar!("Asahi residency ready"), }), GFP_KERNEL, @@ -154,7 +168,7 @@ impl Gate { /// The busy gate continues to exclude restoration and metadata changes. pub(crate) fn try_reclaim(gate: &Arc) -> Option { let mut state = gate.state.try_lock()?; - if state.users != 0 || state.busy || state.closed || state.in_restore_backoff(now_ns()) { + if state.users != 0 || state.busy || state.closed || state.reclaim_deferred(now_ns()) { return None; } state.busy = true; @@ -177,7 +191,7 @@ impl Gate { state.users == 0 && !state.busy && !state.closed - && !state.in_restore_backoff(now_ns()) + && !state.reclaim_deferred(now_ns()) }) } @@ -210,6 +224,7 @@ impl Drop for Lease { let mut state = self.gate.state.lock(); debug_assert!(state.users != 0); state.users -= 1; + state.last_active = now_ns(); if state.users == 0 { drop(state); self.gate.ready.notify_all(); From 3202a8b8912f39ff7b7c06acc5fd604d5d0c4769 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 18:33:06 -0400 Subject: [PATCH 0657/1579] drm/asahi: set the idle-VM reclaim quiet window to 2000 ms Idle-VM reclaim now waits 2000 ms after the VM's last job, binding or retirement before releasing its backing (500 ms before). In five Subnautica launch captures, the restore stalls that still followed a real idle gap came after 518 ms (31 ms stall) and 1270 ms (44 ms stall); most longer submit gaps had bindings within 5 ms and are already covered by counting bindings as activity. A truly idle client is still reclaimed, 1.5 s later. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit dec3bc9eb622df05eca30716834c35506ee31f9f) --- drivers/gpu/drm/asahi/mmu/residency.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 6643b27cb0c31..31f47ee032be6 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -18,7 +18,7 @@ const RESTORE_BACKOFF_FACTOR: u64 = 16; /// A VM whose last job, binding or other residency user ended (or which was /// created) more recently than this is in use: a submitting or loading client /// would restore whatever reclaim released, synchronously in its next job. -const RECLAIM_QUIET_NS: u64 = 500_000_000; +const RECLAIM_QUIET_NS: u64 = 2000000000; fn now_ns() -> u64 { ::ktime_get() as u64 From 4963fa334b2439a0fc831ee3a5a8db008712a307 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 16:19:25 -0400 Subject: [PATCH 0658/1579] drm/asahi: release the VM status mirror object when the VM handle closes The VM status is shared with every accepted job and with each retained compute or render graph that last belonged to the VM, so it outlives the client. The client's status-mirror object (GET_PARAMS VM_STATUS_MIRROR) was retained for the status's whole lifetime, keeping a user object alive after the client exited for as long as a retained graph referenced the context. Keep the mirror word and its object under a mutex shared with the failure reporter, and drop both when the file's VM handle is destroyed; later reports find no mirror and only record the error. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 1c1abce16316282e15f5e471b8ef0b5a33acc038) --- drivers/gpu/drm/asahi/file.rs | 5 ++ drivers/gpu/drm/asahi/g17/status.rs | 82 +++++++++++++++-------------- drivers/gpu/drm/asahi/mmu.rs | 2 +- 3 files changed, 48 insertions(+), 41 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index aaf289c411c20..8286755567f90 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -71,6 +71,11 @@ impl Drop for Vm { fn drop(&mut self) { self.shrinker = None; self.vm.close_idle_reclaim(); + if let Some(status) = self.vm.status() { + // Retained graphs may keep this status alive; the client's mirror + // object must not outlive the client's VM handle. + status.release_mirror(); + } if self.vm.status().is_some() { // Tracked jobs can outlive their file's VM handle. Their last guard performs // these unmaps once no accepted work can still dereference the mappings. diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs index e5ac662a4d1aa..c4cd4b1ef0e3d 100644 --- a/drivers/gpu/drm/asahi/g17/status.rs +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -3,40 +3,40 @@ //! VM admission errors and client failure reporting. No firmware state is accessed here. use crate::gem; -use core::cell::UnsafeCell; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicPtr, Ordering}; -use kernel::prelude::*; +use core::sync::atomic::{AtomicI32, Ordering}; +use kernel::{new_mutex, prelude::*, sync::Mutex}; const MIRROR_MAX_SIZE: usize = 64 * 1024; /// A VM and its accepted jobs share this status, including after queue destruction. /// Client failure is separate from hardware admission: recovery may preserve the /// VM while failed accepted work must still be reported before signalling fences. +#[pin_data] pub(crate) struct VmStatus { error: AtomicI32, reported_error: AtomicI32, - mirror: AtomicPtr, - mirror_claimed: AtomicBool, - mirror_owner: UnsafeCell>, + /// The client's mapped status word, retained with its object only while the + /// client's VM handle exists; released by `release_mirror` at VM close. + #[pin] + mirror: Mutex>, } -// SAFETY: Only the winner of mirror_claimed writes mirror_owner. The owner is never -// read or replaced while the status lives, and its mapping is retained until drop. -// Other shared accesses use atomics, including accesses through the mirror pointer. -unsafe impl Sync for VmStatus {} -// SAFETY: The retained GEM reference and mapping can move between threads. Moving -// the status does not move that mapping, and its remaining fields are atomic. -unsafe impl Send for VmStatus {} +/// One aligned word inside the retained, vmapped mirror object. +struct Mirror { + word: *mut i32, + _owner: gem::ObjectRef, +} +// SAFETY: The word points into the mapping the retained owner keeps alive; it is +// only accessed atomically through this pointer while the mutex is held. +unsafe impl Send for Mirror {} impl VmStatus { - pub(crate) fn new() -> Self { - Self { + pub(crate) fn new() -> impl PinInit { + pin_init!(Self { error: AtomicI32::new(0), reported_error: AtomicI32::new(0), - mirror: AtomicPtr::new(core::ptr::null_mut()), - mirror_claimed: AtomicBool::new(false), - mirror_owner: UnsafeCell::new(None), - } + mirror <- new_mutex!(None, "Asahi VM status mirror"), + }) } pub(crate) fn get(&self) -> i32 { @@ -71,11 +71,11 @@ impl VmStatus { Ok(_) => error.to_errno(), Err(first) => first, }; - let word = self.mirror.load(Ordering::SeqCst); - if !word.is_null() { - // SAFETY: attach_mirror publishes only an aligned word within a mapping - // owned by this status. Every kernel access to this word is atomic. - unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); + let mirror = self.mirror.lock(); + if let Some(mirror) = mirror.as_ref() { + // SAFETY: attach_mirror stored only an aligned word within a mapping the + // retained owner keeps alive while this entry exists under the mutex. + unsafe { AtomicI32::from_ptr(mirror.word) }.store(error, Ordering::SeqCst); } } @@ -86,7 +86,7 @@ impl VmStatus { } } - /// Retain one mapped GEM word until the last reference to this status is dropped. + /// Retain one mapped GEM word until the client's VM handle is dropped. pub(crate) fn attach_mirror(&self, mut owner: gem::ObjectRef, offset: usize) -> Result { if owner.size() > MIRROR_MAX_SIZE || offset % core::mem::align_of::() != 0 @@ -99,24 +99,26 @@ impl VmStatus { let base = owner.vmap()?.as_mut_ptr(); // SAFETY: The checked range is inside the retained, page-aligned mapping. let word = unsafe { base.add(offset) }.cast::(); - if self.mirror_claimed.swap(true, Ordering::AcqRel) { + let mut mirror = self.mirror.lock(); + if mirror.is_some() { return Err(EBUSY); } - // SAFETY: This is the only thread that may write mirror_owner. It is not - // read again until the status is dropped, after all shared references end. - unsafe { *self.mirror_owner.get() = Some(owner) }; - // SAFETY: The retained owner keeps this checked, aligned word mapped. - unsafe { AtomicI32::from_ptr(word) }.store(0, Ordering::SeqCst); - - // The pointer publication and reported error form a store/load handshake: - // either this load sees the error or the reporter sees our pointer. Stores - // after publishing the pointer write only the same first error, never zero. - self.mirror.store(word, Ordering::SeqCst); + // No reporter runs while the mutex is held, so the word holds the first + // reported error (or zero) when it becomes visible. let error = self.reported_error.load(Ordering::SeqCst); - if error != 0 { - // SAFETY: The mapping remains owned by this status as above. - unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); - } + // SAFETY: The retained owner keeps this checked, aligned word mapped. + unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); + *mirror = Some(Mirror { + word, + _owner: owner, + }); Ok(()) } + + /// The client's VM handle is gone and nobody reads the mirror any more. The + /// mapping and object reference drop outside the mutex (GEM release may sleep). + pub(crate) fn release_mirror(&self) { + let taken = self.mirror.lock().take(); + drop(taken); + } } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 7db6c5fb1c0bf..1b0eab3bccf37 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1617,7 +1617,7 @@ impl Vm { /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { - self.status = Some(Arc::new(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); self.lifetime = Some(lifetime::VmLifetime::new()?); self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); From 8057982690f1f1954a83f62382873a8baf03bb8e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 20:33:37 -0400 Subject: [PATCH 0659/1579] drm/asahi: unmap detached compute owners and free vacated pools off-lock Idle retained compute queues detach their exited owner's binding and, once the firmware consumed a free-list release, drop the vacated USC backing. Both happened inside the event worker's device-mutex critical section: the binding drop unmaps seven aliases from the exited VM and releases the last reference to that context and its VM, and the vacated backing frees a 25 MiB object. On the ship stack kbench's cold phase showed the first SUBMIT of 1-4 of 48 fresh contexts at 433-741 us (p99 657-701 us) against a 170-260 us body that is faster than the previous release's 224 us median. The slow samples are spread over the run and the second submission is 15 us, which is consistent with a submission waiting for the event worker's critical section while it performs this teardown; the previous release did the same unmapping inside the new owner's own submission, so no other client could observe it. Hand the detached binding and the vacated backing to the worker's deferred batch, which already finishes completion fences after the device mutex is released. The batch reserves one slot per physical queue for each; without room the value is dropped in place as before. No behaviour changes for the firmware: the release witness, the retained scheduler pages and the re-arm path are unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6af79e6e57ef80d2b93d4c428956ec6cdfb11a3a) --- drivers/gpu/drm/asahi/g17/freelist.rs | 7 +-- drivers/gpu/drm/asahi/g17/queue/compute.rs | 51 +++++++++++--------- drivers/gpu/drm/asahi/g17/runtime.rs | 52 +++++++++++++++++---- drivers/gpu/drm/asahi/g17/runtime/events.rs | 2 +- 4 files changed, 79 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index e40dfec7fc5fd..a4d58e71eeccf 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -393,9 +393,10 @@ impl ComputePool { pub(crate) fn release_record(&self, generation: u64) -> super::fw::channels::FreeListRelease { super::fw::channels::FreeListRelease::new(generation, u32::from(self.id)) } - /// The caller has observed the firmware consume this pool's release. - pub(crate) fn drop_pages(&mut self) { - self.pages = None; + /// The caller has observed the firmware consume this pool's release and + /// frees the returned backing after releasing the device mutex. + pub(crate) fn take_pages(&mut self) -> Option { + self.pages.take() } /// Rewrites the shared control header for fresh backing. The caller then /// rewrites the page-pool descriptor row before the queue is kicked again. diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index df38a727c8d60..2cdeb82fbac38 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -51,7 +51,13 @@ static_assert!(core::mem::offset_of!(GraphLayout, items) == ITEMS); /// Aliases are destroyed before the logical root they name. Superseded bindings remain owned /// until the first exact successful publication under their replacement has retired. -struct Binding { +/// Backing of a vacated pool, freed outside the device mutex. +pub(crate) struct Vacated { + _alias: Option, + _pages: Option, +} + +pub(crate) struct Binding { _kick_alias: mmu::KernelMapping, /// Client alias of the USC pool pages; dropped when the pool is vacated and /// recreated when an owner binds a re-armed pool (it pins the backing). @@ -1046,19 +1052,19 @@ impl Queue { /// (`Context::release_execution`), and this queue is idle without an owner. /// Drop the binding (every alias into that VM) and the context, keeping only /// the scheduler page and work storage the installed queue record names. - pub(crate) fn detach_exited_owner(&mut self) -> bool { + /// The returned binding (aliases into the exited VM and the last reference + /// to its context) is dropped by the caller after the device mutex is + /// released: unmapping and VM teardown must not delay other clients. + pub(crate) fn detach_exited_owner(&mut self) -> Option { if !self.reusable() { - return false; + return None; } - let Some(binding) = self.binding.as_ref() else { - return false; - }; - if binding.context.is_current() { - return false; + if self.binding.as_ref()?.context.is_current() { + return None; } + let binding = self.binding.take()?; self.retained_owner = Some(binding.context.retain_firmware_pages()); - self.binding = None; - true + Some(binding) } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { @@ -1084,23 +1090,26 @@ impl Queue { fence(Ordering::SeqCst); Ok(()) } - /// Drops the USC backing once the firmware's control consumer passed the - /// release. Returns true when the backing was freed by this call. - pub(crate) fn observe_vacancy(&mut self, consumer: u32, producer: u32) -> Result { + /// Detaches the USC backing once the firmware's control consumer passed the + /// release. The caller frees the returned backing after releasing the + /// device mutex. + pub(crate) fn observe_vacancy( + &mut self, + consumer: u32, + producer: u32, + ) -> Result> { let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { - return Ok(false); + return Ok(None); }; if !crate::g17::freelist::control_consumed(consumer, producer, cursor)? { - return Ok(false); + return Ok(None); } - // The retained owner binding's client alias pins the backing: drop it + // The retained owner binding's client alias pins the backing: detach it // first (the owner has exited; `reusable()` proved no previous binding). - if let Some(binding) = self.binding.as_mut() { - binding.pool_alias = None; - } - self.pool.drop_pages(); + let alias = self.binding.as_mut().and_then(|binding| binding.pool_alias.take()); + let pages = self.pool.take_pages(); self.pool_phase = PoolPhase::Vacant; - Ok(true) + Ok(Some(Vacated { _alias: alias, _pages: pages })) } /// Installs fresh backing into a vacant pool; returns the descriptor row data /// the caller writes before binding an owner. diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 204a3f97b3c52..2a7c43762e6b3 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -223,7 +223,11 @@ impl Registry { } /// Frees the backing of every pool whose release the firmware consumed. - pub(super) fn observe_pool_releases(&mut self, init: &super::initdata::InitData) -> Result { + pub(super) fn observe_pool_releases( + &mut self, + init: &super::initdata::InitData, + deferred: &mut DeferredBatch, + ) -> Result { fence(Ordering::Acquire); let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); @@ -234,7 +238,8 @@ impl Registry { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - if queue.observe_vacancy(consumer, producer)? { + if let Some(vacated) = queue.observe_vacancy(consumer, producer)? { + deferred.defer_vacated(vacated); freed += 1; } } @@ -632,6 +637,10 @@ impl Backend { /// Collection failure leaves the engine's Active owner intact for a stopped retry. pub(super) struct DeferredBatch { values: KVVec, + /// Exited owners detached from idle retained compute queues and the backing + /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. + bindings: KVVec, + vacated: KVVec, } impl DeferredBatch { const DEPTH: usize = if crate::hw::t8140::queues::RENDER_DEPTH as usize @@ -644,18 +653,32 @@ impl DeferredBatch { /// Reused by the sole event work item; no allocation on a completion pass. fn worker() -> Result { - Self::with_capacity(2 * QIDS * Self::DEPTH) + Self::with_capacity(2 * QIDS * Self::DEPTH, QIDS) } /// One publication can quarantine only its selected physical queue/pair. /// VM-wide settlement is a separate transaction before publishing its error. fn publication() -> Result { - Self::with_capacity(2 * Self::DEPTH) + Self::with_capacity(2 * Self::DEPTH, 0) } - fn with_capacity(capacity: usize) -> Result { + fn with_capacity(capacity: usize, detached: usize) -> Result { Ok(Self { values: KVVec::with_capacity(capacity, GFP_KERNEL)?, + bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, + vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, }) } + /// Keeps the detached owner's unmapping and VM teardown out of the device + /// mutex. Without reserved room the binding is dropped here instead. + pub(super) fn defer_binding(&mut self, binding: super::queue::compute::Binding) { + if let Err(binding) = self.bindings.push_within_capacity(binding) { + drop(binding); + } + } + pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { + if let Err(vacated) = self.vacated.push_within_capacity(vacated) { + drop(vacated); + } + } /// A recovery pass can fail and retire each retained command at most once. /// Reserve before changing queues, plus a full publication for its caller. /// Healthy submissions keep the small publication collector unchanged. @@ -693,6 +716,12 @@ impl DeferredBatch { for value in self.values.drain_all() { value.finish(); } + for binding in self.bindings.drain_all() { + drop(binding); + } + for vacated in self.vacated.drain_all() { + drop(vacated); + } } } @@ -1080,7 +1109,12 @@ const COMPUTE_WARM_CAP: usize = 4; impl super::Firmware { /// Event-worker service step: witness consumed pool releases, then release the /// oldest idle retained pools beyond the warm cap. - pub(in crate::g17) fn service_compute_pools(&mut self) -> Result { + /// Detached owner bindings and vacated pool backing go to `deferred`, which + /// the worker finishes after releasing the device mutex. + pub(in crate::g17) fn service_compute_pools( + &mut self, + deferred: &mut DeferredBatch, + ) -> Result { for queue in self .queues .compute @@ -1088,9 +1122,11 @@ impl super::Firmware { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - queue.detach_exited_owner(); + if let Some(binding) = queue.detach_exited_owner() { + deferred.defer_binding(binding); + } } - self.queues.observe_pool_releases(&self.init)?; + self.queues.observe_pool_releases(&self.init, deferred)?; if self.render_control_backpressured() { return Ok(()); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index f92c493f4813d..4a18aaa05419d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -386,7 +386,7 @@ impl crate::g17::Shared { let idle = firmware.idle_check(now, in_flight)?; firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; - firmware.service_compute_pools()?; + firmware.service_compute_pools(deferred)?; { let mut reclaim = self.reclaim.lock(); firmware.service_teardown(&mut reclaim)?; From 727284277d335215edd48b5f74a6f4d4e949ff3a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 20:08:23 -0400 Subject: [PATCH 0660/1579] arm64: dts: apple: t8140: describe the GPU mailboxes as ASCWrap v6 The GPU firmware mailboxes were described as "apple,asc-mailbox-v4" windows at 0x482608000 and 0x482e08000. With these nodes the mailbox driver binds its v4 variant, is_ascwrap_v6 stays false, and the ASCWrap v6 CPU lifecycle helpers the GPU driver calls (get_lifecycle, require_safe_cpu_lifecycle, start_cpu, stop_cpu) return -EOPNOTSUPP. The device tree J700 boots with describes both mailboxes as "apple,t8140-ascwrap-v6": the 0x88000 wrapper at 0x482600000 (0x482e00000 for GFX1) plus the 8-byte IOP vector base at 0x482050000 (0x482850000), the send-empty and recv-not-empty interrupts (993/996, 997/1000), and the firmware role. Use the same description here, and give the GPU node the same "asc" (0x88000 wrapper) and "sgx" regions; the driver maps only those two, so the unused "asc1" entry is dropped. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 9c10d7198c8e56b16da35914ffde0eecb6bfb525) --- arch/arm64/boot/dts/apple/t8140.dtsi | 33 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 29e8ad3a19ef0..4dbec938def51 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -803,10 +803,9 @@ gpu: gpu@482600000 { compatible = "apple,agx-t8140"; - reg = <0x4 0x82600000 0x0 0x4000>, - <0x4 0x82e00000 0x0 0x4000>, + reg = <0x4 0x82600000 0x0 0x88000>, <0x4 0x80000000 0x0 0x4000000>; - reg-names = "asc", "asc1", "sgx"; + reg-names = "asc", "sgx"; mboxes = <&gfx_mbox>, <&gfx1_mbox>; power-domains = <&ps_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, @@ -816,31 +815,31 @@ status = "disabled"; }; - gfx_mbox: mailbox@482608000 { - compatible = "apple,t8140-gfx-asc-mailbox", "apple,asc-mailbox-v4"; - reg = <0x4 0x82608000 0x0 0x4000>; + gfx_mbox: mailbox@482600000 { + compatible = "apple,t8140-ascwrap-v6"; + reg = <0x4 0x82600000 0x0 0x88000>, + <0x4 0x82050000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; interrupt-parent = <&aic>; interrupts = , - , - , ; - interrupt-names = "send-empty", "send-not-empty", - "recv-empty", "recv-not-empty"; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; power-domains = <&ps_gfx>; #mbox-cells = <0>; status = "disabled"; }; - gfx1_mbox: mailbox@482e08000 { - compatible = "apple,t8140-gfx-asc-mailbox", "apple,asc-mailbox-v4"; - reg = <0x4 0x82e08000 0x0 0x4000>; + gfx1_mbox: mailbox@482e00000 { + compatible = "apple,t8140-ascwrap-v6"; + reg = <0x4 0x82e00000 0x0 0x88000>, + <0x4 0x82850000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; interrupt-parent = <&aic>; interrupts = , - , - , ; - interrupt-names = "send-empty", "send-not-empty", - "recv-empty", "recv-not-empty"; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX1"; power-domains = <&ps_gfx>; #mbox-cells = <0>; status = "disabled"; From 6f6bef1b9fd7c46f0885eea0ece01a811e08c7d1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 02:40:32 -0400 Subject: [PATCH 0661/1579] drm/asahi: retire a compute command on its record when its stamp never follows A G17 compute command retires once its queue's completion stamp covers its kick and the firmware has consumed its items. On J700 (A18 Pro), a client that keeps render and compute work of one VM in flight together loses its device within seconds: the command's completion record arrives with its start and end timestamps and its items are consumed, but the firmware never stores its completion stamp, so the compute watchdog fails the VM. The same client runs clean on an earlier driver for this GPU, which retires such a command on its completion record and waits for the stamp only a bounded time (500 us); that driver observed that the firmware leaves a minority of completion stamps unpublished. Do the same: once a command's exact completion record (non-zero duration) has been present for 1 ms and its items are consumed, retire it even if its stamp has not covered it. The firmware stores the stamp after the cache maintenance that follows the record, so the bounded wait still lets a stamp that is merely late keep that ordering. Count the commands that retire without a stamp and log the first and every 64th, so how often it happens stays visible. With this change that client ran 575,882 verified checks in 60 s; on the same boot without it, the device was lost within 2 s. Why the firmware leaves these stamps unstored is not yet understood. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 0b4bd1537cc71a1796b976c05cdab24e76549303) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 16 +++++++++- .../drm/asahi/g17/queue/compute/retirement.rs | 30 +++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 2cdeb82fbac38..f956e5726a8d6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -325,6 +325,8 @@ struct Active { work: Arc, ticket: Option, observed: Option, + /// Host time the completion record was first observed. + observed_at: Option, failure_deferred: bool, qos_pending: bool, } @@ -589,6 +591,9 @@ impl Queue { }; if active.observed.is_none() { active.observed = batch.take(&ticket)?; + if active.observed.is_some() { + active.observed_at = Some(now_ns()); + } } } Ok(()) @@ -699,15 +704,23 @@ impl Queue { .iter() .skip(1) .find_map(|a| a.observed.map(|o| o.timestamps[1])); + let stamp = self.graph.stamp()?; + let record_aged = front + .observed_at + .is_some_and(|at| now_ns().saturating_sub(at) >= retirement::STAMP_GRACE_NS); let timestamps = retirement::poll( &ticket, front.observed, self.graph.cursors()?, - self.graph.stamp()?, + stamp, recordless, self.last_end, next_end, + record_aged, )?; + if timestamps.is_some() && !retirement::stamp_covers(stamp, ticket.kick) { + retirement::note_stampless(self.qid(), ticket.kick); + } if timestamps.is_some() && self.previous.as_ref().is_some_and(|p| { p.publication != Some((front.packet.order.sequence, ticket.epoch)) @@ -1488,6 +1501,7 @@ impl Queue { work: work.clone(), ticket: None, observed: None, + observed_at: None, failure_deferred: false, qos_pending: false, }) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 766177ebc32c9..b4d4bcef24421 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -65,8 +65,32 @@ pub(crate) enum Recordless { CrossedRecovery, } +/// How long a command whose exact completion record is present waits for its completion stamp +/// before it retires on the record alone. The firmware stores the stamp after the cache +/// maintenance that follows the record; it occasionally never stores it for a command, most +/// visibly while a render of the same VM is running. +pub(crate) const STAMP_GRACE_NS: u64 = 1_000_000; + +static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + +/// Counts a command that retired on its record without its completion stamp; logs the first and +/// every 64th. +pub(crate) fn note_stampless(qid: u8, kick: u64) { + let count = STAMPLESS.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + 1; + if count == 1 || count % 64 == 0 { + pr_info!( + "G17: {} compute commands retired on their completion record without a completion stamp (latest: queue {} kick {:#x})\n", + count, + qid, + kick + ); + } +} + /// A zero-duration record fails before visibility polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. +/// The command is complete when its stamp covers it, or when its exact completion record has +/// been present for [`STAMP_GRACE_NS`] (`record_aged`); in both cases its items must be consumed. pub(crate) fn poll( ticket: &Ticket, observed: Option, @@ -75,13 +99,13 @@ pub(crate) fn poll( recordless: Recordless, floor: u64, next_end: Option, + record_aged: bool, ) -> Result> { if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { return Err(ENODATA); } - if !stamp_covers(stamp, ticket.kick) - || !cursors.consumed(ticket.item_producer, ticket.item_count)? - { + let completed = stamp_covers(stamp, ticket.kick) || (observed.is_some() && record_aged); + if !completed || !cursors.consumed(ticket.item_producer, ticket.item_count)? { return Ok(None); } Ok(Some(match observed { From e978503f00475864733aefc433d758f0254f401f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 00:18:28 -0400 Subject: [PATCH 0662/1579] drm/asahi: log the first GPU work failure reported to each G17 VM A G17 client treats any failure reported for its work (the VM status word or its mapped mirror) as a lost device, but no path that reports one writes to the kernel log: watchdog timeouts, quarantines, cancelled or failed retirements and recovery classification all fail the client silently. On Neo a client lost its device repeatedly with no kernel line at all, so the failing path could not be identified from the log. Log one warning when the first failure is reported to a VM, with the error and the reporting site. Later failures of the same VM are not logged, so the volume is bounded by the number of failing clients. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f647d69cc5043bf1d3389c8e250e78935e5afa24) --- drivers/gpu/drm/asahi/g17/status.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs index c4cd4b1ef0e3d..92192d7e20d56 100644 --- a/drivers/gpu/drm/asahi/g17/status.rs +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -44,6 +44,7 @@ impl VmStatus { } /// Record permanent admission failure and report it to the client. + #[track_caller] pub(crate) fn record(&self, error: Error) { self.report_failure(error); let _ = self.error.compare_exchange( @@ -61,6 +62,10 @@ impl VmStatus { /// Report failed accepted work without preventing recovery of the VM. /// Every reporter publishes the first error before returning: a losing /// reporter may signal its fence before the winning reporter resumes. + /// + /// The first failure reported to a VM is logged once, with the reporting site: the client treats + /// any reported failure as a lost device, and nothing else in the kernel log names it. + #[track_caller] pub(crate) fn report_failure(&self, error: Error) { let error = match self.reported_error.compare_exchange( 0, @@ -68,7 +73,16 @@ impl VmStatus { Ordering::SeqCst, Ordering::SeqCst, ) { - Ok(_) => error.to_errno(), + Ok(_) => { + let at = core::panic::Location::caller(); + pr_warn!( + "GPU work of a client failed ({:?}, reported at {}:{}); the client sees a lost device\n", + error, + at.file().rsplit("asahi/").next().unwrap_or(at.file()), + at.line() + ); + error.to_errno() + } Err(first) => first, }; let mirror = self.mirror.lock(); @@ -80,6 +94,7 @@ impl VmStatus { } /// Rejected submissions and contention leave the VM usable. + #[track_caller] pub(crate) fn record_if_device_loss(&self, error: Error) { if error == EIO || error == ETIMEDOUT || error == ENODEV { self.record(error); From 85ab080b193fa481276ebcb9393a93b25024384f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 00:27:46 -0400 Subject: [PATCH 0663/1579] drm/asahi: name the completion path in the first-failure log line The first-failure warning names the code that reported the failure. For most failures that is one of three completion helpers (failure before publication, failure while the queue owns the command, failed retirement), which tells little. Let the reporting site propagate through those helpers, so the line names their caller: for example the publication path for a command that failed before it reached the firmware, which is where a failed render pool re-allocation (ENOMEM) is reported. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c8ea00a9ed621e2e5c7c84461091fda3a9b7cb7b) --- drivers/gpu/drm/asahi/g17/completion.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 7308fde83d568..8338f8984d2d1 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -433,6 +433,7 @@ impl Completion { /// Exact retirement permits the physical lease and VM pin to be released /// before waking userspace. A failure fence alone never establishes this. + #[track_caller] pub(crate) fn complete(&self, result: Result<[u64; 4]>) { if self.signalled.swap(true, Ordering::AcqRel) { return; @@ -453,6 +454,7 @@ impl Completion { } /// A rejected, never-published command cannot poison the VM for contention. + #[track_caller] pub(crate) fn fail_unpublished(&self, error: Error) { if self.signalled.swap(true, Ordering::AcqRel) { return; @@ -468,6 +470,7 @@ impl Completion { /// The physical queue must already be quarantined. Keep mappings pinned while /// waking waiters, even if queue destruction requested cancellation. + #[track_caller] pub(crate) fn fail_while_owned(&self, error: Error) { if self.signalled.swap(true, Ordering::AcqRel) { return; From 66631bc632956bb7dfd1b4fb44e48ae90a10140e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:12:31 -0400 Subject: [PATCH 0664/1579] drm/asahi: describe the G17 word tables without their generation history The comment on the constant word tables recorded how they were produced and named an out-of-tree workspace. State what the tables are instead. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d0aadc1e99e6f11a54bbca5ee685d7c015866789) --- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 8dbb6a81bcd47..94b03ebe284db 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -1533,8 +1533,7 @@ const TEMP_KP: u32 = 0x41a2_b852; const TEMP_KI: u32 = 0x4116_0419; /// Nonzero 32-bit words of a firmware object the host builds from constants: (byte offset, value, repeat). -/// Every other word is zero. Generated from the previously field-by-field builders; the fwdump harness -/// (`neo-smooth-20261007/kernel/fwdump`) verifies the objects byte for byte. +/// Every other word is zero. The table is the sole definition of the object's constant contents. type Words = &'static [(usize, u32, usize)]; fn fill_words(bytes: &mut [u8], words: Words) { From ec743d4d34f89fa2ea639801062c957f807c54aa Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:13:52 -0400 Subject: [PATCH 0665/1579] drm/asahi: use one monotonic clock helper across the G17 runtime The runtime registry and the retained compute queues each defined an identical now_ns() over the monotonic clock source. Define it once in the G17 module; the runtime's submodules keep their existing path through a re-export. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5d2a444327afc38543beb78a252e23661e4d4ecc) --- drivers/gpu/drm/asahi/g17/mod.rs | 6 ++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 4 +--- drivers/gpu/drm/asahi/g17/runtime.rs | 5 +---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 60c963b94a5a4..798eaa3a360aa 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -104,6 +104,12 @@ use object::{ }; /// Expected identification of a G17 GPU. +/// One monotonic domain for recovery grace, idle effort, pool retention and +/// control-ring retirement. +pub(crate) fn now_ns() -> u64 { + ::ktime_get() as u64 +} + pub(crate) struct Identity { /// GPU family field of the ID register. pub(crate) family: u8, diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index f956e5726a8d6..0cb231f7050e4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -17,9 +17,7 @@ use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; use retirement::{Cursors, RECORD_SLOTS}; -fn now_ns() -> u64 { - ::ktime_get() as u64 -} +use crate::g17::now_ns; const DESCRIPTORS: usize = 256; const GRAPH_SIZE: usize = 0x8000; diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 2a7c43762e6b3..6cefa9509de97 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -17,6 +17,7 @@ mod feed; mod render; pub(super) mod teardown; +pub(super) use super::now_ns; use super::{ buffer::{BufferIds, MetricsIds}, context::{Context, QosIds}, @@ -1139,7 +1140,3 @@ impl super::Firmware { } } -/// One monotonic domain for recovery grace, idle effort and control retirement. -fn now_ns() -> u64 { - ::ktime_get() as u64 -} From 388a083cca9a1ff7b98f14dc0e598a63494e7d89 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:13:52 -0400 Subject: [PATCH 0666/1579] drm/asahi: grow the scheduler-release table silently Growing the table of contexts awaiting a scheduler release past its preallocated 128 entries is ordinary behaviour under a burst of closes; the one-shot log line that reported it informed nobody and the table still grows the same way. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6850c91310a913b5ff9646261250e62b129f4ce8) --- drivers/gpu/drm/asahi/g17/teardown.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 5d56931879f41..1a25cd7e3fa3b 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -5,12 +5,11 @@ //! The device mutex serializes this state with the Registry's displaced graph ownership. use super::{context::Context, freelist::control_consumed, fw::channels::SchedulerStateRelease}; -use core::sync::atomic::{fence, AtomicBool, Ordering}; +use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; /// Entries preallocated at device start; a burst of closes beyond it grows the table. const CAPACITY: usize = 128; -static GREW: AtomicBool = AtomicBool::new(false); const POLL_NS: u64 = 2_000_000_000; #[derive(Clone, Copy, PartialEq, Eq)] @@ -91,12 +90,6 @@ impl Pending { armed: now, announce_pending: false, }; - if self.entries.len() >= CAPACITY && !GREW.swap(true, Ordering::Relaxed) { - pr_info!( - "asahi: G17 scheduler-release table grew past {} pending contexts\n", - CAPACITY - ); - } self.entries.push(entry, GFP_KERNEL)?; Ok(()) } From 0c8194353943469158f8651c5ec51723e130afa3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 16:15:36 -0400 Subject: [PATCH 0667/1579] drm/asahi: give back the descriptor ring of vacated compute queues A vacated compute queue (USC pool given back) still owns its 4 MiB descriptor ring. Once the queue is idle without an owner, the firmware's job list and context item only name consumed descriptors, so drop the ring and its aliases together with the pool backing, and allocate a fresh ring when an owner re-arms the queue, before the bind that aliases it into the client VM. Submissions to a vacant queue report EAGAIN until the re-arm, as for the pool. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e60f49eede0ecf38cb206c4f59ff832b2aacc1d6) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 112 ++++++++++++++++----- drivers/gpu/drm/asahi/g17/runtime.rs | 6 +- 2 files changed, 90 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 0cb231f7050e4..3db3be2211eb7 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -49,10 +49,13 @@ static_assert!(core::mem::offset_of!(GraphLayout, items) == ITEMS); /// Aliases are destroyed before the logical root they name. Superseded bindings remain owned /// until the first exact successful publication under their replacement has retired. -/// Backing of a vacated pool, freed outside the device mutex. +/// Backing of a vacated pool and descriptor ring, freed outside the device +/// mutex. Field order drops each client alias before the storage it names. pub(crate) struct Vacated { _alias: Option, _pages: Option, + _descriptors: Option, + _ring: Option, } pub(crate) struct Binding { @@ -60,7 +63,9 @@ pub(crate) struct Binding { /// Client alias of the USC pool pages; dropped when the pool is vacated and /// recreated when an owner binds a re-armed pool (it pins the backing). pool_alias: Option, - _descriptors: mmu::KernelMapping, + /// Client alias of the descriptor ring; dropped with the ring when the queue + /// is vacated and recreated when an owner binds a re-armed queue. + descriptors: Option, preempt: mmu::KernelMapping, usage: mmu::KernelMapping, operand: mmu::KernelMapping, @@ -68,10 +73,34 @@ pub(crate) struct Binding { context: Arc, } +/// Descriptor ring of one compute QID with its kernel-VM alias. Built without +/// the device mutex; installed into a vacant graph before an owner binds. +pub(crate) struct DescriptorRing { + object: KernelObject, + high: mmu::KernelMapping, +} + +impl DescriptorRing { + pub(crate) fn new(alloc: &Allocator<'_>) -> Result { + let page = mmu::UAT_PGSZ as u64; + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let object = alloc.lower( + DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, + page, + prot, + CpuMap::WriteCombined, + )?; + let mut high = alloc.uat.geometry().kernel_range(); + high.start += dynamic::KERNEL_OFFSET; + let high = object.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; + Ok(Self { object, high }) + } +} + /// Firmware-shared storage of one compute QID, separate from its common kick ring. +/// `ring` is `None` while the queue is vacant (idle, no owner, pool given back). struct Graph { - descriptor_high: mmu::KernelMapping, - descriptors: KernelObject, + ring: Option, preempt: KernelObject, usage: KernelObject, operand: KernelObject, @@ -89,10 +118,7 @@ impl Graph { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; - let descriptors = alloc.lower(DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, page, prot, cpu)?; - let mut high = alloc.uat.geometry().kernel_range(); - high.start += dynamic::KERNEL_OFFSET; - let descriptor_high = descriptors.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; + let ring = DescriptorRing::new(alloc)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; let operand = alloc.lower(0x14000, page, prot, cpu)?; @@ -120,8 +146,7 @@ impl Graph { let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; Ok(Self { - descriptor_high, - descriptors, + ring: Some(ring), preempt, usage, operand, @@ -147,9 +172,7 @@ impl Graph { } let vm = context.vm(); vm.install_job_context_aliases()?; - let descriptors = - self.descriptors - .map_alias(vm, self.descriptors.gpu_va(), mmu::PROT_GPU_SHARED_RO)?; + let descriptors = self.descriptor_alias(vm)?; let prot = mmu::PROT_GPU_FW_SHARED_RW; let page = mmu::UAT_PGSZ as u64; let preempt = self @@ -165,7 +188,7 @@ impl Graph { Ok(Binding { _kick_alias: kick_alias, pool_alias: Some(pool_alias), - _descriptors: descriptors, + descriptors: Some(descriptors), preempt, usage, operand, @@ -203,12 +226,29 @@ impl Graph { fence(Ordering::Acquire); Ok(self.support.word(STAMP)?.load(Ordering::Relaxed)) } - fn descriptor_vas(&self, slot: u8) -> (u64, u64) { + /// EAGAIN while the queue is vacant: the owner must re-arm it first. + fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { + let ring = self.ring.as_ref().ok_or(EAGAIN)?; let offset = u64::from(slot) * COMPUTE_DESCRIPTOR_SIZE as u64; - ( - self.descriptor_high.iova() + offset, - self.descriptors.gpu_va() + offset, - ) + Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) + } + fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { + let ring = self.ring.as_ref().ok_or(EAGAIN)?; + ring.object + .map_alias(vm, ring.object.gpu_va(), mmu::PROT_GPU_SHARED_RO) + } + /// Detaches the descriptor ring of an idle vacant queue; the caller frees it + /// off-lock. The job list and the context item only name consumed + /// descriptors at this point. + fn vacate_ring(&mut self) -> Option { + self.ring.take() + } + fn rearm_ring(&mut self, ring: DescriptorRing) -> Result { + if self.ring.is_some() { + return Err(EBUSY); + } + self.ring = Some(ring); + Ok(()) } fn ensure_record_slot(&self, slot: usize) -> Result { if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { @@ -1000,6 +1040,9 @@ impl Queue { // The same VM takes back a re-armed pool: alias the new backing. binding.pool_alias = Some(self.pool.map_client(context.vm())?); } + if binding.descriptors.is_none() { + binding.descriptors = Some(self.graph.descriptor_alias(context.vm())?); + } self.graph.set_owner(&context)?; binding.context = context.clone(); self.previous = Some(Previous { @@ -1119,15 +1162,27 @@ impl Queue { // first (the owner has exited; `reusable()` proved no previous binding). let alias = self.binding.as_mut().and_then(|binding| binding.pool_alias.take()); let pages = self.pool.take_pages(); + let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); + let ring = self.graph.vacate_ring(); self.pool_phase = PoolPhase::Vacant; - Ok(Some(Vacated { _alias: alias, _pages: pages })) + Ok(Some(Vacated { + _alias: alias, + _pages: pages, + _descriptors: descriptors, + _ring: ring, + })) } /// Installs fresh backing into a vacant pool; returns the descriptor row data /// the caller writes before binding an owner. - pub(crate) fn rearm(&mut self, pages: crate::g17::freelist::Pages) -> Result<(u16, u64)> { + pub(crate) fn rearm( + &mut self, + pages: crate::g17::freelist::Pages, + ring: DescriptorRing, + ) -> Result<(u16, u64)> { if self.pool_phase != PoolPhase::Vacant || !self.reusable() { return Err(EAGAIN); } + self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; Ok((self.pool.id(), self.pool.page_list_va()?)) @@ -1405,7 +1460,7 @@ impl Queue { let qid = self.qid(); let ordinal = self.ordinal.checked_add(1).ok_or(EOVERFLOW)?; let (descriptor_high, descriptor_low) = - self.graph.descriptor_vas(kick_timestamp.slot() as u8); + self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; let first = !self.installed && self.ordinal == 0; let entry_signal = first || dependencies @@ -1510,10 +1565,15 @@ impl Queue { let result = (|| -> Result { work.add_submitted_kicks(1)?; accounted = true; - self.graph.descriptors.initialize::( - kick_timestamp.slot() * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write(&args), - )?; + self.graph + .ring + .as_mut() + .ok_or(EAGAIN)? + .object + .initialize::( + kick_timestamp.slot() * COMPUTE_DESCRIPTOR_SIZE, + |descriptor| descriptor.write(&args), + )?; if first { self.graph.context_high.write( 0x200, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 6cefa9509de97..8206df716b6b3 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -153,6 +153,7 @@ impl Registry { &mut self, qid: u8, pages: crate::g17::freelist::Pages, + ring: compute::DescriptorRing, ) -> Result<(u16, u64)> { let queue = self .compute @@ -161,7 +162,7 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid) .ok_or(EAGAIN)?; - queue.rearm(pages) + queue.rearm(pages, ring) } pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { @@ -563,12 +564,13 @@ impl Backend { // Fresh zeroed backing is built without the device mutex; the // queue may have been taken or re-armed by another owner since. let pages = crate::g17::freelist::Pages::compute(&alloc)?; + let ring = compute::DescriptorRing::new(&alloc)?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { continue; } - let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages) { + let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages, ring) { Err(error) if error == EAGAIN => continue, result => result?, }; From 75535b36cc0ea75d394b20e11b993bf22e1efe20 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 18:04:27 -0400 Subject: [PATCH 0668/1579] drm/asahi: return killed compute queues to service with fresh backing A compute queue whose closed owner was killed stayed quarantined with its exit transaction retained for the device's lifetime, so every fault or kill with work in flight consumed one of the finite compute QIDs for good until the device ran out and queue creation failed. The firmware acknowledges each kill (ContextKilled) and the driver settles every killed command before marking the transaction released; use that as the witness: reset the item ring to empty at the firmware's consumer cursor, clear the quarantine and exit state, and treat the queue as a released idle queue that must give its USC pool and descriptor ring back and be re-armed fresh before the next owner binds it. Recycled queues are vacated regardless of the warm cap. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5d630414e13f4e017628430820fff73a86e97de9) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 79 ++++++++++++++++++- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 7 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 16 +++- 3 files changed, 97 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 3db3be2211eb7..57179437f9b40 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -260,6 +260,23 @@ impl Graph { Ok(()) } + /// A killed queue's ring restarts empty at the firmware's consumer cursor: every + /// item slot is cleared and the host producer is set to the consumer. Only + /// valid after the firmware acknowledged the kill and no command is pending. + fn reset_items(&mut self) -> Result { + let cursors = self.cursors()?; + if cursors.count != ITEM_RING_ENTRIES || cursors.consumer >= cursors.count { + return Err(EIO); + } + for index in 0..ITEM_RING_ENTRIES as usize { + self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + self.graph.word(POINTERS + 0x40)?.store(cursors.consumer, Ordering::Relaxed); + self.reclaimed = cursors.consumer; + fence(Ordering::SeqCst); + Ok(()) + } /// Reclaim only consumed item pointers, with individual aligned stores and a release fence. /// This leaves records still named by firmware untouched across arbitrarily many wraps. fn recycle(&mut self, cursors: Cursors) -> Result { @@ -431,6 +448,9 @@ pub(crate) struct Queue { /// the installed queue record still names them until the next owner binds. retained_owner: Option, pool_phase: PoolPhase, + /// A killed queue was recycled: its pool/ring must be vacated and re-armed + /// fresh before any owner binds it (set by `recycle_killed`, cleared by `rearm`). + needs_fresh_backing: bool, /// Monotonic time the last owner released this queue (vacate order). released_at: u64, /// Free-list generation of the last published configuration. @@ -486,6 +506,7 @@ impl Queue { owner: Some(owner), retained_owner: None, pool_phase: PoolPhase::Populated, + needs_fresh_backing: false, released_at: 0, free_list_generation: 0, binding: Some(binding), @@ -1012,6 +1033,8 @@ impl Queue { self.installed // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) + // A recycled killed queue is offered only once its backing is vacant (re-arm path). + && (!self.needs_fresh_backing || self.pool_phase == PoolPhase::Vacant) // An issued kill still names this graph even if its commands retire // before the acknowledgement. It cannot acquire a successor owner. && self.exit.is_none() @@ -1122,7 +1145,58 @@ impl Queue { } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { - self.reusable() && self.pool_phase == PoolPhase::Populated && self.pool.populated() + (self.reusable() || self.recycled_pending()) && self.pool_phase == PoolPhase::Populated && self.pool.populated() + } + /// A recycled killed queue still holding the backing its killed work used. + pub(crate) fn recycled_pending(&self) -> bool { + self.needs_fresh_backing + && self.installed + && self.exit.is_none() + && self.owner.is_none() + && !self.quarantined + && self.active.len == 0 + && self.replays.len == 0 + && self.previous.is_none() + } + /// Recycled queues are vacated regardless of the warm cap. + pub(crate) fn needs_fresh_backing(&self) -> bool { + self.needs_fresh_backing + } + /// The firmware acknowledged the kill of this queue's closed owner (`ContextKilled`) + /// and every killed command was settled: return the QID to service. The item + /// ring is reset to empty at the firmware's consumer cursor; the USC pool and + /// descriptor ring are vacated and re-armed fresh before the next owner binds. + pub(crate) fn recycle_killed(&mut self) -> Result { + if !self.installed + || !self.exit.as_ref().is_some_and(|exit| exit.released()) + || self.active.len != 0 + || self.replays.len != 0 + || self.previous.is_some() + { + return Ok(false); + } + self.graph.reset_items()?; + self.exit = None; + self.quarantined = false; + self.quarantine_error = None; + self.failure_status_pending = false; + self.retired_by_teardown = false; + self.spared_quarantine = false; + self.spared_deferred = false; + self.retire_pending = false; + self.awaiting_witness = false; + self.retirement_proved = false; + self.replay_retirement = false; + self.replay_error = None; + self.replay_prepared = false; + self.submitted = 0; + self.kick.clear_parent_after_recovery(); + self.owner = None; + self.released = true; + self.retirement_ready = true; + self.released_at = now_ns(); + self.needs_fresh_backing = true; + Ok(true) } pub(crate) fn released_at(&self) -> u64 { self.released_at @@ -1179,12 +1253,13 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if self.pool_phase != PoolPhase::Vacant || !self.reusable() { + if self.pool_phase != PoolPhase::Vacant || !(self.reusable() || self.recycled_pending()) { return Err(EAGAIN); } self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; + self.needs_fresh_backing = false; Ok((self.pool.id(), self.pool.page_list_va()?)) } /// Exact host identity remains valid even when several commands share an ioctl ID. diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 08d1013e4762b..10d2a37274800 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -75,6 +75,13 @@ pub(super) struct Transaction { announced: Option, } +impl Transaction { + /// The firmware acknowledged the kill and the killed commands were settled. + pub(super) fn released(&self) -> bool { + self.phase == Phase::Released + } +} + impl Queue { pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 8206df716b6b3..dcfdf43b63792 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -184,6 +184,13 @@ impl Registry { cap: usize, ) -> Result { // Count first: event turns are frequent and the steady state has no excess. + let forced = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.vacate_candidate() && queue.needs_fresh_backing()) + .count(); let count = self .compute .iter() @@ -191,7 +198,7 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref()) .filter(|queue| queue.vacate_candidate()) .count(); - if count <= cap { + if count <= cap && forced == 0 { return Ok(0); } let mut candidates: KVec<(u64, u8)> = KVec::with_capacity(count, GFP_KERNEL)?; @@ -202,10 +209,12 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref()) .filter(|queue| queue.vacate_candidate()) { - candidates.push((queue.released_at(), queue.qid()), GFP_KERNEL)?; + // Recycled killed queues sort first (time 0) and count beyond the cap. + let key = if queue.needs_fresh_backing() { 0 } else { queue.released_at() }; + candidates.push((key, queue.qid()), GFP_KERNEL)?; } candidates.sort_unstable(); - let excess = candidates.len() - cap; + let excess = candidates.len().saturating_sub(cap).max(forced); let mut published = 0; for (_, qid) in candidates.iter().take(excess.min(4)) { let queue = self @@ -1125,6 +1134,7 @@ impl super::Firmware { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + queue.recycle_killed()?; if let Some(binding) = queue.detach_exited_owner() { deferred.defer_binding(binding); } From 4457298833f1001a90c15d9277dee87ad6e11927 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:08:54 -0400 Subject: [PATCH 0669/1579] drm/asahi: give back the render pool backing of evicted idle clients Each logical context that has rendered keeps a render pool with about 24 MiB of pinned USC backing for the rest of its life, also while the client is idle and the machine is reclaiming memory. When pressure reclaim has evicted a client's VM (it was idle for the reclaim quiet window and has no job or binding in progress) and the firmware has consumed the pool's final free-list release, the event worker now detaches the pool's backing and both of its aliases and hands them to its deferred batch, which frees them after the device mutex is released. The pool keeps its ID, control and state objects. Before that client's next render, the submission path installs fresh backing, and the changed backing makes the next staging rewrite the control header and descriptor row, as for a new owner of the slot. A vacated pool is never staged. Clients whose VM was not evicted are unaffected; an evicted client already restores its VM before its next job. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 16822976f9d375ddbdc788d968077a15660be5d7) --- drivers/gpu/drm/asahi/g17/context.rs | 2 + drivers/gpu/drm/asahi/g17/freelist.rs | 139 +++++++++++++++--- drivers/gpu/drm/asahi/g17/runtime.rs | 18 +++ drivers/gpu/drm/asahi/g17/runtime/events.rs | 5 + drivers/gpu/drm/asahi/g17/runtime/render.rs | 31 ++++ .../gpu/drm/asahi/g17/runtime/render/pools.rs | 17 +++ .../drm/asahi/g17/runtime/render/prepare.rs | 11 +- drivers/gpu/drm/asahi/mmu.rs | 2 +- drivers/gpu/drm/asahi/mmu/residency.rs | 14 ++ 9 files changed, 218 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 7076708be9f2f..f8145b1d3b619 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -280,6 +280,8 @@ impl Context { } let mut render = self.render.lock(); if let Some(pool) = &*render { + // A pool vacated under memory pressure is re-armed before reuse. + pool.ensure_backing(alloc, render_global, self.vm())?; return Ok(pool.clone()); } let pool = Arc::new( diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index a4d58e71eeccf..c2c9470d10aa3 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -12,7 +12,11 @@ use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIS use super::object::{Allocator, CpuMap, KernelObject}; use crate::mmu; use core::sync::atomic::{fence, AtomicBool, AtomicU64, Ordering}; -use kernel::{prelude::*, sync::Arc}; +use kernel::{ + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; /// Render pools occupy 129..255; compute uses its QID plus one, in 1..128. /// First-free allocation keeps the shared firmware namespace deterministic. @@ -115,27 +119,21 @@ impl Pages { /// Pool-object identities are independent of recycled firmware IDs and VAs. static RENDER_POOL_SERIAL: AtomicU64 = AtomicU64::new(0); -/// A render pool retained by a logical context and every graph referencing it. -pub(crate) struct RenderPool { - // Remove all aliases before dropping the backing or returning the ID. +/// The USC backing of a render pool and its two aliases. Field order drops +/// both aliases before the backing. +pub(crate) struct RenderBacking { _client: mmu::KernelMapping, _global: mmu::KernelMapping, pages: Pages, - control: KernelObject, - state: KernelObject, - served: AtomicBool, - serial: u64, - id: RenderId, } -impl RenderPool { +impl RenderBacking { + /// Fresh populated backing, built without the device mutex. pub(crate) fn new( alloc: &Allocator<'_>, - ids: &Arc, render_global: &mmu::Vm, client: &mmu::Vm, ) -> Result { - let id = RenderId::new(ids)?; let pages = Pages::new(alloc, RENDER_BLOCKS, RunLayout::Render)?; let address = pages.page_list_va(); let global = pages @@ -144,6 +142,45 @@ impl RenderPool { let client = pages .backing .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; + Ok(Self { + _client: client, + _global: global, + pages, + }) + } +} + +/// A render pool retained by a logical context and every graph referencing it. +/// Its backing can be given back while the pool's free-list release has been +/// consumed and the client VM is evicted; a fresh backing is installed before +/// the pool is staged again. +pub(crate) struct RenderPool { + backing: Arc>>, + /// Page-list VA of the installed backing; zero while vacant. + page_list: AtomicU64, + /// Counts installed backings, so a re-armed backing is a new free-list + /// owner even if its addresses repeat. + epoch: AtomicU64, + /// Residency of the client VM the backing is aliased into. + client_residency: Option>, + control: KernelObject, + state: KernelObject, + served: AtomicBool, + serial: u64, + id: RenderId, +} + +impl RenderPool { + pub(crate) fn new( + alloc: &Allocator<'_>, + ids: &Arc, + render_global: &mmu::Vm, + client: &mmu::Vm, + ) -> Result { + let id = RenderId::new(ids)?; + let backing = RenderBacking::new(alloc, render_global, client)?; + let address = backing.pages.page_list_va(); + let run_list_va = backing.pages.run_list_va(); let state = alloc.kernel( CONTROL_SIZE, mmu::UAT_PGSZ as u64, @@ -163,7 +200,7 @@ impl RenderPool { buffer_slot: u32::from(id.slot), class: FreeListClass::Two, page_list_va: address, - run_list_va: pages.run_list_va(), + run_list_va, blocks: RENDER_BLOCKS as u32, state_va: state.gpu_va(), }, @@ -174,10 +211,15 @@ impl RenderPool { let serial = RENDER_POOL_SERIAL .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value.checked_add(1)) .map_err(|_| EOVERFLOW)? + 1; + let backing = Arc::pin_init( + new_mutex!(Some(backing), "G17 render pool backing"), + GFP_KERNEL, + )?; Ok(Self { - _client: client, - _global: global, - pages, + backing, + page_list: AtomicU64::new(address), + epoch: AtomicU64::new(0), + client_residency: client.residency_gate(), control, state, served: AtomicBool::new(false), @@ -186,20 +228,72 @@ impl RenderPool { }) } + /// No backing is installed: the pool must be re-armed before staging. + pub(crate) fn vacant(&self) -> bool { + self.page_list.load(Ordering::Acquire) == 0 + } + /// Pressure reclaim evicted the client VM, which is idle. + pub(crate) fn client_evicted(&self) -> bool { + self.client_residency.as_ref().is_some_and(|gate| gate.evicted()) + } + /// Removes the backing. The caller holds the device mutex and observed this + /// pool's free-list release consumed, so no firmware reference remains; the + /// next staging requires a re-armed backing. The caller drops the result + /// after releasing the device mutex. + pub(crate) fn vacate(&self) -> Option { + let backing = self.backing.lock().take(); + if backing.is_some() { + self.page_list.store(0, Ordering::Release); + } + backing + } + /// Installs fresh backing into a vacant pool, unless another caller did. + /// The next staging rewrites the control header and descriptor row for it + /// (its run-list address changes the free-list owner identity). + pub(crate) fn rearm(&self, fresh: RenderBacking) -> Result { + let mut backing = self.backing.lock(); + if backing.is_some() { + return Ok(()); + } + let address = fresh.pages.page_list_va(); + *backing = Some(fresh); + self.epoch.fetch_add(1, Ordering::AcqRel); + self.served.store(true, Ordering::Release); + self.page_list.store(address, Ordering::Release); + Ok(()) + } + /// Re-arms a vacant pool with fresh backing built here, without the device + /// mutex. `client` is the VM of the context that owns this pool. + pub(crate) fn ensure_backing( + &self, + alloc: &Allocator<'_>, + render_global: &mmu::Vm, + client: &mmu::Vm, + ) -> Result { + if !self.vacant() { + return Ok(()); + } + self.rearm(RenderBacking::new(alloc, render_global, client)?) + } + pub(crate) fn serial(&self) -> u64 { self.serial } + pub(crate) fn epoch(&self) -> u64 { + self.epoch.load(Ordering::Acquire) + } pub(crate) fn id(&self) -> u16 { self.id.slot } pub(crate) fn control_va(&self) -> u64 { self.control.gpu_va() } + /// Zero while vacant. pub(crate) fn page_list_va(&self) -> u64 { - self.pages.page_list_va() + self.page_list.load(Ordering::Acquire) } pub(crate) fn run_list_va(&self) -> u64 { - self.pages.run_list_va() + self.page_list_va() + PAGE_LIST_SIZE as u64 } pub(crate) fn end_va(&self) -> u64 { self.run_list_va() @@ -234,7 +328,12 @@ impl RenderPool { .write_unaligned(header) }; let control = self.control.pointer(0, CONTROL_SIZE)?.cast::(); - self.pages.repopulate()?; + let backing = self.backing.lock(); + let pages = &backing.as_ref().ok_or(EAGAIN)?.pages; + if pages.page_list_va() != self.page_list_va() { + return Err(EIO); + } + pages.repopulate()?; for (index, value) in image.iter().enumerate() { // SAFETY: The entire retained page was checked before any store; // the allocation is page-aligned and release excludes the firmware. @@ -462,6 +561,8 @@ impl ComputePool { pub(crate) struct Owner { /// Identifies the host pool object even when every firmware address recycles. pub(crate) serial: u64, + /// Identifies the pool's installed backing (render pools are re-armed). + pub(crate) epoch: u64, pub(crate) buffer_slot: u32, pub(crate) control_va: u64, pub(crate) ring_va: u64, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index dcfdf43b63792..842e3a8b8df9b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -653,6 +653,8 @@ pub(super) struct DeferredBatch { /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. bindings: KVVec, vacated: KVVec, + /// Backing of render pools vacated for evicted idle clients. + render: KVVec, } impl DeferredBatch { const DEPTH: usize = if crate::hw::t8140::queues::RENDER_DEPTH as usize @@ -677,6 +679,10 @@ impl DeferredBatch { values: KVVec::with_capacity(capacity, GFP_KERNEL)?, bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, + render: KVVec::with_capacity( + if detached != 0 { crate::hw::t8140::queues::RENDER_SLOTS } else { 0 }, + GFP_KERNEL, + )?, }) } /// Keeps the detached owner's unmapping and VM teardown out of the device @@ -686,6 +692,15 @@ impl DeferredBatch { drop(binding); } } + /// Room for one more vacated render backing; the worker vacates no more. + pub(super) fn render_backing_room(&self) -> bool { + self.render.len() < self.render.capacity() + } + pub(super) fn defer_render_backing(&mut self, backing: super::freelist::RenderBacking) { + if let Err(backing) = self.render.push_within_capacity(backing) { + drop(backing); + } + } pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { if let Err(vacated) = self.vacated.push_within_capacity(vacated) { drop(vacated); @@ -734,6 +749,9 @@ impl DeferredBatch { for vacated in self.vacated.drain_all() { drop(vacated); } + for backing in self.render.drain_all() { + drop(backing); + } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 4a18aaa05419d..99f3caf66bebe 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -387,6 +387,11 @@ impl crate::g17::Shared { firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; firmware.service_compute_pools(deferred)?; + if !firmware.recovery.pending() { + firmware + .queues + .vacate_evicted_render_pools(&firmware.init, deferred)?; + } { let mut reclaim = self.reclaim.lock(); firmware.service_teardown(&mut reclaim)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index bf86d46e7ca6b..aabc523e360cd 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -642,6 +642,37 @@ impl Registry { || !self.render.retired.is_empty() || !self.teardown.is_empty() } + /// Gives back the USC backing of render pools whose client VM pressure + /// reclaim evicted (an idle VM) and whose final free-list release was + /// consumed. The vacated backings go to the worker's deferred batch, which + /// frees them after the device mutex; a later job of that client re-arms + /// the pool first. + pub(in crate::g17) fn vacate_evicted_render_pools( + &mut self, + init: &crate::g17::initdata::InitData, + deferred: &mut super::DeferredBatch, + ) -> Result { + let render = &mut self.render; + render.pools.observe(init)?; + for entry in render.entries.iter().flatten() { + if !deferred.render_backing_room() { + break; + } + if entry.constructing || entry.growing || entry.preparing.is_some() { + continue; + } + let Some(pool) = entry.pool.as_ref() else { + continue; + }; + if pool.vacant() || !render.pools.idle(pool.id()) || !pool.client_evicted() { + continue; + } + if let Some(backing) = pool.vacate() { + deferred.defer_render_backing(backing); + } + } + Ok(()) + } pub(in crate::g17) fn render_in_flight(&self) -> bool { self.render.entries.iter().flatten().any(|entry| { entry.pair.as_ref().map_or( diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 7047992754b37..f7f48e853f225 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -52,6 +52,7 @@ impl Pools { fn owner(pool: &RenderPool) -> Owner { Owner { serial: pool.serial(), + epoch: pool.epoch(), buffer_slot: u32::from(pool.id()), control_va: pool.control_va(), ring_va: pool.run_list_va(), @@ -59,6 +60,15 @@ impl Pools { } } + /// The pool's registration has a consumed final release and no references: + /// the firmware holds nothing of its backing. + pub(super) fn idle(&self, id: u16) -> bool { + Self::slot(id) + .ok() + .and_then(|slot| self.leases[slot]) + .is_some_and(|lease| lease.released()) + } + pub(super) fn released(&self, id: u32, _control: u64) -> bool { self.leases .get(id as usize) @@ -68,6 +78,10 @@ impl Pools { /// Install both engine references before either inner queue becomes /// visible. Warm overlap keeps the existing generation and pool contents. pub(super) fn prepare(&mut self, init: &InitData, pool: &RenderPool) -> Result { + // A vacated pool is re-armed by the submission path before it retries. + if pool.vacant() { + return Err(EAGAIN); + } self.observe(init)?; let slot = Self::slot(pool.id())?; let owner = Self::owner(pool); @@ -92,6 +106,9 @@ impl Pools { /// Warming validates the incoming owner and stages its descriptor without /// acquiring command references or publishing an asynchronous request. pub(super) fn warm(&self, init: &InitData, pool: &RenderPool) -> Result { + if pool.vacant() { + return Err(EAGAIN); + } let previous = self.leases[Self::slot(pool.id())?]; let owner = Self::owner(pool); if previous.is_some_and(|lease| lease.owner() == owner) { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 69f8249e0818d..b4813cdba033a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -107,7 +107,16 @@ impl super::super::Backend { } else { let pair = entry.pair.as_ref().ok_or(EBUSY)?; if pair.matches(self.owner, &self.context)? { - return Ok(()); + if !pair.pool().vacant() { + return Ok(()); + } + let pool = pair.pool().clone(); + drop(state); + let alloc = Allocator { + dev: &dev, + uat: &resources.uat, + }; + return pool.ensure_backing(&alloc, &resources.global, self.context.vm()); } if !pair.reusable()? { return Err(EBUSY); diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 1b0eab3bccf37..7a95c9139207a 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -94,7 +94,7 @@ mod context; mod shared; mod lifetime; mod residency; -pub(crate) use residency::VmShrinker; +pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 31f47ee032be6..73a1d163eca82 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -195,6 +195,15 @@ impl Gate { }) } + /// Pressure reclaim released this VM's backing and no user has entered + /// since: its next job restores first. Never blocks; a contended gate + /// reads as not evicted. + pub(crate) fn evicted(&self) -> bool { + self.state + .try_lock() + .is_some_and(|state| state.evicted && !state.busy && state.users == 0) + } + /// A page-table mutation happened outside a `Reclaim` (the purge path runs /// on busy VMs). The next `enter` restores every INVALIDATED mapping. pub(crate) fn note_evicted(&self) { @@ -313,6 +322,11 @@ impl Vm { self.residency.is_some() } + /// The residency gate of a user VM, for observers that must not keep the VM. + pub(crate) fn residency_gate(&self) -> Option> { + self.residency.clone() + } + pub(super) fn enter_residency_metadata(&self) -> Result> { self.residency .as_ref() From 7e8fe19766fd1b895120a8e7bd5fe78f7aca2a7c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 15:17:39 -0400 Subject: [PATCH 0670/1579] drm/asahi: carry a scheduling profile from DRM queue priority to firmware Every queue the driver creates uses firmware priority class 2 and policy 2, written as constants into the queue record, queue configuration, kick announcement, kick header, work-ring selection, doorbells and render registration token, and the QoS row class and share are constants too. The DRM queue priority is validated at queue creation and then dropped. Give each logical queue a scheduling profile (priority class, policy word, QoS class and QoS share), fixed when its context is created, and take every one of those fields from the owning context's profile. LOW queues use priority class 3 and MEDIUM queues keep class 2; the other profile fields are unchanged because no different values are known for class 3. HIGH and REALTIME remain refused. A physical queue takes a new owner's profile only at the existing rebind boundaries, which require it to be idle: a same-VM render rebind also replaces the priority and policy words of both queue records when they differ, compute owner changes do the same in their record, and a render pair whose registration carries another class is registered again at its next publication, under the same power hold as a first install. Work slots mark the first slot naming a queue per priority class, since each class has its own work rings. With only MEDIUM queues, every record, ring, doorbell and token keeps the values and store order it had before. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit bff3853f260b02bdd9605bc6690905cb70aca29f) --- drivers/gpu/drm/asahi/g17/context.rs | 10 +- drivers/gpu/drm/asahi/g17/fw/channels.rs | 17 ++- drivers/gpu/drm/asahi/g17/fw/kick.rs | 16 ++- drivers/gpu/drm/asahi/g17/fw/queue.rs | 105 +++++++++++++++--- drivers/gpu/drm/asahi/g17/initdata.rs | 43 ++++--- drivers/gpu/drm/asahi/g17/mod.rs | 14 ++- drivers/gpu/drm/asahi/g17/qos.rs | 13 ++- drivers/gpu/drm/asahi/g17/queue/compute.rs | 50 ++++++--- drivers/gpu/drm/asahi/g17/queue/render.rs | 54 ++++++--- .../gpu/drm/asahi/g17/queue/render/memory.rs | 21 ++++ .../gpu/drm/asahi/g17/queue/render/parked.rs | 4 +- .../drm/asahi/g17/queue/render/publication.rs | 1 + drivers/gpu/drm/asahi/g17/runtime.rs | 20 +++- .../drm/asahi/g17/runtime/compute_replay.rs | 5 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 13 ++- .../drm/asahi/g17/runtime/render/submit.rs | 5 +- drivers/gpu/drm/asahi/hw/t8140.rs | 11 ++ 17 files changed, 310 insertions(+), 92 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index f8145b1d3b619..afd0d1d22f5bb 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -9,7 +9,7 @@ mod identity; use super::{ freelist::{RenderIds, RenderPool}, - fw::context as fw, + fw::{context as fw, queue::Policy}, object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, status::VmStatus, }; @@ -120,6 +120,9 @@ pub(crate) struct Context { work: Arc>, work_base: u64, render: Arc>>>, + /// Fixed for the context's life. Physical queues bound to it carry this + /// profile in every record, ring, doorbell and registration. + policy: Policy, } impl Context { pub(crate) fn new( @@ -128,6 +131,7 @@ impl Context { qos: &Arc, execution: Arc, owner_pid: u32, + policy: Policy, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; @@ -188,6 +192,7 @@ impl Context { work, work_base, render, + policy, }) } pub(crate) fn vm(&self) -> &Vm { @@ -209,6 +214,9 @@ impl Context { pub(crate) fn owner_pid(&self) -> u32 { self.owner_pid } + pub(crate) fn policy(&self) -> Policy { + self.policy + } pub(crate) fn qos_id(&self) -> u8 { self.qos.id } diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index c2c3b3eb7a80d..c4cf6960db296 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -15,7 +15,7 @@ use kernel::prelude::*; use super::kick::QID_MAX; -use super::queue::{DataMaster, QUEUE_PRIORITY}; +use super::queue::DataMaster; /// Size of a device-control record. pub(crate) const DEVICE_CONTROL_RECORD_SIZE: usize = 0x40; @@ -448,10 +448,13 @@ pub(crate) const WORK_RING_SLOTS: u32 = 256; /// Number of engines with a work ring per priority. const WORK_RINGS_PER_PRIORITY: usize = 3; -/// Index in the channel table of the work ring for `data_master` at the -/// priority of every driver-created queue. -pub(crate) const fn work_ring_index(data_master: DataMaster) -> usize { - QUEUE_PRIORITY as usize * WORK_RINGS_PER_PRIORITY + data_master as usize +/// Index in the channel table of the work ring for `data_master` at priority +/// class `priority`, or `None` if `priority` is not a class. +pub(crate) const fn work_ring_index(priority: u8, data_master: DataMaster) -> Option { + if priority >= super::queue::Policy::PRIORITIES { + return None; + } + Some(priority as usize * WORK_RINGS_PER_PRIORITY + data_master as usize) } /// One work-ring slot: a request to run a queue's newly published commands. @@ -467,7 +470,9 @@ pub(crate) struct WorkSlot { pub(crate) items_end: u16, /// Queue to run. pub(crate) qid: u8, - /// One in the first slot naming this queue on this ring. + /// One in the first slot naming this queue on this ring. Each priority + /// class has its own rings, so a queue whose class changed names the new + /// ring with a first slot again. pub(crate) first: u8, } diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index 2e6133e882b2d..47cdce3ca09e5 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -20,7 +20,7 @@ use kernel::prelude::*; -use super::queue::QUEUE_PRIORITY; +use super::queue::Policy; /// Number of entries in a kick ring. pub(crate) const KICK_SLOTS: usize = 256; @@ -371,6 +371,8 @@ pub(crate) struct KickArgs<'a> { pub(crate) register_arrays: [Option; 4], /// The compute register program includes a shader-context scratch request. pub(crate) compute_scratch: bool, + /// Priority class of the queue. + pub(crate) priority: u8, } /// One kick ring entry. @@ -436,13 +438,14 @@ impl KickEntry { if args.qid > QID_MAX || barriers > KICK_BARRIERS_MAX || args.qos.class > Self::QOS_CLASS_MAX + || args.priority >= Policy::PRIORITIES { return Err(EINVAL); } let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT | (args.qid as u64) << Self::QID_SHIFT - | (QUEUE_PRIORITY as u64) << Self::PRIORITY_SHIFT; + | (args.priority as u64) << Self::PRIORITY_SHIFT; if args.compute_scratch { header |= Self::COMPUTE_SCRATCH; } @@ -506,9 +509,10 @@ impl RenderRegistration { /// Validates both queues without touching MMIO. The resulting transaction /// writes fragment configuration and validity before tiling configuration - /// and validity. Each validity mask is stored high before low. - pub(crate) fn new(fragment: (u8, u64), tiling: (u8, u64)) -> Result { - if fragment.0 == tiling.0 { + /// and validity. Each validity mask is stored high before low. Both queues + /// are registered at priority class `priority`. + pub(crate) fn new(fragment: (u8, u64), tiling: (u8, u64), priority: u8) -> Result { + if fragment.0 == tiling.0 || priority >= Policy::PRIORITIES { return Err(EINVAL); } let mut writes = [(0, 0); 8]; @@ -522,7 +526,7 @@ impl RenderRegistration { if qid > QID_MAX || va & !Self::ADDRESS_MASK != 0 { return Err(EINVAL); } - let completion = (3 * (4 - u64::from(QUEUE_PRIORITY)) + 0x3fe).wrapping_shl(54); + let completion = (3 * (4 - u64::from(priority)) + 0x3fe).wrapping_shl(54); let payload = ((engine as u64 + 1) << 60) | va | completion; let (low, high) = if qid < 64 { (1u64 << qid, 0) diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index db95ba99e0189..48b110a4193e3 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -22,14 +22,59 @@ use kernel::prelude::*; use super::kick::{KickTimestamp, QID_MAX}; -/// Priority of every queue the driver creates. +/// Firmware scheduling profile of a queue. /// -/// The same value appears in the queue record, [`QueueConfig`], -/// [`KickAnnounce`] and the kick entry header; the work rings it selects are -/// listed in `channels.rs`. -pub(crate) const QUEUE_PRIORITY: u8 = 2; -/// Scheduling policy of every queue the driver creates. -pub(crate) const QUEUE_POLICY: u16 = 2; +/// Every record that names a queue repeats its priority class: the queue +/// record (twice), [`QueueConfig`], [`KickAnnounce`], the kick entry header, +/// the work ring the queue is published on (`channels.rs`), the doorbell and +/// the render registration token. The queue record and [`QueueConfig`] also +/// carry the scheduling policy word; the QoS row and share word carry the QoS +/// class and share. All of them must agree for one queue. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Policy { + priority: u8, + policy: u16, + qos_class: u8, + qos_share: u32, +} + +impl Policy { + /// Number of firmware priority classes. + pub(crate) const PRIORITIES: u8 = 4; + + /// Returns the profile with priority class `priority`, policy word + /// `policy`, QoS class `qos_class` and QoS share `qos_share`. + pub(crate) const fn new(priority: u8, policy: u16, qos_class: u8, qos_share: u32) -> Self { + assert!(priority < Self::PRIORITIES); + Self { + priority, + policy, + qos_class, + qos_share, + } + } + + /// Priority class, below [`Self::PRIORITIES`]. + pub(crate) const fn priority(&self) -> u8 { + self.priority + } + + /// Scheduling policy word. + pub(crate) const fn policy(&self) -> u16 { + self.policy + } + + /// QoS class byte of the queue's QoS row. + pub(crate) const fn qos_class(&self) -> u8 { + self.qos_class + } + + /// QoS share word of the owner's QoS slot. + pub(crate) const fn qos_share(&self) -> u32 { + self.qos_share + } +} + /// Number of work pointers in a queue's item ring. pub(crate) const ITEM_RING_ENTRIES: u32 = 0x500; @@ -173,13 +218,15 @@ pub(crate) struct QueueRecordArgs { pub(crate) scheduler_va: u64, /// Thread-group ID of the owning process (0 while unowned). pub(crate) owner_pid: u32, + /// Scheduling profile of the owning logical queue. + pub(crate) policy: Policy, } /// Description of one hardware queue, named by its work-ring slots. /// /// The host writes it whole at queue creation and changes the owner fields of -/// an idle queue with [`QueueRecord::set_owner`] or -/// [`QueueRecord::set_compute_owner`]. +/// an idle queue with [`QueueRecord::set_compute_owner`] or, for render +/// queues, field by field (`render/memory.rs`). #[repr(C, packed)] #[derive(Copy, Clone, Debug)] pub(crate) struct QueueRecord { @@ -224,12 +271,12 @@ impl QueueRecord { job_list_va: args.job_list_va, unk_18: [0; 0xc], unk_24: u32::MAX, - priority: [QUEUE_PRIORITY as u32; 2], + priority: [args.policy.priority() as u32; 2], unk_30: [0; 6], unk_36: u16::MAX, unk_38: 0, unk_3c: 0, - policy: QUEUE_POLICY as u32, + policy: args.policy.policy() as u32, unk_44: u32::MAX, owner_pid: args.owner_pid, unk_4c: [0; 0x50], @@ -241,16 +288,32 @@ impl QueueRecord { /// Hands an idle compute queue to a new owner. /// /// Replaces the job head, process identity and scheduler page, and clears `unk_38`. + /// The priority and policy words are written only when `policy` differs from the + /// profile the record holds. pub(crate) fn set_compute_owner( &mut self, job_list_va: u64, scheduler_va: u64, owner_pid: u32, + policy: Policy, ) { self.job_list_va = job_list_va; self.unk_38 = 0; self.owner_pid = owner_pid; self.scheduler_va = scheduler_va; + self.set_policy(policy); + } + + /// Replaces the priority and policy words of an idle queue's record when they differ + /// from `policy`. + pub(crate) fn set_policy(&mut self, policy: Policy) { + let priority = [policy.priority() as u32; 2]; + if { self.priority } != priority { + self.priority = priority; + } + if { self.policy } != policy.policy() as u32 { + self.policy = policy.policy() as u32; + } } } @@ -310,6 +373,8 @@ pub(crate) struct QueueConfigArgs { pub(crate) free_list: FreeListBinding, /// GPU address of the owner's scheduler-state page. pub(crate) scheduler_va: u64, + /// Scheduling profile of the owning logical queue. + pub(crate) policy: Policy, /// QoS buffer slot of the logical queue. pub(crate) qos_slot: u8, /// The QoS identity or queue membership changed since the last update. @@ -429,7 +494,7 @@ impl QueueConfig { qid: args.qid as u16, install: args.install as u16, unk_1c: 0, - priority: QUEUE_PRIORITY as u16, + priority: args.policy.priority() as u16, unk_20: 0, data_master: data_master as u16, unk_24: 0, @@ -445,7 +510,7 @@ impl QueueConfig { qos_slot: args.qos_slot as u16, unk_58: 0, owner_pid: args.owner_pid, - policy: QUEUE_POLICY, + policy: args.policy.policy(), unk_60: 0, context_update: args.context_update as u16, unk_64: 0, @@ -486,16 +551,22 @@ static_assert!(core::mem::offset_of!(KickAnnounce, timestamp) == 0x09); static_assert!(core::mem::offset_of!(KickAnnounce, data_master) == 0x11); impl KickAnnounce { - /// Announces the entry of queue `qid` written at `timestamp`. - pub(crate) fn new(qid: u8, data_master: DataMaster, timestamp: KickTimestamp) -> Result { - if qid > QID_MAX { + /// Announces the entry of queue `qid` at priority class `priority`, + /// written at `timestamp`. + pub(crate) fn new( + qid: u8, + data_master: DataMaster, + timestamp: KickTimestamp, + priority: u8, + ) -> Result { + if qid > QID_MAX || priority >= Policy::PRIORITIES { return Err(EINVAL); } Ok(Self { tag: CommandTag::KickAnnounce as u32, qid: qid as u16, count: 1, - priority: QUEUE_PRIORITY, + priority, timestamp: timestamp.get(), data_master: data_master as u32, unk_15: [0; 3], diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index aea894ba2c2d8..a41e4de24950d 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -416,14 +416,20 @@ impl InitData { /// The sole host publisher holds the device mutex through the following /// inner and outer publication. Firmware consumers can only free capacity; /// neither this check nor waiting for room reserves or limits other queues. - pub(super) fn work_ready(&self, engine: super::fw::queue::DataMaster) -> Result { - let [consumer, consumer_b, producer] = self.work_cursors(engine)?; + /// `priority` selects the engine's ring of that priority class. + pub(super) fn work_ready( + &self, + priority: u8, + engine: super::fw::queue::DataMaster, + ) -> Result { + let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; let next = (producer + 1) % super::fw::channels::WORK_RING_SLOTS; Ok(next != consumer && next != consumer_b) } fn work_publication<'a>( &'a self, + priority: u8, record: &'a super::fw::channels::WorkSlot, ) -> Result> { use super::fw::{channels, queue::DataMaster}; @@ -433,8 +439,8 @@ impl InitData { 2 => DataMaster::Compute, _ => return Err(EINVAL), }; - let channel = channels::work_ring_index(engine); - let [consumer, consumer_b, producer] = self.work_cursors(engine)?; + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; + let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; let next = (producer + 1) % channels::WORK_RING_SLOTS; if next == consumer || next == consumer_b { return Err(EINVAL); @@ -457,9 +463,13 @@ impl InitData { } /// Two firmware consumers and the host producer, sampled in that order. - pub(super) fn work_cursors(&self, engine: super::fw::queue::DataMaster) -> Result<[u32; 3]> { + pub(super) fn work_cursors( + &self, + priority: u8, + engine: super::fw::queue::DataMaster, + ) -> Result<[u32; 3]> { use super::fw::channels; - let channel = channels::work_ring_index(engine); + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; let mut values = [0; 3]; @@ -481,8 +491,12 @@ impl InitData { /// Inner queue records must be visible before this outer publication. The returned /// producer is the retained consumer witness for this slot. - pub(super) fn publish_work(&self, record: &super::fw::channels::WorkSlot) -> Result { - Ok(self.work_publication(record)?.commit()) + pub(super) fn publish_work( + &self, + priority: u8, + record: &super::fw::channels::WorkSlot, + ) -> Result { + Ok(self.work_publication(priority, record)?.commit()) } /// Validate both destinations before exposing fragment, then tiling. The accounting token @@ -490,6 +504,7 @@ impl InitData { /// Deferred pairs retain both slot bodies without exposing either producer. pub(super) fn publish_work_pair( &self, + priority: u8, fragment: &super::fw::channels::WorkSlot, tiling: &super::fw::channels::WorkSlot, deferred: bool, @@ -502,8 +517,8 @@ impl InitData { { return Err(EINVAL); } - let fragment = self.work_publication(fragment)?; - let tiling = self.work_publication(tiling)?; + let fragment = self.work_publication(priority, fragment)?; + let tiling = self.work_publication(priority, tiling)?; fragment.copy_record(); commit_accounting()?; fence(Ordering::SeqCst); @@ -522,6 +537,7 @@ impl InitData { fn retained_work_publication<'a>( &'a self, + priority: u8, record: &'a super::fw::channels::WorkSlot, next: u32, ) -> Result> { @@ -531,7 +547,7 @@ impl InitData { 1 => DataMaster::Fragment, _ => return Err(EINVAL), }; - let channel = channels::work_ring_index(engine); + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let producer = self.object(self.cluster)?.word( Self::state_offset(Role::Primary) + abi::work_state_offset(channel) @@ -558,6 +574,7 @@ impl InitData { /// witnesses before either copy; both bodies precede the fragment and tiling publications. pub(super) fn publish_retained_work_pair( &self, + priority: u8, fragment: &super::fw::channels::WorkSlot, tiling: &super::fw::channels::WorkSlot, next: [u32; 2], @@ -568,8 +585,8 @@ impl InitData { { return Err(EINVAL); } - let fragment = self.retained_work_publication(fragment, next[0])?; - let tiling = self.retained_work_publication(tiling, next[1])?; + let fragment = self.retained_work_publication(priority, fragment, next[0])?; + let tiling = self.retained_work_publication(priority, tiling, next[1])?; fragment.copy_record(); tiling.copy_record(); fence(Ordering::SeqCst); diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 798eaa3a360aa..23c4fb962153a 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1172,8 +1172,9 @@ impl gpu::Gpu for Gpu { let owner = self.ids.queue.next(); let frontend = queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; + let policy = scheduling_policy(priority)?; let (backend, slot) = loop { - match runtime::Backend::new(&self.shared, &vm, owner) { + match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, result => break result?, } @@ -1214,6 +1215,17 @@ impl gpu::Gpu for Gpu { } } +/// Scheduling profile of a new logical queue. `file.rs` passes +/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses HIGH and REALTIME, so +/// MEDIUM arrives as 2 and LOW as 3. +fn scheduling_policy(priority: u32) -> Result { + match priority { + 2 => Ok(hw::t8140::scheduling::MEDIUM), + 3 => Ok(hw::t8140::scheduling::LOW), + _ => Err(EINVAL), + } +} + impl Drop for Gpu { fn drop(&mut self) { self.shutdown(); diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs index 8a925c23552a7..9a100db4cffdf 100644 --- a/drivers/gpu/drm/asahi/g17/qos.rs +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -6,13 +6,16 @@ //! precedes submitted counters; retiring a final owner clears only its two row //! bytes. Cancellation balances host references without rewinding shared counts. -use super::{fw::qos as fw, object::KernelObject}; +use super::{ + fw::{qos as fw, queue::Policy}, + object::KernelObject, +}; +use crate::hw::t8140::qos as cfg; use core::{ mem::{offset_of, size_of}, sync::atomic::{fence, AtomicU32, AtomicU64, AtomicU8, Ordering}, }; use kernel::prelude::*; -use crate::hw::t8140::qos as cfg; /// Atomic views retain the allocation borrow and never borrow firmware words /// as ordinary mutable memory. The arrays are disjoint apart from the explicit @@ -97,11 +100,13 @@ impl Accounting { /// Called under the device publication lock immediately before exposing /// the work record. The clock is sampled only when its engine becomes busy. + /// The row's class and the owner's share come from the owner's `policy`. pub(crate) fn publish( &mut self, view: &View<'_>, owner: Owner, scheduler: u64, + policy: Policy, clock: impl FnOnce() -> (u64, u64), ) -> Result { let (qid, qos, dm) = owner.indices()?; @@ -124,10 +129,10 @@ impl Accounting { return Err(EBUSY); } view.rows[qid][0].store(owner.qos, Ordering::Relaxed); - view.rows[qid][1].store(cfg::CLASS, Ordering::Relaxed); + view.rows[qid][1].store(policy.qos_class(), Ordering::Relaxed); let record = view.row(qid); view.scheduler[qos].store(scheduler, Ordering::Relaxed); - view.share[qos].store(cfg::SHARE, Ordering::Relaxed); + view.share[qos].store(policy.qos_share(), Ordering::Relaxed); if self.in_flight[qid] == 0 { fence(Ordering::SeqCst); } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 57179437f9b40..2b962e2632606 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -137,6 +137,7 @@ impl Graph { job_list_va: context.work_head_va(), scheduler_va: context.scheduler_va(), owner_pid: context.owner_pid(), + policy: context.policy(), }); view.pointers = PointerBlock::new(); view.jobs = JobListHead::new(graph_va + 0x200); @@ -210,6 +211,7 @@ impl Graph { context.work_head_va(), context.scheduler_va(), context.owner_pid(), + context.policy(), ) }; Ok(()) @@ -468,7 +470,8 @@ pub(crate) struct Queue { replay_prepared: bool, exit: Option, installed: bool, - outer_started: bool, + /// Bit `p` set once a work slot of priority class `p` named this queue. + outer_started: u8, published_config: Option, ordinal: u32, submitted: u32, @@ -521,7 +524,7 @@ impl Queue { replay_prepared: false, exit: None, installed: false, - outer_started: false, + outer_started: 0, published_config: None, ordinal: 0, submitted: 0, @@ -1459,10 +1462,16 @@ pub(crate) trait Host { fn next_compute_publication(&mut self) -> Result; fn epoch(&self) -> Result<(u64, u32)>; fn prepare_compute_shared(&mut self) -> Result; - fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: crate::g17::fw::queue::Policy, + ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; - fn publish_outer(&mut self, slot: &crate::g17::fw::channels::WorkSlot) -> Result; + fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) + -> Result; fn notify(&mut self, message: u64) -> Result; fn note_submission(&mut self) -> Result; } @@ -1580,6 +1589,8 @@ impl Queue { context: packet.context.id(), }; self.free_list_generation = u64::from(packet.context.scheduler_generation()); + let policy = packet.context.policy(); + let ring = 1u8 << policy.priority(); let config = QueueConfig::new(&QueueConfigArgs { target: QueueConfigTarget::Compute, kick_ring_va: self.kick.low_va(), @@ -1594,6 +1605,7 @@ impl Queue { slot: u32::from(self.pool.id()), }, scheduler_va: identity.scheduler, + policy, qos_slot: identity.qos, qos_update: self .published_config @@ -1603,7 +1615,8 @@ impl Queue { .published_config .is_none_or(|p| p.scheduler != identity.scheduler || p.context != identity.context), })?; - let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp)?; + let announce = + KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; let kick_args = KickArgs { qid, @@ -1620,6 +1633,7 @@ impl Queue { event_mask: COMPUTE_KICK_EVENT_MASK, register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch)?, compute_scratch: scratch.enabled(), + priority: policy.priority(), }; let published = host.next_compute_publication()?; self.active @@ -1690,6 +1704,7 @@ impl Queue { data_master: 2, }, identity.scheduler, + policy, )?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; let target = self.pool.reserve_submission().or_else(|error| { @@ -1725,6 +1740,7 @@ impl Queue { data_master: 2, }, identity.scheduler, + policy, )?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; producer = self @@ -1754,15 +1770,18 @@ impl Queue { self.graph.record_ends[slot] = Some(producer); } host.publish_qid(self.kick.id())?; - host.publish_outer(&WorkSlot::new( - DataMaster::Compute, - self.graph.graph.gpu_va(), - qid, - producer.try_into()?, - !self.outer_started, - )?)?; - let notify_activation = !self.outer_started; - self.outer_started = true; + host.publish_outer( + policy.priority(), + &WorkSlot::new( + DataMaster::Compute, + self.graph.graph.gpu_va(), + qid, + producer.try_into()?, + self.outer_started & ring == 0, + )?, + )?; + let notify_activation = self.outer_started & ring == 0; + self.outer_started |= ring; self.installed = true; self.published_config = Some(identity); self.ordinal = ordinal; @@ -1791,7 +1810,8 @@ impl Queue { } if notify_activation { fence(Ordering::SeqCst); - host.notify((0x83 << 48) | 0x0a)?; + // Pipe 2 (compute) in bits 1:0, priority class in bits 3:2. + host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; } fence(Ordering::SeqCst); host.notify((0x83 << 48) | 0x10)?; diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 720f2d8a0897e..7936d253d2411 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -74,12 +74,18 @@ pub(crate) struct PublicationLease { pub(crate) trait Host { fn epoch(&self) -> Result<(u64, u32)>; fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; - fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: crate::g17::fw::queue::Policy, + ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn install_pair(&mut self, registration: &RenderRegistration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; fn publish_outer_pair( &mut self, + priority: u8, fragment: &crate::g17::fw::channels::WorkSlot, tiling: &crate::g17::fw::channels::WorkSlot, commit: impl FnOnce() -> Result, @@ -88,11 +94,12 @@ pub(crate) trait Host { fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result; fn publish_retained_pair( &mut self, + priority: u8, fragment: &WorkSlot, tiling: &WorkSlot, next: [u32; 2], ) -> Result; - fn notify_render(&mut self) -> Result; + fn notify_render(&mut self, priority: u8) -> Result; fn note_submission(&mut self) -> Result; fn complete_render( &mut self, @@ -141,8 +148,11 @@ pub(crate) struct Pair { configs: [Option; 2], cursors: [u32; 2], recovery_generation: u64, - registered: bool, - outer_started: bool, + /// Priority class of the pair's installed registration, if any. A pair + /// rebound to an owner of another class is registered again. + registered: Option, + /// Bit `p` set once a work slot of priority class `p` named the pair. + outer_started: u8, quarantined: bool, terminal: Option<(Arc, Error)>, previous: Option, @@ -223,8 +233,8 @@ impl Pair { configs: [None; 2], cursors: [0; 2], recovery_generation: 0, - registered: false, - outer_started: false, + registered: None, + outer_started: 0, quarantined: false, terminal: None, previous: None, @@ -434,6 +444,12 @@ impl Pair { if packet.completion.status().get() != 0 { return Err(EIO); } + let policy = packet.context.policy(); + // A registration of another class can only follow a rebind, which + // requires the pair to have drained. + if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { + return Err(EIO); + } let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; let mcache = prepared.mcache; @@ -450,6 +466,7 @@ impl Pair { self.memory .completion_links(ordinal, kicks, payload, !covered[1])?; let context = &packet.context; + let ring = 1u8 << policy.priority(); let qos = context.qos_id(); let qos_owners = [ qos::Owner { @@ -492,6 +509,7 @@ impl Pair { slot: self.free_list.id().into(), }, scheduler_va: context.scheduler_va(), + policy, qos_slot: qos, qos_update: self.context_update, owner_pid: context.owner_pid(), @@ -561,6 +579,7 @@ impl Pair { event_mask: abi::RENDER_KICK_EVENT_MASK, register_arrays: arrays, compute_scratch: false, + priority: policy.priority(), }; let ta_args = args( 0, @@ -579,6 +598,7 @@ impl Pair { let registration = RenderRegistration::new( (qids[1], self.kicks[1].low_va()), (qids[0], self.kicks[0].low_va()), + policy.priority(), )?; let [tiling_kick, fragment_kick] = &mut self.kicks; let tiling_kick = tiling_kick.prepare_entry(&ta_args)?; @@ -620,14 +640,14 @@ impl Pair { queues[1], qids[1], windows[1].target().try_into()?, - !self.outer_started, + self.outer_started & ring == 0, )?; let tiling_slot = WorkSlot::new( DataMaster::Tiling, queues[0], qids[0], windows[0].target().try_into()?, - !self.outer_started, + self.outer_started & ring == 0, )?; let event_slot = memory .graph @@ -664,16 +684,20 @@ impl Pair { tiling_kick.commit(); let mut qos_publications = [None; 2]; let result = (|| { - qos_publications[1] = Some(host.qos_publish(qos_owners[1], context.scheduler_va())?); - qos_publications[0] = Some(host.qos_publish(qos_owners[0], context.scheduler_va())?); - if !self.registered { + qos_publications[1] = + Some(host.qos_publish(qos_owners[1], context.scheduler_va(), policy)?); + qos_publications[0] = + Some(host.qos_publish(qos_owners[0], context.scheduler_va(), policy)?); + // A class change happens only at a rebind, after the pair drained. + if self.registered != Some(policy.priority()) { host.install_pair(®istration)?; - self.registered = true; + self.registered = Some(policy.priority()); } for kick in &self.kicks { host.publish_qid(kick.id())?; } host.publish_outer_pair( + policy.priority(), &fragment_slot, &tiling_slot, || { @@ -720,16 +744,16 @@ impl Pair { } self.recovery_generation = epoch; self.installed = [true; 2]; - self.outer_started = true; + self.outer_started |= ring; if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); } packet.context.mark_published(); let publish = (|| { host.activate_render_pool(self.free_list.id(), lease.free_list_generation)?; - host.publish_retained_pair(&fragment_slot, &tiling_slot, outer_next)?; + host.publish_retained_pair(policy.priority(), &fragment_slot, &tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; - host.notify_render()?; + host.notify_render(policy.priority())?; host.note_submission() })(); if let Err(error) = publish { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 859ff61901570..fc5a05e9bf8a9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -32,6 +32,9 @@ pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; const DEPTH: usize = queues::RENDER_DEPTH as usize; +/// Offsets of the priority pair and the policy word in a queue record. +const PRIORITY: usize = core::mem::offset_of!(queue::QueueRecord, priority); +const POLICY: usize = core::mem::offset_of!(queue::QueueRecord, policy); struct Aliases { client: mmu::KernelMapping, @@ -147,6 +150,7 @@ impl Graph { job_list_va: context.work_head_va(), scheduler_va: context.scheduler_va(), owner_pid: context.owner_pid(), + policy: context.policy(), }), )?; queues.initialize::(POINTERS[stage], |view| { @@ -665,6 +669,9 @@ impl Memory { Ok(()) } + /// Retargets both retired queue records to `context`. A context of another + /// scheduling profile also replaces the priority and policy words, after + /// the owner fields; the same profile leaves them untouched. pub(super) fn set_owner(&self, qids: [u8; 2], context: &Context) -> Result { let offsets = qids.map(|qid| usize::from(qid) * size_of::()); let base = self.graph.queues.pointer(0, GRAPH_SIZE)?; @@ -674,7 +681,10 @@ impl Memory { self.graph.queues.pointer(at + 0x10, 8)?; self.graph.queues.pointer(at + 0x48, 4)?; self.graph.queues.pointer(at + 0x9c, 8)?; + self.graph.queues.pointer(at + PRIORITY, 8)?; + self.graph.queues.pointer(at + POLICY, 4)?; } + let policy = context.policy(); // SAFETY: Both queues are retired before retargeting. Complete byte // views were checked above; packed scheduler words use unaligned stores. unsafe { @@ -693,6 +703,17 @@ impl Memory { .cast::() .write_unaligned(context.scheduler_va()); } + for at in offsets { + let words = base.add(at + PRIORITY).cast::<[u32; 2]>(); + let priority = [u32::from(policy.priority()); 2]; + if words.read_unaligned() != priority { + words.write_unaligned(priority); + } + let word = base.add(at + POLICY).cast::(); + if word.read_unaligned() != u32::from(policy.policy()) { + word.write_unaligned(u32::from(policy.policy())); + } + } } Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 3d67578e2aaf9..f712bb7b1b87d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -28,8 +28,8 @@ pub(crate) struct ParkedPair { descriptor_flags: [u32; 2], installed: [bool; 2], recovery_generation: u64, - registered: bool, - outer_started: bool, + registered: Option, + outer_started: u8, previous: Option, } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 3df92062735e3..0fc9c1d9a15dc 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -99,6 +99,7 @@ impl<'a> Lane<'a> { DataMaster::Fragment }, timestamp, + config.priority as u8, )?, announce_destination, prefix: count as usize - 1, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 842e3a8b8df9b..4c7aa6a442cc1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -368,6 +368,7 @@ impl Backend { shared: &Arc, vm: &crate::mmu::Vm, owner: u64, + policy: super::fw::queue::Policy, ) -> Result<(Arc, admission::QueueSlot)> { let lease = shared.preparations.enter(true)?; let dev = shared.drm()?; @@ -394,7 +395,7 @@ impl Backend { let owner_pid = u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; let context = Arc::new( - Context::new(&alloc, &pool, &qos, execution, owner_pid)?, + Context::new(&alloc, &pool, &qos, execution, owner_pid, policy)?, GFP_KERNEL, )?; let state = shared.state.lock(); @@ -776,11 +777,17 @@ impl compute::Host for ComputeHost<'_> { fn prepare_compute_shared(&mut self) -> Result { self.init.activate_compute() } - fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result { + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: super::fw::queue::Policy, + ) -> Result { self.accounting.publish( &qos::View::new(self.init.qos()?)?, owner, scheduler, + policy, qos::clock, ) } @@ -791,8 +798,8 @@ impl compute::Host for ComputeHost<'_> { fn publish_qid(&mut self, id: kick::Id) -> Result { self.qids.publish(id) } - fn publish_outer(&mut self, slot: &super::fw::channels::WorkSlot) -> Result { - self.init.publish_work(slot).map(|_| ()) + fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.init.publish_work(priority, slot).map(|_| ()) } fn notify(&mut self, message: u64) -> Result { self.primary.notify(message) @@ -816,7 +823,10 @@ impl super::Firmware { } // No descriptor, accounting or inner producer changes precede this // capacity check. The same device lock covers queue.publish below. - if !self.init.work_ready(DataMaster::Compute)? { + if !self + .init + .work_ready(packet.context.policy().priority(), DataMaster::Compute)? + { return Err(EAGAIN); } let queues = &mut self.queues; diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 65b00d36714bb..0d4a29e779b4a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -56,7 +56,10 @@ impl Firmware { break; }; let owner = queue.owner(); - let outer_ready = self.init.work_ready(crate::g17::fw::queue::DataMaster::Compute)?; + let outer_ready = self.init.work_ready( + packet.context.policy().priority(), + crate::g17::fw::queue::DataMaster::Compute, + )?; if !outer_ready { let now = super::now_ns(); let since = *self.queues.replay_outer_wait.get_or_insert(now); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index aabc523e360cd..71b21f98479a6 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -377,12 +377,14 @@ impl crate::g17::queue::render::Host for Host<'_> { &mut self, owner: crate::g17::qos::Owner, scheduler: u64, + policy: crate::g17::fw::queue::Policy, ) -> Result { use crate::g17::qos; self.firmware.queues.accounting.publish( &qos::View::new(self.firmware.init.qos()?)?, owner, scheduler, + policy, qos::clock, ) } @@ -418,6 +420,7 @@ impl crate::g17::queue::render::Host for Host<'_> { } fn publish_outer_pair( &mut self, + priority: u8, fragment: &crate::g17::fw::channels::WorkSlot, tiling: &crate::g17::fw::channels::WorkSlot, commit: impl FnOnce() -> Result, @@ -425,7 +428,7 @@ impl crate::g17::queue::render::Host for Host<'_> { ) -> Result<[u32; 2]> { self.firmware .init - .publish_work_pair(fragment, tiling, true, commit, late_tiling) + .publish_work_pair(priority, fragment, tiling, true, commit, late_tiling) } fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { self.release_registration(); @@ -437,6 +440,7 @@ impl crate::g17::queue::render::Host for Host<'_> { } fn publish_retained_pair( &mut self, + priority: u8, fragment: &crate::g17::fw::channels::WorkSlot, tiling: &crate::g17::fw::channels::WorkSlot, next: [u32; 2], @@ -444,18 +448,19 @@ impl crate::g17::queue::render::Host for Host<'_> { let result = self .firmware .init - .publish_retained_work_pair(fragment, tiling, next); + .publish_retained_work_pair(priority, fragment, tiling, next); if result.is_err() { self.firmware.primary.state.shared.lose_device_quietly(); } result } - fn notify_render(&mut self) -> Result { + /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. + fn notify_render(&mut self, priority: u8) -> Result { self.release_registration(); let result = self .firmware .primary - .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | 0x08); + .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | u64::from(priority) << 2); if result.is_err() { self.firmware.primary.state.shared.lose_device_quietly(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 39b8c064d0106..dbf7175c5cd4b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -275,8 +275,9 @@ impl crate::g17::Firmware { } // Both outer destinations must have room before pool accounting or // either inner queue becomes visible. All host producers share this lock. - if !self.init.work_ready(crate::g17::fw::queue::DataMaster::Fragment)? - || !self.init.work_ready(crate::g17::fw::queue::DataMaster::Tiling)? + let priority = packet.context.policy().priority(); + if !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Fragment)? + || !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Tiling)? { return Err(EBUSY); } diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 4a97217a00705..2e73aed1a9797 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -378,6 +378,17 @@ pub(crate) mod qos { pub(crate) const CLOCK_HZ: u64 = 24_000_000; } +/// Firmware scheduling profiles of the DRM queue priorities. Native user +/// queues use priority class 2 with policy 2. +pub(crate) mod scheduling { + use crate::g17::fw::queue::Policy; + + /// `DRM_ASAHI_PRIORITY_MEDIUM`. + pub(crate) const MEDIUM: Policy = Policy::new(2, 2, super::qos::CLASS, super::qos::SHARE); + /// `DRM_ASAHI_PRIORITY_LOW`. + pub(crate) const LOW: Policy = Policy::new(3, 2, super::qos::CLASS, super::qos::SHARE); +} + /// Physical workqueue capacity and logical-owner admission limits. pub(crate) mod queues { pub(crate) const RENDER_SLOTS: usize = 64; From fe676c6f4915ec15bb3ea0880101eb5b29cfc255 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:18:07 -0400 Subject: [PATCH 0671/1579] drm/asahi: drop two defensive clippy allowances in the VM binding paths map_in_range_with_guard_size() and prepare_bind_object() take seven parameters including self, which is clippy's default too_many_arguments threshold, not above it. The allowances silenced a lint that does not fire. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 009b0fb93ba35380563055aa8bc725f36c505262) --- drivers/gpu/drm/asahi/mmu.rs | 1 - drivers/gpu/drm/asahi/mmu/bind.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 7a95c9139207a..5149d7d4b49f3 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1739,7 +1739,6 @@ impl Vm { } /// Reserves a whole-page unmapped guard without adding it to the mapped backing. - #[allow(clippy::too_many_arguments)] pub(crate) fn map_in_range_with_guard_size( &self, gem: &gem::Object, diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index df189e9bc0f5f..adbb609d5a88c 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -56,7 +56,6 @@ impl PreparedUserBindBatch { impl Vm { /// Prepares host state without taking DMA reservations or changing any leaf. - #[allow(clippy::too_many_arguments)] pub(crate) fn prepare_bind_object( &self, gem: &gem::Object, From 3d1be9ce966988190d72dcad1d0fa4dd1b9204c9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:24:25 -0400 Subject: [PATCH 0672/1579] drm/asahi: build the G17 boot objects region by region InitData::build placed and initialized every boot object in one 360-line function with two placement closures. Keep the placement inputs in a small environment and perform the same steps, in the same order, as one function per region: register windows, descriptor tables, render free lists, the render pool, the compute pools, the bundle with its rings and regions, and the firmware-object initialization. The four fences keep their positions and the firmware objects are byte-identical. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 410d8f80e9ab6b405d7f918901fcbe883bb2d9b2) --- drivers/gpu/drm/asahi/g17/initdata.rs | 258 +++++++++++++++++++------- 1 file changed, 187 insertions(+), 71 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index a41e4de24950d..5993be786874e 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -11,6 +11,7 @@ use super::{ Config, Role, }; use crate::{driver::AsahiDevice, hw::t8140, mmu}; +use core::ops::Range; use core::sync::atomic::{fence, AtomicU32, Ordering}; use kernel::{c_str, prelude::*}; use t8140::{kernel_window as high, low}; @@ -166,6 +167,57 @@ pub(super) struct InitData { render_blocks: core::ops::Range, } +/// Placement inputs shared by the build steps. +struct BuildEnv<'a> { + dev: &'a AsahiDevice, + vm: &'a mmu::Vm, + lower: &'a mmu::Vm, + render: &'a mmu::Vm, + base: u64, + dynamic: Range, +} + +impl BuildEnv<'_> { + /// An object at a fixed offset of the kernel window. + fn fixed(&self, offset: u64, size: usize, prot: mmu::Prot, cpu: CpuMap) -> Result { + KernelObject::new( + self.dev, + self.vm, + Placement::At(self.base + offset), + size, + prot, + cpu, + ) + } + + /// An object placed in the dynamic part of the kernel window. + fn dynamic(&self, size: usize, align: u64, prot: mmu::Prot, cpu: CpuMap) -> Result { + KernelObject::new( + self.dev, + self.vm, + Placement::In(self.dynamic.clone(), align), + size, + prot, + cpu, + ) + } +} + +/// Object indices the firmware-object initialization needs from the table step. +struct DescriptorTables { + pb: usize, + page_pool: usize, + fwctl: usize, +} + +/// Object indices and ring addresses the firmware-object initialization needs from the bundle step. +struct BundleObjects { + zero: usize, + completion_rings: [abi::CompletionRing; 2], + region_a: usize, + qos: usize, +} + impl InitData { /// Empty construction state, retained even if graph construction fails. pub(super) fn new() -> Self { @@ -643,46 +695,51 @@ impl InitData { render: &mmu::Vm, metrics: &KernelObject, ) -> Result { - let vm = uat.kernel_vm(); - let lower = uat.kernel_lower_vm(); let range = uat.geometry().kernel_range(); - let base = range.start; - let dynamic = base + t8140::dynamic::KERNEL_OFFSET..range.end; - let fixed = |offset, size, prot, cpu| { - KernelObject::new(dev, vm, Placement::At(base + offset), size, prot, cpu) - }; - let dynamic_object = |size, align, prot, cpu| { - KernelObject::new( - dev, - vm, - Placement::In(dynamic.clone(), align), - size, - prot, - cpu, - ) - }; - use mmu::{ - PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_PRIV_RW as GFP, - PROT_GPU_FW_SHARED_RW as GFS, PROT_GPU_SHARED_RW as GS, + let env = BuildEnv { + dev, + vm: uat.kernel_vm(), + lower: uat.kernel_lower_vm(), + render, + base: range.start, + dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, }; - use CpuMap::{WriteBack as WB, WriteCombined as WC}; + self.map_register_windows(&env)?; + let tables = self.build_descriptor_tables(&env)?; + self.build_render_free_lists(&env)?; + let first_block = self.build_render_pool(&env)?; + self.build_compute_pools(&env, first_block)?; + let bundle = self.build_bundle(&env, cfg, metrics)?; + self.initialize_firmware_objects(&env, cfg, &tables, &bundle) + } + fn map_register_windows(&mut self, env: &BuildEnv<'_>) -> Result { for window in &t8140::REGISTER_WINDOWS { let (phys, va, size) = window.mapping().ok_or(EINVAL)?; self.aliases.push( - vm.map_io(va, phys.try_into()?, size.try_into()?, mmu::PROT_FW_MMIO_RW)?, + env.vm.map_io(va, phys.try_into()?, size.try_into()?, mmu::PROT_FW_MMIO_RW)?, GFP_KERNEL, )?; } - self.cluster = self.add(fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; - self.compute_global = self.add(fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; - let pb = self.add(fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + Ok(()) + } + + /// The private cluster, zero page and the three descriptor tables with their low aliases. + fn build_descriptor_tables(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::{ + PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, + PROT_GPU_SHARED_RW as GS, + }; + use CpuMap::{WriteBack as WB, WriteCombined as WC}; + self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; + self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; + let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(pb)? .initialize::(0, |table| { table.init(); Ok(()) })?; - let page_pool = self.add(fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + let page_pool = self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(page_pool)? .initialize::(0, |table| { table.init(low::RENDER_PAGE_LIST, low::COMPUTE_POOL_PAGE_LIST); @@ -690,17 +747,30 @@ impl InitData { })?; self.pb = pb; self.page_pool = page_pool; - self.alias(pb, lower, low::PB_DESCRIPTORS, GS)?; - self.alias(page_pool, lower, low::PAGE_POOL_DESCRIPTORS, GS)?; - let index = self.add(fixed(high::INDEX, INDEX_SIZE, GFS, WB)?)?; + self.alias(pb, env.lower, low::PB_DESCRIPTORS, GS)?; + self.alias(page_pool, env.lower, low::PAGE_POOL_DESCRIPTORS, GS)?; + let index = self.add(env.fixed(high::INDEX, INDEX_SIZE, GFS, WB)?)?; self.object_mut(index)? .initialize::(0, |table| { table.init(); Ok(()) })?; - self.alias(index, lower, low::INDEX, GFS)?; - let fwctl = self.add(fixed(high::FWCTL, abi::FWCTL_SIZE, FS, WB)?)?; + self.alias(index, env.lower, low::INDEX, GFS)?; + let fwctl = self.add(env.fixed(high::FWCTL, abi::FWCTL_SIZE, FS, WB)?)?; + Ok(DescriptorTables { + pb, + page_pool, + fwctl, + }) + } + /// Render and boot free-list controls, descriptor pages and the two queue contexts. + fn build_render_free_lists(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::{ + PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_PRIV_RW as GFP, + PROT_GPU_FW_SHARED_RW as GFS, + }; + use CpuMap::WriteCombined as WC; for (offset, state, class, prot, initial) in [ ( high::RENDER_FREE_LIST, @@ -717,7 +787,7 @@ impl InitData { abi::FreeListState::BOOT, ), ] { - let control = self.add(fixed(offset, mmu::UAT_PGSZ, prot, WC)?)?; + let control = self.add(env.fixed(offset, mmu::UAT_PGSZ, prot, WC)?)?; self.object_mut(control)?.write( 0, abi::FreeListControl::new(&abi::FreeListArgs { @@ -726,7 +796,7 @@ impl InitData { page_list_va: low::RENDER_PAGE_LIST, run_list_va: low::RENDER_RUN_LIST, blocks: low::RENDER_BLOCKS_INITIAL, - state_va: base + state, + state_va: env.base + state, }), )?; let state_prot = if class == abi::FreeListClass::One { @@ -734,19 +804,26 @@ impl InitData { } else { GFS }; - let state = self.add(fixed(state, mmu::UAT_PGSZ, state_prot, WC)?)?; + let state = self.add(env.fixed(state, mmu::UAT_PGSZ, state_prot, WC)?)?; self.object_mut(state)?.write(0, initial)?; if class == abi::FreeListClass::Two { self.boot_control = control; self.boot_state = state; } } - self.stamps[0] = self.add(fixed(high::DESCRIPTOR_PAGE_A, mmu::UAT_PGSZ, GFS, WC)?)?; - self.stamps[1] = self.add(fixed(high::DESCRIPTOR_PAGE_B, mmu::UAT_PGSZ, GFP, WC)?)?; - self.add(fixed(high::TA_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; - self.add(fixed(high::FRAGMENT_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + self.stamps[0] = self.add(env.fixed(high::DESCRIPTOR_PAGE_A, mmu::UAT_PGSZ, GFS, WC)?)?; + self.stamps[1] = self.add(env.fixed(high::DESCRIPTOR_PAGE_B, mmu::UAT_PGSZ, GFP, WC)?)?; + self.add(env.fixed(high::TA_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + self.add(env.fixed(high::FRAGMENT_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + Ok(()) + } - let pages = self.add(KernelObject::backing(dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; + /// Render page list, run list and blocks aliased into the render-global root; returns + /// the index of the first block. + fn build_render_pool(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::PROT_GPU_SHARED_RW as GS; + use CpuMap::WriteBack as WB; + let pages = self.add(KernelObject::backing(env.dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; self.object_mut(pages)? .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |list| { abi::fill_page_list( @@ -754,21 +831,28 @@ impl InitData { (0..low::RENDER_BLOCKS_INITIAL as usize).map(low::render_block), ) })?; - let runs = self.add(KernelObject::backing(dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; + let runs = self.add(KernelObject::backing(env.dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; let first_block = self.objects.len(); for _ in 0..low::RENDER_BLOCKS_ALLOCATED { - self.add(KernelObject::backing(dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; } self.render_pages = pages; self.render_runs = runs; self.render_blocks = first_block..self.objects.len(); - self.alias(pages, render, low::RENDER_PAGE_LIST, GS)?; - self.alias(runs, render, low::RENDER_RUN_LIST, GS)?; + self.alias(pages, env.render, low::RENDER_PAGE_LIST, GS)?; + self.alias(runs, env.render, low::RENDER_RUN_LIST, GS)?; for block in 0..low::RENDER_BLOCKS_MAPPED { - self.alias(first_block + block, render, low::render_block(block), GS)?; + self.alias(first_block + block, env.render, low::render_block(block), GS)?; } + Ok(first_block) + } - let compute = self.add(fixed(high::COMPUTE_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; + /// Compute free-list controls and states, zero blocks, run list and the compute page + /// list inside the render blocks. + fn build_compute_pools(&mut self, env: &BuildEnv<'_>, first_block: usize) -> Result { + use mmu::PROT_GPU_FW_SHARED_RW as GFS; + use CpuMap::{WriteBack as WB, WriteCombined as WC}; + let compute = self.add(env.fixed(high::COMPUTE_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; self.object_mut(compute)?.write( 0, abi::FreeListControl::new(&abi::FreeListArgs { @@ -777,18 +861,18 @@ impl InitData { page_list_va: low::COMPUTE_PAGE_LIST, run_list_va: low::COMPUTE_RUN_LIST, blocks: low::COMPUTE_BLOCKS, - state_va: base + high::COMPUTE_FREE_LIST_STATE, + state_va: env.base + high::COMPUTE_FREE_LIST_STATE, }), )?; - let compute_state = self.add(KernelObject::backing(dev, mmu::UAT_PGSZ, WB)?)?; + let compute_state = self.add(KernelObject::backing(env.dev, mmu::UAT_PGSZ, WB)?)?; self.object_mut(compute_state)? .write(0, abi::FreeListState::COMPUTE)?; for _ in &low::COMPUTE_ZERO_BLOCKS { - self.add(KernelObject::backing(dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; } let compute_runs = self.add(KernelObject::new( - dev, - lower, + env.dev, + env.lower, Placement::At(low::COMPUTE_RUN_LIST), mmu::UAT_PGSZ, GFS, @@ -798,7 +882,7 @@ impl InitData { .initialize::<[abi::FreeListRunSlot; mmu::UAT_PGSZ / 0x40]>(0, |list| { abi::fill_run_list(list, low::COMPUTE_FREE_LIST_BLOCKS) })?; - let ready = self.add(fixed( + let ready = self.add(env.fixed( high::COMPUTE_READY_FREE_LIST, mmu::UAT_PGSZ, GFS, @@ -812,11 +896,11 @@ impl InitData { page_list_va: low::COMPUTE_POOL_PAGE_LIST, run_list_va: low::COMPUTE_POOL_RUN_LIST, blocks: low::COMPUTE_READY_BLOCKS, - state_va: base + high::COMPUTE_READY_FREE_LIST_STATE, + state_va: env.base + high::COMPUTE_READY_FREE_LIST_STATE, }), )?; for state in [high::COMPUTE_READY_FREE_LIST_STATE, high::COMPUTE_STATE] { - let obj = self.add(fixed(state, mmu::UAT_PGSZ, GFS, WC)?)?; + let obj = self.add(env.fixed(state, mmu::UAT_PGSZ, GFS, WC)?)?; self.object_mut(obj)? .write(0, abi::FreeListState::COMPUTE)?; } @@ -825,9 +909,24 @@ impl InitData { abi::fill_page_list(list, low::COMPUTE_FREE_LIST_BLOCKS) })?; fence(Ordering::SeqCst); + Ok(()) + } - self.bundle = self.add(fixed(high::BUNDLE, abi::bundle::SIZE, FP, WC)?)?; - let zero = self.add(fixed( + /// The bundle with its zero page and completion rings, the QoS table, the metrics alias, + /// the roots and regions A and C. + fn build_bundle( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + metrics: &KernelObject, + ) -> Result { + use mmu::{ + PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, + PROT_GPU_SHARED_RW as GS, + }; + use CpuMap::{WriteBack as WB, WriteCombined as WC}; + self.bundle = self.add(env.fixed(high::BUNDLE, abi::bundle::SIZE, FP, WC)?)?; + let zero = self.add(env.fixed( high::BUNDLE + abi::bundle::ZERO_PAGE as u64, mmu::UAT_PGSZ, GFS, @@ -839,45 +938,62 @@ impl InitData { }; 2]; for slot in 0..2 { let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; - let ring = self.add(fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; + let ring = self.add(env.fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; self.completion_rings[slot] = ring; - self.alias(ring, lower, low::COMPLETION_RINGS[slot], GS)?; + self.alias(ring, env.lower, low::COMPLETION_RINGS[slot], GS)?; completion_rings[slot] = abi::CompletionRing { low_va: low::COMPLETION_RINGS[slot], - high_va: base + offset, + high_va: env.base + offset, }; } - let qos = self.add(dynamic_object(abi::QOS_SIZE, mmu::UAT_PGSZ as u64, FS, WC)?)?; + let qos = self.add(env.dynamic(abi::QOS_SIZE, mmu::UAT_PGSZ as u64, FS, WC)?)?; self.qos = qos; fence(Ordering::SeqCst); let metrics_alias = - metrics.alias_in(vm, dynamic.clone(), cfg.pm_metrics.size as u64, GFS)?; + metrics.alias_in(env.vm, env.dynamic.clone(), cfg.pm_metrics.size as u64, GFS)?; self.metrics_fw_va = metrics_alias.iova(); self.aliases.push(metrics_alias, GFP_KERNEL)?; fence(Ordering::SeqCst); - self.roots = self.add(dynamic_object( + self.roots = self.add(env.dynamic( abi::ROOTS_SIZE, abi::SECONDARY_ROOT_OFFSET as u64, mmu::PROT_FW_SHARED_RO, WB, )?)?; - let region_a = self.add(dynamic_object( + let region_a = self.add(env.dynamic( REGION_A_SIZE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RO, WB, )?)?; - self.region_c = self.add(dynamic_object(REGION_C_SIZE, mmu::UAT_PGSZ as u64, FS, WB)?)?; + self.region_c = self.add(env.dynamic(REGION_C_SIZE, mmu::UAT_PGSZ as u64, FS, WB)?)?; + Ok(BundleObjects { + zero, + completion_rings, + region_a, + qos, + }) + } - let opps = Self::opps(dev)?; + /// Writes every firmware object the host builds from constants and addresses, then the + /// opening control-ring state for both roles. + fn initialize_firmware_objects( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + tables: &DescriptorTables, + bundle: &BundleObjects, + ) -> Result { + let base = env.base; + let opps = Self::opps(env.dev)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, windows: &t8140::REGISTER_WINDOWS, reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, opps: &opps, perf: &t8140::PERF, - completion_rings, - qos_va: self.object(qos)?.gpu_va(), + completion_rings: bundle.completion_rings, + qos_va: self.object(bundle.qos)?.gpu_va(), }; self.object_mut(self.bundle)? .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; @@ -890,7 +1006,7 @@ impl InitData { .write(abi::private::PRIMARY_STATUS_A, abi::StatusBlock::new())?; self.object_mut(self.cluster)? .write(abi::private::SECONDARY_STATUS_A, abi::StatusBlock::new())?; - let fwctl_va = self.object(fwctl)?.gpu_va(); + let fwctl_va = self.object(tables.fwctl)?.gpu_va(); self.object_mut(self.cluster)? .initialize::(abi::private::STATUS_B, |status| { status.init(fwctl_va); @@ -923,9 +1039,9 @@ impl InitData { Ok(()) })?; let views = abi::PrimaryViews { - zero_page_va: self.object(zero)?.gpu_va(), - pb_descriptors_va: self.object(pb)?.gpu_va(), - page_pool_descriptors_va: self.object(page_pool)?.gpu_va(), + zero_page_va: self.object(bundle.zero)?.gpu_va(), + pb_descriptors_va: self.object(tables.pb)?.gpu_va(), + page_pool_descriptors_va: self.object(tables.page_pool)?.gpu_va(), }; for role in [Role::Primary, Role::Secondary] { let slot = role as usize; @@ -958,7 +1074,7 @@ impl InitData { abi::OpeningRecord::new(abi_role), )?; let root = abi::RootArgs { - region_a_va: self.object(region_a)?.gpu_va(), + region_a_va: self.object(bundle.region_a)?.gpu_va(), region_c_va: self.object(self.region_c)?.gpu_va(), main_config_va: main_va, status_a_va: status_va, From 456e6601e69d15067bcccde93498fd58e22977a1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:26:41 -0400 Subject: [PATCH 0673/1579] drm/asahi: split the G17 render pair install paths out of build_render_pair build_render_pair() held the device-locked selection of how a slot is built, the three off-lock-then-locked install paths and the final locked warm-up in one 400-line function. Move each install path into its own function: install_fresh_pair(), install_parked_pair() and rebind_pair() construct off the lock and install under it exactly as before, and finish_render_build() rebases a fresh graph and warms its pool. A small Copy context carries the lease, allocator, resources, slot and first packet to the install steps. Statement order, lock scopes and error handling are unchanged; the selection stays a block expression in build_render_pair() because returning the selected resources and build by value doubled their stack slots (+1.3 KiB on the scheduler run_job path). The three install steps are inlined: as separate frames they deepen the run_job path, which is close to its stack budget. With them inlined the path measures 17,616 of 18,432 bytes against 17,392 before this change. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f5bd61786e3b537fd811f0ea27d2ec33874232b0) --- .../drm/asahi/g17/runtime/render/prepare.rs | 575 ++++++++++-------- 1 file changed, 331 insertions(+), 244 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index b4813cdba033a..06bea3c84fa7a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -6,6 +6,7 @@ use super::{Config, Reservation}; use crate::g17::{ buffer::MetricsLease, + preparation::Lease, job::Packet, object::{Allocator, KernelObject, Pool}, queue::render::{Pair, RenderAddresses}, @@ -59,6 +60,17 @@ enum Build { }, } +/// What every build step needs besides its variant: the preparation lease, the allocator, +/// the device resources, the slot and the first packet. +#[derive(Clone, Copy)] +struct BuildContext<'a> { + lease: &'a Lease, + alloc: &'a Allocator<'a>, + resources: &'a BuildResources, + slot: u8, + first: &'a Arc, +} + impl super::super::Backend { fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { let lease = self.shared.preparations.enter(false)?; @@ -178,264 +190,339 @@ impl super::super::Backend { Build::Fresh(..) | Build::Parked { .. } => true, Build::Rebind { buffer, .. } => buffer.is_some(), }; - let result = match build { + let ctx = BuildContext { + lease: &lease, + alloc: &alloc, + resources: &resources, + slot, + first, + }; + match build { Build::Fresh(reservation, pm_generation, metrics) => { - let built = (|| -> Result> { - let free_list = - self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; - Pair::new( - &alloc, - &resources.objects, - reservation.ids, - slot, - self.owner, - self.context.clone(), - free_list, - metrics, - reservation.buffer, - pm_generation, - first, - resources.config.clusters, - resources.config.descriptor_flags, - ) - })(); - let (mut built, error) = match built { - Ok(pair) => (Some(pair), None), - Err(error) => (None, Some(error)), - }; - let result = (|| { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - return Err(EAGAIN); - } - if let Some(error) = error { - return Err(error); - } - if !self.context.is_current() || self.context.status().get() != 0 { - return Err(EFAULT); - } - if firmware.queues.render.reserved(reservation)?.closed { - return Err(ECANCELED); - } - built - .as_mut() - .ok_or(EIO)? - .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; - firmware.queues.install_render(reservation, &mut built) - })(); - // Fresh backing disappears before its temporary PB reference returns. - drop(built); - if result.is_err() { - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .ok_or(ENODEV)? - .queues - .cancel_render_reservation(reservation)?; - } - result + self.install_fresh_pair(&ctx, reservation, pm_generation, metrics)? } Build::Parked { - mut parked, + parked, buffer, pm_generation, metrics, - } => { - let built = (|| -> Result<_> { - let free_list = - self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; - let prepared = parked.prepare( - &alloc, - &resources.objects, - self.owner, - self.context.clone(), - free_list, - metrics, - buffer, - pm_generation, - first, - )?; - let storage = KBox::::new_uninit(GFP_KERNEL)?; - Ok((prepared, storage)) - })(); - let (mut prepared, mut storage, error) = match built { - Ok((prepared, storage)) => (Some(prepared), Some(storage), None), - Err(error) => (None, None, Some(error)), - }; - let result = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - (|| { - if !lease.is_current() { - return Err(EAGAIN); - } - if let Some(error) = error { - return Err(error); - } - if !self.context.is_current() || self.context.status().get() != 0 { - return Err(EFAULT); - } - let entry = firmware.queues.render.entry(slot)?; - if entry.pending_owner != Some(self.owner) - || entry.pending_owner_closed - || entry.pair.is_some() - || entry.parked.is_some() - { - return Err(ECANCELED); - } - let storage = storage.take().ok_or(EIO)?; - firmware.queues.buffers.can_release(buffer.0)?; - prepared.as_mut().ok_or(EIO)?.finish_install( - &parked, - &alloc, - firmware.init.pb_descriptor_table()?, - )?; - let pair = parked.commit(&mut prepared, storage)?; - firmware.queues.buffers.reserve_owner(buffer.0)?; - firmware.queues.buffers.release(buffer.0)?; - let entry = firmware.queues.render.entry_mut(slot)?; - entry.reservation.owner = self.owner; - entry.reservation.buffer = buffer; - entry.context = Some(self.context.clone()); - entry.pool = Some(pair.pool().clone()); - entry.pair = Some(pair); - entry.closed = false; - entry.published = false; - Ok(()) - })() - }; - // Rejected candidate mappings disappear before the PB identity - // returns, while the original installed kick owner stays live. - drop(prepared); - drop(storage); - let grow_ready = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if result.is_err() { - firmware.queues.render.entry_mut(slot)?.parked = Some(parked); - firmware.queues.buffers.release(buffer.0)?; - } else { - drop(state); - drop(parked); - state = self.shared.state.lock(); - } - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - let entry = firmware.queues.render.entry_mut(slot)?; - entry.preparing = None; - entry.pending_owner = None; - entry.pending_owner_closed = false; - firmware.render_grow_ready() - }; - self.shared.changed.notify_all(); - if grow_ready { - self.shared.queue_grow(); - } - result - } + } => self.install_parked_pair(&ctx, parked, buffer, pm_generation, metrics)?, Build::Rebind { - mut pair, + pair, buffer, pm_generation, metrics, - } => { - let prepared = (|| { - let free_list = - self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; - pair.prepare_bind( - &alloc, - &resources.objects, - self.owner, - self.context.clone(), - free_list, - metrics, - buffer, - pm_generation, - first, - ) - })(); - let (mut prepared, error) = match prepared { - Ok(binding) => (Some(binding), None), - Err(error) => (None, Some(error)), - }; - let (result, grow_ready) = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - let result = (|| { - if !lease.is_current() { - return Err(EAGAIN); - } - if let Some(error) = error { - return Err(error); - } - if !self.context.is_current() || self.context.status().get() != 0 { - return Err(EFAULT); - } - let entry = firmware.queues.render.entry(slot)?; - if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { - return Err(ECANCELED); - } - if firmware.queues.render.retired.len() - == firmware.queues.render.retired.capacity() - { - return Err(ENOSPC); - } - firmware.queues.teardown.displace( - pair.context(), - true, - super::super::now_ns(), - )?; - pair.bind_owner( - prepared.as_mut().ok_or(EIO)?, - &alloc, - firmware.init.pb_descriptor_table()?, - )?; - if let Some((id, _)) = buffer { - firmware.queues.buffers.reserve_owner(id)?; - firmware.queues.buffers.release(id)?; - } - let entry = firmware.queues.render.entry_mut(slot)?; - entry.reservation.owner = self.owner; - if let Some(buffer) = buffer { - entry.reservation.buffer = buffer; - } - entry.context = Some(self.context.clone()); - entry.pool = Some(pair.pool().clone()); - entry.closed = false; - Ok(()) - })(); - // The original installed pair remains owned on every error. - let entry = firmware.queues.render.entry_mut(slot)?; - entry.preparing = None; - entry.pending_owner = None; - entry.pending_owner_closed = false; - entry.pair = Some(pair); - entry.borrowed_dependencies = None; - (result, firmware.render_grow_ready()) - }; - drop(prepared); - if result.is_err() { - if let Some((id, _)) = buffer { - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .ok_or(ENODEV)? - .queues - .buffers - .release(id)?; - } + } => self.rebind_pair(&ctx, pair, buffer, pm_generation, metrics)?, + } + self.finish_render_build(&lease, slot, fresh_graph) + } + + /// Inlined into build_render_pair(): as a separate frame this step deepens the scheduler + /// run_job stack path, which is close to its budget. + #[inline(always)] + /// Off-lock construction of a fresh pair, then its device-locked install. + fn install_fresh_pair( + &self, + ctx: &BuildContext<'_>, + reservation: Reservation, + pm_generation: u64, + metrics: MetricsLease, + ) -> Result { + let BuildContext { + lease, + alloc, + resources, + slot, + first, + } = *ctx; + let built = (|| -> Result> { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + Pair::new( + &alloc, + &resources.objects, + reservation.ids, + slot, + self.owner, + self.context.clone(), + free_list, + metrics, + reservation.buffer, + pm_generation, + first, + resources.config.clusters, + resources.config.descriptor_flags, + ) + })(); + let (mut built, error) = match built { + Ok(pair) => (Some(pair), None), + Err(error) => (None, Some(error)), + }; + let result = (|| { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + if firmware.queues.render.reserved(reservation)?.closed { + return Err(ECANCELED); + } + built + .as_mut() + .ok_or(EIO)? + .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; + firmware.queues.install_render(reservation, &mut built) + })(); + // Fresh backing disappears before its temporary PB reference returns. + drop(built); + if result.is_err() { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .cancel_render_reservation(reservation)?; + } + result + } + + /// Inlined into build_render_pair(): as a separate frame this step deepens the scheduler + /// run_job stack path, which is close to its budget. + #[inline(always)] + /// Off-lock preparation of a parked pair for its new owner, then its device-locked + /// commit; the pair returns to its parked state on failure. + fn install_parked_pair( + &self, + ctx: &BuildContext<'_>, + mut parked: crate::g17::queue::render::parked::ParkedPair, + buffer: (u8, u64), + pm_generation: u64, + metrics: MetricsLease, + ) -> Result { + let BuildContext { + lease, + alloc, + resources, + slot, + first, + } = *ctx; + let built = (|| -> Result<_> { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + let prepared = parked.prepare( + &alloc, + &resources.objects, + self.owner, + self.context.clone(), + free_list, + metrics, + buffer, + pm_generation, + first, + )?; + let storage = KBox::::new_uninit(GFP_KERNEL)?; + Ok((prepared, storage)) + })(); + let (mut prepared, mut storage, error) = match built { + Ok((prepared, storage)) => (Some(prepared), Some(storage), None), + Err(error) => (None, None, Some(error)), + }; + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + (|| { + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); } - self.shared.changed.notify_all(); - if grow_ready { - self.shared.queue_grow(); + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); } - result + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_owner != Some(self.owner) + || entry.pending_owner_closed + || entry.pair.is_some() + || entry.parked.is_some() + { + return Err(ECANCELED); + } + let storage = storage.take().ok_or(EIO)?; + firmware.queues.buffers.can_release(buffer.0)?; + prepared.as_mut().ok_or(EIO)?.finish_install( + &parked, + &alloc, + firmware.init.pb_descriptor_table()?, + )?; + let pair = parked.commit(&mut prepared, storage)?; + firmware.queues.buffers.reserve_owner(buffer.0)?; + firmware.queues.buffers.release(buffer.0)?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.reservation.owner = self.owner; + entry.reservation.buffer = buffer; + entry.context = Some(self.context.clone()); + entry.pool = Some(pair.pool().clone()); + entry.pair = Some(pair); + entry.closed = false; + entry.published = false; + Ok(()) + })() + }; + // Rejected candidate mappings disappear before the PB identity + // returns, while the original installed kick owner stays live. + drop(prepared); + drop(storage); + let grow_ready = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if result.is_err() { + firmware.queues.render.entry_mut(slot)?.parked = Some(parked); + firmware.queues.buffers.release(buffer.0)?; + } else { + drop(state); + drop(parked); + state = self.shared.state.lock(); } + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = None; + entry.pending_owner = None; + entry.pending_owner_closed = false; + firmware.render_grow_ready() }; - result?; + self.shared.changed.notify_all(); + if grow_ready { + self.shared.queue_grow(); + } + result + } + + /// Inlined into build_render_pair(): as a separate frame this step deepens the scheduler + /// run_job stack path, which is close to its budget. + #[inline(always)] + /// Off-lock preparation of an installed pair's rebinding, then its device-locked + /// owner change; the original pair stays installed on failure. + fn rebind_pair( + &self, + ctx: &BuildContext<'_>, + mut pair: KBox, + buffer: Option<(u8, u64)>, + pm_generation: Option, + metrics: Option, + ) -> Result { + let BuildContext { + lease, + alloc, + resources, + slot, + first, + } = *ctx; + let prepared = (|| { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + pair.prepare_bind( + &alloc, + &resources.objects, + self.owner, + self.context.clone(), + free_list, + metrics, + buffer, + pm_generation, + first, + ) + })(); + let (mut prepared, error) = match prepared { + Ok(binding) => (Some(binding), None), + Err(error) => (None, Some(error)), + }; + let (result, grow_ready) = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + let result = (|| { + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { + return Err(ECANCELED); + } + if firmware.queues.render.retired.len() + == firmware.queues.render.retired.capacity() + { + return Err(ENOSPC); + } + firmware.queues.teardown.displace( + pair.context(), + true, + super::super::now_ns(), + )?; + pair.bind_owner( + prepared.as_mut().ok_or(EIO)?, + &alloc, + firmware.init.pb_descriptor_table()?, + )?; + if let Some((id, _)) = buffer { + firmware.queues.buffers.reserve_owner(id)?; + firmware.queues.buffers.release(id)?; + } + let entry = firmware.queues.render.entry_mut(slot)?; + entry.reservation.owner = self.owner; + if let Some(buffer) = buffer { + entry.reservation.buffer = buffer; + } + entry.context = Some(self.context.clone()); + entry.pool = Some(pair.pool().clone()); + entry.closed = false; + Ok(()) + })(); + // The original installed pair remains owned on every error. + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = None; + entry.pending_owner = None; + entry.pending_owner_closed = false; + entry.pair = Some(pair); + entry.borrowed_dependencies = None; + (result, firmware.render_grow_ready()) + }; + drop(prepared); + if result.is_err() { + if let Some((id, _)) = buffer { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .buffers + .release(id)?; + } + } + self.shared.changed.notify_all(); + if grow_ready { + self.shared.queue_grow(); + } + result + } + + /// Under the device mutex: the graph is complete and retained; rebase a fresh graph's + /// cursors and warm its pool. + fn finish_render_build(&self, lease: &Lease, slot: u8, fresh_graph: bool) -> Result { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { From ef1251b62428a647c7ed4311cc6223ca1c8fc80a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:42:34 -0400 Subject: [PATCH 0674/1579] drm/asahi: split the G17 render publication into preflight, barriers and announce Pair::publish() checked the pair, assembled the kick barriers, committed the descriptors and announced the pair to the firmware in one 350-line function with two near-identical failure blocks. Move the entry checks into publication_preflight(), the barrier assembly into render_barriers(), the announcement with its failure path into announce_publication(), and the shared tail of both failure paths (terminal status, deferred failure, failed flag) into record_publication_failure(). The checks, the barrier rules and the order of every effect on failure are unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e0fa28b70ca92d7942809827a7cff7e230d68826) --- drivers/gpu/drm/asahi/g17/queue/render.rs | 193 ++++++++++++++-------- 1 file changed, 128 insertions(+), 65 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 7936d253d2411..d82ee84c1283f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -401,6 +401,47 @@ impl Pair { } } +/// Kick barriers of one render. The fragment kick always waits for its own tiling kick; +/// with the fragment-barriers flag the resolved prefix orders the fragment kick too and the +/// tiling kick keeps only its implicit parent, otherwise the prefix orders the tiling kick. +/// A compute dependency always orders the tiling kick. +struct RenderBarriers { + tiling: [KickDependency; 3], + tiling_count: usize, + fragment: [KickDependency; 3], + fragment_count: usize, +} + +fn render_barriers( + fragment_stage: bool, + dependencies: &RenderDependencies, + own_tiling: KickDependency, +) -> RenderBarriers { + let mut tiling_barriers = [KickDependency::ZERO; 3]; + let mut tiling_count = 0; + let mut fragment_barriers = [own_tiling; 3]; + let mut fragment_count = 1; + for dependency in dependencies.render() { + if fragment_stage { + fragment_barriers[fragment_count] = *dependency; + fragment_count += 1; + } else { + tiling_barriers[tiling_count] = *dependency; + tiling_count += 1; + } + } + if let Some(compute) = dependencies.compute { + tiling_barriers[tiling_count] = compute; + tiling_count += 1; + } + RenderBarriers { + tiling: tiling_barriers, + tiling_count, + fragment: fragment_barriers, + fragment_count, + } +} + impl Pair { /// All descriptor bytes and retained allocations are ready before entry. /// The first inner producer establishes firmware ownership, including on @@ -422,37 +463,9 @@ impl Pair { }, }; use core::sync::atomic::{fence, Ordering}; - if self.pending_aliases.is_some() || self.kick_aliases.is_some() { - return Err(EBUSY); - } - let (epoch, recovery_state) = host.epoch()?; - if recovery_state != 0 - || self.quarantined - || !self.active.room() - || packet.completion.render_slot() != Some(self.slot) - { - return Err(EBUSY); - } - let prepared = self.prepared.as_ref().ok_or(EINVAL)?; - if !Arc::ptr_eq(&prepared.packet, &packet) - || prepared.ordinal != self.ordinal - || !self.memory.owns(self.ordinal) - || !packet.context.is_current() - { - return Err(EIO); - } - if packet.completion.status().get() != 0 { - return Err(EIO); - } - let policy = packet.context.policy(); - // A registration of another class can only follow a rebind, which - // requires the pair to have drained. - if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { - return Err(EIO); - } + let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; - let mcache = prepared.mcache; let qids = self.qids(); let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); @@ -534,32 +547,12 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - // The fragment kick always waits for its own tiling kick. With the - // fragment-barriers flag the resolved prefix orders the fragment kick - // too, and the tiling kick keeps only its implicit parent, so vertex - // work overlaps the predecessors' fragment work as on the native stack. let Validated::Render { pass, .. } = &packet.command else { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; - let mut tiling_barriers = [KickDependency::ZERO; 3]; - let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3]; - let mut fragment_count = 1; - for dependency in dependencies.render() { - if pass.fragment_barriers { - fragment_barriers[fragment_count] = *dependency; - fragment_count += 1; - } else { - tiling_barriers[tiling_count] = *dependency; - tiling_count += 1; - } - } - if let Some(compute) = dependencies.compute { - tiling_barriers[tiling_count] = compute; - tiling_count += 1; - } - let tiling_barriers = &tiling_barriers[..tiling_count]; + let barriers = render_barriers(pass.fragment_barriers, dependencies, own_tiling); + let tiling_barriers = &barriers.tiling[..barriers.tiling_count]; let args = |stage: usize, barriers, arrays, mcache| KickArgs { qid: qids[stage], timestamp: kicks[stage], @@ -589,7 +582,7 @@ impl Pair { ); let fragment_args = args( 1, - &fragment_barriers[..fragment_count], + &barriers.fragment[..barriers.fragment_count], abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), )?, @@ -727,11 +720,7 @@ impl Pair { } self.active.back_mut().ok_or(EIO)?.qos_completed = true; self.quarantined = true; - if !packet.completion.spared() && self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), error)); - } - defer(Deferred::FailedOwned(packet.completion.clone(), error))?; - self.active.back_mut().ok_or(EIO)?.failed = true; + self.record_publication_failure(&packet, error, defer)?; return Err(error); } }; @@ -749,11 +738,89 @@ impl Pair { previous.publication = Some(ordinal); } packet.context.mark_published(); + self.announce_publication( + host, + (&fragment_slot, &tiling_slot, outer_next), + lease.free_list_generation, + policy.priority(), + &packet, + defer, + ) + } + + /// Everything that must hold before a publication changes any state: no alias work + /// pending, a healthy device epoch, room in the active ring, this slot's packet, the + /// prepared descriptors for this ordinal, a clean completion status and a registration + /// class the pair can still take. Returns the epoch, the prepared memory-cache setting + /// and the owner's policy. + fn publication_preflight( + &self, + packet: &Arc, + host: &mut impl Host, + ) -> Result<(u64, Option, crate::g17::fw::queue::Policy)> { + if self.pending_aliases.is_some() || self.kick_aliases.is_some() { + return Err(EBUSY); + } + let (epoch, recovery_state) = host.epoch()?; + if recovery_state != 0 + || self.quarantined + || !self.active.room() + || packet.completion.render_slot() != Some(self.slot) + { + return Err(EBUSY); + } + let prepared = self.prepared.as_ref().ok_or(EINVAL)?; + if !Arc::ptr_eq(&prepared.packet, packet) + || prepared.ordinal != self.ordinal + || !self.memory.owns(self.ordinal) + || !packet.context.is_current() + { + return Err(EIO); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let policy = packet.context.policy(); + // A registration of another class can only follow a rebind, which + // requires the pair to have drained. + if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { + return Err(EIO); + } + Ok((epoch, prepared.mcache, policy)) + } + + /// Records a failed publication on the retained active entry: the terminal status + /// (once), the deferred failure and the entry's failed flag. + fn record_publication_failure( + &mut self, + packet: &Arc, + error: Error, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if !packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), error)); + } + defer(Deferred::FailedOwned(packet.completion.clone(), error))?; + self.active.back_mut().ok_or(EIO)?.failed = true; + Ok(()) + } + + /// Activates the pool lease and announces the retained pair to the firmware; a failure + /// here quarantines the pair with the packet still retained for recovery. + fn announce_publication( + &mut self, + host: &mut impl Host, + (fragment_slot, tiling_slot, outer_next): (&WorkSlot, &WorkSlot, [u32; 2]), + free_list_generation: u64, + priority: u8, + packet: &Arc, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { let publish = (|| { - host.activate_render_pool(self.free_list.id(), lease.free_list_generation)?; - host.publish_retained_pair(policy.priority(), &fragment_slot, &tiling_slot, outer_next)?; + host.activate_render_pool(self.free_list.id(), free_list_generation)?; + host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; - host.notify_render(policy.priority())?; + host.notify_render(priority)?; host.note_submission() })(); if let Err(error) = publish { @@ -764,11 +831,7 @@ impl Pair { .packet .completion .quarantine_render(); - if !packet.completion.spared() && self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), error)); - } - defer(Deferred::FailedOwned(packet.completion.clone(), error))?; - self.active.back_mut().ok_or(EIO)?.failed = true; + self.record_publication_failure(packet, error, defer)?; return Err(error); } Ok(()) From f76a38f8511fd9df3d53e30b899a780c17353d28 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:42:34 -0400 Subject: [PATCH 0675/1579] drm/asahi: split the G17 compute publication preflight and queue configuration Queue::publish() checked the queue, built the descriptor arguments and the queue configuration record, and committed and announced the kick in one 330-line function. Move the entry checks into publication_preflight(), which hands back the epoch and the bound owner's per-queue addresses as values, and the configuration record into queue_config(). The commit and announcement stay one closure: its visible/accounted state decides the rollback for both phases. Checks, record contents and effect order are unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 74bc2f09ab74832be661557cacf374ee5fff11d7) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 144 +++++++++++++-------- 1 file changed, 92 insertions(+), 52 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 2b962e2632606..5347779d629d1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -392,6 +392,13 @@ struct Previous { context: Arc, publication: Option<(u64, u64)>, } +/// Per-queue addresses of the bound owner, read under the publication preflight. +struct BindingAddresses { + preempt: u64, + operand: u64, + usage: u64, +} + #[derive(Copy, Clone, PartialEq, Eq)] struct ConfigIdentity { scheduler: u64, @@ -1495,31 +1502,7 @@ impl Queue { compute::{ComputeArgs, ComputeDescriptor, COMPUTE_KICK_EVENT_MASK, COMPUTE_QOS_CLASS}, kick::{KickArgs, KickQos}, }; - let (epoch, state) = host.epoch()?; - if state != 0 { - return Err(EBUSY); - } - let replay_front = self - .replays - .front() - .is_some_and(|front| Arc::ptr_eq(front, &packet)); - if !self.room() - && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) - { - return Err(EBUSY); - } - if !packet.context.is_current() - || !self - .binding - .as_ref() - .is_some_and(|binding| Arc::ptr_eq(&packet.context, &binding.context)) - { - return Err(EFAULT); - } - let binding = self.binding.as_ref().ok_or(EFAULT)?; - if packet.completion.status().get() != 0 { - return Err(EIO); - } + let (epoch, addresses) = self.publication_preflight(&packet, host)?; let crate::g17::command::Validated::Compute { scratch, usc_base, @@ -1569,9 +1552,9 @@ impl Queue { qid, kick_count: self.ordinal, kick: kick_timestamp, - preempt_va: binding.preempt.iova(), - operand_state_va: binding.operand.iova(), - usage_va: binding.usage.iova(), + preempt_va: addresses.preempt, + operand_state_va: addresses.operand, + usage_va: addresses.usage, usage_fw_va: self.graph.usage.gpu_va(), free_list_slot: u32::from(self.pool.id()), free_list_control_va: self.pool.control_va(), @@ -1591,30 +1574,7 @@ impl Queue { self.free_list_generation = u64::from(packet.context.scheduler_generation()); let policy = packet.context.policy(); let ring = 1u8 << policy.priority(); - let config = QueueConfig::new(&QueueConfigArgs { - target: QueueConfigTarget::Compute, - kick_ring_va: self.kick.low_va(), - kick_ring_fw_va: self.kick.firmware_va(), - qid, - install: first, - completion_seed: None, - context_id: args.context_id, - free_list: FreeListBinding { - va: self.pool.control_va(), - generation: u64::from(packet.context.scheduler_generation()), - slot: u32::from(self.pool.id()), - }, - scheduler_va: identity.scheduler, - policy, - qos_slot: identity.qos, - qos_update: self - .published_config - .is_none_or(|p| p.scheduler != identity.scheduler || p.qos != identity.qos), - owner_pid: packet.context.owner_pid(), - context_update: self - .published_config - .is_none_or(|p| p.scheduler != identity.scheduler || p.context != identity.context), - })?; + let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; @@ -1834,6 +1794,86 @@ impl Queue { } Ok(()) } + + /// Everything that must hold before a publication changes any state: a healthy device + /// epoch, room in the active ring (or this packet at the replay front), the bound + /// owner's current context and a clean completion status. Returns the epoch and the + /// bound owner's per-queue addresses. + fn publication_preflight( + &self, + packet: &Arc, + host: &mut impl Host, + ) -> Result<(u64, BindingAddresses)> { + let (epoch, state) = host.epoch()?; + if state != 0 { + return Err(EBUSY); + } + let replay_front = self + .replays + .front() + .is_some_and(|front| Arc::ptr_eq(front, packet)); + if !self.room() + && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) + { + return Err(EBUSY); + } + if !packet.context.is_current() + || !self + .binding + .as_ref() + .is_some_and(|binding| Arc::ptr_eq(&packet.context, &binding.context)) + { + return Err(EFAULT); + } + let binding = self.binding.as_ref().ok_or(EFAULT)?; + if packet.completion.status().get() != 0 { + return Err(EIO); + } + Ok(( + epoch, + BindingAddresses { + preempt: binding.preempt.iova(), + operand: binding.operand.iova(), + usage: binding.usage.iova(), + }, + )) + } + + /// The queue configuration record for this publication: a fresh install on the first + /// one, and QoS/context updates only when the identity changed since the last publication. + fn queue_config( + &self, + identity: &ConfigIdentity, + first: bool, + context_id: u16, + policy: crate::g17::fw::queue::Policy, + packet: &Arc, + ) -> Result { + QueueConfig::new(&QueueConfigArgs { + target: QueueConfigTarget::Compute, + kick_ring_va: self.kick.low_va(), + kick_ring_fw_va: self.kick.firmware_va(), + qid: self.qid(), + install: first, + completion_seed: None, + context_id, + free_list: FreeListBinding { + va: self.pool.control_va(), + generation: u64::from(packet.context.scheduler_generation()), + slot: u32::from(self.pool.id()), + }, + scheduler_va: identity.scheduler, + policy, + qos_slot: identity.qos, + qos_update: self + .published_config + .is_none_or(|p| p.scheduler != identity.scheduler || p.qos != identity.qos), + owner_pid: packet.context.owner_pid(), + context_update: self + .published_config + .is_none_or(|p| p.scheduler != identity.scheduler || p.context != identity.context), + }) + } } impl Graph { From 49e9337b68a41b5f9a3a26d8d72e8b0eecc16844 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 23:13:38 -0400 Subject: [PATCH 0676/1579] drm/asahi: size the compute descriptor ring to the kicks that can be in flight Each retained compute queue carried a 4 MiB descriptor ring: 256 slots of 16 KiB, one per kick-ring slot, although at most IN_FLIGHT (16) kicks are outstanding and every kick names its descriptor by address. Index the ring by kick slot modulo 32, so a descriptor is rewritten only once the kick 32 publications later is issued, long after its own completion. The ring shrinks to 512 KiB per queue; with the fourteen queues of a desktop and a game that is 49 MiB less pinned memory. Retirement records match on descriptor address and kick timestamp together, so a reused address cannot alias an older kick. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5ad7b191f7e698e3ef140163c6ecf15d3e57ff85) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 5347779d629d1..10b66c9ae811b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -19,7 +19,9 @@ use retirement::{Cursors, RECORD_SLOTS}; use crate::g17::now_ns; -const DESCRIPTORS: usize = 256; +/// Descriptor slots per queue. A kick's descriptor is rewritten when the kick 32 slots later +/// is published; with IN_FLIGHT kicks outstanding at most, that kick has completed by then. +const DESCRIPTORS: usize = 32; const GRAPH_SIZE: usize = 0x8000; const CONTEXT_SIZE: usize = 0x20000; const RECORD_STRIDE: usize = 0x1000; @@ -231,7 +233,7 @@ impl Graph { /// EAGAIN while the queue is vacant: the owner must re-arm it first. fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { let ring = self.ring.as_ref().ok_or(EAGAIN)?; - let offset = u64::from(slot) * COMPUTE_DESCRIPTOR_SIZE as u64; + let offset = (usize::from(slot) % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE) as u64; Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) } fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { @@ -1620,7 +1622,7 @@ impl Queue { .ok_or(EAGAIN)? .object .initialize::( - kick_timestamp.slot() * COMPUTE_DESCRIPTOR_SIZE, + kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, |descriptor| descriptor.write(&args), )?; if first { From 97de25b1fc4afaf2ed16ec7c9139185317406213 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:51:11 +1100 Subject: [PATCH 0677/1579] drm/shmem: revoke CPU PFNs before releasing purge backing Revocable CPU VMAs hold PFN mappings without ordinary rmap references. Dropping either the device page array or sparse CPU page references can make those pages reclaimable while a user PTE still names them. Mark the object purged and unmap all DRM VMAs, including the TLB invalidation, before releasing either array or clearing unevictable protection. This matches the existing idle-release ordering and covers both SG-first faults and sparse CPU backing. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit e6d369606edff3c24dc3b1f5dda036cc77a8887c) From 628bf1df43123ead994b028fd7b82a209e67aa30 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 19:14:45 +1100 Subject: [PATCH 0678/1579] drm/asahi: initialize driver data before claiming destructor ownership Both GPU probe paths allocate a DRM device before creating the GPU manager. A successful no-op PinInit incorrectly claimed that AsahiData was initialized, so ordinary firmware, identity or allocation failures could run its destructor over uninitialized GPU and platform references. Add explicit two-phase allocation to the existing Rust DRM device. Keep the pinned data slot in UnsafeCell> and publish destructor ownership only after the real initializer succeeds. Use a static allocation vtable while release state itself is uninitialized. Preserve the ordinary initialized constructor and inline workqueue offsets. Convert both Asahi probes to the explicit API, avoiding references to uninitialized data. Partial initializer failures unwind their own fields; post-initialization failures retain normal data destruction. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 983575d7b492f6824401cd21fe02278812211e27) --- drivers/gpu/drm/asahi/driver.rs | 32 +++------- rust/kernel/drm/device.rs | 110 +++++++++++++++++++------------- 2 files changed, 75 insertions(+), 67 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index abdef98673f96..a50bb72a28598 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -202,15 +202,9 @@ impl platform::Driver for AsahiDriver { .property_read_array_vec(c_str!("apple,firmware-compat"), 3)? .required_by(pdev.as_ref())?; - // TODO: This is very temporary - // SAFETY: This should be safe as data is not touched by the driver - // untill it gets fully initialised. - // Additionally drm::device::Device::release() will not drop data and - // leaks instead. - let uninit = unsafe { - pin_init::pin_init_from_closure::(|_slot| Ok(())) - }; - let drm: ARef = drm::device::Device::new(pdev.as_ref(), uninit)?; + // SAFETY: GPU construction uses the DRM allocation but not its driver data. + // No userspace or driver-data work is published before init_data() succeeds. + let drm: ARef = unsafe { drm::device::Device::new_uninit(pdev.as_ref())? }; let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { @@ -246,10 +240,8 @@ impl platform::Driver for AsahiDriver { resources: res, }); - let ptr: *const AsahiData = &raw const **drm; - unsafe { - data.__pinned_init(ptr as *mut AsahiData)?; - } + // SAFETY: This is the sole initializer; driver data has not been accessed yet. + unsafe { drm.init_data(data)? }; (*drm).gpu.init()?; @@ -264,12 +256,9 @@ impl AsahiDriver { // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let res = regs::Resources::new(pdev)?; - // SAFETY: As in the legacy probe, the data is inaccessible until the fully initialized - // manager is installed below and the DRM device is registered. - // If construction fails, drm::device::Device::release leaves the data - // untouched rather than dropping an uninitialized AsahiData. - let uninit = unsafe { pin_init::pin_init_from_closure::(|_slot| Ok(())) }; - let drm: ARef = drm::device::Device::new(pdev.as_ref(), uninit)?; + // SAFETY: GPU construction cannot access driver data before init_data(). + // Failed construction releases the allocation without dropping uninitialized data. + let drm: ARef = unsafe { drm::device::Device::new_uninit(pdev.as_ref())? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res.clone())?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiData { @@ -277,9 +266,8 @@ impl AsahiDriver { pdev: pdev.into(), resources: res, }); - let ptr = &raw const **drm; - // SAFETY: The allocation is pinned and its data slot has not been initialized yet. - if let Err(error) = unsafe { data.__pinned_init(ptr as *mut AsahiData) } { + // SAFETY: This is the sole initializer; no driver-data users have been published. + if let Err(error) = unsafe { drm.init_data(data) } { gpu.shutdown(); return Err(error); } diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 6c2cd7459f2b0..e0b73af7fe2f8 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -27,12 +27,14 @@ use crate::{ }; use core::{ alloc::Layout, - mem, + cell::UnsafeCell, + mem::{self, MaybeUninit}, ops::Deref, ptr::{ self, NonNull, // }, + sync::atomic::{AtomicBool, Ordering}, }; #[cfg(CONFIG_DRM_LEGACY)] @@ -73,10 +75,13 @@ macro_rules! drm_legacy_fields { /// # Invariants /// /// `self.dev` is a valid instance of a `struct device`. +/// When `data_initialized` is true, `data` owns initialized, pinned driver data. +/// Otherwise, driver data must not be accessed. #[repr(C)] pub struct Device { dev: Opaque, - data: T::Data, + data: UnsafeCell>, + data_initialized: AtomicBool, } impl Device { @@ -111,59 +116,69 @@ impl Device { fops: &Self::GEM_FOPS, }; + const ALLOC_VTABLE: bindings::drm_driver = bindings::drm_driver { + release: None, + ..Self::VTABLE + }; + const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(T::MODULE); /// Create a new `drm::Device` for a `drm::Driver`. pub fn new(dev: &device::Device, data: impl PinInit) -> Result> { - // `__drm_dev_alloc` uses `kmalloc()` to allocate memory, hence ensure a `kmalloc()` - // compatible `Layout`. - let layout = Kmalloc::aligned_layout(Layout::new::()); - - // Use a temporary vtable without a `release` callback until `data` is initialized, so - // init failure can release the DRM device without dropping uninitialized fields. - let alloc_vtable = bindings::drm_driver { - release: None, - ..Self::VTABLE - }; + // SAFETY: The device remains private until its data is initialized below. + let drm = unsafe { Self::new_uninit(dev)? }; + // SAFETY: This is the only initializer, and no references to the data exist yet. + unsafe { drm.init_data(data)? }; + Ok(drm) + } - // SAFETY: - // - `alloc_vtable` reference remains valid until no longer used, - // - `dev` is valid by its type invarants, + /// Allocate a DRM device whose driver data will be initialized separately. + /// + /// # Safety + /// + /// The caller must not access driver data, publish the device to userspace, or + /// schedule work that accesses driver data before calling [`Self::init_data`]. + pub unsafe fn new_uninit(dev: &device::Device) -> Result> { + let layout = Kmalloc::aligned_layout(Layout::new::()); + // Allocation failure must not inspect fields that we have not initialized yet. let raw_drm: *mut Self = unsafe { bindings::__drm_dev_alloc( dev.as_raw(), - &alloc_vtable, + const { &Self::ALLOC_VTABLE }, layout.size(), mem::offset_of!(Self, dev), ) } .cast(); let raw_drm = NonNull::new(from_err_ptr(raw_drm)?).ok_or(ENOMEM)?; - - // SAFETY: `raw_drm` is a valid pointer to `Self`, given that `__drm_dev_alloc` was - // successful. + // SAFETY: Allocation succeeded, and these fields have no previous value. + unsafe { + ptr::addr_of_mut!((*raw_drm.as_ptr()).data) + .write(UnsafeCell::new(MaybeUninit::uninit())); + ptr::addr_of_mut!((*raw_drm.as_ptr()).data_initialized).write(AtomicBool::new(false)); + } + // SAFETY: The allocation is private and its release state is now initialized. let drm_dev = unsafe { Self::into_drm_device(raw_drm) }; - - // SAFETY: `raw_drm` is a valid pointer to `Self`. - let raw_data = unsafe { ptr::addr_of_mut!((*raw_drm.as_ptr()).data) }; - - // SAFETY: - // - `raw_data` is a valid pointer to uninitialized memory. - // - `raw_data` will not move until it is dropped. - unsafe { data.__pinned_init(raw_data) }.inspect_err(|_| { - // SAFETY: `__drm_dev_alloc()` was successful, hence `drm_dev` must be valid and the - // refcount must be non-zero. - unsafe { bindings::drm_dev_put(drm_dev) }; - })?; - - // SAFETY: `drm_dev` is still private to this function. unsafe { (*drm_dev).driver = const { &Self::VTABLE } }; - - // SAFETY: The reference count is one, and now we take ownership of that reference as a - // `drm::Device`. + // SAFETY: We own the initial DRM reference; uninitialized data is behind UnsafeCell. Ok(unsafe { ARef::from_raw(raw_drm) }) } + /// Initialize the pinned driver data of a device allocated by [`Self::new_uninit`]. + /// + /// # Safety + /// + /// The caller must be the only initializer, the data must be uninitialized, and + /// no driver-data users may run until this method returns successfully. + pub unsafe fn init_data(&self, data: impl PinInit) -> Result { + let raw_data = self.data.get().cast::(); + // SAFETY: The caller guarantees exclusive initialization of this pinned slot. + unsafe { data.__pinned_init(raw_data) }?; + // Publish destructor ownership only after the real initializer succeeded. + self.data_initialized.store(true, Ordering::Release); + Ok(()) + } + pub(crate) fn as_raw(&self) -> *mut bindings::drm_device { self.dev.get() } @@ -210,12 +225,13 @@ impl Device { // SAFETY: `ptr` is a valid pointer to a `struct drm_device` and embedded in `Self`. let this = unsafe { Self::from_drm_device(ptr) }; - // SAFETY: new() installs this callback only after data.__pinned_init - // succeeds. Initialization failures use the temporary release=None - // table instead. The final DRM reference excludes further data users. - // Drop only the Rust data: drm_dev_release still owns the embedded C - // device and subsequently runs its managed cleanup and final kfree. - unsafe { ptr::drop_in_place(ptr::addr_of_mut!((*this).data)) }; + // SAFETY: new_uninit() initialized this flag before enabling this callback. + // A failed initializer unwinds its own partial fields and never sets the flag. + if unsafe { (*this).data_initialized.load(Ordering::Acquire) } { + // SAFETY: Successful initialization published ownership of pinned driver data. + // DRM still owns the embedded C device and its final managed cleanup/kfree. + unsafe { ptr::drop_in_place((*this).data.get().cast::()) }; + } } } @@ -223,7 +239,10 @@ impl Deref for Device { type Target = T::Data; fn deref(&self) -> &Self::Target { - &self.data + debug_assert!(self.data_initialized.load(Ordering::Acquire)); + // SAFETY: Safe constructors initialize data; new_uninit() requires callers + // to prevent data access until init_data() succeeds. + unsafe { &*self.data.get().cast::() } } } @@ -289,7 +308,7 @@ where { unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work, ID> { // SAFETY: The caller promises that `ptr` points to a valid `Device`. - let data_ptr = unsafe { &raw mut (*ptr).data }; + let data_ptr = unsafe { (&raw mut (*ptr).data).cast::() }; // SAFETY: `data_ptr` is a valid pointer to `T::Data`. unsafe { T::Data::raw_get_work(data_ptr) } @@ -298,7 +317,8 @@ where unsafe fn work_container_of(ptr: *mut Work, ID>) -> *mut Self { // SAFETY: The caller promises that `ptr` points at a `Work` field in // `T::Data`. - let data_ptr = unsafe { T::Data::work_container_of(ptr) }; + let data_ptr = unsafe { T::Data::work_container_of(ptr) } + .cast::>>(); // SAFETY: `T::Data` is stored as the `data` field in `Device`. unsafe { crate::container_of!(data_ptr, Self, data) } From f7eb0fe03cf65ebe9046542fdb3c617ccd1eb3a7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:32:03 +1100 Subject: [PATCH 0679/1579] drm: apple: read H17P blend CRCs through the outbound queue Linux measurements show that the H17P blend CRC command takes no input and returns one little-endian 32-bit value. The existing completion path instead supplies a constant zero to the DRM CRC interface. Queue one read for each validated present completion while CRC capture is enabled. Put it before the next pending present so the sample refers to the completed frame, and send it only after the callback is acknowledged and the transport is idle. Use the existing transaction timeout and crash handling, reject malformed replies, and discard queued reads on shutdown. Disabling capture suppresses both unsent requests and returned samples. Older firmware and the H17G method profile retain their current behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 2 +- drivers/gpu/drm/apple/iomfb.c | 69 ++++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 8 ++- 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 15945de4bcd22..af507fde376af 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -420,6 +420,7 @@ int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); +void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id); void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e0082be652f3d..3ceb9f809e262 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -887,7 +887,7 @@ int dcp_set_crc(struct drm_crtc *crtc, bool enabled) struct apple_crtc *ac = to_apple_crtc(crtc); struct apple_dcp *dcp = platform_get_drvdata(ac->dcp); - dcp->crc_enabled = enabled; + WRITE_ONCE(dcp->crc_enabled, enabled); return 0; } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index fe6bcb9703366..b3d14d946aff0 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -408,6 +409,74 @@ int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) return ret; } +struct iomfb_crc_transaction { + struct iomfb_transaction transaction; + u32 swap_id; +}; + +static void iomfb_crc_complete(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct iomfb_crc_transaction *crc = cookie; + struct dcp_packet_header *header; + u32 value; + + /* A106 has no input and a single little-endian blend-output CRC. */ + if (!out) + goto invalid; + header = out - sizeof(*header); + if (header->in_len || header->out_len != sizeof(__le32)) + goto invalid; + value = get_unaligned_le32(out); + if (READ_ONCE(dcp->crc_enabled)) + drm_crtc_add_crc_entry(&dcp->crtc->base, true, crc->swap_id, &value); + return; + +invalid: + dev_err(dcp->dev, "invalid blend CRC response\n"); + WRITE_ONCE(dcp->crashed, true); +} + +static void iomfb_crc_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + static const struct dcp_method_entry method = { + .tag = { 'A', '1', '0', '6' }, + .name = "read_blend_crc", + }; + struct iomfb_crc_transaction *crc = container_of(transaction, + struct iomfb_crc_transaction, + transaction); + + if (READ_ONCE(dcp->crc_enabled)) + dcp_push(dcp, false, &method, 0, sizeof(__le32), NULL, + iomfb_crc_complete, crc); +} + +static void iomfb_crc_release(struct iomfb_transaction *transaction) +{ + kfree(container_of(transaction, struct iomfb_crc_transaction, transaction)); +} + +void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id) +{ + struct iomfb_crc_transaction *crc; + + lockdep_assert_held(&dcp->iomfb.lock); + if (!READ_ONCE(dcp->crc_enabled) || READ_ONCE(dcp->crashed) || + dcp->iomfb.stopped || dcp->iomfb.queued >= 32) + return; + crc = kzalloc_obj(*crc); + if (!crc) + return; + crc->swap_id = swap_id; + crc->transaction.start = iomfb_crc_start; + crc->transaction.release = iomfb_crc_release; + /* Read the completed frame before another queued present changes it. */ + list_add(&crc->transaction.link, &dcp->iomfb.pending); + dcp->iomfb.queued++; + /* The receiver advances the queue only after acknowledging completion. */ +} + struct iomfb_command { struct iomfb_transaction transaction; struct dcp_method_entry method; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index a75ffb818edd4..664077919b5d3 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -265,7 +265,13 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, dcp_release_retained_framebuffers(dcp, resp->swap_id); dcp_drm_crtc_page_flip(dcp, now); } - if (dcp->crc_enabled) { + if (READ_ONCE(dcp->crc_enabled)) { +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + iomfb_queue_crc_h17p(dcp, resp->swap_id); + return; + } +#endif u32 crc32 = 0; drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); } From 5eff5323da0282c8fb9adfc57acdc9217e6d7995 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:32:38 +1100 Subject: [PATCH 0680/1579] soc: apple: mailbox: drain only a present send-empty IRQ PR 65 permits polling when the send-empty IRQ is absent. When composed with the newer provider removal fence, synchronize_irq received that negative optional-resource sentinel even though no IRQ was requested. Drain the TX handler only when its IRQ exists, preserving the RX drain and TX waiter wakeup in both modes. Also return the actual probe IRQ lookup error, including EPROBE_DEFER, while retaining ENXIO as the accepted absent optional TX line. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit c127de8760a8b66cee210702d6cd16d21959e448) --- drivers/soc/apple/mailbox.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 0a9f051a6d9bd..2e22c78f57d69 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -560,7 +560,8 @@ static void apple_mbox_quiesce(void *data) } spin_unlock_irqrestore(&mbox->tx_lock, flags); /* The handler takes tx_lock, so drain it after releasing the lock. */ - synchronize_irq(mbox->irq_send_empty); + if (mbox->irq_send_empty >= 0) + synchronize_irq(mbox->irq_send_empty); complete_all(&mbox->tx_empty); synchronize_irq(mbox->irq_recv_not_empty); @@ -716,7 +717,7 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->irq_recv_not_empty = platform_get_irq_byname(pdev, "recv-not-empty"); if (mbox->irq_recv_not_empty < 0) - return -ENODEV; + return mbox->irq_recv_not_empty; /* * Some coprocessors (e.g. the T6021 ANE ASC) have no send-empty line. @@ -725,7 +726,7 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->irq_send_empty = platform_get_irq_byname_optional(pdev, "send-empty"); if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) - return -ENODEV; + return mbox->irq_send_empty; spin_lock_init(&mbox->rx_lock); spin_lock_init(&mbox->tx_lock); From ae4f6c044821a6c76c2d29b0ab1ea171fec7eab4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:47:57 +1100 Subject: [PATCH 0681/1579] drm/apple: retain the guarded fbdev probe in the display composition Keep the probe and driver callback guarded by CONFIG_DRM_FBDEV_EMULATION when composing the older dual-stream branches with the newer framebuffer helper fix. Enabled and disabled configurations retain the intended DRM helper behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 38c8e25a88675..be8da0dd4915d 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -66,6 +66,7 @@ static int apple_drm_gem_dumb_create(struct drm_file *file_priv, return drm_gem_dma_dumb_create_internal(file_priv, drm, args); } +#ifdef CONFIG_DRM_FBDEV_EMULATION static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, struct drm_fb_helper_surface_size *sizes) { @@ -86,6 +87,7 @@ static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, return drm_fbdev_dma_driver_fbdev_probe(helper, sizes); } +#endif /* * A compositor or boot splash is closing the device. drm_release() has @@ -100,7 +102,11 @@ static void apple_drm_postclose(struct drm_device *drm, struct drm_file *file) static const struct drm_driver apple_drm_driver = { DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), +#ifdef CONFIG_DRM_FBDEV_EMULATION .fbdev_probe = apple_drm_fbdev_probe, +#else + DRM_FBDEV_DMA_DRIVER_OPS, +#endif .postclose = apple_drm_postclose, .name = DRIVER_NAME, .desc = DRIVER_DESC, From b10d60ce3916c44dfb19b1f1b197bdca40bbac34 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:49:30 +1100 Subject: [PATCH 0682/1579] drm/asahi: retain two-phase data ownership with stats registration Keep the reviewed new_uninit/init_data construction sequence when composing per-device statistics with the GPU probe fixes. The stats registration and teardown retain their owned snapshot, while failed GPU construction cannot publish ownership of an uninitialized data slot. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/driver.rs | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 1b59a4fcebfdf..ffa01c6bb60d6 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -210,15 +210,9 @@ impl platform::Driver for AsahiDriver { .property_read_array_vec(c_str!("apple,firmware-compat"), 3)? .required_by(pdev.as_ref())?; - // TODO: This is very temporary - // SAFETY: This should be safe as data is not touched by the driver - // untill it gets fully initialised. - // Additionally drm::device::Device::release() will not drop data and - // leaks instead. - let uninit = unsafe { - pin_init::pin_init_from_closure::(|_slot| Ok(())) - }; - let drm: ARef = drm::device::Device::new(pdev.as_ref(), uninit)?; + // SAFETY: GPU construction uses the DRM allocation but not its driver data. + // No userspace or driver-data work is published before init_data() succeeds. + let drm: ARef = unsafe { drm::device::Device::new_uninit(pdev.as_ref())? }; let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { @@ -254,10 +248,8 @@ impl platform::Driver for AsahiDriver { resources: res, }); - let ptr: *const AsahiData = &raw const **drm; - unsafe { - data.__pinned_init(ptr as *mut AsahiData)?; - } + // SAFETY: This is the sole initializer; driver data has not been accessed yet. + unsafe { drm.init_data(data)? }; (*drm).gpu.init()?; @@ -283,12 +275,9 @@ impl AsahiDriver { // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let res = regs::Resources::new(pdev)?; - // SAFETY: As in the legacy probe, the data is inaccessible until the fully initialized - // manager is installed below and the DRM device is registered. - // If construction fails, drm::device::Device::release leaves the data - // untouched rather than dropping an uninitialized AsahiData. - let uninit = unsafe { pin_init::pin_init_from_closure::(|_slot| Ok(())) }; - let drm: ARef = drm::device::Device::new(pdev.as_ref(), uninit)?; + // SAFETY: GPU construction cannot access driver data before init_data(). + // Failed construction releases the allocation without dropping uninitialized data. + let drm: ARef = unsafe { drm::device::Device::new_uninit(pdev.as_ref())? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res.clone())?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiData { @@ -296,9 +285,8 @@ impl AsahiDriver { pdev: pdev.into(), resources: res, }); - let ptr = &raw const **drm; - // SAFETY: The allocation is pinned and its data slot has not been initialized yet. - if let Err(error) = unsafe { data.__pinned_init(ptr as *mut AsahiData) } { + // SAFETY: This is the sole initializer; no driver-data users have been published. + if let Err(error) = unsafe { drm.init_data(data) } { gpu.shutdown(); return Err(error); } From 19d6664ea33affcceef42d5deef488ce1ba7d50d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:50:29 +1100 Subject: [PATCH 0683/1579] arm64: dts: apple: retain a single J700 PMP alias The peripheral description and PMP integration both provide the same alias. Keep one definition when composing the two series so the J700 DTB builds without a duplicate-property error. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index c4438cc6851e9..05dff1967b376 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -17,7 +17,6 @@ model = "Apple MacBook Neo"; aliases { - pmp = &pmp; nvram = &nvram; spi0 = &qspi; i2c2 = &i2c2; From 42ae23434d4cb42ac1df562b9734db13cedb962d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:57:47 +1100 Subject: [PATCH 0684/1579] soc: apple: compose SEP persistence with authenticated sensor sessions Retain the shared key-store profiles, persistence fields, suspend capture handling and module declarations when adding J700 nonce authentication. Choose the existing communication initialization word for each protocol and keep the qualified older-platform sensor-registration path alongside the authenticated J700 path. The composition uses the public SEP work in pull requests #42, #69, #152 and #202 and their reviewed repairs. Contributor credit includes Chris Kearney, DjDeveloperr, Vincent Fontaine and Ryan Murray. Ryan integrates the existing contributions under DCO (b); original commits and author sign-offs are unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 2 + drivers/soc/apple/Makefile | 3 +- drivers/soc/apple/bio.rs | 23 ++ drivers/soc/apple/p256_shim.c | 1 - drivers/soc/apple/pm.rs | 97 +++++++ drivers/soc/apple/pm_shim.c | 58 ++++ drivers/soc/apple/profile.rs | 167 +++++++++++- drivers/soc/apple/sbio.rs | 462 ++++++++++++++++++++++++++++---- drivers/soc/apple/sensor_shim.c | 1 - drivers/soc/apple/sep-bio.h | 130 +++++++++ drivers/soc/apple/sep.rs | 148 ++++++++-- drivers/soc/apple/shim.h | 5 + drivers/soc/apple/sks.rs | 43 +++ drivers/soc/apple/xarm.rs | 6 + drivers/soc/apple/xart_store.rs | 6 + 15 files changed, 1060 insertions(+), 92 deletions(-) create mode 100644 drivers/soc/apple/pm.rs create mode 100644 drivers/soc/apple/pm_shim.c create mode 100644 drivers/soc/apple/sep-bio.h diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 281e001631fb0..2bbc21c765d49 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -133,6 +133,8 @@ config APPLE_SEP opt-in is accepted only when the driver validates narrow physical extent ownership invariants; this does not prove all macOS-compatible update behavior. Keep it off until the target machine has been checked. + With writes off the driver still attaches, but refuses Touch ID enrol + and verify: the enclave saves anti-replay state after every match. Say Y here if you have an Apple silicon Mac. diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 76aa178f98f75..064e65d0270b9 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -26,6 +26,7 @@ obj-$(CONFIG_APPLE_SEP_FV_INTERFACE) += fv_shim.o obj-$(CONFIG_APPLE_SEP) += apple-sep.o apple-sep-y := sep.o hwrng_shim.o store_shim.o bio_shim.o sha_shim.o \ - crypto_shim.o sensor_shim.o p256_shim.o trusted_shim.o work_shim.o + crypto_shim.o sensor_shim.o p256_shim.o trusted_shim.o work_shim.o \ + pm_shim.o obj-$(CONFIG_APPLE_PMP) += pmp.o diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 464022c681f1f..13d7e99d43fc9 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -927,6 +927,29 @@ fn cancel(ctx: &mut Context<'_>) -> Result { }) } +/// Ends a live enrolment or verify the way a CANCEL does, for the system going +/// to sleep: userspace sees a failure it can retry after resume. Returns +/// whether a waiter needs waking. +pub(crate) fn abort_capture(session: &mut Session) -> bool { + match &mut session.op { + Op::Enrol { terminal, .. } if terminal.is_none() => { + *terminal = Some(EnrolOutcome::Failed(STATUS_LOCAL)) + } + Op::Verify { terminal, .. } if terminal.is_none() => { + *terminal = Some(VerifyOutcome::Failed(STATUS_LOCAL)) + } + _ => return false, + } + session.token = None; + session.unseen = true; + true +} + +/// The ioctls that start a sensor capture. +pub(crate) fn starts_capture(cmd: u32) -> bool { + cmd == IOC_ENROL_START || cmd == IOC_VERIFY_START +} + fn delete(ctx: &mut Context<'_>, user: UserPtr) -> Result { require_admin()?; let request: Delete = UserSlice::new(user, core::mem::size_of::()) diff --git a/drivers/soc/apple/p256_shim.c b/drivers/soc/apple/p256_shim.c index bcf1323e3c58f..837a2e854faf4 100644 --- a/drivers/soc/apple/p256_shim.c +++ b/drivers/soc/apple/p256_shim.c @@ -161,4 +161,3 @@ int sep_p256_sender(const void *peer_pub_be, void *eph_pub_be_out, { return p256_agree(NULL, peer_pub_be, eph_pub_be_out, shared_be_out); } - diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs new file mode 100644 index 0000000000000..27b120d8f77ee --- /dev/null +++ b/drivers/soc/apple/pm.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! System sleep. A Touch ID capture runs on a non-freezable workqueue and keeps +//! talking to the sensor over SPI, so one left running into a suspend is still +//! active while devices go down. A J313 that suspended with the lock screen's +//! capture in flight did not wake. Before the system suspends, end any capture +//! in progress and hold new ones until it has resumed. The notifier itself +//! is in `pm_shim.c`; the capture handling is on [`SepData`]. + +use core::sync::atomic::{AtomicPtr, Ordering}; + +use kernel::prelude::*; +use kernel::sync::Arc; +use kernel::types::ForeignOwnable; + +use crate::SepData; + +/// The notifier has no context pointer, so `SepData` is reached through this +/// global, as in `trusted.rs`. Holds the [`ForeignOwnable::into_foreign`] +/// pointer while registered; `null` otherwise. +static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); + +extern "C" { + fn sep_pm_register(event: unsafe extern "C" fn(bool) -> c_int) -> c_int; + fn sep_pm_unregister(); +} + +/// # Safety +/// Called by `pm_shim.c` from the PM notifier chain, in process context. +unsafe extern "C" fn pm_event(entering: bool) -> c_int { + let ptr = SEP.load(Ordering::Acquire); + if ptr.is_null() { + return 0; + } + // SAFETY: produced by `into_foreign` in `register`, and reclaimed by + // `unregister` only after `sep_pm_unregister` has waited out any running + // notifier call, so it is live for this borrow. + let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; + if entering { + if let Err(error) = sep.sleep_prepare() { + // A failing notifier is not included in the PM chain's rollback. + sep.sleep_finished(); + return error.to_errno(); + } + } else { + sep.sleep_finished(); + } + 0 +} + +pub(crate) fn register(sep: Arc) -> Result<()> { + let ptr = sep.into_foreign(); + + if SEP + .compare_exchange( + core::ptr::null_mut(), + ptr, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_err() + { + // SAFETY: `ptr` was just produced by `into_foreign` and never published. + drop(unsafe { as ForeignOwnable>::from_foreign(ptr) }); + return Err(EBUSY); + } + + // SAFETY: the shim stores the callback and registers a notifier block + // that lives for the module's lifetime. + let rc = unsafe { sep_pm_register(pm_event) }; + if rc != 0 { + let raw = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); + if !raw.is_null() { + // SAFETY: `raw` is the pointer we published; no notifier is wired. + drop(unsafe { as ForeignOwnable>::from_foreign(raw) }); + } + return Err(Error::from_errno(rc)); + } + Ok(()) +} + +/// Safe to call even if [`register`] never ran. Unregisters the notifier, +/// which waits for a running call to finish, before reclaiming the `Arc`. +pub(crate) fn unregister() { + if SEP.load(Ordering::Acquire).is_null() { + return; + } + // SAFETY: no preconditions; matched with `sep_pm_register` above. + unsafe { sep_pm_unregister() }; + + let ptr = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); + if !ptr.is_null() { + // SAFETY: `ptr` came from `into_foreign` in `register`, and the + // notifier can no longer call `pm_event`. + drop(unsafe { as ForeignOwnable>::from_foreign(ptr) }); + } +} diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c new file mode 100644 index 0000000000000..34dd2b3dcf50d --- /dev/null +++ b/drivers/soc/apple/pm_shim.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * System-sleep notifier for the SEP driver. It only forwards "about to sleep" + * and "back from sleep" to Rust, which ends any Touch ID capture before the + * system suspends; see pm.rs. + */ +#include +#include +#include + +#include "shim.h" + +static int (*sep_pm_event)(bool entering); + +static int sep_pm_notify(struct notifier_block *nb, unsigned long action, + void *data) +{ + int (*event)(bool entering) = READ_ONCE(sep_pm_event); + + if (!event) + return NOTIFY_DONE; + + switch (action) { + case PM_SUSPEND_PREPARE: + case PM_HIBERNATION_PREPARE: + case PM_RESTORE_PREPARE: + return notifier_from_errno(event(true)); + case PM_POST_SUSPEND: + case PM_POST_HIBERNATION: + case PM_POST_RESTORE: + event(false); + return NOTIFY_OK; + default: + return NOTIFY_DONE; + } +} + +static struct notifier_block sep_pm_nb = { + .notifier_call = sep_pm_notify, +}; + +int sep_pm_register(int (*event)(bool entering)) +{ + int ret; + + WRITE_ONCE(sep_pm_event, event); + ret = register_pm_notifier(&sep_pm_nb); + if (ret) + WRITE_ONCE(sep_pm_event, NULL); + return ret; +} + +/* Returns only once no notifier call is still running. */ +void sep_pm_unregister(void) +{ + unregister_pm_notifier(&sep_pm_nb); + WRITE_ONCE(sep_pm_event, NULL); +} diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 91ad39a45b6d0..7d8ced489a4a2 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -9,14 +9,18 @@ //! here once. The driver reads addresses from the device tree, never scans for //! them, and never patches properties at runtime. //! -//! Three bring-up targets are modelled: +//! Five bring-up targets are modelled: //! //! * `T8103` / J313 (MacBook Air, M1): the host boots the SEP with the boot //! endpoint handshake over a 0x30000 shared-memory window. +//! * `T6000` / J316s (MacBook Pro 16", M1 Pro): the T8103 boot handshake and +//! key store, with the sensor on the T6020 SPI2 address. //! * `T6020` / J414s (MacBook Pro 14", M2 Pro): the driver does the warm //! single-message registration over a 0x40000 window. //! * `T8140` / J700 (MacBook Neo, A18 Pro): iBoot boots sepOS before the AP //! OS, so the driver uses the warm registration path. +//! * `T8112` / J413, J415 (MacBook Air, M2): iBoot hands over a running SEP and +//! a powered sensor, so the driver uses the warm registration path. // The boot handshake, identity source, sensor and DART fields are consumed by // the boot-endpoint, identity and sensor-transport paths. @@ -89,22 +93,35 @@ pub(crate) struct SensorProfile { pub(crate) capture_qualified: bool, } -/// Identity keybag `CREATE_KEYBAG` field encoding. The strict T8103 enclave -/// reads the request's first word as the bag type and rejects the value the -/// lenient T6020 enclave accepts there; the strict enclave takes the type in -/// the third word instead. Kept per-SoC so the strict path is correct without -/// disturbing the proven T6020 encoding (hardware-verified for enrol, match, -/// and reboot). +/// Identity keybag `CREATE_KEYBAG` field encoding. The first word is the +/// codec's struct version: the 13.5 enclave (T8103) takes versions 0-2 only +/// and macOS creates an identity with version 2, while the T6020 enclave takes +/// variant 5. Kept per-SoC so the T8103 path is correct without disturbing the +/// proven T6020 encoding (hardware-verified for enrol, match, and reboot). pub(crate) struct KeybagCreate { - /// First word: request variant, echoed back in the reply. + /// First word: struct version (request variant), echoed back in the reply. pub(crate) variant: u32, - /// Third word: bag type. Identity is `0x20000` on the strict path; `0` on - /// T6020, which distinguishes the bag by the variant word instead. + /// Third word: bag type. Identity is `0x400000` on 13.5; `0` on T6020, + /// which distinguishes the bag by the variant word instead. pub(crate) bag_type: u32, /// Fourth word: create argument / parent handle. pub(crate) arg: i32, } +/// Which key-store protocol the enclave speaks. It follows the sepOS the +/// firmware hands the SEP, not the SoC alone: the T8103 stub boots macOS 13.5 +/// (22G74)'s sepOS, whose request shapes are those of the 13.5 +/// `AppleSEPKeyStore`; the T6020 sepOS is newer. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum KeyStore { + /// macOS 13.5's key store. Before any other request the endpoint is + /// initialised as `AppleKeyStore::init_sep_endpoint` does it: `0x4d`, then + /// `set_env` carrying the ADT's `/defaults` `cpx-encryption-mode`. + Sepos13 { cpx_encryption_mode: u32 }, + /// The hardware-verified T6020 encoding (variant-5 create). + Variant5, +} + pub(crate) struct PlatformProfile { pub(crate) name: &'static str, /// Capacity of the boot shared-memory window. The layout is still checked @@ -127,6 +144,10 @@ pub(crate) struct PlatformProfile { pub(crate) keybag_create: KeybagCreate, /// Require nonce authentication and signed sensor material before capture. pub(crate) nonce_session: bool, + /// Key-store protocol generation (see [`KeyStore`]). + pub(crate) key_store: KeyStore, + /// Platforms with reboot-tested owner export and user-before-master saves. + pub(crate) persistent_enrol: bool, } const T8103: PlatformProfile = PlatformProfile { @@ -148,12 +169,55 @@ const T8103: PlatformProfile = PlatformProfile { dart_range_required: false, wide_dma_mask: false, firmware_region: c"sepfw", - // Strict enclave: the type goes in the third word; the first word is 0. + // macOS 13.5's AppleKeyStore::identity_create (0xfffffe000994b2d8): + // version 2, type 0x400000, parent -1 (it accepts only -1 or <= -10). + keybag_create: KeybagCreate { + variant: 2, + bag_type: 0x40_0000, + arg: -1, + }, + // The j293 and j313 ADTs both carry `/defaults` `cpx-encryption-mode = 2`. + key_store: KeyStore::Sepos13 { + cpx_encryption_mode: 2, + }, + persistent_enrol: true, + nonce_session: false, +}; + +/// M1 Pro (J316s). The M1 family cold-boots its SEP like T8103, so the boot +/// path, shared-memory geometry and key store follow T8103. The sensor sits on +/// SPI2 at the T6020 address, in the mode T6020 verified: the J313, J316s and +/// J414s platform device trees describe the same sensor (id 0x3352) +/// identically. Enrolment persistence is reboot-tested on J314s. +const T6000: PlatformProfile = PlatformProfile { + name: "T6000/J316s", + shmem_capacity: 0x3_0000, + shmem_first_item: b"CNIP", + bootstrap: Bootstrap::Boot, + identity: IdentitySource::Chosen, + sensor: SensorProfile { + controller_base: 0x3_9b10_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode2, + expected_id: 0x3352, + capture_qualified: false, + }, + dart_range_required: false, + wide_dma_mask: false, + firmware_region: c"sepfw", keybag_create: KeybagCreate { - variant: 0, - bag_type: 0x20000, - arg: 0, + variant: T8103.keybag_create.variant, + bag_type: T8103.keybag_create.bag_type, + arg: T8103.keybag_create.arg, }, + // The J316s ADT carries `/defaults` `cpx-encryption-mode = 2`. + key_store: KeyStore::Sepos13 { + cpx_encryption_mode: 2, + }, + persistent_enrol: true, nonce_session: false, }; @@ -182,6 +246,8 @@ const T6020: PlatformProfile = PlatformProfile { bag_type: 0, arg: -1, }, + key_store: KeyStore::Variant5, + persistent_enrol: true, nonce_session: false, }; @@ -189,6 +255,8 @@ const T6020: PlatformProfile = PlatformProfile { /// (0x3356), and a SEP DART aperture above 4 GiB. These values describe that /// board; device-tree nodes remain authoritative for addresses and resources. const T8140: PlatformProfile = PlatformProfile { + key_store: KeyStore::Variant5, + persistent_enrol: true, name: "T8140/J700", shmem_capacity: 0x4_0000, shmem_first_item: b"CINP", @@ -217,6 +285,48 @@ const T8140: PlatformProfile = PlatformProfile { nonce_session: true, }; +/// MacBook Air M2 (J413, J415). iBoot leaves the SEP running and the Mesa +/// sensor powered (J415: AP GPIO 178 is a driven-high output at handoff), so +/// this is a warm-registration platform like T6020. The J415 ADT describes the +/// sensor exactly as J413's does: spi2 at the T8103 address, power on AP GPIO +/// 178, data-ready on 177, 8 MHz, chip select 0. The SEP DART aperture is +/// 0x4000 + 0xffff0000, below 4 GiB, so no wide DMA mask. `/defaults` +/// carries `cpx-encryption-mode = 2`. +/// +/// The shared-memory geometry and key-store dialect follow the other +/// warm-registration platforms (T6020, T8140). +const T8112: PlatformProfile = PlatformProfile { + name: "T8112/J415", + shmem_capacity: 0x4_0000, + shmem_first_item: b"CINP", + bootstrap: Bootstrap::WarmRegister, + identity: IdentitySource::Chosen, + sensor: SensorProfile { + controller_base: 0x2_3510_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode1, + expected_id: 0x3352, + capture_qualified: false, + }, + dart_range_required: true, + wide_dma_mask: false, + firmware_region: c"sepfw", + // J415 accepted the 13.5 GET_CAPABILITIES/SET_ENV init but refused the + // strict CREATE_KEYBAG (mailbox status -1, empty reply), as J700 did; + // use the T6020 form like the other warm-registration platforms. + keybag_create: KeybagCreate { + variant: 5, + bag_type: 0, + arg: -1, + }, + key_store: KeyStore::Variant5, + persistent_enrol: true, + nonce_session: false, +}; + static_assert!(T8103.shmem_capacity == 0x30000); static_assert!(T6020.shmem_capacity == 0x40000); static_assert!(T8140.shmem_capacity == 0x40000); @@ -224,6 +334,11 @@ static_assert!(T8103.sensor.controller_base == 0x235108000); static_assert!(T6020.sensor.controller_base == 0x39b108000); static_assert!(T8140.sensor.controller_base == 0x385108000); static_assert!(T8103.sensor.expected_id == T6020.sensor.expected_id); +static_assert!(T6000.shmem_capacity == T8103.shmem_capacity); +static_assert!(T6000.sensor.controller_base == T6020.sensor.controller_base); +static_assert!(T6000.sensor.expected_id == T6020.sensor.expected_id); +static_assert!(T8112.shmem_capacity == T6020.shmem_capacity); +static_assert!(T8112.sensor.controller_base == T8103.sensor.controller_base); /// Whether the machine root declares `compatible`. /// @@ -262,8 +377,32 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { if machine_has(b"apple,t6020") { return Ok(&T6020); } + // t6020.dtsi is defined as a cut-down t6021: it includes t6021.dtsi and + // disables the parts the smaller die lacks. Both pull in t602x-die0.dtsi, + // so the SEP, its DART and the SPI controller sit at the same addresses + // with the same interrupts on either part, and the T6020 constants apply + // to the M2 Max unchanged. + if machine_has(b"apple,t6021") { + return Ok(&T6020); + } + if machine_has(b"apple,t6000") { + return Ok(&T6000); + } + // t6000.dtsi is defined as a cut-down t6001: it includes t6001.dtsi and + // deletes the parts the smaller die lacks. Both pull in t600x-die0.dtsi, + // so the SEP, its mailbox, its DART and the SPI controller sit at the same + // addresses with the same interrupts on either part, and the T6000 + // constants apply to the M1 Max unchanged. The Mac Studio (j375c) is also + // a t6001 but leaves the SEP disabled, so nothing binds there; apple,t6002 + // (M1 Ultra, no built-in sensor) stays unmapped. + if machine_has(b"apple,t6001") { + return Ok(&T6000); + } if machine_has(b"apple,t8140") { return Ok(&T8140); } + if machine_has(b"apple,t8112") { + return Ok(&T8112); + } Err(ENODEV) } diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index fd8bbe36ff764..2a19dd2ac216d 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -36,6 +36,37 @@ impl SepData { } } + // A match step whose refusal ends the verify. libfprint reports every + // failed verify as the same unspecified error, so the enclave's exact + // answer has to be recorded here or nowhere. Success is decided as in + // `sbio_call`. + fn sbio_verify_step(&self, op: &crate::sbio::SbioOp) -> Option> { + let done = match self.sbio_transfer(op) { + Ok(done) => done, + Err(e) => { + dev_err!( + self.dev, + "verify: {} transfer failed ({:?})\n", + op.name(), + e + ); + return None; + } + }; + match done.status.answered() { + Some(err) if err as u16 == crate::sbio::SBIO_STATUS_OK => Some(done.payload), + _ => { + dev_err!( + self.dev, + "verify: {} answered status {}\n", + op.name(), + done.status + ); + None + } + } + } + pub(crate) fn sbio_call(&self, op: &crate::sbio::SbioOp) -> SbioOutcome { let done = match self.sbio_transfer(op) { Ok(done) => done, @@ -368,9 +399,11 @@ impl SepData { } } - // J700 needs the owner component even when state 0x3 has no - // save-pending bit. Retain the opt-in policy on older profiles. - if self.profile.nonce_session || *module_parameters::j414s_persistent_enrol.value() != 0 { + // The owner Catacomb must be exported even though state 0x3 has no + // save-pending bit. J313, J414s and J314s each restored a fresh + // enrollment after this export and user-before-master completion. + // Every profile now does this. + if self.profile.persistent_enrol || *module_parameters::j414s_persistent_enrol.value() != 0 { if owner_initial_state == Some(0x3) && !self.save_catacomb( crate::sbio::CatacombUser::OWNER, @@ -759,7 +792,7 @@ impl SepData { } fn save_all_components(&self, user: crate::sbio::UserId) -> bool { - if self.profile.nonce_session || *module_parameters::j414s_persistent_enrol.value() != 0 { + if self.profile.persistent_enrol || *module_parameters::j414s_persistent_enrol.value() != 0 { return self.save_all_components_user_first(user); } // On m2fix11, a completed enrollment marked both master and user 0x7, @@ -826,9 +859,8 @@ impl SepData { dev_err!(self.dev, "enrol: opt-in completion has no owner Catacomb file\n"); return false; } - // Post-match persistence saves the user before the master because the - // user save can change system material. Test that order for a completed - // J414s enrollment, without changing the default path on other Macs. + // Saving the user can change system material. Persist it before the + // master, as in the reboot-tested J313/J414s enrollment paths. for (who, kind, what) in [ ( crate::sbio::CatacombUser::enrolling(user), @@ -1004,14 +1036,20 @@ impl SepData { self.register_ool(&self.ool_sbio)?; self.sbio_ready.store(true, Relaxed); - if self.profile.nonce_session { - let word = 6u32; - // Communication must be initialized before restoring biometric state. + let init_word = if self.profile.nonce_session { + Some(6u32) + } else if matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }) { + Some(1u32) + } else { + None + }; + if let Some(word) = init_word { let result = self.sbio_transfer_raw( 0x73, c"INIT_SBIO_COMMUNICATION", &word.to_le_bytes(), ); match result { - Ok(done) if done.status.is_ok() => { + Ok(done) if done.status.is_ok() + && (self.profile.nonce_session || done.payload.is_empty()) => { dev_info!(self.dev, "sbio: communication initialization accepted\n"); } Ok(done) => { @@ -1022,7 +1060,7 @@ impl SepData { } Err(e) => { self.sbio_ready.store(false, Relaxed); - dev_err!(self.dev, "sbio: communication init transport failed: {:?}\n", e); + dev_err!(self.dev, "sbio: communication init transport failed {:?}; aborting Touch ID setup\n", e); return Err(e); } } @@ -1048,6 +1086,10 @@ impl SepData { return; } let Some(patch) = self.wake_sensor() else { + dev_warn!( + self.dev, + "sbio: the sensor did not wake at attach; the next verify retries the bring-up\n" + ); return; }; if !self.complete_bringup(patch) { @@ -1116,6 +1158,13 @@ impl SepData { dev_warn!(self.dev, "bringup: could not enable the key store ({:?}); keybag and ref-key operations are unavailable\n", e); return; } + if !self.sks_init_endpoint() { + dev_err!( + self.dev, + "bringup: key-store endpoint init failed; the key store stays closed\n" + ); + return; + } if *module_parameters::provision_keybag.value() != 0 { if *module_parameters::xart_writes.value() == 0 { dev_err!( @@ -1206,14 +1255,36 @@ impl SepData { })?; self.sks_designate_user_keybag(handle, stored.secret()); self.sks_machine_refkey(handle, stored.secret()); - let prepared = self.cold_match_continue(handle, uuid); - self.attach_bringup(); + let prepared = self.cold_match_continue(handle, uuid, stored.uuid_provenance()); self.ensure_restored_after(prepared); + self.attach_bringup(); self.probe_owner_export(); + self.refresh_match_credential(); self.touchid_started.store(true, Relaxed); Ok(()) } + // The SCRD credential set up before the Catacomb restore does not survive + // the rest of activation: J414c (variant-5 key store) and J314s (13.5 key + // store) answered every cold-boot MATCH_RESULT with 0x1 on a usable image, + // and a J313 showed the same symptom. Setting it up once more here was + // enough for every match that boot, including verifies that re-register + // the sensor with CLEAR_STATE. It runs on every profile. + fn refresh_match_credential(&self) { + if !self.templates_restored.load(Relaxed) { + return; + } + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + return; + }; + if !self.establish_scrd_match_context(user) { + dev_warn!( + self.dev, + "Touch ID: the match credential did not re-establish after the restore; the enclave may refuse matches this boot\n" + ); + } + } + fn prepare_bio_open(&self) -> Result<()> { if let Err(e) = self.activate_touchid() { self.touchid_failed.store(true, Relaxed); @@ -1227,7 +1298,41 @@ impl SepData { Ok(()) } + // The attach-time bring-up can fail after the Catacomb restore (the + // sensor patch step has no retry), which leaves the device-view proof + // undone and every verify refused until reboot. The restore itself is + // still a valid candidate, so bring the sensor up once more and let + // complete_bringup() finish the proof before refusing. + fn retry_restore_proof(&self) { + if self.templates_restored.load(Relaxed) + || !self.cold_restore_candidate.load(Relaxed) + || self.device_view_synced.load(Relaxed) + { + return; + } + dev_warn!( + self.dev, + "verify: attach-time bring-up left the device-view proof undone; retrying the sensor bring-up\n" + ); + if !self.bring_sensor_online() { + // After a failed first patch, the enclave can refuse the serial + // re-registration that follows CLEAR_STATE, while a reboot's + // fresh registration works. Retry once from the fresh stage. + dev_warn!( + self.dev, + "verify: retrying the sensor bring-up from a fresh registration\n" + ); + self.bringup.store(BRINGUP_FRESH, Relaxed); + if !self.bring_sensor_online() { + dev_err!(self.dev, "verify: the retried sensor bring-up failed\n"); + } + } + let _ = sensor::idle(); + self.refresh_match_credential(); + } + pub(crate) fn run_verify(&self) { + self.retry_restore_proof(); if !self.templates_restored.load(Relaxed) { dev_err!( self.dev, @@ -1241,6 +1346,7 @@ impl SepData { } let Some(token_bytes) = self.mint_token_bytes() else { + dev_err!(self.dev, "verify: could not draw a result token\n"); self.finish_verify( bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), [0u8; bio::TOKEN_LEN], @@ -1252,6 +1358,7 @@ impl SepData { // The enclave refuses a capture from an uncalibrated sensor (0x65 answers 1). if !self.bring_sensor_online() { + dev_err!(self.dev, "verify: the sensor did not come back online\n"); self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); let _ = sensor::idle(); return; @@ -1287,40 +1394,81 @@ impl SepData { self.finish_verify(outcome, token_bytes); } + // Every exit that fails a live verify logs why: userspace sees only a bare + // "unspecified error", which cannot tell a sensor fault from a refused + // match step. fn verify_one_image(&self) -> bio::VerifyOutcome { let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + dev_err!(self.dev, "verify: invalid probe user id\n"); return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); }; - if sensor::start_capture().is_err() { + if let Err(e) = sensor::start_capture() { + dev_err!(self.dev, "verify: could not start the capture ({:?})\n", e); return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); } let advertised = match self.await_capture() { CaptureWait::Ready(n) => n, CaptureWait::Timeout => { + dev_warn!(self.dev, "verify: the capture ended without a frame\n"); return bio::VerifyOutcome::Failed(ENROL_STATUS_TIMEOUT); } - CaptureWait::Fault(_state) => { + CaptureWait::Fault(state) => { + dev_err!( + self.dev, + "verify: the sensor fell to state {} during the capture\n", + state + ); + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); + } + CaptureWait::Abandon => { + // A cancelled verify also abandons; only a live one is a fault. + if bio::verify_is_live( + &self.bio_session.lock(), self.capture_epoch.load(Relaxed), + ) { + dev_err!( + self.dev, + "verify: the sensor status became unreadable during the capture\n" + ); + } return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); } - CaptureWait::Abandon => return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), }; let capture = match sensor::read_capture(advertised) { Ok(capture) => capture, - Err(sensor::CaptureError::Checksum { .. }) | Err(sensor::CaptureError::Length(_)) => { + Err(sensor::CaptureError::Checksum { + advertised: sent, + computed, + }) => { + dev_warn!( + self.dev, + "verify: frame checksum mismatch (sensor sent {:#06x}, computed {:#06x})\n", + sent, + computed + ); return bio::VerifyOutcome::Failed(ENROL_STATUS_RETRY); } - Err(sensor::CaptureError::Bus(_)) => { + Err(sensor::CaptureError::Length(n)) => { + dev_warn!( + self.dev, + "verify: the sensor advertised an unusable frame length {}\n", + n + ); + return bio::VerifyOutcome::Failed(ENROL_STATUS_RETRY); + } + Err(sensor::CaptureError::Bus(e)) => { + dev_err!(self.dev, "verify: reading the frame failed ({:?})\n", e); return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); } Err(sensor::CaptureError::NoMemory) => { - return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR) + dev_err!(self.dev, "verify: no memory for the frame\n"); + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); } }; if self - .sbio_expect_ok(&crate::sbio::sbio_prepare_image_processing()) + .sbio_verify_step(&crate::sbio::sbio_prepare_image_processing()) .is_none() { return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); @@ -1336,7 +1484,7 @@ impl SepData { user, crate::shim::monotonic_ns(), ); - if self.sbio_expect_ok(&init).is_none() { + if self.sbio_verify_step(&init).is_none() { return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); } @@ -1350,21 +1498,35 @@ impl SepData { } drop(capture); - let Some(assessment) = self.sbio_expect_ok(&crate::sbio::sbio_image_assessment()) else { + let Some(assessment) = self.sbio_verify_step(&crate::sbio::sbio_image_assessment()) else { return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); }; if assessment.len() <= crate::sbio::ASSESS_USABLE_MATCH { + dev_err!( + self.dev, + "verify: image assessment is {} bytes, too short to read\n", + assessment.len() + ); return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); } if assessment[crate::sbio::ASSESS_USABLE_MATCH] == 0 { + dev_warn!( + self.dev, + "verify: the image is not usable for matching (partial contact)\n" + ); return bio::VerifyOutcome::Failed(ENROL_STATUS_RETRY); } - let Some(result) = self.sbio_expect_ok(&crate::sbio::sbio_match_result()) else { + let Some(result) = self.sbio_verify_step(&crate::sbio::sbio_match_result()) else { return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); }; let Some(parsed) = crate::sbio::MatchResult::parse(&result) else { + dev_err!( + self.dev, + "verify: the {}-byte match result did not parse\n", + result.len() + ); return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); }; @@ -1377,6 +1539,11 @@ impl SepData { let known = self.bio_index.lock().contains_uuid(&identity); if !known { + dev_err!( + self.dev, + "verify: the enclave matched identity {}, which the host index does not hold\n", + Hex(&identity) + ); return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); } @@ -1433,6 +1600,13 @@ impl SepData { // completion is the flag at offset 0xbfe, not a derived stage count if complete { if !has_template { + dev_err!( + self.dev, + "enrol: enclave flags completion after {} capture(s) (stage {}, {}%) but reports no template\n", + counter, + stage, + percent + ); break Some(Err(ENROL_STATUS_ENCLAVE)); } enrolment_completed = true; @@ -1530,7 +1704,13 @@ impl SepData { let _ = self.log_identity_count(c"after CLEAR_STATE for sensor re-registration"); } - if stage == BRINGUP_FRESH && !self.profile.nonce_session { + if stage == BRINGUP_FRESH + && matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }) + { + // The 13.5 J313 flow begins with REGISTER_SENSOR_SERIAL (0x17). + // The newer REGISTER_SENSOR (0x80) returned 0x16 on this firmware. + self.bringup.store(BRINGUP_IDENTIFIED, Relaxed); + } else if stage == BRINGUP_FRESH && !self.profile.nonce_session { let op = crate::sbio::sbio_register_sensor(id); match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { @@ -1646,6 +1826,7 @@ impl SepData { let st = match sensor::status() { Ok(st) => st, Err(_) => { + dev_warn!(self.dev, "sensor: status read failed on wake\n"); return None; } }; @@ -1669,6 +1850,7 @@ impl SepData { continue; }; if !self.register_sensor(&id) { + dev_warn!(self.dev, "sensor: the enclave did not register the sensor\n"); return None; } @@ -1781,7 +1963,9 @@ impl SepData { Some(0) => { self.templates_restored.store(false, Relaxed); } - Some(n) if self.templates_restored.load(Relaxed) && self.prove_restore() => { + Some(n) if (self.templates_restored.load(Relaxed) + || self.cold_restore_candidate.load(Relaxed)) && self.prove_restore() => { + self.templates_restored.store(true, Relaxed); dev_warn!( self.dev, "Touch ID: restored {} enrolled identity/identities; enrolment survived reboot\n", @@ -1907,48 +2091,63 @@ impl SepData { } } + // Each failure names its step: a report that only says "sequence counter + // init failed" cannot tell the enclave side from the sensor side. fn init_sequence_counter(&self) -> bool { let challenge = match self.sbio_call(&crate::sbio::sbio_request_challenge()) { SbioOutcome::Ok(c) => c, SbioOutcome::PrerequisiteMissing => { if !self.establish_session() { + dev_warn!(self.dev, "sequence counter: session setup failed\n"); return false; } match self.sbio_call(&crate::sbio::sbio_request_challenge()) { SbioOutcome::Ok(c) => c, _ => { + dev_warn!(self.dev, "sequence counter: challenge request failed after session setup\n"); return false; } } } SbioOutcome::Status16 => { + dev_warn!(self.dev, "sequence counter: challenge request answered status 16\n"); return false; } SbioOutcome::Other => { + dev_warn!(self.dev, "sequence counter: challenge request failed\n"); return false; } }; if challenge.len() < sensor::CHALLENGE_LEN { + dev_warn!(self.dev, "sequence counter: short challenge ({} bytes)\n", challenge.len()); return false; } let mut out = [0u8; sensor::CHALLENGE_LEN]; out.copy_from_slice(&challenge[..sensor::CHALLENGE_LEN]); if sensor::send_challenge(&out).is_err() { + dev_warn!(self.dev, "sequence counter: sending the challenge to the sensor failed\n"); return false; } let reply = match sensor::read_challenge_reply() { Ok(r) => r, Err(_) => { + dev_warn!(self.dev, "sequence counter: the sensor did not answer the challenge\n"); return false; } }; match self.sbio_transfer(&crate::sbio::sbio_commit_challenge(&reply)) { Ok(done) if done.status.is_ok() => true, - Ok(_) => false, - Err(_) => false, + Ok(done) => { + dev_warn!(self.dev, "sequence counter: the enclave refused the sensor's reply, status {}\n", done.status); + false + } + Err(e) => { + dev_warn!(self.dev, "sequence counter: committing the reply failed {:?}\n", e); + false + } } } @@ -1957,36 +2156,50 @@ impl SepData { SbioOutcome::Ok(b) => b, SbioOutcome::PrerequisiteMissing => { if !self.init_sequence_counter() { + dev_warn!( + self.dev, + "sensor patch: sequence counter init failed before the patch fetch\n" + ); return None; } match self.sbio_call(&crate::sbio::sbio_fetch_patch()) { SbioOutcome::Ok(b) => b, _ => { + dev_warn!( + self.dev, + "sensor patch: patch fetch failed after the sequence counter init\n" + ); return None; } } } SbioOutcome::Status16 => { + dev_warn!(self.dev, "sensor patch: patch fetch answered status 16\n"); return None; } SbioOutcome::Other => { + dev_warn!(self.dev, "sensor patch: patch fetch failed\n"); return None; } }; if blob.is_empty() { + dev_warn!(self.dev, "sensor patch: the enclave returned an empty patch\n"); return None; } // enable command is mandatory; skipping it leaves the sensor in state 9 if sensor::setup_patch_enable().is_err() { + dev_warn!(self.dev, "sensor patch: the patch enable command failed on the bus\n"); return None; } if !self.await_sensor_state(sensor::STATE_IDLE, c"idle, before sending the patch") { + dev_warn!(self.dev, "sensor patch: the sensor did not reach idle before the patch\n"); return None; } if sensor::send_patch(&blob).is_err() { + dev_warn!(self.dev, "sensor patch: sending the patch failed on the bus\n"); return None; } @@ -1996,12 +2209,20 @@ impl SepData { let st = match sensor::status() { Ok(st) => st, Err(_) => { + dev_warn!( + self.dev, + "sensor patch: status read failed while waiting for the acknowledgment\n" + ); return None; } }; if st.patch_ack() == sensor::PATCH_ACCEPTED { if let Ok(after) = sensor::status() { if after.state == sensor::STATE_NEEDS_PATCH { + dev_warn!( + self.dev, + "sensor patch: acknowledged, but the sensor still asks for a patch\n" + ); return None; } } @@ -2010,6 +2231,11 @@ impl SepData { } let _ = sensor::status(); + dev_warn!( + self.dev, + "sensor patch: no acknowledgment after {} polls\n", + PATCH_POLL_ATTEMPTS + ); None } @@ -2509,6 +2735,82 @@ impl SepData { } } + /// Called by the enrol and verify work items before they touch the sensor. + /// Returns false if the system is going to sleep; the session has then + /// been ended and the capture must not start. A true return must be + /// paired with [`Self::capture_end`]. + pub(crate) fn capture_begin(&self) -> bool { + let mut session = self.bio_session.lock(); + if self.suspending.load(Relaxed) { + let woke = bio::abort_capture(&mut session); + drop(session); + if woke { + self.bio_wake(); + } + return false; + } + self.captures_running.fetch_add(1, Relaxed); + true + } + + pub(crate) fn capture_end(&self) { + self.captures_running.fetch_sub(1, Relaxed); + } + + /// The system is about to suspend. End any capture in progress, hold new + /// ones, and wait for the enrol or verify work to stop touching the sensor + /// before devices go down. + pub(crate) fn sleep_prepare(&self) -> Result<()> { + let woke = { + let mut session = self.bio_session.lock(); + self.suspending.store(true, Relaxed); + bio::abort_capture(&mut session) + }; + if woke { + dev_info!( + self.dev, + "Touch ID: ending the capture in progress before sleep\n" + ); + self.bio_wake(); + } + + let mut waited: u32 = 0; + while self.captures_running.load(Relaxed) != 0 { + if waited >= SLEEP_DRAIN_MS { + dev_warn!( + self.dev, + "Touch ID: capture did not drain within {} ms; aborting suspend\n", + SLEEP_DRAIN_MS + ); + return Err(EBUSY); + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(10)); + waited += 10; + } + Ok(()) + } + + /// The system has resumed, or the suspend was aborted. + pub(crate) fn sleep_finished(&self) { + let _session = self.bio_session.lock(); + self.suspending.store(false, Relaxed); + self.sleep_over.notify_all(); + } + + /// Holds a capture start until the system is awake. Userspace is thawed + /// before the driver hears of the resume, so a lock screen that retries at + /// once would otherwise be refused. A start made just before the system + /// sleeps is interrupted by the freezer and restarted after resume. + fn wait_until_awake(&self) -> Result<()> { + let mut session = self.bio_session.lock(); + while self.suspending.load(Relaxed) { + if self.sleep_over.wait_interruptible(&mut session) { + return Err(ERESTARTSYS); + } + } + Ok(()) + } + pub(crate) fn queue_enrolment(this: Arc) { let Some(epoch) = bio::queued_epoch(&this.bio_session.lock()) else { return; @@ -2551,13 +2853,13 @@ impl SepData { fn mint_token_bytes(&self) -> Option<[u8; bio::TOKEN_LEN]> { let mut bytes = [0u8; bio::TOKEN_LEN]; - for chunk in bytes.chunks_mut(4) { - match self.get_entropy_word() { - Ok(word) => chunk.copy_from_slice(&word.to_le_bytes()[..chunk.len()]), - Err(_) => { - return None; - } - } + // This is a host-issued, per-operation capability, not SEP match + // evidence. Use the initialized kernel CSPRNG, as for other host + // secrets, rather than trusting firmware control replies as entropy. + // J313/13.5 answered those draws with zero and caused valid matches + // to carry an invalid all-zero token to userspace. + if shim::random_bytes(&mut bytes).is_err() || bytes.iter().all(|b| *b == 0) { + return None; } Some(bytes) } @@ -2569,6 +2871,24 @@ impl SepData { if cmd == bio::IOC_ATTEST { return self.bio_attest(arg); } + // With xART writes off the enclave cannot persist what an enrolment or + // a match changes. It saves anti-replay state after every match, and a + // refused save has left it unresponsive until reboot. Refuse the + // capture instead; attach and the read-only ioctls still work. + if bio::starts_capture(cmd) && !self.xart_writable() { + if !self.capture_refusal_logged().xchg(true, Relaxed) { + dev_warn!( + self.dev, + "Touch ID: enrol and verify need xart_writes=1; the enclave saves anti-replay state after every match and stops responding when that save is refused\n" + ); + } + return Err(EROFS); + } + // A capture started while the system goes to sleep would run into the + // suspend; `capture_begin` repeats this check under the session lock. + if bio::starts_capture(cmd) { + self.wait_until_awake()?; + } // Query SEP outside the bio session/index locks. A host index can // outlive its SEP identity after a failed cold restore; userspace // must not mistake that index for proof of a live template. @@ -2656,9 +2976,18 @@ impl SepData { if let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) { let special = user.special_handle(); if let Ok(keybag::State::Present(stored)) = keybag::read(keybag::Slot::Identity) { - unlocked = self.sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { - self.sks_req_unlock_special(special, stored.secret(), healthy) - }).is_some(); + unlocked = match self.profile.key_store { + profile::KeyStore::Sepos13 { .. } => self + .sks_step(c"DEVICE_STATE_TRANSITION", |healthy| { + self.sks_req_unlock_special(special, stored.secret(), healthy) + }) + .is_some(), + profile::KeyStore::Variant5 => self + .sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { + self.sks_req_unlock_special(special, stored.secret(), healthy) + }) + .is_some(), + }; } } } @@ -2682,17 +3011,20 @@ impl SepData { }; let identities = self.log_identity_count(c"after Catacomb restore with sensor registered"); let identity_ready = matches!(identities, Some(n) if n > 0); - let ready = prepared && match_context && restored && identity_ready - && self.device_view_synced.load(Relaxed) && self.prove_restore(); - self.templates_restored.store(ready, Relaxed); - if !ready { + let candidate = prepared && match_context && restored && identity_ready; + self.cold_restore_candidate.store(candidate, Relaxed); + // COMPLETE_INIT has not synchronized the device view yet. Never make + // verification available on the preliminary Catacomb proof alone; + // complete_bringup finishes the proof and sets templates_restored. + self.templates_restored.store(false, Relaxed); + if !candidate { dev_err!( self.dev, - "matching unavailable this boot: cold-match preparation, SCRD credential, Catacomb restore, device view, or identity proof did not complete; template state is unproven\n" + "matching unavailable this boot: cold-match preparation, SCRD credential, Catacomb restore, or identity proof did not complete; template state is unproven\n" ); return; } - self.reconcile_identities(); + dev_info!(self.dev, "sbio: cold restore candidate ready; waiting for device-view proof\n"); } @@ -2700,6 +3032,7 @@ impl SepData { &self, source: crate::sks::KeyBagHandle, uuid: [u8; keybag::UUID_LEN], + provenance: keybag::UuidProvenance, ) -> bool { if !self.keybag_designated.load(Relaxed) { return false; @@ -2711,32 +3044,47 @@ impl SepData { let special = user.special_handle(); self.log_identity_count(c"before the cold-match preparation"); + let Some(source_uuid) = self.sks_read_uuid(source) else { + return false; + }; + let generated_lookup = matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }) + && provenance == keybag::UuidProvenance::AsGenerated; + let Some(identity) = keybag::SnapshotIdentity::from_loaded( + uuid, provenance, source_uuid, generated_lookup, + ) else { + dev_err!(self.dev, "sks: recovered source bag does not match its stored identity\n"); + return false; + }; let uuid_ok = self .sks_send(self.sks_req_copy_uuid_special(special)) .and_then(|out| self.sks_uuid_from_reply(&out)); - match uuid_ok { - Some(got) if got == uuid => {} - Some(_) => { - return false; - } - None => { - return false; - } + if !uuid_ok.is_some_and(|got| identity.accepts(&uuid, provenance, &got)) { + dev_err!(self.dev, "sks: designated bag does not match the recovered source bag\n"); + return false; } if self.sks_send(self.sks_req_unload_keybag(source)).is_none() { return false; } + let mut material = self.enrol_material.lock(); + let Some(material) = &mut *material else { + return false; + }; + material.snapshot_identity = Some(identity); + dev_info!(self.dev, "sks: snapshot identity bound to recovered and designated bag; generated lookup {}\n", generated_lookup); + true } pub(crate) fn sep_random(&self, buf: &mut [u8]) -> Result<()> { - for chunk in buf.chunks_mut(4) { - let word = self.get_entropy_word()?; - let bytes = word.to_le_bytes(); - let take = chunk.len(); - chunk.copy_from_slice(&bytes[..take]); + // The trusted-key framework asks for a fresh plaintext key before + // passing it to SEP for sealing. The host CSPRNG is appropriate for + // that key; J313/13.5's unverified SEP control entropy replies have + // returned all zeros and must not be used as key material. + shim::random_bytes(buf)?; + if !buf.is_empty() && buf.iter().all(|b| *b == 0) { + return Err(EIO); } Ok(()) } diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index e52acb69b04f0..85190a5e14e7d 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -20,7 +20,6 @@ #include #include -#include #include "shim.h" diff --git a/drivers/soc/apple/sep-bio.h b/drivers/soc/apple/sep-bio.h new file mode 100644 index 0000000000000..d52b880ac9332 --- /dev/null +++ b/drivers/soc/apple/sep-bio.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* + * Userspace interface for the SEP biometric device. + * + * The enclave does the matching. Nothing biometric crosses this interface: only + * an operation, a stage, a status, an opaque identity UUID, and an opaque host + * label userspace chooses. + */ + +#pragma once + +#include +#include + +#define SEP_BIO_IFACE_VERSION 4 + +#define SEP_BIO_UUID_LEN 16 +#define SEP_BIO_LABEL_LEN 128 +#define SEP_BIO_NONCE_LEN 32 +#define SEP_BIO_TOKEN_LEN 32 +#define SEP_BIO_MAX_IDENTITIES 32 +#define SEP_BIO_CHALLENGE_LEN 32 +#define SEP_BIO_ATTEST_PUB_LEN 65 +#define SEP_BIO_ATTEST_SIG_MAX 72 + +enum { + SEP_BIO_STATE_IDLE = 0, + SEP_BIO_STATE_PENDING = 1, + SEP_BIO_STATE_PROGRESS = 2, + SEP_BIO_STATE_DONE = 3, + SEP_BIO_STATE_FAILED = 4, +}; + + +enum { + SEP_BIO_NO_MATCH = 0, + SEP_BIO_MATCH = 1, + SEP_BIO_NOT_COMPARED = 2, +}; + +struct sep_bio_identity { + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +struct sep_bio_info { + __u32 version; + __u32 sensor_present; + __u32 enrolled; + __u32 capacity; /* usable slots in this SEP user context, not ABI list size */ + __u32 enroll_stages; + __u32 reserved[3]; +}; + +/* reserved[0] flags; reserved[1] is a live SEP identity count only when valid. */ +#define SEP_BIO_INFO_LIVE_COUNT_VALID 1u + +struct sep_bio_list { + __u32 count; + __u32 reserved; + struct sep_bio_identity id[SEP_BIO_MAX_IDENTITIES]; +}; + +struct sep_bio_enrol_start { + __u32 flags; + __u32 reserved; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +#define SEP_BIO_GUIDANCE_NONE 0 +#define SEP_BIO_GUIDANCE_PLACE 1 +#define SEP_BIO_GUIDANCE_LIFT_AND_MOVE 2 +#define SEP_BIO_GUIDANCE_HOLD_STILL 3 + +struct sep_bio_enrol_poll { + __u32 state; + __u32 stage; + __u32 stages_total; + __u32 status; + __u8 uuid[SEP_BIO_UUID_LEN]; + __u32 guidance; + __u32 progress_percent; +}; +struct sep_bio_verify_start { + __u32 flags; + __u32 reserved; + __u8 nonce[SEP_BIO_NONCE_LEN]; +}; + +struct sep_bio_verify_poll { + __u32 state; + __u32 result; + __u32 status; + __u32 guidance; /* SEP_BIO_GUIDANCE_*; status only, no capture side effect */ + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 token[SEP_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ + __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ +}; + +struct sep_bio_delete { + __u8 uuid[SEP_BIO_UUID_LEN]; +}; + +/* + * Device attestation of key possession: the enclave signs 'challenge' with the + * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and + * returns the DER signature and public point. The private key never leaves the + * enclave. + */ +struct sep_bio_attest { + __u32 sig_len; /* out: DER signature length */ + __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ + __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ + __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ + __u8 reserved[3]; +}; + +#define SEP_BIO_IOC_MAGIC 0xB1 + +#define SEP_BIO_GET_INFO _IOR (SEP_BIO_IOC_MAGIC, 0x01, struct sep_bio_info) +#define SEP_BIO_LIST _IOR (SEP_BIO_IOC_MAGIC, 0x02, struct sep_bio_list) +#define SEP_BIO_ENROL_START _IOW (SEP_BIO_IOC_MAGIC, 0x03, struct sep_bio_enrol_start) +#define SEP_BIO_ENROL_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x04, struct sep_bio_enrol_poll) +#define SEP_BIO_VERIFY_START _IOW (SEP_BIO_IOC_MAGIC, 0x05, struct sep_bio_verify_start) +#define SEP_BIO_VERIFY_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x06, struct sep_bio_verify_poll) +#define SEP_BIO_CANCEL _IO (SEP_BIO_IOC_MAGIC, 0x07) +#define SEP_BIO_DELETE _IOW (SEP_BIO_IOC_MAGIC, 0x08, struct sep_bio_delete) +#define SEP_BIO_DELETE_ALL _IO (SEP_BIO_IOC_MAGIC, 0x09) +#define SEP_BIO_ATTEST _IOWR(SEP_BIO_IOC_MAGIC, 0x0a, struct sep_bio_attest) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 392ad9f05d452..61d6d47769971 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -18,6 +18,8 @@ mod fv; mod hwrng; mod image; mod keybag; +mod keybag_identity; +mod pm; mod profile; mod proto; mod refkey; @@ -106,6 +108,11 @@ const OOL_SIZE_SCRD: usize = 0x4000; const SBIO_TIMEOUT_MS: time::Msecs = 5000; +// How long going to sleep waits for an ended capture to stop: one enclave +// request in flight, plus the capture loop's interrupt-wait backstop. +const SLEEP_DRAIN_MS: u32 = 6000; +static_assert!(SLEEP_DRAIN_MS as u64 > SBIO_TIMEOUT_MS as u64); + const SKS_ALLOC: usize = 0x8000; const SKS_SECRET_LEN: usize = 32; @@ -116,6 +123,8 @@ const SCRD_MAX_CAPTURE: usize = 8; const SCRD_TIMEOUT_MS: time::Msecs = 2000; const SKS_TIMEOUT_MS: time::Msecs = 2000; +/// The 13.5 kext's per-try key-store wait. +const SKS_TIMEOUT_13_MS: time::Msecs = 6000; const SKS_TIMEOUT_PER_KIB_MS: time::Msecs = 6000; @@ -258,6 +267,7 @@ impl CalibrationBlob { struct EnrolMaterial { special: crate::sks::SpecialHandle, secret: Secret, + snapshot_identity: Option, } struct ImageContext<'a> { @@ -555,6 +565,12 @@ struct SepData { sks_seq: Atomic, + // IPC header version negotiated by the 13.5 endpoint init; 1 otherwise. + sks_header_version: Atomic, + + // The 13.5 endpoint init (0x4d, set_env) completed. + sks_initialized: Atomic, + // u32, not u8: the kernel's Rust atomics have no u8 AtomicType phase: Atomic, @@ -576,12 +592,19 @@ struct SepData { sbio_ready: Atomic, templates_restored: Atomic, + // Provisional cold restore; COMPLETE_INIT must still prove the device view. + cold_restore_candidate: Atomic, sensor_calibrated: Atomic, sensor_key_loaded: Atomic, enrol_open: Atomic, + // One warning each per boot: an xART write refused because writes are + // off, and a Touch ID capture refused for the same reason. + xart_refusal_logged: Atomic, + capture_refusal_logged: Atomic, + #[pin] enrol_material: Mutex>, @@ -663,6 +686,16 @@ struct SepData { shutting_down: Atomic, capture_epoch: Atomic, + // The system is going to sleep: no capture may start. Changed under the + // bio session lock; `sleep_over` is signalled when it clears. + suspending: Atomic, + + #[pin] + sleep_over: CondVar, + + // Enrol and verify work items between `capture_begin` and `capture_end`. + captures_running: Atomic, + #[pin] rx_work: Work, @@ -781,6 +814,8 @@ impl SepData { control <- new_mutex!(control::ControlState::new()), control_wq <- new_condvar!("SepData::control_wq"), sks_seq: Atomic::new(0), + sks_header_version: Atomic::new(1), + sks_initialized: Atomic::new(false), sks_wedged: Atomic::new(0), bringup: Atomic::new(BRINGUP_FRESH), keybag_designated: Atomic::new(false), @@ -788,9 +823,12 @@ impl SepData { touchid_started: Atomic::new(false), touchid_failed: Atomic::new(false), enrol_open: Atomic::new(false), + xart_refusal_logged: Atomic::new(false), + capture_refusal_logged: Atomic::new(false), sensor_calibrated: Atomic::new(false), sensor_key_loaded: Atomic::new(false), templates_restored: Atomic::new(false), + cold_restore_candidate: Atomic::new(false), enrol_material <- new_mutex!(None), enrol_identity_candidates <- new_mutex!(KVec::new()), last_capture_end_ns: Atomic::new(0), @@ -829,6 +867,9 @@ impl SepData { registered: Atomic::new(false), shutting_down: Atomic::new(false), capture_epoch: Atomic::new(0), + suspending: Atomic::new(false), + sleep_over <- new_condvar!("SepData::sleep_over"), + captures_running: Atomic::new(0), rx_work <- new_work!("SepData::rx_work"), enrol_work <- new_work!("SepData::enrol_work"), verify_work <- new_work!("SepData::verify_work"), @@ -1120,11 +1161,15 @@ impl SepData { } fn survey_hwrng(&self) -> Result<()> { - // A working control endpoint must answer four consecutive draws before - // it is exposed to the kernel RNG core. Early attach can beat the SEP - // endpoint exchange, so this runs only after that exchange settles. + // Successful transport is not proof of entropy: J313/13.5 has returned + // four zero words here. Do not advertise that source to the RNG core. + // This is a minimal sanity check, not a complete RNG health test. + let mut any_bits = 0u32; for _ in 0..4 { - let _ = self.get_entropy_word()?; + any_bits |= self.get_entropy_word()?; + } + if any_bits == 0 { + return Err(EIO); } self.register_hwrng() } @@ -1234,6 +1279,16 @@ impl SepData { store.as_mut().map(f) } + /// Whether the enclave's xART writes reach the store. + pub(crate) fn xart_writable(&self) -> bool { + self.with_store(|store| store.writes_enabled()) + .unwrap_or(false) + } + + pub(crate) fn capture_refusal_logged(&self) -> &Atomic { + &self.capture_refusal_logged + } + fn with_host_store(&self, f: impl FnOnce(&mut store::Store) -> R) -> Option { let mut guard = self.host_store.lock(); let store: &mut Option = &mut guard; @@ -1315,20 +1370,33 @@ impl SepData { let os_uuid = self.xarm.lock().os_uuid; let serviced = self.with_store(|store| { - xarm::service( + let done = xarm::service( req, &payload, &mut staging, store, PROTECTED_DATA_AVAILABLE, os_uuid, - ) + ); + (done, store.writes_enabled()) }); - let Some(done) = serviced else { + let Some((done, writable)) = serviced else { self.fail_xarm(req.tag); return; }; + if !writable + && xarm::is_write(req.opcode) + && done.reply.status != xarm::STATUS_OK + && !self.xart_refusal_logged.xchg(true, Relaxed) + { + dev_warn!( + self.dev, + "xART: refused the enclave's write (op 0x{:02x}): writes are disabled (xart_writes=0)\n", + req.opcode + ); + } + if *module_parameters::xarm_trace.value() != 0 { dev_info!( self.dev, @@ -1792,8 +1860,11 @@ impl SepData { fn tick_attach(this: &Arc) { let now = this.rx_count.load(Relaxed); - // HW: the SEP takes ~265 ms to answer a registration - if now == 0 { + // HW: the SEP takes ~265 ms to answer a warm registration, and on the + // cold-boot path J316s starts discovery ~370 ms after the IMG4 ack. + // The boot acks count as traffic, so wait for the first endpoint, not + // the first message, or a quiet gap after IMG4 ends the attach early. + if this.endpoint_count() == 0 { let ticks = this.settle_idle_ticks.load(Relaxed).wrapping_add(1); this.settle_idle_ticks.store(ticks, Relaxed); if ticks.saturating_mul(u64::from(SETTLE_MS)) < u64::from(FIRST_RESPONSE_MS) { @@ -1809,9 +1880,20 @@ impl SepData { } if this.endpoint_count() == 0 { + dev_err!( + this.dev, + "attach: {} messages received but no endpoint advertised; xART and key store cannot run\n", + now + ); this.phase.store(PHASE_READY, Relaxed); return; } + dev_info!( + this.dev, + "attach: {} endpoints advertised in {} messages\n", + this.endpoint_count(), + now + ); this.prepare_os_uuid(); @@ -1895,7 +1977,7 @@ impl SepData { proto::EP_BOOT => self.on_boot(msg), - _ep => {}, + ep => dev_info!(self.dev, "rx: unhandled endpoint {:#04x} (msg0 {:#018x})\n", ep, msg.msg0), } } @@ -1916,13 +1998,21 @@ impl SepData { } } - fn on_discovery(&self, _msg: Message, f: proto::Fields) { + fn on_discovery(&self, msg: Message, f: proto::Fields) { let mut table = self.endpoints.lock(); match f.ty { proto::DISCOVER_TYPE_DESCRIPTOR | proto::DISCOVER_TYPE_CONFIG => { let _ = table.slot(f.param); + dev_info!( + self.dev, + "discover: endpoint {:#04x} (type {}, msg0 {:#018x}); {} known\n", + f.param, + f.ty, + msg.msg0, + table.eps.len() + ); } - _ => {} + ty => dev_info!(self.dev, "discover: other type {} (msg0 {:#018x})\n", ty, msg.msg0), } } @@ -1935,6 +2025,10 @@ impl SepData { self.sbio_wq.notify_all(); self.sks_wq.notify_all(); self.scrd_wq.notify_all(); + pm::unregister(); + // Release any capture start still held for a resume that will not be + // reported now. + self.sleep_finished(); trusted::unregister(); self.unregister_fv_kernel(); @@ -2120,8 +2214,9 @@ impl WorkItem for SepData { fn run(this: Arc) { let epoch = this.capture_epoch.load(Relaxed); - if !this.shutting_down.load(Relaxed) { + if !this.shutting_down.load(Relaxed) && this.capture_begin() { this.run_enrolment(); + this.capture_end(); } bio::worker_stopped(&mut this.bio_session.lock(), epoch); } @@ -2132,8 +2227,9 @@ impl WorkItem for SepData { fn run(this: Arc) { let epoch = this.capture_epoch.load(Relaxed); - if !this.shutting_down.load(Relaxed) { + if !this.shutting_down.load(Relaxed) && this.capture_begin() { this.run_verify(); + this.capture_end(); } bio::worker_stopped(&mut this.bio_session.lock(), epoch); } @@ -2192,6 +2288,10 @@ impl platform::Driver for SepDriver { dev_warn!(data.dev, "trusted-keys: registration failed ({:?})\n", e); } + if let Err(e) = pm::register(data.clone()) { + dev_warn!(data.dev, "sleep notifier registration failed ({:?}); a capture may run into a suspend\n", e); + } + if data.registered.load(Relaxed) { SepData::arm_settle(&data); } @@ -2223,10 +2323,18 @@ impl kernel::InPlaceModule for SepModule { // changeset; the cold-boot target describes them statically (its DART // is enabled before probe), so no changeset runs there. _dt: { - if let Ok(p) = profile::detect() { - if matches!(p.bootstrap, profile::Bootstrap::WarmRegister) { - dt::enable_sep_and_dart()?; + match profile::detect() { + Ok(p) => { + if matches!(p.bootstrap, profile::Bootstrap::WarmRegister) { + dt::enable_sep_and_dart()?; + } } + // Without this the module loads, probes nothing and logs + // nothing on an unlisted SoC, which is indistinguishable + // from a driver that started correctly. + Err(_) => pr_info!( + "no platform profile for this SoC; the SEP stays disabled\n" + ), } }, @@ -2246,7 +2354,7 @@ module! { params: { xart_writes: u8 { default: 0, - description: "Opt in to shared xART writes only if APFS proves a single unsnapshotted .gl extent", + description: "Opt in to shared xART writes only if APFS proves a single unsnapshotted .gl extent; Touch ID enrol and verify need it", }, xart_start_sector: u64 { default: 0, @@ -2272,6 +2380,10 @@ module! { default: 0, description: "Opt-in J414s diagnostic: with zero live identities, select the system context and try saving the missing owner Catacomb", }, + refkey_v2: u8 { + default: 0, + description: "Explicit recovery slot for a new identity-bag context; preserves the original machine ref-key", + }, j414s_persistent_enrol: u8 { default: 0, description: "Opt-in J414s enrollment test: export owner from the fresh context, then save user before master at completion", diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index e13ca258d92f8..b8592883ce2d3 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -23,6 +23,11 @@ struct apple_sep_fv_new_file_key; void sep_cancel_work_sync(void *work); void sep_cancel_delayed_work_sync(void *work); +/* -- pm_shim.c ---------------------------------------------------------- */ + +int sep_pm_register(int (*event)(bool entering)); +void sep_pm_unregister(void); + /* -- hwrng_shim.c ------------------------------------------------------- */ void *sep_hwrng_alloc(void); diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 5266541882a17..2d29fa7598577 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -609,6 +609,23 @@ impl SepData { }) } + // A completed CREATE_KEYBAG that the enclave answered with a non-zero + // status created nothing, so the persisted intent must not block the next + // attempt: without this, every later seal fails with EEXIST. Any other + // failure (timeout, no answer) keeps the intent and stays fail-closed. + fn sks_mark_create_refused(&self, slot: keybag::Slot, status: i8) { + if status == 0 { + return; + } + if let Err(e) = keybag::mark_refused(slot) { + dev_err!( + self.dev, + "sks: cannot mark refused keybag create retryable: {:?}\n", + e + ); + } + } + pub(crate) fn sks_provision_identity_keybag(&self) -> bool { let proof = match keybag::read(keybag::Slot::Identity) { Ok(keybag::State::Present(_)) => return true, @@ -654,9 +671,21 @@ impl SepData { }; intent.sending(); let Some(out) = self.sks_exchange(request.name, request.msg, &request.img) else { + dev_err!( + self.dev, + "sks: CREATE_KEYBAG exchange did not complete (buffers unregistered, wedged, or OOL write failed)\n" + ); return false; }; let Some(body) = self.sks_report_response(crate::sks::SKS_CREATE_NAME, &out) else { + dev_err!( + self.dev, + "sks: CREATE_KEYBAG reply carried no parsable body: mailbox status {}, response {} bytes, response_size {}\n", + out.reply.status, + out.response.len(), + out.reply.response_size + ); + self.sks_mark_create_refused(slot, out.reply.status); return false; }; // 13.5 replies with the struct version and the handle alone @@ -672,6 +701,7 @@ impl SepData { out.reply.status, body.len() ); + self.sks_mark_create_refused(slot, out.reply.status); return false; } let variant = u32::from_le_bytes(body[0..4].try_into().unwrap()); @@ -680,6 +710,11 @@ impl SepData { profile::KeyStore::Sepos13 { .. } => 8, profile::KeyStore::Variant5 => { let Some((_fv_data, end)) = image::read_blob(body, 8) else { + dev_err!( + self.dev, + "sks: CREATE_KEYBAG reply has no fv_data blob at offset 8 (body {} bytes)\n", + body.len() + ); return false; }; end @@ -703,15 +738,23 @@ impl SepData { profile::KeyStore::Sepos13 { .. } => None, profile::KeyStore::Variant5 => { let Some(bag_uuid) = self.sks_read_uuid(handle) else { + dev_err!(self.dev, "sks: could not read the new keybag's UUID back\n"); return false; }; Some(bag_uuid) } }; let Some(out) = self.sks_send(self.sks_req_copy_keybag(handle)) else { + dev_err!(self.dev, "sks: COPY_KEYBAG of the new identity keybag got no reply\n"); return false; }; let Some(wrapped) = self.wrapped_from_copy_reply(&out, c"new identity keybag") else { + dev_err!( + self.dev, + "sks: COPY_KEYBAG reply unusable: mailbox status {}, response {} bytes\n", + out.reply.status, + out.response.len() + ); return false; }; let committed = match bag_uuid { diff --git a/drivers/soc/apple/xarm.rs b/drivers/soc/apple/xarm.rs index ae9fe46728020..3e34b34c8951b 100644 --- a/drivers/soc/apple/xarm.rs +++ b/drivers/soc/apple/xarm.rs @@ -83,6 +83,12 @@ pub(crate) struct Serviced { pub(crate) reply_bytes: usize, } +/// Requests that change the store: a refusal of one means the enclave's state +/// was not persisted. +pub(crate) fn is_write(opcode: u8) -> bool { + matches!(opcode, OP_ROOT_WRITE | OP_SESSION_WRITE | OP_SESSION_DELETE) +} + pub(crate) fn service( req: &Request, outbound: &[u8], diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index c487824f7d9a8..e47275ccb62d5 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -401,6 +401,12 @@ impl Store { Ok(true) } + /// Whether this store accepts writes: `xart_writes=1` and APFS proved the + /// extent writable. + pub(crate) fn writes_enabled(&self) -> bool { + self.writes_enabled + } + /// Byte offset of the APFS-resolved gigalocker extent in the container. pub(crate) fn base(&self) -> u64 { self.base From 93bc234385f5bf51b276e9d2e5abe8280e2ff92c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:19:05 +1100 Subject: [PATCH 0685/1579] arm64: dts: apple: t8140: Leave display coprocessor power to firmware Booted with this tree's device tree, the J700 resets about 1.5 seconds into boot: SError Interrupt on CPU5, code 0x00000000be000000 -- SError Call trace: ... apple_pmgr_ps_set apple_pmgr_ps_power_off _genpd_power_off genpd_power_off genpd_sync_state device_links_supplier_sync_state_resume of_platform_sync_state_init The firmware leaves the external display coprocessor running at boot, and the device tree registers its three power states (dispext0_sys, _fe and _cpu) with nothing that keeps them in use: the only consumer is its mailbox, which is runtime suspended because no driver talks to that coprocessor on T8140. Once the mailbox has probed, sync_state powers dispext0_cpu off underneath the running firmware, and that power-state write faults. Stop registering the three external display states, and stop enabling the mailbox and the two DARTs that only that coprocessor would use. The firmware keeps owning them until there is a driver for it. The internal display coprocessor is handed over running as well, and its power states were described disabled, so nothing could depend on them. Register disp_sys, disp_fe and disp_cpu, and mark disp_cpu always-on: genpd then never writes that state, neither at sync_state nor on system suspend, and its parents stay on through it. The bootloader leaves the panel lit, so the domain is on when it probes. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 12 ------------ arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 19 +++++++++++++------ 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 05dff1967b376..146ac921bbbb7 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -454,18 +454,6 @@ }; }; -&dispext0_dart { - status = "okay"; -}; - -&dcpext0_dart { - status = "okay"; -}; - -&dcpext_mbox { - status = "okay"; -}; - &dpxbar0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 02ea254c12502..a8cea58a82da4 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -96,7 +96,6 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "disp_sys"; - status = "disabled"; }; ps_dispext0_sys: power-controller@288 { @@ -149,7 +148,6 @@ #reset-cells = <0>; label = "disp_fe"; power-domains = <&ps_disp_sys>; - status = "disabled"; }; ps_dispext0_fe: power-controller@320 { @@ -373,7 +371,11 @@ #reset-cells = <0>; label = "disp_cpu"; power-domains = <&ps_disp_fe>; - status = "disabled"; + /* + * The display coprocessor is handed over running and its + * firmware owns this state; writing it faults. + */ + apple,always-on; }; @@ -801,13 +803,18 @@ status = "okay"; }; + /* + * The external display coprocessor has no Linux consumer yet. Its + * firmware keeps it running, so leave these three states to it: + * powering dispext0_cpu down underneath that firmware faults. + */ ps_dispext0_sys: power-controller@288 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x288 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = "dispext0_sys"; - status = "okay"; + status = "disabled"; }; ps_ane_sys: power-controller@290 { @@ -945,7 +952,7 @@ #reset-cells = <0>; label = "dispext0_fe"; power-domains = <&ps_dispext0_sys>; - status = "okay"; + status = "disabled"; }; ps_dispext0_cpu: power-controller@328 { @@ -955,7 +962,7 @@ #reset-cells = <0>; label = "dispext0_cpu"; power-domains = <&ps_dispext0_fe>; - status = "okay"; + status = "disabled"; }; ps_spi0: power-controller@330 { From 9e6acbbd21ba92409dd4f24074532b0bad997792 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:40:09 +1100 Subject: [PATCH 0686/1579] arm64: dts: apple: t8140: Describe the internal display coprocessor The T8140 device tree has the display DARTs and power states but no display coprocessor, so the loader finds nothing to hand the running display over to and Linux is left with the boot framebuffer. Add the internal DCP, its ASC mailbox, the PMGR bandwidth registers it names, its two display clocks and the display subsystem node, following the layout used for the other Apple SoCs: - The DCP register ranges are the coprocessor control block and its I/O vector base from the Apple Device Tree's dcp node, and the first five disp0 ranges. Its DART stream (23) and the scanout and PIODMA streams (0 and 16) come from the matching DART mapper nodes. - The mailbox interrupts are the four dcp interrupts, in the same send-empty, send-not-empty, recv-empty, recv-not-empty order as the external mailbox. - Both display DARTs get the 64 GiB VM window at 0x10000000000 that the Apple Device Tree gives those streams. - The bandwidth scratch and request registers sit 0x20000 and 0x68000 into the display clock window at 0x302800000, the last disp0 range. - The clock rates are disp0's clock IDs 0x154 and 0x194: 300 MHz and 450 MHz. The DCP is handed over running with its firmware mapped through locked DARTs, so it stays disabled here, like its DARTs. The loader enables both once it has reserved those mappings; the display subsystem and the DCP carry explicit phandles because the loader refers to them. A single zero in apple,iomfb-surfaces selects surface 0. Kernels that read the property as a list of surfaces select the same one. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 99 ++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 7e8ffb3e4b663..fd419d1fe8a49 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -389,6 +389,34 @@ clock-output-names = "clkref"; }; + /* disp0 clock ID 0x154; the rate is the Apple Device Tree's */ + clk_disp0_154: clock-disp0-154 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <300000000>; + clock-output-names = "clk_disp0_154"; + }; + + /* disp0 clock ID 0x194 */ + clk_disp0_194: clock-disp0-194 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <450000000>; + clock-output-names = "clk_disp0_194"; + }; + + /* + * Display component master. It has no registers, so it sits outside + * the simple-bus. + */ + display: display-subsystem { + compatible = "apple,display-subsystem"; + /* disp0_dart is locked by the bootloader and must come first */ + iommus = <&disp0_dart 0>; + /* generate a phandle for the loader */ + phandle = <&display>; + }; + reserved-memory { #address-cells = <2>; #size-cells = <2>; @@ -1128,6 +1156,8 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; + /* VM window of streams 0 and 16 */ + apple,dma-range = <0x100 0x0 0x10 0x0>; status = "disabled"; }; @@ -1146,7 +1176,76 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; + /* VM window of stream 23 */ + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + dcp_mbox: mailbox@412e08000 { + compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x12e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp_cpu>; + status = "disabled"; + }; + + /* + * Bandwidth scratch and request registers, 0x20000 and 0x68000 + * into the display clock window at 0x302800000 (the sixth disp0 + * register range in the Apple Device Tree). + */ + pmgr_dcp: power-management@302820000 { + reg = <0x3 0x02820000 0x0 0x8>, + <0x3 0x02868000 0x0 0x4>; + reg-names = "dcp-bw-scratch", "dcp-bw-doorbell"; + #apple,bw-scratch-cells = <3>; + #apple,bw-doorbell-cells = <2>; + }; + + dcp: dcp@412e00000 { + compatible = "apple,t8140-dcp", "apple,dcp"; + reg = <0x4 0x12e00000 0x0 0x4000>, + <0x4 0x10000000 0x0 0xafc000>, + <0x4 0x12240000 0x0 0x10000>, + <0x4 0x123a8000 0x0 0x4000>, + <0x4 0x123ac000 0x0 0x4000>, + <0x4 0x12800000 0x0 0x800000>, + <0x4 0x12850000 0x0 0x4000>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4", "iop-vbar"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 23>; + power-domains = <&ps_disp_cpu>; + clocks = <&clk_disp0_154>, <&clk_disp0_194>; + clock-names = "clock-154", "clock-194"; + apple,bw-scratch = <&pmgr_dcp 0 5 0>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + /* + * A single zero selects surface 0; J700 kernels that + * read this as a surface list take it the same way. + */ + apple,iomfb-surfaces = <0>; + /* generate a phandle for the loader */ + phandle = <&dcp>; + /* bus properties for the piodma child device */ + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* the loader enables it when it hands the display over */ status = "disabled"; + + disp0_piodma: piodma { + iommus = <&disp0_dart 16>; + phandle = <&disp0_piodma>; + }; }; dispext0_dart: iommu@416300000 { From 061370b4c70fd888ea6dc520a8988208f4aebca8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:40:10 +1100 Subject: [PATCH 0687/1579] arm64: dts: apple: t8140: Hand the J700 panel over to the DCP The loader only hands the running internal display over when the device tree has dcp and disp0 aliases. Without them it keeps the boot framebuffer, reserves none of the display memory and leaves the DCP disabled, so the internal panel never reaches the display driver. Add the aliases the loader uses for the hand-off, plus the PIODMA alias in its hyphenated form, enable the DCP mailbox and describe the panel: 2408x1506 at the Apple Device Tree's 219 ppi dot pitch is 279 x 175 mm, and the backlight's 16.16 maximum luminance is 525 nits. Give the boot framebuffer the display CPU power domain, as on the other Apple laptops, so that simpledrm depends on the domain that keeps the display running whether or not the hand-off happens. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 146ac921bbbb7..4742ce8aaca35 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -27,6 +27,10 @@ serial0 = &serial0; pmp = &pmp; wifi0 = &wifi0; + /* used by the loader to hand over the display */ + dcp = &dcp; + disp0 = &display; + disp0-piodma = &disp0_piodma; }; chosen { @@ -43,6 +47,7 @@ framebuffer0: framebuffer@0 { compatible = "apple,simple-framebuffer", "simple-framebuffer"; reg = <0 0 0 0>; /* To be filled by loader */ + power-domains = <&ps_disp_cpu>; /* Format properties will be added by loader */ status = "disabled"; }; @@ -454,6 +459,20 @@ }; }; +&dcp_mbox { + status = "okay"; +}; + +&dcp { + /* 2408x1506 at 219 ppi; 525 nits peak from the backlight node */ + panel: panel { + compatible = "apple,panel-j700", "apple,panel"; + width-mm = <279>; + height-mm = <175>; + apple,max-brightness = <525>; + }; +}; + &dpxbar0 { status = "okay"; }; From 2bad3dba632ff85537392b79255fc527baf01746 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:30:45 +1100 Subject: [PATCH 0688/1579] iommu: apple-dart: keep inherited firmware translations visible before attach On T8140 the internal display coprocessor's DART stream is locked, and its root already holds firmware translations for the regions the loader re-exports as translated reservations. Domain setup imports those roots so that apple_dart_iova_to_phys() reports the inherited translations, and the IOMMU core then skips building replacement mappings for them. Since domain finalization stopped pre-populating the attached-stream map, because reserved mappings can be installed before any device is attached, that lookup finds no stream and reports nothing. The core then builds read-write, uncached host leaves for the reservations. Those differ from the firmware's cached leaves, and from its read-only code region, so publishing them into the firmware-owned root slots fails, and the core's must-succeed default-domain attach returns -EBUSY: WARNING: drivers/iommu/iommu.c:3243 at iommu_setup_default_domain apple-dart 412340000.iommu: probe with driver apple-dart failed with error -16 The display coprocessor then never binds. Record, per stream map, the streams whose firmware roots finalization imported, and use them in addition to the attached streams when resolving inherited translations and when validating software mappings against firmware-owned slots. The firmware leaves are left untouched, including the read-only region. Also keep DARTs with imported firmware roots out of the legacy path that replaces a locked root's entries wholesale. That path is meant for older display DARTs whose root Linux takes over; on these DARTs it would overwrite live firmware slots. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 67b30e9435f44..38075e072f18e 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -383,6 +383,9 @@ struct apple_dart_atomic_stream_map { * @finalized: true if the domain has been completely initialized * @init_lock: protects domain initialization * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only) + * @fw_sidmap: streams whose locked firmware roots the domain imported at + * finalization, per entry of @stream_maps; unlike @stream_maps + * they are known before any device is attached * @domain: core iommu domain pointer */ struct apple_dart_domain { @@ -396,6 +399,7 @@ struct apple_dart_domain { dma_addr_t dma_offset; struct mutex init_lock; struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; + DECLARE_BITMAP(fw_sidmap[MAX_DARTS_PER_DEVICE], DART_MAX_STREAMS); struct iommu_domain domain; }; @@ -831,7 +835,8 @@ apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, * before attachment, so replace the old root entries. * T8110 v2.2+ instead retains firmware-owned entries. */ - if (dart->hw->type != DART_T8110 || dart->version < 0x0202) { + if (!dart->fw_mirror && + (dart->hw->type != DART_T8110 || dart->version < 0x0202)) { size_t entry; dma_wmb(); @@ -1180,6 +1185,20 @@ static int apple_dart_iotlb_sync_map(struct iommu_domain *domain, iova, iova + size - 1); } +/* + * The core installs reserved mappings before it attaches the first device, + * while the attached-stream map is still empty. A locked stream whose + * firmware root the domain imported keeps its inherited translations from + * then on, so treat it as part of the domain for firmware lookups. + */ +static bool apple_dart_domain_uses_fw(struct apple_dart_domain *domain, int i, + struct apple_dart_atomic_stream_map *map, + int sid) +{ + return test_bit(sid, domain->fw_sidmap[i]) || + (atomic_long_read(&map->sidmap[BIT_WORD(sid)]) & BIT_MASK(sid)); +} + /* * Hardware continues to use every firmware-owned slot. Accept a private * software mapping only when every stream using such a slot agrees on the @@ -1217,8 +1236,7 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, u64 pte; int fw_prot; - if (!(atomic_long_read(&map->sidmap[BIT_WORD(sid)]) & - BIT_MASK(sid))) + if (!apple_dart_domain_uses_fw(domain, i, map, sid)) continue; fw = dart->locked_fw[sid][idx]; if (!dart->locked_ttbr[sid][idx] || @@ -1265,7 +1283,7 @@ apple_dart_inherited_phys(struct apple_dart_domain *domain, dma_addr_t address) phys_addr_t phys; int prot; - if (!(atomic_long_read(&map->sidmap[BIT_WORD(sid)]) & BIT_MASK(sid))) + if (!apple_dart_domain_uses_fw(domain, i, map, sid)) continue; fw = dart->locked_fw[sid][0]; if (!apple_dart_fw_slot_owned(fw, dart->locked_ttbr[sid][0], slot) || @@ -1524,6 +1542,8 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, goto done; } } + bitmap_copy(dart_domain->fw_sidmap[i], stream->sidmap, + stream->dart->num_streams); } } From 854f2c4c8ada7a960761c52344ec5171ee0ceff0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:04:57 +1100 Subject: [PATCH 0689/1579] drm: apple: accept H17P runtime callbacks within their reply bounds The H17P callback dispatcher only accepts callbacks whose record lengths appear in the startup observations. Every other callback that has a handler is treated as unqualified and marks the coprocessor as crashed, which stops all presents until the next reboot. Twenty-six handled callbacks are missing from those observations because they arrive only after startup: present notifications, swap aborts, the backlight re-send request, the wall-clock and clock-rate queries, and several acknowledgements. A long session that sees any of them loses the internal panel even though the driver knows how to answer. Record, for each of them, the input its handler reads and the reply it writes, and accept any record at least that large. The dispatcher already zeroes the whole reply before the handler runs, so a longer reply carries zeroes beyond the handler's fields. The measured startup tuples stay exact, and tags without a handler are still answered with a zeroed reply as before. Add a KUnit case that every installed handler has bounds and that the exact and bounded rules hold. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 57 ++++++++++++++++++++++++- drivers/gpu/drm/apple/iomfb_h17p.h | 4 ++ drivers/gpu/drm/apple/iomfb_h17p_test.c | 28 ++++++++++++ 3 files changed, 88 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 2de0fbbd4620a..cf708655b3b34 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -14,18 +14,32 @@ struct dcp_callback_size { u32 in_len; u32 out_len; bool valid; + /* Lengths bound what the handler reads and writes; no capture exists. */ + bool minimum; }; +/* + * Callbacks that only arrive after startup: present notifications, swap + * aborts, the backlight re-send request, wall-clock and clock-rate queries. + * Their record lengths are not in the startup observations, so accept any + * record that holds what the handler reads and the reply it writes. The + * dispatcher zeroes the rest of a longer reply. + */ +#define DCP_CB_AT_LEAST(in, out) \ + { .in_len = (in), .out_len = (out), .valid = true, .minimum = true } + /* R-IOMFB-startup-run21 and R-ANALYTICS Linux boundary observations. */ static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { [0] = { 0x0, 0x4, true }, [1] = { 0x0, 0x4, true }, + [2] = DCP_CB_AT_LEAST(0x0, 0x0), [3] = { 0x4, 0x14, true }, [6] = { 0x54, 0x50, true }, [100] = { 0x0, 0x0, true }, [101] = { 0x0, 0x4, true }, [102] = { 0x44, 0x0, true }, [104] = { 0x44, 0x0, true }, + [107] = DCP_CB_AT_LEAST(0x0, 0x0), [108] = { 0x0, 0x4, true }, [109] = { 0x0, 0x4, true }, [110] = { 0x0, 0x4, true }, @@ -33,34 +47,58 @@ static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { [112] = { 0x0, 0x4, true }, [113] = { 0x0, 0x4, true }, [114] = { 0x1044, 0x1004, true }, + [117] = DCP_CB_AT_LEAST(0x0, 0x0), + [118] = DCP_CB_AT_LEAST(0x0, 0x4), + [119] = DCP_CB_AT_LEAST(0x0, 0x4), + [120] = DCP_CB_AT_LEAST(0x0, 0x4), [121] = { 0x0, 0x4, true }, + [122] = DCP_CB_AT_LEAST(0x0, 0x1), [123] = { 0x0, 0x4, true }, + [124] = DCP_CB_AT_LEAST(0x0, 0x4), [125] = { 0x64, 0x24, true }, [127] = { 0x4, 0x4, true }, [128] = { 0x1008, 0x4, true }, [129] = { 0x40, 0x4, true }, + [130] = DCP_CB_AT_LEAST(0x22, 0x14), [201] = { 0xc, 0x10, true }, [202] = { 0x1a, 0x0, true }, [206] = { 0x0, 0x4, true }, [207] = { 0x0, 0x4, true }, + [208] = DCP_CB_AT_LEAST(0x0, 0x0), + [209] = DCP_CB_AT_LEAST(0x0, 0x8), [300] = { 0x10, 0x0, true }, [400] = { 0x4c, 0xc04, true }, [401] = { 0x50, 0xc, true }, + [404] = DCP_CB_AT_LEAST(0x0, 0x0), [406] = { 0x48, 0x0, true }, + [408] = DCP_CB_AT_LEAST(0x8, 0x8), [411] = { 0x10, 0x1c, true }, [413] = { 0x1048, 0x4, true }, [414] = { 0x50, 0x4, true }, [415] = { 0x4c, 0x4, true }, [451] = { 0x14, 0x1c, true }, + [452] = DCP_CB_AT_LEAST(0x18, 0x14), [454] = { 0x4, 0x1, true }, [552] = { 0x1044, 0x4, true }, [561] = { 0x1044, 0x4, true }, [563] = { 0x4c, 0x4, true }, [565] = { 0x48, 0x4, true }, + [567] = DCP_CB_AT_LEAST(0x0, 0x1), + [572] = DCP_CB_AT_LEAST(0x0, 0x4), [574] = { 0x4, 0x8, true }, [575] = { 0x58, 0x4c, true }, + [576] = DCP_CB_AT_LEAST(0x0, 0x0), [582] = { 0x8, 0x4, true }, + [583] = DCP_CB_AT_LEAST(0x0, 0x0), + [584] = DCP_CB_AT_LEAST(0x0, 0x0), + [585] = DCP_CB_AT_LEAST(0xe4, 0xe0), [590] = { 0x730, 0x0, true }, + [592] = DCP_CB_AT_LEAST(0x11, 0x0), + [593] = DCP_CB_AT_LEAST(0x4, 0x0), + [594] = DCP_CB_AT_LEAST(0x1, 0x0), + [595] = DCP_CB_AT_LEAST(0x0, 0x0), + [597] = DCP_CB_AT_LEAST(0x0, 0x1), + [598] = DCP_CB_AT_LEAST(0x0, 0x1), [599] = { 0x0, 0x0, true }, }; @@ -71,7 +109,11 @@ bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len) if (tag < 0 || tag >= ARRAY_SIZE(callback_sizes)) return false; size = &callback_sizes[tag]; - return size->valid && size->in_len == in_len && size->out_len == out_len; + if (!size->valid) + return false; + if (size->minimum) + return in_len >= size->in_len && out_len >= size->out_len; + return size->in_len == in_len && size->out_len == out_len; } struct dcp_h17p_hotplug_request { @@ -659,5 +701,18 @@ void iomfb_start_h17p(struct apple_dcp *dcp) dcp_start_signal(dcp, false, dcp_started, NULL); } +#if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) +/* Returns the first handled tag without reply bounds, or -1. */ +int iomfb_h17p_first_unbounded_callback(void) +{ + int tag; + + for (tag = 0; tag < IOMFB_MAX_CB; tag++) + if (cb_handlers[tag] && !callback_sizes[tag].valid) + return tag; + return -1; +} +#endif + #undef DCP_FW_VER #undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 9473940250297..44e524abd368a 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -88,6 +88,10 @@ void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, const struct dcp_swap_submit_req_h17p *request); +#if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) +int iomfb_h17p_first_unbounded_callback(void); +#endif + #undef DCP_FW_VER #undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c index c2a71ffa8cdbf..dcd26fb1a9ed3 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p_test.c +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -68,9 +68,37 @@ static void h17p_opaque_x_parameter_test(struct kunit *test) KUNIT_EXPECT_EQ(test, property.value, 0U); } +static void h17p_callback_bounds_test(struct kunit *test) +{ + /* Every installed handler is reachable without failing the session. */ + KUNIT_EXPECT_EQ(test, iomfb_h17p_first_unbounded_callback(), -1); + + /* Startup observations stay exact. */ + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(114, 0x1044, 0x1004)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(114, 0x1044, 0x1008)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(590, 0x734, 0x0)); + + /* Runtime callbacks must hold what their handlers read and write. */ + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(209, 0x0, 0x8)); + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(209, 0x4, 0xc)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(209, 0x0, 0x4)); + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(593, 0x4, 0x0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(593, 0x2, 0x0)); + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(594, 0x4, 0x0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(594, 0x0, 0x0)); + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(585, 0xe4, 0xe0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(585, 0xe0, 0xe0)); + + /* Tags without a handler or out of range never validate. */ + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(5, 0x0, 0x0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(-1, 0x0, 0x0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(IOMFB_MAX_CB, 0x0, 0x0)); +} + static struct kunit_case h17p_present_cases[] = { KUNIT_CASE(h17p_surface_wire_test), KUNIT_CASE(h17p_opaque_x_parameter_test), + KUNIT_CASE(h17p_callback_bounds_test), {} }; From 532179dd9fdd7a3e1bf20797aefafb7ad93119b9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:06:36 +1100 Subject: [PATCH 0690/1579] drm: apple: bound firmware register mapping requests The prepare callback maps a physical display register range for the coprocessor. Its size arrives from the firmware as a 64-bit value but is rounded into an int, so a large size wraps negative. The window check then compares a wrapped end address, and dma_map_resource() receives the sign-extended size. The descriptor ID is also taken from find_first_zero_bit() without checking for a full table, so a mapping requested while all 128 descriptors are in use writes past the array. A failed DMA mapping is only warned about and returned as the device address. Keep the size in 64 bits, refuse a zero size or a range that wraps, refuse when no descriptor is free, and refuse a failed DMA mapping before claiming a descriptor. Each refusal is the existing all-zero reply already used for addresses outside the display windows. A valid request is mapped and answered exactly as before. H17P accepts this callback at runtime from now on, which makes the bounds matter there too. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 38eb01e814021..3b17a9569c4f4 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -777,24 +777,37 @@ static bool is_disp_register(struct apple_dcp *dcp, u64 start, u64 end) static struct dcp_map_physical_resp dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) { - int size = ALIGN(req->size, 4096); + u64 size, end; dma_addr_t dva; u32 id; - if (!is_disp_register(dcp, req->paddr, req->paddr + size - 1)) { + /* Both values come from the coprocessor; reject wrapping ranges. */ + if (!req->size || check_add_overflow(req->size, 4095ULL, &size) || + check_add_overflow(req->paddr, round_down(size, 4096) - 1, &end) || + !is_disp_register(dcp, req->paddr, end)) { dev_err(dcp->dev, "refusing to map phys address %llx size %llx\n", req->paddr, req->size); return (struct dcp_map_physical_resp){}; } + size = round_down(size, 4096); id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + if (id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring\n"); + return (struct dcp_map_physical_resp){}; + } + + dva = dma_map_resource(dcp->dev, req->paddr, size, DMA_BIDIRECTIONAL, 0); + if (dma_mapping_error(dcp->dev, dva)) { + dev_err(dcp->dev, "failed to map phys address %llx size %llx\n", + req->paddr, size); + return (struct dcp_map_physical_resp){}; + } + set_bit(id, dcp->memdesc_map); dcp->memdesc[id].size = size; dcp->memdesc[id].reg = req->paddr; - dva = dma_map_resource(dcp->dev, req->paddr, size, DMA_BIDIRECTIONAL, 0); - WARN_ON(dva == DMA_MAPPING_ERROR); - return (struct dcp_map_physical_resp){ .dva_size = size, .mem_desc_id = id, From 239607cb12c3acbf7c424f77b3a8e6580a0b2d49 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:09:58 +1100 Subject: [PATCH 0691/1579] drm: apple: finish H17P presents that the firmware aborts H17P reports an aborted present through its abort callback and never sends a completion for it. The driver only traced the callback, so the present stayed pending: the outbound queue waited until its ten-second timeout declared the coprocessor crashed, and the page-flip event was never signalled. Track the abort in the present state. After the submit reply, an abort ends the present like a rejected submission: the brightness reservation is retried, framebuffers displaced by the present are unarmed and stay retained for the next completed present, and the DRM event is signalled. An abort that arrives before the submit reply is recorded, and the reply then fails the present instead of waiting for a completion. An abort naming a present that is not in flight is only logged. Older firmware still only traces the callback. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.h | 23 +++++++++++++++- drivers/gpu/drm/apple/iomfb_h17p_test.c | 33 +++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 36 +++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 44e524abd368a..94ac90be4247a 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -21,6 +21,8 @@ struct dcp_present_state_h17p { u32 swap_id; bool pending; bool accepted; + /* The firmware aborted the present before its submit reply. */ + bool aborted; }; static inline bool @@ -31,6 +33,7 @@ dcp_present_begin_h17p(struct dcp_present_state_h17p *state, u32 swap_id) state->swap_id = swap_id; state->pending = true; state->accepted = false; + state->aborted = false; return true; } @@ -41,11 +44,29 @@ dcp_present_submit_h17p(struct dcp_present_state_h17p *state, u32 swap_id, if (!state->pending || state->accepted || state->swap_id != swap_id) return false; state->accepted = accepted; - if (!accepted) + if (!accepted || state->aborted) state->pending = false; return true; } +/* + * An aborted present never completes. Before its submit reply, the abort is + * recorded and the reply ends the present instead. + */ +static inline bool +dcp_present_abort_h17p(struct dcp_present_state_h17p *state, u32 swap_id) +{ + if (!state->pending || state->swap_id != swap_id) + return false; + if (state->accepted) { + state->pending = false; + state->accepted = false; + } else { + state->aborted = true; + } + return true; +} + static inline bool dcp_present_complete_h17p(struct dcp_present_state_h17p *state, u32 swap_id) { diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c index dcd26fb1a9ed3..70fecfe52c0ff 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p_test.c +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -95,10 +95,43 @@ static void h17p_callback_bounds_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(IOMFB_MAX_CB, 0x0, 0x0)); } +static void h17p_present_abort_test(struct kunit *test) +{ + struct dcp_present_state_h17p state = {}; + + /* An abort after acceptance ends the present without a completion. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 7)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 7, true)); + KUNIT_EXPECT_FALSE(test, dcp_present_abort_h17p(&state, 6)); + KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_TRUE(test, dcp_present_abort_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, state.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, dcp_present_abort_h17p(&state, 7)); + + /* An abort before the submit reply leaves the reply to finish it. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, dcp_present_abort_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_TRUE(test, state.aborted); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 8, true)); + KUNIT_EXPECT_FALSE(test, state.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 8)); + + /* The next present starts clean and completes normally. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 9)); + KUNIT_EXPECT_FALSE(test, state.aborted); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 9, true)); + KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_TRUE(test, dcp_present_complete_h17p(&state, 9)); + KUNIT_EXPECT_FALSE(test, state.pending); +} + static struct kunit_case h17p_present_cases[] = { KUNIT_CASE(h17p_surface_wire_test), KUNIT_CASE(h17p_opaque_x_parameter_test), KUNIT_CASE(h17p_callback_bounds_test), + KUNIT_CASE(h17p_present_abort_test), {} }; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 3b17a9569c4f4..dfbcbee60a5c7 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1386,6 +1386,12 @@ static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted dcp->crashed = true; return false; } + /* An abort received before this reply turns acceptance into failure. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + accepted && dcp->present_state_h17p.aborted) { + dev_warn(dcp->dev, "firmware aborted present %u\n", swap_id); + return false; + } #endif return true; } @@ -1842,10 +1848,40 @@ dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, info->width, info->height); } +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER +/* + * An aborted H17P present never completes. Finish it like a rejected one: + * keep the displaced framebuffers for the next completed present and signal + * the DRM event, so neither the queue nor userspace waits for a completion. + */ +static void dcp_present_aborted(struct apple_dcp *dcp, u32 swap_id) +{ + if (!dcp_present_abort_h17p(&dcp->present_state_h17p, swap_id)) { + dev_warn(dcp->dev, "abort for present %u, which is not in flight\n", + swap_id); + return; + } + /* The submit reply has not arrived; it finishes the present. */ + if (dcp->present_state_h17p.pending) + return; + + dev_warn(dcp->dev, "firmware aborted present %u\n", swap_id); + dcp_present_failed(dcp); + if (dcp_present_retires(dcp)) { + dcp_unarm_retained_framebuffers(dcp, swap_id); + dcp_drm_crtc_vblank(dcp->crtc); + } +} +#endif + static void dcpep_cb_abort_swap_ap_gated(struct apple_dcp *dcp, u32 *swap_id) { trace_iomfb_abort_swap_ap_gated(dcp, *swap_id); +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + dcp_present_aborted(dcp, *swap_id); +#endif } static struct dcpep_get_tiling_state_resp From 3bb44edf85fd0ddd4b9101b1780343d290e509cb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:11:50 +1100 Subject: [PATCH 0692/1579] drm: apple: re-send the H17P brightness when the firmware asks The enable-backlight-message callback asks the AP to send the panel level again. On H17P the level travels only inside a present, so the existing handler, which sets the older firmware's update flag, sends nothing: the brightness policy sees no pending change and the queued brightness transaction returns without presenting. On the measured H17P profile, mark the effective level dirty and let the update worker queue a brightness present, which re-presents the retained scanout as for any other level change. A blanked panel is re-sent at zero and keeps its restore target. Nothing is sent before Linux controls the level, while a present already carries it, or more than once a second, so a request repeated after every brightness present cannot keep the queue busy. The receive path only schedules the worker; it never queues a transaction itself. Older firmware keeps its existing handling. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 3 +++ drivers/gpu/drm/apple/dcp_backlight.c | 24 +++++++++++++++++ drivers/gpu/drm/apple/dcp_backlight.h | 14 ++++++++++ drivers/gpu/drm/apple/dcp_backlight_test.c | 31 ++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 9 +++++++ 5 files changed, 81 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 8e9e13e16488e..92b0275fbdea7 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -409,6 +409,9 @@ struct apple_dcp { spinlock_t lock; struct dcp_backlight_state state; void (*kick)(struct apple_dcp *dcp); + /* Rate limit for firmware re-send requests. */ + unsigned long resent_at; + bool resent; } backlight; /* Staging wire record; serialization never mutates the atomic inputs. */ diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index ef891ef860230..20662ca107d77 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -224,6 +224,30 @@ bool dcp_backlight_pending(struct apple_dcp *dcp) return pending; } +/* + * Called from the receive path, which must not queue a transaction itself; + * the caller schedules the update worker when this returns true. Requests + * are honoured at most once per second, so a firmware that repeats the + * request after every brightness present cannot keep the queue busy. + */ +bool dcp_backlight_resend(struct apple_dcp *dcp) +{ + unsigned long flags; + bool resend = false; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + if (!dcp->backlight.resent || + time_after(jiffies, dcp->backlight.resent_at + HZ)) { + resend = dcp_bl_resend(&dcp->backlight.state); + if (resend) { + dcp->backlight.resent = true; + dcp->backlight.resent_at = jiffies; + } + } + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return resend; +} + unsigned int dcp_backlight_retry_delay(struct apple_dcp *dcp) { unsigned long flags; diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 2ad4abb4acd37..cb6dafd6bdf42 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -114,6 +114,19 @@ static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) return 0; } +/* + * The firmware asked for the level again. Re-send the effective level once + * Linux controls it, unless a present already carries or will carry it. + */ +static inline bool dcp_bl_resend(struct dcp_backlight_state *state) +{ + if (!state->ready || !state->controlled || state->dirty || + state->in_flight) + return false; + state->dirty = true; + return true; +} + /* * Called by the single outbound queue, with a pinned, accepted scanout or a * prepared replacement. Soft-off never releases it. A brightness-only present @@ -189,6 +202,7 @@ int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, struct dcp_backlight_present *present); bool dcp_backlight_complete(struct apple_dcp *dcp, u64 sequence, bool accepted); bool dcp_backlight_pending(struct apple_dcp *dcp); +bool dcp_backlight_resend(struct apple_dcp *dcp); unsigned int dcp_backlight_retry_delay(struct apple_dcp *dcp); #endif /* __APPLE_DCP_BACKLIGHT_H__ */ diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index a130f6f414712..441410bd5a84a 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -214,6 +214,36 @@ static void backlight_inflight_dpms_test(struct kunit *test) KUNIT_EXPECT_EQ(test, present.nits, 140U); } +static void backlight_resend_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* Nothing is re-sent before Linux controls the level. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, true, 140, false, 0), 0); + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); + KUNIT_EXPECT_FALSE(test, state.dirty); + + state.controlled = true; + KUNIT_EXPECT_TRUE(test, dcp_bl_resend(&state)); + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 140U); + /* The present in flight already carries the level. */ + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_FALSE(test, state.dirty); + + /* A blanked panel is re-sent dark, keeping the restore target. */ + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_TRUE(test, dcp_bl_resend(&state)); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_EQ(test, state.target, 140U); +} + static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), @@ -224,6 +254,7 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_surface_and_busy_test), KUNIT_CASE(backlight_reject_test), KUNIT_CASE(backlight_inflight_dpms_test), + KUNIT_CASE(backlight_resend_test), {} }; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index dfbcbee60a5c7..d6f754c0bb712 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -873,6 +873,15 @@ dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) static void iomfbep_cb_enable_backlight_message_ap_gated(struct apple_dcp *dcp, u8 *enabled) { +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + /* H17P carries the level only in presents; ask for one more. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + if (dcp_has_panel(dcp) && dcp_backlight_active(dcp) && + dcp_backlight_resend(dcp)) + schedule_work(&dcp->bl_update_wq); + return; + } +#endif /* * update backlight brightness on next swap, on non mini-LED displays * DCP seems to set an invalid iDAC value after coming out of DPMS. From 07a6dd6631fc15de3b23938339d486bb33d79f7d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:12:30 +1100 Subject: [PATCH 0693/1579] drm: apple: expose the full H17P panel brightness range The H17P backlight is commanded in nits, and the firmware converts them with its own tables up to the panel ceiling from the device tree, 525 nits on J700. Takeover still clamps the exposed range to 509 nits, the end of the older firmware's DAC interpolation table, which H17P never uses. The top of the panel's range is therefore unreachable, and a loader level reported above 509 nits rejects the takeover hint, which leaves the backlight unregistered until a later report falls below it. Use the panel ceiling as the H17P maximum, and clamp a powerlog report above it to the ceiling instead of rejecting it. The present encoding already writes the same ceiling into every present. Older firmware keeps its existing range. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.c | 3 ++- drivers/gpu/drm/apple/dcp_backlight.h | 10 ++++++---- drivers/gpu/drm/apple/dcp_backlight_test.c | 9 +++++++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 20662ca107d77..3528f7a9dbc36 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -96,7 +96,8 @@ int dcp_backlight_takeover(struct apple_dcp *dcp, u32 millinits) if (dcp_backlight_active(dcp)) return 0; - maximum = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + /* H17P takes nits up to the panel ceiling, not the older DAC table's. */ + maximum = dcp->brightness.maximum; ret = dcp_bl_takeover_nits(maximum, millinits, &nits); if (ret) return ret; diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index cb6dafd6bdf42..75242111bd1de 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -6,6 +6,7 @@ #include #include +#include #include /* Commanded nits, independent of firmware encoding. No fade policy. */ @@ -30,14 +31,15 @@ struct dcp_backlight_present { u32 nits; }; -/* Public powerlog reports use millinits; validate before rounding to nits. */ +/* + * Public powerlog reports use millinits. A report above the panel ceiling + * describes the brightest level the panel can show, so clamp it. + */ static inline int dcp_bl_takeover_nits(u32 maximum, u32 millinits, u32 *nits) { if (!maximum || maximum > INT_MAX) return -EINVAL; - if ((u64)millinits > (u64)maximum * 1000) - return -ERANGE; - *nits = millinits / 1000; + *nits = min(millinits / 1000, maximum); return 0; } diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index 441410bd5a84a..12e039d5ce0a1 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -13,9 +13,14 @@ static void backlight_millinits_takeover_test(struct kunit *test) KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(0, 100000, &nits), -EINVAL); KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(U32_MAX, 100000, &nits), -EINVAL); - KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, 509001, &nits), -ERANGE); - KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), -ERANGE); KUNIT_EXPECT_EQ(test, nits, 123U); + /* Reports above the panel ceiling clamp to it. */ + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(525, 525999, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 525U); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 509001, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 509U); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 509U); KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 100999, &nits), 0); KUNIT_EXPECT_EQ(test, nits, 100U); KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 509, true, nits, false, 0), 0); From 34cd8f3032b337cf4b50f4d40ebf3a9ce55ee716 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:14:54 +1100 Subject: [PATCH 0694/1579] drm: apple: register the H17P backlight without a loader level H17P never publishes the panel level. The driver registers its backlight only when the system endpoint forwards a powerlog report with the loader's level. That report is optional: when it does not arrive, or arrives in a form the parser does not accept, no backlight device is ever registered and desktops offer no brightness control. Two seconds after the first pipe enable, register the backlight anyway if no report has configured it. Expose the middle of the panel range as the current level, but present nothing: the policy starts clean, so the panel keeps the loader's level until userspace writes one. A report that arrives later is ignored, as are reports after a hint-based registration. The worker is cancelled with the other backlight work on unbind and shutdown. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 38 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 92b0275fbdea7..2f74527821944 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -335,6 +335,8 @@ struct apple_dcp { struct mutex bl_register_mutex; /* Workqueue for updating the brightness */ struct work_struct bl_update_wq; + /* Registers the H17P backlight when the loader reports no level. */ + struct delayed_work bl_fallback_wq; /* integrated panel if present */ struct dcp_panel panel; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a5f138f7cf6d7..280e2872683d8 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2661,6 +2661,8 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) } } +#define DCP_BL_FALLBACK_DELAY msecs_to_jiffies(2000) + static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) { return (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) || @@ -2686,6 +2688,10 @@ void dcp_poweron(struct platform_device *pdev) WRITE_ONCE(dcp->crashed, true); return; } + /* The powerlog takeover report normally precedes this. */ + if (dcp_has_panel(dcp)) + schedule_delayed_work(&dcp->bl_fallback_wq, + DCP_BL_FALLBACK_DELAY); } ret = dcp_backlight_dpms(dcp, true); if (ret) @@ -2833,6 +2839,34 @@ static void dcp_work_update_backlight(struct work_struct *work) dcp_backlight_update(dcp); } +/* + * H17P never publishes the panel level, and the loader's level arrives only + * as an optional powerlog report. Without one, register the backlight at + * the middle of the panel range. That value is only reported: no level is + * presented until userspace writes one, so the panel keeps the loader's + * level. + */ +static void dcp_work_backlight_fallback(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), + struct apple_dcp, bl_fallback_wq); + u32 nits = dcp->brightness.maximum / 2; + int ret; + + if (READ_ONCE(dcp->quiescing) || READ_ONCE(dcp->crashed) || + dcp_backlight_active(dcp)) + return; + + ret = iomfb_configure_backlight_h17p(dcp, dcp->brightness.maximum, + false, 0, true, nits); + if (!ret) + dev_info(dcp->dev, + "no loader brightness reported; backlight starts at %u nits without a panel change\n", + nits); + else if (ret != -EBUSY) + dev_warn(dcp->dev, "backlight registration failed: %d\n", ret); +} + static void dcp_release_piodma_iommu_dev(struct apple_dcp *dcp) { if (dcp->retain_dma) { @@ -3443,6 +3477,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) afk_quiesce(dcp->ibootep); afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); + cancel_delayed_work_sync(&dcp->bl_fallback_wq); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); @@ -3497,6 +3532,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) iomfb_queue_stop(dcp); dcp_release_piodma_iommu_dev(dcp); + cancel_delayed_work_sync(&dcp->bl_fallback_wq); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); @@ -3585,6 +3621,7 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp_placeholder_edid_work); INIT_DELAYED_WORK(&dcp->typec_fabric_retrain_wq, dcp_typec_retrain_work); + INIT_DELAYED_WORK(&dcp->bl_fallback_wq, dcp_work_backlight_fallback); /* Balanced by enable at successful component bind. */ disable_delayed_work(&dcp->typec_reconnect_wq); disable_delayed_work(&dcp->placeholder_edid_wq); @@ -3748,6 +3785,7 @@ static void dcp_platform_shutdown(struct platform_device *pdev) * its DMA resources alive until the system resets. */ iomfb_queue_stop(dcp); + cancel_delayed_work_sync(&dcp->bl_fallback_wq); cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); return; From e7750ad831eecb499fbe84b8c5ec933db6f1c3ab Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:15:31 +1100 Subject: [PATCH 0695/1579] drm: apple: drain H17P display updates before system sleep H17P keeps its pipe and display domains running across system sleep, and blanks the panel with a brightness present instead. The DRM and backlight suspend handlers only queue that present: the serialized IOMFB queue sends it, and finishes it when the firmware reports the completion. The DCP suspend handler returned without waiting, so the blanking present could still be in flight when interrupts are suspended, leaving the panel lit through the sleep and the transaction stalled until resume. Before the DCP suspends, flush the brightness worker and wait up to half a second for the queue to go idle, twice, since a completed present can schedule one more brightness present. A crashed or stopped queue counts as idle. If the queue does not drain, log it and continue suspending. Older firmware does not use the queue and is unaffected. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 25 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 26 ++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 2f74527821944..aa0a4096e23cd 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -457,6 +457,7 @@ int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, bool default_valid, u32 default_nits); void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); +bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 280e2872683d8..ab0ed76191c13 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3794,10 +3794,35 @@ static void dcp_platform_shutdown(struct platform_device *pdev) component_del(&pdev->dev, &dcp_comp_ops); } +/* + * The display stays powered across system sleep, and the blanking present + * queued by the DRM and backlight suspend handlers must reach the firmware + * before its mailbox interrupt is suspended. + */ +static void dcp_drain_for_sleep(struct apple_dcp *dcp) +{ + int pass; + + if (!dcp->hw.adopt_live_session || dcp->fw_compat != DCP_FIRMWARE_H17P || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return; + + /* A completed present can schedule one more brightness present. */ + for (pass = 0; pass < 2; pass++) { + if (dcp_has_panel(dcp)) + flush_work(&dcp->bl_update_wq); + if (!iomfb_queue_drain(dcp, msecs_to_jiffies(500))) { + dev_warn(dcp->dev, "display updates still pending at suspend\n"); + return; + } + } +} + static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + dcp_drain_for_sleep(dcp); /* * The Type-C route reports cable removal through * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index de78df01c5edc..0a8b5aaea5f11 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -388,6 +388,32 @@ void iomfb_queue_stop(struct apple_dcp *dcp) cancel_delayed_work_sync(&dcp->iomfb.backlight_retry); } +/* + * Wait until every queued transaction, including a present's completion, has + * finished. A stopped or crashed queue has nothing left to deliver. + */ +bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout) +{ + unsigned long deadline = jiffies + timeout; + bool idle; + + if (!iomfb_uses_queue(dcp)) + return true; + + for (;;) { + mutex_lock(&dcp->iomfb.lock); + idle = READ_ONCE(dcp->crashed) || dcp->iomfb.stopped || + (!dcp->iomfb.active && list_empty(&dcp->iomfb.pending) && + !dcp->present_state_h17p.pending); + mutex_unlock(&dcp->iomfb.lock); + if (idle) + return true; + if (time_after(jiffies, deadline)) + return false; + usleep_range(2000, 4000); + } +} + int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) { int ret = 0; From 21a68c947936c2b070b2b1425c4d3c03ec415e09 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:15:58 +1100 Subject: [PATCH 0696/1579] drm: apple: signal pending DRM events when H17P firmware crashes When the H17P firmware crashes, the driver marks the coprocessor failed and reports the connector disconnected. A present in flight at that moment never completes, so its page-flip event is only released by the DRM core's flip timeout, and the compositor stalls on it first. Schedule the existing delayed vblank work from the crash handler on H17P so the pending event is signalled. Framebuffers stay retained, since the display engine may still be reading the last surface. H17P has no in-place restart, so also log that a reboot is needed to restore the display. Older firmware keeps its existing crash handling. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index ab0ed76191c13..9a9399fb51ba6 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1968,6 +1968,12 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ dcp->crashed = true; dev_err(dcp->dev, "DCP has crashed\n"); + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + /* No in-place restart exists; the last scanout stays retained. */ + dev_err(dcp->dev, "a reboot is required to restore the display\n"); + /* Nothing will complete a present that was in flight. */ + schedule_work(&dcp->vblank_wq); + } if (dcp->connector) { dcp->connector->connected = 0; drm_edid_free(dcp->connector->drm_edid); From 22d54d0a012a50c0e05ece648add66d14582d858 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:16:49 +1100 Subject: [PATCH 0697/1579] drm: apple: keep the internal panel when the T8140 external DCP is enabled The display subsystem builds its component match from every available DCP node and binds only when all of them have bound. The T8140 external DCP runs firmware this driver cannot drive yet: its node falls back to the generic DCP match, which has no firmware profile for it. Its probe then either fails, leaving the component master waiting forever, or selects an older firmware interface from the loader's properties. Enabling that node in a device tree would therefore take the internal panel down with it. Leave T8140 external DCP nodes out of the component match and the pipeline scan, and refuse them in the DCP probe with an informational message, so they are never touched. Other SoCs are unaffected. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 15 +++++++++++++-- drivers/gpu/drm/apple/dcp.c | 6 ++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index be8da0dd4915d..991517b43dcec 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -640,6 +640,17 @@ static const struct of_device_id apple_dcp_id_tbl[] = { {}, }; +/* + * The T8140 external display coprocessor needs firmware support this driver + * does not have yet. Leave it out of the display subsystem, so that an + * enabled node cannot hold back the internal panel's component master. + */ +static bool apple_dcp_usable(struct device_node *np) +{ + return of_device_is_available(np) && + !of_device_is_compatible(np, "apple,t8140-dcpext"); +} + static int apple_drm_init_dcp(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); @@ -652,7 +663,7 @@ static int apple_drm_init_dcp(struct device *dev) for_each_matching_node(np, apple_dcp_id_tbl) { bool dcp_ext; - if (!of_device_is_available(np)) { + if (!apple_dcp_usable(np)) { of_node_put(np); continue; } @@ -831,7 +842,7 @@ static int add_dcp_components(struct device *dev, int num = 0; for_each_matching_node(np, apple_dcp_id_tbl) { - if (of_device_is_available(np)) { + if (apple_dcp_usable(np)) { drm_of_component_match_add(dev, matchptr, component_compare_of, np); num++; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 9a9399fb51ba6..06d2d31feb051 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3570,6 +3570,12 @@ static int dcp_platform_probe(struct platform_device *pdev) if (dcp_session_retained(dev)) return dev_err_probe(dev, -EBUSY, "Previous live DCP session requires a reboot\n"); + /* Not part of the display subsystem yet; see apple_dcp_usable(). */ + if (of_device_is_compatible(dev->of_node, "apple,t8140-dcpext")) { + dev_info(dev, "external display coprocessor not supported yet\n"); + return -ENODEV; + } + fw_compat = dcp_check_firmware_version(dev); if (fw_compat == DCP_FIRMWARE_UNKNOWN) return -ENODEV; From 0abe58c8aec29bcd517a54464aadfe1132653f20 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:20:17 +1100 Subject: [PATCH 0698/1579] soc: apple: sep: remove the resurrected private copy of the UAPI header drivers/soc/apple/sep-bio.h became include/uapi/linux/apple-sep.h when the biometric interface was exported, but composing the persistence work with the authenticated sensor sessions brought the old private copy back. Nothing includes it: the Rust driver mirrors the layout itself, and user space uses the UAPI header. It only gives the interface a second definition that has to be kept in step by hand. Remove it again. Fixes: 42ae23434d4c ("soc: apple: compose SEP persistence with authenticated sensor sessions") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sep-bio.h | 130 ------------------------------------ 1 file changed, 130 deletions(-) delete mode 100644 drivers/soc/apple/sep-bio.h diff --git a/drivers/soc/apple/sep-bio.h b/drivers/soc/apple/sep-bio.h deleted file mode 100644 index d52b880ac9332..0000000000000 --- a/drivers/soc/apple/sep-bio.h +++ /dev/null @@ -1,130 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Copyright 2026 Dj */ -/* - * Userspace interface for the SEP biometric device. - * - * The enclave does the matching. Nothing biometric crosses this interface: only - * an operation, a stage, a status, an opaque identity UUID, and an opaque host - * label userspace chooses. - */ - -#pragma once - -#include -#include - -#define SEP_BIO_IFACE_VERSION 4 - -#define SEP_BIO_UUID_LEN 16 -#define SEP_BIO_LABEL_LEN 128 -#define SEP_BIO_NONCE_LEN 32 -#define SEP_BIO_TOKEN_LEN 32 -#define SEP_BIO_MAX_IDENTITIES 32 -#define SEP_BIO_CHALLENGE_LEN 32 -#define SEP_BIO_ATTEST_PUB_LEN 65 -#define SEP_BIO_ATTEST_SIG_MAX 72 - -enum { - SEP_BIO_STATE_IDLE = 0, - SEP_BIO_STATE_PENDING = 1, - SEP_BIO_STATE_PROGRESS = 2, - SEP_BIO_STATE_DONE = 3, - SEP_BIO_STATE_FAILED = 4, -}; - - -enum { - SEP_BIO_NO_MATCH = 0, - SEP_BIO_MATCH = 1, - SEP_BIO_NOT_COMPARED = 2, -}; - -struct sep_bio_identity { - __u8 uuid[SEP_BIO_UUID_LEN]; - __u8 label[SEP_BIO_LABEL_LEN]; -}; - -struct sep_bio_info { - __u32 version; - __u32 sensor_present; - __u32 enrolled; - __u32 capacity; /* usable slots in this SEP user context, not ABI list size */ - __u32 enroll_stages; - __u32 reserved[3]; -}; - -/* reserved[0] flags; reserved[1] is a live SEP identity count only when valid. */ -#define SEP_BIO_INFO_LIVE_COUNT_VALID 1u - -struct sep_bio_list { - __u32 count; - __u32 reserved; - struct sep_bio_identity id[SEP_BIO_MAX_IDENTITIES]; -}; - -struct sep_bio_enrol_start { - __u32 flags; - __u32 reserved; - __u8 label[SEP_BIO_LABEL_LEN]; -}; - -#define SEP_BIO_GUIDANCE_NONE 0 -#define SEP_BIO_GUIDANCE_PLACE 1 -#define SEP_BIO_GUIDANCE_LIFT_AND_MOVE 2 -#define SEP_BIO_GUIDANCE_HOLD_STILL 3 - -struct sep_bio_enrol_poll { - __u32 state; - __u32 stage; - __u32 stages_total; - __u32 status; - __u8 uuid[SEP_BIO_UUID_LEN]; - __u32 guidance; - __u32 progress_percent; -}; -struct sep_bio_verify_start { - __u32 flags; - __u32 reserved; - __u8 nonce[SEP_BIO_NONCE_LEN]; -}; - -struct sep_bio_verify_poll { - __u32 state; - __u32 result; - __u32 status; - __u32 guidance; /* SEP_BIO_GUIDANCE_*; status only, no capture side effect */ - __u8 uuid[SEP_BIO_UUID_LEN]; - __u8 token[SEP_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ - __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ -}; - -struct sep_bio_delete { - __u8 uuid[SEP_BIO_UUID_LEN]; -}; - -/* - * Device attestation of key possession: the enclave signs 'challenge' with the - * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and - * returns the DER signature and public point. The private key never leaves the - * enclave. - */ -struct sep_bio_attest { - __u32 sig_len; /* out: DER signature length */ - __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ - __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ - __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ - __u8 reserved[3]; -}; - -#define SEP_BIO_IOC_MAGIC 0xB1 - -#define SEP_BIO_GET_INFO _IOR (SEP_BIO_IOC_MAGIC, 0x01, struct sep_bio_info) -#define SEP_BIO_LIST _IOR (SEP_BIO_IOC_MAGIC, 0x02, struct sep_bio_list) -#define SEP_BIO_ENROL_START _IOW (SEP_BIO_IOC_MAGIC, 0x03, struct sep_bio_enrol_start) -#define SEP_BIO_ENROL_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x04, struct sep_bio_enrol_poll) -#define SEP_BIO_VERIFY_START _IOW (SEP_BIO_IOC_MAGIC, 0x05, struct sep_bio_verify_start) -#define SEP_BIO_VERIFY_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x06, struct sep_bio_verify_poll) -#define SEP_BIO_CANCEL _IO (SEP_BIO_IOC_MAGIC, 0x07) -#define SEP_BIO_DELETE _IOW (SEP_BIO_IOC_MAGIC, 0x08, struct sep_bio_delete) -#define SEP_BIO_DELETE_ALL _IO (SEP_BIO_IOC_MAGIC, 0x09) -#define SEP_BIO_ATTEST _IOWR(SEP_BIO_IOC_MAGIC, 0x0a, struct sep_bio_attest) From 1b53d187515033890b9b2a608415eb6a35450a54 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:20:20 +1100 Subject: [PATCH 0699/1579] soc: apple: sep: rename the attestation public key member struct sep_bio_attest calls the attested public point "public", which is a C++ keyword. The UAPI header test compiles every exported header as C++ as well as C, so a configuration with CONFIG_UAPI_HEADER_TEST=y fails to build: ./usr/include/linux/apple-sep.h:115:8: error: expected member name or ';' after declaration specifiers and C++ programs cannot include the header at all. Rename the member to public_key, and rename the Rust mirror of the structure to match. Only the member name changes: every structure keeps its size, alignment and member offsets, and the ioctl numbers are unchanged. No in-tree user space accesses the member. Fixes: 5b4a74c4e0aa ("soc: apple: sep: export the biometric ioctl header through UAPI") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/bio.rs | 2 +- drivers/soc/apple/sbio.rs | 4 ++-- include/uapi/linux/apple-sep.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 13d7e99d43fc9..752e9ea7a6e12 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -149,7 +149,7 @@ pub(crate) struct Attest { /// in: challenge to sign. pub(crate) challenge: [u8; ATTEST_CHALLENGE_LEN], /// out: ref-key public point. - pub(crate) public: [u8; ATTEST_PUB_LEN], + pub(crate) public_key: [u8; ATTEST_PUB_LEN], /// out: DER signature, zero-padded to `ATTEST_SIG_MAX`. pub(crate) signature: [u8; ATTEST_SIG_MAX], pub(crate) reserved: [u8; 3], diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 2a19dd2ac216d..b45158ad5502d 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2948,11 +2948,11 @@ impl SepData { let mut out = bio::Attest { sig_len: sig.len() as u32, challenge: req.challenge, - public: [0u8; bio::ATTEST_PUB_LEN], + public_key: [0u8; bio::ATTEST_PUB_LEN], signature: [0u8; bio::ATTEST_SIG_MAX], reserved: [0u8; 3], }; - out.public.copy_from_slice(&pubk); + out.public_key.copy_from_slice(&pubk); out.signature[..sig.len()].copy_from_slice(&sig); kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) .writer() diff --git a/include/uapi/linux/apple-sep.h b/include/uapi/linux/apple-sep.h index 81467fe91f36a..394b497ff7a59 100644 --- a/include/uapi/linux/apple-sep.h +++ b/include/uapi/linux/apple-sep.h @@ -112,7 +112,7 @@ struct sep_bio_delete { struct sep_bio_attest { __u32 sig_len; /* out: DER signature length */ __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ - __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ + __u8 public_key[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ __u8 reserved[3]; }; From 90849763ad836fa850d0fd33a81409d6b0adde38 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:20:24 +1100 Subject: [PATCH 0700/1579] PCI/DOE: Export pci_doe_disconnected() for modular host controllers pci_dev_set_disconnected() is a static inline in drivers/pci/pci.h that a host controller driver can pass to pci_walk_bus() to mark a hierarchy it has lost as permanently disconnected before tearing it down. Besides setting the error state, it cancels pending DOE exchanges through pci_doe_disconnected(), which is not exported, so modpost rejects a modular caller with 'ERROR: modpost: "pci_doe_disconnected" [drivers/pci/controller/pcie-apple.ko] undefined!'. The Apple PCIe driver is such a caller since it learned to quiesce tunneled PCIe-C links, and it is tristate. Distribution configurations build APPLE_DART as a module, and "depends on APPLE_DART || !APPLE_DART" then makes PCIE_APPLE a module too. Export pci_doe_disconnected() GPL-only, like the rest of the DOE API. Open-coding the walk in the driver would skip the DOE cancellation and leave exchanges waiting on a device that can no longer answer. Fixes: 544b71a5d144 ("PCI: apple: quiesce and restore tunneled PCIe-C links") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/doe.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/pci/doe.c b/drivers/pci/doe.c index 7b41da4ec11a3..44be0d513a893 100644 --- a/drivers/pci/doe.c +++ b/drivers/pci/doe.c @@ -891,3 +891,4 @@ void pci_doe_disconnected(struct pci_dev *pdev) xa_for_each(&pdev->doe_mbs, index, doe_mb) pci_doe_cancel_tasks(doe_mb); } +EXPORT_SYMBOL_GPL(pci_doe_disconnected); From 85ee7788c7e336f18bfafed3f9837225654101a8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:17:16 +1100 Subject: [PATCH 0701/1579] HID: magicmouse: depend on a reachable MTP haptic actuator driver hid-magicmouse drives the trackpad actuator through apple_taptic_send() and apple_taptic_switch_modes(), which hid-apple-mtp-haptic exports, and guards that code with IS_REACHABLE(). With HID_MAGICMOUSE=y and HID_APPLE_MTP_HAPTIC=m the helpers are not reachable from built-in code, so the driver silently compiles the haptics out: the actuator driver binds, but the trackpad never gets a force feedback device or FF_HAPTIC, and nothing in the log says why. Use the usual optional-dependency idiom so that Kconfig limits hid-magicmouse to a module whenever the actuator driver is one, and test the helpers with IS_ENABLED(), which now matches what can be linked. Every valid configuration gets haptics when the actuator driver is enabled, and none when it is not. Fixes: cf248ccdaabc ("HID: magicmouse: add support for controlling MTP touchpad haptics") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/Kconfig | 1 + drivers/hid/hid-magicmouse.c | 4 ++-- include/linux/soc/apple/actuator.h | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index 3371082e799fb..12856c9352d70 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -749,6 +749,7 @@ config LOGIWHEELS_FF config HID_MAGICMOUSE tristate "Apple Magic Mouse/Trackpad multi-touch support" + depends on HID_APPLE_MTP_HAPTIC || !HID_APPLE_MTP_HAPTIC help Support for the Apple Magic Mouse/Trackpad multi-touch. diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index c382f940ff094..70d066a43279b 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -1402,7 +1402,7 @@ static int magicmouse_input_mapping(struct hid_device *hdev, static int magicmouse_init_haptics(struct magicmouse_sc *msc, struct hid_device *hdev); -#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +#if IS_ENABLED(CONFIG_HID_APPLE_MTP_HAPTIC) static int match_actuator(struct device *dev, const void *data) { struct hid_device *hdev; @@ -1880,7 +1880,7 @@ static void magicmouse_battery_timer_tick(struct timer_list *t) } } -#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +#if IS_ENABLED(CONFIG_HID_APPLE_MTP_HAPTIC) static bool magicmouse_effect_is_deep(const struct ff_effect *effect) { return effect->u.haptic.hid_usage == diff --git a/include/linux/soc/apple/actuator.h b/include/linux/soc/apple/actuator.h index 0b565d5567eb0..18e1bee0e266c 100644 --- a/include/linux/soc/apple/actuator.h +++ b/include/linux/soc/apple/actuator.h @@ -19,7 +19,7 @@ static inline bool apple_taptic_is_actuator(struct hid_device *hdev) !strcmp(hdev->name, "Apple SPI Actuator"); } -#if IS_REACHABLE(CONFIG_HID_APPLE_MTP_HAPTIC) +#if IS_ENABLED(CONFIG_HID_APPLE_MTP_HAPTIC) int apple_taptic_send(struct hid_device *hdev, u16 effect_type, u8 strength, u8 softness); int apple_taptic_switch_modes(struct hid_device *hdev, bool host_controlled); From 47d81d1ce33d9cbe354846294bc44b8a88112be4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 21:57:12 +1100 Subject: [PATCH 0702/1579] soc: apple: sep: rearm the cold-restore device-view proof The nonce-session restore path synchronizes the device view before COMPLETE_INIT to refresh the SCRD credential. Deferred restore readiness also uses device_view_synced to guard its final proof after COMPLETE_INIT. The preliminary sync leaves that flag set, so both finish_sensor_setup and retry_restore_proof skip the final proof and templates_restored remains false after a successful cold restore. Clear device_view_synced when deferring template readiness. Keep the preliminary credential refresh, but require the final device-view proof before marking templates ready. A failed final sync can still take the existing bounded sensor bring-up retry. Legacy profiles retain their normal proof path, and refused or empty identity proofs stay unready. Fixes: 42ae23434d4c ("soc: apple: compose SEP persistence with authenticated sensor sessions") Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit b4c0368990089654d5e1212926473137ac314956) --- drivers/soc/apple/sbio.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index b45158ad5502d..deff86802e440 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -3017,6 +3017,9 @@ impl SepData { // verification available on the preliminary Catacomb proof alone; // complete_bringup finishes the proof and sets templates_restored. self.templates_restored.store(false, Relaxed); + // The nonce path synced the view before COMPLETE_INIT to refresh SCRD. + // Re-arm the final proof and its bounded retry on every cold restore. + self.device_view_synced.store(false, Relaxed); if !candidate { dev_err!( self.dev, From 4f3e0ed14a9c280aafb3642619e7335413592e06 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 21:34:23 +1100 Subject: [PATCH 0703/1579] pmdomain: apple: drain PMP report users before teardown The report mapping and allocation are managed by the platform device, but the global RCU list and fast-die debugfs file outlive driver removal. Withdraw both publications and drain readers before devres releases the mapping. Install the teardown action before publishing the region so probe failures also revoke it. The existing DVFS debugfs action still runs before the mapping is released. Depopulate child providers first. Their power callbacks now look up the report inside an RCU read-side section instead of following a child's unmanaged parent pointer. Remove the child provider and generic domain on teardown, including failed probe paths. If genpd refuses removal of a busy domain, keep its independently allocated storage, name and node references alive, with callbacks returning ENODEV. This bounds the leak to the removed domain and avoids reclaiming memory still owned by genpd. Publish the report before populating children so their initial power requests can use the same lookup. Validate report bit indices and use a 64-bit mask for the 64-bit request register. Fixes: f69f1b76fcb0 ("pmdomain: apple: pmp-report: expose T8140 fast-die effort") Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 206b9b56f4b2bbdfc15efee3f23f5e5167f76bf8) --- drivers/pmdomain/apple/pmp-report.c | 151 ++++++++++++++++++++++------ 1 file changed, 119 insertions(+), 32 deletions(-) diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 1882654698771..2f17df448a1a8 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #define PMP_REPORT_READY 0x1 @@ -50,12 +51,10 @@ struct apple_pmp_report { void __iomem *base; spinlock_t lock; struct list_head list; + struct dentry *debugfs; }; -/* - * Bound report regions, for apple_pmp_report_fast_die_effort(). Regions are - * added once they are set up and never removed: the driver cannot be unbound. - */ +/* Bound report regions, protected by RCU until managed teardown drains readers. */ static LIST_HEAD(apple_pmp_reports); static DEFINE_MUTEX(apple_pmp_reports_lock); static struct dentry *apple_pmp_report_debugfs; @@ -132,6 +131,21 @@ static int apple_pmp_report_status_show(struct seq_file *s, void *unused) } DEFINE_SHOW_ATTRIBUTE(apple_pmp_report_status); +static void apple_pmp_report_release(void *data) +{ + struct apple_pmp_report *rep = data; + + /* Withdraw child providers while their power callbacks can still use MMIO. */ + of_platform_depopulate(rep->dev); + mutex_lock(&apple_pmp_reports_lock); + if (!list_empty(&rep->list)) + list_del_rcu(&rep->list); + mutex_unlock(&apple_pmp_reports_lock); + synchronize_rcu(); + /* The debugfs proxy drains active reads and rejects subsequent accesses. */ + debugfs_remove(rep->debugfs); +} + static int apple_pmp_report_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -154,17 +168,24 @@ static int apple_pmp_report_probe(struct platform_device *pdev) if (ret) return ret; - ret = of_platform_populate(np, NULL, NULL, dev); + INIT_LIST_HEAD(&rep->list); + ret = devm_add_action_or_reset(dev, apple_pmp_report_release, rep); if (ret) - return dev_err_probe(dev, ret, "failed to create child devices\n"); + return ret; mutex_lock(&apple_pmp_reports_lock); list_add_tail_rcu(&rep->list, &apple_pmp_reports); mutex_unlock(&apple_pmp_reports_lock); if (rep->offsets->fast_die) - debugfs_create_file(dev_name(dev), 0400, apple_pmp_report_debugfs, - rep, &apple_pmp_report_status_fops); + rep->debugfs = debugfs_create_file(dev_name(dev), 0400, + apple_pmp_report_debugfs, rep, + &apple_pmp_report_status_fops); + + /* Publish the region before child power callbacks start looking it up. */ + ret = of_platform_populate(np, NULL, NULL, dev); + if (ret) + return dev_err_probe(dev, ret, "failed to create child devices\n"); return 0; } @@ -213,28 +234,26 @@ static struct platform_driver apple_pmp_report_driver = { .driver = { .name = "apple-pmp-report", .of_match_table = apple_pmp_report_of_match, - /* - * Its child power domains and the region list keep using the - * mapping, and nothing tears them down. - */ + /* Power-domain providers are not intended for manual unbinding. */ .suppress_bind_attrs = true, }, }; struct apple_pmp_report_entry { - struct device *dev; + struct device_node *node; + struct device_node *report_node; struct generic_pm_domain genpd; u32 id; + bool removed; }; #define genpd_to_apple_pmp_report_entry(_genpd) \ container_of(_genpd, struct apple_pmp_report_entry, genpd) -static int apple_pmp_report_set_state(struct generic_pm_domain *genpd, bool enable) +static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id, + bool enable) { - struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); - struct apple_pmp_report *rep = dev_get_drvdata(ent->dev->parent); - u64 bit_val = 1 << ent->id; + u64 bit_val = BIT_ULL(id); u64 val; unsigned long flags; @@ -256,14 +275,34 @@ static int apple_pmp_report_set_state(struct generic_pm_domain *genpd, bool enab 50000); } +static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, bool enable) +{ + struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); + struct apple_pmp_report *rep; + int ret = -ENODEV; + + if (READ_ONCE(ent->removed)) + return ret; + + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node == ent->report_node) { + ret = apple_pmp_report_set_state(rep, ent->id, enable); + break; + } + } + rcu_read_unlock(); + return ret; +} + static int apple_pmp_report_entry_power_on(struct generic_pm_domain *genpd) { - return apple_pmp_report_set_state(genpd, true); + return apple_pmp_report_entry_set_state(genpd, true); } static int apple_pmp_report_entry_power_off(struct generic_pm_domain *genpd) { - return apple_pmp_report_set_state(genpd, false); + return apple_pmp_report_entry_set_state(genpd, false); } /* ADT DVFS-STATE: four 16-byte entries beginning at entry 8. */ @@ -336,6 +375,28 @@ static int apple_pmp_dvfs_create(struct apple_pmp_report *report) return 0; } +static void apple_pmp_report_entry_free(struct apple_pmp_report_entry *ent) +{ + of_node_put(ent->report_node); + of_node_put(ent->node); + kfree(ent->genpd.name); + kfree(ent); +} + +static void apple_pmp_report_entry_remove(struct platform_device *pdev) +{ + struct apple_pmp_report_entry *ent = platform_get_drvdata(pdev); + + WRITE_ONCE(ent->removed, true); + of_genpd_del_provider(ent->node); + if (pm_genpd_remove(&ent->genpd)) { + /* The core still owns this domain. Keep its now inert storage alive. */ + dev_warn(&pdev->dev, "retaining busy power domain %s\n", ent->genpd.name); + return; + } + apple_pmp_report_entry_free(ent); +} + static int apple_pmp_report_entry_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -345,36 +406,53 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) const char *name; struct of_phandle_iterator it; - ent = devm_kzalloc(dev, sizeof(*ent), GFP_KERNEL); + ent = kzalloc_obj(*ent); if (!ent) return -ENOMEM; - ent->dev = dev; + ent->node = of_node_get(node); + ent->report_node = of_node_get(dev->parent->of_node); ret = of_property_read_u32(node, "reg", &ent->id); - if (ret) - return dev_err_probe(dev, ret, "missing reg property\n"); + if (ret) { + ret = dev_err_probe(dev, ret, "missing reg property\n"); + goto err_free; + } + if (ent->id >= 64) { + ret = -EINVAL; + goto err_free; + } ret = of_property_read_string(node, "label", &name); - if (ret < 0) - return dev_err_probe(dev, ret, "missing label property\n"); + if (ret < 0) { + ret = dev_err_probe(dev, ret, "missing label property\n"); + goto err_free; + } + ent->genpd.name = kstrdup(name, GFP_KERNEL); + if (!ent->genpd.name) { + ret = -ENOMEM; + goto err_free; + } if (of_property_read_bool(node, "apple,always-on")) { ent->genpd.flags |= GENPD_FLAG_ACTIVE_WAKEUP; - apple_pmp_report_set_state(&ent->genpd, true); + apple_pmp_report_entry_set_state(&ent->genpd, true); } - ent->genpd.name = name; ent->genpd.power_on = apple_pmp_report_entry_power_on; ent->genpd.power_off = apple_pmp_report_entry_power_off; ret = pm_genpd_init(&ent->genpd, NULL, true); - if (ret) - return dev_err_probe(dev, ret, "pm_genpd_init failed\n"); + if (ret) { + ret = dev_err_probe(dev, ret, "pm_genpd_init failed\n"); + goto err_free; + } ret = of_genpd_add_provider_simple(node, &ent->genpd); - if (ret) - return dev_err_probe(dev, ret, "of_genpd_add_provider_simple failed\n"); + if (ret) { + ret = dev_err_probe(dev, ret, "of_genpd_add_provider_simple failed\n"); + goto err_remove_domain; + } of_for_each_phandle(&it, ret, node, "power-domains", "#power-domain-cells", -1) { struct of_phandle_args parent, child; @@ -397,11 +475,19 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) } pm_genpd_remove_device(dev); + platform_set_drvdata(pdev, ent); return 0; err_remove: of_genpd_del_provider(node); - pm_genpd_remove(&ent->genpd); +err_remove_domain: + WRITE_ONCE(ent->removed, true); + if (pm_genpd_remove(&ent->genpd)) { + dev_warn(dev, "retaining busy power domain %s\n", ent->genpd.name); + return ret; + } +err_free: + apple_pmp_report_entry_free(ent); return ret; } @@ -412,6 +498,7 @@ static const struct of_device_id apple_pmp_report_entry_of_match[] = { static struct platform_driver apple_pmp_report_entry_driver = { .probe = apple_pmp_report_entry_probe, + .remove = apple_pmp_report_entry_remove, .driver = { .name = "apple-pmp-report-entry", .of_match_table = apple_pmp_report_entry_of_match, From 820fac02c57e7f0cf693b07e33309ab3c1fa7ef2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 21:34:26 +1100 Subject: [PATCH 0704/1579] rtc: apple-pmu: distinguish alarm range errors from expiration The RTC core treats ETIME as an expired timer, retaining it on enqueue and retrying from irqwork without using its bounded error retry count. Returning it for a future alarm beyond the one-hour limit therefore keeps the worker under ops_lock until the alarm enters that horizon. Reserve ETIME for deadlines at or before the current counter. Return ERANGE for unsupported future deadlines so enqueue removes the timer and returns an error, and worker reprogramming uses its bounded error path. Advertise the same one-hour horizon through alarm_offset_max. Fixes: ff07618c82bb ("rtc: add opt-in J700 Baku PMU alarms with retained s2idle interrupts") Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit d3bf9897f2d5cbb6a9169d776b7d946fafb927c1) --- drivers/rtc/rtc-apple-pmu.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index 7853b72a1b552..a684233205803 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -126,8 +126,10 @@ static int baku_rtc_validate_alarm(struct baku_rtc *baku, u64 comparator) error = baku_rtc_seconds(baku, &counter); if (error) return error; - if (comparator <= counter || comparator - counter > BAKU_RTC_MAX_DELAY) + if (comparator <= counter) return -ETIME; + if (comparator - counter > BAKU_RTC_MAX_DELAY) + return -ERANGE; return 0; } @@ -314,6 +316,7 @@ static int baku_rtc_probe(struct platform_device *pdev) baku->clock->ops = &baku_rtc_ops; baku->clock->range_min = 0; baku->clock->range_max = RTC_TIMESTAMP_END_9999; + baku->clock->alarm_offset_max = BAKU_RTC_MAX_DELAY; clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, baku->clock->features); error = device_init_wakeup(dev, true); if (error) From a24ec6e1d7adfefb13ccd8db65e2c1fde3370dd7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:46:35 +1100 Subject: [PATCH 0705/1579] PCI: apple: run the J700 radio bootstrap by default The J700 Wi-Fi and Bluetooth functions sit behind port 0 of the T8140 PCIe host, which only opens downstream configuration access after the PIODMA bootstrap has completed. The bootstrap provider is already selected per board: it only binds on J700, and only when the device tree points the host's "apple,piodma" property at the engine. It nevertheless also required pcie_apple_piodma_diag.enumerate=1 on the kernel command line, so a kernel built with the provider still left both radio functions undiscovered unless the boot entry carried that option. Make the device tree the switch and run the bootstrap by default. Keep the parameter so enumerate=0 still selects the root-port-only path for diagnosis, and update the Kconfig help and documentation to match. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/PCI/controller/apple-t8140.rst | 11 +++++++---- .../networking/device_drivers/wifi/mt7932-neo.rst | 4 ++-- drivers/pci/controller/Kconfig | 6 ++++-- drivers/pci/controller/pcie-apple-piodma-diag.c | 10 +++++++--- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 24213c2cc3aa7..fd01165f6cfae 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -7,10 +7,13 @@ The J700 path trains port 0, validates a bounded PIODMA bootstrap request, and enables native ECAM for the MediaTek Wi-Fi and Bluetooth functions. The root bridge forwards their BARs and DART translates endpoint DMA. -This is an opt-in bring-up implementation, not general T8140 PCIe support. -Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y`` and pass -``pcie_apple_piodma_diag.enumerate=1`` to the kernel. Without the opt-in, the -root-only diagnostic path continues to reject downstream configuration access. +This is a bring-up implementation, not general T8140 PCIe support. +Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``. The board device tree selects the +bootstrap by pointing the host's ``apple,piodma`` property at the engine, and +the kernel then runs it during the host probe. Passing +``pcie_apple_piodma_diag.enumerate=0`` keeps the root-only diagnostic path, +which rejects downstream configuration access. A kernel built without the +option always uses that root-only path. Build the host and its DART in as well: the supplier retains published DMA and has no removal contract. Keeping the supplier and host built in avoids a module dependency on a provider whose lifetime cannot end at unload. diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index c8d641aadfda2..7cce2e3798d65 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -16,8 +16,8 @@ Platform and configuration -------------------------- Use the separately reviewed T8140 PCIe bootstrap described in -``Documentation/PCI/controller/apple-t8140.rst``. Its explicit kernel opt-in is -``pcie_apple_piodma_diag.enumerate=1``. Public m1n1 must populate the J700 +``Documentation/PCI/controller/apple-t8140.rst``. It runs during boot when +``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``. Public m1n1 must populate the J700 ``wifi0`` endpoint's own ``local-mac-address``; a zero placeholder is rejected. The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index a3f642fccdb69..82f0dd1769059 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -63,8 +63,10 @@ config PCIE_APPLE_PIODMA_DIAG depends on !KEXEC_CORE help Build the experimental SID17 retained bootstrap provider. It submits - one fixed request, validates typed ECAM and permits enumeration of - the two Neo functions only with the explicit enumerate=1 parameter. + one fixed request, validates typed ECAM and then enumerates the two + Neo radio functions when the board device tree links the PCIe host + to this engine. The enumerate=0 parameter keeps the host on its + root-port-only path instead. The host and supplier must both be built in: supplier removal cannot release a published DMA arena safely. Suspend-to-idle sleep retains that arena and restores the configuration without repeating a request. diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 5b1e27ad29e52..c0a85e04b6602 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -7,7 +7,7 @@ * to the reviewed Aurora reference and fresh ADT observations. * * This built-in driver deliberately retains its coherent arena and device - * power reference until a coordinator-controlled full hardware reset. There + * power reference until a full hardware reset. There * is no unbind interface, kexec or retry. Suspend-to-idle sleep preserves the * arena and restores the register banks without resubmitting its request. * Native ECAM is only @@ -51,9 +51,13 @@ #define PIODMA_FIFO_MASK GENMASK(5, 0) #define PIODMA_IRQ_LIMIT 16 -static bool enumerate; +/* + * The board DT opts in by pointing the host's "apple,piodma" at this engine. + * enumerate=0 keeps the host on its root-port-only path for diagnosis. + */ +static bool enumerate = true; module_param(enumerate, bool, 0400); -MODULE_PARM_DESC(enumerate, "Prime once, validate native ECAM, and enumerate the two Neo functions"); +MODULE_PARM_DESC(enumerate, "Prime once, validate native ECAM, and enumerate the two Neo functions (default: on)"); struct apple_piodma_request { u16 offset; From cc08268cac88c3860dff76ad84941cda9eca0c44 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:47:09 +1100 Subject: [PATCH 0706/1579] PCI: apple: let drivers bind the J700 radio functions After the J700 bootstrap, the host adds both radio functions to the bus and only then checks that neither is bus mastering and that neither has a driver. If either function has a driver by then, the host reports -EBUSY and closes downstream configuration access again, under a driver that has already bound and started using the function. That happens with a built-in driver, or with a module that udev loaded while the bus was being scanned. Check the cold state, with bus mastering off, before the functions are added and their drivers can probe. Treat drivers binding during publication as the normal outcome. Do not report the host as root-port-only once a bootstrap supplier has been found. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple.c | 65 ++++++++++++++++------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index c011d7be54059..3e416b53eb3f5 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2224,12 +2224,40 @@ static int apple_pcie_neo_check_memory(struct pci_dev *root) return 0; } +/* + * Both functions must come out of the bootstrap cold. Check that before they + * are published, because publication lets their drivers bind and take over. + */ +static int apple_pcie_neo_check_cold(struct apple_pcie *pcie, struct pci_dev *root) +{ + struct pci_dev *endpoint; + u16 command; + int function, ret = 0; + + for (function = 0; function < 2 && !ret; function++) { + endpoint = pci_get_slot(root->subordinate, PCI_DEVFN(0, function)); + if (!endpoint) + return -ENODEV; + if (pci_read_config_word(endpoint, PCI_COMMAND, &command)) { + ret = -EIO; + } else { + dev_info(pcie->dev, "enumerated %s id=%04x:%04x command=%#x\n", + pci_name(endpoint), endpoint->vendor, endpoint->device, command); + if (command & PCI_COMMAND_MASTER) { + pci_clear_master(endpoint); + ret = -EIO; + } + } + pci_dev_put(endpoint); + } + return ret; +} + static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) { struct apple_pcie *pcie = pci_host_bridge_priv(bridge); - struct pci_dev *root, *endpoint; - u16 command; - int function, ret; + struct pci_dev *root; + int ret; root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); if (!root) @@ -2260,37 +2288,16 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) dev_err(pcie->dev, "root memory forwarding unavailable after assignment: %d\n", ret); goto close_config; } + ret = apple_pcie_neo_check_cold(pcie, root); + if (ret) + goto close_config; pci_bus_add_devices(root->subordinate); - for (function = 0; function < 2; function++) { - endpoint = pci_get_slot(root->subordinate, PCI_DEVFN(0, function)); - if (!endpoint) { - ret = -ENODEV; - break; - } - if (pci_read_config_word(endpoint, PCI_COMMAND, &command)) { - ret = -EIO; - } else { - dev_info(pcie->dev, "enumerated %s id=%04x:%04x command=%#x driver=%s\n", - pci_name(endpoint), endpoint->vendor, endpoint->device, command, - endpoint->driver ? endpoint->driver->name : "unbound"); - if (command & PCI_COMMAND_MASTER) { - /* This experiment never permits endpoint bus mastering. */ - pci_clear_master(endpoint); - ret = -EIO; - } - if (endpoint->driver) - ret = -EBUSY; - } - pci_dev_put(endpoint); - if (ret) - break; - } close_config: if (ret) WRITE_ONCE(pcie->neo_config_ready, false); pci_unlock_rescan_remove(); if (!ret) - dev_info(pcie->dev, "NEO_ENUMERATION_ONLY_READY; functions0/1 unbound, bus mastering off\n"); + dev_info(pcie->dev, "radio functions 0/1 published\n"); out: pci_dev_put(root); return ret; @@ -2342,7 +2349,7 @@ static int apple_pcie_probe(struct platform_device *pdev) pcie->dev = dev; pcie->hw = hw; pcie->piodma_supplier = piodma_supplier; - if (hw->root_bus_only) + if (hw->root_bus_only && !piodma_supplier) dev_info(dev, "root-port probe only; downstream config is blocked\n"); pcie->base = devm_platform_ioremap_resource(pdev, 1); if (IS_ERR(pcie->base)) From bcb6eaa5c9ff120d459a3f6c00c382cf1a123bb8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:48:09 +1100 Subject: [PATCH 0707/1579] wifi: mt7932: disable ASPM on the J700 radio link On devicetree platforms the PCI core enables ASPM L0s/L1 by default. The station driver was qualified with the PCIe ASPM "performance" policy selected before it was loaded, so the documented admission sequence had userspace change that global policy first. If the module is autoloaded, nothing changes the policy. Disable all ASPM and clock power management states on the link before the driver starts the device, as other MediaTek PCIe Wi-Fi drivers do. This removes the userspace step and leaves the policy for other links alone. The function is probed before the root filesystem is mounted, and the driver reads its firmware while probing, so recommend building it as a module. Document that, and how cfg80211 finds the regulatory database. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../networking/device_drivers/wifi/mt7932-neo.rst | 15 +++++++++++---- drivers/net/wireless/mediatek/mt7932/Kconfig | 2 ++ drivers/net/wireless/mediatek/mt7932/mt7932.c | 2 ++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 7cce2e3798d65..160b8e6c49f80 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -28,10 +28,17 @@ sleep. The separately selected bootstrap provider has its own lifecycle restrictions. Kexec is excluded while either retained experiment is enabled. Use the ordinary cfg80211 regulatory database and applicable country policy. The validated first-admission fallback is kernel country 00 with firmware XZ. - -After the root filesystem and the local firmware packages are available, -select PCIe ASPM performance policy before loading ``mt7932-fullmac``. This -matches the tested admission sequence. Bluetooth's gate defaults closed; +A modular cfg80211 loads ``regulatory.db`` from the root filesystem; a +built-in one tries before the root filesystem is mounted and then needs +``iw reg reload``. + +The radio functions appear while the kernel boots, before the root +filesystem is mounted, and ``mt7932-fullmac`` reads its firmware while +probing. Build it as a module so that udev loads it from the root +filesystem, or provide its firmware in the initramfs. The driver disables +ASPM and clock power management on its link before starting the device, +matching the tested admission sequence, which used the PCIe ASPM +performance policy. Bluetooth's gate defaults closed; validate the cold, unbound ``14c3:793b`` function, then load ``mt7932_bt_pcie`` with ``enable=1`` after its local inputs are installed. A built-in transport can instead open its enable parameter before requesting diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index d435dbe24c32e..0747ed320c968 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -9,6 +9,8 @@ config MT7932_FULLMAC WPA2-CCMP station operation on 2.4 GHz and channels 36..48 at 20 MHz. Requires original firmware, own calibration and original-derived country policy packages in addition to the cfg80211 regulatory database. + The device is probed before the root filesystem is mounted, so say M + unless those files are in the initramfs. Requires a trained retained PCI link and a translated SID1 DMA domain. Firmware, runtime DMA and calibration negotiation start automatically; cfg80211 and userspace select the country and network. No test boot diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index b422d5acb5b9d..0549ca8575dc1 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1181,6 +1181,8 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) domain = iommu_get_domain_for_dev(&pdev->dev); if (!domain || (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) return dev_err_probe(&pdev->dev, -EINVAL, "translated DMA domain required\n"); + /* Station operation was qualified with ASPM and clock PM off on this link. */ + pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); ret = pcim_enable_device(pdev); if (ret) return ret; From 8beb2f16cc6a7df81e933acb390ada999e228e06 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:48:39 +1100 Subject: [PATCH 0708/1579] Bluetooth: mt7932: probe the J700 transport by default The transport only probes after userspace loads it with enable=1, so an autoloaded module leaves the Bluetooth function unbound. The gate was meant to keep the probe away until the root filesystem holds the unit's firmware and calibration. A module already loads from the root filesystem, and the probe validates a cold, unbound function before it starts. Open the gate by default and keep enable=0 to leave the function unbound. The documented admission sequence also selected the PCIe ASPM "performance" policy and loaded the Wi-Fi driver first. Disable ASPM and clock power management on the link in this probe as well, so the transport does not depend on the Wi-Fi driver for it. Add a module soft dependency so that modprobe loads mt7932_fullmac, which probes function 0, before the transport, keeping the tested firmware start order when udev loads both. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../networking/device_drivers/wifi/mt7932-neo.rst | 13 ++++++++----- drivers/bluetooth/Kconfig | 3 ++- drivers/bluetooth/mt7932_bt_pcie.c | 10 +++++++--- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 160b8e6c49f80..da200c49de793 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -38,11 +38,14 @@ probing. Build it as a module so that udev loads it from the root filesystem, or provide its firmware in the initramfs. The driver disables ASPM and clock power management on its link before starting the device, matching the tested admission sequence, which used the PCIe ASPM -performance policy. Bluetooth's gate defaults closed; -validate the cold, unbound ``14c3:793b`` function, then load -``mt7932_bt_pcie`` with ``enable=1`` after its local inputs are installed. -A built-in transport can instead open its enable parameter before requesting -the PCI probe. +performance policy. + +``mt7932_bt_pcie`` probes the cold ``14c3:793b`` function when it loads and +rejects a function that is not cold; ``enable=0`` leaves it unbound. Its +module soft dependency loads ``mt7932-fullmac`` first, so the Bluetooth +firmware starts after the Wi-Fi firmware, as in the tested sequence. The +transport also disables ASPM on the shared link. Like the Wi-Fi driver, it +reads its firmware while probing, so build it as a module. Do not reprobe after a failed or uncertain Bluetooth admission. The tested Wi-Fi driver owns function 0 and Bluetooth owns function 1. diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 3d350157a6123..94946efbb22e6 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -30,7 +30,8 @@ config BT_MTK7932_PCIE select FW_LOADER help Support the J700 Bluetooth PCIe function with locally installed firmware - and calibration. The late-probe enable parameter defaults off. + and calibration. The function is probed when the driver loads; the + enable=0 parameter leaves it unbound. System suspend uses the firmware sleep handshake and preserves DMA storage. Hibernation and automatic firmware recovery are unsupported. Removal drains the PCI function before releasing device-visible memory; diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index ab41a52ac656c..2078316ac5ff4 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -64,9 +64,9 @@ #define BT7932_CAL "mediatek/j700-mt7932-btcal.bin" #define BT7932_ADDR "mediatek/j700-mt7932-bdaddr.bin" -static bool enable; +static bool enable = true; module_param(enable, bool, 0600); -MODULE_PARM_DESC(enable, "Diagnostic late-probe gate; default closed, no effect on an already bound device"); +MODULE_PARM_DESC(enable, "Probe the transport (default: on); no effect on an already bound device"); struct bt7932_geometry { u16 stride; @@ -1041,13 +1041,15 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) u16 command; int ret; - /* The coordinator opens this only after the real root has its inputs. */ + /* enable=0 leaves the function unbound for diagnosis. */ if (!READ_ONCE(enable)) return -ENODEV; if (!of_machine_is_compatible("apple,j700") || !of_machine_is_compatible("apple,t8140") || pci_domain_nr(pdev->bus) || pdev->bus->number != 1 || pdev->devfn != PCI_DEVFN(0, 1)) return -ENODEV; + /* Admission was qualified with ASPM and clock PM off on this link. */ + pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); bt = kzalloc_obj(*bt); if (!bt) return -ENOMEM; @@ -1349,6 +1351,8 @@ module_pci_driver(bt7932_driver); MODULE_DESCRIPTION("Experimental standalone J700 MT7932 Bluetooth PCIe transport"); MODULE_LICENSE("GPL"); +/* Admission was qualified after the Wi-Fi function had started its firmware. */ +MODULE_SOFTDEP("pre: mt7932_fullmac"); MODULE_FIRMWARE(BT7932_FW_B0); MODULE_FIRMWARE(BT7932_FW_B1); MODULE_FIRMWARE(BT7932_FW_B1_FALLBACK); From 2b8d51be4f0ab98d229701bf4b5cb3afd0c9e75a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 23:48:39 +1100 Subject: [PATCH 0709/1579] drm: apple: locate legacy command replies in the saved record DCP firmware answers an AP command by writing the reply in place into the command record, then sending an ack whose offset and length fields are zero. This holds for H17P and, as traced on t8112 and t6000 with 13.5 firmware, for older firmware too. The legacy path instead located the reply through the ack: it read a packet header at offset zero of the ack context's area. That only finds the right record for an unnested command, which dcp_push() places at offset zero of the command area. A command sent while a callback is being handled goes after the callback in the command area, but its ack arrives on the callback context, whose offset zero holds the callback record. Since commit c529eadaf40a ("drm: apple: retain displaced framebuffers until confirmed completion") the lengths in that header are compared with the command's, so the first command the boot sequence sends from a callback fails the check. The DCP is then marked crashed and the internal panel is left without modes. Use the record address saved by dcp_push() on all firmware, as H17P already does. On older firmware, keep validating the reply before it is dispatched by requiring the record header to still match the one that was sent. Ignore an ack on an idle context on all firmware instead of underflowing the call depth. Fixes: c529eadaf40a ("drm: apple: retain displaced framebuffers until confirmed completion") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 4 +-- drivers/gpu/drm/apple/iomfb.c | 51 +++++++++++++++------------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 8e9e13e16488e..50ff687cfe82b 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -125,8 +125,8 @@ struct dcp_channel { dcp_callback_t callbacks[DCP_MAX_CALL_DEPTH]; void *cookies[DCP_MAX_CALL_DEPTH]; void *output[DCP_MAX_CALL_DEPTH]; - u32 in_len[DCP_MAX_CALL_DEPTH]; - u32 out_len[DCP_MAX_CALL_DEPTH]; + /* Header of each pending AP command record, as dcp_push() wrote it */ + struct dcp_packet_header header[DCP_MAX_CALL_DEPTH]; u16 end[DCP_MAX_CALL_DEPTH]; /* Current depth of the call stack. Less than DCP_MAX_CALL_DEPTH */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index de78df01c5edc..c273d4aae3689 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -603,8 +603,7 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca ch->callbacks[depth] = cb; ch->cookies[depth] = cookie; ch->output[depth] = out + sizeof(header) + in_len; - ch->in_len[depth] = in_len; - ch->out_len[depth] = out_len; + ch->header[depth] = header; ch->end[depth] = offset + ALIGN(data_len, DCP_PACKET_ALIGNMENT); dcp_send_message(dcp, IOMFB_ENDPOINT, @@ -817,12 +816,10 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, dcp_ack(dcp, context); } -static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, - void *data, u32 length) +static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context) { - struct dcp_packet_header *header = data; struct dcp_channel *ch = dcp_get_channel(dcp, context); - bool h17p = dcp->fw_compat == DCP_FIRMWARE_H17P; + const struct dcp_packet_header *record, *sent; void *cookie, *out; dcp_callback_t cb; @@ -831,7 +828,7 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, return; } - if (h17p && !ch->depth) { + if (!ch->depth) { dev_warn(dcp->dev, "ignoring ack on idle context %X\n", context); return; } @@ -840,27 +837,35 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, cb = ch->callbacks[ch->depth]; cookie = ch->cookies[ch->depth]; + out = ch->output[ch->depth]; + sent = &ch->header[ch->depth]; ch->callbacks[ch->depth] = NULL; ch->cookies[ch->depth] = NULL; + ch->output[ch->depth] = NULL; - if (h17p) { - /* - * H17P acks carry no payload and offset zero, including for - * nested calls, so the output cannot be located from the ack - * message. It is in the original AP command record, whose - * address dcp_push() saved. - */ - out = ch->output[ch->depth]; - ch->output[ch->depth] = NULL; - } else { - if (header->in_len != ch->in_len[ch->depth] || - header->out_len != ch->out_len[ch->depth]) { - dev_err(dcp->dev, "invalid command response lengths\n"); - dcp->crashed = true; + /* + * An ack completes the innermost pending command on its context. The + * firmware writes the reply in place into that AP command record, whose + * address dcp_push() saved; the ack itself carries neither offset nor + * length on any firmware. A command sent from within a callback is + * acked on the callback context, whose receive area holds the callback + * record rather than the reply, so the ack cannot be used to find it. + * + * Older firmware leaves the record header untouched. If it no longer + * matches what was sent, the record was overwritten and its output + * cannot be trusted. + */ + if (dcp->fw_compat != DCP_FIRMWARE_H17P) { + record = out - sent->in_len - sizeof(*record); + if (memcmp(record, sent, sizeof(*record))) { + dev_err(dcp->dev, + "corrupt %c%c%c%c command record on context %X\n", + sent->tag[3], sent->tag[2], sent->tag[1], + sent->tag[0], context); + WRITE_ONCE(dcp->crashed, true); return; } - out = data + sizeof(*header) + header->in_len; } if (cb) @@ -889,7 +894,7 @@ static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) data = dcp->shmem + channel_offset + offset; if (FIELD_GET(IOMFB_MSG_ACK, message)) - dcpep_handle_ack(dcp, ctx_id, data, length); + dcpep_handle_ack(dcp, ctx_id); else dcpep_handle_cb(dcp, ctx_id, data, length, offset); } From 3f0f25f4a2c800ab84651fe2daa7d418eb9576a7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:51:05 +1100 Subject: [PATCH 0710/1579] drm: apple: read the main display reply as a boolean The firmware answers is_main_display by writing a single byte into the 32-bit reply slot. Since commit c529eadaf40a ("drm: apple: retain displaced framebuffers until confirmed completion") dcp_push() fills reply slots with 0xff so that an unwritten status cannot pass for success, and the three bytes the firmware does not write stay set. An integrated panel's DCP now reads back -255, and an external display DCP, which answers 0, reads back -256 and is treated as the main display. Its hotplug callbacks are then ignored and power-on takes the integrated panel path. Read only the byte the firmware writes. Fixes: c529eadaf40a ("drm: apple: retain displaced framebuffers until confirmed completion") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 38eb01e814021..82bb6a720fc6c 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2293,10 +2293,15 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) { struct apple_connector *connector; - int result = *(int *)out; + /* + * The reply is a one-byte boolean. The firmware leaves the rest of + * the 32-bit slot alone, and dcp_push() fills that with 0xff. + */ + bool result = *(u8 *)out; + dev_info(dcp->dev, "DCP is_main_display: %d\n", result); - dcp->main_display = result != 0; + dcp->main_display = result; connector = dcp->connector; if (connector) { From 81cdd0987ba999a21c1eced2a0a7c19045c9c95c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:15:45 +1100 Subject: [PATCH 0711/1579] wifi: mt7932: depend on cfg80211 instead of selecting it MT7932_FULLMAC selects CFG80211. Built in, that forces CFG80211=y even when RFKILL=m, and Kconfig then warns about an unmet dependency, because CFG80211 depends on RFKILL || !RFKILL. Wireless drivers normally depend on CFG80211 rather than select it. Do the same, so that cfg80211's own constraints decide whether the driver can be built in. The J700 radio fragment relied on the select. Enable cfg80211 there as a module, which also lets it load regulatory.db from the root filesystem. Record the modular cfg80211 in the tested configuration. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/networking/device_drivers/wifi/mt7932-neo.rst | 2 +- arch/arm64/configs/j700-radio.config | 1 + drivers/net/wireless/mediatek/mt7932/Kconfig | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index da200c49de793..f8ea3dfcc9680 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -21,7 +21,7 @@ Use the separately reviewed T8140 PCIe bootstrap described in ``wifi0`` endpoint's own ``local-mac-address``; a zero placeholder is rejected. The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, -``CONFIG_CFG80211=y``, ``CONFIG_BT_MTK7932_PCIE=m``, ``CONFIG_BT_BREDR=y``, +``CONFIG_CFG80211=m``, ``CONFIG_BT_MTK7932_PCIE=m``, ``CONFIG_BT_BREDR=y``, ``CONFIG_BT_LE=y``, ``CONFIG_CRYPTO_AES=y`` and ``CONFIG_CRYPTO_CMAC=y``. The Bluetooth transport can be built with modular Bluetooth and system sleep. The separately selected bootstrap provider has its own lifecycle diff --git a/arch/arm64/configs/j700-radio.config b/arch/arm64/configs/j700-radio.config index 87b09a42d3cc6..f5c5d3c3ec0bf 100644 --- a/arch/arm64/configs/j700-radio.config +++ b/arch/arm64/configs/j700-radio.config @@ -8,6 +8,7 @@ CONFIG_PCI=y CONFIG_PCIE_APPLE=y CONFIG_APPLE_DART=y CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_CFG80211=m CONFIG_WLAN=y CONFIG_WLAN_VENDOR_MEDIATEK=y CONFIG_MT7932_FULLMAC=m diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index 0747ed320c968..fa18a3357d10c 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -2,8 +2,8 @@ config MT7932_FULLMAC tristate "Experimental J700 MT7932 fullmac wireless support" depends on PCI && OF && ARM64 && IOMMU_DMA + depends on CFG80211 select FW_LOADER - select CFG80211 help Native support for the built-in J700 Wi-Fi endpoint: managed WPA2-CCMP station operation on 2.4 GHz and channels 36..48 at 20 MHz. From 51e42c5366b63a6a46b18d8bc6ee0ba04eb6bd8f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:20:12 +1100 Subject: [PATCH 0712/1579] drm: apple: only signal crash events on a linked H17P pipeline The H17P crash handler schedules the delayed vblank work, so that a present in flight at the time of the crash completes its page-flip event. That work dereferences dcp->crtc, which is set only when the display subsystem links the pipeline. RTKit is already running before that: component binding wakes the coprocessor before the DRM device creates its CRTCs, and a pipeline whose CRTC setup failed stays bound with no CRTC at all. A firmware crash in either state then oopses the workqueue on a NULL CRTC. Schedule the work from the crash handler only once a CRTC is linked, and make the delayed vblank work itself return early without one, which also covers its other callers. Publish the CRTC with WRITE_ONCE() to match the lockless readers. Also limit the crash-time completion to the measured H17P profile, as every other H17P change in this series is. The H17G profile (T8142) keeps its existing crash handling. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 06d2d31feb051..d2abcc392016b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1843,9 +1843,15 @@ static void dcp_delayed_vblank(struct work_struct *work) { struct apple_dcp *dcp; + struct apple_crtc *crtc; + dcp = container_of(work, struct apple_dcp, vblank_wq); + /* RTKit can report a crash before the pipeline has a CRTC. */ + crtc = READ_ONCE(dcp->crtc); + if (!crtc) + return; mdelay(5); - dcp_drm_crtc_vblank(dcp->crtc); + dcp_drm_crtc_vblank(crtc); } static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) @@ -1968,11 +1974,16 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ dcp->crashed = true; dev_err(dcp->dev, "DCP has crashed\n"); - if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { /* No in-place restart exists; the last scanout stays retained. */ dev_err(dcp->dev, "a reboot is required to restore the display\n"); - /* Nothing will complete a present that was in flight. */ - schedule_work(&dcp->vblank_wq); + /* + * Nothing will complete a present that was in flight. Before + * the pipeline is linked there is no event to complete. + */ + if (READ_ONCE(dcp->crtc)) + schedule_work(&dcp->vblank_wq); } if (dcp->connector) { dcp->connector->connected = 0; @@ -2465,7 +2476,7 @@ void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, { struct apple_dcp *dcp = platform_get_drvdata(pdev); - dcp->crtc = crtc; + WRITE_ONCE(dcp->crtc, crtc); /* * Type-C connectors belong to physical ports and are bound by the From 77291fb31e7bf782724c1196ebfe3033698596bf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:20:22 +1100 Subject: [PATCH 0713/1579] drm: apple: keep the H17P backlight policy when registration fails When the backlight class device cannot be registered, the worker zeroes the panel maximum so that older firmware stops encoding brightness. On H17P the brightness policy is already active by then, and the present path needs that maximum for the ceiling it writes into every present: with it zeroed, every present is refused and the panel freezes on its last frame. Registration is attempted on every H17P boot now, so this error path is always reachable. Keep the maximum while the H17P policy is active. Presents keep their ceiling and the policy keeps applying DPMS and the firmware's requests; only the sysfs control is missing. Log the error code in either case. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d2abcc392016b..f2e3475d2ee01 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2839,8 +2839,14 @@ static void dcp_work_register_backlight(struct work_struct *work) if (ret == -ENODATA) goto out_unlock; if (ret) { - dev_err(dcp->dev, "Unable to register backlight device\n"); - dcp->brightness.maximum = 0; + dev_err(dcp->dev, "Unable to register backlight device: %d\n", ret); + /* + * The H17P policy encodes the panel ceiling into every present + * and keeps working without a class device; only older + * firmware stops sending brightness here. + */ + if (!dcp_backlight_active(dcp)) + dcp->brightness.maximum = 0; } out_unlock: From 73dd5dedcc77218926a77bdc77cc665f84f00e33 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:21:34 +1100 Subject: [PATCH 0714/1579] drm: apple: never re-send a guessed H17P backlight level Without a loader brightness report, the H17P backlight is registered with a mid-range default that is only reported to userspace; the panel keeps the loader's level. Registration also marks the policy as controlled, which is the only condition the firmware's re-send request checked. A re-send request before userspace writes a level therefore presented the default and moved the panel from the loader's level to mid-range, contrary to the intent of the re-send and fallback registration changes. Track whether the level the panel shows is known: the loader's reported level, or one a completed present carried. Require it for a re-send, so a re-send request now leaves a panel showing an unknown level alone. A rejected present does not make a level known. A DPMS blank and wake, or a suspend and resume, after a fallback registration still has to light the panel at some level and uses the reported default. The panel keeps the loader's level only until the first such transition or userspace write. Also reject powerlog reports above twice the panel ceiling instead of clamping them, so a corrupt sample cannot become a known level. Add a KUnit case for the fallback registration, the re-send request, rejected and completed presents, and the DPMS wake level. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.h | 29 ++++++++++---- drivers/gpu/drm/apple/dcp_backlight_test.c | 46 +++++++++++++++++++++- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 75242111bd1de..73683cb0322a3 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -24,6 +24,12 @@ struct dcp_backlight_state { bool suspended; bool dirty; bool in_flight; + /* + * The panel shows a level Linux knows: the loader's reported level, or + * one a completed present carried. A default used for registration + * without a loader report is not known until a present carries a level. + */ + bool level_known; }; struct dcp_backlight_present { @@ -32,18 +38,24 @@ struct dcp_backlight_present { }; /* - * Public powerlog reports use millinits. A report above the panel ceiling - * describes the brightest level the panel can show, so clamp it. + * Public powerlog reports use millinits. A report slightly above the panel + * ceiling describes the brightest level the panel can show, so clamp it; one + * more than twice the ceiling is not a credible panel level. */ static inline int dcp_bl_takeover_nits(u32 maximum, u32 millinits, u32 *nits) { - if (!maximum || maximum > INT_MAX) + if (!maximum || maximum > INT_MAX / 2000) return -EINVAL; + if (millinits / 1000 > 2 * maximum) + return -ERANGE; *nits = min(millinits / 1000, maximum); return 0; } -/* Inputs must come from admitted panel/loader records, never a guess. */ +/* + * An inherited level comes from a loader report. A default level is only + * reported to userspace; it is never presented on its own (see level_known). + */ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, bool inherited_valid, u32 inherited, bool default_valid, u32 default_nits) @@ -63,6 +75,7 @@ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, .target = nits, .actual = nits, .ready = true, + .level_known = inherited_valid, }; return 0; } @@ -118,12 +131,13 @@ static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) /* * The firmware asked for the level again. Re-send the effective level once - * Linux controls it, unless a present already carries or will carry it. + * Linux controls it and knows it, unless a present already carries or will + * carry it. A default level is never sent this way. */ static inline bool dcp_bl_resend(struct dcp_backlight_state *state) { - if (!state->ready || !state->controlled || state->dirty || - state->in_flight) + if (!state->ready || !state->controlled || !state->level_known || + state->dirty || state->in_flight) return false; state->dirty = true; return true; @@ -168,6 +182,7 @@ static inline bool dcp_bl_complete(struct dcp_backlight_state *state, if (accepted) { state->actual = state->sent_nits; state->retries = 0; + state->level_known = true; } else { state->dirty = true; if (state->retries < 4) diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index 12e039d5ce0a1..0b94b41b41482 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -19,8 +19,13 @@ static void backlight_millinits_takeover_test(struct kunit *test) KUNIT_EXPECT_EQ(test, nits, 525U); KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 509001, &nits), 0); KUNIT_EXPECT_EQ(test, nits, 509U); - KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 1018999, &nits), 0); KUNIT_EXPECT_EQ(test, nits, 509U); + /* More than twice the ceiling is not a panel level. */ + nits = 123; + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, 1019000, &nits), -ERANGE); + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), -ERANGE); + KUNIT_EXPECT_EQ(test, nits, 123U); KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 100999, &nits), 0); KUNIT_EXPECT_EQ(test, nits, 100U); KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 509, true, nits, false, 0), 0); @@ -249,6 +254,44 @@ static void backlight_resend_test(struct kunit *test) KUNIT_EXPECT_EQ(test, state.target, 140U); } +static void backlight_fallback_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* A registration default is reported, never presented on its own. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_EXPECT_FALSE(test, state.level_known); + state.controlled = true; + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &present), -EALREADY); + + /* A rejected userspace level does not become known. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 300, false, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 300U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, false)); + KUNIT_EXPECT_FALSE(test, state.level_known); + + /* Once a present carrying it completes, the level is re-sendable. */ + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_TRUE(test, state.level_known); + KUNIT_EXPECT_TRUE(test, dcp_bl_resend(&state)); + + /* DPMS has to light the panel at some level: wake uses the default. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + state.controlled = true; + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 262U); +} + static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), @@ -260,6 +303,7 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_reject_test), KUNIT_CASE(backlight_inflight_dpms_test), KUNIT_CASE(backlight_resend_test), + KUNIT_CASE(backlight_fallback_test), {} }; From 7399a69c8c812ec7ac320ca946adbde4e91ed9fb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:22:15 +1100 Subject: [PATCH 0715/1579] drm: apple: tolerate late H17P completions of aborted presents The H17P abort callback is handled as "this present never completes", but no capture confirms that. If the firmware still completes an aborted present, the completion matches no present in flight and the driver declares the coprocessor crashed, turning the first real abort into a dead display. Remember the most recently aborted present. A completion for it is logged and otherwise ignored: its displaced framebuffers were already unarmed and its DRM event signalled when it was aborted, and it does not disturb a later present in flight. Also settle the swap watchdog when an accepted present is aborted. Otherwise a Type-C output would report a missing completion and retrain one second later. Rate-limit the abort messages, so that a firmware that aborts every frame cannot flood the log. Extend the KUnit present-state case with stale completions. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.h | 13 +++++++++++++ drivers/gpu/drm/apple/iomfb_h17p_test.c | 9 +++++++++ drivers/gpu/drm/apple/iomfb_template.c | 24 ++++++++++++++++++++---- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 94ac90be4247a..75856d1921562 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -23,6 +23,9 @@ struct dcp_present_state_h17p { bool accepted; /* The firmware aborted the present before its submit reply. */ bool aborted; + /* The most recent aborted present, whose late completion is ignored. */ + bool has_last_aborted; + u32 last_aborted; }; static inline bool @@ -64,9 +67,19 @@ dcp_present_abort_h17p(struct dcp_present_state_h17p *state, u32 swap_id) } else { state->aborted = true; } + state->has_last_aborted = true; + state->last_aborted = swap_id; return true; } +/* Whether the firmware aborted @swap_id; a completion for it is stale. */ +static inline bool +dcp_present_was_aborted_h17p(const struct dcp_present_state_h17p *state, + u32 swap_id) +{ + return state->has_last_aborted && state->last_aborted == swap_id; +} + static inline bool dcp_present_complete_h17p(struct dcp_present_state_h17p *state, u32 swap_id) { diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c index 70fecfe52c0ff..c017651945a4e 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p_test.c +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -104,9 +104,13 @@ static void h17p_present_abort_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 7, true)); KUNIT_EXPECT_FALSE(test, dcp_present_abort_h17p(&state, 6)); KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&state, 7)); KUNIT_EXPECT_TRUE(test, dcp_present_abort_h17p(&state, 7)); KUNIT_EXPECT_FALSE(test, state.pending); KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 7)); + /* A late completion for it is recognisable as stale, not unexpected. */ + KUNIT_EXPECT_TRUE(test, dcp_present_was_aborted_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&state, 6)); KUNIT_EXPECT_FALSE(test, dcp_present_abort_h17p(&state, 7)); /* An abort before the submit reply leaves the reply to finish it. */ @@ -123,8 +127,13 @@ static void h17p_present_abort_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, state.aborted); KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 9, true)); KUNIT_EXPECT_TRUE(test, state.pending); + /* A stale completion for 8 neither matches nor disturbs present 9. */ + KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_aborted_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, state.pending); KUNIT_EXPECT_TRUE(test, dcp_present_complete_h17p(&state, 9)); KUNIT_EXPECT_FALSE(test, state.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&state, 9)); } static struct kunit_case h17p_present_cases[] = { diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index d6f754c0bb712..e03069b52a2fd 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -252,6 +252,18 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !dcp_present_complete_h17p(&dcp->present_state_h17p, resp->swap_id)) { + /* + * Abort semantics are not captured on H17P. If an aborted + * present completes after all, its framebuffers were already + * unarmed and its event signalled; record it and carry on. + */ + if (dcp_present_was_aborted_h17p(&dcp->present_state_h17p, + resp->swap_id)) { + dev_warn_ratelimited(dcp->dev, + "completion for aborted present %u ignored\n", + resp->swap_id); + return; + } dev_err(dcp->dev, "unexpected present completion %u\n", resp->swap_id); dcp->crashed = true; return; @@ -1398,7 +1410,8 @@ static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted /* An abort received before this reply turns acceptance into failure. */ if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && accepted && dcp->present_state_h17p.aborted) { - dev_warn(dcp->dev, "firmware aborted present %u\n", swap_id); + dev_warn_ratelimited(dcp->dev, "firmware aborted present %u\n", + swap_id); return false; } #endif @@ -1866,15 +1879,18 @@ dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, static void dcp_present_aborted(struct apple_dcp *dcp, u32 swap_id) { if (!dcp_present_abort_h17p(&dcp->present_state_h17p, swap_id)) { - dev_warn(dcp->dev, "abort for present %u, which is not in flight\n", - swap_id); + dev_warn_ratelimited(dcp->dev, + "abort for present %u, which is not in flight\n", + swap_id); return; } /* The submit reply has not arrived; it finishes the present. */ if (dcp->present_state_h17p.pending) return; - dev_warn(dcp->dev, "firmware aborted present %u\n", swap_id); + dev_warn_ratelimited(dcp->dev, "firmware aborted present %u\n", swap_id); + /* No completion follows, so the swap watchdog must not wait for one. */ + dcp_swap_watchdog_complete(dcp); dcp_present_failed(dcp); if (dcp_present_retires(dcp)) { dcp_unarm_retained_framebuffers(dcp, swap_id); From 6d4dc0a94dc95615e268b1bcc29bf1e1e0719d87 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:22:39 +1100 Subject: [PATCH 0716/1579] drm: apple: settle H17P brightness retries before system sleep The sleep drain waited for the IOMFB queue to go idle, but a rejected blanking present leaves the queue idle with a delayed retry scheduled. The drain then returned, and the retry ran after the coprocessor's mailbox had suspended, leaving the panel lit through sleep. Run a pending brightness retry immediately in each drain pass, and keep draining while the brightness policy still has a level to send. Four passes cover the first attempt and the policy's three retries, each bounded as before; a level that is still pending after them is logged. Also queue the fallback backlight registration on the freezable system workqueue, so that it cannot register a class device in the middle of a suspend that starts within two seconds of the first pipe enable. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f2e3475d2ee01..c6924e2d32982 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2707,8 +2707,9 @@ void dcp_poweron(struct platform_device *pdev) } /* The powerlog takeover report normally precedes this. */ if (dcp_has_panel(dcp)) - schedule_delayed_work(&dcp->bl_fallback_wq, - DCP_BL_FALLBACK_DELAY); + queue_delayed_work(system_freezable_wq, + &dcp->bl_fallback_wq, + DCP_BL_FALLBACK_DELAY); } ret = dcp_backlight_dpms(dcp, true); if (ret) @@ -3836,15 +3837,24 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) return; - /* A completed present can schedule one more brightness present. */ - for (pass = 0; pass < 2; pass++) { + /* + * A completed present can schedule one more brightness present, and a + * rejected one a delayed retry; run a pending retry now instead of + * letting it fire after the mailbox has suspended. Four passes cover + * the first attempt and the policy's three retries. + */ + for (pass = 0; pass < 4; pass++) { + flush_delayed_work(&dcp->iomfb.backlight_retry); if (dcp_has_panel(dcp)) flush_work(&dcp->bl_update_wq); if (!iomfb_queue_drain(dcp, msecs_to_jiffies(500))) { dev_warn(dcp->dev, "display updates still pending at suspend\n"); return; } + if (!dcp_backlight_active(dcp) || !dcp_backlight_pending(dcp)) + return; } + dev_warn(dcp->dev, "brightness change still pending at suspend\n"); } static int dcp_platform_suspend(struct device *dev) From 09c4dc6048b76a881e7869b2e62bbf109730a676 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:23:14 +1100 Subject: [PATCH 0717/1579] drm: apple: report H17P runtime callback sizes that exceed their bounds The runtime callbacks accepted within reply bounds are validated against what their handlers read and write, because no capture of their record lengths exists. A longer record is accepted silently, so a layout that differs from the handler's assumption, for instance a status word at another offset, would go unnoticed. The first time each of these callbacks arrives with lengths other than its handler's, log the lengths seen next to the handler's. Each tag is reported at most once per boot, and the measured startup callbacks are unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/iomfb.c | 2 +- drivers/gpu/drm/apple/iomfb.h | 2 ++ drivers/gpu/drm/apple/iomfb_h17p.c | 22 ++++++++++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index aa0a4096e23cd..b78fe1431a2c5 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -420,6 +420,8 @@ struct apple_dcp { struct dcp_present_h17p present_h17p; struct dcp_present_state_h17p present_state_h17p; DECLARE_BITMAP(unknown_callbacks, IOMFB_MAX_CB); + /* Runtime callbacks whose record size has been reported once. */ + DECLARE_BITMAP(sized_callbacks, IOMFB_MAX_CB); /* Serializes H17P transmit preparation with RTKit receive callbacks. */ struct { diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 0a8b5aaea5f11..803affbc65e26 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -798,7 +798,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, (u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && handled && - !iomfb_validate_callback_h17p(tag, hdr->in_len, hdr->out_len)))) { + !iomfb_check_callback_h17p(dcp, tag, hdr->in_len, hdr->out_len)))) { dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", tag, hdr->in_len, hdr->out_len); dcp->crashed = true; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index d17704012b1c6..efed16220d947 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -209,6 +209,8 @@ struct dcp_method_entry { bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len); struct apple_dcp; +bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, + u32 out_len); typedef bool (*iomfb_cb_handler)(struct apple_dcp *, int, void *, void *); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index cf708655b3b34..682a92879f8ef 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -694,6 +694,28 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [599] = trampoline_nop, /* find_swap_function_gated */ }; +/* + * Validate a callback record and, the first time each runtime callback whose + * length is only bounded arrives with a different length, report the length + * actually seen. Those reports are the first capture of these records. + */ +bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, + u32 out_len) +{ + const struct dcp_callback_size *size; + + if (!iomfb_validate_callback_h17p(tag, in_len, out_len)) + return false; + size = &callback_sizes[tag]; + if (size->minimum && + (in_len != size->in_len || out_len != size->out_len) && + !test_and_set_bit(tag, dcp->sized_callbacks)) + dev_info(dcp->dev, + "callback D%03d record is %#x/%#x bytes; its handler uses %#x/%#x\n", + tag, in_len, out_len, size->in_len, size->out_len); + return true; +} + void iomfb_start_h17p(struct apple_dcp *dcp) { WRITE_ONCE(dcp->pipe_enabled_h17p, false); From a371243eff0d61e6601188a392cc18c29399b158 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:35:10 +1100 Subject: [PATCH 0718/1579] wifi: mt7932: support the 5 GHz channels above channel 48 The station only advertises 5 GHz channels 36 to 48, so access points on UNII-2, UNII-2e or UNII-3 channels are invisible. On the MacBook Neo used for testing, the network's 5 GHz radio is on channel 149, and the station falls back to its 2.4 GHz radio. Advertise every 20 MHz primary from 36 to 64, 100 to 144 and 149 to 165, and let the cfg80211 regulatory domain decide which are usable. Each gets its 40 and 80 MHz pairing (165 stays at 20 MHz). The same pairing is used to decode association channel events and to pick the unit's own calibration records, which cover the whole band. Radar channels were previously rejected by refusing the whole regulatory domain. Pass them to the firmware with cfg80211's flags instead, and keep the station passive there. cfg80211 marks them RADAR, not NO_IR, so scans listen on them without probing. Scans made while associated skip them, and connects never target them: DFS client operation is not qualified, and a passive dwell only takes time away from the home channel. A channel whose limits the driver cannot apply, such as a power cap below the qualified 20 dBm, is now disabled on its own rather than blocking every channel. The firmware scan request has 32 channel slots, which holds the larger 5 GHz list in one batch. A scan request left with no eligible channel now completes at once. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 7 ++ drivers/net/wireless/mediatek/mt7932/Kconfig | 4 +- .../net/wireless/mediatek/mt7932/bandwidth.h | 9 +-- .../wireless/mediatek/mt7932/calibration.h | 7 +- .../net/wireless/mediatek/mt7932/channels.h | 66 ++++++++++++++++++- .../net/wireless/mediatek/mt7932/frontend.c | 23 +++++-- drivers/net/wireless/mediatek/mt7932/link.c | 2 +- drivers/net/wireless/mediatek/mt7932/mt7932.h | 4 +- .../net/wireless/mediatek/mt7932/regulatory.h | 6 +- drivers/net/wireless/mediatek/mt7932/scan.h | 5 +- .../net/wireless/mediatek/mt7932/startup.c | 43 ++++++++---- 11 files changed, 142 insertions(+), 34 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index f8ea3dfcc9680..b7675f57e8350 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -26,6 +26,13 @@ The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, The Bluetooth transport can be built with modular Bluetooth and system sleep. The separately selected bootstrap provider has its own lifecycle restrictions. Kexec is excluded while either retained experiment is enabled. +The station uses 2.4 GHz channels 1-13 and the 5 GHz channels 36-64, +100-144 and 149-165 that the cfg80211 regulatory domain permits. Radar (DFS) +channels stay passive: the firmware listens on them while the station is not +associated, but never probes or joins an access point there, and scans made +while associated skip them. A channel whose restrictions the driver cannot +apply (reduced power, no OFDM, PSD limits) is disabled rather than used. + Use the ordinary cfg80211 regulatory database and applicable country policy. The validated first-admission fallback is kernel country 00 with firmware XZ. A modular cfg80211 loads ``regulatory.db`` from the root filesystem; a diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index fa18a3357d10c..32e6608851326 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -6,7 +6,9 @@ config MT7932_FULLMAC select FW_LOADER help Native support for the built-in J700 Wi-Fi endpoint: managed - WPA2-CCMP station operation on 2.4 GHz and channels 36..48 at 20 MHz. + WPA2-CCMP station operation on 2.4 GHz and on the 5 GHz UNII-1, -2, + -2e and -3 channels that the regulatory domain permits. Radar (DFS) + channels are scanned passively but not joined. Requires original firmware, own calibration and original-derived country policy packages in addition to the cfg80211 regulatory database. The device is probed before the root filesystem is mounted, so say M diff --git a/drivers/net/wireless/mediatek/mt7932/bandwidth.h b/drivers/net/wireless/mediatek/mt7932/bandwidth.h index c5efe6599e0a4..07c281e0fc589 100644 --- a/drivers/net/wireless/mediatek/mt7932/bandwidth.h +++ b/drivers/net/wireless/mediatek/mt7932/bandwidth.h @@ -13,13 +13,14 @@ static inline u8 mt7932_domain_5g_bw(void) } /* EE4d width 0 combines 20/40; primary and extension select the center. - * Keep all operational primaries inside 36..48, and 2 GHz at 20 MHz. + * Extension 1 places the secondary above the primary, 3 below it. Keep 2 GHz + * at 20 MHz, and 165, which has no 40 MHz pair, at 20 MHz too. */ static inline bool mt7932_channel_layout(u8 primary, u8 width, u8 extension, u8 maximum, u8 *center, u8 *bw) { u8 band = mt7932_channel_band(primary); - u8 expected = primary == 36 || primary == 44 ? 1 : 3; + u8 pair = mt7932_channel_ht40_centre(primary); if (!band || maximum > 2 || width > 1) return false; @@ -28,12 +29,12 @@ static inline bool mt7932_channel_layout(u8 primary, u8 width, u8 extension, *bw = 0; return true; } - if (band != 2 || extension != expected) + if (band != 2 || !pair || extension != (pair > primary ? 1 : 3)) return false; *bw = width ? 2 : 1; if (*bw > maximum) return false; - *center = width ? 42 : extension == 1 ? primary + 2 : primary - 2; + *center = width ? mt7932_channel_vht80_centre(primary) : pair; return true; } diff --git a/drivers/net/wireless/mediatek/mt7932/calibration.h b/drivers/net/wireless/mediatek/mt7932/calibration.h index f008e34146d64..afb4cdc57ed5d 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration.h +++ b/drivers/net/wireless/mediatek/mt7932/calibration.h @@ -2,6 +2,7 @@ #ifndef MT7932_CALIBRATION_H #define MT7932_CALIBRATION_H +#include "channels.h" #include "protocol.h" /* Protected by the RX/procedure lock. One active logical procedure only; @@ -146,7 +147,7 @@ static inline int mt7932_cal_association(const u8 *data, size_t size, u8 channel if (channel >= 1 && channel <= 13) group = 0; - else if (channel >= 36 && channel <= 48 && !(channel % 2)) + else if (mt7932_channel_5g_cal(channel)) group = 1; else return -EOPNOTSUPP; @@ -228,7 +229,7 @@ static inline int mt7932_cal_requested_5g(const u8 *data, size_t size, int ret; if (upper != 0x1000 || version != 12 || module != 0x89 || - channel < 36 || channel > 48 || channel % 2) + !mt7932_channel_5g_cal(channel)) return -EOPNOTSUPP; ret = mt7932_cal_validate(data, size); if (ret) @@ -243,7 +244,7 @@ static inline int mt7932_cal_requested_5g(const u8 *data, size_t size, if (segment.data[i]) return -EOPNOTSUPP; /* Own row610f07 does not remap center selectors1/2; only primaries. */ - selected = channel % 4 ? channel : channel <= 40 ? 38 : 46; + selected = mt7932_channel_band(channel) ? mt7932_channel_cal_centre(channel) : channel; params[0] = 0x10000001; params[1] = 0x10001000 | channel; params[2] = 0x10001000 | (channel << 17) | selected; diff --git a/drivers/net/wireless/mediatek/mt7932/channels.h b/drivers/net/wireless/mediatek/mt7932/channels.h index 8c4b61e4a3ec7..bae04c245d72a 100644 --- a/drivers/net/wireless/mediatek/mt7932/channels.h +++ b/drivers/net/wireless/mediatek/mt7932/channels.h @@ -4,14 +4,78 @@ /* Bounded operational whitelist, not the full calibration channel namespace. * Firmware RF bands1/2 are distinct from single-interface MAC index0. + * 5 GHz covers the 20 MHz primaries of UNII-1/2 (36..64), UNII-2e (100..144) + * and UNII-3 (149..165); regulatory flags decide which of them are usable. */ +#define MT7932_CHANNELS_2G 13 +#define MT7932_CHANNELS_5G 25 +#define MT7932_CHANNELS (MT7932_CHANNELS_2G + MT7932_CHANNELS_5G) + static inline unsigned int mt7932_channel_band(unsigned int channel) { if (channel >= 1 && channel <= 13) return 1; - if (channel >= 36 && channel <= 48 && !(channel % 4)) + if (((channel >= 36 && channel <= 64) || (channel >= 100 && channel <= 144)) && + !(channel % 4)) + return 2; + if (channel >= 149 && channel <= 165 && !((channel - 149) % 4)) return 2; return 0; } +/* The nth 5 GHz primary, in the order the wiphy advertises them. */ +static inline unsigned int mt7932_channel_5g(unsigned int index) +{ + if (index < 8) + return 36 + 4 * index; + if (index < 20) + return 100 + 4 * (index - 8); + return 149 + 4 * (index - 20); +} + +/* Offset of a 5 GHz primary within its 80 MHz block numbering. */ +static inline unsigned int mt7932_channel_slot(unsigned int primary, unsigned int *base) +{ + *base = primary >= 149 ? 149 : primary >= 100 ? 100 : 36; + return (primary - *base) / 4; +} + +/* 40 MHz pair centre of a 5 GHz primary; 0 where no pair exists (165). */ +static inline unsigned int mt7932_channel_ht40_centre(unsigned int primary) +{ + unsigned int base, slot; + + if (mt7932_channel_band(primary) != 2 || primary == 165) + return 0; + slot = mt7932_channel_slot(primary, &base); + return base + 4 * (slot & ~1U) + 2; +} + +/* 80 MHz block centre of a 5 GHz primary; 0 where no block exists (165). */ +static inline unsigned int mt7932_channel_vht80_centre(unsigned int primary) +{ + unsigned int base, slot; + + if (mt7932_channel_band(primary) != 2 || primary == 165) + return 0; + slot = mt7932_channel_slot(primary, &base); + return base + 4 * (slot & ~3U) + 6; +} + +/* A 5 GHz primary or a 40/80 MHz centre that calibration can be asked for. */ +static inline bool mt7932_channel_5g_cal(unsigned int channel) +{ + if ((channel >= 36 && channel <= 64) || (channel >= 100 && channel <= 144)) + return !(channel % 2); + return channel >= 149 && channel <= 167 && (channel % 2); +} + +/* Calibration centre for a 5 GHz primary. 165 has no 40 MHz pair; its own + * calibration package carries the record for the 165/169 centre, 167. + */ +static inline unsigned int mt7932_channel_cal_centre(unsigned int primary) +{ + return primary == 165 ? 167 : mt7932_channel_ht40_centre(primary); +} + #endif diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 464dca910a978..0bb2761cf934c 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -145,7 +145,7 @@ static int mt_scan_submit_next(struct mt7932 *m) { struct cfg80211_scan_request *request = m->scan_request; const struct cfg80211_ssid *ssid = NULL; - u8 body[0x4d4], channels[13]; + u8 body[0x4d4], channels[MT7932_CHANNELS_5G]; unsigned int count = 0, i; unsigned long flags; bool passive = false; @@ -159,11 +159,15 @@ static int mt_scan_submit_next(struct mt7932 *m) passive = !(m->scan_batch & 1); for (i = 0; i < request->n_channels; i++) { struct ieee80211_channel *channel = request->channels[i]; - bool listen = !request->n_ssids || + bool radar = channel->flags & IEEE80211_CHAN_RADAR; + bool listen = !request->n_ssids || radar || (channel->flags & IEEE80211_CHAN_NO_IR); + /* Radar channels are never joined; while associated, + * a passive dwell there only delays the home channel. + */ if (mt7932_channel_band(channel->hw_value) != m->scan_band || - listen != passive) + listen != passive || (radar && m->scan_home_channel)) continue; if (channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_20MHZ) || count == ARRAY_SIZE(channels)) @@ -215,7 +219,7 @@ static int mt_scan_submit_next(struct mt7932 *m) */ int mt_connect_discover(struct mt7932 *m) { - u8 body[0x4d4], channels[13], cancel[4] = {}; + u8 body[0x4d4], channels[MT7932_CHANNELS_5G], cancel[4] = {}; unsigned long flags; unsigned int attempt, band, i, count; int ret = 0; @@ -236,7 +240,8 @@ int mt_connect_discover(struct mt7932 *m) if (m->connect_channel_req && channel != m->connect_channel_req) continue; - if (channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_20MHZ)) + if (channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_20MHZ | + IEEE80211_CHAN_RADAR)) continue; passive |= !!(channel->flags & IEEE80211_CHAN_NO_IR); channels[count++] = channel->hw_value; @@ -328,7 +333,10 @@ int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) dev_info(&m->pdev->dev, "SCAN_REQUEST: channels=%u ssids=%d ies=%zu flags=%08x duration=%u/%u\n", request->n_channels, request->n_ssids, request->ie_len, request->flags, request->duration, request->duration_mandatory); - if (!request->n_channels || request->n_channels > 17 || request->ie_len > 600 || + /* cfg80211 lists each advertised channel at most once. */ + if (request->n_channels > MT7932_CHANNELS) + return -EOPNOTSUPP; + if (!request->n_channels || request->ie_len > 600 || request->duration_mandatory || (request->flags & ~(NL80211_SCAN_FLAG_FLUSH | NL80211_SCAN_FLAG_COLOCATED_6GHZ)) || request->n_ssids > 1 || (request->n_ssids && request->ssids[0].ssid_len > 32)) @@ -362,6 +370,9 @@ int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) spin_unlock_irqrestore(&m->response_lock, flags); cancel_delayed_work(&m->scan_timeout_work); } else { + /* Nothing eligible to submit: complete the request. */ + if (ret) + schedule_work(&m->scan_finish_work); ret = 0; } out: diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 8caab22dbbb34..fc2d333519f98 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -343,7 +343,7 @@ void mt_connect_work(struct work_struct *work) goto free; } if (m->connect_bss->channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_IR | - IEEE80211_CHAN_NO_20MHZ)) { + IEEE80211_CHAN_NO_20MHZ | IEEE80211_CHAN_RADAR)) { ret = -EINVAL; goto free; } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 3e30ec3aff659..f6fc1f235e619 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -94,8 +94,8 @@ struct mt7932 { struct ieee80211_supported_band band2; struct cfg80211_chan_def connect_chandef; u8 connect_center; - struct ieee80211_channel channels[13]; - struct ieee80211_channel channels5[4]; + struct ieee80211_channel channels[MT7932_CHANNELS_2G]; + struct ieee80211_channel channels5[MT7932_CHANNELS_5G]; struct ieee80211_supported_band band5; u8 scan_band, scan_batch; struct ieee80211_rate rates[12]; diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory.h b/drivers/net/wireless/mediatek/mt7932/regulatory.h index 6f3869fe7200e..0ea60d9d7abcb 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory.h +++ b/drivers/net/wireless/mediatek/mt7932/regulatory.h @@ -5,6 +5,8 @@ /* Private, versioned packaging of original-derived country policy. This is * not firmware code and does not replace cfg80211's regulatory database. */ +#include "channels.h" + #define MT7932_POLICY_TABLES 13 #define MT7932_POLICY_MODES 3 @@ -20,12 +22,12 @@ struct mt7932_reg_snapshot { * profile. Stock unknown-country startup uses XZ while Linux stays 00. */ u8 alpha2[2]; - u8 domain[148]; + u8 domain[12 + 8 * MT7932_CHANNELS]; u16 length; /* Keep numeric limits in equality checks even though CID0f contains only * flags. An unsupported reduction must not be mistaken for no change. */ - s32 power[17]; + s32 power[MT7932_CHANNELS]; int error; }; diff --git a/drivers/net/wireless/mediatek/mt7932/scan.h b/drivers/net/wireless/mediatek/mt7932/scan.h index 4899cbb6c1654..ae26645cc3693 100644 --- a/drivers/net/wireless/mediatek/mt7932/scan.h +++ b/drivers/net/wireless/mediatek/mt7932/scan.h @@ -5,6 +5,9 @@ #include "protocol.h" #include "channels.h" +/* CID03 carries this many band/channel slots, from 0xa0 up to 0xe0. */ +#define MT7932_SCAN_CHANNELS 32 + /* Selected ordinary TX17/fullmac wildcard scan, not a pre-RF gate. * Caller must own BSS0 and filter cfg80211 DISABLED channels first. NO_IR * channels require mt7932_scan_passive() before submission. @@ -17,7 +20,7 @@ static inline int mt7932_scan_body(u8 *body, size_t capacity, u8 seq, 0x6b,7,0x0f,0xff,0xff,0xff,0xff,0xff,0xff}; unsigned int i, j; - if (capacity < 0x4d4 || !seq || seq > 127 || !count || count > 13) + if (capacity < 0x4d4 || !seq || seq > 127 || !count || count > MT7932_SCAN_CHANNELS) return -EINVAL; for (i = 0; i < count; i++) { if (!mt7932_channel_band(channels[i]) || diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index d189f4ad13943..a24597fe0b369 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -334,22 +334,29 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request memcpy(reg.domain, "XZ", 2); reg.domain[4] = 1; /* 2 GHz remains 20 MHz. */ reg.domain[5] = mt7932_domain_5g_bw(); - for (i = 0; i < 17; i++) { - struct ieee80211_channel *channel = i < 13 ? &m->channels[i] : &m->channels5[i - 13]; + for (i = 0; i < MT7932_CHANNELS; i++) { + struct ieee80211_channel *channel = i < MT7932_CHANNELS_2G ? + &m->channels[i] : &m->channels5[i - MT7932_CHANNELS_2G]; u32 flags = channel->flags; reg.power[i] = channel->max_power; if (flags & IEEE80211_CHAN_DISABLED) continue; - if (flags & (IEEE80211_CHAN_RADAR | IEEE80211_CHAN_NO_OFDM | - IEEE80211_CHAN_NO_20MHZ | IEEE80211_CHAN_PSD | IEEE80211_CHAN_IR_CONCURRENT)) - reg.error = -EOPNOTSUPP; /* Native per-rate units are not a proven dBm conversion. Do not * silently ignore a numeric reduction below the qualified 20 dBm * channel envelope or reinterpret sentinel bytes as power caps. + * A channel with a restriction the driver cannot honour is left + * out of the firmware domain and disabled for cfg80211 instead. + */ + if (flags & (IEEE80211_CHAN_NO_OFDM | IEEE80211_CHAN_NO_20MHZ | + IEEE80211_CHAN_PSD | IEEE80211_CHAN_IR_CONCURRENT) || + channel->max_power < 20) { + channel->flags |= IEEE80211_CHAN_DISABLED; + continue; + } + /* Radar channels carry cfg80211's RADAR flag: the firmware may + * listen there, but the station never probes or joins them. */ - if (channel->max_power < 20) - reg.error = -EOPNOTSUPP; put_unaligned_le16(channel->hw_value, reg.domain + 12 + count * 8); /* Driver bandwidth limits are already in channel->flags. Adding * the old blanket 20 MHz mask here would contradict the wiphy @@ -357,7 +364,7 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request */ put_unaligned_le32(flags, reg.domain + 16 + count * 8); count++; - reg.domain[i < 13 ? 8 : 9]++; + reg.domain[i < MT7932_CHANNELS_2G ? 8 : 9]++; } reg.length = 12 + count * 8; if (!count) @@ -439,7 +446,7 @@ int mt_register_regulatory_gate(struct mt7932 *m) * Global hints from other radios still reach the notifier and are handled. */ m->wiphy->regulatory_flags |= REGULATORY_COUNTRY_IE_IGNORE; - for (i = 0; i < 13; i++) { + for (i = 0; i < MT7932_CHANNELS_2G; i++) { m->channels[i].band = NL80211_BAND_2GHZ; m->channels[i].center_freq = 2412 + 5 * i; m->channels[i].hw_value = i + 1; @@ -457,12 +464,22 @@ int mt_register_regulatory_gate(struct mt7932 *m) m->wiphy->bands[NL80211_BAND_2GHZ] = &m->band2; { for (i = 0; i < ARRAY_SIZE(m->channels5); i++) { + unsigned int primary = mt7932_channel_5g(i); + unsigned int pair = mt7932_channel_ht40_centre(primary); + u32 flags = IEEE80211_CHAN_NO_160MHZ; + + /* The secondary channel sits on the far side of the pair centre. */ + if (!pair) + flags |= IEEE80211_CHAN_NO_HT40 | IEEE80211_CHAN_NO_80MHZ; + else if (pair < primary) + flags |= IEEE80211_CHAN_NO_HT40PLUS; + else + flags |= IEEE80211_CHAN_NO_HT40MINUS; m->channels5[i].band = NL80211_BAND_5GHZ; - m->channels5[i].center_freq = 5180 + 20 * i; - m->channels5[i].hw_value = 36 + 4 * i; + m->channels5[i].center_freq = 5000 + 5 * primary; + m->channels5[i].hw_value = primary; m->channels5[i].max_power = 20; - m->channels5[i].flags = IEEE80211_CHAN_NO_160MHZ | - (i % 2 ? IEEE80211_CHAN_NO_HT40PLUS : IEEE80211_CHAN_NO_HT40MINUS); + m->channels5[i].flags = flags; if (MT7932_MAX_5G_BW < 2) m->channels5[i].flags |= IEEE80211_CHAN_NO_80MHZ; if (!MT7932_MAX_5G_BW) From 6e3838e91a3bc5c7e5c08c7fe5b4cf96edca7110 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:35:51 +1100 Subject: [PATCH 0719/1579] wifi: mt7932: honour the channel permissions of the country package The country package that the driver sends to the firmware has a per-rate power limit table for every channel. Where a country does not permit a channel, every limit of that channel is undefined. Examples are UNII-3 in Japan, UNII-2e in China, 120 to 128 in Australia and 144 in most of Europe. The driver still listed such channels in the firmware domain whenever the cfg80211 regulatory domain allowed them. Drop channels the loaded package does not permit from the firmware domain, and disable them in cfg80211 too, so that scans and connects do not offer them. cfg80211 recomputes channel flags on each regulatory change, and the package is reloaded for each change, so the two stay in step. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 4 +- .../net/wireless/mediatek/mt7932/regulatory.h | 50 +++++++++++++++++++ .../net/wireless/mediatek/mt7932/startup.c | 28 +++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index b7675f57e8350..9e590ac89a12e 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -31,7 +31,9 @@ The station uses 2.4 GHz channels 1-13 and the 5 GHz channels 36-64, channels stay passive: the firmware listens on them while the station is not associated, but never probes or joins an access point there, and scans made while associated skip them. A channel whose restrictions the driver cannot -apply (reduced power, no OFDM, PSD limits) is disabled rather than used. +apply (reduced power, no OFDM, PSD limits) is disabled rather than used, and +so is a channel the country package does not permit: every power limit of +that channel in ``policy/.bin`` is left undefined. Use the ordinary cfg80211 regulatory database and applicable country policy. The validated first-admission fallback is kernel country 00 with firmware XZ. diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory.h b/drivers/net/wireless/mediatek/mt7932/regulatory.h index 0ea60d9d7abcb..a49885cdb9788 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory.h +++ b/drivers/net/wireless/mediatek/mt7932/regulatory.h @@ -102,4 +102,54 @@ static inline int mt7932_policy_parse(struct mt7932_policy *out, return at == size ? 0 : -EINVAL; } +/* Every entry of a validated package starts with its channel number and is + * followed by per-rate power limits. A country that does not permit a + * channel leaves all of that channel's limits at this byte. + */ +#define MT7932_POLICY_NO_LIMIT 0xc4 + +static inline bool mt7932_policy_permits(const struct mt7932_policy *policy, + unsigned int channel) +{ + unsigned int i, j, k; + + for (i = 1; i <= 9; i++) { + const u8 *table = policy->table[i]; + + for (j = 0; j < table[4]; j++) { + const u8 *entry = table + 44 + j * 122; + + if (entry[0] != channel) + continue; + for (k = 1; k < 122; k++) + if (entry[k] != MT7932_POLICY_NO_LIMIT) + return true; + return false; + } + } + return false; +} + +/* Remove the channels the country package forbids from a CID0f domain. */ +static inline void mt7932_policy_filter(struct mt7932_reg_snapshot *reg, + const struct mt7932_policy *policy) +{ + unsigned int i, kept = 0, count = (reg->length - 12) / 8; + + reg->domain[8] = 0; + reg->domain[9] = 0; + for (i = 0; i < count; i++) { + const u8 *entry = reg->domain + 12 + i * 8; + unsigned int channel = get_unaligned_le16(entry); + + if (!mt7932_policy_permits(policy, channel)) + continue; + memmove(reg->domain + 12 + kept * 8, entry, 8); + reg->domain[channel <= 14 ? 8 : 9]++; + kept++; + } + memset(reg->domain + 12 + kept * 8, 0, (count - kept) * 8); + reg->length = 12 + kept * 8; +} + #endif diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index a24597fe0b369..9b54240339526 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -1,4 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ +#include + #include "mt7932.h" static int mt_stock_config(struct mt7932 *m); @@ -221,6 +223,28 @@ static int mt_startup_once(struct mt7932 *m) return ret; } +/* Keep cfg80211's channel list in step with the channels the country + * package leaves out of the firmware domain. cfg80211 recomputes the flags + * on its next regulatory change, which runs this again. + */ +static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *policy) +{ + unsigned int i; + + rtnl_lock(); + wiphy_lock(m->wiphy); + for (i = 0; i < MT7932_CHANNELS; i++) { + struct ieee80211_channel *channel = i < MT7932_CHANNELS_2G ? + &m->channels[i] : &m->channels5[i - MT7932_CHANNELS_2G]; + + if (!(channel->flags & IEEE80211_CHAN_DISABLED) && + !mt7932_policy_permits(policy, channel->hw_value)) + channel->flags |= IEEE80211_CHAN_DISABLED; + } + wiphy_unlock(m->wiphy); + rtnl_unlock(); +} + static void mt_startup_work(struct work_struct *work) { struct mt7932 *m = container_of(work, struct mt7932, startup_work); @@ -275,6 +299,10 @@ static void mt_startup_work(struct work_struct *work) } if (!ret) ret = mt7932_policy_parse(&policy, file->data, file->size, reg.domain); + if (!ret) { + mt7932_policy_filter(®, &policy); + mt_policy_disable(m, &policy); + } mutex_lock(&m->command_mutex); if (READ_ONCE(m->stopping) || generation != READ_ONCE(m->reg_generation)) goto next; From 22fc65ce67093ebc6ecb5a4af8065fe6449820a2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:37:23 +1100 Subject: [PATCH 0720/1579] wifi: mt7932: let an address change wait for a retiring scan NetworkManager sets a per-network station address before each connection by default: Fedora uses a stable per-SSID address. It takes the interface down, changes the address and brings the interface back up. Taking the interface down cancels any running scan, but the firmware completes the cancellation asynchronously. Until then the driver refuses the address change with -EBUSY, so the activation fails with "set-hw-addr: failed to set-cloned MAC address" and NetworkManager moves on to another profile. When the interface is down, wait for the scan to retire, for up to two seconds, before checking whether the address can change. Changes on a running interface are still refused, as eth_mac_addr() requires. Log the busy state at debug level when a change is still refused. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/frontend.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 0bb2761cf934c..47d0c9e399ce6 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -447,12 +447,25 @@ static int mt_set_mac_address(struct net_device *netdev, void *address) /* No live-MAC change: the next BSS activation and connect commands use * this address together. A failed retirement must not change identity. + * NetworkManager takes the interface down before changing the address, + * which cancels any scan; let that scan retire instead of failing. */ + if (!netif_running(netdev)) { + unsigned long deadline = jiffies + msecs_to_jiffies(2000); + + mt_scan_quiesce(m); + while (READ_ONCE(m->retired_scan_seq) && time_before(jiffies, deadline)) + wait_for_completion_timeout(&m->scan_done, msecs_to_jiffies(100)); + } mutex_lock(&m->command_mutex); spin_lock_irqsave(&m->response_lock, flags); if (m->stopping || m->link_failed || m->bss_active || m->connecting || m->connected || m->disconnecting || m->peer_valid || m->scan_request || m->retired_scan_seq || m->discovering) { + dev_dbg(&m->pdev->dev, + "address change refused: bss=%u connecting=%u connected=%u peer=%u scan=%u retired=%u\n", + m->bss_active, m->connecting, m->connected, m->peer_valid, + !!m->scan_request, m->retired_scan_seq); ret = -EBUSY; } else { ret = eth_mac_addr(netdev, address); From 6f8eb48033910a5431b4e7fdc4d3ab17798fb585 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 04:02:25 +1100 Subject: [PATCH 0721/1579] PCI: apple: keep T8140 hosts without a bootstrap link root-port-only With the J700 bootstrap on by default, every T8140 host now looks up its "apple,piodma" engine. A host whose device tree has no such property fails its probe with -EINVAL instead of falling back to the root-port-only path. That is the opposite of the intent that the board device tree selects the bootstrap. Only look up a supplier when the property is present. A property that names a missing or wrong engine still fails the probe. Also stop logging "downstream config blocked" for the port of a host whose bootstrap is about to open it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple-piodma-diag.c | 5 +++-- drivers/pci/controller/pcie-apple.c | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index c0a85e04b6602..983b4c7ba706f 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -52,8 +52,9 @@ #define PIODMA_IRQ_LIMIT 16 /* - * The board DT opts in by pointing the host's "apple,piodma" at this engine. - * enumerate=0 keeps the host on its root-port-only path for diagnosis. + * The board DT opts in by pointing the host's "apple,piodma" at this engine; + * a host without that property stays root-port-only. enumerate=0 keeps every + * host on its root-port-only path for diagnosis. */ static bool enumerate = true; module_param(enumerate, bool, 0400); diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 3e416b53eb3f5..08c00e008efbc 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -1613,10 +1613,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (pcie->hw->tunneled) WRITE_ONCE(port->started, true); if (pcie->hw->root_bus_only) - dev_info(pcie->dev, - "port %pOF RUN=%#x link=%#x; downstream config blocked\n", + dev_info(pcie->dev, "port %pOF RUN=%#x link=%#x; downstream config %s\n", np, apple_pcie_port_readl(port, PORT_STATUS), - apple_pcie_port_readl(port, PORT_LINKSTS)); + apple_pcie_port_readl(port, PORT_LINKSTS), + pcie->piodma_supplier ? "waits for the bootstrap" : "blocked"); return 0; @@ -2335,7 +2335,9 @@ static int apple_pcie_probe(struct platform_device *pdev) } } - if (hw->root_bus_only && apple_piodma_bootstrap_enabled()) { + /* Only a host that the board links to a bootstrap engine opens bus 1. */ + if (hw->root_bus_only && apple_piodma_bootstrap_enabled() && + of_property_present(dev->of_node, "apple,piodma")) { ret = apple_piodma_bootstrap_get(dev, &piodma_supplier); if (ret) return dev_err_probe(dev, ret, "PIODMA supplier unavailable\n"); From 7487713a0b371e3b25333b7a33bd70b868b810c9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 04:02:43 +1100 Subject: [PATCH 0722/1579] PCI: apple: remove the J700 radio functions when publication fails If the root memory-window check or the cold check fails after the downstream scan, the host closes downstream configuration access but leaves both functions on the bus, scanned and unpublished. A later bus rescan would publish them and let their drivers probe through a closed config window, with memory decoding off. Remove the scanned functions, while still holding the rescan lock and before closing configuration access. A later rescan then finds nothing. Also check and quiesce both functions in the cold check instead of stopping at the first failure, so that a second function left bus mastering is cleared too. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple.c | 30 +++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 08c00e008efbc..81642e1769d74 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2234,18 +2234,21 @@ static int apple_pcie_neo_check_cold(struct apple_pcie *pcie, struct pci_dev *ro u16 command; int function, ret = 0; - for (function = 0; function < 2 && !ret; function++) { + /* Check and quiesce both functions; keep the first error. */ + for (function = 0; function < 2; function++) { endpoint = pci_get_slot(root->subordinate, PCI_DEVFN(0, function)); - if (!endpoint) - return -ENODEV; + if (!endpoint) { + ret = ret ?: -ENODEV; + continue; + } if (pci_read_config_word(endpoint, PCI_COMMAND, &command)) { - ret = -EIO; + ret = ret ?: -EIO; } else { dev_info(pcie->dev, "enumerated %s id=%04x:%04x command=%#x\n", pci_name(endpoint), endpoint->vendor, endpoint->device, command); if (command & PCI_COMMAND_MASTER) { pci_clear_master(endpoint); - ret = -EIO; + ret = ret ?: -EIO; } } pci_dev_put(endpoint); @@ -2253,6 +2256,19 @@ static int apple_pcie_neo_check_cold(struct apple_pcie *pcie, struct pci_dev *ro return ret; } +/* + * Rescan lock held, downstream config still open. Remove functions that were + * scanned but will not be published, so that a later rescan cannot publish + * them after config access has been closed again. + */ +static void apple_pcie_neo_unscan(struct pci_dev *root) +{ + struct pci_dev *dev, *tmp; + + list_for_each_entry_safe_reverse(dev, tmp, &root->subordinate->devices, bus_list) + pci_stop_and_remove_bus_device(dev); +} + static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) { struct apple_pcie *pcie = pci_host_bridge_priv(bridge); @@ -2293,8 +2309,10 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) goto close_config; pci_bus_add_devices(root->subordinate); close_config: - if (ret) + if (ret) { + apple_pcie_neo_unscan(root); WRITE_ONCE(pcie->neo_config_ready, false); + } pci_unlock_rescan_remove(); if (!ret) dev_info(pcie->dev, "radio functions 0/1 published\n"); From 920e89a2e3df1a0fd7a87ca24d1f139d8913ae53 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 04:03:41 +1100 Subject: [PATCH 0723/1579] Bluetooth: mt7932: wait for the Wi-Fi function before starting Bluetooth admission was qualified with the Wi-Fi function already running its firmware. The module soft dependency added for that only orders a modprobe of both modules. A built-in transport, insmod, asynchronous probing or a failed Wi-Fi probe all bypass it. Removal is affected too: "modprobe -r mt7932_bt_pcie" now also unloads the Wi-Fi driver, as kmod removes pre-softdeps after the module. Enforce the order in the driver instead. Defer the probe until mt7932-fullmac is bound to function 0, and add a device link so that the driver core unbinds Bluetooth before Wi-Fi. Drop the soft dependency. Bluetooth now stays unbound while the Wi-Fi driver is missing or failed, which matches the qualified sequence. Also warn when the link's power management cannot be disabled, and update the help text and documentation, which still called the transport opt-in. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 22 +++++++------ drivers/bluetooth/Kconfig | 6 ++-- drivers/bluetooth/mt7932_bt_pcie.c | 31 +++++++++++++++++-- 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 9e590ac89a12e..0387cb98098a3 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -4,7 +4,8 @@ MT7932 radio bring-up on the MacBook Neo ====================================== J700 MT7932 Wi-Fi uses cfg80211 and NetworkManager with firmware-managed -WPA2-CCMP authentication. Bluetooth uses an opt-in PCIe transport. +WPA2-CCMP authentication. Bluetooth uses a transport on the radio's second +PCIe function. The Wi-Fi import is pinned to ``aurora-silicon/linux-neo-cleanroom-eryk-with-wifi``, commit @@ -49,14 +50,17 @@ ASPM and clock power management on its link before starting the device, matching the tested admission sequence, which used the PCIe ASPM performance policy. -``mt7932_bt_pcie`` probes the cold ``14c3:793b`` function when it loads and -rejects a function that is not cold; ``enable=0`` leaves it unbound. Its -module soft dependency loads ``mt7932-fullmac`` first, so the Bluetooth -firmware starts after the Wi-Fi firmware, as in the tested sequence. The -transport also disables ASPM on the shared link. Like the Wi-Fi driver, it -reads its firmware while probing, so build it as a module. -Do not reprobe after a failed or uncertain Bluetooth admission. The tested -Wi-Fi driver owns function 0 and Bluetooth owns function 1. +``mt7932_bt_pcie`` defers its probe until ``mt7932-fullmac`` is bound to +function 0, so the Bluetooth firmware always starts after the Wi-Fi firmware, +as in the tested sequence, however the drivers are loaded. Bluetooth therefore +stays unbound while the Wi-Fi driver is missing or has failed. A device link +makes the driver core unbind Bluetooth before Wi-Fi; unloading +``mt7932_bt_pcie`` does not affect Wi-Fi. The probe rejects a function that is +not cold, which is what now prevents a reprobe after a failed or uncertain +admission; ``enable=0`` leaves the function unbound. The transport also +disables ASPM on the shared link. Like the Wi-Fi driver, it reads its firmware +while probing, so build it as a module. The tested Wi-Fi driver owns function +0 and Bluetooth owns function 1. Use a saved NetworkManager WPA2 profile matching the current interface. KDE audio requires BlueZ, PipeWire, its PulseAudio compatibility service, diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 94946efbb22e6..a7d3ef19c6ef7 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -30,8 +30,10 @@ config BT_MTK7932_PCIE select FW_LOADER help Support the J700 Bluetooth PCIe function with locally installed firmware - and calibration. The function is probed when the driver loads; the - enable=0 parameter leaves it unbound. + and calibration. The function is probed once the Wi-Fi function is + bound to the mt7932-fullmac driver; the enable=0 parameter leaves it + unbound. Both functions appear before the root filesystem is mounted + and read their firmware when probed, so say M, like the Wi-Fi driver. System suspend uses the firmware sleep handshake and preserves DMA storage. Hibernation and automatic firmware recovery are unsupported. Removal drains the PCI function before releasing device-visible memory; diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 2078316ac5ff4..f3ac19f810736 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1034,6 +1034,28 @@ static void bt7932_cleanup(struct bt7932 *bt, bool release_dma) kfree_sensitive(bt); } +/* + * Admission was only qualified with the Wi-Fi function already running its + * firmware. Wait until mt7932-fullmac is bound to function 0, then link the + * two functions so that the driver core unbinds this one first. + */ +static int bt7932_wait_for_wifi(struct pci_dev *pdev) +{ + struct pci_dev *wifi = pci_get_slot(pdev->bus, PCI_DEVFN(0, 0)); + int ret = 0; + + if (!wifi) + return -ENODEV; + if (!device_is_bound(&wifi->dev) || + strcmp(dev_driver_string(&wifi->dev), "mt7932-fullmac")) + ret = dev_err_probe(&pdev->dev, -EPROBE_DEFER, + "waiting for the Wi-Fi function\n"); + else if (!device_link_add(&pdev->dev, &wifi->dev, DL_FLAG_AUTOREMOVE_CONSUMER)) + ret = -EINVAL; + pci_dev_put(wifi); + return ret; +} + static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct bt7932 *bt; @@ -1048,8 +1070,13 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) !of_machine_is_compatible("apple,t8140") || pci_domain_nr(pdev->bus) || pdev->bus->number != 1 || pdev->devfn != PCI_DEVFN(0, 1)) return -ENODEV; + ret = bt7932_wait_for_wifi(pdev); + if (ret) + return ret; /* Admission was qualified with ASPM and clock PM off on this link. */ - pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); + ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); + if (ret) + dev_warn(&pdev->dev, "link power management left as found: %d\n", ret); bt = kzalloc_obj(*bt); if (!bt) return -ENOMEM; @@ -1351,8 +1378,6 @@ module_pci_driver(bt7932_driver); MODULE_DESCRIPTION("Experimental standalone J700 MT7932 Bluetooth PCIe transport"); MODULE_LICENSE("GPL"); -/* Admission was qualified after the Wi-Fi function had started its firmware. */ -MODULE_SOFTDEP("pre: mt7932_fullmac"); MODULE_FIRMWARE(BT7932_FW_B0); MODULE_FIRMWARE(BT7932_FW_B1); MODULE_FIRMWARE(BT7932_FW_B1_FALLBACK); From cadb688379b17dd14ae89c8b24915caa717c9fd7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 04:03:48 +1100 Subject: [PATCH 0724/1579] wifi: mt7932: warn when link power management cannot be disabled pci_disable_link_state() fails when the OS has no ASPM control, for example with pcie_aspm=off. The link is then left as the firmware set it up, while station operation was qualified with it disabled. Report that instead of ignoring the error. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 0549ca8575dc1..85440558e95a4 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1182,7 +1182,9 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!domain || (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) return dev_err_probe(&pdev->dev, -EINVAL, "translated DMA domain required\n"); /* Station operation was qualified with ASPM and clock PM off on this link. */ - pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); + ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); + if (ret) + dev_warn(&pdev->dev, "link power management left as found: %d\n", ret); ret = pcim_enable_device(pdev); if (ret) return ret; From 99d44aaa0a860f9b7b4a1bd79782e736751048a2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:03:09 +1100 Subject: [PATCH 0725/1579] wifi: mt7932: skip a country-package pass that a newer request superseded The startup work applies the country package to cfg80211's channel list before it checks whether the request it is serving is still current. If the country changes again while the work is loading the first package, cfg80211 has already recomputed the channel flags for the second country, and the stale pass then disables the first country's forbidden channels on top of them. The next pass adds the second country's own disables, so the union of both stays in place until the following regulatory change. The result is only ever more restrictive than intended, and the boot sequence cannot hit it because the world package forbids nothing, but a user who changes country twice in quick succession loses channels that the final country permits. The regulatory notifier runs under RTNL and starts a new generation there, after cfg80211 has recomputed the flags. Check the generation under RTNL in mt_policy_disable() and leave the channel list alone when it has moved on; the newer generation runs its own pass. Fixes: 6e3838e91a3b ("wifi: mt7932: honour the channel permissions of the country package") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/startup.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 9b54240339526..631a77aa684cd 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -227,11 +227,23 @@ static int mt_startup_once(struct mt7932 *m) * package leaves out of the firmware domain. cfg80211 recomputes the flags * on its next regulatory change, which runs this again. */ -static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *policy) +static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *policy, + u32 generation) { unsigned int i; rtnl_lock(); + /* cfg80211 recomputes the channel flags for a new country under RTNL + * and only then calls the notifier, which starts a new generation. + * Once that has happened this package belongs to an older request, + * and applying it would disable channels on top of the new country's + * flags until the next regulatory change. The newer generation runs + * its own pass. + */ + if (generation != READ_ONCE(m->reg_generation)) { + rtnl_unlock(); + return; + } wiphy_lock(m->wiphy); for (i = 0; i < MT7932_CHANNELS; i++) { struct ieee80211_channel *channel = i < MT7932_CHANNELS_2G ? @@ -301,7 +313,7 @@ static void mt_startup_work(struct work_struct *work) ret = mt7932_policy_parse(&policy, file->data, file->size, reg.domain); if (!ret) { mt7932_policy_filter(®, &policy); - mt_policy_disable(m, &policy); + mt_policy_disable(m, &policy, generation); } mutex_lock(&m->command_mutex); if (READ_ONCE(m->stopping) || generation != READ_ONCE(m->reg_generation)) From e2c9e544b9b0447d405b48e75a3714452dfc615a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:03:36 +1100 Subject: [PATCH 0726/1579] wifi: mt7932: bound the address-change wait to two seconds An address change on a stopped interface waits for a cancelled scan to retire before it checks whether the address can change. The change runs under RTNL, and the wait is documented as lasting at most two seconds, but it is longer in two cases: - mt_scan_quiesce() first waits up to five seconds for the scan to complete, and the two-second loop only starts after that, so a firmware that never retires the scan holds RTNL for about seven seconds; - once the driver is stopping, mt_stop_host() has completed scan_done for good, so every wait in the loop returns at once and the loop spins for two seconds. The interface is already down at this point, so its scan has been cancelled and only the retirement is left to wait for. Cancel any scan that is still recorded, then wait for the retirement against a single two-second deadline, and stop waiting as soon as the driver is stopping. The address change itself is unchanged: it is still refused with -EBUSY while a retirement is outstanding. Fixes: 22fc65ce6709 ("wifi: mt7932: let an address change wait for a retiring scan") Assisted-by: LLM Signed-off-by: Ryan Murray --- .../net/wireless/mediatek/mt7932/frontend.c | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 47d0c9e399ce6..50f424cdf3c03 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -137,6 +137,24 @@ int mt_scan_quiesce(struct mt7932 *m) return READ_ONCE(m->retired_scan_seq) ? -EBUSY : 0; } +/* Wait at most @timeout in total for a cancelled scan to retire. Callers + * may hold RTNL, so this never waits longer than asked. A stopping driver + * completes scan_done for good, which would turn the wait into a spin. + */ +static void mt_scan_retire_wait(struct mt7932 *m, unsigned long timeout) +{ + unsigned long deadline = jiffies + timeout; + long left; + + mt_abort_scan(m->wiphy, &m->wdev); + flush_work(&m->scan_finish_work); + while (READ_ONCE(m->retired_scan_seq) && !READ_ONCE(m->stopping)) { + left = (long)(deadline - jiffies); + if (left <= 0 || !wait_for_completion_timeout(&m->scan_done, left)) + break; + } +} + /* command_mutex held: one RF-band/type batch at a time, with matching EID0d * completion before the next batch. A request without SSIDs is passive; * active requests must also listen passively on NO_IR channels. @@ -450,13 +468,8 @@ static int mt_set_mac_address(struct net_device *netdev, void *address) * NetworkManager takes the interface down before changing the address, * which cancels any scan; let that scan retire instead of failing. */ - if (!netif_running(netdev)) { - unsigned long deadline = jiffies + msecs_to_jiffies(2000); - - mt_scan_quiesce(m); - while (READ_ONCE(m->retired_scan_seq) && time_before(jiffies, deadline)) - wait_for_completion_timeout(&m->scan_done, msecs_to_jiffies(100)); - } + if (!netif_running(netdev)) + mt_scan_retire_wait(m, msecs_to_jiffies(2000)); mutex_lock(&m->command_mutex); spin_lock_irqsave(&m->response_lock, flags); if (m->stopping || m->link_failed || m->bss_active || m->connecting || From 19766a407ad14f61fd8527b1084686d20a891bb7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:03:49 +1100 Subject: [PATCH 0727/1579] wifi: mt7932: name the forbidden-channel marker after what it means A country package marks a channel the country does not permit by setting every per-rate limit of the channel to 0xc4. The constant for that byte is called MT7932_POLICY_NO_LIMIT, which reads as "unrestricted", the opposite of its meaning. Rename it to MT7932_POLICY_NOT_PERMITTED. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/regulatory.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory.h b/drivers/net/wireless/mediatek/mt7932/regulatory.h index a49885cdb9788..fd65e07d9506a 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory.h +++ b/drivers/net/wireless/mediatek/mt7932/regulatory.h @@ -104,9 +104,9 @@ static inline int mt7932_policy_parse(struct mt7932_policy *out, /* Every entry of a validated package starts with its channel number and is * followed by per-rate power limits. A country that does not permit a - * channel leaves all of that channel's limits at this byte. + * channel sets all of that channel's limits to this marker. */ -#define MT7932_POLICY_NO_LIMIT 0xc4 +#define MT7932_POLICY_NOT_PERMITTED 0xc4 static inline bool mt7932_policy_permits(const struct mt7932_policy *policy, unsigned int channel) @@ -122,7 +122,7 @@ static inline bool mt7932_policy_permits(const struct mt7932_policy *policy, if (entry[0] != channel) continue; for (k = 1; k < 122; k++) - if (entry[k] != MT7932_POLICY_NO_LIMIT) + if (entry[k] != MT7932_POLICY_NOT_PERMITTED) return true; return false; } From b2f686e884399dac0a2462ba44370767753a1589 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:05:49 +1100 Subject: [PATCH 0728/1579] arm64: configs: asahi: enable the J700 Wi-Fi driver asahi.config asks for the J700 Bluetooth transport, but not for the Wi-Fi driver whose bound function the transport waits for before it starts. Ask for both, as j700-radio.config already does. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index ed937d8d20bd4..5650fd8192dbc 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -15,6 +15,7 @@ CONFIG_PCIE_APPLE=y CONFIG_NVME_APPLE=m CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y +CONFIG_MT7932_FULLMAC=m CONFIG_TOUCHSCREEN_APPLE_Z2=m CONFIG_INPUT_MACSMC_INPUT=m CONFIG_I2C_APPLE=m From a0ad38bdec9d1d14ffabebb1c480ff9b814287e1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:05:50 +1100 Subject: [PATCH 0729/1579] Bluetooth: mt7932: depend on the Wi-Fi driver The Bluetooth transport defers its probe until the Wi-Fi function is bound to mt7932-fullmac. Without that driver it can never bind, yet it can still be selected with MT7932_FULLMAC=n and then sits in the deferred probe list for good. Make the option depend on the Wi-Fi driver being enabled. Use "!= n" rather than a plain dependency so that the transport can still be built in while the Wi-Fi driver is a module; the deferred probe handles that order. Fixes: 920e89a2e3df ("Bluetooth: mt7932: wait for the Wi-Fi function before starting") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index a7d3ef19c6ef7..da96e2f2c05e3 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -27,6 +27,7 @@ config BT_MTK7932_PCIE tristate "J700 MT7932 PCIe Bluetooth transport" depends on BT && PCI && ARCH_APPLE && IOMMU_DMA depends on !KEXEC_CORE + depends on MT7932_FULLMAC != n select FW_LOADER help Support the J700 Bluetooth PCIe function with locally installed firmware From 744c2c7abc15c0acd9f73f70cb4e0f698ab113f2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:07:06 +1100 Subject: [PATCH 0730/1579] drm: apple: skip the H17P sleep drain on a crashed or stopped DCP Before system sleep, the H17P drain runs up to four passes to settle a pending brightness change. A crashed or stopped DCP rejects every present, and the queue drain reports it idle at once, so a level that was dirty when it stopped stays pending: all four passes run and the suspend logs "brightness change still pending at suspend", although there is nothing left that could be sent. Return from the drain as soon as the DCP has crashed or its queue has stopped. The crash itself is already reported. Fixes: 6d4dc0a94dc9 ("drm: apple: settle H17P brightness retries before system sleep") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c6924e2d32982..b5eaac2964445 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3844,6 +3844,12 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) * the first attempt and the policy's three retries. */ for (pass = 0; pass < 4; pass++) { + /* + * A crashed or stopped DCP takes no more presents, so a level + * still pending there can never be sent and is not waited for. + */ + if (READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.stopped)) + return; flush_delayed_work(&dcp->iomfb.backlight_retry); if (dcp_has_panel(dcp)) flush_work(&dcp->bl_update_wq); From 5aa3c2bbf8352e5d84ac455887ad3c95a442b532 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:07:35 +1100 Subject: [PATCH 0731/1579] drm: apple: treat a seeded H17P backlight level as known The H17P backlight policy re-sends a level on the firmware's request only when Linux knows what the panel shows: the loader's reported level or one that a completed present carried. dcp_bl_seed() puts a decoded powerlog sample, which is the loader's level, into the state before Linux takes control, but it does not mark the level as known. A seed over a registration default therefore leaves a real level that cannot be re-sent, and the state disagrees with its own definition. The seeding helper has no caller in the driver yet, so nothing goes wrong today, but wiring it up later, for example to let a late powerlog report correct a registration default, would hit this. Mark a seeded level as known and cover both the accepted and the refused seed in KUnit. Fixes: 73dd5dedcc77 ("drm: apple: never re-send a guessed H17P backlight level") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.h | 6 ++++- drivers/gpu/drm/apple/dcp_backlight_test.c | 26 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 73683cb0322a3..2c9a0a7ab97be 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -80,13 +80,17 @@ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, return 0; } -/* A decoded powerlog sample can seed takeover only before Linux control. */ +/* + * A decoded powerlog sample can seed takeover only before Linux control. + * It reports the level the panel shows, so the level is then known. + */ static inline bool dcp_bl_seed(struct dcp_backlight_state *state, u32 nits) { if (!state->ready || state->controlled || nits > state->maximum) return false; state->target = nits; state->actual = nits; + state->level_known = true; return true; } diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index 0b94b41b41482..a0aaad3ddd232 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -292,6 +292,31 @@ static void backlight_fallback_test(struct kunit *test) KUNIT_EXPECT_EQ(test, present.nits, 262U); } +static void backlight_seed_known_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* A loader sample seeded over a registration default is known. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_EXPECT_FALSE(test, state.level_known); + KUNIT_EXPECT_TRUE(test, dcp_bl_seed(&state, 180)); + KUNIT_EXPECT_TRUE(test, state.level_known); + state.controlled = true; + KUNIT_EXPECT_TRUE(test, dcp_bl_resend(&state)); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 180U); + + /* A refused sample leaves the default unknown. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 526)); + KUNIT_EXPECT_FALSE(test, state.level_known); + state.controlled = true; + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 180)); + KUNIT_EXPECT_FALSE(test, state.level_known); + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); +} + static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), @@ -304,6 +329,7 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_inflight_dpms_test), KUNIT_CASE(backlight_resend_test), KUNIT_CASE(backlight_fallback_test), + KUNIT_CASE(backlight_seed_known_test), {} }; From fd7b445d2459361f30cb3099945d7c0c4eb047c8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:08:20 +1100 Subject: [PATCH 0732/1579] drm: apple: stop dropping DCP node references inside the match loops for_each_matching_node() hands each node to the loop body with a reference held and drops that reference itself when it moves on to the next node. apple_drm_init_dcp() also calls of_node_put() on every node it visits, and add_dcp_components() does the same, so every probe of the display subsystem drops one reference more than it took on each DCP node, and binding it drops another. The nodes are static and their platform devices keep further references, so nothing has been freed, but the counts are wrong, and repeated probe and bind attempts walk them towards a "Bad of_node_put()" error with CONFIG_OF_DYNAMIC. Leave the references to the iterator. Fixes: f2bc92ff49a9 ("drm/apple: Add DCP display driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 991517b43dcec..df31009e3ff6b 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -663,15 +663,12 @@ static int apple_drm_init_dcp(struct device *dev) for_each_matching_node(np, apple_dcp_id_tbl) { bool dcp_ext; - if (!apple_dcp_usable(np)) { - of_node_put(np); + if (!apple_dcp_usable(np)) continue; - } dcp_ext = of_device_is_compatible(np, "apple,dcpext") || of_property_present(np, "phys"); dcp[num_dcp] = of_find_device_by_node(np); - of_node_put(np); if (!dcp[num_dcp]) continue; @@ -875,7 +872,6 @@ static int add_dcp_components(struct device *dev, of_node_put(port); } } - of_node_put(np); } return num; From aa6b682cb7be04da688ff2fe2e4c43fb911dc2e0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:08:39 +1100 Subject: [PATCH 0733/1579] drm: apple: do not log a missing EDID source for internal panels dcp_get_modes() used to ask for an EDID only once the DCP had announced its AV service. Since the EDID read pins the service itself, it is asked on every mode probe and returns -ENODEV when there is no service. Internal panels never have one, so every mode probe on a laptop now logs "copy_edid failed: -ENODEV" at info level. Treat -ENODEV on a DCP that drives an internal panel as the expected "no EDID" answer and log it at debug level. External outputs, and any other failure, are reported as before, and the EDID read itself is unchanged. Fixes: 5837706b10e3 ("drm/apple: pin DP service owners before recycling retired AFK slots") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index c52fceb5c7391..3ee1e95482d19 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -970,7 +970,14 @@ int dcp_get_modes(struct drm_connector *connector) const struct drm_edid *edid; edid = dcpavserv_copy_edid(dcp); if (IS_ERR_OR_NULL(edid)) { - dev_info(dcp->dev, "copy_edid failed: %pe\n", edid); + /* + * An internal panel has no AV service and so no EDID; + * do not report that on every mode probe. + */ + if (dcp_has_panel(dcp) && PTR_ERR(edid) == -ENODEV) + dev_dbg(dcp->dev, "no EDID source for the panel\n"); + else + dev_info(dcp->dev, "copy_edid failed: %pe\n", edid); } else { drm_edid_free(apple_connector->drm_edid); apple_connector->drm_edid = edid; From a7fa3d8fbf5690df6553896a4f984b8cdc8069c2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:10:10 +1100 Subject: [PATCH 0734/1579] arm64: dts: apple: t8140: describe each power state once t8140-pmgr.dtsi extends &pmgr in two blocks, and four power states are described in both: dispext0_sys, dispext0_fe, dispext0_cpu and spi_p. dtc merges the two copies, and where they disagree the second wins silently. That already bit once: the external display states had to be disabled in the second copy because only the first one said so. Keep the first copy of each state, which is where dtc places the merged node, and drop the second. Carry over the only property that differed, spi_p's explicit status, and the comment explaining why the external display states are left to the firmware. The compiled t8140-j700.dtb is byte-identical before and after. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 50 ++++------------------- 1 file changed, 7 insertions(+), 43 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index a8cea58a82da4..91353ac335431 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -98,6 +98,12 @@ label = "disp_sys"; }; + /* + * The external display coprocessor has no Linux consumer yet. Its + * firmware keeps it running, so leave its three states, dispext0_sys, + * dispext0_fe and dispext0_cpu, to it: powering dispext0_cpu down + * underneath that firmware faults. + */ ps_dispext0_sys: power-controller@288 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x288 4>; @@ -130,6 +136,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "spi_p"; + status = "okay"; }; ps_apcie_st: power-controller@310 { @@ -803,20 +810,6 @@ status = "okay"; }; - /* - * The external display coprocessor has no Linux consumer yet. Its - * firmware keeps it running, so leave these three states to it: - * powering dispext0_cpu down underneath that firmware faults. - */ - ps_dispext0_sys: power-controller@288 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x288 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "dispext0_sys"; - status = "disabled"; - }; - ps_ane_sys: power-controller@290 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x290 4>; @@ -908,15 +901,6 @@ status = "okay"; }; - ps_spi_p: power-controller@2f0 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x2f0 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "spi_p"; - status = "okay"; - }; - ps_uart_p: power-controller@2f8 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x2f8 4>; @@ -945,26 +929,6 @@ status = "okay"; }; - ps_dispext0_fe: power-controller@320 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x320 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "dispext0_fe"; - power-domains = <&ps_dispext0_sys>; - status = "disabled"; - }; - - ps_dispext0_cpu: power-controller@328 { - compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x328 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "dispext0_cpu"; - power-domains = <&ps_dispext0_fe>; - status = "disabled"; - }; - ps_spi0: power-controller@330 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x330 4>; From 953220ab6d748bd05db3854f374b701696f6c795 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:10:11 +1100 Subject: [PATCH 0735/1579] arm64: dts: apple: t8140: describe the external display DARTs once dispext0_dart and dcpext0_dart are described twice, identically: once in the soc node and again in the &soc extension further down. Drop the second copy. The compiled t8140-j700.dtb is byte-identical before and after. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index fd419d1fe8a49..b4f6219510a15 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1248,24 +1248,6 @@ }; }; - dispext0_dart: iommu@416300000 { - compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x4 0x16300000 0x0 0xc000>; - interrupt-parent = <&aic>; - interrupts = ; - #iommu-cells = <1>; - status = "disabled"; - }; - - dcpext0_dart: iommu@416340000 { - compatible = "apple,t8140-dart", "apple,t8110-dart"; - reg = <0x4 0x16340000 0x0 0xc000>; - interrupt-parent = <&aic>; - interrupts = ; - #iommu-cells = <1>; - status = "disabled"; - }; - pinctrl_ap: pinctrl@31a000000 { compatible = "apple,t8140-pinctrl", "apple,t8103-pinctrl"; reg = <0x3 0x1a000000 0x0 0x100000>; From 39a036ead934c701af76e05eebd72b102810861c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:10:12 +1100 Subject: [PATCH 0736/1579] arm64: dts: apple: t8140-j700: enable the PMP nodes once The PMP, its DART, mailbox and report region are enabled twice in the J700 device tree, by two identical sets of status overrides. Drop the second set. The compiled t8140-j700.dtb is byte-identical before and after. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 4742ce8aaca35..216f96dc645b1 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -416,22 +416,6 @@ status = "okay"; }; -&pmp_dart { - status = "okay"; -}; - -&pmp_report { - status = "okay"; -}; - -&pmp_mbox { - status = "okay"; -}; - -&pmp { - status = "okay"; -}; - &i2c2 { status = "okay"; }; From b4cecd3f5fd98933e75cbdef69eef2e3ebbb3bb6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:08:47 +1100 Subject: [PATCH 0737/1579] HID: magicmouse: shorten the haptics workqueue name Workqueue names are limited to 31 characters. "Apple trackpad haptics workqueue" is 32, so the first MTP trackpad probe of every boot warns that the name exceeds WQ_NAME_LEN and truncates it to "Apple trackpad haptics workqueu". Name it "apple-trackpad-haptics" instead. Fixes: cf248ccdaabc ("HID: magicmouse: add support for controlling MTP touchpad haptics") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/hid-magicmouse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 70d066a43279b..e417a65acd7e7 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -2127,7 +2127,7 @@ static int apple_taptic_init_mtp(struct magicmouse_sc *msc, return -ENOMEM; } - haptic_dev->wq = create_singlethread_workqueue("Apple trackpad haptics workqueue"); + haptic_dev->wq = create_singlethread_workqueue("apple-trackpad-haptics"); if (!haptic_dev->wq) { dev_err(&trackpad_hdev->dev, "Cannot allocate haptic workqueue\n"); kfree(msc->haptic_effects); From ec78f8fa20a1740c08613eb5620531040e618920 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 11:12:21 +1100 Subject: [PATCH 0738/1579] dt-bindings: cpu: apple,t8140-cpuidle: Correct the idle description The description calls the T8140 WFI state "clock-gated" and says the power-down states are "not enabled". On a J700, WFI already lets the hardware power-gate idle cores: the PMGR power-state registers of the idle E cores and of the second P core read as power-gated in 94% to 99.9% of 1 kHz samples. What T8140 lacks is a way for software to ask for power-down, so there is no separate state to enable. Describe that instead. No property changes. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/cpu/apple,t8140-cpuidle.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml b/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml index aa27ec55309c8..7d4f9cdbc9182 100644 --- a/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml +++ b/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml @@ -10,9 +10,10 @@ maintainers: - Ryan Murray description: - Selects the Apple CPU idle driver on T8140 systems. Only the WFI - clock-gated state is supported; CPU and cluster power-down states - are not enabled on this SoC. + Selects the Apple CPU idle driver on T8140 systems. On this SoC the + hardware decides when an idle core is power-gated after WFI; software + cannot request core or cluster power-down, so WFI is the only idle + state. properties: $nodename: From df6bc3dde208453f701206041c3ef7d6565feb00 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:28:29 -0600 Subject: [PATCH 0739/1579] wifi: brcmfmac: log AWDL tx completions behind a runtime parameter Commit bc1823a897a7 ("wifi: brcmfmac: bound AWDL creation and validate its data path") moved the per-frame AWDL tx-completion line from pr_info() to brcmf_dbg(MSGBUF). brcmf_dbg() only prints with DEBUG or CONFIG_BRCM_TRACING; with neither, as in the linux-aurora config, it is no_printk() and the line is not compiled in at all. That line is not only a measurement trace. omdrop-discoverable sends a burst of frames on awdl0 and counts these completions to decide whether the firmware is transmitting. A firmware that has parked the AWDL data path parses peers, fills its peer table and elects normally while completing nothing, so no other signal tells the two apart. Without the line, every window fails the check. Add an awdl_trace module parameter, off by default and writable at runtime (0644), and print the completion only while it is set. A default build stays quiet, and userspace turns the trace on only for the seconds it needs it. This is the same parameter the out-of-tree omdrop-awdl driver already carries, so userspace works with either module. Fixes: bc1823a897a7 ("wifi: brcmfmac: bound AWDL creation and validate its data path") Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/common.c | 9 +++++++++ .../broadcom/brcm80211/brcmfmac/debug.h | 1 + .../broadcom/brcm80211/brcmfmac/msgbuf.c | 17 +++++++++-------- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index e3a9ca5526c45..2c99a99031568 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -43,6 +43,15 @@ int brcmf_msg_level; module_param_named(debug, brcmf_msg_level, int, 0600); MODULE_PARM_DESC(debug, "Level of debug output"); +/* AWDL tx completions are logged per frame. Userspace counts them to tell a + * radio that is transmitting from one whose firmware has parked the AWDL data + * path: a parked radio completes nothing, and it looks healthy by every other + * measure. Off by default, writable at runtime so no reload is needed. + */ +int brcmf_awdl_trace; +module_param_named(awdl_trace, brcmf_awdl_trace, int, 0644); +MODULE_PARM_DESC(awdl_trace, "Log AWDL tx completions"); + static int brcmf_p2p_enable; module_param_named(p2pon, brcmf_p2p_enable, int, 0); MODULE_PARM_DESC(p2pon, "Enable legacy p2p management functionality"); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h index 432d93ae8fb85..4c7bd41c704c0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h @@ -117,6 +117,7 @@ do { \ } while (0) extern int brcmf_msg_level; +extern int brcmf_awdl_trace; struct brcmf_pub; #ifdef DEBUG diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index c3dfd76c4dbfa..c9dcba08072a3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -968,14 +968,15 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) */ ring_ifidx = msgbuf->flow->rings[flowid] ? brcmf_flowring_ifidx_get(msgbuf->flow, flowid) : 0; - if (ring_ifidx && __ratelimit(&brcmf_awdl_txs_rs)) - brcmf_dbg(MSGBUF, "awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", - ring_ifidx, tx_status->msg.ifidx, - ifp ? ifp->is_awdl : -1, flowid, - (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), - le16_to_cpu(tx_status->tx_status), - le16_to_cpu(tx_status->metadata_len), - skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); + if (ring_ifidx && READ_ONCE(brcmf_awdl_trace) && + __ratelimit(&brcmf_awdl_txs_rs)) + pr_info("brcmfmac: awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", + ring_ifidx, tx_status->msg.ifidx, + ifp ? ifp->is_awdl : -1, flowid, + (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), + le16_to_cpu(tx_status->tx_status), + le16_to_cpu(tx_status->metadata_len), + skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); brcmf_txfinalize(ifp, skb, true); } From 02b9d52358e90fc8ced576a8acf4b3a4fd6a2917 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 11:46:12 +1100 Subject: [PATCH 0740/1579] media: apple: isp: Let userspace select the capture frame rate The driver reports a single 30 fps frame interval and ignores VIDIOC_S_PARM, so applications cannot ask for a lower rate. Offer 30, 25, 24, 20 and 15 fps as discrete intervals for every frame size. VIDIOC_S_PARM picks the closest of them, a zero interval selects the nominal 30 fps, and a change is refused while streaming because the rate is configured when the stream starts. VIDIOC_G_PARM reports the selected interval. Frame interval enumeration now also rejects pixel formats and frame sizes that the driver does not offer. The selected rate becomes the auto-exposure frame-rate ceiling. The firmware takes rates in units of 1/256 frame per second, as the existing 30 and 15 fps values (7680 and 3840) show. The auto-exposure floor stays at 15 fps, so the default behaviour is unchanged. On the J700 (T8140) 1080p NV12 streams run at 30.01, 25.00, 24.01, 20.01 and 15.00 fps for the five intervals, a 1001/30000 s request selects 30 fps, and VIDIOC_S_PARM returns -EBUSY during a stream. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-cam.c | 8 ++- drivers/media/platform/apple/isp/isp-cam.h | 9 ++- drivers/media/platform/apple/isp/isp-drv.h | 2 + drivers/media/platform/apple/isp/isp-v4l2.c | 70 ++++++++++++++++++--- 4 files changed, 76 insertions(+), 13 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 6dc262912d3e5..834faf0c57b3e 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -381,11 +381,15 @@ static int isp_ch_configure_frame_rate(struct apple_isp *isp, u32 ch) { int err; - err = isp_cmd_ch_ae_frame_rate_max_set(isp, ch, ISP_FRAME_RATE_DEN); + err = isp_cmd_ch_ae_frame_rate_max_set(isp, ch, + isp->frame_rate * + ISP_FRAME_RATE_SCALE); if (err) return err; - return isp_cmd_ch_ae_frame_rate_min_set(isp, ch, ISP_FRAME_RATE_DEN2); + return isp_cmd_ch_ae_frame_rate_min_set(isp, ch, + ISP_FRAME_RATE_MIN * + ISP_FRAME_RATE_SCALE); } static int isp_ch_configure_pools(struct apple_isp *isp, u32 ch) diff --git a/drivers/media/platform/apple/isp/isp-cam.h b/drivers/media/platform/apple/isp/isp-cam.h index 97b3940139663..51c2f2ec6c079 100644 --- a/drivers/media/platform/apple/isp/isp-cam.h +++ b/drivers/media/platform/apple/isp/isp-cam.h @@ -6,9 +6,12 @@ #include "isp-drv.h" -#define ISP_FRAME_RATE_NUM 256 -#define ISP_FRAME_RATE_DEN 7680 -#define ISP_FRAME_RATE_DEN2 3840 +/* The firmware takes frame rates in units of 1/256 frame per second. */ +#define ISP_FRAME_RATE_SCALE 256 +/* The slowest rate auto exposure may fall to, in frames per second */ +#define ISP_FRAME_RATE_MIN 15 +/* The nominal capture rate, in frames per second */ +#define ISP_FRAME_RATE_DEFAULT 30 int apple_isp_detect_camera(struct apple_isp *isp); diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 10dc441a030d1..96ab4a8a24343 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -277,6 +277,8 @@ struct apple_isp { struct mutex video_lock; unsigned int sequence; bool multiplanar; + /* capture rate in frames per second, changes under video_lock */ + unsigned int frame_rate; int pd_count; struct device **pd_dev; diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index ec3189f912b4c..d390e14a683dc 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright 2023 Eileen Yoon */ +#include #include #include @@ -22,6 +23,9 @@ #define ISP_BUFFER_TIMEOUT msecs_to_jiffies(1500) #define ISP_STRIDE_ALIGNMENT 64 +/* Capture rates offered to userspace, in frames per second */ +static const unsigned int isp_frame_rates[] = { 30, 25, 24, 20, 15 }; + static bool multiplanar = false; module_param(multiplanar, bool, 0644); MODULE_PARM_DESC(multiplanar, "Enable multiplanar API"); @@ -641,18 +645,56 @@ static int isp_vidioc_enum_framesizes(struct file *file, void *fh, return 0; } -static int isp_vidioc_enum_frameintervals(struct file *filp, void *priv, +static int isp_vidioc_enum_frameintervals(struct file *file, void *fh, struct v4l2_frmivalenum *interval) { - if (interval->index != 0) + struct apple_isp *isp = video_drvdata(file); + int i; + + if (interval->index >= ARRAY_SIZE(isp_frame_rates)) + return -EINVAL; + + if (interval->pixel_format != V4L2_PIX_FMT_NV12 && + interval->pixel_format != V4L2_PIX_FMT_NV12M) + return -EINVAL; + + for (i = 0; i < isp->num_presets; i++) { + if (isp->presets[i].output_dim.x == interval->width && + isp->presets[i].output_dim.y == interval->height) + break; + } + if (i == isp->num_presets) return -EINVAL; interval->type = V4L2_FRMIVAL_TYPE_DISCRETE; interval->discrete.numerator = 1; - interval->discrete.denominator = 30; + interval->discrete.denominator = isp_frame_rates[interval->index]; return 0; } +/* Returns the supported rate whose frame interval is closest to @tpf. */ +static unsigned int isp_closest_frame_rate(const struct v4l2_fract *tpf) +{ + unsigned int best = isp_frame_rates[0]; + + /* + * |n/d - 1/r| = |n * r - d| / (d * r), so the closest rate has the + * smallest |n * r - d| / r; compare those by cross-multiplying. + */ + for (int i = 1; i < ARRAY_SIZE(isp_frame_rates); i++) { + unsigned int rate = isp_frame_rates[i]; + u64 err = abs_diff((u64)tpf->numerator * rate, + (u64)tpf->denominator); + u64 best_err = abs_diff((u64)tpf->numerator * best, + (u64)tpf->denominator); + + if (err * best < best_err * rate) + best = rate; + } + + return best; +} + static inline void isp_get_sp_pix_format(struct apple_isp *isp, struct v4l2_format *f, struct isp_format *fmt) @@ -836,8 +878,8 @@ static int isp_vidioc_get_param(struct file *file, void *fh, a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; a->parm.capture.readbuffers = ISP_MIN_FRAMES; - a->parm.capture.timeperframe.numerator = ISP_FRAME_RATE_NUM; - a->parm.capture.timeperframe.denominator = ISP_FRAME_RATE_DEN; + a->parm.capture.timeperframe.numerator = 1; + a->parm.capture.timeperframe.denominator = isp->frame_rate; return 0; } @@ -846,17 +888,27 @@ static int isp_vidioc_set_param(struct file *file, void *fh, struct v4l2_streamparm *a) { struct apple_isp *isp = video_drvdata(file); + struct v4l2_fract *tpf = &a->parm.capture.timeperframe; if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE && (!isp->multiplanar || a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) return -EINVAL; - /* Not supporting frame rate sets. No use. Plus floats. */ + /* The rate is configured when the stream starts. */ + if (vb2_is_streaming(&isp->vbq)) + return -EBUSY; + + /* A zero interval selects the nominal rate. */ + if (!tpf->numerator || !tpf->denominator) + isp->frame_rate = ISP_FRAME_RATE_DEFAULT; + else + isp->frame_rate = isp_closest_frame_rate(tpf); + a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; a->parm.capture.readbuffers = ISP_MIN_FRAMES; - a->parm.capture.timeperframe.numerator = ISP_FRAME_RATE_NUM; - a->parm.capture.timeperframe.denominator = ISP_FRAME_RATE_DEN; + tpf->numerator = 1; + tpf->denominator = isp->frame_rate; return 0; } @@ -933,6 +985,8 @@ int apple_isp_setup_video(struct apple_isp *isp) return err; } + isp->frame_rate = ISP_FRAME_RATE_DEFAULT; + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { isp->meta_surfs[i] = isp_alloc_surface_vmap(isp, isp->hw->meta_size); From 3f2aa7f390f8cb3c1846fc52b9d4b6337d7e8c9f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 11:48:31 +1100 Subject: [PATCH 0741/1579] media: apple: isp: Hold the frame rate in low light by default Auto exposure may lower the frame rate to 15 fps to lengthen the exposure. On the J700 (T8140) a 1080p stream runs at 30 fps in a lit room but falls to between 23.5 and 15 fps in the dark, while V4L2 still reports a 30 fps interval. V4L2 has a control for this choice. V4L2_CID_EXPOSURE_AUTO_PRIORITY, which UVC webcams also expose, says whether auto exposure may vary the frame rate, and it is off by default so that the rate stays constant. Add it, and set the auto-exposure frame-rate floor to the selected rate unless it is on, in which case the floor stays at 15 fps. The firmware takes the frame-rate limits before the stream starts, so the control is locked while streaming. This changes the default low-light behaviour on every supported ISP: auto exposure now raises the gain instead of slowing down, which keeps motion smooth at the cost of more noise. Setting the control to 1 restores the previous behaviour. On the J700 the control reads 0 by default, is reported as grabbed with VIDIOC_S_EXT_CTRLS returning -EBUSY during a stream, and 1080p streams run at 30.0 fps with it at 0 or 1 in a 60 lux room. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-cam.c | 17 ++++++---- drivers/media/platform/apple/isp/isp-drv.h | 3 ++ drivers/media/platform/apple/isp/isp-v4l2.c | 35 +++++++++++++++++++-- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 834faf0c57b3e..5090ed2ed5b98 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -379,17 +379,22 @@ static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) static int isp_ch_configure_frame_rate(struct apple_isp *isp, u32 ch) { + u32 rate = isp->frame_rate * ISP_FRAME_RATE_SCALE; + u32 min_rate = rate; int err; - err = isp_cmd_ch_ae_frame_rate_max_set(isp, ch, - isp->frame_rate * - ISP_FRAME_RATE_SCALE); + /* + * Auto exposure lengthens the frame time in low light only if it may + * vary the frame rate; otherwise it holds the rate and raises the gain. + */ + if (v4l2_ctrl_g_ctrl(isp->exposure_priority)) + min_rate = ISP_FRAME_RATE_MIN * ISP_FRAME_RATE_SCALE; + + err = isp_cmd_ch_ae_frame_rate_max_set(isp, ch, rate); if (err) return err; - return isp_cmd_ch_ae_frame_rate_min_set(isp, ch, - ISP_FRAME_RATE_MIN * - ISP_FRAME_RATE_SCALE); + return isp_cmd_ch_ae_frame_rate_min_set(isp, ch, min_rate); } static int isp_ch_configure_pools(struct apple_isp *isp, u32 ch) diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 96ab4a8a24343..ac93aba409840 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -273,6 +274,8 @@ struct apple_isp { struct video_device vdev; struct media_device mdev; struct v4l2_device v4l2_dev; + struct v4l2_ctrl_handler ctrl_handler; + struct v4l2_ctrl *exposure_priority; struct vb2_queue vbq; struct mutex video_lock; unsigned int sequence; diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index d390e14a683dc..299177b9075cf 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -6,6 +6,8 @@ #include #include +#include +#include #include #include #include @@ -469,9 +471,14 @@ static int isp_vb2_start_streaming(struct vb2_queue *q, unsigned int count) isp->sequence = 0; + /* The exposure priority is applied when the stream starts. */ + v4l2_ctrl_grab(isp->exposure_priority, true); + err = apple_isp_start_streaming(isp); - if (err) + if (err) { + v4l2_ctrl_grab(isp->exposure_priority, false); isp_vb2_release_buffers(isp, VB2_BUF_STATE_QUEUED); + } return err; } @@ -482,6 +489,7 @@ static void isp_vb2_stop_streaming(struct vb2_queue *q) apple_isp_stop_streaming(isp); isp_vb2_release_buffers(isp, VB2_BUF_STATE_ERROR); + v4l2_ctrl_grab(isp->exposure_priority, false); } int apple_isp_video_suspend(struct apple_isp *isp) @@ -941,6 +949,10 @@ static const struct v4l2_ioctl_ops isp_v4l2_ioctl_ops = { .vidioc_prepare_buf = vb2_ioctl_prepare_buf, .vidioc_streamon = vb2_ioctl_streamon, .vidioc_streamoff = vb2_ioctl_streamoff, + + .vidioc_log_status = v4l2_ctrl_log_status, + .vidioc_subscribe_event = v4l2_ctrl_subscribe_event, + .vidioc_unsubscribe_event = v4l2_event_unsubscribe, }; static const struct v4l2_file_operations isp_v4l2_fops = { @@ -1023,10 +1035,25 @@ int apple_isp_setup_video(struct apple_isp *isp) isp->multiplanar = multiplanar; + /* + * Auto exposure is always on. By default it holds the frame rate, as + * V4L2 specifies; with exposure priority it may slow down in low light. + */ + v4l2_ctrl_handler_init(&isp->ctrl_handler, 1); + isp->exposure_priority = + v4l2_ctrl_new_std(&isp->ctrl_handler, NULL, + V4L2_CID_EXPOSURE_AUTO_PRIORITY, 0, 1, 1, 0); + if (isp->ctrl_handler.error) { + err = isp->ctrl_handler.error; + dev_err(isp->dev, "failed to create controls: %d\n", err); + goto ctrl_free; + } + isp->v4l2_dev.ctrl_handler = &isp->ctrl_handler; + err = v4l2_device_register(isp->dev, &isp->v4l2_dev); if (err) { dev_err(isp->dev, "failed to register v4l2 device: %d\n", err); - goto media_unregister; + goto ctrl_free; } vbq->drv_priv = isp; @@ -1070,7 +1097,8 @@ int apple_isp_setup_video(struct apple_isp *isp) v4l2_unregister: v4l2_device_unregister(&isp->v4l2_dev); -media_unregister: +ctrl_free: + v4l2_ctrl_handler_free(&isp->ctrl_handler); media_device_unregister(&isp->mdev); media_cleanup: media_device_cleanup(&isp->mdev); @@ -1084,6 +1112,7 @@ void apple_isp_remove_video(struct apple_isp *isp) { vb2_video_unregister_device(&isp->vdev); v4l2_device_unregister(&isp->v4l2_dev); + v4l2_ctrl_handler_free(&isp->ctrl_handler); media_device_unregister(&isp->mdev); media_device_cleanup(&isp->mdev); } From 9e0b9d41836ee1edcacda6876cb17207e9e2af7f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 11:12:26 +1100 Subject: [PATCH 0742/1579] cpuidle: apple: Explain why T8140 has no CPU PD state Commit 0a1ed8ce6d3c ("cpuidle: apple: Restrict T8140 to WFI until CPU PD is fixed") hid the CPU PD state on T8140 after an opt-in hung a J700, with the cause unknown. The cause is now known, and there is nothing in the sequence to fix, so say why T8140 keeps only WFI. CPU PD requests core power-down by setting bits 25:24 of CYC_OVRD (s3_5_c15_c5_0) before WFI. On a J700 that register reads as zero on both E and P cores, and an MSR to it, even of the value just read, takes an undefined-instruction exception (ESR 0x02000000). In the idle path that exception hits the idle task after cpu_pm_enter(), and the oops becomes a panic. The register is not needed there. Sampling the PMGR power-state registers of the six cores at 1 kHz on an idle J700, the E cores and the second P core read as power-gated in 94% to 99.9% of samples after plain WFI. The boot P core never did, with any idle state tried, and it is not its interrupt load: the second P core stays power-gated in 98.7% of samples while it is woken about 1500 times a second. A CPU PD state without the CYC_OVRD write was also tried: the same WFI between cpu_pm_enter() and cpu_pm_exit(). It ran a 31-minute load and idle soak on all six CPUs without errors, and the governor put 21% to 98% of each CPU's time in it. On the same boot, three paired two-minute idle windows each, against WFI alone: AC input power, display on 7.173 W vs 7.178 W P-cluster rail, median 1.259 W vs 1.259 W hottest P-cluster zone 65.1 C vs 65.0 C It would add CPU PM notifier work and save nothing, so it is not added. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpuidle/cpuidle-apple.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c index 6c0e622a0975f..6c02bd4197c26 100644 --- a/drivers/cpuidle/cpuidle-apple.c +++ b/drivers/cpuidle/cpuidle-apple.c @@ -124,7 +124,11 @@ static struct cpuidle_driver apple_idle_driver = { static int apple_cpuidle_probe(struct platform_device *pdev) { - /* A T8140 CPU PD opt-in hung J700; expose only WFI. */ + /* + * CPU PD writes CYC_OVRD, which is an undefined instruction on + * T8140. Its WFI already lets the hardware power-gate idle + * secondary cores, so expose only the WFI state. + */ if (of_machine_is_compatible("apple,t8140")) apple_idle_driver.state_count = STATE_WFI + 1; From 163c8d784d4ca354f863260905c18673c43f576c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 08:38:36 -0400 Subject: [PATCH 0743/1579] mm: reuse backing folios for small-granule anonymous COW Closing a subpage pool after each COW fault leaves every remaining slot unavailable for later faults. On a 16K kernel with a 4K process ABI, a 64MiB densely modified fork child therefore uses 256MiB of backing. Retain bounded, address-group pools for eligible faults in each mm. Keep charge domains and memory policy explicit, preserve independent fork ownership, and close pools after exit_mmap while referenced owners remain alive. Copy into a reserved slot under the folio lock, decide first rmap/LRU publication after relocking, and fall back to private backing if nonlinear rmap metadata cannot be allocated. Distribute concurrent faults across 16 address-group pools, skip busy backing on the normal path, and use exact per-offset free counts to avoid repeatedly scanning incompatible partial owners. Speculative allocation and charge do not invoke OOM; under pressure, retry reusable backing before ordinary allocation. Existing policy-sensitive, locked, droppable and multi-node faults keep private backing. Add a user4k selftest covering concurrent, sparse, phased and random COW, mlock and lazy-free transitions, and a descendant that retains data after the pool-owning mm exits. Each case checks untouched bytes and parent isolation. Physical density and reclaim were tested separately on Neo hardware; the selftest does not infer them from logical RSS. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5469651a0899f3bf776c90f4a24e25e952fa0ba2) --- include/linux/mm_subpage.h | 15 + include/linux/mm_types.h | 2 + kernel/fork.c | 3 + mm/memory.c | 152 +++++++-- mm/user-subpage.c | 297 +++++++++++++++++- tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../testing/selftests/arm64/user4k/cow-pool.c | 256 +++++++++++++++ 7 files changed, 690 insertions(+), 39 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/cow-pool.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index 13344e3a92384..c4ac2eb6c4236 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -45,6 +45,21 @@ struct mm_subpage_pool *mm_subpage_pool_create_granule(gfp_t gfp, unsigned int s void mm_subpage_pool_get(struct mm_subpage_pool *pool); void mm_subpage_pool_put(struct mm_subpage_pool *pool); void mm_subpage_pool_close(struct mm_subpage_pool *pool); +#ifdef CONFIG_MM_SUBPAGE +/* Optional per-mm pool: default policy, one memory node, matching memcg. */ +struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long address); +void mm_subpage_cow_pool_exit(struct mm_struct *mm); +bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio); +bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset); +/* Return an initialized slot with its folio locked; may wait without a pool lock. */ +struct mm_subpage *mm_subpage_alloc_at_locked(struct mm_subpage_pool *pool, + unsigned int offset); +struct mm_subpage *mm_subpage_copy_at_locked(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset); +#else +static inline void mm_subpage_cow_pool_exit(struct mm_struct *mm) { } +#endif /* Success consumes the folio reference. On error the caller retains it. */ int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 7f201dda7a13b..46d6f13f6f303 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1303,6 +1303,8 @@ struct mm_struct { #ifdef CONFIG_MM_SUBPAGE /* VM_ACCOUNT granules; global commitment rounds this sum once. */ unsigned long committed_user_pages; + /* Shared COW slots; never inherited by a forked mm. */ + struct mm_subpage_cow_pools *cow_subpage_pool; #endif unsigned long locked_vm; /* Locked bytes >> MM_ACCOUNT_SHIFT */ atomic64_t pinned_vm; /* Pinned native physical pages */ diff --git a/kernel/fork.c b/kernel/fork.c index 88788baaff964..a9fedd8b8abac 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -1097,6 +1098,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, mm->map_count = 0; #ifdef CONFIG_MM_SUBPAGE mm->committed_user_pages = 0; + mm->cow_subpage_pool = NULL; #endif mm->locked_vm = 0; atomic64_set(&mm->pinned_vm, 0); @@ -1212,6 +1214,7 @@ static inline void __mmput(struct mm_struct *mm) ksm_exit(mm); khugepaged_exit(mm); /* must run before exit_mmap */ exit_mmap(mm); + mm_subpage_cow_pool_exit(mm); mm_put_huge_zero_folio(mm); set_mm_exe_file(mm, NULL); if (!list_empty(&mm->mmlist)) { diff --git a/mm/memory.c b/mm/memory.c index ade3dbadb409a..2fb430e842603 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -88,6 +88,7 @@ #include "pgalloc-track.h" #include +#include "user-subpage-internal.h" #include "internal.h" #include "swap.h" @@ -1325,6 +1326,98 @@ static inline struct folio *folio_prealloc(struct mm_struct *src_mm, } #ifdef CONFIG_MM_SUBPAGE +/* Skipping a busy owner must not trigger an avoidable global or memcg OOM. */ +static struct folio *cow_subpage_speculative_folio(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr) +{ + gfp_t alloc_gfp = GFP_HIGHUSER_MOVABLE | __GFP_NORETRY | __GFP_NOWARN; + gfp_t charge_gfp = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN; + unsigned int order = max_t(unsigned int, mm_page_shift(mm), PAGE_SHIFT) - PAGE_SHIFT; + struct folio *folio = vma_alloc_folio(alloc_gfp, order, vma, addr); + + if (!folio) + return NULL; + if (mem_cgroup_charge(folio, mm, charge_gfp)) { + folio_put(folio); + return NULL; + } + folio_throttle_swaprate(folio, charge_gfp); + return folio; +} + +/* The returned initialized slot owns a reference and has its folio locked. */ +static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr, + struct mm_subpage *source, bool force_private, bool *pooled) +{ + struct mm_subpage_pool *pool; + struct mm_subpage *slot; + struct folio *folio; + unsigned int offset = vma_page_offset_at(vma, addr).offset; + int err; + +retry_private: + pool = force_private ? NULL : mm_subpage_cow_pool_get(mm, vma, addr); + *pooled = pool != NULL; + if (!pool) + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return ERR_PTR(-ENOMEM); + for (;;) { + slot = source ? mm_subpage_copy_at_locked(pool, source, offset) : + mm_subpage_alloc_at_locked(pool, offset); + if (!IS_ERR(slot) || PTR_ERR(slot) != -EAGAIN) + break; + folio = NULL; + if (*pooled) { + folio = cow_subpage_speculative_folio(mm, vma, addr); + if (!folio && mm_subpage_cow_wait_busy_at(pool, offset)) { + cond_resched(); + continue; + } + } + /* No reusable owner exists after speculative allocation failed. + * Ordinary allocation/charge semantics are now appropriate. + */ + if (!folio) + folio = folio_prealloc(mm, vma, addr, false); + if (!folio) { + slot = ERR_PTR(-ENOMEM); + break; + } + /* A concurrent cgroup move may change the fresh folio's charge. */ + if (*pooled && !mm_subpage_cow_folio_matches(pool, folio)) { + folio_put(folio); + mm_subpage_pool_put(pool); + force_private = true; + goto retry_private; + } + err = mm_subpage_pool_add_folio_at(pool, folio, offset, + *pooled ? GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN : GFP_KERNEL); + if (*pooled && err == -ENOMEM) { + if (mm_subpage_cow_wait_busy_at(pool, offset)) { + folio_put(folio); + cond_resched(); + continue; + } + err = mm_subpage_pool_add_folio_at(pool, folio, offset, GFP_KERNEL); + } + if (err) { + folio_put(folio); + if (err == -EEXIST) { + cond_resched(); + continue; + } + slot = ERR_PTR(err); + break; + } + } + if (!*pooled) + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + return slot; +} + static struct mm_subpage *anon_subpage_prealloc(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr) { @@ -4541,11 +4634,11 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; struct mm_struct *mm = vma->vm_mm; struct mm_subpage *source = NULL, *dest = NULL; - struct mm_subpage_pool *pool = NULL; struct folio *old = vmf->page ? page_folio(vmf->page) : NULL; struct folio *new = NULL; struct mmu_notifier_range range; bool copied = false, unshare = vmf->flags & FAULT_FLAG_UNSHARE; + bool force_private = false, pooled, fresh, retry_private = false; unsigned int offset; vm_fault_t ret = VM_FAULT_SIGBUS; pte_t entry; @@ -4598,35 +4691,29 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) goto out; offset = vma_page_offset_at(vma, vmf->address).offset; ret = VM_FAULT_OOM; - pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); - if (!pool) - goto out; - new = folio_prealloc(mm, vma, vmf->address, false); - if (!new) - goto out; - err = mm_subpage_pool_add_folio(pool, new, GFP_KERNEL); - if (err) { - folio_put(new); - goto out; - } - dest = source ? mm_subpage_copy_at(pool, source, offset) : - mm_subpage_alloc_at(pool, offset); +retry_copy: + dest = cow_subpage_prealloc(mm, vma, vmf->address, source, force_private, &pooled); if (IS_ERR(dest)) { if (PTR_ERR(dest) == -EHWPOISON) ret = VM_FAULT_HWPOISON; dest = NULL; goto out; } + new = mm_subpage_folio(dest); if (old && !source) { err = copy_user_subpage_range(&new->page, offset, vmf->page, pte_phys_mm(mm, vmf->orig_pte) & ~PAGE_MASK, mm_page_size(mm)); if (err) { + folio_unlock(new); ret = VM_FAULT_HWPOISON; goto out; } } - mm_subpage_pool_close(pool); + /* Initialisation is complete; the slot reference prevents migration/free. + * Do not carry a folio lock through MMU notifier callbacks. + */ + folio_unlock(new); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, vmf->address, vmf->address + mm_page_size(mm)); mmu_notifier_invalidate_range_start(&range); @@ -4634,6 +4721,19 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) folio_get(new); folio_lock(new); ret = 0; + /* A different fault may publish this owner's first PTE while we wait. + * Only the first publisher establishes folio rmap/LRU. Preflight nonlinear + * slot identity before taking PTL, with a fresh-folio fallback on ENOMEM. + */ + fresh = !mm_subpage_anon_root(dest); + if (!fresh) { + err = mm_subpage_bind_rmap(dest, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + retry_private = pooled && err == -ENOMEM; + goto unlock_folio; + } + } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) goto unlock_folio; @@ -4642,7 +4742,8 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) goto unlock_pte; } mm_subpage_set_exclusive(dest); - err = mm_subpage_add_new_anon_rmap(dest, vma, vmf->address); + err = fresh ? mm_subpage_add_new_anon_rmap(dest, vma, vmf->address) : + mm_subpage_add_anon_rmap(dest, vma, vmf->address); if (WARN_ON_ONCE(err)) { ret = VM_FAULT_SIGBUS; goto unlock_pte; @@ -4660,7 +4761,8 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); /* Break-before-make also orders removal of the old slot's rmap. */ ptep_clear_flush(vma, vmf->address, vmf->pte); - subpage_add_new_lru(new, vma, 1); + if (fresh) + subpage_add_new_lru(new, vma, 1); set_pte_at(mm, vmf->address, vmf->pte, entry); update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); if (source) { @@ -4683,13 +4785,17 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) folio_unlock(new); folio_put(new); mmu_notifier_invalidate_range_end(&range); -out: - if (pool) { - mm_subpage_pool_close(pool); - if (dest) - mm_subpage_put(dest); - mm_subpage_pool_put(pool); + if (retry_private) { + mm_subpage_put(dest); + dest = NULL; + force_private = true; + retry_private = false; + ret = VM_FAULT_OOM; + goto retry_copy; } +out: + if (dest) + mm_subpage_put(dest); if (source) { if (copied) mm_subpage_put(source); /* Retired PTE reference, after TLBI. */ diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 319d792ab5e9a..3adfb04ccdca1 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -222,6 +224,17 @@ struct mm_subpage_pool { unsigned long all_slots; u8 shift; bool closed; + /* Only a per-mm COW pool owns this charge-domain reference. */ + struct mem_cgroup *cow_memcg; + int cow_nid; + bool cow_shared; + /* Allocated only for shared COW pools; protected by pool->lock. */ + unsigned long *cow_free_slots; +}; + +#define COW_POOL_BITS 4 +struct mm_subpage_cow_pools { + struct mm_subpage_pool *pool[1U << COW_POOL_BITS]; }; static void owner_init(struct mm_subpage_owner *owner, struct mm_subpage_pool *pool, @@ -277,13 +290,147 @@ void mm_subpage_pool_put(struct mm_subpage_pool *pool) if (!refcount_dec_and_test(&pool->refs)) return; WARN_ON_ONCE(!pool->closed || !list_empty(&pool->owners)); + mem_cgroup_put(pool->cow_memcg); + kfree(pool->cow_free_slots); kfree_rcu(pool, rcu); } +/* Keep the first implementation within one unambiguous allocation domain. + * Policy-sensitive, locked, droppable and multi-node faults keep fresh backing. + * An mm can move cgroups; old charged slots must not satisfy its new faults. + */ +struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long address) +{ + struct mm_subpage_cow_pools *pools, *new_pools; + struct mm_subpage_pool *pool, *new; + unsigned int bucket = hash_long(address >> PAGE_SHIFT, COW_POOL_BITS); + struct mem_cgroup *memcg; + bool matches; + int nid; + + if ((vma->vm_flags & (VM_LOCKED | VM_DROPPABLE | VM_SPECIAL)) || + num_online_nodes() != 1 || num_node_state(N_MEMORY) != 1) + return NULL; +#ifdef CONFIG_NUMA + if (vma_policy(vma) || current->mempolicy) + return NULL; +#endif + nid = first_node(node_states[N_MEMORY]); + if (!node_isset(nid, cpuset_current_mems_allowed)) + return NULL; + /* Publication pairs with cmpxchg: every bucket starts empty. Adjacent + * 4K leaves in one native virtual group always choose the same pool. + * Faulting threads can migrate CPUs without changing that identity. + */ + pools = smp_load_acquire(&mm->cow_subpage_pool); + if (!pools) { + new_pools = kzalloc(sizeof(*new_pools), GFP_KERNEL | __GFP_ACCOUNT); + if (!new_pools) + return NULL; + pools = cmpxchg(&mm->cow_subpage_pool, NULL, new_pools); + if (!pools) + pools = new_pools; + else + kfree(new_pools); + } + memcg = get_mem_cgroup_from_mm(mm); + /* Acquire the immutable charge domain published with this bucket. */ + pool = smp_load_acquire(&pools->pool[bucket]); + if (!pool) { + new = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!new) { + mem_cgroup_put(memcg); + return NULL; + } + new->cow_free_slots = kcalloc(SUBPAGES_PER_FOLIO, + sizeof(*new->cow_free_slots), + GFP_KERNEL | __GFP_ACCOUNT); + if (!new->cow_free_slots) { + mm_subpage_pool_close(new); + mm_subpage_pool_put(new); + mem_cgroup_put(memcg); + return NULL; + } + new->cow_memcg = memcg; + new->cow_nid = nid; + new->cow_shared = true; + pool = cmpxchg(&pools->pool[bucket], NULL, new); + if (!pool) { + pool = new; + /* The mm owns the initial pool reference and its memcg ref. */ + mm_subpage_pool_get(pool); + return pool; + } + matches = pool->cow_memcg == memcg && pool->cow_nid == nid; + mm_subpage_pool_close(new); + mm_subpage_pool_put(new); + } else { + matches = pool->cow_memcg == memcg && pool->cow_nid == nid; + mem_cgroup_put(memcg); + } + if (!matches) + return NULL; + mm_subpage_pool_get(pool); + return pool; +} + +bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio) +{ + return pool->cow_memcg == folio_memcg(folio) && pool->cow_nid == folio_nid(folio); +} + +void mm_subpage_cow_pool_exit(struct mm_struct *mm) +{ + struct mm_subpage_cow_pools *pools = xchg(&mm->cow_subpage_pool, NULL); + unsigned int i; + + /* No faults remain once mm_users reaches zero. Slot owners can outlive mm. */ + if (!pools) + return; + for (i = 0; i < ARRAY_SIZE(pools->pool); i++) { + if (!pools->pool[i]) + continue; + mm_subpage_pool_close(pools->pool[i]); + mm_subpage_pool_put(pools->pool[i]); + } + kfree(pools); +} + +/* Counts include unavailable (busy/migrating) owners, so zero is conclusive. + * A nonzero count still requires all the normal owner eligibility checks. + */ +static void cow_update_free(struct mm_subpage_pool *pool, unsigned long mask, int delta) +{ + unsigned int i; + + lockdep_assert_held(&pool->lock); + if (!pool->cow_shared) + return; + for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { + VM_BUG_ON(delta < 0 && !pool->cow_free_slots[i]); + pool->cow_free_slots[i] += delta; + } +} + +static bool cow_has_free(struct mm_subpage_pool *pool, unsigned long eligible) +{ + unsigned int i; + + lockdep_assert_held(&pool->lock); + if (!pool->cow_shared) + return true; + for_each_set_bit(i, &eligible, SUBPAGES_PER_FOLIO) + if (pool->cow_free_slots[i]) + return true; + return false; +} + static void owner_remove(struct mm_subpage_owner *owner) { lockdep_assert_held(&owner->pool->lock); owner->detached = true; + cow_update_free(owner->pool, owner->pool->all_slots & ~owner->used, -1); list_del(&owner->all); list_del_init(&owner->available); if (owner->pool->unissued == owner) @@ -677,17 +824,81 @@ void mm_subpage_pool_close(struct mm_subpage_pool *pool) owner_free(owner, true); } +static bool cow_owner_eligible(struct mm_subpage_pool *pool, struct folio *folio) +{ + if (!pool->cow_shared) + return true; + /* A previously unlocked owner can be mlocked or fully MADV_FREE'd + * after entering the pool. Never place a new live slot in such backing. + */ + return !folio_test_mlocked(folio) && !folio_test_unevictable(folio) && + !folio_test_ksm(folio) && + (!folio_test_anon(folio) || folio_test_swapbacked(folio)) && + mm_subpage_cow_folio_matches(pool, folio); +} + +/* Pressure-only fallback after a non-OOM speculative allocation failed. + * Return true when existing compatible backing can be retried. Unlike the + * normal bounded search, inspect all owners before allowing an OOM-capable + * allocation: a busy slot farther down the list may require no new charge. + */ +bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + struct mm_subpage_owner *owner; + struct folio *wait = NULL; + unsigned long flags, eligible = offset_mask(pool->shift, offset); + + spin_lock_irqsave(&pool->lock, flags); + if (!pool->closed && pool->cow_shared && cow_has_free(pool, eligible)) { + list_for_each_entry(owner, &pool->available, available) { + if (!owner->migrating && (~owner->used & eligible) && + cow_owner_eligible(pool, owner->folio) && + !folio_test_swapcache(owner->folio) && + !folio_test_writeback(owner->folio)) { + wait = owner->folio; + folio_get(wait); + break; + } + } + } + spin_unlock_irqrestore(&pool->lock, flags); + if (wait) { + folio_wait_locked(wait); + folio_put(wait); + return true; + } + return false; +} + static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, unsigned long eligible) { struct mm_subpage_owner *owner; + unsigned int visited = 0; lockdep_assert_held(&pool->lock); - list_for_each_entry(owner, &pool->available, available) + if (!cow_has_free(pool, eligible)) + return NULL; + list_for_each_entry(owner, &pool->available, available) { + /* A fixed-offset stream must not scan every partially used owner + * on each fault. New COW supply is inserted first, so a bounded + * miss can always make progress using a fresh folio. + */ + if (pool->cow_shared && visited++ == 32) { + /* The next search starts past this incompatible prefix. A + * phased offset stream must not strand older reusable slots. + */ + list_rotate_to_front(&owner->available, &pool->available); + break; + } /* A nonpresent PTE may still need any of this folio's slot bytes. */ if (!owner->migrating && (~owner->used & eligible) && - !folio_test_swapcache(owner->folio)) + (!pool->cow_shared || !folio_test_locked(owner->folio)) && + cow_owner_eligible(pool, owner->folio) && + !folio_test_swapcache(owner->folio) && + !folio_test_writeback(owner->folio)) return owner; + } return NULL; } @@ -720,13 +931,20 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, spin_lock_irqsave(&pool->lock, flags); if (pool->closed) ret = -ESHUTDOWN; - else if (find_available(pool, eligible)) - /* Another allocator supplied backing while we allocated metadata. */ + else if (pool->unissued || find_available(pool, eligible)) + /* Another allocator supplied backing while we allocated metadata. + * Preserve the single-unissued-owner invariant even when a shared + * COW lookup skips a concurrently locked or rotating list prefix. + */ ret = -EEXIST; else { mm_subpage_pool_get(pool); list_add_tail(&owner->all, &pool->owners); - list_add_tail(&owner->available, &pool->available); + cow_update_free(pool, pool->all_slots, 1); + if (pool->cow_shared) + list_add(&owner->available, &pool->available); + else + list_add_tail(&owner->available, &pool->available); pool->backing_pages++; pool->unissued = owner; } @@ -765,13 +983,14 @@ unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) } static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, - unsigned long eligible, bool *needs_zero) + unsigned long eligible, bool *needs_zero, bool lock_folio) { struct mm_subpage_owner *owner; struct mm_subpage *subpage; unsigned long flags; unsigned int index; +retry: spin_lock_irqsave(&pool->lock, flags); if (pool->closed) { subpage = ERR_PTR(-ESHUTDOWN); @@ -782,6 +1001,34 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, subpage = ERR_PTR(-EAGAIN); goto out; } + if (lock_folio) { + if (!folio_trylock(owner->folio)) { + struct folio *wait; + + if (pool->cow_shared) { + /* Another COW fault won this folio after the lookup. + * Try other backing, or supply another folio, rather + * than making parallel faults queue on its lock. + */ + spin_unlock_irqrestore(&pool->lock, flags); + cond_resched(); + goto retry; + } + wait = owner->folio; + folio_get(wait); + spin_unlock_irqrestore(&pool->lock, flags); + folio_wait_locked(wait); + folio_put(wait); + goto retry; + } + /* Reclaim may have acquired the folio after find_available's check. */ + if (folio_test_swapcache(owner->folio) || folio_test_writeback(owner->folio) || + !cow_owner_eligible(pool, owner->folio)) { + folio_unlock(owner->folio); + spin_unlock_irqrestore(&pool->lock, flags); + return ERR_PTR(-EAGAIN); + } + } if (pool->unissued == owner) /* Transfer the supplied reference to the first reservation. */ pool->unissued = NULL; @@ -789,6 +1036,7 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, folio_get(owner->folio); index = __ffs(~owner->used & eligible); __set_bit(index, &owner->used); + cow_update_free(pool, BIT(index), -1); if (needs_zero) *needs_zero = !test_bit(index, &owner->zeroed); __clear_bit(index, &owner->zeroed); @@ -804,10 +1052,10 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, return subpage; } -static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible) +static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible, bool lock_folio) { bool needs_zero; - struct mm_subpage *subpage = subpage_reserve(pool, eligible, &needs_zero); + struct mm_subpage *subpage = subpage_reserve(pool, eligible, &needs_zero, lock_folio); if (IS_ERR(subpage)) return subpage; @@ -822,7 +1070,7 @@ static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned l struct mm_subpage *mm_subpage_alloc(struct mm_subpage_pool *pool) { - return subpage_alloc(pool, pool->all_slots); + return subpage_alloc(pool, pool->all_slots, false); } struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned int offset) @@ -831,7 +1079,7 @@ struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned in if (!eligible) return ERR_PTR(-EINVAL); - return subpage_alloc(pool, eligible); + return subpage_alloc(pool, eligible, false); } static void subpage_release_locked(struct mm_subpage *subpage, unsigned long flags) @@ -851,6 +1099,7 @@ static void subpage_release_locked(struct mm_subpage *subpage, unsigned long fla subpage->rmap_bound = false; } __clear_bit(subpage->index, &owner->used); + cow_update_free(pool, BIT(subpage->index), 1); release = !owner->used && !owner->migrating; if (release) owner_remove(owner); @@ -872,14 +1121,14 @@ static void subpage_release(struct mm_subpage *subpage) } static struct mm_subpage *subpage_copy(struct mm_subpage_pool *pool, - const struct mm_subpage *source, unsigned long eligible) + const struct mm_subpage *source, unsigned long eligible, bool lock_folio) { struct mm_subpage *dest; int err; if (pool->shift != mm_subpage_shift(source)) return ERR_PTR(-EINVAL); - dest = subpage_reserve(pool, eligible, NULL); + dest = subpage_reserve(pool, eligible, NULL, lock_folio); if (IS_ERR(dest)) return dest; @@ -888,6 +1137,8 @@ static struct mm_subpage *subpage_copy(struct mm_subpage_pool *pool, mm_subpage_size(source)); if (err) { /* Never publish a reservation whose copy did not finish. */ + if (lock_folio) + folio_unlock(dest->owner->folio); subpage_release(dest); return ERR_PTR(err); } @@ -898,7 +1149,7 @@ static struct mm_subpage *subpage_copy(struct mm_subpage_pool *pool, struct mm_subpage *mm_subpage_copy(struct mm_subpage_pool *pool, const struct mm_subpage *source) { - return subpage_copy(pool, source, pool->all_slots); + return subpage_copy(pool, source, pool->all_slots, false); } struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, @@ -908,7 +1159,23 @@ struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, if (!eligible) return ERR_PTR(-EINVAL); - return subpage_copy(pool, source, eligible); + return subpage_copy(pool, source, eligible, false); +} + +struct mm_subpage *mm_subpage_alloc_at_locked(struct mm_subpage_pool *pool, + unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + return eligible ? subpage_alloc(pool, eligible, true) : ERR_PTR(-EINVAL); +} + +struct mm_subpage *mm_subpage_copy_at_locked(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + return eligible ? subpage_copy(pool, source, eligible, true) : ERR_PTR(-EINVAL); } /* The page-extension RCU section also protects the pool until its lock is held. */ @@ -956,6 +1223,7 @@ static struct mm_subpage *restore_bound_slot(struct folio *folio, unsigned int o else folio_get(folio); __set_bit(index, &owner->used); + cow_update_free(pool, BIT(index), -1); __clear_bit(index, &owner->zeroed); if (owner->used == pool->all_slots) list_del_init(&owner->available); @@ -1117,6 +1385,7 @@ int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) mm_subpage_pool_get(pool); spin_lock_irqsave(&pool->lock, flags); list_add_tail(&new->all, &pool->owners); + cow_update_free(pool, pool->all_slots, 1); pool->backing_pages++; ret = owner_bind(new); if (ret) diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index cc89cc93293fe..4b7c4cf17006d 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -2,7 +2,7 @@ TEST_GEN_PROGS := page-contract proc-args TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init -TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init +TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init cow-pool CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe @@ -12,7 +12,7 @@ include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init: LDLIBS += -pthread +$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/cow-pool.c b/tools/testing/selftests/arm64/user4k/cow-pool.c new file mode 100644 index 0000000000000..bb25904d68c04 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/cow-pool.c @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Exercise shared COW backing under a selected 4K userspace ABI. Run through + * the platform's ABI launcher; this test never changes its own page contract. + * Every selected leaf changes one byte, and all other bytes must survive. + * Physical density and reclaim require privileged, platform-specific controls + * and are deliberately not inferred from this unprivileged contents test. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define BYTES (64UL << 20) +#define LEAF 4096UL +#define LEAVES (BYTES / LEAF) +#define WORKERS 8 + +enum mode { DENSE, SPARSE, PHASED, RANDOM, LOCKED, LATE_LOCKED, LAZYFREE }; +struct scenario { + const char *name; + enum mode mode; + unsigned int workers; +}; + +static unsigned char *data; +static size_t order[LEAVES]; +static pthread_barrier_t barrier; +static const struct scenario *current; + +static unsigned char initial(size_t leaf, size_t byte) +{ + return leaf * 17 + byte * 3 + 0x35; +} + +static unsigned char changed(size_t leaf) +{ + return initial(leaf, 0) ^ 0xa5; +} + +static bool verify(unsigned int generation, bool discard) +{ + unsigned int stride = current->mode == SPARSE ? 4 : 1; + size_t leaf, byte; + + for (leaf = 0; leaf < LEAVES; leaf++) { + if (generation && discard && leaf % 4 == 0) + continue; + for (byte = 0; byte < LEAF; byte++) { + unsigned char want = initial(leaf, byte); + + if (generation && byte == 0 && leaf % stride == 0) + want ^= generation == 1 ? 0xa5 : 0x5a; + if (data[leaf * LEAF + byte] != want) { + ksft_print_msg("pid=%d leaf=%zu byte=%zu got=%u want=%u\n", + getpid(), leaf, byte, data[leaf * LEAF + byte], want); + return false; + } + } + } + return true; +} + +static bool wait_ok(pid_t pid) +{ + pid_t result; + int status; + + do { + result = waitpid(pid, &status, 0); + } while (result < 0 && errno == EINTR); + return result > 0 && WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +static void *writer(void *arg) +{ + unsigned int id = (uintptr_t)arg; + size_t leaf, index; + unsigned int phase; + + pthread_barrier_wait(&barrier); + if (current->mode == PHASED || current->mode == LATE_LOCKED || + current->mode == LAZYFREE) { + for (phase = 0; phase < 4; phase++) { + for (leaf = phase + 4UL * id; leaf < LEAVES; + leaf += 4UL * current->workers) + data[leaf * LEAF] = changed(leaf); + pthread_barrier_wait(&barrier); + if (phase == 0 && id == 0) { + if (current->mode == LATE_LOCKED && mlock(data, BYTES)) + _exit(2); + if (current->mode == LAZYFREE) + for (leaf = 0; leaf < LEAVES; leaf += 4) + if (madvise(data + leaf * LEAF, LEAF, MADV_FREE)) + _exit(2); + } + pthread_barrier_wait(&barrier); + } + } else { + unsigned int stride = current->mode == SPARSE ? 4 : 1; + + for (index = id; index < LEAVES / stride; index += current->workers) { + leaf = current->mode == RANDOM ? order[index] : index * stride; + data[leaf * LEAF] = changed(leaf); + } + } + return NULL; +} + +static bool exercise(const struct scenario *scenario) +{ + pthread_t workers[WORKERS]; + struct timespec start, end; + unsigned int t; + pid_t child; + + current = scenario; + child = fork(); + if (child < 0) + return false; + if (!child) { + if (current->mode == LOCKED && mlock(data, BYTES)) + _exit(2); + if (pthread_barrier_init(&barrier, NULL, current->workers)) + _exit(2); + clock_gettime(CLOCK_MONOTONIC, &start); + for (t = 0; t < current->workers; t++) + if (pthread_create(&workers[t], NULL, writer, (void *)(uintptr_t)t)) + _exit(2); + for (t = 0; t < current->workers; t++) + if (pthread_join(workers[t], NULL)) + _exit(2); + clock_gettime(CLOCK_MONOTONIC, &end); + ksft_print_msg("%s fault/write phase %.6f s\n", current->name, + end.tv_sec - start.tv_sec + (end.tv_nsec - start.tv_nsec) * 1e-9); + _exit(verify(1, current->mode == LAZYFREE) ? 0 : 3); + } + return wait_ok(child) && verify(0, false); +} + +static bool owner_exit(void) +{ + static const struct scenario scenario = { "owner-mm exit", DENSE, 1 }; + int ready[2]; + pid_t child, descendant; + size_t leaf; + char signal; + + current = &scenario; + if (prctl(PR_SET_CHILD_SUBREAPER, 1) || pipe(ready)) + return false; + child = fork(); + if (child < 0) { + close(ready[0]); + close(ready[1]); + return false; + } + if (!child) { + close(ready[1]); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = changed(leaf); + if (!verify(1, false)) + _exit(3); + descendant = fork(); + if (descendant < 0) + _exit(2); + if (descendant) + _exit(0); + /* Root signals only after the original pool-owning mm has exited. */ + if (read(ready[0], &signal, 1) != 1 || signal != 'g') + _exit(2); + close(ready[0]); + if (!verify(1, false)) + _exit(3); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] ^= 0xff; + _exit(verify(2, false) ? 0 : 3); + } + close(ready[0]); + if (!wait_ok(child)) { + close(ready[1]); + return false; + } + if (write(ready[1], "g", 1) != 1) { + close(ready[1]); + return false; + } + close(ready[1]); + return wait_ok(-1) && verify(0, false); +} + +int main(void) +{ + static const struct scenario scenarios[] = { + { "dense single-thread COW", DENSE, 1 }, + { "dense concurrent COW", DENSE, WORKERS }, + { "single-offset sparse COW", SPARSE, 1 }, + { "phased-offset concurrent COW", PHASED, WORKERS }, + { "random-order concurrent COW", RANDOM, WORKERS }, + { "locked COW", LOCKED, WORKERS }, + { "mlock after first offset", LATE_LOCKED, WORKERS }, + { "MADV_FREE after first offset", LAZYFREE, WORKERS }, + }; + struct rlimit limit; + uint32_t seed = 0x9148140; + size_t leaf, byte, i, j, tmp; + bool can_lock; + + ksft_print_header(); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("run under a selected 4K userspace ABI\n"); + ksft_set_plan(9); + alarm(180); + data = mmap(NULL, BYTES, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (data == MAP_FAILED) + ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); + for (leaf = 0; leaf < LEAVES; leaf++) + for (byte = 0; byte < LEAF; byte++) + data[leaf * LEAF + byte] = initial(leaf, byte); + for (i = 0; i < LEAVES; i++) + order[i] = i; + for (i = LEAVES; i > 1; i--) { + seed = seed * 1664525 + 1013904223; + j = seed % i; + tmp = order[j]; + order[j] = order[i - 1]; + order[i - 1] = tmp; + } + can_lock = !getrlimit(RLIMIT_MEMLOCK, &limit) && limit.rlim_cur >= BYTES; + for (i = 0; i < sizeof(scenarios) / sizeof(scenarios[0]); i++) { + if (!can_lock && (scenarios[i].mode == LOCKED || + scenarios[i].mode == LATE_LOCKED)) { + ksft_test_result_skip("%s needs a 64MiB memlock limit\n", scenarios[i].name); + continue; + } + ksft_test_result(exercise(&scenarios[i]), "%s preserves contents and parent\n", + scenarios[i].name); + } + ksft_test_result(owner_exit(), "descendant retains COW data after owner mm exits\n"); + munmap(data, BYTES); + ksft_finished(); +} From 7bfcdc23ebd5224a109efa80afd896d0c9c3dc53 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 15:16:11 +1100 Subject: [PATCH 0744/1579] dt-bindings: soc: apple: Add the T8140 memory cache controller The memory cache controller of the Apple T8140 holds the system level cache in front of DRAM, split into planes. The boot firmware enables and powers the cache ways of every plane but leaves allocation disabled, so the operating system has to allow it. Describe one register block per plane, and the devices that take over DMA the bootloader left running: allocation must wait until they are bound. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/soc/apple/apple,t8140-mcc.yaml | 58 +++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 59 insertions(+) create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml new file mode 100644 index 0000000000000..e4c5688a01d7f --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,t8140-mcc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 memory cache controller + +maintainers: + - Ryan Murray + +description: + The memory cache controller of the Apple T8140 SoC holds the system level + cache in front of DRAM, split into planes. The boot firmware enables and + powers the cache ways of every plane but leaves allocation disabled, so the + cache holds no data until the operating system allows allocation. + +properties: + compatible: + const: apple,t8140-mcc + + reg: + description: The register block of each cache plane. + minItems: 1 + maxItems: 8 + + apple,boot-dma-devices: + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + maxItems: 1 + description: + Devices that take over DMA the bootloader left running. Cache allocation + is enabled only once each of them is bound to its driver. + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + display: display-subsystem { + }; + + memory-controller@220000000 { + compatible = "apple,t8140-mcc"; + reg = <0x2 0x20000000 0x0 0x4000>, + <0x2 0x20040000 0x0 0x4000>, + <0x2 0x20080000 0x0 0x4000>, + <0x2 0x200c0000 0x0 0x4000>; + apple,boot-dma-devices = <&display>; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index f1ab45ee78b4c..8b6842321e0a5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1990,6 +1990,7 @@ F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml F: Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml F: Documentation/gpu/apple-dcp-backlight.rst From f4ec00ff19dbfdc6d9bdbf03e8d7ec2798502f1a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 15:16:21 +1100 Subject: [PATCH 0745/1579] soc: apple: Enable T8140 system level cache allocation On T8140 the boot firmware enables and powers the 12 ways of each of the 4 system level cache planes, but leaves the allocation-disable word of every plane set. The cache therefore holds no data, and every GPU and CPU access that misses the core caches goes to DRAM. Clear the allocation-disable word of each plane once every plane reports as many ways powered as enabled and no flush in progress, and read it back. Do it only after the devices named in the device tree as having taken over DMA from the bootloader are bound, so the cache starts allocating after their hand-off rather than during it. On shutdown, stop allocating and flush the cache before the next boot stage runs. With allocation disabled the cache holds no data, so no maintenance is needed before enabling it. Measured on J700 (MacBook Neo), toggling only this word on a running system: glmark2 build+texture 902 -> 1886, vkcube 2141 -> 5584 fps, glmark2 terrain at 1920x1200 41 -> 66 fps, and the submit-to-completion latency of an isolated GPU job 560 -> 210 us. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/soc/apple/Kconfig | 12 +++ drivers/soc/apple/Makefile | 3 + drivers/soc/apple/mcc.c | 189 +++++++++++++++++++++++++++++++++++++ 4 files changed, 205 insertions(+) create mode 100644 drivers/soc/apple/mcc.c diff --git a/MAINTAINERS b/MAINTAINERS index 8b6842321e0a5..72b6e43702b5d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2004,6 +2004,7 @@ F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs +F: drivers/soc/apple/mcc.c F: drivers/tty/serial/apple_dockchannel* F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 2bbc21c765d49..c50eb9779d96e 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -25,6 +25,18 @@ config APPLE_MAILBOX Say Y here if you have an Apple SoC. +config APPLE_MCC + tristate "Apple T8140 system level cache" + depends on ARCH_APPLE || COMPILE_TEST + depends on OF + default ARCH_APPLE + help + The boot firmware of the Apple T8140 SoC enables the system level + cache without letting it allocate. This driver allows allocation + once the devices that took over DMA from the bootloader are bound. + + Say 'y' or 'm' here if you have an Apple T8140 SoC. + config APPLE_PMGR_MISC bool "Apple SoC PMGR miscellaneous support" depends on PM diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 064e65d0270b9..91f7bcd43bbbe 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -30,3 +30,6 @@ apple-sep-y := sep.o hwrng_shim.o store_shim.o bio_shim.o sha_shim.o \ pm_shim.o obj-$(CONFIG_APPLE_PMP) += pmp.o + +obj-$(CONFIG_APPLE_MCC) += apple-mcc.o +apple-mcc-y := mcc.o diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c new file mode 100644 index 0000000000000..10f5e6e01e8cd --- /dev/null +++ b/drivers/soc/apple/mcc.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple T8140 memory cache controller: system level cache allocation + * + * The boot firmware enables and powers the cache ways of every plane but + * leaves allocation disabled, so the cache holds no data. This driver allows + * allocation once the devices that took over DMA from the bootloader are + * bound, and disables allocation and flushes the cache again on shutdown. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MCC_ALLOC_DISABLE 0x82c +#define MCC_CACHE_WAYS 0x2800 +#define MCC_CACHE_WAYS_ENABLED GENMASK(4, 0) +#define MCC_CACHE_POWER 0x2804 +#define MCC_CACHE_WAYS_POWERED GENMASK(30, 24) +#define MCC_CACHE_FLUSH 0x2844 +#define MCC_CACHE_FLUSH_BUSY BIT(0) + +#define MCC_MAX_PLANES 8 +#define MCC_FLUSH_TIMEOUT_US 100000 + +struct apple_mcc { + struct device *dev; + void __iomem *planes[MCC_MAX_PLANES]; + unsigned int nr_planes; + bool enabled; +}; + +static int apple_mcc_wait_boot_dma(struct device *dev) +{ + struct of_phandle_iterator it; + int ret; + + of_for_each_phandle(&it, ret, dev->of_node, "apple,boot-dma-devices", NULL, 0) { + struct platform_device *pdev = of_find_device_by_node(it.node); + bool bound = pdev && device_is_bound(&pdev->dev); + + if (pdev) + put_device(&pdev->dev); + if (!bound) { + of_node_put(it.node); + return dev_err_probe(dev, -EPROBE_DEFER, "waiting for %pOF to bind\n", + it.node); + } + } + + return ret == -ENOENT ? 0 : ret; +} + +static bool apple_mcc_plane_ready(struct apple_mcc *mcc, unsigned int plane) +{ + void __iomem *regs = mcc->planes[plane]; + u32 ways = FIELD_GET(MCC_CACHE_WAYS_ENABLED, readl(regs + MCC_CACHE_WAYS)); + u32 powered = FIELD_GET(MCC_CACHE_WAYS_POWERED, readl(regs + MCC_CACHE_POWER)); + u32 flush = readl(regs + MCC_CACHE_FLUSH); + + if (ways && powered == ways && !(flush & MCC_CACHE_FLUSH_BUSY)) + return true; + + dev_warn(mcc->dev, "plane %u not ready: %u ways enabled, %u powered, flush %#x\n", + plane, ways, powered, flush); + return false; +} + +static int apple_mcc_enable(struct apple_mcc *mcc) +{ + unsigned int i; + + for (i = 0; i < mcc->nr_planes; i++) + if (!apple_mcc_plane_ready(mcc, i)) + return -EBUSY; + + for (i = 0; i < mcc->nr_planes; i++) { + writel(0, mcc->planes[i] + MCC_ALLOC_DISABLE); + if (readl(mcc->planes[i] + MCC_ALLOC_DISABLE)) { + dev_err(mcc->dev, "plane %u did not enable allocation\n", i); + return -EIO; + } + } + + return 0; +} + +/* Stop allocating, then write back and invalidate the cache. */ +static void apple_mcc_quiesce(struct apple_mcc *mcc) +{ + unsigned int i; + u32 val; + + if (!mcc->enabled) + return; + + for (i = 0; i < mcc->nr_planes; i++) { + writel(1, mcc->planes[i] + MCC_ALLOC_DISABLE); + writel(1, mcc->planes[i] + MCC_CACHE_FLUSH); + } + + for (i = 0; i < mcc->nr_planes; i++) + if (readl_poll_timeout_atomic(mcc->planes[i] + MCC_CACHE_FLUSH, val, + !(val & MCC_CACHE_FLUSH_BUSY), 10, + MCC_FLUSH_TIMEOUT_US)) + dev_err(mcc->dev, "plane %u flush timed out\n", i); + + mcc->enabled = false; +} + +static int apple_mcc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_mcc *mcc; + int ret; + + mcc = devm_kzalloc(dev, sizeof(*mcc), GFP_KERNEL); + if (!mcc) + return -ENOMEM; + + mcc->dev = dev; + while (platform_get_resource(pdev, IORESOURCE_MEM, mcc->nr_planes)) { + if (mcc->nr_planes == MCC_MAX_PLANES) + return dev_err_probe(dev, -EINVAL, "too many planes\n"); + mcc->planes[mcc->nr_planes] = devm_platform_ioremap_resource(pdev, mcc->nr_planes); + if (IS_ERR(mcc->planes[mcc->nr_planes])) + return PTR_ERR(mcc->planes[mcc->nr_planes]); + mcc->nr_planes++; + } + if (!mcc->nr_planes) + return dev_err_probe(dev, -EINVAL, "no planes\n"); + + ret = apple_mcc_wait_boot_dma(dev); + if (ret) + return ret; + + platform_set_drvdata(pdev, mcc); + + /* Allocation is only a performance setting: leave a cache that is not ready alone. */ + ret = apple_mcc_enable(mcc); + if (ret == -EBUSY) + return 0; + if (ret) + return ret; + + mcc->enabled = true; + dev_info(dev, "system level cache allocation enabled on %u planes\n", mcc->nr_planes); + return 0; +} + +static void apple_mcc_remove(struct platform_device *pdev) +{ + apple_mcc_quiesce(platform_get_drvdata(pdev)); +} + +static void apple_mcc_shutdown(struct platform_device *pdev) +{ + struct apple_mcc *mcc = platform_get_drvdata(pdev); + + if (mcc) + apple_mcc_quiesce(mcc); +} + +static const struct of_device_id apple_mcc_of_match[] = { + { .compatible = "apple,t8140-mcc" }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_mcc_of_match); + +static struct platform_driver apple_mcc_driver = { + .driver = { + .name = "apple-mcc", + .of_match_table = apple_mcc_of_match, + }, + .probe = apple_mcc_probe, + .remove = apple_mcc_remove, + .shutdown = apple_mcc_shutdown, +}; +module_platform_driver(apple_mcc_driver); + +MODULE_AUTHOR("Ryan Murray "); +MODULE_DESCRIPTION("Apple T8140 system level cache allocation"); +MODULE_LICENSE("Dual MIT/GPL"); From 2d51c6aada40a6c97ae81464eea6935ebf70ba95 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 15:16:31 +1100 Subject: [PATCH 0746/1579] arm64: dts: apple: t8140: Describe the memory cache controller Add the four system level cache planes of the T8140 memory cache controller, with the display as the device that must take over its bootloader DMA before cache allocation is enabled, and enable it on J700. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 4 ++++ arch/arm64/boot/dts/apple/t8140.dtsi | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 216f96dc645b1..7a9474a0849c5 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -400,6 +400,10 @@ status = "okay"; }; +&mcc { + status = "okay"; +}; + &pmp { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index b4f6219510a15..d4ce3b6dddf79 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -598,6 +598,20 @@ apple,fast-die-effort = <&pmp_report 1>; }; + /* + * System level cache planes. Allocation waits for the display, which + * takes over DMA the bootloader left running. + */ + mcc: memory-controller@220000000 { + compatible = "apple,t8140-mcc"; + reg = <0x2 0x20000000 0x0 0x4000>, + <0x2 0x20040000 0x0 0x4000>, + <0x2 0x20080000 0x0 0x4000>, + <0x2 0x200c0000 0x0 0x4000>; + apple,boot-dma-devices = <&display>; + status = "disabled"; + }; + sep_dart: iommu@280b80000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x2 0x80b80000 0x0 0x20000>; From d30b0dfcf35813ce52d935e24e7e6d46349ec3cd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 11:51:37 +1100 Subject: [PATCH 0747/1579] arm64: dts: apple: t8140-j700: Keep the idle E cluster out of its lowest OPP The power allocator grants no power to a cooling device that requests none. Above the 58 C switch-on trip it therefore caps an idle E cluster at its lowest OPP, 816 MHz, and work that wakes there runs at that speed until the next evaluation, up to 500 ms later. With the performance cpufreq governor the J700 die idles at 58-70 C, above that trip: in a ten-minute idle the E cooling device sat at state 5 in 39% of the samples, and 35 of 120 short E-core bursts started capped and took 107-114 ms instead of 36 ms. Stop the E-cluster cooling maps at state 3, 1464 MHz. The energy model puts the cost of the cluster's work there within 0.5% of 816 MHz, and with the P cluster at its lowest OPP the floor of both clusters is 0.73 W, well under the 1.1 W sustainable on floating-point loads, so the governor keeps the authority it needs. With this change the same idle never takes the E cluster below 1464 MHz and the slowest burst takes 61 ms, with the P-cluster rail unchanged at 1.21 W. With schedutil the die idles at 41-49 C and the governor never runs, before or after. A ten-minute all-core load keeps the E cluster at 2424 MHz throughout either way, since its weight makes the governor trim the P cores; the P cluster settles at 1260-1370 MHz and the hottest sensor peaks at 93.9 C (95.6 C before). Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 64 ++++++++++++++---------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 216f96dc645b1..f0c6d917595ee 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -82,8 +82,20 @@ * P-state on both kinds of load (higher gains oscillate on the heavy * one). 1.5 W lies inside the sustainable range measured at ~93 C: * 1.1 W for floating-point and 2.5 W for integer loads. The switch-on - * trip sits above the idle temperature (48-53 C), so the controller, - * and with it any integral wind-up, resets whenever the machine rests. + * trip sits above the idle temperature with schedutil (48-53 C), so the + * controller, and with it any integral wind-up, resets whenever the + * machine rests. + * + * The governor grants no power to a cooling device that requests none, + * so above the switch-on trip an idle E cluster is sent to its lowest + * OPP, and work that wakes there runs at 816 MHz until the next + * evaluation. With the performance governor the die idles at 58-70 C, + * above that trip, and in a ten-minute idle the E cluster spent 40% of + * its time capped this way. The E maps therefore stop at cooling + * state 3, 1464 MHz, where the energy model puts the cost of the + * cluster's work within 0.5% of 816 MHz. The lowest power of both + * clusters, 0.73 W, stays well under the 1.1 W sustainable on + * floating-point loads. */ thermal-zones { cpu-e0-thermal { @@ -124,10 +136,10 @@ map1 { trip = <&cpu_e0_control>; cooling-device = - <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; contribution = <3072>; }; }; @@ -171,10 +183,10 @@ map1 { trip = <&cpu_e1_control>; cooling-device = - <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; contribution = <3072>; }; }; @@ -218,10 +230,10 @@ map1 { trip = <&cpu_p0_control>; cooling-device = - <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; contribution = <3072>; }; }; @@ -265,10 +277,10 @@ map1 { trip = <&cpu_p1_control>; cooling-device = - <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; contribution = <3072>; }; }; @@ -312,10 +324,10 @@ map1 { trip = <&cpu_p2_control>; cooling-device = - <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; contribution = <3072>; }; }; @@ -359,10 +371,10 @@ map1 { trip = <&cpu_p3_control>; cooling-device = - <&cpu_e0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_e3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; contribution = <3072>; }; }; From 94f215eaa6424e8f077f1447e78e35b3ebeb6542 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:11:22 -0400 Subject: [PATCH 0748/1579] dt-bindings: power: apple,t8140-pmp-report: describe report entries The PMP chooses DRAM and fabric performance states partly from the power state that the host reports for the devices it tracks. On macOS, ApplePMGR sets a device's bit in the PMP's power-state request word after it powers the device on, clears it before it powers the device off, and waits for the PMP to acknowledge the devices that require it. The pmp-report driver already provides such report entries as power domains on earlier SoCs. Allow them as children of the T8140 report aperture, with the device's PMP soc-device index as the unit address. Assisted-by: LLM Signed-off-by: Diljit Singh --- .../power/apple,t8140-pmp-report.yaml | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml index 9b515685eef00..0cddaf6136379 100644 --- a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -27,6 +27,51 @@ properties: power-domains: maxItems: 1 + '#address-cells': + const: 1 + + '#size-cells': + const: 0 + +patternProperties: + '^report@[0-9a-f]+$': + type: object + description: | + A device whose power state the host reports to the PMP, provided as a + power domain. The host sets the device's request bit after its parent + domain powers on and clears it before the parent powers off, and waits + for the PMP to acknowledge while the PMP runs. + additionalProperties: false + + properties: + compatible: + items: + - const: apple,t8140-pmp-v2-report-entry + - const: apple,t6000-pmp-v2-report-entry + + reg: + maxItems: 1 + description: PMP soc-device index of the device. + + label: + description: Name of the power domain. + + '#power-domain-cells': + const: 0 + + power-domains: + maxItems: 1 + + apple,always-on: + type: boolean + description: Report the device as on from probe. + + required: + - compatible + - reg + - label + - '#power-domain-cells' + required: - compatible - reg @@ -42,5 +87,15 @@ examples: pmp-report@3003c0000 { compatible = "apple,t8140-pmp-v2-report"; reg = <3 0x003c0000 0 0x14000>; + #address-cells = <1>; + #size-cells = <0>; + + report@9 { + compatible = "apple,t8140-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x9>; + label = "pmp-gfx"; + #power-domain-cells = <0>; + }; }; }; From aaea0b3a4fcf5cb371b735c6b595a2919a6a4d07 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:11:22 -0400 Subject: [PATCH 0749/1579] dt-bindings: power: add Apple PMC voter interface The PMC arbitrates the performance states of shared rails, such as the DRAM controller and the SoC fabric, between voter agents: the PMP, the host performance controller (CLPC), the GPU and the application processor. Each agent has a perf-state floor word with one 4-bit state per rail, and each rail has an interface enable word with one bit per agent; the PMC ignores the floor of an agent that is not enabled on a rail. Describe the voter and rail windows, the static agents, the dynamic agents as power domains, and the host performance controller's agent. Assisted-by: LLM Signed-off-by: Diljit Singh --- .../devicetree/bindings/power/apple,pmc.yaml | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 Documentation/devicetree/bindings/power/apple,pmc.yaml diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml new file mode 100644 index 0000000000000..69cdc1cb87197 --- /dev/null +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/power/apple,pmc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SoC PMC voter interface + +maintainers: + - Diljit Singh + +description: | + The PMC arbitrates the performance states of shared rails, such as the + DRAM controller and the SoC fabric, between voter agents. Each agent has a + perf-state floor word with one 4-bit state per rail, and each rail has an + interface enable word with one bit per agent. The PMC only honours an + agent's floor on rails where its interface is enabled. + + Static agents are enabled for as long as the PMC is bound. A dynamic + agent is a power domain: it is enabled after its parent domain powers on + and disabled before its parent domain powers off. The host performance + controller's agent is described by a child node that raises its floor + while the CPU clusters or the GPU are busy. + +properties: + compatible: + const: apple,t8140-pmc + + reg: + items: + - description: Perf-state floor words of the voter agents + - description: Rail state and interface enable words + + reg-names: + items: + - const: voters + - const: rails + + power-domains: + maxItems: 1 + + '#address-cells': + const: 1 + + '#size-cells': + const: 0 + + apple,static-voters: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Voter agents to enable on every rail. + +patternProperties: + '^voter@[0-9a-f]+$': + type: object + description: A dynamic voter agent, provided as a power domain. + additionalProperties: false + + properties: + reg: + maxItems: 1 + description: Voter agent index. + + label: + description: Name of the power domain. + + '#power-domain-cells': + const: 0 + + power-domains: + maxItems: 1 + description: Domain that must be powered while the agent is enabled. + + required: + - reg + - '#power-domain-cells' + + '^performance-controller@[0-9a-f]+$': + type: object + description: The host performance controller's voter agent. + additionalProperties: false + + properties: + compatible: + const: apple,t8140-clpc + + reg: + maxItems: 1 + description: Voter agent index. + + apple,gpu-power-state: + $ref: /schemas/types.yaml#/definitions/phandle + description: PMGR power state of the GPU, used as a demand input. + + required: + - compatible + - reg + +required: + - compatible + - reg + - reg-names + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + power-management@302800000 { + compatible = "apple,t8140-pmc"; + reg = <0x3 0x02800000 0x0 0x10000>, + <0x3 0x028aa000 0x0 0x3000>; + reg-names = "voters", "rails"; + #address-cells = <1>; + #size-cells = <0>; + apple,static-voters = <0 1 3>; + + performance-controller@1 { + compatible = "apple,t8140-clpc"; + reg = <1>; + }; + + voter@2 { + reg = <2>; + label = "gfx"; + #power-domain-cells = <0>; + }; + }; + }; From 8b7d7f837ff68f7fa37f6aaae55114bbd15efb43 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:11:39 -0400 Subject: [PATCH 0750/1579] pmdomain: apple: add PMC voter interface driver The PMC only honours a voter agent's perf-state floor on rails where that agent's interface is enabled. macOS enables the static agents (PMP, CLPC and AP) on all their rails when it restores the PMGR (ApplePMC::restorePMC), and enables a dynamic agent such as the GPU only while the host keeps the agent's PMGR device powered: after the power-on state write and before the power-off one (ApplePMC::stateChange). Without these enables, the PMC ignores those agents' floors. Enable the static agents listed in the device tree for as long as the PMC is bound, and restore the saved enable words when it is unbound. Provide each dynamic agent as a power domain whose parent is its device's power domain, so that genpd enables the agent after the parent powers on and disables it before the parent powers off, as macOS does. Every enable write is read back. Export the floor words so that a host performance controller can set its agent's floor. Assisted-by: LLM Signed-off-by: Diljit Singh --- drivers/pmdomain/apple/Kconfig | 12 + drivers/pmdomain/apple/Makefile | 1 + drivers/pmdomain/apple/pmc.c | 492 ++++++++++++++++++++++++++++++++ include/linux/soc/apple/pmc.h | 55 ++++ 4 files changed, 560 insertions(+) create mode 100644 drivers/pmdomain/apple/pmc.c create mode 100644 include/linux/soc/apple/pmc.h diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index 6e9deb9de0739..7145a073429aa 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -20,4 +20,16 @@ config APPLE_PMP_REPORT depends on PM select PM_GENERIC_DOMAINS +config APPLE_PMC + bool "Apple PMC voter interface" + depends on OF + depends on PM + select PM_GENERIC_DOMAINS + help + The PMC in some Apple SoCs arbitrates the performance states of + shared rails, such as the DRAM controller and the SoC fabric, + between voter agents. This driver enables the static agents and + provides power domains that enable a dynamic agent, such as the + GPU, while its device is powered on. + endif diff --git a/drivers/pmdomain/apple/Makefile b/drivers/pmdomain/apple/Makefile index b098b204d9868..b25665119a2fa 100644 --- a/drivers/pmdomain/apple/Makefile +++ b/drivers/pmdomain/apple/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_APPLE_PMGR_PWRSTATE) += pmgr-pwrstate.o obj-$(CONFIG_APPLE_PMP_REPORT) += pmp-report.o +obj-$(CONFIG_APPLE_PMC) += pmc.o diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c new file mode 100644 index 0000000000000..027fa33a6f8fb --- /dev/null +++ b/drivers/pmdomain/apple/pmc.c @@ -0,0 +1,492 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC PMC voter interface driver + * + * The PMC arbitrates the performance state of shared rails, such as the + * DRAM controller (DCS) and the SoC fabric, between voter agents: the PMP, + * the host performance controller (CLPC), the GPU and the application + * processor. Each agent owns a perf-state floor word holding one 4-bit + * state per rail, and each rail has an interface enable word with one bit + * per agent. The PMC only honours the floor of an agent whose interface is + * enabled on that rail. + * + * At startup macOS enables the static agents (PMP, CLPC and AP) on their + * rails, and enables a dynamic agent such as the GPU only while the PMGR + * device it belongs to is powered on: after the device has been powered + * on, and before it is powered off. This driver does the former at probe + * and models each dynamic agent as a power domain that sits between the + * device and its PMGR power state, which gives the same ordering. + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_PMC_MAX_RAILS 8 +#define APPLE_PMC_MAX_AGENTS 32 + +/** + * struct apple_pmc_hw - per-SoC layout of the PMC voter interface + * @agent_stride: distance between the floor words of consecutive agents + * in the "voters" region + * @agents: number of voter agents + * @rails: number of rails + * @rail_state: offset of the per-rail state words in the "rails" region + * @rail_enable: offset of the per-rail interface enable words in the + * "rails" region + */ +struct apple_pmc_hw { + u32 agent_stride; + u32 agents; + u32 rails; + u32 rail_state; + u32 rail_enable; +}; + +struct apple_pmc { + struct device *dev; + const struct apple_pmc_hw *hw; + void __iomem *voters; + void __iomem *rails; + /* Serializes read-modify-write cycles on enable and floor words. */ + raw_spinlock_t lock; + u32 saved_enable[APPLE_PMC_MAX_RAILS]; + u32 static_agents; + struct list_head domains; + struct dentry *debugfs; +}; + +struct apple_pmc_domain { + struct generic_pm_domain genpd; + struct apple_pmc *pmc; + struct device_node *np; + struct list_head list; + unsigned int agent; + bool provider; +}; + +#define genpd_to_apple_pmc_domain(_genpd) \ + container_of(_genpd, struct apple_pmc_domain, genpd) + +static struct dentry *apple_pmc_debugfs_root; + +static u32 apple_pmc_enable_read(struct apple_pmc *pmc, unsigned int rail) +{ + return readl(pmc->rails + pmc->hw->rail_enable + 4 * rail); +} + +/* Caller holds pmc->lock. Every write is read back, as macOS does. */ +static int apple_pmc_enable_update(struct apple_pmc *pmc, unsigned int rail, + u32 set, u32 clear) +{ + void __iomem *reg = pmc->rails + pmc->hw->rail_enable + 4 * rail; + u32 old = readl(reg); + u32 val = (old | set) & ~clear; + + if (val == old) + return 0; + + writel(val, reg); + if (readl(reg) != val) { + dev_err_ratelimited(pmc->dev, + "rail %u enable %#x -> %#x did not stick\n", + rail, old, val); + return -EIO; + } + + return 0; +} + +static int apple_pmc_agent_enable(struct apple_pmc *pmc, unsigned int agent, + bool enable) +{ + unsigned long flags; + unsigned int rail; + int ret = 0; + + raw_spin_lock_irqsave(&pmc->lock, flags); + for (rail = 0; rail < pmc->hw->rails && !ret; rail++) + ret = apple_pmc_enable_update(pmc, rail, + enable ? BIT(agent) : 0, + enable ? 0 : BIT(agent)); + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + return ret; +} + +static struct apple_pmc *apple_pmc_from_dev(struct device *dev) +{ + return dev ? dev_get_drvdata(dev) : NULL; +} + +/** + * apple_pmc_rail_count() - number of rails of a PMC + * @dev: the PMC device + * + * Return: the number of rails, or 0 if @dev is not a bound PMC. + */ +unsigned int apple_pmc_rail_count(struct device *dev) +{ + struct apple_pmc *pmc = apple_pmc_from_dev(dev); + + return pmc ? pmc->hw->rails : 0; +} +EXPORT_SYMBOL_GPL(apple_pmc_rail_count); + +/** + * apple_pmc_voter_enabled() - whether the PMC honours an agent on a rail + * @dev: the PMC device + * @agent: voter agent + * @rail: rail + * + * Context: Any context. + */ +bool apple_pmc_voter_enabled(struct device *dev, unsigned int agent, + unsigned int rail) +{ + struct apple_pmc *pmc = apple_pmc_from_dev(dev); + + if (!pmc || agent >= pmc->hw->agents || rail >= pmc->hw->rails) + return false; + + return apple_pmc_enable_read(pmc, rail) & BIT(agent); +} +EXPORT_SYMBOL_GPL(apple_pmc_voter_enabled); + +/** + * apple_pmc_floor_read() - read the perf-state floor word of an agent + * @dev: the PMC device + * @agent: voter agent + * + * Context: Any context. + * Return: the floor word, or 0 if @agent is out of range. + */ +u32 apple_pmc_floor_read(struct device *dev, unsigned int agent) +{ + struct apple_pmc *pmc = apple_pmc_from_dev(dev); + + if (!pmc || agent >= pmc->hw->agents) + return 0; + + return readl(pmc->voters + agent * pmc->hw->agent_stride); +} +EXPORT_SYMBOL_GPL(apple_pmc_floor_read); + +/** + * apple_pmc_floor_update() - update the perf-state floor of an agent + * @dev: the PMC device + * @agent: voter agent + * @mask: bits of the floor word to change, see APPLE_PMC_RAIL_MASK() + * @value: new value of the bits in @mask + * + * The floor only takes effect on rails where the agent's interface is + * enabled. + * + * Context: Any context. + * Return: 0 on success, -EINVAL if @agent is out of range, or -EIO if the + * PMC did not accept the new floor. + */ +int apple_pmc_floor_update(struct device *dev, unsigned int agent, u32 mask, + u32 value) +{ + struct apple_pmc *pmc = apple_pmc_from_dev(dev); + void __iomem *reg; + unsigned long flags; + u32 old, val; + int ret = 0; + + if (!pmc || agent >= pmc->hw->agents) + return -EINVAL; + + reg = pmc->voters + agent * pmc->hw->agent_stride; + raw_spin_lock_irqsave(&pmc->lock, flags); + old = readl(reg); + val = (old & ~mask) | (value & mask); + if (val != old) { + writel(val, reg); + if ((readl(reg) & mask) != (value & mask)) + ret = -EIO; + } + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmc_floor_update); + +static int apple_pmc_domain_power_on(struct generic_pm_domain *genpd) +{ + struct apple_pmc_domain *dom = genpd_to_apple_pmc_domain(genpd); + + return apple_pmc_agent_enable(dom->pmc, dom->agent, true); +} + +static int apple_pmc_domain_power_off(struct generic_pm_domain *genpd) +{ + struct apple_pmc_domain *dom = genpd_to_apple_pmc_domain(genpd); + + return apple_pmc_agent_enable(dom->pmc, dom->agent, false); +} + +static void apple_pmc_domain_free(struct apple_pmc_domain *dom) +{ + if (dom->provider) + of_genpd_del_provider(dom->np); + if (pm_genpd_remove(&dom->genpd)) { + /* The core still owns this domain. Keep its storage alive. */ + dev_warn(dom->pmc->dev, "retaining busy power domain %s\n", + dom->genpd.name); + return; + } + of_node_put(dom->np); + kfree(dom->genpd.name); + kfree(dom); +} + +static int apple_pmc_domain_add(struct apple_pmc *pmc, struct device_node *np) +{ + struct device *dev = pmc->dev; + struct apple_pmc_domain *dom; + struct of_phandle_args parent; + const char *label; + u32 agent; + int i, ret; + + ret = of_property_read_u32(np, "reg", &agent); + if (ret) + return dev_err_probe(dev, ret, "%pOF: missing reg\n", np); + if (agent >= pmc->hw->agents) + return dev_err_probe(dev, -EINVAL, "%pOF: invalid agent %u\n", + np, agent); + if (pmc->static_agents & BIT(agent)) + return dev_err_probe(dev, -EINVAL, + "%pOF: agent %u is also static\n", np, + agent); + + dom = kzalloc_obj(*dom); + if (!dom) + return -ENOMEM; + + dom->pmc = pmc; + dom->np = of_node_get(np); + dom->agent = agent; + if (of_property_read_string(np, "label", &label)) + label = np->name; + dom->genpd.name = kasprintf(GFP_KERNEL, "pmc-%s", label); + if (!dom->genpd.name) { + of_node_put(dom->np); + kfree(dom); + return -ENOMEM; + } + dom->genpd.power_on = apple_pmc_domain_power_on; + dom->genpd.power_off = apple_pmc_domain_power_off; + + /* + * Start from the hardware state, so that an agent enabled by the + * bootloader is not disabled under a device that is still running. + */ + ret = pm_genpd_init(&dom->genpd, NULL, + !(apple_pmc_enable_read(pmc, 0) & BIT(agent))); + if (ret) { + of_node_put(dom->np); + kfree(dom->genpd.name); + kfree(dom); + return dev_err_probe(dev, ret, "%pOF: genpd init failed\n", np); + } + list_add_tail(&dom->list, &pmc->domains); + + ret = of_genpd_add_provider_simple(np, &dom->genpd); + if (ret) + return dev_err_probe(dev, ret, "%pOF: provider failed\n", np); + dom->provider = true; + + for (i = 0; !of_parse_phandle_with_args(np, "power-domains", + "#power-domain-cells", i, + &parent); i++) { + struct of_phandle_args child = { .np = np }; + + ret = of_genpd_add_subdomain(&parent, &child); + of_node_put(parent.np); + if (ret) + return dev_err_probe(dev, ret, + "%pOF: failed to join parent domain\n", + np); + } + + return 0; +} + +static int apple_pmc_status_show(struct seq_file *s, void *unused) +{ + struct apple_pmc *pmc = s->private; + const struct apple_pmc_hw *hw = pmc->hw; + unsigned int i; + + seq_printf(s, "static agents: %#x\n", pmc->static_agents); + for (i = 0; i < hw->agents; i++) + seq_printf(s, "agent%u floor: %#010x\n", i, + readl(pmc->voters + i * hw->agent_stride)); + for (i = 0; i < hw->rails; i++) + seq_printf(s, "rail%u state: %#010x enable: %#010x (saved %#010x)\n", + i, readl(pmc->rails + hw->rail_state + 4 * i), + apple_pmc_enable_read(pmc, i), pmc->saved_enable[i]); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(apple_pmc_status); + +static void apple_pmc_release(void *data) +{ + struct apple_pmc *pmc = data; + struct apple_pmc_domain *dom, *tmp; + unsigned long flags; + unsigned int rail; + + of_platform_depopulate(pmc->dev); + debugfs_remove_recursive(pmc->debugfs); + list_for_each_entry_safe_reverse(dom, tmp, &pmc->domains, list) { + list_del(&dom->list); + apple_pmc_domain_free(dom); + } + + raw_spin_lock_irqsave(&pmc->lock, flags); + for (rail = 0; rail < pmc->hw->rails; rail++) + writel(pmc->saved_enable[rail], + pmc->rails + pmc->hw->rail_enable + 4 * rail); + raw_spin_unlock_irqrestore(&pmc->lock, flags); +} + +static int apple_pmc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + u32 agents[APPLE_PMC_MAX_AGENTS]; + struct device_node *child; + struct apple_pmc *pmc; + unsigned int rail; + int count, i, ret; + + pmc = devm_kzalloc(dev, sizeof(*pmc), GFP_KERNEL); + if (!pmc) + return -ENOMEM; + + pmc->dev = dev; + pmc->hw = of_device_get_match_data(dev); + raw_spin_lock_init(&pmc->lock); + INIT_LIST_HEAD(&pmc->domains); + + if (pmc->hw->rails > APPLE_PMC_MAX_RAILS || + pmc->hw->agents > APPLE_PMC_MAX_AGENTS) + return -EINVAL; + + pmc->voters = devm_platform_ioremap_resource_byname(pdev, "voters"); + if (IS_ERR(pmc->voters)) + return PTR_ERR(pmc->voters); + pmc->rails = devm_platform_ioremap_resource_byname(pdev, "rails"); + if (IS_ERR(pmc->rails)) + return PTR_ERR(pmc->rails); + + /* An unpowered PMC reads all-ones; do not write to it. */ + for (rail = 0; rail < pmc->hw->rails; rail++) { + pmc->saved_enable[rail] = apple_pmc_enable_read(pmc, rail); + if (pmc->saved_enable[rail] == ~0U) + return dev_err_probe(dev, -ENODEV, + "PMC is not responding\n"); + } + + count = of_property_count_u32_elems(np, "apple,static-voters"); + if (count > 0) { + if (count > ARRAY_SIZE(agents)) + return -EINVAL; + ret = of_property_read_u32_array(np, "apple,static-voters", + agents, count); + if (ret) + return ret; + for (i = 0; i < count; i++) { + if (agents[i] >= pmc->hw->agents) + return dev_err_probe(dev, -EINVAL, + "invalid static voter %u\n", + agents[i]); + pmc->static_agents |= BIT(agents[i]); + } + } + + platform_set_drvdata(pdev, pmc); + ret = devm_add_action_or_reset(dev, apple_pmc_release, pmc); + if (ret) + return ret; + + for (i = 0; i < pmc->hw->agents; i++) { + if (!(pmc->static_agents & BIT(i))) + continue; + ret = apple_pmc_agent_enable(pmc, i, true); + if (ret) + return dev_err_probe(dev, ret, + "failed to enable static voter %d\n", + i); + } + + for_each_available_child_of_node(np, child) { + if (!of_property_present(child, "#power-domain-cells")) + continue; + ret = apple_pmc_domain_add(pmc, child); + if (ret) { + of_node_put(child); + return ret; + } + } + + pmc->debugfs = debugfs_create_dir(dev_name(dev), apple_pmc_debugfs_root); + debugfs_create_file("status", 0400, pmc->debugfs, pmc, + &apple_pmc_status_fops); + + ret = of_platform_populate(np, NULL, NULL, dev); + if (ret) + return dev_err_probe(dev, ret, "failed to create child devices\n"); + + return 0; +} + +static const struct apple_pmc_hw apple_pmc_hw_t8140 = { + .agent_stride = 0x4000, + .agents = 4, + .rails = 4, + .rail_state = 0x0, + .rail_enable = 0x2000, +}; + +static const struct of_device_id apple_pmc_of_match[] = { + { .compatible = "apple,t8140-pmc", .data = &apple_pmc_hw_t8140 }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_pmc_of_match); + +static struct platform_driver apple_pmc_driver = { + .probe = apple_pmc_probe, + .driver = { + .name = "apple-pmc", + .of_match_table = apple_pmc_of_match, + /* Power-domain providers are not intended for manual unbinding. */ + .suppress_bind_attrs = true, + }, +}; + +static int __init apple_pmc_init(void) +{ + apple_pmc_debugfs_root = debugfs_create_dir("apple-pmc", NULL); + return platform_driver_register(&apple_pmc_driver); +} +module_init(apple_pmc_init); + +MODULE_DESCRIPTION("Apple SoC PMC voter interface driver"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/include/linux/soc/apple/pmc.h b/include/linux/soc/apple/pmc.h new file mode 100644 index 0000000000000..be391b83d9752 --- /dev/null +++ b/include/linux/soc/apple/pmc.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple SoC PMC voter interface + * Copyright (C) The Asahi Linux Contributors + */ + +#ifndef _LINUX_SOC_APPLE_PMC_H_ +#define _LINUX_SOC_APPLE_PMC_H_ + +#include +#include + +struct device; + +/* + * A PMC perf-state floor word holds one 4-bit performance state per rail, + * rail N in bits [4N+3:4N]. + */ +#define APPLE_PMC_RAIL_SHIFT(rail) (4 * (rail)) +#define APPLE_PMC_RAIL_MASK(rail) (0xfU << APPLE_PMC_RAIL_SHIFT(rail)) + +#if IS_ENABLED(CONFIG_APPLE_PMC) +unsigned int apple_pmc_rail_count(struct device *pmc); +bool apple_pmc_voter_enabled(struct device *pmc, unsigned int agent, + unsigned int rail); +u32 apple_pmc_floor_read(struct device *pmc, unsigned int agent); +int apple_pmc_floor_update(struct device *pmc, unsigned int agent, u32 mask, + u32 value); +#else +static inline unsigned int apple_pmc_rail_count(struct device *pmc) +{ + return 0; +} + +static inline bool apple_pmc_voter_enabled(struct device *pmc, + unsigned int agent, + unsigned int rail) +{ + return false; +} + +static inline u32 apple_pmc_floor_read(struct device *pmc, unsigned int agent) +{ + return 0; +} + +static inline int apple_pmc_floor_update(struct device *pmc, + unsigned int agent, u32 mask, + u32 value) +{ + return -ENODEV; +} +#endif + +#endif /* _LINUX_SOC_APPLE_PMC_H_ */ From 6454e02db2a562d82adfbc4447ffe093f43dae0a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:11:39 -0400 Subject: [PATCH 0751/1579] soc: apple: add host performance controller floors On macOS the CLPC raises the DRAM controller (DCS) and SoC fabric floors of its PMC voter agent while the CPU clusters or the GPU are busy (AppleT8140CLPC: clpc::memory::DVFManager and clpc::soc::DVFManager, both ending in the PMC CLPC perf-state floor), and lets them fall at idle. Linux has no equivalent, so CPU-bound work runs with the DRAM controller at whatever state the other agents happen to request. Sample every cpufreq policy every 4 ms, the CLPC sampling interval from the Apple device tree. A policy whose busiest CPU was busy for at least half of the period requests a DCS and fabric state from a table indexed by its frequency quartile. Optionally, while the GPU's PMGR state reports the GPU as actually powered, request a fixed GPU floor. Each rail keeps its last request for 32 ms after demand stops, then returns to 0. The native mapping from CPU and GPU demand to DCS and fabric states is not decoded. The default tables (DCS 1-4 by quartile, no fabric request, no GPU floor) are measured policy and can be changed with module parameters. A debugfs file holds a rail at a fixed state for diagnosis only. Assisted-by: LLM Signed-off-by: Diljit Singh --- drivers/soc/apple/Kconfig | 12 + drivers/soc/apple/Makefile | 2 + drivers/soc/apple/clpc.c | 531 +++++++++++++++++++++++++++++++++++++ 3 files changed, 545 insertions(+) create mode 100644 drivers/soc/apple/clpc.c diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index c50eb9779d96e..39036f1565403 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -163,6 +163,18 @@ config APPLE_PMP Say 'y' here if you have an Apple SoC. +config APPLE_CLPC + tristate "Apple host performance controller floors" + depends on ARCH_APPLE || COMPILE_TEST + depends on APPLE_PMC && CPU_FREQ + select MFD_SYSCON + help + Raises the DRAM controller and SoC fabric performance floors of the + host's PMC voter agent while CPU clusters, and optionally the GPU, + are busy, and releases them at idle. + + Say 'y' or 'm' here if you have an Apple SoC with a PMC. + endmenu endif diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 91f7bcd43bbbe..e6cb2e917bc0a 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -33,3 +33,5 @@ obj-$(CONFIG_APPLE_PMP) += pmp.o obj-$(CONFIG_APPLE_MCC) += apple-mcc.o apple-mcc-y := mcc.o + +obj-$(CONFIG_APPLE_CLPC) += clpc.o diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c new file mode 100644 index 0000000000000..32b88c7c968ba --- /dev/null +++ b/drivers/soc/apple/clpc.c @@ -0,0 +1,531 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC host performance controller (CLPC) floors + * + * On macOS the CLPC raises the floor of the memory controller (DCS) and the + * SoC fabric while the CPU clusters or the GPU are busy, through its own + * voter agent in the PMC. This driver keeps that agent's floor in step with + * Linux CPU load and, optionally, GPU activity: + * + * - every sampling period, each cpufreq policy reports how busy its + * busiest CPU was and how fast the policy runs relative to its maximum; + * a busy policy requests a DCS and fabric state from a table indexed by + * that frequency quartile; + * - while the GPU's PMGR power state reports the GPU as actually powered, + * a fixed GPU floor is requested; + * - each rail keeps its last request for a hold time after demand stops, + * then drops back to 0, so idle releases the floor. + * + * The native mapping from CPU and GPU demand to DCS and fabric states has + * not been decoded. The default tables are measured policy, not a copy of + * macOS, and can be changed with module parameters. + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_CLPC_MAX_CLUSTERS 8 +#define APPLE_CLPC_LEVELS 4 + +/* PMGR power state: ACTUAL is bits [7:4]; 0xf means powered on. */ +#define APPLE_PMGR_PS_ACTUAL GENMASK(7, 4) +#define APPLE_PMGR_PS_ACTIVE 0xf + +enum apple_clpc_rail { + APPLE_CLPC_DCS, + APPLE_CLPC_SOC, + APPLE_CLPC_RAILS, +}; + +static unsigned int sample_ms = 4; +module_param(sample_ms, uint, 0444); +MODULE_PARM_DESC(sample_ms, "Sampling period in ms"); + +static unsigned int busy_pct = 50; +module_param(busy_pct, uint, 0444); +MODULE_PARM_DESC(busy_pct, "Busy threshold of a CPU cluster, percent of the sampling period"); + +static unsigned int hold_ms = 32; +module_param(hold_ms, uint, 0444); +MODULE_PARM_DESC(hold_ms, "Time a rail keeps its last request after demand stops, ms"); + +static char *dcs_map = "1,2,3,4"; +module_param(dcs_map, charp, 0444); +MODULE_PARM_DESC(dcs_map, "DCS state for a busy cluster below 25, 50, 75 and 100 percent of its maximum frequency"); + +static char *soc_map = "0,0,0,0"; +module_param(soc_map, charp, 0444); +MODULE_PARM_DESC(soc_map, "Fabric state for a busy cluster below 25, 50, 75 and 100 percent of its maximum frequency"); + +static unsigned int gpu_dcs; +module_param(gpu_dcs, uint, 0444); +MODULE_PARM_DESC(gpu_dcs, "DCS state while the GPU is powered (0: none)"); + +static unsigned int gpu_soc; +module_param(gpu_soc, uint, 0444); +MODULE_PARM_DESC(gpu_soc, "Fabric state while the GPU is powered (0: none)"); + +/** + * struct apple_clpc_hw - per-SoC CLPC parameters + * @rail: PMC rail of the DCS and of the fabric + * @max_state: highest state of the DCS and of the fabric + */ +struct apple_clpc_hw { + unsigned int rail[APPLE_CLPC_RAILS]; + unsigned int max_state[APPLE_CLPC_RAILS]; +}; + +struct apple_clpc_cluster { + unsigned int policy_cpu; + cpumask_var_t cpus; + unsigned int busy; + unsigned int level; + unsigned int request[APPLE_CLPC_RAILS]; +}; + +struct apple_clpc_vote { + unsigned int state; + unsigned long last_demand; +}; + +struct apple_clpc { + struct device *dev; + struct device *pmc; + const struct apple_clpc_hw *hw; + unsigned int agent; + struct regmap *gpu_regmap; + u32 gpu_offset; + struct delayed_work work; + struct apple_clpc_cluster clusters[APPLE_CLPC_MAX_CLUSTERS]; + unsigned int nr_clusters; + cpumask_var_t covered; + unsigned long next_scan; + u64 *idle; + u64 *wall; + u8 map[APPLE_CLPC_RAILS][APPLE_CLPC_LEVELS]; + unsigned int gpu_state[APPLE_CLPC_RAILS]; + struct apple_clpc_vote vote[APPLE_CLPC_RAILS]; + int force[APPLE_CLPC_RAILS]; + u32 mask; + u32 saved_floor; + bool gpu_on; + unsigned long samples; + unsigned long update_errors; + struct dentry *debugfs; +}; + +static int apple_clpc_parse_map(struct apple_clpc *clpc, const char *s, + enum apple_clpc_rail rail) +{ + unsigned int v[APPLE_CLPC_LEVELS]; + unsigned int i; + + if (sscanf(s, "%u,%u,%u,%u", &v[0], &v[1], &v[2], &v[3]) != 4) + return -EINVAL; + for (i = 0; i < APPLE_CLPC_LEVELS; i++) { + if (v[i] > clpc->hw->max_state[rail]) + return -EINVAL; + clpc->map[rail][i] = v[i]; + } + + return 0; +} + +/* cpufreq policies can appear after probe; pick up new ones as they do. */ +static void apple_clpc_find_clusters(struct apple_clpc *clpc) +{ + unsigned int cpu, i; + + for_each_possible_cpu(cpu) { + struct cpufreq_policy *policy; + struct apple_clpc_cluster *cl; + + if (cpumask_test_cpu(cpu, clpc->covered)) + continue; + policy = cpufreq_cpu_get(cpu); + if (!policy) + continue; + for (i = 0; i < clpc->nr_clusters; i++) + if (clpc->clusters[i].policy_cpu == policy->cpu) + break; + if (i == clpc->nr_clusters && i < APPLE_CLPC_MAX_CLUSTERS) { + cl = &clpc->clusters[clpc->nr_clusters++]; + cl->policy_cpu = policy->cpu; + } + if (i < clpc->nr_clusters) { + cl = &clpc->clusters[i]; + cpumask_set_cpu(cpu, cl->cpus); + cpumask_set_cpu(cpu, clpc->covered); + clpc->idle[cpu] = get_cpu_idle_time(cpu, &clpc->wall[cpu], 0); + } + cpufreq_cpu_put(policy); + } +} + +static void apple_clpc_sample_cluster(struct apple_clpc *clpc, + struct apple_clpc_cluster *cl) +{ + struct cpufreq_policy *policy; + unsigned int cpu, busy = 0, q, rail; + + for_each_cpu(cpu, cl->cpus) { + u64 wall, idle = get_cpu_idle_time(cpu, &wall, 0); + u64 dw = wall - clpc->wall[cpu], di = idle - clpc->idle[cpu]; + + clpc->wall[cpu] = wall; + clpc->idle[cpu] = idle; + if (dw && di <= dw) + busy = max_t(unsigned int, busy, + div64_u64((dw - di) * 100, dw)); + } + cl->busy = busy; + + cl->level = 0; + policy = cpufreq_cpu_get(cl->policy_cpu); + if (policy) { + if (policy->cpuinfo.max_freq) + cl->level = policy->cur * 100 / policy->cpuinfo.max_freq; + cpufreq_cpu_put(policy); + } + + q = min_t(unsigned int, cl->level / (100 / APPLE_CLPC_LEVELS), + APPLE_CLPC_LEVELS - 1); + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + cl->request[rail] = busy >= busy_pct ? clpc->map[rail][q] : 0; +} + +static bool apple_clpc_gpu_powered(struct apple_clpc *clpc) +{ + u32 ps; + + /* + * Hardware auto-gating powers the GPU down while the requested state + * stays on, so only the actual state says whether the GPU runs. + */ + if (regmap_read(clpc->gpu_regmap, clpc->gpu_offset, &ps)) + return false; + + return FIELD_GET(APPLE_PMGR_PS_ACTUAL, ps) == APPLE_PMGR_PS_ACTIVE; +} + +static unsigned int apple_clpc_vote(struct apple_clpc_vote *vote, + unsigned int demand, unsigned long now) +{ + if (demand) { + vote->last_demand = now; + vote->state = demand; + } else if (time_after_eq(now, vote->last_demand + + msecs_to_jiffies(hold_ms))) { + vote->state = 0; + } + + return vote->state; +} + +static u32 apple_clpc_floor(struct apple_clpc *clpc, const unsigned int *state) +{ + u32 floor = 0; + unsigned int rail; + + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + floor |= (state[rail] & 0xf) << + APPLE_PMC_RAIL_SHIFT(clpc->hw->rail[rail]); + + return floor; +} + +static void apple_clpc_work(struct work_struct *work) +{ + struct apple_clpc *clpc = container_of(to_delayed_work(work), + struct apple_clpc, work); + unsigned int demand[APPLE_CLPC_RAILS] = {}; + unsigned int state[APPLE_CLPC_RAILS]; + unsigned long now = jiffies; + unsigned int i, rail; + int force; + + clpc->samples++; + if (!cpumask_equal(clpc->covered, cpu_possible_mask) && + time_after_eq(now, clpc->next_scan)) { + apple_clpc_find_clusters(clpc); + clpc->next_scan = now + HZ; + } + + for (i = 0; i < clpc->nr_clusters; i++) { + struct apple_clpc_cluster *cl = &clpc->clusters[i]; + + apple_clpc_sample_cluster(clpc, cl); + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + demand[rail] = max(demand[rail], cl->request[rail]); + } + + clpc->gpu_on = clpc->gpu_regmap && + (clpc->gpu_state[APPLE_CLPC_DCS] || + clpc->gpu_state[APPLE_CLPC_SOC]) && + apple_clpc_gpu_powered(clpc); + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) { + if (clpc->gpu_on) + demand[rail] = max(demand[rail], clpc->gpu_state[rail]); + state[rail] = apple_clpc_vote(&clpc->vote[rail], demand[rail], + now); + force = READ_ONCE(clpc->force[rail]); + if (force >= 0) + state[rail] = force; + } + + if (apple_pmc_floor_update(clpc->pmc, clpc->agent, clpc->mask, + apple_clpc_floor(clpc, state))) + clpc->update_errors++; + + queue_delayed_work(system_highpri_wq, &clpc->work, + msecs_to_jiffies(sample_ms)); +} + +static int apple_clpc_status_show(struct seq_file *s, void *unused) +{ + struct apple_clpc *clpc = s->private; + unsigned int i, rail; + + seq_printf(s, "agent: %u floor: %#010x (saved %#010x)\n", clpc->agent, + apple_pmc_floor_read(clpc->pmc, clpc->agent), + clpc->saved_floor); + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + seq_printf(s, "%s: rail %u state %u enabled %u force %d map %u,%u,%u,%u gpu %u\n", + rail == APPLE_CLPC_DCS ? "dcs" : "soc", + clpc->hw->rail[rail], clpc->vote[rail].state, + apple_pmc_voter_enabled(clpc->pmc, clpc->agent, + clpc->hw->rail[rail]), + READ_ONCE(clpc->force[rail]), clpc->map[rail][0], + clpc->map[rail][1], clpc->map[rail][2], + clpc->map[rail][3], clpc->gpu_state[rail]); + seq_printf(s, "gpu: %s\n", !clpc->gpu_regmap ? "no input" : + !clpc->gpu_state[APPLE_CLPC_DCS] && + !clpc->gpu_state[APPLE_CLPC_SOC] ? "no floor" : + clpc->gpu_on ? "powered" : "off"); + seq_printf(s, "samples: %lu update errors: %lu\n", clpc->samples, + clpc->update_errors); + for (i = 0; i < clpc->nr_clusters; i++) + seq_printf(s, "cluster cpu%u: busy %u%% level %u%% -> dcs %u soc %u\n", + clpc->clusters[i].policy_cpu, clpc->clusters[i].busy, + clpc->clusters[i].level, + clpc->clusters[i].request[APPLE_CLPC_DCS], + clpc->clusters[i].request[APPLE_CLPC_SOC]); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(apple_clpc_status); + +/* + * Diagnostic override: "DCS SOC" holds each rail at a fixed state, -1 + * returns that rail to demand control; "-1" alone clears both. + */ +static ssize_t apple_clpc_force_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *pos) +{ + struct apple_clpc *clpc = file->private_data; + int v[APPLE_CLPC_RAILS], n; + unsigned int rail; + char buf[32]; + + if (len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + + n = sscanf(buf, "%d %d", &v[APPLE_CLPC_DCS], &v[APPLE_CLPC_SOC]); + if (n == 1 && v[APPLE_CLPC_DCS] == -1) + v[APPLE_CLPC_SOC] = -1; + else if (n != 2) + return -EINVAL; + + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + if (v[rail] < -1 || v[rail] > (int)clpc->hw->max_state[rail]) + return -EINVAL; + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + WRITE_ONCE(clpc->force[rail], v[rail]); + + dev_info(clpc->dev, "diagnostic override: dcs %d soc %d\n", + v[APPLE_CLPC_DCS], v[APPLE_CLPC_SOC]); + return len; +} + +static const struct file_operations apple_clpc_force_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = apple_clpc_force_write, +}; + +static int apple_clpc_gpu_input(struct apple_clpc *clpc) +{ + struct device_node *ps, *pmgr; + int ret; + + ps = of_parse_phandle(clpc->dev->of_node, "apple,gpu-power-state", 0); + if (!ps) + return 0; + + ret = of_property_read_u32(ps, "reg", &clpc->gpu_offset); + pmgr = of_get_parent(ps); + of_node_put(ps); + if (ret) { + of_node_put(pmgr); + return ret; + } + + clpc->gpu_regmap = syscon_node_to_regmap(pmgr); + of_node_put(pmgr); + if (IS_ERR(clpc->gpu_regmap)) { + ret = PTR_ERR(clpc->gpu_regmap); + clpc->gpu_regmap = NULL; + return ret; + } + + return 0; +} + +static void apple_clpc_free_clusters(struct apple_clpc *clpc) +{ + unsigned int i; + + for (i = 0; i < APPLE_CLPC_MAX_CLUSTERS; i++) + free_cpumask_var(clpc->clusters[i].cpus); + free_cpumask_var(clpc->covered); +} + +static void apple_clpc_stop(void *data) +{ + struct apple_clpc *clpc = data; + + cancel_delayed_work_sync(&clpc->work); + debugfs_remove_recursive(clpc->debugfs); + apple_pmc_floor_update(clpc->pmc, clpc->agent, clpc->mask, + clpc->saved_floor); + apple_clpc_free_clusters(clpc); +} + +static int apple_clpc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_clpc *clpc; + unsigned int rail, i; + int ret; + + clpc = devm_kzalloc(dev, sizeof(*clpc), GFP_KERNEL); + if (!clpc) + return -ENOMEM; + + clpc->dev = dev; + clpc->pmc = dev->parent; + clpc->hw = of_device_get_match_data(dev); + if (!apple_pmc_rail_count(clpc->pmc)) + return dev_err_probe(dev, -ENODEV, "parent is not a PMC\n"); + + ret = of_property_read_u32(dev->of_node, "reg", &clpc->agent); + if (ret) + return dev_err_probe(dev, ret, "missing voter agent\n"); + + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) { + if (clpc->hw->rail[rail] >= apple_pmc_rail_count(clpc->pmc)) + return -EINVAL; + clpc->mask |= APPLE_PMC_RAIL_MASK(clpc->hw->rail[rail]); + clpc->force[rail] = -1; + } + + ret = apple_clpc_parse_map(clpc, dcs_map, APPLE_CLPC_DCS) ?: + apple_clpc_parse_map(clpc, soc_map, APPLE_CLPC_SOC); + if (ret) + return dev_err_probe(dev, ret, "invalid state map\n"); + if (gpu_dcs > clpc->hw->max_state[APPLE_CLPC_DCS] || + gpu_soc > clpc->hw->max_state[APPLE_CLPC_SOC]) + return dev_err_probe(dev, -EINVAL, "invalid GPU state\n"); + clpc->gpu_state[APPLE_CLPC_DCS] = gpu_dcs; + clpc->gpu_state[APPLE_CLPC_SOC] = gpu_soc; + + ret = apple_clpc_gpu_input(clpc); + if (ret) + return dev_err_probe(dev, ret, "invalid GPU power state\n"); + + clpc->idle = devm_kcalloc(dev, nr_cpu_ids, sizeof(*clpc->idle), + GFP_KERNEL); + clpc->wall = devm_kcalloc(dev, nr_cpu_ids, sizeof(*clpc->wall), + GFP_KERNEL); + if (!clpc->idle || !clpc->wall) + return -ENOMEM; + + for (i = 0; i < APPLE_CLPC_MAX_CLUSTERS; i++) { + if (!zalloc_cpumask_var(&clpc->clusters[i].cpus, GFP_KERNEL)) { + apple_clpc_free_clusters(clpc); + return -ENOMEM; + } + } + if (!zalloc_cpumask_var(&clpc->covered, GFP_KERNEL)) { + apple_clpc_free_clusters(clpc); + return -ENOMEM; + } + + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + if (!apple_pmc_voter_enabled(clpc->pmc, clpc->agent, + clpc->hw->rail[rail])) + dev_warn(dev, "voter agent %u is not enabled on rail %u\n", + clpc->agent, clpc->hw->rail[rail]); + + clpc->saved_floor = apple_pmc_floor_read(clpc->pmc, clpc->agent) & + clpc->mask; + apple_clpc_find_clusters(clpc); + clpc->next_scan = jiffies + HZ; + INIT_DELAYED_WORK(&clpc->work, apple_clpc_work); + + clpc->debugfs = debugfs_create_dir(dev_name(dev), NULL); + debugfs_create_file("status", 0400, clpc->debugfs, clpc, + &apple_clpc_status_fops); + debugfs_create_file("force", 0200, clpc->debugfs, clpc, + &apple_clpc_force_fops); + + ret = devm_add_action_or_reset(dev, apple_clpc_stop, clpc); + if (ret) + return ret; + + queue_delayed_work(system_highpri_wq, &clpc->work, + msecs_to_jiffies(sample_ms)); + return 0; +} + +/* T8140: DCS states 0-4 on rail 0, fabric states 0-3 on rail 1. */ +static const struct apple_clpc_hw apple_clpc_hw_t8140 = { + .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, + .max_state = { [APPLE_CLPC_DCS] = 4, [APPLE_CLPC_SOC] = 3 }, +}; + +static const struct of_device_id apple_clpc_of_match[] = { + { .compatible = "apple,t8140-clpc", .data = &apple_clpc_hw_t8140 }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_clpc_of_match); + +static struct platform_driver apple_clpc_driver = { + .probe = apple_clpc_probe, + .driver = { + .name = "apple-clpc", + .of_match_table = apple_clpc_of_match, + }, +}; +module_platform_driver(apple_clpc_driver); + +MODULE_DESCRIPTION("Apple SoC host performance controller floors"); +MODULE_LICENSE("Dual MIT/GPL"); From b5b71dfc50b2bfbf5800a76c540f878924a678cb Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:11:40 -0400 Subject: [PATCH 0752/1579] arm64: dts: apple: t8140: describe the PMC and report GFX to the PMP Describe the PMC voter interface with the PMP, CLPC and AP agents as static voters, the GPU's dynamic agent as a power domain, and the host performance controller's agent. Add the PMC's PMGR power state. Report the GPU to the PMP through a report entry for PMP soc-device 9, whose type in the Apple device tree requires an acknowledgment. The GPU now sits on the PMC GFX agent, which sits on the report entry, which sits on the GFX PMGR state. Powering the GPU on therefore writes the PMGR state, then reports it to the PMP, then enables its PMC agent, and powering it off undoes them in reverse order, as macOS does. Enable the PMC on J700. Assisted-by: LLM Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t8140-j700.dts | 4 ++ arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 9 +++++ arch/arm64/boot/dts/apple/t8140.dtsi | 45 ++++++++++++++++++++++- 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 931fe13b0b17f..e433db4472941 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -420,6 +420,10 @@ status = "okay"; }; +&pmc { + status = "okay"; +}; + &pinctrl_ap { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 91353ac335431..93c0bebbb1b82 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -32,6 +32,15 @@ label = "gpio"; }; + ps_pmc: power-controller@138 { + compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x138 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmc"; + apple,always-on; + }; + ps_afggfxni0: power-controller@1b8 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x1b8 4>; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index d4ce3b6dddf79..b643f4dbfa161 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -485,7 +485,19 @@ compatible = "apple,t8140-pmp-v2-report"; reg = <0x3 0x003c0000 0x0 0x14000>; power-domains = <&ps_pms_sram>; + #address-cells = <1>; + #size-cells = <0>; status = "disabled"; + + /* PMP soc-device 9, which the PMP acknowledges */ + pmp_report_gfx: report@9 { + compatible = "apple,t8140-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x9>; + label = "pmp-gfx"; + #power-domain-cells = <0>; + power-domains = <&ps_gfx>; + }; }; pmp: pmp@300500000 { @@ -1123,7 +1135,7 @@ <0x4 0x80000000 0x0 0x4000000>; reg-names = "asc", "sgx"; mboxes = <&gfx_mbox>, <&gfx1_mbox>; - power-domains = <&ps_gfx>; + power-domains = <&pmc_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>; memory-region-names = "ttbs", "pagetables", "handoff"; @@ -1210,6 +1222,37 @@ status = "disabled"; }; + /* + * Perf-state floors of voter agents 0-3, and the rail state and + * interface enable words, in the PMC window at 0x302800000. + */ + pmc: power-management@302800000 { + compatible = "apple,t8140-pmc"; + reg = <0x3 0x02800000 0x0 0x10000>, + <0x3 0x028aa000 0x0 0x3000>; + reg-names = "voters", "rails"; + power-domains = <&ps_pmc>; + #address-cells = <1>; + #size-cells = <0>; + /* PMP, CLPC and AP agents */ + apple,static-voters = <0 1 3>; + status = "disabled"; + + clpc: performance-controller@1 { + compatible = "apple,t8140-clpc"; + reg = <1>; + apple,gpu-power-state = <&ps_gfx>; + }; + + /* Dynamic agent of the GFX PMGR device */ + pmc_gfx: voter@2 { + reg = <2>; + label = "gfx"; + #power-domain-cells = <0>; + power-domains = <&pmp_report_gfx>; + }; + }; + /* * Bandwidth scratch and request registers, 0x20000 and 0x68000 * into the display clock window at 0x302800000 (the sixth disp0 From 0272b63e30f07b8c1a496cb01f5aa74e2cc1ff93 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:41:18 +1100 Subject: [PATCH 0753/1579] arm64: dts: apple: t8140: keep the GPU on its PMGR state With the previous commit the GPU's only power domain is the PMC's GFX voter domain, which only the new PMC driver provides. A kernel without that driver, or built without CONFIG_APPLE_PMC, never probes the GPU: platform 482600000.gpu: deferred probe pending: platform: wait for supplier /soc/power-management@302800000/voter@2 A device tree has to keep working with kernels that predate a driver it describes, so leave the GPU on its PMGR power state as before. The GFX voter does nothing measurable on the J700 anyway: its floor stayed 0 under every GPU load we sampled, and the GPU firmware has no mapping of the PMC window to write it through. The GPU throughput gain comes from the report entry alone. Once the PMP is told that the GPU is on, it raises its own DRAM controller (DCS) vote under GPU load, which the boot firmware already lets through on the DCS rail. With only the report bit set, a kernel without any PMC driver runs glmark2's build scene at 2199 instead of 1453; enabling every PMC agent on every rail on top of that changes it to 2218. Report the GPU as on from probe instead, as t8112 does for its display. The host never turns the GPU's PMGR state off at runtime, and hardware auto-PM already gates the GPU while it stays requested on, so the PMP sees the same thing it does with the report in the GPU's power path: a reported-on GPU that is gated whenever it idles. It votes the DCS up only under GPU load. Every kernel with the PMP report driver supports the entry. Drop the unused GFX voter node. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index b643f4dbfa161..edd81b9b10c22 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -497,6 +497,7 @@ label = "pmp-gfx"; #power-domain-cells = <0>; power-domains = <&ps_gfx>; + apple,always-on; }; }; @@ -1135,7 +1136,7 @@ <0x4 0x80000000 0x0 0x4000000>; reg-names = "asc", "sgx"; mboxes = <&gfx_mbox>, <&gfx1_mbox>; - power-domains = <&pmc_gfx>; + power-domains = <&ps_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>; memory-region-names = "ttbs", "pagetables", "handoff"; @@ -1243,14 +1244,6 @@ reg = <1>; apple,gpu-power-state = <&ps_gfx>; }; - - /* Dynamic agent of the GFX PMGR device */ - pmc_gfx: voter@2 { - reg = <2>; - label = "gfx"; - #power-domain-cells = <0>; - power-domains = <&pmp_report_gfx>; - }; }; /* From e736b2a049bf708512f845093f712c431c20da3b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:42:02 +1100 Subject: [PATCH 0754/1579] pmdomain: apple: pmc: report the honoured floor of each agent The words at the start of the "rails" region are not per-rail states. There is one per voter agent, and each holds that agent's floor as the PMC honours it: masked by the agent's interface enables and clamped to each rail's highest state. On the J700, with agent 1's floor set to 0x40 and its interface enabled only on rail 1, word 1 reads 0x30 (the fabric rail tops out at 3) and word 0 stays 0; enabling agent 1 on rail 0 instead with a floor of 4 makes word 1 read 4. Name the offset after what it holds and print each agent's floor next to what the PMC honours of it, and the rail enables on their own lines. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmc.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 027fa33a6f8fb..fac0b1dfc02b6 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -42,7 +42,9 @@ * in the "voters" region * @agents: number of voter agents * @rails: number of rails - * @rail_state: offset of the per-rail state words in the "rails" region + * @agent_state: offset of the per-agent state words in the "rails" region; + * each holds the agent's floor as the PMC honours it: masked by the + * agent's interface enables and clamped to each rail's highest state * @rail_enable: offset of the per-rail interface enable words in the * "rails" region */ @@ -50,7 +52,7 @@ struct apple_pmc_hw { u32 agent_stride; u32 agents; u32 rails; - u32 rail_state; + u32 agent_state; u32 rail_enable; }; @@ -334,12 +336,12 @@ static int apple_pmc_status_show(struct seq_file *s, void *unused) seq_printf(s, "static agents: %#x\n", pmc->static_agents); for (i = 0; i < hw->agents; i++) - seq_printf(s, "agent%u floor: %#010x\n", i, - readl(pmc->voters + i * hw->agent_stride)); + seq_printf(s, "agent%u floor: %#010x honoured: %#010x\n", i, + readl(pmc->voters + i * hw->agent_stride), + readl(pmc->rails + hw->agent_state + 4 * i)); for (i = 0; i < hw->rails; i++) - seq_printf(s, "rail%u state: %#010x enable: %#010x (saved %#010x)\n", - i, readl(pmc->rails + hw->rail_state + 4 * i), - apple_pmc_enable_read(pmc, i), pmc->saved_enable[i]); + seq_printf(s, "rail%u enable: %#010x (saved %#010x)\n", + i, apple_pmc_enable_read(pmc, i), pmc->saved_enable[i]); return 0; } @@ -461,7 +463,7 @@ static const struct apple_pmc_hw apple_pmc_hw_t8140 = { .agent_stride = 0x4000, .agents = 4, .rails = 4, - .rail_state = 0x0, + .agent_state = 0x0, .rail_enable = 0x2000, }; From 351fc1af825c49264de7e171a983d2c0c83f1601 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:48:27 +1100 Subject: [PATCH 0755/1579] dt-bindings: power: apple,pmc: describe the DCS limit The second register region holds one word per voter agent with the floor the PMC honours for it, followed by one interface enable word per rail; say so instead of calling the first words rail states. The PMC can also limit the DRAM controller (DCS) rail on behalf of the host, which makes it a thermal cooling device. Add #cooling-cells, and an optional power model for the power allocator governor: the system power at each DCS state above the lowest one. Bound the static voter list by the 32 agents an enable word can hold. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/power/apple,pmc.yaml | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index 69cdc1cb87197..91c77695178c4 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -29,7 +29,9 @@ properties: reg: items: - description: Perf-state floor words of the voter agents - - description: Rail state and interface enable words + - description: | + Per-agent words holding the floor the PMC honours for each agent, + and per-rail interface enable words reg-names: items: @@ -48,6 +50,26 @@ properties: apple,static-voters: $ref: /schemas/types.yaml#/definitions/uint32-array description: Voter agents to enable on every rail. + minItems: 1 + maxItems: 32 + items: + maximum: 31 + + '#cooling-cells': + const: 2 + description: | + The PMC can limit the DRAM controller (DCS) rail. Cooling state 0 leaves + the rail to the agents' votes; each further state lowers the highest + DCS state the PMC honours for them by one, down to the lowest state. + + apple,dcs-power-microwatt: + description: | + Power the system draws at each DCS performance state above what it + draws at the lowest one, from the lowest state up, measured under a + load whose speed follows the DCS state. With it, the DCS limit is a + power actor that the power allocator thermal governor can drive. + minItems: 2 + maxItems: 16 patternProperties: '^voter@[0-9a-f]+$': @@ -115,7 +137,9 @@ examples: reg-names = "voters", "rails"; #address-cells = <1>; #size-cells = <0>; + #cooling-cells = <2>; apple,static-voters = <0 1 3>; + apple,dcs-power-microwatt = <0 919000 1453000 2058000 2407000>; performance-controller@1 { compatible = "apple,t8140-clpc"; From cf8edfc522ce82871acc421119e324b56ab2813f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:48:30 +1100 Subject: [PATCH 0756/1579] pmdomain: apple: pmc: limit the DCS rail as a cooling device Once the PMP is told that the GPU is on, it votes the DRAM controller (DCS) up with GPU load. On the J700 that is worth up to 2.4x in GPU throughput, but it also lets the GPU and the memory system draw up to 2.4 W more, and nothing on the host can hold that back: the CPU cooling devices do not touch it. Register the PMC as a cooling device that limits the DCS rail. At cooling state N, only the AP's own agent stays enabled on the DCS rail, and it votes what the other enabled agents ask for, capped N states below the top. A delayed work re-reads their votes every 20 ms while a limit is in place, so the rail still follows demand, including down to 0 when the GPU idles. The capped vote is in place before the others stop being honoured and is withdrawn only once they are honoured again, so the rail never dips on a change of state. Enables that the static and dynamic agents request while a limit is in place are recorded and take effect once it is lifted. With a power model in the device tree, the cooling device is a power actor for the power allocator governor: it requests the power of the state the other agents ask for and converts a granted power into the highest DCS state that fits. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmc.c | 289 ++++++++++++++++++++++++++++++++++- 1 file changed, 283 insertions(+), 6 deletions(-) diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index fac0b1dfc02b6..397fce7fc8ad2 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -17,6 +17,11 @@ * and models each dynamic agent as a power domain that sits between the * device and its PMGR power state, which gives the same ordering. * + * Under thermal pressure the driver can also limit the DRAM controller + * (DCS) rail: it stops honouring the other agents on that rail and votes + * on their behalf through the AP's own agent, capped. This bounds the heat + * of the PMP's DCS vote, which follows GPU load. + * * Copyright The Asahi Linux Contributors */ @@ -32,9 +37,15 @@ #include #include #include +#include +#include #define APPLE_PMC_MAX_RAILS 8 #define APPLE_PMC_MAX_AGENTS 32 +#define APPLE_PMC_MAX_STATES 16 + +/* How often a DCS limit re-reads the votes it stands in for. */ +#define APPLE_PMC_DCS_POLL_MS 20 /** * struct apple_pmc_hw - per-SoC layout of the PMC voter interface @@ -47,6 +58,10 @@ * agent's interface enables and clamped to each rail's highest state * @rail_enable: offset of the per-rail interface enable words in the * "rails" region + * @dcs_rail: rail of the DRAM controller (DCS) + * @dcs_max: highest DCS performance state + * @ap_agent: voter agent of the application processor, which a DCS limit + * votes through */ struct apple_pmc_hw { u32 agent_stride; @@ -54,6 +69,9 @@ struct apple_pmc_hw { u32 rails; u32 agent_state; u32 rail_enable; + u32 dcs_rail; + u32 dcs_max; + u32 ap_agent; }; struct apple_pmc { @@ -61,12 +79,22 @@ struct apple_pmc { const struct apple_pmc_hw *hw; void __iomem *voters; void __iomem *rails; - /* Serializes read-modify-write cycles on enable and floor words. */ + /* + * Serializes read-modify-write cycles on enable and floor words, and + * protects @enable and @dcs_state. + */ raw_spinlock_t lock; u32 saved_enable[APPLE_PMC_MAX_RAILS]; + /* Agents enabled on each rail, before any DCS limit is applied. */ + u32 enable[APPLE_PMC_MAX_RAILS]; u32 static_agents; struct list_head domains; struct dentry *debugfs; + /* Cooling state of the DCS limit: 0 is none, dcs_max holds DCS at 0. */ + unsigned long dcs_state; + /* Power cost of each DCS state in mW, if a model is described. */ + u32 dcs_power[APPLE_PMC_MAX_STATES]; + struct delayed_work dcs_work; }; struct apple_pmc_domain { @@ -88,14 +116,18 @@ static u32 apple_pmc_enable_read(struct apple_pmc *pmc, unsigned int rail) return readl(pmc->rails + pmc->hw->rail_enable + 4 * rail); } -/* Caller holds pmc->lock. Every write is read back, as macOS does. */ -static int apple_pmc_enable_update(struct apple_pmc *pmc, unsigned int rail, - u32 set, u32 clear) +/* + * Caller holds pmc->lock. While the DCS rail is limited, only the AP's agent + * is enabled on it. Every write is read back, as macOS does. + */ +static int apple_pmc_enable_write(struct apple_pmc *pmc, unsigned int rail) { void __iomem *reg = pmc->rails + pmc->hw->rail_enable + 4 * rail; u32 old = readl(reg); - u32 val = (old | set) & ~clear; + u32 val = pmc->enable[rail]; + if (pmc->dcs_state && rail == pmc->hw->dcs_rail) + val = BIT(pmc->hw->ap_agent); if (val == old) return 0; @@ -110,6 +142,15 @@ static int apple_pmc_enable_update(struct apple_pmc *pmc, unsigned int rail, return 0; } +/* Caller holds pmc->lock. */ +static int apple_pmc_enable_update(struct apple_pmc *pmc, unsigned int rail, + u32 set, u32 clear) +{ + pmc->enable[rail] = (pmc->enable[rail] | set) & ~clear; + + return apple_pmc_enable_write(pmc, rail); +} + static int apple_pmc_agent_enable(struct apple_pmc *pmc, unsigned int agent, bool enable) { @@ -226,6 +267,226 @@ int apple_pmc_floor_update(struct device *dev, unsigned int agent, u32 mask, } EXPORT_SYMBOL_GPL(apple_pmc_floor_update); +/* + * Caller holds pmc->lock. The highest DCS state that the agents enabled on + * the DCS rail ask for, the AP's own agent excepted. + */ +static unsigned int apple_pmc_dcs_demand(struct apple_pmc *pmc) +{ + const struct apple_pmc_hw *hw = pmc->hw; + unsigned long agents = pmc->enable[hw->dcs_rail] & ~BIT(hw->ap_agent); + unsigned int agent, demand = 0; + u32 floor; + + for_each_set_bit(agent, &agents, hw->agents) { + floor = readl(pmc->voters + agent * hw->agent_stride); + demand = max(demand, (floor & APPLE_PMC_RAIL_MASK(hw->dcs_rail)) >> + APPLE_PMC_RAIL_SHIFT(hw->dcs_rail)); + } + + return min(demand, hw->dcs_max); +} + +/* + * Caller holds pmc->lock. Vote through the AP's agent for what the other + * agents ask for, capped by the limit; vote nothing without a limit. + */ +static void apple_pmc_dcs_vote(struct apple_pmc *pmc) +{ + const struct apple_pmc_hw *hw = pmc->hw; + void __iomem *reg = pmc->voters + hw->ap_agent * hw->agent_stride; + u32 mask = APPLE_PMC_RAIL_MASK(hw->dcs_rail); + unsigned int state = 0; + u32 old, val; + + if (pmc->dcs_state) + state = min_t(unsigned int, apple_pmc_dcs_demand(pmc), + hw->dcs_max - pmc->dcs_state); + + old = readl(reg); + val = (old & ~mask) | (state << APPLE_PMC_RAIL_SHIFT(hw->dcs_rail)); + if (val != old) + writel(val, reg); +} + +static void apple_pmc_dcs_work(struct work_struct *work) +{ + struct apple_pmc *pmc = container_of(to_delayed_work(work), + struct apple_pmc, dcs_work); + unsigned long flags; + bool limited; + + raw_spin_lock_irqsave(&pmc->lock, flags); + apple_pmc_dcs_vote(pmc); + limited = pmc->dcs_state; + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + if (limited) + queue_delayed_work(system_freezable_power_efficient_wq, + &pmc->dcs_work, + msecs_to_jiffies(APPLE_PMC_DCS_POLL_MS)); +} + +static int apple_pmc_dcs_get_max_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + struct apple_pmc *pmc = cdev->devdata; + + *state = pmc->hw->dcs_max; + return 0; +} + +static int apple_pmc_dcs_get_cur_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + struct apple_pmc *pmc = cdev->devdata; + + *state = READ_ONCE(pmc->dcs_state); + return 0; +} + +static int apple_pmc_dcs_set_cur_state(struct thermal_cooling_device *cdev, + unsigned long state) +{ + struct apple_pmc *pmc = cdev->devdata; + unsigned int rail = pmc->hw->dcs_rail; + unsigned long flags; + int ret = 0; + + if (state > pmc->hw->dcs_max) + return -EINVAL; + + raw_spin_lock_irqsave(&pmc->lock, flags); + if (state != pmc->dcs_state) { + /* + * Vote the capped demand before the other agents stop being + * honoured, and drop that vote only once they are honoured + * again, so that the rail never dips below either. + */ + WRITE_ONCE(pmc->dcs_state, state); + if (state) { + apple_pmc_dcs_vote(pmc); + ret = apple_pmc_enable_write(pmc, rail); + } else { + ret = apple_pmc_enable_write(pmc, rail); + apple_pmc_dcs_vote(pmc); + } + } + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + if (state) + queue_delayed_work(system_freezable_power_efficient_wq, + &pmc->dcs_work, + msecs_to_jiffies(APPLE_PMC_DCS_POLL_MS)); + else + cancel_delayed_work(&pmc->dcs_work); + + return ret; +} + +static int apple_pmc_dcs_get_requested_power(struct thermal_cooling_device *cdev, + u32 *power) +{ + struct apple_pmc *pmc = cdev->devdata; + unsigned long flags; + unsigned int demand; + + raw_spin_lock_irqsave(&pmc->lock, flags); + demand = apple_pmc_dcs_demand(pmc); + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + *power = pmc->dcs_power[demand]; + return 0; +} + +static int apple_pmc_dcs_state2power(struct thermal_cooling_device *cdev, + unsigned long state, u32 *power) +{ + struct apple_pmc *pmc = cdev->devdata; + + if (state > pmc->hw->dcs_max) + return -EINVAL; + + *power = pmc->dcs_power[pmc->hw->dcs_max - state]; + return 0; +} + +static int apple_pmc_dcs_power2state(struct thermal_cooling_device *cdev, + u32 power, unsigned long *state) +{ + struct apple_pmc *pmc = cdev->devdata; + unsigned long s; + + for (s = 0; s < pmc->hw->dcs_max; s++) + if (pmc->dcs_power[pmc->hw->dcs_max - s] <= power) + break; + + *state = s; + return 0; +} + +static const struct thermal_cooling_device_ops apple_pmc_dcs_ops = { + .get_max_state = apple_pmc_dcs_get_max_state, + .get_cur_state = apple_pmc_dcs_get_cur_state, + .set_cur_state = apple_pmc_dcs_set_cur_state, +}; + +static const struct thermal_cooling_device_ops apple_pmc_dcs_power_ops = { + .get_max_state = apple_pmc_dcs_get_max_state, + .get_cur_state = apple_pmc_dcs_get_cur_state, + .set_cur_state = apple_pmc_dcs_set_cur_state, + .get_requested_power = apple_pmc_dcs_get_requested_power, + .state2power = apple_pmc_dcs_state2power, + .power2state = apple_pmc_dcs_power2state, +}; + +/* + * Register the DCS limit as a cooling device. With a power model it is a + * power actor that the power allocator governor can drive. + */ +static int apple_pmc_dcs_init(struct apple_pmc *pmc) +{ + const struct thermal_cooling_device_ops *ops = &apple_pmc_dcs_ops; + const struct apple_pmc_hw *hw = pmc->hw; + struct device *dev = pmc->dev; + struct device_node *np = dev->of_node; + struct thermal_cooling_device *cdev; + u32 uw[APPLE_PMC_MAX_STATES]; + unsigned int i; + int count, ret; + + if (!IS_ENABLED(CONFIG_THERMAL) || + !of_property_present(np, "#cooling-cells")) + return 0; + + count = of_property_count_u32_elems(np, "apple,dcs-power-microwatt"); + if (count > 0) { + if (count != hw->dcs_max + 1) + return dev_err_probe(dev, -EINVAL, + "need %u DCS power values\n", + hw->dcs_max + 1); + ret = of_property_read_u32_array(np, "apple,dcs-power-microwatt", + uw, count); + if (ret) + return ret; + for (i = 0; i < count; i++) { + pmc->dcs_power[i] = uw[i] / 1000; + if (i && pmc->dcs_power[i] < pmc->dcs_power[i - 1]) + return dev_err_probe(dev, -EINVAL, + "DCS power must not decrease\n"); + } + ops = &apple_pmc_dcs_power_ops; + } + + cdev = devm_thermal_of_cooling_device_register(dev, np, "pmc-dcs", pmc, + ops); + if (IS_ERR(cdev)) + return dev_err_probe(dev, PTR_ERR(cdev), + "failed to register the DCS limit\n"); + + return 0; +} + static int apple_pmc_domain_power_on(struct generic_pm_domain *genpd) { struct apple_pmc_domain *dom = genpd_to_apple_pmc_domain(genpd); @@ -342,6 +603,8 @@ static int apple_pmc_status_show(struct seq_file *s, void *unused) for (i = 0; i < hw->rails; i++) seq_printf(s, "rail%u enable: %#010x (saved %#010x)\n", i, apple_pmc_enable_read(pmc, i), pmc->saved_enable[i]); + seq_printf(s, "dcs limit: state %lu of %u\n", READ_ONCE(pmc->dcs_state), + hw->dcs_max); return 0; } @@ -361,10 +624,14 @@ static void apple_pmc_release(void *data) apple_pmc_domain_free(dom); } + /* The cooling device is gone by now; drop any limit it left. */ + cancel_delayed_work_sync(&pmc->dcs_work); raw_spin_lock_irqsave(&pmc->lock, flags); for (rail = 0; rail < pmc->hw->rails; rail++) writel(pmc->saved_enable[rail], pmc->rails + pmc->hw->rail_enable + 4 * rail); + pmc->dcs_state = 0; + apple_pmc_dcs_vote(pmc); raw_spin_unlock_irqrestore(&pmc->lock, flags); } @@ -388,8 +655,10 @@ static int apple_pmc_probe(struct platform_device *pdev) INIT_LIST_HEAD(&pmc->domains); if (pmc->hw->rails > APPLE_PMC_MAX_RAILS || - pmc->hw->agents > APPLE_PMC_MAX_AGENTS) + pmc->hw->agents > APPLE_PMC_MAX_AGENTS || + pmc->hw->dcs_max >= APPLE_PMC_MAX_STATES) return -EINVAL; + INIT_DELAYED_WORK(&pmc->dcs_work, apple_pmc_dcs_work); pmc->voters = devm_platform_ioremap_resource_byname(pdev, "voters"); if (IS_ERR(pmc->voters)) @@ -404,6 +673,7 @@ static int apple_pmc_probe(struct platform_device *pdev) if (pmc->saved_enable[rail] == ~0U) return dev_err_probe(dev, -ENODEV, "PMC is not responding\n"); + pmc->enable[rail] = pmc->saved_enable[rail]; } count = of_property_count_u32_elems(np, "apple,static-voters"); @@ -448,6 +718,10 @@ static int apple_pmc_probe(struct platform_device *pdev) } } + ret = apple_pmc_dcs_init(pmc); + if (ret) + return ret; + pmc->debugfs = debugfs_create_dir(dev_name(dev), apple_pmc_debugfs_root); debugfs_create_file("status", 0400, pmc->debugfs, pmc, &apple_pmc_status_fops); @@ -465,6 +739,9 @@ static const struct apple_pmc_hw apple_pmc_hw_t8140 = { .rails = 4, .agent_state = 0x0, .rail_enable = 0x2000, + .dcs_rail = 0, + .dcs_max = 4, + .ap_agent = 3, }; static const struct of_device_id apple_pmc_of_match[] = { From 99bf4fc4c39cbe59493531b7682f0a7eb60ac2e0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:48:31 +1100 Subject: [PATCH 0757/1579] pmdomain: apple: build the PMC driver by default The J700 device tree relies on the PMC driver for its DCS thermal limit and for letting the host performance controller's floors through, so a kernel built without it runs hotter under GPU load. Build it by default on ARCH_APPLE, as the T8140 memory cache controller driver is; it binds to nothing on SoCs without a PMC node. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/Kconfig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index 7145a073429aa..2494e7aece065 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -25,11 +25,13 @@ config APPLE_PMC depends on OF depends on PM select PM_GENERIC_DOMAINS + default ARCH_APPLE help The PMC in some Apple SoCs arbitrates the performance states of shared rails, such as the DRAM controller and the SoC fabric, between voter agents. This driver enables the static agents and provides power domains that enable a dynamic agent, such as the - GPU, while its device is powered on. + GPU, while its device is powered on. It can also limit the DRAM + controller rail as a thermal cooling device. endif From d0f012d7b92f9967d42c35a70bc09c2b87274c0b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:48:58 +1100 Subject: [PATCH 0758/1579] soc: apple: clpc: do not count offline CPUs as busy An offline CPU accrues no idle time while the wall clock keeps running, so its idle delta is zero and the sampler counted it as 100% busy. With any CPU of a policy offline, that policy then requested its top DCS state on every sample for as long as the CPU stayed down. Skip offline CPUs, and take a fresh baseline for a CPU when it comes back instead of charging it with the whole time it was away. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/clpc.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 32b88c7c968ba..02a9df787a642 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -183,12 +183,26 @@ static void apple_clpc_sample_cluster(struct apple_clpc *clpc, unsigned int cpu, busy = 0, q, rail; for_each_cpu(cpu, cl->cpus) { - u64 wall, idle = get_cpu_idle_time(cpu, &wall, 0); - u64 dw = wall - clpc->wall[cpu], di = idle - clpc->idle[cpu]; + u64 wall, idle, dw, di; + bool valid; + + /* + * An offline CPU accrues no idle time while the wall clock runs + * on, so it would look fully busy. Skip it, and take a fresh + * baseline once it is back. + */ + if (!cpu_online(cpu)) { + clpc->wall[cpu] = 0; + continue; + } + idle = get_cpu_idle_time(cpu, &wall, 0); + dw = wall - clpc->wall[cpu]; + di = idle - clpc->idle[cpu]; + valid = clpc->wall[cpu]; clpc->wall[cpu] = wall; clpc->idle[cpu] = idle; - if (dw && di <= dw) + if (valid && dw && di <= dw) busy = max_t(unsigned int, busy, div64_u64((dw - di) * 100, dw)); } From ddc216cd42cb7c7ebc831d3fafca17df244b280e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:49:13 +1100 Subject: [PATCH 0759/1579] soc: apple: clpc: stop waking idle CPUs and release the floor in sleep The sampler ran every 4 ms on a high-priority, non-freezable workqueue with a normal timer. That woke an idle CPU 250 times a second even when no vote was held, kept running through system sleep, and, once tasks are frozen and sampling stops, would leave the agent's floor at whatever the last sample before suspend asked for, typically a busy one, for the whole sleep. Sample on a freezable, power-efficient workqueue. While a vote is held, keep the normal timer so the vote still drops on time after demand stops; with no vote held, use a deferrable timer that waits for a CPU that is awake anyway. Before system sleep, stop sampling and give the floor back; resume sampling from fresh baselines afterwards. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/clpc.c | 94 +++++++++++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 10 deletions(-) diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 02a9df787a642..2d66b98c3a9be 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -109,7 +109,10 @@ struct apple_clpc { unsigned int agent; struct regmap *gpu_regmap; u32 gpu_offset; + /* Samples while a vote is held; idle_work defers to busy CPUs. */ struct delayed_work work; + struct delayed_work idle_work; + bool stopped; struct apple_clpc_cluster clusters[APPLE_CLPC_MAX_CLUSTERS]; unsigned int nr_clusters; cpumask_var_t covered; @@ -262,10 +265,23 @@ static u32 apple_clpc_floor(struct apple_clpc *clpc, const unsigned int *state) return floor; } -static void apple_clpc_work(struct work_struct *work) +static void apple_clpc_queue(struct apple_clpc *clpc, bool idle) +{ + if (READ_ONCE(clpc->stopped)) + return; + + /* + * A held vote has to be sampled on time so that it drops once demand + * stops; with no vote held, sampling can wait for a CPU that wakes up + * anyway rather than waking an idle one every period. + */ + queue_delayed_work(system_freezable_power_efficient_wq, + idle ? &clpc->idle_work : &clpc->work, + msecs_to_jiffies(sample_ms)); +} + +static void apple_clpc_sample(struct apple_clpc *clpc) { - struct apple_clpc *clpc = container_of(to_delayed_work(work), - struct apple_clpc, work); unsigned int demand[APPLE_CLPC_RAILS] = {}; unsigned int state[APPLE_CLPC_RAILS]; unsigned long now = jiffies; @@ -305,8 +321,19 @@ static void apple_clpc_work(struct work_struct *work) apple_clpc_floor(clpc, state))) clpc->update_errors++; - queue_delayed_work(system_highpri_wq, &clpc->work, - msecs_to_jiffies(sample_ms)); + apple_clpc_queue(clpc, !apple_clpc_floor(clpc, state)); +} + +static void apple_clpc_work(struct work_struct *work) +{ + apple_clpc_sample(container_of(to_delayed_work(work), struct apple_clpc, + work)); +} + +static void apple_clpc_idle_work(struct work_struct *work) +{ + apple_clpc_sample(container_of(to_delayed_work(work), struct apple_clpc, + idle_work)); } static int apple_clpc_status_show(struct seq_file *s, void *unused) @@ -422,14 +449,33 @@ static void apple_clpc_free_clusters(struct apple_clpc *clpc) free_cpumask_var(clpc->covered); } +/* Stop sampling and give the floor back, before suspend and on removal. */ +static void apple_clpc_halt(struct apple_clpc *clpc) +{ + unsigned int rail, i; + + /* + * Each pass queues one of the two works unless stopped. A pass that + * began before the flag was set can still queue the other work once, + * which the second round cancels. + */ + WRITE_ONCE(clpc->stopped, true); + for (i = 0; i < 2; i++) { + cancel_delayed_work_sync(&clpc->work); + cancel_delayed_work_sync(&clpc->idle_work); + } + apple_pmc_floor_update(clpc->pmc, clpc->agent, clpc->mask, + clpc->saved_floor); + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + clpc->vote[rail].state = 0; +} + static void apple_clpc_stop(void *data) { struct apple_clpc *clpc = data; - cancel_delayed_work_sync(&clpc->work); + apple_clpc_halt(clpc); debugfs_remove_recursive(clpc->debugfs); - apple_pmc_floor_update(clpc->pmc, clpc->agent, clpc->mask, - clpc->saved_floor); apple_clpc_free_clusters(clpc); } @@ -504,6 +550,7 @@ static int apple_clpc_probe(struct platform_device *pdev) apple_clpc_find_clusters(clpc); clpc->next_scan = jiffies + HZ; INIT_DELAYED_WORK(&clpc->work, apple_clpc_work); + INIT_DEFERRABLE_WORK(&clpc->idle_work, apple_clpc_idle_work); clpc->debugfs = debugfs_create_dir(dev_name(dev), NULL); debugfs_create_file("status", 0400, clpc->debugfs, clpc, @@ -515,11 +562,37 @@ static int apple_clpc_probe(struct platform_device *pdev) if (ret) return ret; - queue_delayed_work(system_highpri_wq, &clpc->work, - msecs_to_jiffies(sample_ms)); + platform_set_drvdata(pdev, clpc); + apple_clpc_queue(clpc, true); return 0; } +/* + * Tasks are frozen before devices suspend, and with them the sampling, so + * the floor would otherwise stay at whatever the last busy sample asked for + * through the whole sleep. + */ +static int apple_clpc_suspend(struct device *dev) +{ + apple_clpc_halt(dev_get_drvdata(dev)); + return 0; +} + +static int apple_clpc_resume(struct device *dev) +{ + struct apple_clpc *clpc = dev_get_drvdata(dev); + unsigned int cpu; + + for_each_possible_cpu(cpu) + clpc->wall[cpu] = 0; + WRITE_ONCE(clpc->stopped, false); + apple_clpc_queue(clpc, true); + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(apple_clpc_pm_ops, apple_clpc_suspend, + apple_clpc_resume); + /* T8140: DCS states 0-4 on rail 0, fabric states 0-3 on rail 1. */ static const struct apple_clpc_hw apple_clpc_hw_t8140 = { .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, @@ -537,6 +610,7 @@ static struct platform_driver apple_clpc_driver = { .driver = { .name = "apple-clpc", .of_match_table = apple_clpc_of_match, + .pm = pm_sleep_ptr(&apple_clpc_pm_ops), }, }; module_platform_driver(apple_clpc_driver); From defdcb8017744fe0aa57061ed304e793935b52bc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 22:24:11 +1100 Subject: [PATCH 0760/1579] soc: apple: build the CLPC floors by default Without a floor from the host, the J700's DRAM controller stays at its lowest state for CPU work: nothing else votes it up unless the GPU is busy. A single thread copying a 256 MiB buffer moves 6.88 GB/s (sd 0.02, three runs of 12) without the CLPC floors and 28.4 GB/s (sd 0.1-0.3) with them, the floor following the copy to DCS 4. Build the floors by default on ARCH_APPLE; the driver binds only to a host performance controller node under a PMC. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 39036f1565403..49152c8b5e1f0 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -168,6 +168,7 @@ config APPLE_CLPC depends on ARCH_APPLE || COMPILE_TEST depends on APPLE_PMC && CPU_FREQ select MFD_SYSCON + default ARCH_APPLE help Raises the DRAM controller and SoC fabric performance floors of the host's PMC voter agent while CPU clusters, and optionally the GPU, From ae8e780a5b7dedcb4a31adbd106fb9f9dbff8be0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:49:16 +1100 Subject: [PATCH 0761/1579] arm64: dts: apple: t8140: describe the PMC's DCS limit The PMC can limit the DRAM controller rail as a cooling device, so give it cooling cells. Also correct the comment on its second register region, which holds the floor the PMC honours for each agent and the interface enable word of each rail rather than per-rail states. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index edd81b9b10c22..db2caf6194a6b 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1224,8 +1224,9 @@ }; /* - * Perf-state floors of voter agents 0-3, and the rail state and - * interface enable words, in the PMC window at 0x302800000. + * Perf-state floors of voter agents 0-3, and the floor the PMC + * honours for each agent and the interface enable word of each + * rail, in the PMC window at 0x302800000. */ pmc: power-management@302800000 { compatible = "apple,t8140-pmc"; @@ -1235,6 +1236,7 @@ power-domains = <&ps_pmc>; #address-cells = <1>; #size-cells = <0>; + #cooling-cells = <2>; /* PMP, CLPC and AP agents */ apple,static-voters = <0 1 3>; status = "disabled"; From a037985ac9605eb76fe51a05b172b022ac1f0005 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:49:17 +1100 Subject: [PATCH 0762/1579] arm64: dts: apple: t8140-j700: limit the DCS vote when the die runs hot Once the PMP knows that the GPU is on, it votes the DRAM controller (DCS) up with GPU load. On a GPU-bound load the CPU zones cannot hold the resulting heat back, because the CPUs request little power: with the performance governor the CPU die sensors passed 95 C within 35 s and reached 104 C before the CPUs throttled. Add a zone that limits that vote through the PMC. Its sensor, the SMC's Tp0t, reads at or above every CPU die sensor under GPU load. Describe the measured system power of each DCS state so that the power allocator can drive the limit, and make the sustainable power the whole DCS range, so that the vote is only trimmed above the control trip. With the zone, three minutes of mixed GL and Vulkan load under the performance governor peak at 97 C on the CPU zones, as the same load does without any DCS vote, and ten minutes of it under schedutil keep within 1-2% of the throughput of the uncontrolled vote. Tp0t becomes channel 6 of the SMC hwmon so that the zone can refer to it by index, as the CPU zones do. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 63 ++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index e433db4472941..1a06bce2f9dc9 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -379,6 +379,52 @@ }; }; }; + + /* + * Once the PMP knows the GPU is on, it votes the DRAM controller + * (DCS) up with GPU load, and the GPU and memory system draw up + * to 2.4 W more. The CPU zones above cannot hold that back: on a + * GPU-bound load the CPUs request little power, so with the + * performance governor the die passed 95 C within 35 s and + * reached 104 C before they throttled. This zone limits the DCS + * vote instead, through the PMC. With it the same load peaks at + * 97 C, as it does without any DCS vote, and a ten-minute + * balanced GPU load loses 1-2% of its throughput. + * + * Tp0t reads at or above every CPU die sensor under GPU load, + * and leads them by about 1 C when the CPUs also run hot. The + * sustainable power is the whole DCS range, so the vote is only + * trimmed above the control trip; the PID gains follow from it + * and the 6 C between the trips. + */ + soc-dcs-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 6>; /* Tp0t */ + sustainable-power = <2407>; + + trips { + soc_dcs_alert: trip-alert { + temperature = <89000>; + hysteresis = <2000>; + type = "passive"; + }; + + soc_dcs_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&soc_dcs_control>; + cooling-device = + <&pmc THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; }; }; @@ -421,6 +467,11 @@ }; &pmc { + /* + * System power above DCS state 0 at each DCS state, measured with + * glmark2's terrain scene, GPU-bound, on the J700. + */ + apple,dcs-power-microwatt = <0 919000 1453000 2058000 2407000>; status = "okay"; }; @@ -622,6 +673,13 @@ * float, reads 10-99 C and moves between samples. The names follow * the SMC key groups and are not otherwise verified. */ + + /* Channel 6, the sensor of soc-dcs-thermal: keep it first here. */ + temperature-Tp0t { + apple,key-id = "Tp0t"; + label = "SMC Tp0t"; + }; + temperature-TH0p { apple,key-id = "TH0p"; label = "Storage (NAND)"; @@ -898,11 +956,6 @@ label = "SMC Tp0q"; }; - temperature-Tp0t { - apple,key-id = "Tp0t"; - label = "SMC Tp0t"; - }; - temperature-Tpx0 { apple,key-id = "Tpx0"; label = "SMC Tpx0"; From a64a4771b27602bf5a72b413b6bc9a868892992a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:12:37 +1100 Subject: [PATCH 0763/1579] iommu: apple-dart: invalidate T8110 ranges by device DVA The range invalidate for locked T8110 v2.2+ streams passes the IOVA masked to the page-table input width. The TLB range registers match the DVA the device actually issues, though. On T8140 the display and DCP streams sit in a window at 1 TiB (apple,dma-range <0x100 0x0 0x10 0x0>), so the masked range never matches a cached translation there: - a range invalidate over [0, 64 GiB) during the boot scanout does nothing observable, even after 2500 repetitions; - the same range over [1 TiB, 1 TiB + 64 GiB) hits the scanout's live translations at once. Every unmap on these streams therefore left its translations in the TLB. Invalidate the DART-visible range, IOVA plus the DART offset, as the map path already uses it. Windows below the table width are unaffected, because masking was a no-op there. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 38075e072f18e..1e4fec909643b 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -1144,10 +1144,12 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, dev_err_ratelimited(stream.dart->dev, "failed to publish host page-table entries: %d\n", ret); else { + /* The range registers take the DVA the device issues. */ if (range && stream.dart->locked && stream.dart->version >= 0x0202) ret = apple_dart_t8110_hw_tlb_command_range(&stream, DART_T8110_TLB_CMD_OP_FLUSH_SID, true, - first & domain->mask, last & domain->mask); + first + domain->dma_offset, + last + domain->dma_offset); else ret = stream.dart->hw->invalidate_tlb(&stream); } From dd5ac6fdffb03d048df510198af6e8d53dceb17d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:14:04 +1100 Subject: [PATCH 0764/1579] iommu: apple-dart: leave live firmware translations out of invalidates On T8140 the boot loader locks the display DART, and the display controller keeps scanning out the boot framebuffer through the firmware translations while Linux attaches the default domain. The attach ends with a whole-stream TLB invalidate. Now and then the next table walk after that invalidate reports an invalid PTE, although the firmware entry is valid and unchanged: apple-dart 412300000.iommu: translation fault: SID:0 read PTE invalid at 0x10002a20800 (status 0x00003200) The display coprocessor then panics on the failed read, apple-dcp cannot boot RTKit (-ETIME), and the panel stays on simpledrm without backlight control. This hit about one boot in twenty. During the boot scanout, 500 whole-stream invalidates reproduce the fault within 30 ms, with the faulting leaf entry still valid when read back. One invalidate by DVA range over the scanout's window faults as well, with "STT mismatch". So any invalidate of translations that the hardware is using can break the scanout. Linux never changes what a firmware-owned root slot resolves to: it publishes only into vacant slots, and accepts a firmware slot only when its own mappings match it. On DARTs that keep firmware roots (T8140 and T8142), invalidate by DVA range only the parts of the stream's window that are reached through other root slots. That covers the whole-stream invalidate at attach and teardown, and the range invalidates on map and unmap. Slots that Linux retires are invalidated while the root is still known. Clear the root pointer and the firmware snapshot under the same lock, so that an invalidate never sees one without the other. Four-level streams, older DART revisions and other SoCs keep their current invalidates. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 96 +++++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 1e4fec909643b..4cc5aac4f97c2 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -709,6 +709,9 @@ apple_dart_snapshot_fw_root(struct apple_dart *dart, int sid, const u64 *live) return fw; } +static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, + u64 lo, u64 hi); + static int apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) { @@ -775,14 +778,20 @@ apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx if (!live) continue; apple_dart_retire_root(live, owned, dart->pgsize / sizeof(*live)); + if (dart->fw_mirror && dart->version >= 0x0202 && !idx) { + struct apple_dart_stream_map stream = { .dart = dart }; + + __set_bit(sid, stream.sidmap); + (void)apple_dart_fw_invalidate(&stream, 0, U64_MAX); + } spin_lock_irqsave(&dart->lock, flags); fw = dart->locked_fw[sid][idx]; dart->locked_fw[sid][idx] = NULL; + dart->locked_ttbr[sid][idx] = NULL; spin_unlock_irqrestore(&dart->lock, flags); apple_dart_free_fw_root(dart, fw); devm_memunmap(dart->dev, live); devm_kfree(dart->dev, owned); - dart->locked_ttbr[sid][idx] = NULL; dart->locked_owned[sid][idx] = NULL; } return 0; @@ -952,6 +961,84 @@ apple_dart_t8110_hw_tlb_command_range(struct apple_dart_stream_map *stream_map, return ret; } +/* A root table is at most 16 KiB of 64-bit entries. */ +#define DART_MAX_ROOT_ENTRIES (SZ_16K / sizeof(u64)) + +/* + * On DARTs that keep firmware roots (T8140 and T8142), the hardware keeps + * using the translations behind the firmware-owned root slots, e.g. the + * display controller scanning out the boot framebuffer while Linux attaches. + * Invalidating those translations, for the whole stream or by DVA range, can + * make the scanout fail a translation, and the display coprocessor panics on + * the failed read. Linux never changes what those slots resolve to, so only + * invalidate the parts of [lo, hi] reached through other root slots. + */ +static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, + u64 lo, u64 hi) +{ + const u32 op = DART_T8110_TLB_CMD_OP_FLUSH_SID; + struct apple_dart *dart = stream_map->dart; + struct apple_dart_stream_map stream = { .dart = dart }; + size_t entries = dart->pgsize / sizeof(u64); + u32 slot_shift = ilog2(dart->pgsize) + ilog2(entries); + u64 span = BIT_ULL(slot_shift + ilog2(entries)); + u64 base = (dart->dma_min + dart->dma_offset) & ~(span - 1); + DECLARE_BITMAP(fw_slots, DART_MAX_ROOT_ENTRIES); + size_t first, last, i, end; + unsigned long flags; + int sid, ret = 0; + + if (entries > DART_MAX_ROOT_ENTRIES || hi < base || lo > base + span - 1) + return 0; + lo = max(lo, base); + hi = min(hi, base + span - 1); + first = (lo - base) >> slot_shift; + last = (hi - base) >> slot_shift; + + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + const struct apple_dart_fw_root *fw; + const u64 *live; + + __set_bit(sid, stream.sidmap); + if (apple_dart_readl(dart, DART_TCR(dart, sid)) & + dart->hw->tcr_4level) { + ret = apple_dart_t8110_hw_tlb_command_range(&stream, op, + false, 0, 0); + goto next; + } + + /* Nothing of Linux reaches the hardware without the root. */ + bitmap_zero(fw_slots, entries); + spin_lock_irqsave(&dart->lock, flags); + live = dart->locked_ttbr[sid][0]; + fw = dart->locked_fw[sid][0]; + for (i = first; live && i <= last; i++) + if (apple_dart_fw_slot_owned(fw, live, i)) + __set_bit(i, fw_slots); + spin_unlock_irqrestore(&dart->lock, flags); + if (!live) + goto next; + + for (i = find_next_zero_bit(fw_slots, last + 1, first); i <= last; + i = find_next_zero_bit(fw_slots, last + 1, end + 1)) { + u64 start, stop; + + end = find_next_bit(fw_slots, last + 1, i) - 1; + start = max(lo, base + ((u64)i << slot_shift)); + stop = min(hi, base + ((u64)(end + 1) << slot_shift) - 1); + ret = apple_dart_t8110_hw_tlb_command_range(&stream, op, true, + start, stop); + if (ret) + break; + } +next: + __clear_bit(sid, stream.sidmap); + if (ret) + return ret; + } + return 0; +} + static int apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) { @@ -962,6 +1049,8 @@ apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) static int apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) { + if (stream_map->dart->fw_mirror && stream_map->dart->version >= 0x0202) + return apple_dart_fw_invalidate(stream_map, 0, U64_MAX); return apple_dart_t8110_hw_tlb_command_range( stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID, false, 0, 0); } @@ -1145,7 +1234,10 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, "failed to publish host page-table entries: %d\n", ret); else { /* The range registers take the DVA the device issues. */ - if (range && stream.dart->locked && stream.dart->version >= 0x0202) + if (range && stream.dart->fw_mirror && stream.dart->version >= 0x0202) + ret = apple_dart_fw_invalidate(&stream, first + domain->dma_offset, + last + domain->dma_offset); + else if (range && stream.dart->locked && stream.dart->version >= 0x0202) ret = apple_dart_t8110_hw_tlb_command_range(&stream, DART_T8110_TLB_CMD_OP_FLUSH_SID, true, first + domain->dma_offset, From 4f13ab717cb3a5ab19e5c097084ff2c676457f37 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:06:36 +1100 Subject: [PATCH 0765/1579] pmdomain: apple: pmp-report: keep entry domains bound Each PMP report entry is a power-domain provider, but unlike the report driver itself the entry driver still exposes the sysfs bind and unbind attributes. Unbinding an entry whose domain has consumers (the display, camera or video decoder, or the GPU on t8112) cannot remove the domain: pm_genpd_remove() fails while devices are attached, so the entry keeps an inert domain whose power callbacks return -ENODEV, and the consumers can no longer runtime-resume until the next boot. Before commit 4f3e0ed14a9c ("pmdomain: apple: drain PMP report users before teardown") added a .remove callback, the same unbind left the domain pointing at freed memory. Suppress the bind attributes, as the report driver already does. The .remove callback stays for OF depopulation of the entries. Fixes: e5f8c27073d6 ("pmdomain: apple: Add PMP reporting interface") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmp-report.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 2f17df448a1a8..318cefe61a89c 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -502,6 +502,8 @@ static struct platform_driver apple_pmp_report_entry_driver = { .driver = { .name = "apple-pmp-report-entry", .of_match_table = apple_pmp_report_entry_of_match, + /* Power-domain providers are not intended for manual unbinding. */ + .suppress_bind_attrs = true, }, }; From 5a651e466b5fb95e84e1f8278e8e2ac38c88695e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:07:58 +1100 Subject: [PATCH 0766/1579] drm: apple: check only the lengths of a legacy reply record Commit 2b8d51be4f0a ("drm: apple: locate legacy command replies in the saved record") validates a reply on firmware older than H17P by comparing the whole record header, tag included, with the one that was sent. The lengths are what the check before it compared; requiring the firmware to leave the tag alone as well is a new assumption, and it was only confirmed by traces of 13.5 firmware on t8112 and t6000. The same path serves the 12.3 and 13.3 firmware interfaces. If either rewrote the tag when writing its reply, every ack would fail the check, the DCP would be marked crashed during start-up, and the display would stay dark. Compare only the in and out lengths, which is enough to tell that the record was overwritten. Fixes: 2b8d51be4f0a ("drm: apple: locate legacy command replies in the saved record") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 3ee1e95482d19..d563e747dcd7c 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -878,13 +878,15 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context) * acked on the callback context, whose receive area holds the callback * record rather than the reply, so the ack cannot be used to find it. * - * Older firmware leaves the record header untouched. If it no longer - * matches what was sent, the record was overwritten and its output - * cannot be trusted. + * On older firmware the lengths in the record header must still be + * the ones that were sent; otherwise the record was overwritten and + * its output cannot be trusted. The tag is left out of the check: it + * has only been seen preserved on 13.5 firmware. */ if (dcp->fw_compat != DCP_FIRMWARE_H17P) { record = out - sent->in_len - sizeof(*record); - if (memcmp(record, sent, sizeof(*record))) { + if (record->in_len != sent->in_len || + record->out_len != sent->out_len) { dev_err(dcp->dev, "corrupt %c%c%c%c command record on context %X\n", sent->tag[3], sent->tag[2], sent->tag[1], From 3ec50f1cc9dd81b526f19f300e8b244ab7a69c3f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:44 +1100 Subject: [PATCH 0767/1579] nvme: apple: keep the post-M4 ANS link domains powered across sleep A post-M4 ANS cannot be started again once CPU_CONTROL.RUN is cleared, so apple_nvme_suspend() only disables the NVMe controller and leaves the coprocessor running. The running firmware keeps using its PCIe link, whose power domains on T8140 are apcie_sys_st and apcie_phy_sw. genpd powers a domain off in the noirq phase once every device attached to it has suspended, whatever the runtime PM state. The controller holds its domains through per-domain virtual devices, which suspend right after the controller, so both link domains went down underneath the running ANS. On the J700, a pm_test=platform cycle, which holds the devices suspended for five seconds, then hung the SoC during resume until the watchdog reset it. Put the devices through which the controller holds its domains on the awake path when the ANS is left running. The pmgr power-state driver marks these boot-active domains GENPD_FLAG_ACTIVE_WAKEUP, so genpd keeps them powered until resume, as it already does for the domains of the display coprocessor. With this change the same cycle resumes with the root filesystem intact, and genpd records no power-off of either link domain across it or across real suspend-to-idle cycles. Controllers before M4 shut the ANS down and are not affected. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 431a128b81a2c..f6c18221ded3e 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -2582,6 +2582,28 @@ static int apple_nvme_resume(struct device *dev) return nvme_reset_ctrl(&anv->ctrl); } +/* + * The ANS stays up across system sleep on post-M4 hardware, and it keeps + * using its PCIe link while it runs. genpd, however, switches a domain off + * in the noirq phase once every device attached to it has suspended, no + * matter what runtime PM says. Put the devices that hold the controller's + * domains on the awake path so that genpd leaves those domains powered + * until resume. + */ +static void apple_nvme_keep_link_powered(struct apple_nvme *anv) +{ + int i; + + /* A single domain is attached to the controller device itself. */ + if (anv->pd_count <= 1) { + device_set_awake_path(anv->dev); + return; + } + + for (i = 0; i < anv->pd_count; i++) + device_set_awake_path(anv->pd_dev[i]); +} + static int apple_nvme_suspend(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); @@ -2601,6 +2623,7 @@ static int apple_nvme_suspend(struct device *dev) */ if (anv->hw->needs_ioq_registers) { cancel_work_sync(&anv->recovery_work); + apple_nvme_keep_link_powered(anv); return 0; } From 51bb5b50bbc8e92116933b3f582cb4becc3bf210 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:08:33 +1100 Subject: [PATCH 0768/1579] drm: apple: complete a legacy command whose reply record is corrupt When the reply record of a command on firmware older than H17P fails validation, dcpep_handle_ack() marks the DCP crashed and returns without calling the command's callback. The callback owns a reference to its cookie: for power and modeset commands that is a wait cookie whose caller then waits out its full timeout (1 to 10 seconds), and the reference is never dropped, so each such command leaks its cookie. The start-up wait in dcp_wait_ready() likewise runs to its timeout when the is_main_display reply is the one rejected. Call the callback with no output instead. The callbacks used on this firmware either ignore their output or already treat a missing one as a failed command, except two that dereferenced it unconditionally: make dcp_swapped() report a failed swap, as it already does on H17P, and make res_is_main_display() end the start-up wait, which then returns -ENODEV for the crashed DCP. Fixes: c529eadaf40a ("drm: apple: retain displaced framebuffers until confirmed completion") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 7 ++++++- drivers/gpu/drm/apple/iomfb_template.c | 14 +++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index d563e747dcd7c..8c8cfbb5cc579 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -892,7 +892,12 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context) sent->tag[3], sent->tag[2], sent->tag[1], sent->tag[0], context); WRITE_ONCE(dcp->crashed, true); - return; + /* + * Still complete the command, without an output, so + * that its callback drops the cookie reference and + * wakes the waiter instead of leaving it to time out. + */ + out = NULL; } } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index e9001d7af82d9..78d95c072ffe7 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2006,11 +2006,7 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER u32 status = resp ? resp->ret : ~0U; -#else - u32 status = resp->ret; -#endif if (!dcp_present_submit(dcp, swap_id, !status)) status = ~0U; @@ -2367,11 +2363,19 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) { struct apple_connector *connector; + bool result; + + /* No reply: the DCP is marked crashed, so only end the start-up wait. */ + if (!out) { + complete(&dcp->start_done); + return; + } + /* * The reply is a one-byte boolean. The firmware leaves the rest of * the 32-bit slot alone, and dcp_push() fills that with 0xff. */ - bool result = *(u8 *)out; + result = *(u8 *)out; dev_info(dcp->dev, "DCP is_main_display: %d\n", result); From c1bae63595e8fc7b18e79664ee6aa6b4536f5d95 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:46 +1100 Subject: [PATCH 0769/1579] arm64: dts: apple: t8140: Keep the GPU power state on across sleep The GPU driver leaves the GPU firmware running across system sleep, but once the GPU device has suspended nothing keeps the gfx power state in use, and genpd switches it off in the noirq phase. The firmware does not survive that for long. On the J700, a pm_test=platform cycle that holds the devices suspended for five seconds comes back with working GPU rendering, but holding them for a minute, or a real 30 or 90 second suspend-to-idle, hangs the SoC during resume until the watchdog resets it. With the GPU driver unbound, the one-minute cycle resumes. Mark the state always-on, as ANS already is. Runtime behaviour does not change: the GPU device never runtime suspends, so genpd only ever switches this state off for system sleep. Keeping it on also matches the PMP report entry for the GPU, which already reports the GPU as on from probe. With the state kept on, the one-minute cycle and real 30 and 90 second suspend-to-idle cycles woken by the RTC all resume, and the GPU renders afterwards. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 93c0bebbb1b82..6e6098144b757 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -56,6 +56,11 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "gfx"; + /* + * The GPU firmware keeps running across system sleep and + * does not survive this state going off for long. + */ + apple,always-on; }; ps_venc0_sys: power-controller@240 { From b4d89a96d6b0823770e787a16e55d7c8af84274a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:48 +1100 Subject: [PATCH 0770/1579] wifi: mt7932: keep the function running across suspend-to-idle The driver has no dev_pm_ops, so the PCI core's default system-suspend callback clears bus mastering on the function while its firmware keeps running. On the J700 a pm_test=devices cycle then hangs the SoC until the watchdog resets it, every time; with the function removed, the same cycle resumes. Stopping the firmware for sleep would take the remove and probe sequence on every suspend: retiring its DMA, a function-level reset and a new firmware start. Leave it running across suspend-to-idle instead. Saving the PCI state in .suspend keeps the PCI core from changing the function's power state or command register, and putting the device on the awake path keeps the PCIe controller's power state on until resume. Refuse any other sleep state. With this change, the pm_test=devices cycle, a one-minute pm_test=platform cycle and real 30 and 90 second suspend-to-idle cycles all resume with the station still associated and passing traffic, and genpd records no power-off of the controller's power state. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 85440558e95a4..4ca56767f671f 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -5,6 +5,9 @@ */ #include "mt7932.h" +#include +#include + static int mt_ring_alloc(struct mt7932 *m, struct mt7932_ring *q, unsigned int n, bool rx, unsigned int count); static void mt_rx_post(struct mt7932_ring *q, unsigned int index); @@ -1279,6 +1282,25 @@ static void mt_shutdown(struct pci_dev *pdev) /* Storage remains allocated through reset, even if the engine is stuck. */ } +/* + * Stopping the firmware for sleep would take the whole remove and probe + * sequence, so leave it running across suspend-to-idle. A saved state keeps + * the PCI core from changing the function's power state or command register, + * and the awake path keeps the PCIe controller powered until resume. + */ +static int mt_suspend(struct device *dev) +{ + if (!pm_suspend_no_platform()) + return -EOPNOTSUPP; + pci_save_state(to_pci_dev(dev)); + device_set_awake_path(dev); + return 0; +} + +static const struct dev_pm_ops mt_pm_ops = { + .suspend = mt_suspend, +}; + static const struct pci_device_id mt_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7932) }, {} @@ -1292,6 +1314,7 @@ static struct pci_driver mt_driver = { .remove = mt_remove, .shutdown = mt_shutdown, .driver.suppress_bind_attrs = true, + .driver.pm = pm_sleep_ptr(&mt_pm_ops), }; module_pci_driver(mt_driver); MODULE_LICENSE("GPL"); From c80113dfaab9e985b9f617bfde06c481fd17376c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:49 +1100 Subject: [PATCH 0771/1579] arm64: dts: apple: t8140-j700: give the two RTCs fixed numbers The J700 has two RTCs on the same Baku PMU counter: the SMC RTC, which sets the clock, and the PMU RTC, which provides the wake alarm and cannot set the time. Both drivers are modules and register in whichever order they happen to load, so either one can end up as rtc0. rtc0 is what hctosys, NTP's RTC sync and tools such as hwclock and rtcwake use by default, and setting the time through the PMU RTC fails. Over six boots of the same kernel, the PMU RTC registered first, as rtc0, on four, and set the system clock. Add rtc aliases so that the SMC RTC is always rtc0 and the PMU RTC rtc1. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 2 ++ arch/arm64/boot/dts/apple/t8140.dtsi | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 1a06bce2f9dc9..842e1b2bcf572 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -26,6 +26,8 @@ sep = &sep; serial0 = &serial0; pmp = &pmp; + rtc0 = &smc_rtc; + rtc1 = &pmu_rtc; wifi0 = &wifi0; /* used by the loader to hand over the display */ dcp = &dcp; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index db2caf6194a6b..cc071157054b8 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -832,7 +832,7 @@ "boot_error_count", "panic_count"; }; - rtc { + smc_rtc: rtc { compatible = "apple,smc-rtc"; nvmem-cells = <&rtc_offset>; nvmem-cell-names = "rtc_offset"; From 809042160372b84371b4b690466a173a1bc0d8e7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:51 +1100 Subject: [PATCH 0772/1579] rtc: apple-pmu: take over an alarm that firmware left armed The probe refused to bind with -EBUSY whenever it found the alarm armed, so that it would not overwrite a wake scheduled by firmware. On the J700 the alarm is armed at every boot, with the comparator at about 2^32 - 1 day in wall-clock seconds, a time in early 2106. Nothing is due then, but the check means the driver never binds. Keep the alarm only when it is unmasked and due within the range the driver supports, so that the RTC core picks it up at registration. Otherwise disarm it, as probe already did for an alarm it found disarmed. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/rtc/rtc-apple-pmu.c | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index a684233205803..8c7e4156b015f 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -247,6 +247,32 @@ static irqreturn_t baku_rtc_interrupt(int irq, void *context) return IRQ_HANDLED; } +/* + * Firmware leaves the alarm armed even when it has no wake scheduled, with + * the comparator set close to the end of 32-bit time. Keep an alarm that is + * still pending within the supported range, so that the RTC core picks it + * up at registration, and disarm anything else. + */ +static int baku_rtc_init_alarm(struct baku_rtc *baku) +{ + unsigned int control, mask; + __le32 comparator; + int error; + + error = regmap_read(baku->map, BAKU_RTC_CONTROL, &control); + if (!error) + error = regmap_read(baku->map, BAKU_RTC_MASK, &mask); + if (!error) + error = regmap_bulk_read(baku->map, BAKU_RTC_COMPARE, + &comparator, sizeof(comparator)); + if (error) + return error; + if ((control & BAKU_RTC_ARM) && !(mask & BAKU_RTC_ALARM) && + !baku_rtc_validate_alarm(baku, le32_to_cpu(comparator))) + return 0; + return baku_rtc_stop_alarm(baku); +} + static const struct rtc_class_ops baku_rtc_ops = { .read_time = baku_rtc_read_time, .read_alarm = baku_rtc_read_alarm, @@ -284,7 +310,6 @@ static int baku_rtc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct baku_rtc *baku; - unsigned int control; int error; /* Retention has only been measured on J700 with the explicit opt-in. */ @@ -304,12 +329,6 @@ static int baku_rtc_probe(struct platform_device *pdev) return baku->interrupt; mutex_init(&baku->transaction); platform_set_drvdata(pdev, baku); - error = regmap_read(baku->map, BAKU_RTC_CONTROL, &control); - if (error) - return error; - /* Respect an alarm left armed by firmware. */ - if (control & BAKU_RTC_ARM) - return -EBUSY; baku->clock = devm_rtc_allocate_device(dev); if (IS_ERR(baku->clock)) return PTR_ERR(baku->clock); @@ -332,7 +351,7 @@ static int baku_rtc_probe(struct platform_device *pdev) dev_name(dev), dev); if (error) goto disable_wakeup; - error = baku_rtc_stop_alarm(baku); + error = baku_rtc_init_alarm(baku); if (error) goto disable_wakeup; error = devm_rtc_register_device(baku->clock); From eb4213570886631fbcac8057b5d6ff07a1c07d6c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:52 +1100 Subject: [PATCH 0773/1579] rtc: apple-pmu: enable the alarm without a module parameter The driver only probed with experimental=1, because alarm delivery during suspend-to-idle had not been measured. Nothing on that path is switched off by s2idle on the J700: the SPMI controller and the AIC sit in always-on power domains, the SPMI controller's chained interrupt is never disabled by the suspend core, and the alarm interrupt is requested with IRQF_NO_SUSPEND and reports a hard wakeup. Measured on the J700: alarms set 30 and 90 seconds ahead ended suspend-to-idle 29.8 to 30.5 and 90.3 seconds after it began. Drop the parameter so that the PMU RTC binds by default. It is the only wake source on the J700 that does not need a person at the machine. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/rtc/Kconfig | 8 ++++---- drivers/rtc/rtc-apple-pmu.c | 7 ------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index 5488a2dd5c86e..e6077d109afb4 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -2104,10 +2104,10 @@ config RTC_DRV_APPLE_PMU tristate "Apple J700 Baku PMU RTC alarm" depends on ARCH_APPLE && NVMEM_APPLE_SPMI && PM_SLEEP help - Read the Baku PMU clock and use alarms with retained-power - suspend-to-idle. The module requires experimental=1. Alarms are - limited to one hour; deeper suspend states are not supported. - Continue setting the shared clock through the SMC RTC. + Read the Baku PMU clock and use its alarm to wake the system + from suspend-to-idle. Alarms are limited to one hour; deeper + suspend states are not supported. Continue setting the shared + clock through the SMC RTC. This driver can be built as the module rtc-apple-pmu. diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index 8c7e4156b015f..2e3161a914c8a 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -24,10 +24,6 @@ #define BAKU_RTC_HZ_SHIFT 16 #define BAKU_RTC_MAX_DELAY 3600 -static bool experimental; -module_param(experimental, bool, 0444); -MODULE_PARM_DESC(experimental, "Enable alarms with retained SPMI/AIC power in s2idle"); - struct baku_rtc { struct regmap *map; struct nvmem_cell *offset; @@ -312,9 +308,6 @@ static int baku_rtc_probe(struct platform_device *pdev) struct baku_rtc *baku; int error; - /* Retention has only been measured on J700 with the explicit opt-in. */ - if (!experimental) - return -ENODEV; baku = devm_kzalloc(dev, sizeof(*baku), GFP_KERNEL); if (!baku) return -ENOMEM; From 0c07089bbdf7e63b81124e439160141dbcafa0e1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:12:11 +1100 Subject: [PATCH 0774/1579] drm: apple: unmap a released DCP register range The DCP firmware asks for display register ranges to be mapped through map_physical and gives them back through release_mem_desc. The address mapped with dma_map_resource() is returned to the firmware but never stored, and releasing the descriptor only clears it, so every released range leaves its mapping in place: the IOVA space stays allocated and the translation onto the registers stays live. A firmware that maps and releases a range repeatedly grows the leak for as long as the system runs. H17P firmware has been able to reach this path since its map_physical callback was admitted. Record the mapped address and unmap the range when its descriptor is released. The reserved descriptor 0 has no range and is left alone. Fixes: f2bc92ff49a9 ("drm/apple: Add DCP display driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index e9001d7af82d9..3d6b3c86f8a57 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -756,9 +756,13 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) dma_free_coherent(dcp->dev, size, memdesc->buf, memdesc->dva); memdesc->buf = NULL; memset(&memdesc->map, 0, sizeof(memdesc->map)); - } else { + } else if (memdesc->size) { + /* A register range mapped by map_physical */ + dma_unmap_resource(dcp->dev, memdesc->dva, memdesc->size, + DMA_BIDIRECTIONAL, 0); memdesc->reg = 0; } + memdesc->dva = 0; memdesc->size = 0; @@ -819,6 +823,7 @@ dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) set_bit(id, dcp->memdesc_map); dcp->memdesc[id].size = size; dcp->memdesc[id].reg = req->paddr; + dcp->memdesc[id].dva = dva; return (struct dcp_map_physical_resp){ .dva_size = size, From 67091aa86554621624a2daf46f465e983a5b5ff1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:14:46 +1100 Subject: [PATCH 0775/1579] drm: apple: present a backlight write while the H17P level is unknown Without a loader brightness report the H17P backlight registers with the middle of the panel range as its reported level, while the panel keeps the loader's level until a present carries one. dcp_bl_request() only marks a new request dirty when it differs from the target, so a write of exactly the reported level, as systemd-backlight makes when it restores a level saved on an earlier boot without a report, presents nothing. The panel then stays at the loader's level, the reported level stays wrong, and the first DPMS or suspend cycle jumps to it. Also mark a request dirty while the level is not known, so the first write is always presented and makes the level known. Add a KUnit case. Fixes: 34cd8f3032b3 ("drm: apple: register the H17P backlight without a loader level") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.h | 6 +++- drivers/gpu/drm/apple/dcp_backlight_test.c | 36 ++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 2c9a0a7ab97be..1232ff40a84ba 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -110,8 +110,12 @@ static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, return -ERANGE; state->controlled = true; state->retries = 0; + /* + * Until a present carries a level, the panel does not show the target, + * so even a request for the reported default has to be presented. + */ if (state->target != nits || state->core_blank != core_blank || - state->suspended != suspended) { + state->suspended != suspended || !state->level_known) { state->target = nits; state->core_blank = core_blank; state->suspended = suspended; diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index a0aaad3ddd232..41ac118f26cee 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -292,6 +292,41 @@ static void backlight_fallback_test(struct kunit *test) KUNIT_EXPECT_EQ(test, present.nits, 262U); } +static void backlight_fallback_same_level_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* A userspace write of the reported default is still presented. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 262, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 262U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_TRUE(test, state.level_known); + + /* Once the level is known, repeating it presents nothing. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 262, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &present), -EALREADY); + + /* A rejected present leaves it unknown, so the same write is retried. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 262, false, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, false)); + KUNIT_EXPECT_FALSE(test, state.level_known); + state.dirty = false; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 262, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.dirty); + + /* An inherited level is known from the start. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, true, 180, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 180, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.dirty); +} + static void backlight_seed_known_test(struct kunit *test) { struct dcp_backlight_state state; @@ -329,6 +364,7 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_inflight_dpms_test), KUNIT_CASE(backlight_resend_test), KUNIT_CASE(backlight_fallback_test), + KUNIT_CASE(backlight_fallback_same_level_test), KUNIT_CASE(backlight_seed_known_test), {} }; From de4bea8e895d1bc8a72c366711cd768a93fd1862 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:18:29 +1100 Subject: [PATCH 0776/1579] drm: apple: do not report an aborted H17P present as a failed swap When H17P firmware aborts a present before its submit reply arrives, dcp_present_submit() logs the abort, rate-limited, and the reply handler then turns the accepted reply into a failure and logs it again, without a rate limit, as "swap failed! status 4294967295". A firmware that aborted every frame would log that error once per frame, with a status the firmware never returned. Let dcp_present_submit() return why a present does not stand, and skip the failure message for an abort that has already been reported. The present is still finished as a failure. Fixes: 239607cb12c3 ("drm: apple: finish H17P presents that the firmware aborts") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 29 +++++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index e9001d7af82d9..7744393249d54 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1398,24 +1398,29 @@ static bool dcp_present_begin(struct apple_dcp *dcp, u32 swap_id) return true; } -static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted) +/* + * Returns 0 if the submitted present stands, -ECANCELED if the firmware + * aborted it before replying (already reported), or -EPROTO if no present + * was expecting this reply. + */ +static int dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted) { #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !dcp_present_submit_h17p(&dcp->present_state_h17p, swap_id, accepted)) { dev_err(dcp->dev, "unexpected present submission %u\n", swap_id); dcp->crashed = true; - return false; + return -EPROTO; } /* An abort received before this reply turns acceptance into failure. */ if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && accepted && dcp->present_state_h17p.aborted) { dev_warn_ratelimited(dcp->dev, "firmware aborted present %u\n", swap_id); - return false; + return -ECANCELED; } #endif - return true; + return 0; } static void dcp_prepare_clear_swap(struct apple_dcp *dcp, void *request) @@ -1461,12 +1466,16 @@ static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) struct dcp_swap_cookie *info = cookie; u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; u32 status = resp ? resp->ret : ~0U; + int ret; - if (!dcp_present_submit(dcp, swap_id, !status)) + ret = dcp_present_submit(dcp, swap_id, !status); + if (ret) status = ~0U; if (status) { - dev_err(dcp->dev, "swap_clear failed! status %u\n", status); + /* An aborted present has already been reported. */ + if (ret != -ECANCELED) + dev_err(dcp->dev, "swap_clear failed! status %u\n", status); dcp_present_failed(dcp); if (dcp_present_retires(dcp)) { dcp_unarm_retained_framebuffers(dcp, swap_id); @@ -2004,6 +2013,7 @@ TRAMPOLINE_OUT(trampoline_create_backlight_service, dcpep_cb_create_backlight_se static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) { + int ret; struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER @@ -2012,11 +2022,14 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) u32 status = resp->ret; #endif - if (!dcp_present_submit(dcp, swap_id, !status)) + ret = dcp_present_submit(dcp, swap_id, !status); + if (ret) status = ~0U; if (status) { - dev_err(dcp->dev, "swap failed! status %u\n", status); + /* An aborted present has already been reported. */ + if (ret != -ECANCELED) + dev_err(dcp->dev, "swap failed! status %u\n", status); dcp_present_failed(dcp); if (dcp_present_retires(dcp)) { dcp_unarm_retained_framebuffers(dcp, swap_id); From b4b3efea0a116f7f1f9dac2b4305d9794017b364 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:18:52 +1100 Subject: [PATCH 0777/1579] drm: apple: check for a dead DCP after each H17P sleep drain pass Commit 744c2c7abc15 ("drm: apple: skip the H17P sleep drain on a crashed or stopped DCP") checks for a crashed or stopped DCP at the start of each drain pass. A crash during the last pass is not seen: the queue drain then reports the DCP idle at once, the level is still pending, and the loop ends with the "brightness change still pending at suspend" warning that commit set out to avoid. Returning before the first pass also skips the queue drain, whose lock orders the suspend after a queued transaction that is still being started. Check after the queue drain instead, before the pending test. A crashed or stopped DCP now always goes through the drain, which returns at once for it. Fixes: 744c2c7abc15 ("drm: apple: skip the H17P sleep drain on a crashed or stopped DCP") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index b5eaac2964445..2096f86022e09 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3844,12 +3844,6 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) * the first attempt and the policy's three retries. */ for (pass = 0; pass < 4; pass++) { - /* - * A crashed or stopped DCP takes no more presents, so a level - * still pending there can never be sent and is not waited for. - */ - if (READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.stopped)) - return; flush_delayed_work(&dcp->iomfb.backlight_retry); if (dcp_has_panel(dcp)) flush_work(&dcp->bl_update_wq); @@ -3857,6 +3851,14 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) dev_warn(dcp->dev, "display updates still pending at suspend\n"); return; } + /* + * A crashed or stopped DCP takes no more presents, so a level + * still pending there can never be sent and is not waited for. + * Checking after the drain, which returns at once for such a + * DCP, also covers a crash during the pass. + */ + if (READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.stopped)) + return; if (!dcp_backlight_active(dcp) || !dcp_backlight_pending(dcp)) return; } From 826c3346b3ca39728f568e5f6ff0ec78e6c6199d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:23:20 +1100 Subject: [PATCH 0778/1579] wifi: mt7932: report whether the Wi-Fi function initialized When initialization fails and the DMA of function 0 cannot be stopped, mt7932-fullmac keeps the binding to retain that DMA. Whether the Wi-Fi firmware is running therefore cannot be told from the binding, which is what the Bluetooth transport on function 1 waits for before it starts its own firmware on the shared radio. Record when the probe has finished initialization and export mt7932_fullmac_ready() to report it. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/net/wireless/mediatek/mt7932/mt7932.c | 24 +++++++++++++++++++ drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 ++ include/linux/mt7932.h | 14 +++++++++++ 4 files changed, 41 insertions(+) create mode 100644 include/linux/mt7932.h diff --git a/MAINTAINERS b/MAINTAINERS index 72b6e43702b5d..99987f9eca95b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1975,6 +1975,7 @@ F: Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml F: Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml F: drivers/pmdomain/apple/pmp-report.c F: drivers/soc/apple/pmp.rs +F: include/linux/mt7932.h F: include/linux/soc/apple/pmp-report.h F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 85440558e95a4..fa759a964e06c 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -3,6 +3,7 @@ * Clean-room contract: NEO_MT7932_DOWNLOAD_AND_NIC_CAP_CONTRACT.md. * All addresses here are BAR0 offsets, never chip/CPU physical addresses. */ +#include #include "mt7932.h" static int mt_ring_alloc(struct mt7932 *m, struct mt7932_ring *q, @@ -1251,6 +1252,8 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = mt_reset_retained(&m); if (!ret) ret = mt_register_regulatory_gate(m); + if (!ret) + WRITE_ONCE(m->ready, true); retain: if (ret) { int error = ret; @@ -1293,6 +1296,27 @@ static struct pci_driver mt_driver = { .shutdown = mt_shutdown, .driver.suppress_bind_attrs = true, }; + +/** + * mt7932_fullmac_ready() - check that the Wi-Fi function is up + * @pdev: function 0 of the radio + * + * A Wi-Fi function whose initialization failed stays bound when its DMA + * could not be stopped, so being bound does not mean its firmware runs. + * + * Return: true if @pdev is bound to this driver and finished initialization. + */ +bool mt7932_fullmac_ready(struct pci_dev *pdev) +{ + struct mt7932 *m; + + if (!device_is_bound(&pdev->dev) || pdev->dev.driver != &mt_driver.driver) + return false; + m = pci_get_drvdata(pdev); + return m && READ_ONCE(m->ready); +} +EXPORT_SYMBOL_GPL(mt7932_fullmac_ready); + module_pci_driver(mt_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("J700 MT7932 firmware-owned cfg80211 station driver"); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index f6fc1f235e619..93e7fa5629acb 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -76,6 +76,8 @@ struct mt7932 { u64 interrupts, packets; bool running; bool stopping; + /* Probe finished initialization; the Bluetooth function may start. */ + bool ready; bool irq_requested, vectors_allocated, wiphy_registered; bool dma_owned; struct completion reset_retry; diff --git a/include/linux/mt7932.h b/include/linux/mt7932.h new file mode 100644 index 0000000000000..9ab60efa49786 --- /dev/null +++ b/include/linux/mt7932.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Interface between the functions of the J700 MT7932 radio. + */ +#ifndef __LINUX_MT7932_H +#define __LINUX_MT7932_H + +#include + +struct pci_dev; + +bool mt7932_fullmac_ready(struct pci_dev *pdev); + +#endif /* __LINUX_MT7932_H */ From f2e1106bad9f035aa078e4aff0f745c97a429231 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:24:23 +1100 Subject: [PATCH 0779/1579] Bluetooth: mt7932: start only after the Wi-Fi function initialized The Bluetooth transport waits until function 0 is bound to mt7932-fullmac, so that its firmware starts after the Wi-Fi firmware as in the tested sequence. Being bound does not mean the Wi-Fi firmware is running: when the Wi-Fi initialization fails and its DMA cannot be stopped, mt7932-fullmac keeps the binding to retain that DMA. Bluetooth then probes anyway and starts its firmware on the shared radio next to a failed Wi-Fi function. Once function 0 is bound, also require mt7932_fullmac_ready(). The Wi-Fi probe finishes initialization before it returns, so a bound function that is not ready has failed, and the Bluetooth probe fails with -ENODEV. The transport now calls into the Wi-Fi module, so it depends on MT7932_FULLMAC plainly: it can no longer be built in while the Wi-Fi driver is a module. Fixes: 920e89a2e3df ("Bluetooth: mt7932: wait for the Wi-Fi function before starting") Assisted-by: LLM Signed-off-by: Ryan Murray --- .../networking/device_drivers/wifi/mt7932-neo.rst | 3 ++- drivers/bluetooth/Kconfig | 11 ++++++----- drivers/bluetooth/mt7932_bt_pcie.c | 5 +++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 0387cb98098a3..220eff6fbbcad 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -53,7 +53,8 @@ performance policy. ``mt7932_bt_pcie`` defers its probe until ``mt7932-fullmac`` is bound to function 0, so the Bluetooth firmware always starts after the Wi-Fi firmware, as in the tested sequence, however the drivers are loaded. Bluetooth therefore -stays unbound while the Wi-Fi driver is missing or has failed. A device link +stays unbound while the Wi-Fi driver is missing or has failed, including when +a failed Wi-Fi function stays bound to keep DMA it could not stop. A device link makes the driver core unbind Bluetooth before Wi-Fi; unloading ``mt7932_bt_pcie`` does not affect Wi-Fi. The probe rejects a function that is not cold, which is what now prevents a reprobe after a failed or uncertain diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index da96e2f2c05e3..0456ec45b5ec0 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -27,14 +27,15 @@ config BT_MTK7932_PCIE tristate "J700 MT7932 PCIe Bluetooth transport" depends on BT && PCI && ARCH_APPLE && IOMMU_DMA depends on !KEXEC_CORE - depends on MT7932_FULLMAC != n + depends on MT7932_FULLMAC select FW_LOADER help Support the J700 Bluetooth PCIe function with locally installed firmware - and calibration. The function is probed once the Wi-Fi function is - bound to the mt7932-fullmac driver; the enable=0 parameter leaves it - unbound. Both functions appear before the root filesystem is mounted - and read their firmware when probed, so say M, like the Wi-Fi driver. + and calibration. The function is probed once the Wi-Fi function has + initialized under the mt7932-fullmac driver; the enable=0 parameter + leaves it unbound. Both functions appear before the root filesystem + is mounted and read their firmware when probed, so say M, like the + Wi-Fi driver. System suspend uses the firmware sleep handshake and preserves DMA storage. Hibernation and automatic firmware recovery are unsupported. Removal drains the PCI function before releasing device-visible memory; diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index f3ac19f810736..36655c25f88f5 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -1050,6 +1051,10 @@ static int bt7932_wait_for_wifi(struct pci_dev *pdev) strcmp(dev_driver_string(&wifi->dev), "mt7932-fullmac")) ret = dev_err_probe(&pdev->dev, -EPROBE_DEFER, "waiting for the Wi-Fi function\n"); + /* Initialization ends before the Wi-Fi probe returns. */ + else if (!mt7932_fullmac_ready(wifi)) + ret = dev_err_probe(&pdev->dev, -ENODEV, + "the Wi-Fi function failed to initialize\n"); else if (!device_link_add(&pdev->dev, &wifi->dev, DL_FLAG_AUTOREMOVE_CONSUMER)) ret = -EINVAL; pci_dev_put(wifi); From 5b02758233b31a2ebb49a5fb8a2fb472043cda55 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:27:33 +1100 Subject: [PATCH 0780/1579] wifi: mt7932: keep the country package's channel disables The channels a country package leaves out of the firmware domain are disabled only in cfg80211's channel flags, once per package pass. cfg80211 rebuilds those flags from the original ones on every regulatory update it applies, and then calls the notifier. The notifier skips an update whose snapshot matches the request already applied, so if an update ever leaves the snapshot unchanged, no pass runs and the disables are gone: the driver again offers channels its firmware domain excludes, where scans and joins fail. Do not depend on every such update producing a new snapshot. Record the channels each package pass disables and apply them again when the notifier finds the package still current. A new request clears the set; its own pass records a new one. Fixes: 6e3838e91a3b ("wifi: mt7932: honour the channel permissions of the country package") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 ++ .../net/wireless/mediatek/mt7932/startup.c | 29 ++++++++++++++----- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index f6fc1f235e619..57c90b7521cee 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -105,6 +105,8 @@ struct mt7932 { bool interface_registered, interface_up; u32 reg_generation, reg_attempted; struct mt7932_reg_snapshot reg_desired; + /* Channels the applied country package leaves out; under RTNL. */ + DECLARE_BITMAP(policy_disabled, MT7932_CHANNELS); struct net_device *netdev; struct wireless_dev wdev; struct cfg80211_scan_request *scan_request; diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 631a77aa684cd..225e5a0344fc4 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -223,9 +223,16 @@ static int mt_startup_once(struct mt7932 *m) return ret; } +static struct ieee80211_channel *mt_channel(struct mt7932 *m, unsigned int i) +{ + return i < MT7932_CHANNELS_2G ? &m->channels[i] : + &m->channels5[i - MT7932_CHANNELS_2G]; +} + /* Keep cfg80211's channel list in step with the channels the country - * package leaves out of the firmware domain. cfg80211 recomputes the flags - * on its next regulatory change, which runs this again. + * package leaves out of the firmware domain. cfg80211 rebuilds the flags + * on every regulatory update; the notifier applies the recorded set again + * when the update keeps this package, or starts a new pass otherwise. */ static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *policy, u32 generation) @@ -246,11 +253,11 @@ static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *poli } wiphy_lock(m->wiphy); for (i = 0; i < MT7932_CHANNELS; i++) { - struct ieee80211_channel *channel = i < MT7932_CHANNELS_2G ? - &m->channels[i] : &m->channels5[i - MT7932_CHANNELS_2G]; + struct ieee80211_channel *channel = mt_channel(m, i); + bool forbidden = !mt7932_policy_permits(policy, channel->hw_value); - if (!(channel->flags & IEEE80211_CHAN_DISABLED) && - !mt7932_policy_permits(policy, channel->hw_value)) + __assign_bit(i, m->policy_disabled, forbidden); + if (forbidden) channel->flags |= IEEE80211_CHAN_DISABLED; } wiphy_unlock(m->wiphy); @@ -375,8 +382,7 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request reg.domain[4] = 1; /* 2 GHz remains 20 MHz. */ reg.domain[5] = mt7932_domain_5g_bw(); for (i = 0; i < MT7932_CHANNELS; i++) { - struct ieee80211_channel *channel = i < MT7932_CHANNELS_2G ? - &m->channels[i] : &m->channels5[i - MT7932_CHANNELS_2G]; + struct ieee80211_channel *channel = mt_channel(m, i); u32 flags = channel->flags; reg.power[i] = channel->max_power; @@ -416,10 +422,17 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request } if (!memcmp(®, &m->reg_desired, sizeof(reg)) && !m->reg_pending) { spin_unlock_irqrestore(&m->response_lock, irqflags); + /* The applied package still holds, but cfg80211 has just + * rebuilt the channel flags without its disables. + */ + for_each_set_bit(i, m->policy_disabled, MT7932_CHANNELS) + mt_channel(m, i)->flags |= IEEE80211_CHAN_DISABLED; return; } m->reg_desired = reg; m->reg_generation++; + /* The next package pass records its own set. */ + bitmap_zero(m->policy_disabled, MT7932_CHANNELS); WRITE_ONCE(m->reg_pending, true); if (m->netdev) { netif_stop_queue(m->netdev); From cd0b91764e1e04a5e03464fd3e9a834480313251 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 05:43:54 +1100 Subject: [PATCH 0781/1579] media: apple: isp: Keep system sleep off resident firmware's domains The driver switches the power domains of resident firmware through runtime PM alone, because the firmware does not start again once they have been off. Their virtual genpd devices still take part in system sleep, though: genpd counts them as suspended in the noirq phase of suspend, switches the domains off when nothing else holds them, and switches every one of them back on in the noirq phase of resume. The firmware is stopped for system sleep, so on T8140 the CPU core domains of the stopped coprocessor fail to reach the active state on every resume, and genpd turns them off again when the resume completes: PS isp_cpu_core0: Failed to reach power state 0xf (now: 0xf000b4f) PS isp_cpu_core1: Failed to reach power state 0xf (now: 0xf000b4f) Mark the domain devices as syscore devices for as long as they are attached, so that system sleep leaves the domains to runtime PM. The Apple ACIO driver does the same for the domains of its coprocessor. SoCs whose firmware is started for each stream are unchanged. On J700 (T8140) with this change, suspend-to-idle with a PMU RTC wake no longer logs the errors; the camera is still stopped for sleep. Fixes: 62b659e9e5c7 ("media: apple: isp: Keep resident firmware running between streams") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index aaa809aeb1680..436ffb78af943 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -28,8 +28,11 @@ static void apple_isp_detach_genpd(struct apple_isp *isp) for (int i = isp->pd_count - 1; i >= 0; i--) { if (isp->pd_link[i]) device_link_del(isp->pd_link[i]); - if (!IS_ERR_OR_NULL(isp->pd_dev[i])) - dev_pm_domain_detach(isp->pd_dev[i], true); + if (IS_ERR_OR_NULL(isp->pd_dev[i])) + continue; + if (isp->hw->resident_fw) + dev_pm_syscore_device(isp->pd_dev[i], false); + dev_pm_domain_detach(isp->pd_dev[i], true); } return; @@ -74,6 +77,18 @@ static int apple_isp_attach_genpd(struct apple_isp *isp) apple_isp_detach_genpd(isp); return -EINVAL; } + + /* + * The domains of resident firmware are switched by the + * driver alone, through runtime PM: the firmware does not + * start again once they have been off. System sleep would + * otherwise also have genpd switch them off in the noirq + * phase of suspend and back on in the noirq phase of resume, + * where the CPU core domains of a stopped coprocessor fail + * to reach the active state. + */ + if (isp->hw->resident_fw) + dev_pm_syscore_device(isp->pd_dev[i], true); } return 0; From 8db122016f998ad82162dd2b0a4319a401210512 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 05:43:54 +1100 Subject: [PATCH 0782/1579] media: apple: isp: Keep resident firmware running across suspend Resident firmware is stopped for every system sleep, because its power domains used to go off under it. It does not start again, so on T8140, where suspend-to-idle now works, the camera is unavailable after the first suspend until the next boot, and every VIDIOC_STREAMON fails with -EIO. System sleep now leaves those domains to the driver, which keeps them on while the firmware runs. Keep the firmware running, idle, across suspend. The stream is stopped before the system sleeps and restarted when it resumes, as before, with its buffers kept. The awake path, which keeps the NVMe controller's link domains on across sleep, would not be enough here: genpd honours it only for a domain that was on when it was registered, and nine of the fourteen ISP domains are first switched on by this driver. Keeping the domains on has a cost. With the display off and the firmware idle, the J700 draws 30 to 50 mW more than with the firmware stopped and its domains off (3.553 W against 3.523 W, and 3.564 W against 3.512 W, means of 300 and 360 samples). The idle firmware raised no interrupt in 30 seconds, and the ISP DARTs, which are not locked, are reset and restored by their own resume callback while it is idle. SoCs whose firmware is started for each stream are unchanged. Hibernation still stops the firmware: the image is restored by another kernel, and firmware left running would go on using the memory of the kernel that started it. Tested on J700 with a PMU RTC wake: five suspend-to-idle cycles of 30 and 90 seconds. 1080p NV12 capture runs at 30 fps before and after each cycle. A capture left running across two of the cycles resumes by itself at 30 fps and completes all its frames. No ISP, power domain or DART errors are logged, and the ISP domains stay on throughout. Fixes: 62b659e9e5c7 ("media: apple: isp: Keep resident firmware running between streams") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 29 ++++++++++++++++++---- drivers/media/platform/apple/isp/isp-fw.c | 5 ++-- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 436ffb78af943..9f5f8f58b3748 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -801,14 +801,28 @@ static __maybe_unused int apple_isp_suspend(struct device *dev) apple_isp_video_suspend(isp); /* - * The ISP's power domains go off during system sleep, and resident - * firmware would not survive that. Stop it cleanly instead; resuming - * a stream then fails with an error until the next system boot. + * Resident firmware stays up, idle, across suspend-to-idle and + * suspend-to-RAM: its domains stay on, see apple_isp_attach_genpd(). + */ + return 0; +} + +static __maybe_unused int apple_isp_freeze(struct device *dev) +{ + struct apple_isp *isp = dev_get_drvdata(dev); + + apple_isp_video_suspend(isp); + + /* + * Hibernation hands the system from one kernel to another, and + * firmware left running would go on using the memory of the kernel + * that started it. Stop it cleanly instead; the camera is then + * unavailable until the next system boot. */ if (isp->hw->resident_fw) { mutex_lock(&isp->video_lock); if (isp->fw_state == ISP_FW_RUNNING) - dev_warn(dev, "stopping the firmware for system sleep, the camera is unavailable until the next boot\n"); + dev_warn(dev, "stopping the firmware for hibernation, the camera is unavailable until the next boot\n"); apple_isp_firmware_halt(isp); mutex_unlock(&isp->video_lock); } @@ -826,7 +840,12 @@ static __maybe_unused int apple_isp_resume(struct device *dev) } static const struct dev_pm_ops apple_isp_pm_ops = { - SYSTEM_SLEEP_PM_OPS(apple_isp_suspend, apple_isp_resume) + .suspend = pm_sleep_ptr(apple_isp_suspend), + .resume = pm_sleep_ptr(apple_isp_resume), + .freeze = pm_sleep_ptr(apple_isp_freeze), + .thaw = pm_sleep_ptr(apple_isp_resume), + .poweroff = pm_sleep_ptr(apple_isp_freeze), + .restore = pm_sleep_ptr(apple_isp_resume), RUNTIME_PM_OPS(apple_isp_runtime_suspend, apple_isp_runtime_resume, NULL) }; diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 7ece3219b2ea0..ed35f4701c04a 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -940,8 +940,9 @@ static void isp_firmware_shutdown(struct apple_isp *isp) * does not start again once it has been stopped: after CISP_CMD_SUSPEND, * a coprocessor reset and a power cycle of its domains it never completes * the first handshake. It is booted once, at probe, and runs until the - * driver is unbound, probe fails or the system goes to sleep; after that - * the device stays unusable until the next system boot. + * driver is unbound, probe fails or the system hibernates, also across + * suspend; after it has stopped the device stays unusable until the next + * system boot. */ int apple_isp_firmware_boot(struct apple_isp *isp) { From 0358f59f7d85fbdb3bd812aae771dbef90e5112f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 14:36:32 +1100 Subject: [PATCH 0783/1579] Bluetooth: mt7932: don't cancel HCI commands at suspend entry Every system suspend logs Bluetooth: hci0: Opcode 0x0c52 failed: -112 Bluetooth: hci0: unexpected event for opcode 0x0c1a and sometimes the same for 0x0c24. These are Write EIR and Write Class of Device commands that bluetoothd issues at suspend entry: when the session goes inactive, the sound server unregisters its Bluetooth profiles, and removing their UUIDs rewrites the class and EIR. hci_suspend_dev() cancels a command that is still waiting for its completion with -EHOSTDOWN, and the controller's late completion then no longer matches the command that was sent next. The transport calls hci_suspend_dev() twice: the HCI PM notifier calls it at PM_SUSPEND_PREPARE, before tasks are frozen and while bluetoothd is still issuing these commands, and bt7932_suspend() calls it again, which cancels whatever was queued in between. Suspend HCI only from bt7932_suspend(), as the transport's sleep handshake requires anyway, by setting HCI_QUIRK_NO_SUSPEND_NOTIFIER. By then userspace is frozen, so first let the HCI work that it queued finish: queue an empty hci_cmd_sync entry and wait for it, bounded by the HCI command timeout. Leave a user channel device alone, as the notifier does. Fixes: 8896264380c3 ("Bluetooth: mt7932: support modules and system suspend") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 51 +++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 36655c25f88f5..6c8378e495950 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -8,6 +8,7 @@ * transport arena and uses the firmware quiesce/restore handshake. Unload * releases DMA only after the PCI function has stopped bus mastering. */ +#include #include #include #include @@ -112,6 +113,7 @@ struct bt7932 { struct work_struct tx_work; struct sk_buff_head tx_queue; wait_queue_head_t wake_wait; + struct completion hci_idle; struct bt7932_ring rings[BT7932_CHANNELS]; void *arena; dma_addr_t arena_dma; @@ -1089,6 +1091,7 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) bt->last_sleep = U32_MAX; mutex_init(&bt->lock); init_waitqueue_head(&bt->wake_wait); + init_completion(&bt->hci_idle); skb_queue_head_init(&bt->tx_queue); INIT_WORK(&bt->tx_work, bt7932_tx_work); msleep(100); @@ -1164,6 +1167,8 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) bt->hdev->hw_error = bt7932_hw_error; hci_set_quirk(bt->hdev, HCI_QUIRK_RESET_ON_CLOSE); hci_set_quirk(bt->hdev, HCI_QUIRK_NON_PERSISTENT_SETUP); + /* System sleep suspends HCI from bt7932_suspend(), after the freeze. */ + hci_set_quirk(bt->hdev, HCI_QUIRK_NO_SUSPEND_NOTIFIER); ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); if (ret < 0) goto fail; @@ -1269,6 +1274,38 @@ static int bt7932_restore_transport(struct bt7932 *bt) return ret; } +static int bt7932_hci_idle(struct hci_dev *hdev, void *data) +{ + return 0; +} + +static void bt7932_hci_idle_done(struct hci_dev *hdev, void *data, int err) +{ + complete(data); +} + +/* + * hci_suspend_dev() cancels a command that is still waiting for its + * completion. Userspace can queue HCI work right up to the freeze (sound + * servers unregister their profiles when the session goes inactive, which + * rewrites the class and EIR), so let that work finish first. + */ +static void bt7932_wait_hci_idle(struct bt7932 *bt) +{ + reinit_completion(&bt->hci_idle); + if (hci_cmd_sync_queue(bt->hdev, bt7932_hci_idle, &bt->hci_idle, + bt7932_hci_idle_done)) + return; + if (!wait_for_completion_timeout(&bt->hci_idle, HCI_CMD_TIMEOUT)) + dev_warn(&bt->pdev->dev, "HCI work still pending at suspend\n"); +} + +/* Like the HCI PM notifier, leave a user channel device to userspace. */ +static bool bt7932_hci_pm(struct bt7932 *bt) +{ + return !hci_dev_test_flag(bt->hdev, HCI_USER_CHANNEL); +} + static int bt7932_suspend(struct device *dev) { struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); @@ -1278,9 +1315,12 @@ static int bt7932_suspend(struct device *dev) return -EIO; if (!bt->registered) return 0; - ret = hci_suspend_dev(bt->hdev); - if (ret) - return ret; + if (bt7932_hci_pm(bt)) { + bt7932_wait_hci_idle(bt); + ret = hci_suspend_dev(bt->hdev); + if (ret) + return ret; + } ret = bt7932_quiesce(bt); if (!ret) { dev_dbg(dev, "system sleep quiesce acknowledged\n"); @@ -1290,7 +1330,8 @@ static int bt7932_suspend(struct device *dev) /* Undo our request before the PM core rolls back the rest of the bus. */ if (!READ_ONCE(bt->fault)) bt7932_restore_transport(bt); - hci_resume_dev(bt->hdev); + if (bt7932_hci_pm(bt)) + hci_resume_dev(bt->hdev); return ret; } @@ -1309,7 +1350,7 @@ static int bt7932_resume(struct device *dev) return ret; } dev_dbg(dev, "transport restored after system sleep\n"); - return hci_resume_dev(bt->hdev); + return bt7932_hci_pm(bt) ? hci_resume_dev(bt->hdev) : 0; } static int bt7932_freeze(struct device *dev) From 6d54c0fd885ceded70867de874f27987fc8f69b4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 03:44:23 -0400 Subject: [PATCH 0784/1579] drm/asahi: pack G17 tiler heap blocks contiguously in their backing Each tiler heap (TVB) block is 128 KiB and is mapped at a 160 KiB VA stride, so that an unmapped 32 KiB guard follows it. The backing object used the same stride: block i was mapped from offset i * 160 KiB, and an object of n blocks was (n - 1) * 160 KiB + 128 KiB long. Every block but the last therefore carried 32 KiB of allocated, zeroed and pinned backing that no GPU mapping covers, 20 % of all tiler heap memory. Map block i from offset i * 128 KiB and size the backing to n * 128 KiB. The VA placement, the guards and the addresses handed to the firmware are unchanged. At the in-world peak of a measured Subnautica session the render pairs held 2,057 blocks in 49 objects, so this returns 62.7 MiB. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5f6490414a683eef149ba6de38cbc27ba62f9a26) --- drivers/gpu/drm/asahi/g17/buffer.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 90ab8ae21aa14..e3387228093f7 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -340,21 +340,17 @@ pub(crate) struct Tvb { extensions: KVec, } impl Tvb { + /// Blocks are packed in the backing; only their VA placement keeps the + /// stride and its trailing guard. fn bytes(blocks: usize) -> Result { - blocks - .checked_sub(1) - .and_then(|v| v.checked_mul(cfg::TVB_BLOCK_STRIDE as usize)) - .and_then(|v| v.checked_add(cfg::TVB_BLOCK_SIZE)) - .ok_or(EOVERFLOW) + blocks.checked_mul(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW) } fn map_block( object: &KernelObject, vm: &mmu::Vm, relative: usize, ) -> Result { - let start = relative - .checked_mul(cfg::TVB_BLOCK_STRIDE as usize) - .ok_or(EOVERFLOW)?; + let start = relative.checked_mul(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; let source = start..start.checked_add(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; let mapping = compact_alias( object, From 29ff1f70b4880e79a8b980adc4d44d4eacb7c7e1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 03:51:06 -0400 Subject: [PATCH 0785/1579] drm/asahi: allocate only the G17 boot pool blocks that are used Boot allocates three groups of 1 MiB pool blocks that no GPU mapping ever covers and nothing writes: - render blocks 23..27: the render pool maps blocks 0..21 and keeps the compute page list in block 22, but allocated 28 blocks; - the 23rd block of the device-wide compute backing: compute VMs map 22 of the 23 blocks; - the backing of the two zeroed compute blocks: their VAs fall inside the compute pool's page-list mapping, and the objects were never aliased anywhere. Each block is a zeroed, pinned kernel object for the life of the device. Allocate only the mapped render blocks plus the page-list block, only the mapped compute blocks, and no zero-block backing. Every firmware-visible address and table is unchanged; this returns 8 MiB. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d2a2c2fa97b994595209843720ad78b99b93f16a) --- drivers/gpu/drm/asahi/g17/initdata.rs | 8 ++------ drivers/gpu/drm/asahi/hw/t8140.rs | 5 +++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 5993be786874e..46a38c5ad16ec 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -23,7 +23,6 @@ const CONTEXT_SIZE: usize = 0x20000; const REGION_A_SIZE: usize = core::mem::size_of::(); const REGION_C_SIZE: usize = 0x1000; -const COMPUTE_BLOCKS_ALLOCATED: usize = 23; const COMPUTE_BLOCKS_MAPPED: usize = 22; const COMPUTE_BLOCKS_POPULATED: usize = 21; const COMPUTE_CONTROL_BLOCKS: u32 = 25; @@ -65,8 +64,8 @@ impl ComputeShared { KernelObject::backing(alloc.dev, low::RENDER_PAGE_LIST_SIZE, CpuMap::WriteCombined)?; let runs = KernelObject::backing(alloc.dev, low::RENDER_RUN_LIST_SIZE, CpuMap::WriteCombined)?; - let mut blocks = KVec::with_capacity(COMPUTE_BLOCKS_ALLOCATED, GFP_KERNEL)?; - for _ in 0..COMPUTE_BLOCKS_ALLOCATED { + let mut blocks = KVec::with_capacity(COMPUTE_BLOCKS_MAPPED, GFP_KERNEL)?; + for _ in 0..COMPUTE_BLOCKS_MAPPED { blocks.push( KernelObject::backing(alloc.dev, abi::FREE_LIST_BLOCK_SIZE, CpuMap::WriteCombined)?, GFP_KERNEL, @@ -867,9 +866,6 @@ impl InitData { let compute_state = self.add(KernelObject::backing(env.dev, mmu::UAT_PGSZ, WB)?)?; self.object_mut(compute_state)? .write(0, abi::FreeListState::COMPUTE)?; - for _ in &low::COMPUTE_ZERO_BLOCKS { - self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; - } let compute_runs = self.add(KernelObject::new( env.dev, env.lower, diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 2e73aed1a9797..143ba720d06f2 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -196,8 +196,9 @@ pub(crate) mod low { pub(crate) const RENDER_BLOCKS: u64 = 0x70_0022_0000; /// Distance between consecutive blocks. pub(crate) const BLOCK_STRIDE: u64 = 0x10_8000; - /// Render blocks allocated at boot. - pub(crate) const RENDER_BLOCKS_ALLOCATED: usize = 28; + /// Render blocks allocated at boot: the mapped blocks and the block that + /// holds the compute page list. + pub(crate) const RENDER_BLOCKS_ALLOCATED: usize = COMPUTE_PAGE_LIST_BLOCK + 1; /// Render blocks mapped at boot. pub(crate) const RENDER_BLOCKS_MAPPED: usize = 22; /// Render blocks in the free list at boot. From c304acb009f0264d1485596f651495d66efb0fec Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 04:49:41 -0400 Subject: [PATCH 0786/1579] drm/asahi: give back idle G17 tiler heap growth beyond recent use A render pair's tiler heap (TVB) grows on demand, by firmware request or from the sampled page high-water, and never shrinks. A pair that grew for one heavy scene keeps that growth for its whole life, also after the client returns to a light scene or goes idle; in a measured Subnautica session the game's pairs held 260 MiB of growth until it exited. The macOS driver shrinks its parameter buffers (AGXParameterManagement::checkForShrink). It folds the page high-water into an average once per check, average = max((sample + 4 * average) / 5, sample). For each check in which the average stays below a quarter of the capacity it removes one 512 KiB block, and a buffer that has been idle for ten seconds returns to its minimum. Apply the same policy from the event worker, with a minimum that keeps what recent work used, so a shrink never makes the next pass at the recent load grow again: - once per second, sample the page high-water (sharing the sample with the growth check) and update the average; a period with a sample or a prepared pass also enters a window of the last ten such periods; - each second the average is below a quarter of the capacity earns 512 KiB of shrink; - a pair with no pass prepared or in flight for ten seconds may shrink to its floor; - the floor is the largest of the primary backing, the blocks of the last pass layout, and the size below which the automatic growth check would grow again for the largest sample in the window. No shrink goes below it. A shrink removes whole trailing growth extensions only, and only while the pair is idle and every block is back in the inventory ring. The kept blocks are laid out as a fresh heap is (ring entries and page list from the read cursor, state counts, both inventory cursors), and the PB descriptor row is reloaded, as an idle-host growth does. The removed blocks' client aliases leave the VM cache under the device mutex; the worker unmaps and frees them after it. A shrink is best effort and leaves the heap unchanged if anything fails. While some heap can still shrink, the worker wakes at least once per second. If the inventory of an idle heap does not hold every block, the heap is left as it is and the first such heaps are reported once each with their inventory cursors. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 272955b009fe844d4d06361365f166d75b6b451b) --- drivers/gpu/drm/asahi/g17/buffer.rs | 259 +++++++++++++++++++- drivers/gpu/drm/asahi/g17/queue/render.rs | 1 + drivers/gpu/drm/asahi/g17/runtime.rs | 24 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 14 ++ drivers/gpu/drm/asahi/g17/runtime/render.rs | 74 ++++++ drivers/gpu/drm/asahi/mmu/lifetime.rs | 34 +++ 6 files changed, 401 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index e3387228093f7..e8b742bcd1f64 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -333,6 +333,13 @@ pub(crate) struct TvbGrowth { /// Sparse 128-KiB data blocks at 160-KiB backing strides. Firmware owns their /// contents; CPU zeroing occurs only on fresh backing, never on pool reuse. +/// Aliases and backing of growth extensions a shrink removed from an idle +/// heap. Field order unmaps the aliases before freeing their backing. +pub(crate) struct TvbRelease { + _mappings: KVec, + _extensions: KVec, +} + pub(crate) struct Tvb { addresses: KVec, // Retain sparse-block backing until the pool and its aliases retire. @@ -394,6 +401,18 @@ impl Tvb { pub(crate) fn addresses(&self) -> &[u64] { &self.addresses } + /// Blocks of each growth extension; extensions are packed like the primary. + fn extension_blocks(extension: &KernelObject) -> usize { + extension.size() / cfg::TVB_BLOCK_SIZE + } + /// Blocks of the primary backing, which a heap never gives back. + fn primary_blocks(&self) -> usize { + self.extensions + .iter() + .fold(self.blocks(), |blocks, extension| { + blocks.saturating_sub(Self::extension_blocks(extension)) + }) + } pub(crate) fn prepare_growth( &mut self, @@ -536,8 +555,50 @@ pub(crate) struct Manager { scratch: u64, buffer_id: u8, generation: u64, + shrink: Shrink, +} + +/// Native tiler-heap shrink policy, after macOS +/// `AGXParameterManagement::checkForShrink` (AGXG17X 0x8bd5e20). Once per +/// period the page high-water folds into an average, +/// `max((sample + 4 * average) / 5, sample)`. Each period whose average +/// stays below a quarter of the capacity earns one 512 KiB block of shrink, +/// and a heap idle for ten seconds returns to its minimum. Only an idle heap +/// is rebuilt here, by whole growth extensions. +/// +/// Unlike macOS, the minimum keeps what the recent work needs: never below +/// the size at which the automatic growth check would grow the heap again +/// for the largest high-water sample of the last ten active periods. A +/// shrink then never makes the next pass at the recent load grow back. +const SHRINK_PERIOD_NS: u64 = 1_000_000_000; +const SHRINK_IDLE_NS: u64 = 10_000_000_000; +/// Blocks in one native 512 KiB parameter-buffer block. +const SHRINK_STEP_BLOCKS: usize = 0x8_0000 / cfg::TVB_BLOCK_SIZE; +/// Active periods in the recent-load window. +const SHRINK_WINDOW: usize = (SHRINK_IDLE_NS / SHRINK_PERIOD_NS) as usize; + +#[derive(Default)] +struct Shrink { + last_busy: u64, + period_start: u64, + /// High-water samples since the last period and the last growth check. + period_max: u32, + growth_max: u32, + average: u32, + credit: usize, + /// Blocks the most recent pass layout required. + required: usize, + /// High-water samples of the last active periods (with a sample or a + /// pass) and their largest value. Idle periods leave both unchanged. + window: [u32; SHRINK_WINDOW], + window_next: usize, + busy_peak: u32, + reported: bool, } +/// Bounds the one-line report of heaps that could not be rebuilt. +static SHRINK_REPORTS: AtomicU32 = AtomicU32::new(0); + pub(crate) struct Submitted<'a> { word: &'a AtomicU32, next: u32, @@ -599,6 +660,7 @@ impl Manager { scratch: args.scratch, buffer_id: args.buffer_id, generation: args.generation, + shrink: Shrink::default(), }) } pub(crate) fn buffer_id(&self) -> u8 { @@ -658,7 +720,9 @@ impl Manager { next: word.load(Ordering::Relaxed).wrapping_add(1), }) } - pub(crate) fn automatic_target(&self) -> Result> { + /// Reads the firmware page high-water and requests its reset. Both the + /// growth check and the shrink period see every sample. + fn sample_max_pages(&mut self) -> Result { let max_pages = self .graph .word(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages))?; @@ -669,8 +733,83 @@ impl Manager { let pages = max_pages.load(Ordering::Relaxed); fence(Ordering::Release); reset.store(1, Ordering::Relaxed); + self.shrink.period_max = self.shrink.period_max.max(pages); + self.shrink.growth_max = self.shrink.growth_max.max(pages); + Ok(pages) + } + pub(crate) fn automatic_target(&mut self) -> Result> { + self.sample_max_pages()?; + let pages = core::mem::take(&mut self.shrink.growth_max); Ok(automatic_growth_target(pages as usize, self.blocks())) } + /// Records the blocks a pass layout requires, the floor of any shrink. + /// A pass is being prepared at `now`; the heap counts as busy. + pub(crate) fn note_required(&mut self, blocks: usize, now: u64) { + self.shrink.required = blocks; + self.shrink.last_busy = now; + } + /// The smallest heap a shrink may leave: the primary backing, the blocks + /// of the last pass layout, and the size below which the automatic growth + /// check would grow again for the recent peak. + fn shrink_floor(&self) -> usize { + let recent = (self.shrink.busy_peak as usize) + .saturating_mul(2) + .div_ceil(cfg::TVB_PAGES_PER_BLOCK); + self.tvb.primary_blocks().max(self.shrink.required).max(recent) + } + /// Whether the heap holds growth beyond its primary backing. + pub(crate) fn grown(&self) -> bool { + !self.tvb.extensions.is_empty() + } + /// Whether the heap holds growth that a shrink could give back. + pub(crate) fn shrinkable(&self) -> bool { + self.tvb.extensions.last().is_some_and(|extension| { + self.blocks() - Tvb::extension_blocks(extension) >= self.shrink_floor() + }) + } + /// Advances the shrink policy at `now` and returns the block count an idle + /// heap should shrink to. `busy` is true while the pair has work in flight + /// or is borrowed; a busy heap only updates its average and recent peak. + /// Call it for every grown heap, so the recent peak can also fall. + pub(crate) fn shrink_target(&mut self, now: u64, busy: bool) -> Result> { + if busy || self.shrink.last_busy == 0 { + self.shrink.last_busy = now; + } + if self.shrink.period_start == 0 { + self.shrink.period_start = now; + } + if now.saturating_sub(self.shrink.period_start) >= SHRINK_PERIOD_NS { + self.sample_max_pages()?; + let sample = core::mem::take(&mut self.shrink.period_max); + let average = (u64::from(sample) + 4 * u64::from(self.shrink.average)) / 5; + self.shrink.average = (average as u32).max(sample); + if sample != 0 || self.shrink.last_busy >= self.shrink.period_start { + let next = self.shrink.window_next; + self.shrink.window[next] = sample; + self.shrink.window_next = (next + 1) % SHRINK_WINDOW; + self.shrink.busy_peak = self.shrink.window.iter().copied().max().unwrap_or(0); + } + self.shrink.period_start = now; + let capacity = self.blocks() * cfg::TVB_PAGES_PER_BLOCK; + if (self.shrink.average as usize).saturating_mul(4) < capacity { + self.shrink.credit = self.shrink.credit.saturating_add(SHRINK_STEP_BLOCKS); + } else { + self.shrink.credit = 0; + } + } + if !self.shrinkable() { + self.shrink.credit = 0; + return Ok(None); + } + if busy { + return Ok(None); + } + if now.saturating_sub(self.shrink.last_busy) >= SHRINK_IDLE_NS { + return Ok(Some(self.shrink_floor())); + } + let last = self.tvb.extensions.last().map_or(0, Tvb::extension_blocks); + Ok((last != 0 && self.shrink.credit >= last).then(|| self.blocks() - last)) + } fn cursors(&self) -> Result { Ok(Cursors { total: self.graph.word(fw::CONTROL)?.load(Ordering::Relaxed), @@ -872,6 +1011,124 @@ impl Manager { } } +impl Manager { + /// Rebuilds an idle heap with whole trailing growth extensions removed, + /// keeping at least `target` blocks. The kept blocks are laid out as + /// `Graph::init` lays out a fresh heap, starting at the firmware read + /// cursor. This runs only while every block is back in the inventory, as + /// an idle-host growth requires the complete snapshot unchanged. + /// + /// `truncate(blocks, kept, out)` moves the removed blocks' client aliases + /// out of the VM cache. Returns the reloaded PB row and the released + /// aliases and backing, which the caller frees after the device mutex. + /// None leaves the heap, the firmware records and the VM unchanged. + pub(crate) fn shrink( + &mut self, + target: usize, + descriptor: [u32; 4], + truncate: impl FnOnce(usize, usize, &mut KVec) -> Result, + ) -> Result> { + let blocks = self.blocks(); + let mut kept = blocks; + let mut dropped = 0; + for extension in self.tvb.extensions.iter().rev() { + let size = Tvb::extension_blocks(extension); + if kept - size < target.max(self.tvb.primary_blocks()) { + break; + } + kept -= size; + dropped += 1; + } + if dropped == 0 { + return Ok(None); + } + let before = self.cursors()?; + let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) + % cfg::TVB_BLOCK_SLOTS; + if !before.valid() || before.total != before.committed || retained != blocks { + if !self.shrink.reported && SHRINK_REPORTS.fetch_add(1, Ordering::Relaxed) < 8 { + self.shrink.reported = true; + pr_info!( + "G17: tiler heap {} kept {} blocks: inventory {}/{}/{} holds {}\n", + self.buffer_id, + blocks, + before.total, + before.committed, + before.read, + retained + ); + } + return Ok(None); + } + // Allocation failure only postpones the shrink to a later pass. + let (Ok(mappings), Ok(extensions)) = ( + KVec::with_capacity(blocks - kept, GFP_NOWAIT), + KVec::with_capacity(dropped, GFP_NOWAIT), + ) else { + return Ok(None); + }; + let mut release = TvbRelease { + _mappings: mappings, + _extensions: extensions, + }; + for &address in &self.tvb.addresses[..kept] { + fw::compact_page(address)? + .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .ok_or(EOVERFLOW)?; + } + let base = self.graph.pointer(0, fw::SIZE)?.cast::(); + self.graph.word(0)?; + // The callback moves aliases only after checking the cached owner, + // its block count and the spare capacity. + truncate(blocks, kept, &mut release._mappings)?; + + // From here every index, address and CPU view has been checked. + let write = |offset: usize, value: u32| { + // SAFETY: The full graph and base alignment were checked above. + // Every offset is an aligned field or a checked array index of + // Graph. No mutable reference aliases firmware-owned fields. + unsafe { AtomicU32::from_ptr(base.add(offset / 4)).store(value, Ordering::Relaxed) }; + }; + for (index, &address) in self.tvb.addresses[..kept].iter().enumerate() { + // The same pure conversion succeeded for every kept block above. + let Ok(first) = fw::compact_page(address) else { + continue; + }; + let slot = (before.read as usize + index) % cfg::TVB_BLOCK_SLOTS; + let at = fw::BLOCK_TABLE + slot * size_of::(); + write(at, first); + write(at + 4, 0); + for page in 0..cfg::TVB_PAGES_PER_BLOCK { + let entry = index * cfg::TVB_PAGES_PER_BLOCK + page; + write(fw::PAGE_LIST + entry * 4, first + page as u32); + } + } + let total = ((before.read as usize + kept) % cfg::TVB_BLOCK_SLOTS) as u32; + let pages = (kept * cfg::TVB_PAGES_PER_BLOCK) as u32; + write(core::mem::offset_of!(fw::State, pages), pages); + write(core::mem::offset_of!(fw::State, producer), total); + write(core::mem::offset_of!(fw::State, consumer), before.read); + write(core::mem::offset_of!(fw::State, last_page), pages - 1); + write(fw::CONTROL, total); + fence(Ordering::SeqCst); + write(fw::CONTROL + 4, total); + fence(Ordering::SeqCst); + + self.tvb.addresses.truncate(kept); + for _ in 0..dropped { + if let Some(extension) = self.tvb.extensions.pop() { + append_reserved(&mut release._extensions, extension); + } + } + self.shrink.credit = self.shrink.credit.saturating_sub(blocks - kept); + self.shrink.reported = false; + Ok(Some(( + fw::reload_descriptor(descriptor, self.page_list_client, pages), + release, + ))) + } +} + /// Caller reserves all needed entries before any firmware-visible mutation. fn append_reserved(values: &mut KVec, value: T) { let length = values.len(); diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index d82ee84c1283f..0bf477c0344fb 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -909,6 +909,7 @@ impl Pair { return Err(EINVAL); }; let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; + self.memory.manager.note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 4c7aa6a442cc1..e4f19397e5f92 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -656,6 +656,8 @@ pub(super) struct DeferredBatch { vacated: KVVec, /// Backing of render pools vacated for evicted idle clients. render: KVVec, + /// Tiler-heap extensions removed from idle heaps, with their aliases. + tvb: KVVec, } impl DeferredBatch { const DEPTH: usize = if crate::hw::t8140::queues::RENDER_DEPTH as usize @@ -676,14 +678,17 @@ impl DeferredBatch { Self::with_capacity(2 * Self::DEPTH, 0) } fn with_capacity(capacity: usize, detached: usize) -> Result { + let render_slots = if detached != 0 { + crate::hw::t8140::queues::RENDER_SLOTS + } else { + 0 + }; Ok(Self { values: KVVec::with_capacity(capacity, GFP_KERNEL)?, bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, - render: KVVec::with_capacity( - if detached != 0 { crate::hw::t8140::queues::RENDER_SLOTS } else { 0 }, - GFP_KERNEL, - )?, + render: KVVec::with_capacity(render_slots, GFP_KERNEL)?, + tvb: KVVec::with_capacity(render_slots, GFP_KERNEL)?, }) } /// Keeps the detached owner's unmapping and VM teardown out of the device @@ -702,6 +707,14 @@ impl DeferredBatch { drop(backing); } } + pub(super) fn tvb_room(&self) -> bool { + self.tvb.len() < self.tvb.capacity() + } + pub(super) fn defer_tvb(&mut self, release: super::buffer::TvbRelease) { + if let Err(release) = self.tvb.push_within_capacity(release) { + drop(release); + } + } pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { if let Err(vacated) = self.vacated.push_within_capacity(vacated) { drop(vacated); @@ -753,6 +766,9 @@ impl DeferredBatch { for backing in self.render.drain_all() { drop(backing); } + for release in self.tvb.drain_all() { + drop(release); + } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 99f3caf66bebe..1d1f551377145 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -13,6 +13,9 @@ use crate::g17::{ }; use kernel::{prelude::*, sync::Arc}; +/// Wake interval while a tiler heap may still shrink: the native check period. +const SHRINK_WAKE_NS: u64 = 1_000_000_000; + struct Service<'a> { firmware: &'a mut Firmware, deferred: &'a mut DeferredBatch, @@ -387,10 +390,14 @@ impl crate::g17::Shared { firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; firmware.service_compute_pools(deferred)?; + let mut shrinking = false; if !firmware.recovery.pending() { firmware .queues .vacate_evicted_render_pools(&firmware.init, deferred)?; + shrinking = firmware + .queues + .shrink_render_heaps(&firmware.init, now, deferred); } { let mut reclaim = self.reclaim.lock(); @@ -414,6 +421,13 @@ impl crate::g17::Shared { if polling { self.queue_poll(); } + // A heap with growth to give back is re-evaluated at least once per + // shrink period, also while the device is otherwise idle. + let idle = if shrinking { + Some(idle.map_or(SHRINK_WAKE_NS, |delay| delay.min(SHRINK_WAKE_NS))) + } else { + idle + }; if let Some(delay) = idle { self.queue_idle(delay); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 71b21f98479a6..460889be64e2b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -647,6 +647,80 @@ impl Registry { || !self.render.retired.is_empty() || !self.teardown.is_empty() } + /// Applies the native tiler-heap shrink policy (`buffer::Manager:: + /// shrink_target`) to every grown heap at `now` and rebuilds the heaps of + /// idle pairs that are due, removing whole growth extensions. The removed + /// aliases and backing go to the worker's deferred batch, which frees them + /// after the device mutex. A shrink is best effort: any failure leaves the + /// heap as it was. Returns whether a heap still holds growth it could give + /// back, so the worker keeps a periodic wake. + pub(in crate::g17) fn shrink_render_heaps( + &mut self, + init: &crate::g17::initdata::InitData, + now: u64, + deferred: &mut super::DeferredBatch, + ) -> bool { + let mut pending = false; + for (slot, entry) in self.render.entries.iter_mut().enumerate() { + let Some(entry) = entry.as_mut() else { + continue; + }; + let borrowed = entry.constructing || entry.growing || entry.preparing.is_some(); + let Some(pair) = entry.pair.as_mut() else { + continue; + }; + if !pair.manager().grown() { + continue; + } + let busy = borrowed || !pair.idle_for_context(pair.context()).unwrap_or(false); + if let Ok(Some(target)) = pair.manager().shrink_target(now, busy) { + if deferred.tvb_room() { + if let Ok(slot) = u8::try_from(slot) { + // Ignoring the result keeps the heap unchanged on failure. + let _ = Self::shrink_pair(init, slot, pair, target, deferred); + } + } + } + pending |= pair.manager().shrinkable(); + } + pending + } + + fn shrink_pair( + init: &crate::g17::initdata::InitData, + slot: u8, + pair: &mut Pair, + target: usize, + deferred: &mut super::DeferredBatch, + ) -> Result { + use core::sync::atomic::{fence, Ordering}; + let offset = usize::from(pair.buffer_id()) * 16; + let table = init.pb_descriptor_table()?; + let words = [ + table.word(offset)?, + table.word(offset + 4)?, + table.word(offset + 8)?, + table.word(offset + 12)?, + ]; + fence(Ordering::Acquire); + let descriptor = words.each_ref().map(|word| word.load(Ordering::Relaxed)); + let vm = pair.context().vm().clone(); + let pool = pair.pool_id(); + let shrunk = pair + .manager() + .shrink(target, descriptor, |blocks, kept, out| { + vm.truncate_render_pool_mappings(slot, pool, blocks, kept, out) + })?; + if let Some((reloaded, release)) = shrunk { + for (word, value) in words.into_iter().zip(reloaded) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + deferred.defer_tvb(release); + } + Ok(()) + } + /// Gives back the USC backing of render pools whose client VM pressure /// reclaim evicted (an idle VM) and whose final free-list release was /// consumed. The vacated backings go to the worker's deferred batch, which diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index d53c19b93bfdc..975126e0cc76b 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -238,6 +238,40 @@ impl Vm { Ok(()) } + /// Moves the aliases of the last `blocks - kept` heap blocks of the exact + /// cached owner into `out`, which the caller drops after its locks. The + /// growth path appends one alias per added block, so these are the + /// blocks a shrink removed. Nothing moves unless all of them fit in `out`. + pub(crate) fn truncate_render_pool_mappings( + &self, + slot: u8, + id: u64, + blocks: usize, + kept: usize, + out: &mut KVec, + ) -> Result { + let count = blocks.checked_sub(kept).ok_or(EINVAL)?; + if id == 0 || count == 0 || out.capacity() - out.len() < count { + return Err(EINVAL); + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let pool = Option::as_mut(&mut *driver) + .and_then(|driver| driver.render.get_mut(usize::from(slot))) + .and_then(Option::as_mut) + .filter(|pool| pool.id == id) + .ok_or(EINVAL)?; + if pool.blocks != blocks || pool.mappings.len() < count { + return Err(EBUSY); + } + for _ in 0..count { + let mapping = pool.mappings.pop().ok_or(EIO)?; + out.push_within_capacity(mapping).map_err(|_| EIO)?; + } + pool.blocks = kept; + Ok(()) + } + /// A delayed teardown must not remove a successor generation. Remove /// under the cache lock, then unmap after dropping that lock. pub(crate) fn clear_render_pool_mappings_if_owner(&self, slot: u8, id: u64) { From 1464c4bb1ade2f3fe0625eb93776b26a7f91db96 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 04:50:48 -0400 Subject: [PATCH 0787/1579] drm/asahi: give back warm G17 compute pools after ten idle seconds A compute queue whose owner closed or exited stays installed for reuse, and up to four such queues (the warm cap) keep their 23 MiB USC private-memory pool populated for as long as no newer queue displaces them. On J700, the kernel's GEM after each game exit sat 54-78 MiB above its level before the game started, mostly in these warm pools. The macOS driver frees a client's USC pools with the client and also shrinks idle pools. Releasing every pool as soon as its owner is gone (a cap of zero) would make each new client re-arm fresh backing in its first submission, which costs about 2.3 ms on a cold start (measured at cap 0 on hardware). Keep the warm cap for bursts of short-lived clients, and additionally release a retained pool once it has been idle for ten seconds, the same period as the idle tiler-heap shrink. The event worker schedules its wake for the next expiry, and retries 100 ms later when a pass could not release every due pool. Queues without a release time stay bounded by the cap only, as before. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c01af7a9e73c789c6a9f98547585a7048927af30) --- drivers/gpu/drm/asahi/g17/runtime.rs | 66 +++++++++++++++------ drivers/gpu/drm/asahi/g17/runtime/events.rs | 7 ++- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index e4f19397e5f92..be22a0cab1321 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -176,30 +176,42 @@ impl Registry { queue.bind_owner(owner, context.clone()) } - /// Publishes releases for the oldest idle retained pools beyond `cap`, at - /// most four per pass. Returns the number published. + /// Delay before retrying pools a turn could not release. + const POOL_RETRY_NS: u64 = 100_000_000; + /// Publishes releases for the oldest idle retained pools beyond `cap` and + /// for every pool released at least `expiry` before `now`, at most four + /// per pass. Returns the number published and the delay until the next + /// retained pool expires or a retry is due. pub(super) fn vacate_idle_pools( &mut self, init: &super::initdata::InitData, cap: usize, - ) -> Result { + now: u64, + expiry: u64, + ) -> Result<(usize, Option)> { // Count first: event turns are frequent and the steady state has no excess. - let forced = self - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .filter(|queue| queue.vacate_candidate() && queue.needs_fresh_backing()) - .count(); - let count = self + let mut forced = 0; + let mut count = 0; + let mut next = None::; + for queue in self .compute .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) .filter(|queue| queue.vacate_candidate()) - .count(); + { + count += 1; + let released = queue.released_at(); + let age = now.saturating_sub(released); + if queue.needs_fresh_backing() || (released != 0 && age >= expiry) { + forced += 1; + } else if released != 0 { + let left = expiry - age; + next = Some(next.map_or(left, |next| next.min(left))); + } + } if count <= cap && forced == 0 { - return Ok(0); + return Ok((0, next)); } let mut candidates: KVec<(u64, u8)> = KVec::with_capacity(count, GFP_KERNEL)?; for queue in self @@ -213,6 +225,7 @@ impl Registry { let key = if queue.needs_fresh_backing() { 0 } else { queue.released_at() }; candidates.push((key, queue.qid()), GFP_KERNEL)?; } + // Recycled and expired pools are the oldest keys and sort first. candidates.sort_unstable(); let excess = candidates.len().saturating_sub(cap).max(forced); let mut published = 0; @@ -230,7 +243,11 @@ impl Registry { } published += 1; } - Ok(published) + if published < excess { + // Beyond this turn's bound, or not yet releasable: retry soon. + next = Some(next.map_or(Self::POOL_RETRY_NS, |next| next.min(Self::POOL_RETRY_NS))); + } + Ok((published, next)) } /// Frees the backing of every pool whose release the firmware consumed. @@ -1161,16 +1178,24 @@ impl super::Shared { /// Idle retained compute queues that keep their USC pool populated; older idle /// pools give their backing back. const COMPUTE_WARM_CAP: usize = 4; +/// A retained pool idle this long gives its backing back regardless of the +/// cap: a burst of short-lived clients still finds warm pools, but the pools +/// of clients that are gone do not stay resident. The macOS driver frees a +/// client's USC pools with the client and has an idle shrink; ten seconds +/// matches its idle parameter-buffer shrink. +const COMPUTE_WARM_EXPIRY_NS: u64 = 10_000_000_000; impl super::Firmware { /// Event-worker service step: witness consumed pool releases, then release the /// oldest idle retained pools beyond the warm cap. /// Detached owner bindings and vacated pool backing go to `deferred`, which /// the worker finishes after releasing the device mutex. + /// Returns the delay until a retained pool expires, for the worker's wake. pub(in crate::g17) fn service_compute_pools( &mut self, deferred: &mut DeferredBatch, - ) -> Result { + now: u64, + ) -> Result> { for queue in self .queues .compute @@ -1185,14 +1210,19 @@ impl super::Firmware { } self.queues.observe_pool_releases(&self.init, deferred)?; if self.render_control_backpressured() { - return Ok(()); + return Ok(None); } - let published = self.queues.vacate_idle_pools(&self.init, COMPUTE_WARM_CAP)?; + let (published, next) = self.queues.vacate_idle_pools( + &self.init, + COMPUTE_WARM_CAP, + now, + COMPUTE_WARM_EXPIRY_NS, + )?; if published != 0 { fence(Ordering::SeqCst); self.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; } - Ok(()) + Ok(next) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 1d1f551377145..a7ff3e4f0ecc3 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -389,7 +389,7 @@ impl crate::g17::Shared { let idle = firmware.idle_check(now, in_flight)?; firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; - firmware.service_compute_pools(deferred)?; + let pools = firmware.service_compute_pools(deferred, now)?; let mut shrinking = false; if !firmware.recovery.pending() { firmware @@ -428,6 +428,11 @@ impl crate::g17::Shared { } else { idle }; + // A retained compute pool gives its backing back when it expires. + let idle = match (idle, pools) { + (Some(delay), Some(expiry)) => Some(delay.min(expiry)), + (delay, expiry) => delay.or(expiry), + }; if let Some(delay) = idle { self.queue_idle(delay); } From 57e6637d9f087775ee87dcab22dd6c4bb0d4f685 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 04:26:27 -0400 Subject: [PATCH 0788/1579] drm/asahi: keep compaction away from GEM pages held for the GPU Every populated asahi GEM object keeps its shmem pages referenced: by the device SG table, a kernel vmap or a PFN CPU mapping. Those pages sit on the unevictable LRU and cannot migrate, but the compaction migrate scanner still isolates them (compact_unevictable_allowed) and every migration fails. In-world on J700, with about 5 GB of GEM held, background compaction isolated about 3,400 pages a second and 97 % of the migrations failed (pgmigrate_fail 3,203/s against pgmigrate_success 104/s). Mark the shmem mapping of a held object inaccessible. Compaction skips folios of inaccessible mappings before isolating them, and migration refuses them; the flag comes with unevictable, as mapping_set_inaccessible() sets it. Nothing reads or writes a GEM shmem file through its mapping. Kernel objects are marked when created; other objects when they take a device SG table. Releasing idle backing to reclaim, or purging, clears the mark, so released pages can be swapped and migrated again. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 2d7de7690cc0f9c8d3a74b08c33999593b720aac) --- drivers/gpu/drm/asahi/gem.rs | 49 +++++++++++++++++++++++++- drivers/gpu/drm/asahi/mmu.rs | 6 ++-- drivers/gpu/drm/asahi/mmu/bind.rs | 2 +- drivers/gpu/drm/asahi/mmu/residency.rs | 4 ++- 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 6c05da6ff096b..bc14223a837d5 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -13,7 +13,8 @@ use kernel::{ shmem, shmem::VMap, BaseObject, - DriverObject, // + DriverObject, + IntoGEMObject, // }, error::Result, prelude::*, @@ -24,6 +25,7 @@ use kernel::{ use core::ops::Range; use core::sync::atomic::{ AtomicU64, + AtomicUsize, Ordering, // }; @@ -60,6 +62,50 @@ pub(crate) struct AsahiObject { /// Type alias for the shmem GEM object type for this driver. pub(crate) type Object = shmem::Object; +/// Marks whether the pages of `gem` are held for the device. +/// +/// Populated asahi objects keep their pages referenced (device SG table, kernel vmap or a PFN +/// CPU mapping), so the pages cannot migrate. Unmarked, the compaction migrate scanner still +/// isolates them from the unevictable LRU and every migration fails. The shmem mapping of a +/// held object is therefore marked inaccessible, which compaction and migration check and which +/// comes with unevictable, as `mapping_set_inaccessible()` sets it; nothing reads or writes a GEM +/// shmem file through its mapping. Releasing idle backing or purging clears the mark, so the +/// released pages can be swapped and migrated again. +pub(crate) fn set_mapping_held(gem: &Object, held: bool) { + let raw = gem.as_raw(); + // SAFETY: `gem` keeps the GEM object alive. A shmem-backed object's `filp` and its + // `f_mapping` are set at initialization and never change; imported objects have none. + let mapping = unsafe { + let filp = (*raw).filp; + if filp.is_null() { + return; + } + (*filp).f_mapping + }; + if mapping.is_null() { + return; + } + // SAFETY: `flags` is the aligned flag word of the live mapping. The kernel updates it only + // with atomic bit operations on the same word. + let flags = + unsafe { AtomicUsize::from_ptr(core::ptr::addr_of_mut!((*mapping).flags).cast::()) }; + let inaccessible = 1usize << kernel::bindings::mapping_flags_AS_INACCESSIBLE; + let unevictable = 1usize << kernel::bindings::mapping_flags_AS_UNEVICTABLE; + if held { + flags.fetch_or(inaccessible | unevictable, Ordering::Relaxed); + } else { + flags.fetch_and(!inaccessible, Ordering::Relaxed); + } +} + +/// Takes an owned SG table of `gem`, which holds its pages for the device, and marks the +/// object held (see [`set_mapping_held`]). +pub(crate) fn held_sg_table(gem: &Object) -> Result> { + let sgt = gem.owned_sg_table()?; + set_mapping_held(gem, true); + Ok(sgt) +} + /// VM-private BOs share their assigned VM's reservation object. Validate this /// before acquiring mapping state; generic GPUVM accepts external shareable BOs. pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { @@ -192,6 +238,7 @@ fn new_kernel_object_mapped(dev: &AsahiDevice, size: usize, map_wc: bool) -> Res }, )?; + set_mapping_held(&gem, true); mod_pr_debug!("AsahiObject new kernel object id={}\n", gem.id); Ok(ObjectRef::new(gem)) } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 5149d7d4b49f3..eb18f26afb38c 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1761,7 +1761,7 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - let sgt = gem.owned_sg_table()?; + let sgt = crate::gem::held_sg_table(&gem)?; let mut inner = self.inner.exec_lock(Some(gem), false)?; let vm_bo = self.inner.obtain_bo(gem)?; @@ -1828,7 +1828,7 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - let sgt = gem.owned_sg_table()?; + let sgt = crate::gem::held_sg_table(&gem)?; let mut inner = self.inner.exec_lock(Some(&gem), false)?; let vm_bo = self.inner.obtain_bo(&gem)?; @@ -1907,7 +1907,7 @@ impl Vm { if needs_sgt { // Pin without the BO mutex: alias construction can hold the GEM // reservation while acquiring this mutex. - let sgt = gem.owned_sg_table()?; + let sgt = crate::gem::held_sg_table(&gem)?; let mut vm_bo_guard = vm_bo.inner().inner.lock(); if vm_bo_guard.sgt.is_none() { if vm_bo_guard.sg_vec.is_none() { diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index adbb609d5a88c..824a960f6530b 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -81,7 +81,7 @@ impl Vm { if needs_sgt { // Pin without the BO mutex: alias construction can hold the GEM // reservation while acquiring this mutex. - let sgt = gem.owned_sg_table()?; + let sgt = crate::gem::held_sg_table(&gem)?; let mut bo = vm_bo.inner().inner.lock(); if bo.sgt.is_none() { if bo.sg_vec.is_none() { diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 73a1d163eca82..afc9255100fa2 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -440,6 +440,7 @@ impl Vm { bo_inner.sgt = None; } if object.purge_locked().is_ok() { + crate::gem::set_mapping_held(object, false); purged = purged.saturating_add(object.size() / kernel::page::PAGE_SIZE); } else { gate.note_evicted(); @@ -519,6 +520,7 @@ impl Vm { // tables above. Kernel mappings and other SG owners are excluded; // the C/Rust backing helper rechecks exclusion before release. if object.release_idle_pages_locked().is_ok() { + crate::gem::set_mapping_held(object, false); unpinned = unpinned.saturating_add(object.size() / kernel::page::PAGE_SIZE); } Ok(unpinned < target) @@ -566,7 +568,7 @@ impl Vm { if needs_sgt { // Preserve review1's reservation/BO lock order: the getter // may acquire the object reservation; never hold BO here. - let sgt = map.object.owned_sg_table()?; + let sgt = crate::gem::held_sg_table(&map.object)?; let mut bo = bo_owner.inner().inner.lock(); if bo.sgt.is_none() { bo.sgt = Some(sgt); } drop(bo); From 76b1755a06109a66416af0282ec8310dc8a3292b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 10 Oct 2026 00:18:43 -0400 Subject: [PATCH 0789/1579] drm/asahi: complete page-table stores before GPU TLB invalidation Use the existing DSB helper before outer-shareable TLB invalidations. A sequentially consistent Rust fence emits DMB and does not provide the completion required before the GPU discards its cached translations. Publish new page-table leaves even when no context currently names the root, so a later binding cannot expose incomplete table stores. Keep invalidation ranges, queue admission and submission batching intact. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6dd0f73fd3b2bade5415b31346aadeb2f38d446e) --- drivers/gpu/drm/asahi/g17/kick.rs | 2 +- drivers/gpu/drm/asahi/mmu.rs | 36 ++++++++++++++++++------------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index f109c8261476e..3b78b9443a86d 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -308,7 +308,7 @@ impl Queue { if self.id.engine != DataMaster::Compute { return Err(EINVAL); } - fence(Ordering::SeqCst); + mem::sync(); for asid in 0..4 { mem::tlbi_range_or_asid(asid, self.low_va() as usize, KICK_RING_SIZE); mem::tlbi_range_or_asid(asid, self.firmware_va() as usize, KICK_RING_SIZE); diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index eb18f26afb38c..96eb5cd3a1202 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -528,7 +528,7 @@ impl gpuvm::DriverGpuVm for VmInner { } if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); + mem::sync(); self.tlbi_range(asid as u8, unmap_start, unmap_range as usize); mod_dev_dbg!( self.dev, @@ -587,7 +587,7 @@ impl VmInner { } if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); + mem::sync(); self.tlbi_range(asid as u8, va.addr(), va.range() as usize); mod_dev_dbg!( self.dev, @@ -612,7 +612,7 @@ impl VmInner { let result = self.page_table.discard_partial_map(addr..addr + size); if self.uat_inner.firmware == UatFirmware::Handoff { if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); + mem::sync(); self.tlbi_range(asid as u8, addr, size as usize); mem::sync(); } @@ -685,6 +685,7 @@ impl VmInner { /// already invalidate. fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { if self.uat_inner.firmware == UatFirmware::Handoff { + mem::sync(); return; } @@ -693,11 +694,14 @@ impl VmInner { } fn tlbi_context_mask(&self, iova: u64, size: usize, contexts: u64) { + // Complete new leaves even if no context currently names this root: + // a later context can publish it without repeating the map operation. + // DMB orders accesses but does not complete stores before TLBI; the + // GPU uses outer-shareable invalidation and the barrier covers it. + mem::sync(); if contexts == 0 { return; } - - fence(Ordering::SeqCst); for ctx in 0..UAT_NUM_CTX { if contexts & (1 << ctx) != 0 { mem::tlbi_range_or_asid(ctx as u8, iova as usize, size); @@ -871,7 +875,7 @@ impl ExecutionRoot { owned || (low == 0 && high == 0) }; if release { - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_asid(self.id); mem::sync(); self.inner.lock().execution_contexts &= !(1u64 << id); @@ -1106,7 +1110,7 @@ impl KernelMapping { self.iova()..self.iova() + prefix as u64, PROT_GPU_FW_PRIV_RW, )?; - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_all(); mem::sync(); let mut after = [0; MAX_PAGES]; @@ -1156,7 +1160,7 @@ impl KernelMapping { self.size() ); } - fence(Ordering::SeqCst); + mem::sync(); // If we don't have (and have never had) a VM slot, just return let slot = match owner.slot() { @@ -1314,7 +1318,7 @@ impl Drop for KernelMapping { } if let Some(asid) = owner.slot() { - fence(Ordering::SeqCst); + mem::sync(); owner.tlbi_range(asid as u8, self.iova(), self.size()); mod_dev_dbg!( owner.dev, @@ -2188,6 +2192,7 @@ impl Drop for VmInner { self.id, idx ); + mem::sync(); mem::tlbi_asid(idx as u8); mem::sync(); } @@ -2269,7 +2274,7 @@ impl Uat { if contexts == 0 { return Err(ENOENT); } - fence(Ordering::SeqCst); + mem::sync(); for context in 0..UAT_NUM_CTX { if contexts & (1u64 << context) != 0 { mem::tlbi_asid(context as u8); @@ -2335,6 +2340,7 @@ impl Uat { core::mem::drop(uat_inner); // Make sure all TLB entries from the previous owner of this ASID are gone + mem::sync(); mem::tlbi_asid(idx as u8); mem::sync(); } @@ -2402,7 +2408,7 @@ impl Uat { roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); } - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_asid(id as u8); mem::sync(); Ok(context) @@ -2449,7 +2455,7 @@ impl Uat { }) }?; - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_all(); mem::sync(); Ok(snapshot) @@ -2494,7 +2500,7 @@ impl Uat { }) }??; - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_all(); mem::sync(); Ok(()) @@ -2519,7 +2525,7 @@ impl Uat { .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); inner.handoff().unlock(); core::mem::drop(inner); - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); mem::sync(); Ok(()) @@ -2668,7 +2674,7 @@ impl Uat { impl Drop for Uat { fn drop(&mut self) { // Make sure we flush the TLBs - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_all(); mem::sync(); } From 91e15ed8e7a681ca293f94e337f59563cd03879d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 10 Oct 2026 00:18:43 -0400 Subject: [PATCH 0790/1579] drm/asahi: remove tiler-shrink diagnostic logging Keep an incomplete idle inventory unchanged without a global report counter or per-heap diagnostic state. Correct the backing-layout comment and place it on Tvb rather than the deferred release container. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 019ad857ddfa08ed2473eae34db5b8b8d98b068e) --- drivers/gpu/drm/asahi/g17/buffer.rs | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index e8b742bcd1f64..739d4c2fdb1cf 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -331,8 +331,6 @@ pub(crate) struct TvbGrowth { extension: KernelObject, } -/// Sparse 128-KiB data blocks at 160-KiB backing strides. Firmware owns their -/// contents; CPU zeroing occurs only on fresh backing, never on pool reuse. /// Aliases and backing of growth extensions a shrink removed from an idle /// heap. Field order unmaps the aliases before freeing their backing. pub(crate) struct TvbRelease { @@ -340,6 +338,8 @@ pub(crate) struct TvbRelease { _extensions: KVec, } +/// Packed 128-KiB data blocks mapped at guarded 160-KiB virtual strides. +/// Firmware owns their contents; CPU zeroing occurs only on fresh backing. pub(crate) struct Tvb { addresses: KVec, // Retain sparse-block backing until the pool and its aliases retire. @@ -593,12 +593,8 @@ struct Shrink { window: [u32; SHRINK_WINDOW], window_next: usize, busy_peak: u32, - reported: bool, } -/// Bounds the one-line report of heaps that could not be rebuilt. -static SHRINK_REPORTS: AtomicU32 = AtomicU32::new(0); - pub(crate) struct Submitted<'a> { word: &'a AtomicU32, next: u32, @@ -1046,18 +1042,6 @@ impl Manager { let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) % cfg::TVB_BLOCK_SLOTS; if !before.valid() || before.total != before.committed || retained != blocks { - if !self.shrink.reported && SHRINK_REPORTS.fetch_add(1, Ordering::Relaxed) < 8 { - self.shrink.reported = true; - pr_info!( - "G17: tiler heap {} kept {} blocks: inventory {}/{}/{} holds {}\n", - self.buffer_id, - blocks, - before.total, - before.committed, - before.read, - retained - ); - } return Ok(None); } // Allocation failure only postpones the shrink to a later pass. @@ -1121,7 +1105,6 @@ impl Manager { } } self.shrink.credit = self.shrink.credit.saturating_sub(blocks - kept); - self.shrink.reported = false; Ok(Some(( fw::reload_descriptor(descriptor, self.page_list_client, pages), release, From c58ea1758e0a3f0d9a9edc9a6e8af71f2a4d054b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 17:37:36 -0400 Subject: [PATCH 0791/1579] mm: reuse subpage backing when refilling anonymous holes A missing 4K PTE in a partially populated native 16K window currently gets a fresh private folio. Existing neighbors are skipped and the unused quarters are stranded when its pool closes. Repeated per-quarter hole refills can leave 16 MiB of logical data backed by 64 MiB of native pages. Reserve the missing-quarter mask from one compatible shared owner, keeping whole-window private allocation for initial dense faults. Initialize and hold the selected folio through reverse-map setup and PTE publication. Revalidate the missing mask under the PTE lock, release surplus or losing reservations, and retain the private fallback for metadata allocation failure. Existing pool charge, NUMA, locking and droppable exclusions apply. Add density, neighbor-prefault, concurrent refill, owner-exit, and controlled reclaim selftests. On native 16K Neo hardware with the 4K userspace ABI, the 16 MiB refill fixture finishes at 16.09 MiB instead of 64 MiB, with byte-exact contents. Initial dense backing stays at 16 MiB. Native/4K ABI, fork/exec, mprotect, existing concurrent COW and GPU quarter-refault controls pass. Controlled reclaim swaps all 4096 test leaves after original-owner exit, then passes byte-exact readback. Subnautica menu/world backing snapshots show lower native backing overhead; this is not an FPS qualification. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e8fb3863c6103fa0150ba98979756e6685e533d9) --- include/linux/mm_subpage.h | 9 + mm/memory.c | 183 ++++++++--- mm/user-subpage.c | 133 ++++++-- tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../selftests/arm64/user4k/README.hole-refill | 29 ++ .../arm64/user4k/README.hole-refill-race | 33 ++ .../arm64/user4k/README.hole-refill-reclaim | 36 +++ .../arm64/user4k/anon-hole-refill-race.c | 298 +++++++++++++++++ .../arm64/user4k/anon-hole-refill-reclaim.c | 299 ++++++++++++++++++ .../selftests/arm64/user4k/anon-hole-refill.c | 179 +++++++++++ 10 files changed, 1140 insertions(+), 63 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.hole-refill create mode 100644 tools/testing/selftests/arm64/user4k/README.hole-refill-race create mode 100644 tools/testing/selftests/arm64/user4k/README.hole-refill-reclaim create mode 100644 tools/testing/selftests/arm64/user4k/anon-hole-refill-race.c create mode 100644 tools/testing/selftests/arm64/user4k/anon-hole-refill-reclaim.c create mode 100644 tools/testing/selftests/arm64/user4k/anon-hole-refill.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index c4ac2eb6c4236..9a1d33166db30 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -52,6 +52,15 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, void mm_subpage_cow_pool_exit(struct mm_struct *mm); bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio); bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset); +/* Exact physical-slot mask, one owner and folio lock for the entire batch. + * slots is indexed by physical offset >> pool shift; only mask bits are set. + * Success returns initialized owned slots; errors leave slots untouched. + */ +int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots); +int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *folio, + unsigned long mask, gfp_t gfp); +bool mm_subpage_cow_wait_busy_mask(struct mm_subpage_pool *pool, unsigned long mask); /* Return an initialized slot with its folio locked; may wait without a pool lock. */ struct mm_subpage *mm_subpage_alloc_at_locked(struct mm_subpage_pool *pool, unsigned int offset); diff --git a/mm/memory.c b/mm/memory.c index 2fb430e842603..4006d3cef29ff 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6375,17 +6375,95 @@ static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, } #ifdef CONFIG_MM_SUBPAGE +/* One native-window batch, initialized and locked before PTE publication. */ +static int anon_subpage_prealloc_mask(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr, + unsigned long mask, bool use_pool, bool *pooled, + struct mm_subpage **slots) +{ + struct mm_subpage_pool *pool; + struct folio *folio; + int err; + +retry_private: + pool = use_pool ? mm_subpage_cow_pool_get(mm, vma, addr) : NULL; + *pooled = pool != NULL; + if (!pool) + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return -ENOMEM; + for (;;) { + err = mm_subpage_alloc_mask_locked(pool, mask, slots); + if (err != -EAGAIN) + break; + folio = NULL; + if (*pooled) { + folio = cow_subpage_speculative_folio(mm, vma, addr); + if (!folio && mm_subpage_cow_wait_busy_mask(pool, mask)) { + cond_resched(); + continue; + } + } + if (!folio) + folio = folio_prealloc(mm, vma, addr, false); + if (!folio) { + err = -ENOMEM; + break; + } + if (*pooled && !mm_subpage_cow_folio_matches(pool, folio)) { + folio_put(folio); + mm_subpage_pool_put(pool); + use_pool = false; + goto retry_private; + } + err = mm_subpage_pool_add_folio_mask(pool, folio, mask, + *pooled ? GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN : GFP_KERNEL); + if (*pooled && err == -ENOMEM) { + if (mm_subpage_cow_wait_busy_mask(pool, mask)) { + folio_put(folio); + cond_resched(); + continue; + } + err = mm_subpage_pool_add_folio_mask(pool, folio, mask, GFP_KERNEL); + } + if (err) { + folio_put(folio); + if (err == -EEXIST) { + cond_resched(); + continue; + } + break; + } + } + if (!*pooled) + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + return err; +} + +static unsigned long anon_subpage_missing_mask(pte_t *first, unsigned int nr, + unsigned int target, unsigned int offset) +{ + unsigned long mask = 0; + unsigned int i; + + for (i = 0; i < nr; i++) + if (i == target || pte_none(ptep_get(first + i))) + mask |= BIT(offset + i); + return mask; +} + static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct mm_struct *mm = vma->vm_mm; struct mm_subpage *slots[PAGE_SIZE / MM_SUBPAGE_SIZE] = {}; struct vm_page_offset pos = vma_page_offset_at(vma, vmf->address); - struct mm_subpage_pool *pool; struct folio *folio; - unsigned long start, end, lower; - unsigned int first_offset, nr, target, i, mapped = 0; - vm_fault_t ret = VM_FAULT_OOM; + unsigned long start, end, lower, mask, current_mask, window_mask; + unsigned int first_offset, nr, target, i, index, mapped; + bool pooled, fresh, force_private = false, retry_private; + vm_fault_t ret; pte_t *first; int err; @@ -6394,34 +6472,48 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) start = vmf->address - min_t(unsigned long, pos.offset, vmf->address - lower); end = vmf->address + min(PAGE_SIZE - pos.offset, pmd_addr_end_mm(mm, vmf->address, vma->vm_end) - vmf->address); - first_offset = pos.offset - (vmf->address - start); + first_offset = (pos.offset - (vmf->address - start)) >> mm_page_shift(mm); nr = (end - start) >> mm_page_shift(mm); target = (vmf->address - start) >> mm_page_shift(mm); - - pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); - if (!pool) + window_mask = GENMASK(first_offset + nr - 1, first_offset); +retry: + /* This optimistic snapshot only selects backing; PTL revalidates it. + * Initial dense faults retain one-folio, whole-window faultaround. + */ + vmf->pte = pte_offset_map_mm(mm, vmf->pmd, vmf->address); + if (!vmf->pte) + return 0; + mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset); + pte_unmap(vmf->pte); + vmf->pte = NULL; + err = anon_subpage_prealloc_mask(mm, vma, vmf->address, mask, + mask != window_mask && !force_private && !userfaultfd_armed(vma), + &pooled, slots); + if (err) return VM_FAULT_OOM; - /* Supply initialises every byte once, including the unissued slots. */ - folio = folio_prealloc(mm, vma, vmf->address, false); - if (!folio) - goto release; - err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); - if (err) { - folio_put(folio); - goto release; - } - for (i = 0; i < nr; i++) { - slots[i] = mm_subpage_alloc_at(pool, first_offset + i * mm_page_size(mm)); - if (IS_ERR(slots[i])) { - slots[i] = NULL; - goto release; - } - } - + folio = mm_subpage_folio(slots[__ffs(mask)]); /* PTE references may disappear as soon as we drop the PTL. */ folio_get(folio); - folio_lock(folio); + fresh = !mm_subpage_anon_root(slots[__ffs(mask)]); + mapped = 0; + retry_private = false; ret = 0; + /* Existing owners may need nonlinear identities. Allocate metadata before + * PTL and keep the established fresh-folio fallback on metadata ENOMEM. + */ + if (!fresh) { + for_each_set_bit(index, &mask, ARRAY_SIZE(slots)) { + unsigned long address = start + + ((index - first_offset) << mm_page_shift(mm)); + + err = mm_subpage_bind_rmap(slots[index], vma, address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + retry_private = pooled && err == -ENOMEM; + goto unlock_folio; + } + } + } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) goto unlock_folio; @@ -6442,20 +6534,30 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) } first = vmf->pte - target; + current_mask = anon_subpage_missing_mask(first, nr, target, first_offset); + /* Do not lose newly missing neighbors or publish sparse dense-path supply. + * Return to the fault handler on a racing mask change instead of spinning + * under repeated zaps. No mapping has been published yet. + */ + if ((current_mask & ~mask) || + (mask == window_mask && current_mask != mask && + !force_private && !userfaultfd_armed(vma))) + goto unlock_pte; for (i = 0; i < nr; i++) { unsigned long address = start + i * mm_page_size(mm); pte_t entry; - /* Do not replace existing mappings or userfaultfd markers nearby. */ - if (i != target && !pte_none(ptep_get(first + i))) + index = first_offset + i; + if (!(current_mask & BIT(index))) continue; - mm_subpage_set_exclusive(slots[i]); - err = mm_subpage_add_new_anon_rmap(slots[i], vma, address); + mm_subpage_set_exclusive(slots[index]); + err = fresh ? mm_subpage_add_new_anon_rmap(slots[index], vma, address) : + mm_subpage_add_anon_rmap(slots[index], vma, address); if (WARN_ON_ONCE(err)) { ret = VM_FAULT_SIGBUS; break; } - entry = phys_pte_mm(mm, mm_subpage_phys(slots[i]), vma->vm_page_prot); + entry = phys_pte_mm(mm, mm_subpage_phys(slots[index]), vma->vm_page_prot); entry = pte_sw_mkyoung(entry); if (vma->vm_flags & VM_WRITE) entry = pte_mkwrite(pte_mkdirty(entry), vma); @@ -6463,11 +6565,12 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) entry = pte_mkuffd_wp(entry); set_pte_at(mm, address, first + i, entry); update_mmu_cache(vma, address, first + i); - slots[i] = NULL; /* The PTE now owns the reference. */ + slots[index] = NULL; /* The PTE now owns the reference. */ mapped++; } if (mapped) { - subpage_add_new_lru(folio, vma, mapped); + if (fresh) + subpage_add_new_lru(folio, vma, mapped); add_mm_counter(mm, MM_ANONPAGES, mapped); count_mthp_stat(0, MTHP_STAT_ANON_FAULT_ALLOC); } @@ -6478,13 +6581,19 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) folio_unlock(folio); folio_put(folio); release: - mm_subpage_pool_close(pool); - for (i = 0; i < ARRAY_SIZE(slots); i++) - if (slots[i]) + for (i = 0; i < ARRAY_SIZE(slots); i++) { + if (slots[i]) { mm_subpage_put(slots[i]); - mm_subpage_pool_put(pool); + slots[i] = NULL; + } + } + if (retry_private) { + force_private = true; + goto retry; + } return ret; } + #endif /* diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 3adfb04ccdca1..c7876fd34a091 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -842,16 +842,18 @@ static bool cow_owner_eligible(struct mm_subpage_pool *pool, struct folio *folio * normal bounded search, inspect all owners before allowing an OOM-capable * allocation: a busy slot farther down the list may require no new charge. */ -bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) +static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, + bool require_all) { struct mm_subpage_owner *owner; struct folio *wait = NULL; - unsigned long flags, eligible = offset_mask(pool->shift, offset); + unsigned long flags; spin_lock_irqsave(&pool->lock, flags); if (!pool->closed && pool->cow_shared && cow_has_free(pool, eligible)) { list_for_each_entry(owner, &pool->available, available) { if (!owner->migrating && (~owner->used & eligible) && + (!require_all || !(owner->used & eligible)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && !folio_test_writeback(owner->folio)) { @@ -870,8 +872,18 @@ bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offs return false; } +bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + return cow_wait_busy(pool, offset_mask(pool->shift, offset), false); +} + +bool mm_subpage_cow_wait_busy_mask(struct mm_subpage_pool *pool, unsigned long mask) +{ + return cow_wait_busy(pool, mask, true); +} + static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, - unsigned long eligible) + unsigned long eligible, bool require_all) { struct mm_subpage_owner *owner; unsigned int visited = 0; @@ -893,6 +905,7 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, } /* A nonpresent PTE may still need any of this folio's slot bytes. */ if (!owner->migrating && (~owner->used & eligible) && + (!require_all || !(owner->used & eligible)) && (!pool->cow_shared || !folio_test_locked(owner->folio)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && @@ -903,7 +916,7 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, } static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, - unsigned long eligible, gfp_t gfp) + unsigned long eligible, bool require_all, gfp_t gfp) { struct mm_subpage_owner *owner; unsigned long flags; @@ -931,7 +944,7 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, spin_lock_irqsave(&pool->lock, flags); if (pool->closed) ret = -ESHUTDOWN; - else if (pool->unissued || find_available(pool, eligible)) + else if (pool->unissued || find_available(pool, eligible, require_all)) /* Another allocator supplied backing while we allocated metadata. * Preserve the single-unissued-owner invariant even when a shared * COW lookup skips a concurrently locked or rotating list prefix. @@ -959,7 +972,7 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, gfp_t gfp) { - return pool_add_folio(pool, folio, pool->all_slots, gfp); + return pool_add_folio(pool, folio, pool->all_slots, false, gfp); } int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, @@ -969,7 +982,15 @@ int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *fol if (!eligible) return -EINVAL; - return pool_add_folio(pool, folio, eligible, gfp); + return pool_add_folio(pool, folio, eligible, false, gfp); +} + +int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *folio, + unsigned long mask, gfp_t gfp) +{ + if (!mask || (mask & ~pool->all_slots)) + return -EINVAL; + return pool_add_folio(pool, folio, mask, true, gfp); } unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) @@ -982,6 +1003,34 @@ unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) return pages; } +/* Caller holds pool lock; each issued slot owns one native folio reference. */ +static struct mm_subpage *reserve_owner_slot(struct mm_subpage_owner *owner, + unsigned int index, bool *needs_zero) +{ + struct mm_subpage_pool *pool = owner->pool; + struct mm_subpage *subpage; + + lockdep_assert_held(&pool->lock); + if (pool->unissued == owner) + /* Transfer the supplied reference to the first reservation. */ + pool->unissued = NULL; + else + folio_get(owner->folio); + __set_bit(index, &owner->used); + cow_update_free(pool, BIT(index), -1); + if (needs_zero) + *needs_zero = !test_bit(index, &owner->zeroed); + __clear_bit(index, &owner->zeroed); + if (owner->used == pool->all_slots) + list_del_init(&owner->available); + subpage = &owner->slots[index]; + WRITE_ONCE(subpage->lazyfree, false); + /* The used bit keeps backing alive while data is initialised. */ + VM_BUG_ON(refcount_read(&subpage->refs)); + VM_BUG_ON(atomic_read(&subpage->mapcount)); + return subpage; +} + static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, unsigned long eligible, bool *needs_zero, bool lock_folio) { @@ -996,7 +1045,7 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, subpage = ERR_PTR(-ESHUTDOWN); goto out; } - owner = find_available(pool, eligible); + owner = find_available(pool, eligible, false); if (!owner) { subpage = ERR_PTR(-EAGAIN); goto out; @@ -1029,29 +1078,63 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, return ERR_PTR(-EAGAIN); } } - if (pool->unissued == owner) - /* Transfer the supplied reference to the first reservation. */ - pool->unissued = NULL; - else - folio_get(owner->folio); index = __ffs(~owner->used & eligible); - __set_bit(index, &owner->used); - cow_update_free(pool, BIT(index), -1); - if (needs_zero) - *needs_zero = !test_bit(index, &owner->zeroed); - __clear_bit(index, &owner->zeroed); - if (owner->used == pool->all_slots) - list_del_init(&owner->available); - subpage = &owner->slots[index]; - WRITE_ONCE(subpage->lazyfree, false); - /* The used bit keeps backing alive while data is initialised. */ - VM_BUG_ON(refcount_read(&subpage->refs)); - VM_BUG_ON(atomic_read(&subpage->mapcount)); + subpage = reserve_owner_slot(owner, index, needs_zero); out: spin_unlock_irqrestore(&pool->lock, flags); return subpage; } +/* Reserve one exact-offset batch from one owner, with one folio lock. */ +int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots) +{ + struct mm_subpage_owner *owner; + unsigned long flags, zero = 0; + unsigned int i; + bool needs_zero; + + if (!mask || (mask & ~pool->all_slots)) + return -EINVAL; +retry: + spin_lock_irqsave(&pool->lock, flags); + if (pool->closed) { + spin_unlock_irqrestore(&pool->lock, flags); + return -ESHUTDOWN; + } + owner = find_available(pool, mask, true); + if (!owner) { + spin_unlock_irqrestore(&pool->lock, flags); + return -EAGAIN; + } + if (!folio_trylock(owner->folio)) { + /* Parallel faults use another owner rather than queue behind it. */ + spin_unlock_irqrestore(&pool->lock, flags); + cond_resched(); + goto retry; + } + if (folio_test_swapcache(owner->folio) || folio_test_writeback(owner->folio) || + !cow_owner_eligible(pool, owner->folio)) { + folio_unlock(owner->folio); + spin_unlock_irqrestore(&pool->lock, flags); + return -EAGAIN; + } + for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { + slots[i] = reserve_owner_slot(owner, i, &needs_zero); + if (needs_zero) + zero |= BIT(i); + } + spin_unlock_irqrestore(&pool->lock, flags); + for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { + if (zero & BIT(i)) + clear_user_subpage_range(&owner->folio->page, + i << pool->shift, 1U << pool->shift); + /* Zeroing must precede publication, including reused slots. */ + refcount_set_release(&slots[i]->refs, 1); + } + return 0; +} + static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible, bool lock_folio) { bool needs_zero; diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 4b7c4cf17006d..083ce9dc56bfd 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -7,12 +7,14 @@ CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe TEST_GEN_FILES += compat-shm-probe compat-perf-probe compat-numa-probe compat-numa-balancing-probe +TEST_GEN_PROGS_EXTENDED += anon-hole-refill anon-hole-refill-race anon-hole-refill-reclaim +TEST_FILES += README.hole-refill README.hole-refill-race README.hole-refill-reclaim TEST_FILES += compat-script runtime-app.py include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool: LDLIBS += -pthread +$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/README.hole-refill b/tools/testing/selftests/arm64/user4k/README.hole-refill new file mode 100644 index 0000000000000..e547a6bfc05eb --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.hole-refill @@ -0,0 +1,29 @@ +Anonymous hole-refill regression +================================ + +Run anon-hole-refill through the platform's 4K userspace ABI launcher on a +kernel with a native 16K page size. The test does not change its page ABI. + + ./anon-hole-refill + sudo ./anon-hole-refill --native-16k + +The optional --native-16k argument enables physical density checks and requires +visible pagemap PFNs (CAP_SYS_ADMIN). Without it, four density results are +skipped. Do not use it with a different native page size. Pagemap must be +readable even without privileged PFNs: present bits verify neighbor prefault. + +The test owns 16 MiB of logical data. It discards individual 4K leaves and +writes before reading them, exercising missing-PTE refill rather than zero-page +COW. It checks zero initialization, all retained data and neighboring mappings. +Two phased rounds check slot reuse. A two-hole case checks that one fault +prefaults both missing neighbors before the second is accessed. + +Initial backing must be within one native page of 16 MiB. Final backing after +each refill sequence must be at most 20 MiB, allowing bounded pool slack. The +old fresh-private-folio path consumes 64 MiB after single-hole refill rounds +and 32 MiB after the paired-hole sequence. Temporary coexistence of original +and replacement backing is allowed; the test does not require 16 MiB at every +intermediate step. + +Expect eight TAP results. A 30-second alarm bounds the test. It does not fork, +induce pressure or test concurrent faults; see the companion lifetime tests. diff --git a/tools/testing/selftests/arm64/user4k/README.hole-refill-race b/tools/testing/selftests/arm64/user4k/README.hole-refill-race new file mode 100644 index 0000000000000..96f2994a4765b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.hole-refill-race @@ -0,0 +1,33 @@ +Concurrent hole-refill lifetime test +=================================== + + sudo ./anon-hole-refill-race + +Run with a 4K userspace ABI on the native 16K kernel. CAP_SYS_ADMIN is required +to distinguish real swap entries from migration and poison markers. The target +configuration has MAX_SWAPFILES=27; types 27 through 31 are excluded and counted +separately. + +Four workers own disjoint 16K virtual windows in a 16 MiB region. Two phased +rounds discard and refill individual 4K leaves. Barriers synchronize quarter +phases, exercising concurrent shared-pool reservations without a userspace data +race. Every refill checks zeroing and unchanged neighbors. This covers pool +contention across windows, not simultaneous PTE changes within one window. + +After joining workers and verifying the region, the allocating process forks a +survivor, modifies one native window privately and exits. A supervisor waits +for that exit before releasing the survivor. The survivor verifies original +contents, rewrites all leaf tags, and verifies them again. This checks that +backing survives its original allocating mm and preserves fork isolation. + +The survivor then applies MADV_COLD/PAGEOUT only to its own region, at most 20 +times with 10 ms between attempts. At least one real swapped leaf and complete +byte-exact readback are required for the reclaim result. The reported count +states coverage; one swapped leaf is not broad reclaim qualification. These +advice operations can skip nonlinear pooled folios because their mappings are +not contiguous. Use README.hole-refill-reclaim for broader controlled coverage. + +Expect three TAP passes and no skips or unexpected signals. The supervisor has +a 35-second alarm and children have 30-second alarms. Failure or timeout kills +the dedicated process group; the supervisor reaps surviving descendants. An +external runner should also bound the entire fixture cgroup. diff --git a/tools/testing/selftests/arm64/user4k/README.hole-refill-reclaim b/tools/testing/selftests/arm64/user4k/README.hole-refill-reclaim new file mode 100644 index 0000000000000..6ede9503192b8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.hole-refill-reclaim @@ -0,0 +1,36 @@ +Controlled reclaim of hole-refill backing +======================================== + + sudo ./anon-hole-refill-reclaim --external-reclaim + +This separate fixture repeats the concurrent allocation and owner-exit checks +in README.hole-refill-race. It requires the same page ABI, native granule and +visible swap metadata. After the survivor rewrites and verifies its region, it +prints: + + RECLAIM_READY pid= base= bytes=16777216 owner_mm_exited=1 + +It then waits up to 20 seconds for byte r on stdin. Place only the fixture and +its descendants in a fresh cgroup, with the controller outside that cgroup. +The controller may make at most three writes of "24M swappiness=200" to that +cgroup's memory.reclaim. Leave memory.max/high unchanged and do not reclaim +other cgroups or create global pressure. Bound controller work within the +20-second window. + +Record memory.current, memory.swap.current, memory.stat and memory.events, plus +present and valid-swap PTE counts for the reported region. EAGAIN or a partial +reclaim result does not determine success: actual coverage does. Send r after +the bounded attempts, even if coverage is below the required threshold. + +Before reading data back, the survivor snapshots pagemap. It requires at least +3072 of 4096 leaves (75%) to contain real swap entries and verifies every byte +after refault. Insufficient coverage fails this qualification rather than +skipping it. Expect three TAP passes, no skips and no unexpected signals or +kernel warnings. Supervisor and child alarms are 35 and 30 seconds; the +fixture's dedicated process group is killed on failure. + +MADV_COLD/PAGEOUT compares native folio mapcount with a contiguous PTE batch. +Nonlinear pooled quarters can belong to the same requested region while +occupying separate virtual locations, so those advice operations skip them. +This fixture exercises ordinary cgroup reclaim and nonlinear reverse mapping +without removing the advice path's protection for other mappings. diff --git a/tools/testing/selftests/arm64/user4k/anon-hole-refill-race.c b/tools/testing/selftests/arm64/user4k/anon-hole-refill-race.c new file mode 100644 index 0000000000000..965141e52e6f7 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/anon-hole-refill-race.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Bounded partial-window sharing, original-owner exit and real pageout. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define LEAF 4096UL +#define NATIVE 16384UL +#define BYTES (16UL << 20) +#define LEAVES (BYTES / LEAF) +#define WORKERS 4 +#define ROUNDS 2 + +struct result { + bool concurrent, lifetime, reclaimed; + size_t swapped, non_swap_markers; + unsigned int attempts; +}; + +static unsigned char *data; +static pthread_barrier_t barrier; +static volatile sig_atomic_t fixture_group; + +static void stop_fixture(int signal_number) +{ + if (fixture_group > 0) + kill(-fixture_group, SIGKILL); + _exit(128 + signal_number); +} + +static unsigned char tag(size_t leaf, unsigned int generation) +{ + unsigned char value = leaf % 251 + 1; + + if (generation == 1 && leaf < 4) + value ^= 0xa5; + else if (generation == 2) + value ^= 0x5a; + return value; +} + +static bool verify_leaf(size_t leaf, unsigned int generation) +{ + size_t byte; + + if (data[leaf * LEAF] != tag(leaf, generation)) + return false; + for (byte = 1; byte < LEAF; byte++) + if (data[leaf * LEAF + byte] != 0x5a) + return false; + return true; +} + +static bool verify(unsigned int generation) +{ + size_t leaf; + + for (leaf = 0; leaf < LEAVES; leaf++) + if (!verify_leaf(leaf, generation)) + return false; + return true; +} + +static void *worker(void *arg) +{ + unsigned int id = (uintptr_t)arg, round, quarter, q; + size_t group, leaf, byte; + bool ok = true; + + for (round = 0; round < ROUNDS; round++) { + for (quarter = 0; quarter < 4; quarter++) { + pthread_barrier_wait(&barrier); + for (group = id; group < BYTES / NATIVE; group += WORKERS) { + unsigned char *p; + + leaf = group * 4 + quarter; + p = data + leaf * LEAF; + if (madvise(p, LEAF, MADV_DONTNEED)) { + ok = false; + continue; + } + /* A write fault, without a preceding zero-page read. */ + *(volatile unsigned char *)p = tag(leaf, 0); + for (byte = 1; byte < LEAF; byte++) + if (p[byte]) + ok = false; + memset(p + 1, 0x5a, LEAF - 1); + /* This worker owns the entire native virtual window. */ + for (q = 0; q < 4; q++) + ok &= verify_leaf(group * 4 + q, 0); + } + } + } + return (void *)(uintptr_t)!ok; +} + +static size_t swapped_leaves(size_t *markers) +{ + uint64_t entries[LEAVES]; + size_t i, count = 0; + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + ssize_t n; + + if (fd < 0) + return SIZE_MAX; + n = pread(fd, entries, sizeof(entries), (uintptr_t)data / LEAF * 8); + close(fd); + if (n != sizeof(entries)) + return SIZE_MAX; + *markers = 0; + for (i = 0; i < LEAVES; i++) { + if ((entries[i] & (UINT64_C(1) << 63)) || + !(entries[i] & (UINT64_C(1) << 62))) + continue; + /* Audited target config:27..31 are poison/migration/markers. */ + if ((entries[i] & 31) < 27) + count++; + else + (*markers)++; + } + return count; +} + +static bool wait_ok(pid_t pid) +{ + int status; + pid_t got; + + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + return got > 0 && WIFEXITED(status) && !WEXITSTATUS(status); +} + +static void owner(int go, int result_fd) +{ + struct result result = {}; + pthread_t threads[WORKERS]; + void *mapping, *status; + unsigned int i; + bool advice_ok = true, readback_ok; + size_t leaf; + pid_t survivor; + char token; + + alarm(30); + mapping = mmap(NULL, BYTES + NATIVE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + _exit(2); + data = (void *)(((uintptr_t)mapping + NATIVE - 1) & ~(NATIVE - 1)); + memset(data, 0x5a, BYTES); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, 0); + if (pthread_barrier_init(&barrier, NULL, WORKERS)) + _exit(2); + for (i = 0; i < WORKERS; i++) + if (pthread_create(&threads[i], NULL, worker, (void *)(uintptr_t)i)) + _exit(2); + result.concurrent = true; + for (i = 0; i < WORKERS; i++) { + if (pthread_join(threads[i], &status)) + _exit(2); + result.concurrent &= !status; + } + pthread_barrier_destroy(&barrier); + result.concurrent &= verify(0); + if (!result.concurrent) + _exit(3); + survivor = fork(); + if (survivor < 0) + _exit(2); + if (survivor) { + /* The survivor must retain the original contents across this COW. */ + for (leaf = 0; leaf < 4; leaf++) + data[leaf * LEAF] = tag(leaf, 1); + _exit(verify(1) ? 0 : 3); + } + alarm(30); + /* Supervisor waits for the original allocator's mm to disappear first. */ + if (read(go, &token, 1) != 1 || token != 'g') + _exit(2); + close(go); + result.lifetime = verify(0); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, 2); + result.lifetime &= verify(2); + /* No memory.max reduction, memory.reclaim or unrelated memory pressure. */ + for (i = 0; i < 20; i++) { + result.attempts = i + 1; + if (madvise(data, BYTES, MADV_COLD) || madvise(data, BYTES, MADV_PAGEOUT)) { + advice_ok = false; + break; + } + usleep(10000); + result.swapped = swapped_leaves(&result.non_swap_markers); + if (result.swapped == SIZE_MAX) + break; + if (result.swapped == LEAVES) + break; + } + readback_ok = verify(2); + result.reclaimed = advice_ok && result.swapped && + result.swapped != SIZE_MAX && readback_ok; + if (write(result_fd, &result, sizeof(result)) != sizeof(result)) + _exit(2); + close(result_fd); + munmap(mapping, BYTES + NATIVE); + _exit(result.concurrent && result.lifetime && result.reclaimed ? 0 : 3); +} + +static bool visible_swap_metadata(void) +{ + FILE *file = fopen("/proc/self/status", "re"); + char line[256]; + unsigned long long caps = 0; + + if (!file) + return false; + while (fgets(line, sizeof(line), file)) + if (sscanf(line, "CapEff: %llx", &caps) == 1) + break; + fclose(file); + return caps & (1ULL << 21); /* CAP_SYS_ADMIN, needed for swap type bits. */ +} + +int main(void) +{ + struct result result = {}; + int go[2], results[2]; + pid_t child; + bool owner_ok, survivor_ok = false; + ssize_t n = 0; + + ksft_print_header(); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("Run through the established 4K ABI launcher\n"); + if (!visible_swap_metadata()) + ksft_exit_skip("CAP_SYS_ADMIN required to distinguish real swap entries\n"); + signal(SIGPIPE, SIG_IGN); + signal(SIGALRM, stop_fixture); + signal(SIGTERM, stop_fixture); + alarm(35); + if (prctl(PR_SET_CHILD_SUBREAPER, 1) || pipe(go) || pipe(results)) + ksft_exit_fail_msg("setup: %s\n", strerror(errno)); + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork: %s\n", strerror(errno)); + if (!child) { + if (setpgid(0, 0)) + _exit(2); + close(go[1]); + close(results[0]); + owner(go[0], results[1]); + } + fixture_group = child; + if (setpgid(child, child) && errno != EACCES && errno != ESRCH) + stop_fixture(SIGTERM); + close(go[0]); + close(results[1]); + owner_ok = wait_ok(child); + if (owner_ok && write(go[1], "g", 1) == 1) { + n = read(results[0], &result, sizeof(result)); + survivor_ok = wait_ok(-1); + } + close(go[1]); + close(results[0]); + if (!owner_ok || !survivor_ok) { + kill(-child, SIGKILL); + while (waitpid(-1, NULL, 0) > 0 || errno == EINTR) + ; + } + fixture_group = 0; + alarm(0); + ksft_set_plan(3); + ksft_test_result(owner_ok && n == sizeof(result) && result.concurrent, + "four-worker partial-hole refill zeroing and neighbors\n"); + ksft_test_result(owner_ok && n == sizeof(result) && result.lifetime, + "original pool owner exits; descendant fork isolation/readwrite\n"); + ksft_print_msg("pageout attempts=%u real swapped=%zu/%lu excluded markers=%zu\n", + result.attempts, result.swapped, LEAVES, result.non_swap_markers); + ksft_test_result(survivor_ok && result.reclaimed, + "actual bounded pageout and full survivor readback\n"); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/anon-hole-refill-reclaim.c b/tools/testing/selftests/arm64/user4k/anon-hole-refill-reclaim.c new file mode 100644 index 0000000000000..4b15b076b0c1c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/anon-hole-refill-reclaim.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Opt-in own-cgroup reclaim of shared partial-hole backing after owner exit. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define LEAF 4096UL +#define NATIVE 16384UL +#define BYTES (16UL << 20) +#define LEAVES (BYTES / LEAF) +#define WORKERS 4 +#define ROUNDS 2 + +struct result { + bool concurrent, lifetime, reclaimed; + size_t swapped, non_swap_markers; + unsigned int attempts; +}; + +static unsigned char *data; +static pthread_barrier_t barrier; +static volatile sig_atomic_t fixture_group; + +static void stop_fixture(int signal_number) +{ + if (fixture_group > 0) + kill(-fixture_group, SIGKILL); + _exit(128 + signal_number); +} + +static unsigned char tag(size_t leaf, unsigned int generation) +{ + unsigned char value = leaf % 251 + 1; + + if (generation == 1 && leaf < 4) + value ^= 0xa5; + else if (generation == 2) + value ^= 0x5a; + return value; +} + +static bool verify_leaf(size_t leaf, unsigned int generation) +{ + size_t byte; + + if (data[leaf * LEAF] != tag(leaf, generation)) + return false; + for (byte = 1; byte < LEAF; byte++) + if (data[leaf * LEAF + byte] != 0x5a) + return false; + return true; +} + +static bool verify(unsigned int generation) +{ + size_t leaf; + + for (leaf = 0; leaf < LEAVES; leaf++) + if (!verify_leaf(leaf, generation)) + return false; + return true; +} + +static void *worker(void *arg) +{ + unsigned int id = (uintptr_t)arg, round, quarter, q; + size_t group, leaf, byte; + bool ok = true; + + for (round = 0; round < ROUNDS; round++) { + for (quarter = 0; quarter < 4; quarter++) { + pthread_barrier_wait(&barrier); + for (group = id; group < BYTES / NATIVE; group += WORKERS) { + unsigned char *p; + + leaf = group * 4 + quarter; + p = data + leaf * LEAF; + if (madvise(p, LEAF, MADV_DONTNEED)) { + ok = false; + continue; + } + /* A write fault, without a preceding zero-page read. */ + *(volatile unsigned char *)p = tag(leaf, 0); + for (byte = 1; byte < LEAF; byte++) + if (p[byte]) + ok = false; + memset(p + 1, 0x5a, LEAF - 1); + /* This worker owns the entire native virtual window. */ + for (q = 0; q < 4; q++) + ok &= verify_leaf(group * 4 + q, 0); + } + } + } + return (void *)(uintptr_t)!ok; +} + +static size_t swapped_leaves(size_t *markers) +{ + uint64_t entries[LEAVES]; + size_t i, count = 0; + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + ssize_t n; + + if (fd < 0) + return SIZE_MAX; + n = pread(fd, entries, sizeof(entries), (uintptr_t)data / LEAF * 8); + close(fd); + if (n != sizeof(entries)) + return SIZE_MAX; + *markers = 0; + for (i = 0; i < LEAVES; i++) { + if ((entries[i] & (UINT64_C(1) << 63)) || + !(entries[i] & (UINT64_C(1) << 62))) + continue; + /* Audited target config:27..31 are poison/migration/markers. */ + if ((entries[i] & 31) < 27) + count++; + else + (*markers)++; + } + return count; +} + +static bool wait_ok(pid_t pid) +{ + int status; + pid_t got; + + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + return got > 0 && WIFEXITED(status) && !WEXITSTATUS(status); +} + +static void owner(int go, int result_fd) +{ + struct result result = {}; + pthread_t threads[WORKERS]; + void *mapping, *status; + unsigned int i; + bool readback_ok; + struct pollfd control = { .fd = STDIN_FILENO, .events = POLLIN }; + size_t leaf; + pid_t survivor; + char token; + + alarm(30); + mapping = mmap(NULL, BYTES + NATIVE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + _exit(2); + data = (void *)(((uintptr_t)mapping + NATIVE - 1) & ~(NATIVE - 1)); + memset(data, 0x5a, BYTES); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, 0); + if (pthread_barrier_init(&barrier, NULL, WORKERS)) + _exit(2); + for (i = 0; i < WORKERS; i++) + if (pthread_create(&threads[i], NULL, worker, (void *)(uintptr_t)i)) + _exit(2); + result.concurrent = true; + for (i = 0; i < WORKERS; i++) { + if (pthread_join(threads[i], &status)) + _exit(2); + result.concurrent &= !status; + } + pthread_barrier_destroy(&barrier); + result.concurrent &= verify(0); + if (!result.concurrent) + _exit(3); + survivor = fork(); + if (survivor < 0) + _exit(2); + if (survivor) { + /* The survivor must retain the original contents across this COW. */ + for (leaf = 0; leaf < 4; leaf++) + data[leaf * LEAF] = tag(leaf, 1); + _exit(verify(1) ? 0 : 3); + } + alarm(30); + /* Supervisor waits for the original allocator's mm to disappear first. */ + if (read(go, &token, 1) != 1 || token != 'g') + _exit(2); + close(go); + result.lifetime = verify(0); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, 2); + result.lifetime &= verify(2); + /* Controller acts only on this fixture's cgroup. No pressure here. */ + printf("RECLAIM_READY pid=%d base=%p bytes=%lu owner_mm_exited=1\n", + getpid(), data, BYTES); + fflush(stdout); + if (poll(&control, 1, 20000) != 1 || + read(STDIN_FILENO, &token, 1) != 1 || token != 'r') + _exit(2); + result.attempts = 1; /* One post-controller pagemap snapshot. */ + result.swapped = swapped_leaves(&result.non_swap_markers); + readback_ok = verify(2); + result.reclaimed = result.swapped != SIZE_MAX && + result.swapped >= LEAVES * 3 / 4 && readback_ok; + + if (write(result_fd, &result, sizeof(result)) != sizeof(result)) + _exit(2); + close(result_fd); + munmap(mapping, BYTES + NATIVE); + _exit(result.concurrent && result.lifetime && result.reclaimed ? 0 : 3); +} + +static bool visible_swap_metadata(void) +{ + FILE *file = fopen("/proc/self/status", "re"); + char line[256]; + unsigned long long caps = 0; + + if (!file) + return false; + while (fgets(line, sizeof(line), file)) + if (sscanf(line, "CapEff: %llx", &caps) == 1) + break; + fclose(file); + return caps & (1ULL << 21); /* CAP_SYS_ADMIN, needed for swap type bits. */ +} + +int main(int argc, char **argv) +{ + struct result result = {}; + int go[2], results[2]; + pid_t child; + bool owner_ok, survivor_ok = false; + ssize_t n = 0; + + ksft_print_header(); + fflush(stdout); + if (argc != 2 || strcmp(argv[1], "--external-reclaim")) + ksft_exit_skip("Needs explicit --external-reclaim and a bounded own-cgroup controller\n"); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("Run through the established 4K ABI launcher\n"); + if (!visible_swap_metadata()) + ksft_exit_skip("CAP_SYS_ADMIN required to distinguish real swap entries\n"); + signal(SIGPIPE, SIG_IGN); + signal(SIGALRM, stop_fixture); + signal(SIGTERM, stop_fixture); + alarm(35); + if (prctl(PR_SET_CHILD_SUBREAPER, 1) || pipe(go) || pipe(results)) + ksft_exit_fail_msg("setup: %s\n", strerror(errno)); + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork: %s\n", strerror(errno)); + if (!child) { + if (setpgid(0, 0)) + _exit(2); + close(go[1]); + close(results[0]); + owner(go[0], results[1]); + } + fixture_group = child; + if (setpgid(child, child) && errno != EACCES && errno != ESRCH) + stop_fixture(SIGTERM); + close(go[0]); + close(results[1]); + owner_ok = wait_ok(child); + if (owner_ok && write(go[1], "g", 1) == 1) { + n = read(results[0], &result, sizeof(result)); + survivor_ok = wait_ok(-1); + } + close(go[1]); + close(results[0]); + if (!owner_ok || !survivor_ok) { + kill(-child, SIGKILL); + while (waitpid(-1, NULL, 0) > 0 || errno == EINTR) + ; + } + fixture_group = 0; + alarm(0); + ksft_set_plan(3); + ksft_test_result(owner_ok && n == sizeof(result) && result.concurrent, + "four-worker partial-hole refill zeroing and neighbors\n"); + ksft_test_result(owner_ok && n == sizeof(result) && result.lifetime, + "original pool owner exits; descendant fork isolation/readwrite\n"); + ksft_print_msg("post-controller snapshots=%u real swapped=%zu/%lu excluded markers=%zu\n", + result.attempts, result.swapped, LEAVES, result.non_swap_markers); + ksft_test_result(survivor_ok && result.reclaimed, + "own-cgroup reclaim swaps at least75percent, full survivor readback\n"); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/anon-hole-refill.c b/tools/testing/selftests/arm64/user4k/anon-hole-refill.c new file mode 100644 index 0000000000000..e2d71e5cce602 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/anon-hole-refill.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Missing-PTE refill must retain contents, faultaround and native density. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define LEAF 4096UL +#define NATIVE 16384UL +#define BYTES (16UL << 20) +#define LEAVES (BYTES / LEAF) + +static unsigned char *data; +static int pm = -1; +static bool density; + +static unsigned char tag(size_t leaf) +{ + return leaf % 251 + 1; +} + +static bool refill(size_t leaf) +{ + unsigned char *p = data + leaf * LEAF; + size_t i; + + /* Write first: a read would test zero-page COW instead of hole refill. */ + *(volatile unsigned char *)p = tag(leaf); + for (i = 1; i < LEAF; i++) + if (p[i]) + return false; + memset(p + 1, 0x5a, LEAF - 1); + return true; +} + +static bool contents(void) +{ + size_t leaf, i; + + for (leaf = 0; leaf < LEAVES; leaf++) { + if (data[leaf * LEAF] != tag(leaf)) + return false; + for (i = 1; i < LEAF; i++) + if (data[leaf * LEAF + i] != 0x5a) + return false; + } + return true; +} + +static int cmp(const void *a, const void *b) +{ + uint64_t x = *(const uint64_t *)a, y = *(const uint64_t *)b; + + return (x > y) - (x < y); +} + +static int native_bytes(size_t *bytes) +{ + uint64_t entries[LEAVES]; + size_t i, unique = 0; + off_t offset = (uintptr_t)data / LEAF * sizeof(uint64_t); + + if (pm < 0 || pread(pm, entries, sizeof(entries), offset) != sizeof(entries)) + return -1; + for (i = 0; i < LEAVES; i++) { + uint64_t pfn = entries[i] & ((UINT64_C(1) << 55) - 1); + + if (!(entries[i] & (UINT64_C(1) << 63)) || !pfn) + return -1; + entries[i] = pfn / (NATIVE / LEAF); + } + qsort(entries, LEAVES, sizeof(*entries), cmp); + for (i = 0; i < LEAVES; i++) + unique += !i || entries[i] != entries[i - 1]; + *bytes = unique * NATIVE; + return 0; +} + +static void check_density(const char *stage, size_t limit) +{ + size_t bytes; + + if (!density) { + ksft_test_result_skip("%s physical density (needs --native-16k and visible PFNs)\n", stage); + return; + } + if (native_bytes(&bytes)) { + ksft_test_result_fail("%s pagemap unavailable, swapped or hidden\n", stage); + return; + } + ksft_print_msg("%s logical=%lu native=%zu limit=%zu\n", stage, BYTES, bytes, limit); + ksft_test_result(bytes <= limit, "%s physical density\n", stage); +} + +static bool phased_refill(void) +{ + unsigned int quarter; + size_t leaf; + + for (quarter = 0; quarter < 4; quarter++) { + for (leaf = quarter; leaf < LEAVES; leaf += 4) + if (madvise(data + leaf * LEAF, LEAF, MADV_DONTNEED) || !refill(leaf)) + return false; + if (!contents()) + return false; + } + return true; +} + +static bool paired_refill(void) +{ + unsigned int parity; + size_t leaf; + + for (parity = 0; parity < 2; parity++) { + for (leaf = parity; leaf < LEAVES; leaf += 4) { + uint64_t entry; + + if (madvise(data + leaf * LEAF, LEAF, MADV_DONTNEED) || + madvise(data + (leaf + 2) * LEAF, LEAF, MADV_DONTNEED) || + !refill(leaf)) + return false; + /* Present is visible without privileged PFNs. Prove the other + * missing neighbor was prefaulted before touching it ourselves. + */ + if (pm < 0 || pread(pm, &entry, sizeof(entry), + ((uintptr_t)data / LEAF + leaf + 2) * 8) != sizeof(entry) || + !(entry & (UINT64_C(1) << 63)) || !refill(leaf + 2)) + return false; + } + } + return contents(); +} + +int main(int argc, char **argv) +{ + void *mapping; + size_t leaf; + + ksft_print_header(); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("Run through the established 4K ABI launcher\n"); + if (argc == 2 && !strcmp(argv[1], "--native-16k")) + density = true; + else if (argc != 1) + ksft_exit_fail_msg("Usage: %s [--native-16k]\n", argv[0]); + alarm(30); + mapping = mmap(NULL, BYTES + NATIVE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); + data = (void *)(((uintptr_t)mapping + NATIVE - 1) & ~(NATIVE - 1)); + pm = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + memset(data, 0x5a, BYTES); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf); + ksft_set_plan(8); + ksft_test_result(contents(), "initial dense content\n"); + check_density("initial dense", BYTES + NATIVE); + ksft_test_result(phased_refill(), "phased hole refill zeroing and neighbors\n"); + /* Allow bounded pool-bucket slop, but reject the old 64MiB outcome. */ + check_density("phased refill", BYTES + (4UL << 20)); + ksft_test_result(phased_refill(), "reused-slot refill zeroing and neighbors\n"); + check_density("reused-slot refill", BYTES + (4UL << 20)); + ksft_test_result(paired_refill(), "two-hole mask retains missing-neighbor prefault\n"); + check_density("two-hole refill", BYTES + (4UL << 20)); + munmap(mapping, BYTES + NATIVE); + if (pm >= 0) + close(pm); + ksft_finished(); +} From 3b4ae4bb7b24589b80040c81b54ba43daebb802b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:32:47 -0400 Subject: [PATCH 0792/1579] mm: pack single-slot COW across native page quarters A workload writing the same logical quarter in each native window used 16 MiB of backing for 4 MiB of changed data because COW only reused a matching physical quarter. Allow eligible pooled single-slot copies to use any free quarter, retaining exact-offset private fallbacks and dense anonymous batching. Prepare nonlinear reverse mappings before taking PTE locks, including anonymous hole and swap-in publication. Restore swapoff from a locked swap-PTE snapshot and recheck it before installation. Use actual source and destination slot offsets for file and anonymous copies. On Neo, the same-quarter case now uses 4.09375 MiB with byte-exact parent isolation. Hardware checks cover same-native-folio COW, typed GUP and shifted mremap, complete owned-cgroup reclaim/readback, selected-page swapoff, native/4K ABI, hole refill, GPU quarter refault and normal game correctness. Maintained regression binaries match those tested exactly. Ongoing game memory pressure and other first-write allocation paths are separate unresolved issues; this change makes no FPS claim. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6748c940b0e8fd5287a85ad9bda11e254c8901cf) --- include/linux/mm_subpage.h | 7 + mm/memory.c | 58 +-- mm/swapfile.c | 34 +- mm/user-subpage.c | 32 ++ .../testing/selftests/arm64/user4k/.gitignore | 3 + tools/testing/selftests/arm64/user4k/Makefile | 2 + .../selftests/arm64/user4k/README.any-quarter | 37 ++ .../arm64/user4k/check-any-quarter.py | 24 ++ .../arm64/user4k/file-cow-consumers.c | 319 +++++++++++++++++ .../selftests/arm64/user4k/same-native.c | 176 +++++++++ .../selftests/arm64/user4k/same-quarter.c | 335 ++++++++++++++++++ 11 files changed, 991 insertions(+), 36 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.any-quarter create mode 100644 tools/testing/selftests/arm64/user4k/check-any-quarter.py create mode 100644 tools/testing/selftests/arm64/user4k/file-cow-consumers.c create mode 100644 tools/testing/selftests/arm64/user4k/same-native.c create mode 100644 tools/testing/selftests/arm64/user4k/same-quarter.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index 9a1d33166db30..e892ae3e70bee 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -52,6 +52,13 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, void mm_subpage_cow_pool_exit(struct mm_struct *mm); bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio); bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset); +/* COW slots may use any physical quarter; rmap retains the virtual position. */ +bool mm_subpage_cow_wait_busy(struct mm_subpage_pool *pool); +struct mm_subpage *mm_subpage_alloc_locked(struct mm_subpage_pool *pool); +struct mm_subpage *mm_subpage_copy_locked(struct mm_subpage_pool *pool, + const struct mm_subpage *source); +int mm_subpage_prepare_anon_rmap(struct mm_subpage *slot, + struct vm_area_struct *vma, unsigned long address); /* Exact physical-slot mask, one owner and folio lock for the entire batch. * slots is indexed by physical offset >> pool shift; only mask bits are set. * Success returns initialized owned slots; errors leave slots untouched. diff --git a/mm/memory.c b/mm/memory.c index 4006d3cef29ff..cde03950db1e1 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1364,14 +1364,18 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, if (!pool) return ERR_PTR(-ENOMEM); for (;;) { - slot = source ? mm_subpage_copy_at_locked(pool, source, offset) : - mm_subpage_alloc_at_locked(pool, offset); + if (*pooled) + slot = source ? mm_subpage_copy_locked(pool, source) : + mm_subpage_alloc_locked(pool); + else + slot = source ? mm_subpage_copy_at_locked(pool, source, offset) : + mm_subpage_alloc_at_locked(pool, offset); if (!IS_ERR(slot) || PTR_ERR(slot) != -EAGAIN) break; folio = NULL; if (*pooled) { folio = cow_subpage_speculative_folio(mm, vma, addr); - if (!folio && mm_subpage_cow_wait_busy_at(pool, offset)) { + if (!folio && mm_subpage_cow_wait_busy(pool)) { cond_resched(); continue; } @@ -1392,15 +1396,16 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, force_private = true; goto retry_private; } - err = mm_subpage_pool_add_folio_at(pool, folio, offset, - *pooled ? GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN : GFP_KERNEL); + err = *pooled ? mm_subpage_pool_add_folio(pool, folio, + GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN) : + mm_subpage_pool_add_folio_at(pool, folio, offset, GFP_KERNEL); if (*pooled && err == -ENOMEM) { - if (mm_subpage_cow_wait_busy_at(pool, offset)) { + if (mm_subpage_cow_wait_busy(pool)) { folio_put(folio); cond_resched(); continue; } - err = mm_subpage_pool_add_folio_at(pool, folio, offset, GFP_KERNEL); + err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); } if (err) { folio_put(folio); @@ -4639,7 +4644,6 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) struct mmu_notifier_range range; bool copied = false, unshare = vmf->flags & FAULT_FLAG_UNSHARE; bool force_private = false, pooled, fresh, retry_private = false; - unsigned int offset; vm_fault_t ret = VM_FAULT_SIGBUS; pte_t entry; int err; @@ -4689,7 +4693,6 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) ret = vmf_anon_prepare(vmf); if (ret) goto out; - offset = vma_page_offset_at(vma, vmf->address).offset; ret = VM_FAULT_OOM; retry_copy: dest = cow_subpage_prealloc(mm, vma, vmf->address, source, force_private, &pooled); @@ -4701,7 +4704,7 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) } new = mm_subpage_folio(dest); if (old && !source) { - err = copy_user_subpage_range(&new->page, offset, vmf->page, + err = copy_user_subpage_range(&new->page, mm_subpage_offset(dest), vmf->page, pte_phys_mm(mm, vmf->orig_pte) & ~PAGE_MASK, mm_page_size(mm)); if (err) { @@ -4726,13 +4729,11 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) * slot identity before taking PTL, with a fresh-folio fallback on ENOMEM. */ fresh = !mm_subpage_anon_root(dest); - if (!fresh) { - err = mm_subpage_bind_rmap(dest, vma, vmf->address); - if (err) { - ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; - retry_private = pooled && err == -ENOMEM; - goto unlock_folio; - } + err = mm_subpage_prepare_anon_rmap(dest, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + retry_private = pooled && err == -ENOMEM; + goto unlock_folio; } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) @@ -5552,6 +5553,11 @@ static vm_fault_t do_swap_subpage(struct vm_fault *vmf, swp_entry_t entry) ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; goto unlock_folio; } + err = mm_subpage_prepare_anon_rmap(slot, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_folio; + } folio_throttle_swaprate(folio, GFP_KERNEL); vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) @@ -6501,17 +6507,15 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) /* Existing owners may need nonlinear identities. Allocate metadata before * PTL and keep the established fresh-folio fallback on metadata ENOMEM. */ - if (!fresh) { - for_each_set_bit(index, &mask, ARRAY_SIZE(slots)) { - unsigned long address = start + - ((index - first_offset) << mm_page_shift(mm)); + for_each_set_bit(index, &mask, ARRAY_SIZE(slots)) { + unsigned long address = start + + ((index - first_offset) << mm_page_shift(mm)); - err = mm_subpage_bind_rmap(slots[index], vma, address); - if (err) { - ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; - retry_private = pooled && err == -ENOMEM; - goto unlock_folio; - } + err = mm_subpage_prepare_anon_rmap(slots[index], vma, address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + retry_private = pooled && err == -ENOMEM; + goto unlock_folio; } } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); diff --git a/mm/swapfile.c b/mm/swapfile.c index e5faba70227c0..c2c49b9fb3f36 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2453,33 +2453,50 @@ static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte) #ifdef CONFIG_MM_SUBPAGE /* Caller holds the swap-cache folio lock and the VMA's mmap read lock. */ static int unuse_subpage(struct vm_area_struct *vma, pmd_t *pmd, - unsigned long addr, swp_entry_t entry, struct folio *folio, - unsigned int offset) + unsigned long addr, swp_entry_t entry, struct folio *folio) { struct mm_struct *mm = vma->vm_mm; + unsigned int offset; struct mm_subpage *slot = NULL; spinlock_t *ptl; - pte_t *ptep, old, pte; + pte_t *ptep, old, snapshot, pte; bool poisoned = PageHWPoison(&folio->page) || !folio_test_uptodate(folio); int ret = 0; if (folio_order(folio) || folio_test_ksm(folio)) return -EOPNOTSUPP; + /* A swap PTE records the physical quarter, independently of the VMA. + * Snapshot under PTL, then allocate metadata while only folio is locked. + */ + ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!ptep) + return 0; + snapshot = ptep_get(ptep); + if (!softleaf_is_swap(softleaf_from_pte(snapshot)) || + softleaf_from_pte(snapshot).val != entry.val) { + pte_unmap_unlock(ptep, ptl); + return 0; + } + offset = pte_swp_subpage_offset(snapshot); + pte_unmap_unlock(ptep, ptl); + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(mm))) + return -EINVAL; if (!poisoned) { arch_swap_restore(folio_swap(entry, folio), folio); /* Allocate metadata before taking the PTE spinlock. */ slot = mm_subpage_restore_granule(folio, offset, mm_page_shift(mm), GFP_KERNEL); if (IS_ERR(slot)) return PTR_ERR(slot) == -EAGAIN ? 0 : PTR_ERR(slot); + ret = mm_subpage_prepare_anon_rmap(slot, vma, addr); + if (ret) + goto out; } ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); if (!ptep) goto out; old = ptep_get(ptep); - if (softleaf_from_pte(old).val != entry.val) - goto unlock; - /* A moved swap PTE retains its backing offset, not its VMA offset. */ - if (pte_swp_subpage_offset(old) != offset) + /* Includes quarter and flag changes while metadata was prepared. */ + if (!pte_same(old, snapshot)) goto unlock; if (poisoned) { pte = swp_entry_to_pte(PageHWPoison(&folio->page) ? @@ -2540,8 +2557,7 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, #ifdef CONFIG_MM_SUBPAGE if (mm_page_size(vma->vm_mm) < PAGE_SIZE) - return unuse_subpage(vma, pmd, addr, entry, folio, - pte_swp_subpage_offset(orig_pte)); + return unuse_subpage(vma, pmd, addr, entry, folio); #endif swapcache = folio; diff --git a/mm/user-subpage.c b/mm/user-subpage.c index c7876fd34a091..30fb3669458cb 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -630,6 +630,22 @@ int mm_subpage_bind_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, return ret; } +/* Preflight publication without making an unexposed owner look published. + * A fresh linear slot needs no sidecar: add_new_anon_rmap will set its anchor. + * Otherwise bind the independent virtual position before any PTE lock. + */ +int mm_subpage_prepare_anon_rmap(struct mm_subpage *slot, + struct vm_area_struct *vma, unsigned long address) +{ + struct mm_subpage_owner *owner = slot->owner; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + if (!owner->anon_root && !owner->rmap && + mm_subpage_offset(slot) == vma_page_offset_at(vma, address).offset) + return 0; + return mm_subpage_bind_rmap(slot, vma, address); +} + static bool subpage_move_allowed(struct mm_subpage *slot, bool swap) { lockdep_assert_held(&slot->owner->pool->lock); @@ -872,6 +888,11 @@ static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, return false; } +bool mm_subpage_cow_wait_busy(struct mm_subpage_pool *pool) +{ + return cow_wait_busy(pool, pool->all_slots, false); +} + bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) { return cow_wait_busy(pool, offset_mask(pool->shift, offset), false); @@ -1261,6 +1282,17 @@ struct mm_subpage *mm_subpage_copy_at_locked(struct mm_subpage_pool *pool, return eligible ? subpage_copy(pool, source, eligible, true) : ERR_PTR(-EINVAL); } +struct mm_subpage *mm_subpage_alloc_locked(struct mm_subpage_pool *pool) +{ + return subpage_alloc(pool, pool->all_slots, true); +} + +struct mm_subpage *mm_subpage_copy_locked(struct mm_subpage_pool *pool, + const struct mm_subpage *source) +{ + return subpage_copy(pool, source, pool->all_slots, true); +} + /* The page-extension RCU section also protects the pool until its lock is held. */ static struct mm_subpage *restore_bound_slot(struct folio *folio, unsigned int offset, unsigned int shift) diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index 2e6224b832f98..86bb27433258e 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -1,3 +1,6 @@ page-contract swap-fixture-init +same-quarter +file-cow-consumers +same-native diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 083ce9dc56bfd..0c22734f5ad03 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -10,6 +10,8 @@ TEST_GEN_FILES += compat-shm-probe compat-perf-probe compat-numa-probe compat-nu TEST_GEN_PROGS_EXTENDED += anon-hole-refill anon-hole-refill-race anon-hole-refill-reclaim TEST_FILES += README.hole-refill README.hole-refill-race README.hole-refill-reclaim TEST_FILES += compat-script runtime-app.py +TEST_GEN_PROGS_EXTENDED += same-quarter file-cow-consumers same-native +TEST_FILES += README.any-quarter check-any-quarter.py include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S diff --git a/tools/testing/selftests/arm64/user4k/README.any-quarter b/tools/testing/selftests/arm64/user4k/README.any-quarter new file mode 100644 index 0000000000000..305441c8a9db4 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.any-quarter @@ -0,0 +1,37 @@ +Arbitrary-quarter anonymous COW qualification +============================================= + +Run on the audited native16K kernel through an established4K ABI launcher, +with CAP_SYS_ADMIN for visible pagemap PFNs: + userpages 4k ./same-quarter --native-16k > same-quarter.log + python3 check-any-quarter.py same-quarter.log + userpages 4k ./file-cow-consumers --native-16k + userpages 4k ./same-native --native-16k + +same-quarter: two fresh owner+child arms compare4MiB changed data in same versus +balanced virtual quarters of a16MiB dense parent. Exact nativePFN union, no +modified-child alias to parent/unchangedchild, all-byte isolation and histograms. +The binary reports measurements; the checker requires both arms <=6MiB native +for4MiB changed and actual nonzero physical-quarter use in the same-quarter arm. +This catches the old16MiB same-quarter result while allowing bounded pool slack. +30s supervisor/20s child alarms, group/subreaper cleanup, no pressure. + +file-cow-consumers:1MiB owned memfd,two private views; child explicitly READ- +prefaults both views AFTER fork (fork may omit reconstructible file PTEs). +CPU COW, remote typed GUP read-unshare/write-COW each require actual physical/ +logical quarter mismatch and correct file-to-anonymous transition. Then shifted +VA mremap must preserve every actualPFN and byte; post-move GUP and parent/file +isolation are checked.30s supervisor/20s child. Exact stage/byte/PTE diagnostics. +This is typed GUP coverage, not long-term DMA pin or migration qualification. + +same-native:16fresh three-generation trials/16KiB logical payload. B's first COW +is retained by held C while B COWs again. Required distinct4KPFNs sharing native +PFN/4 prove source/destination use one nativefolio; every generation and neighbor +byte stays correct. At least1actual pair is required for PASS; exit77 means +UNEXERCISED, never a coverage pass.20/5/4s supervisor/owner/held-child alarms. + +No fixture changes memory policy, system swap, kernel or GPU state. Native16K +assumption is explicit; do not run density math on an unaudited kernel. Metadata +--selftest modes (where provided) are not kernel-path qualification. UFFD_MOVE +and targeted migration are separate gates; disabled UFFD/single-node systems do +not acquire those qualifications merely by passing these tests. diff --git a/tools/testing/selftests/arm64/user4k/check-any-quarter.py b/tools/testing/selftests/arm64/user4k/check-any-quarter.py new file mode 100644 index 0000000000000..1b71b3922783f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/check-any-quarter.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Validate native backing density from the unchanged same-quarter oracle.""" +import json,sys +from pathlib import Path + +def check(path): + rows=[json.loads(x) for x in Path(path).read_text().splitlines() if x.startswith('{')] + assert len(rows)==2, 'Require exactly two measured arms' + arms={x['arm']:x for x in rows} + assert set(arms)=={'same-quarter-zero','balanced'} + for name,r in arms.items(): + assert r['logical_parent_bytes']==16*1024*1024 and r['modified_child_bytes']==4*1024*1024 + assert r['content_ok'] and r['disjoint_ok'] + assert r['modified_parent_native_overlap']==0 and r['modified_unchanged_native_overlap']==0 + assert r['modified_child']['represented_bytes']==4*1024*1024 + n=r['modified_child']['native_bytes'] + assert 4*1024*1024<=n<=6*1024*1024, (name,'native backing outside packed bound',n) + assert sum(arms['same-quarter-zero']['modified_physical_quarter_counts'][1:])>0 + print('PASS same-quarter/balanced native density, actual relocation and isolation') + +if __name__=='__main__': + if len(sys.argv)!=2:sys.exit('usage: check-any-quarter.py same-quarter.log') + try:check(sys.argv[1]) + except (AssertionError,KeyError,ValueError,OSError) as e:sys.exit('FAIL '+str(e)) diff --git a/tools/testing/selftests/arm64/user4k/file-cow-consumers.c b/tools/testing/selftests/arm64/user4k/file-cow-consumers.c new file mode 100644 index 0000000000000..7d2f82500f637 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/file-cow-consumers.c @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Owned memfd only: file COW, typed remote GUP and shifted mremap. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define LEAF 4096UL +#define NATIVE 16384UL +#define WINDOWS 64UL +#define BYTES (WINDOWS * NATIVE) +#define LEAVES (BYTES / LEAF) +#define PRESENT (UINT64_C(1) << 63) +#define FILE_PAGE (UINT64_C(1) << 61) +#define PFN_MASK ((UINT64_C(1) << 55) - 1) + +struct message { uintptr_t address; uint64_t phase, detail; }; +static volatile sig_atomic_t child_group; +static const char *stage = "startup"; +static bool child_reaped; +static int child_status; +static void report_wait(pid_t pid, int status) +{ + fprintf(stderr, "CHILD_WAIT pid=%ld raw_status=%d exited=%d exit=%d signaled=%d signal=%d\n", + (long)pid, status, WIFEXITED(status), WIFEXITED(status) ? WEXITSTATUS(status) : -1, + WIFSIGNALED(status), WIFSIGNALED(status) ? WTERMSIG(status) : 0); +} + +static void stop(int sig) +{ + if (child_group > 0) kill(-child_group, SIGKILL); + _exit(128 + sig); +} +static bool io_full(int fd, void *buf, size_t n, bool write_it) +{ + char *p = buf; + while (n) { + ssize_t got = write_it ? write(fd, p, n) : read(fd, p, n); + if (got < 0 && errno == EINTR) continue; + if (got <= 0) return false; + p += got; n -= (size_t)got; + } + return true; +} +static unsigned char original(size_t offset) +{ + size_t leaf = offset / LEAF, byte = offset % LEAF; + return (unsigned char)(leaf * 17 + byte * 13 + byte / 256); +} +static bool verify(const unsigned char *data, unsigned kind, size_t gup_a_leaf) +{ + for (size_t at = 0; at < BYTES; at++) { + size_t leaf = at / LEAF, byte = at % LEAF; + unsigned char value = original(at); + if (kind == 1 && leaf % 4 == 3 && byte == 0) value ^= 0x5a; + if (kind == 1 && leaf == gup_a_leaf && byte >= 17 && byte < 49) value = 0xe3; + if (kind == 2 && leaf % 4 == 2 && byte >= 37 && byte < 69) value = 0xc7; + if (data[at] != value) { + fprintf(stderr, "VERIFY_FAIL stage=%s kind=%u offset=0x%zx leaf=%zu byte=%zu expected=%02x actual=%02x\n", + stage, kind, at, leaf, byte, value, data[at]); + return false; + } + } + return true; +} +static bool visible_pfns(void) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256]; unsigned long long caps = 0; + if (!f) return false; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "CapEff: %llx", &caps) == 1) break; + fclose(f); return !!(caps & (1ULL << 21)); +} +static bool entries(pid_t pid, const void *base, uint64_t out[LEAVES]) +{ + char path[64]; snprintf(path, sizeof(path), "/proc/%ld/pagemap", (long)pid); + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) { fprintf(stderr, "PAGEMAP_OPEN_FAIL stage=%s pid=%ld errno=%d\n", stage, (long)pid, errno); return false; } + ssize_t n = pread(fd, out, LEAVES * 8, (uintptr_t)base / LEAF * 8); + close(fd); + if (n != LEAVES * 8) { fprintf(stderr, "PAGEMAP_READ_FAIL stage=%s bytes=%zd errno=%d\n", stage, n, errno); return false; } + for (size_t i = 0; i < LEAVES; i++) + if (!(out[i] & PRESENT) || !(out[i] & PFN_MASK)) { + fprintf(stderr, "PAGEMAP_ENTRY_FAIL stage=%s pid=%ld leaf=%zu raw=%016llx present=%d file=%d pfn_visible=%d\n", + stage, (long)pid, i, (unsigned long long)out[i], !!(out[i] & PRESENT), + !!(out[i] & FILE_PAGE), !!(out[i] & PFN_MASK)); + return false; + } + return true; +} +static bool quarter_file_state(const uint64_t *map, unsigned quarter, bool file) +{ + for (size_t leaf = quarter; leaf < LEAVES; leaf += 4) + if (!!(map[leaf] & FILE_PAGE) != file) { + fprintf(stderr, "PTE_KIND_FAIL stage=%s leaf=%zu expected_file=%d raw=%016llx\n", + stage, leaf, file, (unsigned long long)map[leaf]); + return false; + } + return true; +} +static unsigned mismatches(const uint64_t *map, unsigned logical_quarter, size_t *first) +{ + unsigned count = 0; + for (size_t leaf = logical_quarter; leaf < LEAVES; leaf += 4) { + if ((map[leaf] & PFN_MASK) % 4 != logical_quarter) { + if (!count && first) *first = leaf; + count++; + } + } + fprintf(stderr, "MISMATCH_WITNESS stage=%s logical_quarter=%u mismatches=%u/%lu\n", + stage, logical_quarter, count, WINDOWS); + return count; +} +/* Does not fault payload or require present entries: proves the fork premise. */ +static bool entry_summary(const void *base, const char *view) +{ + uint64_t map[LEAVES]; unsigned present = 0, file = 0; + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + if (fd < 0) return false; + ssize_t n = pread(fd, map, sizeof(map), (uintptr_t)base / LEAF * 8); + close(fd); + if (n != sizeof(map)) return false; + for (size_t i = 0; i < LEAVES; i++) { + present += !!(map[i] & PRESENT); + file += !!(map[i] & PRESENT) && !!(map[i] & FILE_PAGE); + } + fprintf(stderr, "PREFLIGHT stage=%s view=%s present=%u file_present=%u leaves=%lu first_raw=%016llx\n", + stage, view, present, file, LEAVES, (unsigned long long)map[0]); + return true; +} +static bool token(int fd, char wanted) +{ + char got; + return io_full(fd, &got, 1, false) && got == wanted; +} +#define CHILD_CHECK(test, name, code) do { stage = name; if (!(test)) { \ + fprintf(stderr, "CHILD_FAIL stage=%s exit=%d errno=%d\n", stage, code, errno); _exit(code); } } while (0) +static void child(unsigned char *a, unsigned char *b, int control, int status) +{ + struct message msg; + uint64_t before[LEAVES], after[LEAVES]; + alarm(20); + CHILD_CHECK(entry_summary(a, "A") && entry_summary(b, "B"), "before-child-prefault", 2); + /* Fork may skip reconstructible file PTEs. Establish them in this mm. */ + CHILD_CHECK(verify(a, 0, SIZE_MAX), "child-prefault-A-original", 3); + CHILD_CHECK(verify(b, 0, SIZE_MAX), "child-prefault-B-original", 3); + CHILD_CHECK(entry_summary(a, "A") && entry_summary(b, "B"), "after-child-prefault", 2); + CHILD_CHECK(entries(getpid(), a, before), "CPU-source-present", 3); + CHILD_CHECK(quarter_file_state(before, 3, true), "CPU-source-file-PTE", 3); + for (size_t window = 0; window < WINDOWS; window++) { + size_t at = (window * 4 + 3) * LEAF; + *(volatile unsigned char *)(a + at) = original(at) ^ 0x5a; + } + CHILD_CHECK(verify(a, 1, SIZE_MAX), "CPU-COW-full-bytes", 3); + CHILD_CHECK(entries(getpid(), a, after), "CPU-destination-present", 3); + CHILD_CHECK(quarter_file_state(after, 3, false), "CPU-destination-anon-PTE", 3); + size_t first = SIZE_MAX; + unsigned cpu_mismatch = mismatches(after, 3, &first); + CHILD_CHECK(cpu_mismatch, "CPU-COW-actual-quarter-mismatch", 5); + msg = (struct message){ .address = first, .phase = 1, .detail = cpu_mismatch }; + CHILD_CHECK(io_full(status, &msg, sizeof(msg), true), "CPU-result-write", 2); + CHILD_CHECK(token(control, 'm'), "wait-mremap-command", 2); + CHILD_CHECK(verify(a, 1, first), "post-GUP-A-full-bytes", 3); + CHILD_CHECK(verify(b, 2, SIZE_MAX), "post-GUP-B-full-bytes", 3); + CHILD_CHECK(entries(getpid(), b, before), "before-mremap-PFNs", 3); + void *reserve = mmap(NULL, BYTES + 2 * NATIVE, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHILD_CHECK(reserve != MAP_FAILED, "mremap-reservation", 2); + unsigned long desired = ((uintptr_t)b + LEAF) % NATIVE; + unsigned char *destination = (void *)((((uintptr_t)reserve + NATIVE - 1) & ~(NATIVE - 1)) + desired); + unsigned char *moved = mremap(b, BYTES, BYTES, MREMAP_MAYMOVE | MREMAP_FIXED, destination); + CHILD_CHECK(moved != MAP_FAILED && moved == destination && + (uintptr_t)moved % NATIVE != (uintptr_t)b % NATIVE, "mremap-shifted-VA", 3); + CHILD_CHECK(verify(moved, 2, SIZE_MAX), "mremap-full-bytes", 3); + CHILD_CHECK(entries(getpid(), moved, after), "after-mremap-PFNs", 3); + for (size_t leaf = 0; leaf < LEAVES; leaf++) { + if ((before[leaf] & PFN_MASK) != (after[leaf] & PFN_MASK)) { + fprintf(stderr, "PFN_CHANGE stage=mremap leaf=%zu before=%016llx after=%016llx\n", + leaf, (unsigned long long)before[leaf], (unsigned long long)after[leaf]); + CHILD_CHECK(false, "mremap-preserved-PFNs", 6); + } + } + msg = (struct message){ .address = (uintptr_t)moved, .phase = 2, .detail = desired }; + CHILD_CHECK(io_full(status, &msg, sizeof(msg), true), "mremap-result-write", 2); + CHILD_CHECK(token(control, 'x'), "wait-final-release", 2); + CHILD_CHECK(verify(a, 1, first), "final-A-full-bytes", 3); + CHILD_CHECK(verify(moved, 2, SIZE_MAX), "final-B-full-bytes", 3); + munmap(reserve, BYTES + 2 * NATIVE); munmap(a, BYTES); + close(control); close(status); _exit(0); +} +static bool remote_one(pid_t pid, void *local, void *remote, size_t bytes, bool write_it) +{ + struct iovec l = { .iov_base = local, .iov_len = bytes }; + struct iovec r = { .iov_base = remote, .iov_len = bytes }; + ssize_t n = write_it ? process_vm_writev(pid, &l, 1, &r, 1, 0) : + process_vm_readv(pid, &l, 1, &r, 1, 0); + return n == (ssize_t)bytes; +} +static bool remote_quarters(pid_t pid, void *local, unsigned char *remote, + unsigned quarter, size_t offset, size_t bytes, bool write_it) +{ + struct iovec l = { .iov_base = local, .iov_len = WINDOWS * bytes }; + struct iovec r[WINDOWS]; + for (size_t window = 0; window < WINDOWS; window++) + r[window] = (struct iovec){ .iov_base = remote + (window * 4 + quarter) * LEAF + offset, .iov_len = bytes }; + ssize_t n = write_it ? process_vm_writev(pid, &l, 1, r, WINDOWS, 0) : + process_vm_readv(pid, &l, 1, r, WINDOWS, 0); + return n == (ssize_t)(WINDOWS * bytes); +} +static bool wait_ok(pid_t pid) +{ + int status; pid_t n; + do { n = waitpid(pid, &status, 0); } while (n < 0 && errno == EINTR); + if (n == pid) { child_reaped = true; child_status = status; report_wait(pid, status); } + else fprintf(stderr, "CHILD_WAIT_ERROR pid=%ld errno=%d\n", (long)pid, errno); + return n == pid && WIFEXITED(status) && !WEXITSTATUS(status); +} +#define CHECK(test, message) do { stage = message; if (!(test)) { fprintf(stderr, "FAIL: %s (errno=%d)\n", message, errno); goto cleanup; } } while (0) +int main(int argc, char **argv) +{ + unsigned char *buffer = NULL, *a = MAP_FAILED, *b = MAP_FAILED; + int fd = -1, control[2] = {-1,-1}, status[2] = {-1,-1}; + pid_t pid = -1; bool ok = false; + uint64_t map[LEAVES]; struct message msg; + unsigned cpu_mismatch = 0, read_mismatch = 0, write_mismatch = 0; + size_t first_a = SIZE_MAX, first_b = SIZE_MAX; + if (argc != 2 || strcmp(argv[1], "--native-16k")) { + fprintf(stderr, "usage: %s --native-16k\n", argv[0]); return 2; + } + if (getauxval(AT_PAGESZ) != LEAF || !visible_pfns()) { + fprintf(stderr, "SKIP: established4K launcher/native16K kernel/visiblePFNs required\n"); return 4; + } + signal(SIGPIPE, SIG_IGN); signal(SIGALRM, stop); signal(SIGTERM, stop); signal(SIGINT, stop); + alarm(30); + buffer = malloc(BYTES); + CHECK(buffer, "buffer allocation"); + for (size_t i = 0; i < BYTES; i++) buffer[i] = original(i); + fd = memfd_create("own-file-cow-consumers", MFD_CLOEXEC); + CHECK(fd >= 0 && !ftruncate(fd, BYTES), "owned memfd"); + CHECK(pwrite(fd, buffer, BYTES, 0) == BYTES, "initialize memfd"); + a = mmap(NULL, BYTES, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + b = mmap(NULL, BYTES, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + CHECK(a != MAP_FAILED && b != MAP_FAILED, "private file views"); + /* Full reads establish file PTEs before COW and typed pin operations. */ + CHECK(verify(a, 0, SIZE_MAX) && verify(b, 0, SIZE_MAX), "prefault originals"); + CHECK(!pipe(control) && !pipe(status), "control pipes"); + pid = fork(); CHECK(pid >= 0, "fork"); + if (!pid) { + CHILD_CHECK(!setpgid(0, 0), "set-child-group", 2); + close(control[1]); close(status[0]); close(fd); + child(a, b, control[0], status[1]); + } + child_group = pid; + if (setpgid(pid, pid) && errno != EACCES && errno != ESRCH) CHECK(false, "child group"); + close(control[0]); control[0] = -1; close(status[1]); status[1] = -1; + bool received = io_full(status[0], &msg, sizeof(msg), false); + if (!received) { fprintf(stderr, "RESULT_EOF stage=CPU-COW; waiting for exact child status\n"); (void)wait_ok(pid); } + CHECK(received && msg.phase == 1, "CPU COW result"); + cpu_mismatch = msg.detail; first_a = msg.address; + CHECK(cpu_mismatch && first_a < LEAVES && first_a % 4 == 3, "actual CPU COW offset mismatch"); + CHECK(remote_one(pid, buffer, a + first_a * LEAF, LEAF, false), "GUP read of mismatched anonymous slot"); + for (size_t i = 0; i < LEAF; i++) + CHECK(buffer[i] == (unsigned char)(original(first_a * LEAF + i) ^ (i == 0 ? 0x5a : 0)), "GUP read bytes"); + memset(buffer, 0xe3, 32); + CHECK(remote_one(pid, buffer, a + first_a * LEAF + 17, 32, true), "GUP write of mismatched anonymous slot"); + CHECK(entries(pid, b, map) && quarter_file_state(map, 3, true), "read-pin sources remain file PTEs"); + CHECK(remote_quarters(pid, buffer, b, 3, 0, LEAF, false), "typed read-pin file unshare"); + for (size_t w = 0; w < WINDOWS; w++) + for (size_t i = 0; i < LEAF; i++) + CHECK(buffer[w * LEAF + i] == original((w * 4 + 3) * LEAF + i), "read-pin copied bytes"); + CHECK(entries(pid, b, map) && quarter_file_state(map, 3, false), "read-pin produced anonymous slots"); + read_mismatch = mismatches(map, 3, &first_b); + CHECK(read_mismatch && first_b < LEAVES, "actual read-pin file COW offset mismatch"); + CHECK(quarter_file_state(map, 2, true), "write-pin sources remain file PTEs"); + memset(buffer, 0xc7, WINDOWS * 32); + CHECK(remote_quarters(pid, buffer, b, 2, 37, 32, true), "typed write-pin file COW"); + CHECK(entries(pid, b, map) && quarter_file_state(map, 2, false), "write-pin produced anonymous slots"); + write_mismatch = mismatches(map, 2, NULL); + CHECK(write_mismatch, "actual write-pin file COW offset mismatch"); + char command = 'm'; CHECK(io_full(control[1], &command, 1, true), "mremap command"); + received = io_full(status[0], &msg, sizeof(msg), false); + if (!received) { fprintf(stderr, "RESULT_EOF stage=mremap; waiting for exact child status\n"); (void)wait_ok(pid); } + CHECK(received && msg.phase == 2, "mremap content/PFN result"); + unsigned char *moved = (void *)msg.address; + CHECK(remote_one(pid, buffer, moved + first_b * LEAF, LEAF, false), "GUP read after shifted mremap"); + for (size_t i = 0; i < LEAF; i++) CHECK(buffer[i] == original(first_b * LEAF + i), "moved GUP bytes"); + CHECK(verify(a, 0, SIZE_MAX) && verify(b, 0, SIZE_MAX), "parent private views unchanged"); + CHECK(pread(fd, buffer, BYTES, 0) == BYTES && verify(buffer, 0, SIZE_MAX), "original memfd byte exact"); + command = 'x'; CHECK(io_full(control[1], &command, 1, true), "child release"); + CHECK(wait_ok(pid), "child final integrity"); pid = -1; child_group = 0; + printf("{\"file_bytes\":%lu,\"cpu_cow_mismatches\":%u,\"read_pin_cow_mismatches\":%u," + "\"write_pin_cow_mismatches\":%u,\"targets_per_path\":%lu,\"mremap_shifted_va_quarter\":true," + "\"mremap_preserved_pfns\":true,\"gup_after_mremap\":true,\"parent_and_file_unchanged\":true,\"content_ok\":true}\n", + BYTES, cpu_mismatch, read_mismatch, write_mismatch, WINDOWS); + ok = true; +cleanup: + if (pid > 0 && !child_reaped) { + pid_t got = waitpid(pid, &child_status, WNOHANG); + if (got == pid) { child_reaped = true; report_wait(pid, child_status); } + else { kill(-pid, SIGKILL); (void)wait_ok(pid); } + } + child_group = 0; alarm(0); + for (unsigned i = 0; i < 2; i++) { if (control[i] >= 0) close(control[i]); if (status[i] >= 0) close(status[i]); } + if (a != MAP_FAILED) munmap(a, BYTES); + if (b != MAP_FAILED) munmap(b, BYTES); + if (fd >= 0) close(fd); + free(buffer); + return !ok; +} diff --git a/tools/testing/selftests/arm64/user4k/same-native.c b/tools/testing/selftests/arm64/user4k/same-native.c new file mode 100644 index 0000000000000..9c592ee98c3a7 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/same-native.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Three generations; actual PFNs distinguish same-native-folio COW coverage. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define LEAF 4096UL +#define NATIVE 16384UL +#define TRIALS 16 +#define PFN_MASK ((UINT64_C(1) << 55) - 1) +struct result { uint64_t old_pfn, new_pfn, held_pfn; unsigned quarter; }; +static volatile sig_atomic_t group; +static unsigned char *data; +static void stop(int sig) +{ + if (group > 0) kill(-group, SIGKILL); + _exit(128 + sig); +} +static bool io_full(int fd, void *buf, size_t bytes, bool output) +{ + unsigned char *p = buf; + while (bytes) { + ssize_t n = output ? write(fd, p, bytes) : read(fd, p, bytes); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) return false; + p += n; bytes -= n; + } + return true; +} +static unsigned char pattern(size_t at, unsigned generation, unsigned quarter) +{ + unsigned salt = at / LEAF == quarter ? generation * 53 : 0; + return (unsigned char)((at * 17 + at / LEAF * 29 + at / 256 + salt) % 251 + 1); +} +static void change(unsigned generation, unsigned quarter) +{ + size_t at; + for (at = quarter * LEAF; at < (quarter + 1) * LEAF; at++) + data[at] = pattern(at, generation, quarter); +} +static bool verify(unsigned generation, unsigned quarter) +{ + size_t at; + for (at = 0; at < NATIVE; at++) + if (data[at] != pattern(at, generation, quarter)) return false; + return true; +} +static bool pfn(int fd, unsigned quarter, uint64_t *out) +{ + uint64_t entry; + off_t where = ((uintptr_t)data / LEAF + quarter) * 8; + if (pread(fd, &entry, 8, where) != 8 || !(entry & (UINT64_C(1) << 63)) || + (entry & (UINT64_C(1) << 62)) || !(entry & PFN_MASK)) return false; + *out = entry & PFN_MASK; + return true; +} +static bool wait_ok(pid_t child) +{ + int status; pid_t got; + do { got = waitpid(child, &status, 0); } while (got < 0 && errno == EINTR); + return got == child && WIFEXITED(status) && WEXITSTATUS(status) == 0; +} +static bool caps(void) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256]; unsigned long long effective = 0; + if (!f) return false; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "CapEff: %llx", &effective) == 1) break; + fclose(f); return !!(effective & (1ULL << 21)); +} +static void owner(int output, unsigned quarter) +{ + struct result result = { .quarter = quarter }; + int go[2], answer[2], mapfd; + char token; + pid_t held; + alarm(5); + if (pipe(go) || pipe(answer)) _exit(10); + /* Open in this mm, never inherit an ancestor's already-open pagemap fd. */ + mapfd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + if (mapfd < 0) _exit(11); + change(1, quarter); + if (!pfn(mapfd, quarter, &result.old_pfn) || !verify(1, quarter)) _exit(12); + held = fork(); + if (held < 0) _exit(13); + if (!held) { + uint64_t held_pfn; + close(go[1]); close(answer[0]); close(output); close(mapfd); + alarm(4); + if (!io_full(go[0], &token, 1, false) || token != 'r') _exit(14); + mapfd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + if (mapfd < 0 || !pfn(mapfd, quarter, &held_pfn) || + held_pfn != result.old_pfn || !verify(1, quarter) || + !io_full(answer[1], &held_pfn, sizeof(held_pfn), true)) _exit(15); + close(mapfd); close(go[0]); close(answer[1]); _exit(0); + } + /* Before libc/pipe bookkeeping, consume a free sibling if allocator picks it. */ + change(2, quarter); + if (!pfn(mapfd, quarter, &result.new_pfn) || + result.new_pfn == result.old_pfn || !verify(2, quarter)) _exit(16); + close(go[0]); close(answer[1]); token = 'r'; + if (!io_full(go[1], &token, 1, true) || + !io_full(answer[0], &result.held_pfn, sizeof(result.held_pfn), false) || + !wait_ok(held) || !verify(2, quarter)) _exit(17); + close(go[1]); close(answer[0]); close(mapfd); + if (!io_full(output, &result, sizeof(result), true)) _exit(18); + close(output); _exit(0); +} +int main(int argc, char **argv) +{ + unsigned same = 0; + if (argc == 2 && !strcmp(argv[1], "--selftest")) { + for (unsigned q = 0; q < 4; q++) + for (size_t at = 0; at < NATIVE; at++) + if ((pattern(at, 1, q) != pattern(at, 2, q)) != (at / LEAF == q)) return 1; + puts("PASS generation pattern selftest"); return 0; + } + if (argc != 2 || strcmp(argv[1], "--native-16k") || + getauxval(AT_PAGESZ) != LEAF || !caps()) { + fputs("Requires established4K launcher/native16K kernel and visible PFNs\n", stderr); return 77; + } + if (prctl(PR_SET_CHILD_SUBREAPER, 1)) return 1; + signal(SIGPIPE, SIG_IGN); signal(SIGALRM, stop); signal(SIGTERM, stop); signal(SIGINT, stop); + setvbuf(stdout, NULL, _IONBF, 0); alarm(20); + void *map = mmap(NULL, 2 * NATIVE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (map == MAP_FAILED) return 1; + data = (void *)(((uintptr_t)map + NATIVE - 1) & ~(NATIVE - 1)); + for (size_t at = 0; at < NATIVE; at++) data[at] = pattern(at, 0, 0); + for (unsigned trial = 0; trial < TRIALS; trial++) { + int output[2]; struct result result; pid_t pid; + if (pipe(output)) return 1; + pid = fork(); + if (pid < 0) return 1; + if (!pid) { + group = 0; + if (setpgid(0, 0)) _exit(1); + close(output[0]); owner(output[1], trial % 4); + } + group = pid; + if (setpgid(pid, pid) && errno != EACCES && errno != ESRCH) { + kill(pid, SIGKILL); stop(SIGTERM); + } + close(output[1]); + bool received = io_full(output[0], &result, sizeof(result), false); + close(output[0]); + bool good = wait_ok(pid); + /* Reap any orphaned held generation on failure; no child can escape group. */ + kill(-pid, SIGKILL); + while (waitpid(-1, NULL, 0) > 0 || errno == EINTR) {} + group = 0; + if (!received || !good || result.held_pfn != result.old_pfn || + result.new_pfn == result.old_pfn || !verify(0, 0)) { + fprintf(stderr, "FAIL trial %u generation/PFN/isolation/cleanup\n", trial); return 1; + } + bool native_equal = result.old_pfn / 4 == result.new_pfn / 4; + same += native_equal; + printf("{\"trial\":%u,\"logical_quarter\":%u,\"old_pfn4k\":%"PRIu64",\"new_pfn4k\":%"PRIu64",\"held_pfn4k\":%"PRIu64",\"same_native\":%s,\"three_generation_bytes_ok\":true}\n", + trial, result.quarter, result.old_pfn, result.new_pfn, result.held_pfn, native_equal ? "true" : "false"); + } + munmap(map, 2 * NATIVE); alarm(0); + printf("{\"result\":\"%s\",\"trials\":%u,\"same_native_exercised\":%u,\"payload_bytes\":%lu}\n", same ? "PASS" : "UNEXERCISED", TRIALS, same, NATIVE); + return same ? 0 : 77; +} diff --git a/tools/testing/selftests/arm64/user4k/same-quarter.c b/tools/testing/selftests/arm64/user4k/same-quarter.c new file mode 100644 index 0000000000000..ce1f7e80081d0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/same-quarter.c @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Metadata-only native-backing comparison. Never changes memory policy. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LEAF 4096UL +#define NATIVE 16384UL +#define BYTES (16UL << 20) +#define LEAVES (BYTES / LEAF) +#define WINDOWS (BYTES / NATIVE) +#define MAX_FRAMES (LEAVES * 2) + +struct coverage { + uint64_t native_units, quarters, occupancy[4]; +}; +struct result { + unsigned arm; + bool content_ok, disjoint_ok; + struct coverage parent, child, modified, unchanged, combined; + uint64_t modified_parent_native_overlap; + uint64_t modified_unchanged_native_overlap; + uint64_t modified_quarter_counts[4]; +}; +static volatile sig_atomic_t fixture_group; + +static void stop_fixture(int sig) +{ + if (fixture_group > 0) + kill(-fixture_group, SIGKILL); + _exit(128 + sig); +} +static bool full_io(int fd, void *buffer, size_t bytes, bool writing) +{ + char *p = buffer; + while (bytes) { + ssize_t n = writing ? write(fd, p, bytes) : read(fd, p, bytes); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + return false; + p += n; + bytes -= (size_t)n; + } + return true; +} +static bool wait_ok(pid_t pid) +{ + int status; + pid_t got; + do { got = waitpid(pid, &status, 0); } while (got < 0 && errno == EINTR); + return got > 0 && WIFEXITED(status) && WEXITSTATUS(status) == 0; +} +static int compare_u64(const void *a, const void *b) +{ + uint64_t x = *(const uint64_t *)a, y = *(const uint64_t *)b; + return (x > y) - (x < y); +} +/* Input is actual4K PFN, never VMA-derived quarter position. Duplicates count once. */ +static struct coverage summarize(uint64_t *frames, size_t count) +{ + struct coverage out = {}; + uint64_t last = UINT64_MAX; + unsigned mask = 0; + qsort(frames, count, sizeof(*frames), compare_u64); + for (size_t i = 0; i < count; i++) { + uint64_t native = frames[i] / 4; + if (native != last) { + if (mask) { + unsigned n = __builtin_popcount(mask); + out.native_units++; + out.quarters += n; + out.occupancy[n - 1]++; + } + last = native; + mask = 0; + } + mask |= 1U << (frames[i] % 4); + } + if (mask) { + unsigned n = __builtin_popcount(mask); + out.native_units++; + out.quarters += n; + out.occupancy[n - 1]++; + } + return out; +} +/* Both arrays were sorted by summarize. Count each common nativeunit once. */ +static uint64_t native_overlap(const uint64_t *a, size_t an, const uint64_t *b, size_t bn) +{ + size_t ai = 0, bi = 0; + uint64_t count = 0; + while (ai < an && bi < bn) { + uint64_t av = a[ai] / 4, bv = b[bi] / 4; + if (av == bv) + count++; + if (av <= bv) + while (ai < an && a[ai] / 4 == av) ai++; + if (bv <= av) + while (bi < bn && b[bi] / 4 == bv) bi++; + } + return count; +} +static bool read_frames(pid_t pid, const unsigned char *data, uint64_t frames[LEAVES]) +{ + char path[64]; + uint64_t entries[LEAVES]; + snprintf(path, sizeof(path), "/proc/%ld/pagemap", (long)pid); + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return false; + ssize_t n = pread(fd, entries, sizeof(entries), (uintptr_t)data / LEAF * 8); + close(fd); + if (n != (ssize_t)sizeof(entries)) + return false; + for (size_t i = 0; i < LEAVES; i++) { + uint64_t pfn = entries[i] & ((UINT64_C(1) << 55) - 1); + if (!(entries[i] & (UINT64_C(1) << 63)) || !pfn) + return false; /* Swap/hiddenPFNs invalidate this bounded resident comparison. */ + frames[i] = pfn; + } + return true; +} +static unsigned chosen_quarter(unsigned arm, size_t window) +{ + return arm ? window % 4 : 0; +} +static unsigned char tag(size_t leaf, bool changed) +{ + unsigned char value = (unsigned char)(leaf % 251 + 1); + return changed ? value ^ 0xa5 : value; +} +static bool contents(const unsigned char *data, unsigned arm, bool child) +{ + for (size_t leaf = 0; leaf < LEAVES; leaf++) { + bool changed = child && leaf % 4 == chosen_quarter(arm, leaf / 4); + if (data[leaf * LEAF] != tag(leaf, changed)) + return false; + for (size_t byte = 1; byte < LEAF; byte++) + if (data[leaf * LEAF + byte] != 0x5a) + return false; + } + return true; +} +static void owner(unsigned arm, int output) +{ + uint64_t parent[LEAVES], child_frames[LEAVES], modified[WINDOWS]; + uint64_t unchanged[LEAVES - WINDOWS], combined[MAX_FRAMES]; + struct result result = {.arm = arm}; + int ready[2], release[2]; + char token; + size_t mi = 0, ui = 0; + alarm(20); + void *mapping = mmap(NULL, BYTES + 2 * NATIVE, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + _exit(2); + unsigned char *data = (void *)(((uintptr_t)mapping + NATIVE - 1) & ~(NATIVE - 1)); + if (mprotect(data, BYTES, PROT_READ | PROT_WRITE)) + _exit(2); + memset(data, 0x5a, BYTES); + for (size_t leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, false); + if (!contents(data, arm, false) || !read_frames(getpid(), data, parent)) + _exit(3); + struct coverage initial = summarize(parent, LEAVES); + if (initial.native_units * NATIVE != BYTES || initial.quarters != LEAVES) + _exit(4); /* Stop beforefork if the32MiB payload ceiling is not established. */ + if (pipe(ready) || pipe(release)) + _exit(2); + pid_t child = fork(); + if (child < 0) + _exit(2); + if (!child) { + alarm(20); + close(ready[0]); close(release[1]); close(output); + for (size_t window = 0; window < WINDOWS; window++) { + size_t leaf = window * 4 + chosen_quarter(arm, window); + *(volatile unsigned char *)(data + leaf * LEAF) = tag(leaf, true); + } + if (!contents(data, arm, true)) + _exit(3); + token = 'r'; + if (!full_io(ready[1], &token, 1, true)) + _exit(2); + close(ready[1]); + if (!full_io(release[0], &token, 1, false) || token != 'x') + _exit(2); + close(release[0]); + _exit(contents(data, arm, true) ? 0 : 3); + } + close(ready[1]); close(release[0]); + if (!full_io(ready[0], &token, 1, false) || token != 'r') + _exit(2); + close(ready[0]); + result.content_ok = contents(data, arm, false); + if (!result.content_ok || !read_frames(getpid(), data, parent) || + !read_frames(child, data, child_frames)) + _exit(3); + result.disjoint_ok = true; + for (size_t leaf = 0; leaf < LEAVES; leaf++) { + bool changed = leaf % 4 == chosen_quarter(arm, leaf / 4); + if (changed) { + modified[mi++] = child_frames[leaf]; + result.modified_quarter_counts[child_frames[leaf] % 4]++; + result.disjoint_ok &= parent[leaf] != child_frames[leaf]; + } else { + unchanged[ui++] = child_frames[leaf]; + result.disjoint_ok &= parent[leaf] == child_frames[leaf]; + } + } + memcpy(combined, parent, sizeof(parent)); + memcpy(combined + LEAVES, child_frames, sizeof(child_frames)); + result.parent = summarize(parent, LEAVES); + result.child = summarize(child_frames, LEAVES); + result.modified = summarize(modified, mi); + result.unchanged = summarize(unchanged, ui); + result.combined = summarize(combined, MAX_FRAMES); + result.modified_parent_native_overlap = native_overlap(modified, mi, parent, LEAVES); + result.modified_unchanged_native_overlap = native_overlap(modified, mi, unchanged, ui); + result.disjoint_ok &= mi == WINDOWS && result.modified.quarters == WINDOWS && + result.combined.quarters == LEAVES + WINDOWS && + result.modified_parent_native_overlap == 0 && + result.modified_unchanged_native_overlap == 0 && + result.combined.native_units * NATIVE <= 2 * BYTES; + token = 'x'; + if (!full_io(release[1], &token, 1, true)) + _exit(2); + close(release[1]); + bool child_ok = wait_ok(child); + result.content_ok &= child_ok && contents(data, arm, false); + munmap(mapping, BYTES + 2 * NATIVE); + if (!full_io(output, &result, sizeof(result), true)) + _exit(2); + close(output); + _exit(result.content_ok && result.disjoint_ok ? 0 : 3); +} +static bool visible_pfns(void) +{ + FILE *file = fopen("/proc/self/status", "re"); + char line[256]; + unsigned long long caps = 0; + if (!file) return false; + while (fgets(line, sizeof(line), file)) + if (sscanf(line, "CapEff: %llx", &caps) == 1) break; + fclose(file); + return !!(caps & (1ULL << 21)); +} +static void print_coverage(const char *name, struct coverage c) +{ + printf("\"%s\":{\"native_bytes\":%llu,\"represented_bytes\":%llu," + "\"quarter_occupancy\":[%llu,%llu,%llu,%llu]}", name, + (unsigned long long)(c.native_units * NATIVE), + (unsigned long long)(c.quarters * LEAF), + (unsigned long long)c.occupancy[0], (unsigned long long)c.occupancy[1], + (unsigned long long)c.occupancy[2], (unsigned long long)c.occupancy[3]); +} +static int selftest(void) +{ + uint64_t a[] = {400, 401, 401, 403, 404}, b[] = {402, 405}; + uint64_t combined[] = {400, 401, 401, 403, 404, 402, 405}; + struct coverage ca = summarize(a, 5), cb = summarize(b, 2); + struct coverage cc = summarize(combined, 7); + bool ok = ca.native_units == 2 && ca.quarters == 4 && ca.occupancy[0] == 1 && + ca.occupancy[2] == 1 && cb.native_units == 2 && cb.quarters == 2 && + cc.native_units == 2 && cc.quarters == 6 && cc.occupancy[1] == 1 && + cc.occupancy[3] == 1 && native_overlap(a, 5, b, 2) == 2; + uint64_t sparse[] = {400, 404, 408, 412}, dense[] = {400, 401, 402, 403}; + ca = summarize(sparse, 4); cb = summarize(dense, 4); + ok &= ca.native_units == 4 && cb.native_units == 1 && ca.quarters == cb.quarters; + printf("metadata selftest: %s\n", ok ? "PASS" : "FAIL"); + return !ok; +} +int main(int argc, char **argv) +{ + if (argc == 2 && !strcmp(argv[1], "--selftest")) return selftest(); + if (argc != 2 || strcmp(argv[1], "--native-16k")) { fprintf(stderr, "usage: %s --native-16k | --selftest\n", argv[0]); return 2; } + if (getauxval(AT_PAGESZ) != LEAF || !visible_pfns()) { + fprintf(stderr, "SKIP: established4K launcher, native16K kernel and visiblePFNs required\n"); + return 4; + } + signal(SIGPIPE, SIG_IGN); signal(SIGALRM, stop_fixture); signal(SIGTERM, stop_fixture); + signal(SIGINT, stop_fixture); + if (prctl(PR_SET_CHILD_SUBREAPER, 1)) return 2; + alarm(30); + for (unsigned arm = 0; arm < 2; arm++) { + int pipefd[2]; struct result result = {}; bool copied, success; + if (pipe(pipefd)) return 2; + pid_t child = fork(); + if (child < 0) return 2; + if (!child) { + if (setpgid(0, 0)) _exit(2); + close(pipefd[0]); owner(arm, pipefd[1]); + } + fixture_group = child; + if (setpgid(child, child) && errno != EACCES && errno != ESRCH) stop_fixture(SIGTERM); + close(pipefd[1]); + copied = full_io(pipefd[0], &result, sizeof(result), false); + close(pipefd[0]); success = wait_ok(child); + if (!copied || !success) { + kill(-child, SIGKILL); + while (waitpid(-1, NULL, 0) > 0 || errno == EINTR) {} + fprintf(stderr, "FAIL: arm%u incomplete or integrity/coverage check failed\n", arm); + return 1; + } + fixture_group = 0; + printf("{\"arm\":\"%s\",\"logical_parent_bytes\":%lu,\"modified_child_bytes\":%lu,", + arm ? "balanced" : "same-quarter-zero", BYTES, WINDOWS * LEAF); + print_coverage("parent", result.parent); printf(","); + print_coverage("child", result.child); printf(","); + print_coverage("modified_child", result.modified); printf(","); + print_coverage("unchanged_child", result.unchanged); printf(","); + print_coverage("parent_child_union", result.combined); + printf(",\"modified_parent_native_overlap\":%llu,\"modified_unchanged_native_overlap\":%llu," + "\"modified_physical_quarter_counts\":[%llu,%llu,%llu,%llu],\"content_ok\":true,\"disjoint_ok\":true}\n", + (unsigned long long)result.modified_parent_native_overlap, + (unsigned long long)result.modified_unchanged_native_overlap, + (unsigned long long)result.modified_quarter_counts[0], (unsigned long long)result.modified_quarter_counts[1], + (unsigned long long)result.modified_quarter_counts[2], (unsigned long long)result.modified_quarter_counts[3]); + fflush(stdout); + } + alarm(0); + return 0; +} From 155f70716a8e0921f8adfd51c7ca954866c39457 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:49:51 -0400 Subject: [PATCH 0793/1579] mm: share backing for clipped anonymous and file first writes Short anonymous windows and untouched private-file writes allocated a fresh 16 KiB folio for individual 4 KiB destinations. They consumed 64 MiB of backing for 16 MiB of written data on a 16 KiB kernel. Reserve clipped anonymous batches from one shared owner using a matching free-slot count throughout lookup, waiting and supply. Keep slots indexed by their logical mask while zeroing and publishing their actual physical quarters. Preserve full native-window batching and private fallbacks. Pool private-file first writes only for known filemap and shmem callbacks. Release destination locks before invoking the source callback, serialize publication and nonlinear rmap preparation, and mark shared folios uptodate atomically. Retry publication metadata OOM once with private backing, without internally retrying callbacks that dropped the VMA lock. Hardware first-write cases now use 16.06-16.19 MiB for 16 MiB of data. Clipped-neighbor prefaulting, full contents, source isolation, repeated eight-writer first faults, COW/GUP/mremap and actual reclaim/readback pass. Dense allocation remains 16 MiB; sparse wide-VMA prefaulting is unchanged. Maintained regression binaries match the hardware-tested artifacts. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit a0e28af0b29548a42148ce7e2c5054e24b89883a) --- include/linux/mm_subpage.h | 9 + mm/memory.c | 104 +++++++-- mm/user-subpage.c | 73 ++++-- .../testing/selftests/arm64/user4k/.gitignore | 2 + tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../selftests/arm64/user4k/README.first-write | 28 +++ .../arm64/user4k/clipped-firstwrite.c | 141 ++++++++++++ .../selftests/arm64/user4k/first-write.c | 216 ++++++++++++++++++ 8 files changed, 538 insertions(+), 39 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.first-write create mode 100644 tools/testing/selftests/arm64/user4k/clipped-firstwrite.c create mode 100644 tools/testing/selftests/arm64/user4k/first-write.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index e892ae3e70bee..910ed2b36bcfd 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -68,6 +68,15 @@ int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mas int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *folio, unsigned long mask, gfp_t gfp); bool mm_subpage_cow_wait_busy_mask(struct mm_subpage_pool *pool, unsigned long mask); +/* Arbitrary physical offsets, one owner/folio lock for the whole logical mask. + * Output slots are indexed by logical mask bits; use each slot's actual offset. + * Count-aware supply/wait must accompany this allocation to guarantee progress. + */ +int mm_subpage_alloc_any_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots); +int mm_subpage_pool_add_folio_count(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int count, gfp_t gfp); +bool mm_subpage_cow_wait_busy_count(struct mm_subpage_pool *pool, unsigned int count); /* Return an initialized slot with its folio locked; may wait without a pool lock. */ struct mm_subpage *mm_subpage_alloc_at_locked(struct mm_subpage_pool *pool, unsigned int offset); diff --git a/mm/memory.c b/mm/memory.c index cde03950db1e1..77735be88aed5 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6384,28 +6384,33 @@ static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, /* One native-window batch, initialized and locked before PTE publication. */ static int anon_subpage_prealloc_mask(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, - unsigned long mask, bool use_pool, bool *pooled, - struct mm_subpage **slots) + unsigned long mask, bool use_pool, bool any_offset, + bool *pooled, struct mm_subpage **slots) { struct mm_subpage_pool *pool; struct folio *folio; + unsigned int count = hweight_long(mask); + bool arbitrary; int err; retry_private: pool = use_pool ? mm_subpage_cow_pool_get(mm, vma, addr) : NULL; *pooled = pool != NULL; + arbitrary = *pooled && any_offset; if (!pool) pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); if (!pool) return -ENOMEM; for (;;) { - err = mm_subpage_alloc_mask_locked(pool, mask, slots); + err = arbitrary ? mm_subpage_alloc_any_mask_locked(pool, mask, slots) : + mm_subpage_alloc_mask_locked(pool, mask, slots); if (err != -EAGAIN) break; folio = NULL; if (*pooled) { folio = cow_subpage_speculative_folio(mm, vma, addr); - if (!folio && mm_subpage_cow_wait_busy_mask(pool, mask)) { + if (!folio && (arbitrary ? mm_subpage_cow_wait_busy_count(pool, count) : + mm_subpage_cow_wait_busy_mask(pool, mask))) { cond_resched(); continue; } @@ -6422,15 +6427,20 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm, use_pool = false; goto retry_private; } - err = mm_subpage_pool_add_folio_mask(pool, folio, mask, + err = arbitrary ? mm_subpage_pool_add_folio_count(pool, folio, count, + GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN) : + mm_subpage_pool_add_folio_mask(pool, folio, mask, *pooled ? GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN : GFP_KERNEL); if (*pooled && err == -ENOMEM) { - if (mm_subpage_cow_wait_busy_mask(pool, mask)) { + if (arbitrary ? mm_subpage_cow_wait_busy_count(pool, count) : + mm_subpage_cow_wait_busy_mask(pool, mask)) { folio_put(folio); cond_resched(); continue; } - err = mm_subpage_pool_add_folio_mask(pool, folio, mask, GFP_KERNEL); + err = arbitrary ? + mm_subpage_pool_add_folio_count(pool, folio, count, GFP_KERNEL) : + mm_subpage_pool_add_folio_mask(pool, folio, mask, GFP_KERNEL); } if (err) { folio_put(folio); @@ -6493,8 +6503,9 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) pte_unmap(vmf->pte); vmf->pte = NULL; err = anon_subpage_prealloc_mask(mm, vma, vmf->address, mask, - mask != window_mask && !force_private && !userfaultfd_armed(vma), - &pooled, slots); + (mask != window_mask || nr < (PAGE_SIZE >> mm_page_shift(mm))) && + !force_private && !userfaultfd_armed(vma), + nr < (PAGE_SIZE >> mm_page_shift(mm)), &pooled, slots); if (err) return VM_FAULT_OOM; folio = mm_subpage_folio(slots[__ffs(mask)]); @@ -7092,7 +7103,9 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) struct mm_subpage *slot = vmf->cow_subpage; bool is_cow = (vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED); struct folio *folio = page_folio(is_cow ? vmf->cow_page : vmf->page); + bool fresh = false; vm_fault_t ret; + int err; pte_t entry; if (!(vma->vm_flags & VM_SHARED)) { @@ -7106,10 +7119,18 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) else if (pte_alloc(mm, vmf->pmd)) return VM_FAULT_OOM; } - /* The private destination is not yet visible through a PTE. */ + /* A reserved destination slot is private; sibling slots may be published. */ if (is_cow) { folio_get(folio); folio_lock(folio); + if (slot) { + fresh = !mm_subpage_anon_root(slot); + err = mm_subpage_prepare_anon_rmap(slot, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_folio; + } + } } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); ret = VM_FAULT_NOPAGE; @@ -7121,7 +7142,9 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) } if (slot) { mm_subpage_set_exclusive(slot); - if (WARN_ON_ONCE(mm_subpage_add_new_anon_rmap(slot, vma, vmf->address))) { + err = fresh ? mm_subpage_add_new_anon_rmap(slot, vma, vmf->address) : + mm_subpage_add_anon_rmap(slot, vma, vmf->address); + if (WARN_ON_ONCE(err)) { ret = VM_FAULT_SIGBUS; goto unlock; } @@ -7129,7 +7152,8 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) entry = maybe_mkwrite(pte_mkdirty(pte_sw_mkyoung(entry)), vma); if (vmf_orig_pte_uffd_wp(vmf)) entry = pte_mkuffd_wp(entry); - subpage_add_new_lru(folio, vma, 1); + if (fresh) + subpage_add_new_lru(folio, vma, 1); set_pte_at(mm, vmf->address, vmf->pte, entry); update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); inc_mm_counter(mm, MM_ANONPAGES); @@ -7476,6 +7500,10 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; struct folio *folio; vm_fault_t ret; +#ifdef CONFIG_MM_SUBPAGE + bool pooled = false, force_private = false; + vm_fault_t earlier = 0; +#endif ret = vmf_can_call_fault(vmf); if (!ret) @@ -7484,11 +7512,24 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) return ret; #ifdef CONFIG_MM_SUBPAGE +retry_subpage: if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { - vmf->cow_subpage = anon_subpage_prealloc(vma->vm_mm, vma, vmf->address); + /* Known callbacks only supply a locked source page for the core. + * Custom/DAX callbacks may fill a native cow_page or finish COW. + */ + if (!force_private && !userfaultfd_armed(vma) && + (vma->vm_ops->fault == filemap_fault || vma_is_shmem(vma))) { + vmf->cow_subpage = cow_subpage_prealloc(vma->vm_mm, vma, + vmf->address, NULL, false, &pooled); + if (!IS_ERR(vmf->cow_subpage)) + folio_unlock(mm_subpage_folio(vmf->cow_subpage)); + } else { + pooled = false; + vmf->cow_subpage = anon_subpage_prealloc(vma->vm_mm, vma, vmf->address); + } if (IS_ERR(vmf->cow_subpage)) { vmf->cow_subpage = NULL; - return VM_FAULT_OOM; + return earlier | VM_FAULT_OOM; } folio = mm_subpage_folio(vmf->cow_subpage); goto allocated; @@ -7513,12 +7554,22 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) #ifdef CONFIG_MM_SUBPAGE if (vmf->cow_subpage) { - if (copy_user_subpage_range(vmf->cow_page, mm_subpage_offset(vmf->cow_subpage), - vmf->page, vma_page_offset_at(vma, vmf->address).offset, - mm_page_size(vma->vm_mm))) { + int err; + + /* No destination lock crosses the source fault callback or retry. */ + if (pooled) + folio_lock(folio); + err = copy_user_subpage_range(vmf->cow_page, mm_subpage_offset(vmf->cow_subpage), + vmf->page, vma_page_offset_at(vma, vmf->address).offset, + mm_page_size(vma->vm_mm)); + if (pooled) + folio_unlock(folio); + if (err) { ret = VM_FAULT_HWPOISON; goto unlock; } + /* Preserve concurrent sibling lock/LRU/waiter flag updates. */ + folio_mark_uptodate(folio); goto copied; } #endif @@ -7539,21 +7590,36 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) goto unlock; } } + __folio_mark_uptodate(folio); #ifdef CONFIG_MM_SUBPAGE copied: #endif - __folio_mark_uptodate(folio); - ret |= finish_fault(vmf); unlock: unlock_page(vmf->page); put_page(vmf->page); +#ifdef CONFIG_MM_SUBPAGE + vmf->page = NULL; + /* Publication OOM left the PTE untouched. Retry with private metadata, + * outside the source lock; never loop internally on callback RETRY. + */ + if (pooled && (ret & VM_FAULT_OOM)) { + mm_subpage_put(vmf->cow_subpage); + vmf->cow_subpage = NULL; + vmf->cow_page = NULL; + earlier |= ret & VM_FAULT_MAJOR; + force_private = true; + goto retry_subpage; + } + ret |= earlier; +#endif if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) goto uncharge_out; return ret; uncharge_out: #ifdef CONFIG_MM_SUBPAGE + ret |= earlier; if (vmf->cow_subpage) { mm_subpage_put(vmf->cow_subpage); vmf->cow_subpage = NULL; diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 30fb3669458cb..fd553493b3de4 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -859,7 +859,7 @@ static bool cow_owner_eligible(struct mm_subpage_pool *pool, struct folio *folio * allocation: a busy slot farther down the list may require no new charge. */ static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, - bool require_all) + bool require_all, unsigned int count) { struct mm_subpage_owner *owner; struct folio *wait = NULL; @@ -870,6 +870,7 @@ static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, list_for_each_entry(owner, &pool->available, available) { if (!owner->migrating && (~owner->used & eligible) && (!require_all || !(owner->used & eligible)) && + (count == 1 || hweight_long(~owner->used & eligible) >= count) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && !folio_test_writeback(owner->folio)) { @@ -890,21 +891,29 @@ static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, bool mm_subpage_cow_wait_busy(struct mm_subpage_pool *pool) { - return cow_wait_busy(pool, pool->all_slots, false); + return cow_wait_busy(pool, pool->all_slots, false, 1); } bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) { - return cow_wait_busy(pool, offset_mask(pool->shift, offset), false); + return cow_wait_busy(pool, offset_mask(pool->shift, offset), false, 1); } bool mm_subpage_cow_wait_busy_mask(struct mm_subpage_pool *pool, unsigned long mask) { - return cow_wait_busy(pool, mask, true); + return cow_wait_busy(pool, mask, true, 1); +} + +bool mm_subpage_cow_wait_busy_count(struct mm_subpage_pool *pool, unsigned int count) +{ + if (!count || count > hweight_long(pool->all_slots)) + return false; + return cow_wait_busy(pool, pool->all_slots, false, count); } static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, - unsigned long eligible, bool require_all) + unsigned long eligible, bool require_all, + unsigned int count) { struct mm_subpage_owner *owner; unsigned int visited = 0; @@ -927,6 +936,7 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, /* A nonpresent PTE may still need any of this folio's slot bytes. */ if (!owner->migrating && (~owner->used & eligible) && (!require_all || !(owner->used & eligible)) && + (count == 1 || hweight_long(~owner->used & eligible) >= count) && (!pool->cow_shared || !folio_test_locked(owner->folio)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && @@ -937,7 +947,8 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, } static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, - unsigned long eligible, bool require_all, gfp_t gfp) + unsigned long eligible, bool require_all, + unsigned int count, gfp_t gfp) { struct mm_subpage_owner *owner; unsigned long flags; @@ -965,7 +976,7 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, spin_lock_irqsave(&pool->lock, flags); if (pool->closed) ret = -ESHUTDOWN; - else if (pool->unissued || find_available(pool, eligible, require_all)) + else if (pool->unissued || find_available(pool, eligible, require_all, count)) /* Another allocator supplied backing while we allocated metadata. * Preserve the single-unissued-owner invariant even when a shared * COW lookup skips a concurrently locked or rotating list prefix. @@ -993,7 +1004,7 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, gfp_t gfp) { - return pool_add_folio(pool, folio, pool->all_slots, false, gfp); + return pool_add_folio(pool, folio, pool->all_slots, false, 1, gfp); } int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, @@ -1003,7 +1014,7 @@ int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *fol if (!eligible) return -EINVAL; - return pool_add_folio(pool, folio, eligible, false, gfp); + return pool_add_folio(pool, folio, eligible, false, 1, gfp); } int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *folio, @@ -1011,7 +1022,15 @@ int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *f { if (!mask || (mask & ~pool->all_slots)) return -EINVAL; - return pool_add_folio(pool, folio, mask, true, gfp); + return pool_add_folio(pool, folio, mask, true, 1, gfp); +} + +int mm_subpage_pool_add_folio_count(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int count, gfp_t gfp) +{ + if (!count || count > hweight_long(pool->all_slots)) + return -EINVAL; + return pool_add_folio(pool, folio, pool->all_slots, false, count, gfp); } unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) @@ -1066,7 +1085,7 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, subpage = ERR_PTR(-ESHUTDOWN); goto out; } - owner = find_available(pool, eligible, false); + owner = find_available(pool, eligible, false, 1); if (!owner) { subpage = ERR_PTR(-EAGAIN); goto out; @@ -1106,13 +1125,13 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, return subpage; } -/* Reserve one exact-offset batch from one owner, with one folio lock. */ -int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, - struct mm_subpage **slots) +/* One owner/folio lock; output remains indexed by the caller's logical mask. */ +static int subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + bool any_offset, struct mm_subpage **slots) { struct mm_subpage_owner *owner; - unsigned long flags, zero = 0; - unsigned int i; + unsigned long flags, zero = 0, available; + unsigned int i, physical, count = hweight_long(mask); bool needs_zero; if (!mask || (mask & ~pool->all_slots)) @@ -1123,7 +1142,8 @@ int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mas spin_unlock_irqrestore(&pool->lock, flags); return -ESHUTDOWN; } - owner = find_available(pool, mask, true); + owner = find_available(pool, any_offset ? pool->all_slots : mask, + !any_offset, any_offset ? count : 1); if (!owner) { spin_unlock_irqrestore(&pool->lock, flags); return -EAGAIN; @@ -1140,8 +1160,11 @@ int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mas spin_unlock_irqrestore(&pool->lock, flags); return -EAGAIN; } + available = ~owner->used & pool->all_slots; for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { - slots[i] = reserve_owner_slot(owner, i, &needs_zero); + physical = any_offset ? __ffs(available) : i; + available &= ~BIT(physical); + slots[i] = reserve_owner_slot(owner, physical, &needs_zero); if (needs_zero) zero |= BIT(i); } @@ -1149,13 +1172,25 @@ int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mas for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { if (zero & BIT(i)) clear_user_subpage_range(&owner->folio->page, - i << pool->shift, 1U << pool->shift); + mm_subpage_offset(slots[i]), 1U << pool->shift); /* Zeroing must precede publication, including reused slots. */ refcount_set_release(&slots[i]->refs, 1); } return 0; } +int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots) +{ + return subpage_alloc_mask_locked(pool, mask, false, slots); +} + +int mm_subpage_alloc_any_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots) +{ + return subpage_alloc_mask_locked(pool, mask, true, slots); +} + static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible, bool lock_folio) { bool needs_zero; diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index 86bb27433258e..85e45b6ce0399 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -4,3 +4,5 @@ swap-fixture-init same-quarter file-cow-consumers same-native +first-write +clipped-firstwrite diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 0c22734f5ad03..dd9d4f233b056 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -12,11 +12,13 @@ TEST_FILES += README.hole-refill README.hole-refill-race README.hole-refill-recl TEST_FILES += compat-script runtime-app.py TEST_GEN_PROGS_EXTENDED += same-quarter file-cow-consumers same-native TEST_FILES += README.any-quarter check-any-quarter.py +TEST_GEN_PROGS_EXTENDED += first-write clipped-firstwrite +TEST_FILES += README.first-write include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread +$(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/README.first-write b/tools/testing/selftests/arm64/user4k/README.first-write new file mode 100644 index 0000000000000..2ec69368a6c70 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.first-write @@ -0,0 +1,28 @@ +First-write and clipped-native-window backing tests +================================================== + +Run only on the audited native16K kernel through the established4K ABI launcher: + cd /var/tmp/owned-test-dir + userpages 4k ./first-write --native-16k + userpages 4k ./clipped-firstwrite --native-16k + +first-write: six fresh child arms (anonymous,shmem,regular file x dense/guarded), +eight threads, absent-PTE direct first writes, full untouched-byte verification, +DONTNEED and distinct-byte same-PTE refault racing, all source-file bytes verified. +Regular-file working directory MUST be non-tmpfs; otherwise that arm SKIPs. +All six must pass for qualification.4MiB useful payload/arm,<=16MiB native private +backing under a wasteful baseline plus source file.40s supervisor/15s child alarm +kills the fixture child/threads on timeout. No global pressure or kernel change. +This content/race fixture does not itself assert physical density. + +clipped-firstwrite: root CAP_SYS_ADMIN pagemap visibility required.11 fresh child +arms cover all valid starts for1/2/3leaf windows,3+1tail fill and full4dense control; +two rounds with DONTNEED between. A single initial byte write must prefault every +neighbor in its clipped batch, proven by pagemap before any neighbor read. Full +zero/content checks then native PFN union enforce density.256groups/arm, +<=8MiB payload backing even on baseline;16MiB reserved VA.1MiB native slack bounds +pool sharding/background COW. Count3 alone allows natural16K/12K packing; mixed3+1 +must fill tails.40s supervisor/8s child alarm, no GPU/reclaim/pressure. + +The fixture metadata selftest (--selftest) checks tags/sorting only. It is not a +kernel-path test and does not replace running both fixtures on the target kernel. diff --git a/tools/testing/selftests/arm64/user4k/clipped-firstwrite.c b/tools/testing/selftests/arm64/user4k/clipped-firstwrite.c new file mode 100644 index 0000000000000..c07cb1a8a980f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/clipped-firstwrite.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real clipped-window batching, zeroing, reuse and physical backing density. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define LEAF 4096UL +#define NATIVE 16384UL +#define GROUPS 256UL +#define STRIDE (4 * NATIVE) +#define PFN_MASK ((UINT64_C(1)<<55)-1) +#define PRESENT (UINT64_C(1)<<63) +#define SWAPPED (UINT64_C(1)<<62) +static volatile sig_atomic_t child; +static void stop(int sig) { if(child>0)kill(child,SIGKILL);_exit(128+sig); } +static bool visible(void) +{ + FILE *f=fopen("/proc/self/status","re");char line[256];unsigned long long caps=0; + if(!f)return false; + while(fgets(line,sizeof(line),f))if(sscanf(line,"CapEff: %llx",&caps)==1)break; + fclose(f);return !!(caps&(1ULL<<21)); +} +static bool entries(int fd,unsigned char *base,unsigned n,uint64_t *out) +{ + if(pread(fd,out,n*8,(uintptr_t)base/LEAF*8)!=(ssize_t)n*8)return false; + for(unsigned i=0;iy)-(x>2; + if(mix){ + p=base+g*STRIDE+2*NATIVE; + if(!check_window(p,1,g+GROUPS,round)||!entries(fd,p,1,map))return 4; + native[used++]=(map[0]&PFN_MASK)>>2; + } + } + qsort(native,used,sizeof(*native),compare);size_t unique=0; + for(size_t i=0;itarget+(1UL<<20)){fprintf(stderr,"FAIL physical density\n");return 5;} + if(!round)for(unsigned g=0;g +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define LEAF 4096UL +#define LEAVES 1024UL +#define WORKERS 8 +#define TMPFS_MAGIC 0x01021994 + +enum backing { ANONYMOUS, SHMEM, REGULAR }; +static unsigned char *data; +static size_t stride; +static pthread_barrier_t barrier; +static bool race; +static volatile sig_atomic_t running_child; +static void stop_fixture(int sig) +{ + if (running_child > 0) + kill(running_child, SIGKILL); + _exit(128 + sig); +} + +static unsigned char initial(size_t leaf, size_t byte) +{ + return leaf * 17 + byte * 3 + 0x39; +} + +static void *writer(void *arg) +{ + size_t leaf, id = (uintptr_t)arg; + + pthread_barrier_wait(&barrier); + if (race) { + /* Distinct bytes avoid a C data race while faulting the same PTE. */ + for (leaf = 0; leaf < LEAVES; leaf += 3) + data[leaf * stride + id] = initial(leaf, id) ^ 0xa5; + } else { + for (leaf = id; leaf < LEAVES; leaf += WORKERS) + data[leaf * stride] = initial(leaf, 0) ^ 0xa5; + } + return NULL; +} + +static bool verify(enum backing backing) +{ + size_t leaf, byte; + + for (leaf = 0; leaf < LEAVES; leaf++) { + for (byte = 0; byte < LEAF; byte++) { + unsigned char want = backing == ANONYMOUS ? 0 : initial(leaf, byte); + + if (!byte || (race && leaf % 3 == 0 && byte < WORKERS)) + want = initial(leaf, byte) ^ 0xa5; + if (data[leaf * stride + byte] != want) { + ksft_print_msg("leaf=%zu byte=%zu got=%u want=%u\n", leaf, byte, + data[leaf * stride + byte], want); + return false; + } + } + } + return true; +} + +static int exercise(enum backing backing, bool small) +{ + unsigned char buffer[LEAF]; + char name[] = "first-write-XXXXXX"; + pthread_t workers[WORKERS]; + struct statfs fs; + int fd = -1, flags = MAP_PRIVATE; + size_t leaf, byte, span; + unsigned int t; + + stride = small ? 2 * LEAF : LEAF; + span = LEAVES * stride; + if (backing != ANONYMOUS) { + fd = backing == SHMEM ? memfd_create("first-write", MFD_CLOEXEC) : + mkstemp(name); + if (fd < 0) + return 2; + if (backing == REGULAR) + unlink(name); + if (fstatfs(fd, &fs)) + return 2; + if (backing == REGULAR && fs.f_type == TMPFS_MAGIC) { + ksft_print_msg("regular-file arm needs a non-tmpfs working directory\n"); + return KSFT_SKIP; + } + if (ftruncate(fd, span)) + return 2; + for (leaf = 0; leaf < LEAVES; leaf++) { + for (byte = 0; byte < LEAF; byte++) + buffer[byte] = initial(leaf, byte); + if (pwrite(fd, buffer, LEAF, leaf * stride) != LEAF) + return 2; + } + } else { + flags |= MAP_ANONYMOUS; + } + data = mmap(NULL, span, small ? PROT_NONE : PROT_READ | PROT_WRITE, + small ? MAP_PRIVATE | MAP_ANONYMOUS : flags, small ? -1 : fd, 0); + if (data == MAP_FAILED) + return 2; + if (small) + for (leaf = 0; leaf < LEAVES; leaf++) + if (mmap(data + leaf * stride, LEAF, PROT_READ | PROT_WRITE, + flags | MAP_FIXED, fd, backing == ANONYMOUS ? 0 : leaf * stride) == + MAP_FAILED) + return 2; + + /* No reads through the mapping before concurrent direct first writes. */ + if (pthread_barrier_init(&barrier, NULL, WORKERS)) + return 2; + for (t = 0; t < WORKERS; t++) + if (pthread_create(&workers[t], NULL, writer, (void *)(uintptr_t)t)) + return 2; + for (t = 0; t < WORKERS; t++) + if (pthread_join(workers[t], NULL)) + return 2; + if (!verify(backing)) + return 3; + /* Drop selected private slots, leaving the other slots live. */ + for (leaf = 0; leaf < LEAVES; leaf += 3) + if (madvise(data + leaf * stride, LEAF, MADV_DONTNEED)) + return 2; + /* Again no reads before the first writes: exercise abort and slot reuse. */ + race = true; + for (t = 0; t < WORKERS; t++) + if (pthread_create(&workers[t], NULL, writer, (void *)(uintptr_t)t)) + return 2; + for (t = 0; t < WORKERS; t++) + if (pthread_join(workers[t], NULL)) + return 2; + pthread_barrier_destroy(&barrier); + if (!verify(backing)) + return 3; + if (backing != ANONYMOUS) { + for (leaf = 0; leaf < LEAVES; leaf++) { + if (pread(fd, buffer, LEAF, leaf * stride) != LEAF) + return 2; + for (byte = 0; byte < LEAF; byte++) + if (buffer[byte] != initial(leaf, byte)) + return 3; + } + close(fd); + } + munmap(data, span); + return 0; +} + +int main(int argc, char **argv) +{ + static const char * const names[] = { "anonymous", "shmem", "regular-file" }; + unsigned int backing, small; + int status; + pid_t child, result; + + if (argc != 2 || strcmp(argv[1], "--native-16k")) + return 2; + signal(SIGALRM, stop_fixture); + signal(SIGTERM, stop_fixture); + signal(SIGINT, stop_fixture); + ksft_print_header(); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("run under a selected 4K userspace ABI\n"); + ksft_set_plan(6); + fflush(NULL); + alarm(40); + for (backing = ANONYMOUS; backing <= REGULAR; backing++) { + for (small = 0; small <= 1; small++) { + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork: %s\n", strerror(errno)); + if (!child) { + running_child = 0; + alarm(15); + _exit(exercise(backing, small)); + } + running_child = child; + do { + result = waitpid(child, &status, 0); + } while (result < 0 && errno == EINTR); + running_child = 0; + if (result > 0 && WIFEXITED(status) && WEXITSTATUS(status) == KSFT_SKIP) + ksft_test_result_skip("%s %s first writes\n", names[backing], + small ? "guarded" : "dense"); + else + ksft_test_result(result > 0 && WIFEXITED(status) && + !WEXITSTATUS(status), + "%s %s first writes and slot reuse preserve contents\n", + names[backing], small ? "guarded" : "dense"); + } + } + ksft_finished(); +} From 6c877b84d1cb763a039078df4c280f76464da32b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 19:28:15 -0400 Subject: [PATCH 0794/1579] mm: pack pinned-fork subpage copies without changing their charge domain A fork that must copy pinned 4 KiB leaves previously allocated one native 16 KiB folio per leaf. Reserve copies from the child mm's subpage pools, while retaining the source mm for the existing fork allocation charge. Preflight non-linear rmap before taking the page-table locks and copy from the actual source offset after revalidating the parent PTE. Add a bounded real io_uring buffer-pin regression that verifies distinct parent/child native owners, full two-way byte isolation and packed density. On Neo, a 1 MiB copy fell from 4 MiB to 1 MiB of native backing. A held non-root cgroup check verified every payload native PFN retained the correct charge. Retained MM, actual reclaim/readback and GPU controls pass. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6ec0621cc97679bfa90f005c4e0666b319783842) --- include/linux/mm_subpage.h | 4 + mm/memory.c | 74 ++++++-- mm/user-subpage.c | 13 +- .../testing/selftests/arm64/user4k/.gitignore | 1 + tools/testing/selftests/arm64/user4k/Makefile | 2 + .../selftests/arm64/user4k/README.pinned-fork | 37 ++++ .../selftests/arm64/user4k/pinned-fork.c | 172 ++++++++++++++++++ 7 files changed, 285 insertions(+), 18 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.pinned-fork create mode 100644 tools/testing/selftests/arm64/user4k/pinned-fork.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index 910ed2b36bcfd..4a6224f8cc95a 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -49,6 +49,10 @@ void mm_subpage_pool_close(struct mm_subpage_pool *pool); /* Optional per-mm pool: default policy, one memory node, matching memcg. */ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address); +/* Fork's child owns the pool before it joins a cgroup; charge the source mm. */ +struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, + struct mm_struct *charge_mm, struct vm_area_struct *vma, + unsigned long address); void mm_subpage_cow_pool_exit(struct mm_struct *mm); bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio); bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset); diff --git a/mm/memory.c b/mm/memory.c index 77735be88aed5..ea603a72be93d 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1125,7 +1125,7 @@ static void subpage_add_new_lru(struct folio *folio, struct vm_area_struct *vma, mlock_folio(folio); } -/* The destination slot and its charge/metadata were allocated outside PTLs. */ +/* Destination allocation, folio locking and rmap preflight precede both PTLs. */ static int copy_pinned_subpage(struct vm_area_struct *dst_vma, pte_t *dst_pte, pte_t source_pte, unsigned long addr, int *rss, struct mm_subpage *source, struct mm_subpage **prealloc) @@ -1133,23 +1133,26 @@ static int copy_pinned_subpage(struct vm_area_struct *dst_vma, pte_t *dst_pte, struct mm_subpage *dest = *prealloc; struct folio *new; pte_t pte; + bool fresh; int ret; if (!dest) return -EAGAIN; new = mm_subpage_folio(dest); + VM_BUG_ON_FOLIO(!folio_test_locked(new), new); ret = copy_user_subpage_range(&new->page, mm_subpage_offset(dest), &mm_subpage_folio(source)->page, mm_subpage_offset(source), mm_subpage_size(source)); if (ret) return ret; - /* Fresh, unmapped backing is private to this preallocation. */ - if (WARN_ON_ONCE(!folio_trylock(new))) - return -EBUSY; + fresh = !mm_subpage_anon_root(dest); + mm_subpage_set_exclusive(dest); - ret = mm_subpage_add_new_anon_rmap(dest, dst_vma, addr); + ret = fresh ? mm_subpage_add_new_anon_rmap(dest, dst_vma, addr) : + mm_subpage_add_anon_rmap(dest, dst_vma, addr); if (!ret) { - subpage_add_new_lru(new, dst_vma, 1); + if (fresh) + subpage_add_new_lru(new, dst_vma, 1); rss[MM_ANONPAGES]++; pte = phys_pte_mm(dst_vma->vm_mm, mm_subpage_phys(dest), dst_vma->vm_page_prot); pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); @@ -1157,8 +1160,9 @@ static int copy_pinned_subpage(struct vm_area_struct *dst_vma, pte_t *dst_pte, pte = pte_mkuffd_wp(pte); set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); *prealloc = NULL; + folio_unlock(new); } - folio_unlock(new); + /* An unconsumed reservation remains locked until the caller drops PTLs. */ return ret; } #endif @@ -1348,7 +1352,8 @@ static struct folio *cow_subpage_speculative_folio(struct mm_struct *mm, /* The returned initialized slot owns a reference and has its folio locked. */ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, - struct mm_subpage *source, bool force_private, bool *pooled) + struct mm_subpage *source, bool force_private, bool *pooled, + struct mm_struct *charge_mm) { struct mm_subpage_pool *pool; struct mm_subpage *slot; @@ -1357,7 +1362,8 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, int err; retry_private: - pool = force_private ? NULL : mm_subpage_cow_pool_get(mm, vma, addr); + pool = force_private ? NULL : + mm_subpage_cow_pool_get_charged(mm, charge_mm, vma, addr); *pooled = pool != NULL; if (!pool) pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); @@ -1374,7 +1380,7 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, break; folio = NULL; if (*pooled) { - folio = cow_subpage_speculative_folio(mm, vma, addr); + folio = cow_subpage_speculative_folio(charge_mm, vma, addr); if (!folio && mm_subpage_cow_wait_busy(pool)) { cond_resched(); continue; @@ -1384,7 +1390,7 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, * Ordinary allocation/charge semantics are now appropriate. */ if (!folio) - folio = folio_prealloc(mm, vma, addr, false); + folio = folio_prealloc(charge_mm, vma, addr, false); if (!folio) { slot = ERR_PTR(-ENOMEM); break; @@ -1423,6 +1429,30 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, return slot; } +/* The child owns the pool; fork retains the source mm allocation charge. */ +static struct mm_subpage *fork_subpage_prealloc(struct vm_area_struct *dst_vma, + struct mm_struct *src_mm, unsigned long addr) +{ + struct mm_subpage *slot; + bool pooled, force_private = false; + int err; + + for (;;) { + slot = cow_subpage_prealloc(dst_vma->vm_mm, dst_vma, addr, + NULL, force_private, &pooled, src_mm); + if (IS_ERR(slot)) + return slot; + err = mm_subpage_prepare_anon_rmap(slot, dst_vma, addr); + if (!err) + return slot; + folio_unlock(mm_subpage_folio(slot)); + mm_subpage_put(slot); + if (!pooled || err != -ENOMEM) + return ERR_PTR(err); + force_private = true; + } +} + static struct mm_subpage *anon_subpage_prealloc(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr) { @@ -1467,6 +1497,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, softleaf_t entry = softleaf_mk_none(); struct folio *prealloc = NULL; struct mm_subpage *subprealloc = NULL; + unsigned long subprealloc_addr __maybe_unused = 0; int nr; again: @@ -1507,6 +1538,14 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, do { nr = 1; +#ifdef CONFIG_MM_SUBPAGE + /* A vanished source PTE must not move a prepared slot to another VA. */ + if (subprealloc && addr != subprealloc_addr) { + folio_unlock(mm_subpage_folio(subprealloc)); + mm_subpage_put(subprealloc); + subprealloc = NULL; + } +#endif /* * We are holding two locks at this point - either of them @@ -1560,6 +1599,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, break; #ifdef CONFIG_MM_SUBPAGE if (subprealloc) { + folio_unlock(mm_subpage_folio(subprealloc)); mm_subpage_put(subprealloc); subprealloc = NULL; } @@ -1597,7 +1637,8 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } else if (ret == -EAGAIN) { #ifdef CONFIG_MM_SUBPAGE if (mm_page_size(src_mm) < PAGE_SIZE) { - subprealloc = anon_subpage_prealloc(src_mm, src_vma, addr); + subprealloc_addr = addr; + subprealloc = fork_subpage_prealloc(dst_vma, src_mm, addr); if (IS_ERR(subprealloc)) return PTR_ERR(subprealloc); ret = 0; @@ -1621,8 +1662,10 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, goto again; out: #ifdef CONFIG_MM_SUBPAGE - if (subprealloc) + if (subprealloc) { + folio_unlock(mm_subpage_folio(subprealloc)); mm_subpage_put(subprealloc); + } #endif if (unlikely(prealloc)) folio_put(prealloc); @@ -4695,7 +4738,8 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) goto out; ret = VM_FAULT_OOM; retry_copy: - dest = cow_subpage_prealloc(mm, vma, vmf->address, source, force_private, &pooled); + dest = cow_subpage_prealloc(mm, vma, vmf->address, source, force_private, + &pooled, mm); if (IS_ERR(dest)) { if (PTR_ERR(dest) == -EHWPOISON) ret = VM_FAULT_HWPOISON; @@ -7520,7 +7564,7 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) if (!force_private && !userfaultfd_armed(vma) && (vma->vm_ops->fault == filemap_fault || vma_is_shmem(vma))) { vmf->cow_subpage = cow_subpage_prealloc(vma->vm_mm, vma, - vmf->address, NULL, false, &pooled); + vmf->address, NULL, false, &pooled, vma->vm_mm); if (!IS_ERR(vmf->cow_subpage)) folio_unlock(mm_subpage_folio(vmf->cow_subpage)); } else { diff --git a/mm/user-subpage.c b/mm/user-subpage.c index fd553493b3de4..acfa80848e6e3 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -299,8 +299,9 @@ void mm_subpage_pool_put(struct mm_subpage_pool *pool) * Policy-sensitive, locked, droppable and multi-node faults keep fresh backing. * An mm can move cgroups; old charged slots must not satisfy its new faults. */ -struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, - struct vm_area_struct *vma, unsigned long address) +struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, + struct mm_struct *charge_mm, struct vm_area_struct *vma, + unsigned long address) { struct mm_subpage_cow_pools *pools, *new_pools; struct mm_subpage_pool *pool, *new; @@ -334,7 +335,7 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, else kfree(new_pools); } - memcg = get_mem_cgroup_from_mm(mm); + memcg = get_mem_cgroup_from_mm(charge_mm); /* Acquire the immutable charge domain published with this bucket. */ pool = smp_load_acquire(&pools->pool[bucket]); if (!pool) { @@ -375,6 +376,12 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, return pool; } +struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long address) +{ + return mm_subpage_cow_pool_get_charged(mm, mm, vma, address); +} + bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio) { return pool->cow_memcg == folio_memcg(folio) && pool->cow_nid == folio_nid(folio); diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index 85e45b6ce0399..a5b225dbaf61d 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -6,3 +6,4 @@ file-cow-consumers same-native first-write clipped-firstwrite +pinned-fork diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index dd9d4f233b056..a6e4854f454c2 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -14,6 +14,8 @@ TEST_GEN_PROGS_EXTENDED += same-quarter file-cow-consumers same-native TEST_FILES += README.any-quarter check-any-quarter.py TEST_GEN_PROGS_EXTENDED += first-write clipped-firstwrite TEST_FILES += README.first-write +TEST_GEN_PROGS_EXTENDED += pinned-fork +TEST_FILES += README.pinned-fork include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S diff --git a/tools/testing/selftests/arm64/user4k/README.pinned-fork b/tools/testing/selftests/arm64/user4k/README.pinned-fork new file mode 100644 index 0000000000000..129adde8b005c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.pinned-fork @@ -0,0 +1,37 @@ +Pinned anonymous fork density and pin isolation +============================================= + +Build pinned-fork with the other arm64/user4k tests. Run as root through the +established 4K userspace launcher on a native16K kernel: + + userpages 4k ./pinned-fork --native-16k --expect-packed + +The fixture initializes one native-aligned1MiB anonymous range, registers real +writable long-term io_uring buffer pins, then forks while the parent retains the +registration. It requires every copied child native PFN to be disjoint from the +pinned parent's native PFNs. It checks every byte before and after distinct +parent/child writes; registration and PFN disjointness, not mlock or VmPin alone, +establish copy-path coverage. Kernel/pin-policy limitations return77 UNEXERCISED. + +--expect-packed requires child backing <=1.25MiB for1MiB copied data, with bounded +pool-shard slack. Without this option the fixture also accepts the old4MiB +backing so a baseline can record both content and density separately. Successful +contents alone do not establish packed density. --selftest checks metadata PFN +grouping only, with no dynamic pin/fork qualification. + +Optional --hold reports the parent PID, child PID and range base after fork PFN +verification, then waits for the byte 'c' on stdin. A controller can hold both +processes in one fresh uncapped non-root cgroup and verify exact payload charges: +read each target's pagemap in4KiB units; normalize PFN>>2 to native16KiB PFN; +read /proc/kpagecgroup at native_PFN*8 and require each returned inode equals +stat(the fixture cgroup).st_ino. Re-read pagemap to reject migration during the +sample. This directly catches incorrectly root-charged fork copies; total +memory.current includes unrelated fixture code/stack/table costs and is not an +exact payload charge oracle. Verify both PIDs belong to the same owned cgroup, +record memory.stat/status, and release promptly (within3seconds). + +Parent15s/child10s alarms bound the protocol; parent exit kills/reaps the child +and the child uses PDEATHSIG. Payload native backing is at most5MiB on the old +path. No forced reclaim, memory.max/high changes, GPU operations, or swap-area +changes are used. Controllers should additionally enforce their own process-group +or service-unit lifetime and preserve logs on any failure. diff --git a/tools/testing/selftests/arm64/user4k/pinned-fork.c b/tools/testing/selftests/arm64/user4k/pinned-fork.c new file mode 100644 index 0000000000000..ec91136af8de4 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/pinned-fork.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LEAF 4096UL +#define NATIVE 16384UL +#define SIZE (1024UL * 1024) +#define COUNT (SIZE / LEAF) +#define PFNMASK ((1ULL << 55) - 1) +static pid_t child = -1; +static uint64_t parent_pfn[COUNT], child_pfn[COUNT]; +static void die(const char *s) { perror(s); exit(1); } +static void cleanup(void) +{ + if (child > 0) { + kill(child, SIGKILL); + while (waitpid(child, NULL, 0) < 0 && errno == EINTR) {} + } +} +static void timeout(int sig) +{ + (void)sig; + if (child > 0) kill(child, SIGKILL); + _exit(124); +} +static void transfer(int fd, void *buffer, size_t len, int writing) +{ + unsigned char *p = buffer; + while (len) { + ssize_t n = writing ? write(fd, p, len) : read(fd, p, len); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) die(writing ? "write protocol" : "read protocol"); + p += n; len -= n; + } +} +static unsigned char pattern(size_t i) { return (unsigned char)(1 + (i * 17 + i / LEAF) % 251); } +static void check(unsigned char *p, int mode) +{ + for (size_t i = 0; i < SIZE; i++) { + unsigned char expected = pattern(i); + if ((mode == 1 && i % LEAF == 0) || (mode == 2 && i % LEAF == 1)) expected ^= 0x80; + if (p[i] != expected) { + fprintf(stderr, "FAIL bytes mode=%d offset=%zu expected=%u actual=%u\n", mode, i, expected, p[i]); + exit(1); + } + } +} +static void pfns(unsigned char *p, uint64_t *out) +{ + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + if (fd < 0) die("pagemap open"); + for (size_t i = 0; i < COUNT; i++) { + uint64_t e; + if (pread(fd, &e, sizeof(e), (((uintptr_t)p / LEAF) + i) * sizeof(e)) != sizeof(e)) die("pagemap read"); + if (!(e >> 63) || (e & (1ULL << 62)) || !(e & PFNMASK)) { + fprintf(stderr, "FAIL pagemap leaf=%zu entry=%016llx (root PFNs required)\n", i, (unsigned long long)e); exit(1); + } + out[i] = e & PFNMASK; + } + close(fd); +} +static size_t native_count(uint64_t *pf) +{ + size_t n = 0; + for (size_t i = 0; i < COUNT; i++) { + size_t j; + for (j = 0; j < i; j++) if (pf[j] / 4 == pf[i] / 4) break; + if (j == i) n++; + } + return n; +} +int main(int argc, char **argv) +{ + if (argc == 2 && !strcmp(argv[1], "--selftest")) { + for (size_t i = 0; i < COUNT; i++) parent_pfn[i] = 4 + i; + if (native_count(parent_pfn) != COUNT / 4) return 1; + for (size_t i = 0; i < COUNT; i++) parent_pfn[i] = 4 + 4 * i; + if (native_count(parent_pfn) != COUNT) return 1; + puts("PASS native PFN grouping selftest (no pin/fork coverage)"); return 0; + } + int packed = 0, hold = 0; + if (argc < 2 || strcmp(argv[1], "--native-16k") || getauxval(AT_PAGESZ) != LEAF) return 77; + for (int i = 2; i < argc; i++) { + if (!strcmp(argv[i], "--expect-packed")) packed = 1; + else if (!strcmp(argv[i], "--hold")) hold = 1; + else return 77; + } + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGALRM, timeout); signal(SIGPIPE, SIG_IGN); alarm(15); atexit(cleanup); + unsigned char *raw = mmap(NULL, SIZE + NATIVE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) die("mmap"); + unsigned char *p = (void *)(((uintptr_t)raw + NATIVE - 1) & ~(NATIVE - 1)); + for (size_t i = 0; i < SIZE; i++) p[i] = pattern(i); + check(p, 0); pfns(p, parent_pfn); + if (native_count(parent_pfn) * NATIVE != SIZE) die("initial dense native backing"); + struct io_uring_params params = {0}; + int ring = syscall(__NR_io_uring_setup, 2, ¶ms); + if (ring < 0) { perror("UNEXERCISED io_uring_setup"); return 77; } + struct iovec iov = { .iov_base = p, .iov_len = SIZE }; + if (syscall(__NR_io_uring_register, ring, IORING_REGISTER_BUFFERS, &iov, 1)) { + perror("UNEXERCISED register real long-term pins"); close(ring); return 77; + } + /* Long-term pin setup can migrate backing; capture the pinned identity. */ + pfns(p, parent_pfn); check(p, 0); + if (native_count(parent_pfn) * NATIVE != SIZE) die("pinned dense native backing"); + int to_child[2], from_child[2]; + if (pipe(to_child) || pipe(from_child)) die("pipe"); + pid_t parent = getpid(); + child = fork(); + if (child < 0) die("fork"); + if (!child) { + prctl(PR_SET_PDEATHSIG, SIGKILL); + if (getppid() != parent) _exit(1); + alarm(10); close(ring); close(to_child[1]); close(from_child[0]); + pfns(p, child_pfn); check(p, 0); + transfer(from_child[1], child_pfn, sizeof(child_pfn), 1); + char token; transfer(to_child[0], &token, 1, 0); + if (token != 'w') die("child token"); + check(p, 0); + for (size_t i = 0; i < COUNT; i++) p[i * LEAF + 1] ^= 0x80; + check(p, 2); + token = 'c'; transfer(from_child[1], &token, 1, 1); + transfer(to_child[0], &token, 1, 0); + if (token != 'e') die("child exit token"); + check(p, 2); _exit(0); + } + close(to_child[0]); close(from_child[1]); + transfer(from_child[0], child_pfn, sizeof(child_pfn), 0); + for (size_t i = 0; i < COUNT; i++) + for (size_t j = 0; j < COUNT; j++) + if (child_pfn[i] / 4 == parent_pfn[j] / 4) die("child copy overlaps pinned parent backing"); + printf("METADATA real_registered_pin=1 parent_native=%zu child_native=%zu copied_leaves=%lu native_overlap=0\n", native_count(parent_pfn) * NATIVE, native_count(child_pfn) * NATIVE, COUNT); + if (hold) { + printf("HOLD parent=%d child=%d base=%lx bytes=%lu (write c to continue)\n", getpid(), child, (unsigned long)p, SIZE); + char ack; transfer(STDIN_FILENO, &ack, 1, 0); + if (ack != 'c') die("controller hold token"); + } + size_t child_bytes = native_count(child_pfn) * NATIVE; + if (child_bytes < SIZE || child_bytes > COUNT * NATIVE || + (packed && child_bytes > SIZE + 256 * 1024)) die("child density bound"); + for (size_t i = 0; i < COUNT; i++) p[i * LEAF] ^= 0x80; + check(p, 1); + char token = 'w'; transfer(to_child[1], &token, 1, 1); + transfer(from_child[0], &token, 1, 0); + if (token != 'c') die("parent token"); + check(p, 1); + token = 'e'; transfer(to_child[1], &token, 1, 1); + int status; + if (waitpid(child, &status, 0) != child) die("waitpid"); + child = -1; + if (!WIFEXITED(status) || WEXITSTATUS(status)) die("child failed"); + if (syscall(__NR_io_uring_register, ring, IORING_UNREGISTER_BUFFERS, NULL, 0)) die("unregister"); + close(ring); close(to_child[1]); close(from_child[0]); + if (munmap(raw, SIZE + NATIVE)) die("munmap"); + puts("PASS real pin/fork copy and full-byte two-way isolation; density is separately reported"); + return 0; +} From adec17b402197bcf2b56ca10ba0872cb2d128b3f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 19:29:34 -0400 Subject: [PATCH 0795/1579] arm64: reuse cached table fragments after allocation races Concurrent small-table allocators can all miss the fragment cache before allocating a native owner. Recheck the cache after construction and consume its reserved fragment if another allocator supplied it. Destroy the unused fresh owner instead of returning one table and stranding its other slots. Add bounded serial and concurrent backing fixtures with separate logical VmPTE and physical-charge checkpoints. On Neo the same concurrent fixture fell from 5632 to 4192 KiB physical pagetables for approximately 4160 KiB logical tables. All contents, retained MM, reclaim/readback and GPU controls pass. Issued-fragment retention after unmap is a separate unresolved case. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit fd8b40cc03751f3d4128be7d09c855ae1c7e5498) --- arch/arm64/mm/user4k-pgalloc.c | 26 ++- .../testing/selftests/arm64/user4k/.gitignore | 2 + tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../arm64/user4k/README.pagetable-backing | 33 ++++ .../arm64/user4k/pagetable-concurrent.c | 161 ++++++++++++++++++ .../arm64/user4k/pagetable-retention.c | 118 +++++++++++++ 6 files changed, 339 insertions(+), 5 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.pagetable-backing create mode 100644 tools/testing/selftests/arm64/user4k/pagetable-concurrent.c create mode 100644 tools/testing/selftests/arm64/user4k/pagetable-retention.c diff --git a/arch/arm64/mm/user4k-pgalloc.c b/arch/arm64/mm/user4k-pgalloc.c index 0bcaab5c3a8d2..21d27c1656f76 100644 --- a/arch/arm64/mm/user4k-pgalloc.c +++ b/arch/arm64/mm/user4k-pgalloc.c @@ -98,12 +98,30 @@ void *arm64_user_pt_alloc_granule(struct mm_struct *mm, atomic_set(&pt->pt_frag_refcount, 1); table = ptdesc_address(pt); spin_lock(&mm->page_table_lock); - if (nr_fragments > 1 && !mm->context.user4k_pt_frag[level]) { - atomic_set(&pt->pt_frag_refcount, nr_fragments); - mm->context.user4k_pt_frag[level] = table + frag_size; + if (nr_fragments > 1) { + next = mm->context.user4k_pt_frag[level]; + if (next) { + /* Another allocator supplied backing while we constructed + * a table owner. Consume its unused fragment rather than + * stranding the rest of our new native page. + */ + if (virt_to_ptdesc(next)->pt_frag_shift != shift) { + spin_unlock(&mm->page_table_lock); + fragment_owner_free(pt); + return NULL; + } + table = next; + next += frag_size; + mm->context.user4k_pt_frag[level] = + ((unsigned long)next & ~PAGE_MASK) ? next : NULL; + } else { + atomic_set(&pt->pt_frag_refcount, nr_fragments); + mm->context.user4k_pt_frag[level] = table + frag_size; + } } - /* A racing allocation may own the cache; this page then has one user. */ spin_unlock(&mm->page_table_lock); + if (table != ptdesc_address(pt)) + fragment_owner_free(pt); return table; } diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index a5b225dbaf61d..3c4c8c86eefe2 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -7,3 +7,5 @@ same-native first-write clipped-firstwrite pinned-fork +pagetable-retention +pagetable-concurrent diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index a6e4854f454c2..d306deb1f0054 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -16,11 +16,13 @@ TEST_GEN_PROGS_EXTENDED += first-write clipped-firstwrite TEST_FILES += README.first-write TEST_GEN_PROGS_EXTENDED += pinned-fork TEST_FILES += README.pinned-fork +TEST_GEN_PROGS_EXTENDED += pagetable-retention pagetable-concurrent +TEST_FILES += README.pagetable-backing include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread +$(OUTPUT)/pagetable-concurrent $(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/README.pagetable-backing b/tools/testing/selftests/arm64/user4k/README.pagetable-backing new file mode 100644 index 0000000000000..2c6b16b17465a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.pagetable-backing @@ -0,0 +1,33 @@ +Page-table physical backing and fragment retention +================================================== + +These bounded fixtures reserve 2 GiB of virtual address space and write one +4 KiB leaf in each of 1,024 distinct 2 MiB table windows. Native payload +backing is at most 16 MiB, with approximately 4 MiB of logical PTE tables. +The concurrent variant uses 32 threads, 32 synchronized fault rounds and +128 KiB thread stacks. Both verify all payload bytes across six checkpoints: +baseline, full, quarter_survivors, refilled, quarter_survivors_again, unmapped. + +Run through the 4 KiB ABI launcher on a native 16 KiB kernel. Each JSON +checkpoint requires one byte 'c' on stdin before continuing. For contents +only, with no physical charge observation: + + yes c | userpages 4k ./pagetable-retention --native-16k + yes c | userpages 4k ./pagetable-concurrent --native-16k + +For physical accounting, run each fixture alone in a fresh uncapped cgroup. +At each checkpoint verify the fixture is the only member, record memory.stat +pagetables and memory.swap.current, and sample at 0, 1 and 3 seconds before +acknowledging. Record logical VmPTE separately: it is not physical backing. +Use an external service lifetime below the fixture's 60-second alarm and +verify all processes exit. Do not change memory.max/high or force reclaim. + +Compare the same complete fixture across kernels. Concurrent thread stacks +and upper-level tables contribute to the measurements; do not subtract them +as though they were pure payload. Deferred table releases can occur between +samples. Contents PASS does not assert a density or performance result. + +The allocation-race fix reduces initial concurrent physical backing. It does +not reclaim partially occupied table owners or reuse issued fragments after +unmap. Retention/refill checkpoints deliberately preserve that separate +coverage gap rather than declaring it fixed. diff --git a/tools/testing/selftests/arm64/user4k/pagetable-concurrent.c b/tools/testing/selftests/arm64/user4k/pagetable-concurrent.c new file mode 100644 index 0000000000000..2573a8882e165 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/pagetable-concurrent.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LEAF 4096UL +#define WINDOW (2UL * 1024 * 1024) +#define WINDOWS 1024UL +#define LENGTH (WINDOWS * WINDOW) + +static void fail(const char *why) +{ + perror(why); + exit(1); +} + +static unsigned char tag(size_t i) +{ + return (unsigned char)(1 + i % 251); +} + +static void checkpoint(const char *phase) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256], ack; + unsigned long pte = 0; + if (!f) + fail("status"); + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmPTE: %lu kB", &pte) == 1) + break; + fclose(f); + printf("{\"phase\":\"%s\",\"pid\":%d,\"VmPTE_bytes\":%lu}\n", + phase, getpid(), pte * 1024); + if (read(STDIN_FILENO, &ack, 1) != 1 || ack != 'c') + fail("controller ack"); +} + +static void touch_serial(unsigned char *base, int missing_only) +{ + for (size_t i = 0; i < WINDOWS; i++) { + if (missing_only && !(i % 4)) + continue; + if (missing_only) { + void *p = mmap(base + i * WINDOW, WINDOW, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, + -1, 0); + if (p != base + i * WINDOW) + fail("refill mmap"); + } + memset(base + i * WINDOW, tag(i), LEAF); + } +} + +#define WORKERS 32 +static pthread_barrier_t step; +static atomic_int start_workers; +struct worker { unsigned char *base; unsigned id; }; +static void *fault_worker(void *arg) +{ + struct worker *w = arg; + int start; + do { start = atomic_load_explicit(&start_workers, memory_order_acquire); + if (!start) sched_yield(); } while (!start); + if (start != 1) return NULL; + for (unsigned round = 0; round < WINDOWS / WORKERS; round++) { + int ret = pthread_barrier_wait(&step); + if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) abort(); + size_t i = round * WORKERS + w->id; + memset(w->base + i * WINDOW, tag(i), LEAF); + } + return NULL; +} +static void touch(unsigned char *base, int missing_only) +{ + if (missing_only) { touch_serial(base, missing_only); return; } + pthread_t threads[WORKERS]; + struct worker args[WORKERS]; + pthread_attr_t attr; + if (pthread_attr_init(&attr) || pthread_attr_setstacksize(&attr, 128 * 1024) || + pthread_barrier_init(&step, NULL, WORKERS)) fail("thread setup"); + unsigned made = 0; + for (; made < WORKERS; made++) { + args[made] = (struct worker){.base = base, .id = made}; + int err = pthread_create(&threads[made], &attr, fault_worker, &args[made]); + if (err) { errno = err; break; } + } + pthread_attr_destroy(&attr); + atomic_store_explicit(&start_workers, made == WORKERS ? 1 : 2, memory_order_release); + for (unsigned i = 0; i < made; i++) if (pthread_join(threads[i], NULL)) abort(); + pthread_barrier_destroy(&step); + if (made != WORKERS) fail("pthread_create"); +} + +static void verify(unsigned char *base, int sparse) +{ + for (size_t i = 0; i < WINDOWS; i++) { + if (sparse && i % 4) + continue; + for (size_t j = 0; j < LEAF; j++) + if (base[i * WINDOW + j] != tag(i)) { + fprintf(stderr, "FAIL content window=%zu byte=%zu\n", i, j); + exit(1); + } + } +} + +static void thin(unsigned char *base) +{ + for (size_t i = 0; i < WINDOWS; i++) + if (i % 4 && munmap(base + i * WINDOW, WINDOW)) + fail("thin munmap"); + verify(base, 1); +} + +int main(int argc, char **argv) +{ + if (argc != 2 || strcmp(argv[1], "--native-16k") || + getauxval(AT_PAGESZ) != LEAF) { + fputs("Requires established 4K launcher on native16K kernel\n", stderr); + return 77; + } + setvbuf(stdout, NULL, _IONBF, 0); + alarm(60); + checkpoint("baseline"); + unsigned char *raw = mmap(NULL, LENGTH + WINDOW, + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) + fail("reserve mmap"); + unsigned char *base = (void *)(((uintptr_t)raw + WINDOW - 1) & ~(WINDOW - 1)); + size_t head = base - raw, tail = WINDOW - head; + if (head && munmap(raw, head)) + fail("trim head"); + if (tail && munmap(base + LENGTH, tail)) + fail("trim tail"); + touch(base, 0); + verify(base, 0); + checkpoint("full"); + thin(base); + checkpoint("quarter_survivors"); + touch(base, 1); + verify(base, 0); + checkpoint("refilled"); + thin(base); + checkpoint("quarter_survivors_again"); + if (munmap(base, LENGTH)) + fail("final munmap"); + checkpoint("unmapped"); + puts("PASS contents, unmap and refill; concurrent firstfaults, accounting is a separate observation"); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/pagetable-retention.c b/tools/testing/selftests/arm64/user4k/pagetable-retention.c new file mode 100644 index 0000000000000..09f18c5424ef8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/pagetable-retention.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +#define LEAF 4096UL +#define WINDOW (2UL * 1024 * 1024) +#define WINDOWS 1024UL +#define LENGTH (WINDOWS * WINDOW) + +static void fail(const char *why) +{ + perror(why); + exit(1); +} + +static unsigned char tag(size_t i) +{ + return (unsigned char)(1 + i % 251); +} + +static void checkpoint(const char *phase) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256], ack; + unsigned long pte = 0; + if (!f) + fail("status"); + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmPTE: %lu kB", &pte) == 1) + break; + fclose(f); + printf("{\"phase\":\"%s\",\"pid\":%d,\"VmPTE_bytes\":%lu}\n", + phase, getpid(), pte * 1024); + if (read(STDIN_FILENO, &ack, 1) != 1 || ack != 'c') + fail("controller ack"); +} + +static void touch(unsigned char *base, int missing_only) +{ + for (size_t i = 0; i < WINDOWS; i++) { + if (missing_only && !(i % 4)) + continue; + if (missing_only) { + void *p = mmap(base + i * WINDOW, WINDOW, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, + -1, 0); + if (p != base + i * WINDOW) + fail("refill mmap"); + } + memset(base + i * WINDOW, tag(i), LEAF); + } +} + +static void verify(unsigned char *base, int sparse) +{ + for (size_t i = 0; i < WINDOWS; i++) { + if (sparse && i % 4) + continue; + for (size_t j = 0; j < LEAF; j++) + if (base[i * WINDOW + j] != tag(i)) { + fprintf(stderr, "FAIL content window=%zu byte=%zu\n", i, j); + exit(1); + } + } +} + +static void thin(unsigned char *base) +{ + for (size_t i = 0; i < WINDOWS; i++) + if (i % 4 && munmap(base + i * WINDOW, WINDOW)) + fail("thin munmap"); + verify(base, 1); +} + +int main(int argc, char **argv) +{ + if (argc != 2 || strcmp(argv[1], "--native-16k") || + getauxval(AT_PAGESZ) != LEAF) { + fputs("Requires established 4K launcher on native16K kernel\n", stderr); + return 77; + } + setvbuf(stdout, NULL, _IONBF, 0); + alarm(60); + checkpoint("baseline"); + unsigned char *raw = mmap(NULL, LENGTH + WINDOW, + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) + fail("reserve mmap"); + unsigned char *base = (void *)(((uintptr_t)raw + WINDOW - 1) & ~(WINDOW - 1)); + size_t head = base - raw, tail = WINDOW - head; + if (head && munmap(raw, head)) + fail("trim head"); + if (tail && munmap(base + LENGTH, tail)) + fail("trim tail"); + touch(base, 0); + verify(base, 0); + checkpoint("full"); + thin(base); + checkpoint("quarter_survivors"); + touch(base, 1); + verify(base, 0); + checkpoint("refilled"); + thin(base); + checkpoint("quarter_survivors_again"); + if (munmap(base, LENGTH)) + fail("final munmap"); + checkpoint("unmapped"); + puts("PASS contents, unmap and refill; accounting is a separate observation"); + return 0; +} From 5ba433b1b4cb1998f78019fd1ec4084d990e8ecf Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 20:29:26 -0400 Subject: [PATCH 0796/1579] arm64: reuse retired small page-table fragments Keep an independent per-mm, per-level pool of native table owners so holes left by unmap can satisfy subsequent small-table allocations. Retain issued references until RCU retirement, clear dirty slots only when reused, and release empty owners. Pool references outlive mm teardown for callbacks. On Neo, serial refill physical tables fall from 7,421,952 to 4,276,224 bytes; concurrent refill falls from 6,275,072 to 4,276,224. Contents, teardown, pinned-copy charges, repeated MM, actual swap readback and GPU controls pass. Host allocator checks cover native 4K/16K/64K primitives; maintained KUnit code compiles but was not enabled in the hardware kernel. Seven-round operation timing has overlapping ranges and about 1% higher medians; no zero-overhead claim. Live-table compaction remains separate. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e08fea2abe7ea51544718e47d1f14706ce4c630e) --- arch/arm64/include/asm/mmu.h | 2 +- arch/arm64/mm/user4k-pgalloc.c | 282 +++++++++++++----- arch/arm64/mm/user4k-test.c | 160 +++------- .../arm64/user4k/README.pagetable-backing | 20 +- 4 files changed, 255 insertions(+), 209 deletions(-) diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 15943bec53392..da82ed5f0c41f 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -39,7 +39,7 @@ typedef struct { unsigned long flags; u8 pkey_allocation_map; #ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL - /* Unissued PTE/PMD/PUD fragments; protected by page_table_lock. */ + /* PTE/PMD/PUD fragment pools; initialized before atomic publication. */ void *user4k_pt_frag[3]; #endif } mm_context_t; diff --git a/arch/arm64/mm/user4k-pgalloc.c b/arch/arm64/mm/user4k-pgalloc.c index 21d27c1656f76..7a3e34205760c 100644 --- a/arch/arm64/mm/user4k-pgalloc.c +++ b/arch/arm64/mm/user4k-pgalloc.c @@ -1,20 +1,38 @@ // SPDX-License-Identifier: GPL-2.0-only -/* - * Hardware tables have immutable granule/owner metadata. Smaller tables share - * a native page; larger tables own a compound allocation. Only unissued slots - * in a shared native page are cached by the mm. - */ +/* Smaller hardware tables share native owners, including safely retired slots. */ #include #include #include #include +#include #include +struct fragment_pool { + spinlock_t lock; + refcount_t refs; + struct list_head available; + unsigned int shift; + bool closed; +}; + +struct fragment_slot { + struct rcu_head rcu; + void *table; +}; + +struct fragment_owner { + struct fragment_pool *pool; + struct ptdesc *pt; + struct list_head available; + unsigned long free_mask; + unsigned long dirty_mask; + struct fragment_slot slots[]; +}; + unsigned int arm64_ptep_page_shift(pte_t *ptep) { struct ptdesc *pt = virt_to_ptdesc(ptep); - /* Table owners retain geometry through their last RCU reader. */ return test_bit(PT_fragmented, &pt->pt_flags.f) ? pt->pt_frag_shift : PAGE_SHIFT; } EXPORT_SYMBOL_GPL(arm64_ptep_page_shift); @@ -30,49 +48,47 @@ static void fragment_owner_free_rcu(struct rcu_head *rcu) fragment_owner_free(container_of(rcu, struct ptdesc, pt_rcu_head)); } -static void fragment_put(struct ptdesc *pt, unsigned int nr) +static void fragment_pool_put(struct fragment_pool *pool) { - if (!atomic_sub_and_test(nr, &pt->pt_frag_refcount)) - return; - if (test_and_clear_bit(PT_frag_defer, &pt->pt_flags.f)) - call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); - else - fragment_owner_free(pt); + if (refcount_dec_and_test(&pool->refs)) { + WARN_ON_ONCE(!list_empty(&pool->available)); + kfree(pool); + } } -void *arm64_user_pt_alloc_granule(struct mm_struct *mm, - enum arm64_user4k_pt_level level, unsigned int shift) +/* Active table allocation already holds the mm alive. Its cache reference + * remains until destroy_context, after all allocators have stopped. Owners + * independently retain the pool for callbacks after that point. + */ +static struct fragment_pool *fragment_pool_get(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift) { - struct ptdesc *pt; - void *table, *next; - bool constructed; - unsigned int frag_size, nr_fragments, order; + struct fragment_pool *pool, *new; - if (shift != 12 && shift != 14 && shift != 16) - return NULL; - frag_size = 1U << shift; - order = shift > PAGE_SHIFT ? shift - PAGE_SHIFT : 0; - nr_fragments = (PAGE_SIZE << order) >> shift; - if (WARN_ON_ONCE((unsigned int)level >= ARRAY_SIZE(mm->context.user4k_pt_frag))) + pool = smp_load_acquire(&mm->context.user4k_pt_frag[level]); + if (pool) + return pool->shift == shift ? pool : NULL; + new = kzalloc(sizeof(*new), GFP_KERNEL | __GFP_ACCOUNT); + if (!new) return NULL; + spin_lock_init(&new->lock); + refcount_set(&new->refs, 1); + INIT_LIST_HEAD(&new->available); + new->shift = shift; + pool = cmpxchg(&mm->context.user4k_pt_frag[level], NULL, new); + if (!pool) + return new; + fragment_pool_put(new); + return pool->shift == shift ? pool : NULL; +} - spin_lock(&mm->page_table_lock); - table = mm->context.user4k_pt_frag[level]; - if (table) { - /* An mm's geometry is immutable; never consume another granule. */ - if (virt_to_ptdesc(table)->pt_frag_shift != shift) { - spin_unlock(&mm->page_table_lock); - return NULL; - } - next = table + frag_size; - mm->context.user4k_pt_frag[level] = - ((unsigned long)next & ~PAGE_MASK) ? next : NULL; - } - spin_unlock(&mm->page_table_lock); - if (table) - return table; +static struct ptdesc *fragment_pt_alloc(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift) +{ + struct ptdesc *pt; + bool constructed; - pt = pagetable_alloc(GFP_PGTABLE_USER, order); + pt = pagetable_alloc(GFP_PGTABLE_USER, shift > PAGE_SHIFT ? shift - PAGE_SHIFT : 0); if (!pt) return NULL; switch (level) { @@ -96,32 +112,95 @@ void *arm64_user_pt_alloc_granule(struct mm_struct *mm, pt->pt_frag_shift = shift; set_bit(PT_fragmented, &pt->pt_flags.f); atomic_set(&pt->pt_frag_refcount, 1); - table = ptdesc_address(pt); - spin_lock(&mm->page_table_lock); - if (nr_fragments > 1) { - next = mm->context.user4k_pt_frag[level]; - if (next) { - /* Another allocator supplied backing while we constructed - * a table owner. Consume its unused fragment rather than - * stranding the rest of our new native page. - */ - if (virt_to_ptdesc(next)->pt_frag_shift != shift) { - spin_unlock(&mm->page_table_lock); - fragment_owner_free(pt); - return NULL; - } - table = next; - next += frag_size; - mm->context.user4k_pt_frag[level] = - ((unsigned long)next & ~PAGE_MASK) ? next : NULL; - } else { - atomic_set(&pt->pt_frag_refcount, nr_fragments); - mm->context.user4k_pt_frag[level] = table + frag_size; - } + return pt; +} + +/* The caller holds pool->lock; its active mm retains the pool. */ +static void *fragment_take(struct fragment_pool *pool, bool *zero) +{ + struct fragment_owner *owner; + unsigned int slot; + + if (pool->closed || list_empty(&pool->available)) + return NULL; + owner = list_first_entry(&pool->available, struct fragment_owner, available); + slot = __ffs(owner->free_mask); + owner->free_mask &= ~BIT(slot); + *zero = owner->dirty_mask & BIT(slot); + owner->dirty_mask &= ~BIT(slot); + atomic_inc(&owner->pt->pt_frag_refcount); + if (!owner->free_mask) + list_del_init(&owner->available); + return owner->slots[slot].table; +} + +void *arm64_user_pt_alloc_granule(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift) +{ + struct fragment_pool *pool; + struct fragment_owner *owner; + struct ptdesc *pt; + void *table; + unsigned int nr, i; + unsigned long flags; + bool zero = false; + + if (shift != 12 && shift != 14 && shift != 16) + return NULL; + if (WARN_ON_ONCE((unsigned int)level >= ARRAY_SIZE(mm->context.user4k_pt_frag))) + return NULL; + if (shift >= PAGE_SHIFT) { + pt = fragment_pt_alloc(mm, level, shift); + return pt ? ptdesc_address(pt) : NULL; + } + pool = fragment_pool_get(mm, level, shift); + if (!pool) + return NULL; + spin_lock_irqsave(&pool->lock, flags); + table = fragment_take(pool, &zero); + spin_unlock_irqrestore(&pool->lock, flags); + if (table) + goto out; + + nr = PAGE_SIZE >> shift; + owner = kzalloc(struct_size(owner, slots, nr), GFP_KERNEL | __GFP_ACCOUNT); + if (!owner) + goto out; + pt = fragment_pt_alloc(mm, level, shift); + if (!pt) { + kfree(owner); + goto out; + } + owner->pool = pool; + owner->pt = pt; + INIT_LIST_HEAD(&owner->available); + owner->free_mask = GENMASK(nr - 1, 1); + for (i = 0; i < nr; i++) + owner->slots[i].table = ptdesc_address(pt) + (i << shift); + /* This word is unused by arm64 page-table constructors and walkers. */ + pt->__page_mapping = (unsigned long)owner; + + spin_lock_irqsave(&pool->lock, flags); + /* Reuse supply published while allocation slept, including retired slots. */ + table = fragment_take(pool, &zero); + if (!table && !pool->closed) { + refcount_inc(&pool->refs); /* Native owner's lifetime reference. */ + list_add_tail(&owner->available, &pool->available); + table = owner->slots[0].table; + owner = NULL; } - spin_unlock(&mm->page_table_lock); - if (table != ptdesc_address(pt)) + spin_unlock_irqrestore(&pool->lock, flags); + if (owner) { + pt->__page_mapping = 0; fragment_owner_free(pt); + kfree(owner); + } +out: + /* Clear only recycled slots that are actually allocated again. The issued + * reference excludes other allocators and keeps this owner alive here. + */ + if (table && zero) + memset(table, 0, 1UL << shift); return table; } @@ -130,14 +209,45 @@ void *arm64_user4k_pt_alloc(struct mm_struct *mm, enum arm64_user4k_pt_level lev return arm64_user_pt_alloc_granule(mm, level, 12); } +/* Only unpublished tables or tables past the required walker grace arrive here. */ void arm64_user4k_pt_free(void *table) { struct ptdesc *pt = virt_to_ptdesc(table); + struct fragment_owner *owner; + struct fragment_pool *pool; + unsigned int slot; + unsigned long flags; + bool last; VM_BUG_ON(!test_bit(PT_fragmented, &pt->pt_flags.f)); VM_BUG_ON(!IS_ALIGNED((unsigned long)table, 1UL << pt->pt_frag_shift)); VM_BUG_ON(atomic_read(&pt->pt_frag_refcount) <= 0); - fragment_put(pt, 1); + if (pt->pt_frag_shift >= PAGE_SHIFT) { + if (test_and_clear_bit(PT_frag_defer, &pt->pt_flags.f)) + call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); + else + fragment_owner_free(pt); + return; + } + owner = (void *)pt->__page_mapping; + pool = owner->pool; + slot = ((unsigned long)table & ~PAGE_MASK) >> pt->pt_frag_shift; + spin_lock_irqsave(&pool->lock, flags); + VM_BUG_ON(owner->free_mask & BIT(slot)); + owner->free_mask |= BIT(slot); + owner->dirty_mask |= BIT(slot); + last = atomic_dec_and_test(&pt->pt_frag_refcount); + if (last) + list_del_init(&owner->available); + else if (!pool->closed && list_empty(&owner->available)) + list_add_tail(&owner->available, &pool->available); + spin_unlock_irqrestore(&pool->lock, flags); + if (last) { + pt->__page_mapping = 0; + fragment_owner_free(pt); + kfree(owner); + fragment_pool_put(pool); + } } void arm64_pgtable_free(void *table) @@ -154,32 +264,46 @@ void arm64_pgtable_free(void *table) void arm64_user4k_pt_cache_destroy(struct mm_struct *mm) { + struct fragment_pool *pool; unsigned int level; + unsigned long flags; - /* The mm is private or dying; no allocator may still access its cache. */ + /* No allocator may still access this mm; deferred owners are independent. */ for (level = 0; level < ARRAY_SIZE(mm->context.user4k_pt_frag); level++) { - void *next = mm->context.user4k_pt_frag[level]; - struct ptdesc *pt; - unsigned int issued, shift; - - if (!next) + pool = mm->context.user4k_pt_frag[level]; + if (!pool) continue; mm->context.user4k_pt_frag[level] = NULL; - pt = virt_to_ptdesc(next); - shift = pt->pt_frag_shift; - issued = ((unsigned long)next & ~PAGE_MASK) >> shift; - fragment_put(pt, (PAGE_SIZE >> shift) - issued); + spin_lock_irqsave(&pool->lock, flags); + pool->closed = true; + spin_unlock_irqrestore(&pool->lock, flags); + fragment_pool_put(pool); } } +static void fragment_slot_free_rcu(struct rcu_head *rcu) +{ + struct fragment_slot *slot = container_of(rcu, struct fragment_slot, rcu); + + arm64_user4k_pt_free(slot->table); +} + void arm64_pte_free_defer(struct mm_struct *mm, pgtable_t table) { struct ptdesc *pt = virt_to_ptdesc(table); if (test_bit(PT_fragmented, &pt->pt_flags.f)) { - /* No freed slot is reused. Delay the final owner release for RCU. */ - set_bit(PT_frag_defer, &pt->pt_flags.f); - arm64_user4k_pt_free(table); + if (pt->pt_frag_shift < PAGE_SHIFT) { + struct fragment_owner *owner = (void *)pt->__page_mapping; + unsigned int slot = ((unsigned long)table & ~PAGE_MASK) >> + pt->pt_frag_shift; + + /* Pending callbacks retain their issued references and contents. */ + call_rcu(&owner->slots[slot].rcu, fragment_slot_free_rcu); + } else { + set_bit(PT_frag_defer, &pt->pt_flags.f); + arm64_user4k_pt_free(table); + } } else { call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); } diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index e406b97ea6235..4e5b2befe18f5 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include @@ -858,6 +857,7 @@ static void test_fragment_lifetime(struct kunit *test, unsigned int shift) unsigned int i, slots = PAGE_SIZE >> shift; unsigned long size = 1UL << shift; unsigned int read_shift = 0; + int pending_refs; bool live; KUNIT_ASSERT_NOT_NULL(test, f); @@ -925,20 +925,33 @@ static void test_fragment_lifetime(struct kunit *test, unsigned int shift) KUNIT_EXPECT_PTR_NE(test, f->pinned[1], f->pinned[2]); pte_owner = page_ptdesc(f->pinned[0]); KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots); + /* Refill holes in the same owner, without retaining old table contents. */ + for (i = 1; i < slots; i++) { + void *old = f->table[i]; + + memset(old, 0x6a, size); + arm64_pgtable_free(old); + f->table[i] = arm64_user_pt_alloc_granule(f->mm, USER4K_PT_PTE, shift); + KUNIT_ASSERT_NOT_NULL(test, f->table[i]); + KUNIT_EXPECT_PTR_EQ(test, f->table[i], old); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0, size), NULL); + } + /* Begin the reader before queuing a callback, so it cannot finish early. */ + rcu_read_lock(); arm64_pte_free_defer(f->mm, f->table[0]); f->table[0] = NULL; - KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots - 1); + pending_refs = atomic_read(&pte_owner->pt_frag_refcount); for (i = 1; i < slots - 1; i++) { arm64_pgtable_free(f->table[i]); f->table[i] = NULL; } - /* A pre-existing software walker must survive the last fragment release. */ - rcu_read_lock(); + /* Pending slot zero keeps the owner alive through this reader. */ arm64_pgtable_free(f->table[slots - 1]); f->table[slots - 1] = NULL; live = folio_test_pgtable(page_folio(f->pinned[0])); read_shift = arm64_ptep_page_shift(page_address(f->pinned[0])); rcu_read_unlock(); + KUNIT_EXPECT_EQ(test, pending_refs, (int)slots); KUNIT_EXPECT_TRUE(test, live); KUNIT_EXPECT_EQ(test, read_shift, shift); rcu_barrier(); @@ -949,8 +962,9 @@ static void test_fragment_lifetime(struct kunit *test, unsigned int shift) arm64_pgtable_free(f->table[i]); f->table[i] = NULL; - KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), (int)slots - 1); - KUNIT_EXPECT_TRUE(test, folio_test_pgtable(ptdesc_folio(pt))); + /* A wholly empty owner is released without waiting for mm teardown. */ + KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), 0); + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(ptdesc_folio(pt))); } mmput(f->mm); f->mm = NULL; @@ -5213,14 +5227,13 @@ static void user4k_exec_arguments_test(struct kunit *test) } for (native = 0; native < 2; native++) { struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); - struct mm_struct *mm; + struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); unsigned long size, mask, initial, copied, new_top; unsigned long a_len = 2 * PAGE_SIZE + SZ_4K + 31, b_len = SZ_4K + 43; char *a, *b, *buf, *too_long; int ret; KUNIT_ASSERT_NOT_NULL(test, bprm); - mm = native ? mm_alloc() : test_vma_mm(test); KUNIT_ASSERT_NOT_NULL(test, mm); if (native) KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); @@ -5293,88 +5306,6 @@ static void user4k_exec_arguments_test(struct kunit *test) } } -/* The transfer can replace bprm->mm, so release its final owner. */ -static void test_exec_mm_free(void *data) -{ - struct linux_binprm *bprm = data; - - mmput(bprm->mm); -} - -static void user64k_exec_arguments_test(struct kunit *test) -{ - unsigned int shifts[] = { 16, 12, PAGE_SHIFT }; - unsigned int mode; - - if (PAGE_SHIFT >= 16 || - (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { - kunit_skip(test, "requires internal larger user granule and MM opt-in"); - return; - } - for (mode = 0; mode < ARRAY_SIZE(shifts); mode++) { - struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); - unsigned long a_len = mode ? SZ_64K + SZ_16K + 31 : 32 * PAGE_SIZE + 31; - unsigned long b_len = PAGE_SIZE + 17, initial, copied; - char *a, *b, *buf; - unsigned long i; - - KUNIT_ASSERT_NOT_NULL(test, bprm); - bprm->mm = mm_alloc(); - KUNIT_ASSERT_NOT_NULL(test, bprm->mm); - KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_exec_mm_free, bprm), 0); - KUNIT_ASSERT_EQ(test, test_mm_select_granule(bprm->mm, 16), 0); - KUNIT_ASSERT_EQ(test, create_init_stack_vma(bprm->mm, &bprm->vma, &bprm->p), 0); - initial = bprm->p; - bprm->argmin = initial - SZ_2M; - bprm->rlim_stack.rlim_cur = SZ_8M; - bprm->rlim_stack.rlim_max = RLIM_INFINITY; - a = kunit_kmalloc(test, a_len + 1, GFP_KERNEL); - b = kunit_kmalloc(test, b_len + 1, GFP_KERNEL); - buf = kunit_kmalloc(test, a_len + b_len + 7, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, a); - KUNIT_ASSERT_NOT_NULL(test, b); - KUNIT_ASSERT_NOT_NULL(test, buf); - for (i = 0; i < a_len; i++) - a[i] = 'A' + i % 23; - a[a_len] = 0; - memset(b, 'b', b_len); - b[b_len] = 0; - KUNIT_ASSERT_EQ(test, copy_string_kernel("tail", bprm), 0); - KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, 5, 0), 5); - KUNIT_ASSERT_STREQ(test, buf, "tail"); - KUNIT_ASSERT_EQ(test, copy_string_kernel(a, bprm), 0); - KUNIT_ASSERT_EQ(test, copy_string_kernel(b, bprm), 0); - bprm->argc = 3; - copied = initial - bprm->p; - KUNIT_ASSERT_EQ(test, copied, a_len + b_len + 7); - KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), - (int)copied); - KUNIT_ASSERT_EQ(test, memcmp(buf, b, b_len + 1), 0); - KUNIT_ASSERT_EQ(test, memcmp(buf + b_len + 1, a, a_len + 1), 0); - KUNIT_ASSERT_STREQ(test, buf + b_len + a_len + 2, "tail"); - /* Exercise copying out of every native page within a coarse leaf. */ - if (shifts[mode] != 16) { - KUNIT_ASSERT_EQ(test, bprm_set_page_shift(bprm, shifts[mode]), 0); - KUNIT_ASSERT_EQ(test, mm_page_shift(bprm->mm), shifts[mode]); - memset(buf, 0, copied); - KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), - (int)copied); - KUNIT_ASSERT_EQ(test, memcmp(buf, b, b_len + 1), 0); - KUNIT_ASSERT_EQ(test, memcmp(buf + b_len + 1, a, a_len + 1), 0); - KUNIT_ASSERT_STREQ(test, buf + b_len + a_len + 2, "tail"); - } - initial = bprm->p; - KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); - KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + 1); - KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); - KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + a_len + 2); - KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); - KUNIT_EXPECT_EQ(test, bprm->p, initial + copied); - KUNIT_EXPECT_EQ(test, bprm->argc, 0); - KUNIT_EXPECT_EQ(test, bprm->vma_pages, 0UL); - } -} - static struct vm_area_struct *test_special_vma(struct mm_struct *mm, const struct vm_special_mapping *sm, unsigned long start, unsigned long len, vm_flags_t flags) @@ -6286,10 +6217,10 @@ static void user4k_file_mapcounts_test(struct kunit *test) folio = filemap_get_folio(file->f_mapping, 0); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 2 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 2, 2, 2 }, 1); KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA + 2 * SZ_4K, true) & VM_FAULT_ERROR, 0U); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 2, 2, 1 }, 1); child = test_vma_mm(test); KUNIT_ASSERT_NOT_NULL(test, child); uprobe_start_dup_mmap(); @@ -6298,17 +6229,17 @@ static void user4k_file_mapcounts_test(struct kunit *test) KUNIT_ASSERT_EQ(test, ret, 0); cvma = test_vma_lookup(child, TEST_VA); KUNIT_ASSERT_NOT_NULL(test, cvma); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 3, 3, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 3, 3, 1 }, 1); KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 3, 2, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 3, 2, 1 }, 1); kunit_release_action(test, user4k_vma_mm_free, child); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 2, 2, 1 }, 1); kunit_release_action(test, user4k_vma_mm_free, private->vm_mm); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 1, 1, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 1, 1, 1 }, 1); kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 1, 1, 1 }, 0); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 1, 1, 1 }, 0); kunit_release_action(test, user4k_vma_mm_free, shared->vm_mm); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 0, 0, 0 }, 0); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 0, 0, 0 }, 0); } /* A unique RAM-backed test inode: exercise generic file faults without @@ -8890,10 +8821,6 @@ static void user_subpage_numa_cow_batch_test(struct kunit *test) { unsigned int shift; - if (!(READ_ONCE(sysctl_numa_balancing_mode) & NUMA_BALANCING_NORMAL)) { - kunit_skip(test, "requires normal NUMA balancing; use numa_balancing=enable"); - return; - } for (shift = 12; shift < PAGE_SHIFT; shift += 2) { unsigned long size = 1UL << shift, start = TEST_VA; struct vm_area_struct *vma; @@ -10092,7 +10019,7 @@ static void user4k_file_migration_test(struct kunit *test) lru_add_drain_all(); folio = filemap_get_folio(file->f_mapping, 0); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { [0 ... TEST_SLOTS - 1] = 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ [0 ... TEST_SLOTS - 1] = 1 }, 1); old_phys = PFN_PHYS(folio_pfn(folio)); isolated = isolate_folio_to_list(folio, &folios); if (!fail || !isolated) @@ -10107,7 +10034,7 @@ static void user4k_file_migration_test(struct kunit *test) KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : 1U); folio = filemap_get_folio(file->f_mapping, 0); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { [0 ... TEST_SLOTS - 1] = 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ [0 ... TEST_SLOTS - 1] = 1 }, 1); for (i = 0; i < 4; i++) { pte_t entry = test_fault_entry(test, small->vm_mm, small->vm_start + i * SZ_4K); @@ -10125,7 +10052,7 @@ static void user4k_file_migration_test(struct kunit *test) folio_lock(folio); try_to_unmap(folio, TTU_SYNC); KUNIT_EXPECT_FALSE(test, folio_mapped(folio)); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 0, 0, 0 }, 0); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 0, 0, 0 }, 0); folio_unlock(folio); folio_put(folio); for (i = 0; i < 4; i++) { @@ -14103,7 +14030,6 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_mremap_resize_test), KUNIT_CASE(user4k_stack_relocate_test), KUNIT_CASE(user4k_exec_arguments_test), - KUNIT_CASE(user64k_exec_arguments_test), KUNIT_CASE(user4k_special_mapping_test), KUNIT_CASE(user4k_insert_fragment_test), KUNIT_CASE(user16k_insert_shared_test), @@ -14142,6 +14068,9 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_vm_mmap_test), KUNIT_CASE(user4k_mprotect_test), KUNIT_CASE(user4k_mprotect_cow_batch_test), +#ifdef CONFIG_NUMA_BALANCING + KUNIT_CASE(user_subpage_numa_cow_batch_test), +#endif KUNIT_CASE(user4k_mprotect_accounting_test), KUNIT_CASE(user4k_file_cow_test), KUNIT_CASE(user4k_file_eof_test), @@ -14202,25 +14131,6 @@ static struct kunit_case user4k_test_cases[] = { {} }; -#ifdef CONFIG_NUMA_BALANCING -/* - * The COW hinting expectations require a stable normal-balancing policy. - * Runtime sysctl writers do not take the fixture's mm or PTE locks, so run - * this case at boot and do not expose it through the debugfs rerun control. - */ -static struct kunit_case user4k_numa_init_cases[] = { - KUNIT_CASE(user_subpage_numa_cow_batch_test), - {} -}; - -static struct kunit_suite user4k_numa_init_suite = { - .name = "arm64-user4k-numa-init", - .test_cases = user4k_numa_init_cases, -}; - -kunit_test_init_section_suite(user4k_numa_init_suite); -#endif - static struct kunit_suite user4k_test_suite = { .name = "arm64-user4k", .test_cases = user4k_test_cases, diff --git a/tools/testing/selftests/arm64/user4k/README.pagetable-backing b/tools/testing/selftests/arm64/user4k/README.pagetable-backing index 2c6b16b17465a..cf87f2a64f5ae 100644 --- a/tools/testing/selftests/arm64/user4k/README.pagetable-backing +++ b/tools/testing/selftests/arm64/user4k/README.pagetable-backing @@ -27,7 +27,19 @@ and upper-level tables contribute to the measurements; do not subtract them as though they were pure payload. Deferred table releases can occur between samples. Contents PASS does not assert a density or performance result. -The allocation-race fix reduces initial concurrent physical backing. It does -not reclaim partially occupied table owners or reuse issued fragments after -unmap. Retention/refill checkpoints deliberately preserve that separate -coverage gap rather than declaring it fixed. +The allocator reuses safely retired fragments, including slots from owners +that still contain live tables. RCU-pending slots remain unavailable until +their callback runs; an empty owner is freed. A reused dirty slot is cleared +before it is returned. This avoids allocating new owners while old ones have +reusable holes. It does not compact surviving live tables. + +On Neo, the serial refill checkpoint fell from 7,421,952 to 4,276,224 bytes of +physical page tables; the concurrent refill fell from 6,275,072 to 4,276,224. +The same six-phase contents and post-unmap cleanup passed. Whole-cgroup slab +usage increased and includes allocator metadata plus other kernel objects. +Do not report page-table bytes as the complete memory cost. + +Seven operation-timed rounds after one warmup had serial/concurrent total +medians of 15.625/16.865 ms, versus 15.434/16.679 ms before reuse. The ranges +substantially overlap; a small cost remains possible. These measurements do +not establish a speedup or zero overhead. They exclude checkpoint I/O. From 619af6d48cfdcbce9ccbb1619b5c5f95777f9ff2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 20:29:40 -0400 Subject: [PATCH 0797/1579] mm: retain compatible subpage pools across allocation-domain changes Index per-mm subpage pools by charge domain, node and VA shard instead of keeping only the first domain. Retain discoverable pools while their native owners survive, so A-to-B-to-A moves can reuse each domain's free quarters. Pin charge identities and registry lifetime, retire empty pools safely, and retry lookup when allocation races with retirement. Preserve private fallback and existing allocation policy restrictions. Add a bounded fixture for bulk moves, alternating single-slot tails and concurrent moves followed by steady allocation. On Neo, the previously 4 MiB backing for a 1 MiB steady allocation becomes approximately 1 MiB; alternating eight 4K leaves per domain fit two 16K owners. Actual kpagecgroup charges, old/new PFN stability, disjoint quarters and complete bytes pass. The corrected race oracle measures added owners rather than counting reused live backing again. Retained MM, pin, real swap readback and GPU controls also pass. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 453b475f491683d207313c66736ca4eef4028c16) --- mm/memory.c | 12 + mm/user-subpage.c | 225 ++++++++++++++---- .../testing/selftests/arm64/user4k/.gitignore | 2 + tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../arm64/user4k/README.domain-cycle | 78 ++++++ .../selftests/arm64/user4k/domain-cycle.c | 155 ++++++++++++ 6 files changed, 432 insertions(+), 44 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.domain-cycle create mode 100644 tools/testing/selftests/arm64/user4k/domain-cycle.c diff --git a/mm/memory.c b/mm/memory.c index ea603a72be93d..22e3ea5356680 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1359,6 +1359,7 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, struct mm_subpage *slot; struct folio *folio; unsigned int offset = vma_page_offset_at(vma, addr).offset; + unsigned int retired_retries = 0; int err; retry_private: @@ -1426,6 +1427,11 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, if (!*pooled) mm_subpage_pool_close(pool); mm_subpage_pool_put(pool); + if (*pooled && IS_ERR(slot) && PTR_ERR(slot) == -ESHUTDOWN) { + if (++retired_retries > 1) + force_private = true; + goto retry_private; + } return slot; } @@ -6434,6 +6440,7 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm, struct mm_subpage_pool *pool; struct folio *folio; unsigned int count = hweight_long(mask); + unsigned int retired_retries = 0; bool arbitrary; int err; @@ -6498,6 +6505,11 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm, if (!*pooled) mm_subpage_pool_close(pool); mm_subpage_pool_put(pool); + if (*pooled && err == -ESHUTDOWN) { + if (++retired_retries > 1) + use_pool = false; + goto retry_private; + } return err; } diff --git a/mm/user-subpage.c b/mm/user-subpage.c index acfa80848e6e3..cff26e5b5c803 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -213,6 +214,12 @@ struct mm_subpage *mm_subpage_get_from_phys(phys_addr_t phys) return slot; } +struct cow_pool_key { + struct mem_cgroup *memcg; + int nid; + u32 bucket; +}; + struct mm_subpage_pool { struct rcu_head rcu; spinlock_t lock; @@ -230,13 +237,101 @@ struct mm_subpage_pool { bool cow_shared; /* Allocated only for shared COW pools; protected by pool->lock. */ unsigned long *cow_free_slots; + struct cow_pool_key cow_key; + struct rhash_head cow_hash; + struct list_head cow_domain; + /* Protected by this pool's lock, never used after registry detachment. */ + struct mm_subpage_cow_pools *cow_registry; }; #define COW_POOL_BITS 4 struct mm_subpage_cow_pools { - struct mm_subpage_pool *pool[1U << COW_POOL_BITS]; + struct rhashtable domains; + spinlock_t lock; + struct list_head pools; + refcount_t refs; + struct rcu_head rcu; + struct work_struct free_work; }; +static const struct rhashtable_params cow_domain_params = { + .head_offset = offsetof(struct mm_subpage_pool, cow_hash), + .key_offset = offsetof(struct mm_subpage_pool, cow_key), + .key_len = sizeof(struct cow_pool_key), + .nelem_hint = 16, + .automatic_shrinking = true, +}; + +static void cow_registry_free_work(struct work_struct *work) +{ + struct mm_subpage_cow_pools *pools = + container_of(work, struct mm_subpage_cow_pools, free_work); + + WARN_ON_ONCE(!list_empty(&pools->pools)); + rhashtable_destroy(&pools->domains); + kfree(pools); +} + +static void cow_registry_free_rcu(struct rcu_head *rcu) +{ + struct mm_subpage_cow_pools *pools = + container_of(rcu, struct mm_subpage_cow_pools, rcu); + + queue_work(system_unbound_wq, &pools->free_work); +} + +static void cow_registry_put(struct mm_subpage_cow_pools *pools) +{ + if (refcount_dec_and_test(&pools->refs)) + call_rcu(&pools->rcu, cow_registry_free_rcu); +} + +static void pool_put_raw(struct mm_subpage_pool *pool); +static void owner_remove(struct mm_subpage_owner *owner); +static void owner_free(struct mm_subpage_owner *owner, bool put_folio); + +/* The caller retains a pool reference independently of registry membership. */ +static void cow_pool_detach_locked(struct mm_subpage_pool *pool) +{ + struct mm_subpage_cow_pools *pools = pool->cow_registry; + + lockdep_assert_held(&pool->lock); + if (!pools) + return; + pool->cow_registry = NULL; + /* A failed prospective insertion may not have entered the hash/list. */ + rhashtable_remove_fast(&pools->domains, &pool->cow_hash, cow_domain_params); + spin_lock(&pools->lock); + list_del_init(&pool->cow_domain); + spin_unlock(&pools->lock); + pool_put_raw(pool); /* Removed (or unpublished prospective) membership. */ + cow_registry_put(pools); +} + +static void cow_pool_retire_empty(struct mm_subpage_pool *pool) +{ + struct mm_subpage_owner *owner = NULL; + unsigned long flags; + + if (!pool->cow_shared) + return; + spin_lock_irqsave(&pool->lock, flags); + if (!pool->backing_pages || + (pool->backing_pages == 1 && pool->unissued)) { + pool->closed = true; + cow_pool_detach_locked(pool); + /* A failed allocation may leave wholly unissued supply, not a + * live domain. Retire it under the same reservation lock. + */ + owner = pool->unissued; + if (owner) + owner_remove(owner); + } + spin_unlock_irqrestore(&pool->lock, flags); + if (owner) + owner_free(owner, true); +} + static void owner_init(struct mm_subpage_owner *owner, struct mm_subpage_pool *pool, struct folio *folio) { @@ -285,7 +380,7 @@ void mm_subpage_pool_get(struct mm_subpage_pool *pool) refcount_inc(&pool->refs); } -void mm_subpage_pool_put(struct mm_subpage_pool *pool) +static void pool_put_raw(struct mm_subpage_pool *pool) { if (!refcount_dec_and_test(&pool->refs)) return; @@ -295,19 +390,25 @@ void mm_subpage_pool_put(struct mm_subpage_pool *pool) kfree_rcu(pool, rcu); } -/* Keep the first implementation within one unambiguous allocation domain. - * Policy-sensitive, locked, droppable and multi-node faults keep fresh backing. - * An mm can move cgroups; old charged slots must not satisfy its new faults. - */ +void mm_subpage_pool_put(struct mm_subpage_pool *pool) +{ + /* Failed allocations must not leave an empty per-domain cache behind. */ + cow_pool_retire_empty(pool); + pool_put_raw(pool); +} + +/* Per-mm pools retain each still-backed charge/node/shard domain separately. */ struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, struct mm_struct *charge_mm, struct vm_area_struct *vma, unsigned long address) { struct mm_subpage_cow_pools *pools, *new_pools; struct mm_subpage_pool *pool, *new; - unsigned int bucket = hash_long(address >> PAGE_SHIFT, COW_POOL_BITS); + struct cow_pool_key key = {}; struct mem_cgroup *memcg; - bool matches; + unsigned long flags; + unsigned int attempt; + void *result; int nid; if ((vma->vm_flags & (VM_LOCKED | VM_DROPPABLE | VM_SPECIAL)) || @@ -320,60 +421,85 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, nid = first_node(node_states[N_MEMORY]); if (!node_isset(nid, cpuset_current_mems_allowed)) return NULL; - /* Publication pairs with cmpxchg: every bucket starts empty. Adjacent - * 4K leaves in one native virtual group always choose the same pool. - * Faulting threads can migrate CPUs without changing that identity. - */ + /* Acquire initialization published by the mm's cmpxchg below. */ pools = smp_load_acquire(&mm->cow_subpage_pool); if (!pools) { new_pools = kzalloc(sizeof(*new_pools), GFP_KERNEL | __GFP_ACCOUNT); if (!new_pools) return NULL; + spin_lock_init(&new_pools->lock); + INIT_LIST_HEAD(&new_pools->pools); + refcount_set(&new_pools->refs, 1); + INIT_WORK(&new_pools->free_work, cow_registry_free_work); + if (rhashtable_init(&new_pools->domains, &cow_domain_params)) { + kfree(new_pools); + return NULL; + } pools = cmpxchg(&mm->cow_subpage_pool, NULL, new_pools); if (!pools) pools = new_pools; else - kfree(new_pools); + cow_registry_put(new_pools); } memcg = get_mem_cgroup_from_mm(charge_mm); - /* Acquire the immutable charge domain published with this bucket. */ - pool = smp_load_acquire(&pools->pool[bucket]); - if (!pool) { - new = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); - if (!new) { + key.memcg = memcg; + key.nid = nid; + key.bucket = hash_long(address >> PAGE_SHIFT, COW_POOL_BITS); + for (attempt = 0; attempt < 3; attempt++) { + rcu_read_lock(); + pool = rhashtable_lookup(&pools->domains, &key, cow_domain_params); + if (pool && !refcount_inc_not_zero(&pool->refs)) + pool = NULL; + rcu_read_unlock(); + if (pool) { mem_cgroup_put(memcg); - return NULL; + return pool; } + new = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!new) + break; new->cow_free_slots = kcalloc(SUBPAGES_PER_FOLIO, sizeof(*new->cow_free_slots), GFP_KERNEL | __GFP_ACCOUNT); if (!new->cow_free_slots) { mm_subpage_pool_close(new); mm_subpage_pool_put(new); - mem_cgroup_put(memcg); - return NULL; + break; } + if (memcg) + css_get(&memcg->css); new->cow_memcg = memcg; new->cow_nid = nid; new->cow_shared = true; - pool = cmpxchg(&pools->pool[bucket], NULL, new); - if (!pool) { - pool = new; - /* The mm owns the initial pool reference and its memcg ref. */ - mm_subpage_pool_get(pool); - return pool; + new->cow_key = key; + INIT_LIST_HEAD(&new->cow_domain); + new->cow_registry = pools; + /* Hold caller, prospective membership and registry lifetime refs + * before a concurrent lookup can observe this pool. + */ + mm_subpage_pool_get(new); + refcount_inc(&pools->refs); + spin_lock_irqsave(&new->lock, flags); + result = rhashtable_lookup_get_insert_fast(&pools->domains, + &new->cow_hash, cow_domain_params); + if (!result) { + spin_lock(&pools->lock); + list_add_tail(&new->cow_domain, &pools->pools); + spin_unlock(&pools->lock); } - matches = pool->cow_memcg == memcg && pool->cow_nid == nid; + spin_unlock_irqrestore(&new->lock, flags); + if (!result) { + mem_cgroup_put(memcg); + return new; + } + /* Do not dereference an unreferenced object returned by insert. */ mm_subpage_pool_close(new); mm_subpage_pool_put(new); - } else { - matches = pool->cow_memcg == memcg && pool->cow_nid == nid; - mem_cgroup_put(memcg); + if (IS_ERR(result)) + break; } - if (!matches) - return NULL; - mm_subpage_pool_get(pool); - return pool; + mem_cgroup_put(memcg); + return NULL; } struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, @@ -390,18 +516,26 @@ bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *fo void mm_subpage_cow_pool_exit(struct mm_struct *mm) { struct mm_subpage_cow_pools *pools = xchg(&mm->cow_subpage_pool, NULL); - unsigned int i; + struct mm_subpage_pool *pool; + unsigned long flags; - /* No faults remain once mm_users reaches zero. Slot owners can outlive mm. */ + /* No faults remain; outstanding owners/callbacks never use this mm. */ if (!pools) return; - for (i = 0; i < ARRAY_SIZE(pools->pool); i++) { - if (!pools->pool[i]) - continue; - mm_subpage_pool_close(pools->pool[i]); - mm_subpage_pool_put(pools->pool[i]); + for (;;) { + spin_lock_irqsave(&pools->lock, flags); + pool = list_first_entry_or_null(&pools->pools, + struct mm_subpage_pool, cow_domain); + if (pool) + mm_subpage_pool_get(pool); + spin_unlock_irqrestore(&pools->lock, flags); + if (!pool) + break; + /* Lock order is pool -> registry list, never the reverse. */ + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); } - kfree(pools); + cow_registry_put(pools); } /* Counts include unavailable (busy/migrating) owners, so zero is conclusive. @@ -443,6 +577,10 @@ static void owner_remove(struct mm_subpage_owner *owner) if (owner->pool->unissued == owner) owner->pool->unissued = NULL; owner->pool->backing_pages--; + if (!owner->pool->backing_pages && owner->pool->cow_shared) { + owner->pool->closed = true; + cow_pool_detach_locked(owner->pool); + } } static void owner_free_rmap(struct mm_subpage_owner *owner) @@ -838,6 +976,7 @@ void mm_subpage_pool_close(struct mm_subpage_pool *pool) spin_lock_irqsave(&pool->lock, flags); pool->closed = true; + cow_pool_detach_locked(pool); /* At most one newly supplied owner has no issued slots. Close is O(1). */ owner = pool->unissued; if (owner) diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index 3c4c8c86eefe2..e3e47c9305901 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -9,3 +9,5 @@ clipped-firstwrite pinned-fork pagetable-retention pagetable-concurrent + +/domain-cycle diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index d306deb1f0054..0350be7a57912 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -18,11 +18,13 @@ TEST_GEN_PROGS_EXTENDED += pinned-fork TEST_FILES += README.pinned-fork TEST_GEN_PROGS_EXTENDED += pagetable-retention pagetable-concurrent TEST_FILES += README.pagetable-backing +TEST_GEN_PROGS_EXTENDED += domain-cycle +TEST_FILES += README.domain-cycle include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/pagetable-concurrent $(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread +$(OUTPUT)/domain-cycle $(OUTPUT)/pagetable-concurrent $(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/README.domain-cycle b/tools/testing/selftests/arm64/user4k/README.domain-cycle new file mode 100644 index 0000000000000..3e7425f4dbffb --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.domain-cycle @@ -0,0 +1,78 @@ +Cgroup allocation-domain pool reuse +================================== + +Build domain-cycle with -pthread. Run in the established4K userspace ABI on a +native16K kernel. Root-visible pagemap and kpagecgroup are required for the +controller's density/charge proof. The fixture itself checks every payload byte. +No memory.high/max limits, reclaim, swap-area or GPU operations are needed. + + userpages 4k ./domain-cycle --native-16k [--tail | --race] + +The program prints READY and waits for stdin byte 'c' before allocating payload. +A controller places only this fresh fixture in owned cgroup A, then acknowledges. +Each PHASE prints its index/PID/base/leaf count/stride and waits for 'c'. Both +cgroups must remain alive until all prior ranges have been checked and unmapped. + +Default: three1MiB useful guarded allocation phases. Run in A, move to B, move +back to A between phases. Check all256 written leaves/range (4K every8K), excluding +the inaccessible guards. The old sticky-domain implementation can allocate16K +native backing per4K leaf after a move. Packed acceptance is1MiB..1.25MiB native +PFN union per1MiB phase; the allowance covers bounded shard/partial-owner tails. + +--tail:16 independent4K allocations, switching A/B after EVERY leaf. All16VAs are +selected in the same current16-way allocator shard using its documented hash. +Eight leaves/domain should occupy at most3native folios: two useful plus one +possible preexisting partial boundary. This catches a latest-domain-only cache +that closes old A pools when switching to B, then cannot use A's surviving free +quarters on return. Preserve every earlier leaf throughout all switches. Useful +payload64KiB, maximum256KiB private backing, about8MiB virtual reservation. + +--race:4workers allocate64guarded leaves each, with bounded test-only2ms pacing +per leaf. RACE prints four bases and waits for 'c'. After acknowledgement, move +only this thread group between A/B32times while allocation is active; record +monotonic move intervals. RACE_DONE prints the allocation interval and waits. +Require at least one move overlapping that interval before claiming concurrency +coverage (this does not prove a particular RCU/CAS interleaving occurred). Check +all race PFNs are charged to A or B, never outside those owned domains. Then move +the stopped fixture to B and acknowledge: it allocates one additional1MiB guarded +PHASE. Require packed density after transitions stop, plus unchanged earlier +race PFNs/charges and all bytes. Worst native payload8MiB, useful2MiB; threads +have128KiB stacks. No tight density claim applies during arbitrary in-flight moves. + +For each held range, read pagemap indexed by target4K VA. A present PFN is in4K +units; native PFN=PFN>>2. Read /proc/kpagecgroup at native_PFN*8 and compare with +stat(the intended cgroup).st_ino. Old phases keep their original allocation-time +charge after process moves. During racing allocations either A/B is valid. +Re-read PFNs to reject migration during the charge sample. Do not use total +memory.current as exact payload accounting: it includes code/stacks/page tables. +Normal8KiB guard stride and --tail4KiB stride are emitted explicitly. + +Use an owned fresh uncapped cgroup hierarchy and a bounded controller. If using +a parent unit and childA/B groups, move the held PID into childA BEFORE enabling +memory in that owned parent's subtree_control (parent must be empty). Do not +change unrelated cgroups or global controller settings. Parent15s alarm kills +all fixture threads; partial thread creation releases waiting workers before +joining. Controller should independently terminate/reap the owned process/unit +on protocol error, timeout or failed metadata, and retain the evidence. + +--selftest only validates byte patterns and same-shard address selection; it does +not exercise cgroup moves, kernel pool allocation or RCU. Hardware qualification +must include bulk/tail/race content and exact-charge/density gates, mm-exit cleanup, +existing COW/fork/reclaim controls, and dense/concurrent timing. No source or +parser-only pass closes these runtime/performance requirements. + +Race density must count newly added native owners. Before and after sampling +the final steady range, recheck the original race PFNs and allocation charges. +Reject old/new physical 4K aliases and inconsistent charges in shared native +owners. Subtract only those native owners actually observed in both stable +snapshots. The 80-owner/1,310,720-byte limit applies to the added native PFN +union for the new 1 MiB payload. Reusing an old owner's free quarters should +not count its existing live quarters as additional backing. Record total, +reused and added owners separately, plus combined per-domain quarter masks. +This is payload accounting, not proof of whole-mm alias closure. + +On Neo the corrected controller observed 77 owners in the final range, of +which 14 were reused and 63 new (1,032,192 added bytes). Old mappings and +charges stayed unchanged, reused slots were disjoint and all bytes passed. +The earlier range-only oracle failed at 81 seen owners before its final +checks; that run remains failed, not retrospectively qualified. diff --git a/tools/testing/selftests/arm64/user4k/domain-cycle.c b/tools/testing/selftests/arm64/user4k/domain-cycle.c new file mode 100644 index 0000000000000..e67b9f1b22b33 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/domain-cycle.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define LEAF 4096UL +#define NATIVE 16384UL +#define COUNT 256UL +#define STRIDE (2 * LEAF) +#define LENGTH (COUNT * STRIDE) +#define TAIL_SPAN (8UL * 1024 * 1024) +static unsigned char *maps[16]; +static unsigned long leaves = COUNT, stride = STRIDE; +static void fail(const char *s) { perror(s); exit(1); } +static unsigned bucket(uintptr_t p) +{ return ((uint64_t)(p >> 14) * UINT64_C(0x61c8864680b583eb)) >> 60; } +static unsigned char value(unsigned phase, size_t leaf, size_t byte) +{ return (unsigned char)(1 + (phase * 53 + leaf * 7 + byte) % 251); } +static void check(unsigned count) +{ + for (unsigned phase = 0; phase < count; phase++) + for (size_t leaf = 0; leaf < leaves; leaf++) + for (size_t byte = 0; byte < LEAF; byte++) + if (maps[phase][leaf * stride + byte] != value(phase, leaf, byte)) { + fprintf(stderr, "FAIL bytes phase=%u leaf=%zu byte=%zu\n", phase, leaf, byte); exit(1); + } +} +static void ack(void) +{ + char c; ssize_t n; + do { n = read(0, &c, 1); } while (n < 0 && errno == EINTR); + if (n != 1 || c != 'c') fail("controller ack"); +} +#define WORKERS 4 +#define RACE_LEAVES 64UL +static unsigned char *race_maps[WORKERS]; +static atomic_int race_start; +static uint64_t monotonic_ns(void) +{ + struct timespec t; + if (clock_gettime(CLOCK_MONOTONIC, &t)) fail("clock_gettime"); + return (uint64_t)t.tv_sec * 1000000000 + t.tv_nsec; +} +static void race_check(void) +{ + for (unsigned w = 0; w < WORKERS; w++) + for (size_t leaf = 0; leaf < RACE_LEAVES; leaf++) + for (size_t byte = 0; byte < LEAF; byte++) + if (race_maps[w][leaf * STRIDE + byte] != value(w + 32, leaf, byte)) fail("race bytes"); +} +static void *race_worker(void *arg) +{ + unsigned w = (uintptr_t)arg; + int start; + do { start = atomic_load_explicit(&race_start, memory_order_acquire); if (!start) sched_yield(); } while (!start); + if (start != 1) return NULL; + for (size_t leaf = 0; leaf < RACE_LEAVES; leaf++) { + unsigned char *p = race_maps[w] + leaf * STRIDE; + if (mprotect(p, LEAF, PROT_READ | PROT_WRITE)) fail("race mprotect"); + for (size_t byte = 0; byte < LEAF; byte++) p[byte] = value(w + 32, leaf, byte); + /* Test-only pacing keeps allocation active during bounded controller moves. */ + struct timespec delay = {.tv_nsec = 2000000}; + while (nanosleep(&delay, &delay) && errno == EINTR) {} + } + return NULL; +} +static void race_run(void) +{ + pthread_t threads[WORKERS]; + pthread_attr_t attr; + if (pthread_attr_init(&attr) || pthread_attr_setstacksize(&attr, 128 * 1024)) fail("thread attributes"); + unsigned made = 0; + for (; made < WORKERS; made++) { + race_maps[made] = mmap(NULL, RACE_LEAVES * STRIDE, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (race_maps[made] == MAP_FAILED) fail("race mmap"); + int err = pthread_create(&threads[made], &attr, race_worker, (void *)(uintptr_t)made); + if (err) { errno = err; break; } + } + pthread_attr_destroy(&attr); + if (made != WORKERS) { + atomic_store_explicit(&race_start, 2, memory_order_release); + for (unsigned i = 0; i < made; i++) pthread_join(threads[i], NULL); + fail("pthread_create"); + } + printf("RACE pid=%d base0=%lx base1=%lx base2=%lx base3=%lx leaves=%lu stride=%lu\n", getpid(), (unsigned long)race_maps[0], (unsigned long)race_maps[1], (unsigned long)race_maps[2], (unsigned long)race_maps[3], RACE_LEAVES, STRIDE); + ack(); + uint64_t begin = monotonic_ns(); + atomic_store_explicit(&race_start, 1, memory_order_release); + for (unsigned i = 0; i < WORKERS; i++) if (pthread_join(threads[i], NULL)) abort(); + uint64_t end = monotonic_ns(); + race_check(); + printf("RACE_DONE pid=%d begin_ns=%llu end_ns=%llu\n", getpid(), (unsigned long long)begin, (unsigned long long)end); + ack(); +} +int main(int argc, char **argv) +{ + if (argc == 2 && !strcmp(argv[1], "--selftest")) { + unsigned count = 0; + for (uintptr_t p = 0x100000; p < 0x100000 + TAIL_SPAN; p += NATIVE) count += bucket(p) == 0; + if (count < 16 || value(0, 0, 0) == value(1, 0, 0)) return 1; + puts("PASS patterns/hash selection only; no domain allocation coverage"); return 0; + } + int tail = argc == 3 && !strcmp(argv[2], "--tail"); + int race = argc == 3 && !strcmp(argv[2], "--race"); + if ((argc != 2 && !tail && !race) || strcmp(argv[1], "--native-16k") || getauxval(AT_PAGESZ) != LEAF) return 77; + unsigned phases = tail ? 16 : race ? 1 : 3; + unsigned char *reservation = NULL; + alarm(15); setvbuf(stdout, NULL, _IONBF, 0); + printf("READY pid=%d\n", getpid()); ack(); + if (race) race_run(); + if (tail) { + leaves = 1; stride = LEAF; + reservation = mmap(NULL, TAIL_SPAN + NATIVE, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reservation == MAP_FAILED) fail("tail reserve"); + uintptr_t begin = ((uintptr_t)reservation + NATIVE - 1) & ~(NATIVE - 1); + unsigned found = 0; + for (uintptr_t p = begin; p < begin + TAIL_SPAN && found < phases; p += NATIVE) + if (!bucket(p)) maps[found++] = (void *)p; + if (found != phases) fail("same-shard tail addresses"); + } + for (unsigned phase = 0; phase < phases; phase++) { + check(phase); + if (!tail) { + maps[phase] = mmap(NULL, LENGTH, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (maps[phase] == MAP_FAILED) fail("mmap"); + } + for (size_t leaf = 0; leaf < leaves; leaf++) { + unsigned char *p = maps[phase] + leaf * stride; + if (mprotect(p, LEAF, PROT_READ | PROT_WRITE)) fail("mprotect"); + for (size_t byte = 0; byte < LEAF; byte++) p[byte] = value(phase, leaf, byte); + } + check(phase + 1); + printf("PHASE index=%u pid=%d base=%lx leaves=%lu stride=%lu\n", phase, getpid(), (unsigned long)maps[phase], leaves, stride); + ack(); + } + check(phases); + if (race) { + race_check(); + for (unsigned w = 0; w < WORKERS; w++) if (munmap(race_maps[w], RACE_LEAVES * STRIDE)) fail("race munmap"); + } + if (tail) { + if (munmap(reservation, TAIL_SPAN + NATIVE)) fail("tail munmap"); + } else for (unsigned phase = 0; phase < phases; phase++) if (munmap(maps[phase], LENGTH)) fail("munmap"); + puts("PASS domain allocation phases and all prior bytes; charge/density separately verified"); + return 0; +} From 9c512b25f55eaf6332b590e5a205ca7d55de12c8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 17:10:58 +1100 Subject: [PATCH 0798/1579] iommu: apple-dart: invalidate client-ranged T8140 streams again On T8140 the AOP DART has no apple,dma-range of its own. Its clients carry their own windows (apple,dma-range on the AOP, its setup mailbox, the audio node and the base-ns and leap-ns ADMACs, all at or above 1 TiB), and the leap-s ADMAC gets its IOVAs from the top of the DART's 42-bit aperture. apple_dart_fw_invalidate() takes its window from the DART node instead, so it works out [0, 64 GiB), finds every client DVA outside that window and returns before it looks at any stream. That includes the four-level streams (TCR 0x9 on every AOP DART stream Linux uses), which should get a whole-stream invalidate. So no unmap on this DART invalidates the TLB. The IOMMU core then hands the freed IOVA to the next buffer, and the DART still translates it to the pages that were just freed. A capture loop on the high-quality microphone shows it: every new stream's DMA landed in the first, long-freed buffer, never in its own buffer. Once those pages are back in the allocator: - the speaker sense capture zeroes whatever reuses them (struct file and fdtable objects, btrfs tree blocks); - the speaker DMA plays freed memory at full scale. The volume limit only applies to the new buffer, which the DMA no longer reads. Decide per stream instead: - a four-level stream gets the whole-stream invalidate first; - a three-level stream is invalidated by range inside the window of the domain attached to it. The driver records that window when the domain imports the stream's root and when it attaches. Streams whose domain uses the DART's own window, such as the display and DCP streams, keep the window they had before. Fixes: dd5ac6fdffb0 ("iommu: apple-dart: leave live firmware translations out of invalidates") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 50 +++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 4cc5aac4f97c2..00d42c36c0372 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -301,6 +301,12 @@ struct apple_dart { u64 *locked_owned[DART_MAX_STREAMS][DART_MAX_TTBR]; /* Leaf tables behind the root slots firmware held when we attached. */ struct apple_dart_fw_root *locked_fw[DART_MAX_STREAMS][DART_MAX_TTBR]; + /* + * First DVA of the window of the domain using each locked stream. The + * window can be the client's own (apple,dma-range on the client node) + * rather than the DART's. + */ + u64 locked_window[DART_MAX_STREAMS]; }; /* @@ -982,22 +988,18 @@ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, size_t entries = dart->pgsize / sizeof(u64); u32 slot_shift = ilog2(dart->pgsize) + ilog2(entries); u64 span = BIT_ULL(slot_shift + ilog2(entries)); - u64 base = (dart->dma_min + dart->dma_offset) & ~(span - 1); DECLARE_BITMAP(fw_slots, DART_MAX_ROOT_ENTRIES); size_t first, last, i, end; unsigned long flags; int sid, ret = 0; - if (entries > DART_MAX_ROOT_ENTRIES || hi < base || lo > base + span - 1) + if (entries > DART_MAX_ROOT_ENTRIES) return 0; - lo = max(lo, base); - hi = min(hi, base + span - 1); - first = (lo - base) >> slot_shift; - last = (hi - base) >> slot_shift; for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { const struct apple_dart_fw_root *fw; const u64 *live; + u64 base, start_dva, end_dva; __set_bit(sid, stream.sidmap); if (apple_dart_readl(dart, DART_TCR(dart, sid)) & @@ -1007,6 +1009,19 @@ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, goto next; } + /* + * A three-level stream walks the low bits of the DVA; the TLB is + * tagged with the whole DVA, inside the window of the stream's + * domain. + */ + base = READ_ONCE(dart->locked_window[sid]) & ~(span - 1); + if (hi < base || lo > base + span - 1) + goto next; + start_dva = max(lo, base); + end_dva = min(hi, base + span - 1); + first = (start_dva - base) >> slot_shift; + last = (end_dva - base) >> slot_shift; + /* Nothing of Linux reaches the hardware without the root. */ bitmap_zero(fw_slots, entries); spin_lock_irqsave(&dart->lock, flags); @@ -1024,8 +1039,8 @@ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, u64 start, stop; end = find_next_bit(fw_slots, last + 1, i) - 1; - start = max(lo, base + ((u64)i << slot_shift)); - stop = min(hi, base + ((u64)(end + 1) << slot_shift) - 1); + start = max(start_dva, base + ((u64)i << slot_shift)); + stop = min(end_dva, base + ((u64)(end + 1) << slot_shift) - 1); ret = apple_dart_t8110_hw_tlb_command_range(&stream, op, true, start, stop); if (ret) @@ -1481,6 +1496,18 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, stream_map->dart->hw->invalidate_tlb(stream_map); } +/* Record the DVA window of @domain for invalidates of @stream_map's streams. */ +static void apple_dart_set_locked_window(struct apple_dart_domain *domain, + struct apple_dart_stream_map *stream_map) +{ + struct apple_dart *dart = stream_map->dart; + u64 start = domain->domain.geometry.aperture_start + domain->dma_offset; + int sid; + + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) + WRITE_ONCE(dart->locked_window[sid], start); +} + static int apple_dart_setup_translation_locked(struct apple_dart_domain *domain, struct apple_dart_stream_map *stream_map) @@ -1488,6 +1515,7 @@ apple_dart_setup_translation_locked(struct apple_dart_domain *domain, struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; int i, ret; + apple_dart_set_locked_window(domain, stream_map); for (i = 0; i < cfg->apple_dart_cfg.n_ttbrs; i++) { ret = apple_dart_hw_map_locked_ttbr(stream_map, i); if (ret) @@ -1628,6 +1656,7 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, for_each_stream_map(i, cfg, stream) { if (!stream->dart->locked || !stream->dart->fw_mirror) continue; + apple_dart_set_locked_window(dart_domain, stream); for (index = 0; index < pgtbl_cfg.apple_dart_cfg.n_ttbrs; index++) { ret = apple_dart_hw_map_locked_ttbr(stream, index); if (ret) { @@ -2414,7 +2443,7 @@ apple_dart_platform_ioremap_np_resource(struct platform_device *pdev, static int apple_dart_probe(struct platform_device *pdev) { - int ret; + int ret, sid; u32 dart_params[4]; struct device_node *pd_np; struct resource *res; @@ -2644,6 +2673,9 @@ static int apple_dart_probe(struct platform_device *pdev) &dart->dma_min, &dart->dma_max, &dart->dma_offset); } + for (sid = 0; sid < DART_MAX_STREAMS; sid++) + dart->locked_window[sid] = dart->dma_min + dart->dma_offset; + if (dart->num_streams > DART_MAX_STREAMS) { dev_err(&pdev->dev, "Too many streams (%d > %d)\n", dart->num_streams, DART_MAX_STREAMS); From 27178c4e373a23ba1015647f774a3102265c1b27 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 10 Oct 2026 04:45:59 -0400 Subject: [PATCH 0799/1579] drm/asahi: reuse the render state mapping for auxiliary framebuffers Each render pair keeps sixteen generations, with a 32 KiB auxiliary framebuffer in each generation. Allocating a second alias from the fixed 4 MiB auxiliary arena exhausts that arena as logical contexts accumulate, returning ENOSPC and stale GL readback despite spare backing memory. Use the existing full state mapping and its auxiliary offset. The state owner already retains both the pages and their client-VM mapping for the required lifetime. This preserves queue depth and concurrent contexts. Tested on A18 Pro with 16 retained contexts, 128 immediate context recreations, targeted native GL/Zink rendering, native/x86-64/i686 compute oracles and MRT, and GL/GLES glmark2 scene validation. Unknown scene validations remain unknown; this is not a CTS conformance claim. Assisted-by: LLM Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/g17/queue/render/memory.rs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index fc5a05e9bf8a9..85319585b3d13 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -259,7 +259,6 @@ impl Compact { pub(super) struct Generation { deflake: mmu::KernelMapping, - auxiliary: mmu::KernelMapping, fragment_client: mmu::KernelMapping, state: KernelObject, fragment: KernelObject, @@ -285,15 +284,6 @@ impl Generation { mmu::PROT_GPU_FW_SHARED_RW, mmu::UAT_PGSZ, )?; - let auxiliary_range = cfg::AUXILIARY; - let auxiliary = state.map_range( - vm, - storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE, - auxiliary_range, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - 0, - )?; let fragment = alloc.kernel( mmu::UAT_PGSZ, mmu::UAT_PGSZ as u64, @@ -311,7 +301,6 @@ impl Generation { KBox::new( Self { deflake, - auxiliary, fragment_client, state, fragment, @@ -349,7 +338,10 @@ impl Generation { self.state.gpu_va() + storage::TA_STATUS as u64, self.fragment_client.iova(), self.fragment.gpu_va(), - self.auxiliary.iova(), + // The retained state already maps these pages into the client VM. + // The auxiliary framebuffer accepts a full GPU address; reusing this + // view avoids exhausting the fixed 4 MiB alias arena across pairs. + self.state.gpu_va() + storage::AUXILIARY as u64, ] } } From 9266541b8d35c1cd7146eec9a9b31b39fb7f29cc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 22:12:20 +1100 Subject: [PATCH 0800/1579] treewide: describe hardware behaviour in comments Rewrite a set of comments so they state the hardware and firmware behaviour directly: - soc/apple SEP (sks.rs, image.rs, profile.rs, sep.rs): key-store request layouts, timeouts and endpoint init described as the enclave protocol. - soc/apple mailbox: the ASC wrapper CPU start/stop sequences. - accel/ane: register, handshake, ring and command-format notes for the T602x/T8112 Neural Engine, plus the scratch3_ack parameter text. - drm/asahi G17: the tiler-heap shrink policy. - mt7932, reset-apple-cio and two documentation pages: keep the descriptions self-contained. Every edit keeps its line count, so line-dependent data (panic locations, WARN/BUG and dynamic-debug records) is unchanged. Apart from the scratch3_ack module parameter description, the change is comments only; there is no functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/admin-guide/media/apple-ave.rst | 6 +- .../device_drivers/wifi/mt7932-neo.rst | 6 +- drivers/accel/ane/ane_fw_validate.h | 12 +- drivers/accel/ane/ane_t6021.h | 230 +++++++++--------- drivers/accel/ane/ane_t6021_boot.c | 128 +++++----- drivers/accel/ane/ane_t6021_boot.h | 132 +++++----- drivers/accel/ane/ane_t6021_fwload.c | 6 +- drivers/accel/ane/ane_t6021_rtclient_main.c | 16 +- drivers/gpu/drm/asahi/g17/buffer.rs | 4 +- drivers/net/wireless/mediatek/mt7932/mt7932.c | 2 +- drivers/net/wireless/mediatek/mt7932/reset.c | 4 +- drivers/reset/reset-apple-cio.c | 2 +- drivers/soc/apple/image.rs | 4 +- drivers/soc/apple/mailbox.c | 4 +- drivers/soc/apple/profile.rs | 8 +- drivers/soc/apple/sep.rs | 2 +- drivers/soc/apple/sks.rs | 64 ++--- 17 files changed, 314 insertions(+), 316 deletions(-) diff --git a/Documentation/admin-guide/media/apple-ave.rst b/Documentation/admin-guide/media/apple-ave.rst index 228ea84db2b86..d7e63d029de15 100644 --- a/Documentation/admin-guide/media/apple-ave.rst +++ b/Documentation/admin-guide/media/apple-ave.rst @@ -58,6 +58,6 @@ kernel health and the existing desktop/radio stack. Keep the generated inputs, bitstreams, decoded pixels, logs and hashes as a repeatable artifact. The prior implementation was authored by Ace (Acelogic). This contribution -ports that existing GPL work; it does not claim independent clean-room -provenance. Arbitrary resolutions, standard media APIs, throughput, -concurrent clients and other codecs or SoCs are outside this experiment. +ports that existing GPL work. Arbitrary resolutions, standard media APIs, +throughput, concurrent clients and other codecs or SoCs are outside this +experiment. diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 220eff6fbbcad..996fbe5d34f65 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -7,10 +7,8 @@ J700 MT7932 Wi-Fi uses cfg80211 and NetworkManager with firmware-managed WPA2-CCMP authentication. Bluetooth uses a transport on the radio's second PCIe function. -The Wi-Fi import is pinned to -``aurora-silicon/linux-neo-cleanroom-eryk-with-wifi``, commit -``610cb463c03f9bc68a5d020e7ca443af1fda4856``, with configuration, scan IE and -WPA2 AKM compatibility fixes. Existing source notices are retained. +The Wi-Fi driver carries configuration, scan IE and WPA2 AKM compatibility +fixes on top of its original import. Existing source notices are retained. Co-authored by DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. Platform and configuration diff --git a/drivers/accel/ane/ane_fw_validate.h b/drivers/accel/ane/ane_fw_validate.h index f4f723a3f33df..3f8e82978b4de 100644 --- a/drivers/accel/ane/ane_fw_validate.h +++ b/drivers/accel/ane/ane_fw_validate.h @@ -22,7 +22,7 @@ #ifndef __ANE_FW_VALIDATE_H__ #define __ANE_FW_VALIDATE_H__ -/* FWIM surface size = config+0x138 byte-count = 0x500000 (Main +/* FWIM surface size = 0x500000 bytes (Main * 2026-09-20, audit 751caa4, T6021). Covers the selene vmsize 0x4fc000 * and the bia vmsize 0x4ec000; NOT derivable from the blob length. */ @@ -269,10 +269,10 @@ ane_fw_validate_blob(const u8 *blob, size_t size, * whose key <= the chip revision: header {type 1, flags 3 (20-byte * records), capacity 0x24, count, u32 key}, then {u32 offset, u64 mask, * u64 value} per record (iBoot table offsets carry flag bit 30; the - * block does not). Source: iBootStage2 macOS 26.6.2 (25G83), LZFSE, not - * encrypted (receipts/2026-10-01-t8112-optin). The 13.5 iBoot2 that - * preloads the 13.5 image is encrypted; on T6021 its block (the - * captures) equals the 25G83 records exactly. + * block does not). Source: the 25G83 iBoot's records + * (receipts/2026-10-01-t8112-optin). On T6021 the block that the 13.5 + * iBoot leaves in the preloaded image (the captures) equals the 25G83 + * records exactly. */ struct ane_asc_tunable { u32 off; @@ -395,7 +395,7 @@ static const struct ane_asc_tunables ane_t8112_asc_tunables = { }; /* T8112 RTK_soc_revision from its two eFuse words at 0x23d2c8060, as - * iBootStage2 25G83 j413 0x40270 (and j415/j473/j493) computes it: + * iBoot (25G83, on j413 and j415/j473/j493) computes it: * bits 2:0 = w0[29:27], bits 6:4 = {w1[0], w0[31:30]}. */ static inline u32 ane_t8112_fuse_revision(u32 w0, u32 w1) diff --git a/drivers/accel/ane/ane_t6021.h b/drivers/accel/ane/ane_t6021.h index 03d9ee35cdd96..4de6c0dfe6b53 100644 --- a/drivers/accel/ane/ane_t6021.h +++ b/drivers/accel/ane/ane_t6021.h @@ -1,18 +1,18 @@ /* SPDX-License-Identifier: GPL-2.0-only OR MIT */ /* T6021 (H14 / J414c) Apple Neural Engine — skeleton constants and types. * - * Every constant below is static-decode provenance, no device writes: + * Where the constants below come from (none of them needs a device write): * - reg windows, IRQ, pmgr islands: ADT j414c (DeviceTree.j414cap.im4p, * receipts/2026-09-18-t6021-engine-layout-mined §2), Linux translation * +0x200000000 (proven class, pmgr low-32 match). * - ASC cpu block (+0x1400000 h14g) + MBI transport (SCRATCH, channel - * table, +0x1844000 doorbell): K14 kext disasm, W4-fix receipt - * (initializeANESoCConfig h14g blocks + InitializeRTBuddy + - * doorbell setter 0x…95ebdd0); m1n1 ASCRegs = t8103 cross-ref only + * table, +0x1844000 doorbell): the W4-fix receipt plus the live reads + * and the handshake described below; m1n1 ASCRegs = t8103 cross-ref + * only. * - RTKit MGMT protocol: Asahi rtkit.c semantics, u64 message halves as * staged in omarchy-ane rtkit/h14_rtkit_hello.py (commit 6ad26b7). - * - RTBuddy endpoint table + doorbell word: K14 kext cfg table - * __DATA_CONST.__const+0x814e520 and HandleRTBuddyMessage + * - RTBuddy endpoint table + doorbell word: the per-endpoint ring sizes, + * fourccs and doorbell bits listed with ane_t6021_eps below * (receipts/2026-09-18-h14-w2-protocol-decode §3). * * DT binding (driver + packaging/dt/t6021-ane.dts are the two halves): @@ -25,7 +25,7 @@ * fourth window "fuse" (0x23d2c8060+8, chip revision). * reg/reg-names = "engine" (whole 32 MiB ADT range0, 0x284000000; * the H13-style +0x1c04000 engine delta does not exist - * on this SoC — kext never computes it and first touch + * on this SoC — it is never applied here; a first touch * external-aborted, proven 2026-09-18), * "pmgr" (0x28e080000+0x4034, pmgr1,t6021 island words), * "set" (0x28e08c000+0x4000, SET window — read-only by @@ -43,8 +43,8 @@ * ane_cpu last): ane_sys_mpm@4000, ane_td@4008, * ane_base@4010, ane_set1..4@4018-4030, ane_cpu@2e0. * ane_td/ane_base were the W3 chain gap (six consumed, - * block access reset the machine); receipt - * 2026-09-19-h14-init-sequence-kext-trace §3. + * block access reset the machine); see the + * 2026-09-19 init-sequence receipt §3. */ #ifndef __ANE_T6021_H__ @@ -80,20 +80,20 @@ enum { #define ANE_PS_AUTO_ENABLE BIT(28) /* ANE-block-relative ASC/RTBuddy addresses. CPU block = ANE+0x1400000 - * for h14g (K14 initializeANESoCConfig h14g blocks 0x…961365c / - * 0x…9614084 build 0x1400044 with NO 0x200000 orr — the 0x1600044 - * phase-1 §2 quote is the h16g/h17/h18g variant block 0x…9613458; - * runtime h14g-shaped sites 0x…95d2968 write RUN there). CPU_STATUS - * +0x48 keeps the m1n1 ASCRegs +4 shape (config field dev+0x49c = - * 0x1400048; kext poll 0x…95ecfb4). + * for h14g: CPU_CONTROL is 0x1400044, without the 0x200000 offset of + * the h16g/h17/h18g parts (whose block is the 0x1600044 one that the + * phase-1 §2 note quoted). + * RUN is set at 0x1400044 on this part. + * CPU_STATUS +0x48 keeps the m1n1 ASCRegs +4 shape (0x1400048, polled + * while the ASC CPU starts). * * W10 (2026-09-19) CORRECTS the claim that stood here — that the m1n1 * t8103 mailbox (INBOX_CTRL +0x8110 / INBOX0 +0x8800) "has NO h14g * analog". It has one, and it is live: see ANE_ASC_MBOX_* below. The * earlier live test read +0x1608114, the h16g base, which is simply - * the wrong address on this part; and the absence of 0x1408xxx - * constants from the kext text means only that the mailbox belongs to - * the RTBuddy provider kext, not to AppleH11ANEInterface. + * the wrong address on this part. The mailbox belongs to the RTBuddy + * transport rather than to the ANE engine interface, which is why the + * engine-side constants never refer to 0x1408xxx. * * W10 also read the CPU block for the first time, on a Linux boot with * all eight pmgr domains at ACTUAL=0xf and the W8 grant applied: @@ -108,7 +108,7 @@ enum { #define ANE_ASC_RVBAR 0x1050000 /* fw entry | valid bit0 */ #define ANE_ASC_EDPRCR 0x1010310 /* phase1 S2 whitelist */ #define ANE_ASC_VERS 0x1840000 -#define ANE_ASC_RTB_STATUS 0x1840088 /* K14 poll: value < 2 */ +#define ANE_ASC_RTB_STATUS 0x1840088 /* polled: value < 2 */ #define ANE_ASC_RTB_STATUS_UNK7C 0x184007c /* phase1 S2 whitelist */ /* ASC blocks iBoot names in the firmware's __rtk_patch records @@ -164,36 +164,36 @@ enum { #define ANE_FATAL_READ_LO 0x1854000 #define ANE_FATAL_READ_HI 0x1c04000 -/* MBI transport (K14 h14g config: socinit stores the SCRATCH register - * offsets as q-blobs @0x…7503a40/0x…7503a50/0x…7503a60 into dev+0x438 - * and the message-register offsets @0x…7503860 into dev+0x488; runtime - * handshake in InitializeRTBuddy 0x…95e942c: cmd buffer base -> - * SCRATCH0/1 (0x…95eaa94), wake 0xf7fbdff9 -> SCRATCH7 (0x…95eab24), - * poll SCRATCH7 == 0x08042006 "channel description table ready" - * (0x…95eab74), table base read back from SCRATCH0/1 (0x…95ead04), - * per-channel {type,bit,size,phys} entries registered with the - * doorbell setter 0x…95ebdd0 writing (1 << bit) to +0x1844000. Host - * ack = SCRATCH3 = 0x08042006 (0x…95eaee4). +/* MBI transport (h14g): the SCRATCH registers and the message + * registers below are the host side of the handshake. The host writes + * the command buffer base to SCRATCH0/1, then the wake word 0xf7fbdff9 + * to SCRATCH7, and polls until SCRATCH7 == 0x08042006 ("channel + * description table ready"). It then reads the table base back from + * SCRATCH0/1; the table holds per-channel {type,bit,size,phys} + * entries, and a channel is signalled by writing (1 << bit) to the + * +0x1844000 doorbell. + * Host ack = SCRATCH3 = 0x08042006. This driver runs the handshake + * only behind the transport opt-in (struct ane_t6021, transport). + * + * RTBuddy mode: each ANE channel is an RTBuddy endpoint. A message to + * an endpoint is sent by copying it into that endpoint's shared ring + * and then ringing the AP mailbox doorbell (+0x1844000) with the + * endpoint's bit; the firmware's mailbox block has inbox and outbox + * control registers and refuses a message while its inbox is not + * ready or would overflow. + * + * Endpoint number = doorbell bit: EP0 is RTBuddy management (hello, + * endpoint roll call, power acknowledgement), EP1..5 are the ANE data + * channels. The host side of each endpoint is a ring allocated by + * this driver (struct ane_t6021_ep below), with the write cursor + * advanced as described further down. + * + * Responses travel the other way on the fw->host channels (T2F*, + * see ane_t6021_eps), which this driver does not walk yet. * - * PROVIDER DECODE (2026-09-19, kernelcache.release.mac14j): in - * RTBuddy mode the TX gate lives in com.apple.driver.RTBuddy 1.0.0 - * (carved from the KC, 46088 B, __TEXT_EXEC 0x…b696860). RTBuddy - * publishes per-ANE nubs named "%sEndpoint%u" (0x…7cc7d3f) with the - * role string "ANE" -> ANEEndpoint1..ANEEndpoint5, class - * RTBuddyEndpointService (0x98 B) wrapping RTBuddyEndpoint (0xE8 B), - * over the kernel IOSlaveEndpoint family; K14 EnableRTBuddyEndpoints - * (0x…95feb30) formats the name ("%s%d" @0x…74c481f), waits 10 s - * (x1 = 10^10 ns) for the match and takes the gate from [svc+0x88], - * registering its rx callback at gate+0xd0. The gate's per-message - * send is the vtable+0x1e8 slot; its body ends in the AKF mailbox - * write (AKF_AP_MAILBOX_SET = the +0x1844000 doorbell; RTBuddy dump - * fn 0x…b69d9c8 prints "AKF_KIC_INBOX_CTRL / AKF_KIC_MAILBOX_SET / - * AKF_AP_OUTBOX_CTRL / AKF_AP_MAILBOX_SET" and guards - * "INBOX%d not ready"/"Inbox%d overflow"). Endpoint number = - * doorbell bit: EP0 is RTBuddyManagementEndpoint (_handleHello / - * _handleEPRollCall / _handlePowerAck), EP1..5 the ANE data channels. * [INFERENCE: the fw->host doorbell/IRQ bit numbering mirrors the * host->fw SET bit per Asahi rtkit semantics; pinned live by W5.] + * */ #define ANE_MBI_SCRATCH0 0x1840048 /* SCRATCH0..7 = +0x48..+0x64 */ #define ANE_MBI_SCRATCH6 0x1840060 @@ -220,14 +220,14 @@ enum { #define ANE_MBI_MSG_A2I_RD 0x184c000 /* host->fw message read peer */ #define ANE_MBI_MSG_A2I_WR 0x1850000 /* host->fw message write */ -/* Per-message MBI word (48-bit ring notification, K14 - * rtbuddyEndpointSendMessage 0x…95f3bf4-c30): offset = ring write +/* Per-message MBI word (48-bit ring notification, sent once for every + * endpoint message): offset = ring write * cursor [23:0], length [47:24]. NOT the 54-bit surface-announce - * word (ane_ep_doorbell_encode below) — that one rides SetupEndpoints - * buffer mapping; this one is what the gate sends per command: the - * gate call is send(&msg48, 0, 1) at vtable+0x1e8, after the command - * bytes are already memcpy'd into the shared ring at ring_base + - * cursor. Length is capped at 0xffffff by the encoder field. + * word (ane_ep_doorbell_encode below) — that one rides the endpoint + * buffer setup; this one is sent once per command, after the command + * bytes are already copied into the shared ring at ring_base + + * cursor. + * Length is capped at 0xffffff by the encoder field. */ #define ANE_MBI_MSG48_OFF GENMASK_ULL(23, 0) #define ANE_MBI_MSG48_LEN GENMASK_ULL(47, 24) @@ -238,18 +238,18 @@ static inline u64 ane_mbi_msg48_encode(u32 cursor, u32 len) FIELD_PREP(ANE_MBI_MSG48_LEN, len); } -/* Host->fw TX sequence (RTBuddy mode, all static-decode proven): - * 1. fail if len > ring_size (K14 0xe00002c2 @0x…95f3a24) +/* Host->fw TX sequence (RTBuddy mode): + * 1. fail if len > ring_size * 2. cursor = write_cursor; if (cursor + len >= ring_size) cursor = 0 - * (exact fit wraps: csel lo @0x…95f3b04) + * (an exact fit wraps too) * 3. memcpy(ring + cursor, cmd, len) (DMA-coherent ring) * 4. dma_wmb() (ring visible before bell) * 5. write32(ANE_MBI_MSG_A2I_WR, lo) + write32(+4, hi) — 32-bit * halves (the AKF message register is word-shaped; W6 proved the * W5-live abort was NOT the 64-bit writeq — the halves abort too) * 6. write32(ANE_MBI_DOORBELL, 1 << ep) - * 7. on send success only: write_cursor = cursor + len (K14 - * 0x…95f3c70-c); the cursors at rec+0x40/rec+0x58 track the last + * 7. on send success only: write_cursor = cursor + len, so a + * failed send leaves the cursor at the last * issued slot. * CSNE_CMD_PING = header-only 0x11 on EP1 (INIT) -> doorbell bit * 1 << 1 = 0x2. Only the a2i message register + doorbell are @@ -258,9 +258,9 @@ static inline u64 ane_mbi_msg48_encode(u32 cursor, u32 len) * host-writable granted, W9 nonzero latch). */ -/* MBI channel-table entry (kext stride 0x100, fields at +0x40 family: - * type @+0x40, doorbell bit @+0x44, size @+0x48, phys @+0x50 — - * InitializeRTBuddy 0x…95eade8-0x…95eae58) +/* MBI channel-table entry as the firmware publishes it (stride 0x100): + * type @+0x40, doorbell bit @+0x44, size @+0x48, phys @+0x50 within + * each entry. */ #define ANE_MBI_CHAN_STRIDE 0x100 #define ANE_MBI_CHAN_MAX_DUMP 8 @@ -298,7 +298,7 @@ static inline u64 ane_mbi_msg48_encode(u32 cursor, u32 len) #define ANE_RTKIT_EP_OSLOG 8 #define ANE_RTKIT_EP_TRACEKIT 0xa -/* RTBuddy app endpoints — K14 InitializeRTBuddyEndpoints opens ids 1..6; +/* RTBuddy app endpoints — the host opens ids 1..6; * ring sizes + fourccs from the per-EP config table (W2 §3). fourcc is * byte-reversed in the table (0x54324643 = "T2FC"). */ @@ -312,11 +312,11 @@ enum ane_t6021_eps { ANE_T6021_EP_COUNT = ANE_T6021_EP_T2HT + 1 /* arrays index by id */ }; -/* CSNE command ids — selene id->name table, vaddr 0xea430, 96 entries +/* CSNE command ids — the firmware knows 96 of them * (full set: receipts/2026-09-18-h14-w2-protocol-decode §4 and * receipts/2026-09-18-h14-w2-protocol-decode/fw_cmd_table.json). The * fw parses the id as u16 at wire offset +4 (W4 correction of the W2 - * §4 "offset 0" claim: fw sites 0x4d244/0x4d264/0x5246c); the kext + * §4 "offset 0" claim); the * controller header for the fw->host direction is a different shape * (u32 id @ +0x8, 0x24 bytes) and never rides host->fw submission. */ @@ -337,12 +337,12 @@ enum ane_t6021_csne_cmd { CSNE_CMD_BACK_CHANNEL_RPC = 0x7000, }; -/* fw buffer word (selene builder 0x982b4/0x98f84, decoder 0x98fc8; the - * K14 SetupEndpoints word is the same shape): addr[43:0] | +/* fw buffer word (the firmware encodes and decodes buffer words this + * way, and the endpoint setup word has the same shape): addr[43:0] | * size_code[51:44] | unit[53:52], unit 0 = no size, 1 = code*4K, * 2 = code*1M, 3 = code*2M. The fw encoder picks unit 1 below 1 MiB - * and unit 2 at or above, size code = CEILING division (cinc on the - * remainder), so the decoded size is the rounded-up value. rtkit.c's + * and unit 2 at or above, size code = CEILING division (rounded up, + * never down), so the decoded size is the rounded-up value. rtkit.c's * BUFFER_REQUEST is this word with unit 1. */ #define ANE_EP_DOORBELL_OFFSET GENMASK_ULL(43, 0) @@ -373,7 +373,7 @@ struct ane_t6021_ep { const char *name; u32 fourcc; u32 ring_size; - u32 write_cursor; /* next ring slot (K14 rec+0x20) */ + u32 write_cursor; /* next ring slot */ void *ring; /* dma_alloc_coherent, ring_size */ dma_addr_t ring_iova; bool started; @@ -414,7 +414,7 @@ struct ane_t6021 { * live, enabled, permanently-empty ASC v4 control pair. It is * empty because the ASC CPU is STOPPED with RUN clear and RVBAR * entry 0, i.e. no firmware was ever started. Default off = - * status-only bring-up. Opt-in runs the kext-evidenced MBI + * status-only bring-up. Opt-in runs the documented MBI * handshake (SCRATCH wake -> fw channel table), capture-only. */ bool transport; @@ -519,25 +519,25 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, /* ---- CSNE_CMD wire structs (host->fw on the INIT channel) ---- * - * W4 static decode, extending W2 §4 (the receipt's "fw parses a u16 id - * at header offset 0" was WRONG — three independent fw parse sites read - * the id as u16 at offset +4: the generic CSNE processor @0x4d264, its - * preload @0x4d244, and the LOAD_PROGRAM/CREATE_PROCESS/PROCEDURE_CALL - * pre-parse @0x5246c, all `ldrh wN, [xM, #4]` in selene). Wire is + * W4, extending W2 §4 (the receipt's "fw parses a u16 id + * at header offset 0" was WRONG — the firmware reads + * the id as u16 at offset +4, for generic commands as well as for + * LOAD_PROGRAM/CREATE_PROCESS/PROCEDURE_CALL). + * Wire is * little-endian; this module is arm64-only so host order is wire order. * - * The generic processor takes commands < 0x1b89 bytes (work-item size - * bound @0x4d134-0x4d158) and writes completion state back into the - * command block (strb @0x4d324 byte +6, str @0x4d3b0 qword +8) — the - * ring slot doubles as the response area. The processor's id tree - * sends PING/BUILDINFO/BOOT/REG_FILE_LOAD/IPC_ENDPOINT_SET on to the - * default path (0x4e65c): carried without field parsing, so + * The firmware takes commands < 0x1b89 bytes and writes completion + * state back into the + * command block (byte +6 and the qword at +8) — the + * ring slot doubles as the response area. The firmware handles + * PING/BUILDINFO/BOOT/REG_FILE_LOAD/IPC_ENDPOINT_SET on its + * default path, without parsing their fields, so * their payloads beyond the header are opaque until the W1 live * exchange pins them. */ struct ane_csne_hdr { u32 rsvd0; /* bytes 0..3: never read by the fw processor; 0 */ - u16 id; /* PROVEN u16 @ +4 (sites above) */ + u16 id; /* PROVEN u16 @ +4 (see above) */ u8 flags; /* fw-written byte @ +6 (status/scratch) [INFERENCE] */ u8 rsvd7; }; @@ -551,12 +551,12 @@ static inline void ane_csne_hdr_init(struct ane_csne_hdr *h, u16 id) } /* Header-only CSNE_CMDs: BOOT (0x10 — boot-arg surfaces ride - * SCRATCH0-7 / SetupFWInitBootArgs, not the command, phase1 §2.5), + * SCRATCH0-7 / the firmware init boot args, not the command, phase1 §2.5), * PING (0x11), BUILDINFO (0x06). sizeof(struct ane_csne_hdr) bytes. */ /* REG_FILE_LOAD (0x05) payload: the 1456 B blob is the fw's own - * __DATA._rtk_tunables section (@0x100590, size 0x5b0) — its transport + * _rtk_tunables section (0x5b0 bytes) — its transport * (inline vs shared-memory iova) is [INFERENCE], pinned by W1. */ struct ane_csne_cmd_reg_file_load { @@ -573,21 +573,21 @@ struct ane_csne_cmd_ipc_endpoint_set { u8 payload[]; }; -/* PROCEDURE_CALL (0x204) / INFERENCE_CALL (0x404) — the kext - * CANE_SUB_PACKET_CMD_PROCEDURE_CALL family. Field offsets PROVEN from - * the fw parse; field semantics named only where the kext asserts name - * them (programId, procedureId, numIoBuffers in assert order, W2 §4): - * +0x08/+0x0c u32 pair validated together (ldp @0x4d654) - * +0x10 u64 passed to the fw validator (0x4d6d4) +/* PROCEDURE_CALL (0x204) / INFERENCE_CALL (0x404) — the procedure-call + * command family. Field offsets follow what the firmware accepts; + * field names (programId, procedureId, numIoBuffers, W2 §4) are given + * only where the field's role is known: + * +0x08/+0x0c u32 pair validated together + * +0x10 u64 passed to the fw validator * +0x18 u32 stats/priority, fw-valid [2,7]; 2 is the value * proven end-to-end (binding.json stats_type; the * first-inference receipt's bare calls) - * +0x28 u32 element count (loop bound @0x524bc-0x524c4) - * +0x60 count × 0x30-byte io-buffer records (@0x524a8-0x524b4) - * The kext `size % sizeof(sCSneCmdProcedureCall) == 0` assert says - * commands form same-shape arrays in the ring; this driver submits one + * +0x28 u32 element count + * +0x60 count × 0x30-byte io-buffer records + * Commands are whole multiples of the procedure-call size, so they + * form same-shape arrays in the ring; this driver submits one * command per slot. INFERENCE_CALL shares the shape [INFERENCE: the - * 0x404 id is not in the decoded id tree — it is the W4 submission + * 0x404 id is not among the decoded ids — it is the W4 submission * endpoint per the phase-1 workstream plan]. */ struct ane_csne_io_elem { @@ -622,23 +622,23 @@ ane_csne_cmd_procedure_call_size(unsigned int num_io_buffers) num_io_buffers * sizeof(struct ane_csne_io_elem); } -/* Fw-side bound: the generic CSNE processor rejects work items of - * 0x1b89 bytes and above (@0x4d134 cmp x2, #0x1b89). What x2 names - * beyond "the command's size" is [INFERENCE] — enforced here as a +/* Fw-side bound: the firmware rejects work items of + * 0x1b89 bytes and above. That the bound applies to the whole command + * and not to some other size is [INFERENCE] — enforced here as a * fail-fast so an oversized command cannot enter the ring. */ #define ANE_CSNE_CMD_MAX_SIZE 0x1b88 -/* ---- LOAD_PROGRAM (0x200) wire contract (selene 0x4e44c hook + - * 0x4e53c worker + 0x4e640 registration; decode 2026-09-26, entry - * item 23/23b/23c/23e): the payload is NINE 0x30-byte named section - * records at +0x08. Registration publishes ".bin" into the fw - * program-info symbol table (format "%s.bin" @0xa733f, - * CAneProgramInfo.cpp) with value = record+0x18 and aux = - * u32(record+0x20). The fw dereferences record+0x18 (0x5d0c8), so it +/* ---- LOAD_PROGRAM (0x200) wire contract (2026-09-26, entry item + * 23/23b/23c/23e): the payload is NINE 0x30-byte named section + * records at +0x08. Loading publishes ".bin" for each section + * into the fw program-info symbol table, with value = record+0x18 + * and aux = u32(record+0x20). + * + * The fw dereferences record+0x18, so it * carries a fw-addressable pointer — in this driver, the IOVA of a * host-authored section object (below). Absent sections (flags bit0 - * clear) are skipped by every decoded parser. ---- + * clear) are skipped by the firmware. ---- */ enum ane_t6021_load_section { ANE_SEC_GENERIC = 0, @@ -671,9 +671,9 @@ static_assert(sizeof(struct ane_csne_cmd_load_program) == 0x08 + 9 * 0x30); /* Section-record wire accessors over the 0x30-byte io_elem (the only - * bytes any decoded parser reads; the rest is unread pass-through): + * bytes the firmware reads; the rest is unread pass-through): * +0x00 u8 flags — bit0 present; +0x18 u64 obj — fw-addressable - * pointer, validated by 0x5d0c8 to land inside a registered program + * pointer, which the firmware checks lands inside a registered program * object's entry table; +0x20 u64 key — per-section lookup key. */ #define ANE_SEC_F_PRESENT BIT(0) @@ -689,7 +689,7 @@ static inline void ane_sec_record_init(void *rec, u64 obj, u64 key) } /* Host-authored PROGRAM OBJECT (the thing section records point at; - * 0x5d0c8 validates: magic 1 @+0, count <= 0x10 @+4, table entry + * the firmware validates: magic 1 @+0, count <= 0x10 @+4, table entry * count in [0x201, 0x400] @+0x204, 0x30-byte entry table @+0x208 with * the recorded pointer bounded inside it). Minimum object = header + * 0x201 zeroed entries; entries are the same 0x30-byte shape. @@ -716,7 +716,7 @@ static inline void ane_progobj_init(void *obj, u32 entries) /* caller zeroes the tail: entries table starts at +0x208 */ } -/* Host-authored OPERATION-SECTION image (0x5d290: u32 op_count @+0, +/* Host-authored OPERATION-SECTION image (u32 op_count @+0, * <= 0x80; then op_count 0x40c-byte operation records at +4; the * section KEY is an offset past the array, >= op_count*0x40c + 4). * Operation record fields decoded: +0x00 u32 type <= 4; +0x04 u16 @@ -731,8 +731,8 @@ static inline size_t ane_opsec_size(u32 ops) return 4 + (size_t)ops * ANE_OPSEC_OP_REC_SIZE; } -/* Submit one CSNE command block on the INIT (EP1) ring: K14 - * rtbuddyEndpointSendMessage semantics (W2 §3) — slot alloc with wrap, +/* Submit one CSNE command block on the INIT (EP1) ring: RTBuddy + * endpoint send semantics (W2 §3) — slot alloc with wrap, * memcpy into the ring, 54-bit doorbell word, cursor * advanced only on doorbell success. No synchronous response matching: * fw->host responses arrive on the T2F* channels and are not walked @@ -747,8 +747,8 @@ void ane_t6021_csne_ping_attempt(struct ane_t6021 *ane); /* W13/W14 firmware loader (ane_t6021_fwload.c): validate + stage + * dart-ane0-map the 13.5 PRELOAD payload behind fw_load=1. This is - * the staging half of the boot contract: the Params+0x18 producer - * chain and the RVBAR fold are closed (mapper-callchain audit, commits + * the staging half of the boot contract: the fw DVA source + * and the RVBAR fold are closed (audit commits * 3762aee/12be074); ane_t6021_boot.c consumes the staged surface. */ int ane_t6021_fwload_probe(struct ane_t6021 *ane); @@ -768,7 +768,7 @@ struct ane_asc_tunables; * for this SoC (T6021 only), so fw_alias_reserved=1 may map it; the * other SoCs always run the firmware from driver-owned memory. * ps_cpu_off: the ANE CPU ps word in the DT "pmgr" window (probe guard). - * pwgate_off: the kext's PWGATE word in the DT "set" window, checked + * pwgate_off: the PWGATE word in the DT "set" window, checked * open before any engine read; 0 = not checked. * pmu_pa: the page of the seven ANE ps words; the firmware's power * service writes them through its DART at IOVA == PA. ps_off: the diff --git a/drivers/accel/ane/ane_t6021_boot.c b/drivers/accel/ane/ane_t6021_boot.c index ec2bb466133c4..ccb87ec65b2ff 100644 --- a/drivers/accel/ane/ane_t6021_boot.c +++ b/drivers/accel/ane/ane_t6021_boot.c @@ -14,25 +14,25 @@ * * Blocked boot writes, with the blocking prerequisite each one waits on: * - RVBAR entry fold (ENTRY_BASE | fw DVA & ADDR_MASK; single u64 - * store, width_proof 0x95e9860→0x9622b0c read / 0x95e9988→0x9622c94 - * write): semantics of the observed RVBAR read 0x1 are unresolved + * store; the read and the write are both 64-bit, per width_proof): + * semantics of the observed RVBAR read 0x1 are unresolved * (see the rvbar lifecycle fork) AND the pre-CPU block * below applies regardless. * - CPU_CONTROL write32 0 then 0x10 (rvbar-width local order): waits * on the same pre-CPU block; releasing RUN with an unresolved * entry state is the exact blind start the lane forbids. * - SCRATCH0/1 publication + SCRATCH7 wake: init-structure fields - * [0x08] and [0x10,0x38) have no pinned selene fn 0x71A4 consumer + * [0x08] and [0x10,0x38) have no pinned firmware consumer * read-set; zero-filling unknown fields and calling the structure * valid is forbidden (Main 2026-09-20). * * Pre-CPU prerequisites, updated for pass4 (receipts 2026-09-20 - * mapper-callchain pass4 + tools/check_power_anchors.py 33/33, commit + * pass4 + tools/check_power_anchors.py 33/33, commit * 04630ef): - * RESOLVED (pass4): the CPU-power branch selector — H14g SoCConfig - * zeroes dev+0x3EA0 (0x9613e14, the block that also sets - * dev+0x4A0=0x01400044), so EnableCPUClocksAndPower takes the - * PLAIN branch: write32(PMGR+0x2e0, 0x0000000f), validate 0xff — + * RESOLVED (pass4): the CPU-power branch — the h14g configuration + * selects the PLAIN branch (the same configuration that puts + * CPU_CONTROL at 0x01400044), so the CPU island is powered with + * write32(PMGR+0x2e0, 0x0000000f), validate 0xff — * AUTO_ENABLE bit28 never set. On Linux this END STATE is exactly * what the first_resume stage-2/3 gate verifies (act=0xf, * AUTO_ENABLE clear) before any of this code runs — the write is @@ -41,29 +41,29 @@ * equivalence check. * STILL OPEN (each blocks its write below): * 1. CLOSED (pass5 c364f24): the pre-CPU engine table writels - * eng+0xb38/0xb98/0xbf8 <- 0x01ff01ff are REQUIRED every - * EnableANEClocksAndPower — dev+0x784 bit0 has no writer, so the - * always-open gate runs the loop unconditionally; the + * eng+0xb38/0xb98/0xbf8 <- 0x01ff01ff are REQUIRED every time + * the ANE clocks and power are enabled — they are written + * unconditionally, before the CPU start; the * order-vs-start question is moot. Receiver PROVEN pass4 - * (dev+0x188 accessor = engine reg[0]); count = config+0x150 = + * (the engine register window, reg[0]); count = * 3. Armed below; the first live firing must ride netconsole * behind the seam logs. - * 2. Provider first-enable enableDeviceClock(1,dev+0x8F0)/ - * enableDevicePower(1,&out,dev+0x8F0) (0x95d1d48/0x95d1d90): - * internal gate-ID arrays are runtime-populated (open). Linux + * 2. Provider first-enable of the device clock and power: the + * set of clock and power gates it enables is only known at run + * time (open). Linux * substitutes the genpd raise + supplier links (islands verified * ACTUAL=0xf by the gate); the equivalence datum is the open * half. * 3. RVBAR lifecycle — RESOLVED (M2ResetLifecycle, * rvbar-lifecycle-evidence.json, anchors 38/38 * check_rvbar_lifecycle_anchors.py): live bit0-set + entry 0 is - * the STALE state; the kext-lawful transition is power_off - * (ANE_deInit when dev+0x3FB, then PMGR ps 0x2e0 ← 0x00000000 via - * the dev+0x190 PMGR accessor) → power_on → re-init, NEVER an - * RVBAR write while bit0 is set. RVBAR (engine accessor, + * the STALE state; the valid transition is power_off + * (de-init, then PMGR ps 0x2e0 ← 0x00000000 through the PMGR + * window) → power_on → re-init, NEVER an + * RVBAR write while bit0 is set. RVBAR (engine window, * 0x1050000, u64) is written only after a read64 with bit0 == 0; * CPU_CONTROL write32 0 then 0x10 runs on BOTH paths; success = - * SCRATCH7 (dev+0x454 selector) read32 == 0x08042006. Open edge: + * SCRATCH7 read32 == 0x08042006. Open edge: * whether ps-off clears bit0 — one live read-only read64 after a * domain-off answers it. IMPLEMENTATION CAVEAT: the ps-word write * is the class that froze this host from kernel context @@ -81,37 +81,37 @@ * poll A; then wake 0xf7fbdff9 -> SCRATCH7 releases the fw; poll * B awaits SCRATCH7 DONE 0x08042006 (booted) and reads back * SCRATCH0/1. Because of the pulse, a post-CPU READY is - * unambiguous — no stale-ack hazard. The SCRATCH0 marker (fn - * 0x86EC) is not a gate. - * Selene raw anchors: READY write idx7 @0x7360-0x7388 (movz - * 0x2006 @0x7364, mov w1,#7 @0x7368, movk 0x804 @0x736c, slot - * byte48 @0x7380, salt 0x5bdd @0x7384, blraa @0x7388); poll loop - * READ32 idx7 @0x73c4-0x73e4, cmp wake 0xf7fbdff9 @0x73e8, - * sleep(1000) @0x73f8-0x73fc; DONE ack @0x77cc-0x77f0. + * unambiguous — no stale-ack hazard. The SCRATCH0 marker + * is not a gate. + * On the firmware side: it writes READY (0x08042006) to + * SCRATCH7, then polls SCRATCH7 every 1 ms until it reads the + * wake word 0xf7fbdff9, and writes DONE (0x08042006 again) once + * it has booted; the host's poll B waits for that second + * 0x08042006. * 4. Linux allocation map (legacy branch sizes): FWIM surface = - * config+0x138 byte-count = 0x500000 (Main, audit 751caa4); + * 0x500000 bytes (Main, audit 751caa4); * Linux fw_buf is ANE_FW_BUF_SIZE = 0x500000 — the semantic is * honored directly, covering the image vmsize 0x36c000. 'IPC ' - * surface = min(config+4, dev+0x3A70 - * cap) — the numeric cap needs the h14g config blob field map + * surface = a configured size, capped + * — the numeric cap needs the h14g config field map * (open prerequisite). Init suballoc = 0x174 from a Linux-owned - * pool standing in for the kext dev+0x968/dev+0x980 pool (pool + * pool (pool * total size open). RTBuddy FW_INIT sizes (64K…) are the OTHER * branch — never used here. * 5. Init publication: the fw consumes [0x08]..[0x68] (pass5) and * the Linux sources of those fields are not pinned ([0x08] = - * *(dev+0x988+0x18) = the 'IPC ' surface DVA — the surface + * the 'IPC ' surface DVA — the surface * itself is CLOSED (pass5c/5d), its Linux allocation size open * per item 4; [0x10]/[0x18] = config-size terms with - * config+0x138 = image byte-count closed; [0x30] = dev+0x990 + * the image byte-count closed; [0x30] = the * load-progress word) — publication cannot fire until every * fw-read field has a pinned Linux source. * * Read whitelist — every MMIO read below, individually: * - RVBAR eng+0x01050000: read64 width-proven (rvbar-width * width_proof); W8 and W10 live Linux reads, value 0x1 each. - * - CPU_STATUS eng+0x1400048: phase-1 S2 whitelist; kext poll site - * 0x…95ecfb4 (config field dev+0x49c = 0x1400048); W10 live read + * - CPU_STATUS eng+0x1400048: phase-1 S2 whitelist; polled while + * the CPU starts; W10 live read * 0x2a. * - ASC mailbox controls eng+0x1408110/0x1408114: W10 live reads * (both 0x00020001; 600 polls / 30 s, zero changes, zero aborts). @@ -166,23 +166,23 @@ MODULE_PARM_DESC(boot_prevent_nap, * every probe and the driver performs no direct kernel PMGR writes. */ static const bool pf_provider_genpd_strategy = true; -static const bool pf_pool_word0_proven = true; /* CONFIRMED (Main raw + - * Reset 183fd50, chain - * 0x67dc/0x6824/0x68d8/ - * 0x70dc/0x736c): DDM Params +static const bool pf_pool_word0_proven = true; /* CONFIRMED (Reset + * 183fd50, traced through + * the firmware's pool + * setup): DDM Params * word0 = requested bytes = * 0x40000 — sourced, not * synthesized */ static const bool pf_heap_floor_pinned = true; /* CLOSED: floor = - * getPageSize() return = - * DART page size 0x4000 - * (store chain 0x9602e68-90 - * + dart-ane0 page-size - * 0x4000 node, pass6h); + * the DART page size = + * 0x4000 (the dart-ane0 + * node's page-size + * property is 0x4000, + * pass6h); * sourced in boot_sources */ -static const bool pf_dart_page_floor = true; /* CLOSED (Main raw, +static const bool pf_dart_page_floor = true; /* CLOSED ( * pass6h + exact-node * linkage): dtree-j414c.txt * sha256 dd2955…d504, ane0 @@ -207,7 +207,7 @@ static const bool pf_pass6_init_contract = true; /* cd25b46, 87/87 */ */ /* Table-block mode selection (2026-09-20 16:23:07 wedge): * mode 0 = ABORT before any write (accidental-repeat prevention), - * mode 1 = write the table (kext-faithful; selected per-run via the + * mode 1 = write the table (as on macOS; selected per-run via the * rtclient fw_start_table_mode param), * mode 2 = SKIP the table (default: the shipped diagnostic). * W8 write-grant tunables are mode-independent (proven no-abort @@ -230,12 +230,12 @@ static bool ane_t6021_boot_preflight_complete(void) } /* STATIC boot sources — populated per the pinned contract (Main - * authorization 2026-09-20). Object layout (Main raw correction, - * superseding audit 018abdb): the OSValueObject at +0x10 POINTS to - * the Params; Params+0x00 = size, +0x18 = DVA, +0x38 = hostVA. - * dev+0x980 is the 'DDM ' Params pool (size 0x40000); dev+0x968 is a - * SEPARATE DMM manager constructed over the pool hostVA — never - * conflate the two. Runtime fields (fw_dva, ipc_dva, pool_dma) are + * authorization 2026-09-20). The 'DDM ' pool (size 0x40000) and the + * memory manager built on top of it are separate objects, which + * audit 018abdb had confused; the pool's first word is its + * requested size (see pool_word0 below). + * + * Runtime fields (fw_dva, ipc_dva, pool_dma) are * filled by ane_t6021_boot_prepare() at sequence time (after poll A, * per "dynamic allocations occur after READY"); heap_floor stays 0 * here and is supplied together with pf_heap_floor_pinned — NEVER @@ -243,12 +243,12 @@ static bool ane_t6021_boot_preflight_complete(void) * then). */ static const struct ane_t6021_init_sources boot_sources = { - .cfg_size = 0x500000, /* config+0x138 (0x9613da8/ dac) */ + .cfg_size = 0x500000, /* FWIM surface size */ .prev_fw_len = 0, /* first boot; static per reload */ - .heap_floor = 0x4000, /* DART page size: dev+0x3A90 = - * getPageSize() return; node - * authority dart-ane0 page-size - * 0x4000 (exact linkage check + .heap_floor = 0x4000, /* DART page size; node + * authority: the dart-ane0 + * page-size property, 0x4000 + * (exact linkage check * passing) */ .pool_word0 = 0x40000, /* DDM Params word0 = requested @@ -256,8 +256,8 @@ static const struct ane_t6021_init_sources boot_sources = { */ }; -/* Poll A/B bound: the kext polls <=1000 x sleep(1ms) (selene poll - * loop 0x73c4-0x73fc analog; rvbar-lifecycle step 6/10). +/* Poll A/B bound: up to 1000 polls with a 1 ms sleep, the same bound + * the firmware's own handshake loop uses (rvbar-lifecycle step 6/10). */ #define ANE_BOOT_ACK_POLL_US 1000 #define ANE_BOOT_POLL_MS 1000 @@ -310,7 +310,7 @@ static void ane_boot_publish_barrier(void *ctx) { (void)ctx; /* dma_wmb() = dmb oshst on arm64: orders the coherent pool fill - * before the device publish. ORDERING guarantee — not the kext's + * before the device publish. ORDERING guarantee — not a * dsb st (completion). Sufficient for the Linux coherent-DMA + * writel doorbell contract (writel orders prior accesses before * the MMIO store). @@ -321,7 +321,7 @@ static void ane_boot_publish_barrier(void *ctx) static void ane_boot_wait(void *ctx) { (void)ctx; - usleep_range(1000, 1500); /* kext poll: sleep(1000us) */ + usleep_range(1000, 1500); /* 1 ms between polls */ } static void ane_boot_phase(void *ctx, const char *what) @@ -385,7 +385,7 @@ static int ane_t6021_boot_prepare(void *ctx, u32 *lo, u32 *hi) return -ENODATA; /* belt: run() already gated */ /* dynamic reads (post-READY, live cells — never hardcoded): - * read order SCRATCH0 then SCRATCH1 (0x95ea0d8/0x95ea100); + * read order SCRATCH0 then SCRATCH1; * SCRATCH0 >= 0x21 refuses before allocations/publication; * SCRATCH3 = fw extra-heap request; SCRATCH1+1 = ordinal. */ @@ -435,8 +435,8 @@ static int ane_t6021_boot_prepare(void *ctx, u32 *lo, u32 *hi) * 2026-09-25 handshake receipt; a static tick separates "ASC domain * unclocked" from "clocked but silent"). * - * Deliberately NOT read: engine+0x1140008 (the 13.5 stub's VM 0x30c - * store target). Its readability, width and reset value are unproven + * Deliberately NOT read: engine+0x1140008 (a word the 13.5 stub + * writes). Its readability, width and reset value are unproven * and no pre-release baseline exists, so an after-value would not * locate PC (could be preexisting iBoot state, write-only, or * normalized). diff --git a/drivers/accel/ane/ane_t6021_boot.h b/drivers/accel/ane/ane_t6021_boot.h index aba267e453f13..0677dd046172e 100644 --- a/drivers/accel/ane/ane_t6021_boot.h +++ b/drivers/accel/ane/ane_t6021_boot.h @@ -3,29 +3,29 @@ * ane_t6021_boot.h — T6021 legacy-boot contract units (pure, shared by * the kernel sequencer and the userspace regression). * - * Byte anchors (KC kernelcache.release.mac14j sha256 8304156f…, selene - * sha256 9f7915c4…; receipts under ane-linux-experiments/receipts/, - * commits 3762aee + 12be074): - * - RVBAR fold: ANEHWDevice::ANE_Init 0x95e9850–0x95e9988 (K13 - * 0x93203b0–0x93204e0 identical). Read64 of eng+0x01050000; bit0 + * Evidence: live reads and traces under Linux, plus the receipts in + * ane-linux-experiments/receipts/ (commits 3762aee + 12be074), for + * the firmware image with sha256 9f7915c4…. + * + * - RVBAR fold: read64 of eng+0x01050000; bit0 * set skips the write (live jw14m2 read 0x1). Otherwise write64 * ENTRY_BASE | (fw DVA & ADDR_MASK) — single u64 store * (2026-09-20-h14-rvbar-width.json width_proof). - * - fw DVA = ANESharedMemorySurfaceParams("FWIM")+0x18, produced by - * dartMapMemoryDescriptor (vtable slot 0x8a8, genIOVMSegments) for - * the out-param installed at start+0x336c (mapper-callchain audit - * pass1–3). Linux correspondence: dma_alloc_coherent on the ane + * - fw DVA = the device address of the staged firmware ("FWIM") + * surface, mapped through the ANE DART; the RVBAR fold above starts + * the ASC at this address. + * Linux correspondence: dma_alloc_coherent on the ane * platform device is dart-ane0-translated (live iommu group 6, * three apple,t6020-dart streams, DMA domain). * - SCRATCH0/1 u64 split: low32 → 0x01840048 first, high32 → - * 0x0184004c, dsb st before (legacy-init-publication receipt, - * accessor write32 width-proven at 0x9622d4c). + * 0x0184004c, ordered after the writes that fill the published + * data (legacy-init-publication receipt; each half is a write32). * - 0x174 init suballocation: header[0x00] u64 = fw DVA (legacy - * branch, dev+0x780 bit0 clear = observed AppleARMIODevice - * provider), [0x68] u32 = 64 count, template 256 B at - * [0x6C,0x16C) with template+0xC0 = 4 RESOLVED (Main raw anchors - * 0x9612b78/7c/80 on the dev+0x998 template; conditional |=0x10 - * separate, unqualified, not set). + * branch, the one taken on this provider), [0x68] u32 = 64 count, + * template 256 B at [0x6C,0x16C) with template+0xC0 = 4 RESOLVED + * (the template's initial state); the conditional |=0x10 is + * separate, unqualified and not set. The rest of the block is + * zeroed by the caller (see ane_t6021_init_struct_fill). * * This header intentionally includes nothing. Kernel consumers have * already; userspace consumers typedef u8/u32/u64 and @@ -35,7 +35,7 @@ #ifndef __ANE_T6021_BOOT_H__ #define __ANE_T6021_BOOT_H__ -/* RVBAR entry fold (ANE_Init 0x95e9868–0x95e988c). The mask clears +/* RVBAR entry fold (single u64 store, see above). The mask clears * bits 0–10, 48 and 55 — bit 11 IS retained (0xf800 keeps 0x800); the * base supplies bit0 and the fixed upper pattern. */ @@ -47,10 +47,10 @@ */ #define ANE_T6021_CPU_RUN_RELEASE 0x10 -/* SCRATCH7 handshake words (InitializeRTBuddy / legacy ANE_Init - * publication; selene ack writer 0x77c8). +/* SCRATCH7 handshake words (RTBuddy and legacy init publication; the + * firmware writes the ack). */ -/* SCRATCH0 request band (pass9-corrected, Main raw trace): post-READY +/* SCRATCH0 request band (pass9-corrected): post-READY * SCRATCH0 >= 0x21 REFUSES the boot preparation; below 0x21 proceeds * to the MAX/alloc path. */ @@ -59,7 +59,7 @@ #define ANE_T6021_BOOT_WAKE_REQ 0xf7fbdff9U #define ANE_T6021_BOOT_ACK 0x08042006U -/* Init suballocation geometry (ANE_Init 0x95ea710–0x95ea97c; +/* Init suballocation geometry (legacy init path; * legacy-init-publication receipt). */ #define ANE_T6021_INIT_STRUCT_SIZE 0x174 @@ -69,8 +69,8 @@ #define ANE_T6021_INIT_TEMPLATE_OFF 0x6c /* 256 B at [0x6C,0x16C) */ #define ANE_T6021_INIT_TEMPLATE_SIZE 0x100 #define ANE_T6021_INIT_TBIT_OFF 0xc0 /* template-relative */ -/* RESOLVED (Main raw anchors 0x9612b78/7c/80: ldr/orr#4/str on the - * dev+0x998 template); the conditional |=0x10 is a separate, still +/* RESOLVED (the init path ORs 4 into this template word, whose + * initial value is zero); the conditional |=0x10 is a separate, still * unqualified mutation and is NOT set. */ #define ANE_T6021_INIT_TBIT_VAL 0x4 @@ -80,8 +80,8 @@ static inline u64 ane_t6021_rvbar_compose(u64 iova) return ANE_T6021_RVBAR_ENTRY_BASE | (iova & ANE_T6021_RVBAR_ADDR_MASK); } -/* The kext skip condition: bit0 of the RVBAR read (tbnz w0,#0 at - * ANE_Init 0x95e9878; live jw14m2 reads have been 0x1). What a set +/* The skip condition: bit0 of the RVBAR read (a set bit skips the + * fold; live jw14m2 reads have been 0x1). What a set * bit names in boot-mode terms is the open lifecycle fork — see * ane_t6021_boot.c; this predicate only decodes the bit. */ @@ -125,14 +125,14 @@ static inline u64 ane_t6021_scratch64_join(u32 lo, u32 hi) * 68/68). EVERY fw-read field is explicitly sourced from the struct * below — nothing is silently zero: [0x00] FWIM DVA, [0x08] 'IPC ' * surface DVA (Linux alloc, size = ane_t6021_ipc_size), [0x10] - * config+0x138 = 0x500000, [0x18] = 0x10000000 - size, [0x20] obj2 - * DVA, [0x28] w22, [0x30] fw-load progress, [0x50] zext [x23+4], + * cfg_size = 0x500000, [0x18] = 0x10000000 - size, [0x20] heap + * DVA, [0x28] heap size, [0x30] fw-load progress, [0x50] boot ordinal, * [0x58] pool DMA base, [0x60] pool word0, [0x68] count 64, * template[0x00] = 0, template+0xC0 = 4. The struct values pin BEFORE * the preflight gate can open (Main: no unresolved field silently * zero; no partial boot). The caller MUST zero the full 0x174 block - * first (coherent alloc): gaps [0x34..37]/[0x64..67] get no kext - * store and sit inside fw-read u64 units. + * first (coherent alloc): gaps [0x34..37]/[0x64..67] are never + * written and sit inside fw-read u64 units. */ /* Init-suballocation contract — pass6 producer semantics * (rvbar-lifecycle-evidence.json pass6_init_header_producers, commit @@ -142,25 +142,25 @@ static inline u64 ane_t6021_scratch64_join(u32 lo, u32 hi) * fw_dva -> [0x00] staged selene ('FWIM') surface DVA, * ipc_dva -> [0x08] 'IPC ' surface DVA (its size calc remains * the audit lane's open item), - * cfg_size -> [0x10] config+0x138 = 0x500000; [0x18] = + * cfg_size -> [0x10] = 0x500000; [0x18] = * 0x10000000 - cfg_size = 0x0fb00000, * prev_fw_len -> [0x30] previous fw image length (0 on first boot; * driver-static, updated on each reload), - * heap_floor -> [0x28] floor = config field dev+0x3A90 (value + * heap_floor -> [0x28] floor = the DART page size (value * pinned by the audit lane at preflight close), * pool_dma -> [0x58] the 'DDM ' pool surface DVA (Linux allocates - * its own 0x40000-byte DDM pool — the kext size), + * its own 0x40000-byte DDM pool — the size macOS uses), * pool_word0 -> [0x60] first qword of the Linux pool-descriptor * struct. NOT CLOSED: the fw-side consumer of - * pool+0x00 is untraced and the kext never stores it + * pool+0x00 is untraced and the host never stores it * (free-list head at manager+0 untouched by * Allocate). This value is a HARD-GATED source — the * sequence cannot run until a producer/consumer proof * pins it; no hypothesis value is shipped. * PRODUCER OPEN (Main, superseding b21e23f): the * "pool+0x00 = manager free-list head" claim is - * REJECTED — manager dev+0x968 != Params pool - * dev+0x980 (wrong-object alias likely), and + * REJECTED — the pool manager and the Params pool + * are different objects (wrong-object alias likely), and * Reset6cb048d shows the consumer stores word0 * VERBATIM at image ops 68 (possible extent use). * Wait for the exact same-Params producer. @@ -187,24 +187,24 @@ struct ane_t6021_init_sources { }; /* 'IPC ' surface size (CURRENT_CONTRACT f9602e0; operands proven: - * u64[dev+0x3A90] = IODARTMapper::getPageSize() — 0x4000, one 16K - * DART page, typical; u32[x23+4] = captured SCRATCH1 + 1, the boot + * page_size = the DART page size — 0x4000, one 16K + * DART page, typical; ordinal = captured SCRATCH1 + 1, the boot * ordinal): size = max(page_size, ordinal + 1). Used by the kernel * prepare hook to size the 'IPC ' allocation. */ /* SCRATCH0 read-order note (pass9 phase-1: SCRATCH0 then SCRATCH1 - * after READY): Main raw branch proof SUPERSEDES the audit's - * "SCRATCH0 >= 0x21 arms IPC" reading — 0x95ea1dc b.cc(low) branches - * TO the MAX+IPC-alloc path and >=0x21 goes to logging/error, so + * after READY): later evidence SUPERSEDES the audit's + * "SCRATCH0 >= 0x21 arms IPC" reading — a value below 0x21 leads + * to the MAX+IPC-alloc path and >=0x21 goes to logging/error, so * scratch0 < 33 is likely a VALIDITY gate, not an optional-IPC - * condition. Header [0x08] unconditionally dereferences - * dev+0x988+0x18. Allocation stays UNCONDITIONAL until Reset corrects + * condition. Header [0x08] is always filled with the IPC surface + * DVA. Allocation stays UNCONDITIONAL until Reset corrects * the source; the threshold is NOT implemented. */ static inline u64 ane_t6021_ipc_size(u64 page_size, u32 captured) { /* EXACT u32 semantics: the fw-visible ordinal is the captured - * SCRATCH1 value + 1 in 32-bit arithmetic (kext add w8,w0,#1) — + * SCRATCH1 value + 1 in 32-bit arithmetic (as the firmware sees it) — * it wraps. Callers pass the CAPTURED value, never pre-incremented. * The result is then bounded by ANE_T6021_BOOT_IPC_CEILING at the * allocation site: captured is fw-supplied and untrusted. @@ -250,7 +250,7 @@ ane_t6021_heap_size(u32 scratch3_req, u64 floor, u64 max_size) /* Fill the COMPLETE init suballocation from the static sources plus * the post-READY dynamic values. The caller MUST zero the full 0x174 * block first (coherent alloc): the gap bytes [0x34..37]/[0x64..67] - * get no kext store and sit inside fw-read u64 units. + * are never written and sit inside fw-read u64 units. */ static inline void ane_t6021_init_struct_fill(u8 *buf, const struct ane_t6021_init_sources *s, @@ -290,7 +290,7 @@ ane_t6021_init_struct_fill(u8 *buf, const struct ane_t6021_init_sources *s, buf[ANE_T6021_INIT_COUNT_OFF + 2] = 0; buf[ANE_T6021_INIT_COUNT_OFF + 3] = 0; - /* template[0x00] = config word [dev+0x1D8], pinned value 0 + /* template[0x00] = a config word, pinned value 0 * (pass5d); [0x04..0x0B] zero via the caller's zeroing. */ buf[ANE_T6021_INIT_TEMPLATE_OFF + 0] = 0; @@ -298,8 +298,8 @@ ane_t6021_init_struct_fill(u8 *buf, const struct ane_t6021_init_sources *s, buf[ANE_T6021_INIT_TEMPLATE_OFF + 2] = 0; buf[ANE_T6021_INIT_TEMPLATE_OFF + 3] = 0; - /* template+0xC0 = 4: RESOLVED initial state (Main raw anchors - * 0x9612b78/7c/80). The conditional |=0x10 remains unset. + /* template+0xC0 = 4: RESOLVED initial state (see + * ANE_T6021_INIT_TBIT_VAL). The conditional |=0x10 remains unset. */ buf[ANE_T6021_INIT_TEMPLATE_OFF + ANE_T6021_INIT_TBIT_OFF + 0] = ANE_T6021_INIT_TBIT_VAL; @@ -345,10 +345,10 @@ ane_t6021_boot_prepare_publish(const struct ane_t6021_init_sources *s, ((u64 *)a)[i] = 0; /* SCRATCH0 band check — REFUSAL before allocations/publication - * (Main raw trace: >= 0x21 -> 0x95ea2a0 log -> 0x2c4 -> 0x3330 - * error, w21 = 0xE00002DA; < 0x21 -> 0x214 MAX -> 0x25c alloc -> - * 0x260 -> 0x2c8). Boundary: 0x20 accepted, 0x21 refused. - * Read order: SCRATCH0 first, then SCRATCH1 (0x95ea0d8/0x95ea100). + * (>= 0x21 is logged and fails the boot with an error; < 0x21 + * goes on to the MAX sizing and the allocations). + * Boundary: 0x20 accepted, 0x21 refused. + * Read order: SCRATCH0 first, then SCRATCH1. * Untrusted-input sizing after the band check; heap_size() * returns SIGNED (negative = refused) — kept local so a refusal * cannot wrap into the u64 record. @@ -380,8 +380,8 @@ ane_t6021_boot_prepare_publish(const struct ane_t6021_init_sources *s, return -ENOMEM; } - /* ordinal: exact u32 +1 on the READ SCRATCH1 value (kext add - * w8,w0,#1; callers pass the read value, never pre-incremented). + /* ordinal: exact u32 +1 on the READ SCRATCH1 value (wrapping; + * callers pass the read value, never pre-incremented). * HEADER SOURCES come from the ALLOCATIONS: s->ipc_dva/pool_dma * may be zero in the caller's static sources — fill from the * allocator outputs (a->ipc_dva/a->pool_dva), never from s @@ -477,11 +477,11 @@ struct ane_t6021_boot_cfg { * after the table-base analysis), * 2 = SKIP the table block, run the rest * (diagnostic: tests fw-alive without - * the kext pre-CPU config). + * the pre-CPU table config). */ u64 fw_dva; /* staged surface DVA (fold input) */ int rtb_mode; /* 1 = S1 writes SCRATCH6=0 (RTBuddy/RTKit-app- - * endpoint select, fw 0x42c8) instead of 1 + * endpoint select) instead of 1 * (legacy ChMan/MBI). In RTBuddy mode the fw * may skip READY/DONE; listen on mailbox. * Run contract: P4 READY is recorded, not @@ -544,7 +544,7 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, return -ENODATA; /* pre-CPU engine table (pass4 receiver, pass5 gate: REQUIRED - * every EnableANEClocksAndPower; residual alias/indirect writer + * every clock/power enable; residual alias/indirect writer * risk documented in the preflight list). * LIVE-FAULT 2026-09-20: this block wedged the machine (P0 then * silence — stall could be the FIRST write; P1 never reached; @@ -571,7 +571,7 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, case 2: default: /* diagnostic skip: NO table writes — tests fw-alive - * without the kext pre-CPU config (user override + * without the pre-CPU table config (user override * 2026-09-20). */ io->phase(io->ctx, "P0 table SKIPPED (diagnostic)"); @@ -619,7 +619,7 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, return -ECANCELED; /* tunables done, nothing else fired */ io->phase(io->ctx, "P1 scratch-clear+pulse"); - /* S1: InitANEScratchRegisters — clear ALL cells, SCRATCH6 = rtb_mode ? 0 (RTBuddy) : 1 + /* S1: scratch init — clear ALL cells, SCRATCH6 = rtb_mode ? 0 (RTBuddy) : 1 * (legacy ChMan/MBI); pulse SCRATCH7 1 -> 0 (stale cleared). */ for (i = 0; i < 8; i++) @@ -656,11 +656,11 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, io->phase(io->ctx, "P4 pollA-READY"); /* S4: poll A — FRESH READY (the pulse made it unambiguous). - * RTBuddy mode (SCRATCH6=0, kext fw 0x42c8: rtbuddyFW when - * SCRATCH6 == 0): the fw does not publish BOOT_ACK to SCRATCH7 + * RTBuddy mode (SCRATCH6=0; the firmware runs in RTBuddy mode + * when SCRATCH6 == 0): the fw does not publish BOOT_ACK to SCRATCH7 * and no legacy publication follows — aliveness is gated by the - * RTKit HELLO after boot_run returns (selene InitializeRTBuddy - * 0x95ead04). A missing READY is recorded, not fatal, and + * RTKit HELLO after boot_run returns. + * A missing READY is recorded, not fatal, and * *fw_alive stays 0 — never fabricated. */ for (i = 0; i < ANE_T6021_BOOT_TABLE_POLLS; i++) { @@ -689,11 +689,11 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, * publish barrier, then publish suballoc DVA low32/high32, then * wake. Barrier semantics (Main review 2026-09-20): the kernel * backend's dma_wmb() is dmb oshst — it ORDERS the coherent fill - * before the device publish; it is NOT the kext's dsb st (which + * before the device publish; it is NOT a dsb st (which * additionally waits for completion). Ordering suffices for the * Linux coherent-DMA + writel doorbell contract (writel itself - * orders prior accesses before the MMIO store); the kext dsb st - * choice is a completion guarantee this driver does not claim. + * orders prior accesses before the MMIO store); a dsb st would add + * a completion guarantee this driver does not claim. */ { u32 lo = 0, hi = 0; @@ -725,7 +725,7 @@ ane_t6021_boot_run(const struct ane_t6021_boot_io *io, /* HANDSHAKE COMPLETE != firmware result success: the DONE ack is * the fw's handshake word; the SCRATCH0/1 result u64 it leaves * behind has UNSOURCED success semantics (pass6: the fw writes a - * "result" at 0x77a0 whose meaning is not decoded). Expose it + * "result" whose meaning is not decoded). Expose it * raw; never infer success from booted alone. */ *scratch_result = diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index 44be0d3e72ecc..59688070ada62 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -34,7 +34,7 @@ * RVBAR entry on the device's default DMA domain. Placement * contract, receipt receipts/2026-09-20-t6021-entry-alias.md: * - live RVBAR read64 = 0x10000000001 (bit0 latched, entry - * bits 0x10000000000 = dart-ane vm base); kext law + * bits 0x10000000000 = dart-ane vm base); lifecycle rule * (rvbar-lifecycle 6288b0b): bit0 set => the RVBAR write * branch is skipped, so the ASC fetches AT the latched entry; * - netconsole 2026-09-20T17:48:19: apple-dart 285800000.iommu @@ -193,14 +193,14 @@ bool ane_t6021_fwload_options_ok(void) * die-0 addresses (13.5 ADTs, t602x-ane.dtsi). * * pmu_pa: the firmware's power service programs the seven ANE ps words - * through its DART at IOVA == PA (SetPMUBaseAddress stores the ANE_TD + * through its DART at IOVA == PA (the PMU base it is given is the ANE_TD * word: selene 0x28e084008, bia 0x23b70c010, receipts/2026-10-01-t8112-kit). * macOS maps the page first; without it the first access faults (NO PMD * FOR IOVA 0x28e084008, 2026-09-29) and the firmware halts. * * T8112 (receipts/2026-10-01-t8112-optin): the revision comes from the * eFuse words iBoot reads, the ANE_SYS_CPU word is pmgr +0xc008, and the - * kext (type 0x70) opens PWGATE "set" +0x8b8 before the ps words. The TM + * macOS (type 0x70) opens PWGATE "set" +0x8b8 before the ps words. The TM * TD word is not known there, so trace_td is off. */ const struct ane_t602x_soc ane_t6020_soc = { diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 90bb89339486b..eb4ae9e383e23 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -103,8 +103,8 @@ #include /* Doorbell (IPI) block, engine + 0x1844000: set +0, pending +0x8000, - * ack +0xc000 (kext aneInterruptHandler reads +0x184c000 and writes - * +0x1850000 with no SoC branch, receipts/2026-10-01-t8112-ane). + * ack +0xc000 (an interrupt is handled by reading +0x184c000 and writing + * +0x1850000, with no SoC difference, receipts/2026-10-01-t8112-ane). */ #define ANE_IPI_OFF 0x1844000 @@ -155,7 +155,7 @@ MODULE_PARM_DESC(start_app_eps, static bool scratch3_ack = true; module_param(scratch3_ack, bool, 0444); MODULE_PARM_DESC(scratch3_ack, - "After DONE, write SCRATCH3 = 0x08042006 (selene 0x7edc, kext 0x95eaee4). Default on; 0 withholds."); + "After DONE, write SCRATCH3 = 0x08042006 (host ack). Default on; 0 withholds."); static bool legacy_only = true; module_param(legacy_only, bool, 0444); @@ -479,7 +479,7 @@ static void ane_rtclient_validate_chman(struct ane_rtclient *ane) /* Legacy ChMan exchange (post-DONE 13.5 transport). Resends to the same * 64-byte IO slot would wedge the ring; cursor advances after each - * completed command (selene fw decodes bit0 = host-owned). + * completed command (the fw treats bit0 as host-owned). */ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, struct ane_legacy_buffer *command, @@ -1672,9 +1672,9 @@ static int ane_t6021_prog_load_ioctl(struct drm_device *drm, void *data, } /* Firmware perf mode: CSNE_CMD_CH_PROPERTY_WRITE (0x1f), channel 0, - * property 0x10aa, value 1. macOS sends it once during power-on (H13 - * kext evidence in the omarchy-ane perf-mode receipt); the selene 13.5 - * fw routes it to CAneEngineExeLoop::setPerfMode. It is a runtime + * property 0x10aa, value 1. macOS sends it once during power-on (see + * the omarchy-ane perf-mode receipt); the 13.5 firmware treats it as + * its engine perf-mode switch. It is a runtime * switch (write 1 to /sys/module/ane_t6021/parameters/fw_perf_mode) so * its effect on call time can be measured on one boot. Only 1 is * accepted: no other value is known to be safe. @@ -2098,7 +2098,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) pm_runtime_disable(dev); return -EPROBE_DEFER; } - /* T8112: the kext (type 0x70) opens PWGATE (bits 29:28 = 0) before + /* T8112: macOS (type 0x70) opens PWGATE (bits 29:28 = 0) before * the ps words. This driver only reads it: an engine read behind a * closed gate is untested, so it refuses first. */ diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 739d4c2fdb1cf..015245f061cca 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -558,8 +558,8 @@ pub(crate) struct Manager { shrink: Shrink, } -/// Native tiler-heap shrink policy, after macOS -/// `AGXParameterManagement::checkForShrink` (AGXG17X 0x8bd5e20). Once per +/// Native tiler-heap shrink policy, matching the heap behaviour seen +/// under macOS. Once per /// period the page high-water folds into an average, /// `max((sample + 4 * average) / 5, sample)`. Each period whose average /// stays below a quarter of the capacity earns one 512 KiB block of shrink, diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index a10e4d2010a10..4e225bb049539 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* J700 firmware-owned Wi-Fi: native transport and cfg80211 station support. - * Clean-room contract: NEO_MT7932_DOWNLOAD_AND_NIC_CAP_CONTRACT.md. + * Covers firmware download and the NIC capability exchange. * All addresses here are BAR0 offsets, never chip/CPU physical addresses. */ #include diff --git a/drivers/net/wireless/mediatek/mt7932/reset.c b/drivers/net/wireless/mediatek/mt7932/reset.c index 0f384dbdaed83..09f3cee068d6c 100644 --- a/drivers/net/wireless/mediatek/mt7932/reset.c +++ b/drivers/net/wireless/mediatek/mt7932/reset.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* J700 MT7932 mapped preparation and function-scoped reset. - * Clean-room contracts: NEO_MT7932_DEVICE_RESET_UNLOAD_CONTRACT.md and - * NEO_MT7932_FLR_PREPARATION_RESOLVED.md. No secondary-bus reset fallback. + * Covers device reset, firmware unload and FLR preparation. + * No secondary-bus reset fallback. */ #include "mt7932.h" diff --git a/drivers/reset/reset-apple-cio.c b/drivers/reset/reset-apple-cio.c index 04a985b7f6b44..1dd1d6df8cc56 100644 --- a/drivers/reset/reset-apple-cio.c +++ b/drivers/reset/reset-apple-cio.c @@ -30,7 +30,7 @@ * "port = slot % 4", which the device tree expresses as one controller node * per die instead. The others index by slot directly. * - * "regmap" is an ApplePMGR::RegMap enum rather than an address. RegMap 0 is + * "regmap" is a PMGR register-map index rather than an address. RegMap 0 is * the PMGR block itself: t8103 resolves to 0x23b784000 and * t6020 to the pmgr node plus 0xa02c, both of which match this driver. The * t8122 and t6030 selectors are unresolved, so they cannot be wired up diff --git a/drivers/soc/apple/image.rs b/drivers/soc/apple/image.rs index 290a9ea43dd3e..c465ffddfbd70 100644 --- a/drivers/soc/apple/image.rs +++ b/drivers/soc/apple/image.rs @@ -60,8 +60,8 @@ static_assert!(HEADER_WIRE == 4 + HEADER_SIZE as usize); pub(crate) enum Version { V1, /// Negotiated from the `0x4d` capability word. Laid out as version 1, but - /// `__payload_hash` (0xfffffe000996b970) digests the header through the - /// trailer (0x…ba68-ba80) where version 1 stops before it (0x…ba84-ba90). + /// the payload hash covers the header through the + /// trailer, where version 1 stops before it. V2, } diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 2e22c78f57d69..229ebcf90c788 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -179,7 +179,7 @@ static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) cpu_control = (u8 __iomem *)mbox->wrapper_regs + APPLE_ASCWRAP_V6_CPU_CONTROL; - /* AppleASCWrapV6::_runCPU(false): clear RUN, reread, clear bit 5. */ + /* Stop: clear RUN, reread, clear bit 5. */ val = readl(cpu_control); writel(val & ~APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); val = readl(cpu_control); @@ -239,7 +239,7 @@ int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) cpu_control = (u8 __iomem *)mbox->wrapper_regs + APPLE_ASCWRAP_V6_CPU_CONTROL; - /* AppleASCWrapV6::_runCPU(true): read, set RUN, write. */ + /* Start: read, set RUN, write. */ val = readl(cpu_control); writel(val | APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); mbox->ascwrap_v6_cpu_running = true; diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 7d8ced489a4a2..82ad95613c4b1 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -111,11 +111,11 @@ pub(crate) struct KeybagCreate { /// Which key-store protocol the enclave speaks. It follows the sepOS the /// firmware hands the SEP, not the SoC alone: the T8103 stub boots macOS 13.5 /// (22G74)'s sepOS, whose request shapes are those of the 13.5 -/// `AppleSEPKeyStore`; the T6020 sepOS is newer. +/// key store; the T6020 sepOS is newer. #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum KeyStore { /// macOS 13.5's key store. Before any other request the endpoint is - /// initialised as `AppleKeyStore::init_sep_endpoint` does it: `0x4d`, then + /// initialised with `0x4d`, then /// `set_env` carrying the ADT's `/defaults` `cpx-encryption-mode`. Sepos13 { cpx_encryption_mode: u32 }, /// The hardware-verified T6020 encoding (variant-5 create). @@ -169,7 +169,7 @@ const T8103: PlatformProfile = PlatformProfile { dart_range_required: false, wide_dma_mask: false, firmware_region: c"sepfw", - // macOS 13.5's AppleKeyStore::identity_create (0xfffffe000994b2d8): + // The 13.5 key store creates an identity bag with // version 2, type 0x400000, parent -1 (it accepts only -1 or <= -10). keybag_create: KeybagCreate { variant: 2, @@ -276,7 +276,7 @@ const T8140: PlatformProfile = PlatformProfile { wide_dma_mask: true, firmware_region: c"sepfw", // J700 hardware refused the strict encoding and accepted this T6020 form - // (linux-aurora commit 18165cf1de, 2026-09-19). + // (first verified on hardware 2026-09-19). keybag_create: KeybagCreate { variant: 5, bag_type: 0, diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 61d6d47769971..2b1562fae288c 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -123,7 +123,7 @@ const SCRD_MAX_CAPTURE: usize = 8; const SCRD_TIMEOUT_MS: time::Msecs = 2000; const SKS_TIMEOUT_MS: time::Msecs = 2000; -/// The 13.5 kext's per-try key-store wait. +/// The per-try wait for the 13.5 key store. const SKS_TIMEOUT_13_MS: time::Msecs = 6000; const SKS_TIMEOUT_PER_KIB_MS: time::Msecs = 6000; diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 2d29fa7598577..f1fabe2791a76 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -164,8 +164,8 @@ impl SepData { } let sized_ms = sks_timeout_for(img.len()); - // The 13.5 key store is waited on as sep_deliver_msg_gated does: - // getAdjustedTimeout(0x1770), 6000 ms per wait (0xfffffe0009945a14). + // The 13.5 key store is given a 0x1770 (6000 ms) wait per delivered + // request. // On a j313 a CREATE took // 1.6 s and a COPY 1.66 s, close enough to the 2 s base that a slow // one would be abandoned mid-way through its xART writes. @@ -313,10 +313,10 @@ impl SepData { self.sks_seal(&op, &body) } - /// `0x2a` set_env, as `AppleKeyStore::set_env(false)` (0xfffffe000994a168) - /// builds it: a 0x40c-byte environment `{1, mode, 4, 0x400 zero bytes}` - /// (0x…a1a0-a1e0) sent by `__ipc_set_env` (0x…9ddc) as struct version 0, - /// u64 1, blob (`_code_ipc_set_env` 0x…4b64). + /// `0x2a` set_env in its "false" form, the environment the key store is + /// given at endpoint init: a 0x40c-byte environment + /// `{1, mode, 4, 0x400 zero bytes}`, sent as struct version 0, u64 1 and + /// the environment as a blob. fn sks_req_set_env(&self, mode: u32) -> Result { let op = crate::sks::sks_set_env(); let mut env = KVec::new(); @@ -331,10 +331,10 @@ impl SepData { self.sks_seal(&op, &body) } - /// `AppleKeyStore::init_sep_endpoint` (0xfffffe0009946cdc) on the 13.5 key - /// store, before any other request: `0x4d`, whose capability word sets the - /// IPC header version to `min(word, 2)` (0x…6dc0-6dd0), then `set_env` - /// (0x…6e64). Returns false if the key store must stay closed this boot. + /// Endpoint init for the 13.5 key store, which must precede any other + /// request: `0x4d`, whose capability word sets the IPC header version to + /// `min(word, 2)`, then `set_env`. + /// Returns false if the key store must stay closed this boot. /// The T6020 path sends nothing here. pub(crate) fn sks_init_endpoint(&self) -> bool { let profile::KeyStore::Sepos13 { @@ -354,7 +354,7 @@ impl SepData { .filter(|body| out.reply.status == 0 && body.len() >= 12) .map(|body| u64::from_le_bytes(body[4..12].try_into().unwrap())); match word { - // Version 0 leaves out the header fields __payload_hash needs. + // Version 0 leaves out the header fields the payload hash covers. Some(0) | None => { dev_err!( self.dev, @@ -396,9 +396,9 @@ impl SepData { true } - /// The designate struct version, which the reply echoes. 13.5's - /// `_code_ipc_make_system_keybag` (0xfffffe000995fd8c) takes version 0 - /// only (0x…fe78). + /// The designate struct version, which the reply echoes. On 13.5, + /// designate uses struct version 0; the other profile uses its own + /// variant. fn sks_designate_variant(&self) -> u32 { match self.profile.key_store { profile::KeyStore::Sepos13 { .. } => crate::sks::SKS_DESIGNATE_VERSION_13, @@ -406,8 +406,8 @@ impl SepData { } } - /// `0x0d` designate. On 13.5 this is `identity_load`'s make-system-keybag - /// (0xfffffe000994ba48-ba5c): source handle, session, an empty blob and + /// `0x0d` designate, which makes the system key bag. On 13.5 the body is: + /// source handle, session, an empty blob and /// nothing after it. fn sks_req_designate(&self, d: &crate::sks::Designation, secret: &[u8]) -> Result { let mut body = image::Body::new(); @@ -494,12 +494,12 @@ impl SepData { } /// `0x18` device-state transition to unlocked, the 13.5 identity-session - /// unlock. `AKSIdentityUnlockSession` (user-client selector 0x7b) reaches - /// `unlock_the_device(client, h, secret, 5)` (0xfffffe000993cee0, called at - /// 0x…15a4), which sends `device_state_transition` with state 0 and flags 0 - /// (0x…cf04-cf0c) through `__ipc_device_state_transition` - /// (0xfffffe0009958204): struct version 0 only (0x…200c), u64 client, u32 - /// handle, u32 state, u64 flags, blob secret. The reply is the version word + /// unlock: `device_state_transition` for the session's handle and secret, + /// with state 0 and flags 0. + /// + /// It uses struct version 0. The body is the version, u64 client, + /// u32 handle, u32 state, u64 flags and the secret as a blob. + /// The reply is the version word /// and two u64. fn sks_req_device_state_unlock( &self, @@ -580,10 +580,10 @@ impl SepData { } // The 13.5 (T8103) and T6020 enclaves encode the identity bag // differently; the per-SoC profile carries the field values (see - // profile::KeybagCreate). On 13.5 this is identity_create's - // __ipc_create_keybag_v2 (0xfffffe0009955ce0): secret, an empty second - // blob and the 16-byte identity UUID, nothing after it; the codec - // (_code_ipc_create_keybag 0xfffffe000995de6c) decodes exact-length. + // profile::KeybagCreate). On 13.5 the identity-bag create carries: + // secret, an empty second + // blob and the 16-byte identity UUID, nothing after it, so the body + // has exactly this length. let enc = &self.profile.keybag_create; let mut body = image::Body::new(); body.put_u32(enc.variant)?; @@ -688,8 +688,8 @@ impl SepData { self.sks_mark_create_refused(slot, out.reply.status); return false; }; - // 13.5 replies with the struct version and the handle alone - // (0x…e11c-e12c); the T6020 reply adds a blob. + // 13.5 replies with the struct version and the handle alone; + // the T6020 reply adds a blob. let reply_min = match self.profile.key_store { profile::KeyStore::Sepos13 { .. } => 8, profile::KeyStore::Variant5 => 12, @@ -731,8 +731,8 @@ impl SepData { return false; } let handle = crate::sks::KeyBagHandle::from_create_reply(raw_handle); - // identity_create reads no UUID back: 13.5 names the identity by the - // UUID it was created with, and identity_save (0xfffffe000994b1cc) is + // An identity create reads no UUID back: 13.5 names the identity by the + // UUID it was created with, and saving the identity is // just the copy below, then the unload. let bag_uuid = match self.profile.key_store { profile::KeyStore::Sepos13 { .. } => None, @@ -926,7 +926,7 @@ impl SepData { &self, stored: &keybag::StoredKeyBag, ) -> Option<(crate::sks::KeyBagHandle, [u8; keybag::UUID_LEN])> { - // 13.5 identity_open addresses an enclave-resident identity bag by + // Opening a 13.5 identity addresses an enclave-resident identity bag by // its generated UUID. Older J313 records instead store the bag's // read-back UUID and a wrapped bag, which must be loaded as a blob. let uuid_load = matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }) @@ -1236,7 +1236,7 @@ pub(crate) fn sks_set_env() -> SksOp { } } pub(crate) const SKS_SET_ENV_LEN: usize = 12 + 0x400; -/// Third environment word for `set_env(false)`, the call init_sep_endpoint makes. +/// Third environment word for the "false" `set_env` sent at endpoint init. pub(crate) const SKS_SET_ENV_FALSE: u32 = 4; const OP_SKS_COPY_UUID: u8 = 0x06; From cb222e7f649ddba4d50dd0ec9ca0c3f3200ce336 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 01:44:15 +1100 Subject: [PATCH 0801/1579] dt-bindings: soc: apple: sep: Allow a power domain the SEP depends on On T8140 the SEP depends on the GPIO power state while it runs: once that state is switched off, the enclave stops reading its mailbox and does not recover until the next boot. Let the node list the power domain the SEP depends on after its own. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/soc/apple/apple,sep.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml b/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml index 989ee4b9e9078..83ad3aa31a4b7 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml @@ -33,7 +33,13 @@ properties: maxItems: 1 power-domains: - maxItems: 1 + minItems: 1 + items: + - description: The SEP's own power state. + - description: + A power state the SEP depends on while it runs. On T8140 this is + the GPIO power state; the SEP stops servicing its mailbox for good + once that is switched off. memory-region: maxItems: 1 From 4ceb4f7689ba21aafaa000ae779795d41aee9ac8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 20:53:24 +1100 Subject: [PATCH 0802/1579] soc: apple: dockchannel: Let received data wake the system The DockChannel FIFO carries the MTP coprocessor's keyboard and trackpad reports on the M2 and later laptops. The coprocessor keeps running across suspend-to-idle, so a key press does reach the FIFO, but it cannot end the sleep: suspend_device_irqs() disables the receive interrupt like any other device interrupt, and enable_irq_wake() on it fails with -ENXIO because the DockChannel interrupt controller implements neither irq_set_wake() nor IRQCHIP_SKIP_SET_WAKE. Nothing needs configuring for a FIFO interrupt to wake the system. The controller is chained to a single AIC interrupt, which the IRQ core never disables for system sleep, and the AIC keeps delivering it in suspend-to-idle. Mark the irqchip IRQCHIP_SKIP_SET_WAKE. Add dockchannel_init_wakeup() for clients whose remote side is a wakeup source. It makes the client device wakeup capable and enabled, and registers the receive interrupt as its wake interrupt, so that the PM core arms it across system sleep whenever wakeup is enabled in sysfs. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/dockchannel.c | 33 +++++++++++++++++++++++++++ include/linux/soc/apple/dockchannel.h | 1 + 2 files changed, 34 insertions(+) diff --git a/drivers/soc/apple/dockchannel.c b/drivers/soc/apple/dockchannel.c index 3557b1e4e0558..3072284a911a7 100644 --- a/drivers/soc/apple/dockchannel.c +++ b/drivers/soc/apple/dockchannel.c @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include @@ -263,6 +265,30 @@ struct dockchannel *dockchannel_init(struct platform_device *pdev) } EXPORT_SYMBOL(dockchannel_init); +/** + * dockchannel_init_wakeup() - Let data from the remote side wake the system + * @dockchannel: FIFO returned by dockchannel_init() + * + * Make the FIFO's device a wakeup source, enabled by default, with the + * receive interrupt as its wake interrupt. While wakeup is enabled for the + * device, the PM core keeps that interrupt armed across system sleep, and + * data that arrives then wakes the system. The data itself is read after + * resume, as usual. + * + * Return: 0 on success, or a negative error code. + */ +int dockchannel_init_wakeup(struct dockchannel *dockchannel) +{ + int ret; + + ret = devm_device_init_wakeup(dockchannel->dev); + if (ret) + return ret; + + return devm_pm_set_wake_irq(dockchannel->dev, dockchannel->rx_irq); +} +EXPORT_SYMBOL(dockchannel_init_wakeup); + /* Dockchannel IRQchip */ @@ -308,11 +334,18 @@ static void dockchannel_irq_unmask(struct irq_data *data) writel_relaxed(val | BIT(hwirq), dcc->irq_base + IRQ_MASK); } +/* + * The controller hangs off a single chained AIC interrupt, which the IRQ core + * never disables for system sleep and which keeps being delivered in + * suspend-to-idle. A FIFO interrupt left unmasked can therefore end a + * suspend-to-idle without configuring anything, here or in the AIC. + */ static const struct irq_chip dockchannel_irqchip = { .name = "dockchannel-irqc", .irq_ack = dockchannel_irq_ack, .irq_mask = dockchannel_irq_mask, .irq_unmask = dockchannel_irq_unmask, + .flags = IRQCHIP_SKIP_SET_WAKE, }; static int dockchannel_irq_domain_map(struct irq_domain *d, unsigned int virq, diff --git a/include/linux/soc/apple/dockchannel.h b/include/linux/soc/apple/dockchannel.h index 0b7093935ddf4..759b5dda2be1f 100644 --- a/include/linux/soc/apple/dockchannel.h +++ b/include/linux/soc/apple/dockchannel.h @@ -15,6 +15,7 @@ struct dockchannel; struct dockchannel *dockchannel_init(struct platform_device *pdev); +int dockchannel_init_wakeup(struct dockchannel *dockchannel); int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count); int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count); From af2907b3ff96fbb33236b9bb88de8151bd0f6085 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 01:44:15 +1100 Subject: [PATCH 0803/1579] soc: apple: sep: Keep the enclave's power domains on across sleep On the J700 the SEP stops working after suspend-to-idle: hwrng reads return nothing, trusted keys can be sealed but no longer loaded ("sks: LOAD_KEYBAG got no reply"), and Touch ID cannot take its sensor back. Nothing short of a reboot recovers it. The enclave has stopped reading its mailbox: the AP-to-SEP FIFO fills up with every message sent after the fault and no reply ever comes back. It happens when the enclave is sent a message while devices are in the noirq phase of suspend, which the hwrng fill thread does: it is not freezable and keeps drawing on its timer through the whole sleep. Without a message in that window the SEP survives the sleep, and pm_test=devices with messages is fine where pm_test=platform with messages is not. In the noirq phase genpd switches off each power domain whose devices have all suspended. Refusing that for one domain at a time showed that the SEP needs the GPIO power state to stay on: with only that one kept on, it answers during the sleep and after it. The pin controller is the only Linux user of that domain, and the J700's device tree from Apple lists no power state dependency between the SEP and GPIO, so this is a dependency of the enclave as a consumer. Let a SEP node list the power domains the enclave depends on after its own, attach all of them when it lists more than one, keep them powered through device links for as long as the driver is bound, and mark the virtual domain devices as syscore devices. The links are not enough on their own: they order PM transitions and carry runtime PM, but in the noirq phase genpd runs its system sleep callbacks on every device attached to a domain, virtual ones included, counts each as suspended whatever its runtime state, and switches the domain off once all of them are. A syscore device is skipped by system sleep and so never counted, which is how the ISP keeps the domains of its resident firmware on. A node with a single domain is still attached by the platform core, so device trees without a second domain behave as before. On the J700, after each of ten 70 s suspend-to-idle cycles in which the hwrng thread drew from the SEP during the sleep, hwrng reads returned data, a trusted key could be sealed and loaded again, and a Touch ID verify started without sensor errors; genpd never switched the GPIO power state off. Before this change one such sleep was enough to wedge the enclave until reboot. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/pm.rs | 14 ++++++++++ drivers/soc/apple/pm_shim.c | 51 +++++++++++++++++++++++++++++++++++-- drivers/soc/apple/sep.rs | 5 ++++ drivers/soc/apple/shim.h | 1 + 4 files changed, 69 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index 27b120d8f77ee..a517e6e448732 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -23,6 +23,20 @@ static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); extern "C" { fn sep_pm_register(event: unsafe extern "C" fn(bool) -> c_int) -> c_int; fn sep_pm_unregister(); + fn sep_pm_keep_domains(dev: *mut c_void) -> c_int; +} + +/// Attaches every power domain of a multi-domain SEP node and keeps them on +/// across system sleep; see `pm_shim.c`. Must run before the enclave is first +/// spoken to. +pub(crate) fn keep_domains(dev: &kernel::device::Device) -> Result<()> { + // SAFETY: `dev` is the bound SEP device; the shim ties what it attaches to + // that device's lifetime through devres. + let rc = unsafe { sep_pm_keep_domains(dev.as_raw().cast()) }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok(()) } /// # Safety diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index 34dd2b3dcf50d..4a46088681767 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -1,11 +1,15 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * System-sleep notifier for the SEP driver. It only forwards "about to sleep" + * System sleep for the SEP driver: the notifier only forwards "about to sleep" * and "back from sleep" to Rust, which ends any Touch ID capture before the - * system suspends; see pm.rs. + * system suspends (see pm.rs), and the enclave's power domains are kept on + * across sleep. */ #include +#include #include +#include +#include #include #include "shim.h" @@ -56,3 +60,46 @@ void sep_pm_unregister(void) unregister_pm_notifier(&sep_pm_nb); WRITE_ONCE(sep_pm_event, NULL); } + +static void sep_pm_release_domains(void *data) +{ + struct dev_pm_domain_list *list = data; + u32 i; + + for (i = 0; i < list->num_pds; i++) + dev_pm_syscore_device(list->pd_devs[i], false); +} + +/* + * The enclave runs from boot to shutdown, and on some SoCs it needs power + * domains besides its own: once one of them is switched off, it stops + * reading its mailbox for good. A node that lists more than one domain gets + * all of them attached here, powered on through device links for as long as + * the driver is bound, and taken out of system sleep, which would otherwise + * switch a domain off in the noirq phase once its other users had suspended. + * A node with a single domain is attached by the platform core as before. + */ +int sep_pm_keep_domains(void *ptr) +{ + struct device *dev = ptr; + struct dev_pm_domain_attach_data data = { + .pd_flags = PD_FLAG_DEV_LINK_ON, + }; + struct dev_pm_domain_list *list; + int count, ret; + u32 i; + + count = of_count_phandle_with_args(dev->of_node, "power-domains", + "#power-domain-cells"); + if (count <= 1) + return 0; + + ret = devm_pm_domain_attach_list(dev, &data, &list); + if (ret < 0) + return ret; + + for (i = 0; i < list->num_pds; i++) + dev_pm_syscore_device(list->pd_devs[i], true); + + return devm_add_action_or_reset(dev, sep_pm_release_domains, list); +} diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 2b1562fae288c..cecffaa55377d 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2274,6 +2274,11 @@ impl platform::Driver for SepDriver { return Err(EBUSY); } + if let Err(e) = pm::keep_domains(dev) { + dev_err!(dev, "could not keep the power domains on ({:?})\n", e); + return Err(e); + } + let data = SepData::new(pdev)?; *data.mbox.lock() = Some(Mailbox::new_byname(dev, c"mbox", data.clone())?); diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index b8592883ce2d3..e7711068cb4ef 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -27,6 +27,7 @@ void sep_cancel_delayed_work_sync(void *work); int sep_pm_register(int (*event)(bool entering)); void sep_pm_unregister(void); +int sep_pm_keep_domains(void *dev); /* -- hwrng_shim.c ------------------------------------------------------- */ From ba5fe11dbd21a92d9ab474444179dab5cac7800e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 20:58:39 +1100 Subject: [PATCH 0804/1579] HID: dockchannel: Wake the system on keyboard and trackpad input A key press or a trackpad click on the built-in keyboard of an M2 or later laptop does not wake it from suspend-to-idle. The MTP coprocessor keeps running and sends the report, but the DockChannel receive interrupt is not a wake interrupt, so it is disabled for the sleep like any other device interrupt and the report waits in the FIFO until something else wakes the system. Register the receive interrupt as the device's wake interrupt with dockchannel_init_wakeup(). Wakeup is enabled by default, as for other keyboards, and can be turned off through the device's power/wakeup attribute. Reports that arrive during sleep are processed after resume, as before. All interfaces share the FIFO, so any trackpad input wakes the system too, a touch as well as a click. On the J700 a key press, a trackpad click and a trackpad touch each end suspend-to-idle at once, with "Triggering wakeup from IRQ" naming the DockChannel receive interrupt. With the lid closed the machine stays asleep until the lid is opened. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index d03589ad506d8..bd529afd3d4a8 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -1372,6 +1372,17 @@ static int dockchannel_hid_probe(struct platform_device *pdev) if (IS_ERR_OR_NULL(dchid->dc)) { return PTR_ERR(dchid->dc); } + + /* + * Keyboard and trackpad input wakes the system. The coprocessor keeps + * running across suspend-to-idle and keeps sending reports through + * the FIFO, so its receive interrupt is the wake interrupt. Without + * it the machine still works, it just cannot be woken from here. + */ + ret = dockchannel_init_wakeup(dchid->dc); + if (ret) + dev_warn(dev, "Failed to set up wakeup: %d\n", ret); + dchid->new_iface_wq = alloc_workqueue("dchid-new", WQ_MEM_RECLAIM, 0); if (!dchid->new_iface_wq) return -ENOMEM; From ea044d49c76c18079848672c2779c7521a6b8fc3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 01:44:16 +1100 Subject: [PATCH 0805/1579] arm64: dts: apple: t8140: Keep the GPIO power state on for the SEP The SEP stops reading its mailbox for good once the GPIO power state is switched off while it runs, which genpd does in the noirq phase of every suspend after the pin controller has suspended. The next message to the enclave, typically from the hwrng fill thread, then wedges it until reboot. Give the SEP the GPIO power state as its second power domain, so that the driver keeps it on for as long as the enclave is in use. Fixes: aec8cb1999f7 ("arm64: dts: apple: t8140: Add the SEP, its DART and mailbox, and SPI2") Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index cc071157054b8..89f12fbd90632 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -645,7 +645,11 @@ mboxes = <&sep_mbox>; mbox-names = "mbox"; iommus = <&sep_dart 0>; - power-domains = <&ps_sep>; + /* + * The enclave stops reading its mailbox for good once + * the GPIO power state is switched off under it. + */ + power-domains = <&ps_sep>, <&ps_gpio>; status = "disabled"; }; From 1e79a47b63f07b33fdf5df220e35392ba7ac46c0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 20:59:06 +1100 Subject: [PATCH 0806/1579] usb: dwc3: apple: Keep a fixed-hub controller running across sleep Commit 11585d940629 ("usb: dwc3: apple: Add system sleep support for a fixed-hub controller") exits the core of a fixed-hub controller for system sleep and initialises it again on resume. Every resume then costs a full USB re-enumeration: xHCI finds its saved state gone, reports "xHC error in resume, USBSTS 0x411, Reinit", and resets the controller and every device behind the fixed hub. On the J700 that is most of the 2.8 s device resume, and the network behind the hub goes away meanwhile. On the way down the core exit also powers the USB3 PHY off, which routes the PIPE back to the dummy PHY while the SuperSpeed port is suspended; the pipehandler lock is then never acknowledged ("Pipehandler lock not acked", "Failed to lock pipehandler"). Left running, the core keeps its state across suspend-to-idle. Mark a fixed-hub controller as a syscore device while its core is up, as a Type-C controller is while a cable is connected since commit 3b7f11e579c4 ("usb: dwc3-apple: preserve live ports across system sleep"), and drop the sleep callbacks and the DWC3_APPLE_SUSPENDED state. xHCI saves its state on suspend and restores it on resume, and the devices behind the hub are resumed instead of reset. As a syscore device the core is also never counted as suspended in its power domain, so genpd cannot switch the domain off underneath it in the noirq phase. On the J700, ten 70 s suspend-to-idle cycles resumed their devices in 293-297 ms instead of 2.8 s, with no xHCI reinit, no pipehandler message and none of the devices behind the hub reset or re-enumerated. Fixes: 11585d940629 ("usb: dwc3: apple: Add system sleep support for a fixed-hub controller") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple-state-test.c | 2 +- drivers/usb/dwc3/dwc3-apple-state.h | 1 - drivers/usb/dwc3/dwc3-apple.c | 100 +++-------------------- 3 files changed, 11 insertions(+), 92 deletions(-) diff --git a/drivers/usb/dwc3/dwc3-apple-state-test.c b/drivers/usb/dwc3/dwc3-apple-state-test.c index bf9a539a53e3f..7a04604ddd4a1 100644 --- a/drivers/usb/dwc3/dwc3-apple-state-test.c +++ b/drivers/usb/dwc3/dwc3-apple-state-test.c @@ -26,7 +26,7 @@ static void dwc3_apple_device_selected_before_reset_test(struct kunit *test) static void dwc3_apple_non_role_test(struct kunit *test) { static const enum dwc3_apple_state states[] = { - DWC3_APPLE_PROBE_PENDING, DWC3_APPLE_NO_CABLE, DWC3_APPLE_SUSPENDED, + DWC3_APPLE_PROBE_PENDING, DWC3_APPLE_NO_CABLE, }; enum phy_mode mode; int i; diff --git a/drivers/usb/dwc3/dwc3-apple-state.h b/drivers/usb/dwc3/dwc3-apple-state.h index 89ce0b416bb40..42c13d900428e 100644 --- a/drivers/usb/dwc3/dwc3-apple-state.h +++ b/drivers/usb/dwc3/dwc3-apple-state.h @@ -16,7 +16,6 @@ enum dwc3_apple_state { DWC3_APPLE_NO_CABLE, /* No cable connected, dwc3 suspended after dwc3_core_exit */ DWC3_APPLE_HOST, /* Cable connected, dwc3 in host mode */ DWC3_APPLE_DEVICE, /* Cable connected, dwc3 in device mode */ - DWC3_APPLE_SUSPENDED, /* Fixed-hub host asleep: core exited, xhci still registered */ }; /** diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index da01a43877473..232a4de95bb75 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include "dwc3-apple-state.h" @@ -76,8 +75,9 @@ * a host whatever happens at the connectors and never sees a role change: it is brought up in * DWC3_APPLE_HOST at probe and stays there. The Type-C port controllers only drive the PHY's * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. - * Across system sleep such a controller goes through DWC3_APPLE_SUSPENDED: the core is exited - * and initialised again around the sleep while xhci stays registered, see dwc3_apple_suspend(). + * + * Whenever the core is up, for either kind of port, it is kept running across system sleep, see + * dwc3_apple_init(). * * enum dwc3_apple_state is defined in dwc3-apple-state.h. */ @@ -240,7 +240,6 @@ static int dwc3_apple_core_init(struct dwc3_apple *appledwc) dev_err(appledwc->dev, "Failed to probe DWC3 Core, err=%d\n", ret); break; case DWC3_APPLE_NO_CABLE: - case DWC3_APPLE_SUSPENDED: ret = dwc3_core_init(&appledwc->dwc); if (ret) dev_err(appledwc->dev, "Failed to initialize DWC3 Core, err=%d\n", ret); @@ -378,11 +377,12 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st * role change or disconnect. Do not let generic system PM independently * power-gate the DWC3 device in between: xHCI then observes a dead core on * resume and cannot restore the root hubs without a full cable teardown. - * A fixed-hub controller is never torn down by a role change and has - * sleep callbacks of its own instead, see dwc3_apple_suspend(). + * The same holds for a fixed-hub controller, whose core is up for as long + * as the driver is bound. As a syscore device the core is not counted as + * suspended in its power domain, so the domain stays on, and xHCI saves + * its state on suspend and restores it on resume. */ - if (!appledwc->fixed_hub) - dev_pm_syscore_device(appledwc->dev, true); + dev_pm_syscore_device(appledwc->dev, true); return 0; core_exit: @@ -406,20 +406,13 @@ static int dwc3_apple_exit(struct dwc3_apple *appledwc) /* Nothing to do if we're already off */ return 0; case DWC3_APPLE_DEVICE: - if (!appledwc->fixed_hub) - dev_pm_syscore_device(appledwc->dev, false); + dev_pm_syscore_device(appledwc->dev, false); dwc3_gadget_exit(&appledwc->dwc); break; case DWC3_APPLE_HOST: - if (!appledwc->fixed_hub) - dev_pm_syscore_device(appledwc->dev, false); + dev_pm_syscore_device(appledwc->dev, false); dwc3_host_exit(&appledwc->dwc); break; - case DWC3_APPLE_SUSPENDED: - /* The core is exited and the reset asserted already, see dwc3_apple_suspend() */ - dwc3_host_exit(&appledwc->dwc); - appledwc->state = DWC3_APPLE_NO_CABLE; - return 0; } /* @@ -498,7 +491,6 @@ static enum usb_role dwc3_usb_role_switch_get(struct usb_role_switch *sw) switch (appledwc->state) { case DWC3_APPLE_HOST: - case DWC3_APPLE_SUSPENDED: return USB_ROLE_HOST; case DWC3_APPLE_DEVICE: return USB_ROLE_DEVICE; @@ -639,84 +631,12 @@ static const struct of_device_id dwc3_apple_of_match[] = { }; MODULE_DEVICE_TABLE(of, dwc3_apple_of_match); -/* - * System sleep of a fixed-hub controller. - * - * A controller with a role switch is a syscore device while a cable is connected, see - * dwc3_apple_init(). A fixed-hub controller is always up, and its core does not keep its state - * across s2idle, so it gets what dwc3_suspend_common() and dwc3_resume_common() give a host: - * the core is exited on the way down and initialised again on the way up. The xhci child stays - * registered throughout. It suspends before this device and resumes after it, and then resets - * the re-initialised controller. Adding or removing the child from inside a PM callback is not - * an option, since that re-enters the PM core's device list. - * - * No wakeup path is offered: the interrupt controller implements no irq_set_wake, so no - * interrupt on this SoC can be armed to end s2idle, and a remote-wakeup capable USB device - * could not end it either. Once that changes, the wake IRQ belongs on the xhci device - * (dev_pm_set_wake_irq()) and a keep-powered branch like dwc3_suspend_common()'s belongs here. - */ -static int dwc3_apple_suspend(struct device *dev) -{ - struct dwc3 *dwc = dev_get_drvdata(dev); - struct dwc3_apple *appledwc; - int ret; - - if (!dwc) - return 0; - appledwc = to_dwc3_apple(dwc); - - guard(mutex)(&appledwc->lock); - - if (!appledwc->fixed_hub || appledwc->state != DWC3_APPLE_HOST) - return 0; - - /* SUSPHY has to be enabled for the PHY to power down properly, as in dwc3_apple_exit() */ - dwc3_enable_susphy(&appledwc->dwc, true); - dwc3_core_exit(&appledwc->dwc); - appledwc->state = DWC3_APPLE_SUSPENDED; - - ret = reset_control_assert(appledwc->reset); - if (ret) - dev_err(appledwc->dev, "Failed to assert reset, err=%d\n", ret); - - return ret; -} - -static int dwc3_apple_resume(struct device *dev) -{ - struct dwc3 *dwc = dev_get_drvdata(dev); - struct dwc3_apple *appledwc; - int ret; - - if (!dwc) - return 0; - appledwc = to_dwc3_apple(dwc); - - guard(mutex)(&appledwc->lock); - - if (appledwc->state != DWC3_APPLE_SUSPENDED) - return 0; - - /* A failure leaves the controller in DWC3_APPLE_SUSPENDED, which remove() can tear down */ - ret = dwc3_apple_core_start(appledwc, DWC3_APPLE_HOST); - if (ret) - return ret; - - dwc3_apple_set_role(appledwc, DWC3_APPLE_HOST); - appledwc->state = DWC3_APPLE_HOST; - - return 0; -} - -static DEFINE_SIMPLE_DEV_PM_OPS(dwc3_apple_pm_ops, dwc3_apple_suspend, dwc3_apple_resume); - static struct platform_driver dwc3_apple_driver = { .probe = dwc3_apple_probe, .remove = dwc3_apple_remove, .driver = { .name = "dwc3-apple", .of_match_table = dwc3_apple_of_match, - .pm = pm_sleep_ptr(&dwc3_apple_pm_ops), }, }; From 4ca4ba3805528f5370bbe6bab3d8dbb8f7f9226c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 20:57:31 +1100 Subject: [PATCH 0807/1579] iommu: apple-dart: invalidate four-level T8140 streams by DVA range Since commit 9c512b25f55e ("iommu: apple-dart: invalidate client-ranged T8140 streams again"), every map and unmap on a four-level stream of a DART that keeps firmware roots flushes the whole stream. Every stream Linux uses on the T8140 AOP DART is four-level, so each audio buffer that is allocated or freed flushes its ADMAC's stream whole. That has two problems: - On streams that also hold the AOP firmware's own root slots (stream 0 slot 16, stream 1 slot 33, stream 8 slot 16), it drops translations the firmware set up and may be using. apple_dart_fw_invalidate() exists precisely to leave those alone; only three-level streams got that treatment. - On the MacBook Neo, a capture open/close loop on the microphone array and the headset microphone once left such a flush busy for good: apple-dart 310fc0000.iommu: TLB command 0x1 for stream 11 timed out (0x8700810b) Every later TLB command found the engine busy, buffer mappings failed with -ENOMEM, and the AOP rejected every audio run request until reboot. A whole-stream flush in flight reads 0x8400xxxx and a finished one 0x0400xxxx; the range-invalidate bits 24-25 that the stuck command also showed never appear in a whole-stream flush otherwise. Invalidate four-level streams the way three-level streams already are: by DVA range, through root slots that are not the firmware's. A four-level root translates the whole input address space from DVA 0, so each root slot covers 2^(page shift + 2 * table bits) bytes. Four-level roots are not snapshotted, so a slot is the firmware's when it is valid and does not hold the entry Linux published. Three-level streams, and with them the display and DCP streams, are unchanged. Fixes: 9c512b25f55e ("iommu: apple-dart: invalidate client-ranged T8140 streams again") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 68 ++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 00d42c36c0372..9cdbcc9798e6d 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -970,14 +970,30 @@ apple_dart_t8110_hw_tlb_command_range(struct apple_dart_stream_map *stream_map, /* A root table is at most 16 KiB of 64-bit entries. */ #define DART_MAX_ROOT_ENTRIES (SZ_16K / sizeof(u64)) +/* + * A slot of a four-level locked root that firmware holds: valid, and not an + * entry Linux published. The loader's four-level roots are not snapshotted, + * so this is the only firmware state Linux knows about them. + */ +static bool apple_dart_fw_slot_foreign(const u64 *live, const u64 *owned, + size_t i) +{ + u64 pte = READ_ONCE(live[i]); + + return (pte & APPLE_DART_PTE_VALID) && (!owned || pte != READ_ONCE(owned[i])); +} + /* * On DARTs that keep firmware roots (T8140 and T8142), the hardware keeps * using the translations behind the firmware-owned root slots, e.g. the - * display controller scanning out the boot framebuffer while Linux attaches. + * display controller scanning out the boot framebuffer while Linux attaches, + * or the AOP firmware's own DMA on the AOP DART streams that Linux shares. * Invalidating those translations, for the whole stream or by DVA range, can - * make the scanout fail a translation, and the display coprocessor panics on - * the failed read. Linux never changes what those slots resolve to, so only - * invalidate the parts of [lo, hi] reached through other root slots. + * break that DMA: the scanout fails a translation and the display + * coprocessor panics on the failed read, and a whole-stream invalidate on a + * busy AOP stream can stay busy for good. Linux never changes what those + * slots resolve to, so only invalidate the parts of [lo, hi] reached through + * other root slots, by DVA range, on three-level and four-level streams alike. */ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, u64 lo, u64 hi) @@ -986,8 +1002,7 @@ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, struct apple_dart *dart = stream_map->dart; struct apple_dart_stream_map stream = { .dart = dart }; size_t entries = dart->pgsize / sizeof(u64); - u32 slot_shift = ilog2(dart->pgsize) + ilog2(entries); - u64 span = BIT_ULL(slot_shift + ilog2(entries)); + u32 table_bits = ilog2(entries); DECLARE_BITMAP(fw_slots, DART_MAX_ROOT_ENTRIES); size_t first, last, i, end; unsigned long flags; @@ -998,23 +1013,32 @@ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { const struct apple_dart_fw_root *fw; - const u64 *live; - u64 base, start_dva, end_dva; + const u64 *live, *owned; + u64 base, span, start_dva, end_dva; + bool four_level; + u32 slot_shift; __set_bit(sid, stream.sidmap); - if (apple_dart_readl(dart, DART_TCR(dart, sid)) & - dart->hw->tcr_4level) { - ret = apple_dart_t8110_hw_tlb_command_range(&stream, op, - false, 0, 0); - goto next; + four_level = apple_dart_readl(dart, DART_TCR(dart, sid)) & + dart->hw->tcr_4level; + if (four_level) { + /* + * A four-level stream walks the whole DVA: its root slots + * cover the input address space from zero. + */ + slot_shift = ilog2(dart->pgsize) + 2 * table_bits; + span = BIT_ULL(min(slot_shift + table_bits, dart->ias)); + base = 0; + } else { + /* + * A three-level stream walks the low bits of the DVA; the + * TLB is tagged with the whole DVA, inside the window of + * the stream's domain. + */ + slot_shift = ilog2(dart->pgsize) + table_bits; + span = BIT_ULL(slot_shift + table_bits); + base = READ_ONCE(dart->locked_window[sid]) & ~(span - 1); } - - /* - * A three-level stream walks the low bits of the DVA; the TLB is - * tagged with the whole DVA, inside the window of the stream's - * domain. - */ - base = READ_ONCE(dart->locked_window[sid]) & ~(span - 1); if (hi < base || lo > base + span - 1) goto next; start_dva = max(lo, base); @@ -1026,9 +1050,11 @@ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, bitmap_zero(fw_slots, entries); spin_lock_irqsave(&dart->lock, flags); live = dart->locked_ttbr[sid][0]; + owned = dart->locked_owned[sid][0]; fw = dart->locked_fw[sid][0]; for (i = first; live && i <= last; i++) - if (apple_dart_fw_slot_owned(fw, live, i)) + if (four_level ? apple_dart_fw_slot_foreign(live, owned, i) : + apple_dart_fw_slot_owned(fw, live, i)) __set_bit(i, fw_slots); spin_unlock_irqrestore(&dart->lock, flags); if (!live) From d1c974abcc34f6976203754e7063ab91515ab034 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 19:05:02 +1100 Subject: [PATCH 0808/1579] ASoC: apple: t8140-aop-audio: count the holders of the lpai run state The low-power microphone's runn state is requested by its own PCM and, around every back-end and microphone-array startup, by with_fabric_raised(), which runs the stream to bring a gated audio fabric back. Each request reads the state back from the firmware and fails on a mismatch, but nothing orders the requests. When two services start at the same time, one raise returns lpai to idle between the other's run request and its readback, and that open fails with EIO: audio.3.auto: lpai power state 0x69646c65 after requesting 0x72756e6e A raise that finishes while the PCM starts likewise idles the capture the PCM just started, and a raise that skips the run request because the PCM holds it powers the leaves while the PCM's close idles the stream under it. Count the holders of runn under a mutex: the first holder requests runn, the last one requests idle, and every lpai power request is made by one caller at a time. Fixes: 656867bfe3ec ("ASoC: apple: t8140-aop-audio: raise a gated audio fabric before the leaves") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 78 ++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 15 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 069656e5e6d98..2d657cc62e5ef 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -52,6 +52,7 @@ use kernel::{ }, error::from_err_ptr, module_platform_driver, + new_mutex, new_spinlock, of, platform, @@ -73,6 +74,7 @@ use kernel::{ }, Arc, ArcBorrow, + Mutex, SpinLock, // }, types::{ForeignOwnable, Opaque}, @@ -432,6 +434,14 @@ struct StreamState { // sets it and close clears it under the same lock. unsafe impl Send for StreamState {} +/// The holders of the low-power microphone's `runn` state: its own PCM +/// while prepared, and every service startup that raises the fabric. +struct LpaiRun { + users: u32, + /// `runn` is the last state the firmware confirmed. + running: bool, +} + #[pin_data] struct SndSocT8140AopData { dev: ARef, @@ -440,7 +450,7 @@ struct SndSocT8140AopData { ring: Arc, ring_bytes: Atomic, sequence: Atomic, - /// lpai is in `runn`. + /// The low-power microphone PCM holds a reference on `lpai_run`. powered: AtomicFlag, /// A low-power microphone report arrived since the last run request. reported: AtomicFlag, @@ -478,6 +488,10 @@ struct SndSocT8140AopData { hpai_start_work: Work, #[pin] stream: SpinLock, + /// Every lpai power request goes through here, see + /// [`SndSocT8140AopData::lpai_run_get`]. + #[pin] + lpai_run: Mutex, } const SPKR_START_WORK: u64 = 0; @@ -680,6 +694,39 @@ impl SndSocT8140AopData { Ok(()) } + /// Takes a reference on the low-power microphone's `runn` state, + /// requesting it for the first holder. + /// + /// The PCM and the fabric raises share the stream, and every power + /// request is checked against the state the firmware reports back: a + /// raise that returns lpai to `idle` while another service starts up + /// makes that one read `idle` after requesting `runn`, and idles a + /// capture the PCM just started. So they count their holds here and + /// only the first and last holder change the state. + fn lpai_run_get(&self) -> Result<()> { + let mut run = self.lpai_run.lock(); + if !run.running { + self.reported.store(false, Relaxed); + self.set_power(POWER_STATE_RUN)?; + run.running = true; + } + run.users += 1; + Ok(()) + } + + /// Drops a reference taken by [`Self::lpai_run_get`], returning lpai to + /// `idle` after the last holder. If that fails, the next holder finds + /// it still running. + fn lpai_run_put(&self) -> Result<()> { + let mut run = self.lpai_run.lock(); + run.users -= 1; + if run.users == 0 && run.running { + self.set_power(POWER_STATE_IDLE)?; + run.running = false; + } + Ok(()) + } + // ---- the back-end services ---- /// Property 202 with the service's sequence and a 24 MHz timestamp; @@ -785,14 +832,10 @@ impl SndSocT8140AopData { /// does not do that (measured: `audio_p` stays gated for the whole leaf /// retry window), only a running stream does, after which a service in /// `pw0 ` keeps it up only while its stream runs. The low-power - /// microphone is attached for the firmware's lifetime, so run it briefly - /// around `f` when it is idle, once its first report has arrived. + /// microphone is attached for the firmware's lifetime, so hold it in + /// `runn` around `f`, once its first report has arrived. fn with_fabric_raised(&self, f: impl FnOnce() -> Result) -> Result { - if self.powered.load(Relaxed) { - return f(); - } - self.reported.store(false, Relaxed); - self.set_power(POWER_STATE_RUN)?; + self.lpai_run_get()?; let mut polls = 0; while !self.reported.load(Relaxed) && polls < FABRIC_RAISE_POLLS { kernel::time::delay::fsleep(kernel::time::Delta::from_millis(FABRIC_RAISE_POLL_MS)); @@ -802,7 +845,7 @@ impl SndSocT8140AopData { dev_warn!(self.dev, "lpai: no report while raising the fabric\n"); } let ret = f(); - if let Err(e) = self.set_power(POWER_STATE_IDLE) { + if let Err(e) = self.lpai_run_put() { dev_warn!(self.dev, "lpai: unable to idle after raising the fabric: {:?}\n", e); } ret @@ -1192,12 +1235,11 @@ unsafe extern "C" fn lpai_pcm_close(substream: *mut bindings::snd_pcm_substream) let mut st = data.stream.lock(); st.substream = ptr::null_mut(); } - if data.powered.load(Relaxed) { - if let Err(e) = data.set_power(POWER_STATE_IDLE) { + if data.powered.xchg(false, Relaxed) { + if let Err(e) = data.lpai_run_put() { dev_err!(data.dev, "unable to return lpai to idle\n"); return e.to_errno(); } - data.powered.store(false, Relaxed); } 0 } @@ -1221,7 +1263,7 @@ unsafe extern "C" fn lpai_pcm_prepare(substream: *mut bindings::snd_pcm_substrea data.hw_ptr_frames.store(0, Relaxed); } if !data.powered.load(Relaxed) { - if let Err(e) = data.set_power(POWER_STATE_RUN) { + if let Err(e) = data.lpai_run_get() { dev_err!(data.dev, "unable to run lpai\n"); return e.to_errno(); } @@ -2620,8 +2662,10 @@ impl SndSocT8140AopDriver { // SAFETY: the card is freed once, while the binding remains live. unsafe { bindings::snd_card_free(self.card) }; } - if self.data.powered.load(Relaxed) && self.data.set_power(POWER_STATE_IDLE).is_ok() { - self.data.powered.store(false, Relaxed); + // Every stream is closed now; retry an idle request that failed then. + let mut run = self.data.lpai_run.lock(); + if run.running && self.data.set_power(POWER_STATE_IDLE).is_ok() { + run.running = false; } } } @@ -2750,6 +2794,10 @@ impl platform::Driver for SndSocT8140AopDriver { period_acc: 0, buffer_bytes: 0, }), + lpai_run <- new_mutex!(LpaiRun { + users: 0, + running: false, + }), }), GFP_KERNEL, )?; From bb51fe91cd3e298d032521c672122c1a7b9de204 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:23:53 +1100 Subject: [PATCH 0809/1579] Input: macsmc-input - unregister the SMC event notifier on unbind Probe adds a notifier block to the SMC's event chain, but nothing ever takes it off again. The block is embedded in devm-allocated driver data, so after an unbind the chain still links freed memory, and every SMC notification walks through it: a power button press, a lid change or a charger event is then a use-after-free, and once the module has been unloaded the chain calls into unloaded module text. A rebind can also reuse the freed memory for the new block, which zeroes the old link and cuts any handler registered after it, such as macsmc-power's, off the chain. Unregister the notifier from a devm action added right after the registration, so it comes off the chain before the input device and the driver data are released, and stop ignoring the registration result. Fixes: e89b625fdbce ("input: macsmc-input: New driver to handle the Apple Mac SMC buttons/lid") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/input/misc/macsmc-input.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/input/misc/macsmc-input.c b/drivers/input/misc/macsmc-input.c index 2cead3b7f45fe..e149b043fef82 100644 --- a/drivers/input/misc/macsmc-input.c +++ b/drivers/input/misc/macsmc-input.c @@ -106,6 +106,13 @@ static int macsmc_input_event(struct notifier_block *nb, unsigned long event, vo } } +static void macsmc_input_unregister_notifier(void *data) +{ + struct macsmc_input *smcin = data; + + blocking_notifier_chain_unregister(&smcin->smc->event_handlers, &smcin->nb); +} + static int macsmc_input_probe(struct platform_device *pdev) { struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); @@ -168,7 +175,19 @@ static int macsmc_input_probe(struct platform_device *pdev) input_sync(smcin->input); smcin->nb.notifier_call = macsmc_input_event; - blocking_notifier_chain_register(&smc->event_handlers, &smcin->nb); + error = blocking_notifier_chain_register(&smc->event_handlers, &smcin->nb); + if (error) + return error; + + /* + * The notifier block lives in devm-allocated memory and the callback + * reports to a devm-managed input device: take it off the SMC's chain + * before either goes away on unbind. + */ + error = devm_add_action_or_reset(&pdev->dev, + macsmc_input_unregister_notifier, smcin); + if (error) + return error; device_init_wakeup(&pdev->dev, true); From 52fc78534cb5214138c1076d405cff0190787070 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:25:16 +1100 Subject: [PATCH 0810/1579] wifi: mt7932: report the receive bitrate on 40 and 80 MHz channels The station's receive rate is discarded whenever the RX vector's bandwidth field is not 20 MHz. Associations on 5 GHz use 40 and 80 MHz channels where the access point offers them, and most frames on those channels are received at the full width, so "iw dev wlan0 station dump" shows no rx bitrate on most 5 GHz networks. mt7932_rx_rate() already decodes the bandwidth field into rate_info.bw, and cfg80211 computes the bitrate for every width it produces. Keep the rate unless it is wider than the channel the association uses, which a frame from the associated access point cannot be, and keep refusing a CCK rate on 5 GHz. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/data.c | 28 +++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 25a7bcbf31238..6008d3fab33ac 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -164,6 +164,23 @@ netdev_tx_t mt_net_xmit(struct sk_buff *skb, struct net_device *netdev) return NETDEV_TX_OK; } +/* Width in MHz of a rate decoded by mt7932_rx_rate(). */ +static int mt_rate_width(enum rate_info_bw bw) +{ + switch (bw) { + case RATE_INFO_BW_20: + return 20; + case RATE_INFO_BW_40: + return 40; + case RATE_INFO_BW_80: + return 80; + case RATE_INFO_BW_160: + return 160; + default: + return INT_MAX; + } +} + void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) { struct sk_buff *skb; @@ -202,11 +219,12 @@ void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) m->station_signal_valid = true; } if (frame->unicast && frame->rate_valid) { - m->station_rx_rate_valid = mt7932_rx_rate(frame->rxv, m->phy_cap[4], - &m->station_rx_rate); - /* Current association contract admits only 20 MHz and no 5 GHz CCK. */ - if (((frame->rxv >> 12) & 7) || - (frame->channel > 14 && !((frame->rxv >> 24) & 15))) + struct rate_info *rate = &m->station_rx_rate; + + m->station_rx_rate_valid = mt7932_rx_rate(frame->rxv, m->phy_cap[4], rate); + /* No CCK on 5 GHz, and no rate wider than the associated channel. */ + if ((frame->channel > 14 && !((frame->rxv >> 24) & 15)) || + mt_rate_width(rate->bw) > cfg80211_chandef_get_width(&m->connect_chandef)) m->station_rx_rate_valid = false; m->station_rx_rate_time = jiffies; } From 261d6b450ed6edeaee885665f80ed287748018fc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:29:01 +1100 Subject: [PATCH 0811/1579] wifi: mt7932: monitor the connection's signal against a threshold The driver does not implement set_cqm_rssi_config, so userspace cannot ask to be told when the signal crosses a threshold. wpa_supplicant's "simple" background scan module, which NetworkManager enables, then picks its scan interval from one signal reading at association and never revisits it: when that reading is below the threshold it keeps scanning at the short interval, 30 seconds in NetworkManager's configuration, for as long as the long interval lasts. Each scan takes the station off channel for about a second, which shows up as a latency spike every 31 seconds or so. iwd uses the same events to decide when to look for a better access point. The firmware reports no signal threshold events, so run the monitor on the host, the way mac80211 runs it for drivers without one: average the signal of the frames the associated access point sends, both its beacons and its data, and report a crossing of the threshold through cfg80211_cqm_rssi_notify(). Take at most one sample per 100 ms, about a beacon interval, so that the average follows time rather than traffic, and wait for four samples before the first event. As in mac80211, the first event after a new association or configuration reports the side of the threshold the signal is on, and a later event needs the average to be across the threshold and more than the hysteresis away from the level last reported. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/data.c | 1 + .../net/wireless/mediatek/mt7932/frontend.c | 11 ++-- drivers/net/wireless/mediatek/mt7932/link.c | 1 + drivers/net/wireless/mediatek/mt7932/mt7932.h | 13 ++++ .../net/wireless/mediatek/mt7932/startup.c | 1 + .../net/wireless/mediatek/mt7932/station.c | 66 +++++++++++++++++++ 6 files changed, 89 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 25a7bcbf31238..8a396d10fcc63 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -200,6 +200,7 @@ void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) if (frame->signal_valid && signal > -128) { m->station_signal = signal; m->station_signal_valid = true; + mt_cqm_signal(m, signal); } if (frame->unicast && frame->rate_valid) { m->station_rx_rate_valid = mt7932_rx_rate(frame->rxv, m->phy_cap[4], diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 50f424cdf3c03..282404beac17b 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -631,11 +631,14 @@ void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length) unsigned long flags; spin_lock_irqsave(&m->response_lock, flags); - if (tim && tim->datalen >= 4 && m->connected && !m->disconnecting && - !m->stopping && !m->power_tim && frame.channel == m->connect_channel && + if (m->connected && !m->disconnecting && !m->stopping && + frame.channel == m->connect_channel && ether_addr_equal(frame.data + 16, m->connect_bssid)) { - m->power_tim = true; - schedule_work(&m->power_work); + mt_cqm_signal(m, mt7932_rx_signal(&frame, m->antenna_mask)); + if (tim && tim->datalen >= 4 && !m->power_tim) { + m->power_tim = true; + schedule_work(&m->power_work); + } } spin_unlock_irqrestore(&m->response_lock, flags); } diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index fc2d333519f98..6869e7ff85c8a 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -471,6 +471,7 @@ void mt_connect_work(struct work_struct *work) m->station_rx_rate_valid = false; m->station_tx_rate_valid = false; spin_unlock(&m->data_lock); + mt_cqm_reset(m); } m->connected = !ret; m->disconnecting = false; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 261199230ac79..fa312ecca622c 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -3,6 +3,7 @@ #define MT7932_H +#include #include #include #include @@ -40,6 +41,8 @@ #define STRIDE 8192 #define RX_STRIDE ALIGN(MT7932_RX_CAPACITY, 64) +DECLARE_EWMA(mt7932_signal, 4, 4) + struct mt7932; int mt_dma_stop(struct mt7932 *m); int mt_function_reset(struct mt7932 *m); @@ -156,6 +159,12 @@ struct mt7932 { struct rate_info station_tx_rate; unsigned long station_tx_rate_time, station_rate_query_time; bool station_tx_rate_valid, station_rate_query_failed; + /* Connection quality monitor, protected by response_lock. */ + struct ewma_mt7932_signal cqm_signal; + unsigned long cqm_sample_time; + s32 cqm_threshold, cqm_last_event; + u32 cqm_hysteresis; + u8 cqm_samples; u8 *data_headers, *data_payloads; dma_addr_t data_headers_dma, data_payloads_dma; bool data_live[512], data_token[512], data_retired[512]; @@ -219,6 +228,10 @@ int mt_get_station(struct wiphy *wiphy, struct wireless_dev *wdev, const u8 *mac, struct station_info *sinfo); int mt_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev, int idx, u8 *mac, struct station_info *sinfo); +int mt_set_cqm_rssi_config(struct wiphy *wiphy, struct net_device *netdev, + s32 rssi_thold, u32 rssi_hyst); +void mt_cqm_reset(struct mt7932 *m); +void mt_cqm_signal(struct mt7932 *m, int signal); int mt_set_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, bool enabled, int timeout); void mt_power_work(struct work_struct *work); diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 225e5a0344fc4..696b79e48ddfe 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -459,6 +459,7 @@ static const struct cfg80211_ops mt_cfg_ops = { .get_channel = mt_get_channel, .dump_station = mt_dump_station, .set_power_mgmt = mt_set_power_mgmt, + .set_cqm_rssi_config = mt_set_cqm_rssi_config, }; int mt_register_regulatory_gate(struct mt7932 *m) diff --git a/drivers/net/wireless/mediatek/mt7932/station.c b/drivers/net/wireless/mediatek/mt7932/station.c index c9d388e50d8d5..83cee4b650926 100644 --- a/drivers/net/wireless/mediatek/mt7932/station.c +++ b/drivers/net/wireless/mediatek/mt7932/station.c @@ -103,3 +103,69 @@ int mt_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev, spin_unlock_irqrestore(&m->response_lock, flags); return ret; } + +/* + * The firmware reports no signal threshold events, so the connection quality + * monitor runs on the host, as mac80211 runs it for drivers without one: + * average the signal of the frames the associated access point sends and + * report when the average crosses the threshold. Sample at most once per + * beacon interval, so that the average follows time rather than traffic. + */ +#define MT7932_CQM_INTERVAL (HZ / 10) +#define MT7932_CQM_MIN_SAMPLES 4 + +/* Caller holds response_lock; a new association starts a new average. */ +void mt_cqm_reset(struct mt7932 *m) +{ + ewma_mt7932_signal_init(&m->cqm_signal); + m->cqm_samples = 0; + m->cqm_last_event = 0; +} + +/* Caller holds response_lock; @signal is a frame from the associated AP. */ +void mt_cqm_signal(struct mt7932 *m, int signal) +{ + enum nl80211_cqm_rssi_threshold_event event; + int average, last = m->cqm_last_event; + int hysteresis = m->cqm_hysteresis; + + if (signal >= 0 || signal <= -128) + return; + if (m->cqm_samples && + time_before(jiffies, m->cqm_sample_time + MT7932_CQM_INTERVAL)) + return; + m->cqm_sample_time = jiffies; + ewma_mt7932_signal_add(&m->cqm_signal, -signal); + if (m->cqm_samples < MT7932_CQM_MIN_SAMPLES) + m->cqm_samples++; + if (!m->cqm_threshold || m->cqm_samples < MT7932_CQM_MIN_SAMPLES) + return; + + /* The first event after (re)configuration reports the current side. */ + average = -(int)ewma_mt7932_signal_read(&m->cqm_signal); + if (average < m->cqm_threshold && (!last || average < last - hysteresis)) + event = NL80211_CQM_RSSI_THRESHOLD_EVENT_LOW; + else if (average > m->cqm_threshold && (!last || average > last + hysteresis)) + event = NL80211_CQM_RSSI_THRESHOLD_EVENT_HIGH; + else + return; + m->cqm_last_event = average; + cfg80211_cqm_rssi_notify(m->netdev, event, average, GFP_ATOMIC); +} + +int mt_set_cqm_rssi_config(struct wiphy *wiphy, struct net_device *netdev, + s32 rssi_thold, u32 rssi_hyst) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + unsigned long flags; + + if (netdev != m->netdev) + return -ENODEV; + spin_lock_irqsave(&m->response_lock, flags); + /* A zero threshold disables the monitor. */ + m->cqm_threshold = rssi_thold; + m->cqm_hysteresis = min_t(u32, rssi_hyst, S8_MAX); + m->cqm_last_event = 0; + spin_unlock_irqrestore(&m->response_lock, flags); + return 0; +} From 9aa531dee41919fc10404dfb6b0877b1182de815 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:30:48 +1100 Subject: [PATCH 0812/1579] dmaengine: apple-admac: Wait for the channel to stop in synchronize device_synchronize() must not return while the controller can still access the memory of terminated descriptors, but admac_synchronize() only kills the tasklet and frees the descriptors. terminate_all() posts the channel's STOP write and resets the rings without waiting for the channel to wind down, so a client that reuses or frees its buffer right after dmaengine_synchronize() races the channel's last burst, and nothing even ensures that the STOP write has reached the controller by then. The START registers hold the per-channel enable bits that the STOP registers clear (m1n1's register map calls them TX_EN/RX_EN and TX_EN_CLR/RX_EN_CLR). Poll the channel's bit until it reads back clear before synchronize goes on; the read also pushes out the posted STOP write. A channel only has a burst to finish, so bound the wait generously and warn instead of hanging the caller if it never clears. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/dma/apple-admac.c | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index a1aeaf3e4dc8f..dfaeeba9d8e83 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +43,10 @@ #define T603X_UNK_28_VAL 0x200000 +/* + * One bit per channel: a write to START sets the channel's enable bit, a write + * to STOP clears it, and START reads back the channels that are enabled. + */ #define REG_TX_START 0x0000 #define REG_TX_STOP 0x0004 #define REG_RX_START 0x0008 @@ -474,6 +479,35 @@ static void admac_stop_chan(struct admac_chan *adchan) dev_dbg(adchan->host->dev, "ch%d stop\n", adchan->no); } +/* + * Generous bound for a channel to wind down after its enable bit was cleared: + * it only has to finish the burst in flight, a few frames at audio rates. + */ +#define ADMAC_STOP_TIMEOUT_US 100000 + +static void admac_wait_stopped(struct admac_chan *adchan) +{ + struct admac_data *ad = adchan->host; + u32 bit = BIT(adchan->no / 2); + void __iomem *reg; + u32 val; + + switch (admac_chan_direction(adchan->no)) { + case DMA_MEM_TO_DEV: + reg = ad->base + REG_TX_START; + break; + case DMA_DEV_TO_MEM: + reg = ad->base + REG_RX_START; + break; + default: + return; + } + + if (readl_poll_timeout(reg, val, !(val & bit), 10, ADMAC_STOP_TIMEOUT_US)) + dev_warn_ratelimited(ad->dev, "ch%d still enabled %d us after stop\n", + adchan->no, ADMAC_STOP_TIMEOUT_US); +} + static void admac_reset_rings(struct admac_chan *adchan) { struct admac_data *ad = adchan->host; @@ -564,6 +598,13 @@ static void admac_synchronize(struct dma_chan *chan) unsigned long flags; LIST_HEAD(head); + /* + * terminate_all() only requests the stop: make sure the channel no + * longer touches the memory of the terminated descriptors before the + * client is allowed to reuse or free it. + */ + admac_wait_stopped(adchan); + spin_lock_irqsave(&adchan->lock, flags); list_splice_tail_init(&adchan->to_free, &head); spin_unlock_irqrestore(&adchan->lock, flags); From fc58f598aae29364282925a68bc67880c2d6b7fd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:31:58 +1100 Subject: [PATCH 0813/1579] wifi: mt7932: ring the TX doorbell once per burst mt_net_xmit() writes the TX0 CPU index after every packet, even when the stack is handing over a burst. Each write is an uncached MMIO store across the PCIe link, and each makes the device fetch descriptors for a single packet. Use netdev_xmit_more() to publish a burst with one doorbell write: ring it when the stack has nothing more queued, when the queue has just stopped, and at least every 64 packets so that the device starts on a long burst early. Descriptors stay published in order behind the doorbell, and the TX0 accounting is unchanged. The queue can also stop from outside ndo_start_xmit(), when the access point goes absent or the link goes down, which ends a burst early. Ring the doorbell in those paths too, before the connection teardown waits for TX0 to drain, and on every interrupt, so that no published descriptor waits behind a doorbell that is never written. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/data.c | 41 +++++++++++++++++-- drivers/net/wireless/mediatek/mt7932/link.c | 6 ++- drivers/net/wireless/mediatek/mt7932/mt7932.h | 3 ++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 25a7bcbf31238..e75b666b30baf 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -35,6 +35,25 @@ void mt_data_clean_locked(struct mt7932 *m) netif_wake_queue(m->netdev); } +/* + * The stack marks a packet that has another one right behind it with + * netdev_xmit_more(), so the TX0 doorbell is rung once per burst instead of + * once per packet, and at least every MT7932_TX_BATCH packets so that the + * device starts on a long burst early. + */ +#define MT7932_TX_BATCH 64 + +/* data_lock held: hand every published TX0 descriptor to the device. */ +void mt_data_kick_locked(struct mt7932 *m) +{ + struct mt7932_ring *q = &m->tx[0]; + + if (!m->data_unkicked) + return; + m->data_unkicked = 0; + mt_write(m, q->reg + 8, q->head); +} + void mt_data_clean(struct mt7932 *m) { unsigned long flags; @@ -42,6 +61,10 @@ void mt_data_clean(struct mt7932 *m) if (!READ_ONCE(m->data_ready)) return; spin_lock_irqsave(&m->data_lock, flags); + /* The queue can be stopped from outside ndo_start_xmit() in the middle + * of a burst; no descriptor waits longer than the next interrupt. + */ + mt_data_kick_locked(m); mt_data_clean_locked(m); spin_unlock_irqrestore(&m->data_lock, flags); } @@ -59,10 +82,12 @@ void mt_bss_presence(struct mt7932 *m, const struct mt7932_event *event) spin_lock_irqsave(&m->data_lock, flags); m->bss_absent = !!body[1]; m->bss_quota = body[2]; - if (!mt_bss_tx_allowed(m)) + if (!mt_bss_tx_allowed(m)) { netif_stop_queue(m->netdev); - else if (READ_ONCE(m->data_ready)) + mt_data_kick_locked(m); + } else if (READ_ONCE(m->data_ready)) { mt_data_clean_locked(m); + } spin_unlock_irqrestore(&m->data_lock, flags); dev_dbg_ratelimited(&m->pdev->dev, "BSS_PRESENCE: absent=%u quota=%u\n", !!body[1], body[2]); @@ -122,6 +147,7 @@ netdev_tx_t mt_net_xmit(struct sk_buff *skb, struct net_device *netdev) slot = q->head; if (!mt_bss_tx_allowed(m) || m->data_live[slot] || q->queued >= q->count - 1) { netif_stop_queue(netdev); + mt_data_kick_locked(m); spin_unlock_irqrestore(&m->data_lock, flags); return NETDEV_TX_BUSY; } @@ -147,18 +173,27 @@ netdev_tx_t mt_net_xmit(struct sk_buff *skb, struct net_device *netdev) q->head = (slot + 1) % q->count; q->queued++; dma_wmb(); - mt_write(m, q->reg + 8, q->head); + m->data_unkicked++; mt_bss_tx_account(m); m->station_tx_packets++; m->station_tx_bytes += skb->len; if (!mt_bss_tx_allowed(m) || m->data_live[q->head] || q->queued >= q->count - 1) netif_stop_queue(netdev); + if (!netdev_xmit_more() || netif_queue_stopped(netdev) || + m->data_unkicked >= MT7932_TX_BATCH) + mt_data_kick_locked(m); spin_unlock_irqrestore(&m->data_lock, flags); netdev->stats.tx_packets++; netdev->stats.tx_bytes += skb->len; dev_kfree_skb_any(skb); return NETDEV_TX_OK; drop: + /* The burst ends here: publish what earlier packets queued. */ + if (!netdev_xmit_more() && READ_ONCE(m->data_ready)) { + spin_lock_irqsave(&m->data_lock, flags); + mt_data_kick_locked(m); + spin_unlock_irqrestore(&m->data_lock, flags); + } netdev->stats.tx_dropped++; dev_kfree_skb_any(skb); return NETDEV_TX_OK; diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index fc2d333519f98..058592ecec467 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -224,8 +224,12 @@ static int mt_retire_connection(struct mt7932 *m) return -ESHUTDOWN; netif_carrier_off(m->netdev); netif_tx_disable(m->netdev); - /* ndo_start_xmit rechecks the closed admission gate under data_lock. */ + /* ndo_start_xmit rechecks the closed admission gate under data_lock. + * The queue may have stopped in the middle of a burst; hand the device + * what it already holds so that the drain below can complete. + */ spin_lock_irqsave(&m->data_lock, flags); + mt_data_kick_locked(m); spin_unlock_irqrestore(&m->data_lock, flags); flush_work(&m->cal_work); mutex_lock(&m->command_mutex); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 261199230ac79..10ce8f21e6e53 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -161,6 +161,8 @@ struct mt7932 { bool data_live[512], data_token[512], data_retired[512]; struct mt7932_duplicate duplicates[17]; u64 tx_released, rx_ethernet; + /* TX0 descriptors published since the last doorbell; data_lock. */ + unsigned int data_unkicked; bool data_ready; bool bss_absent; u8 bss_quota; @@ -186,6 +188,7 @@ int mt_cal_procedure(struct mt7932 *m, struct mt7932_cal_piece *pieces, unsigned int count, unsigned int logical, u8 context_version); int mt_calibration_gate(struct mt7932 *m); void mt_data_clean_locked(struct mt7932 *m); +void mt_data_kick_locked(struct mt7932 *m); void mt_data_clean(struct mt7932 *m); void mt_bss_presence(struct mt7932 *m, const struct mt7932_event *event); void mt_data_complete(struct mt7932 *m, const u8 *packet, size_t length); From b24c12fad32aa6fe9d6164e768215fb0b2f5a068 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:31:42 +1100 Subject: [PATCH 0814/1579] media: apple: isp: Stop the coprocessor before unwinding a failed boot A firmware boot that fails after the first handshake unwinds stage by stage: it frees the channel table and the extra heap, and only then stops the coprocessor and powers its domains down. Stage 2 also frees the extra heap itself when the second handshake does not come. The firmware is running from the first handshake on, so it can still access the heap while its pages are released; the pages are unmapped from the DART first, so a late access faults instead of reaching freed memory, but it should not happen at all. isp_firmware_shutdown() already stops the coprocessor before freeing anything. Share that order with the failed boot: stop the coprocessor and power its domains down, then free the channel table, the extra heap and the surfaces the firmware requested. Stage 2 leaves the heap to the caller, and the heap pointer is cleared once it has been freed, so an unwind that runs before the heap is allocated has nothing to free. The runtime PM reference of the ISP and the references on its other power domains are dropped as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 57 +++++++++++++---------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index ed35f4701c04a..65c760de616f1 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -455,7 +455,11 @@ void apple_isp_free_firmware_surface(struct apple_isp *isp) static void isp_firmware_shutdown_stage2(struct apple_isp *isp) { + if (!isp->extra_surf) + return; + isp_free_surface(isp, isp->extra_surf); + isp->extra_surf = NULL; } static void isp_write_bootargs(struct apple_isp *isp, void *virt, @@ -504,7 +508,6 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) size_t cmd_size = ISP_CMD_AREA_SIZE(isp_num_capmeta(isp)); dma_addr_t args_iova, cmd_iova; void *args_virt, *cmd_virt; - int err; u32 num_ipc_chans = isp_gpio_read32(isp, ISP_GPIO_0); u32 args_offset = isp_gpio_read32(isp, ISP_GPIO_1); @@ -575,19 +578,18 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) /* Wait for ISP_GPIO_7 to 0xf7fbdff9 -> 0x8042006 */ isp_gpio_write32(isp, ISP_GPIO_7, 0xf7fbdff9); + /* + * The firmware may already use the extra heap: it is freed only once + * the coprocessor has stopped, see isp_firmware_boot(). + */ if (isp_gpio_wait(isp, ISP_GPIO_7, 0x8042006)) { isp_err(isp, "never received second magic number from firmware\n"); - err = -ENODEV; - goto free_extra; + return -ENODEV; } isp_dbg(isp, "got second magic number from firmware\n"); return 0; - -free_extra: - isp_free_surface(isp, isp->extra_surf); - return err; } static inline struct isp_channel *isp_get_chan_index(struct apple_isp *isp, @@ -866,32 +868,49 @@ static void isp_collect_gc_surface(struct apple_isp *isp) } } +/* + * Stop the coprocessor and power its domains down, then release the memory + * the firmware may still be using: its channel table, extra heap and the + * surfaces it requested. + */ +static void isp_firmware_stop(struct apple_isp *isp) +{ + isp_firmware_shutdown_stage1(isp); + + isp_firmware_shutdown_stage3(isp); + isp_firmware_shutdown_stage2(isp); + isp_collect_gc_surface(isp); +} + static int isp_firmware_boot(struct apple_isp *isp) { int err; err = isp_firmware_boot_stage1(isp); if (err < 0) { + /* Stage 1 stops the coprocessor and powers it down itself. */ isp_err(isp, "failed firmware boot stage 1: %d\n", err); - goto garbage_collect; + isp_collect_gc_surface(isp); + return err; } + /* From here on the firmware runs, and may use what it was given. */ err = isp_firmware_boot_stage2(isp); if (err < 0) { isp_err(isp, "failed firmware boot stage 2: %d\n", err); - goto shutdown_stage1; + goto stop; } err = isp_firmware_boot_stage3(isp); if (err < 0) { isp_err(isp, "failed firmware boot stage 3: %d\n", err); - goto shutdown_stage2; + goto stop; } err = isp_enable_irq(isp); if (err < 0) { isp_err(isp, "failed to enable interrupts: %d\n", err); - goto shutdown_stage3; + goto stop; } err = isp_start_command_processor(isp); @@ -906,14 +925,8 @@ static int isp_firmware_boot(struct apple_isp *isp) disable_irqs: isp_disable_irq(isp); -shutdown_stage3: - isp_firmware_shutdown_stage3(isp); -shutdown_stage2: - isp_firmware_shutdown_stage2(isp); -shutdown_stage1: - isp_firmware_shutdown_stage1(isp); -garbage_collect: - isp_collect_gc_surface(isp); +stop: + isp_firmware_stop(isp); return err; } @@ -928,11 +941,7 @@ static void isp_firmware_shutdown(struct apple_isp *isp) if (isp_stop_command_processor(isp)) dev_warn(isp->dev, "firmware did not suspend, stopping it\n"); isp_disable_irq(isp); - isp_firmware_shutdown_stage1(isp); - - isp_firmware_shutdown_stage3(isp); - isp_firmware_shutdown_stage2(isp); - isp_collect_gc_surface(isp); + isp_firmware_stop(isp); } /* From a680729a168fabfd6f5247e8f640f98615cdc8de Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:32:41 +1100 Subject: [PATCH 0815/1579] media: apple: isp: Report a power domain that fails to go down as such apple_isp_power_down_domains() reports a failed runtime PM put of an ISP power domain as a failure to power it up, which points at the wrong transition when a domain stays on after the firmware has stopped. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 65c760de616f1..46b4513b39df3 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -95,7 +95,7 @@ static void apple_isp_power_down_domains(struct apple_isp *isp) ret = pm_runtime_put_sync(isp->pd_dev[i]); if (ret < 0) dev_err(isp->dev, - "Failed to power up power domain %d: %d\n", i, ret); + "Failed to power down power domain %d: %d\n", i, ret); } isp->pds_active = false; From d35e09d2d2501c909f2dea30beb82c866d327e7d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:35:02 +1100 Subject: [PATCH 0816/1579] ASoC: apple: t8140-aop-audio: wait for stopped streams in sync_stop None of the driver's PCMs has a sync_stop op, so after a stop ALSA moves on to hw_params, hw_free or prepare without waiting for the stream to have really stopped. The microphone array and the front-ends stop their ADMAC channel from the trigger with dmaengine_terminate_async(), and the channel is only synchronized at close. hw_free comes first and releases the buffer, which these PCMs allocate at every hw_params, so the channel's last burst and a pending period callback can land after the buffer is gone. Give them the dmaengine helper's sync_stop, as the generic dmaengine PCM does. The low-power microphone has no DMA: its report handler copies each reported span into the ALSA buffer under the stream state lock, after checking the run flag, and the trigger only clears that flag. A report that passed the check before the stop can still be copying when hw_free releases a buffer larger than the preallocated one. Its sync_stop takes the state lock once, which waits for such a copy; any later report sees the flag clear. Fixes: eeb112648f2a ("ASoC: apple: Add T8140 AOP audio driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 58 ++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 069656e5e6d98..bf76e35d0af25 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -1256,6 +1256,24 @@ unsafe extern "C" fn lpai_pcm_trigger( } } +/// Called after a stop, without the ALSA stream lock, before the buffer is +/// reallocated or freed (hw_params, hw_free) or the counters are reset +/// (prepare). The trigger only clears the run flag, so a report that saw +/// the stream running may still be copying into the buffer: it holds the +/// stream lock for the whole copy, so taking that lock waits for it, and +/// every report after it sees the flag clear. +/// +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM, whose private +/// data is the driver's `Arc`. +unsafe extern "C" fn lpai_pcm_sync_stop(substream: *mut bindings::snd_pcm_substream) -> i32 { + // SAFETY: ALSA calls this op with a live substream of the LPAI PCM. + let data = unsafe { pcm_data(substream) }; + drop(data.stream.lock()); + 0 +} + /// # Safety /// /// Called by ALSA with a live substream of this card's PCM, whose private @@ -1374,6 +1392,19 @@ unsafe extern "C" fn hpai_pcm_trigger( 0 } +/// The trigger stops the DMA asynchronously: wait for the channel to stop +/// and for its completion callbacks to finish before ALSA reuses or frees +/// the buffer. +/// +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM. +unsafe extern "C" fn hpai_pcm_sync_stop(substream: *mut bindings::snd_pcm_substream) -> i32 { + // SAFETY: the substream is live and its dmaengine runtime was set up by + // `snd_dmaengine_pcm_open` in `hpai_pcm_open`. + unsafe { bindings::snd_dmaengine_pcm_sync_stop(substream) } +} + /// # Safety /// /// Called once by ALSA when the PCM goes away, with the `Arc` leaked into @@ -1953,6 +1984,28 @@ unsafe extern "C" fn asoc_pcm_trigger( unsafe { bindings::snd_dmaengine_pcm_trigger(substream, cmd) } } +/// Front-end stop synchronization, as for the microphone array: the trigger +/// stops the DMA asynchronously, so wait for the channel to stop and for its +/// completion callbacks to finish before ALSA reuses or frees the buffer. +/// +/// # Safety +/// +/// Called by ASoC on this driver's registered component, with live +/// arguments belonging to one of its links. +unsafe extern "C" fn asoc_pcm_sync_stop( + _component: *mut bindings::snd_soc_component, + substream: *mut bindings::snd_pcm_substream, +) -> i32 { + // SAFETY: ASoC hands its ops a live substream whose private data is the + // `snd_soc_pcm_runtime` of one of this component's links. + if unsafe { rtd_is_be(substream_rtd(substream)) } { + return 0; + } + // SAFETY: the substream is live and its dmaengine runtime was set up by + // `snd_dmaengine_pcm_open` in `asoc_pcm_open`. + unsafe { bindings::snd_dmaengine_pcm_sync_stop(substream) } +} + /// # Safety /// /// Called by ASoC on this driver's registered component, with live @@ -2389,6 +2442,7 @@ impl AsocContext { hw_params: Some(asoc_pcm_hw_params), prepare: Some(asoc_pcm_prepare), trigger: Some(asoc_pcm_trigger), + sync_stop: Some(asoc_pcm_sync_stop), pointer: Some(asoc_pcm_pointer), copy: Some(asoc_pcm_copy), ..Default::default() @@ -2432,7 +2486,7 @@ impl SndSocT8140AopDriver { ioctl: None, hw_params: None, hw_free: None, - sync_stop: None, + sync_stop: Some(lpai_pcm_sync_stop), get_time_info: None, fill_silence: None, copy: None, @@ -2450,7 +2504,7 @@ impl SndSocT8140AopDriver { ioctl: None, hw_params: Some(hpai_pcm_hw_params), hw_free: None, - sync_stop: None, + sync_stop: Some(hpai_pcm_sync_stop), get_time_info: None, fill_silence: None, copy: None, From 778e39c13d98e0476bff5d10fe8a80e0a1e6ea3d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:35:42 +1100 Subject: [PATCH 0817/1579] Bluetooth: mt7932: reset the controller instead of faulting the transport Any controller error ends the Bluetooth function for the rest of the boot. A command timeout, a Hardware Error event from the controller, the core's own recovery when the controller stops accepting commands, and a write to the device's "reset" sysfs attribute all reach bt7932_reset() or bt7932_hw_error(), which mark the transport faulted, stop bus mastering and leave the adapter dead until a cold boot. A failed setup command does the same. None of these mean that the transport is broken: the DMA rings were consistent when the controller stopped responding. Recover the way other Bluetooth drivers do. On a command timeout, inject a Hardware Error event with hci_reset_dev(); the core then closes the device, which resets the controller, and opens it again, which repeats setup and sends the calibration, PTX and address again. A Hardware Error event from the controller takes the same path. Let a setup that fails return its error, so that a later power-on can try again. Bound it so that a controller that no longer answers cannot cause a reset loop: while a recovery is pending or the device is being reopened, further timeouts do not start another one, and if the setup that follows a recovery fails, the transport faults as before. The next successful setup ends the recovery. Transport faults, where the ring counters or the firmware state are inconsistent, still fault the function: reloading the firmware needs a reset of the PCI function, which this driver does not perform. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 7 ++- drivers/bluetooth/Kconfig | 4 +- drivers/bluetooth/mt7932_bt_pcie.c | 61 ++++++++++++++++++- 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 996fbe5d34f65..1a22836af939d 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -121,8 +121,11 @@ Limitations Removal first quiesces firmware, then stops work and interrupts, disables bus mastering and drains pending PCI transactions before freeing DMA. An unconfirmed drain retains storage - until reset. Firmware recovery and warm reprobe are unsupported; use a fresh - cold boot after removing or faulting an active transport. + until reset. A controller that stops answering commands, or reports a + hardware error, is reset over HCI and set up again; if that setup fails, + the transport faults. Firmware reloading and warm reprobe are + unsupported; use a fresh cold boot after removing or faulting an active + transport. * The Bluetooth software queue limit does not provide HCI backpressure; saturation can drop an accounted frame. * Arbitrary scan IEs, WPA3/SAE, required MFP, AP/P2P, general country-package diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 0456ec45b5ec0..3fe1e488c807b 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -37,7 +37,9 @@ config BT_MTK7932_PCIE is mounted and read their firmware when probed, so say M, like the Wi-Fi driver. System suspend uses the firmware sleep handshake and preserves DMA - storage. Hibernation and automatic firmware recovery are unsupported. + storage. Hibernation and firmware reloading are unsupported. A + controller that stops answering commands is reset over HCI and set + up again; if that setup fails, the transport stays faulted. Removal drains the PCI function before releasing device-visible memory; an unconfirmed drain retains it until a full hardware reset. diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 6c8378e495950..c1db653929871 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -104,6 +104,14 @@ enum bt7932_pm_state { BT7932_RESTORING, }; +enum bt7932_recovery { + BT7932_RECOVERY_NONE, + /* A Hardware Error event was injected and not yet handled. */ + BT7932_RECOVERY_REQUESTED, + /* The core is closing and reopening the device. */ + BT7932_RECOVERY_REOPENING, +}; + struct bt7932 { struct pci_dev *pdev; void __iomem *bar; @@ -133,6 +141,7 @@ struct bt7932 { bool enabled, regions, vectors, irq_requested, retained; bool ipc_ready, opened, awake, waking, fault, registered; bool stopping, runtime_forbidden; + enum bt7932_recovery recovery; enum bt7932_pm_state pm_state; u64 tx_packets, rx_packets, diagnostics; }; @@ -833,6 +842,9 @@ static int bt7932_open(struct hci_dev *hdev) ret = -EHOSTDOWN; else WRITE_ONCE(bt->opened, true); + /* Closing the device drops an injected event that was still queued. */ + if (bt->recovery == BT7932_RECOVERY_REQUESTED) + bt->recovery = BT7932_RECOVERY_NONE; mutex_unlock(&bt->lock); return ret; } @@ -940,28 +952,71 @@ static int bt7932_setup(struct hci_dev *hdev) if (ret) goto fault; dev_info(&bt->pdev->dev, "BT_HCI_SETUP_OK Reset/calibration/PTX/address completed in order\n"); + mutex_lock(&bt->lock); + if (bt->recovery == BT7932_RECOVERY_REOPENING) + dev_info(&bt->pdev->dev, "controller recovered\n"); + bt->recovery = BT7932_RECOVERY_NONE; + mutex_unlock(&bt->lock); return 0; fault: dev_err(&bt->pdev->dev, "BT_HCI_SETUP_FAILED stage=%s opcode=%04x status=%d\n", stage, opcode, ret); + /* + * The transport is intact, so a requested recovery or a later power-on + * may try again. A controller that fails the setup of its recovery, + * though, does not come back without a reset of the PCI function. + */ mutex_lock(&bt->lock); - bt7932_fault_locked(bt, "HCI setup command failed; no retry"); + if (bt->recovery == BT7932_RECOVERY_REOPENING) + bt7932_fault_locked(bt, "controller recovery failed"); mutex_unlock(&bt->lock); return ret; } +/* + * Called when a command times out, and through the "reset" sysfs attribute. + * While the transport rings are consistent, recover the controller the + * standard way: inject a Hardware Error event, so that the core closes the + * device, resetting the controller, and opens it again, repeating setup. + * Allow one attempt until a setup succeeds, so that a controller that no + * longer answers ends in a fault instead of a reset loop. + */ static void bt7932_reset(struct hci_dev *hdev) { struct bt7932 *bt = hci_get_drvdata(hdev); + bool recover; mutex_lock(&bt->lock); - bt7932_fault_locked(bt, "HCI requested recovery; reset is not qualified"); + recover = !bt->fault && !bt->stopping && + bt->recovery == BT7932_RECOVERY_NONE && + bt->pm_state == BT7932_RUNNING; + if (recover) + bt->recovery = BT7932_RECOVERY_REQUESTED; mutex_unlock(&bt->lock); + if (!recover) + return; + dev_warn(&bt->pdev->dev, "resetting the controller\n"); + if (hci_reset_dev(hdev)) { + mutex_lock(&bt->lock); + if (bt->recovery == BT7932_RECOVERY_REQUESTED) + bt->recovery = BT7932_RECOVERY_NONE; + mutex_unlock(&bt->lock); + } } +/* + * The core closes and reopens the device after this, for a Hardware Error + * event from the controller as for one injected above. That reopening is + * the recovery attempt. + */ static void bt7932_hw_error(struct hci_dev *hdev, u8 code) { - bt7932_reset(hdev); + struct bt7932 *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->recovery = BT7932_RECOVERY_REOPENING; + mutex_unlock(&bt->lock); + bt_dev_err(hdev, "hardware error 0x%2.2x", code); } static int bt7932_iommu(struct bt7932 *bt) From ba350456a9a90c22b49f1cda2e1a8f62731a932d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:23:41 +1100 Subject: [PATCH 0818/1579] wifi: mt7932: acknowledge interrupts before servicing the rings mt_service() drains the RX rings and retires TX descriptors first and only then writes the causes it read back to the interrupt status register. A completion that the device posts after a ring has been drained but before that write sets a status bit that the write clears again. Its descriptor then waits, unserviced, until some unrelated interrupt arrives; for a command reply that can be the full command timeout. Acknowledge the causes, then service the rings, as other MediaTek PCIe drivers do. Read the status register back after the write so that the acknowledgment has reached the device, and the descriptors written before it are visible, before the rings are read. A completion that arrives while the rings are being serviced now leaves its status bit set, and the device interrupts again once the mask is restored. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 4e225bb049539..92fbad6f648f9 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -277,16 +277,27 @@ static void mt_service(struct mt7932 *m) u32 causes = mt_read(m, W + 0x200) & m->irq_mask; unsigned int i; - mt_data_clean(m); - for (i = 0; i < ARRAY_SIZE(m->rx); i++) - if (m->rx[i].count) - mt_rx_drain(m, &m->rx[i]); + /* + * Acknowledge before servicing the rings. A completion that lands + * while they are being drained then sets its status bit again and + * raises a new interrupt once the mask is restored, instead of being + * cleared by a late acknowledgment and left waiting for an unrelated + * interrupt. Read the status back so that the acknowledgment has + * reached the device, and every descriptor it covers is visible, + * before the rings are read. + */ if (causes & BIT(29)) { u32 sw = mt_read(m, W + 0x1f0); mt_write(m, W + 0x1f0, sw); } mt_write(m, W + 0x200, causes); + mt_read(m, W + 0x200); + + mt_data_clean(m); + for (i = 0; i < ARRAY_SIZE(m->rx); i++) + if (m->rx[i].count) + mt_rx_drain(m, &m->rx[i]); if (READ_ONCE(m->running) && !poll_test) mt_write(m, W + 0x204, m->irq_mask); } From 62b5b9a080a5a651b7f364cd2b075887565128d5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:20:16 +1100 Subject: [PATCH 0819/1579] drm: apple: fade between H17P backlight levels The H17P backlight presents every new level in one step, so each press of a brightness key and every change from an ambient light policy shows as a visible jump on the panel. Fade between levels instead. The policy now keeps the level it presents separate from the requested target: a new target for a panel that stays lit at a known level starts a linear fade from the level presented last, and a delayed work item moves it every 16 ms for 250 ms, the last step being the target itself. Each step reuses the existing brightness-only present, so steps that come faster than presents complete merge into the next one, and a step does not bypass the backoff of a rejected present. A new level during a fade restarts it from where it is. Blanking, unblanking, suspend and a level the panel may not show yet still jump as before, and a fade still running at system sleep ends at its target before the queue is drained. The KUnit cases cover the fade steps, restarts, merging, overflow and the cases that still jump. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 3 +- drivers/gpu/drm/apple/dcp-internal.h | 6 + drivers/gpu/drm/apple/dcp.c | 12 +- drivers/gpu/drm/apple/dcp_backlight.c | 94 ++++++++++++- drivers/gpu/drm/apple/dcp_backlight.h | 76 ++++++++++- drivers/gpu/drm/apple/dcp_backlight_test.c | 152 +++++++++++++++++++++ 6 files changed, 330 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 4416c3e6bd729..d74b2dfdb0f88 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -35,7 +35,8 @@ config DRM_APPLE_KUNIT_TEST default KUNIT_ALL_TESTS help Build tests for the Apple display driver's AFK ring parser and - backlight policy (takeover, DPMS, suspend and present completion). + backlight policy (takeover, DPMS, suspend, fades and present + completion). The tests cover the header strides used by older firmware and larger rings, as well as malformed ring size combinations. This option is intended for developer test kernels. diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index df01bdd9092e4..d38c62964887d 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -411,6 +411,9 @@ struct apple_dcp { spinlock_t lock; struct dcp_backlight_state state; void (*kick)(struct apple_dcp *dcp); + /* Steps a running fade; fade_start is its start in jiffies. */ + struct delayed_work fade_work; + unsigned long fade_start; /* Rate limit for firmware re-send requests. */ unsigned long resent_at; bool resent; @@ -466,6 +469,9 @@ void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id); void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); +int dcp_backlight_init(struct apple_dcp *dcp, struct device *dev); +void dcp_backlight_stop(struct apple_dcp *dcp); +void dcp_backlight_finish_fade(struct apple_dcp *dcp); int dcp_backlight_register(struct apple_dcp *dcp); int dcp_backlight_update(struct apple_dcp *dcp); bool dcp_has_panel(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 2096f86022e09..b3f013f2b5da0 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3502,6 +3502,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); cancel_delayed_work_sync(&dcp->bl_fallback_wq); + dcp_backlight_stop(dcp); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); @@ -3557,6 +3558,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) dcp_release_piodma_iommu_dev(dcp); cancel_delayed_work_sync(&dcp->bl_fallback_wq); + dcp_backlight_stop(dcp); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); @@ -3616,7 +3618,9 @@ static int dcp_platform_probe(struct platform_device *pdev) INIT_LIST_HEAD(&dcp->swapped_out_fbs); mutex_init(&dcp->swapped_out_fbs_lock); - spin_lock_init(&dcp->backlight.lock); + ret = dcp_backlight_init(dcp, dev); + if (ret) + return ret; iomfb_queue_init(dcp); dcp->fw_compat = fw_compat; @@ -3816,6 +3820,7 @@ static void dcp_platform_shutdown(struct platform_device *pdev) */ iomfb_queue_stop(dcp); cancel_delayed_work_sync(&dcp->bl_fallback_wq); + dcp_backlight_stop(dcp); cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); return; @@ -3841,8 +3846,11 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) * A completed present can schedule one more brightness present, and a * rejected one a delayed retry; run a pending retry now instead of * letting it fire after the mailbox has suspended. Four passes cover - * the first attempt and the policy's three retries. + * the first attempt and the policy's three retries. A fade still + * running ends at its target first. */ + if (dcp_has_panel(dcp)) + dcp_backlight_finish_fade(dcp); for (pass = 0; pass < 4; pass++) { flush_delayed_work(&dcp->iomfb.backlight_retry); if (dcp_has_panel(dcp)) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 3528f7a9dbc36..51f270d21545f 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -82,6 +82,14 @@ static u32 brightness_part2[] = { }; +/* + * A new level fades in over DCP_BL_FADE_MS, one brightness-only present per + * step at most. Steps that come faster than presents complete merge, so the + * step only bounds how smooth the fade can be. + */ +#define DCP_BL_FADE_MS 250U +#define DCP_BL_FADE_STEP_MS 16U + bool dcp_backlight_active(struct apple_dcp *dcp) { return dcp->fw_compat == DCP_FIRMWARE_H17P && @@ -145,8 +153,10 @@ int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, ret = dcp_bl_init(&dcp->backlight.state, maximum, inherited_valid, inherited, default_valid, default_nits); - if (!ret) + if (!ret) { + dcp->backlight.state.fade = true; dcp->backlight.kick = kick; + } } spin_unlock_irqrestore(&dcp->backlight.lock, flags); mutex_unlock(&dcp->bl_register_mutex); @@ -174,6 +184,71 @@ static void dcp_backlight_kick(struct apple_dcp *dcp) kick(dcp); } +static void dcp_backlight_fade_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), + struct apple_dcp, + backlight.fade_work); + unsigned long flags; + bool running, kick; + u32 elapsed; + + if (READ_ONCE(dcp->quiescing) || READ_ONCE(dcp->crashed) || + READ_ONCE(dcp->iomfb.stopped)) + return; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + elapsed = jiffies_to_msecs(jiffies - dcp->backlight.fade_start); + running = dcp_bl_fade_step(&dcp->backlight.state, elapsed, + DCP_BL_FADE_MS); + /* A rejected present is retried with backoff, carrying the new level. */ + kick = dcp->backlight.state.dirty && !dcp->backlight.state.retries; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + + if (kick) + dcp_backlight_kick(dcp); + if (running) + queue_delayed_work(system_wq, &dcp->backlight.fade_work, + msecs_to_jiffies(DCP_BL_FADE_STEP_MS)); +} + +void dcp_backlight_stop(struct apple_dcp *dcp) +{ + cancel_delayed_work_sync(&dcp->backlight.fade_work); +} + +static void dcp_backlight_release(void *data) +{ + dcp_backlight_stop(data); +} + +/* + * The backlight device is a devres of @dev, registered after probe, so it is + * unregistered before this action runs, and no request can start a fade + * afterwards. + */ +int dcp_backlight_init(struct apple_dcp *dcp, struct device *dev) +{ + spin_lock_init(&dcp->backlight.lock); + INIT_DELAYED_WORK(&dcp->backlight.fade_work, dcp_backlight_fade_work); + return devm_add_action_or_reset(dev, dcp_backlight_release, dcp); +} + +/* Present the target of a running fade at once, e.g. before system sleep. */ +void dcp_backlight_finish_fade(struct apple_dcp *dcp) +{ + unsigned long flags; + bool kick; + + cancel_delayed_work_sync(&dcp->backlight.fade_work); + spin_lock_irqsave(&dcp->backlight.lock, flags); + dcp_bl_fade_step(&dcp->backlight.state, DCP_BL_FADE_MS, DCP_BL_FADE_MS); + kick = dcp->backlight.state.dirty; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + if (kick) + dcp_backlight_kick(dcp); +} + int dcp_backlight_dpms(struct apple_dcp *dcp, bool on) { unsigned long flags; @@ -395,14 +470,25 @@ static int dcp_set_brightness(struct backlight_device *bd) return -ENODEV; if (dcp_backlight_active(dcp)) { + struct dcp_backlight_state *state = &dcp->backlight.state; + bool fade; + u32 old; + /* Preserve the requested level even while the core forces zero. */ spin_lock_irqsave(&dcp->backlight.lock, flags); - ret = dcp_bl_request(&dcp->backlight.state, - bd->props.brightness, + old = state->target; + ret = dcp_bl_request(state, bd->props.brightness, backlight_is_blank(bd), bd->props.state & BL_CORE_SUSPENDED); + /* A new level restarts the fade from the level presented last. */ + fade = !ret && state->fading && state->target != old; + if (fade) + dcp->backlight.fade_start = jiffies; spin_unlock_irqrestore(&dcp->backlight.lock, flags); - if (!ret) + if (fade) + queue_delayed_work(system_wq, &dcp->backlight.fade_work, + msecs_to_jiffies(DCP_BL_FADE_STEP_MS)); + else if (!ret) dcp_backlight_kick(dcp); return ret; } diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 1232ff40a84ba..92ac07b2e9d9d 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -6,13 +6,20 @@ #include #include +#include #include #include -/* Commanded nits, independent of firmware encoding. No fade policy. */ +/* + * Commanded nits, independent of firmware encoding. target is the requested + * level and level the one presented while the panel is lit. They differ only + * while a fade runs; the caller supplies the fade's clock. + */ struct dcp_backlight_state { u32 maximum; u32 target; + u32 level; + u32 fade_from; u32 actual; u32 sent_nits; u64 sequence; @@ -24,6 +31,10 @@ struct dcp_backlight_state { bool suspended; bool dirty; bool in_flight; + /* Fade between lit levels instead of jumping to them. */ + bool fade; + /* A fade from fade_from to target is running. */ + bool fading; /* * The panel shows a level Linux knows: the loader's reported level, or * one a completed present carried. A default used for registration @@ -73,6 +84,7 @@ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, *state = (struct dcp_backlight_state) { .maximum = maximum, .target = nits, + .level = nits, .actual = nits, .ready = true, .level_known = inherited_valid, @@ -89,21 +101,40 @@ static inline bool dcp_bl_seed(struct dcp_backlight_state *state, u32 nits) if (!state->ready || state->controlled || nits > state->maximum) return false; state->target = nits; + state->level = nits; state->actual = nits; state->level_known = true; return true; } +static inline bool dcp_bl_lit(const struct dcp_backlight_state *state) +{ + return !state->dpms_off && !state->core_blank && !state->suspended; +} + static inline u32 dcp_bl_effective(const struct dcp_backlight_state *state) { - if (state->dpms_off || state->core_blank || state->suspended) - return 0; - return state->target; + return dcp_bl_lit(state) ? state->level : 0; } +/* Stop a running fade and present the target directly. */ +static inline void dcp_bl_jump(struct dcp_backlight_state *state) +{ + state->fading = false; + state->level = state->target; + state->dirty = true; +} + +/* + * With fades enabled, a new level for a panel that stays lit at a known level + * starts a fade from the level presented last; dcp_bl_fade_step() moves it. + * Blanking, unblanking and a level the panel may not show still jump. + */ static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, bool core_blank, bool suspended) { + bool was_lit = dcp_bl_lit(state); + if (!state->ready) return -ENODATA; if (nits > state->maximum) @@ -119,11 +150,44 @@ static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, state->target = nits; state->core_blank = core_blank; state->suspended = suspended; - state->dirty = true; + if (state->fade && state->level_known && was_lit && + dcp_bl_lit(state) && state->level != nits) { + state->fade_from = state->level; + state->fading = true; + } else { + dcp_bl_jump(state); + } } return 0; } +/* + * Move a running fade to @elapsed of @duration, in any one unit. The level + * follows a straight line from the level presented when the fade started, + * and the last step is the target itself. Returns true while the fade runs. + */ +static inline bool dcp_bl_fade_step(struct dcp_backlight_state *state, + u32 elapsed, u32 duration) +{ + u32 from = state->fade_from, to = state->target, level; + + if (!state->fading) + return false; + if (elapsed >= duration) { + level = to; + state->fading = false; + } else if (to >= from) { + level = from + div_u64(mul_u32_u32(to - from, elapsed), duration); + } else { + level = from - div_u64(mul_u32_u32(from - to, elapsed), duration); + } + if (state->level != level) { + state->level = level; + state->dirty = true; + } + return state->fading; +} + static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) { if (!state->ready) @@ -132,7 +196,7 @@ static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) state->retries = 0; if (state->dpms_off != !on) { state->dpms_off = !on; - state->dirty = true; + dcp_bl_jump(state); } return 0; } diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index 41ac118f26cee..a6fb45958b168 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -352,6 +352,155 @@ static void backlight_seed_known_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); } +static void backlight_fade_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 100, false, 0), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.fading); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 100U); + + /* The start of the fade is the level already shown. */ + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 0, 250)); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 125, 250)); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 250U); + + /* Steps during a present merge into the next one. */ + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 150, 250)); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 200, 250)); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &present), -EBUSY); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 250U); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 340U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + + /* The last step is exactly the target, however late it runs. */ + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 1000, 250)); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 400U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 400U); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 250, 250)); + KUNIT_EXPECT_FALSE(test, state.dirty); + + /* Fading down reaches the target too, including zero. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 0, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 249, 250)); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 2U); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 250, 250)); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); +} + +static void backlight_fade_restart_test(struct kunit *test) +{ + struct dcp_backlight_state state; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 100, false, 0), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 100, 250)); + KUNIT_EXPECT_EQ(test, state.level, 260U); + + /* Repeating the target keeps the running fade. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.fading); + KUNIT_EXPECT_EQ(test, state.fade_from, 100U); + + /* A new level replaces it, starting from the level presented last. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 60, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.fading); + KUNIT_EXPECT_EQ(test, state.fade_from, 260U); + KUNIT_EXPECT_EQ(test, state.level, 260U); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 125, 250)); + KUNIT_EXPECT_EQ(test, state.level, 160U); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 250, 250)); + KUNIT_EXPECT_EQ(test, state.level, 60U); + + /* A new level equal to the one presented ends the fade there. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 300, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 125, 250)); + KUNIT_EXPECT_EQ(test, state.level, 180U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 180, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 180U); + + /* No overflow over the whole range a panel ceiling may have. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, INT_MAX, true, 0, false, 0), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, INT_MAX, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 249, 250)); + KUNIT_EXPECT_EQ(test, state.level, (u32)div_u64((u64)INT_MAX * 249, 250)); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 250, 250)); + KUNIT_EXPECT_EQ(test, state.level, (u32)INT_MAX); +} + +static void backlight_fade_blank_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* DPMS off and on during a fade jump, as without fades. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 100, false, 0), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 125, 250)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 200, 250)); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 400U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + + /* A level set while blanked is shown at once on unblank. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, true, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 200, true, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 200, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 200U); + + /* Suspend during a fade blanks at once; resume shows the target. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 50, 250)); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, true, true), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 500U); + + /* A level the panel may not show yet is presented directly. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 100, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 100U); + + /* Without fades a new level is presented directly. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 100, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 400U); +} + static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), @@ -366,6 +515,9 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_fallback_test), KUNIT_CASE(backlight_fallback_same_level_test), KUNIT_CASE(backlight_seed_known_test), + KUNIT_CASE(backlight_fade_test), + KUNIT_CASE(backlight_fade_restart_test), + KUNIT_CASE(backlight_fade_blank_test), {} }; From 9cb00d244c757259a2283c7890c257eebf3bc780 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:37:55 +1100 Subject: [PATCH 0820/1579] btrfs: keep the extent map referenced until a compressed read is submitted When btrfs_do_readpage() adds blocks of a compressed extent to a bio it drops its reference on the extent map, and btrfs_submit_compressed_read() looks the extent map up again in the inode's extent map tree, by the bio's file offset, to learn where the extent starts in the file and how it is compressed. If the lookup misses, the whole bio fails with -EIO. The lookup can miss. The bio is submitted when a later block does not fit it, or at the end of the read_folio/readahead call after the range has been unlocked, and the extent map shrinker can remove the extent map from the tree in between: apart from pinned extent maps it drops whatever it finds, regardless of locked io tree ranges or of references held by readers. Under memory pressure, in particular in a memory cgroup at its limit whose reclaim keeps invoking the fs shrinker, this shows up as spurious -EIO on reads of compressed files, and on buffered writes that have to read a partial folio first. Have the bio_ctrl hold a reference on the extent map of the compressed extent its bio reads, handed over by btrfs_do_readpage() once the block is in the bio and dropped once the bio is submitted, and pass it to btrfs_submit_compressed_read() instead of looking it up again. Fixes: 956a17d9d050 ("btrfs: add a shrinker for extent maps") Assisted-by: LLM Signed-off-by: Ryan Murray --- fs/btrfs/compression.c | 18 +++++++--------- fs/btrfs/compression.h | 3 ++- fs/btrfs/extent_io.c | 47 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 2ceb5661e0718..96ae95c3029bc 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -540,11 +540,10 @@ static noinline int add_ra_bio_pages(struct inode *inode, * After the compressed pages are read, we copy the bytes into the * bio we were passed and then call the bio end_io calls */ -void btrfs_submit_compressed_read(struct btrfs_bio *bbio) +void btrfs_submit_compressed_read(struct btrfs_bio *bbio, struct extent_map *em) { struct btrfs_inode *inode = bbio->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct extent_map_tree *em_tree = &inode->extent_tree; struct compressed_bio *cb; unsigned int compressed_len; const u32 min_folio_size = btrfs_min_folio_size(fs_info); @@ -552,7 +551,6 @@ void btrfs_submit_compressed_read(struct btrfs_bio *bbio) gfp_t gfp; u64 em_len; u64 em_start; - struct extent_map *em; unsigned long pflags; int memstall = 0; int ret; @@ -568,16 +566,18 @@ void btrfs_submit_compressed_read(struct btrfs_bio *bbio) else gfp = GFP_NOFS; - /* we need the actual starting offset of this extent in the file */ - read_lock(&em_tree->lock); - em = btrfs_lookup_extent_mapping(em_tree, file_offset, fs_info->sectorsize); - read_unlock(&em_tree->lock); - if (!em) { + /* + * We need the actual starting offset of this extent in the file. The + * caller holds a reference on the extent map the bio was built from; + * it may no longer be in the inode's extent map tree. + */ + if (WARN_ON_ONCE(!em)) { ret = -EIO; goto out; } ASSERT(btrfs_extent_map_is_compressed(em)); + ASSERT(em->start <= file_offset && file_offset < btrfs_extent_map_end(em)); compressed_len = em->disk_num_bytes; cb = alloc_compressed_bio(inode, file_offset, REQ_OP_READ, @@ -592,8 +592,6 @@ void btrfs_submit_compressed_read(struct btrfs_bio *bbio) cb->orig_bbio = bbio; cb->bbio.csum_search_commit_root = bbio->csum_search_commit_root; - btrfs_free_extent_map(em); - for (int i = 0; i * min_folio_size < compressed_len; i++) { struct folio *folio; u32 cur_len = min(compressed_len - i * min_folio_size, min_folio_size); diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h index 1022dc53ec51e..6ccbcfaf3280e 100644 --- a/fs/btrfs/compression.h +++ b/fs/btrfs/compression.h @@ -20,6 +20,7 @@ struct address_space; struct inode; struct btrfs_inode; struct btrfs_ordered_extent; +struct extent_map; /* * We want to make sure that amount of RAM required to uncompress an extent is @@ -94,7 +95,7 @@ struct compressed_bio *btrfs_alloc_compressed_write(struct btrfs_inode *inode, u64 start, u64 len); void btrfs_submit_compressed_write(struct btrfs_ordered_extent *ordered, struct compressed_bio *cb); -void btrfs_submit_compressed_read(struct btrfs_bio *bbio); +void btrfs_submit_compressed_read(struct btrfs_bio *bbio, struct extent_map *em); int btrfs_compress_str2level(unsigned int type, const char *str, int *level_ret); diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 741fd1d4ab2ea..eddc6a834f214 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -150,6 +150,16 @@ struct btrfs_bio_ctrl { * inside the same inode. */ u64 last_em_start; + + /* + * For a compressed read, the extent map of the compressed extent that + * @bbio reads, referenced until the bio is submitted. + * + * btrfs_submit_compressed_read() needs it, but must not look it up in + * the inode's extent map tree again: by then the extent map shrinker + * may have removed it from the tree. + */ + struct extent_map *compressed_em; }; /* @@ -176,8 +186,10 @@ static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) { struct btrfs_bio *bbio = bio_ctrl->bbio; - if (!bbio) + if (!bbio) { + ASSERT(!bio_ctrl->compressed_em); return; + } /* Caller should ensure the bio has at least some range added */ ASSERT(bbio->bio.bi_iter.bi_size); @@ -186,12 +198,14 @@ static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) if (btrfs_op(&bbio->bio) == BTRFS_MAP_READ && bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) - btrfs_submit_compressed_read(bbio); + btrfs_submit_compressed_read(bbio, bio_ctrl->compressed_em); else btrfs_submit_bbio(bbio, 0); /* The bbio is owned by the end_io handler now */ bio_ctrl->bbio = NULL; + btrfs_free_extent_map(bio_ctrl->compressed_em); + bio_ctrl->compressed_em = NULL; /* * We used the generation to decide whether to lookup csums in the * commit_root or not when we called bio_set_csum_search_commit_root() @@ -963,6 +977,26 @@ static struct extent_map *get_extent_map(struct btrfs_inode *inode, return em; } +/* + * A block of the compressed extent mapped by @em has just been added to + * @bio_ctrl->bbio, which only ever reads blocks of one compressed extent. + * Take over the caller's reference on @em: bio_ctrl keeps one reference on + * the extent map of its bio until submit_one_bio() passes it on. + */ +static void bio_ctrl_hold_compressed_em(struct btrfs_bio_ctrl *bio_ctrl, + struct extent_map *em) +{ + struct extent_map *old = bio_ctrl->compressed_em; + + /* + * Any reference held so far is for an earlier block of the same bio, + * on the same extent map or on a copy of it loaded again after the + * shrinker dropped it: one is enough. + */ + bio_ctrl->compressed_em = em; + btrfs_free_extent_map(old); +} + static void btrfs_readahead_expand(struct readahead_control *ractl, const struct extent_map *em) { @@ -1121,17 +1155,17 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, bio_ctrl->last_em_start = em->start; em_gen = em->generation; - btrfs_free_extent_map(em); - em = NULL; /* we've found a hole, just zero and go on */ if (block_start == EXTENT_MAP_HOLE) { + btrfs_free_extent_map(em); folio_zero_range(folio, pg_offset, blocksize); end_folio_read(vi, folio, true, cur, blocksize); continue; } /* the get_extent function already copied into the folio */ if (block_start == EXTENT_MAP_INLINE) { + btrfs_free_extent_map(em); end_folio_read(vi, folio, true, cur, blocksize); continue; } @@ -1145,6 +1179,11 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, submit_one_bio(bio_ctrl); submit_extent_folio(bio_ctrl, disk_bytenr, folio, blocksize, pg_offset, em_gen); + + if (compress_type != BTRFS_COMPRESS_NONE) + bio_ctrl_hold_compressed_em(bio_ctrl, em); + else + btrfs_free_extent_map(em); } return 0; } From a45a44cc6e2b6bbdb189debda045a2174d07e039 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 19:02:56 +1100 Subject: [PATCH 0821/1579] rust: drm: gem: shmem: let drivers allocate the objects the core creates The DRM core allocates some GEM objects itself. drm_gem_shmem_prime_import_sg_table() and drm_gem_shmem_dumb_create() both go through __drm_gem_shmem_create(), which allocates a bare struct drm_gem_shmem_object unless the driver provides gem_create_object. The shmem abstraction offers both helpers but no gem_create_object, while lookup_handle() and the GEM object callbacks take every object of the driver to be an Object. An imported dma-buf or a dumb buffer therefore reaches the driver as an Object whose Rust fields lie past the end of its allocation. Add an optional DriverObject::core_object_args() that gives the arguments for such objects. When a driver provides it, a gem_create_object callback allocates the whole Object and leaves the shmem part to the core, as the C shmem drivers do with their own gem_create_object. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/drm/gem/mod.rs | 9 +++++++ rust/kernel/drm/gem/shmem.rs | 48 +++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index bea304395d67c..42892c71f84d0 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -109,6 +109,15 @@ pub trait DriverObject: Sync + Send + Sized { ) -> Result::Object>> { unimplemented!() } + + /// Arguments for an object that the DRM core allocates on the driver's behalf, such as a + /// dma-buf imported from another device or a dumb buffer. + /// + /// The shmem helpers build the driver data of such an object from these arguments. If the + /// core then fails to initialize the object, it frees it without dropping the driver data. + fn core_object_args(_dev: &drm::Device, _size: usize) -> Result { + build_error!(crate::error::VTABLE_DEFAULT_ERROR) + } } /// Trait that represents a GEM object subtype diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index f0e49537e0c3a..5ca039e949637 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -202,6 +202,48 @@ impl Object { let _ = unsafe { KBox::from_raw(this) }; } + /// `gem_create_object` callback, for the objects the DRM core allocates on the driver's + /// behalf (dma-buf imports and dumb buffers) before it initializes their shmem part. + /// + /// Every GEM object of the driver is taken to be an `Object`, these included, so build one + /// with the driver data from [`DriverObject::core_object_args`]. + extern "C" fn create_object_callback( + raw_dev: *mut bindings::drm_device, + size: usize, + ) -> *mut bindings::drm_gem_object { + // SAFETY: The DRM core only calls this for a device of `T::Driver`, which outlives the + // call. + let dev = unsafe { device::Device::::from_raw(raw_dev) }; + + let new = T::core_object_args(dev, size).and_then(|args| { + KBox::try_pin_init( + try_pin_init!(Self { + obj <- Opaque::init_zeroed(), + parent_resv_obj: None, + sgt_access: AtomicUsize::new(0), + inner <- T::new(dev, size, args), + }), + GFP_KERNEL, + ) + }); + let new = match new { + Ok(new) => new, + Err(e) => return e.to_ptr(), + }; + + // SAFETY: We never move out of `self`. The DRM core owns the object from here on and + // releases it through `free_callback()`. An object the core then fails to initialize is + // freed with `kfree()`, which the `KBox` allocator allows, without dropping `inner`. + let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); + + // SAFETY: `new` is a valid object that nothing else references yet. + unsafe { + let obj = (*new).obj.get(); + (*obj).base.funcs = &Self::VTABLE; + &raw mut (*obj).base + } + } + /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA /// pages for this object. /// @@ -446,7 +488,11 @@ impl driver::AllocImpl for Object { type Driver = T::Driver; const ALLOC_OPS: driver::AllocOps = driver::AllocOps { - gem_create_object: None, + gem_create_object: if T::HAS_CORE_OBJECT_ARGS { + Some(Self::create_object_callback) + } else { + None + }, prime_handle_to_fd: None, prime_fd_to_handle: None, gem_prime_import: None, From c0e47197c71c80b57974130a06386e101bbdc7ee Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 19:02:58 +1100 Subject: [PATCH 0822/1579] drm/asahi: allocate imported and dumb GEM objects as asahi objects A dma-buf imported from another device comes back from drm_gem_shmem_prime_import_sg_table() as a bare 456-byte struct drm_gem_shmem_object. gnome-shell imports one at startup and binds it on every frame. ASAHI_VM_BIND then reads the object's flags and the SG lease counter behind its end, and updates the counter there. KASAN reports it on every boot: BUG: KASAN: slab-out-of-bounds in ...ASAHI_VM_BIND+0xb7c/0x1840 [asahi] Read of size 4 at addr ffff00002ac179f0 by task gnome-shell/1175 BUG: KASAN: slab-out-of-bounds in ...Object::owned_sg_table+0x48/0x11c [asahi] Write of size 8 at addr ffff00002ac179d0 by task gnome-shell/1175 ... The buggy address is located 40 bytes to the right of allocated 456-byte region [ffff00002ac17800, ffff00002ac179c8) A dumb buffer created on the card node has the same problem. Give the DRM core the arguments for the objects it allocates, so that imports and dumb buffers are allocated as whole asahi objects: shared, neither VM-private nor kernel-owned, like a GEM_CREATE object without flags. They now also get the driver's GEM callbacks, so closing the last handle in a file unbinds them from that file's VMs, as it does for every other object. Fixes: 80df573af9ef ("rust: drm: gem: shmem: Add DRM shmem helper abstraction") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/gem.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index bc14223a837d5..7b0a933cdc725 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -296,6 +296,16 @@ impl DriverObject for AsahiObject { }) } + /// Objects the DRM core allocates for us (dma-bufs imported from other devices and dumb + /// buffers) are shared user objects, like a GEM_CREATE object without flags. + fn core_object_args(_dev: &AsahiDevice, _size: usize) -> Result { + Ok(AsahiObjConfig { + flags: 0, + exportable: true, + kernel: false, + }) + } + /// Callback to drop all mappings for a GEM object owned by a given `File` fn close(obj: &::Object, file: &drm::gem::DriverFile) { // fn close(obj: &Object, file: &DrmFile) { From eba6449864e8e41f891e95acbf9ce0afeedb458f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 14:03:58 +1100 Subject: [PATCH 0823/1579] media: apple: isp: Keep the driver state until the last file is closed The driver state, which embeds the video device, its queue and lock, the control handler and the media device, is allocated with devm_kzalloc() and freed as soon as the driver is unbound. A process that still has the video device open at unbind keeps a reference to the video device, and when it closes the file, vb2_fop_release() takes the queue lock and v4l2_fh_release() walks the file handle lists, all in freed memory. A control event subscription also reaches the freed control handler. Tie the state to the v4l2_device reference count as V4L2 drivers do: allocate it with kzalloc(), free it, the control handler and the media device from the v4l2_device release callback, and drop the driver's own reference at the end of remove. The video device holds the other one while it is registered or open, so the state lives until the last file is closed. Before the video device is registered nothing else can reach the state, so a failed probe still frees it directly. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 17 ++++++++++---- drivers/media/platform/apple/isp/isp-v4l2.c | 25 +++++++++++++++++++-- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 9f5f8f58b3748..71ba9c7939470 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -380,7 +380,11 @@ static int apple_isp_probe(struct platform_device *pdev) if (err) return err; - isp = devm_kzalloc(dev, sizeof(*isp), GFP_KERNEL); + /* + * Not devres: an open video device file keeps this until it is + * closed, also after unbind. See apple_isp_release(). + */ + isp = kzalloc_obj(*isp); if (!isp) return -ENOMEM; @@ -399,7 +403,7 @@ static int apple_isp_probe(struct platform_device *pdev) if (err) { dev_err(dev, "failed to get 'apple,platform-id' property: %d\n", err); - return err; + goto free_isp; } err = of_property_read_u32(dev->of_node, "apple,temporal-filter", @@ -410,13 +414,13 @@ static int apple_isp_probe(struct platform_device *pdev) err = apple_isp_init_presets(isp); if (err) { dev_err(dev, "failed to initialize presets\n"); - return err; + goto free_isp; } err = apple_isp_attach_genpd(isp); if (err) { dev_err(dev, "failed to attatch power domains\n"); - return err; + goto free_isp; } isp->coproc = devm_platform_ioremap_resource_byname(pdev, "coproc"); @@ -529,6 +533,9 @@ static int apple_isp_probe(struct platform_device *pdev) destroy_workqueue(isp->wq); detach_genpd: apple_isp_detach_genpd(isp); +free_isp: + /* Nothing else can reach it before the video device is registered. */ + kfree(isp); return err; } @@ -545,6 +552,8 @@ static void apple_isp_remove(struct platform_device *pdev) apple_isp_free_iommu(isp); destroy_workqueue(isp->wq); apple_isp_detach_genpd(isp); + /* Freed now, or when the last open file is closed. */ + v4l2_device_put(&isp->v4l2_dev); } static const struct apple_isp_hw apple_isp_hw_t8103 = { diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 299177b9075cf..f3056e78a0a17 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -969,6 +969,22 @@ static const struct media_device_ops isp_media_device_ops = { .link_notify = v4l2_pipeline_link_notify, }; +/* + * The video device's file operations can run until its last open file is + * closed, after the driver has been unbound, so the state they reach is freed + * only with the last reference to the v4l2_device: the one remove drops, or + * the one the video device holds while it is open. + */ +static void apple_isp_release(struct v4l2_device *v4l2_dev) +{ + struct apple_isp *isp = container_of(v4l2_dev, struct apple_isp, + v4l2_dev); + + v4l2_ctrl_handler_free(&isp->ctrl_handler); + media_device_cleanup(&isp->mdev); + kfree(isp); +} + static void isp_free_meta_surfaces(struct apple_isp *isp) { for (int i = 0; i < ARRAY_SIZE(isp->capmeta_surfs); i++) { @@ -1093,6 +1109,9 @@ int apple_isp_setup_video(struct apple_isp *isp) goto v4l2_unregister; } + /* From here on, the last v4l2_device reference frees the state. */ + isp->v4l2_dev.release = apple_isp_release; + return 0; v4l2_unregister: @@ -1108,13 +1127,15 @@ int apple_isp_setup_video(struct apple_isp *isp) return err; } +/* + * Unregister the device nodes. An open file keeps the video device, and + * with it the driver state, until it is closed: see apple_isp_release(). + */ void apple_isp_remove_video(struct apple_isp *isp) { vb2_video_unregister_device(&isp->vdev); v4l2_device_unregister(&isp->v4l2_dev); - v4l2_ctrl_handler_free(&isp->ctrl_handler); media_device_unregister(&isp->mdev); - media_device_cleanup(&isp->mdev); } /* After apple_isp_remove_video() and with the firmware stopped */ From 87eadf24b7dc989d8877ecb50554977be193b8ed Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 15:36:14 +1100 Subject: [PATCH 0824/1579] tools: aurora-sep: report unusable captures to libfprint as retries The kernel can tell userspace that an enrolment capture was not usable: the next poll while the enrolment runs carries SEP_BIO_STATUS_RETRY, and a run of unusable captures ends the enrolment with SEP_BIO_STATUS_REJECTED. A verify whose capture was not usable already ends with SEP_BIO_STATUS_RETRY. The aurora driver ignored the status while an enrolment ran and reported any failure only as a driver status number, so nobody was asked to touch the sensor again, and a match that compared nothing ended the authentication attempt as an error. Report a rejected enrolment capture through fpi_device_enroll_progress() with FP_DEVICE_RETRY_GENERAL, which fprintd shows as enroll-retry-scan. The kernel does not say why a capture was unusable, so the general code is the most specific one that is always true. Report a verify or identify that ended with RETRY as a retry too; fprintd answers that by starting the match again. Fail a rejected enrolment with a message that says why and what to do, and name the reason for the other failure statuses. The status values are defined here when the UAPI header predates them. With a kernel that never sends RETRY during an enrolment, nor REJECTED, the driver behaves as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 6 + .../libfprint-1.94.100-apple-sep.patch | 136 ++++++++++++++++-- 2 files changed, 129 insertions(+), 13 deletions(-) diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 300fa97c935a5..8e85dec103414 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -73,6 +73,12 @@ number increases; guidance and stage-zero events are not captured samples. fprintd 1.94.5 itself emits an initial `enroll-stage-passed` after its duplicate check, **before** starting enrollment. That first line does not prove a finger was captured. +A capture the enclave cannot use is reported as a retry, both while enrolling +and while matching: fprintd shows `enroll-retry-scan` or `verify-retry-scan`, +and starts a match again by itself. When the kernel ends an enrolment because +too many captures in a row were unusable (`SEP_BIO_STATUS_REJECTED`), the +enrolment fails with that reason in fprintd's log, and fprintd reports +`enroll-unknown-error`. Kernels from before these statuses never send them. The device-specific `apple/mesa_calibration.bin` on m2 was recovered using the bundled [`extract-mesa-calibration.py`](extract-mesa-calibration.py), copied from [Gist revision `eb079a8007985d04e75182f20994f1a1c496f12e`](https://gist.github.com/DjDeveloperr/867a1961b861c570442724f48f770158/eb079a8007985d04e75182f20994f1a1c496f12e), diff --git a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch index bdb15bb1c2b27..5163351683362 100644 --- a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch +++ b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch @@ -2,7 +2,7 @@ diff --git a/libfprint/drivers/aurora/aurora.c b/libfprint/drivers/aurora/aurora new file mode 100644 --- /dev/null +++ b/libfprint/drivers/aurora/aurora.c -@@ -0,0 +1,881 @@ +@@ -0,0 +1,991 @@ +/* + * Aurora SEP fingerprint driver + * @@ -28,6 +28,20 @@ new file mode 100644 +#include +#include + ++/* Why the kernel ended an enrol or verify, in a poll's 'status'. Headers from ++ * before these were named carry the same numbers; those kernels never report ++ * RETRY while an enrolment is running, nor REJECTED at all. */ ++#ifndef SEP_BIO_STATUS_REJECTED ++#define SEP_BIO_STATUS_NONE 0 ++#define SEP_BIO_STATUS_SENSOR 1 ++#define SEP_BIO_STATUS_ENCLAVE 2 ++#define SEP_BIO_STATUS_TOO_MANY 3 ++#define SEP_BIO_STATUS_TIMEOUT 4 ++#define SEP_BIO_STATUS_RETRY 5 ++#define SEP_BIO_STATUS_UNFILED 6 ++#define SEP_BIO_STATUS_REJECTED 7 ++#endif ++ +struct _FpiDeviceAurora +{ + FpDevice parent; @@ -95,6 +109,65 @@ new file mode 100644 + return TRUE; +} + ++static const gchar * ++aurora_status_reason (guint32 status) ++{ ++ switch (status) ++ { ++ case SEP_BIO_STATUS_NONE: ++ return "it was cancelled or ended by the system"; ++ ++ case SEP_BIO_STATUS_SENSOR: ++ return "the fingerprint sensor failed"; ++ ++ case SEP_BIO_STATUS_ENCLAVE: ++ return "the secure enclave refused a step"; ++ ++ case SEP_BIO_STATUS_TOO_MANY: ++ return "the enclave did not complete the enrolment within its captures"; ++ ++ case SEP_BIO_STATUS_TIMEOUT: ++ return "no finger was placed on the sensor in time"; ++ ++ case SEP_BIO_STATUS_RETRY: ++ return "the capture was not usable"; ++ ++ case SEP_BIO_STATUS_UNFILED: ++ return "the fingerprint was enrolled but could not be recorded"; ++ ++ case SEP_BIO_STATUS_REJECTED: ++ return "too many captures in a row were not usable; " ++ "rest more of the finger flat on the sensor and enrol again"; ++ ++ default: ++ return NULL; ++ } ++} ++ ++static GError * ++aurora_failure_new (const gchar *operation, guint32 status) ++{ ++ const gchar *reason = aurora_status_reason (status); ++ ++ if (reason == NULL) ++ return fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, ++ "%s failed, driver status 0x%x", ++ operation, status); ++ ++ return fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, ++ "%s failed: %s (driver status %u)", ++ operation, reason, status); ++} ++ ++/* The kernel reports an unusable capture without saying why it was unusable, ++ * so the general retry is the most specific code that is always true. */ ++static GError * ++aurora_retry_new (void) ++{ ++ return fpi_device_retry_new_msg (FP_DEVICE_RETRY_GENERAL, ++ "the capture was not usable; touch the sensor again"); ++} ++ +/* The label is how the host recognises its own records in the device's list. + * libfprint already has an encoding for exactly this -- it carries the enroll + * date, the finger and the username -- so use it rather than inventing one. */ @@ -402,16 +475,23 @@ new file mode 100644 + switch (poll_res.state) + { + case SEP_BIO_STATE_PENDING: -+ return G_SOURCE_CONTINUE; -+ + case SEP_BIO_STATE_PROGRESS: + /* fprintd calls this "enroll-stage-passed". A guidance wakeup or an + * initial stage-zero event must not masquerade as a captured sample. */ -+ if (poll_res.stage > self->enroll_last_reported_stage) ++ if (poll_res.state == SEP_BIO_STATE_PROGRESS && ++ poll_res.stage > self->enroll_last_reported_stage) + { + self->enroll_last_reported_stage = poll_res.stage; + fpi_device_enroll_progress (device, poll_res.stage, NULL, NULL); + } ++ ++ /* The latest capture did not count, and the enrolment goes on. The ++ * kernel reports each rejection in one poll only, and an accepted ++ * capture clears it, so a rejection that shares a poll with a new stage ++ * came after that stage. */ ++ if (poll_res.status == SEP_BIO_STATUS_RETRY) ++ fpi_device_enroll_progress (device, self->enroll_last_reported_stage, ++ NULL, aurora_retry_new ()); + return G_SOURCE_CONTINUE; + + case SEP_BIO_STATE_DONE: @@ -434,12 +514,17 @@ new file mode 100644 + return G_SOURCE_REMOVE; + + case SEP_BIO_STATE_FAILED: ++ self->watch_id = 0; ++ fpi_device_enroll_complete (device, NULL, ++ aurora_failure_new ("enrolment", poll_res.status)); ++ return G_SOURCE_REMOVE; ++ + default: + self->watch_id = 0; + fpi_device_enroll_complete (device, NULL, + fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "enrolment failed, driver status 0x%x", -+ poll_res.status)); ++ "enrolment failed, state %u, driver status 0x%x", ++ poll_res.state, poll_res.status)); + return G_SOURCE_REMOVE; + } +} @@ -526,6 +611,17 @@ new file mode 100644 + return TRUE; +} + ++/* A verify that ended without a comparison. When the kernel says why, pass ++ * that on: otherwise the log shows only an unspecified error. */ ++static GError * ++aurora_not_compared_new (const struct sep_bio_verify_poll *res) ++{ ++ if (res->state == SEP_BIO_STATE_FAILED) ++ return aurora_failure_new ("verification", res->status); ++ ++ return fpi_device_error_new (FP_DEVICE_ERROR_GENERAL); ++} ++ +static gboolean +aurora_verify_ready (gint fd, GIOCondition condition, gpointer user_data) +{ @@ -539,6 +635,7 @@ new file mode 100644 + * reference under libfprint's feet, and it frees the print again when the + * match completes. */ + FpPrint *matched = NULL; ++ GError *retry = NULL; + + if (!aurora_ioctl (self, SEP_BIO_VERIFY_POLL, &res, &error)) + { @@ -567,7 +664,12 @@ new file mode 100644 + + if (aurora_result_is_match (self, &res)) + matched = aurora_print_from_uuid (self, res.uuid, NULL); -+ ++ /* The capture was not usable, so nothing was compared. A retry tells the ++ * caller to ask for another touch: fprintd starts the match again on its ++ * own, where a failure would end the authentication attempt. */ ++ else if (res.state == SEP_BIO_STATE_FAILED && ++ res.status == SEP_BIO_STATUS_RETRY) ++ retry = aurora_retry_new (); + + if (action == FPI_DEVICE_ACTION_VERIFY) + { @@ -575,7 +677,12 @@ new file mode 100644 + + fpi_device_get_verify_data (device, &expected); + -+ if (matched != NULL && aurora_print_has_uuid (expected, res.uuid)) ++ if (retry != NULL) ++ { ++ fpi_device_verify_report (device, FPI_MATCH_ERROR, NULL, retry); ++ fpi_device_verify_complete (device, NULL); ++ } ++ else if (matched != NULL && aurora_print_has_uuid (expected, res.uuid)) + { + fpi_device_verify_report (device, FPI_MATCH_SUCCESS, matched, NULL); + fpi_device_verify_complete (device, NULL); @@ -585,8 +692,7 @@ new file mode 100644 + /* No comparison happened, so there is no verdict to report. Saying + * FAIL here would tell the user their finger was rejected when it + * was never examined. */ -+ fpi_device_verify_complete (device, -+ fpi_device_error_new (FP_DEVICE_ERROR_GENERAL)); ++ fpi_device_verify_complete (device, aurora_not_compared_new (&res)); + } + else + { @@ -617,10 +723,14 @@ new file mode 100644 + } + } + -+ if (matched == NULL && res.result == SEP_BIO_NOT_COMPARED) ++ if (retry != NULL) ++ { ++ fpi_device_identify_report (device, NULL, NULL, retry); ++ fpi_device_identify_complete (device, NULL); ++ } ++ else if (matched == NULL && res.result == SEP_BIO_NOT_COMPARED) + { -+ fpi_device_identify_complete (device, -+ fpi_device_error_new (FP_DEVICE_ERROR_GENERAL)); ++ fpi_device_identify_complete (device, aurora_not_compared_new (&res)); + } + else + { From 89288388f2119d2295191cdbfa31706a83aa1fff Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 15:36:15 +1100 Subject: [PATCH 0825/1579] tools: aurora-sep: give the SEP loader time for a slow enclave load-driver tries to seal and unseal a trusted key 300 times, 0.1 s apart, then waits as long again for /dev/sep-bio. On J700 the enclave's key store arrives around 43 s after boot, and the driver itself waits up to 90 s for it, so the loader can give up before Touch ID comes up. The service then fails, and fprintd, which requires it, fails with it. Measure the waits in seconds of uptime rather than in attempts, wait up to 120 s for the trusted key and 60 s for the device, and let AURORA_SEP_READY_TIMEOUT and AURORA_SEP_DEVICE_TIMEOUT override them. Log why the loader gives up and how long each step took, so a slow boot can be told from a broken one. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 11 +++++++ tools/aurora-sep/load-driver | 59 ++++++++++++++++++++++++++++++------ 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 8e85dec103414..8cc7fc4279419 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -17,6 +17,17 @@ systemctl daemon-reload systemctl enable aurora-sep.service ``` +The loader waits up to 120 s for the enclave to seal and unseal a trusted key, +then up to 60 s for `/dev/sep-bio`, and logs how long each took. On J700 the +enclave's key store arrives around 43 s after boot. To change a wait, set +`AURORA_SEP_READY_TIMEOUT` or `AURORA_SEP_DEVICE_TIMEOUT` in whole seconds, +for example with `systemctl edit aurora-sep.service`: + +```ini +[Service] +Environment=AURORA_SEP_READY_TIMEOUT=180 +``` + `xart_writes=0` remains the default. `load-driver` explicitly opts into writes and must only be enabled on a machine whose APFS `.gl` extent passes the driver's narrow ownership checks: one physical extent, no snapshots or revert diff --git a/tools/aurora-sep/load-driver b/tools/aurora-sep/load-driver index a70db75e6b25b..fbf43e5093f00 100755 --- a/tools/aurora-sep/load-driver +++ b/tools/aurora-sep/load-driver @@ -4,6 +4,34 @@ set -eu keybag=/var/lib/aurora-sep-keybag.bin refkey=/var/lib/aurora-sep-refkey.bin +# Seconds to wait for the enclave to seal and unseal a trusted key, then for +# /dev/sep-bio. On J700 the key store arrives around 43 s after boot, and the +# driver waits up to 90 s for it before it brings Touch ID up, so the first +# wait has to outlast that. Set either in the environment, for example with a +# drop-in for aurora-sep.service. +ready_timeout=${AURORA_SEP_READY_TIMEOUT:-120} +device_timeout=${AURORA_SEP_DEVICE_TIMEOUT:-60} + +log() { + echo "aurora-sep-load: $*" >&2 +} + +for budget in "$ready_timeout" "$device_timeout"; do + case "$budget" in + '' | *[!0-9]*) + log "timeouts must be whole seconds, not '$budget'" + exit 2 + ;; + esac +done + +# Seconds since boot. Unlike the wall clock it cannot jump when the time is +# first synchronised during boot. +uptime_s() { + read -r up _ /dev/null || true) if [ -n "$key_id" ]; then if keyctl pipe "$key_id" >"$work/sealed"; then @@ -37,17 +66,27 @@ while [ "$attempts" -gt 0 ]; do keyctl revoke "$key_id" >/dev/null 2>&1 || true fi [ "$verified" = true ] && break + [ "$(uptime_s)" -lt "$deadline" ] || break sleep 0.1 - attempts=$((attempts - 1)) done -[ "$verified" = true ] || exit 1 -[ -s "$keybag" ] || exit 1 -[ -s "$refkey" ] || exit 1 +if [ "$verified" != true ]; then + log "the enclave did not seal and unseal a trusted key within ${ready_timeout} s" + exit 1 +fi +log "trusted keys work after $(($(uptime_s) - start)) s" + +if [ ! -s "$keybag" ] || [ ! -s "$refkey" ]; then + log "$keybag or $refkey is missing or empty" + exit 1 +fi -attempts=300 -while [ "$attempts" -gt 0 ] && [ ! -c /dev/sep-bio ]; do +deadline=$(($(uptime_s) + device_timeout)) +while [ ! -c /dev/sep-bio ] && [ "$(uptime_s)" -lt "$deadline" ]; do sleep 0.1 - attempts=$((attempts - 1)) done -[ -c /dev/sep-bio ] || exit 1 +if [ ! -c /dev/sep-bio ]; then + log "/dev/sep-bio did not appear within ${device_timeout} s" + exit 1 +fi +log "/dev/sep-bio is ready after $(($(uptime_s) - start)) s" From 8087ed3a7ccbf2712a0fe14e17a78549826f6936 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 15:39:02 +1100 Subject: [PATCH 0826/1579] tools: aurora-sep: add Fedora and Arch Linux packaging A manual install leaves libfprint under /usr/local, where fprintd only finds it through a library path drop-in, and puts the loader and the units under /usr/local and /etc, where no package manager tracks them. Add packaging for the distributions the driver runs on: - packaging/fedora/build.sh rebuilds the release's own libfprint source package with the aurora patch added and an ".aurora1" release suffix, so it replaces the distribution's library in place. It also builds aurora-sep.spec. - packaging/arch has libfprint-aurora, which provides and conflicts with libfprint, and aurora-sep, for Arch Linux and Arch Linux ARM. - aurora-sep installs the loader, its service pointed at the packaged loader, and the fprintd policy as a drop-in under /usr/lib. On Fedora a preset enables the service at boot when the package is first installed; Arch leaves that to the user. The PKGBUILDs take their local sources through links into this tree, and a test checks their checksums against it. The README gives the build, install and rollback steps. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 52 ++++++++++ .../packaging/arch/aurora-sep/PKGBUILD | 44 +++++++++ .../packaging/arch/aurora-sep/README.md | 1 + .../packaging/arch/aurora-sep/XART-SAFETY.md | 1 + .../arch/aurora-sep/aurora-sep.service | 1 + .../arch/aurora-sep/fprintd-aurora.conf | 1 + .../packaging/arch/aurora-sep/load-driver | 1 + .../packaging/arch/libfprint-aurora/PKGBUILD | 73 ++++++++++++++ .../arch/libfprint-aurora/apple-sep.uapi | 1 + .../libfprint-1.94.100-apple-sep.patch | 1 + .../packaging/fedora/aurora-sep.spec | 74 ++++++++++++++ tools/aurora-sep/packaging/fedora/build.sh | 98 +++++++++++++++++++ tools/aurora-sep/tests/test_packaging.py | 72 ++++++++++++++ 13 files changed, 420 insertions(+) create mode 100644 tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/README.md create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/XART-SAFETY.md create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep.service create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/fprintd-aurora.conf create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/load-driver create mode 100644 tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD create mode 120000 tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi create mode 120000 tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch create mode 100644 tools/aurora-sep/packaging/fedora/aurora-sep.spec create mode 100755 tools/aurora-sep/packaging/fedora/build.sh create mode 100644 tools/aurora-sep/tests/test_packaging.py diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 8cc7fc4279419..942d4002683e7 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -144,6 +144,58 @@ lock screen then selects `omarchy-lock-fingerprint` automatically and accepts Touch ID through fprintd. Hyprlock's separate fingerprint switch is not used by the Quickshell lock screen. +## Packages + +`packaging/` builds the same pieces as distribution packages: libfprint with +the aurora driver, which replaces the distribution's libfprint so fprintd needs +no library path, and `aurora-sep` with the loader, its service and the fprintd +policy. Installing `aurora-sep` lets fprintd load the driver with xART writes +on, as above. + +On Fedora, `packaging/fedora/build.sh` rebuilds the release's own libfprint +source package with the patch added (1.94.100 on Fedora 45) and builds +`aurora-sep`. It installs build dependencies, so run it in a throwaway +container from the top of this tree: + +```sh +mkdir rpms +podman run --rm -v "$PWD":/src:ro,z -v "$PWD/rpms":/out:z \ + registry.fedoraproject.org/fedora:45 \ + /src/tools/aurora-sep/packaging/fedora/build.sh /out +sudo dnf install rpms/libfprint-1.94.100-*.aurora1.*.rpm \ + rpms/aurora-sep-1-*.noarch.rpm +``` + +A later Fedora libfprint update replaces the rebuilt one; rebuild after it, or +hold it with `dnf versionlock add libfprint`. + +On Arch Linux and Arch Linux ARM, build `libfprint-aurora` (it replaces +`libfprint`) and then `aurora-sep`: + +```sh +cd tools/aurora-sep/packaging/arch/libfprint-aurora && makepkg -si +cd ../aurora-sep && makepkg -si +``` + +fprintd starts the loader the first time it runs. Loading the driver at boot +instead has the enclave ready before the login screen asks for a fingerprint. +The Fedora package enables that with a systemd preset when it is first +installed; on Arch, enable it yourself: + +```sh +sudo systemctl enable aurora-sep.service +``` + +The driver attaches to the enclave once per boot, so reboot rather than +restarting the service or reloading `apple_sep`. To move from the manual +install above, remove `/usr/local/sbin/aurora-sep-load`, +`/etc/systemd/system/aurora-sep.service`, +`/etc/systemd/system/fprintd.service.d/aurora.conf` and any drop-in that points +fprintd at another libfprint, then run `systemctl daemon-reload` and +`systemctl reenable aurora-sep.service`. To go back to the distribution's +libfprint, remove `aurora-sep` and run `dnf distro-sync libfprint` or +`pacman -S libfprint`. + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD new file mode 100644 index 0000000000000..d7a921a5501cf --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# The loader, service and fprintd policy for the Apple SEP fingerprint driver. +# The local sources are links into tools/aurora-sep; after changing any of +# them, run updpkgsums here. libfprint-aurora provides the libfprint driver. + +pkgname=aurora-sep +pkgver=1 +pkgrel=1 +pkgdesc="Loader and fprintd policy for the Apple SEP (Touch ID) driver" +url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" +arch=(any) +license=(GPL-2.0-only) +depends=( + coreutils + fprintd + keyutils + kmod + sh +) +source=(load-driver + aurora-sep.service + fprintd-aurora.conf + README.md + XART-SAFETY.md) +sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' + 'd3441eeb06c930fcc13314d18012b1f5d71cc1a821ceaf44de13caafb2931d73' + 'f8d0c4b9385e6716d5ac5dd9941a57459fbc551a8238383bcde36cd929d0523f' + 'd6e5c68bab7e0d71da48820da75d4ff7e67a06e7970961377ed28d30c2b66c08' + '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') + +package() { + install -Dm755 load-driver "$pkgdir/usr/lib/aurora-sep/aurora-sep-load" + + install -Dm644 aurora-sep.service "$pkgdir/usr/lib/systemd/system/aurora-sep.service" + sed -i 's|^ExecStart=.*|ExecStart=/usr/lib/aurora-sep/aurora-sep-load|' \ + "$pkgdir/usr/lib/systemd/system/aurora-sep.service" + grep -qx 'ExecStart=/usr/lib/aurora-sep/aurora-sep-load' \ + "$pkgdir/usr/lib/systemd/system/aurora-sep.service" + + install -Dm644 fprintd-aurora.conf \ + "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora-sep.conf" + install -Dm644 -t "$pkgdir/usr/share/doc/$pkgname" README.md XART-SAFETY.md +} diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/README.md b/tools/aurora-sep/packaging/arch/aurora-sep/README.md new file mode 120000 index 0000000000000..8a33348c7d811 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/README.md @@ -0,0 +1 @@ +../../../README.md \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/packaging/arch/aurora-sep/XART-SAFETY.md new file mode 120000 index 0000000000000..45a7d77d7e92e --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/XART-SAFETY.md @@ -0,0 +1 @@ +../../../XART-SAFETY.md \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep.service b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep.service new file mode 120000 index 0000000000000..72bc9aad30d16 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep.service @@ -0,0 +1 @@ +../../../aurora-sep.service \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-aurora.conf b/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-aurora.conf new file mode 120000 index 0000000000000..41efea20ba890 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-aurora.conf @@ -0,0 +1 @@ +../../../fprintd-aurora.conf \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/load-driver b/tools/aurora-sep/packaging/arch/aurora-sep/load-driver new file mode 120000 index 0000000000000..3f9fc69e38042 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/load-driver @@ -0,0 +1 @@ +../../../load-driver \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD b/tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD new file mode 100644 index 0000000000000..12bfdd7222953 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# libfprint with the aurora driver for the Apple SEP fingerprint sensor. +# The local sources are links into tools/aurora-sep and to +# include/uapi/linux/apple-sep.h; after changing either file, run updpkgsums +# here. + +pkgname=libfprint-aurora +_pkgname=libfprint +pkgver=1.94.100 +pkgrel=1 +pkgdesc="Library for fingerprint readers, with the Apple SEP (Touch ID) driver" +url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" +arch=(aarch64 x86_64) +license=(LGPL-2.1-or-later) +depends=( + libgcc + glib2 + glibc + libgudev + libgusb + openssl + pixman +) +makedepends=( + glib2-devel + gobject-introspection + gtk-doc + meson + python-cairo + python-gobject + systemd +) +checkdepends=( + cairo + umockdev +) +provides=("$_pkgname=$pkgver" libfprint-2.so) +conflicts=("$_pkgname") +source=("https://gitlab.freedesktop.org/libfprint/libfprint/-/archive/v$pkgver/$_pkgname-v$pkgver.tar.gz" + "libfprint-$pkgver-apple-sep.patch" + apple-sep.uapi) +sha256sums=('edc90e02f330a7595ceaf37f2c6ec32ed43541347fe936d3273b0bb2524fd19c' + '00f999dab395c0f6cd5ffda94099f898eb48d001e19036c500fe2713f76aa076' + '00cb09430461e0b0772619bc2054c8f9794484fbc3d807268955c0fb2de45c10') + +prepare() { + cd "$_pkgname-v$pkgver" + patch -Np1 -i "../libfprint-$pkgver-apple-sep.patch" + + # The driver includes , which the distribution's kernel + # headers do not carry. + install -Dm644 ../apple-sep.uapi ../aurora-include/linux/apple-sep.h +} + +build() { + local meson_options=( + -D drivers=all + -D installed-tests=false + ) + + CFLAGS+=" -I$srcdir/aurora-include" + arch-meson "$_pkgname-v$pkgver" build "${meson_options[@]}" + meson compile -C build +} + +check() { + meson test -C build --print-errorlogs +} + +package() { + meson install -C build --destdir "$pkgdir" +} diff --git a/tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi b/tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi new file mode 120000 index 0000000000000..3ba9077515e02 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi @@ -0,0 +1 @@ +../../../../../include/uapi/linux/apple-sep.h \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch new file mode 120000 index 0000000000000..40c7665bd03ef --- /dev/null +++ b/tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch @@ -0,0 +1 @@ +../../../patches/libfprint-1.94.100-apple-sep.patch \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec new file mode 100644 index 0000000000000..9d09b92727479 --- /dev/null +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Built from tools/aurora-sep: rpmbuild --define "_sourcedir " +# (build.sh does this). + +Name: aurora-sep +Version: 1 +Release: 1%{?dist} +Summary: Loader and fprintd policy for the Apple SEP fingerprint driver + +License: GPL-2.0-only +URL: https://github.com/aurora-silicon/linux +Source0: load-driver +Source1: aurora-sep.service +Source2: fprintd-aurora.conf +Source3: README.md +Source4: XART-SAFETY.md + +BuildArch: noarch +BuildRequires: systemd-rpm-macros +Requires: coreutils +Requires: fprintd +Requires: keyutils +Requires: kmod +%{?systemd_requires} + +%description +Loads the Apple SEP driver (apple_sep) with xART writes enabled, waits until +the secure enclave seals and unseals a trusted key and /dev/sep-bio appears, +and lets fprintd reach /dev/sep-bio. The loader runs at boot, and fprintd +also pulls it in when it starts. libfprint needs the aurora driver, which +this package does not provide. Read XART-SAFETY.md before installing it on +a machine that also runs macOS. + +%prep +%setup -q -c -T +cp -p %{SOURCE3} %{SOURCE4} . + +%build + +%install +install -Dpm 0755 %{SOURCE0} %{buildroot}%{_libexecdir}/aurora-sep-load +install -Dpm 0644 %{SOURCE1} %{buildroot}%{_unitdir}/aurora-sep.service +sed -i 's|^ExecStart=.*|ExecStart=%{_libexecdir}/aurora-sep-load|' \ + %{buildroot}%{_unitdir}/aurora-sep.service +grep -qx 'ExecStart=%{_libexecdir}/aurora-sep-load' \ + %{buildroot}%{_unitdir}/aurora-sep.service +install -Dpm 0644 %{SOURCE2} \ + %{buildroot}%{_unitdir}/fprintd.service.d/aurora-sep.conf +# Load the driver at boot, so the enclave is up before the login screen asks. +install -d %{buildroot}%{_presetdir} +echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset + +%post +%systemd_post aurora-sep.service + +%preun +%systemd_preun aurora-sep.service + +%postun +# Never restart it: the driver attaches to the enclave once per boot. +%systemd_postun aurora-sep.service + +%files +%doc README.md XART-SAFETY.md +%{_libexecdir}/aurora-sep-load +%{_unitdir}/aurora-sep.service +%dir %{_unitdir}/fprintd.service.d +%{_unitdir}/fprintd.service.d/aurora-sep.conf +%{_presetdir}/80-aurora-sep.preset + +%changelog +* Sun Oct 11 2026 Ryan Murray - 1-1 +- Package the loader, its service and the fprintd policy diff --git a/tools/aurora-sep/packaging/fedora/build.sh b/tools/aurora-sep/packaging/fedora/build.sh new file mode 100755 index 0000000000000..32d9f30761d02 --- /dev/null +++ b/tools/aurora-sep/packaging/fedora/build.sh @@ -0,0 +1,98 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +# +# Build the Fedora packages for Touch ID on the Apple SEP: +# +# - libfprint: Fedora's own source package, rebuilt with the aurora driver +# patch from ../../patches and nothing else changed. The release gains an +# ".aurora1" suffix, so it replaces the distribution's build in place. +# - aurora-sep: the driver loader, its service and the fprintd policy. +# +# It installs the build dependencies with dnf, so run it as root in a +# throwaway container of the Fedora release you are building for: +# +# podman run --rm -v :/src:ro,z -v :/out:z \ +# registry.fedoraproject.org/fedora:45 /src/ /out +# +# The binary and source packages land in the output directory. + +set -eu + +out=${1:?usage: build.sh } +here=$(cd "$(dirname "$0")" && pwd) +sep=$(cd "$here/../.." && pwd) +patch=$sep/patches/libfprint-1.94.100-apple-sep.patch +header=${APPLE_SEP_HEADER:-$sep/../../include/uapi/linux/apple-sep.h} +patched_version=1.94.100 + +for f in "$patch" "$header"; do + [ -f "$f" ] || { echo "build.sh: $f is missing" >&2; exit 1; } +done +mkdir -p "$out" + +top=$(mktemp -d) +trap 'rm -rf "$top"' EXIT +rpmtop=$top/rpmbuild + +dnf -y install rpm-build systemd-rpm-macros + +# Fedora's libfprint, exactly as the distribution builds it. +(cd "$top" && dnf -y download --source libfprint) +srpm=$(ls "$top"/libfprint-*.src.rpm) +rpm -i --define "_topdir $rpmtop" "$srpm" +spec=$rpmtop/SPECS/libfprint.spec + +version=$(rpmspec -q --srpm --qf '%{version}\n' "$spec") +if [ "$version" != "$patched_version" ]; then + echo "build.sh: this Fedora ships libfprint $version, the patch is for $patched_version" >&2 + exit 1 +fi +cp "$patch" "$header" "$rpmtop/SOURCES/" + +# Add the patch and the header, mark the release, and point the compiler at +# the header: is not in the distribution's kernel headers. +# %autosetup applies every PatchN. Stop if any anchor has moved. +awk -v patch="$(basename "$patch")" -v date="$(LC_ALL=C date -u '+%a %b %d %Y')" ' + /^Release:/ && !release { + sub(/[[:space:]]*$/, ".aurora1"); print; release = 1; next + } + /^Source0:/ && !source { + print + print "Source9000: apple-sep.h" + print "Patch9000: " patch + source = 1; next + } + /^%meson[[:space:]]/ && !build { + print "# The aurora driver includes from the Aurora kernel." + print "mkdir -p aurora-include/linux" + print "cp -p %{SOURCE9000} aurora-include/linux/" + print "export CFLAGS=\"%{build_cflags} -I$PWD/aurora-include\"" + print; build = 1; next + } + /^%changelog/ && !changelog { + print + print "* " date " Aurora Silicon - " "%{version}-%{release}" + print "- Add the Apple SEP (aurora) driver from aurora-silicon/linux" + print " tools/aurora-sep/patches/" patch + print "" + changelog = 1; next + } + { print } + END { + if (!(release && source && build && changelog)) { + print "build.sh: libfprint.spec changed shape; update the edit" > "/dev/stderr" + exit 1 + } + } +' "$spec" >"$spec.aurora" +mv "$spec.aurora" "$spec" + +dnf -y builddep "$spec" +rpmbuild --define "_topdir $rpmtop" -ba "$spec" + +# The loader, its service and the fprintd policy, from this tree. +rpmbuild --define "_topdir $rpmtop" --define "_sourcedir $sep" \ + -ba "$here/aurora-sep.spec" + +cp -p "$rpmtop"/RPMS/*/*.rpm "$rpmtop"/SRPMS/*.rpm "$out/" +ls -l "$out" diff --git a/tools/aurora-sep/tests/test_packaging.py b/tools/aurora-sep/tests/test_packaging.py new file mode 100644 index 0000000000000..f5aa1b124d060 --- /dev/null +++ b/tools/aurora-sep/tests/test_packaging.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""The packaging must describe the files in this tree, not stale copies.""" + +import hashlib +import re +import unittest +from pathlib import Path + +SEP = Path(__file__).resolve().parents[1] +ARCH = SEP / "packaging" / "arch" +FEDORA = SEP / "packaging" / "fedora" + + +def bash_array(text, name): + match = re.search(r"^%s=\((.*?)\)" % re.escape(name), text, re.M | re.S) + if match is None: + raise AssertionError("no %s array" % name) + return re.findall(r"'([^']*)'|\"([^\"]*)\"|(\S+)", match.group(1)) + + +def words(text, name): + return ["".join(parts) for parts in bash_array(text, name)] + + +class PkgbuildTest(unittest.TestCase): + def check(self, directory): + text = (directory / "PKGBUILD").read_text() + sources = words(text, "source") + sums = words(text, "sha256sums") + self.assertEqual(len(sources), len(sums)) + pkgver = re.search(r"^pkgver=(\S+)$", text, re.M).group(1) + for source, expected in zip(sources, sums): + if "://" in source: + continue + name = source.replace("$pkgver", pkgver) + path = directory / name + self.assertTrue(path.is_file(), "%s is not a file" % path) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + self.assertEqual( + digest, expected, "%s changed: run updpkgsums in %s" % (name, directory) + ) + + def test_libfprint_sums_match_tree(self): + self.check(ARCH / "libfprint-aurora") + + def test_aurora_sep_sums_match_tree(self): + self.check(ARCH / "aurora-sep") + + def test_patch_version_matches_pkgver(self): + text = (ARCH / "libfprint-aurora" / "PKGBUILD").read_text() + pkgver = re.search(r"^pkgver=(\S+)$", text, re.M).group(1) + self.assertTrue((SEP / "patches" / ("libfprint-%s-apple-sep.patch" % pkgver)).is_file()) + + +class FedoraSpecTest(unittest.TestCase): + def test_sources_are_in_tree(self): + text = (FEDORA / "aurora-sep.spec").read_text() + sources = re.findall(r"^Source\d+:\s+(\S+)$", text, re.M) + self.assertTrue(sources) + for source in sources: + self.assertTrue((SEP / source).is_file(), source) + + def test_build_script_patches_the_shipped_version(self): + text = (FEDORA / "build.sh").read_text() + version = re.search(r"^patched_version=(\S+)$", text, re.M).group(1) + self.assertIn("libfprint-%s-apple-sep.patch" % version, text) + self.assertTrue((SEP / "patches" / ("libfprint-%s-apple-sep.patch" % version)).is_file()) + + +if __name__ == "__main__": + unittest.main() From 59540c7f72b3721d293fb36cc5fd5bf79e1f47bc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 15:42:14 +1100 Subject: [PATCH 0827/1579] pmdomain: apple: Undo a power state transition that times out apple_pmgr_ps_set() writes the new target state and polls the actual state for 100 us. When the poll times out it logs and returns the error, but leaves the new target requested, and a power-on also sets AUTO_ENABLE. genpd then keeps the domain in its previous state: off after a failed ->power_on(), on after a failed ->power_off(). From then on the register and genpd disagree, and genpd does not write the domain again until its next transition. On T8140 (J700), rebinding the ISP after an unbind fails because the resident firmware does not boot a second time, and the CPU core domain stops short of ACTIVE: [ 760.990199] apple-pmgr-pwrstate 300700000.power-management:power-controller@4008: PS isp_cpu_core0: Failed to reach power state 0xf (requested 0xf00000f, now: 0xf000b4f) genpd and the ISP driver unwind as if the domain were off, but its register keeps requesting ACTIVE with auto-PM enabled, so the core sits clock-gated instead of power-gated until the next boot. A failed power-off is the opposite case: genpd still treats the domain as on and will not power it on again for its next consumer, while the register asks for it to be power-gated. Follow a failed transition with the opposite one: power the domain back off after a failed power-on, and back on after a failed power-off. Both are the normal sequences, so a power-off still sets the force-disable and force-reset bits first, and the domain is down before genpd powers off its parents. genpd ignores power-on errors in the noirq and syscore phases of system sleep (genpd_sync_power_on()) and records the domain as on there, so powering it back off would leave a domain off that genpd and its consumers believe is on. Slow domains do time out there: the gfx domain misses the poll window on resume from s2idle on M1 Pro (t6000), and did on T8140 in genpd's noirq resume before it became always-on, then reaches ACTIVE, and the GPU keeps working. The PM core runs those phases between the suspend_late and resume_early callbacks of every device. Add these callbacks to mark the window and keep requesting the domain on inside it, as before. The probe-time power-on of an always-on domain also keeps the current behaviour, as genpd never powers those domains. Transitions that succeed are unchanged. Fixes: 6df9d38f9146 ("soc: apple: Add driver for Apple PMGR power state controls") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmgr-pwrstate.c | 68 ++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 42511ed7ef89c..c5699abae3d93 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -48,6 +48,7 @@ struct apple_pmgr_ps { bool force_disable; bool force_reset; bool externally_clocked; + bool system_sleep; }; #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) @@ -143,14 +144,62 @@ static bool apple_pmgr_ps_is_active(struct apple_pmgr_ps *ps) reg & APPLE_PMGR_AUTO_ENABLE)); } +/* + * genpd keeps a domain off when ->power_on() fails and on when ->power_off() + * fails. Follow a failed transition with the opposite one, so the domain is + * not left requesting a state genpd does not know it is in. + * + * The exception is a failed power-on in the noirq and syscore phases of a + * system sleep transition: genpd ignores the error there and records the + * domain as on. The PM core runs those phases after the suspend_late callback + * of every device and before any resume_early callback, so keep requesting + * the domain on between this device's two callbacks. + */ static int apple_pmgr_ps_power_on(struct generic_pm_domain *genpd) { - return apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); + struct apple_pmgr_ps *ps = genpd_to_apple_pmgr_ps(genpd); + int ret; + + ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); + if (ret && !ps->system_sleep) + apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); + + return ret; } static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd) { - return apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); + int ret; + + ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); + if (ret) + apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); + + return ret; +} + +static void apple_pmgr_ps_set_system_sleep(struct device *dev, bool sleep) +{ + struct apple_pmgr_ps *ps = dev_get_drvdata(dev); + unsigned long flags; + + spin_lock_irqsave(&ps->genpd.slock, flags); + ps->system_sleep = sleep; + spin_unlock_irqrestore(&ps->genpd.slock, flags); +} + +static int apple_pmgr_ps_suspend_late(struct device *dev) +{ + apple_pmgr_ps_set_system_sleep(dev, true); + + return 0; +} + +static int apple_pmgr_ps_resume_early(struct device *dev) +{ + apple_pmgr_ps_set_system_sleep(dev, false); + + return 0; } static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) @@ -253,6 +302,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) ps->dev = dev; ps->regmap = regmap; + platform_set_drvdata(pdev, ps); ret = of_property_read_string(node, "label", &name); if (ret < 0) { @@ -290,8 +340,13 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; if (!active) { dev_warn(dev, "always-on domain %s is not on at boot\n", name); - /* Turn it on so pm_genpd_init does not fail */ - active = apple_pmgr_ps_power_on(&ps->genpd) == 0; + /* + * Turn it on so pm_genpd_init does not fail. genpd never + * powers an always-on domain, so keep requesting it on + * even if it does not get there in time. + */ + active = apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_ACTIVE, + true) == 0; } } else if (active) { ps->genpd.flags |= GENPD_FLAG_DEFER_OFF | GENPD_FLAG_ACTIVE_WAKEUP; @@ -366,11 +421,16 @@ static const struct of_device_id apple_pmgr_ps_of_match[] = { MODULE_DEVICE_TABLE(of, apple_pmgr_ps_of_match); +static const struct dev_pm_ops apple_pmgr_ps_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_pmgr_ps_suspend_late, apple_pmgr_ps_resume_early) +}; + static struct platform_driver apple_pmgr_ps_driver = { .probe = apple_pmgr_ps_probe, .driver = { .name = "apple-pmgr-pwrstate", .of_match_table = apple_pmgr_ps_of_match, + .pm = pm_sleep_ptr(&apple_pmgr_ps_pm_ops), }, }; From 6a18494793ca0ccaff6014acec1613123d07bbf7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:03:09 +1100 Subject: [PATCH 0828/1579] soc: apple: sep: let a start wait for the last capture to stop The enrol and verify work items report their result and wake the poller before they return and clear the session's busy marker. After a cancel or a close they stop only once the capture loop notices. An ENROL_START or VERIFY_START in that window, or an open right after a close, is refused with EBUSY. Userspace cannot tell that from a device in use by someone else: libfprint reports a protocol error, and fprintd turns it into verify-disconnected. fprintd starts a match again as soon as a capture is reported unusable, so it lands in that window. Hold such an open or start until the work item has stopped, for as long as a capture may take to drain before sleep. Only an open with no other opener, or a start whose last operation has ended, waits; a start while an operation is still running is refused at once, as before. If the work item has not stopped in time, the caller is refused as before, and the driver logs it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/bio.rs | 7 ++++++ drivers/soc/apple/sbio.rs | 47 +++++++++++++++++++++++++++++++++++++-- drivers/soc/apple/sep.rs | 18 ++++++++++++--- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 752e9ea7a6e12..9ba7722f5cf4e 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -846,6 +846,13 @@ pub(crate) fn worker_stopped(session: &mut Session, epoch: u64) { } } +/// Whether an open (`opening`) or a capture start by the current opener would +/// be refused only because the last enrol or verify work item has not stopped +/// yet, although its operation has ended or its opener has gone. +pub(crate) fn waiting_on_worker(session: &Session, opening: bool) -> bool { + session.worker_epoch.is_some() && session.op.may_start() && session.open != opening +} + fn verify_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { let mut out = VerifyPoll::default(); // Spend the token only after the copy to user succeeds; a fault must not lose a match. diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index deff86802e440..9d626bb275a6d 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2719,6 +2719,7 @@ impl SepData { return Err(ENODEV); } let mut session = self.bio_session.lock(); + self.wait_for_worker_exit(&mut session, true)?; bio::open(&mut session)?; drop(session); if let Err(e) = self.prepare_bio_open() { @@ -2757,6 +2758,45 @@ impl SepData { self.captures_running.fetch_sub(1, Relaxed); } + /// The enrol or verify work item `epoch` has stopped. + pub(crate) fn capture_worker_stopped(&self, epoch: u64) { + bio::worker_stopped(&mut self.bio_session.lock(), epoch); + self.worker_gone.notify_all(); + } + + /// Holds an open or a capture start while the last enrol or verify work + /// item is still on its way out. It reports its result before it stops, + /// and after a cancel or a close it stops only once the capture notices, + /// so an open or a start straight after either would be refused with + /// EBUSY, which the caller cannot tell from a device in use. If it has + /// not stopped in time, the caller refuses as before. + fn wait_for_worker_exit( + &self, + session: &mut MutexGuard<'_, bio::Session>, + opening: bool, + ) -> Result<()> { + let mut remaining = time::msecs_to_jiffies(WORKER_EXIT_WAIT_MS); + while bio::waiting_on_worker(session, opening) { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + if remaining == 0 { + dev_warn!( + self.dev, + "Touch ID: the last capture had not stopped after {} ms\n", + WORKER_EXIT_WAIT_MS + ); + return Ok(()); + } + match self.worker_gone.wait_interruptible_timeout(session, remaining) { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + } + } + Ok(()) + } + /// The system is about to suspend. End any capture in progress, hold new /// ones, and wait for the enrol or verify work to stop touching the sensor /// before devices go down. @@ -2821,7 +2861,7 @@ impl SepData { .is_err() { this.finish_enrolment(Err(ENROL_STATUS_SENSOR)); - bio::worker_stopped(&mut this.bio_session.lock(), epoch); + this.capture_worker_stopped(epoch); } } @@ -2838,7 +2878,7 @@ impl SepData { bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), [0u8; bio::TOKEN_LEN], ); - bio::worker_stopped(&mut this.bio_session.lock(), epoch); + this.capture_worker_stopped(epoch); } } @@ -2904,6 +2944,9 @@ impl SepData { let sensor_present = sensor::is_bound(); let handled = { let mut session = self.bio_session.lock(); + if bio::starts_capture(cmd) { + self.wait_for_worker_exit(&mut session, false)?; + } let index = self.bio_index.lock(); let mut ctx = bio::Context { session: &mut session, diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index cecffaa55377d..ce1cc461a6ef2 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -63,7 +63,8 @@ use kernel::{ Arc, CondVar, CondVarTimeoutResult, - Mutex, // + Mutex, + MutexGuard, // }, time, types::ForeignOwnable, @@ -113,6 +114,11 @@ const SBIO_TIMEOUT_MS: time::Msecs = 5000; const SLEEP_DRAIN_MS: u32 = 6000; static_assert!(SLEEP_DRAIN_MS as u64 > SBIO_TIMEOUT_MS as u64); +// How long an open or a capture start waits for the last enrol or verify work +// item to stop once its operation has ended. It is stopped in the same way, +// so it gets the same bound as a capture draining before sleep. +const WORKER_EXIT_WAIT_MS: time::Msecs = SLEEP_DRAIN_MS; + const SKS_ALLOC: usize = 0x8000; const SKS_SECRET_LEN: usize = 32; @@ -696,6 +702,10 @@ struct SepData { // Enrol and verify work items between `capture_begin` and `capture_end`. captures_running: Atomic, + // Signalled when an enrol or verify work item has stopped. + #[pin] + worker_gone: CondVar, + #[pin] rx_work: Work, @@ -870,6 +880,7 @@ impl SepData { suspending: Atomic::new(false), sleep_over <- new_condvar!("SepData::sleep_over"), captures_running: Atomic::new(0), + worker_gone <- new_condvar!("SepData::worker_gone"), rx_work <- new_work!("SepData::rx_work"), enrol_work <- new_work!("SepData::enrol_work"), verify_work <- new_work!("SepData::verify_work"), @@ -2020,6 +2031,7 @@ impl SepData { fn remove(&self) { self.shutting_down.store(true, Relaxed); bio::release(&mut self.bio_session.lock()); + self.worker_gone.notify_all(); self.rng_shutdown.store(true, Relaxed); self.control_wq.notify_all(); self.sbio_wq.notify_all(); @@ -2218,7 +2230,7 @@ impl WorkItem for SepData { this.run_enrolment(); this.capture_end(); } - bio::worker_stopped(&mut this.bio_session.lock(), epoch); + this.capture_worker_stopped(epoch); } } @@ -2231,7 +2243,7 @@ impl WorkItem for SepData { this.run_verify(); this.capture_end(); } - bio::worker_stopped(&mut this.bio_session.lock(), epoch); + this.capture_worker_stopped(epoch); } } From 11fa59a387fcb3e452008f4a8b85454b2ab934fa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:34:30 +1100 Subject: [PATCH 0829/1579] tools: aurora-sep: activate Touch ID at boot, before fprintd The first open of /dev/sep-bio activates Touch ID: the driver restores the enrolled fingerprints from the enclave before the open returns, which takes a few seconds. That open is fprintd's first probe of the reader, so the first fingerprint request after boot, at the lock screen or for sudo, waits for the restore. Add aurora-sep-ready.service, which opens the device once after the loader and before fprintd, and pull it in from aurora-sep.service. It ends as soon as the open returns; a failed open is only logged, and a job timeout stops anything ordered after it from waiting beyond 90 s after boot. Install it in the Fedora and Arch packages. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 16 +++++++ tools/aurora-sep/aurora-sep-ready | 44 +++++++++++++++++++ tools/aurora-sep/aurora-sep-ready.service | 18 ++++++++ tools/aurora-sep/aurora-sep.service | 2 + .../packaging/arch/aurora-sep/PKGBUILD | 36 ++++++++++----- .../arch/aurora-sep/aurora-sep-ready | 1 + .../arch/aurora-sep/aurora-sep-ready.service | 1 + .../packaging/fedora/aurora-sep.spec | 41 +++++++++++------ 8 files changed, 133 insertions(+), 26 deletions(-) create mode 100644 tools/aurora-sep/aurora-sep-ready create mode 100644 tools/aurora-sep/aurora-sep-ready.service create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready.service diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 942d4002683e7..eb9af9803d3c0 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -196,6 +196,22 @@ fprintd at another libfprint, then run `systemctl daemon-reload` and libfprint, remove `aurora-sep` and run `dnf distro-sync libfprint` or `pacman -S libfprint`. +## Desktop integration + +The first open of `/dev/sep-bio` activates Touch ID: the driver restores the +enrolled fingerprints from the enclave before the open returns, which takes +a few seconds. `aurora-sep-ready.service`, which `aurora-sep.service` pulls +in, makes that open at boot, before fprintd starts. The lock screen, sudo +and polkit then find the reader and the fingerprints ready, and the first +fingerprint request does not wait for the restore. A failed open is only +logged. The packages install it; by hand: + +```sh +install -Dm755 aurora-sep-ready /usr/local/sbin/aurora-sep-ready +install -Dm644 aurora-sep-ready.service /etc/systemd/system/aurora-sep-ready.service +systemctl daemon-reload +``` + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/aurora-sep-ready b/tools/aurora-sep/aurora-sep-ready new file mode 100644 index 0000000000000..0363a3a298ecb --- /dev/null +++ b/tools/aurora-sep/aurora-sep-ready @@ -0,0 +1,44 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +# +# Bring Touch ID up before anyone asks for a fingerprint. The first open of +# /dev/sep-bio activates it: the driver restores the enrolled fingerprints +# from the enclave and returns only when that is done. Opened here at boot, +# ahead of fprintd, the restore no longer delays the first fingerprint +# request. A failure is only logged. +set -u + +dev=/dev/sep-bio + +log() { + echo "aurora-sep-ready: $*" >&2 +} + +uptime_s() { + read -r up _ &1); then + log "Touch ID is ready after $(($(uptime_s) - start)) s" + exit 0 + fi + case "$err" in + # Another opener is activating it or using it; the unit's start + # timeout bounds this. + *[Bb]usy*) sleep 0.2 ;; + *) + log "could not open $dev: ${err##*: }" + exit 0 + ;; + esac +done diff --git a/tools/aurora-sep/aurora-sep-ready.service b/tools/aurora-sep/aurora-sep-ready.service new file mode 100644 index 0000000000000..91e5a7bce3815 --- /dev/null +++ b/tools/aurora-sep/aurora-sep-ready.service @@ -0,0 +1,18 @@ +[Unit] +Description=Activate Touch ID and restore its fingerprints +Documentation=file:///usr/share/doc/aurora-sep/README.md +Requires=aurora-sep.service +After=aurora-sep.service +Before=fprintd.service +ConditionPathExists=/dev/sep-bio + +# Whatever is ordered after this waits for the enclave, which can take most +# of a minute after boot. Stop waiting 90 s after boot, so that a broken +# enclave cannot hold anything back for longer. +JobTimeoutSec=90 + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/aurora-sep-ready +TimeoutStartSec=60 diff --git a/tools/aurora-sep/aurora-sep.service b/tools/aurora-sep/aurora-sep.service index 4287247cba0da..34a9f1cd69eb7 100644 --- a/tools/aurora-sep/aurora-sep.service +++ b/tools/aurora-sep/aurora-sep.service @@ -2,6 +2,8 @@ Description=Aurora SEP services After=local-fs.target RequiresMountsFor=/var/lib +# Activate Touch ID as soon as the driver is up. +Wants=aurora-sep-ready.service [Service] Type=oneshot diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index d7a921a5501cf..e33322557b247 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -1,13 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-only # -# The loader, service and fprintd policy for the Apple SEP fingerprint driver. +# The loader and desktop integration for the Apple SEP fingerprint driver. # The local sources are links into tools/aurora-sep; after changing any of # them, run updpkgsums here. libfprint-aurora provides the libfprint driver. pkgname=aurora-sep pkgver=1 -pkgrel=1 -pkgdesc="Loader and fprintd policy for the Apple SEP (Touch ID) driver" +pkgrel=2 +pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) license=(GPL-2.0-only) @@ -21,22 +21,34 @@ depends=( source=(load-driver aurora-sep.service fprintd-aurora.conf + aurora-sep-ready + aurora-sep-ready.service README.md XART-SAFETY.md) sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' - 'd3441eeb06c930fcc13314d18012b1f5d71cc1a821ceaf44de13caafb2931d73' + '447f99cc3c7e683134f31b993aa825fadde24e4854954a3035c96e6631d4d4d0' 'f8d0c4b9385e6716d5ac5dd9941a57459fbc551a8238383bcde36cd929d0523f' - 'd6e5c68bab7e0d71da48820da75d4ff7e67a06e7970961377ed28d30c2b66c08' + '2db07d3a0191e8d14581a3056505a8c4225b2fc8e6ea4a1956ae5493abcbb0ae' + '5a5e5c7e93d1462d9d1023446fdb297926aeb93d8988867b515aca7adafe4487' + '29a8fb4970f7e550310a1c11d6aa0ad2b840b0307f4be0d1ad76875bed1a4ec2' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') -package() { - install -Dm755 load-driver "$pkgdir/usr/lib/aurora-sep/aurora-sep-load" +# A unit and the helper it runs ($1, installed as $3), pointed at the +# packaged helper. +_unit() { + local unit="$pkgdir/usr/lib/systemd/system/$2" + + install -Dm755 "$1" "$pkgdir/usr/lib/aurora-sep/$3" + install -Dm644 "$2" "$unit" + sed -i "s|^\(Exec[A-Za-z]*=\)/usr/local/sbin/$3|\1/usr/lib/aurora-sep/$3|" "$unit" + if grep -q /usr/local "$unit"; then + return 1 + fi +} - install -Dm644 aurora-sep.service "$pkgdir/usr/lib/systemd/system/aurora-sep.service" - sed -i 's|^ExecStart=.*|ExecStart=/usr/lib/aurora-sep/aurora-sep-load|' \ - "$pkgdir/usr/lib/systemd/system/aurora-sep.service" - grep -qx 'ExecStart=/usr/lib/aurora-sep/aurora-sep-load' \ - "$pkgdir/usr/lib/systemd/system/aurora-sep.service" +package() { + _unit load-driver aurora-sep.service aurora-sep-load + _unit aurora-sep-ready aurora-sep-ready.service aurora-sep-ready install -Dm644 fprintd-aurora.conf \ "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora-sep.conf" diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready new file mode 120000 index 0000000000000..55fb2a87e5d43 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready @@ -0,0 +1 @@ +../../../aurora-sep-ready \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready.service b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready.service new file mode 120000 index 0000000000000..7ab622ca2d833 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready.service @@ -0,0 +1 @@ +../../../aurora-sep-ready.service \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index 9d09b92727479..57ae4ba402c1a 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -5,8 +5,8 @@ Name: aurora-sep Version: 1 -Release: 1%{?dist} -Summary: Loader and fprintd policy for the Apple SEP fingerprint driver +Release: 2%{?dist} +Summary: Loader and desktop integration for the Apple SEP fingerprint driver License: GPL-2.0-only URL: https://github.com/aurora-silicon/linux @@ -15,6 +15,8 @@ Source1: aurora-sep.service Source2: fprintd-aurora.conf Source3: README.md Source4: XART-SAFETY.md +Source5: aurora-sep-ready +Source6: aurora-sep-ready.service BuildArch: noarch BuildRequires: systemd-rpm-macros @@ -28,8 +30,9 @@ Requires: kmod Loads the Apple SEP driver (apple_sep) with xART writes enabled, waits until the secure enclave seals and unseals a trusted key and /dev/sep-bio appears, and lets fprintd reach /dev/sep-bio. The loader runs at boot, and fprintd -also pulls it in when it starts. libfprint needs the aurora driver, which -this package does not provide. Read XART-SAFETY.md before installing it on +also pulls it in when it starts. Touch ID is then activated, before fprintd +starts. libfprint needs the aurora driver, which this package does not +provide. Read XART-SAFETY.md before installing it on a machine that also runs macOS. %prep @@ -39,12 +42,17 @@ cp -p %{SOURCE3} %{SOURCE4} . %build %install -install -Dpm 0755 %{SOURCE0} %{buildroot}%{_libexecdir}/aurora-sep-load -install -Dpm 0644 %{SOURCE1} %{buildroot}%{_unitdir}/aurora-sep.service -sed -i 's|^ExecStart=.*|ExecStart=%{_libexecdir}/aurora-sep-load|' \ - %{buildroot}%{_unitdir}/aurora-sep.service -grep -qx 'ExecStart=%{_libexecdir}/aurora-sep-load' \ - %{buildroot}%{_unitdir}/aurora-sep.service +# A unit and the helper it runs ($1, installed as $3), pointed at the +# packaged helper. +unit() { + install -Dpm 0755 "$1" %{buildroot}%{_libexecdir}/"$3" + install -Dpm 0644 "$2" %{buildroot}%{_unitdir}/"${2##*/}" + sed -i "s|^\(Exec[A-Za-z]*=\)/usr/local/sbin/$3|\1%{_libexecdir}/$3|" \ + %{buildroot}%{_unitdir}/"${2##*/}" + if grep -q /usr/local %{buildroot}%{_unitdir}/"${2##*/}"; then exit 1; fi +} +unit %{SOURCE0} %{SOURCE1} aurora-sep-load +unit %{SOURCE5} %{SOURCE6} aurora-sep-ready install -Dpm 0644 %{SOURCE2} \ %{buildroot}%{_unitdir}/fprintd.service.d/aurora-sep.conf # Load the driver at boot, so the enclave is up before the login screen asks. @@ -52,23 +60,28 @@ install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %post -%systemd_post aurora-sep.service +%systemd_post aurora-sep.service aurora-sep-ready.service %preun -%systemd_preun aurora-sep.service +%systemd_preun aurora-sep.service aurora-sep-ready.service %postun -# Never restart it: the driver attaches to the enclave once per boot. -%systemd_postun aurora-sep.service +# Never restart them: the driver attaches to the enclave once per boot. +%systemd_postun aurora-sep.service aurora-sep-ready.service %files %doc README.md XART-SAFETY.md %{_libexecdir}/aurora-sep-load +%{_libexecdir}/aurora-sep-ready %{_unitdir}/aurora-sep.service +%{_unitdir}/aurora-sep-ready.service %dir %{_unitdir}/fprintd.service.d %{_unitdir}/fprintd.service.d/aurora-sep.conf %{_presetdir}/80-aurora-sep.preset %changelog +* Sun Oct 11 2026 Ryan Murray - 1-2 +- Activate Touch ID at boot, before fprintd starts + * Sun Oct 11 2026 Ryan Murray - 1-1 - Package the loader, its service and the fprintd policy From 61e027296f96f564d84582029d769aee1599b11c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:34:51 +1100 Subject: [PATCH 0830/1579] tools: aurora-sep: keep fprintd running fprintd exits after 30 s without a client, and each new start probes the reader again. The login and lock screens ask fprintd for the reader as they come up, so they wait for that start, and for a reader that is still being probed. Add a drop-in that runs fprintd with --no-timeout. Its path differs between distributions, so the drop-in is a template: the packages fill it in and check it against fprintd's own unit at build time. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 11 +++++++++++ tools/aurora-sep/fprintd-no-timeout.conf.in | 7 +++++++ tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD | 7 ++++++- .../arch/aurora-sep/fprintd-no-timeout.conf.in | 1 + tools/aurora-sep/packaging/fedora/aurora-sep.spec | 8 ++++++++ tools/aurora-sep/packaging/fedora/build.sh | 3 ++- 6 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 tools/aurora-sep/fprintd-no-timeout.conf.in create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/fprintd-no-timeout.conf.in diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index eb9af9803d3c0..671531f0729d6 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -212,6 +212,17 @@ install -Dm644 aurora-sep-ready.service /etc/systemd/system/aurora-sep-ready.ser systemctl daemon-reload ``` +fprintd exits after 30 s without a client, and each new start probes the +reader again, while the login and lock screens ask for the reader as they +come up. `fprintd-no-timeout.conf.in` keeps it running so that it answers +them at once. The packages fill in where the distribution keeps fprintd; by +hand, with `/usr/lib/fprintd` on Arch: + +```sh +sed 's|@FPRINTD@|/usr/libexec/fprintd|' fprintd-no-timeout.conf.in \ + >/etc/systemd/system/fprintd.service.d/aurora-sep-no-timeout.conf +``` + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/fprintd-no-timeout.conf.in b/tools/aurora-sep/fprintd-no-timeout.conf.in new file mode 100644 index 0000000000000..35f70927831fa --- /dev/null +++ b/tools/aurora-sep/fprintd-no-timeout.conf.in @@ -0,0 +1,7 @@ +# Keep fprintd running instead of letting it exit after 30 s without a +# client. Each new start probes the reader again, and the login and lock +# screens ask it for the reader when they come up, so a resident fprintd +# answers them at once. The packages fill in fprintd's path. +[Service] +ExecStart= +ExecStart=@FPRINTD@ --no-timeout diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index e33322557b247..01d0b410d2fff 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -23,6 +23,7 @@ source=(load-driver fprintd-aurora.conf aurora-sep-ready aurora-sep-ready.service + fprintd-no-timeout.conf.in README.md XART-SAFETY.md) sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' @@ -30,7 +31,8 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' 'f8d0c4b9385e6716d5ac5dd9941a57459fbc551a8238383bcde36cd929d0523f' '2db07d3a0191e8d14581a3056505a8c4225b2fc8e6ea4a1956ae5493abcbb0ae' '5a5e5c7e93d1462d9d1023446fdb297926aeb93d8988867b515aca7adafe4487' - '29a8fb4970f7e550310a1c11d6aa0ad2b840b0307f4be0d1ad76875bed1a4ec2' + '8d8df0ba2a6861b2edb2110e606906d84ca4848617621ad3cb48493a1522971d' + '9d8caa6ad9be48cdb9331a5498eafad2b7db82764fe41b74bc97ba3fcccfbb2b' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') # A unit and the helper it runs ($1, installed as $3), pointed at the @@ -52,5 +54,8 @@ package() { install -Dm644 fprintd-aurora.conf \ "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora-sep.conf" + grep -qx 'ExecStart=/usr/lib/fprintd' /usr/lib/systemd/system/fprintd.service + sed 's|@FPRINTD@|/usr/lib/fprintd|' fprintd-no-timeout.conf.in \ + >"$pkgdir/usr/lib/systemd/system/fprintd.service.d/50-aurora-sep-no-timeout.conf" install -Dm644 -t "$pkgdir/usr/share/doc/$pkgname" README.md XART-SAFETY.md } diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-no-timeout.conf.in b/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-no-timeout.conf.in new file mode 120000 index 0000000000000..2e8a7a43c8830 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-no-timeout.conf.in @@ -0,0 +1 @@ +../../../fprintd-no-timeout.conf.in \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index 57ae4ba402c1a..59a0ade7ed2d4 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -17,9 +17,12 @@ Source3: README.md Source4: XART-SAFETY.md Source5: aurora-sep-ready Source6: aurora-sep-ready.service +Source7: fprintd-no-timeout.conf.in BuildArch: noarch BuildRequires: systemd-rpm-macros +# Only to check where fprintd's unit runs it from. +BuildRequires: fprintd Requires: coreutils Requires: fprintd Requires: keyutils @@ -55,6 +58,9 @@ unit %{SOURCE0} %{SOURCE1} aurora-sep-load unit %{SOURCE5} %{SOURCE6} aurora-sep-ready install -Dpm 0644 %{SOURCE2} \ %{buildroot}%{_unitdir}/fprintd.service.d/aurora-sep.conf +grep -qx 'ExecStart=%{_libexecdir}/fprintd' %{_unitdir}/fprintd.service +sed 's|@FPRINTD@|%{_libexecdir}/fprintd|' %{SOURCE7} \ + >%{buildroot}%{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf # Load the driver at boot, so the enclave is up before the login screen asks. install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset @@ -77,11 +83,13 @@ echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %{_unitdir}/aurora-sep-ready.service %dir %{_unitdir}/fprintd.service.d %{_unitdir}/fprintd.service.d/aurora-sep.conf +%{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf %{_presetdir}/80-aurora-sep.preset %changelog * Sun Oct 11 2026 Ryan Murray - 1-2 - Activate Touch ID at boot, before fprintd starts +- Keep fprintd running * Sun Oct 11 2026 Ryan Murray - 1-1 - Package the loader, its service and the fprintd policy diff --git a/tools/aurora-sep/packaging/fedora/build.sh b/tools/aurora-sep/packaging/fedora/build.sh index 32d9f30761d02..27fb64e28aafd 100755 --- a/tools/aurora-sep/packaging/fedora/build.sh +++ b/tools/aurora-sep/packaging/fedora/build.sh @@ -90,7 +90,8 @@ mv "$spec.aurora" "$spec" dnf -y builddep "$spec" rpmbuild --define "_topdir $rpmtop" -ba "$spec" -# The loader, its service and the fprintd policy, from this tree. +# The loader, its units and the desktop integration, from this tree. +dnf -y builddep "$here/aurora-sep.spec" rpmbuild --define "_topdir $rpmtop" --define "_sourcedir $sep" \ -ba "$here/aurora-sep.spec" From a3d58f5a3f0f619bbbd006b4e573cc34d3e662ff Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:35:10 +1100 Subject: [PATCH 0831/1579] tools: aurora-sep: suspend rather than power off on the Touch ID button On Macs with Touch ID the sensor is the power button. Touching it to authenticate goes to the enclave and never produces a key event, but pressing it does, and logind cannot tell a press meant for the sensor from a press meant for the button. Its default action is to power off, so pressing while authenticating outside GNOME ends the session. Add a logind drop-in that suspends on a short press instead: a press while authenticating then costs nothing, and the next press wakes the machine. Holding the button for 5 s powers off cleanly. GNOME sessions and the GDM login screen take a short press over and suspend by default already, so this only changes consoles and other desktops. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 14 ++++++++++++++ tools/aurora-sep/logind-aurora-sep.conf | 11 +++++++++++ .../aurora-sep/packaging/arch/aurora-sep/PKGBUILD | 6 +++++- .../arch/aurora-sep/logind-aurora-sep.conf | 1 + tools/aurora-sep/packaging/fedora/aurora-sep.spec | 9 +++++++-- 5 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 tools/aurora-sep/logind-aurora-sep.conf create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/logind-aurora-sep.conf diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 671531f0729d6..5f0e9056ab749 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -223,6 +223,20 @@ sed 's|@FPRINTD@|/usr/libexec/fprintd|' fprintd-no-timeout.conf.in \ >/etc/systemd/system/fprintd.service.d/aurora-sep-no-timeout.conf ``` +The Touch ID sensor is the power button. A touch to authenticate goes to the +enclave and never reaches logind as a key press, but a press does, and logind +cannot tell a press meant for the sensor from one meant for the button. Its +default is to power off. `logind-aurora-sep.conf` makes a short press +suspend instead, so pressing while authenticating loses nothing and the next +press wakes the machine, and holding the button for 5 s powers off cleanly. +GNOME sessions and the GDM login screen handle a short press themselves +(`power-button-action`, `suspend` by default); the setting covers consoles +and other desktops. It takes effect at the next boot. By hand: + +```sh +install -Dm644 logind-aurora-sep.conf /etc/systemd/logind.conf.d/aurora-sep.conf +``` + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/logind-aurora-sep.conf b/tools/aurora-sep/logind-aurora-sep.conf new file mode 100644 index 0000000000000..a175c39f6ca73 --- /dev/null +++ b/tools/aurora-sep/logind-aurora-sep.conf @@ -0,0 +1,11 @@ +# The Touch ID sensor is the power button. Touching it to authenticate goes to +# the enclave and is never a key press, but pressing it is, and logind cannot +# tell a press meant for the sensor from one meant for the button. Suspend +# rather than power off on a short press, so a press while authenticating +# costs nothing: the next press wakes the machine. Holding the button for +# 5 s still powers off, cleanly. GNOME sessions and the GDM login screen +# handle a short press themselves (power-button-action, suspend by default); +# this covers everything else. +[Login] +HandlePowerKey=suspend +HandlePowerKeyLongPress=poweroff diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 01d0b410d2fff..3c093add292ba 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -24,6 +24,7 @@ source=(load-driver aurora-sep-ready aurora-sep-ready.service fprintd-no-timeout.conf.in + logind-aurora-sep.conf README.md XART-SAFETY.md) sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' @@ -32,7 +33,8 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '2db07d3a0191e8d14581a3056505a8c4225b2fc8e6ea4a1956ae5493abcbb0ae' '5a5e5c7e93d1462d9d1023446fdb297926aeb93d8988867b515aca7adafe4487' '8d8df0ba2a6861b2edb2110e606906d84ca4848617621ad3cb48493a1522971d' - '9d8caa6ad9be48cdb9331a5498eafad2b7db82764fe41b74bc97ba3fcccfbb2b' + '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' + '3f5a8a4a19e15268552264da95d4848be31629c49f3f64f377c590cf6ef9e1cb' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') # A unit and the helper it runs ($1, installed as $3), pointed at the @@ -57,5 +59,7 @@ package() { grep -qx 'ExecStart=/usr/lib/fprintd' /usr/lib/systemd/system/fprintd.service sed 's|@FPRINTD@|/usr/lib/fprintd|' fprintd-no-timeout.conf.in \ >"$pkgdir/usr/lib/systemd/system/fprintd.service.d/50-aurora-sep-no-timeout.conf" + install -Dm644 logind-aurora-sep.conf \ + "$pkgdir/usr/lib/systemd/logind.conf.d/50-aurora-sep.conf" install -Dm644 -t "$pkgdir/usr/share/doc/$pkgname" README.md XART-SAFETY.md } diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/logind-aurora-sep.conf b/tools/aurora-sep/packaging/arch/aurora-sep/logind-aurora-sep.conf new file mode 120000 index 0000000000000..db6dc4e953022 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/logind-aurora-sep.conf @@ -0,0 +1 @@ +../../../logind-aurora-sep.conf \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index 59a0ade7ed2d4..a916df31cc0a5 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -18,6 +18,7 @@ Source4: XART-SAFETY.md Source5: aurora-sep-ready Source6: aurora-sep-ready.service Source7: fprintd-no-timeout.conf.in +Source8: logind-aurora-sep.conf BuildArch: noarch BuildRequires: systemd-rpm-macros @@ -34,8 +35,8 @@ Loads the Apple SEP driver (apple_sep) with xART writes enabled, waits until the secure enclave seals and unseals a trusted key and /dev/sep-bio appears, and lets fprintd reach /dev/sep-bio. The loader runs at boot, and fprintd also pulls it in when it starts. Touch ID is then activated, before fprintd -starts. libfprint needs the aurora driver, which this package does not -provide. Read XART-SAFETY.md before installing it on +starts, and pressing the Touch ID button suspends rather than powers off. +libfprint needs the aurora driver, which this package does not provide. Read XART-SAFETY.md before installing it on a machine that also runs macOS. %prep @@ -61,6 +62,8 @@ install -Dpm 0644 %{SOURCE2} \ grep -qx 'ExecStart=%{_libexecdir}/fprintd' %{_unitdir}/fprintd.service sed 's|@FPRINTD@|%{_libexecdir}/fprintd|' %{SOURCE7} \ >%{buildroot}%{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf +install -Dpm 0644 %{SOURCE8} \ + %{buildroot}%{_prefix}/lib/systemd/logind.conf.d/50-aurora-sep.conf # Load the driver at boot, so the enclave is up before the login screen asks. install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset @@ -84,12 +87,14 @@ echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %dir %{_unitdir}/fprintd.service.d %{_unitdir}/fprintd.service.d/aurora-sep.conf %{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf +%{_prefix}/lib/systemd/logind.conf.d/50-aurora-sep.conf %{_presetdir}/80-aurora-sep.preset %changelog * Sun Oct 11 2026 Ryan Murray - 1-2 - Activate Touch ID at boot, before fprintd starts - Keep fprintd running +- Suspend rather than power off on the Touch ID button * Sun Oct 11 2026 Ryan Murray - 1-1 - Package the loader, its service and the fprintd policy From 57bd2e0bb964833139ee7e02f969c30db6b79db2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:35:38 +1100 Subject: [PATCH 0832/1579] tools: aurora-sep: offer fingerprint at the GNOME login screen GNOME Shell 51 connects the greeter's fingerprint reader only when it re-authenticates a session that is already running, so the login screen never offers fingerprint, whatever org.gnome.login-screen says. That is deliberate: a fingerprint cannot unlock the login keyring, and the first application that wants a stored secret then asks for the password. Some people want fingerprint login anyway. Add aurora-sep-greeter-fingerprint.service, off by default. At boot it takes the greeter source file that holds the check from the installed GNOME Shell, drops the check, and serves the copy to the login screen's shell alone, through a GResource overlay set in a runtime drop-in for org.gnome.Shell@gdm.service. When the check is not exactly as expected, it changes nothing, so a GNOME update that moves it leaves the login screen as GNOME ships it. enable-fingerprint-authentication still turns fingerprint off. The login screen asks fprintd for the reader once, as it comes up, and offers no fingerprint until the user is selected again if Touch ID is not up yet. The unit therefore also orders the display manager after aurora-sep-ready.service. That holds the login screen back by a few seconds on a healthy boot, and up to 90 s after boot when the enclave is slow, which only makes sense when it offers fingerprint, so both come with the one switch. It needs gresource, which the packages suggest rather than require. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 30 +++++++ .../aurora-sep/aurora-sep-greeter-fingerprint | 85 +++++++++++++++++++ .../aurora-sep-greeter-fingerprint.service | 17 ++++ .../packaging/arch/aurora-sep/PKGBUILD | 9 +- .../aurora-sep/aurora-sep-greeter-fingerprint | 1 + .../aurora-sep-greeter-fingerprint.service | 1 + .../packaging/fedora/aurora-sep.spec | 14 ++- 7 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 tools/aurora-sep/aurora-sep-greeter-fingerprint create mode 100644 tools/aurora-sep/aurora-sep-greeter-fingerprint.service create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint.service diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 5f0e9056ab749..23c3f589b5289 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -237,6 +237,36 @@ and other desktops. It takes effect at the next boot. By hand: install -Dm644 logind-aurora-sep.conf /etc/systemd/logind.conf.d/aurora-sep.conf ``` +GNOME Shell 51 offers fingerprint only on the lock screen, never on the +login screen, and `org.gnome.login-screen enable-fingerprint-authentication` +(true by default) cannot change that: the greeter connects its fingerprint +reader only when it re-authenticates a running session. That is GNOME's +choice, because a fingerprint cannot unlock the login keyring: after a +fingerprint login, the first application that wants a stored secret asks +for the password. `aurora-sep-greeter-fingerprint.service` lifts the +restriction for those who want it. At boot it copies the one greeter source +file that holds the check out of the installed GNOME Shell, drops the check, +and serves the copy to the login screen's shell alone through a GResource +overlay. When the check is not exactly as expected, for example after a +GNOME update, it changes nothing. The enable key above still applies. + +The login screen asks fprintd for the reader once, as it comes up, so the +same unit also holds it back until `aurora-sep-ready.service` has finished: +a few seconds on a healthy boot, but as long as the enclave takes when it is +slow, up to 90 s after boot. Without fingerprint at the login screen that +wait buys nothing, which is why both are one switch, off by default. It +needs `gresource` from glib2-devel. With the packages, or after installing +it by hand, turn it on with `systemctl enable`: + +```sh +install -Dm755 aurora-sep-greeter-fingerprint /usr/local/sbin/aurora-sep-greeter-fingerprint +install -Dm644 aurora-sep-greeter-fingerprint.service /etc/systemd/system/ +systemctl daemon-reload +systemctl enable aurora-sep-greeter-fingerprint.service +``` + +Disabling it and rebooting returns the login screen to GNOME's behaviour. + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/aurora-sep-greeter-fingerprint b/tools/aurora-sep/aurora-sep-greeter-fingerprint new file mode 100644 index 0000000000000..7f04a19d24d20 --- /dev/null +++ b/tools/aurora-sep/aurora-sep-greeter-fingerprint @@ -0,0 +1,85 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +# +# Offer fingerprint at the GNOME login screen, or with "stop", stop offering +# it from the next time the login screen starts. +# +# GNOME Shell 51 connects the fingerprint reader only when it re-authenticates +# a session that is already running, which is the lock screen. The login +# screen never offers fingerprint, whatever org.gnome.login-screen says: a +# fingerprint cannot unlock the login keyring, so a session started with one +# asks for the password the first time an application wants a stored secret. +# +# This takes the greeter source file that holds that check from the installed +# GNOME Shell, drops the check, and serves the copy to the login screen's +# shell alone through a GResource overlay. When the check is not exactly as +# expected, for example after a GNOME update, it changes nothing and the login +# screen stays as GNOME ships it. Run it before the display manager starts. +set -eu + +res=/org/gnome/shell/gdm/authServicesLegacy.js +out=/run/aurora-sep/greeter +dropin=/run/systemd/user/org.gnome.Shell@gdm.service.d/aurora-sep-fingerprint.conf + +log() { + echo "aurora-sep-greeter-fingerprint: $*" >&2 +} + +rm -rf "$out" "$dropin" +[ "${1:-}" != stop ] || exit 0 + +if ! command -v gresource >/dev/null; then + log "gresource is not installed (it comes with glib2-devel); the login screen stays as it is" + exit 0 +fi + +lib= +for f in /usr/lib64/gnome-shell/libshell-*.so /usr/lib/gnome-shell/libshell-*.so \ + /usr/lib/*/gnome-shell/libshell-*.so; do + [ -f "$f" ] || continue + if gresource list "$f" 2>/dev/null | grep -qxF "$res"; then + lib=$f + break + fi +done +if [ -z "$lib" ]; then + log "no installed GNOME Shell carries $res; the login screen stays as it is" + exit 0 +fi + +install -d -m 0755 "$out" +gresource extract "$lib" "$res" >"$out/stock.js" +if ! awk ' + state == 0 && $0 == " _connectFingerprintManager() {" { + print + state = 1 + next + } + state == 1 { + if ($0 == " if (!this._reauthOnly)") { + state = 2 + next + } + state = 9 + } + state == 2 { + if ($0 == " return;") { + state = 3 + next + } + state = 9 + } + { print } + END { exit state != 3 } +' "$out/stock.js" >"$out/authServicesLegacy.js"; then + log "$lib does not have the expected check; the login screen stays as it is" + rm -rf "$out" + exit 0 +fi +rm -f "$out/stock.js" +chmod 0644 "$out/authServicesLegacy.js" + +install -d -m 0755 "${dropin%/*}" +printf '[Service]\nEnvironment=G_RESOURCE_OVERLAYS=%s=%s\n' \ + "$res" "$out/authServicesLegacy.js" >"$dropin" +log "the login screen offers fingerprint ($res from $lib)" diff --git a/tools/aurora-sep/aurora-sep-greeter-fingerprint.service b/tools/aurora-sep/aurora-sep-greeter-fingerprint.service new file mode 100644 index 0000000000000..108b1e4ccbc87 --- /dev/null +++ b/tools/aurora-sep/aurora-sep-greeter-fingerprint.service @@ -0,0 +1,17 @@ +[Unit] +Description=Offer fingerprint at the GNOME login screen +Documentation=file:///usr/share/doc/aurora-sep/README.md +# The login screen asks fprintd for the reader once, as it comes up. Hold it +# back until Touch ID is up, so that it finds the reader and the fingerprints. +Wants=aurora-sep-ready.service +After=aurora-sep-ready.service +Before=gdm.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/aurora-sep-greeter-fingerprint +ExecStop=/usr/local/sbin/aurora-sep-greeter-fingerprint stop + +[Install] +WantedBy=gdm.service diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 3c093add292ba..d27422443be35 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -18,6 +18,7 @@ depends=( kmod sh ) +optdepends=('glib2-devel: fingerprint at the GNOME login screen') source=(load-driver aurora-sep.service fprintd-aurora.conf @@ -25,6 +26,8 @@ source=(load-driver aurora-sep-ready.service fprintd-no-timeout.conf.in logind-aurora-sep.conf + aurora-sep-greeter-fingerprint + aurora-sep-greeter-fingerprint.service README.md XART-SAFETY.md) sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' @@ -34,7 +37,9 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '5a5e5c7e93d1462d9d1023446fdb297926aeb93d8988867b515aca7adafe4487' '8d8df0ba2a6861b2edb2110e606906d84ca4848617621ad3cb48493a1522971d' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' - '3f5a8a4a19e15268552264da95d4848be31629c49f3f64f377c590cf6ef9e1cb' + '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' + 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' + '226a5f5f2244583b5b10d9a5b3e134f5b155065fd8dce850c828f98f085fc931' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') # A unit and the helper it runs ($1, installed as $3), pointed at the @@ -53,6 +58,8 @@ _unit() { package() { _unit load-driver aurora-sep.service aurora-sep-load _unit aurora-sep-ready aurora-sep-ready.service aurora-sep-ready + _unit aurora-sep-greeter-fingerprint aurora-sep-greeter-fingerprint.service \ + aurora-sep-greeter-fingerprint install -Dm644 fprintd-aurora.conf \ "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora-sep.conf" diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint new file mode 120000 index 0000000000000..dd1bb06545313 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint @@ -0,0 +1 @@ +../../../aurora-sep-greeter-fingerprint \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint.service b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint.service new file mode 120000 index 0000000000000..bf6aeca94eb78 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint.service @@ -0,0 +1 @@ +../../../aurora-sep-greeter-fingerprint.service \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index a916df31cc0a5..648409fb7c463 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -19,6 +19,8 @@ Source5: aurora-sep-ready Source6: aurora-sep-ready.service Source7: fprintd-no-timeout.conf.in Source8: logind-aurora-sep.conf +Source9: aurora-sep-greeter-fingerprint +Source10: aurora-sep-greeter-fingerprint.service BuildArch: noarch BuildRequires: systemd-rpm-macros @@ -28,6 +30,8 @@ Requires: coreutils Requires: fprintd Requires: keyutils Requires: kmod +# Only for aurora-sep-greeter-fingerprint.service, which is off by default. +Suggests: /usr/bin/gresource %{?systemd_requires} %description @@ -57,6 +61,7 @@ unit() { } unit %{SOURCE0} %{SOURCE1} aurora-sep-load unit %{SOURCE5} %{SOURCE6} aurora-sep-ready +unit %{SOURCE9} %{SOURCE10} aurora-sep-greeter-fingerprint install -Dpm 0644 %{SOURCE2} \ %{buildroot}%{_unitdir}/fprintd.service.d/aurora-sep.conf grep -qx 'ExecStart=%{_libexecdir}/fprintd' %{_unitdir}/fprintd.service @@ -69,21 +74,23 @@ install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %post -%systemd_post aurora-sep.service aurora-sep-ready.service +%systemd_post aurora-sep.service aurora-sep-ready.service aurora-sep-greeter-fingerprint.service %preun -%systemd_preun aurora-sep.service aurora-sep-ready.service +%systemd_preun aurora-sep.service aurora-sep-ready.service aurora-sep-greeter-fingerprint.service %postun # Never restart them: the driver attaches to the enclave once per boot. -%systemd_postun aurora-sep.service aurora-sep-ready.service +%systemd_postun aurora-sep.service aurora-sep-ready.service aurora-sep-greeter-fingerprint.service %files %doc README.md XART-SAFETY.md %{_libexecdir}/aurora-sep-load %{_libexecdir}/aurora-sep-ready +%{_libexecdir}/aurora-sep-greeter-fingerprint %{_unitdir}/aurora-sep.service %{_unitdir}/aurora-sep-ready.service +%{_unitdir}/aurora-sep-greeter-fingerprint.service %dir %{_unitdir}/fprintd.service.d %{_unitdir}/fprintd.service.d/aurora-sep.conf %{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf @@ -95,6 +102,7 @@ echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset - Activate Touch ID at boot, before fprintd starts - Keep fprintd running - Suspend rather than power off on the Touch ID button +- Offer fingerprint at the GNOME 51 login screen, off by default * Sun Oct 11 2026 Ryan Murray - 1-1 - Package the loader, its service and the fprintd policy From f2760d2f42ec4ede63abb18918ef4abeb4b3de35 Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:06:15 -0400 Subject: [PATCH 0833/1579] drm/asahi: preserve TA prefix dependencies across render pairs A fragment-only dependency leaves TA with its physical ring parent. When covered renders migrate between pairs, that parent does not order their TA work. In the Neo Canvas Arcs reproducer, normal two-pair execution stalls, while retaining the cross-pair TA predecessor allows progress. Collect the actual TA QID and kick timestamp alongside the fragment frontier, including predecessors whose graphs are borrowed for preparation. Attach foreign TA predecessors to TA while keeping the fragment frontier on FRG. Same-pair TA dependencies remain implicit; independent prefixes, compute dependencies, pair admission and queue depth are unchanged. Neo validation with production Mesa: the clean four-file patch completed 60 seconds of the failing 18,000-arc workload, and 2,196 targeted CTS outcomes produced 1,026 Pass / 1,170 NotSupported / zero failures or GPU faults. Host checks cover borrowed/resident frontiers, nonadjacent QIDs, unequal and wrapped timestamps, failed producers and barrier capacity. Same-binary opt-in comparison completed pixel-checked resolve repetitions with overlapping timing ranges (median 6.203s off / 6.178s on); capped 2,000-arc browser runs retained their 60 Hz pacing. The tested module uses the deployed651 kernel/PRIME API. Its four changed dependency files and their pre-patch bases match this public tree exactly. The precise firmware resource cycle is not independently established. --- drivers/gpu/drm/asahi/g17/queue/render.rs | 19 ++++++++-- .../asahi/g17/queue/render/dependencies.rs | 36 +++++++++++++++++++ .../asahi/g17/runtime/render/dependencies.rs | 25 +++++++++++++ .../drm/asahi/g17/runtime/render/submit.rs | 13 +++++++ 4 files changed, 91 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 0bf477c0344fb..24a274e1af62a 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -43,6 +43,8 @@ use retirement::{Ticket, Tracker}; pub(crate) struct RenderDependencies { pub(crate) render: [KickDependency; 2], pub(crate) render_count: usize, + pub(crate) tiling: [KickDependency; 2], + pub(crate) tiling_count: usize, pub(crate) compute: Option, } @@ -50,6 +52,8 @@ impl RenderDependencies { pub(crate) const EMPTY: Self = Self { render: [KickDependency::ZERO; 2], render_count: 0, + tiling: [KickDependency::ZERO; 2], + tiling_count: 0, compute: None, }; pub(crate) fn render(&self) -> &[KickDependency] { @@ -403,8 +407,8 @@ impl Pair { /// Kick barriers of one render. The fragment kick always waits for its own tiling kick; /// with the fragment-barriers flag the resolved prefix orders the fragment kick too and the -/// tiling kick keeps only its implicit parent, otherwise the prefix orders the tiling kick. -/// A compute dependency always orders the tiling kick. +/// tiling kick preserves the TA prefix across physical pairs. Otherwise the render prefix +/// orders the tiling kick. A compute dependency always orders the tiling kick. struct RenderBarriers { tiling: [KickDependency; 3], tiling_count: usize, @@ -430,6 +434,17 @@ fn render_barriers( tiling_count += 1; } } + if fragment_stage { + // The implicit parent covers only this physical ring. Preserve the logical + // TA prefix when a predecessor was published on another pair, without + // making this TA kick wait for that predecessor's fragment completion. + for dependency in &dependencies.tiling[..dependencies.tiling_count] { + if dependency.qid() != own_tiling.qid() { + tiling_barriers[tiling_count] = *dependency; + tiling_count += 1; + } + } + } if let Some(compute) = dependencies.compute { tiling_barriers[tiling_count] = compute; tiling_count += 1; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs index be1d9fc9c902c..53a0fa6e449ed 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs @@ -15,6 +15,7 @@ use kernel::{prelude::*, sync::Arc}; struct Predecessor { packet: Arc, word: u64, + ta_word: u64, } /// A loan of the graph cannot hide work already published on that pair. Fence @@ -68,6 +69,21 @@ impl Dependencies { } Ok(()) } + pub(crate) fn collect_tiling( + &self, + owner: u64, + status: &Arc, + prefix: Option, + frontier: &mut Frontier, + ) -> Result { + if self.owner != Some(owner) { + return Ok(()); + } + for entry in self.entries.iter().flatten() { + include(frontier, status, prefix, &entry.packet, entry.ta_word)?; + } + Ok(()) + } } impl Pair { @@ -81,6 +97,8 @@ impl Pair { entries: core::array::from_fn(|_| { active.next().map(|entry| Predecessor { packet: entry.packet.clone(), + ta_word: KickDependency::new(self.qids()[0], entry.kicks[0]) + .map_or(u64::MAX, KickDependency::word), word: KickDependency::new(qid, entry.kicks[1]) .map_or(u64::MAX, KickDependency::word), }) @@ -106,4 +124,22 @@ impl Pair { } Ok(()) } + pub(crate) fn collect_tiling_dependencies( + &self, + owner: u64, + status: &Arc, + prefix: Option, + frontier: &mut Frontier, + ) -> Result { + if self.owner != Some(owner) { + return Ok(()); + } + let qid = self.qids()[0]; + for entry in self.active.iter() { + let word = + KickDependency::new(qid, entry.kicks[0]).map_or(u64::MAX, KickDependency::word); + include(frontier, status, prefix, &entry.packet, word)?; + } + Ok(()) + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs index 09e4322bf42ce..2b14b4548e0db 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs @@ -31,4 +31,29 @@ impl Registry { } Ok(frontier) } + pub(in crate::g17::runtime) fn tiling_prefix( + &self, + owner: u64, + packet: &Arc, + ) -> Result { + let mut frontier = Frontier::new(); + for entry in self.render.entries.iter().flatten() { + if let Some(pair) = entry.pair.as_ref() { + pair.collect_tiling_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + )?; + } else if let Some(dependencies) = entry.borrowed_dependencies.as_ref() { + dependencies.collect_tiling( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + )?; + } + } + Ok(frontier) + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index dbf7175c5cd4b..13381b62da86c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -187,6 +187,19 @@ impl Registry { .ok_or(EOVERFLOW)? = *dependency; dependencies.render_count += 1; } + if dependencies.render_count != 0 + && matches!(&packet.command, crate::g17::command::Validated::Render { pass, .. } + if pass.fragment_barriers) + { + let tiling = self.tiling_prefix(owner, packet)?; + for dependency in tiling.as_slice() { + *dependencies + .tiling + .get_mut(dependencies.tiling_count) + .ok_or(EOVERFLOW)? = *dependency; + dependencies.tiling_count += 1; + } + } if let Some(queue) = self.compute(owner) { dependencies.compute = queue.dependency( owner, From 27b4c674f0bb8f58790e09aa75d210cff3516225 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 20:27:16 +1100 Subject: [PATCH 0834/1579] wifi: mt7932: ring the TX doorbell when a regulatory change stops the queue A regulatory change stops the data queue from the cfg80211 notifier, outside ndo_start_xmit() and outside data_lock. If that lands in the middle of a burst, the TX0 descriptors the burst already published stay behind the doorbell until the next interrupt rings it. Ring the doorbell there as well, under data_lock nested inside response_lock in the same order as the receive path, so that every path that stops the queue from outside ndo_start_xmit() hands the device what it already holds. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/startup.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 225e5a0344fc4..cd0942687b059 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -436,6 +436,14 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request WRITE_ONCE(m->reg_pending, true); if (m->netdev) { netif_stop_queue(m->netdev); + /* The stop can land in the middle of a burst: hand the device the + * TX0 descriptors already published behind the doorbell. + */ + if (READ_ONCE(m->data_ready)) { + spin_lock(&m->data_lock); + mt_data_kick_locked(m); + spin_unlock(&m->data_lock); + } if (m->connecting) { m->connect_error = -ECANCELED; complete(&m->assoc_start); From 666591569cc4319cf429da79ccd85b2dc988222e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 20:31:59 +1100 Subject: [PATCH 0835/1579] wifi: mt7932: keep RX0 status in the RAM-config acknowledgment Since "acknowledge interrupts before servicing the rings", mt_service() clears the interrupt causes it is about to service, RX0 included, before it drains the rings, and restores the interrupt mask after the last ring. The RAM-config acknowledgment in mt_rx_drain() still clears RX0's status bit and rewrites the mask in the middle of the drain. The status write can clear the bit of an RX0 completion that arrived after the up-front acknowledgment, so the device no longer shows work that is still pending, and the unmask lets the interrupt fire again while the rings are still being serviced, with the fixed IRQ_MASK rather than m->irq_mask. Clear only the acknowledgment's other two status bits there, and leave the mask to mt_service(). The diagnostic drain on a command timeout runs with the interrupt line disabled; a bit it leaves set is serviced once the line is enabled again. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 92fbad6f648f9..c06fc92243b9f 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -241,12 +241,17 @@ static void mt_rx_drain(struct mt7932 *m, struct mt7932_ring *q) if (READ_ONCE(m->ram_config) && length >= 33 && length <= MT7932_RX_CAPACITY) { u8 *packet = q->buffers + q->tail * RX_STRIDE; - /* Source-specific RAM-config acknowledgment before RX0 refill. */ + /* + * Source-specific RAM-config acknowledgment before RX0 + * refill. mt_service() has already acknowledged this + * pass's causes, RX0 (bit 0) included, and restores the + * mask once every ring is drained. Leave bit 0 alone, so + * that an RX0 completion arriving now keeps its status, + * and leave the mask alone while the rings are serviced. + */ if (q == &m->rx[0] && packet[28] == 1 && packet[29] == m->waiting && !packet[30] && packet[32] == 1) { - mt_write(m, W + 0x204, 0); - mt_write(m, W + 0x200, 0x0c000001); - mt_write(m, W + 0x204, poll_test ? 0 : IRQ_MASK); + mt_write(m, W + 0x200, 0x0c000000); mt_write(m, W + 0x1f4, 0xffff); } } From 69b264dc00b60aa612f2d7bc9f214a3e877cb8c8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 20:57:42 +1100 Subject: [PATCH 0836/1579] arm64: configs: add a J700 base fragment defconfig, asahi.config and j700-radio.config give a J700 kernel that boots, but not the configuration it is tested with. The boot path (NVMe/ANS, btrfs, the DockChannel keyboard and trackpad, SMC, SPMI, the USB-C PHY and Type-C) ends up modular and so depends on what the initramfs carries, the per-process 4K user page mode stays off, and several J700 driver choices differ. Add j700.config, to be merged after j700-radio.config, with each group commented with its reason, and cover the J700 fragments with one MAINTAINERS pattern. Together with the device-parity fragment added next, the fragments reproduce the tested configuration. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 2 +- arch/arm64/configs/j700.config | 113 +++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/configs/j700.config diff --git a/MAINTAINERS b/MAINTAINERS index 99987f9eca95b..96e7fc1ab1bc7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1998,7 +1998,7 @@ F: Documentation/gpu/apple-dcp-backlight.rst F: Documentation/hid/apple-j700-multitouch.rst F: Documentation/soc/apple/t8140-pmp-dvfs.rst F: arch/arm64/boot/dts/apple/t8140* -F: arch/arm64/configs/j700-radio.config +F: arch/arm64/configs/j700*.config F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* diff --git a/arch/arm64/configs/j700.config b/arch/arm64/configs/j700.config new file mode 100644 index 0000000000000..976fd65b7161c --- /dev/null +++ b/arch/arm64/configs/j700.config @@ -0,0 +1,113 @@ +# J700 (MacBook Neo, T8140) base fragment. +# +# Merge after defconfig, asahi.config and j700-radio.config, then add the device-parity fragment: +# make defconfig +# scripts/kconfig/merge_config.sh -m .config arch/arm64/configs/asahi.config arch/arm64/configs/j700-radio.config +# make olddefconfig +# scripts/kconfig/merge_config.sh -m .config arch/arm64/configs/j700.config arch/arm64/configs/j700-device-parity.config +# make olddefconfig +# The result is the configuration the J700 kernel is tested with. + +# Boot NOR (nvram) access for boot-volume switching, and the PMU RTC wake alarm +CONFIG_SPI_APPLE_QSPI=m +CONFIG_RTC_DRV_APPLE_PMU=m + +# Classic Bluetooth profiles (headsets, tethering) +CONFIG_BT_RFCOMM=m +CONFIG_BT_RFCOMM_TTY=y +CONFIG_BT_BNEP=m +CONFIG_BT_BNEP_MC_FILTER=y +CONFIG_BT_BNEP_PROTO_FILTER=y + +# Distribution firmware files are compressed +CONFIG_FW_LOADER_COMPRESS=y +CONFIG_FW_LOADER_COMPRESS_ZSTD=y + +# 48-bit virtual addresses; KASLR is untested on this machine +CONFIG_ARM64_VA_BITS_48=y +# CONFIG_RANDOMIZE_BASE is not set + +# Boot path and platform built in, so storage, keyboard, SMC and USB do not depend on the initramfs +CONFIG_BLK_DEV_NVME=y +CONFIG_NVME_APPLE=y +CONFIG_APPLE_SART=y +CONFIG_APPLE_RTKIT_HELPER=y +CONFIG_BTRFS_FS=y +CONFIG_APPLE_DOCKCHANNEL=y +CONFIG_HID_DOCKCHANNEL=y +CONFIG_HID_APPLE=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_SPMI_APPLE=y +CONFIG_NVMEM_APPLE_SPMI=y +CONFIG_MFD_MACSMC=y +CONFIG_GPIO_MACSMC=y +CONFIG_POWER_RESET_MACSMC=y +CONFIG_SENSORS_MACSMC_HWMON=y +CONFIG_I2C_APPLE=y +CONFIG_PINCTRL_APPLE_GPIO=y +CONFIG_APPLE_WATCHDOG=y +CONFIG_ARM_APPLE_SOC_CPUFREQ=y +CONFIG_PHY_APPLE_ATC=y +CONFIG_USB_DWC3_APPLE=y +CONFIG_TYPEC=y +CONFIG_TYPEC_SN201202X=y +CONFIG_MUX_APPLE_DPXBAR=y +CONFIG_BACKLIGHT_CLASS_DEVICE=y +CONFIG_SYSFB_SIMPLEFB=y +CONFIG_ZRAM=y +CONFIG_UHID=y +CONFIG_RFKILL=y +CONFIG_THERMAL_NETLINK=y +CONFIG_NVME_HWMON=y + +# J700 driver choices +CONFIG_APPLE_CLPC=m +CONFIG_APPLE_PMP=y +CONFIG_DRM_ACCEL=y +CONFIG_DRM_ACCEL_ANE=m +# CONFIG_HID_APPLE_MTP_HAPTIC is not set +CONFIG_IIO=m +# CONFIG_IIO_AOP_SENSOR_LAS is not set +# CONFIG_USB4 is not set + +# Per-process 4K user pages on the 16K kernel (x86 emulation) +CONFIG_ARM64_USER4K_EXPERIMENTAL=y + +# USB Ethernet built in, for a wired link before modules load +CONFIG_USB_USBNET=y +CONFIG_USB_NET_AX8817X=y +CONFIG_USB_NET_AX88179_178A=y +CONFIG_USB_NET_CDC_NCM=y +CONFIG_USB_RTL8152=y +CONFIG_USB_UAS=m + +# Crypto that Bluetooth and Wi-Fi need early +CONFIG_CRYPTO_CCM=y +CONFIG_CRYPTO_ECDH=y +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_SHA256=y + +# Utilisation clamping +CONFIG_UCLAMP_TASK=y +CONFIG_UCLAMP_TASK_GROUP=y + +# LSMs named in CONFIG_LSM +CONFIG_SECURITY_LANDLOCK=y +CONFIG_SECURITY_YAMA=y + +# dma-buf heaps and udmabuf (camera and GPU buffer sharing) +CONFIG_DMABUF_HEAPS=y +CONFIG_DMABUF_HEAPS_SYSTEM=y +CONFIG_UDMABUF=y + +# Debugging aids kept in the tested configuration +CONFIG_DYNAMIC_DEBUG=y +CONFIG_FTRACE=y +CONFIG_ENABLE_DEFAULT_TRACERS=y +CONFIG_PM_DEBUG=y +CONFIG_PM_ADVANCED_DEBUG=y + +# Match the tested kernel image: no relative relocation packing, no SCM version suffix +# CONFIG_RELR is not set +# CONFIG_LOCALVERSION_AUTO is not set From d96ce2dea7fe5c23188253d7e998aad0b82d38b6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 20:57:43 +1100 Subject: [PATCH 0837/1579] arm64: configs: add a J700 device-parity fragment The J700 configuration lacks much of what a general-purpose distribution kernel provides: game controllers and other USB or Bluetooth HID devices, most USB-serial and USB network adapters, filesystems found on removable media, dm-crypt, VPN protocols, the nftables and xtables extensions used by firewall and container tools, and binfmt_misc. Add j700-device-parity.config, to be merged after j700.config. It takes the Fedora Asahi Remix 44 kernel configuration as the reference and enables what can attach to or run on the J700, mostly as modules. It changes no option that the other fragments set. Options that change scheduler, memory, security or boot behaviour are left out. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/j700-device-parity.config | 1053 ++++++++++++++++++ 1 file changed, 1053 insertions(+) create mode 100644 arch/arm64/configs/j700-device-parity.config diff --git a/arch/arm64/configs/j700-device-parity.config b/arch/arm64/configs/j700-device-parity.config new file mode 100644 index 0000000000000..55bce7a9ac0af --- /dev/null +++ b/arch/arm64/configs/j700-device-parity.config @@ -0,0 +1,1053 @@ +# J700 (MacBook Neo, T8140) device-parity fragment. +# +# Brings the J700 configuration to device and feature parity with a general-purpose +# distribution kernel (reference: Fedora Asahi Remix 44, 7.1.13-402.asahi.fc44.aarch64+16k): +# drivers for devices that can attach over USB or Bluetooth, and common protocols, +# filesystems and features. Most entries are modules. Nothing here changes an option +# that defconfig, asahi.config, j700-radio.config or j700.config set. +# +# Merge after j700.config; see that file for the full sequence. + +# Prerequisites of the entries below: nftables core, hidraw, uinput and the ALSA sequencer +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_IPV4=y +CONFIG_NF_TABLES_IPV6=y +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_CT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_NAT=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_REJECT_INET=m +CONFIG_NFT_FIB_INET=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NFT_FIB_IPV6=m +CONFIG_HIDRAW=y +CONFIG_INPUT_UINPUT=m +CONFIG_SND_SEQUENCER=m +CONFIG_SND_HRTIMER=m +CONFIG_SND_SEQ_DUMMY=m +CONFIG_SND_SEQ_HRTIMER_DEFAULT=y + +# HID / input / game controllers (USB + Bluetooth HID devices, xpad, joydev, USB tablets/touchscreens) +CONFIG_DRAGONRISE_FF=y +CONFIG_GREENASIA_FF=y +CONFIG_HID_A4TECH=m +CONFIG_HID_ACCUTOUCH=m +CONFIG_HID_ACRUX=m +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_ALPS=m +CONFIG_HID_APPLEIR=m +CONFIG_HID_AUREAL=m +CONFIG_HID_BATTERY_STRENGTH=y +CONFIG_HID_BELKIN=m +CONFIG_HID_BETOP_FF=m +CONFIG_HID_BIGBEN_FF=m +CONFIG_HID_CHERRY=m +CONFIG_HID_CHICONY=m +CONFIG_HID_CMEDIA=m +CONFIG_HID_CORSAIR=m +CONFIG_HID_COUGAR=m +CONFIG_HID_CP2112=m +CONFIG_HID_CREATIVE_SB0540=m +CONFIG_HID_CYPRESS=m +CONFIG_HID_DRAGONRISE=m +CONFIG_HID_ELAN=m +CONFIG_HID_ELECOM=m +CONFIG_HID_ELO=m +CONFIG_HID_EMS_FF=m +CONFIG_HID_EVISION=m +CONFIG_HID_EZKEY=m +CONFIG_HID_FT260=m +CONFIG_HID_GEMBIRD=m +CONFIG_HID_GFRM=m +CONFIG_HID_GLORIOUS=m +CONFIG_HID_GOOGLE_STADIA_FF=m +CONFIG_HID_GREENASIA=m +CONFIG_HID_GT683R=m +CONFIG_HID_GYRATION=m +CONFIG_HID_HOLTEK=m +CONFIG_HID_HUAWEI=m +CONFIG_HID_ICADE=m +CONFIG_HID_ITE=m +CONFIG_HID_JABRA=m +CONFIG_HID_KENSINGTON=m +CONFIG_HID_KEYTOUCH=m +CONFIG_HID_KYE=m +CONFIG_HID_KYSONA=m +CONFIG_HID_LCPOWER=m +CONFIG_HID_LED=m +CONFIG_HID_LENOVO=m +CONFIG_HID_LENOVO_GO=m +CONFIG_HID_LENOVO_GO_S=m +CONFIG_HID_LETSKETCH=m +CONFIG_HID_LOGITECH=m +CONFIG_HID_LOGITECH_DJ=m +CONFIG_HID_LOGITECH_HIDPP=m +CONFIG_HID_MACALLY=m +CONFIG_HID_MALTRON=m +CONFIG_HID_MAYFLASH=m +CONFIG_HID_MCP2221=m +CONFIG_HID_MEGAWORLD_FF=m +CONFIG_HID_MICROSOFT=m +CONFIG_HID_MONTEREY=m +CONFIG_HID_NINTENDO=m +CONFIG_HID_NTI=m +CONFIG_HID_NTRIG=m +CONFIG_HID_NVIDIA_SHIELD=m +CONFIG_HID_ORTEK=m +CONFIG_HID_PANTHERLORD=m +CONFIG_HID_PENMOUNT=m +CONFIG_HID_PETALYNX=m +CONFIG_HID_PICOLCD=m +CONFIG_HID_PICOLCD_BACKLIGHT=y +CONFIG_HID_PICOLCD_FB=y +CONFIG_HID_PICOLCD_LEDS=y +CONFIG_HID_PID=y +CONFIG_HID_PLANTRONICS=m +CONFIG_HID_PLAYSTATION=m +CONFIG_HID_PRIMAX=m +CONFIG_HID_PRODIKEYS=m +CONFIG_HID_PXRC=m +CONFIG_HID_RAPOO=m +CONFIG_HID_RAZER=m +CONFIG_HID_RETRODE=m +CONFIG_HID_RMI=m +CONFIG_HID_ROCCAT=m +CONFIG_HID_SAITEK=m +CONFIG_HID_SAMSUNG=m +CONFIG_HID_SEMITEK=m +CONFIG_HID_SIGMAMICRO=m +CONFIG_HID_SMARTJOYPLUS=m +CONFIG_HID_SONY=m +CONFIG_HID_SPEEDLINK=m +CONFIG_HID_STEAM=m +CONFIG_HID_STEELSERIES=m +CONFIG_HID_SUNPLUS=m +CONFIG_HID_THINGM=m +CONFIG_HID_THRUSTMASTER=m +CONFIG_HID_TIVO=m +CONFIG_HID_TOPRE=m +CONFIG_HID_TOPSEED=m +CONFIG_HID_TWINHAN=m +CONFIG_HID_U2FZERO=m +CONFIG_HID_UCLOGIC=m +CONFIG_HID_UDRAW_PS3=m +CONFIG_HID_UNIVERSAL_PIDFF=m +CONFIG_HID_VIEWSONIC=m +CONFIG_HID_VIVALDI=m +CONFIG_HID_WACOM=m +CONFIG_HID_WALTOP=m +CONFIG_HID_WIIMOTE=m +CONFIG_HID_WINWING=m +CONFIG_HID_XIAOMI=m +CONFIG_HID_XINMO=m +CONFIG_HID_ZEROPLUS=m +CONFIG_HID_ZYDACRON=m +CONFIG_HOLTEK_FF=y +CONFIG_INPUT_JOYDEV=m +CONFIG_INPUT_JOYSTICK=y +CONFIG_INPUT_TABLET=y +CONFIG_JOYSTICK_IFORCE=m +CONFIG_JOYSTICK_IFORCE_USB=m +CONFIG_JOYSTICK_PXRC=m +CONFIG_JOYSTICK_XPAD=m +CONFIG_JOYSTICK_XPAD_FF=y +CONFIG_JOYSTICK_XPAD_LEDS=y +CONFIG_LOGIG940_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGITECH_FF=y +CONFIG_LOGIWHEELS_FF=y +CONFIG_MOUSE_SYNAPTICS_USB=m +CONFIG_NINTENDO_FF=y +CONFIG_NVIDIA_SHIELD_FF=y +CONFIG_PANTHERLORD_FF=y +CONFIG_PLAYSTATION_FF=y +CONFIG_RMI4_CORE=m +CONFIG_RMI4_F03=y +CONFIG_RMI4_F11=y +CONFIG_RMI4_F12=y +CONFIG_RMI4_F1A=y +CONFIG_RMI4_F21=y +CONFIG_RMI4_F30=y +CONFIG_RMI4_F34=y +CONFIG_RMI4_F3A=y +CONFIG_RMI4_F55=y +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_SONY_FF=y +CONFIG_STEAM_FF=y +CONFIG_TABLET_USB_ACECAD=m +CONFIG_TABLET_USB_AIPTEK=m +CONFIG_TABLET_USB_HANWANG=m +CONFIG_TABLET_USB_KBTAB=m +CONFIG_TABLET_USB_PEGASUS=m +CONFIG_THRUSTMASTER_FF=y +CONFIG_TOUCHSCREEN_USB_3M=y +CONFIG_TOUCHSCREEN_USB_COMPOSITE=m +CONFIG_TOUCHSCREEN_USB_DMC_TSC10=y +CONFIG_TOUCHSCREEN_USB_E2I=y +CONFIG_TOUCHSCREEN_USB_EASYTOUCH=y +CONFIG_TOUCHSCREEN_USB_EGALAX=y +CONFIG_TOUCHSCREEN_USB_ELO=y +CONFIG_TOUCHSCREEN_USB_ETT_TC45USB=y +CONFIG_TOUCHSCREEN_USB_ETURBO=y +CONFIG_TOUCHSCREEN_USB_GENERAL_TOUCH=y +CONFIG_TOUCHSCREEN_USB_GOTOP=y +CONFIG_TOUCHSCREEN_USB_GUNZE=y +CONFIG_TOUCHSCREEN_USB_IDEALTEK=y +CONFIG_TOUCHSCREEN_USB_IRTOUCH=y +CONFIG_TOUCHSCREEN_USB_ITM=y +CONFIG_TOUCHSCREEN_USB_JASTEC=y +CONFIG_TOUCHSCREEN_USB_NEXIO=y +CONFIG_TOUCHSCREEN_USB_PANJIT=y +CONFIG_TOUCHSCREEN_USB_ZYTRONIC=y +CONFIG_USB_HIDDEV=y +CONFIG_ZEROPLUS_FF=y + +# USB device classes: serial adapters, storage sub-drivers, printers/TMC/WDM, misc USB gadgets, usbmon, USB/IP, card readers, maker adapters +CONFIG_APPLE_MFI_FASTCHARGE=m +CONFIG_DLN2_ADC=m +CONFIG_GNSS_USB=m +CONFIG_GPIB=m +CONFIG_GPIB_AGILENT_82357A=m +CONFIG_GPIB_COMMON=m +CONFIG_GPIB_LPVO=m +CONFIG_GPIB_NI_USB=m +CONFIG_GPIO_DLN2=m +CONFIG_GPIO_MPSSE=m +CONFIG_I2C_CP2615=m +CONFIG_I2C_DIOLAN_U2C=m +CONFIG_I2C_DLN2=m +CONFIG_I2C_TINY_USB=m +CONFIG_MEMSTICK=m +CONFIG_MEMSTICK_REALTEK_USB=m +CONFIG_MFD_DLN2=m +CONFIG_MISC_RTSX_USB=m +CONFIG_MMC_REALTEK_USB=m +CONFIG_MMC_USHC=m +CONFIG_MMC_VUB300=m +CONFIG_MSPRO_BLOCK=m +CONFIG_REALTEK_AUTOPM=y +CONFIG_SENSORS_AQUACOMPUTER_D5NEXT=m +CONFIG_SENSORS_ASUS_ROG_RYUJIN=m +CONFIG_SENSORS_CORSAIR_CPRO=m +CONFIG_SENSORS_CORSAIR_PSU=m +CONFIG_SENSORS_GIGABYTE_WATERFORCE=m +CONFIG_SENSORS_NZXT_KRAKEN2=m +CONFIG_SENSORS_NZXT_KRAKEN3=m +CONFIG_SENSORS_NZXT_SMART2=m +CONFIG_SENSORS_POWERZ=m +CONFIG_SPI_CH341=m +CONFIG_SPI_DLN2=m +CONFIG_USBIP_CORE=m +CONFIG_USBIP_HOST=m +CONFIG_USBIP_VHCI_HCD=m +CONFIG_USBPCWATCHDOG=m +CONFIG_USB_ADUTUX=m +CONFIG_USB_APPLEDISPLAY=m +CONFIG_USB_CHAOSKEY=m +CONFIG_USB_EMI26=m +CONFIG_USB_EMI62=m +CONFIG_USB_EZUSB_FX2=m +CONFIG_USB_IDMOUSE=m +CONFIG_USB_IOWARRIOR=m +CONFIG_USB_ISIGHTFW=m +CONFIG_USB_LCD=m +CONFIG_USB_LD=m +CONFIG_USB_LEGOTOWER=m +CONFIG_USB_MDC800=m +CONFIG_USB_MICROTEK=m +CONFIG_USB_MON=m +CONFIG_USB_PRINTER=m +CONFIG_USB_SERIAL_AIRCABLE=m +CONFIG_USB_SERIAL_ARK3116=m +CONFIG_USB_SERIAL_BELKIN=m +CONFIG_USB_SERIAL_CH341=m +CONFIG_USB_SERIAL_CYBERJACK=m +CONFIG_USB_SERIAL_CYPRESS_M8=m +CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m +CONFIG_USB_SERIAL_EDGEPORT=m +CONFIG_USB_SERIAL_EDGEPORT_TI=m +CONFIG_USB_SERIAL_EMPEG=m +CONFIG_USB_SERIAL_F81232=m +CONFIG_USB_SERIAL_F8153X=m +CONFIG_USB_SERIAL_GARMIN=m +CONFIG_USB_SERIAL_GENERIC=y +CONFIG_USB_SERIAL_IPAQ=m +CONFIG_USB_SERIAL_IPW=m +CONFIG_USB_SERIAL_IR=m +CONFIG_USB_SERIAL_IUU=m +CONFIG_USB_SERIAL_KEYSPAN=m +CONFIG_USB_SERIAL_KEYSPAN_PDA=m +CONFIG_USB_SERIAL_KLSI=m +CONFIG_USB_SERIAL_KOBIL_SCT=m +CONFIG_USB_SERIAL_MCT_U232=m +CONFIG_USB_SERIAL_MOS7720=m +CONFIG_USB_SERIAL_MOS7840=m +CONFIG_USB_SERIAL_NAVMAN=m +CONFIG_USB_SERIAL_OMNINET=m +CONFIG_USB_SERIAL_OPTICON=m +CONFIG_USB_SERIAL_OTI6858=m +CONFIG_USB_SERIAL_PL2303=m +CONFIG_USB_SERIAL_QCAUX=m +CONFIG_USB_SERIAL_QT2=m +CONFIG_USB_SERIAL_QUALCOMM=m +CONFIG_USB_SERIAL_SAFE=m +CONFIG_USB_SERIAL_SAFE_PADDED=y +CONFIG_USB_SERIAL_SIERRAWIRELESS=m +CONFIG_USB_SERIAL_SIMPLE=m +CONFIG_USB_SERIAL_SPCP8X5=m +CONFIG_USB_SERIAL_SSU100=m +CONFIG_USB_SERIAL_SYMBOL=m +CONFIG_USB_SERIAL_TI=m +CONFIG_USB_SERIAL_UPD78F0730=m +CONFIG_USB_SERIAL_VISOR=m +CONFIG_USB_SERIAL_WHITEHEAT=m +CONFIG_USB_SERIAL_XR=m +CONFIG_USB_SERIAL_XSENS_MT=m +CONFIG_USB_SEVSEG=m +CONFIG_USB_SISUSBVGA=m +CONFIG_USB_STORAGE_ALAUDA=m +CONFIG_USB_STORAGE_CYPRESS_ATACB=m +CONFIG_USB_STORAGE_DATAFAB=m +CONFIG_USB_STORAGE_ENE_UB6250=m +CONFIG_USB_STORAGE_FREECOM=m +CONFIG_USB_STORAGE_ISD200=m +CONFIG_USB_STORAGE_JUMPSHOT=m +CONFIG_USB_STORAGE_KARMA=m +CONFIG_USB_STORAGE_ONETOUCH=m +CONFIG_USB_STORAGE_REALTEK=m +CONFIG_USB_STORAGE_SDDR09=m +CONFIG_USB_STORAGE_SDDR55=m +CONFIG_USB_STORAGE_USBAT=m +CONFIG_USB_TMC=m +CONFIG_USB_TRANCEVIBRATOR=m +CONFIG_USB_WDM=m +CONFIG_USB_YUREX=m +CONFIG_W1=m +CONFIG_W1_MASTER_DS2490=m +CONFIG_W1_SLAVE_DS2405=m +CONFIG_W1_SLAVE_DS2406=m +CONFIG_W1_SLAVE_DS2408=m +CONFIG_W1_SLAVE_DS2413=m +CONFIG_W1_SLAVE_DS2423=m +CONFIG_W1_SLAVE_DS2430=m +CONFIG_W1_SLAVE_DS2431=m +CONFIG_W1_SLAVE_DS2433=m +CONFIG_W1_SLAVE_DS2433_CRC=y +CONFIG_W1_SLAVE_DS2438=m +CONFIG_W1_SLAVE_DS2780=m +CONFIG_W1_SLAVE_DS2781=m +CONFIG_W1_SLAVE_DS2805=m +CONFIG_W1_SLAVE_DS28E04=m +CONFIG_W1_SLAVE_SMEM=m +CONFIG_W1_SLAVE_THERM=m +CONFIG_XILLYUSB=m + +# USB network adapters: USB Ethernet/tethering/mobile broadband, USB Wi-Fi dongles, USB CAN adapters +CONFIG_AR5523=m +CONFIG_ATH10K_USB=m +CONFIG_ATH6KL=m +CONFIG_ATH6KL_USB=m +CONFIG_ATH9K_HTC=m +CONFIG_BRCMFMAC_USB=y +CONFIG_CAN_8DEV_USB=m +CONFIG_CAN_EMS_USB=m +CONFIG_CAN_ESD_USB=m +CONFIG_CAN_F81604=m +CONFIG_CAN_GS_USB=m +CONFIG_CAN_ISOTP=m +CONFIG_CAN_J1939=m +CONFIG_CAN_KVASER_USB=m +CONFIG_CAN_MCBA_USB=m +CONFIG_CAN_PEAK_USB=m +CONFIG_CARL9170=m +CONFIG_CARL9170_LEDS=y +CONFIG_LIBERTAS_THINFIRM=m +CONFIG_LIBERTAS_THINFIRM_USB=m +CONFIG_MT7601U=m +CONFIG_MT7663U=m +CONFIG_MT76x0U=m +CONFIG_MT76x2U=m +CONFIG_MT7921U=m +CONFIG_MT7925U=m +CONFIG_RT2500USB=m +CONFIG_RT2800USB=m +CONFIG_RT2800USB_RT33XX=y +CONFIG_RT2800USB_RT3573=y +CONFIG_RT2800USB_RT35XX=y +CONFIG_RT2800USB_RT53XX=y +CONFIG_RT2800USB_RT55XX=y +CONFIG_RT2800USB_UNKNOWN=y +CONFIG_RT2X00=m +CONFIG_RT73USB=m +CONFIG_RTL8187=m +CONFIG_RTL8192DU=m +CONFIG_RTL8XXXU=m +CONFIG_RTW88=m +CONFIG_RTW88_8723DU=m +CONFIG_RTW88_8812AU=m +CONFIG_RTW88_8814AU=m +CONFIG_RTW88_8821AU=m +CONFIG_RTW88_8821CU=m +CONFIG_RTW88_8822BU=m +CONFIG_RTW88_8822CU=m +CONFIG_RTW89=m +CONFIG_RTW89_8851BU=m +CONFIG_RTW89_8852AU=m +CONFIG_RTW89_8852BU=m +CONFIG_RTW89_8852CU=m +CONFIG_USB_ALI_M5632=y +CONFIG_USB_AN2720=y +CONFIG_USB_CATC=m +CONFIG_USB_EPSON2888=y +CONFIG_USB_HSO=m +CONFIG_USB_IPHETH=m +CONFIG_USB_KAWETH=m +CONFIG_USB_KC2190=y +CONFIG_USB_NET_AQC111=m +CONFIG_USB_NET_CDC_EEM=m +CONFIG_USB_NET_CDC_MBIM=m +CONFIG_USB_NET_CH9200=m +CONFIG_USB_NET_CX82310_ETH=m +CONFIG_USB_NET_GL620A=m +CONFIG_USB_NET_HUAWEI_CDC_NCM=m +CONFIG_USB_NET_INT51X1=m +CONFIG_USB_NET_KALMIA=m +CONFIG_USB_NET_QMI_WWAN=m +CONFIG_USB_NET_RNDIS_HOST=m +CONFIG_USB_NET_SR9700=m +CONFIG_USB_SIERRA_NET=m +CONFIG_USB_VL600=m +CONFIG_ZD1211RW=m + +# Storage: optical/SCSI generic, device-mapper (LUKS, LVM, verity, integrity), MD RAID, partition tables +CONFIG_AIX_PARTITION=y +CONFIG_BLK_DEV_SR=m +CONFIG_BSD_DISKLABEL=y +CONFIG_CHR_DEV_SG=m +CONFIG_DM_CACHE=m +CONFIG_DM_CACHE_SMQ=m +CONFIG_DM_CRYPT=m +CONFIG_DM_INTEGRITY=m +CONFIG_DM_RAID=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_UEVENT=y +CONFIG_DM_VERITY=m +CONFIG_DM_VERITY_FEC=y +CONFIG_DM_WRITECACHE=m +CONFIG_LDM_PARTITION=y +CONFIG_MAC_PARTITION=y +CONFIG_MD_LINEAR=m +CONFIG_MD_RAID0=m +CONFIG_MD_RAID1=m +CONFIG_MD_RAID10=m +CONFIG_MD_RAID456=m +CONFIG_MINIX_SUBPARTITION=y +CONFIG_OSF_PARTITION=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_SGI_PARTITION=y +CONFIG_SOLARIS_X86_PARTITION=y +CONFIG_SUN_PARTITION=y +CONFIG_UNIXWARE_DISKLABEL=y + +# Filesystems for removable media and images, binfmt_misc, SMB client +CONFIG_BINFMT_MISC=m +CONFIG_CIFS=m +CONFIG_CIFS_ALLOW_INSECURE_LEGACY=y +CONFIG_CIFS_DFS_UPCALL=y +CONFIG_CIFS_POSIX=y +CONFIG_CIFS_SWN_UPCALL=y +CONFIG_CIFS_UPCALL=y +CONFIG_CIFS_XATTR=y +CONFIG_EROFS_FS=m +CONFIG_EROFS_FS_BACKED_BY_FILE=y +CONFIG_EROFS_FS_POSIX_ACL=y +CONFIG_EROFS_FS_SECURITY=y +CONFIG_EROFS_FS_XATTR=y +CONFIG_EROFS_FS_ZIP=y +CONFIG_EROFS_FS_ZIP_ACCEL=y +CONFIG_EROFS_FS_ZIP_DEFLATE=y +CONFIG_EROFS_FS_ZIP_LZMA=y +CONFIG_EROFS_FS_ZIP_ZSTD=y +CONFIG_EXFAT_FS=m +CONFIG_F2FS_FS=m +CONFIG_F2FS_FS_COMPRESSION=y +CONFIG_F2FS_FS_LZ4=y +CONFIG_F2FS_FS_LZ4HC=y +CONFIG_F2FS_FS_LZO=y +CONFIG_F2FS_FS_LZORLE=y +CONFIG_F2FS_FS_POSIX_ACL=y +CONFIG_F2FS_FS_SECURITY=y +CONFIG_F2FS_FS_XATTR=y +CONFIG_F2FS_FS_ZSTD=y +CONFIG_F2FS_IOSTAT=y +CONFIG_F2FS_STAT_FS=y +CONFIG_F2FS_UNFAIR_RWSEM=y +CONFIG_HFSPLUS_FS=m +CONFIG_HFS_FS=m +CONFIG_ISO9660_FS=m +CONFIG_JOLIET=y +CONFIG_MSDOS_FS=m +CONFIG_NLS_ASCII=m +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_936=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_950=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_NLS_UTF8=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_FS_POSIX_ACL=y +CONFIG_NTFS3_LZX_XPRESS=y +CONFIG_SQUASHFS_LZ4=y +CONFIG_SQUASHFS_LZO=y +CONFIG_SQUASHFS_XATTR=y +CONFIG_SQUASHFS_XZ=y +CONFIG_SQUASHFS_ZSTD=y +CONFIG_UDF_FS=m +CONFIG_XFS_FS=m +CONFIG_XFS_ONLINE_REPAIR=y +CONFIG_XFS_ONLINE_SCRUB=y +CONFIG_XFS_ONLINE_SCRUB_STATS=y +CONFIG_XFS_POSIX_ACL=y +CONFIG_XFS_QUOTA=y +CONFIG_XFS_RT=y +CONFIG_XFS_SUPPORT_ASCII_CI=y +CONFIG_XFS_SUPPORT_V4=y +CONFIG_ZISOFS=y + +# Network protocols: WireGuard/OpenVPN-DCO/L2TP/PPP/IPsec/kTLS, tunnels, policy routing, nftables+xtables+ipset+ebtables, tc, TCP CC, ss diag, virtual links +CONFIG_BONDING=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_IP6=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_NF_EBTABLES_LEGACY=m +CONFIG_CLS_U32_MARK=y +CONFIG_CLS_U32_PERF=y +CONFIG_DEFAULT_CUBIC=y +CONFIG_DUMMY=m +CONFIG_GACT_PROB=y +CONFIG_GENEVE=m +CONFIG_IFB=m +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_ESPINTCP=y +CONFIG_INET6_ESP_OFFLOAD=m +CONFIG_INET6_IPCOMP=m +CONFIG_INET_AH=m +CONFIG_INET_DIAG_DESTROY=y +CONFIG_INET_ESP=m +CONFIG_INET_ESPINTCP=y +CONFIG_INET_ESP_OFFLOAD=m +CONFIG_INET_IPCOMP=m +CONFIG_INET_RAW_DIAG=m +CONFIG_INET_UDP_DIAG=m +CONFIG_IP6_NF_FILTER=m +CONFIG_IP6_NF_IPTABLES_LEGACY=m +CONFIG_IP6_NF_MANGLE=m +CONFIG_IP6_NF_MATCH_AH=m +CONFIG_IP6_NF_MATCH_EUI64=m +CONFIG_IP6_NF_MATCH_FRAG=m +CONFIG_IP6_NF_MATCH_HL=m +CONFIG_IP6_NF_MATCH_IPV6HEADER=m +CONFIG_IP6_NF_MATCH_MH=m +CONFIG_IP6_NF_MATCH_OPTS=m +CONFIG_IP6_NF_MATCH_RPFILTER=m +CONFIG_IP6_NF_MATCH_RT=m +CONFIG_IP6_NF_MATCH_SRH=m +CONFIG_IP6_NF_NAT=m +CONFIG_IP6_NF_RAW=m +CONFIG_IP6_NF_TARGET_HL=m +CONFIG_IP6_NF_TARGET_MASQUERADE=m +CONFIG_IP6_NF_TARGET_NPT=m +CONFIG_IP6_NF_TARGET_REJECT=m +CONFIG_IPV6_GRE=m +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_IPV6_TUNNEL=m +CONFIG_IPV6_VTI=m +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_IPTABLES_LEGACY=m +CONFIG_IP_NF_MANGLE=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMAC=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_L2TP=m +CONFIG_L2TP_ETH=m +CONFIG_L2TP_IP=m +CONFIG_L2TP_V3=y +CONFIG_NETFILTER_NETLINK_ACCT=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NETFILTER_NETLINK_HOOK=m +CONFIG_NETFILTER_NETLINK_LOG=m +CONFIG_NETFILTER_NETLINK_OSF=m +CONFIG_NETFILTER_NETLINK_QUEUE=m +CONFIG_NETFILTER_XTABLES_LEGACY=y +CONFIG_NETFILTER_XT_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DCCP=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ECN=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_HL=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_L2TP=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SCTP=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_NETFILTER_XT_NAT=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_AUDIT=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_CT=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HL=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LED=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_MASQUERADE=m +CONFIG_NETFILTER_XT_TARGET_NETMAP=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_NOTRACK=m +CONFIG_NETFILTER_XT_TARGET_RATEEST=m +CONFIG_NETFILTER_XT_TARGET_REDIRECT=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETLINK_DIAG=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_CONNMARK=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_CT=m +CONFIG_NET_ACT_CTINFO=m +CONFIG_NET_ACT_MPLS=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_U32=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_IPSET=m +CONFIG_NET_EMATCH_IPT=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_FOU=m +CONFIG_NET_FOU_IP_TUNNELS=y +CONFIG_NET_IPGRE=m +CONFIG_NET_IPGRE_BROADCAST=y +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_NET_IPIP=m +CONFIG_NET_IPVTI=m +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_NET_SCH_CAKE=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_DUALPI2=m +CONFIG_NET_SCH_ETS=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_FQ_PIE=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_TEQL=m +CONFIG_NET_TEAM=m +CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m +CONFIG_NET_TEAM_MODE_BROADCAST=m +CONFIG_NET_TEAM_MODE_LOADBALANCE=m +CONFIG_NET_TEAM_MODE_RANDOM=m +CONFIG_NET_TEAM_MODE_ROUNDROBIN=m +CONFIG_NFT_BRIDGE_META=m +CONFIG_NFT_BRIDGE_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_CONNLIMIT=m +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_DUP_IPV6=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FIB_NETDEV=m +CONFIG_NFT_FLOW_OFFLOAD=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NFT_HASH=m +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_OSF=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_REJECT_NETDEV=m +CONFIG_NFT_SOCKET=m +CONFIG_NFT_TPROXY=m +CONFIG_NFT_TUNNEL=m +CONFIG_NFT_XFRM=m +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_BRIDGE=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_LABELS=y +CONFIG_NF_CONNTRACK_MARK=y +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_PROCFS=y +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_DUP_IPV4=m +CONFIG_NF_DUP_IPV6=m +CONFIG_NF_DUP_NETDEV=m +CONFIG_NF_FLOW_TABLE=m +CONFIG_NF_FLOW_TABLE_INET=m +CONFIG_NF_FLOW_TABLE_PROCFS=y +CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m +CONFIG_NF_NAT_SNMP_BASIC=m +CONFIG_NF_SOCKET_IPV4=m +CONFIG_NF_SOCKET_IPV6=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_TABLES_BRIDGE=m +CONFIG_NF_TPROXY_IPV4=m +CONFIG_NF_TPROXY_IPV6=m +CONFIG_OVPN=m +CONFIG_PACKET_DIAG=m +CONFIG_PPP=m +CONFIG_PPPOE=m +CONFIG_PPPOE_HASH_BITS_4=y +CONFIG_PPPOL2TP=m +CONFIG_PPP_ASYNC=m +CONFIG_PPP_BSDCOMP=m +CONFIG_PPP_DEFLATE=m +CONFIG_PPP_FILTER=y +CONFIG_PPP_MPPE=m +CONFIG_PPP_MULTILINK=y +CONFIG_PPP_SYNC_TTY=m +CONFIG_PPTP=m +CONFIG_PSAMPLE=m +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_CONG_BBR=m +CONFIG_TCP_CONG_BIC=m +CONFIG_TCP_CONG_CDG=m +CONFIG_TCP_CONG_DCTCP=m +CONFIG_TCP_CONG_HSTCP=m +CONFIG_TCP_CONG_HTCP=m +CONFIG_TCP_CONG_HYBLA=m +CONFIG_TCP_CONG_ILLINOIS=m +CONFIG_TCP_CONG_LP=m +CONFIG_TCP_CONG_NV=m +CONFIG_TCP_CONG_SCALABLE=m +CONFIG_TCP_CONG_VEGAS=m +CONFIG_TCP_CONG_VENO=m +CONFIG_TCP_CONG_WESTWOOD=m +CONFIG_TCP_CONG_YEAH=m +CONFIG_TLS=m +CONFIG_UNIX_DIAG=m +CONFIG_VXLAN=m +CONFIG_WIREGUARD=m +CONFIG_XFRM_INTERFACE=m +CONFIG_XFRM_MIGRATE=y +CONFIG_XFRM_STATISTICS=y +CONFIG_XFRM_USER=m + +# Bluetooth: USB dongle drivers, vhci, MSFT/AOSP vendor extensions (USB controllers only) +CONFIG_BT_AOSPEXT=y +CONFIG_BT_ATH3K=m +CONFIG_BT_HCIBCM203X=m +CONFIG_BT_HCIBFUSB=m +CONFIG_BT_HCIBPA10X=m +CONFIG_BT_HCIBTUSB_AUTOSUSPEND=y +CONFIG_BT_HCIVHCI=m +CONFIG_BT_MSFTEXT=y + +# Containers / virtualisation: CRIU/checkpoint, userfaultfd, misc cgroup, KVM host vhost-net/vhost-vsock/vsock +CONFIG_CGROUP_MISC=y +CONFIG_CHECKPOINT_RESTORE=y +CONFIG_USERFAULTFD=y +CONFIG_VHOST_NET=m +CONFIG_VHOST_VSOCK=m +CONFIG_VSOCKETS=m +CONFIG_VSOCKETS_DIAG=m +CONFIG_VSOCKETS_LOOPBACK=m + +# Crypto: AF_ALG user API (cryptsetup/iwd), dm-crypt/fscrypt modes, VeraCrypt/TrueCrypt ciphers+hashes, IPsec AEADs +CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128_SIMD=y +CONFIG_CRYPTO_BLAKE2B=m +CONFIG_CRYPTO_BLOWFISH=m +CONFIG_CRYPTO_CAMELLIA=m +CONFIG_CRYPTO_CAST5=m +CONFIG_CRYPTO_CAST6=m +CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_CRC32=m +CONFIG_CRYPTO_CRC32C=m +CONFIG_CRYPTO_CRYPTD=m +CONFIG_CRYPTO_CTS=m +CONFIG_CRYPTO_ESSIV=m +CONFIG_CRYPTO_HCTR2=m +CONFIG_CRYPTO_LRW=m +CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_NULL=m +CONFIG_CRYPTO_PCBC=m +CONFIG_CRYPTO_RMD160=m +CONFIG_CRYPTO_SEQIV=m +CONFIG_CRYPTO_SERPENT=m +CONFIG_CRYPTO_STREEBOG=m +CONFIG_CRYPTO_TWOFISH=m +CONFIG_CRYPTO_USER_API_AEAD=m +CONFIG_CRYPTO_USER_API_HASH=m +CONFIG_CRYPTO_USER_API_SKCIPHER=m +CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_XCBC=m +CONFIG_CRYPTO_XXHASH=m + +# Sound: USB audio/MIDI device drivers, MIDI 2.0, virtual MIDI, dummy card, userspace timers +CONFIG_SND_BCD2000=m +CONFIG_SND_DUMMY=m +CONFIG_SND_SEQ_UMP=y +CONFIG_SND_UMP_LEGACY_RAWMIDI=y +CONFIG_SND_USB_6FIRE=m +CONFIG_SND_USB_AUDIO_MIDI_V2=y +CONFIG_SND_USB_CAIAQ=m +CONFIG_SND_USB_CAIAQ_INPUT=y +CONFIG_SND_USB_HIFACE=m +CONFIG_SND_USB_POD=m +CONFIG_SND_USB_PODHD=m +CONFIG_SND_USB_TONEPORT=m +CONFIG_SND_USB_UA101=m +CONFIG_SND_USB_VARIAX=m +CONFIG_SND_UTIMER=y +CONFIG_SND_VIRMIDI=m + +# Video: non-UVC USB webcams (gspca family, Philips pwc), USB displays (DisplayLink USB2 udl, GUD, GM12U320) +CONFIG_DRM_GM12U320=m +CONFIG_DRM_GUD=m +CONFIG_DRM_UDL=m +CONFIG_USB_GL860=m +CONFIG_USB_GSPCA=m +CONFIG_USB_GSPCA_BENQ=m +CONFIG_USB_GSPCA_CONEX=m +CONFIG_USB_GSPCA_CPIA1=m +CONFIG_USB_GSPCA_DTCS033=m +CONFIG_USB_GSPCA_ETOMS=m +CONFIG_USB_GSPCA_FINEPIX=m +CONFIG_USB_GSPCA_JEILINJ=m +CONFIG_USB_GSPCA_JL2005BCD=m +CONFIG_USB_GSPCA_KINECT=m +CONFIG_USB_GSPCA_KONICA=m +CONFIG_USB_GSPCA_MARS=m +CONFIG_USB_GSPCA_MR97310A=m +CONFIG_USB_GSPCA_NW80X=m +CONFIG_USB_GSPCA_OV519=m +CONFIG_USB_GSPCA_OV534=m +CONFIG_USB_GSPCA_OV534_9=m +CONFIG_USB_GSPCA_PAC207=m +CONFIG_USB_GSPCA_PAC7302=m +CONFIG_USB_GSPCA_PAC7311=m +CONFIG_USB_GSPCA_SE401=m +CONFIG_USB_GSPCA_SN9C2028=m +CONFIG_USB_GSPCA_SN9C20X=m +CONFIG_USB_GSPCA_SONIXB=m +CONFIG_USB_GSPCA_SONIXJ=m +CONFIG_USB_GSPCA_SPCA1528=m +CONFIG_USB_GSPCA_SPCA500=m +CONFIG_USB_GSPCA_SPCA501=m +CONFIG_USB_GSPCA_SPCA505=m +CONFIG_USB_GSPCA_SPCA506=m +CONFIG_USB_GSPCA_SPCA508=m +CONFIG_USB_GSPCA_SPCA561=m +CONFIG_USB_GSPCA_SQ905=m +CONFIG_USB_GSPCA_SQ905C=m +CONFIG_USB_GSPCA_SQ930X=m +CONFIG_USB_GSPCA_STK014=m +CONFIG_USB_GSPCA_STK1135=m +CONFIG_USB_GSPCA_STV0680=m +CONFIG_USB_GSPCA_SUNPLUS=m +CONFIG_USB_GSPCA_T613=m +CONFIG_USB_GSPCA_TOPRO=m +CONFIG_USB_GSPCA_TOUPTEK=m +CONFIG_USB_GSPCA_TV8532=m +CONFIG_USB_GSPCA_VC032X=m +CONFIG_USB_GSPCA_VICAM=m +CONFIG_USB_GSPCA_XIRLINK_CIT=m +CONFIG_USB_GSPCA_ZC3XX=m +CONFIG_USB_M5602=m +CONFIG_USB_PWC=m +CONFIG_USB_PWC_INPUT_EVDEV=y +CONFIG_USB_STV06XX=m + +# Other user-facing: Kerberos persistent keyrings, Speakup screen reader, Wine/Proton ntsync +CONFIG_ACCESSIBILITY=y +CONFIG_NTSYNC=m +CONFIG_PERSISTENT_KEYRINGS=y +CONFIG_SPEAKUP=m +CONFIG_SPEAKUP_SYNTH_ACNTSA=m +CONFIG_SPEAKUP_SYNTH_APOLLO=m +CONFIG_SPEAKUP_SYNTH_AUDPTR=m +CONFIG_SPEAKUP_SYNTH_BNS=m +CONFIG_SPEAKUP_SYNTH_DECTLK=m +CONFIG_SPEAKUP_SYNTH_LTLK=m +CONFIG_SPEAKUP_SYNTH_SOFT=m +CONFIG_SPEAKUP_SYNTH_SPKOUT=m +CONFIG_SPEAKUP_SYNTH_TXPRT=m From c260a185e22ce966a75b21896b780ca1cd3e6b66 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 23:08:47 +1100 Subject: [PATCH 0838/1579] tools: aurora-sep: leave libfprint to aurora-sep-userspace libfprint with the aurora driver, as a source patch and as Fedora and Arch Linux packages, is now maintained in https://github.com/aurora-silicon/aurora-sep-userspace. Two copies of the userspace driver, built from different trees with colliding package releases, would drift apart. Drop the libfprint patch, the Arch libfprint-aurora package and the libfprint rebuild from the Fedora build script. This tree keeps the kernel side's userspace pieces: the loader, its services, the fprintd policy and the desktop integration, packaged as aurora-sep. The README and the package descriptions point to aurora-sep-userspace for the driver. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/M1-SUPPORT.md | 4 +- tools/aurora-sep/README.md | 62 +- .../packaging/arch/aurora-sep/PKGBUILD | 7 +- .../packaging/arch/libfprint-aurora/PKGBUILD | 73 - .../arch/libfprint-aurora/apple-sep.uapi | 1 - .../libfprint-1.94.100-apple-sep.patch | 1 - .../packaging/fedora/aurora-sep.spec | 8 +- tools/aurora-sep/packaging/fedora/build.sh | 69 +- .../libfprint-1.94.100-apple-sep.patch | 1215 ----------------- tools/aurora-sep/tests/test_packaging.py | 15 +- 10 files changed, 36 insertions(+), 1419 deletions(-) delete mode 100644 tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD delete mode 120000 tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi delete mode 120000 tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch delete mode 100644 tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch diff --git a/tools/aurora-sep/M1-SUPPORT.md b/tools/aurora-sep/M1-SUPPORT.md index 774af1ef10ca0..6d3c6fa93985a 100644 --- a/tools/aurora-sep/M1-SUPPORT.md +++ b/tools/aurora-sep/M1-SUPPORT.md @@ -75,8 +75,8 @@ capture-guidance fixes that the older J313 test checkout did not contain. Build/test results for this consolidation are not a new hardware regression test on Neo, M2, or every M1 board. The consolidated SEP module compiled against the prepared J313 kernel build. Its four identity-binding tests, ten extracted -entropy-method tests, and eight calibration/xART tests passed. The bundled -patch applied to clean libfprint 1.94.100 and built with `drivers=aurora`; +entropy-method tests, and eight calibration/xART tests passed. The libfprint +patch (now maintained in aurora-silicon/aurora-sep-userspace) applied to clean libfprint 1.94.100 and built with `drivers=aurora`; the three core unit-test suites passed. In that minimal build, 33 unrelated driver/hwdb tests were skipped and AppStream validation rejected the generated empty `` device list. No full-kernel rebuild or new hardware run of diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 23c3f589b5289..96632f81c585d 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -53,32 +53,13 @@ window shifted 90 APFS blocks beyond the actual `.gl` extent. This proves the locator can misidentify the extent; it does not identify the historical cause of any particular damaged container. -For fingerprint support, apply -`patches/libfprint-1.94.100-apple-sep.patch` to libfprint 1.94.100, build and -install libfprint, then install the fprintd device policy: +For fingerprint support, install libfprint with the aurora driver, then the +fprintd device policy below. The userspace driver is maintained in +https://github.com/aurora-silicon/aurora-sep-userspace (libfprint with the +aurora driver, as source and as distribution packages); this tree carries only +the kernel driver, its loader and its service. The driver behaviour fprintd +sees: -```sh -git clone --depth 1 --branch v1.94.100 \ - https://gitlab.freedesktop.org/libfprint/libfprint.git libfprint-aurora -cd libfprint-aurora -git apply ../linux/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch -meson setup build -Dprefix=/usr -Ddrivers=default -Dintrospection=false \ - -Dgtk-examples=false -Ddoc=false -Dinstalled-tests=false -ninja -C build -meson test -C build --print-errorlogs -``` - -Adjust the patch path for your checkout. The patch includes a fix for an -upstream Meson test loop when introspection is disabled. In the Aurora -`linux-build` guest, the additional packages needed for this build are -`libgudev libgusb pixman cairo glib2-devel`; `meson` and `ninja` are already -in the release image. The default driver set retains all normal libfprint -drivers and adds Aurora. This build was checked on aarch64: the shared library -and all default drivers compiled, with 0 test failures (most device tests -skip without their optional mocks/hardware). That does **not** prove Touch ID -works on m2. The patched library is installed privately under -`/opt/aurora-sep` there, with fprintd loading it via a service drop-in while -the distro libfprint remains untouched. fprintd discovers one Apple SEP sensor. The Aurora driver now reports stage progress only when the kernel's stage number increases; guidance and stage-zero events are not captured samples. fprintd 1.94.5 itself emits an initial `enroll-stage-passed` after its @@ -146,35 +127,26 @@ by the Quickshell lock screen. ## Packages -`packaging/` builds the same pieces as distribution packages: libfprint with -the aurora driver, which replaces the distribution's libfprint so fprintd needs -no library path, and `aurora-sep` with the loader, its service and the fprintd -policy. Installing `aurora-sep` lets fprintd load the driver with xART writes -on, as above. +`packaging/` builds `aurora-sep`, with the loader, its service and the fprintd +policy, as a distribution package. libfprint with the aurora driver comes from +https://github.com/aurora-silicon/aurora-sep-userspace; install both. Installing +`aurora-sep` lets fprintd load the driver with xART writes on, as above. -On Fedora, `packaging/fedora/build.sh` rebuilds the release's own libfprint -source package with the patch added (1.94.100 on Fedora 45) and builds -`aurora-sep`. It installs build dependencies, so run it in a throwaway -container from the top of this tree: +On Fedora, `packaging/fedora/build.sh` builds `aurora-sep`. It installs build +dependencies, so run it in a throwaway container from the top of this tree: ```sh mkdir rpms podman run --rm -v "$PWD":/src:ro,z -v "$PWD/rpms":/out:z \ registry.fedoraproject.org/fedora:45 \ /src/tools/aurora-sep/packaging/fedora/build.sh /out -sudo dnf install rpms/libfprint-1.94.100-*.aurora1.*.rpm \ - rpms/aurora-sep-1-*.noarch.rpm +sudo dnf install rpms/aurora-sep-1-*.noarch.rpm ``` -A later Fedora libfprint update replaces the rebuilt one; rebuild after it, or -hold it with `dnf versionlock add libfprint`. - -On Arch Linux and Arch Linux ARM, build `libfprint-aurora` (it replaces -`libfprint`) and then `aurora-sep`: +On Arch Linux and Arch Linux ARM, build `aurora-sep`: ```sh -cd tools/aurora-sep/packaging/arch/libfprint-aurora && makepkg -si -cd ../aurora-sep && makepkg -si +cd tools/aurora-sep/packaging/arch/aurora-sep && makepkg -si ``` fprintd starts the loader the first time it runs. Loading the driver at boot @@ -272,5 +244,5 @@ Disabling it and rebooting returns the login screen to GNOME's behaviour. The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this checkout's headers with `make headers_install INSTALL_HDR_PATH=` and add that directory's `include` to the libfprint build include path. The -libfprint patch uses `` directly; it has no copied ABI -header. Interface version, ioctl numbers and fixed-width layouts are unchanged. +aurora driver in aurora-sep-userspace uses `` directly; it +has no copied ABI header. Interface version, ioctl numbers and fixed-width layouts are unchanged. diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index d27422443be35..15712b3bdf4e2 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -2,11 +2,12 @@ # # The loader and desktop integration for the Apple SEP fingerprint driver. # The local sources are links into tools/aurora-sep; after changing any of -# them, run updpkgsums here. libfprint-aurora provides the libfprint driver. +# them, run updpkgsums here. libfprint with the aurora driver comes from +# https://github.com/aurora-silicon/aurora-sep-userspace. pkgname=aurora-sep pkgver=1 -pkgrel=2 +pkgrel=3 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -39,7 +40,7 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' - '226a5f5f2244583b5b10d9a5b3e134f5b155065fd8dce850c828f98f085fc931' + 'd7a09f31367c181a2cbda63cd58104d6117136fb5db5b20f4a3996ea3af7025e' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') # A unit and the helper it runs ($1, installed as $3), pointed at the diff --git a/tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD b/tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD deleted file mode 100644 index 12bfdd7222953..0000000000000 --- a/tools/aurora-sep/packaging/arch/libfprint-aurora/PKGBUILD +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# libfprint with the aurora driver for the Apple SEP fingerprint sensor. -# The local sources are links into tools/aurora-sep and to -# include/uapi/linux/apple-sep.h; after changing either file, run updpkgsums -# here. - -pkgname=libfprint-aurora -_pkgname=libfprint -pkgver=1.94.100 -pkgrel=1 -pkgdesc="Library for fingerprint readers, with the Apple SEP (Touch ID) driver" -url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" -arch=(aarch64 x86_64) -license=(LGPL-2.1-or-later) -depends=( - libgcc - glib2 - glibc - libgudev - libgusb - openssl - pixman -) -makedepends=( - glib2-devel - gobject-introspection - gtk-doc - meson - python-cairo - python-gobject - systemd -) -checkdepends=( - cairo - umockdev -) -provides=("$_pkgname=$pkgver" libfprint-2.so) -conflicts=("$_pkgname") -source=("https://gitlab.freedesktop.org/libfprint/libfprint/-/archive/v$pkgver/$_pkgname-v$pkgver.tar.gz" - "libfprint-$pkgver-apple-sep.patch" - apple-sep.uapi) -sha256sums=('edc90e02f330a7595ceaf37f2c6ec32ed43541347fe936d3273b0bb2524fd19c' - '00f999dab395c0f6cd5ffda94099f898eb48d001e19036c500fe2713f76aa076' - '00cb09430461e0b0772619bc2054c8f9794484fbc3d807268955c0fb2de45c10') - -prepare() { - cd "$_pkgname-v$pkgver" - patch -Np1 -i "../libfprint-$pkgver-apple-sep.patch" - - # The driver includes , which the distribution's kernel - # headers do not carry. - install -Dm644 ../apple-sep.uapi ../aurora-include/linux/apple-sep.h -} - -build() { - local meson_options=( - -D drivers=all - -D installed-tests=false - ) - - CFLAGS+=" -I$srcdir/aurora-include" - arch-meson "$_pkgname-v$pkgver" build "${meson_options[@]}" - meson compile -C build -} - -check() { - meson test -C build --print-errorlogs -} - -package() { - meson install -C build --destdir "$pkgdir" -} diff --git a/tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi b/tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi deleted file mode 120000 index 3ba9077515e02..0000000000000 --- a/tools/aurora-sep/packaging/arch/libfprint-aurora/apple-sep.uapi +++ /dev/null @@ -1 +0,0 @@ -../../../../../include/uapi/linux/apple-sep.h \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch deleted file mode 120000 index 40c7665bd03ef..0000000000000 --- a/tools/aurora-sep/packaging/arch/libfprint-aurora/libfprint-1.94.100-apple-sep.patch +++ /dev/null @@ -1 +0,0 @@ -../../../patches/libfprint-1.94.100-apple-sep.patch \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index 648409fb7c463..b4dd9ca4391b6 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -5,7 +5,7 @@ Name: aurora-sep Version: 1 -Release: 2%{?dist} +Release: 3%{?dist} Summary: Loader and desktop integration for the Apple SEP fingerprint driver License: GPL-2.0-only @@ -40,7 +40,8 @@ the secure enclave seals and unseals a trusted key and /dev/sep-bio appears, and lets fprintd reach /dev/sep-bio. The loader runs at boot, and fprintd also pulls it in when it starts. Touch ID is then activated, before fprintd starts, and pressing the Touch ID button suspends rather than powers off. -libfprint needs the aurora driver, which this package does not provide. Read XART-SAFETY.md before installing it on +libfprint needs the aurora driver, which this package does not provide; it is +packaged in https://github.com/aurora-silicon/aurora-sep-userspace. Read XART-SAFETY.md before installing it on a machine that also runs macOS. %prep @@ -98,6 +99,9 @@ echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %{_presetdir}/80-aurora-sep.preset %changelog +* Sun Oct 11 2026 Ryan Murray - 1-3 +- libfprint with the aurora driver is now built in aurora-sep-userspace + * Sun Oct 11 2026 Ryan Murray - 1-2 - Activate Touch ID at boot, before fprintd starts - Keep fprintd running diff --git a/tools/aurora-sep/packaging/fedora/build.sh b/tools/aurora-sep/packaging/fedora/build.sh index 27fb64e28aafd..a1331bbdb266b 100755 --- a/tools/aurora-sep/packaging/fedora/build.sh +++ b/tools/aurora-sep/packaging/fedora/build.sh @@ -1,13 +1,13 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0-only # -# Build the Fedora packages for Touch ID on the Apple SEP: +# Build the Fedora package for Touch ID on the Apple SEP: # -# - libfprint: Fedora's own source package, rebuilt with the aurora driver -# patch from ../../patches and nothing else changed. The release gains an -# ".aurora1" suffix, so it replaces the distribution's build in place. # - aurora-sep: the driver loader, its service and the fprintd policy. # +# libfprint with the aurora driver is built and packaged in +# https://github.com/aurora-silicon/aurora-sep-userspace. +# # It installs the build dependencies with dnf, so run it as root in a # throwaway container of the Fedora release you are building for: # @@ -21,13 +21,6 @@ set -eu out=${1:?usage: build.sh } here=$(cd "$(dirname "$0")" && pwd) sep=$(cd "$here/../.." && pwd) -patch=$sep/patches/libfprint-1.94.100-apple-sep.patch -header=${APPLE_SEP_HEADER:-$sep/../../include/uapi/linux/apple-sep.h} -patched_version=1.94.100 - -for f in "$patch" "$header"; do - [ -f "$f" ] || { echo "build.sh: $f is missing" >&2; exit 1; } -done mkdir -p "$out" top=$(mktemp -d) @@ -36,60 +29,6 @@ rpmtop=$top/rpmbuild dnf -y install rpm-build systemd-rpm-macros -# Fedora's libfprint, exactly as the distribution builds it. -(cd "$top" && dnf -y download --source libfprint) -srpm=$(ls "$top"/libfprint-*.src.rpm) -rpm -i --define "_topdir $rpmtop" "$srpm" -spec=$rpmtop/SPECS/libfprint.spec - -version=$(rpmspec -q --srpm --qf '%{version}\n' "$spec") -if [ "$version" != "$patched_version" ]; then - echo "build.sh: this Fedora ships libfprint $version, the patch is for $patched_version" >&2 - exit 1 -fi -cp "$patch" "$header" "$rpmtop/SOURCES/" - -# Add the patch and the header, mark the release, and point the compiler at -# the header: is not in the distribution's kernel headers. -# %autosetup applies every PatchN. Stop if any anchor has moved. -awk -v patch="$(basename "$patch")" -v date="$(LC_ALL=C date -u '+%a %b %d %Y')" ' - /^Release:/ && !release { - sub(/[[:space:]]*$/, ".aurora1"); print; release = 1; next - } - /^Source0:/ && !source { - print - print "Source9000: apple-sep.h" - print "Patch9000: " patch - source = 1; next - } - /^%meson[[:space:]]/ && !build { - print "# The aurora driver includes from the Aurora kernel." - print "mkdir -p aurora-include/linux" - print "cp -p %{SOURCE9000} aurora-include/linux/" - print "export CFLAGS=\"%{build_cflags} -I$PWD/aurora-include\"" - print; build = 1; next - } - /^%changelog/ && !changelog { - print - print "* " date " Aurora Silicon - " "%{version}-%{release}" - print "- Add the Apple SEP (aurora) driver from aurora-silicon/linux" - print " tools/aurora-sep/patches/" patch - print "" - changelog = 1; next - } - { print } - END { - if (!(release && source && build && changelog)) { - print "build.sh: libfprint.spec changed shape; update the edit" > "/dev/stderr" - exit 1 - } - } -' "$spec" >"$spec.aurora" -mv "$spec.aurora" "$spec" - -dnf -y builddep "$spec" -rpmbuild --define "_topdir $rpmtop" -ba "$spec" - # The loader, its units and the desktop integration, from this tree. dnf -y builddep "$here/aurora-sep.spec" rpmbuild --define "_topdir $rpmtop" --define "_sourcedir $sep" \ diff --git a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch deleted file mode 100644 index 5163351683362..0000000000000 --- a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch +++ /dev/null @@ -1,1215 +0,0 @@ -diff --git a/libfprint/drivers/aurora/aurora.c b/libfprint/drivers/aurora/aurora.c -new file mode 100644 ---- /dev/null -+++ b/libfprint/drivers/aurora/aurora.c -@@ -0,0 +1,991 @@ -+/* -+ * Aurora SEP fingerprint driver -+ * -+ * The sensor is matched inside Apple's secure enclave. This driver never sees -+ * an image, a template, or key material: it starts operations on the kernel -+ * device and reports the enclave's verdict. That is why it can be short. -+ * -+ * SPDX-License-Identifier: LGPL-2.1-or-later -+ */ -+ -+#define FP_COMPONENT "apple-sep" -+ -+#include "drivers_api.h" -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+/* Why the kernel ended an enrol or verify, in a poll's 'status'. Headers from -+ * before these were named carry the same numbers; those kernels never report -+ * RETRY while an enrolment is running, nor REJECTED at all. */ -+#ifndef SEP_BIO_STATUS_REJECTED -+#define SEP_BIO_STATUS_NONE 0 -+#define SEP_BIO_STATUS_SENSOR 1 -+#define SEP_BIO_STATUS_ENCLAVE 2 -+#define SEP_BIO_STATUS_TOO_MANY 3 -+#define SEP_BIO_STATUS_TIMEOUT 4 -+#define SEP_BIO_STATUS_RETRY 5 -+#define SEP_BIO_STATUS_UNFILED 6 -+#define SEP_BIO_STATUS_REJECTED 7 -+#endif -+ -+struct _FpiDeviceAurora -+{ -+ FpDevice parent; -+ -+ gint fd; -+ guint watch_id; -+ guint32 enroll_last_reported_stage; -+ -+ guint8 nonce[SEP_BIO_NONCE_LEN]; -+ gboolean nonce_valid; -+}; -+ -+G_DECLARE_FINAL_TYPE (FpiDeviceAurora, fpi_device_aurora, FPI, DEVICE_AURORA, FpDevice) -+G_DEFINE_TYPE (FpiDeviceAurora, fpi_device_aurora, FP_TYPE_DEVICE) -+ -+/* ------------------------------------------------------------------ */ -+/* helpers */ -+/* ------------------------------------------------------------------ */ -+ -+static gboolean -+aurora_ioctl (FpiDeviceAurora *self, unsigned long req, void *arg, GError **error) -+{ -+ if (self->fd < 0) -+ { -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_NOT_OPEN, -+ "device is not open"); -+ return FALSE; -+ } -+ -+ if (ioctl (self->fd, req, arg) < 0) -+ { -+ gint err = errno; -+ -+ switch (err) -+ { -+ case ENODEV: -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_REMOVED, -+ "the enclave reports no fingerprint sensor"); -+ break; -+ -+ case EPERM: -+ case EACCES: -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "not permitted"); -+ break; -+ -+ case ENOSPC: -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_FULL, -+ "three SEP fingerprint slots are full; delete one before enrolling"); -+ break; -+ -+ case ENOENT: -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_NOT_FOUND, -+ "no such identity"); -+ break; -+ -+ default: -+ g_set_error (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_PROTO, -+ "ioctl failed: %s", g_strerror (err)); -+ break; -+ } -+ return FALSE; -+ } -+ -+ return TRUE; -+} -+ -+static const gchar * -+aurora_status_reason (guint32 status) -+{ -+ switch (status) -+ { -+ case SEP_BIO_STATUS_NONE: -+ return "it was cancelled or ended by the system"; -+ -+ case SEP_BIO_STATUS_SENSOR: -+ return "the fingerprint sensor failed"; -+ -+ case SEP_BIO_STATUS_ENCLAVE: -+ return "the secure enclave refused a step"; -+ -+ case SEP_BIO_STATUS_TOO_MANY: -+ return "the enclave did not complete the enrolment within its captures"; -+ -+ case SEP_BIO_STATUS_TIMEOUT: -+ return "no finger was placed on the sensor in time"; -+ -+ case SEP_BIO_STATUS_RETRY: -+ return "the capture was not usable"; -+ -+ case SEP_BIO_STATUS_UNFILED: -+ return "the fingerprint was enrolled but could not be recorded"; -+ -+ case SEP_BIO_STATUS_REJECTED: -+ return "too many captures in a row were not usable; " -+ "rest more of the finger flat on the sensor and enrol again"; -+ -+ default: -+ return NULL; -+ } -+} -+ -+static GError * -+aurora_failure_new (const gchar *operation, guint32 status) -+{ -+ const gchar *reason = aurora_status_reason (status); -+ -+ if (reason == NULL) -+ return fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "%s failed, driver status 0x%x", -+ operation, status); -+ -+ return fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "%s failed: %s (driver status %u)", -+ operation, reason, status); -+} -+ -+/* The kernel reports an unusable capture without saying why it was unusable, -+ * so the general retry is the most specific code that is always true. */ -+static GError * -+aurora_retry_new (void) -+{ -+ return fpi_device_retry_new_msg (FP_DEVICE_RETRY_GENERAL, -+ "the capture was not usable; touch the sensor again"); -+} -+ -+/* The label is how the host recognises its own records in the device's list. -+ * libfprint already has an encoding for exactly this -- it carries the enroll -+ * date, the finger and the username -- so use it rather than inventing one. */ -+static gboolean -+aurora_label_from_print (FpPrint *print, guint8 *label_out, GError **error) -+{ -+ g_autofree gchar *user_id = NULL; -+ -+ if (print == NULL) -+ { -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, -+ "no print to label"); -+ return FALSE; -+ } -+ -+ user_id = fpi_print_generate_user_id (print); -+ -+ if (user_id == NULL || strlen (user_id) >= SEP_BIO_LABEL_LEN) -+ { -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, -+ "print label does not fit the device record"); -+ return FALSE; -+ } -+ -+ memset (label_out, 0, SEP_BIO_LABEL_LEN); -+ memcpy (label_out, user_id, strlen (user_id)); -+ return TRUE; -+} -+ -+static void -+aurora_stop_watch (FpiDeviceAurora *self) -+{ -+ if (self->watch_id != 0) -+ { -+ g_source_remove (self->watch_id); -+ self->watch_id = 0; -+ } -+} -+ -+/* Build the FpPrint that represents one enclave-stored identity. The identity -+ * UUID is all we hold; there is no template on this side to store. */ -+static FpPrint * -+aurora_print_from_uuid (FpiDeviceAurora *self, -+ const guint8 *uuid, -+ const guint8 *label) -+{ -+ FpPrint *print = fp_print_new (FP_DEVICE (self)); -+ GVariant *data; -+ -+ data = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, uuid, -+ SEP_BIO_UUID_LEN, 1); -+ -+ fpi_print_set_type (print, FPI_PRINT_RAW); -+ fpi_print_set_device_stored (print, TRUE); -+ g_object_set (print, "fpi-data", data, NULL); -+ -+ /* The label came from the device. Treat it as untrusted bytes: bound it to -+ * the buffer before anything reads it as a string. */ -+ if (label != NULL) -+ { -+ gchar safe[SEP_BIO_LABEL_LEN]; -+ -+ memcpy (safe, label, SEP_BIO_LABEL_LEN); -+ safe[SEP_BIO_LABEL_LEN - 1] = '\0'; -+ -+ if (safe[0] != '\0') -+ fpi_print_fill_from_user_id (print, safe); -+ } -+ -+ return print; -+} -+ -+static gboolean -+aurora_uuid_from_print (FpPrint *print, guint8 *uuid_out, GError **error) -+{ -+ g_autoptr(GVariant) data = NULL; -+ const guint8 *raw; -+ gsize len = 0; -+ -+ g_object_get (print, "fpi-data", &data, NULL); -+ -+ if (data == NULL || !g_variant_is_of_type (data, G_VARIANT_TYPE ("ay"))) -+ { -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, -+ "print does not carry an enclave identity"); -+ return FALSE; -+ } -+ -+ raw = g_variant_get_fixed_array (data, &len, 1); -+ if (raw == NULL || len != SEP_BIO_UUID_LEN) -+ { -+ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, -+ "enclave identity is the wrong size"); -+ return FALSE; -+ } -+ -+ memcpy (uuid_out, raw, SEP_BIO_UUID_LEN); -+ return TRUE; -+} -+ -+/* Compare by enclave identity rather than by whole print. -+ * -+ * fp_print_equal() compares host-side labels too -- the username and finger a -+ * print was filed under. Those are ours, not the enclave's: the device knows -+ * only the identity it stored. Comparing whole prints made a correct match -+ * from fprintd, which files prints under a username, read as a non-match -+ * against the bare print the device hands back. The UUID is the ground truth, -+ * so compare that. */ -+static gboolean -+aurora_print_has_uuid (FpPrint *print, const guint8 *uuid) -+{ -+ guint8 stored[SEP_BIO_UUID_LEN]; -+ -+ if (print == NULL) -+ return FALSE; -+ -+ if (!aurora_uuid_from_print (print, stored, NULL)) -+ return FALSE; -+ -+ return memcmp (stored, uuid, SEP_BIO_UUID_LEN) == 0; -+} -+ -+/* ------------------------------------------------------------------ */ -+/* probe / open / close */ -+/* ------------------------------------------------------------------ */ -+ -+static void -+aurora_probe (FpDevice *device) -+{ -+ struct sep_bio_info info = { 0 }; -+ const gchar *path; -+ g_autofree gchar *serial = NULL; -+ gint fd; -+ -+ path = fpi_device_get_udev_data (device, FPI_DEVICE_UDEV_SUBTYPE_MISC); -+ if (path == NULL) -+ { -+ fpi_device_probe_complete (device, NULL, NULL, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "no sep-bio node")); -+ return; -+ } -+ -+ fd = open (path, O_RDWR | O_CLOEXEC); -+ if (fd < 0) -+ { -+ fpi_device_probe_complete (device, NULL, NULL, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "cannot open %s: %s", -+ path, g_strerror (errno))); -+ return; -+ } -+ -+ if (ioctl (fd, SEP_BIO_GET_INFO, &info) < 0) -+ { -+ gint err = errno; -+ -+ close (fd); -+ fpi_device_probe_complete (device, NULL, NULL, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "cannot query device: %s", -+ g_strerror (err))); -+ return; -+ } -+ -+ close (fd); -+ -+ if (info.version != SEP_BIO_IFACE_VERSION) -+ { -+ fpi_device_probe_complete (device, NULL, NULL, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "interface version %u, expected %u", -+ info.version, -+ SEP_BIO_IFACE_VERSION)); -+ return; -+ } -+ -+ /* A driver that claims a device it cannot use makes the whole stack look -+ * broken to the user. If the enclave has no sensor, say so plainly and let -+ * fprintd report "no devices" rather than a device that fails every touch. */ -+ if (!info.sensor_present) -+ { -+ fp_dbg ("enclave reports no fingerprint sensor attached"); -+ fpi_device_probe_complete (device, NULL, NULL, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "the secure enclave reports no " -+ "fingerprint sensor attached")); -+ return; -+ } -+ -+ if (info.enroll_stages > 0) -+ fpi_device_set_nr_enroll_stages (device, info.enroll_stages); -+ -+ serial = g_strdup ("apple-sep"); -+ fpi_device_probe_complete (device, serial, NULL, NULL); -+} -+ -+static void -+aurora_open (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ const gchar *path; -+ -+ path = fpi_device_get_udev_data (device, FPI_DEVICE_UDEV_SUBTYPE_MISC); -+ if (path == NULL) -+ { -+ fpi_device_open_complete (device, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "no sep-bio node")); -+ return; -+ } -+ -+ /* The kernel admits one opener at a time; EBUSY means something else holds -+ * the sensor, which is a real condition and not a driver fault. */ -+ self->fd = open (path, O_RDWR | O_CLOEXEC); -+ if (self->fd < 0) -+ { -+ gint err = errno; -+ -+ fpi_device_open_complete (device, -+ fpi_device_error_new_msg (err == EBUSY -+ ? FP_DEVICE_ERROR_BUSY -+ : FP_DEVICE_ERROR_NOT_SUPPORTED, -+ "cannot open %s: %s", -+ path, g_strerror (err))); -+ return; -+ } -+ -+ fpi_device_open_complete (device, NULL); -+} -+ -+static void -+aurora_close (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ -+ aurora_stop_watch (self); -+ -+ if (self->fd >= 0) -+ { -+ /* Closing the description cancels anything in flight, by contract. */ -+ close (self->fd); -+ self->fd = -1; -+ } -+ -+ self->nonce_valid = FALSE; -+ memset (self->nonce, 0, sizeof (self->nonce)); -+ fpi_device_report_finger_status (device, FP_FINGER_STATUS_NONE); -+ -+ fpi_device_close_complete (device, NULL); -+} -+ -+static void -+aurora_cancel (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ -+ fpi_device_report_finger_status (device, FP_FINGER_STATUS_NONE); -+ if (self->fd >= 0) -+ ioctl (self->fd, SEP_BIO_CANCEL); -+} -+ -+static void -+aurora_report_guidance (FpDevice *device, guint32 guidance, gboolean active) -+{ -+ FpFingerStatusFlags status = FP_FINGER_STATUS_NONE; -+ -+ if (active) -+ { -+ if (guidance == SEP_BIO_GUIDANCE_HOLD_STILL) -+ status = FP_FINGER_STATUS_PRESENT; -+ else if (guidance == SEP_BIO_GUIDANCE_PLACE || -+ guidance == SEP_BIO_GUIDANCE_LIFT_AND_MOVE) -+ status = FP_FINGER_STATUS_NEEDED; -+ } -+ -+ fpi_device_report_finger_status (device, status); -+} -+ -+/* ------------------------------------------------------------------ */ -+/* enrolment */ -+/* ------------------------------------------------------------------ */ -+ -+static gboolean -+aurora_enrol_ready (gint fd, GIOCondition condition, gpointer user_data) -+{ -+ FpDevice *device = FP_DEVICE (user_data); -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct sep_bio_enrol_poll poll_res = { 0 }; -+ g_autoptr(GError) error = NULL; -+ FpPrint *print = NULL; -+ -+ if (!aurora_ioctl (self, SEP_BIO_ENROL_POLL, &poll_res, &error)) -+ { -+ self->watch_id = 0; -+ fpi_device_report_finger_status (device, FP_FINGER_STATUS_NONE); -+ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); -+ return G_SOURCE_REMOVE; -+ } -+ -+ aurora_report_guidance (device, poll_res.guidance, -+ poll_res.state == SEP_BIO_STATE_PENDING || -+ poll_res.state == SEP_BIO_STATE_PROGRESS); -+ -+ switch (poll_res.state) -+ { -+ case SEP_BIO_STATE_PENDING: -+ case SEP_BIO_STATE_PROGRESS: -+ /* fprintd calls this "enroll-stage-passed". A guidance wakeup or an -+ * initial stage-zero event must not masquerade as a captured sample. */ -+ if (poll_res.state == SEP_BIO_STATE_PROGRESS && -+ poll_res.stage > self->enroll_last_reported_stage) -+ { -+ self->enroll_last_reported_stage = poll_res.stage; -+ fpi_device_enroll_progress (device, poll_res.stage, NULL, NULL); -+ } -+ -+ /* The latest capture did not count, and the enrolment goes on. The -+ * kernel reports each rejection in one poll only, and an accepted -+ * capture clears it, so a rejection that shares a poll with a new stage -+ * came after that stage. */ -+ if (poll_res.status == SEP_BIO_STATUS_RETRY) -+ fpi_device_enroll_progress (device, self->enroll_last_reported_stage, -+ NULL, aurora_retry_new ()); -+ return G_SOURCE_CONTINUE; -+ -+ case SEP_BIO_STATE_DONE: -+ fpi_device_get_enroll_data (device, &print); -+ if (print != NULL) -+ { -+ GVariant *data = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, -+ poll_res.uuid, -+ SEP_BIO_UUID_LEN, 1); -+ fpi_print_set_type (print, FPI_PRINT_RAW); -+ fpi_print_set_device_stored (print, TRUE); -+ g_object_set (print, "fpi-data", data, NULL); -+ } -+ self->watch_id = 0; -+ fpi_device_enroll_complete (device, -+ print ? g_object_ref (print) : NULL, -+ print ? NULL -+ : fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "enrolment produced no print")); -+ return G_SOURCE_REMOVE; -+ -+ case SEP_BIO_STATE_FAILED: -+ self->watch_id = 0; -+ fpi_device_enroll_complete (device, NULL, -+ aurora_failure_new ("enrolment", poll_res.status)); -+ return G_SOURCE_REMOVE; -+ -+ default: -+ self->watch_id = 0; -+ fpi_device_enroll_complete (device, NULL, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "enrolment failed, state %u, driver status 0x%x", -+ poll_res.state, poll_res.status)); -+ return G_SOURCE_REMOVE; -+ } -+} -+ -+static void -+aurora_enroll (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct sep_bio_enrol_start start = { 0 }; -+ g_autoptr(GError) error = NULL; -+ FpPrint *print = NULL; -+ -+ fpi_device_get_enroll_data (device, &print); -+ -+ if (!aurora_label_from_print (print, start.label, &error)) -+ { -+ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); -+ return; -+ } -+ -+ if (!aurora_ioctl (self, SEP_BIO_ENROL_START, &start, &error)) -+ { -+ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); -+ return; -+ } -+ -+ self->enroll_last_reported_stage = 0; -+ aurora_report_guidance (device, SEP_BIO_GUIDANCE_PLACE, TRUE); -+ aurora_stop_watch (self); -+ self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_enrol_ready, device); -+} -+ -+/* ------------------------------------------------------------------ */ -+/* verify and identify */ -+/* ------------------------------------------------------------------ */ -+ -+/* The single place a match may be declared. -+ * -+ * Everything else in this driver reports failure. A result counts as a match -+ * only if the enclave said DONE and said MATCH, we require it to have bound the -+ * answer to the nonce we generated for this operation (a binding the host -+ * cannot verify cryptographically -- it is enforced by the enclave), and it -+ * arrived within the token's deadline. Any other combination -- including a -+ * state we do not recognise -- is a non-match, never an error that a caller -+ * might interpret loosely. */ -+static gboolean -+aurora_result_is_match (FpiDeviceAurora *self, -+ const struct sep_bio_verify_poll *res) -+{ -+ struct timespec now; -+ guint64 now_ns; -+ gboolean token_set = FALSE; -+ gsize i; -+ -+ if (res->state != SEP_BIO_STATE_DONE) -+ return FALSE; -+ -+ if (res->result != SEP_BIO_MATCH) -+ return FALSE; -+ -+ if (!self->nonce_valid) -+ return FALSE; -+ -+ for (i = 0; i < SEP_BIO_TOKEN_LEN; i++) -+ if (res->token[i] != 0) -+ { -+ token_set = TRUE; -+ break; -+ } -+ -+ if (!token_set) -+ return FALSE; -+ -+ if (clock_gettime (CLOCK_MONOTONIC, &now) != 0) -+ return FALSE; -+ -+ now_ns = (guint64) now.tv_sec * 1000000000ull + (guint64) now.tv_nsec; -+ if (res->deadline_ns == 0 || now_ns > res->deadline_ns) -+ { -+ fp_dbg ("match result arrived after its deadline; refusing it"); -+ return FALSE; -+ } -+ -+ return TRUE; -+} -+ -+/* A verify that ended without a comparison. When the kernel says why, pass -+ * that on: otherwise the log shows only an unspecified error. */ -+static GError * -+aurora_not_compared_new (const struct sep_bio_verify_poll *res) -+{ -+ if (res->state == SEP_BIO_STATE_FAILED) -+ return aurora_failure_new ("verification", res->status); -+ -+ return fpi_device_error_new (FP_DEVICE_ERROR_GENERAL); -+} -+ -+static gboolean -+aurora_verify_ready (gint fd, GIOCondition condition, gpointer user_data) -+{ -+ FpDevice *device = FP_DEVICE (user_data); -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct sep_bio_verify_poll res = { 0 }; -+ FpiDeviceAction action = fpi_device_get_current_action (device); -+ g_autoptr(GError) error = NULL; -+ /* Not a g_autoptr: FpPrint is a GInitiallyUnowned and both report functions -+ * take the floating reference. Unreffing it here as well drops the last -+ * reference under libfprint's feet, and it frees the print again when the -+ * match completes. */ -+ FpPrint *matched = NULL; -+ GError *retry = NULL; -+ -+ if (!aurora_ioctl (self, SEP_BIO_VERIFY_POLL, &res, &error)) -+ { -+ self->watch_id = 0; -+ fpi_device_report_finger_status (device, FP_FINGER_STATUS_NONE); -+ /* Terminal path: void the per-operation nonce like every other exit, so -+ * a nonce cannot be carried into a later operation. */ -+ self->nonce_valid = FALSE; -+ memset (self->nonce, 0, sizeof (self->nonce)); -+ if (action == FPI_DEVICE_ACTION_VERIFY) -+ fpi_device_verify_complete (device, g_steal_pointer (&error)); -+ else -+ fpi_device_identify_complete (device, g_steal_pointer (&error)); -+ return G_SOURCE_REMOVE; -+ } -+ -+ aurora_report_guidance (device, res.guidance, -+ res.state == SEP_BIO_STATE_PENDING || -+ res.state == SEP_BIO_STATE_PROGRESS); -+ -+ if (res.state == SEP_BIO_STATE_PENDING || -+ res.state == SEP_BIO_STATE_PROGRESS) -+ return G_SOURCE_CONTINUE; -+ -+ self->watch_id = 0; -+ -+ if (aurora_result_is_match (self, &res)) -+ matched = aurora_print_from_uuid (self, res.uuid, NULL); -+ /* The capture was not usable, so nothing was compared. A retry tells the -+ * caller to ask for another touch: fprintd starts the match again on its -+ * own, where a failure would end the authentication attempt. */ -+ else if (res.state == SEP_BIO_STATE_FAILED && -+ res.status == SEP_BIO_STATUS_RETRY) -+ retry = aurora_retry_new (); -+ -+ if (action == FPI_DEVICE_ACTION_VERIFY) -+ { -+ FpPrint *expected = NULL; -+ -+ fpi_device_get_verify_data (device, &expected); -+ -+ if (retry != NULL) -+ { -+ fpi_device_verify_report (device, FPI_MATCH_ERROR, NULL, retry); -+ fpi_device_verify_complete (device, NULL); -+ } -+ else if (matched != NULL && aurora_print_has_uuid (expected, res.uuid)) -+ { -+ fpi_device_verify_report (device, FPI_MATCH_SUCCESS, matched, NULL); -+ fpi_device_verify_complete (device, NULL); -+ } -+ else if (res.result == SEP_BIO_NOT_COMPARED) -+ { -+ /* No comparison happened, so there is no verdict to report. Saying -+ * FAIL here would tell the user their finger was rejected when it -+ * was never examined. */ -+ fpi_device_verify_complete (device, aurora_not_compared_new (&res)); -+ } -+ else -+ { -+ fpi_device_verify_report (device, FPI_MATCH_FAIL, matched, NULL); -+ fpi_device_verify_complete (device, NULL); -+ } -+ } -+ else -+ { -+ GPtrArray *gallery = NULL; -+ FpPrint *hit = NULL; -+ -+ fpi_device_get_identify_data (device, &gallery); -+ -+ if (matched != NULL && gallery != NULL) -+ { -+ guint i; -+ -+ for (i = 0; i < gallery->len; i++) -+ { -+ FpPrint *candidate = g_ptr_array_index (gallery, i); -+ -+ if (aurora_print_has_uuid (candidate, res.uuid)) -+ { -+ hit = candidate; -+ break; -+ } -+ } -+ } -+ -+ if (retry != NULL) -+ { -+ fpi_device_identify_report (device, NULL, NULL, retry); -+ fpi_device_identify_complete (device, NULL); -+ } -+ else if (matched == NULL && res.result == SEP_BIO_NOT_COMPARED) -+ { -+ fpi_device_identify_complete (device, aurora_not_compared_new (&res)); -+ } -+ else -+ { -+ fpi_device_identify_report (device, hit, matched, NULL); -+ fpi_device_identify_complete (device, NULL); -+ } -+ } -+ -+ self->nonce_valid = FALSE; -+ memset (self->nonce, 0, sizeof (self->nonce)); -+ -+ return G_SOURCE_REMOVE; -+} -+ -+static void -+aurora_match_start (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct sep_bio_verify_start start = { 0 }; -+ struct sep_bio_info info = { 0 }; -+ FpiDeviceAction action = fpi_device_get_current_action (device); -+ g_autoptr(GError) error = NULL; -+ FpPrint *print = NULL; -+ -+ if (action == FPI_DEVICE_ACTION_VERIFY) -+ fpi_device_get_verify_data (device, &print); -+ -+ /* An identify against an EMPTY gallery is "no match" by definition, and -+ * answering it needs no hardware at all. -+ * -+ * This is not an optimisation, it is what unblocks enrolment. fprintd runs a -+ * duplicate check before each enrol by identifying the incoming finger -+ * against the prints already stored. On a device with nothing enrolled that -+ * gallery is empty, so the question has only one possible answer -- but we -+ * were forwarding it to the kernel, whose verify ioctl returns ENOSYS -+ * because matching is not implemented yet. fprintd then reported -+ * "ioctl failed: Function not implemented", which GNOME shows to the user as -+ * "Fingerprint device disconnected". -+ * -+ * So the first enrolment on a fresh device was impossible, and the message -+ * blamed the cable. Answer the empty case here and the enrol proceeds. -+ * -+ * A NON-empty gallery normally goes to the enclave for matching. A stale -+ * host index may outlive its SEP identity after a failed cold restore; only -+ * an explicit, valid zero live-identity count from SEP lets us answer that -+ * gallery without capture. An unavailable count must never be treated as -+ * zero, and VERIFY still goes through the kernel's restore proof gate. -+ */ -+ if (action == FPI_DEVICE_ACTION_IDENTIFY) -+ { -+ GPtrArray *gallery = NULL; -+ -+ fpi_device_get_identify_data (device, &gallery); -+ if (gallery == NULL || gallery->len == 0) -+ { -+ fp_dbg ("identify against an empty gallery: reporting no match " -+ "without touching the sensor"); -+ fpi_device_identify_report (device, NULL, NULL, NULL); -+ fpi_device_identify_complete (device, NULL); -+ return; -+ } -+ -+ if (ioctl (self->fd, SEP_BIO_GET_INFO, &info) == 0 && -+ info.version == SEP_BIO_IFACE_VERSION && -+ (info.reserved[0] & SEP_BIO_INFO_LIVE_COUNT_VALID) != 0 && -+ info.reserved[1] == 0) -+ { -+ fp_dbg ("SEP proved zero live identities despite a non-empty host " -+ "gallery: reporting no match for identify only"); -+ fpi_device_identify_report (device, NULL, NULL, NULL); -+ fpi_device_identify_complete (device, NULL); -+ return; -+ } -+ } -+ -+ /* A fresh nonce per operation is what makes the result non-replayable. If we -+ * cannot get one we must not fall back to anything weaker. */ -+ if (getrandom (self->nonce, sizeof (self->nonce), 0) != (gssize) sizeof (self->nonce)) -+ { -+ error = fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, -+ "cannot obtain a nonce for the match"); -+ if (action == FPI_DEVICE_ACTION_VERIFY) -+ fpi_device_verify_complete (device, g_steal_pointer (&error)); -+ else -+ fpi_device_identify_complete (device, g_steal_pointer (&error)); -+ return; -+ } -+ self->nonce_valid = TRUE; -+ -+ memcpy (start.nonce, self->nonce, sizeof (start.nonce)); -+ -+ if (!aurora_ioctl (self, SEP_BIO_VERIFY_START, &start, &error)) -+ { -+ self->nonce_valid = FALSE; -+ if (action == FPI_DEVICE_ACTION_VERIFY) -+ fpi_device_verify_complete (device, g_steal_pointer (&error)); -+ else -+ fpi_device_identify_complete (device, g_steal_pointer (&error)); -+ return; -+ } -+ -+ aurora_report_guidance (device, SEP_BIO_GUIDANCE_PLACE, TRUE); -+ aurora_stop_watch (self); -+ self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_verify_ready, device); -+} -+ -+/* ------------------------------------------------------------------ */ -+/* storage */ -+/* ------------------------------------------------------------------ */ -+ -+static void -+aurora_list (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct sep_bio_list list = { 0 }; -+ g_autoptr(GError) error = NULL; -+ GPtrArray *prints; -+ guint32 i; -+ -+ if (!aurora_ioctl (self, SEP_BIO_LIST, &list, &error)) -+ { -+ fpi_device_list_complete (device, NULL, g_steal_pointer (&error)); -+ return; -+ } -+ -+ if (list.count > SEP_BIO_MAX_IDENTITIES) -+ { -+ fpi_device_list_complete (device, NULL, -+ fpi_device_error_new_msg (FP_DEVICE_ERROR_PROTO, -+ "device reported %u identities, " -+ "more than the interface allows", -+ list.count)); -+ return; -+ } -+ -+ prints = g_ptr_array_new_with_free_func (g_object_unref); -+ -+ for (i = 0; i < list.count; i++) -+ g_ptr_array_add (prints, -+ g_object_ref_sink (aurora_print_from_uuid (self, -+ list.id[i].uuid, -+ list.id[i].label))); -+ -+ fpi_device_list_complete (device, prints, NULL); -+} -+ -+static void -+aurora_delete (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ struct sep_bio_delete del = { 0 }; -+ g_autoptr(GError) error = NULL; -+ FpPrint *print = NULL; -+ -+ fpi_device_get_delete_data (device, &print); -+ -+ if (print == NULL || !aurora_uuid_from_print (print, del.uuid, &error)) -+ { -+ if (error == NULL) -+ error = fpi_device_error_new_msg (FP_DEVICE_ERROR_DATA_INVALID, -+ "no print to delete"); -+ fpi_device_delete_complete (device, g_steal_pointer (&error)); -+ return; -+ } -+ -+ if (!aurora_ioctl (self, SEP_BIO_DELETE, &del, &error)) -+ { -+ fpi_device_delete_complete (device, g_steal_pointer (&error)); -+ return; -+ } -+ -+ fpi_device_delete_complete (device, NULL); -+} -+ -+static void -+aurora_clear_storage (FpDevice *device) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); -+ g_autoptr(GError) error = NULL; -+ -+ if (!aurora_ioctl (self, SEP_BIO_DELETE_ALL, NULL, &error)) -+ { -+ fpi_device_clear_storage_complete (device, g_steal_pointer (&error)); -+ return; -+ } -+ -+ fpi_device_clear_storage_complete (device, NULL); -+} -+ -+/* ------------------------------------------------------------------ */ -+/* class */ -+/* ------------------------------------------------------------------ */ -+ -+static const FpIdEntry aurora_id_table[] = { -+ { .udev_types = FPI_DEVICE_UDEV_SUBTYPE_MISC, .misc_name = "sep-bio" }, -+ { .udev_types = 0 } -+}; -+ -+static void -+fpi_device_aurora_init (FpiDeviceAurora *self) -+{ -+ self->fd = -1; -+ self->watch_id = 0; -+ self->nonce_valid = FALSE; -+} -+ -+static void -+fpi_device_aurora_finalize (GObject *object) -+{ -+ FpiDeviceAurora *self = FPI_DEVICE_AURORA (object); -+ -+ aurora_stop_watch (self); -+ -+ if (self->fd >= 0) -+ { -+ close (self->fd); -+ self->fd = -1; -+ } -+ -+ memset (self->nonce, 0, sizeof (self->nonce)); -+ -+ G_OBJECT_CLASS (fpi_device_aurora_parent_class)->finalize (object); -+} -+ -+static void -+fpi_device_aurora_class_init (FpiDeviceAuroraClass *klass) -+{ -+ GObjectClass *object_class = G_OBJECT_CLASS (klass); -+ FpDeviceClass *dev_class = FP_DEVICE_CLASS (klass); -+ -+ object_class->finalize = fpi_device_aurora_finalize; -+ -+ dev_class->id = FP_COMPONENT; -+ dev_class->full_name = "Apple secure enclave fingerprint sensor"; -+ dev_class->type = FP_DEVICE_TYPE_UDEV; -+ dev_class->id_table = aurora_id_table; -+ dev_class->scan_type = FP_SCAN_TYPE_PRESS; -+ dev_class->nr_enroll_stages = 8; -+ dev_class->temp_hot_seconds = -1; -+ -+ dev_class->probe = aurora_probe; -+ dev_class->open = aurora_open; -+ dev_class->close = aurora_close; -+ dev_class->cancel = aurora_cancel; -+ dev_class->enroll = aurora_enroll; -+ dev_class->verify = aurora_match_start; -+ dev_class->identify = aurora_match_start; -+ dev_class->list = aurora_list; -+ dev_class->delete = aurora_delete; -+ dev_class->clear_storage = aurora_clear_storage; -+ -+ fpi_device_class_auto_initialize_features (dev_class); -+ -+ /* DUPLICATES_CHECK is deliberately NOT advertised. -+ * -+ * It tells libfprint the device can say whether a finger being enrolled is -+ * already enrolled, and libfprint implements that by running a full IDENTIFY -+ * pass immediately before every enrol. Matching is implemented, but the -+ * enclave's enrol result does not identify an already-enrolled finger in a -+ * way that satisfies libfprint's pre-enrol duplicate-check contract. Keep -+ * the feature unset until that narrower contract is implemented. -+ */ -+} -diff --git a/libfprint/fp-context.c b/libfprint/fp-context.c -index 70d4062..6fdf53b 100644 ---- a/libfprint/fp-context.c -+++ b/libfprint/fp-context.c -@@ -479,6 +479,7 @@ fp_context_enumerate (FpContext *context) - - g_autoptr(GList) spidev_devices = g_udev_client_query_by_subsystem (udev_client, "spidev"); - g_autoptr(GList) hidraw_devices = g_udev_client_query_by_subsystem (udev_client, "hidraw"); -+ g_autoptr(GList) misc_devices = g_udev_client_query_by_subsystem (udev_client, "misc"); - - /* for each potential driver, try to match all requested resources. */ - for (i = 0; i < priv->drivers->len; i++) -@@ -492,7 +493,7 @@ fp_context_enumerate (FpContext *context) - - for (entry = cls->id_table; entry->udev_types; entry++) - { -- GList *matched_spidev = NULL, *matched_hidraw = NULL; -+ GList *matched_spidev = NULL, *matched_hidraw = NULL, *matched_misc = NULL; - - if (entry->udev_types & FPI_DEVICE_UDEV_SUBTYPE_SPIDEV) - { -@@ -530,6 +531,20 @@ fp_context_enumerate (FpContext *context) - if (matched_hidraw == NULL) - continue; - } -+ if (entry->udev_types & FPI_DEVICE_UDEV_SUBTYPE_MISC) -+ { -+ for (matched_misc = misc_devices; matched_misc; matched_misc = matched_misc->next) -+ { -+ const gchar * name = g_udev_device_get_name (matched_misc->data); -+ if (!name || !entry->misc_name) -+ continue; -+ if (g_strcmp0 (name, entry->misc_name) == 0) -+ break; -+ } -+ /* If match was not found exit */ -+ if (matched_misc == NULL) -+ continue; -+ } - priv->pending_devices++; - g_async_initable_new_async (driver, - G_PRIORITY_LOW, -@@ -539,6 +554,7 @@ fp_context_enumerate (FpContext *context) - "fpi-driver-data", entry->driver_data, - "fpi-udev-data-spidev", (matched_spidev ? g_udev_device_get_device_file (matched_spidev->data) : NULL), - "fpi-udev-data-hidraw", (matched_hidraw ? g_udev_device_get_device_file (matched_hidraw->data) : NULL), -+ "fpi-udev-data-misc", (matched_misc ? g_udev_device_get_device_file (matched_misc->data) : NULL), - NULL); - /* remove entries from list to avoid conflicts */ - if (matched_spidev) -@@ -551,6 +567,11 @@ fp_context_enumerate (FpContext *context) - g_object_unref (matched_hidraw->data); - hidraw_devices = g_list_delete_link (hidraw_devices, matched_hidraw); - } -+ if (matched_misc) -+ { -+ g_object_unref (matched_misc->data); -+ misc_devices = g_list_delete_link (misc_devices, matched_misc); -+ } - } - } - -diff --git a/libfprint/fp-device-private.h b/libfprint/fp-device-private.h -index 1c3702f..c8253c6 100644 ---- a/libfprint/fp-device-private.h -+++ b/libfprint/fp-device-private.h -@@ -49,6 +49,7 @@ typedef struct - { - gchar *spidev_path; - gchar *hidraw_path; -+ gchar *misc_path; - } udev_data; - - gboolean is_removed; -diff --git a/libfprint/fp-device.c b/libfprint/fp-device.c -index 115063d..70b10f1 100644 ---- a/libfprint/fp-device.c -+++ b/libfprint/fp-device.c -@@ -53,6 +53,7 @@ enum { - PROP_FPI_USB_DEVICE, - PROP_FPI_UDEV_DATA_SPIDEV, - PROP_FPI_UDEV_DATA_HIDRAW, -+ PROP_FPI_UDEV_DATA_MISC, - PROP_FPI_DRIVER_DATA, - N_PROPS - }; -@@ -237,6 +238,7 @@ fp_device_finalize (GObject *object) - g_clear_pointer (&priv->virtual_env, g_free); - g_clear_pointer (&priv->udev_data.spidev_path, g_free); - g_clear_pointer (&priv->udev_data.hidraw_path, g_free); -+ g_clear_pointer (&priv->udev_data.misc_path, g_free); - - G_OBJECT_CLASS (fp_device_parent_class)->finalize (object); - } -@@ -307,6 +309,13 @@ fp_device_get_property (GObject *object, - g_value_set_string (value, NULL); - break; - -+ case PROP_FPI_UDEV_DATA_MISC: -+ if (cls->type == FP_DEVICE_TYPE_UDEV) -+ g_value_set_string (value, priv->udev_data.misc_path); -+ else -+ g_value_set_string (value, NULL); -+ break; -+ - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - } -@@ -353,6 +362,13 @@ fp_device_set_property (GObject *object, - g_assert (g_value_get_string (value) == NULL); - break; - -+ case PROP_FPI_UDEV_DATA_MISC: -+ if (cls->type == FP_DEVICE_TYPE_UDEV) -+ priv->udev_data.misc_path = g_value_dup_string (value); -+ else -+ g_assert (g_value_get_string (value) == NULL); -+ break; -+ - case PROP_FPI_DRIVER_DATA: - priv->driver_data = g_value_get_uint64 (value); - break; -@@ -583,6 +599,19 @@ fp_device_class_init (FpDeviceClass *klass) - "Private: The path to /dev/hidrawN", - NULL, - G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY); -+ /** -+ * FpDevice::fpi-udev-data-misc: (skip) -+ * -+ * This property is only for internal purposes. -+ * -+ * Stability: private -+ */ -+ properties[PROP_FPI_UDEV_DATA_MISC] = -+ g_param_spec_string ("fpi-udev-data-misc", -+ "Udev data: misc path", -+ "Private: The path to the misc character device", -+ NULL, -+ G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY); - - /** - * FpDevice::fpi-driver-data: (skip) -diff --git a/libfprint/fpi-device.c b/libfprint/fpi-device.c -index 9500b3a..a70db5c 100644 ---- a/libfprint/fpi-device.c -+++ b/libfprint/fpi-device.c -@@ -521,6 +521,9 @@ fpi_device_get_udev_data (FpDevice *device, FpiDeviceUdevSubtypeFlags subtype) - case FPI_DEVICE_UDEV_SUBTYPE_SPIDEV: - return priv->udev_data.spidev_path; - -+ case FPI_DEVICE_UDEV_SUBTYPE_MISC: -+ return priv->udev_data.misc_path; -+ - default: - g_return_val_if_reached (NULL); - return NULL; -diff --git a/libfprint/fpi-device.h b/libfprint/fpi-device.h -index b17c10d..7671ad6 100644 ---- a/libfprint/fpi-device.h -+++ b/libfprint/fpi-device.h -@@ -28,12 +28,14 @@ - * FpiDeviceUdevSubtypeFlags: - * @FPI_DEVICE_UDEV_SUBTYPE_SPIDEV: The device requires an spidev node - * @FPI_DEVICE_UDEV_SUBTYPE_HIDRAW: The device requires a hidraw node -+ * @FPI_DEVICE_UDEV_SUBTYPE_MISC: The device requires a misc character node - * - * Bitfield of required hardware resources for a udev-backed device. - */ - typedef enum { - FPI_DEVICE_UDEV_SUBTYPE_SPIDEV = 1 << 0, - FPI_DEVICE_UDEV_SUBTYPE_HIDRAW = 1 << 1, -+ FPI_DEVICE_UDEV_SUBTYPE_MISC = 1 << 2, - } FpiDeviceUdevSubtypeFlags; - - /** -@@ -71,6 +73,7 @@ struct _FpIdEntry - guint pid; - guint vid; - } hid_id; -+ const gchar *misc_name; - }; - }; - guint64 driver_data; -diff --git a/libfprint/meson.build b/libfprint/meson.build -index f11533c..400c5b7 100644 ---- a/libfprint/meson.build -+++ b/libfprint/meson.build -@@ -116,6 +116,7 @@ driver_sources = { - ), - 'etes603' : files('drivers/etes603.c'), - 'egis0570' : files('drivers/egis0570.c'), -+ 'aurora' : files('drivers/aurora/aurora.c'), - 'egismoc' : files('drivers/egismoc/egismoc.c'), - 'egis_etu905' : files('drivers/egismoc/egis_etu905.c'), - 'vfs0050' : files('drivers/vfs0050.c'), -diff --git a/meson.build b/meson.build -index ab09dc1..dfa7460 100644 ---- a/meson.build -+++ b/meson.build -@@ -151,6 +151,9 @@ drivers_info = { - # SPI driver (non-optional if SPI is available) - 'elanspi': { 'spi': true, 'helper': ['udev'], 'optional': not have_spi }, - -+ # Apple secure enclave, reached through a kernel misc device -+ 'aurora': { 'helper': ['udev'], 'optional': host_machine.system() != 'linux' }, -+ - # Virtual drivers (test-only, optional) - 'virtual_image': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, - 'virtual_device': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, -diff --git a/tests/meson.build b/tests/meson.build -index aad0675..2147ef3 100644 ---- a/tests/meson.build -+++ b/tests/meson.build -@@ -295,4 +295,4 @@ else -- foreach driver_test: drivers_tests -+ foreach driver_test, args: drivers_tests - test(driver_test, - sh, - args: ['-c', 'exit 77'] diff --git a/tools/aurora-sep/tests/test_packaging.py b/tools/aurora-sep/tests/test_packaging.py index f5aa1b124d060..961985758ba1e 100644 --- a/tools/aurora-sep/tests/test_packaging.py +++ b/tools/aurora-sep/tests/test_packaging.py @@ -41,17 +41,9 @@ def check(self, directory): digest, expected, "%s changed: run updpkgsums in %s" % (name, directory) ) - def test_libfprint_sums_match_tree(self): - self.check(ARCH / "libfprint-aurora") - def test_aurora_sep_sums_match_tree(self): self.check(ARCH / "aurora-sep") - def test_patch_version_matches_pkgver(self): - text = (ARCH / "libfprint-aurora" / "PKGBUILD").read_text() - pkgver = re.search(r"^pkgver=(\S+)$", text, re.M).group(1) - self.assertTrue((SEP / "patches" / ("libfprint-%s-apple-sep.patch" % pkgver)).is_file()) - class FedoraSpecTest(unittest.TestCase): def test_sources_are_in_tree(self): @@ -61,11 +53,10 @@ def test_sources_are_in_tree(self): for source in sources: self.assertTrue((SEP / source).is_file(), source) - def test_build_script_patches_the_shipped_version(self): + def test_build_script_builds_only_aurora_sep(self): text = (FEDORA / "build.sh").read_text() - version = re.search(r"^patched_version=(\S+)$", text, re.M).group(1) - self.assertIn("libfprint-%s-apple-sep.patch" % version, text) - self.assertTrue((SEP / "patches" / ("libfprint-%s-apple-sep.patch" % version)).is_file()) + self.assertIn("aurora-sep.spec", text) + self.assertNotIn("dnf -y download --source libfprint", text) if __name__ == "__main__": From 5c84e0c4ebf8110f2ef7212debc8c42b3d4691d0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 04:46:20 -0400 Subject: [PATCH 0839/1579] mm: preserve clean subpage lazyfree state on retained reads When a retained reference prevents discarding a clean lazyfree slot, try_to_unmap_subpage() restores the PTE but then marks the native folio swap-backed. The extra reference alone does not cancel MADV_FREE, so this can prevent reclaim without swap and cause unnecessary swap traffic. Leave the classification unchanged after restoring a clean PTE. Dirty entries still cancel lazyfree and follow the existing swap-backed path. Extend the lazyfree KUnit case to retain a reader after whole-folio advice, check the clean classification, then release the reader and reclaim. The expanded test passes on 16K and 64K KASAN guests without warnings. Signed-off-by: Diljit Singh (cherry picked from commit e9a25aae4e7509bc40554dac0a23240d77363110) --- arch/arm64/mm/user4k-test.c | 18 ++++++++++++++---- mm/rmap.c | 12 +++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 4e5b2befe18f5..f9dcaf932557f 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -9380,7 +9380,7 @@ static void user4k_lazyfree_test(struct kunit *test) return; } /* Partial, redirty, retained GET, typed write, all leaves, migration. */ - for (mode = 0; mode < 10; mode++) { + for (mode = 0; mode < 11; mode++) { struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); struct mm_struct *mm; struct user_page_fragment held = {}; @@ -9401,7 +9401,8 @@ static void user4k_lazyfree_test(struct kunit *test) } kthread_use_mm(mm); ret = test_madvise(TEST_VA, - (mode == 4 || mode == 9) ? PAGE_SIZE : SZ_4K, MADV_FREE); + (mode == 4 || mode == 9 || mode == 10) ? PAGE_SIZE : SZ_4K, + MADV_FREE); kthread_unuse_mm(mm); KUNIT_ASSERT_EQ(test, ret, 0L); pte = test_fault_entry(test, mm, TEST_VA); @@ -9413,7 +9414,7 @@ static void user4k_lazyfree_test(struct kunit *test) if (mode == 1) KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); - if (mode == 2 || mode == 3 || mode == 8) { + if (mode == 2 || mode == 3 || mode == 8 || mode == 10) { ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, mode == 3 ? FOLL_WRITE : 0, &held, 1); KUNIT_ASSERT_EQ(test, ret, 1L); @@ -9444,11 +9445,20 @@ static void user4k_lazyfree_test(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, folio); folio_lock(folio); try_to_unmap(folio, TTU_SYNC); + if (mode == 10) { + /* A temporary reader leaves clean whole-folio advice intact. */ + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + KUNIT_EXPECT_FALSE(test, folio_test_dirty(folio)); + KUNIT_EXPECT_FALSE(test, pte_none(test_fault_entry(test, mm, TEST_VA))); + release_user_fragments(&held, 1, false); + try_to_unmap(folio, TTU_SYNC); + } folio_unlock(folio); folio_put(folio); for (i = 0; i < 4; i++) { pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); - KUNIT_EXPECT_EQ(test, pte_none(pte), (mode == 4 || mode == 9) || + KUNIT_EXPECT_EQ(test, pte_none(pte), + (mode == 4 || mode == 9 || mode == 10) || (i == 0 && (mode == 0 || mode == 5))); } if (mode == 2 || mode == 6 || mode == 7) diff --git a/mm/rmap.c b/mm/rmap.c index 795b568b9f7c5..bf11cc857cb45 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2176,16 +2176,14 @@ static bool try_to_unmap_subpage(struct folio *folio, struct vm_area_struct *vma goto out; } set_pte_at(mm, address, pvmw->pte, pte); - /* Retained references must not turn a droppable folio into swap data. */ - if (vma->vm_flags & VM_DROPPABLE) + /* A retained reference alone does not cancel clean lazyfree data. */ + if ((vma->vm_flags & VM_DROPPABLE) || !pte_dirty(pte)) goto out; - if (pte_dirty(pte)) { - mm_subpage_cancel_lazyfree(slot); - folio_mark_dirty(folio); - } + mm_subpage_cancel_lazyfree(slot); + folio_mark_dirty(folio); } if (!folio_test_swapbacked(folio)) { - /* A write or retained reference cancelled whole-folio lazy reclaim. */ + /* A write cancelled whole-folio lazy reclaim. */ folio_set_swapbacked(folio); goto out; } From f42ef849f09f86f95311589a59b8ecda9513ba28 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 04:56:16 -0400 Subject: [PATCH 0840/1579] arm64: preserve dependencies in mixed-page test selectors The IOMMUFD harness selects IOMMU_PT even when IOMMU_SUPPORT is disabled, excluding the page allocation implementation its selected code requires. The boolean VFIO harness selectors can also force VFIO built-in while IOMMUFD is modular, bypassing VFIO's own tristate constraint. Require IOMMU_SUPPORT for the IOMMUFD harness. Require non-modular IOMMUFD for both VFIO selectors, including the ioctl selector that selects the accounting harness. Built-in and disabled IOMMUFD configurations remain available for VFIO testing. Six before/after configuration checks cover the invalid combinations and the retained valid selections. Integrated 16K/64K KASAN images build. Signed-off-by: Diljit Singh (cherry picked from commit cf3e9884efb7a5cd06cb06d7ca632bd1ceef22d4) --- arch/arm64/Kconfig.debug | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 58d7a1dc98405..52fced2ab8355 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -65,6 +65,7 @@ config ARM64_USER4K_IOMMUFD_TEST bool "IOMMUFD ioctl harness for mixed userspace granules" depends on ARM64_USER4K_EXPERIMENTAL && DEBUG_KERNEL depends on FAULT_INJECTION && RUNTIME_TESTING_MENU + depends on IOMMU_SUPPORT select GENERIC_PT select IOMMU_PT select IOMMU_PT_AMDV1 @@ -80,6 +81,7 @@ config ARM64_USER4K_IOMMUFD_TEST config ARM64_USER4K_VFIO_ACCOUNT_TEST bool "VFIO type1 pinning harness for mixed userspace granules" depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y + depends on IOMMUFD != m select VFIO select VFIO_GROUP select VFIO_CONTAINER @@ -96,6 +98,7 @@ config ARM64_USER4K_VFIO_IOCTL_TEST bool "Private VFIO type1 ioctl harness for mixed userspace granules" depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y && DEBUG_KERNEL depends on RUNTIME_TESTING_MENU + depends on IOMMUFD != m select ARM64_USER4K_VFIO_ACCOUNT_TEST default n help From 120704985f8bcfe7ddbac9d5ffccedb3eb11ab13 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 04:56:16 -0400 Subject: [PATCH 0841/1579] arm64/mm: avoid table metadata lookup at the exclusive PTE endpoint contpte_try_unfold_partial() checks its exclusive endpoint with contpte_align_down(). That helper derives granule geometry from the PTE's backing metadata, but ptep + nr can already lie beyond that backing. Use the mm's contiguous-group size to test alignment at both ends. This preserves partial-group unfolding while avoiding a metadata lookup for a pointer that need not identify an allocated page-table entry. 16K/64K KASAN images build and their existing native/4K contiguous-PTE functional tests pass without warnings. The endpoint defect is established by source inspection; no runtime reproduction is claimed. Signed-off-by: Diljit Singh (cherry picked from commit f6e776e813d9dec5da55a0b2eb779eeca145c471) --- arch/arm64/mm/contpte.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index f8faf1a99eeca..e78e0d4265218 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -68,15 +68,18 @@ static inline pte_t *contpte_align_addr_ptep(struct mm_struct *mm, unsigned long static void contpte_try_unfold_partial(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr) { + unsigned int cont_bytes = sizeof(*ptep) * cont_ptes_mm(mm); + /* * Unfold any partially covered contpte block at the beginning and end * of the range. */ - if (ptep != contpte_align_down(ptep) || nr < cont_ptes_mm(mm)) + if (!IS_ALIGNED((unsigned long)ptep, cont_bytes) || nr < cont_ptes_mm(mm)) contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); - if (ptep + nr != contpte_align_down(ptep + nr)) { + /* The exclusive end can lie beyond the allocated page-table backing. */ + if (!IS_ALIGNED((unsigned long)(ptep + nr), cont_bytes)) { unsigned long last_addr = addr + mm_page_size(mm) * (nr - 1); pte_t *last_ptep = ptep + nr - 1; From 2266c8c46863614fed00c7115810c4a62da0bf47 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:09:14 -0400 Subject: [PATCH 0842/1579] perf: sample page sizes using the target mm geometry perf_get_pgtable_size() starts with an mm-aware PGD lookup but continues with native table offsets and leaf sizes. A smaller userspace granule can therefore select an unrelated entry or report the native page size. Add mm-aware lockless offset and leaf-size helpers with native fallbacks. Use the target mm throughout perf's walk, including runtime folding and contiguous mappings, while preserving the existing interrupt exclusion and descriptor-snapshot semantics. Expose the walker only for KUnit testing and cover ordinary mapped and unmapped addresses across table levels. Extend contiguous-PTE fixtures to check perf's reported size and cover 16K userspace on a 64K kernel. The tests pass in 16K/64K KASAN guests for native, 4K and supported 16K userspace granules. An x86_64 perf-object build checks the native fallbacks. Hardware PMU event delivery was not tested. Signed-off-by: Diljit Singh (cherry picked from commit 4538d92c925d9adc7af502c9df8aec52c56df322) --- arch/arm64/include/asm/user-pgtable.h | 23 +++++++ arch/arm64/mm/user4k-test.c | 91 +++++++++++++++++++++++++-- include/linux/perf_event.h | 5 ++ include/linux/pgtable.h | 29 +++++++++ kernel/events/core.c | 24 +++---- 5 files changed, 156 insertions(+), 16 deletions(-) diff --git a/arch/arm64/include/asm/user-pgtable.h b/arch/arm64/include/asm/user-pgtable.h index b244f2387a7b4..7b1d1a94515d0 100644 --- a/arch/arm64/include/asm/user-pgtable.h +++ b/arch/arm64/include/asm/user-pgtable.h @@ -113,6 +113,17 @@ static inline pmd_t *pmd_offset_mm(struct mm_struct *mm, pud_t *pud, #define pmd_offset_mm pmd_offset_mm +/* Use the caller's descriptor snapshots throughout a lockless walk. */ +#define p4d_offset_lockless_mm(mm, pgdp, pgd, addr) \ + (arm64_mm_alt_granule(mm) ? p4d_offset_mm(mm, &(pgd), addr) : \ + p4d_offset_lockless(pgdp, pgd, addr)) +#define pud_offset_lockless_mm(mm, p4dp, p4d, addr) \ + (arm64_mm_alt_granule(mm) ? pud_offset_mm(mm, &(p4d), addr) : \ + pud_offset_lockless(p4dp, p4d, addr)) +#define pmd_offset_lockless_mm(mm, pudp, pud, addr) \ + (arm64_mm_alt_granule(mm) ? pmd_offset_mm(mm, &(pud), addr) : \ + pmd_offset_lockless(pudp, pud, addr)) + static inline pte_t *pte_offset_kernel_mm(struct mm_struct *mm, pmd_t *pmd, unsigned long addr) { @@ -133,6 +144,18 @@ static inline pte_t *pte_offset_kernel_mm(struct mm_struct *mm, pmd_t *pmd, #define pud_size_mm(mm) (1UL << pud_shift_mm(mm)) #define p4d_size_mm(mm) (1UL << p4d_shift_mm(mm)) #define pgd_size_mm(mm) (arm64_mm_alt_granule(mm) ? (1UL << arm64_user_level_shift(mm, 0)) : PGDIR_SIZE) +#define pgd_leaf_size_mm(mm, pgd) \ + (arm64_mm_alt_granule(mm) ? pgd_size_mm(mm) : pgd_leaf_size(pgd)) +#define p4d_leaf_size_mm(mm, p4d) \ + (arm64_mm_alt_granule(mm) ? p4d_size_mm(mm) : p4d_leaf_size(p4d)) +#define pud_leaf_size_mm(mm, pud) \ + (arm64_mm_alt_granule(mm) ? pud_size_mm(mm) : pud_leaf_size(pud)) +#define pmd_leaf_size_mm(mm, pmd) \ + (arm64_mm_alt_granule(mm) ? pmd_size_mm(mm) * \ + (pmd_cont(pmd) ? (mm_page_shift(mm) == 12 ? 16 : 32) : 1) : pmd_leaf_size(pmd)) +#define __pte_leaf_size_mm(mm, pmd, pte) \ + (arm64_mm_alt_granule(mm) ? \ + (pte_cont(pte) ? cont_pte_size_mm(mm) : mm_page_size(mm)) : __pte_leaf_size(pmd, pte)) #define pte_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PTE * sizeof(pte_t)) #define pmd_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PMD * sizeof(pmd_t)) #define pud_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PUD * sizeof(pud_t)) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index f9dcaf932557f..69456ba03bac5 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -1324,6 +1325,58 @@ static int test_map_alias(struct test_mm *ctx, unsigned int index, return 0; } +#ifdef CONFIG_PERF_EVENTS +static void test_perf_page_size(struct kunit *test, unsigned int shift) +{ + const unsigned long addresses[] = { + SZ_1M, SZ_1G + SZ_1M, (1UL << 39) + SZ_1M, + }; + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + void *data = test_alloc_granule(test, PAGE_SHIFT); + unsigned long flags; + u64 present, absent; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, data); + ctx->test = test; + ctx->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, ctx->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + if (shift != PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, test_mm_select_granule(ctx->mm, shift), 0); + for (i = 0; i < ARRAY_SIZE(addresses); i++) + KUNIT_ASSERT_EQ(test, test_map_alias(ctx, i, addresses[i], virt_to_phys(data)), 0); + for (i = 0; i < ARRAY_SIZE(addresses); i++) { + local_irq_save(flags); + present = perf_get_pgtable_size(ctx->mm, addresses[i]); + absent = perf_get_pgtable_size(ctx->mm, addresses[i] + (1UL << shift)); + local_irq_restore(flags); + KUNIT_EXPECT_EQ(test, present, 1ULL << shift); + KUNIT_EXPECT_EQ(test, absent, 0ULL); + } +} + +static void user4k_perf_page_size_test(struct kunit *test) +{ + test_perf_page_size(test, 12); +} + +static void user16k_perf_page_size_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_perf_page_size(test, 14); +} + +static void native_perf_page_size_test(struct kunit *test) +{ + test_perf_page_size(test, PAGE_SHIFT); +} +#endif + /* Physical-range lookup only; coarse anonymous ownership is a separate step. */ static void user64k_rmap_lookup_test(struct kunit *test) { @@ -7569,7 +7622,7 @@ static void contpte_fixture_free(void *arg) folio_put(f->folio); } -static void test_contpte_granule(struct kunit *test, bool small) +static void test_contpte_granule(struct kunit *test, unsigned int shift) { struct contpte_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); struct vm_area_struct vma = {}; @@ -7585,13 +7638,17 @@ static void test_contpte_granule(struct kunit *test, bool small) spinlock_t *ptl; unsigned int i; int access_ret, access_same_ret, access_noop_ret; +#ifdef CONFIG_PERF_EVENTS + unsigned long irqflags; + u64 reported_size; +#endif KUNIT_ASSERT_NOT_NULL(test, f); KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, contpte_fixture_free, f), 0); f->mm = mm_alloc(); KUNIT_ASSERT_NOT_NULL(test, f->mm); - if (small) - KUNIT_ASSERT_EQ(test, test_mm_select4k(f->mm), 0); + if (shift != PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, test_mm_select_granule(f->mm, shift), 0); f->nr = cont_ptes_mm(f->mm); size = mm_page_size(f->mm); f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, @@ -7621,6 +7678,11 @@ static void test_contpte_granule(struct kunit *test, bool small) ok.addresses &= pte_phys_mm(f->mm, observed) == phys + i * size; ok.hints &= pte_batch_hint(pte + i, observed) == f->nr - i; } +#ifdef CONFIG_PERF_EVENTS + local_irq_save(irqflags); + reported_size = perf_get_pgtable_size(f->mm, TEST_VA + size); + local_irq_restore(irqflags); +#endif __set_pte(pte + 3, pte_mkdirty(__ptep_get(pte + 3))); observed = ptep_get(pte + 1); ok.gathered &= pte_dirty(observed) && pte_young(observed); @@ -7693,16 +7755,28 @@ static void test_contpte_granule(struct kunit *test, bool small) KUNIT_EXPECT_TRUE(test, ok.holes); KUNIT_EXPECT_TRUE(test, ok.full_clear); KUNIT_EXPECT_TRUE(test, ok.sentinel); +#ifdef CONFIG_PERF_EVENTS + KUNIT_EXPECT_EQ(test, reported_size, (u64)cont_pte_size_mm(f->mm)); +#endif } static void user4k_contpte_test(struct kunit *test) { - test_contpte_granule(test, true); + test_contpte_granule(test, 12); +} + +static void user16k_contpte_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_contpte_granule(test, 14); } static void native_contpte_test(struct kunit *test) { - test_contpte_granule(test, false); + test_contpte_granule(test, PAGE_SHIFT); } #endif @@ -14121,7 +14195,13 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_anon_remote_zap_test), #ifdef CONFIG_ARM64_CONTPTE KUNIT_CASE(user4k_contpte_test), + KUNIT_CASE(user16k_contpte_test), KUNIT_CASE(native_contpte_test), +#endif +#ifdef CONFIG_PERF_EVENTS + KUNIT_CASE(user4k_perf_page_size_test), + KUNIT_CASE(user16k_perf_page_size_test), + KUNIT_CASE(native_perf_page_size_test), #endif KUNIT_CASE(user4k_mm_tables_test), KUNIT_CASE(user16k_mm_tables_test), @@ -14155,3 +14235,4 @@ kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_nonlinear_swap #endif MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 48d851fbd8ea5..14a21f187e9f4 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1209,6 +1209,11 @@ perf_cgroup_from_task(struct task_struct *task, struct perf_event_context *ctx) extern struct perf_event_context *perf_cpu_task_ctx(void); +#if IS_ENABLED(CONFIG_KUNIT) +/* The caller must disable IRQs, as for the sampling path. */ +u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr); +#endif + extern void *perf_aux_output_begin(struct perf_output_handle *handle, struct perf_event *event); extern void perf_aux_output_end(struct perf_output_handle *handle, diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index f5ef44f25c52b..5c67d9b5d2f8d 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -2385,6 +2385,19 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level) #define pmd_offset_lockless(pudp, pud, address) pmd_offset(&(pud), address) #endif +#ifndef p4d_offset_lockless_mm +#define p4d_offset_lockless_mm(mm, pgdp, pgd, address) \ + p4d_offset_lockless(pgdp, pgd, address) +#endif +#ifndef pud_offset_lockless_mm +#define pud_offset_lockless_mm(mm, p4dp, p4d, address) \ + pud_offset_lockless(p4dp, p4d, address) +#endif +#ifndef pmd_offset_lockless_mm +#define pmd_offset_lockless_mm(mm, pudp, pud, address) \ + pmd_offset_lockless(pudp, pud, address) +#endif + /* * pXd_leaf() is the API to check whether a pgtable entry is a huge page * mapping. It should work globally across all archs, without any @@ -2438,6 +2451,22 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level) #define __pte_leaf_size(x,y) pte_leaf_size(y) #endif +#ifndef pgd_leaf_size_mm +#define pgd_leaf_size_mm(mm, pgd) pgd_leaf_size(pgd) +#endif +#ifndef p4d_leaf_size_mm +#define p4d_leaf_size_mm(mm, p4d) p4d_leaf_size(p4d) +#endif +#ifndef pud_leaf_size_mm +#define pud_leaf_size_mm(mm, pud) pud_leaf_size(pud) +#endif +#ifndef pmd_leaf_size_mm +#define pmd_leaf_size_mm(mm, pmd) pmd_leaf_size(pmd) +#endif +#ifndef __pte_leaf_size_mm +#define __pte_leaf_size_mm(mm, pmd, pte) __pte_leaf_size(pmd, pte) +#endif + /* * We always define pmd_pfn for all archs as it's used in lots of generic * code. Now it happens too for pud_pfn (and can happen for larger diff --git a/kernel/events/core.c b/kernel/events/core.c index 343dc1af5a308..4b1b0f8b15cf2 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -8,6 +8,7 @@ * Copyright © 2009 Paul Mackerras, IBM Corp. */ +#include #include #include #include @@ -8491,7 +8492,7 @@ static u64 perf_virt_to_phys(u64 virt) /* * Return the pagetable size of a given virtual address. */ -static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) +VISIBLE_IF_KUNIT u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) { u64 size = 0; @@ -8508,45 +8509,46 @@ static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) return 0; if (pgd_leaf(pgd)) - return pgd_leaf_size(pgd); + return pgd_leaf_size_mm(mm, pgd); - p4dp = p4d_offset_lockless(pgdp, pgd, addr); + p4dp = p4d_offset_lockless_mm(mm, pgdp, pgd, addr); p4d = p4dp_get(p4dp); - if (!p4d_present(p4d)) + if (!p4d_present_mm(mm, p4d)) return 0; if (p4d_leaf(p4d)) - return p4d_leaf_size(p4d); + return p4d_leaf_size_mm(mm, p4d); - pudp = pud_offset_lockless(p4dp, p4d, addr); + pudp = pud_offset_lockless_mm(mm, p4dp, p4d, addr); pud = pudp_get(pudp); if (!pud_present(pud)) return 0; if (pud_leaf(pud)) - return pud_leaf_size(pud); + return pud_leaf_size_mm(mm, pud); - pmdp = pmd_offset_lockless(pudp, pud, addr); + pmdp = pmd_offset_lockless_mm(mm, pudp, pud, addr); again: pmd = pmdp_get_lockless(pmdp); if (!pmd_present(pmd)) return 0; if (pmd_leaf(pmd)) - return pmd_leaf_size(pmd); + return pmd_leaf_size_mm(mm, pmd); - ptep = pte_offset_map(&pmd, addr); + ptep = pte_offset_map_mm(mm, &pmd, addr); if (!ptep) goto again; pte = ptep_get_lockless(ptep); if (pte_present(pte)) - size = __pte_leaf_size(pmd, pte); + size = __pte_leaf_size_mm(mm, pmd, pte); pte_unmap(ptep); #endif /* CONFIG_HAVE_GUP_FAST */ return size; } +EXPORT_SYMBOL_IF_KUNIT(perf_get_pgtable_size); static u64 perf_get_page_size(unsigned long addr) { From 73c4b4d5360baf722a06376894e65a400247e10c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:23:12 -0400 Subject: [PATCH 0843/1579] uprobes: preserve architecture metadata when comparing subpage ranges The smaller-page orig_page_is_identical() path compares bytes directly, bypassing the architecture checks used by pages_identical(). On arm64, byte equality does not make distinct tagged ranges interchangeable. Add memcmp_pages_range() with a generic mapped-byte implementation and an arm64 implementation retaining the existing MTE policy. Distinct tagged ranges remain non-identical, including different offsets within one page. Keep full-page comparison as the zero-offset full-length case, and use the range API for the smaller-page uprobe comparison. Add direct tests for offset ranges, bytes outside the compared range, unequal contents, empty ranges and tagged-range identity. These and the existing ownership tests pass in 16K/64K KASAN guests. The generic util and uprobe objects also compile for x86. Actual uprobe removal with private tag state was not runtime reproduced. Signed-off-by: Diljit Singh (cherry picked from commit 0c0dccf36f0a795fb0e52aa74ca1eb51c75f659e) --- arch/arm64/kernel/mte.c | 21 ++++++---- arch/arm64/mm/user4k-data-test.c | 72 ++++++++++++++++++++++++++++++++ include/linux/mm.h | 2 + kernel/events/uprobes.c | 6 +-- mm/util.c | 11 ++++- 5 files changed, 99 insertions(+), 13 deletions(-) diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index 6874b16d06576..645f549d9fed9 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -69,21 +69,23 @@ void mte_sync_tags(pte_t pte, unsigned int nr_pages) smp_wmb(); } -int memcmp_pages(struct page *page1, struct page *page2) +int memcmp_pages_range(struct page *page1, unsigned int offset1, + struct page *page2, unsigned int offset2, size_t len) { char *addr1, *addr2; int ret; - addr1 = page_address(page1); - addr2 = page_address(page2); - ret = memcmp(addr1, addr2, PAGE_SIZE); + addr1 = page_address(page1) + offset1; + addr2 = page_address(page2) + offset2; + ret = memcmp(addr1, addr2, len); - if (!system_supports_mte() || ret) + if (!len || !system_supports_mte() || ret) return ret; /* - * If the page content is identical but at least one of the pages is - * tagged, return non-zero to avoid KSM merging. If only one of the + * If the byte ranges are identical but at least one page is tagged, + * only the same physical range is interchangeable. Avoid merging or + * replacing independently tagged ranges. If only one of the * pages is tagged, __set_ptes() may zero or change the tags of the * other page via mte_sync_tags(). */ @@ -93,6 +95,11 @@ int memcmp_pages(struct page *page1, struct page *page2) return ret; } +int memcmp_pages(struct page *page1, struct page *page2) +{ + return memcmp_pages_range(page1, 0, page2, 0, PAGE_SIZE); +} + static inline void __mte_enable_kernel(const char *mode, unsigned long tcf) { /* Enable MTE Sync Mode for EL1. */ diff --git a/arch/arm64/mm/user4k-data-test.c b/arch/arm64/mm/user4k-data-test.c index d7d550c9e1420..76656736074b1 100644 --- a/arch/arm64/mm/user4k-data-test.c +++ b/arch/arm64/mm/user4k-data-test.c @@ -1084,6 +1084,76 @@ static void extent_folio_put(void *arg) folio_put(arg); } +static struct folio *compare_folio_alloc(struct kunit *test) +{ + struct folio *folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + + if (folio && kunit_add_action_or_reset(test, extent_folio_put, folio)) + return NULL; + return folio; +} + +static void user_page_range_compare_test(struct kunit *test) +{ + const unsigned int sizes[] = { SZ_4K, SZ_16K, PAGE_SIZE }; + struct folio *a = compare_folio_alloc(test), *b = compare_folio_alloc(test); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + for (i = 0; i < ARRAY_SIZE(sizes); i++) { + unsigned int len = sizes[i], offset = PAGE_SIZE - len; + char *from = folio_address(a), *to = folio_address(b); + + if (len > PAGE_SIZE) + continue; + memset(from, 0x11, PAGE_SIZE); + memset(to, 0x22, PAGE_SIZE); + memset(from, 0x5a, len); + memset(to + offset, 0x5a, len); + KUNIT_EXPECT_EQ(test, memcmp_pages_range(&a->page, 0, &b->page, offset, len), 0); + KUNIT_EXPECT_EQ(test, !memcmp_pages(&a->page, &b->page), len == PAGE_SIZE); + to[PAGE_SIZE - 1] ^= 1; + KUNIT_EXPECT_NE(test, memcmp_pages_range(&a->page, 0, &b->page, offset, len), 0); + } +} + +static void user_page_tagged_range_compare_test(struct kunit *test) +{ +#ifdef CONFIG_ARM64_MTE + struct folio *a, *b; + unsigned int len; + + if (!system_supports_mte()) { + kunit_skip(test, "MTE is not available on this CPU"); + return; + } + a = compare_folio_alloc(test); + b = compare_folio_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_EQ(test, memcmp_pages(&a->page, &b->page), 0); + KUNIT_ASSERT_TRUE(test, try_page_mte_tagging(&a->page)); + mte_clear_page_tags(folio_address(a)); + set_page_mte_tagged(&a->page); + KUNIT_EXPECT_NE(test, memcmp_pages(&a->page, &b->page), 0); + for (len = SZ_4K; len <= PAGE_SIZE; len *= 4) { + KUNIT_EXPECT_EQ(test, memcmp_pages_range(&a->page, 0, &a->page, 0, len), 0); + KUNIT_EXPECT_NE(test, memcmp_pages_range(&a->page, 0, &b->page, 0, len), 0); + if (len < PAGE_SIZE) + KUNIT_EXPECT_NE(test, + memcmp_pages_range(&a->page, 0, &a->page, len, len), 0); + } + KUNIT_EXPECT_EQ(test, memcmp_pages_range(&a->page, 0, &b->page, 0, 0), 0); + KUNIT_ASSERT_TRUE(test, try_page_mte_tagging(&b->page)); + mte_clear_page_tags(folio_address(b)); + set_page_mte_tagged(&b->page); + KUNIT_EXPECT_NE(test, memcmp_pages_range(&a->page, 0, &b->page, 0, SZ_4K), 0); +#else + kunit_skip(test, "MTE is not configured"); +#endif +} + static void user4k_data_copy_extent_test(struct kunit *test) { const unsigned int sizes[] = { SZ_4K, SZ_16K, SZ_64K }; @@ -1798,6 +1868,8 @@ static struct kunit_case user4k_data_cases[] = { #endif KUNIT_CASE(user4k_data_mte_test), KUNIT_CASE(user4k_data_copy_extent_test), + KUNIT_CASE(user_page_range_compare_test), + KUNIT_CASE(user_page_tagged_range_compare_test), KUNIT_CASE(user16k_data_owner_test), KUNIT_CASE(user16k_file_coverage_test), KUNIT_CASE(user16k_data_refset_test), diff --git a/include/linux/mm.h b/include/linux/mm.h index 6c01377914fe8..9e621134c8fb1 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -5256,6 +5256,8 @@ static inline void setup_nr_node_ids(void) {} #endif extern int memcmp_pages(struct page *page1, struct page *page2); +int memcmp_pages_range(struct page *page1, unsigned int offset1, + struct page *page2, unsigned int offset2, size_t len); static inline int pages_identical(struct page *page1, struct page *page2) { diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 640991b1c25cb..0d8dc2ae9fdd5 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -425,12 +425,10 @@ static bool orig_page_is_identical(struct vm_area_struct *vma, folio_test_pmd_mappable(orig_folio); if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { unsigned int offset = vma_page_offset_at(vma, vaddr).offset; - void *a = kmap_local_page(page), *b = kmap_local_page(orig_page); identical = folio_test_uptodate(orig_folio) && - !memcmp(a + page_offset, b + offset, mm_page_size(vma->vm_mm)); - kunmap_local(b); - kunmap_local(a); + !memcmp_pages_range(page, page_offset, orig_page, offset, + mm_page_size(vma->vm_mm)); } else { identical = folio_test_uptodate(orig_folio) && pages_identical(page, orig_page); diff --git a/mm/util.c b/mm/util.c index 1f41dfdd3ad13..2adb3d08660d6 100644 --- a/mm/util.c +++ b/mm/util.c @@ -1094,19 +1094,26 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen) return res; } -int __weak memcmp_pages(struct page *page1, struct page *page2) +/* Both byte ranges must fit within their respective native pages. */ +int __weak memcmp_pages_range(struct page *page1, unsigned int offset1, + struct page *page2, unsigned int offset2, size_t len) { char *addr1, *addr2; int ret; addr1 = kmap_local_page(page1); addr2 = kmap_local_page(page2); - ret = memcmp(addr1, addr2, PAGE_SIZE); + ret = memcmp(addr1 + offset1, addr2 + offset2, len); kunmap_local(addr2); kunmap_local(addr1); return ret; } +int __weak memcmp_pages(struct page *page1, struct page *page2) +{ + return memcmp_pages_range(page1, 0, page2, 0, PAGE_SIZE); +} + #ifdef CONFIG_PRINTK /** * mem_dump_obj - Print available provenance information From 89dbfab60e8c97e7c0c04fc5fc74fa2f032b419f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:44:23 -0400 Subject: [PATCH 0844/1579] arm64: preserve subpage offsets and lengths in MTE core dumps Size MTE tag segments from dumped bytes rather than a truncated native page count. For alternative userspace geometry, retain the selected fragment and save only its tags, preserving the physical subpage offset. Keep the native page path and share its assembly packing loop through a bounded tag-range helper. Add a disposable-guest core-file fixture checking actual ELF data and tag headers and bytes for split read-only, sparse and excluded mappings. Native and 4K processes pass on a 16K KASAN kernel; native, 4K and 16K processes pass on a 64K KASAN kernel. These are emulated MTE checks, not physical-hardware validation. Signed-off-by: Diljit Singh (cherry picked from commit 886de7154672d656b031d3ee5a1cda03b7f3168f) --- arch/arm64/include/asm/mte.h | 2 + arch/arm64/kernel/elfcore.c | 60 ++++- arch/arm64/lib/mte.S | 20 +- tools/testing/selftests/arm64/user4k/Makefile | 2 + .../selftests/arm64/user4k/mte-core-init.c | 206 ++++++++++++++++++ 5 files changed, 286 insertions(+), 4 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/mte-core-init.c diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 890d49a4398b2..c454f5ea8c3d1 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -25,6 +25,8 @@ unsigned long mte_copy_tags_to_user(void __user *to, void *from, unsigned long n); int mte_save_tags(struct page *page); void mte_save_page_tags(const void *page_addr, void *tag_storage); +/* Address and size must be aligned to 256 bytes; size may be zero. */ +void mte_save_tags_range(const void *addr, void *tag_storage, unsigned long size); void mte_restore_tags(swp_entry_t entry, struct page *page); void mte_restore_page_tags(void *page_addr, const void *tag_storage); void mte_invalidate_tags(int type, pgoff_t offset); diff --git a/arch/arm64/kernel/elfcore.c b/arch/arm64/kernel/elfcore.c index b735f4c2fe5ea..32ddc36838fe3 100644 --- a/arch/arm64/kernel/elfcore.c +++ b/arch/arm64/kernel/elfcore.c @@ -2,8 +2,10 @@ #include #include +#include #include #include +#include #include #include @@ -17,9 +19,60 @@ static unsigned long mte_vma_tag_dump_size(struct core_vma_metadata *m) { - return (m->dump_size >> PAGE_SHIFT) * MTE_PAGE_TAG_STORAGE; + return (m->dump_size / MTE_GRANULE_SIZE) * MTE_TAG_SIZE / 8; } +#ifdef CONFIG_MM_SUBPAGE +static int mte_dump_fragment_tags(struct coredump_params *cprm, + unsigned long start, unsigned long len) +{ + unsigned long size = mm_user_fragment_size(current->mm); + unsigned long tag_size = (size / MTE_GRANULE_SIZE) * MTE_TAG_SIZE / 8; + unsigned long addr; + void *tags = NULL; + int ret = 1; + + for (addr = start; addr < start + len; addr += size) { + struct user_page_fragment fragment; + struct page *page; + + if (get_dump_fragment(addr, &fragment) != 1) { + dump_skip(cprm, tag_size); + goto next; + } + page = folio_page(fragment.folio, fragment.offset >> PAGE_SHIFT); + if (!page_mte_tagged(page)) { + release_user_fragments(&fragment, 1, false); + dump_skip(cprm, tag_size); + goto next; + } + if (!tags) { + tags = mte_allocate_tag_storage(); + if (!tags) { + release_user_fragments(&fragment, 1, false); + ret = 0; + break; + } + } + mte_save_tags_range(folio_address(fragment.folio) + fragment.offset, tags, size); + release_user_fragments(&fragment, 1, false); + if (!dump_emit(cprm, tags, tag_size)) { + ret = 0; + break; + } +next: + if (fatal_signal_pending(current) || freezing(current)) { + ret = 0; + break; + } + cond_resched(); + } + if (tags) + mte_free_tag_storage(tags); + return ret; +} +#endif + /* Derived from dump_user_range(); start/end must be page-aligned */ static int mte_dump_tag_range(struct coredump_params *cprm, unsigned long start, unsigned long len) @@ -29,6 +82,11 @@ static int mte_dump_tag_range(struct coredump_params *cprm, void *tags = NULL; int locked = 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return mte_dump_fragment_tags(cprm, start, len); +#endif + for (addr = start; addr < start + len; addr += PAGE_SIZE) { struct page *page = get_dump_page(addr, &locked); diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S index c1ce370068cf4..8402e81b71474 100644 --- a/arch/arm64/lib/mte.S +++ b/arch/arm64/lib/mte.S @@ -183,6 +183,19 @@ SYM_FUNC_END(mte_copy_tags_to_user) * x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes */ SYM_FUNC_START(mte_save_page_tags) + mov x2, #PAGE_SIZE + b mte_save_tags_range +SYM_FUNC_END(mte_save_page_tags) + +/* + * Save tags for a 256-byte-aligned range. + * x0 - start address + * x1 - tag storage, x2 / 32 bytes + * x2 - byte length, a multiple of 256 + */ +SYM_FUNC_START(mte_save_tags_range) + cbz x2, 3f + add x6, x0, x2 multitag_transfer_size x7, x5 1: mov x2, #0 @@ -195,11 +208,12 @@ SYM_FUNC_START(mte_save_page_tags) str x2, [x1], #8 - tst x0, #(PAGE_SIZE - 1) - b.ne 1b + cmp x0, x6 + b.lo 1b +3: ret -SYM_FUNC_END(mte_save_page_tags) +SYM_FUNC_END(mte_save_tags_range) /* * Restore the tags in a page diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 0350be7a57912..889386ca79ffc 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -3,6 +3,7 @@ TEST_GEN_PROGS := page-contract proc-args TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init cow-pool +TEST_GEN_PROGS_EXTENDED += mte-core-init CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe @@ -60,6 +61,7 @@ $(OUTPUT)/libfex-abi.so: fex-native-abi.c $(OUTPUT)/exec-policy-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 $(OUTPUT)/exec-policy-init: LDLIBS += -pthread +$(OUTPUT)/mte-core-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 $(OUTPUT)/compat-policy-probe: compat-policy-probe.c $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ diff --git a/tools/testing/selftests/arm64/user4k/mte-core-init.c b/tools/testing/selftests/arm64/user4k/mte-core-init.c new file mode 100644 index 0000000000000..3a6cf5e0723b7 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mte-core-init.c @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* MTE core-file validation: PID 1 and its children in a disposable VM only. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef PT_AARCH64_MEMTAG_MTE +#define PT_AARCH64_MEMTAG_MTE (PT_LOPROC + 0x2) +#endif +#define TAGGED_BASE 0x200000000UL +#define ZERO_BASE 0x210000000UL +#define EXCLUDED_BASE 0x220000000UL +#define REQUIRE(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "FAIL line %d: %s errno=%d\n", __LINE__, #condition, errno); \ + exit(1); \ + } \ +} while (0) + +static void write_setting(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + + REQUIRE(fd >= 0); + REQUIRE(write(fd, value, strlen(value)) == (ssize_t)strlen(value)); + REQUIRE(!close(fd)); +} + +static unsigned char *tagged_map(unsigned long address, size_t size) +{ + void *p = mmap((void *)address, size, PROT_READ | PROT_WRITE | PROT_MTE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + + REQUIRE(p == (void *)address); + return p; +} + +static void set_tags(unsigned char *p, size_t size, unsigned int tag) +{ + for (size_t i = 0; i < size; i += 16) { + uintptr_t address = (uintptr_t)(p + i) | ((uintptr_t)tag << 56); + + asm volatile(".arch_extension memtag\n\tstg %0, [%0]" + : : "r" (address) : "memory"); + } +} + +static void child(unsigned long ps) +{ + struct stat self, init; + struct rlimit limit = { 64UL << 20, 64UL << 20 }; + unsigned char *p; + + REQUIRE(getppid() == 1); + REQUIRE(!stat("/proc/self/exe", &self) && !stat("/proc/1/exe", &init)); + REQUIRE(self.st_dev == init.st_dev && self.st_ino == init.st_ino); + REQUIRE(getauxval(AT_PAGESZ) == ps); + REQUIRE(!setrlimit(RLIMIT_CORE, &limit)); + REQUIRE(!prctl(PR_SET_DUMPABLE, 1UL, 0UL, 0UL, 0UL)); + REQUIRE(!prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 0UL, 0UL, 0UL)); + write_setting("/proc/self/coredump_filter", "1"); + p = tagged_map(TAGGED_BASE, 3 * ps); + for (unsigned int i = 0; i < 3; i++) { + memset(p + i * ps, 0x40 + i, ps); + set_tags(p + i * ps, ps, i + 1); + } + /* Split metadata at a non-native boundary and retain the physical offset. */ + REQUIRE(!mprotect(p + ps, ps, PROT_READ | PROT_MTE)); + p = tagged_map(ZERO_BASE, ps); + *p = 0; + /* Keep anonymous dump eligibility while leaving the leaf absent. */ + REQUIRE(!madvise(p, ps, MADV_DONTNEED)); + p = tagged_map(EXCLUDED_BASE, ps); + memset(p, 0x55, ps); + set_tags(p, ps, 7); + REQUIRE(!madvise(p, ps, MADV_DONTDUMP)); + raise(SIGABRT); + _exit(2); +} + +static void inspect_core(pid_t pid, unsigned long ps) +{ + char path[80]; + Elf64_Ehdr eh; + struct stat st; + unsigned int tagged = 0, loads = 0, zero = 0, excluded = 0; + unsigned char *bytes = malloc(ps); + int fd; + + REQUIRE(bytes); + snprintf(path, sizeof(path), "/tmp/core.%d", pid); + fd = open(path, O_RDONLY); + REQUIRE(fd >= 0 && !fstat(fd, &st)); + REQUIRE(pread(fd, &eh, sizeof(eh), 0) == sizeof(eh)); + REQUIRE(!memcmp(eh.e_ident, ELFMAG, SELFMAG)); + REQUIRE(eh.e_ident[EI_CLASS] == ELFCLASS64 && eh.e_type == ET_CORE); + REQUIRE(eh.e_phentsize == sizeof(Elf64_Phdr) && eh.e_phnum < 4096); + for (unsigned int n = 0; n < eh.e_phnum; n++) { + Elf64_Phdr ph; + unsigned char expected; + unsigned int index; + size_t length; + + REQUIRE(pread(fd, &ph, sizeof(ph), eh.e_phoff + n * sizeof(ph)) == sizeof(ph)); + if (ph.p_type != PT_LOAD && ph.p_type != PT_AARCH64_MEMTAG_MTE) + continue; + if (ph.p_vaddr >= TAGGED_BASE && ph.p_vaddr < TAGGED_BASE + 3 * ps) { + index = (ph.p_vaddr - TAGGED_BASE) / ps; + REQUIRE(ph.p_vaddr == TAGGED_BASE + index * ps && ph.p_memsz == ps); + if (ph.p_type == PT_LOAD) { + loads++; + length = ps; + expected = 0x40 + index; + } else { + tagged++; + length = ps / 32; + expected = (index + 1) * 0x11; + } + } else if (ph.p_vaddr == ZERO_BASE && ph.p_type == PT_AARCH64_MEMTAG_MTE) { + zero++; + length = ps / 32; + expected = 0; + REQUIRE(ph.p_memsz == ps); + } else if (ph.p_vaddr == EXCLUDED_BASE && ph.p_type == PT_AARCH64_MEMTAG_MTE) { + excluded++; + REQUIRE(ph.p_filesz == 0 && ph.p_memsz == ps); + continue; + } else { + continue; + } + if (ph.p_filesz != length) + fprintf(stderr, "segment type=%#x address=%#lx size=%lu expected=%zu\n", + ph.p_type, (unsigned long)ph.p_vaddr, + (unsigned long)ph.p_filesz, length); + REQUIRE(ph.p_filesz == length); + REQUIRE(ph.p_offset <= (uint64_t)st.st_size && length <= st.st_size - ph.p_offset); + REQUIRE(pread(fd, bytes, length, ph.p_offset) == (ssize_t)length); + for (size_t i = 0; i < length; i++) + REQUIRE(bytes[i] == expected); + } + REQUIRE(tagged == 3 && loads == 3 && zero == 1 && excluded == 1); + REQUIRE(!close(fd) && !unlink(path)); + free(bytes); + printf("MTE CORE CASE PASS page-size=%lu\n", ps); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + unsigned long sizes[] = { native, 4096, 16384 }; + unsigned int count = native > 16384 ? 3 : native > 4096 ? 2 : 1; + + setbuf(stdout, NULL); + setbuf(stderr, NULL); + if (argc == 3 && !strcmp(argv[1], "--child")) { + child(strtoul(argv[2], NULL, 10)); + return 1; + } + if (getpid() != 1) { + fprintf(stderr, "SKIP: disposable guest PID1 only\n"); + return 4; + } + REQUIRE(getauxval(AT_HWCAP2) & HWCAP2_MTE); + mkdir("/proc", 0755); + mkdir("/tmp", 0700); + REQUIRE(!mount("proc", "/proc", "proc", 0, NULL)); + REQUIRE(!mount("tmpfs", "/tmp", "tmpfs", 0, "mode=0700")); + write_setting("/proc/sys/kernel/core_pattern", "/tmp/core.%p"); + for (unsigned int i = 0; i < count; i++) { + pid_t pid = fork(); + int status; + + REQUIRE(pid >= 0); + if (!pid) { + char value[32]; + + snprintf(value, sizeof(value), "%lu", sizes[i]); + REQUIRE(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)); + execl("/init", "/init", "--child", value, NULL); + _exit(3); + } + REQUIRE(waitpid(pid, &status, 0) == pid); + REQUIRE(WIFSIGNALED(status) && WTERMSIG(status) == SIGABRT && WCOREDUMP(status)); + inspect_core(pid, sizes[i]); + } + printf("MTE CORE PASS cases=%u\n", count); + reboot(RB_POWER_OFF); + return 0; +} From ded687fd2535ca26af03486ae61039b3a380163c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:52:13 -0400 Subject: [PATCH 0845/1579] mm: distinguish callback-supplied pages from page-cache faults A VMA can retain vm_file while its fault callback returns independent pages. Use cache-indexed fault installation only when the returned folio belongs to that file mapping; otherwise preserve the callback page and its requested subpage offset. Allow ordinary non-anonymous fragment references to follow these mappings without requiring page-cache membership. Keep permission checks and private read-pin COW rules unchanged. Add native, 4K and supported 16K callback tests with reversed backing pages, nonzero offsets and remote reads. They pass on 16K/64K KASAN guests, along with existing file-fragment lifetime/COW and special-mapping tests. This tests the callback contract, not a physical usbmon device. Signed-off-by: Diljit Singh (cherry picked from commit 09dae19a275153e1bbfe13e4b31f3fa8d94f72e8) --- arch/arm64/mm/user4k-test.c | 81 +++++++++++++++++++++++++++++++++++++ mm/gup.c | 4 -- mm/memory.c | 4 +- 3 files changed, 83 insertions(+), 6 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 69456ba03bac5..de2d47a5e569e 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -5439,6 +5439,84 @@ static void user4k_special_mapping_test(struct kunit *test) } } +static vm_fault_t test_callback_fault(struct vm_fault *vmf) +{ + struct page **pages = vmf->vma->vm_private_data; + + if (vmf->pgoff >= 2) + return VM_FAULT_SIGBUS; + + vmf->page = pages[vmf->pgoff]; + get_page(vmf->page); + return 0; +} + +static const struct vm_operations_struct test_callback_vm_ops = { + .fault = test_callback_fault, +}; + +static void test_callback_mapping(struct kunit *test, unsigned int shift) +{ + void *data[2] = { test_alloc_page(test), test_alloc_page(test) }; + struct page **pages = kunit_kcalloc(test, 2, sizeof(*pages), GFP_KERNEL); + struct file *file = test_cache_file(test, 0, false); + struct vm_area_struct *vma; + unsigned long size = 1UL << shift; + unsigned long offset = shift == PAGE_SHIFT ? 0 : size; + unsigned long len = 2 * PAGE_SIZE - offset; + unsigned long addr; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, data[0]); + KUNIT_ASSERT_NOT_NULL(test, data[1]); + KUNIT_ASSERT_NOT_NULL(test, pages); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + pages[0] = virt_to_page(data[1]); + pages[1] = virt_to_page(data[0]); + for (i = 0; i < 2 * PAGE_SIZE / size; i++) + memset(page_address(pages[(i * size) >> PAGE_SHIFT]) + + offset_in_page(i * size), 0x51 + i, size); + + /* The file identifies the VMA; the callback supplies independent pages. */ + vma = test_file_vma_granule(test, file, shift, false, TEST_VA, len, offset); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_write_lock(vma->vm_mm); + vma->vm_ops = &test_callback_vm_ops; + vma->vm_private_data = pages; + mmap_write_unlock(vma->vm_mm); + for (addr = vma->vm_start; addr < vma->vm_end; addr += size) { + unsigned long source = offset + addr - vma->vm_start; + u8 byte; + + KUNIT_ASSERT_EQ(test, test_file_fault(vma, addr, false) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, + test_fault_entry(test, vma->vm_mm, addr)), + page_to_phys(pages[source >> PAGE_SHIFT]) + offset_in_page(source)); + KUNIT_ASSERT_EQ(test, access_remote_vm(vma->vm_mm, addr + 7, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0x51 + source / size)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_FILEPAGES), len / size); +} + +static void native_callback_mapping_test(struct kunit *test) +{ + test_callback_mapping(test, PAGE_SHIFT); +} + +static void user4k_callback_mapping_test(struct kunit *test) +{ + test_callback_mapping(test, 12); +} + +static void user16k_callback_mapping_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative requires a larger native granule"); + return; + } + test_callback_mapping(test, 14); +} + static void user4k_insert_range_test(struct kunit *test) { unsigned int native; @@ -14115,6 +14193,9 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_stack_relocate_test), KUNIT_CASE(user4k_exec_arguments_test), KUNIT_CASE(user4k_special_mapping_test), + KUNIT_CASE(native_callback_mapping_test), + KUNIT_CASE(user4k_callback_mapping_test), + KUNIT_CASE(user16k_callback_mapping_test), KUNIT_CASE(user4k_insert_fragment_test), KUNIT_CASE(user16k_insert_shared_test), KUNIT_CASE(user4k_insert_range_test), diff --git a/mm/gup.c b/mm/gup.c index 8d9bbc80546fa..b7738d87d55b8 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -3521,10 +3521,6 @@ static int follow_subpage_fragment(struct vm_area_struct *vma, unsigned long add goto acquired; } if (!folio_test_anon(folio)) { - if (vma->vm_file && folio_mapping(folio) != vma->vm_file->f_mapping) { - ret = -EOPNOTSUPP; - goto out; - } if ((flags & FOLL_WRITE) && !pte_write(pte)) goto out; /* Private non-anonymous read pins must first acquire COW ownership. */ diff --git a/mm/memory.c b/mm/memory.c index 22e3ea5356680..a2ca51ea01939 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -7096,7 +7096,7 @@ vm_fault_t map_file_user_pages(struct vm_fault *vmf, struct folio *folio, } /* - * A non-file fault callback supplies one native page, not a cache-indexed + * A non-page-cache fault callback supplies one native page, not a cache-indexed * folio. In particular, VVAR callbacks may reorder native pages. Only map * quarters of this supplied page; never infer adjacent pages from its folio. */ @@ -7232,7 +7232,7 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) inc_mm_counter(mm, MM_ANONPAGES); ret = 0; } else { - if (vma->vm_file) + if (vma->vm_file && folio_mapping(folio) == vma->vm_file->f_mapping) ret = map_file_user_pages(vmf, folio, folio->index, folio_next_index(folio) - 1, true); else if (mm_page_size(mm) < PAGE_SIZE) From bfd5ef2860605d0c17ddd24e27019f98eada5d51 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:58:29 -0400 Subject: [PATCH 0846/1579] mm: clip mapping walks to exact VMA byte intervals walk_page_mapping() truncates a VMA length to native pages and ignores its subpage file offset. Short mappings can disappear from the walk and partial first/last native pages can produce incorrect virtual bounds. Use the existing byte-offset translation and last-index helpers to clip the requested native file indices to the VMA. Handle an empty request and reject an overflowing index interval before the interval-tree walk. Validated native and smaller-granule walks over short mappings and partial first/last pages in 16K/64K KASAN guests. The temporary fixture and results remain in the local validation records instead of adding another permanent arm64 test block. The generic x86 pagewalk object also compiles. Signed-off-by: Diljit Singh (cherry picked from commit 0a085675da3ec34e40f94c8d9af6194cfd133c91) --- mm/pagewalk.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/mm/pagewalk.c b/mm/pagewalk.c index c3216b4852399..54bc7ac30befa 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -809,27 +810,26 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index, .private = private, }; struct vm_area_struct *vma; - pgoff_t vba, vea, cba, cea; + pgoff_t last_index; unsigned long start_addr, end_addr; int err = 0; if (!check_ops_safe(ops)) return -EINVAL; + if (!nr) + return 0; + if (check_add_overflow(first_index, nr - 1, &last_index)) + return -EINVAL; lockdep_assert_held(&mapping->i_mmap_rwsem); vma_interval_tree_foreach(vma, &mapping->i_mmap, first_index, - first_index + nr - 1) { - /* Clip to the vma */ - vba = vma->vm_pgoff; - vea = vba + vma_pages(vma); - cba = first_index; - cba = max(cba, vba); - cea = first_index + nr; - cea = min(cea, vea); - - start_addr = ((cba - vba) << PAGE_SHIFT) + vma->vm_start; - end_addr = ((cea - vba) << PAGE_SHIFT) + vma->vm_start; - if (start_addr >= end_addr) + last_index) { + /* Clip native file indices to the VMA's exact byte interval. */ + start_addr = first_index <= vma->vm_pgoff ? vma->vm_start : + vma_address_at_offset(vma, (struct vm_page_offset) { first_index, 0 }); + end_addr = last_index >= vma_last_pgoff(vma) ? vma->vm_end : + vma_address_at_offset(vma, (struct vm_page_offset) { last_index + 1, 0 }); + if (IS_ERR_VALUE(start_addr) || IS_ERR_VALUE(end_addr) || start_addr >= end_addr) continue; walk.vma = vma; From f44e5019c23c06a6fd889a190b2c8ae78f3b791b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:02:48 -0400 Subject: [PATCH 0847/1579] mm: round up mapping lengths when checking I/O resource bounds A mapping can be shorter than a native page when its MM uses a smaller granule. Count a partial final native page in __simple_ioremap_prep() instead of truncating it away. Native-aligned mappings retain their existing calculation, and the quotient/remainder form avoids a rounding addition overflow. Both 16K/64K KASAN kernels build and pass the existing PFN-remap suite; the x86 memory object also builds. No device-boundary reproducer was run. This bounds correction does not qualify every device offset interface. Signed-off-by: Diljit Singh (cherry picked from commit d4e50b58b49627c95920d2f106da600ead1254ff) --- mm/memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/memory.c b/mm/memory.c index a2ca51ea01939..dd2916612f9c7 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3696,7 +3696,7 @@ static int __simple_ioremap_prep(unsigned long vm_len, pgoff_t vm_pgoff, pages -= vm_pgoff; /* Can we fit all of the mapping? */ - if ((vm_len >> PAGE_SHIFT) > pages) + if ((vm_len >> PAGE_SHIFT) + !!offset_in_page(vm_len) > pages) return -EINVAL; *pfnp = pfn; From e9628b1a52bb35824180722a2b4469da2894d372 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:07:24 -0400 Subject: [PATCH 0848/1579] arm64: retain exact fragments for remote MTE tag access The legacy remote-page interface cannot represent smaller-granule backing. Use the stable-VMA fragment interface for alternative MMs, take the actual physical offset from the returned record, and bound each transfer by its length. Release the same retained fragment and dirty its ownership record on writes. Preserve the native page path and existing ptrace permission and partial-transfer semantics. Reused the local core-file fixture to validate own-child ptrace tag reads, writes and restoration across three differently tagged mappings, including a read-only mapping. Native/4K cases pass in a 16K KASAN guest and native/4K/16K cases pass in a 64K guest; resulting core files also pass. The feature-disabled native MTE object builds. No additional submitted test program is introduced, and physical MTE hardware remains untested. Signed-off-by: Diljit Singh (cherry picked from commit f41afd84696d9b7381b25f105f58caa2c0590365) --- arch/arm64/kernel/mte.c | 44 +++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index 645f549d9fed9..fb4eba0626e5a 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -463,11 +464,24 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, while (len) { struct vm_area_struct *vma; - unsigned long tags, offset; + unsigned long tags = 0, offset; void *maddr; - struct page *page = get_user_page_vma_remote(mm, addr, - gup_flags, &vma); + struct page *page; struct folio *folio; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment = {}; + + if (mm_page_size(mm) != PAGE_SIZE) { + unsigned long bytes = min(len, PAGE_SIZE / MTE_GRANULE_SIZE) * + MTE_GRANULE_SIZE; + int got = get_user_fragment_vma_remote(mm, addr, bytes, + gup_flags, &fragment, &vma); + + page = got > 0 ? folio_page(fragment.folio, + fragment.offset >> PAGE_SHIFT) : ERR_PTR(got ?: -EFAULT); + } else +#endif + page = get_user_page_vma_remote(mm, addr, gup_flags, &vma); if (IS_ERR(page)) { err = PTR_ERR(page); @@ -483,8 +497,7 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, */ if (!(vma->vm_flags & VM_MTE)) { err = -EOPNOTSUPP; - put_page(page); - break; + goto release; } folio = page_folio(page); @@ -497,18 +510,33 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, /* limit access to the end of the page */ offset = offset_in_page(addr); tags = min(len, (PAGE_SIZE - offset) / MTE_GRANULE_SIZE); +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) { + offset = offset_in_page(fragment.offset); + tags = fragment.length / MTE_GRANULE_SIZE; + } +#endif maddr = page_address(page); if (write) { tags = mte_copy_tags_from_user(maddr + offset, buf, tags); - set_page_dirty_lock(page); +#ifdef CONFIG_MM_SUBPAGE + if (!fragment.folio) +#endif + set_page_dirty_lock(page); } else { tags = mte_copy_tags_to_user(buf, maddr + offset, tags); } - put_page(page); +release: +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) + release_user_fragments(&fragment, 1, write && !err); + else +#endif + put_page(page); /* error accessing the tracer's buffer */ - if (!tags) + if (err || !tags) break; len -= tags; From e4b30f2f526d40c7d83e3861b172029717a46c90 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:18:24 -0400 Subject: [PATCH 0849/1579] iommu/arm-smmu-v3: use the bound mm granule for SVA Build the SVA context descriptor and invalidation geometry from the bound MM rather than PAGE_SIZE. Check the target device's granule support at allocation and attachment, and record the same granule in the domain so range invalidation commands agree with the CPU page-table geometry. Extend the existing SVA descriptor transition test to cover the enabled alternative granules. The existing 25-case SMMUv3 suite passes in both 16K and 64K KASAN guests; native feature-disabled SVA and test objects also compile. Actual SVA device DMA and invalidation delivery remain untested. The driver's existing global native-granule capability prerequisite is preserved. Signed-off-by: Diljit Singh (cherry picked from commit 4ccea57e713c22800c059cad4662abef494dd054) --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 16 ++++++++----- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 23 +++++++++++++++---- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index f1f8e01a7e914..5706967254517 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -36,13 +36,13 @@ arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain) spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); } -static u64 page_size_to_cd(void) +static u64 page_size_to_cd(unsigned long page_size) { static_assert(PAGE_SIZE == SZ_4K || PAGE_SIZE == SZ_16K || PAGE_SIZE == SZ_64K); - if (PAGE_SIZE == SZ_64K) + if (page_size == SZ_64K) return ARM_LPAE_TCR_TG0_64K; - if (PAGE_SIZE == SZ_16K) + if (page_size == SZ_16K) return ARM_LPAE_TCR_TG0_16K; return ARM_LPAE_TCR_TG0_4K; } @@ -83,7 +83,7 @@ void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, target->data[0] |= cpu_to_le64( FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 64ULL - vabits_actual) | - FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd()) | + FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd(mm_page_size(mm))) | FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, ARM_LPAE_TCR_RGN_WBWA) | FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, @@ -138,7 +138,7 @@ static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, */ size = end - start; - arm_smmu_domain_inv_range(smmu_domain, start, size, PAGE_SIZE, false); + arm_smmu_domain_inv_range(smmu_domain, start, size, mm_page_size(mm), false); } static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) @@ -256,6 +256,8 @@ static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain, if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) return -EOPNOTSUPP; + if (!(master->smmu->pgsize_bitmap & mm_page_size(domain->mm))) + return -EOPNOTSUPP; /* Prevent arm_smmu_mm_release from being called while we are attaching */ if (!mmget_not_zero(domain->mm)) @@ -312,6 +314,8 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) return ERR_PTR(-EOPNOTSUPP); + if (!(smmu->pgsize_bitmap & mm_page_size(mm))) + return ERR_PTR(-EOPNOTSUPP); smmu_domain = arm_smmu_domain_alloc(); if (IS_ERR(smmu_domain)) @@ -323,7 +327,7 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, * Choose page_size as the leaf page size for invalidation when * ARM_SMMU_FEAT_RANGE_INV is present */ - smmu_domain->domain.pgsize_bitmap = PAGE_SIZE; + smmu_domain->domain.pgsize_bitmap = mm_page_size(mm); smmu_domain->stage = ARM_SMMU_DOMAIN_SVA; smmu_domain->smmu = smmu; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index add671363c828..fa1c7b03394d6 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -616,12 +616,25 @@ static void arm_smmu_v3_write_cd_test_sva_clear(struct kunit *test) { struct arm_smmu_cd cd = {}; struct arm_smmu_cd cd_2; + const unsigned long flags[] = { 0, MMCF_USER_4K, MMCF_USER_16K }; + /* TG0 encodings: 4K = 0, 64K = 1, 16K = 2. */ + const unsigned int tg[] = { PAGE_SHIFT == 16 ? 1 : PAGE_SHIFT == 14 ? 2 : 0, 0, 2 }; + unsigned int i; - arm_smmu_test_make_sva_cd(&cd_2, 1997); - arm_smmu_v3_test_cd_expect_non_hitless_transition( - test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); - arm_smmu_v3_test_cd_expect_non_hitless_transition( - test, &cd_2, &cd, NUM_EXPECTED_SYNCS(2)); + for (i = 0; i < ARRAY_SIZE(flags); i++) { + if (i && (!IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) || + (i == 2 && PAGE_SHIFT <= 14))) + continue; + sva_mm.context.flags = flags[i]; + arm_smmu_test_make_sva_cd(&cd_2, 1997); + KUNIT_EXPECT_EQ(test, FIELD_GET(CTXDESC_CD_0_TCR_TG0, + le64_to_cpu(cd_2.data[0])), tg[i]); + arm_smmu_v3_test_cd_expect_non_hitless_transition( + test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); + arm_smmu_v3_test_cd_expect_non_hitless_transition( + test, &cd_2, &cd, NUM_EXPECTED_SYNCS(2)); + } + sva_mm.context.flags = 0; } static void arm_smmu_v3_write_cd_test_sva_release(struct kunit *test) From 581b1ace0c990677574011ea8c2a6eafce1118a0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:21:46 -0400 Subject: [PATCH 0850/1579] ocfs2: use truncate_pagecache after shrinking an extent-backed file After ocfs2_orphan_for_truncate() updates i_size, use the generic pagecache truncation helper. Its invalidation covers smaller userspace leaves in the last native page and repeats after pagecache truncation to remove COW mappings created during the operation. Keep the inline-data path's ordering unchanged. OCFS2 blocks are at most 4K, so inline data cannot span multiple leaves of the exposed alternative userspace configurations. The changed OCFS2 file object builds for 16K and 64K arm64 configurations. A mounted OCFS2 filesystem has not been runtime-tested for this change. Signed-off-by: Diljit Singh (cherry picked from commit a6c5a0042f88694ebedb8b622cd0ca17896ba474) --- fs/ocfs2/file.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index 7df9921c1a389..21153bd53f66e 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -504,8 +504,7 @@ int ocfs2_truncate_file(struct inode *inode, goto bail_unlock_sem; } - unmap_mapping_range(inode->i_mapping, new_i_size + PAGE_SIZE - 1, 0, 1); - truncate_inode_pages(inode->i_mapping, new_i_size); + truncate_pagecache(inode, new_i_size); status = ocfs2_commit_truncate(osb, inode, di_bh); if (status < 0) { From f03bbf7cdc8d5a33a98b89ac9ca63239b0cde451 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:35:39 -0400 Subject: [PATCH 0851/1579] dma-buf: retain subpage offsets in CMA and udmabuf faults CMA and udmabuf fault handlers identify the native backing page but lose the smaller leaf's byte offset. Pass the explicit file subpage offset to PFN insertion. In udmabuf, use the same translation for demand faults and prefaults, advancing by the target MM granule so prefault coverage remains complete. Validated CPU mappings with the existing dma-buf heap and udmabuf selftests in 16K/64K KASAN guests for native, 4K and supported 16K processes. The local udmabuf test uses native-aligned buffer sizes and distinct per-leaf bytes; its two hugetlb cases are explicitly skipped without a reserved pool. Both CMA heap names and vgem import pass. No additional test source is submitted and physical device DMA remains unqualified. Signed-off-by: Diljit Singh (cherry picked from commit ac963beadc4754a08163d643e26b23f0d1c31bd5) --- drivers/dma-buf/heaps/cma_heap.c | 5 ++++- drivers/dma-buf/udmabuf.c | 38 ++++++++++++++++---------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c index a359aac455799..37ec7bc84a24e 100644 --- a/drivers/dma-buf/heaps/cma_heap.c +++ b/drivers/dma-buf/heaps/cma_heap.c @@ -174,7 +174,10 @@ static vm_fault_t cma_heap_vm_fault(struct vm_fault *vmf) if (vmf->pgoff >= buffer->pagecount) return VM_FAULT_SIGBUS; - return vmf_insert_pfn(vma, vmf->address, page_to_pfn(buffer->pages[vmf->pgoff])); + return vmf_insert_pfn_prot_mkwrite_offset(vma, vmf->address, + page_to_pfn(buffer->pages[vmf->pgoff]), + vma_page_offset_at(vma, vmf->address).offset, + vma->vm_page_prot, false); } static const struct vm_operations_struct dma_heap_vm_ops = { diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c index 2e7b5d4d11f42..71efcf40d9169 100644 --- a/drivers/dma-buf/udmabuf.c +++ b/drivers/dma-buf/udmabuf.c @@ -45,45 +45,45 @@ struct udmabuf { pgoff_t *offsets; }; -static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf) +static vm_fault_t udmabuf_insert_pfn(struct vm_area_struct *vma, unsigned long addr) { - struct vm_area_struct *vma = vmf->vma; struct udmabuf *ubuf = vma->vm_private_data; - pgoff_t pgoff = vmf->pgoff; - unsigned long addr, pfn; - vm_fault_t ret; + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + unsigned long pfn; - if (pgoff >= ubuf->pagecount) + if (pos.index >= ubuf->pagecount) return VM_FAULT_SIGBUS; - pfn = folio_pfn(ubuf->folios[pgoff]); - pfn += ubuf->offsets[pgoff] >> PAGE_SHIFT; + pfn = folio_pfn(ubuf->folios[pos.index]); + pfn += ubuf->offsets[pos.index] >> PAGE_SHIFT; + return vmf_insert_pfn_prot_mkwrite_offset(vma, addr, pfn, pos.offset, + vma->vm_page_prot, false); +} + +static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + unsigned long addr; + vm_fault_t ret; - ret = vmf_insert_pfn(vma, vmf->address, pfn); + ret = udmabuf_insert_pfn(vma, vmf->address); if (ret & VM_FAULT_ERROR) return ret; /* pre fault */ - pgoff = vma->vm_pgoff; addr = vma->vm_start; - for (; addr < vma->vm_end; pgoff++, addr += PAGE_SIZE) { + for (; addr < vma->vm_end; addr += mm_page_size(vma->vm_mm)) { if (addr == vmf->address) continue; - if (WARN_ON(pgoff >= ubuf->pagecount)) - break; - - pfn = folio_pfn(ubuf->folios[pgoff]); - pfn += ubuf->offsets[pgoff] >> PAGE_SHIFT; - /** - * If the below vmf_insert_pfn() fails, we do not return an + * If the below insertion fails, we do not return an * error here during this pre-fault step. However, an error * will be returned if the failure occurs when the addr is * truly accessed. */ - if (vmf_insert_pfn(vma, addr, pfn) & VM_FAULT_ERROR) + if (udmabuf_insert_pfn(vma, addr) & VM_FAULT_ERROR) break; } From c663e6321a81fd3978e93b6c9690d8bf35d88622 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:51:16 -0400 Subject: [PATCH 0852/1579] arm64/mm: preserve neighboring subpage dirty state when clearing PTEs Contiguous-PTE access/dirty clearing can expand to the full group because native MM tracks those bits per folio. Smaller userspace leaves can be reclaimed independently, so extending a dirty clear can erase evidence belonging to a different retained fragment. Unfold partially covered groups for smaller-granule dirty clearing and operate on the requested entries only. Keep native handling, full-group operations and the existing folding policy. Update the existing contiguous-PTE expectations for fragment isolation. The lazyfree migration case accepts dirty retention because migration can refold a group and conservatively aggregate a neighbor's dirty state; it still rejects a leftover migration entry or clean retained slot. The existing contiguous-PTE and lazyfree suites pass on 16K/64K KASAN kernels. No data-loss reproducer was run. Signed-off-by: Diljit Singh (cherry picked from commit 425488825ee3b258100181b75295215cf50397fe) --- arch/arm64/mm/contpte.c | 12 +++++++++--- arch/arm64/mm/user4k-test.c | 20 ++++++++++++++++---- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index e78e0d4265218..b3fcca267148b 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -614,6 +614,15 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, unsigned int nr, cydp_t flags) { struct mm_struct *mm = vma->vm_mm; + unsigned long start = addr; + unsigned long end = start + nr * mm_page_size(mm); + + /* Smaller leaves may be discarded independently within the folio. */ + if (mm_page_size(mm) < PAGE_SIZE && (flags & CYDP_CLEAR_DIRTY)) { + contpte_try_unfold_partial(mm, addr, ptep, nr); + __clear_young_dirty_ptes(vma, addr, ptep, nr, flags); + return; + } /* * We can safely clear access/dirty without needing to unfold from @@ -625,9 +634,6 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, * when it is covered by a single folio, we can get away with * clearing access/dirty for the whole block. */ - unsigned long start = addr; - unsigned long end = start + nr * mm_page_size(mm); - ptep = contpte_align_addr_ptep(vma->vm_mm, &start, &end, ptep, nr); __clear_young_dirty_ptes(vma, start, ptep, (end - start) >> mm_page_shift(mm), flags); } diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index de2d47a5e569e..bad3eee2c79ee 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -7771,14 +7771,19 @@ static void test_contpte_granule(struct kunit *test, unsigned int shift) CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY); for (i = 0; i < f->nr; i++) { observed = __ptep_get(pte + i); - ok.aged &= !pte_young(observed) && !pte_dirty(observed) && pte_cont(observed); + if (size < PAGE_SIZE) + ok.aged &= !pte_cont(observed) && + pte_young(observed) == (i != 3) && pte_dirty(observed) == (i != 3); + else + ok.aged &= !pte_young(observed) && !pte_dirty(observed) && + pte_cont(observed); } observed = pte_mkdirty(pte_mkyoung(__ptep_get(pte + 3))); access_same_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, pte + 3, observed, 1); for (i = 0; i < f->nr; i++) { observed = __ptep_get(pte + i); - ok.access_all &= pte_cont(observed) && pte_young(observed) && + ok.access_all &= pte_cont(observed) == (size >= PAGE_SIZE) && pte_young(observed) && pte_dirty(observed) && !pte_write(observed); } access_noop_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, @@ -9609,9 +9614,16 @@ static void user4k_lazyfree_test(struct kunit *test) folio_put(folio); for (i = 0; i < 4; i++) { pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); - KUNIT_EXPECT_EQ(test, pte_none(pte), + /* Migration may refold dirty neighbors and retain an advised slot. */ + if (mode == 5 && !i && pte_present(pte)) { + KUNIT_EXPECT_TRUE(test, pte_dirty(pte)); + continue; + } + KUNIT_EXPECT_EQ_MSG(test, pte_none(pte), (mode == 4 || mode == 9 || mode == 10) || - (i == 0 && (mode == 0 || mode == 5))); + (i == 0 && (mode == 0 || mode == 5)), + "mode=%u slot=%u pte=%llx", mode, i, + (unsigned long long)pte_val(pte)); } if (mode == 2 || mode == 6 || mode == 7) release_user_fragments(&held, 1, false); From 5756a5f8fcb385fce46e8fa9eab3f8fdc3c8a3f0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:58:30 -0400 Subject: [PATCH 0853/1579] mm: determine swap allocation requirements before taking the local lock The minimum anonymous folio order lookup temporarily holds an anon-vma reference. Dropping its last reference may sleep, so eligibility checks must not repeat that lookup under the per-CPU swap-cluster local lock. Reuse the mandatory-allocation decision already calculated by folio_alloc_swap() and pass it through the fast and slow allocation paths. The required fallback uses the same decision; hibernation has no folio. Document the lookup's sleepable context. 16K/64K KASAN builds and the existing swap/swapoff and UFFD move contract checks pass. The x86 swapfile object compiles. These checks do not reproduce the destructor lifetime race or qualify regular-file swap transport. Signed-off-by: Diljit Singh (cherry picked from commit 5f06041cbb3a178674d5aae5eb40483980175fca) --- mm/rmap.c | 5 ++++- mm/swapfile.c | 29 ++++++++++++++++------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index bf11cc857cb45..9241060489f58 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -629,7 +629,10 @@ struct anon_vma *folio_get_anon_vma(const struct folio *folio) } #ifdef CONFIG_MM_SUBPAGE -/* The caller retains the folio; splitting rechecks the order under its lock. */ +/* + * The caller retains the folio; splitting rechecks the order under its lock. + * May sleep when dropping the temporary anon-vma reference. + */ unsigned int folio_anon_min_order(struct folio *folio) { struct anon_vma *anon_vma = NULL; diff --git a/mm/swapfile.c b/mm/swapfile.c index c2c49b9fb3f36..ca11d95c2d037 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1083,7 +1083,8 @@ static void swap_reclaim_work(struct work_struct *work) swap_reclaim_full_clusters(si, true); } -static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio *folio) +static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio *folio, + bool mandatory) { unsigned int flags; @@ -1093,7 +1094,7 @@ static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio * if (flags & SWP_BLKDEV) return true; /* Filesystem swap_rw implementations retain their existing contract. */ - return si->bdev && !(flags & SWP_FS_OPS) && folio_anon_min_order(folio); + return si->bdev && !(flags & SWP_FS_OPS) && mandatory; } /* @@ -1101,13 +1102,13 @@ static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio * * cluster for current CPU too. */ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, - struct folio *folio) + struct folio *folio, bool mandatory) { struct swap_cluster_info *ci; unsigned int order = likely(folio) ? folio_order(folio) : 0; unsigned int offset = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID; - if (!swap_folio_alloc_allowed(si, folio)) + if (!swap_folio_alloc_allowed(si, folio, mandatory)) return 0; if (!(si->flags & SWP_SOLIDSTATE)) { @@ -1371,7 +1372,7 @@ static bool get_swap_device_info(struct swap_info_struct *si) * Fast path try to get swap entries with specified order from current * CPU's swap entry pool (a cluster). */ -static bool swap_alloc_fast(struct folio *folio) +static bool swap_alloc_fast(struct folio *folio, bool mandatory) { unsigned int order = folio_order(folio); struct swap_cluster_info *ci; @@ -1386,7 +1387,7 @@ static bool swap_alloc_fast(struct folio *folio) offset = this_cpu_read(percpu_swap_cluster.offset[order]); if (!si || !offset || !get_swap_device_info(si)) return false; - if (!swap_folio_alloc_allowed(si, folio)) { + if (!swap_folio_alloc_allowed(si, folio, mandatory)) { put_swap_device(si); return false; } @@ -1405,7 +1406,7 @@ static bool swap_alloc_fast(struct folio *folio) } /* Rotate the device and switch to a new cluster */ -static void swap_alloc_slow(struct folio *folio) +static void swap_alloc_slow(struct folio *folio, bool mandatory) { struct swap_info_struct *si, *next; @@ -1416,7 +1417,7 @@ static void swap_alloc_slow(struct folio *folio) plist_requeue(&si->avail_list, &swap_avail_head); spin_unlock(&swap_avail_lock); if (get_swap_device_info(si)) { - cluster_alloc_swap_entry(si, folio); + cluster_alloc_swap_entry(si, folio, mandatory); put_swap_device(si); if (folio_test_swapcache(folio)) return; @@ -1808,7 +1809,7 @@ static void swap_alloc_required_from_device(struct swap_info_struct *si, unsigned int order = folio_order(folio), nr = folio_nr_pages(folio); unsigned long start; - if (!swap_folio_alloc_allowed(si, folio)) + if (!swap_folio_alloc_allowed(si, folio, true)) return; for (start = 0; start < si->max; start += SWAPFILE_CLUSTER) { struct swap_cluster_info *ci = __swap_offset_to_cluster(si, start); @@ -1886,10 +1887,12 @@ int folio_alloc_swap(struct folio *folio) { unsigned int order = folio_order(folio); unsigned int size = 1 << order; - bool mandatory = order && folio_anon_min_order(folio); + bool mandatory; VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); VM_BUG_ON_FOLIO(!folio_test_uptodate(folio), folio); + /* The temporary anon-vma reference may require sleepable final release. */ + mandatory = order && folio_anon_min_order(folio); if (order) { /* @@ -1911,8 +1914,8 @@ int folio_alloc_swap(struct folio *folio) again: local_lock(&percpu_swap_cluster.lock); - if (!swap_alloc_fast(folio)) - swap_alloc_slow(folio); + if (!swap_alloc_fast(folio, mandatory)) + swap_alloc_slow(folio, mandatory); local_unlock(&percpu_swap_cluster.lock); if (mandatory && unlikely(!folio_test_swapcache(folio))) @@ -2318,7 +2321,7 @@ swp_entry_t swap_alloc_hibernation_slot(int type) swap_cluster_unlock(ci); } if (!offset) - offset = cluster_alloc_swap_entry(si, NULL); + offset = cluster_alloc_swap_entry(si, NULL, false); local_unlock(&percpu_swap_cluster.lock); if (offset) entry = swp_entry(si->type, offset); From 229284442eea6c1277af638f5a2035cf6ee5f108 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:17:39 -0400 Subject: [PATCH 0854/1579] mm: return the mapped fragment address from page_mapped_in_vma Supply the native page offset when bounding the reverse-map walk. For a smaller-granule MM, return the matching leaf found by the walk instead of the initial address, which can refer to an unmapped leading fragment. Keep the byte address within a covering leaf for native and coarser MMs. Extend the existing boundary case with a nonzero file offset and an absent leading fragment. Reverse-map tests pass in 16K/64K KASAN guests; native x86 objects build with CONFIG_MEMORY_FAILURE enabled. Signed-off-by: Diljit Singh (cherry picked from commit c11e970bc8c123095f2deb62b40abe29cb8d129d) --- arch/arm64/mm/user4k-test.c | 16 ++++++++++++++++ mm/page_vma_mapped.c | 8 ++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index bad3eee2c79ee..42719000b5a1d 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -1507,11 +1507,14 @@ static void user4k_rmap_boundary_test(struct kunit *test) KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); vma->vm_start = SZ_2M - SZ_4K; vma->vm_end = vma->vm_start + PAGE_SIZE; + vma->vm_pgoff = 37; + page_folio(pfn_to_page(ctx->phys[0] >> PAGE_SHIFT))->index = vma->vm_pgoff; for (i = 0; i < 3; i++) KUNIT_ASSERT_EQ(test, test_map_alias(ctx, 4 + i, vma->vm_start + i * SZ_4K, ctx->phys[i]), 0); walk = (struct page_vma_mapped_walk) { .pfn = ctx->phys[0] >> PAGE_SHIFT, + .pgoff = vma->vm_pgoff, .nr_pages = 1, .vma = vma, .address = vma->vm_start, @@ -1529,6 +1532,19 @@ static void user4k_rmap_boundary_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, valid); KUNIT_EXPECT_EQ(test, count, 3U); KUNIT_EXPECT_TRUE(test, walk.flags & PVMW_PGTABLE_CROSSED); +#ifdef CONFIG_MEMORY_FAILURE + { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(ctx->mm, vma->vm_start, &ptl); + + KUNIT_ASSERT_NOT_NULL(test, pte); + ptep_get_and_clear(ctx->mm, vma->vm_start, pte); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, + page_mapped_in_vma(pfn_to_page(ctx->phys[0] >> PAGE_SHIFT), vma), + vma->vm_start + SZ_4K); + } +#endif } static void native_rmap_walk_test(struct kunit *test) diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index e3b0b64fbfba2..9377be895563a 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -378,20 +378,24 @@ unsigned long page_mapped_in_vma(const struct page *page, unsigned long address; struct page_vma_mapped_walk pvmw = { .pfn = page_to_pfn(page), + .pgoff = page_pgoff(folio, page), .nr_pages = 1, .vma = vma, .flags = PVMW_SYNC, }; - address = vma_address(vma, page_pgoff(folio, page), 1); + address = vma_address(vma, pvmw.pgoff, 1); pvmw.address = address; if (pvmw.address == -EFAULT) goto out; if (!page_vma_mapped_walk(&pvmw)) return -EFAULT; + /* A smaller leaf can map the page after an unmapped leading fragment. */ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + address = pvmw.address; page_vma_mapped_walk_done(&pvmw); out: - /* Report this native page's byte address, not the covering leaf start. */ + /* Coarser leaves retain this native page's byte address within the leaf. */ return address; } #endif From b66b6e809c828fa6f33a12f209e8627742b8e167 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:17:39 -0400 Subject: [PATCH 0855/1579] mm: clear poison entries on subpage MADV_FREE Match the native MADV_FREE handling of hardware-poison and software poison markers. Clearing these non-swap entries must not change swap accounting. Extend the existing userfaultfd marker case to check MADV_FREE recovery and neighboring data. The suite passes for native/4K processes on 16K and native/4K/16K processes on 64K KASAN kernels. Hardware memory errors were not injected. Signed-off-by: Diljit Singh (cherry picked from commit f86041e8396472c60b6a4adf063c4d93b99082d5) --- mm/madvise.c | 3 +++ tools/testing/selftests/arm64/user4k/uffd-wp-init.c | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/mm/madvise.c b/mm/madvise.c index 1d15da9d1200d..e746437c6b1f3 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -690,6 +690,9 @@ static int madvise_free_subpage_range(pmd_t *pmd, unsigned long addr, pte_swp_subpage_offset(pte), true); pte_clear_not_present_full(mm, addr, ptep, tlb->fullmm); swapped--; + } else if (softleaf_is_hwpoison(entry) || + softleaf_is_poison_marker(entry)) { + pte_clear_not_present_full(mm, addr, ptep, tlb->fullmm); } continue; } diff --git a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c index 19747755c4c6c..f34ab8328721d 100644 --- a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c +++ b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c @@ -432,8 +432,12 @@ static void markers(size_t ps) CHECK(!sigaction(SIGBUS, &old, NULL)); p[17] = 0x81; p[2 * ps + 17] = 0x82; - CHECK(!close(fd) && !munmap(p, 3 * ps)); - printf("ok - uffd ABI %zuK poisoned leaf signals SIGBUS and preserves neighbours\n", + CHECK(!madvise(p + ps, ps, MADV_FREE)); + CHECK(!close(fd)); + CHECK(p[ps + 17] == 0); + CHECK(p[17] == 0x81 && p[2 * ps + 17] == 0x82); + CHECK(!munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK poison: SIGBUS, MADV_FREE recovery, neighbours\n", ps / 1024); } struct pager_request { From db14f30251da58b14ccf9f518a6b6c8fed9a11ec Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:17:40 -0400 Subject: [PATCH 0856/1579] arm64: retain the fragment pin test workers through cleanup A pin worker can exit on an error before cleanup calls kthread_stop(). Create it stopped, acquire a task reference, register cleanup, and then wake it. Release the reference after kthread_stop(), including the cleanup registration failure path. The existing retained-pin test passes on 16K and 64K KASAN kernels. No additional test fixture is introduced. Signed-off-by: Diljit Singh (cherry picked from commit e7d10fd228c70c65e932a002296d07a48a0bcdfd) --- arch/arm64/mm/user4k-test.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 42719000b5a1d..369eb0d459ee5 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -10406,6 +10406,7 @@ static void fragment_pin_worker_stop(void *arg) if (worker->task) { kthread_stop(worker->task); + put_task_struct(worker->task); worker->task = NULL; } } @@ -10433,13 +10434,16 @@ static void user4k_fragment_pin_race_test(struct kunit *test) for (i = 0; i < 2; i++) { workers[i].mm = vma->vm_mm; init_completion(&workers[i].ready); - workers[i].task = kthread_run(fragment_pin_worker_run, &workers[i], "user4k-pin-%u", i); + workers[i].task = kthread_create(fragment_pin_worker_run, &workers[i], + "user4k-pin-%u", i); if (IS_ERR(workers[i].task)) { workers[i].task = NULL; KUNIT_FAIL(test, "pin worker creation failed"); return; } + get_task_struct(workers[i].task); KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_pin_worker_stop, &workers[i]), 0); + wake_up_process(workers[i].task); KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); } for (round = 0; round < 32; round++) { From faa543fceb07d682d04f55fd9ab39ac8849c161b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:39:15 -0400 Subject: [PATCH 0857/1579] mm: preserve physical subpage offsets in batched PFN remaps Add an explicit physical offset to the PFN range mapping path. Carry it through the existing page-table batching and error cleanup, record it in COW mapping offsets, and track the complete covering native-page range. The existing remap_pfn_range() interface selects offset zero. Use the offset-aware range interface in the system dma-buf heap. Shorten the first scatterlist extent by the starting offset and reset that offset when advancing to the next extent. Physical identity remains independent of the target virtual address. Extend the existing PFN mapping test with noncongruent physical/virtual offsets. 16K/64K KASAN builds and tests pass. Locally adapted existing dma-buf heap tests also verify offset aliases across backing segments for native/4K/16K processes, including coherent writes. Native x86 PFN tracking and system-heap objects build. Physical DMA hardware remains unqualified. Signed-off-by: Diljit Singh (cherry picked from commit eadcabe164613222b98854a43e75b1b35d8c8e0a) --- arch/arm64/mm/user4k-test.c | 15 ++++-- drivers/dma-buf/heaps/system_heap.c | 10 ++-- include/linux/mm.h | 2 + mm/memory.c | 72 ++++++++++++++++++++--------- 4 files changed, 70 insertions(+), 29 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 369eb0d459ee5..887b2b599bc80 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -5650,6 +5650,7 @@ static void user4k_remap_pfn_range_test(struct kunit *test) for (unsigned int shape = 0; shape < ARRAY_SIZE(boundaries); shape++) { unsigned long start = boundaries[shape] - size; unsigned long length = PAGE_SIZE + size; + unsigned int physical_offset = native || !shape ? 0 : size; struct vm_area_struct *vma; int ret; @@ -5661,8 +5662,12 @@ static void user4k_remap_pfn_range_test(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); mmap_write_lock(mm); - ret = remap_pfn_range(vma, start, folio_pfn(folio), length, - vma->vm_page_prot); + if (physical_offset) + ret = remap_pfn_range_offset(vma, start, folio_pfn(folio), + physical_offset, length, vma->vm_page_prot); + else + ret = remap_pfn_range(vma, start, folio_pfn(folio), length, + vma->vm_page_prot); mmap_write_unlock(mm); KUNIT_ASSERT_EQ(test, ret, 0); for (unsigned long offset = 0; offset < length; offset += size) { @@ -5671,13 +5676,15 @@ static void user4k_remap_pfn_range_test(struct kunit *test) struct follow_pfnmap_args args = { .vma = vma, .address = start + offset + 31, }; - phys_addr_t expected = PFN_PHYS(folio_pfn(folio)) + offset + 31; + phys_addr_t expected = PFN_PHYS(folio_pfn(folio)) + + physical_offset + offset + 31; int ret; KUNIT_EXPECT_TRUE(test, pte_present(pte)); KUNIT_EXPECT_TRUE(test, pte_special(pte)); KUNIT_EXPECT_TRUE(test, pte_write(pte)); KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), - PFN_PHYS(folio_pfn(folio)) + offset); + PFN_PHYS(folio_pfn(folio)) + + physical_offset + offset); mmap_read_lock(mm); ret = follow_pfnmap_start(&args); diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c index 03c2b87cb1112..a1797ae903004 100644 --- a/drivers/dma-buf/heaps/system_heap.c +++ b/drivers/dma-buf/heaps/system_heap.c @@ -228,6 +228,7 @@ static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) struct sg_table *table = &buffer->sg_table; unsigned long addr = vma->vm_start; unsigned long pgoff = vma->vm_pgoff; + unsigned int offset = vma_subpage_offset(vma); struct scatterlist *sg; pgprot_t prot; int i, ret; @@ -247,17 +248,18 @@ static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) for (; sg && addr < vma->vm_end; sg = sg_next(sg)) { unsigned long n = (sg->length >> PAGE_SHIFT) - pgoff; struct page *page = sg_page(sg) + pgoff; - unsigned long size = n << PAGE_SHIFT; + unsigned long size = (n << PAGE_SHIFT) - offset; - if (addr + size > vma->vm_end) - size = vma->vm_end - addr; + size = min(size, vma->vm_end - addr); - ret = remap_pfn_range(vma, addr, page_to_pfn(page), size, prot); + ret = remap_pfn_range_offset(vma, addr, page_to_pfn(page), offset, + size, prot); if (ret) return ret; addr += size; pgoff = 0; + offset = 0; } return 0; diff --git a/include/linux/mm.h b/include/linux/mm.h index 9e621134c8fb1..a49f279b934ff 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4591,6 +4591,8 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, unsigned long addr); int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t pgprot); +int remap_pfn_range_offset(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot); int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, diff --git a/mm/memory.c b/mm/memory.c index dd2916612f9c7..e302129c5ba49 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3466,13 +3466,13 @@ static int get_remap_pgoff(bool is_cow, unsigned long addr, } static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { pgd_t *pgd; unsigned long next; struct mm_struct *mm = vma->vm_mm; unsigned long end = addr + mm_page_align(mm, size); - phys_addr_t phys = PFN_PHYS(pfn); + phys_addr_t phys = PFN_PHYS(pfn) + offset; int err; if (WARN_ON_ONCE(!IS_ALIGNED(addr, mm_page_size(mm)))) @@ -3500,9 +3500,9 @@ static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long ad * must have pre-validated the caching bits of the pgprot_t. */ static int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { - int error = remap_pfn_range_internal(vma, addr, pfn, size, prot); + int error = remap_pfn_range_internal(vma, addr, pfn, offset, size, prot); if (!error) return 0; @@ -3546,12 +3546,17 @@ void pfnmap_track_ctx_release(struct kref *ref) } static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { struct pfnmap_track_ctx *ctx = NULL; + unsigned long track_size; int err; size = mm_page_align(vma->vm_mm, size); + if (check_add_overflow(size, (unsigned long)offset, &track_size) || + track_size > ULONG_MAX - (PAGE_SIZE - 1)) + return -EINVAL; + track_size = PAGE_ALIGN(track_size); /* * If we cover the full VMA, we'll perform actual tracking, and @@ -3565,14 +3570,14 @@ static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, if (addr == vma->vm_start && addr + size == vma->vm_end) { if (vma->pfnmap_track_ctx) return -EINVAL; - ctx = pfnmap_track_ctx_alloc(pfn, size, &prot); + ctx = pfnmap_track_ctx_alloc(pfn, track_size, &prot); if (IS_ERR(ctx)) return PTR_ERR(ctx); - } else if (pfnmap_setup_cachemode(pfn, size, &prot)) { + } else if (pfnmap_setup_cachemode(pfn, track_size, &prot)) { return -EINVAL; } - err = remap_pfn_range_notrack(vma, addr, pfn, size, prot); + err = remap_pfn_range_notrack(vma, addr, pfn, offset, size, prot); if (ctx) { if (err) kref_put(&ctx->kref, pfnmap_track_ctx_release); @@ -3583,15 +3588,15 @@ static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, } static int do_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { - return remap_pfn_range_track(vma, addr, pfn, size, prot); + return remap_pfn_range_track(vma, addr, pfn, offset, size, prot); } #else static int do_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { - return remap_pfn_range_notrack(vma, addr, pfn, size, prot); + return remap_pfn_range_notrack(vma, addr, pfn, offset, size, prot); } #endif @@ -3618,7 +3623,7 @@ int remap_pfn_range_prepare(struct vm_area_desc *desc) static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, - unsigned long size) + unsigned int offset, unsigned long size) { const unsigned long end = addr + mm_page_align(vma->vm_mm, size); const bool is_cow = is_cow_mapping(vma->vm_flags); @@ -3629,10 +3634,41 @@ static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, if (err) return err; + if (is_cow) + vma_set_page_offset(vma, (struct vm_page_offset) { pfn, offset }); vma_set_flags_mask(vma, VMA_REMAP_FLAGS); return 0; } +/** + * remap_pfn_range_offset - remap a physical range starting within a native page + * @vma: user VMA to map to + * @addr: target address, aligned to the VMA's mapping granule + * @pfn: native page frame containing the first byte + * @offset: byte offset within @pfn, aligned to the mapping granule + * @size: size of the mapping, in bytes + * @prot: page protection flags + * + * The caller must hold the mmap lock. The physical offset is independent of + * the target virtual address. References and error cleanup match remap_pfn_range(). + * + * Return: 0 on success, a negative error code otherwise. + */ +int remap_pfn_range_offset(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) +{ + int err; + + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(vma->vm_mm))) + return -EINVAL; + err = remap_pfn_range_prepare_vma(vma, addr, pfn, offset, size); + if (err) + return err; + + return do_remap_pfn_range(vma, addr, pfn, offset, size, prot); +} +EXPORT_SYMBOL(remap_pfn_range_offset); + /** * remap_pfn_range - remap kernel memory to userspace * @vma: user vma to map to @@ -3648,13 +3684,7 @@ static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t prot) { - int err; - - err = remap_pfn_range_prepare_vma(vma, addr, pfn, size); - if (err) - return err; - - return do_remap_pfn_range(vma, addr, pfn, size, prot); + return remap_pfn_range_offset(vma, addr, pfn, 0, size, prot); } EXPORT_SYMBOL(remap_pfn_range); @@ -3666,7 +3696,7 @@ int remap_pfn_range_complete(struct vm_area_struct *vma, const unsigned long size = action->remap.size; const pgprot_t prot = action->remap.pgprot; - return do_remap_pfn_range(vma, start, pfn, size, prot); + return do_remap_pfn_range(vma, start, pfn, 0, size, prot); } static int __simple_ioremap_prep(unsigned long vm_len, pgoff_t vm_pgoff, From 9245cf76b2a82371e4dd5f6f8fc52b917be9224e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 08:00:36 -0400 Subject: [PATCH 0858/1579] mm: retain physical offsets through I/O mapping preparation Round resource coverage in the target MM's granule and combine the resource start with the VMA's complete byte offset. Validate lengths and offset arithmetic before publishing mappings. Native-page rounding is unchanged. Carry the resulting physical offset through both vm_iomap_memory() and mmap_prepare actions. Keep the native architecture PFN conversion and protection hooks, include the offset in their range, and preserve COW physical identity. The action field is absent without CONFIG_MM_SUBPAGE. Extend the existing PFN mapping test across native, 4K and supported 16K process geometries. Direct and descriptor I/O mappings check noncongruent offsets and reject insufficient resource ranges. 16K/64K KASAN builds and tests pass; native x86 mapping/action objects compile. Physical MMIO device access is not qualified by these RAM-backed fixtures. Signed-off-by: Diljit Singh (cherry picked from commit fa6b5cff34588782b063a10d40df4a4ecf882db8) --- arch/arm64/mm/user4k-test.c | 35 +++++++++++++++---- include/linux/mm.h | 30 +++++++++++++--- include/linux/mm_types.h | 3 ++ mm/internal.h | 6 +++- mm/memory.c | 70 +++++++++++++++++++++++-------------- 5 files changed, 106 insertions(+), 38 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 887b2b599bc80..89b2bb46f6dc2 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -5635,18 +5635,17 @@ static void user4k_remap_pfn_range_test(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, folio); KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); - for (unsigned int native = 0; native < 2; native++) { - struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); - unsigned long size, boundaries[3]; + for (unsigned int shift = 12; shift <= PAGE_SHIFT; shift += 2) { + struct mm_struct *mm = test_vma_mm_granule(test, shift); + bool native = shift == PAGE_SHIFT; + unsigned long size, boundaries[4]; KUNIT_ASSERT_NOT_NULL(test, mm); - if (native) - KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, - user4k_vma_mm_free, mm), 0); size = mm_page_size(mm); boundaries[0] = pmd_size_mm(mm); boundaries[1] = pud_size_mm(mm); boundaries[2] = pgd_size_mm(mm); + boundaries[3] = 2 * pmd_size_mm(mm); for (unsigned int shape = 0; shape < ARRAY_SIZE(boundaries); shape++) { unsigned long start = boundaries[shape] - size; unsigned long length = PAGE_SIZE + size; @@ -5662,7 +5661,29 @@ static void user4k_remap_pfn_range_test(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); mmap_write_lock(mm); - if (physical_offset) + if (shape >= 2) { + phys_addr_t base = PFN_PHYS(folio_pfn(folio)) + physical_offset; + unsigned long available = folio_size(folio) - physical_offset; + + vma_set_page_offset(vma, (struct vm_page_offset) { + .offset = physical_offset, + }); + if (shape == 2) { + KUNIT_EXPECT_EQ(test, + vm_iomap_memory(vma, base, PAGE_SIZE), -EINVAL); + ret = vm_iomap_memory(vma, base, available); + } else { + struct vm_area_desc desc; + + compat_set_desc_from_vma(&desc, NULL, vma); + mmap_action_simple_ioremap(&desc, base, PAGE_SIZE); + KUNIT_EXPECT_EQ(test, + __compat_vma_mmap(&desc, vma), -EINVAL); + mmap_action_simple_ioremap(&desc, base, available); + ret = __compat_vma_mmap(&desc, vma); + } + physical_offset *= 2; + } else if (physical_offset) ret = remap_pfn_range_offset(vma, start, folio_pfn(folio), physical_offset, length, vma->vm_page_prot); else diff --git a/include/linux/mm.h b/include/linux/mm.h index a49f279b934ff..86241653559ad 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4353,6 +4353,15 @@ static inline unsigned long vma_desc_pages(const struct vm_area_desc *desc) return vma_desc_size(desc) >> PAGE_SHIFT; } +static inline unsigned int mmap_action_remap_offset(const struct mmap_action *action) +{ +#ifdef CONFIG_MM_SUBPAGE + return action->remap.offset; +#else + return 0; +#endif +} + /** * mmap_action_remap - helper for mmap_prepare hook to specify that a pure PFN * remap is required. @@ -4378,6 +4387,9 @@ static inline void mmap_action_remap(struct vm_area_desc *desc, action->remap.start_pfn = start_pfn; action->remap.size = size; action->remap.pgprot = desc->page_prot; +#ifdef CONFIG_MM_SUBPAGE + action->remap.offset = 0; +#endif } /** @@ -4728,14 +4740,24 @@ static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, } #endif +static inline int io_remap_pfn_range_offset(struct vm_area_struct *vma, + unsigned long addr, unsigned long orig_pfn, unsigned int offset, + unsigned long size, pgprot_t orig_prot) +{ + const pgprot_t prot = pgprot_decrypted(orig_prot); + unsigned long pfn; + + if (size > ULONG_MAX - offset) + return -EINVAL; + pfn = io_remap_pfn_range_pfn(orig_pfn, size + offset); + return remap_pfn_range_offset(vma, addr, pfn, offset, size, prot); +} + static inline int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long orig_pfn, unsigned long size, pgprot_t orig_prot) { - const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size); - const pgprot_t prot = pgprot_decrypted(orig_prot); - - return remap_pfn_range(vma, addr, pfn, size, prot); + return io_remap_pfn_range_offset(vma, addr, orig_pfn, 0, size, orig_prot); } static inline vm_fault_t vmf_error(int err) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 46d6f13f6f303..180207618dbae 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -838,6 +838,9 @@ struct mmap_action { unsigned long start_pfn; unsigned long size; pgprot_t pgprot; +#ifdef CONFIG_MM_SUBPAGE + unsigned int offset; +#endif } remap; struct { phys_addr_t start_phys_addr; diff --git a/mm/internal.h b/mm/internal.h index 5ad4e57daedc6..bd953128ed6f1 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1923,9 +1923,13 @@ static inline int io_remap_pfn_range_prepare(struct vm_area_desc *desc) const unsigned long orig_pfn = action->remap.start_pfn; const pgprot_t orig_pgprot = action->remap.pgprot; const unsigned long size = action->remap.size; - const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size); + const unsigned int offset = mmap_action_remap_offset(action); + unsigned long pfn; int err; + if (size > ULONG_MAX - offset) + return -EINVAL; + pfn = io_remap_pfn_range_pfn(orig_pfn, size + offset); action->remap.start_pfn = pfn; action->remap.pgprot = pgprot_decrypted(orig_pgprot); err = remap_pfn_range_prepare(desc); diff --git a/mm/memory.c b/mm/memory.c index e302129c5ba49..8e58ea71b88c0 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3606,16 +3606,23 @@ int remap_pfn_range_prepare(struct vm_area_desc *desc) const unsigned long start = action->remap.start; const unsigned long end = start + action->remap.size; const unsigned long pfn = action->remap.start_pfn; + const unsigned int offset = mmap_action_remap_offset(action); const bool is_cow = vma_desc_is_cow_mapping(desc); int err; if (!range_in_vma_desc(desc, start, end)) return -EFAULT; + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(desc->mm))) + return -EINVAL; err = get_remap_pgoff(is_cow, start, end, desc->start, desc->end, pfn, &desc->pgoff); if (err) return err; +#ifdef CONFIG_MM_SUBPAGE + if (is_cow) + desc->subpage_offset = offset; +#endif vma_desc_set_flags_mask(desc, VMA_REMAP_FLAGS); return 0; @@ -3696,40 +3703,43 @@ int remap_pfn_range_complete(struct vm_area_struct *vma, const unsigned long size = action->remap.size; const pgprot_t prot = action->remap.pgprot; - return do_remap_pfn_range(vma, start, pfn, 0, size, prot); + return do_remap_pfn_range(vma, start, pfn, mmap_action_remap_offset(action), size, prot); } -static int __simple_ioremap_prep(unsigned long vm_len, pgoff_t vm_pgoff, +static int __simple_ioremap_prep(struct mm_struct *mm, unsigned long vm_len, + struct vm_page_offset pos, phys_addr_t start_phys, unsigned long size, - unsigned long *pfnp) + struct vm_page_offset *physical) { - unsigned long pfn, pages; + unsigned long granule = mm_page_size(mm); + unsigned long offset, bias = start_phys & (granule - 1); /* Check that the physical memory area passed in looks valid */ if (start_phys + size < start_phys) return -EINVAL; /* - * You *really* shouldn't map things that aren't page-aligned, - * but we've historically allowed it because IO memory might - * just have smaller alignment. + * Preserve the historical allowance for unaligned I/O resources, using + * the target MM's granule for the prefix and final partial mapping. */ - size += start_phys & ~PAGE_MASK; - pfn = start_phys >> PAGE_SHIFT; - pages = (size + ~PAGE_MASK) >> PAGE_SHIFT; - if (pfn + pages < pfn) + if (check_add_overflow(size, bias, &size) || size > ULONG_MAX - (granule - 1)) return -EINVAL; - - /* We start the mapping 'vm_pgoff' pages into the area */ - if (vm_pgoff > pages) + size = ALIGN(size, granule); + start_phys -= bias; + if (start_phys + size < start_phys) + return -EINVAL; + if (pos.offset >= PAGE_SIZE || !IS_ALIGNED(pos.offset, granule) || + check_mul_overflow(pos.index, PAGE_SIZE, &offset) || + check_add_overflow(offset, (unsigned long)pos.offset, &offset)) return -EINVAL; - pfn += vm_pgoff; - pages -= vm_pgoff; - /* Can we fit all of the mapping? */ - if ((vm_len >> PAGE_SHIFT) + !!offset_in_page(vm_len) > pages) + if (offset > size || vm_len > size - offset) return -EINVAL; - *pfnp = pfn; + start_phys += offset; + physical->index = PHYS_PFN(start_phys); + physical->offset = offset_in_page(start_phys); + if (PFN_PHYS(physical->index) + physical->offset != start_phys) + return -EINVAL; return 0; } @@ -3738,16 +3748,22 @@ int simple_ioremap_prepare(struct vm_area_desc *desc) struct mmap_action *action = &desc->action; const phys_addr_t start = action->simple_ioremap.start_phys_addr; const unsigned long size = action->simple_ioremap.size; - unsigned long pfn; + struct vm_page_offset physical, pos = { .index = desc->pgoff }; int err; - err = __simple_ioremap_prep(vma_desc_size(desc), desc->pgoff, - start, size, &pfn); +#ifdef CONFIG_MM_SUBPAGE + pos.offset = desc->subpage_offset; +#endif + err = __simple_ioremap_prep(desc->mm, vma_desc_size(desc), pos, + start, size, &physical); if (err) return err; /* The I/O remap logic does the heavy lifting. */ - mmap_action_ioremap_full(desc, pfn); + mmap_action_ioremap_full(desc, physical.index); +#ifdef CONFIG_MM_SUBPAGE + action->remap.offset = physical.offset; +#endif return io_remap_pfn_range_prepare(desc); } @@ -3771,15 +3787,17 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long const unsigned long vm_start = vma->vm_start; const unsigned long vm_end = vma->vm_end; const unsigned long vm_len = vm_end - vm_start; - unsigned long pfn; + struct vm_page_offset physical; int err; - err = __simple_ioremap_prep(vm_len, vma->vm_pgoff, start, len, &pfn); + err = __simple_ioremap_prep(vma->vm_mm, vm_len, vma_page_offset_at(vma, vm_start), + start, len, &physical); if (err) return err; /* Ok, let it rip */ - return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot); + return io_remap_pfn_range_offset(vma, vm_start, physical.index, physical.offset, + vm_len, vma->vm_page_prot); } EXPORT_SYMBOL(vm_iomap_memory); From e5c8028e5c777e89442929c82c04ff95bf2e4cc1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 08:05:14 -0400 Subject: [PATCH 0859/1579] mm: preserve the PFN remap interface on no-MMU kernels Provide a zero-offset wrapper for remap_pfn_range_offset() without an MMU. It retains the existing identity-mapping checks and rejects offsets that this configuration cannot represent. MMU kernels keep the range mapper. ARM MPS2 no-MMU builds of mm/nommu.o and the system heap pass; their symbols contain no unresolved offset-remap reference. Native x86 mapping and action objects also build. No no-MMU runtime test was performed. Signed-off-by: Diljit Singh (cherry picked from commit 207036cd20c2f5b43958146873fdba2d0d1d6b3b) --- include/linux/mm.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/linux/mm.h b/include/linux/mm.h index 86241653559ad..0536bb678c2e9 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4603,8 +4603,18 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, unsigned long addr); int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t pgprot); +#ifdef CONFIG_MMU int remap_pfn_range_offset(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot); +#else +static inline int remap_pfn_range_offset(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) +{ + if (offset) + return -EINVAL; + return remap_pfn_range(vma, addr, pfn, size, prot); +} +#endif int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, From ac1720fb87e970b8b8db3eec48c5f50c431749a3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 08:13:02 -0400 Subject: [PATCH 0860/1579] userfaultfd: update published folio flags atomically Shared shmem backing and neighboring retained slots can already expose the folio while a fill operation updates it. Holding the folio lock and PTL does not exclude other atomic changes to the folio flags word. Use folio_mark_uptodate() for both published-backing paths, preserving the write barrier while avoiding a non-atomic flags update. Keep private allocation initialization unchanged. 16K/64K KASAN builds and the existing userfaultfd contract suite pass for native/4K and supported 16K processes. No new fixture was added and the specific flag-word race was not forced at runtime. Signed-off-by: Diljit Singh (cherry picked from commit f2928b082c7a0e6e5b7588180daebce0f1efbea8) --- mm/userfaultfd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 8d72d991b05e5..e7327a6749962 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -438,7 +438,7 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, } kunmap_local(dst); flush_dcache_folio(folio); - __folio_mark_uptodate(folio); + folio_mark_uptodate(folio); folio_mark_dirty(folio); } #endif @@ -813,7 +813,7 @@ static int mfill_atomic_subpage(struct mfill_state *state) } } /* The allocator zeroed every byte, including unissued neighbours. */ - __folio_mark_uptodate(folio); + folio_mark_uptodate(folio); vma = state->vma; pte = pte_offset_map_lock(mm, state->pmd, state->dst_addr, &ptl); err = -EAGAIN; From 86ccb92f789b7095570db8460c228b9e716f413d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 08:48:13 -0400 Subject: [PATCH 0861/1579] tomoyo: inspect exec arguments through virtual-byte access Argument staging can use a smaller userspace granule, where legacy page-array GUP cannot represent the backing. Read the requested virtual bytes through access_remote_vm(), retaining the existing native-page buffer and caller iteration units. Keep the no-MMU page-array path. Cache the first copied virtual position instead of a backing-page pointer. Refill on a backward access outside the cached tail and invalidate before a potentially partial copy. Multiple fragments may contribute to one cached native-sized virtual range. Local extensions of existing exec tests verify argument bytes and cache rewinds on 16K/64K KASAN kernels for all supported process granules. Native x86 and ARM no-MMU TOMOYO objects compile. The temporary test hooks are not submitted; full policy enforcement was not exercised. Signed-off-by: Diljit Singh (cherry picked from commit 3001cf4667e728ce38f3453aa98e5995af1ca098) --- security/tomoyo/common.h | 4 ++-- security/tomoyo/domain.c | 38 +++++++++++++++----------------------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/security/tomoyo/common.h b/security/tomoyo/common.h index d098cf8aae610..b859f099ae9ea 100644 --- a/security/tomoyo/common.h +++ b/security/tomoyo/common.h @@ -575,8 +575,8 @@ struct tomoyo_mini_stat { /* Structure for dumping argv[] and envp[] of "struct linux_binprm". */ struct tomoyo_page_dump { - struct page *page; /* Previously dumped page. */ - char *data; /* Contents of "page". Size is PAGE_SIZE. */ + unsigned long pos; /* First cached virtual byte, or ULONG_MAX. */ + char *data; /* Buffer for one native page of virtual bytes. */ }; /* Structure for attribute checks in addition to pathname checks. */ diff --git a/security/tomoyo/domain.c b/security/tomoyo/domain.c index eeaad421f15ba..2ff8d783f54cc 100644 --- a/security/tomoyo/domain.c +++ b/security/tomoyo/domain.c @@ -906,45 +906,37 @@ int tomoyo_find_next_domain(struct linux_binprm *bprm) bool tomoyo_dump_page(struct linux_binprm *bprm, unsigned long pos, struct tomoyo_page_dump *dump) { - struct page *page; -#ifdef CONFIG_MMU - int ret; -#endif + const unsigned int offset = offset_in_page(pos); + const unsigned int length = PAGE_SIZE - offset; /* dump->data is released by tomoyo_find_next_domain(). */ if (!dump->data) { dump->data = kzalloc(PAGE_SIZE, GFP_NOFS); if (!dump->data) return false; + dump->pos = ULONG_MAX; } - /* Same with get_arg_page(bprm, pos, 0) in fs/exec.c */ + if (pos >= dump->pos && (pos & PAGE_MASK) == (dump->pos & PAGE_MASK)) + return true; + /* A failed partial copy must not leave the previous cache valid. */ + dump->pos = ULONG_MAX; #ifdef CONFIG_MMU /* - * This is called at execve() time in order to dig around - * in the argv/environment of the new process - * (represented by bprm). + * Inspect the nascent MM's virtual bytes. Smaller userspace mappings + * can use several independently retained fragments within this range. */ - mmap_read_lock(bprm->mm); - ret = get_user_pages_remote(bprm->mm, pos, 1, - FOLL_FORCE, &page, NULL); - mmap_read_unlock(bprm->mm); - if (ret <= 0) + if (access_remote_vm(bprm->mm, pos, dump->data + offset, length, + FOLL_FORCE) != length) return false; #else - page = bprm->page[pos / PAGE_SIZE]; -#endif - if (page != dump->page) { - const unsigned int offset = pos % PAGE_SIZE; + { + struct page *page = bprm->page[pos / PAGE_SIZE]; char *kaddr = kmap_local_page(page); - dump->page = page; - memcpy(dump->data + offset, kaddr + offset, - PAGE_SIZE - offset); + memcpy(dump->data + offset, kaddr + offset, length); kunmap_local(kaddr); } - /* Same with put_arg_page(page) in fs/exec.c */ -#ifdef CONFIG_MMU - put_page(page); #endif + dump->pos = pos; return true; } From 2843ef2c9df87fb25dd3d2e0fbe2d2ddbb3a02d0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 08:48:13 -0400 Subject: [PATCH 0862/1579] exec: retain the selected user namespace when replacing the staging mm would_dump() can refine the staging MM's namespace for the interpreter before final ELF geometry selection. A replacement MM allocated from the current credentials must not discard that earlier policy decision. Transfer a namespace reference from the old MM after preparation succeeds, before publishing the replacement. Failed preparation leaves the old MM and namespace untouched. Local extensions of existing exec tests verify retention of a distinct namespace during granule changes on 16K/64K KASAN kernels. Native x86 and ARM no-MMU exec objects build. No privileged-exec experiment was performed. Signed-off-by: Diljit Singh (cherry picked from commit a8100f86f408900dc7f2a75c0b0f62a1b3021d8f) --- fs/exec.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/exec.c b/fs/exec.c index cb5cda32f8e30..c9e6ecf187b7b 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -439,6 +439,8 @@ int bprm_set_page_shift(struct linux_binprm *bprm, unsigned int page_shift) } old_mm = bprm->mm; + put_user_ns(replacement.mm->user_ns); + replacement.mm->user_ns = get_user_ns(old_mm->user_ns); acct_arg_size(bprm, 0); bprm->mm = replacement.mm; bprm->vma = replacement.vma; From a6bcbd28b959dde1e39d9f9103bf410b83a6d823 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 09:37:17 -0400 Subject: [PATCH 0863/1579] block: estimate user vectors with the process fragment size Fragment pinning emits records in mm_user_fragment_size() intervals. Count the same intervals when reserving vectors, including unaligned segments and saturation at the caller's limit. Fixed 4K counting otherwise overestimates 16K userspace buffers on a 64K kernel, affecting allocation and direct-I/O path selection. Local extensions of the existing BIO tests cover native and smaller granules on 16K/64K KASAN kernels. The native x86 BIO object builds. Signed-off-by: Diljit Singh (cherry picked from commit e139642c1fac2cfce399d3b6cd8d229fbe7cf93c) --- block/bio.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/block/bio.c b/block/bio.c index 846417dec7d16..37fed25556398 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1247,10 +1247,12 @@ static bool bio_subpage_iter(const struct iov_iter *iter) int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs) { struct iov_iter walk = *iter; + unsigned long size; unsigned int count = 0; if (!bio_subpage_iter(iter)) return iov_iter_npages(iter, max_vecs); + size = mm_user_fragment_size(current->mm); while (iov_iter_count(&walk) && count < max_vecs) { size_t len, offset, nr; @@ -1258,9 +1260,8 @@ int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs) len = min(iter_iov_len(&walk), iov_iter_count(&walk)); if (WARN_ON_ONCE(!len)) return max_vecs; - offset = (unsigned long)iter_iov_addr(&walk) & (MM_SUBPAGE_SIZE - 1); - nr = (len >> MM_SUBPAGE_SHIFT) + - DIV_ROUND_UP(offset + (len & (MM_SUBPAGE_SIZE - 1)), MM_SUBPAGE_SIZE); + offset = (unsigned long)iter_iov_addr(&walk) & (size - 1); + nr = len / size + DIV_ROUND_UP(offset + (len & (size - 1)), size); if (nr >= max_vecs - count) return max_vecs; count += nr; From 33d0c9789f30ca4196278be12a83a028d02615ca Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 09:37:17 -0400 Subject: [PATCH 0864/1579] block: preserve nofault when pinning user fragments Pass FOLL_NOFAULT when the iterator forbids faulting, matching ordinary iterator extraction. Filesystems set this flag while holding locks that must not be reacquired through a user-buffer fault. Keep existing partial-progress and cleanup behavior. Local extensions of the existing BIO tests check missing and resident mappings with nofault set on 16K/64K KASAN kernels. Filesystem-specific locking paths have not been exercised. Signed-off-by: Diljit Singh (cherry picked from commit 1de4d3c0c6bdd60c98f992aa259ab3ed1eb4c397) --- block/bio.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/block/bio.c b/block/bio.c index 37fed25556398..19805b79cf0b2 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1283,6 +1283,8 @@ static int bio_iov_iter_get_fragments(struct bio *bio, struct iov_iter *iter, if (WARN_ON_ONCE(bio->bi_vcnt || bio->bi_user_fragments || !capacity)) return -EINVAL; + if (iter->nofault) + flags |= FOLL_NOFAULT; bio->bi_user_fragments = kmalloc_array(capacity, sizeof(*bio->bi_user_fragments), GFP_KERNEL); if (!bio->bi_user_fragments) From d0b4b3f943e5d9f50ed159e6a091c649b3704b4a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:02:10 -0400 Subject: [PATCH 0865/1579] net: allocate a new stream zerocopy owner after subpage splice A queued TCP skb can carry retained subpage references in its ubuf_info. A subsequent MSG_ZEROCOPY send cannot extend that owner as a message completion record. Allocate a fresh record for stream sockets and let the existing incompatible-owner handling select another skb, as it already does when a completion record reaches its size limit. Keep the datagram behavior unchanged. Mixed-granule pipe producers and native TCP consumers passed both send orders on IPv4/IPv6 loopback in 16K/64K KASAN guests, including completion notifications. Signed-off-by: Diljit Singh (cherry picked from commit 4043902ee814ee5c94e1bd9c97f35dd81493a743) --- net/core/skbuff.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 0bc6467aae5d4..91ee22a7c6cba 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1767,8 +1767,11 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, u32 bytelen, next; /* there might be non MSG_ZEROCOPY users */ - if (uarg->ops != &msg_zerocopy_ubuf_ops) + if (uarg->ops != &msg_zerocopy_ubuf_ops) { + if (sk->sk_type == SOCK_STREAM) + goto new_alloc; return NULL; + } /* realloc only when socket is locked (TCP, UDP cork), * so uarg->len and sk_zckey access is serialized From 061ef42edb4843754c0fd5a42c411eabaadd84b0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:02:10 -0400 Subject: [PATCH 0866/1579] tcp: start a new segment for incompatible splice ownership Splicing a retained subpage into a queued MSG_ZEROCOPY skb needs its own fragment owner. Report this conflict as EEXIST and retry on a new TCP segment, matching the reverse zerocopy transition. Preserve the iterator position and the existing owner's completion references. IPv4/IPv6 loopback checks cover both operation orders, exact bytes and completion notifications on 16K/64K KASAN guests. Existing SKB clone, trim and retained-reference tests also pass. No NIC throughput claim is made by these checks. Signed-off-by: Diljit Singh (cherry picked from commit f22a44fa973b4b96923246686d8d277ef13f8785) --- net/core/skbuff.c | 5 +++-- net/ipv4/tcp.c | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 91ee22a7c6cba..090a26ed9f2db 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -7427,7 +7427,7 @@ static int skb_splice_hold_subpages(struct sk_buff *skb, struct page *page, return 0; mm_subpage_put(probe); if (old && (skb_zcopy_is_nouarg(skb) || old->ops != &skb_subpage_ops)) - return -EOPNOTSUPP; + return -EEXIST; refs = old ? container_of(old, struct skb_subpage_refs, ubuf) : NULL; if (!refs || refcount_read(&old->refcnt) != 1) { new = kzalloc(sizeof(*new), GFP_ATOMIC); @@ -7481,7 +7481,8 @@ static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, * pages). * * Returns the amount of data spliced/copied or -EMSGSIZE if there's - * insufficient space in the buffer to transfer anything. + * insufficient space in the buffer to transfer anything, or -EEXIST if the + * buffer already has an incompatible fragment owner. */ ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, ssize_t maxsize) diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 389a7cc17110d..05c8c9504bb3c 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1356,7 +1356,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) err = skb_splice_from_iter(skb, &msg->msg_iter, copy); if (err < 0) { - if (err == -EMSGSIZE) { + if (err == -EMSGSIZE || err == -EEXIST) { tcp_mark_push(tp, skb); goto new_segment; } From e3983bfa520ccee76e987007655624c4001b954e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:02:10 -0400 Subject: [PATCH 0867/1579] selftests/arm64: derive splice fixture sizes from runtime limits The splice fixture assumes a 16K native pipe page, a corked payload that fits below TCP's MSS, and a process page smaller than a UDP datagram. Those assumptions fail on a 64K kernel even for native processes. Discover the native pipe allocation unit, size the test mapping to hold it, and bound corked TCP and UDP payloads by their protocol limits. Retain the data, ownership, teardown and capacity assertions. The same existing fixture now passes its 148 checks on both 16K and 64K kernels. Signed-off-by: Diljit Singh (cherry picked from commit 260567108121b13af59df7430fb2137b6558e2cb) --- .../arm64/user4k/splice-fixture-init.c | 50 +++++++++++++------ 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/arm64/user4k/splice-fixture-init.c b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c index 282ba6268b25d..bac72da15f40e 100644 --- a/tools/testing/selftests/arm64/user4k/splice-fixture-init.c +++ b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c @@ -336,8 +336,12 @@ static void tls_checks(size_t ps, unsigned char *p, size_t len, static int exercise(void) { - const size_t ps = getauxval(AT_PAGESZ), len = 8 * ps; - unsigned char *p, *back = malloc(len); + const size_t ps = getauxval(AT_PAGESZ); + size_t len; + unsigned char *p, *back; + int native; + unsigned int mss = 0; + socklen_t mss_len = sizeof(mss); int fds[2], dup[2], file, sockets[2]; struct iovec v[3]; struct stat self, init; @@ -348,9 +352,17 @@ static int exercise(void) if (stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || self.st_dev != init.st_dev || self.st_ino != init.st_ino) return 2; + if (pipe(fds) || pipe(dup)) + ksft_exit_fail_msg("pipe setup\n"); + /* A pipe has at least one native page, independently of this MM. */ + native = fcntl(fds[1], F_SETPIPE_SZ, 1); + if (native <= 0 || fcntl(fds[1], F_SETPIPE_SZ, 16 * native) < 0) + ksft_exit_fail_msg("pipe capacity setup\n"); + len = 8 * ps > (size_t)native ? 8 * ps : (size_t)native; + back = malloc(len); p = mmap((void *)0x300000000UL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); - if (p == MAP_FAILED || !back || pipe(fds) || pipe(dup)) + if (p == MAP_FAILED || !back) ksft_exit_fail_msg("setup\n"); ksft_print_header(); ksft_set_plan(use_tcp ? 22 : 10); @@ -370,21 +382,21 @@ static int exercise(void) ok = read_exact(fds[0], back, result) && check(back, ps - 17, 17) && check(back + ps - 17, 2 * ps - 31, 2 * ps + 31); report(ok, ps, "unaligned fragmented iovecs and empty segment"); - v[0] = (struct iovec){ p, 16384 }; - ok = fcntl(fds[1], F_SETPIPE_SZ, 16384) == 16384; + v[0] = (struct iovec){ p, native }; + ok = fcntl(fds[1], F_SETPIPE_SZ, native) == native; result = vmsplice(fds[1], v, 1, SPLICE_F_NONBLOCK); { int pending = 0; - ok &= result == 16384 && !ioctl(fds[0], FIONREAD, &pending) && - pending == 16384; + ok &= result == native && !ioctl(fds[0], FIONREAD, &pending) && + pending == native; } if (result > 0) ok &= read_exact(fds[0], back, result) && check(back, result, 0); report(ok, ps, - "contiguous quarters retain one-native-page pipe capacity"); - fcntl(fds[1], F_SETPIPE_SZ, 16 * 16384); + "contiguous fragments retain one-native-page pipe capacity"); + fcntl(fds[1], F_SETPIPE_SZ, 16 * native); v[0] = (struct iovec){ p + 11, 3 * ps - 11 }; result = vmsplice(fds[1], v, 1, 0); if (result != (ssize_t)(3 * ps - 11) || @@ -422,14 +434,22 @@ static int exercise(void) ksft_exit_fail_msg("socketpair errno=%d\n", errno); setsockopt(sockets[1], SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + if (use_tcp && + (getsockopt(sockets[0], IPPROTO_TCP, TCP_MAXSEG, &mss, &mss_len) || mss <= 1)) + ksft_exit_fail_msg("TCP_MAXSEG setup\n"); for (unsigned int mode = 0; mode < 3; mode++) { - const size_t bytes = mode == 1 ? ps : + size_t bytes = mode == 1 ? ps : (use_udp && ps == 4096 ? 6 * ps - 256 : 2 * ps - 256); const size_t offset = mode == 1 ? 0 : ps + 29; int cork = 1; + /* Keep the corked payload below one segment; UDP has a byte limit. */ + if (use_tcp && mode < 2 && bytes >= mss) + bytes = mss - 1; + if (use_udp && bytes > 65507) + bytes = 65507; if ((use_tcp || use_udp) && mode < 2 && setsockopt(sockets[0], use_tcp ? IPPROTO_TCP : IPPROTO_UDP, use_tcp ? TCP_CORK : UDP_CORK, &cork, @@ -497,12 +517,14 @@ static int exercise(void) close(sockets[1]); ok = true; for (unsigned int round = 0; round < 128 && ok; round++) { + const size_t bytes = use_udp && ps > 65507 ? 65507 : ps; + fill(p, len); ok = !connected_pair(sockets); - v[0] = (struct iovec){ p + ps, ps }; - ok &= vmsplice(fds[1], v, 1, 0) == (ssize_t)ps && - splice(fds[0], NULL, sockets[0], NULL, ps, 0) == - (ssize_t)ps; + v[0] = (struct iovec){ p + ps, bytes }; + ok &= vmsplice(fds[1], v, 1, 0) == (ssize_t)bytes && + splice(fds[0], NULL, sockets[0], NULL, bytes, 0) == + (ssize_t)bytes; close(sockets[1]); close(sockets[0]); ok &= !madvise(p + ps, ps, MADV_DONTNEED); From cb9b70b2a128d6694b916f9eacbaca98d572420a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 12:24:35 -0400 Subject: [PATCH 0868/1579] arm64: align mixed-granule KUnit cases with migration dependencies The migration guard accidentally encloses the VMA split and brk tests, but their registrations are unconditional. Keep those definitions available without migration and guard the reclaim cases which explicitly exercise migration. Guard the userfaultfd MOVE/migration fixture and its registration too; a migration-disabled build exposed its unguarded reference to the migration allocator. The existing KUnit object builds on 16K and 64K configurations with migration both enabled and disabled. No new tests are added. Signed-off-by: Diljit Singh (cherry picked from commit 76088e05aa1e391fdcf2d19944c2bf2f2412e3a7) --- arch/arm64/mm/user4k-test.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 89b2bb46f6dc2..38a6897e090e2 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -9387,6 +9387,7 @@ static long test_madvise(unsigned long addr, unsigned long size, unsigned long b return __arm64_sys_madvise(®s); } +#endif static void user_granule_large_vma_split_test(struct kunit *test) { @@ -9475,6 +9476,7 @@ static void user_granule_brk_extent_test(struct kunit *test) } } +#ifdef CONFIG_MIGRATION static void user4k_droppable_reclaim_test(struct kunit *test) { unsigned int mode; @@ -11708,6 +11710,7 @@ static ssize_t test_uffd_move(struct vm_area_struct *dst, unsigned long dst_addr return ret; } +#ifdef CONFIG_MIGRATION struct move_migration_worker { struct userfaultfd_ctx ctx; struct vm_area_struct *vma[2]; @@ -11863,6 +11866,8 @@ static void user4k_move_migration_race_test(struct kunit *test) } } +#endif + static void user4k_move_swap_get_test(struct kunit *test) { unsigned int variant; @@ -13420,7 +13425,9 @@ static struct kunit_suite user4k_shmem_swap_suite = { static struct kunit_case user4k_nonlinear_swap_cases[] = { #ifdef CONFIG_USERFAULTFD KUNIT_CASE(user4k_move_swap_get_test), +#ifdef CONFIG_MIGRATION KUNIT_CASE(user4k_move_migration_race_test), +#endif #endif KUNIT_CASE(user4k_nonlinear_swap_cached_test), KUNIT_CASE(user4k_nonlinear_swap_disk_test), @@ -14214,8 +14221,10 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_mlock_repair_test), KUNIT_CASE(user_granule_large_vma_split_test), KUNIT_CASE(user_granule_brk_extent_test), +#ifdef CONFIG_MIGRATION KUNIT_CASE(user4k_droppable_reclaim_test), KUNIT_CASE(user4k_lazyfree_test), +#endif KUNIT_CASE(user4k_zero_cow_test), KUNIT_CASE(user16k_zero_cow_test), KUNIT_CASE(user4k_shmem_swap_usage_test), From 8ec095d57d3b5685aaf07550560070d4e8c853ec Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 13:40:54 -0400 Subject: [PATCH 0869/1579] io_uring: retain shared fragment charges through buffer teardown - Track fragment-backed registrations in a refcounted account for each user/MM counter pair. Charge each backing folio once and release its charge with the final retained fragment, including cloned buffers. - Keep the account alive independently of the originating ring. Roll back only the references acquired by failed preparation. - Serialize account updates locally; cache the registry lookup per ring. Preserve existing ring-context member offsets and keep native backing owners in their separate accounting path. - Validate with 16K/64K KASAN and lockdep builds, 110 guest lifecycle checks, and an x86 io_uring build. Warmed emulated registration measurements do not reproduce the earlier native timing discrepancy; hardware overhead and total metadata costs remain unqualified. Signed-off-by: Diljit Singh (cherry picked from commit d90fab78cd1b2044701c39b8593891a9b514b05f) --- include/linux/io_uring_types.h | 3 + io_uring/rsrc.c | 161 ++++++++++++++++++++++++++++++--- 2 files changed, 151 insertions(+), 13 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index f61b6cad229b4..e29fcd83060eb 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -512,6 +512,9 @@ struct io_ring_ctx { struct io_mapped_region ring_region; /* used for optimised request parameter and wait argument passing */ struct io_mapped_region param_region; +#ifdef CONFIG_MM_SUBPAGE + struct io_fragment_account *fragment_account; +#endif }; /* diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 33cc1e2688480..b442d5cf6c7f5 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,10 @@ struct io_rsrc_update { static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, struct page **last_hpage); +#ifdef CONFIG_MM_SUBPAGE +static void io_fragment_account_put(struct io_fragment_account *account); +#endif + /* only define max */ #define IORING_MAX_FIXED_FILES (1U << 20) #define IORING_MAX_REG_BUFFERS (1U << 14) @@ -168,6 +173,10 @@ bool io_rsrc_cache_init(struct io_ring_ctx *ctx) void io_rsrc_cache_free(struct io_ring_ctx *ctx) { +#ifdef CONFIG_MM_SUBPAGE + io_fragment_account_put(ctx->fragment_account); + ctx->fragment_account = NULL; +#endif io_alloc_cache_free(&ctx->node_cache, kfree); io_alloc_cache_free(&ctx->imu_cache, kvfree); } @@ -635,6 +644,9 @@ static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages, if (!node) continue; imu = node->buf; + /* Other backing owners have independent charge lifetimes. */ + if (imu->release != io_release_ubuf) + continue; for (j = 0; j < imu->nr_bvecs; j++) { if (!PageCompound(imu->bvec[j].bv_page)) continue; @@ -764,14 +776,145 @@ bool io_check_coalesce_buffer(struct page **page_array, int nr_pages, } #ifdef CONFIG_MM_SUBPAGE +/* + * Shared by rings charging the same counters, including cloned buffers. + * uring_lock nests outside lock; the registry lock is never held with lock. + * Long-term pins keep each indexed folio and its size stable until release. + */ +struct io_fragment_account { + struct hlist_node node; + refcount_t refs; + struct mutex lock; + struct xarray folios; + struct user_struct *user; + struct mm_struct *mm; +}; + +static DEFINE_HASHTABLE(io_fragment_accounts, 6); +static DEFINE_MUTEX(io_fragment_accounts_lock); + +static void io_fragment_account_put(struct io_fragment_account *account) +{ + if (!account || !refcount_dec_and_mutex_lock(&account->refs, + &io_fragment_accounts_lock)) + return; + hash_del(&account->node); + mutex_unlock(&io_fragment_accounts_lock); + WARN_ON_ONCE(!xa_empty(&account->folios)); + xa_destroy(&account->folios); + if (account->mm) + mmdrop(account->mm); + if (account->user) + free_uid(account->user); + kfree(account); +} + +static struct io_fragment_account *io_fragment_account_get(struct io_ring_ctx *ctx) +{ + unsigned long key = (unsigned long)ctx->mm_account ^ (unsigned long)ctx->user; + struct io_fragment_account *account; + + lockdep_assert_held(&ctx->uring_lock); + account = ctx->fragment_account; + if (account) + goto found; + + mutex_lock(&io_fragment_accounts_lock); + hash_for_each_possible(io_fragment_accounts, account, node, key) { + if (account->user == ctx->user && account->mm == ctx->mm_account) { + refcount_inc(&account->refs); + goto install; + } + } + account = kzalloc(sizeof(*account), GFP_KERNEL_ACCOUNT); + if (!account) { + mutex_unlock(&io_fragment_accounts_lock); + return NULL; + } + refcount_set(&account->refs, 1); + mutex_init(&account->lock); + xa_init(&account->folios); + account->user = ctx->user ? get_uid(ctx->user) : NULL; + account->mm = ctx->mm_account; + if (account->mm) + mmgrab(account->mm); + hash_add(io_fragment_accounts, &account->node, key); +install: + ctx->fragment_account = account; + mutex_unlock(&io_fragment_accounts_lock); +found: + /* The buffer retains its account after the originating ring exits. */ + refcount_inc(&account->refs); + return account; +} + struct io_fragment_buffer { unsigned int count; + unsigned int accounted; + struct io_fragment_account *account; struct user_page_fragment fragments[]; }; +static int io_account_fragment_buffer(struct io_fragment_buffer *buffer) +{ + struct io_fragment_account *account = buffer->account; + unsigned int i; + int ret = 0; + + mutex_lock(&account->lock); + for (i = 0; i < buffer->count; i++) { + struct folio *folio = buffer->fragments[i].folio; + unsigned long pfn = folio_pfn(folio); + void *entry = xa_load(&account->folios, pfn); + unsigned long refs = entry ? xa_to_value(entry) : 0; + + if (refs == LONG_MAX) { + ret = -EOVERFLOW; + break; + } + ret = xa_err(xa_store(&account->folios, pfn, xa_mk_value(refs + 1), + GFP_KERNEL_ACCOUNT)); + if (ret) + break; + if (!refs) { + ret = io_account_mem(account->user, account->mm, + folio_nr_pages(folio)); + if (ret) { + xa_erase(&account->folios, pfn); + break; + } + } + buffer->accounted++; + } + mutex_unlock(&account->lock); + return ret; +} + static void io_release_fragment_buffer(void *priv) { struct io_fragment_buffer *buffer = priv; + struct io_fragment_account *account = buffer->account; + unsigned int i; + + if (account) { + mutex_lock(&account->lock); + for (i = 0; i < buffer->accounted; i++) { + struct folio *folio = buffer->fragments[i].folio; + unsigned long pfn = folio_pfn(folio); + unsigned long refs = xa_to_value(xa_load(&account->folios, pfn)); + + if (refs == 1) { + xa_erase(&account->folios, pfn); + io_unaccount_mem(account->user, account->mm, + folio_nr_pages(folio)); + } else { + xa_store(&account->folios, pfn, xa_mk_value(refs - 1), + GFP_NOWAIT); + } + } + mutex_unlock(&account->lock); + io_fragment_account_put(account); + } release_user_fragments(buffer->fragments, buffer->count, false); kvfree(buffer); @@ -786,9 +929,7 @@ static struct io_rsrc_node *io_register_fragment_buffer(struct io_ring_ctx *ctx, struct io_fragment_buffer *buffer; struct io_mapped_ubuf *imu = NULL; struct io_rsrc_node *node; - unsigned long charged = 0; unsigned int i; - struct xarray seen; long nr; int ret = -ENOMEM; @@ -799,6 +940,8 @@ static struct io_rsrc_node *io_register_fragment_buffer(struct io_ring_ctx *ctx, if (!buffer) goto free_node; buffer->count = 0; + buffer->accounted = 0; + buffer->account = NULL; nr = pin_user_fragments_remote(current->mm, (unsigned long)iov->iov_base, iov->iov_len, FOLL_WRITE | FOLL_LONGTERM, buffer->fragments, count); if (nr > 0) @@ -812,26 +955,18 @@ static struct io_rsrc_node *io_register_fragment_buffer(struct io_ring_ctx *ctx, goto free_buffer; imu->nr_bvecs = count; imu->acct_pages = 0; - /* Several user leaves can share native backing, including aliases. */ - xa_init(&seen); for (i = 0; i < count; i++) { struct user_page_fragment *f = &buffer->fragments[i]; - ret = xa_insert(&seen, folio_pfn(f->folio), xa_mk_value(1), GFP_KERNEL); - if (!ret) - charged += folio_nr_pages(f->folio); - else if (ret != -EBUSY) - break; bvec_set_page(&imu->bvec[i], folio_page(f->folio, f->offset >> PAGE_SHIFT), f->length, offset_in_page(f->offset)); } - xa_destroy(&seen); - if (ret && ret != -EBUSY) + buffer->account = io_fragment_account_get(ctx); + if (!buffer->account) goto free_imu; - ret = io_account_mem(ctx->user, ctx->mm_account, charged); + ret = io_account_fragment_buffer(buffer); if (ret) goto free_imu; - imu->acct_pages = charged; imu->ubuf = (unsigned long)iov->iov_base; imu->len = iov->iov_len; imu->folio_shift = mm_page_shift(current->mm); From c9abc19a7ec8050fb8906e37673285de99e5c88b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 13:46:31 -0400 Subject: [PATCH 0870/1579] mm: serialize subpage lazyfree cancellation with discard checks - Take the pool lock when cancelling lazyfree so discard eligibility cannot combine an earlier lazyfree state with a later reference count. - Keep cancellation ordered before releasing the caller's retained reference. Existing callers acquire the helper outside the pool lock. - Validate with 16K/64K KASAN and lockdep builds and existing lazyfree and ownership suites; no concurrent-race or hardware reproduction claimed. Signed-off-by: Diljit Singh (cherry picked from commit 808513ac0c8a339799e0eb8650d2e2b0fd388e38) --- include/linux/mm_subpage.h | 1 + mm/user-subpage.c | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index 4a6224f8cc95a..81f70bed1f2c8 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -251,6 +251,7 @@ bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage); bool mm_subpage_all_lazyfree(struct mm_subpage *subpage); bool mm_subpage_discardable(struct mm_subpage *subpage); bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage); +/* Hold a slot reference through cancellation; serializes with discard checks. */ void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage); /* * Caller holds a live slot reference and its PTE lock. Success takes an diff --git a/mm/user-subpage.c b/mm/user-subpage.c index cff26e5b5c803..c08a816b90b20 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -1884,7 +1884,12 @@ bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage) void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage) { + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + + spin_lock_irqsave(&pool->lock, flags); WRITE_ONCE(subpage->lazyfree, false); + spin_unlock_irqrestore(&pool->lock, flags); } bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage) From f85e30bac1e1bb6af6958cd601605314b5394743 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 13:46:32 -0400 Subject: [PATCH 0871/1579] arm64: manage lazyfree test fragments through assertion cleanup - Keep retained fragments in KUnit-managed storage with a release action so a fatal assertion cannot strand the backing reference. - Use the already-retained folio when exercising reclaim; transient migration PTEs must not turn a successful GET into a failed lookup. - Keep the existing eleven test modes. Both 16K/64K lazyfree runs pass; no concurrent memory-offline experiment was performed. Signed-off-by: Diljit Singh (cherry picked from commit 5c9f905211e7abb99fe60c9eb21872938d1e67b2) --- arch/arm64/mm/user4k-test.c | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 38a6897e090e2..6b5aeef7be61d 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -9574,6 +9574,11 @@ static void user4k_droppable_reclaim_test(struct kunit *test) } } +static void test_release_fragment(void *arg) +{ + release_user_fragments(arg, 1, false); +} + static void user4k_lazyfree_test(struct kunit *test) { unsigned int mode; @@ -9586,7 +9591,7 @@ static void user4k_lazyfree_test(struct kunit *test) for (mode = 0; mode < 11; mode++) { struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); struct mm_struct *mm; - struct user_page_fragment held = {}; + struct user_page_fragment *held; struct folio *folio; struct mm_subpage *slot; unsigned int i, succeeded = 0; @@ -9595,11 +9600,15 @@ static void user4k_lazyfree_test(struct kunit *test) LIST_HEAD(folios); KUNIT_ASSERT_NOT_NULL(test, vma); + held = kunit_kzalloc(test, sizeof(*held), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, held); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, test_release_fragment, held), 0); mm = vma->vm_mm; KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); lru_add_drain_all(); if (mode == 6) { - ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, held, 1); KUNIT_ASSERT_EQ(test, ret, 1L); } kthread_use_mm(mm); @@ -9619,15 +9628,15 @@ static void user4k_lazyfree_test(struct kunit *test) if (mode == 2 || mode == 3 || mode == 8 || mode == 10) { ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, - mode == 3 ? FOLL_WRITE : 0, &held, 1); + mode == 3 ? FOLL_WRITE : 0, held, 1); KUNIT_ASSERT_EQ(test, ret, 1L); if (mode == 3 || mode == 8) { - memset(folio_address(held.folio) + held.offset, 0xd3, SZ_4K); - release_user_fragments(&held, 1, true); + memset(folio_address(held->folio) + held->offset, 0xd3, SZ_4K); + release_user_fragments(held, 1, true); } } if (mode == 7) { - ret = pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + ret = pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, held, 1); KUNIT_ASSERT_EQ(test, ret, 1L); } if (mode == 5 || mode == 9) { @@ -9644,7 +9653,12 @@ static void user4k_lazyfree_test(struct kunit *test) KUNIT_EXPECT_EQ(test, succeeded, 1U); } lru_add_drain_all(); - folio = test_fault_folio(mm, TEST_VA); + /* A retained GET can coexist with a temporary migration PTE. */ + folio = held->folio; + if (folio) + folio_get(folio); + else + folio = test_fault_folio(mm, TEST_VA); KUNIT_ASSERT_NOT_NULL(test, folio); folio_lock(folio); try_to_unmap(folio, TTU_SYNC); @@ -9653,7 +9667,7 @@ static void user4k_lazyfree_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); KUNIT_EXPECT_FALSE(test, folio_test_dirty(folio)); KUNIT_EXPECT_FALSE(test, pte_none(test_fault_entry(test, mm, TEST_VA))); - release_user_fragments(&held, 1, false); + release_user_fragments(held, 1, false); try_to_unmap(folio, TTU_SYNC); } folio_unlock(folio); @@ -9672,7 +9686,9 @@ static void user4k_lazyfree_test(struct kunit *test) (unsigned long long)pte_val(pte)); } if (mode == 2 || mode == 6 || mode == 7) - release_user_fragments(&held, 1, false); + release_user_fragments(held, 1, false); + kunit_release_action(test, test_release_fragment, held); + kunit_kfree(test, held); kunit_release_action(test, user4k_vma_mm_free, mm); } } From 8d840bec47b1565e0ead508190e1ea69f68e9627 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:22:28 -0400 Subject: [PATCH 0872/1579] xen/gntdev: retain typed fragments for mixed-granule grant copies Pin alternative-granule local ranges with the fragment API and derive the Xen frame from the retained backing offset. Keep native fast GUP and the existing hypercall batch size. Cache the fragment array with each batch, selecting its use from the calling MM on every ioctl. Track write intent per local pin so a later source-only segment cannot suppress dirty release for an earlier destination. Release adjacent pins with the same intent together, including on preparation failures. Signed-off-by: Diljit Singh (cherry picked from commit acebd9789c415af2f3e5492d839b8231e930671f) --- drivers/xen/gntdev.c | 82 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 14 deletions(-) diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index 1dcc4675580ed..c35d81d26e981 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -64,7 +65,11 @@ struct gntdev_copy_batch { s16 __user *status[GNTDEV_COPY_BATCH]; unsigned int nr_ops; unsigned int nr_pages; - bool writeable; + unsigned long writeable; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *fragments; + bool use_fragments; +#endif struct gntdev_copy_batch *next; }; @@ -631,6 +636,9 @@ static int gntdev_release(struct inode *inode, struct file *flip) while (priv->batch) { batch = priv->batch; priv->batch = batch->next; +#ifdef CONFIG_MM_SUBPAGE + kfree(batch->fragments); +#endif kfree(batch); } mutex_unlock(&priv->batch_lock); @@ -805,20 +813,45 @@ static long gntdev_ioctl_notify(struct gntdev_priv *priv, void __user *u) } static int gntdev_get_page(struct gntdev_copy_batch *batch, void __user *virt, - unsigned long *gfn) + size_t len, bool writeable, unsigned long *gfn) { unsigned long addr = (unsigned long)virt; struct page *page; unsigned long xen_pfn; int ret; - ret = pin_user_pages_fast(addr, 1, batch->writeable ? FOLL_WRITE : 0, &page); - if (ret < 0) - return ret; - - batch->pages[batch->nr_pages++] = page; +#ifdef CONFIG_MM_SUBPAGE + if (batch->use_fragments) { + struct user_page_fragment *fragment; - xen_pfn = page_to_xen_pfn(page) + XEN_PFN_DOWN(addr & ~PAGE_MASK); + if (!batch->fragments) { + batch->fragments = kcalloc(GNTDEV_COPY_BATCH, + sizeof(*fragment), GFP_KERNEL); + if (!batch->fragments) + return -ENOMEM; + } + fragment = &batch->fragments[batch->nr_pages]; + ret = pin_user_fragments_remote(current->mm, addr, len, + writeable ? FOLL_WRITE : 0, fragment, 1); + if (ret != 1) + return ret < 0 ? ret : -EFAULT; + if (fragment->length != len) { + release_user_fragments(fragment, 1, false); + return -EFAULT; + } + xen_pfn = XEN_PFN_DOWN(PFN_PHYS(folio_pfn(fragment->folio)) + fragment->offset); + } else +#endif + { + ret = pin_user_pages_fast(addr, 1, writeable ? FOLL_WRITE : 0, &page); + if (ret != 1) + return ret < 0 ? ret : -EFAULT; + batch->pages[batch->nr_pages] = page; + xen_pfn = page_to_xen_pfn(page) + XEN_PFN_DOWN(addr & ~PAGE_MASK); + } + if (writeable) + __set_bit(batch->nr_pages, &batch->writeable); + batch->nr_pages++; *gfn = pfn_to_gfn(xen_pfn); return 0; @@ -826,9 +859,24 @@ static int gntdev_get_page(struct gntdev_copy_batch *batch, void __user *virt, static void gntdev_put_pages(struct gntdev_copy_batch *batch) { - unpin_user_pages_dirty_lock(batch->pages, batch->nr_pages, batch->writeable); + unsigned int first = 0; + + while (first < batch->nr_pages) { + bool dirty = test_bit(first, &batch->writeable); + unsigned int next = dirty ? + find_next_zero_bit(&batch->writeable, batch->nr_pages, first) : + find_next_bit(&batch->writeable, batch->nr_pages, first); + +#ifdef CONFIG_MM_SUBPAGE + if (batch->use_fragments) + release_user_fragments(batch->fragments + first, next - first, dirty); + else +#endif + unpin_user_pages_dirty_lock(batch->pages + first, next - first, dirty); + first = next; + } batch->nr_pages = 0; - batch->writeable = false; + batch->writeable = 0; } static int gntdev_copy(struct gntdev_copy_batch *batch) @@ -917,9 +965,8 @@ static int gntdev_grant_copy_seg(struct gntdev_copy_batch *batch, virt = seg->source.virt + copied; off = (unsigned long)virt & ~XEN_PAGE_MASK; len = min(len, (size_t)XEN_PAGE_SIZE - off); - batch->writeable = false; - ret = gntdev_get_page(batch, virt, &gfn); + ret = gntdev_get_page(batch, virt, len, false, &gfn); if (ret < 0) return ret; @@ -937,9 +984,8 @@ static int gntdev_grant_copy_seg(struct gntdev_copy_batch *batch, virt = seg->dest.virt + copied; off = (unsigned long)virt & ~XEN_PAGE_MASK; len = min(len, (size_t)XEN_PAGE_SIZE - off); - batch->writeable = true; - ret = gntdev_get_page(batch, virt, &gfn); + ret = gntdev_get_page(batch, virt, len, true, &gfn); if (ret < 0) return ret; @@ -971,6 +1017,10 @@ static long gntdev_ioctl_grant_copy(struct gntdev_priv *priv, void __user *u) mutex_lock(&priv->batch_lock); if (!priv->batch) { batch = kmalloc_obj(*batch); +#ifdef CONFIG_MM_SUBPAGE + if (batch) + batch->fragments = NULL; +#endif } else { batch = priv->batch; priv->batch = batch->next; @@ -981,6 +1031,10 @@ static long gntdev_ioctl_grant_copy(struct gntdev_priv *priv, void __user *u) batch->nr_ops = 0; batch->nr_pages = 0; + batch->writeable = 0; +#ifdef CONFIG_MM_SUBPAGE + batch->use_fragments = mm_page_size(current->mm) < PAGE_SIZE; +#endif for (i = 0; i < copy.count; i++) { struct gntdev_grant_copy_segment seg; From 67ebb1ddb573088978bba7c4c922ab9d99252ee0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:34:29 -0400 Subject: [PATCH 0873/1579] mm: retire poisoned subpage mappings and exclude poisoned backing - Replace poisoned anonymous subpage PTEs with hardware-poison entries under the existing folio/PTE locks and MMU notifier scope. - Complete invalidation before dropping mapping ownership, preserving exact slot references retained by pins or asynchronous users. - Reject poisoned pool supply and skip poisoned owners during allocation; unused neighboring slots share the same native hardware-error granule. - Preserve the existing swap-cache containment path when TTU_HWPOISON is cleared by unmap_poisoned_folio(). Signed-off-by: Diljit Singh (cherry picked from commit ab3d927968dd9fb0563b26db4cc5fb7fb40a3d34) --- mm/rmap.c | 18 +++++++++++++++++- mm/user-subpage.c | 6 ++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index 9241060489f58..84aba8ec5fa41 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2157,11 +2157,27 @@ static bool try_to_unmap_subpage(struct folio *folio, struct vm_area_struct *vma swp_entry_t entry; bool exclusive, ret = false; - if (!pte_present(pte) || (flags & TTU_HWPOISON)) + if (!pte_present(pte)) return false; slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); if (WARN_ON_ONCE(!slot)) return false; + if (folio_test_hwpoison(folio) && (flags & TTU_HWPOISON)) { + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + /* Retained references keep their slot, but no CPU may keep its PTE. */ + flush_tlb_range(vma, address, end); + if (pte_dirty(pte)) + folio_mark_dirty(folio); + update_hiwater_rss(mm); + dec_mm_counter(mm, MM_ANONPAGES); + set_pte_at(mm, address, pvmw->pte, + swp_entry_to_pte(make_hwpoison_entry(&folio->page))); + mm_subpage_remove_anon_rmap(slot, vma); + mm_subpage_put(slot); /* Removed PTE reference, after invalidation. */ + ret = true; + goto out; + } if ((vma->vm_flags & VM_DROPPABLE) || mm_subpage_is_lazyfree(slot)) { flush_cache_range(vma, address, end); pte = ptep_get_and_clear(mm, address, pvmw->pte); diff --git a/mm/user-subpage.c b/mm/user-subpage.c index c08a816b90b20..bcb375cf53eca 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -1086,7 +1086,8 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, (!pool->cow_shared || !folio_test_locked(owner->folio)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && - !folio_test_writeback(owner->folio)) + !folio_test_writeback(owner->folio) && + !folio_test_hwpoison(owner->folio)) return owner; } return NULL; @@ -1103,7 +1104,8 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, /* Only an unexposed order-0 allocation can become a slot owner. */ if (folio_order(folio) || folio_ref_count(folio) != 1 || folio->mapping || folio_mapped(folio) || folio_test_lru(folio) || - folio_test_private(folio) || folio_test_swapcache(folio)) + folio_test_private(folio) || folio_test_swapcache(folio) || + folio_test_hwpoison(folio)) return -EINVAL; owner = kzalloc(sizeof(*owner), gfp | __GFP_ACCOUNT); if (!owner) From 31e4552d075fd1928309d6fc244176c859ff8a03 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:50:27 -0400 Subject: [PATCH 0874/1579] mm: reject non-native geometry in native PFN array interfaces - HMM results represent whole native pages without fragment offsets or per-leaf permissions. Reject non-native MMs before walking or changing the output PFN array. - Device VMA migration collects, owns and restores native pages. Reject non-native MMs before rounding endpoints or modifying mappings/arrays. - Return -EOPNOTSUPP as a non-retry error. Native MMs keep the existing paths; enabling alternatives requires converting the complete API and its consumers together. Signed-off-by: Diljit Singh (cherry picked from commit 39c50fbe0687b81bdc2664da3dd051b3ebc43a5a) --- mm/hmm.c | 4 ++++ mm/migrate_device.c | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/mm/hmm.c b/mm/hmm.c index 5955f2f0c83db..c4b96191ce09f 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -652,6 +652,7 @@ static const struct mm_walk_ops hmm_walk_ops = { * the invalidation to finish. * -EFAULT: A page was requested to be valid and could not be made valid * ie it has no backing VMA or it is illegal to access + * -EOPNOTSUPP: The MM uses a non-native mapping granule. * * This is similar to get_user_pages(), except that it can read the page tables * without mutating them (ie causing faults). @@ -666,6 +667,9 @@ int hmm_range_fault(struct hmm_range *range) int ret; mmap_assert_locked(mm); + /* Each output PFN describes a whole native page, without a byte offset. */ + if (mm_page_size(mm) != PAGE_SIZE) + return -EOPNOTSUPP; do { /* If range is no longer valid force retry. */ diff --git a/mm/migrate_device.c b/mm/migrate_device.c index bae35a327cf1b..4d72fb07810ab 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -668,7 +668,7 @@ static void migrate_vma_unmap(struct migrate_vma *migrate) * @args: contains the vma, start, and pfns arrays for the migration * * Returns: negative errno on failures, 0 when 0 or more pages were migrated - * without an error. + * without an error. Non-native MM granules return -EOPNOTSUPP. * * Prepare to migrate a range of memory virtual address range by collecting all * the pages backing each virtual address in the range, saving them inside the @@ -731,6 +731,10 @@ int migrate_vma_setup(struct migrate_vma *args) { long nr_pages = (args->end - args->start) >> PAGE_SHIFT; + /* PFN arrays and migration ownership describe whole native pages. */ + if (args->vma && mm_page_size(args->vma->vm_mm) != PAGE_SIZE) + return -EOPNOTSUPP; + args->start &= PAGE_MASK; args->end &= PAGE_MASK; if (!args->vma || is_vm_hugetlb_page(args->vma) || From cb4cee88da407639f3f2be270e97361c9d22cbea Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 17:26:09 -0400 Subject: [PATCH 0875/1579] mm: exclude poisoned backing from COW pool retry and revalidation The published COW pool work added a pressure retry scan and folio-locked eligibility rechecks after the original poison-containment fix was prepared. Checking hwpoison only in find_available leaves those paths inconsistent: the pressure path can report unusable poisoned backing as retryable. Put the exclusion in the shared eligibility predicate, before the non-COW early return, so ordinary allocation, pressure retry and locked revalidation all use the same condition. Keep the fresh-supply rejection. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c404199c7213c475f36788bd9336e2f9ebb89651) --- mm/user-subpage.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/user-subpage.c b/mm/user-subpage.c index bcb375cf53eca..88f043bae28b2 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -988,6 +988,9 @@ void mm_subpage_pool_close(struct mm_subpage_pool *pool) static bool cow_owner_eligible(struct mm_subpage_pool *pool, struct folio *folio) { + /* All allocation and retry paths must exclude poisoned backing. */ + if (folio_test_hwpoison(folio)) + return false; if (!pool->cow_shared) return true; /* A previously unlocked owner can be mlocked or fully MADV_FREE'd @@ -1086,8 +1089,7 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, (!pool->cow_shared || !folio_test_locked(owner->folio)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && - !folio_test_writeback(owner->folio) && - !folio_test_hwpoison(owner->folio)) + !folio_test_writeback(owner->folio)) return owner; } return NULL; From 5e18522b727c8741aeab026b30cab308607a9a0e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 17:32:45 -0400 Subject: [PATCH 0876/1579] mm: protect COW backing charge comparisons with RCU The remote fragment-pin/COW test reaches mm_subpage_cow_folio_matches() without an RCU read lock. With lockdep enabled, folio_memcg() warns in obj_cgroup_memcg() because the charge's memcg can be reparented. Hold RCU across the comparison as required by folio_memcg(). The pool's retained memcg reference does not protect a different memcg obtained from the folio. No returned memcg pointer escapes the read-side section. Fixes: 5469651a0899 ("mm: reuse backing folios for small-granule anonymous COW") Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 613963008f40faa6787e0a678afd7a008ca52554) --- mm/user-subpage.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 88f043bae28b2..68e410e546302 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -510,7 +510,14 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio) { - return pool->cow_memcg == folio_memcg(folio) && pool->cow_nid == folio_nid(folio); + bool matches; + + /* The folio charge may be reparented while this pool remains alive. */ + rcu_read_lock(); + matches = pool->cow_memcg == folio_memcg(folio) && + pool->cow_nid == folio_nid(folio); + rcu_read_unlock(); + return matches; } void mm_subpage_cow_pool_exit(struct mm_struct *mm) From 1fd7ceec883eac2272f45ca1532b7f6d2a244147 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 17:38:01 -0400 Subject: [PATCH 0877/1579] arm64: validate COW tests against the mapped physical quarter COW backing can now use a physical quarter unrelated to the fault's virtual quarter. Update the zero, fork, file and fragment-permission fixtures to validate the actual PTE and slot rather than requiring the old fixed-offset allocation policy. Read zeroed bytes and write child-private bytes at the actual slot offset. Keep the checks for untouched neighboring PTEs, copied data, parent and file-cache isolation, mapping ownership and access permissions. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit ef7f72d427299dd14bfed784ba917d19967f0bb3) --- arch/arm64/mm/user4k-test.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 6b5aeef7be61d..a18801a3fa44e 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -7096,7 +7096,7 @@ static void user4k_file_cow_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, pte_present(entry)); KUNIT_EXPECT_TRUE(test, pte_write(entry)); KUNIT_EXPECT_TRUE(test, folio_test_anon(page_folio(pte_page(entry)))); - KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, entry) & ~PAGE_MASK, 2UL * SZ_4K); + /* Private COW can pack into any physical quarter of its backing. */ KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), 0x33, SZ_4K), NULL); memset(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), 0x91, SZ_4K); @@ -7314,11 +7314,12 @@ static void test_zero_cow_granule(struct kunit *test, unsigned int shift) KUNIT_ASSERT_TRUE(test, pte_write(after)); slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); KUNIT_ASSERT_NOT_NULL(test, slot); - KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)size); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), + (unsigned int)offset_in_page(pte_phys_mm(mm, after))); KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + - size, 0, size), NULL); + mm_subpage_offset(slot), 0, size), NULL); mm_subpage_put(slot); } KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1UL); @@ -7459,13 +7460,15 @@ static void test_fork_cow_granule(struct kunit *test, unsigned int shift) KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after) != mm_subpage_phys(held[i]), changed); slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); KUNIT_ASSERT_NOT_NULL(test, slot); - KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)size); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), + (unsigned int)offset_in_page(pte_phys_mm(mm, after))); KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot), 0x31 + i, size), NULL); if (changed) { KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); - memset(folio_address(mm_subpage_folio(slot)) + size, 0x77, size); + memset(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0x77, size); } mm_subpage_put(slot); } @@ -9225,7 +9228,9 @@ static void user4k_fragment_permissions_test(struct kunit *test) ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE, 8); KUNIT_ASSERT_EQ(test, ret, 1L); KUNIT_EXPECT_NOT_NULL(test, fragment->subpage); - KUNIT_EXPECT_EQ(test, fragment->offset, SZ_4K + 17UL); + /* COW backing may use a different quarter from the virtual address. */ + KUNIT_EXPECT_EQ(test, page_to_phys(&fragment->folio->page) + fragment->offset, + pte_phys_mm(mm, test_fault_entry(test, mm, start)) + 17); memset(folio_address(fragment->folio) + fragment->offset, 0xa4, fragment->length); fragment_test_unpin(pins); mmap_write_lock(mm); From dc37a22b290d4fb0cc42d8143fce8fd0ff6de2d7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 19:36:01 -0400 Subject: [PATCH 0878/1579] mm: guard subpage COW pool charge references without MEMCG get_mem_cgroup_from_mm() returns NULL without CONFIG_MEMCG, but the COW pool still references the incomplete struct mem_cgroup in the css_get() expression. Guard the retained-reference operation so mixed-granule kernels build without memory cgroups. Keep the reference acquisition unchanged for enabled memory cgroups. --- mm/user-subpage.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 68e410e546302..c9101990099df 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -466,8 +466,10 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, mm_subpage_pool_put(new); break; } +#ifdef CONFIG_MEMCG if (memcg) css_get(&memcg->css); +#endif new->cow_memcg = memcg; new->cow_nid = nid; new->cow_shared = true; From 4bed9fe05ec15856f2004e028dde1db0ec8da973 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 19:42:05 -0400 Subject: [PATCH 0879/1579] arm64: validate packed fragments through mapped physical offsets The shared COW pool may place a private fragment in another physical quarter of the same native folio. Sparse anonymous allocations likewise need not match their virtual-page index. Follow actual PTE/subpage offsets when checking pinned copies, live COW writes and restored swap data; record pre-migration and pre-swap offsets where the operation must preserve them. Verify tags at the actual copied slot and require child read pins and swap COW writes to retain distinct physical backing from the parent. Keep data, permissions, ownership and accounting assertions intact. Validated with 233 passing ARM64 KUnit tests (39 skips), all 17 disposable swap tests, and native16K/4K exec and thread VM fixtures. A separate memory-cgroup-enabled COW object build also passes. --- arch/arm64/mm/user4k-test.c | 72 +++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index a18801a3fa44e..d5349f54f6d7c 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -3019,7 +3019,9 @@ static void test_anon_write_fault_granule(struct kunit *test, unsigned int shift if (!backing) backing = folio; identity = folio == backing && - mm_subpage_offset(slot) == i * size; + mm_subpage_phys(slot) == + pte_phys_mm(vma->vm_mm, ptep_get(pte)) && + (shape || mm_subpage_offset(slot) == i * size); zero = !memchr_inv(folio_address(folio) + mm_subpage_offset(slot), 0, size); mm_subpage_put(slot); @@ -8553,7 +8555,8 @@ static void test_swap_tags_set(struct folio *folio) #endif } -static bool test_swap_tags_match(struct folio *folio, unsigned int slot) +static bool test_swap_tags_match_at(struct folio *folio, unsigned int start, + unsigned int slot) { #ifdef CONFIG_ARM64_MTE unsigned int offset; @@ -8562,7 +8565,7 @@ static bool test_swap_tags_match(struct folio *folio, unsigned int slot) return true; if (!page_mte_tagged(&folio->page)) return false; - for (offset = slot * SZ_4K; offset < (slot + 1) * SZ_4K; offset += MTE_GRANULE_SIZE) { + for (offset = start; offset < start + SZ_4K; offset += MTE_GRANULE_SIZE) { unsigned long tag = 0; asm volatile(".arch_extension memtag\n\tldg %0, [%1]" @@ -8574,6 +8577,11 @@ static bool test_swap_tags_match(struct folio *folio, unsigned int slot) return true; } +static bool test_swap_tags_match(struct folio *folio, unsigned int slot) +{ + return test_swap_tags_match_at(folio, slot * SZ_4K, slot); +} + struct fragment_test_pins { struct user_page_fragment fragments[2 * TEST_SLOTS]; unsigned long count; @@ -8681,9 +8689,14 @@ static void user4k_fragment_pin_test(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, 0x71 + i, f->length), NULL); if (!native) { + pte_t parent_pte = test_fault_entry(test, mm, TEST_VA + i * size); + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(f->subpage)); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), + pte_phys_mm(mm, parent_pte)); if (i) - KUNIT_EXPECT_PTR_NE(test, f->folio, pins->fragments[0].folio); + KUNIT_EXPECT_NE(test, mm_subpage_phys(f->subpage), + mm_subpage_phys(pins->fragments[0].subpage)); } } fragment_test_unpin(pins); @@ -9337,9 +9350,10 @@ static void user4k_slot_pin_test(struct kunit *test) copy = mm_subpage_get_from_phys(pte_phys_mm(child, pte)); KUNIT_ASSERT_NOT_NULL(test, copy); KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(copy), i == 1); - KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(copy)) + i * SZ_4K, + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(copy)) + mm_subpage_offset(copy), 0x61 + i, SZ_4K), NULL); - KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(copy), i)); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match_at(mm_subpage_folio(copy), + mm_subpage_offset(copy), i)); mm_subpage_put(copy); } /* A read pin on a shared slot must request unsharing, not pin shared COW data. */ @@ -10324,6 +10338,7 @@ static void user4k_migration_test(struct kunit *test) unsigned long start = edge ? boundaries[mode - 6] : TEST_VA + (sparse ? SZ_4K : 0); unsigned int nr = sparse ? 1 : 4, i, side, succeeded = 0; + unsigned int offsets[TEST_SLOTS]; struct vm_area_struct *vma = test_fault_vma(test, start, nr * SZ_4K); struct mm_struct *child = NULL; struct folio *folio; @@ -10350,6 +10365,9 @@ static void user4k_migration_test(struct kunit *test) for (i = 0; i < TEST_SLOTS; i++) memset(folio_address(folio) + i * SZ_4K, 0x41 + i + mode, SZ_4K); test_swap_tags_set(folio); + for (i = 0; i < nr; i++) + offsets[i] = offset_in_page(pte_phys_mm(vma->vm_mm, + test_fault_entry(test, vma->vm_mm, start + i * SZ_4K))); folio_put(folio); if (forked) { child = test_vma_mm(test); @@ -10379,7 +10397,7 @@ static void user4k_migration_test(struct kunit *test) struct mm_struct *mm = side ? child : vma->vm_mm; for (i = 0; i < nr; i++) { - unsigned int index = sparse ? offset_in_page(start) / SZ_4K : i; + unsigned int index = offsets[i] / SZ_4K; pte_t pte = test_fault_entry(test, mm, start + i * SZ_4K); struct mm_subpage *slot; @@ -10866,13 +10884,27 @@ static void test_user4k_swap_roundtrip(struct kunit *test, unsigned int mode, u8 KUNIT_EXPECT_TRUE(test, pte_write(pte)); folio = test_fault_folio(child, TEST_VA + SZ_4K); KUNIT_ASSERT_NOT_NULL(test, folio); - KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); - KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, 1)); - memset(folio_address(folio) + SZ_4K, 0xa7, SZ_4K); + { + unsigned int offset = pte_phys_mm(child, pte) - PFN_PHYS(folio_pfn(folio)); + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + base + 1, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match_at(folio, offset, 1)); + memset(folio_address(folio) + offset, 0xa7, SZ_4K); + } folio_put(folio); folio = test_fault_folio(parent->vm_mm, TEST_VA + SZ_4K); KUNIT_ASSERT_NOT_NULL(test, folio); - KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); + { + pte_t parent_pte = test_fault_entry(test, parent->vm_mm, TEST_VA + SZ_4K); + unsigned int offset = pte_phys_mm(parent->vm_mm, parent_pte) - + PFN_PHYS(folio_pfn(folio)); + + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), + pte_phys_mm(parent->vm_mm, parent_pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + base + 1, SZ_4K), NULL); + } folio_put(folio); KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1UL); KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), TEST_SLOTS - 1UL); @@ -11247,12 +11279,13 @@ static void user4k_swapoff_boundaries_test(struct kunit *test) struct vm_area_struct *parent = test_fault_vma(test, start, 2 * SZ_4K); struct mm_struct *child; unsigned int i, side; + unsigned int offsets[2]; int ret; KUNIT_ASSERT_NOT_NULL(test, parent); for (i = 0; i < 2; i++) { unsigned long addr = start + i * SZ_4K; - unsigned int offset = vma_page_offset_at(parent, addr).offset; + unsigned int offset; struct folio *folio; vm_fault_t fault; swp_entry_t entry; @@ -11262,6 +11295,9 @@ static void user4k_swapoff_boundaries_test(struct kunit *test) fault = handle_mm_fault(parent, addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); mmap_read_unlock(parent->vm_mm); KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + offset = offset_in_page(pte_phys_mm(parent->vm_mm, + test_fault_entry(test, parent->vm_mm, addr))); + offsets[i] = offset; lru_add_drain_all(); folio = test_fault_folio(parent->vm_mm, addr); KUNIT_ASSERT_NOT_NULL(test, folio); @@ -11293,7 +11329,7 @@ static void user4k_swapoff_boundaries_test(struct kunit *test) KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 2UL); for (i = 0; i < 2; i++) { unsigned long addr = start + i * SZ_4K; - unsigned int offset = vma_page_offset_at(parent, addr).offset; + unsigned int offset = offsets[i]; pte_t pte = test_fault_entry(test, mm, addr); struct folio *folio; @@ -11520,8 +11556,14 @@ static void test_live_fault_roundtrip(struct kunit *test, unsigned int mode) } folio = test_fault_folio(child, TEST_VA + SZ_4K); KUNIT_ASSERT_NOT_NULL(test, folio); - for (i = 0; i < 2; i++) - KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + SZ_4K + 64 + i), (u8)(0xe0 + i)); + { + pte_t pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + unsigned long offset = pte_phys_mm(child, pte) - PFN_PHYS(folio_pfn(folio)); + + for (i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + offset + 64 + i), + (u8)(0xe0 + i)); + } folio_put(folio); mmap_read_lock(parent->vm_mm); fault = handle_mm_fault(parent, TEST_VA + SZ_4K, FAULT_FLAG_REMOTE, NULL); From 1af8de012d2221144395f900729619b5065b14fd Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:48:24 -0400 Subject: [PATCH 0880/1579] drm/asahi: provision G17P ray traversal pools from Gen1 SCS demand Program the Gen1 scratch and intersection-table registers from validated compute requests. Publish the optional compute command tail in the UAPI and retain zero-tail compatibility for ordinary compute submissions. Include each enabled USC's SCS small-page, large-page and tail inventories in physical compute pool admission. Allocate the admitted capacity for new or vacant queues, reuse retained pools only when they fit, and grow owned queues after prior work drains and firmware consumes the free-list release. Keep old aliases and backing until the replacement is published. Preserve the pending capacity high-water mark across competing requests. This replaces the private fixed 64MiB RT pool experiment. Ordinary compute keeps its existing 21MiB floor, while the captured 304-byte Gen1 profile requires 41MiB on Neo's five enabled cores. Diagnostic logging and RT-unit control experiments are excluded. Validation: seven actual-source host capacity/lifecycle tests; matched651 module import/modinfo audit; Neo hardware 99 RT CTS passes (two unsupported), repeated 16384-ray dispatches at local 64/256 and four concurrent clients, with fourteen checked runtime outputs and no new captured kernel faults. --- drivers/gpu/drm/asahi/g17/command.rs | 22 ++- drivers/gpu/drm/asahi/g17/freelist.rs | 26 ++- drivers/gpu/drm/asahi/g17/fw/compute.rs | 187 +++++++++++++++++-- drivers/gpu/drm/asahi/g17/fw/compute_pool.rs | 25 +++ drivers/gpu/drm/asahi/g17/fw/initdata.rs | 6 +- drivers/gpu/drm/asahi/g17/fw/mod.rs | 1 + drivers/gpu/drm/asahi/g17/initdata.rs | 9 +- drivers/gpu/drm/asahi/g17/job.rs | 2 +- drivers/gpu/drm/asahi/g17/mod.rs | 3 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 61 +++++- drivers/gpu/drm/asahi/g17/queue/mod.rs | 7 +- drivers/gpu/drm/asahi/g17/runtime.rs | 69 ++++++- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 4 +- include/uapi/drm/asahi_drm.h | 23 ++- 14 files changed, 388 insertions(+), 57 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/fw/compute_pool.rs diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 83191ddbcb1c3..dd3be1a81e1d8 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -3,7 +3,7 @@ //! Decode a submission and validate its user resources before publishing any work. use super::fw::{ - compute::ScratchRequest, + compute::{IntersectionTables, ScratchRequest}, kick::WriteRange, render::{RenderPass, UscProgram, ZlsSurface}, }; @@ -14,6 +14,7 @@ use kernel::{prelude::*, uapi}; const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; const USC_WINDOW_SIZE: u64 = 1 << 32; const MAX_SAMPLERS: u32 = 1024; +const COMPUTE_BASE_SIZE: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_compute, prior_cdm) + size_of::(); const SAMPLER_SIZE: u64 = 8; const PAGE_SIZE: u64 = 0x4000; const ZLS_METADATA_STRIDE: u64 = 0x80; @@ -78,7 +79,7 @@ impl Attachments { #[allow(clippy::large_enum_variant)] pub(crate) enum Payload { Render(uapi::drm_asahi_cmd_render, [Attachments; 2]), - Compute(uapi::drm_asahi_cmd_compute, Attachments), + Compute(uapi::drm_asahi_cmd_compute, Attachments, IntersectionTables), } pub(crate) struct Command { @@ -130,7 +131,11 @@ impl<'a> Parser<'a> { Payload::Render(payload(bytes)?, [self.attachments[0], self.attachments[1]]) } else { self.counts[1] += 1; - Payload::Compute(payload(bytes)?, self.attachments[2]) + Payload::Compute( + payload(&bytes[..bytes.len().min(COMPUTE_BASE_SIZE)])?, + self.attachments[2], + IntersectionTables::parse(bytes.get(COMPUTE_BASE_SIZE..).unwrap_or(&[]))?, + ) }; return Ok(Some(Command { barriers, payload })); } @@ -283,6 +288,7 @@ pub(crate) enum Validated { timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { + intersection: IntersectionTables, scratch: ScratchRequest, usc_base: u64, cdm_va: u64, @@ -297,7 +303,7 @@ impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { Self::Render(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], - Self::Compute(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Compute(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], } } @@ -415,7 +421,7 @@ impl Payload { timestamps: [cmd.ts_vtx, cmd.ts_frag], }) } - Self::Compute(cmd, attachments) => { + Self::Compute(cmd, attachments, intersection) => { if cmd.flags != 0 || cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 @@ -432,7 +438,13 @@ impl Payload { )?; sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; + if intersection.enabled() { + require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; + let code = intersection.default_program & 0x0000_ffff_ffff_ffff; + if code != 0 { require_range(space, code, 64, Access::Read)?; } + } Ok(Validated::Compute { + intersection: *intersection, scratch: ScratchRequest::new(cmd.scs_layout)?, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index c2c9470d10aa3..2612a9302ce28 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -8,7 +8,7 @@ //! reference and are released only after the owner's teardown is acknowledged. use super::fw::freelist::*; -use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIST_BLOCK_SIZE}; +use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; use super::object::{Allocator, CpuMap, KernelObject}; use crate::mmu; use core::sync::atomic::{fence, AtomicBool, AtomicU64, Ordering}; @@ -90,9 +90,15 @@ impl Pages { layout, }) } - /// Fresh zeroed compute backing, built without the device mutex. - pub(crate) fn compute(alloc: &Allocator<'_>) -> Result { - Self::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute) + /// Fresh compute backing at the admitted capacity, built off-lock. + pub(crate) fn compute_sized(alloc: &Allocator<'_>, blocks: usize) -> Result { + if blocks < COMPUTE_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { + return Err(EINVAL); + } + Self::new(alloc, blocks, RunLayout::Compute) + } + pub(crate) fn map_client(&self, vm: &mmu::Vm) -> Result { + self.backing.map_alias(vm, self.page_list_va(), mmu::PROT_GPU_SHARED_RW) } fn page_list_va(&self) -> u64 { self.backing.gpu_va() @@ -437,12 +443,16 @@ pub(crate) struct ComputePool { id: u16, } impl ComputePool { - pub(crate) fn new(alloc: &Allocator<'_>, qid: u8) -> Result { + pub(crate) fn fits(&self, blocks: usize) -> bool { + self.pages.as_ref().is_some_and(|pages| pages.blocks >= blocks) + } + + pub(crate) fn new(alloc: &Allocator<'_>, qid: u8, blocks: usize) -> Result { if qid > super::fw::kick::QID_MAX { return Err(EINVAL); } let id = u16::from(qid) + 1; - let pages = Pages::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute)?; + let pages = Pages::compute_sized(alloc, blocks)?; let mut control = alloc.kernel( CONTROL_SIZE, mmu::UAT_PGSZ as u64, @@ -462,7 +472,7 @@ impl ComputePool { class: FreeListClass::One, page_list_va: pages.page_list_va(), run_list_va: pages.run_list_va(), - blocks: COMPUTE_BLOCKS as u32, + blocks: blocks as u32, state_va: state.gpu_va(), }), )?; @@ -517,7 +527,7 @@ impl ComputePool { class: FreeListClass::One, page_list_va: pages.page_list_va(), run_list_va: pages.run_list_va(), - blocks: COMPUTE_BLOCKS as u32, + blocks: pages.blocks as u32, state_va: self.state.gpu_va(), }); header.unk_54[..8].copy_from_slice(&bytes); diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 5b7489cfe93f4..a4775abc6b7eb 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -23,57 +23,114 @@ pub(crate) const COMPUTE_QOS_CLASS: u8 = 0x08; /// Event mask of compute kicks. pub(crate) const COMPUTE_KICK_EVENT_MASK: [u64; 4] = [0, 0, 4, 0]; -const COMPUTE_REGISTERS: usize = 40; +const COMPUTE_REGISTERS: usize = 41; const COMPUTE_REGISTERS_MAX: usize = COMPUTE_REGISTERS + 1; /// Validated per-command shader-context scratch request. +/// +/// The layout word is Apple's packed SCS layout: depth (bits 0:15), levels +/// (16:23), cliques (24:31), user (32:47) and driver (48:63). The region the +/// hardware needs is derived with G17P's Gen1 section model: fixed prefixes +/// totalling [`Self::FIXED_BYTES`], then per-thread stack, per-thread driver +/// and per-thread user bytes. Gen1 has no per-clique level payload. The scratch +/// registers the program writes are the native `generateRegisterList` +/// formulas use `com.apple.AGXG17P`; the +/// disabled program keeps the native disabled constants. #[derive(Clone, Copy)] pub(crate) struct ScratchRequest { pages: u16, + window: u64, + stack: u64, + driver: u64, + tail: u64, } impl ScratchRequest { + /// Native ScsLayoutGen1::getSectionEnd(4). const FIXED_BYTES: u64 = 0x58000; const THREADS_PER_CLIQUE: u64 = 32; const BYTES_PER_STACK_ENTRY: u64 = 8; const PAGE_SHIFT: u32 = 12; const PAGE_SIZE: u64 = 1 << Self::PAGE_SHIFT; const PAGES_PER_MIB: u64 = 1 << (20 - Self::PAGE_SHIFT); + /// Window word when enabled: bit 24 plus the clique count at 16:23. const WINDOW_ENABLED: u64 = 1 << 24; + /// Native disabled window word, [`UNK_10791`]. + const WINDOW_DISABLED: u64 = UNK_10791; + /// Native disabled stack word, [`UNK_1A0E9`]. + const STACK_DISABLED: u64 = UNK_1A0E9; + /// Native tail reservation when enabled. + const TAIL_ENABLED: u64 = 8; /// Payload-free layouts request no scratch even if depth/cliques are set. - /// Otherwise round the fixed and per-thread sections and add a guard page. + /// Otherwise derive the Gen1 end and round it up with the native two-page + /// guard; the per-register words are the native formulas, so they are + /// checked here against Apple's own values before they reach hardware. pub(crate) fn new(layout: u64) -> Result { let depth = layout & u16::MAX as u64; let cliques = (layout >> 24) & u8::MAX as u64; let user = (layout >> 32) & u16::MAX as u64; let driver = (layout >> 48) & u16::MAX as u64; if user == 0 && driver == 0 { - return Ok(Self { pages: 0 }); + return Ok(Self { + pages: 0, + window: Self::WINDOW_DISABLED, + stack: Self::STACK_DISABLED, + driver: 0, + tail: 0, + }); } let threads = cliques.checked_mul(Self::THREADS_PER_CLIQUE).ok_or(EINVAL)?; - let per_thread = depth.checked_mul(Self::BYTES_PER_STACK_ENTRY) - .and_then(|bytes| bytes.checked_add(user)) - .and_then(|bytes| bytes.checked_add(driver)).ok_or(EINVAL)?; - let bytes = threads.checked_mul(per_thread) - .and_then(|bytes| bytes.checked_add(Self::FIXED_BYTES)) + let stack_bytes = threads + .checked_mul(depth.checked_mul(Self::BYTES_PER_STACK_ENTRY).ok_or(EINVAL)?) + .ok_or(EINVAL)?; + let driver_bytes = threads.checked_mul(driver).ok_or(EINVAL)?; + let user_bytes = threads.checked_mul(user).ok_or(EINVAL)?; + let end10 = Self::FIXED_BYTES + .checked_add(stack_bytes) + .and_then(|bytes| bytes.checked_add(driver_bytes)) + .ok_or(EINVAL)?; + let end = end10 + .checked_add(user_bytes) .and_then(|bytes| bytes.checked_add(2 * Self::PAGE_SIZE - 1)) .ok_or(EINVAL)?; - let pages = bytes >> Self::PAGE_SHIFT; - Ok(Self { pages: u16::try_from(pages).map_err(|_| EINVAL)? }) + let pages = end >> Self::PAGE_SHIFT; + Ok(Self { + pages: u16::try_from(pages).map_err(|_| EINVAL)?, + window: Self::WINDOW_ENABLED | (cliques << 16), + stack: depth & !7, + // G17P's native register list does not publish the Gen2 level + // region register. Preserve the historical zero slot. + driver: 0, + tail: Self::TAIL_ENABLED, + }) } pub(crate) fn enabled(self) -> bool { self.pages != 0 } + pub(crate) fn pages(self) -> u16 { self.pages } + fn size_word(self) -> u64 { let pages = u64::from(self.pages); pages | (pages.div_ceil(Self::PAGES_PER_MIB) << 16) } fn window_word(self) -> u64 { - if self.enabled() { Self::WINDOW_ENABLED } else { UNK_10791 } + self.window + } + + fn stack_word(self) -> u64 { + self.stack + } + + fn driver_word(self) -> u64 { + self.driver + } + + fn tail_word(self) -> u64 { + self.tail } fn checkpoints(self) -> [u16; 4] { @@ -106,10 +163,94 @@ const OPERAND_STATE_0A5C9: u64 = 0x9000; /// Checkpoints into the main register array, as entry counts: after the /// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full /// count. -const CHECKPOINTS: [u16; 4] = [38, 39, 39, COMPUTE_REGISTERS as u16]; +const CHECKPOINTS: [u16; 4] = [39, 40, 40, COMPUTE_REGISTERS as u16]; + +#[derive(Clone, Copy)] +pub(crate) struct IntersectionTables { + pub(crate) pool: u64, + pub(crate) pool_bytes: u64, + auxiliary: u32, + first: u32, + count: u32, + pub(crate) default_program: u64, + default_control: u64, +} + +impl IntersectionTables { + pub(crate) const NONE: Self = Self { + pool: 0, pool_bytes: 0, auxiliary: 0, first: 0, count: 0, + default_program: 0, default_control: 0, + }; + + pub(crate) fn new(pool: u64, pool_bytes: u64, first: u32, count: u32, + default_program: u64, default_control: u64) -> Result { + if pool == 0 { + if pool_bytes != 0 || first != 0 || count != 0 + || default_program != 0 || default_control != 0 { + return Err(EINVAL); + } + return Ok(Self::NONE); + } + // G17P HAL200 Gen1 publishes a byte-sized table length (25 bits), + // with a plain aligned address. Unlike G17X HAL300, its address has + // no bit 51 and its two pool registers use fixed native values. + // Keep the shared UAPI's 128-byte blocks and 16 KiB allocation units. + let end = pool.checked_add(pool_bytes).ok_or(EINVAL)?; + let blocks_end = u64::from(first) + u64::from(count); + let code = default_program & 0x0000_ffff_ffff_ffff; + if pool & 0x3fff != 0 || pool_bytes == 0 || pool_bytes & 0x3fff != 0 + || end > (1u64 << 48) || (pool_bytes >> 14) >= (1u64 << 25) + || count == 0 || count >= (1 << 18) || blocks_end * 128 > pool_bytes + || code & 63 != 0 + || default_control & !0x0000_7fff_0000_3fff != 0 + || (code == 0 && (default_program != 0 || default_control != 0)) + || (code != 0 && default_control & (1u64 << 40) == 0) { + return Err(EINVAL); + } + Ok(Self { pool, pool_bytes, auxiliary: 0, first, count, default_program, default_control }) + } + + pub(crate) fn parse(bytes: &[u8]) -> Result { + if bytes.get(48..).is_some_and(|tail| tail.iter().any(|&b| b != 0)) { + return Err(EINVAL); + } + let mut tail = [0u8; 48]; + let count = bytes.len().min(tail.len()); + tail[..count].copy_from_slice(&bytes[..count]); + let auxiliary = u32::from_le_bytes(tail[..4].try_into().unwrap()); + if tail[4..8].iter().any(|&b| b != 0) + || (auxiliary != 0 && !(8..=32).contains(&auxiliary)) { + return Err(EINVAL); + } + let mut tables = Self::new( + u64::from_le_bytes(tail[8..16].try_into().unwrap()), + u64::from_le_bytes(tail[16..24].try_into().unwrap()), + u32::from_le_bytes(tail[24..28].try_into().unwrap()), + u32::from_le_bytes(tail[28..32].try_into().unwrap()), + u64::from_le_bytes(tail[32..40].try_into().unwrap()), + u64::from_le_bytes(tail[40..48].try_into().unwrap()), + )?; + if auxiliary != 0 && !tables.enabled() { return Err(EINVAL); } + tables.auxiliary = auxiliary; + Ok(tables) + } + + pub(crate) fn enabled(self) -> bool { self.pool != 0 } + + pub(crate) fn words(self) -> [u64; 6] { + if !self.enabled() { return [0; 6]; } + // AGXMetalG17P 26A428 endComputePass, both command encodings; + // pinned AGXG17P processComputeSetup copies these six words as-is. + [self.pool + u64::from(self.first) * 128, + 0xc0000000 | (u64::from(self.count) << 7), 0x6f00000000, + 0x3feff, self.default_program, self.default_control] + } +} + /// Inputs of one compute descriptor. pub(crate) struct ComputeArgs { + pub(crate) intersection: IntersectionTables, /// Scratch request validated with the userspace command. pub(crate) scratch: ScratchRequest, /// Base of the queue's USC window. @@ -166,6 +307,7 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS let operand = args.operand_state_va; let key = work_key(args.context_id, args.work_state); let ids = OBJECT_IDS.word(); + let intersection = args.intersection.words(); Ok([ RegisterWrite::new(0x017d9, 1), @@ -183,15 +325,20 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS RegisterWrite::new(0x1a440, UNK_1A440), RegisterWrite::new(0x1a458, UNK_1A458), RegisterWrite::new(0x101d9, UNK_101D9), - RegisterWrite::new(0x1a089, 0), - RegisterWrite::new(0x1a091, 0), - RegisterWrite::new(0x1a059, 0), - RegisterWrite::new(0x1a061, 0), - RegisterWrite::new(0x1a0b9, 0), - RegisterWrite::new(0x1a0c1, 0), + RegisterWrite::new(0x1a089, intersection[0]), + RegisterWrite::new(0x1a091, intersection[1]), + RegisterWrite::new(0x1a059, intersection[2]), + RegisterWrite::new(0x1a061, intersection[3]), + RegisterWrite::new(0x1a0b9, intersection[4]), + RegisterWrite::new(0x1a0c1, intersection[5]), RegisterWrite::new(0x101d1, args.scratch.size_word()), - RegisterWrite::new(0x0d479, 0), - RegisterWrite::new(0x1a0e9, UNK_1A0E9), + RegisterWrite::new(0x0d479, if args.intersection.auxiliary == 0 { + args.scratch.tail_word() + } else { + u64::from(args.intersection.auxiliary) + }), + RegisterWrite::new(0x1a0e9, args.scratch.stack_word()), + RegisterWrite::new(0x1a141, args.scratch.driver_word()), RegisterWrite::new(0x107a1, args.scratch.window_word()), RegisterWrite::new(0x0a599, UNK_0A599), RegisterWrite::new(0x0d411, UNK_0D411), diff --git a/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs b/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs new file mode 100644 index 0000000000000..4005daa4e9832 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! G17P compute pool capacity for a validated Gen1 SCS request. + +use kernel::prelude::*; +use super::initdata::{FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; +use super::freelist::{COMPUTE_BLOCKS, PAGE_LIST_SIZE}; + +pub(crate) fn required_blocks(active_cores: u32, scs_pages: u16) -> Result { + if active_cores == 0 { return Err(EINVAL); } + if scs_pages == 0 { return Ok(COMPUTE_BLOCKS); } + // Register0x101d1 counts 4-KiB pages and their 1-MiB large-page inventory. + // Register0x0d479 reserves eight further pages for each enabled USC. + let small = u64::from(scs_pages); + let per_core = small.checked_add(small.div_ceil(256)) + .and_then(|pages| pages.checked_add(8)).ok_or(EOVERFLOW)?; + let extra = per_core.checked_mul(u64::from(active_cores)) + .and_then(|pages| pages.checked_mul(4096)).ok_or(EOVERFLOW)?; + let bytes = extra.checked_add((COMPUTE_BLOCKS * FREE_LIST_BLOCK_SIZE) as u64) + .ok_or(EOVERFLOW)?; + let blocks = usize::try_from(bytes.div_ceil(FREE_LIST_BLOCK_SIZE as u64)) + .map_err(|_| EOVERFLOW)?; + let maximum = PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8); + if blocks > maximum { return Err(E2BIG); } + Ok(blocks) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 94b03ebe284db..1f25bd0fc1c0d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -2239,7 +2239,11 @@ static_assert!(size_of::() == 0x20); impl PagePoolDescriptor { /// Complete row for a physical compute queue's independent 21-block pool. pub(crate) const fn compute(page_list_va: u64) -> Self { - let pages = 21 * FREE_LIST_BLOCK_PAGES as u64; + Self::compute_blocks(page_list_va, 21) + } + + pub(crate) const fn compute_blocks(page_list_va: u64, blocks: usize) -> Self { + let pages = blocks as u64 * FREE_LIST_BLOCK_PAGES as u64; Self::new(page_list_va, pages << PAGE_POOL_FREE_PAGES_SHIFT, pages) } diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index 4f0ae8aa253ed..479bc91d21ca5 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -14,6 +14,7 @@ use kernel::prelude::*; pub(crate) mod buffer; pub(crate) mod channels; pub(crate) mod compute; +pub(crate) mod compute_pool; pub(crate) mod context; pub(crate) mod freelist; pub(crate) mod initdata; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 46a38c5ad16ec..4df78cca06d46 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -285,11 +285,12 @@ impl InitData { /// Initialize a fresh physical compute QID's row before exposing its queue. An installed /// or rebound queue keeps the row's firmware-owned counters and must never call this again. - pub(super) fn initialize_compute_pool(&self, slot: u16, page_list_va: u64) -> Result { + /// Called for a fresh row or after its free-list release was consumed. + pub(super) fn initialize_compute_pool_blocks(&self, slot: u16, page_list_va: u64, blocks: usize) -> Result { if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { return Err(EINVAL); } - let entry = abi::PagePoolDescriptor::compute(page_list_va); + let entry = abi::PagePoolDescriptor::compute_blocks(page_list_va, blocks); let size = core::mem::size_of::(); let offset = usize::from(slot) * size; let object = self.object(self.page_pool)?; @@ -299,8 +300,8 @@ impl InitData { object.dword(offset + core::mem::offset_of!(abi::PagePoolDescriptor, counters))?; let count = object.dword(offset + core::mem::offset_of!(abi::PagePoolDescriptor, page_count))?; - // SAFETY: the whole fresh row was bounds checked, the QID has never been published, - // and the device mutex excludes another host publisher. No other row is touched. + // SAFETY: the whole fresh row was bounds checked, this is a fresh QID or its release + // was consumed, and the device mutex excludes another host publisher. No other row is touched. unsafe { pointer.write_bytes(0, size); } diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 34b6308620644..b85d794ace512 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -238,7 +238,7 @@ pub(crate) trait Backend: Send + Sync { } /// Lazily create or rebind this logical queue's physical compute graph. - fn ensure_compute(&self) -> Result; + fn ensure_compute(&self, scs_pages: u16) -> Result; /// Settle outstanding work of a terminal VM before an ioctl error is published. fn fail_vm(&self, error: Error); /// Called after both scheduler entities have cancelled their queued jobs. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 23c4fb962153a..3cfa2c1d71655 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -603,6 +603,7 @@ struct Shared { #[pin] drm: Mutex>, clusters: u32, + active_cores: u32, descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, @@ -880,11 +881,13 @@ impl Gpu { let preparations = preparation::Gate::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; + let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), dev: dev.as_ref().into(), clusters: id.num_clusters, + active_cores, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 10b66c9ae811b..cae576fe183af 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -416,8 +416,8 @@ pub(crate) struct PoolBacking { } impl PoolBacking { - pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id) -> Result { - let pool = ComputePool::new(alloc, id.qid())?; + pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id, blocks: usize) -> Result { + let pool = ComputePool::new(alloc, id.qid(), blocks)?; Ok(Self { pool, id }) } @@ -459,6 +459,8 @@ pub(crate) struct Queue { /// the installed queue record still names them until the next owner binds. retained_owner: Option, pool_phase: PoolPhase, + growing: bool, + growth_blocks: usize, /// A killed queue was recycled: its pool/ring must be vacated and re-armed /// fresh before any owner binds it (set by `recycle_killed`, cleared by `rearm`). needs_fresh_backing: bool, @@ -518,6 +520,8 @@ impl Queue { owner: Some(owner), retained_owner: None, pool_phase: PoolPhase::Populated, + growing: false, + growth_blocks: 0, needs_fresh_backing: false, released_at: 0, free_list_generation: 0, @@ -1042,6 +1046,7 @@ impl Queue { Ok(true) } pub(crate) fn reusable(&self) -> bool { + if self.growing { return false; } self.installed // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) @@ -1238,6 +1243,7 @@ impl Queue { consumer: u32, producer: u32, ) -> Result> { + if self.growing { return Ok(None); } let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { return Ok(None); }; @@ -1274,6 +1280,55 @@ impl Queue { self.needs_fresh_backing = false; Ok((self.pool.id(), self.pool.page_list_va()?)) } + pub(crate) fn pool_fits(&self, required: usize) -> bool { + !self.growing && self.pool_phase == PoolPhase::Populated && self.pool.fits(required) + } + + /// Only a capacity increase waits for this queue's previously published work. + /// Other queues and ordinary submissions keep their existing concurrency. + pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { + if self.growing { + if blocks < self.growth_blocks { return Err(EAGAIN); } + self.growth_blocks = blocks; + return Ok(false); + } + if self.quarantined || self.released || self.exit.is_some() + || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() + || (self.installed && !self.retirement_ready) + || self.pool_phase != PoolPhase::Populated { return Err(EAGAIN); } + if self.installed { + let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); + let cursor = Rings::publish_control(init, &record)?; + self.pool_phase = PoolPhase::ReleaseSent(cursor); + } else { self.pool_phase = PoolPhase::Vacant; } + self.growing = true; + self.growth_blocks = blocks; + Ok(self.installed) + } + + pub(crate) fn finish_growth(&mut self, init: &crate::g17::initdata::InitData, + pages: &mut Option, + alias: &mut Option, blocks: usize) -> Result { + if !self.growing || blocks < self.growth_blocks { return Err(EAGAIN); } + if let PoolPhase::ReleaseSent(cursor) = self.pool_phase { + let consumer = init.control_consumer()?.load(Ordering::Acquire); + let producer = init.control_producer()?.load(Ordering::Acquire); + if !crate::g17::freelist::control_consumed(consumer, producer, cursor)? { + return Err(EAGAIN); + } + } + let binding = self.binding.as_mut().ok_or(EFAULT)?; + let old_alias = binding.pool_alias.take(); + let old_pages = self.pool.take_pages(); + self.pool.rearm(pages.take().ok_or(EINVAL)?)?; + init.initialize_compute_pool_blocks(self.pool.id(), self.pool.page_list_va()?, blocks)?; + binding.pool_alias = alias.take(); + self.pool_phase = PoolPhase::Populated; + self.growing = false; + self.growth_blocks = 0; + Ok(Vacated { _alias: old_alias, _pages: old_pages, _descriptors: None, _ring: None }) + } + /// Exact host identity remains valid even when several commands share an ioctl ID. pub(crate) fn owns_packet(&self, packet: &Arc) -> bool { self.active @@ -1506,6 +1561,7 @@ impl Queue { }; let (epoch, addresses) = self.publication_preflight(&packet, host)?; let crate::g17::command::Validated::Compute { + intersection, scratch, usc_base, cdm_va, @@ -1540,6 +1596,7 @@ impl Queue { self.graph.ensure_record_slot(slot)?; } let args = ComputeArgs { + intersection: *intersection, scratch: *scratch, usc_base: *usc_base, cdm_va: *cdm_va, diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 5f535a8321cf4..31f2fcd6d016a 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -127,9 +127,9 @@ impl Queue { }) } - fn ensure_compute(&mut self) -> Result { + fn ensure_compute(&mut self, scs_pages: u16) -> Result { // Admission is checked even when this queue already owns a scheduler. - self.backend.ensure_compute()?; + self.backend.ensure_compute(scs_pages)?; if self.compute.is_none() { self.compute = Some(Scheduler::new( &self.dev, @@ -174,7 +174,8 @@ impl Queue { self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { - self.ensure_compute()?; + let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; + self.ensure_compute(scratch.pages())?; } // Only earlier render ordinals covered by the explicit logical prefix // can delay host publication. These links never become GPU dependencies. diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index be22a0cab1321..ecfa57cd9a5a1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -121,10 +121,10 @@ impl Registry { /// replacement context must succeed before its old binding is displaced. /// A vacant queue (USC backing given back) is offered for re-arming only /// when no populated idle queue exists. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc) -> Result { + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize) -> Result { if let Some(queue) = self.compute(owner) { return Ok(if queue.matches(owner, queue.qid(), context)? { - Rebind::Done + if queue.pool_fits(required) { Rebind::Done } else { Rebind::Grow } } else { Rebind::Fresh }); @@ -141,6 +141,7 @@ impl Registry { vacant.get_or_insert(queue.qid()); continue; } + if !queue.pool_fits(required) { continue; } queue.bind_owner(owner, context.clone())?; return Ok(Rebind::Done); } @@ -365,6 +366,7 @@ impl Registry { pub(super) enum Rebind { /// The owner now holds an installed, populated queue. Done, + Grow, /// Only a vacant queue is available; re-arm `qid` with fresh backing. Rearm(u8), /// No retained queue suits the owner: reserve a fresh QID. @@ -549,7 +551,52 @@ impl Backend { result } - fn build_compute(&self) -> Result { + fn grow_compute(&self, required: usize) -> Result { + use kernel::time::{delay::fsleep, Delta}; + { + let state = self.shared.state.lock(); + let firmware = state.as_ref().ok_or(ENODEV)?; + if firmware.queues.compute(self.owner).is_some_and(|queue| queue.pool_fits(required)) { + return Ok(()); + } + } + let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; + let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); + let alloc = super::object::Allocator { dev: &dev, uat: &uat }; + let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); + let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); + let mut deferred = DeferredBatch::publication()?; + let started = now_ns(); + loop { + let result = (|| { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { return Err(EAGAIN); } + if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } + firmware.scan_compute(None, &mut deferred)?; + let queue = firmware.queues.compute_mut(self.owner).ok_or(EIO)?; + if queue.pool_fits(required) { return Ok(None); } + if queue.begin_growth(&firmware.init, required)? { + fence(Ordering::SeqCst); + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } + queue.finish_growth(&firmware.init, &mut pages, &mut alias, required).map(Some) + })(); + deferred.finish(); + match result { + Ok(old) => { drop(old); return Ok(()); } + Err(error) if error == EAGAIN && lease.is_current() + && now_ns().saturating_sub(started) < WORK_RING_WAIT_NS => { + self.shared.queue_events(); + fsleep(Delta::from_millis(1)); + } + Err(error) => return Err(error), + } + } + } + + fn build_compute(&self, required: usize) -> Result { loop { match self.shared_compute() { Err(error) if error == EAGAIN => continue, @@ -572,8 +619,9 @@ impl Backend { { return Err(EIO); } - match firmware.queues.rebind_compute(self.owner, &self.context)? { + match firmware.queues.rebind_compute(self.owner, &self.context, required)? { Rebind::Done => return Ok(()), + Rebind::Grow => { drop(state); return self.grow_compute(required); } Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), Rebind::Fresh => { let reservation = firmware.queues.reserve_compute(self.owner)?; @@ -590,7 +638,7 @@ impl Backend { Err(qid) => { // Fresh zeroed backing is built without the device mutex; the // queue may have been taken or re-armed by another owner since. - let pages = crate::g17::freelist::Pages::compute(&alloc)?; + let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; let ring = compute::DescriptorRing::new(&alloc)?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -603,13 +651,13 @@ impl Backend { }; // The firmware consumed this slot's release before the backing // was dropped; the row is rewritten before the queue is kicked. - firmware.init.initialize_compute_pool(slot, page_list)?; + firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; firmware.queues.bind_rearmed(qid, self.owner, &self.context)?; return Ok(()); } }; let built = (|| { - let pool = compute::PoolBacking::new(&alloc, reservation.id)?; + let pool = compute::PoolBacking::new(&alloc, reservation.id, required)?; { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -618,7 +666,7 @@ impl Backend { } firmware.queues.reserved(reservation)?; let (slot, page_list) = pool.pool_descriptor()?; - firmware.init.initialize_compute_pool(slot, page_list)?; + firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; } let backing = compute::Backing::new(&alloc, pool)?; // The global scheduler is created after pool/kick backing and @@ -1005,7 +1053,8 @@ impl super::Firmware { } impl Backend { - fn ensure_compute(&self) -> Result { + fn ensure_compute(&self, scs_pages: u16) -> Result { + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; use kernel::time::{delay::fsleep, Delta}; const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; @@ -1021,7 +1070,7 @@ impl Backend { deferred.finish(); let pending = pending?; if !pending { - return self.build_compute(); + return self.build_compute(required); } // Other owners continue submitting while this owner gives its // queue-local drain witness time to arrive. diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index ef72699e8e7ce..7509cf539c278 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -70,8 +70,8 @@ impl job::Backend for Backend { Some(self.shared.feed.clone()) } - fn ensure_compute(&self) -> Result { - Backend::ensure_compute(self) + fn ensure_compute(&self, scs_pages: u16) -> Result { + Backend::ensure_compute(self, scs_pages) } fn fail_vm(&self, error: Error) { diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 437ad42aeafda..1053fc26d17c0 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -1453,7 +1453,11 @@ struct drm_asahi_cmd_compute { /** @ts: Timestamps for the compute command */ struct drm_asahi_timestamps ts; - /** @scs_layout: Reserved; must be zero. */ + /** + * @scs_layout: Packed G17P Gen1 shader-context scratch layout, or zero. + * Bits 0:15 contain stack depth, 16:23 levels, 24:31 cliques, + * 32:47 user bytes and 48:63 driver bytes. Non-G17P paths require zero. + */ __u64 scs_layout; /** @@ -1473,6 +1477,23 @@ struct drm_asahi_cmd_compute { * queue's compute command count and the header's cdm_barrier instead. */ __u64 prior_cdm; + + /** @cdm_auxiliary_allocation: G17P auxiliary allocation, zero or 8..32. */ + __u32 cdm_auxiliary_allocation; + /** @reserved: MBZ. */ + __u32 reserved; + /** @intersection_pool: 16-KiB aligned G17P intersection-table address. */ + __u64 intersection_pool; + /** @intersection_pool_bytes: Mapped pool size in bytes, a multiple of 16 KiB. */ + __u64 intersection_pool_bytes; + /** @intersection_first: First 128-byte entry in @intersection_pool. */ + __u32 intersection_first; + /** @intersection_count: Number of 128-byte entries, less than 2^18. */ + __u32 intersection_count; + /** @intersection_default_program: Encoded default G17P program address. */ + __u64 intersection_default_program; + /** @intersection_default_control: Default G17P program control word. */ + __u64 intersection_default_control; }; /** From f64c279049b2c334fb2651812de701f6e6bf16bc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:15:50 +1100 Subject: [PATCH 0881/1579] drm: apple: isolate an experimental external H17P transport Add a separate external H17P method profile and compact service serializer for bounded trials. Keep the existing internal H17P send rejection and boot-mode inheritance unchanged. Restrict the new serializer to the existing DisplayPort and EDID service calls, validate ring occupancy, and reject unknown callbacks without fabricating success. The compact transmit header and existing command descriptor are protocol hypotheses derived from public interfaces. No hardware compatibility is claimed. This commit does not enable or match an external controller. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 130 +++++++++++++++++++------ drivers/gpu/drm/apple/afk.h | 2 + drivers/gpu/drm/apple/dcp-internal.h | 3 + drivers/gpu/drm/apple/dcp.c | 10 ++ drivers/gpu/drm/apple/dcp.h | 2 + drivers/gpu/drm/apple/dptxep.c | 6 +- drivers/gpu/drm/apple/iomfb.c | 19 +++- drivers/gpu/drm/apple/iomfb_template.c | 8 +- 8 files changed, 148 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index ff1f0f57afd9f..8161db7eb440f 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -572,8 +573,10 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %d received unhandled standard service message: %x / %x\n", ep->endpoint, channel, type, eshdr->category); - print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, - payload_size, true); + if (ep->dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, + payload, payload_size, true); } static bool afk_validate_h17p_header(const u8 *data, size_t size) @@ -645,6 +648,16 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, payload_size = data_size - hdr_len; trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); + if (ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + ep->experimental_rx_logs < 32) { + ep->experimental_rx_logs++; + dev_info(ep->dcp->dev, + "external AFK ep=%u channel=%u type=%u subtype=%u category=%u bytes=%zu\n", + ep->endpoint, channel, type, subtype, eshdr->category, + data_size); + } + service = afk_epic_find_service(ep, channel); @@ -698,8 +711,10 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %d received unhandled message " "(type %x subtype %x)\n", ep->endpoint, channel, type, subtype); - print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, - payload_size, true); + if (ep->dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, + payload, payload_size, true); return; too_small: @@ -878,6 +893,8 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, struct epic_sub_hdr *eshdr; unsigned long flags; size_t total_epic_size, total_size; + bool compact = ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; int ret; /* @@ -885,30 +902,73 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, * 8-byte sub-header, with the service channel carried in the EPIC * header on queue channel 0. Only the receive side of that format is * known, and the 24-byte framing below would be misparsed, so send - * nothing. The display path needs no AP-initiated EPIC traffic: the + * nothing for the established internal profile. The external-only + * experiment below tests receive-layout symmetry. Internal callers are: the * callers are optional (verbose firmware logging, EDID copy, audio, * DPTX, debugfs) or replies to calls only DPTX services make. */ - if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && !compact) { dev_dbg_once(ep->dcp->dev, "AFK[ep:%02x]: not sending EPIC message on H17P\n", ep->endpoint); return -EOPNOTSUPP; } + /* Only existing DPTX/EDID service commands participate in this trial. */ + if (compact && + (channel > U16_MAX || payload_len > SZ_1M || + (payload_len && !payload) || stype != EPIC_SUBTYPE_STD_SERVICE || + (ep->endpoint != DPTX_ENDPOINT && ep->endpoint != DPAVSERV_ENDPOINT) || + !((ecat == EPIC_CAT_COMMAND && etype == EPIC_TYPE_COMMAND && + payload_len == sizeof(struct epic_cmd)) || + (ecat == EPIC_CAT_REPLY && + (etype == EPIC_TYPE_REPLY || etype == EPIC_TYPE_NOTIFY_ACK))))) + return -EOPNOTSUPP; + spin_lock_irqsave(&ep->lock, flags); if (ep->stopping) { ret = -ESHUTDOWN; goto out; } + if (compact && (!ep->txbfr.ready || !ep->txbfr.rptr || + !ep->txbfr.wptr || !ep->txbfr.buf)) { + ret = -ESHUTDOWN; + goto out; + } + dma_rmb(); rptr = le32_to_cpu(*ep->txbfr.rptr); wptr = le32_to_cpu(*ep->txbfr.wptr); trace_afk_send_rwptr_pre(ep, rptr, wptr); - total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; + total_epic_size = sizeof(*ehdr) + + (compact ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + + payload_len; total_size = sizeof(*hdr) + total_epic_size; + if (compact) { + u32 available, needed; + + if (!ep->txbfr.ready || !ep->txbfr.stride || + (ep->txbfr.stride & (ep->txbfr.stride - 1)) || + rptr >= ep->txbfr.bufsz || wptr >= ep->txbfr.bufsz || + !IS_ALIGNED(rptr, ep->txbfr.stride) || + !IS_ALIGNED(wptr, ep->txbfr.stride) || + total_size >= ep->txbfr.bufsz) { + ret = -EPROTO; + goto out; + } + available = rptr > wptr ? rptr - wptr : ep->txbfr.bufsz - wptr + rptr; + needed = ALIGN(total_size, ep->txbfr.stride); + if (total_size > ep->txbfr.bufsz - wptr) + needed += ep->txbfr.bufsz - wptr; + /* Retain an empty slot; equal read/write pointers mean empty. */ + if (needed >= available) { + ret = -ENOSPC; + goto out; + } + } + hdr = hdr2 = NULL; /* @@ -989,31 +1049,45 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, hdr->magic = cpu_to_le32(QE_MAGIC); hdr->size = cpu_to_le32(total_epic_size); - hdr->channel = cpu_to_le32(channel); + hdr->channel = cpu_to_le32(compact ? 0 : channel); hdr->type = cpu_to_le32(etype); if (hdr2) memcpy(hdr2, hdr, sizeof(*hdr)); - ehdr = ep->txbfr.buf + wptr; - memset(ehdr, 0, sizeof(*ehdr)); - ehdr->version = 2; - ehdr->seq = cpu_to_le16(ep->qe_seq++); - ehdr->timestamp = cpu_to_le64(0); - wptr += sizeof(*ehdr); - - eshdr = ep->txbfr.buf + wptr; - memset(eshdr, 0, sizeof(*eshdr)); - eshdr->length = cpu_to_le32(payload_len); - eshdr->version = 4; - eshdr->category = ecat; - eshdr->type = cpu_to_le16(stype); - eshdr->timestamp = cpu_to_le64(0); - eshdr->tag = cpu_to_le16(tag); - if (ecat == EPIC_CAT_REPLY) - eshdr->inline_len = cpu_to_le32(payload_len - 4); - else - eshdr->inline_len = cpu_to_le32(0); - wptr += sizeof(*eshdr); + if (compact) { + u8 *wire = ep->txbfr.buf + wptr; + + /* Experimental symmetry with the published H17P receive layout. */ + memset(wire, 0, sizeof(*ehdr) + sizeof(struct epic_sub_hdr_h17p)); + wire[0] = ep->qe_seq++; + put_unaligned_le16(channel, wire + 2); + put_unaligned_le32(total_epic_size - 8, wire + 4); + wire[16] = stype; + wire[17] = ecat; + put_unaligned_le32(tag, wire + 20); + wptr += sizeof(*ehdr) + sizeof(struct epic_sub_hdr_h17p); + } else { + ehdr = ep->txbfr.buf + wptr; + memset(ehdr, 0, sizeof(*ehdr)); + ehdr->version = 2; + ehdr->seq = cpu_to_le16(ep->qe_seq++); + ehdr->timestamp = cpu_to_le64(0); + wptr += sizeof(*ehdr); + + eshdr = ep->txbfr.buf + wptr; + memset(eshdr, 0, sizeof(*eshdr)); + eshdr->length = cpu_to_le32(payload_len); + eshdr->version = 4; + eshdr->category = ecat; + eshdr->type = cpu_to_le16(stype); + eshdr->timestamp = cpu_to_le64(0); + eshdr->tag = cpu_to_le16(tag); + if (ecat == EPIC_CAT_REPLY) + eshdr->inline_len = cpu_to_le32(payload_len - 4); + else + eshdr->inline_len = cpu_to_le32(0); + wptr += sizeof(*eshdr); + } memcpy(ep->txbfr.buf + wptr, payload, payload_len); wptr += payload_len; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 0cef14fa6dfe8..8eb797e12ef07 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -291,6 +291,8 @@ struct afk_ringbuffer { struct apple_dcp_afkep { struct apple_dcp *dcp; + /* Bounded external protocol observations, owned by the RX worker. */ + u8 experimental_rx_logs; u32 endpoint; struct workqueue_struct *wq; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 8be2f4f267cfa..3f4a5620999e3 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -174,6 +174,7 @@ struct dcp_panel { enum dcp_iomfb_method_profile { DCP_IOMFB_METHODS_DEFAULT, DCP_IOMFB_METHODS_H17G, + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL, }; struct apple_dcp_hw_data { @@ -208,6 +209,8 @@ struct apple_dcp { struct apple_connector *connector; struct apple_dcp_hw_data hw; + /* Protected by the H17P IOMFB receive mutex. */ + u8 experimental_callback_logs; /* firmware version and compatible firmware version */ enum dcp_firmware_version fw_compat; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 9228483aed8d6..8cccb95efdfc3 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -130,6 +130,16 @@ static void dcp_release_rtkit(void *data) #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) +static bool experimental_h17p_external; +module_param(experimental_h17p_external, bool, 0444); +MODULE_PARM_DESC(experimental_h17p_external, + "Enable experimental T8140 external display protocol"); + +bool dcp_h17p_external_enabled(void) +{ + return experimental_h17p_external; +} + static bool show_notch; module_param(show_notch, bool, 0644); MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index a2ef5176f3fee..bbd14f3316e6e 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -12,6 +12,8 @@ #include "dcp-internal.h" #include "parser.h" +bool dcp_h17p_external_enabled(void); + struct apple_crtc { struct drm_crtc base; struct drm_pending_vblank_event *event; diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 11091084f05e2..ec190a7461df9 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -694,7 +694,8 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, trace_dptxport_apcall(dptx, idx, data_size); dev_dbg(service->ep->dcp->dev, "DPTXPort: APCALL %u (%zu bytes)\n", idx, data_size); - if (data_size) + if (data_size && service->ep->dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) print_hex_dump_debug("DPTXPort: apcall data: ", DUMP_PREFIX_OFFSET, 16, 1, data, min(data_size, (size_t)64), true); @@ -792,6 +793,9 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, memset(reply, 0, 4); return 0; default: + if (service->ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + return -EOPNOTSUPP; /* just try to ACK and hope for the best... */ dev_info(service->ep->dcp->dev, "DPTXPort: acking unhandled call %u\n", idx); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8c8cfbb5cc579..ac23f92329f74 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -790,6 +790,14 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, } tag = dcp_parse_tag(hdr->tag); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + dcp->experimental_callback_logs < 64) { + dcp->experimental_callback_logs++; + dev_info(dev, "external IOMFB callback=%d in=%u out=%u context=%u\n", + tag, hdr->in_len, hdr->out_len, context); + } + handled = tag >= 0 && tag < IOMFB_MAX_CB && dcp->cb_handlers && dcp->cb_handlers[tag]; if (dcp->fw_compat == DCP_FIRMWARE_H17P && @@ -812,6 +820,13 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, !test_and_set_bit(tag, dcp->unknown_callbacks)) dev_warn(dev, "received unknown callback %c%c%c%c\n", hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + /* Retain the failed session without inventing a successful reply. */ + dcp->crashed = true; + complete(&dcp->start_done); + return; + } /* * Leaving a callback unanswered wedges the coprocessor: it * waits for the ack forever and the outer call never returns. @@ -1041,7 +1056,9 @@ static bool dcp_mode_settable(struct apple_dcp *dcp, if (!dcp_mode) return false; - if (dcp->fw_compat == DCP_FIRMWARE_H17P) + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) return dcp_mode->mode.type & DRM_MODE_TYPE_PREFERRED; return true; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index f14031085142f..575b99606e1ac 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2194,10 +2194,14 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, dcp->swap_submit_timestamp = 0; - if (DCP_INHERIT_BOOT_MODE) { + if (DCP_INHERIT_BOOT_MODE && + dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { dev_dbg(dcp->dev, "inheriting the bootloader's display mode\n"); complete_set_digital_out_mode(dcp, NULL, cookie); - } else if (mode->vrr) + } else if (mode->vrr && + dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) dcp_set_adaptive_sync(dcp, crtc_state->vrr_enabled ? mode->min_vrr : 0, cookie); From b65e24146a0ef604d3b1a6ae96c30c39888ee8e9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:15:50 +1100 Subject: [PATCH 0882/1579] drm: apple: prepare optional external H17P before component matching Add a default-off T8140 external-controller experiment. Require the inherited START, STOPPED and IDLE state before the standard start/wake path and pin its DMA lifetime before any startup write. Complete this optional preparation during platform probe so failed or deferred controllers do not become prerequisites for the internal DRM master. Only successfully bound, prepared controllers are included. Preserve started-session mappings on failure and quiesce retained external work during shutdown. Internal H17P lifecycle remains unchanged. Hardware qualification and validated external resource mappings remain required before this experiment can be used as a supported display path. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 3 +- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 100 ++++++++++++++++++++++++++- drivers/gpu/drm/apple/dcp.h | 1 + 4 files changed, 101 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index df31009e3ff6b..3e1cfa1a69b2c 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -648,7 +648,8 @@ static const struct of_device_id apple_dcp_id_tbl[] = { static bool apple_dcp_usable(struct device_node *np) { return of_device_is_available(np) && - !of_device_is_compatible(np, "apple,t8140-dcpext"); + (!of_device_is_compatible(np, "apple,t8140-dcpext") || + dcp_h17p_external_ready(np)); } static int apple_drm_init_dcp(struct device *dev) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 3f4a5620999e3..f552a89bda977 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -211,6 +211,7 @@ struct apple_dcp { struct apple_dcp_hw_data hw; /* Protected by the H17P IOMFB receive mutex. */ u8 experimental_callback_logs; + bool external_prepared; /* firmware version and compatible firmware version */ enum dcp_firmware_version fw_compat; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8cccb95efdfc3..c973d15a7b9c1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -50,6 +51,9 @@ #define APPLE_DCP_COPROC_CPU_CONTROL 0x44 #define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) +#define APPLE_DCP_COPROC_CPU_STATUS 0x48 +#define APPLE_DCP_COPROC_CPU_STATUS_STOPPED BIT(1) +#define APPLE_DCP_COPROC_CPU_STATUS_IDLE BIT(5) struct dcp_session_ref { struct list_head link; @@ -80,7 +84,10 @@ static int dcp_pin_live_session(struct apple_dcp *dcp) { struct dcp_session_ref *session; - if (!dcp->hw.adopt_live_session || dcp->fw_compat != DCP_FIRMWARE_H17P || + if ((!dcp->hw.adopt_live_session && + dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) || + dcp->fw_compat != DCP_FIRMWARE_H17P || dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || dcp->retain_dma) return 0; session = kzalloc_obj(*session); @@ -140,6 +147,27 @@ bool dcp_h17p_external_enabled(void) return experimental_h17p_external; } +bool dcp_h17p_external_ready(struct device_node *np) +{ + struct platform_device *pdev; + struct apple_dcp *dcp; + bool ready = false; + + if (!dcp_h17p_external_enabled()) + return false; + pdev = of_find_device_by_node(np); + if (!pdev) + return false; + /* Failed and deferred probes must never become required components. */ + if (device_is_bound(&pdev->dev)) { + dcp = platform_get_drvdata(pdev); + ready = dcp && READ_ONCE(dcp->external_prepared) && + !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->crashed); + } + put_device(&pdev->dev); + return ready; +} + static bool show_notch; module_param(show_notch, bool, 0644); MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); @@ -3286,6 +3314,17 @@ static void dcp_enable_typec_work(struct apple_dcp *dcp) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); } +static bool dcp_external_is_idle(struct apple_dcp *dcp) +{ + u32 control = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + u32 status = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS); + + /* Same externally inherited idle state accepted by the public loader. */ + return (control & APPLE_DCP_COPROC_CPU_CONTROL_RUN) && + (status & APPLE_DCP_COPROC_CPU_STATUS_STOPPED) && + (status & APPLE_DCP_COPROC_CPU_STATUS_IDLE); +} + static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; @@ -3297,6 +3336,9 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) return dev_err_probe(dev, -EBUSY, "Live DCP session cannot be rebound; reboot required\n"); + if (dcp->external_prepared) + return 0; + /* A timed-out prior session may still scan these mappings. */ mutex_lock(&dcp->swapped_out_fbs_lock); ret = list_empty(&dcp->swapped_out_fbs) ? 0 : -EBUSY; @@ -3327,6 +3369,12 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) "DCP was not left running by the bootloader\n"); } + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + !dcp_external_is_idle(dcp)) + return dev_err_probe(dev, -EBUSY, + "External DCP is not in the inherited idle state\n"); + if (dcp->index || dcp->dptx_phy || dcp->dptx_die) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); @@ -3407,6 +3455,19 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + /* Pin before the first write or mailbox wake can expose DMA. */ + ret = dcp_pin_live_session(dcp); + if (ret) + goto err_piodma; + if (!dcp_external_is_idle(dcp)) { + ret = dev_err_probe(dev, -EBUSY, + "External DCP changed state before startup\n"); + goto err_piodma; + } + } + if (!dcp->hw.adopt_live_session) { cpu_ctrl = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); @@ -3557,7 +3618,8 @@ static int dcp_platform_probe(struct platform_device *pdev) return dev_err_probe(dev, -EBUSY, "Previous live DCP session requires a reboot\n"); /* Not part of the display subsystem yet; see apple_dcp_usable(). */ - if (of_device_is_compatible(dev->of_node, "apple,t8140-dcpext")) { + if (of_device_is_compatible(dev->of_node, "apple,t8140-dcpext") && + !dcp_h17p_external_enabled()) { dev_info(dev, "external display coprocessor not supported yet\n"); return -ENODEV; } @@ -3759,7 +3821,24 @@ static int dcp_platform_probe(struct platform_device *pdev) if (ret) return ret; - return component_add(&pdev->dev, &dcp_comp_ops); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + /* Complete optional startup before the DRM master can depend on it. */ + ret = dcp_comp_bind(dev, NULL, NULL); + if (ret) { + dcp_disable_typec_work(dcp, true); + disable_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + return ret; + } + WRITE_ONCE(dcp->external_prepared, true); + } + + ret = component_add(&pdev->dev, &dcp_comp_ops); + if (ret && dcp->external_prepared) { + dcp_disable_typec_work(dcp, true); + disable_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + } + return ret; } static void dcp_platform_remove(struct platform_device *pdev) @@ -3771,6 +3850,12 @@ static void dcp_platform_shutdown(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (dcp && dcp->retain_dma && dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + dcp_comp_unbind(&pdev->dev, NULL, NULL); + return; + } + if (dcp && dcp->hw.adopt_live_session && dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { @@ -3909,6 +3994,14 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { .firmware_request = 0x68000, }; +/* Separate from the established internal-panel H17P profile. */ +static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external = { + .num_dptx_ports = 1, + .iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL, + .adopt_live_session = false, + .firmware_compat = DCP_FIRMWARE_H17P, +}; + /* * M5 (T8142) runs H17-generation DCP firmware with the H17G method numbering * and is left running by the bootloader. @@ -3932,6 +4025,7 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, { .compatible = "apple,t8140-dcp", .data = &apple_dcp_hw_t8140, }, + { .compatible = "apple,t8140-dcpext", .data = &apple_dcp_hw_t8140_external, }, { .compatible = "apple,t8142-dcp", .data = &apple_dcp_hw_t8142, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index bbd14f3316e6e..df1e46c439697 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -13,6 +13,7 @@ #include "parser.h" bool dcp_h17p_external_enabled(void); +bool dcp_h17p_external_ready(struct device_node *np); struct apple_crtc { struct drm_crtc base; From c67346da36d614fd267714dbc634d7489e58dce0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:18:18 +1100 Subject: [PATCH 0883/1579] drm: apple: drain the work items present in the pinned base The optional external error path referenced a retrain work item absent from this source base. Use dcp_disable_typec_work(), which already blocks and drains both existing Type-C work items. Preserve the initial compilation failure for review. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c973d15a7b9c1..96ce0e741c055 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3827,7 +3827,6 @@ static int dcp_platform_probe(struct platform_device *pdev) ret = dcp_comp_bind(dev, NULL, NULL); if (ret) { dcp_disable_typec_work(dcp, true); - disable_delayed_work_sync(&dcp->typec_fabric_retrain_wq); return ret; } WRITE_ONCE(dcp->external_prepared, true); @@ -3836,7 +3835,6 @@ static int dcp_platform_probe(struct platform_device *pdev) ret = component_add(&pdev->dev, &dcp_comp_ops); if (ret && dcp->external_prepared) { dcp_disable_typec_work(dcp, true); - disable_delayed_work_sync(&dcp->typec_fabric_retrain_wq); } return ret; } From 40a9c929c55f2b13c39e732fc2ae3f10e1506634 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:20:49 +1100 Subject: [PATCH 0884/1579] arm64: dts: apple: describe disabled J700 external resources Describe the external display controller and four display apertures plus its fifth clock aperture from bounded read-only ADT observations. Select the first observed DCP mapper and the observed PIODMA stream. Add the reciprocal rear Type-C graph while retaining disabled controller and route status. Register names follow existing public bindings. Mailbox, firmware handoff, routing and bandwidth integration remain qualification requirements. This resource description is not a display readiness claim. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- .../boot/dts/apple/t8140-j700-dcpext.dtsi | 69 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8140-j700.dts | 3 + 2 files changed, 72 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi new file mode 100644 index 0000000000000..7f3adb838473f --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * T8140 external display resources measured through the public ADT API. + * Generated-by: LLM + * Assisted-by: LLM + * + * The controller and route remain disabled. Their live firmware + * profile, bandwidth request, routing selector and handoff must be qualified + * before enabling the controller. The existing T8140 refusal remains in force. + */ + +&{/soc} { + dcpext: dcp@416e00000 { + compatible = "apple,t8140-dcpext"; + reg = <0x4 0x16e00000 0x0 0x4000>, + <0x4 0x14000000 0x0 0xafc000>, + <0x4 0x16240000 0x0 0x10000>, + <0x4 0x163ac000 0x0 0x4000>, + <0x4 0x16800000 0x0 0x800000>, + <0x3 0x02800000 0x0 0xbc000>, + <0x4 0x16850000 0x0 0x4000>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4", "iop-vbar"; + mboxes = <&dcpext_mbox>; + mbox-names = "mbox"; + /* First mapper in the measured ADT iommu-parent list. */ + iommus = <&dcpext0_dart 23>; + power-domains = <&ps_dispext0_cpu>; + apple,connector-type = "USB-C"; + status = "disabled"; + + #address-cells = <2>; + #size-cells = <2>; + ranges; + + dispext0_piodma: piodma { + iommus = <&dispext0_dart 16>; + }; + + typec-routes { + #address-cells = <1>; + #size-cells = <0>; + + route@0 { + reg = <0>; + status = "disabled"; + mode-switch; + + port { + dcpext_typec0: endpoint { + remote-endpoint = <&typec0_dcp>; + }; + }; + }; + }; + }; +}; + +&typec0 { + ports { + port@3 { + reg = <3>; + + typec0_dcp: endpoint { + remote-endpoint = <&dcpext_typec0>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 842e1b2bcf572..5b102fe5fcbba 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -1918,3 +1918,6 @@ local-mac-address = [00 00 00 00 00 00]; }; }; + +/* External resources stay disabled pending the separate firmware profile. */ +#include "t8140-j700-dcpext.dtsi" From 98038492dc481d784c76fb2c52f3dd030dfad8f5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:30:45 +1100 Subject: [PATCH 0885/1579] drm: apple: recognize the T8140 external binding in the display master Use the generic DCP fallback required by the published T8140 external binding and explicitly classify that specific compatible as an external pipeline. Otherwise the display master never discovers a successfully prepared external node. The device and route remain disabled by default. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi | 2 +- drivers/gpu/drm/apple/apple_drv.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi index 7f3adb838473f..cb3e7ea4ebdc8 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -11,7 +11,7 @@ &{/soc} { dcpext: dcp@416e00000 { - compatible = "apple,t8140-dcpext"; + compatible = "apple,t8140-dcpext", "apple,dcp"; reg = <0x4 0x16e00000 0x0 0x4000>, <0x4 0x14000000 0x0 0xafc000>, <0x4 0x16240000 0x0 0x10000>, diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 3e1cfa1a69b2c..eda6ea33fa225 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -667,6 +667,7 @@ static int apple_drm_init_dcp(struct device *dev) if (!apple_dcp_usable(np)) continue; dcp_ext = of_device_is_compatible(np, "apple,dcpext") || + of_device_is_compatible(np, "apple,t8140-dcpext") || of_property_present(np, "phys"); dcp[num_dcp] = of_find_device_by_node(np); From 205a56d33cb07184d7dfe17ab8aba0996cff3efe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:31:34 +1100 Subject: [PATCH 0886/1579] arm64: dts: apple: retain the inherited external display supply Keep the external CPU supply on after the loader validates and publishes its power-domain chain. The supply remains disabled in the static tree, so a refused handoff does not register or power it. This preserves firmware ownership while allowing mailbox consumers to bind. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi index cb3e7ea4ebdc8..6a6538dc6dba6 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -67,3 +67,8 @@ }; }; }; + +/* Retain the inherited firmware supply once the loader publishes it. */ +&ps_dispext0_cpu { + apple,always-on; +}; From 24c20cf0924ec8a4096cb41d013713edbbdd88c2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:40:04 +1100 Subject: [PATCH 0887/1579] drm: apple: validate external display clock and bandwidth resources Separate the number of display windows from the shared firmware clock aperture. Preserve the internal five-window profile and validate the external four-window profile against its existing final clock resource. Require explicit read-only experimental bandwidth offsets for the external profile, with alignment, distinctness and aperture bounds checks before startup. Keep both offsets unset by default and reject missing external clock rates rather than guessing them. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-external-bw.h | 24 ++++++++ drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 56 ++++++++++++++++++- .../drivers/gpu/test_t8140_external_bw.py | 45 +++++++++++++++ 4 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-external-bw.h create mode 100644 tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py diff --git a/drivers/gpu/drm/apple/dcp-external-bw.h b/drivers/gpu/drm/apple/dcp-external-bw.h new file mode 100644 index 0000000000000..439db008b5084 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-external-bw.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Aurora Silicon */ +#ifndef APPLE_DCP_EXTERNAL_BW_H +#define APPLE_DCP_EXTERNAL_BW_H + +#include + +static inline bool apple_dcp_external_bw_valid(u64 size, int scratch, int request) +{ + if (size < sizeof(u32) || scratch < 0 || request < 0) + return false; + if ((scratch & 3) || (request & 3) || scratch == request) + return false; + return (u64)scratch <= size - sizeof(u32) && + (u64)request <= size - sizeof(u32); +} + +static inline bool apple_dcp_external_clock_window(u64 start, u64 size, + u64 expected_start, u64 expected_size) +{ + return start == expected_start && size == expected_size; +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index f552a89bda977..ba790968a2dd1 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -188,6 +188,7 @@ struct apple_dcp_hw_data { enum dcp_firmware_version firmware_compat; /* Firmware-visible clock aperture, independent of the AP PMGR window. */ const struct resource *firmware_clock; + u32 firmware_display_windows; u32 firmware_scratch; u32 firmware_request; }; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 96ce0e741c055..3f60b70042202 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -45,6 +45,7 @@ #include "av.h" #include "dcp.h" #include "dcp-internal.h" +#include "dcp-external-bw.h" #include "iomfb.h" #include "parser.h" #include "trace.h" @@ -142,6 +143,16 @@ module_param(experimental_h17p_external, bool, 0444); MODULE_PARM_DESC(experimental_h17p_external, "Enable experimental T8140 external display protocol"); +static int experimental_h17p_external_bw_scratch = -1; +module_param(experimental_h17p_external_bw_scratch, int, 0444); +MODULE_PARM_DESC(experimental_h17p_external_bw_scratch, + "Experimental external bandwidth scratch offset; unset by default"); + +static int experimental_h17p_external_bw_request = -1; +module_param(experimental_h17p_external_bw_request, int, 0444); +MODULE_PARM_DESC(experimental_h17p_external_bw_request, + "Experimental external bandwidth request offset; unset by default"); + bool dcp_h17p_external_enabled(void) { return experimental_h17p_external; @@ -3158,8 +3169,25 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) } if (dcp->hw.firmware_clock) { - /* The internal firmware's last aperture follows five display windows. */ - if (count != 5 || resource_size(dcp->hw.firmware_clock) < 4 || + unsigned int windows = dcp->hw.firmware_display_windows; + bool external = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + + if (!windows || windows >= MAX_DISP_REGISTERS) + return -EINVAL; + if (external) { + /* The external clock aperture is already described by disp-4. */ + if (count != windows + 1) + return -EINVAL; + res = dcp->disp_registers[windows]; + if (!apple_dcp_external_clock_window(res->start, + resource_size(res), + dcp->hw.firmware_clock->start, + resource_size(dcp->hw.firmware_clock))) + return -EINVAL; + count = windows; + } + if (count != windows || resource_size(dcp->hw.firmware_clock) < 4 || dcp->hw.firmware_scratch > resource_size(dcp->hw.firmware_clock) - 4 || dcp->hw.firmware_request > resource_size(dcp->hw.firmware_clock) - 4) return -EINVAL; @@ -3448,6 +3476,15 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) goto err_piodma; } + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + (!dcp->clk_194 || !clk_get_rate(dcp->clk) || + !clk_get_rate(dcp->clk_194))) { + ret = dev_err_probe(dev, -EINVAL, + "External display clocks are unavailable\n"); + goto err_piodma; + } + bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry set_bit(0, dcp->memdesc_map); @@ -3662,6 +3699,18 @@ static int dcp_platform_probe(struct platform_device *pdev) spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + if (!dcp->hw.firmware_clock || + !apple_dcp_external_bw_valid(resource_size(dcp->hw.firmware_clock), + experimental_h17p_external_bw_scratch, + experimental_h17p_external_bw_request)) + return dev_err_probe(dev, -EINVAL, + "External bandwidth offsets are unset or invalid\n"); + dcp->hw.firmware_scratch = experimental_h17p_external_bw_scratch; + dcp->hw.firmware_request = experimental_h17p_external_bw_request; + } + /* The domain use must survive driver-core DMA cleanup for a live session. */ ret = iommu_device_use_default_domain(dev); @@ -3988,6 +4037,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { .adopt_live_session = true, .firmware_compat = DCP_FIRMWARE_H17P, .firmware_clock = &t8140_firmware_clock, + .firmware_display_windows = 5, .firmware_scratch = 0x20000, .firmware_request = 0x68000, }; @@ -3998,6 +4048,8 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external = { .iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL, .adopt_live_session = false, .firmware_compat = DCP_FIRMWARE_H17P, + .firmware_clock = &t8140_firmware_clock, + .firmware_display_windows = 4, }; /* diff --git a/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py new file mode 100644 index 0000000000000..73f0dba8d4d22 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: MIT +import pathlib +import subprocess +import tempfile +import unittest + + +class BandwidthBounds(unittest.TestCase): + def test_shared_header_boundaries(self): + root = pathlib.Path(__file__).resolve().parents[5] + code = r""" +#include +#include +#include "dcp-external-bw.h" +int main(void) { + const u64 n=0xbc000; + assert(!apple_dcp_external_bw_valid(n,-1,-1)); + assert(!apple_dcp_external_bw_valid(n,0,-1)); + assert(!apple_dcp_external_bw_valid(n,-1,0)); + assert(!apple_dcp_external_bw_valid(n,4,4)); + assert(!apple_dcp_external_bw_valid(n,2,4)); + assert(!apple_dcp_external_bw_valid(n,4,6)); + assert(!apple_dcp_external_bw_valid(n,n,4)); + assert(!apple_dcp_external_bw_valid(n,0,n)); + assert(!apple_dcp_external_bw_valid(3,0,4)); + assert(!apple_dcp_external_bw_valid(n,INT_MAX,0)); + assert(apple_dcp_external_bw_valid(n,0,4)); + assert(apple_dcp_external_bw_valid(n,n-4,0)); + assert(apple_dcp_external_bw_valid(n,0,n-4)); + assert(apple_dcp_external_clock_window(0x302800000ULL,n,0x302800000ULL,n)); + assert(!apple_dcp_external_clock_window(0x302800000ULL,n-4,0x302800000ULL,n)); + assert(!apple_dcp_external_clock_window(0x302800004ULL,n,0x302800000ULL,n)); + return 0; +} +""" + with tempfile.TemporaryDirectory() as directory: + tmp=pathlib.Path(directory);(tmp/'linux').mkdir() + (tmp/'linux/types.h').write_text('#include \n#include \ntypedef uint64_t u64; typedef uint32_t u32;\n') + (tmp/'test.c').write_text(code) + subprocess.run(['clang','-std=c11','-Wall','-Wextra','-Werror','-pedantic','-I'+str(tmp),'-I'+str(root/'drivers/gpu/drm/apple'),str(tmp/'test.c'),'-o',str(tmp/'test')],check=True) + subprocess.run([str(tmp/'test')],check=True) + + +if __name__=='__main__': + unittest.main() From 2e6003f6db07b0e0a6b2bf2b977753bee47556b0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:40:12 +1100 Subject: [PATCH 0888/1579] arm64: dts: apple: add an explicit external display trial tree Describe the rear Type-C route and both scanout DARTs in a separate test DTB. Keep the normal J700 tree unchanged. The external DART must be withdrawn by the paired loader when external validation fails. Route selectors and external bandwidth parameters still require hardware qualification. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/Makefile | 1 + .../dts/apple/t8140-j700-external-test.dts | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8140-j700-external-test.dts diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index 06e84a336b5b0..27cb75f1e2856 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -111,4 +111,5 @@ dtb-$(CONFIG_ARCH_APPLE) += t8122-j504.dtb dtb-$(CONFIG_ARCH_APPLE) += t8122-j613.dtb dtb-$(CONFIG_ARCH_APPLE) += t8122-j615.dtb dtb-$(CONFIG_ARCH_APPLE) += t8140-j700.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8140-j700-external-test.dtb dtb-$(CONFIG_ARCH_APPLE) += t8142-j813.dtb diff --git a/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts b/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts new file mode 100644 index 0000000000000..6dde297589d65 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* Explicit external-display trial; the ordinary J700 tree is unchanged. */ +#include "t8140-j700.dts" +#include + +&display { + iommus = <&disp0_dart 0>, <&dispext0_dart 0>; +}; + +&dcpext { + clocks = <&clk_disp0_154>, <&clk_disp0_194>; + clock-names = "clock-154", "clock-194"; + phys = <&atcphy0 PHY_TYPE_DP>; + phy-names = "typec0"; + mux-controls = <&dpxbar0 0>; + mux-control-names = "typec0"; + apple,typec-mux-indices = <0>; + apple,typec-dptx-phys = <0>; + apple,dcp-index = <1>; +}; + +&{/soc/dcp@416e00000/typec-routes/route@0} { + status = "okay"; +}; From b5c2960b4b441fa51b7f0ccdbaa17eae8a9de4bc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:47:51 +1100 Subject: [PATCH 0889/1579] soc: apple: rtkit: identify failed boot handshake stages Report which bounded startup wait failed so endpoint discovery, host readiness and IOP acknowledgment failures can be distinguished without logging message payloads. Successful handshakes remain unchanged. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index eb0da50d092a2..85d70c46015db 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -1034,22 +1034,28 @@ static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) dev_dbg(rtk->dev, "RTKit: waiting for boot to finish\n"); ret = apple_rtkit_wait_for_completion(&rtk->epmap_completion); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: endpoint discovery failed: %d\n", ret); return ret; + } if (rtk->boot_result) return rtk->boot_result; /* Some helpers need AP readiness before acknowledging IOP power. */ if (early_ap_power) { ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: early AP readiness failed: %d\n", ret); return ret; + } } dev_dbg(rtk->dev, "RTKit: waiting for IOP power state ACK\n"); ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: IOP power acknowledgment failed: %d\n", ret); return ret; + } return early_ap_power ? 0 : apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); From 5da6455cdd5182bdd70160d036cd45723e9f5448 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:47:51 +1100 Subject: [PATCH 0890/1579] drm/apple: announce AP readiness early for experimental external H17P The external H17P trial completes endpoint setup but its normal RTKit wake sequence times out. Use the existing early-AP wake helper for this experimental profile, which announces host readiness before waiting for IOP readiness. Keep the normal wake sequence for other profiles. This ordering is a hardware qualification candidate. The previous trial retained the internal panel but did not expose an external connector. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 3f60b70042202..6a9286a2cf562 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3529,7 +3529,12 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (dcp->hw.adopt_live_session) dev_info(dev, "negotiating RTKit with the running DCP\n"); - ret = apple_rtkit_wake(dcp->rtk); + /* Announce host readiness before awaiting external IOP readiness. */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + ret = apple_rtkit_wake_early_ap(dcp->rtk); + else + ret = apple_rtkit_wake(dcp->rtk); if (ret) { ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); goto err_piodma; From e00ebb9cde3c14fc30e252965969b232c683b139 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:54:13 +1100 Subject: [PATCH 0891/1579] drm/apple: acknowledge firmware-owned external H17P buffers The external H17P startup requests a shared buffer at a firmware-provided IOVA. Its setup callback maps that buffer, but the standard RTKit ops do not acknowledge mapped buffers. Select the existing co-processor-owned buffer protocol for this experimental profile so RTKit acknowledges the request and starts the crashlog endpoint before its endpoint-map reply. Keep other display profiles on their existing ops. Restore the normal wake ordering after the earlier AP-readiness trial also timed out, while retaining diagnostics for subsequent qualification. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 6a9286a2cf562..d884a5aeb91bb 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2092,6 +2092,14 @@ static struct apple_rtkit_ops rtkit_ops = { .shmem_destroy = dcp_rtk_shmem_destroy, }; +static const struct apple_rtkit_ops external_h17p_rtkit_ops = { + .flags = APPLE_RTKIT_COPROC_PLACES_BUFFERS, + .crashed = dcp_rtk_crashed, + .recv_message = dcp_recv_msg, + .shmem_setup = dcp_rtk_shmem_setup, + .shmem_destroy = dcp_rtk_shmem_destroy, +}; + void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message) { if (READ_ONCE(dcp->quiescing)) @@ -3516,7 +3524,13 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) ret = dcp_pin_live_session(dcp); if (ret) goto err_piodma; - dcp->rtk = apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); + /* External H17P supplies shared-buffer addresses during RTKit startup. */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + dcp->rtk = apple_rtkit_init(dev, dcp, "mbox", 0, + &external_h17p_rtkit_ops); + else + dcp->rtk = apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); if (IS_ERR(dcp->rtk)) { ret = dev_err_probe(dev, PTR_ERR(dcp->rtk), "Failed to initialize RTKit\n"); @@ -3529,12 +3543,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (dcp->hw.adopt_live_session) dev_info(dev, "negotiating RTKit with the running DCP\n"); - /* Announce host readiness before awaiting external IOP readiness. */ - if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) - ret = apple_rtkit_wake_early_ap(dcp->rtk); - else - ret = apple_rtkit_wake(dcp->rtk); + ret = apple_rtkit_wake(dcp->rtk); if (ret) { ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); goto err_piodma; From fa66e6f2c3899d58951b63cfb47ac1ca932b1955 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:00:10 +1100 Subject: [PATCH 0892/1579] drm/apple: pair external buffer acknowledgments with early AP readiness The early-AP trial still lacked acknowledgments for firmware-owned shared buffers. The buffer-protocol trial supplied those acknowledgments but kept waiting for IOP readiness before announcing AP readiness. Neither trial established that one change alone could complete the external handshake. Combine the existing early-AP helper with the corrected buffer protocol for the experimental external profile. Keep normal display profiles and all timeout and lifetime rules unchanged. Hardware qualification of the combined sequence remains required. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d884a5aeb91bb..5a29d432c7967 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3543,7 +3543,12 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (dcp->hw.adopt_live_session) dev_info(dev, "negotiating RTKit with the running DCP\n"); - ret = apple_rtkit_wake(dcp->rtk); + /* Announce host readiness before awaiting external IOP readiness. */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + ret = apple_rtkit_wake_early_ap(dcp->rtk); + else + ret = apple_rtkit_wake(dcp->rtk); if (ret) { ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); goto err_piodma; From 78fc11047efa814a238113c46b4bb07aa7d641fb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:04:31 +1100 Subject: [PATCH 0893/1579] arm64: dts: apple: give the external J700 trial its firmware DMA window The internal display DART already describes a 1 TiB-based, 64 GiB DMA window. The external trial omitted that range, so its shared buffer was allocated below the firmware window while the internal controller used the high window and completed its power handshake. Describe the same window on the external DART in the test tree. This matches the high address of the external firmware-owned buffer observed during startup. Leave the normal J700 tree unchanged. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700-external-test.dts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts b/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts index 6dde297589d65..d3965ab2c5061 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts @@ -22,3 +22,8 @@ &{/soc/dcp@416e00000/typec-routes/route@0} { status = "okay"; }; + +/* Match the external firmware buffer window rather than allocating below it. */ +&dcpext0_dart { + apple,dma-range = <0x100 0x0 0x10 0x0>; +}; From 60f78724df46169d3d583e5f97a71444c11083da Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:07:14 +1100 Subject: [PATCH 0894/1579] drm/apple: recognize the external H17P display service The external H17P firmware announces dispext-service on the existing IBOOT endpoint. Match that measured service name for the experimental external profile, keeping the normal disp0-service table unchanged. Propagate endpoint allocation and startup failures to the caller. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/ibootep.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/ibootep.c b/drivers/gpu/drm/apple/ibootep.c index ae4bc8a69f2a8..79d5dc19569c3 100644 --- a/drivers/gpu/drm/apple/ibootep.c +++ b/drivers/gpu/drm/apple/ibootep.c @@ -20,10 +20,23 @@ static const struct apple_epic_service_ops ibootep_ops[] = { {} }; +static const struct apple_epic_service_ops external_ibootep_ops[] = { + { + .name = "dispext-service", + .init = disp_service_init, + }, + {} +}; + int ibootep_init(struct apple_dcp *dcp) { - dcp->ibootep = afk_init(dcp, DISP0_ENDPOINT, ibootep_ops); - afk_start(dcp->ibootep); + const struct apple_epic_service_ops *ops = ibootep_ops; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + ops = external_ibootep_ops; + dcp->ibootep = afk_init(dcp, DISP0_ENDPOINT, ops); + if (IS_ERR(dcp->ibootep)) + return PTR_ERR(dcp->ibootep); - return 0; + return afk_start(dcp->ibootep); } From 7739e55cdbb669a3add3cd8b5ee3063377697eb8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:07:15 +1100 Subject: [PATCH 0895/1579] drm/apple: report external AFK startup progress Report external-profile control-message types and ring readiness when startup times out. Exclude normal receive notifications and rate-limit control diagnostics; no payload contents or addresses are logged. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 8161db7eb440f..c7e2baf56a93a 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -146,6 +146,11 @@ int afk_start(struct apple_dcp_afkep *ep) afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_INIT)); ret = wait_for_completion_timeout(&ep->started, msecs_to_jiffies(1000)); + if (ret <= 0 && ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + dev_warn(ep->dcp->dev, + "external AFK startup ep=%#x tx_ready=%u rx_ready=%u bytes=%zu\n", + ep->endpoint, ep->txbfr.ready, ep->rxbfr.ready, ep->bfr_size); if (ret <= 0) return -ETIMEDOUT; else @@ -818,6 +823,11 @@ static void afk_receive_message_worker(struct work_struct *work_) work = container_of(work_, struct afk_receive_message_work, work); type = FIELD_GET(RBEP_TYPE, work->message); + if (work->ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && type != RBEP_RECV) + dev_info_ratelimited(work->ep->dcp->dev, + "external AFK control ep=%#x type=%#x\n", + work->ep->endpoint, type); switch (type) { case RBEP_INIT_ACK: break; From ca886226f17cd09a1ec844da2c944a81dd181d99 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:11:52 +1100 Subject: [PATCH 0896/1579] drm/apple: start the external H17P allocator before service discovery External H17P reaches AFK transport readiness but does not announce the DPTX port service. Its existing allocator endpoint is currently started only later with IOMFB. Make that known endpoint available before external service discovery, and skip its duplicate start during IOMFB startup. Keep normal H17P ordering unchanged. This external-only ordering change requires hardware validation; no new endpoint or message is introduced. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 7 +++++++ drivers/gpu/drm/apple/iomfb.c | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 5a29d432c7967..84761884261e0 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2546,6 +2546,13 @@ int dcp_start(struct platform_device *pdev) return -ENODEV; } + /* Make the H17P allocator available before external service discovery. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); + if (ret) + return dev_err_probe(dcp->dev, ret, "Failed to start remote allocator\n"); + } + /* start RTKit endpoints */ ret = systemep_init(dcp); if (ret) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index ac23f92329f74..1950599effd74 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1563,7 +1563,8 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) * has nothing answering it on the AP side, and without this one the * display DART faults. */ - if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); if (ret) return ret; From f58c17ce883538c28cf14000844e47ed903947a7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:11:52 +1100 Subject: [PATCH 0897/1579] drm/apple: isolate DPTX discovery diagnostics Limit external AFK control diagnostics to DPTX so unrelated endpoint startup cannot consume their rate limit. Distinguish a ready transport with no announced port service from a transport-start timeout. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 3 ++- drivers/gpu/drm/apple/dptxep.c | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index c7e2baf56a93a..c8a6d1426171a 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -824,7 +824,8 @@ static void afk_receive_message_worker(struct work_struct *work_) type = FIELD_GET(RBEP_TYPE, work->message); if (work->ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && type != RBEP_RECV) + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + work->ep->endpoint == DPTX_ENDPOINT && type != RBEP_RECV) dev_info_ratelimited(work->ep->dcp->dev, "external AFK control ep=%#x type=%#x\n", work->ep->endpoint, type); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index ec190a7461df9..571f6f1b03d3e 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -871,10 +871,13 @@ int dptxep_init(struct apple_dcp *dcp) for (port = 0; port < dcp->hw.num_dptx_ports; port++) { ret = wait_for_completion_timeout(&dcp->dptxport[port].enable_completion, timeout); - if (!ret) + if (!ret) { + dev_warn(dcp->dev, "DPTX transport ready but port %u service not announced\n", + port); return -ETIMEDOUT; - else if (ret < 0) + } else if (ret < 0) { return ret; + } timeout = ret; } From 79f124c8e43c4c8bee7879723a88611f539f76f7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:17:13 +1100 Subject: [PATCH 0898/1579] drm/apple: initialize the external DPTX controller before port discovery The public m1n1 display startup initializes the DPTX controller endpoint before the port endpoint. Add that existing AFK controller interface to the experimental external H17P path, wait for its service announcement, and then continue port discovery. Send no controller service commands. Use a dedicated endpoint pointer and route its incoming messages through the existing AFK worker. Cover both retained-session quiescing and normal teardown, and never publish an allocation error as an endpoint pointer. Normal display profiles retain their existing startup sequence. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 3 +++ drivers/gpu/drm/apple/dcp.c | 19 +++++++++++++++ drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/dptxep.c | 35 ++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index ba790968a2dd1..3b99d5b403e9f 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -89,6 +89,7 @@ enum { TEST_ENDPOINT = 0x21, DCP_EXPERT_ENDPOINT = 0x22, DISP0_ENDPOINT = 0x23, + DPTX_CONTROLLER_ENDPOINT = 0x24, DPAVSERV_ENDPOINT = 0x28, AV_ENDPOINT = 0x29, DPTX_ENDPOINT = 0x2a, @@ -357,6 +358,8 @@ struct apple_dcp { struct audiosrv_data *audiosrv; struct apple_dcp_afkep *dptxep; + struct apple_dcp_afkep *dptx_controller_ep; + struct completion dptx_controller_ready; struct dptx_port dptxport[2]; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 84761884261e0..d426bf4c90c14 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1875,6 +1875,8 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) return dcp->dcpavservep; case DPTX_ENDPOINT: return dcp->dptxep; + case DPTX_CONTROLLER_ENDPOINT: + return dcp->dptx_controller_ep; default: return NULL; } @@ -1958,6 +1960,9 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) case DPTX_ENDPOINT: ep = dcp->dptxep; break; + case DPTX_CONTROLLER_ENDPOINT: + ep = dcp->dptx_controller_ep; + break; default: ep = NULL; break; @@ -2571,6 +2576,14 @@ int dcp_start(struct platform_device *pdev) dev_warn(dcp->dev, "Failed to start IBOOT endpoint: %d\n", ret); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + ret = dptx_controller_init(dcp); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to start DPTX controller\n"); + } + ret = dptxep_init(dcp); if (ret) { dev_warn(dcp->dev, "Failed to start DPTX endpoint: %d\n", @@ -3589,6 +3602,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) iomfb_queue_stop(dcp); afk_quiesce(dcp->avep); afk_quiesce(dcp->dptxep); + afk_quiesce(dcp->dptx_controller_ep); afk_quiesce(dcp->ibootep); afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); @@ -3625,6 +3639,11 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) dcp->dptxep = NULL; } + if (!IS_ERR_OR_NULL(dcp->dptx_controller_ep)) { + afk_shutdown(dcp->dptx_controller_ep); + dcp->dptx_controller_ep = NULL; + } + if (dcp->ibootep) { afk_shutdown(dcp->ibootep); dcp->ibootep = NULL; diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index df1e46c439697..600899c71546b 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -97,6 +97,7 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message); int systemep_init(struct apple_dcp *dcp); int dptxep_init(struct apple_dcp *dcp); +int dptx_controller_init(struct apple_dcp *dcp); int ibootep_init(struct apple_dcp *dcp); int dpavservep_init(struct apple_dcp *dcp); int avep_init(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 571f6f1b03d3e..e1eeb8aa6bd47 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -849,6 +849,41 @@ static const struct apple_epic_service_ops dptxep_ops[] = { {} }; +static void dptx_controller_service_init(struct apple_epic_service *service, + const char *name, const char *class, s64 unit) +{ + complete(&service->ep->dcp->dptx_controller_ready); +} + +static const struct apple_epic_service_ops dptx_controller_ops[] = { + { + .name = "AppleDCPDPTXController", + .init = dptx_controller_service_init, + }, + {} +}; + +int dptx_controller_init(struct apple_dcp *dcp) +{ + struct apple_dcp_afkep *ep; + int ret; + + init_completion(&dcp->dptx_controller_ready); + ep = afk_init(dcp, DPTX_CONTROLLER_ENDPOINT, dptx_controller_ops); + if (IS_ERR(ep)) + return PTR_ERR(ep); + dcp->dptx_controller_ep = ep; + + ret = afk_start(ep); + if (ret) + return ret; + if (!wait_for_completion_timeout(&dcp->dptx_controller_ready, + msecs_to_jiffies(1000))) + return -ETIMEDOUT; + + return 0; +} + int dptxep_init(struct apple_dcp *dcp) { int ret; From 892ceabbc9dd110edda363c8e62e99069a2c57e7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:27:24 +1100 Subject: [PATCH 0899/1579] drm/apple: discover the version-two external DPTX service The external H17P test reaches transport readiness on the legacy DPTX endpoint without receiving a port announcement. The public version-two firmware implementation uses endpoint 0x2c, which this firmware advertises. Use that endpoint for an isolated service-discovery test. Keep a separate endpoint object with an initialization-only service table, RX routing and lifecycle handling. This does not enable operational port callbacks or transmit service commands; the discovered identity and framing must be validated before extending the command path. Normal profiles retain the legacy DPTX endpoint. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 3 +- drivers/gpu/drm/apple/dcp-internal.h | 3 ++ drivers/gpu/drm/apple/dcp.c | 12 ++++++++ drivers/gpu/drm/apple/dptxep.c | 41 ++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index c8a6d1426171a..546774c00f45c 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -825,7 +825,8 @@ static void afk_receive_message_worker(struct work_struct *work_) type = FIELD_GET(RBEP_TYPE, work->message); if (work->ep->dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && - work->ep->endpoint == DPTX_ENDPOINT && type != RBEP_RECV) + (work->ep->endpoint == DPTX_ENDPOINT || + work->ep->endpoint == DPTX_ENDPOINT_V2) && type != RBEP_RECV) dev_info_ratelimited(work->ep->dcp->dev, "external AFK control ep=%#x type=%#x\n", work->ep->endpoint, type); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 3b99d5b403e9f..700512815c2af 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -93,6 +93,7 @@ enum { DPAVSERV_ENDPOINT = 0x28, AV_ENDPOINT = 0x29, DPTX_ENDPOINT = 0x2a, + DPTX_ENDPOINT_V2 = 0x2c, HDCP_ENDPOINT = 0x2b, REMOTE_ALLOC_ENDPOINT = 0x2d, IOMFB_ENDPOINT = 0x37, @@ -360,6 +361,8 @@ struct apple_dcp { struct apple_dcp_afkep *dptxep; struct apple_dcp_afkep *dptx_controller_ep; struct completion dptx_controller_ready; + struct apple_dcp_afkep *dptx_v2_probe_ep; + struct completion dptx_v2_probe_ready; struct dptx_port dptxport[2]; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d426bf4c90c14..2b5b6bdef87e1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1875,6 +1875,9 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) return dcp->dcpavservep; case DPTX_ENDPOINT: return dcp->dptxep; + case DPTX_ENDPOINT_V2: + return dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? dcp->dptx_v2_probe_ep : NULL; case DPTX_CONTROLLER_ENDPOINT: return dcp->dptx_controller_ep; default: @@ -1960,6 +1963,9 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) case DPTX_ENDPOINT: ep = dcp->dptxep; break; + case DPTX_ENDPOINT_V2: + ep = dcp->dptx_v2_probe_ep; + break; case DPTX_CONTROLLER_ENDPOINT: ep = dcp->dptx_controller_ep; break; @@ -3603,6 +3609,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) afk_quiesce(dcp->avep); afk_quiesce(dcp->dptxep); afk_quiesce(dcp->dptx_controller_ep); + afk_quiesce(dcp->dptx_v2_probe_ep); afk_quiesce(dcp->ibootep); afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); @@ -3639,6 +3646,11 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) dcp->dptxep = NULL; } + if (!IS_ERR_OR_NULL(dcp->dptx_v2_probe_ep)) { + afk_shutdown(dcp->dptx_v2_probe_ep); + dcp->dptx_v2_probe_ep = NULL; + } + if (!IS_ERR_OR_NULL(dcp->dptx_controller_ep)) { afk_shutdown(dcp->dptx_controller_ep); dcp->dptx_controller_ep = NULL; diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index e1eeb8aa6bd47..e00c4cbcea171 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -884,12 +884,53 @@ int dptx_controller_init(struct apple_dcp *dcp) return 0; } +static void dptx_v2_probe_service(struct apple_epic_service *service, + const char *name, const char *class, s64 unit) +{ + dev_info(service->ep->dcp->dev, + "DPTX v2 discovery name=%s class=%s unit=%lld channel=%u\n", + name ?: "(none)", class ?: "(none)", unit, service->channel); + complete(&service->ep->dcp->dptx_v2_probe_ready); +} + +static const struct apple_epic_service_ops dptx_v2_probe_ops[] = { + { + .name = "AppleDCPDPTXRemotePort", + .init = dptx_v2_probe_service, + }, + {} +}; + +static int dptx_v2_discover(struct apple_dcp *dcp) +{ + struct apple_dcp_afkep *ep; + int ret; + + init_completion(&dcp->dptx_v2_probe_ready); + ep = afk_init(dcp, DPTX_ENDPOINT_V2, dptx_v2_probe_ops); + if (IS_ERR(ep)) + return PTR_ERR(ep); + dcp->dptx_v2_probe_ep = ep; + ret = afk_start(ep); + if (ret) + return ret; + if (!wait_for_completion_timeout(&dcp->dptx_v2_probe_ready, + msecs_to_jiffies(1000))) + return -ETIMEDOUT; + + /* Discovery does not enable port callbacks or transmit service commands. */ + return -EOPNOTSUPP; +} + int dptxep_init(struct apple_dcp *dcp) { int ret; u32 port; unsigned long timeout = msecs_to_jiffies(1000); + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + return dptx_v2_discover(dcp); + init_completion(&dcp->dptxport[0].enable_completion); init_completion(&dcp->dptxport[1].enable_completion); init_completion(&dcp->dptxport[0].linkcfg_completion); From d063a2ab3be82f5aca9267725e1b84060314489e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:39:11 +1100 Subject: [PATCH 0900/1579] drm/apple: operate the external DPTX service over AFKV2 The external T8140 controller announces its port on endpoint 0x2c using version-two frames. Select that transport per endpoint, open announced interfaces and map the single physical DCP unit to logical port zero. Encode bounded command descriptors and validate replies against pending owned buffers before completing them. Preserve callback tokens and limit peer-buffer access to active allocations of the same service. Keep the legacy transport for other display profiles. Add host selftests for frame encoding, malformed records, response correlation, command identifiers and the production owned-copy helper. These pass 905 codec assertions and owned-copy cases with sanitizers; hardware qualification of the operational transport remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/gpu/drm/apple/afk.c | 355 +++++++++++++++++- drivers/gpu/drm/apple/afk.h | 6 + drivers/gpu/drm/apple/afkv2-codec.h | 270 +++++++++++++ drivers/gpu/drm/apple/dcp-internal.h | 2 - drivers/gpu/drm/apple/dcp.c | 10 +- drivers/gpu/drm/apple/dptxep.c | 81 ++-- .../drivers/gpu/apple-afkv2/Makefile | 9 + .../selftests/drivers/gpu/apple-afkv2/README | 24 ++ .../selftests/drivers/gpu/apple-afkv2/codec.c | 183 +++++++++ .../drivers/gpu/apple-afkv2/extract-owned.py | 28 ++ .../apple-afkv2/host-include/linux/errno.h | 5 + .../apple-afkv2/host-include/linux/string.h | 2 + .../apple-afkv2/host-include/linux/types.h | 8 + .../drivers/gpu/apple-afkv2/owned-prefix.c | 50 +++ .../drivers/gpu/apple-afkv2/owned-suffix.c | 56 +++ .../gpu/apple-afkv2/run-afkv2-tests.sh | 40 ++ 17 files changed, 1050 insertions(+), 80 deletions(-) create mode 100644 drivers/gpu/drm/apple/afkv2-codec.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/README create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-owned.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/string.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/types.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c create mode 100755 tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh diff --git a/MAINTAINERS b/MAINTAINERS index c68108f464848..3f1523e9fdb31 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1952,6 +1952,7 @@ L: dri-devel@lists.freedesktop.org S: Maintained T: git git://anongit.freedesktop.org/drm/drm-misc F: drivers/gpu/drm/apple/ +F: tools/testing/selftests/drivers/gpu/apple-afkv2/ APPLE PCIE CONTROLLER DRIVER M: Marc Zyngier diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 546774c00f45c..e7c76e2d8a610 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -14,6 +14,7 @@ #include #include "afk.h" +#include "afkv2-codec.h" #include "trace.h" struct afk_receive_message_work { @@ -305,6 +306,10 @@ static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep * return NULL; } +static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, + enum epic_type etype, enum epic_category ecat, u8 stype, + const void *payload, size_t payload_len, u64 token); + static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *payload, size_t payload_size) { @@ -383,6 +388,22 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, goto free; } ep->num_channels = max(ep->num_channels, ch_idx + 1); + if (ep->wire_v2) { + int ret = afk_send_epic(ep, channel, 0, EPIC_TYPE_NOTIFY, + EPIC_CAT_REPORT, AFKV2_OPEN, NULL, 0); + + if (ret) { + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + service->torndown = true; + spin_unlock_irqrestore(&service->lock, flags); + dev_err(ep->dcp->dev, "AFKV2 interface %u OPEN failed: %d\n", + channel, ret); + goto free; + } + } ops->init(service, epic_name, epic_class, epic_unit); dev_info(ep->dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", ep->endpoint, service_name, channel); @@ -430,7 +451,7 @@ static void afk_recv_handle_reply(struct apple_dcp_afkep *ep, u32 channel, struct epic_cmd *cmd = payload; struct apple_epic_service *service; unsigned long flags; - u8 idx = tag & 0xff; + u8 idx = tag & (ep->wire_v2 ? 0xf : 0xff); void *rxbuf, *txbuf; dma_addr_t rxbuf_dma, txbuf_dma; size_t rxlen, txlen; @@ -592,6 +613,233 @@ static bool afk_validate_h17p_header(const u8 *data, size_t size) get_unaligned_le32(data + 4) == size - 8; } +static void afk_v2_status(struct apple_dcp_afkep *ep, + const struct afkv2_frame *frame, + const struct afkv2_command *command, u32 status) +{ + u8 record[AFKV2_HEADER_SIZE + AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE]; + size_t size; + int ret; + + ret = afkv2_encode_error_reply(record, sizeof(record), 0, frame, command, + status, &size); + if (!ret) + ret = afk_send_epic_token(ep, frame->interface, command->id, + EPIC_TYPE_REPLY, EPIC_CAT_REPLY, frame->type, + record + AFKV2_HEADER_SIZE, + size - AFKV2_HEADER_SIZE, frame->token); + if (ret) + dev_warn(ep->dcp->dev, "AFKV2 error reply failed: %d\n", ret); +} + +static int afk_v2_access_owned(struct apple_epic_service *service, u64 dva, + void *buffer, size_t size, bool to_peer, bool copy) +{ + unsigned long flags; + unsigned int i; + int ret = -EACCES; + + if (!service || !service->ep->wire_v2 || size > SZ_1M || (copy && size && !buffer)) + return -EINVAL; + if (!size) + return 0; + spin_lock_irqsave(&service->lock, flags); + for (i = 0; i < MAX_PENDING_CMDS; i++) { + struct epic_cmd_info *cmd = &service->cmds[i]; + void *owned = NULL; + u64 offset; + + if (!test_bit(i, service->cmd_map) || cmd->done) + continue; + if (cmd->rxbuf && afkv2_owned_span(cmd->rxbuf_dma, cmd->rxlen, dva, size)) { + offset = dva - cmd->rxbuf_dma; + owned = cmd->rxbuf + offset; + } else if (cmd->txbuf && afkv2_owned_span(cmd->txbuf_dma, cmd->txlen, dva, size)) { + offset = dva - cmd->txbuf_dma; + owned = cmd->txbuf + offset; + } + if (!owned) + continue; + if (!copy) { + ret = 0; + break; + } + if (to_peer) { + memcpy(owned, buffer, size); + dma_wmb(); + } else { + dma_rmb(); + memcpy(buffer, owned, size); + } + ret = 0; + break; + } + spin_unlock_irqrestore(&service->lock, flags); + if (ret) + dev_warn_ratelimited(service->ep->dcp->dev, + "AFKV2 unowned buffer: ep=%u interface=%u size=%zu write=%u\n", + service->ep->endpoint, service->channel, size, to_peer); + return ret; +} + +int afk_v2_copy_owned(struct apple_epic_service *service, u64 dva, + void *buffer, size_t size, bool to_peer) +{ + return afk_v2_access_owned(service, dva, buffer, size, to_peer, true); +} + +static void afk_v2_callback(struct apple_dcp_afkep *ep, + const struct afkv2_frame *frame, + const struct afkv2_command *command) +{ + struct apple_epic_service *service; + const struct epic_service_call *call; + const u8 *request = command->inline_data; + u8 *owned_request = NULL, *reply = NULL; + size_t request_size = command->oob ? command->tx_size : command->inline_size; + size_t body_size, reply_size; + u32 status = 0xe00002c7; + int ret; + + if (frame->type != EPIC_SUBTYPE_STD_SERVICE || + request_size < sizeof(*call) + sizeof(u32) || + request_size > SZ_1M - AFKV2_COMMAND_SIZE || + request_size > command->value || + (command->oob && request_size > command->rx_size)) + goto reject; + service = afk_service_get(afk_epic_find_service(ep, frame->interface)); + if (!service || READ_ONCE(service->torndown) || !service->ops->call) { + if (service) + afk_service_put(service); + goto reject; + } + if (command->oob) { + /* Reject an unowned response destination before any callback side effect. */ + ret = afk_v2_access_owned(service, command->rx_address, NULL, + request_size, true, false); + if (ret) + goto release; + owned_request = kmalloc(request_size, GFP_KERNEL); + if (!owned_request) { + status = 0xe00002bd; + goto release; + } + ret = afk_v2_copy_owned(service, command->tx_address, + owned_request, request_size, false); + if (ret) + goto release; + request = owned_request; + } + call = (const void *)request; + body_size = request_size - sizeof(*call); + if (le16_to_cpu(call->group) || + le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || + le32_to_cpu(call->data_len) != body_size) + goto release; + reply = kzalloc(AFKV2_COMMAND_SIZE + request_size, GFP_KERNEL); + if (!reply) { + status = 0xe00002bd; + goto release; + } + reply[1] = command->id; + memcpy(reply + AFKV2_COMMAND_SIZE, call, sizeof(*call)); + ret = service->ops->call(service, le32_to_cpu(call->command), + request + sizeof(*call), body_size, + reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + if (ret) + goto release; + reply_size = AFKV2_COMMAND_SIZE + request_size; + if (command->oob) { + ret = afk_v2_copy_owned(service, command->rx_address, + reply + AFKV2_COMMAND_SIZE, request_size, true); + if (ret) + goto release; + ret = afkv2_encode_oob_command(reply, reply_size, command->id, + command->rx_address, request_size, + command->tx_address, command->tx_size); + if (ret) + goto release; + afkv2_put32(reply + 4, 0); + reply_size = AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE; + } + ret = afk_send_epic_token(ep, frame->interface, command->id, EPIC_TYPE_REPLY, + EPIC_CAT_REPLY, frame->type, reply, reply_size, frame->token); + kfree(reply); + kfree(owned_request); + afk_service_put(service); + if (ret) + dev_warn(ep->dcp->dev, "AFKV2 callback reply failed: %d\n", ret); + return; +release: + kfree(reply); + kfree(owned_request); + afk_service_put(service); +reject: + afk_v2_status(ep, frame, command, status); +} + +static void afk_v2_receive(struct apple_dcp_afkep *ep, u32 channel, u32 type, + const void *data, size_t size) +{ + struct afkv2_frame frame; + struct afkv2_command response; + struct apple_epic_service *service; + struct epic_cmd legacy = {}; + unsigned long flags; + unsigned int slot; + int ret; + + if (channel || type || afkv2_decode_frame(data, size, false, &frame) || + !frame.interface) + return; + if (frame.category == AFKV2_REPORT) { + if (frame.type == EPIC_SUBTYPE_ANNOUNCE_H17P) + afk_recv_handle_init(ep, frame.interface, (void *)frame.payload, + frame.payload_size); + else if (frame.type == AFKV2_CLOSE && frame.interface_flags == 1) + afk_recv_handle_teardown(ep, frame.interface); + return; + } + if (afkv2_decode_command(frame.payload, frame.payload_size, &response)) + return; + if (frame.category == AFKV2_COMMAND) { + afk_v2_callback(ep, &frame, &response); + return; + } + service = afk_epic_find_service(ep, frame.interface); + if (!service) + return; + spin_lock_irqsave(&service->lock, flags); + if (bitmap_empty(service->cmd_map, MAX_PENDING_CMDS)) { + spin_unlock_irqrestore(&service->lock, flags); + return; + } + service->users++; + spin_unlock_irqrestore(&service->lock, flags); + slot = response.id & 0xf; + spin_lock_irqsave(&service->lock, flags); + ret = -EPROTO; + if (slot < MAX_PENDING_CMDS && test_bit(slot, service->cmd_map) && + !service->cmds[slot].done && response.id == service->cmds[slot].tag) + ret = afkv2_check_response(&frame, &response, service->channel, + service->cmds[slot].type, service->cmds[slot].tag, + service->cmds[slot].rxbuf_dma, + service->cmds[slot].rxlen, + service->cmds[slot].txbuf_dma, + service->cmds[slot].txlen); + if (!ret) + service->cmds[slot].reply_length = response.value ? 0 : response.rx_size; + spin_unlock_irqrestore(&service->lock, flags); + if (!ret) { + legacy.retcode = cpu_to_le32(response.value); + afk_recv_handle_reply(ep, frame.interface, response.id, + &legacy, sizeof(legacy)); + } else { + dev_warn(ep->dcp->dev, "AFKV2 reply rejected: %d\n", ret); + } + afk_service_put(service); +} + static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, u8 *data, size_t data_size) { @@ -605,6 +853,11 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, u8 *payload; size_t payload_size; + if (ep->wire_v2) { + afk_v2_receive(ep, channel, type, data, data_size); + return; + } + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P) { const struct epic_sub_hdr_h17p *c = (const struct epic_sub_hdr_h17p *)(data + sizeof(*ehdr)); @@ -895,9 +1148,9 @@ int afk_receive_message(struct apple_dcp_afkep *ep, u64 message) return 0; } -int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, - enum epic_type etype, enum epic_category ecat, u8 stype, - const void *payload, size_t payload_len) +static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, + enum epic_type etype, enum epic_category ecat, u8 stype, + const void *payload, size_t payload_len, u64 token) { u32 rptr, wptr; struct afk_qe *hdr, *hdr2; @@ -905,7 +1158,10 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, struct epic_sub_hdr *eshdr; unsigned long flags; size_t total_epic_size, total_size; - bool compact = ep->dcp->hw.iomfb_method_profile == + u8 v2_command[AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE]; + u8 v2_category = 0; + u16 v2_flags = 0; + bool compact = !ep->wire_v2 && ep->dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; int ret; @@ -919,7 +1175,7 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, * callers are optional (verbose firmware logging, EDID copy, audio, * DPTX, debugfs) or replies to calls only DPTX services make. */ - if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && !compact) { + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && !compact && !ep->wire_v2) { dev_dbg_once(ep->dcp->dev, "AFK[ep:%02x]: not sending EPIC message on H17P\n", ep->endpoint); @@ -937,14 +1193,44 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, (etype == EPIC_TYPE_REPLY || etype == EPIC_TYPE_NOTIFY_ACK))))) return -EOPNOTSUPP; + if (ep->wire_v2) { + if (!channel || channel > U16_MAX || tag > U8_MAX || payload_len > SZ_1M) + return -EINVAL; + if (ecat == EPIC_CAT_REPORT) { + if ((stype != AFKV2_OPEN && stype != AFKV2_CLOSE) || payload_len) + return -EOPNOTSUPP; + v2_flags = 1; + } else if (ecat == EPIC_CAT_COMMAND) { + const struct epic_cmd *cmd = payload; + + if (stype != EPIC_SUBTYPE_STD_SERVICE || !cmd || + payload_len != sizeof(*cmd)) + return -EINVAL; + ret = afkv2_encode_oob_command(v2_command, sizeof(v2_command), tag, + le64_to_cpu(cmd->rxbuf), + le32_to_cpu(cmd->rxlen), + le64_to_cpu(cmd->txbuf), + le32_to_cpu(cmd->txlen)); + if (ret) + return ret; + payload = v2_command; + payload_len = sizeof(v2_command); + v2_category = AFKV2_COMMAND; + } else if (ecat == EPIC_CAT_REPLY) { + v2_category = AFKV2_RESPONSE; + } else { + return -EOPNOTSUPP; + } + } + spin_lock_irqsave(&ep->lock, flags); if (ep->stopping) { ret = -ESHUTDOWN; goto out; } - if (compact && (!ep->txbfr.ready || !ep->txbfr.rptr || - !ep->txbfr.wptr || !ep->txbfr.buf)) { + if ((compact || ep->wire_v2) && + (!ep->txbfr.ready || !ep->txbfr.rptr || !ep->txbfr.wptr || !ep->txbfr.buf)) { ret = -ESHUTDOWN; goto out; } @@ -954,11 +1240,11 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, wptr = le32_to_cpu(*ep->txbfr.wptr); trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + - (compact ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + + ((compact || ep->wire_v2) ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + payload_len; total_size = sizeof(*hdr) + total_epic_size; - if (compact) { + if (compact || ep->wire_v2) { u32 available, needed; if (!ep->txbfr.ready || !ep->txbfr.stride || @@ -1061,12 +1347,21 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, hdr->magic = cpu_to_le32(QE_MAGIC); hdr->size = cpu_to_le32(total_epic_size); - hdr->channel = cpu_to_le32(compact ? 0 : channel); - hdr->type = cpu_to_le32(etype); + hdr->channel = cpu_to_le32((compact || ep->wire_v2) ? 0 : channel); + hdr->type = cpu_to_le32(ep->wire_v2 ? 0 : etype); if (hdr2) memcpy(hdr2, hdr, sizeof(*hdr)); - if (compact) { + if (ep->wire_v2) { + size_t written; + + ret = afkv2_encode_frame(ep->txbfr.buf + wptr, total_epic_size, + ep->qe_seq++, channel, token, stype, v2_category, + v2_flags, payload, payload_len, &written); + if (ret) + goto out; + wptr += written; + } else if (compact) { u8 *wire = ep->txbfr.buf + wptr; /* Experimental symmetry with the published H17P receive layout. */ @@ -1101,8 +1396,10 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, wptr += sizeof(*eshdr); } - memcpy(ep->txbfr.buf + wptr, payload, payload_len); - wptr += payload_len; + if (!ep->wire_v2) { + memcpy(ep->txbfr.buf + wptr, payload, payload_len); + wptr += payload_len; + } wptr = afk_ring_advance(wptr, 0, ep->txbfr.bufsz, ep->txbfr.stride); trace_afk_send_rwptr_post(ep, rptr, wptr); @@ -1119,6 +1416,14 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, return ret; } +int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, + enum epic_type etype, enum epic_category ecat, u8 stype, + const void *payload, size_t payload_len) +{ + return afk_send_epic_token(ep, channel, tag, etype, ecat, stype, + payload, payload_len, 0); +} + int afk_send_command(struct apple_epic_service *service, u8 type, const void *payload, size_t payload_len, void *output, size_t output_len, u32 *retcode) @@ -1151,6 +1456,12 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, goto err_put_service; } + if (ep->wire_v2 && (payload_len > SZ_1M || output_len > SZ_1M || + (payload_len && !payload))) { + ret = -EINVAL; + goto err_put_service; + } + rxbuf = dma_alloc_coherent(ep->dcp->dev, output_len, &rxbuf_dma, GFP_KERNEL); if (!rxbuf) { @@ -1184,11 +1495,13 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, goto err_unlock; } - tag = (service->cmd_tag & 0xff) << 8; - tag |= idx & 0xff; + tag = ep->wire_v2 ? afkv2_command_id(service->cmd_tag, idx) : + ((service->cmd_tag & 0xff) << 8) | idx; service->cmd_tag++; service->cmds[idx].tag = tag; + service->cmds[idx].type = type; + service->cmds[idx].reply_length = 0; service->cmds[idx].rxbuf = rxbuf; service->cmds[idx].txbuf = txbuf; service->cmds[idx].rxbuf_dma = rxbuf_dma; @@ -1234,8 +1547,14 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, ret = 0; if (retcode) *retcode = service->cmds[idx].retcode; - if (output && output_len) + if (ep->wire_v2 && !service->cmds[idx].retcode && + service->cmds[idx].reply_length < output_len) + ret = -EMSGSIZE; + else if (output && output_len && (!ep->wire_v2 || !service->cmds[idx].retcode)) { + if (ep->wire_v2) + dma_rmb(); memcpy(output, rxbuf, output_len); + } err_free_cmd: spin_lock_irqsave(&service->lock, flags); diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 8eb797e12ef07..a4de9090681c2 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -25,6 +25,8 @@ struct apple_dcp_afkep; struct epic_cmd_info { u16 tag; + u8 type; + size_t reply_length; void *rxbuf; void *txbuf; @@ -295,6 +297,8 @@ struct apple_dcp_afkep { u8 experimental_rx_logs; u32 endpoint; + /* Set before afk_start; protocol selection is per endpoint. */ + bool wire_v2; struct workqueue_struct *wq; struct completion started; @@ -325,6 +329,8 @@ struct apple_dcp_afkep { struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, const struct apple_epic_service_ops *ops); +int afk_v2_copy_owned(struct apple_epic_service *service, u64 dva, + void *buffer, size_t size, bool to_peer); int afk_start(struct apple_dcp_afkep *ep); void afk_shutdown(struct apple_dcp_afkep *ep); void afk_quiesce(struct apple_dcp_afkep *ep); diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h new file mode 100644 index 0000000000000..cdd5d5ad20b22 --- /dev/null +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -0,0 +1,270 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Aurora Silicon */ +/* Wire layout: public Linux PR254 and m1n1 AFKV2 implementations. */ +#ifndef AURORA_AFKV2_CODEC_H +#define AURORA_AFKV2_CODEC_H + +#include +#include +#include + +#define AFKV2_HEADER_SIZE 24U +#define AFKV2_COMMAND_SIZE 8U +#define AFKV2_OOB_SIZE 24U +#define AFKV2_MAX_PAYLOAD 1048576U +#define AFKV2_REPORT 0U +#define AFKV2_COMMAND 1U +#define AFKV2_RESPONSE 2U +#define AFKV2_OPEN 0x12U +#define AFKV2_CLOSE 0x13U + +struct afkv2_frame { + u8 sequence; + u8 transport_flags; + u16 interface; + u64 token; + u8 type; + u8 category; + u16 interface_flags; + u32 reserved; + const u8 *payload; + size_t payload_size; + size_t padding_size; +}; + +struct afkv2_command { + u8 id; + u32 value; + bool oob; + u64 rx_address; + u64 tx_address; + u32 rx_size; + u32 tx_size; + const u8 *inline_data; + size_t inline_size; +}; + +static inline u16 afkv2_get16(const u8 *p) +{ + return (u16)p[0] | (u16)p[1] << 8; +} + +static inline u32 afkv2_get32(const u8 *p) +{ + return (u32)p[0] | (u32)p[1] << 8 | (u32)p[2] << 16 | (u32)p[3] << 24; +} + +static inline u64 afkv2_get64(const u8 *p) +{ + return (u64)afkv2_get32(p) | (u64)afkv2_get32(p + 4) << 32; +} + +static inline void afkv2_put16(u8 *p, u16 value) +{ + p[0] = value; + p[1] = value >> 8; +} + +static inline void afkv2_put32(u8 *p, u32 value) +{ + p[0] = value; + p[1] = value >> 8; + p[2] = value >> 16; + p[3] = value >> 24; +} + +static inline void afkv2_put64(u8 *p, u64 value) +{ + afkv2_put32(p, value); + afkv2_put32(p + 4, value >> 32); +} + +/* Decode metadata only. Never follow an address supplied by the peer. */ +static inline int afkv2_decode_frame(const void *record, size_t size, + bool allow_padding, struct afkv2_frame *out) +{ + const u8 *p = record; + u32 length; + struct afkv2_frame result = {0}; + + if (!record || !out || size < AFKV2_HEADER_SIZE) + return -EINVAL; + if (size > AFKV2_HEADER_SIZE + AFKV2_MAX_PAYLOAD) + return -E2BIG; + length = afkv2_get32(p + 4); + if (length < 16 || length - 16 > AFKV2_MAX_PAYLOAD || length > size - 8) + return -EINVAL; + if (!allow_padding && length != size - 8) + return -EINVAL; + if (p[17] > AFKV2_RESPONSE) + return -EPROTO; + result.sequence = p[0]; + result.transport_flags = p[1]; + result.interface = afkv2_get16(p + 2); + result.token = afkv2_get64(p + 8); + result.type = p[16]; + result.category = p[17]; + result.interface_flags = afkv2_get16(p + 18); + result.reserved = afkv2_get32(p + 20); + result.payload = p + AFKV2_HEADER_SIZE; + result.payload_size = length - 16; + result.padding_size = size - 8 - length; + *out = result; + return 0; +} + +static inline int afkv2_encode_frame(void *record, size_t capacity, u8 sequence, + u16 interface, u64 token, u8 type, u8 category, + u16 flags, const void *payload, size_t payload_size, + size_t *written) +{ + u8 *p = record; + + if (!record || !written || !interface || category > AFKV2_RESPONSE || + (payload_size && !payload)) + return -EINVAL; + if (payload_size > AFKV2_MAX_PAYLOAD) + return -E2BIG; + if (capacity < AFKV2_HEADER_SIZE || payload_size > capacity - AFKV2_HEADER_SIZE) + return -ENOSPC; + /* Move first so an in-buffer payload is safe even when it overlaps. */ + if (payload_size) + memmove(p + AFKV2_HEADER_SIZE, payload, payload_size); + memset(p, 0, AFKV2_HEADER_SIZE); + p[0] = sequence; + afkv2_put16(p + 2, interface); + afkv2_put32(p + 4, payload_size + 16); + afkv2_put64(p + 8, token); + p[16] = type; + p[17] = category; + afkv2_put16(p + 18, flags); + *written = AFKV2_HEADER_SIZE + payload_size; + return 0; +} + +static inline int afkv2_encode_open(void *record, size_t capacity, u8 sequence, + u16 interface, size_t *written) +{ + return afkv2_encode_frame(record, capacity, sequence, interface, 0, + AFKV2_OPEN, AFKV2_REPORT, 1, NULL, 0, written); +} + +static inline bool afkv2_valid_span(u64 address, u32 size) +{ + if (!size) + return !address; + return address && size <= AFKV2_MAX_PAYLOAD && + address <= ~(u64)0 - (size - 1); +} + +static inline int afkv2_encode_oob_command(void *payload, size_t capacity, + u8 id, u64 rx_address, u32 rx_size, + u64 tx_address, u32 tx_size) +{ + u8 *p = payload; + + if (!payload || !afkv2_valid_span(rx_address, rx_size) || + !afkv2_valid_span(tx_address, tx_size)) + return -EINVAL; + if (capacity < AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE) + return -ENOSPC; + memset(p, 0, AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE); + p[0] = 1; + p[1] = id; + afkv2_put32(p + 4, rx_size); + afkv2_put64(p + 8, rx_address); + afkv2_put64(p + 16, tx_address); + afkv2_put32(p + 24, rx_size); + afkv2_put32(p + 28, tx_size); + return 0; +} + +static inline int afkv2_decode_command(const void *payload, size_t size, + struct afkv2_command *out) +{ + const u8 *p = payload; + struct afkv2_command result = {0}; + size_t header = AFKV2_COMMAND_SIZE; + + if (!payload || !out || size < header || p[0] & ~1U || afkv2_get16(p + 2)) + return -EINVAL; + result.id = p[1]; + result.value = afkv2_get32(p + 4); + result.oob = p[0] & 1; + if (result.oob) { + header += AFKV2_OOB_SIZE; + if (size < header) + return -EINVAL; + result.rx_address = afkv2_get64(p + 8); + result.tx_address = afkv2_get64(p + 16); + result.rx_size = afkv2_get32(p + 24); + result.tx_size = afkv2_get32(p + 28); + /* A failed response may retain addresses with zero lengths. */ + if ((result.rx_size && !afkv2_valid_span(result.rx_address, result.rx_size)) || + (result.tx_size && !afkv2_valid_span(result.tx_address, result.tx_size))) + return -EINVAL; + } + result.inline_data = p + header; + result.inline_size = size - header; + if (result.inline_size > AFKV2_MAX_PAYLOAD) + return -E2BIG; + *out = result; + return 0; +} + +/* Correlate only; no buffer is read, written or freed by this helper. */ +static inline int afkv2_check_response(const struct afkv2_frame *frame, + const struct afkv2_command *response, + u16 interface, u8 type, u8 id, + u64 rx_address, u32 rx_capacity, + u64 tx_address, u32 tx_capacity) +{ + if (!frame || !response || frame->category != AFKV2_RESPONSE || + frame->interface != interface || frame->type != type || response->id != id) + return -EPROTO; + if (response->value) + return 0; + if (!response->oob || response->inline_size || + response->rx_address != rx_address || response->tx_address != tx_address || + response->rx_size > rx_capacity || response->tx_size > tx_capacity) + return -EPROTO; + return 0; +} + +/* Reject a peer command without touching its advertised DMA buffers. */ +static inline int afkv2_encode_error_reply(void *record, size_t capacity, u8 sequence, + const struct afkv2_frame *request, + const struct afkv2_command *command, + u32 status, size_t *written) +{ + u8 payload[AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE] = {0}; + size_t size = AFKV2_COMMAND_SIZE; + + if (!request || !command || request->category != AFKV2_COMMAND || !status) + return -EINVAL; + payload[0] = command->oob ? 1 : 0; + payload[1] = command->id; + afkv2_put32(payload + 4, status); + if (command->oob) { + afkv2_put64(payload + 8, command->rx_address); + afkv2_put64(payload + 16, command->tx_address); + size += AFKV2_OOB_SIZE; + } + return afkv2_encode_frame(record, capacity, sequence, request->interface, + request->token, request->type, AFKV2_RESPONSE, 0, + payload, size, written); +} + +static inline u8 afkv2_command_id(u8 generation, u8 slot) +{ + return ((generation & 15) << 4) | (slot & 15); +} + +static inline bool afkv2_owned_span(u64 base, size_t length, u64 address, size_t size) +{ + if (!length || base > ~(u64)0 - (length - 1)) + return false; + return address >= base && size <= length && address - base <= length - size; +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 700512815c2af..883d4f9ce2e80 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -361,8 +361,6 @@ struct apple_dcp { struct apple_dcp_afkep *dptxep; struct apple_dcp_afkep *dptx_controller_ep; struct completion dptx_controller_ready; - struct apple_dcp_afkep *dptx_v2_probe_ep; - struct completion dptx_v2_probe_ready; struct dptx_port dptxport[2]; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 2b5b6bdef87e1..7edee07d7bb10 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1877,7 +1877,7 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) return dcp->dptxep; case DPTX_ENDPOINT_V2: return dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? dcp->dptx_v2_probe_ep : NULL; + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? dcp->dptxep : NULL; case DPTX_CONTROLLER_ENDPOINT: return dcp->dptx_controller_ep; default: @@ -1964,7 +1964,7 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) ep = dcp->dptxep; break; case DPTX_ENDPOINT_V2: - ep = dcp->dptx_v2_probe_ep; + ep = dcp->dptxep; break; case DPTX_CONTROLLER_ENDPOINT: ep = dcp->dptx_controller_ep; @@ -3609,7 +3609,6 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) afk_quiesce(dcp->avep); afk_quiesce(dcp->dptxep); afk_quiesce(dcp->dptx_controller_ep); - afk_quiesce(dcp->dptx_v2_probe_ep); afk_quiesce(dcp->ibootep); afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); @@ -3646,11 +3645,6 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) dcp->dptxep = NULL; } - if (!IS_ERR_OR_NULL(dcp->dptx_v2_probe_ep)) { - afk_shutdown(dcp->dptx_v2_probe_ep); - dcp->dptx_v2_probe_ep = NULL; - } - if (!IS_ERR_OR_NULL(dcp->dptx_controller_ep)) { afk_shutdown(dcp->dptx_controller_ep); dcp->dptx_controller_ep = NULL; diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index e00c4cbcea171..8d605beb7faeb 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -812,27 +812,40 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, static void dptxport_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { + struct apple_dcp *dcp = service->ep->dcp; + s64 port = unit; + if (!name || !class) + return; if (strcmp(name, "dcpdptx-port-epic")) return; if (strcmp(class, "AppleDCPDPTXRemotePort")) return; - trace_dptxport_init(service->ep->dcp, unit); + trace_dptxport_init(dcp, unit); + if (service->ep->wire_v2) { + /* The single external service reports its physical DCP unit. */ + if (dcp->hw.num_dptx_ports != 1 || unit != dcp->index) { + dev_err(dcp->dev, "Unexpected external DPTX unit %lld\n", unit); + return; + } + port = 0; + dev_info(dcp->dev, "DPTX physical unit %lld maps to logical port 0\n", unit); + } - switch (unit) { + switch (port) { case 0: case 1: - if (service->ep->dcp->dptxport[unit].enabled) { + if (service->ep->dcp->dptxport[port].enabled) { dev_err(service->ep->dcp->dev, "DPTXPort: unit %lld already exists\n", unit); return; } - service->ep->dcp->dptxport[unit].unit = unit; - service->ep->dcp->dptxport[unit].service = service; - service->ep->dcp->dptxport[unit].enabled = true; - service->cookie = (void *)&service->ep->dcp->dptxport[unit]; - complete(&service->ep->dcp->dptxport[unit].enable_completion); + service->ep->dcp->dptxport[port].unit = unit; + service->ep->dcp->dptxport[port].service = service; + service->ep->dcp->dptxport[port].enabled = true; + service->cookie = (void *)&service->ep->dcp->dptxport[port]; + complete(&service->ep->dcp->dptxport[port].enable_completion); break; default: dev_err(service->ep->dcp->dev, "DPTXPort: invalid unit %lld\n", @@ -884,61 +897,25 @@ int dptx_controller_init(struct apple_dcp *dcp) return 0; } -static void dptx_v2_probe_service(struct apple_epic_service *service, - const char *name, const char *class, s64 unit) -{ - dev_info(service->ep->dcp->dev, - "DPTX v2 discovery name=%s class=%s unit=%lld channel=%u\n", - name ?: "(none)", class ?: "(none)", unit, service->channel); - complete(&service->ep->dcp->dptx_v2_probe_ready); -} - -static const struct apple_epic_service_ops dptx_v2_probe_ops[] = { - { - .name = "AppleDCPDPTXRemotePort", - .init = dptx_v2_probe_service, - }, - {} -}; - -static int dptx_v2_discover(struct apple_dcp *dcp) -{ - struct apple_dcp_afkep *ep; - int ret; - - init_completion(&dcp->dptx_v2_probe_ready); - ep = afk_init(dcp, DPTX_ENDPOINT_V2, dptx_v2_probe_ops); - if (IS_ERR(ep)) - return PTR_ERR(ep); - dcp->dptx_v2_probe_ep = ep; - ret = afk_start(ep); - if (ret) - return ret; - if (!wait_for_completion_timeout(&dcp->dptx_v2_probe_ready, - msecs_to_jiffies(1000))) - return -ETIMEDOUT; - - /* Discovery does not enable port callbacks or transmit service commands. */ - return -EOPNOTSUPP; -} - int dptxep_init(struct apple_dcp *dcp) { + struct apple_dcp_afkep *ep; + bool wire_v2 = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; int ret; u32 port; unsigned long timeout = msecs_to_jiffies(1000); - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) - return dptx_v2_discover(dcp); - init_completion(&dcp->dptxport[0].enable_completion); init_completion(&dcp->dptxport[1].enable_completion); init_completion(&dcp->dptxport[0].linkcfg_completion); init_completion(&dcp->dptxport[1].linkcfg_completion); - dcp->dptxep = afk_init(dcp, DPTX_ENDPOINT, dptxep_ops); - if (IS_ERR(dcp->dptxep)) - return PTR_ERR(dcp->dptxep); + ep = afk_init(dcp, wire_v2 ? DPTX_ENDPOINT_V2 : DPTX_ENDPOINT, dptxep_ops); + if (IS_ERR(ep)) + return PTR_ERR(ep); + ep->wire_v2 = wire_v2; + dcp->dptxep = ep; ret = afk_start(dcp->dptxep); if (ret) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile new file mode 100644 index 0000000000000..65780f3d5003f --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +HOSTCC ?= cc +export HOSTCC +export AFKV2_SOURCE_ROOT := $(abspath ../../../../../..) + +TEST_PROGS := run-afkv2-tests.sh +TEST_FILES := codec.c owned-prefix.c owned-suffix.c extract-owned.py host-include + +include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/README b/tools/testing/selftests/drivers/gpu/apple-afkv2/README new file mode 100644 index 0000000000000..42eb152d716a2 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/README @@ -0,0 +1,24 @@ +AFKV2 host selftests +=================== + +Run ./run-afkv2-tests.sh from this directory, or use make run_tests in a +complete kernel source checkout. HOSTCC, CFLAGS and PYTHON select host tools +(standalone CC is also accepted). +AFKV2_SANITIZERS=1 additionally enables AddressSanitizer and UBSan. +AFKV2_SOURCE_ROOT can name the source checkout when running a copied test +package; the Makefile exports it automatically for separate output builds. + +The codec tests include the production header directly and compare invented, +independently specified golden bytes. They cover truncation, category ranges, +length/padding/overflow, unaligned input, OOB descriptor ownership, response +correlation, token preservation and command ID packing. + +The owned-copy test extracts the actual production function from afk.c, prints +its SHA256, and compiles it with synthetic command slots and CPU buffers. It +checks active/inactive/done slots, high device addresses, boundaries and copies. +Locks and DMA barriers are mocked: this test does not qualify concurrency, +DMA coherency, firmware, a graphics device or any hardware display behavior. + +All build products are temporary. No device node is opened and no GPU, firmware +or target machine is contacted. The extractor fails if production function +boundaries change, requiring review rather than silently testing a stale copy. diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c new file mode 100644 index 0000000000000..de49360bdabea --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Aurora Silicon */ +#undef NDEBUG +#include +#include +#include +#include +#include "afkv2-codec.h" + +static unsigned int assertions; +#define CHECK(x) \ + do { \ + assertions++; \ + assert(x); \ + } while (0) + +/* Invented fixtures with independently specified wire bytes; no device data. */ +static const u8 open_fixture[24] = { 0x7b, 0, 0x34, 0x12, 0x10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0x12, 0, 1, 0, 0, 0, 0, 0 }; +static const u8 descriptor_fixture[32] = { + 1, 0xab, 0, 0, 0x40, 0, 0, 0, 0x88, 0x77, 0x66, + 0x55, 0x44, 0x33, 0x22, 0x11, 8, 7, 6, 5, 4, 3, + 2, 1, 0x40, 0, 0, 0, 0x10, 0, 0, 0 +}; + +int main(void) +{ + u8 data[128], command[32], unaligned[129], saved[128]; + struct afkv2_frame frame, reply; + struct afkv2_command parsed, response; + size_t written = 999; + int ret; + + memset(data, 0xa5, sizeof(data)); + CHECK(!afkv2_encode_open(data, sizeof(data), 0x7b, 0x1234, &written)); + CHECK(written == sizeof(open_fixture)); + CHECK(!memcmp(data, open_fixture, sizeof(open_fixture))); + CHECK(data[24] == 0xa5); + CHECK(!afkv2_decode_frame(data, written, false, &frame)); + CHECK(frame.interface == 0x1234 && frame.type == AFKV2_OPEN); + CHECK(frame.category == AFKV2_REPORT && frame.interface_flags == 1); + CHECK(frame.payload_size == 0 && frame.padding_size == 0); + memcpy(unaligned + 1, open_fixture, sizeof(open_fixture)); + CHECK(!afkv2_decode_frame(unaligned + 1, sizeof(open_fixture), false, + &frame)); + CHECK(afkv2_decode_frame(data, SIZE_MAX, true, &frame) == -E2BIG); + CHECK(afkv2_decode_frame(NULL, 24, false, &frame) == -EINVAL); + CHECK(afkv2_decode_frame(data, 24, false, NULL) == -EINVAL); + for (size_t i = 0; i < 24; i++) + CHECK(afkv2_decode_frame(data, i, false, &frame) == -EINVAL); + CHECK(afkv2_decode_frame(data, 32, false, &frame) == -EINVAL); + CHECK(!afkv2_decode_frame(data, 32, true, &frame) && + frame.padding_size == 8); + for (u32 i = 0; i < 16; i++) { + afkv2_put32(data + 4, i); + CHECK(afkv2_decode_frame(data, 24, false, &frame) == -EINVAL); + } + afkv2_put32(data + 4, 17); + CHECK(afkv2_decode_frame(data, 24, false, &frame) == -EINVAL); + afkv2_put32(data + 4, UINT32_MAX); + CHECK(afkv2_decode_frame(data, 24, false, &frame) == -EINVAL); + memcpy(data, open_fixture, sizeof(open_fixture)); + for (unsigned int category = 0; category < 256; category++) { + data[17] = category; + ret = afkv2_decode_frame(data, 24, false, &frame); + CHECK(category <= 2 ? ret == 0 : ret == -EPROTO); + } + CHECK(!afkv2_encode_oob_command(command, sizeof(command), 0xab, + 0x1122334455667788ULL, 64, + 0x0102030405060708ULL, 16)); + CHECK(!memcmp(command, descriptor_fixture, sizeof(command))); + CHECK(!afkv2_decode_command(command, sizeof(command), &parsed)); + CHECK(parsed.oob && parsed.id == 0xab && parsed.value == 64); + CHECK(parsed.rx_address == 0x1122334455667788ULL && + parsed.tx_size == 16); + CHECK(parsed.inline_size == 0); + for (size_t i = 0; i < sizeof(command); i++) + CHECK(afkv2_decode_command(command, i, &response) == -EINVAL); + command[0] = 2; + CHECK(afkv2_decode_command(command, sizeof(command), &response) == + -EINVAL); + memcpy(command, descriptor_fixture, sizeof(command)); + command[2] = 1; + CHECK(afkv2_decode_command(command, sizeof(command), &response) == + -EINVAL); + memcpy(command, descriptor_fixture, sizeof(command)); + CHECK(afkv2_encode_oob_command(command, 31, 0, 1, 1, 2, 1) == -ENOSPC); + CHECK(afkv2_encode_oob_command(command, 32, 0, UINT64_MAX, 2, 0, 0) == + -EINVAL); + CHECK(afkv2_encode_oob_command(command, 32, 0, 1, AFKV2_MAX_PAYLOAD + 1, + 0, 0) == -EINVAL); + CHECK(afkv2_encode_oob_command(command, 32, 0, 1, 0, 0, 0) == -EINVAL); + memcpy(command, descriptor_fixture, sizeof(command)); + CHECK(!afkv2_encode_frame(data, sizeof(data), 1, 0x1234, + 0xaabbccddeeff0011ULL, 0xc0, AFKV2_COMMAND, 0, + command, sizeof(command), &written)); + CHECK(written == 56 && afkv2_get32(data + 4) == 48); + CHECK(!afkv2_decode_frame(data, written, false, &frame)); + CHECK(frame.token == 0xaabbccddeeff0011ULL && + frame.category == AFKV2_COMMAND); + CHECK(!afkv2_encode_error_reply(unaligned, sizeof(unaligned), 2, &frame, + &parsed, 0xe00002c7U, &written)); + CHECK(!afkv2_decode_frame(unaligned, written, false, &reply)); + CHECK(reply.token == frame.token && reply.interface == frame.interface); + CHECK(reply.type == frame.type && reply.category == AFKV2_RESPONSE); + CHECK(!afkv2_decode_command(reply.payload, reply.payload_size, + &response)); + CHECK(response.value == 0xe00002c7U && response.rx_size == 0 && + response.tx_size == 0); + CHECK(response.rx_address == parsed.rx_address && + response.tx_address == parsed.tx_address); + CHECK(!afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16)); + response.value = 0; + response.rx_size = 64; + response.tx_size = 16; + CHECK(!afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16)); + response.rx_address++; + CHECK(afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + response.rx_address--; + response.rx_size++; + CHECK(afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + response.rx_size--; + CHECK(afkv2_check_response(&reply, &response, 0x1235, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + CHECK(afkv2_check_response(&reply, &response, 0x1234, 0xc1, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + CHECK(afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xaa, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + memset(data, 0xa5, sizeof(data)); + memcpy(saved, data, sizeof(data)); + CHECK(afkv2_encode_open(data, 23, 0, 1, &written) == -ENOSPC); + CHECK(!memcmp(data, saved, sizeof(data))); + CHECK(afkv2_encode_open(data, sizeof(data), 0, 0, &written) == -EINVAL); + CHECK(!memcmp(data, saved, sizeof(data))); + CHECK(afkv2_encode_frame(data, sizeof(data), 0, 1, 0, 0, 0, 0, data, + SIZE_MAX, &written) == -E2BIG); + CHECK(!memcmp(data, saved, sizeof(data))); + u8 *large = malloc(AFKV2_HEADER_SIZE + AFKV2_MAX_PAYLOAD); + + CHECK(large); + memset(large, 0x5a, AFKV2_HEADER_SIZE + AFKV2_MAX_PAYLOAD); + CHECK(!afkv2_encode_frame(large, AFKV2_HEADER_SIZE + AFKV2_MAX_PAYLOAD, + 0, 1, 0, 0xc0, 1, 0, large, AFKV2_MAX_PAYLOAD, + &written)); + CHECK(!afkv2_decode_frame(large, written, false, &frame)); + CHECK(frame.payload_size == AFKV2_MAX_PAYLOAD && + frame.payload[0] == 0x5a); + free(large); + bool ids[256] = { 0 }; + + for (unsigned int generation = 0; generation < 16; generation++) { + for (unsigned int slot = 0; slot < 16; slot++) { + u8 id = afkv2_command_id(generation, slot); + + CHECK(!ids[id]); + ids[id] = true; + CHECK((id & 15) == slot); + } + } + CHECK(afkv2_command_id(16, 3) == afkv2_command_id(0, 3)); + CHECK(afkv2_owned_span(0x1000, 32, 0x1000, 32)); + CHECK(afkv2_owned_span(0x1000, 32, 0x101c, 4)); + CHECK(!afkv2_owned_span(0x1000, 32, 0x0fff, 1)); + CHECK(!afkv2_owned_span(0x1000, 32, 0x101d, 4)); + CHECK(!afkv2_owned_span(0x1000, 32, 0x1020, 1)); + CHECK(!afkv2_owned_span(0x1000, 0, 0x1000, 0)); + CHECK(afkv2_owned_span(UINT64_MAX - 15, 16, UINT64_MAX, 1)); + CHECK(!afkv2_owned_span(UINT64_MAX - 15, 17, UINT64_MAX, 1)); + printf("PASS %u assertions\n", assertions); + return 0; +} diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-owned.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-owned.py new file mode 100644 index 0000000000000..21fe63b6d3823 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-owned.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Compile the production ownership checker with host-only synchronization mocks.""" +import hashlib +from pathlib import Path +import sys + + +def main(): + source = Path(sys.argv[1]).read_text(encoding="utf-8") + output = Path(sys.argv[2]) + first = "static int afk_v2_access_owned(" + last = "static void afk_v2_callback(" + if source.count(first) != 1 or source.count(last) != 1: + raise SystemExit("FAIL: ownership function boundaries changed; review extractor") + start = source.index(first) + end = source.index(last, start) + body = source[start:end] + if "int afk_v2_copy_owned(" not in body: + raise SystemExit("FAIL: production ownership wrapper missing") + directory = Path(__file__).resolve().parent + output.write_text((directory / "owned-prefix.c").read_text() + body + + (directory / "owned-suffix.c").read_text(), encoding="utf-8") + print("Production ownership function SHA256:", hashlib.sha256(body.encode()).hexdigest()) + + +if __name__ == "__main__": + main() diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h new file mode 100644 index 0000000000000..a5c3a3f44a591 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#define EINVAL 22 +#define ENOSPC 28 +#define E2BIG 7 +#define EPROTO 71 diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/string.h b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/string.h new file mode 100644 index 0000000000000..e74214a66ed83 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/string.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/types.h b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/types.h new file mode 100644 index 0000000000000..4d490ef828ccd --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/types.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c new file mode 100644 index 0000000000000..8ff88ec1ec5b0 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Aurora Silicon */ +#undef NDEBUG +#include +#include +#include "afkv2-codec.h" +#define MAX_PENDING_CMDS 16 +#define SZ_1M 1048576U +#define EACCES 13 +#define spin_lock_irqsave(lock, flags) \ + do { \ + (void)(lock); \ + (flags) = 0; \ + } while (0) +#define spin_unlock_irqrestore(lock, flags) \ + do { \ + (void)(lock); \ + (void)(flags); \ + } while (0) +#define dma_rmb() \ + do { \ + } while (0) +#define dma_wmb() \ + do { \ + } while (0) +#define dev_warn_ratelimited(...) \ + do { \ + } while (0) +static int test_bit(unsigned int bit, const unsigned long *map) +{ + return !!(*map & (1UL << bit)); +} + +struct epic_cmd_info { + bool done; + void *rxbuf, *txbuf; + u64 rxbuf_dma, txbuf_dma; + size_t rxlen, txlen; +}; + +struct apple_dcp_afkep { + bool wire_v2; +}; + +struct apple_epic_service { + int lock; + struct apple_dcp_afkep *ep; + unsigned long cmd_map[1]; + struct epic_cmd_info cmds[MAX_PENDING_CMDS]; +}; diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c new file mode 100644 index 0000000000000..770b16bc2fa94 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Aurora Silicon */ +int main(void) +{ + u8 rx[16], tx[16], output[16]; + struct apple_dcp_afkep ep = { .wire_v2 = true }; + struct apple_epic_service service = { .ep = &ep }; + struct epic_cmd_info *cmd = &service.cmds[0]; + + memset(rx, 0x12, sizeof(rx)); + memset(tx, 0x34, sizeof(tx)); + memset(output, 0x56, sizeof(output)); + cmd->rxbuf = rx; + cmd->rxbuf_dma = 0x10000001000ULL; + cmd->rxlen = sizeof(rx); + cmd->txbuf = tx; + cmd->txbuf_dma = 0x10000002000ULL; + cmd->txlen = sizeof(tx); + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma, output, 4, false) == + -EACCES); + service.cmd_map[0] = 1; + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma + 12, output, 4, + false) == 0); + assert(output[0] == 0x12 && output[3] == 0x12 && output[4] == 0x56); + assert(afk_v2_copy_owned(&service, cmd->txbuf_dma + 4, output, 4, + false) == 0); + assert(output[0] == 0x34 && output[4] == 0x56); + memset(output, 0x78, 4); + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma + 8, output, 4, + true) == 0); + assert(rx[7] == 0x12 && rx[8] == 0x78 && rx[11] == 0x78 && + rx[12] == 0x12); + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma - 1, output, 1, + true) == -EACCES); + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma + 13, output, 4, + true) == -EACCES); + assert(afk_v2_copy_owned(&service, cmd->txbuf_dma + 16, output, 1, + true) == -EACCES); + cmd->done = true; + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma, output, 1, false) == + -EACCES); + cmd->done = false; + ep.wire_v2 = false; + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma, output, 1, false) == + -EINVAL); + ep.wire_v2 = true; + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma, NULL, 1, false) == + -EINVAL); + assert(afk_v2_copy_owned(&service, 0, NULL, 0, false) == 0); + assert(afk_v2_access_owned(&service, cmd->rxbuf_dma + 1, NULL, 15, true, + false) == 0); + assert(afk_v2_access_owned(&service, cmd->rxbuf_dma + 1, NULL, 16, true, + false) == -EACCES); + puts("PASS owned-copy boundaries (actual extracted function; lock/barrier mocks)"); + return 0; +} diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh new file mode 100755 index 0000000000000..4e13f6d05d986 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu + +afkv2_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +afkv2_root=${AFKV2_SOURCE_ROOT:-$(CDPATH= cd -- "$afkv2_dir/../../../../../.." && pwd)} +afkv2_cc=${HOSTCC:-${CC:-cc}} +afkv2_cc_command=${afkv2_cc%% *} +afkv2_python=${PYTHON:-python3} +if ! command -v "$afkv2_cc_command" >/dev/null 2>&1 || + ! command -v "$afkv2_python" >/dev/null 2>&1; then + echo "SKIP: a host C compiler and Python 3 are required" + exit 4 +fi +if [ ! -f "$afkv2_root/drivers/gpu/drm/apple/afkv2-codec.h" ]; then + echo "SKIP: run from a source tree containing the AFKV2 codec" + exit 4 +fi +afkv2_tmp=$(mktemp -d "${TMPDIR:-/tmp}/apple-afkv2.XXXXXX") +trap 'rm -rf -- "$afkv2_tmp"' EXIT +trap 'exit 1' HUP INT TERM +afkv2_sanitizers= +if [ "${AFKV2_SANITIZERS:-0}" = 1 ]; then + afkv2_sanitizers="-fsanitize=address,undefined -fno-omit-frame-pointer" +fi + +# Compiler arguments, CFLAGS and sanitizers use normal compiler word splitting. +# shellcheck disable=SC2086 +$afkv2_cc -std=c11 -Wall -Wextra -Werror -pedantic ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_dir/codec.c" -o "$afkv2_tmp/codec" +"$afkv2_tmp/codec" +"$afkv2_python" "$afkv2_dir/extract-owned.py" \ + "$afkv2_root/drivers/gpu/drm/apple/afk.c" "$afkv2_tmp/owned.c" +# The extracted kernel function uses GNU C void-pointer arithmetic. +# shellcheck disable=SC2086 +$afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/owned.c" -o "$afkv2_tmp/owned" +"$afkv2_tmp/owned" From b4926f7a70ed407ff316e8ab339623e7cb62d219 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:44:13 +1100 Subject: [PATCH 0901/1579] dt-bindings: phy: apple,atcphy: describe T8140 DP setup lists Document the three bounded calibration lists used for T8140 DisplayPort setup. Describe their register regions and ordering, and reject these properties on other PHY variants. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/phy/apple,atcphy.yaml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml index 2484121870400..fef0605343e8f 100644 --- a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -50,6 +50,9 @@ allOf: reg-names: maxItems: 5 apple,tunable-usb2phy-reg-dflt: false + apple,tunable-dp-common-pre: false + apple,tunable-dp-lpdptx: false + apple,tunable-dp-common-post: false $defs: apple,tunable: @@ -179,6 +182,27 @@ properties: Note that lane here refers to a USB RX and TX pair re-used for DisplayPort and not to an individual DisplayPort differential lane. + apple,tunable-dp-common-pre: + $ref: "#/$defs/apple,tunable" + maxItems: 5461 + description: > + T8140 DisplayPort core setup applied before the LPDPTX list. The + bootloader supplies device-specific values for the core register region. + + apple,tunable-dp-lpdptx: + $ref: "#/$defs/apple,tunable" + maxItems: 5461 + description: > + T8140 DisplayPort setup for the lpdptx register region, applied between + the common-pre and common-post lists. + + apple,tunable-dp-common-post: + $ref: "#/$defs/apple,tunable" + maxItems: 5461 + description: > + T8140 DisplayPort core setup applied after the LPDPTX list. All three + DisplayPort lists are needed before selecting a DisplayPort PHY mode. + apple,tunable-usb2phy-reg-dflt: $ref: "#/$defs/apple,tunable" description: > From 7f67eba0ba5fe4a99b9ac9925dcdcf2a4a9e927b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:44:13 +1100 Subject: [PATCH 0902/1579] phy: apple: apply the T8140 DisplayPort setup lists The T8140 variant currently returns from AUX setup without configuring it. The bootloader supplies three ordered DisplayPort setup lists, but the PHY driver does not read or apply them. Validate every list against its mapped register resource and apply the complete bundle at AUX setup. Reject DP selection if the bundle is missing or invalid, while keeping ordinary USB operation independent. Other PHY generations keep their existing setup paths. Drive-level training programming remains unchanged. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-dp-tunables.h | 13 ++++++ drivers/phy/apple/atc.c | 68 ++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 drivers/phy/apple/atc-dp-tunables.h diff --git a/drivers/phy/apple/atc-dp-tunables.h b/drivers/phy/apple/atc-dp-tunables.h new file mode 100644 index 0000000000000..6d0cb00158c71 --- /dev/null +++ b/drivers/phy/apple/atc-dp-tunables.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* Copyright 2026 Aurora Silicon */ +#ifndef APPLE_ATC_DP_TUNABLES_H +#define APPLE_ATC_DP_TUNABLES_H + +#include + +static inline bool atc_dp_list_length_valid(int length) +{ + return length >= 0 && length <= 65536 && length % 12 == 0; +} + +#endif diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index a5f8b7c0d5e5c..2fec5c7821388 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -46,6 +46,8 @@ #include #include +#include "atc-dp-tunables.h" + #define AUSPLL_FSM_CTRL 0x1014 #define AUSPLL_APB_CMD_OVERRIDE 0x2000 @@ -617,6 +619,7 @@ struct atcphy_mode_configuration { * @pipe_mux_clk: PIPE clock selection field mask * @pipe_mux_data: PIPE data selection field mask * @has_usb2phy_reg: The PHY has the secondary eUSB2 register bank (T8140) + * @has_t8140_dp: Uses the T8140 DisplayPort tunable bundle * @optional_tunables: The bootloader may leave out the common-a tunables (this * generation has none) and the SuperSpeed tunables; USB2 * still works without the latter @@ -630,6 +633,7 @@ struct atcphy_hw { u32 pipe_mux_data; bool has_usb2phy_reg; bool optional_tunables; + bool has_t8140_dp; }; /** @@ -642,9 +646,13 @@ struct atcphy_hw { * @tunables.lane_usb3: USB3 lane-specific tunables * @tunables.lane_dp: DisplayPort lane-specific tunables * @tunables.lane_usb4: USB4 lane-specific tunables + * @tunables.dp_common_pre: T8140 DisplayPort common setup + * @tunables.dp_lpdptx: T8140 DisplayPort LPDPTX setup + * @tunables.dp_common_post: T8140 DisplayPort common completion * @tunables.usb2phy_reg_dflt: Defaults for the secondary eUSB2 register bank * @hw: SoC-specific PHY description * @ss_tunables: The complete SuperSpeed tunable set was supplied + * @dp_tunables: The complete T8140 DisplayPort bundle was supplied * @fixed_usb2: The USB2 pairs go to a fixed hub on the USB controller, which * stays in host mode; the PHY provides USB2 from probe on * @typec_mode: Mode the Type-C mux last asked for; a fixed-hub port returns to @@ -671,6 +679,7 @@ struct atcphy_hw { * @regs.lpdptx: DisplayPort registers * @res: Resources for memory-mapped registers, used to verify that tunables aren't out of bounds * @res.core: Core register resource + * @res.lpdptx: DisplayPort register resource * @res.axi2af: AXI to Apple Fabric interface resource * @res.usb2phy_reg: Secondary eUSB2 register bank resource * @phys: PHY instances @@ -694,10 +703,14 @@ struct apple_atcphy { struct apple_tunable *lane_dp[2]; struct apple_tunable *lane_usb4[2]; struct apple_tunable *usb2phy_reg_dflt; + struct apple_tunable *dp_common_pre; + struct apple_tunable *dp_lpdptx; + struct apple_tunable *dp_common_post; } tunables; const struct atcphy_hw *hw; bool ss_tunables; + bool dp_tunables; bool fixed_usb2; enum atcphy_mode typec_mode; bool host_active; @@ -724,6 +737,7 @@ struct apple_atcphy { struct { struct resource *core; + struct resource *lpdptx; struct resource *axi2af; struct resource *usb2phy_reg; } res; @@ -1482,6 +1496,13 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) { + if (atcphy->hw->has_t8140_dp) { + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); + apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); + return; + } + /* FIXME */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) return; @@ -2523,6 +2544,9 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) u32 reg; lockdep_assert_held(&atcphy->lock); + if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && + !atcphy->dp_tunables) + return -EOPNOTSUPP; /* the mode change below may power the PHY down: stop the tunnel clock first */ atc_tunnel_stop_t8103(atcphy); atc_tunnel_restore(atcphy); @@ -3087,6 +3111,9 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta case APPLE_ATCPHY_MODE_USB3: case APPLE_ATCPHY_MODE_USB3_DP: case APPLE_ATCPHY_MODE_DP: + if (atcphy->hw->has_t8140_dp && atcphy_modes[target_mode].enable_dp_aux && + !atcphy->dp_tunables) + return -EOPNOTSUPP; /* Without the lane tunables the SuperSpeed lanes are not calibrated */ if (!atcphy->ss_tunables) return -EOPNOTSUPP; @@ -3151,6 +3178,37 @@ static int atcphy_probe_mux(struct apple_atcphy *atcphy) return devm_add_action_or_reset(atcphy->dev, atcphy_typec_mux_unregister, mux); } +static int atcphy_load_t8140_dp(struct apple_atcphy *atcphy) +{ + struct { + const char *name; + struct resource *resource; + struct apple_tunable **out; + } lists[] = { + { "apple,tunable-dp-common-pre", atcphy->res.core, + &atcphy->tunables.dp_common_pre }, + { "apple,tunable-dp-lpdptx", atcphy->res.lpdptx, + &atcphy->tunables.dp_lpdptx }, + { "apple,tunable-dp-common-post", atcphy->res.core, + &atcphy->tunables.dp_common_post }, + }; + if (!of_machine_is_compatible("apple,j700")) + return -EOPNOTSUPP; + for (unsigned int i = 0; i < ARRAY_SIZE(lists); i++) { + struct property *property = of_find_property(atcphy->np, lists[i].name, NULL); + + if (!property || !atc_dp_list_length_valid(property->length)) + return -EINVAL; + *lists[i].out = devm_apple_tunable_parse(atcphy->dev, atcphy->np, + lists[i].name, lists[i].resource); + if (IS_ERR(*lists[i].out)) + return PTR_ERR(*lists[i].out); + } + + atcphy->dp_tunables = true; + return 0; +} + static int atcphy_load_tunables(struct apple_atcphy *atcphy) { size_t tunable_count; @@ -3215,6 +3273,13 @@ static int atcphy_load_tunables(struct apple_atcphy *atcphy) atcphy->tunables.usb2phy_reg_dflt = tunable; } + if (atcphy->hw->has_t8140_dp) { + int ret = atcphy_load_t8140_dp(atcphy); + + if (ret) + dev_warn(atcphy->dev, "DisplayPort tunable bundle unavailable: %d\n", ret); + } + atcphy->ss_tunables = !ss_missing; if (ss_missing) dev_warn(atcphy->dev, "SuperSpeed tunables missing, USB2 only\n"); @@ -3230,7 +3295,7 @@ static int atcphy_map_resources(struct platform_device *pdev, struct apple_atcph struct resource **res; } resources[] = { { "core", &atcphy->regs.core, &atcphy->res.core }, - { "lpdptx", &atcphy->regs.lpdptx, NULL }, + { "lpdptx", &atcphy->regs.lpdptx, &atcphy->res.lpdptx }, { "axi2af", &atcphy->regs.axi2af, &atcphy->res.axi2af }, { "usb2phy", &atcphy->regs.usb2phy, NULL }, { "pipehandler", &atcphy->regs.pipehandler, NULL }, @@ -3399,6 +3464,7 @@ static const struct atcphy_hw atcphy_hw_t8140 = { .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, .has_usb2phy_reg = true, + .has_t8140_dp = true, .optional_tunables = true, }; From f019831620ddbe8d8eea7173cab766c21154cc97 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:54:12 +1100 Subject: [PATCH 0903/1579] drm/apple: open the version-two DPTX controller interfaces The experimental external profile uses AFKV2 for its DPTX port, but the companion controller endpoint still uses the legacy path. Its interfaces are discovered without receiving the version-two OPEN handshake. Select AFKV2 for the controller before starting it, so controller publication performs the same OPEN-before-ready sequence as the port. Add a one-time PHY setup message to distinguish successful calibration application from controller activation during hardware qualification. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 3 +++ drivers/phy/apple/atc.c | 1 + 2 files changed, 4 insertions(+) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 8d605beb7faeb..856a9a75a389d 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -885,6 +885,9 @@ int dptx_controller_init(struct apple_dcp *dcp) ep = afk_init(dcp, DPTX_CONTROLLER_ENDPOINT, dptx_controller_ops); if (IS_ERR(ep)) return PTR_ERR(ep); + /* Controller interfaces require the same OPEN handshake as the port. */ + ep->wire_v2 = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; dcp->dptx_controller_ep = ep; ret = afk_start(ep); diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 2fec5c7821388..6a2020e36eed2 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1500,6 +1500,7 @@ static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); + dev_info_once(atcphy->dev, "Applied T8140 DisplayPort setup lists\n"); return; } From 294891122c094263fa45fc23404064b553e0b8ed Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:59:36 +1100 Subject: [PATCH 0904/1579] usb: typec: expose mux callback results through dynamic debug A DisplayPort mode can reach the display route without the PHY setup function being called. Add default-off dynamic-debug messages for each mux callback result and for the ATC requested, current and target modes. This distinguishes callback dispatch, unavailable setup data and an unchanged PHY mode without accessing additional hardware registers. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 4 ++++ drivers/usb/typec/mux.c | 1 + 2 files changed, 5 insertions(+) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 6a2020e36eed2..e5e26712c5a4d 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -3040,6 +3040,9 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta int ret; guard(mutex)(&atcphy->lock); + dev_dbg(atcphy->dev, "mux mode=%lu svid=%04x current=%u fixed=%u dp=%u ss=%u\n", + state->mode, state->alt ? state->alt->svid : 0, atcphy->mode, + atcphy->fixed_usb2, atcphy->dp_tunables, atcphy->ss_tunables); if (state->mode == TYPEC_STATE_SAFE) { target_mode = APPLE_ATCPHY_MODE_OFF; @@ -3100,6 +3103,7 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta if (atcphy->fixed_usb2 && target_mode == APPLE_ATCPHY_MODE_OFF) target_mode = APPLE_ATCPHY_MODE_USB2; + dev_dbg(atcphy->dev, "mux target=%u\n", target_mode); if (atcphy->mode == target_mode) return 0; diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index 7e327118cbd71..dc763766012f7 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -388,6 +388,7 @@ int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) mux_dev = mux->mux_devs[i]; ret = mux_dev->set(mux_dev, state); + dev_dbg(&mux_dev->dev, "mode=%lu callback result=%d\n", state->mode, ret); if (ret && ret != -EOPNOTSUPP) return ret; } From 0be54abf018f7b3a2deee17f8eec50536de3b063 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:05:27 +1100 Subject: [PATCH 0905/1579] tools: add bounded J700 ATC setup readback Add an explicit read-only diagnostic for the exact words selected by the T8140 DisplayPort setup lists. Validate the machine, mapped resource layout, list alignment and every offset before opening the memory device. Compose overlapping writes and report only mismatching masks and offsets, without exposing calibration values or writing registers. Four hardware-independent tests cover write composition, separate resources and malformed or out-of-range lists. Hardware readback remains an observation rather than a display qualification result. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + tools/diagnostics/apple-atc-readback.py | 111 ++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 tools/diagnostics/apple-atc-readback.py diff --git a/MAINTAINERS b/MAINTAINERS index 3f1523e9fdb31..f299bbe5c35b9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2636,6 +2636,7 @@ F: drivers/nvme/host/apple.c F: drivers/nvmem/apple-efuses.c F: drivers/nvmem/apple-spmi-nvmem.c F: drivers/phy/apple/ +F: tools/diagnostics/apple-atc-readback.py F: drivers/pinctrl/pinctrl-apple-gpio.c F: drivers/pmdomain/apple/ F: drivers/power/reset/macsmc-reboot.c diff --git a/tools/diagnostics/apple-atc-readback.py b/tools/diagnostics/apple-atc-readback.py new file mode 100644 index 0000000000000..e0443446fea8f --- /dev/null +++ b/tools/diagnostics/apple-atc-readback.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Aurora Silicon +"""Read back only the J700 ATC words selected by its DP setup lists. + +No register is written. Output excludes calibration values. Run --selftest +without hardware access; --read-hardware requires permission to read /dev/mem. +""" +import argparse +import json +import os +from pathlib import Path +import struct +import unittest + +ROOT = Path('/sys/firmware/devicetree/base') +PHY = ROOT / 'soc/phy@40b000000' +GROUPS = ( + ('apple,tunable-dp-common-pre', 'core'), + ('apple,tunable-dp-lpdptx', 'lpdptx'), + ('apple,tunable-dp-common-post', 'core'), +) + + +def decode(data, size): + if len(data) > 65536 or len(data) % 12 or size < 4: + raise ValueError('invalid setup list size') + result = list(struct.iter_unpack('>III', data)) + for offset, mask, value in result: + if offset % 4 or offset > size - 4: + raise ValueError('setup word outside resource') + return result + + +def compose(groups): + expected = {} + for base, words in groups: + for offset, mask, value in words: + address = base + offset + known, prior = expected.get(address, (0, 0)) + expected[address] = (known | mask | value, (prior & ~mask) | value) + if len(expected) > 256: + raise ValueError('readback word limit exceeded') + return expected + + +def acquire(): + if b'apple,j700' not in (ROOT / 'compatible').read_bytes().split(b'\0'): + raise ValueError('requires J700') + if b'apple,t8140-atcphy' not in (PHY / 'compatible').read_bytes().split(b'\0'): + raise ValueError('requires T8140 ATC') + names = (PHY / 'reg-names').read_bytes().rstrip(b'\0').decode().split('\0') + raw = (PHY / 'reg').read_bytes() + if len(raw) != 16 * len(names): + raise ValueError('unexpected resource layout') + regs = dict(zip(names, struct.iter_unpack('>QQ', raw))) + if regs.get('core') != (0x40b000000, 0x4c000): + raise ValueError('unexpected core resource') + if regs.get('lpdptx') != (0x40b050000, 0x4000): + raise ValueError('unexpected LPDPTX resource') + groups = [] + for prop, region in GROUPS: + base, size = regs[region] + groups.append((base, decode((PHY / prop).read_bytes(), size))) + expected = compose(groups) + failures = [] + fd = os.open('/dev/mem', os.O_RDONLY | os.O_CLOEXEC) + try: + for address, (mask, value) in sorted(expected.items()): + data = os.pread(fd, 4, address) + if len(data) != 4: + raise OSError('short register read') + mismatch = (int.from_bytes(data, 'little') ^ value) & mask + if mismatch: + region = 'core' if address < regs['lpdptx'][0] else 'lpdptx' + failures.append({'region': region, + 'offset': hex(address - regs[region][0]), + 'mismatch_mask': hex(mismatch)}) + finally: + os.close(fd) + print(json.dumps({'checked_words': len(expected), 'mismatches': failures})) + + +class CodecTests(unittest.TestCase): + def test_overlap(self): + self.assertEqual(compose([(16, [(0, 15, 5), (0, 3, 2)])]), {16: (15, 6)}) + + def test_set_outside_mask(self): + self.assertEqual(compose([(0, [(0, 1, 8)])]), {0: (9, 8)}) + + def test_regions(self): + self.assertEqual(len(compose([(0, [(0, 1, 1)]), (16, [(0, 1, 0)])])), 2) + + def test_bounds(self): + for data, size in [(b'x', 4), (struct.pack('>III', 4, 1, 1), 4), + (struct.pack('>III', 1, 1, 1), 8)]: + with self.assertRaises(ValueError): + decode(data, size) + self.assertEqual(decode(struct.pack('>III', 4, 1, 1), 8), [(4, 1, 1)]) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--selftest', action='store_true') + group.add_argument('--read-hardware', action='store_true') + args = parser.parse_args() + if args.selftest: + unittest.main(argv=['apple-atc-readback'], exit=True) + else: + acquire() From 373e947ee1766c743283e43ecc5987863df4bd6f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:08:39 +1100 Subject: [PATCH 0906/1579] tools: map ATC readback words as read-only MMIO Use a page-aligned read-only mapping for each selected MMIO word. The memory-device read syscall does not provide the device mapping required by this target. Preserve exact resource and offset admission and never create a writable mapping. Test the same mapped-word reader against a temporary file, including its alignment refusal, alongside the existing setup-list composition tests. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/diagnostics/apple-atc-readback.py | 26 +++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tools/diagnostics/apple-atc-readback.py b/tools/diagnostics/apple-atc-readback.py index e0443446fea8f..8eb3c49c508cc 100644 --- a/tools/diagnostics/apple-atc-readback.py +++ b/tools/diagnostics/apple-atc-readback.py @@ -8,9 +8,11 @@ """ import argparse import json +import mmap import os from pathlib import Path import struct +import tempfile import unittest ROOT = Path('/sys/firmware/devicetree/base') @@ -44,6 +46,15 @@ def compose(groups): return expected +def read_word(fd, address): + if address % 4: + raise ValueError('unaligned word address') + page = address - address % mmap.PAGESIZE + with mmap.mmap(fd, mmap.PAGESIZE, flags=mmap.MAP_SHARED, + prot=mmap.PROT_READ, offset=page) as view: + return struct.unpack_from(' Date: Mon, 12 Oct 2026 18:10:36 +1100 Subject: [PATCH 0907/1579] tools: report the J700 AUX power control state Report the analog shutdown, supply override and clamp control bits beside the setup-list readback. Keep the read-only resource checks and avoid printing complete register values or calibration data. Add a decoder test covering asserted and cleared states. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/diagnostics/apple-atc-readback.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tools/diagnostics/apple-atc-readback.py b/tools/diagnostics/apple-atc-readback.py index 8eb3c49c508cc..3e67f49cc1c85 100644 --- a/tools/diagnostics/apple-atc-readback.py +++ b/tools/diagnostics/apple-atc-readback.py @@ -55,6 +55,18 @@ def read_word(fd, address): return struct.unpack_from(' Date: Mon, 12 Oct 2026 18:12:31 +1100 Subject: [PATCH 0908/1579] phy: apple: sequence J700 AUX power separately from calibration J700 reaches DisplayPort mode with its setup-list values present, but readback shows AUX still clamped and both supply overrides unset. The calibration lists do not perform the AUX power transition. Request the small and large supplies in order, release the clamp, then release analog shutdown. Check each masked state with a bounded poll. Track incomplete startup so a later shutdown can unwind it, and refuse to stop the domain if AUX cleanup fails. Leave ordinary USB operation and other PHY generations on their existing paths. Add host tests of the production transition helpers with register mocks, covering phase ordering, unrelated bits and failure at every stage. These pass with AddressSanitizer and UBSan; hardware qualification is pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/phy/apple/atc.c | 94 +++++++++++++++++++-- tools/diagnostics/test-apple-atc-aux.py | 106 ++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 7 deletions(-) create mode 100644 tools/diagnostics/test-apple-atc-aux.py diff --git a/MAINTAINERS b/MAINTAINERS index f299bbe5c35b9..42369620cc9a2 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2637,6 +2637,7 @@ F: drivers/nvmem/apple-efuses.c F: drivers/nvmem/apple-spmi-nvmem.c F: drivers/phy/apple/ F: tools/diagnostics/apple-atc-readback.py +F: tools/diagnostics/test-apple-atc-aux.py F: drivers/pinctrl/pinctrl-apple-gpio.c F: drivers/pmdomain/apple/ F: drivers/power/reset/macsmc-reboot.c diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index e5e26712c5a4d..013d2f36d4d3b 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -653,6 +653,7 @@ struct atcphy_hw { * @hw: SoC-specific PHY description * @ss_tunables: The complete SuperSpeed tunable set was supplied * @dp_tunables: The complete T8140 DisplayPort bundle was supplied + * @aux_power_requested: AUX power-up began and needs matching shutdown * @fixed_usb2: The USB2 pairs go to a fixed hub on the USB controller, which * stays in host mode; the PHY provides USB2 from probe on * @typec_mode: Mode the Type-C mux last asked for; a fixed-hub port returns to @@ -711,6 +712,7 @@ struct apple_atcphy { const struct atcphy_hw *hw; bool ss_tunables; bool dp_tunables; + bool aux_power_requested; bool fixed_usb2; enum atcphy_mode typec_mode; bool host_active; @@ -1494,19 +1496,81 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode } } -static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) +#define J700_AUX_ANALOG_CONTROL 0x16000 +#define J700_AUX_SUPPLY_CONTROL 0x16400 +#define J700_AUX_ANALOG_OFF BIT(0) +#define J700_AUX_CLAMP_SELECT BIT(8) +#define J700_AUX_CLAMP_ENABLE BIT(9) + +static int j700_aux_update(struct apple_atcphy *atcphy, u32 offset, u32 mask, u32 value) +{ + u32 observed; + + core_mask32(atcphy, offset, mask, value); + usleep_range(10, 20); + return readl_poll_timeout_atomic(atcphy->regs.core + offset, observed, + (observed & mask) == value, 1, 100); +} + +static int j700_aux_enable(struct apple_atcphy *atcphy) +{ + u32 clamp = J700_AUX_CLAMP_SELECT | J700_AUX_CLAMP_ENABLE; + int ret; + + /* Even an incomplete power-up must be unwound before the domain is stopped. */ + atcphy->aux_power_requested = true; + for (unsigned int stage = 0; stage < 2; stage++) { + u32 fields = 3U << (2 * stage); + + ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, fields, fields); + if (ret) + return ret; + } + ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, J700_AUX_CLAMP_SELECT); + if (ret) + return ret; + + return j700_aux_update(atcphy, J700_AUX_ANALOG_CONTROL, J700_AUX_ANALOG_OFF, 0); +} + +static int j700_aux_disable(struct apple_atcphy *atcphy) +{ + u32 clamp = J700_AUX_CLAMP_SELECT | J700_AUX_CLAMP_ENABLE; + int ret; + + if (!atcphy->aux_power_requested) + return 0; + ret = j700_aux_update(atcphy, J700_AUX_ANALOG_CONTROL, + J700_AUX_ANALOG_OFF, J700_AUX_ANALOG_OFF); + if (ret) + return ret; + ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, clamp); + if (ret) + return ret; + for (unsigned int remaining = 2; remaining; remaining--) { + u32 override = BIT(2 * remaining - 1); + + ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, override, 0); + if (ret) + return ret; + } + atcphy->aux_power_requested = false; + return 0; +} + +static int atcphy_enable_dp_aux(struct apple_atcphy *atcphy) { if (atcphy->hw->has_t8140_dp) { apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); dev_info_once(atcphy->dev, "Applied T8140 DisplayPort setup lists\n"); - return; + return j700_aux_enable(atcphy); } /* FIXME */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) - return; + return 0; core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N); core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N_OV); @@ -1556,10 +1620,14 @@ static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) FIELD_PREP(LPDPTX_AUX_MARGIN_RCAL_TXSWING, 12)); atcphy->dp_link_rate = -1; + return 0; } -static void atcphy_disable_dp_aux(struct apple_atcphy *atcphy) +static int atcphy_disable_dp_aux(struct apple_atcphy *atcphy) { + if (atcphy->hw->has_t8140_dp) + return j700_aux_disable(atcphy); + /* FIXME */ if (atcphy->hw->gen != ATCPHY_GENERATION_T8122) { set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_PWN_DOWN); @@ -1574,6 +1642,7 @@ static void atcphy_disable_dp_aux(struct apple_atcphy *atcphy) core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPRX_PCLK_ENABLE); core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK2_ENABLE); + return 0; } static int atcphy_dp_configure_lane(struct apple_atcphy *atcphy, enum atcphy_lane lane, @@ -1940,7 +2009,9 @@ static int atcphy_power_off(struct apple_atcphy *atcphy) u32 reg; int ret; - atcphy_disable_dp_aux(atcphy); + ret = atcphy_disable_dp_aux(atcphy); + if (ret) + return ret; /* Enable all reset lines */ core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); @@ -2548,6 +2619,12 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && !atcphy->dp_tunables) return -EOPNOTSUPP; + if (atcphy->hw->has_t8140_dp && !atcphy_modes[mode].enable_dp_aux && + atcphy->aux_power_requested) { + ret = j700_aux_disable(atcphy); + if (ret) + return ret; + } /* the mode change below may power the PHY down: stop the tunnel clock first */ atc_tunnel_stop_t8103(atcphy); atc_tunnel_restore(atcphy); @@ -2612,8 +2689,11 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) } /* Setup AUX channel if DP altmode is requested */ - if (atcphy_modes[mode].enable_dp_aux) - atcphy_enable_dp_aux(atcphy); + if (atcphy_modes[mode].enable_dp_aux) { + ret = atcphy_enable_dp_aux(atcphy); + if (ret) + return ret; + } /* Enable clocks and configure lanes */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8103) { diff --git a/tools/diagnostics/test-apple-atc-aux.py b/tools/diagnostics/test-apple-atc-aux.py new file mode 100644 index 0000000000000..8442694fd44ff --- /dev/null +++ b/tools/diagnostics/test-apple-atc-aux.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Compile the production J700 AUX transitions against a bounded register mock.""" +import hashlib +import os +from pathlib import Path +import shlex +import subprocess +import tempfile + +source = Path(__file__).resolve().parents[2] / 'drivers/phy/apple/atc.c' +text = source.read_text() +parts = [line for line in text.splitlines() if line.startswith('#define J700_AUX_')] +for name in ('j700_aux_update', 'j700_aux_enable', 'j700_aux_disable'): + start = text.index('static int ' + name + '(') + begin = text.index('{', start) + depth = 1 + end = begin + 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + parts.append(text[start:end]) +production = '\n'.join(parts) +print('Production transition SHA256:', hashlib.sha256(production.encode()).hexdigest()) +prefix = r''' +#include +#include +#include +#include +#include +typedef uint32_t u32; +#define BIT(n) (1U << (n)) +struct apple_atcphy { struct { unsigned char *core; } regs; bool aux_power_requested; }; +struct event { u32 offset, mask, value; } events[32]; +static u32 registers[0x18000 / 4]; +static unsigned int writes, reject; +static void core_mask32(struct apple_atcphy *phy, u32 offset, u32 mask, u32 value) +{ + assert(offset < sizeof(registers) && offset % 4 == 0 && writes < 32); + assert(!(value & ~mask)); + events[writes] = (struct event){offset, mask, value}; + writes++; + if (writes != reject) + registers[offset / 4] = (registers[offset / 4] & ~mask) | value; + (void)phy; +} +static void usleep_range(unsigned int low, unsigned int high) +{ + assert(low && high >= low); +} +#define readl_poll_timeout_atomic(address, value, condition, interval, timeout) \ + ({ (value) = *(u32 *)(address); (condition) ? 0 : -110; }) +''' +suffix = r''' +static void initial(struct apple_atcphy *phy) +{ + memset(registers, 0, sizeof(registers)); + memset(events, 0, sizeof(events)); + registers[J700_AUX_SUPPLY_CONTROL / 4] = 0x80000200; + registers[J700_AUX_ANALOG_CONTROL / 4] = 0x40000001; + *phy = (struct apple_atcphy){ .regs.core = (unsigned char *)registers }; + writes = reject = 0; +} +int main(void) +{ + struct apple_atcphy phy; + initial(&phy); + assert(j700_aux_disable(&phy) == 0 && writes == 0); + assert(j700_aux_enable(&phy) == 0 && phy.aux_power_requested && writes == 4); + assert(registers[J700_AUX_SUPPLY_CONTROL / 4] == 0x8000010f); + assert(registers[J700_AUX_ANALOG_CONTROL / 4] == 0x40000000); + assert(events[0].mask == 3 && events[1].mask == 12); + assert(events[2].mask == 0x300 && events[2].value == 0x100); + assert(events[3].offset == J700_AUX_ANALOG_CONTROL); + writes = 0; + assert(j700_aux_disable(&phy) == 0 && !phy.aux_power_requested && writes == 4); + assert(events[0].offset == J700_AUX_ANALOG_CONTROL); + assert(events[1].mask == 0x300 && events[2].mask == 8 && events[3].mask == 2); + assert(registers[J700_AUX_SUPPLY_CONTROL / 4] == 0x80000305); + assert(registers[J700_AUX_ANALOG_CONTROL / 4] == 0x40000001); + for (unsigned int fault = 1; fault <= 4; fault++) { + initial(&phy); + reject = fault; + assert(j700_aux_enable(&phy) == -110 && phy.aux_power_requested); + reject = 0; + writes = 0; + assert(j700_aux_disable(&phy) == 0 && !phy.aux_power_requested); + initial(&phy); + assert(j700_aux_enable(&phy) == 0); + writes = 0; + reject = fault; + assert(j700_aux_disable(&phy) == -110 && phy.aux_power_requested); + } + puts("PASS: sequencing, unrelated bits, every failed phase, retained cleanup ownership"); + return 0; +} +''' +with tempfile.TemporaryDirectory(prefix='apple-atc-aux-') as directory: + path = Path(directory) + (path / 'test.c').write_text(prefix + production + suffix) + command = shlex.split(os.environ.get('CC', 'cc')) + command += ['-std=gnu11', '-Wall', '-Wextra', '-Werror', '-O1'] + if os.environ.get('ATC_SANITIZERS') == '1': + command += ['-fsanitize=address,undefined', '-fno-omit-frame-pointer'] + subprocess.run(command + [str(path / 'test.c'), '-o', str(path / 'test')], check=True) + subprocess.run([str(path / 'test')], check=True) From 39af890eb392cea1ff58cc0fc4e47ebff6e8f3db Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:18:18 +1100 Subject: [PATCH 0909/1579] tools: report J700 lane protocol and power controls Extend the read-only state report with both lane protocol selections, power-control nibbles and receive override bits. Keep the fixed resource admission and add a synthetic two-lane decoding test. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/diagnostics/apple-atc-readback.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tools/diagnostics/apple-atc-readback.py b/tools/diagnostics/apple-atc-readback.py index 3e67f49cc1c85..6929e38064e25 100644 --- a/tools/diagnostics/apple-atc-readback.py +++ b/tools/diagnostics/apple-atc-readback.py @@ -67,6 +67,16 @@ def aux_state(analog, supply): } +def lane_state(protocol, power, receiver): + return [{'lane': lane, + 'rx_protocol': (protocol >> (6 * lane)) & 7, + 'tx_protocol': (protocol >> (6 * lane + 3)) & 7, + 'power_control': (power >> (4 * lane)) & 15, + 'receiver_small_override': bool(receiver & (1 << (2 + lane))), + 'receiver_large_override': bool(receiver & (1 << (6 + lane)))} + for lane in range(2)] + + def acquire(): if b'apple,j700' not in (ROOT / 'compatible').read_bytes().split(b'\0'): raise ValueError('requires J700') @@ -88,6 +98,7 @@ def acquire(): expected = compose(groups) failures = [] power = None + lanes = None fd = os.open('/dev/mem', os.O_RDONLY | os.O_CLOEXEC) try: for address, (mask, value) in sorted(expected.items()): @@ -99,12 +110,23 @@ def acquire(): 'mismatch_mask': hex(mismatch)}) power = aux_state(read_word(fd, regs['core'][0] + 0x16000), read_word(fd, regs['core'][0] + 0x16400)) + lanes = lane_state(read_word(fd, regs['core'][0] + 0x60), + read_word(fd, regs['core'][0] + 0x200), + read_word(fd, regs['core'][0] + 0xc)) finally: os.close(fd) - print(json.dumps({'checked_words': len(expected), 'mismatches': failures, 'aux_state': power})) + print(json.dumps({'checked_words': len(expected), 'mismatches': failures, 'aux_state': power, 'lanes': lanes})) class CodecTests(unittest.TestCase): + def test_lane_state(self): + lanes = lane_state(0x489, 0x10, 0x88) + self.assertEqual((lanes[0]['rx_protocol'], lanes[0]['tx_protocol']), (1, 1)) + self.assertEqual((lanes[1]['rx_protocol'], lanes[1]['tx_protocol']), (2, 2)) + self.assertEqual(lanes[1]['power_control'], 1) + self.assertTrue(lanes[1]['receiver_small_override']) + self.assertTrue(lanes[1]['receiver_large_override']) + def test_aux_state(self): self.assertTrue(all(aux_state(1, 0x30f).values())) self.assertFalse(any(aux_state(0, 0).values())) From f19979190af80c37fdaea4b30ef13dca3852f037 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:19:18 +1100 Subject: [PATCH 0910/1579] phy: apple: park the J700 DP lane before link training The selected DP lane still reports the active power-control nibble after its protocol changes, with both receive overrides clear. The J700 path requires this lane to reach its quiescent state before link training. Assert only the selected lane's receive overrides, then sequence its power-control field into idle with bounded readback checks. Preserve the USB lane and unrelated bits. Return failures to the mux caller instead of publishing a completed mode transition. Host tests of the production helpers cover both lanes, invalid indices, unrelated-bit preservation and a failed write at every phase, under AddressSanitizer and UBSan. Hardware qualification remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 88 +++++++++++++++++++------ tools/diagnostics/test-apple-atc-aux.py | 23 ++++++- 2 files changed, 90 insertions(+), 21 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 013d2f36d4d3b..600a3f09dcecf 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1440,7 +1440,50 @@ static void atcphy_setup_pipehandler(struct apple_atcphy *atcphy) atcphy_enable_dummy_phy(atcphy); } -static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode mode) +#define J700_PMA_RECEIVER_CONTROL 0x000c +#define J700_PMA_POWER_CONTROL 0x0200 + +static int j700_field_update(struct apple_atcphy *atcphy, u32 offset, u32 mask, u32 value) +{ + u32 observed; + + core_mask32(atcphy, offset, mask, value); + usleep_range(10, 20); + return readl_poll_timeout_atomic(atcphy->regs.core + offset, observed, + (observed & mask) == value, 1, 100); +} + +static int j700_pma_idle(struct apple_atcphy *atcphy, unsigned int lane) +{ + u32 field; + int ret; + + if (lane >= 2) + return -EINVAL; + for (unsigned int supply = 0; supply < 2; supply++) { + field = BIT(2 + lane + 4 * supply); + ret = j700_field_update(atcphy, J700_PMA_RECEIVER_CONTROL, field, field); + if (ret) + return ret; + } + field = BIT(4 * lane + 3); + ret = j700_field_update(atcphy, J700_PMA_POWER_CONTROL, field, 0); + if (ret) + return ret; + field = BIT(4 * lane); + ret = j700_field_update(atcphy, J700_PMA_POWER_CONTROL, field, field); + if (ret) + return ret; + for (unsigned int control = 1; control <= 2; control++) { + field = BIT(4 * lane + control); + ret = j700_field_update(atcphy, J700_PMA_POWER_CONTROL, field, 0); + if (ret) + return ret; + } + return 0; +} + +static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode mode) { const struct atcphy_mode_configuration *mode_cfg = atcphy_get_mode_config(atcphy, mode); @@ -1467,9 +1510,22 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode else core_clear32(atcphy, atcphy->hw->aciophy_crossbar, ACIOPHY_CROSSBAR_DP_BOTH_PMA); + if (atcphy->hw->has_t8140_dp) { + for (unsigned int lane = 0; lane < 2; lane++) { + int ret; + + if (!mode_cfg->dp_lane[lane]) + continue; + ret = j700_pma_idle(atcphy, lane); + if (ret) + return ret; + } + return 0; + } + /* unclear if we need the remainder for t8122 */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) - return; + return 0; if (mode_cfg->dp_lane[0]) { core_set32(atcphy, LN0_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, @@ -1494,6 +1550,7 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV); udelay(10); } + return 0; } #define J700_AUX_ANALOG_CONTROL 0x16000 @@ -1502,15 +1559,6 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode #define J700_AUX_CLAMP_SELECT BIT(8) #define J700_AUX_CLAMP_ENABLE BIT(9) -static int j700_aux_update(struct apple_atcphy *atcphy, u32 offset, u32 mask, u32 value) -{ - u32 observed; - - core_mask32(atcphy, offset, mask, value); - usleep_range(10, 20); - return readl_poll_timeout_atomic(atcphy->regs.core + offset, observed, - (observed & mask) == value, 1, 100); -} static int j700_aux_enable(struct apple_atcphy *atcphy) { @@ -1522,15 +1570,15 @@ static int j700_aux_enable(struct apple_atcphy *atcphy) for (unsigned int stage = 0; stage < 2; stage++) { u32 fields = 3U << (2 * stage); - ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, fields, fields); + ret = j700_field_update(atcphy, J700_AUX_SUPPLY_CONTROL, fields, fields); if (ret) return ret; } - ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, J700_AUX_CLAMP_SELECT); + ret = j700_field_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, J700_AUX_CLAMP_SELECT); if (ret) return ret; - return j700_aux_update(atcphy, J700_AUX_ANALOG_CONTROL, J700_AUX_ANALOG_OFF, 0); + return j700_field_update(atcphy, J700_AUX_ANALOG_CONTROL, J700_AUX_ANALOG_OFF, 0); } static int j700_aux_disable(struct apple_atcphy *atcphy) @@ -1540,17 +1588,17 @@ static int j700_aux_disable(struct apple_atcphy *atcphy) if (!atcphy->aux_power_requested) return 0; - ret = j700_aux_update(atcphy, J700_AUX_ANALOG_CONTROL, - J700_AUX_ANALOG_OFF, J700_AUX_ANALOG_OFF); + ret = j700_field_update(atcphy, J700_AUX_ANALOG_CONTROL, + J700_AUX_ANALOG_OFF, J700_AUX_ANALOG_OFF); if (ret) return ret; - ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, clamp); + ret = j700_field_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, clamp); if (ret) return ret; for (unsigned int remaining = 2; remaining; remaining--) { u32 override = BIT(2 * remaining - 1); - ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, override, 0); + ret = j700_field_update(atcphy, J700_AUX_SUPPLY_CONTROL, override, 0); if (ret) return ret; } @@ -2700,7 +2748,9 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) core_set32(atcphy, CIO3PLL_CLK_CTRL, CIO3PLL_CLK_PCLK_EN); core_set32(atcphy, CIO3PLL_CLK_CTRL, CIO3PLL_CLK_REFCLK_EN); } - atcphy_configure_lanes(atcphy, mode); + ret = atcphy_configure_lanes(atcphy, mode); + if (ret) + return ret; /* Take the USB3 PHY out of reset */ core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); diff --git a/tools/diagnostics/test-apple-atc-aux.py b/tools/diagnostics/test-apple-atc-aux.py index 8442694fd44ff..00f29b0a410b3 100644 --- a/tools/diagnostics/test-apple-atc-aux.py +++ b/tools/diagnostics/test-apple-atc-aux.py @@ -10,8 +10,8 @@ source = Path(__file__).resolve().parents[2] / 'drivers/phy/apple/atc.c' text = source.read_text() -parts = [line for line in text.splitlines() if line.startswith('#define J700_AUX_')] -for name in ('j700_aux_update', 'j700_aux_enable', 'j700_aux_disable'): +parts = [line for line in text.splitlines() if line.startswith(('#define J700_AUX_', '#define J700_PMA_'))] +for name in ('j700_field_update', 'j700_pma_idle', 'j700_aux_enable', 'j700_aux_disable'): start = text.index('static int ' + name + '(') begin = text.index('{', start) depth = 1 @@ -24,6 +24,7 @@ print('Production transition SHA256:', hashlib.sha256(production.encode()).hexdigest()) prefix = r''' #include +#include #include #include #include @@ -91,6 +92,24 @@ reject = fault; assert(j700_aux_disable(&phy) == -110 && phy.aux_power_requested); } + initial(&phy); + assert(j700_pma_idle(&phy, 2) == -EINVAL && writes == 0); + for (unsigned int lane = 0; lane < 2; lane++) { + initial(&phy); + registers[J700_PMA_POWER_CONTROL / 4] = 0x800000ee; + registers[J700_PMA_RECEIVER_CONTROL / 4] = 0x5000; + assert(j700_pma_idle(&phy, lane) == 0 && writes == 6); + assert(((registers[J700_PMA_POWER_CONTROL / 4] >> (4 * lane)) & 15) == 1); + assert(((registers[J700_PMA_POWER_CONTROL / 4] >> (4 * (1 - lane))) & 15) == 14); + assert(registers[J700_PMA_POWER_CONTROL / 4] & 0x80000000); + assert(registers[J700_PMA_RECEIVER_CONTROL / 4] == (0x5000U | (0x44U << lane))); + for (unsigned int fault = 1; fault <= 6; fault++) { + initial(&phy); + registers[J700_PMA_POWER_CONTROL / 4] = 0xee; + reject = fault; + assert(j700_pma_idle(&phy, lane) == -110); + } + } puts("PASS: sequencing, unrelated bits, every failed phase, retained cleanup ownership"); return 0; } From bc3b8aaa560f23e069a7ea2d774645ea5fec39cc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:24:32 +1100 Subject: [PATCH 0911/1579] arm64: dts: apple: select the second external DCP source on J700 Route the test connector to dispext1,0, matching its external DCP index. The selector denotes the crossbar source and is independent of physical Type-C port zero. Hardware routing facts were audited with user authorization; this DT change is independently written. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700-external-test.dts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts b/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts index d3965ab2c5061..fbd8e83738f0c 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts @@ -14,7 +14,7 @@ phy-names = "typec0"; mux-controls = <&dpxbar0 0>; mux-control-names = "typec0"; - apple,typec-mux-indices = <0>; + apple,typec-mux-indices = <2>; apple,typec-dptx-phys = <0>; apple,dcp-index = <1>; }; From ea3d81e81f781c0f9f73748f5e7d405e232e0f39 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:34:02 +1100 Subject: [PATCH 0912/1579] dt-bindings: phy: apple: describe T8140 DisplayPort drive presets Document the firmware-provided fallback and per-rate drive tables. Each has sixteen entries indexed by voltage swing and pre-emphasis; other PHYs do not accept these properties. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/phy/apple,atcphy.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml index fef0605343e8f..3e36244ef4d5b 100644 --- a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -53,6 +53,8 @@ allOf: apple,tunable-dp-common-pre: false apple,tunable-dp-lpdptx: false apple,tunable-dp-common-post: false + patternProperties: + "^apple,dp-training-table(-rbr|-hbr|-hbr2|-hbr3)?$": false $defs: apple,tunable: @@ -210,6 +212,16 @@ properties: T8140, copied from Apple's Device Tree by the bootloader. Unlike the other tunables these are applied to the usb2phy-reg region. +patternProperties: + "^apple,dp-training-table(-rbr|-hbr|-hbr2|-hbr3)?$": + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 16 + maxItems: 16 + description: > + Firmware-provided T8140 drive presets indexed by voltage swing times + four plus pre-emphasis. The unsuffixed table is the required fallback; + a link-rate suffix selects an optional rate-specific override. + required: - compatible - reg From 85d81b76238d5a3a4782d0ec443b7274cb1f72e0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:34:02 +1100 Subject: [PATCH 0913/1579] phy: apple: implement bounded J700 DisplayPort main-link training After correcting the crossbar route, J700 reaches SET_LINK_RATE but the legacy PHY path waits for the wrong command-ready register. Add an isolated T8140 main-link sequence, firmware drive presets, and cleanup ownership that survives failed setup or shutdown. Preserve the other PMA used for USB. Handle the version-two group-one drive command with complete bounds and range checks before changing the PHY. Cache training values only after a successful PHY operation and retain the legacy endpoint behavior. Hardware register facts were audited with explicit user authorization. The implementation and tests are independently written. The copy audit matched only three hardware-field expressions and standard C test headers. No private implementation patch was applied. Validation: 29 PHY state scenarios, 112 callback boundary cases, 100 group routing cases, 905 AFK assertions and production owned-buffer/AUX tests under ASan/UBSan; schema validation and strict checkpatch. Hardware qualification remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 4 + drivers/gpu/drm/apple/afk.c | 17 +- drivers/gpu/drm/apple/afk.h | 2 + drivers/gpu/drm/apple/dptx-j700-training.h | 54 +++ drivers/gpu/drm/apple/dptxep.c | 30 +- drivers/gpu/drm/apple/dptxep.h | 1 + drivers/phy/apple/atc-j700-dp.h | 333 +++++++++++++++++++ drivers/phy/apple/atc.c | 45 ++- tools/diagnostics/apple-dptx-training-test.c | 131 ++++++++ tools/diagnostics/apple-j700-dp-test.c | 226 +++++++++++++ tools/diagnostics/test-apple-j700-dp.py | 26 ++ 11 files changed, 861 insertions(+), 8 deletions(-) create mode 100644 drivers/gpu/drm/apple/dptx-j700-training.h create mode 100644 drivers/phy/apple/atc-j700-dp.h create mode 100644 tools/diagnostics/apple-dptx-training-test.c create mode 100644 tools/diagnostics/apple-j700-dp-test.c create mode 100644 tools/diagnostics/test-apple-j700-dp.py diff --git a/MAINTAINERS b/MAINTAINERS index 42369620cc9a2..4db1e45c648ce 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2006,6 +2006,7 @@ F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ +F: drivers/phy/apple/atc-j700-dp.h F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs F: drivers/soc/apple/mcc.c @@ -2015,6 +2016,9 @@ F: include/linux/soc/apple/pci-apple-piodma.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs +F: tools/diagnostics/apple-dptx-training-test.c +F: tools/diagnostics/apple-j700-dp-test.c +F: tools/diagnostics/test-apple-j700-dp.py F: tools/testing/apple/ APPLETALK NETWORK LAYER diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index e7c76e2d8a610..b7c3905303eab 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -708,7 +708,8 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, (command->oob && request_size > command->rx_size)) goto reject; service = afk_service_get(afk_epic_find_service(ep, frame->interface)); - if (!service || READ_ONCE(service->torndown) || !service->ops->call) { + if (!service || READ_ONCE(service->torndown) || + (!service->ops->call && !service->ops->call_v2)) { if (service) afk_service_put(service); goto reject; @@ -732,7 +733,7 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, } call = (const void *)request; body_size = request_size - sizeof(*call); - if (le16_to_cpu(call->group) || + if ((le16_to_cpu(call->group) && !service->ops->call_v2) || le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || le32_to_cpu(call->data_len) != body_size) goto release; @@ -743,9 +744,15 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, } reply[1] = command->id; memcpy(reply + AFKV2_COMMAND_SIZE, call, sizeof(*call)); - ret = service->ops->call(service, le32_to_cpu(call->command), - request + sizeof(*call), body_size, - reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + if (service->ops->call_v2) + ret = service->ops->call_v2(service, le16_to_cpu(call->group), + le32_to_cpu(call->command), + request + sizeof(*call), body_size, + reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + else + ret = service->ops->call(service, le32_to_cpu(call->command), + request + sizeof(*call), body_size, + reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); if (ret) goto release; reply_size = AFKV2_COMMAND_SIZE + request_size; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index a4de9090681c2..3d1f975d798a8 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -76,6 +76,8 @@ struct apple_epic_service_ops { int (*call)(struct apple_epic_service *service, u32 idx, const void *data, size_t data_size, void *reply, size_t reply_size); + int (*call_v2)(struct apple_epic_service *service, u16 group, u32 command, + const void *data, size_t size, void *reply, size_t room); int (*report)(struct apple_epic_service *service, enum epic_subtype type, const void *data, size_t data_size); void (*teardown)(struct apple_epic_service *service); diff --git a/drivers/gpu/drm/apple/dptx-j700-training.h b/drivers/gpu/drm/apple/dptx-j700-training.h new file mode 100644 index 0000000000000..09a5d1018a7fb --- /dev/null +++ b/drivers/gpu/drm/apple/dptx-j700-training.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Aurora Silicon */ +#ifndef APPLE_DPTX_J700_TRAINING_H +#define APPLE_DPTX_J700_TRAINING_H + +static int dptx_j700_training(struct apple_epic_service *service, bool write, + const void *data, size_t size, void *reply, size_t room) +{ + struct dptx_port *port = service->cookie; + union phy_configure_opts options = {}; + u32 count; + int ret; + + if (!port || !port->atcphy) + return -ENODEV; + if (size < 32 || room < 32) + return -EINVAL; + count = get_unaligned_le32(data + (write ? 16 : 0)); + if (count > 4 || (write && (!count || count == 3)) || room < 32 + 12 * count || + (write && size < 32 + 12 * count)) + return -EINVAL; + if (write) { + ret = phy_validate(port->atcphy, PHY_MODE_DP, 0, &options); + if (ret) + return ret; + if (count > options.dp.lanes) + return -EINVAL; + for (unsigned int i = 0; i < count; i++) { + const u8 *entry = data + 32 + 12 * i; + + if (get_unaligned_le32(entry) || get_unaligned_le32(entry + 4) > 3 || + get_unaligned_le32(entry + 8) > 3) + return -EINVAL; + options.dp.voltage[i] = get_unaligned_le32(entry + 4); + options.dp.pre[i] = get_unaligned_le32(entry + 8); + } + options.dp.lanes = count; + options.dp.set_voltages = true; + ret = phy_configure(port->atcphy, &options); + if (ret) + return ret; + memcpy(port->trained_voltage, options.dp.voltage, sizeof(port->trained_voltage)); + memcpy(port->trained_pre, options.dp.pre, sizeof(port->trained_pre)); + } + memset(reply, 0, room); + put_unaligned_le32(count, reply + (write ? 16 : 0)); + for (unsigned int i = 0; i < count; i++) { + put_unaligned_le32(port->trained_voltage[i], reply + 36 + 12 * i); + put_unaligned_le32(port->trained_pre[i], reply + 40 + 12 * i); + } + return 0; +} + +#endif diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 856a9a75a389d..d2ab05ef9e57a 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -17,6 +18,8 @@ #include "parser.h" #include "trace.h" +#include "dptx-j700-training.h" + struct dcpdptx_connection_cmd { __le32 unk; __le32 target; @@ -264,6 +267,13 @@ dptxport_call_get_max_drive_settings(struct apple_epic_service *service, return -EINVAL; reply->retcode = cpu_to_le32(0); + if (service->ep->wire_v2) { + memset(reply, 0, reply_size); + put_unaligned_le32(3, reply_ + 20); + put_unaligned_le32(3, reply_ + 24); + return 0; + } + reply->max_drive_settings[0] = cpu_to_le32(0x3); reply->max_drive_settings[1] = cpu_to_le32(0x3); @@ -405,6 +415,9 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic u64 lane_count = le64_to_cpu(request->lane_count); + if (service->ep->wire_v2 && lane_count > dptx->lane_count) + return -EINVAL; + if (dptx->lane_count < lane_count) dev_err(dcp->dev, "set_active_lane_count: unexpected lane " "count:%llu phy: %d\n", lane_count, dptx->lane_count); @@ -415,7 +428,7 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic dptx->phy_ops.dp.lanes = lane_count; /* USB4 DP IN has no ATC PHY; still accept the lane count. */ dptx->phy_ops.dp.set_lanes = - dcp_is_usb4_output(dcp) || (dcp->dptx_phy > 3); + service->ep->wire_v2 || dcp_is_usb4_output(dcp) || (dcp->dptx_phy > 3); break; default: dev_err(dcp->dev, "set_active_lane_count: invalid lane count:%llu\n", lane_count); @@ -853,11 +866,26 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, } } +static int dptxport_group_call(struct apple_epic_service *service, u16 group, u32 command, + const void *data, size_t size, void *reply, size_t room) +{ + if (group == 1 && command == DPTX_APCALL_SET_DRIVE_SETTINGS) + return dptx_j700_training(service, true, data, size, reply, room); + if (group) + return -EOPNOTSUPP; + if (command == DPTX_APCALL_SET_DRIVE_SETTINGS) + return -EOPNOTSUPP; + if (command == DPTX_APCALL_GET_DRIVE_SETTINGS) + return dptx_j700_training(service, false, data, size, reply, room); + return dptxport_call(service, command, data, size, reply, room); +} + static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", .init = dptxport_init, .call = dptxport_call, + .call_v2 = dptxport_group_call, }, {} }; diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 885e3675f9db5..3eb71cf00f9ba 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -59,6 +59,7 @@ struct dptx_port { u32 lane_count; u32 link_rate, pending_link_rate; u32 drive_settings[2]; + u32 trained_voltage[4], trained_pre[4]; /* * Per-boot invocation counters for the validate/connect/request/ * release AFK calls, used together with the caller return address diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h new file mode 100644 index 0000000000000..185551e5a589e --- /dev/null +++ b/drivers/phy/apple/atc-j700-dp.h @@ -0,0 +1,333 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* Copyright 2026 Aurora Silicon */ +#ifndef APPLE_ATC_J700_DP_H +#define APPLE_ATC_J700_DP_H + +/* Register facts from the authorized J700 hardware audit. */ +struct j700_dp_step { + u32 offset, mask, value; + u16 delay_ns; +}; + +static void j700_dp_steps(struct apple_atcphy *p, u32 base, + const struct j700_dp_step *steps, size_t count) +{ + for (size_t i = 0; i < count; i++) { + core_mask32(p, base + steps[i].offset, steps[i].mask, steps[i].value); + if (steps[i].delay_ns) + ndelay(steps[i].delay_ns); + } +} + +static int j700_dp_wait(struct apple_atcphy *p, u32 offset, u32 mask, bool set) +{ + u32 value; + + return readl_poll_timeout_atomic(p->regs.core + offset, value, + !!(value & mask) == set, 1, 100); +} + +static void j700_dp_power(struct apple_atcphy *p, unsigned int lane) +{ + unsigned int shift = 4 * lane; + const u32 bits[] = { BIT(2), BIT(1), BIT(0), BIT(3) }; + + for (unsigned int i = 0; i < ARRAY_SIZE(bits); i++) { + u32 mask = bits[i] << shift; + + core_mask32(p, 0x200, mask, i == 2 ? 0 : mask); + udelay(1); + } + for (unsigned int bank = 0; bank < 2; bank++) { + core_clear32(p, 0xc, BIT(6 - 4 * bank + lane)); + udelay(1); + } +} + +static void j700_dp_output_power(struct apple_atcphy *p, unsigned int lane, bool tx) +{ + u32 reg = tx ? 0x10c : 0xc; + + core_set32(p, reg, (BIT(4) | BIT(6)) << lane); + udelay(1); + core_set32(p, reg, (BIT(0) | BIT(2)) << lane); + udelay(1); + if (!tx) { + core_set32(p, 0x8, (BIT(10) | BIT(12)) << lane); + udelay(1); + core_set32(p, 0x8, (BIT(6) | BIT(8)) << lane); + udelay(1); + } + reg = tx ? 0x10c : 0x8; + core_clear32(p, reg, BIT(tx ? 8 : 14) << lane); + core_set32(p, reg, BIT(tx ? 10 : 16) << lane); + udelay(1); +} + +static void j700_dp_equalize(struct apple_atcphy *p, unsigned int lane, + bool tx, u32 preset) +{ + u32 base = (tx ? 0xc000 : 0x9000) + lane * 0x7000; + u32 swing = ((preset & 3) << 14) | ((preset >> 2) & 15); + u32 taps = ((preset >> 6) & 0x3ffff) * 2 + 1; + + if (tx) + core_set32(p, base + 0xd0, BIT(20)); + core_mask32(p, base + (tx ? 0x50 : 0x210), 0xc00f, swing); + core_set32(p, base + (tx ? 0xd4 : 0x248), BIT(28)); + core_mask32(p, base + (tx ? 0xd0 : 0x244), 0x7ffff, taps); +} + +static void j700_dp_transmitter(struct apple_atcphy *p, unsigned int lane, int rate) +{ + static const struct j700_dp_step start[] = { + { 0x44, ~0U, 0x0aafb800, 500 }, + { 0x44, 0x1000000, 0x1000000, 250 }, + { 0x44, 0x4000000, 0x4000000, 250 }, + { 0x44, 0x7c0, 0x300, 250 }, + { 0x44, 0x6c000, 0, 250 }, + }; + static const struct j700_dp_step divide[] = { + { 0x44, 0x100000, 0x100000, 250 }, + { 0x44, 0x400000, 0x400000, 0 }, + }; + static const struct j700_dp_step finish[] = { + { 0x04, 0x33000000, 0x32000000, 0 }, + { 0x10, 0x600, 0x600, 0 }, + { 0x1c, 0xc00000, 0x800000, 0 }, + }; + u32 base = 0xd000 + lane * 0x7000; + + j700_dp_output_power(p, lane, true); + j700_dp_steps(p, base, start, ARRAY_SIZE(start)); + j700_dp_equalize(p, lane, true, 0); + if (rate != ATCPHY_DP_LINK_RATE_HBR3) + j700_dp_steps(p, base, divide, ARRAY_SIZE(divide)); + j700_dp_steps(p, base, finish, ARRAY_SIZE(finish)); + core_mask32(p, base + 0xc, 0x7000000, + rate == ATCPHY_DP_LINK_RATE_RBR ? 0x5000000 : 0x4000000); +} + +static void j700_dp_receiver(struct apple_atcphy *p, unsigned int lane, int rate) +{ + static const struct j700_dp_step setup[] = { + { 0xa4, 3, 2, 0 }, { 0xa8, 0x18000, 0x10000, 0 }, + { 0x6c, 0x18000, 0x10000, 0 }, + { 0xd4, 0x1f1f80, 0x121180, 0 }, { 0x64, 0xfc00, 0xb800, 0 }, + { 0xd8, 7, 4, 0 }, { 0xdc, 0x18, 0, 0 }, + { 0xd4, 0xe000, 0xc000, 100 }, + }; + static const struct j700_dp_step settle[] = { + { 0x6c, 0x600, 0x400, 100 }, { 0xd4, 0xd0000, 0x50000, 0 }, + { 0xd8, 3, 1, 500 }, { 0xd4, 0xc0000, 0, 200 }, + { 0x68, 0x180, 0x180, 200 }, { 0x68, 0x60, 0x60, 200 }, + { 0xd8, 3, 3, 1000 }, { 0xd8, 3, 2, 1000 }, + { 0xd8, 3, 0, 750 }, { 0xa8, 0x60000, 0x60000, 1000 }, + { 0x6c, 0x6000, 0x4000, 0 }, + }; + u32 top = 0x9000 + lane * 0x7000; + u32 base = top + 0x2000; + + core_set32(p, top + 0x160, 1); + core_set32(p, top + 0x1dc, 0x18000); + j700_dp_steps(p, base, setup, ARRAY_SIZE(setup)); + j700_dp_output_power(p, lane, false); + core_set32(p, base + 0x6c, BIT(11)); + core_mask32(p, top + 0x1d4, 3, 1); + ndelay(10); + core_set32(p, top + 0x1d4, 2); + j700_dp_steps(p, base, settle, ARRAY_SIZE(settle)); + j700_dp_equalize(p, lane, false, 0x3f000); + core_mask32(p, base + 0xa8, 0x380000, + rate == ATCPHY_DP_LINK_RATE_RBR ? 0x280000 : 0x200000); + core_mask32(p, base + 0xa8, 0xc00000, 0x800000); + ndelay(50); + core_set32(p, base + 0x30, 0xc00); + ndelay(100); +} + +static void j700_dp_outputs_off(struct apple_atcphy *p, unsigned int lane) +{ + static const struct j700_dp_step tx[] = { + { 0x1c, 0xc00000, 0xc00000, 0 }, { 0x10, 0x600, 0x400, 0 }, + { 0x04, 0x30000000, 0x20000000, 250 }, + { 0x44, 0x100000, 0, 250 }, { 0x44, 0x400000, 0, 250 }, + { 0x44, 0x4000000, 0, 250 }, { 0x44, 0x1000000, 0, 250 }, + { 0x44, 0x1000, 0, 0 }, + }; + static const struct j700_dp_step rx[] = { + { 0x30, 0xc00, 0x800, 50 }, { 0xa8, 0xc00000, 0xc00000, 500 }, + { 0xa8, 0x60000, 0x40000, 0 }, { 0x68, 0x60, 0x40, 200 }, + { 0x68, 0x180, 0x100, 200 }, { 0xd4, 0x10000, 0, 100 }, + { 0x6c, 0x600, 0x600, 0 }, + }; + const u32 tx_bits[] = { BIT(8), BIT(0), BIT(4) }; + const u32 rx_bits[] = { BIT(14), BIT(6), BIT(10) }; + + j700_dp_steps(p, 0xd000 + lane * 0x7000, tx, ARRAY_SIZE(tx)); + for (unsigned int i = 0; i < ARRAY_SIZE(tx_bits); i++) { + core_mask32(p, 0x10c, tx_bits[i] << lane, i ? 0 : tx_bits[i] << lane); + udelay(1); + } + j700_dp_steps(p, 0xb000 + lane * 0x7000, rx, ARRAY_SIZE(rx)); + core_mask32(p, 0x91d4 + lane * 0x7000, 3, 1); + core_clear32(p, 0xb06c + lane * 0x7000, BIT(11)); + for (unsigned int i = 0; i < ARRAY_SIZE(rx_bits); i++) { + core_mask32(p, 0x8, rx_bits[i] << lane, i ? 0 : rx_bits[i] << lane); + udelay(1); + } + for (unsigned int i = 0; i < 2; i++) { + core_clear32(p, 0xc, BIT(4 * i + lane)); + udelay(1); + } +} + +static int j700_dp_stop(struct apple_atcphy *p) +{ + int ret; + + if (!p->dp_owned_pmas) + return 0; + core_set32(p, 0x7000, BIT(0)); + for (unsigned int i = 0; i < 2; i++) + if (p->dp_owned_pmas & BIT(i)) + j700_dp_outputs_off(p, i); + core_mask32(p, 0x7000, 0xc, 8); + core_mask32(p, 0x2000, 0x1ffffff9, 0x10000001); + ret = j700_dp_wait(p, 0x2000, BIT(1), true); + if (ret) + return ret; + core_mask32(p, 0x2000, 0x1ffffff9, 0x10000018); + ret = j700_dp_wait(p, 0x2000, BIT(1), false); + if (!ret) + ret = j700_dp_wait(p, 0x7034, BIT(3), false); + if (ret) + return ret; + core_clear32(p, 0x2200, BIT(3)); + for (unsigned int i = 0; i < 2; i++) { + if (!(p->dp_owned_pmas & BIT(i))) + continue; + ret = j700_pma_idle(p, i); + if (ret) + return ret; + } + p->dp_owned_pmas = 0; + p->dp_link_rate = -1; + return 0; +} + +static int j700_dp_start(struct apple_atcphy *p, unsigned int pmas, int rate, bool ssc) +{ + static const u32 parameters[4][4] = { + { 0x1e0e021c, 0, 0x156600, 0x15671b }, + { 0x1e0e01c2, 0x07fffffe, 0x155200, 0x155316 }, + { 0x1e0e01c2, 0x07fffffe, 0x554800, 0x554916 }, + { 0x1e0e02a3, 0x0bff7ffc, 0x554810, 0x564922 }, + }; + int ret; + + if (!pmas || (pmas & ~3U) || rate < 0 || (unsigned int)rate >= ARRAY_SIZE(parameters)) + return -EINVAL; + /* Claim before the first write: failed setup must still be cleaned up. */ + p->dp_owned_pmas = pmas; + p->dp_link_rate = -1; + for (unsigned int i = 0; i < 2; i++) + if (pmas & BIT(i)) + j700_dp_power(p, i); + core_clear32(p, 0x2234, 3); + core_mask32(p, 0x7000, 0x1ff0, 0x490); + for (unsigned int i = 0; i < 3; i++) + writel(parameters[rate][i == 2 ? 2 + ssc : i], p->regs.core + 0x2080 + 4 * i); + core_mask32(p, 0x2208, 0x1f0000, 0x70000); + core_set32(p, 0x2218, 1); + core_set32(p, 0x2200, BIT(3)); + core_mask32(p, 0x2000, 0x1ffffff9, 0x10000001); + ret = j700_dp_wait(p, 0x2000, BIT(1), true); + if (!ret) + ret = j700_dp_wait(p, 0x7034, BIT(3), true); + if (ret) + return ret; + core_mask32(p, 0x2000, 0x1ffffff9, 0x10014000); + ret = j700_dp_wait(p, 0x2000, BIT(1), false); + if (ret) + return ret; + for (unsigned int i = 0; i < 2; i++) + if (pmas & BIT(i)) + core_mask32(p, 0xb064 + i * 0x7000, 0xc, 8); + core_set32(p, 0x7000, 0xc); + udelay(1); + for (unsigned int i = 0; i < 2; i++) { + if (!(pmas & BIT(i))) + continue; + j700_dp_transmitter(p, i, rate); + j700_dp_receiver(p, i, rate); + } + core_clear32(p, 0x7000, 1); + p->dp_link_rate = rate; + p->dp_spread_spectrum = ssc; + return 0; +} + +static int j700_dp_configure(struct apple_atcphy *p, const struct phy_configure_opts_dp *o) +{ + const struct atcphy_mode_configuration *mode; + unsigned int pmas, capacity, first, table; + int rate = -1, ret; + + guard(mutex)(&p->lock); + if ((o->set_rate && !o->link_rate) || (o->set_lanes && !o->lanes)) + return o->set_voltages ? -EINVAL : j700_dp_stop(p); + if (p->mode != APPLE_ATCPHY_MODE_DP && p->mode != APPLE_ATCPHY_MODE_USB3_DP) + return -EINVAL; + mode = atcphy_get_mode_config(p, p->mode); + pmas = !!mode->dp_lane[0] | (!!mode->dp_lane[1] << 1); + capacity = 2 * hweight32(pmas); + if (o->lanes > capacity || (o->set_lanes && o->lanes != 1 && + o->lanes != 2 && o->lanes != 4)) + return -EINVAL; + if (!p->dp_presets_valid[0]) + return -ENOENT; + if (o->set_voltages) { + if (!o->lanes) + return -EINVAL; + for (unsigned int i = 0; i < o->lanes; i++) + if (o->voltage[i] > 3 || o->pre[i] > 3) + return -EINVAL; + } + if (o->set_rate) { + const unsigned int rates[] = { 1620, 2700, 5400, 8100 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(rates); i++) + if (o->link_rate == rates[i]) + rate = i; + if (rate < 0) + return -EINVAL; + if (p->dp_link_rate != rate || p->dp_owned_pmas != pmas || + p->dp_spread_spectrum != o->ssc) { + ret = j700_dp_stop(p); + if (ret) + return ret; + ret = j700_dp_start(p, pmas, rate, o->ssc); + if (ret) { + int cleanup = j700_dp_stop(p); + + dev_err(p->dev, "DP setup failed: %d, cleanup: %d\n", ret, cleanup); + return ret; + } + } + } + if (!o->set_voltages) + return 0; + if (!p->dp_owned_pmas || p->dp_link_rate < 0) + return -EINVAL; + table = p->dp_link_rate + 1; + if (!p->dp_presets_valid[table]) + table = 0; + first = pmas == 3 ? !p->swap_lanes : (pmas == 2); + for (unsigned int i = 0; i < o->lanes; i++) + j700_dp_equalize(p, first ^ (i / 2), i % 2, + p->dp_presets[table][4 * o->voltage[i] + o->pre[i]]); + return 0; +} + +#endif diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 600a3f09dcecf..4d5bba1db5c3d 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -654,6 +654,10 @@ struct atcphy_hw { * @ss_tunables: The complete SuperSpeed tunable set was supplied * @dp_tunables: The complete T8140 DisplayPort bundle was supplied * @aux_power_requested: AUX power-up began and needs matching shutdown + * @dp_owned_pmas: J700 PMAs requiring main-link cleanup + * @dp_spread_spectrum: J700 main-link spread-spectrum state + * @dp_presets: Firmware-derived per-rate drive presets + * @dp_presets_valid: Firmware supplied each drive-preset table * @fixed_usb2: The USB2 pairs go to a fixed hub on the USB controller, which * stays in host mode; the PHY provides USB2 from probe on * @typec_mode: Mode the Type-C mux last asked for; a fixed-hub port returns to @@ -713,6 +717,10 @@ struct apple_atcphy { bool ss_tunables; bool dp_tunables; bool aux_power_requested; + u8 dp_owned_pmas; + bool dp_spread_spectrum; + u32 dp_presets[5][16]; + bool dp_presets_valid[5]; bool fixed_usb2; enum atcphy_mode typec_mode; bool host_active; @@ -1483,6 +1491,8 @@ static int j700_pma_idle(struct apple_atcphy *atcphy, unsigned int lane) return 0; } +#include "atc-j700-dp.h" + static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode mode) { const struct atcphy_mode_configuration *mode_cfg = atcphy_get_mode_config(atcphy, mode); @@ -1673,8 +1683,11 @@ static int atcphy_enable_dp_aux(struct apple_atcphy *atcphy) static int atcphy_disable_dp_aux(struct apple_atcphy *atcphy) { - if (atcphy->hw->has_t8140_dp) - return j700_aux_disable(atcphy); + if (atcphy->hw->has_t8140_dp) { + int ret = j700_dp_stop(atcphy); + + return ret ? ret : j700_aux_disable(atcphy); + } /* FIXME */ if (atcphy->hw->gen != ATCPHY_GENERATION_T8122) { @@ -2664,6 +2677,11 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) u32 reg; lockdep_assert_held(&atcphy->lock); + if (atcphy->hw->has_t8140_dp && atcphy->dp_owned_pmas && mode != atcphy->mode) { + ret = j700_dp_stop(atcphy); + if (ret) + return ret; + } if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && !atcphy->dp_tunables) return -EOPNOTSUPP; @@ -2909,6 +2927,9 @@ static int atcphy_dpphy_configure(struct phy *phy, union phy_configure_opts *opt struct apple_atcphy *atcphy = phy_get_drvdata(phy); enum atcphy_dp_link_rate link_rate; + if (atcphy->hw->has_t8140_dp) + return j700_dp_configure(atcphy, opts); + if (opts->set_voltages) return -EINVAL; if (opts->set_lanes) @@ -3340,6 +3361,26 @@ static int atcphy_load_t8140_dp(struct apple_atcphy *atcphy) return PTR_ERR(*lists[i].out); } + for (unsigned int i = 0; i < ARRAY_SIZE(atcphy->dp_presets); i++) { + static const char * const properties[] = { + "apple,dp-training-table", "apple,dp-training-table-rbr", + "apple,dp-training-table-hbr", "apple,dp-training-table-hbr2", + "apple,dp-training-table-hbr3", + }; + int count = of_property_count_u32_elems(atcphy->np, properties[i]); + int ret; + + if (i && !of_find_property(atcphy->np, properties[i], NULL)) + continue; + if (count != ARRAY_SIZE(atcphy->dp_presets[i])) + return -EINVAL; + ret = of_property_read_u32_array(atcphy->np, properties[i], + atcphy->dp_presets[i], count); + if (ret) + return ret; + atcphy->dp_presets_valid[i] = true; + } + atcphy->dp_tunables = true; return 0; } diff --git a/tools/diagnostics/apple-dptx-training-test.c b/tools/diagnostics/apple-dptx-training-test.c new file mode 100644 index 0000000000000..53ae4419f293d --- /dev/null +++ b/tools/diagnostics/apple-dptx-training-test.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2026 Aurora Silicon */ +#include +#include +#include +#include +#include +#include + +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint16_t u16; +struct phy { int unused; }; +struct settings { + unsigned int lanes, voltage[4], pre[4]; + bool set_voltages; +}; + +union phy_configure_opts { struct settings dp; }; +struct dptx_port { + struct phy *atcphy; + u32 trained_voltage[4], trained_pre[4]; +}; + +struct apple_epic_service { struct dptx_port *cookie; }; +static unsigned int calls; +static int failure; +#define PHY_MODE_DP 1 + +static u32 get_unaligned_le32(const void *p) +{ + u32 v; + + memcpy(&v, p, sizeof(v)); + return v; +} + +static void put_unaligned_le32(u32 v, void *p) +{ + memcpy(p, &v, sizeof(v)); +} + +static int phy_validate(struct phy *p, int mode, int sub, union phy_configure_opts *o) +{ + assert(p && mode == PHY_MODE_DP && !sub); + o->dp.lanes = 2; + return 0; +} + +static int phy_configure(struct phy *p, union phy_configure_opts *o) +{ + assert(p && o->dp.set_voltages && o->dp.lanes <= 2); + calls++; + return failure; +} + +#include "drivers/gpu/drm/apple/dptx-j700-training.h" + +#ifdef DPTX_GROUP_TEST +#define DPTX_APCALL_SET_DRIVE_SETTINGS 3 +#define DPTX_APCALL_GET_DRIVE_SETTINGS 4 +static int dptxport_call(struct apple_epic_service *service, u32 command, + const void *data, size_t size, void *reply, size_t room) +{ + (void)service; + (void)command; + (void)data; + (void)size; + (void)reply; + (void)room; + return 7; +} + +#include "group-call.h" +#endif + +int main(void) +{ + struct phy phy = {}; + struct dptx_port port = { .atcphy = &phy }; + struct apple_epic_service service = { .cookie = &port }; + u8 input[81] = {}, output[81]; + u8 *request = input + 1, *reply = output + 1; + unsigned int cases = 0; + + put_unaligned_le32(2, request + 16); + put_unaligned_le32(1, request + 36); + put_unaligned_le32(2, request + 52); + for (unsigned int size = 0; size < 56; size++) { + assert(dptx_j700_training(&service, true, request, size, reply, 80) == -EINVAL); + assert(dptx_j700_training(&service, true, request, 80, reply, size) == -EINVAL); + assert(!calls); + cases += 2; + } + put_unaligned_le32(9, request + 52); + assert(dptx_j700_training(&service, true, request, 80, reply, 80) == -EINVAL); + assert(!calls && !port.trained_voltage[0]); + put_unaligned_le32(2, request + 52); + failure = -EIO; + assert(dptx_j700_training(&service, true, request, 80, reply, 80) == -EIO); + assert(!port.trained_voltage[0]); + failure = 0; + assert(!dptx_j700_training(&service, true, request, 80, reply, 80)); + assert(port.trained_voltage[0] == 1 && port.trained_pre[1] == 2); + assert(get_unaligned_le32(reply + 16) == 2 && !get_unaligned_le32(reply)); + put_unaligned_le32(4, request); + assert(!dptx_j700_training(&service, false, request, 32, reply, 80)); + assert(get_unaligned_le32(reply) == 4 && !get_unaligned_le32(reply + 16)); + assert(get_unaligned_le32(reply + 36) == 1 && get_unaligned_le32(reply + 52) == 2); + assert(!get_unaligned_le32(reply + 76)); + +#ifdef DPTX_GROUP_TEST + for (unsigned int group = 0; group < 4; group++) { + for (unsigned int command = 0; command < 25; command++) { + int expected = group ? -EOPNOTSUPP : 7; + + if (group == 0 && command == 3) + expected = -EOPNOTSUPP; + if ((group == 1 && command == 3) || (group == 0 && command == 4)) + expected = 0; + assert(dptxport_group_call(&service, group, command, + request, 80, reply, 80) == expected); + } + } +#endif + port.atcphy = NULL; + assert(dptx_j700_training(&service, true, request, 80, reply, 80) == -ENODEV); + printf("DPTX training: %u boundary cases plus failure/cache/readback checks passed\n", + cases); + return 0; +} diff --git a/tools/diagnostics/apple-j700-dp-test.c b/tools/diagnostics/apple-j700-dp-test.c new file mode 100644 index 0000000000000..d6a6219fe9ad1 --- /dev/null +++ b/tools/diagnostics/apple-j700-dp-test.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2026 Aurora Silicon */ +#include +#include +#include +#include +#include +#include + +typedef uint32_t u32; +typedef uint16_t u16; +#define BIT(n) (1U << (n)) +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define hweight32(v) __builtin_popcount(v) +static void test_guard_mutex(void *p) { (void)p; } +#define guard(kind) test_guard_##kind +static void report(void *p, const char *s, ...) { (void)p; (void)s; } +#define dev_err report +#define ndelay(n) ((void)(n)) +#define udelay(n) ((void)(n)) + +enum { + ATCPHY_DP_LINK_RATE_RBR, ATCPHY_DP_LINK_RATE_HBR, + ATCPHY_DP_LINK_RATE_HBR2, ATCPHY_DP_LINK_RATE_HBR3, +}; + +enum { APPLE_ATCPHY_MODE_OFF, APPLE_ATCPHY_MODE_DP, APPLE_ATCPHY_MODE_USB3_DP }; +struct phy_configure_opts_dp { + unsigned int link_rate, lanes, voltage[4], pre[4]; + bool set_rate, set_lanes, set_voltages, ssc; +}; + +struct atcphy_mode_configuration { bool dp_lane[2]; }; +struct apple_atcphy { + struct { void *core; } regs; + void *dev; + int lock, mode, dp_link_rate; + unsigned int dp_owned_pmas; + bool dp_spread_spectrum, swap_lanes, dp_presets_valid[5]; + u32 dp_presets[5][16]; +}; + +static u32 registers[0x20000 / 4]; +static unsigned int writes, polls, fail_poll, pin_mask = 2; +static bool fail_all; + +static void core_mask32(struct apple_atcphy *p, u32 off, u32 mask, u32 value) +{ + (void)p; + assert(!(off % 4) && off / 4 < ARRAY_SIZE(registers)); + assert(!(value & ~mask)); + registers[off / 4] = (registers[off / 4] & ~mask) | value; + writes++; + if (off == 0x2000) { + if (value == 0x10000001) { + registers[off / 4] |= BIT(1); + registers[0x7034 / 4] |= BIT(3); + } else { + registers[off / 4] &= ~BIT(1); + if (value == 0x10000018) + registers[0x7034 / 4] &= ~BIT(3); + } + } +} + +static void core_set32(struct apple_atcphy *p, u32 o, u32 v) +{ + core_mask32(p, o, v, v); +} + +static void core_clear32(struct apple_atcphy *p, u32 o, u32 v) +{ + core_mask32(p, o, v, 0); +} + +static void writel(u32 value, void *ptr) +{ + u32 off = (char *)ptr - (char *)registers; + + core_mask32(NULL, off, ~0U, value); +} + +static u32 readl(void *ptr) { return *(u32 *)ptr; } +#define readl_poll_timeout_atomic(addr, val, condition, delay, timeout) \ + ({ \ + int rc; \ + (void)(delay); \ + (void)(timeout); \ + polls++; \ + val = readl(addr); \ + rc = fail_all || polls == fail_poll || !(condition) ? -ETIMEDOUT : 0; \ + rc; \ + }) + +static int j700_pma_idle(struct apple_atcphy *p, unsigned int lane) +{ + core_set32(p, 0xc, BIT(2 + lane) | BIT(6 + lane)); + core_mask32(p, 0x200, 15U << (4 * lane), 1U << (4 * lane)); + return 0; +} + +static const struct atcphy_mode_configuration * +atcphy_get_mode_config(struct apple_atcphy *p, int mode) +{ + static struct atcphy_mode_configuration config; + + (void)p; + (void)mode; + config.dp_lane[0] = pin_mask & 1; + config.dp_lane[1] = pin_mask & 2; + return &config; +} + +#include "drivers/phy/apple/atc-j700-dp.h" + +static struct apple_atcphy device(void) +{ + struct apple_atcphy p = { + .regs.core = registers, .mode = APPLE_ATCPHY_MODE_USB3_DP, + .dp_link_rate = -1, + }; + + memset(registers, 0xa5, sizeof(registers)); + for (unsigned int i = 0; i < ARRAY_SIZE(p.dp_presets_valid); i++) + p.dp_presets_valid[i] = true; + writes = 0; + polls = 0; + fail_poll = 0; + fail_all = false; + return p; +} + +int main(void) +{ + const unsigned int rates[] = { 1620, 2700, 5400, 8100 }; + unsigned int cases = 0; + + for (pin_mask = 1; pin_mask <= 3; pin_mask++) { + for (unsigned int rate = 0; rate < ARRAY_SIZE(rates); rate++) { + for (unsigned int ssc = 0; ssc < 2; ssc++) { + struct apple_atcphy p = device(); + struct phy_configure_opts_dp o = { + .set_rate = true, .link_rate = rates[rate], .ssc = ssc, + }; + u32 before = registers[0x200 / 4]; + + assert(!j700_dp_configure(&p, &o)); + assert(p.dp_owned_pmas == pin_mask && p.dp_link_rate == (int)rate); + if (pin_mask != 3) { + u32 other = pin_mask == 1 ? 0xf0 : 0xf; + + assert(!((registers[0x200 / 4] ^ before) & other)); + } + unsigned int first_writes = writes; + + assert(!j700_dp_configure(&p, &o) && writes == first_writes); + struct phy_configure_opts_dp voltage = { + .set_voltages = true, .lanes = 2 * hweight32(pin_mask), + }; + + for (unsigned int lane = 0; lane < voltage.lanes; lane++) { + unsigned int index = 4 * lane + 1; + + voltage.voltage[lane] = lane; + voltage.pre[lane] = 1; + p.dp_presets[rate + 1][index] = 0x12340 + lane; + } + assert(!j700_dp_configure(&p, &voltage)); + for (unsigned int lane = 0; lane < voltage.lanes; lane++) { + unsigned int first = pin_mask == 1 ? 0 : 1; + u32 base = (lane % 2 ? 0xc000 : 0x9000) + + (first ^ (lane / 2)) * 0x7000; + u32 offset = lane % 2 ? 0x50 : 0x210; + u32 preset = 0x12340 + lane; + + assert((registers[(base + offset) / 4] & 0xc00f) == + (((preset & 3) << 14) | ((preset >> 2) & 15))); + } + + assert(!j700_dp_stop(&p)); + assert(!p.dp_owned_pmas && p.dp_link_rate == -1); + cases++; + } + } + } + pin_mask = 2; + for (unsigned int fault = 1; fault <= 3; fault++) { + struct apple_atcphy p = device(); + struct phy_configure_opts_dp o = { .set_rate = true, .link_rate = 8100 }; + + fail_poll = fault; + assert(j700_dp_configure(&p, &o) == -ETIMEDOUT); + assert(!p.dp_owned_pmas && p.dp_link_rate == -1); + cases++; + } + { + struct apple_atcphy p = device(); + struct phy_configure_opts_dp o = { .set_rate = true, .link_rate = 8100 }; + + fail_all = true; + assert(j700_dp_configure(&p, &o) == -ETIMEDOUT && p.dp_owned_pmas == 2); + fail_all = false; + assert(!j700_dp_stop(&p) && !p.dp_owned_pmas); + cases++; + } + { + struct apple_atcphy p = device(); + struct phy_configure_opts_dp o = { .set_rate = true, .link_rate = 1234 }; + + assert(j700_dp_configure(&p, &o) == -EINVAL && !writes); + o.link_rate = 8100; + o.lanes = 3; + assert(j700_dp_configure(&p, &o) == -EINVAL && !writes); + o.lanes = 0; + p.dp_presets_valid[0] = false; + assert(j700_dp_configure(&p, &o) == -ENOENT && !writes); + p.dp_presets_valid[0] = true; + o.set_voltages = true; + o.lanes = 2; + o.voltage[1] = 4; + assert(j700_dp_configure(&p, &o) == -EINVAL && !writes); + cases++; + } + printf("J700 DP: %u register/state cases passed\n", cases); + return 0; +} diff --git a/tools/diagnostics/test-apple-j700-dp.py b/tools/diagnostics/test-apple-j700-dp.py new file mode 100644 index 0000000000000..41836939f6eae --- /dev/null +++ b/tools/diagnostics/test-apple-j700-dp.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Compile the production J700 helpers against bounded host register models.""" +import os +from pathlib import Path +import shlex +import subprocess +import tempfile + + +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/gpu/drm/apple/dptxep.c").read_text() +start = source.index("static int dptxport_group_call(") +end = source.index("\n}\n", start) + 3 +with tempfile.TemporaryDirectory(prefix="j700-dp-") as work: + work = Path(work) + (work / "group-call.h").write_text(source[start:end]) + for name in ("apple-j700-dp-test", "apple-dptx-training-test"): + command = shlex.split(os.environ.get("CC", "clang")) + command += ["-std=gnu11", "-O1", "-g", "-Wall", "-Wextra", "-Werror"] + command += ["-fsanitize=address,undefined", "-fno-omit-frame-pointer"] + command += ["-I" + str(root), "-I" + str(work), "-DDPTX_GROUP_TEST"] + command += [str(root / "tools/diagnostics" / (name + ".c"))] + command += ["-o", str(work / name)] + subprocess.run(command, check=True) + subprocess.run([str(work / name)], check=True) From a004aa36383b8b39ad522fab3ed22d31f769fdab Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:36:12 +1100 Subject: [PATCH 0914/1579] drm: apple: honor separate AFK V2 callback reply capacity A drive-settings read has a short request and a larger response. Validate both owned-buffer lengths before dispatch, allocate the response capacity, and report its body length without extending the input read. Keep the full response range ownership check before any callback side effect. The 917 codec assertions now include unequal request/response lengths, undersized buffers, limits, and invalid output pointers. Production owned copy tests and all J700 PHY/training/group tests pass under sanitizers. Strict checkpatch and the opaque copy audit are clean. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 27 ++++++++++--------- drivers/gpu/drm/apple/afkv2-codec.h | 19 +++++++++++++ .../selftests/drivers/gpu/apple-afkv2/codec.c | 24 +++++++++++++++++ 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index b7c3905303eab..8bcdb8ec24033 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -696,16 +696,15 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, const struct epic_service_call *call; const u8 *request = command->inline_data; u8 *owned_request = NULL, *reply = NULL; - size_t request_size = command->oob ? command->tx_size : command->inline_size; + size_t request_size, response_size, response_body; size_t body_size, reply_size; u32 status = 0xe00002c7; int ret; if (frame->type != EPIC_SUBTYPE_STD_SERVICE || - request_size < sizeof(*call) + sizeof(u32) || - request_size > SZ_1M - AFKV2_COMMAND_SIZE || - request_size > command->value || - (command->oob && request_size > command->rx_size)) + afkv2_callback_lengths(command, sizeof(*call) + sizeof(u32), + SZ_1M - AFKV2_COMMAND_SIZE, + &request_size, &response_size)) goto reject; service = afk_service_get(afk_epic_find_service(ep, frame->interface)); if (!service || READ_ONCE(service->torndown) || @@ -717,7 +716,7 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, if (command->oob) { /* Reject an unowned response destination before any callback side effect. */ ret = afk_v2_access_owned(service, command->rx_address, NULL, - request_size, true, false); + response_size, true, false); if (ret) goto release; owned_request = kmalloc(request_size, GFP_KERNEL); @@ -737,32 +736,36 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || le32_to_cpu(call->data_len) != body_size) goto release; - reply = kzalloc(AFKV2_COMMAND_SIZE + request_size, GFP_KERNEL); + response_body = response_size - sizeof(*call); + reply = kzalloc(AFKV2_COMMAND_SIZE + response_size, GFP_KERNEL); if (!reply) { status = 0xe00002bd; goto release; } reply[1] = command->id; memcpy(reply + AFKV2_COMMAND_SIZE, call, sizeof(*call)); + ((struct epic_service_call *)(reply + AFKV2_COMMAND_SIZE))->data_len = + cpu_to_le32(response_body); if (service->ops->call_v2) ret = service->ops->call_v2(service, le16_to_cpu(call->group), le32_to_cpu(call->command), request + sizeof(*call), body_size, - reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + reply + AFKV2_COMMAND_SIZE + sizeof(*call), + response_body); else ret = service->ops->call(service, le32_to_cpu(call->command), request + sizeof(*call), body_size, - reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + reply + AFKV2_COMMAND_SIZE + sizeof(*call), response_body); if (ret) goto release; - reply_size = AFKV2_COMMAND_SIZE + request_size; + reply_size = AFKV2_COMMAND_SIZE + response_size; if (command->oob) { ret = afk_v2_copy_owned(service, command->rx_address, - reply + AFKV2_COMMAND_SIZE, request_size, true); + reply + AFKV2_COMMAND_SIZE, response_size, true); if (ret) goto release; ret = afkv2_encode_oob_command(reply, reply_size, command->id, - command->rx_address, request_size, + command->rx_address, response_size, command->tx_address, command->tx_size); if (ret) goto release; diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h index cdd5d5ad20b22..8990182275354 100644 --- a/drivers/gpu/drm/apple/afkv2-codec.h +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -212,6 +212,25 @@ static inline int afkv2_decode_command(const void *payload, size_t size, return 0; } +/* Requests and responses may have different owned-buffer capacities. */ +static inline int afkv2_callback_lengths(const struct afkv2_command *command, + size_t minimum, size_t limit, + size_t *request, size_t *response) +{ + size_t in, out; + + if (!command || !request || !response || minimum > limit) + return -EINVAL; + in = command->oob ? command->tx_size : command->inline_size; + out = command->oob ? command->rx_size : command->inline_size; + if (in < minimum || out < minimum || in > limit || out > limit || + in > command->value || out > command->value) + return -EINVAL; + *request = in; + *response = out; + return 0; +} + /* Correlate only; no buffer is read, written or freed by this helper. */ static inline int afkv2_check_response(const struct afkv2_frame *frame, const struct afkv2_command *response, diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c index de49360bdabea..79598033c084a 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c @@ -178,6 +178,30 @@ int main(void) CHECK(!afkv2_owned_span(0x1000, 0, 0x1000, 0)); CHECK(afkv2_owned_span(UINT64_MAX - 15, 16, UINT64_MAX, 1)); CHECK(!afkv2_owned_span(UINT64_MAX - 15, 17, UINT64_MAX, 1)); + struct afkv2_command callback = { .oob = true, .tx_size = 96, + .rx_size = 144, .value = 144 }; + size_t input = 0, output = 0; + + CHECK(!afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + CHECK(input == 96 && output == 144); + callback.value = 143; + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + callback.value = 2048; + callback.rx_size = 2048; + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + callback.rx_size = 67; + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + callback.rx_size = 144; + callback.tx_size = 67; + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + callback.oob = false; + callback.inline_size = 96; + CHECK(!afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + CHECK(input == 96 && output == 96); + CHECK(afkv2_callback_lengths(NULL, 68, 1024, &input, &output)); + CHECK(afkv2_callback_lengths(&callback, 68, 1024, NULL, &output)); + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, NULL)); + CHECK(afkv2_callback_lengths(&callback, 1025, 1024, &input, &output)); printf("PASS %u assertions\n", assertions); return 0; } From d863129e9cf58169827cb04ba21a3411898a134a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:42:34 +1100 Subject: [PATCH 0915/1579] phy: apple: preserve J700 lane-routing state during DP setup Use the audited T8140 routing masks for USB3/DP and four-lane DP. The generic sequence changes the miscellaneous lane-swap control and clears routing state that this generation preserves. Update the low protocol fields in order and idle only the PMAs assigned to DisplayPort. Add opt-in callback and PHY completion diagnostics to distinguish firmware waiting from a blocked PHY operation. No calibration values are logged. The routing values are hardware facts from the user-authorized audit; this implementation is independently written. Copy audit clean. Host checks pass: 33 PHY routing/state cases, 112 training bounds, 100 group routing cases, 917 codec assertions, owned-copy and AUX transition tests. Strict checkpatch passes. External-display qualification remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 6 ++++++ drivers/phy/apple/atc-j700-dp.h | 27 ++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 17 +++++----------- tools/diagnostics/apple-j700-dp-test.c | 19 ++++++++++++++++++ 4 files changed, 57 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 8bcdb8ec24033..38b44c2d6d785 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -701,6 +701,8 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, u32 status = 0xe00002c7; int ret; + dev_dbg(ep->dcp->dev, "AFKV2 callback sizes: tx=%u rx=%u value=%u inline=%zu\n", + command->tx_size, command->rx_size, command->value, command->inline_size); if (frame->type != EPIC_SUBTYPE_STD_SERVICE || afkv2_callback_lengths(command, sizeof(*call) + sizeof(u32), SZ_1M - AFKV2_COMMAND_SIZE, @@ -746,6 +748,8 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, memcpy(reply + AFKV2_COMMAND_SIZE, call, sizeof(*call)); ((struct epic_service_call *)(reply + AFKV2_COMMAND_SIZE))->data_len = cpu_to_le32(response_body); + dev_dbg(ep->dcp->dev, "AFKV2 callback begin: group=%u command=%u in=%zu out=%zu\n", + le16_to_cpu(call->group), le32_to_cpu(call->command), body_size, response_body); if (service->ops->call_v2) ret = service->ops->call_v2(service, le16_to_cpu(call->group), le32_to_cpu(call->command), @@ -756,6 +760,8 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, ret = service->ops->call(service, le32_to_cpu(call->command), request + sizeof(*call), body_size, reply + AFKV2_COMMAND_SIZE + sizeof(*call), response_body); + dev_dbg(ep->dcp->dev, "AFKV2 callback end: group=%u command=%u result=%d\n", + le16_to_cpu(call->group), le32_to_cpu(call->command), ret); if (ret) goto release; reply_size = AFKV2_COMMAND_SIZE + response_size; diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h index 185551e5a589e..5ce78d085159d 100644 --- a/drivers/phy/apple/atc-j700-dp.h +++ b/drivers/phy/apple/atc-j700-dp.h @@ -3,6 +3,29 @@ #ifndef APPLE_ATC_J700_DP_H #define APPLE_ATC_J700_DP_H +/* Preserve the upper routing state established by the platform firmware. */ +static int j700_dp_route(struct apple_atcphy *p, bool four_lanes) +{ + u32 protocols = four_lanes ? 0x492 : (p->swap_lanes ? 0x252 : 0x489); + int ret; + + core_mask32(p, 0x60, 0xfff, protocols); + if (four_lanes) { + core_mask32(p, 0x64, 0x1f, 0x14); + core_mask32(p, 0x64, 0x3ffe0, p->swap_lanes ? 0x100 : 0x22000); + } else { + core_mask32(p, 0x64, 0x1ffff, 0x110 | p->swap_lanes); + } + for (unsigned int lane = 0; lane < 2; lane++) { + if (!four_lanes && lane != !p->swap_lanes) + continue; + ret = j700_pma_idle(p, lane); + if (ret) + return ret; + } + return 0; +} + /* Register facts from the authorized J700 hardware audit. */ struct j700_dp_step { u32 offset, mask, value; @@ -213,6 +236,7 @@ static int j700_dp_stop(struct apple_atcphy *p) } p->dp_owned_pmas = 0; p->dp_link_rate = -1; + dev_dbg(p->dev, "J700 DP main stopped\n"); return 0; } @@ -265,6 +289,7 @@ static int j700_dp_start(struct apple_atcphy *p, unsigned int pmas, int rate, bo core_clear32(p, 0x7000, 1); p->dp_link_rate = rate; p->dp_spread_spectrum = ssc; + dev_dbg(p->dev, "J700 DP main ready: PMAs=%u rate=%d ssc=%u\n", pmas, rate, ssc); return 0; } @@ -274,7 +299,9 @@ static int j700_dp_configure(struct apple_atcphy *p, const struct phy_configure_ unsigned int pmas, capacity, first, table; int rate = -1, ret; + dev_dbg(p->dev, "J700 DP configure waiting for PHY lock\n"); guard(mutex)(&p->lock); + dev_dbg(p->dev, "J700 DP configure acquired PHY lock\n"); if ((o->set_rate && !o->link_rate) || (o->set_lanes && !o->lanes)) return o->set_voltages ? -EINVAL : j700_dp_stop(p); if (p->mode != APPLE_ATCPHY_MODE_DP && p->mode != APPLE_ATCPHY_MODE_USB3_DP) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 4d5bba1db5c3d..0e2d74959b41c 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1497,6 +1497,11 @@ static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode { const struct atcphy_mode_configuration *mode_cfg = atcphy_get_mode_config(atcphy, mode); + if (atcphy->hw->has_t8140_dp && + (mode == APPLE_ATCPHY_MODE_USB3 || mode == APPLE_ATCPHY_MODE_USB3_DP || + mode == APPLE_ATCPHY_MODE_DP)) + return j700_dp_route(atcphy, mode == APPLE_ATCPHY_MODE_DP); + core_mask32(atcphy, atcphy->hw->aciophy_lane_mode, ACIOPHY_LANE_MODE_RX0, FIELD_PREP(ACIOPHY_LANE_MODE_RX0, mode_cfg->lane_mode[0])); core_mask32(atcphy, atcphy->hw->aciophy_lane_mode, ACIOPHY_LANE_MODE_TX0, @@ -1520,18 +1525,6 @@ static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode else core_clear32(atcphy, atcphy->hw->aciophy_crossbar, ACIOPHY_CROSSBAR_DP_BOTH_PMA); - if (atcphy->hw->has_t8140_dp) { - for (unsigned int lane = 0; lane < 2; lane++) { - int ret; - - if (!mode_cfg->dp_lane[lane]) - continue; - ret = j700_pma_idle(atcphy, lane); - if (ret) - return ret; - } - return 0; - } /* unclear if we need the remainder for t8122 */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) diff --git a/tools/diagnostics/apple-j700-dp-test.c b/tools/diagnostics/apple-j700-dp-test.c index d6a6219fe9ad1..0b3e24067ac05 100644 --- a/tools/diagnostics/apple-j700-dp-test.c +++ b/tools/diagnostics/apple-j700-dp-test.c @@ -16,6 +16,7 @@ static void test_guard_mutex(void *p) { (void)p; } #define guard(kind) test_guard_##kind static void report(void *p, const char *s, ...) { (void)p; (void)s; } #define dev_err report +#define dev_dbg report #define ndelay(n) ((void)(n)) #define udelay(n) ((void)(n)) @@ -183,6 +184,24 @@ int main(void) } } } + for (unsigned int full = 0; full < 2; full++) { + for (unsigned int swap = 0; swap < 2; swap++) { + struct apple_atcphy p = device(); + u32 old_selector = registers[0x64 / 4]; + u32 old_misc = registers[4 / 4]; + u32 mask = full ? 0x3ffff : 0x1ffff; + + p.swap_lanes = swap; + assert(!j700_dp_route(&p, full)); + assert(!((registers[0x64 / 4] ^ old_selector) & ~mask)); + assert(registers[4 / 4] == old_misc); + assert((registers[0x60 / 4] & 0xfff) == + (full ? 0x492 : (swap ? 0x252 : 0x489))); + assert((registers[0x64 / 4] & mask) == + (full ? (swap ? 0x114 : 0x22014) : (0x110 | swap))); + cases++; + } + } pin_mask = 2; for (unsigned int fault = 1; fault <= 3; fault++) { struct apple_atcphy p = device(); From 7a294dd592be90d4b6322ead8a6a5a241a095a63 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:45:36 +1100 Subject: [PATCH 0916/1579] mux: apple: retain the physical J700 crossbar gate sequence Keep the newer T602x DP IN clock controls for existing users, but exclude them from the J700 physical DP output. That output uses the original T602x gate sequence and read-clock reset location. The previous generic path wrote the additional source/output gate registers on J700 too. This small condition is derived from audited hardware facts; no private implementation is imported. The production change has no copy-audit matches. Shared generic test-extractor/header boilerplate was reviewed. Validate the actual production mux function for every source on all three outputs, both with and without the J700 condition: 54 select/disconnect scenarios pass under ASan/UBSan. Existing PHY, training and AFK checks pass. Strict checkpatch passes; external hardware qualification remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/mux/apple-display-crossbar.c | 17 ++- tools/diagnostics/test-apple-j700-crossbar.py | 119 ++++++++++++++++++ 3 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 tools/diagnostics/test-apple-j700-crossbar.py diff --git a/MAINTAINERS b/MAINTAINERS index 4db1e45c648ce..c157347692c6f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2018,6 +2018,7 @@ F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs F: tools/diagnostics/apple-dptx-training-test.c F: tools/diagnostics/apple-j700-dp-test.c +F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-dp.py F: tools/testing/apple/ diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index f49977b846f04..a0a7ac57ccf00 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -210,6 +210,8 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) unsigned int dispext_bit_en; u32 atc_bit, mux_mask, mux_val; bool enable; + /* J700 physical DP retains the original T602x clock-gate sequence. */ + bool j700_physical = index == MUX_DPPHY && of_machine_is_compatible("apple,j700"); int ret = 0; if (index >= MUX_MAX) @@ -271,11 +273,14 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) dpxbar_set32(dpxbar, T602X_REG_014, 0x4); /* Native T602x DPIN0 teardown restores the read reset at +0x24. */ - dpxbar_set32(dpxbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); + dpxbar_set32(dpxbar, j700_physical ? FIFO_RD_PCLK1_EN : + T602X_FIFO_RD_PCLK2_EN, atc_bit); dpxbar_clear32(dpxbar, T602X_REG_034, atc_bit); - dpxbar_clear32(dpxbar, CROSSBAR_ATC_EN, atc_bit); - dpxbar_clear32(dpxbar, CROSSBAR_DISPEXT_EN, prev_dispext_bit); + if (!j700_physical) { + dpxbar_clear32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + dpxbar_clear32(dpxbar, CROSSBAR_DISPEXT_EN, prev_dispext_bit); + } dpxbar_mask32(dpxbar, T602X_REG_030, mux_mask, 0); dpxbar->selected_dispext[index] = -1; @@ -320,13 +325,15 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) index == MUX_DPIN1 ? FIFO_RD_PCLK1_EN : T602X_REG_01C, atc_bit); dpxbar_set32(dpxbar, T602X_REG_034, atc_bit); - dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + if (!j700_physical) + dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); /* * t8103 enables the dispext source at 0x050. T602x left * that register at 0, so dpin0 was selected with no * source clock into the ACIO analog PHY. */ - dpxbar_set32(dpxbar, CROSSBAR_DISPEXT_EN, dispext_bit); + if (!j700_physical) + dpxbar_set32(dpxbar, CROSSBAR_DISPEXT_EN, dispext_bit); dpxbar_set32(dpxbar, T602X_FIFO_RD_UNK_EN, dispext_bit); diff --git a/tools/diagnostics/test-apple-j700-crossbar.py b/tools/diagnostics/test-apple-j700-crossbar.py new file mode 100644 index 0000000000000..0bc7577127196 --- /dev/null +++ b/tools/diagnostics/test-apple-j700-crossbar.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise the actual T602x mux function for J700 and the other outputs.""" +from pathlib import Path +import os +import re +import shlex +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/mux/apple-display-crossbar.c").read_text() + + +def function(name): + start = source.index("static ", source.index(name) - 30) + opening = source.index("{", start) + depth = 1 + end = opening + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +#define BIT(n) (1U << (n)) +#define GENMASK(h, l) ((~0U << (l)) & (~0U >> (31 - (h)))) +#define FIELD_PREP(mask, value) (((u32)(value) << __builtin_ctz(mask)) & (mask)) +#define MUX_IDLE_DISCONNECT (-2) +#define dev_info(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define spin_lock_irqsave(lock, flags) do { (void)(lock); (flags) = 0; } while (0) +#define spin_unlock_irqrestore(lock, flags) do { (void)(lock); (void)(flags); } while (0) +#define udelay(value) ((void)(value)) +#define t602x_dump(...) ((void)0) +struct apple_dpxbar { int selected_dispext[3], lock; }; +struct mux_chip { struct apple_dpxbar *data; }; +struct mux_control { struct mux_chip *chip; unsigned int index; }; +static struct apple_dpxbar *mux_chip_priv(struct mux_chip *chip) { return chip->data; } +static unsigned int mux_control_get_index(struct mux_control *mux) { return mux->index; } +static bool j700; +static u32 regs[1024], writes[1024]; +static bool of_machine_is_compatible(const char *name) +{ + assert(!strcmp(name, "apple,j700")); + return j700; +} +static void dpxbar_mask32(struct apple_dpxbar *x, u32 reg, u32 mask, u32 value) +{ + (void)x; + assert(reg < sizeof(regs) && !(reg % 4) && !(value & ~mask)); + regs[reg / 4] = (regs[reg / 4] & ~mask) | value; + writes[reg / 4]++; +} +static void dpxbar_set32(struct apple_dpxbar *x, u32 reg, u32 value) +{ + dpxbar_mask32(x, reg, value, value); +} +static void dpxbar_clear32(struct apple_dpxbar *x, u32 reg, u32 value) +{ + dpxbar_mask32(x, reg, value, 0); +} +''' +definitions = "\n".join(re.findall(r"^#define (?:T602X_|CROSSBAR_|ATC_|FIFO_).*$", + source, re.M)) +enum = re.search(r"enum \{ MUX_DPPHY.*?;", source).group() +body = "\n".join(function(name) for name in ( + "t602x_atc_bit(", "t602x_mux_mask(", "t602x_mux_set(", + "apple_dpxbar_set_t602x(")) +suffix = r''' +int main(void) +{ + unsigned int cases = 0; + for (unsigned int machine = 0; machine < 2; machine++) { + for (unsigned int output = 0; output < 3; output++) { + for (int state = 0; state < 9; state++) { + struct apple_dpxbar data = { .selected_dispext = { -1, -1, -1 } }; + struct mux_chip chip = { .data = &data }; + struct mux_control mux = { .chip = &chip, .index = output }; + j700 = machine; + memset(regs, 0xa5, sizeof(regs)); + memset(writes, 0, sizeof(writes)); + assert(!apple_dpxbar_set_t602x(&mux, state)); + assert(data.selected_dispext[output] == state); + assert(!!writes[0x50 / 4] == !(j700 && output == 0)); + assert(!!writes[0x70 / 4] == !(j700 && output == 0)); + memset(writes, 0, sizeof(writes)); + assert(!apple_dpxbar_set_t602x(&mux, MUX_IDLE_DISCONNECT)); + assert(data.selected_dispext[output] == -1); + assert(!!writes[0x50 / 4] == !(j700 && output == 0)); + assert(!!writes[0x70 / 4] == !(j700 && output == 0)); + assert(writes[(j700 && output == 0 ? 0x20 : 0x24) / 4]); + assert(apple_dpxbar_set_t602x(&mux, 9) == -EINVAL); + cases++; + } + } + } + printf("PASS %u actual crossbar select/disconnect scenarios\n", cases); + return 0; +} +''' +with tempfile.TemporaryDirectory(prefix="j700-crossbar-") as work: + work = Path(work) + unit = work / "test.c" + unit.write_text(prefix + definitions + "\n" + enum + "\n" + body + suffix) + command = shlex.split(os.environ.get("CC", "clang")) + # The existing mux implementation compares an int loop index with an unsigned port. + command += ["-std=gnu11", "-Wall", "-Wextra", "-Werror", "-Wno-sign-compare", "-O1", "-g"] + command += ["-fsanitize=address,undefined", str(unit), "-o", str(work / "test")] + subprocess.run(command, check=True) + subprocess.run([str(work / "test")], check=True) From 011471c6d6dbfd7268497e7bb6c65abaf474e51a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:50:55 +1100 Subject: [PATCH 0917/1579] drm: apple: validate J700 active lanes against PHY capacity Firmware resets the active lane count to zero before setting its link rate. Comparing the following two-lane request against that cached zero rejected a valid activation with EINVAL and prevented link completion. Query the PHY capacity for each request instead. Validate impossible counts before changing state, and propagate validation failures. Cover the zero-to-two reset sequence, unsupported counts and missing PHY. Validation: J700 PHY/training tests, 917 AFK assertions and owned-buffer checks pass under sanitizers; strict checkpatch and copy audit clean. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptx-j700-training.h | 16 ++++++++++++++++ drivers/gpu/drm/apple/dptxep.c | 7 +++++-- tools/diagnostics/apple-dptx-training-test.c | 10 ++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dptx-j700-training.h b/drivers/gpu/drm/apple/dptx-j700-training.h index 09a5d1018a7fb..a4632cd363abd 100644 --- a/drivers/gpu/drm/apple/dptx-j700-training.h +++ b/drivers/gpu/drm/apple/dptx-j700-training.h @@ -3,6 +3,22 @@ #ifndef APPLE_DPTX_J700_TRAINING_H #define APPLE_DPTX_J700_TRAINING_H +/* Active lanes may be zero during the firmware's training reset. */ +static int dptx_j700_validate_lanes(struct dptx_port *port, u64 requested) +{ + union phy_configure_opts capacity = {}; + int ret; + + if (!port || !port->atcphy) + return -ENODEV; + if (requested > 4 || requested == 3) + return -EINVAL; + ret = phy_validate(port->atcphy, PHY_MODE_DP, 0, &capacity); + if (ret) + return ret; + return requested <= capacity.dp.lanes ? 0 : -EINVAL; +} + static int dptx_j700_training(struct apple_epic_service *service, bool write, const void *data, size_t size, void *reply, size_t room) { diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index d2ab05ef9e57a..eaa3a3b26bf61 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -415,8 +415,11 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic u64 lane_count = le64_to_cpu(request->lane_count); - if (service->ep->wire_v2 && lane_count > dptx->lane_count) - return -EINVAL; + if (service->ep->wire_v2) { + ret = dptx_j700_validate_lanes(dptx, lane_count); + if (ret) + return ret; + } if (dptx->lane_count < lane_count) dev_err(dcp->dev, "set_active_lane_count: unexpected lane " diff --git a/tools/diagnostics/apple-dptx-training-test.c b/tools/diagnostics/apple-dptx-training-test.c index 53ae4419f293d..72d682820363d 100644 --- a/tools/diagnostics/apple-dptx-training-test.c +++ b/tools/diagnostics/apple-dptx-training-test.c @@ -10,6 +10,7 @@ typedef uint8_t u8; typedef uint32_t u32; typedef uint16_t u16; +typedef uint64_t u64; struct phy { int unused; }; struct settings { unsigned int lanes, voltage[4], pre[4]; @@ -26,6 +27,7 @@ struct apple_epic_service { struct dptx_port *cookie; }; static unsigned int calls; static int failure; #define PHY_MODE_DP 1 +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) static u32 get_unaligned_le32(const void *p) { @@ -123,7 +125,15 @@ int main(void) } } #endif + const unsigned int transitions[] = { 0, 2, 0, 2, 1, 2 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(transitions); i++) + assert(!dptx_j700_validate_lanes(&port, transitions[i])); + assert(dptx_j700_validate_lanes(&port, 3) == -EINVAL); + assert(dptx_j700_validate_lanes(&port, 4) == -EINVAL); + assert(dptx_j700_validate_lanes(&port, UINT64_MAX) == -EINVAL); port.atcphy = NULL; + assert(dptx_j700_validate_lanes(&port, 2) == -ENODEV); assert(dptx_j700_training(&service, true, request, 80, reply, 80) == -ENODEV); printf("DPTX training: %u boundary cases plus failure/cache/readback checks passed\n", cases); From ea06f4a6afef7aa15ec85c772a2da8ac290e4334 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:52:49 +1100 Subject: [PATCH 0918/1579] drm: apple: avoid stale active-lane warnings after V2 validation The V2 request has already passed the physical-capacity check. Restrict the older cached-count warning to the legacy path so a valid zero-to-two transition does not emit an incorrect error. No state transition changes. Validation: J700 host regression tests, strict checkpatch and copy audit. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index eaa3a3b26bf61..4935539bd2920 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -419,9 +419,7 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic ret = dptx_j700_validate_lanes(dptx, lane_count); if (ret) return ret; - } - - if (dptx->lane_count < lane_count) + } else if (dptx->lane_count < lane_count) dev_err(dcp->dev, "set_active_lane_count: unexpected lane " "count:%llu phy: %d\n", lane_count, dptx->lane_count); From 09f9dd589378484133747489f2e8f34a424001db Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:57:03 +1100 Subject: [PATCH 0919/1579] phy: apple: preserve J700 transmitter programming order Keep the separate supply request/override writes and program the link-rate field before the final transmitter output controls. Combining those writes and moving the rate field after output setup did not preserve the audited hardware sequence. The host test now checks the actual production register transaction order, including supply writes. It rejects the previous implementation and passes the corrected sequence for both PMAs and every supported rate/SSC pair. Strict checkpatch and the expanded copy audit pass. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-j700-dp.h | 24 +++++++++++-------- tools/diagnostics/apple-j700-dp-test.c | 33 +++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h index 5ce78d085159d..11d7b32003748 100644 --- a/drivers/phy/apple/atc-j700-dp.h +++ b/drivers/phy/apple/atc-j700-dp.h @@ -71,16 +71,18 @@ static void j700_dp_output_power(struct apple_atcphy *p, unsigned int lane, bool { u32 reg = tx ? 0x10c : 0xc; - core_set32(p, reg, (BIT(4) | BIT(6)) << lane); - udelay(1); - core_set32(p, reg, (BIT(0) | BIT(2)) << lane); - udelay(1); - if (!tx) { - core_set32(p, 0x8, (BIT(10) | BIT(12)) << lane); - udelay(1); - core_set32(p, 0x8, (BIT(6) | BIT(8)) << lane); + for (unsigned int pair = 0; pair < 2; pair++) { + core_set32(p, reg, BIT(4 - 4 * pair + lane)); + core_set32(p, reg, BIT(6 - 4 * pair + lane)); udelay(1); } + if (!tx) { + for (unsigned int pair = 0; pair < 2; pair++) { + core_set32(p, 0x8, BIT(10 - 4 * pair + lane)); + core_set32(p, 0x8, BIT(12 - 4 * pair + lane)); + udelay(1); + } + } reg = tx ? 0x10c : 0x8; core_clear32(p, reg, BIT(tx ? 8 : 14) << lane); core_set32(p, reg, BIT(tx ? 10 : 16) << lane); @@ -115,7 +117,8 @@ static void j700_dp_transmitter(struct apple_atcphy *p, unsigned int lane, int r { 0x44, 0x400000, 0x400000, 0 }, }; static const struct j700_dp_step finish[] = { - { 0x04, 0x33000000, 0x32000000, 0 }, + { 0x04, 0x03000000, 0x02000000, 0 }, + { 0x04, 0x30000000, 0x30000000, 0 }, { 0x10, 0x600, 0x600, 0 }, { 0x1c, 0xc00000, 0x800000, 0 }, }; @@ -126,9 +129,10 @@ static void j700_dp_transmitter(struct apple_atcphy *p, unsigned int lane, int r j700_dp_equalize(p, lane, true, 0); if (rate != ATCPHY_DP_LINK_RATE_HBR3) j700_dp_steps(p, base, divide, ARRAY_SIZE(divide)); - j700_dp_steps(p, base, finish, ARRAY_SIZE(finish)); + j700_dp_steps(p, base, finish, 2); core_mask32(p, base + 0xc, 0x7000000, rate == ATCPHY_DP_LINK_RATE_RBR ? 0x5000000 : 0x4000000); + j700_dp_steps(p, base, finish + 2, ARRAY_SIZE(finish) - 2); } static void j700_dp_receiver(struct apple_atcphy *p, unsigned int lane, int rate) diff --git a/tools/diagnostics/apple-j700-dp-test.c b/tools/diagnostics/apple-j700-dp-test.c index 0b3e24067ac05..d8e565491f96e 100644 --- a/tools/diagnostics/apple-j700-dp-test.c +++ b/tools/diagnostics/apple-j700-dp-test.c @@ -43,6 +43,7 @@ struct apple_atcphy { static u32 registers[0x20000 / 4]; static unsigned int writes, polls, fail_poll, pin_mask = 2; +static u32 transactions[2048], transaction_masks[2048]; static bool fail_all; static void core_mask32(struct apple_atcphy *p, u32 off, u32 mask, u32 value) @@ -51,7 +52,9 @@ static void core_mask32(struct apple_atcphy *p, u32 off, u32 mask, u32 value) assert(!(off % 4) && off / 4 < ARRAY_SIZE(registers)); assert(!(value & ~mask)); registers[off / 4] = (registers[off / 4] & ~mask) | value; - writes++; + assert(writes < ARRAY_SIZE(transactions)); + transaction_masks[writes] = mask; + transactions[writes++] = off; if (off == 0x2000) { if (value == 0x10000001) { registers[off / 4] |= BIT(1); @@ -131,6 +134,33 @@ static struct apple_atcphy device(void) return p; } +static void check_transmitter_order(unsigned int pmas) +{ + for (unsigned int lane = 0; lane < 2; lane++) { + u32 base = 0xd000 + lane * 0x7000; + bool found_rate = false, found_power = false; + + if (!(pmas & BIT(lane))) + continue; + for (unsigned int t = 0; t + 4 < writes; t++) { + if (transactions[t] == base + 4 && transactions[t + 1] == base + 4) { + assert(transactions[t + 2] == base + 0xc); + assert(transactions[t + 3] == base + 0x10); + assert(transactions[t + 4] == base + 0x1c); + found_rate = true; + } + if (transactions[t] == 0x10c && transaction_masks[t] == BIT(4 + lane)) { + assert(transactions[t + 1] == 0x10c); + assert(transaction_masks[t + 1] == BIT(6 + lane)); + assert(transaction_masks[t + 2] == BIT(lane)); + assert(transaction_masks[t + 3] == BIT(2 + lane)); + found_power = true; + } + } + assert(found_rate && found_power); + } +} + int main(void) { const unsigned int rates[] = { 1620, 2700, 5400, 8100 }; @@ -147,6 +177,7 @@ int main(void) assert(!j700_dp_configure(&p, &o)); assert(p.dp_owned_pmas == pin_mask && p.dp_link_rate == (int)rate); + check_transmitter_order(pin_mask); if (pin_mask != 3) { u32 other = pin_mask == 1 ? 0xf0 : 0xf; From b96ac1e25ab65e229dc817546b9a3f8b79502259 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:57:03 +1100 Subject: [PATCH 0920/1579] drm: apple: answer the V2 physical-device mode query The external firmware asks whether the connection is a virtual display. Return a bounded, zero-initialized physical-device answer instead of unsupported, matching the public m1n1 V2 DPTX protocol. Keep other unknown commands rejected and retain the callback group restrictions. Validation: query size/group checks, J700 training tests, 54 crossbar scenarios, 917 codec assertions and owned-buffer tests; checkpatch and copy audit pass. Hardware qualification remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 6 ++++++ drivers/gpu/drm/apple/dptxep.h | 1 + tools/diagnostics/apple-dptx-training-test.c | 11 +++++++++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 4935539bd2920..bae72b7dc7dfb 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -876,6 +876,12 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, u3 return -EOPNOTSUPP; if (command == DPTX_APCALL_SET_DRIVE_SETTINGS) return -EOPNOTSUPP; + if (command == DPTX_APCALL_GET_VIRTUAL_DEVICE_MODE) { + if (room < sizeof(struct dptxport_apcall_get_support)) + return -EINVAL; + memset(reply, 0, room); + return 0; + } if (command == DPTX_APCALL_GET_DRIVE_SETTINGS) return dptx_j700_training(service, false, data, size, reply, room); return dptxport_call(service, command, data, size, reply, room); diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 3eb71cf00f9ba..14530c99567b1 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -30,6 +30,7 @@ enum dptx_apcall { DPTX_APCALL_DEVICE_NOT_RESPONDING = 22, DPTX_APCALL_DEVICE_BUSY_TIMEOUT = 23, DPTX_APCALL_DEVICE_NOT_STARTED = 24, + DPTX_APCALL_GET_VIRTUAL_DEVICE_MODE = 26, }; #define DCPDPTX_REMOTE_PORT_CORE GENMASK(3, 0) diff --git a/tools/diagnostics/apple-dptx-training-test.c b/tools/diagnostics/apple-dptx-training-test.c index 72d682820363d..cb1cea55a6452 100644 --- a/tools/diagnostics/apple-dptx-training-test.c +++ b/tools/diagnostics/apple-dptx-training-test.c @@ -61,6 +61,8 @@ static int phy_configure(struct phy *p, union phy_configure_opts *o) #ifdef DPTX_GROUP_TEST #define DPTX_APCALL_SET_DRIVE_SETTINGS 3 #define DPTX_APCALL_GET_DRIVE_SETTINGS 4 +#define DPTX_APCALL_GET_VIRTUAL_DEVICE_MODE 26 +struct dptxport_apcall_get_support { u8 bytes[32]; }; static int dptxport_call(struct apple_epic_service *service, u32 command, const void *data, size_t size, void *reply, size_t room) { @@ -113,17 +115,22 @@ int main(void) #ifdef DPTX_GROUP_TEST for (unsigned int group = 0; group < 4; group++) { - for (unsigned int command = 0; command < 25; command++) { + for (unsigned int command = 0; command < 27; command++) { int expected = group ? -EOPNOTSUPP : 7; if (group == 0 && command == 3) expected = -EOPNOTSUPP; - if ((group == 1 && command == 3) || (group == 0 && command == 4)) + if ((group == 1 && command == 3) || + (group == 0 && (command == 4 || command == 26))) expected = 0; assert(dptxport_group_call(&service, group, command, request, 80, reply, 80) == expected); } } + assert(dptxport_group_call(&service, 0, 26, request, 80, reply, 31) == -EINVAL); + memset(reply, 0xff, 80); + assert(!dptxport_group_call(&service, 0, 26, request, 80, reply, 80)); + assert(!get_unaligned_le32(reply + 16)); #endif const unsigned int transitions[] = { 0, 2, 0, 2, 1, 2 }; From 29ebc07b7e66e05462dcccf9b6771c864655f7f8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:02:26 +1100 Subject: [PATCH 0921/1579] phy: apple: apply J700 DP calibration before common power-up Move the firmware DP calibration bundle into the tunable-application phase, before the common supply/clamp sequence. Applying it later from AUX enable preserved the final register values but not the required initialization order. Full DP uses its own common bundle; shared USB/DP keeps the USB lane calibration and then applies the DP bundle. The phase ordering follows the authorized hardware audit. Existing public tunable operations are reused; no private implementation is imported. Actual production-dispatch tests cover full/shared/reversed DP, the legacy path and fixed USB2. Existing PHY, codec and crossbar tests pass. Strict checkpatch passes; only generic test scaffolding matched in the reviewed copy audit. Hardware qualification remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/phy/apple/atc.c | 19 +++- tools/diagnostics/test-apple-j700-tuning.py | 103 ++++++++++++++++++++ 3 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 tools/diagnostics/test-apple-j700-tuning.py diff --git a/MAINTAINERS b/MAINTAINERS index c157347692c6f..e68f9960b6e67 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2020,6 +2020,7 @@ F: tools/diagnostics/apple-dptx-training-test.c F: tools/diagnostics/apple-j700-dp-test.c F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-dp.py +F: tools/diagnostics/test-apple-j700-tuning.py F: tools/testing/apple/ APPLETALK NETWORK LAYER diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 0e2d74959b41c..2ec021c2e2c79 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1007,6 +1007,14 @@ static const struct atcphy_mode_configuration *atcphy_get_mode_config(struct app return &atcphy_modes[mode].normal; } +static void j700_dp_tunables(struct apple_atcphy *atcphy) +{ + apple_tunable_apply(atcphy->regs.axi2af, atcphy->tunables.axi2af); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); + apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); +} + static void atcphy_apply_tunables(struct apple_atcphy *atcphy, enum atcphy_mode mode) { const int lane0 = atcphy->swap_lanes ? 1 : 0; @@ -1021,6 +1029,11 @@ static void atcphy_apply_tunables(struct apple_atcphy *atcphy, enum atcphy_mode if (mode == APPLE_ATCPHY_MODE_USB2 && atcphy->fixed_usb2) return; + if (atcphy->hw->has_t8140_dp && mode == APPLE_ATCPHY_MODE_DP) { + j700_dp_tunables(atcphy); + return; + } + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.common[0]); apple_tunable_apply(atcphy->regs.axi2af, atcphy->tunables.axi2af); apple_tunable_apply(atcphy->regs.core, atcphy->tunables.common[1]); @@ -1057,6 +1070,9 @@ static void atcphy_apply_tunables(struct apple_atcphy *atcphy, enum atcphy_mode case APPLE_ATCPHY_MODE_USB2: break; } + if (atcphy->hw->has_t8140_dp && mode == APPLE_ATCPHY_MODE_USB3_DP) + j700_dp_tunables(atcphy); + } static int atcphy_pipehandler_lock(struct apple_atcphy *atcphy) @@ -1612,9 +1628,6 @@ static int j700_aux_disable(struct apple_atcphy *atcphy) static int atcphy_enable_dp_aux(struct apple_atcphy *atcphy) { if (atcphy->hw->has_t8140_dp) { - apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); - apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); - apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); dev_info_once(atcphy->dev, "Applied T8140 DisplayPort setup lists\n"); return j700_aux_enable(atcphy); } diff --git a/tools/diagnostics/test-apple-j700-tuning.py b/tools/diagnostics/test-apple-j700-tuning.py new file mode 100644 index 0000000000000..b6dc336eb71f6 --- /dev/null +++ b/tools/diagnostics/test-apple-j700-tuning.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Check the production calibration dispatch before common-power setup.""" +from pathlib import Path +import os +import shlex +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/phy/apple/atc.c").read_text() + + +def function(name): + start = source.index("static void " + name + "(") + brace = source.index("{", start) + end, depth = brace + 1, 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +prefix = r''' +#include +#include +#include +#include +#include +enum atcphy_mode { + APPLE_ATCPHY_MODE_OFF, APPLE_ATCPHY_MODE_USB2, APPLE_ATCPHY_MODE_USB3, + APPLE_ATCPHY_MODE_USB3_DP, APPLE_ATCPHY_MODE_DP, + APPLE_ATCPHY_MODE_TBT, APPLE_ATCPHY_MODE_USB4, +}; +struct apple_atcphy { + bool swap_lanes, fixed_usb2; + struct { bool has_t8140_dp; } *hw; + struct { void *core, *axi2af, *lpdptx; } regs; + struct { + int *common[2], *axi2af, *lane_usb3[2], *lane_dp[2], *lane_usb4[2]; + int *dp_common_pre, *dp_lpdptx, *dp_common_post; + } tunables; +}; +static unsigned int count; +static int events[20]; +static void apple_tunable_apply(void *base, int *table) +{ + (void)base; + if (table) + events[count++] = *table; +} +''' +suffix = r''' +static void check(struct apple_atcphy *p, enum atcphy_mode mode, + const int *expected, unsigned int size) +{ + count = 0; + atcphy_apply_tunables(p, mode); + assert(count == size); + if (size) + assert(!memcmp(events, expected, sizeof(int) * size)); +} +int main(void) +{ + int ids[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }; + struct apple_atcphy p = {}; + typeof(*p.hw) hw = { .has_t8140_dp = true }; + p.hw = &hw; + p.tunables.common[0] = &ids[0]; p.tunables.common[1] = &ids[1]; + p.tunables.axi2af = &ids[2]; + p.tunables.lane_usb3[0] = &ids[3]; p.tunables.lane_usb3[1] = &ids[4]; + p.tunables.lane_dp[0] = &ids[5]; p.tunables.lane_dp[1] = &ids[6]; + p.tunables.dp_common_pre = &ids[7]; p.tunables.dp_lpdptx = &ids[8]; + p.tunables.dp_common_post = &ids[9]; + p.tunables.lane_usb4[0] = &ids[10]; p.tunables.lane_usb4[1] = &ids[11]; + const int dp[] = { 3, 8, 9, 10 }; + const int shared[] = { 1, 3, 2, 4, 7, 3, 8, 9, 10 }; + const int reversed[] = { 1, 3, 2, 5, 6, 3, 8, 9, 10 }; + const int legacy[] = { 1, 3, 2, 6, 7 }; + check(&p, APPLE_ATCPHY_MODE_DP, dp, 4); + check(&p, APPLE_ATCPHY_MODE_USB3_DP, shared, 9); + p.swap_lanes = true; + check(&p, APPLE_ATCPHY_MODE_USB3_DP, reversed, 9); + p.swap_lanes = false; + hw.has_t8140_dp = false; + check(&p, APPLE_ATCPHY_MODE_DP, legacy, 5); + p.fixed_usb2 = true; + check(&p, APPLE_ATCPHY_MODE_USB2, NULL, 0); + puts("PASS calibration order: full DP, shared, reverse, legacy and fixed USB2"); + return 0; +} +''' +with tempfile.TemporaryDirectory(prefix="j700-tuning-") as work: + work = Path(work) + test = work / "test.c" + test.write_text(prefix + function("j700_dp_tunables") + + function("atcphy_apply_tunables") + suffix) + command = shlex.split(os.environ.get("CC", "clang")) + command += ["-std=gnu11", "-O1", "-g", "-Wall", "-Wextra", "-Werror"] + command += ["-fsanitize=address,undefined", str(test), "-o", str(work / "test")] + subprocess.run(command, check=True) + subprocess.run([str(work / "test")], check=True, + env={**os.environ, "UBSAN_OPTIONS": "halt_on_error=1"}) From dc41df1a19d970134fa28ff0ee4f8ef3fc994c42 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:12:35 +1100 Subject: [PATCH 0922/1579] drm: apple: allow the qualified J700 provider fallback on DCPEXT Missing power-LUT properties on the external controller took the provider failure path and set dcp->crashed before display modes could be published. The existing J700 compatibility payloads were accepted only on the internal controller. Allow those same validated payloads for the explicitly experimental J700 external profile. Keep all request, capacity and property-length checks, and keep malformed properties and other machines/profiles fail-closed. The five existing public payloads match the audited working handoff; no new calibration data is added. Validation: actual provider-callback host tests cover the six known properties, internal compatibility, experimental external compatibility, other profiles/machines, malformed lengths and unknown requests under ASan/UBSan. The old implementation rejects the external regression case. Strict checkpatch and copy audit pass; hardware qualification is pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/gpu/drm/apple/iomfb_h17p.c | 4 +- tools/diagnostics/test-apple-j700-provider.py | 123 ++++++++++++++++++ 3 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 tools/diagnostics/test-apple-j700-provider.py diff --git a/MAINTAINERS b/MAINTAINERS index e68f9960b6e67..0ceb4b28e0fef 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2020,6 +2020,7 @@ F: tools/diagnostics/apple-dptx-training-test.c F: tools/diagnostics/apple-j700-dp-test.c F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-dp.py +F: tools/diagnostics/test-apple-j700-provider.py F: tools/diagnostics/test-apple-j700-tuning.py F: tools/testing/apple/ diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 682a92879f8ef..49fcabaeb7c66 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -445,7 +445,9 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, goto fail; } else if (!of_property_present(node, property->dt_name) && of_machine_is_compatible("apple,j700") && - of_device_is_compatible(node, "apple,t8140-dcp")) { + (of_device_is_compatible(node, "apple,t8140-dcp") || + (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + of_device_is_compatible(node, "apple,t8140-dcpext")))) { /* Qualified older loaders lack the newly described table properties. */ memset(reply, 0, sizeof(*reply)); memcpy(reply->data, property->legacy_data, property->length); diff --git a/tools/diagnostics/test-apple-j700-provider.py b/tools/diagnostics/test-apple-j700-provider.py new file mode 100644 index 0000000000000..9872c6d55be3b --- /dev/null +++ b/tools/diagnostics/test-apple-j700-provider.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Run the actual provider callback against missing and malformed properties.""" +from pathlib import Path +import os +import shlex +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/gpu/drm/apple/iomfb_h17p.c").read_text() +body = source[source.index("struct dcp_h17p_provider_request {"): + source.index("static bool trampoline_luma_span_h17p")] +prefix = r''' +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint32_t __le32; +#define __packed __attribute__((packed)) +#undef static_assert +#define static_assert(cond) _Static_assert(cond, "") +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define sizeof_field(t, f) sizeof(((t *)0)->f) +#define le32_to_cpu(x) (x) +#define cpu_to_le32(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +#define DCP_IOMFB_METHODS_H17G 1 +#define DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL 2 +#define trace_iomfb_callback(d, t, n) ((void)(d), (void)(t)) +#define dev_err(...) ((void)0) +#define dev_warn_once(...) ((void)0) +struct device_node { bool internal, external, present; int length; }; +struct device { struct device_node *of_node; }; +struct apple_dcp { struct { int iomfb_method_profile; } hw; struct device *dev; bool crashed; }; +static bool j700 = true; +static bool of_machine_is_compatible(const char *name) +{ + assert(!strcmp(name, "apple,j700")); + return j700; +} +static bool of_device_is_compatible(struct device_node *n, const char *name) +{ + return (!strcmp(name, "apple,t8140-dcp") && n->internal) || + (!strcmp(name, "apple,t8140-dcpext") && n->external); +} +static bool of_property_present(struct device_node *n, const char *name) +{ + (void)name; + return n->present; +} +static int of_property_count_u8_elems(struct device_node *n, const char *name) +{ + (void)name; + return n->present ? n->length : -EINVAL; +} +static int of_property_read_u8_array(struct device_node *n, const char *name, void *p, unsigned int len) +{ + (void)n; (void)name; + memset(p, 0x5a, len); + return 0; +} +static const void *memchr_inv(const void *p, int value, size_t size) +{ + const u8 *b = p; + for (size_t i = 0; i < size; i++) + if (b[i] != value) + return b + i; + return NULL; +} +''' +suffix = r''' +int main(void) +{ + struct device_node node = { .external = true }; + struct device dev = { .of_node = &node }; + struct apple_dcp dcp = { .dev = &dev, .hw.iomfb_method_profile = 2 }; + struct dcp_h17p_provider_request req = { .scope = { 'V', 'O', 'R', 'P' }, .capacity = 0xc00 }; + struct dcp_h17p_provider_reply reply; + unsigned int cases = 0; + for (unsigned int i = 0; i < ARRAY_SIZE(provider_properties); i++) { + memset(&reply, 0xff, sizeof(reply)); + strcpy(req.name, provider_properties[i].name); + assert(trampoline_provider_property_h17p(&dcp, 400, &reply, &req)); + assert(!dcp.crashed && reply.length == provider_properties[i].length); + if (reply.length) + assert(!memcmp(reply.data, provider_properties[i].legacy_data, reply.length)); + cases++; + } + strcpy(req.name, "power-lut-data-x"); + j700 = false; + assert(!trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && dcp.crashed); + j700 = true; dcp.crashed = false; dcp.hw.iomfb_method_profile = 3; + assert(!trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && dcp.crashed); + node.external = false; node.internal = true; dcp.crashed = false; + assert(trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && !dcp.crashed); + node.internal = false; node.external = true; node.present = true; node.length = 3; + dcp.hw.iomfb_method_profile = 2; dcp.crashed = false; + assert(!trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && dcp.crashed); + node.length = 4; dcp.crashed = false; + assert(trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && !dcp.crashed); + assert(reply.length == 4 && reply.data[0] == 0x5a); + strcpy(req.name, "unqualified-property"); + assert(!trampoline_provider_property_h17p(&dcp, 400, &reply, &req)); + printf("PASS %u provider payloads and scope/length rejection cases\n", cases); + return 0; +} +''' +with tempfile.TemporaryDirectory(prefix="j700-provider-") as work: + work = Path(work) + test = work / "test.c" + test.write_text(prefix + body + suffix) + command = shlex.split(os.environ.get("CC", "clang")) + command += ["-std=gnu11", "-O1", "-g", "-Wall", "-Wextra", "-Werror"] + command += ["-fsanitize=address,undefined", str(test), "-o", str(work / "test")] + subprocess.run(command, check=True) + subprocess.run([str(work / "test")], check=True, + env={**os.environ, "UBSAN_OPTIONS": "halt_on_error=1"}) From e8fe156f2d3dcc0058228abc0ebb9f7c1c4e3300 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:19:22 +1100 Subject: [PATCH 0923/1579] drm/apple: add scoped J700 link-rate diagnostic ceiling Preserve HBR3 by default and on other outputs. Permit known lower DP rates only for the experimental J700 physical output so training failures can be isolated without changing PHY tables. Exercise the actual callback over all ceilings and scope combinations. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 19 ++++- .../diagnostics/test-apple-j700-link-rate.py | 70 +++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 tools/diagnostics/test-apple-j700-link-rate.py diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index bae72b7dc7dfb..964f2896b6373 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,12 @@ #include "dptx-j700-training.h" +/* Diagnostic ceiling for the experimental physical J700 output only. */ +static unsigned int j700_dp_max_link_rate = LINK_RATE_HBR3; +module_param(j700_dp_max_link_rate, uint, 0600); +MODULE_PARM_DESC(j700_dp_max_link_rate, + "J700 external DP ceiling: 6=RBR, 10=HBR, 20=HBR2, 30=HBR3"); + struct dcpdptx_connection_cmd { __le32 unk; __le32 target; @@ -343,12 +350,22 @@ static int dptxport_call_get_max_link_rate(struct apple_epic_service *service, void *reply_, size_t reply_size) { struct dptxport_apcall_link_rate *reply = reply_; + struct apple_dcp *dcp = service->ep->dcp; + u32 ceiling = READ_ONCE(j700_dp_max_link_rate); if (reply_size < sizeof(*reply)) return -EINVAL; reply->retcode = cpu_to_le32(0); - reply->link_rate = cpu_to_le32(LINK_RATE_HBR3); + if (service->cookie && ((struct dptx_port *)service->cookie)->atcphy && + dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + of_machine_is_compatible("apple,j700") && + (ceiling == LINK_RATE_RBR || ceiling == LINK_RATE_HBR || + ceiling == LINK_RATE_HBR2 || ceiling == LINK_RATE_HBR3)) + reply->link_rate = cpu_to_le32(ceiling); + else + reply->link_rate = cpu_to_le32(LINK_RATE_HBR3); return 0; } diff --git a/tools/diagnostics/test-apple-j700-link-rate.py b/tools/diagnostics/test-apple-j700-link-rate.py new file mode 100644 index 0000000000000..7bef0cc936918 --- /dev/null +++ b/tools/diagnostics/test-apple-j700-link-rate.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise the production link ceiling callback and its isolation boundaries.""" +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/gpu/drm/apple/dptxep.c").read_text() +start = source.index("static int dptxport_call_get_max_link_rate(") +end = source.index("\n}\n", start) + 3 +stub = r""" +#include +#include +#include +#include +#include +typedef uint32_t u32; +#define LINK_RATE_RBR 6 +#define LINK_RATE_HBR 10 +#define LINK_RATE_HBR2 20 +#define LINK_RATE_HBR3 30 +#define DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL 7 +#define READ_ONCE(x) (x) +#define cpu_to_le32(x) (x) +struct apple_dcp { struct { int iomfb_method_profile; } hw; }; +struct apple_epic_endpoint { struct apple_dcp *dcp; }; +struct apple_epic_service { struct apple_epic_endpoint *ep; void *cookie; }; +struct dptx_port { void *atcphy; }; +struct dptxport_apcall_link_rate { u32 retcode; char pad[12]; u32 link_rate; char tail[12]; }; +static u32 j700_dp_max_link_rate; +static bool j700; +static bool of_machine_is_compatible(const char *name) { (void)name; return j700; } +""" +main = r""" +int main(void) { + struct apple_dcp dcp = {}; + struct apple_epic_endpoint ep = { &dcp }; + struct dptx_port port = {}; + struct apple_epic_service service = { &ep, &port }; + struct dptxport_apcall_link_rate reply; + unsigned cases = 0; + for (unsigned rate = 0; rate <= 40; rate++) + for (unsigned scope = 0; scope < 8; scope++) { + j700 = scope & 1; + port.atcphy = (scope & 2) ? &port : NULL; + dcp.hw.iomfb_method_profile = (scope & 4) ? 7 : 0; + j700_dp_max_link_rate = rate; + assert(dptxport_call_get_max_link_rate(&service, &reply, sizeof(reply)) == 0); + bool valid = rate == 6 || rate == 10 || rate == 20 || rate == 30; + assert(reply.link_rate == ((scope == 7 && valid) ? rate : 30)); + assert(reply.retcode == 0); + cases++; + } + service.cookie = NULL; + assert(!dptxport_call_get_max_link_rate(&service, &reply, sizeof(reply))); + assert(reply.link_rate == 30); + for (unsigned size = 0; size < sizeof(reply); size++) + assert(dptxport_call_get_max_link_rate(&service, &reply, size) == -EINVAL); + return cases != 328; +} +""" +with tempfile.TemporaryDirectory(prefix="j700-rate-") as tmp: + c = Path(tmp) / "test.c" + exe = Path(tmp) / "test" + c.write_text(stub + source[start:end] + main) + subprocess.run(["clang", "-Wall", "-Wextra", "-Werror", + "-fsanitize=address,undefined", str(c), "-o", str(exe)], check=True) + subprocess.run([str(exe)], check=True) +print("328 link ceilings and scope cases, null port and all short replies: PASS") From f7ced60fe8b08c9b45c439c5c3b6abff25bcbe62 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:20:51 +1100 Subject: [PATCH 0924/1579] drm/apple: report unknown callback bounds for diagnosis Include validated input/output lengths and channel context in the existing warning. Retain the external fail-closed behavior while identifying the D134 callback observed after link setup. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 1950599effd74..5bd735f9299c4 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -818,8 +818,9 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, if (!handled) { if (!iomfb_uses_queue(dcp) || !test_and_set_bit(tag, dcp->unknown_callbacks)) - dev_warn(dev, "received unknown callback %c%c%c%c\n", - hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + dev_warn(dev, "received unknown callback %c%c%c%c (%u, %u) context=%u\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0], + hdr->in_len, hdr->out_len, context); if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { /* Retain the failed session without inventing a successful reply. */ From db547bff7d79d0df818afac3d668c10280e78145 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:25:59 +1100 Subject: [PATCH 0925/1579] drm/apple: answer the measured J700 external D134 record The external controller reaches video-interface publication, then sends D134 on the async channel with no input and a four-byte output. Its unanswered callback marks the strict external session crashed. Permit the existing zero reply only for that exact record on J700 T8140 DCPEXT. Retain fail-closed behavior for every other unknown external callback and preserve legacy handling. Test the production dispatcher under ASan/UBSan across 15552 tag, size, context and device/profile combinations, including output bounds and unchanged legacy replies. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 28 +++++- tools/diagnostics/test-apple-j700-d134.py | 111 ++++++++++++++++++++++ 2 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 tools/diagnostics/test-apple-j700-d134.py diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5bd735f9299c4..5fe53684795fc 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -770,6 +770,23 @@ void dcp_hotplug(struct work_struct *work) drm_kms_helper_connector_hotplug_event(&connector->base); } +/* + * J700/25G83 asks D134 after publishing its external video interface. + * The observed async record has no input and a four-byte result. Use the + * existing zero reply for this record only; its feature remains unsupported. + */ +static bool dcp_external_zero_reply(struct apple_dcp *dcp, int tag, + enum dcp_context_id context, + u32 in_len, u32 out_len) +{ + return tag == 134 && context == DCP_CONTEXT_ASYNC && + in_len == 0 && out_len == sizeof(u32) && + dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + of_machine_is_compatible("apple,j700") && + of_device_is_compatible(dcp->dev->of_node, "apple,t8140-dcpext"); +} + static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, void *data, u32 length, u16 offset) { @@ -779,7 +796,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, int tag; struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; - bool handled; + bool handled, zero_reply; if (dcp->fw_compat == DCP_FIRMWARE_H17P && (length < sizeof(*hdr) || !ch || ch->depth >= DCP_MAX_CALL_DEPTH || @@ -816,13 +833,16 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, out = in + hdr->in_len; if (!handled) { - if (!iomfb_uses_queue(dcp) || - !test_and_set_bit(tag, dcp->unknown_callbacks)) + zero_reply = dcp_external_zero_reply(dcp, tag, context, + hdr->in_len, hdr->out_len); + if (!zero_reply && + (!iomfb_uses_queue(dcp) || + !test_and_set_bit(tag, dcp->unknown_callbacks))) dev_warn(dev, "received unknown callback %c%c%c%c (%u, %u) context=%u\n", hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0], hdr->in_len, hdr->out_len, context); if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && !zero_reply) { /* Retain the failed session without inventing a successful reply. */ dcp->crashed = true; complete(&dcp->start_done); diff --git a/tools/diagnostics/test-apple-j700-d134.py b/tools/diagnostics/test-apple-j700-d134.py new file mode 100644 index 0000000000000..d29debcd45634 --- /dev/null +++ b/tools/diagnostics/test-apple-j700-d134.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Compile the production dispatcher and check the sole D134 reply exception.""" +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/gpu/drm/apple/iomfb.c").read_text() +start = source.index("static bool dcp_external_zero_reply(") +end = source.index("static void dcpep_handle_ack(", start) +prefix = r""" +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +#define DCP_FIRMWARE_H17P 1 +#define DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL 2 +#define DCP_IOMFB_METHODS_H17G 3 +#define DCP_MAX_CALL_DEPTH 8 +#define IOMFB_MAX_CB 600 +#define DCP_PACKET_ALIGNMENT 64 +#define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1)) +#define dev_err(...) ((void)dev) +#define dev_info(...) ((void)dev) +#define dev_warn(...) ((void)dev) +enum dcp_context_id { DCP_CONTEXT_CB = 0, DCP_CONTEXT_ASYNC = 3 }; +struct device_node { bool external; }; +struct device { struct device_node *of_node; }; +struct dcp_packet_header { char tag[4]; u32 in_len, out_len; }; +struct dcp_channel { unsigned depth; void *output[8]; unsigned end[8]; }; +struct apple_dcp { + struct { int iomfb_method_profile; } hw; + struct device *dev; + int fw_compat; + unsigned experimental_callback_logs; + bool crashed; + bool (**cb_handlers)(struct apple_dcp *, int, void *, void *); + unsigned long unknown_callbacks[16]; + int start_done; + struct dcp_channel channel; +}; +static bool machine; +static unsigned acks, case_tag; +static struct dcp_channel *dcp_get_channel(struct apple_dcp *d, enum dcp_context_id c) +{ (void)c; return &d->channel; } +static int dcp_parse_tag(char *p) { (void)p; return case_tag; } +static bool of_machine_is_compatible(const char *n) { (void)n; return machine; } +static bool of_device_is_compatible(struct device_node *n, const char *s) +{ (void)s; return n->external; } +static bool iomfb_check_callback_h17p(struct apple_dcp *d, int t, u32 i, u32 o) +{ (void)d; (void)t; (void)i; (void)o; return true; } +static bool iomfb_uses_queue(struct apple_dcp *d) { (void)d; return false; } +static bool test_and_set_bit(int t, unsigned long *b) { (void)t; (void)b; return false; } +static void complete(int *p) { *p = 1; } +static u8 dcp_push_depth(unsigned *d) { return (*d)++; } +static void dcp_ack(struct apple_dcp *d, enum dcp_context_id c) +{ (void)c; acks++; d->channel.depth--; } +""" +suffix = r""" +int main(void) +{ + struct device_node node; + struct device dev = { &node }; + bool (*handlers[IOMFB_MAX_CB])(struct apple_dcp *, int, void *, void *) = {}; + unsigned cases = 0; + for (unsigned scope = 0; scope < 8; scope++) + for (unsigned tag = 133; tag <= 135; tag++) + for (unsigned context = 0; context < 8; context++) + for (unsigned in = 0; in <= 8; in++) + for (unsigned out = 0; out <= 8; out++) { + struct apple_dcp d = { + .dev = &dev, .fw_compat = 1, .experimental_callback_logs = 64, + .cb_handlers = handlers, .hw.iomfb_method_profile = 2, + }; + union { uint64_t align; unsigned char data[64]; } packet; + struct dcp_packet_header *h = (void *)packet.data; + memset(&packet, 0xa5, sizeof(packet)); + h->in_len = in; h->out_len = out; + machine = scope & 1; node.external = scope & 2; + if (!(scope & 4)) + d.hw.iomfb_method_profile = 0; + case_tag = tag; acks = 0; + dcpep_handle_cb(&d, context, h, sizeof(*h) + in + out, 0); + bool accepted = scope == 7 && tag == 134 && context == 3 && !in && out == 4; + if (scope & 4) { + assert(acks == accepted && d.crashed != accepted); + for (unsigned i = 0; i < out; i++) + assert(packet.data[sizeof(*h) + in + i] == (accepted ? 0 : 0xa5)); + } else { + assert(acks == 1 && !d.crashed); /* Existing legacy behavior. */ + } + assert(packet.data[sizeof(*h) + in + out] == 0xa5); + cases++; + } + return cases != 15552; +} +""" +with tempfile.TemporaryDirectory(prefix="j700-d134-") as tmp: + c = Path(tmp) / "test.c" + exe = Path(tmp) / "test" + c.write_text(prefix + source[start:end] + suffix) + subprocess.run(["clang", "-std=gnu11", "-Wall", "-Wextra", "-Werror", + "-fsanitize=address,undefined", str(c), "-o", str(exe)], check=True) + subprocess.run([str(exe)], check=True) +print("15552 dispatcher cases: D134 bounds/scope, rejection and legacy replies PASS") From 1a317d5142ea6ff0e563403d51a0b738ce01927c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:30:46 +1100 Subject: [PATCH 0926/1579] drm/apple: route native training readback during modeset R24 reaches external mode export and a visible but frozen image. Firmware uses group 1 command 4 to read training settings during modeset; route it to the bounded native readback helper instead of rejecting it. Test returned cached lane values, no PHY writes and short-reply rejection. Include pending command and present-state information in the existing transaction timeout diagnostic. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 2 ++ drivers/gpu/drm/apple/iomfb.c | 14 +++++++++++++- tools/diagnostics/apple-dptx-training-test.c | 14 +++++++++++++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 964f2896b6373..34c49d078533e 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -889,6 +889,8 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, u3 { if (group == 1 && command == DPTX_APCALL_SET_DRIVE_SETTINGS) return dptx_j700_training(service, true, data, size, reply, room); + if (group == 1 && command == DPTX_APCALL_GET_DRIVE_SETTINGS) + return dptx_j700_training(service, false, data, size, reply, room); if (group) return -EOPNOTSUPP; if (command == DPTX_APCALL_SET_DRIVE_SETTINGS) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5fe53684795fc..8d1fecfec763f 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -312,7 +312,19 @@ static void iomfb_queue_timeout(struct work_struct *work) dcp->iomfb.deadline - jiffies); } else if (dcp->iomfb.active) { WRITE_ONCE(dcp->crashed, true); - dev_err(dcp->dev, "IOMFB transaction timed out\n"); + dev_err(dcp->dev, + "IOMFB transaction timed out: cmd=%u oob=%u present=%u accepted=%u swap=%u\n", + dcp->ch_cmd.depth, dcp->ch_oobcmd.depth, + dcp->present_state_h17p.pending, dcp->present_state_h17p.accepted, + dcp->present_state_h17p.swap_id); + if (dcp->ch_cmd.depth && dcp->ch_cmd.depth <= DCP_MAX_CALL_DEPTH) { + const struct dcp_packet_header *h = + &dcp->ch_cmd.header[dcp->ch_cmd.depth - 1]; + + dev_err(dcp->dev, "pending IOMFB %c%c%c%c in=%u out=%u\n", + h->tag[3], h->tag[2], h->tag[1], h->tag[0], + h->in_len, h->out_len); + } iomfb_discard_pending(dcp); schedule_work(&dcp->vblank_wq); } diff --git a/tools/diagnostics/apple-dptx-training-test.c b/tools/diagnostics/apple-dptx-training-test.c index cb1cea55a6452..7e3a7f3b59790 100644 --- a/tools/diagnostics/apple-dptx-training-test.c +++ b/tools/diagnostics/apple-dptx-training-test.c @@ -120,13 +120,25 @@ int main(void) if (group == 0 && command == 3) expected = -EOPNOTSUPP; - if ((group == 1 && command == 3) || + if ((group == 1 && (command == 3 || command == 4)) || (group == 0 && (command == 4 || command == 26))) expected = 0; assert(dptxport_group_call(&service, group, command, request, 80, reply, 80) == expected); } } + put_unaligned_le32(2, request); + unsigned int before_readback = calls; + + memset(reply, 0xff, 80); + assert(!dptxport_group_call(&service, 1, 4, request, 56, reply, 56)); + assert(calls == before_readback); + assert(get_unaligned_le32(reply) == 2 && !get_unaligned_le32(reply + 16)); + assert(get_unaligned_le32(reply + 36) == port.trained_voltage[0]); + assert(get_unaligned_le32(reply + 52) == port.trained_pre[1]); + assert(reply[56] == 0xff); + assert(dptxport_group_call(&service, 1, 4, request, 56, reply, 55) == -EINVAL); + assert(calls == before_readback); assert(dptxport_group_call(&service, 0, 26, request, 80, reply, 31) == -EINVAL); memset(reply, 0xff, 80); assert(!dptxport_group_call(&service, 0, 26, request, 80, reply, 80)); From 4b0a450bef3a82e3453e9ee28c817d03e98743d9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:35:59 +1100 Subject: [PATCH 0927/1579] drm/apple: keep opaque X property off experimental DCPEXT The external controller rejects the post-first-present enableGPAlphaDiv property with status 4. Treating that panel workaround as required marks the external session crashed and freezes a visible image. Leave the experimental external controller on its native presentation path while preserving the existing internal-panel property queue. Cover 396 production-queue scope/state/capacity cases and allocation failure under ASan/UBSan. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 4 + .../test-apple-j700-opaque-scope.py | 83 +++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 tools/diagnostics/test-apple-j700-opaque-scope.py diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8d1fecfec763f..e7eefa6abf91a 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -193,6 +193,10 @@ static int iomfb_enqueue_opaque_x(struct apple_dcp *dcp) struct iomfb_opaque_x_transaction *opaque; lockdep_assert_held(&dcp->iomfb.lock); + /* DCPEXT rejects enableGPAlphaDiv; retain the panel-only workaround. */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + return 0; if (atomic_read(&dcp->iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) return 0; if (dcp->iomfb.queued >= 32) diff --git a/tools/diagnostics/test-apple-j700-opaque-scope.py b/tools/diagnostics/test-apple-j700-opaque-scope.py new file mode 100644 index 0000000000000..d9827134366b2 --- /dev/null +++ b/tools/diagnostics/test-apple-j700-opaque-scope.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Check that the actual property queue preserves panel behavior, excludes DCPEXT.""" +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/gpu/drm/apple/iomfb.c").read_text() +start = source.index("static int iomfb_enqueue_opaque_x(") +end = source.index("\n}\n", start) + 3 +prefix = r""" +#include +#include +#include +#define DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL 2 +enum { IOMFB_OPAQUE_X_WAITING, IOMFB_OPAQUE_X_NEEDED, + IOMFB_OPAQUE_X_QUEUED, IOMFB_OPAQUE_X_READY }; +struct list_head { unsigned count; }; +struct apple_dcp; +struct iomfb_transaction { + void (*start)(struct apple_dcp *, struct iomfb_transaction *); + void (*release)(struct iomfb_transaction *); + struct list_head link; +}; +struct iomfb_opaque_x_transaction { struct iomfb_transaction transaction; }; +struct apple_dcp { + struct { int iomfb_method_profile; } hw; + struct { int lock, opaque_x_state; unsigned queued; struct list_head pending; } iomfb; +}; +static unsigned allocations; +static int fail_alloc; +static void *last_allocation; +static void *allocate(size_t n) { allocations++; return last_allocation = fail_alloc ? NULL : calloc(1, n); } +#define kzalloc_obj(x) allocate(sizeof(x)) +#define kfree(x) free(x) +#define lockdep_assert_held(x) ((void)(x)) +#define atomic_read(x) (*(x)) +static int atomic_cmpxchg(int *p, int from, int to) +{ int old = *p; if (old == from) *p = to; return old; } +static void list_add(struct list_head *n, struct list_head *h) +{ (void)n; h->count++; } +static void iomfb_opaque_x_start(struct apple_dcp *d, struct iomfb_transaction *t) +{ (void)d; (void)t; } +static void iomfb_opaque_x_release(struct iomfb_transaction *t) { (void)t; } +""" +suffix = r""" +int main(void) +{ + for (int state = 0; state < 4; state++) + for (unsigned queued = 0; queued <= 32; queued++) + for (int profile = 0; profile < 3; profile++) { + struct apple_dcp d = { .hw.iomfb_method_profile = profile, + .iomfb = { .opaque_x_state = state, .queued = queued } }; + last_allocation = NULL; + unsigned before = allocations; + int ret = iomfb_enqueue_opaque_x(&d); + if (profile == 2 || state != IOMFB_OPAQUE_X_NEEDED) { + assert(!ret && allocations == before); + assert(d.iomfb.queued == queued && d.iomfb.opaque_x_state == state); + } else if (queued == 32) { + assert(ret == -EBUSY && allocations == before); + } else { + assert(!ret && allocations == before + 1); + assert(d.iomfb.queued == queued + 1 && d.iomfb.pending.count == 1); + assert(d.iomfb.opaque_x_state == IOMFB_OPAQUE_X_QUEUED); + } + free(last_allocation); + } + struct apple_dcp d = { .iomfb.opaque_x_state = IOMFB_OPAQUE_X_NEEDED }; + fail_alloc = 1; + assert(iomfb_enqueue_opaque_x(&d) == -ENOMEM); + return 0; +} +""" +with tempfile.TemporaryDirectory(prefix="j700-property-") as tmp: + c = Path(tmp) / "test.c" + exe = Path(tmp) / "test" + c.write_text(prefix + source[start:end] + suffix) + subprocess.run(["clang", "-std=gnu11", "-Wall", "-Wextra", "-Werror", + "-fsanitize=address,undefined", str(c), "-o", str(exe)], check=True) + subprocess.run([str(exe)], check=True) +print("396 actual queue scope/state/capacity cases and allocation failure: PASS") From 649b825945882c0389c3ca5ea51453e6a3435f0c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:40:33 +1100 Subject: [PATCH 0928/1579] drm/apple: add output-targeted Wayland presentation check Run a bounded GTK4 animation on the selected Wayland connector and report draw, frame callback and presentation-feedback counts. The window closes automatically without changing persistent display settings. This allows external presentation to be checked while the operator is away. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- .../diagnostics/wayland-output-frame-test.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 tools/diagnostics/wayland-output-frame-test.py diff --git a/tools/diagnostics/wayland-output-frame-test.py b/tools/diagnostics/wayland-output-frame-test.py new file mode 100644 index 0000000000000..107fad56f539f --- /dev/null +++ b/tools/diagnostics/wayland-output-frame-test.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Show a bounded animation on one Wayland output and report frame feedback.""" +import argparse +import json +import time + +import gi + +gi.require_version("Gtk", "4.0") +gi.require_version("Gdk", "4.0") +from gi.repository import Gdk, GLib, Gtk # noqa: E402 + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("--connector", default="DP-1") +parser.add_argument("--seconds", type=int, default=8, choices=range(3, 31)) +args = parser.parse_args() +Gtk.init() +display = Gdk.Display.get_default() +if display is None: + raise SystemExit("No display connection") +monitors = display.get_monitors() +matches = [monitors.get_item(i) for i in range(monitors.get_n_items()) + if monitors.get_item(i).get_connector() == args.connector] +if len(matches) != 1: + raise SystemExit("Requested output is absent or ambiguous") + +loop = GLib.MainLoop() +window = Gtk.Window(title="Aurora external display frame test") +area = Gtk.DrawingArea() +window.set_child(area) +started = time.monotonic() +stats = {"connector": args.connector, "ticks": 0, "draws": 0} +feedback = set() + + +def draw(_area, context, width, height): + phase = (time.monotonic() - started) % 2 / 2 + context.set_source_rgb(0.06, 0.10, 0.17) + context.paint() + context.set_source_rgb(0.15, 0.80, 0.55) + context.rectangle(phase * width, height * 0.25, width * 0.15, height * 0.5) + context.fill() + stats["draws"] += 1 + + +def tick(widget, clock): + stats["ticks"] += 1 + previous = clock.get_timings(clock.get_frame_counter() - 1) + if previous is not None and previous.get_complete(): + presented = previous.get_presentation_time() + if presented: + feedback.add(presented) + widget.queue_draw() + return GLib.SOURCE_CONTINUE + + +def finish(): + window.destroy() + loop.quit() + return GLib.SOURCE_REMOVE + + +area.set_draw_func(draw) +area.add_tick_callback(tick) +window.connect("close-request", lambda _window: finish()) +window.fullscreen_on_monitor(matches[0]) +window.present() +GLib.timeout_add_seconds(args.seconds, finish) +loop.run() +stats["elapsed_s"] = round(time.monotonic() - started, 3) +stats["presentation_feedback_frames"] = len(feedback) +stats["callback_pass"] = stats["ticks"] >= args.seconds * 15 +stats["draw_pass"] = stats["draws"] >= args.seconds * 15 +stats["presentation_pass"] = len(feedback) >= args.seconds * 10 +print(json.dumps(stats, sort_keys=True)) +raise SystemExit(0 if all(stats[k] for k in + ("callback_pass", "draw_pass", "presentation_pass")) else 1) From 77c3d30fd99b2fd98138d1384aa733d33c3c4892 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:41:23 +1100 Subject: [PATCH 0929/1579] diagnostics: collect delayed Wayland frame feedback Inspect the retained frame history instead of only the immediately preceding frame, whose presentation feedback may still be pending. Report completed timing records separately from presentation timestamps. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/diagnostics/wayland-output-frame-test.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tools/diagnostics/wayland-output-frame-test.py b/tools/diagnostics/wayland-output-frame-test.py index 107fad56f539f..e9f0893ead65d 100644 --- a/tools/diagnostics/wayland-output-frame-test.py +++ b/tools/diagnostics/wayland-output-frame-test.py @@ -32,6 +32,7 @@ started = time.monotonic() stats = {"connector": args.connector, "ticks": 0, "draws": 0} feedback = set() +complete_frames = set() def draw(_area, context, width, height): @@ -46,11 +47,14 @@ def draw(_area, context, width, height): def tick(widget, clock): stats["ticks"] += 1 - previous = clock.get_timings(clock.get_frame_counter() - 1) - if previous is not None and previous.get_complete(): - presented = previous.get_presentation_time() - if presented: - feedback.add(presented) + counter = clock.get_frame_counter() + for frame in range(max(0, counter - 16), counter): + previous = clock.get_timings(frame) + if previous is not None and previous.get_complete(): + complete_frames.add(frame) + presented = previous.get_presentation_time() + if presented: + feedback.add(presented) widget.queue_draw() return GLib.SOURCE_CONTINUE @@ -70,6 +74,7 @@ def finish(): loop.run() stats["elapsed_s"] = round(time.monotonic() - started, 3) stats["presentation_feedback_frames"] = len(feedback) +stats["complete_timings"] = len(complete_frames) stats["callback_pass"] = stats["ticks"] >= args.seconds * 15 stats["draw_pass"] = stats["draws"] >= args.seconds * 15 stats["presentation_pass"] = len(feedback) >= args.seconds * 10 From 9190b4b742cfc4d95bc1b5dea00c8c96820d441e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:45:26 +1100 Subject: [PATCH 0930/1579] drm/apple: remove temporary external bring-up diagnostics Drop temporary protocol capture counters, PHY lock chatter, the global Type-C trace addition and unused link-rate override. Preserve standard tracepoints, failure diagnostics and the validated HBR3 behavior. Register the remaining focused tests and output-frame utility with MAINTAINERS. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 3 + drivers/gpu/drm/apple/afk.c | 17 +---- drivers/gpu/drm/apple/afk.h | 2 - drivers/gpu/drm/apple/dcp-internal.h | 2 - drivers/gpu/drm/apple/dptxep.c | 21 +----- drivers/gpu/drm/apple/iomfb.c | 8 +-- drivers/phy/apple/atc-j700-dp.h | 2 - drivers/usb/typec/mux.c | 1 - .../diagnostics/test-apple-j700-link-rate.py | 70 ------------------- 9 files changed, 7 insertions(+), 119 deletions(-) delete mode 100644 tools/diagnostics/test-apple-j700-link-rate.py diff --git a/MAINTAINERS b/MAINTAINERS index 0ceb4b28e0fef..f7a8a7dd863ca 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2019,9 +2019,12 @@ F: sound/soc/apple/t8140_aop_audio.rs F: tools/diagnostics/apple-dptx-training-test.c F: tools/diagnostics/apple-j700-dp-test.c F: tools/diagnostics/test-apple-j700-crossbar.py +F: tools/diagnostics/test-apple-j700-d134.py F: tools/diagnostics/test-apple-j700-dp.py +F: tools/diagnostics/test-apple-j700-opaque-scope.py F: tools/diagnostics/test-apple-j700-provider.py F: tools/diagnostics/test-apple-j700-tuning.py +F: tools/diagnostics/wayland-output-frame-test.py F: tools/testing/apple/ APPLETALK NETWORK LAYER diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 38b44c2d6d785..b45695ddf9836 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -922,15 +922,6 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, payload_size = data_size - hdr_len; trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); - if (ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && - ep->experimental_rx_logs < 32) { - ep->experimental_rx_logs++; - dev_info(ep->dcp->dev, - "external AFK ep=%u channel=%u type=%u subtype=%u category=%u bytes=%zu\n", - ep->endpoint, channel, type, subtype, eshdr->category, - data_size); - } service = afk_epic_find_service(ep, channel); @@ -1092,13 +1083,7 @@ static void afk_receive_message_worker(struct work_struct *work_) work = container_of(work_, struct afk_receive_message_work, work); type = FIELD_GET(RBEP_TYPE, work->message); - if (work->ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && - (work->ep->endpoint == DPTX_ENDPOINT || - work->ep->endpoint == DPTX_ENDPOINT_V2) && type != RBEP_RECV) - dev_info_ratelimited(work->ep->dcp->dev, - "external AFK control ep=%#x type=%#x\n", - work->ep->endpoint, type); + switch (type) { case RBEP_INIT_ACK: break; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 3d1f975d798a8..c6a34fa67399e 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -295,8 +295,6 @@ struct afk_ringbuffer { struct apple_dcp_afkep { struct apple_dcp *dcp; - /* Bounded external protocol observations, owned by the RX worker. */ - u8 experimental_rx_logs; u32 endpoint; /* Set before afk_start; protocol selection is per endpoint. */ diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 883d4f9ce2e80..48f4112151dd3 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -212,8 +212,6 @@ struct apple_dcp { struct apple_connector *connector; struct apple_dcp_hw_data hw; - /* Protected by the H17P IOMFB receive mutex. */ - u8 experimental_callback_logs; bool external_prepared; /* firmware version and compatible firmware version */ diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 34c49d078533e..6c4012d6ad752 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -21,12 +20,6 @@ #include "dptx-j700-training.h" -/* Diagnostic ceiling for the experimental physical J700 output only. */ -static unsigned int j700_dp_max_link_rate = LINK_RATE_HBR3; -module_param(j700_dp_max_link_rate, uint, 0600); -MODULE_PARM_DESC(j700_dp_max_link_rate, - "J700 external DP ceiling: 6=RBR, 10=HBR, 20=HBR2, 30=HBR3"); - struct dcpdptx_connection_cmd { __le32 unk; __le32 target; @@ -350,22 +343,12 @@ static int dptxport_call_get_max_link_rate(struct apple_epic_service *service, void *reply_, size_t reply_size) { struct dptxport_apcall_link_rate *reply = reply_; - struct apple_dcp *dcp = service->ep->dcp; - u32 ceiling = READ_ONCE(j700_dp_max_link_rate); if (reply_size < sizeof(*reply)) return -EINVAL; reply->retcode = cpu_to_le32(0); - if (service->cookie && ((struct dptx_port *)service->cookie)->atcphy && - dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && - of_machine_is_compatible("apple,j700") && - (ceiling == LINK_RATE_RBR || ceiling == LINK_RATE_HBR || - ceiling == LINK_RATE_HBR2 || ceiling == LINK_RATE_HBR3)) - reply->link_rate = cpu_to_le32(ceiling); - else - reply->link_rate = cpu_to_le32(LINK_RATE_HBR3); + reply->link_rate = cpu_to_le32(LINK_RATE_HBR3); return 0; } @@ -861,7 +844,7 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, return; } port = 0; - dev_info(dcp->dev, "DPTX physical unit %lld maps to logical port 0\n", unit); + dev_dbg(dcp->dev, "DPTX physical unit %lld maps to logical port 0\n", unit); } switch (port) { diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index e7eefa6abf91a..786eb53383329 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -823,13 +823,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, } tag = dcp_parse_tag(hdr->tag); - if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && - dcp->experimental_callback_logs < 64) { - dcp->experimental_callback_logs++; - dev_info(dev, "external IOMFB callback=%d in=%u out=%u context=%u\n", - tag, hdr->in_len, hdr->out_len, context); - } + handled = tag >= 0 && tag < IOMFB_MAX_CB && dcp->cb_handlers && dcp->cb_handlers[tag]; diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h index 11d7b32003748..9071587bd5e8d 100644 --- a/drivers/phy/apple/atc-j700-dp.h +++ b/drivers/phy/apple/atc-j700-dp.h @@ -303,9 +303,7 @@ static int j700_dp_configure(struct apple_atcphy *p, const struct phy_configure_ unsigned int pmas, capacity, first, table; int rate = -1, ret; - dev_dbg(p->dev, "J700 DP configure waiting for PHY lock\n"); guard(mutex)(&p->lock); - dev_dbg(p->dev, "J700 DP configure acquired PHY lock\n"); if ((o->set_rate && !o->link_rate) || (o->set_lanes && !o->lanes)) return o->set_voltages ? -EINVAL : j700_dp_stop(p); if (p->mode != APPLE_ATCPHY_MODE_DP && p->mode != APPLE_ATCPHY_MODE_USB3_DP) diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index dc763766012f7..7e327118cbd71 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -388,7 +388,6 @@ int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) mux_dev = mux->mux_devs[i]; ret = mux_dev->set(mux_dev, state); - dev_dbg(&mux_dev->dev, "mode=%lu callback result=%d\n", state->mode, ret); if (ret && ret != -EOPNOTSUPP) return ret; } diff --git a/tools/diagnostics/test-apple-j700-link-rate.py b/tools/diagnostics/test-apple-j700-link-rate.py deleted file mode 100644 index 7bef0cc936918..0000000000000 --- a/tools/diagnostics/test-apple-j700-link-rate.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-only -"""Exercise the production link ceiling callback and its isolation boundaries.""" -from pathlib import Path -import subprocess -import tempfile - -root = Path(__file__).resolve().parents[2] -source = (root / "drivers/gpu/drm/apple/dptxep.c").read_text() -start = source.index("static int dptxport_call_get_max_link_rate(") -end = source.index("\n}\n", start) + 3 -stub = r""" -#include -#include -#include -#include -#include -typedef uint32_t u32; -#define LINK_RATE_RBR 6 -#define LINK_RATE_HBR 10 -#define LINK_RATE_HBR2 20 -#define LINK_RATE_HBR3 30 -#define DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL 7 -#define READ_ONCE(x) (x) -#define cpu_to_le32(x) (x) -struct apple_dcp { struct { int iomfb_method_profile; } hw; }; -struct apple_epic_endpoint { struct apple_dcp *dcp; }; -struct apple_epic_service { struct apple_epic_endpoint *ep; void *cookie; }; -struct dptx_port { void *atcphy; }; -struct dptxport_apcall_link_rate { u32 retcode; char pad[12]; u32 link_rate; char tail[12]; }; -static u32 j700_dp_max_link_rate; -static bool j700; -static bool of_machine_is_compatible(const char *name) { (void)name; return j700; } -""" -main = r""" -int main(void) { - struct apple_dcp dcp = {}; - struct apple_epic_endpoint ep = { &dcp }; - struct dptx_port port = {}; - struct apple_epic_service service = { &ep, &port }; - struct dptxport_apcall_link_rate reply; - unsigned cases = 0; - for (unsigned rate = 0; rate <= 40; rate++) - for (unsigned scope = 0; scope < 8; scope++) { - j700 = scope & 1; - port.atcphy = (scope & 2) ? &port : NULL; - dcp.hw.iomfb_method_profile = (scope & 4) ? 7 : 0; - j700_dp_max_link_rate = rate; - assert(dptxport_call_get_max_link_rate(&service, &reply, sizeof(reply)) == 0); - bool valid = rate == 6 || rate == 10 || rate == 20 || rate == 30; - assert(reply.link_rate == ((scope == 7 && valid) ? rate : 30)); - assert(reply.retcode == 0); - cases++; - } - service.cookie = NULL; - assert(!dptxport_call_get_max_link_rate(&service, &reply, sizeof(reply))); - assert(reply.link_rate == 30); - for (unsigned size = 0; size < sizeof(reply); size++) - assert(dptxport_call_get_max_link_rate(&service, &reply, size) == -EINVAL); - return cases != 328; -} -""" -with tempfile.TemporaryDirectory(prefix="j700-rate-") as tmp: - c = Path(tmp) / "test.c" - exe = Path(tmp) / "test" - c.write_text(stub + source[start:end] + main) - subprocess.run(["clang", "-Wall", "-Wextra", "-Werror", - "-fsanitize=address,undefined", str(c), "-o", str(exe)], check=True) - subprocess.run([str(exe)], check=True) -print("328 link ceilings and scope cases, null port and all short replies: PASS") From 18d69dc8064845d35c8d868efc917682b7171a62 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:51:31 +1100 Subject: [PATCH 0931/1579] drm/apple: close external admission and recovery races Make explicitly enabled external providers participate in component matching before readiness, then validate prepared state after component binding. Reject unpublished or retired DPTX services and preserve stable ownership rather than reusing a published service generation. Retain failed J700 PHY transition state so SAFE retries cleanup, and reject normal training while that state is dirty. Add fault, admission/lifetime and provider-order regressions. A published DPTX service loss remains fail-closed until recovery; these changes do not claim physical hotplug qualification. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 2 + drivers/gpu/drm/apple/Makefile | 2 + drivers/gpu/drm/apple/afk-service-test.c | 99 ++++++ drivers/gpu/drm/apple/afk.c | 27 +- drivers/gpu/drm/apple/afk.h | 32 +- drivers/gpu/drm/apple/apple_drv.c | 23 +- drivers/gpu/drm/apple/dcp.c | 27 +- drivers/gpu/drm/apple/dptx-service.h | 63 ++++ drivers/gpu/drm/apple/dptxep.c | 55 ++-- drivers/phy/apple/atc-j700-dp.h | 26 +- drivers/phy/apple/atc.c | 28 +- tools/diagnostics/apple-j700-dp-test.c | 4 +- tools/diagnostics/test-apple-atc-unwind.py | 295 ++++++++++++++++++ tools/diagnostics/test-apple-dcp-readiness.py | 246 +++++++++++++++ 14 files changed, 856 insertions(+), 73 deletions(-) create mode 100644 drivers/gpu/drm/apple/dptx-service.h create mode 100644 tools/diagnostics/test-apple-atc-unwind.py create mode 100644 tools/diagnostics/test-apple-dcp-readiness.py diff --git a/MAINTAINERS b/MAINTAINERS index f7a8a7dd863ca..e7b22a9c9b774 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2018,6 +2018,8 @@ F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs F: tools/diagnostics/apple-dptx-training-test.c F: tools/diagnostics/apple-j700-dp-test.c +F: tools/diagnostics/test-apple-atc-unwind.py +F: tools/diagnostics/test-apple-dcp-readiness.py F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-d134.py F: tools/diagnostics/test-apple-j700-dp.py diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 837aa713ad435..4980b2863225b 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -21,3 +21,5 @@ appledrm-$(CONFIG_TRACING) += trace.o obj-$(CONFIG_DRM_APPLE) += appledrm.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk_test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_backlight_test.o + +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk-service-test.o diff --git a/drivers/gpu/drm/apple/afk-service-test.c b/drivers/gpu/drm/apple/afk-service-test.c index 4a5f07fa9b15f..a86df389deb17 100644 --- a/drivers/gpu/drm/apple/afk-service-test.c +++ b/drivers/gpu/drm/apple/afk-service-test.c @@ -3,6 +3,7 @@ #include #include "afk.h" +#include "dptx-service.h" static const struct apple_epic_service_ops reusable_ops = { .name = "reusable", @@ -122,7 +123,105 @@ static void afk_service_unconverted_owner_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); } +static int afk_test_prepare(struct apple_epic_service *service, + const char *name, const char *class, s64 unit) +{ + return 0; +} + +static const struct apple_epic_service_ops prepared_ops = { + .name = "prepared", + .prepare = afk_test_prepare, + .teardown = dptx_service_unbind, +}; + +static void afk_service_admission_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &prepared_ops, 1)); + /* Neither callback dispatch nor command callers can observe preparation. */ + KUNIT_EXPECT_FALSE(test, afk_service_matches(service, 1)); + KUNIT_EXPECT_PTR_EQ(test, afk_service_get(service), NULL); + /* A rejected announcement or failed OPEN consumes no service slot. */ + afk_service_discard(service); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &prepared_ops, 2)); + /* Cookie-less services are valid: admission does not test cookie != NULL. */ + afk_service_publish(service); + KUNIT_EXPECT_TRUE(test, afk_service_matches(service, 2)); + KUNIT_ASSERT_PTR_EQ(test, afk_service_get(service), service); + afk_service_put(service); +} + +static void dptx_service_identity_test(struct kunit *test) +{ + const char *name = "dcpdptx-port-epic"; + const char *class = "AppleDCPDPTXRemotePort"; + + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 3, true, 3, 1), 0); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 0, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, -1, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 3, true, 3, 2), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(NULL, class, 3, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, NULL, 3, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port("wrong", class, 3, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, "wrong", 3, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 1, false, 3, 2), 1); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 2, false, 3, 2), -EINVAL); +} + +static void dptx_service_duplicate_close_test(struct kunit *test) +{ + struct apple_epic_service *first = test->priv; + struct apple_epic_service *next; + struct dptx_port port = {}; + + next = kunit_kzalloc(test, sizeof(*next), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, next); + spin_lock_init(&next->lock); + init_completion(&port.enable_completion); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(first, NULL, &prepared_ops, 1)); + KUNIT_ASSERT_EQ(test, dptx_service_bind(first, &port, 3), 0); + /* An observer during preparation/OPEN sees no available port or owner. */ + KUNIT_EXPECT_FALSE(test, dptx_service_available(&port)); + KUNIT_EXPECT_PTR_EQ(test, port.service, NULL); + KUNIT_EXPECT_PTR_EQ(test, first->cookie, &port); + /* Failed OPEN does not publish anything and permits a fresh attempt. */ + afk_service_discard(first); + KUNIT_EXPECT_FALSE(test, port.enabled); + KUNIT_EXPECT_PTR_EQ(test, port.service, NULL); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(first, NULL, &prepared_ops, 1)); + KUNIT_ASSERT_EQ(test, dptx_service_bind(first, &port, 3), 0); + afk_service_publish(first); + KUNIT_EXPECT_FALSE(test, port.enabled); + dptx_service_publish(first); + KUNIT_EXPECT_TRUE(test, dptx_service_available(&port)); + KUNIT_EXPECT_PTR_EQ(test, port.service->cookie, &port); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(next, NULL, &prepared_ops, 2)); + KUNIT_EXPECT_EQ(test, dptx_service_bind(next, &port, 3), -EBUSY); + afk_service_discard(next); + KUNIT_EXPECT_TRUE(test, port.enabled); + KUNIT_EXPECT_PTR_EQ(test, port.service, first); + /* CLOSE invalidates a connected session without publishing a replacement. */ + port.connected = true; + afk_test_retire(first); + dptx_service_unbind(first); + KUNIT_EXPECT_FALSE(test, port.enabled); + KUNIT_EXPECT_FALSE(test, port.connected); + KUNIT_EXPECT_PTR_EQ(test, first->cookie, &port); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(next, NULL, &prepared_ops, 3)); + KUNIT_EXPECT_EQ(test, dptx_service_bind(next, &port, 3), -EBUSY); + afk_service_discard(next); + /* Old work between validate/connect/request still names the old session. */ + KUNIT_EXPECT_PTR_EQ(test, port.service, first); + KUNIT_EXPECT_PTR_EQ(test, afk_service_get(port.service), NULL); + KUNIT_EXPECT_FALSE(test, port.enabled); +} + static struct kunit_case afk_service_cases[] = { + KUNIT_CASE(afk_service_admission_test), + KUNIT_CASE(dptx_service_identity_test), + KUNIT_CASE(dptx_service_duplicate_close_test), KUNIT_CASE(afk_service_many_generations_test), KUNIT_CASE(afk_service_reader_pin_test), KUNIT_CASE(afk_service_pending_reply_test), diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index b45695ddf9836..3fecf8d931db5 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -388,6 +388,16 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, goto free; } ep->num_channels = max(ep->num_channels, ch_idx + 1); + if (ops->prepare) { + int ret = ops->prepare(service, epic_name, epic_class, epic_unit); + + if (ret) { + dev_err(ep->dcp->dev, "AFK service %s admission failed: %d\n", + service_name, ret); + afk_service_discard(service); + goto free; + } + } if (ep->wire_v2) { int ret = afk_send_epic(ep, channel, 0, EPIC_TYPE_NOTIFY, EPIC_CAT_REPORT, AFKV2_OPEN, NULL, 0); @@ -395,16 +405,23 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, if (ret) { unsigned long flags; - spin_lock_irqsave(&service->lock, flags); - service->enabled = false; - service->torndown = true; - spin_unlock_irqrestore(&service->lock, flags); + if (ops->prepare) { + afk_service_discard(service); + } else { + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + service->torndown = true; + spin_unlock_irqrestore(&service->lock, flags); + } dev_err(ep->dcp->dev, "AFKV2 interface %u OPEN failed: %d\n", channel, ret); goto free; } } - ops->init(service, epic_name, epic_class, epic_unit); + if (ops->prepare) + afk_service_publish(service); + if (ops->init) + ops->init(service, epic_name, epic_class, epic_unit); dev_info(ep->dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", ep->endpoint, service_name, channel); diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index c6a34fa67399e..b8c889da5c356 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -71,6 +71,12 @@ struct apple_epic_service_ops { /* All retained pointers must be pinned before enabling slot reuse. */ bool reusable; + /* + * Optional admission, before OPEN/publication. Must not retain service + * on failure; teardown must accept a failed/partial preparation. + */ + int (*prepare)(struct apple_epic_service *service, const char *name, + const char *class, s64 unit); void (*init)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); int (*call)(struct apple_epic_service *service, u32 idx, @@ -177,7 +183,7 @@ afk_service_reinit(struct apple_epic_service *service, spin_lock_irqsave(&service->lock, flags); reusable = afk_service_reclaimable(service); if (reusable) { - service->enabled = true; + service->enabled = !ops->prepare; service->torndown = false; service->ops = ops; service->ep = ep; @@ -189,6 +195,30 @@ afk_service_reinit(struct apple_epic_service *service, return reusable; } +/* Only used before a prepared service has been published to callers. */ +static inline void afk_service_discard(struct apple_epic_service *service) +{ + unsigned long flags; + + if (service->ops->teardown) + service->ops->teardown(service); + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + service->torndown = true; + service->cookie = NULL; + service->ops = NULL; + spin_unlock_irqrestore(&service->lock, flags); +} + +static inline void afk_service_publish(struct apple_epic_service *service) +{ + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + service->enabled = true; + spin_unlock_irqrestore(&service->lock, flags); +} + struct afk_ringbuffer_header { __le32 bufsz; u32 unk; diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index eda6ea33fa225..d0462690c25c2 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -640,16 +640,12 @@ static const struct of_device_id apple_dcp_id_tbl[] = { {}, }; -/* - * The T8140 external display coprocessor needs firmware support this driver - * does not have yet. Leave it out of the display subsystem, so that an - * enabled node cannot hold back the internal panel's component master. - */ +/* An explicitly enabled external provider participates in component matching. */ static bool apple_dcp_usable(struct device_node *np) { return of_device_is_available(np) && (!of_device_is_compatible(np, "apple,t8140-dcpext") || - dcp_h17p_external_ready(np)); + dcp_h17p_external_enabled()); } static int apple_drm_init_dcp(struct device *dev) @@ -670,6 +666,13 @@ static int apple_drm_init_dcp(struct device *dev) of_device_is_compatible(np, "apple,t8140-dcpext") || of_property_present(np, "phys"); + /* component_bind_all() established the matched provider lifetime. */ + if (of_device_is_compatible(np, "apple,t8140-dcpext") && + !dcp_h17p_external_ready(np)) { + of_node_put(np); + return -ENODEV; + } + dcp[num_dcp] = of_find_device_by_node(np); if (!dcp[num_dcp]) continue; @@ -679,8 +682,14 @@ static int apple_drm_init_dcp(struct device *dev) ret = apple_probe_per_dcp(dev, &apple->drm, dcp[num_dcp], num_dcp, dcp_ext, &encoder[num_dcp], &crtc_mask[num_dcp]); - if (ret) + if (ret) { + if (of_device_is_compatible(np, "apple,t8140-dcpext")) { + put_device(&dcp[num_dcp]->dev); + of_node_put(np); + return ret; + } continue; + } num_dcp++; } diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7edee07d7bb10..4f2b4923135af 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -46,6 +46,7 @@ #include "dcp.h" #include "dcp-internal.h" #include "dcp-external-bw.h" +#include "dptx-service.h" #include "iomfb.h" #include "parser.h" #include "trace.h" @@ -169,12 +170,14 @@ bool dcp_h17p_external_ready(struct device_node *np) pdev = of_find_device_by_node(np); if (!pdev) return false; - /* Failed and deferred probes must never become required components. */ - if (device_is_bound(&pdev->dev)) { - dcp = platform_get_drvdata(pdev); - ready = dcp && READ_ONCE(dcp->external_prepared) && - !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->crashed); - } + /* + * Called after component binding. component_add() may synchronously + * bind the master before the platform probe itself returns, so the + * driver core's bound flag is not a readiness condition here. + */ + dcp = platform_get_drvdata(pdev); + ready = dcp && READ_ONCE(dcp->external_prepared) && + !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->crashed); put_device(&pdev->dev); return ready; } @@ -2185,7 +2188,7 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) dcp->connector_type, dcp->dptxport[port].connected); mutex_lock(&dcp->hpd_mutex); - if (!dcp->dptxport[port].enabled) { + if (!dptx_service_available(&dcp->dptxport[port])) { dev_warn(dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); ret = -ENODEV; goto out_unlock; @@ -2322,7 +2325,7 @@ static void dcp_placeholder_edid_work(struct work_struct *work) mutex_lock(&dcp->hpd_mutex); generation = dcp->placeholder_generation; if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || - !dcp->dptxport[0].enabled || generation != dcp->typec_generation) + !dptx_service_available(&dcp->dptxport[0]) || generation != dcp->typec_generation) goto out_unlock; service = dcp->dptxport[0].service; @@ -2397,7 +2400,7 @@ static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); mutex_lock(&dcp->hpd_mutex); - if (dcp->dptxport[port].enabled && dcp->dptxport[port].connected) { + if (dptx_service_available(&dcp->dptxport[port]) && dcp->dptxport[port].connected) { dptxport_release_display(dcp->dptxport[port].service); dcp->dptxport[port].connected = false; } @@ -2449,7 +2452,7 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) if (dcp->avep) av_service_disconnect(dcp); - if (dcp->dptxport[port].enabled) + if (dptx_service_available(&dcp->dptxport[port])) dptxport_set_hpd(dcp->dptxport[port].service, false); return dcp_dptx_disconnect(dcp, port); @@ -2601,7 +2604,7 @@ int dcp_start(struct platform_device *pdev) * Since dcp0 is not used at the moment let's avoid this * since it is possibly the cause for startup issues. */ - } else if (dcp->dptxport[0].enabled) { + } else if (dptx_service_available(&dcp->dptxport[0])) { bool connected; /* force disconnect on start - necessary if the display * is already up from m1n1 @@ -2850,7 +2853,7 @@ void dcp_poweroff(struct platform_device *pdev) if (dcp_is_typec_output(dcp)) { /* DCP owns a synthetic HPD for Type-C. Release it with the CRTC. */ - if (dcp->dptxport[0].enabled && dcp->dptxport[0].connected) { + if (dptx_service_available(&dcp->dptxport[0]) && dcp->dptxport[0].connected) { ret = dptxport_set_hpd(dcp->dptxport[0].service, false); if (ret) dev_warn(dcp->dev, diff --git a/drivers/gpu/drm/apple/dptx-service.h b/drivers/gpu/drm/apple/dptx-service.h new file mode 100644 index 0000000000000..f806e3253999a --- /dev/null +++ b/drivers/gpu/drm/apple/dptx-service.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DPTX_SERVICE_H +#define APPLE_DPTX_SERVICE_H + +#include "afk.h" +#include "dptxep.h" + +static inline int dptx_service_port(const char *name, const char *class, + s64 unit, bool wire_v2, u32 index, + u32 num_ports) +{ + if (!name || !class || strcmp(name, "dcpdptx-port-epic") || + strcmp(class, "AppleDCPDPTXRemotePort")) + return -EINVAL; + if (wire_v2) + return num_ports == 1 && unit == index ? 0 : -EINVAL; + return unit >= 0 && unit < 2 ? unit : -EINVAL; +} + +static inline bool dptx_service_available(struct dptx_port *port) +{ + /* Pairs with publication before readers use the service and its cookie. */ + return smp_load_acquire(&port->enabled); +} + +static inline int dptx_service_bind(struct apple_epic_service *service, + struct dptx_port *port, s64 unit) +{ + if (port->service) + return -EBUSY; + port->unit = unit; + /* No owner is published until OPEN and AFK publication succeed. */ + service->cookie = port; + return 0; +} + +static inline void dptx_service_publish(struct apple_epic_service *service) +{ + struct dptx_port *port = service->cookie; + + port->service = service; + port->connected = false; + /* Pair with dcp.c readers before they dereference the service cookie. */ + smp_store_release(&port->enabled, true); +} + +static inline void dptx_service_unbind(struct apple_epic_service *service) +{ + struct dptx_port *port = service->cookie; + + if (port && port->service == service) { + WRITE_ONCE(port->enabled, false); + WRITE_ONCE(port->connected, false); + reinit_completion(&port->enable_completion); + } + /* + * Keep the published owner and cookie valid for retained callers. Once + * this session closes, bind fails closed until reboot: + * an in-flight connect must never switch to a new firmware session. + */ +} + +#endif diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 6c4012d6ad752..451820389f91c 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -15,6 +15,7 @@ #include "afk.h" #include "dcp.h" #include "dptxep.h" +#include "dptx-service.h" #include "parser.h" #include "trace.h" @@ -823,48 +824,26 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, } } -static void dptxport_init(struct apple_epic_service *service, const char *name, - const char *class, s64 unit) +static int dptxport_prepare(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; - s64 port = unit; - - if (!name || !class) - return; - if (strcmp(name, "dcpdptx-port-epic")) - return; - if (strcmp(class, "AppleDCPDPTXRemotePort")) - return; + int port = dptx_service_port(name, class, unit, service->ep->wire_v2, + dcp->index, dcp->hw.num_dptx_ports); + if (port < 0) + return port; trace_dptxport_init(dcp, unit); - if (service->ep->wire_v2) { - /* The single external service reports its physical DCP unit. */ - if (dcp->hw.num_dptx_ports != 1 || unit != dcp->index) { - dev_err(dcp->dev, "Unexpected external DPTX unit %lld\n", unit); - return; - } - port = 0; - dev_dbg(dcp->dev, "DPTX physical unit %lld maps to logical port 0\n", unit); - } + return dptx_service_bind(service, &dcp->dptxport[port], unit); +} - switch (port) { - case 0: - case 1: - if (service->ep->dcp->dptxport[port].enabled) { - dev_err(service->ep->dcp->dev, - "DPTXPort: unit %lld already exists\n", unit); - return; - } - service->ep->dcp->dptxport[port].unit = unit; - service->ep->dcp->dptxport[port].service = service; - service->ep->dcp->dptxport[port].enabled = true; - service->cookie = (void *)&service->ep->dcp->dptxport[port]; - complete(&service->ep->dcp->dptxport[port].enable_completion); - break; - default: - dev_err(service->ep->dcp->dev, "DPTXPort: invalid unit %lld\n", - unit); - } +static void dptxport_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct dptx_port *port = service->cookie; + + dptx_service_publish(service); + complete(&port->enable_completion); } static int dptxport_group_call(struct apple_epic_service *service, u16 group, u32 command, @@ -892,6 +871,8 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, u3 static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", + .prepare = dptxport_prepare, + .teardown = dptx_service_unbind, .init = dptxport_init, .call = dptxport_call, .call_v2 = dptxport_group_call, diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h index 9071587bd5e8d..ead9ea6be3f3a 100644 --- a/drivers/phy/apple/atc-j700-dp.h +++ b/drivers/phy/apple/atc-j700-dp.h @@ -3,11 +3,26 @@ #ifndef APPLE_ATC_J700_DP_H #define APPLE_ATC_J700_DP_H +/* Retry only the idle operations claimed by an incomplete route change. */ +static int j700_dp_idle_pending(struct apple_atcphy *p) +{ + int ret; + + for (unsigned int lane = 0; lane < 2; lane++) { + if (!(p->dp_idle_pmas & BIT(lane))) + continue; + ret = j700_pma_idle(p, lane); + if (ret) + return ret; + p->dp_idle_pmas &= ~BIT(lane); + } + return 0; +} + /* Preserve the upper routing state established by the platform firmware. */ static int j700_dp_route(struct apple_atcphy *p, bool four_lanes) { u32 protocols = four_lanes ? 0x492 : (p->swap_lanes ? 0x252 : 0x489); - int ret; core_mask32(p, 0x60, 0xfff, protocols); if (four_lanes) { @@ -19,11 +34,9 @@ static int j700_dp_route(struct apple_atcphy *p, bool four_lanes) for (unsigned int lane = 0; lane < 2; lane++) { if (!four_lanes && lane != !p->swap_lanes) continue; - ret = j700_pma_idle(p, lane); - if (ret) - return ret; + p->dp_idle_pmas |= BIT(lane); } - return 0; + return j700_dp_idle_pending(p); } /* Register facts from the authorized J700 hardware audit. */ @@ -306,6 +319,9 @@ static int j700_dp_configure(struct apple_atcphy *p, const struct phy_configure_ guard(mutex)(&p->lock); if ((o->set_rate && !o->link_rate) || (o->set_lanes && !o->lanes)) return o->set_voltages ? -EINVAL : j700_dp_stop(p); + /* The last completed mode cannot describe a partially changed lane route. */ + if (p->mode_dirty) + return -EBUSY; if (p->mode != APPLE_ATCPHY_MODE_DP && p->mode != APPLE_ATCPHY_MODE_USB3_DP) return -EINVAL; mode = atcphy_get_mode_config(p, p->mode); diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 2ec021c2e2c79..ea04dec0f0d05 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -654,6 +654,8 @@ struct atcphy_hw { * @ss_tunables: The complete SuperSpeed tunable set was supplied * @dp_tunables: The complete T8140 DisplayPort bundle was supplied * @aux_power_requested: AUX power-up began and needs matching shutdown + * @mode_dirty: J700 mode transition needs cleanup before it can be retried + * @dp_idle_pmas: J700 PMAs with an incomplete routing idle sequence * @dp_owned_pmas: J700 PMAs requiring main-link cleanup * @dp_spread_spectrum: J700 main-link spread-spectrum state * @dp_presets: Firmware-derived per-rate drive presets @@ -717,6 +719,8 @@ struct apple_atcphy { bool ss_tunables; bool dp_tunables; bool aux_power_requested; + bool mode_dirty; + u8 dp_idle_pmas; u8 dp_owned_pmas; bool dp_spread_spectrum; u32 dp_presets[5][16]; @@ -2683,28 +2687,43 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) u32 reg; lockdep_assert_held(&atcphy->lock); - if (atcphy->hw->has_t8140_dp && atcphy->dp_owned_pmas && mode != atcphy->mode) { + if (atcphy->hw->has_t8140_dp && atcphy->dp_owned_pmas && + (mode != atcphy->mode || atcphy->mode_dirty)) { + atcphy->mode_dirty = true; ret = j700_dp_stop(atcphy); if (ret) return ret; } + if (atcphy->hw->has_t8140_dp && atcphy->mode_dirty) { + ret = j700_dp_idle_pending(atcphy); + if (ret) + return ret; + } if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && !atcphy->dp_tunables) return -EOPNOTSUPP; - if (atcphy->hw->has_t8140_dp && !atcphy_modes[mode].enable_dp_aux && + if (atcphy->hw->has_t8140_dp && + (!atcphy_modes[mode].enable_dp_aux || atcphy->mode_dirty) && atcphy->aux_power_requested) { + atcphy->mode_dirty = true; ret = j700_aux_disable(atcphy); if (ret) return ret; } + /* Keep failed transitions retryable even when the requested mode is unchanged. */ + if (atcphy->hw->has_t8140_dp) + atcphy->mode_dirty = true; /* the mode change below may power the PHY down: stop the tunnel clock first */ atc_tunnel_stop_t8103(atcphy); atc_tunnel_restore(atcphy); if (mode == APPLE_ATCPHY_MODE_OFF) { ret = atcphy_power_off(atcphy); + if (ret) + return ret; atcphy->mode = mode; - return ret; + atcphy->mode_dirty = false; + return 0; } ret = atcphy_power_on(atcphy); @@ -2788,6 +2807,7 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) } atcphy->mode = mode; + atcphy->mode_dirty = false; return 0; } @@ -3261,7 +3281,7 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta target_mode = APPLE_ATCPHY_MODE_USB2; dev_dbg(atcphy->dev, "mux target=%u\n", target_mode); - if (atcphy->mode == target_mode) + if (atcphy->mode == target_mode && !atcphy->mode_dirty) return 0; switch (target_mode) { diff --git a/tools/diagnostics/apple-j700-dp-test.c b/tools/diagnostics/apple-j700-dp-test.c index d8e565491f96e..23caf5df299c0 100644 --- a/tools/diagnostics/apple-j700-dp-test.c +++ b/tools/diagnostics/apple-j700-dp-test.c @@ -36,8 +36,8 @@ struct apple_atcphy { struct { void *core; } regs; void *dev; int lock, mode, dp_link_rate; - unsigned int dp_owned_pmas; - bool dp_spread_spectrum, swap_lanes, dp_presets_valid[5]; + unsigned int dp_owned_pmas, dp_idle_pmas; + bool dp_spread_spectrum, swap_lanes, mode_dirty, dp_presets_valid[5]; u32 dp_presets[5][16]; }; diff --git a/tools/diagnostics/test-apple-atc-unwind.py b/tools/diagnostics/test-apple-atc-unwind.py new file mode 100644 index 0000000000000..0494b16fb3bc7 --- /dev/null +++ b/tools/diagnostics/test-apple-atc-unwind.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Fault-inject production J700 mode, mux, AUX and PMA transitions (no hardware).""" +import hashlib +import os +from pathlib import Path +import re +import shlex +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +atc = (root / 'drivers/phy/apple/atc.c').read_text() +dp = (root / 'drivers/phy/apple/atc-j700-dp.h').read_text() + + +def block(source, marker): + start = source.index(marker) + opening = source.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def function(source, name): + return block(source, re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) + + +# Use the production mode table and wrappers too, so SAFE mapping and fast paths +# are tested rather than mirrored by a second implementation. +parts = re.findall(r'^#define [^\n]*(?:\\\n[^\n]*)*', atc, re.M) +for name in ('atcphy_generation', 'atcphy_lane_mode', 'atcphy_pipehandler_state', 'atcphy_mode', + 'atcphy_dp_link_rate'): + parts.append(block(atc, 'enum ' + name + ' {') + ';') +for name in ('atcphy_mode_configuration', 'atcphy_hw'): + parts.append(block(atc, 'struct ' + name + ' {') + ';') +parts.append(r''' +struct apple_atcphy { + const struct atcphy_hw *hw; + struct { unsigned char *core, *lpdptx; } regs; + void *dev; + int lock, dp_link_rate; + bool fixed_usb2, host_active, swap_lanes, ss_tunables, dp_tunables; + bool aux_power_requested, mode_dirty, dp_spread_spectrum, dp_presets_valid[5]; + u32 dp_presets[5][16]; + unsigned int dp_idle_pmas, dp_owned_pmas; + enum atcphy_mode mode, typec_mode; + enum atcphy_pipehandler_state pipe_state; +}; +''') +start = atc.index('static const struct {\n\tconst struct atcphy_mode_configuration normal;') +parts.append(atc[start:atc.index('\n};', start) + 3]) +parts.append(r''' +struct phy_configure_opts_dp { + unsigned int link_rate, lanes, voltage[4], pre[4]; + bool set_rate, set_lanes, set_voltages, ssc; +}; +static u32 registers[0x30000 / 4]; +static unsigned int polls, fail_poll, writes, usb2_on; +static u32 readl(void *address) { return *(u32 *)address; } +static void core_mask32(struct apple_atcphy *p, u32 offset, u32 mask, u32 value) +{ + (void)p; + assert(offset < sizeof(registers) && offset % 4 == 0); + registers[offset / 4] = (registers[offset / 4] & ~mask) | value; + writes++; + if (offset == ATCPHY_POWER_CTRL) + registers[ATCPHY_POWER_STAT / 4] = registers[offset / 4]; + if (offset == 0x2000) { + if (value == 0x10000001) + registers[offset / 4] |= BIT(1); + else { + registers[offset / 4] &= ~BIT(1); + registers[0x7034 / 4] &= ~BIT(3); + } + } +} +static void core_set32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, 0, v); } +static void core_clear32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, v, 0); } +static void writel(u32 value, void *address) { *(u32 *)address = value; } +static void mask32(void *p, u32 m, u32 v) { * (u32 *)p = (*(u32 *)p & ~m) | v; } +static void set32(void *p, u32 v) { mask32(p, 0, v); } +static void clear32(void *p, u32 v) { mask32(p, v, 0); } +#define readl_poll_timeout_atomic(address, value, condition, interval, timeout) \ + ({ (value) = readl(address); polls++; \ + polls == fail_poll || !(condition) ? -ETIMEDOUT : 0; }) +#define readl_poll_timeout readl_poll_timeout_atomic +static void atcphy_usb2_power_on(struct apple_atcphy *p) { (void)p; usb2_on++; } +static void atcphy_apply_tunables(struct apple_atcphy *p, enum atcphy_mode mode) +{ (void)p; (void)mode; } +static void atc_tunnel_stop_t8103(struct apple_atcphy *p) { (void)p; } +static void atc_tunnel_restore(struct apple_atcphy *p) { (void)p; } +static void atcphy_park_pipehandler(struct apple_atcphy *p) +{ p->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; } +static int atcphy_configure_pipehandler(struct apple_atcphy *p, bool host) +{ (void)host; p->pipe_state = atcphy_modes[p->mode].pipehandler_state; return 0; } +''') +for name in ('atcphy_get_mode_config', 'j700_field_update', 'j700_pma_idle'): + parts.append(function(atc, name)) +parts.append(dp) +for name in ('atcphy_configure_lanes', 'j700_aux_enable', 'j700_aux_disable', + 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', + 'atcphy_power_on', 'atcphy_configure', 'atcphy_mux_set'): + parts.append(function(atc, name)) +production = '\n'.join(parts) +print('Production transition SHA256:', hashlib.sha256(production.encode()).hexdigest()) +prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint16_t u16; +#define BIT(n) (1U << (n)) +#define GENMASK(h, l) ((~0U << (l)) & (~0U >> (31 - (h)))) +#define FIELD_PREP(m, v) (((u32)(v) << __builtin_ctz(m)) & (m)) +#define FIELD_GET(m, v) (((v) & (m)) >> __builtin_ctz(m)) +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define hweight32(v) __builtin_popcount(v) +#define dev_info_once(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_warn(...) ((void)0) +static void report(void *dev, const char *fmt, ...) { (void)dev; (void)fmt; } +#define dev_err report +#define lockdep_assert_held(...) ((void)0) +#define WARN_ON_ONCE(v) assert(!(v)) +#define guard(kind) (void) +#define fallthrough __attribute__((fallthrough)) +#define ndelay(n) ((void)(n)) +#define udelay(n) ((void)(n)) +#define usleep_range(a, b) ((void)(a), (void)(b)) +enum { TYPEC_STATE_SAFE, TYPEC_STATE_USB, TYPEC_MODE_USB4, + TYPEC_DP_STATE_C, TYPEC_DP_STATE_D, TYPEC_DP_STATE_E }; +enum { USB_TYPEC_TBT_SID = 0x8087, USB_TYPEC_DP_SID = 0xff01 }; +#define EUDO_USB_MODE_MASK 3U +enum { EUDO_USB_MODE_USB2, EUDO_USB_MODE_USB3, EUDO_USB_MODE_USB4 }; +struct enter_usb_data { u32 eudo; }; +struct typec_altmode { unsigned int svid; }; +struct typec_mux_state { unsigned long mode; struct typec_altmode *alt; void *data; }; +struct typec_mux_dev { struct apple_atcphy *data; }; +#define typec_mux_get_drvdata(mux) ((mux)->data) +''' +suffix = r''' +static struct atcphy_hw hw = { + .has_t8140_dp = true, .gen = ATCPHY_GENERATION_T8122, + .aciophy_lane_mode = 0x60, .aciophy_crossbar = 0x64, +}; +static void initial(struct apple_atcphy *p, bool fixed, bool swapped) +{ + memset(registers, 0, sizeof(registers)); + registers[AUS_COMMON_DIG_RCAL1 / 4] = AUS_COMMON_DIG_RCAL1_ALL_CODES_DONE; + *p = (struct apple_atcphy) { + .hw = &hw, .regs.core = (unsigned char *)registers, + .regs.lpdptx = (unsigned char *)registers, + .fixed_usb2 = fixed, .swap_lanes = swapped, + .ss_tunables = true, .dp_tunables = true, .dp_presets_valid = { true }, + .mode = fixed ? APPLE_ATCPHY_MODE_USB2 : APPLE_ATCPHY_MODE_OFF, + .pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY, + }; + polls = fail_poll = writes = usb2_on = 0; +} +static int safe(struct apple_atcphy *p) +{ + struct typec_mux_dev mux = { p }; + struct typec_mux_state state = { .mode = TYPEC_STATE_SAFE }; + return atcphy_mux_set(&mux, &state); +} +int main(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + for (unsigned int fixed = 0; fixed < 2; fixed++) { + for (unsigned int swapped = 0; swapped < 2; swapped++) { + const enum atcphy_mode modes[] = { + APPLE_ATCPHY_MODE_DP, APPLE_ATCPHY_MODE_USB3_DP, APPLE_ATCPHY_MODE_USB3, + }; + for (unsigned int m = 0; m < ARRAY_SIZE(modes); m++) { + unsigned int aux = modes[m] != APPLE_ATCPHY_MODE_USB3 ? 4 : 0; + unsigned int idle = modes[m] == APPLE_ATCPHY_MODE_DP ? 12 : 6; + for (unsigned int fault = 3; fault <= 2 + aux + idle; fault++) { + enum atcphy_mode old = fixed ? APPLE_ATCPHY_MODE_USB2 : APPLE_ATCPHY_MODE_OFF; + initial(&p, fixed, swapped); + fail_poll = fault; + assert(atcphy_configure(&p, modes[m]) == -ETIMEDOUT); + assert(p.mode == old && p.mode_dirty); + if (fault <= 2 + aux) + assert(p.aux_power_requested && !p.dp_idle_pmas); + else + assert(p.dp_idle_pmas); + bool owned_aux = p.aux_power_requested; + unsigned int owned_idle = p.dp_idle_pmas; + polls = 0; + fail_poll = 1; + usb2_on = 0; + assert(safe(&p) == -ETIMEDOUT); + assert(p.mode == old && p.mode_dirty); + assert(p.aux_power_requested == owned_aux && p.dp_idle_pmas == owned_idle); + assert(usb2_on == 0); /* Cleanup failure must not continue into power changes. */ + polls = fail_poll = 0; + assert(safe(&p) == 0); + assert(p.mode == old && !p.mode_dirty); + assert(!p.aux_power_requested && !p.dp_idle_pmas && !p.dp_owned_pmas); + assert(usb2_on == fixed); /* Fixed-hub SAFE restores USB2, never powers it off. */ + polls = writes = 0; + assert(safe(&p) == 0 && polls == 0 && writes == 0); + cases++; + } + } + } + } + /* A failed route change must not expose the old DP mode to link training. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_DP; + p.aux_power_requested = true; + p.dp_owned_pmas = 3; + assert(atcphy_configure(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + unsigned int last_idle_poll = polls - 1; /* The final poll is RCAL, after route idle. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_DP; + p.aux_power_requested = true; + p.dp_owned_pmas = 3; + fail_poll = last_idle_poll; + assert(atcphy_configure(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + assert(p.mode == APPLE_ATCPHY_MODE_DP && p.mode_dirty && p.dp_idle_pmas); + struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 4, + }; + struct phy_configure_opts_dp training = { .set_voltages = true, .lanes = 4 }; + unsigned int writes_before = writes, idle_before = p.dp_idle_pmas; + unsigned int owned_before = p.dp_owned_pmas; + assert(j700_dp_configure(&p, &rate) == -EBUSY); + assert(j700_dp_configure(&p, &training) == -EBUSY); + assert(writes == writes_before && p.dp_idle_pmas == idle_before); + assert(p.dp_owned_pmas == owned_before && p.aux_power_requested && p.mode_dirty); + /* Explicit stop is allowed, but must leave the route cleanup obligation. */ + struct phy_configure_opts_dp stop = { .set_rate = true }; + assert(j700_dp_configure(&p, &stop) == 0); + assert(p.dp_idle_pmas == idle_before && p.mode_dirty); + fail_poll = polls = 0; + assert(safe(&p) == 0 && !p.mode_dirty && !p.dp_idle_pmas && !p.aux_power_requested); + /* If teardown itself fails, reject training without dropping live ownership. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_DP; + p.aux_power_requested = true; + p.dp_owned_pmas = 3; + fail_poll = 1; + assert(atcphy_configure(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + assert(p.mode_dirty && p.dp_owned_pmas == 3); + writes_before = writes; + assert(j700_dp_configure(&p, &rate) == -EBUSY); + assert(j700_dp_configure(&p, &training) == -EBUSY); + assert(writes == writes_before && p.dp_owned_pmas == 3 && p.aux_power_requested); + fail_poll = polls = 0; + assert(safe(&p) == 0 && !p.mode_dirty && !p.dp_owned_pmas && !p.aux_power_requested); + /* A cleanup failure also dirties an initially successful mode transition. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_DP; + p.aux_power_requested = true; + fail_poll = 1; + assert(safe(&p) == -ETIMEDOUT && p.mode_dirty && p.aux_power_requested); + fail_poll = polls = 0; + assert(safe(&p) == 0 && !p.mode_dirty && !p.aux_power_requested); + /* Existing main-link ownership survives a failed cleanup of a dirty mode. */ + initial(&p, true, false); + p.mode_dirty = p.aux_power_requested = true; + p.dp_owned_pmas = 3; + fail_poll = 1; + assert(safe(&p) == -ETIMEDOUT); + assert(p.mode_dirty && p.aux_power_requested && p.dp_owned_pmas == 3); + fail_poll = polls = 0; + assert(safe(&p) == 0 && !p.mode_dirty && !p.aux_power_requested && !p.dp_owned_pmas); + /* OFF is not committed when domain sleep fails, and SAFE retries it. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_USB2; + fail_poll = 1; + assert(safe(&p) == -ETIMEDOUT && p.mode == APPLE_ATCPHY_MODE_USB2 && p.mode_dirty); + fail_poll = polls = 0; + assert(safe(&p) == 0 && p.mode == APPLE_ATCPHY_MODE_OFF && !p.mode_dirty); + printf("PASS: %u AUX/PMA fault scenarios, failed cleanup, dirty training guard, SAFE retry, fixed USB2, OFF commit\n", cases); + return 0; +} +''' +with tempfile.TemporaryDirectory(prefix='apple-atc-unwind-') as directory: + path = Path(directory) + unit = path / 'test.c' + unit.write_text(prefix + production + suffix) + command = shlex.split(os.environ.get('CC', 'clang')) + command += ['-std=gnu11', '-Wall', '-Wextra', '-Werror', '-O1', '-g', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer'] + subprocess.run(command + [str(unit), '-o', str(path / 'test')], check=True) + subprocess.run([str(path / 'test')], check=True) diff --git a/tools/diagnostics/test-apple-dcp-readiness.py b/tools/diagnostics/test-apple-dcp-readiness.py new file mode 100644 index 0000000000000..3d6e4cb5593e0 --- /dev/null +++ b/tools/diagnostics/test-apple-dcp-readiness.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise production DCP matching/readiness and publication against host stubs.""" +import argparse +import hashlib +import os +from pathlib import Path +import shlex +import subprocess +import tempfile + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--source-root', type=Path, default=Path(__file__).resolve().parents[2]) +args = parser.parse_args() +dcp = (args.source_root / 'drivers/gpu/drm/apple/dcp.c').read_text() +drm = (args.source_root / 'drivers/gpu/drm/apple/apple_drv.c').read_text() + + +def function(source, marker): + start = source.index(marker) + opening = source.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +# This slice is the production probe's publication boundary, including its +# failure exits and prepared-before-component_add ordering, not a copied model. +probe = function(dcp, 'static int dcp_platform_probe(') +publish = probe[probe.rindex('\tif (dcp->hw.iomfb_method_profile ==\n\t DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) {'):] +production = '\n'.join(( + function(dcp, 'bool dcp_h17p_external_ready('), + function(drm, 'static bool apple_dcp_usable('), + function(drm, 'static int add_dcp_components('), + function(drm, 'static int apple_drm_init_dcp('), + 'static int publish_provider(struct platform_device *pdev, struct apple_dcp *dcp)\n' + '{\n\tstruct device *dev = &pdev->dev;\n\tint ret;\n' + publish, +)) +print('Production readiness/publication SHA256:', hashlib.sha256(production.encode()).hexdigest()) +prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint64_t u64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +#define IS_ENABLED(x) 0 +#define MAX_COPROCESSORS 4 +#define DL_FLAG_AUTOREMOVE_SUPPLIER 0 +#define DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL 2 +#define dev_warn(...) ((void)0) +#define msleep(x) ((void)(x)) +#define component_compare_of 0 +#define apple_dcp_id_tbl 0 +static int dcp_comp_ops; +struct device_node { bool available, external, phys; unsigned int index; }; +struct device { struct device_node *of_node; void *data; }; +struct platform_device { struct device dev; bool bound, registered; }; +struct apple_dcp { + bool external_prepared, quiescing, crashed; + struct { int iomfb_method_profile; } hw; +}; +struct drm_device { int unused; }; +struct drm_encoder { int unused; }; +struct apple_drm_private { struct drm_device drm; }; +struct component_match { bool includes[2]; }; +static struct device_node nodes[2]; +static struct platform_device devices[2]; +static struct apple_dcp dcps[2]; +static struct apple_drm_private priv; +static struct device master = { .data = &priv }; +static struct component_match matches; +static bool enabled, absent_external; +static unsigned int references, released, node_puts, probes[2], bind_attempts, disables; +static int preflight_error, external_probe_error, bind_result; +static void try_bind(void); +static bool dcp_h17p_external_enabled(void) { return enabled; } +static bool of_device_is_available(struct device_node *n) { return n->available; } +static bool of_device_is_compatible(struct device_node *n, const char *s) +{ + return n->external && (!strcmp(s, "apple,t8140-dcpext") || !strcmp(s, "apple,dcpext")); +} +static bool of_property_present(struct device_node *n, const char *s) +{ assert(!strcmp(s, "phys")); return n->phys; } +#define for_each_matching_node(np, table) \ + for (unsigned int node_i = 0; node_i < 2 && ((np) = &nodes[node_i]); node_i++) +#define for_each_endpoint_of_node(np, endpoint) \ + for ((endpoint) = NULL; (endpoint) != NULL; (endpoint) = NULL) +static struct device_node *of_graph_get_remote_port_parent(struct device_node *n) +{ return n; } +static void of_node_put(struct device_node *n) { (void)n; node_puts++; } +static struct platform_device *of_find_device_by_node(struct device_node *n) +{ + if (n->external && absent_external) + return NULL; + references++; + return &devices[n->index]; +} +static void *platform_get_drvdata(struct platform_device *p) { return p->dev.data; } +static void *dev_get_drvdata(struct device *d) { return d->data; } +static void put_device(struct device *d) { (void)d; released++; } +/* Deliberately provided: using this during synchronous bind is a regression. */ +#define device_is_bound(dev) (devices[(dev)->of_node->index].bound) +static void drm_of_component_match_add(struct device *d, struct component_match **m, + int compare, struct device_node *n) +{ (void)d; (void)compare; *m = &matches; matches.includes[n->index] = true; } +static void device_link_add(struct device *a, struct device *b, int flags) +{ (void)a; (void)b; (void)flags; } +static int apple_probe_per_dcp(struct device *d, struct drm_device *drm, + struct platform_device *p, int index, bool external, + struct drm_encoder **encoder, u32 *mask) +{ + (void)d; (void)drm; (void)index; (void)encoder; (void)mask; + assert(external == p->dev.of_node->external); + probes[p->dev.of_node->index]++; + return external ? external_probe_error : 0; +} +static int apple_probe_typec_ports(struct drm_device *d, struct platform_device **p, + u32 *masks, int count) +{ (void)d; (void)p; (void)masks; (void)count; return 0; } +static int dcp_start(struct platform_device *p) { (void)p; return 0; } +static int dcp_wait_ready(struct platform_device *p, u64 timeout) +{ (void)p; (void)timeout; return 0; } +static u64 get_jiffies_64(void) { return 0; } +static u64 msecs_to_jiffies(unsigned int ms) { return ms; } +static bool time_after_eq64(u64 a, u64 b) { return a >= b; } +static int dcp_comp_bind(struct device *d, void *master_data, void *data) +{ (void)d; (void)master_data; (void)data; return preflight_error; } +static void dcp_disable_typec_work(struct apple_dcp *d, bool quiesce) +{ disables++; d->quiescing = quiesce; } +static int component_add(struct device *d, const void *ops) +{ + (void)ops; + struct platform_device *p = &devices[d->of_node->index]; + assert(!p->bound); + p->registered = true; + try_bind(); /* Linux component_add may bind the master before probe returns. */ + return 0; +} +''' +suffix = r''' +static void try_bind(void) +{ + for (unsigned int i = 0; i < 2; i++) + if (matches.includes[i] && !devices[i].registered) + return; + bind_attempts++; + bind_result = apple_drm_init_dcp(&master); +} +static void initial(bool opt_in) +{ + memset(nodes, 0, sizeof(nodes)); + memset(devices, 0, sizeof(devices)); + memset(dcps, 0, sizeof(dcps)); + memset(&matches, 0, sizeof(matches)); + memset(probes, 0, sizeof(probes)); + for (unsigned int i = 0; i < 2; i++) { + nodes[i] = (struct device_node){ .available = true, .external = i == 1, .index = i }; + devices[i].dev = (struct device){ .of_node = &nodes[i], .data = &dcps[i] }; + } + dcps[1].hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + devices[0].registered = devices[0].bound = true; + enabled = opt_in; + absent_external = false; + references = released = node_puts = bind_attempts = disables = 0; + preflight_error = external_probe_error = bind_result = 0; +} +static int match_providers(void) +{ + struct component_match *match = NULL; + return add_dcp_components(&master, &match); +} +int main(void) +{ + /* Default internal display does not wait for an unsupported external DCP. */ + initial(false); + assert(match_providers() == 1 && matches.includes[0] && !matches.includes[1]); + assert(!dcp_h17p_external_ready(&nodes[1]) && references == 0); + try_bind(); + assert(bind_attempts == 1 && bind_result == 0 && probes[0] == 1 && !probes[1]); + + /* Delayed external provider remains in the match; the master must wait. */ + initial(true); + assert(match_providers() == 2 && matches.includes[1]); + try_bind(); + assert(bind_attempts == 0 && !probes[0] && !probes[1]); + /* Last provider is prepared, but driver core bound is still false here. */ + assert(publish_provider(&devices[1], &dcps[1]) == 0); + assert(dcps[1].external_prepared && !devices[1].bound); + assert(bind_attempts == 1 && bind_result == 0 && probes[0] == 1 && probes[1] == 1); + devices[1].bound = true; + + /* Permanent preflight failure cannot silently drop a requested provider. */ + initial(true); + assert(match_providers() == 2); + preflight_error = -EIO; + assert(publish_provider(&devices[1], &dcps[1]) == -EIO); + assert(!devices[1].registered && !dcps[1].external_prepared && disables == 1); + try_bind(); + assert(matches.includes[1] && bind_attempts == 0 && !probes[0]); + + /* An unavailable DT node is excluded; a matched but absent device is fatal. */ + initial(true); + nodes[1].available = false; + assert(!apple_dcp_usable(&nodes[1]) && match_providers() == 1); + initial(true); + absent_external = true; + assert(match_providers() == 2); + assert(apple_drm_init_dcp(&master) == -ENODEV && !probes[1]); + + /* Readiness rejects every unavailable provider state and releases lookup refs. */ + for (unsigned int state = 0; state < 4; state++) { + initial(true); + dcps[1].external_prepared = true; + if (state == 0) devices[1].dev.data = NULL; + if (state == 1) dcps[1].external_prepared = false; + if (state == 2) dcps[1].quiescing = true; + if (state == 3) dcps[1].crashed = true; + assert(!dcp_h17p_external_ready(&nodes[1])); + assert(references == 1 && released == 1); + assert(apple_drm_init_dcp(&master) == -ENODEV && !probes[1]); + } + initial(true); + dcps[1].external_prepared = true; + external_probe_error = -EIO; + assert(apple_drm_init_dcp(&master) == -EIO); + assert(probes[1] == 1 && node_puts == 1); + puts("PASS: opt-in matching, delayed/preflight-failed providers, synchronous publication, readiness rejection"); + return 0; +} +''' +with tempfile.TemporaryDirectory(prefix='apple-dcp-readiness-') as directory: + path = Path(directory) + unit = path / 'test.c' + unit.write_text(prefix + production + suffix) + command = shlex.split(os.environ.get('CC', 'clang')) + command += ['-std=gnu11', '-Wall', '-Wextra', '-Werror', '-O1', '-g', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer'] + subprocess.run(command + [str(unit), '-o', str(path / 'test')], check=True) + subprocess.run([str(path / 'test')], check=True) From cf6198d365e9060bc5f4516926b50e0d10600667 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:56:39 +1100 Subject: [PATCH 0932/1579] tests: publish standalone AFK admission regression harness Make the host-extracted admission/lifetime checks reproducible alongside the kernel KUnit suite, with repository-relative source selection and an optional source-root override. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + tools/diagnostics/test-apple-afk-admission.py | 118 ++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 tools/diagnostics/test-apple-afk-admission.py diff --git a/MAINTAINERS b/MAINTAINERS index e7b22a9c9b774..9e9fbe808e1bc 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2018,6 +2018,7 @@ F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs F: tools/diagnostics/apple-dptx-training-test.c F: tools/diagnostics/apple-j700-dp-test.c +F: tools/diagnostics/test-apple-afk-admission.py F: tools/diagnostics/test-apple-atc-unwind.py F: tools/diagnostics/test-apple-dcp-readiness.py F: tools/diagnostics/test-apple-j700-crossbar.py diff --git a/tools/diagnostics/test-apple-afk-admission.py b/tools/diagnostics/test-apple-afk-admission.py new file mode 100644 index 0000000000000..fc5669048bdc9 --- /dev/null +++ b/tools/diagnostics/test-apple-afk-admission.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the actual inline admission helpers against host-only kernel mocks.""" +from pathlib import Path +import subprocess +import tempfile +import argparse + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("--source-root", type=Path, + default=Path(__file__).resolve().parents[2], + help="kernel source tree (defaults to this script's tree)") +root = parser.parse_args().source_root.resolve() +def extract(file, name): + s = (root / 'drivers/gpu/drm/apple' / file).read_text() + endname = s.index(name + '(') + start = s.rfind('static inline', 0, endname) + brace = s.index('{', endname) + depth = 1 + end = brace + 1 + while depth: + depth += (s[end] == '{') - (s[end] == '}') + end += 1 + return s[start:end] +preamble = r''' +#include +#include +#include +#include +#include +#include +typedef int64_t s64; +typedef uint32_t u32; +#define MAX_PENDING_CMDS 16 +#define spin_lock_irqsave(lock, flags) ((void)(lock), (flags)=0) +#define spin_unlock_irqrestore(lock, flags) ((void)(lock), (void)(flags)) +#define WARN_ON(x) (x) +#define smp_store_release(p,v) (*(p)=(v)) +#define smp_load_acquire(p) (*(p)) +#define WRITE_ONCE(p,v) ((p)=(v)) +#define bitmap_empty(p, n) (!*(p)) +struct apple_epic_service; +struct apple_dcp_afkep { int unused; }; +struct apple_epic_service_ops { + bool reusable; + int (*prepare)(struct apple_epic_service *,const char *,const char *,s64); + void (*teardown)(struct apple_epic_service *); +}; +struct apple_epic_service { + int lock; unsigned int users; u32 channel; bool enabled, torndown; + void *cookie; const struct apple_epic_service_ops *ops; + struct apple_dcp_afkep *ep; unsigned long cmd_map[1]; +}; +struct dptx_port { + bool enabled, connected; int enable_completion; u32 unit; + struct apple_epic_service *service; +}; +static void reinit_completion(int *p) { *p = 0; } +''' +functions = [('afk.h', n) for n in ['afk_service_get','afk_service_put','afk_service_matches','afk_service_reclaimable','afk_service_reinit','afk_service_discard','afk_service_publish']] +functions += [('dptx-service.h', n) for n in ['dptx_service_port','dptx_service_available','dptx_service_bind','dptx_service_publish','dptx_service_unbind']] +body = r''' +static int prepare(struct apple_epic_service *s,const char *n,const char *c,s64 u) { return 0; } +int main(void) { + const char *n="dcpdptx-port-epic", *c="AppleDCPDPTXRemotePort"; + assert(dptx_service_port(n,c,3,true,3,1)==0); + assert(dptx_service_port(n,c,0,true,3,1)==-EINVAL); + assert(dptx_service_port(n,c,-1,true,3,1)==-EINVAL); + assert(dptx_service_port(n,c,3,true,3,2)==-EINVAL); + assert(dptx_service_port(NULL,c,3,true,3,1)==-EINVAL); + assert(dptx_service_port(n,NULL,3,true,3,1)==-EINVAL); + assert(dptx_service_port("bad",c,3,true,3,1)==-EINVAL); + assert(dptx_service_port(n,"bad",3,true,3,1)==-EINVAL); + assert(dptx_service_port(n,c,1,false,0,1)==1); + struct apple_epic_service_ops ops={.prepare=prepare,.teardown=dptx_service_unbind}; + struct apple_epic_service_ops legacy={0}; + struct apple_epic_service a={0}, b={0}, old={0}; struct dptx_port p={0}; + assert(afk_service_reinit(&old,NULL,&legacy,8)); + assert(afk_service_matches(&old,8)); + assert(afk_service_reinit(&a,NULL,&ops,1)); + assert(!afk_service_get(&a) && !afk_service_matches(&a,1)); + afk_service_discard(&a); + assert(afk_service_reinit(&a,NULL,&ops,1)); + assert(dptx_service_bind(&a,&p,3)==0); + assert(!dptx_service_available(&p) && !p.service && a.cookie==&p); + assert(!afk_service_get(&a)); + afk_service_discard(&a); /* failed OPEN */ + assert(!p.enabled && !p.service); + assert(afk_service_reinit(&a,NULL,&ops,1)); + assert(dptx_service_bind(&a,&p,3)==0); + afk_service_publish(&a); + assert(!p.enabled && !p.service); + dptx_service_publish(&a); + assert(dptx_service_available(&p) && p.service==&a && p.service->cookie==&p); + assert(afk_service_get(&a)==&a); afk_service_put(&a); + assert(afk_service_reinit(&b,NULL,&ops,2)); + assert(dptx_service_bind(&b,&p,3)==-EBUSY); + afk_service_discard(&b); assert(p.enabled && p.service==&a); + p.connected=true; + a.enabled=false; a.torndown=true; dptx_service_unbind(&a); + assert(!p.enabled && !p.connected && a.cookie==&p); + assert(!afk_service_reinit(&a,NULL,&ops,3)); + assert(afk_service_reinit(&b,NULL,&ops,3)); + assert(dptx_service_bind(&b,&p,3)==-EBUSY); + afk_service_discard(&b); + assert(p.service==&a && !afk_service_get(p.service) && !p.enabled); + assert(afk_service_reinit(&b,NULL,&ops,4)); + afk_service_publish(&b); assert(!b.cookie && afk_service_get(&b)==&b); + afk_service_put(&b); + return 0; +} +''' +with tempfile.TemporaryDirectory() as d: + source=Path(d)/'test.c' + source.write_text(preamble+'\n'.join(extract(*x) for x in functions)+body) + subprocess.run(['cc','-std=gnu11','-Wall','-Werror',str(source),'-o',str(Path(d)/'test')],check=True) + subprocess.run([str(Path(d)/'test')],check=True) +print('PASS: host-extracted admission/identity/lifetime assertions (kernel mocks, no target kernel build)') From 35f4ccf26053b10c3f9750a218b9328239bd8eec Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:59:19 +1100 Subject: [PATCH 0933/1579] drm/apple: add bounded external stop qualification path Pause new presentation work while callbacks and control RPCs remain available; settle the active queue, serialize external lifecycle operations and retain resources on failure. Preserve first-start preparation and generation-aware CRTC recovery. OFF replies are captured but deliberately return EOPNOTSUPP until their contract is qualified; no successful production teardown or DMA retirement is claimed. Host tests exercise admission, drain, exact response bounds, timeout ownership and lifecycle ordering. A hypothetical qualified-OFF adapter exists only inside the host test. Astra reviewed this revision for one initialized DPMS-OFF capture followed by reboot, excluding unplug/suspend/restart qualification. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/gpu/drm/apple/apple_drv.c | 3 + drivers/gpu/drm/apple/dcp-internal.h | 25 + drivers/gpu/drm/apple/dcp.c | 281 +++++++++- drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/iomfb.c | 64 ++- drivers/gpu/drm/apple/iomfb_h17p.c | 99 ++++ drivers/gpu/drm/apple/iomfb_template.c | 3 + .../test-apple-external-lifecycle.py | 529 ++++++++++++++++++ 9 files changed, 999 insertions(+), 7 deletions(-) create mode 100644 tools/diagnostics/test-apple-external-lifecycle.py diff --git a/MAINTAINERS b/MAINTAINERS index 9e9fbe808e1bc..508097a2e7db3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2021,6 +2021,7 @@ F: tools/diagnostics/apple-j700-dp-test.c F: tools/diagnostics/test-apple-afk-admission.py F: tools/diagnostics/test-apple-atc-unwind.py F: tools/diagnostics/test-apple-dcp-readiness.py +F: tools/diagnostics/test-apple-external-lifecycle.py F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-d134.py F: tools/diagnostics/test-apple-j700-dp.py diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index d0462690c25c2..9a98818391aa2 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -152,6 +152,9 @@ static void apple_crtc_atomic_enable(struct drm_crtc *crtc, dcp_poweron(apple_crtc->dcp); /* Force the CTM to be set on first swap */ crtc_state->color_mgmt_changed = true; + } else if (crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state)) { + /* A cable generation can change while the CRTC stays active. */ + dcp_external_recover(to_apple_crtc(crtc)->dcp); } if (crtc_state->active) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 48f4112151dd3..685c985680ba0 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -46,6 +46,7 @@ struct iomfb_transaction { bool backlight_reserved; bool backlight_failed; bool brightness_only; + bool presentation; bool completed; }; struct apple_dcp_typec_port; @@ -213,6 +214,25 @@ struct apple_dcp { struct apple_dcp_hw_data hw; bool external_prepared; + /* Serializes external connection, power, and CRTC generation changes. */ + struct mutex external_lock; + u64 external_generation; + u64 external_crtc_generation; + /* iomfb.lock: epoch used by receive-side deferred stop requests. */ + u64 external_rx_generation; + u64 external_stop_generation; + bool external_cable; + bool external_running; + bool external_power_attempted; + /* iomfb.lock: replay the latest callback after activation succeeds. */ + bool external_hotplug_pending; + bool external_hotplug_connected; + bool external_stopped; + int external_error; + u64 external_power_reply; + bool external_power_reply_valid; + struct work_struct external_stop_work; + /* firmware version and compatible firmware version */ enum dcp_firmware_version fw_compat; @@ -445,6 +465,7 @@ struct apple_dcp { unsigned long deadline; unsigned int queued; bool stopped; + bool presentation_paused; bool backlight_queued; atomic_t opaque_x_state; struct iomfb_scanout_h17p *scanout; @@ -469,11 +490,15 @@ int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout); +int iomfb_queue_settle(struct apple_dcp *dcp, unsigned long timeout); +void iomfb_pause_presentation(struct apple_dcp *dcp, bool pause); +int iomfb_external_power_h17p(struct apple_dcp *dcp, bool on); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id); void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); +bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected); int dcp_backlight_init(struct apple_dcp *dcp, struct device *dev); void dcp_backlight_stop(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4f2b4923135af..da176a910f46b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -301,6 +301,14 @@ static bool dcp_typec_route_fits(struct apple_dcp_typec_route *route, return connector->candidate_crtcs & drm_crtc_mask(&dcp->crtc->base); } +static bool dcp_is_external_h17p(struct apple_dcp *dcp) +{ + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; +} + +static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation); +static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical); + static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); @@ -349,7 +357,9 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * can still be marked connected to a display that is gone. Release it * (a no-op otherwise), or connecting the borrowed route returns early. */ - dcp_dptx_disconnect(dcp, 0); + ret = dcp_dptx_disconnect(dcp, 0); + if (ret) + return ret; if (dcp->fixed_route_selected) { ret = mux_control_deselect(dcp->xbar); @@ -439,6 +449,11 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) bool was_tunnel = route->tunnel; int ret = 0; + guard(mutex)(&dcp->external_lock); + if (dcp_is_external_h17p(dcp) && + (dcp->external_error || (dcp->external_power_attempted && !dcp->external_stopped))) + return dcp->external_error ?: -EBUSY; + /* * Under tb_lock so a DCP link (re)configuration can neither select the * crossbar nor restart the tunnel pixel clock behind our back. @@ -1557,7 +1572,8 @@ static void dcp_typec_route_unregister(void *data) if (port->hpd || dcp->typec_cable_connected) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); port->hpd = false; - dcp_typec_route_deactivate(route); + if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) + return; port->owner = NULL; } if (port->secondary_owner == route) { @@ -1565,7 +1581,8 @@ static void dcp_typec_route_unregister(void *data) if (port->hpd || dcp->typec_cable_connected) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - dcp_typec_route_deactivate(route); + if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) + return; port->secondary_owner = NULL; } port->hpd = !!(port->owner || port->secondary_owner); @@ -2171,7 +2188,7 @@ bool dcp_has_typec_routes(struct platform_device *pdev) #define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) #define DPTX_RECONNECT_RETRIES 5 -static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) +static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port) { unsigned long timeout; int ret = 0; @@ -2280,6 +2297,186 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) return ret; } +static void dcp_external_report_disconnected(struct apple_dcp *dcp) +{ + unsigned int action; + + guard(mutex)(&dcp->iomfb.lock); + action = dcp_mode_hotplug(&dcp->mode_state, false, + dcp->connector ? &dcp->connector->connected : NULL); + /* The existing worker delivers DRM notifications outside our locks. */ + dcp_handle_hotplug_actions(dcp, action); +} + +static void dcp_external_publish_generation(struct apple_dcp *dcp) +{ + lockdep_assert_held(&dcp->external_lock); + guard(mutex)(&dcp->iomfb.lock); + dcp->external_rx_generation = dcp->external_generation; +} + +/* external_lock is never taken by the serialized IOMFB receive callbacks. */ +static int dcp_external_link_locked(struct apple_dcp *dcp) +{ + int ret; + + lockdep_assert_held(&dcp->external_lock); + if (!dcp->external_prepared) + return -EAGAIN; + if (dcp->external_error) + return dcp->external_error; + if (!dcp->external_cable) + return -ENODEV; + ret = dcp_dptx_connect_raw(dcp, 0); + if (ret) + dcp->external_error = ret; + return ret; +} + +static int dcp_external_start_locked(struct apple_dcp *dcp) +{ + int ret; + + lockdep_assert_held(&dcp->external_lock); + ret = dcp_external_link_locked(dcp); + if (ret || dcp->external_running) + return ret; + iomfb_pause_presentation(dcp, true); + /* Even a timed-out first activation can have changed firmware state. */ + dcp->external_power_attempted = true; + ret = iomfb_external_power_h17p(dcp, true); + if (ret) { + dcp->external_error = ret; + return ret; + } + dcp->external_running = true; + dcp->external_stopped = false; + WRITE_ONCE(dcp->pipe_enabled_h17p, true); + WRITE_ONCE(dcp->typec_crtc_off, false); + iomfb_pause_presentation(dcp, false); + return 0; +} + +static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) +{ + if (!dcp_is_external_h17p(dcp)) + return dcp_dptx_connect_raw(dcp, port); + guard(mutex)(&dcp->external_lock); + return dcp_external_link_locked(dcp); +} + +/* The generation token belongs to the CRTC that was actually enabled. */ +static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) +{ + int ret; + + lockdep_assert_held(&dcp->external_lock); + if (generation != dcp->external_generation) + return 0; + if (dcp->external_error) + return dcp->external_error; + if (dcp->external_stopped) + return 0; + iomfb_pause_presentation(dcp, true); + dcp->external_running = false; + /* Route setup can call this before IOMFB or its shared memory exists. */ + if (!dcp->external_power_attempted) + goto release_link; + ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); + if (!ret) + ret = iomfb_external_power_h17p(dcp, false); + if (!ret) + ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); + if (ret) + goto failed; + /* This is a transport acknowledgment, not permission to free scanout. */ + WRITE_ONCE(dcp->pipe_enabled_h17p, false); +release_link: + scoped_guard(mutex, &dcp->hpd_mutex) { + if (dcp->dptxport[0].connected) { + if (!dptx_service_available(&dcp->dptxport[0])) { + ret = -ENODEV; + goto failed; + } + ret = dptxport_set_hpd(dcp->dptxport[0].service, false); + if (!ret) + ret = dptxport_release_display(dcp->dptxport[0].service); + if (ret) + goto failed; + dcp->dptxport[0].connected = false; + } + } + dcp->external_stopped = true; + WRITE_ONCE(dcp->typec_crtc_off, true); + if (dcp->avep) + av_service_disconnect(dcp); + return 0; +failed: + /* A retry could race a late reply. Keep this session blocked and owned. */ + dcp->external_error = ret; + return ret; +} + +static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) +{ + int ret; + + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->external_lock); + ret = dcp_external_stop_locked(dcp, dcp->external_generation); + if (physical) { + dcp->external_cable = false; + dcp->external_generation++; + dcp_external_publish_generation(dcp); + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, false); + dcp->typec_generation++; + } + /* A missing cable is reported even when its hardware remains owned. */ + dcp_external_report_disconnected(dcp); + } + return ret; +} + +static void dcp_external_stop_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, external_stop_work); + + u64 generation; + + scoped_guard(mutex, &dcp->iomfb.lock) + generation = dcp->external_stop_generation; + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->external_lock); + if (generation == dcp->external_generation) { + dcp_external_stop_locked(dcp, generation); + dcp_external_report_disconnected(dcp); + } +} + +bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) +{ + if (!dcp_is_external_h17p(dcp)) + return false; + /* No lifecycle or HPD mutex may be acquired from the RX callback. */ + lockdep_assert_held(&dcp->iomfb.lock); + if (READ_ONCE(dcp->quiescing)) + return true; + if (dcp->iomfb.presentation_paused) { + dcp->external_hotplug_pending = true; + dcp->external_hotplug_connected = connected; + return true; + } + if (!connected && !READ_ONCE(dcp->iomfb.presentation_paused)) { + dcp->external_stop_generation = dcp->external_rx_generation; + schedule_work(&dcp->external_stop_work); + return true; + } + return READ_ONCE(dcp->iomfb.presentation_paused); +} + static bool dcp_edid_is_placeholder(const struct drm_edid *drm_edid) { const u8 *raw = (const u8 *)drm_edid_raw(drm_edid); @@ -2397,6 +2594,9 @@ static void disconnected_hpd_event(struct apple_connector *con) static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) { + if (dcp_is_external_h17p(dcp)) + return dcp_external_disconnect(dcp, false); + dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); mutex_lock(&dcp->hpd_mutex); @@ -2414,6 +2614,23 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + if (dcp_is_external_h17p(dcp)) { + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + cancel_work_sync(&dcp->external_stop_work); + guard(mutex)(&dcp->external_lock); + if (!dcp->external_cable) + dcp->external_generation++; + dcp->external_cable = true; + dcp_external_publish_generation(dcp); + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, true); + dcp->typec_generation++; + dcp->placeholder_retried = false; + } + return dcp_external_link_locked(dcp); + } + if (dcp_is_typec_output(dcp)) { cancel_delayed_work_sync(&dcp->placeholder_edid_wq); guard(mutex)(&dcp->hpd_mutex); @@ -2436,6 +2653,9 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (dcp_is_external_h17p(dcp)) + return dcp_external_disconnect(dcp, true); + if (dcp_is_typec_output(dcp)) { scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, false); @@ -2740,6 +2960,19 @@ void dcp_poweron(struct platform_device *pdev) unsigned long remaining; int ret; + if (dcp_is_external_h17p(dcp)) { + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + cancel_work_sync(&dcp->external_stop_work); + guard(mutex)(&dcp->external_lock); + ret = dcp_external_start_locked(dcp); + if (!ret) + dcp->external_crtc_generation = dcp->external_generation; + else + dev_err(dcp->dev, "external display activation failed: %d\n", ret); + return; + } + if (dcp_uses_soft_dpms(dcp)) { if (dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && @@ -2814,11 +3047,30 @@ void dcp_poweron(struct platform_device *pdev) av_service_connect(dcp); } +/* Same-CRTC helper dependencies order this token refresh before later disables. */ +void dcp_external_recover(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp_is_external_h17p(dcp)) + dcp_poweron(pdev); +} + void dcp_poweroff(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + if (dcp_is_external_h17p(dcp)) { + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->external_lock); + ret = dcp_external_stop_locked(dcp, dcp->external_crtc_generation); + if (ret) + dev_err(dcp->dev, "external display stop failed: %d\n", ret); + return; + } + cancel_delayed_work(&dcp->swap_watchdog_wq); /* Internal H17P DPMS is a brightness present, never a pipe stop. */ @@ -3368,6 +3620,7 @@ static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); disable_delayed_work_sync(&dcp->placeholder_edid_wq); + cancel_work_sync(&dcp->external_stop_work); } static void dcp_enable_typec_work(struct apple_dcp *dcp) @@ -3603,7 +3856,9 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->retain_dma) { /* No firmware stop is acknowledged: detach host users, never DMA. */ - WRITE_ONCE(dcp->quiescing, true); + scoped_guard(mutex, &dcp->iomfb.lock) + WRITE_ONCE(dcp->quiescing, true); + cancel_work_sync(&dcp->external_stop_work); if (dcp->crtc && !dcp->drm_retained) { drm_dev_get(dcp->crtc->base.dev); dcp->drm_retained = true; @@ -3749,6 +4004,8 @@ static int dcp_platform_probe(struct platform_device *pdev) * registered below, before the DRM device binds. */ mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->external_lock); + INIT_WORK(&dcp->external_stop_work, dcp_external_stop_work); mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->dcpavserv.lock); @@ -3761,6 +4018,7 @@ static int dcp_platform_probe(struct platform_device *pdev) experimental_h17p_external_bw_request)) return dev_err_probe(dev, -EINVAL, "External bandwidth offsets are unset or invalid\n"); + iomfb_pause_presentation(dcp, true); dcp->hw.firmware_scratch = experimental_h17p_external_bw_scratch; dcp->hw.firmware_request = experimental_h17p_external_bw_request; } @@ -4026,6 +4284,19 @@ static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + if (dcp_is_external_h17p(dcp)) { + int ret; + + dcp_disable_typec_work(dcp, false); + ret = dcp_external_disconnect(dcp, false); + if (ret) { + dcp_enable_typec_work(dcp); + return ret; + } + device_set_wakeup_path(dev); + return 0; + } + dcp_drain_for_sleep(dcp); /* * The Type-C route reports cable removal through diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 600899c71546b..c2b02a5309eab 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -34,6 +34,7 @@ struct apple_encoder { void dcp_poweroff(struct platform_device *pdev); void dcp_poweron(struct platform_device *pdev); +void dcp_external_recover(struct platform_device *pdev); int dcp_set_crc(struct drm_crtc *crtc, bool enabled); int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state); int dcp_get_connector_type(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 786eb53383329..cc0989199b714 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -430,6 +430,45 @@ bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout) } } +/* Admission is independent of transport: replies and control RPCs remain live. */ +void iomfb_pause_presentation(struct apple_dcp *dcp, bool pause) +{ + guard(mutex)(&dcp->iomfb.lock); + dcp->iomfb.presentation_paused = pause; + if (!pause && dcp->external_hotplug_pending) { + unsigned int action; + + dcp->external_hotplug_pending = false; + action = dcp_mode_hotplug(&dcp->mode_state, dcp->external_hotplug_connected, + dcp->connector ? &dcp->connector->connected : NULL); + dcp_handle_hotplug_actions(dcp, action); + } +} + +/* A dead transport is not proof that DMA or a pending present has stopped. */ +int iomfb_queue_settle(struct apple_dcp *dcp, unsigned long timeout) +{ + unsigned long deadline = jiffies + timeout; + bool idle; + + for (;;) { + mutex_lock(&dcp->iomfb.lock); + if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { + mutex_unlock(&dcp->iomfb.lock); + return -EIO; + } + idle = !dcp->iomfb.active && !dcp->iomfb.queued && + list_empty(&dcp->iomfb.pending) && + !dcp->present_state_h17p.pending && iomfb_channels_idle(dcp); + mutex_unlock(&dcp->iomfb.lock); + if (idle) + return 0; + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; + usleep_range(2000, 4000); + } +} + int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) { int ret = 0; @@ -437,6 +476,8 @@ int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) mutex_lock(&dcp->iomfb.lock); if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { ret = -EIO; + } else if (transaction->presentation && dcp->iomfb.presentation_paused) { + ret = -EAGAIN; } else if (transaction->brightness_only && dcp->iomfb.backlight_queued) { ret = -EALREADY; } else if (dcp->iomfb.queued >= 32) { @@ -506,7 +547,7 @@ void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id) lockdep_assert_held(&dcp->iomfb.lock); if (!READ_ONCE(dcp->crc_enabled) || READ_ONCE(dcp->crashed) || - dcp->iomfb.stopped || dcp->iomfb.queued >= 32) + dcp->iomfb.stopped || dcp->iomfb.presentation_paused || dcp->iomfb.queued >= 32) return; crc = kzalloc_obj(*crc); if (!crc) @@ -1119,6 +1160,13 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, int ret = -EIO; bool modeset; + /* Disconnect cannot pass the admission check before these RPCs finish. */ + guard(mutex)(&dcp->external_lock); + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + (!dcp->external_running || dcp->external_error || READ_ONCE(dcp->quiescing) || + READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.presentation_paused))) + return -EPIPE; + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); if (!crtc_state) return 0; @@ -1374,6 +1422,7 @@ static void iomfb_backlight_kick(struct apple_dcp *dcp) transaction->start = iomfb_backlight_start; transaction->release = iomfb_backlight_release; transaction->brightness_only = true; + transaction->presentation = true; if (iomfb_queue(dcp, transaction)) kfree(transaction); } @@ -1434,18 +1483,29 @@ static void iomfb_queue_atomic(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_state *state) { struct iomfb_atomic_transaction *atomic; + int ret; + if (READ_ONCE(dcp->iomfb.presentation_paused)) { + schedule_work(&dcp->vblank_wq); + return; + } atomic = kzalloc_obj(*atomic); if (!atomic) goto failed; atomic->transaction.start = iomfb_atomic_start; + atomic->transaction.presentation = true; atomic->transaction.release = iomfb_atomic_release; atomic->state = drm_atomic_state_get(state); atomic->crtc = crtc; atomic->dcp = dcp; - if (!iomfb_queue(dcp, &atomic->transaction)) + ret = iomfb_queue(dcp, &atomic->transaction); + if (!ret) return; iomfb_atomic_release(&atomic->transaction); + if (ret == -EAGAIN) { + schedule_work(&dcp->vblank_wq); + return; + } failed: WRITE_ONCE(dcp->crashed, true); schedule_work(&dcp->vblank_wq); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 49fcabaeb7c66..b3ae896426e25 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -718,6 +718,105 @@ bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, return true; } +static_assert(sizeof(struct dcp_set_power_state_resp) == 8); + +struct dcp_external_power_cookie { + struct dcp_wait_cookie wait; + bool on; +}; + +/* A473's envelope is known; no OFF reply has been qualified as a stop. */ +static void iomfb_external_power_complete(struct apple_dcp *dcp, void *out, void *data) +{ + struct dcp_external_power_cookie *cookie = data; + const struct dcp_packet_header *header; + int ret = -EPROTO; + + if (out) { + header = out - sizeof(struct dcp_set_power_state_req_h17p) - sizeof(*header); + if (header->in_len == sizeof(struct dcp_set_power_state_req_h17p) && + header->out_len == sizeof(struct dcp_set_power_state_resp)) { + u64 reply = get_unaligned_le64(out); + + WRITE_ONCE(dcp->external_power_reply, reply); + WRITE_ONCE(dcp->external_power_reply_valid, true); + dev_info(dcp->dev, "A473 power=%u reply=%#llx%s\n", + cookie->on, reply, cookie->on ? "" : " (stop unqualified)"); + ret = cookie->on ? (reply == 1 ? 0 : -EREMOTEIO) : -EOPNOTSUPP; + } + } + WRITE_ONCE(cookie->wait.status, ret); + complete(&cookie->wait.done); + kref_put(&cookie->wait.refcount, release_wait_cookie); +} + +static bool iomfb_external_reply_size(void *out, u32 in_len, u32 out_len) +{ + const struct dcp_packet_header *header; + + if (!out) + return false; + header = out - in_len - sizeof(*header); + return header->in_len == in_len && header->out_len == out_len; +} + +static void iomfb_external_parameter_complete(struct apple_dcp *dcp, void *out, void *data) +{ + struct dcp_set_power_state_req_h17p request = { .unklong = 1 }; + + if (!iomfb_external_reply_size(out, sizeof(struct dcp_set_parameter_dcp), sizeof(u32))) { + iomfb_external_power_complete(dcp, NULL, data); + return; + } + dcp_set_power_state(dcp, true, &request, iomfb_external_power_complete, data); +} + +static void iomfb_external_device_complete(struct apple_dcp *dcp, void *out, void *data) +{ + struct dcp_set_parameter_dcp parameter = { + .param = IOMFBPARAM_ADAPTIVE_SYNC, + .count = 3, + }; + + if (!iomfb_external_reply_size(out, sizeof(u32), sizeof(u32))) { + iomfb_external_power_complete(dcp, NULL, data); + return; + } + /* Preserve the existing external power-on preparation and payload semantics. */ + dcp_set_parameter_dcp(dcp, false, ¶meter, iomfb_external_parameter_complete, data); +} + +int iomfb_external_power_h17p(struct apple_dcp *dcp, bool on) +{ + struct dcp_set_power_state_req_h17p request = { .unklong = on }; + struct dcp_external_power_cookie *cookie; + int ret; + + if (READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.stopped)) + return -EIO; + cookie = kzalloc_obj(*cookie); + if (!cookie) + return -ENOMEM; + cookie->on = on; + WRITE_ONCE(dcp->external_power_reply_valid, false); + cookie->wait.status = -ETIMEDOUT; + init_completion(&cookie->wait.done); + kref_init(&cookie->wait.refcount); + kref_get(&cookie->wait.refcount); + if (on) { + u32 handle = 2; + + iomfb_opaque_x_reset_h17p(dcp); + dcp_set_display_device(dcp, false, &handle, iomfb_external_device_complete, cookie); + } else { + dcp_set_power_state(dcp, true, &request, iomfb_external_power_complete, cookie); + } + ret = wait_for_completion_timeout(&cookie->wait.done, msecs_to_jiffies(500)); + ret = ret ? (int)READ_ONCE(cookie->wait.status) : -ETIMEDOUT; + kref_put(&cookie->wait.refcount, release_wait_cookie); + return ret; +} + void iomfb_start_h17p(struct apple_dcp *dcp) { WRITE_ONCE(dcp->pipe_enabled_h17p, false); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 575b99606e1ac..dd57ffaaafe09 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1860,6 +1860,9 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) if (dcp_is_typec_output(dcp) && *connected && dcp->nr_modes) complete_all(&dcp->typec_iomfb_hpd_ready); + if (dcp_external_hotplug(dcp, !!(*connected))) + return; + action = dcp_mode_hotplug(&dcp->mode_state, !!(*connected), connector ? &connector->connected : NULL); /* diff --git a/tools/diagnostics/test-apple-external-lifecycle.py b/tools/diagnostics/test-apple-external-lifecycle.py new file mode 100644 index 0000000000000..35649c07c3412 --- /dev/null +++ b/tools/diagnostics/test-apple-external-lifecycle.py @@ -0,0 +1,529 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Host-only interleavings through production external lifecycle and queue code.""" +import hashlib +import os +from pathlib import Path +import shlex +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +base = root / 'drivers/gpu/drm/apple' +dcp = (base / 'dcp.c').read_text() +iomfb = (base / 'iomfb.c').read_text() +h17p = (base / 'iomfb_h17p.c').read_text() +drm = (base / 'apple_drv.c').read_text() + + +def block(source, marker): + start = source.index(marker) + begin = source.index('{', start) + end, depth = begin + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def function(source, name): + import re + marker = re.search(r'^(?:static )?(?:bool|void|int) ' + name + r'\([^;{]*\)\n\{', source, re.M) + assert marker, name + return block(source, marker.group()[:-1]) + + +def external_branch(name, result): + body = function(dcp, name) + branch = block(body, 'if (dcp_is_external_h17p(dcp)) {') + return body[:body.index('if (dcp_is_external_h17p(dcp)) {')] + branch + result + '\n}' + + +parts = [function(iomfb, 'dcp_handle_hotplug_actions')] +for name in ('iomfb_channels_idle', 'iomfb_pause_presentation', 'iomfb_queue_settle', 'iomfb_queue'): + parts.append(function(iomfb, name)) +parts.append(block(h17p, 'struct dcp_external_power_cookie {') + ';') +for name in ('iomfb_external_power_complete', 'iomfb_external_reply_size', + 'iomfb_external_parameter_complete', 'iomfb_external_device_complete'): + parts.append(function(h17p, name)) +parts.append(function(h17p, 'iomfb_external_power_h17p').replace( + 'int iomfb_external_power_h17p(', 'static int test_wire_power(')) +# Model a future qualified OFF policy only in this harness. The production +# decoder is also tested directly and currently MUST reject every OFF reply. +parts.append(r''' +static int iomfb_external_power_h17p(struct apple_dcp *p, bool on) +{ + int ret = test_wire_power(p, on); + return test_qualified_off && !on && ret == -EOPNOTSUPP && !p->external_power_reply ? 0 : ret; +} +''') +for name in ('dcp_is_external_h17p', 'dcp_external_report_disconnected', + 'dcp_external_publish_generation', 'dcp_external_link_locked', + 'dcp_external_start_locked', 'dcp_external_stop_locked', + 'dcp_external_disconnect', 'dcp_external_stop_work', 'dcp_external_hotplug'): + parts.append(function(dcp, name)) +parts += [external_branch('dcp_dptx_connect_oob', '\nreturn -ENOSYS;'), + external_branch('dcp_poweron', ''), external_branch('dcp_poweroff', ''), + function(dcp, 'dcp_external_recover')] +# Include real admission call sites; only their allocation/start/release dependencies are mocked. +parts += [function(iomfb, 'iomfb_queue_atomic'), function(iomfb, 'iomfb_backlight_kick'), + function(iomfb, 'iomfb_queue_crc_h17p')] +parts += [function(iomfb, 'dcp_crtc_atomic_modeset')] +parts += [function(drm, 'apple_crtc_atomic_enable'), function(drm, 'apple_crtc_atomic_disable')] +route_admission = function(dcp, 'dcp_typec_route_deactivate') +route_admission = route_admission[:route_admission.index('\n\t/*')] + '\n\treturn 0;\n}' +route_admission = route_admission.replace('dcp_typec_route_deactivate(', 'test_route_deactivate_admission(') +parts.append(route_admission) +production = '\n'.join(parts) +print('Production lifecycle SHA256:', hashlib.sha256(production.encode()).hexdigest()) +prefix = r''' +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint64_t u64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +#define msecs_to_jiffies(x) (x) +#define time_after_eq(a, b) ((a) >= (b)) +#define DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL 2 +#define dev_err(...) ((void)0) +#define dev_info(...) ((void)0) +#define WARN_ONCE(...) ((void)0) +enum { DCP_FIRMWARE_V_12_3, DCP_FIRMWARE_V_13_5, DCP_FIRMWARE_H17P }; +#define IOMFBPARAM_ADAPTIVE_SYNC 1 +struct dcp_set_parameter_dcp { u32 param, value[8], count; }; +struct mutex { bool held; }; +static void mutex_lock(struct mutex *m) { assert(!m->held); m->held = true; } +static void mutex_unlock(struct mutex *m) { assert(m->held); m->held = false; } +static struct mutex *acquire(struct mutex *m) { mutex_lock(m); return m; } +static void release(struct mutex **m) { if (*m) mutex_unlock(*m); } +static struct mutex *once(struct mutex *m) { mutex_unlock(m); return NULL; } +#define JOIN2(a,b) a##b +#define JOIN(a,b) JOIN2(a,b) +#define guard(kind) struct mutex *JOIN(guard_, __COUNTER__) __attribute__((cleanup(release))) = acquire +#define scoped_guard(kind, value) for (struct mutex *g __attribute__((cleanup(release))) = acquire(value); g; g = once(g)) +#define lockdep_assert_held(m) assert((m)->held) +#define container_of(p, type, member) ((type *)((char *)(p) - offsetof(type, member))) +struct list_head { unsigned int count; }; +static bool list_empty(struct list_head *h) { return !h->count; } +static void list_add_tail(struct list_head *n, struct list_head *h) { (void)n; h->count++; } +#define list_add list_add_tail +struct work_struct { unsigned int scheduled; }; +struct delayed_work { unsigned int cancelled; }; +struct completion { bool done; }; +struct kref { unsigned int count; }; +struct dcp_wait_cookie { struct kref refcount; struct completion done; u32 status; }; +struct dcp_packet_header { char tag[4]; u32 in_len, out_len; }; +struct dcp_set_power_state_req_h17p { u64 unklong; u32 unkint; unsigned char null, padding[3]; } __attribute__((packed)); +struct dcp_set_power_state_resp { u32 unknown, ret; } __attribute__((packed)); +struct device { int unused; }; +struct apple_connector { bool connected; struct work_struct hotplug_wq; }; +struct dptx_port { bool connected; void *service; }; +struct iomfb_transaction { struct list_head link; void (*start)(void *,void *), (*release)(void *); bool brightness_only, presentation; }; +struct platform_device; +struct drm_crtc_state { bool active, active_changed, mode_changed, color_mgmt_changed; void *event; struct { int hdisplay, vdisplay; } mode; }; +struct drm_device { int event_lock; }; +struct drm_crtc { struct drm_crtc_state *state; struct drm_device *dev; }; +struct drm_atomic_state { struct drm_crtc_state *next; }; +struct apple_crtc { struct drm_crtc base; struct platform_device *dcp; }; +#define to_apple_crtc(p) container_of(p, struct apple_crtc, base) +static struct drm_crtc_state *drm_atomic_get_new_crtc_state(struct drm_atomic_state *s, struct drm_crtc *c) { (void)c; return s->next; } +static bool drm_atomic_crtc_needs_modeset(struct drm_crtc_state *s) { return s->mode_changed; } +static unsigned int modesets, vblanks; +static int dcp_crtc_atomic_modeset(struct drm_crtc *, struct drm_atomic_state *); +#define spin_lock_irq(lock) ((void)(lock)) +#define spin_unlock_irq(lock) ((void)(lock)) +static void drm_crtc_send_vblank_event(struct drm_crtc *c, void *e) { (void)c; assert(e); vblanks++; } +struct apple_dcp; +struct iomfb_atomic_transaction { struct iomfb_transaction transaction; struct drm_crtc *crtc; struct drm_atomic_state *state; struct apple_dcp *dcp; }; +struct iomfb_crc_transaction { struct iomfb_transaction transaction; u32 swap_id; }; +struct apple_dcp { + struct device *dev; + struct { int iomfb_method_profile; } hw; + struct mutex external_lock, hpd_mutex; + u64 external_generation, external_crtc_generation, typec_generation, external_rx_generation, external_stop_generation, external_power_reply; + bool external_power_reply_valid, external_power_attempted, quiescing; + int fw_compat; + bool external_prepared, external_cable, external_running, external_stopped; + bool external_hotplug_pending, external_hotplug_connected; + int external_error; + bool crashed, pipe_enabled_h17p, typec_crtc_off, typec_cable_connected, placeholder_retried, crc_enabled; + struct delayed_work typec_reconnect_wq, placeholder_edid_wq; + struct work_struct external_stop_work, vblank_wq; + struct apple_connector *connector, *typec_connector; + struct dptx_port dptxport[1]; + void *avep; + struct { bool pending; } present_state_h17p; + struct { unsigned int depth; } ch_cmd, ch_cb, ch_oobcmd, ch_oobcb, ch_async, ch_oobasync; + struct { + struct mutex lock; + struct list_head pending; + struct iomfb_transaction *active; + unsigned int queued; + bool stopped, presentation_paused, backlight_queued; + struct work_struct work; + } iomfb; + struct { bool valid; } mode_state; + unsigned int retained_dma, retained_fb; +}; +struct platform_device { struct apple_dcp *data; }; +struct mux_control { int unused; }; +struct apple_dcp_typec_route { struct apple_dcp *dcp; struct mux_control *active_xbar; bool tunnel; }; +#define platform_get_drvdata(p) ((p)->data) +static struct apple_dcp *current_dcp; +static unsigned long jiffies; +static unsigned int sleeps, power0, power1, drops, releases, connects, allocations; +static int connect_error, hpd_error, release_error, wire_error; +static bool drain_callback, timeout_reply, absent_reply, test_qualified_off; +static int prep_error; +static bool on_zero_reply; +static u32 wire_in, wire_out; +static char events[256]; +static unsigned int event_count; +static void event(char c) { assert(event_count < sizeof(events) - 1); events[event_count++] = c; } +static bool dcp_external_hotplug(struct apple_dcp *, bool); +static void usleep_range(unsigned int a, unsigned int b) +{ + (void)a; (void)b; + assert(!current_dcp->iomfb.lock.held); + jiffies += 2; sleeps++; + if (drain_callback) { + mutex_lock(¤t_dcp->iomfb.lock); + assert(dcp_external_hotplug(current_dcp, false)); + current_dcp->iomfb.active = NULL; + current_dcp->iomfb.pending.count = current_dcp->iomfb.queued = 0; + current_dcp->present_state_h17p.pending = false; + current_dcp->ch_cb.depth = 0; + mutex_unlock(¤t_dcp->iomfb.lock); + drain_callback = false; + } +} +static unsigned int dcp_mode_hotplug(void *state, bool connected, bool *result) +{ (void)state; if (result) *result = connected; return 1; } +#define DCP_HOTPLUG_NOTIFY 1 +#define DCP_HOTPLUG_VBLANK 2 +static void dcp_handle_hotplug_actions(struct apple_dcp *, unsigned int); +static void schedule_work(struct work_struct *); +static void schedule_work(struct work_struct *w) +{ + w->scheduled++; + if (current_dcp && w == ¤t_dcp->connector->hotplug_wq) event('N'); +} +static void cancel_delayed_work_sync(struct delayed_work *w) +{ assert(!current_dcp->external_lock.held); w->cancelled++; } +static void cancel_work_sync(struct work_struct *w) +{ assert(!current_dcp->external_lock.held); w->scheduled = 0; } +static bool dptx_service_available(struct dptx_port *p) { return p->service != NULL; } +static int dptxport_set_hpd(void *s, bool hpd) +{ (void)s; assert(!hpd); event('H'); drops++; return hpd_error; } +static int dptxport_release_display(void *s) +{ (void)s; event('R'); releases++; return release_error; } + +static void av_service_disconnect(struct apple_dcp *p) { (void)p; } +static int dcp_dptx_connect_raw(struct apple_dcp *p, u32 port) +{ + assert(!port && p->external_lock.held); + if (p->dptxport[0].connected) return 0; + event('C'); connects++; + if (connect_error) return connect_error; + p->dptxport[0].connected = true; + mutex_lock(&p->iomfb.lock); + assert(dcp_external_hotplug(p, true)); + mutex_unlock(&p->iomfb.lock); + return 0; +} +static void kref_init(struct kref *r) { r->count = 1; } +static void kref_get(struct kref *r) { r->count++; } +static void kref_put(struct kref *r, void (*fn)(struct kref *)) { assert(r->count); if (!--r->count) fn(r); } +static void release_wait_cookie(struct kref *r) { allocations--; free(container_of(r, struct dcp_wait_cookie, refcount)); } +static void *allocate(size_t size) { allocations++; return calloc(1, size); } +#define kzalloc_obj(obj) allocate(sizeof(obj)) +static void kfree(void *p) { if (p) { allocations--; free(p); } } +static void init_completion(struct completion *c) { c->done = false; } +static void complete(struct completion *c) { c->done = true; } +static unsigned long wait_for_completion_timeout(struct completion *c, unsigned long timeout) +{ (void)timeout; assert(!current_dcp->iomfb.lock.held); return c->done; } +static u64 get_unaligned_le64(const void *p) { u64 n; memcpy(&n, p, sizeof(n)); return n; } +static struct dcp_wait_cookie *late_cookie; +static void iomfb_external_power_complete(struct apple_dcp *, void *, void *); +static void dcp_set_power_state(struct apple_dcp *p, bool oob, + struct dcp_set_power_state_req_h17p *req, + void (*cb)(struct apple_dcp *, void *, void *), void *cookie) +{ + struct { struct dcp_packet_header header; struct dcp_set_power_state_req_h17p req; struct dcp_set_power_state_resp resp; } packet = {0}; + assert(oob && p->iomfb.presentation_paused && !p->iomfb.lock.held); + assert(req->unklong <= 1 && !req->unkint && !req->null); + event(req->unklong ? '1' : '0'); + if (req->unklong) power1++; else power0++; + packet.header.in_len = wire_in; + packet.header.out_len = wire_out; + packet.resp.unknown = req->unklong && !on_zero_reply ? 1 : 0; + packet.resp.ret = wire_error; + if (timeout_reply) { late_cookie = cookie; return; } + mutex_lock(&p->iomfb.lock); + assert(dcp_external_hotplug(p, req->unklong)); + cb(p, absent_reply ? NULL : &packet.resp, cookie); + mutex_unlock(&p->iomfb.lock); +} +static void iomfb_opaque_x_reset_h17p(struct apple_dcp *p) { (void)p; } +static void dcp_set_display_device(struct apple_dcp *p, bool oob, u32 *handle, + void (*cb)(struct apple_dcp *, void *, void *), void *cookie) +{ + struct { struct dcp_packet_header h; u32 input, output; } packet = {0}; + assert(!oob && *handle == 2); event('D'); + packet.h.in_len = 4; packet.h.out_len = prep_error == 1 ? 0 : 4; + cb(p, &packet.output, cookie); +} +static void dcp_set_parameter_dcp(struct apple_dcp *p, bool oob, struct dcp_set_parameter_dcp *param, + void (*cb)(struct apple_dcp *, void *, void *), void *cookie) +{ + struct { struct dcp_packet_header h; struct dcp_set_parameter_dcp input; u32 output; } packet = {0}; + assert(!oob && param->param == IOMFBPARAM_ADAPTIVE_SYNC && param->count == 3); event('A'); + for (unsigned int i = 0; i < 8; i++) assert(!param->value[i]); + packet.h.in_len = 40; packet.h.out_len = prep_error == 2 ? 0 : 4; + cb(p, &packet.output, cookie); +} +static void dcp_mode_begin(void *state) { (void)state; } +static unsigned int dcp_mode_finish(void *state, bool success, bool *connected) +{ (void)state; (void)success; (void)connected; return 0; } +static int iomfb_modeset_h17p(struct apple_dcp *p, struct drm_crtc_state *s) +{ (void)s; assert(p->external_lock.held && p->external_running); modesets++; return 0; } +#define iomfb_modeset_v12_3 iomfb_modeset_h17p +#define iomfb_modeset_v13_3 iomfb_modeset_h17p +static void iomfb_atomic_start(void *a, void *b) { (void)a; (void)b; } +static void iomfb_atomic_release(void *p) { kfree(p); } +static struct drm_atomic_state *drm_atomic_state_get(struct drm_atomic_state *s) { return s; } +static void iomfb_backlight_start(void *a, void *b) { (void)a; (void)b; } +static void iomfb_backlight_release(void *p) { kfree(p); } +static void iomfb_crc_start(void *a, void *b) { (void)a; (void)b; } +static void iomfb_crc_release(void *p) { kfree(p); } +''' +suffix = r''' +static struct apple_connector connector; +static void initial(struct apple_dcp *p) +{ + assert(!allocations); + memset(p, 0, sizeof(*p)); + memset(&connector, 0, sizeof(connector)); connector.connected = true; + p->connector = &connector; + p->external_prepared = p->external_cable = p->external_running = p->external_power_attempted = true; + p->fw_compat = DCP_FIRMWARE_H17P; + p->external_generation = p->external_crtc_generation = p->external_rx_generation = 1; + p->hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + p->dptxport[0] = (struct dptx_port){ .connected = true, .service = p }; + p->retained_dma = 7; p->retained_fb = 11; + current_dcp = p; + jiffies = sleeps = power0 = power1 = drops = releases = connects = event_count = 0; + connect_error = hpd_error = release_error = wire_error = 0; + drain_callback = timeout_reply = absent_reply = false; + wire_in = 16; wire_out = 8; + late_cookie = NULL; test_qualified_off = true; prep_error = 0; on_zero_reply = false; + memset(events, 0, sizeof(events)); +} +static int stop(struct apple_dcp *p, u64 generation) +{ + guard(mutex)(&p->external_lock); + return dcp_external_stop_locked(p, generation); +} +static void owned(struct apple_dcp *p) +{ assert(p->retained_dma == 7 && p->retained_fb == 11 && !allocations); } +int main(void) +{ + struct apple_dcp p; + struct platform_device dev = { &p }; + initial(&p); + assert(stop(&p, 1) == 0); + assert(!strcmp(events, "0HR") && power0 == 1 && releases == 1); + assert(p.iomfb.presentation_paused && p.external_stopped && !p.crashed); + assert(stop(&p, 1) == 0 && power0 == 1 && releases == 1); + owned(&p); + + /* Atomic, backlight and CRC are rejected; control transactions remain admissible. */ + iomfb_queue_atomic(&p, NULL, NULL); + iomfb_backlight_kick(&p); + p.crc_enabled = true; + mutex_lock(&p.iomfb.lock); iomfb_queue_crc_h17p(&p, 1); mutex_unlock(&p.iomfb.lock); + assert(!p.crashed && !p.iomfb.queued && !allocations); + struct iomfb_transaction presentation = { .presentation = true }; + struct iomfb_transaction control = {0}; + assert(iomfb_queue(&p, &presentation) == -EAGAIN); + assert(iomfb_queue(&p, &control) == 0); + p.iomfb.queued = p.iomfb.pending.count = 0; + + /* A callback must be able to acquire iomfb.lock and complete a drain. */ + initial(&p); + p.iomfb.active = &presentation; p.iomfb.pending.count = p.iomfb.queued = 1; + p.present_state_h17p.pending = true; p.ch_cb.depth = 1; drain_callback = true; + assert(stop(&p, 1) == 0 && sleeps && power0 == 1 && !p.crashed); + owned(&p); + + /* Every channel, queued-count, active, pending and present state blocks strict drain. */ + for (unsigned int field = 0; field < 10; field++) { + initial(&p); + if (field == 0) p.ch_cmd.depth = 1; + if (field == 1) p.ch_cb.depth = 1; + if (field == 2) p.ch_oobcmd.depth = 1; + if (field == 3) p.ch_oobcb.depth = 1; + if (field == 4) p.ch_async.depth = 1; + if (field == 5) p.ch_oobasync.depth = 1; + if (field == 6) p.iomfb.active = &presentation; + if (field == 7) p.iomfb.pending.count = 1; + if (field == 8) p.present_state_h17p.pending = true; + if (field == 9) p.iomfb.queued = 1; + assert(stop(&p, 1) == -ETIMEDOUT && !power0 && !drops && !releases); + assert(p.iomfb.presentation_paused && p.external_error == -ETIMEDOUT); + assert(stop(&p, 1) == -ETIMEDOUT && !power0); + dcp_external_recover(&dev); + assert(!power1 && !p.crashed); + owned(&p); + } + for (unsigned int fault = 0; fault < 7; fault++) { + initial(&p); + if (fault == 0) p.crashed = true; + if (fault == 1) p.iomfb.stopped = true; + if (fault == 2) wire_out = 12; + if (fault == 3) wire_in = 12; + if (fault == 4) { wire_error = 5; test_qualified_off = false; } + if (fault == 5) hpd_error = -EIO; + if (fault == 6) release_error = -EIO; + assert(stop(&p, 1) < 0 && p.external_error && p.iomfb.presentation_paused); + assert(p.dptxport[0].connected && !p.external_stopped); + assert(power0 == (fault > 1)); + assert(releases == (fault == 6)); + owned(&p); + } + initial(&p); timeout_reply = true; + assert(stop(&p, 1) == -ETIMEDOUT && late_cookie && allocations == 1); + assert(!drops && !releases && p.dptxport[0].connected); + iomfb_external_power_complete(&p, NULL, late_cookie); /* late callback owns its reference */ + owned(&p); + + /* Physical removal + new cable before old CRTC disable: stale token cannot stop new link. */ + initial(&p); + assert(dcp_external_disconnect(&p, true) == 0); + assert(!p.external_cable && !connector.connected && p.external_generation == 2); + assert(dcp_dptx_connect_oob(&dev, 0) == 0); + assert(p.external_generation == 3 && !p.external_running && !connector.connected); + assert(p.iomfb.presentation_paused && power1 == 0); + dcp_poweroff(&dev); + assert(power0 == 1 && !p.external_running); /* token still belongs to generation 1 */ + /* A stale disable must still deliver host DRM event bookkeeping. */ + struct drm_crtc_state inactive = { .active_changed = true, .event = &p }; + struct drm_device drm_dev = {0}; + struct apple_crtc crtc = { .dcp = &dev, .base = { .state = &inactive, .dev = &drm_dev } }; + struct drm_atomic_state atomic = { .next = &inactive }; + apple_crtc_atomic_disable(&crtc.base, &atomic); + assert(power0 == 1 && !inactive.event && vblanks == 1); + /* Active -> active recovery invokes the real CRTC hook and refreshes the token. */ + struct drm_crtc_state active = { .active = true, .mode_changed = true, .mode = { 1920, 1080 } }; + atomic.next = &active; + apple_crtc_atomic_enable(&crtc.base, &atomic); + assert(modesets == 1); + assert(p.external_crtc_generation == 3 && power1 == 1); + dcp_poweroff(&dev); + assert(power0 == 2 && p.external_stopped); + dcp_poweroff(&dev); assert(power0 == 2); + owned(&p); + + /* Uncertain physical stop reports cable loss but keeps the resource and blocks restart. */ + initial(&p); test_qualified_off = false; + assert(dcp_external_disconnect(&p, true) == -EOPNOTSUPP); + assert(!connector.connected && p.dptxport[0].connected && !releases); + assert(dcp_dptx_connect_oob(&dev, 0) == -EOPNOTSUPP && !power1); + assert(p.external_crtc_generation == 1 && p.iomfb.presentation_paused); + owned(&p); + /* A deferred old RX disconnect cannot stop a later connection epoch. */ + initial(&p); + mutex_lock(&p.iomfb.lock); + assert(dcp_external_hotplug(&p, false)); + mutex_unlock(&p.iomfb.lock); + assert(p.external_stop_generation == 1); + p.external_generation = p.external_rx_generation = p.external_crtc_generation = 3; + dcp_external_stop_work(&p.external_stop_work); + assert(!power0 && !drops && p.external_running); + owned(&p); + + /* Before initialization there is no firmware/shared-memory transport to touch. */ + initial(&p); + p.external_prepared = p.external_power_attempted = p.external_running = false; + p.dptxport[0].connected = false; + assert(stop(&p, 1) == 0 && !power0 && !power1 && !releases); + assert(dcp_dptx_connect_oob(&dev, 0) == -EAGAIN && !connects); + owned(&p); + + /* Link hookup alone never powers the pipe. A first activation preserves DA1 order. */ + initial(&p); + p.external_power_attempted = p.external_running = false; + p.iomfb.presentation_paused = true; + p.dptxport[0].connected = false; + assert(dcp_dptx_connect_oob(&dev, 0) == 0 && connects == 1 && !power1); + dcp_external_recover(&dev); + assert(strstr(events, "DA1") && p.external_running && power1 == 1); + owned(&p); + + /* Exact ON requires the 64-bit reply 1; all-zero must not activate modesets. */ + initial(&p); + p.external_running = false; p.external_stopped = true; + p.iomfb.presentation_paused = true; on_zero_reply = true; + unsigned int previous_modesets = modesets; + dcp_external_recover(&dev); + assert(p.external_error == -EREMOTEIO && !p.external_running); + assert(p.external_power_reply_valid && p.external_power_reply == 0); + assert(dcp_crtc_atomic_modeset(&crtc.base, &atomic) == -EPIPE); + apple_crtc_atomic_enable(&crtc.base, &atomic); + assert(modesets == previous_modesets && power1 == 1); + owned(&p); + + /* Production OFF policy always records but refuses to qualify its 8-byte reply. */ + initial(&p); p.iomfb.presentation_paused = true; wire_error = 0x1234; + assert(test_wire_power(&p, false) == -EOPNOTSUPP); + assert(p.external_power_reply_valid && p.external_power_reply == (0x1234ULL << 32)); + assert(!drops && !releases); + owned(&p); + /* Notification delivery is deferred: a client can reenter the lifecycle lock. */ + initial(&p); + assert(dcp_external_disconnect(&p, true) == 0); + assert(p.connector->hotplug_wq.scheduled && !p.external_lock.held); + mutex_lock(&p.external_lock); mutex_unlock(&p.external_lock); + + /* Fabric cleanup cannot release a failed or still-running external route. */ + initial(&p); + struct apple_dcp_typec_route route = { .dcp = &p }; + assert(test_route_deactivate_admission(&route) == -EBUSY); + p.external_error = -EOPNOTSUPP; + assert(test_route_deactivate_admission(&route) == -EOPNOTSUPP); + p.external_error = 0; p.external_stopped = true; + assert(test_route_deactivate_admission(&route) == 0); + owned(&p); + + /* Missing replies and malformed first-start preparation fail closed. */ + initial(&p); p.iomfb.presentation_paused = true; absent_reply = true; + assert(test_wire_power(&p, false) == -EPROTO && !p.external_power_reply_valid); + owned(&p); + for (int stage = 1; stage <= 2; stage++) { + initial(&p); p.external_running = false; p.iomfb.presentation_paused = true; + prep_error = stage; + dcp_external_recover(&dev); + assert(p.external_error == -EPROTO && !power1 && p.iomfb.presentation_paused); + owned(&p); + } + puts("PASS: strict drain, callbacks, admission, exact RPC, stop/restart, stale epochs, modeset gate, retained ownership"); + return 0; +} +''' +with tempfile.TemporaryDirectory(prefix='apple-external-lifecycle-') as tmp: + tmp = Path(tmp) + unit = tmp / 'test.c' + unit.write_text(prefix + production + suffix) + command = shlex.split(os.environ.get('CC', 'clang')) + command += ['-std=gnu11', '-Wall', '-Wextra', '-Werror', + '-Wno-unused-variable', '-Wno-unused-parameter', '-O1', '-g', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer'] + subprocess.run(command + [str(unit), '-o', str(tmp / 'test')], check=True) + subprocess.run([str(tmp / 'test')], check=True) From 22d427d629caba743a74648e13699655b0a1a94c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:13:34 +1100 Subject: [PATCH 0934/1579] drm/apple: retry initial external DPTX service discovery Distinguish a service not yet published from terminal session closure before issuing firmware commands. Keep initial publication retryable, preserve sticky failures after real connection commands, and schedule asynchronous reconnect when the service arrives late. The R28 hardware trial exposed the startup ordering regression. Extracted production tests cover initial and partial publication, closure, late arrival, and retained firmware failures. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/gpu/drm/apple/dcp.c | 16 +- drivers/gpu/drm/apple/dptx-service.h | 4 +- drivers/gpu/drm/apple/dptxep.c | 8 + drivers/gpu/drm/apple/dptxep.h | 2 + tools/diagnostics/test-apple-afk-admission.py | 2 +- .../diagnostics/test-apple-dptx-readiness.py | 241 ++++++++++++++++++ .../test-apple-external-lifecycle.py | 3 +- 8 files changed, 270 insertions(+), 7 deletions(-) create mode 100644 tools/diagnostics/test-apple-dptx-readiness.py diff --git a/MAINTAINERS b/MAINTAINERS index 508097a2e7db3..fede536d3e31e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2021,6 +2021,7 @@ F: tools/diagnostics/apple-j700-dp-test.c F: tools/diagnostics/test-apple-afk-admission.py F: tools/diagnostics/test-apple-atc-unwind.py F: tools/diagnostics/test-apple-dcp-readiness.py +F: tools/diagnostics/test-apple-dptx-readiness.py F: tools/diagnostics/test-apple-external-lifecycle.py F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-d134.py diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index da176a910f46b..8da531f342478 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2188,11 +2188,13 @@ bool dcp_has_typec_routes(struct platform_device *pdev) #define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) #define DPTX_RECONNECT_RETRIES 5 -static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port) +static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_service) { unsigned long timeout; int ret = 0; + if (waiting_service) + *waiting_service = false; if (!dcp->phy) { dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); return -ENODEV; @@ -2208,6 +2210,11 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port) if (!dptx_service_available(&dcp->dptxport[port])) { dev_warn(dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); ret = -ENODEV; + /* No firmware request has been made; the first publication can retry. */ + if (waiting_service && !READ_ONCE(dcp->dptxport[port].closed)) { + *waiting_service = true; + ret = -EAGAIN; + } goto out_unlock; } @@ -2318,6 +2325,7 @@ static void dcp_external_publish_generation(struct apple_dcp *dcp) /* external_lock is never taken by the serialized IOMFB receive callbacks. */ static int dcp_external_link_locked(struct apple_dcp *dcp) { + bool waiting_service; int ret; lockdep_assert_held(&dcp->external_lock); @@ -2327,8 +2335,8 @@ static int dcp_external_link_locked(struct apple_dcp *dcp) return dcp->external_error; if (!dcp->external_cable) return -ENODEV; - ret = dcp_dptx_connect_raw(dcp, 0); - if (ret) + ret = dcp_dptx_connect_raw(dcp, 0, &waiting_service); + if (ret && !waiting_service) dcp->external_error = ret; return ret; } @@ -2360,7 +2368,7 @@ static int dcp_external_start_locked(struct apple_dcp *dcp) static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { if (!dcp_is_external_h17p(dcp)) - return dcp_dptx_connect_raw(dcp, port); + return dcp_dptx_connect_raw(dcp, port, NULL); guard(mutex)(&dcp->external_lock); return dcp_external_link_locked(dcp); } diff --git a/drivers/gpu/drm/apple/dptx-service.h b/drivers/gpu/drm/apple/dptx-service.h index f806e3253999a..3d990a9ca1ea1 100644 --- a/drivers/gpu/drm/apple/dptx-service.h +++ b/drivers/gpu/drm/apple/dptx-service.h @@ -49,7 +49,9 @@ static inline void dptx_service_unbind(struct apple_epic_service *service) struct dptx_port *port = service->cookie; if (port && port->service == service) { - WRITE_ONCE(port->enabled, false); + WRITE_ONCE(port->closed, true); + /* Make terminal state visible to readers of availability. */ + smp_store_release(&port->enabled, false); WRITE_ONCE(port->connected, false); reinit_completion(&port->enable_completion); } diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 451820389f91c..e1d9dfb34dca5 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -844,6 +844,14 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, dptx_service_publish(service); complete(&port->enable_completion); + /* Publication may follow the startup wait or the bounded retry window. */ + if (service->ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + READ_ONCE(service->ep->dcp->typec_cable_connected) && + !READ_ONCE(service->ep->dcp->quiescing) && + !READ_ONCE(service->ep->dcp->external_error)) + mod_delayed_work(system_freezable_wq, + &service->ep->dcp->typec_reconnect_wq, 0); } static int dptxport_group_call(struct apple_epic_service *service, u16 group, u32 command, diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 14530c99567b1..e731604fb6768 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -50,6 +50,8 @@ struct apple_epic_service; struct dptx_port { bool enabled, connected; + /* A published session closed; unlike initial discovery, this is terminal. */ + bool closed; struct completion enable_completion; struct completion linkcfg_completion; u32 unit; diff --git a/tools/diagnostics/test-apple-afk-admission.py b/tools/diagnostics/test-apple-afk-admission.py index fc5669048bdc9..feb21910537dd 100644 --- a/tools/diagnostics/test-apple-afk-admission.py +++ b/tools/diagnostics/test-apple-afk-admission.py @@ -52,7 +52,7 @@ def extract(file, name): struct apple_dcp_afkep *ep; unsigned long cmd_map[1]; }; struct dptx_port { - bool enabled, connected; int enable_completion; u32 unit; + bool enabled, connected, closed; int enable_completion; u32 unit; struct apple_epic_service *service; }; static void reinit_completion(int *p) { *p = 0; } diff --git a/tools/diagnostics/test-apple-dptx-readiness.py b/tools/diagnostics/test-apple-dptx-readiness.py new file mode 100644 index 0000000000000..d86f1505f7cf3 --- /dev/null +++ b/tools/diagnostics/test-apple-dptx-readiness.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise production DPTX publication, retries and terminal service failures.""" +import hashlib +import os +from pathlib import Path +import re +import shlex +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +base = root / 'drivers/gpu/drm/apple' +dcp = (base / 'dcp.c').read_text() +ep = (base / 'dptxep.c').read_text() +service = (base / 'dptx-service.h').read_text() + + +def function(source, name): + match = re.search(r'^static (?:inline )?(?:bool|int|void) ' + name + r'\([^;{]*\)\n\{', source, re.M) + assert match, name + start, opening = match.start(), match.end() - 1 + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +body = '\n'.join(function(service, n) for n in ( + 'dptx_service_available', 'dptx_service_bind', 'dptx_service_publish', 'dptx_service_unbind')) +body += '\n' + '\n'.join(function(dcp, n) for n in ( + 'dcp_is_external_h17p', 'dcp_dptx_connect_raw', 'dcp_external_link_locked', + 'dcp_dptx_connect', 'dcp_typec_reconnect_work')) +body += '\n' + function(ep, 'dptxport_init') +print('Production readiness SHA256:', hashlib.sha256(body.encode()).hexdigest()) +prefix = r''' +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef int64_t s64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +#define smp_load_acquire(p) (*(p)) +#define smp_store_release(p, v) (*(p) = (v)) +#define DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL 2 +#define DPTX_CONNECT_TIMEOUT 2000 +#define DPTX_TUNNEL_CONNECT_TIMEOUT 8000 +#define DPTX_RECONNECT_DELAY 1000 +#define DPTX_RECONNECT_RETRIES 5 +#define DRM_MODE_CONNECTOR_DisplayPort 10 +#define dev_warn(...) ((void)0) +#define dev_info(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_err(...) ((void)0) +#define jiffies_to_msecs(x) (x) +#define usleep_range(a,b) ((void)(a), (void)(b)) +#define system_freezable_wq NULL +#define container_of(p,t,m) ((t *)((char *)(p) - offsetof(t,m))) +#define to_delayed_work(p) container_of(p, struct delayed_work, work) +struct mutex { bool held; }; +static void mutex_lock(struct mutex *m) { assert(!m->held); m->held = true; } +static void mutex_unlock(struct mutex *m) { assert(m->held); m->held = false; } +static struct mutex *acquire(struct mutex *m) { mutex_lock(m); return m; } +static void release(struct mutex **m) { if (*m) mutex_unlock(*m); } +#define guard(kind) struct mutex *g __attribute__((cleanup(release))) = acquire +#define lockdep_assert_held(m) assert((m)->held) +struct completion { bool done; }; +struct work_struct { int unused; }; +struct delayed_work { struct work_struct work; unsigned int scheduled, delay; }; +struct apple_dcp; +struct apple_dcp_afkep { struct apple_dcp *dcp; }; +struct apple_epic_service { void *cookie; struct apple_dcp_afkep *ep; }; +struct dptx_port { + struct apple_epic_service *service; + bool enabled, connected, closed; + s64 unit; + void *atcphy; + struct completion enable_completion, linkcfg_completion; +}; +struct apple_dcp { + struct { int iomfb_method_profile; } hw; + bool external_prepared, external_cable, quiescing, typec_cable_connected; + int external_error; + unsigned int typec_reconnect_tries; + struct mutex external_lock, hpd_mutex; + struct delayed_work typec_reconnect_wq; + void *phy, *dev, *active_typec_route, *avep; + unsigned int dptx_die, dptx_phy, dptx_dfp_port; + int connector_type; + struct dptx_port dptxport[1]; +}; +static unsigned int raw_attempts, firmware_calls, release_calls; +static int injected_error; +static unsigned int fail_stage; +static bool dcp_is_typec_output(struct apple_dcp *p) { (void)p; return true; } +static bool dcp_is_usb4_output(struct apple_dcp *p) { (void)p; return false; } +static bool apple_dp_tunnel_t602x(void) { return false; } +static void reinit_completion(struct completion *c) { c->done = false; } +static void complete(struct completion *c) { c->done = true; } +static unsigned long wait_for_completion_timeout(struct completion *c, unsigned long timeout) +{ (void)timeout; return c->done ? 1 : 0; } +static void mod_delayed_work(void *queue, struct delayed_work *w, unsigned int delay) +{ + struct apple_dcp *p = container_of(w, struct apple_dcp, typec_reconnect_wq); + (void)queue; + assert(!p->external_lock.held && !p->hpd_mutex.held); + w->scheduled++; + w->delay = delay; +} +static int stage(unsigned int step) +{ firmware_calls++; return fail_stage == step ? injected_error : 0; } +static int dptxport_validate_connection(struct apple_epic_service *s, u32 a, u32 b, u32 c) +{ (void)s; (void)a; (void)b; (void)c; raw_attempts++; return stage(1); } +static int dptxport_connect(struct apple_epic_service *s, u32 a, u32 b, u32 c, bool typec) +{ (void)s; (void)a; (void)b; (void)c; (void)typec; return stage(2); } +static int dptxport_request_display(struct apple_epic_service *s) +{ + struct dptx_port *p = s->cookie; + int ret = stage(3); + if (!ret) complete(&p->linkcfg_completion); + return ret; +} +static int dptxport_set_hpd(struct apple_epic_service *s, bool on) +{ (void)s; (void)on; return stage(4); } +static int dptxport_set_hpd_timeout(struct apple_epic_service *s, bool on, unsigned int timeout) +{ (void)timeout; return dptxport_set_hpd(s, on); } +static int dptxport_release_display(struct apple_epic_service *s) +{ (void)s; release_calls++; return 0; } +static void av_service_connect(struct apple_dcp *p) { (void)p; } +''' +suffix = r''' +static void initial(struct apple_dcp *p, struct apple_epic_service *s, struct apple_dcp_afkep *ep) +{ + *p = (struct apple_dcp){ + .hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL, + .external_prepared = true, .external_cable = true, .typec_cable_connected = true, + .phy = p, + }; + *ep = (struct apple_dcp_afkep){ .dcp = p }; + *s = (struct apple_epic_service){ .ep = ep }; + assert(dptx_service_bind(s, &p->dptxport[0], 0) == 0); + raw_attempts = firmware_calls = release_calls = fail_stage = 0; + injected_error = 0; +} +static int link(struct apple_dcp *p) +{ + guard(mutex)(&p->external_lock); + return dcp_external_link_locked(p); +} +static void publish(struct apple_epic_service *s) +{ dptxport_init(s, "dcpdptx-port-epic", "AppleDCPDPTXRemotePort", 0); } +int main(void) +{ + struct apple_dcp p; + struct apple_epic_service service; + struct apple_dcp_afkep ep; + initial(&p, &service, &ep); + /* Reproduce the observed initial call before the first AFK publication. */ + assert(link(&p) == -EAGAIN && !p.external_error); + assert(!firmware_calls && !release_calls && !p.dptxport[0].connected); + /* Even publication later than the bounded retry window must restart work. */ + for (unsigned int i = 0; i < DPTX_RECONNECT_RETRIES; i++) + dcp_typec_reconnect_work(&p.typec_reconnect_wq.work); + assert(p.typec_reconnect_tries == DPTX_RECONNECT_RETRIES && !p.external_error); + unsigned int queued = p.typec_reconnect_wq.scheduled; + publish(&service); + assert(p.dptxport[0].enabled && p.dptxport[0].enable_completion.done); + assert(p.typec_reconnect_wq.scheduled == queued + 1 && !p.typec_reconnect_wq.delay); + assert(!firmware_calls); /* Publication only schedules; never calls firmware inline. */ + dcp_typec_reconnect_work(&p.typec_reconnect_wq.work); + assert(raw_attempts == 1 && p.dptxport[0].connected && !p.external_error); + assert(!p.typec_reconnect_tries); + + /* A reader between owner publication and enabled=true must also retry. */ + initial(&p, &service, &ep); + p.dptxport[0].service = &service; + assert(link(&p) == -EAGAIN && !p.external_error && !firmware_calls); + publish(&service); + assert(link(&p) == 0 && p.dptxport[0].connected); + + /* A closed published session must remain terminal, even before any power request. */ + dptx_service_unbind(&service); + assert(link(&p) == -ENODEV && p.external_error == -ENODEV); + assert(p.dptxport[0].service == &service); + assert(dptx_service_bind(&service, &p.dptxport[0], 0) == -EBUSY); + unsigned int attempts = raw_attempts; + assert(link(&p) == -ENODEV && raw_attempts == attempts); + + /* Neither ENODEV nor EAGAIN from a real operation can masquerade as unpublished. */ + for (unsigned int phase = 1; phase <= 4; phase++) { + for (unsigned int transient = 0; transient < 2; transient++) { + initial(&p, &service, &ep); + publish(&service); + fail_stage = phase; + injected_error = transient ? -EAGAIN : -ENODEV; + assert(link(&p) == injected_error && p.external_error == injected_error); + unsigned int calls = firmware_calls; + fail_stage = 0; + assert(link(&p) == injected_error && firmware_calls == calls); + } + } + /* Missing configuration is terminal; only the first unpublished service is retryable. */ + initial(&p, &service, &ep); + p.phy = NULL; + assert(link(&p) == -ENODEV && p.external_error == -ENODEV && !firmware_calls); + + /* Legacy raw callers keep ENODEV and do not acquire external retry semantics. */ + initial(&p, &service, &ep); + p.hw.iomfb_method_profile = 0; + assert(dcp_dptx_connect(&p, 0) == -ENODEV && !firmware_calls); + publish(&service); + assert(!p.typec_reconnect_wq.scheduled); + + /* Publication cannot revive a removed cable, failed session, or quiescing device. */ + for (unsigned int state = 0; state < 3; state++) { + initial(&p, &service, &ep); + if (state == 0) p.typec_cable_connected = false; + if (state == 1) p.external_error = -EIO; + if (state == 2) p.quiescing = true; + publish(&service); + assert(!p.typec_reconnect_wq.scheduled && !firmware_calls); + } + puts("PASS: delayed publication, exhausted retry recovery, terminal CLOSE/operation errors, async notification"); + return 0; +} +''' +with tempfile.TemporaryDirectory(prefix='apple-dptx-readiness-') as directory: + directory = Path(directory) + unit = directory / 'test.c' + unit.write_text(prefix + body + suffix) + command = shlex.split(os.environ.get('CC', 'clang')) + command += ['-std=gnu11', '-Wall', '-Wextra', '-Werror', '-Wno-unused-parameter', '-O1', '-g', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer'] + subprocess.run(command + [str(unit), '-o', str(directory / 'test')], check=True) + subprocess.run([str(directory / 'test')], check=True) diff --git a/tools/diagnostics/test-apple-external-lifecycle.py b/tools/diagnostics/test-apple-external-lifecycle.py index 35649c07c3412..e0ab24bcd07b7 100644 --- a/tools/diagnostics/test-apple-external-lifecycle.py +++ b/tools/diagnostics/test-apple-external-lifecycle.py @@ -226,8 +226,9 @@ def external_branch(name, result): { (void)s; event('R'); releases++; return release_error; } static void av_service_disconnect(struct apple_dcp *p) { (void)p; } -static int dcp_dptx_connect_raw(struct apple_dcp *p, u32 port) +static int dcp_dptx_connect_raw(struct apple_dcp *p, u32 port, bool *waiting_service) { + if (waiting_service) *waiting_service = false; assert(!port && p->external_lock.held); if (p->dptxport[0].connected) return 0; event('C'); connects++; From dcd9693d6f455bf20b24d0dca24a608e930ac6cc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:14:49 +1100 Subject: [PATCH 0935/1579] drm/apple: select compatible SDR color modes for external H17P Choose advertised full-range 8/10-bit SDR RGB modes for the experimental external H17P profile. Prefer sRGB or BT.709 over default RGB, then use firmware scores. Reject timings that only offer incompatible color modes instead of selecting BT.2020 for native SDR scanout. Leave internal and legacy policy unchanged. The working J700 selected BT.2020 RGB while displaying an SDR desktop. Production-parser tests cover 65 valid, malformed, legacy and high-refresh cases. Hardware color appearance still requires qualification. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/gpu/drm/apple/iomfb_template.c | 4 +- drivers/gpu/drm/apple/parser.c | 44 ++++- drivers/gpu/drm/apple/parser.h | 2 +- tools/diagnostics/test-apple-external-sdr.py | 188 +++++++++++++++++++ 5 files changed, 232 insertions(+), 7 deletions(-) create mode 100644 tools/diagnostics/test-apple-external-sdr.py diff --git a/MAINTAINERS b/MAINTAINERS index fede536d3e31e..38e9edaa75fdd 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2023,6 +2023,7 @@ F: tools/diagnostics/test-apple-atc-unwind.py F: tools/diagnostics/test-apple-dcp-readiness.py F: tools/diagnostics/test-apple-dptx-readiness.py F: tools/diagnostics/test-apple-external-lifecycle.py +F: tools/diagnostics/test-apple-external-sdr.py F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-d134.py F: tools/diagnostics/test-apple-j700-dp.py diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index dd57ffaaafe09..3c580c6a6b08e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -985,7 +985,9 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, dcp->width_mm, dcp->height_mm, dcp->notch_height, dcp->fixed_connector_type == - DRM_MODE_CONNECTOR_eDP); + DRM_MODE_CONNECTOR_eDP, + dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL); if (IS_ERR(dcp->modes)) { dev_warn(dcp->dev, "failed to parse modes\n"); diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 16027c1923369..73e7e1cf315c6 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -390,10 +390,32 @@ static int fill_color_mode(struct dcp_color_mode *color, return 0; } +/* External SDR scanout must not select a wide-gamut mode by score alone. */ +static unsigned int external_sdr_rank(const struct color_mode *color) +{ + if (color->id < 0 || color->id > U32_MAX || + color->eotf != DCP_EOTF_SDR_GAMMA || + color->pixel_encoding != DCP_COLOR_FORMAT_RGB || + color->dynamic_range != DCP_COLOR_YCBCR_RANGE_FULL || + (color->depth != 8 && color->depth != 10)) + return 0; + + switch (color->colorimetry) { + case DCP_COLORIMETRY_SRGB: + case DCP_COLORIMETRY_BT709: + return 2; + case DCP_COLORIMETRY_RGB: + return 1; + default: + return 0; + } +} + static int parse_color_modes(struct dcp_parse_ctx *handle, - struct dcp_display_mode *out) + struct dcp_display_mode *out, bool external_sdr) { struct iterator outer_it; + unsigned int best_sdr_rank = 0; int ret = 0; out->sdr_444.score = -1; out->sdr_rgb.score = -1; @@ -461,6 +483,18 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, cmode.eotf, cmode.dynamic_range, cmode.pixel_encoding); + if (external_sdr) { + unsigned int rank = external_sdr_rank(&cmode); + + if (!rank || rank < best_sdr_rank) + continue; + if (rank > best_sdr_rank) + out->sdr_rgb.score = -1; + fill_color_mode(&out->sdr_rgb, &cmode); + best_sdr_rank = rank; + continue; + } + if (cmode.eotf == DCP_EOTF_SDR_GAMMA) { if (cmode.pixel_encoding == DCP_COLOR_FORMAT_RGB && cmode.depth <= 10) @@ -492,7 +526,7 @@ static u64 calculate_clock(struct dimension *horiz, struct dimension *vert) static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *out, s64 *score, int width_mm, - int height_mm, unsigned notch_height, bool internal) + int height_mm, unsigned notch_height, bool internal, bool external_sdr) { int ret = 0; struct iterator it; @@ -532,7 +566,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, else if (!strcmp(key, "MaximumVariableRefreshRate")) ret = parse_int_bound(it.handle, &max_vrr, 0, U32_MAX); else if (!strcmp(key, "ColorModes")) - ret = parse_color_modes(it.handle, out); + ret = parse_color_modes(it.handle, out, external_sdr); else if (!strcmp(key, "ID")) ret = parse_int_bound(it.handle, &id, 0, U32_MAX); else if (!strcmp(key, "IsVirtual")) @@ -638,7 +672,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, - bool internal) + bool internal, bool external_sdr) { struct iterator it; int ret; @@ -663,7 +697,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, mode = &modes[*count]; ret = parse_mode(it.handle, mode, &score, width_mm, height_mm, - notch_height, internal); + notch_height, internal, external_sdr); /* Errors for a single mode are recoverable -- just skip it. */ if (ret) { diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index bfb0a06c8760a..1f3b8782b7bfc 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -105,7 +105,7 @@ int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, - bool internal); + bool internal, bool external_sdr); int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, diff --git a/tools/diagnostics/test-apple-external-sdr.py b/tools/diagnostics/test-apple-external-sdr.py new file mode 100644 index 0000000000000..f72c26e38108c --- /dev/null +++ b/tools/diagnostics/test-apple-external-sdr.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Parse real-format TimingElements blobs with the production SDR selection code.""" +import hashlib +import os +from pathlib import Path +import re +import shlex +import struct +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +base = root / 'drivers/gpu/drm/apple' +source = (base / 'parser.c').read_text() +header = (base / 'parser.h').read_text() +body = source[source.index('#define DCP_PARSE_HEADER'):source.index('int parse_display_attributes(')] +header = header[header.index('struct apple_dcp;'):header.index('int parse(')] +# Pin the integration scope: the template passes its explicit external profile, +# independent of the existing internal-panel boolean. +template = (base / 'iomfb_template.c').read_text() +call = re.search(r'enumerate_modes\(&ctx,.*?\);', template, re.S).group() +assert re.search(r'dcp->hw\.iomfb_method_profile\s*==\s*DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL\)', call) +print('Production mode parser SHA256:', hashlib.sha256((header + body).encode()).hexdigest()) + + +def wire(value, out): + out += b'\0' * (-len(out) % 4) + if isinstance(value, bool): + out += struct.pack(' +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint64_t u64; +typedef int64_t s64; +#define __packed __attribute__((packed)) +#define IS_ENABLED(x) 0 +#define U32_MAX UINT32_MAX +#define U16_MAX UINT16_MAX +#define S64_MAX INT64_MAX +#define S64_MIN INT64_MIN +#define round_up(n, a) (((n) + (a) - 1) & ~((a) - 1)) +#define ERR_PTR(n) ((void *)(intptr_t)(n)) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define IS_ERR(p) ((uintptr_t)(p) > (uintptr_t)-4096) +#define IS_ERR_OR_NULL(p) (!(p) || IS_ERR(p)) +#define GFP_KERNEL 0 +#define kmalloc(n, f) malloc(n) +#define kcalloc(n, s, f) calloc(n,s) +#define kfree(p) free(p) +#define mul_u32_u32(a, b) ((u64)(u32)(a) * (u32)(b)) +#define DIV_ROUND_CLOSEST_ULL(n, d) (((n) + (d) / 2) / (d)) +#define trace_iomfb_color_mode(...) ((void)0) +#define trace_iomfb_parse_mode_fail(...) ((void)0) +#define trace_iomfb_parse_mode_success(...) ((void)0) +#define trace_iomfb_timing_mode(...) ((void)0) +#define DRM_MODE_TYPE_DRIVER 1 +#define DRM_MODE_TYPE_PREFERRED 2 +struct drm_display_mode { + unsigned int type; + int clock, vdisplay, vsync_start, vsync_end, vtotal; + int hdisplay, hsync_start, hsync_end, htotal, width_mm, height_mm; +}; +static void drm_mode_set_name(struct drm_display_mode *mode) { (void)mode; } +''' +data = '\n'.join('static const unsigned char blob_%u[] = {%s};' % + (i, ','.join(map(str, case[1]))) for i, case in enumerate(cases)) +suffix = r''' +int main(void) +{ +''' +for i, (name, blob, external, count, identity, depth, clock) in enumerate(cases): + suffix += f''' + {{ + struct dcp_parse_ctx ctx; + unsigned int count = 99; + assert(parse(blob_{i}, sizeof(blob_{i}), &ctx) == 0); + struct dcp_display_mode *modes = enumerate_modes(&ctx, &count, 600, 340, 0, + {str(name.startswith('internal')).lower()}, {str(external).lower()}); + if (IS_ERR(modes) || count != {count}) {{ + fprintf(stderr, "case {i}: {name} count=%u\\n", count); + abort(); + }} +''' + if count: + suffix += f''' + assert(modes[0].color_mode_id == {identity}); + assert(modes[0].sdr_rgb.depth == {depth}); + assert(modes[0].mode.clock == {clock}); + assert(modes[0].mode.type & DRM_MODE_TYPE_PREFERRED); +''' + suffix += ' free(modes);\n }\n' +suffix += f' puts("PASS: {len(cases)} production binary-parser SDR, malformed-input, legacy and high-refresh cases");\n return 0;\n}}\n' +with tempfile.TemporaryDirectory(prefix='apple-external-sdr-') as directory: + directory = Path(directory) + unit = directory / 'test.c' + unit.write_text(prefix + header + body + data + suffix) + command = shlex.split(os.environ.get('CC', 'clang')) + command += ['-std=gnu11', '-Wall', '-Wextra', '-Werror', '-Wno-sign-compare', '-O1', '-g', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer'] + subprocess.run(command + [str(unit), '-o', str(directory / 'test')], check=True) + subprocess.run([str(directory / 'test')], check=True) From 468d76e876c03021a87f3be0628e84002447bb9a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:14:49 +1100 Subject: [PATCH 0936/1579] drm/apple: select compatible SDR color modes for external H17P Choose advertised full-range 8/10-bit SDR RGB modes for the experimental external H17P profile. Prefer sRGB or BT.709 over default RGB, then use firmware scores. Reject timings that only offer incompatible color modes instead of selecting BT.2020 for native SDR scanout. Leave internal and legacy policy unchanged. The working J700 selected BT.2020 RGB while displaying an SDR desktop. Production-parser tests cover 65 valid, malformed, legacy and high-refresh cases. Hardware color appearance still requires qualification. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/gpu/drm/apple/iomfb_template.c | 4 +- drivers/gpu/drm/apple/parser.c | 44 ++++- drivers/gpu/drm/apple/parser.h | 2 +- tools/diagnostics/test-apple-external-sdr.py | 188 +++++++++++++++++++ 5 files changed, 232 insertions(+), 7 deletions(-) create mode 100644 tools/diagnostics/test-apple-external-sdr.py diff --git a/MAINTAINERS b/MAINTAINERS index 9e9fbe808e1bc..90a72c4fd6b16 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2021,6 +2021,7 @@ F: tools/diagnostics/apple-j700-dp-test.c F: tools/diagnostics/test-apple-afk-admission.py F: tools/diagnostics/test-apple-atc-unwind.py F: tools/diagnostics/test-apple-dcp-readiness.py +F: tools/diagnostics/test-apple-external-sdr.py F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-d134.py F: tools/diagnostics/test-apple-j700-dp.py diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 575b99606e1ac..31764f1bc3497 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -985,7 +985,9 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, dcp->width_mm, dcp->height_mm, dcp->notch_height, dcp->fixed_connector_type == - DRM_MODE_CONNECTOR_eDP); + DRM_MODE_CONNECTOR_eDP, + dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL); if (IS_ERR(dcp->modes)) { dev_warn(dcp->dev, "failed to parse modes\n"); diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 16027c1923369..73e7e1cf315c6 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -390,10 +390,32 @@ static int fill_color_mode(struct dcp_color_mode *color, return 0; } +/* External SDR scanout must not select a wide-gamut mode by score alone. */ +static unsigned int external_sdr_rank(const struct color_mode *color) +{ + if (color->id < 0 || color->id > U32_MAX || + color->eotf != DCP_EOTF_SDR_GAMMA || + color->pixel_encoding != DCP_COLOR_FORMAT_RGB || + color->dynamic_range != DCP_COLOR_YCBCR_RANGE_FULL || + (color->depth != 8 && color->depth != 10)) + return 0; + + switch (color->colorimetry) { + case DCP_COLORIMETRY_SRGB: + case DCP_COLORIMETRY_BT709: + return 2; + case DCP_COLORIMETRY_RGB: + return 1; + default: + return 0; + } +} + static int parse_color_modes(struct dcp_parse_ctx *handle, - struct dcp_display_mode *out) + struct dcp_display_mode *out, bool external_sdr) { struct iterator outer_it; + unsigned int best_sdr_rank = 0; int ret = 0; out->sdr_444.score = -1; out->sdr_rgb.score = -1; @@ -461,6 +483,18 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, cmode.eotf, cmode.dynamic_range, cmode.pixel_encoding); + if (external_sdr) { + unsigned int rank = external_sdr_rank(&cmode); + + if (!rank || rank < best_sdr_rank) + continue; + if (rank > best_sdr_rank) + out->sdr_rgb.score = -1; + fill_color_mode(&out->sdr_rgb, &cmode); + best_sdr_rank = rank; + continue; + } + if (cmode.eotf == DCP_EOTF_SDR_GAMMA) { if (cmode.pixel_encoding == DCP_COLOR_FORMAT_RGB && cmode.depth <= 10) @@ -492,7 +526,7 @@ static u64 calculate_clock(struct dimension *horiz, struct dimension *vert) static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *out, s64 *score, int width_mm, - int height_mm, unsigned notch_height, bool internal) + int height_mm, unsigned notch_height, bool internal, bool external_sdr) { int ret = 0; struct iterator it; @@ -532,7 +566,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, else if (!strcmp(key, "MaximumVariableRefreshRate")) ret = parse_int_bound(it.handle, &max_vrr, 0, U32_MAX); else if (!strcmp(key, "ColorModes")) - ret = parse_color_modes(it.handle, out); + ret = parse_color_modes(it.handle, out, external_sdr); else if (!strcmp(key, "ID")) ret = parse_int_bound(it.handle, &id, 0, U32_MAX); else if (!strcmp(key, "IsVirtual")) @@ -638,7 +672,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, - bool internal) + bool internal, bool external_sdr) { struct iterator it; int ret; @@ -663,7 +697,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, mode = &modes[*count]; ret = parse_mode(it.handle, mode, &score, width_mm, height_mm, - notch_height, internal); + notch_height, internal, external_sdr); /* Errors for a single mode are recoverable -- just skip it. */ if (ret) { diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index bfb0a06c8760a..1f3b8782b7bfc 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -105,7 +105,7 @@ int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, - bool internal); + bool internal, bool external_sdr); int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, diff --git a/tools/diagnostics/test-apple-external-sdr.py b/tools/diagnostics/test-apple-external-sdr.py new file mode 100644 index 0000000000000..f72c26e38108c --- /dev/null +++ b/tools/diagnostics/test-apple-external-sdr.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Parse real-format TimingElements blobs with the production SDR selection code.""" +import hashlib +import os +from pathlib import Path +import re +import shlex +import struct +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +base = root / 'drivers/gpu/drm/apple' +source = (base / 'parser.c').read_text() +header = (base / 'parser.h').read_text() +body = source[source.index('#define DCP_PARSE_HEADER'):source.index('int parse_display_attributes(')] +header = header[header.index('struct apple_dcp;'):header.index('int parse(')] +# Pin the integration scope: the template passes its explicit external profile, +# independent of the existing internal-panel boolean. +template = (base / 'iomfb_template.c').read_text() +call = re.search(r'enumerate_modes\(&ctx,.*?\);', template, re.S).group() +assert re.search(r'dcp->hw\.iomfb_method_profile\s*==\s*DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL\)', call) +print('Production mode parser SHA256:', hashlib.sha256((header + body).encode()).hexdigest()) + + +def wire(value, out): + out += b'\0' * (-len(out) % 4) + if isinstance(value, bool): + out += struct.pack(' +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint64_t u64; +typedef int64_t s64; +#define __packed __attribute__((packed)) +#define IS_ENABLED(x) 0 +#define U32_MAX UINT32_MAX +#define U16_MAX UINT16_MAX +#define S64_MAX INT64_MAX +#define S64_MIN INT64_MIN +#define round_up(n, a) (((n) + (a) - 1) & ~((a) - 1)) +#define ERR_PTR(n) ((void *)(intptr_t)(n)) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define IS_ERR(p) ((uintptr_t)(p) > (uintptr_t)-4096) +#define IS_ERR_OR_NULL(p) (!(p) || IS_ERR(p)) +#define GFP_KERNEL 0 +#define kmalloc(n, f) malloc(n) +#define kcalloc(n, s, f) calloc(n,s) +#define kfree(p) free(p) +#define mul_u32_u32(a, b) ((u64)(u32)(a) * (u32)(b)) +#define DIV_ROUND_CLOSEST_ULL(n, d) (((n) + (d) / 2) / (d)) +#define trace_iomfb_color_mode(...) ((void)0) +#define trace_iomfb_parse_mode_fail(...) ((void)0) +#define trace_iomfb_parse_mode_success(...) ((void)0) +#define trace_iomfb_timing_mode(...) ((void)0) +#define DRM_MODE_TYPE_DRIVER 1 +#define DRM_MODE_TYPE_PREFERRED 2 +struct drm_display_mode { + unsigned int type; + int clock, vdisplay, vsync_start, vsync_end, vtotal; + int hdisplay, hsync_start, hsync_end, htotal, width_mm, height_mm; +}; +static void drm_mode_set_name(struct drm_display_mode *mode) { (void)mode; } +''' +data = '\n'.join('static const unsigned char blob_%u[] = {%s};' % + (i, ','.join(map(str, case[1]))) for i, case in enumerate(cases)) +suffix = r''' +int main(void) +{ +''' +for i, (name, blob, external, count, identity, depth, clock) in enumerate(cases): + suffix += f''' + {{ + struct dcp_parse_ctx ctx; + unsigned int count = 99; + assert(parse(blob_{i}, sizeof(blob_{i}), &ctx) == 0); + struct dcp_display_mode *modes = enumerate_modes(&ctx, &count, 600, 340, 0, + {str(name.startswith('internal')).lower()}, {str(external).lower()}); + if (IS_ERR(modes) || count != {count}) {{ + fprintf(stderr, "case {i}: {name} count=%u\\n", count); + abort(); + }} +''' + if count: + suffix += f''' + assert(modes[0].color_mode_id == {identity}); + assert(modes[0].sdr_rgb.depth == {depth}); + assert(modes[0].mode.clock == {clock}); + assert(modes[0].mode.type & DRM_MODE_TYPE_PREFERRED); +''' + suffix += ' free(modes);\n }\n' +suffix += f' puts("PASS: {len(cases)} production binary-parser SDR, malformed-input, legacy and high-refresh cases");\n return 0;\n}}\n' +with tempfile.TemporaryDirectory(prefix='apple-external-sdr-') as directory: + directory = Path(directory) + unit = directory / 'test.c' + unit.write_text(prefix + header + body + data + suffix) + command = shlex.split(os.environ.get('CC', 'clang')) + command += ['-std=gnu11', '-Wall', '-Wextra', '-Werror', '-Wno-sign-compare', '-O1', '-g', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer'] + subprocess.run(command + [str(unit), '-o', str(directory / 'test')], check=True) + subprocess.run([str(directory / 'test')], check=True) From 014aed99ea14aa654fa4d0b9472e577702e55ed5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:18:06 +1100 Subject: [PATCH 0937/1579] drm/apple: retrieve external H17P EDID through bounded AFKV2 inline calls Use the external DPAV endpoint's inline framing with independent request length and response capacity. Validate actual service and EDID lengths before reading them, retain timeout ownership, and queue a connector reprobe when the service appears late. Preserve the legacy padded OOB path and EDID reader, and keep internal H17P excluded. Host tests exercise the production sender, service envelope, EDID reader, ring admission and late notification with malformed replies and timeout cases. Hardware identity and color qualification remain pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 168 ++++++++++++++---- drivers/gpu/drm/apple/afk.h | 7 + drivers/gpu/drm/apple/afkv2-codec.h | 52 ++++++ drivers/gpu/drm/apple/epic/dpavserv-edid.h | 48 +++++ drivers/gpu/drm/apple/epic/dpavservep.c | 74 ++++++-- drivers/gpu/drm/apple/iomfb.c | 7 +- .../drivers/gpu/apple-afkv2/Makefile | 2 +- .../selftests/drivers/gpu/apple-afkv2/README | 16 ++ .../drivers/gpu/apple-afkv2/extract-edid.py | 79 ++++++++ .../drivers/gpu/apple-afkv2/extract-notify.py | 50 ++++++ .../drivers/gpu/apple-afkv2/extract-ring.py | 50 ++++++ .../drivers/gpu/apple-afkv2/extract-send.py | 118 ++++++++++++ .../gpu/apple-afkv2/extract-service.py | 86 +++++++++ .../apple-afkv2/host-include/linux/errno.h | 5 + .../drivers/gpu/apple-afkv2/inline-edid.c | 113 ++++++++++++ .../gpu/apple-afkv2/run-afkv2-tests.sh | 42 +++++ 16 files changed, 869 insertions(+), 48 deletions(-) create mode 100644 drivers/gpu/drm/apple/epic/dpavserv-edid.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 3fecf8d931db5..2d7fe02b99ab1 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -853,15 +853,23 @@ static void afk_v2_receive(struct apple_dcp_afkep *ep, u32 channel, u32 type, spin_lock_irqsave(&service->lock, flags); ret = -EPROTO; if (slot < MAX_PENDING_CMDS && test_bit(slot, service->cmd_map) && - !service->cmds[slot].done && response.id == service->cmds[slot].tag) - ret = afkv2_check_response(&frame, &response, service->channel, - service->cmds[slot].type, service->cmds[slot].tag, - service->cmds[slot].rxbuf_dma, - service->cmds[slot].rxlen, - service->cmds[slot].txbuf_dma, - service->cmds[slot].txlen); - if (!ret) - service->cmds[slot].reply_length = response.value ? 0 : response.rx_size; + !service->cmds[slot].done && response.id == service->cmds[slot].tag) { + struct epic_cmd_info *cmd = &service->cmds[slot]; + + if (ep->inline_v2) + ret = afkv2_check_inline_response(&frame, &response, service->channel, + cmd->type, cmd->tag, cmd->rxlen); + else + ret = afkv2_check_response(&frame, &response, service->channel, + cmd->type, cmd->tag, cmd->rxbuf_dma, + cmd->rxlen, cmd->txbuf_dma, cmd->txlen); + if (!ret) { + cmd->reply_length = response.value ? 0 : + (ep->inline_v2 ? response.inline_size : response.rx_size); + if (ep->inline_v2 && !response.value && cmd->reply_length) + memcpy(cmd->rxbuf, response.inline_data, cmd->reply_length); + } + } spin_unlock_irqrestore(&service->lock, flags); if (!ret) { legacy.retcode = cpu_to_le32(response.value); @@ -1219,20 +1227,29 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, return -EOPNOTSUPP; v2_flags = 1; } else if (ecat == EPIC_CAT_COMMAND) { - const struct epic_cmd *cmd = payload; - - if (stype != EPIC_SUBTYPE_STD_SERVICE || !cmd || - payload_len != sizeof(*cmd)) - return -EINVAL; - ret = afkv2_encode_oob_command(v2_command, sizeof(v2_command), tag, - le64_to_cpu(cmd->rxbuf), - le32_to_cpu(cmd->rxlen), - le64_to_cpu(cmd->txbuf), - le32_to_cpu(cmd->txlen)); - if (ret) - return ret; - payload = v2_command; - payload_len = sizeof(v2_command); + if (ep->inline_v2) { + struct afkv2_command command; + + if (stype != EPIC_SUBTYPE_STD_SERVICE || + afkv2_decode_command(payload, payload_len, &command) || + command.oob || command.id != tag) + return -EINVAL; + } else { + const struct epic_cmd *cmd = payload; + + if (stype != EPIC_SUBTYPE_STD_SERVICE || !cmd || + payload_len != sizeof(*cmd)) + return -EINVAL; + ret = afkv2_encode_oob_command(v2_command, sizeof(v2_command), tag, + le64_to_cpu(cmd->rxbuf), + le32_to_cpu(cmd->rxlen), + le64_to_cpu(cmd->txbuf), + le32_to_cpu(cmd->txlen)); + if (ret) + return ret; + payload = v2_command; + payload_len = sizeof(v2_command); + } v2_category = AFKV2_COMMAND; } else if (ecat == EPIC_CAT_REPLY) { v2_category = AFKV2_RESPONSE; @@ -1451,12 +1468,28 @@ int afk_send_command(struct apple_epic_service *service, u8 type, MSEC_PER_SEC); } +static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, + void *output, size_t output_len, u32 *retcode, + unsigned int timeout_ms, size_t *received); + int afk_send_command_timeout(struct apple_epic_service *service, u8 type, const void *payload, size_t payload_len, void *output, size_t output_len, u32 *retcode, unsigned int timeout_ms) +{ + return afk_send_command_timeout_len(service, type, payload, payload_len, + output, output_len, retcode, timeout_ms, NULL); +} + +static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, + void *output, size_t output_len, u32 *retcode, + unsigned int timeout_ms, size_t *received) { struct epic_cmd cmd; + void *inline_command = NULL; + size_t copy_len = output_len; void *rxbuf, *txbuf; dma_addr_t rxbuf_dma, txbuf_dma; unsigned long flags; @@ -1465,6 +1498,8 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, struct apple_dcp_afkep *ep; DECLARE_COMPLETION_ONSTACK(completion); + if (received) + *received = 0; service = afk_service_get(service); if (!service) return -ENODEV; @@ -1494,6 +1529,17 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, } memcpy(txbuf, payload, payload_len); + if (ep->inline_v2) { + if (payload_len > AFKV2_MAX_PAYLOAD - AFKV2_COMMAND_SIZE) { + ret = -E2BIG; + goto err_free_txbuf; + } + inline_command = kmalloc(AFKV2_COMMAND_SIZE + payload_len, GFP_KERNEL); + if (!inline_command) { + ret = -ENOMEM; + goto err_free_txbuf; + } + } memset(&cmd, 0, sizeof(cmd)); cmd.retcode = cpu_to_le32(0); @@ -1531,9 +1577,21 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, service->cmds[idx].completion = &completion; init_completion(&completion); - ret = afk_send_epic(service->ep, service->channel, tag, - EPIC_TYPE_COMMAND, EPIC_CAT_COMMAND, type, &cmd, - sizeof(cmd)); + if (ep->inline_v2) { + ret = afkv2_encode_inline_command(inline_command, + AFKV2_COMMAND_SIZE + payload_len, + tag, payload, payload_len, output_len); + if (!ret) + ret = afk_send_epic(ep, service->channel, tag, EPIC_TYPE_COMMAND, + EPIC_CAT_COMMAND, type, inline_command, + AFKV2_COMMAND_SIZE + payload_len); + } else { + ret = afk_send_epic(service->ep, service->channel, tag, + EPIC_TYPE_COMMAND, EPIC_CAT_COMMAND, type, &cmd, + sizeof(cmd)); + } + kfree(inline_command); + inline_command = NULL; spin_unlock_irqrestore(&service->lock, flags); if (ret) goto err_free_cmd; @@ -1565,13 +1623,17 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, ret = 0; if (retcode) *retcode = service->cmds[idx].retcode; + if (ep->inline_v2 && received) + copy_len = service->cmds[idx].reply_length; if (ep->wire_v2 && !service->cmds[idx].retcode && - service->cmds[idx].reply_length < output_len) + service->cmds[idx].reply_length < copy_len) ret = -EMSGSIZE; else if (output && output_len && (!ep->wire_v2 || !service->cmds[idx].retcode)) { if (ep->wire_v2) dma_rmb(); - memcpy(output, rxbuf, output_len); + memcpy(output, rxbuf, copy_len); + if (received) + *received = copy_len; } err_free_cmd: @@ -1580,6 +1642,8 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, bitmap_release_region(service->cmd_map, idx, 0); err_unlock: spin_unlock_irqrestore(&service->lock, flags); +err_free_txbuf: + kfree(inline_command); dma_free_coherent(ep->dcp->dev, payload_len, txbuf, txbuf_dma); err_free_rxbuf: dma_free_coherent(ep->dcp->dev, output_len, rxbuf, rxbuf_dma); @@ -1597,12 +1661,40 @@ int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, output_pad, MSEC_PER_SEC); } +static int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, unsigned int timeout_ms, + size_t *received); + +int afk_service_call_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, size_t *received) +{ + return afk_service_call_timeout_len(service, group, command, data, data_len, + data_pad, output, output_len, output_pad, + MSEC_PER_SEC, received); +} + int afk_service_call_timeout(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, size_t output_len, size_t output_pad, unsigned int timeout_ms) +{ + return afk_service_call_timeout_len(service, group, command, data, data_len, + data_pad, output, output_len, output_pad, + timeout_ms, NULL); +} + +static int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, unsigned int timeout_ms, + size_t *received) { struct epic_service_call *call; + size_t wire_received = 0, result_size; void *bfr; size_t bfr_len = max(data_len + data_pad, output_len + output_pad) + sizeof(*call); @@ -1610,6 +1702,8 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, u32 retcode; u32 retlen; + if (received) + *received = 0; if (!service) return -ENODEV; @@ -1622,14 +1716,17 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, memset(call, 0, sizeof(*call)); call->group = cpu_to_le16(group); call->command = cpu_to_le32(command); - call->data_len = cpu_to_le32(data_len + data_pad); + call->data_len = cpu_to_le32(service->ep->inline_v2 ? data_len : + data_len + data_pad); call->magic = cpu_to_le32(EPIC_SERVICE_CALL_MAGIC); memcpy(bfr + sizeof(*call), data, data_len); - ret = afk_send_command_timeout(service, EPIC_SUBTYPE_STD_SERVICE, bfr, - bfr_len, bfr, bfr_len, &retcode, - timeout_ms); + /* Inline carries only the request; its response capacity is independent. */ + ret = afk_send_command_timeout_len(service, EPIC_SUBTYPE_STD_SERVICE, bfr, + service->ep->inline_v2 ? sizeof(*call) + data_len : + bfr_len, bfr, bfr_len, &retcode, timeout_ms, + received ? &wire_received : NULL); if (ret) { dev_warn(service->ep->dcp->dev, "AFK[ep:%02x]: %s (chan:%u) service call group %u cmd %u failed to complete: %d\n", @@ -1646,6 +1743,13 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, ret = -EINVAL; goto out; } + if (received) { + ret = afkv2_service_reply_size(bfr, wire_received, group, command, + output_len, &result_size); + if (ret) + goto out; + *received = result_size; + } if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || le16_to_cpu(call->group) != group || le32_to_cpu(call->command) != command) { diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index b8c889da5c356..45fe9db64d30f 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -329,6 +329,8 @@ struct apple_dcp_afkep { u32 endpoint; /* Set before afk_start; protocol selection is per endpoint. */ bool wire_v2; + /* External H17P DPAV only; other endpoints retain owned-DMA commands. */ + bool inline_v2; struct workqueue_struct *wq; struct completion started; @@ -382,4 +384,9 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, size_t output_len, size_t output_pad, unsigned int timeout_ms); +int afk_service_call_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, size_t *received); + #endif diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h index 8990182275354..9ef52b2e70ea2 100644 --- a/drivers/gpu/drm/apple/afkv2-codec.h +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -179,6 +179,26 @@ static inline int afkv2_encode_oob_command(void *payload, size_t capacity, return 0; } +/* Inline requests carry a response capacity separately from their body. */ +static inline int afkv2_encode_inline_command(void *payload, size_t capacity, + u8 id, const void *body, + size_t size, u32 response_capacity) +{ + u8 *p = payload; + + if (!payload || (size && !body) || size > AFKV2_MAX_PAYLOAD - AFKV2_COMMAND_SIZE || + response_capacity > AFKV2_MAX_PAYLOAD) + return -EINVAL; + if (capacity < AFKV2_COMMAND_SIZE || size > capacity - AFKV2_COMMAND_SIZE) + return -ENOSPC; + if (size) + memmove(p + AFKV2_COMMAND_SIZE, body, size); + memset(p, 0, AFKV2_COMMAND_SIZE); + p[1] = id; + afkv2_put32(p + 4, response_capacity); + return 0; +} + static inline int afkv2_decode_command(const void *payload, size_t size, struct afkv2_command *out) { @@ -250,6 +270,38 @@ static inline int afkv2_check_response(const struct afkv2_frame *frame, return 0; } +/* Kept separate so DPAV inline replies cannot loosen owned-DMA checks. */ +static inline int afkv2_check_inline_response(const struct afkv2_frame *frame, + const struct afkv2_command *response, + u16 interface, u8 type, u8 id, + size_t capacity) +{ + if (!frame || !response || frame->category != AFKV2_RESPONSE || + frame->interface != interface || frame->type != type || response->id != id || + frame->token || frame->transport_flags || frame->interface_flags || + frame->reserved || response->oob || response->inline_size > capacity) + return -EPROTO; + return 0; +} + +/* The standard service envelope precedes the actual, possibly short reply. */ +static inline int afkv2_service_reply_size(const void *body, size_t received, + u16 group, u32 command, size_t capacity, + size_t *size) +{ + const u8 *p = body; + u32 length; + + if (!body || !size || received < 64 || afkv2_get32(p + 12) != 0x69706378 || + afkv2_get16(p + 2) != group || afkv2_get32(p + 4) != command) + return -EPROTO; + length = afkv2_get32(p + 8); + if (length > received - 64 || length > capacity) + return -EMSGSIZE; + *size = length; + return 0; +} + /* Reject a peer command without touching its advertised DMA buffers. */ static inline int afkv2_encode_error_reply(void *record, size_t capacity, u8 sequence, const struct afkv2_frame *request, diff --git a/drivers/gpu/drm/apple/epic/dpavserv-edid.h b/drivers/gpu/drm/apple/epic/dpavserv-edid.h new file mode 100644 index 0000000000000..31baaaea7070d --- /dev/null +++ b/drivers/gpu/drm/apple/epic/dpavserv-edid.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DPAVSERV_EDID_H +#define APPLE_DPAVSERV_EDID_H + +#include "../afkv2-codec.h" + +#define DPAV_EDID_HEADER_SIZE 48U +#define EDID_LEADING_DATA_SIZE 8U +#define EDID_BLOCK_SIZE 128U +#define EDID_MAX_SIZE 32768U +#define EDID_BUF_SIZE (EDID_LEADING_DATA_SIZE + EDID_MAX_SIZE) + +/* Validate only bytes actually returned by firmware, never zero-fill padding. */ +static inline int dpavserv_edid_size(const void *response, size_t received, + size_t *edid_size) +{ + static const u8 header[] = { 0, 255, 255, 255, 255, 255, 255, 0 }; + const u8 *p = response; + u64 used; + size_t length, offset, i; + + if (!response || !edid_size || received < DPAV_EDID_HEADER_SIZE) + return -EIO; + if (afkv2_get64(p) != EDID_BUF_SIZE) + return -EIO; + used = afkv2_get64(p + 32); + if (used < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE || used > EDID_BUF_SIZE || + used > received - DPAV_EDID_HEADER_SIZE) + return -EIO; + length = used - EDID_LEADING_DATA_SIZE; + p += DPAV_EDID_HEADER_SIZE + EDID_LEADING_DATA_SIZE; + if (length % EDID_BLOCK_SIZE || memcmp(p, header, sizeof(header)) || + ((size_t)p[126] + 1) * EDID_BLOCK_SIZE > length) + return -EIO; + /* Permit extra whole blocks for HF-EEODB, as drm_edid supports it. */ + for (offset = 0; offset < length; offset += EDID_BLOCK_SIZE) { + u8 sum = 0; + + for (i = 0; i < EDID_BLOCK_SIZE; i++) + sum += p[offset + i]; + if (sum) + return -EIO; + } + *edid_size = length; + return 0; +} + +#endif diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 4ddce69860379..1b4019cc159f9 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -2,6 +2,9 @@ /* Copyright The Asahi Linux Contributors */ #include "dpavservep.h" +#include "dpavserv-edid.h" + +#define EDID_EXT_BLOCK_COUNT_OFFSET 0x7E #include @@ -12,8 +15,23 @@ #include "../afk.h" #include "../dcp.h" #include "../dcp-internal.h" +#include "../connector.h" #include "../trace.h" +static void dcpavserv_notify_edid(struct apple_epic_service *service) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct apple_connector *connector = READ_ONCE(dcp->connector); + + if (!service->ep->inline_v2 || READ_ONCE(dcp->quiescing) || !connector || + !READ_ONCE(dcp->nr_modes) || !READ_ONCE(connector->connected) || + READ_ONCE(connector->base.registration_state) != DRM_CONNECTOR_REGISTERED || + READ_ONCE(connector->dcp) != to_platform_device(dcp->dev)) + return; + /* get_modes may already have run before this service was announced. */ + schedule_work(&connector->hotplug_wq); +} + static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -35,6 +53,11 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, service->cookie = &dcp->dcpavserv; complete(&dcp->dcpavserv.enable_completion); spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + if (service->ep->inline_v2) { + dev_info(dcp->dev, "DCPAVSERV: inline EDID service ready on channel %u\n", + service->channel); + dcpavserv_notify_edid(service); + } } } @@ -100,12 +123,6 @@ struct dpavserv_copy_edid_cmd { u8 _pad2[8]; } __packed; -#define EDID_LEADING_DATA_SIZE 8 -#define EDID_BLOCK_SIZE 128 -#define EDID_EXT_BLOCK_COUNT_OFFSET 0x7E -#define EDID_MAX_SIZE SZ_32K -#define EDID_BUF_SIZE (EDID_LEADING_DATA_SIZE + EDID_MAX_SIZE) - struct dpavserv_copy_edid_resp { __le64 max_size; u8 _pad1[24]; @@ -182,7 +199,7 @@ static int dcpdpserv_report(struct apple_epic_service *service, return parse_report(service, type, data, data_size); } -static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *service) +static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *service) { struct dpavserv_copy_edid_cmd cmd; struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; @@ -229,6 +246,32 @@ static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *ser data_size - EDID_LEADING_DATA_SIZE); } +static const struct drm_edid *dcpavserv_read_edid_inline(struct apple_epic_service *service) +{ + struct dpavserv_copy_edid_cmd cmd; + struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; + size_t resp_size = sizeof(*resp) + EDID_BUF_SIZE; + size_t received, edid_size; + int ret; + + memset(&cmd, 0, sizeof(cmd)); + cmd.max_size = cpu_to_le64(EDID_BUF_SIZE); + resp = kzalloc(resp_size, GFP_KERNEL); + if (!resp) + return ERR_PTR(-ENOMEM); + + ret = afk_service_call_len(service, 1, 7, &cmd, sizeof(cmd), EDID_BUF_SIZE, + resp, resp_size, 0, &received); + if (ret < 0) + return ERR_PTR(ret); + + ret = dpavserv_edid_size(resp, received, &edid_size); + if (ret) + return ERR_PTR(ret); + + return drm_edid_alloc(resp->data + EDID_LEADING_DATA_SIZE, edid_size); +} + const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) { struct apple_epic_service *service; @@ -236,12 +279,13 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) unsigned long flags; spin_lock_irqsave(&dcp->dcpavserv.lock, flags); - service = afk_service_get(dcp->dcpavserv.service); + service = dcp->dcpavserv.enabled ? afk_service_get(dcp->dcpavserv.service) : NULL; spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); if (!service) return ERR_PTR(-ENODEV); - edid = dcpavserv_read_edid(service); + edid = service->ep->inline_v2 ? dcpavserv_read_edid_inline(service) : + dcpavserv_read_edid_legacy(service); spin_lock_irqsave(&dcp->dcpavserv.lock, flags); if (dcp->dcpavserv.service != service) { if (!IS_ERR(edid)) @@ -281,6 +325,10 @@ int dpavservep_init(struct apple_dcp *dcp) ep = afk_init(dcp, DPAVSERV_ENDPOINT, dpavservep_ops); if (IS_ERR(ep)) return PTR_ERR(ep); + /* Only the external H17P DPAV endpoint uses inline AFKV2 service calls. */ + ep->wire_v2 = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + ep->inline_v2 = ep->wire_v2; dcp->dcpavservep = ep; dcp->dcpavservep->match_epic_name = true; @@ -295,8 +343,10 @@ int dpavservep_init(struct apple_dcp *dcp) ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, msecs_to_jiffies(1000)); - if (ret >= 0) - return 0; + if (!ret) { + dev_warn(dcp->dev, "DCPAVSERV: timed out waiting for EDID service announcement\n"); + return -ETIMEDOUT; + } - return ret; + return 0; } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 786eb53383329..50b240511442b 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1017,9 +1017,10 @@ int dcp_get_modes(struct drm_connector *connector) } drm_connector_set_vrr_capable_property(connector, vrr_capable); - /* H17P sends no EPIC commands; see afk_send_epic(). */ - if (dcp->nr_modes && dcp->fw_compat != DCP_FIRMWARE_H17P && - !apple_connector->drm_edid) { + /* Internal H17P still has no supported EDID command transport. */ + if (dcp->nr_modes && !apple_connector->drm_edid && + (dcp->fw_compat != DCP_FIRMWARE_H17P || + (dcp->dcpavservep && dcp->dcpavservep->inline_v2))) { const struct drm_edid *edid; edid = dcpavserv_copy_edid(dcp); if (IS_ERR_OR_NULL(edid)) { diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile index 65780f3d5003f..df1b033d9cb32 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile @@ -4,6 +4,6 @@ export HOSTCC export AFKV2_SOURCE_ROOT := $(abspath ../../../../../..) TEST_PROGS := run-afkv2-tests.sh -TEST_FILES := codec.c owned-prefix.c owned-suffix.c extract-owned.py host-include +TEST_FILES := codec.c owned-prefix.c owned-suffix.c extract-owned.py inline-edid.c extract-service.py extract-send.py extract-edid.py extract-ring.py extract-notify.py host-include include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/README b/tools/testing/selftests/drivers/gpu/apple-afkv2/README index 42eb152d716a2..064cdcf018595 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/README +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/README @@ -22,3 +22,19 @@ DMA coherency, firmware, a graphics device or any hardware display behavior. All build products are temporary. No device node is opened and no GPU, firmware or target machine is contacted. The extractor fails if production function boundaries change, requiring review rather than silently testing a stale copy. + +The inline EDID tests cover response context, status, declared/actual lengths, +EDID header/checksum/block limits, and continued rejection of inline responses +by the OOB matcher. The actual service-call body is extracted for short reply, +error status and timeout checks; its wire sender is mocked. + +The real sender is also extracted to test a 128-byte mocked command payload +with a 32 KiB response capacity, OOB behavior, send failures and timeout +retention. The two production EDID readers are tested independently to +prove legacy checksum tolerance remains unchanged. + +The complete encoded EDID request is 160 bytes including the queue entry, +consumes 192 bytes at a 64-byte stride, and needs a 256-byte empty ring. The +ring-admission block is extracted from the real transmitter to check this. +The late-publication notifier is extracted to test asynchronous hotplug +queueing and connector-registration, quiesce, routing and legacy guards. diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py new file mode 100644 index 0000000000000..0ab0f5eda8cac --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Verify legacy EDID tolerance and stricter external-inline parsing separately.""" +import pathlib +import re +import sys + +source = pathlib.Path(sys.argv[1]).read_text() + +def function(name): + match = re.search(r"static const struct drm_edid \*" + name + r"\([^;]*?\)\n\{", source) + if not match: + raise RuntimeError("production reader missing: " + name) + end, depth = match.end(), 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start():end] + +assert "service->ep->inline_v2 ? dcpavserv_read_edid_inline(service)" in source +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include +#include "epic/dpavserv-edid.h" +#define GFP_KERNEL 0 +#define kzalloc(n,f) calloc(1,n) +#define cpu_to_le64(x) (x) +#define le64_to_cpu(x) (x) +#define EDID_EXT_BLOCK_COUNT_OFFSET 0x7e +#define ERR_PTR(x) ((void*)(intptr_t)(x)) +#define IS_ERR(p) ((uintptr_t)(p)>(uintptr_t)-4096) +static void cleanup_free(void *p) { free(*(void**)p); } +#define __free(x) __attribute__((cleanup(cleanup_free))) +struct apple_epic_service { int unused; }; +struct drm_edid { size_t size; }; +struct dpavserv_copy_edid_cmd { u64 max_size;u8 pad1[24];u64 used_size;u8 pad2[8]; }; +struct dpavserv_copy_edid_resp { u64 max_size;u8 pad1[24];u64 used_size;u8 pad2[8];u8 data[]; }; +static u8 fixture[48+8+256]; +static bool truncated; +static const struct drm_edid *drm_edid_alloc(const void *p,size_t n) +{ struct drm_edid *e=malloc(sizeof(*e));(void)p;e->size=n;return e; } +static int afk_service_call(struct apple_epic_service *s,u16 group,u32 command, + const void *data,size_t data_len,size_t data_pad,void *output,size_t output_len,size_t output_pad) +{ + const struct dpavserv_copy_edid_cmd *cmd=data; + (void)s;(void)data_pad;(void)output_pad; + assert(group==1 && command==7 && data_len==48 && cmd->max_size==EDID_BUF_SIZE); + assert(output_len>=sizeof(fixture));memcpy(output,fixture,sizeof(fixture));return 0; +} +static int afk_service_call_len(struct apple_epic_service *s,u16 group,u32 command, + const void *data,size_t data_len,size_t data_pad,void *output,size_t output_len, + size_t output_pad,size_t *received) +{ int ret=afk_service_call(s,group,command,data,data_len,data_pad,output,output_len,output_pad); + *received=sizeof(fixture)-(truncated?1:0);return ret; } +static void fix_checksum(u8 *p) +{ unsigned int i;u8 sum=0;p[127]=0;for(i=0;i<127;i++)sum+=p[i];p[127]=-sum; } +''' +suffix = r''' +int main(void) +{ + struct apple_epic_service service={0};const struct drm_edid *e; + u8 *base=fixture+56,*extension=base+128; + afkv2_put64(fixture,EDID_BUF_SIZE);afkv2_put64(fixture+32,8+256); + memset(base+1,255,6);base[126]=1;fix_checksum(base); + extension[0]=2; /* deliberately invalid extension checksum */ + e=dcpavserv_read_edid_legacy(&service);assert(!IS_ERR(e) && e->size==256);free((void*)e); + e=dcpavserv_read_edid_inline(&service);assert(IS_ERR(e)); + fix_checksum(extension); + e=dcpavserv_read_edid_inline(&service);assert(!IS_ERR(e) && e->size==256);free((void*)e); + truncated=true; + e=dcpavserv_read_edid_inline(&service);assert(IS_ERR(e)); + e=dcpavserv_read_edid_legacy(&service);assert(!IS_ERR(e));free((void*)e); + puts("PASS: actual legacy EDID checksum tolerance preserved; inline checksum/actual-length validation enforced"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + function("dcpavserv_read_edid_legacy") + function("dcpavserv_read_edid_inline") + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py new file mode 100644 index 0000000000000..26579f8399cb1 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise actual late-EDID publication notification guards with mock work.""" +import pathlib +import re +import sys +source = pathlib.Path(sys.argv[1]).read_text() +match = re.search(r"static void dcpavserv_notify_edid\([^;]*?\)\n\{", source) +if not match: + raise RuntimeError("EDID notification helper missing") +end, depth = match.end(), 1 +while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include +#define READ_ONCE(x) (x) +#define DRM_CONNECTOR_REGISTERED 1 +struct base { int registration_state; }; +struct apple_connector { bool connected;struct base base;void *dcp;int hotplug_wq; }; +struct apple_dcp { struct apple_connector *connector;bool quiescing;unsigned int nr_modes;void *dev; }; +struct ep { struct apple_dcp *dcp;bool inline_v2; }; +struct apple_epic_service { struct ep *ep; }; +static unsigned int queued; +static void *to_platform_device(void *p) { return p; } +static void schedule_work(int *w) { (*w)++;queued++; } +''' +suffix = r''' +int main(void) +{ + int device,other;struct apple_dcp dcp={.dev=&device}; + struct ep ep={.dcp=&dcp,.inline_v2=true};struct apple_epic_service service={.ep=&ep}; + struct apple_connector connector={.dcp=&device}; + dcpavserv_notify_edid(&service);assert(queued==0); + dcp.connector=&connector;dcpavserv_notify_edid(&service);assert(queued==0); + dcp.nr_modes=1;connector.connected=true;dcpavserv_notify_edid(&service);assert(queued==0); + connector.base.registration_state=DRM_CONNECTOR_REGISTERED; + dcpavserv_notify_edid(&service);assert(queued==1 && connector.hotplug_wq==1); + dcp.quiescing=true;dcpavserv_notify_edid(&service);assert(queued==1); + dcp.quiescing=false;connector.dcp=&other;dcpavserv_notify_edid(&service);assert(queued==1); + connector.dcp=&device;ep.inline_v2=false;dcpavserv_notify_edid(&service);assert(queued==1); + ep.inline_v2=true;connector.connected=false;dcpavserv_notify_edid(&service);assert(queued==1); + puts("PASS: actual late-EDID notification queues ready connector only; legacy/quiesce/unrouted/unregistered guarded"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + source[match.start():end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py new file mode 100644 index 0000000000000..65d6fdeca7a54 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Check full AFKV2 frame size against the production ring-admission block.""" +import pathlib +import sys + +source = pathlib.Path(sys.argv[1]).read_text() +start = source.index("\ttotal_epic_size = sizeof(*ehdr) +", source.index("static int afk_send_epic_token", source.index("int afk_receive_message"))) +end = source.index("\n\thdr = hdr2 = NULL;", start) +block = source[start:end] +prefix = r''' +#undef NDEBUG +#include +#include +#include "afkv2-codec.h" +#define IS_ALIGNED(x,a) (!((x)&((a)-1))) +#define ALIGN(x,a) (((x)+(a)-1)&~((a)-1)) +struct epic_hdr { u8 data[16]; }; +struct epic_sub_hdr { u8 data[24]; }; +struct epic_sub_hdr_h17p { u8 data[8]; }; +struct afk_qe { u8 data[16]; }; +struct ring { bool ready; u32 stride,bufsz; }; +struct ep { bool wire_v2; struct ring txbfr; }; +static int check_ring(struct ep *ep,u32 rptr,u32 wptr,size_t payload_len,size_t *total) +{ + struct epic_hdr *ehdr;struct epic_sub_hdr *eshdr;struct afk_qe *hdr; + bool compact=false;size_t total_epic_size,total_size;int ret=0; +''' +suffix = r''' + *total=total_size; +out: + return ret; +} +int main(void) +{ + u8 request[112]={0},command[120],frame[144];size_t written,total; + struct ep ep={.wire_v2=true,.txbfr={.ready=true,.stride=64,.bufsz=256}}; + assert(!afkv2_encode_inline_command(command,sizeof(command),1,request,sizeof(request),32888)); + assert(!afkv2_encode_frame(frame,sizeof(frame),0,1,0,0xc0,AFKV2_COMMAND,0,command,sizeof(command),&written)); + assert(written==144 && written+sizeof(struct afk_qe)==160); + assert(!check_ring(&ep,0,0,sizeof(command),&total)); + assert(total==160 && ALIGN(total,64)==192); + ep.txbfr.bufsz=192;assert(check_ring(&ep,0,0,sizeof(command),&total)==-ENOSPC); + ep.txbfr.bufsz=128;assert(check_ring(&ep,0,0,sizeof(command),&total)==-EPROTO); + ep.txbfr.bufsz=256;assert(check_ring(&ep,192,0,sizeof(command),&total)==-ENOSPC); + assert(check_ring(&ep,0,1,sizeof(command),&total)==-EPROTO); + puts("PASS: full request 160 bytes / 192-byte aligned ring consumption; 256-byte empty ring accepted, smaller capacity rejected"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + block + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py new file mode 100644 index 0000000000000..92ca8857cc104 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the real command sender against synthetic ring and DMA operations.""" +import pathlib +import re +import sys + +source = pathlib.Path(sys.argv[1]).read_text() +match = re.search(r"static int afk_send_command_timeout_len\([^;]*?\)\n\{", source) +if not match: + raise RuntimeError("production command sender not found") +start, end, depth = match.start(), match.end(), 1 +while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include +#include "afkv2-codec.h" +#define SZ_1M 1048576U +#define GFP_KERNEL 0 +#define MAX_PENDING_CMDS 16 +#define cpu_to_le32(x) (x) +#define cpu_to_le64(x) (x) +#define kmalloc(n,f) malloc(n) +#define kfree(p) free(p) +#define READ_ONCE(x) (x) +#define spin_lock_irqsave(l,f) ((void)(l),(f)=0) +#define spin_unlock_irqrestore(l,f) ((void)(l),(void)(f)) +#define dma_rmb() do {} while(0) +#define dev_warn(...) do {} while(0) +#define EPIC_TYPE_COMMAND 3 +#define EPIC_CAT_COMMAND 0x30 +#define DECLARE_COMPLETION_ONSTACK(n) int n=0 +#define init_completion(p) (*(p)=0) +#define msecs_to_jiffies(x) (x) +#define wait_for_completion_timeout(p,t) ((void)(t),*(p)) +typedef u64 dma_addr_t; +struct epic_cmd { u32 retcode; u64 rxbuf,txbuf; u32 rxlen,txlen; u8 rxcookie,txcookie; }; +struct cmd_info { + u16 tag; u8 type; size_t reply_length; void *rxbuf,*txbuf; + dma_addr_t rxbuf_dma,txbuf_dma; size_t rxlen,txlen; u32 retcode; + bool done,free_on_ack; int *completion; +}; +struct dcp { void *dev; bool quiescing; }; +struct apple_dcp_afkep { struct dcp *dcp; bool wire_v2,inline_v2; int endpoint; }; +struct ops { const char *name; }; +struct apple_epic_service { + struct apple_dcp_afkep *ep; struct ops *ops; unsigned int channel; unsigned long cmd_map[1]; + int lock; bool enabled,torndown; u8 cmd_tag; struct cmd_info cmds[16]; +}; +static struct apple_epic_service *active; +static unsigned int frees, sends; +static int wire_fail, timeout, reply_status; +static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) { return s->enabled?s:NULL; } +static void afk_service_put(struct apple_epic_service *s) { (void)s; } +static void *dma_alloc_coherent(void *dev,size_t n,dma_addr_t *address,int flags) +{ void *p=calloc(1,n);(void)dev;(void)flags;*address=(uintptr_t)p;return p; } +static void dma_free_coherent(void *dev,size_t n,void *p,dma_addr_t a) +{ (void)dev;(void)n;assert((uintptr_t)p==a);free(p);frees++; } +static int bitmap_find_free_region(unsigned long *map,unsigned int n,int order) +{ (void)n;(void)order;assert(!*map);*map=1;return 0; } +static void bitmap_release_region(unsigned long *map,int i,int order) +{ (void)i;(void)order;*map=0; } +static int afk_send_epic(struct apple_dcp_afkep *ep,u32 channel,u16 tag,int type,int category,u8 subtype,const void *payload,size_t length) +{ + struct cmd_info *c=&active->cmds[0];struct afkv2_command parsed; + assert(channel==4 && type==3 && category==0x30 && subtype==0xc0); + sends++; + /* The payload-envelope mock bounds the command separately from its response. */ + assert(length<=128); + if(ep->inline_v2) { + assert(length==120); + assert(!afkv2_decode_command(payload,length,&parsed)); + assert(!parsed.oob && parsed.id==tag && parsed.value==32888 && parsed.inline_size==112); + assert(parsed.inline_data[0]==0x5a); + } else { + const struct epic_cmd *legacy=payload; + assert(length==sizeof(*legacy) && legacy->txlen==112 && legacy->rxlen==32888); + } + if(wire_fail)return -ENOSPC; + if(timeout)return 0; + c->reply_length=ep->inline_v2?4:32888;c->retcode=reply_status; + memset(c->rxbuf,0x77,c->reply_length);c->done=true;*c->completion=1; + return 0; +} +''' +suffix = r''' +int main(void) +{ + struct dcp dcp={0};struct apple_dcp_afkep ep={.dcp=&dcp,.wire_v2=true,.inline_v2=true}; + struct ops ops={.name="test"};struct apple_epic_service s={.ep=&ep,.ops=&ops,.channel=4,.enabled=true}; + u8 request[112]={0x5a},*output=calloc(1,32888);size_t received;u32 status=99; + active=&s; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==4 && status==0 && output[0]==0x77 && frees==2 && sends==1 && !s.cmd_map[0]); + /* Existing strict callers still reject a short inline result. */ + assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,NULL)==-EMSGSIZE); + reply_status=5;received=99; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(status==5 && received==0);reply_status=0; + wire_fail=1; + assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)==-ENOSPC && !s.cmd_map[0]); + wire_fail=0;ep.inline_v2=false; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==32888 && status==0); + ep.inline_v2=true;timeout=1; + assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)==-ETIMEDOUT); + assert(s.cmd_map[0] && s.cmds[0].free_on_ack && !s.cmds[0].completion); + /* Mock the late-reply owner's release, avoiding leaks in this host test. */ + free(s.cmds[0].rxbuf);free(s.cmds[0].txbuf);free(output); + puts("PASS: extracted production sender, 128-byte command envelope / 32888-byte reply, strict OOB and timeout ownership"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py new file mode 100644 index 0000000000000..68e3f5e6ba980 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production service-call body with a mocked wire reply.""" +import pathlib +import re +import sys + +source = pathlib.Path(sys.argv[1]).read_text() +match = re.search(r"static int afk_service_call_timeout_len\([^;]*?\)\n\{", source) +if not match: + raise RuntimeError("production service-call definition not found") +start = match.start() +end = match.end() +depth = 1 +while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include "afkv2-codec.h" +#define GFP_KERNEL 0 +#define kzalloc(n,f) calloc(1,n) +#define kfree(p) free(p) +#define max(a,b) ((a)>(b)?(a):(b)) +#define cpu_to_le16(x) (x) +#define cpu_to_le32(x) (x) +#define le16_to_cpu(x) (x) +#define le32_to_cpu(x) (x) +#define EPIC_SERVICE_CALL_MAGIC 0x69706378 +#define EPIC_SUBTYPE_STD_SERVICE 0xc0 +#define dev_warn(...) do {} while(0) +#define dev_dbg(...) do {} while(0) +struct epic_service_call { + u8 pad0[2]; u16 group; u32 command, data_len, magic; u8 pad1[48]; +}; +struct dcp { void *dev; }; +struct ops { const char *name; }; +struct ep { struct dcp *dcp; int endpoint; bool inline_v2; }; +struct apple_epic_service { struct ep *ep; struct ops *ops; u32 channel; }; +static int error_status, malformed, short_header, too_long, wire_error; +static int afk_send_command_timeout_len(struct apple_epic_service *s, u8 type, + const void *payload, size_t payload_len, void *output, size_t output_len, + u32 *retcode, unsigned int timeout_ms, size_t *received) +{ + struct epic_service_call *call=output; + (void)payload; + assert(payload_len==(s->ep->inline_v2?68:output_len)); + assert(call->data_len==(s->ep->inline_v2?4:4100)); + assert(type==0xc0 && timeout_ms==1000 && output_len>=68); + if(wire_error) return -ETIMEDOUT; + call->group=1;call->command=7;call->magic=EPIC_SERVICE_CALL_MAGIC; + call->data_len=too_long?5:4; + ((u8*)output)[64]=0xde;((u8*)output)[65]=0xad; + ((u8*)output)[66]=0xbe;((u8*)output)[67]=0xef; + if(malformed)call->command=8; + *retcode=error_status; + if(received)*received=short_header?63:68; + return 0; +} +''' +suffix = r''' +int main(void) +{ + struct ep ep={.inline_v2=true};struct apple_epic_service service={.ep=&ep}; + u8 input[4]={0}, output[128]={0}; size_t received=999; + assert(!afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + assert(received==4 && output[0]==0xde && output[3]==0xef && output[4]==0); + error_status=5;received=999; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)==-EINVAL && received==0); + error_status=0;malformed=1; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + malformed=0;short_header=1; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + short_header=0;too_long=1; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + too_long=0;wire_error=1; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)==-ETIMEDOUT); + wire_error=0; ep.inline_v2=false; + assert(!afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,NULL)); + puts("PASS: extracted production service-call actual request / legacy padding / short reply / status / context / length tests"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h index a5c3a3f44a591..23e11801f1e76 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h @@ -3,3 +3,8 @@ #define ENOSPC 28 #define E2BIG 7 #define EPROTO 71 +#define EIO 5 +#define EMSGSIZE 90 +#define ENODEV 19 +#define ETIMEDOUT 110 +#define ENOMEM 12 diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c new file mode 100644 index 0000000000000..7ce16fd2b6a8b --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#undef NDEBUG +#include +#include +#include "epic/dpavserv-edid.h" + +static unsigned int assertions; +#define CHECK(x) do { assertions++; assert(x); } while (0) + +static void checksum(u8 *block) +{ + u8 sum = 0; + unsigned int i; + + block[127] = 0; + for (i = 0; i < 127; i++) + sum += block[i]; + block[127] = -sum; +} + +int main(void) +{ + u8 command[192], packet[216], body[136] = {0}; + u8 edid[DPAV_EDID_HEADER_SIZE + 8 + 3 * 128] = {0}; + struct afkv2_frame frame; + struct afkv2_command reply; + size_t written, length = 0; + u8 *base = edid + DPAV_EDID_HEADER_SIZE + 8; + + CHECK(!afkv2_encode_inline_command(command, sizeof(command), 0x91, + body, sizeof(body), 400)); + CHECK(command[0] == 0 && command[1] == 0x91); + CHECK(afkv2_get32(command + 4) == 400); + CHECK(!afkv2_encode_frame(packet, sizeof(packet), 5, 4, 0, 0xc0, + AFKV2_RESPONSE, 0, command, 8 + sizeof(body), &written)); + /* Response uses status in the request-capacity field. */ + afkv2_put32(packet + AFKV2_HEADER_SIZE + 4, 0); + CHECK(!afkv2_decode_frame(packet, written, false, &frame)); + CHECK(!afkv2_decode_command(frame.payload, frame.payload_size, &reply)); + CHECK(!afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 135)); + CHECK(afkv2_check_inline_response(&frame, &reply, 5, 0xc0, 0x91, 136)); + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x92, 136)); + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc1, 0x91, 136)); + frame.category = AFKV2_COMMAND; + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + frame.category = AFKV2_RESPONSE; + frame.token = 1; + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + frame.token = 0; + frame.reserved = 1; + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + frame.reserved = 0; + reply.oob = true; + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + reply.oob = false; + /* The original OOB matcher must continue rejecting inline replies. */ + CHECK(afkv2_check_response(&frame, &reply, 4, 0xc0, 0x91, 0, 136, 0, 0)); + reply.value = 7; + CHECK(!afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + CHECK(reply.value != 0); /* Error status is preserved for the caller. */ + CHECK(afkv2_encode_inline_command(command, 7, 1, body, 1, 1)); + CHECK(afkv2_encode_inline_command(command, sizeof(command), 1, NULL, 1, 1)); + CHECK(afkv2_encode_inline_command(command, sizeof(command), 1, body, 1, + AFKV2_MAX_PAYLOAD + 1)); + + afkv2_put16(body + 2, 1); + afkv2_put32(body + 4, 7); + afkv2_put32(body + 8, 72); + afkv2_put32(body + 12, 0x69706378); + CHECK(!afkv2_service_reply_size(body, 136, 1, 7, 72, &length) && length == 72); + CHECK(afkv2_service_reply_size(body, 135, 1, 7, 72, &length)); + CHECK(afkv2_service_reply_size(body, 136, 1, 7, 71, &length)); + CHECK(afkv2_service_reply_size(body, 63, 1, 7, 72, &length)); + CHECK(afkv2_service_reply_size(body, 136, 2, 7, 72, &length)); + CHECK(afkv2_service_reply_size(body, 136, 1, 8, 72, &length)); + body[12] ^= 1; + CHECK(afkv2_service_reply_size(body, 136, 1, 7, 72, &length)); + + afkv2_put64(edid, EDID_BUF_SIZE); + afkv2_put64(edid + 32, 8 + 128); + memset(base + 1, 255, 6); + base[18] = 1; + base[19] = 4; + checksum(base); + CHECK(!dpavserv_edid_size(edid, 48 + 8 + 128, &length) && length == 128); + CHECK(dpavserv_edid_size(edid, 47, &length)); + CHECK(dpavserv_edid_size(edid, 48 + 8 + 127, &length)); + base[0] = 1; + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + base[0] = 0; + base[24] ^= 1; + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + base[24] ^= 1; + base[126] = 1; + checksum(base); + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + afkv2_put64(edid + 32, 8 + 256); + CHECK(!dpavserv_edid_size(edid, 48 + 8 + 256, &length) && length == 256); + /* HF-EEODB can supply more complete blocks than base[126] declares. */ + base[126] = 0; + checksum(base); + CHECK(!dpavserv_edid_size(edid, sizeof(edid), &length) && length == 256); + afkv2_put64(edid + 32, 8 + 255); + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + afkv2_put64(edid + 32, EDID_BUF_SIZE + 1); + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + afkv2_put64(edid + 32, 8 + 128); + afkv2_put64(edid, EDID_BUF_SIZE - 1); + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + printf("PASS: inline AFKV2 and EDID validation (%u checks)\n", assertions); + return 0; +} diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh index 4e13f6d05d986..4348eebc78246 100755 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh @@ -38,3 +38,45 @@ $afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ "$afkv2_tmp/owned.c" -o "$afkv2_tmp/owned" "$afkv2_tmp/owned" + +# Inline DPAV transport and actual-length EDID response validation. +# shellcheck disable=SC2086 +$afkv2_cc -std=c11 -Wall -Wextra -Werror -pedantic ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_dir/inline-edid.c" -o "$afkv2_tmp/inline-edid" +"$afkv2_tmp/inline-edid" +"$afkv2_python" "$afkv2_dir/extract-service.py" \ + "$afkv2_root/drivers/gpu/drm/apple/afk.c" "$afkv2_tmp/service.c" +# shellcheck disable=SC2086 +$afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/service.c" -o "$afkv2_tmp/service" +"$afkv2_tmp/service" + +"$afkv2_python" "$afkv2_dir/extract-send.py" \ + "$afkv2_root/drivers/gpu/drm/apple/afk.c" "$afkv2_tmp/send.c" +# shellcheck disable=SC2086 +$afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/send.c" -o "$afkv2_tmp/send" +"$afkv2_tmp/send" + +"$afkv2_python" "$afkv2_dir/extract-edid.py" \ + "$afkv2_root/drivers/gpu/drm/apple/epic/dpavservep.c" "$afkv2_tmp/edid.c" +# shellcheck disable=SC2086 +$afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/edid.c" -o "$afkv2_tmp/edid" +"$afkv2_tmp/edid" + +"$afkv2_python" "$afkv2_dir/extract-ring.py" \ + "$afkv2_root/drivers/gpu/drm/apple/afk.c" "$afkv2_tmp/ring.c" +"$afkv2_python" "$afkv2_dir/extract-notify.py" \ + "$afkv2_root/drivers/gpu/drm/apple/epic/dpavservep.c" "$afkv2_tmp/notify.c" +for afkv2_test in ring notify; do + # shellcheck disable=SC2086 + $afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/$afkv2_test.c" -o "$afkv2_tmp/$afkv2_test" + "$afkv2_tmp/$afkv2_test" +done From e9ee1ffbcb4ca2f3f103329804ebe76b6de741fc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:28:37 +1100 Subject: [PATCH 0938/1579] drm/apple: add bounded tracepoints for external output qualification Expose AFK headers before protocol decoding, submitted surface metadata, CTM values, and A423/A428 reply values through disabled-by-default tracepoints. Capture at most 32 received header bytes and do not dereference firmware-advertised addresses. Preserve all firmware requests and callback results. Host tests exercise the actual trace assignments and callback continuation with zero/short/null and unaligned inputs. These observers support investigating the unqualified DPAV announcement and output-color behavior without changing protocol policy. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/gpu/drm/apple/afk.c | 2 + drivers/gpu/drm/apple/iomfb_template.c | 19 ++- drivers/gpu/drm/apple/trace.h | 86 ++++++++++++++ tools/diagnostics/test-apple-display-trace.py | 108 ++++++++++++++++++ 5 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 tools/diagnostics/test-apple-display-trace.py diff --git a/MAINTAINERS b/MAINTAINERS index 90a72c4fd6b16..62910290250a9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2021,6 +2021,7 @@ F: tools/diagnostics/apple-j700-dp-test.c F: tools/diagnostics/test-apple-afk-admission.py F: tools/diagnostics/test-apple-atc-unwind.py F: tools/diagnostics/test-apple-dcp-readiness.py +F: tools/diagnostics/test-apple-display-trace.py F: tools/diagnostics/test-apple-external-sdr.py F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-d134.py diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 2d7fe02b99ab1..aab67b94afc64 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -894,6 +894,8 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, u8 *payload; size_t payload_size; + trace_afk_recv_header(ep, channel, type, data_size, data); + if (ep->wire_v2) { afk_v2_receive(ep, channel, type, data, data_size); return; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 31764f1bc3497..248ee966e33a6 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -68,8 +69,16 @@ static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, void *cookie) { const void *data = request; + unsigned int i; if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + for (i = 0; i < SWAP_SURFACES; i++) { + const struct dcp_surface *surface = &request->surf[i].base; + + if (!request->surf_null[i]) + trace_iomfb_surface(dcp, i, surface->format, surface->xfer_func, + surface->colorspace, surface->is_premultiplied); + } iomfb_serialize_present_h17p(&dcp->present_h17p, request); if (!iomfb_apply_backlight_h17p(dcp, request, &dcp->present_h17p)) { if (cb) @@ -2093,11 +2102,18 @@ void iomfb_present_backlight_h17p(struct apple_dcp *dcp) } #endif +static void dcp_post_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) +{ + trace_iomfb_control_result(dcp, 428, !!out, out ? get_unaligned_le32(out) : 0); + do_swap(dcp, out, cookie); +} + static void dcp_pre_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) { u32 identity = 0x10000; - dcp_pipe_set_428(dcp, false, &identity, do_swap, cookie); + trace_iomfb_control_result(dcp, 423, !!out, out ? get_unaligned_le32(out) : 0); + dcp_pipe_set_428(dcp, false, &identity, dcp_post_swap_a428, cookie); } #endif @@ -2375,6 +2391,7 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru mat.matrix[0] = mat.matrix[4] = mat.matrix[8] = 1LLU << 32; } + trace_iomfb_ctm(dcp, mat.location, mat.matrix, mat.matrix_null); #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) iomfb_set_matrix(dcp, false, &mat, dcp_pre_swap_a428, NULL); #else diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index d3c5106e5d66d..085f28ac44fae 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -205,6 +205,92 @@ TRACE_EVENT( print_epic_category(__entry->category), __entry->subtype, __entry->tag)); +/* Raw prefix before any protocol-specific decode; no peer address is followed. */ +TRACE_EVENT(afk_recv_header, + TP_PROTO(struct apple_dcp_afkep *ep, u32 channel, u32 type, + size_t data_size, const void *data), + TP_ARGS(ep, channel, type, data_size, data), + TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) + __field(u8, endpoint) + __field(u32, channel) + __field(u32, type) + __field(size_t, data_size) + __field(u8, captured) + __array(u8, header, 32)), + TP_fast_assign(__assign_str(devname); + __entry->endpoint = ep->endpoint; + __entry->channel = channel; + __entry->type = type; + __entry->data_size = data_size; + __entry->captured = data ? min_t(size_t, data_size, 32) : 0; + memset(__entry->header, 0, sizeof(__entry->header)); + if (__entry->captured) + memcpy(__entry->header, data, __entry->captured); + ), + TP_printk("%s: endpoint 0x%x channel 0x%x type 0x%x size %zu header[%u]=%s", + __get_str(devname), __entry->endpoint, __entry->channel, + __entry->type, __entry->data_size, __entry->captured, + __print_hex(__entry->header, __entry->captured)) +); + +TRACE_EVENT(iomfb_ctm, + TP_PROTO(struct apple_dcp *dcp, u32 location, const void *matrix, u8 matrix_null), + TP_ARGS(dcp, location, matrix, matrix_null), + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u32, location) + __field(u8, matrix_null) + __array(u64, matrix, 9)), + TP_fast_assign(__assign_str(devname); + __entry->location = location; + __entry->matrix_null = matrix_null; + if (matrix) + memcpy(__entry->matrix, matrix, sizeof(__entry->matrix)); + else + memset(__entry->matrix, 0, sizeof(__entry->matrix)); + ), + TP_printk("%s: location %u null %u matrix=%s", + __get_str(devname), __entry->location, __entry->matrix_null, + __print_array(__entry->matrix, 9, sizeof(u64))) +); + +TRACE_EVENT(iomfb_control_result, + TP_PROTO(struct apple_dcp *dcp, u32 method, bool valid, u32 result), + TP_ARGS(dcp, method, valid, result), + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u32, method) + __field(bool, valid) + __field(u32, result)), + TP_fast_assign(__assign_str(devname); + __entry->method = method; + __entry->valid = valid; + __entry->result = result; + ), + TP_printk("%s: A%u reply_valid %u result 0x%08x", + __get_str(devname), __entry->method, __entry->valid, __entry->result) +); + +TRACE_EVENT(iomfb_surface, + TP_PROTO(struct apple_dcp *dcp, u32 index, u32 format, u8 xfer_func, + u8 colorspace, u8 premultiplied), + TP_ARGS(dcp, index, format, xfer_func, colorspace, premultiplied), + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u32, index) + __field(u32, format) + __field(u8, xfer_func) + __field(u8, colorspace) + __field(u8, premultiplied)), + TP_fast_assign(__assign_str(devname); + __entry->index = index; + __entry->format = format; + __entry->xfer_func = xfer_func; + __entry->colorspace = colorspace; + __entry->premultiplied = premultiplied; + ), + TP_printk("%s: surface %u format 0x%08x xfer %u colorspace %u source_premul %u", + __get_str(devname), __entry->index, __entry->format, + __entry->xfer_func, __entry->colorspace, __entry->premultiplied) +); + TRACE_EVENT(iomfb_callback, TP_PROTO(struct apple_dcp *dcp, int tag, const char *name), TP_ARGS(dcp, tag, name), diff --git a/tools/diagnostics/test-apple-display-trace.py b/tools/diagnostics/test-apple-display-trace.py new file mode 100644 index 0000000000000..c4b5d041d6401 --- /dev/null +++ b/tools/diagnostics/test-apple-display-trace.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Host checks for bounded trace assignments and unchanged color callback flow.""" +import argparse +from pathlib import Path +import re +import subprocess +import tempfile + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--source-root', type=Path, default=Path(__file__).resolve().parents[2]) +args = parser.parse_args() +root = args.source_root / 'drivers/gpu/drm/apple' +trace = (root / 'trace.h').read_text() +template = (root / 'iomfb_template.c').read_text() + +def trace_assignment(name): + start = trace.index('TRACE_EVENT(' + name + ',') + start = trace.index('TP_fast_assign(', start) + len('TP_fast_assign(') + end, depth = start, 1 + while depth: + depth += (trace[end] == '(') - (trace[end] == ')') + end += 1 + return trace[start:end - 1] + +def function(name): + match = re.search(r'static void ' + name + r'\([^;]*?\)\n\{', template) + if not match: + raise RuntimeError('missing callback: ' + name) + end, depth = match.end(), 1 + while depth: + depth += (template[end] == '{') - (template[end] == '}') + end += 1 + return template[match.start():end] + +prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; typedef uint32_t u32; typedef uint64_t u64; +#define __assign_str(x) ((void)0) +#define min_t(t,a,b) ((t)(a)<(t)(b)?(t)(a):(t)(b)) +struct apple_dcp { int unused; }; +struct apple_dcp_afkep { u8 endpoint; }; +struct header_entry { u8 endpoint;u32 channel,type;size_t data_size;u8 captured,header[32]; }; +struct ctm_entry { u32 location;u8 matrix_null;u64 matrix[9]; }; +static unsigned int calls, swapped;static u32 methods[2],results[2];static bool valids[2]; +static void *swap_cookie,*reply_data;static u8 ack[5];static bool null_ack; +static u32 get_unaligned_le32(const void *p) { const u8 *b=p;return b[0]|(u32)b[1]<<8|(u32)b[2]<<16|(u32)b[3]<<24; } +static void trace_iomfb_control_result(struct apple_dcp *d,u32 m,bool valid,u32 result) +{ (void)d;assert(calls<2);methods[calls]=m;valids[calls]=valid;results[calls++]=result; } +static void do_swap(struct apple_dcp *d,void *out,void *cookie) +{ (void)d;swapped++;swap_cookie=cookie;reply_data=out; } +static void dcp_pipe_set_428(struct apple_dcp *d,bool oob,const u32 *identity, + void (*cb)(struct apple_dcp*,void*,void*),void *cookie) +{ assert(!oob && *identity==0x10000);cb(d,null_ack?NULL:ack+1,cookie); } +''' +code = prefix +code += 'static void capture(struct header_entry *__entry,struct apple_dcp_afkep *ep,u32 channel,u32 type,size_t data_size,const void *data) {\n' + trace_assignment('afk_recv_header') + '\n}\n' +code += 'static void capture_ctm(struct ctm_entry *__entry,u32 location,const void *matrix,u8 matrix_null) {\n' + trace_assignment('iomfb_ctm') + '\n}\n' +code += function('dcp_post_swap_a428') + '\n' + function('dcp_pre_swap_a428') +code += r''' +int main(void) +{ + struct apple_dcp_afkep ep={.endpoint=0x28};struct header_entry e;size_t n,i; + for(n=0;n<=64;n++) { + u8 *p=malloc(n?n:1);for(i=0;i Date: Mon, 12 Oct 2026 20:39:37 +1100 Subject: [PATCH 0939/1579] drm/apple: refine inline reply validation and trace silent DPAV rings Treat the transport/interface/reserved header fields as opaque, matching captured firmware responses and the owned-DMA validator, while retaining identity, type, tag, category and payload-bound checks. Add an optional read-only ring snapshot at the DPAV announcement timeout; it does not consume data or send another command. Host tests preserve response mismatch rejection, short-reply checks and timeout ownership, and verify the snapshot's enable, readiness, stop and pointer guards. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 31 +++++++++++++++ drivers/gpu/drm/apple/afk.h | 1 + drivers/gpu/drm/apple/afkv2-codec.h | 7 ++-- drivers/gpu/drm/apple/epic/dpavservep.c | 1 + drivers/gpu/drm/apple/trace.h | 34 +++++++++++++++++ tools/diagnostics/test-apple-display-trace.py | 38 +++++++++++++++++-- .../drivers/gpu/apple-afkv2/inline-edid.c | 12 +++++- 7 files changed, 116 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index aab67b94afc64..e1c87840a6c89 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -158,6 +158,37 @@ int afk_start(struct apple_dcp_afkep *ep) return 0; } +/* Read only host-owned ring headers; never consume an entry or notify firmware. */ +void afk_trace_ring_snapshot(struct apple_dcp_afkep *ep) +{ + unsigned long flags; + u32 rx_rptr = 0, rx_wptr = 0, tx_rptr = 0, tx_wptr = 0; + bool rx_ready, tx_ready; + + if (!ep || !ep->inline_v2 || !trace_afk_ring_snapshot_enabled()) + return; + spin_lock_irqsave(&ep->lock, flags); + if (ep->stopping) { + spin_unlock_irqrestore(&ep->lock, flags); + return; + } + rx_ready = READ_ONCE(ep->rxbfr.ready) && ep->rxbfr.rptr && ep->rxbfr.wptr; + tx_ready = READ_ONCE(ep->txbfr.ready) && ep->txbfr.rptr && ep->txbfr.wptr; + dma_rmb(); + if (rx_ready) { + rx_rptr = le32_to_cpu(READ_ONCE(*ep->rxbfr.rptr)); + rx_wptr = le32_to_cpu(READ_ONCE(*ep->rxbfr.wptr)); + } + if (tx_ready) { + tx_rptr = le32_to_cpu(READ_ONCE(*ep->txbfr.rptr)); + tx_wptr = le32_to_cpu(READ_ONCE(*ep->txbfr.wptr)); + } + trace_afk_ring_snapshot(ep, rx_ready, rx_rptr, rx_wptr, + READ_ONCE(ep->rxbfr.bufsz), tx_ready, tx_rptr, + tx_wptr, READ_ONCE(ep->txbfr.bufsz)); + spin_unlock_irqrestore(&ep->lock, flags); +} + static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) { u32 size = FIELD_GET(GETBUF_SIZE, message) << BLOCK_SHIFT; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 45fe9db64d30f..36b07a556f6d4 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -364,6 +364,7 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, int afk_v2_copy_owned(struct apple_epic_service *service, u64 dva, void *buffer, size_t size, bool to_peer); int afk_start(struct apple_dcp_afkep *ep); +void afk_trace_ring_snapshot(struct apple_dcp_afkep *ep); void afk_shutdown(struct apple_dcp_afkep *ep); void afk_quiesce(struct apple_dcp_afkep *ep); int afk_receive_message(struct apple_dcp_afkep *ep, u64 message); diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h index 9ef52b2e70ea2..a06a1efd5fc7e 100644 --- a/drivers/gpu/drm/apple/afkv2-codec.h +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -270,7 +270,9 @@ static inline int afkv2_check_response(const struct afkv2_frame *frame, return 0; } -/* Kept separate so DPAV inline replies cannot loosen owned-DMA checks. */ +/* Kept separate so DPAV inline replies cannot loosen owned-DMA checks. + * Transport/interface flags and the reserved word are opaque firmware data. + */ static inline int afkv2_check_inline_response(const struct afkv2_frame *frame, const struct afkv2_command *response, u16 interface, u8 type, u8 id, @@ -278,8 +280,7 @@ static inline int afkv2_check_inline_response(const struct afkv2_frame *frame, { if (!frame || !response || frame->category != AFKV2_RESPONSE || frame->interface != interface || frame->type != type || response->id != id || - frame->token || frame->transport_flags || frame->interface_flags || - frame->reserved || response->oob || response->inline_size > capacity) + frame->token || response->oob || response->inline_size > capacity) return -EPROTO; return 0; } diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 1b4019cc159f9..64e836ef48811 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -344,6 +344,7 @@ int dpavservep_init(struct apple_dcp *dcp) ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, msecs_to_jiffies(1000)); if (!ret) { + afk_trace_ring_snapshot(ep); dev_warn(dcp->dev, "DCPAVSERV: timed out waiting for EDID service announcement\n"); return -ETIMEDOUT; } diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index 085f28ac44fae..d19609f116f2c 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -233,6 +233,40 @@ TRACE_EVENT(afk_recv_header, __print_hex(__entry->header, __entry->captured)) ); +TRACE_EVENT(afk_ring_snapshot, + TP_PROTO(struct apple_dcp_afkep *ep, bool rx_ready, u32 rx_rptr, + u32 rx_wptr, u32 rx_size, bool tx_ready, u32 tx_rptr, + u32 tx_wptr, u32 tx_size), + TP_ARGS(ep, rx_ready, rx_rptr, rx_wptr, rx_size, + tx_ready, tx_rptr, tx_wptr, tx_size), + TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) + __field(u8, endpoint) + __field(bool, rx_ready) + __field(u32, rx_rptr) + __field(u32, rx_wptr) + __field(u32, rx_size) + __field(bool, tx_ready) + __field(u32, tx_rptr) + __field(u32, tx_wptr) + __field(u32, tx_size)), + TP_fast_assign(__assign_str(devname); + __entry->endpoint = ep->endpoint; + __entry->rx_ready = rx_ready; + __entry->rx_rptr = rx_rptr; + __entry->rx_wptr = rx_wptr; + __entry->rx_size = rx_size; + __entry->tx_ready = tx_ready; + __entry->tx_rptr = tx_rptr; + __entry->tx_wptr = tx_wptr; + __entry->tx_size = tx_size; + ), + TP_printk("%s: endpoint 0x%x rx_ready %u rx_rptr %#x rx_wptr %#x rx_size %#x tx_ready %u tx_rptr %#x tx_wptr %#x tx_size %#x", + __get_str(devname), __entry->endpoint, __entry->rx_ready, + __entry->rx_rptr, __entry->rx_wptr, __entry->rx_size, + __entry->tx_ready, __entry->tx_rptr, __entry->tx_wptr, + __entry->tx_size) +); + TRACE_EVENT(iomfb_ctm, TP_PROTO(struct apple_dcp *dcp, u32 location, const void *matrix, u8 matrix_null), TP_ARGS(dcp, location, matrix, matrix_null), diff --git a/tools/diagnostics/test-apple-display-trace.py b/tools/diagnostics/test-apple-display-trace.py index c4b5d041d6401..f0c32db9e769f 100644 --- a/tools/diagnostics/test-apple-display-trace.py +++ b/tools/diagnostics/test-apple-display-trace.py @@ -13,6 +13,7 @@ root = args.source_root / 'drivers/gpu/drm/apple' trace = (root / 'trace.h').read_text() template = (root / 'iomfb_template.c').read_text() +afk = (root / 'afk.c').read_text() def trace_assignment(name): start = trace.index('TRACE_EVENT(' + name + ',') @@ -44,7 +45,19 @@ def function(name): #define __assign_str(x) ((void)0) #define min_t(t,a,b) ((t)(a)<(t)(b)?(t)(a):(t)(b)) struct apple_dcp { int unused; }; -struct apple_dcp_afkep { u8 endpoint; }; +struct ring { bool ready; u32 *rptr,*wptr,bufsz; }; +struct apple_dcp_afkep { u8 endpoint; bool inline_v2,stopping; int lock; struct ring rxbfr,txbfr; }; +#define READ_ONCE(x) (x) +#define le32_to_cpu(x) (x) +#define spin_lock_irqsave(l,f) ((void)(l),(f)=0) +#define spin_unlock_irqrestore(l,f) ((void)(l),(void)(f)) +#define dma_rmb() do {} while(0) +static bool enabled; +static unsigned int snapshots; +static u32 rx_seen,rx_write_seen,tx_seen,tx_write_seen; +static bool trace_afk_ring_snapshot_enabled(void) { return enabled; } +static void trace_afk_ring_snapshot(struct apple_dcp_afkep *ep,bool rr,u32 rp,u32 wp,u32 rs,bool tr,u32 tp,u32 tw,u32 ts) +{ (void)ep;(void)rr;(void)rs;(void)tr;(void)ts;snapshots++;rx_seen=rp;rx_write_seen=wp;tx_seen=tp;tx_write_seen=tw; } struct header_entry { u8 endpoint;u32 channel,type;size_t data_size;u8 captured,header[32]; }; struct ctm_entry { u32 location;u8 matrix_null;u64 matrix[9]; }; static unsigned int calls, swapped;static u32 methods[2],results[2];static bool valids[2]; @@ -58,7 +71,13 @@ def function(name): void (*cb)(struct apple_dcp*,void*,void*),void *cookie) { assert(!oob && *identity==0x10000);cb(d,null_ack?NULL:ack+1,cookie); } ''' -code = prefix +match = re.search(r"void afk_trace_ring_snapshot\([^;]*?\)\n\{", afk) +assert match +end, depth = match.end(), 1 +while depth: + depth += (afk[end] == '{') - (afk[end] == '}') + end += 1 +code = prefix + afk[match.start():end] + '\n' code += 'static void capture(struct header_entry *__entry,struct apple_dcp_afkep *ep,u32 channel,u32 type,size_t data_size,const void *data) {\n' + trace_assignment('afk_recv_header') + '\n}\n' code += 'static void capture_ctm(struct ctm_entry *__entry,u32 location,const void *matrix,u8 matrix_null) {\n' + trace_assignment('iomfb_ctm') + '\n}\n' code += function('dcp_post_swap_a428') + '\n' + function('dcp_pre_swap_a428') @@ -95,7 +114,20 @@ def function(name): assert(calls==2 && !valids[0] && !valids[1] && !results[0] && !results[1]); assert(swapped==2 && swap_cookie==&cookie && reply_data==NULL); } - puts("PASS: actual trace assignments bounded/null-safe; CTM nine coefficients; A423/A428 callback flow preserved"); + { + u32 r=128,w=256,tr=64,tw=192;ep.inline_v2=true; + afk_trace_ring_snapshot(&ep);assert(!snapshots); + enabled=true;ep.rxbfr.rptr=(u32*)1;ep.txbfr.rptr=(u32*)1; + afk_trace_ring_snapshot(&ep);assert(snapshots==1 && !rx_seen && !tx_seen); + ep.rxbfr=(struct ring){.ready=true,.rptr=&r,.wptr=&w,.bufsz=8192}; + ep.txbfr=(struct ring){.ready=true,.rptr=&tr,.wptr=&tw,.bufsz=8192}; + afk_trace_ring_snapshot(&ep);assert(snapshots==2 && rx_seen==128 && rx_write_seen==256 && tx_seen==64 && tx_write_seen==192); + assert(r==128 && w==256 && tr==64 && tw==192); + ep.stopping=true;afk_trace_ring_snapshot(&ep);assert(snapshots==2); + ep.stopping=false;ep.inline_v2=false;afk_trace_ring_snapshot(&ep);assert(snapshots==2); + afk_trace_ring_snapshot(NULL);assert(snapshots==2); + } + puts("PASS: bounded trace/callback flow and read-only ring snapshot enable/ready/stop guards"); } ''' with tempfile.TemporaryDirectory(prefix='apple-display-trace-') as tmp: diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c index 7ce16fd2b6a8b..b711adf804794 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c @@ -48,8 +48,16 @@ int main(void) frame.token = 1; CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); frame.token = 0; - frame.reserved = 1; - CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + /* R30 responses carry these opaque nonzero header fields. */ + frame.transport_flags = 0x18; + frame.interface_flags = 0x0400; + frame.reserved = 0x3607ebe0; + CHECK(!afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + /* Context and bounds remain mandatory even with opaque header values. */ + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x92, 136)); + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 135)); + frame.transport_flags = 0; + frame.interface_flags = 0; frame.reserved = 0; reply.oob = true; CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); From cb49a7e2b4ba09ac47e6b780692f33aeac241421 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:42:08 +1100 Subject: [PATCH 0940/1579] drm/apple: honor external timing color exclusions before SDR selection Exclude firmware-listed unsafe and DSC-required color IDs before choosing an external H17P SDR candidate. Read restrictions independently of property order and reject malformed or duplicate lists instead of silently treating them as empty. Preserve internal and legacy parsing. The captured dock route permits 60 Hz at 10-bit and 120 Hz at 8-bit; its 144/180 Hz timings have no eligible nonvirtual SDR candidate after exclusions. This is metadata admission, not a new physical-link qualification or a global refresh cap. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 104 ++++++++++++++++++- tools/diagnostics/test-apple-external-sdr.py | 61 +++++++++++ 2 files changed, 162 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 73e7e1cf315c6..691bc91e3542b 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -411,8 +411,92 @@ static unsigned int external_sdr_rank(const struct color_mode *color) } } +/* Firmware lists color element IDs which this timing cannot safely use. */ +#define DCP_TIMING_COLOR_ID_LIMIT 64 + +struct timing_color_ids { + u32 ids[DCP_TIMING_COLOR_ID_LIMIT]; + unsigned int count; +}; + +struct timing_color_restrictions { + struct timing_color_ids unsafe; + struct timing_color_ids dsc_required; +}; + +static bool timing_color_contains(const struct timing_color_ids *ids, u32 id) +{ + unsigned int i; + + for (i = 0; i < ids->count; i++) + if (ids->ids[i] == id) + return true; + return false; +} + +static int parse_timing_color_ids(struct dcp_parse_ctx *handle, + struct timing_color_ids *ids) +{ + struct iterator it; + int ret; + + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + if (it.len > DCP_TIMING_COLOR_ID_LIMIT) + return -E2BIG; + for (; it.idx < it.len; it.idx++) { + s64 id; + + ret = parse_int_bound(handle, &id, 0, U32_MAX); + if (ret) + return ret; + if (timing_color_contains(ids, id)) + return -EINVAL; + ids->ids[ids->count++] = id; + } + return 0; +} + +static int parse_timing_color_restrictions(struct dcp_parse_ctx *handle, + struct timing_color_restrictions *limits) +{ + struct iterator it; + bool unsafe_seen = false, dsc_seen = false, colors_seen = false; + int ret; + + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char *key = parse_string(handle); + + if (IS_ERR(key)) + return PTR_ERR(key); + if (!strcmp(key, "UnsafeColorElementIDs")) { + ret = unsafe_seen ? -EINVAL : + parse_timing_color_ids(handle, &limits->unsafe); + unsafe_seen = true; + } else if (!strcmp(key, "DSCRequiredColorElementIDs")) { + ret = dsc_seen ? -EINVAL : + parse_timing_color_ids(handle, &limits->dsc_required); + dsc_seen = true; + } else if (!strcmp(key, "ColorModes")) { + ret = colors_seen ? -EINVAL : skip(handle); + colors_seen = true; + } else { + ret = skip(handle); + } + kfree(key); + if (ret) + return ret; + } + return 0; +} + static int parse_color_modes(struct dcp_parse_ctx *handle, - struct dcp_display_mode *out, bool external_sdr) + struct dcp_display_mode *out, bool external_sdr, + const struct timing_color_restrictions *limits) { struct iterator outer_it; unsigned int best_sdr_rank = 0; @@ -484,8 +568,13 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, cmode.pixel_encoding); if (external_sdr) { - unsigned int rank = external_sdr_rank(&cmode); + unsigned int rank; + if (cmode.id <= U32_MAX && + (timing_color_contains(&limits->unsafe, cmode.id) || + timing_color_contains(&limits->dsc_required, cmode.id))) + continue; + rank = external_sdr_rank(&cmode); if (!rank || rank < best_sdr_rank) continue; if (rank > best_sdr_rank) @@ -531,6 +620,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, int ret = 0; struct iterator it; struct dimension horiz = { .active = -1 }, vert = { .active = -1 }; + struct timing_color_restrictions limits = {}; s64 min_vrr = 0, max_vrr = 0; u64 clock; s64 id = -1; @@ -546,6 +636,14 @@ static int parse_mode(struct dcp_parse_ctx *handle, }; *score = -1; + if (external_sdr) { + struct dcp_parse_ctx scan = *handle; + + /* ColorModes can precede either list: collect restrictions first. */ + ret = parse_timing_color_restrictions(&scan, &limits); + if (ret) + return ret; + } ret = iterator_begin(handle, &it, true); if (ret) return ret; @@ -566,7 +664,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, else if (!strcmp(key, "MaximumVariableRefreshRate")) ret = parse_int_bound(it.handle, &max_vrr, 0, U32_MAX); else if (!strcmp(key, "ColorModes")) - ret = parse_color_modes(it.handle, out, external_sdr); + ret = parse_color_modes(it.handle, out, external_sdr, &limits); else if (!strcmp(key, "ID")) ret = parse_int_bound(it.handle, &id, 0, U32_MAX); else if (!strcmp(key, "IsVirtual")) diff --git a/tools/diagnostics/test-apple-external-sdr.py b/tools/diagnostics/test-apple-external-sdr.py index f72c26e38108c..1ece679366267 100644 --- a/tools/diagnostics/test-apple-external-sdr.py +++ b/tools/diagnostics/test-apple-external-sdr.py @@ -24,6 +24,10 @@ print('Production mode parser SHA256:', hashlib.sha256((header + body).encode()).hexdigest()) +class Pairs(list): + """A wire dictionary which preserves duplicate keys for rejection tests.""" + + def wire(value, out): out += b'\0' * (-len(out) % 4) if isinstance(value, bool): @@ -35,6 +39,11 @@ def wire(value, out): out += struct.pack(' #include From 9303ecf2c6ab901cdf9697b24d21632807c1a7c3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:49:18 +1100 Subject: [PATCH 0941/1579] drm/apple: derive bounded external monitor metadata for initial attachment Expose the firmware-reported identity, primaries and preferred timing while raw DPAV EDID is unavailable on the experimental H17P external path. Keep this synthetic base block out of the raw cache, require complete metadata from the same initial connection, and invalidate it on route or cable loss. Do not synthesize extra timings, audio, HDR, VRR or link capabilities. Serialize EDID property and cache changes with connector probing. Defer commit-context disconnect cleanup through a persistent invalidation flag, so rapid reconnection cannot lose it or invert DRM modeset lock ordering. Include bounded-parser, independent EDID decoding and concurrent lifetime tests. Hardware qualification remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 2 + drivers/gpu/drm/apple/Makefile | 2 +- drivers/gpu/drm/apple/connector.h | 3 + drivers/gpu/drm/apple/dcp-display-metadata.c | 204 +++++++++ drivers/gpu/drm/apple/dcp-display-metadata.h | 24 ++ drivers/gpu/drm/apple/dcp-internal.h | 23 + drivers/gpu/drm/apple/dcp-metadata-epoch.h | 48 +++ drivers/gpu/drm/apple/dcp.c | 183 +++++++- drivers/gpu/drm/apple/iomfb.c | 66 ++- drivers/gpu/drm/apple/iomfb_template.c | 20 +- drivers/gpu/drm/apple/parser.c | 274 +++++++++++- drivers/gpu/drm/apple/parser.h | 6 + .../display-attributes-fixture.json | 23 + .../test-apple-display-metadata-lifetime.py | 395 ++++++++++++++++++ .../test-apple-display-metadata.py | 363 ++++++++++++++++ tools/diagnostics/test-apple-external-sdr.py | 6 +- 16 files changed, 1625 insertions(+), 17 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-display-metadata.c create mode 100644 drivers/gpu/drm/apple/dcp-display-metadata.h create mode 100644 drivers/gpu/drm/apple/dcp-metadata-epoch.h create mode 100644 tools/diagnostics/display-attributes-fixture.json create mode 100644 tools/diagnostics/test-apple-display-metadata-lifetime.py create mode 100644 tools/diagnostics/test-apple-display-metadata.py diff --git a/MAINTAINERS b/MAINTAINERS index 62910290250a9..cff04513aaad7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2018,9 +2018,11 @@ F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs F: tools/diagnostics/apple-dptx-training-test.c F: tools/diagnostics/apple-j700-dp-test.c +F: tools/diagnostics/display-attributes-fixture.json F: tools/diagnostics/test-apple-afk-admission.py F: tools/diagnostics/test-apple-atc-unwind.py F: tools/diagnostics/test-apple-dcp-readiness.py +F: tools/diagnostics/test-apple-display-metadata*.py F: tools/diagnostics/test-apple-display-trace.py F: tools/diagnostics/test-apple-external-sdr.py F: tools/diagnostics/test-apple-j700-crossbar.py diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 4980b2863225b..601689ee3b172 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -6,7 +6,7 @@ appledrm-y := apple_drv.o appledrm-y += afk.o dcp.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o -appledrm-y += connector.o +appledrm-y += connector.o dcp-display-metadata.o appledrm-y += ibootep.o appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index 4cb58502c1ef8..6f73e975ebdc1 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -43,6 +43,9 @@ struct apple_connector { u32 candidate_crtcs; const struct drm_edid *drm_edid; + bool drm_edid_firmware_derived; + bool edid_invalidation_pending; + u64 drm_edid_generation; /* Workqueue for sending hotplug events to the associated device */ struct work_struct hotplug_wq; diff --git a/drivers/gpu/drm/apple/dcp-display-metadata.c b/drivers/gpu/drm/apple/dcp-display-metadata.c new file mode 100644 index 0000000000000..cc4c69736d098 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-display-metadata.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include + +#include "dcp-display-metadata.h" + +static int dcp_metadata_timing(struct detailed_timing *timing, + const struct drm_display_mode *mode, + const struct dcp_display_metadata *metadata) +{ + struct detailed_pixel_timing *pixel = &timing->data.pixel_data; + u32 hblank, vblank, hoff, voff, hsync, vsync; + u32 width = metadata->width_cm * 10, height = metadata->height_cm * 10; + u32 hflags = mode->flags & (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC); + u32 vflags = mode->flags & (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC); + + if (mode->clock <= 0 || mode->clock > 655350 || mode->clock % 10 || + mode->hdisplay <= 0 || mode->vdisplay <= 0 || + mode->hdisplay > 4095 || mode->vdisplay > 4095 || + mode->htotal <= mode->hdisplay || mode->vtotal <= mode->vdisplay || + mode->hsync_start < mode->hdisplay || mode->hsync_end <= mode->hsync_start || + mode->hsync_end > mode->htotal || mode->vsync_start < mode->vdisplay || + mode->vsync_end <= mode->vsync_start || mode->vsync_end > mode->vtotal || + (hflags != DRM_MODE_FLAG_PHSYNC && hflags != DRM_MODE_FLAG_NHSYNC) || + (vflags != DRM_MODE_FLAG_PVSYNC && vflags != DRM_MODE_FLAG_NVSYNC) || + (mode->flags & ~(DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC | + DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC))) + return -EINVAL; + hblank = mode->htotal - mode->hdisplay; + vblank = mode->vtotal - mode->vdisplay; + hoff = mode->hsync_start - mode->hdisplay; + voff = mode->vsync_start - mode->vdisplay; + hsync = mode->hsync_end - mode->hsync_start; + vsync = mode->vsync_end - mode->vsync_start; + if (hblank > 4095 || vblank > 4095 || hoff > 1023 || hsync > 1023 || + voff > 63 || vsync > 63 || width > 4095 || height > 4095) + return -EINVAL; + + timing->pixel_clock = cpu_to_le16(mode->clock / 10); + pixel->hactive_lo = mode->hdisplay; + pixel->hblank_lo = hblank; + pixel->hactive_hblank_hi = (mode->hdisplay >> 8) << 4 | hblank >> 8; + pixel->vactive_lo = mode->vdisplay; + pixel->vblank_lo = vblank; + pixel->vactive_vblank_hi = (mode->vdisplay >> 8) << 4 | vblank >> 8; + pixel->hsync_offset_lo = hoff; + pixel->hsync_pulse_width_lo = hsync; + pixel->vsync_offset_pulse_width_lo = (voff & 15) << 4 | (vsync & 15); + pixel->hsync_vsync_offset_pulse_width_hi = (hoff >> 8) << 6 | (hsync >> 8) << 4 | + (voff >> 4) << 2 | vsync >> 4; + pixel->width_mm_lo = width; + pixel->height_mm_lo = height; + pixel->width_height_mm_hi = (width >> 8) << 4 | height >> 8; + pixel->misc = DRM_EDID_PT_SEPARATE_SYNC; + if (hflags == DRM_MODE_FLAG_PHSYNC) + pixel->misc |= DRM_EDID_PT_HSYNC_POSITIVE; + if (vflags == DRM_MODE_FLAG_PVSYNC) + pixel->misc |= DRM_EDID_PT_VSYNC_POSITIVE; + return 0; +} + +static int dcp_metadata_text(struct detailed_timing *timing, u8 type, const char *text) +{ + size_t length = strnlen(text, 14); + u8 *out = timing->data.other_data.data.str.str; + + if (!length || length > 13) + return -EINVAL; + for (size_t i = 0; i < length; i++) + if (text[i] < 0x20 || text[i] > 0x7e) + return -EINVAL; + timing->data.other_data.type = type; + memset(out, ' ', 13); + memcpy(out, text, length); + if (length < 13) + out[length] = '\n'; + return 0; +} + +static s64 dcp_metadata_side(const u32 *a, const u32 *b, const u32 *point) +{ + return ((s64)b[0] - a[0]) * ((s64)point[1] - a[1]) - + ((s64)b[1] - a[1]) * ((s64)point[0] - a[0]); +} + +/* Build only a base block: the authoritative firmware mode list stays separate. */ +static int dcp_metadata_block(struct edid *edid, const struct dcp_display_metadata *metadata, + const struct drm_display_mode *mode) +{ + u32 coordinate[8], manufacturer = 0, gamma; + u8 checksum = 0; + int ret; + + if (!metadata->width_cm || metadata->width_cm > 255 || + !metadata->height_cm || metadata->height_cm > 255 || + metadata->product > U16_MAX || metadata->week > 53 || + metadata->year < 1990 || metadata->year > 2245 || + metadata->gamma < 65536 || metadata->gamma > (354U << 16) / 100 || + strnlen(metadata->manufacturer, 4) != 3) + return -EINVAL; + for (unsigned int i = 0; i < 3; i++) { + if (metadata->manufacturer[i] < 'A' || metadata->manufacturer[i] > 'Z') + return -EINVAL; + manufacturer = manufacturer << 5 | (metadata->manufacturer[i] - '@'); + } + for (unsigned int i = 0; i < 8; i++) { + u32 value = i < 6 ? metadata->chromaticity[i / 2][i % 2] : metadata->white[i - 6]; + + if (!value || value >= 65536) + return -EINVAL; + coordinate[i] = DIV_ROUND_CLOSEST(value, 64); + if (!coordinate[i] || coordinate[i] > 1023) + return -EINVAL; + if ((i & 1) && coordinate[i - 1] + coordinate[i] > 1024) + return -EINVAL; + } + /* Degenerate primaries or a white point outside their triangle are not usable. */ + { + s64 area = dcp_metadata_side(coordinate, coordinate + 2, coordinate + 4); + + if (!area) + return -EINVAL; + for (unsigned int i = 0; i < 3; i++) { + s64 side = dcp_metadata_side(coordinate + 2 * i, + coordinate + 2 * ((i + 1) % 3), coordinate + 6); + + if ((area > 0 && side < 0) || (area < 0 && side > 0)) + return -EINVAL; + } + } + gamma = DIV_ROUND_CLOSEST(metadata->gamma * 100, 65536) - 100; + if (gamma > 254) + return -EINVAL; + + memset(edid, 0, sizeof(*edid)); + memcpy(edid->header, "\x00\xff\xff\xff\xff\xff\xff\x00", 8); + edid->product_id.manufacturer_name = cpu_to_be16(manufacturer); + edid->product_id.product_code = cpu_to_le16(metadata->product); + edid->product_id.serial_number = cpu_to_le32(metadata->serial); + edid->product_id.week_of_manufacture = metadata->week; + edid->product_id.year_of_manufacture = metadata->year - 1990; + edid->version = 1; + edid->revision = 4; + /* No claim about connector interface, bit depth, HDR, audio or VRR. */ + edid->input = DRM_EDID_INPUT_DIGITAL; + edid->width_cm = metadata->width_cm; + edid->height_cm = metadata->height_cm; + edid->gamma = gamma; + edid->features = DRM_EDID_FEATURE_PREFERRED_TIMING | DRM_EDID_FEATURE_RGB; + edid->red_green_lo = (coordinate[0] & 3) << 6 | (coordinate[1] & 3) << 4 | + (coordinate[2] & 3) << 2 | (coordinate[3] & 3); + edid->blue_white_lo = (coordinate[4] & 3) << 6 | (coordinate[5] & 3) << 4 | + (coordinate[6] & 3) << 2 | (coordinate[7] & 3); + edid->red_x = coordinate[0] >> 2; + edid->red_y = coordinate[1] >> 2; + edid->green_x = coordinate[2] >> 2; + edid->green_y = coordinate[3] >> 2; + edid->blue_x = coordinate[4] >> 2; + edid->blue_y = coordinate[5] >> 2; + edid->white_x = coordinate[6] >> 2; + edid->white_y = coordinate[7] >> 2; + memset(edid->standard_timings, 1, sizeof(edid->standard_timings)); + ret = dcp_metadata_timing(&edid->detailed_timings[0], mode, metadata); + if (!ret) + ret = dcp_metadata_text(&edid->detailed_timings[1], EDID_DETAIL_MONITOR_NAME, + metadata->name); + if (!ret) + ret = dcp_metadata_text(&edid->detailed_timings[2], EDID_DETAIL_MONITOR_STRING, + "DCP metadata"); + if (!ret) + ret = dcp_metadata_text(&edid->detailed_timings[3], + metadata->serial_text[0] ? EDID_DETAIL_MONITOR_SERIAL : + EDID_DETAIL_MONITOR_STRING, + metadata->serial_text[0] ? + metadata->serial_text : "FW derived"); + if (ret) + return ret; + for (unsigned int i = 0; i < sizeof(*edid) - 1; i++) + checksum += ((const u8 *)edid)[i]; + edid->checksum = -checksum; + return 0; +} + +const struct drm_edid *dcp_metadata_edid(const struct dcp_display_metadata *metadata, + const struct drm_display_mode *mode) +{ + struct edid block; + const struct drm_edid *edid; + int ret = dcp_metadata_block(&block, metadata, mode); + + if (ret) + return ERR_PTR(ret); + edid = drm_edid_alloc(&block, sizeof(block)); + if (!edid) + return ERR_PTR(-ENOMEM); + if (!drm_edid_valid(edid)) { + drm_edid_free(edid); + return ERR_PTR(-EINVAL); + } + return edid; +} diff --git a/drivers/gpu/drm/apple/dcp-display-metadata.h b/drivers/gpu/drm/apple/dcp-display-metadata.h new file mode 100644 index 0000000000000..537994420ebd6 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-display-metadata.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_DISPLAY_METADATA_H +#define APPLE_DCP_DISPLAY_METADATA_H + +#include + +/* Values reported by DisplayAttributes, not a copy of the sink's EDID. */ +struct dcp_display_metadata { + u32 width_cm, height_cm; + u32 chromaticity[3][2]; + u32 white[2]; + u32 gamma; + u32 product, serial, week, year; + char manufacturer[4]; + char name[14]; + char serial_text[14]; +}; + +struct drm_edid; +struct drm_display_mode; +const struct drm_edid *dcp_metadata_edid(const struct dcp_display_metadata *metadata, + const struct drm_display_mode *mode); + +#endif diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 48f4112151dd3..cd265c6ced986 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -18,6 +18,8 @@ #include "dptxep.h" #include "dcp_backlight.h" +#include "dcp-display-metadata.h" +#include "dcp-metadata-epoch.h" #include "iomfb.h" #include "iomfb_h17p.h" #include "iomfb-state.h" @@ -103,6 +105,9 @@ enum { struct dcp_chunks { size_t length; void *data; + size_t received; + bool contiguous; + struct dcp_metadata_ticket metadata_ticket; }; #define DCP_MAX_MAPPINGS (128) /* should be enough */ @@ -264,6 +269,15 @@ struct apple_dcp { /* Active chunked transfer. There can only be one at a time. */ struct dcp_chunks chunks; + struct { + /* Protects connection tickets and cached metadata snapshots. */ + spinlock_t lock; + struct dcp_metadata_epoch epoch; + struct dcp_display_metadata attributes; + struct drm_display_mode preferred; + struct dcp_metadata_ticket attributes_ticket, timing_ticket; + bool attributes_valid, timing_valid, hpd_seen; + } metadata; /* Queued swap. Owned by the DCP to avoid per-swap memory allocation */ union { @@ -452,6 +466,15 @@ struct apple_dcp { } iomfb; }; +void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected); +void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected); +void dcp_firmware_metadata_begin(struct apple_dcp *dcp); +void dcp_firmware_metadata_attributes(struct apple_dcp *dcp, + const struct dcp_display_metadata *attributes); +void dcp_firmware_metadata_timing(struct apple_dcp *dcp); +bool dcp_firmware_metadata_current(struct apple_dcp *dcp, u64 generation); +const struct drm_edid *dcp_firmware_edid(struct apple_dcp *dcp, u64 *generation); + void iomfb_scanout_complete_h17p(struct apple_dcp *dcp); void iomfb_present_backlight_h17p(struct apple_dcp *dcp); bool iomfb_present_brightness_only_h17p(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp-metadata-epoch.h b/drivers/gpu/drm/apple/dcp-metadata-epoch.h new file mode 100644 index 0000000000000..c2ab0c22a74e0 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-metadata-epoch.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_METADATA_EPOCH_H +#define APPLE_DCP_METADATA_EPOCH_H + +#include + +/* Caller serializes these fields with the metadata snapshot lock. */ +struct dcp_metadata_epoch { + u64 generation; + bool connected; + bool associated_once; + bool invalidated; +}; + +struct dcp_metadata_ticket { + u64 generation; + bool associated; +}; + +static inline void dcp_metadata_epoch_connection(struct dcp_metadata_epoch *epoch, + u64 generation, bool connected) +{ + if (epoch->associated_once && + (!connected || generation != epoch->generation)) + epoch->invalidated = true; + if (connected) + epoch->associated_once = true; + epoch->generation = generation; + epoch->connected = connected; +} + +static inline struct dcp_metadata_ticket +dcp_metadata_begin(const struct dcp_metadata_epoch *epoch) +{ + return (struct dcp_metadata_ticket) { + .generation = epoch->generation, + .associated = epoch->connected && !epoch->invalidated, + }; +} + +static inline bool dcp_metadata_matches(const struct dcp_metadata_epoch *epoch, + struct dcp_metadata_ticket ticket) +{ + return ticket.associated && epoch->connected && !epoch->invalidated && + ticket.generation == epoch->generation; +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4f2b4923135af..6a6ad92c0a891 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -344,6 +344,8 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, route->port->secondary_connector : route->port->connector; int ret; + dcp_firmware_metadata_connection(dcp, READ_ONCE(dcp->typec_generation), false); + /* * The fixed output's HPD handler leaves disconnects to DCP, so the port * can still be marked connected to a display that is gone. Release it @@ -439,6 +441,8 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) bool was_tunnel = route->tunnel; int ret = 0; + dcp_firmware_metadata_connection(dcp, READ_ONCE(dcp->typec_generation), false); + /* * Under tb_lock so a DCP link (re)configuration can neither select the * crossbar nor restart the tunnel pixel clock behind our back. @@ -1798,6 +1802,168 @@ void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); } +static bool dcp_firmware_metadata_enabled(struct apple_dcp *dcp) +{ + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; +} + +void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected) +{ + unsigned long flags; + + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (generation != dcp->metadata.epoch.generation || !connected) { + dcp->metadata.attributes_valid = false; + dcp->metadata.timing_valid = false; + dcp->metadata.hpd_seen = false; + } + dcp_metadata_epoch_connection(&dcp->metadata.epoch, generation, connected); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); +} + +void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) +{ + unsigned long flags; + + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (connected) { + dcp->metadata.hpd_seen = true; + } else if (dcp->metadata.hpd_seen) { + dcp_metadata_epoch_connection(&dcp->metadata.epoch, + dcp->metadata.epoch.generation, false); + dcp->metadata.attributes_valid = false; + dcp->metadata.timing_valid = false; + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); +} + +void dcp_firmware_metadata_begin(struct apple_dcp *dcp) +{ + unsigned long flags; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->chunks.metadata_ticket = dcp_metadata_begin(&dcp->metadata.epoch); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp->chunks.received = 0; + dcp->chunks.contiguous = true; +} + +static void dcp_firmware_metadata_notify(struct apple_dcp *dcp) +{ + unsigned long flags; + bool ready; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + ready = dcp->metadata.attributes_valid && dcp->metadata.timing_valid && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.attributes_ticket) && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.timing_ticket); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (ready && dcp->connector && READ_ONCE(dcp->connector->connected)) + schedule_work(&dcp->connector->hotplug_wq); +} + +static bool dcp_firmware_metadata_complete(struct apple_dcp *dcp) +{ + return dcp->chunks.contiguous && dcp->chunks.received == dcp->chunks.length && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket); +} + +void dcp_firmware_metadata_attributes(struct apple_dcp *dcp, + const struct dcp_display_metadata *attributes) +{ + unsigned long flags; + + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket)) { + dcp->metadata.attributes_valid = attributes && dcp_firmware_metadata_complete(dcp); + if (dcp->metadata.attributes_valid) { + dcp->metadata.attributes = *attributes; + dcp->metadata.attributes_ticket = dcp->chunks.metadata_ticket; + } + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp_firmware_metadata_notify(dcp); +} + +void dcp_firmware_metadata_timing(struct apple_dcp *dcp) +{ + unsigned long flags; + unsigned int i; + + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (!dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket)) + goto unlock; + dcp->metadata.timing_valid = false; + if (!dcp_firmware_metadata_complete(dcp)) + goto unlock; + for (i = 0; i < dcp->nr_modes; i++) { + if (!(dcp->modes[i].mode.type & DRM_MODE_TYPE_PREFERRED) || + !dcp->modes[i].metadata_timing_valid) + continue; + dcp->metadata.preferred = dcp->modes[i].mode; + dcp->metadata.timing_ticket = dcp->chunks.metadata_ticket; + dcp->metadata.timing_valid = true; + break; + } +unlock: + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp_firmware_metadata_notify(dcp); +} + +bool dcp_firmware_metadata_current(struct apple_dcp *dcp, u64 generation) +{ + unsigned long flags; + bool valid; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + valid = !dcp->metadata.epoch.invalidated && dcp->metadata.epoch.connected && + generation == dcp->metadata.epoch.generation; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + return valid; +} + +const struct drm_edid *dcp_firmware_edid(struct apple_dcp *dcp, u64 *generation) +{ + struct dcp_display_metadata attributes; + struct drm_display_mode preferred; + const struct drm_edid *edid; + unsigned long flags; + bool valid; + + if (!dcp_firmware_metadata_enabled(dcp)) + return ERR_PTR(-ENODEV); + spin_lock_irqsave(&dcp->metadata.lock, flags); + valid = dcp->metadata.attributes_valid && dcp->metadata.timing_valid && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.attributes_ticket) && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.timing_ticket); + if (valid) { + attributes = dcp->metadata.attributes; + preferred = dcp->metadata.preferred; + *generation = dcp->metadata.epoch.generation; + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (!valid) + return ERR_PTR(-ENODATA); + edid = dcp_metadata_edid(&attributes, &preferred); + spin_lock_irqsave(&dcp->metadata.lock, flags); + valid = !dcp->metadata.epoch.invalidated && dcp->metadata.epoch.connected && + *generation == dcp->metadata.epoch.generation; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (!valid && !IS_ERR(edid)) { + drm_edid_free(edid); + return ERR_PTR(-ESTALE); + } + return edid; +} + void dcp_set_dimensions(struct apple_dcp *dcp) { int i; @@ -2387,12 +2553,13 @@ static void dcp_typec_reconnect_work(struct work_struct *work) static void disconnected_hpd_event(struct apple_connector *con) { - if (con && con->connected) { - con->connected = 0; - drm_edid_free(con->drm_edid); - con->drm_edid = NULL; - drm_kms_helper_connector_hotplug_event(&con->base); - } + if (!con) + return; + + /* May run with DRM modeset locks held; defer property/cache cleanup. */ + WRITE_ONCE(con->connected, false); + WRITE_ONCE(con->edid_invalidation_pending, true); + schedule_work(&con->hotplug_wq); } static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) @@ -2419,6 +2586,7 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) guard(mutex)(&dcp->hpd_mutex); dcp->typec_generation++; WRITE_ONCE(dcp->typec_cable_connected, true); + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); dcp->typec_reconnect_tries = 0; dcp->placeholder_retried = false; cancel_delayed_work(&dcp->typec_reconnect_wq); @@ -2440,6 +2608,7 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } WRITE_ONCE(dcp->typec_crtc_off, false); reinit_completion(&dcp->typec_iomfb_hpd_ready); @@ -3364,6 +3533,7 @@ static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) if (release_cable) WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); @@ -3751,6 +3921,7 @@ static int dcp_platform_probe(struct platform_device *pdev) mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); + spin_lock_init(&dcp->metadata.lock); spin_lock_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); if (dcp->hw.iomfb_method_profile == diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 50b240511442b..c10821fe2a538 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -736,6 +736,23 @@ void dcp_retrain_oob(struct apple_connector *connector) schedule_work(&connector->hotplug_wq); } +static void dcp_clear_connector_edid(struct apple_connector *connector) +{ + lockdep_assert_held(&connector->base.dev->mode_config.mutex); + drm_edid_connector_update(&connector->base, NULL); + drm_edid_free(connector->drm_edid); + connector->drm_edid = NULL; + connector->drm_edid_firmware_derived = false; +} + +static void dcp_consume_edid_invalidation(struct apple_connector *connector) +{ + lockdep_assert_held(&connector->base.dev->mode_config.mutex); + /* Preserve a disconnect even if the port reconnected before probing. */ + if (xchg(&connector->edid_invalidation_pending, false)) + dcp_clear_connector_edid(connector); +} + void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; @@ -744,9 +761,17 @@ void dcp_hotplug(struct work_struct *work) int ret; connector = container_of(work, struct apple_connector, hotplug_wq); + scoped_guard(mutex, &connector->base.dev->mode_config.mutex) + dcp_consume_edid_invalidation(connector); pdev = READ_ONCE(connector->dcp); if (!pdev) { /* a Type-C port unrouted after this was queued */ + scoped_guard(mutex, &connector->base.dev->mode_config.mutex) { + if (!READ_ONCE(connector->dcp) && connector->drm_edid_firmware_derived) { + drm_edid_connector_update(&connector->base, NULL); + connector->drm_edid_firmware_derived = false; + } + } drm_kms_helper_connector_hotplug_event(&connector->base); return; } @@ -754,9 +779,9 @@ void dcp_hotplug(struct work_struct *work) dev_info(dcp->dev, "%s() connected:%d valid_mode:%d nr_modes:%u\n", __func__, connector->connected, READ_ONCE(dcp->mode_state.valid), dcp->nr_modes); - if (!connector->connected) { - drm_edid_free(connector->drm_edid); - connector->drm_edid = NULL; + scoped_guard(mutex, &connector->base.dev->mode_config.mutex) { + if (!READ_ONCE(connector->connected)) + dcp_clear_connector_edid(connector); } /* @@ -990,9 +1015,16 @@ int dcp_get_modes(struct drm_connector *connector) bool vrr_capable = false; int i; + dcp_consume_edid_invalidation(apple_connector); + /* A Type-C port has no pipeline while the fabric is moving it. */ - if (!pdev) + if (!pdev) { + if (apple_connector->drm_edid_firmware_derived) { + drm_edid_connector_update(connector, NULL); + apple_connector->drm_edid_firmware_derived = false; + } return 0; + } dcp = platform_get_drvdata(pdev); for (i = 0; i < dcp->nr_modes; ++i) { @@ -1038,8 +1070,34 @@ int dcp_get_modes(struct drm_connector *connector) } } if (dcp->nr_modes && apple_connector->drm_edid) { + apple_connector->drm_edid_firmware_derived = false; drm_edid_connector_update(connector, apple_connector->drm_edid); dcp_retry_placeholder_edid(dcp, apple_connector->drm_edid); + } else { + u64 generation = 0; + const struct drm_edid *derived = dcp->nr_modes ? + dcp_firmware_edid(dcp, &generation) : ERR_PTR(-ENODATA); + + if (!IS_ERR(derived)) { + bool log = !apple_connector->drm_edid_firmware_derived || + apple_connector->drm_edid_generation != generation; + int update; + + apple_connector->drm_edid_firmware_derived = true; + apple_connector->drm_edid_generation = generation; + update = drm_edid_connector_update(connector, derived); + drm_edid_free(derived); + if (update || !dcp_firmware_metadata_current(dcp, generation)) { + drm_edid_connector_update(connector, NULL); + apple_connector->drm_edid_firmware_derived = false; + } else if (log) { + dev_info(dcp->dev, "using firmware-derived EDID metadata for connection %llu\n", + generation); + } + } else if (apple_connector->drm_edid_firmware_derived) { + drm_edid_connector_update(connector, NULL); + apple_connector->drm_edid_firmware_derived = false; + } } /* diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 248ee966e33a6..03438d24a7557 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -941,6 +941,7 @@ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) #endif dcp->chunks.length = *length; + dcp_firmware_metadata_begin(dcp); dcp->chunks.data = kzalloc(*length, GFP_KERNEL); if (!dcp->chunks.data) { @@ -959,11 +960,17 @@ static u8 dcpep_cb_prop_chunk(struct apple_dcp *dcp, return false; } - if (req->offset + req->length > dcp->chunks.length) { + if (req->offset > dcp->chunks.length || req->length > sizeof(req->data) || + req->length > dcp->chunks.length - req->offset) { + dcp->chunks.contiguous = false; dev_warn(dcp->dev, "ignoring overflowing chunk\n"); return false; } + if (req->offset != dcp->chunks.received) + dcp->chunks.contiguous = false; + if (dcp->chunks.contiguous) + dcp->chunks.received += req->length; memcpy(dcp->chunks.data + req->offset, req->data, req->length); return true; } @@ -1002,11 +1009,15 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, dev_warn(dcp->dev, "failed to parse modes\n"); dcp->modes = NULL; dcp->nr_modes = 0; + dcp_firmware_metadata_timing(dcp); return false; } if (dcp->nr_modes == 0) dev_warn(dcp->dev, "TimingElements without valid modes!\n"); + dcp_firmware_metadata_timing(dcp); } else if (!strcmp(req->key, "DisplayAttributes")) { + struct dcp_parse_ctx metadata_ctx = ctx; + ret = parse_display_attributes(&ctx, &dcp->width_mm, &dcp->height_mm); @@ -1016,6 +1027,12 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, } dcp_set_dimensions(dcp); + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + struct dcp_display_metadata metadata; + + ret = parse_display_metadata(&metadata_ctx, &metadata); + dcp_firmware_metadata_attributes(dcp, ret ? NULL : &metadata); + } } return true; @@ -1871,6 +1888,7 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) if (dcp_is_typec_output(dcp) && *connected && dcp->nr_modes) complete_all(&dcp->typec_iomfb_hpd_ready); + dcp_firmware_metadata_hotplug(dcp, !!(*connected)); action = dcp_mode_hotplug(&dcp->mode_state, !!(*connected), connector ? &connector->connected : NULL); /* diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 691bc91e3542b..b723fba76d456 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -299,7 +299,7 @@ int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx) return 0; } -static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) +static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, bool polarity) { struct iterator it; int ret = 0; @@ -310,6 +310,8 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) .front_porch = -1, .sync_width = -1, .precise_sync_rate = -1, + .sync_polarity = -1, + .pixel_repetition = -1, }; ret = iterator_begin(handle, &it, true); @@ -331,6 +333,18 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) ret = parse_int_bound(it.handle, &dim->sync_width, 0, U16_MAX); else if (!strcmp(key, "PreciseSyncRate")) ret = parse_int_bound(it.handle, &dim->precise_sync_rate, 1, U32_MAX); + else if (polarity && !strcmp(key, "SyncPolarity")) { + if (dim->sync_polarity >= 0) + ret = -EINVAL; + else + ret = parse_int_bound(it.handle, &dim->sync_polarity, 0, 1); + } else if (polarity && !strcmp(key, "PixelRepetition")) { + if (dim->pixel_repetition >= 0) + ret = -EINVAL; + else + ret = parse_int_bound(it.handle, &dim->pixel_repetition, + 0, U32_MAX); + } else ret = skip(it.handle); @@ -626,6 +640,8 @@ static int parse_mode(struct dcp_parse_ctx *handle, s64 id = -1; s64 best_color_mode = -1; bool is_virtual = false; + bool preferred = false, interlaced = false; + bool have_preferred = false, have_interlaced = false; struct drm_display_mode *mode = &out->mode; *out = (struct dcp_display_mode) { @@ -656,9 +672,9 @@ static int parse_mode(struct dcp_parse_ctx *handle, else if (is_virtual) ret = skip(it.handle); else if (!strcmp(key, "HorizontalAttributes")) - ret = parse_dimension(it.handle, &horiz); + ret = parse_dimension(it.handle, &horiz, external_sdr); else if (!strcmp(key, "VerticalAttributes")) - ret = parse_dimension(it.handle, &vert); + ret = parse_dimension(it.handle, &vert, external_sdr); else if (!strcmp(key, "MinimumVariableRefreshRate")) ret = parse_int_bound(it.handle, &min_vrr, 0, U32_MAX); else if (!strcmp(key, "MaximumVariableRefreshRate")) @@ -669,7 +685,19 @@ static int parse_mode(struct dcp_parse_ctx *handle, ret = parse_int_bound(it.handle, &id, 0, U32_MAX); else if (!strcmp(key, "IsVirtual")) ret = parse_bool(it.handle, &is_virtual); - else if (!strcmp(key, "Score")) + else if (external_sdr && !strcmp(key, "IsPreferred")) { + if (have_preferred) + ret = -EINVAL; + else + ret = parse_bool(it.handle, &preferred); + have_preferred = true; + } else if (external_sdr && !strcmp(key, "IsInterlaced")) { + if (have_interlaced) + ret = -EINVAL; + else + ret = parse_bool(it.handle, &interlaced); + have_interlaced = true; + } else if (!strcmp(key, "Score")) ret = parse_int_bound(it.handle, score, 0, S64_MAX); else ret = skip(it.handle); @@ -755,6 +783,12 @@ static int parse_mode(struct dcp_parse_ctx *handle, .height_mm = height_mm, }; + if (external_sdr && horiz.sync_polarity >= 0 && vert.sync_polarity >= 0) + mode->flags = (horiz.sync_polarity ? DRM_MODE_FLAG_PHSYNC : DRM_MODE_FLAG_NHSYNC) | + (vert.sync_polarity ? DRM_MODE_FLAG_PVSYNC : DRM_MODE_FLAG_NVSYNC); + out->metadata_timing_valid = external_sdr && have_preferred && preferred && + have_interlaced && !interlaced && horiz.pixel_repetition == 0 && + vert.pixel_repetition == 0 && horiz.sync_polarity >= 0 && vert.sync_polarity >= 0; drm_mode_set_name(mode); out->timing_mode_id = id; @@ -861,6 +895,238 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, return 0; } +/* Metadata strings must fit an EDID text descriptor without hidden suffixes. */ +static int parse_metadata_string(struct dcp_parse_ctx *handle, char *out, size_t size) +{ + const struct dcp_parse_tag *tag = parse_tag_of_type(handle, DCP_TYPE_STRING); + const u8 *value; + + if (IS_ERR(tag)) + return PTR_ERR(tag); + if (!tag->size || tag->size >= size) + return -EINVAL; + value = parse_bytes(handle, tag->size); + if (IS_ERR(value)) + return PTR_ERR(value); + for (unsigned int i = 0; i < tag->size; i++) + if (value[i] < 0x20 || value[i] > 0x7e) + return -EINVAL; + memcpy(out, value, tag->size); + out[tag->size] = 0; + return 0; +} + +static int parse_metadata_number(struct dcp_parse_ctx *handle, u32 *out, u32 min, u32 max) +{ + s64 value; + int ret = parse_int_bound(handle, &value, min, max); + + if (!ret) + *out = value; + return ret; +} + +static int parse_metadata_point(struct dcp_parse_ctx *handle, u32 point[2], u32 *gamma) +{ + struct iterator it; + unsigned int seen = 0; + int ret = iterator_begin(handle, &it, true); + + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char key[64]; + + ret = parse_metadata_string(handle, key, sizeof(key)); + if (ret) + return ret; + if (!strcmp(key, "X")) { + if (seen & 1) + return -EINVAL; + seen |= 1; + ret = parse_metadata_number(handle, &point[0], 1, 65535); + } else if (!strcmp(key, "Y")) { + if (seen & 2) + return -EINVAL; + seen |= 2; + ret = parse_metadata_number(handle, &point[1], 1, 65535); + } else if (gamma && !strcmp(key, "Gamma")) { + if (seen & 4) + return -EINVAL; + seen |= 4; + ret = parse_metadata_number(handle, gamma, 65536, (354U << 16) / 100); + } else { + ret = skip(handle); + } + if (ret) + return ret; + } + if (seen != (gamma ? 7 : 3) || point[0] + point[1] > 65536) + return -EINVAL; + return 0; +} + +static int parse_metadata_chromaticity(struct dcp_parse_ctx *handle, + struct dcp_display_metadata *out) +{ + static const char * const names[] = { "Red", "Green", "Blue" }; + struct iterator it; + unsigned int seen = 0; + int ret = iterator_begin(handle, &it, true); + + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char key[64]; + unsigned int color; + + ret = parse_metadata_string(handle, key, sizeof(key)); + if (ret) + return ret; + for (color = 0; color < ARRAY_SIZE(names); color++) + if (!strcmp(key, names[color])) + break; + if (color < ARRAY_SIZE(names)) { + if (seen & 1U << color) + return -EINVAL; + seen |= 1U << color; + ret = parse_metadata_point(handle, out->chromaticity[color], NULL); + } else { + ret = skip(handle); + } + if (ret) + return ret; + } + return seen == 7 ? 0 : -EINVAL; +} + +static int parse_metadata_product(struct dcp_parse_ctx *handle, struct dcp_display_metadata *out) +{ + struct iterator it; + unsigned int seen = 0; + u32 legacy = 0; + int ret = iterator_begin(handle, &it, true); + + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char key[64]; + + ret = parse_metadata_string(handle, key, sizeof(key)); + if (ret) + return ret; + if (!strcmp(key, "ManufacturerID")) { + if (seen & 1) + return -EINVAL; + seen |= 1; + ret = parse_metadata_string(handle, out->manufacturer, + sizeof(out->manufacturer)); + } else if (!strcmp(key, "ProductName")) { + if (seen & 2) + return -EINVAL; + seen |= 2; + ret = parse_metadata_string(handle, out->name, sizeof(out->name)); + } else if (!strcmp(key, "ProductID")) { + if (seen & 4) + return -EINVAL; + seen |= 4; + ret = parse_metadata_number(handle, &out->product, 0, U16_MAX); + } else if (!strcmp(key, "WeekOfManufacture")) { + if (seen & 8) + return -EINVAL; + seen |= 8; + ret = parse_metadata_number(handle, &out->week, 0, 53); + } else if (!strcmp(key, "YearOfManufacture")) { + if (seen & 16) + return -EINVAL; + seen |= 16; + ret = parse_metadata_number(handle, &out->year, 1990, 2245); + } else if (!strcmp(key, "SerialNumber")) { + if (seen & 64) + return -EINVAL; + seen |= 64; + ret = parse_metadata_number(handle, &out->serial, 0, U32_MAX); + } else if (!strcmp(key, "AlphanumericSerialNumber")) { + if (seen & 128) + return -EINVAL; + seen |= 128; + ret = parse_metadata_string(handle, out->serial_text, + sizeof(out->serial_text)); + } else if (!strcmp(key, "LegacyManufacturerID")) { + if (seen & 32) + return -EINVAL; + seen |= 32; + ret = parse_metadata_number(handle, &legacy, 0, 0x7fff); + } else { + ret = skip(handle); + } + if (ret) + return ret; + } + if ((seen & 31) != 31 || strlen(out->manufacturer) != 3) + return -EINVAL; + for (unsigned int i = 0; i < 3; i++) + if (out->manufacturer[i] < 'A' || out->manufacturer[i] > 'Z') + return -EINVAL; + if ((seen & 32) && legacy != ((out->manufacturer[0] - '@') << 10 | + (out->manufacturer[1] - '@') << 5 | + (out->manufacturer[2] - '@'))) + return -EINVAL; + return 0; +} + +int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metadata *out) +{ + struct dcp_display_metadata parsed = {}; + struct iterator it; + unsigned int seen = 0; + int ret = iterator_begin(handle, &it, true); + + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char key[64]; + + ret = parse_metadata_string(handle, key, sizeof(key)); + if (ret) + return ret; + if (!strcmp(key, "MaxHorizontalImageSize")) { + if (seen & 1) + return -EINVAL; + seen |= 1; + ret = parse_metadata_number(handle, &parsed.width_cm, 1, 255); + } else if (!strcmp(key, "MaxVerticalImageSize")) { + if (seen & 2) + return -EINVAL; + seen |= 2; + ret = parse_metadata_number(handle, &parsed.height_cm, 1, 255); + } else if (!strcmp(key, "Chromaticity")) { + if (seen & 4) + return -EINVAL; + seen |= 4; + ret = parse_metadata_chromaticity(handle, &parsed); + } else if (!strcmp(key, "DefaultWhitePoint")) { + if (seen & 8) + return -EINVAL; + seen |= 8; + ret = parse_metadata_point(handle, parsed.white, &parsed.gamma); + } else if (!strcmp(key, "ProductAttributes")) { + if (seen & 16) + return -EINVAL; + seen |= 16; + ret = parse_metadata_product(handle, &parsed); + } else { + ret = skip(handle); + } + if (ret) + return ret; + } + if (seen != 31) + return -EINVAL; + *out = parsed; + return 0; +} + int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, const char **class, s64 *unit, bool h17p_keys) { diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 1f3b8782b7bfc..50e0580f98080 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -7,6 +7,8 @@ /* For mode parsing */ #include +#include "dcp-display-metadata.h" + struct apple_dcp; struct dcp_parse_ctx { @@ -94,11 +96,14 @@ struct dcp_display_mode { struct dcp_color_mode sdr; struct dcp_color_mode best; bool vrr; + bool metadata_timing_valid; }; struct dimension { s64 total, front_porch, sync_width, active; s64 precise_sync_rate; + s64 sync_polarity; + s64 pixel_repetition; }; int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); @@ -106,6 +111,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, bool internal, bool external_sdr); +int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metadata *out); int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, diff --git a/tools/diagnostics/display-attributes-fixture.json b/tools/diagnostics/display-attributes-fixture.json new file mode 100644 index 0000000000000..9324ae496d516 --- /dev/null +++ b/tools/diagnostics/display-attributes-fixture.json @@ -0,0 +1,23 @@ +{ + "MaxHorizontalImageSize": 60, + "MaxVerticalImageSize": 33, + "Chromaticity": { + "Red": {"X": 44608, "Y": 20288}, + "Green": {"X": 16960, "Y": 40448}, + "Blue": {"X": 9344, "Y": 5184} + }, + "DefaultWhitePoint": {"X": 20544, "Y": 21568, "Gamma": 144179}, + "WhitePoints": [{"X": 20544, "Y": 21568, "Gamma": 144179}], + "DefaultColorSpaceIsSRGB": true, + "SupportsPQEOTF": true, + "ProductAttributes": { + "LegacyManufacturerID": 1138, + "ProductID": 2895, + "SerialNumber": 305419896, + "WeekOfManufacture": 7, + "YearOfManufacture": 2013, + "AlphanumericSerialNumber": "ANON000000001", + "ProductName": "QG270 S3", + "ManufacturerID": "ACR" + } +} diff --git a/tools/diagnostics/test-apple-display-metadata-lifetime.py b/tools/diagnostics/test-apple-display-metadata-lifetime.py new file mode 100644 index 0000000000000..bc64678c26d5d --- /dev/null +++ b/tools/diagnostics/test-apple-display-metadata-lifetime.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise actual metadata epoch/cache and connector raw-EDID precedence paths.""" +from pathlib import Path +import os +import shlex +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +base = root / 'drivers/gpu/drm/apple' +dcp = (base / 'dcp.c').read_text() +iomfb = (base / 'iomfb.c').read_text() + + +def function(text, signature): + start = text.index(signature) + opening = text.index('{', start) + while ';' in text[start:opening]: + start = text.index(signature, start + 1) + opening = text.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + + +metadata = function((base / 'dcp-display-metadata.h').read_text(), 'struct dcp_display_metadata {') + ';' +epoch = (base / 'dcp-metadata-epoch.h').read_text().replace('#include ', '') +cache = dcp[dcp.index('static bool dcp_firmware_metadata_enabled('):dcp.index('void dcp_set_dimensions(')] +getter = function(iomfb, 'int dcp_get_modes(') +cleanup = function(iomfb, 'static void dcp_clear_connector_edid(') + '\n' + function(iomfb, 'static void dcp_consume_edid_invalidation(') +hotplug = function(iomfb, 'void dcp_hotplug(') +disconnect = function(dcp, 'static void disconnected_hpd_event(') +chunk_callbacks = function((base / 'iomfb_template.c').read_text(), 'static u8 dcpep_cb_prop_start(') + '\n' + function((base / 'iomfb_template.c').read_text(), 'static u8 dcpep_cb_prop_chunk(') +# Check the actual ingress/egress hooks as well as executing the shared helpers. +assert dcp.count('dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true)') == 1 +assert dcp.count('dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false)') == 2 +assert dcp.count('dcp_firmware_metadata_connection(dcp, READ_ONCE(dcp->typec_generation), false)') == 2 +template = (base / 'iomfb_template.c').read_text() +assert 'dcp_firmware_metadata_begin(dcp);' in template +assert 'dcp_firmware_metadata_hotplug(dcp, !!(*connected));' in template +assert 'dcp_firmware_metadata_attributes(dcp, ret ? NULL : &metadata);' in template +assert 'dcp->chunks.received += req->length;' in template +prefix = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define xchg(p,v) __atomic_exchange_n(p,v,__ATOMIC_SEQ_CST) +#define lockdep_assert_held(p) assert(held_mutex==(p)) +#define ERR_PTR(n) ((void *)(intptr_t)(n)) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define IS_ERR(p) ((uintptr_t)(p) > (uintptr_t)-4096) +#define IS_ERR_OR_NULL(p) (!(p) || IS_ERR(p)) +#define DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL 2 +#define DCP_FIRMWARE_H17P 2 +#define DRM_MODE_TYPE_PREFERRED 2 +#define DCP_FW_VER 26 +#define DCP_FW_VERSION(a,b,c) (a) +#define SZ_1M (1024*1024) +#define GFP_KERNEL 0 +#define kzalloc(n,f) calloc(1,n) +#define dev_warn(...) ((void)0) +#define dev_err(...) ((void)0) +#define dev_info(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define container_of(p,t,m) ((t *)((char *)(p) - offsetof(t,m))) +typedef struct { bool held; } spinlock_t; +#define spin_lock_irqsave(p, flags) do { assert(!(p)->held); (p)->held=true; (flags)=0; } while (0) +#define spin_unlock_irqrestore(p, flags) do { (void)(flags); assert((p)->held); (p)->held=false; } while (0) +struct drm_display_mode { unsigned int type; int clock; }; +struct dcp_display_mode { struct drm_display_mode mode; bool vrr, metadata_timing_valid; u32 min_vrr, max_vrr; }; +struct work_struct { unsigned int scheduled; }; +struct mutex { pthread_mutex_t native; }; +static _Thread_local struct mutex *held_mutex; +static _Thread_local bool modeset_held; +static void mutex_lock(struct mutex *m) { assert(!modeset_held); assert(held_mutex != m); assert(!pthread_mutex_lock(&m->native)); held_mutex=m; } +static void mutex_unlock(struct mutex *m) { assert(held_mutex==m); held_mutex=NULL; assert(!pthread_mutex_unlock(&m->native)); } +static struct mutex *acquire(struct mutex *m) { mutex_lock(m); return m; } +static void release(struct mutex **m) { if (*m) mutex_unlock(*m); } +static struct mutex *once(struct mutex *m) { mutex_unlock(m); return NULL; } +#define scoped_guard(kind,value) for(struct mutex *g __attribute__((cleanup(release)))=acquire(value); g; g=once(g)) +struct test_blob { unsigned int identity, references, magic; }; +struct drm_device { void *dev; struct { struct mutex mutex; } mode_config; }; +struct drm_connector { struct drm_device *dev; void *state; struct test_blob *edid_blob_ptr; struct { struct { unsigned int min_vfreq, max_vfreq; } monitor_range; } display_info; }; +struct dcp_set_dcpav_prop_chunk_req { u8 data[4096]; u32 offset, length; }; +struct apple_dcp; +struct platform_device { struct apple_dcp *data; }; +struct drm_edid { unsigned int identity; }; +struct apple_connector { + struct drm_connector base; + struct platform_device *dcp; + const struct drm_edid *drm_edid; + bool drm_edid_firmware_derived; + bool edid_invalidation_pending; + u64 drm_edid_generation; + bool connected; + struct work_struct hotplug_wq; +}; +#define to_apple_connector(p) container_of(p, struct apple_connector, base) +#define platform_get_drvdata(p) ((p)->data) +''' +state = r''' +struct apple_dcp { + void *dev; + struct { int iomfb_method_profile; } hw; + struct apple_connector *connector; + struct { void *data; size_t length, received; bool contiguous; struct dcp_metadata_ticket metadata_ticket; } chunks; + struct { + spinlock_t lock; + struct dcp_metadata_epoch epoch; + struct dcp_display_metadata attributes; + struct drm_display_mode preferred; + struct dcp_metadata_ticket attributes_ticket, timing_ticket; + bool attributes_valid, timing_valid, hpd_seen; + } metadata; + struct dcp_display_mode *modes; + unsigned int nr_modes; + int fw_compat; + struct { bool valid; } mode_state; + struct { bool inline_v2; } *dcpavservep; +}; +static struct apple_dcp *current_for_update; +static unsigned int allocations, raw_reads, mode_adds, clears, property_identity, blob_allocations, notifications, retrains; +static bool invalidate_during_update; +static bool update_error; +static bool raw_available; +static const struct drm_edid *allocate(unsigned int identity) +{ + struct drm_edid *edid = malloc(sizeof(*edid)); assert(edid); + edid->identity = identity; allocations++; return edid; +} +static void drm_edid_free(const struct drm_edid *edid) +{ if (edid) { assert(allocations); allocations--; free((void *)edid); } } +static const struct drm_edid *dcp_metadata_edid(const struct dcp_display_metadata *m, + const struct drm_display_mode *mode) +{ assert(mode->clock); return allocate(m->serial); } +static void schedule_work(struct work_struct *w) { w->scheduled++; } +static struct drm_display_mode *drm_mode_duplicate(struct drm_device *d, const struct drm_display_mode *m) +{ (void)d; struct drm_display_mode *copy=malloc(sizeof(*copy)); *copy=*m; return copy; } +static void drm_mode_probed_add(struct drm_connector *c, struct drm_display_mode *m) +{ (void)c; mode_adds++; free(m); } +static void drm_connector_set_vrr_capable_property(struct drm_connector *c, bool v) +{ (void)c; (void)v; } +static bool dcp_has_panel(struct apple_dcp *p) { (void)p; return false; } +static const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *p) +{ (void)p; raw_reads++; return raw_available ? allocate(0xfeed) : ERR_PTR(-ENODEV); } +static int drm_edid_connector_update(struct drm_connector *c, const struct drm_edid *e) +{ + assert(held_mutex==&c->dev->mode_config.mutex); + if (e && update_error) return -ENOMEM; + struct test_blob *old=c->edid_blob_ptr; + struct test_blob *next=NULL; + if (e) { + next=malloc(sizeof(*next)); assert(next); + *next=(struct test_blob){ .identity=e->identity, .references=1, .magic=0xcafe }; + blob_allocations++; + } else clears++; + if (old) { assert(old->magic==0xcafe && old->references==1); old->magic=0; free(old); blob_allocations--; } + c->edid_blob_ptr=next; + property_identity=e ? e->identity : 0; + if (invalidate_during_update && e && e->identity!=0xfeed) current_for_update->metadata.epoch.invalidated=true; + return 0; +} +static bool dcp_is_usb4_output(struct apple_dcp *p) { (void)p; return false; } +static bool apple_dp_tunnel_t602x(void) { return false; } +#define DRM_MODE_LINK_STATUS_BAD 1 +static void drm_connector_set_link_status_property(struct drm_connector *c, int status) { (void)c; (void)status; } +static int dcp_retrain_active_crtc(struct apple_connector *c) +{ + assert(held_mutex!=&c->base.dev->mode_config.mutex); + retrains++; + return 0; +} +static void drm_kms_helper_connector_hotplug_event(struct drm_connector *c) +{ + assert(held_mutex!=&c->dev->mode_config.mutex); + mutex_lock(&c->dev->mode_config.mutex); + if (c->edid_blob_ptr) assert(c->edid_blob_ptr->magic==0xcafe); + notifications++; + mutex_unlock(&c->dev->mode_config.mutex); +} +static void dcp_retry_placeholder_edid(struct apple_dcp *p, const struct drm_edid *e) +{ (void)p; assert(e->identity==0xfeed); } +''' +suffix = r''' +static struct apple_dcp p; +static struct apple_connector c; +static struct platform_device platform; +static struct drm_device drm={ .mode_config.mutex.native=PTHREAD_MUTEX_INITIALIZER }; +static struct dcp_display_mode modes[1] = {{ .mode={ .type=DRM_MODE_TYPE_PREFERRED, .clock=148500 }, .metadata_timing_valid=true }}; +static struct dcp_display_metadata attrs = { .serial=1234 }; +static int probe(struct drm_connector *connector) +{ + mutex_lock(&connector->dev->mode_config.mutex); + int ret=dcp_get_modes(connector); + mutex_unlock(&connector->dev->mode_config.mutex); + return ret; +} +static void initial(void) +{ + mutex_lock(&drm.mode_config.mutex); + drm_edid_free(c.drm_edid); + c.base.dev=&drm; + drm_edid_connector_update(&c.base,NULL); + mutex_unlock(&drm.mode_config.mutex); + free(p.chunks.data); + assert(!allocations); + memset(&p,0,sizeof(p)); memset(&c,0,sizeof(c)); + p.hw.iomfb_method_profile=DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + p.fw_compat=DCP_FIRMWARE_H17P; + static typeof(*p.dcpavservep) av={true}; p.dcpavservep=&av; + p.connector=&c; p.modes=modes; p.nr_modes=1; + platform.data=&p; c.dcp=&platform; c.base.dev=&drm; c.connected=true; + p.mode_state.valid=true; + raw_reads=mode_adds=clears=property_identity=0; raw_available=false; + current_for_update=&p; invalidate_during_update=false; update_error=false; modes[0].metadata_timing_valid=true; +} +static void begin(bool complete) +{ + dcp_firmware_metadata_begin(&p); + p.chunks.length=100; p.chunks.received=complete ? 100 : 99; +} +static bool eligible(void) +{ + u64 generation; + const struct drm_edid *edid=dcp_firmware_edid(&p,&generation); + bool valid=!IS_ERR(edid); + if (valid) { assert(generation==1 && edid->identity==attrs.serial); drm_edid_free(edid); } + return valid; +} +static void publish(void) +{ + begin(true); dcp_firmware_metadata_attributes(&p,&attrs); + begin(true); dcp_firmware_metadata_timing(&p); +} +static void *probe_thread(void *unused) +{ + (void)unused; + for(unsigned int i=0;i<1000;i++) { + mutex_lock(&drm.mode_config.mutex); + c.connected=true; raw_available=!!(i&1); + assert(dcp_get_modes(&c.base)==1); + mutex_unlock(&drm.mode_config.mutex); + } + return NULL; +} +static void *hotplug_thread(void *unused) +{ + (void)unused; + for(unsigned int i=0;i<1000;i++) { + mutex_lock(&drm.mode_config.mutex); c.connected=false; mutex_unlock(&drm.mode_config.mutex); + dcp_hotplug(&c.hotplug_wq); + } + return NULL; +} +static void *reader_thread(void *unused) +{ + (void)unused; + for(unsigned int i=0;i<2000;i++) { + mutex_lock(&drm.mode_config.mutex); + struct test_blob *b=c.base.edid_blob_ptr; + if(b) { + assert(b->magic==0xcafe && b->references==1); + b->references++; + assert(b->identity==1234 || b->identity==0xfeed); + b->references--; + } + mutex_unlock(&drm.mode_config.mutex); + } + return NULL; +} +int main(void) +{ + initial(); begin(true); /* A pre-association blob cannot become current later. */ + dcp_firmware_metadata_connection(&p,1,true); + dcp_firmware_metadata_attributes(&p,&attrs); + begin(true); dcp_firmware_metadata_timing(&p); + assert(!eligible()); + begin(true); dcp_firmware_metadata_attributes(&p,&attrs); + assert(eligible() && c.hotplug_wq.scheduled); + /* Raw failure admits fallback, but fallback never blocks a later raw read. */ + assert(probe(&c.base)==1 && c.drm_edid_firmware_derived && raw_reads==1); + assert(!c.drm_edid && property_identity==1234 && !allocations && mode_adds==1); + raw_available=true; + assert(probe(&c.base)==1 && !c.drm_edid_firmware_derived && raw_reads==2); + assert(c.drm_edid->identity==0xfeed && mode_adds==2); + assert(probe(&c.base)==1 && raw_reads==2); /* A raw cache remains authoritative. */ + + initial(); dcp_firmware_metadata_connection(&p,1,true); + begin(false); dcp_firmware_metadata_attributes(&p,&attrs); + begin(true); dcp_firmware_metadata_timing(&p); assert(!eligible()); + begin(true); p.chunks.contiguous=false; dcp_firmware_metadata_attributes(&p,&attrs); + assert(!eligible()); + begin(true); dcp_firmware_metadata_attributes(&p,&attrs); assert(eligible()); + assert(probe(&c.base)==1 && c.drm_edid_firmware_derived); + dcp_firmware_metadata_connection(&p,2,false); + dcp_firmware_metadata_connection(&p,3,true); + publish(); assert(!eligible()); /* First connection only, even with later complete records. */ + assert(probe(&c.base)==1 && !c.drm_edid && !c.drm_edid_firmware_derived && clears && !allocations); + + initial(); dcp_firmware_metadata_connection(&p,1,true); begin(true); + dcp_firmware_metadata_connection(&p,2,true); /* Change while a transfer is in flight. */ + dcp_firmware_metadata_attributes(&p,&attrs); assert(!eligible()); + initial(); dcp_firmware_metadata_connection(&p,1,true); + dcp_firmware_metadata_hotplug(&p,false); /* Initial no-HPD state is not a second attachment. */ + publish(); assert(eligible()); + dcp_firmware_metadata_hotplug(&p,true); + dcp_firmware_metadata_hotplug(&p,false); assert(!eligible()); + initial(); p.hw.iomfb_method_profile=0; + dcp_firmware_metadata_connection(&p,1,true); publish(); assert(!eligible()); + assert(probe(&c.base)==1 && !c.drm_edid); + initial(); dcp_firmware_metadata_connection(&p,1,true); + modes[0].metadata_timing_valid=false; publish(); assert(!eligible()); + initial(); dcp_firmware_metadata_connection(&p,1,true); publish(); + invalidate_during_update=true; + assert(probe(&c.base)==1 && !c.drm_edid_firmware_derived && !property_identity && !allocations); + initial(); dcp_firmware_metadata_connection(&p,1,true); publish(); update_error=true; + assert(probe(&c.base)==1 && !c.drm_edid_firmware_derived && !allocations); + initial(); dcp_firmware_metadata_connection(&p,1,true); + u32 length=64; + struct dcp_set_dcpav_prop_chunk_req chunk={ .length=32 }; + assert(dcpep_cb_prop_start(&p,&length)); + assert(dcpep_cb_prop_chunk(&p,&chunk)); + assert(p.chunks.received==32 && p.chunks.contiguous); + dcp_firmware_metadata_attributes(&p,&attrs); assert(!p.metadata.attributes_valid); + chunk.offset=32; assert(dcpep_cb_prop_chunk(&p,&chunk)); + dcp_firmware_metadata_attributes(&p,&attrs); assert(p.metadata.attributes_valid); + initial(); dcp_firmware_metadata_connection(&p,1,true); + assert(dcpep_cb_prop_start(&p,&length)); + assert(dcpep_cb_prop_chunk(&p,&chunk)); /* Out-of-order first chunk. */ + chunk.offset=0; assert(dcpep_cb_prop_chunk(&p,&chunk)); + assert(!p.chunks.contiguous); + dcp_firmware_metadata_attributes(&p,&attrs); assert(!p.metadata.attributes_valid); + chunk.offset=UINT32_MAX; chunk.length=8; + assert(!dcpep_cb_prop_chunk(&p,&chunk)); + initial(); dcp_firmware_metadata_connection(&p,1,true); publish(); + assert(probe(&c.base)==1); + c.dcp=NULL; + dcp_hotplug(&c.hotplug_wq); + assert(!c.drm_edid_firmware_derived && !c.base.edid_blob_ptr); + c.dcp=&platform; c.connected=true; c.base.state=&p; p.mode_state.valid=false; + dcp_hotplug(&c.hotplug_wq); assert(retrains); + p.mode_state.valid=true; + assert(probe(&c.base)==1); + modeset_held=true; disconnected_hpd_event(&c); modeset_held=false; + assert(!c.connected && c.edid_invalidation_pending && c.base.edid_blob_ptr); + dcp_hotplug(&c.hotplug_wq); + assert(!c.edid_invalidation_pending && !c.drm_edid_firmware_derived && !c.base.edid_blob_ptr); + /* A fast reconnect cannot retain a raw EDID from the previous attachment. */ + c.connected=true; raw_available=true; + assert(probe(&c.base)==1 && c.drm_edid); + unsigned int previous_reads=raw_reads; + modeset_held=true; disconnected_hpd_event(&c); modeset_held=false; + c.connected=true; + assert(probe(&c.base)==1 && raw_reads==previous_reads+1); + assert(!c.edid_invalidation_pending); + /* Worker-first ordering also consumes the invalidation after reconnect. */ + modeset_held=true; disconnected_hpd_event(&c); modeset_held=false; + c.connected=true; + dcp_hotplug(&c.hotplug_wq); + assert(!c.drm_edid && !c.edid_invalidation_pending); + assert(probe(&c.base)==1 && c.drm_edid); + initial(); dcp_firmware_metadata_connection(&p,1,true); publish(); + pthread_t a,b,reader; + assert(!pthread_create(&a,NULL,probe_thread,NULL)); + assert(!pthread_create(&b,NULL,hotplug_thread,NULL)); + assert(!pthread_create(&reader,NULL,reader_thread,NULL)); + assert(!pthread_join(a,NULL) && !pthread_join(b,NULL) && !pthread_join(reader,NULL)); + initial(); assert(!blob_allocations); + puts("PASS: mutex-serialized EDID/blob get_modes, hotplug, sysfs reader and same-epoch publication, first-connection scope, raw precedence, invalidation, no added modes"); + return 0; +} +''' +with tempfile.TemporaryDirectory(prefix='dcp-metadata-lifetime-') as tmp: + tmp = Path(tmp) + (tmp / 'test.c').write_text(prefix + metadata + epoch + state + cache + chunk_callbacks + cleanup + getter + hotplug + disconnect + suffix) + command = shlex.split(os.environ.get('CC','clang')) + command += ['-std=gnu11','-Wall','-Wextra','-Werror','-Wno-sign-compare','-O1','-g', + '-fsanitize=address,undefined','-fno-omit-frame-pointer','-pthread'] + subprocess.run(command + [str(tmp / 'test.c'),'-o',str(tmp / 'test')],check=True) + subprocess.run([str(tmp / 'test')],check=True) diff --git a/tools/diagnostics/test-apple-display-metadata.py b/tools/diagnostics/test-apple-display-metadata.py new file mode 100644 index 0000000000000..e0508bbf431d8 --- /dev/null +++ b/tools/diagnostics/test-apple-display-metadata.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise the actual metadata parser/builder and independently decode base EDID.""" +import argparse +import copy +import json +import os +from pathlib import Path +import shlex +import struct +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +base = root / 'drivers/gpu/drm/apple' +args = argparse.ArgumentParser(description=__doc__) +args.add_argument('--attributes-json', type=Path, + default=Path(__file__).with_name('display-attributes-fixture.json')) +args.add_argument('--timings-bin', type=Path, help='Optional captured TimingElements binary (not embedded in public fixtures)') +args = args.parse_args() +fixture = json.loads(args.attributes_json.read_text()) + + +def block(text, marker): + start = text.index(marker) + opening = text.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + + +class ObjectPairs(list): + pass + + +def wire(value, out): + out += b'\0' * (-len(out) % 4) + if isinstance(value, bool): + out += struct.pack('', '') +timing_fixture = [{ + "HorizontalAttributes": {"Total": 2200, "FrontPorch": 88, "SyncWidth": 44, + "Active": 1920, "PreciseSyncRate": 4423680, "SyncPolarity": 1, "PixelRepetition": 0}, + "VerticalAttributes": {"Total": 1125, "FrontPorch": 4, "SyncWidth": 5, + "Active": 1080, "PreciseSyncRate": 3932160, "SyncPolarity": 1, "PixelRepetition": 0}, + "ColorModes": [{"ID": 85, "Score": 1, "IsVirtual": False, "PixelEncoding": 0, + "Colorimetry": 10, "Depth": 10, "DynamicRange": 0, "EOTF": 0}], + "ID": 62, "Score": 16224, "IsPreferred": True, "IsInterlaced": False, "IsVirtual": False, +}] +timing_raw = bytearray(struct.pack(' +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +typedef int64_t s64; +typedef u16 __le16; +typedef u16 __be16; +typedef u32 __le32; +#define __packed __attribute__((packed)) +#undef static_assert +#define static_assert(c) _Static_assert(c, #c) +#define IS_ENABLED(x) 0 +#define U32_MAX UINT32_MAX +#define U16_MAX UINT16_MAX +#define S64_MAX INT64_MAX +#define S64_MIN INT64_MIN +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define round_up(n, a) (((n) + (a) - 1) & ~((a) - 1)) +#define ERR_PTR(n) ((void *)(intptr_t)(n)) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define IS_ERR(p) ((uintptr_t)(p) > (uintptr_t)-4096) +#define IS_ERR_OR_NULL(p) (!(p) || IS_ERR(p)) +#define GFP_KERNEL 0 +#define kmalloc(n, f) malloc(n) +#define kcalloc(n, s, f) calloc(n,s) +#define kfree(p) free(p) +#define mul_u32_u32(a, b) ((u64)(u32)(a) * (u32)(b)) +#define DIV_ROUND_CLOSEST_ULL(n, d) (((n) + (d) / 2) / (d)) +#define DIV_ROUND_CLOSEST(n, d) (((n) + (d) / 2) / (d)) +#define trace_iomfb_color_mode(...) ((void)0) +#define trace_iomfb_parse_mode_fail(...) ((void)0) +#define trace_iomfb_parse_mode_success(...) ((void)0) +#define trace_iomfb_timing_mode(...) ((void)0) +#define DRM_MODE_TYPE_DRIVER 1 +#define DRM_MODE_TYPE_PREFERRED 2 +#define DRM_MODE_FLAG_PHSYNC 1 +#define DRM_MODE_FLAG_NHSYNC 2 +#define DRM_MODE_FLAG_PVSYNC 4 +#define DRM_MODE_FLAG_NVSYNC 8 +#define cpu_to_le16(v) (v) +#define cpu_to_le32(v) (v) +#define cpu_to_be16(v) __builtin_bswap16(v) +struct drm_display_mode { + unsigned int type, flags; + int clock, vdisplay, vsync_start, vsync_end, vtotal; + int hdisplay, hsync_start, hsync_end, htotal, width_mm, height_mm; +}; +static void drm_mode_set_name(struct drm_display_mode *mode) { (void)mode; } +''' +stubs = r''' +struct drm_edid { struct edid raw; }; +static const struct drm_edid *drm_edid_alloc(const void *data, size_t size) +{ + assert(size == 128); + struct drm_edid *result = malloc(sizeof(*result)); + memcpy(&result->raw, data, size); + return result; +} +static void drm_edid_free(const struct drm_edid *edid) { free((void *)edid); } +static bool drm_edid_valid(const struct drm_edid *edid) +{ + const u8 *raw = (const u8 *)&edid->raw; + unsigned int sum = 0; + for (unsigned int i = 0; i < 128; i++) sum += raw[i]; + return !(sum & 255) && !raw[126] && !memcmp(raw, "\0\xff\xff\xff\xff\xff\xff\0", 8); +} +''' +suffix = r''' +static void epochs(void) +{ + struct dcp_metadata_epoch e = {0}; + struct dcp_metadata_ticket before = dcp_metadata_begin(&e); + assert(!dcp_metadata_matches(&e, before)); + dcp_metadata_epoch_connection(&e, 1, true); + struct dcp_metadata_ticket current = dcp_metadata_begin(&e); + assert(dcp_metadata_matches(&e, current)); + assert(!dcp_metadata_matches(&e, before)); /* Never retag pre-association metadata. */ + dcp_metadata_epoch_connection(&e, 1, true); + assert(dcp_metadata_matches(&e, current)); + dcp_metadata_epoch_connection(&e, 2, false); + assert(!dcp_metadata_matches(&e, current)); + dcp_metadata_epoch_connection(&e, 3, true); + assert(!dcp_metadata_matches(&e, dcp_metadata_begin(&e))); /* Later attachment ineligible. */ + e = (struct dcp_metadata_epoch){0}; + dcp_metadata_epoch_connection(&e, 9, true); + current = dcp_metadata_begin(&e); + dcp_metadata_epoch_connection(&e, 10, true); /* A route change also invalidates. */ + assert(!dcp_metadata_matches(&e, current)); +} +int main(int argc, char **argv) +{ + assert(argc == 4 || argc == 5); + epochs(); + timing_proof(); + FILE *f = fopen(argv[1], "rb"); + assert(f && !fseek(f, 0, SEEK_END)); + long length = ftell(f); + assert(length > 0 && !fseek(f, 0, SEEK_SET)); + void *data = malloc(length); + assert(fread(data, length, 1, f) == 1); + fclose(f); + struct dcp_parse_ctx ctx; + struct dcp_display_metadata metadata, unchanged; + memset(&metadata, 0xa5, sizeof(metadata)); unchanged = metadata; + assert(!parse(data, length, &ctx)); + int ret = parse_display_metadata(&ctx, &metadata); + if (atoi(argv[2])) { + assert(ret == -EINVAL && !memcmp(&metadata, &unchanged, sizeof(metadata))); + free(data); return 0; + } + assert(!ret); + struct dcp_parse_ctx timing_ctx; + unsigned int count = 0; + assert(!parse(timing_blob, sizeof(timing_blob), &timing_ctx)); + struct dcp_display_mode *timings = enumerate_modes(&timing_ctx, &count, + metadata.width_cm * 10, metadata.height_cm * 10, 0, false, true); + assert(!IS_ERR(timings) && count == 1 && timings[0].metadata_timing_valid); + struct drm_display_mode mode = timings[0].mode; + assert(mode.flags == (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC)); + assert(mode.clock == 148500 && mode.hdisplay == 1920 && mode.vdisplay == 1080); + free(timings); + const struct drm_edid *edid = dcp_metadata_edid(&metadata, &mode); + assert(!IS_ERR(edid)); + f = fopen(argv[3], "wb"); assert(f); + assert(fwrite(&edid->raw, 128, 1, f) == 1); fclose(f); + drm_edid_free(edid); + mode.flags = 0; + assert(IS_ERR(dcp_metadata_edid(&metadata, &mode))); + mode.flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC; + mode.clock++; + assert(IS_ERR(dcp_metadata_edid(&metadata, &mode))); + free(data); + if (argc == 5) { + f=fopen(argv[4],"rb"); assert(f && !fseek(f,0,SEEK_END)); + length=ftell(f); assert(length>0 && !fseek(f,0,SEEK_SET)); + data=malloc(length); assert(fread(data,length,1,f)==1); fclose(f); + assert(!parse(data,length,&timing_ctx)); count=0; + timings=enumerate_modes(&timing_ctx,&count,600,330,0,false,true); + assert(!IS_ERR(timings)); bool found60=false, found120=false; + for(unsigned int i=0;iH', 1138) + assert struct.unpack_from('> 4) << 8) == 1920 + assert raw[57] + ((raw[58] & 15) << 8) == 280 + assert raw[59] + ((raw[61] >> 4) << 8) == 1080 + assert raw[60] + ((raw[61] & 15) << 8) == 45 + assert raw[62] == 88 and raw[63] == 44 and raw[64] == 0x45 + assert raw[71] == 0x1e + chroma = [raw[27] * 4 + ((raw[25] >> 6) & 3), + raw[28] * 4 + ((raw[25] >> 4) & 3), + raw[29] * 4 + ((raw[25] >> 2) & 3), raw[30] * 4 + (raw[25] & 3), + raw[31] * 4 + ((raw[26] >> 6) & 3), + raw[32] * 4 + ((raw[26] >> 4) & 3), + raw[33] * 4 + ((raw[26] >> 2) & 3), raw[34] * 4 + (raw[26] & 3)] + assert chroma == [697,317,265,632,146,81,321,337] + assert raw[72:77] == bytes.fromhex('000000fc00') + assert b'QG270 S3' in raw and b'DCP metadata' in raw + result = subprocess.run(['edid-decode', '--check', str(tmp / 'edid')], capture_output=True, text=True) + if result.returncode: + print(result.stdout) + print(result.stderr) + assert result.returncode == 0 + print(f'PASS: {len(cases)} actual schema cases, epoch transitions, exact base block and independent edid-decode') diff --git a/tools/diagnostics/test-apple-external-sdr.py b/tools/diagnostics/test-apple-external-sdr.py index 1ece679366267..695a6059da419 100644 --- a/tools/diagnostics/test-apple-external-sdr.py +++ b/tools/diagnostics/test-apple-external-sdr.py @@ -201,10 +201,14 @@ def restricted(rate=60, unsafe=None, dsc=None, colors=None, before=False): #define trace_iomfb_parse_mode_fail(...) ((void)0) #define trace_iomfb_parse_mode_success(...) ((void)0) #define trace_iomfb_timing_mode(...) ((void)0) +#define DRM_MODE_FLAG_PHSYNC 1 +#define DRM_MODE_FLAG_NHSYNC 2 +#define DRM_MODE_FLAG_PVSYNC 4 +#define DRM_MODE_FLAG_NVSYNC 8 #define DRM_MODE_TYPE_DRIVER 1 #define DRM_MODE_TYPE_PREFERRED 2 struct drm_display_mode { - unsigned int type; + unsigned int type, flags; int clock, vdisplay, vsync_start, vsync_end, vtotal; int hdisplay, hsync_start, hsync_end, htotal, width_mm, height_mm; }; From 070123c2917ecc01a397df0162feca2295df7d8d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:54:54 +1100 Subject: [PATCH 0942/1579] drm/apple: trace external metadata admission state Record bounded connection tickets and validity flags while holding the metadata lock. Trace initial association, property receipt and probing to distinguish an ineligible connection from incomplete or rejected metadata without logging monitor contents or weakening admission. The tracepoint is disabled by default. Exercise the observation path in the existing metadata lifetime test. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 25 ++++++++++++++++- drivers/gpu/drm/apple/trace.h | 28 +++++++++++++++++++ .../test-apple-display-metadata-lifetime.py | 8 +++++- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 6a6ad92c0a891..3c3be773c9db7 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1807,6 +1807,23 @@ static bool dcp_firmware_metadata_enabled(struct apple_dcp *dcp) return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; } +/* Called under metadata.lock; no monitor contents or pointers are exposed. */ +static void dcp_trace_metadata(struct apple_dcp *dcp, const char *phase, bool valid) +{ + u32 flags = (dcp->metadata.epoch.connected ? BIT(0) : 0) | + (dcp->metadata.epoch.associated_once ? BIT(1) : 0) | + (dcp->metadata.epoch.invalidated ? BIT(2) : 0) | + (dcp->metadata.attributes_valid ? BIT(3) : 0) | + (dcp->metadata.timing_valid ? BIT(4) : 0) | + (dcp->chunks.metadata_ticket.associated ? BIT(5) : 0) | + (dcp->chunks.contiguous ? BIT(6) : 0) | (valid ? BIT(7) : 0); + + if (dcp_firmware_metadata_enabled(dcp)) + trace_iomfb_metadata(dcp, phase, dcp->metadata.epoch.generation, + dcp->chunks.metadata_ticket.generation, flags, + dcp->chunks.received, dcp->chunks.length); +} + void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected) { unsigned long flags; @@ -1820,6 +1837,7 @@ void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, boo dcp->metadata.hpd_seen = false; } dcp_metadata_epoch_connection(&dcp->metadata.epoch, generation, connected); + dcp_trace_metadata(dcp, "connection", connected); spin_unlock_irqrestore(&dcp->metadata.lock, flags); } @@ -1838,6 +1856,7 @@ void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) dcp->metadata.attributes_valid = false; dcp->metadata.timing_valid = false; } + dcp_trace_metadata(dcp, "hotplug", connected); spin_unlock_irqrestore(&dcp->metadata.lock, flags); } @@ -1847,9 +1866,10 @@ void dcp_firmware_metadata_begin(struct apple_dcp *dcp) spin_lock_irqsave(&dcp->metadata.lock, flags); dcp->chunks.metadata_ticket = dcp_metadata_begin(&dcp->metadata.epoch); - spin_unlock_irqrestore(&dcp->metadata.lock, flags); dcp->chunks.received = 0; dcp->chunks.contiguous = true; + dcp_trace_metadata(dcp, "begin", dcp->chunks.metadata_ticket.associated); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); } static void dcp_firmware_metadata_notify(struct apple_dcp *dcp) @@ -1887,6 +1907,7 @@ void dcp_firmware_metadata_attributes(struct apple_dcp *dcp, dcp->metadata.attributes_ticket = dcp->chunks.metadata_ticket; } } + dcp_trace_metadata(dcp, "attributes", !!attributes); spin_unlock_irqrestore(&dcp->metadata.lock, flags); dcp_firmware_metadata_notify(dcp); } @@ -1914,6 +1935,7 @@ void dcp_firmware_metadata_timing(struct apple_dcp *dcp) break; } unlock: + dcp_trace_metadata(dcp, "timing", dcp->metadata.timing_valid); spin_unlock_irqrestore(&dcp->metadata.lock, flags); dcp_firmware_metadata_notify(dcp); } @@ -1944,6 +1966,7 @@ const struct drm_edid *dcp_firmware_edid(struct apple_dcp *dcp, u64 *generation) valid = dcp->metadata.attributes_valid && dcp->metadata.timing_valid && dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.attributes_ticket) && dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.timing_ticket); + dcp_trace_metadata(dcp, "probe", valid); if (valid) { attributes = dcp->metadata.attributes; preferred = dcp->metadata.preferred; diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index d19609f116f2c..0af0e0d0d654a 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -76,6 +76,34 @@ { DPTX_APCALL_DEVICE_BUSY_TIMEOUT, "device_busy_timeout" }, \ { DPTX_APCALL_DEVICE_NOT_STARTED, "device_not_started" }) +TRACE_EVENT(iomfb_metadata, + TP_PROTO(struct apple_dcp *dcp, const char *phase, u64 generation, + u64 ticket_generation, u32 flags, u32 received, u32 length), + TP_ARGS(dcp, phase, generation, ticket_generation, flags, received, length), + TP_STRUCT__entry( + __string(devname, dev_name(dcp->dev)) + __string(phase, phase) + __field(u64, generation) + __field(u64, ticket_generation) + __field(u32, flags) + __field(u32, received) + __field(u32, length) + ), + TP_fast_assign( + __assign_str(devname); + __assign_str(phase); + __entry->generation = generation; + __entry->ticket_generation = ticket_generation; + __entry->flags = flags; + __entry->received = received; + __entry->length = length; + ), + TP_printk("%s: %s generation=%llu ticket=%llu flags=%#x received=%u length=%u", + __get_str(devname), __get_str(phase), __entry->generation, + __entry->ticket_generation, __entry->flags, + __entry->received, __entry->length) +); + TRACE_EVENT(dcp_recv_msg, TP_PROTO(struct apple_dcp *dcp, u8 endpoint, u64 message), TP_ARGS(dcp, endpoint, message), diff --git a/tools/diagnostics/test-apple-display-metadata-lifetime.py b/tools/diagnostics/test-apple-display-metadata-lifetime.py index bc64678c26d5d..d85daa509ef8c 100644 --- a/tools/diagnostics/test-apple-display-metadata-lifetime.py +++ b/tools/diagnostics/test-apple-display-metadata-lifetime.py @@ -57,6 +57,12 @@ def function(text, signature): typedef uint16_t u16; typedef uint32_t u32; typedef uint64_t u64; +#define BIT(n) (1u << (n)) +static unsigned int trace_observations; +#define trace_iomfb_metadata(d,p,g,t,f,r,l) do { \ + (void)(d); (void)(p); (void)(g); (void)(t); (void)(f); (void)(r); (void)(l); \ + trace_observations++; \ +} while (0) #define READ_ONCE(x) (x) #define WRITE_ONCE(x,v) ((x)=(v)) #define xchg(p,v) __atomic_exchange_n(p,v,__ATOMIC_SEQ_CST) @@ -380,7 +386,7 @@ def function(text, signature): assert(!pthread_create(&b,NULL,hotplug_thread,NULL)); assert(!pthread_create(&reader,NULL,reader_thread,NULL)); assert(!pthread_join(a,NULL) && !pthread_join(b,NULL) && !pthread_join(reader,NULL)); - initial(); assert(!blob_allocations); + initial(); assert(!blob_allocations && trace_observations); puts("PASS: mutex-serialized EDID/blob get_modes, hotplug, sysfs reader and same-epoch publication, first-connection scope, raw precedence, invalidation, no added modes"); return 0; } From b44fc82491ed70a81f1d514492c76d4df73f6df2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:59:53 +1100 Subject: [PATCH 0943/1579] drm/apple: validate delivered property extent before metadata admission H17P advertises one extra byte after the word-aligned serialized property, but does not send that byte. Requiring received == advertised discarded complete same-connection TimingElements and DisplayAttributes. Validate the complete serialized object against bytes actually received. Accept exact lengths or the observed one-byte framing convention only; reject gaps, excess trailing content and truncated values. Perform the bounded structural walk outside the metadata spinlock, then retain the existing connection-ticket checks. No absent byte is used as payload. Exercise captured property extents, truncation mutations and a missing final string byte alongside metadata lifetime and decoder tests. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 6 ++-- drivers/gpu/drm/apple/iomfb_template.c | 6 ++++ drivers/gpu/drm/apple/parser.c | 16 ++++++++++ drivers/gpu/drm/apple/parser.h | 1 + .../test-apple-display-metadata-lifetime.py | 5 +++- .../test-apple-display-metadata.py | 30 +++++++++++++++++++ 7 files changed, 62 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index cd265c6ced986..0f108bc2a1880 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -107,6 +107,7 @@ struct dcp_chunks { void *data; size_t received; bool contiguous; + bool payload_complete; struct dcp_metadata_ticket metadata_ticket; }; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 3c3be773c9db7..0bffc92c19959 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1816,7 +1816,8 @@ static void dcp_trace_metadata(struct apple_dcp *dcp, const char *phase, bool va (dcp->metadata.attributes_valid ? BIT(3) : 0) | (dcp->metadata.timing_valid ? BIT(4) : 0) | (dcp->chunks.metadata_ticket.associated ? BIT(5) : 0) | - (dcp->chunks.contiguous ? BIT(6) : 0) | (valid ? BIT(7) : 0); + (dcp->chunks.contiguous ? BIT(6) : 0) | (valid ? BIT(7) : 0) | + (dcp->chunks.payload_complete ? BIT(8) : 0); if (dcp_firmware_metadata_enabled(dcp)) trace_iomfb_metadata(dcp, phase, dcp->metadata.epoch.generation, @@ -1868,6 +1869,7 @@ void dcp_firmware_metadata_begin(struct apple_dcp *dcp) dcp->chunks.metadata_ticket = dcp_metadata_begin(&dcp->metadata.epoch); dcp->chunks.received = 0; dcp->chunks.contiguous = true; + dcp->chunks.payload_complete = false; dcp_trace_metadata(dcp, "begin", dcp->chunks.metadata_ticket.associated); spin_unlock_irqrestore(&dcp->metadata.lock, flags); } @@ -1888,7 +1890,7 @@ static void dcp_firmware_metadata_notify(struct apple_dcp *dcp) static bool dcp_firmware_metadata_complete(struct apple_dcp *dcp) { - return dcp->chunks.contiguous && dcp->chunks.received == dcp->chunks.length && + return dcp->chunks.contiguous && dcp->chunks.payload_complete && dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket); } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 03438d24a7557..cfbea578769fa 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -986,6 +986,12 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, return false; } + /* Validate serialized extent outside the metadata spinlock. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + dcp->chunks.payload_complete = dcp->chunks.contiguous && + dcp_property_complete(dcp->chunks.data, dcp->chunks.received, + dcp->chunks.length); + /* used just as opaque pointer for tracing */ ctx.dcp = dcp; diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index b723fba76d456..8c2a1069417b6 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -299,6 +299,22 @@ int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx) return 0; } +bool dcp_property_complete(const void *blob, size_t received, size_t advertised) +{ + struct dcp_parse_ctx ctx; + + if (!blob || !received || received > U32_MAX - 3 || + received > advertised || advertised - received > 1) + return false; + /* H17P may count one unsent byte after the aligned serialized object. */ + if (advertised != received && (received & 3)) + return false; + /* Bound decoding to bytes actually delivered, never allocation padding. */ + if (parse(blob, received, &ctx) || skip(&ctx)) + return false; + return ctx.pos == received || round_up(ctx.pos, 4) == received; +} + static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, bool polarity) { struct iterator it; diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 50e0580f98080..0392b09751ac9 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -107,6 +107,7 @@ struct dimension { }; int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); +bool dcp_property_complete(const void *blob, size_t received, size_t advertised); struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, diff --git a/tools/diagnostics/test-apple-display-metadata-lifetime.py b/tools/diagnostics/test-apple-display-metadata-lifetime.py index d85daa509ef8c..407c8a2da0229 100644 --- a/tools/diagnostics/test-apple-display-metadata-lifetime.py +++ b/tools/diagnostics/test-apple-display-metadata-lifetime.py @@ -43,6 +43,7 @@ def function(text, signature): assert 'dcp_firmware_metadata_hotplug(dcp, !!(*connected));' in template assert 'dcp_firmware_metadata_attributes(dcp, ret ? NULL : &metadata);' in template assert 'dcp->chunks.received += req->length;' in template +assert 'dcp_property_complete(dcp->chunks.data, dcp->chunks.received,' in template prefix = r''' #include #include @@ -124,7 +125,7 @@ def function(text, signature): void *dev; struct { int iomfb_method_profile; } hw; struct apple_connector *connector; - struct { void *data; size_t length, received; bool contiguous; struct dcp_metadata_ticket metadata_ticket; } chunks; + struct { void *data; size_t length, received; bool contiguous, payload_complete; struct dcp_metadata_ticket metadata_ticket; } chunks; struct { spinlock_t lock; struct dcp_metadata_epoch epoch; @@ -239,6 +240,7 @@ def function(text, signature): { dcp_firmware_metadata_begin(&p); p.chunks.length=100; p.chunks.received=complete ? 100 : 99; + p.chunks.payload_complete=complete; } static bool eligible(void) { @@ -344,6 +346,7 @@ def function(text, signature): assert(p.chunks.received==32 && p.chunks.contiguous); dcp_firmware_metadata_attributes(&p,&attrs); assert(!p.metadata.attributes_valid); chunk.offset=32; assert(dcpep_cb_prop_chunk(&p,&chunk)); + p.chunks.payload_complete=true; /* Structural validator tested separately. */ dcp_firmware_metadata_attributes(&p,&attrs); assert(p.metadata.attributes_valid); initial(); dcp_firmware_metadata_connection(&p,1,true); assert(dcpep_cb_prop_start(&p,&length)); diff --git a/tools/diagnostics/test-apple-display-metadata.py b/tools/diagnostics/test-apple-display-metadata.py index e0508bbf431d8..8a5e34e6c006f 100644 --- a/tools/diagnostics/test-apple-display-metadata.py +++ b/tools/diagnostics/test-apple-display-metadata.py @@ -17,6 +17,8 @@ args.add_argument('--attributes-json', type=Path, default=Path(__file__).with_name('display-attributes-fixture.json')) args.add_argument('--timings-bin', type=Path, help='Optional captured TimingElements binary (not embedded in public fixtures)') +args.add_argument('--property-bin', type=Path, action='append', default=[], + help='Optional captured property to validate against delivered extent') args = args.parse_args() fixture = json.loads(args.attributes_json.read_text()) @@ -219,7 +221,23 @@ def wire(value, out): } int main(int argc, char **argv) { + if (argc == 3 && !strcmp(argv[1], "--property")) { + FILE *f=fopen(argv[2],"rb"); assert(f && !fseek(f,0,SEEK_END)); + long n=ftell(f); assert(n>20 && !fseek(f,0,SEEK_SET)); + unsigned char *b=malloc(n); assert(fread(b,n,1,f)==1); fclose(f); + assert(dcp_property_complete(b,n-1,n)); + assert(dcp_property_complete(b,n-1,n-1)); + for(unsigned int cut=1;cut<=16;cut++) + assert(!dcp_property_complete(b,n-1-cut,n-cut)); + assert(!dcp_property_complete(b,n-1,n+1)); + free(b); return 0; + } assert(argc == 4 || argc == 5); + /* A missing final string byte must not be supplied by zero allocation. */ + unsigned char incomplete[]={0xd3,0,0,0,1,0,0,0x82,1,0,0,0x89,0x42}; + assert(dcp_property_complete(incomplete,sizeof(incomplete),sizeof(incomplete))); + assert(!dcp_property_complete(incomplete,12,13)); + assert(!dcp_property_complete(NULL,0,0)); epochs(); timing_proof(); FILE *f = fopen(argv[1], "rb"); @@ -239,6 +257,16 @@ def wire(value, out): free(data); return 0; } assert(!ret); + assert(dcp_property_complete(data,length,length)); + size_t aligned=round_up((size_t)length,4); + unsigned char *padded=malloc(aligned+4); + memset(padded,0xa5,aligned+4); memcpy(padded,data,length); + assert(dcp_property_complete(padded,aligned,aligned+1)); + assert(!dcp_property_complete(data,length-1,length)); + assert(!dcp_property_complete(padded,aligned,aligned+2)); + assert(!dcp_property_complete(padded,aligned+4,aligned+5)); + assert(!dcp_property_complete(padded,aligned-1,aligned+1)); + free(padded); struct dcp_parse_ctx timing_ctx; unsigned int count = 0; assert(!parse(timing_blob, sizeof(timing_blob), &timing_ctx)); @@ -331,6 +359,8 @@ def wire(value, out): raw = bytearray(struct.pack(' Date: Mon, 12 Oct 2026 21:49:16 +1100 Subject: [PATCH 0944/1579] drm/apple: publish initial external discovery while presents remain paused The lifecycle gate cached the first connected HPD while waiting for DRM activation to unpause presentation. DRM could not activate a connector that had never been published, leaving the R35 external output absent even with a connected DPTX link and valid mode metadata. Publish cable-backed initial discovery before the first power attempt, while retaining the presentation pause. Sticky failures, absent cables and OFF/restart callbacks remain withheld. Receive callbacks still acquire no lifecycle mutex or issue firmware control requests. The regression test fails on the prior production function and passes with this change; it verifies that discovery does not power or unpause the pipe. ASan/UBSan lifecycle, metadata and delayed-service tests pass. Hardware activation and OFF-response qualification remain pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 9 +++++++++ .../diagnostics/test-apple-external-lifecycle.py | 16 ++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 148f3ee692a0f..d01725a99103f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2671,6 +2671,15 @@ bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) if (dcp->iomfb.presentation_paused) { dcp->external_hotplug_pending = true; dcp->external_hotplug_connected = connected; + /* Publish discovery so DRM can activate the first CRTC; keep presents paused. */ + if (connected && READ_ONCE(dcp->external_cable) && + !READ_ONCE(dcp->external_power_attempted) && !READ_ONCE(dcp->external_error)) { + unsigned int action = dcp_mode_hotplug(&dcp->mode_state, true, + dcp->connector ? + &dcp->connector->connected : NULL); + + dcp_handle_hotplug_actions(dcp, action); + } return true; } if (!connected && !READ_ONCE(dcp->iomfb.presentation_paused)) { diff --git a/tools/diagnostics/test-apple-external-lifecycle.py b/tools/diagnostics/test-apple-external-lifecycle.py index b676b4b45076c..ca24415533bc3 100644 --- a/tools/diagnostics/test-apple-external-lifecycle.py +++ b/tools/diagnostics/test-apple-external-lifecycle.py @@ -475,16 +475,28 @@ def external_branch(name, result): assert(dcp_dptx_connect_oob(&dev,0)==-EAGAIN && p.typec_generation==1); assert(dcp_metadata_matches(&p.metadata_epoch,first)); - /* Link hookup alone never powers the pipe. A first activation preserves DA1 order. */ + /* Discovery publishes physical HPD before DRM activation, without unpausing the pipe. */ initial(&p); p.external_power_attempted = p.external_running = false; p.iomfb.presentation_paused = true; - p.dptxport[0].connected = false; + p.dptxport[0].connected = false; connector.connected=false; assert(dcp_dptx_connect_oob(&dev, 0) == 0 && connects == 1 && !power1); + assert(connector.connected && p.iomfb.presentation_paused); dcp_external_recover(&dev); assert(strstr(events, "DA1") && p.external_running && power1 == 1); owned(&p); + /* Discovery cannot publish a missing cable, a sticky error, or an OFF/restart callback. */ + for(unsigned int reason=0;reason<3;reason++) { + initial(&p); connector.connected=false; p.iomfb.presentation_paused=true; + p.external_power_attempted=false; + if(reason==0)p.external_cable=false; + if(reason==1)p.external_error=-EIO; + if(reason==2)p.external_power_attempted=true; + mutex_lock(&p.iomfb.lock); assert(dcp_external_hotplug(&p,true)); mutex_unlock(&p.iomfb.lock); + assert(!connector.connected && p.iomfb.presentation_paused && !power1); + } + /* Exact ON requires the 64-bit reply 1; all-zero must not activate modesets. */ initial(&p); p.external_running = false; p.external_stopped = true; From 30812f491575ede27e5be059fd6afc4e985166f1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 12 Oct 2026 14:30:09 -0400 Subject: [PATCH 0945/1579] arm64: dts: apple: t8140-j700: keep the DCS rail out of thermal control The soc-dcs-thermal zone let the power allocator cap the DRAM controller (DCS) rail through the PMC whenever Tp0t passed 95 C. Most of that heat comes from the GPU and CPUs rather than from the DCS rail, so the governor could not settle. Under a sustained Minecraft load it stepped the cap between states 1 and 4 every 1-2 s, and at state 4 the rail was held at its lowest state while the PMP and CLPC both voted the highest. The game's GPU fence rate followed the cap (Pearson r = -0.98): about 2,000/s at state 4 and 5,200/s at state 1. In game the frame rate was seen swinging from about 150 FPS down to 30-50 FPS and back every few seconds. With the zone disabled and the cap lifted on the same session, the rate held between 4,500 and 4,900/s. Tp0t settled near 100 C (peak 105 C), and the CPU die sensors stayed at 88-97 C, under the 110 C critical trips of the CPU zones. The PMP already drops its DCS vote when the GPU idles, so the rail only needs to follow demand. Drop the zone. The PMC keeps its cooling cells and power model. Assisted-by: LLM Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t8140-j700.dts | 56 ++++-------------------- 1 file changed, 9 insertions(+), 47 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 842e1b2bcf572..066175edbc83a 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -381,52 +381,6 @@ }; }; }; - - /* - * Once the PMP knows the GPU is on, it votes the DRAM controller - * (DCS) up with GPU load, and the GPU and memory system draw up - * to 2.4 W more. The CPU zones above cannot hold that back: on a - * GPU-bound load the CPUs request little power, so with the - * performance governor the die passed 95 C within 35 s and - * reached 104 C before they throttled. This zone limits the DCS - * vote instead, through the PMC. With it the same load peaks at - * 97 C, as it does without any DCS vote, and a ten-minute - * balanced GPU load loses 1-2% of its throughput. - * - * Tp0t reads at or above every CPU die sensor under GPU load, - * and leads them by about 1 C when the CPUs also run hot. The - * sustainable power is the whole DCS range, so the vote is only - * trimmed above the control trip; the PID gains follow from it - * and the 6 C between the trips. - */ - soc-dcs-thermal { - polling-delay-passive = <500>; - polling-delay = <500>; - thermal-sensors = <&smc_hwmon 6>; /* Tp0t */ - sustainable-power = <2407>; - - trips { - soc_dcs_alert: trip-alert { - temperature = <89000>; - hysteresis = <2000>; - type = "passive"; - }; - - soc_dcs_control: trip-control { - temperature = <95000>; - hysteresis = <2000>; - type = "passive"; - }; - }; - - cooling-maps { - map0 { - trip = <&soc_dcs_control>; - cooling-device = - <&pmc THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; - }; }; }; @@ -472,6 +426,14 @@ /* * System power above DCS state 0 at each DCS state, measured with * glmark2's terrain scene, GPU-bound, on the J700. + * + * No thermal zone limits the DCS rail. The PMP already drops its + * DCS vote when the GPU idles, and under load the rail must stay + * where the PMP puts it: a power allocator zone on Tp0t at 95 C + * stepped the limit between states 1 and 4 every 1-2 s in + * Minecraft, swinging the game's GPU submission rate about 2.5x. + * With the limit lifted the rate held within 8% and Tp0t settled + * near 100 C, under the 110 C critical trips of the CPU zones. */ apple,dcs-power-microwatt = <0 919000 1453000 2058000 2407000>; status = "okay"; @@ -676,7 +638,7 @@ * the SMC key groups and are not otherwise verified. */ - /* Channel 6, the sensor of soc-dcs-thermal: keep it first here. */ + /* Channel 6: reads at or above every CPU die sensor under GPU load. */ temperature-Tp0t { apple,key-id = "Tp0t"; label = "SMC Tp0t"; From a7cf0b40d0d2ada11558835cbdc177547de232f6 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Mon, 12 Oct 2026 21:32:58 +0200 Subject: [PATCH 0946/1579] cpuidle: apple: retain a selectable WFI state for T8140 s2idle T8140 exposes only the index-0 WFI fallback to avoid the unsupported CPU power-down path. The generic s2idle selector skips index 0, so this prevents tick_freeze() and timekeeping suspension from being reached. Copy the safe WFI state over the T8140 power-down entry instead of reducing state_count. This preserves an eligible nonzero s2idle entry without using CYC_OVRD, while leaving other Apple SoCs unchanged. On MacBook Neo, three 20-second RTC sleeps accounted for approximately zero suspended time before this change and 20.198, 19.808 and 20.419 seconds afterward using the identical BOOTTIME-minus-MONOTONIC test. Signed-off-by: Bartosz Nitka --- drivers/cpuidle/cpuidle-apple.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c index 6c02bd4197c26..64b45dcbafb31 100644 --- a/drivers/cpuidle/cpuidle-apple.c +++ b/drivers/cpuidle/cpuidle-apple.c @@ -127,10 +127,12 @@ static int apple_cpuidle_probe(struct platform_device *pdev) /* * CPU PD writes CYC_OVRD, which is an undefined instruction on * T8140. Its WFI already lets the hardware power-gate idle - * secondary cores, so expose only the WFI state. + * secondary cores, so both entries must use WFI. Keep index 1: + * the generic s2idle selector skips fallback index 0, otherwise + * tick_freeze() and timekeeping suspension are never reached. */ if (of_machine_is_compatible("apple,t8140")) - apple_idle_driver.state_count = STATE_WFI + 1; + apple_idle_driver.states[STATE_PWRDOWN] = apple_idle_driver.states[STATE_WFI]; return cpuidle_register(&apple_idle_driver, NULL); } From 47d7269a5e2bc7a59a1309c31cd967bb2af3f457 Mon Sep 17 00:00:00 2001 From: nekolyanich Date: Mon, 12 Oct 2026 22:46:12 +0300 Subject: [PATCH 0947/1579] drm: apple: answer the DPTX port's group 1 calls as macOS does On t6020 with the 13.5 firmware, DCP's DPTX port sends AP calls of group 1, commands 3 and 4, around link (re)configuration, audio prepare included. The AFK v1 notify path dropped the group, so they reached the group 0 handlers of the same numbers, SET/GET_DRIVE_SETTINGS: DCP addresses from the payload were stored as drive settings. Behind a USB-C to HDMI adapter DCP then refused the audio start ("Capabilities changed") or never confirmed the stream. Pass the group through a call_v2 op, with the signature of call_v2 on aurora's J700 test branch (85d81b76, not in aurora-wip), and keep .call for group 0 only. A group call to a service without call_v2 gets a warning and no reply: AFK has no handler and does not guess one. The DPTX port answers group 1 commands 3 and 4 the way macOS 13 does in 42 of 42 traced pairs (TV through the adapter, Dell over Thunderbolt): an echo of the request with +0x00..0x03 cleared for (1, 3), and +0x10..0x13 and +0x20 to the end cleared for (1, 4). The lane count sits at +0x10 in (1, 3) and at +0x00 in (1, 4); macOS keeps it in all 42 and clears the other slot. The J700 code reads (1, 3) the same way, with per-lane levels from +0x20, which macOS does not clear there; it refuses (1, 4), and its group 0 GET (0, 4) has the count at +0x00. Other groups are echoed with a rate-limited warning. Testing: MacBook Pro 16" M2 Pro (j416s), DCP firmware 13.5, a TV on a USB-C to HDMI adapter and a Dell over Thunderbolt, 8 audio starts on the TV per boot. Cold boots under the m1n1 hypervisor, aurora-wip with PRs #275-#277, the AFK slots series and a bench DT change, interleaved: with this commit the first start was accepted in 5 of 5 boots, with 0 refusals and 8 of 8 starts played; without it, 3 of 3 boots refused the first start, 2 refusals each, 4 of 8 played. The Dell had no refusals in any boot, and no group warning was logged. Natively, on a downstream tree with the same reply rule, the boot connect and 3 of 8 TV replugs reached DCP (the rest failed earlier, in the PHY): 0 refusals in 4 connects; before, 10 refusals in 5 connects. Not tested: other SoCs, other firmware, AFK v2. Assisted-by: Claude:claude-opus-5-5 checkpatch Signed-off-by: nekolyanich --- drivers/gpu/drm/apple/afk.c | 48 ++++++++++++++++++++------- drivers/gpu/drm/apple/afk.h | 2 ++ drivers/gpu/drm/apple/dptxep.c | 60 +++++++++++++++++++++++++++++++++- drivers/gpu/drm/apple/trace.h | 17 ++++++---- 4 files changed, 107 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index ff1f0f57afd9f..18e85187660f0 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -493,14 +493,12 @@ static void afk_recv_handle_reply(struct apple_dcp_afkep *ep, u32 channel, dma_free_coherent(ep->dcp->dev, txlen, txbuf, txbuf_dma); } +/* A call from DCP: one word ahead of the call header AP sends. */ struct epic_std_service_ap_call { __le32 unk0; - __le32 unk1; - __le32 type; - __le32 len; - __le32 magic; - u8 _unk[48]; + struct epic_service_call call; } __attribute__((packed)); +static_assert(sizeof(struct epic_std_service_ap_call) == 68); static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, u32 type, struct epic_hdr *ehdr, @@ -525,30 +523,56 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, if (type == EPIC_TYPE_NOTIFY && eshdr->category == EPIC_CAT_NOTIFY) { struct epic_std_service_ap_call *call = payload; size_t call_size; + u32 command; void *reply; + u16 group; int ret; if (payload_size < sizeof(*call)) return; - call_size = le32_to_cpu(call->len); + call_size = le32_to_cpu(call->call.data_len); if (payload_size < sizeof(*call) + call_size) return; - if (!service->ops->call) + group = le16_to_cpu(call->call.group); + command = le32_to_cpu(call->call.command); + + /* + * Command numbers are per group: a group-less .call handler + * would take another group's call for a group 0 one. Without + * call_v2 there is no handler for it, and AFK cannot guess a + * reply, so it gets none, as a call without any handler. + */ + if (!service->ops->call_v2 && (group || !service->ops->call)) { + if (group) + dev_warn_ratelimited(ep->dcp->dev, + "AFK[ep:%02x]: %s (chan:%u) apcall %u:%u has no handler, sending NO reply\n", + ep->endpoint, service->ops->name, + channel, group, command); return; + } reply = kzalloc(payload_size, GFP_KERNEL); if (!reply) return; - ret = service->ops->call(service, le32_to_cpu(call->type), - payload + sizeof(*call), call_size, - reply + sizeof(*call), call_size); + if (service->ops->call_v2) + ret = service->ops->call_v2(service, group, command, + payload + sizeof(*call), + call_size, + reply + sizeof(*call), + call_size); + else + ret = service->ops->call(service, command, + payload + sizeof(*call), + call_size, + reply + sizeof(*call), + call_size); if (ret) { dev_warn(ep->dcp->dev, - "AFK[ep:%02x]: %s (chan:%u) apcall %u handler returned %d, sending NO reply\n", + "AFK[ep:%02x]: %s (chan:%u) apcall %u:%u handler returned %d, sending NO reply\n", ep->endpoint, service->ops->name, channel, - le32_to_cpu(call->type), ret); + group, command, ret); kfree(reply); return; } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 0cef14fa6dfe8..65696d6ac689e 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -74,6 +74,8 @@ struct apple_epic_service_ops { int (*call)(struct apple_epic_service *service, u32 idx, const void *data, size_t data_size, void *reply, size_t reply_size); + int (*call_v2)(struct apple_epic_service *service, u16 group, u32 command, + const void *data, size_t size, void *reply, size_t room); int (*report)(struct apple_epic_service *service, enum epic_subtype type, const void *data, size_t data_size); void (*teardown)(struct apple_epic_service *service); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 11091084f05e2..946c195520cbc 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -691,7 +691,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, size_t reply_size) { struct dptx_port *dptx = service->cookie; - trace_dptxport_apcall(dptx, idx, data_size); + trace_dptxport_apcall(dptx, 0, idx, data_size); dev_dbg(service->ep->dcp->dev, "DPTXPort: APCALL %u (%zu bytes)\n", idx, data_size); if (data_size) @@ -836,11 +836,69 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, } } +/* + * Group 1 calls 3 and 4 come around link (re)configuration, audio prepare + * included. They carry a lane count at +0x10 in (1, 3) and at +0x00 in + * (1, 4); the J700 code reads (1, 3) as drive settings with levels per + * lane from +0x20, and its group 0 GET (0, 4) has the count at +0x00. + * macOS 13 acks both with the request, (1, 3) with +0x00..0x03 + * cleared and (1, 4) with +0x10..0x13 and +0x20 to the end cleared, so + * what DCP left there does not go back (42 of 42 traced pairs on t6020, + * 13.5 firmware: TV through a USB-C to HDMI adapter, Dell over TB). + */ +static int dptxport_call_group1(u32 command, const void *data, + size_t data_size, void *reply, size_t reply_size) +{ + size_t size = min(reply_size, data_size); + + memcpy(reply, data, size); + if (command == 3) { + memset(reply, 0, min_t(size_t, size, 0x4)); + } else if (size > 0x10) { + memset(reply + 0x10, 0, min_t(size_t, size - 0x10, 0x4)); + if (size > 0x20) + memset(reply + 0x20, 0, size - 0x20); + } + return 0; +} + +static int dptxport_group_call(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t size, + void *reply, size_t room) +{ + struct dptx_port *dptx = service->cookie; + + if (!group) + return dptxport_call(service, command, data, size, reply, room); + + trace_dptxport_apcall(dptx, group, command, size); + dev_dbg(service->ep->dcp->dev, "DPTXPort: APCALL %u:%u (%zu bytes)\n", + group, command, size); + if (size) + print_hex_dump_debug("DPTXPort: apcall data: ", + DUMP_PREFIX_OFFSET, 16, 1, data, + min(size, (size_t)64), true); + + if (group == 1 && (command == 3 || command == 4)) + return dptxport_call_group1(command, data, size, reply, room); + + /* + * Echo it, as group 0 acks an unhandled call: no reply might stall + * DCP's caller. + */ + dev_warn_ratelimited(service->ep->dcp->dev, + "DPTXPort: unknown AP call %u:%u, echoing it\n", + group, command); + memcpy(reply, data, min(room, size)); + return 0; +} + static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", .init = dptxport_init, .call = dptxport_call, + .call_v2 = dptxport_group_call, }, {} }; diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index d3c5106e5d66d..cbb3f4080fd57 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -398,18 +398,21 @@ TRACE_EVENT(dptxport_init, TP_PROTO(struct apple_dcp *dcp, u64 unit), TRACE_EVENT( dptxport_apcall, - TP_PROTO(struct dptx_port *dptx, int idx, size_t len), - TP_ARGS(dptx, idx, len), + TP_PROTO(struct dptx_port *dptx, u16 group, int idx, size_t len), + TP_ARGS(dptx, group, idx, len), TP_STRUCT__entry(__string(devname, dev_name(dptx->service->ep->dcp->dev)) - __field(u32, unit) __field(int, idx) __field(size_t, len)), + __field(u32, unit) __field(u16, group) __field(int, idx) + __field(size_t, len)), TP_fast_assign(__assign_str(devname); - __entry->unit = dptx->unit; __entry->idx = idx; __entry->len = len;), + __entry->unit = dptx->unit; __entry->group = group; + __entry->idx = idx; __entry->len = len;), - TP_printk("%s: dptx%d: AP Call %d (%s) with len %lu", __get_str(devname), - __entry->unit, - __entry->idx, show_dptxport_apcall(__entry->idx), __entry->len)); + TP_printk("%s: dptx%d: AP Call %u:%d (%s) with len %lu", __get_str(devname), + __entry->unit, __entry->group, __entry->idx, + __entry->group ? "-" : show_dptxport_apcall(__entry->idx), + __entry->len)); TRACE_EVENT( dptxport_validate_connection, From cfa762bbc3d0bb0011e9b9abcf048947e3e887a7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 08:55:03 +1100 Subject: [PATCH 0948/1579] Exercise unified public DPTX dispatch and isolate owned colour-test pixel evidence Retain all original J700 boundary/cache/refusal cases with native wire_v2, extract all three production routing functions, and add legacy48/64byte reply/canary/no-PHY assertions. Add optional bounded GSK snapshot of only the owned diagnostic widget to distinguish actual rendered values from compositor/physicalcolour; no desktop capture or output-setting changes. Preserve first extraction shim compile failures and strict Werror/sanitizers. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/diagnostics/apple-dptx-training-test.c | 35 +++++++++++++++++- tools/diagnostics/test-apple-j700-dp.py | 9 +++-- .../diagnostics/wayland-output-frame-test.py | 37 +++++++++++++++++++ 3 files changed, 76 insertions(+), 5 deletions(-) diff --git a/tools/diagnostics/apple-dptx-training-test.c b/tools/diagnostics/apple-dptx-training-test.c index 7e3a7f3b59790..fd2bed592f4c3 100644 --- a/tools/diagnostics/apple-dptx-training-test.c +++ b/tools/diagnostics/apple-dptx-training-test.c @@ -23,11 +23,19 @@ struct dptx_port { u32 trained_voltage[4], trained_pre[4]; }; -struct apple_epic_service { struct dptx_port *cookie; }; +struct endpoint { bool wire_v2; }; +struct apple_epic_service { struct dptx_port *cookie; struct endpoint *ep; }; static unsigned int calls; static int failure; #define PHY_MODE_DP 1 #define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define min(a, b) ((a) < (b) ? (a) : (b)) +#define min_t(t, a, b) min((t)(a), (t)(b)) +#define trace_dptxport_apcall(port, ...) ((void)(port)) +#define dev_dbg(...) ((void)0) +#define dev_warn_ratelimited(...) ((void)0) +#define print_hex_dump_debug(...) ((void)0) +#define DUMP_PREFIX_OFFSET 0 static u32 get_unaligned_le32(const void *p) { @@ -82,7 +90,8 @@ int main(void) { struct phy phy = {}; struct dptx_port port = { .atcphy = &phy }; - struct apple_epic_service service = { .cookie = &port }; + struct endpoint ep = { .wire_v2 = true }; + struct apple_epic_service service = { .cookie = &port, .ep = &ep }; u8 input[81] = {}, output[81]; u8 *request = input + 1, *reply = output + 1; unsigned int cases = 0; @@ -127,6 +136,28 @@ int main(void) request, 80, reply, 80) == expected); } } + /* Real unified dispatcher: legacy replies retain fields/canaries and + * never enter native PHY training. All original v2 cases above stay. + */ + ep.wire_v2 = false; + for (unsigned int command = 3; command <= 4; command++) { + for (unsigned int length = 48; length <= 64; length += 16) { + unsigned int before = calls; + memset(request, 0xa5, 80); + memset(output, 0xc3, sizeof(output)); + assert(!dptxport_group_call(&service, 1, command, + request, length, reply, length)); + for (unsigned int i = 0; i < length; i++) { + bool cleared = command == 3 ? i < 4 : + (i >= 16 && i < 20) || i >= 32; + assert(reply[i] == (cleared ? 0 : 0xa5)); + } + assert(output[0] == 0xc3 && output[length + 1] == 0xc3); + assert(calls == before); + } + } + assert(dptxport_group_call(&service, 0, 3, request, 80, reply, 80) == 7); + ep.wire_v2 = true; put_unaligned_le32(2, request); unsigned int before_readback = calls; diff --git a/tools/diagnostics/test-apple-j700-dp.py b/tools/diagnostics/test-apple-j700-dp.py index 41836939f6eae..1bcb4014b82c6 100644 --- a/tools/diagnostics/test-apple-j700-dp.py +++ b/tools/diagnostics/test-apple-j700-dp.py @@ -10,11 +10,14 @@ root = Path(__file__).resolve().parents[2] source = (root / "drivers/gpu/drm/apple/dptxep.c").read_text() -start = source.index("static int dptxport_group_call(") -end = source.index("\n}\n", start) + 3 +functions = [] +for name in ("dptxport_v2_group_call", "dptxport_call_group1", "dptxport_group_call"): + start = source.index("static int " + name + "(") + end = source.index("\n}\n", start) + 3 + functions.append(source[start:end]) with tempfile.TemporaryDirectory(prefix="j700-dp-") as work: work = Path(work) - (work / "group-call.h").write_text(source[start:end]) + (work / "group-call.h").write_text("\n\n".join(functions)) for name in ("apple-j700-dp-test", "apple-dptx-training-test"): command = shlex.split(os.environ.get("CC", "clang")) command += ["-std=gnu11", "-O1", "-g", "-Wall", "-Wextra", "-Werror"] diff --git a/tools/diagnostics/wayland-output-frame-test.py b/tools/diagnostics/wayland-output-frame-test.py index e9f0893ead65d..e915806ab3544 100644 --- a/tools/diagnostics/wayland-output-frame-test.py +++ b/tools/diagnostics/wayland-output-frame-test.py @@ -3,6 +3,8 @@ """Show a bounded animation on one Wayland output and report frame feedback.""" import argparse import json +import os +import tempfile import time import gi @@ -65,11 +67,46 @@ def finish(): return GLib.SOURCE_REMOVE +def save_widget_pixels(): + try: + if stats["actual_connector"] != args.connector or not stats["draws"]: + raise RuntimeError("test widget not ready on requested connector") + width, height = area.get_width(), area.get_height() + if width <= 0 or height <= 0 or width > 8192 or height > 8192: + raise RuntimeError("widget dimensions out of diagnostic bounds") + paintable = Gtk.WidgetPaintable.new(area) + snapshot = Gtk.Snapshot.new() + paintable.snapshot(snapshot, width, height) + node = snapshot.to_node() + if node is None: + raise RuntimeError("widget snapshot empty") + renderer = window.get_renderer() + texture = renderer.render_texture(node, None) + directory = tempfile.mkdtemp(prefix="aurora-widget-pixels-", dir="/var/tmp") + path = os.path.join(directory, "widget.png") + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "wb") as output: + output.write(texture.save_to_png_bytes().get_data()) + output.flush() + os.fsync(output.fileno()) + stats["widget_png"] = path + stats["widget_dimensions"] = [width, height] + stats["renderer_type"] = renderer.__gtype__.name + stats["widget_pixels_pass"] = True + stats["widget_scope"] = "owned test widget only; before compositor and physical scanout" + except Exception as error: + stats["widget_pixels_pass"] = False + stats["widget_error_type"] = type(error).__name__ + return GLib.SOURCE_REMOVE + + area.set_draw_func(draw) area.add_tick_callback(tick) window.connect("close-request", lambda _window: finish()) window.fullscreen_on_monitor(matches[0]) window.present() +if args.widget_pixels: + GLib.timeout_add_seconds(2, save_widget_pixels) GLib.timeout_add_seconds(args.seconds, finish) loop.run() stats["elapsed_s"] = round(time.monotonic() - started, 3) From f40dc94b73108f1ed461d22bd0cb2fcbb6378be3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 08:58:48 +1100 Subject: [PATCH 0949/1579] Carry current public diagnostic placement and gate requested widget snapshots Use public91997e1s full gamma/output placement contract as the diagnostic base, preserving moving/gamma inputs. Register optionalwidget-pixels, populate actualconnector beforecapture, and require requested snapshot success in exitgrading. Independentreview caught the older mergedhelper mismatch before any execution; olduploadedcfa helper staysunqualified andunused. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- .../diagnostics/wayland-output-frame-test.py | 55 +++++++++++++++++-- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/tools/diagnostics/wayland-output-frame-test.py b/tools/diagnostics/wayland-output-frame-test.py index e915806ab3544..d9fa634ee9cf2 100644 --- a/tools/diagnostics/wayland-output-frame-test.py +++ b/tools/diagnostics/wayland-output-frame-test.py @@ -15,7 +15,9 @@ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--connector", default="DP-1") -parser.add_argument("--seconds", type=int, default=8, choices=range(3, 31)) +parser.add_argument("--seconds", type=int, default=8, choices=range(3, 61)) +parser.add_argument("--pattern", choices=("moving", "gamma"), default="moving") +parser.add_argument("--widget-pixels", action="store_true", help="Save only this test widget, before compositor/scanout") args = parser.parse_args() Gtk.init() display = Gdk.Display.get_default() @@ -32,12 +34,44 @@ area = Gtk.DrawingArea() window.set_child(area) started = time.monotonic() -stats = {"connector": args.connector, "ticks": 0, "draws": 0} +stats = {"connector": args.connector, "pattern": args.pattern, "ticks": 0, "draws": 0, + "actual_connector": None, "physical_visibility": "requires user observation"} feedback = set() complete_frames = set() def draw(_area, context, width, height): + surface = window.get_surface() + monitor = display.get_monitor_at_surface(surface) if surface else None + if monitor: + stats["actual_connector"] = monitor.get_connector() + if args.pattern == "gamma": + context.set_source_rgb(0.08, 0.08, 0.08) + context.paint() + context.set_source_rgb(1, 1, 1) + context.select_font_face("Sans") + context.set_font_size(24) + context.move_to(24, 48) + context.show_text("Match a solid grey to the checkerboard brightness; view from a distance") + y, h = int(height * 0.25), int(height * 0.5) + w = int(width * 0.15) + left = int(width * 0.04) + for row in range(0, h, 2): + for col in range(0, w, 2): + value = ((row // 2) + (col // 2)) % 2 + context.set_source_rgb(value, value, value) + context.rectangle(left + col, y + row, 2, 2) + context.fill() + for index, value in enumerate((0.50, 0.67, 0.735, 0.80)): + x = int(width * (0.24 + index * 0.19)) + context.set_source_rgb(value, value, value) + context.rectangle(x, y, w, h) + context.fill() + context.set_source_rgb(1, 1, 1) + context.move_to(x, y + h + 40) + context.show_text(f"{value * 100:g}%") + stats["draws"] += 1 + return phase = (time.monotonic() - started) % 2 / 2 context.set_source_rgb(0.06, 0.10, 0.17) context.paint() @@ -57,7 +91,12 @@ def tick(widget, clock): presented = previous.get_presentation_time() if presented: feedback.add(presented) - widget.queue_draw() + surface = window.get_surface() + monitor = display.get_monitor_at_surface(surface) if surface else None + if monitor: + stats["actual_connector"] = monitor.get_connector() + if args.pattern == "moving" or stats["actual_connector"] != args.connector: + widget.queue_draw() return GLib.SOURCE_CONTINUE @@ -113,8 +152,12 @@ def save_widget_pixels(): stats["presentation_feedback_frames"] = len(feedback) stats["complete_timings"] = len(complete_frames) stats["callback_pass"] = stats["ticks"] >= args.seconds * 15 -stats["draw_pass"] = stats["draws"] >= args.seconds * 15 +stats["draw_pass"] = stats["draws"] >= (args.seconds * 15 if args.pattern == "moving" else 1) stats["presentation_pass"] = len(feedback) >= args.seconds * 10 +stats["target_pass"] = stats["actual_connector"] == args.connector +stats["verification_scope"] = "compositor feedback and output placement; physical scanout is not measured" print(json.dumps(stats, sort_keys=True)) -raise SystemExit(0 if all(stats[k] for k in - ("callback_pass", "draw_pass", "presentation_pass")) else 1) +checks = ("callback_pass", "draw_pass", "presentation_pass", "target_pass") if args.pattern == "moving" else ("draw_pass", "target_pass") +if args.widget_pixels: + checks += ("widget_pixels_pass",) +raise SystemExit(0 if all(stats[k] for k in checks) else 1) From db1f25f068cf9772e77d3da0e46564c648662537 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 09:40:54 +1100 Subject: [PATCH 0950/1579] drm/vc4: convert encoded match data through pointer-width integer Clang22 rejects a direct void-pointer-to-enum cast under the existing Fedora WERROR configuration. Preserve numeric generation match data by converting through uintptr_t before the enum; retain all drivers and warnings. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/vc4/vc4_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c index a14ecb7694613..2454ce0b0bc47 100644 --- a/drivers/gpu/drm/vc4/vc4_drv.c +++ b/drivers/gpu/drm/vc4/vc4_drv.c @@ -299,7 +299,7 @@ static int vc4_drm_bind(struct device *dev) dev->coherent_dma_mask = DMA_BIT_MASK(32); - gen = (enum vc4_gen)of_device_get_match_data(dev); + gen = (enum vc4_gen)(uintptr_t)of_device_get_match_data(dev); if (gen > VC4_GEN_4) driver = &vc5_drm_driver; From b1418dba05c50e8ef06c6f8a707927e95ec24182 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 09:45:37 +1100 Subject: [PATCH 0951/1579] drm/pl111: convert encoded match data through pointer-width integer Preserve Versatile and IM-PD1 generation values while avoiding Clang22 direct pointer-to-enum errors under the unchanged Fedora WERROR configuration. Keep all driver selections and strict compiler checks. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/pl111/pl111_versatile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/pl111/pl111_versatile.c b/drivers/gpu/drm/pl111/pl111_versatile.c index 0d8331a3909f4..93ef58ffc91d3 100644 --- a/drivers/gpu/drm/pl111/pl111_versatile.c +++ b/drivers/gpu/drm/pl111/pl111_versatile.c @@ -486,7 +486,7 @@ int pl111_versatile_init(struct drm_device *dev, struct pl111_drm_dev_private *p return 0; } - versatile_clcd_type = (enum versatile_clcd)clcd_id->data; + versatile_clcd_type = (enum versatile_clcd)(uintptr_t)clcd_id->data; /* Versatile Express special handling */ if (versatile_clcd_type == VEXPRESS_CLCD_V2M) { @@ -506,7 +506,7 @@ int pl111_versatile_init(struct drm_device *dev, struct pl111_drm_dev_private *p np = of_find_matching_node_and_match(NULL, impd1_clcd_of_match, &clcd_id); if (np) - versatile_clcd_type = (enum versatile_clcd)clcd_id->data; + versatile_clcd_type = (enum versatile_clcd)(uintptr_t)clcd_id->data; } map = syscon_node_to_regmap(np); From 40bc3b97c1d6a75ff5f8af9bc75322388a5381e1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 12 Oct 2026 16:39:47 -0400 Subject: [PATCH 0952/1579] soc: apple: rtkit-helper: adopt a running T8140 MTP session T8140 MTP firmware faults or times out when it is woken after a boot stage has taken it to QUIESCED or SLEEP, so a boot stage that drives the keyboard has to hand the session over running. When the node carries apple,rtkit-handoff = "mtp-live-v1" and the ASC CPU is still running, adopt the session with apple_rtkit_init_adopted() instead of setting RUN and waking the coprocessor. The boot stage reserves the buffers the firmware holds and maps them 1:1 through the DART (memory-region with iommu-addresses), so they stay valid; the RTKit core acknowledges the inherited system endpoints without a new handshake. An adopted session cannot be restarted, so remove() leaves it running and the transport is released with apple_rtkit_free_retaining_buffers(). Without the property, or with a stopped CPU, the cold-boot path is unchanged. Co-Authored-By: Claude Opus 5.5 --- drivers/soc/apple/rtkit-helper.c | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index 89e5642444367..3dd941241893b 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -12,13 +12,19 @@ #include #include #include +#include #define APPLE_ASC_CPU_CONTROL 0x44 #define APPLE_ASC_CPU_CONTROL_RUN BIT(4) +/* A running T8140 MTP session described by the boot stage that started it. */ +#define APPLE_RTKIT_HELPER_HANDOFF "mtp-live-v1" + struct apple_rtkit_helper { struct device *dev; struct apple_rtkit *rtk; + /* Inherited from the boot stage; it cannot be restarted. */ + bool adopted; void __iomem *asc_base; @@ -73,6 +79,39 @@ static const struct apple_rtkit_ops apple_rtkit_helper_ops = { .shmem_destroy = apple_rtkit_helper_shmem_destroy, }; +static void apple_rtkit_helper_release(void *rtk) +{ + apple_rtkit_free_retaining_buffers(rtk); +} + +/* + * T8140 MTP firmware cannot be restarted once it has been taken to QUIESCED + * or SLEEP, so a boot stage that used it hands the session over running. It + * reserves the buffers the firmware holds and maps them 1:1 through the + * DART. Only adopt a session it described whose CPU still runs. + */ +static bool apple_rtkit_helper_live(struct apple_rtkit_helper *helper) +{ + const char *handoff; + + if (of_property_read_string(helper->dev->of_node, "apple,rtkit-handoff", + &handoff)) + return false; + + if (strcmp(handoff, APPLE_RTKIT_HELPER_HANDOFF)) { + dev_warn(helper->dev, "unknown RTKit handoff \"%s\"\n", handoff); + return false; + } + + if (!(readl_relaxed(helper->asc_base + APPLE_ASC_CPU_CONTROL) & + APPLE_ASC_CPU_CONTROL_RUN)) { + dev_warn(helper->dev, "RTKit handoff without a running CPU\n"); + return false; + } + + return true; +} + static int apple_rtkit_helper_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -103,6 +142,23 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) "Failed to map SRAM region"); } + if (apple_rtkit_helper_live(helper)) { + helper->rtk = apple_rtkit_init_adopted(dev, helper, NULL, 0, + &apple_rtkit_helper_ops); + if (IS_ERR(helper->rtk)) + return dev_err_probe(dev, PTR_ERR(helper->rtk), + "Failed to adopt RTKit"); + + ret = devm_add_action_or_reset(dev, apple_rtkit_helper_release, + helper->rtk); + if (ret) + return ret; + + helper->adopted = true; + dev_info(dev, "adopted the running RTKit session\n"); + return 0; + } + helper->rtk = devm_apple_rtkit_init(dev, helper, NULL, 0, &apple_rtkit_helper_ops); if (IS_ERR(helper->rtk)) @@ -127,6 +183,10 @@ static void apple_rtkit_helper_remove(struct platform_device *pdev) { struct apple_rtkit_helper *helper = platform_get_drvdata(pdev); + /* An inherited session would not come back from QUIESCED. */ + if (helper->adopted) + return; + if (apple_rtkit_is_running(helper->rtk)) apple_rtkit_quiesce(helper->rtk); From 0a0d25f4a99b66a08a2ab4a22314782aaad7b3e2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 12 Oct 2026 19:11:08 -0400 Subject: [PATCH 0953/1579] arm64: dts: apple: t8140-j700: drive the SPI NOR on a single data line With spi-rx-bus-width and spi-tx-bus-width set to 4, the first command spi-nor sends at probe (READ ID) times out on J700 with every byte moved but no completion: "transfer timed out: p=0x100000 c=0x700 ... tx=0 rx=0", so no MTD device and no NVRAM partition appear and asahi-bless cannot read or write the boot volume selection. The same driver probes the flash, creates the nvram partition and lets asahi-bless list volumes when the node describes single data lines, at either 25 or 40 MHz. The ADT's spinor node gives 25 MHz and does not describe quad lanes, and the J700 kernels that probed this flash never declared them. Drop the quad widths and keep the ADT frequency. Tested on J700 (7.1.12, QSPIMC 0x20000): 25 MHz single-line probes and creates "nvram" (0x1d0000 bytes); asahi-bless --list-volumes works. 25 MHz quad fails at the first command. Co-Authored-By: Claude Opus 5.5 --- arch/arm64/boot/dts/apple/t8140-j700.dts | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index beacbf3168957..44143ae11092e 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -462,8 +462,6 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <25000000>; - spi-rx-bus-width = <4>; - spi-tx-bus-width = <4>; partitions { compatible = "fixed-partitions"; From 3dbd78f23e014ee346e286f287d8c7039c4a54c2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 10:24:00 +1100 Subject: [PATCH 0954/1579] drm/apple: accept exact H17P power replies and republish reconnected displays Live R36 evidence records OFF state zero with zero status and a stopped external pipeline. Validate that exact echo while retaining malformed reply, timeout, error, queue drain and ownership guards. Publish discovery only for a cleanly stopped new physical cable epoch, and release a reattached link if unplugged before CRTC activation. Exercise production reply decoding and reconnect paths without the previous test-only OFF policy shim. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 13 ++-- drivers/gpu/drm/apple/iomfb_h17p.c | 8 +-- .../test-apple-external-lifecycle.py | 72 +++++++++++++------ 3 files changed, 62 insertions(+), 31 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d01725a99103f..bcad3cf6f3c86 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2578,12 +2578,12 @@ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) return 0; if (dcp->external_error) return dcp->external_error; - if (dcp->external_stopped) + if (dcp->external_stopped && !READ_ONCE(dcp->dptxport[0].connected)) return 0; iomfb_pause_presentation(dcp, true); dcp->external_running = false; /* Route setup can call this before IOMFB or its shared memory exists. */ - if (!dcp->external_power_attempted) + if (dcp->external_stopped || !dcp->external_power_attempted) goto release_link; ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); if (!ret) @@ -2671,9 +2671,14 @@ bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) if (dcp->iomfb.presentation_paused) { dcp->external_hotplug_pending = true; dcp->external_hotplug_connected = connected; - /* Publish discovery so DRM can activate the first CRTC; keep presents paused. */ + /* Publish initial/new-cable discovery, never same-cable DPMS wake. */ if (connected && READ_ONCE(dcp->external_cable) && - !READ_ONCE(dcp->external_power_attempted) && !READ_ONCE(dcp->external_error)) { + !READ_ONCE(dcp->external_error) && + (!READ_ONCE(dcp->external_power_attempted) || + (READ_ONCE(dcp->external_stopped) && + READ_ONCE(dcp->external_rx_generation) != + READ_ONCE(dcp->external_crtc_generation) && + READ_ONCE(dcp->dptxport[0].connected)))) { unsigned int action = dcp_mode_hotplug(&dcp->mode_state, true, dcp->connector ? &dcp->connector->connected : NULL); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index b3ae896426e25..02bb44d76c29b 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -725,7 +725,7 @@ struct dcp_external_power_cookie { bool on; }; -/* A473's envelope is known; no OFF reply has been qualified as a stop. */ +/* A473 echoes the accepted state with zero status; DMA ownership is unchanged. */ static void iomfb_external_power_complete(struct apple_dcp *dcp, void *out, void *data) { struct dcp_external_power_cookie *cookie = data; @@ -740,9 +740,9 @@ static void iomfb_external_power_complete(struct apple_dcp *dcp, void *out, void WRITE_ONCE(dcp->external_power_reply, reply); WRITE_ONCE(dcp->external_power_reply_valid, true); - dev_info(dcp->dev, "A473 power=%u reply=%#llx%s\n", - cookie->on, reply, cookie->on ? "" : " (stop unqualified)"); - ret = cookie->on ? (reply == 1 ? 0 : -EREMOTEIO) : -EOPNOTSUPP; + dev_info(dcp->dev, "A473 power=%u reply=%#llx\n", + cookie->on, reply); + ret = reply == (u64)cookie->on ? 0 : -EREMOTEIO; } } WRITE_ONCE(cookie->wait.status, ret); diff --git a/tools/diagnostics/test-apple-external-lifecycle.py b/tools/diagnostics/test-apple-external-lifecycle.py index ca24415533bc3..de81ae88691cd 100644 --- a/tools/diagnostics/test-apple-external-lifecycle.py +++ b/tools/diagnostics/test-apple-external-lifecycle.py @@ -46,17 +46,7 @@ def external_branch(name, result): for name in ('iomfb_external_power_complete', 'iomfb_external_reply_size', 'iomfb_external_parameter_complete', 'iomfb_external_device_complete'): parts.append(function(h17p, name)) -parts.append(function(h17p, 'iomfb_external_power_h17p').replace( - 'int iomfb_external_power_h17p(', 'static int test_wire_power(')) -# Model a future qualified OFF policy only in this harness. The production -# decoder is also tested directly and currently MUST reject every OFF reply. -parts.append(r''' -static int iomfb_external_power_h17p(struct apple_dcp *p, bool on) -{ - int ret = test_wire_power(p, on); - return test_qualified_off && !on && ret == -EOPNOTSUPP && !p->external_power_reply ? 0 : ret; -} -''') +parts.append(function(h17p, 'iomfb_external_power_h17p')) for name in ('dcp_is_external_h17p', 'dcp_external_report_disconnected', 'dcp_external_publish_generation', 'dcp_external_link_locked', 'dcp_external_start_locked', 'dcp_external_stop_locked', @@ -183,9 +173,9 @@ def external_branch(name, result): static unsigned long jiffies; static unsigned int sleeps, power0, power1, drops, releases, connects, allocations; static int connect_error, hpd_error, release_error, wire_error; -static bool drain_callback, timeout_reply, absent_reply, test_qualified_off; +static bool drain_callback, timeout_reply, absent_reply; static int prep_error; -static bool on_zero_reply; +static bool on_zero_reply, off_one_reply; static u32 wire_in, wire_out; static char events[256]; static unsigned int event_count; @@ -267,7 +257,7 @@ def external_branch(name, result): if (req->unklong) power1++; else power0++; packet.header.in_len = wire_in; packet.header.out_len = wire_out; - packet.resp.unknown = req->unklong && !on_zero_reply ? 1 : 0; + packet.resp.unknown = ((req->unklong && !on_zero_reply) || (!req->unklong && off_one_reply)) ? 1 : 0; packet.resp.ret = wire_error; if (timeout_reply) { late_cookie = cookie; return; } mutex_lock(&p->iomfb.lock); @@ -328,7 +318,7 @@ def external_branch(name, result): connect_error = hpd_error = release_error = wire_error = 0; drain_callback = timeout_reply = absent_reply = false; wire_in = 16; wire_out = 8; - late_cookie = NULL; test_qualified_off = true; prep_error = 0; on_zero_reply = false; + late_cookie = NULL; prep_error = 0; on_zero_reply = off_one_reply = false; memset(events, 0, sizeof(events)); } static int stop(struct apple_dcp *p, u64 generation) @@ -349,6 +339,34 @@ def external_branch(name, result): assert(stop(&p, 1) == 0 && power0 == 1 && releases == 1); owned(&p); + /* A new physical cable epoch publishes HPD before the CRTC powers on. */ + initial(&p); + assert(dcp_external_disconnect(&p, true) == 0); + assert(!connector.connected && p.external_stopped && power0 == 1); + assert(dcp_dptx_connect_oob(&dev, 0) == 0); + assert(connector.connected && p.iomfb.presentation_paused && !power1); + assert(p.external_power_attempted && p.external_stopped); + dcp_external_recover(&dev); + assert(p.external_running && !p.iomfb.presentation_paused && power1 == 1); + assert(p.external_crtc_generation == p.external_rx_generation); + owned(&p); + + /* Replug/unplug before CRTC activation releases the new link without a second OFF. */ + initial(&p); + assert(dcp_external_disconnect(&p, true) == 0); + assert(dcp_dptx_connect_oob(&dev, 0) == 0 && releases == 1); + assert(dcp_external_disconnect(&p, true) == 0); + assert(releases == 2 && power0 == 1 && !power1); + assert(!p.dptxport[0].connected && !connector.connected); + owned(&p); + + /* Same-cable DPMS OFF must not publish a synthetic wake. */ + initial(&p); assert(stop(&p, 1) == 0); connector.connected = false; + assert(dcp_dptx_connect_oob(&dev, 0) == 0); + assert(!connector.connected && p.iomfb.presentation_paused && !power1); + owned(&p); + + initial(&p); assert(stop(&p, 1) == 0); /* Atomic, backlight and CRC are rejected; control transactions remain admissible. */ iomfb_queue_atomic(&p, NULL, NULL); iomfb_backlight_kick(&p); @@ -394,7 +412,7 @@ def external_branch(name, result): if (fault == 1) p.iomfb.stopped = true; if (fault == 2) wire_out = 12; if (fault == 3) wire_in = 12; - if (fault == 4) { wire_error = 5; test_qualified_off = false; } + if (fault == 4) { wire_error = 5; } if (fault == 5) hpd_error = -EIO; if (fault == 6) release_error = -EIO; assert(stop(&p, 1) < 0 && p.external_error && p.iomfb.presentation_paused); @@ -415,7 +433,7 @@ def external_branch(name, result): assert(!p.external_cable && !connector.connected && p.external_generation == 2); assert(p.metadata_epoch.invalidated && !p.metadata_epoch.connected && connector.edid_invalidation_pending); assert(dcp_dptx_connect_oob(&dev, 0) == 0); - assert(p.external_generation == 3 && !p.external_running && !connector.connected); + assert(p.external_generation == 3 && !p.external_running && connector.connected); assert(p.metadata_epoch.connected && p.metadata_epoch.invalidated && p.metadata_epoch.generation==3); assert(dcp_dptx_connect_oob(&dev,0)==0 && p.typec_generation==3); assert(p.iomfb.presentation_paused && power1 == 0); @@ -440,10 +458,10 @@ def external_branch(name, result): owned(&p); /* Uncertain physical stop reports cable loss but keeps the resource and blocks restart. */ - initial(&p); test_qualified_off = false; - assert(dcp_external_disconnect(&p, true) == -EOPNOTSUPP); + initial(&p); wire_error = 5; + assert(dcp_external_disconnect(&p, true) == -EREMOTEIO); assert(!connector.connected && p.dptxport[0].connected && !releases); - assert(dcp_dptx_connect_oob(&dev, 0) == -EOPNOTSUPP && !power1); + assert(dcp_dptx_connect_oob(&dev, 0) == -EREMOTEIO && !power1); assert(p.external_crtc_generation == 1 && p.iomfb.presentation_paused); owned(&p); /* A deferred old RX disconnect cannot stop a later connection epoch. */ @@ -510,9 +528,17 @@ def external_branch(name, result): assert(modesets == previous_modesets && power1 == 1); owned(&p); - /* Production OFF policy always records but refuses to qualify its 8-byte reply. */ + /* Production accepts only an exact echoed OFF state with zero status. */ + initial(&p); p.iomfb.presentation_paused = true; + assert(iomfb_external_power_h17p(&p, false) == 0); + assert(p.external_power_reply_valid && p.external_power_reply == 0); + owned(&p); + initial(&p); p.iomfb.presentation_paused = true; off_one_reply = true; + assert(iomfb_external_power_h17p(&p, false) == -EREMOTEIO); + assert(p.external_power_reply_valid && p.external_power_reply == 1); + owned(&p); initial(&p); p.iomfb.presentation_paused = true; wire_error = 0x1234; - assert(test_wire_power(&p, false) == -EOPNOTSUPP); + assert(iomfb_external_power_h17p(&p, false) == -EREMOTEIO); assert(p.external_power_reply_valid && p.external_power_reply == (0x1234ULL << 32)); assert(!drops && !releases); owned(&p); @@ -534,7 +560,7 @@ def external_branch(name, result): /* Missing replies and malformed first-start preparation fail closed. */ initial(&p); p.iomfb.presentation_paused = true; absent_reply = true; - assert(test_wire_power(&p, false) == -EPROTO && !p.external_power_reply_valid); + assert(iomfb_external_power_h17p(&p, false) == -EPROTO && !p.external_power_reply_valid); owned(&p); for (int stage = 1; stage <= 2; stage++) { initial(&p); p.external_running = false; p.iomfb.presentation_paused = true; From a4ba95c5083c68b65b9b5b55063a61014203be40 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 12 Oct 2026 19:11:08 -0400 Subject: [PATCH 0955/1579] PCI: apple: piodma-diag: decide suspend-to-idle admission in prepare Commit 2c815e33ece ("PCI: apple: preserve the retained bootstrap across suspend-to-idle sleep") admits suspend-to-idle once the bootstrap has completed, but its PM_SUSPEND_PREPARE notifier tests pm_suspend_no_platform(). enter_state() clears the suspend flags and runs these notifiers in suspend_prepare() before suspend_devices_and_enter() marks the suspend as suspend-to-idle, so the test is false for every suspend and the guard refuses s2idle too: apple-piodma-diag 390030000.piodma: sleep refused: Neo guard active during radio admission or retained DMA lifetime Leave the suspend decision to apple_piodma_pm_prepare(), which runs after the target state is set, makes the same ready/admitted check and still refuses anything but suspend-to-idle. As a supplier of the radio's PCI devices it is prepared before them. Hibernation and restore remain refused in the notifier. Co-Authored-By: Claude Opus 5.5 --- drivers/pci/controller/pcie-apple-piodma-diag.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 7684f6aec5eb6..6f5d9bac0e64e 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -168,10 +168,14 @@ static int apple_piodma_pm_notify(struct notifier_block *nb, switch (action) { case PM_SUSPEND_PREPARE: - if (pm_suspend_no_platform() && READ_ONCE(diag->ready) && - smp_load_acquire(&diag->radio_admitted)) - return NOTIFY_DONE; - fallthrough; + /* + * The suspend core marks suspend-to-idle only after these + * notifiers have run, so pm_suspend_no_platform() is still + * false here for every suspend. apple_piodma_pm_prepare() + * makes the same check once the target state is known, and + * still vetoes everything but suspend-to-idle. + */ + return NOTIFY_DONE; case PM_HIBERNATION_PREPARE: case PM_RESTORE_PREPARE: dev_warn_ratelimited(diag->dev, From 7468369c1aa1754f5a42a0d4dde18838de4a6fbb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 11:59:20 +1100 Subject: [PATCH 0956/1579] drm/apple: tag H17P external SDR RGB sources as sRGB Native-tagged RGB surfaces produce washed-out external output on J700 H17P despite correct KMS pixel values. The same MSI and cable render normally in macOS and M2 Fairydust. A controlled J700 trial changing only the source tag to sRGB restores normal colours. Apply that tag to native RGB surfaces on the external H17P profile. Preserve transfer, pixel format, output modes and other profiles. No diagnostic switch is needed. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p_test.c | 8 ++++++++ drivers/gpu/drm/apple/iomfb_template.c | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c index c017651945a4e..c5955836700ca 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p_test.c +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -45,6 +45,8 @@ static void h17p_surface_wire_test(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, wire->swap, &request->swap, sizeof(wire->swap)); KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x593), DCP_FORMAT_BGRA); + KUNIT_EXPECT_EQ(test, ((u8 *)wire)[0x59b], (u8)DCP_XFER_FUNC_SDR); + KUNIT_EXPECT_EQ(test, ((u8 *)wire)[0x59c], (u8)DCP_COLORSPACE_NATIVE); KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x59d), 10176U); KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5a9), 2408U); KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5ad), 1506U); @@ -58,6 +60,12 @@ static void h17p_surface_wire_test(struct kunit *test) KUNIT_EXPECT_EQ(test, get_unaligned_le64((u8 *)wire + 0xe38), iova); KUNIT_EXPECT_EQ(test, wire->tail[sizeof(wire->tail) - 2], 0U); KUNIT_EXPECT_EQ(test, wire->tail[sizeof(wire->tail) - 1], 0U); + + /* Source-tag changes must reach the wire without changing transfer. */ + surface->colorspace = DCP_COLORSPACE_BG_SRGB; + iomfb_serialize_present_h17p(wire, request); + KUNIT_EXPECT_EQ(test, ((u8 *)wire)[0x59b], (u8)DCP_XFER_FUNC_SDR); + KUNIT_EXPECT_EQ(test, ((u8 *)wire)[0x59c], (u8)DCP_COLORSPACE_BG_SRGB); } static void h17p_opaque_x_parameter_test(struct kunit *test) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 2d41aa4697b41..34557f7ed1f4e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2353,6 +2353,18 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP && req->surf[l].base.colorspace == DCP_COLORSPACE_BG_SRGB) req->surf[l].base.colorspace = DCP_COLORSPACE_NATIVE; + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* H17P external SDR RGB needs an sRGB source tag. Native + * tagging washes out the same pixel values on this profile. + * Keep YUV, other source spaces and other profiles unchanged. + */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + !new_state->fb->format->is_yuv && + req->surf[l].base.colorspace == DCP_COLORSPACE_NATIVE) + req->surf[l].base.colorspace = DCP_COLORSPACE_BG_SRGB; +#endif } if (!has_surface && !crtc_state->color_mgmt_changed) { From bfab2e611aec8a472c49cec96ddb59d081d3df68 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 13:47:16 +1100 Subject: [PATCH 0957/1579] drm/apple: use the observed H17P DPAV service for fresh EDID The external H17P DPAV service is on endpoint 0x2a with the identity dcpav-service-epic / AppleDCPDP2HDMI / unit 1. Select that endpoint only for the external profile and route AFK receives by the instantiated DPAV and DPTX endpoints, preserving legacy 0x28 DPAV and 0x2a DPTX routing and native H17P DPTX at 0x2c. Admit the exact service before the existing AFKV2 OPEN. Keep the bounded read-only group 1 command 7 EDID request and its response validation. Reject in-flight EDID and cached/publication results when the physical Type-C generation changes. Invalidate that generation before waiting for firmware disconnect teardown. Release hpd_mutex during EDID requests and hold it for publication under the existing mode_config.mutex ordering. Keep the conservative firmware metadata epoch guard and service owner/reader pins. Host tests exercise actual production routing, admission, reader ownership and publication with synthetic disconnected/delayed responses. Firmware wire behavior still requires owner qualification. Complete existing kernel-doc parameter descriptions for a clean W=1 compile without warning suppression. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 57 ++---- drivers/gpu/drm/apple/epic/dpavservep.c | 60 +++++- drivers/gpu/drm/apple/epic/dpavservep.h | 3 +- drivers/gpu/drm/apple/iomfb.c | 40 +++- .../test-apple-display-metadata-lifetime.py | 46 ++++- tools/diagnostics/test-apple-h17p-edid.py | 172 ++++++++++++++++++ 7 files changed, 327 insertions(+), 52 deletions(-) create mode 100644 tools/diagnostics/test-apple-h17p-edid.py diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 16e24034aff94..2007129a3ac12 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -94,6 +94,7 @@ enum { DISP0_ENDPOINT = 0x23, DPTX_CONTROLLER_ENDPOINT = 0x24, DPAVSERV_ENDPOINT = 0x28, + DPAVSERV_ENDPOINT_V2 = 0x2a, AV_ENDPOINT = 0x29, DPTX_ENDPOINT = 0x2a, DPTX_ENDPOINT_V2 = 0x2c, diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index bcad3cf6f3c86..ef220f77c6fc8 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1751,6 +1751,10 @@ static void send_vblank_event(struct drm_device *dev, /** * dcp_crtc_send_page_flip_event - helper to send vblank event after pageflip + * @crtc: CRTC whose flip completed + * @e: pending vblank event + * @now: swap completion timestamp + * @start: swap submission timestamp * * Compensate for unknown slack between page flip and arrival of the * swap_complete callback. Minimal observed duration on DCP with HDMI output @@ -2075,6 +2079,11 @@ static void dcp_delayed_vblank(struct work_struct *work) static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) { + /* DPAV 0x2a on H17P overlaps the legacy DPTX endpoint number. */ + if (dcp->dcpavservep && endpoint == dcp->dcpavservep->endpoint) + return dcp->dcpavservep; + if (dcp->dptxep && endpoint == dcp->dptxep->endpoint) + return dcp->dptxep; switch (endpoint) { case AV_ENDPOINT: return dcp->avep; @@ -2082,13 +2091,6 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) return dcp->systemep; case DISP0_ENDPOINT: return dcp->ibootep; - case DPAVSERV_ENDPOINT: - return dcp->dcpavservep; - case DPTX_ENDPOINT: - return dcp->dptxep; - case DPTX_ENDPOINT_V2: - return dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? dcp->dptxep : NULL; case DPTX_CONTROLLER_ENDPOINT: return dcp->dptx_controller_ep; default: @@ -2156,34 +2158,10 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) return; } - switch (endpoint) { - case IOMFB_ENDPOINT: + if (endpoint == IOMFB_ENDPOINT) return iomfb_recv_msg(dcp, message); - case AV_ENDPOINT: - ep = dcp->avep; - break; - case SYSTEM_ENDPOINT: - ep = dcp->systemep; - break; - case DISP0_ENDPOINT: - ep = dcp->ibootep; - break; - case DPAVSERV_ENDPOINT: - ep = dcp->dcpavservep; - break; - case DPTX_ENDPOINT: - ep = dcp->dptxep; - break; - case DPTX_ENDPOINT_V2: - ep = dcp->dptxep; - break; - case DPTX_CONTROLLER_ENDPOINT: - ep = dcp->dptx_controller_ep; - break; - default: - ep = NULL; - break; - } + + ep = dcp_afkep(dcp, endpoint); if (!ep) { dev_warn_ratelimited(dcp->dev, @@ -2627,16 +2605,19 @@ static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_delayed_work_sync(&dcp->placeholder_edid_wq); guard(mutex)(&dcp->external_lock); - ret = dcp_external_stop_locked(dcp, dcp->external_generation); + /* Invalidate physical EDID before waiting for firmware power/link teardown. */ if (physical) { - dcp->external_cable = false; - dcp->external_generation++; - dcp_external_publish_generation(dcp); scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } + } + ret = dcp_external_stop_locked(dcp, dcp->external_generation); + if (physical) { + dcp->external_cable = false; + dcp->external_generation++; + dcp_external_publish_generation(dcp); /* A missing cable is reported even when its hardware remains owned. */ dcp_external_report_disconnected(dcp); } diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 64e836ef48811..9787c48cb5fa3 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -32,6 +32,29 @@ static void dcpavserv_notify_edid(struct apple_epic_service *service) schedule_work(&connector->hotplug_wq); } +/* Admission runs before AFKV2 OPEN and before readers can pin the service. */ +static int dcpavserv_prepare(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + unsigned long flags; + int ret; + + if (!name || strcmp(name, "dcpav-service-epic")) + return -ENODEV; + if (service->ep->inline_v2) { + if (service->ep->endpoint != DPAVSERV_ENDPOINT_V2 || !class || + strcmp(class, "AppleDCPDP2HDMI") || unit != 1) + return -ENODEV; + } else if (unit != 0) { + return -ENODEV; + } + spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + ret = dcp->dcpavserv.enabled ? -EBUSY : 0; + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + return ret; +} + static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -40,7 +63,7 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, trace_dcpavserv_init(dcp, unit); - if (unit == 0 && name && !strcmp(name, "dcpav-service-epic")) { + if (!dcpavserv_prepare(service, name, class, unit)) { spin_lock_irqsave(&dcp->dcpavserv.lock, flags); if (dcp->dcpavserv.enabled) { spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); @@ -95,6 +118,13 @@ void dpavservep_detach(struct apple_dcp *dcp) } } +static int dcpdpserv_prepare(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + /* No production command transport is needed for this H17P service. */ + return service->ep->inline_v2 ? -ENODEV : 0; +} + static void dcpdpserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -272,12 +302,25 @@ static const struct drm_edid *dcpavserv_read_edid_inline(struct apple_epic_servi return drm_edid_alloc(resp->data + EDID_LEADING_DATA_SIZE, edid_size); } -const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) +bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation) +{ + lockdep_assert_held(&dcp->hpd_mutex); + return dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL || + (dcp->typec_cable_connected && generation == dcp->typec_generation && + !READ_ONCE(dcp->quiescing)); +} + +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation) { struct apple_epic_service *service; const struct drm_edid *edid; unsigned long flags; + scoped_guard(mutex, &dcp->hpd_mutex) { + if (!dcpavserv_edid_current(dcp, generation)) + return ERR_PTR(-ESTALE); + } spin_lock_irqsave(&dcp->dcpavserv.lock, flags); service = dcp->dcpavserv.enabled ? afk_service_get(dcp->dcpavserv.service) : NULL; spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); @@ -294,6 +337,13 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) } spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); afk_service_put(service); + scoped_guard(mutex, &dcp->hpd_mutex) { + if (!dcpavserv_edid_current(dcp, generation)) { + if (!IS_ERR(edid)) + drm_edid_free(edid); + edid = ERR_PTR(-ESTALE); + } + } return edid; } @@ -301,6 +351,7 @@ static const struct apple_epic_service_ops dpavservep_ops[] = { { .name = "dcpav-service-epic", .reusable = true, + .prepare = dcpavserv_prepare, .init = dcpavserv_init, .teardown = dcpavserv_teardown, .report = dcpavserv_report, @@ -308,6 +359,7 @@ static const struct apple_epic_service_ops dpavservep_ops[] = { { .name = "dcpdp-service-epic", .reusable = true, + .prepare = dcpdpserv_prepare, .init = dcpdpserv_init, .teardown = dcpdpserv_teardown, .report = dcpdpserv_report, @@ -322,7 +374,9 @@ int dpavservep_init(struct apple_dcp *dcp) init_completion(&dcp->dcpavserv.enable_completion); - ep = afk_init(dcp, DPAVSERV_ENDPOINT, dpavservep_ops); + ep = afk_init(dcp, dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? + DPAVSERV_ENDPOINT_V2 : DPAVSERV_ENDPOINT, dpavservep_ops); if (IS_ERR(ep)) return PTR_ERR(ep); /* Only the external H17P DPAV endpoint uses inline AFKV2 service calls. */ diff --git a/drivers/gpu/drm/apple/epic/dpavservep.h b/drivers/gpu/drm/apple/epic/dpavservep.h index e9ab17a7302c1..7e2972984c101 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.h +++ b/drivers/gpu/drm/apple/epic/dpavservep.h @@ -20,7 +20,8 @@ struct dcpavserv { struct apple_epic_service *service; }; -const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp); +bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation); +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation); void dpavservep_detach(struct apple_dcp *dcp); #endif /* _DRM_APPLE_EPIC_DPAVSERV_H */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index f68a20e0675b9..642cac5e070d5 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1054,6 +1054,8 @@ int dcp_get_modes(struct drm_connector *connector) struct drm_display_mode *mode; u16 min_vfreq = 0, max_vfreq = 0; bool vrr_capable = false; + bool physical_edid; + u64 edid_generation = 0; int i; dcp_consume_edid_invalidation(apple_connector); @@ -1090,12 +1092,40 @@ int dcp_get_modes(struct drm_connector *connector) } drm_connector_set_vrr_capable_property(connector, vrr_capable); + physical_edid = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + /* get_modes holds mode_config.mutex; HPD paths never acquire it. */ + if (physical_edid) { + mutex_lock(&dcp->hpd_mutex); + edid_generation = dcp->typec_generation; + if (apple_connector->drm_edid && + apple_connector->drm_edid_generation != edid_generation) + dcp_clear_connector_edid(apple_connector); + if (!dcpavserv_edid_current(dcp, edid_generation)) { + dcp_clear_connector_edid(apple_connector); + goto edid_done; + } + } + /* Internal H17P still has no supported EDID command transport. */ if (dcp->nr_modes && !apple_connector->drm_edid && (dcp->fw_compat != DCP_FIRMWARE_H17P || (dcp->dcpavservep && dcp->dcpavservep->inline_v2))) { const struct drm_edid *edid; - edid = dcpavserv_copy_edid(dcp); + /* Let physical disconnect advance the generation during the request. */ + if (physical_edid) + mutex_unlock(&dcp->hpd_mutex); + edid = dcpavserv_copy_edid(dcp, edid_generation); + if (physical_edid) { + mutex_lock(&dcp->hpd_mutex); + if (!dcpavserv_edid_current(dcp, edid_generation) || + READ_ONCE(apple_connector->dcp) != pdev) { + if (!IS_ERR_OR_NULL(edid)) + drm_edid_free(edid); + dcp_clear_connector_edid(apple_connector); + goto edid_done; + } + } if (IS_ERR_OR_NULL(edid)) { /* * An internal panel has no AV service and so no EDID; @@ -1108,12 +1138,12 @@ int dcp_get_modes(struct drm_connector *connector) } else { drm_edid_free(apple_connector->drm_edid); apple_connector->drm_edid = edid; + apple_connector->drm_edid_generation = edid_generation; } } if (dcp->nr_modes && apple_connector->drm_edid) { apple_connector->drm_edid_firmware_derived = false; drm_edid_connector_update(connector, apple_connector->drm_edid); - dcp_retry_placeholder_edid(dcp, apple_connector->drm_edid); } else { u64 generation = 0; const struct drm_edid *derived = dcp->nr_modes ? @@ -1141,6 +1171,12 @@ int dcp_get_modes(struct drm_connector *connector) } } +edid_done: + if (physical_edid) + mutex_unlock(&dcp->hpd_mutex); + if (apple_connector->drm_edid) + dcp_retry_placeholder_edid(dcp, apple_connector->drm_edid); + /* * An internal panel has no EDID, so nothing fills in the refresh range * that userspace needs before it will drive VRR. Supply the range DCP diff --git a/tools/diagnostics/test-apple-display-metadata-lifetime.py b/tools/diagnostics/test-apple-display-metadata-lifetime.py index baf70b6cb4cae..3b9ed5738caa0 100644 --- a/tools/diagnostics/test-apple-display-metadata-lifetime.py +++ b/tools/diagnostics/test-apple-display-metadata-lifetime.py @@ -30,6 +30,7 @@ def function(text, signature): epoch = (base / 'dcp-metadata-epoch.h').read_text().replace('#include ', '') cache = dcp[dcp.index('static bool dcp_firmware_metadata_enabled('):dcp.index('void dcp_set_dimensions(')] getter = function(iomfb, 'int dcp_get_modes(') +physical_current = function((base / 'epic/dpavservep.c').read_text(), 'bool dcpavserv_edid_current(') cleanup = function(iomfb, 'static void dcp_clear_connector_edid(') + '\n' + function(iomfb, 'static void dcp_consume_edid_invalidation(') hotplug = function(iomfb, 'void dcp_hotplug(') disconnect = function(dcp, 'static void disconnected_hpd_event(') @@ -67,7 +68,7 @@ def function(text, signature): #define READ_ONCE(x) (x) #define WRITE_ONCE(x,v) ((x)=(v)) #define xchg(p,v) __atomic_exchange_n(p,v,__ATOMIC_SEQ_CST) -#define lockdep_assert_held(p) assert(held_mutex==(p)) +#define lockdep_assert_held(p) assert(mutex_held(p)) #define ERR_PTR(n) ((void *)(intptr_t)(n)) #define PTR_ERR(p) ((int)(intptr_t)(p)) #define IS_ERR(p) ((uintptr_t)(p) > (uintptr_t)-4096) @@ -94,8 +95,11 @@ def function(text, signature): struct mutex { pthread_mutex_t native; }; static _Thread_local struct mutex *held_mutex; static _Thread_local bool modeset_held; -static void mutex_lock(struct mutex *m) { assert(!modeset_held); assert(held_mutex != m); assert(!pthread_mutex_lock(&m->native)); held_mutex=m; } -static void mutex_unlock(struct mutex *m) { assert(held_mutex==m); held_mutex=NULL; assert(!pthread_mutex_unlock(&m->native)); } +static _Thread_local struct mutex *held_stack[4]; +static _Thread_local unsigned int held_depth; +static bool mutex_held(struct mutex *m) { for(unsigned int i=0;inative)); held_stack[held_depth++]=m; held_mutex=m; } +static void mutex_unlock(struct mutex *m) { assert(held_mutex==m); assert(held_depth); held_depth--; held_mutex=held_depth ? held_stack[held_depth-1] : NULL; assert(!pthread_mutex_unlock(&m->native)); } static struct mutex *acquire(struct mutex *m) { mutex_lock(m); return m; } static void release(struct mutex **m) { if (*m) mutex_unlock(*m); } static struct mutex *once(struct mutex *m) { mutex_unlock(m); return NULL; } @@ -123,6 +127,9 @@ def function(text, signature): state = r''' struct apple_dcp { void *dev; + struct mutex hpd_mutex; + bool typec_cable_connected, quiescing; + u64 typec_generation; struct { int iomfb_method_profile; } hw; struct apple_connector *connector; struct { void *data; size_t length, received; bool contiguous, payload_complete; struct dcp_metadata_ticket metadata_ticket; } chunks; @@ -145,6 +152,7 @@ def function(text, signature): static bool invalidate_during_update; static bool update_error; static bool raw_available; +static bool detach_during_read, replug_during_read; static const struct drm_edid *allocate(unsigned int identity) { struct drm_edid *edid = malloc(sizeof(*edid)); assert(edid); @@ -163,11 +171,20 @@ def function(text, signature): static void drm_connector_set_vrr_capable_property(struct drm_connector *c, bool v) { (void)c; (void)v; } static bool dcp_has_panel(struct apple_dcp *p) { (void)p; return false; } -static const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *p) -{ (void)p; raw_reads++; return raw_available ? allocate(0xfeed) : ERR_PTR(-ENODEV); } +static const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *p, u64 generation) +{ + assert(!mutex_held(&p->hpd_mutex)); + (void)generation; raw_reads++; + if (detach_during_read) { + mutex_lock(&p->hpd_mutex); p->typec_generation++; + p->typec_cable_connected=replug_during_read; + mutex_unlock(&p->hpd_mutex); + } + return raw_available ? allocate(0xfeed) : ERR_PTR(-ENODEV); +} static int drm_edid_connector_update(struct drm_connector *c, const struct drm_edid *e) { - assert(held_mutex==&c->dev->mode_config.mutex); + assert(mutex_held(&c->dev->mode_config.mutex)); if (e && update_error) return -ENOMEM; struct test_blob *old=c->edid_blob_ptr; struct test_blob *next=NULL; @@ -227,6 +244,9 @@ def function(text, signature): free(p.chunks.data); assert(!allocations); memset(&p,0,sizeof(p)); memset(&c,0,sizeof(c)); + assert(!pthread_mutex_init(&p.hpd_mutex.native,NULL)); + p.typec_cable_connected=true; p.typec_generation=1; + detach_during_read=replug_during_read=false; p.hw.iomfb_method_profile=DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; p.fw_compat=DCP_FIRMWARE_H17P; static typeof(*p.dcpavservep) av={true}; p.dcpavservep=&av; @@ -389,14 +409,24 @@ def function(text, signature): assert(!pthread_create(&b,NULL,hotplug_thread,NULL)); assert(!pthread_create(&reader,NULL,reader_thread,NULL)); assert(!pthread_join(a,NULL) && !pthread_join(b,NULL) && !pthread_join(reader,NULL)); + initial(); raw_available=true; detach_during_read=true; + assert(probe(&c.base)==1 && !c.drm_edid && !property_identity && !allocations); + assert(raw_reads==1 && !p.typec_cable_connected); + assert(probe(&c.base)==1 && raw_reads==1); /* No request while disconnected. */ + initial(); raw_available=true; detach_during_read=replug_during_read=true; + assert(probe(&c.base)==1 && !c.drm_edid && !property_identity && !allocations); + detach_during_read=false; + assert(probe(&c.base)==1 && c.drm_edid && c.drm_edid_generation==2); + p.typec_generation++; /* Missed worker: generation alone rejects the old cache. */ + assert(probe(&c.base)==1 && raw_reads==3 && c.drm_edid_generation==3); initial(); assert(!blob_allocations && trace_observations); - puts("PASS: mutex-serialized EDID/blob get_modes, hotplug, sysfs reader and same-epoch publication, first-connection scope, raw precedence, invalidation, no added modes"); + puts("PASS: mutex-serialized EDID/blob get_modes, hotplug, sysfs reader and same-epoch publication, first-connection scope, raw precedence, delayed disconnect/replug rejection, generation cache invalidation, no added modes"); return 0; } ''' with tempfile.TemporaryDirectory(prefix='dcp-metadata-lifetime-') as tmp: tmp = Path(tmp) - (tmp / 'test.c').write_text(prefix + metadata + epoch + state + cache + chunk_callbacks + cleanup + getter + hotplug + disconnect + suffix) + (tmp / 'test.c').write_text(prefix + metadata + epoch + state + cache + chunk_callbacks + cleanup + physical_current + getter + hotplug + disconnect + suffix) command = shlex.split(os.environ.get('CC','clang')) command += ['-std=gnu11','-Wall','-Wextra','-Werror','-Wno-sign-compare','-O1','-g', '-fsanitize=address,undefined','-fno-omit-frame-pointer','-pthread'] diff --git a/tools/diagnostics/test-apple-h17p-edid.py b/tools/diagnostics/test-apple-h17p-edid.py new file mode 100644 index 0000000000000..1ac755f6f69e9 --- /dev/null +++ b/tools/diagnostics/test-apple-h17p-edid.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise production DPAV routing, admission and in-flight EDID ownership.""" +from pathlib import Path +import os +import shlex +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +base = root / 'drivers/gpu/drm/apple' + +def function(file, signature): + source = (base / file).read_text() + start = source.index(signature) + opening = source.index('{', start) + while ';' in source[start:opening]: + start = source.index(signature, start + 1) + opening = source.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + +av = (base / 'epic/dpavservep.c').read_text() +af = (base / 'afk.c').read_text() +announce = af[af.index('static void afk_recv_handle_init('):af.index('static void afk_recv_handle_init(') + 6500] +assert announce.index('ops->prepare(service,') < announce.index('EPIC_CAT_REPORT, AFKV2_OPEN') +assert '.prepare = dcpavserv_prepare,' in av and '.prepare = dcpdpserv_prepare,' in av +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint64_t u64; +typedef int64_t s64; +#define READ_ONCE(x) (x) +#define ERR_PTR(x) ((void *)(intptr_t)(x)) +#define PTR_ERR(x) ((int)(intptr_t)(x)) +#define IS_ERR(x) ((uintptr_t)(x) > (uintptr_t)-4096) +#define spin_lock_irqsave(p,f) do { assert(!*(p)); *(p)=true; (f)=0; } while(0) +#define spin_unlock_irqrestore(p,f) do { (void)(f); assert(*(p)); *(p)=false; } while(0) +#define lockdep_assert_held(p) assert(*(p)) +static bool *acquire(bool *p) { assert(!*p); *p=true; return p; } +static void release(bool **p) { if(*p) { assert(**p); **p=false; } } +static bool *once(bool *p) { assert(*p); *p=false; return NULL; } +#define scoped_guard(k,p) for(bool *_lock __attribute__((cleanup(release)))=acquire(p); _lock; _lock=once(_lock)) +#define dev_dbg_ratelimited(...) ((void)0) +#define dev_warn_ratelimited(...) ((void)0) +#define trace_dcp_recv_msg(...) ((void)0) +#define DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL 2 +#define DCP_FIRMWARE_H17P 2 +#define AV_ENDPOINT 0x29 +#define SYSTEM_ENDPOINT 0x20 +#define DISP0_ENDPOINT 0x23 +#define DPAVSERV_ENDPOINT 0x28 +#define DPAVSERV_ENDPOINT_V2 0x2a +#define DPTX_ENDPOINT 0x2a +#define DPTX_ENDPOINT_V2 0x2c +#define DPTX_CONTROLLER_ENDPOINT 0x24 +#define IOMFB_ENDPOINT 0x37 +struct apple_dcp; +struct apple_dcp_afkep { struct apple_dcp *dcp; u8 endpoint; bool inline_v2; }; +struct apple_epic_service { struct apple_dcp_afkep *ep; unsigned int users; bool enabled; }; +struct apple_dcp { + struct apple_dcp_afkep *dcpavservep, *dptxep, *avep, *systemep, *ibootep, *dptx_controller_ep; + struct { bool lock, enabled; struct apple_epic_service *service; } dcpavserv; + struct { int iomfb_method_profile; } hw; + bool hpd_mutex, typec_cable_connected, quiescing; + u64 typec_generation; + int fw_compat; + void *dev; +}; +struct drm_edid { int identity; }; +static unsigned int reads, allocations, legacy_reads, receives; +static unsigned int event; +static struct apple_dcp_afkep *received_ep; +static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) +{ if(!s || !s->enabled) return NULL; s->users++; return s; } +static void afk_service_put(struct apple_epic_service *s) +{ assert(s->users); s->users--; } +static void drm_edid_free(const struct drm_edid *e) +{ if(e) { assert(allocations); allocations--; free((void *)e); } } +static const struct drm_edid *dcpavserv_read_edid_inline(struct apple_epic_service *s) +{ + struct apple_dcp *d=s->ep->dcp; + assert(!d->hpd_mutex && !d->dcpavserv.lock && s->users==2); + reads++; + if(event==1 || event==2) { d->typec_generation++; d->typec_cable_connected=(event==2); } + if(event==3) { d->dcpavserv.service=NULL; d->dcpavserv.enabled=false; s->enabled=false; afk_service_put(s); } + struct drm_edid *e=malloc(sizeof(*e)); assert(e); e->identity=42; allocations++; return e; +} +static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *s) +{ legacy_reads++; return dcpavserv_read_edid_inline(s); } +static void iomfb_recv_msg(struct apple_dcp *d, u64 message) { (void)d; (void)message; } +static void afk_receive_message(struct apple_dcp_afkep *ep, u64 message) +{ (void)message; received_ep=ep; receives++; } +''' +parts = [function('dcp.c', 'static struct apple_dcp_afkep *dcp_afkep('), + function('dcp.c', 'static void dcp_recv_msg('), + function('epic/dpavservep.c', 'static int dcpavserv_prepare('), + function('epic/dpavservep.c', 'static int dcpdpserv_prepare('), + function('epic/dpavservep.c', 'bool dcpavserv_edid_current('), + function('epic/dpavservep.c', 'const struct drm_edid *dcpavserv_copy_edid(')] +suffix = r''' +int main(void) +{ + struct apple_dcp d={.fw_compat=DCP_FIRMWARE_H17P}; + struct apple_dcp_afkep av={.dcp=&d,.endpoint=0x28}, tx={.dcp=&d,.endpoint=0x2a}; + d.dcpavservep=&av; d.dptxep=&tx; + assert(dcp_afkep(&d,0x28)==&av && dcp_afkep(&d,0x2a)==&tx && !dcp_afkep(&d,0x2c)); + dcp_recv_msg(&d,0x28,1); assert(received_ep==&av && receives==1); + dcp_recv_msg(&d,0x2a,2); assert(received_ep==&tx && receives==2); + d.hw.iomfb_method_profile=DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + av.endpoint=0x2a; av.inline_v2=true; tx.endpoint=0x2c; + assert(!dcp_afkep(&d,0x28) && dcp_afkep(&d,0x2a)==&av && dcp_afkep(&d,0x2c)==&tx); + dcp_recv_msg(&d,0x28,3); assert(receives==2); + dcp_recv_msg(&d,0x2a,4); assert(received_ep==&av && receives==3); + dcp_recv_msg(&d,0x2c,5); assert(received_ep==&tx && receives==4); + d.dcpavservep=NULL; dcp_recv_msg(&d,0x2a,6); assert(receives==4); d.dcpavservep=&av; + struct apple_epic_service s={.ep=&av,.users=1,.enabled=true}; + const char *name="dcpav-service-epic", *class="AppleDCPDP2HDMI"; + assert(!dcpavserv_prepare(&s,name,class,1)); + assert(dcpavserv_prepare(&s,name,class,0)==-ENODEV); + assert(dcpavserv_prepare(&s,name,class,3)==-ENODEV); + assert(dcpavserv_prepare(&s,name,class,-1)==-ENODEV); + assert(dcpavserv_prepare(&s,"dcpdp-service-epic",class,1)==-ENODEV); + assert(dcpavserv_prepare(&s,name,"wrong",1)==-ENODEV); + assert(dcpavserv_prepare(&s,NULL,class,1)==-ENODEV); + assert(dcpavserv_prepare(&s,name,NULL,1)==-ENODEV); + av.endpoint=0x28; assert(dcpavserv_prepare(&s,name,class,1)==-ENODEV); av.endpoint=0x2a; + assert(dcpdpserv_prepare(&s,"dcpdp-service-epic",class,3)==-ENODEV); + d.dcpavserv.enabled=true; assert(dcpavserv_prepare(&s,name,class,1)==-EBUSY); + d.dcpavserv.service=&s; d.typec_generation=1; d.typec_cable_connected=true; + const struct drm_edid *e=dcpavserv_copy_edid(&d,1); + assert(!IS_ERR(e) && s.users==1 && reads==1); drm_edid_free(e); + d.typec_cable_connected=false; + assert(PTR_ERR(dcpavserv_copy_edid(&d,1))==-ESTALE && reads==1); + d.typec_cable_connected=true; + assert(PTR_ERR(dcpavserv_copy_edid(&d,0))==-ESTALE && reads==1); + event=1; assert(PTR_ERR(dcpavserv_copy_edid(&d,1))==-ESTALE && !allocations && s.users==1); + event=2; d.typec_cable_connected=true; + assert(PTR_ERR(dcpavserv_copy_edid(&d,2))==-ESTALE && !allocations && s.users==1); + event=0; e=dcpavserv_copy_edid(&d,3); assert(!IS_ERR(e)); drm_edid_free(e); + event=3; assert(PTR_ERR(dcpavserv_copy_edid(&d,3))==-ENODEV && !allocations && !s.users); + assert(PTR_ERR(dcpavserv_copy_edid(&d,3))==-ENODEV); + av.inline_v2=false; av.endpoint=0x28; d.hw.iomfb_method_profile=0; + d.dcpavserv.enabled=false; assert(!dcpavserv_prepare(&s,name,NULL,0)); + assert(dcpavserv_prepare(&s,name,class,1)==-ENODEV); + assert(!dcpdpserv_prepare(&s,NULL,NULL,3)); + d.dcpavserv.service=&s; d.dcpavserv.enabled=true; s.enabled=true; s.users=1; event=0; + d.typec_cable_connected=false; e=dcpavserv_copy_edid(&d,0); + assert(!IS_ERR(e) && legacy_reads==1 && s.users==1); drm_edid_free(e); + assert(!allocations && !d.hpd_mutex && !d.dcpavserv.lock); + puts("PASS: production old/new lookup and receive, exact service admission before OPEN, delayed disconnect/replug, service detach pins, legacy transport"); +} +''' +with tempfile.TemporaryDirectory(prefix='apple-h17p-edid-') as tmp: + source=Path(tmp)/'test.c' + source.write_text(prefix+'\n'.join(parts)+suffix) + cc=shlex.split(os.environ.get('HOSTCC','clang')) + subprocess.run(cc+['-std=gnu11','-Wall','-Werror', + '-O1','-g','-fsanitize=address,undefined','-fno-omit-frame-pointer', + str(source),'-o',str(Path(tmp)/'test')],check=True) + subprocess.run([str(Path(tmp)/'test')],check=True) From 57f9f3623299185d7b834ecb892e96ed0a7ea819 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 13:55:51 +1100 Subject: [PATCH 0958/1579] drm/apple: fence EDID publication by service lifetime and unblock RX A service CLOSE/replacement can leave a raw EDID cache valid even when the physical Type-C generation does not change. Replacement after the reader's last pointer check can also publish data from the retired service. Give DPAV an epoch that advances on service publication and retirement. Carry it from the pinned reader into the raw connector cache and validate it again while serializing publication against teardown. Use a process context mutex for the DPAV owner state, nesting below the DRM and HPD mutexes and releasing it during firmware waits. Defer CLOSE cache/property cleanup through the existing hotplug worker. Preserve the physical connection checks, owner/reader pins and firmware metadata epoch guard. DisplayAttributes is handled synchronously on RTKit's ordered RX worker. Taking mode_config.mutex there can block dispatch of the AFK reply that a mode probe is waiting for while it holds that mutex. Defer connector physical-dimension updates to the existing hotplug worker instead. Add a threaded host regression through the actual RTKit, IOMFB receive, dimensions and AFK enqueue functions. It fails on the parent with RX blocked before the AFK reply, and passes with the deferred update. Extend service and publication tests for same-slot CLOSE/re-OPEN, unchanged physical generations and replacement after the reader's last check. No command ABI or activation-readiness assumption is changed. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/connector.h | 1 + drivers/gpu/drm/apple/dcp.c | 33 ++- drivers/gpu/drm/apple/dcp.h | 2 + drivers/gpu/drm/apple/epic/dpavservep.c | 66 ++++-- drivers/gpu/drm/apple/epic/dpavservep.h | 12 +- drivers/gpu/drm/apple/iomfb.c | 32 ++- .../test-apple-display-metadata-lifetime.py | 39 +++- .../test-apple-edid-rx-progress.py | 203 ++++++++++++++++++ tools/diagnostics/test-apple-h17p-edid.py | 65 ++++-- 9 files changed, 394 insertions(+), 59 deletions(-) create mode 100644 tools/diagnostics/test-apple-edid-rx-progress.py diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index 6f73e975ebdc1..e04bc2e119a5e 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -46,6 +46,7 @@ struct apple_connector { bool drm_edid_firmware_derived; bool edid_invalidation_pending; u64 drm_edid_generation; + u64 drm_edid_service_epoch; /* Workqueue for sending hotplug events to the associated device */ struct work_struct hotplug_wq; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index ef220f77c6fc8..20115ce416842 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2021,15 +2021,13 @@ void dcp_set_dimensions(struct apple_dcp *dcp) height_mm = dcp->panel.height_mm; } - /* Set the connector info */ - if (dcp->connector) { - struct drm_connector *connector = &dcp->connector->base; - - mutex_lock(&connector->dev->mode_config.mutex); - connector->display_info.width_mm = width_mm; - connector->display_info.height_mm = height_mm; - mutex_unlock(&connector->dev->mode_config.mutex); - } + /* + * This callback runs on the ordered RTKit RX worker. A mode probe + * can hold mode_config.mutex while waiting for an AFK reply which + * that same worker still has to dispatch. Defer the DRM update. + */ + if (dcp->connector && !READ_ONCE(dcp->quiescing)) + schedule_work(&dcp->connector->hotplug_wq); /* * Fix up any probed modes. Modes are created when parsing @@ -2042,6 +2040,21 @@ void dcp_set_dimensions(struct apple_dcp *dcp) } } +void dcp_update_connector_dimensions(struct apple_dcp *dcp, + struct drm_connector *connector) +{ + int width_mm = READ_ONCE(dcp->width_mm); + int height_mm = READ_ONCE(dcp->height_mm); + + lockdep_assert_held(&connector->dev->mode_config.mutex); + if (!width_mm || !height_mm) { + width_mm = dcp->panel.width_mm; + height_mm = dcp->panel.height_mm; + } + connector->display_info.width_mm = width_mm; + connector->display_info.height_mm = height_mm; +} + bool dcp_has_panel(struct apple_dcp *dcp) { return dcp->panel.width_mm > 0; @@ -4218,7 +4231,7 @@ static int dcp_platform_probe(struct platform_device *pdev) mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->metadata.lock); - spin_lock_init(&dcp->dcpavserv.lock); + mutex_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index c2b02a5309eab..411240d3e5f42 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -78,6 +78,8 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, const struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode); void dcp_set_dimensions(struct apple_dcp *dcp); +void dcp_update_connector_dimensions(struct apple_dcp *dcp, + struct drm_connector *connector); void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 9787c48cb5fa3..a43e94f7b978a 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -37,7 +37,6 @@ static int dcpavserv_prepare(struct apple_epic_service *service, const char *nam const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; - unsigned long flags; int ret; if (!name || strcmp(name, "dcpav-service-epic")) @@ -49,9 +48,9 @@ static int dcpavserv_prepare(struct apple_epic_service *service, const char *nam } else if (unit != 0) { return -ENODEV; } - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + mutex_lock(&dcp->dcpavserv.lock); ret = dcp->dcpavserv.enabled ? -EBUSY : 0; - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); return ret; } @@ -59,23 +58,23 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; - unsigned long flags; trace_dcpavserv_init(dcp, unit); if (!dcpavserv_prepare(service, name, class, unit)) { - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + mutex_lock(&dcp->dcpavserv.lock); if (dcp->dcpavserv.enabled) { - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); dev_err(dcp->dev, "DCPAVSERV: unit %lld already exists\n", unit); return; } dcp->dcpavserv.service = afk_service_get(service); dcp->dcpavserv.enabled = true; + dcp->dcpavserv.epoch++; service->cookie = &dcp->dcpavserv; complete(&dcp->dcpavserv.enable_completion); - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); if (service->ep->inline_v2) { dev_info(dcp->dev, "DCPAVSERV: inline EDID service ready on channel %u\n", service->channel); @@ -84,20 +83,35 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, } } +/* Defer DRM cleanup: AFK callbacks must not take mode_config.mutex. */ +static void dcpavserv_invalidate_edid(struct apple_dcp *dcp) +{ + struct apple_connector *connector = READ_ONCE(dcp->connector); + + lockdep_assert_held(&dcp->dcpavserv.lock); + if (!connector || READ_ONCE(connector->dcp) != to_platform_device(dcp->dev)) + return; + WRITE_ONCE(connector->edid_invalidation_pending, true); + if (!READ_ONCE(dcp->quiescing) && + READ_ONCE(connector->base.registration_state) == DRM_CONNECTOR_REGISTERED) + schedule_work(&connector->hotplug_wq); +} + static void dcpavserv_teardown(struct apple_epic_service *service) { struct apple_dcp *dcp = service->ep->dcp; - unsigned long flags; bool owned = false; - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + mutex_lock(&dcp->dcpavserv.lock); if (dcp->dcpavserv.service == service) { dcp->dcpavserv.enabled = false; dcp->dcpavserv.service = NULL; + dcp->dcpavserv.epoch++; + dcpavserv_invalidate_edid(dcp); reinit_completion(&dcp->dcpavserv.enable_completion); owned = true; } - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); if (owned) afk_service_put(service); } @@ -105,13 +119,16 @@ static void dcpavserv_teardown(struct apple_epic_service *service) void dpavservep_detach(struct apple_dcp *dcp) { struct apple_epic_service *service; - unsigned long flags; - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + mutex_lock(&dcp->dcpavserv.lock); service = dcp->dcpavserv.service; dcp->dcpavserv.service = NULL; dcp->dcpavserv.enabled = false; - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + if (service) { + dcp->dcpavserv.epoch++; + dcpavserv_invalidate_edid(dcp); + } + mutex_unlock(&dcp->dcpavserv.lock); if (service) { afk_service_disable(service); afk_service_put(service); @@ -311,31 +328,40 @@ bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation) !READ_ONCE(dcp->quiescing)); } -const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation) +bool dcpavserv_edid_service_current(struct apple_dcp *dcp, u64 epoch) +{ + lockdep_assert_held(&dcp->dcpavserv.lock); + return dcp->dcpavserv.enabled && dcp->dcpavserv.service && + epoch == dcp->dcpavserv.epoch; +} + +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation, + u64 *epoch) { struct apple_epic_service *service; const struct drm_edid *edid; - unsigned long flags; scoped_guard(mutex, &dcp->hpd_mutex) { if (!dcpavserv_edid_current(dcp, generation)) return ERR_PTR(-ESTALE); } - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + mutex_lock(&dcp->dcpavserv.lock); service = dcp->dcpavserv.enabled ? afk_service_get(dcp->dcpavserv.service) : NULL; - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + *epoch = dcp->dcpavserv.epoch; + mutex_unlock(&dcp->dcpavserv.lock); if (!service) return ERR_PTR(-ENODEV); edid = service->ep->inline_v2 ? dcpavserv_read_edid_inline(service) : dcpavserv_read_edid_legacy(service); - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); - if (dcp->dcpavserv.service != service) { + mutex_lock(&dcp->dcpavserv.lock); + if (dcp->dcpavserv.service != service || + !dcpavserv_edid_service_current(dcp, *epoch)) { if (!IS_ERR(edid)) drm_edid_free(edid); edid = ERR_PTR(-ENODEV); } - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); afk_service_put(service); scoped_guard(mutex, &dcp->hpd_mutex) { if (!dcpavserv_edid_current(dcp, generation)) { diff --git a/drivers/gpu/drm/apple/epic/dpavservep.h b/drivers/gpu/drm/apple/epic/dpavservep.h index 7e2972984c101..b824a59918ea3 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.h +++ b/drivers/gpu/drm/apple/epic/dpavservep.h @@ -5,7 +5,7 @@ #define _DRM_APPLE_EPIC_DPAVSERV_H #include -#include +#include #include struct drm_edid; @@ -13,7 +13,11 @@ struct apple_epic_service; struct apple_dcp; struct dcpavserv { - spinlock_t lock; /* Protects the service pointer and its owner pin. */ + /* Process context only: serializes retirement and EDID publication. + * Nest below mode_config.mutex and hpd_mutex; never wait for firmware. + */ + struct mutex lock; + u64 epoch; /* Changes even when the same service slot is reused. */ bool enabled; struct completion enable_completion; u32 unit; @@ -21,7 +25,9 @@ struct dcpavserv { }; bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation); -const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation); +bool dcpavserv_edid_service_current(struct apple_dcp *dcp, u64 epoch); +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation, + u64 *epoch); void dpavservep_detach(struct apple_dcp *dcp); #endif /* _DRM_APPLE_EPIC_DPAVSERV_H */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 642cac5e070d5..21a70f893f2e3 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -821,6 +821,8 @@ void dcp_hotplug(struct work_struct *work) connector->connected, READ_ONCE(dcp->mode_state.valid), dcp->nr_modes); scoped_guard(mutex, &connector->base.dev->mode_config.mutex) { + if (READ_ONCE(connector->dcp) == pdev) + dcp_update_connector_dimensions(dcp, &connector->base); if (!READ_ONCE(connector->connected)) dcp_clear_connector_edid(connector); } @@ -1056,6 +1058,7 @@ int dcp_get_modes(struct drm_connector *connector) bool vrr_capable = false; bool physical_edid; u64 edid_generation = 0; + u64 service_epoch = 0; int i; dcp_consume_edid_invalidation(apple_connector); @@ -1098,9 +1101,15 @@ int dcp_get_modes(struct drm_connector *connector) if (physical_edid) { mutex_lock(&dcp->hpd_mutex); edid_generation = dcp->typec_generation; - if (apple_connector->drm_edid && - apple_connector->drm_edid_generation != edid_generation) - dcp_clear_connector_edid(apple_connector); + } + mutex_lock(&dcp->dcpavserv.lock); + /* Consume CLOSE invalidation under the publication/retirement lock. */ + dcp_consume_edid_invalidation(apple_connector); + if (apple_connector->drm_edid && + (!dcpavserv_edid_service_current(dcp, apple_connector->drm_edid_service_epoch) || + (physical_edid && apple_connector->drm_edid_generation != edid_generation))) + dcp_clear_connector_edid(apple_connector); + if (physical_edid) { if (!dcpavserv_edid_current(dcp, edid_generation)) { dcp_clear_connector_edid(apple_connector); goto edid_done; @@ -1113,11 +1122,14 @@ int dcp_get_modes(struct drm_connector *connector) (dcp->dcpavservep && dcp->dcpavservep->inline_v2))) { const struct drm_edid *edid; /* Let physical disconnect advance the generation during the request. */ + mutex_unlock(&dcp->dcpavserv.lock); if (physical_edid) mutex_unlock(&dcp->hpd_mutex); - edid = dcpavserv_copy_edid(dcp, edid_generation); - if (physical_edid) { + edid = dcpavserv_copy_edid(dcp, edid_generation, &service_epoch); + if (physical_edid) mutex_lock(&dcp->hpd_mutex); + mutex_lock(&dcp->dcpavserv.lock); + if (physical_edid) { if (!dcpavserv_edid_current(dcp, edid_generation) || READ_ONCE(apple_connector->dcp) != pdev) { if (!IS_ERR_OR_NULL(edid)) @@ -1126,6 +1138,14 @@ int dcp_get_modes(struct drm_connector *connector) goto edid_done; } } + if (!IS_ERR_OR_NULL(edid) && + !dcpavserv_edid_service_current(dcp, service_epoch)) { + drm_edid_free(edid); + dcp_clear_connector_edid(apple_connector); + goto edid_done; + } + /* Retirement may have queued cleanup while this request waited. */ + dcp_consume_edid_invalidation(apple_connector); if (IS_ERR_OR_NULL(edid)) { /* * An internal panel has no AV service and so no EDID; @@ -1139,6 +1159,7 @@ int dcp_get_modes(struct drm_connector *connector) drm_edid_free(apple_connector->drm_edid); apple_connector->drm_edid = edid; apple_connector->drm_edid_generation = edid_generation; + apple_connector->drm_edid_service_epoch = service_epoch; } } if (dcp->nr_modes && apple_connector->drm_edid) { @@ -1172,6 +1193,7 @@ int dcp_get_modes(struct drm_connector *connector) } edid_done: + mutex_unlock(&dcp->dcpavserv.lock); if (physical_edid) mutex_unlock(&dcp->hpd_mutex); if (apple_connector->drm_edid) diff --git a/tools/diagnostics/test-apple-display-metadata-lifetime.py b/tools/diagnostics/test-apple-display-metadata-lifetime.py index 3b9ed5738caa0..4fae50d0698c0 100644 --- a/tools/diagnostics/test-apple-display-metadata-lifetime.py +++ b/tools/diagnostics/test-apple-display-metadata-lifetime.py @@ -31,6 +31,7 @@ def function(text, signature): cache = dcp[dcp.index('static bool dcp_firmware_metadata_enabled('):dcp.index('void dcp_set_dimensions(')] getter = function(iomfb, 'int dcp_get_modes(') physical_current = function((base / 'epic/dpavservep.c').read_text(), 'bool dcpavserv_edid_current(') +service_current = function((base / 'epic/dpavservep.c').read_text(), 'bool dcpavserv_edid_service_current(') cleanup = function(iomfb, 'static void dcp_clear_connector_edid(') + '\n' + function(iomfb, 'static void dcp_consume_edid_invalidation(') hotplug = function(iomfb, 'void dcp_hotplug(') disconnect = function(dcp, 'static void disconnected_hpd_event(') @@ -117,7 +118,7 @@ def function(text, signature): const struct drm_edid *drm_edid; bool drm_edid_firmware_derived; bool edid_invalidation_pending; - u64 drm_edid_generation; + u64 drm_edid_generation, drm_edid_service_epoch; bool connected; struct work_struct hotplug_wq; }; @@ -128,6 +129,7 @@ def function(text, signature): struct apple_dcp { void *dev; struct mutex hpd_mutex; + struct { struct mutex lock; bool enabled; void *service; u64 epoch; } dcpavserv; bool typec_cable_connected, quiescing; u64 typec_generation; struct { int iomfb_method_profile; } hw; @@ -153,6 +155,7 @@ def function(text, signature): static bool update_error; static bool raw_available; static bool detach_during_read, replug_during_read; +static bool retire_after_read; static const struct drm_edid *allocate(unsigned int identity) { struct drm_edid *edid = malloc(sizeof(*edid)); assert(edid); @@ -171,9 +174,11 @@ def function(text, signature): static void drm_connector_set_vrr_capable_property(struct drm_connector *c, bool v) { (void)c; (void)v; } static bool dcp_has_panel(struct apple_dcp *p) { (void)p; return false; } -static const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *p, u64 generation) +static const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *p, u64 generation, u64 *epoch) { - assert(!mutex_held(&p->hpd_mutex)); + assert(!mutex_held(&p->hpd_mutex) && !mutex_held(&p->dcpavserv.lock)); + *epoch=p->dcpavserv.epoch; + if (retire_after_read) { p->dcpavserv.epoch+=2; p->connector->edid_invalidation_pending=true; } (void)generation; raw_reads++; if (detach_during_read) { mutex_lock(&p->hpd_mutex); p->typec_generation++; @@ -185,6 +190,11 @@ def function(text, signature): static int drm_edid_connector_update(struct drm_connector *c, const struct drm_edid *e) { assert(mutex_held(&c->dev->mode_config.mutex)); + if (e && e->identity==0xfeed) { + assert(mutex_held(¤t_for_update->dcpavserv.lock)); + if(current_for_update->hw.iomfb_method_profile==DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + assert(mutex_held(¤t_for_update->hpd_mutex)); + } if (e && update_error) return -ENOMEM; struct test_blob *old=c->edid_blob_ptr; struct test_blob *next=NULL; @@ -199,6 +209,7 @@ def function(text, signature): if (invalidate_during_update && e && e->identity!=0xfeed) current_for_update->metadata.epoch.invalidated=true; return 0; } +static void dcp_update_connector_dimensions(struct apple_dcp *p, struct drm_connector *c) { (void)p; assert(mutex_held(&c->dev->mode_config.mutex)); } static bool dcp_is_usb4_output(struct apple_dcp *p) { (void)p; return false; } static bool apple_dp_tunnel_t602x(void) { return false; } #define DRM_MODE_LINK_STATUS_BAD 1 @@ -245,8 +256,10 @@ def function(text, signature): assert(!allocations); memset(&p,0,sizeof(p)); memset(&c,0,sizeof(c)); assert(!pthread_mutex_init(&p.hpd_mutex.native,NULL)); + assert(!pthread_mutex_init(&p.dcpavserv.lock.native,NULL)); + p.dcpavserv.enabled=true; p.dcpavserv.service=&p; p.dcpavserv.epoch=1; p.typec_cable_connected=true; p.typec_generation=1; - detach_during_read=replug_during_read=false; + detach_during_read=replug_during_read=retire_after_read=false; p.hw.iomfb_method_profile=DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; p.fw_compat=DCP_FIRMWARE_H17P; static typeof(*p.dcpavservep) av={true}; p.dcpavservep=&av; @@ -419,14 +432,28 @@ def function(text, signature): assert(probe(&c.base)==1 && c.drm_edid && c.drm_edid_generation==2); p.typec_generation++; /* Missed worker: generation alone rejects the old cache. */ assert(probe(&c.base)==1 && raw_reads==3 && c.drm_edid_generation==3); + initial(); raw_available=true; + assert(probe(&c.base)==1 && c.drm_edid && c.drm_edid_service_epoch==1); + p.dcpavserv.epoch+=2; /* Replacement without physical generation or worker. */ + assert(probe(&c.base)==1 && raw_reads==2 && c.drm_edid_service_epoch==3); + p.dcpavserv.enabled=false; p.dcpavserv.epoch++; + c.edid_invalidation_pending=true; raw_available=false; + assert(probe(&c.base)==1 && !c.drm_edid && !property_identity && !allocations); + initial(); raw_available=true; retire_after_read=true; + assert(probe(&c.base)==1 && !c.drm_edid && !property_identity && !allocations); + assert(p.typec_generation==1); /* Replacement in the check/publication gap. */ + retire_after_read=false; + assert(probe(&c.base)==1 && c.drm_edid_service_epoch==3); + assert(!c.edid_invalidation_pending); /* Old CLOSE cannot erase the fresh cache. */ + dcp_hotplug(&c.hotplug_wq); assert(c.drm_edid); initial(); assert(!blob_allocations && trace_observations); - puts("PASS: mutex-serialized EDID/blob get_modes, hotplug, sysfs reader and same-epoch publication, first-connection scope, raw precedence, delayed disconnect/replug rejection, generation cache invalidation, no added modes"); + puts("PASS: mutex-serialized EDID/blob get_modes, hotplug, sysfs reader and same-epoch publication, first-connection scope, raw precedence, delayed disconnect/replug rejection, physical and service cache invalidation, service replacement publication gap, no added modes"); return 0; } ''' with tempfile.TemporaryDirectory(prefix='dcp-metadata-lifetime-') as tmp: tmp = Path(tmp) - (tmp / 'test.c').write_text(prefix + metadata + epoch + state + cache + chunk_callbacks + cleanup + physical_current + getter + hotplug + disconnect + suffix) + (tmp / 'test.c').write_text(prefix + metadata + epoch + state + cache + chunk_callbacks + cleanup + physical_current + service_current + getter + hotplug + disconnect + suffix) command = shlex.split(os.environ.get('CC','clang')) command += ['-std=gnu11','-Wall','-Wextra','-Werror','-Wno-sign-compare','-O1','-g', '-fsanitize=address,undefined','-fno-omit-frame-pointer','-pthread'] diff --git a/tools/diagnostics/test-apple-edid-rx-progress.py b/tools/diagnostics/test-apple-edid-rx-progress.py new file mode 100644 index 0000000000000..9f4d99ddf5683 --- /dev/null +++ b/tools/diagnostics/test-apple-edid-rx-progress.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Prove DisplayAttributes cannot block a later AFK reply on ordered RTKit RX.""" +from pathlib import Path +import argparse +import os +import shlex +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--ref', help='public git revision for a negative baseline test') +args = parser.parse_args() + +def source(path): + if args.ref: + return subprocess.check_output(['git', '-C', str(root), 'show', args.ref+':'+path], text=True) + return (root/path).read_text() + +def function(text, signature): + start = text.index(signature) + opening = text.index('{', start) + while ';' in text[start:opening]: + start = text.index(signature, start+1) + opening = text.index('{', start) + end, depth = opening+1, 1 + while depth: + depth += (text[end]=='{')-(text[end]=='}') + end += 1 + return text[start:end] + +dcp=source('drivers/gpu/drm/apple/dcp.c') +iomfb=source('drivers/gpu/drm/apple/iomfb.c') +af=source('drivers/gpu/drm/apple/afk.c') +rtkit=source('drivers/soc/apple/rtkit.c') +template=source('drivers/gpu/drm/apple/iomfb_template.c') +assert 'alloc_ordered_workqueue("rtkit-%s"' in rtkit +assert '.recv_message = dcp_recv_msg' in dcp +assert 'dcp_set_dimensions(dcp);' in template[template.index('"DisplayAttributes"'):] +assert 'queue_work(ep->wq, &work->work)' in af +prefix=r''' +#undef NDEBUG +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint64_t u64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define GFP_KERNEL 0 +#define kzalloc(n,f) calloc(1,n) +#define kfree(p) free(p) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define INIT_WORK(p,f) ((p)->fn=(f)) +#define dev_warn(...) ((void)0) +#define dev_warn_ratelimited(...) ((void)0) +#define dev_dbg_ratelimited(...) ((void)0) +#define trace_dcp_recv_msg(...) ((void)0) +#define DCP_FIRMWARE_H17P 2 +#define IOMFB_ENDPOINT 0x37 +#define AV_ENDPOINT 0x29 +#define SYSTEM_ENDPOINT 0x20 +#define DISP0_ENDPOINT 0x23 +#define DPTX_CONTROLLER_ENDPOINT 0x24 +#define APPLE_RTKIT_EP_MGMT 0 +#define APPLE_RTKIT_EP_CRASHLOG 1 +#define APPLE_RTKIT_EP_SYSLOG 2 +#define APPLE_RTKIT_EP_IOREPORT 4 +#define APPLE_RTKIT_EP_OSLOG 8 +#define APPLE_RTKIT_APP_ENDPOINT_START 0x20 +struct mutex { pthread_mutex_t native; }; +static _Thread_local struct mutex *locks[4]; +static _Thread_local unsigned int depth; +static bool held(struct mutex *m) { for(unsigned int i=0;inative)); locks[depth++]=m; } +static void mutex_unlock(struct mutex *m) { assert(depth && locks[depth-1]==m); depth--; assert(!pthread_mutex_unlock(&m->native)); } +#define lockdep_assert_held(p) assert(held(p)) +#define spin_lock_irqsave(p,f) do { mutex_lock(p); (f)=0; } while(0) +#define spin_unlock_irqrestore(p,f) do { (void)(f); mutex_unlock(p); } while(0) +struct work_struct { void (*fn)(struct work_struct *); }; +struct workqueue_struct { struct work_struct *pending; pthread_cond_t ready; pthread_mutex_t lock; }; +struct drm_device { struct { struct mutex mutex; } mode_config; }; +struct drm_connector { struct drm_device *dev; struct { int width_mm,height_mm; } display_info; }; +struct apple_connector { struct drm_connector base; struct work_struct hotplug_wq; }; +struct apple_dcp; +struct apple_dcp_afkep { struct apple_dcp *dcp; u8 endpoint; struct mutex lock; bool stopping; struct workqueue_struct *wq; }; +struct apple_dcp { + bool quiescing; int fw_compat; void *dev; + int width_mm,height_mm; struct { int width_mm,height_mm; } panel; + unsigned int nr_modes; struct { struct { int width_mm,height_mm; } mode; } modes[1]; + struct apple_connector *connector; + struct apple_dcp_afkep *dcpavservep,*dptxep,*avep,*systemep,*ibootep,*dptx_controller_ep; + struct { struct mutex lock; bool stopped; void *owner; } iomfb; +}; +struct afk_receive_message_work { struct apple_dcp_afkep *ep; u64 message; struct work_struct work; }; +struct apple_rtkit_ops { void (*recv_message)(void *,u8,u64); }; +struct apple_rtkit { bool shutting_down; const struct apple_rtkit_ops *ops; void *cookie,*dev; }; +struct apple_rtkit_rx_work { struct apple_rtkit *rtk; u8 ep; u64 msg; struct work_struct work; }; +#define current ((void *)1) +static pthread_mutex_t reply_lock=PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t reply_ready=PTHREAD_COND_INITIALIZER; +static bool reply_done; +static unsigned int advances, dimension_updates; +static struct workqueue_struct dimensions_queue={.ready=PTHREAD_COND_INITIALIZER,.lock=PTHREAD_MUTEX_INITIALIZER}; +static struct workqueue_struct afk_queue={.ready=PTHREAD_COND_INITIALIZER,.lock=PTHREAD_MUTEX_INITIALIZER}; +static void queue_work(struct workqueue_struct *q,struct work_struct *w) +{ assert(!pthread_mutex_lock(&q->lock)); assert(!q->pending); q->pending=w; pthread_cond_signal(&q->ready); assert(!pthread_mutex_unlock(&q->lock)); } +static void schedule_work(struct work_struct *w) { queue_work(&dimensions_queue,w); } +static void afk_receive_message_worker(struct work_struct *w) +{ + struct afk_receive_message_work *a=container_of(w,struct afk_receive_message_work,work); + assert(a->ep->endpoint==0x2a && a->message==7); + assert(!pthread_mutex_lock(&reply_lock)); reply_done=true; + pthread_cond_signal(&reply_ready); assert(!pthread_mutex_unlock(&reply_lock)); free(a); +} +static bool iomfb_uses_queue(struct apple_dcp *d) { (void)d; return true; } +static void iomfb_queue_advance(struct apple_dcp *d) { assert(held(&d->iomfb.lock)); advances++; } +static void dcp_set_dimensions(struct apple_dcp *d); +static void iomfb_recv_message(struct apple_dcp *d,u64 message) +{ assert(message==1 && held(&d->iomfb.lock)); dcp_set_dimensions(d); } +#define rx_unused(name) static void name(struct apple_rtkit *r,u64 m) { (void)r;(void)m; } +rx_unused(apple_rtkit_management_rx) +rx_unused(apple_rtkit_crashlog_rx) +rx_unused(apple_rtkit_syslog_rx) +rx_unused(apple_rtkit_ioreport_rx) +rx_unused(apple_rtkit_oslog_rx) +''' +parts=[function(dcp,'void dcp_set_dimensions(')] +if 'void dcp_update_connector_dimensions(' in dcp: + parts.append(function(dcp,'void dcp_update_connector_dimensions(')) +else: + parts.append('static void dcp_update_connector_dimensions(struct apple_dcp *d, struct drm_connector *c) { (void)d; (void)c; }') +parts += [function(iomfb,'void iomfb_recv_msg('),function(af,'int afk_receive_message('), + function(dcp,'static struct apple_dcp_afkep *dcp_afkep('), + function(dcp,'static void dcp_recv_msg('),function(rtkit,'static void apple_rtkit_rx_work(')] +suffix=r''' +static struct drm_device drm={.mode_config.mutex.native=PTHREAD_MUTEX_INITIALIZER}; +static struct apple_connector connector={.base.dev=&drm}; +static struct apple_dcp d={.connector=&connector,.width_mm=600,.height_mm=340,.nr_modes=1,.fw_compat=DCP_FIRMWARE_H17P,.iomfb.lock.native=PTHREAD_MUTEX_INITIALIZER}; +static struct apple_dcp_afkep av={.dcp=&d,.endpoint=0x2a,.lock.native=PTHREAD_MUTEX_INITIALIZER,.wq=&afk_queue}; +static void dimension_work(struct work_struct *w) +{ + (void)w; + mutex_lock(&drm.mode_config.mutex); + dcp_update_connector_dimensions(&d,&connector.base); dimension_updates++; + mutex_unlock(&drm.mode_config.mutex); +} +static void *worker(void *arg) +{ + struct workqueue_struct *q=arg; + assert(!pthread_mutex_lock(&q->lock)); + while(!q->pending) assert(!pthread_cond_wait(&q->ready,&q->lock)); + struct work_struct *w=q->pending; q->pending=NULL; + assert(!pthread_mutex_unlock(&q->lock)); w->fn(w); return NULL; +} +static const struct apple_rtkit_ops ops={.recv_message=dcp_recv_msg}; +static struct apple_rtkit rtk={.ops=&ops,.cookie=&d}; +static void *ordered_rx(void *arg) +{ + (void)arg; + /* One ordered RX worker must complete DisplayAttributes before AFK. */ + struct apple_rtkit_rx_work *a=calloc(1,sizeof(*a)); assert(a); + a->rtk=&rtk; a->ep=IOMFB_ENDPOINT; a->msg=1; apple_rtkit_rx_work(&a->work); + a=calloc(1,sizeof(*a)); assert(a); + a->rtk=&rtk; a->ep=0x2a; a->msg=7; apple_rtkit_rx_work(&a->work); + return NULL; +} +int main(void) +{ + pthread_t rx,afk,dimensions; + (void)schedule_work; /* The negative baseline does not defer dimensions. */ + d.dcpavservep=&av; connector.hotplug_wq.fn=dimension_work; + mutex_lock(&drm.mode_config.mutex); /* DRM get_modes holds this while waiting. */ + assert(!pthread_create(&afk,NULL,worker,&afk_queue)); + assert(!pthread_create(&dimensions,NULL,worker,&dimensions_queue)); + assert(!pthread_mutex_lock(&reply_lock)); + assert(!pthread_create(&rx,NULL,ordered_rx,NULL)); + struct timespec deadline; assert(!clock_gettime(CLOCK_REALTIME,&deadline)); deadline.tv_sec+=2; + int rc=0; + while(!reply_done && !rc) rc=pthread_cond_timedwait(&reply_ready,&reply_lock,&deadline); + if(!reply_done) { fputs("FAIL: ordered RTKit RX blocked before AFK reply while mode_config.mutex held\n",stderr); fflush(stderr); _Exit(1); } + assert(!pthread_mutex_unlock(&reply_lock)); + assert(!dimension_updates); /* Deferred worker cannot take the probe's mutex. */ + mutex_unlock(&drm.mode_config.mutex); + assert(!pthread_join(rx,NULL) && !pthread_join(afk,NULL) && !pthread_join(dimensions,NULL)); + assert(advances==1 && dimension_updates==1 && connector.base.display_info.width_mm==600 && connector.base.display_info.height_mm==340); + assert(d.modes[0].mode.width_mm==600 && d.modes[0].mode.height_mm==340); + puts("PASS: actual ordered RTKit RX -> synchronous IOMFB dimensions callback -> AFK enqueue makes reply progress while DRM mutex held; deferred dimensions update completes after release"); +} +''' +with tempfile.TemporaryDirectory(prefix='apple-edid-rx-') as tmp: + test=Path(tmp)/'test.c'; test.write_text(prefix+'\n'.join(parts)+suffix) + cc=shlex.split(os.environ.get('HOSTCC','clang')) + subprocess.run(cc+['-std=gnu11','-Wall','-Werror','-O1','-g','-pthread', + '-fsanitize=address,undefined','-fno-omit-frame-pointer',str(test),'-o',str(Path(tmp)/'test')],check=True) + subprocess.run([str(Path(tmp)/'test')],check=True) diff --git a/tools/diagnostics/test-apple-h17p-edid.py b/tools/diagnostics/test-apple-h17p-edid.py index 1ac755f6f69e9..e95ed887aee80 100644 --- a/tools/diagnostics/test-apple-h17p-edid.py +++ b/tools/diagnostics/test-apple-h17p-edid.py @@ -42,6 +42,7 @@ def function(file, signature): typedef uint64_t u64; typedef int64_t s64; #define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) #define ERR_PTR(x) ((void *)(intptr_t)(x)) #define PTR_ERR(x) ((int)(intptr_t)(x)) #define IS_ERR(x) ((uintptr_t)(x) > (uintptr_t)-4096) @@ -52,6 +53,16 @@ def function(file, signature): static void release(bool **p) { if(*p) { assert(**p); **p=false; } } static bool *once(bool *p) { assert(*p); *p=false; return NULL; } #define scoped_guard(k,p) for(bool *_lock __attribute__((cleanup(release)))=acquire(p); _lock; _lock=once(_lock)) +static void mutex_lock(bool *p) { (void)acquire(p); } +static void mutex_unlock(bool *p) { (void)once(p); } +#define dev_info(...) ((void)0) +#define dev_err(...) ((void)0) +#define trace_dcpavserv_init(...) ((void)0) +#define DRM_CONNECTOR_REGISTERED 1 +#define to_platform_device(x) (x) +static void reinit_completion(int *p) { *p=0; } +static void complete(int *p) { *p=1; } +static void schedule_work(int *p) { (*p)++; } #define dev_dbg_ratelimited(...) ((void)0) #define dev_warn_ratelimited(...) ((void)0) #define trace_dcp_recv_msg(...) ((void)0) @@ -67,11 +78,13 @@ def function(file, signature): #define DPTX_CONTROLLER_ENDPOINT 0x24 #define IOMFB_ENDPOINT 0x37 struct apple_dcp; +struct apple_connector { void *dcp; struct { int registration_state; } base; bool connected, edid_invalidation_pending; int hotplug_wq; }; struct apple_dcp_afkep { struct apple_dcp *dcp; u8 endpoint; bool inline_v2; }; -struct apple_epic_service { struct apple_dcp_afkep *ep; unsigned int users; bool enabled; }; +struct apple_epic_service { struct apple_dcp_afkep *ep; unsigned int users; bool enabled; void *cookie; unsigned int channel; }; struct apple_dcp { struct apple_dcp_afkep *dcpavservep, *dptxep, *avep, *systemep, *ibootep, *dptx_controller_ep; - struct { bool lock, enabled; struct apple_epic_service *service; } dcpavserv; + struct { bool lock, enabled; u64 epoch; int enable_completion; struct apple_epic_service *service; } dcpavserv; + struct apple_connector *connector; unsigned int nr_modes; struct { int iomfb_method_profile; } hw; bool hpd_mutex, typec_cable_connected, quiescing; u64 typec_generation; @@ -88,13 +101,14 @@ def function(file, signature): { assert(s->users); s->users--; } static void drm_edid_free(const struct drm_edid *e) { if(e) { assert(allocations); allocations--; free((void *)e); } } +static void dcpavserv_teardown(struct apple_epic_service *s); static const struct drm_edid *dcpavserv_read_edid_inline(struct apple_epic_service *s) { struct apple_dcp *d=s->ep->dcp; assert(!d->hpd_mutex && !d->dcpavserv.lock && s->users==2); reads++; if(event==1 || event==2) { d->typec_generation++; d->typec_cable_connected=(event==2); } - if(event==3) { d->dcpavserv.service=NULL; d->dcpavserv.enabled=false; s->enabled=false; afk_service_put(s); } + if(event==3) { s->enabled=false; dcpavserv_teardown(s); } struct drm_edid *e=malloc(sizeof(*e)); assert(e); e->identity=42; allocations++; return e; } static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *s) @@ -103,16 +117,24 @@ def function(file, signature): static void afk_receive_message(struct apple_dcp_afkep *ep, u64 message) { (void)message; received_ep=ep; receives++; } ''' -parts = [function('dcp.c', 'static struct apple_dcp_afkep *dcp_afkep('), +parts = [function('epic/dpavservep.c', 'static void dcpavserv_notify_edid('), + function('epic/dpavservep.c', 'static void dcpavserv_invalidate_edid('), + function('epic/dpavservep.c', 'static void dcpavserv_teardown('), + function('dcp.c', 'static struct apple_dcp_afkep *dcp_afkep('), function('dcp.c', 'static void dcp_recv_msg('), function('epic/dpavservep.c', 'static int dcpavserv_prepare('), function('epic/dpavservep.c', 'static int dcpdpserv_prepare('), + function('epic/dpavservep.c', 'static void dcpavserv_init('), function('epic/dpavservep.c', 'bool dcpavserv_edid_current('), + function('epic/dpavservep.c', 'bool dcpavserv_edid_service_current('), function('epic/dpavservep.c', 'const struct drm_edid *dcpavserv_copy_edid(')] suffix = r''' int main(void) { - struct apple_dcp d={.fw_compat=DCP_FIRMWARE_H17P}; + u64 epoch=0; + int device; + struct apple_connector connector={.dcp=&device,.base.registration_state=DRM_CONNECTOR_REGISTERED,.connected=true}; + struct apple_dcp d={.fw_compat=DCP_FIRMWARE_H17P,.dev=&device,.connector=&connector,.nr_modes=1}; struct apple_dcp_afkep av={.dcp=&d,.endpoint=0x28}, tx={.dcp=&d,.endpoint=0x2a}; d.dcpavservep=&av; d.dptxep=&tx; assert(dcp_afkep(&d,0x28)==&av && dcp_afkep(&d,0x2a)==&tx && !dcp_afkep(&d,0x2c)); @@ -138,25 +160,38 @@ def function(file, signature): av.endpoint=0x28; assert(dcpavserv_prepare(&s,name,class,1)==-ENODEV); av.endpoint=0x2a; assert(dcpdpserv_prepare(&s,"dcpdp-service-epic",class,3)==-ENODEV); d.dcpavserv.enabled=true; assert(dcpavserv_prepare(&s,name,class,1)==-EBUSY); - d.dcpavserv.service=&s; d.typec_generation=1; d.typec_cable_connected=true; - const struct drm_edid *e=dcpavserv_copy_edid(&d,1); + d.dcpavserv.service=&s; d.dcpavserv.epoch=1; d.typec_generation=1; d.typec_cable_connected=true; + const struct drm_edid *e=dcpavserv_copy_edid(&d,1,&epoch); assert(!IS_ERR(e) && s.users==1 && reads==1); drm_edid_free(e); d.typec_cable_connected=false; - assert(PTR_ERR(dcpavserv_copy_edid(&d,1))==-ESTALE && reads==1); + assert(PTR_ERR(dcpavserv_copy_edid(&d,1,&epoch))==-ESTALE && reads==1); d.typec_cable_connected=true; - assert(PTR_ERR(dcpavserv_copy_edid(&d,0))==-ESTALE && reads==1); - event=1; assert(PTR_ERR(dcpavserv_copy_edid(&d,1))==-ESTALE && !allocations && s.users==1); + assert(PTR_ERR(dcpavserv_copy_edid(&d,0,&epoch))==-ESTALE && reads==1); + event=1; assert(PTR_ERR(dcpavserv_copy_edid(&d,1,&epoch))==-ESTALE && !allocations && s.users==1); event=2; d.typec_cable_connected=true; - assert(PTR_ERR(dcpavserv_copy_edid(&d,2))==-ESTALE && !allocations && s.users==1); - event=0; e=dcpavserv_copy_edid(&d,3); assert(!IS_ERR(e)); drm_edid_free(e); - event=3; assert(PTR_ERR(dcpavserv_copy_edid(&d,3))==-ENODEV && !allocations && !s.users); - assert(PTR_ERR(dcpavserv_copy_edid(&d,3))==-ENODEV); + assert(PTR_ERR(dcpavserv_copy_edid(&d,2,&epoch))==-ESTALE && !allocations && s.users==1); + event=0; e=dcpavserv_copy_edid(&d,3,&epoch); assert(!IS_ERR(e)); drm_edid_free(e); + event=3; assert(PTR_ERR(dcpavserv_copy_edid(&d,3,&epoch))==-ENODEV && !allocations && !s.users); + assert(PTR_ERR(dcpavserv_copy_edid(&d,3,&epoch))==-ENODEV); + assert(connector.edid_invalidation_pending && connector.hotplug_wq && d.dcpavserv.epoch==2); + /* Same slot and physical generation: CLOSE/re-OPEN gets a fresh ticket. */ + s.enabled=true; dcpavserv_init(&s,name,class,1); + assert(d.dcpavserv.service==&s && s.users==1 && d.dcpavserv.epoch==3); + mutex_lock(&d.dcpavserv.lock); + assert(!dcpavserv_edid_service_current(&d,1) && dcpavserv_edid_service_current(&d,3)); + mutex_unlock(&d.dcpavserv.lock); + event=0; e=dcpavserv_copy_edid(&d,3,&epoch); assert(!IS_ERR(e) && epoch==3); + /* Simulate retirement in the final validation-to-publication gap. */ + s.enabled=false; dcpavserv_teardown(&s); s.enabled=true; dcpavserv_init(&s,name,class,1); + mutex_lock(&d.dcpavserv.lock); assert(!dcpavserv_edid_service_current(&d,epoch)); mutex_unlock(&d.dcpavserv.lock); + drm_edid_free(e); assert(d.typec_generation==3 && d.dcpavserv.epoch==5 && s.users==1); + s.enabled=false; dcpavserv_teardown(&s); av.inline_v2=false; av.endpoint=0x28; d.hw.iomfb_method_profile=0; d.dcpavserv.enabled=false; assert(!dcpavserv_prepare(&s,name,NULL,0)); assert(dcpavserv_prepare(&s,name,class,1)==-ENODEV); assert(!dcpdpserv_prepare(&s,NULL,NULL,3)); d.dcpavserv.service=&s; d.dcpavserv.enabled=true; s.enabled=true; s.users=1; event=0; - d.typec_cable_connected=false; e=dcpavserv_copy_edid(&d,0); + d.typec_cable_connected=false; e=dcpavserv_copy_edid(&d,0,&epoch); assert(!IS_ERR(e) && legacy_reads==1 && s.users==1); drm_edid_free(e); assert(!allocations && !d.hpd_mutex && !d.dcpavserv.lock); puts("PASS: production old/new lookup and receive, exact service admission before OPEN, delayed disconnect/replug, service detach pins, legacy transport"); From d2a2d2bbd46a9692c915ce090d0caf3ad12f4191 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 14:32:53 +1100 Subject: [PATCH 0959/1579] drm/apple: read external H17P EDID through owned AFKV2 buffers Use the existing host-owned AFKV2 OOB command transport for the observed external H17P DPAV service. Select service admission, notification and the strict length-aware EDID reader by AFKV2 framing rather than inline payload placement. Keep legacy profiles on their existing endpoint and reader. Length-aware AFKV2 callers must copy only the validated actual reply length for both inline and owned OOB replies. Keep strict callers' full-length contract and non-AFKV2 behavior unchanged. Preserve exact original DMA address/bounds matching, command slots, timeout retention and late-ACK release, without following peer-provided pointers. Apply the public owned transport correction from 821e79667f4d935fb50e1c6bc95beb4710f316cd on the service-lifetime/RX-progress base. Retain physical generation and service epoch checks, strict EDID header/length/block/checksum validation, and the conservative firmware metadata guard. No one-shot diagnostic permission, enlarged trace capture, timeout snapshot or refresh-rate admission is included. Extend actual sender/service extraction fixtures for short owned OOB responses, strict and legacy contracts, and the real late-ACK completer. A mismatched or duplicate reply cannot release retained DMA; the matching late reply retires the pending slot even after service retirement. Adapt lifetime/notification fixtures to the framing guard. Validation: full AFKV2 suite with ASan/UBSan (917 codec assertions), EDID lifetime, threaded RX-progress and metadata regressions pass. Targeted ARM64 W=1 module compile/modpost/link passes without warnings. Initial boot-time reads and physical hotplug remain owner qualification work. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 4 +- drivers/gpu/drm/apple/epic/dpavservep.c | 16 ++--- drivers/gpu/drm/apple/iomfb.c | 2 +- .../test-apple-display-metadata-lifetime.py | 2 +- tools/diagnostics/test-apple-h17p-edid.py | 6 +- .../drivers/gpu/apple-afkv2/extract-edid.py | 2 +- .../drivers/gpu/apple-afkv2/extract-notify.py | 8 +-- .../drivers/gpu/apple-afkv2/extract-send.py | 60 +++++++++++++++---- .../gpu/apple-afkv2/extract-service.py | 5 +- 9 files changed, 74 insertions(+), 31 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 946cdfb946a03..e182105c47faa 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -165,7 +165,7 @@ void afk_trace_ring_snapshot(struct apple_dcp_afkep *ep) u32 rx_rptr = 0, rx_wptr = 0, tx_rptr = 0, tx_wptr = 0; bool rx_ready, tx_ready; - if (!ep || !ep->inline_v2 || !trace_afk_ring_snapshot_enabled()) + if (!ep || !ep->wire_v2 || !trace_afk_ring_snapshot_enabled()) return; spin_lock_irqsave(&ep->lock, flags); if (ep->stopping) { @@ -1680,7 +1680,7 @@ static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 t ret = 0; if (retcode) *retcode = service->cmds[idx].retcode; - if (ep->inline_v2 && received) + if (ep->wire_v2 && received) copy_len = service->cmds[idx].reply_length; if (ep->wire_v2 && !service->cmds[idx].retcode && service->cmds[idx].reply_length < copy_len) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index a43e94f7b978a..0f00f80da716e 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -23,7 +23,7 @@ static void dcpavserv_notify_edid(struct apple_epic_service *service) struct apple_dcp *dcp = service->ep->dcp; struct apple_connector *connector = READ_ONCE(dcp->connector); - if (!service->ep->inline_v2 || READ_ONCE(dcp->quiescing) || !connector || + if (!service->ep->wire_v2 || READ_ONCE(dcp->quiescing) || !connector || !READ_ONCE(dcp->nr_modes) || !READ_ONCE(connector->connected) || READ_ONCE(connector->base.registration_state) != DRM_CONNECTOR_REGISTERED || READ_ONCE(connector->dcp) != to_platform_device(dcp->dev)) @@ -41,7 +41,7 @@ static int dcpavserv_prepare(struct apple_epic_service *service, const char *nam if (!name || strcmp(name, "dcpav-service-epic")) return -ENODEV; - if (service->ep->inline_v2) { + if (service->ep->wire_v2) { if (service->ep->endpoint != DPAVSERV_ENDPOINT_V2 || !class || strcmp(class, "AppleDCPDP2HDMI") || unit != 1) return -ENODEV; @@ -75,8 +75,8 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, service->cookie = &dcp->dcpavserv; complete(&dcp->dcpavserv.enable_completion); mutex_unlock(&dcp->dcpavserv.lock); - if (service->ep->inline_v2) { - dev_info(dcp->dev, "DCPAVSERV: inline EDID service ready on channel %u\n", + if (service->ep->wire_v2) { + dev_info(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", service->channel); dcpavserv_notify_edid(service); } @@ -139,7 +139,7 @@ static int dcpdpserv_prepare(struct apple_epic_service *service, const char *nam const char *class, s64 unit) { /* No production command transport is needed for this H17P service. */ - return service->ep->inline_v2 ? -ENODEV : 0; + return service->ep->wire_v2 ? -ENODEV : 0; } static void dcpdpserv_init(struct apple_epic_service *service, const char *name, @@ -352,7 +352,7 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation if (!service) return ERR_PTR(-ENODEV); - edid = service->ep->inline_v2 ? dcpavserv_read_edid_inline(service) : + edid = service->ep->wire_v2 ? dcpavserv_read_edid_inline(service) : dcpavserv_read_edid_legacy(service); mutex_lock(&dcp->dcpavserv.lock); if (dcp->dcpavserv.service != service || @@ -405,10 +405,10 @@ int dpavservep_init(struct apple_dcp *dcp) DPAVSERV_ENDPOINT_V2 : DPAVSERV_ENDPOINT, dpavservep_ops); if (IS_ERR(ep)) return PTR_ERR(ep); - /* Only the external H17P DPAV endpoint uses inline AFKV2 service calls. */ + /* External H17P uses AFKV2 service calls with host-owned OOB buffers. */ ep->wire_v2 = dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; - ep->inline_v2 = ep->wire_v2; + ep->inline_v2 = false; dcp->dcpavservep = ep; dcp->dcpavservep->match_epic_name = true; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 21a70f893f2e3..2d7b42869dc68 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1119,7 +1119,7 @@ int dcp_get_modes(struct drm_connector *connector) /* Internal H17P still has no supported EDID command transport. */ if (dcp->nr_modes && !apple_connector->drm_edid && (dcp->fw_compat != DCP_FIRMWARE_H17P || - (dcp->dcpavservep && dcp->dcpavservep->inline_v2))) { + (dcp->dcpavservep && dcp->dcpavservep->wire_v2))) { const struct drm_edid *edid; /* Let physical disconnect advance the generation during the request. */ mutex_unlock(&dcp->dcpavserv.lock); diff --git a/tools/diagnostics/test-apple-display-metadata-lifetime.py b/tools/diagnostics/test-apple-display-metadata-lifetime.py index 4fae50d0698c0..fa1ac9605f2c4 100644 --- a/tools/diagnostics/test-apple-display-metadata-lifetime.py +++ b/tools/diagnostics/test-apple-display-metadata-lifetime.py @@ -147,7 +147,7 @@ def function(text, signature): unsigned int nr_modes; int fw_compat; struct { bool valid; } mode_state; - struct { bool inline_v2; } *dcpavservep; + struct { bool wire_v2; } *dcpavservep; }; static struct apple_dcp *current_for_update; static unsigned int allocations, raw_reads, mode_adds, clears, property_identity, blob_allocations, notifications, retrains; diff --git a/tools/diagnostics/test-apple-h17p-edid.py b/tools/diagnostics/test-apple-h17p-edid.py index e95ed887aee80..75a1ef8275b22 100644 --- a/tools/diagnostics/test-apple-h17p-edid.py +++ b/tools/diagnostics/test-apple-h17p-edid.py @@ -79,7 +79,7 @@ def function(file, signature): #define IOMFB_ENDPOINT 0x37 struct apple_dcp; struct apple_connector { void *dcp; struct { int registration_state; } base; bool connected, edid_invalidation_pending; int hotplug_wq; }; -struct apple_dcp_afkep { struct apple_dcp *dcp; u8 endpoint; bool inline_v2; }; +struct apple_dcp_afkep { struct apple_dcp *dcp; u8 endpoint; bool wire_v2; }; struct apple_epic_service { struct apple_dcp_afkep *ep; unsigned int users; bool enabled; void *cookie; unsigned int channel; }; struct apple_dcp { struct apple_dcp_afkep *dcpavservep, *dptxep, *avep, *systemep, *ibootep, *dptx_controller_ep; @@ -141,7 +141,7 @@ def function(file, signature): dcp_recv_msg(&d,0x28,1); assert(received_ep==&av && receives==1); dcp_recv_msg(&d,0x2a,2); assert(received_ep==&tx && receives==2); d.hw.iomfb_method_profile=DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; - av.endpoint=0x2a; av.inline_v2=true; tx.endpoint=0x2c; + av.endpoint=0x2a; av.wire_v2=true; tx.endpoint=0x2c; assert(!dcp_afkep(&d,0x28) && dcp_afkep(&d,0x2a)==&av && dcp_afkep(&d,0x2c)==&tx); dcp_recv_msg(&d,0x28,3); assert(receives==2); dcp_recv_msg(&d,0x2a,4); assert(received_ep==&av && receives==3); @@ -186,7 +186,7 @@ def function(file, signature): mutex_lock(&d.dcpavserv.lock); assert(!dcpavserv_edid_service_current(&d,epoch)); mutex_unlock(&d.dcpavserv.lock); drm_edid_free(e); assert(d.typec_generation==3 && d.dcpavserv.epoch==5 && s.users==1); s.enabled=false; dcpavserv_teardown(&s); - av.inline_v2=false; av.endpoint=0x28; d.hw.iomfb_method_profile=0; + av.wire_v2=false; av.endpoint=0x28; d.hw.iomfb_method_profile=0; d.dcpavserv.enabled=false; assert(!dcpavserv_prepare(&s,name,NULL,0)); assert(dcpavserv_prepare(&s,name,class,1)==-ENODEV); assert(!dcpdpserv_prepare(&s,NULL,NULL,3)); diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py index 0ab0f5eda8cac..0d554d1c82b5d 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py @@ -17,7 +17,7 @@ def function(name): end += 1 return source[match.start():end] -assert "service->ep->inline_v2 ? dcpavserv_read_edid_inline(service)" in source +assert "service->ep->wire_v2 ? dcpavserv_read_edid_inline(service)" in source prefix = r''' #undef NDEBUG #include diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py index 26579f8399cb1..481aac7d0246a 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py @@ -23,7 +23,7 @@ struct base { int registration_state; }; struct apple_connector { bool connected;struct base base;void *dcp;int hotplug_wq; }; struct apple_dcp { struct apple_connector *connector;bool quiescing;unsigned int nr_modes;void *dev; }; -struct ep { struct apple_dcp *dcp;bool inline_v2; }; +struct ep { struct apple_dcp *dcp;bool wire_v2; }; struct apple_epic_service { struct ep *ep; }; static unsigned int queued; static void *to_platform_device(void *p) { return p; } @@ -33,7 +33,7 @@ int main(void) { int device,other;struct apple_dcp dcp={.dev=&device}; - struct ep ep={.dcp=&dcp,.inline_v2=true};struct apple_epic_service service={.ep=&ep}; + struct ep ep={.dcp=&dcp,.wire_v2=true};struct apple_epic_service service={.ep=&ep}; struct apple_connector connector={.dcp=&device}; dcpavserv_notify_edid(&service);assert(queued==0); dcp.connector=&connector;dcpavserv_notify_edid(&service);assert(queued==0); @@ -42,8 +42,8 @@ dcpavserv_notify_edid(&service);assert(queued==1 && connector.hotplug_wq==1); dcp.quiescing=true;dcpavserv_notify_edid(&service);assert(queued==1); dcp.quiescing=false;connector.dcp=&other;dcpavserv_notify_edid(&service);assert(queued==1); - connector.dcp=&device;ep.inline_v2=false;dcpavserv_notify_edid(&service);assert(queued==1); - ep.inline_v2=true;connector.connected=false;dcpavserv_notify_edid(&service);assert(queued==1); + connector.dcp=&device;ep.wire_v2=false;dcpavserv_notify_edid(&service);assert(queued==1); + ep.wire_v2=true;connector.connected=false;dcpavserv_notify_edid(&service);assert(queued==1); puts("PASS: actual late-EDID notification queues ready connector only; legacy/quiesce/unrouted/unregistered guarded"); } ''' diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py index 92ca8857cc104..0d80a729a3342 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py @@ -13,6 +13,13 @@ while depth: depth += (source[end] == "{") - (source[end] == "}") end += 1 +reply_match = re.search(r"static void afk_recv_handle_reply\([^;]*?\)\n\{", source) +if not reply_match: + raise RuntimeError("production reply completer not found") +reply_end, reply_depth = reply_match.end(), 1 +while reply_depth: + reply_depth += (source[reply_end] == "{") - (source[reply_end] == "}") + reply_end += 1 prefix = r''' #undef NDEBUG #include @@ -24,6 +31,7 @@ #define GFP_KERNEL 0 #define MAX_PENDING_CMDS 16 #define cpu_to_le32(x) (x) +#define le32_to_cpu(x) (x) #define cpu_to_le64(x) (x) #define kmalloc(n,f) malloc(n) #define kfree(p) free(p) @@ -32,6 +40,8 @@ #define spin_unlock_irqrestore(l,f) ((void)(l),(void)(f)) #define dma_rmb() do {} while(0) #define dev_warn(...) do {} while(0) +#define dev_err(...) do {} while(0) +#define complete(p) (*(p)=1) #define EPIC_TYPE_COMMAND 3 #define EPIC_CAT_COMMAND 0x30 #define DECLARE_COMPLETION_ONSTACK(n) int n=0 @@ -50,13 +60,18 @@ struct ops { const char *name; }; struct apple_epic_service { struct apple_dcp_afkep *ep; struct ops *ops; unsigned int channel; unsigned long cmd_map[1]; - int lock; bool enabled,torndown; u8 cmd_tag; struct cmd_info cmds[16]; + int lock; unsigned int users; bool enabled,torndown; u8 cmd_tag; struct cmd_info cmds[16]; }; static struct apple_epic_service *active; static unsigned int frees, sends; static int wire_fail, timeout, reply_status; -static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) { return s->enabled?s:NULL; } -static void afk_service_put(struct apple_epic_service *s) { (void)s; } +static size_t owned_reply_size=32888; +static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) +{ if(!s->enabled) return NULL; s->users++; return s; } +static void afk_service_put(struct apple_epic_service *s) { assert(s->users); s->users--; } +static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep *ep,u32 channel) +{ return active->ep==ep && active->channel==channel && (active->enabled || active->cmd_map[0]) ? active : NULL; } +static int test_bit(unsigned int bit,const unsigned long *map) { return !!(*map & (1UL<reply_length=ep->inline_v2?4:32888;c->retcode=reply_status; + c->reply_length=ep->inline_v2?4:owned_reply_size;c->retcode=reply_status; memset(c->rxbuf,0x77,c->reply_length);c->done=true;*c->completion=1; return 0; } @@ -92,7 +107,7 @@ int main(void) { struct dcp dcp={0};struct apple_dcp_afkep ep={.dcp=&dcp,.wire_v2=true,.inline_v2=true}; - struct ops ops={.name="test"};struct apple_epic_service s={.ep=&ep,.ops=&ops,.channel=4,.enabled=true}; + struct ops ops={.name="test"};struct apple_epic_service s={.ep=&ep,.ops=&ops,.channel=4,.enabled=true,.users=1}; u8 request[112]={0x5a},*output=calloc(1,32888);size_t received;u32 status=99; active=&s; assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); @@ -107,12 +122,37 @@ wire_fail=0;ep.inline_v2=false; assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); assert(received==32888 && status==0); - ep.inline_v2=true;timeout=1; + /* Length-aware owned OOB callers copy only the bounded actual reply. */ + owned_reply_size=4;memset(output,0x19,32888); + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==4 && output[0]==0x77 && output[3]==0x77 && output[4]==0x19 && s.users==1); + memset(output,0x19,32888); + assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,NULL)==-EMSGSIZE); + assert(output[0]==0x19 && !s.cmd_map[0]); + owned_reply_size=0;received=99; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==0 && output[0]==0x19); + /* Legacy framing retains its full-buffer copy contract. */ + ep.wire_v2=false;owned_reply_size=4; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==32888 && s.users==1); + ep.wire_v2=true;owned_reply_size=32888;timeout=1; assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)==-ETIMEDOUT); assert(s.cmd_map[0] && s.cmds[0].free_on_ack && !s.cmds[0].completion); - /* Mock the late-reply owner's release, avoiding leaks in this host test. */ - free(s.cmds[0].rxbuf);free(s.cmds[0].txbuf);free(output); - puts("PASS: extracted production sender, 128-byte command envelope / 32888-byte reply, strict OOB and timeout ownership"); + /* Execute the real late-ACK completer: mismatch/duplicate cannot free DMA. */ + unsigned int before_late=frees;u16 retained_tag=s.cmds[0].tag; + struct epic_cmd reply={.retcode=0}; + s.enabled=false;s.torndown=true; /* Retired service still owns the pending slot. */ + afk_recv_handle_reply(&ep,4,retained_tag^0x10,&reply,sizeof(reply)); + assert(frees==before_late && s.cmd_map[0] && !s.cmds[0].done && s.users==1); + afk_recv_handle_reply(&ep,4,retained_tag,&reply,sizeof(reply)); + assert(frees==before_late+2 && !s.cmd_map[0] && s.cmds[0].done && s.users==1); + afk_recv_handle_reply(&ep,4,retained_tag,&reply,sizeof(reply)); + assert(frees==before_late+2); + s.enabled=true;s.torndown=false;timeout=0; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(!s.cmd_map[0] && s.users==1);free(output); + puts("PASS: extracted production sender, 128-byte command envelope / 32888-byte reply, short OOB, strict/legacy callers and actual late-ACK ownership"); } ''' -pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + suffix) +pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + source[reply_match.start():reply_end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py index 68e3f5e6ba980..ab95f5c422710 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py @@ -79,8 +79,11 @@ too_long=0;wire_error=1; assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)==-ETIMEDOUT); wire_error=0; ep.inline_v2=false; + received=999; + assert(!afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + assert(received==4 && output[0]==0xde && output[3]==0xef && output[4]==0); assert(!afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,NULL)); - puts("PASS: extracted production service-call actual request / legacy padding / short reply / status / context / length tests"); + puts("PASS: extracted production service-call actual request / OOB padding and short reply / legacy / status / context / length tests"); } ''' pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + suffix) From bccf1fc991a1a86c40e3e3ee198af95d053720ed Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 14:48:51 +1100 Subject: [PATCH 0960/1579] drm/apple: admit external AFKV2 EDID services asynchronously External H17P service announcements can arrive after the AFK endpoint starts. Waiting synchronously for the EDID service can time out just before exact admission completes, leaving a misleading startup failure despite a usable service and delaying other endpoint initialization. After a successful afk_start, return for the external AFKV2 profile and let the existing guarded admission and late connector notification publish the service. Do not claim a missing service is enabled. Retain the legacy completion wait, timeout behavior and AFK-start failure cleanup unchanged. Add host tests executing actual endpoint startup, admission, notification and retirement for early, late, permanently missing and wrong-identity announcements, plus legacy waits and endpoint-start failure. No protocol command, service identity, EDID validation, generation or lifetime fence is changed. Persistent Offline replies after physical reconnect remain separate session/attachment qualification work. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/epic/dpavservep.c | 4 + .../test-apple-dpav-announcement.py | 138 ++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 tools/diagnostics/test-apple-dpav-announcement.py diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 0f00f80da716e..5a7b402492b12 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -421,6 +421,10 @@ int dpavservep_init(struct apple_dcp *dcp) return ret; } + /* Admission may arrive after startup; exact guards and late notify own it. */ + if (ep->wire_v2) + return 0; + ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, msecs_to_jiffies(1000)); if (!ret) { diff --git a/tools/diagnostics/test-apple-dpav-announcement.py b/tools/diagnostics/test-apple-dpav-announcement.py new file mode 100644 index 0000000000000..292f9f3f12f65 --- /dev/null +++ b/tools/diagnostics/test-apple-dpav-announcement.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Execute actual endpoint startup and early/late/missing DPAV admission paths.""" +from pathlib import Path +import os +import shlex +import subprocess +import tempfile + +root=Path(__file__).resolve().parents[2] +source=(root/'drivers/gpu/drm/apple/epic/dpavservep.c').read_text() +def function(signature): + start=source.index(signature);opening=source.index('{',start) + end,depth=opening+1,1 + while depth: + depth+=(source[end]=='{')-(source[end]=='}');end+=1 + return source[start:end] + +prefix=r''' +#undef NDEBUG +#include +#include +#include +#include +#include +#include +#include +typedef int64_t s64;typedef uint32_t u32;typedef uint64_t u64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define IS_ERR(p) ((uintptr_t)(p)>(uintptr_t)-4096) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL 2 +#define DPAVSERV_ENDPOINT 0x28 +#define DPAVSERV_ENDPOINT_V2 0x2a +#define DRM_CONNECTOR_REGISTERED 1 +#define to_platform_device(p) (p) +#define trace_dcpavserv_init(...) ((void)0) +#define dev_info(...) ((void)0) +#define dev_err(...) ((void)0) +static unsigned int warnings; +#define dev_warn(...) (warnings++) +static void mutex_lock(bool *p) { assert(!*p);*p=true; } +static void mutex_unlock(bool *p) { assert(*p);*p=false; } +#define lockdep_assert_held(p) assert(*(p)) +static unsigned int waits,snapshots,shutdowns,opens; +static void init_completion(bool *p) { *p=false; } +static void reinit_completion(bool *p) { *p=false; } +static void complete(bool *p) { *p=true; } +static unsigned int msecs_to_jiffies(unsigned int n) { return n; } +static int wait_for_completion_timeout(bool *p,unsigned int timeout) +{ assert(timeout==1000);waits++;return *p; } +static void schedule_work(unsigned int *p) { (*p)++; } +struct apple_dcp; +struct apple_dcp_afkep { struct apple_dcp *dcp; u32 endpoint; bool wire_v2,inline_v2,match_epic_name; }; +struct apple_epic_service { struct apple_dcp_afkep *ep; bool enabled; unsigned int users,channel; void *cookie; }; +struct apple_connector { void *dcp; bool connected,edid_invalidation_pending; struct { int registration_state; } base; unsigned int hotplug_wq; }; +struct apple_dcp { + void *dev; struct { int iomfb_method_profile; } hw; + bool quiescing; unsigned int nr_modes; + struct apple_connector *connector; + struct apple_dcp_afkep *dcpavservep; + struct { bool lock,enabled,enable_completion; u64 epoch; struct apple_epic_service *service; } dcpavserv; +}; +static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) +{ if(!s || !s->enabled) return NULL; s->users++;return s; } +static void afk_service_put(struct apple_epic_service *s) { assert(s->users);s->users--; } +static void afk_service_disable(struct apple_epic_service *s) { s->enabled=false; } +static const int dpavservep_ops[1]={0}; +static struct apple_dcp_afkep ep; +static struct apple_epic_service service; +static int start_error,announce_early; +static bool announce(struct apple_dcp *d,const char *name,const char *class,s64 unit); +static struct apple_dcp_afkep *afk_init(struct apple_dcp *d,u32 endpoint,const void *ops) +{ assert(ops==dpavservep_ops);ep=(struct apple_dcp_afkep){.dcp=d,.endpoint=endpoint};return &ep; } +static int afk_start(struct apple_dcp_afkep *e) +{ assert(e==&ep && e->dcp->dcpavservep==e);if(announce_early) assert(announce(e->dcp,"dcpav-service-epic","AppleDCPDP2HDMI",e->wire_v2?1:0));return start_error; } +static void afk_shutdown(struct apple_dcp_afkep *e) { assert(e==&ep);shutdowns++; } +static void afk_trace_ring_snapshot(struct apple_dcp_afkep *e) { assert(e==&ep);snapshots++; } +''' +parts=[function('static void dcpavserv_notify_edid('),function('static int dcpavserv_prepare('), + function('static void dcpavserv_init('),function('static void dcpavserv_invalidate_edid('), + function('static void dcpavserv_teardown('),function('void dpavservep_detach('), + function('int dpavservep_init(')] +suffix=r''' +static bool announce(struct apple_dcp *d,const char *name,const char *class,s64 unit) +{ + assert(!d->dcpavserv.lock);service=(struct apple_epic_service){.ep=&ep,.channel=1}; + if(dcpavserv_prepare(&service,name,class,unit)) return false; + if(ep.wire_v2) opens++; /* Standard OPEN after exact prepare, before publish. */ + service.enabled=true;dcpavserv_init(&service,name,class,unit); + return d->dcpavserv.enabled; +} +static struct apple_dcp d; +static struct apple_connector c; +static int device; +static void initial(bool v2) +{ + if(service.users) dcpavserv_teardown(&service); + assert(!service.users); + memset(&d,0,sizeof(d));memset(&c,0,sizeof(c));memset(&service,0,sizeof(service)); + d.dev=&device;d.connector=&c;d.nr_modes=1; + d.hw.iomfb_method_profile=v2?2:0;c.dcp=&device;c.connected=true;c.base.registration_state=1; + waits=snapshots=warnings=shutdowns=opens=0;start_error=announce_early=0; +} +int main(void) +{ + initial(true);announce_early=1; + assert(dpavservep_init(&d)==0 && !waits && !warnings && !snapshots); + assert(d.dcpavserv.enabled && service.users==1 && d.dcpavserv.epoch==1 && opens==1 && c.hotplug_wq==1); + assert(ep.endpoint==0x2a && ep.wire_v2 && !ep.inline_v2); + initial(true); /* Late announcement after successful nonblocking startup. */ + assert(dpavservep_init(&d)==0 && !waits && !d.dcpavserv.enabled && !opens); + assert(announce(&d,"dcpav-service-epic","AppleDCPDP2HDMI",1)); + assert(service.users==1 && d.dcpavserv.enable_completion && c.hotplug_wq==1 && opens==1); + initial(true); /* Permanently missing service: no fake admission or timeout. */ + assert(dpavservep_init(&d)==0 && !waits && !warnings && !d.dcpavserv.enabled && !d.dcpavserv.service && !c.hotplug_wq); + assert(!announce(&d,"dcpav-service-epic","wrong",1)); + assert(!announce(&d,"dcpav-service-epic","AppleDCPDP2HDMI",0)); + assert(!announce(&d,"dcpdp-service-epic","AppleDCPDP2HDMI",3)); + assert(!d.dcpavserv.enabled && !service.users && !opens); + assert(announce(&d,"dcpav-service-epic","AppleDCPDP2HDMI",1)); + assert(opens==1 && d.dcpavserv.enabled); + initial(false);announce_early=1; + assert(dpavservep_init(&d)==0 && waits==1 && !warnings && !opens && ep.endpoint==0x28); + assert(service.users==1 && d.dcpavserv.enabled && !c.hotplug_wq); + initial(false); + assert(dpavservep_init(&d)==-ETIMEDOUT && waits==1 && warnings==1 && snapshots==1 && !d.dcpavserv.enabled); + initial(true);start_error=-EIO; + assert(dpavservep_init(&d)==-EIO && shutdowns==1 && !d.dcpavservep && !waits && !service.users); + initial(true); puts("PASS: actual DPAV init/admission/notify early, late, missing, wrong identity, legacy waits and AFK startup failure"); +} +''' +with tempfile.TemporaryDirectory(prefix='dpav-announcement-') as tmp: + p=Path(tmp)/'test.c';p.write_text(prefix+'\n'.join(parts)+suffix) + cc=shlex.split(os.environ.get('HOSTCC','clang')) + subprocess.run(cc+['-std=gnu11','-Wall','-Werror','-O1','-g','-fsanitize=address,undefined','-fno-omit-frame-pointer',str(p),'-o',str(Path(tmp)/'test')],check=True) + subprocess.run([str(Path(tmp)/'test')],check=True) From d2635e490a611e03ec6dd68ecf79814f374ac4f4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 15:15:51 +1100 Subject: [PATCH 0961/1579] drm/apple: replace H17P AV bindings across physical reconnect H17P parse_epic_service_init maps interface-id into the variable named unit. That value is a dynamic AFKV2 interface identity, not a physical-port number. Requiring unit 1 rejects fresh AV announcements after reconnect while retaining the old offline interface. Require the exact DPAV endpoint/compartment/class and a nonzero bounded interface-id equal to the validated service channel. Preserve legacy EPICUnit 0 admission. Retire the current AV registry binding on physical disconnect, current-generation deferred disconnect, route withdrawal and cable-release teardown, outside hpd_mutex. Ordinary DPMS poweroff does not withdraw it. Carry physical-generation and registry-epoch tickets from prepare through OPEN/init so a disconnect or retirement during admission cannot publish stale work. Local retirement disables the service and marks its slot torn down; reader pins and outstanding commands still prevent reuse, and timed-out DMA remains owned until a matching late ACK. Old CLOSE/ACK handling cannot revoke a fresh binding. Recognize the observed DPAV CLOSED report (0x14, zero token, low flags byte 1, one zero payload byte) only on the instantiated DPAV endpoint. Keep the existing 0x13 report handling and other endpoints unchanged. No new host opcode or reopen of an offline interface is introduced. Host regressions execute the production parser, ownership helpers, prepare/ init, retirement, receive dispatch and late-ACK completer for dynamic interfaces 1/5/9/13, forged identities, independent generation/epoch changes, old pins/CLOSED/replies and legacy unit 0. Full AFKV2 ASan/UBSan suite, startup, EDID lifetime, external lifecycle, metadata and RX-progress tests pass. Targeted ARM64 W=1 module compile/modpost/link passes without warnings. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 3 +- drivers/gpu/drm/apple/afk.h | 13 ++ drivers/gpu/drm/apple/afkv2-codec.h | 10 + drivers/gpu/drm/apple/dcp.c | 8 + drivers/gpu/drm/apple/epic/dpavservep.c | 101 +++++--- .../test-apple-dpav-announcement.py | 12 +- tools/diagnostics/test-apple-dpav-rebind.py | 216 ++++++++++++++++++ .../test-apple-external-lifecycle.py | 11 + tools/diagnostics/test-apple-h17p-edid.py | 14 +- 9 files changed, 345 insertions(+), 43 deletions(-) create mode 100644 tools/diagnostics/test-apple-dpav-rebind.py diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index e182105c47faa..50f6e796d7908 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -884,7 +884,8 @@ static void afk_v2_receive(struct apple_dcp_afkep *ep, u32 channel, u32 type, if (frame.type == EPIC_SUBTYPE_ANNOUNCE_H17P) afk_recv_handle_init(ep, frame.interface, (void *)frame.payload, frame.payload_size); - else if (frame.type == AFKV2_CLOSE && frame.interface_flags == 1) + else if ((frame.type == AFKV2_CLOSE && frame.interface_flags == 1) || + (ep == ep->dcp->dcpavservep && afkv2_closed_report(&frame))) afk_recv_handle_teardown(ep, frame.interface); return; } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 36b07a556f6d4..686b5e1881779 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -56,6 +56,8 @@ struct apple_epic_service { bool enabled; bool torndown; + /* Provider-owned prepare/init ticket; AFK core never interprets it. */ + struct { u64 generation, epoch; } admission; void *cookie; struct { @@ -148,6 +150,17 @@ static inline void afk_service_disable(struct apple_epic_service *service) spin_unlock_irqrestore(&service->lock, flags); } +/* Retire local ownership while keeping timed-out commands discoverable. */ +static inline void afk_service_retire(struct apple_epic_service *service) +{ + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + service->torndown = true; + spin_unlock_irqrestore(&service->lock, flags); +} + static inline bool afk_service_matches(struct apple_epic_service *service, u32 channel) { diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h index a06a1efd5fc7e..c42788a1b2a74 100644 --- a/drivers/gpu/drm/apple/afkv2-codec.h +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -17,6 +17,7 @@ #define AFKV2_RESPONSE 2U #define AFKV2_OPEN 0x12U #define AFKV2_CLOSE 0x13U +#define AFKV2_CLOSED 0x14U struct afkv2_frame { u8 sequence; @@ -113,6 +114,15 @@ static inline int afkv2_decode_frame(const void *record, size_t size, return 0; } +/* Closed notification: flags occupy the low byte; the high byte is opaque. */ +static inline bool afkv2_closed_report(const struct afkv2_frame *frame) +{ + return frame && !frame->token && frame->category == AFKV2_REPORT && + frame->type == AFKV2_CLOSED && + (frame->interface_flags & 0xff) == 1 && frame->payload_size == 1 && + frame->payload && !frame->payload[0]; +} + static inline int afkv2_encode_frame(void *record, size_t capacity, u8 sequence, u16 interface, u64 token, u8 type, u8 category, u16 flags, const void *payload, size_t payload_size, diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 20115ce416842..1b1d1d18a78e2 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -495,6 +495,8 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) if (sel && sel != -EOPNOTSUPP) dev_warn(dcp->dev, "DP tunnel source reset failed: %d\n", sel); } + if (dcp_is_external_h17p(dcp)) + dpavservep_detach(dcp); route->selected = false; if (dcp->active_typec_route == route) dcp->active_typec_route = NULL; @@ -2625,6 +2627,8 @@ static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } + /* HPD is fenced; retire the AV owner after releasing hpd_mutex. */ + dpavservep_detach(dcp); } ret = dcp_external_stop_locked(dcp, dcp->external_generation); if (physical) { @@ -2649,6 +2653,8 @@ static void dcp_external_stop_work(struct work_struct *work) cancel_delayed_work_sync(&dcp->placeholder_edid_wq); guard(mutex)(&dcp->external_lock); if (generation == dcp->external_generation) { + /* The deferred disconnect also covers missing out-of-band HPD. */ + dpavservep_detach(dcp); dcp_external_stop_locked(dcp, generation); dcp_external_report_disconnected(dcp); } @@ -3839,6 +3845,8 @@ static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } + if (release_cable && dcp_is_external_h17p(dcp)) + dpavservep_detach(dcp); /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); disable_delayed_work_sync(&dcp->placeholder_edid_wq); diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 5a7b402492b12..229680c6c265c 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -32,25 +32,51 @@ static void dcpavserv_notify_edid(struct apple_epic_service *service) schedule_work(&connector->hotplug_wq); } +static bool dcpavserv_identity(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + if (!name || strcmp(name, "dcpav-service-epic")) + return false; + if (!service->ep->wire_v2) + return unit == 0; + /* H17P's lower-case interface-id is dynamic, not a physical-port unit. */ + return service->ep->endpoint == DPAVSERV_ENDPOINT_V2 && class && + !strcmp(class, "AppleDCPDP2HDMI") && service->channel && + service->channel <= U16_MAX && unit == service->channel; +} + +static bool dcpavserv_admission_current(struct apple_dcp *dcp, u64 generation) +{ + lockdep_assert_held(&dcp->hpd_mutex); + /* Initial service discovery may precede the first Type-C attachment. */ + return !READ_ONCE(dcp->quiescing) && generation == dcp->typec_generation && + (!generation || dcp->typec_cable_connected); +} + /* Admission runs before AFKV2 OPEN and before readers can pin the service. */ static int dcpavserv_prepare(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; - int ret; + bool v2 = service->ep->wire_v2; + int ret = 0; - if (!name || strcmp(name, "dcpav-service-epic")) - return -ENODEV; - if (service->ep->wire_v2) { - if (service->ep->endpoint != DPAVSERV_ENDPOINT_V2 || !class || - strcmp(class, "AppleDCPDP2HDMI") || unit != 1) - return -ENODEV; - } else if (unit != 0) { + if (!dcpavserv_identity(service, name, class, unit)) return -ENODEV; - } + if (v2) + mutex_lock(&dcp->hpd_mutex); mutex_lock(&dcp->dcpavserv.lock); - ret = dcp->dcpavserv.enabled ? -EBUSY : 0; + if (dcp->dcpavserv.enabled) + ret = -EBUSY; + else if (v2 && !dcpavserv_admission_current(dcp, dcp->typec_generation)) + ret = -ENODEV; + else { + service->admission.generation = v2 ? dcp->typec_generation : 0; + service->admission.epoch = dcp->dcpavserv.epoch; + } mutex_unlock(&dcp->dcpavserv.lock); + if (v2) + mutex_unlock(&dcp->hpd_mutex); return ret; } @@ -58,29 +84,39 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; + bool v2 = service->ep->wire_v2; + bool published = false; trace_dcpavserv_init(dcp, unit); - - if (!dcpavserv_prepare(service, name, class, unit)) { - mutex_lock(&dcp->dcpavserv.lock); - if (dcp->dcpavserv.enabled) { - mutex_unlock(&dcp->dcpavserv.lock); - dev_err(dcp->dev, - "DCPAVSERV: unit %lld already exists\n", unit); - return; - } + if (v2) + mutex_lock(&dcp->hpd_mutex); + mutex_lock(&dcp->dcpavserv.lock); + if (dcpavserv_identity(service, name, class, unit) && + !dcp->dcpavserv.enabled && + service->admission.epoch == dcp->dcpavserv.epoch && + (!v2 || dcpavserv_admission_current(dcp, service->admission.generation))) { dcp->dcpavserv.service = afk_service_get(service); - dcp->dcpavserv.enabled = true; - dcp->dcpavserv.epoch++; - service->cookie = &dcp->dcpavserv; - complete(&dcp->dcpavserv.enable_completion); - mutex_unlock(&dcp->dcpavserv.lock); - if (service->ep->wire_v2) { - dev_info(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", - service->channel); - dcpavserv_notify_edid(service); + if (dcp->dcpavserv.service) { + dcp->dcpavserv.enabled = true; + dcp->dcpavserv.epoch++; + service->cookie = &dcp->dcpavserv; + complete(&dcp->dcpavserv.enable_completion); + published = true; } } + mutex_unlock(&dcp->dcpavserv.lock); + if (v2) + mutex_unlock(&dcp->hpd_mutex); + if (!published) { + /* A generation/registry change during OPEN cannot enable stale work. */ + afk_service_retire(service); + return; + } + if (v2) { + dev_info(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", + service->channel); + dcpavserv_notify_edid(service); + } } /* Defer DRM cleanup: AFK callbacks must not take mode_config.mutex. */ @@ -124,15 +160,16 @@ void dpavservep_detach(struct apple_dcp *dcp) service = dcp->dcpavserv.service; dcp->dcpavserv.service = NULL; dcp->dcpavserv.enabled = false; + /* Also reject an in-flight prepare/OPEN with no published owner yet. */ + dcp->dcpavserv.epoch++; + reinit_completion(&dcp->dcpavserv.enable_completion); if (service) { - dcp->dcpavserv.epoch++; + afk_service_retire(service); dcpavserv_invalidate_edid(dcp); } mutex_unlock(&dcp->dcpavserv.lock); - if (service) { - afk_service_disable(service); + if (service) afk_service_put(service); - } } static int dcpdpserv_prepare(struct apple_epic_service *service, const char *name, diff --git a/tools/diagnostics/test-apple-dpav-announcement.py b/tools/diagnostics/test-apple-dpav-announcement.py index 292f9f3f12f65..256d1b2eb8d88 100644 --- a/tools/diagnostics/test-apple-dpav-announcement.py +++ b/tools/diagnostics/test-apple-dpav-announcement.py @@ -26,6 +26,7 @@ def function(signature): #include #include typedef int64_t s64;typedef uint32_t u32;typedef uint64_t u64; +#define U16_MAX UINT16_MAX #define READ_ONCE(x) (x) #define WRITE_ONCE(x,v) ((x)=(v)) #define IS_ERR(p) ((uintptr_t)(p)>(uintptr_t)-4096) @@ -53,19 +54,19 @@ def function(signature): static void schedule_work(unsigned int *p) { (*p)++; } struct apple_dcp; struct apple_dcp_afkep { struct apple_dcp *dcp; u32 endpoint; bool wire_v2,inline_v2,match_epic_name; }; -struct apple_epic_service { struct apple_dcp_afkep *ep; bool enabled; unsigned int users,channel; void *cookie; }; +struct apple_epic_service { struct apple_dcp_afkep *ep; bool enabled,torndown; struct { u64 generation,epoch; } admission; unsigned int users,channel; void *cookie; }; struct apple_connector { void *dcp; bool connected,edid_invalidation_pending; struct { int registration_state; } base; unsigned int hotplug_wq; }; struct apple_dcp { void *dev; struct { int iomfb_method_profile; } hw; - bool quiescing; unsigned int nr_modes; + bool quiescing,hpd_mutex,typec_cable_connected; u64 typec_generation; unsigned int nr_modes; struct apple_connector *connector; struct apple_dcp_afkep *dcpavservep; struct { bool lock,enabled,enable_completion; u64 epoch; struct apple_epic_service *service; } dcpavserv; }; static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) -{ if(!s || !s->enabled) return NULL; s->users++;return s; } +{ if(!s || !s->enabled || s->torndown) return NULL; s->users++;return s; } static void afk_service_put(struct apple_epic_service *s) { assert(s->users);s->users--; } -static void afk_service_disable(struct apple_epic_service *s) { s->enabled=false; } +static void afk_service_retire(struct apple_epic_service *s) { s->enabled=false;s->torndown=true; } static const int dpavservep_ops[1]={0}; static struct apple_dcp_afkep ep; static struct apple_epic_service service; @@ -78,7 +79,8 @@ def function(signature): static void afk_shutdown(struct apple_dcp_afkep *e) { assert(e==&ep);shutdowns++; } static void afk_trace_ring_snapshot(struct apple_dcp_afkep *e) { assert(e==&ep);snapshots++; } ''' -parts=[function('static void dcpavserv_notify_edid('),function('static int dcpavserv_prepare('), +parts=[function('static void dcpavserv_notify_edid('),function('static bool dcpavserv_identity('), + function('static bool dcpavserv_admission_current('),function('static int dcpavserv_prepare('), function('static void dcpavserv_init('),function('static void dcpavserv_invalidate_edid('), function('static void dcpavserv_teardown('),function('void dpavservep_detach('), function('int dpavservep_init(')] diff --git a/tools/diagnostics/test-apple-dpav-rebind.py b/tools/diagnostics/test-apple-dpav-rebind.py new file mode 100644 index 0000000000000..278b2e52c88b5 --- /dev/null +++ b/tools/diagnostics/test-apple-dpav-rebind.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run actual AFK ownership/receive and DPAV admission against physical epochs.""" +from pathlib import Path +import os,re,shlex,subprocess,tempfile +root=Path(__file__).resolve().parents[2];base=root/'drivers/gpu/drm/apple' +def function(file,signature): + text=(base/file).read_text();start=text.index(signature);opening=text.index('{',start) + while ';' in text[start:opening]: + start=text.index(signature,start+1);opening=text.index('{',start) + end,depth=opening+1,1 + while depth: + depth+=(text[end]=='{')-(text[end]=='}');end+=1 + return text[start:end] +prefix=r''' +#undef NDEBUG +#include +#include +#include +#include +#include "afkv2-codec.h" +typedef int64_t s64; +typedef u64 dma_addr_t; +#define U16_MAX UINT16_MAX +#define EBUSY 16 +#define ENOENT 2 +#define MAX_PENDING_CMDS 16 +#define AFK_MAX_CHANNEL 16 +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define WARN_ON(x) (x) +#define bitmap_empty(p,n) (!*(p)) +#define spin_lock_irqsave(p,f) do { assert(!*(p));*(p)=true;(f)=0; } while(0) +#define spin_unlock_irqrestore(p,f) do { (void)(f);assert(*(p));*(p)=false; } while(0) +static int test_bit(unsigned int bit,const unsigned long *p) { return !!(*p & (1UL<(uintptr_t)-4096) +#define lockdep_assert_held(p) assert(*(p)) +static bool *hpd_lock,*av_lock; +static void mutex_lock(bool *p) { if(p==hpd_lock) assert(!*av_lock);assert(!*p);*p=true; } +static void mutex_unlock(bool *p) { assert(*p);*p=false; } +static void complete(int *p) { (*p)++; } +static void reinit_completion(int *p) { *p=0; } +static void schedule_work(unsigned int *p) { (*p)++; } +struct apple_dcp;struct apple_dcp_afkep;struct apple_epic_service; +struct apple_epic_service_ops { bool reusable;int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; +struct epic_cmd_info { u16 tag;u8 type;size_t reply_length;void *rxbuf,*txbuf;dma_addr_t rxbuf_dma,txbuf_dma;size_t rxlen,txlen;u32 retcode;bool done,free_on_ack;int *completion; }; +struct apple_epic_service { + struct apple_dcp_afkep *ep;const struct apple_epic_service_ops *ops; + bool lock,enabled,torndown;unsigned int users;u32 channel;u8 cmd_tag; + unsigned long cmd_map[1];struct epic_cmd_info cmds[MAX_PENDING_CMDS]; + struct { u64 generation,epoch; } admission;void *cookie; +}; +struct apple_dcp_afkep { struct apple_dcp *dcp;bool wire_v2,inline_v2;u32 endpoint;unsigned int num_channels;struct apple_epic_service services[AFK_MAX_CHANNEL]; }; +struct apple_connector { void *dcp;bool connected,edid_invalidation_pending;struct { int registration_state; } base;unsigned int hotplug_wq; }; +struct apple_dcp { + void *dev;struct apple_dcp_afkep *dcpavservep;struct apple_connector *connector; + bool hpd_mutex,quiescing,typec_cable_connected;u64 typec_generation;unsigned int nr_modes; + struct { int iomfb_method_profile; } hw; + struct { bool lock,enabled;u64 epoch;int enable_completion;struct apple_epic_service *service; } dcpavserv; +}; +struct epic_cmd { u32 retcode;u64 rxbuf,txbuf;u32 rxlen,txlen; }; +static unsigned int dma_frees,announces; +static void dma_free_coherent(void *dev,size_t size,void *p,dma_addr_t address) +{ (void)dev;assert(size && (uintptr_t)p==address);free(p);dma_frees++; } +static void afk_remove_service_debugfs(struct apple_epic_service *s) { (void)s; } +static void afk_recv_handle_init(struct apple_dcp_afkep *e,u32 channel,u8 *p,size_t n) { (void)e;(void)channel;(void)p;(void)n;announces++; } +static void afk_v2_callback(struct apple_dcp_afkep *e,const struct afkv2_frame *f,const struct afkv2_command *c) { (void)e;(void)f;(void)c;assert(!"unexpected callback"); } + +struct item { const char *key,*value; s64 integer; }; +struct dcp_parse_ctx { struct item *items;unsigned int count,index;bool key; }; +struct iterator { struct dcp_parse_ctx *handle;unsigned int idx,len; }; +#define kfree(p) free((void *)(p)) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +static int iterator_begin(struct dcp_parse_ctx *h,struct iterator *it,bool dict) +{ assert(dict);h->index=0;h->key=true;*it=(struct iterator){.handle=h,.len=h->count};return 0; } +static char *parse_string(struct dcp_parse_ctx *h) +{ + struct item *i=&h->items[h->index];const char *p; + if(h->key) { p=i->key;h->key=false; } else { p=i->value;h->index++;h->key=true; } + if(!p) return ERR_PTR(-EINVAL);char *copy=malloc(strlen(p)+1);assert(copy);strcpy(copy,p);return copy; +} +static int parse_int(struct dcp_parse_ctx *h,s64 *out) +{ assert(!h->key);*out=h->items[h->index++].integer;h->key=true;return 0; } +static int skip(struct dcp_parse_ctx *h) { assert(!h->key);h->index++;h->key=true;return 0; } +''' +parts=[function('parser.c','int parse_epic_service_init('),function('afk.h','static inline struct apple_epic_service *afk_service_get('), + function('afk.h','static inline void afk_service_put('),function('afk.h','static inline void afk_service_retire('), + function('afk.h','static inline bool afk_service_matches('),function('afk.h','static inline bool afk_service_reclaimable('), + function('afk.h','static inline bool\nafk_service_reinit('),function('afk.h','static inline void afk_service_discard('), + function('afk.h','static inline void afk_service_publish('), + function('epic/dpavservep.c','static void dcpavserv_notify_edid('),function('epic/dpavservep.c','static bool dcpavserv_identity('), + function('epic/dpavservep.c','static bool dcpavserv_admission_current('),function('epic/dpavservep.c','static int dcpavserv_prepare('), + function('epic/dpavservep.c','static void dcpavserv_init('),function('epic/dpavservep.c','static void dcpavserv_invalidate_edid('), + function('epic/dpavservep.c','static void dcpavserv_teardown('),function('epic/dpavservep.c','void dpavservep_detach('), + function('afk.c','static struct apple_epic_service *afk_epic_find_service('),function('afk.c','static void afk_recv_handle_teardown('), + function('afk.c','static void afk_recv_handle_reply('),function('afk.c','static void afk_v2_receive(')] +suffix=r''' +static struct apple_dcp d; +static struct apple_dcp_afkep ep; +static struct apple_connector con; +static const struct apple_epic_service_ops ops={.reusable=true,.prepare=dcpavserv_prepare,.teardown=dcpavserv_teardown}; +static const char *name="dcpav-service-epic",*class="AppleDCPDP2HDMI"; +static struct apple_epic_service *prepare(u32 channel,s64 unit,const char *n,const char *c) +{ + for(unsigned int i=0;iusers==1); + assert(afk_service_get(old)==old); /* Delayed EDID reader pins the retired slot. */ + old->cmds[0]=(struct epic_cmd_info){.tag=0,.type=0xc0,.rxlen=32,.txlen=32,.free_on_ack=true}; + old->cmds[0].rxbuf=calloc(1,32);old->cmds[0].txbuf=calloc(1,32);assert(old->cmds[0].rxbuf && old->cmds[0].txbuf); + old->cmds[0].rxbuf_dma=(uintptr_t)old->cmds[0].rxbuf;old->cmds[0].txbuf_dma=(uintptr_t)old->cmds[0].txbuf;old->cmd_map[0]=1; + disconnect();assert(!d.dcpavserv.service && !d.dcpavserv.enabled && old->users==1 && old->torndown && !old->enabled); + assert(con.edid_invalidation_pending && !afk_service_get(old)); + assert(!afk_service_reinit(old,&ep,&ops,5)); + assert(!prepare(5,5,name,class)); /* No admission while physically disconnected. */ + connect();struct apple_epic_service *fresh=prepare(5,5,name,class);assert(fresh && fresh!=old);publish(fresh,5); + assert(d.dcpavserv.service==fresh && fresh->users==1); + u64 epoch=d.dcpavserv.epoch; + dcpavserv_teardown(old);assert(d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch); + closed(1,0x6a01,0);assert(d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch && !dma_frees); + u8 oob[32],reply[56];size_t written; + assert(!afkv2_encode_oob_command(oob,sizeof(oob),0,old->cmds[0].rxbuf_dma,32,old->cmds[0].txbuf_dma,32));afkv2_put32(oob+4,0); + assert(!afkv2_encode_frame(reply,sizeof(reply),1,1,0,0xc0,AFKV2_RESPONSE,0,oob,sizeof(oob),&written)); + afk_v2_receive(&ep,0,0,reply,written); + assert(dma_frees==2 && !old->cmd_map[0] && old->users==1 && d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch); + assert(!afk_service_reinit(old,&ep,&ops,9));afk_service_put(old); + afk_v2_receive(&ep,0,0,reply,written);assert(dma_frees==2); /* Duplicate late ACK. */ + disconnect();connect();struct apple_epic_service *next=prepare(9,9,name,class);assert(next);publish(next,9); + assert(d.dcpavserv.service==next); + closed(1,0x6a01,0);closed(5,0x6a01,0);assert(d.dcpavserv.service==next); + disconnect();connect();next=prepare(13,13,name,class);assert(next);publish(next,13);assert(d.dcpavserv.service==next); + closed(13,0x6a01,1);assert(d.dcpavserv.service==next); /* Nonzero result rejected. */ + closed(13,0x6a00,0);assert(d.dcpavserv.service==next); /* Wrong low flags rejected. */ + d.dcpavservep=NULL;closed(13,0x6a01,0);assert(d.dcpavserv.service==next);d.dcpavservep=&ep; + closed(13,0x6a01,0);assert(!d.dcpavserv.service && !d.dcpavserv.enabled); + u8 result_byte=0;struct afkv2_frame notification={.category=AFKV2_REPORT,.type=AFKV2_CLOSED,.interface_flags=1,.payload=&result_byte,.payload_size=1}; + assert(afkv2_closed_report(¬ification));notification.token=1;assert(!afkv2_closed_report(¬ification)); + notification.token=0;notification.payload_size=0;assert(!afkv2_closed_report(¬ification)); + assert(!prepare(0,0,name,class));assert(!prepare(65536,65536,name,class)); + assert(!prepare(9,1,name,class));assert(!prepare(9,9,"wrong",class));assert(!prepare(9,9,name,"wrong")); + struct apple_epic_service *pending=prepare(9,9,name,class);assert(pending); + disconnect();connect();publish(pending,9); + assert(!d.dcpavserv.service && pending->torndown && !pending->enabled && !pending->users); + pending=prepare(9,9,name,class);assert(pending); + mutex_lock(&d.hpd_mutex);d.typec_generation++;mutex_unlock(&d.hpd_mutex); + publish(pending,9); /* Generation change alone, without registry retirement. */ + assert(!d.dcpavserv.service && pending->torndown && !pending->enabled); + pending=prepare(13,13,name,class);assert(pending); + dpavservep_detach(&d);publish(pending,13); /* Registry retirement with no physical change. */ + assert(!d.dcpavserv.service && pending->torndown && !pending->enabled); + ep.wire_v2=false;ep.endpoint=0x28;d.hw.iomfb_method_profile=0; + assert(!prepare(7,7,name,class));pending=prepare(7,0,name,NULL);assert(pending);publish(pending,0); + assert(d.dcpavserv.service==pending && pending->users==1);dpavservep_detach(&d); + assert(!pending->users && !d.hpd_mutex && !d.dcpavserv.lock); + puts("PASS: actual dynamic 1/5/9/13 binding, forged identities, prepare/init epochs, physical retirement, reader pins, real old CLOSED/late ACK isolation and legacy unit 0"); +} +''' +with tempfile.TemporaryDirectory(prefix='dpav-rebind-') as tmp: + p=Path(tmp)/'test.c';p.write_text(prefix+'\n'.join(parts)+suffix) + cc=shlex.split(os.environ.get('HOSTCC','clang')) + subprocess.run(cc+['-std=gnu11','-Wall','-Werror','-O1','-g','-fsanitize=address,undefined','-fno-omit-frame-pointer', + '-I'+str(root/'tools/testing/selftests/drivers/gpu/apple-afkv2/host-include'), + '-I'+str(base),str(p),'-o',str(Path(tmp)/'test')],check=True) + subprocess.run([str(Path(tmp)/'test')],check=True) diff --git a/tools/diagnostics/test-apple-external-lifecycle.py b/tools/diagnostics/test-apple-external-lifecycle.py index de81ae88691cd..60d3d61deacd3 100644 --- a/tools/diagnostics/test-apple-external-lifecycle.py +++ b/tools/diagnostics/test-apple-external-lifecycle.py @@ -218,6 +218,8 @@ def external_branch(name, result): static int dptxport_release_display(void *s) { (void)s; event('R'); releases++; return release_error; } +static unsigned int av_retirements; +static void dpavservep_detach(struct apple_dcp *p) { assert(!p->hpd_mutex.held);av_retirements++; } static void av_service_disconnect(struct apple_dcp *p) { (void)p; } static int dcp_dptx_connect_raw(struct apple_dcp *p, u32 port, bool *waiting_service) { @@ -569,6 +571,15 @@ def external_branch(name, result): assert(p.external_error == -EPROTO && !power1 && p.iomfb.presentation_paused); owned(&p); } + /* AV retirement is physical/fallback disconnect, not ordinary DPMS. */ + initial(&p); unsigned int av_before=av_retirements; + assert(dcp_external_disconnect(&p, false)==0 && av_retirements==av_before); + initial(&p); p.external_error=-EIO; av_before=av_retirements; + assert(dcp_external_disconnect(&p, true)==-EIO && av_retirements==av_before+1); + initial(&p); p.external_stop_generation=p.external_generation-1; av_before=av_retirements; + dcp_external_stop_work(&p.external_stop_work); assert(av_retirements==av_before); + p.external_stop_generation=p.external_generation; + dcp_external_stop_work(&p.external_stop_work); assert(av_retirements==av_before+1); puts("PASS: strict drain, callbacks, admission, exact RPC, stop/restart, stale epochs, modeset gate, retained ownership"); return 0; } diff --git a/tools/diagnostics/test-apple-h17p-edid.py b/tools/diagnostics/test-apple-h17p-edid.py index 75a1ef8275b22..57cac9f7e1635 100644 --- a/tools/diagnostics/test-apple-h17p-edid.py +++ b/tools/diagnostics/test-apple-h17p-edid.py @@ -41,6 +41,7 @@ def function(file, signature): typedef uint8_t u8; typedef uint64_t u64; typedef int64_t s64; +#define U16_MAX UINT16_MAX #define READ_ONCE(x) (x) #define WRITE_ONCE(x,v) ((x)=(v)) #define ERR_PTR(x) ((void *)(intptr_t)(x)) @@ -80,7 +81,7 @@ def function(file, signature): struct apple_dcp; struct apple_connector { void *dcp; struct { int registration_state; } base; bool connected, edid_invalidation_pending; int hotplug_wq; }; struct apple_dcp_afkep { struct apple_dcp *dcp; u8 endpoint; bool wire_v2; }; -struct apple_epic_service { struct apple_dcp_afkep *ep; unsigned int users; bool enabled; void *cookie; unsigned int channel; }; +struct apple_epic_service { struct apple_dcp_afkep *ep; unsigned int users; bool enabled,torndown; struct { u64 generation,epoch; } admission; void *cookie; unsigned int channel; }; struct apple_dcp { struct apple_dcp_afkep *dcpavservep, *dptxep, *avep, *systemep, *ibootep, *dptx_controller_ep; struct { bool lock, enabled; u64 epoch; int enable_completion; struct apple_epic_service *service; } dcpavserv; @@ -96,9 +97,10 @@ def function(file, signature): static unsigned int event; static struct apple_dcp_afkep *received_ep; static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) -{ if(!s || !s->enabled) return NULL; s->users++; return s; } +{ if(!s || !s->enabled || s->torndown) return NULL; s->users++; return s; } static void afk_service_put(struct apple_epic_service *s) { assert(s->users); s->users--; } +static void afk_service_retire(struct apple_epic_service *s) { s->enabled=false;s->torndown=true; } static void drm_edid_free(const struct drm_edid *e) { if(e) { assert(allocations); allocations--; free((void *)e); } } static void dcpavserv_teardown(struct apple_epic_service *s); @@ -122,6 +124,8 @@ def function(file, signature): function('epic/dpavservep.c', 'static void dcpavserv_teardown('), function('dcp.c', 'static struct apple_dcp_afkep *dcp_afkep('), function('dcp.c', 'static void dcp_recv_msg('), + function('epic/dpavservep.c', 'static bool dcpavserv_identity('), + function('epic/dpavservep.c', 'static bool dcpavserv_admission_current('), function('epic/dpavservep.c', 'static int dcpavserv_prepare('), function('epic/dpavservep.c', 'static int dcpdpserv_prepare('), function('epic/dpavservep.c', 'static void dcpavserv_init('), @@ -147,7 +151,7 @@ def function(file, signature): dcp_recv_msg(&d,0x2a,4); assert(received_ep==&av && receives==3); dcp_recv_msg(&d,0x2c,5); assert(received_ep==&tx && receives==4); d.dcpavservep=NULL; dcp_recv_msg(&d,0x2a,6); assert(receives==4); d.dcpavservep=&av; - struct apple_epic_service s={.ep=&av,.users=1,.enabled=true}; + struct apple_epic_service s={.ep=&av,.users=1,.enabled=true,.channel=1}; const char *name="dcpav-service-epic", *class="AppleDCPDP2HDMI"; assert(!dcpavserv_prepare(&s,name,class,1)); assert(dcpavserv_prepare(&s,name,class,0)==-ENODEV); @@ -175,14 +179,14 @@ def function(file, signature): assert(PTR_ERR(dcpavserv_copy_edid(&d,3,&epoch))==-ENODEV); assert(connector.edid_invalidation_pending && connector.hotplug_wq && d.dcpavserv.epoch==2); /* Same slot and physical generation: CLOSE/re-OPEN gets a fresh ticket. */ - s.enabled=true; dcpavserv_init(&s,name,class,1); + s.enabled=true; assert(!dcpavserv_prepare(&s,name,class,1)); dcpavserv_init(&s,name,class,1); assert(d.dcpavserv.service==&s && s.users==1 && d.dcpavserv.epoch==3); mutex_lock(&d.dcpavserv.lock); assert(!dcpavserv_edid_service_current(&d,1) && dcpavserv_edid_service_current(&d,3)); mutex_unlock(&d.dcpavserv.lock); event=0; e=dcpavserv_copy_edid(&d,3,&epoch); assert(!IS_ERR(e) && epoch==3); /* Simulate retirement in the final validation-to-publication gap. */ - s.enabled=false; dcpavserv_teardown(&s); s.enabled=true; dcpavserv_init(&s,name,class,1); + s.enabled=false; dcpavserv_teardown(&s); s.enabled=true; assert(!dcpavserv_prepare(&s,name,class,1)); dcpavserv_init(&s,name,class,1); mutex_lock(&d.dcpavserv.lock); assert(!dcpavserv_edid_service_current(&d,epoch)); mutex_unlock(&d.dcpavserv.lock); drm_edid_free(e); assert(d.typec_generation==3 && d.dcpavserv.epoch==5 && s.users==1); s.enabled=false; dcpavserv_teardown(&s); From d08fd52e161953a481a464716dd68aae85338044 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 13 Oct 2026 07:38:47 -0400 Subject: [PATCH 0962/1579] drm/apple: enable the validated J700 external display profile by default Use native PMGR bandwidth scratch and clock-request offsets from the T8140 hardware profile. Keep diagnostic overrides bounded and retain the disable switch and loader firmware validation. Put the existing tested external route and scanout window in the normal J700 tree so users need no manual command line or alternate DTB. --- .../boot/dts/apple/t8140-j700-dcpext.dtsi | 5 ++-- .../dts/apple/t8140-j700-external-test.dts | 28 +------------------ arch/arm64/boot/dts/apple/t8140-j700.dts | 27 +++++++++++++++++- drivers/gpu/drm/apple/dcp-external-bw.h | 15 ++++++++++ drivers/gpu/drm/apple/dcp.c | 23 +++++++++------ .../drivers/gpu/test_t8140_external_bw.py | 12 ++++++++ 6 files changed, 70 insertions(+), 40 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi index 6a6538dc6dba6..3b8be06ed22f5 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -4,9 +4,8 @@ * Generated-by: LLM * Assisted-by: LLM * - * The controller and route remain disabled. Their live firmware - * profile, bandwidth request, routing selector and handoff must be qualified - * before enabling the controller. The existing T8140 refusal remains in force. + * The controller remains disabled until the loader validates and preserves + * the inherited firmware and its complete dependency group. */ &{/soc} { diff --git a/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts b/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts index fbd8e83738f0c..5b9b3d78c8160 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700-external-test.dts @@ -1,29 +1,3 @@ // SPDX-License-Identifier: GPL-2.0+ OR MIT -/* Explicit external-display trial; the ordinary J700 tree is unchanged. */ +/* Compatibility alias: external display is now part of the ordinary J700 tree. */ #include "t8140-j700.dts" -#include - -&display { - iommus = <&disp0_dart 0>, <&dispext0_dart 0>; -}; - -&dcpext { - clocks = <&clk_disp0_154>, <&clk_disp0_194>; - clock-names = "clock-154", "clock-194"; - phys = <&atcphy0 PHY_TYPE_DP>; - phy-names = "typec0"; - mux-controls = <&dpxbar0 0>; - mux-control-names = "typec0"; - apple,typec-mux-indices = <2>; - apple,typec-dptx-phys = <0>; - apple,dcp-index = <1>; -}; - -&{/soc/dcp@416e00000/typec-routes/route@0} { - status = "okay"; -}; - -/* Match the external firmware buffer window rather than allocating below it. */ -&dcpext0_dart { - apple,dma-range = <0x100 0x0 0x10 0x0>; -}; diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 44143ae11092e..033272cdb32d9 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -1879,5 +1879,30 @@ }; }; -/* External resources stay disabled pending the separate firmware profile. */ +/* The loader publishes external resources after validating their firmware. */ #include "t8140-j700-dcpext.dtsi" + +&display { + iommus = <&disp0_dart 0>, <&dispext0_dart 0>; +}; + +&dcpext { + clocks = <&clk_disp0_154>, <&clk_disp0_194>; + clock-names = "clock-154", "clock-194"; + phys = <&atcphy0 PHY_TYPE_DP>; + phy-names = "typec0"; + mux-controls = <&dpxbar0 0>; + mux-control-names = "typec0"; + apple,typec-mux-indices = <2>; + apple,typec-dptx-phys = <0>; + apple,dcp-index = <1>; +}; + +&{/soc/dcp@416e00000/typec-routes/route@0} { + status = "okay"; +}; + +/* Allocate scanout buffers in the window used by the inherited firmware. */ +&dcpext0_dart { + apple,dma-range = <0x100 0x0 0x10 0x0>; +}; diff --git a/drivers/gpu/drm/apple/dcp-external-bw.h b/drivers/gpu/drm/apple/dcp-external-bw.h index 439db008b5084..a3a799ddefbd7 100644 --- a/drivers/gpu/drm/apple/dcp-external-bw.h +++ b/drivers/gpu/drm/apple/dcp-external-bw.h @@ -21,4 +21,19 @@ static inline bool apple_dcp_external_clock_window(u64 start, u64 size, return start == expected_start && size == expected_size; } +/* Resolve optional diagnostic overrides without changing the SoC defaults. */ +static inline bool apple_dcp_external_bw_resolve(u64 size, int default_scratch, + int default_request, int override_scratch, + int override_request, u32 *scratch, u32 *request) +{ + int s = override_scratch == -1 ? default_scratch : override_scratch; + int r = override_request == -1 ? default_request : override_request; + + if (!apple_dcp_external_bw_valid(size, s, r)) + return false; + *scratch = s; + *request = r; + return true; +} + #endif diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 1b1d1d18a78e2..e208e102949f9 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -139,20 +139,20 @@ static void dcp_release_rtkit(void *data) #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) -static bool experimental_h17p_external; +static bool experimental_h17p_external = true; module_param(experimental_h17p_external, bool, 0444); MODULE_PARM_DESC(experimental_h17p_external, - "Enable experimental T8140 external display protocol"); + "Enable T8140 external display protocol (default: enabled)"); static int experimental_h17p_external_bw_scratch = -1; module_param(experimental_h17p_external_bw_scratch, int, 0444); MODULE_PARM_DESC(experimental_h17p_external_bw_scratch, - "Experimental external bandwidth scratch offset; unset by default"); + "Override external bandwidth scratch offset (-1 uses the hardware profile)"); static int experimental_h17p_external_bw_request = -1; module_param(experimental_h17p_external_bw_request, int, 0444); MODULE_PARM_DESC(experimental_h17p_external_bw_request, - "Experimental external bandwidth request offset; unset by default"); + "Override external bandwidth request offset (-1 uses the hardware profile)"); bool dcp_h17p_external_enabled(void) { @@ -4244,14 +4244,16 @@ static int dcp_platform_probe(struct platform_device *pdev) if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { if (!dcp->hw.firmware_clock || - !apple_dcp_external_bw_valid(resource_size(dcp->hw.firmware_clock), + !apple_dcp_external_bw_resolve(resource_size(dcp->hw.firmware_clock), + dcp->hw.firmware_scratch, + dcp->hw.firmware_request, experimental_h17p_external_bw_scratch, - experimental_h17p_external_bw_request)) + experimental_h17p_external_bw_request, + &dcp->hw.firmware_scratch, + &dcp->hw.firmware_request)) return dev_err_probe(dev, -EINVAL, - "External bandwidth offsets are unset or invalid\n"); + "External bandwidth offsets are invalid\n"); iomfb_pause_presentation(dcp, true); - dcp->hw.firmware_scratch = experimental_h17p_external_bw_scratch; - dcp->hw.firmware_request = experimental_h17p_external_bw_request; } @@ -4606,6 +4608,9 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external = { .firmware_compat = DCP_FIRMWARE_H17P, .firmware_clock = &t8140_firmware_clock, .firmware_display_windows = 4, + /* Native PMGR BWR agent 2 and boost agent 3 for DISPEXT0. */ + .firmware_scratch = 0x28000, + .firmware_request = 0x70000, }; /* diff --git a/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py index 73f0dba8d4d22..f55107c0ddaa6 100644 --- a/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py +++ b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py @@ -14,6 +14,18 @@ def test_shared_header_boundaries(self): #include "dcp-external-bw.h" int main(void) { const u64 n=0xbc000; + u32 scratch=1, request=2; + assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,-1,&scratch,&request)); + assert(scratch==0x28000 && request==0x70000); + assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,0x30000,-1,&scratch,&request)); + assert(scratch==0x30000 && request==0x70000); + assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,0x74000,&scratch,&request)); + assert(scratch==0x28000 && request==0x74000); + assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-2,-1,&scratch,&request)); + assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,0x28000,&scratch,&request)); + assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,0x28002,-1,&scratch,&request)); + assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,n,&scratch,&request)); + assert(scratch==0x28000 && request==0x74000); assert(!apple_dcp_external_bw_valid(n,-1,-1)); assert(!apple_dcp_external_bw_valid(n,0,-1)); assert(!apple_dcp_external_bw_valid(n,-1,0)); From ea2e6a147844998a489da8e06fb176f2894b736f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 21:51:16 +1100 Subject: [PATCH 0963/1579] drm/apple: admit the native DP EDID provider in the guarded AV service H17P can announce DCPDP13Service in the EDID AV compartment after an attachment change. The HDMI-only class check rejects that otherwise valid identity before OPEN, leaving the connector without an EDID owner. Accept that exact native class alongside AppleDCPDP2HDMI under the existing AFKV2 AV compartment, endpoint, channel and unit constraints. Keep legacy handling, auxiliary-service policy and all attachment, generation, epoch, ownership and retirement fences unchanged. Exercise the actual announcement and admission functions with both providers over 64 flip/revalidation/return cycles, including same-generation revalidation. Retain forged identity, stale OPEN, owner, auxiliary, reader/pending-command and legacy controls. The exact parent fails the first legitimate native publication; the correction passes. Independently authored from authorized hardware observations of provider identifiers and the existing licensed source. No private implementation, firmware control flow or raw trace content was copied. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/epic/dpavservep.c | 3 +- tools/diagnostics/test-apple-dpav-rebind.py | 191 ++++++++++++++++++-- 2 files changed, 182 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 229680c6c265c..903d9578b0f80 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -41,7 +41,8 @@ static bool dcpavserv_identity(struct apple_epic_service *service, const char *n return unit == 0; /* H17P's lower-case interface-id is dynamic, not a physical-port unit. */ return service->ep->endpoint == DPAVSERV_ENDPOINT_V2 && class && - !strcmp(class, "AppleDCPDP2HDMI") && service->channel && + (!strcmp(class, "AppleDCPDP2HDMI") || + !strcmp(class, "DCPDP13Service")) && service->channel && service->channel <= U16_MAX && unit == service->channel; } diff --git a/tools/diagnostics/test-apple-dpav-rebind.py b/tools/diagnostics/test-apple-dpav-rebind.py index 278b2e52c88b5..fda6fdf7396a0 100644 --- a/tools/diagnostics/test-apple-dpav-rebind.py +++ b/tools/diagnostics/test-apple-dpav-rebind.py @@ -1,9 +1,17 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Run actual AFK ownership/receive and DPAV admission against physical epochs.""" +"""Run actual AFK announcements, ownership and DPAV admission across epochs. + +Wire frames and the property interpreter are production code; the binary +property decoder receives a synthetic dictionary. Locks, DMA and OPEN sending +are host mocks. This does not qualify firmware, concurrency or display output. +""" from pathlib import Path -import os,re,shlex,subprocess,tempfile -root=Path(__file__).resolve().parents[2];base=root/'drivers/gpu/drm/apple' +import argparse, hashlib, os, shlex, subprocess, tempfile +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("--source-root", type=Path, default=Path(__file__).resolve().parents[2]) +root = parser.parse_args().source_root.resolve() +base = root / 'drivers/gpu/drm/apple' def function(file,signature): text=(base/file).read_text();start=text.index(signature);opening=text.index('{',start) while ';' in text[start:opening]: @@ -39,6 +47,11 @@ def function(file,signature): #define dev_warn(...) ((void)0) #define dev_err(...) ((void)0) #define dev_info(...) ((void)0) +#define DCP_FIRMWARE_H17P 17 +#define EPIC_TYPE_NOTIFY 0 +#define EPIC_CAT_REPORT 0 +#define max(a,b) ((a)>(b)?(a):(b)) +#define strscpy(dst,src,n) snprintf(dst,n,"%s",(char *)(src)) #define trace_dcpavserv_init(...) ((void)0) #define dma_rmb() ((void)0) #define DRM_CONNECTOR_REGISTERED 1 @@ -57,7 +70,7 @@ def function(file,signature): static void reinit_completion(int *p) { *p=0; } static void schedule_work(unsigned int *p) { (*p)++; } struct apple_dcp;struct apple_dcp_afkep;struct apple_epic_service; -struct apple_epic_service_ops { bool reusable;int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; +struct apple_epic_service_ops { char name[32];bool reusable;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; struct epic_cmd_info { u16 tag;u8 type;size_t reply_length;void *rxbuf,*txbuf;dma_addr_t rxbuf_dma,txbuf_dma;size_t rxlen,txlen;u32 retcode;bool done,free_on_ack;int *completion; }; struct apple_epic_service { struct apple_dcp_afkep *ep;const struct apple_epic_service_ops *ops; @@ -65,20 +78,43 @@ def function(file,signature): unsigned long cmd_map[1];struct epic_cmd_info cmds[MAX_PENDING_CMDS]; struct { u64 generation,epoch; } admission;void *cookie; }; -struct apple_dcp_afkep { struct apple_dcp *dcp;bool wire_v2,inline_v2;u32 endpoint;unsigned int num_channels;struct apple_epic_service services[AFK_MAX_CHANNEL]; }; +struct apple_dcp_afkep { struct apple_dcp *dcp;const struct apple_epic_service_ops *ops;bool wire_v2,inline_v2,match_epic_name;u32 endpoint;unsigned int num_channels;struct apple_epic_service services[AFK_MAX_CHANNEL]; }; struct apple_connector { void *dcp;bool connected,edid_invalidation_pending;struct { int registration_state; } base;unsigned int hotplug_wq; }; struct apple_dcp { - void *dev;struct apple_dcp_afkep *dcpavservep;struct apple_connector *connector; + void *dev;int fw_compat;struct apple_dcp_afkep *dcpavservep;struct apple_connector *connector; bool hpd_mutex,quiescing,typec_cable_connected;u64 typec_generation;unsigned int nr_modes; struct { int iomfb_method_profile; } hw; struct { bool lock,enabled;u64 epoch;int enable_completion;struct apple_epic_service *service; } dcpavserv; }; struct epic_cmd { u32 retcode;u64 rxbuf,txbuf;u32 rxlen,txlen; }; -static unsigned int dma_frees,announces; +static unsigned int dma_frees,opens; +static int open_error,open_action; +void dpavservep_detach(struct apple_dcp *dcp); +static void inject_owner(struct apple_dcp_afkep *ep,u32 pending_channel); static void dma_free_coherent(void *dev,size_t size,void *p,dma_addr_t address) { (void)dev;assert(size && (uintptr_t)p==address);free(p);dma_frees++; } static void afk_remove_service_debugfs(struct apple_epic_service *s) { (void)s; } -static void afk_recv_handle_init(struct apple_dcp_afkep *e,u32 channel,u8 *p,size_t n) { (void)e;(void)channel;(void)p;(void)n;announces++; } +static void afk_populate_service_debugfs(struct apple_epic_service *s) { (void)s; } +static int afk_send_epic(struct apple_dcp_afkep *e,u32 channel,u16 tag,int type,int category,u8 subtype,const void *p,size_t n) +{ + assert(channel && !tag && type==EPIC_TYPE_NOTIFY && category==EPIC_CAT_REPORT && subtype==AFKV2_OPEN && !p && !n); + opens++; + assert(!e->dcp->dcpavserv.enabled && !e->dcp->dcpavserv.service); + for(unsigned int i=0;inum_channels;i++) { + struct apple_epic_service *s=&e->services[i]; + if(s->channel==channel && !s->torndown) assert(!s->enabled && !s->users); + } + if(open_action==1) e->dcp->typec_generation++; + if(open_action==2) dpavservep_detach(e->dcp); + if(open_action==4) inject_owner(e,channel); + if(open_action==5) e->dcp->typec_cable_connected=false; + if(open_action==6) e->dcp->quiescing=true; + if(open_action==3) { + for(unsigned int i=0;inum_channels;i++) + if(e->services[i].channel==channel && !e->services[i].torndown) e->services[i].channel++; + } + return open_error; +} static void afk_v2_callback(struct apple_dcp_afkep *e,const struct afkv2_frame *f,const struct afkv2_command *c) { (void)e;(void)f;(void)c;assert(!"unexpected callback"); } struct item { const char *key,*value; s64 integer; }; @@ -86,6 +122,9 @@ def function(file,signature): struct iterator { struct dcp_parse_ctx *handle;unsigned int idx,len; }; #define kfree(p) free((void *)(p)) #define PTR_ERR(p) ((int)(intptr_t)(p)) +static struct item announcement_items[3]; +static int parse(u8 *p,size_t n,struct dcp_parse_ctx *ctx) +{ (void)p;assert(n==37);*ctx=(struct dcp_parse_ctx){.items=announcement_items,.count=3};return 0; } static int iterator_begin(struct dcp_parse_ctx *h,struct iterator *it,bool dict) { assert(dict);h->index=0;h->key=true;*it=(struct iterator){.handle=h,.len=h->count};return 0; } static char *parse_string(struct dcp_parse_ctx *h) @@ -99,7 +138,7 @@ def function(file,signature): static int skip(struct dcp_parse_ctx *h) { assert(!h->key);h->index++;h->key=true;return 0; } ''' parts=[function('parser.c','int parse_epic_service_init('),function('afk.h','static inline struct apple_epic_service *afk_service_get('), - function('afk.h','static inline void afk_service_put('),function('afk.h','static inline void afk_service_retire('), + function('afk.h','static inline void afk_service_put('),function('afk.h','static inline void afk_service_retire('),function('afk.h','static inline void afk_service_disable('), function('afk.h','static inline bool afk_service_matches('),function('afk.h','static inline bool afk_service_reclaimable('), function('afk.h','static inline bool\nafk_service_reinit('),function('afk.h','static inline void afk_service_discard('), function('afk.h','static inline void afk_service_publish('), @@ -107,14 +146,25 @@ def function(file,signature): function('epic/dpavservep.c','static bool dcpavserv_admission_current('),function('epic/dpavservep.c','static int dcpavserv_prepare('), function('epic/dpavservep.c','static void dcpavserv_init('),function('epic/dpavservep.c','static void dcpavserv_invalidate_edid('), function('epic/dpavservep.c','static void dcpavserv_teardown('),function('epic/dpavservep.c','void dpavservep_detach('), - function('afk.c','static struct apple_epic_service *afk_epic_find_service('),function('afk.c','static void afk_recv_handle_teardown('), + function('afk.c','static const struct apple_epic_service_ops *\nafk_match_service('), + function('afk.c','static struct apple_epic_service *afk_epic_find_service('), + function('afk.c','static void afk_recv_handle_init('), + function('epic/dpavservep.c','static int dcpdpserv_prepare('), + function('epic/dpavservep.c','static void dcpdpserv_teardown('),function('afk.c','static void afk_recv_handle_teardown('), function('afk.c','static void afk_recv_handle_reply('),function('afk.c','static void afk_v2_receive(')] +print('Production lifecycle functions SHA256:', hashlib.sha256('\n'.join(parts).encode()).hexdigest(), flush=True) suffix=r''' static struct apple_dcp d; static struct apple_dcp_afkep ep; static struct apple_connector con; -static const struct apple_epic_service_ops ops={.reusable=true,.prepare=dcpavserv_prepare,.teardown=dcpavserv_teardown}; +static const struct apple_epic_service_ops ops={.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.init=dcpavserv_init,.teardown=dcpavserv_teardown}; +static const struct apple_epic_service_ops announcement_ops[]={ + {.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.init=dcpavserv_init,.teardown=dcpavserv_teardown}, + {.name="dcpdp-service-epic",.reusable=true,.prepare=dcpdpserv_prepare,.teardown=dcpdpserv_teardown}, + {0} +}; static const char *name="dcpav-service-epic",*class="AppleDCPDP2HDMI"; +static const char *native_class="DCPDP13Service"; static struct apple_epic_service *prepare(u32 channel,s64 unit,const char *n,const char *c) { for(unsigned int i=0;iusers==1); +} +static void repeated_announcements(void) +{ + ep.ops=announcement_ops;ep.match_epic_name=true;ep.num_channels=0;d.fw_compat=DCP_FIRMWARE_H17P; + /* More than the entire slot budget; wrap channel identity as well. */ + for(unsigned int i=0;i<4*AFK_MAX_CHANNEL;i++) { + u16 channel=1+4*(i%4);unsigned int before=opens; + /* Flip, same-attachment revalidation, return flip; swap both providers. */ + bool native=((i%4==1 || i%4==2) != !!((i/4)%2)); + const char *provider=native?native_class:class; + announce(channel,name,provider,channel); + struct apple_epic_service *owner=d.dcpavserv.service; + if(native) assert(owner && "native DP EDID provider must publish"); + assert(owner && owner->channel==channel && owner->users==1 && opens==before+1); + assert(ep.num_channels<=2); /* Real allocator reuses, no slot exhaustion. */ + u64 epoch=d.dcpavserv.epoch; + announce(channel,name,provider,channel); /* Active duplicate: no extra OPEN/pin. */ + assert(opens==before+1 && owner->users==1 && d.dcpavserv.epoch==epoch); + announce(channel+2,"dcpdp-service-epic",provider,channel+2); + assert(opens==before+1 && d.dcpavserv.service==owner && d.dcpavserv.epoch==epoch); + if(i%4==1) { + u64 generation=d.typec_generation; + closed(channel,0x6a01,0); /* Revalidation retires service, not attachment. */ + assert(!d.dcpavserv.service && d.typec_generation==generation && d.dcpavserv.epoch>epoch); + } else { + disconnect();connect(); + } + } + assert(ep.num_channels==2); /* Auxiliary rejection leaves a reusable slot. */ + unsigned int before=opens; + announce(13,name,"synthetic-wrong-class",13);assert(opens==before && !d.dcpavserv.service); + char long_class[161];memset(long_class,'x',160);long_class[160]=0; + announce(13,name,long_class,13);assert(opens==before && !d.dcpavserv.service); + for(unsigned int variant=0;variant<2;variant++) { + const char *provider=variant?native_class:class; + assert(!prepare(0,0,name,provider));assert(!prepare(65536,65536,name,provider)); + assert(!prepare(13,-1,name,provider));assert(!prepare(13,0,name,provider)); + assert(!prepare(13,1,name,provider));assert(!prepare(13,13,NULL,provider)); + assert(!prepare(13,13,"dcpdp-service-epic",provider)); + announce(13,"wrong-compartment",provider,13);assert(opens==before && !d.dcpavserv.service); + ep.endpoint=0x2c;announce(13,name,provider,13);assert(opens==before && !d.dcpavserv.service);ep.endpoint=0x2a; + d.quiescing=true;announce(13,name,provider,13);assert(opens==before && !d.dcpavserv.service);d.quiescing=false; + disconnect();announce(13,name,provider,13);assert(opens==before && !d.dcpavserv.service);connect(); + } + for(unsigned int variant=0;variant<4;variant++) { + const char *forged[]={"DCPDP13ServiceExtra","dcpdp13service","AppleDCPDP2HDMIExtra","unrelated-provider"}; + announce(13,name,forged[variant],13);assert(opens==before && !d.dcpavserv.service); + } + assert(!prepare(13,13,name,NULL)); + open_error=-EIO;announce(13,name,native_class,13);assert(opens==++before && !d.dcpavserv.service);open_error=0; + for(unsigned int window=0;window<5;window++) { + const int actions[]={1,2,3,5,6};open_action=actions[window]; + announce(13,name,native_class,13); + assert(opens==++before && !d.dcpavserv.service && !d.dcpavserv.enabled); + d.typec_cable_connected=true;d.quiescing=false; + } + open_action=4;announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service==injected_owner); + assert(injected_owner->users==1);dpavservep_detach(&d);assert(!injected_owner->users); + open_action=0;announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service); + struct apple_epic_service *old=d.dcpavserv.service; + assert(afk_service_get(old)==old);old->cmd_map[0]=1; + disconnect();connect(); + announce(13,name,native_class,13); /* Same channel still owns a pending reply. */ + assert(opens==before && !d.dcpavserv.service && old->users==1 && old->channel==13); + announce(1,name,class,1);assert(opens==++before && d.dcpavserv.service!=old); + struct apple_epic_service *fresh=d.dcpavserv.service;u64 epoch=d.dcpavserv.epoch; + dcpavserv_teardown(old);assert(d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch); + old->cmd_map[0]=0;afk_service_put(old);disconnect();connect(); + announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service); + struct apple_epic_service *owner=d.dcpavserv.service; + u64 owner_epoch=d.dcpavserv.epoch; + announce(5,name,class,5); /* A second admitted identity cannot displace the owner. */ + assert(opens==before && d.dcpavserv.service==owner && d.dcpavserv.epoch==owner_epoch); + dpavservep_detach(&d); + /* All slots held by retired readers: fail safely, then recover after puts. */ + for(unsigned int i=0;itorndown); + legacy=prepare(13,0,name,native_class);assert(legacy); + d.typec_generation++;afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); + assert(d.dcpavserv.service==legacy && legacy->users==1);dpavservep_detach(&d); + assert(dcpdpserv_prepare(legacy,"dcpdp-service-epic",native_class,0)==0); + ep.wire_v2=true;ep.endpoint=0x2a; + announce(U16_MAX,name,native_class,U16_MAX);assert(d.dcpavserv.service && d.dcpavserv.service->channel==U16_MAX); + dpavservep_detach(&d); + puts("PASS: actual AFKV2 HDMI/native DP alternating over 64 cycles: flip/revalidation/return, strict identity, stale OPEN generation/epoch/owner, auxiliary isolation, reader/command pins and legacy controls"); +} int main(void) { d.hw.iomfb_method_profile=2;d.typec_cable_connected=true;d.typec_generation=1;d.nr_modes=1;d.connector=&con;d.dev=&d; @@ -204,6 +371,8 @@ def function(file,signature): assert(!prepare(7,7,name,class));pending=prepare(7,0,name,NULL);assert(pending);publish(pending,0); assert(d.dcpavserv.service==pending && pending->users==1);dpavservep_detach(&d); assert(!pending->users && !d.hpd_mutex && !d.dcpavserv.lock); + memset(&ep.services,0,sizeof(ep.services));ep.wire_v2=true;ep.endpoint=0x2a;d.hw.iomfb_method_profile=2; + repeated_announcements(); puts("PASS: actual dynamic 1/5/9/13 binding, forged identities, prepare/init epochs, physical retirement, reader pins, real old CLOSED/late ACK isolation and legacy unit 0"); } ''' From 7357e51053e4dc9e326e0cafc888b5c6e8bc5077 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 19:16:00 +0000 Subject: [PATCH 0964/1579] drm/apple: scope deferred DPAV recovery to current interfaces Retain admitted reannouncements while old commands drain, cancel pending OPEN when CLOSED supersedes it, and preserve interface tag history across reader-pinned slot transitions. Fence explicit physical absence during admission, publication and EDID-current validation. Preserve old command buffers, reader pins and existing endpoint teardown ownership. Keep actual-function regressions in the compact standard selftests. Independent review verified corrected CLOSED/history/absence/fixture controls and strict ARM objects. Exact source tree f14f599948a8ccae7362c4d328dddea623442e54 passed the full strict R66 Image/modules/dtbs build. This is a functional qualification candidate. The inherited four-bit command identity still cannot distinguish a firmware-republished consumed reply when all correlation values recur; retained command-17 counterexamples remain unresolved. No new peer-token semantics, packet experiment, cached EDID override or production diagnostic knob is introduced. No release or hardware qualification is claimed for this new candidate. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 165 ++++++- drivers/gpu/drm/apple/afk.h | 18 +- drivers/gpu/drm/apple/epic/dpavservep.c | 30 +- .../drivers/gpu/apple-afkv2/extract-send.py | 16 +- .../selftests/drivers/gpu/apple-dcp/Makefile | 2 +- .../selftests/drivers/gpu/apple-dcp/README | 14 + .../selftests/drivers/gpu/apple-dcp/edid.py | 135 +++++- .../drivers/gpu/apple-dcp/recovery.py | 439 ++++++++++++++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 3 + 9 files changed, 795 insertions(+), 27 deletions(-) create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 8a42985a091bb..a617d35fd49f2 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -52,6 +52,21 @@ enum rbep_msg_type { #define SEND_WPTR GENMASK(31, 0) +static void afk_service_worker(struct work_struct *work); +static void afk_discard_pending_open(struct apple_epic_service *service); + +void afk_service_retry(struct apple_dcp_afkep *ep) +{ + unsigned long flags; + + if (!ep || !ep->wire_v2 || ep != ep->dcp->dcpavservep) + return; + spin_lock_irqsave(&ep->lock, flags); + if (!ep->stopping) + queue_work(ep->wq, &ep->service_work); + spin_unlock_irqrestore(&ep->lock, flags); +} + static void afk_send(struct apple_dcp_afkep *ep, u64 message) { dcp_send_message(ep->dcp, ep->endpoint, message); @@ -62,8 +77,10 @@ static void afk_release_context(void *data) struct apple_dcp_afkep *afkep = data; afk_quiesce(afkep); - if (!afkep->dcp->retain_dma) + if (!afkep->dcp->retain_dma) { + kvfree(afkep->command_tags); kfree(afkep); + } } struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, @@ -86,6 +103,7 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, goto out_free_afkep; } + INIT_WORK(&afkep->service_work, afk_service_worker); init_completion(&afkep->started); init_completion(&afkep->stopped); spin_lock_init(&afkep->lock); @@ -104,12 +122,21 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, void afk_quiesce(struct apple_dcp_afkep *afkep) { + unsigned long flags; + u32 i; + if (IS_ERR_OR_NULL(afkep)) return; + spin_lock_irqsave(&afkep->lock, flags); + afkep->stopping = true; + spin_unlock_irqrestore(&afkep->lock, flags); if (afkep->wq) { destroy_workqueue(afkep->wq); afkep->wq = NULL; } + for (i = 0; i < afkep->num_channels; i++) + if (afkep->services[i].pending_open) + afk_discard_pending_open(&afkep->services[i]); debugfs_remove_recursive(afkep->debugfs_entry); afkep->debugfs_entry = NULL; } @@ -140,6 +167,14 @@ int afk_start(struct apple_dcp_afkep *ep) { int ret; + /* Allocate before firmware can announce services or issue commands. */ + if (ep->wire_v2 && !ep->command_tags) { + ep->command_tags = kvcalloc(AFKV2_INTERFACES, sizeof(*ep->command_tags), + GFP_KERNEL); + if (!ep->command_tags) + return -ENOMEM; + } + reinit_completion(&ep->started); ret = apple_rtkit_start_ep(ep->dcp->rtk, ep->endpoint); if (ret) @@ -310,12 +345,74 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, enum epic_type etype, enum epic_category ecat, u8 stype, const void *payload, size_t payload_len, u64 token); +/* Pending identities are owned only by the ordered endpoint workqueue. */ +static void afk_discard_pending_open(struct apple_epic_service *service) +{ + service->pending_open = false; + kfree(service->pending_name); + kfree(service->pending_class); + service->pending_name = NULL; + service->pending_class = NULL; + afk_service_discard(service); +} + +static void afk_retry_service_init(struct apple_dcp_afkep *ep) +{ + u32 i; + + for (i = 0; i < ep->num_channels; i++) { + struct apple_epic_service *service = &ep->services[i]; + const struct apple_epic_service_ops *ops = service->ops; + int ret; + + if (!service->pending_open) + continue; + ret = ops->revalidate(service, service->pending_name, + service->pending_class, service->pending_unit); + if (ret) { + afk_discard_pending_open(service); + continue; + } + /* Old replies keep their original slot, tag and buffers until drained. */ + if (afk_epic_find_service(ep, service->channel)) + continue; + ret = afk_send_epic(ep, service->channel, 0, EPIC_TYPE_NOTIFY, + EPIC_CAT_REPORT, AFKV2_OPEN, NULL, 0); + if (ret) { + dev_err(ep->dcp->dev, "AFKV2 interface %u deferred OPEN failed: %d\n", + service->channel, ret); + afk_discard_pending_open(service); + continue; + } + service->pending_open = false; + afk_service_publish(service); + ops->init(service, service->pending_name, service->pending_class, + service->pending_unit); + if (service->enabled) + afk_populate_service_debugfs(service); + kfree(service->pending_name); + kfree(service->pending_class); + service->pending_name = NULL; + service->pending_class = NULL; + } +} + +static void afk_service_worker(struct work_struct *work) +{ + struct apple_dcp_afkep *ep = container_of(work, struct apple_dcp_afkep, service_work); + + if (!READ_ONCE(ep->stopping)) + afk_retry_service_init(ep); +} + static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *payload, size_t payload_size) { char name[32]; s64 epic_unit = -1; - struct apple_epic_service *service = NULL; + struct apple_epic_service *service = NULL, *previous; + unsigned long flags; + bool retired = false; u32 ch_idx; const char *service_name = name; const char *epic_name = NULL, *epic_class = NULL; @@ -324,8 +421,25 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *props = payload + sizeof(name); size_t props_size = payload_size - sizeof(name); - if (afk_epic_find_service(ep, channel)) - return; + previous = afk_epic_find_service(ep, channel); + if (previous) { + spin_lock_irqsave(&previous->lock, flags); + retired = previous->torndown; + spin_unlock_irqrestore(&previous->lock, flags); + if (!ep->wire_v2 || ep != ep->dcp->dcpavservep || + !retired || !previous->ops->revalidate) + return; + } + for (ch_idx = 0; ch_idx < ep->num_channels; ch_idx++) { + struct apple_epic_service *pending = &ep->services[ch_idx]; + + if (!pending->pending_open || pending->channel != channel) + continue; + if (!pending->ops->revalidate(pending, pending->pending_name, + pending->pending_class, pending->pending_unit)) + return; + afk_discard_pending_open(pending); + } if (payload_size < sizeof(name)) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", @@ -398,6 +512,15 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, goto free; } } + if (previous && retired && ops->revalidate) { + /* Preserve the admitted identity and its original provider ticket. */ + service->pending_open = true; + service->pending_name = epic_name; + service->pending_class = epic_class; + service->pending_unit = epic_unit; + afk_service_retry(ep); + return; + } if (ep->wire_v2) { int ret = afk_send_epic(ep, channel, 0, EPIC_TYPE_NOTIFY, EPIC_CAT_REPORT, AFKV2_OPEN, NULL, 0); @@ -438,6 +561,14 @@ static void afk_recv_handle_teardown(struct apple_dcp_afkep *ep, u32 channel) const struct apple_epic_service_ops *ops; unsigned long flags; + /* Accepted CLOSE/CLOSED supersedes unpublished recovery for this interface. */ + for (u32 i = 0; i < ep->num_channels; i++) { + struct apple_epic_service *pending = &ep->services[i]; + + if (pending->pending_open && pending->channel == channel) + afk_discard_pending_open(pending); + } + service = afk_epic_find_service(ep, channel); if (!service) { dev_warn(ep->dcp->dev, "AFK[ep:%02x]: teardown for disabled channel %u\n", @@ -1459,6 +1590,23 @@ int afk_send_command(struct apple_epic_service *service, u8 type, MSEC_PER_SEC); } +static u16 afk_command_tag(struct apple_epic_service *service, u8 slot) +{ + struct apple_dcp_afkep *ep = service->ep; + unsigned long flags; + u16 tag; + + lockdep_assert_held(&service->lock); + if (!ep->wire_v2) + return ((service->cmd_tag++ & 0xff) << 8) | slot; + + /* Send already nests the endpoint lock below the service lock. */ + spin_lock_irqsave(&ep->lock, flags); + tag = afkv2_command_id(ep->command_tags[service->channel]++, slot); + spin_unlock_irqrestore(&ep->lock, flags); + return tag; +} + static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 type, const void *payload, size_t payload_len, void *output, size_t output_len, u32 *retcode, @@ -1495,6 +1643,11 @@ static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 t if (!service) return -ENODEV; ep = service->ep; + if (ep->wire_v2 && (!service->channel || service->channel >= AFKV2_INTERFACES || + !ep->command_tags)) { + ret = -ENODEV; + goto err_put_service; + } if (READ_ONCE(ep->dcp->quiescing)) { ret = -ENODEV; goto err_put_service; @@ -1550,9 +1703,7 @@ static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 t goto err_unlock; } - tag = ep->wire_v2 ? afkv2_command_id(service->cmd_tag, idx) : - ((service->cmd_tag & 0xff) << 8) | idx; - service->cmd_tag++; + tag = afk_command_tag(service, idx); service->cmds[idx].tag = tag; service->cmds[idx].type = type; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index ed0e3443a2969..b51ac5ca8b9a4 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -19,10 +19,13 @@ #define AFK_MAX_CHANNEL 16 #define MAX_PENDING_CMDS 16 +#define AFKV2_INTERFACES BIT(16) struct apple_epic_service_ops; struct apple_dcp_afkep; +void afk_service_retry(struct apple_dcp_afkep *ep); + struct epic_cmd_info { u16 tag; u8 type; @@ -47,6 +50,7 @@ struct apple_epic_service { struct epic_cmd_info cmds[MAX_PENDING_CMDS]; DECLARE_BITMAP(cmd_map, MAX_PENDING_CMDS); + /* Legacy EPIC sequence; AFKV2 history belongs to the interface. */ u8 cmd_tag; spinlock_t lock; /* Pins held by owners and synchronous callers; protected by lock. */ @@ -60,6 +64,11 @@ struct apple_epic_service { struct { u64 generation, epoch; } admission; void *cookie; + /* Admitted reannouncement waiting for the previous channel to drain. */ + bool pending_open; + const char *pending_name, *pending_class; + s64 pending_unit; + struct { struct dentry *entry; u8 *scratch; @@ -79,6 +88,9 @@ struct apple_epic_service_ops { */ int (*prepare)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); + /* Check the original prepare ticket before a deferred OPEN. */ + int (*revalidate)(struct apple_epic_service *service, const char *name, + const char *class, s64 unit); void (*init)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); int (*call)(struct apple_epic_service *service, u32 idx, @@ -139,6 +151,7 @@ static inline void afk_service_put(struct apple_epic_service *service) if (!WARN_ON(!service->users)) service->users--; spin_unlock_irqrestore(&service->lock, flags); + afk_service_retry(service->ep); } static inline void afk_service_disable(struct apple_epic_service *service) @@ -202,7 +215,7 @@ afk_service_reinit(struct apple_epic_service *service, service->ep = ep; service->channel = channel; service->cookie = NULL; - /* Keep the command-tag sequence across service generations. */ + /* Keep legacy slot history; AFKV2 history belongs to the interface. */ } spin_unlock_irqrestore(&service->lock, flags); return reusable; @@ -344,7 +357,10 @@ struct apple_dcp_afkep { bool wire_v2; /* External H17P DPAV only; other endpoints retain owned-DMA commands. */ bool inline_v2; + /* Fixed 16-bit interface domain; survives slot and channel reuse. */ + u8 *command_tags; struct workqueue_struct *wq; + struct work_struct service_work; struct completion started; struct completion stopped; diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 2fb18d1eb512c..037a6388ffea2 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -50,7 +50,8 @@ static bool dcpavserv_admission_current(struct apple_dcp *dcp, u64 generation) { lockdep_assert_held(&dcp->hpd_mutex); /* Initial service discovery may precede the first Type-C attachment. */ - return !READ_ONCE(dcp->quiescing) && generation == dcp->typec_generation && + return !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->external_physically_absent) && + generation == dcp->typec_generation && (!generation || dcp->typec_cable_connected); } @@ -81,6 +82,26 @@ static int dcpavserv_prepare(struct apple_epic_service *service, const char *nam return ret; } +static int dcpavserv_revalidate(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + int ret = 0; + + if (!dcpavserv_identity(service, name, class, unit)) + return -ENODEV; + mutex_lock(&dcp->hpd_mutex); + mutex_lock(&dcp->dcpavserv.lock); + if (dcp->dcpavserv.enabled) + ret = -EBUSY; + else if (service->admission.epoch != dcp->dcpavserv.epoch || + !dcpavserv_admission_current(dcp, service->admission.generation)) + ret = -ENODEV; + mutex_unlock(&dcp->dcpavserv.lock); + mutex_unlock(&dcp->hpd_mutex); + return ret; +} + static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -363,14 +384,16 @@ bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation) return dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL || (dcp->typec_cable_connected && generation == dcp->typec_generation && - !READ_ONCE(dcp->quiescing)); + !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->external_physically_absent)); } bool dcpavserv_edid_service_current(struct apple_dcp *dcp, u64 epoch) { lockdep_assert_held(&dcp->dcpavserv.lock); return dcp->dcpavserv.enabled && dcp->dcpavserv.service && - epoch == dcp->dcpavserv.epoch; + epoch == dcp->dcpavserv.epoch && + (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL || + !READ_ONCE(dcp->external_physically_absent)); } const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation, @@ -416,6 +439,7 @@ static const struct apple_epic_service_ops dpavservep_ops[] = { .name = "dcpav-service-epic", .reusable = true, .prepare = dcpavserv_prepare, + .revalidate = dcpavserv_revalidate, .init = dcpavserv_init, .teardown = dcpavserv_teardown, .report = dcpavserv_report, diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py index 0d80a729a3342..70dbd5d3a69f4 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py @@ -30,6 +30,8 @@ #define SZ_1M 1048576U #define GFP_KERNEL 0 #define MAX_PENDING_CMDS 16 +#define AFKV2_INTERFACES (1U << 16) +#define lockdep_assert_held(l) ((void)(l)) #define cpu_to_le32(x) (x) #define le32_to_cpu(x) (x) #define cpu_to_le64(x) (x) @@ -56,7 +58,7 @@ bool done,free_on_ack; int *completion; }; struct dcp { void *dev; bool quiescing; }; -struct apple_dcp_afkep { struct dcp *dcp; bool wire_v2,inline_v2; int endpoint; }; +struct apple_dcp_afkep { struct dcp *dcp; bool wire_v2,inline_v2; int endpoint,lock;u8 *command_tags; }; struct ops { const char *name; }; struct apple_epic_service { struct apple_dcp_afkep *ep; struct ops *ops; unsigned int channel; unsigned long cmd_map[1]; @@ -106,7 +108,8 @@ suffix = r''' int main(void) { - struct dcp dcp={0};struct apple_dcp_afkep ep={.dcp=&dcp,.wire_v2=true,.inline_v2=true}; + u8 tags[AFKV2_INTERFACES]={0}; + struct dcp dcp={0};struct apple_dcp_afkep ep={.dcp=&dcp,.wire_v2=true,.inline_v2=true,.command_tags=tags}; struct ops ops={.name="test"};struct apple_epic_service s={.ep=&ep,.ops=&ops,.channel=4,.enabled=true,.users=1}; u8 request[112]={0x5a},*output=calloc(1,32888);size_t received;u32 status=99; active=&s; @@ -155,4 +158,11 @@ puts("PASS: extracted production sender, 128-byte command envelope / 32888-byte reply, short OOB, strict/legacy callers and actual late-ACK ownership"); } ''' -pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + source[reply_match.start():reply_end] + suffix) +tag_match = re.search(r"static u16 afk_command_tag\([^;]*?\)\n\{", source) +if not tag_match: + raise RuntimeError("production interface tag allocator not found") +tag_end, tag_depth = tag_match.end(), 1 +while tag_depth: + tag_depth += (source[tag_end] == "{") - (source[tag_end] == "}") + tag_end += 1 +pathlib.Path(sys.argv[2]).write_text(prefix + source[tag_match.start():tag_end] + source[start:end] + source[reply_match.start():reply_end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile index 0d6d406ef5b9d..cdc821ce4be25 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT TEST_PROGS := run.sh -TEST_FILES := run.py common.py edid.py rate.py phy.py lifecycle.py pm.py idle.py dwc3 README +TEST_FILES := run.py common.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py dwc3 README include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index a61543d1168f8..e9bf9bdabcb67 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -54,3 +54,17 @@ The cpuidle group executes the actual Apple table/probe, generic registration and s2idle selector/entry, and tick_freeze/unfreeze. It checks source behavior only; RTC, elapsed suspend time and simultaneous hardware execution remain separate qualification gates. The original human cpuidle patch is preserved. + +The service-rebind group also runs retained reviewer controls through the +actual sender, receive worker, recovery, quiesce and EDID wrapper. It covers +CLOSED cancellation before lookup, first late ACK versus already consumed +duplicates, reader-pinned cross-slot/channel reuse, and the explicit physical +absence window. Per-interface sequence history has a fixed 64-KiB bound per +AFKV2 endpoint; reply routing still ignores reader-only retired services. + +Two wrap controls intentionally retain the command-17 replay assertion +failure, for OOB and inline. The group verifies that these counterexamples +still reproduce, alongside nonvacuous mutations and passing corrected safety +controls. A passing host suite does not establish a replay-exclusion contract +or admit a hardware trial. Token/sequence semantics remain a separate protocol +fact to establish; the tests do not manufacture a firmware echo guarantee. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py index 55a16678b97be..7d892663ff675 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -21,6 +21,8 @@ def function(file,signature): #include #include #include +#include +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) #include "afkv2-codec.h" typedef int64_t s64; typedef u64 dma_addr_t; @@ -29,6 +31,7 @@ def function(file,signature): #define ENOENT 2 #define MAX_PENDING_CMDS 16 #define AFK_MAX_CHANNEL 16 +#define AFKV2_INTERFACES (1U << 16) #define READ_ONCE(x) (x) #define WRITE_ONCE(x,v) ((x)=(v)) #define WARN_ON(x) (x) @@ -65,25 +68,30 @@ def function(file,signature): static void complete(int *p) { (*p)++; } static void reinit_completion(int *p) { *p=0; } static void schedule_work(unsigned int *p) { (*p)++; } +struct work_struct { bool pending; }; +static bool queue_work(void *wq,struct work_struct *w) { assert(wq);bool fresh=!w->pending;w->pending=true;return fresh; } struct apple_dcp;struct apple_dcp_afkep;struct apple_epic_service; -struct apple_epic_service_ops { char name[32];bool reusable;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; +void afk_service_retry(struct apple_dcp_afkep *ep); +struct apple_epic_service_ops { char name[32];bool reusable;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);int (*revalidate)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; struct epic_cmd_info { u16 tag;u8 type;size_t reply_length;void *rxbuf,*txbuf;dma_addr_t rxbuf_dma,txbuf_dma;size_t rxlen,txlen;u32 retcode;bool done,free_on_ack;int *completion; }; struct apple_epic_service { struct apple_dcp_afkep *ep;const struct apple_epic_service_ops *ops; bool lock,enabled,torndown;unsigned int users;u32 channel;u8 cmd_tag; unsigned long cmd_map[1];struct epic_cmd_info cmds[MAX_PENDING_CMDS]; struct { u64 generation,epoch; } admission;void *cookie; + bool pending_open;const char *pending_name,*pending_class;s64 pending_unit; }; -struct apple_dcp_afkep { struct apple_dcp *dcp;const struct apple_epic_service_ops *ops;bool wire_v2,inline_v2,match_epic_name;u32 endpoint;unsigned int num_channels;struct apple_epic_service services[AFK_MAX_CHANNEL]; }; +struct apple_dcp_afkep { u8 *command_tags;bool lock,stopping;void *wq;struct work_struct service_work;struct apple_dcp *dcp;const struct apple_epic_service_ops *ops;bool wire_v2,inline_v2,match_epic_name;u32 endpoint;unsigned int num_channels;struct apple_epic_service services[AFK_MAX_CHANNEL]; }; struct apple_connector { void *dcp;bool connected,edid_invalidation_pending;struct { int registration_state; } base;unsigned int hotplug_wq; }; struct apple_dcp { void *dev;int fw_compat;struct apple_dcp_afkep *dcpavservep;struct apple_connector *connector; - bool hpd_mutex,quiescing,typec_cable_connected;u64 typec_generation;unsigned int nr_modes; + bool external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;u64 typec_generation;unsigned int nr_modes; struct { int iomfb_method_profile; } hw; struct { bool lock,enabled;u64 epoch;int enable_completion;struct apple_epic_service *service; } dcpavserv; }; struct epic_cmd { u32 retcode;u64 rxbuf,txbuf;u32 rxlen,txlen; }; static unsigned int dma_frees,opens; + static int open_error,open_action; void dpavservep_detach(struct apple_dcp *dcp); static void inject_owner(struct apple_dcp_afkep *ep,u32 pending_channel); @@ -105,6 +113,7 @@ def function(file,signature): if(open_action==4) inject_owner(e,channel); if(open_action==5) e->dcp->typec_cable_connected=false; if(open_action==6) e->dcp->quiescing=true; + if(open_action==7) e->dcp->external_physically_absent=true; if(open_action==3) { for(unsigned int i=0;inum_channels;i++) if(e->services[i].channel==channel && !e->services[i].torndown) e->services[i].channel++; @@ -133,18 +142,18 @@ def function(file,signature): { assert(!h->key);*out=h->items[h->index++].integer;h->key=true;return 0; } static int skip(struct dcp_parse_ctx *h) { assert(!h->key);h->index++;h->key=true;return 0; } ''' - parts=[function('parser.c','int parse_epic_service_init('),function('afk.h','static inline struct apple_epic_service *afk_service_get('), + parts=[function('afk.c','static u16 afk_command_tag('),function('afk.c','void afk_service_retry('),function('parser.c','int parse_epic_service_init('),function('afk.h','static inline struct apple_epic_service *afk_service_get('), function('afk.h','static inline void afk_service_put('),function('afk.h','static inline void afk_service_retire('),function('afk.h','static inline void afk_service_disable('), function('afk.h','static inline bool afk_service_matches('),function('afk.h','static inline bool afk_service_reclaimable('), function('afk.h','static inline bool\nafk_service_reinit('),function('afk.h','static inline void afk_service_discard('), function('afk.h','static inline void afk_service_publish('), function('epic/dpavservep.c','static void dcpavserv_notify_edid('),function('epic/dpavservep.c','static bool dcpavserv_identity('), function('epic/dpavservep.c','static bool dcpavserv_admission_current('),function('epic/dpavservep.c','static int dcpavserv_prepare('), - function('epic/dpavservep.c','static void dcpavserv_init('),function('epic/dpavservep.c','static void dcpavserv_invalidate_edid('), + function('epic/dpavservep.c','static int dcpavserv_revalidate('),function('epic/dpavservep.c','static void dcpavserv_init('),function('epic/dpavservep.c','static void dcpavserv_invalidate_edid('), function('epic/dpavservep.c','static void dcpavserv_teardown('),function('epic/dpavservep.c','void dpavservep_detach('), function('afk.c','static const struct apple_epic_service_ops *\nafk_match_service('), function('afk.c','static struct apple_epic_service *afk_epic_find_service('), - function('afk.c','static void afk_recv_handle_init('), + function('afk.c','static void afk_discard_pending_open('),function('afk.c','static void afk_retry_service_init('),function('afk.c','static void afk_service_worker('),function('afk.c','static void afk_recv_handle_init('), function('epic/dpavservep.c','static int dcpdpserv_prepare('), function('epic/dpavservep.c','static void dcpdpserv_teardown('),function('afk.c','static void afk_recv_handle_teardown('), function('afk.c','static void afk_recv_handle_reply('),function('afk.c','static void afk_v2_receive(')] @@ -152,9 +161,10 @@ def function(file,signature): static struct apple_dcp d; static struct apple_dcp_afkep ep; static struct apple_connector con; -static const struct apple_epic_service_ops ops={.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.init=dcpavserv_init,.teardown=dcpavserv_teardown}; +static void run_retries(void) { assert(!ep.lock);if(ep.service_work.pending) { ep.service_work.pending=false;afk_service_worker(&ep.service_work); } } +static const struct apple_epic_service_ops ops={.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.revalidate=dcpavserv_revalidate,.init=dcpavserv_init,.teardown=dcpavserv_teardown}; static const struct apple_epic_service_ops announcement_ops[]={ - {.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.init=dcpavserv_init,.teardown=dcpavserv_teardown}, + {.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.revalidate=dcpavserv_revalidate,.init=dcpavserv_init,.teardown=dcpavserv_teardown}, {.name="dcpdp-service-epic",.reusable=true,.prepare=dcpdpserv_prepare,.teardown=dcpdpserv_teardown}, {0} }; @@ -268,14 +278,39 @@ def function(file,signature): assert(injected_owner->users==1);dpavservep_detach(&d);assert(!injected_owner->users); open_action=0;announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service); struct apple_epic_service *old=d.dcpavserv.service; - assert(afk_service_get(old)==old);old->cmd_map[0]=1; + assert(afk_service_get(old)==old);ep.command_tags[13]=1;old->cmd_map[0]=1; disconnect();connect(); announce(13,name,native_class,13); /* Same channel still owns a pending reply. */ assert(opens==before && !d.dcpavserv.service && old->users==1 && old->channel==13); + /* The firmware need not announce again after the old reply drains. */ + old->cmds[0]=(struct epic_cmd_info){.tag=0,.type=0xc0,.rxlen=32,.txlen=32,.free_on_ack=true}; + old->cmds[0].rxbuf=calloc(1,32);old->cmds[0].txbuf=calloc(1,32); + old->cmds[0].rxbuf_dma=(uintptr_t)old->cmds[0].rxbuf; + old->cmds[0].txbuf_dma=(uintptr_t)old->cmds[0].txbuf; + u8 drain_oob[32],drain_reply[56];size_t drain_written; + assert(!afkv2_encode_oob_command(drain_oob,sizeof(drain_oob),0,old->cmds[0].rxbuf_dma,32,old->cmds[0].txbuf_dma,32)); + afkv2_put32(drain_oob+4,0); + assert(!afkv2_encode_frame(drain_reply,sizeof(drain_reply),1,13,0,0xc0,AFKV2_RESPONSE,0,drain_oob,sizeof(drain_oob),&drain_written)); + afk_v2_receive(&ep,0,0,drain_reply,drain_written); + run_retries(); + assert(d.dcpavserv.service && "a blocked reannouncement must recover after the old ACK without another announcement"); + assert(d.dcpavserv.service!=old && opens==++before && old->users==1 && !old->cmd_map[0]); + struct apple_epic_service *resumed=d.dcpavserv.service; + resumed->cmd_map[0]=1; + /* The DMA allocator can reuse both old addresses for the new command. */ + resumed->lock=true; + resumed->cmds[0]=(struct epic_cmd_info){.tag=afk_command_tag(resumed,0),.type=0xc0, + .rxbuf_dma=old->cmds[0].rxbuf_dma,.txbuf_dma=old->cmds[0].txbuf_dma,.rxlen=32,.txlen=32}; + resumed->lock=false; + afk_v2_receive(&ep,0,0,drain_reply,drain_written); + assert(!resumed->cmds[0].done && resumed->cmd_map[0] && "old ACK must not complete the next owner's command"); + resumed->cmd_map[0]=0; + dpavservep_detach(&d); announce(1,name,class,1);assert(opens==++before && d.dcpavserv.service!=old); struct apple_epic_service *fresh=d.dcpavserv.service;u64 epoch=d.dcpavserv.epoch; dcpavserv_teardown(old);assert(d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch); old->cmd_map[0]=0;afk_service_put(old);disconnect();connect(); + run_retries(); announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service); struct apple_epic_service *owner=d.dcpavserv.service; u64 owner_epoch=d.dcpavserv.epoch; @@ -294,7 +329,7 @@ def function(file,signature): dpavservep_detach(&d); assert(!d.hpd_mutex && !d.dcpavserv.lock); /* Legacy remains unit-zero and class-agnostic, with its original epoch fence. */ - ep.wire_v2=false;ep.endpoint=0x28; + ep.wire_v2=false;ep.endpoint=0x28;d.external_physically_absent=true; assert(!prepare(13,13,name,native_class)); struct apple_epic_service *legacy=prepare(13,0,name,native_class);assert(legacy); dpavservep_detach(&d);afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); @@ -303,11 +338,82 @@ def function(file,signature): d.typec_generation++;afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); assert(d.dcpavserv.service==legacy && legacy->users==1);dpavservep_detach(&d); assert(dcpdpserv_prepare(legacy,"dcpdp-service-epic",native_class,0)==0); - ep.wire_v2=true;ep.endpoint=0x2a; + ep.wire_v2=true;ep.endpoint=0x2a;d.external_physically_absent=false; announce(U16_MAX,name,native_class,U16_MAX);assert(d.dcpavserv.service && d.dcpavserv.service->channel==U16_MAX); dpavservep_detach(&d); puts("PASS: actual AFKV2 HDMI/native DP alternating over 64 cycles: flip/revalidation/return, strict identity, stale OPEN generation/epoch/owner, auxiliary isolation, reader/command pins and legacy controls"); } + +static void deferred_announcements(void) +{ + for(unsigned int cycle=0;cycle<64;cycle++) { + unsigned int event=cycle%10,before=opens,freed=dma_frees; + u16 channel=1+4*(cycle%4); + const char *provider=(cycle%2)?native_class:class; + announce(channel,name,provider,channel); + struct apple_epic_service *old=d.dcpavserv.service; + assert(old && opens==++before && afk_service_get(old)==old); + old->cmd_map[0]=1; + old->cmds[0]=(struct epic_cmd_info){.tag=0,.type=0xc0,.rxlen=32,.txlen=32,.free_on_ack=true}; + old->cmds[0].rxbuf=calloc(1,32);old->cmds[0].txbuf=calloc(1,32); + old->cmds[0].rxbuf_dma=(uintptr_t)old->cmds[0].rxbuf; + old->cmds[0].txbuf_dma=(uintptr_t)old->cmds[0].txbuf; + u8 oob[32],reply[56];size_t written; + assert(!afkv2_encode_oob_command(oob,sizeof(oob),0,old->cmds[0].rxbuf_dma,32,old->cmds[0].txbuf_dma,32)); + afkv2_put32(oob+4,0); + assert(!afkv2_encode_frame(reply,sizeof(reply),1,channel,0,0xc0,AFKV2_RESPONSE,0,oob,sizeof(oob),&written)); + u64 generation=d.typec_generation; + closed(channel,0x6a01,0); + assert(!d.dcpavserv.service && old->users==1 && old->torndown && d.typec_generation==generation); + u64 epoch=d.dcpavserv.epoch; + announce(channel,name,provider,channel); + run_retries(); + assert(opens==before && !d.dcpavserv.service && dma_frees==freed); + for(unsigned int i=0;ipending_open) continue; + assert(dcpavserv_revalidate(pending,pending->pending_name,pending->pending_class,pending->pending_unit)==0); + d.dcpavserv.enabled=true; + assert(dcpavserv_revalidate(pending,pending->pending_name,pending->pending_class,pending->pending_unit)==-EBUSY); + d.dcpavserv.enabled=false; + } + if(event==1) d.typec_generation++; + if(event==2) dpavservep_detach(&d); + if(event==3) d.typec_cable_connected=false; + if(event==4) d.quiescing=true; + struct apple_epic_service *other=NULL; + if(event==5) { other=prepare(channel+2,channel+2,name,class);publish(other,channel+2); } + if(event==6) assert(!ep.stopping); + if(event==7) open_error=-EIO; + if(event==8) announce(channel,name,provider,channel); + if(event==9) { d.typec_generation++;announce(channel,name,provider,channel); } + afk_v2_receive(&ep,0,0,reply,written); + if(event==6) { assert(ep.service_work.pending);ep.stopping=true; } + run_retries(); + assert(dma_frees==freed+2 && old->users==1 && !old->cmd_map[0]); + if(event==0 || event==8 || event==9) { + assert(d.dcpavserv.service && d.dcpavserv.service!=old); + assert(d.dcpavserv.service->channel==channel && opens==++before); + assert(d.dcpavserv.epoch>epoch && d.dcpavserv.service->users==1); + } else if(event==5) { + assert(d.dcpavserv.service==other && opens==before); + } else { + assert(!d.dcpavserv.service && opens==before+(event==7)); + } + afk_v2_receive(&ep,0,0,reply,written); /* Duplicate late ACK cannot touch the new owner. */ + assert(dma_frees==freed+2); + afk_service_put(old);run_retries(); + assert(!old->users); + if(event==6) { + for(unsigned int i=0;iusers==1); assert(afk_service_get(old)==old); /* Delayed EDID reader pins the retired slot. */ @@ -375,6 +482,8 @@ def function(file,signature): assert(!pending->users && !d.hpd_mutex && !d.dcpavserv.lock); memset(&ep.services,0,sizeof(ep.services));ep.wire_v2=true;ep.endpoint=0x2a;d.hw.iomfb_method_profile=2; repeated_announcements(); + deferred_announcements(); + free(ep.command_tags);ep.command_tags=NULL; puts("PASS: actual dynamic 1/5/9/13 binding, forged identities, prepare/init epochs, physical retirement, reader pins, real old CLOSED/late ACK isolation and legacy unit 0"); } ''' @@ -461,7 +570,7 @@ def function(file, signature): struct { bool lock, enabled; u64 epoch; int enable_completion; struct apple_epic_service *service; } dcpavserv; struct apple_connector *connector; unsigned int nr_modes; struct { int iomfb_method_profile; } hw; - bool hpd_mutex, typec_cable_connected, quiescing; + bool external_physically_absent,hpd_mutex, typec_cable_connected, quiescing; u64 typec_generation; int fw_compat; void *dev; @@ -571,6 +680,8 @@ def function(file, signature): d.dcpavserv.service=&s; d.dcpavserv.enabled=true; s.enabled=true; s.users=1; event=0; d.typec_cable_connected=false; e=dcpavserv_copy_edid(&d,0,&epoch); assert(!IS_ERR(e) && legacy_reads==1 && s.users==1); drm_edid_free(e); + d.hw.iomfb_method_profile=0;av.wire_v2=false;d.external_physically_absent=true; + mutex_lock(&d.hpd_mutex);assert(dcpavserv_edid_current(&d,0));mutex_unlock(&d.hpd_mutex); assert(!allocations && !d.hpd_mutex && !d.dcpavserv.lock); puts("PASS: production old/new lookup and receive, exact service admission before OPEN, delayed disconnect/replug, service detach pins, legacy transport"); } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py b/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py new file mode 100644 index 0000000000000..310af408b0f96 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py @@ -0,0 +1,439 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright 2026 Ryan Murray +"""Retained reviewer controls: actual AFK sender/receive, queue and EDID functions. + +The wrap controls deliberately retain an expected replay assertion failure. +They document an unresolved peer contract, not successful replay protection. +""" +from pathlib import Path +import json, hashlib, subprocess + +def check(root, output): + import edid + out = output / 'service-recovery-controls' + def extract(text,signature): + start=text.index(signature);opening=text.index('{',start) + while ';' in text[start:opening]: + start=text.index(signature,start+1);opening=text.index('{',start) + end,depth=opening+1,1 + while depth: + depth+=(text[end]=='{')-(text[end]=='}');end+=1 + return text[start:end] + + unit=edid.rebind(root) + # Retain the original exact fixture separately. Correct only its causal ordering. + corrected=unit.replace('assert(afk_service_get(old)==old);old->cmd_map[0]=1;', + 'assert(afk_service_get(old)==old);old->cmd_tag=1;old->cmd_map[0]=1;') + corrected=corrected.replace(' old->cmd_tag=1;\n','') + + sender=extract((root/'drivers/gpu/drm/apple/afk.c').read_text(), + 'static int afk_send_command_timeout_len(') + unit=unit[:unit.index('int main(void)')] + unit=unit.replace('static unsigned int dma_frees,opens;',r''' + static unsigned int dma_frees,opens; + #define SZ_1M (1U<<20) + #define GFP_KERNEL 0 + #define DECLARE_COMPLETION_ONSTACK(c) int c=0 + #define init_completion(c) (*(c)=0) + #define msecs_to_jiffies(x) (x) + #define cpu_to_le64(x) (x) + #define kmalloc(n,g) malloc(n) + #define EPIC_TYPE_COMMAND 3 + #define EPIC_CAT_COMMAND 3 + static int bitmap_find_free_region(unsigned long *p,int n,int order) { + (void)order;for(int i=0;iinline_v2) { + struct afkv2_command decoded;u8 response[40]={0}; + assert(!afkv2_decode_command(p,n,&decoded) && !decoded.oob && decoded.value==32 && decoded.id==tag); + response[1]=decoded.id; + assert(!afkv2_encode_frame(current_reply,sizeof(current_reply),1,channel,0,subtype,AFKV2_RESPONSE,0,response,sizeof(response),¤t_size)); + current_channel=channel;current_tag=tag;return 0; + } + assert(n==sizeof(struct epic_cmd)); + const struct epic_cmd *cmd=p;u8 oob[32]; + assert(!afkv2_encode_oob_command(oob,sizeof(oob),tag,cmd->rxbuf,cmd->rxlen,cmd->txbuf,cmd->txlen)); + afkv2_put32(oob+4,0); + assert(!afkv2_encode_frame(current_reply,sizeof(current_reply),1,channel,0,subtype,AFKV2_RESPONSE,0,oob,sizeof(oob),¤t_size)); + current_channel=channel;current_tag=tag;return 0; + } + '''+marker) + + tests=r''' + static void setup(void) { + d.hw.iomfb_method_profile=2;d.typec_cable_connected=true;d.typec_generation=1; + d.nr_modes=1;d.connector=&con;d.dev=&d; + con.dcp=&d;con.connected=true;con.base.registration_state=1; + ep.dcp=&d;ep.wire_v2=true;ep.endpoint=0x2a;ep.wq=&ep;ep.command_tags=calloc(AFKV2_INTERFACES,1);assert(ep.command_tags); + d.dcpavservep=&ep;ep.ops=announcement_ops;ep.match_epic_name=true; + d.fw_compat=DCP_FIRMWARE_H17P;hpd_lock=&d.hpd_mutex;av_lock=&d.dcpavserv.lock;active_ep=&ep; + } + static int send_actual(struct apple_epic_service *s,bool ack,bool late) { + u8 payload[32]={0},output[32]={0};u32 code; + ack_current=ack;inject_late=late; + int result=afk_send_command_timeout_len(s,0xc0,payload,32,output,32,&code,1,NULL); + inject_late=false;return result; + } + static void save_ack(void) { memcpy(saved_reply,current_reply,current_size);saved_size=current_size; } + static void cleanup(void) { + dpavservep_detach(&d);run_retries(); + for(unsigned int i=0;ichannel==5); + assert(send_actual(middle,true,true)==0); + closed(5,0x6a01,0);announce(13,name,native_class,13);run_retries(); + assert(d.dcpavserv.service && d.dcpavserv.service==middle && d.dcpavserv.service!=old); + assert(send_actual(d.dcpavserv.service,true,true)==0); + afk_service_put(old); + } else if(!strcmp(argv[1],"multiple-first-late-acks")) { + reuse_dma=false;assert(afk_service_get(old)==old); + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + u8 first[56];size_t first_size=saved_size;memcpy(first,saved_reply,first_size); + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + assert(old->cmd_map[0]==3); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries();assert(!d.dcpavserv.service); + afk_v2_receive(&ep,0,0,first,first_size);run_retries(); + assert(!d.dcpavserv.service && old->cmd_map[0]==2 && old->users==1); + unsigned int freed=dma_frees; + afk_v2_receive(&ep,0,0,first,first_size);run_retries(); + assert(dma_frees==freed && old->cmd_map[0]==2 && !d.dcpavserv.service); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(d.dcpavserv.service && d.dcpavserv.service!=old && !old->cmd_map[0]); + afk_service_put(old); + } else if(!strcmp(argv[1],"tag-wrap") || !strcmp(argv[1],"inline-tag-wrap")) { + assert(send_actual(old,true,false)==0);save_ack(); + for(unsigned int i=0;i<15;i++) assert(send_actual(old,true,true)==0); + assert(send_actual(old,true,true)==0); + } else if(!strcmp(argv[1],"pending-closed")) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries(); + assert(!d.dcpavserv.service); + closed(13,0x6a01,0); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(!d.dcpavserv.service && "CLOSED during pending recovery must invalidate the pending OPEN"); + } else if(!strcmp(argv[1],"actual-tag-before-announcement")) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack();assert(ep.command_tags[13]==1); + closed(13,0x6a01,0);announce(13,name,native_class,13); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(d.dcpavserv.service && send_actual(d.dcpavserv.service,true,true)==0); + } else if(!strncmp(argv[1],"open-race-",10)) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries(); + if(!strcmp(argv[1],"open-race-generation")) open_action=1; + else if(!strcmp(argv[1],"open-race-epoch")) open_action=2; + else if(!strcmp(argv[1],"open-race-owner")) open_action=4; + else if(!strcmp(argv[1],"open-race-cable")) open_action=5; + else if(!strcmp(argv[1],"open-race-quiescing")) open_action=6; + else if(!strcmp(argv[1],"open-race-absence")) open_action=7; + else assert(!"unknown OPEN race"); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + if(open_action==4) assert(d.dcpavserv.service==injected_owner); + else assert(!d.dcpavserv.service); + open_action=0;d.quiescing=false;d.typec_cable_connected=true;d.external_physically_absent=false; + } else if(!strcmp(argv[1],"physical-absence")) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries(); + d.external_physically_absent=true; + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(!d.dcpavserv.service && "physical absence authority must fence deferred OPEN before cable/generation update"); + } else if(!strncmp(argv[1],"fence-",6)) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries(); + unsigned int before=opens; + if(!strcmp(argv[1],"fence-epoch")) dpavservep_detach(&d); + else if(!strcmp(argv[1],"fence-generation")) d.typec_generation++; + else if(!strcmp(argv[1],"fence-cable")) d.typec_cable_connected=false; + else if(!strcmp(argv[1],"fence-quiescing")) d.quiescing=true; + else if(!strcmp(argv[1],"fence-owner")) inject_owner(&ep,13); + else if(!strcmp(argv[1],"fence-open-failure")) open_error=-EIO; + else assert(!"unknown fence"); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(opens==before+(!strcmp(argv[1],"fence-open-failure"))); + if(!strcmp(argv[1],"fence-owner")) assert(d.dcpavserv.service==injected_owner); + else assert(!d.dcpavserv.service); + for(unsigned int i=0;ifn=(f);(w)->pending=false;(w)->next=NULL; } while(0) + static struct work_struct *fifo_head,*fifo_tail;static bool destroyed,rx_running; + ''') + queued=queued.replace('static bool queue_work(void *wq,struct work_struct *w) { assert(wq);bool fresh=!w->pending;w->pending=true;return fresh; }',r''' + static bool queue_work(void *wq,struct work_struct *w) { + assert(wq && !destroyed);if(w->pending) return false; + w->pending=true;w->next=NULL;if(fifo_tail) fifo_tail->next=w;else fifo_head=w;fifo_tail=w;return true; + } + static void run_one_work(void) { + assert(fifo_head);struct work_struct *w=fifo_head;fifo_head=w->next;if(!fifo_head) fifo_tail=NULL; + w->pending=false;w->next=NULL;assert(w->fn);w->fn(w); + } + static void destroy_workqueue(void *wq) { assert(wq && !destroyed);while(fifo_head) run_one_work();destroyed=true; } + #define IS_ERR_OR_NULL(p) (!(p) || IS_ERR(p)) + #define debugfs_remove_recursive(p) ((void)(p)) + #define kzalloc(n,g) calloc(1,n) + #define kvfree(p) free(p) + #define ESHUTDOWN 108 + #define RBEP_TYPE 0xffff + #define FIELD_GET(mask,val) ((val)&(mask)) + #define RBEP_INIT_ACK 0xa0 + #define RBEP_START_ACK 0x86 + #define RBEP_SHUTDOWN_ACK 0xc1 + #define RBEP_GETBUF 0x89 + #define RBEP_INIT_TX 0x8a + #define RBEP_INIT_RX 0x8b + #define RBEP_RECV 0x85 + #define complete_all(p) complete(p) + ''') + queued=queued.replace('struct apple_dcp_afkep { u8 *command_tags;bool lock,stopping;', + 'struct apple_dcp_afkep { u8 *command_tags;int started,stopped,txbfr,rxbfr;void *debugfs_entry;bool lock,stopping;') + queued=queued.replace('bool external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;', + 'bool retain_dma,external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;') + queued=queued.replace('static void run_retries(void) { assert(!ep.lock);if(ep.service_work.pending) { ep.service_work.pending=false;afk_service_worker(&ep.service_work); } }', + 'static void run_retries(void) { assert(!ep.lock);while(fifo_head) run_one_work(); }') + queued=queued.replace(' if(type==EPIC_TYPE_COMMAND) {', + ' if(e->stopping) return -ESHUTDOWN;\n if(type==EPIC_TYPE_COMMAND) {') + queued=queued[:queued.index('int main(int argc')] + queued+=r''' + struct afk_receive_message_work { struct apple_dcp_afkep *ep;u64 message;struct work_struct work; }; + static void afk_getbuf(struct apple_dcp_afkep *e,u64 m) { (void)e;(void)m;assert(!"unexpected GETBUF"); } + static void afk_init_rxtx(struct apple_dcp_afkep *e,u64 m,int *r) { (void)e;(void)m;(void)r;assert(!"unexpected INIT"); } + static unsigned int rx_step;static bool close_pending; + static bool afk_recv(struct apple_dcp_afkep *e) { + assert(e==&ep);rx_running=true; + if(rx_step==0) closed(13,0x6a01,0); + else if(rx_step==1) announce(13,name,native_class,13); + else if(rx_step==2) afk_v2_receive(e,0,0,saved_reply,saved_size); + else if(rx_step==3 && close_pending) closed(13,0x6a01,0); + else { assert(!d.dcpavserv.service && "recovery must wait for the RX drain loop to finish");rx_running=false;return false; } + assert(!d.dcpavserv.service && "recovery must remain behind RX on the ordered queue");rx_step++;return true; + } + ''' + af=(root/'drivers/gpu/drm/apple/afk.c').read_text() + for signature in ['static void afk_receive_message_worker(', 'int afk_receive_message(', 'void afk_quiesce(', 'static void afk_release_context(']: + queued+=extract(af,signature)+'\n' + queued+=r''' + int main(int argc,char **argv) { + assert(argc==2);setup();reuse_dma=true;INIT_WORK(&ep.service_work,afk_service_worker); + announce(13,name,native_class,13);struct apple_epic_service *old=d.dcpavserv.service;assert(old); + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack();run_retries(); + unsigned int before=opens; + if(!strncmp(argv[1],"ordered-",8)) { + close_pending=!strcmp(argv[1],"ordered-pending-closed"); + assert(!afk_receive_message(&ep,RBEP_RECV));assert(fifo_head); + run_one_work();assert(!rx_running && opens==before && !d.dcpavserv.service); + assert(fifo_head==&ep.service_work);run_retries(); + if(close_pending) assert(!d.dcpavserv.service && "ordered CLOSED must cancel pending OPEN"); + else { assert(d.dcpavserv.service && opens==before+1);assert(send_actual(d.dcpavserv.service,true,true)==0); } + cleanup(); + } else { + assert(!strcmp(argv[1],"quiesce-pending"));assert(afk_service_get(old)==old); + closed(13,0x6a01,0);announce(13,name,native_class,13); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);assert(ep.service_work.pending); + afk_quiesce(&ep);assert(destroyed && !ep.wq && ep.stopping && !fifo_head); + assert(opens==before && !d.dcpavserv.service && old->users==1); + for(unsigned int i=0;iusers); + assert(afk_receive_message(&ep,RBEP_RECV)==-ESHUTDOWN);assert(!fifo_head); + afk_quiesce(&ep);assert(!fifo_head); + struct apple_dcp_afkep *empty=calloc(1,sizeof(*empty));assert(empty);empty->dcp=&d; + afk_release_context(empty); + cleanup(); + } + printf("PASS actual RX worker/quiesce: %s\n",argv[1]); + } + ''' + run('actual-queue-quiesce',queued,[[c] for c in ['ordered-recovery','ordered-pending-closed','quiesce-pending']]) + + # Actual EDID wrapper exercises its owner/reader pins, epoch and generation exits. + reader=unit[:unit.index('int main(int argc')] + reader+=r''' + #define ESTALE 116 + struct drm_edid { int marker; }; + static unsigned int edid_allocations,edid_frees;static int read_action; + struct reader_guard { bool *lock;bool once; }; + static struct reader_guard lock_reader_guard(bool *lock) { mutex_lock(lock);return (struct reader_guard){lock,true}; } + static void unlock_reader_guard(struct reader_guard *guard) { mutex_unlock(guard->lock); } + #define scoped_guard(kind,p) for(struct reader_guard _guard __attribute__((cleanup(unlock_reader_guard)))=lock_reader_guard(p);_guard.once;_guard.once=false) + static void drm_edid_free(const struct drm_edid *edid) { assert(edid && !IS_ERR(edid));free((void *)edid);edid_frees++; } + static const struct drm_edid *dcpavserv_read_edid_inline(struct apple_epic_service *service) { + assert(service->users==2 && "owner plus actual EDID wrapper reader pin"); + assert(send_actual(service,true,false)==0); + if(read_action==1) { closed(service->channel,0x6a01,0);announce(5,name,class,5);assert(d.dcpavserv.service!=service); } + if(read_action==2) d.typec_generation++; + if(read_action==3) d.external_physically_absent=true; + if(read_action==4) { dpavservep_detach(&d);d.quiescing=true; } + struct drm_edid *edid=malloc(sizeof(*edid));assert(edid);edid->marker=1;edid_allocations++;return edid; + } + static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *service) { return dcpavserv_read_edid_inline(service); } + ''' + dpav=(root/'drivers/gpu/drm/apple/epic/dpavservep.c').read_text() + for signature in ['bool dcpavserv_edid_current(', 'bool dcpavserv_edid_service_current(', 'const struct drm_edid *dcpavserv_copy_edid(']: + reader+=extract(dpav,signature)+'\n' + reader+=r''' + int main(int argc,char **argv) { + assert(argc==2);setup();reuse_dma=true;announce(13,name,class,13); + struct apple_epic_service *old=d.dcpavserv.service;assert(old); + if(!strcmp(argv[1],"reader-owner")) read_action=1; + else if(!strcmp(argv[1],"reader-generation")) read_action=2; + else if(!strcmp(argv[1],"reader-physical-absence")) read_action=3; + else if(!strcmp(argv[1],"reader-quiescing")) read_action=4; + else assert(!strcmp(argv[1],"reader-success")); + u64 epoch;const struct drm_edid *edid=dcpavserv_copy_edid(&d,1,&epoch); + if(!read_action) { assert(!IS_ERR(edid));drm_edid_free(edid); } + else { assert(IS_ERR(edid) && "actual EDID wrapper must reject invalidated ownership/physical state"); } + if(read_action==1 || read_action==4) assert(!old->users);else assert(old->users==1); + assert(edid_allocations==edid_frees && !d.hpd_mutex && !d.dcpavserv.lock); + d.quiescing=false;cleanup();printf("PASS actual copy_edid: %s\n",argv[1]); + } + ''' + run('actual-edid-reader',reader,[[c] for c in ['reader-success','reader-owner','reader-generation','reader-quiescing','reader-physical-absence']]) + mutations=[ + ('restart-tags','afkv2_command_id(ep->command_tags[service->channel]++, slot)','afkv2_command_id(0, slot)'), + ('no-drain-retry','\tafk_service_retry(service->ep);',''), + ('ignore-old-command','\t\tif (afk_epic_find_service(ep, service->channel))\n\t\t\tcontinue;',''), + ('ignore-epoch','service->admission.epoch != dcp->dcpavserv.epoch ||\n\t\t ','false ||\n\t\t '), + ('refresh-generation','!dcpavserv_admission_current(dcp, service->admission.generation))\n\t\tret = -ENODEV;','!dcpavserv_admission_current(dcp, dcp->typec_generation))\n\t\tret = -ENODEV;'), + ('ignore-owner','if (dcp->dcpavserv.enabled)\n\t\tret = -EBUSY;\n\telse if (service->admission.epoch','if (false)\n\t\tret = -EBUSY;\n\telse if (service->admission.epoch'), + ('ignore-stop','if (!READ_ONCE(ep->stopping))\n\t\tafk_retry_service_init(ep);','if (true)\n\t\tafk_retry_service_init(ep);'), + ] + + for name,text,case in [ + ('cancel-closed',unit.replace('if (pending->pending_open && pending->channel == channel)','if (false && pending->pending_open && pending->channel == channel)'), 'pending-closed'), + ('reader-lookup',unit.replace('(service->enabled ||','(service->users || service->enabled ||'), 'drained-reader-rebind'), + ('reset-interface-history',unit.replace('service->cookie = NULL;', 'service->cookie = NULL; if(ep->wire_v2 && ep->command_tags) ep->command_tags[channel]=0;',1), 'channel-reuse'), + ('absence-admission',unit.replace('!READ_ONCE(dcp->external_physically_absent) &&','true &&',1), 'initial-absence'), + ('absence-publication',unit.replace('!READ_ONCE(dcp->external_physically_absent) &&','true &&',1), 'initial-open-absence'), + ('absence-reader',reader.replace('!READ_ONCE(dcp->external_physically_absent)', 'true'), 'reader-physical-absence'), + ]: + run('mutation-'+name,text,[[case]]) + + for name,old,new in mutations: + assert corrected.count(old)==1,(name,corrected.count(old)) + run('mutation-'+name,corrected.replace(old,new),[[]]) + negative={ + 'actual-sender':{'tag-wrap','inline-tag-wrap'}, + 'actual-queue-quiesce':set(), + 'actual-edid-reader':set(), + } + for result in results: + case=result.get('case',[]) + fail_expected=bool('case' in result and (result['name'].startswith('mutation-') or (case and case[0] in negative.get(result['name'],set())))) + result['assertion_failure_expected']=fail_expected + result['expectation_met']=result['exit']==(-6 if fail_expected else 0) + if fail_expected: result['expectation_met'] = result['expectation_met'] and 'Assertion' in result.get('output','') + if case and case[0] in {'tag-wrap','inline-tag-wrap'}: + result['expectation_met'] = result['expectation_met'] and 'late ACK completed an actual new production command' in result.get('output','') + (out/'results.json').write_text(json.dumps(results,indent=2)+'\n') + + assert all(r['expectation_met'] for r in results), "retained recovery control expectation failed; see results.json" diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index fe62df8d78129..94349823fa03c 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -18,6 +18,7 @@ from common import run_unit import edid +import recovery import lifecycle import phy import pm @@ -90,6 +91,8 @@ def main(): unit, cases = unit if unit is not None: result['translation_unit_sha256'] = run_unit(name, unit, output, cases, includes if name == 'service-rebind' else (), flags) + if name == 'service-rebind': + recovery.check(root, output) if name == 'metadata-decoder': rate.check_metadata(output) result['passed'] = True From 83adf0412237691b61b804c08a9e39ccd95116c5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 23:21:04 +0000 Subject: [PATCH 0965/1579] dt-bindings: display: apple: allow T8140 provider data on DCPEXT The shared H17P provider callback consumes the same five opaque, exact-size table inputs for T8140 DCP and DCPEXT. Permit them on both controllers; keep the existing byte lengths and restrictions on every other compatible. No requiredness or runtime consumer change is made. Independent validation reproduces five DCPEXT errors on the parent and passes the correction, 113 contract controls and dt-doc validation. Live firmware presence and loader/FDT qualification remain separate. Signed-off-by: Ryan Murray Assisted-by: LLM --- Documentation/devicetree/bindings/display/apple,dcp.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index bba5d64f50451..0fe31267f99e1 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -320,7 +320,7 @@ allOf: properties: compatible: contains: - const: apple,t8140-dcp + enum: ['apple,t8140-dcp', 'apple,t8140-dcpext'] then: properties: apple,power-lut-data-x: false From 6f004361c61bf176e5cd502c59b531e2e8c648e7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 14 Oct 2026 01:22:15 +0000 Subject: [PATCH 0966/1579] usb: typec: serialize display teardown and preserve recovery ownership Quiesce the display before SAFE provider writes and role-clock shutdown. Serialize connector transitions across mux handles, retain failed SAFE ownership, and check finalization before admitting a new DP route. Keep SRCU reader accounting local and preserve module/provider lifetime. Track consumed mux release separately so retries cannot release twice. Refuse reset when provider acquisition cannot establish SAFE. Retain compact actual-source regression coverage for transaction ordering, error paths, reference lifetime and release accounting. Independent review passes all five retained failure witnesses. Complete ARM64 Image/modules/ DTB build passes. Target recovery qualification is pending; this is not a hardware hotplug or suspend/wake pass. Signed-off-by: Ryan Murray Assisted-by: LLM --- drivers/gpu/drm/apple/dcp-internal.h | 3 + drivers/gpu/drm/apple/dcp.c | 133 +++++- drivers/usb/typec/mux.c | 228 +++++++++- drivers/usb/typec/mux.h | 6 + drivers/usb/typec/tipd/core.c | 130 ++++-- drivers/usb/typec/tipd/tps6598x.h | 5 +- include/linux/usb/typec_mux.h | 32 ++ .../drivers/gpu/apple-dcp/accounting.py | 318 ++++++++++++++ .../selftests/drivers/gpu/apple-dcp/pm.py | 24 +- .../drivers/gpu/apple-dcp/quiesce.py | 147 +++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 11 +- .../drivers/gpu/apple-dcp/transaction.py | 406 ++++++++++++++++++ 12 files changed, 1390 insertions(+), 53 deletions(-) create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/accounting.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index eeea8591b0466..95881b9cb79fe 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -70,6 +70,9 @@ struct apple_dcp_typec_route { bool tunnel; /* tunnel: crossbar brought up (at DidChangeLinkConfiguration) */ bool xbar_up; + /* Lower mux release is consumed even when idle programming fails. */ + bool deactivate_xbar_done; + int deactivate_error; }; bool dcp_is_typec_output(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 06e7e3e78ce1c..ca5988f81fe20 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -210,6 +210,9 @@ struct apple_dcp_typec_port { u32 applied_status; u32 applied_conf; bool applied_valid; + /* SAFE preparation retains this owner until the set phase completes. */ + bool safe_pending; + bool safe_finish_ready; bool hpd; /* Direct DP-alt: the port is in DP mode, routed or not, and its HPD. */ bool dp_wanted; @@ -262,6 +265,13 @@ static bool dcp_typec_route_fixed_output_busy(struct apple_dcp_typec_route *rout static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) { + struct apple_dcp_typec_port *port; + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port->safe_pending && port->owner && + port->owner->dcp == route->dcp) + return false; + } return !route->dcp->active_typec_route && !dcp_typec_route_fixed_output_busy(route); } @@ -363,9 +373,9 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, if (dcp->fixed_route_selected) { ret = mux_control_deselect(dcp->xbar); + dcp->fixed_route_selected = false; if (ret) return ret; - dcp->fixed_route_selected = false; } /* @@ -427,6 +437,8 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, } dcp->active_typec_route = route; scoped_guard(mutex, &dcp->tb_lock) { + route->deactivate_xbar_done = false; + route->deactivate_error = 0; route->active_xbar = xbar; route->tunnel = xbar != route->xbar; route->xbar_up = !route->tunnel; @@ -461,13 +473,22 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) * crossbar nor restart the tunnel pixel clock behind our back. */ scoped_guard(mutex, &dcp->tb_lock) { - if (!route->tunnel || route->xbar_up) - ret = mux_control_deselect(route->active_xbar ?: route->xbar); - else - dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); - if (ret) - return ret; - route->xbar_up = false; + if (route->deactivate_error) + return route->deactivate_error; + if (!route->deactivate_xbar_done) { + if (!route->tunnel || route->xbar_up) + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + else + ret = dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); + /* Deselect always unlocks: never consume that selection twice. */ + route->deactivate_xbar_done = true; + route->xbar_up = false; + if (ret) { + /* Idle is unproven; retain the owner and the real error. */ + route->deactivate_error = ret; + return ret; + } + } if (route->tunnel && dcp->phy) { /* the tunnel pixel clock must not outlive the tunnel */ @@ -537,7 +558,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) dcp->phy = dcp->fixed_phy; dcp->dptx_phy = dcp->fixed_dptx_phy; - if (dcp->xbar) { + if (dcp->xbar && !dcp->fixed_route_selected) { ret = mux_control_select(dcp->xbar, dcp->fixed_mux_index); if (ret) @@ -856,6 +877,8 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, struct apple_dcp_typec_route *owner = port->owner; struct apple_dcp *dcp; + if (port->safe_pending) + continue; if (!owner || owner->tunnel || owner == port->target) continue; @@ -940,6 +963,86 @@ void dcp_typec_reorder(void) dcp_typec_rebalance_locked(NULL, 0); } +static int dcp_typec_route_prepare(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); + struct apple_dcp_typec_port *port = route->port; + struct apple_dcp *dcp; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE) + return port->safe_pending ? -EBUSY : 0; + if (!port->owner || port->owner->tunnel) + return 0; + dcp = port->owner->dcp; + /* Stop new admission, but retain the owner until the set phase succeeds. */ + if (!port->safe_pending) + port->safe_finish_ready = false; + port->safe_pending = true; + port->applied_valid = false; + port->dp_wanted = false; + port->dp_hpd = false; + /* HPD absence alone is not proof of a completed firmware shutdown. */ + if (dcp_is_external_h17p(dcp)) { + scoped_guard(mutex, &dcp->external_lock) { + if (dcp->external_error) + return dcp->external_error; + if (!dcp->external_cable && dcp->external_stopped && + !READ_ONCE(dcp->dptxport[0].connected)) { + port->hpd = false; + return 0; + } + } + } + ret = dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + if (ret) + return ret; + port->hpd = false; + return 0; +} + +static int dcp_typec_route_finish(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_port *port = + ((struct apple_dcp_typec_route *)typec_mux_get_drvdata(mux))->port; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE || !port->safe_pending || + port->safe_finish_ready) + return 0; + ret = port->owner ? dcp_typec_route_deactivate(port->owner) : 0; + if (ret) + return ret; + port->safe_finish_ready = true; + return 0; +} + +static void dcp_typec_route_commit(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_port *port = + ((struct apple_dcp_typec_route *)typec_mux_get_drvdata(mux))->port; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE || !port->safe_pending) + return; + if (port->owner) + port->preferred_route = port->owner; + port->owner = NULL; + port->safe_pending = false; + port->safe_finish_ready = false; + port->applied_alt = state->alt; + port->applied_mode = state->mode; + port->applied_status = 0; + port->applied_conf = 0; + port->applied_valid = true; + dcp_typec_pipeline_freed(); +} + static int dcp_typec_route_set(struct typec_mux_dev *mux, struct typec_mux_state *state) { @@ -955,6 +1058,11 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode == TYPEC_STATE_SAFE && port->safe_pending) + return 0; + if (port->safe_pending) + return -EBUSY; + /* * Every candidate route for this port is notified with the same state, * so only the first to arrive does the work; the others return early. @@ -1011,6 +1119,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, dcp_typec_rebalance_locked(NULL, 0); } + port->safe_pending = false; /* Data-only USB4 changes do not invalidate other display routes. */ port->applied_valid = true; return 0; @@ -1181,7 +1290,8 @@ int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) guard(mutex)(&dcp->tb_lock); route = dcp->active_typec_route; - if (!route || !route->tunnel || !route->active_xbar) + if (!route || !route->tunnel || !route->active_xbar || + route->deactivate_xbar_done) return -ENODEV; if (!dcp->tb_clock_ok) { dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); @@ -1686,6 +1796,9 @@ static int dcp_register_typec_routes(struct apple_dcp *dcp) desc.fwnode = of_fwnode_handle(route_np); desc.set = dcp_typec_route_set; + desc.prepare = dcp_typec_route_prepare; + desc.finish = dcp_typec_route_finish; + desc.commit = dcp_typec_route_commit; desc.name = mux_name; desc.drvdata = route; route->typec_mux = typec_mux_register(dev, &desc); diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index 7e327118cbd71..a6dd8284ff1b7 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include "class.h" #include "mux.h" @@ -259,11 +261,60 @@ EXPORT_SYMBOL_GPL(typec_switch_get_drvdata); /* ------------------------------------------------------------------------- */ +/* Class, altmode and controller handles for one connector share this domain. */ +struct typec_mux_context { + struct list_head list; + struct fwnode_handle *fwnode; + struct mutex lock; + unsigned int users; +}; + +static LIST_HEAD(typec_mux_contexts); +static DEFINE_MUTEX(typec_mux_contexts_lock); +DEFINE_STATIC_SRCU(typec_mux_srcu); + struct typec_mux { struct typec_mux_dev *mux_devs[TYPEC_MUX_MAX_DEVS]; unsigned int num_mux_devs; + struct typec_mux_context *context; + struct typec_mux_state transaction; + struct task_struct *owner; + int srcu_index; + unsigned int prepared; + bool applied; }; +static struct typec_mux_context *typec_mux_context_get(struct fwnode_handle *fwnode) +{ + struct typec_mux_context *context; + + guard(mutex)(&typec_mux_contexts_lock); + list_for_each_entry(context, &typec_mux_contexts, list) { + if (context->fwnode == fwnode) { + context->users++; + return context; + } + } + context = kzalloc_obj(*context); + if (!context) + return ERR_PTR(-ENOMEM); + context->fwnode = fwnode_handle_get(fwnode); + context->users = 1; + mutex_init(&context->lock); + list_add_tail(&context->list, &typec_mux_contexts); + return context; +} + +static void typec_mux_context_put(struct typec_mux_context *context) +{ + guard(mutex)(&typec_mux_contexts_lock); + if (--context->users) + return; + list_del(&context->list); + fwnode_handle_put(context->fwnode); + kfree(context); +} + static int mux_fwnode_match(struct device *dev, const void *fwnode) { if (!is_typec_mux_dev(dev)) @@ -310,16 +361,20 @@ struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode) int count; int err; int i; + int index; mux = kzalloc_obj(*mux); if (!mux) return ERR_PTR(-ENOMEM); + /* Unregister must drain lookup and module-reference conversion too. */ + index = srcu_read_lock(&typec_mux_srcu); count = fwnode_connection_find_matches(fwnode, "mode-switch", NULL, typec_mux_match, (void **)mux_devs, ARRAY_SIZE(mux_devs)); if (count <= 0) { + srcu_read_unlock(&typec_mux_srcu, index); kfree(mux); return NULL; } @@ -332,12 +387,26 @@ struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode) } for (i = 0; i < count; i++) { - WARN_ON(!try_module_get(mux_devs[i]->dev.parent->driver->owner)); + if (READ_ONCE(mux_devs[i]->removed) || + !try_module_get(mux_devs[i]->owner)) { + err = -ENODEV; + while (i--) + module_put(mux_devs[i]->owner); + goto put_mux_devs; + } mux->mux_devs[i] = mux_devs[i]; } mux->num_mux_devs = count; + mux->context = typec_mux_context_get(fwnode); + if (IS_ERR(mux->context)) { + err = PTR_ERR(mux->context); + for (i = 0; i < count; i++) + module_put(mux_devs[i]->owner); + goto put_mux_devs; + } + srcu_read_unlock(&typec_mux_srcu, index); return mux; put_mux_devs: @@ -346,6 +415,7 @@ struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode) put_device(&mux_devs[i]->dev); } + srcu_read_unlock(&typec_mux_srcu, index); kfree(mux); return ERR_PTR(err); @@ -368,31 +438,162 @@ void typec_mux_put(struct typec_mux *mux) for (i = 0; i < mux->num_mux_devs; i++) { mux_dev = mux->mux_devs[i]; - module_put(mux_dev->dev.parent->driver->owner); + module_put(mux_dev->owner); put_device(&mux_dev->dev); } + typec_mux_context_put(mux->context); kfree(mux); } EXPORT_SYMBOL_GPL(typec_mux_put); -int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) +static void typec_mux_unlock(struct typec_mux *mux) +{ + int index = mux->srcu_index; + + mux->prepared = 0; + mux->applied = false; + mux->owner = NULL; + mutex_unlock(&mux->context->lock); + srcu_read_unlock(&typec_mux_srcu, index); +} + +void typec_mux_abort(struct typec_mux *mux) { - struct typec_mux_dev *mux_dev; unsigned int i; - int ret; + + if (IS_ERR_OR_NULL(mux)) + return; + if (WARN_ON_ONCE(mux->owner != current)) + return; + for (i = mux->prepared; i > 0; i--) { + struct typec_mux_dev *dev = mux->mux_devs[i - 1]; + + if (dev->abort) + dev->abort(dev, &mux->transaction); + } + typec_mux_unlock(mux); +} +EXPORT_SYMBOL_GPL(typec_mux_abort); + +int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state *state) +{ + unsigned int i; + int index, ret; if (IS_ERR_OR_NULL(mux)) return 0; + index = srcu_read_lock(&typec_mux_srcu); + /* Nonblocking acquisition also rejects same-domain forwarding recursion. */ + if (!mutex_trylock(&mux->context->lock)) { + srcu_read_unlock(&typec_mux_srcu, index); + return -EBUSY; + } + mux->srcu_index = index; + mux->owner = current; + mux->transaction = *state; + for (i = 0; i < mux->num_mux_devs; i++) { + struct typec_mux_dev *dev = mux->mux_devs[i]; + if (READ_ONCE(dev->removed)) { + typec_mux_unlock(mux); + return -ENODEV; + } + } for (i = 0; i < mux->num_mux_devs; i++) { - mux_dev = mux->mux_devs[i]; + struct typec_mux_dev *dev = mux->mux_devs[i]; + + mux->prepared = i + 1; + if (!dev->prepare) + continue; + ret = dev->prepare(dev, &mux->transaction); + if (ret) + goto abort; + } + return 0; +abort: + typec_mux_abort(mux); + return ret; +} +EXPORT_SYMBOL_GPL(typec_mux_begin); - ret = mux_dev->set(mux_dev, state); - if (ret && ret != -EOPNOTSUPP) +int typec_mux_apply(struct typec_mux *mux) +{ + unsigned int i; + int ret; + + if (IS_ERR_OR_NULL(mux)) + return 0; + if (WARN_ON_ONCE(mux->owner != current)) + return -EINVAL; + if (mux->applied) + return 0; + for (i = 0; i < mux->num_mux_devs; i++) { + struct typec_mux_dev *dev = mux->mux_devs[i]; + + ret = dev->set(dev, &mux->transaction); + if (ret && ret != -EOPNOTSUPP) { + typec_mux_abort(mux); return ret; + } + } + mux->applied = true; + return 0; +} +EXPORT_SYMBOL_GPL(typec_mux_apply); + +int typec_mux_end(struct typec_mux *mux) +{ + unsigned int i; + int ret; + + if (IS_ERR_OR_NULL(mux)) + return 0; + ret = typec_mux_apply(mux); + if (ret) + return ret; + /* Finalizers can fail, but must not release their software reservation. */ + for (i = 0; i < mux->num_mux_devs; i++) { + struct typec_mux_dev *dev = mux->mux_devs[i]; + + if (!dev->finish) + continue; + ret = dev->finish(dev, &mux->transaction); + if (ret) + goto abort; } + /* Only this no-fail publication phase may release owner reservations. */ + for (i = 0; i < mux->num_mux_devs; i++) { + struct typec_mux_dev *dev = mux->mux_devs[i]; + if (dev->commit) + dev->commit(dev, &mux->transaction); + } + typec_mux_unlock(mux); return 0; +abort: + typec_mux_abort(mux); + return ret; +} +EXPORT_SYMBOL_GPL(typec_mux_end); + +/* Preparation-only callers leave recovery ownership pending, never committed. */ +int typec_mux_prepare(struct typec_mux *mux, struct typec_mux_state *state) +{ + int ret = typec_mux_begin(mux, state); + + if (!ret) + typec_mux_abort(mux); + return ret; +} +EXPORT_SYMBOL_GPL(typec_mux_prepare); + +int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) +{ + int ret = typec_mux_begin(mux, state); + + if (ret) + return ret; + return typec_mux_end(mux); } EXPORT_SYMBOL_GPL(typec_mux_set); @@ -429,7 +630,12 @@ typec_mux_register(struct device *parent, const struct typec_mux_desc *desc) if (!mux_dev) return ERR_PTR(-ENOMEM); + mux_dev->owner = parent->driver ? parent->driver->owner : NULL; mux_dev->set = desc->set; + mux_dev->prepare = desc->prepare; + mux_dev->finish = desc->finish; + mux_dev->commit = desc->commit; + mux_dev->abort = desc->abort; device_initialize(&mux_dev->dev); mux_dev->dev.parent = parent; @@ -462,8 +668,12 @@ EXPORT_SYMBOL_GPL(typec_mux_register); */ void typec_mux_unregister(struct typec_mux_dev *mux_dev) { - if (!IS_ERR_OR_NULL(mux_dev)) + if (!IS_ERR_OR_NULL(mux_dev)) { + WRITE_ONCE(mux_dev->removed, true); + /* Callbacks may nest through forwarders; SRCU permits that nesting. */ + synchronize_srcu(&typec_mux_srcu); device_unregister(&mux_dev->dev); + } } EXPORT_SYMBOL_GPL(typec_mux_unregister); diff --git a/drivers/usb/typec/mux.h b/drivers/usb/typec/mux.h index 12e1ac47a11f1..42e44211fd553 100644 --- a/drivers/usb/typec/mux.h +++ b/drivers/usb/typec/mux.h @@ -12,8 +12,14 @@ struct typec_switch_dev { }; struct typec_mux_dev { + struct module *owner; struct device dev; typec_mux_set_fn_t set; + typec_mux_set_fn_t prepare; + typec_mux_set_fn_t finish; + typec_mux_notify_fn_t commit; + typec_mux_notify_fn_t abort; + bool removed; }; #define to_typec_switch_dev(_dev_) container_of(_dev_, struct typec_switch_dev, dev) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 6e001be91b874..f7fbee64a344d 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -212,10 +212,11 @@ static int tps6598x_read_partner_identity(struct tps6598x *tps) return 0; } -static void tps6598x_set_data_role(struct tps6598x *tps, +static int tps6598x_set_data_role(struct tps6598x *tps, enum typec_data_role role, bool connected) { enum usb_role role_val; + int ret; if (role == TYPEC_HOST) role_val = USB_ROLE_HOST; @@ -225,8 +226,11 @@ static void tps6598x_set_data_role(struct tps6598x *tps, if (!connected) role_val = USB_ROLE_NONE; - usb_role_switch_set_role(tps->role_sw, role_val); + ret = usb_role_switch_set_role(tps->role_sw, role_val); + if (ret) + return ret; typec_set_data_role(tps->port, role); + return 0; } static int tps6598x_connect(struct tps6598x *tps, u32 status) @@ -259,7 +263,9 @@ static int tps6598x_connect(struct tps6598x *tps, u32 status) else typec_set_orientation(tps->port, TYPEC_ORIENTATION_NORMAL); typec_set_mode(tps->port, TYPEC_STATE_USB); - tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), true); + ret = tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), true); + if (ret) + return ret; tps->partner = typec_register_partner(tps->port, &desc); if (IS_ERR(tps->partner)) @@ -273,7 +279,7 @@ static int tps6598x_connect(struct tps6598x *tps, u32 status) return 0; } -static void tps6598x_disconnect(struct tps6598x *tps, u32 status) +static void tps6598x_drop_partner(struct tps6598x *tps, u32 status) { if (!IS_ERR(tps->partner)) typec_unregister_partner(tps->partner); @@ -281,11 +287,37 @@ static void tps6598x_disconnect(struct tps6598x *tps, u32 status) typec_set_pwr_opmode(tps->port, TYPEC_PWR_MODE_USB); typec_set_pwr_role(tps->port, TPS_STATUS_TO_TYPEC_PORTROLE(status)); typec_set_vconn_role(tps->port, TPS_STATUS_TO_TYPEC_VCONN(status)); - typec_set_orientation(tps->port, TYPEC_ORIENTATION_NONE); - typec_set_mode(tps->port, TYPEC_STATE_SAFE); - tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), false); +} + +static int tps6598x_disconnect(struct tps6598x *tps, u32 status) +{ + struct typec_mux_state safe = { .mode = TYPEC_STATE_SAFE }; + bool prepared = tps->remove_safe_prepared; + int ret; + + tps->remove_safe_prepared = false; + tps6598x_drop_partner(tps, status); + if (!prepared) { + typec_set_orientation(tps->port, TYPEC_ORIENTATION_NONE); + ret = typec_mux_begin(tps->mux, &safe); + if (ret) + return ret; + /* Park while the host still supplies the PIPE backend clock. */ + ret = typec_mux_apply(tps->mux); + if (ret) + return ret; + } + ret = tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), false); + if (ret) { + typec_mux_abort(tps->mux); + return ret; + } + ret = typec_mux_end(tps->mux); + if (ret) + return ret; power_supply_changed(tps->psy); + return 0; } int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, @@ -529,7 +561,9 @@ static void tps6598x_handle_plug_event(struct tps6598x *tps, u32 status) if (ret) dev_err(tps->dev, "failed to register partner\n"); } else { - tps6598x_disconnect(tps, status); + ret = tps6598x_disconnect(tps, status); + if (ret) + dev_err(tps->dev, "hardware disconnect failed: %d\n", ret); } } @@ -1983,12 +2017,13 @@ tps25750_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) return 0; } -static void cd321x_remove(struct tps6598x *tps) +static int cd321x_remove(struct tps6598x *tps) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); struct typec_thunderbolt_switch_data tbt_switch_data = { .state = TYPEC_THUNDERBOLT_SWITCH_OFF, }; + struct typec_mux_state safe = { .mode = TYPEC_STATE_SAFE }; int ret; cd321x_stop_work(cd321x); @@ -2005,10 +2040,24 @@ static void cd321x_remove(struct tps6598x *tps) ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); if (ret) - dev_warn(tps->dev, - "failed to close Thunderbolt/USB4 session on remove: %d\n", - ret); - usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); + return ret; + /* Display quiescence must precede even an ordinary host-role shutdown. */ + typec_set_orientation(tps->port, TYPEC_ORIENTATION_NONE); + ret = typec_mux_begin(tps->mux, &safe); + if (ret) + return ret; + /* Park while the host still supplies the PIPE backend clock. */ + ret = typec_mux_apply(tps->mux); + if (ret) + return ret; + ret = usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); + if (ret) { + typec_mux_abort(tps->mux); + return ret; + } + /* Same task/handle completes without another post-shutdown set callback. */ + tps->remove_safe_prepared = true; + return 0; } @@ -2017,7 +2066,8 @@ int tipd_init(struct tps6598x *tps) struct fwnode_handle *fwnode; u32 status; u32 vid; - int ret; + int ret, cleanup_ret; + bool reset_allowed = true; tps->reset = devm_gpiod_get_optional(tps->dev, "reset", GPIOD_OUT_LOW); if (IS_ERR(tps->reset)) @@ -2090,19 +2140,32 @@ int tipd_init(struct tps6598x *tps) if (ret) goto err_role_put; - ret = tps->data->register_port(tps, fwnode); - if (ret) + tps->mux = fwnode_typec_mux_get(fwnode); + if (IS_ERR(tps->mux)) { + ret = PTR_ERR(tps->mux); + /* No provider reference means rollback cannot prove SAFE. */ + reset_allowed = false; goto err_role_put; + } + + ret = tps->data->register_port(tps, fwnode); + if (ret) { + /* No live port remains to prove SAFE on failed registration. */ + reset_allowed = false; + goto err_mux_put; + } if (status & TPS_STATUS_PLUG_PRESENT) { ret = -EINVAL; if (!tps6598x_read_power_status(tps)) - goto err_unregister_port; + goto err_disconnect; if (!tps->data->read_data_status(tps)) - goto err_unregister_port; + goto err_disconnect; ret = tps->data->connect(tps, status); - if (ret) + if (ret) { dev_err(tps->dev, "failed to register partner\n"); + goto err_disconnect; + } } if (tps->irq) { @@ -2135,9 +2198,14 @@ int tipd_init(struct tps6598x *tps) return 0; err_disconnect: - tps6598x_disconnect(tps, 0); -err_unregister_port: + cleanup_ret = tps6598x_disconnect(tps, 0); + if (cleanup_ret) { + dev_err(tps->dev, "probe hardware rollback failed: %d\n", cleanup_ret); + reset_allowed = false; + } tps->data->unregister_port(tps); +err_mux_put: + typec_mux_put(tps->mux); err_role_put: usb_role_switch_put(tps->role_sw); err_fwnode_put: @@ -2146,7 +2214,8 @@ int tipd_init(struct tps6598x *tps) tps6598x_write64(tps, TPS_REG_INT_MASK1, 0); err_reset_controller: /* Reset PD controller to remove any applied patch */ - tps->data->reset(tps); + if (reset_allowed) + tps->data->reset(tps); return ret; } @@ -2154,6 +2223,8 @@ EXPORT_SYMBOL_GPL(tipd_init); void tipd_remove(struct tps6598x *tps) { + int ret = 0; + tipd_debugfs_unregister(tps); if (!tps->irq) @@ -2162,16 +2233,23 @@ void tipd_remove(struct tps6598x *tps) devm_free_irq(tps->dev, tps->irq, tps); if (tps->data->remove) - tps->data->remove(tps); + ret = tps->data->remove(tps); - tps6598x_disconnect(tps, 0); + if (!ret) + ret = tps6598x_disconnect(tps, 0); + else + tps6598x_drop_partner(tps, 0); + if (ret) + dev_err(tps->dev, "hardware teardown failed on remove: %d\n", ret); tps->data->unregister_port(tps); + typec_mux_put(tps->mux); usb_role_switch_put(tps->role_sw); /* Reset PD controller to remove any applied patch */ - tps->data->reset(tps); + if (!ret) + tps->data->reset(tps); - if (tps->reset) + if (!ret && tps->reset) gpiod_set_value_cansleep(tps->reset, 1); } EXPORT_SYMBOL_GPL(tipd_remove); diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 3a807f1def6b2..ef3f7ff6531d9 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -295,7 +295,7 @@ struct tipd_data { irq_handler_t irq_handler; u64 irq_mask1; size_t tps_struct_size; - void (*remove)(struct tps6598x *tps); + int (*remove)(struct tps6598x *tps); int (*register_port)(struct tps6598x *tps, struct fwnode_handle *node); void (*unregister_port)(struct tps6598x *tps); void (*ready)(struct tps6598x *tps); @@ -313,6 +313,9 @@ struct tipd_data { }; struct tps6598x { + struct typec_mux *mux; + /* CD remove leaves one open SAFE transaction for generic completion. */ + bool remove_safe_prepared; struct device *dev; struct regmap *regmap; struct mutex lock; /* device lock */ diff --git a/include/linux/usb/typec_mux.h b/include/linux/usb/typec_mux.h index d58d60fcae21f..a18e4e0141adf 100644 --- a/include/linux/usb/typec_mux.h +++ b/include/linux/usb/typec_mux.h @@ -86,6 +86,9 @@ struct typec_mux_state { void *data; }; +typedef void (*typec_mux_notify_fn_t)(struct typec_mux_dev *mux, + struct typec_mux_state *state); + typedef int (*typec_mux_set_fn_t)(struct typec_mux_dev *mux, struct typec_mux_state *state); @@ -94,12 +97,26 @@ struct typec_mux_desc { typec_mux_set_fn_t set; const char *name; void *drvdata; + /* Quiesce consumers before any provider changes the shared route. */ + typec_mux_set_fn_t prepare; + /* Fallible finalization retains ownership; commit cannot fail. */ + typec_mux_set_fn_t finish; + typec_mux_notify_fn_t commit; + typec_mux_notify_fn_t abort; }; #if IS_ENABLED(CONFIG_TYPEC) struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode); void typec_mux_put(struct typec_mux *mux); +/* begin/end must run in the same task; failure retains provider recovery state. */ +/* Same-task begin/end; retain the handle and state pointees until end/abort. */ +int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state *state); +/* Apply the providers while retaining the transaction and ownership. */ +int typec_mux_apply(struct typec_mux *mux); +int typec_mux_end(struct typec_mux *mux); +void typec_mux_abort(struct typec_mux *mux); +int typec_mux_prepare(struct typec_mux *mux, struct typec_mux_state *state); int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state); struct typec_mux_dev * @@ -118,6 +135,21 @@ static inline struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnod static inline void typec_mux_put(struct typec_mux *mux) {} +static inline int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state *state) +{ + return 0; +} + +static inline int typec_mux_apply(struct typec_mux *mux) { return 0; } + +static inline int typec_mux_end(struct typec_mux *mux) { return 0; } +static inline void typec_mux_abort(struct typec_mux *mux) {} + +static inline int typec_mux_prepare(struct typec_mux *mux, struct typec_mux_state *state) +{ + return 0; +} + static inline int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) { return 0; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/accounting.py b/tools/testing/selftests/drivers/gpu/apple-dcp/accounting.py new file mode 100644 index 0000000000000..ce2791cc3a379 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/accounting.py @@ -0,0 +1,318 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Source-connected accounting regressions retaining adverse-review assertions. + +SRCU, module and hardware leaves are models; the transaction, DCP and mux core +functions are extracted from the selected public source. No device is accessed. +""" +import quiesce +import transaction +from common import block + +def srcu_index(ROOT): + unit=quiesce.make_unit(ROOT) + main=block(unit,'int main(') + unit=unit.replace(main,main.replace('int main(','int __attribute__((unused)) prior_main(',1),1) + old_lock=block(unit,'static int srcu_read_lock(') + old_unlock=block(unit,'static void srcu_read_unlock(') + old_mutex_unlock=block(unit,'static void mutex_unlock(') + leaves=r''' + static pthread_mutex_t schedule_mutex=PTHREAD_MUTEX_INITIALIZER; + static pthread_cond_t schedule_cond=PTHREAD_COND_INITIALIZER; + static int srcu_bank,bank_readers[2],released_index=-1; + static bool pause_unlock,pause_seen,resume_unlock; + static _Thread_local int expected_index=-1; + static int srcu_read_lock(int *s) { + (void)s;pthread_mutex_lock(&schedule_mutex);int i=srcu_bank;bank_readers[i]++;expected_index=i;pthread_mutex_unlock(&schedule_mutex);return i; + } + static void srcu_read_unlock(int *s,int i) { + (void)s;pthread_mutex_lock(&schedule_mutex);released_index=i; + fprintf(stderr,"SRCU unlock: acquired=%d released=%d bank0=%d bank1=%d\n",expected_index,i,bank_readers[0],bank_readers[1]); + assert(i==expected_index&&bank_readers[i]>0);bank_readers[i]--;expected_index=-1; + pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_mutex); + } + static void mutex_unlock(struct mutex *m) { + assert(!pthread_mutex_unlock(&m->value)); + if(m==&shared_context.lock&&pause_unlock) { + pthread_mutex_lock(&schedule_mutex);pause_seen=true;pause_unlock=false;pthread_cond_broadcast(&schedule_cond); + while(!resume_unlock)pthread_cond_wait(&schedule_cond,&schedule_mutex); + pthread_mutex_unlock(&schedule_mutex); + } + } + ''' + unit=unit.replace(old_lock,'',1).replace(old_unlock,'',1).replace(old_mutex_unlock,leaves,1) + unit+=r''' + static void *first_actor(void *p) { + struct typec_mux *m=p;struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE};assert(typec_mux_set(m,&safe)==0);return NULL; + } + int main(void) { + struct cd321x c;initial(&c);mux.num_mux_devs=0;pause_unlock=true; + pthread_t thread;assert(!pthread_create(&thread,NULL,first_actor,&mux)); + pthread_mutex_lock(&schedule_mutex);while(!pause_seen)pthread_cond_wait(&schedule_cond,&schedule_mutex); + srcu_bank=1;pthread_mutex_unlock(&schedule_mutex); + struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE};assert(typec_mux_begin(&mux,&safe)==0); + assert(mux.srcu_index==1&&bank_readers[0]==1&&bank_readers[1]==1); + pthread_mutex_lock(&schedule_mutex);resume_unlock=true;pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_mutex); + assert(!pthread_join(thread,NULL));typec_mux_abort(&mux);assert(bank_readers[0]==0&&bank_readers[1]==0); + puts("PASS same-handle SRCU read-index balance across context unlock"); + } + ''' + return unit + +def resources(ROOT): + actual=next(u for n,u,c,f in transaction.controls(ROOT) if n=='actual-dcp-normal-safe') + main=block(actual,'int main(') + actual=actual.replace(main,main.replace('int main(','int __attribute__((unused)) prior_route_main(',1),1) + actual=actual.replace('struct mux_control { int unused; };',r''' + struct mux_control; + struct mux_ops { int (*set)(struct mux_control *,int); }; + struct mux_chip { const struct mux_ops *ops; }; + struct mux_control { int lock,idle_state,cached_state; unsigned last_change;struct mux_chip *chip; }; + ''',1) + muxcore=(ROOT/'drivers/mux/core.c').read_text() + resource=r''' + #define MUX_IDLE_AS_IS -1 + #define MUX_CACHE_UNKNOWN -2 + static int hw_calls,hw_failures,fixed_restore_calls,fixed_restore_failures; + static unsigned ktime_get(void) { return 0; } + static void up(int *s) { (*s)++; } + static int hw_set(struct mux_control *m,int state) { (void)m;(void)state;hw_calls++;return hw_calls<=hw_failures?-EIO:0; } + ''' + resource+=block(muxcore,'static int mux_control_set(')+'\n'+block(muxcore,'int mux_control_deselect(')+'\n' + actual=actual.replace(block(actual,'static int mux_control_deselect('),resource,1) + actual=actual.replace(block(actual,'static int mux_control_select('),r'''static int mux_control_select(struct mux_control *m,unsigned n) { (void)m;(void)n;fixed_restore_calls++;return fixed_restore_calls<=fixed_restore_failures?-EIO:0; }''',1) + actual=actual.replace('static int gpiod_get_value_cansleep(void *p) { (void)p;return 0; }', 'static int gpio_level;\nstatic int gpiod_get_value_cansleep(void *p) { (void)p;return gpio_level; }',1) + actual+=r''' + static int peer_error; + static int peer_finish(struct typec_mux_dev *p,struct typec_mux_state *s) { (void)p;(void)s;return peer_error; } + int main(int argc,char **argv) { + (void)argc;struct cd321x c;initial(&c); + struct mux_ops ops={.set=hw_set};struct mux_chip chip={.ops=&ops}; + struct mux_control xbar={.lock=0,.idle_state=0,.cached_state=1,.chip=&chip}; + struct mux_control fixed_xbar={.lock=1,.idle_state=0,.cached_state=0,.chip=&chip}; + route.xbar=route.active_xbar=&xbar; + struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE}; + if(argv[1][0]=='n') { + assert(typec_mux_set(&mux,&safe)==0&&xbar.lock==1&&!port.owner&&!port.safe_pending); + puts("PASS actual DCP finalizer + real mux deselect resource balance");return 0; + } + if(argv[1][0]=='l') { + phy_provider.finish=peer_finish;mux.mux_devs[0]=&display_provider;mux.mux_devs[1]=&phy_provider;peer_error=-EIO; + assert(typec_mux_set(&mux,&safe)==-EIO&&xbar.lock==1&&port.owner==&route&&port.safe_pending&&port.safe_finish_ready); + peer_error=0;assert(typec_mux_set(&mux,&safe)==0&&xbar.lock==1&&!port.owner&&!port.safe_pending); + puts("PASS successful DCP finalizer is not repeated after a later provider failure");return 0; + } + if(argv[1][0]=='h')hw_failures=1; + if(argv[1][0]=='f') {display.fixed_connector_type=2;display.hdmi_hpd=(void *)1;gpio_level=1;display.xbar=&fixed_xbar;fixed_restore_failures=1;} + assert(typec_mux_set(&mux,&safe)==-EIO&&xbar.lock==1&&port.owner==&route&&port.safe_pending&&!port.safe_finish_ready); + bool selected_after_error=route.selected; + int ret=typec_mux_set(&mux,&safe); + fprintf(stderr,"finalizer retry: kind=%c ret=%d permits=%d selected_after_error=%d owner=%p pending=%d hw_calls=%d fixed_calls=%d\n",argv[1][0],ret,xbar.lock,selected_after_error,(void *)port.owner,port.safe_pending,hw_calls,fixed_restore_calls); + assert(xbar.lock==1); + if(argv[1][0]=='h')assert(ret==-EIO&&hw_calls==1&&port.owner==&route&&port.safe_pending); + if(argv[1][0]=='f')assert(ret==0&&!port.owner&&!port.safe_pending&&fixed_restore_calls==2); + return 0; + } + ''' + return actual + +def getter_unit(ROOT): + getter=transaction.lifetime(ROOT) + main=block(getter,'int main(') + renamed=main.replace('int main(','int __attribute__((unused)) prior_lifetime_main(',1) + renamed=renamed[:-1]+'return 0;\n}' + getter=getter.replace(main,renamed,1) + getter=getter.replace('struct typec_mux_dev {', 'struct module { unsigned refs;bool going; };\nstruct device { unsigned refs; };\nstruct typec_mux_dev { struct module *owner;struct device dev;',1) + getter=getter.replace(block(getter,'static void typec_mux_put('),'void typec_mux_put(struct typec_mux *mux);',1) + getter=getter.replace('#include ',r''' + #include + static int allocation_count,allocation_failure; + static void *review_alloc(size_t size) { if(++allocation_count==allocation_failure)return NULL;return calloc(1,size); } + ''',1) + getter=getter.replace('#define kzalloc_obj(x) calloc(1,sizeof(x))','#define kzalloc_obj(x) review_alloc(sizeof(x))',1) + getter+=r''' + #undef IS_ERR + #define IS_ERR(p) ((uintptr_t)(p)>=(uintptr_t)-4095) + #define PTR_ERR(p) ((int)(intptr_t)(p)) + #define TYPEC_MUX_MAX_DEVS 3 + #define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) + static struct typec_mux_dev graph_providers[2]; + static struct module modules[2]; + static bool graph_second_error; + static void *typec_mux_match(const struct fwnode_handle *f,const char *id,void *data) { (void)f;(void)id;(void)data;return NULL; } + static int fwnode_connection_find_matches(struct fwnode_handle *f,const char *id,void *data,void *(*fn)(const struct fwnode_handle *,const char *,void *),void **matches,unsigned max) { + (void)f;(void)id;(void)data;(void)fn;assert(max>=2); + graph_providers[0].dev.refs++;matches[0]=&graph_providers[0]; + if(graph_second_error)matches[1]=ERR_PTR(-EPROBE_DEFER); + else {graph_providers[1].dev.refs++;matches[1]=&graph_providers[1];} + return 2; + } + static bool try_module_get(struct module *m) { if(!m)return true;if(m->going)return false;m->refs++;return true; } + static void module_put(struct module *m) { if(m){assert(m->refs);m->refs--;} } + static void put_device(struct device *d) { assert(d->refs);d->refs--; } + ''' + getter='#define EPROBE_DEFER 517\n'+getter + mux=(ROOT/'drivers/usb/typec/mux.c').read_text() + getter+='\n'+block(mux,'struct typec_mux *fwnode_typec_mux_get(')+'\n'+block(mux,'void typec_mux_put(')+'\n' + getter+=r''' + static void graph_setup(void) { + graph_second_error=false;allocation_count=allocation_failure=0; + for(unsigned i=0;i<2;i++){modules[i]=(struct module){0};graph_providers[i]=(struct typec_mux_dev){.owner=&modules[i],.set=phy_set};} + } + static void graph_balanced(struct fwnode_handle *f) { + assert(!f->refs&&typec_mux_contexts.next==&typec_mux_contexts); + for(unsigned i=0;i<2;i++)assert(!modules[i].refs&&!graph_providers[i].dev.refs); + } + int main(void) { + struct fwnode_handle f={0};graph_setup(); + struct typec_mux *a=fwnode_typec_mux_get(&f),*b=fwnode_typec_mux_get(&f); + assert(!IS_ERR(a)&&!IS_ERR(b)&&a->context==b->context&&a->context->users==2&&f.refs==1); + for(unsigned i=0;i<2;i++)assert(modules[i].refs==2&&graph_providers[i].dev.refs==2); + typec_mux_put(a);typec_mux_put(b);graph_balanced(&f); + graph_setup();modules[1].going=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENODEV);graph_balanced(&f); + graph_setup();graph_second_error=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-EPROBE_DEFER);graph_balanced(&f); + graph_setup();allocation_failure=2;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENOMEM);graph_balanced(&f); + graph_setup();a=fwnode_typec_mux_get(&f);graph_providers[0].removed=true; + struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE};unsigned before=phy_calls; + assert(typec_mux_set(a,&safe)==-ENODEV&&phy_calls==before);typec_mux_put(a);graph_balanced(&f); + puts("PASS actual getter/context/module/device balance: same fwnode, failed module conversion, deferred provider, context allocation and stale-provider handle"); + return 0; + } + ''' + return getter + +def controls(root): + clock = transaction.clock_order(root).replace('if(parked)return 0;if(!backend_clock)', 'if(!backend_clock)', 1) + clock = clock.replace('assert(!port.owner&&!port.safe_pending);', 'assert(!port.owner&&!port.safe_pending&&phy_calls==1&&role_calls==2);', 1) + return [ + ('same-handle-srcu-index', srcu_index(root), ((),), ('-pthread',)), + ('mux-release-stages', resources(root), (('normal',), ('late',), ('hardware',), ('fixed',)), ('-pthread',)), + ('single-safe-before-role-exit', clock, ((),), ('-pthread',)), + ('mux-get-put-balance', getter_unit(root), ((),), ('-pthread',)), + ('lookup-unregister-module-lifetime', lookup_lifetime(root), ((),), ('-pthread',)), + ('kernel-module-conversion-balance', module_conversion(root), ((),), ('-pthread',)), + ] + +def lookup_lifetime(ROOT): + getter = getter_unit(ROOT) + mux = (ROOT / "drivers/usb/typec/mux.c").read_text() + module_unit=getter.replace(block(getter,'int main('),block(getter,'int main(').replace('int main(','int __attribute__((unused)) prior_getter_main(',1),1) + module_unit=module_unit.replace('struct module { unsigned refs;bool going; };','struct module { unsigned refs,refcnt;bool going; };',1) + module_unit=module_unit.replace('struct device { unsigned refs; };','struct device { unsigned refs;bool unregistered; };',1) + sync=r''' + static pthread_mutex_t conversion_mutex=PTHREAD_MUTEX_INITIALIZER; + static pthread_cond_t conversion_cond=PTHREAD_COND_INITIALIZER; + static bool lookup_ready,lookup_resume; + static unsigned active_readers; + static void synchronize_srcu(int *s) { (void)s;pthread_mutex_lock(&conversion_mutex);assert(active_readers==0);pthread_mutex_unlock(&conversion_mutex); } + static void device_unregister(struct device *d) { d->unregistered=true;assert(d->refs);d->refs--; } + static bool module_is_live(struct module *m) { return !m->going; } + static bool atomic_inc_not_zero(unsigned *p) { if(!*p)return false;(*p)++;return true; } + #define likely(x) (x) + #define trace_module_get(...) ((void)0) + #define _RET_IP_ 0 + ''' + pos=module_unit.index('static bool try_module_get(') + module_unit=module_unit[:pos]+sync+module_unit[pos:] + modulemain=(ROOT/'kernel/module/main.c').read_text() + module_unit=module_unit.replace(block(module_unit,'static bool try_module_get('),block(modulemain,'bool try_module_get('),1) + needle='return 2;\n }' + assert module_unit.count(needle)==1 + module_unit=module_unit.replace(needle,r''' + pthread_mutex_lock(&conversion_mutex);lookup_ready=true;pthread_cond_broadcast(&conversion_cond); + while(!lookup_resume)pthread_cond_wait(&conversion_cond,&conversion_mutex); + pthread_mutex_unlock(&conversion_mutex);return 2; + }''',1) + + declarations = r''' +static pthread_mutex_t conversion_mutex=PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t conversion_cond=PTHREAD_COND_INITIALIZER; +static bool lookup_ready,lookup_resume,draining; +static unsigned active_readers; +#define WRITE_ONCE(x,v) ((x)=(v)) +''' + start = module_unit.index('static pthread_mutex_t conversion_mutex=') + end = module_unit.index('static void synchronize_srcu(', start) + module_unit = module_unit[:start] + module_unit[end:] + pos = module_unit.index('static int srcu_read_lock(') + module_unit = module_unit[:pos] + declarations + module_unit[pos:] + module_unit = module_unit.replace(block(module_unit, 'static int srcu_read_lock('), r''' +static int srcu_read_lock(int *s) { (void)s;pthread_mutex_lock(&conversion_mutex);active_readers++;pthread_mutex_unlock(&conversion_mutex);return 0; } +''', 1) + module_unit = module_unit.replace(block(module_unit, 'static void srcu_read_unlock('), r''' +static void srcu_read_unlock(int *s,int index) { (void)s;assert(index==0);pthread_mutex_lock(&conversion_mutex);assert(active_readers);active_readers--;pthread_cond_broadcast(&conversion_cond);pthread_mutex_unlock(&conversion_mutex); } +''', 1) + module_unit = module_unit.replace(block(module_unit, 'static void synchronize_srcu('), r''' +static void synchronize_srcu(int *s) { (void)s;pthread_mutex_lock(&conversion_mutex);draining=true;pthread_cond_broadcast(&conversion_cond);while(active_readers)pthread_cond_wait(&conversion_cond,&conversion_mutex);pthread_mutex_unlock(&conversion_mutex); } +''', 1) + module_unit += '\n' + block(mux, 'void typec_mux_unregister(') + r''' +static struct typec_mux *lookup_result; +static void *getter_actor(void *node) { lookup_result=fwnode_typec_mux_get(node);return NULL; } +static void *unregister_actor(void *provider) { typec_mux_unregister(provider);return NULL; } +int main(void) { + struct fwnode_handle f={0};graph_setup(); + struct module *departing=calloc(1,sizeof(*departing));departing->going=true;graph_providers[0].owner=departing;graph_providers[0].dev.refs=1; + pthread_t getter_thread,remove_thread;assert(!pthread_create(&getter_thread,NULL,getter_actor,&f)); + pthread_mutex_lock(&conversion_mutex);while(!lookup_ready)pthread_cond_wait(&conversion_cond,&conversion_mutex); + assert(active_readers==1&&graph_providers[0].dev.refs==2);pthread_mutex_unlock(&conversion_mutex); + assert(!pthread_create(&remove_thread,NULL,unregister_actor,&graph_providers[0])); + pthread_mutex_lock(&conversion_mutex);while(!draining)pthread_cond_wait(&conversion_cond,&conversion_mutex); + assert(active_readers==1&&!graph_providers[0].dev.unregistered); + lookup_resume=true;pthread_cond_broadcast(&conversion_cond);pthread_mutex_unlock(&conversion_mutex); + assert(!pthread_join(getter_thread,NULL));assert(!pthread_join(remove_thread,NULL)); + assert(PTR_ERR(lookup_result)==-ENODEV&&graph_providers[0].dev.unregistered&&graph_providers[0].dev.refs==0&&active_readers==0); + assert(graph_providers[1].dev.refs==0&&!f.refs); + free(departing); + puts("PASS lookup-to-module conversion holds a reader until removed-provider rejection and balanced cleanup"); + return 0; +} +''' + return module_unit + + +def module_conversion(ROOT): + unit = getter_unit(ROOT) + main = block(unit, 'int main(') + unit = unit.replace(main, main.replace('int main(', 'int __attribute__((unused)) prior_getter_main(', 1), 1) + unit = unit.replace('struct module { unsigned refs;bool going; };', 'struct module { unsigned refs,refcnt;bool going; };', 1) + source = (ROOT / 'kernel/module/main.c').read_text() + pos = unit.index('static bool try_module_get(') + leaves = r""" +static bool module_is_live(struct module *m) { return !m->going; } +static bool atomic_inc_not_zero(unsigned *p) { if(!*p)return false;(*p)++;return true; } +static int atomic_dec_if_positive(unsigned *p) { if(!*p)return -1;return --(*p); } +#define likely(x) (x) +#define trace_module_get(...) ((void)0) +#define trace_module_put(...) ((void)0) +#define preempt_disable() ((void)0) +#define preempt_enable() ((void)0) +#define WARN_ON(v) assert(!(v)) +#define _RET_IP_ 0 +""" + unit = unit[:pos] + leaves + unit[pos:] + unit = unit.replace(block(unit, 'static bool try_module_get('), block(source, 'bool try_module_get('), 1) + unit = unit.replace(block(unit, 'static void module_put('), block(source, 'void module_put('), 1) + old = block(unit, 'static int srcu_read_lock(') + '\n' + block(unit, 'static void srcu_read_unlock(') + unit = unit.replace(old, r""" +static unsigned conversion_readers; +static int srcu_read_lock(int *s) { (void)s;conversion_readers++;return 0; } +static void srcu_read_unlock(int *s,int i) { (void)s;assert(i==0&&conversion_readers);conversion_readers--; } +""", 1) + unit += r""" +static void live_graph(void) { graph_setup();for(unsigned i=0;i<2;i++)modules[i].refcnt=1; } +static void module_balance(struct fwnode_handle *f) { graph_balanced(f);assert(!conversion_readers);for(unsigned i=0;i<2;i++)assert(modules[i].refcnt==1); } +int main(void) { + struct fwnode_handle f={0};live_graph(); + struct typec_mux *a=fwnode_typec_mux_get(&f),*b=fwnode_typec_mux_get(&f); + assert(!IS_ERR(a)&&!IS_ERR(b)&&a->context==b->context&&!conversion_readers); + for(unsigned i=0;i<2;i++)assert(modules[i].refcnt==3&&graph_providers[i].dev.refs==2); + typec_mux_put(a);typec_mux_put(b);module_balance(&f); + live_graph();modules[1].going=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENODEV);module_balance(&f); + live_graph();allocation_failure=2;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENOMEM);module_balance(&f); + live_graph();graph_second_error=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-EPROBE_DEFER);module_balance(&f); + live_graph();graph_providers[0].removed=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENODEV);module_balance(&f); + puts("PASS actual kernel module get/put and mux get/put: shared handles and every failed conversion unwind"); + return 0; +} +""" + return unit diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 3c2913281ba5e..6125dbf312b8f 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -16,9 +16,22 @@ def ownership(root, usb_unreachable=False): prefix=ns['prefix']+'\nstruct device_node;\nstatic int atcphy_mux_set(struct typec_mux_dev *, struct typec_mux_state *);\n' prefix=prefix.replace('struct typec_mux_dev { struct apple_atcphy *data; };', '''struct typec_mux_dev { struct apple_atcphy *data; - int (*set)(struct typec_mux_dev *, struct typec_mux_state *); }; - struct typec_mux { unsigned int num_mux_devs; struct typec_mux_dev **mux_devs; }; + int (*set)(struct typec_mux_dev *, struct typec_mux_state *); + int (*prepare)(struct typec_mux_dev *, struct typec_mux_state *); + int (*finish)(struct typec_mux_dev *, struct typec_mux_state *); + void (*commit)(struct typec_mux_dev *, struct typec_mux_state *); + void (*abort)(struct typec_mux_dev *, struct typec_mux_state *);bool removed; }; + struct typec_mux_context { struct { bool held; } lock; }; + struct task_struct { int unused; };static struct task_struct test_task; + #define current (&test_task) + static int typec_mux_srcu; + static int srcu_read_lock(int *s) { (void)s;return 0; } + static void srcu_read_unlock(int *s,int i) { (void)s;(void)i; } + #define mutex_trylock(m) ((m)->held ? 0 : ((m)->held=true)) + #define mutex_unlock(m) ((m)->held=false) + struct typec_mux { unsigned int num_mux_devs; struct typec_mux_dev **mux_devs; struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied; }; #define IS_ERR_OR_NULL(p) (!(p))''') + prefix=prefix.replace('#define WARN_ON_ONCE(v) assert(!(v))', 'static bool checked_warn(bool v) { assert(!v);return v; }\n#define WARN_ON_ONCE(v) checked_warn(v)') prefix=prefix.replace('#define guard(kind) (void)',r''' struct lock_guard { int *lock; }; static struct lock_guard lock_take(int *lock) { assert(!*lock); *lock=1; return (struct lock_guard){lock}; } @@ -177,7 +190,7 @@ def ownership(root, usb_unreachable=False): #if !USB_REACHABLE { prepare(&p,true,true,false);unsigned int before=writes; struct typec_mux_dev provider={.data=&p,.set=atcphy_mux_set}; - struct typec_mux_dev *providers[]={&provider};struct typec_mux mux={1,providers}; + struct typec_mux_dev *providers[]={&provider};struct typec_mux_context typec_test_context={0};struct typec_mux mux={.num_mux_devs=1,.mux_devs=providers,.context=&typec_test_context}; struct typec_mux_state state={.mode=TYPEC_STATE_SAFE}; assert(typec_mux_set(&mux,&state)==-ENODEV&&writes==before);balanced(&p); puts("PASS USB-unreachable fixed route blocks before MMIO");return 0; } @@ -298,7 +311,8 @@ def ownership(root, usb_unreachable=False): if usb_unreachable: pmfixture=pmfixture.replace('static ', 'static __attribute__((unused)) ') typec_source=(root/'drivers/usb/typec/mux.c').read_text() - typec_caller=ns['block'](typec_source,'int typec_mux_set(')+'\n' + typec_caller='\n'.join(ns['block'](typec_source, marker) for marker in + ('static void typec_mux_unlock(', 'void typec_mux_abort(', 'int typec_mux_begin(', 'int typec_mux_apply(', 'int typec_mux_end(', 'int typec_mux_set('))+'\n' hcd_source=(root/'drivers/usb/core/hcd.c').read_text() hcd_death=ns['block'](hcd_source,'void usb_hc_died (')+'\n' text=prefix+prod+'\n#define REQUIRES_PLATFORM_LOCKS '+str(int('device_lock(&pm->controller->dev)' in s))+'\n#define USB_REACHABLE '+str(int(not usb_unreachable))+'\n#define HCD_DEAD_ONLY '+str(int(False))+'\n'+pmfixture+helpers+typec_caller+hcd_death+suffix+controls @@ -531,7 +545,7 @@ def probe_dispatch(root): struct cd321x c={}; struct typec_mux_dev provider={.data=&p,.set=atcphy_mux_set}; struct typec_mux_dev *providers[]={&provider}; - struct typec_mux route={1,providers};probe_port.mux=&route; + struct typec_mux_context typec_test_context={0};struct typec_mux route={.num_mux_devs=1,.mux_devs=providers,.context=&typec_test_context};probe_port.mux=&route; prepare(&p,true,true,false);root_missing=true;unsigned int before=writes; assert(cd321x_register_port(&c.tps,&connector)==-EPROBE_DEFER); assert(!c.state_valid&&!resumes&&writes==before);balanced(&p);port_balance(); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py new file mode 100644 index 0000000000000..6e7b9f69b2a25 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Actual mux prepare/set and TPS removal ordering; firmware/PHY leaves are mocks.""" +from common import block + +def make_unit(source, parent=False, reference=None): + reference=source if reference is None else reference + core=(source/'drivers/usb/typec/tipd/core.c').read_text();mux=(source/'drivers/usb/typec/mux.c').read_text() + dcp=(reference/'drivers/gpu/drm/apple/dcp.c').read_text() + prefix=r''' +#include +#include +#include +#include +#include +#include +typedef unsigned u32; +#define IS_ERR(p) false +#define IS_ERR_OR_NULL(p) (!(p)) +#define EXPORT_SYMBOL_GPL(x) +#define guard(k) (void) +#define scoped_guard(k,l) for(int once=((void)(l),1);once;once=0) +#define READ_ONCE(x) (x) +#define WARN_ON_ONCE(x) (x) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define to_platform_device(p) (p) +#define typec_mux_get_drvdata(m) ((m)->data) +#define dev_warn(...) ((void)0) +#define dev_info(...) ((void)0) +#define dev_name(p) "device" +#define list_for_each_entry(p,h,m) for((void)(h),(p)=sole_port();(p);(p)=NULL) +#define lockdep_assert_held(p) ((void)(p)) +#define dev_err(dev,fmt,ret) do { (void)(dev);last_error=(ret); } while(0) +enum {TYPEC_STATE_SAFE,TYPEC_STATE_USB,TYPEC_ORIENTATION_NONE,TYPEC_PWR_MODE_USB,TYPEC_THUNDERBOLT_SWITCH_OFF}; +enum usb_role {USB_ROLE_NONE,USB_ROLE_HOST,USB_ROLE_DEVICE}; +enum typec_data_role {TYPEC_HOST,TYPEC_DEVICE}; +#define TPS_STATUS_TO_TYPEC_PORTROLE(s) (s) +#define TPS_STATUS_TO_TYPEC_VCONN(s) (s) +#define TPS_STATUS_TO_TYPEC_DATAROLE(s) (s) +struct typec_mux_state { void *alt; unsigned long mode; void *data; }; +struct typec_mux_dev { bool removed; void (*commit)(struct typec_mux_dev *,struct typec_mux_state *); void (*abort)(struct typec_mux_dev *,struct typec_mux_state *); int (*finish)(struct typec_mux_dev *,struct typec_mux_state *); void *data; int (*set)(struct typec_mux_dev *,struct typec_mux_state *); int (*prepare)(struct typec_mux_dev *,struct typec_mux_state *); }; +struct task_struct { int unused; }; +static _Thread_local struct task_struct this_task; +#define current (&this_task) +struct mutex { pthread_mutex_t value; }; +struct typec_mux_context { struct mutex lock; }; +static struct typec_mux_context shared_context={.lock={.value=PTHREAD_MUTEX_INITIALIZER}}; +static int typec_mux_srcu; +static int srcu_read_lock(int *s) { (void)s;return 0; } +static void srcu_read_unlock(int *s,int i) { (void)s;(void)i; } +static int mutex_trylock(struct mutex *m) { return pthread_mutex_trylock(&m->value)==0; } +static void mutex_unlock(struct mutex *m) { assert(!pthread_mutex_unlock(&m->value)); } +struct typec_mux { struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied; unsigned num_mux_devs; struct typec_mux_dev *mux_devs[3]; }; +struct connector { bool connected; }; +struct apple_dcp { int external_lock,external_error; bool external_cable,external_stopped; struct { bool connected; } dptxport[1]; void *hdmi_hpd; bool active; bool external,typec_cable_connected; void *dev; struct connector *typec_connector; }; +struct apple_dcp_typec_route; +struct apple_dcp_typec_port { int link; void *connector_np; struct apple_dcp_typec_route *owner,*target,*secondary_target,*preferred_route; void *applied_alt; unsigned long applied_mode; unsigned applied_status,applied_conf; bool applied_valid,safe_pending,safe_finish_ready,dp_wanted,dp_hpd,hpd; }; +struct apple_dcp_typec_route { struct apple_dcp_typec_port *port; struct apple_dcp *dcp; void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel; }; +static int dcp_typec_fabric_lock,dcp_typec_ports,deactivate_calls; +struct drm_device { int unused; }; +static struct apple_dcp display; +static struct connector connector; +static struct apple_dcp_typec_port port; +static struct apple_dcp_typec_route route; +static struct apple_dcp_typec_port *sole_port(void) { return &port; } +static char events[100];static unsigned count; +static int power_error,phy_error,tbt_error,role_error,last_error,phy_calls,reset_calls,gpio_calls,role_calls,stops,unregisters; +static void event(char c) { assert(count+1external; } +static int dcp_dptx_disconnect_oob(void *dev,u32 index) { (void)dev;assert(index==0);event('Q');stops++;if(power_error){display.external_error=power_error;return power_error;}display.external_cable=false;display.external_stopped=true;display.dptxport[0].connected=false;display.typec_cable_connected=false;connector.connected=false;return 0; } +static struct drm_device *dcp_typec_drm(void) { return NULL; } +static void dcp_typec_plan(struct drm_device *d,struct apple_dcp_typec_port *p,unsigned n) { (void)d;(void)p;(void)n; } +static bool dcp_typec_tunnel_held(struct apple_dcp *d) { (void)d;return false; } +static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *r) { deactivate_calls++;r->selected=false;return 0; } +static void dcp_typec_port_attach(struct apple_dcp_typec_port *p) { (void)p; } +static int gpiod_get_value_cansleep(void *p) { (void)p;return 0; } +static int dcp_dptx_connect(struct apple_dcp *d,unsigned n) { (void)d;(void)n;return 0; } +static bool dcp_typec_route_available(struct apple_dcp_typec_route *r) { return !r->selected; } +static int dcp_typec_route_activate(struct apple_dcp_typec_route *r,void *x) { (void)x;r->selected=true;return 0; } +struct tps6598x; +struct tipd_data { REMOVE_RETURN (*remove)(struct tps6598x *); void (*unregister_port)(struct tps6598x *); int (*reset)(struct tps6598x *); }; +struct typec_port { struct typec_mux *mux; }; +struct tps6598x { bool remove_safe_prepared; struct typec_mux *mux; void *partner,*dev,*psy,*role_sw,*reset; int irq,wq_poll; struct typec_port *port; struct tipd_data *data; }; +struct cd321x { struct tps6598x tps; struct typec_mux *mux; void *tbt_switch; }; +struct typec_thunderbolt_switch_data { int state; }; +static void cd321x_stop_work(struct cd321x *c) { (void)c;event('W'); } +static int typec_thunderbolt_switch_set(void *p,void *s) { (void)p;(void)s;event('B');return tbt_error; } +static int usb_role_switch_set_role(void *p,int r) { (void)p;assert(r==USB_ROLE_NONE);event('R');role_calls++;return role_error; } +static void typec_unregister_partner(void *p) { (void)p;event('P'); } +#define typec_set_pwr_opmode(...) ((void)0) +#define typec_set_pwr_role(...) ((void)0) +#define typec_set_vconn_role(...) ((void)0) +static int typec_set_orientation(void *p,int o) { (void)p;assert(o==TYPEC_ORIENTATION_NONE);return 0; } +static void typec_set_data_role(struct typec_port *p,enum typec_data_role r) { (void)p;(void)r;event('D'); } +static void dcp_typec_pipeline_freed(void) { event('F'); } +static void power_supply_changed(void *p) { (void)p; } +static void tipd_debugfs_unregister(struct tps6598x *t) { (void)t; } +static void cancel_delayed_work_sync(void *w) { (void)w; } +static void devm_free_irq(void *d,int i,void *t) { (void)d;(void)i;(void)t; } +static void typec_mux_put(struct typec_mux *m) { (void)m; } +static void usb_role_switch_put(void *p) { (void)p;event('V'); } +static void gpiod_set_value_cansleep(void *p,int v) { (void)p;assert(v==1);gpio_calls++;event('G'); } + +'''.replace('REMOVE_RETURN','void' if parent else 'int') + functions='\n'.join(block(mux, marker) for marker in ('static void typec_mux_unlock(', 'void typec_mux_abort(', 'int typec_mux_begin(', 'int typec_mux_apply(', 'int typec_mux_end(', 'int typec_mux_prepare(')) if not parent else block((reference/'drivers/usb/typec/mux.c').read_text(),'int typec_mux_prepare(')+'\n' + functions+=block(mux,'int typec_mux_set(')+'\n'+block(dcp,'static int dcp_typec_route_prepare(')+'\n'+block(dcp,'static int dcp_typec_route_finish(')+'\n'+block(dcp,'static void dcp_typec_route_commit(')+'\n' + functions+=block(dcp,'static struct apple_dcp_typec_route *\ndcp_typec_rebalance_locked(')+'\n' + if not parent:functions+=block(core,'static void tps6598x_drop_partner(')+'\n'+block(core,'static int tps6598x_set_data_role(')+'\n' + functions+=block(core,'static '+('void' if parent else 'int')+' tps6598x_disconnect(')+'\n'+block(core,'static '+('void' if parent else 'int')+' cd321x_remove(')+'\n'+block(core,'void tipd_remove(')+'\n' + controls=r''' +static struct typec_mux_dev phy_provider,display_provider; +static struct typec_mux mux; +static struct typec_port class_port; +static struct tipd_data data; + +static int phy_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;assert(s->mode==TYPEC_STATE_SAFE);event('L');phy_calls++;return phy_error; } +static int display_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;event('S');if(port.owner&&display.external&&connector.connected) {int r=dcp_dptx_disconnect_oob(display.dev,0);if(r)return r;}if(port.safe_pending)return 0;port.owner=NULL;return 0; } +static void unregister_port(struct tps6598x *t) { (void)t;unregisters++;event('U'); } +static int reset(struct tps6598x *t) { (void)t;reset_calls++;event('X');return 0; } +static void initial(struct cd321x *c) { + count=0;events[0]=0;power_error=phy_error=tbt_error=role_error=last_error=phy_calls=reset_calls=gpio_calls=role_calls=stops=unregisters=deactivate_calls=0; + connector=(struct connector){.connected=true};display=(struct apple_dcp){.external=true,.external_cable=true,.dptxport={{.connected=true}},.typec_cable_connected=true,.typec_connector=&connector}; + port=(struct apple_dcp_typec_port){.owner=&route,.applied_valid=true,.dp_wanted=true,.dp_hpd=true,.hpd=true};route=(struct apple_dcp_typec_route){.port=&port,.dcp=&display,.selected=true}; + phy_provider=(struct typec_mux_dev){.set=phy_set};display_provider=(struct typec_mux_dev){.data=&route,.set=display_set,.prepare=dcp_typec_route_prepare,.finish=dcp_typec_route_finish,.commit=dcp_typec_route_commit}; + mux=(struct typec_mux){.context=&shared_context,.num_mux_devs=2,.mux_devs={&phy_provider,&display_provider}};class_port.mux=&mux; + data=(struct tipd_data){.remove=cd321x_remove,.unregister_port=unregister_port,.reset=reset}; + *c=(struct cd321x){.tps={.mux=&mux,.partner=(void *)1,.port=&class_port,.data=&data,.reset=(void *)1},.mux=&mux}; +} +static int position(char c) { for(unsigned i=0;i=0&&position('Q')mode==TYPEC_STATE_SAFE);event('L');", "(void)m;(void)s;event('L');",1) + actual=actual.replace('connector=(struct connector){.connected=true};','connector=(struct connector){.connected=true};',1) + actual=actual.replace('phy_provider=(struct typec_mux_dev){.set=phy_set};','display.typec_routes=&route;display.fixed_connector_type=DRM_MODE_CONNECTOR_USB;display.active_typec_route=&route;\n phy_provider=(struct typec_mux_dev){.set=phy_set};',1) + controls=r''' +static int late_finish_error(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;return -EIO; } + int main(int argc,char **argv) { + (void)argc;struct cd321x c;initial(&c);struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE}; + if(argv[1][0]=='o') { + assert(typec_mux_set(&mux,&safe)==0); + assert(stops==1&&!port.owner&&!port.safe_pending&&!route.selected); + puts("PASS actual DCP prepare + route_set + route_deactivate + framework normal SAFE"); + } + if(argv[1][0]=='r') { + mux.mux_devs[0]=&display_provider;mux.mux_devs[1]=&phy_provider;phy_error=-ETIMEDOUT; + assert(typec_mux_set(&mux,&safe)==-ETIMEDOUT); + fprintf(stderr,"display-first failed SAFE: owner=%p lease=%d selected=%d events=%s\n",(void *)port.owner,port.safe_pending,route.selected,events); + assert(port.owner==&route&&port.safe_pending&&route.selected); + } + if(argv[1][0]=='d') { + phy_error=-ETIMEDOUT;assert(typec_mux_set(&mux,&safe)==-ETIMEDOUT&&port.safe_pending&&port.owner==&route); + phy_error=0;struct typec_altmode alt={.svid=USB_TYPEC_DP_SID}; + struct typec_displayport_data data={.status=DP_STATUS_HPD_STATE}; + struct typec_mux_state dp={.alt=&alt,.mode=TYPEC_DP_STATE_A,.data=&data}; + int ret=typec_mux_set(&mux,&dp); + fprintf(stderr,"DP after failed SAFE: ret=%d connects=%d pending=%d wanted=%d hpd=%d\n",ret,connect_calls,port.safe_pending,port.dp_wanted,port.hpd); + assert(ret<0&&connect_calls==0&&!port.dp_wanted); + } + if(argv[1][0]=='a') { + struct altmode alt={.mux=&mux,.adev={.svid=USB_TYPEC_DP_SID}};phy_error=-ETIMEDOUT; + int ret=typec_altmode_set_mux(&alt,TYPEC_STATE_SAFE,NULL); + fprintf(stderr,"altmode SAFE: ret=%d stops=%d phy_calls=%d lease=%d\n",ret,stops,phy_calls,port.safe_pending); + assert(ret==-ETIMEDOUT&&stops==1&&port.safe_pending); + } + if(argv[1][0]=='x') { + xbar_error=-EIO;assert(typec_mux_set(&mux,&safe)==-EIO); + assert(port.owner==&route&&port.safe_pending&&!port.safe_finish_ready&&route.selected&&route.deactivate_xbar_done&&route.deactivate_error==-EIO); + xbar_error=0;assert(typec_mux_set(&mux,&safe)==-EIO&&port.owner==&route&&port.safe_pending&&xbar_calls==1); + } + if(argv[1][0]=='f') { + mux.mux_devs[0]=&display_provider;mux.mux_devs[1]=&phy_provider;phy_provider.finish=late_finish_error; + assert(typec_mux_set(&mux,&safe)==-EIO); + assert(port.owner==&route&&port.safe_pending&&port.safe_finish_ready&&!route.selected&&xbar_calls==1); + assert(!dcp_typec_route_available(&route)); + struct apple_dcp peer={0};struct apple_dcp_typec_route peer_route={.dcp=&peer}; + assert(dcp_typec_route_available(&peer_route)); + phy_provider.finish=NULL; + assert(typec_mux_set(&mux,&safe)==0&&!port.owner&&!port.safe_pending&&xbar_calls==1); + } + return 0; + } + ''' + bus=(ROOT/'drivers/usb/typec/bus.c').read_text() + actual+='\nstruct altmode { struct typec_mux *mux; struct typec_altmode adev; };\n'+block(bus,'static inline int\ntypec_altmode_set_mux(')+'\n'+controls + check('actual-dcp-normal-safe',actual,(('ordered',),)) + check('actual-dcp-finalizer-recovery',actual,(('xbar-error',),('finalizer',))) + check('provider-order-releases-failed-lease',actual,(('release',),)) + check('failed-safe-allows-new-dp-admission',actual,(('dp',),)) + check('altmode-safe-skips-quiescence',actual,(('altmode',),)) + + # Real pthread fabric/consumer mutexes. Pause the SAFE actor after its actual + # DCP prepare has returned and released its locks, before any set callback. + # A second handle can reference the same providers (class/altmode/CD handles + # are separately acquired in the real framework). + concurrent=actual.replace(block(actual,'int main('),block(actual,'int main(').replace('int main(','int __attribute__((unused)) route_controls_main(',1),1) + concurrent=concurrent.replace('#define guard(k) (void)','') + concurrent=concurrent.replace('#define scoped_guard(k,l) for(int once=((void)(l),1);once;once=0)','') + locks=r''' + #include + static pthread_mutex_t fabric_mutex=PTHREAD_MUTEX_INITIALIZER,external_mutex=PTHREAD_MUTEX_INITIALIZER,tb_mutex=PTHREAD_MUTEX_INITIALIZER; + struct real_guard { pthread_mutex_t *mutex; bool held; }; + static struct real_guard take_lock(int *key) { + pthread_mutex_t *m=key==&dcp_typec_fabric_lock?&fabric_mutex:key==&display.external_lock?&external_mutex:&tb_mutex; + assert(pthread_mutex_lock(m)==0);return (struct real_guard){m,true}; + } + static void drop_lock(struct real_guard *g) { if(g->held){g->held=false;assert(pthread_mutex_unlock(g->mutex)==0);} } + #define guard(k) struct real_guard __attribute__((cleanup(drop_lock))) _guard = take_lock + #define scoped_guard(k,l) for(struct real_guard __attribute__((cleanup(drop_lock))) _scope=take_lock(l);_scope.held;drop_lock(&_scope)) + static bool observed_unquiesced_safe; + ''' + pos=concurrent.index('static int dcp_typec_route_deactivate(') + concurrent=concurrent[:pos]+locks+concurrent[pos:] + concurrent=concurrent.replace("(void)m;(void)s;event('L');phy_calls++;", "(void)m;if(s->mode==TYPEC_STATE_SAFE&&display.dptxport[0].connected)observed_unquiesced_safe=true;event('L');phy_calls++;",1) + concurrent+=r''' + static pthread_mutex_t schedule_lock=PTHREAD_MUTEX_INITIALIZER; + static pthread_cond_t schedule_cond=PTHREAD_COND_INITIALIZER; + static bool prepared,continue_safe; + static int safe_ret,dp_ret; + static int scheduled_prepare(struct typec_mux_dev *m,struct typec_mux_state *s) { + int ret=dcp_typec_route_prepare(m,s); + if(s->mode==TYPEC_STATE_SAFE&&!s->alt) { + pthread_mutex_lock(&schedule_lock);prepared=true;pthread_cond_broadcast(&schedule_cond); + while(!continue_safe)pthread_cond_wait(&schedule_cond,&schedule_lock); + pthread_mutex_unlock(&schedule_lock); + } + return ret; + } + static void *safe_actor(void *unused) { + (void)unused;struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE};safe_ret=typec_mux_set(&mux,&safe);return NULL; + } + int main(void) { + struct cd321x c;initial(&c);display_provider.prepare=scheduled_prepare; + pthread_t thread;assert(pthread_create(&thread,NULL,safe_actor,NULL)==0); + pthread_mutex_lock(&schedule_lock);while(!prepared)pthread_cond_wait(&schedule_cond,&schedule_lock);pthread_mutex_unlock(&schedule_lock); + struct typec_mux second_handle=mux; + struct typec_altmode alt={.svid=USB_TYPEC_DP_SID};struct typec_displayport_data data={.status=DP_STATUS_HPD_STATE}; + struct typec_mux_state dp={.alt=&alt,.mode=TYPEC_DP_STATE_A,.data=&data}; + dp_ret=typec_mux_set(&second_handle,&dp); + pthread_mutex_lock(&schedule_lock);continue_safe=true;pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_lock); + assert(pthread_join(thread,NULL)==0); + fprintf(stderr,"overlapping handles: dp_ret=%d safe_ret=%d connected_at_SAFE_PHY=%d events=%s\n",dp_ret,safe_ret,observed_unquiesced_safe,events); + assert(!observed_unquiesced_safe); + } + ''' + check('concurrent-prepare-set-stale-quiescence',concurrent,flags=('-pthread',)) + check('connector-context-lifetime', lifetime(ROOT), flags=('-pthread',)) + check('probe-transaction-rollback', probe(ROOT)) + check('provider-unregister-drain', unregister(ROOT), flags=('-pthread',)) + check('park-before-host-role-exit', clock_order(ROOT)) + return tests + + +def lifetime(root): + mux = (root / 'drivers/usb/typec/mux.c').read_text() + unit = quiesce.make_unit(root) + unit = unit.replace(block(unit, 'int main('), + block(unit, 'int main(').replace('int main(', 'int __attribute__((unused)) prior_main(', 1), 1) + unit = unit.replace('struct typec_mux_context { struct mutex lock; };', r""" +struct list_head { struct list_head *next,*prev; }; +struct fwnode_handle { unsigned refs; }; +struct typec_mux_context { struct list_head list; struct fwnode_handle *fwnode; struct mutex lock; unsigned users; }; +""") + helpers = r""" +#include +#include +#undef guard +#undef list_for_each_entry +static struct mutex typec_mux_contexts_lock={.value=PTHREAD_MUTEX_INITIALIZER}; +static struct list_head typec_mux_contexts={&typec_mux_contexts,&typec_mux_contexts}; +static struct fwnode_handle *fwnode_handle_get(struct fwnode_handle *f) { f->refs++;return f; } +static void fwnode_handle_put(struct fwnode_handle *f) { assert(f->refs);f->refs--; } +static void mutex_init(struct mutex *m) { assert(!pthread_mutex_init(&m->value,NULL)); } +static struct mutex *registry_lock(struct mutex *m) { assert(!pthread_mutex_lock(&m->value));return m; } +static void registry_unlock(struct mutex **m) { mutex_unlock(*m); } +#define guard(k) struct mutex * __attribute__((cleanup(registry_unlock))) held=registry_lock +#define kzalloc_obj(x) calloc(1,sizeof(x)) +#define kfree(x) free(x) +#define ERR_PTR(e) ((void *)(intptr_t)(e)) +#define list_for_each_entry(p,h,m) for(p=container_of((h)->next,__typeof__(*p),m); &(p)->m!=(h);p=container_of((p)->m.next,__typeof__(*p),m)) +static void list_add_tail(struct list_head *n,struct list_head *h) { n->prev=h->prev;n->next=h;h->prev->next=n;h->prev=n; } +static void list_del(struct list_head *n) { n->prev->next=n->next;n->next->prev=n->prev; } +""" + unit += helpers + block(mux, 'static struct typec_mux_context *typec_mux_context_get(') + '\n' + unit += block(mux, 'static void typec_mux_context_put(') + '\n' + unit += r""" +static struct typec_mux downstream; +static int forwarded,commits,aborts,finish_error; +static int forward_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;forwarded++;return typec_mux_set(&downstream,s); } +static int finish(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;return finish_error; } +static void commit(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;commits++; } +static void abort_callback(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;aborts++; } +int main(void) { + struct fwnode_handle a={0},b={0}; + struct typec_mux_context *one=typec_mux_context_get(&a),*two=typec_mux_context_get(&a),*other=typec_mux_context_get(&b); + assert(one==two&&one!=other&&one->users==2&&a.refs==1&&b.refs==1); + struct cd321x c;initial(&c);struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE}; + struct typec_mux second=mux; mux.context=one;second.context=two; + assert(typec_mux_begin(&mux,&safe)==0); + unsigned before=phy_calls;assert(typec_mux_set(&second,&safe)==-EBUSY&&phy_calls==before); + typec_mux_abort(&mux);assert(port.safe_pending&&port.owner==&route); + phy_error=-ETIMEDOUT;assert(typec_mux_set(&second,&safe)==-ETIMEDOUT&&port.owner==&route); + phy_error=0;assert(typec_mux_set(&second,&safe)==0&&!port.owner&&!port.safe_pending); + struct typec_mux_dev leaf={.set=phy_set,.finish=finish,.commit=commit,.abort=abort_callback}; + downstream=(struct typec_mux){.context=other,.num_mux_devs=1,.mux_devs={&leaf}}; + struct typec_mux_dev forward={.set=forward_set}; + mux=(struct typec_mux){.context=one,.num_mux_devs=1,.mux_devs={&forward}}; + assert(typec_mux_set(&mux,&safe)==0&&forwarded==1&&commits==1); + finish_error=-EIO;assert(typec_mux_set(&downstream,&safe)==-EIO&&commits==1&&aborts==1); + finish_error=0;assert(typec_mux_set(&downstream,&safe)==0&&commits==2); + downstream.context=one;before=phy_calls; + assert(typec_mux_set(&mux,&safe)==-EBUSY&&phy_calls==before); // cyclic same-connector forwarding + leaf.removed=true;downstream.context=other;before=phy_calls; + assert(typec_mux_set(&downstream,&safe)==-ENODEV&&phy_calls==before&&aborts==1); + typec_mux_context_put(one);assert(a.refs==1&&two->users==1); + typec_mux_context_put(two);assert(a.refs==0); + typec_mux_context_put(other);assert(b.refs==0&&typec_mux_contexts.next==&typec_mux_contexts); + puts("PASS actual connector registry, shared handles, forwarding, finalize/abort and removed-provider guards"); +} +""" + return unit + + +def probe(root): + core = (root / 'drivers/usb/typec/tipd/core.c').read_text() + unit = quiesce.make_unit(root) + unit = unit.replace(block(unit, 'int main('), block(unit, 'int main(').replace('int main(', 'int __attribute__((unused)) prior_main(', 1), 1) + unit = unit.replace('struct tipd_data {', 'struct fwnode_handle;\nstruct tipd_data { unsigned irq_mask1; int (*init)(struct tps6598x *); int (*switch_power_state)(struct tps6598x *, int); int (*register_port)(struct tps6598x *, struct fwnode_handle *); bool (*read_data_status)(struct tps6598x *); int (*connect)(struct tps6598x *,u32); void (*ready)(struct tps6598x *); void *irq_handler;') + unit = unit.replace('struct tps6598x {', 'struct tps6598x { bool dbma_at_probe,wakeup;') + helpers = r""" +#undef dev_err +#define dev_err(...) ((void)0) +#define TPS_SETUP_MS 1000 +#define TPS_SYSTEM_POWER_STATE_S0 0 +#define TPS_MODE_PTCH 2 +#define TPS_MODE_DBMA 3 +#define TPS_REG_VID 0 +#define TPS_REG_INT_MASK1 1 +#define TPS_STATUS_PLUG_PRESENT 1 +#define GPIOD_OUT_LOW 0 +#include +#undef IS_ERR +#define IS_ERR(p) ((uintptr_t)(p)>=(uintptr_t)-4095) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define IRQF_SHARED 0 +#define IRQF_ONESHOT 0 +#define POLL_INTERVAL 1000 +#define INIT_DELAYED_WORK(...) ((void)0) +#define tps6598x_poll_work 0 +#define devm_device_init_wakeup(...) ((void)0) +#define enable_irq_wake(...) ((void)0) +#define dev_err_probe(...) (-EINVAL) +struct fwnode_handle { int unused; }; +static int connect_error,irq_error,register_error,node_puts,mux_get_error; +static void *devm_gpiod_get_optional(void *d,const char *s,int v) { (void)d;(void)s;(void)v;return NULL; } +static void msleep(unsigned ms) { (void)ms; } +static bool device_is_compatible(void *d,const char *s) { (void)d;(void)s;return false; } +static int tps6598x_read32(struct tps6598x *t,int reg,u32 *v) { (void)t;(void)reg;*v=1;return 0; } +static int tps6598x_check_mode(struct tps6598x *t) { (void)t;return 0; } +static int tps6598x_write64(struct tps6598x *t,int reg,unsigned v) { (void)t;(void)reg;(void)v;return 0; } +static bool tps6598x_read_status(struct tps6598x *t,u32 *s) { (void)t;*s=TPS_STATUS_PLUG_PRESENT;return true; } +static bool tps6598x_read_power_status(struct tps6598x *t) { (void)t;return true; } +static struct fwnode_handle node; +static struct fwnode_handle *device_get_named_child_node(void *d,const char *s) { (void)d;(void)s;return &node; } +static void fw_devlink_purge_absent_suppliers(struct fwnode_handle *f) { (void)f; } +static void *fwnode_usb_role_switch_get(struct fwnode_handle *f) { (void)f;return (void *)1; } +static struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *f) { (void)f;return mux_get_error?(struct typec_mux *)(intptr_t)mux_get_error:&mux; } +static void fwnode_handle_put(struct fwnode_handle *f) { (void)f;node_puts++; } +static int devm_tps6598_psy_register(struct tps6598x *t) { (void)t;return 0; } +static int devm_request_threaded_irq(void *d,int irq,void *h,void *th,int flags,const char *name,void *arg) { (void)d;(void)irq;(void)h;(void)th;(void)flags;(void)name;(void)arg;return irq_error; } +static int system_power_efficient_wq; +static unsigned msecs_to_jiffies(unsigned ms) { return ms; } +static void queue_delayed_work(int wq,int *work,unsigned delay) { (void)wq;(void)work;(void)delay; } +static bool device_property_read_bool(void *d,const char *p) { (void)d;(void)p;return false; } +static void tipd_debugfs_register(struct tps6598x *t) { (void)t; } +static int register_port(struct tps6598x *t,struct fwnode_handle *f) { (void)t;(void)f;return register_error; } +static bool read_data_status(struct tps6598x *t) { (void)t;return true; } +static int connect(struct tps6598x *t,u32 status) { (void)t;(void)status;return connect_error; } +""" + unit += helpers + block(core, 'int tipd_init(') + r""" +static void probe_initial(struct cd321x *c) { + initial(c);c->tps.irq=1;data.register_port=register_port;data.read_data_status=read_data_status;data.connect=connect; + connect_error=irq_error=register_error=node_puts=mux_get_error=0; +} +int main(void) { + struct cd321x c; + probe_initial(&c);irq_error=-ENOSPC;phy_error=-ETIMEDOUT; + assert(tipd_init(&c.tps)==-ENOSPC&&reset_calls==0&&gpio_calls==0&&port.owner==&route&&port.safe_pending&&unregisters==1&&node_puts==1); + probe_initial(&c);connect_error=-EAGAIN;role_error=-EBUSY; + assert(tipd_init(&c.tps)==-EAGAIN&&reset_calls==0&&gpio_calls==0&&phy_calls==1&&port.owner==&route&&port.safe_pending); + probe_initial(&c);irq_error=-ENOSPC; + assert(tipd_init(&c.tps)==-ENOSPC&&reset_calls==1&&!port.owner&&!port.safe_pending&&unregisters==1); + probe_initial(&c);mux_get_error=-ENODEV; + assert(tipd_init(&c.tps)==-ENODEV&&reset_calls==0&&stops==0&&phy_calls==0&&unregisters==0&&node_puts==1); + probe_initial(&c);register_error=-EIO; + assert(tipd_init(&c.tps)==-EIO&&reset_calls==0&&stops==0&&phy_calls==0&&unregisters==0); + puts("PASS actual probe rollback preserves primary errors and blocks reset after SAFE/role/registration errors"); +} +""" + return unit + + +def unregister(root): + """Actual unregister/begin/end with sleeping SRCU leaves and two threads.""" + source = (root / 'drivers/usb/typec/mux.c').read_text() + unit = quiesce.make_unit(root) + unit = unit.replace(block(unit, 'int main('), block(unit, 'int main(').replace('int main(', 'int __attribute__((unused)) prior_main(', 1), 1) + unit = unit.replace('struct typec_mux_dev {', 'struct device { bool unregistered; };\nstruct typec_mux_dev { struct device dev;') + old = block(unit, 'static int srcu_read_lock(') + '\n' + block(unit, 'static void srcu_read_unlock(') + leaves = r""" +static pthread_mutex_t lifetime_lock=PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t lifetime_cond=PTHREAD_COND_INITIALIZER; +static unsigned readers; +static bool draining; +static int srcu_read_lock(int *s) { (void)s;pthread_mutex_lock(&lifetime_lock);readers++;pthread_mutex_unlock(&lifetime_lock);return 0; } +static void srcu_read_unlock(int *s,int i) { (void)s;(void)i;pthread_mutex_lock(&lifetime_lock);assert(readers);readers--;pthread_cond_broadcast(&lifetime_cond);pthread_mutex_unlock(&lifetime_lock); } +static void synchronize_srcu(int *s) { (void)s;pthread_mutex_lock(&lifetime_lock);draining=true;pthread_cond_broadcast(&lifetime_cond);while(readers)pthread_cond_wait(&lifetime_cond,&lifetime_lock);pthread_mutex_unlock(&lifetime_lock); } +static void device_unregister(struct device *d) { assert(!readers);d->unregistered=true; } +#define WRITE_ONCE(x,v) ((x)=(v)) +""" + unit = unit.replace(old, leaves, 1) + unit += block(source, 'void typec_mux_unregister(') + r""" +static void *remove_provider(void *arg) { typec_mux_unregister(arg);return NULL; } +int main(void) { + struct cd321x c;initial(&c);struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE}; + assert(typec_mux_begin(&mux,&safe)==0&&readers==1); + pthread_t thread;assert(!pthread_create(&thread,NULL,remove_provider,&phy_provider)); + pthread_mutex_lock(&lifetime_lock);while(!draining)pthread_cond_wait(&lifetime_cond,&lifetime_lock);pthread_mutex_unlock(&lifetime_lock); + assert(phy_provider.removed&&!phy_provider.dev.unregistered&&readers==1); + assert(typec_mux_end(&mux)==0); // pre-existing transaction finishes before removal returns + assert(!pthread_join(thread,NULL)&&phy_provider.dev.unregistered&&readers==0); + unsigned before=phy_calls; + assert(typec_mux_set(&mux,&safe)==-ENODEV&&phy_calls==before&&readers==0); + puts("PASS actual unregister drains the open transaction and tombstone blocks stale handles"); +} +""" + return unit + + +def clock_order(root): + unit = quiesce.make_unit(root) + unit = unit.replace(block(unit, 'int main('), block(unit, 'int main(').replace('int main(', 'int __attribute__((unused)) prior_main(', 1), 1) + unit = unit.replace('static int power_error,', 'static bool backend_clock=true,parked;\nstatic int power_error,') + unit = unit.replace("event('R');role_calls++;return role_error;", "event('R');role_calls++;if(!role_error)backend_clock=false;return role_error;", 1) + unit = unit.replace("event('L');phy_calls++;return phy_error;", "event('L');phy_calls++;if(parked)return 0;if(!backend_clock)return -ENODEV;if(!phy_error)parked=true;return phy_error;", 1) + unit += r""" +int main(void) { + (void)current;struct cd321x c;initial(&c); + tipd_remove(&c.tps); + assert(!last_error&&reset_calls==1&&gpio_calls==1&&parked&&!backend_clock); + assert(position('Q') Date: Wed, 14 Oct 2026 04:00:16 -0400 Subject: [PATCH 0967/1579] soc: apple: enable SEP autoload and writable xART defaults Select the SEP driver as a module by default and export its OF device table for modpost. The missing alias left SEP unbound on a fresh Neo install until apple_sep was loaded manually. Default xart_writes to 1 while preserving APFS extent ownership, record validation and write checks. Keep xart_writes=0 as the read-only override and leave identity-keybag provisioning opt-in. Update policy documentation and the Arch package checksums. Validation: 12 SEP host tests pass; the module builds with W=1 using the prepared J700 kernel build. Neo rebooted with the exact candidate module loading automatically, attached, and reporting validated writable xART. Two 64-byte hardware RNG reads passed. The shared iBoot-container hash remained unchanged; no actual xART write or Touch ID operation is claimed. Assisted-by: LLM Signed-off-by: DjDeveloperr --- drivers/soc/apple/Kconfig | 9 +++++---- drivers/soc/apple/sep.rs | 12 ++++++++---- tools/aurora-sep/README.md | 11 +++++++---- tools/aurora-sep/XART-SAFETY.md | 11 +++++++++++ tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD | 6 +++--- 5 files changed, 34 insertions(+), 15 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 49152c8b5e1f0..8ef7f37c0661d 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -116,6 +116,7 @@ config APPLE_SEP_FV_INTERFACE config APPLE_SEP tristate "Apple SEP (Secure Enclave Processor)" + default m if ARCH_APPLE depends on ARCH_APPLE depends on PM depends on RUST @@ -141,10 +142,10 @@ config APPLE_SEP the iBoot system container, locating its .gl extent through read-only, checksum-validated APFS metadata. It validates the complete store and both root records before registering with SEP. - Raw APFS xART writes are disabled by default. An explicit xart_writes - opt-in is accepted only when the driver validates narrow physical - extent ownership invariants; this does not prove all macOS-compatible - update behavior. Keep it off until the target machine has been checked. + xART writes are enabled by default, but accepted only when the driver + validates the physical extent ownership invariants. Unsupported APFS + layouts fail closed. Set xart_writes=0 for read-only operation. + Identity-keybag provisioning remains a separate opt-in. With writes off the driver still attaches, but refuses Touch ID enrol and verify: the enclave saves anti-replay state after every match. diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index c5b12c8214416..d58ea01064fc4 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2316,8 +2316,12 @@ impl WorkItem for SepData { struct SepDriver(Arc); -const OF_TABLE: kernel::device_id::IdArray = - kernel::device_id::IdArray::new([(of::DeviceId::new(c"apple,sep"), ())]); +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c"apple,sep"), ())] +); impl platform::Driver for SepDriver { type IdInfo = (); @@ -2442,8 +2446,8 @@ module! { license: "Dual MIT/GPL", params: { xart_writes: u8 { - default: 0, - description: "Opt in to shared xART writes only if APFS proves a single unsnapshotted .gl extent; Touch ID enrol and verify need it", + default: 1, + description: "Allow shared xART writes only if APFS proves a single unsnapshotted .gl extent; set to 0 for read-only operation", }, xart_start_sector: u64 { default: 0, diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 96632f81c585d..e56a1311876a3 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -28,12 +28,15 @@ for example with `systemctl edit aurora-sep.service`: Environment=AURORA_SEP_READY_TIMEOUT=180 ``` -`xart_writes=0` remains the default. `load-driver` explicitly opts into writes -and must only be enabled on a machine whose APFS `.gl` extent passes the -driver's narrow ownership checks: one physical extent, no snapshots or revert +The driver defaults to `xart_writes=1` and exports the `apple,sep` module +alias for automatic loading. Writes are accepted only when the APFS `.gl` +extent passes the driver's narrow ownership checks: one physical extent, no snapshots or revert state, and a single extent reference owned by the `.gl` inode. The kernel resolves that extent from checksum-validated APFS metadata; unsupported layouts -fail closed. The opt-in has been tested on m2 with identity-keybag provisioning, +fail closed. Set `options apple_sep xart_writes=0 provision_keybag=0` in a +modprobe configuration to retain read-only operation. Identity-keybag creation +still requires `provision_keybag=1`; `load-driver` supplies that option. +Enabling writes has been tested on m2 with identity-keybag provisioning, trusted-key seal/unseal, and reboot reload. The raw iBoot container hash did not change in those tests, so an actual xART write was **not** validated. Do not infer that all SEP or Touch ID operations work with a read-only GigaLocker. diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md index 52a976c63e5c4..33af5b7b97651 100644 --- a/tools/aurora-sep/XART-SAFETY.md +++ b/tools/aurora-sep/XART-SAFETY.md @@ -1,5 +1,16 @@ # Shared xART gigalocker safety gate +## Current default + +`xart_writes=1` is the driver default. The APFS extent-ownership checks, +record validation and write-ordering checks remain mandatory; unsupported +layouts fail closed. `xart_writes=0` explicitly selects read-only operation. +Identity-keybag creation remains a separate `provision_keybag=1` opt-in. +The historical evidence below records the earlier opt-in deployments and +does not establish qualification on every supported Mac or APFS layout. + +## Earlier deployment policy + The iBoot APFS container and its XART `.gl` file are shared with macOS. The deployed `m2fix4` kernel remains read-only. The `m2fix5` source has an explicit `xart_writes=1` opt-in; it accepts writes only when the APFS-resolved extent is diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 15712b3bdf4e2..c0a935fa93e18 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -7,7 +7,7 @@ pkgname=aurora-sep pkgver=1 -pkgrel=3 +pkgrel=4 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -40,8 +40,8 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' - 'd7a09f31367c181a2cbda63cd58104d6117136fb5db5b20f4a3996ea3af7025e' - '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') + '7777f399b05f9ed860687140efdb0e07ea7b2d23939fe59ec4894690dff299cc' + '6628ae9c63265914e7206edf61e42c613810d9d9a61adc6c7c02327504911a42') # A unit and the helper it runs ($1, installed as $3), pointed at the # packaged helper. From 790d5a81066bd922bea78dce43ea7e5fc60d4ca4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 04:28:57 -0400 Subject: [PATCH 0968/1579] soc: apple: provision the Linux SEP identity automatically A fresh installation can attach SEP and use its RNG yet fail keyctl with ENODEV because no Linux identity keybag or machine reference key exists. Enable existing first-boot identity provisioning by default. Recover valid persisted identities on later boots and retain the existing refusal to replace unreadable identities or reference keys. Keep the explicit xart_writes=0 provision_keybag=0 read-only configuration and APFS checks. Update configuration help, documentation and package checksums. Validation: 12 SEP host tests pass and the module builds with W=1. Neo provisioned its missing identity using the existing provisioning path, then loaded the same trusted-key blob after three reboots. An encrypted payload authenticated under that recovered master key, corrupted trusted blobs were rejected, and identity/reference-key hashes stayed unchanged. The final module loaded automatically, with no provisioning override, and recovered the existing identity successfully. Touch ID was not tested. Assisted-by: LLM Signed-off-by: DjDeveloperr --- drivers/soc/apple/Kconfig | 5 +++-- drivers/soc/apple/sep.rs | 4 ++-- tools/aurora-sep/README.md | 6 ++++-- tools/aurora-sep/XART-SAFETY.md | 5 +++-- tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD | 6 +++--- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 8ef7f37c0661d..639435f966c3f 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -144,8 +144,9 @@ config APPLE_SEP both root records before registering with SEP. xART writes are enabled by default, but accepted only when the driver validates the physical extent ownership invariants. Unsupported APFS - layouts fail closed. Set xart_writes=0 for read-only operation. - Identity-keybag provisioning remains a separate opt-in. + layouts fail closed. A Linux identity keybag is created automatically + when none exists; existing identities and reference keys are retained. + Set xart_writes=0 provision_keybag=0 for read-only operation. With writes off the driver still attaches, but refuses Touch ID enrol and verify: the enclave saves anti-replay state after every match. diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index d58ea01064fc4..0cb129220a953 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2454,8 +2454,8 @@ module! { description: "Expected 512-byte start sector of the APFS .gl extent in the iBoot container. Zero accepts the APFS result; nonzero must match it and cannot override it", }, provision_keybag: u8 { - default: 0, - description: "Create the Linux identity keybag when none exists; requires xart_writes=1", + default: 1, + description: "Create the Linux identity keybag when none exists; set to 0 to disable first-boot provisioning; requires xart_writes=1", }, os_uuid_hi: u64 { default: 0, diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index e56a1311876a3..ea9d2704e9e80 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -34,8 +34,10 @@ extent passes the driver's narrow ownership checks: one physical extent, no snap state, and a single extent reference owned by the `.gl` inode. The kernel resolves that extent from checksum-validated APFS metadata; unsupported layouts fail closed. Set `options apple_sep xart_writes=0 provision_keybag=0` in a -modprobe configuration to retain read-only operation. Identity-keybag creation -still requires `provision_keybag=1`; `load-driver` supplies that option. +modprobe configuration to retain read-only operation. `provision_keybag=1` +is also the default: first boot creates the Linux identity needed for trusted +keys, and later boots recover that identity and its machine reference key. +An existing unreadable or invalid identity is not silently replaced. Enabling writes has been tested on m2 with identity-keybag provisioning, trusted-key seal/unseal, and reboot reload. The raw iBoot container hash did not change in those tests, so an actual xART write was **not** validated. Do not diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md index 33af5b7b97651..5dd70e8b26a80 100644 --- a/tools/aurora-sep/XART-SAFETY.md +++ b/tools/aurora-sep/XART-SAFETY.md @@ -4,8 +4,9 @@ `xart_writes=1` is the driver default. The APFS extent-ownership checks, record validation and write-ordering checks remain mandatory; unsupported -layouts fail closed. `xart_writes=0` explicitly selects read-only operation. -Identity-keybag creation remains a separate `provision_keybag=1` opt-in. +layouts fail closed. `xart_writes=0 provision_keybag=0` explicitly selects +read-only operation. `provision_keybag=1` is the default: create the Linux +identity if missing and recover an existing identity without replacing it. The historical evidence below records the earlier opt-in deployments and does not establish qualification on every supported Mac or APFS layout. diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index c0a935fa93e18..5c0f665f9dcfe 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -7,7 +7,7 @@ pkgname=aurora-sep pkgver=1 -pkgrel=4 +pkgrel=5 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -40,8 +40,8 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' - '7777f399b05f9ed860687140efdb0e07ea7b2d23939fe59ec4894690dff299cc' - '6628ae9c63265914e7206edf61e42c613810d9d9a61adc6c7c02327504911a42') + 'ed507587d3356ff2713e988e024b183772bab14061bf787bcaef28691867af0d' + '3d92b3be06bc1b670666dec3d0032893fe8c7be428a4f1200d7c9803980c769a') # A unit and the helper it runs ($1, installed as $3), pointed at the # packaged helper. From 50a711417171e48a07340999bf43bb8c281b14d7 Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Tue, 13 Oct 2026 02:54:20 +0200 Subject: [PATCH 0969/1579] pmdomain: apple: leave firmware-owned power domains untouched On MacBook Neo (J700) m1n1 hands the display coprocessor (DCP) over running, and its firmware owns the DISP_CPU power state: writing that register raises an SError. The domain is marked apple,always-on, but the driver still writes it at probe: it sets automatic gating when the domain is on, or forces it on when it is not. The kernel panicked with an asynchronous SError in apple_pmgr_ps_probe before any console. Add apple,firmware-owned: the driver only reads such a domain, keeps it on for its consumers (the boot framebuffer), and never writes its minimum state, power target, automatic gating or reset. Mark J700's disp_cpu with it. Co-Authored-By: Claude Opus 5.5 --- .../bindings/power/apple,pmgr-pwrstate.yaml | 7 ++++ arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 1 + drivers/pmdomain/apple/pmgr-pwrstate.c | 33 +++++++++++++++++-- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index d131441214db1..d86c8b68d1b48 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -79,6 +79,13 @@ properties: Forces this power domain to always be powered up. type: boolean + apple,firmware-owned: + description: + The boot loader hands over the coprocessor in this domain running, and its + firmware owns the domain's power state; writing it faults. The OS keeps the + domain on and never changes its power target, automatic gating or reset. + type: boolean + apple,min-state: description: Specifies the minimum power state for auto-PM. diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 6e6098144b757..8438e4f17e81f 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -397,6 +397,7 @@ * firmware owns this state; writing it faults. */ apple,always-on; + apple,firmware-owned; }; diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index c5699abae3d93..47f9593ae1221 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -49,6 +49,7 @@ struct apple_pmgr_ps { bool force_reset; bool externally_clocked; bool system_sleep; + bool firmware_owned; }; #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) @@ -160,6 +161,10 @@ static int apple_pmgr_ps_power_on(struct generic_pm_domain *genpd) struct apple_pmgr_ps *ps = genpd_to_apple_pmgr_ps(genpd); int ret; + /* Its coprocessor's firmware owns the state; Linux only holds a vote. */ + if (ps->firmware_owned) + return 0; + ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); if (ret && !ps->system_sleep) apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); @@ -171,6 +176,9 @@ static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd) { int ret; + if (genpd_to_apple_pmgr_ps(genpd)->firmware_owned) + return 0; + ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); if (ret) apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); @@ -207,6 +215,9 @@ static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned struct apple_pmgr_ps *ps = rcdev_to_apple_pmgr_ps(rcdev); unsigned long flags; + if (ps->firmware_owned) + return -EPERM; + spin_lock_irqsave(&ps->genpd.slock, flags); if (ps->genpd.status == GENPD_STATE_OFF) @@ -230,6 +241,9 @@ static int apple_pmgr_reset_deassert(struct reset_controller_dev *rcdev, unsigne struct apple_pmgr_ps *ps = rcdev_to_apple_pmgr_ps(rcdev); unsigned long flags; + if (ps->firmware_owned) + return -EPERM; + spin_lock_irqsave(&ps->genpd.slock, flags); dev_dbg(ps->dev, "PS 0x%x: deassert reset\n", ps->offset); @@ -321,8 +335,15 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) ps->genpd.power_on = apple_pmgr_ps_power_on; ps->genpd.power_off = apple_pmgr_ps_power_off; + /* + * A domain whose coprocessor the boot loader hands over running (the DCP) is + * owned by that coprocessor's firmware: writing its state faults. Only read + * it, keep it on and never change its target or automatic gating. + */ + ps->firmware_owned = of_property_read_bool(node, "apple,firmware-owned"); + ret = of_property_read_u32(node, "apple,min-state", &ps->min_state); - if (ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE) + if (ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE && !ps->firmware_owned) regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_PS_MIN, FIELD_PREP(APPLE_PMGR_PS_MIN, ps->min_state)); @@ -336,7 +357,13 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) ps->externally_clocked = true; active = apple_pmgr_ps_is_active(ps); - if (of_property_read_bool(node, "apple,always-on")) { + if (ps->firmware_owned) { + /* Consumers (the boot framebuffer) need it; never try to turn it on. */ + if (!active) + dev_warn(dev, "firmware-owned domain %s does not read as on\n", name); + ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; + active = true; + } else if (of_property_read_bool(node, "apple,always-on")) { ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; if (!active) { dev_warn(dev, "always-on domain %s is not on at boot\n", name); @@ -353,7 +380,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) } /* Turn on auto-PM if the domain is already on */ - if (active) + if (active && !ps->firmware_owned) regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_AUTO_ENABLE, APPLE_PMGR_AUTO_ENABLE); From 4db9a3896199981ad42597ac61e9c77c87aba4f4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 08:41:03 +1100 Subject: [PATCH 0970/1579] soc: apple: recheck sensor status after SEP registration Registration can clear and authenticate the sensor session after the wake status was sampled. Read status again before trusting the patch acknowledgement, refuse unreadable or silent status, and send an explicit needs-patch state through the existing patch sequence. Preserve capture failures and enrolled identities. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index cff44167e20cd..1dd46c3195918 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1882,8 +1882,18 @@ impl SepData { return None; } - // The patch decision is made on status[8], not on state 9. - if st.patch_ack() == sensor::PATCH_ACCEPTED { + // Registration can reset the sensor session. Do not reuse the + // acknowledgement sampled before CLEAR_STATE and authentication. + let st = match sensor::status() { + Ok(st) if !st.is_silent() => st, + _ => { + dev_warn!(self.dev, "sensor: status unavailable after registration\n"); + return None; + } + }; + if st.patch_ack() == sensor::PATCH_ACCEPTED + && st.state != sensor::STATE_NEEDS_PATCH + { return Some(PatchLoaded(())); } return self.apply_sensor_patch(st.patch_ack()); From 26c640bc3f1ed16c1cbecb5e3dbde85a7a44f797 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 08:47:09 +1100 Subject: [PATCH 0971/1579] tools: aurora-sep: exercise post-registration wake status Run the actual wake_sensor body with synthetic registration and status mocks. Check register-before-status ordering, stale accepted acknowledgement followed by needs-patch, unreadable and silent status, fresh accepted readiness, and missing acknowledgement. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/tests/test_sensor_wake.py | 96 ++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tools/aurora-sep/tests/test_sensor_wake.py diff --git a/tools/aurora-sep/tests/test_sensor_wake.py b/tools/aurora-sep/tests/test_sensor_wake.py new file mode 100644 index 0000000000000..c9bb8c6af7579 --- /dev/null +++ b/tools/aurora-sep/tests/test_sensor_wake.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: MIT +"""Execute the actual wake_sensor body with synthetic status/registration mocks.""" +from pathlib import Path +import subprocess +import tempfile +import unittest + +APPLE = Path(__file__).resolve().parents[3] / 'drivers/soc/apple' + + +class SensorWakeTest(unittest.TestCase): + def test_post_registration_status_drives_patch_readiness(self): + source = (APPLE / 'sbio.rs').read_text() + start = source.index(' fn wake_sensor(') + opening = source.index('{', start) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + function = source[start:end] + program = r''' +#![allow(dead_code, unused_variables)] +use std::cell::RefCell; +macro_rules! dev_warn { ($($args:tt)*) => {} } +macro_rules! dev_err { ($($args:tt)*) => {} } +mod kernel { pub mod time { + pub struct Delta; + impl Delta { pub fn from_millis(_:i64)->Self {Self} } + pub mod delay { pub fn fsleep(_:super::Delta) {} } +}} +#[derive(Clone, Copy)] struct PatchLoaded(()); +#[derive(Default)] struct Mock { + reads: usize, registered: bool, patch: Option, next: Option, +} +thread_local! { static MOCK: RefCell = RefCell::new(Mock::default()); } +mod sensor { + pub const POWER_ON_READ_DELAYS_MS: [u32;1] = [0]; + pub const KNOWN_IDENTIFIERS: &[u16] = &[1]; + pub const PATCH_ACCEPTED: u8 = 0x29; + pub const STATE_NEEDS_PATCH: u8 = 9; + #[derive(Clone,Copy)] pub struct Status { pub state:u8, pub ack:u8, pub silent:bool } + pub struct Identifier; + pub enum Offset12 { Identifier(u16), Other } + pub struct Timing; + impl Timing { pub fn is_hardware(&self)->bool {true} pub fn name(&self)->&'static str {"mock"} } + pub fn power_source() {} + pub fn power_cycle()->bool {true} + pub fn cs_timing()->Timing {Timing} + pub fn status()->Result { + super::MOCK.with(|m| { let mut m=m.borrow_mut(); m.reads+=1; + if m.reads==1 {return Ok(Status{state:0,ack:PATCH_ACCEPTED,silent:false})} + assert!(m.registered, "post-registration read preceded registration"); + m.next.ok_or(()) + }) + } + impl Status { + pub fn is_silent(&self)->bool {self.silent} + pub fn offset12(&self)->Offset12 {Offset12::Identifier(1)} + pub fn identifier_to_register(&self)->Option {Some(Identifier)} + pub fn patch_ack(&self)->u8 {self.ack} + } +} +struct Driver; +impl Driver { + fn register_sensor(&self,_:&sensor::Identifier)->bool { + MOCK.with(|m| {let mut m=m.borrow_mut();assert_eq!(m.reads,1);m.registered=true;}); true + } + fn apply_sensor_patch(&self,ack:u8)->Option { + MOCK.with(|m| {let mut m=m.borrow_mut();assert_eq!(m.reads,2);m.patch=Some(ack);}); + Some(PatchLoaded(())) + } +''' + program += function + '\n}\n' + r''' +fn main() { + for (next,ready,patch) in [ + (Some(sensor::Status{state:9,ack:0x29,silent:false}),true,Some(0x29)), + (None,false,None), + (Some(sensor::Status{state:0,ack:0x29,silent:true}),false,None), + (Some(sensor::Status{state:0,ack:0x29,silent:false}),true,None), + (Some(sensor::Status{state:0,ack:0,silent:false}),true,Some(0)), + ] { + MOCK.with(|m| *m.borrow_mut()=Mock{next,..Mock::default()}); + assert_eq!(Driver.wake_sensor().is_some(),ready); + MOCK.with(|m| {let m=m.borrow();assert_eq!(m.reads,2);assert!(m.registered);assert_eq!(m.patch,patch);}); + } +} +''' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'test.rs').write_text(program) + subprocess.run(['rustc', '--edition=2021', str(root / 'test.rs'), '-o', str(root / 'test')], check=True) + subprocess.run([str(root / 'test')], check=True) + + +if __name__ == '__main__': + unittest.main() From 6ffb2dc7a658e2e6bcaf48a75461b8d7e84549bf Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 19:46:45 -0400 Subject: [PATCH 0972/1579] drm/asahi: recycle settled quarantined G17 compute queues A failed client's compute queue is quarantined. Once its owner closes and the retirement witness has settled every retained command, the queue has no active command, so exit_candidate() never issues a kill for it, and reusable() refuses it because it is still quarantined. It therefore keeps its QID and its USC pool backing until the processor stops. Each Gen1-capable compute pool takes about 43 MiB of the T8140 dynamic lower VA window. On Neo, after roughly 42 failed clients the window is exhausted and every new compute submission fails with ENOSPC until reboot. Return such a queue to service the way recycle_killed() already does after an acknowledged kill: reset the item ring at the firmware consumer cursor and require fresh backing, so the existing vacate path publishes the free-list release and frees the VA. A kill acknowledged before the scheduler page showed the killed state was left Unproven forever, retaining its queue as well. Re-read such kills on later passes and settle them only once the page shows the killed state, also when their commands were retired by other means. Validation on Neo with deliberately failing compute clients: unmodified aurora-wip reaches ENOSPC after about 42 failed clients. With this change, a fresh boot handled 84 failed clients, including a replay of 28 failed device creations, with no ENOSPC and no firmware crash, and the same build passed more than 150 failed clients on one boot. Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 29 ++++++++++++++++++- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 10 +++++-- drivers/gpu/drm/asahi/g17/runtime.rs | 1 + 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index cae576fe183af..f3f050d482bef 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1192,6 +1192,33 @@ impl Queue { { return Ok(false); } + self.recycle_settled()?; + Ok(true) + } + /// A quarantined queue whose owner closed and whose every retained command was + /// already settled, by the exact retirement witness or by an acknowledged kill, + /// holds no pending command: the firmware consumed its item ring exactly as after + /// a kill. Neither `reusable` nor `exit_candidate` matches it, so without this it + /// keeps its QID and USC pool backing until processor stop, and repeated client + /// failures exhaust the compute pool VA range. Return it to service as + /// `recycle_killed` does. + pub(crate) fn recycle_quarantined_idle(&mut self) -> Result { + if !self.installed + || !self.quarantined + || !self.released + || self.growing + || self.exit.is_some() + || self.awaiting_witness + || self.active.len != 0 + || self.replays.len != 0 + || self.previous.is_some() + { + return Ok(false); + } + self.recycle_settled()?; + Ok(true) + } + fn recycle_settled(&mut self) -> Result { self.graph.reset_items()?; self.exit = None; self.quarantined = false; @@ -1213,7 +1240,7 @@ impl Queue { self.retirement_ready = true; self.released_at = now_ns(); self.needs_fresh_backing = true; - Ok(true) + Ok(()) } pub(crate) fn released_at(&self) -> u64 { self.released_at diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 10d2a37274800..877fc935dc909 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -156,10 +156,16 @@ impl Queue { complete_qos: &mut impl FnMut(crate::g17::qos::Owner) -> Result, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { - if !self.exit.as_ref().is_some_and(|exit| exit.phase == Phase::Acknowledged) { + let Some(phase) = self.exit.as_ref().map(|exit| exit.phase) else { + return Ok(()); + }; + if !matches!(phase, Phase::Acknowledged | Phase::Unproven) { return Ok(()); } - if self.active.len != 0 { + // An unproven kill (acknowledged before the scheduler page showed the killed + // state) is re-read on later passes and settles only once the page proves it, + // even after its commands were retired by other means. + if self.active.len != 0 || phase == Phase::Unproven { if !recovery_closed { return Ok(()); } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index ecfa57cd9a5a1..ffaff6d80dbf9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1253,6 +1253,7 @@ impl super::Firmware { .filter_map(|entry| entry.queue.as_deref_mut()) { queue.recycle_killed()?; + queue.recycle_quarantined_idle()?; if let Some(binding) = queue.detach_exited_owner() { deferred.defer_binding(binding); } From 4152bb971d6ed080fede5eb6539a4aaff475d548 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 00:16:24 +0000 Subject: [PATCH 0973/1579] arm64: configs: add aurora_j700_defconfig for the MacBook Neo Add the configuration the Omarchy/Arch beta kernel for the MacBook Neo (J700, T8140) is built with, as `make savedefconfig` output, so that the package can be rebuilt from public sources alone. It is the configuration that booted aurora-wip 5f6c917 on the Neo on 2026-10-15, with LOCALVERSION reduced to "-aurora"; the package sets the full release. With Arch Linux ARM's LLVM 23 and Rust 1.99, `make aurora_j700_defconfig` gives that configuration back exactly, apart from the symbols that follow the toolchain (the version strings, CC_MS_EXTENSIONS and RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS). `make defconfig` followed by asahi.config, j700-radio.config, j700.config and j700-device-parity.config gives the same configuration except for WERROR, LOCALVERSION and the boot logo. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/aurora_j700_defconfig | 2895 ++++++++++++++++++++++ 1 file changed, 2895 insertions(+) create mode 100644 arch/arm64/configs/aurora_j700_defconfig diff --git a/arch/arm64/configs/aurora_j700_defconfig b/arch/arm64/configs/aurora_j700_defconfig new file mode 100644 index 0000000000000..49e5c03d106ea --- /dev/null +++ b/arch/arm64/configs/aurora_j700_defconfig @@ -0,0 +1,2895 @@ +CONFIG_WERROR=y +CONFIG_LOCALVERSION="-aurora" +# CONFIG_LOCALVERSION_AUTO is not set +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +CONFIG_AUDIT=y +CONFIG_NO_HZ_IDLE=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_BPF_SYSCALL=y +CONFIG_BPF_JIT=y +CONFIG_PREEMPT=y +CONFIG_IRQ_TIME_ACCOUNTING=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_BSD_PROCESS_ACCT_V3=y +CONFIG_TASKSTATS=y +CONFIG_TASK_XACCT=y +CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_UCLAMP_TASK=y +CONFIG_NUMA_BALANCING=y +CONFIG_MEMCG=y +CONFIG_BLK_CGROUP=y +CONFIG_UCLAMP_TASK_GROUP=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_HUGETLB=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_PERF=y +CONFIG_CGROUP_BPF=y +CONFIG_CGROUP_MISC=y +CONFIG_USER_NS=y +CONFIG_CHECKPOINT_RESTORE=y +CONFIG_SCHED_AUTOGROUP=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_KALLSYMS_ALL=y +CONFIG_PROFILING=y +CONFIG_RUST=y +CONFIG_ARCH_ACTIONS=y +CONFIG_ARCH_AIROHA=y +CONFIG_ARCH_SUNXI=y +CONFIG_ARCH_ALPINE=y +CONFIG_ARCH_APPLE=y +CONFIG_ARCH_ARTPEC=y +CONFIG_ARCH_AXIADO=y +CONFIG_ARCH_BCM=y +CONFIG_ARCH_BCM2835=y +CONFIG_ARCH_BCM_IPROC=y +CONFIG_ARCH_BCMBCA=y +CONFIG_ARCH_BRCMSTB=y +CONFIG_ARCH_BERLIN=y +CONFIG_ARCH_BLAIZE=y +CONFIG_ARCH_BST=y +CONFIG_ARCH_CIX=y +CONFIG_ARCH_EXYNOS=y +CONFIG_ARCH_K3=y +CONFIG_ARCH_LG1K=y +CONFIG_ARCH_HISI=y +CONFIG_ARCH_KEEMBAY=y +CONFIG_ARCH_MEDIATEK=y +CONFIG_ARCH_MESON=y +CONFIG_ARCH_MICROCHIP=y +CONFIG_ARCH_SPARX5=y +CONFIG_ARCH_MVEBU=y +CONFIG_ARCH_NXP=y +CONFIG_ARCH_LAYERSCAPE=y +CONFIG_ARCH_MXC=y +CONFIG_ARCH_S32=y +CONFIG_ARCH_MA35=y +CONFIG_ARCH_NPCM=y +CONFIG_ARCH_QCOM=y +CONFIG_ARCH_REALTEK=y +CONFIG_ARCH_RENESAS=y +CONFIG_ARCH_ROCKCHIP=y +CONFIG_ARCH_SEATTLE=y +CONFIG_ARCH_INTEL_SOCFPGA=y +CONFIG_ARCH_SOPHGO=y +CONFIG_ARCH_STM32=y +CONFIG_ARCH_SYNQUACER=y +CONFIG_ARCH_TEGRA=y +CONFIG_ARCH_TESLA_FSD=y +CONFIG_ARCH_SPRD=y +CONFIG_ARCH_THUNDER=y +CONFIG_ARCH_THUNDER2=y +CONFIG_ARCH_UNIPHIER=y +CONFIG_ARCH_VEXPRESS=y +CONFIG_ARCH_VISCONTI=y +CONFIG_ARCH_XGENE=y +CONFIG_ARCH_ZYNQMP=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_NUMA=y +CONFIG_XEN=y +CONFIG_ARM64_MEMORY_MODEL_CONTROL=y +CONFIG_HIBERNATION=y +CONFIG_PM_DEBUG=y +CONFIG_PM_ADVANCED_DEBUG=y +CONFIG_WQ_POWER_EFFICIENT_DEFAULT=y +CONFIG_ENERGY_MODEL=y +CONFIG_ARM_PSCI_CPUIDLE=y +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_FREQ_GOV_POWERSAVE=m +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_CPU_FREQ_GOV_ONDEMAND=y +CONFIG_CPU_FREQ_GOV_CONSERVATIVE=m +CONFIG_CPUFREQ_DT=y +CONFIG_ARM_ALLWINNER_SUN50I_CPUFREQ_NVMEM=m +CONFIG_ARM_APPLE_SOC_CPUFREQ=y +CONFIG_ARM_ARMADA_37XX_CPUFREQ=y +CONFIG_ARM_SCPI_CPUFREQ=y +CONFIG_ARM_IMX_CPUFREQ_DT=m +CONFIG_ARM_MEDIATEK_CPUFREQ=y +CONFIG_ARM_QCOM_CPUFREQ_NVMEM=y +CONFIG_ARM_QCOM_CPUFREQ_HW=y +CONFIG_ARM_RASPBERRYPI_CPUFREQ=m +CONFIG_ARM_SCMI_CPUFREQ=y +CONFIG_ARM_TEGRA186_CPUFREQ=y +CONFIG_QORIQ_CPUFREQ=y +CONFIG_ACPI_CPPC_CPUFREQ=m +CONFIG_ACPI=y +CONFIG_ACPI_HOTPLUG_MEMORY=y +CONFIG_ACPI_HMAT=y +CONFIG_ACPI_APEI=y +CONFIG_ACPI_APEI_GHES=y +CONFIG_ACPI_APEI_PCIEAER=y +CONFIG_ACPI_APEI_MEMORY_FAILURE=y +CONFIG_ACPI_APEI_EINJ=y +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=y +CONFIG_COMPAT_32BIT_TIME=y +# CONFIG_RELR is not set +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_AIX_PARTITION=y +CONFIG_OSF_PARTITION=y +CONFIG_MAC_PARTITION=y +CONFIG_BSD_DISKLABEL=y +CONFIG_MINIX_SUBPARTITION=y +CONFIG_SOLARIS_X86_PARTITION=y +CONFIG_UNIXWARE_DISKLABEL=y +CONFIG_LDM_PARTITION=y +CONFIG_SGI_PARTITION=y +CONFIG_SUN_PARTITION=y +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_BINFMT_MISC=m +# CONFIG_COMPAT_BRK is not set +CONFIG_MEMORY_HOTPLUG=y +CONFIG_MEMORY_HOTREMOVE=y +CONFIG_KSM=y +CONFIG_MEMORY_FAILURE=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_PAGE_BLOCK_MAX_ORDER=10 +CONFIG_USERFAULTFD=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_PACKET_DIAG=m +CONFIG_UNIX_DIAG=m +CONFIG_TLS=m +CONFIG_XFRM_USER=m +CONFIG_XFRM_INTERFACE=m +CONFIG_XFRM_STATISTICS=y +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_NET_IPIP=m +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_NET_IPGRE=m +CONFIG_NET_IPGRE_BROADCAST=y +CONFIG_NET_IPVTI=m +CONFIG_NET_FOU_IP_TUNNELS=y +CONFIG_INET_AH=m +CONFIG_INET_ESP=m +CONFIG_INET_ESP_OFFLOAD=m +CONFIG_INET_ESPINTCP=y +CONFIG_INET_IPCOMP=m +CONFIG_INET_UDP_DIAG=m +CONFIG_INET_RAW_DIAG=m +CONFIG_INET_DIAG_DESTROY=y +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_CONG_HSTCP=m +CONFIG_TCP_CONG_HYBLA=m +CONFIG_TCP_CONG_NV=m +CONFIG_TCP_CONG_SCALABLE=m +CONFIG_TCP_CONG_LP=m +CONFIG_TCP_CONG_VENO=m +CONFIG_TCP_CONG_YEAH=m +CONFIG_TCP_CONG_ILLINOIS=m +CONFIG_TCP_CONG_DCTCP=m +CONFIG_TCP_CONG_CDG=m +CONFIG_TCP_CONG_BBR=m +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_ESP_OFFLOAD=m +CONFIG_INET6_ESPINTCP=y +CONFIG_INET6_IPCOMP=m +CONFIG_IPV6_VTI=m +CONFIG_IPV6_GRE=m +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_NETFILTER=y +CONFIG_BRIDGE_NETFILTER=m +CONFIG_NETFILTER_NETLINK_HOOK=m +CONFIG_NF_CONNTRACK=m +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CONNTRACK_PROCFS=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_CT=m +CONFIG_NFT_FLOW_OFFLOAD=m +CONFIG_NFT_CONNLIMIT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_NAT=m +CONFIG_NFT_TUNNEL=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_HASH=m +CONFIG_NFT_FIB_INET=m +CONFIG_NFT_XFRM=m +CONFIG_NFT_SOCKET=m +CONFIG_NFT_OSF=m +CONFIG_NFT_TPROXY=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NFT_FIB_NETDEV=m +CONFIG_NFT_REJECT_NETDEV=m +CONFIG_NF_FLOW_TABLE_INET=m +CONFIG_NF_FLOW_TABLE=m +CONFIG_NF_FLOW_TABLE_PROCFS=y +CONFIG_NETFILTER_XTABLES_LEGACY=y +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_AUDIT=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LED=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_NOTRACK=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DCCP=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SCTP=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_IPMAC=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_IP_VS=m +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_NFT_DUP_IPV6=m +CONFIG_NFT_FIB_IPV6=m +CONFIG_IP6_NF_IPTABLES=m +CONFIG_IP6_NF_MATCH_AH=m +CONFIG_IP6_NF_MATCH_EUI64=m +CONFIG_IP6_NF_MATCH_FRAG=m +CONFIG_IP6_NF_MATCH_OPTS=m +CONFIG_IP6_NF_MATCH_HL=m +CONFIG_IP6_NF_MATCH_IPV6HEADER=m +CONFIG_IP6_NF_MATCH_MH=m +CONFIG_IP6_NF_MATCH_RPFILTER=m +CONFIG_IP6_NF_MATCH_RT=m +CONFIG_IP6_NF_MATCH_SRH=m +CONFIG_IP6_NF_TARGET_HL=m +CONFIG_IP6_NF_TARGET_REJECT=m +CONFIG_IP6_NF_RAW=m +CONFIG_IP6_NF_NAT=m +CONFIG_IP6_NF_TARGET_MASQUERADE=m +CONFIG_IP6_NF_TARGET_NPT=m +CONFIG_NF_TABLES_BRIDGE=m +CONFIG_NFT_BRIDGE_META=m +CONFIG_NFT_BRIDGE_REJECT=m +CONFIG_NF_CONNTRACK_BRIDGE=m +CONFIG_BRIDGE_NF_EBTABLES_LEGACY=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_IP6=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_L2TP=m +CONFIG_L2TP_V3=y +CONFIG_L2TP_IP=m +CONFIG_L2TP_ETH=m +CONFIG_BRIDGE=m +CONFIG_BRIDGE_VLAN_FILTERING=y +CONFIG_NET_DSA=m +CONFIG_VLAN_8021Q=m +CONFIG_VLAN_8021Q_GVRP=y +CONFIG_VLAN_8021Q_MVRP=y +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TEQL=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_CBS=m +CONFIG_NET_SCH_ETF=m +CONFIG_NET_SCH_TAPRIO=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_MQPRIO=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_CAKE=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_FQ_PIE=m +CONFIG_NET_SCH_INGRESS=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_SCH_ETS=m +CONFIG_NET_SCH_DUALPI2=m +CONFIG_NET_CLS_BASIC=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_U32=m +CONFIG_CLS_U32_PERF=y +CONFIG_CLS_U32_MARK=y +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_EMATCH_IPSET=m +CONFIG_NET_EMATCH_IPT=m +CONFIG_NET_CLS_ACT=y +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_GACT=m +CONFIG_GACT_PROB=y +CONFIG_NET_ACT_MIRRED=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_MPLS=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_CONNMARK=m +CONFIG_NET_ACT_CTINFO=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_ACT_CT=m +CONFIG_NET_ACT_GATE=m +CONFIG_VSOCKETS=m +CONFIG_NETLINK_DIAG=m +CONFIG_HSR=m +CONFIG_QRTR_SMD=m +CONFIG_QRTR_TUN=m +CONFIG_CAN=m +CONFIG_CAN_J1939=m +CONFIG_CAN_ISOTP=m +CONFIG_BT=m +CONFIG_BT_RFCOMM=m +CONFIG_BT_RFCOMM_TTY=y +CONFIG_BT_BNEP=m +CONFIG_BT_BNEP_MC_FILTER=y +CONFIG_BT_BNEP_PROTO_FILTER=y +CONFIG_BT_HIDP=m +CONFIG_BT_LEDS=y +CONFIG_BT_MSFTEXT=y +CONFIG_BT_AOSPEXT=y +CONFIG_BT_BRCMEXT=y +# CONFIG_BT_DEBUGFS is not set +CONFIG_BT_MTK7932_PCIE=m +CONFIG_BT_HCIBTUSB=m +CONFIG_BT_HCIBTUSB_AUTOSUSPEND=y +CONFIG_BT_HCIBTUSB_MTK=y +CONFIG_BT_HCIUART=m +CONFIG_BT_HCIUART_LL=y +CONFIG_BT_HCIUART_BCM=y +CONFIG_BT_HCIUART_QCA=y +CONFIG_BT_HCIUART_MRVL=y +CONFIG_BT_HCIBCM203X=m +CONFIG_BT_HCIBCM4377=m +CONFIG_BT_HCIBPA10X=m +CONFIG_BT_HCIBFUSB=m +CONFIG_BT_HCIVHCI=m +CONFIG_BT_MRVL=m +CONFIG_BT_MRVL_SDIO=m +CONFIG_BT_ATH3K=m +CONFIG_BT_QCOMSMD=m +CONFIG_BT_NXPUART=m +CONFIG_CFG80211=m +CONFIG_MAC80211=m +CONFIG_RFKILL=y +CONFIG_RFKILL_GPIO=m +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y +CONFIG_NFC=m +CONFIG_NFC_NCI=m +CONFIG_NFC_NXP_NCI=m +CONFIG_NFC_NXP_NCI_I2C=m +CONFIG_NFC_S3FWRN5_I2C=m +CONFIG_PCI=y +CONFIG_PCIEPORTBUS=y +CONFIG_PCIEAER=y +CONFIG_PCI_IOV=y +CONFIG_PCI_PASID=y +CONFIG_HOTPLUG_PCI=y +CONFIG_HOTPLUG_PCI_ACPI=y +CONFIG_PCI_AARDVARK=y +CONFIG_PCIE_ALTERA=y +CONFIG_PCIE_ALTERA_MSI=y +CONFIG_PCIE_APPLE=y +CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_PCIE_BRCMSTB=m +CONFIG_PCI_HOST_THUNDER_PEM=y +CONFIG_PCI_HOST_THUNDER_ECAM=y +CONFIG_PCI_HOST_GENERIC=y +CONFIG_PCIE_MEDIATEK_GEN3=m +CONFIG_PCI_TEGRA=y +CONFIG_PCIE_RCAR_HOST=y +CONFIG_PCIE_RCAR_EP=y +CONFIG_PCIE_RENESAS_RZG3S_HOST=y +CONFIG_PCIE_ROCKCHIP_HOST=m +CONFIG_PCI_XGENE=y +CONFIG_PCIE_XILINX=y +CONFIG_PCIE_XILINX_DMA_PL=y +CONFIG_PCIE_XILINX_NWL=y +CONFIG_PCIE_XILINX_CPM=y +CONFIG_PCI_J721E_HOST=m +CONFIG_PCI_IMX6_HOST=y +CONFIG_PCI_LAYERSCAPE=y +CONFIG_PCI_HISI=y +CONFIG_PCIE_KIRIN=y +CONFIG_PCIE_HISI_STB=y +CONFIG_PCIE_ARMADA_8K=y +CONFIG_PCIE_TEGRA194_HOST=m +CONFIG_PCIE_TEGRA194_EP=m +CONFIG_PCIE_QCOM=y +CONFIG_PCIE_RCAR_GEN4_HOST=m +CONFIG_PCIE_RCAR_GEN4_EP=m +CONFIG_PCIE_ROCKCHIP_DW_HOST=y +CONFIG_PCIE_VISCONTI_HOST=y +CONFIG_PCIE_LAYERSCAPE_GEN4=y +CONFIG_PCI_ENDPOINT=y +CONFIG_PCI_ENDPOINT_CONFIGFS=y +CONFIG_PCI_EPF_TEST=m +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_FW_LOADER_USER_HELPER=y +CONFIG_FW_LOADER_COMPRESS=y +CONFIG_FW_LOADER_COMPRESS_ZSTD=y +CONFIG_HISILICON_LPC=y +CONFIG_IMX_AIPSTZ=m +CONFIG_TEGRA_ACONNECT=m +CONFIG_MHI_BUS_PCI_GENERIC=m +CONFIG_ARM_SCPI_PROTOCOL=y +CONFIG_RASPBERRYPI_FIRMWARE=y +CONFIG_INTEL_STRATIX10_SERVICE=y +CONFIG_INTEL_STRATIX10_RSU=m +CONFIG_MTK_ADSP_IPC=m +CONFIG_SYSFB_SIMPLEFB=y +CONFIG_GOOGLE_FIRMWARE=y +CONFIG_GOOGLE_CBMEM=m +CONFIG_GOOGLE_COREBOOT_TABLE=m +CONFIG_EFI_CAPSULE_LOADER=y +CONFIG_IMX_SCU=y +CONFIG_QCOM_TZMEM_MODE_SHMBRIDGE=y +CONFIG_QCOM_QSEECOM=y +CONFIG_QCOM_QSEECOM_UEFISECAPP=y +CONFIG_EXYNOS_ACPM_PROTOCOL=m +CONFIG_TEGRA_BPMP=y +CONFIG_ZYNQMP_FIRMWARE_DEBUG=y +CONFIG_GNSS=m +CONFIG_GNSS_MTK_SERIAL=m +CONFIG_GNSS_USB=m +CONFIG_MTD=y +CONFIG_MTD_BLOCK=y +CONFIG_MTD_CFI=y +CONFIG_MTD_CFI_ADV_OPTIONS=y +CONFIG_MTD_CFI_INTELEXT=y +CONFIG_MTD_CFI_AMDSTD=y +CONFIG_MTD_CFI_STAA=y +CONFIG_MTD_PHYSMAP=y +CONFIG_MTD_PHYSMAP_OF=y +CONFIG_MTD_DATAFLASH=y +CONFIG_MTD_SST25L=y +CONFIG_MTD_RAW_NAND=y +CONFIG_MTD_NAND_DENALI_DT=y +CONFIG_MTD_NAND_MARVELL=y +CONFIG_MTD_NAND_BRCMNAND=m +CONFIG_MTD_NAND_FSL_IFC=y +CONFIG_MTD_NAND_QCOM=y +CONFIG_MTD_SPI_NAND=m +CONFIG_MTD_SPI_NOR=y +CONFIG_MTD_UBI=m +CONFIG_MTD_HYPERBUS=m +CONFIG_HBMC_AM654=m +CONFIG_OF_OVERLAY=y +CONFIG_ZRAM=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_NBD=m +CONFIG_VIRTIO_BLK=y +CONFIG_BLK_DEV_NVME=y +CONFIG_NVME_HWMON=y +CONFIG_NVME_APPLE=y +CONFIG_QCOM_COINCELL=m +CONFIG_QCOM_FASTRPC=m +CONFIG_SRAM=y +CONFIG_PCI_ENDPOINT_TEST=m +CONFIG_XILINX_SDFEC=m +CONFIG_NTSYNC=m +CONFIG_EEPROM_AT24=m +CONFIG_EEPROM_AT25=m +CONFIG_MISC_RTSX_USB=m +CONFIG_UACCE=m +CONFIG_MISC_RP1=m +# CONFIG_SCSI_PROC_FS is not set +CONFIG_BLK_DEV_SD=y +CONFIG_BLK_DEV_SR=m +CONFIG_CHR_DEV_SG=m +CONFIG_SCSI_SAS_ATA=y +CONFIG_SCSI_HISI_SAS=y +CONFIG_SCSI_HISI_SAS_PCI=y +CONFIG_MEGARAID_SAS=y +CONFIG_SCSI_MPT3SAS=m +CONFIG_ATA=y +CONFIG_SATA_AHCI=y +CONFIG_SATA_AHCI_PLATFORM=y +CONFIG_AHCI_BRCM=m +CONFIG_AHCI_DWC=m +CONFIG_AHCI_CEVA=y +CONFIG_AHCI_MVEBU=y +CONFIG_AHCI_XGENE=y +CONFIG_AHCI_QORIQ=y +CONFIG_SATA_SIL24=y +CONFIG_SATA_RCAR=y +CONFIG_PATA_OF_PLATFORM=y +CONFIG_MD=y +CONFIG_MD_LINEAR=m +CONFIG_BLK_DEV_DM=m +CONFIG_DM_CRYPT=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_CACHE=m +CONFIG_DM_WRITECACHE=m +CONFIG_DM_MIRROR=m +CONFIG_DM_RAID=m +CONFIG_DM_ZERO=m +CONFIG_DM_UEVENT=y +CONFIG_DM_VERITY=m +CONFIG_DM_VERITY_FEC=y +CONFIG_DM_INTEGRITY=m +CONFIG_NETDEVICES=y +CONFIG_BONDING=m +CONFIG_DUMMY=m +CONFIG_WIREGUARD=m +CONFIG_OVPN=m +CONFIG_IFB=m +CONFIG_NET_TEAM=m +CONFIG_NET_TEAM_MODE_BROADCAST=m +CONFIG_NET_TEAM_MODE_ROUNDROBIN=m +CONFIG_NET_TEAM_MODE_RANDOM=m +CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m +CONFIG_NET_TEAM_MODE_LOADBALANCE=m +CONFIG_MACVLAN=m +CONFIG_MACVTAP=m +CONFIG_VXLAN=m +CONFIG_GENEVE=m +CONFIG_TUN=y +CONFIG_VETH=m +CONFIG_VIRTIO_NET=y +CONFIG_MHI_NET=m +CONFIG_NET_DSA_BCM_SF2=m +CONFIG_NET_DSA_MSCC_FELIX=m +CONFIG_ENA_ETHERNET=m +CONFIG_AMD_XGBE=y +CONFIG_NET_XGENE=y +CONFIG_ATL1C=m +CONFIG_BCMGENET=m +CONFIG_BNX2X=m +CONFIG_SYSTEMPORT=m +CONFIG_MACB=y +CONFIG_THUNDER_NIC_PF=y +CONFIG_FEC=y +CONFIG_FSL_FMAN=y +CONFIG_FSL_DPAA_ETH=y +CONFIG_FSL_DPAA2_ETH=y +CONFIG_FSL_ENETC=y +CONFIG_NXP_ENETC4=m +CONFIG_FSL_ENETC_VF=y +CONFIG_FSL_ENETC_QOS=y +CONFIG_NXP_NETC_BLK_CTRL=m +CONFIG_HIX5HD2_GMAC=y +CONFIG_HNS_DSAF=y +CONFIG_HNS_ENET=y +CONFIG_HNS3=y +CONFIG_HNS3_HCLGE=y +CONFIG_HNS3_ENET=y +CONFIG_E1000=y +CONFIG_E1000E=y +CONFIG_IGB=m +CONFIG_IGBVF=y +CONFIG_IGC=m +CONFIG_MVNETA=y +CONFIG_MVPP2=y +CONFIG_SKY2=y +CONFIG_NET_VENDOR_MEDIATEK=y +CONFIG_NET_MEDIATEK_STAR_EMAC=m +CONFIG_MLX4_EN=m +CONFIG_MLX5_CORE=m +CONFIG_MLX5_CORE_EN=y +CONFIG_QCOM_EMAC=m +CONFIG_RMNET=m +CONFIG_R8169=m +CONFIG_SH_ETH=y +CONFIG_RAVB=y +CONFIG_RENESAS_ETHER_SWITCH=y +CONFIG_RTSN=y +CONFIG_SMC91X=y +CONFIG_SMSC911X=y +CONFIG_SNI_AVE=y +CONFIG_SNI_NETSEC=y +CONFIG_STMMAC_ETH=m +CONFIG_DWMAC_MEDIATEK=m +CONFIG_DWMAC_TEGRA=m +CONFIG_TI_K3_AM65_CPSW_NUSS=y +CONFIG_TI_ICSSG_PRUETH=m +CONFIG_XILINX_AXI_EMAC=m +CONFIG_QCOM_IPA=m +CONFIG_MESON_GXL_PHY=m +CONFIG_AQUANTIA_PHY=y +CONFIG_BCM54140_PHY=m +CONFIG_MARVELL_PHY=m +CONFIG_MARVELL_10G_PHY=y +CONFIG_MARVELL_88Q2XXX_PHY=y +CONFIG_MICREL_PHY=y +CONFIG_MICROSEMI_PHY=y +CONFIG_AT803X_PHY=y +CONFIG_QCA808X_PHY=m +CONFIG_REALTEK_PHY=y +CONFIG_ROCKCHIP_PHY=y +CONFIG_DP83867_PHY=y +CONFIG_DP83869_PHY=m +CONFIG_DP83TD510_PHY=y +CONFIG_DP83TG720_PHY=m +CONFIG_VITESSE_PHY=y +CONFIG_XILINX_GMII2RGMII=m +CONFIG_CAN_FLEXCAN=m +CONFIG_CAN_XILINXCAN=m +CONFIG_CAN_M_CAN=m +CONFIG_CAN_M_CAN_PLATFORM=m +CONFIG_CAN_RCAR=m +CONFIG_CAN_RCAR_CANFD=m +CONFIG_CAN_MCP251XFD=m +CONFIG_CAN_8DEV_USB=m +CONFIG_CAN_EMS_USB=m +CONFIG_CAN_ESD_USB=m +CONFIG_CAN_F81604=m +CONFIG_CAN_GS_USB=m +CONFIG_CAN_KVASER_USB=m +CONFIG_CAN_MCBA_USB=m +CONFIG_CAN_PEAK_USB=m +CONFIG_MDIO_GPIO=y +CONFIG_MDIO_BUS_MUX_MULTIPLEXER=y +CONFIG_MDIO_BUS_MUX_MMIOREG=y +CONFIG_PPP=m +CONFIG_PPP_BSDCOMP=m +CONFIG_PPP_DEFLATE=m +CONFIG_PPP_FILTER=y +CONFIG_PPP_MPPE=m +CONFIG_PPP_MULTILINK=y +CONFIG_PPPOE=m +CONFIG_PPTP=m +CONFIG_PPPOL2TP=m +CONFIG_PPP_ASYNC=m +CONFIG_PPP_SYNC_TTY=m +CONFIG_USB_CATC=m +CONFIG_USB_KAWETH=m +CONFIG_USB_PEGASUS=m +CONFIG_USB_RTL8150=m +CONFIG_USB_RTL8152=y +CONFIG_USB_LAN78XX=m +CONFIG_USB_USBNET=y +CONFIG_USB_NET_CDC_EEM=m +CONFIG_USB_NET_HUAWEI_CDC_NCM=m +CONFIG_USB_NET_CDC_MBIM=m +CONFIG_USB_NET_DM9601=m +CONFIG_USB_NET_SR9700=m +CONFIG_USB_NET_SR9800=m +CONFIG_USB_NET_SMSC75XX=m +CONFIG_USB_NET_SMSC95XX=m +CONFIG_USB_NET_GL620A=m +CONFIG_USB_NET_NET1080=m +CONFIG_USB_NET_PLUSB=m +CONFIG_USB_NET_MCS7830=m +CONFIG_USB_NET_RNDIS_HOST=m +CONFIG_USB_NET_CDC_SUBSET=m +CONFIG_USB_ALI_M5632=y +CONFIG_USB_AN2720=y +CONFIG_USB_EPSON2888=y +CONFIG_USB_KC2190=y +CONFIG_USB_NET_ZAURUS=m +CONFIG_USB_NET_CX82310_ETH=m +CONFIG_USB_NET_KALMIA=m +CONFIG_USB_NET_QMI_WWAN=m +CONFIG_USB_HSO=m +CONFIG_USB_NET_INT51X1=m +CONFIG_USB_IPHETH=m +CONFIG_USB_SIERRA_NET=m +CONFIG_USB_VL600=m +CONFIG_USB_NET_CH9200=m +CONFIG_USB_NET_AQC111=m +CONFIG_ATH9K_HTC=m +CONFIG_CARL9170=m +CONFIG_ATH6KL=m +CONFIG_ATH6KL_USB=m +CONFIG_AR5523=m +CONFIG_ATH10K=m +CONFIG_ATH10K_PCI=m +CONFIG_ATH10K_SDIO=m +CONFIG_ATH10K_USB=m +CONFIG_ATH10K_SNOC=m +CONFIG_WCN36XX=m +CONFIG_ATH11K=m +CONFIG_ATH11K_AHB=m +CONFIG_ATH11K_PCI=m +CONFIG_ATH12K=m +CONFIG_BRCMFMAC=m +CONFIG_BRCMFMAC_USB=y +CONFIG_BRCMFMAC_PCIE=y +CONFIG_IWLWIFI=m +CONFIG_IWLDVM=m +CONFIG_IWLMVM=m +CONFIG_LIBERTAS_THINFIRM=m +CONFIG_LIBERTAS_THINFIRM_USB=m +CONFIG_MWIFIEX=m +CONFIG_MWIFIEX_SDIO=m +CONFIG_MWIFIEX_PCIE=m +CONFIG_MWIFIEX_USB=m +CONFIG_MT7601U=m +CONFIG_MT76x0U=m +CONFIG_MT76x2U=m +CONFIG_MT7663U=m +CONFIG_MT7921E=m +CONFIG_MT7921U=m +CONFIG_MT7925U=m +CONFIG_MT7932_FULLMAC=m +CONFIG_RT2X00=m +CONFIG_RT2500USB=m +CONFIG_RT73USB=m +CONFIG_RT2800USB=m +CONFIG_RT2800USB_RT3573=y +CONFIG_RT2800USB_RT53XX=y +CONFIG_RT2800USB_RT55XX=y +CONFIG_RT2800USB_UNKNOWN=y +CONFIG_RTL8187=m +CONFIG_RTL8192DU=m +CONFIG_RTL8XXXU=m +CONFIG_RTW88=m +CONFIG_RTW88_8822BU=m +CONFIG_RTW88_8822CU=m +CONFIG_RTW88_8723DU=m +CONFIG_RTW88_8821CU=m +CONFIG_RTW88_8821AU=m +CONFIG_RTW88_8812AU=m +CONFIG_RTW88_8814AU=m +CONFIG_RTW89=m +CONFIG_RTW89_8851BU=m +CONFIG_RTW89_8852AU=m +CONFIG_RTW89_8852BU=m +CONFIG_RTW89_8852CU=m +CONFIG_RSI_91X=m +CONFIG_WL18XX=m +CONFIG_WLCORE_SDIO=m +CONFIG_ZD1211RW=m +CONFIG_WWAN=m +CONFIG_MHI_WWAN_CTRL=m +CONFIG_MHI_WWAN_MBIM=m +CONFIG_INPUT_JOYDEV=m +CONFIG_INPUT_EVDEV=y +CONFIG_KEYBOARD_ADC=m +CONFIG_KEYBOARD_GPIO=y +CONFIG_KEYBOARD_GPIO_POLLED=m +CONFIG_KEYBOARD_SNVS_PWRKEY=m +CONFIG_KEYBOARD_IMX_SC_KEY=m +CONFIG_KEYBOARD_CROS_EC=y +CONFIG_KEYBOARD_MTK_PMIC=m +CONFIG_MOUSE_ELAN_I2C=m +CONFIG_MOUSE_SYNAPTICS_USB=m +CONFIG_INPUT_JOYSTICK=y +CONFIG_JOYSTICK_IFORCE=m +CONFIG_JOYSTICK_IFORCE_USB=m +CONFIG_JOYSTICK_XPAD=m +CONFIG_JOYSTICK_XPAD_FF=y +CONFIG_JOYSTICK_XPAD_LEDS=y +CONFIG_JOYSTICK_PXRC=m +CONFIG_INPUT_TABLET=y +CONFIG_TABLET_USB_ACECAD=m +CONFIG_TABLET_USB_AIPTEK=m +CONFIG_TABLET_USB_HANWANG=m +CONFIG_TABLET_USB_KBTAB=m +CONFIG_TABLET_USB_PEGASUS=m +CONFIG_INPUT_TOUCHSCREEN=y +CONFIG_TOUCHSCREEN_APPLE_Z2=m +CONFIG_TOUCHSCREEN_ATMEL_MXT=m +CONFIG_TOUCHSCREEN_GOODIX=m +CONFIG_TOUCHSCREEN_GOODIX_BERLIN_SPI=m +CONFIG_TOUCHSCREEN_ILI210X=m +CONFIG_TOUCHSCREEN_ELAN=m +CONFIG_TOUCHSCREEN_EDT_FT5X06=m +CONFIG_TOUCHSCREEN_USB_COMPOSITE=m +CONFIG_TOUCHSCREEN_HIMAX_HX83112B=m +CONFIG_INPUT_MISC=y +CONFIG_INPUT_AW86927=m +CONFIG_INPUT_BBNSM_PWRKEY=m +CONFIG_INPUT_PM8941_PWRKEY=y +CONFIG_INPUT_PM8XXX_VIBRATOR=m +CONFIG_INPUT_TPS65219_PWRBUTTON=m +CONFIG_INPUT_TPS6594_PWRBUTTON=m +CONFIG_INPUT_UINPUT=m +CONFIG_INPUT_PWM_BEEPER=m +CONFIG_INPUT_PWM_VIBRA=m +CONFIG_INPUT_RK805_PWRKEY=m +CONFIG_INPUT_DA9063_ONKEY=m +CONFIG_INPUT_HISI_POWERKEY=y +CONFIG_INPUT_MACSMC_INPUT=m +CONFIG_RMI4_F1A=y +CONFIG_RMI4_F21=y +CONFIG_RMI4_F34=y +CONFIG_RMI4_F55=y +# CONFIG_SERIO_SERPORT is not set +CONFIG_SERIO_AMBAKMI=y +CONFIG_LEGACY_PTY_COUNT=16 +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_8250_NR_UARTS=8 +CONFIG_SERIAL_8250_EXTENDED=y +CONFIG_SERIAL_8250_SHARE_IRQ=y +CONFIG_SERIAL_8250_BCM2835AUX=y +CONFIG_SERIAL_8250_DW=y +CONFIG_SERIAL_8250_EM=y +CONFIG_SERIAL_8250_OMAP=y +CONFIG_SERIAL_8250_MT6577=y +CONFIG_SERIAL_8250_UNIPHIER=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y +CONFIG_SERIAL_APPLE_DOCKCHANNEL=y +CONFIG_SERIAL_MESON=y +CONFIG_SERIAL_MESON_CONSOLE=y +CONFIG_SERIAL_SAMSUNG=y +CONFIG_SERIAL_SAMSUNG_CONSOLE=y +CONFIG_SERIAL_TEGRA=y +CONFIG_SERIAL_TEGRA_TCU=y +CONFIG_SERIAL_IMX=y +CONFIG_SERIAL_IMX_CONSOLE=y +CONFIG_SERIAL_SH_SCI=y +CONFIG_SERIAL_RSCI=y +CONFIG_SERIAL_MSM=y +CONFIG_SERIAL_MSM_CONSOLE=y +CONFIG_SERIAL_QCOM_GENI=y +CONFIG_SERIAL_QCOM_GENI_CONSOLE=y +CONFIG_SERIAL_XILINX_PS_UART=y +CONFIG_SERIAL_XILINX_PS_UART_CONSOLE=y +CONFIG_SERIAL_FSL_LPUART=y +CONFIG_SERIAL_FSL_LPUART_CONSOLE=y +CONFIG_SERIAL_FSL_LINFLEXUART=y +CONFIG_SERIAL_FSL_LINFLEXUART_CONSOLE=y +CONFIG_SERIAL_STM32=y +CONFIG_SERIAL_STM32_CONSOLE=y +CONFIG_SERIAL_MVEBU_UART=y +CONFIG_SERIAL_OWL=y +CONFIG_SERIAL_DEV_BUS=y +CONFIG_VIRTIO_CONSOLE=y +CONFIG_IPMI_HANDLER=m +CONFIG_IPMI_DEVICE_INTERFACE=m +CONFIG_IPMI_SI=m +CONFIG_HW_RANDOM=y +CONFIG_HW_RANDOM_VIRTIO=y +CONFIG_TCG_TPM=y +CONFIG_TCG_TIS=m +CONFIG_TCG_TIS_SPI=m +CONFIG_TCG_TIS_SPI_CR50=y +CONFIG_TCG_TIS_I2C_CR50=m +CONFIG_TCG_TIS_I2C_INFINEON=y +CONFIG_XILLYUSB=m +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_MUX=y +CONFIG_I2C_MUX_PCA954x=y +CONFIG_I2C_MUX_PINCTRL=m +CONFIG_I2C_BCM2835=m +CONFIG_I2C_CADENCE=m +CONFIG_I2C_DESIGNWARE_CORE=y +CONFIG_I2C_GPIO=m +CONFIG_I2C_IMX=y +CONFIG_I2C_IMX_LPI2C=y +CONFIG_I2C_MESON=y +CONFIG_I2C_MT65XX=y +CONFIG_I2C_MV64XXX=y +CONFIG_I2C_OMAP=y +CONFIG_I2C_OWL=y +CONFIG_I2C_APPLE=y +CONFIG_I2C_PXA=y +CONFIG_I2C_QCOM_CCI=m +CONFIG_I2C_QCOM_GENI=m +CONFIG_I2C_QUP=y +CONFIG_I2C_RIIC=y +CONFIG_I2C_RK3X=y +CONFIG_I2C_RZV2M=m +CONFIG_I2C_S3C2410=y +CONFIG_I2C_SH_MOBILE=y +CONFIG_I2C_TEGRA=y +CONFIG_I2C_UNIPHIER_F=y +CONFIG_I2C_XILINX=m +CONFIG_I2C_RCAR=y +CONFIG_I2C_DIOLAN_U2C=m +CONFIG_I2C_DLN2=m +CONFIG_I2C_CP2615=m +CONFIG_I2C_TINY_USB=m +CONFIG_I2C_CROS_EC_TUNNEL=y +CONFIG_SPI=y +CONFIG_SPI_APPLE=m +CONFIG_SPI_APPLE_QSPI=m +CONFIG_SPI_ARMADA_3700=y +CONFIG_SPI_BCM2835=m +CONFIG_SPI_BCM2835AUX=m +CONFIG_SPI_CADENCE_QUADSPI=y +CONFIG_SPI_CH341=m +CONFIG_SPI_DESIGNWARE=m +CONFIG_SPI_DW_DMA=y +CONFIG_SPI_DW_MMIO=m +CONFIG_SPI_DLN2=m +CONFIG_SPI_FSL_LPSPI=y +CONFIG_SPI_FSL_QUADSPI=y +CONFIG_SPI_NXP_FLEXSPI=y +CONFIG_SPI_IMX=m +CONFIG_SPI_FSL_DSPI=y +CONFIG_SPI_MESON_SPICC=m +CONFIG_SPI_MESON_SPIFC=m +CONFIG_SPI_MT65XX=y +CONFIG_SPI_MTK_NOR=m +CONFIG_SPI_OMAP24XX=m +CONFIG_SPI_ORION=y +CONFIG_SPI_PL022=y +CONFIG_SPI_ROCKCHIP=y +CONFIG_SPI_ROCKCHIP_SFC=m +CONFIG_SPI_RPCIF=m +CONFIG_SPI_RSPI=m +CONFIG_SPI_RZV2H_RSPI=m +CONFIG_SPI_RZV2M_CSI=m +CONFIG_SPI_QCOM_QSPI=m +CONFIG_SPI_QUP=y +CONFIG_SPI_QCOM_GENI=m +CONFIG_SPI_S3C64XX=y +CONFIG_SPI_SH_MSIOF=m +CONFIG_SPI_STM32_OSPI=m +CONFIG_SPI_SUN6I=y +CONFIG_SPI_TEGRA210_QUAD=m +CONFIG_SPI_TEGRA114=m +CONFIG_SPI_XILINX=m +CONFIG_SPI_ZYNQMP_GQSPI=m +CONFIG_SPI_SPIDEV=m +CONFIG_SPMI=y +CONFIG_SPMI_APPLE=y +CONFIG_SPMI_MTK_PMIF=m +CONFIG_PINCTRL_APPLE_GPIO=y +CONFIG_PINCTRL_DA9062=m +CONFIG_PINCTRL_MAX77620=y +CONFIG_PINCTRL_RK805=m +CONFIG_PINCTRL_SINGLE=y +CONFIG_PINCTRL_SX150X=m +CONFIG_PINCTRL_OWL=y +CONFIG_PINCTRL_S700=y +CONFIG_PINCTRL_S900=y +CONFIG_PINCTRL_BRCMSTB=y +CONFIG_PINCTRL_BCM2712=y +CONFIG_PINCTRL_IMX_SCMI=y +CONFIG_PINCTRL_IMX91=y +CONFIG_PINCTRL_MSM=y +CONFIG_PINCTRL_ELIZA=y +CONFIG_PINCTRL_GLYMUR=y +CONFIG_PINCTRL_IPQ5018=y +CONFIG_PINCTRL_IPQ5210=y +CONFIG_PINCTRL_IPQ5332=y +CONFIG_PINCTRL_IPQ5424=y +CONFIG_PINCTRL_IPQ8074=y +CONFIG_PINCTRL_IPQ6018=y +CONFIG_PINCTRL_IPQ9574=y +CONFIG_PINCTRL_KAANAPALI=y +CONFIG_PINCTRL_MSM8916=y +CONFIG_PINCTRL_MSM8953=y +CONFIG_PINCTRL_MSM8976=y +CONFIG_PINCTRL_MSM8994=y +CONFIG_PINCTRL_MSM8996=y +CONFIG_PINCTRL_MSM8998=y +CONFIG_PINCTRL_QCM2290=y +CONFIG_PINCTRL_QCS404=y +CONFIG_PINCTRL_QCS615=y +CONFIG_PINCTRL_QCS8300=y +CONFIG_PINCTRL_QDF2XXX=y +CONFIG_PINCTRL_QDU1000=y +CONFIG_PINCTRL_SA8775P=y +CONFIG_PINCTRL_SC7180=y +CONFIG_PINCTRL_SC7280=y +CONFIG_PINCTRL_SC8180X=y +CONFIG_PINCTRL_SC8280XP=y +CONFIG_PINCTRL_SDM660=y +CONFIG_PINCTRL_SDM670=y +CONFIG_PINCTRL_SDM845=y +CONFIG_PINCTRL_SDX75=y +CONFIG_PINCTRL_SM4450=y +CONFIG_PINCTRL_SM6115=y +CONFIG_PINCTRL_SM6125=y +CONFIG_PINCTRL_SM6350=y +CONFIG_PINCTRL_SM6375=y +CONFIG_PINCTRL_MILOS=y +CONFIG_PINCTRL_SM8150=y +CONFIG_PINCTRL_SM8250=y +CONFIG_PINCTRL_SM8350=y +CONFIG_PINCTRL_SM8450=y +CONFIG_PINCTRL_SM8550=y +CONFIG_PINCTRL_SM8650=y +CONFIG_PINCTRL_SM8750=y +CONFIG_PINCTRL_X1E80100=y +CONFIG_PINCTRL_QCOM_SPMI_PMIC=y +CONFIG_PINCTRL_LPASS_LPI=m +CONFIG_PINCTRL_MILOS_LPASS_LPI=m +CONFIG_PINCTRL_SC7280_LPASS_LPI=m +CONFIG_PINCTRL_SM6115_LPASS_LPI=m +CONFIG_PINCTRL_SM8250_LPASS_LPI=m +CONFIG_PINCTRL_SM8450_LPASS_LPI=m +CONFIG_PINCTRL_SC8280XP_LPASS_LPI=m +CONFIG_PINCTRL_SM8550_LPASS_LPI=m +CONFIG_PINCTRL_SM8650_LPASS_LPI=m +CONFIG_PINCTRL_SOPHGO_SG2000=y +CONFIG_GPIO_ALTERA=m +CONFIG_GPIO_DAVINCI=y +CONFIG_GPIO_DWAPB=y +CONFIG_GPIO_MB86S7X=y +CONFIG_GPIO_MPC8XXX=y +CONFIG_GPIO_MXC=y +CONFIG_GPIO_PL061=y +CONFIG_GPIO_RCAR=y +CONFIG_GPIO_SYSCON=y +CONFIG_GPIO_UNIPHIER=y +CONFIG_GPIO_VF610=y +CONFIG_GPIO_VISCONTI=y +CONFIG_GPIO_WCD934X=m +CONFIG_GPIO_XGENE=y +CONFIG_GPIO_XGENE_SB=y +CONFIG_GPIO_XILINX=m +CONFIG_GPIO_ZYNQ=m +CONFIG_GPIO_MAX732X=y +CONFIG_GPIO_PCA953X=y +CONFIG_GPIO_PCA953X_IRQ=y +CONFIG_GPIO_PCF857X=m +CONFIG_GPIO_TPIC2810=m +CONFIG_GPIO_ADP5585=m +CONFIG_GPIO_BD9571MWV=m +CONFIG_GPIO_DLN2=m +CONFIG_GPIO_MACSMC=y +CONFIG_GPIO_MAX77620=y +CONFIG_GPIO_SL28CPLD=m +CONFIG_GPIO_MPSSE=m +CONFIG_GPIO_AGGREGATOR=m +CONFIG_W1=m +CONFIG_W1_MASTER_DS2490=m +CONFIG_W1_SLAVE_THERM=m +CONFIG_W1_SLAVE_SMEM=m +CONFIG_W1_SLAVE_DS2405=m +CONFIG_W1_SLAVE_DS2408=m +CONFIG_W1_SLAVE_DS2413=m +CONFIG_W1_SLAVE_DS2406=m +CONFIG_W1_SLAVE_DS2423=m +CONFIG_W1_SLAVE_DS2805=m +CONFIG_W1_SLAVE_DS2430=m +CONFIG_W1_SLAVE_DS2431=m +CONFIG_W1_SLAVE_DS2433=m +CONFIG_W1_SLAVE_DS2433_CRC=y +CONFIG_W1_SLAVE_DS2438=m +CONFIG_W1_SLAVE_DS2780=m +CONFIG_W1_SLAVE_DS2781=m +CONFIG_W1_SLAVE_DS28E04=m +CONFIG_POWER_RESET_GPIO=y +CONFIG_POWER_RESET_GPIO_RESTART=y +CONFIG_POWER_RESET_MACSMC=y +CONFIG_POWER_RESET_MSM=y +CONFIG_POWER_RESET_QCOM_PON=m +CONFIG_POWER_RESET_TORADEX_EC=m +CONFIG_POWER_RESET_XGENE=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_POWER_RESET_SYSCON_POWEROFF=y +CONFIG_SYSCON_REBOOT_MODE=y +CONFIG_NVMEM_REBOOT_MODE=m +CONFIG_POWER_SEQUENCING_PCIE_M2=m +CONFIG_BATTERY_QCOM_BATTMGR=m +CONFIG_BATTERY_SBS=m +CONFIG_BATTERY_BQ27XXX=y +CONFIG_BATTERY_MAX17042=m +CONFIG_CHARGER_MT6360=m +CONFIG_CHARGER_BQ25890=m +CONFIG_CHARGER_BQ25980=m +CONFIG_CHARGER_RK817=m +CONFIG_MACSMC_POWER=m +CONFIG_SENSORS_AQUACOMPUTER_D5NEXT=m +CONFIG_SENSORS_ASUS_ROG_RYUJIN=m +CONFIG_SENSORS_ARM_SCMI=y +CONFIG_SENSORS_ARM_SCPI=y +CONFIG_SENSORS_CORSAIR_CPRO=m +CONFIG_SENSORS_CORSAIR_PSU=m +CONFIG_SENSORS_GIGABYTE_WATERFORCE=m +CONFIG_SENSORS_GPIO_FAN=m +CONFIG_SENSORS_JC42=m +CONFIG_SENSORS_POWERZ=m +CONFIG_SENSORS_MACSMC_HWMON=y +CONFIG_SENSORS_LM75=m +CONFIG_SENSORS_LM90=m +CONFIG_SENSORS_NZXT_KRAKEN2=m +CONFIG_SENSORS_NZXT_KRAKEN3=m +CONFIG_SENSORS_NZXT_SMART2=m +CONFIG_SENSORS_PWM_FAN=m +CONFIG_SENSORS_RASPBERRYPI_HWMON=m +CONFIG_SENSORS_SL28CPLD=m +CONFIG_SENSORS_AMC6821=m +CONFIG_SENSORS_INA2XX=m +CONFIG_SENSORS_INA3221=m +CONFIG_SENSORS_TMP102=m +CONFIG_THERMAL_NETLINK=y +CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y +CONFIG_THERMAL_GOV_STEP_WISE=y +CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y +CONFIG_CPU_THERMAL=y +CONFIG_DEVFREQ_THERMAL=y +CONFIG_THERMAL_EMULATION=y +CONFIG_IMX_SC_THERMAL=m +CONFIG_IMX8MM_THERMAL=m +CONFIG_K3_THERMAL=m +CONFIG_QORIQ_THERMAL=m +CONFIG_SUN8I_THERMAL=y +CONFIG_ROCKCHIP_THERMAL=m +CONFIG_ARMADA_THERMAL=y +CONFIG_MTK_THERMAL=m +CONFIG_MTK_LVTS_THERMAL=m +CONFIG_BCM2711_THERMAL=m +CONFIG_BCM2835_THERMAL=m +CONFIG_BRCMSTB_THERMAL=m +CONFIG_EXYNOS_THERMAL=y +CONFIG_RCAR_THERMAL=y +CONFIG_RCAR_GEN3_THERMAL=y +CONFIG_RZG2L_THERMAL=y +CONFIG_RZG3E_THERMAL=y +CONFIG_RZG3S_THERMAL=m +CONFIG_TEGRA_SOCTHERM=m +CONFIG_TEGRA_BPMP_THERMAL=m +CONFIG_GENERIC_ADC_THERMAL=m +CONFIG_QCOM_TSENS=y +CONFIG_QCOM_SPMI_ADC_TM5=m +CONFIG_QCOM_SPMI_TEMP_ALARM=m +CONFIG_QCOM_LMH=m +CONFIG_UNIPHIER_THERMAL=y +CONFIG_KHADAS_MCU_FAN_THERMAL=m +CONFIG_WATCHDOG=y +CONFIG_XILINX_WATCHDOG=m +CONFIG_XILINX_WINDOW_WATCHDOG=m +CONFIG_SL28CPLD_WATCHDOG=m +CONFIG_ARM_SP805_WATCHDOG=y +CONFIG_ARM_SBSA_WATCHDOG=y +CONFIG_S3C2410_WATCHDOG=y +CONFIG_DW_WATCHDOG=y +CONFIG_K3_RTI_WATCHDOG=m +CONFIG_SUNXI_WATCHDOG=m +CONFIG_NPCM7XX_WATCHDOG=y +CONFIG_IMX2_WDT=y +CONFIG_IMX_SC_WDT=m +CONFIG_IMX7ULP_WDT=m +CONFIG_QCOM_WDT=m +CONFIG_MESON_GXBB_WATCHDOG=m +CONFIG_MESON_WATCHDOG=m +CONFIG_ARM_SMC_WATCHDOG=y +CONFIG_RENESAS_WDT=y +CONFIG_RENESAS_RZG2LWDT=y +CONFIG_RENESAS_RZV2HWDT=y +CONFIG_UNIPHIER_WATCHDOG=y +CONFIG_PM8916_WATCHDOG=m +CONFIG_APPLE_WATCHDOG=y +CONFIG_BCM2835_WDT=y +CONFIG_BCM7038_WDT=m +CONFIG_USBPCWATCHDOG=m +CONFIG_MFD_ADP5585=m +CONFIG_MFD_ALTERA_SYSMGR=y +CONFIG_MFD_BD9571MWV=y +CONFIG_MFD_AXP20X_I2C=y +CONFIG_MFD_AXP20X_RSB=y +CONFIG_MFD_MACSMC=y +CONFIG_MFD_DA9062=m +CONFIG_MFD_DLN2=m +CONFIG_MFD_EXYNOS_LPASS=m +CONFIG_MFD_HI6421_PMIC=y +CONFIG_MFD_HI655X_PMIC=y +CONFIG_MFD_MAX77620=y +CONFIG_MFD_MAX77759=m +CONFIG_MFD_MT6360=y +CONFIG_MFD_MT6397=y +CONFIG_MFD_SPMI_PMIC=y +CONFIG_MFD_RK8XX_I2C=y +CONFIG_MFD_RK8XX_SPI=y +CONFIG_MFD_SEC_ACPM=m +CONFIG_MFD_SEC_I2C=y +CONFIG_MFD_SL28CPLD=y +CONFIG_RZ_MTU3=y +CONFIG_MFD_TI_AM335X_TSCADC=m +CONFIG_MFD_TI_LP873X=m +CONFIG_MFD_TPS65219=y +CONFIG_MFD_TPS6594_I2C=m +CONFIG_MFD_ROHM_BD718XX=y +CONFIG_MFD_STM32_LPTIMER=m +CONFIG_MFD_STM32_TIMERS=m +CONFIG_MFD_WCD934X=m +CONFIG_MFD_KHADAS_MCU=m +CONFIG_MFD_QCOM_PM8008=m +CONFIG_REGULATOR_FIXED_VOLTAGE=y +CONFIG_REGULATOR_AXP20X=y +CONFIG_REGULATOR_BD718XX=y +CONFIG_REGULATOR_BD9571MWV=y +CONFIG_REGULATOR_CROS_EC=y +CONFIG_REGULATOR_DA9211=m +CONFIG_REGULATOR_FAN53555=y +CONFIG_REGULATOR_GPIO=y +CONFIG_REGULATOR_HI6421V530=y +CONFIG_REGULATOR_HI655X=y +CONFIG_REGULATOR_LP873X=m +CONFIG_REGULATOR_MAX77620=y +CONFIG_REGULATOR_MAX8973=y +CONFIG_REGULATOR_MAX20411=m +CONFIG_REGULATOR_MP8859=y +CONFIG_REGULATOR_MT6315=m +CONFIG_REGULATOR_MT6357=y +CONFIG_REGULATOR_MT6358=y +CONFIG_REGULATOR_MT6359=y +CONFIG_REGULATOR_MT6360=y +CONFIG_REGULATOR_MT6397=y +CONFIG_REGULATOR_PCA9450=y +CONFIG_REGULATOR_PF8X00=y +CONFIG_REGULATOR_PFUZE100=y +CONFIG_REGULATOR_PWM=y +CONFIG_REGULATOR_QCOM_PM8008=m +CONFIG_REGULATOR_QCOM_REFGEN=m +CONFIG_REGULATOR_QCOM_RPMH=y +CONFIG_REGULATOR_QCOM_SMD_RPM=y +CONFIG_REGULATOR_QCOM_SPMI=y +CONFIG_REGULATOR_QCOM_USB_VBUS=m +CONFIG_REGULATOR_RAA215300=y +CONFIG_REGULATOR_RK808=y +CONFIG_REGULATOR_S2MPS11=y +CONFIG_REGULATOR_TPS65132=m +CONFIG_REGULATOR_TPS65219=y +CONFIG_REGULATOR_VCTRL=m +CONFIG_RC_CORE=m +CONFIG_RC_DECODERS=y +CONFIG_RC_DEVICES=y +CONFIG_IR_GPIO_CIR=m +CONFIG_IR_MESON=m +CONFIG_IR_SUNXI=m +CONFIG_MEDIA_CEC_SUPPORT=y +CONFIG_CEC_MESON_G12A_AO=m +CONFIG_MEDIA_SUPPORT=m +CONFIG_MEDIA_CAMERA_SUPPORT=y +CONFIG_MEDIA_ANALOG_TV_SUPPORT=y +CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y +CONFIG_MEDIA_SDR_SUPPORT=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +# CONFIG_DVB_NET is not set +CONFIG_MEDIA_USB_SUPPORT=y +CONFIG_USB_GSPCA=m +CONFIG_USB_GSPCA_BENQ=m +CONFIG_USB_GSPCA_CONEX=m +CONFIG_USB_GSPCA_CPIA1=m +CONFIG_USB_GSPCA_DTCS033=m +CONFIG_USB_GSPCA_ETOMS=m +CONFIG_USB_GSPCA_FINEPIX=m +CONFIG_USB_GSPCA_JEILINJ=m +CONFIG_USB_GSPCA_JL2005BCD=m +CONFIG_USB_GSPCA_KINECT=m +CONFIG_USB_GSPCA_KONICA=m +CONFIG_USB_GSPCA_MARS=m +CONFIG_USB_GSPCA_MR97310A=m +CONFIG_USB_GSPCA_NW80X=m +CONFIG_USB_GSPCA_OV519=m +CONFIG_USB_GSPCA_OV534=m +CONFIG_USB_GSPCA_OV534_9=m +CONFIG_USB_GSPCA_PAC207=m +CONFIG_USB_GSPCA_PAC7302=m +CONFIG_USB_GSPCA_PAC7311=m +CONFIG_USB_GSPCA_SE401=m +CONFIG_USB_GSPCA_SN9C2028=m +CONFIG_USB_GSPCA_SN9C20X=m +CONFIG_USB_GSPCA_SONIXB=m +CONFIG_USB_GSPCA_SONIXJ=m +CONFIG_USB_GSPCA_SPCA1528=m +CONFIG_USB_GSPCA_SPCA500=m +CONFIG_USB_GSPCA_SPCA501=m +CONFIG_USB_GSPCA_SPCA505=m +CONFIG_USB_GSPCA_SPCA506=m +CONFIG_USB_GSPCA_SPCA508=m +CONFIG_USB_GSPCA_SPCA561=m +CONFIG_USB_GSPCA_SQ905=m +CONFIG_USB_GSPCA_SQ905C=m +CONFIG_USB_GSPCA_SQ930X=m +CONFIG_USB_GSPCA_STK014=m +CONFIG_USB_GSPCA_STK1135=m +CONFIG_USB_GSPCA_STV0680=m +CONFIG_USB_GSPCA_SUNPLUS=m +CONFIG_USB_GSPCA_T613=m +CONFIG_USB_GSPCA_TOPRO=m +CONFIG_USB_GSPCA_TOUPTEK=m +CONFIG_USB_GSPCA_TV8532=m +CONFIG_USB_GSPCA_VC032X=m +CONFIG_USB_GSPCA_VICAM=m +CONFIG_USB_GSPCA_XIRLINK_CIT=m +CONFIG_USB_GSPCA_ZC3XX=m +CONFIG_USB_GL860=m +CONFIG_USB_M5602=m +CONFIG_USB_STV06XX=m +CONFIG_USB_PWC=m +CONFIG_USB_VIDEO_CLASS=m +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_SDR_PLATFORM_DRIVERS=y +CONFIG_V4L_MEM2MEM_DRIVERS=y +CONFIG_VIDEO_AMPHION_VPU=m +CONFIG_VIDEO_APPLE_ISP=m +CONFIG_VIDEO_APPLE_AVD=m +CONFIG_VIDEO_CADENCE_CSI2RX=m +CONFIG_VIDEO_WAVE_VPU=m +CONFIG_VIDEO_E5010_JPEG_ENC=m +CONFIG_VIDEO_MEDIATEK_JPEG=m +CONFIG_VIDEO_MEDIATEK_VCODEC=m +CONFIG_VIDEO_MEDIATEK_MDP3=m +CONFIG_VIDEO_IMX7_CSI=m +CONFIG_VIDEO_IMX_MIPI_CSIS=m +CONFIG_VIDEO_IMX8_ISI=m +CONFIG_VIDEO_IMX8_ISI_M2M=y +CONFIG_VIDEO_IMX8_JPEG=m +CONFIG_VIDEO_QCOM_CAMSS=m +CONFIG_VIDEO_QCOM_IRIS=m +CONFIG_VIDEO_QCOM_VENUS=m +CONFIG_VIDEO_RCAR_CSI2=m +CONFIG_VIDEO_RCAR_ISP=m +CONFIG_VIDEO_RCAR_VIN=m +CONFIG_VIDEO_RZG2L_CSI2=m +CONFIG_VIDEO_RZG2L_CRU=m +CONFIG_VIDEO_RENESAS_FCP=m +CONFIG_VIDEO_RENESAS_FDP1=m +CONFIG_VIDEO_RENESAS_VSP1=m +CONFIG_VIDEO_RCAR_DRIF=m +CONFIG_VIDEO_ROCKCHIP_RGA=m +CONFIG_VIDEO_ROCKCHIP_CIF=m +CONFIG_VIDEO_SAMSUNG_EXYNOS_GSC=m +CONFIG_VIDEO_SAMSUNG_S5P_JPEG=m +CONFIG_VIDEO_SAMSUNG_S5P_MFC=m +CONFIG_VIDEO_SUN6I_CSI=m +CONFIG_VIDEO_SYNOPSYS_HDMIRX=m +CONFIG_VIDEO_SYNOPSYS_HDMIRX_LOAD_DEFAULT_EDID=y +CONFIG_VIDEO_TI_J721E_CSI2RX=m +CONFIG_VIDEO_HANTRO=m +CONFIG_VIDEO_IMX219=m +CONFIG_VIDEO_IMX412=m +CONFIG_VIDEO_OV5640=m +CONFIG_VIDEO_OV5645=m +CONFIG_VIDEO_S5KJN1=m +CONFIG_DRM=y +CONFIG_DRM_ADP=m +CONFIG_DRM_APPLE=m +CONFIG_DRM_HDLCD=m +CONFIG_DRM_MALI_DISPLAY=m +CONFIG_DRM_KOMEDA=m +CONFIG_DRM_ASAHI=m +CONFIG_DRM_FSL_LDB=m +CONFIG_DRM_I2C_NXP_TDA998X=m +CONFIG_DRM_ITE_IT6263=m +CONFIG_DRM_LONTIUM_LT8912B=m +CONFIG_DRM_LONTIUM_LT9611=m +CONFIG_DRM_LONTIUM_LT9611UXC=m +CONFIG_DRM_LONTIUM_LT8713SX=m +CONFIG_DRM_ITE_IT66121=m +CONFIG_DRM_NWL_MIPI_DSI=m +CONFIG_DRM_PARADE_PS8640=m +CONFIG_DRM_SII902X=m +CONFIG_DRM_SIMPLE_BRIDGE=m +CONFIG_DRM_THINE_THC63LVD1024=m +CONFIG_DRM_TOSHIBA_TC358767=m +CONFIG_DRM_TOSHIBA_TC358768=m +CONFIG_DRM_TI_TFP410=m +CONFIG_DRM_TI_SN65DSI83=m +CONFIG_DRM_TI_SN65DSI86=m +CONFIG_DRM_ANALOGIX_ANX7625=m +CONFIG_DRM_I2C_ADV7511=m +CONFIG_DRM_I2C_ADV7511_AUDIO=y +CONFIG_DRM_CDNS_DSI=m +CONFIG_DRM_CDNS_MHDP8546=m +CONFIG_DRM_IMX8MP_DW_HDMI_BRIDGE=m +CONFIG_DRM_DW_HDMI_AHB_AUDIO=m +CONFIG_DRM_DW_HDMI_CEC=m +CONFIG_DRM_DW_HDMI_QP_CEC=y +CONFIG_DRM_ETNAVIV=m +CONFIG_DRM_EXYNOS=m +CONFIG_DRM_EXYNOS5433_DECON=y +CONFIG_DRM_EXYNOS7_DECON=y +CONFIG_DRM_EXYNOS_DSI=y +# CONFIG_DRM_EXYNOS_DP is not set +CONFIG_DRM_EXYNOS_HDMI=y +CONFIG_DRM_EXYNOS_MIC=y +CONFIG_DRM_GUD=m +CONFIG_DRM_HISI_HIBMC=m +CONFIG_DRM_HISI_KIRIN=m +CONFIG_DRM_POWERVR=m +CONFIG_DRM_IMX_DCSS=m +CONFIG_DRM_LIMA=m +CONFIG_DRM_MEDIATEK=m +CONFIG_DRM_MEDIATEK_DP=m +CONFIG_DRM_MEDIATEK_HDMI=m +CONFIG_DRM_MEDIATEK_HDMI_V2=m +CONFIG_DRM_MESON=m +CONFIG_DRM_MSM=m +CONFIG_DRM_MXSFB=m +CONFIG_DRM_IMX_LCDIF=m +CONFIG_DRM_NOUVEAU=m +CONFIG_DRM_PANEL_BOE_TV101WUM_NL6=m +CONFIG_DRM_PANEL_LVDS=m +CONFIG_DRM_PANEL_HIMAX_HX8279=m +CONFIG_DRM_PANEL_HIMAX_HX83112A=m +CONFIG_DRM_PANEL_HIMAX_HX83112B=m +CONFIG_DRM_PANEL_ILITEK_ILI9882T=m +CONFIG_DRM_PANEL_KHADAS_TS050=m +CONFIG_DRM_PANEL_MANTIX_MLAF057WE51=m +CONFIG_DRM_PANEL_NOVATEK_NT36672A=m +CONFIG_DRM_PANEL_NOVATEK_NT36672E=m +CONFIG_DRM_PANEL_NOVATEK_NT37801=m +CONFIG_DRM_PANEL_RAYDIUM_RM67191=m +CONFIG_DRM_PANEL_RAYDIUM_RM692E5=m +CONFIG_DRM_PANEL_SAMSUNG_ATNA33XC20=m +CONFIG_DRM_PANEL_SITRONIX_ST7703=m +CONFIG_DRM_PANEL_STARTEK_KD070FHFID015=m +CONFIG_DRM_PANEL_EDP=m +CONFIG_DRM_PANEL_SIMPLE=m +CONFIG_DRM_PANEL_SUMMIT=m +CONFIG_DRM_PANEL_TRULY_NT35597_WQXGA=m +CONFIG_DRM_PANEL_VISIONOX_VTDR6130=m +CONFIG_DRM_PANFROST=m +CONFIG_DRM_PANTHOR=m +CONFIG_DRM_PL111=m +CONFIG_DRM_RCAR_DU=m +CONFIG_DRM_RCAR_DW_HDMI=m +CONFIG_DRM_RZG2L_DU=m +CONFIG_DRM_ROCKCHIP=m +CONFIG_ROCKCHIP_VOP2=y +CONFIG_ROCKCHIP_ANALOGIX_DP=y +CONFIG_ROCKCHIP_CDN_DP=y +CONFIG_ROCKCHIP_DW_DP=y +CONFIG_ROCKCHIP_DW_HDMI=y +CONFIG_ROCKCHIP_DW_HDMI_QP=y +CONFIG_ROCKCHIP_DW_MIPI_DSI=y +CONFIG_ROCKCHIP_INNO_HDMI=y +CONFIG_ROCKCHIP_LVDS=y +CONFIG_DRM_STM=m +CONFIG_DRM_STM_LVDS=m +CONFIG_DRM_SUN4I=m +CONFIG_DRM_SIMPLEDRM=y +CONFIG_DRM_TEGRA=m +CONFIG_DRM_TIDSS=m +CONFIG_DRM_GM12U320=m +CONFIG_DRM_UDL=m +CONFIG_DRM_V3D=m +CONFIG_DRM_VC4=m +CONFIG_DRM_ZYNQMP_DPSUB=m +CONFIG_DRM_ZYNQMP_DPSUB_AUDIO=y +CONFIG_FB=y +CONFIG_FB_EFI=y +CONFIG_BACKLIGHT_CLASS_DEVICE=y +CONFIG_BACKLIGHT_PWM=m +CONFIG_BACKLIGHT_APPLE_DWI=m +CONFIG_BACKLIGHT_QCOM_WLED=m +CONFIG_BACKLIGHT_LP855X=m +CONFIG_BACKLIGHT_GPIO=m +CONFIG_LOGO=y +CONFIG_LOGO_LINUX_CLUT224_FILE="drivers/video/logo/logo_linux_clut224.ppm" +CONFIG_DRM_ACCEL=y +CONFIG_DRM_ACCEL_ANE=m +CONFIG_SOUND=m +CONFIG_SND=m +CONFIG_SND_UMP_LEGACY_RAWMIDI=y +CONFIG_SND_HRTIMER=m +CONFIG_SND_UTIMER=y +CONFIG_SND_SEQUENCER=m +CONFIG_SND_SEQ_DUMMY=m +CONFIG_SND_DUMMY=m +CONFIG_SND_ALOOP=m +CONFIG_SND_VIRMIDI=m +CONFIG_SND_HDA_TEGRA=m +CONFIG_SND_HDA_CODEC_HDMI=m +CONFIG_SND_USB_AUDIO=m +CONFIG_SND_USB_AUDIO_MIDI_V2=y +CONFIG_SND_USB_UA101=m +CONFIG_SND_USB_CAIAQ=m +CONFIG_SND_USB_CAIAQ_INPUT=y +CONFIG_SND_USB_6FIRE=m +CONFIG_SND_USB_HIFACE=m +CONFIG_SND_BCD2000=m +CONFIG_SND_USB_AUDIO_QMI=m +CONFIG_SND_USB_POD=m +CONFIG_SND_USB_PODHD=m +CONFIG_SND_USB_TONEPORT=m +CONFIG_SND_USB_VARIAX=m +CONFIG_SND_SOC=m +CONFIG_SND_SOC_USB=m +CONFIG_SND_SOC_APPLE_AOP_AUDIO=m +CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m +CONFIG_SND_SOC_APPLE_MACAUDIO=m +CONFIG_SND_BCM2835_SOC_I2S=m +CONFIG_SND_SOC_FSL_ASRC=m +CONFIG_SND_SOC_FSL_MICFIL=m +CONFIG_SND_SOC_FSL_EASRC=m +CONFIG_SND_IMX_SOC=m +CONFIG_SND_SOC_IMX_SGTL5000=m +CONFIG_SND_SOC_FSL_ASOC_CARD=m +CONFIG_SND_SOC_IMX_AUDMIX=m +CONFIG_SND_SOC_MT8183=m +CONFIG_SND_SOC_MT8183_MT6358_TS3A227E_MAX98357A=m +CONFIG_SND_SOC_MT8183_DA7219_MAX98357A=m +CONFIG_SND_SOC_MT8188=m +CONFIG_SND_SOC_MT8188_MT6359=m +CONFIG_SND_SOC_MT8192=m +CONFIG_SND_SOC_MT8192_MT6359_RT1015_RT5682=m +CONFIG_SND_SOC_MT8195=m +CONFIG_SND_SOC_MT8195_MT6359=m +CONFIG_SND_SOC_MT8365=m +CONFIG_SND_SOC_MT8365_MT6357=m +CONFIG_SND_MESON_AXG_SOUND_CARD=m +CONFIG_SND_MESON_GX_SOUND_CARD=m +CONFIG_SND_SOC_QCOM=m +CONFIG_SND_SOC_APQ8016_SBC=m +CONFIG_SND_SOC_QDSP6_USB=m +CONFIG_SND_SOC_MSM8996=m +CONFIG_SND_SOC_SDM845=m +CONFIG_SND_SOC_SM8250=m +CONFIG_SND_SOC_SC8280XP=m +CONFIG_SND_SOC_SC7180=m +CONFIG_SND_SOC_SC7280=m +CONFIG_SND_SOC_X1E80100=m +CONFIG_SND_SOC_RCAR=m +CONFIG_SND_SOC_MSIOF=m +CONFIG_SND_SOC_RZ=m +CONFIG_SND_SOC_ROCKCHIP_I2S_TDM=m +CONFIG_SND_SOC_ROCKCHIP_SAI=m +CONFIG_SND_SOC_ROCKCHIP_SPDIF=m +CONFIG_SND_SOC_ROCKCHIP_RT5645=m +CONFIG_SND_SOC_RK3399_GRU_SOUND=m +CONFIG_SND_SOC_SAMSUNG=m +CONFIG_SND_SUN8I_CODEC=m +CONFIG_SND_SUN8I_CODEC_ANALOG=m +CONFIG_SND_SUN50I_CODEC_ANALOG=m +CONFIG_SND_SUN4I_I2S=m +CONFIG_SND_SUN4I_SPDIF=m +CONFIG_SND_SOC_TEGRA=m +CONFIG_SND_SOC_TEGRA210_AHUB=m +CONFIG_SND_SOC_TEGRA210_DMIC=m +CONFIG_SND_SOC_TEGRA210_I2S=m +CONFIG_SND_SOC_TEGRA210_OPE=m +CONFIG_SND_SOC_TEGRA186_ASRC=m +CONFIG_SND_SOC_TEGRA186_DSPK=m +CONFIG_SND_SOC_TEGRA210_ADMAIF=m +CONFIG_SND_SOC_TEGRA210_MVC=m +CONFIG_SND_SOC_TEGRA210_SFC=m +CONFIG_SND_SOC_TEGRA210_AMX=m +CONFIG_SND_SOC_TEGRA210_ADX=m +CONFIG_SND_SOC_TEGRA210_MIXER=m +CONFIG_SND_SOC_TEGRA_AUDIO_GRAPH_CARD=m +CONFIG_SND_SOC_J721E_EVM=m +CONFIG_SND_SOC_XILINX_I2S=m +CONFIG_SND_SOC_XILINX_AUDIO_FORMATTER=m +CONFIG_SND_SOC_XILINX_SPDIF=m +CONFIG_SND_SOC_SOF_TOPLEVEL=y +CONFIG_SND_SOC_SOF_OF=m +CONFIG_SND_SOC_SOF_MTK_TOPLEVEL=y +CONFIG_SND_SOC_SOF_MT8186=m +CONFIG_SND_SOC_SOF_MT8195=m +CONFIG_SND_SOC_AK4613=m +CONFIG_SND_SOC_AK4619=m +CONFIG_SND_SOC_DA7213=m +CONFIG_SND_SOC_ES7134=m +CONFIG_SND_SOC_ES7241=m +CONFIG_SND_SOC_ES8316=m +CONFIG_SND_SOC_ES8328_I2C=m +CONFIG_SND_SOC_GTM601=m +CONFIG_SND_SOC_MAX98090=m +CONFIG_SND_SOC_MSM8916_WCD_ANALOG=m +CONFIG_SND_SOC_MSM8916_WCD_DIGITAL=m +CONFIG_SND_SOC_RK3308=m +CONFIG_SND_SOC_RK817=m +CONFIG_SND_SOC_RT5640=m +CONFIG_SND_SOC_RT5659=m +CONFIG_SND_SOC_SIMPLE_AMPLIFIER=m +CONFIG_SND_SOC_SIMPLE_MUX=m +CONFIG_SND_SOC_TAS2552=m +CONFIG_SND_SOC_TAS571X=m +CONFIG_SND_SOC_TLV320AIC32X4_I2C=m +CONFIG_SND_SOC_TLV320AIC3X_I2C=m +CONFIG_SND_SOC_WCD9335=m +CONFIG_SND_SOC_WCD934X=m +CONFIG_SND_SOC_WCD937X_SDW=m +CONFIG_SND_SOC_WCD939X_SDW=m +CONFIG_SND_SOC_WM8524=m +CONFIG_SND_SOC_WM8904=m +CONFIG_SND_SOC_WM8960=m +CONFIG_SND_SOC_WM8962=m +CONFIG_SND_SOC_WM8978=m +CONFIG_SND_SOC_WSA881X=m +CONFIG_SND_SOC_WSA883X=m +CONFIG_SND_SOC_WSA884X=m +CONFIG_SND_SOC_NAU8822=m +CONFIG_SND_SOC_LPASS_WSA_MACRO=m +CONFIG_SND_SOC_LPASS_VA_MACRO=m +CONFIG_SND_SIMPLE_CARD=m +CONFIG_SND_AUDIO_GRAPH_CARD=m +CONFIG_SND_AUDIO_GRAPH_CARD2=m +CONFIG_HID_BATTERY_STRENGTH=y +CONFIG_HIDRAW=y +CONFIG_UHID=y +CONFIG_HID_A4TECH=m +CONFIG_HID_ACCUTOUCH=m +CONFIG_HID_ACRUX=m +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_APPLE=y +CONFIG_HID_APPLEIR=m +CONFIG_HID_AUREAL=m +CONFIG_HID_BELKIN=m +CONFIG_HID_BETOP_FF=m +CONFIG_HID_BIGBEN_FF=m +CONFIG_HID_CHERRY=m +CONFIG_HID_CHICONY=m +CONFIG_HID_CORSAIR=m +CONFIG_HID_COUGAR=m +CONFIG_HID_MACALLY=m +CONFIG_HID_PRODIKEYS=m +CONFIG_HID_CMEDIA=m +CONFIG_HID_CP2112=m +CONFIG_HID_CREATIVE_SB0540=m +CONFIG_HID_CYPRESS=m +CONFIG_HID_DRAGONRISE=m +CONFIG_DRAGONRISE_FF=y +CONFIG_HID_EMS_FF=m +CONFIG_HID_ELAN=m +CONFIG_HID_ELECOM=m +CONFIG_HID_ELO=m +CONFIG_HID_EVISION=m +CONFIG_HID_EZKEY=m +CONFIG_HID_FT260=m +CONFIG_HID_GEMBIRD=m +CONFIG_HID_GFRM=m +CONFIG_HID_GLORIOUS=m +CONFIG_HID_HOLTEK=m +CONFIG_HOLTEK_FF=y +CONFIG_HID_GOOGLE_STADIA_FF=m +CONFIG_HID_VIVALDI=m +CONFIG_HID_GT683R=m +CONFIG_HID_KEYTOUCH=m +CONFIG_HID_KYE=m +CONFIG_HID_KYSONA=m +CONFIG_HID_UCLOGIC=m +CONFIG_HID_WALTOP=m +CONFIG_HID_VIEWSONIC=m +CONFIG_HID_XIAOMI=m +CONFIG_HID_GYRATION=m +CONFIG_HID_ICADE=m +CONFIG_HID_ITE=m +CONFIG_HID_JABRA=m +CONFIG_HID_TWINHAN=m +CONFIG_HID_KENSINGTON=m +CONFIG_HID_LCPOWER=m +CONFIG_HID_LENOVO=m +CONFIG_HID_LENOVO_GO=m +CONFIG_HID_LENOVO_GO_S=m +CONFIG_HID_LETSKETCH=m +CONFIG_HID_LOGITECH=m +CONFIG_HID_LOGITECH_DJ=m +CONFIG_LOGITECH_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGIG940_FF=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_HID_MALTRON=m +CONFIG_HID_MAYFLASH=m +CONFIG_HID_MEGAWORLD_FF=m +CONFIG_HID_MICROSOFT=m +CONFIG_HID_MONTEREY=m +CONFIG_HID_MULTITOUCH=m +CONFIG_HID_NINTENDO=m +CONFIG_NINTENDO_FF=y +CONFIG_HID_NTI=m +CONFIG_HID_NTRIG=m +CONFIG_HID_NVIDIA_SHIELD=m +CONFIG_NVIDIA_SHIELD_FF=y +CONFIG_HID_ORTEK=m +CONFIG_HID_PANTHERLORD=m +CONFIG_PANTHERLORD_FF=y +CONFIG_HID_PENMOUNT=m +CONFIG_HID_PETALYNX=m +CONFIG_HID_PICOLCD=m +CONFIG_HID_PLANTRONICS=m +CONFIG_HID_PLAYSTATION=m +CONFIG_PLAYSTATION_FF=y +CONFIG_HID_PXRC=m +CONFIG_HID_RAPOO=m +CONFIG_HID_RAZER=m +CONFIG_HID_PRIMAX=m +CONFIG_HID_RETRODE=m +CONFIG_HID_ROCCAT=m +CONFIG_HID_SAITEK=m +CONFIG_HID_SAMSUNG=m +CONFIG_HID_SEMITEK=m +CONFIG_HID_SIGMAMICRO=m +CONFIG_HID_SONY=m +CONFIG_SONY_FF=y +CONFIG_HID_SPEEDLINK=m +CONFIG_HID_STEAM=m +CONFIG_STEAM_FF=y +CONFIG_HID_STEELSERIES=m +CONFIG_HID_SUNPLUS=m +CONFIG_HID_RMI=m +CONFIG_HID_GREENASIA=m +CONFIG_GREENASIA_FF=y +CONFIG_HID_SMARTJOYPLUS=m +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_HID_TIVO=m +CONFIG_HID_TOPSEED=m +CONFIG_HID_TOPRE=m +CONFIG_HID_THINGM=m +CONFIG_HID_THRUSTMASTER=m +CONFIG_THRUSTMASTER_FF=y +CONFIG_HID_UDRAW_PS3=m +CONFIG_HID_U2FZERO=m +CONFIG_HID_UNIVERSAL_PIDFF=m +CONFIG_HID_WACOM=m +CONFIG_HID_WIIMOTE=m +CONFIG_HID_WINWING=m +CONFIG_HID_XINMO=m +CONFIG_HID_ZEROPLUS=m +CONFIG_ZEROPLUS_FF=y +CONFIG_HID_ZYDACRON=m +CONFIG_HID_ALPS=m +CONFIG_HID_MCP2221=m +CONFIG_HID_HUAWEI=m +CONFIG_I2C_HID_ACPI=m +CONFIG_I2C_HID_OF=m +CONFIG_I2C_HID_OF_ELAN=m +CONFIG_HID_PID=y +CONFIG_USB_HIDDEV=y +CONFIG_SPI_HID_APPLE_OF=m +CONFIG_HID_DOCKCHANNEL=y +CONFIG_USB=y +CONFIG_USB_OTG=y +CONFIG_USB_MON=m +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_XHCI_PCI_RENESAS=m +CONFIG_USB_XHCI_RZV2M=y +CONFIG_USB_XHCI_SIDEBAND=y +CONFIG_USB_XHCI_TEGRA=y +CONFIG_USB_BRCMSTB=m +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_EHCI_EXYNOS=y +CONFIG_USB_EHCI_HCD_PLATFORM=y +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_OHCI_EXYNOS=y +CONFIG_USB_OHCI_HCD_PLATFORM=y +CONFIG_USB_RENESAS_USBHS_HCD=m +CONFIG_USB_RENESAS_USBHS=m +CONFIG_USB_PRINTER=m +CONFIG_USB_TMC=m +CONFIG_USB_STORAGE=y +CONFIG_USB_STORAGE_REALTEK=m +CONFIG_USB_STORAGE_DATAFAB=m +CONFIG_USB_STORAGE_FREECOM=m +CONFIG_USB_STORAGE_ISD200=m +CONFIG_USB_STORAGE_USBAT=m +CONFIG_USB_STORAGE_SDDR09=m +CONFIG_USB_STORAGE_SDDR55=m +CONFIG_USB_STORAGE_JUMPSHOT=m +CONFIG_USB_STORAGE_ALAUDA=m +CONFIG_USB_STORAGE_ONETOUCH=m +CONFIG_USB_STORAGE_KARMA=m +CONFIG_USB_STORAGE_CYPRESS_ATACB=m +CONFIG_USB_STORAGE_ENE_UB6250=m +CONFIG_USB_UAS=m +CONFIG_USB_MDC800=m +CONFIG_USB_MICROTEK=m +CONFIG_USBIP_CORE=m +CONFIG_USBIP_VHCI_HCD=m +CONFIG_USBIP_HOST=m +CONFIG_USB_CDNS_SUPPORT=m +CONFIG_USB_CDNS3=m +CONFIG_USB_CDNS3_GADGET=y +CONFIG_USB_CDNS3_HOST=y +CONFIG_USB_MTU3=y +CONFIG_USB_MUSB_HDRC=y +CONFIG_USB_MUSB_SUNXI=y +CONFIG_USB_DWC3=y +CONFIG_USB_DWC2=y +CONFIG_USB_CHIPIDEA=y +CONFIG_USB_CHIPIDEA_UDC=y +CONFIG_USB_CHIPIDEA_HOST=y +CONFIG_USB_ISP1760=y +CONFIG_USB_SERIAL=m +CONFIG_USB_SERIAL_GENERIC=y +CONFIG_USB_SERIAL_SIMPLE=m +CONFIG_USB_SERIAL_AIRCABLE=m +CONFIG_USB_SERIAL_ARK3116=m +CONFIG_USB_SERIAL_BELKIN=m +CONFIG_USB_SERIAL_CH341=m +CONFIG_USB_SERIAL_WHITEHEAT=m +CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m +CONFIG_USB_SERIAL_CP210X=m +CONFIG_USB_SERIAL_CYPRESS_M8=m +CONFIG_USB_SERIAL_EMPEG=m +CONFIG_USB_SERIAL_FTDI_SIO=m +CONFIG_USB_SERIAL_VISOR=m +CONFIG_USB_SERIAL_IPAQ=m +CONFIG_USB_SERIAL_IR=m +CONFIG_USB_SERIAL_EDGEPORT=m +CONFIG_USB_SERIAL_EDGEPORT_TI=m +CONFIG_USB_SERIAL_F81232=m +CONFIG_USB_SERIAL_F8153X=m +CONFIG_USB_SERIAL_GARMIN=m +CONFIG_USB_SERIAL_IPW=m +CONFIG_USB_SERIAL_IUU=m +CONFIG_USB_SERIAL_KEYSPAN_PDA=m +CONFIG_USB_SERIAL_KEYSPAN=m +CONFIG_USB_SERIAL_KLSI=m +CONFIG_USB_SERIAL_KOBIL_SCT=m +CONFIG_USB_SERIAL_MCT_U232=m +CONFIG_USB_SERIAL_MOS7720=m +CONFIG_USB_SERIAL_MOS7840=m +CONFIG_USB_SERIAL_NAVMAN=m +CONFIG_USB_SERIAL_PL2303=m +CONFIG_USB_SERIAL_OTI6858=m +CONFIG_USB_SERIAL_QCAUX=m +CONFIG_USB_SERIAL_QUALCOMM=m +CONFIG_USB_SERIAL_SPCP8X5=m +CONFIG_USB_SERIAL_SAFE=m +CONFIG_USB_SERIAL_SAFE_PADDED=y +CONFIG_USB_SERIAL_SIERRAWIRELESS=m +CONFIG_USB_SERIAL_SYMBOL=m +CONFIG_USB_SERIAL_TI=m +CONFIG_USB_SERIAL_CYBERJACK=m +CONFIG_USB_SERIAL_OPTION=m +CONFIG_USB_SERIAL_OMNINET=m +CONFIG_USB_SERIAL_OPTICON=m +CONFIG_USB_SERIAL_XSENS_MT=m +CONFIG_USB_SERIAL_SSU100=m +CONFIG_USB_SERIAL_QT2=m +CONFIG_USB_SERIAL_UPD78F0730=m +CONFIG_USB_SERIAL_XR=m +CONFIG_USB_EMI62=m +CONFIG_USB_EMI26=m +CONFIG_USB_ADUTUX=m +CONFIG_USB_SEVSEG=m +CONFIG_USB_LEGOTOWER=m +CONFIG_USB_LCD=m +CONFIG_USB_IDMOUSE=m +CONFIG_USB_APPLEDISPLAY=m +CONFIG_USB_QCOM_EUD=m +CONFIG_APPLE_MFI_FASTCHARGE=m +CONFIG_USB_SISUSBVGA=m +CONFIG_USB_LD=m +CONFIG_USB_TRANCEVIBRATOR=m +CONFIG_USB_IOWARRIOR=m +CONFIG_USB_ISIGHTFW=m +CONFIG_USB_YUREX=m +CONFIG_USB_HSIC_USB3503=y +CONFIG_USB_CHAOSKEY=m +CONFIG_USB_ONBOARD_DEV=m +CONFIG_USB_ONBOARD_DEV_USB5744=y +CONFIG_NOP_USB_XCEIV=y +CONFIG_USB_MXS_PHY=m +CONFIG_USB_GADGET=y +CONFIG_USB_RENESAS_USBHS_UDC=m +CONFIG_USB_RZV2M_USB3DRD=y +CONFIG_USB_RENESAS_USB3=m +CONFIG_USB_TEGRA_XUDC=m +CONFIG_USB_CONFIGFS=m +CONFIG_USB_CONFIGFS_SERIAL=y +CONFIG_USB_CONFIGFS_ACM=y +CONFIG_USB_CONFIGFS_OBEX=y +CONFIG_USB_CONFIGFS_NCM=y +CONFIG_USB_CONFIGFS_ECM=y +CONFIG_USB_CONFIGFS_ECM_SUBSET=y +CONFIG_USB_CONFIGFS_RNDIS=y +CONFIG_USB_CONFIGFS_EEM=y +CONFIG_USB_CONFIGFS_MASS_STORAGE=y +CONFIG_USB_CONFIGFS_F_FS=y +CONFIG_USB_MASS_STORAGE=m +CONFIG_TYPEC=y +CONFIG_TYPEC_TCPM=m +CONFIG_TYPEC_TCPCI=m +CONFIG_TYPEC_RT1711H=m +CONFIG_TYPEC_MT6360=m +CONFIG_TYPEC_TCPCI_MAXIM=m +CONFIG_TYPEC_FUSB302=m +CONFIG_TYPEC_QCOM_PMIC=m +CONFIG_TYPEC_UCSI=m +CONFIG_UCSI_CCG=m +CONFIG_UCSI_PMIC_GLINK=m +CONFIG_TYPEC_TPS6598X=m +CONFIG_TYPEC_SN201202X=y +CONFIG_TYPEC_HD3SS3220=m +CONFIG_TYPEC_MUX_FSA4480=m +CONFIG_TYPEC_MUX_GPIO_SBU=m +CONFIG_TYPEC_MUX_IT5205=m +CONFIG_TYPEC_MUX_NB7VPQ904M=m +CONFIG_TYPEC_MUX_PS883X=m +CONFIG_TYPEC_MUX_PTN36502=m +CONFIG_TYPEC_MUX_WCD939X_USBSS=m +CONFIG_TYPEC_DP_ALTMODE=m +CONFIG_MMC=y +CONFIG_MMC_BLOCK_MINORS=32 +CONFIG_MMC_ARMMMCI=y +CONFIG_MMC_SDHCI=y +CONFIG_MMC_SDHCI_ACPI=y +CONFIG_MMC_SDHCI_PLTFM=y +CONFIG_MMC_SDHCI_OF_ARASAN=y +CONFIG_MMC_SDHCI_OF_ESDHC=y +CONFIG_MMC_SDHCI_OF_DWCMSHC=y +CONFIG_MMC_SDHCI_OF_SPARX5=y +CONFIG_MMC_SDHCI_CADENCE=y +CONFIG_MMC_SDHCI_ESDHC_IMX=y +CONFIG_MMC_SDHCI_TEGRA=y +CONFIG_MMC_SDHCI_F_SDH30=y +CONFIG_MMC_MESON_GX=y +CONFIG_MMC_SDHCI_MSM=y +CONFIG_MMC_SPI=y +CONFIG_MMC_SDHI=y +CONFIG_MMC_UNIPHIER=y +CONFIG_MMC_DW=y +CONFIG_MMC_DW_EXYNOS=y +CONFIG_MMC_DW_HI3798CV200=y +CONFIG_MMC_DW_K3=y +CONFIG_MMC_DW_ROCKCHIP=y +CONFIG_MMC_VUB300=m +CONFIG_MMC_USHC=m +CONFIG_MMC_REALTEK_USB=m +CONFIG_MMC_SUNXI=y +CONFIG_MMC_BCM2835=y +CONFIG_MMC_MTK=y +CONFIG_MMC_SDHCI_XENON=y +CONFIG_MMC_SDHCI_AM654=y +CONFIG_MMC_OWL=y +CONFIG_SCSI_UFSHCD=y +CONFIG_SCSI_UFS_BSG=y +CONFIG_SCSI_UFS_CRYPTO=y +CONFIG_SCSI_UFSHCD_PLATFORM=y +CONFIG_SCSI_UFS_CDNS_PLATFORM=m +CONFIG_SCSI_UFS_QCOM=m +CONFIG_SCSI_UFS_MEDIATEK=y +CONFIG_SCSI_UFS_HISI=y +CONFIG_SCSI_UFS_RENESAS=m +CONFIG_SCSI_UFS_TI_J721E=m +CONFIG_SCSI_UFS_EXYNOS=y +CONFIG_SCSI_UFS_ROCKCHIP=y +CONFIG_MEMSTICK=m +CONFIG_MSPRO_BLOCK=m +CONFIG_MEMSTICK_REALTEK_USB=m +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_CLASS_FLASH=m +CONFIG_LEDS_CLASS_MULTICOLOR=m +CONFIG_LEDS_LM3692X=m +CONFIG_LEDS_PCA9532=m +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_PWM=y +CONFIG_LEDS_SYSCON=y +CONFIG_LEDS_QCOM_FLASH=m +CONFIG_LEDS_QCOM_LPG=m +CONFIG_LEDS_TRIGGER_TIMER=y +CONFIG_LEDS_TRIGGER_DISK=y +CONFIG_LEDS_TRIGGER_HEARTBEAT=y +CONFIG_LEDS_TRIGGER_CPU=y +CONFIG_LEDS_TRIGGER_DEFAULT_ON=y +CONFIG_LEDS_TRIGGER_PANIC=y +CONFIG_ACCESSIBILITY=y +CONFIG_SPEAKUP=m +CONFIG_SPEAKUP_SYNTH_ACNTSA=m +CONFIG_SPEAKUP_SYNTH_APOLLO=m +CONFIG_SPEAKUP_SYNTH_AUDPTR=m +CONFIG_SPEAKUP_SYNTH_BNS=m +CONFIG_SPEAKUP_SYNTH_DECTLK=m +CONFIG_SPEAKUP_SYNTH_LTLK=m +CONFIG_SPEAKUP_SYNTH_SOFT=m +CONFIG_SPEAKUP_SYNTH_SPKOUT=m +CONFIG_SPEAKUP_SYNTH_TXPRT=m +CONFIG_INFINIBAND=m +CONFIG_INFINIBAND_USER_MAD=m +CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_RDMA_RXE=m +CONFIG_RDMA_SIW=m +CONFIG_EDAC=y +CONFIG_EDAC_GHES=y +CONFIG_EDAC_LAYERSCAPE=m +CONFIG_EDAC_ZYNQMP=m +CONFIG_EDAC_VERSAL=m +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_DS1307=m +CONFIG_RTC_DRV_HYM8563=m +CONFIG_RTC_DRV_MAX77686=y +CONFIG_RTC_DRV_NVIDIA_VRS10=m +CONFIG_RTC_DRV_RK808=m +CONFIG_RTC_DRV_ISL1208=m +CONFIG_RTC_DRV_PCF85363=m +CONFIG_RTC_DRV_PCF8563=m +CONFIG_RTC_DRV_M41T80=m +CONFIG_RTC_DRV_BQ32K=m +CONFIG_RTC_DRV_RX8581=m +CONFIG_RTC_DRV_RV3028=m +CONFIG_RTC_DRV_RV8803=m +CONFIG_RTC_DRV_S5M=y +CONFIG_RTC_DRV_DS3232=y +CONFIG_RTC_DRV_PCF2127=m +CONFIG_RTC_DRV_PCF85063=m +CONFIG_RTC_DRV_DA9063=m +CONFIG_RTC_DRV_EFI=y +CONFIG_RTC_DRV_ZYNQMP=m +CONFIG_RTC_DRV_CROS_EC=y +CONFIG_RTC_DRV_FSL_FTM_ALARM=m +CONFIG_RTC_DRV_S3C=y +CONFIG_RTC_DRV_PL031=y +CONFIG_RTC_DRV_SUN6I=y +CONFIG_RTC_DRV_ARMADA38X=y +CONFIG_RTC_DRV_PM8XXX=m +CONFIG_RTC_DRV_TEGRA=y +CONFIG_RTC_DRV_SNVS=m +CONFIG_RTC_DRV_BBNSM=m +CONFIG_RTC_DRV_IMX_SC=m +CONFIG_RTC_DRV_MT6397=m +CONFIG_RTC_DRV_XGENE=y +CONFIG_RTC_DRV_TI_K3=m +CONFIG_RTC_DRV_RENESAS_RTCA3=m +CONFIG_RTC_DRV_APPLE_PMU=m +CONFIG_RTC_DRV_MACSMC=m +CONFIG_RTC_DRV_S32G=m +CONFIG_DMADEVICES=y +CONFIG_APPLE_SIO=m +CONFIG_DMA_BCM2835=y +CONFIG_DMA_SUN6I=m +CONFIG_FSL_EDMA=y +CONFIG_IMX_SDMA=m +CONFIG_K3_DMA=y +CONFIG_MV_XOR=y +CONFIG_MV_XOR_V2=y +CONFIG_OWL_DMA=y +CONFIG_PL330_DMA=y +CONFIG_TEGRA186_GPC_DMA=y +CONFIG_TEGRA20_APB_DMA=y +CONFIG_TEGRA210_ADMA=m +CONFIG_XILINX_DMA=m +CONFIG_XILINX_ZYNQMP_DMA=m +CONFIG_XILINX_ZYNQMP_DPDMA=m +CONFIG_MTK_UART_APDMA=m +CONFIG_QCOM_BAM_DMA=y +CONFIG_QCOM_GPI_DMA=m +CONFIG_QCOM_HIDMA_MGMT=y +CONFIG_QCOM_HIDMA=y +CONFIG_DW_EDMA=m +CONFIG_RCAR_DMAC=y +CONFIG_RENESAS_USB_DMAC=m +CONFIG_RZ_DMAC=y +CONFIG_TI_K3_UDMA=y +CONFIG_TI_K3_UDMA_GLUE_LAYER=y +CONFIG_STM32_DMA3=m +CONFIG_UDMABUF=y +CONFIG_DMABUF_HEAPS=y +CONFIG_DMABUF_HEAPS_SYSTEM=y +CONFIG_VFIO=y +CONFIG_VFIO_PCI=y +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_BALLOON=y +CONFIG_VIRTIO_MMIO=y +CONFIG_VHOST_NET=m +CONFIG_VHOST_VSOCK=m +CONFIG_XEN_GNTDEV=y +CONFIG_XEN_GRANT_DEV_ALLOC=y +CONFIG_GREYBUS=m +CONFIG_GREYBUS_BEAGLEPLAY=m +CONFIG_GPIB=m +CONFIG_GPIB_AGILENT_82357A=m +CONFIG_GPIB_NI_USB=m +CONFIG_GPIB_LPVO=m +CONFIG_STAGING=y +CONFIG_STAGING_MEDIA=y +CONFIG_VIDEO_MAX96712=m +CONFIG_VIDEO_MESON_VDEC=m +CONFIG_SND_BCM2835=m +CONFIG_CHROME_PLATFORMS=y +CONFIG_CROS_EC=y +CONFIG_CROS_EC_I2C=y +CONFIG_CROS_EC_RPMSG=m +CONFIG_CROS_EC_SPI=y +CONFIG_CROS_KBD_LED_BACKLIGHT=m +CONFIG_CROS_EC_CHARDEV=m +CONFIG_CROS_EC_TYPEC=m +CONFIG_CROS_TYPEC_SWITCH=m +CONFIG_EC_ACER_ASPIRE1=m +CONFIG_EC_HUAWEI_GAOKUN=m +CONFIG_EC_LENOVO_YOGA_C630=m +CONFIG_EC_LENOVO_THINKPAD_T14S=m +CONFIG_COMMON_CLK_RK808=y +CONFIG_COMMON_CLK_SCPI=y +CONFIG_COMMON_CLK_CS2000_CP=y +CONFIG_COMMON_CLK_FSL_SAI=y +CONFIG_COMMON_CLK_S2MPS11=y +CONFIG_COMMON_CLK_PWM=y +CONFIG_COMMON_CLK_RS9_PCIE=y +CONFIG_COMMON_CLK_VC3=y +CONFIG_COMMON_CLK_VC5=y +CONFIG_COMMON_CLK_BD718XX=m +CONFIG_CLK_RASPBERRYPI=y +CONFIG_CLK_IMX8MM=y +CONFIG_CLK_IMX8MN=y +CONFIG_CLK_IMX8MP=y +CONFIG_CLK_IMX8MQ=y +CONFIG_CLK_IMX8QXP=y +CONFIG_CLK_IMX8ULP=y +CONFIG_CLK_IMX93=y +CONFIG_CLK_IMX95_BLK_CTL=y +CONFIG_TI_SCI_CLK=y +CONFIG_COMMON_CLK_QCOM=y +CONFIG_CLK_ELIZA_DISPCC=m +CONFIG_CLK_ELIZA_GCC=y +CONFIG_CLK_ELIZA_TCSRCC=m +CONFIG_CLK_GLYMUR_DISPCC=m +CONFIG_CLK_GLYMUR_GCC=y +CONFIG_CLK_GLYMUR_TCSRCC=m +CONFIG_CLK_KAANAPALI_CAMCC=m +CONFIG_CLK_KAANAPALI_DISPCC=m +CONFIG_CLK_KAANAPALI_GCC=y +CONFIG_CLK_KAANAPALI_GPUCC=m +CONFIG_CLK_KAANAPALI_TCSRCC=m +CONFIG_CLK_KAANAPALI_VIDEOCC=m +CONFIG_CLK_X1E80100_CAMCC=m +CONFIG_CLK_X1E80100_DISPCC=m +CONFIG_CLK_X1E80100_GCC=y +CONFIG_CLK_X1E80100_GPUCC=m +CONFIG_CLK_X1E80100_TCSRCC=y +CONFIG_CLK_X1P42100_GPUCC=m +CONFIG_CLK_QCM2290_GPUCC=m +CONFIG_QCOM_A53PLL=y +CONFIG_QCOM_CLK_APCS_MSM8916=y +CONFIG_QCOM_CLK_APCC_MSM8996=y +CONFIG_QCOM_CLK_SMD_RPM=y +CONFIG_QCOM_CLK_RPMH=y +CONFIG_IPQ_APSS_6018=y +CONFIG_IPQ_CMN_PLL=m +CONFIG_IPQ_GCC_5018=y +CONFIG_IPQ_GCC_5210=y +CONFIG_IPQ_GCC_5332=y +CONFIG_IPQ_GCC_5424=y +CONFIG_IPQ_GCC_6018=y +CONFIG_IPQ_GCC_8074=y +CONFIG_IPQ_GCC_9574=y +CONFIG_IPQ_NSSCC_5424=m +CONFIG_IPQ_NSSCC_9574=m +CONFIG_MSM_GCC_8916=y +CONFIG_MSM_GCC_8953=y +CONFIG_MSM_MMCC_8994=m +CONFIG_MSM_GCC_8994=y +CONFIG_MSM_GCC_8996=y +CONFIG_MSM_MMCC_8996=m +CONFIG_MSM_GCC_8998=y +CONFIG_MSM_MMCC_8998=m +CONFIG_QCM_GCC_2290=y +CONFIG_QCM_DISPCC_2290=m +CONFIG_QCS_DISPCC_615=m +CONFIG_QCS_CAMCC_615=m +CONFIG_QCS_GCC_404=y +CONFIG_SA_CAMCC_8775P=m +CONFIG_QCS_GCC_8300=y +CONFIG_QCS_GCC_615=y +CONFIG_QCS_GPUCC_615=m +CONFIG_QCS_VIDEOCC_615=m +CONFIG_SC_CAMCC_7280=m +CONFIG_SC_CAMCC_8280XP=m +CONFIG_SA_DISPCC_8775P=m +CONFIG_SC_DISPCC_7280=m +CONFIG_SC_DISPCC_8280XP=m +CONFIG_SA_GCC_8775P=y +CONFIG_SA_GPUCC_8775P=m +CONFIG_SC_GCC_7180=y +CONFIG_SC_GCC_7280=y +CONFIG_SC_GCC_8180X=y +CONFIG_SC_GCC_8280XP=y +CONFIG_SC_GPUCC_7280=m +CONFIG_SC_GPUCC_8280XP=m +CONFIG_SC_LPASSCC_8280XP=m +CONFIG_SC_LPASS_CORECC_7280=m +CONFIG_SC_VIDEOCC_7280=m +CONFIG_SDM_CAMCC_845=m +CONFIG_QDU_GCC_1000=y +CONFIG_SDM_GPUCC_845=y +CONFIG_SDM_VIDEOCC_845=y +CONFIG_SDM_DISPCC_845=y +CONFIG_SDM_LPASSCC_845=m +CONFIG_SDX_GCC_75=y +CONFIG_SM_CAMCC_6350=m +CONFIG_SM_CAMCC_MILOS=m +CONFIG_SM_CAMCC_8250=m +CONFIG_SM_CAMCC_8550=m +CONFIG_SM_CAMCC_8650=m +CONFIG_SM_CAMCC_8750=m +CONFIG_SM_DISPCC_6115=m +CONFIG_SM_DISPCC_8250=y +CONFIG_SM_DISPCC_6350=m +CONFIG_SM_DISPCC_MILOS=m +CONFIG_SM_DISPCC_8450=m +CONFIG_SM_DISPCC_8550=m +CONFIG_SM_DISPCC_8750=m +CONFIG_SM_GCC_4450=y +CONFIG_SM_GCC_6115=y +CONFIG_SM_GCC_6350=y +CONFIG_SM_GCC_MILOS=y +CONFIG_SM_GCC_8350=y +CONFIG_SM_GCC_8450=y +CONFIG_SM_GCC_8550=y +CONFIG_SM_GCC_8650=y +CONFIG_SM_GCC_8750=y +CONFIG_SM_GPUCC_6115=m +CONFIG_SM_GPUCC_6350=m +CONFIG_SM_GPUCC_MILOS=m +CONFIG_SM_GPUCC_8150=y +CONFIG_SM_GPUCC_8250=y +CONFIG_SM_GPUCC_8350=m +CONFIG_SM_GPUCC_8450=m +CONFIG_SM_GPUCC_8550=m +CONFIG_SM_GPUCC_8650=m +CONFIG_SM_GPUCC_8750=m +CONFIG_SM_TCSRCC_8550=y +CONFIG_SM_TCSRCC_8650=y +CONFIG_SM_TCSRCC_8750=m +CONFIG_SA_VIDEOCC_8775P=m +CONFIG_SM_VIDEOCC_6350=m +CONFIG_SM_VIDEOCC_MILOS=m +CONFIG_SM_VIDEOCC_8250=y +CONFIG_SM_VIDEOCC_8550=m +CONFIG_SM_VIDEOCC_8750=m +CONFIG_QCOM_HFPLL=y +CONFIG_CLK_GFM_LPASS_SM8250=m +CONFIG_SM_VIDEOCC_8450=m +CONFIG_CLK_RCAR_USB2_CLOCK_SEL=y +CONFIG_CLK_RENESAS_VBATTB=m +CONFIG_EXYNOS_ACPM_CLK=m +CONFIG_CLK_SOPHGO_CV1800=y +CONFIG_XILINX_VCU=m +CONFIG_COMMON_CLK_XLNX_CLKWZRD=m +CONFIG_HWSPINLOCK=y +CONFIG_HWSPINLOCK_OMAP=m +CONFIG_HWSPINLOCK_QCOM=y +CONFIG_TEGRA186_TIMER=y +CONFIG_CLKSRC_STM32_LP=y +CONFIG_RENESAS_OSTM=y +CONFIG_ARM_MHU=y +CONFIG_EXYNOS_MBOX=m +CONFIG_IMX_MBOX=y +CONFIG_PLATFORM_MHU=y +CONFIG_OMAP2PLUS_MBOX=m +CONFIG_BCM2835_MBOX=y +CONFIG_QCOM_APCS_IPC=y +CONFIG_TEGRA_HSP_MBOX=y +CONFIG_MTK_ADSP_MBOX=m +CONFIG_QCOM_CPUCP_MBOX=m +CONFIG_QCOM_IPCC=y +CONFIG_CIX_MBOX=y +CONFIG_ARM_SMMU=y +CONFIG_ARM_SMMU_V3=y +CONFIG_QCOM_IOMMU=y +CONFIG_ROCKCHIP_IOMMU=y +CONFIG_TEGRA_IOMMU_SMMU=y +CONFIG_APPLE_DART=y +CONFIG_MTK_IOMMU=y +CONFIG_REMOTEPROC=y +CONFIG_IMX_REMOTEPROC=y +CONFIG_MTK_SCP=m +CONFIG_QCOM_Q6V5_ADSP=m +CONFIG_QCOM_Q6V5_MSS=m +CONFIG_QCOM_Q6V5_PAS=m +CONFIG_QCOM_SYSMON=m +CONFIG_QCOM_WCNSS_PIL=m +CONFIG_TI_K3_DSP_REMOTEPROC=m +CONFIG_TI_K3_M4_REMOTEPROC=m +CONFIG_TI_K3_R5_REMOTEPROC=m +CONFIG_RPMSG_CHAR=m +CONFIG_RPMSG_CTRL=m +CONFIG_RPMSG_QCOM_GLINK_RPM=y +CONFIG_RPMSG_QCOM_GLINK_SMEM=m +CONFIG_RPMSG_QCOM_SMD=y +CONFIG_RPMSG_VIRTIO=y +CONFIG_SOUNDWIRE=m +CONFIG_SOUNDWIRE_QCOM=m +CONFIG_APPLE_DOCKCHANNEL=y +CONFIG_APPLE_PMGR_MISC=y +CONFIG_APPLE_RTKIT_HELPER=y +CONFIG_APPLE_SART=y +CONFIG_APPLE_CLPC=m +CONFIG_FSL_DPAA=y +CONFIG_FSL_MC_DPIO=y +CONFIG_FSL_RCPM=y +CONFIG_MTK_CMDQ=m +CONFIG_MTK_DEVAPC=m +CONFIG_MTK_PMIC_WRAP=y +CONFIG_MTK_SVS=m +CONFIG_QCOM_AOSS_QMP=y +CONFIG_QCOM_COMMAND_DB=y +CONFIG_QCOM_GENI_SE=y +CONFIG_QCOM_LLCC=m +CONFIG_QCOM_OCMEM=m +CONFIG_QCOM_PMIC_GLINK=m +CONFIG_QCOM_RMTFS_MEM=m +CONFIG_QCOM_RPMH=y +CONFIG_QCOM_SMEM=y +CONFIG_QCOM_SMD_RPM=y +CONFIG_QCOM_SMP2P=y +CONFIG_QCOM_SMSM=y +CONFIG_QCOM_SOCINFO=m +CONFIG_QCOM_SPM=m +CONFIG_QCOM_STATS=m +CONFIG_QCOM_WCNSS_CTRL=m +CONFIG_QCOM_APR=m +CONFIG_QCOM_ICC_BWMON=m +CONFIG_QCOM_PBS=m +CONFIG_ROCKCHIP_IODOMAIN=y +CONFIG_TI_PRUSS=m +CONFIG_OWL_PM_DOMAINS=y +CONFIG_RASPBERRYPI_POWER=y +CONFIG_IMX_SCU_PD=y +CONFIG_QCOM_CPR=y +CONFIG_QCOM_RPMHPD=y +CONFIG_QCOM_RPMPD=y +CONFIG_TI_SCI_PM_DOMAINS=y +CONFIG_ARM_IMX_BUS_DEVFREQ=y +CONFIG_ARM_IMX8M_DDRC_DEVFREQ=m +CONFIG_ARM_MEDIATEK_CCI_DEVFREQ=m +CONFIG_PM_DEVFREQ_EVENT=y +CONFIG_DEVFREQ_EVENT_ROCKCHIP_DFI=m +CONFIG_EXTCON_PTN5150=m +CONFIG_EXTCON_USB_GPIO=y +CONFIG_EXTCON_USBC_CROS_EC=y +CONFIG_FSL_IFC=y +CONFIG_RENESAS_RPCIF=m +CONFIG_STM32_OMM=m +CONFIG_DLN2_ADC=m +CONFIG_EXYNOS_ADC=m +CONFIG_IMX8QXP_ADC=m +CONFIG_IMX93_ADC=m +CONFIG_MAX9611=m +CONFIG_MEDIATEK_MT6359_AUXADC=m +CONFIG_MEDIATEK_MT6577_AUXADC=m +CONFIG_QCOM_SPMI_VADC=m +CONFIG_QCOM_SPMI_ADC5=m +CONFIG_ROCKCHIP_SARADC=m +CONFIG_RZG2L_ADC=m +CONFIG_RZT2H_ADC=m +CONFIG_SOPHGO_CV1800B_ADC=m +CONFIG_TI_ADS1015=m +CONFIG_TI_AM335X_ADC=m +CONFIG_XILINX_XADC=m +CONFIG_XILINX_AMS=m +CONFIG_IIO_AOP_SENSOR_ALS=m +CONFIG_IIO_CROS_EC_SENSORS_CORE=m +CONFIG_IIO_CROS_EC_SENSORS=m +CONFIG_IIO_ST_LSM6DSX=m +CONFIG_IIO_CROS_EC_LIGHT_PROX=m +CONFIG_SENSORS_ISL29018=m +CONFIG_VCNL4000=m +CONFIG_IIO_ST_MAGN_3AXIS=m +CONFIG_IIO_STM32_TIMER_TRIGGER=m +CONFIG_IIO_CROS_EC_BARO=m +CONFIG_MPL3115=m +CONFIG_PWM=y +CONFIG_PWM_ADP5585=m +CONFIG_PWM_APPLE=m +CONFIG_PWM_BCM2835=m +CONFIG_PWM_BRCMSTB=m +CONFIG_PWM_CROS_EC=m +CONFIG_PWM_IMX27=m +CONFIG_PWM_MEDIATEK=m +CONFIG_PWM_MESON=m +CONFIG_PWM_MTK_DISP=m +CONFIG_PWM_RENESAS_RCAR=m +CONFIG_PWM_RENESAS_RZG2L_GPT=m +CONFIG_PWM_RENESAS_RZ_MTU3=m +CONFIG_PWM_RENESAS_TPU=m +CONFIG_PWM_ROCKCHIP=y +CONFIG_PWM_SAMSUNG=y +CONFIG_PWM_SL28CPLD=m +CONFIG_PWM_STM32=m +CONFIG_PWM_SUN4I=m +CONFIG_PWM_TEGRA=m +CONFIG_PWM_TIECAP=m +CONFIG_PWM_TIEHRPWM=m +CONFIG_PWM_VISCONTI=m +CONFIG_PWM_XILINX=m +CONFIG_SL28CPLD_INTC=y +CONFIG_XILINX_INTC=y +CONFIG_QCOM_PDC=y +CONFIG_QCOM_MPM=y +CONFIG_TI_SCI_INTR_IRQCHIP=y +CONFIG_TI_SCI_INTA_IRQCHIP=y +CONFIG_RESET_GPIO=m +CONFIG_RESET_IMX7=y +CONFIG_RESET_QCOM_AOSS=y +CONFIG_RESET_QCOM_PDC=m +CONFIG_RESET_RZG2L_USBPHY_CTRL=y +CONFIG_RESET_RZV2H_USB2PHY=m +CONFIG_RESET_TI_SCI=y +CONFIG_PHY_CAN_TRANSCEIVER=m +CONFIG_PHY_NXP_PTN3222=m +CONFIG_PHY_SNPS_EUSB2=m +CONFIG_PHY_XGENE=y +CONFIG_PHY_SUN4I_USB=y +CONFIG_PHY_APPLE_ATC=y +CONFIG_PHY_APPLE_DPTX=m +CONFIG_PHY_CADENCE_TORRENT=m +CONFIG_PHY_CADENCE_DPHY=m +CONFIG_PHY_CADENCE_DPHY_RX=m +CONFIG_PHY_CADENCE_SIERRA=m +CONFIG_PHY_CADENCE_SALVO=m +CONFIG_PHY_FSL_IMX8MQ_USB=m +CONFIG_PHY_MIXEL_MIPI_DPHY=m +CONFIG_PHY_FSL_IMX8M_PCIE=y +CONFIG_PHY_HI6220_USB=y +CONFIG_PHY_HISTB_COMBPHY=y +CONFIG_PHY_HISI_INNO_USB2=y +CONFIG_PHY_MVEBU_CP110_COMPHY=y +CONFIG_PHY_MTK_PCIE=m +CONFIG_PHY_MTK_TPHY=y +CONFIG_PHY_MTK_HDMI=m +CONFIG_PHY_MTK_MIPI_DSI=m +CONFIG_PHY_MTK_DP=m +CONFIG_PHY_QCOM_EDP=m +CONFIG_PHY_QCOM_PCIE2=m +CONFIG_PHY_QCOM_QMP=m +CONFIG_PHY_QCOM_QUSB2=m +CONFIG_PHY_QCOM_EUSB2_REPEATER=m +CONFIG_PHY_QCOM_M31_USB=m +CONFIG_PHY_QCOM_M31_EUSB=m +CONFIG_PHY_QCOM_USB_HS=m +CONFIG_PHY_QCOM_USB_SNPS_FEMTO_V2=m +CONFIG_PHY_QCOM_USB_HS_28NM=m +CONFIG_PHY_QCOM_USB_SS=m +CONFIG_PHY_QCOM_SGMII_ETH=m +CONFIG_PHY_R8A779F0_ETHERNET_SERDES=y +CONFIG_PHY_RCAR_GEN3_PCIE=y +CONFIG_PHY_RCAR_GEN3_USB2=y +CONFIG_PHY_RCAR_GEN3_USB3=m +CONFIG_PHY_RZ_G3E_USB3=m +CONFIG_PHY_ROCKCHIP_EMMC=y +CONFIG_PHY_ROCKCHIP_INNO_HDMI=m +CONFIG_PHY_ROCKCHIP_INNO_USB2=y +CONFIG_PHY_ROCKCHIP_INNO_DSIDPHY=m +CONFIG_PHY_ROCKCHIP_NANENG_COMBO_PHY=m +CONFIG_PHY_ROCKCHIP_PCIE=m +CONFIG_PHY_ROCKCHIP_SAMSUNG_HDPTX=m +CONFIG_PHY_ROCKCHIP_SNPS_PCIE3=y +CONFIG_PHY_ROCKCHIP_TYPEC=y +CONFIG_PHY_ROCKCHIP_USBDP=m +CONFIG_PHY_SAMSUNG_UFS=y +CONFIG_PHY_EXYNOS5_USBDRD=m +CONFIG_PHY_UNIPHIER_USB2=y +CONFIG_PHY_UNIPHIER_USB3=y +CONFIG_PHY_TEGRA_XUSB=y +CONFIG_PHY_AM654_SERDES=m +CONFIG_PHY_J721E_WIZ=m +CONFIG_OMAP_USB2=m +CONFIG_PHY_XILINX_ZYNQMP=m +CONFIG_ARM_CCI_PMU=m +CONFIG_ARM_CCN=m +CONFIG_ARM_CMN=m +CONFIG_ARM_SMMU_V3_PMU=m +CONFIG_ARM_DSU_PMU=m +CONFIG_FSL_IMX8_DDR_PMU=m +CONFIG_QCOM_L2_PMU=y +CONFIG_QCOM_L3_PMU=y +CONFIG_ARM_SPE_PMU=m +CONFIG_ARM_DMC620_PMU=m +CONFIG_APPLE_M1_CPU_PMU=y +CONFIG_HISI_PMU=y +CONFIG_ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU=m +CONFIG_NVIDIA_CORESIGHT_PMU_ARCH_SYSTEM_PMU=m +CONFIG_MESON_DDR_PMU=m +CONFIG_NVMEM_LAYOUT_SL28_VPD=m +CONFIG_NVMEM_APPLE_EFUSES=m +CONFIG_NVMEM_APPLE_SPMI=y +CONFIG_NVMEM_IMX_OCOTP=y +CONFIG_NVMEM_IMX_OCOTP_ELE=m +CONFIG_NVMEM_IMX_OCOTP_SCU=y +CONFIG_NVMEM_LAYERSCAPE_SFP=m +CONFIG_NVMEM_MTK_EFUSE=y +CONFIG_NVMEM_QCOM_QFPROM=y +CONFIG_NVMEM_QCOM_SEC_QFPROM=m +CONFIG_NVMEM_RMEM=m +CONFIG_NVMEM_ROCKCHIP_EFUSE=y +CONFIG_NVMEM_ROCKCHIP_OTP=y +CONFIG_NVMEM_SNVS_LPGPR=y +CONFIG_NVMEM_SPMI_SDAM=m +CONFIG_NVMEM_SUNXI_SID=y +CONFIG_NVMEM_UNIPHIER_EFUSE=y +CONFIG_NVMEM_ZYNQMP=m +CONFIG_FPGA=y +CONFIG_FPGA_MGR_ALTERA_CVP=m +CONFIG_FPGA_MGR_STRATIX10_SOC=m +CONFIG_FPGA_BRIDGE=m +CONFIG_ALTERA_FREEZE_BRIDGE=m +CONFIG_XILINX_PR_DECOUPLER=m +CONFIG_FPGA_REGION=m +CONFIG_OF_FPGA_REGION=m +CONFIG_FPGA_MGR_ZYNQMP_FPGA=m +CONFIG_FPGA_MGR_VERSAL_FPGA=m +CONFIG_TEE=y +CONFIG_OPTEE=y +CONFIG_QCOMTEE=m +CONFIG_MUX_APPLE_DPXBAR=y +CONFIG_MUX_GPIO=m +CONFIG_MUX_MMIO=y +CONFIG_SLIM_QCOM_NGD_CTRL=m +CONFIG_INTERCONNECT_IMX=y +CONFIG_INTERCONNECT_IMX8MM=m +CONFIG_INTERCONNECT_IMX8MN=m +CONFIG_INTERCONNECT_IMX8MQ=m +CONFIG_INTERCONNECT_IMX8MP=y +CONFIG_INTERCONNECT_QCOM=y +CONFIG_INTERCONNECT_QCOM_ELIZA=y +CONFIG_INTERCONNECT_QCOM_GLYMUR=y +CONFIG_INTERCONNECT_QCOM_KAANAPALI=y +CONFIG_INTERCONNECT_QCOM_MSM8916=m +CONFIG_INTERCONNECT_QCOM_MSM8953=y +CONFIG_INTERCONNECT_QCOM_MSM8996=y +CONFIG_INTERCONNECT_QCOM_OSM_L3=m +CONFIG_INTERCONNECT_QCOM_QCM2290=y +CONFIG_INTERCONNECT_QCOM_QCS404=m +CONFIG_INTERCONNECT_QCOM_QCS615=y +CONFIG_INTERCONNECT_QCOM_QCS8300=y +CONFIG_INTERCONNECT_QCOM_QDU1000=y +CONFIG_INTERCONNECT_QCOM_SA8775P=y +CONFIG_INTERCONNECT_QCOM_SC7180=y +CONFIG_INTERCONNECT_QCOM_SC7280=y +CONFIG_INTERCONNECT_QCOM_SC8180X=y +CONFIG_INTERCONNECT_QCOM_SC8280XP=y +CONFIG_INTERCONNECT_QCOM_SDM845=y +CONFIG_INTERCONNECT_QCOM_SDX75=y +CONFIG_INTERCONNECT_QCOM_SM6115=y +CONFIG_INTERCONNECT_QCOM_SM6350=y +CONFIG_INTERCONNECT_QCOM_MILOS=y +CONFIG_INTERCONNECT_QCOM_SM8150=y +CONFIG_INTERCONNECT_QCOM_SM8250=y +CONFIG_INTERCONNECT_QCOM_SM8350=y +CONFIG_INTERCONNECT_QCOM_SM8450=y +CONFIG_INTERCONNECT_QCOM_SM8550=y +CONFIG_INTERCONNECT_QCOM_SM8650=y +CONFIG_INTERCONNECT_QCOM_SM8750=y +CONFIG_INTERCONNECT_QCOM_X1E80100=y +CONFIG_COUNTER=m +CONFIG_RZ_MTU3_CNT=m +CONFIG_STM32_TIMER_CNT=m +CONFIG_TI_EQEP=m +CONFIG_HTE=y +CONFIG_HTE_TEGRA194=y +CONFIG_HTE_TEGRA194_TEST=m +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_XFS_FS=m +CONFIG_XFS_SUPPORT_V4=y +CONFIG_XFS_SUPPORT_ASCII_CI=y +CONFIG_XFS_QUOTA=y +CONFIG_XFS_POSIX_ACL=y +CONFIG_XFS_RT=y +CONFIG_BTRFS_FS=y +CONFIG_BTRFS_FS_POSIX_ACL=y +CONFIG_F2FS_FS=m +CONFIG_F2FS_FS_SECURITY=y +CONFIG_F2FS_FS_COMPRESSION=y +CONFIG_F2FS_UNFAIR_RWSEM=y +CONFIG_FANOTIFY=y +CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y +CONFIG_QUOTA=y +CONFIG_AUTOFS_FS=y +CONFIG_FUSE_FS=m +CONFIG_CUSE=m +CONFIG_OVERLAY_FS=m +CONFIG_ISO9660_FS=m +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_UDF_FS=m +CONFIG_MSDOS_FS=m +CONFIG_VFAT_FS=y +CONFIG_EXFAT_FS=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_LZX_XPRESS=y +CONFIG_NTFS3_FS_POSIX_ACL=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_HUGETLBFS=y +CONFIG_EFIVAR_FS=y +CONFIG_HFS_FS=m +CONFIG_HFSPLUS_FS=m +CONFIG_UBIFS_FS=m +CONFIG_SQUASHFS=y +CONFIG_SQUASHFS_XATTR=y +CONFIG_SQUASHFS_LZ4=y +CONFIG_SQUASHFS_LZO=y +CONFIG_SQUASHFS_XZ=y +CONFIG_SQUASHFS_ZSTD=y +CONFIG_PSTORE_RAM=m +CONFIG_EROFS_FS=m +CONFIG_EROFS_FS_ZIP_DEFLATE=y +CONFIG_EROFS_FS_ZIP_ZSTD=y +CONFIG_EROFS_FS_ZIP_ACCEL=y +CONFIG_NFS_FS=y +CONFIG_NFS_V4=y +CONFIG_NFS_V4_2=y +CONFIG_ROOT_NFS=y +CONFIG_CIFS=m +CONFIG_CIFS_UPCALL=y +CONFIG_CIFS_XATTR=y +CONFIG_CIFS_POSIX=y +CONFIG_CIFS_DFS_UPCALL=y +CONFIG_CIFS_SWN_UPCALL=y +CONFIG_9P_FS=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_936=m +CONFIG_NLS_CODEPAGE_950=m +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_ASCII=m +CONFIG_NLS_ISO8859_1=y +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_PERSISTENT_KEYRINGS=y +CONFIG_TRUSTED_KEYS=y +# CONFIG_TRUSTED_KEYS_TPM is not set +# CONFIG_TRUSTED_KEYS_TEE is not set +CONFIG_SECURITY=y +CONFIG_SECURITY_YAMA=y +CONFIG_SECURITY_LANDLOCK=y +CONFIG_CRYPTO_USER=y +CONFIG_CRYPTO_NULL=m +CONFIG_CRYPTO_CRYPTD=m +CONFIG_CRYPTO_BENCHMARK=m +CONFIG_CRYPTO_ECDH=y +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_BLOWFISH=m +CONFIG_CRYPTO_CAMELLIA=m +CONFIG_CRYPTO_CAST5=m +CONFIG_CRYPTO_CAST6=m +CONFIG_CRYPTO_SERPENT=m +CONFIG_CRYPTO_TWOFISH=m +CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_CTS=m +CONFIG_CRYPTO_HCTR2=m +CONFIG_CRYPTO_LRW=m +CONFIG_CRYPTO_PCBC=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CCM=y +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_ECHAINIV=y +CONFIG_CRYPTO_BLAKE2B=m +CONFIG_CRYPTO_CMAC=y +CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_RMD160=m +CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_STREEBOG=m +CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_XCBC=m +CONFIG_CRYPTO_XXHASH=m +CONFIG_CRYPTO_CRC32C=m +CONFIG_CRYPTO_CRC32=m +CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_USER_API_HASH=m +CONFIG_CRYPTO_USER_API_SKCIPHER=m +CONFIG_CRYPTO_USER_API_RNG=m +CONFIG_CRYPTO_USER_API_AEAD=m +CONFIG_CRYPTO_GHASH_ARM64_CE=y +CONFIG_CRYPTO_AES_ARM64_BS=m +CONFIG_CRYPTO_AES_ARM64_CE_CCM=y +CONFIG_CRYPTO_DEV_SUN8I_CE=m +CONFIG_CRYPTO_DEV_FSL_CAAM=m +CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM=m +CONFIG_CRYPTO_DEV_QCE=m +CONFIG_CRYPTO_DEV_QCOM_RNG=m +CONFIG_CRYPTO_DEV_TEGRA=m +CONFIG_CRYPTO_DEV_XILINX_TRNG=m +CONFIG_CRYPTO_DEV_ZYNQMP_AES=m +CONFIG_CRYPTO_DEV_ZYNQMP_SHA3=m +CONFIG_CRYPTO_DEV_CCREE=m +CONFIG_CRYPTO_DEV_HISI_SEC2=m +CONFIG_CRYPTO_DEV_HISI_ZIP=m +CONFIG_CRYPTO_DEV_HISI_HPRE=m +CONFIG_CRYPTO_DEV_HISI_TRNG=m +CONFIG_CRYPTO_DEV_SA2UL=m +CONFIG_DMA_RESTRICTED_POOL=y +CONFIG_CMA_SIZE_MBYTES=32 +CONFIG_PRINTK_TIME=y +CONFIG_DYNAMIC_DEBUG=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y +CONFIG_DEBUG_INFO_REDUCED=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_DEBUG_FS=y +CONFIG_ENABLE_DEFAULT_TRACERS=y +CONFIG_CORESIGHT=m +CONFIG_CORESIGHT_LINK_AND_SINK_TMC=m +CONFIG_CORESIGHT_CATU=m +CONFIG_CORESIGHT_SINK_TPIU=m +CONFIG_CORESIGHT_SINK_ETBV10=m +CONFIG_CORESIGHT_STM=m +CONFIG_CORESIGHT_CPU_DEBUG=m +CONFIG_CORESIGHT_CTI=m +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_MEMTEST=y From 8ad79fc25869a9a5cd36262cc8a702b4327cf437 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 20:05:00 -0400 Subject: [PATCH 0974/1579] drm/asahi: g17: replay render groups firmware retired without executing A render group can be retired by firmware without its fragment pass ever running: the tiler start record is written, the fragment end record is not, both completion stamps already cover the group, its job list is empty and no recovery is reported. On Neo this happens to an innocent owner (the compositor) at the instant another client's compute command fails with a zero-duration completion record. The front group then polls Pending forever, the 2 s scheduler timeout fires, fail_after_observation takes the strict abandonment witness and fails the owner's VM with EIO. The compositor's device is lost and the desktop freezes. Use the same witness to recognise the abandoned group as soon as it retires, and publish the same command again instead of failing it. In Pair::retire_front a lone, outer-published front group that polls Pending is checked with retirement::abandoned (executed groups are never accepted); the proof has to hold on two observations at least 2 ms apart so a single sample straddling a firmware update cannot trigger it. The dead generation's free-list and QoS accounting is completed exactly as the timeout path does, the record is retired and the packet is parked on the pair with its publication state reset; its completion stays unsignalled and no VM status is recorded. Firmware::replay_render, run by the event worker next to the compute replay and never during recovery, prepares and publishes the parked packet again through the ordinary path (fresh ordinal, descriptors, scene and timestamp record). A closed owner, a cancelled packet or a stale context settle the replay with ECANCELED, which does not poison the VM; other publication failures settle it with their error as before. Completion::begin_replay bounds the attempts to three, after which the previous EIO settlement applies. The original submission's scheduler timeout is ignored while its packet is parked, and a processor stop drains a parked replay like active groups. Nothing of an abandoned group reached its attachments, which only the fragment stage writes, so re-running the tiler and fragment passes from the same command is the first execution as far as render targets are concerned. Tiler-side pipeline statistics may count twice; vertex-stage side effects exist only behind an opt-in diagnostic. Tested on Neo (A18 Pro, G17P) with failing ray-query clients next to an animating compositor: the first incident at 81.455 s was replayed at 81.468 s (slot 4, qids [8, 9], pass [tiler start, 0], stamps covering the group) and the compositor kept rendering with no device loss, where the previous kernels lost the desktop in three out of three runs. With this change and VerboseFaults set, two further boots logged 2 and 3 replayed groups over 13 and 24 failing clients (the first boot differed only in the log string), with no device loss and the compositor still rendering. Boots of earlier builds with the same replay logic ran about 1,500 ray-query, ray-tracing pipeline and mesh conformance cases without failing clients and logged no replay. With the VerboseFaults debug flag, each replay is logged with its slot and the pass record that proved the group was not executed. Co-Authored-By: Claude Fable 5.1 Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/g17/job.rs | 5 + drivers/gpu/drm/asahi/g17/queue/render.rs | 134 +++++++++++++++++- .../gpu/drm/asahi/g17/queue/render/parked.rs | 1 + drivers/gpu/drm/asahi/g17/runtime/events.rs | 2 + drivers/gpu/drm/asahi/g17/runtime/render.rs | 1 + .../drm/asahi/g17/runtime/render/replay.rs | 105 ++++++++++++++ .../drm/asahi/g17/runtime/render/settle.rs | 2 +- 7 files changed, 247 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/replay.rs diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index b85d794ace512..9ad0370edfa2f 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -185,6 +185,11 @@ impl Packet { pub(crate) fn is_published(&self) -> bool { self.publication.load(Ordering::Acquire) == 2 } + /// A published group retired by firmware without executing is published + /// again from the same packet. Publication state returns to "begun". + pub(crate) fn reset_publication_for_replay(&self) { + self.publication.store(1, Ordering::Release); + } pub(crate) fn cancel_requested(&self) -> bool { self.cancelled.load(Ordering::Acquire) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 24a274e1af62a..bbf85695970a0 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -130,8 +130,14 @@ struct Active { context_update: bool, failed: bool, drained_result: Option>, + /// First observation of this group retired and unlinked without execution. + abandoned_since: Option, } +/// A firmware-retired, unexecuted group must hold on two observations this far +/// apart before it is republished; a single sample may straddle an update. +const ABANDON_GRACE_NS: u64 = 2_000_000; + /// Installed kick backing remains owned even when this pair has no logical /// owner. Commands retain their exact descriptor and allocation generations. pub(crate) struct Pair { @@ -162,6 +168,8 @@ pub(crate) struct Pair { previous: Option, displaced: Option, context_update: bool, + /// A retired, never-executed group awaiting republication on this pair. + replay: Option>, } impl Pair { #[inline(never)] @@ -243,6 +251,7 @@ impl Pair { terminal: None, previous: None, displaced: None, + replay: None, context_update: true, })) } @@ -271,7 +280,21 @@ impl Pair { &mut self.memory.manager } pub(crate) fn in_flight(&self) -> bool { - self.active.len() != 0 + self.active.len() != 0 || self.replay.is_some() + } + /// A replay or an abandonment grace period needs the worker to poll. + pub(crate) fn replay_polling(&self) -> bool { + self.replay.is_some() || self.active.iter().any(|a| a.abandoned_since.is_some()) + } + pub(crate) fn replay_front(&self) -> Option> { + self.replay.clone() + } + pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { + if !self.replay.as_ref().is_some_and(|front| Arc::ptr_eq(front, packet)) { + return Err(EIO); + } + self.replay = None; + Ok(()) } pub(crate) fn published_in_flight(&self) -> bool { !self.quarantined && self.active.iter().any(|active| active.outer_published) @@ -682,6 +705,7 @@ impl Pair { context_update: self.context_update, failed: false, drained_result: None, + abandoned_since: None, }) .map_err(|_| EBUSY)?; self.prepared = None; @@ -904,6 +928,9 @@ impl Pair { .as_ref() .is_some_and(|prepared| Arc::ptr_eq(&prepared.packet, packet)) } + pub(crate) fn replay_contains(&self, packet: &Arc) -> bool { + self.replay.as_ref().is_some_and(|replay| Arc::ptr_eq(replay, packet)) + } pub(crate) fn take_deferred_timeout(&self) -> Option<(Arc, Error)> { self.active.iter().find_map(|active| { active @@ -1079,7 +1106,8 @@ impl Pair { }; let timestamps = match result { Ok(retirement::Poll::Complete(timestamps)) => timestamps, - Ok(retirement::Poll::Retry | retirement::Poll::Pending) => return Ok(false), + Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer), + Ok(retirement::Poll::Retry) => return Ok(false), Err(error) => { self.quarantine(error, defer)?; return Ok(false); @@ -1109,6 +1137,105 @@ impl Pair { } } + /// The front group was retired and unlinked by firmware without executing + /// (job list empty, stamps cover it, no recovery, fragment end never + /// written). Nothing of this group reached its attachments, so the same + /// command is published again instead of failing the owner's VM. The + /// proof is the strict timeout witness, taken twice ABANDON_GRACE_NS apart, + /// and only for a lone group whose completion still accepts a replay. + fn replay_abandoned( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + use retirement::Memory; + if self.quarantined || self.active.len() != 1 || self.replay.is_some() { + return Ok(false); + } + let front = self.active.front().ok_or(EIO)?; + if !front.outer_published || front.failed { + return Ok(false); + } + let mut observer = Observer { + memory: &self.memory, + qids: self.qids(), + host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(front.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + let abandoned = + retirement::abandoned(&front.ticket, snapshot, stamps, state, false)?; + let now = crate::g17::now_ns(); + let front = self.active.front_mut().ok_or(EIO)?; + if !abandoned { + front.abandoned_since = None; + return Ok(false); + } + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(false); + }; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(false); + } + if crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { + pr_info!( + "render slot {} retired without execution (pass {:?}); replaying\n", + self.slot, + snapshot.pass + ); + } + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + let front = self.active.front_mut().ok_or(EIO)?; + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + let ordinal = front.ticket.ordinal; + let packet = front.packet.clone(); + if !packet.completion.begin_replay() { + if !packet.completion.spared() { + packet.completion.status().record(EIO); + if self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), EIO)); + } + } + defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + return Ok(true); + } + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + packet.reset_publication_for_replay(); + self.replay = Some(packet); + Ok(true) + } + + /// The pending replay cannot proceed (closed owner, cancellation or a + /// publication refusal). ECANCELED leaves the VM usable; any other error + /// is the ordinary device-loss outcome the replay tried to avoid. + pub(crate) fn fail_replay( + &mut self, + error: Error, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + let Some(packet) = self.replay.take() else { + return Ok(()); + }; + defer(Deferred::Retired(packet.completion.clone(), Err(error))) + } + /// A closed preparation epoch invalidates only this unpublished descriptor. /// Its completion remains available for a retry under the next epoch. pub(crate) fn discard_prepared(&mut self, packet: &Arc) -> Result { @@ -1292,6 +1419,9 @@ impl Pair { self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; } + if let Some(packet) = self.replay.take() { + defer(Deferred::Drained(packet.completion.clone(), Err(EIO)))?; + } if let Some(previous) = self.previous.as_mut() { previous.ready = true; } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index f712bb7b1b87d..39941de815e07 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -213,6 +213,7 @@ impl ParkedPair { terminal: None, previous: self.previous.take(), displaced: None, + replay: None, context_update: true, })) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index a7ff3e4f0ecc3..3ae1d0a609a33 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -368,6 +368,7 @@ impl crate::g17::Shared { // Replay only at this outer boundary, after recovery has settled. // Publication may service events but must never recurse into replay. firmware.replay_compute(deferred)?; + firmware.replay_render(deferred)?; drop(state); self.observe_renders(masks, deferred)?; let mut state = self.state.lock(); @@ -409,6 +410,7 @@ impl crate::g17::Shared { let backpressured = firmware.render_control_backpressured(); let polling = backpressured || firmware.queues.replay_outer_wait.is_some() + || firmware.queues.render.replay_polling() || firmware.queues.teardown.polling(now) || firmware.compute_exit_polling(now) || firmware diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 460889be64e2b..a47abf2045526 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -11,6 +11,7 @@ mod observe; mod pools; mod prepare; mod recovery; +mod replay; mod settle; mod submit; pub(super) mod teardown; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs new file mode 100644 index 0000000000000..ce9771778acd4 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Republication of render groups that firmware retired without executing. +//! The pair proves the abandonment (retired, unlinked, stamps covering it, no +//! recovery, fragment end never written) and parks the packet; this pass +//! prepares and publishes the same command again under the device mutex. + +use super::super::DeferredBatch; +use crate::g17::{object::Allocator, queue::render::RenderAddresses, Firmware}; +use kernel::prelude::*; + +impl super::RenderState { + pub(in crate::g17) fn replay_polling(&self) -> bool { + self.entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .any(|pair| pair.replay_polling()) + } +} + +impl Firmware { + pub(in crate::g17) fn replay_render(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&self.init)? != 0 + { + return Ok(()); + } + for slot in 0..self.queues.render.entries.len() { + let Some((packet, owner, qids, closed, busy)) = + self.queues.render.entries[slot].as_ref().and_then(|entry| { + let pair = entry.pair.as_ref()?; + let packet = pair.replay_front()?; + Some(( + packet, + entry.reservation.owner, + pair.qids(), + entry.closed, + entry.preparing.is_some() + || entry.growing + || entry.pending_timeout + || entry.pending_error.is_some(), + )) + }) + else { + continue; + }; + if busy { + continue; + } + if closed || packet.cancel_requested() || !packet.context.is_current() { + self.with_render(slot, |pair, _| { + pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) + })?; + continue; + } + if packet.completion.status().get() != 0 { + self.with_render(slot, |pair, _| { + pair.fail_replay(EIO, &mut |event| deferred.push(event)) + })?; + continue; + } + let addresses = RenderAddresses { + stamps: [ + self.init.stamp_addresses(qids[0])?, + self.init.stamp_addresses(qids[1])?, + ], + }; + let dev = self.primary.state.shared.drm()?; + let uat = self.uat.clone(); + let prepared = self.with_render(slot, |pair, _| { + if !pair.room() { + return Err(EBUSY); + } + let alloc = Allocator { + dev: &dev, + uat: &uat, + }; + pair.prepare(&alloc, &packet, addresses) + }); + match prepared { + Ok(()) => {} + Err(error) if error == EBUSY || error == EAGAIN => continue, + Err(error) => { + self.with_render(slot, |pair, _| { + pair.fail_replay(error, &mut |event| deferred.push(event)) + })?; + continue; + } + } + match self.publish_render(owner, slot as u8, &packet, deferred) { + Ok(()) => { + self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; + } + Err(error) if error == EBUSY || error == EAGAIN => continue, + Err(error) => { + self.with_render(slot, |pair, _| { + pair.fail_replay(error, &mut |event| deferred.push(event)) + })?; + } + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 415d1f391f601..2e88848b93216 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -70,7 +70,7 @@ impl crate::g17::Firmware { .and_then(Option::as_ref) .and_then(|entry| entry.pair.as_ref()) .ok_or(EIO)?; - if !pair.contains_packet(packet) { + if pair.replay_contains(packet) || !pair.contains_packet(packet) { return Ok(()); } let generation = self.recovery.generation(); From 1678525100652694b0aaee977ea2012b487d6a51 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 21:40:42 -0400 Subject: [PATCH 0975/1579] drm/asahi: g17: never kill compute commands with ray traversal scratch A closed client's quarantined or cancelled compute queue is ended with a firmware context kill once its owner is gone. For commands that use ray traversal scratch this goes wrong on Neo (T8140) when the traversal stopped making progress: - releasing the killed queue's pool right after the kill was acknowledged crashed the GPU firmware about 40 ms later (3 of 3 cases), taking the whole GPU and the desktop down; - keeping the pool instead avoided the crash, but about 12 s later every following render on the device timed out and the compositor lost its context. Leaving such a queue alone, as a diagnostic build that skipped the kill did on five boots, kept the rest of the GPU working each time. Do not issue the kill while any active command of the queue has traversal scratch enabled. The client still sees its device lost through the existing timeout and quarantine path, the queue stays retained until its commands retire, and every other compute queue keeps the existing kill and recycle behaviour. Tested on Neo with this exact change: a ray tracing dispatch that stopped making progress timed out at 69.7 s and was left unkilled; the compositor kept rendering for the following 40 s, and new compute, ray tracing pipeline and amber conformance work ran normally afterwards. A raygen shader looping forever was also left unkilled; later ray tracing dispatches still completed in 20 ms each and the desktop stayed live. Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/g17/queue/compute/exit.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 877fc935dc909..187fd27f8e78e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -83,11 +83,25 @@ impl Transaction { } impl Queue { + /// A command with ray traversal scratch is never killed. On Neo, killing a + /// traversal that stopped making progress either crashed the GPU firmware + /// when the queue's pool was then released, or stalled every later render + /// when it was retained; left alone, the rest of the GPU kept working. Such + /// a queue stays quarantined and retained until its commands retire. + fn active_traversal(&self) -> bool { + self.active.iter().any(|active| { + matches!( + &active.packet.command, + crate::g17::command::Validated::Compute { scratch, .. } if scratch.enabled() + ) + }) + } pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) && self.released && self.active.len != 0 && self.exit.is_none() + && !self.active_traversal() && (self.quarantined || self.retired_by_teardown) && age >= if self.quarantined { From 48b2631fdaab66dc7bd850b3006c0f85a73aba0e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 20:46:34 -0400 Subject: [PATCH 0976/1579] drm/asahi: g17: ray queries in render stages Let render commands carry ray query state for their vertex (tiling) and fragment stages, as compute commands already do. UAPI: - DRM_ASAHI_FEATURE_RENDER_RAY_QUERY advertises the feature. - struct drm_asahi_render_ray_query_stage describes one stage: its ray query scratch layout and intersection tables. drm_asahi_cmd_render gains ray_query[2] at its end; shorter commands from older userspace are read with both stages zeroed. - drm_asahi_queue_create gains render_scs_layout[2], the largest layouts the queue will submit, which sizes the queue's render pool. Zero keeps the existing pool size. Firmware programming: for each stage with ray query scratch, the tiling or fragment register program carries the scratch size, tail, stack and window words and the intersection table registers, and enables SCS with register 0x017d9; its kick sets the SCS-enabled header bit. Stages without ray queries keep the existing register programs unchanged. Validation rejects intersection pools and default programs outside the VM and a scratch layout whose enable state disagrees with the intersection tables. The render pool is sized from both stages' scratch demand at queue creation, and a command needing more than was reserved is rejected. Tested on Neo (A18 Pro) with the matching Mesa series: ray_query builtin.flow and advanced.null_as cases 41 of 48 pass across vertex, tessellation, geometry, fragment, compute and ray tracing stages (the failures are tessellation evaluation and three null acceleration structure cases); position fetch 60 of 60 runnable cases; earlier builds of the same render path passed 883 ray query stage cases. Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/file.rs | 4 +- drivers/gpu/drm/asahi/g17/command.rs | 23 ++++- drivers/gpu/drm/asahi/g17/context.rs | 14 ++- drivers/gpu/drm/asahi/g17/freelist.rs | 19 ++-- drivers/gpu/drm/asahi/g17/fw/compute.rs | 8 +- drivers/gpu/drm/asahi/g17/fw/compute_pool.rs | 9 +- drivers/gpu/drm/asahi/g17/fw/kick.rs | 8 +- drivers/gpu/drm/asahi/g17/fw/mod.rs | 1 + drivers/gpu/drm/asahi/g17/fw/render.rs | 92 +++++++++++++------- drivers/gpu/drm/asahi/g17/fw/render_rt.rs | 48 ++++++++++ drivers/gpu/drm/asahi/g17/job.rs | 4 + drivers/gpu/drm/asahi/g17/mod.rs | 3 +- drivers/gpu/drm/asahi/g17/queue/compute.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 11 +++ drivers/gpu/drm/asahi/g17/queue/render.rs | 13 ++- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 4 + drivers/gpu/drm/asahi/queue/mod.rs | 5 ++ include/uapi/drm/asahi_drm.h | 55 ++++++++++++ 18 files changed, 262 insertions(+), 61 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/fw/render_rt.rs diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 8286755567f90..598d1cdce4c7a 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -1156,7 +1156,7 @@ impl File { let _ = file_vm; core::mem::drop(guard); - let queue = device.gpu.new_queue( + let mut queue = device.gpu.new_queue( vm, ualloc, ualloc_priv, @@ -1165,6 +1165,8 @@ impl File { data.usc_exec_base, )?; + queue.reserve_render_scs(data.render_scs_layout)?; + data.queue_id = resv.index().try_into()?; resv.fill(Arc::pin_init(new_mutex!(queue), GFP_KERNEL)?)?; diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index dd3be1a81e1d8..671e513852c96 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -6,6 +6,7 @@ use super::fw::{ compute::{IntersectionTables, ScratchRequest}, kick::WriteRange, render::{RenderPass, UscProgram, ZlsSurface}, + render_rt::RayQueryStage, }; use crate::util::{AnyBitPattern, Reader}; use core::ops::Range; @@ -14,6 +15,7 @@ use kernel::{prelude::*, uapi}; const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; const USC_WINDOW_SIZE: u64 = 1 << 32; const MAX_SAMPLERS: u32 = 1024; +const RENDER_BASE_SIZE: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_render, prior_cdm) + size_of::(); const COMPUTE_BASE_SIZE: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_compute, prior_cdm) + size_of::(); const SAMPLER_SIZE: u64 = 8; const PAGE_SIZE: u64 = 0x4000; @@ -78,7 +80,7 @@ impl Attachments { // Commands stay inline in the submission plan, avoiding an allocation per render pass. #[allow(clippy::large_enum_variant)] pub(crate) enum Payload { - Render(uapi::drm_asahi_cmd_render, [Attachments; 2]), + Render(uapi::drm_asahi_cmd_render, [Attachments; 2], [RayQueryStage; 2]), Compute(uapi::drm_asahi_cmd_compute, Attachments, IntersectionTables), } @@ -128,7 +130,11 @@ impl<'a> Parser<'a> { == uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER { self.counts[0] += 1; - Payload::Render(payload(bytes)?, [self.attachments[0], self.attachments[1]]) + Payload::Render( + payload(&bytes[..bytes.len().min(RENDER_BASE_SIZE)])?, + [self.attachments[0], self.attachments[1]], + RayQueryStage::parse_pair(bytes.get(RENDER_BASE_SIZE..).unwrap_or(&[]))?, + ) } else { self.counts[1] += 1; Payload::Compute( @@ -284,6 +290,7 @@ fn surface( pub(crate) enum Validated { Render { pass: RenderPass, + ray_query: [RayQueryStage; 2], attachments: [Attachments; 2], timestamps: [uapi::drm_asahi_timestamps; 2], }, @@ -302,7 +309,7 @@ pub(crate) enum Validated { impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { - Self::Render(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Render(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], Self::Compute(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], } } @@ -313,7 +320,7 @@ impl Payload { space: &impl AddressSpace, ) -> Result { match self { - Self::Render(cmd, attachments) => { + Self::Render(cmd, attachments, ray_query) => { if cmd.flags & !RENDER_FLAGS != 0 || !(1..=MAX_DIMENSION).contains(&cmd.width_px) || !(1..=MAX_DIMENSION).contains(&cmd.height_px) @@ -415,8 +422,16 @@ impl Payload { | (u64::from(cmd.bg_eot_partial_rsrc_spec_hi) << 32), rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, }; + for stage in ray_query { + if stage.intersection.enabled() { + require_range(space, stage.intersection.pool, stage.intersection.pool_bytes, Access::Read)?; + let code = stage.intersection.default_program & 0x0000_ffff_ffff_ffff; + if code != 0 { require_range(space, code, 64, Access::Read)?; } + } + } Ok(Validated::Render { pass, + ray_query: *ray_query, attachments: *attachments, timestamps: [cmd.ts_vtx, cmd.ts_frag], }) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index afd0d1d22f5bb..6bc62a7d44bee 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -14,7 +14,7 @@ use super::{ status::VmStatus, }; use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; -use core::sync::atomic::{fence, AtomicBool, AtomicU32, Ordering}; +use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicUsize, Ordering}; use identity::{RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; use kernel::{ new_mutex, @@ -120,6 +120,7 @@ pub(crate) struct Context { work: Arc>, work_base: u64, render: Arc>>>, + render_blocks: AtomicUsize, /// Fixed for the context's life. Physical queues bound to it carry this /// profile in every record, ring, doorbell and registration. policy: Policy, @@ -192,6 +193,7 @@ impl Context { work, work_base, render, + render_blocks: AtomicUsize::new(super::fw::freelist::RENDER_BLOCKS), policy, }) } @@ -277,6 +279,14 @@ impl Context { } Ok(result) } + pub(crate) fn reserve_render_blocks(&self, blocks: usize) -> Result { + if self.is_published() || self.render.lock().is_some() { return Err(EBUSY); } + self.render_blocks.store(blocks, Ordering::Release); + Ok(()) + } + pub(crate) fn render_blocks(&self) -> usize { + self.render_blocks.load(Ordering::Acquire) + } pub(crate) fn render_pool( &self, alloc: &Allocator<'_>, @@ -293,7 +303,7 @@ impl Context { return Ok(pool.clone()); } let pool = Arc::new( - RenderPool::new(alloc, ids, render_global, self.vm())?, + RenderPool::new(alloc, ids, render_global, self.vm(), self.render_blocks())?, GFP_KERNEL, )?; *render = Some(pool.clone()); diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 2612a9302ce28..c9c9ebda7f6bc 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -139,8 +139,12 @@ impl RenderBacking { alloc: &Allocator<'_>, render_global: &mmu::Vm, client: &mmu::Vm, + blocks: usize, ) -> Result { - let pages = Pages::new(alloc, RENDER_BLOCKS, RunLayout::Render)?; + if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { + return Err(EINVAL); + } + let pages = Pages::new(alloc, blocks, RunLayout::Render)?; let address = pages.page_list_va(); let global = pages .backing @@ -161,6 +165,7 @@ impl RenderBacking { /// consumed and the client VM is evicted; a fresh backing is installed before /// the pool is staged again. pub(crate) struct RenderPool { + blocks: usize, backing: Arc>>, /// Page-list VA of the installed backing; zero while vacant. page_list: AtomicU64, @@ -182,9 +187,10 @@ impl RenderPool { ids: &Arc, render_global: &mmu::Vm, client: &mmu::Vm, + blocks: usize, ) -> Result { let id = RenderId::new(ids)?; - let backing = RenderBacking::new(alloc, render_global, client)?; + let backing = RenderBacking::new(alloc, render_global, client, blocks)?; let address = backing.pages.page_list_va(); let run_list_va = backing.pages.run_list_va(); let state = alloc.kernel( @@ -207,7 +213,7 @@ impl RenderPool { class: FreeListClass::Two, page_list_va: address, run_list_va, - blocks: RENDER_BLOCKS as u32, + blocks: blocks as u32, state_va: state.gpu_va(), }, 0, @@ -222,6 +228,7 @@ impl RenderPool { GFP_KERNEL, )?; Ok(Self { + blocks, backing, page_list: AtomicU64::new(address), epoch: AtomicU64::new(0), @@ -279,7 +286,7 @@ impl RenderPool { if !self.vacant() { return Ok(()); } - self.rearm(RenderBacking::new(alloc, render_global, client)?) + self.rearm(RenderBacking::new(alloc, render_global, client, self.blocks)?) } pub(crate) fn serial(&self) -> u64 { @@ -303,7 +310,7 @@ impl RenderPool { } pub(crate) fn end_va(&self) -> u64 { self.run_list_va() - + (RENDER_BLOCKS * size_of::()) as u64 + + (self.blocks * size_of::()) as u64 } /// Fresh pools are already populated. Reuse requires the previous release @@ -319,7 +326,7 @@ impl RenderPool { class: FreeListClass::Two, page_list_va: self.page_list_va(), run_list_va: self.run_list_va(), - blocks: RENDER_BLOCKS as u32, + blocks: self.blocks as u32, state_va: self.state.gpu_va(), }, submitted, diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index a4775abc6b7eb..0633da7f43541 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -112,16 +112,16 @@ impl ScratchRequest { pub(crate) fn pages(self) -> u16 { self.pages } - fn size_word(self) -> u64 { + pub(crate) fn size_word(self) -> u64 { let pages = u64::from(self.pages); pages | (pages.div_ceil(Self::PAGES_PER_MIB) << 16) } - fn window_word(self) -> u64 { + pub(crate) fn window_word(self) -> u64 { self.window } - fn stack_word(self) -> u64 { + pub(crate) fn stack_word(self) -> u64 { self.stack } @@ -129,7 +129,7 @@ impl ScratchRequest { self.driver } - fn tail_word(self) -> u64 { + pub(crate) fn tail_word(self) -> u64 { self.tail } diff --git a/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs b/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs index 4005daa4e9832..0f0614f6155a1 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs @@ -5,9 +5,9 @@ use kernel::prelude::*; use super::initdata::{FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; use super::freelist::{COMPUTE_BLOCKS, PAGE_LIST_SIZE}; -pub(crate) fn required_blocks(active_cores: u32, scs_pages: u16) -> Result { +pub(crate) fn extra_bytes(active_cores: u32, scs_pages: u16) -> Result { if active_cores == 0 { return Err(EINVAL); } - if scs_pages == 0 { return Ok(COMPUTE_BLOCKS); } + if scs_pages == 0 { return Ok(0); } // Register0x101d1 counts 4-KiB pages and their 1-MiB large-page inventory. // Register0x0d479 reserves eight further pages for each enabled USC. let small = u64::from(scs_pages); @@ -15,6 +15,11 @@ pub(crate) fn required_blocks(active_cores: u32, scs_pages: u16) -> Result Result { + let extra = extra_bytes(active_cores, scs_pages)?; let bytes = extra.checked_add((COMPUTE_BLOCKS * FREE_LIST_BLOCK_SIZE) as u64) .ok_or(EOVERFLOW)?; let blocks = usize::try_from(bytes.div_ceil(FREE_LIST_BLOCK_SIZE as u64)) diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index 47cdce3ca09e5..e07fbfb485a10 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -370,7 +370,7 @@ pub(crate) struct KickArgs<'a> { /// Register arrays bound to the kick. pub(crate) register_arrays: [Option; 4], /// The compute register program includes a shader-context scratch request. - pub(crate) compute_scratch: bool, + pub(crate) scs_enabled: bool, /// Priority class of the queue. pub(crate) priority: u8, } @@ -423,7 +423,7 @@ impl KickEntry { const QID_SHIFT: u32 = 42; const MCACHE_PRESENT: u64 = 1 << 58; const PRIORITY_SHIFT: u32 = 59; - const COMPUTE_SCRATCH: u64 = 1 << 61; + const SCS_ENABLED: u64 = 1 << 61; const QOS_SLOT_SHIFT: u32 = 32; const QOS_CLASS_SHIFT: u32 = 40; const QOS_CLASS_MAX: u8 = 0x1f; @@ -446,8 +446,8 @@ impl KickEntry { let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT | (args.qid as u64) << Self::QID_SHIFT | (args.priority as u64) << Self::PRIORITY_SHIFT; - if args.compute_scratch { - header |= Self::COMPUTE_SCRATCH; + if args.scs_enabled { + header |= Self::SCS_ENABLED; } let mcache = match args.mcache { Some(table) => { diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index 479bc91d21ca5..b6bb6aaea860f 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -21,6 +21,7 @@ pub(crate) mod initdata; pub(crate) mod kick; pub(crate) mod queue; pub(crate) mod render; +pub(crate) mod render_rt; pub(crate) mod qos; /// Clears a firmware structure in place. diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 23eaa7e7dfba9..85a2385b421b8 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -23,7 +23,7 @@ use super::kick::{ use super::queue::CommandTag; use super::{ clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, - UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, + UNK_0D411, UNK_101D9, UNK_1C8F8, }; /// Size of a tiling descriptor. @@ -54,7 +54,11 @@ const DEFLAKE_20: u64 = 0x20; const DEFLAKE_2A0: u64 = 0x2a0; const TA_REGISTERS: usize = 73; +const TA_REGISTERS_MAX: usize = TA_REGISTERS + 1; +const TA_SCS_ENABLE_INDEX: usize = 1; const FRAGMENT_REGISTERS: usize = 89; +const FRAGMENT_REGISTERS_MAX: usize = FRAGMENT_REGISTERS + 1; +const FRAGMENT_SCS_ENABLE_INDEX: usize = 1; const PARTIAL_STORE_REGISTERS: usize = 16; const PARTIAL_RESUME_REGISTERS: usize = 23; const PARTIAL_LOAD_REGISTERS: usize = 10; @@ -122,8 +126,6 @@ const UNK_15359_PARTIAL: u64 = 0x20; /// Required value of register 0x15049 and of fragment descriptor field /// +0x1f50. const UNK_15049: u64 = 0x10_0000; -/// Value of the fragment register 0x10791. -const UNK_10791_FRAGMENT: u64 = UNK_10791 | 1 << 9; /// USC program offsets carry tag bits that are not part of the address. const USC_OFFSET_TAG_MASK: u32 = 7; @@ -303,6 +305,7 @@ pub(crate) struct RenderStage { pub(crate) struct RenderArgs<'a> { /// Render-pass state. pub(crate) pass: &'a RenderPass, + pub(crate) ray_query: [super::render_rt::RayQueryStage; 2], /// Kernel-owned objects. pub(crate) resources: &'a RenderResources, /// Submission ordinal of the render queue pair, from zero. @@ -481,7 +484,7 @@ fn ta_registers( args: &RenderArgs<'_>, g: &Geometry, w: &PassWords, -) -> Result<[RegisterWrite; TA_REGISTERS]> { +) -> Result<[RegisterWrite; TA_REGISTERS_MAX]> { let p = args.pass; let r = args.resources; let tilemap = user_offset(r.tilemap_va)?; @@ -497,9 +500,12 @@ fn ta_registers( } let key = work_key(r.context_id, args.ta.state_word); let ids = r.ta_ids.word(); + let rt = args.ray_query[0]; + let ift = rt.intersection.words(); Ok([ RegisterWrite::new(0x01748, 1), + RegisterWrite::new(0x017d9, 1), RegisterWrite::new(0x10141, 0x200), RegisterWrite::new(0x1c039, tilemap), RegisterWrite::new(0x1c9c8, tilemap), @@ -545,16 +551,16 @@ fn ta_registers( RegisterWrite::new(0x1c898, 0), RegisterWrite::new(0x101e1, UNK_101D9), RegisterWrite::new(0x1c9e8, w.layer_mode & LAYER_MODE_PIPE_MASK), - RegisterWrite::new(0x1a099, 0), - RegisterWrite::new(0x1a0a1, 0), - RegisterWrite::new(0x1a069, 0), - RegisterWrite::new(0x1a071, 0), - RegisterWrite::new(0x1a0c9, 0), - RegisterWrite::new(0x1a0d1, 0), - RegisterWrite::new(0x101c9, 0), - RegisterWrite::new(0x0d471, 0), - RegisterWrite::new(0x1a0f1, UNK_1A0E9), - RegisterWrite::new(0x10799, UNK_10791), + RegisterWrite::new(0x1a099, ift[0]), + RegisterWrite::new(0x1a0a1, ift[1]), + RegisterWrite::new(0x1a069, ift[2]), + RegisterWrite::new(0x1a071, ift[3]), + RegisterWrite::new(0x1a0c9, ift[4]), + RegisterWrite::new(0x1a0d1, ift[5]), + RegisterWrite::new(0x101c9, rt.scratch.size_word()), + RegisterWrite::new(0x0d471, rt.scratch.tail_word()), + RegisterWrite::new(0x1a0f1, rt.scratch.stack_word()), + RegisterWrite::new(0x10799, rt.scratch.window_word()), RegisterWrite::new(0x1c830, r.pb_slot as u64), RegisterWrite::new(0x1ca30, r.pm_scratch), RegisterWrite::new(0x16c39, r.pm_scratch), @@ -579,7 +585,7 @@ fn fragment_registers( args: &RenderArgs<'_>, g: &Geometry, w: &PassWords, -) -> [RegisterWrite; FRAGMENT_REGISTERS] { +) -> [RegisterWrite; FRAGMENT_REGISTERS_MAX] { let p = args.pass; let r = args.resources; let mut tile_state = TILE_STATE_BASE @@ -591,9 +597,12 @@ fn fragment_registers( } let key = work_key(r.context_id, args.fragment.state_word); let ids = r.fragment_ids.word(); + let rt = args.ray_query[1]; + let ift = rt.intersection.words(); [ RegisterWrite::new(0x01739, 1), + RegisterWrite::new(0x017d9, 1), RegisterWrite::new(0x10009, w.utile_config), // End-of-tile resource specifier. RegisterWrite::new(0x15379, w.eot_rsrc_spec), @@ -611,7 +620,7 @@ fn fragment_registers( RegisterWrite::new(0x16030, g.macro_size), RegisterWrite::new(0x100d9, g.screen), RegisterWrite::new(0x0a301, 0), - RegisterWrite::new(0x10791, UNK_10791_FRAGMENT), + RegisterWrite::new(0x10791, rt.scratch.window_word() | 1 << 9), RegisterWrite::new(0x16098, r.heapmeta_va), RegisterWrite::new(0x15109, p.scissor_va), RegisterWrite::new(0x15101, p.depth_bias_va), @@ -659,15 +668,15 @@ fn fragment_registers( RegisterWrite::new(0x101e9, UNK_101D9), RegisterWrite::new(0x160a8, 0), RegisterWrite::new(0x16068, tile_state), - RegisterWrite::new(0x1a0a9, 0), - RegisterWrite::new(0x1a0b1, 0), - RegisterWrite::new(0x1a079, 0), - RegisterWrite::new(0x1a081, 0), - RegisterWrite::new(0x1a0d9, 0), - RegisterWrite::new(0x1a0e1, 0), - RegisterWrite::new(0x101c1, 0), - RegisterWrite::new(0x0d469, 0), - RegisterWrite::new(0x1a0f9, UNK_1A0E9), + RegisterWrite::new(0x1a0a9, ift[0]), + RegisterWrite::new(0x1a0b1, ift[1]), + RegisterWrite::new(0x1a079, ift[2]), + RegisterWrite::new(0x1a081, ift[3]), + RegisterWrite::new(0x1a0d9, ift[4]), + RegisterWrite::new(0x1a0e1, ift[5]), + RegisterWrite::new(0x101c1, rt.scratch.size_word()), + RegisterWrite::new(0x0d469, rt.scratch.tail_word()), + RegisterWrite::new(0x1a0f9, rt.scratch.stack_word()), RegisterWrite::new(0x0a5a9, UNK_0A599), RegisterWrite::new(0x0d429, UNK_0D411), RegisterWrite::new(0x160e0, ids), @@ -907,9 +916,16 @@ impl TaDescriptor { self.pb_va = resources.pb_va; self.scene_va = resources.scene_va; self.scene_end_va = resources.scene_end_va; - self.checkpoints = TA_CHECKPOINTS; + let scs_register = args.ray_query[0].scratch.enabled(); + self.checkpoints = TA_CHECKPOINTS.map(|count| count + u16::from(scs_register)); self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); - self.registers.set(array_va, ®isters)?; + if scs_register { + self.registers.set(array_va, ®isters)?; + } else { + let mut trimmed = [registers[0]; TA_REGISTERS]; + trimmed[TA_SCS_ENABLE_INDEX..].copy_from_slice(®isters[TA_SCS_ENABLE_INDEX + 1..]); + self.registers.set(array_va, &trimmed)?; + } self.tpc_va = resources.tpc_va; self.tpc_size = resources.tpc_size; self.fragment_qid = args.fragment.qid.into(); @@ -963,10 +979,11 @@ impl TaDescriptor { /// descriptor's address as seen by the pass's VM. pub(crate) fn register_bindings( descriptor_va: u64, + scs_register: bool, ) -> Result<[Option; 4]> { let array = RegisterArrayBinding::new( offset_va(descriptor_va, Self::ARRAY_OFFSET)?, - TA_CHECKPOINTS[0] as u8, + TA_CHECKPOINTS[0] as u8 + u8::from(scs_register), )?; Ok([Some(array), None, None, None]) } @@ -1153,13 +1170,14 @@ impl FragmentDescriptor { /// the descriptor's address as seen by the pass's VM. pub(crate) fn register_bindings( descriptor_va: u64, + scs_register: bool, ) -> Result<[Option; 4]> { let array = |index, checkpoint: usize| -> Result> { let va = offset_va(descriptor_va, Self::array_offset(index))?; Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) }; Ok([ - array(0, FRAGMENT_CHECKPOINTS[0] as usize)?, + array(0, FRAGMENT_CHECKPOINTS[0] as usize + usize::from(scs_register))?, array(1, PARTIAL_STORE_REGISTERS)?, array(2, PARTIAL_RESUME_REGISTERS)?, array(3, PARTIAL_LOAD_REGISTERS)?, @@ -1226,9 +1244,19 @@ impl FragmentSlot { desc.merge_upper_x = pass.merge_upper_x; desc.merge_upper_y = pass.merge_upper_y; desc.tiles = geometry.tiles_x * geometry.tiles_y; - desc.checkpoints = FRAGMENT_CHECKPOINTS; - desc.entries = FRAGMENT_REGISTERS as u16; - desc.arrays[0].set(arrays[0], &fragment_registers(args, &geometry, &words))?; + let scs_register = args.ray_query[1].scratch.enabled(); + let extra = u16::from(scs_register); + desc.checkpoints = FRAGMENT_CHECKPOINTS.map(|count| count + extra); + desc.entries = FRAGMENT_REGISTERS as u16 + extra; + let registers = fragment_registers(args, &geometry, &words); + if scs_register { + desc.arrays[0].set(arrays[0], ®isters)?; + } else { + let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; + trimmed[FRAGMENT_SCS_ENABLE_INDEX..] + .copy_from_slice(®isters[FRAGMENT_SCS_ENABLE_INDEX + 1..]); + desc.arrays[0].set(arrays[0], &trimmed)?; + } desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; desc.arrays[3].set(arrays[3], &partial_load_registers(pass, &words))?; diff --git a/drivers/gpu/drm/asahi/g17/fw/render_rt.rs b/drivers/gpu/drm/asahi/g17/fw/render_rt.rs new file mode 100644 index 0000000000000..5ae23d84400eb --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/render_rt.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +use super::{compute::{IntersectionTables, ScratchRequest}, compute_pool, + freelist::{PAGE_LIST_SIZE, RENDER_BLOCKS}, + initdata::{FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}}; +use kernel::prelude::*; + +#[derive(Clone, Copy)] +pub(crate) struct RayQueryStage { + pub(crate) intersection: IntersectionTables, + pub(crate) scratch: ScratchRequest, +} + +impl RayQueryStage { + fn parse(bytes: &[u8]) -> Result { + let mut wire = [0u8; 56]; + wire[..bytes.len()].copy_from_slice(bytes); + let layout = u64::from_le_bytes(wire[..8].try_into().unwrap()); + let intersection = IntersectionTables::parse(&wire[8..])?; + let scratch = ScratchRequest::new(layout)?; + if scratch.enabled() != intersection.enabled() { return Err(EINVAL); } + Ok(Self { intersection, scratch }) + } + + pub(crate) fn parse_pair(bytes: &[u8]) -> Result<[Self; 2]> { + if bytes.get(112..).is_some_and(|tail| tail.iter().any(|&b| b != 0)) { + return Err(EINVAL); + } + let mut wire = [0u8; 112]; + let count = bytes.len().min(wire.len()); + wire[..count].copy_from_slice(&bytes[..count]); + Ok([Self::parse(&wire[..56])?, Self::parse(&wire[56..])?]) + } +} + +pub(crate) fn required_blocks(active_cores: u32, pages: [u16; 2]) -> Result { + let extra = compute_pool::extra_bytes(active_cores, pages[0])? + .checked_add(compute_pool::extra_bytes(active_cores, pages[1])?) + .and_then(|bytes| bytes.checked_mul(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u64)) + .ok_or(EOVERFLOW)?; + let bytes = extra.checked_add((RENDER_BLOCKS * FREE_LIST_BLOCK_SIZE) as u64) + .ok_or(EOVERFLOW)?; + let blocks = usize::try_from(bytes.div_ceil(FREE_LIST_BLOCK_SIZE as u64)) + .map_err(|_| EOVERFLOW)?; + let maximum = PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8); + if blocks > maximum { return Err(E2BIG); } + Ok(blocks) +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 9ad0370edfa2f..9293c83c55b61 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -244,6 +244,10 @@ pub(crate) trait Backend: Send + Sync { /// Lazily create or rebind this logical queue's physical compute graph. fn ensure_compute(&self, scs_pages: u16) -> Result; + fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { + if pages != [0; 2] { return Err(EINVAL); } + Ok(super::fw::freelist::RENDER_BLOCKS) + } /// Settle outstanding work of a terminal VM before an ioctl error is published. fn fail_vm(&self, error: Error); /// Called after both scheduler entities have cancelled their queued jobs. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 3cfa2c1d71655..0ffa568d6d4b3 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1102,7 +1102,8 @@ impl gpu::Gpu for Gpu { uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64; + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index f3f050d482bef..b78598cfbb760 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1678,7 +1678,7 @@ impl Queue { mcache: None, event_mask: COMPUTE_KICK_EVENT_MASK, register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch)?, - compute_scratch: scratch.enabled(), + scs_enabled: scratch.enabled(), priority: policy.priority(), }; let published = host.next_compute_publication()?; diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 31f2fcd6d016a..326405f05fdf9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -171,6 +171,10 @@ impl Queue { Validated::Compute { .. } => Engine::Compute, }; let index = engine.index(); + if let Validated::Render { ray_query, .. } = &command { + let blocks = self.backend.render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; + if blocks > self.context.render_blocks() { return Err(EINVAL); } + } self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { @@ -354,6 +358,13 @@ impl Queue { } impl crate::queue::Queue for Queue { + fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { + let pages = [super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), + super::fw::compute::ScratchRequest::new(layouts[1])?.pages()]; + let blocks = self.backend.render_scs_blocks(pages)?; + self.context.reserve_render_blocks(blocks) + } + fn submit( &mut self, id: u64, diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index bbf85695970a0..99be455b48e40 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -321,7 +321,7 @@ impl Pair { return Err(EBUSY); } let Validated::Render { - pass, attachments, .. + pass, attachments, ray_query, .. } = &packet.command else { return Err(EINVAL); @@ -385,6 +385,7 @@ impl Pair { let qids = self.qids(); let args = abi::RenderArgs { pass, + ray_query: *ray_query, resources: &resources, ordinal, ta: abi::RenderStage { @@ -585,7 +586,7 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - let Validated::Render { pass, .. } = &packet.command else { + let Validated::Render { pass, ray_query, .. } = &packet.command else { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; @@ -609,13 +610,16 @@ impl Pair { mcache, event_mask: abi::RENDER_KICK_EVENT_MASK, register_arrays: arrays, - compute_scratch: false, + scs_enabled: ray_query[stage].scratch.enabled(), priority: policy.priority(), }; let ta_args = args( 0, tiling_barriers, - abi::TaDescriptor::register_bindings(self.memory.graph.descriptor_client(0, ordinal))?, + abi::TaDescriptor::register_bindings( + self.memory.graph.descriptor_client(0, ordinal), + ray_query[0].scratch.enabled(), + )?, None, ); let fragment_args = args( @@ -623,6 +627,7 @@ impl Pair { &barriers.fragment[..barriers.fragment_count], abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), + ray_query[1].scratch.enabled(), )?, mcache, ); diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 7509cf539c278..3d5f441d8428f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -70,6 +70,10 @@ impl job::Backend for Backend { Some(self.shared.feed.clone()) } + fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { + crate::g17::fw::render_rt::required_blocks(self.shared.active_cores, pages) + } + fn ensure_compute(&self, scs_pages: u16) -> Result { Backend::ensure_compute(self, scs_pages) } diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 95f0ac9a959bf..27737ac4d863a 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -65,6 +65,11 @@ mod render; /// Trait implemented by all versioned queues. pub(crate) trait Queue: Send + Sync { + fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { + if layouts != [0; 2] { return Err(EINVAL); } + Ok(()) + } + fn submit( &mut self, id: u64, diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 1053fc26d17c0..02e27073cf216 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -298,6 +298,12 @@ enum drm_asahi_feature { * DRM_ASAHI_RENDER_FRAGMENT_BARRIERS. */ DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS = (1UL) << 7, + /** + * @DRM_ASAHI_FEATURE_RENDER_RAY_QUERY: Render commands accept per-stage + * ray query state (struct drm_asahi_render_ray_query_stage) and queues + * accept render_scs_layout. + */ + DRM_ASAHI_FEATURE_RENDER_RAY_QUERY = (1UL) << 8, }; /** @@ -817,6 +823,13 @@ struct drm_asahi_queue_create { * fix a 4GiB VA carveout for USC memory and pass its base address here. */ __u64 usc_exec_base; + + /** + * @render_scs_layout: Largest ray query scratch layouts of the vertex and + * fragment stages this queue will submit, used to size its render pool. + * Requires DRM_ASAHI_FEATURE_RENDER_RAY_QUERY; zero otherwise. + */ + __u64 render_scs_layout[2]; }; /** @@ -1218,6 +1231,42 @@ struct drm_asahi_bg_eot { __u32 rsrc_spec; }; +/** + * struct drm_asahi_render_ray_query_stage - Ray query state of one render stage + * + * Index 0 describes the vertex (tiling) stage and index 1 the fragment stage + * of a render command. An all-zero entry disables ray queries in that stage. + * Requires DRM_ASAHI_FEATURE_RENDER_RAY_QUERY. + */ +struct drm_asahi_render_ray_query_stage { + /** @scs_layout: Per-thread ray query scratch layout, as for compute. */ + __u64 scs_layout; + + /** @intersection_aux: Auxiliary intersection table word. */ + __u32 intersection_aux; + + /** @intersection_pad: MBZ */ + __u32 intersection_pad; + + /** @intersection_pool: GPU VA of the intersection program pool. */ + __u64 intersection_pool; + + /** @intersection_pool_bytes: Size of the intersection program pool. */ + __u64 intersection_pool_bytes; + + /** @intersection_first: First intersection table entry. */ + __u32 intersection_first; + + /** @intersection_count: Number of intersection table entries. */ + __u32 intersection_count; + + /** @intersection_default: GPU VA of the default intersection program. */ + __u64 intersection_default; + + /** @intersection_default_control: Control word of the default program. */ + __u64 intersection_default_control; +}; + /** * struct drm_asahi_cmd_render - Command to submit 3D * @@ -1415,6 +1464,12 @@ struct drm_asahi_cmd_render { * queue's compute command count and the header's cdm_barrier instead. */ __u64 prior_cdm; + + /** + * @ray_query: Ray query state of the vertex and fragment stages. Older + * userspace omits it; a shorter command is read with both stages zeroed. + */ + struct drm_asahi_render_ray_query_stage ray_query[2]; }; /** From 060384dca40d56e2c38c77c67b23fd1fef15d44c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:06:47 +1100 Subject: [PATCH 0977/1579] soc: apple: sep: serialize SBIO exchanges The capture work item and the ioctl path both talk to the SBIO endpoint, which has one out-of-line buffer and one reassembly state. GET_INFO and ENROL_START list the enclave's identities before waiting for the last work item, so libfprint cancelling an identify and starting the next one lands that request in the middle of the previous work item's sensor registration. The second caller is then refused with EBUSY, or, when it starts between a completed reply and its collection, clears that reply and the first caller takes the second caller's answer. Hold an exchange lock for the whole request and reply, as the key store already does with sks_exchange_lock. The receive path does not take it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 5 +++++ drivers/soc/apple/sep.rs | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 1dd46c3195918..ba21814a32a50 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2587,6 +2587,11 @@ impl SepData { name: &'static CStr, payload: &[u8], ) -> Result { + // The capture work item and the ioctl path both reach this endpoint. + // Without this, the second caller is refused with EBUSY, or clears a + // reply the first has not collected yet and takes the next one. Taken + // before the shutdown check, so a waiter never starts after remove(). + let _exchange = self.sbio_exchange_lock.lock(); if self.shutting_down.load(Relaxed) || !self.sbio_ready.load(Relaxed) { return Err(ENODEV); } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 0cb129220a953..0c9bf75a1a843 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -592,6 +592,11 @@ struct SepData { #[pin] dma_ring: Mutex>, + // One SBIO request/reply exchange at a time: the endpoint has a single + // out-of-line buffer and `sbio_rx` tracks a single transfer. + #[pin] + sbio_exchange_lock: Mutex<()>, + #[pin] sbio_rx: Mutex, @@ -858,6 +863,7 @@ impl SepData { scrd_probe <- new_mutex!(ScrdProbe::new()), scrd_wq <- new_condvar!("SepData::scrd_wq"), dma_ring <- new_mutex!(Some(dma_ring)), + sbio_exchange_lock <- new_mutex!(()), sbio_rx <- new_mutex!(transfer::Reassembly::new()), sbio_wq <- new_condvar!("SepData::sbio_wq"), sbio_ready: Atomic::new(false), From 463ba8cc2bd963ab8b772be9ae16d3398ef46eb9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:06:48 +1100 Subject: [PATCH 0978/1579] tools: aurora-sep: run concurrent SBIO exchanges Compile the driver's sbio_transfer_raw against a threaded enclave mock and run a work-item exchange and an ioctl exchange at the same time. Each must get its own reply. Without the exchange lock the ioctl side is refused with EBUSY in every round. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/tests/test_sbio_exchange.py | 174 +++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 tools/aurora-sep/tests/test_sbio_exchange.py diff --git a/tools/aurora-sep/tests/test_sbio_exchange.py b/tools/aurora-sep/tests/test_sbio_exchange.py new file mode 100644 index 0000000000000..177246001f759 --- /dev/null +++ b/tools/aurora-sep/tests/test_sbio_exchange.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: MIT +"""Run the driver's actual SBIO transfer method from two threads at once. + +The verify/enrol work item and the ioctl path (GET_INFO and ENROL_START list +the enclave's identities, DELETE removes one) can reach the single SBIO +endpoint at the same time, e.g. when libfprint cancels an identify and starts +the next one while the previous work item is still registering the sensor. +Each exchange must complete with its own reply; none may be refused with +EBUSY or receive another caller's answer. +""" +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + +APPLE = Path(__file__).resolve().parents[3] / 'drivers/soc/apple' + + +def method(file, name): + source = (APPLE / file).read_text() + match = re.search(r'^ (?:pub\(crate\) )?fn ' + name + r'\(', source, re.M) + if match is None: + raise AssertionError(f'Missing driver method {name}') + opening = source.index('{', match.start()) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[match.start():end] + + +HARNESS = r''' +#![allow(dead_code, unused_variables, unused_imports)] +use std::sync::atomic::{AtomicBool, Ordering::Relaxed}; +use std::sync::mpsc; +use std::time::Duration; + +type CStr = str; +type Result = std::result::Result; +const ENODEV: i32 = -19; +const EBUSY: i32 = -16; +const ETIMEDOUT: i32 = -110; +const SBIO_TIMEOUT_MS: u64 = 2000; + +mod time { pub fn msecs_to_jiffies(ms: u64) -> u64 { ms } } + +pub enum CondVarTimeoutResult { Woken { jiffies: u64 }, Signal { jiffies: u64 }, Timeout } + +/// Kernel-shaped mutex: lock() returns a guard a CondVar can wait on. +struct Mutex(std::sync::Mutex); +struct Guard<'a, T>(Option>); +impl Mutex { + fn new(v: T) -> Self { Mutex(std::sync::Mutex::new(v)) } + fn lock(&self) -> Guard<'_, T> { Guard(Some(self.0.lock().unwrap())) } +} +impl<'a, T> std::ops::Deref for Guard<'a, T> { + type Target = T; + fn deref(&self) -> &T { self.0.as_ref().unwrap() } +} +impl<'a, T> std::ops::DerefMut for Guard<'a, T> { + fn deref_mut(&mut self) -> &mut T { self.0.as_mut().unwrap() } +} +struct CondVar(std::sync::Condvar); +impl CondVar { + fn wait_interruptible_timeout(&self, g: &mut Guard<'_, T>, j: u64) -> CondVarTimeoutResult { + let inner = g.0.take().unwrap(); + let (inner, r) = self.0.wait_timeout(inner, Duration::from_millis(j)).unwrap(); + g.0 = Some(inner); + if r.timed_out() { CondVarTimeoutResult::Timeout } else { CondVarTimeoutResult::Woken { jiffies: j / 2 } } + } + fn notify_all(&self) { self.0.notify_all() } +} + +mod transfer { + pub struct Completed { pub status: u32, pub payload: Vec } + /// Same state rules as drivers/soc/apple/transfer.rs Reassembly. + pub struct Reassembly { pub active: Option, pub done: Option } + impl Reassembly { + pub fn begin(&mut self, opcode: u32) -> super::Result<()> { + if self.active.is_some() { return Err(super::EBUSY); } + self.done = None; + self.active = Some(opcode); + Ok(()) + } + pub fn take_done(&mut self) -> Option { self.done.take() } + pub fn abort(&mut self) { + if self.active.take().is_some() { + self.done = Some(Completed { status: 0xdead, payload: Vec::new() }); + } + } + /// The enclave's reply: it echoes the opcode in the payload. + pub fn on_reply(&mut self, opcode: u32) { + if self.active == Some(opcode) { + self.active = None; + self.done = Some(Completed { status: 0, payload: opcode.to_le_bytes().to_vec() }); + } + } + } +} + +struct SepData { + shutting_down: AtomicBool, + sbio_ready: AtomicBool, + sbio_rx: Mutex, + sbio_wq: CondVar, + sbio_exchange_lock: Mutex<()>, + to_enclave: std::sync::Mutex>, +} + +impl SepData { + fn sbio_start(&self, opcode: u16, _name: &'static CStr, _payload: &[u8]) -> Result<()> { + self.to_enclave.lock().unwrap().send(u32::from(opcode)).unwrap(); + Ok(()) + } +METHOD +} + +fn main() { + let (tx, rx) = mpsc::channel::(); + let sep: &'static SepData = Box::leak(Box::new(SepData { + shutting_down: AtomicBool::new(false), + sbio_ready: AtomicBool::new(true), + sbio_rx: Mutex::new(transfer::Reassembly { active: None, done: None }), + sbio_wq: CondVar(std::sync::Condvar::new()), + sbio_exchange_lock: Mutex::new(()), + to_enclave: std::sync::Mutex::new(tx), + })); + // The enclave answers each request after a short delay, as the mailbox does. + std::thread::spawn(move || { + for opcode in rx { + std::thread::sleep(Duration::from_millis(30)); + sep.sbio_rx.lock().on_reply(opcode); + sep.sbio_wq.notify_all(); + } + }); + let mut failures = 0; + for round in 0..20u16 { + let worker = std::thread::spawn(move || { + // e.g. CLEAR_STATE from the work item's sensor registration + sep.sbio_transfer_raw(0x10 + round, "WORKER", &[]) + .map(|d| (d.status, d.payload)) + }); + std::thread::sleep(Duration::from_millis(5)); + // e.g. LIST_IDENTITIES from GET_INFO on the next identify + let ioctl = sep.sbio_transfer_raw(0x40 + round, "IOCTL", &[]).map(|d| (d.status, d.payload)); + let worker = worker.join().unwrap(); + let want_w = Ok((0, u32::from(0x10 + round).to_le_bytes().to_vec())); + let want_i = Ok((0, u32::from(0x40 + round).to_le_bytes().to_vec())); + if worker != want_w || ioctl != want_i { + eprintln!("round {round}: worker {:?} ioctl {:?}", worker, ioctl); + failures += 1; + } + } + assert_eq!(failures, 0, "concurrent SBIO exchanges interfered"); +} +''' + + +@unittest.skipUnless(shutil.which('rustc'), 'rustc required for driver method tests') +class SbioExchangeTest(unittest.TestCase): + def test_concurrent_exchanges_each_get_their_own_reply(self): + program = HARNESS.replace('METHOD', method('sbio.rs', 'sbio_transfer_raw')) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'test.rs').write_text(program) + subprocess.run(['rustc', '--edition=2021', str(root / 'test.rs'), + '-o', str(root / 'test')], check=True) + subprocess.run([str(root / 'test')], check=True) + + +if __name__ == '__main__': + unittest.main() From ec606af097b9ad4e9bbf9d55d7e25cecf5343b3e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:56:30 +1100 Subject: [PATCH 0979/1579] usb: typec: tipd: Tidy up the CD321x additions to tps6598x.h The CD321x support placed the mux handle and its remove flag ahead of the device pointer in struct tps6598x, and included cd321x-pm.h in the middle of the header between a register define and the bit fields. Keep the includes together at the top, document why TPS_REG_MODE lives in the header, and move the mux handle next to the other port objects. Describe what remove_safe_prepared actually means instead of the previous shorthand. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/tps6598x.h | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index ef3f7ff6531d9..5b394bb40c56d 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -15,11 +15,13 @@ #include #include +#include "cd321x-pm.h" + #ifndef __TPS6598X_H__ #define __TPS6598X_H__ +/* Also read by the SPMI glue to tell when the controller has woken up */ #define TPS_REG_MODE 0x03 -#include "cd321x-pm.h" #define TPS_FIELD_GET(_mask, _reg) ((typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask))) @@ -313,9 +315,6 @@ struct tipd_data { }; struct tps6598x { - struct typec_mux *mux; - /* CD remove leaves one open SAFE transaction for generic completion. */ - bool remove_safe_prepared; struct device *dev; struct regmap *regmap; struct mutex lock; /* device lock */ @@ -329,6 +328,12 @@ struct tps6598x { struct usb_pd_identity partner_identity; struct usb_role_switch *role_sw; struct typec_capability typec_cap; + struct typec_mux *mux; + /* + * Set by ->remove() once it has parked the mux in SAFE, so that + * tps6598x_disconnect() only ends that mux transaction. + */ + bool remove_safe_prepared; struct power_supply *psy; struct power_supply_desc psy_desc; From 21cbed38d507b58251d22e34d840acdaeea3a97c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:57:02 +1100 Subject: [PATCH 0980/1579] usb: typec: tipd: Fix coding style in the CD321x update path Fix the issues checkpatch --strict reports in code added for CD321x: misaligned continuation lines, two lines over 100 columns, an else branch after a goto, and a double blank line. Reword the comment on the shared display connector, which used double spaces and repeated itself, and the comment on the SAFE transaction left open by cd321x_remove(). Move the DRM include below the linux/ ones, where the DisplayPort Alt Mode driver has it, instead of ahead of them. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/core.c | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index f7fbee64a344d..e88a9a90e929b 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -6,8 +6,6 @@ * Author: Heikki Krogerus */ -#include - #include #include #include @@ -22,6 +20,7 @@ #include #include #include +#include #include "tps6598x.h" #include "trace.h" @@ -213,7 +212,7 @@ static int tps6598x_read_partner_identity(struct tps6598x *tps) } static int tps6598x_set_data_role(struct tps6598x *tps, - enum typec_data_role role, bool connected) + enum typec_data_role role, bool connected) { enum usb_role role_val; int ret; @@ -811,21 +810,19 @@ static void cd321x_update_work(struct work_struct *work) was_disconnected |= partner_changed; /* - * Report sink absence before any fallible hardware transition. A failed - * shutdown retains the route/PHY owner, but cannot keep the sink present. - * Every Type-C port references the same external DCP connector. Only the - * port which successfully acquired the shared display route may report a - * disconnect. A second DP sink can negotiate Alt Mode even though its mux - * request is rejected with -EBUSY; treating that sink as the connector - * owner would tear down the display carried by the first port. + * Report the sink as gone before any hardware transition that can fail: + * a failed teardown keeps the route, but the sink must not stay present. + * All ports reference the same external display connector, so only the + * port that owns the display route may report a disconnect. A second DP + * sink can enter Alt Mode while its mux request fails with -EBUSY, and + * must not tear down the display carried by the other port. */ if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && dp_route_was_active && (!new_connected || was_disconnected || !dp_connected || !dp_hpd || - dp_hpd_changed || dp_mode_changed)) { + dp_hpd_changed || dp_mode_changed)) drm_connector_oob_hotplug_event(cd321x->connector_fwnode, - connector_status_disconnected); - } + connector_status_disconnected); /* * ACIO carries the tunneled PCIe reset handshake over the still-live @@ -872,9 +869,8 @@ static void cd321x_update_work(struct work_struct *work) ret); cd321x->state = old_state; goto retry; - } else { - cd321x->display_route_active = false; } + cd321x->display_route_active = false; } /* Update Type-C properties */ @@ -936,7 +932,8 @@ static void cd321x_update_work(struct work_struct *work) if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && cd321x->display_route_active && dp_hpd) - drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_connected); + drm_connector_oob_hotplug_event(cd321x->connector_fwnode, + connector_status_connected); power_supply_changed(tps->psy); cd321x_pm_complete(&cd321x->pm); @@ -950,7 +947,8 @@ static void cd321x_update_work(struct work_struct *work) static void cd321x_queue_status(struct cd321x *cd321x) { cd321x->update_status.status_changed |= cd321x->update_status.status ^ cd321x->tps.status; - cd321x->update_status.data_status_changed |= cd321x->update_status.data_status ^ cd321x->tps.data_status; + cd321x->update_status.data_status_changed |= + cd321x->update_status.data_status ^ cd321x->tps.data_status; cd321x->update_status.status = cd321x->tps.status; cd321x->update_status.pwr_status = cd321x->tps.pwr_status; @@ -2055,12 +2053,11 @@ static int cd321x_remove(struct tps6598x *tps) typec_mux_abort(tps->mux); return ret; } - /* Same task/handle completes without another post-shutdown set callback. */ + /* Leave the SAFE transaction open for tps6598x_disconnect() to end. */ tps->remove_safe_prepared = true; return 0; } - int tipd_init(struct tps6598x *tps) { struct fwnode_handle *fwnode; From 1cd13c2e1c3e95514d7a00699bb445fc8b36670b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:57:22 +1100 Subject: [PATCH 0981/1579] usb: typec: tipd: Simplify the DisplayPort connector lookup fwnode_find_reference() already fails with -ENOENT when the property is missing and never returns NULL, so the separate presence check and the IS_ERR_OR_NULL() test are redundant. Look the reference up directly and drop the error pointer, as the DisplayPort Alt Mode driver does. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/core.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index e88a9a90e929b..4f0bdbda79702 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -1509,7 +1509,6 @@ static int cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); - struct fwnode_handle *connector_fwnode = NULL; int ret; INIT_DELAYED_WORK(&cd321x->update_work, cd321x_update_work); @@ -1531,10 +1530,9 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) goto err_unregister_altmodes; } - if (fwnode_property_present(fwnode, "displayport")) - connector_fwnode = fwnode_find_reference(fwnode, "displayport", 0); - if (!IS_ERR_OR_NULL(connector_fwnode)) - cd321x->connector_fwnode = connector_fwnode; + cd321x->connector_fwnode = fwnode_find_reference(fwnode, "displayport", 0); + if (IS_ERR(cd321x->connector_fwnode)) + cd321x->connector_fwnode = NULL; cd321x->tbt_switch = fwnode_typec_thunderbolt_switch_get(fwnode); if (IS_ERR(cd321x->tbt_switch)) { From b2ffd97a8ce544bfdaca0b04bac70c211ffcb933 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:57:48 +1100 Subject: [PATCH 0982/1579] usb: typec: tipd: Name the sleep hooks after their PM callbacks The tipd_data hooks called from tipd_prepare() and tipd_complete() were named suspend_prepare and resume_reverify. The latter also runs after an aborted suspend, so "resume" is misleading, and neither name matches the dev_pm_ops callback it serves. Call them prepare and complete, and rename the CD321x implementations to match. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/core.c | 20 ++++++++++---------- drivers/usb/typec/tipd/tps6598x.h | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 4f0bdbda79702..7c63f0db91b97 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -1039,7 +1039,7 @@ static void cd321x_resume_work(struct work_struct *work) cd321x_retry_revalidation(cd321x, true); } -static void cd321x_suspend_prepare(struct tps6598x *tps) +static void cd321x_prepare(struct tps6598x *tps) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); @@ -1052,7 +1052,7 @@ static void cd321x_suspend_prepare(struct tps6598x *tps) cancel_delayed_work_sync(&cd321x->update_work); } -static void cd321x_resume_reverify(struct tps6598x *tps) +static void cd321x_complete(struct tps6598x *tps) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); @@ -2300,8 +2300,8 @@ int tipd_prepare(struct device *dev) { struct tps6598x *tps = dev_get_drvdata(dev); - if (tps->data->suspend_prepare) - tps->data->suspend_prepare(tps); + if (tps->data->prepare) + tps->data->prepare(tps); return 0; } @@ -2312,8 +2312,8 @@ void tipd_complete(struct device *dev) { struct tps6598x *tps = dev_get_drvdata(dev); - if (tps->data->resume_reverify) - tps->data->resume_reverify(tps); + if (tps->data->complete) + tps->data->complete(tps); } EXPORT_SYMBOL_GPL(tipd_complete); @@ -2335,8 +2335,8 @@ const struct tipd_data tipd_cd321x_data = { .reset = cd321x_reset, .switch_power_state = cd321x_switch_power_state, .connect = cd321x_connect, - .suspend_prepare = cd321x_suspend_prepare, - .resume_reverify = cd321x_resume_reverify, + .prepare = cd321x_prepare, + .complete = cd321x_complete, }; EXPORT_SYMBOL_GPL(tipd_cd321x_data); @@ -2396,8 +2396,8 @@ const struct tipd_data tipd_sn201202x_data = { .reset = cd321x_reset, .switch_power_state = cd321x_switch_power_state, .connect = cd321x_connect, - .suspend_prepare = cd321x_suspend_prepare, - .resume_reverify = cd321x_resume_reverify, + .prepare = cd321x_prepare, + .complete = cd321x_complete, }; EXPORT_SYMBOL_GPL(tipd_sn201202x_data); diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 5b394bb40c56d..744efb2bdbdef 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -310,8 +310,8 @@ struct tipd_data { bool (*read_data_status)(struct tps6598x *tps); int (*reset)(struct tps6598x *tps); int (*connect)(struct tps6598x *tps, u32 status); - void (*suspend_prepare)(struct tps6598x *tps); - void (*resume_reverify)(struct tps6598x *tps); + void (*prepare)(struct tps6598x *tps); + void (*complete)(struct tps6598x *tps); }; struct tps6598x { From 4da8304f1b51e6493b8d2006439d50b03defc059 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:58:38 +1100 Subject: [PATCH 0983/1579] usb: typec: tipd: Don't warn about deferred CD321x mode changes The ACIO Thunderbolt switch and the ATC PHY return -EAGAIN while the PCIe host behind a USB4 port, or the USB host on the other side of the PHY, is still being set up or torn down. cd321x_update_work() already treats that as a deferral: it retries without forcing a reconnect. Yet cd321x_typec_update_mode() logged every such attempt as a warning, so a dock that comes up a little slowly left "failed to configure Type-C connection: -11" in the log even though it then worked. Log -EAGAIN at debug level and keep the warning for real failures. The revalidation message after a tunnel state change is in the same category: it is a routine, bounded recovery step, so make it debug too. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/core.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 7c63f0db91b97..8c4cd8f954cf6 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -717,7 +717,11 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, if (ret) { /* The mux may have changed before the switch failed. */ cd321x->state = old_state; - dev_warn(tps->dev, "failed to configure Type-C connection: %d\n", ret); + /* A provider that is not ready yet asks for a retry. */ + if (ret == -EAGAIN) + dev_dbg(tps->dev, "Type-C connection deferred\n"); + else + dev_warn(tps->dev, "failed to configure Type-C connection: %d\n", ret); } else if (cd321x->state.alt != cd321x->port_altmode_dp) { cd321x->dp_status = 0; cd321x->dp_conf = 0; @@ -1008,7 +1012,7 @@ static void cd321x_resume_work(struct work_struct *work) if (atomic_xchg(&cd321x->link_event, 0)) { if (cd321x_pm_link_event(&cd321x->pm)) { cd321x->state_valid = false; - dev_info(tps->dev, "revalidating cable after tunnel state change\n"); + dev_dbg(tps->dev, "revalidating cable after tunnel state change\n"); } else { dev_warn_ratelimited(tps->dev, "tunnel link recovery budget exhausted\n"); From 36ae420b3cde6f3e260f17f5abe5742b7efe7b40 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:59:58 +1100 Subject: [PATCH 0984/1579] usb: typec: tipd: Use guard() for the locks in the debugfs interface core.c takes tps->lock with guard() and scoped_guard(), but the debugfs interface added for CD321x open-codes mutex_lock()/mutex_unlock() and needs a goto label to unlock on the -EBUSY path. Use the same guards there, which removes the label and the extra return-value juggling. The critical sections are unchanged: user copies still happen outside tps->lock except for the read of the saved DATA1 result, as before. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/debugfs.c | 56 +++++++++++++------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/drivers/usb/typec/tipd/debugfs.c b/drivers/usb/typec/tipd/debugfs.c index 5a8d576eff0dc..8f35394624fe6 100644 --- a/drivers/usb/typec/tipd/debugfs.c +++ b/drivers/usb/typec/tipd/debugfs.c @@ -122,10 +122,10 @@ static int tipd_debugfs_command_release(struct inode *inode, struct file *file) * may be gone. tipd_debugfs_unregister() has then left "DBMa" itself. */ if (f->holds_dbma && !debugfs_file_get(dentry)) { - mutex_lock(&tps->lock); - if (!--tps->debugfs->dbma_holders) - tipd_debugfs_leave_dbma(tps); - mutex_unlock(&tps->lock); + scoped_guard(mutex, &tps->lock) { + if (!--tps->debugfs->dbma_holders) + tipd_debugfs_leave_dbma(tps); + } debugfs_file_put(dentry); } @@ -166,17 +166,15 @@ static ssize_t tipd_debugfs_command_write(struct file *file, if (!memcmp(cmd->cmd, "DBMa", 4) && buf[4] > 1) return -EINVAL; - mutex_lock(&tps->lock); + guard(mutex)(&tps->lock); /* Preserve inherited mode and the leases held by other open files. */ if ((!memcmp(cmd->cmd, "DBMa", 4) && !buf[4] && (tps->dbma_at_probe || tps->debugfs->dbma_holders > (unsigned int)f->holds_dbma)) || (!memcmp(cmd->cmd, "LOCK", 4) && !memchr_inv(buf + 4, 0, 4) && - (tps->dbma_at_probe || tps->debugfs->dbma_holders))) { - ret = -EBUSY; - goto out; - } + (tps->dbma_at_probe || tps->debugfs->dbma_holders))) + return -EBUSY; ret = tps6598x_exec_cmd_tmo(tps, cmd->cmd, len, len ? &buf[4] : NULL, sizeof(out), out, cmd->timeout_ms, 0); @@ -198,27 +196,20 @@ static ssize_t tipd_debugfs_command_write(struct file *file, } } -out: - mutex_unlock(&tps->lock); - - return ret ? ret : count; + return ret ?: count; } static ssize_t tipd_debugfs_command_read(struct file *file, char __user *ubuf, size_t count, loff_t *ppos) { struct tipd_debugfs_file *f = file->private_data; - ssize_t ret; - mutex_lock(&f->tps->lock); + guard(mutex)(&f->tps->lock); if (!f->have_result) - ret = -ENODATA; - else - ret = simple_read_from_buffer(ubuf, count, ppos, f->result, - sizeof(f->result)); - mutex_unlock(&f->tps->lock); + return -ENODATA; - return ret; + return simple_read_from_buffer(ubuf, count, ppos, f->result, + sizeof(f->result)); } static const struct file_operations tipd_debugfs_command_fops = { @@ -244,9 +235,8 @@ static ssize_t tipd_debugfs_regs_read(struct file *file, char __user *ubuf, if (count > sizeof(buf)) return -EINVAL; - mutex_lock(&tps->lock); - ret = tps6598x_block_read(tps, *ppos, buf, count); - mutex_unlock(&tps->lock); + scoped_guard(mutex, &tps->lock) + ret = tps6598x_block_read(tps, *ppos, buf, count); if (ret) return ret; @@ -268,10 +258,10 @@ void tipd_debugfs_register(struct tps6598x *tps) { struct tipd_debugfs *dbg; - mutex_lock(&tipd_debugfs_root_lock); - if (!tipd_debugfs_root) - tipd_debugfs_root = debugfs_create_dir("tipd", usb_debug_root); - mutex_unlock(&tipd_debugfs_root_lock); + scoped_guard(mutex, &tipd_debugfs_root_lock) { + if (!tipd_debugfs_root) + tipd_debugfs_root = debugfs_create_dir("tipd", usb_debug_root); + } dbg = kzalloc_obj(*dbg); if (!dbg) @@ -295,11 +285,11 @@ void tipd_debugfs_unregister(struct tps6598x *tps) /* Waits for running file operations, later ones fail with -EIO */ debugfs_remove_recursive(dbg->dentry); - mutex_lock(&tps->lock); - if (dbg->dbma_holders) - tipd_debugfs_leave_dbma(tps); - tps->debugfs = NULL; - mutex_unlock(&tps->lock); + scoped_guard(mutex, &tps->lock) { + if (dbg->dbma_holders) + tipd_debugfs_leave_dbma(tps); + tps->debugfs = NULL; + } kfree(dbg); } From 79c568dece2359566c17115e56e4018d578ed332 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 14:00:55 +1100 Subject: [PATCH 0985/1579] usb: typec: tipd: Name the CD321x mux error label after what it does The err_unregister_mux label in cd321x_register_port() drops the references to the mux and the DisplayPort connector; nothing is unregistered there. Call it err_put_mux, like err_put_switch above it. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 8c4cd8f954cf6..d498bb412943c 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -1541,7 +1541,7 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) cd321x->tbt_switch = fwnode_typec_thunderbolt_switch_get(fwnode); if (IS_ERR(cd321x->tbt_switch)) { ret = PTR_ERR(cd321x->tbt_switch); - goto err_unregister_mux; + goto err_put_mux; } cd321x->state.alt = NULL; @@ -1571,7 +1571,7 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) err_put_switch: typec_thunderbolt_switch_put(cd321x->tbt_switch); cd321x->tbt_switch = NULL; -err_unregister_mux: +err_put_mux: fwnode_handle_put(cd321x->connector_fwnode); cd321x->connector_fwnode = NULL; typec_mux_put(cd321x->mux); From eb65fb9952cbeb923b10be0074e06915e7fc4f3c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:09:59 +1100 Subject: [PATCH 0986/1579] HID: dockchannel: Drop reports that arrive before the device is open The MTP firmware starts sending reports for an interface as soon as the host enables it. dchid_create_interface_work() enables the interface first and only then allocates and registers the HID device, while the reports are handled on the interface's own workqueue. The keyboard sends its first report in that window on every boot, so the driver logs Report received but keyboard is not initialized! and reads iface->hid while the create work is still writing it. A report that arrives before the HID device is open has nobody to go to either way: the existing open check already dropped it once the device was registered. Make the open flag the only gate, so this expected state no longer warns, and use release/acquire on it so a reader that sees the device open also sees iface->hid. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index bd529afd3d4a8..ce82e25ce052b 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -675,7 +675,8 @@ static int dchid_open(struct hid_device *hdev) } } - iface->open = true; + /* Pairs with the acquire in dchid_handle_report(); publishes iface->hid. */ + smp_store_release(&iface->open, true); return 0; } @@ -683,7 +684,7 @@ static void dchid_close(struct hid_device *hdev) { struct dchid_iface *iface = hdev->driver_data; - iface->open = false; + WRITE_ONCE(iface->open, false); } static int dchid_parse(struct hid_device *hdev) @@ -1063,14 +1064,12 @@ static void dchid_handle_event(struct dockchannel_hid *dchid, void *data, size_t static void dchid_handle_report(struct dchid_iface *iface, void *data, size_t length) { - struct dockchannel_hid *dchid = iface->dchid; - - if (!iface->hid) { - dev_warn(dchid->dev, "Report received but %s is not initialized!\n", iface->name); - return; - } - - if (!iface->open) + /* + * An interface starts reporting as soon as it is enabled, which is + * before its HID device has been registered and opened. Until then + * there is nobody to deliver the reports to. + */ + if (!smp_load_acquire(&iface->open)) return; hid_input_report(iface->hid, HID_INPUT_REPORT, data, length, 1); From 06914e1d499555a96e0beccccc0b5485051108cd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:10:13 +1100 Subject: [PATCH 0987/1579] soc: apple: rtkit: Handle oslog notifications quietly Besides the buffer request, RTKit firmware sends oslog messages of types 2 to 5 during normal operation. Types 3 to 5 arrive in a burst when SMC, DCP and AOP firmware boot, and were logged at info level, four or five lines per co-processor on every boot. Type 2 arrives once per boot from the AOP and the MTP, and was logged as an unknown message with a warning. None of them expects a reply: the co-processors keep running, and keyboard, trackpad and AOP sensors keep working after they arrive. The payload is not decoded, so the lines say nothing actionable. Log all four types at debug level and keep the warning for types that have not been seen. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 15baa7e4193f3..b4054ec0dfc03 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -608,16 +608,13 @@ static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg) apple_rtkit_common_rx_get_buffer(rtk, &rtk->oslog_buffer, APPLE_RTKIT_EP_OSLOG, msg); break; - case 3: - case 4: - case 5: + case 2 ... 5: /* - * OpenBSD's public RTKit consumer recognizes these opaque events - * without an ACK. Keep their payload visible without interpreting - * it as a buffer address or a decoded firmware error. + * Undocumented notifications sent during normal operation. They + * expect no reply, and the co-processor keeps running without + * one. */ - dev_info(rtk->dev, "RTKit: OSLog event %u: %016llx (undecoded)\n", - type, msg); + dev_dbg(rtk->dev, "RTKit: oslog notification: %016llx\n", msg); break; default: dev_warn(rtk->dev, "RTKit: Unknown oslog message: %llx\n", From 0ad3e7c0ff9e5452f7e2157a3da60f35acbb6058 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:10:28 +1100 Subject: [PATCH 0988/1579] HID: dockchannel: Terminate log messages and fix two typos Eight messages lack a trailing newline, so the next message can be appended to the same console line, and two carry typos ("firmwareS", "expFected"). No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index ce82e25ce052b..6a5e412887082 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -580,7 +580,7 @@ static int dchid_start_interface(struct dchid_iface *iface) int ret; if (iface->starting) { - dev_warn(iface->dchid->dev, "Interface %s is already starting", iface->name); + dev_warn(iface->dchid->dev, "Interface %s is already starting\n", iface->name); return -EINPROGRESS; } @@ -607,7 +607,7 @@ static int dchid_start_interface(struct dchid_iface *iface) dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); ret = dchid_send_firmware(iface, fw, size); if (ret < 0) { - dev_err(iface->dchid->dev, "Failed to send %s firmwareS", iface->name); + dev_err(iface->dchid->dev, "Failed to send %s firmware\n", iface->name); goto err; } @@ -640,7 +640,7 @@ static int dchid_start(struct hid_device *hdev) if (iface->keyboard_layout_id) { int ret = device_create_file(&hdev->dev, &dev_attr_apple_layout_id); if (ret) { - dev_warn(iface->dchid->dev, "Failed to create apple_layout_id: %d", ret); + dev_warn(iface->dchid->dev, "Failed to create apple_layout_id: %d\n", ret); iface->keyboard_layout_id = 0; } } @@ -802,7 +802,7 @@ static void dchid_create_interface_work(struct work_struct *ws) if (ret < 0) { iface->hid = NULL; hid_destroy_device(hid); - dev_warn(iface->dchid->dev, "Failed to register hid device %s", iface->name); + dev_warn(iface->dchid->dev, "Failed to register hid device %s\n", iface->name); } } @@ -1133,7 +1133,7 @@ static void dchid_handle_ack(struct dchid_iface *iface, struct dchid_hdr *hdr, v } if (shdr->flags != iface->out_flags) { dev_err(iface->dchid->dev, - "Received unexpected flags 0x%x on ACK channel (expFected 0x%x)\n", + "Received unexpected flags 0x%x on ACK channel (expected 0x%x)\n", shdr->flags, iface->out_flags); return; } @@ -1344,14 +1344,14 @@ static int dockchannel_hid_probe(struct platform_device *pdev) */ helper = of_parse_phandle(dev->of_node, "apple,helper-cpu", 0); if (!helper) { - dev_err(dev, "Missing apple,helper-cpu property"); + dev_err(dev, "Missing apple,helper-cpu property\n"); return -EINVAL; } helper_pdev = of_find_device_by_node(helper); of_node_put(helper); if (!helper_pdev) { - dev_err(dev, "Failed to find helper device"); + dev_err(dev, "Failed to find helper device\n"); return -EINVAL; } @@ -1359,7 +1359,7 @@ static int dockchannel_hid_probe(struct platform_device *pdev) DL_FLAG_AUTOREMOVE_CONSUMER); put_device(&helper_pdev->dev); if (!dchid->helper_link) { - dev_err(dev, "Failed to link to helper device"); + dev_err(dev, "Failed to link to helper device\n"); return -EINVAL; } @@ -1388,7 +1388,7 @@ static int dockchannel_hid_probe(struct platform_device *pdev) dchid->comm = dchid_get_interface(dchid, IFACE_COMM, "comm"); if (!dchid->comm) { - dev_err(dchid->dev, "Failed to initialize comm interface"); + dev_err(dchid->dev, "Failed to initialize comm interface\n"); destroy_workqueue(dchid->new_iface_wq); return -EIO; } From 83a01c72288bf0f169695d04285998c99e8dbfef Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 14:08:53 +1100 Subject: [PATCH 0989/1579] rust: iosys_map: let the KUnit helper compare with assert_eq VecIoSysMap::assert_eq() uses assert_eq!, which needs T: Debug, but the helper only bounds T by AsBytes + FromBytes + Clone + PartialEq. With CONFIG_KUNIT=y the kernel crate then fails to build (E0277, "T doesn't implement Debug"), which takes every Rust KUnit suite down with it. Add the Debug bound to the helper struct and its impl. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/iosys_map.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/kernel/iosys_map.rs b/rust/kernel/iosys_map.rs index 039137c051a2f..ef31660eefaca 100644 --- a/rust/kernel/iosys_map.rs +++ b/rust/kernel/iosys_map.rs @@ -514,12 +514,12 @@ pub mod tests { use super::*; /// A helper struct for managed IoSysMapRef structs which point to a [`Vec`]. - pub struct VecIoSysMap { + pub struct VecIoSysMap { map: RawIoSysMap, vec: KVec, } - impl VecIoSysMap { + impl VecIoSysMap { pub fn new(src: &[T]) -> Result { let mut vec = KVec::::new(); From f3296ed0b579eab9b341152f28462ed5447fadf5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 14:08:54 +1100 Subject: [PATCH 0990/1579] ASoC: apple: t8140-aop-audio: test the speaker ceiling with KUnit The copy-time conversion is the only digital limit between userspace and the J700 speaker amplifiers: the top of "Speaker Playback Volume" must be the amplifier's full scale (a float of 0.5 on the LEAP; above it the words wrap), and macaudio's lock level (87) must sit 20 dB below it. Nothing checked either outside the compile-time assertion on the table's top entry. Add a KUnit suite, snd_soc_t8140_aop_audio_speaker: - the top of the volume range is the amplifier full scale; - the table falls in 0.5 dB steps to mute; - the locked level is 20 dB below full scale; - no S32 sample at any volume converts above full scale (with the exact results for i32::MAX/MIN at full volume and at the lock); - the float conversion is exact on powers of two. kunit.py on arm64 qemu: 5/5 pass. With the conversion's shift changed from 31 to 30 (6 dB over the ceiling) the full-scale test fails. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 87 ++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index f726c88f0ea04..57dad8c960331 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -2869,3 +2869,90 @@ module_platform_driver! { description: "Apple T8140 AOP audio: microphones, headphone jack and speakers", license: "Dual MIT/GPL", } + +/// The speaker ceiling and the copy-path conversion are the whole digital +/// protection of the J700 speakers below the daemon: check them where they +/// are built, without the hardware. +#[kunit_tests(snd_soc_t8140_aop_audio_speaker)] +mod tests { + use super::*; + + /// macaudio's level for the J700 while no daemon holds the lock: + /// MAX98360A_DB(safe_vol = -20). + const LOCKED_VOLUME: usize = 87; + /// 10^(0.5 / 20) in parts per 10^9. + const HALF_DB_PPB: u128 = 1_059_253_725; + /// 0.5 as a float32: the amplifier's full scale on the wire. + const F32_HALF: u32 = 0x3f00_0000; + + #[test] + fn the_top_of_the_volume_range_is_the_amplifier_full_scale() { + assert_eq!( + SPKR_VOLUME_Q31[SPKR_VOLUME_MAX as usize] as i64, + SPKR_CEILING_Q31 + ); + assert_eq!(SPKR_CEILING_Q31, 1 << 30); + assert_eq!(f32_bits_of_q31(SPKR_CEILING_Q31 as i32), F32_HALF); + } + + #[test] + fn the_volume_table_falls_in_half_db_steps_to_mute() { + assert_eq!(SPKR_VOLUME_Q31[0], 0); + for i in 2..=SPKR_VOLUME_MAX as usize { + let (lo, hi) = (SPKR_VOLUME_Q31[i - 1] as u128, SPKR_VOLUME_Q31[i] as u128); + assert!(hi > lo); + // hi / lo == 10^(0.5/20) to within the table's rounding + let expect = lo * HALF_DB_PPB / 1_000_000_000; + assert!(hi.abs_diff(expect) <= 2); + } + } + + #[test] + fn the_locked_level_is_20_db_below_full_scale() { + let locked = SPKR_VOLUME_Q31[LOCKED_VOLUME] as i64; + // 10 * locked == ceiling to within the table's rounding (0.1 ppm) + assert!((10 * locked - SPKR_CEILING_Q31).abs() <= 10); + } + + #[test] + fn no_sample_at_any_volume_exceeds_the_full_scale() { + for &gain in SPKR_VOLUME_Q31.iter() { + for sample in [ + i32::MAX, + i32::MIN, + i32::MAX - 1, + i32::MIN + 1, + 1 << 30, + -(1 << 30), + 1, + -1, + 0, + ] { + let bits = spkr_f32_bits(sample, gain); + assert!(bits & 0x7fff_ffff <= F32_HALF); + } + } + // the extremes at full volume: just under and exactly at full scale + assert_eq!( + spkr_f32_bits(i32::MAX, SPKR_CEILING_Q31 as u32), + 0x3eff_ffff + ); + assert_eq!( + spkr_f32_bits(i32::MIN, SPKR_CEILING_Q31 as u32), + 0x8000_0000 | F32_HALF + ); + // and 20 dB down at the locked level + let locked = spkr_f32_bits(i32::MIN, SPKR_VOLUME_Q31[LOCKED_VOLUME]); + // 0.05 as a float32 is 0x3d4c_cccd; the conversion truncates + assert!(locked & 0x7fff_ffff <= 0x3d4c_cccd && locked & 0x7fff_ffff > 0x3d4c_cccd - 64); + } + + #[test] + fn the_float_conversion_is_exact_on_powers_of_two() { + assert_eq!(f32_bits_of_q31(0), 0); + assert_eq!(f32_bits_of_q31(1 << 29), 0x3e80_0000); // 0.25 + assert_eq!(f32_bits_of_q31(-(1 << 30)), 0xbf00_0000); // -0.5 + assert_eq!(f32_bits_of_q31(1), 0x3000_0000); // 2^-31 + assert_eq!(f32_bits_of_q31(i32::MIN), 0xbf80_0000); // -1.0 + } +} From a6d525e226bde76eb7f0084c7bc29f5837a80047 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 11:55:26 +1100 Subject: [PATCH 0991/1579] drm/apple: Resume a CRTC whose mode needs a new link disabled Every resume with the external display at 1920x1080@180 logged [drm:drm_mode_config_helper_resume] *ERROR* Failed to resume (-22) The 180 Hz mode is admitted only while the DPTX link training that proves its bandwidth is current. System sleep releases the link, so when drm_atomic_helper_resume() commits the saved state, dcp_crtc_mode_fixup() rejects the mode and the whole commit fails with -EINVAL. Neither CRTC is restored; both outputs come back only when the compositor reacts to the hotplug that follows the relink. Before resuming, disable each CRTC of the saved state whose mode no longer passes the mode fixup, detaching its planes and connectors. The rest of the state is restored as before, and the DCP reports the external display again once its link is back, which is when userspace re-enables it, as after a replug. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 57 ++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 54f8d52c864e5..eeea2a9f31e2a 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -927,12 +928,66 @@ static int apple_platform_suspend(struct device *dev) return 0; } +/* + * A mode can be admissible only while the link it was proven on is up: the + * external H17P 180 Hz mode needs the current DPTX link training, and system + * sleep releases that link. Restoring such a CRTC as it was fails the whole + * resume commit in its mode fixup, which leaves every output dark until + * userspace reacts to the next hotplug. Restore that CRTC disabled instead. + * The DCP reports the display again once its link is back, and userspace + * re-enables it on that hotplug, as it does after a cable is replugged. + * + * The suspend state holds every CRTC, plane and connector already, so it is + * edited directly rather than through helpers that need the modeset locks. + */ +static void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) +{ + struct drm_connector_state *conn_state; + struct drm_plane_state *plane_state; + struct drm_crtc_state *crtc_state; + struct drm_connector *connector; + struct drm_plane *plane; + struct drm_crtc *crtc; + int i, j; + + for_each_new_crtc_in_state(state, crtc, crtc_state, i) { + if (!crtc_state->enable || + dcp_crtc_mode_fixup(crtc, &crtc_state->mode, + &crtc_state->adjusted_mode)) + continue; + + drm_dbg_kms(crtc->dev, + "[CRTC:%d:%s] " DRM_MODE_FMT " needs a new link, resuming disabled\n", + crtc->base.id, crtc->name, DRM_MODE_ARG(&crtc_state->mode)); + + for_each_new_plane_in_state(state, plane, plane_state, j) { + if (plane_state->crtc != crtc) + continue; + crtc_state->plane_mask &= ~drm_plane_mask(plane); + plane_state->crtc = NULL; + drm_atomic_set_fb_for_plane(plane_state, NULL); + } + for_each_new_connector_in_state(state, connector, conn_state, j) { + /* Detaching from a CRTC already in the state cannot fail. */ + if (conn_state->crtc == crtc && + WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) + return; + } + crtc_state->active = false; + if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) + return; + } +} + static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - if (apple) + if (apple) { + if (apple->drm.mode_config.suspend_state) + apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); drm_mode_config_helper_resume(&apple->drm); + } return 0; } From 3f92348a0340ad5a776233850bb372d58dea16d9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:13:05 +1100 Subject: [PATCH 0992/1579] drm/apple: Test the resume of a CRTC whose mode no longer fixes up Resume restores a CRTC whose mode needs a link that system sleep released by bringing it back disabled. That edit of the suspend state detaches planes, connectors and the framebuffer by hand, outside the helpers, and nothing exercises it. Move it into its own file and have it ask the CRTC's own mode fixup, which for every apple-drm CRTC is dcp_crtc_mode_fixup(), so that KUnit can run it against a mock device. Add tests that take that device through drm_mode_config_helper_suspend() and _resume(): an inadmissible mode fails the resume when it is restored as is, and resumes disabled, with its planes, connector and framebuffer reference released, when it goes through the filter; an admissible or DPMS-off CRTC is handled the same way as before. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 4 +- drivers/gpu/drm/apple/Makefile | 3 +- drivers/gpu/drm/apple/apple_drv.c | 53 +---- drivers/gpu/drm/apple/apple_resume.c | 66 ++++++ drivers/gpu/drm/apple/apple_resume.h | 11 + drivers/gpu/drm/apple/apple_resume_test.c | 261 ++++++++++++++++++++++ 6 files changed, 344 insertions(+), 54 deletions(-) create mode 100644 drivers/gpu/drm/apple/apple_resume.c create mode 100644 drivers/gpu/drm/apple/apple_resume.h create mode 100644 drivers/gpu/drm/apple/apple_resume_test.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index d74b2dfdb0f88..9c1ce96bc0dbb 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -32,6 +32,7 @@ config DRM_APPLE_DEBUG config DRM_APPLE_KUNIT_TEST tristate "KUnit tests for Apple display controller" if !KUNIT_ALL_TESTS depends on KUNIT + select DRM_KUNIT_TEST_HELPERS if DRM_APPLE default KUNIT_ALL_TESTS help Build tests for the Apple display driver's AFK ring parser and @@ -40,4 +41,5 @@ config DRM_APPLE_KUNIT_TEST The tests cover the header strides used by older firmware and larger rings, as well as malformed ring size combinations. This option is intended for developer test kernels. - Also test the Type-C modeset, hotplug and service ownership state. + Also test the Type-C modeset, hotplug and service ownership state, + and the system resume of a CRTC whose mode is no longer admissible. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 6b3fbdbc55aa6..10fcd21cee723 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -2,7 +2,7 @@ CFLAGS_trace.o = -I$(src) -appledrm-y := apple_drv.o +appledrm-y := apple_drv.o apple_resume.o appledrm-y += afk.o dcp.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o @@ -12,6 +12,7 @@ appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o appledrm-y += iomfb_h17p.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple_resume_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index eeea2a9f31e2a..2d6173ed9472e 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -21,7 +21,6 @@ #include #include -#include #include #include #include @@ -41,6 +40,7 @@ #include #include +#include "apple_resume.h" #include "dcp.h" #include "plane.h" @@ -928,57 +928,6 @@ static int apple_platform_suspend(struct device *dev) return 0; } -/* - * A mode can be admissible only while the link it was proven on is up: the - * external H17P 180 Hz mode needs the current DPTX link training, and system - * sleep releases that link. Restoring such a CRTC as it was fails the whole - * resume commit in its mode fixup, which leaves every output dark until - * userspace reacts to the next hotplug. Restore that CRTC disabled instead. - * The DCP reports the display again once its link is back, and userspace - * re-enables it on that hotplug, as it does after a cable is replugged. - * - * The suspend state holds every CRTC, plane and connector already, so it is - * edited directly rather than through helpers that need the modeset locks. - */ -static void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) -{ - struct drm_connector_state *conn_state; - struct drm_plane_state *plane_state; - struct drm_crtc_state *crtc_state; - struct drm_connector *connector; - struct drm_plane *plane; - struct drm_crtc *crtc; - int i, j; - - for_each_new_crtc_in_state(state, crtc, crtc_state, i) { - if (!crtc_state->enable || - dcp_crtc_mode_fixup(crtc, &crtc_state->mode, - &crtc_state->adjusted_mode)) - continue; - - drm_dbg_kms(crtc->dev, - "[CRTC:%d:%s] " DRM_MODE_FMT " needs a new link, resuming disabled\n", - crtc->base.id, crtc->name, DRM_MODE_ARG(&crtc_state->mode)); - - for_each_new_plane_in_state(state, plane, plane_state, j) { - if (plane_state->crtc != crtc) - continue; - crtc_state->plane_mask &= ~drm_plane_mask(plane); - plane_state->crtc = NULL; - drm_atomic_set_fb_for_plane(plane_state, NULL); - } - for_each_new_connector_in_state(state, connector, conn_state, j) { - /* Detaching from a CRTC already in the state cannot fail. */ - if (conn_state->crtc == crtc && - WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) - return; - } - crtc_state->active = false; - if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) - return; - } -} - static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); diff --git a/drivers/gpu/drm/apple/apple_resume.c b/drivers/gpu/drm/apple/apple_resume.c new file mode 100644 index 0000000000000..4d17d693fa59b --- /dev/null +++ b/drivers/gpu/drm/apple/apple_resume.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include + +#include +#include +#include +#include +#include +#include + +#include "apple_resume.h" + +/* + * A mode can be admissible only while the link it was proven on is up: the + * external H17P 180 Hz mode needs the current DPTX link training, and system + * sleep releases that link. Restoring such a CRTC as it was fails the whole + * resume commit in its mode fixup, which leaves every output dark until + * userspace reacts to the next hotplug. Restore that CRTC disabled instead. + * The DCP reports the display again once its link is back, and userspace + * re-enables it on that hotplug, as it does after a cable is replugged. + * + * The suspend state holds every CRTC, plane and connector already, so it is + * edited directly rather than through helpers that need the modeset locks. + */ +void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) +{ + const struct drm_crtc_helper_funcs *funcs; + struct drm_connector_state *conn_state; + struct drm_plane_state *plane_state; + struct drm_crtc_state *crtc_state; + struct drm_connector *connector; + struct drm_plane *plane; + struct drm_crtc *crtc; + int i, j; + + for_each_new_crtc_in_state(state, crtc, crtc_state, i) { + funcs = crtc->helper_private; + if (!crtc_state->enable || !funcs || !funcs->mode_fixup || + funcs->mode_fixup(crtc, &crtc_state->mode, + &crtc_state->adjusted_mode)) + continue; + + drm_dbg_kms(crtc->dev, + "[CRTC:%d:%s] " DRM_MODE_FMT " needs a new link, resuming disabled\n", + crtc->base.id, crtc->name, DRM_MODE_ARG(&crtc_state->mode)); + + for_each_new_plane_in_state(state, plane, plane_state, j) { + if (plane_state->crtc != crtc) + continue; + crtc_state->plane_mask &= ~drm_plane_mask(plane); + plane_state->crtc = NULL; + drm_atomic_set_fb_for_plane(plane_state, NULL); + } + for_each_new_connector_in_state(state, connector, conn_state, j) { + /* Detaching from a CRTC already in the state cannot fail. */ + if (conn_state->crtc == crtc && + WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) + return; + } + crtc_state->active = false; + if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) + return; + } +} diff --git a/drivers/gpu/drm/apple/apple_resume.h b/drivers/gpu/drm/apple/apple_resume.h new file mode 100644 index 0000000000000..313e02c12b064 --- /dev/null +++ b/drivers/gpu/drm/apple/apple_resume.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Ryan Murray */ + +#ifndef __APPLE_RESUME_H__ +#define __APPLE_RESUME_H__ + +struct drm_atomic_state; + +void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state); + +#endif /* __APPLE_RESUME_H__ */ diff --git a/drivers/gpu/drm/apple/apple_resume_test.c b/drivers/gpu/drm/apple/apple_resume_test.c new file mode 100644 index 0000000000000..2dd1431bc065f --- /dev/null +++ b/drivers/gpu/drm/apple/apple_resume_test.c @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "apple_resume.h" + +/* One pipe: a primary plane scanning a framebuffer out to one connector. */ +struct resume_test { + struct drm_device drm; + struct drm_plane *plane; + struct drm_crtc *crtc; + struct drm_encoder encoder; + struct drm_connector connector; + struct drm_framebuffer fb; + /* Whether the mode fixup admits the mode, as a current link would. */ + bool admissible; +}; + +static struct resume_test *to_resume_test(struct drm_device *drm) +{ + return container_of(drm, struct resume_test, drm); +} + +static bool resume_test_mode_fixup(struct drm_crtc *crtc, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) +{ + return to_resume_test(crtc->dev)->admissible; +} + +static const struct drm_crtc_helper_funcs resume_test_crtc_helpers = { + .mode_fixup = resume_test_mode_fixup, +}; + +/* Scanout itself is not under test; the plane helpers require an update. */ +static void resume_test_plane_update(struct drm_plane *plane, + struct drm_atomic_state *state) +{ +} + +static const struct drm_plane_helper_funcs resume_test_plane_helpers = { + .atomic_update = resume_test_plane_update, +}; + +static const struct drm_connector_helper_funcs resume_test_connector_helpers = { +}; + +static const struct drm_connector_funcs resume_test_connector_funcs = { + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .reset = drm_atomic_helper_connector_reset, +}; + +static void resume_test_fb_destroy(struct drm_framebuffer *fb) +{ + drm_framebuffer_cleanup(fb); +} + +static const struct drm_framebuffer_funcs resume_test_fb_funcs = { + .destroy = resume_test_fb_destroy, +}; + +static void resume_test_teardown(void *data) +{ + struct resume_test *priv = data; + + /* Release the scanout reference before the device goes away. */ + drm_atomic_helper_shutdown(&priv->drm); + drm_framebuffer_put(&priv->fb); +} + +static struct resume_test *resume_test_init(struct kunit *test) +{ + struct drm_modeset_acquire_ctx ctx; + struct drm_connector_state *conn_state; + struct drm_plane_state *plane_state; + struct drm_crtc_state *crtc_state; + struct drm_atomic_state *state; + struct drm_display_mode *mode; + struct resume_test *priv; + struct drm_device *drm; + struct device *dev; + int ret; + + dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + priv = drm_kunit_helper_alloc_drm_device(test, dev, struct resume_test, drm, + DRIVER_MODESET | DRIVER_ATOMIC); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv); + drm = &priv->drm; + priv->admissible = true; + + priv->plane = drm_kunit_helper_create_primary_plane(test, drm, NULL, + &resume_test_plane_helpers, + NULL, 0, NULL); + priv->crtc = drm_kunit_helper_create_crtc(test, drm, priv->plane, NULL, NULL, + &resume_test_crtc_helpers); + ret = drmm_encoder_init(drm, &priv->encoder, NULL, DRM_MODE_ENCODER_TMDS, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + priv->encoder.possible_crtcs = drm_crtc_mask(priv->crtc); + ret = drmm_connector_init(drm, &priv->connector, &resume_test_connector_funcs, + DRM_MODE_CONNECTOR_USB, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + drm_connector_helper_add(&priv->connector, &resume_test_connector_helpers); + ret = drm_connector_attach_encoder(&priv->connector, &priv->encoder); + KUNIT_ASSERT_EQ(test, ret, 0); + drm_mode_config_reset(drm); + + priv->fb.dev = drm; + priv->fb.format = drm_format_info(DRM_FORMAT_XRGB8888); + priv->fb.width = 1920; + priv->fb.height = 1080; + priv->fb.pitches[0] = 1920 * 4; + priv->fb.modifier = DRM_FORMAT_MOD_LINEAR; + ret = drm_framebuffer_init(drm, &priv->fb, &resume_test_fb_funcs); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, resume_test_teardown, priv); + KUNIT_ASSERT_EQ(test, ret, 0); + + mode = drm_kunit_display_mode_from_cea_vic(test, drm, 16); + KUNIT_ASSERT_NOT_NULL(test, mode); + + /* Put by hand: a state kept to the end of the test pins old scanout. */ + drm_modeset_acquire_init(&ctx, 0); + state = drm_atomic_state_alloc(drm); + KUNIT_ASSERT_NOT_NULL(test, state); + state->acquire_ctx = &ctx; + conn_state = drm_atomic_get_connector_state(state, &priv->connector); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state); + KUNIT_ASSERT_EQ(test, drm_atomic_set_crtc_for_connector(conn_state, priv->crtc), 0); + crtc_state = drm_atomic_get_crtc_state(state, priv->crtc); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state); + KUNIT_ASSERT_EQ(test, drm_atomic_set_mode_for_crtc(crtc_state, mode), 0); + crtc_state->active = true; + plane_state = drm_atomic_get_plane_state(state, priv->plane); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane_state); + KUNIT_ASSERT_EQ(test, drm_atomic_set_crtc_for_plane(plane_state, priv->crtc), 0); + drm_atomic_set_fb_for_plane(plane_state, &priv->fb); + plane_state->crtc_w = 1920; + plane_state->crtc_h = 1080; + plane_state->src_w = 1920 << 16; + plane_state->src_h = 1080 << 16; + ret = drm_atomic_commit(state); + drm_atomic_state_put(state); + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_TRUE(test, priv->crtc->state->active); + + return priv; +} + +/* The failure the stale CRTC filter exists for: one bad mode fails resume. */ +static void resume_stale_mode_fails_unfiltered_test(struct kunit *test) +{ + struct resume_test *priv = resume_test_init(test); + + KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); + priv->admissible = false; + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), -EINVAL); + KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); +} + +static void resume_stale_crtc_disabled_test(struct kunit *test) +{ + struct resume_test *priv = resume_test_init(test); + + KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); + priv->admissible = false; + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); + + KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); + KUNIT_EXPECT_FALSE(test, priv->crtc->state->active); + KUNIT_EXPECT_EQ(test, priv->crtc->state->plane_mask, 0U); + KUNIT_EXPECT_EQ(test, priv->crtc->state->connector_mask, 0U); + KUNIT_EXPECT_NULL(test, priv->plane->state->crtc); + KUNIT_EXPECT_NULL(test, priv->plane->state->fb); + KUNIT_EXPECT_NULL(test, priv->connector.state->crtc); + /* Only the creator's reference is left: the saved scanout was dropped. */ + KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 1U); +} + +static void resume_admissible_crtc_restored_test(struct kunit *test) +{ + struct resume_test *priv = resume_test_init(test); + + KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); + + KUNIT_EXPECT_TRUE(test, priv->crtc->state->enable); + KUNIT_EXPECT_TRUE(test, priv->crtc->state->active); + KUNIT_EXPECT_PTR_EQ(test, priv->plane->state->crtc, priv->crtc); + KUNIT_EXPECT_PTR_EQ(test, priv->plane->state->fb, &priv->fb); + KUNIT_EXPECT_PTR_EQ(test, priv->connector.state->crtc, priv->crtc); + KUNIT_EXPECT_EQ(test, priv->connector.state->link_status, + DRM_LINK_STATUS_GOOD); + KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 2U); +} + +/* A DPMS-off CRTC keeps its mode, so it is checked like an active one. */ +static void resume_stale_dpms_off_crtc_disabled_test(struct kunit *test) +{ + struct resume_test *priv = resume_test_init(test); + struct drm_modeset_acquire_ctx ctx; + struct drm_crtc_state *crtc_state; + struct drm_atomic_state *state; + int ret; + + drm_modeset_acquire_init(&ctx, 0); + state = drm_atomic_state_alloc(&priv->drm); + KUNIT_ASSERT_NOT_NULL(test, state); + state->acquire_ctx = &ctx; + crtc_state = drm_atomic_get_crtc_state(state, priv->crtc); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state); + crtc_state->active = false; + ret = drm_atomic_commit(state); + drm_atomic_state_put(state); + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + KUNIT_ASSERT_EQ(test, ret, 0); + + KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); + priv->admissible = false; + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); + KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); + KUNIT_EXPECT_NULL(test, priv->connector.state->crtc); + KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 1U); +} + +static struct kunit_case resume_cases[] = { + KUNIT_CASE(resume_stale_mode_fails_unfiltered_test), + KUNIT_CASE(resume_stale_crtc_disabled_test), + KUNIT_CASE(resume_admissible_crtc_restored_test), + KUNIT_CASE(resume_stale_dpms_off_crtc_disabled_test), + {} +}; + +static struct kunit_suite resume_suite = { + .name = "apple-drm-resume", + .test_cases = resume_cases, +}; + +kunit_test_suite(resume_suite); From bc4d7b9dd4a7b5a7d1024abfb4dfbbf22f089bd2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:22:06 +1100 Subject: [PATCH 0993/1579] drm/apple: Release the encoder of a connector detached for resume A CRTC restored disabled after system sleep has its connectors detached from it in the suspend state, but each keeps the encoder it was routed through. The commit that restores the state then finds a connector with an encoder and no CRTC, which the atomic helpers only expect from a broken duplicate, and every such resume logs WARNING: drivers/gpu/drm/drm_atomic_helper.c:238 at set_best_encoder The CRTC also keeps the encoder in its encoder mask. Release the encoder together with the CRTC, as the helpers do when they route a connector to no CRTC. The KUnit test now checks the edited suspend state. Fixes: 229d38cd44fd ("drm/apple: Resume a CRTC whose mode needs a new link disabled") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_resume.c | 14 ++++++++++++-- drivers/gpu/drm/apple/apple_resume_test.c | 15 ++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_resume.c b/drivers/gpu/drm/apple/apple_resume.c index 4d17d693fa59b..ec49db7d1d924 100644 --- a/drivers/gpu/drm/apple/apple_resume.c +++ b/drivers/gpu/drm/apple/apple_resume.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -54,10 +55,19 @@ void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) drm_atomic_set_fb_for_plane(plane_state, NULL); } for_each_new_connector_in_state(state, connector, conn_state, j) { + if (conn_state->crtc != crtc) + continue; /* Detaching from a CRTC already in the state cannot fail. */ - if (conn_state->crtc == crtc && - WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) + if (WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) return; + /* + * Nor does a connector without a CRTC keep an encoder: + * the resume check warns about one left routed. + */ + if (conn_state->best_encoder) + crtc_state->encoder_mask &= + ~drm_encoder_mask(conn_state->best_encoder); + conn_state->best_encoder = NULL; } crtc_state->active = false; if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) diff --git a/drivers/gpu/drm/apple/apple_resume_test.c b/drivers/gpu/drm/apple/apple_resume_test.c index 2dd1431bc065f..94f539986e663 100644 --- a/drivers/gpu/drm/apple/apple_resume_test.c +++ b/drivers/gpu/drm/apple/apple_resume_test.c @@ -179,10 +179,23 @@ static void resume_stale_mode_fails_unfiltered_test(struct kunit *test) static void resume_stale_crtc_disabled_test(struct kunit *test) { struct resume_test *priv = resume_test_init(test); + struct drm_connector_state *conn_state; + struct drm_crtc_state *crtc_state; + struct drm_atomic_state *state; KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); priv->admissible = false; - apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + state = priv->drm.mode_config.suspend_state; + apple_drm_drop_stale_crtcs(state); + + /* The resume check warns about an encoder routed without a CRTC. */ + conn_state = drm_atomic_get_new_connector_state(state, &priv->connector); + crtc_state = drm_atomic_get_new_crtc_state(state, priv->crtc); + KUNIT_ASSERT_NOT_NULL(test, conn_state); + KUNIT_ASSERT_NOT_NULL(test, crtc_state); + KUNIT_EXPECT_NULL(test, conn_state->best_encoder); + KUNIT_EXPECT_EQ(test, crtc_state->encoder_mask, 0U); + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); From 813f2a17c2a481f6fda22a2861c80f4498381d87 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:23:56 +1100 Subject: [PATCH 0994/1579] drm/apple: Have userspace restore a CRTC resumed disabled A CRTC whose mode needs a link that system sleep released is restored disabled, on the expectation that the hotplug following the relink makes userspace enable it again. Nothing guarantees that hotplug: the DCP reports a display that comes back on the same cable only for a new cable or the first attachment, and the 180 Hz mode is announced again only if its rate proof is renewed. Even when a hotplug arrives, the connector is still connected with the same modes, so a compositor that tracks its own state sees no change, believes the output is still lit, and has every flip rejected on the disabled CRTC. Use the link-status contract instead: mark the link of each connector detached from such a CRTC bad in the state being restored, and send a hotplug event once resume has committed it. Userspace then sets the mode again, with whatever modes the connector offers at that point, as it does after a failed link retraining. Fixes: 229d38cd44fd ("drm/apple: Resume a CRTC whose mode needs a new link disabled") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 6 +++++- drivers/gpu/drm/apple/apple_resume.c | 19 ++++++++++++++----- drivers/gpu/drm/apple/apple_resume.h | 4 +++- drivers/gpu/drm/apple/apple_resume_test.c | 14 +++++++++++--- 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 2d6173ed9472e..48167f356890c 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -931,11 +931,15 @@ static int apple_platform_suspend(struct device *dev) static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); + bool dropped = false; if (apple) { if (apple->drm.mode_config.suspend_state) - apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); + dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); drm_mode_config_helper_resume(&apple->drm); + /* Have userspace set the mode of a CRTC resumed disabled again. */ + if (dropped) + drm_kms_helper_hotplug_event(&apple->drm); } return 0; diff --git a/drivers/gpu/drm/apple/apple_resume.c b/drivers/gpu/drm/apple/apple_resume.c index ec49db7d1d924..52dabd11d3e51 100644 --- a/drivers/gpu/drm/apple/apple_resume.c +++ b/drivers/gpu/drm/apple/apple_resume.c @@ -19,13 +19,17 @@ * sleep releases that link. Restoring such a CRTC as it was fails the whole * resume commit in its mode fixup, which leaves every output dark until * userspace reacts to the next hotplug. Restore that CRTC disabled instead. - * The DCP reports the display again once its link is back, and userspace - * re-enables it on that hotplug, as it does after a cable is replugged. + * Userspace still believes the CRTC is on, and the DCP need not report the + * same display again when its link comes back, so mark the link of each of + * its connectors bad: userspace then sets the mode again, on the hotplug + * event the caller sends, as after a failed link retraining. * * The suspend state holds every CRTC, plane and connector already, so it is * edited directly rather than through helpers that need the modeset locks. + * + * Returns true if a CRTC is to be resumed disabled. */ -void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) +bool apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) { const struct drm_crtc_helper_funcs *funcs; struct drm_connector_state *conn_state; @@ -34,6 +38,7 @@ void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) struct drm_connector *connector; struct drm_plane *plane; struct drm_crtc *crtc; + bool dropped = false; int i, j; for_each_new_crtc_in_state(state, crtc, crtc_state, i) { @@ -59,7 +64,7 @@ void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) continue; /* Detaching from a CRTC already in the state cannot fail. */ if (WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) - return; + return true; /* * Nor does a connector without a CRTC keep an encoder: * the resume check warns about one left routed. @@ -68,9 +73,13 @@ void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) crtc_state->encoder_mask &= ~drm_encoder_mask(conn_state->best_encoder); conn_state->best_encoder = NULL; + conn_state->link_status = DRM_LINK_STATUS_BAD; } crtc_state->active = false; + dropped = true; if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) - return; + break; } + + return dropped; } diff --git a/drivers/gpu/drm/apple/apple_resume.h b/drivers/gpu/drm/apple/apple_resume.h index 313e02c12b064..de0dffe2dc8bd 100644 --- a/drivers/gpu/drm/apple/apple_resume.h +++ b/drivers/gpu/drm/apple/apple_resume.h @@ -4,8 +4,10 @@ #ifndef __APPLE_RESUME_H__ #define __APPLE_RESUME_H__ +#include + struct drm_atomic_state; -void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state); +bool apple_drm_drop_stale_crtcs(struct drm_atomic_state *state); #endif /* __APPLE_RESUME_H__ */ diff --git a/drivers/gpu/drm/apple/apple_resume_test.c b/drivers/gpu/drm/apple/apple_resume_test.c index 94f539986e663..7f89c07c15294 100644 --- a/drivers/gpu/drm/apple/apple_resume_test.c +++ b/drivers/gpu/drm/apple/apple_resume_test.c @@ -186,7 +186,8 @@ static void resume_stale_crtc_disabled_test(struct kunit *test) KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); priv->admissible = false; state = priv->drm.mode_config.suspend_state; - apple_drm_drop_stale_crtcs(state); + /* The caller tells userspace about the CRTC it had and lost. */ + KUNIT_EXPECT_TRUE(test, apple_drm_drop_stale_crtcs(state)); /* The resume check warns about an encoder routed without a CRTC. */ conn_state = drm_atomic_get_new_connector_state(state, &priv->connector); @@ -205,6 +206,9 @@ static void resume_stale_crtc_disabled_test(struct kunit *test) KUNIT_EXPECT_NULL(test, priv->plane->state->crtc); KUNIT_EXPECT_NULL(test, priv->plane->state->fb); KUNIT_EXPECT_NULL(test, priv->connector.state->crtc); + /* Userspace must modeset again rather than flip onto a dead CRTC. */ + KUNIT_EXPECT_EQ(test, priv->connector.state->link_status, + DRM_LINK_STATUS_BAD); /* Only the creator's reference is left: the saved scanout was dropped. */ KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 1U); } @@ -214,7 +218,8 @@ static void resume_admissible_crtc_restored_test(struct kunit *test) struct resume_test *priv = resume_test_init(test); KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); - apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_FALSE(test, + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state)); KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); KUNIT_EXPECT_TRUE(test, priv->crtc->state->enable); @@ -251,10 +256,13 @@ static void resume_stale_dpms_off_crtc_disabled_test(struct kunit *test) KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); priv->admissible = false; - apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_TRUE(test, + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state)); KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); KUNIT_EXPECT_NULL(test, priv->connector.state->crtc); + KUNIT_EXPECT_EQ(test, priv->connector.state->link_status, + DRM_LINK_STATUS_BAD); KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 1U); } From 03426fb46c9ce29e5b58db52cfefd09fac156fea Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:15:37 +1100 Subject: [PATCH 0995/1579] soc: apple: mailbox, rtkit: Allow holding received messages The receive interrupt of an ASC mailbox is requested IRQF_NO_SUSPEND so that RTKit co-processors stay serviceable during system sleep. A co-processor that keeps sending while the system sleeps then wakes a CPU for every message. Add apple_mbox_hold_rx() and apple_mbox_release_rx(), which mask and unmask the receive interrupt so that messages stay queued in the hardware FIFO, and the RTKit wrappers apple_rtkit_hold_rx() and apple_rtkit_resume_rx() for co-processor drivers. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mailbox.c | 12 ++++++++++++ drivers/soc/apple/rtkit.c | 12 ++++++++++++ include/linux/soc/apple/mailbox.h | 6 ++++++ include/linux/soc/apple/rtkit.h | 7 +++++++ 4 files changed, 37 insertions(+) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 229ebcf90c788..9c448ec1fa9ec 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -512,6 +512,18 @@ void apple_mbox_stop(struct apple_mbox *mbox) } EXPORT_SYMBOL(apple_mbox_stop); +void apple_mbox_hold_rx(struct apple_mbox *mbox) +{ + disable_irq(mbox->irq_recv_not_empty); +} +EXPORT_SYMBOL(apple_mbox_hold_rx); + +void apple_mbox_release_rx(struct apple_mbox *mbox) +{ + enable_irq(mbox->irq_recv_not_empty); +} +EXPORT_SYMBOL(apple_mbox_release_rx); + /* * Stop the provider-owned CPU, then quiesce every transport edge before * devres releases IRQs or MMIO mappings. Device links order consumer removal diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index b4054ec0dfc03..3721c529de915 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -743,6 +743,18 @@ int apple_rtkit_poll(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_poll); +void apple_rtkit_hold_rx(struct apple_rtkit *rtk) +{ + apple_mbox_hold_rx(rtk->mbox); +} +EXPORT_SYMBOL_GPL(apple_rtkit_hold_rx); + +void apple_rtkit_resume_rx(struct apple_rtkit *rtk) +{ + apple_mbox_release_rx(rtk->mbox); +} +EXPORT_SYMBOL_GPL(apple_rtkit_resume_rx); + bool apple_rtkit_has_endpoint(struct apple_rtkit *rtk, u8 ep) { return test_bit(ep, rtk->endpoints); diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index e5697975a50b5..8813438326ffc 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -93,6 +93,12 @@ struct apple_mbox *apple_mbox_get_byname(struct device *dev, const char *name); int apple_mbox_start(struct apple_mbox *mbox); void apple_mbox_stop(struct apple_mbox *mbox); int apple_mbox_poll(struct apple_mbox *mbox); +/* + * Leave received messages in the hardware FIFO, without an interrupt, until + * apple_mbox_release_rx(). For consumers that must not wake the system. + */ +void apple_mbox_hold_rx(struct apple_mbox *mbox); +void apple_mbox_release_rx(struct apple_mbox *mbox); int apple_mbox_send(struct apple_mbox *mbox, struct apple_mbox_msg msg, bool atomic); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index e4b38a8f6e1ba..00b9a21e936d7 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -239,6 +239,13 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, */ int apple_rtkit_poll(struct apple_rtkit *rtk); +/* + * Stop taking interrupts for messages from the co-processor, which then + * stay queued in the mailbox until apple_rtkit_resume_rx(). + */ +void apple_rtkit_hold_rx(struct apple_rtkit *rtk); +void apple_rtkit_resume_rx(struct apple_rtkit *rtk); + /* * Checks if an endpoint with a given index exists * From 1efb7996cdcf6fc12b15a79cad905ff78f04417f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:15:37 +1100 Subject: [PATCH 0996/1579] drm/apple: Hold internal DCP messages during system sleep DPMS off on the internal H17P panel is a brightness present and leaves the pipe scanning out. While it scans out, the firmware publishes a property to the AP on every frame (D300 pr_publish, id 108, about 60 per second), and the DCP mailbox interrupt is IRQF_NO_SUSPEND, so each of them wakes a CPU out of suspend-to-idle: a 60 s RTC sleep took about 3650 interrupts from 412e08000.mailbox-recv, a 360 s one 21676. Hold the coprocessor's messages in the mailbox as the last step of the DCP suspend handler, after presentation is paused and the command queue has settled, and process them first thing on resume. A 360 s sleep now takes 97 interrupts, all of them in the transitions, and the panel and the firmware carry on normally after it. apple-drm is a device-link consumer of each DCP, so its suspend handler has blanked the panel before this runs and its resume handler restores the CRTC after the messages are processed. Powering the pipe off with A473 or idling the coprocessor through RTKit instead are not usable: the first left the firmware crashed after the power on, and the firmware does not acknowledge the second. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 37 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 95881b9cb79fe..169b2d17af867 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -344,6 +344,8 @@ struct apple_dcp { bool active; /* A successful initial enable keeps the H17P pipe running across DPMS. */ bool pipe_enabled_h17p; + /* Messages from the coprocessor are held in the mailbox for sleep. */ + bool rx_held; /* eDP display without DP-HDMI conversion */ bool main_display; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index ca5988f81fe20..1ae27015d6724 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4629,6 +4629,40 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) dev_warn(dcp->dev, "brightness change still pending at suspend\n"); } +/* + * DPMS off leaves the internal H17P pipe scanning out (see dcp_poweroff()), + * and while it scans out the firmware publishes a property to the AP on every + * frame. The mailbox interrupt is not suspended, so each of those messages + * wakes a CPU out of suspend-to-idle about 60 times a second. Hold the + * messages in the mailbox for system sleep and process them at resume. + * apple-drm is a consumer of this device, so its suspend handler has blanked + * the panel before this runs and its resume handler restores the CRTC after. + */ +static void dcp_hold_for_sleep(struct apple_dcp *dcp) +{ + if (!dcp_uses_soft_dpms(dcp) || dcp->fw_compat != DCP_FIRMWARE_H17P || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || + READ_ONCE(dcp->crashed)) + return; + + iomfb_pause_presentation(dcp, true); + if (iomfb_queue_settle(dcp, msecs_to_jiffies(500))) { + iomfb_pause_presentation(dcp, false); + return; + } + apple_rtkit_hold_rx(dcp->rtk); + dcp->rx_held = true; +} + +static void dcp_release_after_sleep(struct apple_dcp *dcp) +{ + if (!dcp->rx_held) + return; + dcp->rx_held = false; + apple_rtkit_resume_rx(dcp->rtk); + iomfb_pause_presentation(dcp, false); +} + static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); @@ -4672,6 +4706,8 @@ static int dcp_platform_suspend(struct device *dev) * shutdown properly yet. */ device_set_wakeup_path(dev); + /* Last: the steps above still talk to the coprocessor. */ + dcp_hold_for_sleep(dcp); return 0; } @@ -4680,6 +4716,7 @@ static int dcp_platform_resume(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); + dcp_release_after_sleep(dcp); dcp_enable_typec_work(dcp); if (dcp->hdmi_hpd_irq) enable_irq(dcp->hdmi_hpd_irq); From e786a6a78a972597b763c82e3968fea5874452df Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:32:20 +1100 Subject: [PATCH 0997/1579] drm/apple, usb: typec: tipd: Retry a display mux that is only busy The CD321x driver treats -EBUSY from typec_mux_set() as "the shared external display is driven by the other port": it sets up the port's USB path without the display and does not retry. -EBUSY has other sources on the same mux chain, though. The Apple Type-C PHY returns it while a mode or pipe transition is pending, and the DCP route returned it while the previous SAFE transition was still running and while the external DCP had not finished stopping. Those conditions clear on their own, but the port was left without its display until the next cable event. Have the DCP route report the lasting condition, every pipeline driving another port's display, as -ENOSPC, and the transient ones as -EAGAIN. The CD321x driver now skips the retry only for -ENOSPC, through a helper with a KUnit case; -EBUSY and -EAGAIN are retried like any other failure, -EAGAIN without counting against the port. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 12 +++++++----- drivers/usb/typec/tipd/cd321x-pm-test.c | 12 ++++++++++++ drivers/usb/typec/tipd/cd321x-pm.h | 13 +++++++++++++ drivers/usb/typec/tipd/core.c | 4 ++-- 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 1ae27015d6724..f60fcb3698e6e 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -464,7 +464,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) guard(mutex)(&dcp->external_lock); if (dcp_is_external_h17p(dcp) && (dcp->external_error || (dcp->external_power_attempted && !dcp->external_stopped))) - return dcp->external_error ?: -EBUSY; + return dcp->external_error ?: -EAGAIN; dcp_firmware_metadata_connection(dcp, READ_ONCE(dcp->typec_generation), false); @@ -973,7 +973,7 @@ static int dcp_typec_route_prepare(struct typec_mux_dev *mux, guard(mutex)(&dcp_typec_fabric_lock); if (state->mode != TYPEC_STATE_SAFE) - return port->safe_pending ? -EBUSY : 0; + return port->safe_pending ? -EAGAIN : 0; if (!port->owner || port->owner->tunnel) return 0; dcp = port->owner->dcp; @@ -1060,8 +1060,9 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (state->mode == TYPEC_STATE_SAFE && port->safe_pending) return 0; + /* The previous SAFE transition has not finished; retry. */ if (port->safe_pending) - return -EBUSY; + return -EAGAIN; /* * Every candidate route for this port is notified with the same state, @@ -1154,7 +1155,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, dcp_typec_rebalance_locked(NULL, 0); if (!port->owner) { if (hpd) - return -EBUSY; + return -ENOSPC; port->applied_valid = true; return 0; } @@ -1173,8 +1174,9 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (!best) best = dcp_typec_lowest_free(port); + /* Every pipeline is driving another port's display. */ if (!best) - return -EBUSY; + return -ENOSPC; ret = dcp_typec_route_activate(best, best->xbar); if (ret) diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index de9db7437dce2..f94f4ad57cb75 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -417,6 +417,17 @@ static void cd321x_busy_provider_keeps_real_failure_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, pm.force_reconnect); } +/* Only a pipeline owned by another port skips the retry. */ +static void cd321x_display_taken_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, cd321x_display_taken(-ENOSPC)); + /* A PHY mid-transition or a pending SAFE state must be retried. */ + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-EBUSY)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-EAGAIN)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-ETIMEDOUT)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(0)); +} + static struct kunit_case cd321x_pm_cases[] = { KUNIT_CASE(cd321x_busy_provider_preserves_connection_test), KUNIT_CASE(cd321x_busy_provider_completion_reopens_test), @@ -440,6 +451,7 @@ static struct kunit_case cd321x_pm_cases[] = { KUNIT_CASE(cd321x_read_and_apply_share_budget_test), KUNIT_CASE(cd321x_status_event_after_exhaustion_test), KUNIT_CASE(cd321x_runtime_failure_revalidates_test), + KUNIT_CASE(cd321x_display_taken_test), {} }; diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h index a96392f88bee4..08421d54c570b 100644 --- a/drivers/usb/typec/tipd/cd321x-pm.h +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -2,6 +2,8 @@ #ifndef __CD321X_PM_H__ #define __CD321X_PM_H__ +#include + #include #define CD321X_RESUME_ATTEMPTS 3 @@ -140,4 +142,15 @@ static inline void cd321x_pm_remove(struct cd321x_pm_state *pm) pm->attempts_left = 0; } +/* + * A display mux reports -ENOSPC when every display pipeline is driving + * another port. That lasts until the other display goes away, so the port's + * USB path is set up without the display and nothing is retried. Any other + * error, such as -EBUSY from a PHY in the middle of a transition, is retried. + */ +static inline bool cd321x_display_taken(int ret) +{ + return ret == -ENOSPC; +} + #endif /* __CD321X_PM_H__ */ diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index d498bb412943c..ca6cc92c8b425 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -692,7 +692,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, if (ret) goto out; ret = typec_mux_set(cd321x->mux, &cd321x->state); - *display_busy = ret == -EBUSY; + *display_busy = cd321x_display_taken(ret); if (!ret) { cd321x->dp_status = dp_data.status; cd321x->dp_conf = dp_data.conf; @@ -818,7 +818,7 @@ static void cd321x_update_work(struct work_struct *work) * a failed teardown keeps the route, but the sink must not stay present. * All ports reference the same external display connector, so only the * port that owns the display route may report a disconnect. A second DP - * sink can enter Alt Mode while its mux request fails with -EBUSY, and + * sink can enter Alt Mode while its mux request fails with -ENOSPC, and * must not tear down the display carried by the other port. */ if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && From 0833360ca84c3882cf0246185276bd1dbc067b7f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:34:20 +1100 Subject: [PATCH 0998/1579] drm/apple: Log routine DPTX and EDID service events at debug level Every hotplug, link training and resume of the external display logged a dozen info lines: each DPTX port call (SET_LINK_RATE, GET_SUPPORTS_HPD, DEACTIVATE, FORCE_HOTPLUG_DETECT, INACTIVE_SINK_DETECTED, the lane count calls and the validate_connection attributes), the AV service announcing its EDID channel, every connect attempt, every boot property and read_edt_data request, and "copy_edid failed: -ENODEV" whenever the EDID was probed while the link was down. None of them is a problem; they are the normal protocol flow. Log them with dev_dbg() instead, so dynamic debug still shows them. -ENODEV from the EDID copy now means no EDID source for any connector, not only the internal panel. A copy or publication failure with any other error is a real problem and is now a warning. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 12 +++++------ drivers/gpu/drm/apple/dptxep.c | 27 ++++++++++++------------- drivers/gpu/drm/apple/epic/dpavservep.c | 4 ++-- drivers/gpu/drm/apple/iomfb.c | 11 +++++----- drivers/gpu/drm/apple/iomfb_template.c | 8 ++++---- 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f60fcb3698e6e..93787252badd4 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2499,12 +2499,12 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_s dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); return -ENODEV; } - dev_info(dcp->dev, - "%s(port=%d) target=%u:%u typec=%d route=%s conn_type=%d connected=%d\n", - __func__, port, dcp->dptx_die, dcp->dptx_phy, - dcp_is_typec_output(dcp), - dcp->active_typec_route ? "borrowed" : "fixed", - dcp->connector_type, dcp->dptxport[port].connected); + dev_dbg(dcp->dev, + "%s(port=%d) target=%u:%u typec=%d route=%s conn_type=%d connected=%d\n", + __func__, port, dcp->dptx_die, dcp->dptx_phy, + dcp_is_typec_output(dcp), + dcp->active_typec_route ? "borrowed" : "fixed", + dcp->connector_type, dcp->dptxport[port].connected); mutex_lock(&dcp->hpd_mutex); if (!dptx_service_available(&dcp->dptxport[port])) { diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 7bc55bfffe213..1e06d9185fa5e 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -152,8 +152,8 @@ int dptxport_validate_connection(struct apple_epic_service *service, u8 core, le32_to_cpu(resp.unk), attrs); return -EINVAL; } - dev_info(service->ep->dcp->dev, - "validate_connection: attrs reply 0x%x (sent 0x%x)\n", + dev_dbg(service->ep->dcp->dev, + "validate_connection: attrs reply 0x%x (sent 0x%x)\n", le32_to_cpu(resp.unk), attrs); } @@ -373,7 +373,7 @@ static int dptxport_call_get_max_lane_count(struct apple_epic_service *service, dptx->lane_count = 4; reply->retcode = cpu_to_le32(0); reply->lane_count = cpu_to_le64(4); - dev_info(dcp->dev, "get_max_lane_count: USB4 DP IN, 4 lanes\n"); + dev_dbg(dcp->dev, "get_max_lane_count: USB4 DP IN, 4 lanes\n"); return 0; } @@ -455,8 +455,8 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic reply->lane_count = cpu_to_le64(lane_count); if (lane_count > 0) { - dev_info(dcp->dev, "USB4/DPTX: SET_ACTIVE_LANE_COUNT %llu\n", - lane_count); + dev_dbg(dcp->dev, "USB4/DPTX: SET_ACTIVE_LANE_COUNT %llu\n", + lane_count); /* * dcp_usb4_drm_allowed() (usb4_force_dptx) used to gate this * for the old manual-training sysfs knob (module_param_cb @@ -545,8 +545,8 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, link_rate = le32_to_cpu(request->link_rate); trace_dptxport_call_set_link_rate(dptx, link_rate); - dev_info(service->ep->dcp->dev, "DPTXPort: SET_LINK_RATE 0x%x\n", - link_rate); + dev_dbg(service->ep->dcp->dev, "DPTXPort: SET_LINK_RATE 0x%x\n", + link_rate); switch (link_rate) { case LINK_RATE_RBR: @@ -624,8 +624,8 @@ static int dptxport_call_get_supports_hpd(struct apple_epic_service *service, * that path instead of waiting for a PHY start. */ reply->supported = cpu_to_le32(dcp_is_typec_output(dcp) ? 1 : 0); - dev_info(dcp->dev, "DPTXPort: GET_SUPPORTS_HPD %u usb4=%d\n", - le32_to_cpu(reply->supported), dcp_is_usb4_output(dcp)); + dev_dbg(dcp->dev, "DPTXPort: GET_SUPPORTS_HPD %u usb4=%d\n", + le32_to_cpu(reply->supported), dcp_is_usb4_output(dcp)); return 0; } @@ -692,7 +692,7 @@ dptxport_call_deactivate(struct apple_epic_service *service, struct apple_dcp *dcp = service->ep->dcp; dcp_180hz_invalidate(dcp); - dev_info(dcp->dev, "DPTXPort: DEACTIVATE\n"); + dev_dbg(dcp->dev, "DPTXPort: DEACTIVATE\n"); if (dcp->dptx_tunnel) dcp_tunnel_dpin_activate(dcp, false); if (dptx->atcphy && !dcp->phy_managed_by_typec) @@ -825,8 +825,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, return dptxport_call_deactivate(service, data, data_size, reply, reply_size); case DPTX_APCALL_FORCE_HOTPLUG_DETECT: - dev_info(service->ep->dcp->dev, - "DPTXPort: FORCE_HOTPLUG_DETECT\n"); + dev_dbg(service->ep->dcp->dev, "DPTXPort: FORCE_HOTPLUG_DETECT\n"); complete(&dptx->linkcfg_completion); memcpy(reply, data, min(reply_size, data_size)); if (reply_size >= 4) @@ -838,8 +837,8 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, * SET_ACTIVE_LANE_COUNT. Treating this as failure made * firmware DEACTIVATE after it had already set lanes/rate. */ - dev_info(service->ep->dcp->dev, - "DPTXPort: INACTIVE_SINK_DETECTED (keep waiting for lanes)\n"); + dev_dbg(service->ep->dcp->dev, + "DPTXPort: INACTIVE_SINK_DETECTED (keep waiting for lanes)\n"); memcpy(reply, data, min(reply_size, data_size)); if (reply_size >= 4) memset(reply, 0, 4); diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index a61d28e5e2edb..2476209719c08 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -135,8 +135,8 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, return; } if (v2) { - dev_info(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", - service->channel); + dev_dbg(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", + service->channel); dcpavserv_notify_edid(service); } } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 1027af19ab49e..1ab457b55b464 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1179,13 +1179,14 @@ int dcp_get_modes(struct drm_connector *connector) dcp_consume_edid_invalidation(apple_connector); if (IS_ERR_OR_NULL(edid)) { /* - * An internal panel has no AV service and so no EDID; + * There is no AV service for an internal panel, nor + * for an external display while its link is down; * do not report that on every mode probe. */ - if (dcp_has_panel(dcp) && PTR_ERR(edid) == -ENODEV) - dev_dbg(dcp->dev, "no EDID source for the panel\n"); + if (PTR_ERR(edid) == -ENODEV) + dev_dbg(dcp->dev, "no EDID source\n"); else - dev_info(dcp->dev, "copy_edid failed: %pe\n", edid); + dev_warn(dcp->dev, "copy_edid failed: %pe\n", edid); } else { drm_edid_free(apple_connector->drm_edid); apple_connector->drm_edid = edid; @@ -1197,7 +1198,7 @@ int dcp_get_modes(struct drm_connector *connector) int update = drm_edid_connector_update(connector, apple_connector->drm_edid); if (update) { - dev_info(dcp->dev, "raw EDID publication failed: %d\n", update); + dev_warn(dcp->dev, "raw EDID publication failed: %d\n", update); dcp_clear_connector_edid(apple_connector); } else { apple_connector->drm_edid_firmware_derived = false; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 88d54cdd1756f..e3d5240b38d29 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -877,8 +877,8 @@ dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) { /* Observe boot-property requests without inventing firmware timings. */ if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) - dev_info(dcp->dev, - "read_edt_data key=%.*s count=%u default0=%#x ret=0\n", + dev_dbg(dcp->dev, + "read_edt_data key=%.*s count=%u default0=%#x ret=0\n", (int)sizeof(req->key), req->key, req->count, req->value[0]); @@ -1109,8 +1109,8 @@ static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, u8 resp = dcpep_process_chunks(dcp, req); if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) - dev_info(dcp->dev, - "DCP property key=%.*s bytes=%zu accepted=%u nr_modes=%u\n", + dev_dbg(dcp->dev, + "DCP property key=%.*s bytes=%zu accepted=%u nr_modes=%u\n", (int)sizeof(req->key), req->key, dcp->chunks.length, resp, dcp->nr_modes); From 47ed92814aac93717d691d5abbf2df41d5f8f4b5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:35:16 +1100 Subject: [PATCH 0999/1579] drm/apple: Describe the H17P support without bring-up terms Comments, function names and a few messages in the H17P support still use the language of its bring-up: "measured" profiles, "qualified" loaders and sequences, an "unqualified" callback, the "USB4 PHY experiment". They tell a reader nothing beyond the code next to them, and "unqualified" in an error suggests a state the driver could get out of rather than an unsupported request. Say what is meant instead: the H17P profile, a known or unknown sequence, an unsupported callback or request, the J700 defaults. Rename dptxport_call_qualified_config() and dptxport_qualified_teardown() after what they do. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-180hz-policy.h | 2 +- drivers/gpu/drm/apple/dcp.c | 6 +++--- drivers/gpu/drm/apple/dptxep.c | 14 ++++++++------ drivers/gpu/drm/apple/iomfb.c | 2 +- drivers/gpu/drm/apple/iomfb_h17p.c | 16 ++++++++-------- drivers/gpu/drm/apple/iomfb_template.c | 12 ++++++------ drivers/gpu/drm/apple/plane.c | 2 +- 7 files changed, 28 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-180hz-policy.h b/drivers/gpu/drm/apple/dcp-180hz-policy.h index a221c0e4a0773..d23aa57c685e2 100644 --- a/drivers/gpu/drm/apple/dcp-180hz-policy.h +++ b/drivers/gpu/drm/apple/dcp-180hz-policy.h @@ -6,7 +6,7 @@ #include #include -/* Only this qualified uncompressed configuration is admitted. */ +/* Only this verified uncompressed configuration is admitted. */ static inline bool dcp_180hz_payload(u32 rate, u32 lanes, u32 clock) { return rate == 0x1e && lanes == 2 && clock == 414090 && diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 93787252badd4..8f601bbb91cbe 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4432,7 +4432,7 @@ static int dcp_platform_probe(struct platform_device *pdev) set_bit(surf, dcp->iomfb_surfaces); surf++; } - /* The qualified J700 loader names surface zero rather than enabling it. */ + /* The J700 loader names surface zero rather than enabling it. */ if (of_machine_is_compatible("apple,j700") && of_device_is_compatible(dev->of_node, "apple,t8140-dcp") && num_surfs == 1 && @@ -4571,7 +4571,7 @@ static void dcp_platform_shutdown(struct platform_device *pdev) dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { /* * This firmware keeps scanning across soft DPMS and has no - * qualified stop sequence. Component unbind releases the RTKit + * known stop sequence. Component unbind releases the RTKit * devres group and the piodma domain while firmware may still * access them. Stop host submissions but keep the component and * its DMA resources alive until the system resets. @@ -4741,7 +4741,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { .num_dptx_ports = 2, }; -/* The internal T8140 endpoint uses the measured H17P method profile. */ +/* The internal T8140 endpoint uses the H17P method profile. */ static const struct resource t8140_firmware_clock = DEFINE_RES_MEM(0x302800000, 0xbc000); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 1e06d9185fa5e..b9d4729407ed6 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -705,8 +705,8 @@ dptxport_call_deactivate(struct apple_epic_service *service, return 0; } -static int dptxport_call_qualified_config(struct apple_epic_service *service, bool lanes, - const void *data, size_t size, void *reply, size_t room) +static int dptxport_call_link_config(struct apple_epic_service *service, bool lanes, + const void *data, size_t size, void *reply, size_t room) { struct dcp_180hz_token token = dcp_180hz_config_begin(service, lanes); struct dptx_port *port = service->cookie; @@ -776,7 +776,8 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_GET_LINK_RATE: return dptxport_call_get_link_rate(service, reply, reply_size); case DPTX_APCALL_SET_LINK_RATE: - return dptxport_call_qualified_config(service, false, data, data_size, reply, reply_size); + return dptxport_call_link_config(service, false, data, data_size, + reply, reply_size); case DPTX_APCALL_GET_MAX_LANE_COUNT: return dptxport_call_get_max_lane_count(service, reply, reply_size); case DPTX_APCALL_GET_ACTIVE_LANE_COUNT: { @@ -789,7 +790,8 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, return 0; } case DPTX_APCALL_SET_ACTIVE_LANE_COUNT: - return dptxport_call_qualified_config(service, true, data, data_size, reply, reply_size); + return dptxport_call_link_config(service, true, data, data_size, + reply, reply_size); case DPTX_APCALL_GET_SUPPORTS_HPD: return dptxport_call_get_supports_hpd(service, reply, reply_size); @@ -974,7 +976,7 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, return 0; } -static void dptxport_qualified_teardown(struct apple_epic_service *service) +static void dptxport_teardown(struct apple_epic_service *service) { dcp_180hz_invalidate(service->ep->dcp); dptx_service_unbind(service); @@ -984,7 +986,7 @@ static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", .prepare = dptxport_prepare, - .teardown = dptxport_qualified_teardown, + .teardown = dptxport_teardown, .init = dptxport_init, .call = dptxport_call, .call_v2 = dptxport_group_call, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 1ab457b55b464..1c97639bb6b6c 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -901,7 +901,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && handled && !iomfb_check_callback_h17p(dcp, tag, hdr->in_len, hdr->out_len)))) { - dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", + dev_err(dev, "unsupported IOMFB callback %d (%u, %u)\n", tag, hdr->in_len, hdr->out_len); dcp->crashed = true; return; diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 45743b4173699..6c0b780674206 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -311,7 +311,7 @@ static bool trampoline_analytics_h17p(struct apple_dcp *dcp, int tag, return trampoline_zero(dcp, tag, out, in); trace_iomfb_callback(dcp, tag, __func__); - /* The dispatcher has validated the measured D114 reply size. */ + /* The dispatcher has validated the D114 reply size. */ memset(out, 0, 0x1004); *(u8 *)out = 'd'; return true; @@ -390,7 +390,7 @@ static const u8 j700_provider_data_4[] = { 0xa2, 0x19, 0x00, 0x00 }; -/* Only these provider transfers have been measured against the disp0 ADT. */ +/* Only these provider transfers are known to match the disp0 ADT. */ static const struct dcp_h17p_provider_property provider_properties[] = { { "power-lut-data-x", "apple,power-lut-data-x", 4, j700_provider_data_0 }, { "power-lut-data-y", "apple,power-lut-data-y", 4, j700_provider_data_1 }, @@ -434,13 +434,13 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, trace_iomfb_callback(dcp, tag, __func__); property = dcp_provider_property_h17p(in); if (!property) { - dev_err(dcp->dev, "unqualified D400 provider property request\n"); + dev_err(dcp->dev, "unsupported D400 provider property request\n"); goto fail; } length = of_property_count_u8_elems(node, property->dt_name); if (!property->length) { - /* Only this absent property has a measured empty reply. */ + /* Only this absent property has a known empty reply. */ if (of_property_present(node, property->dt_name)) goto fail; } else if (!of_property_present(node, property->dt_name) && @@ -448,12 +448,12 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, (of_device_is_compatible(node, "apple,t8140-dcp") || (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && of_device_is_compatible(node, "apple,t8140-dcpext")))) { - /* Qualified older loaders lack the newly described table properties. */ + /* Older loaders lack the newly described table properties. */ memset(reply, 0, sizeof(*reply)); memcpy(reply->data, property->legacy_data, property->length); reply->length = cpu_to_le32(property->length); dev_warn_once(dcp->dev, - "missing provider property %s (%u bytes); using qualified J700 compatibility data\n", + "missing provider property %s (%u bytes); using the J700 defaults\n", property->dt_name, property->length); return true; } else if (length != (int)property->length) { @@ -486,7 +486,7 @@ static bool trampoline_luma_span_h17p(struct apple_dcp *dcp, int tag, trace_iomfb_callback(dcp, tag, __func__); if (get_unaligned_le32(in)) { - dev_err(dcp->dev, "unqualified D574 range selector\n"); + dev_err(dcp->dev, "unsupported D574 range selector\n"); WRITE_ONCE(dcp->crashed, true); return false; } @@ -510,7 +510,7 @@ static bool trampoline_get_uint_prop_h17p(struct apple_dcp *dcp, int tag, strnlen(request->key, sizeof(request->key)) == sizeof(request->key) || strcmp(request->key, "Temperature") || request->value_null || memchr_inv(request->padding, 0, sizeof(request->padding))) { - dev_err(dcp->dev, "unqualified D401 integer property query\n"); + dev_err(dcp->dev, "unsupported D401 integer property query\n"); WRITE_ONCE(dcp->crashed, true); return false; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index e3d5240b38d29..fd985debcdbe5 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -459,7 +459,7 @@ static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *pr case IOMFB_PROPERTY_NITS: { #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - /* Measured H17P takeover comes from the powerlog hint interface. */ + /* The H17P takeover level comes from the powerlog hint interface. */ if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) break; #endif @@ -1407,7 +1407,7 @@ static void release_swap_cookie(struct kref *ref) * format two bytes late and its surface check rejects the swap as an * unsupported format. * - * H17G retains this public packing path. The measured H17P profile uses + * H17G retains this public packing path. The H17P profile uses * a separate wire record with the SPEC's 0x588 swap boundary. */ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) @@ -1430,7 +1430,7 @@ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) req->swap.h17p_pre_swap_id[0x00] = 2; req->swap.h17p_pre_swap_id[0x28] = 3; - /* The measured H17P layout is serialized without shifting its surfaces. */ + /* The H17P layout is serialized without shifting its surfaces. */ if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) return; @@ -1744,7 +1744,7 @@ static void complete_set_powerstate(struct apple_dcp *dcp, void *out, static bool dcp_power_stop_confirmed(struct dcp_wait_cookie *cookie) { - /* H17P stop semantics still require hardware qualification. */ + /* H17P does not confirm that scanout has stopped. */ #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER return false; #else @@ -1782,7 +1782,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) typeof(DCP_FW_UNION(dcp->swap)) *swap = &DCP_FW_UNION(dcp->swap); #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - /* The measured profile has no safe pipe-off or surface-free transition. */ + /* H17P has no known safe pipe-off or surface-free transition. */ if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) return; #endif @@ -1932,7 +1932,7 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) if (dcp->main_display) return; /* - * Report firmware hotplug independently of the USB4 PHY experiment. + * Report firmware hotplug independently of the USB4 PHY. * Reassigning lpdptxphy blanked eDP even with these callbacks ignored; * suppressing connector notifications does not protect the panel. * Mode probing still uses this DCP's firmware modes, and mode_valid diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index b128a7df971da..a94ea4441d04d 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -430,7 +430,7 @@ static const u32 dcp_primary_formats[] = { #endif }; -/* H17P 10-bit XRGB alpha semantics have not been qualified. */ +/* The H17P alpha semantics of 10-bit XRGB are unknown. */ static const u32 dcp_primary_formats_h17p[] = { DRM_FORMAT_ARGB2101010, DRM_FORMAT_XRGB8888, From c2b2c2626c9b6c12c58a328c1e00f70642537e13 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:11:02 +1100 Subject: [PATCH 1000/1579] phy: apple: atc: Quiet the DisplayPort AUX and tunnel clock logging atcphy_enable_dp_aux() logged "Applied T8140 DisplayPort setup lists" once per boot. It comes from enabling the AUX channel, not from where the setup lists are applied, and it reports nothing a user needs. atc_tunnel_start_t602x() logged four raw register values at info level on every tunnel clock preflight. They only help when a preflight is refused, so log them with dev_dbg(). No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index adb2ecdb310f6..6c74ad7595a26 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1744,10 +1744,8 @@ static int j700_aux_disable(struct apple_atcphy *atcphy) static int atcphy_enable_dp_aux(struct apple_atcphy *atcphy) { - if (atcphy->hw->has_t8140_dp) { - dev_info_once(atcphy->dev, "Applied T8140 DisplayPort setup lists\n"); + if (atcphy->hw->has_t8140_dp) return j700_aux_enable(atcphy); - } /* FIXME */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) @@ -2613,9 +2611,9 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy) outputs = readl(atcphy->regs.core + AUSPLL_CLKOUT_MASTER); command = readl(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE); status = readl(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT); - dev_info(atcphy->dev, - "USB4 tunnel clock preflight: +7000=%08x +2200=%08x +2000=%08x +7044=%08x\n", - gates, outputs, command, status); + dev_dbg(atcphy->dev, + "USB4 tunnel clock preflight: +7000=%08x +2200=%08x +2000=%08x +7044=%08x\n", + gates, outputs, command, status); /* * Before the first rate request, TX_DP_CTRL0 gate bits read 0xe001: * the enable gates are set, the byte-clock reset is asserted, and the From ce5577ad5457fb80906c8deed63f380ad7113c38 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:11:02 +1100 Subject: [PATCH 1001/1579] phy: apple: atc: Use the register field names in the tunnel clock code atc_tunnel_restore(), atc_tunnel_start_t602x() and the T602X tunnel rate code write raw masks where the fields already have names: 0x54 and BIT(2), BIT(4), BIT(6) are the AUSPLL_CLKOUT_MASTER output drivers, the ACIOPHY_SLEEP_CTRL values are the TX_SMALL, TX_BIG and TX_CLAMP fields and their overrides, and 0xc00, 0xc0 and 0xc000 in ACIOPHY_CFG0 are RX_BIG, RX_SMALL and RX_CLAMP. The comments calling the last three "additional override bits" of unconfirmed meaning were wrong; the sequence is the same one atc_tunnel_wake_t8103() writes for the TX and RX lanes. Use the names. No functional change: every write is the same value. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 6c74ad7595a26..e8568092bed72 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2546,9 +2546,9 @@ static void atc_tunnel_restore(struct apple_atcphy *atcphy) */ core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE | DPTX_PCLK2_ENABLE); - core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(2)); - core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(4)); - core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, BIT(6)); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_PCLK_DRVR_EN); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_PCLK2_DRVR_EN); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_REFBUFCLK_DRVR_EN); ret = atc_tunnel_command(atcphy, 3); if (ret) dev_warn(atcphy->dev, "USB4 tunnel clock: AUSPLL power-down was not acked\n"); @@ -2621,7 +2621,7 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy) * state only with no enabled PLL output, no outstanding command and * no lock. Any other active gate state is refused. */ - if (outputs & 0x54 || command & AUSPLL_APB_CMD_OVERRIDE_REQ || + if (outputs & AUSPLL_CLKOUT_MASTER_DRVR_EN || command & AUSPLL_APB_CMD_OVERRIDE_REQ || status & ACIOPHY_AUSPLL_LOCK) return -EBUSY; if ((gates & (DPTX_PCLK1_ENABLE | DPTX_PCLK2_ENABLE | DPRX_PCLK_ENABLE)) && @@ -2647,25 +2647,24 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy) core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP_OV); udelay(2); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x30); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0xc0); + /* same for the TX lanes */ + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL_OV); udelay(2); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x03); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x0c); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG_OV); udelay(2); - core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, 0x300); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0xc00); + core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV); udelay(2); - /* Additional override bit alongside RX_BIG_OV; exact meaning not independently confirmed. */ - core_set32(atcphy, ACIOPHY_CFG0, 0xc00); + /* and for the RX lanes */ + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG_OV); udelay(2); - /* Additional override bit alongside RX_SMALL_OV; exact meaning not independently confirmed. */ - core_set32(atcphy, ACIOPHY_CFG0, 0xc0); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL_OV); udelay(2); - /* Additional override bit alongside RX_CLAMP_OV; exact meaning not independently confirmed. */ - core_clear32(atcphy, ACIOPHY_CFG0, 0xc000); + core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP_OV); udelay(2); core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(2)); @@ -2733,7 +2732,7 @@ static void atc_tunnel_stop_t602x(struct apple_atcphy *atcphy, unsigned int dpin * them. A mode change still powers it down in atc_tunnel_restore(). */ if (!atcphy->tunnel_users && atcphy->tunnel_saved) - core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_DRVR_EN); } /* Run one T602X DP IN adapter's pixel clock at @rate; atcphy->lock held. */ @@ -2758,12 +2757,13 @@ static int atc_tunnel_set_t602x(struct apple_atcphy *atcphy, unsigned int dpin, * its descriptor until the USB4 PHY changes mode instead of * reprogramming a still-locked PLL on every DPMS wake. */ - core_set32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_DRVR_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, value, value & ACIOPHY_AUSPLL_LOCK, 10, 10000); if (ret) - core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, 0x54); + core_clear32(atcphy, AUSPLL_CLKOUT_MASTER, + AUSPLL_CLKOUT_MASTER_DRVR_EN); } if (ret) return ret; From 5fe0c2d6c2ed688a3a9874546885df537fdfd606 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:11:02 +1100 Subject: [PATCH 1002/1579] phy: apple: atc: Share the tunnel rate selector table atc_tunnel_start_t8103() mapped the DP link rate code to the pixel clock selector with its own switch, a copy of atc_tunnel_rate_selector() used by the T602X code. Move the helper up and use it for both. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 46 +++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index e8568092bed72..8dbb319184bd2 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2397,28 +2397,32 @@ static void atc_tunnel_wake_t8103(struct apple_atcphy *atcphy) udelay(2); } -static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) +/* The pixel clock selector for a DP link rate code. */ +static int atc_tunnel_rate_selector(u8 rate) { - u32 selector, value; - int ret; - - lockdep_assert_held(&atcphy->lock); switch (rate) { case 0x06: /* RBR */ - selector = 4; - break; + return 4; case 0x0a: /* HBR */ - selector = 3; - break; + return 3; case 0x14: /* HBR2 */ - selector = 1; - break; + return 1; case 0x1e: /* HBR3 */ - selector = 0; - break; + return 0; default: return -EINVAL; } +} + +static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) +{ + int selector, ret; + u32 value; + + lockdep_assert_held(&atcphy->lock); + selector = atc_tunnel_rate_selector(rate); + if (selector < 0) + return selector; if (atcphy->tunnel_clock_on) { if (atcphy->tunnel_rate == rate) return 0; @@ -2574,22 +2578,6 @@ static void atc_tunnel_restore(struct apple_atcphy *atcphy) atcphy->tunnel_rate = 0; } -static int atc_tunnel_rate_selector(u8 rate) -{ - switch (rate) { - case 0x06: - return 4; - case 0x0a: - return 3; - case 0x14: - return 1; - case 0x1e: - return 0; - default: - return -EINVAL; - } -} - /* * Bring the T602X tunnel PLL up with its fixed descriptor. The rate is not * part of it: each DP IN adapter selects its own rate on PCLK1 or PCLK2 in From 1a8a7d1e7d1cfe8966a288cd9c57763a73360381 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:12:38 +1100 Subject: [PATCH 1003/1579] phy: apple: atc: Fix coding style in the AUX and tunnel clock code Align the readl_poll_timeout() continuations in the tunnel clock code with the open parenthesis and drop a doubled blank line after the J700 AUX register definitions, as checkpatch asks. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 8dbb319184bd2..a0ef5ad76a917 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1695,7 +1695,6 @@ static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode #define J700_AUX_CLAMP_SELECT BIT(8) #define J700_AUX_CLAMP_ENABLE BIT(9) - static int j700_aux_enable(struct apple_atcphy *atcphy) { u32 clamp = J700_AUX_CLAMP_SELECT | J700_AUX_CLAMP_ENABLE; @@ -2527,7 +2526,7 @@ static int atc_tunnel_command(struct apple_atcphy *atcphy, u32 command) FIELD_PREP(AUSPLL_APB_CMD_OVERRIDE_CMD, command) | AUSPLL_APB_CMD_OVERRIDE_REQ | AUSPLL_APB_CMD_OVERRIDE_UNK28); ret = readl_poll_timeout(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE, - value, value & AUSPLL_APB_CMD_OVERRIDE_ACK, 10, 10000); + value, value & AUSPLL_APB_CMD_OVERRIDE_ACK, 10, 10000); core_clear32(atcphy, AUSPLL_APB_CMD_OVERRIDE, AUSPLL_APB_CMD_OVERRIDE_REQ); return ret; } @@ -2616,8 +2615,8 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy) gates != 0x0000e001) return -EBUSY; ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_CMN_SHM_STS_REG0, - value, value & ACIOPHY_CMN_SHM_STS_REG0_CMD_READY, - 10, 10000); + value, value & ACIOPHY_CMN_SHM_STS_REG0_CMD_READY, + 10, 10000); if (ret) return ret; atcphy->tunnel_attempted = true; @@ -2676,7 +2675,7 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy) if (ret) goto restore; ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, - value, value & ACIOPHY_AUSPLL_LOCK, 10, 10000); + value, value & ACIOPHY_AUSPLL_LOCK, 10, 10000); if (ret) goto restore; ret = atc_tunnel_command(atcphy, 0x2000); From 820a7cb83800294d42d71ba7710020060515d23c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:12:39 +1100 Subject: [PATCH 1004/1579] phy: apple: atc: Restore the note on the PHY reset ordering atcphy_configure() takes the USB3 PHY out of reset after the lane configuration, while the Asahi driver this is based on carries a note that the reference sequence releases the reset earlier, and that the order is the first thing to revisit if a mode does not come up. The note was lost when the J700 paths were added; put it back. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index a0ef5ad76a917..4bb98a02237eb 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2935,6 +2935,13 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) /* Take the USB3 PHY out of reset */ core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); + /* + * NOTE: Not what macos does, it takes the phy out of reset right before + * writing to ACIOPHY_CFG0_COMMON_SMALL, does lane configuration with phy + * out of reset, and the poll is right after AUS_COMMON_SHIM_BLK_BIAS_REG + * write. Does not seem to matter, but if something does not work, + * consider re-ordering things to macos order. + */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) { ret = readl_poll_timeout(atcphy->regs.core + AUS_COMMON_DIG_RCAL1, reg, (reg & AUS_COMMON_DIG_RCAL1_ALL_CODES_DONE), 10, 100000); From 7ed160b9b5f28df712e98088d4368188b2e28072 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:38 +1100 Subject: [PATCH 1005/1579] soc: apple: mailbox: Tidy T8140 comments and resume error logging Drop the references to private bring-up notes and to vendor function names from the T8140 comments, and describe tx_irq_unmasked by what it guards. A failed runtime resume when starting the mailbox or the ASCWrap CPU printed four runtime PM internals next to the error code. The error code is what tells the cause apart and the PM core already reports the state through sysfs, so print just the error. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mailbox.c | 19 +++++-------------- include/linux/soc/apple/mailbox.h | 8 +++----- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 9c448ec1fa9ec..3d76ebc59d51f 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -70,9 +70,8 @@ static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == 0x8830); /* - * The T8140 AOP "setup port" (ADT aop-exclave-mailbox): the second mailbox - * the J700 AOP firmware boots through. Layout measured on hardware by the - * native m1n1 microphone host (artifacts/j700-native-lpmic-20260906). + * The T8140 AOP "setup port": a second, smaller mailbox the AOP firmware + * uses during its boot. */ #define APPLE_T8140_AOP_SETUP_A2I_CONTROL 0x000 #define APPLE_T8140_AOP_SETUP_I2A_CONTROL 0x004 @@ -227,11 +226,7 @@ int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) ret = pm_runtime_resume_and_get(mbox->dev); if (ret) { - dev_err(mbox->dev, - "ASC CPU start: runtime resume failed: %d (active=%d status=%d disable_depth=%d runtime_error=%d)\n", - ret, mbox->active, mbox->dev->power.runtime_status, - mbox->dev->power.disable_depth, - mbox->dev->power.runtime_error); + dev_err(mbox->dev, "ASC CPU start: runtime resume failed: %d\n", ret); apple_mbox_clear_resume_error(mbox); goto out_unlock; } @@ -455,11 +450,7 @@ int apple_mbox_start(struct apple_mbox *mbox) ret = pm_runtime_resume_and_get(mbox->dev); if (ret) { - dev_err(mbox->dev, - "mailbox start: runtime resume failed: %d (active=%d status=%d disable_depth=%d runtime_error=%d)\n", - ret, mbox->active, mbox->dev->power.runtime_status, - mbox->dev->power.disable_depth, - mbox->dev->power.runtime_error); + dev_err(mbox->dev, "mailbox start: runtime resume failed: %d\n", ret); apple_mbox_clear_resume_error(mbox); goto out_unlock; } @@ -698,7 +689,7 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->ascwrap_v6.wrapper_size = resource_size(res); mbox->ascwrap_v6.iop_vbar_start = iop_vbar_res->start; mbox->ascwrap_v6.iop_vbar_size = resource_size(iop_vbar_res); - /* resetState() is software-only; the matching stop lives above. */ + /* T8140 implements both the CPU start and the CPU stop. */ mbox->ascwrap_v6.missing = 0; mbox->wrapper_regs = diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index 8813438326ffc..fcb17eaf2d5a3 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -42,8 +42,6 @@ enum apple_mbox_ascwrap_v6_role { * * The apple-mailbox provider owns both mappings. Consumers request CPU * transitions through the provider and never map either resource themselves. - * T8140 has a complete matching start and stop transform. AppleA7IOP's - * resetState() only updates software state and performs no reset MMIO. */ struct apple_mbox_ascwrap_v6_lifecycle { enum apple_mbox_ascwrap_v6_role role; @@ -80,9 +78,9 @@ struct apple_mbox { void *cookie; /* - * Protected by tx_lock. IRQ, send-error and provider-removal paths - * mask an outstanding unmask only once; rebuild every consumer when - * changing this internal structure's layout. + * Protected by tx_lock: set while irq_send_empty is unmasked, so that + * the IRQ handler, a failed send and provider removal mask it exactly + * once between them. */ bool tx_irq_unmasked; }; From f80758084279064eb1c60e6bfa56f7649bb86a4d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:39 +1100 Subject: [PATCH 1006/1579] iommu: apple-dart: Demote informational probe messages to debug Every DART with an "apple,dma-offset" property, such as the T8140 SEP DART, prints the aliased DMA window at info level on each boot, and the tunneled PCIe-C DART prints its fixed topology and a pair of messages around applying its tunables. These describe an expected configuration taken from the device tree, so log them at debug level like the plain "Limiting DMA range" message already is, and fold the tunables pair into one message. Also merge the two comments explaining why a tunneled DART's parameter registers are not read, and describe the access ordering requirement without reference to how it was found. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 53 +++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 9cdbcc9798e6d..6a32a91f61d09 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -312,12 +312,11 @@ struct apple_dart { /* * PCIe-C is a cable-powered aperture and can report a failed transaction as * an asynchronous external abort unless each access is completed before the - * next instruction retires. A live /dev/mem diagnostic using the same - * Device-nGnRnE attributes and the exact 64-SID reset sequence is stable when - * every access is followed by dsb sy + isb. ioremap_np() supplies the memory - * type, but writel() only supplies the normal I/O ordering barrier on arm64. - * Match the proven access semantics for the tunneled DART while leaving all - * always-on DARTs on the normal fast path. + * next instruction retires. The DART reset sequence is only stable with + * Device-nGnRnE attributes and a dsb sy + isb after every access. + * ioremap_np() supplies the memory type, but writel() only supplies the + * normal I/O ordering barrier on arm64, so add the barriers here for the + * tunneled DART and leave all always-on DARTs on the normal fast path. */ static inline void apple_dart_writel(struct apple_dart *dart, u32 value, u32 offset) @@ -2502,9 +2501,9 @@ static int apple_dart_probe(struct platform_device *pdev) * PCIe-C is a cable-powered aperture whose writes must complete before * the following transaction. The normal arm64 ioremap() mapping is * Device-nGnRE and allowed the first posted DART reset write to surface as - * an asynchronous external abort at a later instruction. The identical - * reset sequence is stable through /dev/mem's Device-nGnRnE mapping, and - * PCI configuration forwarding already requires the same non-posted + * an asynchronous external abort at a later instruction. The same + * reset sequence is stable with a Device-nGnRnE mapping, and PCI + * configuration forwarding already requires the same non-posted * ordering. Keep ordinary always-on DARTs on their existing mapping. */ if (tunneled) @@ -2562,26 +2561,20 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) goto err_clk_disable; if (tunneled && dart->tunables) { - dev_info(dev, "applying %zu firmware PCIe-C DART tunables\n", - dart->tunables->sz); + dev_dbg(dev, "applying %zu PCIe-C DART tunables\n", + dart->tunables->sz); apple_dart_apply_tunables(dart); - dev_info(dev, "firmware PCIe-C DART tunables applied\n"); } - /* - * The cable-powered T8110 DART does not provide a safely probeable - * parameter block at this point in PCIe tunnel activation. The SError - * raised by these reads is asynchronous, which made the following trace - * boundary look guilty on different boots. The device tree supplies the - * immutable topology: 16 KiB pages, 42-bit addresses and 64 SIDs. Use that - * description before making any DART MMIO access. - */ if (tunneled) { /* - * PARAMS is not safe to read on a tunneled DART. T8110 is - * 16 KiB pages, 42-bit addresses, 64 streams and four levels. - * A T8020 PCIe-C DART is 16 KiB pages, a 32-bit IOVA, a 36-bit - * PA and as many streams as its register block describes. + * The parameter registers of a tunneled DART cannot be read + * safely at this point of PCIe tunnel activation: the read + * raises an asynchronous SError. The topology is fixed + * instead. T8110 is 16 KiB pages, 42-bit addresses, 64 + * streams and four levels. A T8020 PCIe-C DART is 16 KiB + * pages, a 32-bit IOVA, a 36-bit PA and as many streams as its + * register block describes. */ if (dart->hw->type == DART_T8110) { dart->pgsize = 1 << DART_PCIEC_PAGE_SHIFT; @@ -2598,9 +2591,9 @@ static int apple_dart_probe(struct platform_device *pdev) dart->num_streams = dart->hw->max_sid_count; dart->four_level = false; } - dev_info(dev, - "PCIe-C DART using fixed topology: pagesize %#x, %u streams, AS %u -> %u\n", - dart->pgsize, dart->num_streams, dart->ias, dart->oas); + dev_dbg(dev, + "PCIe-C DART using fixed topology: pagesize %#x, %u streams, AS %u -> %u\n", + dart->pgsize, dart->num_streams, dart->ias, dart->oas); goto params_done; } @@ -2672,9 +2665,9 @@ static int apple_dart_probe(struct platform_device *pdev) ret = -EINVAL; goto err_clk_disable; } else { - dev_info(dev, - "aliasing DMA range %pad..%pad at DART offset %pad\n", - &dart->dma_min, &dart->dma_max, &dart->dma_offset); + dev_dbg(dev, + "aliasing DMA range %pad..%pad at DART offset %pad\n", + &dart->dma_min, &dart->dma_max, &dart->dma_offset); } /* From 152cc6b4d737c3523b56d1c4b0b4cd3928e514b5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:39 +1100 Subject: [PATCH 1007/1579] spi: apple-qspi: Do not print a banner on probe The controller version, clock rate and FIFO depths are fixed for a given SoC and are useful only when debugging the driver. Print them at debug level. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index 57a572c1e779d..e0d8653e51984 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -587,9 +587,9 @@ static int apple_qspi_probe(struct platform_device *pdev) "failed to register SPI controller\n"); version = apple_qspi_p_read(qspi, APPLE_QSPI_P_VERSION); - dev_info(dev, "QSPIMC version %#x, %lu Hz clock, txq=%u rxq=%u\n", - version, clk_get_rate(qspi->clk), qspi->tx_depth, - qspi->rx_depth); + dev_dbg(dev, "QSPIMC version %#x, %lu Hz clock, txq=%u rxq=%u\n", + version, clk_get_rate(qspi->clk), qspi->tx_depth, + qspi->rx_depth); return 0; } From 95b26591a1c4b446656c3bc8a257a17792917a78 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:40 +1100 Subject: [PATCH 1008/1579] spi: apple: Reword the hardware CS delay comment Describe what the pre- and post-delay register values do instead of naming the vendor driver they were observed in. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c index 21937bb50ba08..f18cc6e1b8d0e 100644 --- a/drivers/spi/spi-apple.c +++ b/drivers/spi/spi-apple.c @@ -112,10 +112,9 @@ #define APPLE_SPI_DELAY_CYCLES GENMASK(31, 16) /* - * These are the values used by AppleSPIMCController's hardware-delay path. - * The pre-delay asserts active-low CS and the post-delay releases it. The - * otherwise undocumented bit 11 is named for its observed role rather than - * assigning it a speculative electrical meaning. + * Hardware CS delays: the pre-delay asserts the active-low CS and the + * post-delay releases it. Bit 11 is undocumented and named for its observed + * role only. */ #define APPLE_SPI_DELAY_PRE_FLAGS (APPLE_SPI_DELAY_PRE_ARM | \ APPLE_SPI_DELAY_SET_CS | \ From f52e01d6925de5022176c5038ff13cc9ad17559b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:40 +1100 Subject: [PATCH 1009/1579] dt-bindings: gpu: apple,agx: Match the T8140 register layout On T8140 the GFX and GFX1 coprocessor windows are owned by their "apple,t8140-ascwrap-v6" mailbox providers, and the GPU node only keeps the GFX coprocessor control registers ("asc") and the GPU block ("sgx"), which is all the driver requests. The binding still required a third "asc1" region, so dtbs_check rejected t8140-j700.dtb. Describe the two regions the device tree and the driver use. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/gpu/apple,agx.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index cc9c78a7624f8..e7afc4dd87e8f 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -30,12 +30,10 @@ properties: - const: apple,agx-g14s reg: - minItems: 2 - maxItems: 3 + maxItems: 2 reg-names: - minItems: 2 - maxItems: 3 + maxItems: 2 power-domains: maxItems: 1 @@ -93,12 +91,10 @@ allOf: reg: items: - description: GFX coprocessor control registers - - description: GFX1 coprocessor control registers - description: GPU block MMIO registers reg-names: items: - const: asc - - const: asc1 - const: sgx mboxes: items: From 337b14463d5010296efb0b17dfaa11b8f7601aae Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:41 +1100 Subject: [PATCH 1010/1579] arm64: dts: apple: t8140: Fold the trailing soc override into the soc node The display, PMC, pinctrl, I2C, SPI, AOP, ISP and ANE nodes were added through a second "&soc { }" block at the end of the same file, indented as if they were still inside the soc node. Move them into the soc node itself, as the other Apple SoC dtsi files do. The compiled DTB is unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 89f12fbd90632..93a7f22266d68 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1177,10 +1177,7 @@ #mbox-cells = <0>; status = "disabled"; }; - }; -}; -&soc { disp0_dart: iommu@412300000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x4 0x12300000 0x0 0xc000>; @@ -1791,6 +1788,7 @@ power-domains = <&ps_ane_sys>; status = "disabled"; }; + }; }; #include "t8140-pmgr.dtsi" From 6f1b4ab6f2f036dedd104b83df774d6284ecab74 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:18:12 +1100 Subject: [PATCH 1011/1579] arm64: dts: apple: t8140: Tidy the mailbox and pinctrl comments Shorten the AOP mailbox comments to what the nodes need: the register layout of the setup mailbox lives in the driver, and the interrupt and register numbering of the firmware's own device tree does not help a reader of this one. Also drop a stray blank line in the pinctrl and pmgr nodes. The compiled DTB is unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 93a7f22266d68..22b7d0a887d19 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -674,7 +674,6 @@ /* child nodes are added in t8140-pmgr.dtsi */ }; - aic: interrupt-controller@301000000 { compatible = "apple,t8140-aic3", "apple,t8122-aic3"; #interrupt-cells = <3>; @@ -1311,7 +1310,6 @@ gpio-ranges = <&pinctrl_ap 0 0 216>; apple,npins = <216>; - /* Live J700 ADT /arm-io/gpio0 interrupts (7 groups). */ interrupt-controller; #interrupt-cells = <2>; interrupt-parent = <&aic>; @@ -1322,7 +1320,7 @@ , , ; - /* ADT no-restore-gpio-pins entries: 50 and 134. */ + /* Pins the firmware lists as no-restore */ gpio-reserved-ranges = <50 1>, <134 1>; status = "disabled"; @@ -1335,7 +1333,6 @@ pinmux = , ; }; - }; avd: avd@381080000 { @@ -1400,13 +1397,9 @@ }; /* - * AOP (always-on processor). Live J700 ADT /arm-io/aop - * (arm-io child + 0x210000000): reg[2] is the AOP MMIO window, - * reg[0] the ASCWrap v6 block whose mailbox sits at +0x8000. - * ADT interrupts [391, 390, 393, 392] = send-not-empty, - * send-empty, recv-not-empty, recv-empty. The firmware - * (j700aop, RTKit 12) is iBoot-loaded and left stopped; Linux - * adopts it and never reloads it. + * AOP (always-on processor). The bootloader loads its RTKit + * firmware and leaves it stopped; Linux starts it and never + * reloads it. */ aop_mbox: mailbox@310608000 { compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; @@ -1422,14 +1415,7 @@ status = "disabled"; }; - /* - * The AOP "setup port": the second mailbox the J700 AOP firmware - * boots through (ADT /arm-io/aop-exclave-mailbox, absolute - * address 0x35067c000, IRQs [397, 394] = recv-not-empty, - * send-empty). Register layout measured on hardware: - * A2I control +0x0 (FULL bit 16), I2A control +0x4 (EMPTY bit - * 17), send +0x180/+0x188, recv +0x1b0/+0x1b8. - */ + /* The AOP "setup port", a second mailbox used while it boots */ aop_setup_mbox: mailbox@35067c000 { compatible = "apple,t8140-aop-setup-mailbox"; reg = <0x3 0x5067c000 0x0 0x4000>; From d50642d23a7edb9ace6837d6471981e84d0d4ef6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:49:11 +1100 Subject: [PATCH 1012/1579] iommu: apple-dart: Fix the KUnit test after deferred root publication apple_dart_hw_sync_locked() gained a defer_unknown argument when the early TLB sync learnt to skip locked streams whose roots are not mapped yet, but the KUnit test kept calling it with two arguments, so CONFIG_APPLE_DART_KUNIT_TEST no longer builds. The before-attach case also still expected that early sync to fail with -EIO, which is exactly what the change removed. Pass false where the test exercises the attach-time publication, expect the early sync to succeed, and add a case that checks both sides of the new argument: the early sync leaves a stream with a half-known root alone, while the attach-time sync still refuses it. With this the whole suite passes again under QEMU. Fixes: c361702757bf ("iommu/apple-dart: defer publication until locked roots are known") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 49 ++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 0eba2eca17474..86ed79c8d0c1d 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -47,9 +47,12 @@ static void apple_dart_test_before_attach(struct kunit *test) KUNIT_EXPECT_EQ(test, atomic_long_read(&domain->stream_maps[0].sidmap[0]), 0L); KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), 0U); - /* Once attached, missing root mappings must still report an error. */ + /* + * A stream whose locked roots are not mapped yet is left to attach, + * which publishes the complete page table. + */ KUNIT_EXPECT_EQ(test, apple_dart_domain_add_streams(domain, &master), 0); - KUNIT_EXPECT_EQ(test, apple_dart_iotlb_sync_map(&domain->domain, 0, PAGE_SIZE), -EIO); + KUNIT_EXPECT_EQ(test, apple_dart_iotlb_sync_map(&domain->domain, 0, PAGE_SIZE), 0); pm_runtime_disable(dart->dev); free_io_pgtable_ops(ops); @@ -78,7 +81,7 @@ static void apple_dart_test_locked_handoff(struct kunit *test) cfg.apple_dart_cfg.ttbr[0] = ours; /* Legacy firmware mappings are rebuilt by the IOMMU core at handoff. */ - KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), 0); + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, false), 0); KUNIT_EXPECT_MEMEQ(test, live, ours, sizeof(live)); KUNIT_EXPECT_MEMEQ(test, owned, ours, sizeof(owned)); apple_dart_retire_root(live, owned, ARRAY_SIZE(live)); @@ -89,10 +92,41 @@ static void apple_dart_test_locked_handoff(struct kunit *test) dart->hw = &apple_dart_hw_t8110; dart->version = 0x0200; live[0] = 0x1001; - KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), 0); + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, false), 0); KUNIT_EXPECT_MEMEQ(test, live, ours, sizeof(live)); } +static void apple_dart_test_sync_unknown_roots(struct kunit *test) +{ + struct apple_dart_stream_map stream = {}; + struct io_pgtable_cfg cfg = {}; + struct apple_dart *dart; + u64 live[] = { 0x1001, 0, 0, 0 }; + u64 ours[] = { 0, 0x2001, 0, 0 }; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->hw = &apple_dart_hw_t8110; + dart->version = 0x0202; + dart->num_streams = 1; + dart->pgsize = sizeof(live); + spin_lock_init(&dart->lock); + /* The live root is mapped, its ownership shadow is not yet. */ + dart->locked_ttbr[0][0] = live; + stream.dart = dart; + __set_bit(0, stream.sidmap); + cfg.apple_dart_cfg.n_ttbrs = 1; + cfg.apple_dart_cfg.ttbr[0] = ours; + + /* An early TLB sync skips the stream and leaves the root alone... */ + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, true), 0); + KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); + KUNIT_EXPECT_EQ(test, live[1], 0ULL); + /* ...but publishing at attach still requires both. */ + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, false), -EIO); + KUNIT_EXPECT_EQ(test, live[1], 0ULL); +} + static void apple_dart_test_firmware_roots(struct kunit *test) { struct apple_dart_stream_map stream = {}; @@ -116,13 +150,13 @@ static void apple_dart_test_firmware_roots(struct kunit *test) cfg.apple_dart_cfg.n_ttbrs = 1; cfg.apple_dart_cfg.ttbr[0] = ours; - KUNIT_ASSERT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), 0); + KUNIT_ASSERT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, false), 0); KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); KUNIT_EXPECT_EQ(test, live[1], 0x2001ULL); ours[0] = 0x3001; - KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), -EBUSY); + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, false), -EBUSY); dart->version = 0x0203; - KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream), -EBUSY); + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, false), -EBUSY); KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); /* Teardown must preserve a root which firmware has since replaced. */ live[1] = 0x4001; @@ -377,6 +411,7 @@ static void apple_dart_test_gated_irq(struct kunit *test) static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_before_attach), KUNIT_CASE(apple_dart_test_locked_handoff), + KUNIT_CASE(apple_dart_test_sync_unknown_roots), KUNIT_CASE(apple_dart_test_firmware_roots), KUNIT_CASE(apple_dart_test_gated_commands), KUNIT_CASE(apple_dart_test_resume_commands_failure), From 1ef9c2113ecdf2851908e4bd9793876d92152d27 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:48:35 +1100 Subject: [PATCH 1013/1579] soc: apple: rtkit: Do not warn when buffers are retained on purpose apple_rtkit_free_retaining_buffers() prints a warning every time it is called. Its callers use it deliberately for firmware that is left running: the T8140 NVMe controller on every unbind after its firmware reached RUN, the DCP when it has pinned a live firmware session, and the MTP helper for a session adopted from the bootloader. None of these is a problem, yet each unbind left a warning in the log. A caller that gets here after a failed shutdown is better placed to report that failure, so log the retained buffer sizes at debug level and say in the kernel-doc that a running co-processor is an expected case. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 15 ++++++++------- include/linux/soc/apple/rtkit.h | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 3721c529de915..62e7a9add49ff 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -1231,14 +1231,15 @@ void apple_rtkit_free_retaining_buffers(struct apple_rtkit *rtk) apple_rtkit_stop_rx(rtk); /* - * A failed shutdown provides no guarantee that firmware stopped DMA. - * Retain shared buffers, including custom allocation contexts, rather - * than returning their storage to an allocator while it may be live. + * The firmware may still be using the shared buffers, so keep them, + * including custom allocation contexts, rather than returning their + * storage to an allocator. Callers use this on purpose for firmware + * that cannot be stopped, and report a failed shutdown themselves. */ - dev_warn(rtk->dev, - "RTKit: retaining the shared buffers of an unstopped co-processor (ioreport %zu, crashlog %zu, oslog %zu, syslog %zu bytes)\n", - rtk->ioreport_buffer.size, rtk->crashlog_buffer.size, - rtk->oslog_buffer.size, rtk->syslog_buffer.size); + dev_dbg(rtk->dev, + "RTKit: retaining the shared buffers of a running co-processor (ioreport %zu, crashlog %zu, oslog %zu, syslog %zu bytes)\n", + rtk->ioreport_buffer.size, rtk->crashlog_buffer.size, + rtk->oslog_buffer.size, rtk->syslog_buffer.size); kfree(rtk->syslog_msg_buffer); apple_rtkit_release_rx(rtk); kfree(rtk); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 00b9a21e936d7..6f7d59a267877 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -136,8 +136,8 @@ struct apple_rtkit *apple_rtkit_init_adopted(struct device *dev, void *cookie, void apple_rtkit_free(struct apple_rtkit *rtk); /* - * Detach a co-processor that could not be quiesced without releasing its - * shared buffers. + * Detach a co-processor that is still running, or that could not be + * quiesced, without releasing its shared buffers. * * Stops mailbox callbacks and drains the private RTKit workqueue before * freeing the transport state. Firmware-visible mappings and their allocation From b6838aa647e6f609ec3854c4925b7f43034e583d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:22:40 +1100 Subject: [PATCH 1014/1579] ASoC: apple: t8140-aop-audio: Format with rustfmt make rustfmtcheck flags one dev_warn!() call that is wider than the kernel's rustfmt configuration allows. Reformat it. Also move the comment on the float conversion's exponent onto its own line above the statement. rustfmt had aligned its continuation line under the end of the statement, where it reads as a stray comment. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 57dad8c960331..b63c6c8baa084 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -846,7 +846,11 @@ impl SndSocT8140AopData { } let ret = f(); if let Err(e) = self.lpai_run_put() { - dev_warn!(self.dev, "lpai: unable to idle after raising the fabric: {:?}\n", e); + dev_warn!( + self.dev, + "lpai: unable to idle after raising the fabric: {:?}\n", + e + ); } ret } @@ -1907,8 +1911,8 @@ fn f32_bits_of_q31(v: i32) -> u32 { } let sign = if v < 0 { 1u32 << 31 } else { 0 }; let m = v.unsigned_abs(); - let p = 31 - m.leading_zeros(); // position of the leading one, 0..=31 - // value = m * 2^-31 = 1.f * 2^(p - 31) + // The position of the leading one, 0..=31: |v| / 2^31 = 1.f * 2^(p - 31). + let p = 31 - m.leading_zeros(); let exp = (p as i32 - 31 + 127) as u32; let mant = if p >= 23 { (m >> (p - 23)) & 0x7f_ffff From 5953a8e5828879fc98aef07a9f46ff870c923eb2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:44:17 +1100 Subject: [PATCH 1015/1579] ASoC: dt-bindings: cirrus,cs42l42: Reference dai-common.yaml The binding lists '#sound-dai-cells' and sound-name-prefix by hand. Reference dai-common.yaml for the common DAI properties instead, as the other codec bindings do, and use unevaluatedProperties so that what it defines is accepted. While at it, put the note on the CS42L83 supplies of Apple Silicon machines in a paragraph of its own and say why they are optional there. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/sound/cirrus,cs42l42.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml index 1d839e28c0f8a..31cdd309f33ee 100644 --- a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml +++ b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml @@ -14,8 +14,9 @@ description: It provides a high-dynamic range, stereo DAC for audio playback and a mono high-dynamic-range ADC for audio capture. There is an integrated headset detection block. - CS42L83 systems may have platform-managed supplies without regulator - phandles, as on Apple Silicon boards. + + On Apple Silicon machines the supplies of the CS42L83 are platform-managed + rails that are not described as regulators, so they are optional there. properties: compatible: @@ -31,8 +32,6 @@ properties: '#sound-dai-cells': const: 0 - sound-name-prefix: true - VP-supply: description: VP power supply. @@ -194,7 +193,9 @@ properties: required: - compatible - reg + allOf: + - $ref: dai-common.yaml# - if: properties: compatible: @@ -207,7 +208,7 @@ allOf: - VL-supply - VA-supply -additionalProperties: false +unevaluatedProperties: false examples: - | From 07dea1d1040ab6140cfef4e2fb66718d65d05bad Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:44:18 +1100 Subject: [PATCH 1016/1579] ASoC: dt-bindings: apple,t8140-aop-audio: Reference dai-common.yaml The node is a DAI provider, like the MCA. Reference dai-common.yaml as apple,mca.yaml does, and use unevaluatedProperties so that the common DAI properties it defines are accepted. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/sound/apple,t8140-aop-audio.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml index a4e10cbb4deea..b023080e48c7f 100644 --- a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml @@ -20,6 +20,9 @@ description: | The remaining audio data moves over the named ADMAC channels while its AOP service is active. The parent AOP binding describes the firmware transport. +allOf: + - $ref: dai-common.yaml# + properties: compatible: const: apple,t8140-aop-audio @@ -96,7 +99,7 @@ required: - apple,chassis-name - apple,machine-kind -additionalProperties: false +unevaluatedProperties: false examples: - | From 10e1825791523747c5f0d4ac44570fe4418ff586 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:11:48 +1100 Subject: [PATCH 1017/1579] pmdomain: apple: pmgr-pwrstate: log always-on domains left off at info level The boot loader is not required to leave an always-on domain powered. Several T8140 domains are off when Linux starts: the GPU domain, the AOP audio fabric, an SMC I2C master and the NUB OCLA domain. The device tree marks them always-on for good reasons (firmware dependencies, the Apple Device Tree's critical and on flags, and keeping the GPU powered across system sleep), and the driver already turns them on at probe. That is an expected state, not a problem, yet each such domain produces a warning on every boot. Log it at info level and say what the driver does about it. A domain that cannot be turned on is still reported as an error by apple_pmgr_ps_set(). Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmgr-pwrstate.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 47f9593ae1221..053c087c37184 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -366,7 +366,13 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) } else if (of_property_read_bool(node, "apple,always-on")) { ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; if (!active) { - dev_warn(dev, "always-on domain %s is not on at boot\n", name); + /* + * The boot loader need not leave an always-on domain on, + * so this is not an error: apple_pmgr_ps_set() reports a + * domain that cannot be turned on. + */ + dev_info(dev, "always-on domain %s is off at boot, turning it on\n", + name); /* * Turn it on so pm_genpd_init does not fail. genpd never * powers an always-on domain, so keep requesting it on From 21c1d3f21ac49f4223993ed1beefca1d6d6e89cd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:12:20 +1100 Subject: [PATCH 1018/1579] arm64: dts: apple: t8140: state why PMGR domains off at boot are always-on The AOP audio fabric domains (audio_a, audio_fr, audio_p, audio_leap_a and audio_leap_c) are described as "on at boot", but the boot loader leaves them off and the pwrstate driver turns them on at probe. They are always-on because the AOP firmware stops answering LPAI power requests when audio_p goes off underneath it, not because of their boot state. Correct the comments. nub_ocla and smc_i2cm0 are off at boot as well and carry no reason. The Apple Device Tree flags nub_ocla critical, which is what makes a domain always-on in the generated PMGR files of other SoCs (t8122, t8132), and flags smc_i2cm0 on, as it does smc_i2cm1 and smc_i2cm2. Say so. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 8438e4f17e81f..df5ba0ac105b8 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -477,10 +477,11 @@ * aop_he_nrcs0/1, aop_i2cm3 and aop_spi1 are off at boot and the HE * NRCS pair does not power up, so those stay unmanaged). The audio * fabric parents (audio_a, audio_fr, audio_p, audio_leap_a, - * audio_leap_c) are on at boot and the AOP firmware relies on them: - * letting genpd switch audio_p off during a runtime-PM cycle of - * audio_leap_mca left the firmware unable to answer LPAI power - * requests, so they are always-on; only the leaves (audio_leap_tx0, + * audio_leap_c) are off at boot, and the AOP firmware relies on them + * once they are on: letting genpd switch audio_p off during a + * runtime-PM cycle of audio_leap_mca left the firmware unable to + * answer LPAI power requests, so they are always-on and the driver + * turns them on at probe; only the leaves (audio_leap_tx0, * audio_leap_mca, audio_leap_ns_a, ...) are runtime-managed. */ ps_aop_filter: power-controller@4000 { @@ -518,7 +519,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "audio_a"; - /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ + /* AOP audio fabric: the firmware expects it to stay on */ apple,always-on; power-domains = <&ps_aop_filter>; }; @@ -539,7 +540,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "audio_fr"; - /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ + /* AOP audio fabric: the firmware expects it to stay on */ apple,always-on; power-domains = <&ps_audio_a>; }; @@ -550,7 +551,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "audio_p"; - /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ + /* AOP audio fabric: the firmware expects it to stay on */ apple,always-on; power-domains = <&ps_audio_fr>; }; @@ -561,7 +562,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "audio_leap_a"; - /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ + /* AOP audio fabric: the firmware expects it to stay on */ apple,always-on; power-domains = <&ps_audio_fr>; }; @@ -608,7 +609,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "audio_leap_c"; - /* AOP-owned audio fabric: on at boot, the firmware expects it to stay on */ + /* AOP audio fabric: the firmware expects it to stay on */ apple,always-on; power-domains = <&ps_audio_leap_a>; }; @@ -1113,7 +1114,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "nub_ocla"; - apple,always-on; + apple,always-on; /* critical in the Apple Device Tree */ status = "okay"; }; @@ -1493,7 +1494,7 @@ #reset-cells = <0>; label = "smc_i2cm0"; power-domains = <&ps_smc_fabric>; - apple,always-on; + apple,always-on; /* on in the Apple Device Tree, like smc_i2cm1/2 */ status = "okay"; }; From 23b2b6f4a1c93dbc90768e271c92bc914e5d676f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:12:43 +1100 Subject: [PATCH 1019/1579] soc: apple: pmp: log tunables the boot loader does not provide at debug level While it starts, the PMP firmware asks for tunables by name, and the driver answers from the apple,tunable-* properties the boot loader copied from the Apple Device Tree. On T8140 the firmware asks for four names that the Apple Device Tree does not carry (pmc-max-bwr-latency-us, mc-quota-policy, mc-cue-dsid0-quota and lts-config), so every boot logs four "unknown property" lines, although the firmware starts and runs without them. A missing tunable is not something the user can act on. Log it at debug level, where it is still available when bringing up a new SoC. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/pmp.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index 3356fc889b523..7f236f1bd4c6f 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -329,7 +329,9 @@ impl PmpData { } size = len as u32; } else { - dev_info!(self.dev, "unknown property {:?}", name_str); + // Not every boot loader provides every tunable the firmware + // asks for, and the firmware starts without them. + dev_dbg!(self.dev, "unknown property {:?}", name_str); } } state.ioreg_entries.push(size, GFP_KERNEL)?; From e99a6adbeba77b7ed032ef61f864afb3da8fe3a9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:16:31 +1100 Subject: [PATCH 1020/1579] dt-bindings: hwmon: apple,smc-hwmon: drop apple,cpu-thermal-policy The conditional for apple,discover-temperatures forbids an apple,cpu-thermal-policy property, but the binding never defines that property and additionalProperties is false, so it is rejected anyway. No device tree uses it. Remove the clause. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index 3bb7931d648bd..df155f9c34a65 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -103,8 +103,6 @@ allOf: required: - apple,discover-temperatures then: - properties: - apple,cpu-thermal-policy: false patternProperties: "^temperature-": false From 0790f8a546fea7ae46871d857643568c1b3c0c64 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:16:31 +1100 Subject: [PATCH 1021/1579] hwmon: macsmc: drop the check for the undefined apple,cpu-thermal-policy Temperature discovery refuses to run when the hwmon node has an apple,cpu-thermal-policy property. No binding defines that property and no device tree sets it, so the check can never fire. The real guard is the one next to it: discovered channels are not mixed with channels the device tree describes, which thermal zones refer to by index. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hwmon/macsmc-hwmon.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index 36598f06ecaf2..b0172bbb76e98 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -514,9 +514,8 @@ static int macsmc_hwmon_discover_temperatures(struct macsmc_hwmon *hwmon) if (!of_property_read_bool(hwmon->dev->of_node, "apple,discover-temperatures")) return 0; - /* Dynamic channel numbers must not replace a DT-described thermal policy. */ - if (hwmon->temp.count || - of_property_read_bool(hwmon->dev->of_node, "apple,cpu-thermal-policy")) + /* Dynamic channel numbers must not replace DT-described channels. */ + if (hwmon->temp.count) return -EINVAL; /* Bound both allocation and mailbox transactions for invalid firmware data. */ From a61bca6382a765f5b74134dea3b094039f2a15bd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:04 +1100 Subject: [PATCH 1022/1579] cpufreq: apple-soc: tidy includes and the ->exit() callback The headers the fast die-temperature limit needs were added as a second, separately sorted block. Merge them into the existing alphabetical list. ->exit() is a wrapper that stops the limit and then calls apple_soc_cpufreq_exit(), which needs a forward declaration. Stop the limit at the start of apple_soc_cpufreq_exit() instead and use that as the callback. The order of teardown is unchanged. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 36 ++++++++++++----------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 86273ba1d64d8..ce67fa7d62f70 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -7,32 +7,31 @@ * Based on scpi-cpufreq.c */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include #include #include #include +#include #include #include #include +#include #include #include +#include +#include +#include #include #include #include #include +#include #include +#include +#include +#include +#include +#include #define APPLE_DVFS_CMD 0x20 #define APPLE_DVFS_CMD_BUSY BIT(31) @@ -208,7 +207,6 @@ struct apple_cpu_priv { }; static struct cpufreq_driver apple_soc_cpufreq_driver; -static void apple_soc_cpufreq_exit_with_limits(struct cpufreq_policy *policy); static const struct apple_soc_cpufreq_info soc_s5l8960x_info = { .has_ps2 = false, @@ -931,6 +929,9 @@ static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) { struct apple_cpu_priv *priv = policy->driver_data; + /* Drain firmware-limit updates before releasing the OPP tables. */ + apple_fast_die_exit(priv); + dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); /* * The core clears the last CPU from policy->cpus before calling @@ -948,7 +949,7 @@ static struct cpufreq_driver apple_soc_cpufreq_driver = { .verify = cpufreq_generic_frequency_table_verify, .get = apple_soc_cpufreq_get_rate, .init = apple_soc_cpufreq_init, - .exit = apple_soc_cpufreq_exit_with_limits, + .exit = apple_soc_cpufreq_exit, .target_index = apple_soc_cpufreq_set_target, .fast_switch = apple_soc_cpufreq_fast_switch, .register_em = cpufreq_register_em_with_opp, @@ -956,13 +957,6 @@ static struct cpufreq_driver apple_soc_cpufreq_driver = { .suspend = cpufreq_generic_suspend, }; -/* Drain firmware-limit updates before releasing the policy's OPP table. */ -static void apple_soc_cpufreq_exit_with_limits(struct cpufreq_policy *policy) -{ - apple_fast_die_exit(policy->driver_data); - apple_soc_cpufreq_exit(policy); -} - static int __init apple_soc_cpufreq_module_init(void) { struct device_node *np; From e7d3d687d20745259fcb83af83c9ed377ff6a1f3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:18:45 +1100 Subject: [PATCH 1023/1579] cpufreq: apple-soc: describe the unmanaged P-state ceiling as data SoCs whose higher P-states need a thermal policy that is not qualified yet refuse an OPP table that goes above max_unmanaged_pstate. T8140 has one (power_allocator zones and the PMP fast die-temperature limit), so init skips the check for it by comparing the info pointer with &soc_t8140_info, while soc_t8140_info still carries a ceiling that is never used. Drop the unused T8140 ceiling and run the check whenever a SoC has one, so the per-SoC table says which SoCs are capped. Document both fields. The check runs for exactly the same SoCs as before (T8152 only). No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index ce67fa7d62f70..057954affc4fb 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -186,8 +186,10 @@ static int apple_fast_die_hp_state; struct apple_soc_cpufreq_info { bool has_ps2; bool verify_transition; + /* Fail closed when the DT describes no fast die-temperature limit. */ bool needs_thermal_policy; u32 transition_timeout_us; + /* Highest P-state allowed without a thermal policy; 0 if not capped. */ u32 max_unmanaged_pstate; u64 min_pstate; u64 max_pstate; @@ -244,7 +246,6 @@ static const struct apple_soc_cpufreq_info soc_t8112_info = { static const struct apple_soc_cpufreq_info soc_t8140_info = { .verify_transition = true, .needs_thermal_policy = true, - .max_unmanaged_pstate = 2, .min_pstate = 1, .max_pstate = 31, .ps1_mask = APPLE_DVFS_CMD_PS1, @@ -871,8 +872,8 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) priv->expected_pstate = state; } - /* T8140 has IPA and the PMP limit; other SoCs keep their safe ceiling. */ - if (info->needs_thermal_policy && info != &soc_t8140_info) { + /* SoCs without a qualified thermal policy keep their safe ceiling. */ + if (info->max_unmanaged_pstate) { cpufreq_for_each_valid_entry(p, freq_table) { if (p->driver_data <= info->max_unmanaged_pstate) continue; From 742366203cc8a1f9356d82a43d52e2cb6fa97621 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:20:16 +1100 Subject: [PATCH 1024/1579] pmdomain: apple: pmp-report: remove the raw DVFS debugfs dump On T8140 the driver creates /sys/kernel/debug/apple-pmp-/dvfs, which prints four undocumented 16-byte entries of the report region as raw hex. It was a reverse-engineering aid: nothing reads it, it sits in a second debugfs root next to apple-pmp-report/, and it contradicts both the comment on the status file and the binding, which only allow documented entries to be read. The status file already shows what the driver knows about (whether the PMP runs and the fast die-temperature efforts). Remove it together with its forward declarations. While at it, drop a duplicate include and a dev_set_drvdata() call whose value nothing reads since report entries look their region up in the RCU list. No functional change outside debugfs. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmp-report.c | 79 ----------------------------- 1 file changed, 79 deletions(-) diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 318cefe61a89c..0e50a9d56a14b 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -41,10 +40,6 @@ struct apple_pmp_report_offsets { u32 fast_die_lanes; }; -struct apple_pmp_report; - -static int apple_pmp_dvfs_create(struct apple_pmp_report *report); - struct apple_pmp_report { struct device *dev; const struct apple_pmp_report_offsets *offsets; @@ -163,10 +158,6 @@ static int apple_pmp_report_probe(struct platform_device *pdev) if (IS_ERR(rep->base)) return PTR_ERR(rep->base); rep->offsets = of_device_get_match_data(dev); - dev_set_drvdata(dev, rep); - ret = apple_pmp_dvfs_create(rep); - if (ret) - return ret; INIT_LIST_HEAD(&rep->list); ret = devm_add_action_or_reset(dev, apple_pmp_report_release, rep); @@ -305,76 +296,6 @@ static int apple_pmp_report_entry_power_off(struct generic_pm_domain *genpd) return apple_pmp_report_entry_set_state(genpd, false); } -/* ADT DVFS-STATE: four 16-byte entries beginning at entry 8. */ -#define APPLE_PMP_DVFS_OFFSET 0x80 -#define APPLE_PMP_DVFS_STRIDE 16 -#define APPLE_PMP_DVFS_COUNT 4 - -static int apple_pmp_dvfs_show(struct seq_file *seq, void *unused) -{ - struct apple_pmp_report *report = seq->private; - u64 values[APPLE_PMP_DVFS_COUNT][2]; - unsigned int index; - - if (!(readl(report->base + report->offsets->status) & PMP_REPORT_READY)) - return -EAGAIN; - - for (index = 0; index < ARRAY_SIZE(values); index++) { - void __iomem *entry = report->base + APPLE_PMP_DVFS_OFFSET + - index * APPLE_PMP_DVFS_STRIDE; - - values[index][0] = readq(entry); - values[index][1] = readq(entry + sizeof(values[index][0])); - } - - if (!(readl(report->base + report->offsets->status) & PMP_REPORT_READY)) - return -EAGAIN; - - for (index = 0; index < ARRAY_SIZE(values); index++) - seq_printf(seq, "entry%u: 0x%016llx 0x%016llx\n", index + 8, - values[index][0], values[index][1]); - - return 0; -} -DEFINE_SHOW_ATTRIBUTE(apple_pmp_dvfs); - -static void apple_pmp_dvfs_remove(void *directory) -{ - debugfs_remove_recursive(directory); -} - -static int apple_pmp_dvfs_create(struct apple_pmp_report *report) -{ - struct device *dev = report->dev; - struct resource *resource; - struct dentry *directory; - char *name; - int ret; - - if (!of_device_is_compatible(dev->of_node, "apple,t8140-pmp-v2-report")) - return 0; - - resource = platform_get_resource(to_platform_device(dev), IORESOURCE_MEM, 0); - if (resource_size(resource) < APPLE_PMP_DVFS_OFFSET + - APPLE_PMP_DVFS_COUNT * APPLE_PMP_DVFS_STRIDE) - return dev_err_probe(dev, -EINVAL, "DVFS report exceeds resource\n"); - - name = devm_kasprintf(dev, GFP_KERNEL, "apple-pmp-%s", dev_name(dev)); - if (!name) - return -ENOMEM; - - directory = debugfs_create_dir(name, NULL); - if (IS_ERR(directory)) - return 0; - - ret = devm_add_action_or_reset(dev, apple_pmp_dvfs_remove, directory); - if (ret) - return ret; - - debugfs_create_file("dvfs", 0400, directory, report, &apple_pmp_dvfs_fops); - return 0; -} - static void apple_pmp_report_entry_free(struct apple_pmp_report_entry *ent) { of_node_put(ent->report_node); From 83039e319faf3e49ceabc424c7c69180ac997dfd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:20:35 +1100 Subject: [PATCH 1025/1579] dt-bindings: apple: use the canonical SPDX form in PMP, PMC and report bindings Upstream bindings, and checkpatch, expect the dual licence to be written as "(GPL-2.0-only OR BSD-2-Clause)". The PMP, PMP report and PMC bindings leave out the parentheses; the other T8140 power bindings already use them. Add them. The licence is unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/power/apple,pmc.yaml | 2 +- .../devicetree/bindings/power/apple,t8140-pmp-report.yaml | 2 +- Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index 91c77695178c4..a57b90806511b 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- $id: http://devicetree.org/schemas/power/apple,pmc.yaml# diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml index 0cddaf6136379..781afa1ad9d9f 100644 --- a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- $id: http://devicetree.org/schemas/power/apple,t8140-pmp-report.yaml# diff --git a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml index 9784c62fba89e..997761ed5ad4d 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- $id: http://devicetree.org/schemas/soc/apple/apple,pmp.yaml# From dba445e6732685313e036e73dc2753c7ed8cbcb6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:21:26 +1100 Subject: [PATCH 1026/1579] soc: apple: clpc: remove the debugfs override of the floor The driver has a writable debugfs file, force, that pins the DCS and fabric floors to fixed states regardless of CPU demand. It was used to measure what each state costs and gains while the default policy was chosen. Nothing uses it any more, and a debugfs file that changes memory-controller performance behind the driver's back is not something to keep in a driver meant for upstream. Remove it. The read-only status file stays. No functional change outside debugfs. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/clpc.c | 55 ++-------------------------------------- 1 file changed, 2 insertions(+), 53 deletions(-) diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 2d66b98c3a9be..911163f13b0c0 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #define APPLE_CLPC_MAX_CLUSTERS 8 @@ -122,7 +121,6 @@ struct apple_clpc { u8 map[APPLE_CLPC_RAILS][APPLE_CLPC_LEVELS]; unsigned int gpu_state[APPLE_CLPC_RAILS]; struct apple_clpc_vote vote[APPLE_CLPC_RAILS]; - int force[APPLE_CLPC_RAILS]; u32 mask; u32 saved_floor; bool gpu_on; @@ -286,7 +284,6 @@ static void apple_clpc_sample(struct apple_clpc *clpc) unsigned int state[APPLE_CLPC_RAILS]; unsigned long now = jiffies; unsigned int i, rail; - int force; clpc->samples++; if (!cpumask_equal(clpc->covered, cpu_possible_mask) && @@ -312,9 +309,6 @@ static void apple_clpc_sample(struct apple_clpc *clpc) demand[rail] = max(demand[rail], clpc->gpu_state[rail]); state[rail] = apple_clpc_vote(&clpc->vote[rail], demand[rail], now); - force = READ_ONCE(clpc->force[rail]); - if (force >= 0) - state[rail] = force; } if (apple_pmc_floor_update(clpc->pmc, clpc->agent, clpc->mask, @@ -345,12 +339,12 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused) apple_pmc_floor_read(clpc->pmc, clpc->agent), clpc->saved_floor); for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - seq_printf(s, "%s: rail %u state %u enabled %u force %d map %u,%u,%u,%u gpu %u\n", + seq_printf(s, "%s: rail %u state %u enabled %u map %u,%u,%u,%u gpu %u\n", rail == APPLE_CLPC_DCS ? "dcs" : "soc", clpc->hw->rail[rail], clpc->vote[rail].state, apple_pmc_voter_enabled(clpc->pmc, clpc->agent, clpc->hw->rail[rail]), - READ_ONCE(clpc->force[rail]), clpc->map[rail][0], + clpc->map[rail][0], clpc->map[rail][1], clpc->map[rail][2], clpc->map[rail][3], clpc->gpu_state[rail]); seq_printf(s, "gpu: %s\n", !clpc->gpu_regmap ? "no input" : @@ -370,48 +364,6 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused) } DEFINE_SHOW_ATTRIBUTE(apple_clpc_status); -/* - * Diagnostic override: "DCS SOC" holds each rail at a fixed state, -1 - * returns that rail to demand control; "-1" alone clears both. - */ -static ssize_t apple_clpc_force_write(struct file *file, - const char __user *ubuf, size_t len, - loff_t *pos) -{ - struct apple_clpc *clpc = file->private_data; - int v[APPLE_CLPC_RAILS], n; - unsigned int rail; - char buf[32]; - - if (len >= sizeof(buf)) - return -EINVAL; - if (copy_from_user(buf, ubuf, len)) - return -EFAULT; - buf[len] = '\0'; - - n = sscanf(buf, "%d %d", &v[APPLE_CLPC_DCS], &v[APPLE_CLPC_SOC]); - if (n == 1 && v[APPLE_CLPC_DCS] == -1) - v[APPLE_CLPC_SOC] = -1; - else if (n != 2) - return -EINVAL; - - for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - if (v[rail] < -1 || v[rail] > (int)clpc->hw->max_state[rail]) - return -EINVAL; - for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - WRITE_ONCE(clpc->force[rail], v[rail]); - - dev_info(clpc->dev, "diagnostic override: dcs %d soc %d\n", - v[APPLE_CLPC_DCS], v[APPLE_CLPC_SOC]); - return len; -} - -static const struct file_operations apple_clpc_force_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .write = apple_clpc_force_write, -}; - static int apple_clpc_gpu_input(struct apple_clpc *clpc) { struct device_node *ps, *pmgr; @@ -504,7 +456,6 @@ static int apple_clpc_probe(struct platform_device *pdev) if (clpc->hw->rail[rail] >= apple_pmc_rail_count(clpc->pmc)) return -EINVAL; clpc->mask |= APPLE_PMC_RAIL_MASK(clpc->hw->rail[rail]); - clpc->force[rail] = -1; } ret = apple_clpc_parse_map(clpc, dcs_map, APPLE_CLPC_DCS) ?: @@ -555,8 +506,6 @@ static int apple_clpc_probe(struct platform_device *pdev) clpc->debugfs = debugfs_create_dir(dev_name(dev), NULL); debugfs_create_file("status", 0400, clpc->debugfs, clpc, &apple_clpc_status_fops); - debugfs_create_file("force", 0200, clpc->debugfs, clpc, - &apple_clpc_force_fops); ret = devm_add_action_or_reset(dev, apple_clpc_stop, clpc); if (ret) From 9dfd51203aa80ce8867d98ba03c79fbb283039c2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:22:11 +1100 Subject: [PATCH 1027/1579] soc: apple: clpc: make the demand policy per-SoC data, drop the GPU input The sampling period, busy threshold, hold time and the frequency-to-state tables are module parameters, along with an optional GPU floor. They were knobs for choosing the policy; the values the driver ships with are the measured result, and nothing sets the parameters. Module parameters that change memory-controller policy also become ABI once the driver is upstream. Turn the timing values into constants and move the state tables into the per-SoC data, next to the rails they apply to. The map validation against each rail's highest state goes with the parameters, since the tables can no longer be wrong at runtime. The GPU input only acts when gpu_dcs or gpu_soc is set, and both default to 0, so it has never contributed to the floor: the GPU raises the DCS through the PMP's GFX report instead. Remove it, and with it the read of the PMGR GPU power state through the PMGR syscon. The device tree property it read goes in the following patches. No functional change with the default parameters. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/clpc.c | 170 +++++++-------------------------------- 1 file changed, 29 insertions(+), 141 deletions(-) diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 911163f13b0c0..76bd9d3815acb 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -5,34 +5,28 @@ * On macOS the CLPC raises the floor of the memory controller (DCS) and the * SoC fabric while the CPU clusters or the GPU are busy, through its own * voter agent in the PMC. This driver keeps that agent's floor in step with - * Linux CPU load and, optionally, GPU activity: + * Linux CPU load: * * - every sampling period, each cpufreq policy reports how busy its * busiest CPU was and how fast the policy runs relative to its maximum; * a busy policy requests a DCS and fabric state from a table indexed by * that frequency quartile; - * - while the GPU's PMGR power state reports the GPU as actually powered, - * a fixed GPU floor is requested; * - each rail keeps its last request for a hold time after demand stops, * then drops back to 0, so idle releases the floor. * - * The native mapping from CPU and GPU demand to DCS and fabric states has - * not been decoded. The default tables are measured policy, not a copy of - * macOS, and can be changed with module parameters. + * The native mapping from CPU demand to DCS and fabric states is not known. + * The tables below are measured policy, not a copy of macOS. * * Copyright The Asahi Linux Contributors */ -#include #include #include #include #include -#include #include #include #include -#include #include #include #include @@ -40,52 +34,28 @@ #define APPLE_CLPC_MAX_CLUSTERS 8 #define APPLE_CLPC_LEVELS 4 -/* PMGR power state: ACTUAL is bits [7:4]; 0xf means powered on. */ -#define APPLE_PMGR_PS_ACTUAL GENMASK(7, 4) -#define APPLE_PMGR_PS_ACTIVE 0xf - enum apple_clpc_rail { APPLE_CLPC_DCS, APPLE_CLPC_SOC, APPLE_CLPC_RAILS, }; -static unsigned int sample_ms = 4; -module_param(sample_ms, uint, 0444); -MODULE_PARM_DESC(sample_ms, "Sampling period in ms"); - -static unsigned int busy_pct = 50; -module_param(busy_pct, uint, 0444); -MODULE_PARM_DESC(busy_pct, "Busy threshold of a CPU cluster, percent of the sampling period"); - -static unsigned int hold_ms = 32; -module_param(hold_ms, uint, 0444); -MODULE_PARM_DESC(hold_ms, "Time a rail keeps its last request after demand stops, ms"); - -static char *dcs_map = "1,2,3,4"; -module_param(dcs_map, charp, 0444); -MODULE_PARM_DESC(dcs_map, "DCS state for a busy cluster below 25, 50, 75 and 100 percent of its maximum frequency"); - -static char *soc_map = "0,0,0,0"; -module_param(soc_map, charp, 0444); -MODULE_PARM_DESC(soc_map, "Fabric state for a busy cluster below 25, 50, 75 and 100 percent of its maximum frequency"); - -static unsigned int gpu_dcs; -module_param(gpu_dcs, uint, 0444); -MODULE_PARM_DESC(gpu_dcs, "DCS state while the GPU is powered (0: none)"); - -static unsigned int gpu_soc; -module_param(gpu_soc, uint, 0444); -MODULE_PARM_DESC(gpu_soc, "Fabric state while the GPU is powered (0: none)"); +/* Sampling period */ +#define APPLE_CLPC_SAMPLE_MS 4 +/* Busy threshold of a CPU cluster, percent of the sampling period */ +#define APPLE_CLPC_BUSY_PCT 50 +/* Time a rail keeps its last request after demand stops */ +#define APPLE_CLPC_HOLD_MS 32 /** * struct apple_clpc_hw - per-SoC CLPC parameters * @rail: PMC rail of the DCS and of the fabric - * @max_state: highest state of the DCS and of the fabric + * @map: state of each rail for a busy cluster below 25, 50, 75 and 100 + * percent of its maximum frequency */ struct apple_clpc_hw { unsigned int rail[APPLE_CLPC_RAILS]; - unsigned int max_state[APPLE_CLPC_RAILS]; + u8 map[APPLE_CLPC_RAILS][APPLE_CLPC_LEVELS]; }; struct apple_clpc_cluster { @@ -106,8 +76,6 @@ struct apple_clpc { struct device *pmc; const struct apple_clpc_hw *hw; unsigned int agent; - struct regmap *gpu_regmap; - u32 gpu_offset; /* Samples while a vote is held; idle_work defers to busy CPUs. */ struct delayed_work work; struct delayed_work idle_work; @@ -118,34 +86,14 @@ struct apple_clpc { unsigned long next_scan; u64 *idle; u64 *wall; - u8 map[APPLE_CLPC_RAILS][APPLE_CLPC_LEVELS]; - unsigned int gpu_state[APPLE_CLPC_RAILS]; struct apple_clpc_vote vote[APPLE_CLPC_RAILS]; u32 mask; u32 saved_floor; - bool gpu_on; unsigned long samples; unsigned long update_errors; struct dentry *debugfs; }; -static int apple_clpc_parse_map(struct apple_clpc *clpc, const char *s, - enum apple_clpc_rail rail) -{ - unsigned int v[APPLE_CLPC_LEVELS]; - unsigned int i; - - if (sscanf(s, "%u,%u,%u,%u", &v[0], &v[1], &v[2], &v[3]) != 4) - return -EINVAL; - for (i = 0; i < APPLE_CLPC_LEVELS; i++) { - if (v[i] > clpc->hw->max_state[rail]) - return -EINVAL; - clpc->map[rail][i] = v[i]; - } - - return 0; -} - /* cpufreq policies can appear after probe; pick up new ones as they do. */ static void apple_clpc_find_clusters(struct apple_clpc *clpc) { @@ -220,21 +168,8 @@ static void apple_clpc_sample_cluster(struct apple_clpc *clpc, q = min_t(unsigned int, cl->level / (100 / APPLE_CLPC_LEVELS), APPLE_CLPC_LEVELS - 1); for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - cl->request[rail] = busy >= busy_pct ? clpc->map[rail][q] : 0; -} - -static bool apple_clpc_gpu_powered(struct apple_clpc *clpc) -{ - u32 ps; - - /* - * Hardware auto-gating powers the GPU down while the requested state - * stays on, so only the actual state says whether the GPU runs. - */ - if (regmap_read(clpc->gpu_regmap, clpc->gpu_offset, &ps)) - return false; - - return FIELD_GET(APPLE_PMGR_PS_ACTUAL, ps) == APPLE_PMGR_PS_ACTIVE; + cl->request[rail] = busy >= APPLE_CLPC_BUSY_PCT ? + clpc->hw->map[rail][q] : 0; } static unsigned int apple_clpc_vote(struct apple_clpc_vote *vote, @@ -244,7 +179,7 @@ static unsigned int apple_clpc_vote(struct apple_clpc_vote *vote, vote->last_demand = now; vote->state = demand; } else if (time_after_eq(now, vote->last_demand + - msecs_to_jiffies(hold_ms))) { + msecs_to_jiffies(APPLE_CLPC_HOLD_MS))) { vote->state = 0; } @@ -275,7 +210,7 @@ static void apple_clpc_queue(struct apple_clpc *clpc, bool idle) */ queue_delayed_work(system_freezable_power_efficient_wq, idle ? &clpc->idle_work : &clpc->work, - msecs_to_jiffies(sample_ms)); + msecs_to_jiffies(APPLE_CLPC_SAMPLE_MS)); } static void apple_clpc_sample(struct apple_clpc *clpc) @@ -300,16 +235,9 @@ static void apple_clpc_sample(struct apple_clpc *clpc) demand[rail] = max(demand[rail], cl->request[rail]); } - clpc->gpu_on = clpc->gpu_regmap && - (clpc->gpu_state[APPLE_CLPC_DCS] || - clpc->gpu_state[APPLE_CLPC_SOC]) && - apple_clpc_gpu_powered(clpc); - for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) { - if (clpc->gpu_on) - demand[rail] = max(demand[rail], clpc->gpu_state[rail]); + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) state[rail] = apple_clpc_vote(&clpc->vote[rail], demand[rail], now); - } if (apple_pmc_floor_update(clpc->pmc, clpc->agent, clpc->mask, apple_clpc_floor(clpc, state))) @@ -339,18 +267,13 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused) apple_pmc_floor_read(clpc->pmc, clpc->agent), clpc->saved_floor); for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - seq_printf(s, "%s: rail %u state %u enabled %u map %u,%u,%u,%u gpu %u\n", + seq_printf(s, "%s: rail %u state %u enabled %u map %u,%u,%u,%u\n", rail == APPLE_CLPC_DCS ? "dcs" : "soc", clpc->hw->rail[rail], clpc->vote[rail].state, apple_pmc_voter_enabled(clpc->pmc, clpc->agent, clpc->hw->rail[rail]), - clpc->map[rail][0], - clpc->map[rail][1], clpc->map[rail][2], - clpc->map[rail][3], clpc->gpu_state[rail]); - seq_printf(s, "gpu: %s\n", !clpc->gpu_regmap ? "no input" : - !clpc->gpu_state[APPLE_CLPC_DCS] && - !clpc->gpu_state[APPLE_CLPC_SOC] ? "no floor" : - clpc->gpu_on ? "powered" : "off"); + clpc->hw->map[rail][0], clpc->hw->map[rail][1], + clpc->hw->map[rail][2], clpc->hw->map[rail][3]); seq_printf(s, "samples: %lu update errors: %lu\n", clpc->samples, clpc->update_errors); for (i = 0; i < clpc->nr_clusters; i++) @@ -364,34 +287,6 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused) } DEFINE_SHOW_ATTRIBUTE(apple_clpc_status); -static int apple_clpc_gpu_input(struct apple_clpc *clpc) -{ - struct device_node *ps, *pmgr; - int ret; - - ps = of_parse_phandle(clpc->dev->of_node, "apple,gpu-power-state", 0); - if (!ps) - return 0; - - ret = of_property_read_u32(ps, "reg", &clpc->gpu_offset); - pmgr = of_get_parent(ps); - of_node_put(ps); - if (ret) { - of_node_put(pmgr); - return ret; - } - - clpc->gpu_regmap = syscon_node_to_regmap(pmgr); - of_node_put(pmgr); - if (IS_ERR(clpc->gpu_regmap)) { - ret = PTR_ERR(clpc->gpu_regmap); - clpc->gpu_regmap = NULL; - return ret; - } - - return 0; -} - static void apple_clpc_free_clusters(struct apple_clpc *clpc) { unsigned int i; @@ -458,20 +353,6 @@ static int apple_clpc_probe(struct platform_device *pdev) clpc->mask |= APPLE_PMC_RAIL_MASK(clpc->hw->rail[rail]); } - ret = apple_clpc_parse_map(clpc, dcs_map, APPLE_CLPC_DCS) ?: - apple_clpc_parse_map(clpc, soc_map, APPLE_CLPC_SOC); - if (ret) - return dev_err_probe(dev, ret, "invalid state map\n"); - if (gpu_dcs > clpc->hw->max_state[APPLE_CLPC_DCS] || - gpu_soc > clpc->hw->max_state[APPLE_CLPC_SOC]) - return dev_err_probe(dev, -EINVAL, "invalid GPU state\n"); - clpc->gpu_state[APPLE_CLPC_DCS] = gpu_dcs; - clpc->gpu_state[APPLE_CLPC_SOC] = gpu_soc; - - ret = apple_clpc_gpu_input(clpc); - if (ret) - return dev_err_probe(dev, ret, "invalid GPU power state\n"); - clpc->idle = devm_kcalloc(dev, nr_cpu_ids, sizeof(*clpc->idle), GFP_KERNEL); clpc->wall = devm_kcalloc(dev, nr_cpu_ids, sizeof(*clpc->wall), @@ -542,10 +423,17 @@ static int apple_clpc_resume(struct device *dev) static DEFINE_SIMPLE_DEV_PM_OPS(apple_clpc_pm_ops, apple_clpc_suspend, apple_clpc_resume); -/* T8140: DCS states 0-4 on rail 0, fabric states 0-3 on rail 1. */ +/* + * T8140: DCS states 0-4 on rail 0, fabric states 0-3 on rail 1. A busy + * cluster raises the DCS one state per frequency quartile and leaves the + * fabric alone. + */ static const struct apple_clpc_hw apple_clpc_hw_t8140 = { .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, - .max_state = { [APPLE_CLPC_DCS] = 4, [APPLE_CLPC_SOC] = 3 }, + .map = { + [APPLE_CLPC_DCS] = { 1, 2, 3, 4 }, + [APPLE_CLPC_SOC] = { 0, 0, 0, 0 }, + }, }; static const struct of_device_id apple_clpc_of_match[] = { From e666e50ccfe8c58e8fa72bdf7ef6bedffec6551a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:27:48 +1100 Subject: [PATCH 1028/1579] rtc: macsmc: correct the comment on the whole-second offset With apple,rtc-offset-seconds the offset is stored in whole seconds. set_time() subtracts the counter's whole seconds from the requested time, so the fraction of a second the counter had reached is dropped and the next second starts early: the clock runs up to a second ahead. The comment says the fraction "carries into the new time", which reads as the opposite. Say what happens. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/rtc/rtc-macsmc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/rtc/rtc-macsmc.c b/drivers/rtc/rtc-macsmc.c index 25d5af5cfbbcd..1c1c529df8296 100644 --- a/drivers/rtc/rtc-macsmc.c +++ b/drivers/rtc/rtc-macsmc.c @@ -113,7 +113,10 @@ static int macsmc_rtc_set_time(struct device *dev, struct rtc_time *tm) if (rtc->offset_in_seconds) { __le64 off_secs; - /* The counter's fraction of a second carries into the new time */ + /* + * The offset holds whole seconds, so the counter's fraction of a + * second is lost: the clock may run up to a second ahead. + */ off_secs = cpu_to_le64(rtc_tm_to_time64(tm) - (ctr >> RTC_SEC_SHIFT)); ret = nvmem_cell_write(rtc->rtc_offset, &off_secs, sizeof(off_secs)); return ret < 0 ? ret : 0; From 807cb8976697ff272433278b272444ced7e91d57 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:05:55 +1100 Subject: [PATCH 1029/1579] hwmon: macsmc: account for temperature channels in one helper Move the code that turns a temperature node into a channel into macsmc_hwmon_add_temp_channel(), so that the next patch can change how a node that fails to set up is accounted for in one place, and a unit test can exercise it without an SMC. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hwmon/macsmc-hwmon.c | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index b0172bbb76e98..bfdd357849c61 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -501,6 +501,21 @@ static int macsmc_hwmon_create_sensor(struct device *dev, struct apple_smc *smc, return 0; } +/* Account for the temperature sensor just set up at the end of @temp. */ +static void macsmc_hwmon_add_temp_channel(struct macsmc_hwmon_sensors *temp, + int create_ret) +{ + struct macsmc_hwmon_sensor *sensor = &temp->sensors[temp->count]; + + if (create_ret) + return; + + sensor->attrs = HWMON_T_INPUT; + if (*sensor->label) + sensor->attrs |= HWMON_T_LABEL; + temp->count++; +} + /* Discover reporting-only temperatures without a board-specific key table. */ static int macsmc_hwmon_discover_temperatures(struct macsmc_hwmon *hwmon) { @@ -637,6 +652,7 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, struct device_node *key_node __maybe_unused; struct macsmc_hwmon_sensor *sensor; u32 n_current = 0, n_fan = 0, n_power = 0, n_temperature = 0, n_voltage = 0; + int ret; for_each_child_of_node_with_prefix(hwmon_node, key_node, "current-") { n_current++; @@ -713,14 +729,9 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, for_each_child_of_node_with_prefix(hwmon_node, key_node, "temperature-") { sensor = &hwmon->temp.sensors[hwmon->temp.count]; - if (!macsmc_hwmon_create_sensor(hwmon->dev, hwmon->smc, key_node, sensor)) { - sensor->attrs = HWMON_T_INPUT; - - if (*sensor->label) - sensor->attrs |= HWMON_T_LABEL; - - hwmon->temp.count++; - } + ret = macsmc_hwmon_create_sensor(hwmon->dev, hwmon->smc, + key_node, sensor); + macsmc_hwmon_add_temp_channel(&hwmon->temp, ret); } } From cbd157ef2a12a46e3caf5367de912a525d8d4874 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:09:26 +1100 Subject: [PATCH 1030/1579] hwmon: macsmc: keep the channel of a temperature sensor that fails to set up Since the SMC temperatures became thermal sensors, thermal zones refer to them by channel index, and the binding defines that index as the position of the node among the temperature-* nodes. The driver only counts nodes whose key it could look up, so when one fails, for instance because the SMC firmware does not have the key, every later channel moves down by one. Every zone that refers to a later channel then silently reads a different sensor. On T8140 the six CPU die zones use the first six channels, so a missing die key would move every later CPU zone onto its neighbour's sensor and the last one onto a sensor outside the CPU clusters. On a fanless machine that can mean running hot without throttling. Give every temperature node its channel. A sensor that fails to set up keeps its slot with no key; is_visible() hides it, so hwmon creates no attributes and no thermal sensor for it, and only the zones that use that channel fail to register. The config entry stays non-zero, since a zero entry would end the channel list. Say which node failed. The KUnit test sets up three channels with the middle one failing and checks that the third keeps its index and key and that the middle one is hidden. It fails before this change (two channels, the third key in slot 1) and passes after it. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 2 +- drivers/hwmon/Kconfig | 10 ++++ drivers/hwmon/macsmc-hwmon-test.c | 87 +++++++++++++++++++++++++++++++ drivers/hwmon/macsmc-hwmon.c | 28 ++++++++-- 4 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 drivers/hwmon/macsmc-hwmon-test.c diff --git a/MAINTAINERS b/MAINTAINERS index cc6fe5b42f38a..aafd54ac18b65 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2624,7 +2624,7 @@ F: drivers/clk/clk-apple-nco.c F: drivers/cpufreq/apple-soc-cpufreq.c F: drivers/dma/apple-admac.c F: drivers/gpio/gpio-macsmc.c -F: drivers/hwmon/macsmc-hwmon.c +F: drivers/hwmon/macsmc-hwmon*.c F: drivers/i2c/busses/i2c-pasemi-core.c F: drivers/i2c/busses/i2c-pasemi-platform.c F: drivers/input/misc/macsmc-input.c diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index b1f121b84bf68..59e379f867d71 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -1174,6 +1174,16 @@ config SENSORS_MACSMC_HWMON on Apple Silicon devices. Say Y here if you have an Apple Silicon device. +config SENSORS_MACSMC_HWMON_KUNIT_TEST + bool "KUnit tests for the Apple SMC hwmon driver" if !KUNIT_ALL_TESTS + depends on SENSORS_MACSMC_HWMON && KUNIT=y + default KUNIT_ALL_TESTS + help + Unit tests for the channel numbering of the Apple SMC hwmon driver. + Thermal zones refer to temperature channels by index, so a sensor + that cannot be set up must keep its channel; the tests check that. + No SMC is needed. Only useful for kernel developers. + This driver can also be built as a module. If so, the module will be called macsmc-hwmon. diff --git a/drivers/hwmon/macsmc-hwmon-test.c b/drivers/hwmon/macsmc-hwmon-test.c new file mode 100644 index 0000000000000..e7239986ce3c9 --- /dev/null +++ b/drivers/hwmon/macsmc-hwmon-test.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * KUnit tests for the Apple SMC hwmon driver: channel numbering of the + * temperature sensors, which thermal zones depend on. + * + * Included by macsmc-hwmon.c. + */ + +#include + +/* What macsmc_hwmon_create_sensor() leaves behind for one node. */ +static void macsmc_hwmon_test_add(struct macsmc_hwmon_sensors *temp, + const char *key, int create_ret) +{ + struct macsmc_hwmon_sensor *sensor = &temp->sensors[temp->count]; + + sensor->macsmc_key = _SMC_KEY(key); + strscpy_pad(sensor->label, key, sizeof(sensor->label)); + macsmc_hwmon_add_temp_channel(temp, create_ret); +} + +/* A node that fails to set up keeps its channel; later channels do not move. */ +static void macsmc_hwmon_test_failed_temp_keeps_slot(struct kunit *test) +{ + struct macsmc_hwmon_sensor sensors[3] = {}; + struct macsmc_hwmon hwmon = { + .temp = { .sensors = sensors }, + }; + u32 configs[ARRAY_SIZE(sensors) + 1] = {}; + + macsmc_hwmon_test_add(&hwmon.temp, "Te05", 0); + macsmc_hwmon_test_add(&hwmon.temp, "Te0S", -EINVAL); + macsmc_hwmon_test_add(&hwmon.temp, "Tp01", 0); + + KUNIT_ASSERT_EQ(test, hwmon.temp.count, 3); + KUNIT_EXPECT_EQ(test, sensors[0].macsmc_key, _SMC_KEY("Te05")); + KUNIT_EXPECT_EQ(test, sensors[2].macsmc_key, _SMC_KEY("Tp01")); + + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_input, 0), 0444); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_input, 1), 0); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_label, 1), 0); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_input, 2), 0444); + + /* A zero entry would end the channel list at the failed sensor. */ + macsmc_hwmon_populate_configs(configs, &hwmon.temp); + KUNIT_EXPECT_NE(test, configs[0], 0); + KUNIT_EXPECT_NE(test, configs[1], 0); + KUNIT_EXPECT_NE(test, configs[2], 0); + KUNIT_EXPECT_EQ(test, configs[3], 0); +} + +static void macsmc_hwmon_test_temp_label(struct kunit *test) +{ + struct macsmc_hwmon_sensor sensors[2] = {}; + struct macsmc_hwmon hwmon = { + .temp = { .sensors = sensors }, + }; + + macsmc_hwmon_test_add(&hwmon.temp, "Tp05", 0); + sensors[1].macsmc_key = _SMC_KEY("Tp09"); + macsmc_hwmon_add_temp_channel(&hwmon.temp, 0); + + KUNIT_ASSERT_EQ(test, hwmon.temp.count, 2); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_label, 0), 0444); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_label, 1), 0); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_input, 1), 0444); +} + +static struct kunit_case macsmc_hwmon_test_cases[] = { + KUNIT_CASE(macsmc_hwmon_test_failed_temp_keeps_slot), + KUNIT_CASE(macsmc_hwmon_test_temp_label), + {} +}; + +static struct kunit_suite macsmc_hwmon_test_suite = { + .name = "macsmc-hwmon", + .test_cases = macsmc_hwmon_test_cases, +}; + +kunit_test_suite(macsmc_hwmon_test_suite); diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index bfdd357849c61..40b3b0b5fb41e 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -436,6 +436,10 @@ static umode_t macsmc_hwmon_is_visible(const void *data, return 0; } + /* A sensor without a key could not be set up; see macsmc_hwmon_add_temp_channel(). */ + if (!sensor->macsmc_key) + return 0; + /* Sensors only register ro attributes */ if (sensor->attrs & BIT(attr)) return 0444; @@ -501,19 +505,28 @@ static int macsmc_hwmon_create_sensor(struct device *dev, struct apple_smc *smc, return 0; } -/* Account for the temperature sensor just set up at the end of @temp. */ +/* + * Account for the temperature sensor just set up at the end of @temp. + * + * Thermal zones refer to a temperature channel by the index of its node + * among the temperature-* nodes, so every node takes a channel. A sensor + * that could not be set up keeps its channel with no key, which hides it: + * hwmon creates no attributes and no thermal sensor for it, and only the + * zones that use it fail to register, instead of every later channel + * moving onto the wrong sensor. + */ static void macsmc_hwmon_add_temp_channel(struct macsmc_hwmon_sensors *temp, int create_ret) { - struct macsmc_hwmon_sensor *sensor = &temp->sensors[temp->count]; + struct macsmc_hwmon_sensor *sensor = &temp->sensors[temp->count++]; if (create_ret) - return; + memset(sensor, 0, sizeof(*sensor)); + /* A zero entry would end the list of channels here. */ sensor->attrs = HWMON_T_INPUT; if (*sensor->label) sensor->attrs |= HWMON_T_LABEL; - temp->count++; } /* Discover reporting-only temperatures without a board-specific key table. */ @@ -731,6 +744,9 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, sensor = &hwmon->temp.sensors[hwmon->temp.count]; ret = macsmc_hwmon_create_sensor(hwmon->dev, hwmon->smc, key_node, sensor); + if (ret) + dev_warn(hwmon->dev, "%pOFn unavailable (%d), temp%u hidden\n", + key_node, ret, hwmon->temp.count + 1); macsmc_hwmon_add_temp_channel(&hwmon->temp, ret); } } @@ -932,6 +948,10 @@ static struct platform_driver macsmc_hwmon_driver = { }; module_platform_driver(macsmc_hwmon_driver); +#if IS_ENABLED(CONFIG_SENSORS_MACSMC_HWMON_KUNIT_TEST) +#include "macsmc-hwmon-test.c" +#endif + MODULE_DESCRIPTION("Apple Silicon SMC hwmon driver"); MODULE_AUTHOR("James Calligeros "); MODULE_LICENSE("Dual MIT/GPL"); From 4cf2113dc36329544edc54b09dc775c5888c3975 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:10:00 +1100 Subject: [PATCH 1031/1579] soc: apple: mcc: factor out disabling and flushing planes Move the loop that stops allocation and flushes each plane out of apple_mcc_quiesce() into apple_mcc_disable_planes(), which takes the number of planes to act on, so that the next patch can use it on the planes enabled before a failure. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mcc.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c index 10f5e6e01e8cd..d0f6c1362b308 100644 --- a/drivers/soc/apple/mcc.c +++ b/drivers/soc/apple/mcc.c @@ -72,6 +72,24 @@ static bool apple_mcc_plane_ready(struct apple_mcc *mcc, unsigned int plane) return false; } +/* Stop allocating on the first @count planes, then write back and invalidate them. */ +static void apple_mcc_disable_planes(struct apple_mcc *mcc, unsigned int count) +{ + unsigned int i; + u32 val; + + for (i = 0; i < count; i++) { + writel(1, mcc->planes[i] + MCC_ALLOC_DISABLE); + writel(1, mcc->planes[i] + MCC_CACHE_FLUSH); + } + + for (i = 0; i < count; i++) + if (readl_poll_timeout_atomic(mcc->planes[i] + MCC_CACHE_FLUSH, val, + !(val & MCC_CACHE_FLUSH_BUSY), 10, + MCC_FLUSH_TIMEOUT_US)) + dev_err(mcc->dev, "plane %u flush timed out\n", i); +} + static int apple_mcc_enable(struct apple_mcc *mcc) { unsigned int i; @@ -91,26 +109,12 @@ static int apple_mcc_enable(struct apple_mcc *mcc) return 0; } -/* Stop allocating, then write back and invalidate the cache. */ static void apple_mcc_quiesce(struct apple_mcc *mcc) { - unsigned int i; - u32 val; - if (!mcc->enabled) return; - for (i = 0; i < mcc->nr_planes; i++) { - writel(1, mcc->planes[i] + MCC_ALLOC_DISABLE); - writel(1, mcc->planes[i] + MCC_CACHE_FLUSH); - } - - for (i = 0; i < mcc->nr_planes; i++) - if (readl_poll_timeout_atomic(mcc->planes[i] + MCC_CACHE_FLUSH, val, - !(val & MCC_CACHE_FLUSH_BUSY), 10, - MCC_FLUSH_TIMEOUT_US)) - dev_err(mcc->dev, "plane %u flush timed out\n", i); - + apple_mcc_disable_planes(mcc, mcc->nr_planes); mcc->enabled = false; } From 8bf158a74ea7e2a5a7e42082fc882d3508119a09 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:10:03 +1100 Subject: [PATCH 1032/1579] soc: apple: mcc: disable the planes already enabled when enabling fails apple_mcc_enable() clears the allocation-disable bit plane by plane and reads it back. If a later plane does not take it, probe returns -EIO with the earlier planes allocating. The driver does not bind, so neither remove() nor shutdown() runs, and nothing stops allocation or writes the cache back before the next kernel or the firmware takes over, which is what the driver promises to do on shutdown. Disable and flush the planes enabled so far, and the one that failed, whose state is unknown, before returning the error. The failure needs a plane whose register does not take the write, which cannot be produced on a working machine or with memory standing in for the registers, so there is no automated test; the normal path is unchanged, and the on-device check is that probe still reports allocation enabled on every plane. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mcc.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c index d0f6c1362b308..7b8c2e2220c8b 100644 --- a/drivers/soc/apple/mcc.c +++ b/drivers/soc/apple/mcc.c @@ -102,6 +102,12 @@ static int apple_mcc_enable(struct apple_mcc *mcc) writel(0, mcc->planes[i] + MCC_ALLOC_DISABLE); if (readl(mcc->planes[i] + MCC_ALLOC_DISABLE)) { dev_err(mcc->dev, "plane %u did not enable allocation\n", i); + /* + * The driver will not bind, so nothing disables these + * planes on shutdown: disable the ones enabled so far, + * and this one, whose state is unknown, now. + */ + apple_mcc_disable_planes(mcc, i + 1); return -EIO; } } From 4653ef5a873f16c68a1e9e9080a41395628ca9d4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:10:45 +1100 Subject: [PATCH 1033/1579] pmdomain: apple: pmc: undo a partial agent enable apple_pmc_agent_enable() changes the agent's bit rail by rail and stops at the first rail that does not take the write. The rails before it keep the change, and pmc->enable[] records the failed one as changed too. For a dynamic voter domain genpd treats a failed power_on() as the domain staying off and never calls power_off(), so the agent stays enabled on those rails for good; a failed disable likewise leaves it half disabled while genpd keeps the domain on. Snapshot the enables before the change and, on failure, put back the rails changed so far and the one that failed, so the agent is in the same state on every rail and that state matches what genpd records. The failure needs a rail enable word that does not take the write, which memory standing in for the registers cannot produce, so the KUnit test added with the next patch covers the successful path only. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmc.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 397fce7fc8ad2..5ae62280734b0 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -151,18 +151,46 @@ static int apple_pmc_enable_update(struct apple_pmc *pmc, unsigned int rail, return apple_pmc_enable_write(pmc, rail); } +/* + * Caller holds pmc->lock. Put the rails up to and including @last back to + * the enables in @prev, after a change failed on @last. + */ +static void apple_pmc_enable_restore(struct apple_pmc *pmc, const u32 *prev, + unsigned int last) +{ + unsigned int rail; + + for (rail = 0; rail <= last; rail++) { + pmc->enable[rail] = prev[rail]; + apple_pmc_enable_write(pmc, rail); + } +} + +/* + * Enable or disable @agent on every rail. If a rail does not take the + * change, undo it on the rails changed so far: genpd keeps a domain whose + * power_on() fails off, and never calls its power_off(), so an agent left + * enabled on some rails would stay enabled for good. + */ static int apple_pmc_agent_enable(struct apple_pmc *pmc, unsigned int agent, bool enable) { + u32 prev[APPLE_PMC_MAX_RAILS]; unsigned long flags; unsigned int rail; int ret = 0; raw_spin_lock_irqsave(&pmc->lock, flags); - for (rail = 0; rail < pmc->hw->rails && !ret; rail++) + memcpy(prev, pmc->enable, sizeof(prev)); + for (rail = 0; rail < pmc->hw->rails; rail++) { ret = apple_pmc_enable_update(pmc, rail, enable ? BIT(agent) : 0, enable ? 0 : BIT(agent)); + if (ret) { + apple_pmc_enable_restore(pmc, prev, rail); + break; + } + } raw_spin_unlock_irqrestore(&pmc->lock, flags); return ret; From a86e8e5637c07504ca09b7bfa5639492af065d8c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:13:13 +1100 Subject: [PATCH 1034/1579] pmdomain: apple: pmc: restore the AP's own DCS vote when a limit is lifted The DCS limit votes through the AP's agent on behalf of the agents it stops honouring. When the limit is lifted, and when the driver is torn down, apple_pmc_dcs_vote() writes 0 into the AP agent's DCS field. That discards whatever the AP agent voted before Linux took over, so a single limit cycle changes the boot configuration of the memory controller. Read the AP agent's DCS vote at probe, before anything writes it, and put it back instead of 0 whenever no limit applies. Add a KUnit test with memory standing in for the registers. It checks that agent enables only change the agent's own bit, and that after a limit cycle the AP agent's DCS vote is the one it started with; the latter fails without this change (the vote reads 0) and passes with it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/Kconfig | 10 ++++ drivers/pmdomain/apple/pmc-test.c | 95 +++++++++++++++++++++++++++++++ drivers/pmdomain/apple/pmc.c | 22 ++++++- 3 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 drivers/pmdomain/apple/pmc-test.c diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index 2494e7aece065..2a70d364f88c0 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -34,4 +34,14 @@ config APPLE_PMC GPU, while its device is powered on. It can also limit the DRAM controller rail as a thermal cooling device. +config APPLE_PMC_KUNIT_TEST + bool "KUnit tests for the Apple PMC voter interface" if !KUNIT_ALL_TESTS + depends on APPLE_PMC && KUNIT=y + default KUNIT_ALL_TESTS + help + Unit tests for the Apple PMC driver. They check that enabling and + disabling a voter agent only changes that agent on each rail, and + that lifting a DCS limit gives the AP agent its own vote back. The + registers are simulated in memory. Only useful for kernel developers. + endif diff --git a/drivers/pmdomain/apple/pmc-test.c b/drivers/pmdomain/apple/pmc-test.c new file mode 100644 index 0000000000000..3f0e277b21255 --- /dev/null +++ b/drivers/pmdomain/apple/pmc-test.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * KUnit tests for the Apple PMC voter interface: agent enables and the + * AP's DCS vote, with memory standing in for the registers. + * + * Included by pmc.c. + */ + +#include + +#define APPLE_PMC_TEST_RAILS_SIZE 0x4000 + +static struct apple_pmc *apple_pmc_test_alloc(struct kunit *test) +{ + const struct apple_pmc_hw *hw = &apple_pmc_hw_t8140; + struct apple_pmc *pmc; + + pmc = kunit_kzalloc(test, sizeof(*pmc), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pmc); + pmc->hw = hw; + raw_spin_lock_init(&pmc->lock); + pmc->voters = (void __iomem *)kunit_kzalloc(test, + hw->agents * hw->agent_stride, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, (void __force *)pmc->voters); + pmc->rails = (void __iomem *)kunit_kzalloc(test, + APPLE_PMC_TEST_RAILS_SIZE, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, (void __force *)pmc->rails); + + return pmc; +} + +static void apple_pmc_test_set_dcs_vote(struct apple_pmc *pmc, + unsigned int agent, unsigned int state) +{ + void __iomem *reg = pmc->voters + agent * pmc->hw->agent_stride; + + writel(state << APPLE_PMC_RAIL_SHIFT(pmc->hw->dcs_rail), reg); +} + +/* Enabling and disabling an agent changes its bit on every rail only. */ +static void apple_pmc_test_agent_enable(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + unsigned int rail; + + pmc->enable[0] = BIT(0); + writel(BIT(0), pmc->rails + pmc->hw->rail_enable); + + KUNIT_ASSERT_EQ(test, apple_pmc_agent_enable(pmc, 1, true), 0); + for (rail = 0; rail < pmc->hw->rails; rail++) + KUNIT_EXPECT_EQ(test, apple_pmc_enable_read(pmc, rail), + BIT(1) | (rail ? 0 : BIT(0))); + + KUNIT_ASSERT_EQ(test, apple_pmc_agent_enable(pmc, 1, false), 0); + for (rail = 0; rail < pmc->hw->rails; rail++) + KUNIT_EXPECT_EQ(test, apple_pmc_enable_read(pmc, rail), + rail ? 0 : BIT(0)); +} + +/* Lifting a DCS limit gives the AP's agent back its own vote, not 0. */ +static void apple_pmc_test_dcs_limit_restores_ap_vote(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + const struct apple_pmc_hw *hw = pmc->hw; + + apple_pmc_test_set_dcs_vote(pmc, hw->ap_agent, 2); + apple_pmc_test_set_dcs_vote(pmc, 0, 4); + pmc->enable[hw->dcs_rail] = BIT(0) | BIT(hw->ap_agent); + pmc->ap_dcs_vote = apple_pmc_ap_dcs_read(pmc); + KUNIT_ASSERT_EQ(test, pmc->ap_dcs_vote, 2); + + /* A limit of 3 lets the AP vote at most DCS 1 for the PMP's 4. */ + pmc->dcs_state = 3; + apple_pmc_dcs_vote(pmc); + KUNIT_EXPECT_EQ(test, apple_pmc_ap_dcs_read(pmc), 1); + + pmc->dcs_state = 0; + apple_pmc_dcs_vote(pmc); + KUNIT_EXPECT_EQ(test, apple_pmc_ap_dcs_read(pmc), 2); +} + +static struct kunit_case apple_pmc_test_cases[] = { + KUNIT_CASE(apple_pmc_test_agent_enable), + KUNIT_CASE(apple_pmc_test_dcs_limit_restores_ap_vote), + {} +}; + +static struct kunit_suite apple_pmc_test_suite = { + .name = "apple-pmc", + .test_cases = apple_pmc_test_cases, +}; + +kunit_test_suite(apple_pmc_test_suite); diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 5ae62280734b0..d1d37a9917ff4 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -92,6 +92,8 @@ struct apple_pmc { struct dentry *debugfs; /* Cooling state of the DCS limit: 0 is none, dcs_max holds DCS at 0. */ unsigned long dcs_state; + /* DCS state the AP's agent voted for before probe, kept without a limit. */ + unsigned int ap_dcs_vote; /* Power cost of each DCS state in mW, if a model is described. */ u32 dcs_power[APPLE_PMC_MAX_STATES]; struct delayed_work dcs_work; @@ -315,16 +317,27 @@ static unsigned int apple_pmc_dcs_demand(struct apple_pmc *pmc) return min(demand, hw->dcs_max); } +/* The DCS state the AP's agent votes for. */ +static unsigned int apple_pmc_ap_dcs_read(struct apple_pmc *pmc) +{ + const struct apple_pmc_hw *hw = pmc->hw; + u32 floor = readl(pmc->voters + hw->ap_agent * hw->agent_stride); + + return (floor & APPLE_PMC_RAIL_MASK(hw->dcs_rail)) >> + APPLE_PMC_RAIL_SHIFT(hw->dcs_rail); +} + /* * Caller holds pmc->lock. Vote through the AP's agent for what the other - * agents ask for, capped by the limit; vote nothing without a limit. + * agents ask for, capped by the limit; without a limit, put back the vote + * the AP's agent had before probe. */ static void apple_pmc_dcs_vote(struct apple_pmc *pmc) { const struct apple_pmc_hw *hw = pmc->hw; void __iomem *reg = pmc->voters + hw->ap_agent * hw->agent_stride; u32 mask = APPLE_PMC_RAIL_MASK(hw->dcs_rail); - unsigned int state = 0; + unsigned int state = pmc->ap_dcs_vote; u32 old, val; if (pmc->dcs_state) @@ -703,6 +716,7 @@ static int apple_pmc_probe(struct platform_device *pdev) "PMC is not responding\n"); pmc->enable[rail] = pmc->saved_enable[rail]; } + pmc->ap_dcs_vote = apple_pmc_ap_dcs_read(pmc); count = of_property_count_u32_elems(np, "apple,static-voters"); if (count > 0) { @@ -797,3 +811,7 @@ module_init(apple_pmc_init); MODULE_DESCRIPTION("Apple SoC PMC voter interface driver"); MODULE_LICENSE("Dual MIT/GPL"); + +#if IS_ENABLED(CONFIG_APPLE_PMC_KUNIT_TEST) +#include "pmc-test.c" +#endif From 0d1799c7c179d3b6e914eeaee59da7529e824565 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:13:40 +1100 Subject: [PATCH 1035/1579] arm64: dts: apple: t8140-j700: trim measurement notes from comments The PMC node's comment records how the DCS power model and the decision not to limit the DCS rail were measured, down to the benchmark and the game used. Keep what the values are and why no zone uses the limit; the measurements belong in the history, not in the device tree. Two PCIe comments call themselves experimental; describe what the nodes do instead. No functional change: the compiled device tree is identical. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 033272cdb32d9..942e360f57c4e 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -424,16 +424,12 @@ &pmc { /* - * System power above DCS state 0 at each DCS state, measured with - * glmark2's terrain scene, GPU-bound, on the J700. + * System power above DCS state 0 at each DCS state, GPU-bound. * - * No thermal zone limits the DCS rail. The PMP already drops its - * DCS vote when the GPU idles, and under load the rail must stay - * where the PMP puts it: a power allocator zone on Tp0t at 95 C - * stepped the limit between states 1 and 4 every 1-2 s in - * Minecraft, swinging the game's GPU submission rate about 2.5x. - * With the limit lifted the rate held within 8% and Tp0t settled - * near 100 C, under the 110 C critical trips of the CPU zones. + * No thermal zone limits the DCS rail: the PMP drops its DCS vote + * when the GPU idles, and a power allocator zone on Tp0t made the + * limit oscillate between states 1 and 4 under GPU load. Without + * it Tp0t stays below the 110 C critical trips of the CPU zones. */ apple,dcs-power-microwatt = <0 919000 1453000 2058000 2407000>; status = "okay"; @@ -1826,7 +1822,7 @@ #include "hwmon-laptop.dtsi" &pcie0 { - /* Experiment: guard stays closed until the retained bootstrap succeeds. */ + /* The PIODMA guard stays closed until the retained bootstrap succeeds. */ apple,piodma = <&pcie0_piodma_diag>; iommu-map = <0x100 &pcie0_dart 1 2>; iommu-map-mask = <0xffff>; @@ -1848,7 +1844,7 @@ }; }; -/* Experimental SID17 bootstrap; arena ownership lasts until full reset. */ +/* SID17 bootstrap; the arena stays owned until a full reset. */ &pcie0_dart { status = "okay"; apple,dma-range = /bits/ 64 <0 0x1000000000>; From 41097c6b96cd4f7c18445de1c83ba5350b117b1d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1036/1579] wifi: mt7932: drop the diagnostic module parameters legacy_irq_test, poll_test and reset_test were bring-up switches. They forced INTx, replaced interrupts with polling of the rings, and ran an extra function reset and firmware boot before the interface was registered. All three default to off, nothing sets them, and none of them is needed to run or debug the station. Remove them with the code only they reached: the polled completion wait, mt_transport_polled() and mt_reset_retained(). With the defaults the driver behaves as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/Kconfig | 3 +- drivers/net/wireless/mediatek/mt7932/mt7932.c | 106 ++---------------- drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 - .../net/wireless/mediatek/mt7932/recovery.c | 2 +- 4 files changed, 11 insertions(+), 102 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index b12b2362efb97..273c389ad09bd 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -15,8 +15,7 @@ config MT7932_FULLMAC unless those files are in the initramfs. Requires a trained retained PCI link and a translated SID1 DMA domain. Firmware, runtime DMA and calibration negotiation start automatically; - cfg80211 and userspace select the country and network. No test boot - parameters are required for ordinary station operation. + cfg80211 and userspace select the country and network. Once DMA pointers are published the module is pinned until hardware reset. Normal unload is refused, including with active Bluetooth. A failed probe reset retains the binding and DMA; a successful checked diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index ebe7087ee030b..89e36ca28a6d4 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -41,22 +41,6 @@ static int mt_initialize(struct mt7932 *m); static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id); static void mt_shutdown(struct pci_dev *pdev); -static bool legacy_irq_test; -module_param(legacy_irq_test, bool, 0444); -MODULE_PARM_DESC(legacy_irq_test, "Explicit diagnostic use of standard PCI INTx instead of MSI"); -static bool poll_test; -module_param(poll_test, bool, 0444); -MODULE_PARM_DESC(poll_test, "Diagnostic ROM/firmware polling with endpoint IRQ generation masked"); -static bool reset_test; -module_param(reset_test, bool, 0444); -MODULE_PARM_DESC(reset_test, "Qualify one FLR and fresh firmware epoch before publishing an interface, retaining old DMA"); - -bool mt_transport_polled(void) -{ - return poll_test; -} - - bool mt_rf_allowed(struct mt7932 *m) { return !READ_ONCE(m->stopping) && READ_ONCE(m->rf_ready) && !READ_ONCE(m->reg_pending) && @@ -313,7 +297,7 @@ static void mt_service(struct mt7932 *m) for (i = 0; i < ARRAY_SIZE(m->rx); i++) if (m->rx[i].count) mt_rx_drain(m, &m->rx[i]); - if (READ_ONCE(m->running) && !poll_test) + if (READ_ONCE(m->running)) mt_write(m, W + 0x204, m->irq_mask); } @@ -395,22 +379,9 @@ int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, boo m->reply_length = 0; spin_unlock_irqrestore(&m->response_lock, flags); ret = mt_submit(m, q, ret); - if (!ret && wait) { - if (poll_test) { - unsigned long deadline = jiffies + msecs_to_jiffies(5000); - - do { - mt_service(m); - if (completion_done(&m->response)) - break; - usleep_range(100, 200); - } while (time_before(jiffies, deadline)); - if (!completion_done(&m->response)) - ret = -ETIMEDOUT; - } else if (!wait_for_completion_timeout(&m->response, msecs_to_jiffies(5000))) { - ret = -ETIMEDOUT; - } - } + if (!ret && wait && + !wait_for_completion_timeout(&m->response, msecs_to_jiffies(5000))) + ret = -ETIMEDOUT; if (READ_ONCE(m->stopping)) ret = -ESHUTDOWN; if (ret == -ETIMEDOUT) { @@ -504,7 +475,7 @@ static int mt_dma_setup(struct mt7932 *m) u32 glo; mt_write(m, W + 0x204, 0); - mt_write(m, 0x10188, poll_test ? 0 : 0xff); + mt_write(m, 0x10188, 0xff); mt_rmw(m, W + 0x208, BIT(0) | BIT(2) | BIT(15) | BIT(21) | BIT(27) | BIT(28), 0); ret = mt_poll(m, W + 0x208, BIT(1) | BIT(3), 0, 100000); if (ret) @@ -598,7 +569,7 @@ static int mt_dma_setup(struct mt7932 *m) mt_write(m, W + 0x2f0, 0x8032800a); mt_rmw(m, 0x2120, 0, BIT(1)); mt_write(m, W + 0x1f4, 0xffff); - mt_write(m, W + 0x204, poll_test ? 0 : IRQ_MASK); + mt_write(m, W + 0x204, IRQ_MASK); dev_info(&m->pdev->dev, "DMA_PUBLISHED glo=%08x TX17=%pad RX0=%pad IPC=%pad\n", mt_read(m, W + 0x208), &m->tx[17].dma, &m->rx[0].dma, &m->ipc_dma); return 0; @@ -636,8 +607,8 @@ static int mt_rom_gate(struct mt7932 *m) dev_info(&m->pdev->dev, "ROM_READ_OK %08x=%08x seq=%u IRQ=%llu\n", addresses[i], get_unaligned_le32(m->reply + 36), m->seq, m->interrupts); } - dev_info(&m->pdev->dev, "ROM_TRANSPORT_OK mode=%s IRQ=%llu (no firmware download or RF yet)\n", - poll_test ? "polled-test" : "interrupt", m->interrupts); + dev_info(&m->pdev->dev, "ROM_TRANSPORT_OK IRQ=%llu (no firmware download or RF yet)\n", + m->interrupts); return 0; } @@ -1113,63 +1084,6 @@ static int mt_initialize(struct mt7932 *m) return ret; } -static int mt_reset_retained(struct mt7932 **epoch) -{ - struct mt7932 *old = *epoch, *fresh; - struct pci_dev *pdev = old->pdev; - int irq = pci_irq_vector(pdev, 0), ret; - - /* No interface/work producers have been published at this test gate. */ - WRITE_ONCE(old->running, false); - mt_write(old, W + 0x204, 0); - devm_free_irq(&pdev->dev, irq, old); - old->irq_requested = false; - ret = mt_ownership(old); - if (!ret) - ret = mt_dma_stop(old); - if (!ret) - ret = mt_function_reset(old); - if (ret) - return ret; - old->dma_owned = false; - if (old->module_pinned) { - old->module_pinned = false; - module_put(THIS_MODULE); - } - dev_info(&pdev->dev, "RESET_TEST: function reset complete; retaining old DMA\n"); - fresh = devm_kzalloc(&pdev->dev, sizeof(*fresh), GFP_KERNEL); - if (!fresh) - return -ENOMEM; - fresh->pdev = pdev; - fresh->bar = old->bar; - fresh->irq_mask = IRQ_MASK; - fresh->vectors_allocated = true; - ether_addr_copy(fresh->mac, old->mac); - init_completion(&fresh->response); - init_completion(&fresh->cal_response); - init_completion(&fresh->reset_retry); - mutex_init(&fresh->command_mutex); - spin_lock_init(&fresh->response_lock); - *epoch = fresh; - pci_set_drvdata(pdev, fresh); - ret = mt_ownership(fresh); - if (ret) - return ret; - ret = mt_poll(fresh, 0xf0140, BIT(4), BIT(4), 200000); - if (ret) - return ret; - mt_write(fresh, W + 0x204, 0); - ret = devm_request_threaded_irq(&pdev->dev, irq, mt_irq, mt_irq_thread, - IRQF_ONESHOT | IRQF_SHARED, "mt7932", fresh); - if (ret) - return ret; - fresh->irq_requested = true; - ret = mt_initialize(fresh); - if (!ret) - dev_info(&pdev->dev, "RESET_TEST: fresh ROM and firmware gates passed\n"); - return ret; -} - /* A failed PCI reset cannot be turned into successful removal: devres would * free still-owned DMA. Keep the remove callback (and therefore its device, * mappings and module text) alive. This explicit retry does not run hardware @@ -1307,7 +1221,7 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto retain; mt_write(m, W + 0x204, 0); - ret = pci_alloc_irq_vectors(pdev, 1, 1, legacy_irq_test ? PCI_IRQ_INTX : PCI_IRQ_ALL_TYPES); + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); if (ret < 0) goto retain; m->vectors_allocated = true; @@ -1338,8 +1252,6 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) irq->chip->name, irq->hwirq); } ret = mt_initialize(m); - if (!ret && reset_test) - ret = mt_reset_retained(&m); if (!ret) ret = mt_register_regulatory_gate(m); if (!ret) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 8151e794bfabf..1c2262952e82e 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -253,6 +253,4 @@ void mt_power_work(struct work_struct *work); int mt_power_init(struct mt7932 *m); void mt_station_update_rate(struct mt7932 *m, const u8 *mac); -bool mt_transport_polled(void); - #endif diff --git a/drivers/net/wireless/mediatek/mt7932/recovery.c b/drivers/net/wireless/mediatek/mt7932/recovery.c index f0b7cc9714c24..cdb327bb27e39 100644 --- a/drivers/net/wireless/mediatek/mt7932/recovery.c +++ b/drivers/net/wireless/mediatek/mt7932/recovery.c @@ -192,7 +192,7 @@ int mt_recovery_gate(struct mt7932 *m) int ret; /* mt_runtime_start() completed configuration before entering this stage. */ - if (mt_transport_polled() || m->runtime_epoch) + if (m->runtime_epoch) return -EINVAL; for (i = 0; i < ARRAY_SIZE(queue_config); i++) { ret = mt_request(m, 0x6b, true, true, false, queue_config[i], 32); From 08b907740a4ff01fa7c531363747b10ee1a79c75 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1037/1579] wifi: mt7932: remove the unused station rate query mt_station_update_rate() queried the firmware for the transmit rate of the associated access point. It has never been called: the reply comes from a transmit vector that the firmware shares between stations, so station reports leave the transmit bitrate out. Remove it, the reply parser and the station fields that only it used. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/Makefile | 2 +- drivers/net/wireless/mediatek/mt7932/link.c | 3 - drivers/net/wireless/mediatek/mt7932/mt7932.h | 4 -- drivers/net/wireless/mediatek/mt7932/rate.h | 29 --------- .../wireless/mediatek/mt7932/station_query.c | 65 ------------------- 5 files changed, 1 insertion(+), 102 deletions(-) delete mode 100644 drivers/net/wireless/mediatek/mt7932/station_query.c diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index 9e76679c46a8a..f854d21e5db81 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o -mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o station_query.o +mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 9f857417a5ce3..999d0bd16c290 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -532,7 +532,6 @@ void mt_connect_work(struct work_struct *work) m->station_connected = jiffies; m->station_signal_valid = false; m->station_rx_rate_valid = false; - m->station_tx_rate_valid = false; spin_unlock(&m->data_lock); mt_cqm_reset(m); } @@ -643,8 +642,6 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, m->connect_center = 0; memset(&m->connect_chandef, 0, sizeof(m->connect_chandef)); m->connection_generation++; - m->station_rate_query_failed = false; - m->station_rate_query_time = jiffies - HZ; m->connect_cancelled = false; m->peer_valid = false; m->connecting = true; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 1c2262952e82e..688654acac97d 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -161,9 +161,6 @@ struct mt7932 { struct rate_info station_rx_rate; unsigned long station_rx_rate_time; bool station_rx_rate_valid; - struct rate_info station_tx_rate; - unsigned long station_tx_rate_time, station_rate_query_time; - bool station_tx_rate_valid, station_rate_query_failed; /* Connection quality monitor, protected by response_lock. */ struct ewma_mt7932_signal cqm_signal; unsigned long cqm_sample_time; @@ -251,6 +248,5 @@ int mt_set_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, bool enabled, int timeout); void mt_power_work(struct work_struct *work); int mt_power_init(struct mt7932 *m); -void mt_station_update_rate(struct mt7932 *m, const u8 *mac); #endif diff --git a/drivers/net/wireless/mediatek/mt7932/rate.h b/drivers/net/wireless/mediatek/mt7932/rate.h index d570a69f81f44..7e7c5685d40fd 100644 --- a/drivers/net/wireless/mediatek/mt7932/rate.h +++ b/drivers/net/wireless/mediatek/mt7932/rate.h @@ -65,33 +65,4 @@ static inline bool mt7932_rx_rate(u32 rxv, u8 max_nss, struct rate_info *rate) return true; } -/* Stock CID85 reply: exact version1 body, owned BSS/MAC and pending sequence. - * Byte8's station identity and byte0a remain opaque; no invented WTBL test. - */ -static inline bool mt7932_tx_rate_reply(const struct mt7932_event *event, - u8 sequence, const u8 *peer, u8 max_nss, - struct rate_info *rate) -{ - const u8 *body; - u32 a, b, c, rxv; - - if (!mt7932_event_matches(event, sequence) || event->eid != 0x21 || - event->length != 344) - return false; - body = event->packet + 36; - if (body[0] != 1 || - !(get_unaligned_le32(body + 4) & 1) || body[9] || - memcmp(body + 12, peer, 6)) - return false; - a = get_unaligned_le32(body + 0x9c); - b = get_unaligned_le32(body + 0xa0); - c = get_unaligned_le32(body + 0xa4); - if (((a >> 12) & 15) >= 5) - return false; /* TX HE GI/DCM/allocation fields not qualified by this reply. */ - /* Normalize the independently qualified TX words to the shared fields. */ - rxv = (c & 127) | (((c >> 8) & 7) << 7) | - (((a >> 8) & 7) << 12) | (((b >> 26) & 3) << 15) | - (((a >> 6) & 3) << 22) | (((a >> 12) & 15) << 24); - return mt7932_rx_rate(rxv, max_nss, rate); -} #endif diff --git a/drivers/net/wireless/mediatek/mt7932/station_query.c b/drivers/net/wireless/mediatek/mt7932/station_query.c deleted file mode 100644 index 8382768898172..0000000000000 --- a/drivers/net/wireless/mediatek/mt7932/station_query.c +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -#include "mt7932.h" - -/* Research helper, deliberately not called by production station reporting: - * the reply identity is per-peer, but its vector is shared and unattributed. - * Retained while qualifying a real per-peer rate source, not a public metric. - * One serialized query for the owned peer. Keep the command owner until the - * reply has been checked; disconnect may close admission in IRQ context but - * cannot reuse BSS/station resources across this critical section. - */ -void mt_station_update_rate(struct mt7932 *m, const u8 *mac) -{ - struct mt7932_event event; - struct rate_info rate; - u8 body[28] = {}, peer[ETH_ALEN]; - u32 generation; - unsigned long flags; - bool valid = false; - int ret; - - mutex_lock(&m->command_mutex); - spin_lock_irqsave(&m->response_lock, flags); - if (!m->connected || m->disconnecting || !m->peer_valid || !mt_rf_allowed(m) || - m->station_rate_query_failed || (mac && !ether_addr_equal(mac, m->connect_bssid)) || - time_before(jiffies, m->station_rate_query_time + HZ / 2)) { - spin_unlock_irqrestore(&m->response_lock, flags); - goto unlock; - } - generation = m->connection_generation; - ether_addr_copy(peer, m->connect_bssid); - body[0] = m->peer_ids.station[0]; - memcpy(body + 4, peer, ETH_ALEN); - m->station_rate_query_time = jiffies; - spin_unlock_irqrestore(&m->response_lock, flags); - - ret = mt_request(m, 0x85, true, false, true, body, sizeof(body)); - if (!ret) - ret = mt7932_event_parse(m->reply, m->reply_length, &event); - if (!ret) - valid = mt7932_tx_rate_reply(&event, m->seq, peer, m->phy_cap[4], &rate); - if (valid && (rate.bw != RATE_INFO_BW_20 || - (m->connect_channel > 14 && !rate.flags && - (rate.legacy == 10 || rate.legacy == 20 || rate.legacy == 55 || rate.legacy == 110)))) - valid = false; - - spin_lock_irqsave(&m->response_lock, flags); - if (generation == m->connection_generation && m->connected && - !m->disconnecting && m->peer_valid && ether_addr_equal(peer, m->connect_bssid)) { - /* A timed-out command must not turn every station dump into another - * five-second wait. A new connection/module epoch may try again. - */ - if (ret) - m->station_rate_query_failed = true; - spin_lock(&m->data_lock); - m->station_tx_rate_valid = valid; - if (valid) { - m->station_tx_rate = rate; - m->station_tx_rate_time = jiffies; - } - spin_unlock(&m->data_lock); - } - spin_unlock_irqrestore(&m->response_lock, flags); -unlock: - mutex_unlock(&m->command_mutex); -} From a257e5168baa8048f3187203b7e18e53d21db4e2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1038/1579] wifi: mt7932: report driver progress at debug level The driver logged each firmware boot step, every scan request and completion, every association event and each calibration request at info level. A desktop scans every few seconds, so this filled the kernel log with SCAN_REQUEST, SCAN_SUBMITTED, LINK_EVENT and DISCONNECT_EVENT lines, plus hex dumps of firmware replies and DMA addresses at probe. Log routine progress with dev_dbg() and print_hex_dump_debug(), so it is still available through dynamic debug. Errors and warnings keep their level and text. Also drop two diagnostics only used during bring-up: the dump of the MSI configuration and interrupt hierarchy at probe, and the register and descriptor dump on a command timeout. A timed-out command still collects a late reply before reporting the error, and the hex dump of a rejected calibration upload reply goes away, since the error line already has the fields that matter. A few of the demoted messages read device status registers. Those reads now happen only while the message is enabled; each follows the driver's own accesses to the same block, so nothing depends on them. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../mediatek/mt7932/calibration_transport.c | 13 +- drivers/net/wireless/mediatek/mt7932/data.c | 6 +- .../net/wireless/mediatek/mt7932/frontend.c | 24 +-- drivers/net/wireless/mediatek/mt7932/link.c | 64 ++++--- drivers/net/wireless/mediatek/mt7932/mt7932.c | 156 +++++++----------- drivers/net/wireless/mediatek/mt7932/mt7932.h | 1 - .../net/wireless/mediatek/mt7932/recovery.c | 6 +- .../net/wireless/mediatek/mt7932/startup.c | 22 +-- 8 files changed, 132 insertions(+), 160 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c index 1d6d4d6985a45..fb5a33dd5aa61 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c +++ b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c @@ -124,12 +124,10 @@ int mt_calibration_gate(struct mt7932 *m) get_unaligned_le32(m->reply + 40)) { dev_err(&m->pdev->dev, "WCAL_REPLY_REJECTED: length=%zu eid=%02x ext=%02x\n", m->reply_length, m->reply[28], m->reply[32]); - print_hex_dump(KERN_INFO, "WCAL_RESULT: ", DUMP_PREFIX_OFFSET, 16, 1, - m->reply, min_t(size_t, m->reply_length, 64), false); ret = -EPROTO; goto out; } - dev_info(&m->pdev->dev, "OWN_WCAL_ACCEPTED: %zu bytes\n", wcal->size); + dev_dbg(&m->pdev->dev, "OWN_WCAL_ACCEPTED: %zu bytes\n", wcal->size); phase = "power-on D6 command/completion"; for (at = 0, group = 0; at < planned; at += count, group++) { count = pieces[at].fragment >> 4; @@ -140,10 +138,10 @@ int mt_calibration_gate(struct mt7932 *m) ret = mt_cal_procedure(m, pieces + at, count, 1, m->preload_version); if (ret) goto out; - dev_info(&m->pdev->dev, "POWER_ON_CAL_GROUP_OK: %u type=%u parameter=%08x fragments=%u\n", - group, pieces[at].type, pieces[at].parameter, count); + dev_dbg(&m->pdev->dev, "POWER_ON_CAL_GROUP_OK: %u type=%u parameter=%08x fragments=%u\n", + group, pieces[at].type, pieces[at].parameter, count); } - dev_info(&m->pdev->dev, "POWER_ON_CAL_COMPLETE: %u groups, %d fragments\n", group, planned); + dev_dbg(&m->pdev->dev, "POWER_ON_CAL_COMPLETE: %u groups, %d fragments\n", group, planned); phase = "queued D7 planning/command/completion"; ret = mt_cal_drain_requests(m, oca); out: @@ -187,7 +185,8 @@ int mt_cal_drain_requests(struct mt7932 *m, const struct firmware *oca) mt7932_cal_request_replies(request), m->preload_version); if (ret) return ret; - dev_info(&m->pdev->dev, "CAL_REQUEST_SERVICED: channel=%u\n", get_unaligned_le32(request + 12)); + dev_dbg(&m->pdev->dev, "CAL_REQUEST_SERVICED: channel=%u\n", + get_unaligned_le32(request + 12)); } return 0; } diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 9e6a51f8e3987..e4f9d8a9f9cc8 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -313,10 +313,10 @@ int mt_data_prepare(struct mt7932 *m) /* The selected response parser establishes sequence completion only, * not a command-specific firmware status. No payload-status invention. */ - dev_info(&m->pdev->dev, "MAC_RTS_RESPONSE: sequence matched, EID=%02x length=%zu\n", - m->reply[28], m->reply_length); + dev_dbg(&m->pdev->dev, "MAC_RTS_RESPONSE: sequence matched, EID=%02x length=%zu\n", + m->reply[28], m->reply_length); /* Publish the initialized data_lock and DMA state to failure handlers. */ smp_store_release(&m->data_ready, true); - dev_info(&m->pdev->dev, "ETHERNET_PREPARED: TX0 retained coherent slots=512, RX translation enabled\n"); + dev_dbg(&m->pdev->dev, "ETHERNET_PREPARED: TX0 retained coherent slots=512, RX translation enabled\n"); return 0; } diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index aa36c68a10691..7b0259bde004a 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -64,8 +64,8 @@ static void mt_scan_finish_work(struct work_struct *work) } mutex_unlock(&m->command_mutex); if (request) { - dev_info(&m->pdev->dev, "SCAN_FINISHED: seq=%u aborted=%u received-BSS=%llu\n", - cancel[0], info.aborted, m->beacons); + dev_dbg(&m->pdev->dev, "SCAN_FINISHED: seq=%u aborted=%u received-BSS=%llu\n", + cancel[0], info.aborted, m->beacons); cfg80211_scan_done(request, &info); } } @@ -226,8 +226,8 @@ static int mt_scan_submit_next(struct mt7932 *m) ret = mt_request(m, 3, true, true, false, body, sizeof(body)); } if (!ret) - dev_info(&m->pdev->dev, "SCAN_SUBMITTED: seq=%u band=%u channels=%u passive=%u\n", - m->scan_seq, m->scan_band, count, passive); + dev_dbg(&m->pdev->dev, "SCAN_SUBMITTED: seq=%u band=%u channels=%u passive=%u\n", + m->scan_seq, m->scan_band, count, passive); return ret; } @@ -350,9 +350,10 @@ int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) if (request->wdev != &m->wdev) return -ENODEV; mt_retry_missing_policy(m); - dev_info_ratelimited(&m->pdev->dev, "SCAN_REQUEST: channels=%u ssids=%d ies=%zu flags=%08x duration=%u/%u\n", - request->n_channels, request->n_ssids, request->ie_len, request->flags, - request->duration, request->duration_mandatory); + dev_dbg(&m->pdev->dev, + "SCAN_REQUEST: channels=%u ssids=%d ies=%zu flags=%08x duration=%u/%u\n", + request->n_channels, request->n_ssids, request->ie_len, request->flags, + request->duration, request->duration_mandatory); /* cfg80211 lists each advertised channel at most once. */ if (request->n_channels > MT7932_CHANNELS) return -EOPNOTSUPP; @@ -547,7 +548,8 @@ static void mt_cal_work(struct work_struct *work) m->preload_version); if (ret) break; - dev_info(&m->pdev->dev, "D7_CALIBRATION_COMPLETE: channel=%u\n", get_unaligned_le32(request + 12)); + dev_dbg(&m->pdev->dev, "D7_CALIBRATION_COMPLETE: channel=%u\n", + get_unaligned_le32(request + 12)); } release_firmware(oca); if (!ret) @@ -601,7 +603,8 @@ int mt_register_interface(struct mt7932 *m) free_netdev(netdev); return ret; } - dev_info(&m->pdev->dev, "CFG80211_INTERFACE_READY: %s, RF awaits country policy\n", netdev->name); + dev_dbg(&m->pdev->dev, "CFG80211_INTERFACE_READY: %s, RF awaits country policy\n", + netdev->name); return 0; } @@ -637,7 +640,8 @@ int mt_enable_scan(struct mt7932 *m) return ret; } WRITE_ONCE(m->rf_ready, true); - dev_info(&m->pdev->dev, "CFG80211_SCAN_READY: %s, carrier off until WPA2 authorization\n", m->netdev->name); + dev_dbg(&m->pdev->dev, "CFG80211_SCAN_READY: %s, carrier off until WPA2 authorization\n", + m->netdev->name); schedule_work(&m->cal_work); spin_unlock_irqrestore(&m->response_lock, flags); return 0; diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 999d0bd16c290..bcdcb71ce3db9 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -113,11 +113,11 @@ void mt_link_event(struct mt7932 *m, const struct mt7932_event *event) selector = get_unaligned_le32(event->packet + 40); body = event->packet + 48; length = event->length - 48; - dev_info(&m->pdev->dev, "LINK_EVENT: selector=%02x length=%zu\n", selector, length); + dev_dbg(&m->pdev->dev, "LINK_EVENT: selector=%02x length=%zu\n", selector, length); if (selector == 0x44) { if (length < 5 || (!m->connecting && !m->connected && !m->disconnecting)) return; - dev_info(&m->pdev->dev, "DISCONNECT_EVENT: metadata=%u deferred=%u reason-index=%u\n", + dev_dbg(&m->pdev->dev, "DISCONNECT_EVENT: metadata=%u deferred=%u reason-index=%u\n", body[0], body[3], body[4]); if (m->disconnect_seen) return; @@ -179,27 +179,28 @@ void mt_link_event(struct mt7932 *m, const struct mt7932_event *event) * rate-control setting. Bounds were checked by the exporter. */ if (he) - dev_info(&m->pdev->dev, - "ASSOC_HE_CAPABILITIES: epoch=%u rx-map=%04x tx-map=%04x\n", - m->connection_generation, - get_unaligned_le16(he->data + 18), - get_unaligned_le16(he->data + 20)); + dev_dbg(&m->pdev->dev, + "ASSOC_HE_CAPABILITIES: epoch=%u rx-map=%04x tx-map=%04x\n", + m->connection_generation, + get_unaligned_le16(he->data + 18), + get_unaligned_le16(he->data + 20)); } break; } case 0x4d: if (length >= 20) - dev_info(&m->pdev->dev, "ASSOC_START: band=%u width=%u channel=%u extension=%u\n", - get_unaligned_le32(body + 12), body[16], body[17], body[19]); + dev_dbg(&m->pdev->dev, "ASSOC_START: band=%u width=%u channel=%u extension=%u\n", + get_unaligned_le32(body + 12), body[16], body[17], body[19]); if (!mt_channel_event(m, body, length)) m->connect_error = -EPROTO; complete(&m->assoc_start); break; case 0x40: if (length >= 52) - dev_info(&m->pdev->dev, "ASSOC_REPORT: status=%u band=%u channel=%u bss=%u bmc=%u station=%u peer=%u cipher=%08x\n", - get_unaligned_le16(body), body[11], body[12], body[16], body[17], body[20], body[21], - get_unaligned_le32(body + 48)); + dev_dbg(&m->pdev->dev, + "ASSOC_REPORT: status=%u band=%u channel=%u bss=%u bmc=%u station=%u peer=%u cipher=%08x\n", + get_unaligned_le16(body), body[11], body[12], body[16], body[17], + body[20], body[21], get_unaligned_le32(body + 48)); /* OPEN reports retain cipher as numeric metadata only; it neither * installs a key nor changes the immutable requested security mode. */ @@ -213,12 +214,13 @@ void mt_link_event(struct mt7932 *m, const struct mt7932_event *event) m->peer_valid = true; /* HOST decoder confirms the RSN suite namespace, not request enum10. */ m->event_cipher = get_unaligned_le32(body + 48); - dev_info(&m->pdev->dev, "ASSOCIATED: peer=%u station=%u cipher-raw=%u\n", - m->peer_wtbl, body[20], m->event_cipher); + dev_dbg(&m->pdev->dev, "ASSOCIATED: peer=%u station=%u cipher-raw=%u\n", + m->peer_wtbl, body[20], m->event_cipher); break; case 0x42: if (length >= 2) - dev_info(&m->pdev->dev, "CONNECT_REPORT: status=%u peer-owned=%u\n", get_unaligned_le16(body), m->peer_valid); + dev_dbg(&m->pdev->dev, "CONNECT_REPORT: status=%u peer-owned=%u\n", + get_unaligned_le16(body), m->peer_valid); if (length < 2 || get_unaligned_le16(body) || !m->peer_valid) m->connect_error = -ECONNREFUSED; complete(&m->assoc_done); @@ -352,8 +354,9 @@ static int mt_retire_connection(struct mt7932 *m) m->link_failed = true; /* uncertain ownership: recovery required */ spin_unlock_irqrestore(&m->response_lock, flags); mutex_unlock(&m->command_mutex); - dev_info(&m->pdev->dev, "CONNECTION_RETIRED: status=%d%s\n", ret, - ret ? "; recovery required, DMA ownership retained" : "; old TX drained, six IDs released"); + dev_dbg(&m->pdev->dev, "CONNECTION_RETIRED: status=%d%s\n", ret, + ret ? "; recovery required, DMA ownership retained" : + "; old TX drained, six IDs released"); return ret; } @@ -398,7 +401,7 @@ void mt_connect_work(struct work_struct *work) goto free; } if (!m->connect_bss) { - dev_info(&m->pdev->dev, "CONNECT_DISCOVERY: refreshing missing scan candidate\n"); + dev_dbg(&m->pdev->dev, "CONNECT_DISCOVERY: refreshing missing scan candidate\n"); ret = mt_connect_discover(m); if (ret) goto free; @@ -444,7 +447,8 @@ void mt_connect_work(struct work_struct *work) memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); if (ret) goto report; - dev_info(&m->pdev->dev, "CONNECT_SUBMITTED: BSSID=%pM channel=%u\n", m->connect_bssid, m->connect_channel); + dev_dbg(&m->pdev->dev, "CONNECT_SUBMITTED: BSSID=%pM channel=%u\n", + m->connect_bssid, m->connect_channel); /* Genuine D7 requests must remain serviceable during asynchronous waits. */ mutex_unlock(&m->command_mutex); ret = wait_for_completion_timeout(&m->assoc_start, msecs_to_jiffies(5000)) ? 0 : -ETIMEDOUT; @@ -460,8 +464,8 @@ void mt_connect_work(struct work_struct *work) ret = ret ?: READ_ONCE(m->connect_error); if (ret) goto report; - dev_info(&m->pdev->dev, "ASSOCIATION_CALIBRATED: primary=%u center=%u\n", - m->connect_channel, m->connect_center); + dev_dbg(&m->pdev->dev, "ASSOCIATION_CALIBRATED: primary=%u center=%u\n", + m->connect_channel, m->connect_center); mt7932_peer_queue(queue, m->peer_ids.station[0], true); ret = mt_request(m, 0x6b, true, true, false, queue, sizeof(queue)); if (ret) @@ -551,10 +555,14 @@ void mt_connect_work(struct work_struct *work) netif_wake_queue(m->netdev); } if (m->connect_open) { - dev_info(&m->pdev->dev, "OPEN_CONNECTED: owned peer, firmware completion; Ethernet=%u\n", m->data_ready); + dev_dbg(&m->pdev->dev, + "OPEN_CONNECTED: owned peer, firmware completion; Ethernet=%u\n", + m->data_ready); } else { cfg80211_port_authorized(m->netdev, m->connect_bssid, NULL, 0, GFP_ATOMIC); - dev_info(&m->pdev->dev, "WPA2_AUTHORIZED: firmware handshake complete; Ethernet=%u\n", m->data_ready); + dev_dbg(&m->pdev->dev, + "WPA2_AUTHORIZED: firmware handshake complete; Ethernet=%u\n", + m->data_ready); } } else { dev_err(&m->pdev->dev, "CONNECT_FAILED: %d; recovery-required=%u\n", ret, m->link_failed); @@ -596,9 +604,11 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, mt_retry_missing_policy(m); if (!mt_rf_allowed(m) || !m->bss_active || netdev != m->netdev) return -EAGAIN; - dev_info_ratelimited(&m->pdev->dev, "CONNECT_REQUEST: WPA=%u pairwise-count=%d group=%08x AKM-count=%d PMK-present=%u MFP=%u IE-bytes=%zu\n", - params->crypto.wpa_versions, params->crypto.n_ciphers_pairwise, params->crypto.cipher_group, - params->crypto.n_akm_suites, !!params->crypto.psk, params->mfp, params->ie_len); + dev_dbg(&m->pdev->dev, + "CONNECT_REQUEST: WPA=%u pairwise-count=%d group=%08x AKM-count=%d PMK-present=%u MFP=%u IE-bytes=%zu\n", + params->crypto.wpa_versions, params->crypto.n_ciphers_pairwise, + params->crypto.cipher_group, params->crypto.n_akm_suites, + !!params->crypto.psk, params->mfp, params->ie_len); if (!params->ssid_len || params->ssid_len > 32) return -EINVAL; if (!open && (params->crypto.wpa_versions != NL80211_WPA_VERSION_2 || @@ -675,7 +685,7 @@ void mt_disconnect_work(struct work_struct *work) m->disconnecting = false; cfg80211_disconnected(m->netdev, reason, NULL, 0, local, GFP_ATOMIC); spin_unlock_irqrestore(&m->response_lock, flags); - dev_info(&m->pdev->dev, "DISCONNECTED: teardown=%d local=%u\n", ret, local); + dev_dbg(&m->pdev->dev, "DISCONNECTED: teardown=%d local=%u\n", ret, local); } int mt_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 reason) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 89e36ca28a6d4..d8a319cfc12a8 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -169,8 +169,8 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) ret = mt7932_cal_null(&m->cal_state, packet + 36, event.length >= 36 ? event.length - 36 : 0, m->smart_version); - dev_info(&m->pdev->dev, "CAL_EVENT: seq=%u length=%zu result=%d credits=%u/%u\n", - event.seq, event.length, ret, m->cal_state.received, m->cal_state.expected); + dev_dbg(&m->pdev->dev, "CAL_EVENT: seq=%u length=%zu result=%d credits=%u/%u\n", + event.seq, event.length, ret, m->cal_state.received, m->cal_state.expected); if (ret < 0) mt_rf_fail_locked(m, ret); if (ret) @@ -196,9 +196,10 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) memcpy(m->cal_requests[at], packet + 36, 16); m->cal_request_seq[at] = event.seq; m->cal_request_count++; - dev_info(&m->pdev->dev, "CAL_REQUEST: seq=%u words=%08x/%08x/%08x/%08x queued=%u\n", - event.seq, get_unaligned_le32(packet + 36), get_unaligned_le32(packet + 40), - get_unaligned_le32(packet + 44), get_unaligned_le32(packet + 48), m->cal_request_count); + dev_dbg(&m->pdev->dev, "CAL_REQUEST: seq=%u words=%08x/%08x/%08x/%08x queued=%u\n", + event.seq, get_unaligned_le32(packet + 36), + get_unaligned_le32(packet + 40), get_unaligned_le32(packet + 44), + get_unaligned_le32(packet + 48), m->cal_request_count); if (READ_ONCE(m->rf_ready)) schedule_work(&m->cal_work); } @@ -385,38 +386,18 @@ int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, boo if (READ_ONCE(m->stopping)) ret = -ESHUTDOWN; if (ret == -ETIMEDOUT) { - /* Diagnostic only: never turn a polled late reply into IRQ success. */ unsigned int i; - u16 command; - u32 mask, pending; - unsigned int cap = pci_find_capability(m->pdev, PCI_CAP_ID_MSI); - - pci_read_config_word(m->pdev, PCI_COMMAND, &command); - pci_read_config_dword(m->pdev, cap + PCI_MSI_MASK_64, &mask); - pci_read_config_dword(m->pdev, cap + PCI_MSI_PENDING_64, &pending); - dev_info(&m->pdev->dev, "timeout PCI command=%04x MSI mask=%08x pending=%08x\n", - command, mask, pending); - - dev_info(&m->pdev->dev, "timeout gates MAC=%08x WFDMA=%08x SW=%08x\n", - mt_read(m, 0x10188), mt_read(m, W + 0x204), mt_read(m, W + 0x1f4)); + + /* + * Collect a reply that arrived without its interrupt, so that the + * error below reports it. It does not turn the timeout into success. + */ disable_irq(pci_irq_vector(m->pdev, 0)); - for (i = 0; i < ARRAY_SIZE(m->rx); i++) { - struct mt7932_ring *rx = &m->rx[i]; - - if (!rx->count) - continue; - dev_info(&m->pdev->dev, "timeout RX%u ctrl=%08x CPU=%u DMA=%u\n", i, - le32_to_cpu(rx->desc[rx->tail].control), - mt_read(m, rx->reg + 8), mt_read(m, rx->reg + 12)); - if (le32_to_cpu(rx->desc[rx->tail].control) & DONE) { - dma_rmb(); - print_hex_dump(KERN_INFO, "ROM RX prefix: ", DUMP_PREFIX_OFFSET, - 16, 1, rx->buffers + rx->tail * RX_STRIDE, 40, false); - } - mt_rx_drain(m, rx); - } - dev_info(&m->pdev->dev, "late polled reply length=%zu eid=%02x (not IRQ success)\n", - m->reply_length, m->reply_length > 28 ? m->reply[28] : 0); + for (i = 0; i < ARRAY_SIZE(m->rx); i++) + if (m->rx[i].count) + mt_rx_drain(m, &m->rx[i]); + dev_dbg(&m->pdev->dev, "late reply length=%zu eid=%02x\n", + m->reply_length, m->reply_length > 28 ? m->reply[28] : 0); enable_irq(pci_irq_vector(m->pdev, 0)); } spin_lock_irqsave(&m->response_lock, flags); @@ -459,7 +440,7 @@ static int mt_ownership(struct mt7932 *m) udelay(2); if ((mt_read(m, 0xfe250) >> 16) != 0x7000) return -EIO; - dev_info(&m->pdev->dev, "DRIVER_OWN_OK power=%08x\n", mt_read(m, 0xe00f0)); + dev_dbg(&m->pdev->dev, "DRIVER_OWN_OK power=%08x\n", mt_read(m, 0xe00f0)); return 0; } @@ -570,8 +551,8 @@ static int mt_dma_setup(struct mt7932 *m) mt_rmw(m, 0x2120, 0, BIT(1)); mt_write(m, W + 0x1f4, 0xffff); mt_write(m, W + 0x204, IRQ_MASK); - dev_info(&m->pdev->dev, "DMA_PUBLISHED glo=%08x TX17=%pad RX0=%pad IPC=%pad\n", - mt_read(m, W + 0x208), &m->tx[17].dma, &m->rx[0].dma, &m->ipc_dma); + dev_dbg(&m->pdev->dev, "DMA_PUBLISHED glo=%08x TX17=%pad RX0=%pad IPC=%pad\n", + mt_read(m, W + 0x208), &m->tx[17].dma, &m->rx[0].dma, &m->ipc_dma); return 0; } @@ -604,11 +585,11 @@ static int mt_rom_gate(struct mt7932 *m) if (m->reply_length < 40 || m->reply[28] != 2 || get_unaligned_le32(m->reply + 32) != addresses[i]) return -EPROTO; - dev_info(&m->pdev->dev, "ROM_READ_OK %08x=%08x seq=%u IRQ=%llu\n", - addresses[i], get_unaligned_le32(m->reply + 36), m->seq, m->interrupts); + dev_dbg(&m->pdev->dev, "ROM_READ_OK %08x=%08x seq=%u IRQ=%llu\n", + addresses[i], get_unaligned_le32(m->reply + 36), m->seq, m->interrupts); } - dev_info(&m->pdev->dev, "ROM_TRANSPORT_OK IRQ=%llu (no firmware download or RF yet)\n", - m->interrupts); + dev_dbg(&m->pdev->dev, "ROM_TRANSPORT_OK IRQ=%llu (no firmware download or RF yet)\n", + m->interrupts); return 0; } @@ -663,8 +644,8 @@ static int mt_download(struct mt7932 *m, const u8 *data, struct mt7932_region *r if (ret) return ret; ret = mt_scatter(m, data + r->offset, r->length); - dev_info(&m->pdev->dev, "%s region target=%08x length=%u scatter=%d\n", - ram ? "RAM" : "PATCH", r->target, r->length, ret); + dev_dbg(&m->pdev->dev, "%s region target=%08x length=%u scatter=%d\n", + ram ? "RAM" : "PATCH", r->target, r->length, ret); return ret; } @@ -728,8 +709,8 @@ static int mt_capability_gate(struct mt7932 *m) count = mt7932_capabilities(body, m->reply_length - 36); if (count < 0) { dev_err(&m->pdev->dev, "invalid capability reply len=%zu parse=%d\n", m->reply_length, count); - print_hex_dump(KERN_INFO, "NIC_CAP_REJECTED: ", DUMP_PREFIX_OFFSET, - 16, 1, m->reply, m->reply_length, false); + print_hex_dump_debug("NIC_CAP_REJECTED: ", DUMP_PREFIX_OFFSET, + 16, 1, m->reply, m->reply_length, false); return count; } for (i = 0; i < count; i++) { @@ -737,9 +718,9 @@ static int mt_capability_gate(struct mt7932 *m) u32 length = get_unaligned_le32(body + offset + 4); const u8 *value = body + offset + 8; - dev_info(&m->pdev->dev, "capability TLV type=%x bytes=%u\n", type, length); + dev_dbg(&m->pdev->dev, "capability TLV type=%x bytes=%u\n", type, length); if (type == 7) { - dev_info(&m->pdev->dev, "firmware MAC %pM\n", value); + dev_dbg(&m->pdev->dev, "firmware MAC %pM\n", value); mac = true; } else if (type == 8) { if (!value[4] || value[4] > 4) { @@ -748,13 +729,13 @@ static int mt_capability_gate(struct mt7932 *m) } memcpy(m->phy_cap, value, 12); m->antenna_mask = (1U << value[4]) - 1; - dev_info(&m->pdev->dev, "firmware PHY NSS=%u paths=%02x HT=%u VHT=%u HE=%u\n", - value[4], value[10], value[0], value[1], value[11]); + dev_dbg(&m->pdev->dev, "firmware PHY NSS=%u paths=%02x HT=%u VHT=%u HE=%u\n", + value[4], value[10], value[0], value[1], value[11]); phy = true; } else if (type == 0x38 && length) { m->runtime_cal = value[0]; m->runtime_cal_valid = true; - dev_info(&m->pdev->dev, "RUNTIME_CAL_CAPABILITY: %u\n", value[0]); + dev_dbg(&m->pdev->dev, "RUNTIME_CAL_CAPABILITY: %u\n", value[0]); } else if (type == 0x18 && length >= 4) { m->six_ghz = value[0]; m->six_ghz_valid = value[0] <= 1; @@ -767,10 +748,10 @@ static int mt_capability_gate(struct mt7932 *m) dev_err(&m->pdev->dev, "CAPABILITY_MISSING: MAC=%u PHY=%u\n", mac, phy); return -ENODATA; } - print_hex_dump(KERN_INFO, "NIC_CAP: ", DUMP_PREFIX_OFFSET, 16, 1, - m->reply, m->reply_length, false); - dev_info(&m->pdev->dev, "MCU_CAPABILITY_OK eid=%02x seq=%u (no calibration or RF yet)\n", - m->reply[28], m->seq); + print_hex_dump_debug("NIC_CAP: ", DUMP_PREFIX_OFFSET, 16, 1, + m->reply, m->reply_length, false); + dev_dbg(&m->pdev->dev, "MCU_CAPABILITY_OK eid=%02x seq=%u (no calibration or RF yet)\n", + m->reply[28], m->seq); return 0; } @@ -792,7 +773,7 @@ static int mt_config_gate(struct mt7932 *m) do { mt_tx_clean(&m->tx[17]); if (!m->tx[17].queued) { - dev_info(&m->pdev->dev, "BASIC_MLME_SUBMITTED: TX retired, no semantic ACK or RF startup\n"); + dev_dbg(&m->pdev->dev, "BASIC_MLME_SUBMITTED: TX retired, no semantic ACK or RF startup\n"); return 0; } usleep_range(100, 200); @@ -816,7 +797,7 @@ static int mt_board_gate(struct mt7932 *m) return ret; if (m->reply_length < 44 || m->reply[28] != 0xed || m->reply[32] != 0xc1) return -EPROTO; - dev_info(&m->pdev->dev, "CHIP_UID_QUERY_OK: 8 opaque bytes received, not published\n"); + dev_dbg(&m->pdev->dev, "CHIP_UID_QUERY_OK: 8 opaque bytes received, not published\n"); for (i = 0; i < ARRAY_SIZE(addresses); i++) { put_unaligned_le32(addresses[i], request); ret = mt_request_ext(m, 0xed, 1, true, false, true, request, sizeof(request)); @@ -834,8 +815,8 @@ static int mt_board_gate(struct mt7932 *m) candidate = !(module & 15) ? 2 : !board_byte ? 3 : 1; m->board_type = candidate; m->module_byte = module; - dev_info(&m->pdev->dev, "BOARD_CLASSIFICATION_CAPTURED: candidate=%u module=%02x board-byte=%02x status=%08x/%08x; no buffer branch selected\n", - candidate, module, board_byte, status[0], status[1]); + dev_dbg(&m->pdev->dev, "BOARD_CLASSIFICATION_CAPTURED: candidate=%u module=%02x board-byte=%02x status=%08x/%08x; no buffer branch selected\n", + candidate, module, board_byte, status[0], status[1]); return 0; } @@ -874,7 +855,7 @@ static int mt_precal_gate(struct mt7932 *m) return -EPROTO; } status = get_unaligned_le32(m->reply + 40); - dev_info(&m->pdev->dev, "PPR_SOURCE2_RESULT: status=%08x\n", status); + dev_dbg(&m->pdev->dev, "PPR_SOURCE2_RESULT: status=%08x\n", status); if (status) return -EREMOTEIO; memset(request, 0, sizeof(request)); @@ -884,17 +865,17 @@ static int mt_precal_gate(struct mt7932 *m) if (m->reply_length < 40 || m->reply[28] != 0xed || m->reply[32] != 0xaa) return -EPROTO; m->smart_version = m->reply[38] & 15; - dev_info(&m->pdev->dev, "SMART_VERSION_REPLY: eid=%02x ext=%02x version=%u\n", - m->reply[28], m->reply[32], m->smart_version); + dev_dbg(&m->pdev->dev, "SMART_VERSION_REPLY: eid=%02x ext=%02x version=%u\n", + m->reply[28], m->reply[32], m->smart_version); request[3] = 9; ret = mt_request(m, 0xbf, true, false, true, request, 20); if (ret) return ret; if (m->reply_length < 56) return -EPROTO; - dev_info(&m->pdev->dev, "CALTYPE_REPLY: eid=%02x ext=%02x H=%u P=%u O=%u V=%u marker=%08x\n", - m->reply[28], m->reply[32], m->runtime_cal, m->reply[37], - m->reply[44], m->reply[38], get_unaligned_le32(m->reply + 48)); + dev_dbg(&m->pdev->dev, "CALTYPE_REPLY: eid=%02x ext=%02x H=%u P=%u O=%u V=%u marker=%08x\n", + m->reply[28], m->reply[32], m->runtime_cal, m->reply[37], + m->reply[44], m->reply[38], get_unaligned_le32(m->reply + 48)); /* M=0/T=3 exact host predicate. Stage metadata only after acceptance; * do not select the legacy absent-marker D6 compatibility fallback. */ @@ -902,7 +883,7 @@ static int mt_precal_gate(struct mt7932 *m) m->reply[37] != 1 || m->reply[44] != 1) return -EPROTO; m->preload_version = m->reply[38]; - dev_info(&m->pdev->dev, "PRECAL_HANDSHAKE_OK: own calibration and RF policy still required\n"); + dev_dbg(&m->pdev->dev, "PRECAL_HANDSHAKE_OK: own calibration and RF policy still required\n"); return 0; } @@ -958,8 +939,8 @@ static int mt_firmware_gate(struct mt7932 *m) dev_err(&m->pdev->dev, "local input mediatek/mt7932/IZUBA_W7932_2.bin has invalid region %d\n", i); goto out; } - dev_info(&m->pdev->dev, "stock containers patch=%zu/%d regions RAM=%zu/%d regions\n", - patch->size, patches, ram->size, regions); + dev_dbg(&m->pdev->dev, "stock containers patch=%zu/%d regions RAM=%zu/%d regions\n", + patch->size, patches, ram->size, regions); phase = "patch semaphore command/reply"; put_unaligned_le32(2, request); ret = mt_command(m, 0x10, request, 4); @@ -986,7 +967,7 @@ static int mt_firmware_gate(struct mt7932 *m) goto out; mt_secure_release(m); } - dev_info(&m->pdev->dev, "PATCH_READY\n"); + dev_dbg(&m->pdev->dev, "PATCH_READY\n"); phase = "OTP command/profile"; ret = mt_otp_byte(m, 0x7a, &value); if (!ret) @@ -1041,7 +1022,7 @@ static int mt_firmware_gate(struct mt7932 *m) ret = mt_poll(m, 0xe00f0, 3, 3, 5000000); if (ret) goto out; - dev_info(&m->pdev->dev, "FIRMWARE_READY entry=%08x\n", entry); + dev_dbg(&m->pdev->dev, "FIRMWARE_READY entry=%08x\n", entry); phase = "runtime capability/transport gates"; ret = mt_runtime_start(m); out: @@ -1058,16 +1039,16 @@ void mt_transport_snapshot(struct mt7932 *m) unsigned int i; struct mt7932_ring *tx = &m->tx[17]; - dev_info(&m->pdev->dev, "TRANSPORT: interrupts=%llu packets=%llu pending=%08x mask=%08x TX17 head=%u tail=%u queued=%u DMA=%u\n", - m->interrupts, m->packets, mt_read(m, W + 0x200), mt_read(m, W + 0x204), - tx->head, tx->tail, tx->queued, mt_read(m, tx->reg + 12)); + dev_dbg(&m->pdev->dev, "TRANSPORT: interrupts=%llu packets=%llu pending=%08x mask=%08x TX17 head=%u tail=%u queued=%u DMA=%u\n", + m->interrupts, m->packets, mt_read(m, W + 0x200), mt_read(m, W + 0x204), + tx->head, tx->tail, tx->queued, mt_read(m, tx->reg + 12)); for (i = 0; i < ARRAY_SIZE(m->rx); i++) { struct mt7932_ring *rx = &m->rx[i]; if (rx->count) - dev_info(&m->pdev->dev, "TRANSPORT_RX%u: tail=%u producer=%u DMA=%u control=%08x\n", - i, rx->tail, mt_read(m, rx->reg + 8), mt_read(m, rx->reg + 12), - le32_to_cpu(READ_ONCE(rx->desc[rx->tail].control))); + dev_dbg(&m->pdev->dev, "TRANSPORT_RX%u: tail=%u producer=%u DMA=%u control=%08x\n", + i, rx->tail, mt_read(m, rx->reg + 8), mt_read(m, rx->reg + 12), + le32_to_cpu(READ_ONCE(rx->desc[rx->tail].control))); } } @@ -1155,7 +1136,7 @@ static void mt_remove(struct pci_dev *pdev) wiphy_free(m->wiphy); if (m->vectors_allocated) pci_free_irq_vectors(pdev); - dev_info(&pdev->dev, "REMOVE_COMPLETE: host users joined, PCI function reset, DMA retired\n"); + dev_dbg(&pdev->dev, "REMOVE_COMPLETE: host users joined, PCI function reset, DMA retired\n"); /* All coherent allocations/BAR mappings are devres-owned and now safe * to release. No default-domain ownership manipulation is necessary. */ @@ -1230,27 +1211,6 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto retain; m->irq_requested = true; - { - struct irq_data *irq = irq_get_irq_data(pci_irq_vector(pdev, 0)); - unsigned int cap = pci_find_capability(pdev, PCI_CAP_ID_MSI); - u16 control, command; - u32 lo, hi, data, mask; - - pci_read_config_word(pdev, PCI_COMMAND, &command); - pci_read_config_word(pdev, cap + PCI_MSI_FLAGS, &control); - pci_read_config_dword(pdev, cap + PCI_MSI_ADDRESS_LO, &lo); - pci_read_config_dword(pdev, cap + PCI_MSI_ADDRESS_HI, &hi); - pci_read_config_dword(pdev, cap + PCI_MSI_DATA_64, &data); - pci_read_config_dword(pdev, cap + PCI_MSI_MASK_64, &mask); - dev_dbg(&pdev->dev, "PCI command=%04x MSI cap=%x control=%04x addr=%08x:%08x data=%08x\n", - command, cap, control, hi, lo, data); - dev_dbg(&pdev->dev, "MSI mask=%08x MSI-X cap=%x chosen=%s\n", mask, - pci_find_capability(pdev, PCI_CAP_ID_MSIX), - pdev->msix_enabled ? "MSI-X" : pdev->msi_enabled ? "MSI" : "INTx"); - for (; irq; irq = irq->parent_data) - dev_dbg(&pdev->dev, "IRQ %u chip=%s hwirq=%lu\n", irq->irq, - irq->chip->name, irq->hwirq); - } ret = mt_initialize(m); if (!ret) ret = mt_register_regulatory_gate(m); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 688654acac97d..d6c80ecb315ed 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/net/wireless/mediatek/mt7932/recovery.c b/drivers/net/wireless/mediatek/mt7932/recovery.c index cdb327bb27e39..91ae4b0342608 100644 --- a/drivers/net/wireless/mediatek/mt7932/recovery.c +++ b/drivers/net/wireless/mediatek/mt7932/recovery.c @@ -274,7 +274,7 @@ int mt_recovery_gate(struct mt7932 *m) mt_read(m, m->tx[17].reg + 12)); } else { WRITE_ONCE(m->running, true); - dev_info(&m->pdev->dev, "RUNTIME_DMA_READY: full RX epoch, TX17 reset, firmware NORMAL_STATE\n"); + dev_dbg(&m->pdev->dev, "RUNTIME_DMA_READY: full RX epoch, TX17 reset, firmware NORMAL_STATE\n"); } enable_irq(pci_irq_vector(m->pdev, 0)); if (!ret) { @@ -284,8 +284,8 @@ int mt_recovery_gate(struct mt7932 *m) mt7932_capabilities(m->reply + 36, m->reply_length - 36) < 0)) ret = -EPROTO; if (!ret) - dev_info(&m->pdev->dev, "RUNTIME_QUERY_OK: TX17 capability response, IRQ=%llu\n", - m->interrupts); + dev_dbg(&m->pdev->dev, "RUNTIME_QUERY_OK: TX17 capability response, IRQ=%llu\n", + m->interrupts); } return ret; } diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index a4ed7f5b97739..c949c28254355 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -123,7 +123,7 @@ static int mt_stock_config(struct mt7932 *m) if (ret) goto free; } - dev_info(&m->pdev->dev, "STOCK_CONFIG_SUBMITTED: %u original/PHY-derived entries\n", count); + dev_dbg(&m->pdev->dev, "STOCK_CONFIG_SUBMITTED: %u original/PHY-derived entries\n", count); free: kfree(records); out: @@ -165,9 +165,9 @@ static int mt_publish_policy(struct mt7932 *m, const struct mt7932_reg_snapshot /* Native SETs have no response/atomic policy-commit event. Command * submission is not misrepresented as firmware readback or an ACK. */ - dev_info(&m->pdev->dev, "REGULATORY_POLICY_SUBMITTED: %c%c firmware=%c%c channels=%u (native no-response SETs)\n", - reg->alpha2[0], reg->alpha2[1], reg->domain[0], reg->domain[1], - (reg->length - 12) / 8); + dev_dbg(&m->pdev->dev, "REGULATORY_POLICY_SUBMITTED: %c%c firmware=%c%c channels=%u (native no-response SETs)\n", + reg->alpha2[0], reg->alpha2[1], reg->domain[0], reg->domain[1], + (reg->length - 12) / 8); return 0; } @@ -226,7 +226,7 @@ static int mt_startup_once(struct mt7932 *m) ret = mt_calibration_gate(m); if (!ret) ret = mt_request(m, 0x3e, true, true, false, roam_lock, sizeof(roam_lock)); - dev_info(&m->pdev->dev, "OWN_CALIBRATION_STARTUP_RESULT: %d\n", ret); + dev_dbg(&m->pdev->dev, "OWN_CALIBRATION_STARTUP_RESULT: %d\n", ret); if (!ret) ret = mt_enable_scan(m); if (ret) @@ -385,9 +385,9 @@ static void mt_startup_work(struct work_struct *work) reg.alpha2[0], reg.alpha2[1], generation, ret, m->policy_failed || m->link_failed); else - dev_info(&m->pdev->dev, "REGULATORY_READY: %c%c firmware=%c%c generation=%u current=%u\n", - reg.alpha2[0], reg.alpha2[1], reg.domain[0], reg.domain[1], - generation, !m->reg_pending); + dev_dbg(&m->pdev->dev, "REGULATORY_READY: %c%c firmware=%c%c generation=%u current=%u\n", + reg.alpha2[0], reg.alpha2[1], reg.domain[0], reg.domain[1], + generation, !m->reg_pending); spin_unlock_irqrestore(&m->response_lock, flags); next: mutex_unlock(&m->command_mutex); @@ -496,8 +496,8 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request if (READ_ONCE(m->interface_registered)) schedule_work(&m->startup_work); spin_unlock_irqrestore(&m->response_lock, irqflags); - dev_info_ratelimited(&m->pdev->dev, "REGULATORY_REQUEST: %c%c initiator=%u generation=%u\n", - reg.alpha2[0], reg.alpha2[1], request->initiator, m->reg_generation); + dev_dbg(&m->pdev->dev, "REGULATORY_REQUEST: %c%c initiator=%u generation=%u\n", + reg.alpha2[0], reg.alpha2[1], request->initiator, m->reg_generation); } static const struct cfg80211_ops mt_cfg_ops = { @@ -611,6 +611,6 @@ int mt_register_regulatory_gate(struct mt7932 *m) return ret; WRITE_ONCE(m->interface_registered, true); schedule_work(&m->startup_work); - dev_info(&m->pdev->dev, "REGULATORY_GATE_REGISTERED: cfg80211 country policy required\n"); + dev_dbg(&m->pdev->dev, "REGULATORY_GATE_REGISTERED: cfg80211 country policy required\n"); return 0; } From 559e474df2619fee338b5e989280393ae6d1887d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1039/1579] wifi: mt7932: disable link power management only once Probe disabled ASPM and clock PM on the link twice. The first call fails the probe, so that a remove and rescan cannot leave the radio on a link policy it was never tested with. The second call could then only succeed, and its warning for a failure was unreachable. Keep the first call and merge the two comments. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index d8a319cfc12a8..13b8f97ae5910 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1160,17 +1160,16 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = apple_piodma_radio_check(pdev); if (ret) return ret; - /* A remove/rescan may have recreated link state under the global policy. */ + /* + * Station operation was tested with ASPM and clock PM off on this link, + * and a remove/rescan may have restored them under the global policy. + */ ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); if (ret) return ret; domain = iommu_get_domain_for_dev(&pdev->dev); if (!domain || (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) return dev_err_probe(&pdev->dev, -EINVAL, "translated DMA domain required\n"); - /* Station operation was qualified with ASPM and clock PM off on this link. */ - ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); - if (ret) - dev_warn(&pdev->dev, "link power management left as found: %d\n", ret); ret = pcim_enable_device(pdev); if (ret) return ret; From df00dc306318ac4628a2f7ec31ed3e92f5e1a1a5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1040/1579] wifi: mt7932: drop internal document names from comments Several comments named internal design notes (NEO_*_CONTRACT and similar) or used bring-up wording such as "campaign" and "explicit experiment". These names mean nothing outside the project. State the facts they stood for instead. Comment-only change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/association.h | 9 ++++----- drivers/net/wireless/mediatek/mt7932/calibration.h | 2 +- drivers/net/wireless/mediatek/mt7932/frontend.c | 4 ++-- drivers/net/wireless/mediatek/mt7932/link.c | 2 +- drivers/net/wireless/mediatek/mt7932/packet.h | 4 ++-- drivers/net/wireless/mediatek/mt7932/recovery.c | 10 +++++----- drivers/net/wireless/mediatek/mt7932/reset.c | 2 +- drivers/net/wireless/mediatek/mt7932/scan.h | 6 +++--- 8 files changed, 19 insertions(+), 20 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/association.h b/drivers/net/wireless/mediatek/mt7932/association.h index eb207921906e1..ba733a885e261 100644 --- a/drivers/net/wireless/mediatek/mt7932/association.h +++ b/drivers/net/wireless/mediatek/mt7932/association.h @@ -91,8 +91,8 @@ static inline int mt7932_assoc_capabilities(u8 *out, size_t capacity, } /* Original25G83 pure WPA2 binary-key result: structured type6, not a string. - * NEO_TYPE6_BINARY_PMK_NATIVE_CONTRACT: standard nl80211 PMK maps directly; - * no passphrase API, key derivation, duplicate independent PMK or extra flags. + * The standard nl80211 PMK maps directly; no passphrase API, key derivation, + * duplicate independent PMK or extra flags. */ static inline int mt7932_connect_binary_pmk(u8 *body, size_t capacity, const u8 *pmk, size_t length) @@ -111,8 +111,8 @@ static inline int mt7932_connect_binary_pmk(u8 *body, size_t capacity, } /* Explicit OPEN or PMK-only WPA2/CCMP; never an automatic security fallback. - * Caller owns identities/resources and validated BSS state. - * NEO_WPA2_FULLMAC_ASSOCIATION_AND_ETHERNET_CONTRACT. Never log this body. + * Caller owns identities/resources and validated BSS state. Never log this + * body: it carries the PMK. */ static inline int mt7932_connect_body(u8 *body, size_t capacity, const u8 *ssid, size_t ssid_length, const u8 pmk[32], const u8 bssid[6], @@ -141,7 +141,6 @@ static inline int mt7932_connect_body(u8 *body, size_t capacity, const u8 *ssid, /* NORMAL25G83 original strict keyless OPEN branch, without optional * automatic-authentication policy. Keep this coherent version/auth/ * policy tuple together; no dummy keys or stale security state. - * NEO_NORMAL_STRICT_OPEN_PRODUCER_CONTRACT. */ if (open) { body[1] = 1; diff --git a/drivers/net/wireless/mediatek/mt7932/calibration.h b/drivers/net/wireless/mediatek/mt7932/calibration.h index 85075bce7b255..f13ee11866bad 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration.h +++ b/drivers/net/wireless/mediatek/mt7932/calibration.h @@ -80,7 +80,7 @@ struct mt7932_cal_segment { }; /* Validate the entire original container before selecting any data to send. - * NEO_MT7932_STOCK_ARTIFACTS_AND_CAL_CONVERSION: raw BLOB, not a transcript. + * The input is the raw BLOB container, not a converted transcript. */ static inline int mt7932_cal_validate(const u8 *data, size_t size) { diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 7b0259bde004a..0a5e700d16ccd 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -89,7 +89,7 @@ void mt_scan_event(struct mt7932 *m, const struct mt7932_event *event) if (m->stopping) return; /* Abort completes userspace locally, not by an invented cancel ACK. - * Keep the sequence reserved until an actual old-campaign done arrives. + * Keep the sequence reserved until the old scan's own done event arrives. */ if (event->length >= 37 && m->retired_scan_seq && event->packet[36] == m->retired_scan_seq) { @@ -127,7 +127,7 @@ void mt_abort_scan(struct wiphy *wiphy, struct wireless_dev *wdev) } /* No command/response lock held. The caller closes new scan admission first. - * Natural EID0d completion fences the campaign; a local cancel alone does not. + * Natural EID0d completion fences the scan; a local cancel alone does not. */ int mt_scan_quiesce(struct mt7932 *m) { diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index bcdcb71ce3db9..55a08f5b86728 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -423,7 +423,7 @@ void mt_connect_work(struct work_struct *work) goto free; /* Initial station policy: framework M=0 branch and no optional flags. * This yields flag2 independently of credentials; no scan-result count - * belongs in this field. The recovered framework uses listen interval20. + * belongs in this field. The original driver uses listen interval 20. * Other platform flag policies remain unsupported, not guessed mappings. */ ret = mt7932_connect_body(body, 856, m->connect_ssid, m->connect_ssid_length, diff --git a/drivers/net/wireless/mediatek/mt7932/packet.h b/drivers/net/wireless/mediatek/mt7932/packet.h index 303cc87fef8b0..a93d04d91fd41 100644 --- a/drivers/net/wireless/mediatek/mt7932/packet.h +++ b/drivers/net/wireless/mediatek/mt7932/packet.h @@ -14,8 +14,8 @@ struct mt7932_rx_frame { bool translated, group4, signal_valid, rate_valid, unicast; }; -/* NEO_PACKET_RX_GROUPS_AND_DATA_TXP_CONTRACT. Strict non-monitor subset; - * descriptor groups are ordered4,1,2,3,5, not by numerical group identifier. +/* Strict non-monitor subset of the RX descriptor; its groups are ordered + * 4, 1, 2, 3, 5, not by numerical group identifier. */ static inline int mt7932_rx_frame(const u8 *packet, size_t size, struct mt7932_rx_frame *out) { diff --git a/drivers/net/wireless/mediatek/mt7932/recovery.c b/drivers/net/wireless/mediatek/mt7932/recovery.c index 91ae4b0342608..fc60bc05b13d6 100644 --- a/drivers/net/wireless/mediatek/mt7932/recovery.c +++ b/drivers/net/wireless/mediatek/mt7932/recovery.c @@ -6,9 +6,6 @@ static int mt_runtime_scheduler(struct mt7932 *m, bool publish); static int mt_runtime_rebuild(struct mt7932 *m); static void mt_runtime_publish(struct mt7932 *m); static int mt_runtime_ack(struct mt7932 *m, u32 bit); -/* First runtime epoch only. Contracts: NEO_IRQ_SELECTION_AND_RUNTIME_HANDSHAKE - * and NEO_RUNTIME_RING_REBUILD_AND_OBSERVED_PRECAL. - */ static int mt_tx_retire(struct mt7932 *m) { @@ -179,7 +176,7 @@ static int mt_runtime_ack(struct mt7932 *m, u32 bit) int mt_recovery_gate(struct mt7932 *m) { /* Original observed 25F84 CID6b bodies, not PL country tables. - * Explicit experiment; first two action2 field meanings remain opaque. + * The meaning of the first two action2 fields is unknown. */ static const u8 queue_config[3][32] = { {0x10,0x11,2,0,3,0,7,0,0x2f,0,2}, @@ -191,7 +188,10 @@ int mt_recovery_gate(struct mt7932 *m) unsigned int i, phase = 0; int ret; - /* mt_runtime_start() completed configuration before entering this stage. */ + /* + * mt_runtime_start() completed configuration before entering this stage, + * which switches to the runtime ring layout once per firmware boot. + */ if (m->runtime_epoch) return -EINVAL; for (i = 0; i < ARRAY_SIZE(queue_config); i++) { diff --git a/drivers/net/wireless/mediatek/mt7932/reset.c b/drivers/net/wireless/mediatek/mt7932/reset.c index 114da7afd69eb..d93080513c77f 100644 --- a/drivers/net/wireless/mediatek/mt7932/reset.c +++ b/drivers/net/wireless/mediatek/mt7932/reset.c @@ -125,7 +125,7 @@ int mt_function_reset(struct mt7932 *m) if (!peer) return mt_function_reset_locked(m); /* The radio functions may share state beyond PCI's advertised FLR scope. - * No qualified contract permits resetting Wi-Fi while Bluetooth is live. + * Resetting Wi-Fi while Bluetooth is live has never been tested. * Trylock avoids inversion with a concurrent sibling probe/remove; hold * it across FLR to prevent Bluetooth starting after the admission check. */ diff --git a/drivers/net/wireless/mediatek/mt7932/scan.h b/drivers/net/wireless/mediatek/mt7932/scan.h index ae26645cc3693..217fc5c9d1e96 100644 --- a/drivers/net/wireless/mediatek/mt7932/scan.h +++ b/drivers/net/wireless/mediatek/mt7932/scan.h @@ -10,8 +10,8 @@ /* Selected ordinary TX17/fullmac wildcard scan, not a pre-RF gate. * Caller must own BSS0 and filter cfg80211 DISABLED channels first. NO_IR - * channels require mt7932_scan_passive() before submission. - * NEO_FIRST_CFG80211_SCAN_CONTRACT: 20ms dwell and meaningful MT7932 tail. + * channels require mt7932_scan_passive() before submission. The body uses + * a 20 ms dwell and fills the MT7932-specific tail. */ static inline int mt7932_scan_body(u8 *body, size_t capacity, u8 seq, const u8 *channels, unsigned int count) @@ -87,7 +87,7 @@ static inline void mt7932_scan_disconnected_passive(u8 body[0x4d4]) } /* This firmware requires its native wildcard template. Directed scans use - * no probe IEs. Forwarding the normal supplicant extras stalled the campaign + * no probe IEs. Forwarding the normal supplicant extras stalled the scan * on J700; arbitrary caller probe-IE carriage remains unsupported here. */ static inline int mt7932_scan_options(u8 body[0x4d4], const u8 *ssid, From 5913e895a3a4632a4d85c21faae55abc8060ad13 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1041/1579] wifi: mt7932: tidy the Kconfig help and file headers Describe what the driver supports and what it needs in the Kconfig help instead of listing bring-up details, use C++ comment style for the SPDX line of every C file, and remove repeated blank lines. Rename the documentation title, which still called the driver a bring-up. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 4 +-- drivers/net/wireless/mediatek/mt7932/Kconfig | 34 +++++++++---------- .../mediatek/mt7932/calibration_transport.c | 4 +-- drivers/net/wireless/mediatek/mt7932/data.c | 2 +- .../net/wireless/mediatek/mt7932/frontend.c | 5 +-- drivers/net/wireless/mediatek/mt7932/link.c | 2 +- drivers/net/wireless/mediatek/mt7932/mt7932.c | 2 -- drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 -- .../net/wireless/mediatek/mt7932/recovery.c | 2 +- .../net/wireless/mediatek/mt7932/startup.c | 2 +- 10 files changed, 25 insertions(+), 34 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index f4ad65f0815f7..58e97162bdba3 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -1,7 +1,7 @@ .. SPDX-License-Identifier: GPL-2.0-only -MT7932 radio bring-up on the MacBook Neo -====================================== +MediaTek MT7932 radio on the MacBook Neo +======================================== J700 MT7932 Wi-Fi uses cfg80211 and NetworkManager with firmware-managed WPA2-CCMP authentication. Bluetooth uses a transport on the radio's second diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index 273c389ad09bd..c6120372d1f3b 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -1,26 +1,26 @@ # SPDX-License-Identifier: GPL-2.0-only config MT7932_FULLMAC - tristate "Experimental J700 MT7932 fullmac wireless support" + tristate "MediaTek MT7932 Wi-Fi on the Apple MacBook Neo (J700)" depends on PCI && OF && ARM64 && IOMMU_DMA depends on PCIE_APPLE_PIODMA_DIAG && CFG80211 select FW_LOADER help - Native support for the built-in J700 Wi-Fi endpoint: managed - WPA2-CCMP station operation on 2.4 GHz and on the 5 GHz UNII-1, -2, - -2e and -3 channels that the regulatory domain permits. Radar (DFS) - channels are scanned passively but not joined. - Requires original firmware, own calibration and original-derived - country policy packages in addition to the cfg80211 regulatory database. - The device is probed before the root filesystem is mounted, so say M - unless those files are in the initramfs. - Requires a trained retained PCI link and a translated SID1 DMA domain. - Firmware, runtime DMA and calibration negotiation start automatically; - cfg80211 and userspace select the country and network. - Once DMA pointers are published the module is pinned until hardware - reset. Normal unload is refused, including with active Bluetooth. - A failed probe reset retains the binding and DMA; a successful checked - reset retires ownership before releasing probe resources. - Other MediaTek devices and other machines are not matched. + Support for the Wi-Fi function of the MediaTek MT7932 radio in the + Apple MacBook Neo (J700). The firmware runs association and the + WPA2-PSK handshake; the driver is a cfg80211 station on 2.4 GHz and + on the 5 GHz UNII-1, -2, -2e and -3 channels that the regulatory + domain permits. Radar (DFS) channels are scanned passively but not + joined. Other MediaTek devices and other machines are not matched. + + The driver needs the original firmware, the unit's own calibration + and the country policy packages under mediatek/mt7932/, besides the + cfg80211 regulatory database. The device is probed before the root + filesystem is mounted, so say M unless those files are in the + initramfs. + + Once the device has been given DMA addresses, the module cannot be + unloaded. A function that fails to initialize and cannot be reset + stays bound and keeps its DMA memory. config MT7932_RF_KUNIT_TEST bool "KUnit tests for MT7932 terminal RF failure" if !KUNIT_ALL_TESTS diff --git a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c index fb5a33dd5aa61..4efd5bed77479 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c +++ b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c @@ -1,8 +1,6 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" - - /* Called only by the serialized command owner, never from the RX consumer. */ int mt_cal_procedure(struct mt7932 *m, struct mt7932_cal_piece *pieces, unsigned int count, unsigned int logical, u8 context_version) diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index e4f9d8a9f9cc8..99ae52977b582 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" static bool mt_bss_tx_allowed(struct mt7932 *m); diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 0a5e700d16ccd..8d873a4379bfa 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" static void mt_scan_finish_work(struct work_struct *work); @@ -15,9 +15,6 @@ struct mt7932 *mt_from_wiphy(struct wiphy *wiphy) return *(struct mt7932 **)wiphy_priv(wiphy); } - - - static void mt_scan_finish_work(struct work_struct *work) { struct mt7932 *m = container_of(work, struct mt7932, scan_finish_work); diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 55a08f5b86728..a1c345ec1f64e 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" static int mt_peer_reserve(struct mt7932 *m); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 13b8f97ae5910..41d34e9c6b8c7 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -781,7 +781,6 @@ static int mt_config_gate(struct mt7932 *m) return -ETIMEDOUT; } - static int mt_board_gate(struct mt7932 *m) { const u32 addresses[] = {0x70, 0x160}; @@ -1052,7 +1051,6 @@ void mt_transport_snapshot(struct mt7932 *m) } } - static int mt_initialize(struct mt7932 *m) { int ret; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index d6c80ecb315ed..6994f0c208105 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -2,7 +2,6 @@ #ifndef MT7932_H #define MT7932_H - #include #include #include @@ -31,7 +30,6 @@ #include "rate.h" #include "capabilities.h" - #define W 0xd4000 #define H 0xd6000 #define C 0xd7000 diff --git a/drivers/net/wireless/mediatek/mt7932/recovery.c b/drivers/net/wireless/mediatek/mt7932/recovery.c index fc60bc05b13d6..31168bfa2b489 100644 --- a/drivers/net/wireless/mediatek/mt7932/recovery.c +++ b/drivers/net/wireless/mediatek/mt7932/recovery.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" static int mt_tx_retire(struct mt7932 *m); diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index c949c28254355..051ab3793d965 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include #include "mt7932.h" From c333bdf9d6ffa1ce011e368398f3b31df1caa6e4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1042/1579] Bluetooth: mt7932: drop the enable module parameter enable=0 made probe return -ENODEV, to keep the Bluetooth function unbound while the transport was being brought up. Nothing sets it any more, and a function can be kept unbound with the driver core's driver_override or by blacklisting the module instead. Remove the parameter and the documentation that mentions it. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../networking/device_drivers/wifi/mt7932-neo.rst | 2 +- drivers/bluetooth/Kconfig | 7 +++---- drivers/bluetooth/mt7932_bt_pcie.c | 7 ------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 58e97162bdba3..a5f0f16a9d46f 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -59,7 +59,7 @@ makes the driver core unbind Bluetooth before Wi-Fi; unloading DMA pointers, and refuses function reset while its Bluetooth sibling is live. The probe rejects a function that is not cold, which is what now prevents a reprobe after a failed or uncertain -admission; ``enable=0`` leaves the function unbound. The transport also +admission. The transport also disables ASPM on the shared link. Like the Wi-Fi driver, it reads its firmware while probing, so build it as a module. The tested Wi-Fi driver owns function 0 and Bluetooth owns function 1. diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 5f8105fa09ae0..f87544448d2f9 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -31,10 +31,9 @@ config BT_MTK7932_PCIE help Support the J700 Bluetooth PCIe function with locally installed firmware and calibration. The function is probed once the Wi-Fi function has - initialized under the mt7932-fullmac driver; the enable=0 parameter - leaves it unbound. Both functions appear before the root filesystem - is mounted and read their firmware when probed, so say M, like the - Wi-Fi driver. + initialized under the mt7932-fullmac driver. Both functions appear + before the root filesystem is mounted and read their firmware when + probed, so say M, like the Wi-Fi driver. System suspend uses the firmware sleep handshake and preserves DMA storage. Hibernation and firmware reloading are unsupported. A controller that stops answering commands is reset over HCI and set diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 2cebb5c3d6f5a..57c6a3a216bc8 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -67,10 +67,6 @@ #define BT7932_CAL "mediatek/j700-mt7932-btcal.bin" #define BT7932_ADDR "mediatek/j700-mt7932-bdaddr.bin" -static bool enable = true; -module_param(enable, bool, 0600); -MODULE_PARM_DESC(enable, "Probe the transport (default: on); no effect on an already bound device"); - struct bt7932_geometry { u16 stride; u16 payload; @@ -1126,9 +1122,6 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) u16 command; int ret; - /* enable=0 leaves the function unbound for diagnosis. */ - if (!READ_ONCE(enable)) - return -ENODEV; if (!of_machine_is_compatible("apple,j700") || !of_machine_is_compatible("apple,t8140") || pci_domain_nr(pdev->bus) || pdev->bus->number != 1 || pdev->devfn != PCI_DEVFN(0, 1)) From a70f8e8a419cbe16359b49e9ab9144a72e6bf266 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1043/1579] Bluetooth: mt7932: report setup progress at debug level The driver logged eight lines at info level while probing, among them DMA addresses, and another whenever the HCI device was set up, which happens on every power-on because the setup is not persistent. On a transport fault it also printed eleven error lines, one per ring. Log routine progress and the ring state at debug level. Errors keep their level, and a successful controller recovery is still reported at info level. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 57c6a3a216bc8..0ecc88cc87649 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -177,7 +177,7 @@ static void bt7932_fault_locked(struct bt7932 *bt, const char *reason) dev_err(&bt->pdev->dev, "terminal transport fault: %s; owned DMA retained, external reset required\n", reason); for (channel = 0; channel < BT7932_CHANNELS; channel++) - dev_err(&bt->pdev->dev, "ring%u host=%u firmware=%u previous=%u\n", channel, + dev_dbg(&bt->pdev->dev, "ring%u host=%u firmware=%u previous=%u\n", channel, bt7932_counter(bt, bt->host_transfer, channel), bt7932_counter(bt, bt->fw_transfer, channel), bt->rings[channel].previous); pci_clear_master(bt->pdev); @@ -409,8 +409,8 @@ static int bt7932_arena(struct bt7932 *bt) context[0x51] = 13; context[0x53] = 2; dma_wmb(); - dev_info(&bt->pdev->dev, "owned SID2 arena=%pad bytes=%zu image=%pad bytes=%zu, both low32\n", - &bt->arena_dma, bt->arena_size, &bt->image_dma, bt->image_size); + dev_dbg(&bt->pdev->dev, "owned SID2 arena=%pad bytes=%zu image=%pad bytes=%zu, both low32\n", + &bt->arena_dma, bt->arena_size, &bt->image_dma, bt->image_size); return 0; } @@ -724,7 +724,7 @@ static int bt7932_download(struct bt7932 *bt) } writel(0x188d1807, bt->bar + BT7932_CONNECTION_SELECTOR); writel(1, bt->bar + BT7932_SEMAPHORE_RELEASE); - dev_info(&bt->pdev->dev, "BT_IMAGE_ACCEPTED response1 boot0->2; image remains retained\n"); + dev_dbg(&bt->pdev->dev, "BT_IMAGE_ACCEPTED response1 boot0->2; image remains retained\n"); return 0; } @@ -825,7 +825,7 @@ static void bt7932_channels(struct bt7932 *bt) bt->last_sleep = 0; WRITE_ONCE(bt->awake, true); WRITE_ONCE(bt->ipc_ready, true); - dev_info(&bt->pdev->dev, "BT_IPC_READY ten retained advertisements;127 RX offers per channel\n"); + dev_dbg(&bt->pdev->dev, "BT_IPC_READY ten retained advertisements;127 RX offers per channel\n"); } static int bt7932_open(struct hci_dev *hdev) @@ -948,7 +948,7 @@ static int bt7932_setup(struct hci_dev *hdev) HCI_INIT_TIMEOUT); if (ret) goto fault; - dev_info(&bt->pdev->dev, "BT_HCI_SETUP_OK Reset/calibration/PTX/address completed in order\n"); + dev_dbg(&bt->pdev->dev, "BT_HCI_SETUP_OK Reset/calibration/PTX/address completed in order\n"); mutex_lock(&bt->lock); if (bt->recovery == BT7932_RECOVERY_REOPENING) dev_info(&bt->pdev->dev, "controller recovered\n"); @@ -1056,7 +1056,7 @@ static int bt7932_iommu(struct bt7932 *bt) if (ret) dev_err(dev, "BT_IOMMU_REJECT 32-bit DMA mask status=%d\n", ret); else - dev_info(dev, "BT_IOMMU_ADMITTED RID101->SID2 matched translated DART\n"); + dev_dbg(dev, "BT_IOMMU_ADMITTED RID101->SID2 matched translated DART\n"); } of_node_put(target); return ret; @@ -1173,7 +1173,7 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = -ENOMEM; goto fail; } - dev_info(&pdev->dev, "BT_BAR0 resource=%pR\n", &pdev->resource[0]); + dev_dbg(&pdev->dev, "BT_BAR0 resource=%pR\n", &pdev->resource[0]); ret = pci_read_config_dword(pdev, BT7932_BOOT_CONFIG, &boot); if (ret || boot == U32_MAX || (boot & 0xf) || ((boot >> 4) & 7) > 1) { dev_err(&pdev->dev, "BT_IDENTITY_REJECT config_status=%d boot=%#x\n", ret, boot); @@ -1190,7 +1190,7 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = -ENODEV; goto fail; } - dev_info(&pdev->dev, "BT_COLD_ADMISSION ROM=%u bootstate0 chip7932 SID2\n", bt->rom); + dev_dbg(&pdev->dev, "BT_COLD_ADMISSION ROM=%u bootstate0 chip7932 SID2\n", bt->rom); ret = bt7932_read_inputs(bt); if (ret) goto fail; @@ -1251,7 +1251,7 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret < 0) goto retained_fault; bt->registered = true; - dev_info(&pdev->dev, "BT_HCI_REGISTERED; setup/discovery/connection remain separate gates\n"); + dev_dbg(&pdev->dev, "BT_HCI_REGISTERED; setup/discovery/connection remain separate gates\n"); return 0; retained_fault: mutex_lock(&bt->lock); From 5500f5ef688913d1d1718fa6f41124d52dfd979e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1044/1579] Bluetooth: mt7932: remove write-only packet counters tx_packets, rx_packets and diagnostics were counted but never read. The HCI core keeps its own statistics, which the driver updates as before. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 0ecc88cc87649..34647ab577c14 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -140,7 +140,6 @@ struct bt7932 { bool stopping, runtime_forbidden; enum bt7932_recovery recovery; enum bt7932_pm_state pm_state; - u64 tx_packets, rx_packets, diagnostics; }; static void bt7932_ring_doorbell(struct bt7932 *bt, unsigned int bit) @@ -583,7 +582,6 @@ static void bt7932_tx_work(struct work_struct *work) ring->host = (ring->host + 1) & BT7932_MASK; bt7932_publish_counter(bt, bt->host_transfer, channel, ring->host); bt7932_ring_doorbell(bt, geometry[channel].doorbell); - bt->tx_packets++; bt->hdev->stat.byte_tx += skb->len; mutex_unlock(&bt->lock); kfree_skb(skb); @@ -628,10 +626,7 @@ static int bt7932_rx_locked(struct bt7932 *bt, unsigned int channel) hci_skb_pkt_type(skb) = geo->packet; skb_put_data(skb, descriptor + 16, length); bt->hdev->stat.byte_rx += length; - bt->rx_packets++; hci_recv_frame(bt->hdev, skb); - } else if (!geo->packet) { - bt->diagnostics++; } slot = (slot + 1) & BT7932_MASK; } From 08640f32d3eaf3f460f041abe09395a4aec0e5c2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1045/1579] Bluetooth: mt7932: describe the driver without bring-up wording The file header called the driver an experiment reconstructed from "protocol contracts", two comments referred to a review, and the module description said "Experimental standalone". Describe the hardware and the behaviour instead, and keep the credit for the transport research. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 34647ab577c14..57e732df261de 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1,8 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * J700 MT7932 Bluetooth experiment, reconstructed from the attributed - * protocol contracts and public Linux interfaces. Original transport research: - * DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. + * Bluetooth transport for the MediaTek MT7932 radio in the Apple MacBook Neo + * (J700), whose Bluetooth controller is the second function of the radio's + * PCIe device. Transport research by DJ (DjDeveloperr), Ace (Acelogic) and + * Ryan Murray. * * Firmware and board inputs are supplied locally. System sleep preserves the * transport arena and uses the firmware quiesce/restore handshake. Unload @@ -733,7 +734,7 @@ static int bt7932_ipc_state(struct bt7932 *bt, u8 *state) value = readl(bt->bar + BT7932_STATE); if (value == U32_MAX) return -EIO; - /* The reviewed early-state rule replaces memory with the BAR byte. */ + /* Until the state in memory passes 1, the BAR register holds it. */ *state = value & 0xff; } return 0; @@ -816,7 +817,7 @@ static void bt7932_channels(struct bt7932 *bt) bt7932_publish_counter(bt, bt->host_transfer, channel, BT7932_DEPTH - 1); bt7932_ring_doorbell(bt, geometry[channel].doorbell); } - /* The reviewed cold IPC transition starts ACTIVE; no initial sleep ACK. */ + /* The IPC starts active after a cold start; no sleep ACK is due. */ bt->last_sleep = 0; WRITE_ONCE(bt->awake, true); WRITE_ONCE(bt->ipc_ready, true); @@ -1469,7 +1470,7 @@ static struct pci_driver bt7932_driver = { }; module_pci_driver(bt7932_driver); -MODULE_DESCRIPTION("Experimental standalone J700 MT7932 Bluetooth PCIe transport"); +MODULE_DESCRIPTION("MediaTek MT7932 Bluetooth PCIe transport for the Apple MacBook Neo"); MODULE_LICENSE("GPL"); MODULE_FIRMWARE(BT7932_FW_B0); MODULE_FIRMWARE(BT7932_FW_B1); From 7a01b6e775a1e1bc6db5973f5af6f3d935ff385a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1046/1579] Bluetooth: mt7932: add KUnit tests for packet and board file checks bt7932_hci_length() decides whether a packet the controller placed in a receive ring is handed to the HCI core, and bt7932_validate_ptx() whether a board file is sent to the controller. Cover both with KUnit, including truncated and over-long packets and corrupted board files built from synthetic data. The tests are built into the driver when BT_MTK7932_PCIE_KUNIT_TEST is enabled, so that they reach the static helpers. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/Kconfig | 13 +++ drivers/bluetooth/mt7932_bt_pcie.c | 4 + drivers/bluetooth/mt7932_bt_pcie_test.c | 133 ++++++++++++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 drivers/bluetooth/mt7932_bt_pcie_test.c diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index f87544448d2f9..2280f206d0b70 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -43,6 +43,19 @@ config BT_MTK7932_PCIE To build as a module, choose M. The module is mt7932_bt_pcie. +config BT_MTK7932_PCIE_KUNIT_TEST + bool "KUnit tests for the MT7932 Bluetooth transport" if !KUNIT_ALL_TESTS + depends on KUNIT=y && BT_MTK7932_PCIE + default KUNIT_ALL_TESTS + help + Build KUnit tests into the MT7932 Bluetooth transport. They check, + without radio hardware, that HCI packets whose length does not + match their header are rejected in both directions, and that a + board file with a bad layout or checksum is never sent to the + controller. + + If unsure, say N. + config BT_HCIBTUSB tristate "HCI USB driver" depends on USB diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 57e732df261de..19d2d714cdd24 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1478,3 +1478,7 @@ MODULE_FIRMWARE(BT7932_FW_B1_FALLBACK); MODULE_FIRMWARE(BT7932_PTX); MODULE_FIRMWARE(BT7932_CAL); MODULE_FIRMWARE(BT7932_ADDR); + +#if IS_ENABLED(CONFIG_BT_MTK7932_PCIE_KUNIT_TEST) +#include "mt7932_bt_pcie_test.c" +#endif diff --git a/drivers/bluetooth/mt7932_bt_pcie_test.c b/drivers/bluetooth/mt7932_bt_pcie_test.c new file mode 100644 index 0000000000000..748dcabde731c --- /dev/null +++ b/drivers/bluetooth/mt7932_bt_pcie_test.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the checks mt7932_bt_pcie applies to HCI packets and to the + * board file before either reaches the controller or the HCI core. Built as + * part of mt7932_bt_pcie.c, so the tests reach its static helpers. + */ +#include + +static void bt7932_hci_length_command_test(struct kunit *test) +{ + /* HCI_Reset (0x0c03) and a command carrying two parameter bytes. */ + static const u8 reset[] = { 0x03, 0x0c, 0x00 }; + static const u8 command[] = { 0x1a, 0xfc, 0x02, 0x11, 0x22, 0x33 }; + + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_COMMAND_PKT, reset, sizeof(reset))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_COMMAND_PKT, reset, 2)); + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_COMMAND_PKT, command, 5)); + /* One byte short of, and one byte past, the declared length. */ + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_COMMAND_PKT, command, 4)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_COMMAND_PKT, command, 6)); +} + +static void bt7932_hci_length_event_test(struct kunit *test) +{ + /* Command Complete for HCI_Reset with status 0, then one spare byte. */ + static const u8 event[] = { 0x0e, 0x04, 0x01, 0x03, 0x0c, 0x00, 0xff }; + + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_EVENT_PKT, event, 6)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_EVENT_PKT, event, 5)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_EVENT_PKT, event, 7)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_EVENT_PKT, event, 1)); +} + +static void bt7932_hci_length_data_test(struct kunit *test) +{ + /* ACL and ISO lengths are 16-bit; ISO keeps two flag bits on top. */ + static const u8 acl[] = { 0x01, 0x20, 0x02, 0x00, 0xaa, 0xbb }; + static const u8 sco[] = { 0x01, 0x00, 0x01, 0xcc }; + static const u8 iso[] = { 0x01, 0x00, 0x01, 0xc0, 0xdd }; + static const u8 long_acl[] = { 0x01, 0x20, 0x00, 0x01 }; + + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_ACLDATA_PKT, acl, sizeof(acl))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_ACLDATA_PKT, acl, 5)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_ACLDATA_PKT, acl, 3)); + /* A 256-byte ACL payload must not be satisfied by its header alone. */ + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_ACLDATA_PKT, long_acl, sizeof(long_acl))); + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_SCODATA_PKT, sco, sizeof(sco))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_SCODATA_PKT, sco, 3)); + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_ISODATA_PKT, iso, sizeof(iso))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_ISODATA_PKT, iso, 4)); +} + +static void bt7932_hci_length_type_test(struct kunit *test) +{ + static const u8 packet[] = { 0x00, 0x00, 0x00, 0x00 }; + + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_VENDOR_PKT, packet, sizeof(packet))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(0, packet, sizeof(packet))); +} + +#define BT7932_TEST_PTX_SIZE 198 + +/* A well-formed container with synthetic record contents. */ +static void bt7932_test_ptx(u8 *data) +{ + static const u32 types[] = { 0x101, 0x201, 0x301, 0x401 }; + static const u32 sizes[] = { 14, 44, 44, 0 }; + u32 offset = 96, sum, i, j; + + memset(data, 0, BT7932_TEST_PTX_SIZE); + memcpy(data, "BLOB", 4); + put_unaligned_le32(96, data + 4); + put_unaligned_le16(1, data + 8); + put_unaligned_le16(ARRAY_SIZE(types), data + 10); + for (i = 0; i < ARRAY_SIZE(types); i++) { + u8 *record = data + 16 + i * 20; + + for (j = 0, sum = 0; j < sizes[i]; j++) { + data[offset + j] = 0x40 + i * 16 + j; + sum += data[offset + j]; + } + put_unaligned_le32(types[i], record); + put_unaligned_le32(offset, record + 4); + put_unaligned_le32(sizes[i], record + 8); + put_unaligned_le32(sum, record + 12); + offset += sizes[i]; + } +} + +static void bt7932_validate_ptx_test(struct kunit *test) +{ + u8 *data = kunit_kzalloc(test, BT7932_TEST_PTX_SIZE + 1, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, data); + bt7932_test_ptx(data); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), 0); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE - 1), -EINVAL); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE + 1), -EINVAL); + + /* A payload byte that no longer matches its record's checksum. */ + data[120]++; + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), -EINVAL); + + bt7932_test_ptx(data); + data[0] = 'b'; + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), -EINVAL); + + /* A record of the wrong type in the first slot. */ + bt7932_test_ptx(data); + put_unaligned_le32(0x201, data + 16); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), -EINVAL); + + /* A record length other than the one its type has. */ + bt7932_test_ptx(data); + put_unaligned_le32(45, data + 16 + 2 * 20 + 8); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), -EINVAL); +} + +static struct kunit_case bt7932_test_cases[] = { + KUNIT_CASE(bt7932_hci_length_command_test), + KUNIT_CASE(bt7932_hci_length_event_test), + KUNIT_CASE(bt7932_hci_length_data_test), + KUNIT_CASE(bt7932_hci_length_type_test), + KUNIT_CASE(bt7932_validate_ptx_test), + {} +}; + +static struct kunit_suite bt7932_test_suite = { + .name = "mt7932-bt-pcie", + .test_cases = bt7932_test_cases, +}; + +kunit_test_suite(bt7932_test_suite); From dcd65e980ce86644008d63590b14d08fc374b053 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 1047/1579] selftests: mt7932: fail on routine driver messages at info level The radio drivers report scans, association and setup steps at debug level. Make the physical network test fail when either driver logs at info or notice level while it runs, so that a return of per-scan log spam is caught on the hardware. The check reads "dmesg --level=notice,info" and counts the messages of mt7932-fullmac and mt7932_bt_pcie newer than the last kernel message before the test. Add unit tests for the parsing. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/net/mt7932_e2e.py | 27 ++++++++++++++++++- .../selftests/drivers/net/mt7932_e2e_test.py | 22 +++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/net/mt7932_e2e.py b/tools/testing/selftests/drivers/net/mt7932_e2e.py index 258baebce12f3..f66ccaac201a6 100644 --- a/tools/testing/selftests/drivers/net/mt7932_e2e.py +++ b/tools/testing/selftests/drivers/net/mt7932_e2e.py @@ -25,6 +25,12 @@ r'SCAN_INTERFACE_SETUP_FAILED|RF_FAILED|recovery-required=1|WARNING:|' r'UBSAN:|hung_task|blocked for more than|WIFI_FLR_REFUSED_OR_FAILED', re.I) +# Both radio drivers report progress at debug level, so a message of theirs at +# info or notice level during scans, association or traffic is log spam. +driver_message = re.compile(r'^\[\s*(\d+\.\d+)\] (?:mt7932-fullmac|mt7932_bt_pcie) ' + r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]: ', re.M) +log_stamp = re.compile(r'^\[\s*(\d+\.\d+)\]', re.M) + def run(argv, timeout=30): return subprocess.run(argv, text=True, stdout=subprocess.PIPE, @@ -42,6 +48,17 @@ def kernel_ok(): return p.stdout +def last_log_time(log): + """Timestamp of the newest kernel log message, or 0.0 if there is none.""" + stamps = log_stamp.findall(log) + return float(stamps[-1]) if stamps else 0.0 + + +def driver_info_messages(log, since): + """Radio driver messages in a ``dmesg --level=notice,info`` log after @since.""" + return [m.group(0) for m in driver_message.finditer(log) if float(m.group(1)) > since] + + def dns_name(packet, offset): """Decode a bounded DNS name, rejecting bad offsets and compression cycles.""" labels, seen, end, size = [], set(), None, 1 @@ -178,8 +195,9 @@ def main(): compatible = Path('/sys/firmware/devicetree/base/compatible').read_bytes().split(b'\0') require(b'apple,j700' in compatible and b'apple,t8140' in compatible, 'Wrong device') out.mkdir(mode=0o700) + started = None try: - kernel_ok() + started = last_log_time(kernel_ok()) profile = run(['nmcli', '-g', 'connection.uuid', 'connection', 'show', 'id', args.profile]) profile_uuid = profile.stdout.strip() require(profile.returncode == 0 and re.fullmatch(r'[0-9a-f-]{36}', profile_uuid), @@ -272,6 +290,13 @@ def main(): final_log.returncode != 0 or result['kernel_faults']): result['result'] = 'WIFI_NETWORK_E2E_FAILED' result['error'] = 'Final kernel log unavailable or contains a driver fault' + info_log = run(['dmesg', '--level=notice,info']) + if started is not None and info_log.returncode == 0: + result['driver_info_messages'] = len(driver_info_messages(info_log.stdout, started)) + if result['result'] in ('WIFI_SCAN_OBSERVED', 'WIFI_PHYSICAL_NETWORK_E2E_PASS') and ( + info_log.returncode != 0 or result.get('driver_info_messages')): + result['result'] = 'WIFI_NETWORK_E2E_FAILED' + result['error'] = 'Radio drivers logged routine events at info level' (out / 'result.json').write_text(json.dumps(result, indent=2) + '\n') print(json.dumps(result, indent=2)) print('ARTIFACT=' + str(out / 'result.json')) diff --git a/tools/testing/selftests/drivers/net/mt7932_e2e_test.py b/tools/testing/selftests/drivers/net/mt7932_e2e_test.py index 22b45b154f365..75d341cad257f 100644 --- a/tools/testing/selftests/drivers/net/mt7932_e2e_test.py +++ b/tools/testing/selftests/drivers/net/mt7932_e2e_test.py @@ -182,5 +182,27 @@ def test_new_fault_markers_are_terminal(self): self.assertIsNotNone(e2e.fault.search(message)) +class DriverLogTest(unittest.TestCase): + LOG = ('[ 10.000001] mt7932-fullmac 0000:01:00.0: probe message\n' + '[ 20.500000] usb 1-1: new high-speed USB device\n' + '[ 21.000000] mt7932-fullmac 0000:01:00.0: SCAN_REQUEST: channels=38\n' + '[ 21.250000] mt7932_bt_pcie 0000:01:00.1: controller recovered\n' + '[ 22.000000] Bluetooth: hci0: unrelated core message\n') + + def test_last_log_time(self): + self.assertEqual(e2e.last_log_time(self.LOG), 22.0) + self.assertEqual(e2e.last_log_time(''), 0.0) + + def test_only_radio_messages_after_start_count(self): + self.assertEqual(len(e2e.driver_info_messages(self.LOG, 0.0)), 3) + self.assertEqual(e2e.driver_info_messages(self.LOG, 20.5), + ['[ 21.000000] mt7932-fullmac 0000:01:00.0: ', + '[ 21.250000] mt7932_bt_pcie 0000:01:00.1: ']) + self.assertEqual(e2e.driver_info_messages(self.LOG, 22.0), []) + + def test_quiet_log_has_no_driver_messages(self): + self.assertEqual(e2e.driver_info_messages('[ 1.000000] usb 1-1: reset\n', 0.0), []) + + if __name__ == '__main__': unittest.main() From d7d5d6b51f5c0a38b9ed5a140db431005593ac6e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:15:16 +1100 Subject: [PATCH 1048/1579] media: apple: isp: Do not report read() buffers in the stream parameters The driver fills readbuffers in G_PARM and S_PARM with the minimum number of queued buffers. That field is the number of buffers used for read(), and the device does not support read(): it does not set V4L2_CAP_READWRITE and its queue only allows MMAP. A non-zero value there makes v4l2-compliance fail the G_PARM and S_PARM tests. G_PARM is cleared by the V4L2 core before the driver fills it, so it only has to leave the field alone. S_PARM passes in what the caller wrote, so set it to zero there. Fixes: 329bf83de183 ("media: apple: Add Apple ISP driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index f3056e78a0a17..70b5911900a56 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -885,7 +885,6 @@ static int isp_vidioc_get_param(struct file *file, void *fh, return -EINVAL; a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; - a->parm.capture.readbuffers = ISP_MIN_FRAMES; a->parm.capture.timeperframe.numerator = 1; a->parm.capture.timeperframe.denominator = isp->frame_rate; @@ -914,7 +913,7 @@ static int isp_vidioc_set_param(struct file *file, void *fh, isp->frame_rate = isp_closest_frame_rate(tpf); a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; - a->parm.capture.readbuffers = ISP_MIN_FRAMES; + a->parm.capture.readbuffers = 0; tpf->numerator = 1; tpf->denominator = isp->frame_rate; From 8b93f5129c1c8255a267c3fb81a1d2c4b643f297 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:15:34 +1100 Subject: [PATCH 1049/1579] media: apple: isp: Refuse format changes while buffers are allocated S_FMT changes the format of the current channel and the type of the vb2 queue at any time, also while buffers are allocated or the stream runs, for example from a second open file. The running stream keeps the preset it was configured with, but buffer preparation and submission then use the new frame size. With the multiplanar API enabled, an S_FMT for the other buffer type also changes the queue type under the allocated buffers. Return -EBUSY from S_FMT while the queue has buffers, as V4L2 specifies and as uvcvideo does. TRY_FMT and G_FMT are unchanged. Fixes: 329bf83de183 ("media: apple: Add Apple ISP driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 70b5911900a56..e817d2146fe12 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -757,6 +757,10 @@ static int isp_vidioc_set_format(struct file *file, void *fh, struct isp_preset *preset; int err; + /* The buffers and the queue type depend on the format. */ + if (vb2_is_busy(&isp->vbq)) + return -EBUSY; + preset = isp_select_preset(isp, f->fmt.pix.width, f->fmt.pix.height); err = isp_set_preset(isp, fmt, preset); if (err) @@ -812,6 +816,9 @@ static int isp_vidioc_set_format_mplane(struct file *file, void *fh, if (!isp->multiplanar) return -ENOTTY; + if (vb2_is_busy(&isp->vbq)) + return -EBUSY; + preset = isp_select_preset(isp, f->fmt.pix_mp.width, f->fmt.pix_mp.height); err = isp_set_preset(isp, fmt, preset); From 1e821230fc92f1faae815bae254ea7efab33836d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 17:41:51 +1100 Subject: [PATCH 1050/1579] media: apple: isp: Count existing buffers against the buffer pool size The firmware's buffer pools are configured for ISP_MAX_BUFFERS buffers, and queue_setup() caps the number of buffers to allocate to that many. The cap only looks at the current request. REQBUFS frees the old buffers before it calls queue_setup(), but CREATE_BUFS adds to them: REQBUFS for 16 buffers followed by CREATE_BUFS gives the queue more buffers than the pool holds, and once they are all queued the driver hands them all to the firmware. Count the buffers the queue already has. CREATE_BUFS on a full queue now fails with -ENOBUFS, and one that asks for more than the remaining space gets what fits. REQBUFS behaves as before. vb2_queue.max_num_buffers cannot express the limit: vb2_queue_init() refuses any value below VB2_MAX_FRAME (32). Fixes: 6f0201787b3a ("media: apple: isp: Limit maximal number of buffers") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index e817d2146fe12..84b6696a3979c 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -292,13 +292,20 @@ static int isp_vb2_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, { struct apple_isp *isp = vb2_get_drv_priv(vq); struct isp_format *fmt = isp_get_current_format(isp); + unsigned int q_num_bufs = vb2_get_num_buffers(vq); /* This is not strictly neccessary but makes it easy to enforce that * at most 16 buffers are submitted at once. ISP on t6001 (FW 12.3) * times out if more buffers are submitted than set in the buffer pool * config before streaming is started. + * + * REQBUFS frees the old buffers first, CREATE_BUFS adds to them, so + * count the buffers the queue already has. */ - *nbuffers = min_t(unsigned int, *nbuffers, ISP_MAX_BUFFERS); + if (q_num_bufs >= ISP_MAX_BUFFERS) + return -ENOBUFS; + *nbuffers = min_t(unsigned int, *nbuffers, + ISP_MAX_BUFFERS - q_num_bufs); if (*num_planes) { if (sizes[0] < fmt->total_size) From d3aee6db98425d1e45711b7b02b8cc768d57cb2f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:16:11 +1100 Subject: [PATCH 1051/1579] media: apple: isp: Use V4L2_CAP_VIDEO_CAPTURE in the device capabilities device_caps is built from V4L2_BUF_TYPE_VIDEO_CAPTURE, a buffer type, instead of the V4L2_CAP_VIDEO_CAPTURE capability flag. Both are 1, so the reported capabilities are right, but only by coincidence. Use the capability flag. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 84b6696a3979c..cea7ae7e6bbe7 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -1105,7 +1105,7 @@ int apple_isp_setup_video(struct apple_isp *isp) vdev->queue = vbq; vdev->fops = &isp_v4l2_fops; vdev->ioctl_ops = &isp_v4l2_ioctl_ops; - vdev->device_caps = V4L2_BUF_TYPE_VIDEO_CAPTURE | V4L2_CAP_STREAMING; + vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING; if (isp->multiplanar) vdev->device_caps |= V4L2_CAP_VIDEO_CAPTURE_MPLANE; vdev->v4l2_dev = &isp->v4l2_dev; From cccd6a170d0bdde4e11c8ede7c42128ebd30d977 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:16:39 +1100 Subject: [PATCH 1052/1579] media: apple: isp: Skip the firmware version check for the H17 interface On T8140 the command layouts come from the match data, and the firmware version properties are never compared against the version table. isp_read_fw_version() still parsed them, then returned "unknown" through a separate H17 branch that printed each property at info level, two lines on every boot that the other SoCs log only at debug level. Return from isp_check_firmware_version() before reading the properties on H17. isp_read_fw_version() loses its interface argument and the H17 branch. fw_compat stays ISP_FIRMWARE_V_UNKNOWN on T8140, as before; the other SoCs are unchanged. The bootloader-provided versions remain readable in the device tree. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 33 ++++++---------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 76c28c0dc9f7c..d11ce3eff54ba 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -309,8 +309,7 @@ static const char * isp_fw2str(enum isp_firmware_version version) #define ISP_FW_VERSION_MAX_LEN 5 static enum isp_firmware_version isp_read_fw_version(struct device *dev, - const char *name, - enum isp_fw_abi abi) + const char *name) { u32 ver[ISP_FW_VERSION_MAX_LEN]; int size, len; @@ -325,24 +324,6 @@ static enum isp_firmware_version isp_read_fw_version(struct device *dev, ISP_FW_VERSION_MIN_LEN, ISP_FW_VERSION_MAX_LEN); - /* H17 command layouts come from match data, not the legacy version table. */ - if (abi == ISP_FW_ABI_H17) { - switch (len) { - case 3: - dev_info(dev, "%s: %u.%u.%u (H17 interface)\n", - name, ver[0], ver[1], ver[2]); - return ISP_FIRMWARE_V_UNKNOWN; - case 4: - dev_info(dev, "%s: %u.%u.%u.%u (H17 interface)\n", - name, ver[0], ver[1], ver[2], ver[3]); - return ISP_FIRMWARE_V_UNKNOWN; - case 5: - dev_info(dev, "%s: %u.%u.%u.%u.%u (H17 interface)\n", - name, ver[0], ver[1], ver[2], ver[3], ver[4]); - return ISP_FIRMWARE_V_UNKNOWN; - } - } - switch (len) { case -EINVAL: /* not provided, see isp_check_firmware_version() */ @@ -381,16 +362,18 @@ isp_check_firmware_version(struct apple_isp *isp) struct device *dev = isp->dev; enum isp_firmware_version version, compat; + /* The H17 interface is selected by match data, not by version. */ + if (isp->hw->fw_abi == ISP_FW_ABI_H17) + return ISP_FIRMWARE_V_UNKNOWN; + /* firmware version is just informative */ - version = isp_read_fw_version(dev, "apple,firmware-version", isp->hw->fw_abi); - compat = isp_read_fw_version(dev, "apple,firmware-compat", isp->hw->fw_abi); + version = isp_read_fw_version(dev, "apple,firmware-version"); + compat = isp_read_fw_version(dev, "apple,firmware-compat"); dev_dbg(dev, "ISP firmware-compat: %s (FW: %s)\n", isp_fw2str(compat), isp_fw2str(version)); - /* The H17 interface does not depend on it. */ - if (isp->hw->fw_abi == ISP_FW_ABI_LEGACY && - !of_property_present(dev->of_node, "apple,firmware-compat")) + if (!of_property_present(dev->of_node, "apple,firmware-compat")) dev_warn(dev, "firmware compatibility version not provided, assuming 12.x\n"); return compat; From 07a3b11d54a4d9a78667e9303792b34fd121d693 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:16:54 +1100 Subject: [PATCH 1053/1579] media: apple: isp: Name the H17 flag in boot stage 2 after the interface isp_firmware_boot_stage2() keeps "is this the H17 interface" in a variable called h16, after the H16 boot descriptor that interface takes. Everywhere else the same test is called h17. Rename it so that the condition reads the same in every function; the descriptor keeps its H16 name. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 46b4513b39df3..5c3026502e0e5 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -502,8 +502,8 @@ static void isp_write_bootargs_h16(struct apple_isp *isp, void *virt, static int isp_firmware_boot_stage2(struct apple_isp *isp) { - bool h16 = isp->hw->fw_abi == ISP_FW_ABI_H17; - size_t args_size = h16 ? sizeof(struct isp_firmware_bootargs_h16) : + bool h17 = isp->hw->fw_abi == ISP_FW_ABI_H17; + size_t args_size = h17 ? sizeof(struct isp_firmware_bootargs_h16) : sizeof(struct isp_firmware_bootargs); size_t cmd_size = ISP_CMD_AREA_SIZE(isp_num_capmeta(isp)); dma_addr_t args_iova, cmd_iova; @@ -526,7 +526,7 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) num_ipc_chans); /* Only hand the H16 descriptor to firmware that asks for it. */ - if (h16 && !(desc_flags & ISP_GPIO2_H16_DESCRIPTOR)) { + if (h17 && !(desc_flags & ISP_GPIO2_H16_DESCRIPTOR)) { dev_err(isp->dev, "firmware did not request the H16 boot descriptor (0x%x)\n", desc_flags); @@ -562,7 +562,7 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) isp->cmd_iova = cmd_iova; isp->cmd_virt = cmd_virt; - if (h16) + if (h17) isp_write_bootargs_h16(isp, args_virt, args_offset); else isp_write_bootargs(isp, args_virt, args_iova); From ae22f06a3f113ffc4dc6d3f274b705a44a7c8be1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:22:56 +1100 Subject: [PATCH 1054/1579] dt-bindings: media: apple,t8140-isp: Add an example The binding has no example, so dt_binding_check never validates a node against it. Add one that follows the T8140 device tree, with a single sensor preset. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/media/apple,t8140-isp.yaml | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml index dea138b415b7a..59c6c91fb19d7 100644 --- a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml +++ b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml @@ -145,3 +145,45 @@ required: - sensor-presets additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + isp@320000000 { + compatible = "apple,t8140-isp", "apple,isp"; + reg = <0x3 0x20000000 0x0 0x1801000>, + <0x3 0x22920000 0x0 0x1000>, + <0x3 0x22921000 0x0 0x100>, + <0x3 0x63b50000 0x0 0x8000>; + reg-names = "coproc", "mbox", "gpio", "wdt"; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, + <&ps_isp_cpu_core0>, <&ps_isp_cpu_core1>, + <&ps_isp_fe>, <&ps_isp_dprx>, <&ps_isp_secure>, + <&ps_isp_sts0>, <&ps_isp_sts1>, <&ps_isp_pearl>, + <&ps_isp_be>, <&ps_isp_clr>, <&ps_isp_cvd>, + <&ps_isp_cvm>; + memory-region = <&isp_heap>; + memory-region-names = "heap"; + apple,dart-vm-size = <0x0 0xa0000000>; + apple,platform-id = <0x70>; + apple,temporal-filter = <1>; + + sensor-presets { + preset0 { + apple,config-index = <1>; + apple,input-size = <1920 1080>; + apple,output-size = <1920 1080>; + apple,crop = <0 0 1920 1080>; + }; + }; + }; + }; From e5d2287029188859511c427cb9b72c221dc1144d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 17:45:34 +1100 Subject: [PATCH 1055/1579] media: apple: isp: Add KUnit tests for the videobuf2 queue Move the vb2 queue set-up out of apple_isp_setup_video() into isp_vb2_queue_init() and test it, together with the buffer limit in queue_setup(), without hardware: - the queue initialises with the parameters the driver probes with, so a parameter that vb2_queue_init() rejects fails here instead of at probe, where it leaves the machine without a camera; - REQBUFS gets at most ISP_MAX_BUFFERS buffers; - CREATE_BUFS counts the buffers the queue already has, fails with -ENOBUFS on a full queue and still refuses planes smaller than a frame. The tests are built into isp-v4l2.c, as they call its static functions, and run with tools/testing/kunit/kunit.py run --arch=arm64 \ --kunitconfig= No functional change to the driver. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/Kconfig | 12 ++ .../media/platform/apple/isp/isp-v4l2-test.c | 122 ++++++++++++++++++ drivers/media/platform/apple/isp/isp-v4l2.c | 35 +++-- 3 files changed, 157 insertions(+), 12 deletions(-) create mode 100644 drivers/media/platform/apple/isp/isp-v4l2-test.c diff --git a/drivers/media/platform/apple/isp/Kconfig b/drivers/media/platform/apple/isp/Kconfig index 5d818e55448a4..cc2c59b51a441 100644 --- a/drivers/media/platform/apple/isp/Kconfig +++ b/drivers/media/platform/apple/isp/Kconfig @@ -17,3 +17,15 @@ config VIDEO_APPLE_ISP calibration file for the detected camera module, apple/isp__XX.dat, from the firmware search path; if it is missing, the stream runs without calibration. + +config VIDEO_APPLE_ISP_KUNIT_TEST + bool "KUnit tests for the Apple ISP driver" if !KUNIT_ALL_TESTS + depends on VIDEO_APPLE_ISP && KUNIT + depends on KUNIT=y || VIDEO_APPLE_ISP=m + default KUNIT_ALL_TESTS + help + Builds unit tests for the videobuf2 queue set-up of the Apple ISP + driver into the driver. They need no hardware. Only useful for + kernel developers running KUnit. + + If unsure, say N. diff --git a/drivers/media/platform/apple/isp/isp-v4l2-test.c b/drivers/media/platform/apple/isp/isp-v4l2-test.c new file mode 100644 index 0000000000000..895a6d1299c22 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-v4l2-test.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the videobuf2 queue of the Apple ISP driver, built into + * isp-v4l2.c. They check the queue parameters vb2 accepts and the buffer + * limit queue_setup() enforces, without any hardware. + */ + +#include +#include +#include + +#define ISP_TEST_FRAME_SIZE 0x1000 + +static struct apple_isp *isp_test_alloc(struct kunit *test) +{ + struct apple_isp *isp; + + isp = kunit_kzalloc(test, sizeof(*isp), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, isp); + + mutex_init(&isp->video_lock); + isp_get_current_format(isp)->total_size = ISP_TEST_FRAME_SIZE; + + return isp; +} + +/* The queue must initialise with the parameters the driver probes with. */ +static void isp_test_queue_init(struct kunit *test) +{ + struct apple_isp *isp = isp_test_alloc(test); + + isp->dev = kunit_device_register(test, "apple-isp-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, isp->dev); + + KUNIT_ASSERT_EQ(test, isp_vb2_queue_init(isp), 0); + KUNIT_EXPECT_GE(test, isp->vbq.max_num_buffers, VB2_MAX_FRAME); + vb2_queue_release(&isp->vbq); +} + +/* Pretend the queue already has @num buffers, as vb2_get_num_buffers() sees. */ +static void isp_test_set_num_buffers(struct kunit *test, struct vb2_queue *q, + unsigned int num) +{ + q->max_num_buffers = VB2_MAX_FRAME; + q->bufs_bitmap = kunit_kzalloc(test, + BITS_TO_LONGS(VB2_MAX_FRAME) * + sizeof(unsigned long), + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, q->bufs_bitmap); + bitmap_set(q->bufs_bitmap, 0, num); +} + +static int isp_test_setup(struct kunit *test, unsigned int existing, + unsigned int *nbuffers, unsigned int *num_planes, + unsigned int sizes[]) +{ + struct apple_isp *isp = isp_test_alloc(test); + + isp->vbq.drv_priv = isp; + isp_test_set_num_buffers(test, &isp->vbq, existing); + + return isp_vb2_queue_setup(&isp->vbq, nbuffers, num_planes, sizes, + NULL); +} + +/* REQBUFS: an empty queue gets at most one pool of buffers. */ +static void isp_test_reqbufs_limit(struct kunit *test) +{ + unsigned int sizes[VB2_MAX_PLANES] = {}; + unsigned int nbuffers = VB2_MAX_FRAME, num_planes = 0; + + KUNIT_ASSERT_EQ(test, isp_test_setup(test, 0, &nbuffers, &num_planes, + sizes), 0); + KUNIT_EXPECT_EQ(test, nbuffers, ISP_MAX_BUFFERS); + KUNIT_EXPECT_EQ(test, num_planes, 1); + KUNIT_EXPECT_EQ(test, sizes[0], ISP_TEST_FRAME_SIZE); + + nbuffers = 4; + num_planes = 0; + KUNIT_ASSERT_EQ(test, isp_test_setup(test, 0, &nbuffers, &num_planes, + sizes), 0); + KUNIT_EXPECT_EQ(test, nbuffers, 4); +} + +/* CREATE_BUFS: the buffers the queue has count against the pool. */ +static void isp_test_create_bufs_limit(struct kunit *test) +{ + unsigned int sizes[VB2_MAX_PLANES] = { ISP_TEST_FRAME_SIZE }; + unsigned int nbuffers, num_planes; + + nbuffers = 10; + num_planes = 1; + KUNIT_ASSERT_EQ(test, isp_test_setup(test, 10, &nbuffers, &num_planes, + sizes), 0); + KUNIT_EXPECT_EQ(test, nbuffers, ISP_MAX_BUFFERS - 10); + + nbuffers = 1; + num_planes = 1; + KUNIT_EXPECT_EQ(test, isp_test_setup(test, ISP_MAX_BUFFERS, &nbuffers, + &num_planes, sizes), -ENOBUFS); + + /* A plane smaller than a frame is still refused. */ + nbuffers = 1; + num_planes = 1; + sizes[0] = ISP_TEST_FRAME_SIZE - 1; + KUNIT_EXPECT_EQ(test, isp_test_setup(test, 0, &nbuffers, &num_planes, + sizes), -EINVAL); +} + +static struct kunit_case isp_v4l2_test_cases[] = { + KUNIT_CASE(isp_test_queue_init), + KUNIT_CASE(isp_test_reqbufs_limit), + KUNIT_CASE(isp_test_create_bufs_limit), + {} +}; + +static struct kunit_suite isp_v4l2_test_suite = { + .name = "apple-isp-v4l2", + .test_cases = isp_v4l2_test_cases, +}; + +kunit_test_suite(isp_v4l2_test_suite); diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index cea7ae7e6bbe7..4cc79ee8a2307 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -1013,6 +1013,24 @@ static void isp_free_meta_surfaces(struct apple_isp *isp) } } +static int isp_vb2_queue_init(struct apple_isp *isp) +{ + struct vb2_queue *vbq = &isp->vbq; + + vbq->drv_priv = isp; + vbq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + vbq->io_modes = VB2_MMAP; + vbq->dev = isp->dev; + vbq->ops = &isp_vb2_ops; + vbq->mem_ops = &vb2_dma_sg_memops; + vbq->buf_struct_size = sizeof(struct isp_buffer); + vbq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + vbq->min_queued_buffers = ISP_MIN_FRAMES; + vbq->lock = &isp->video_lock; + + return vb2_queue_init(vbq); +} + int apple_isp_setup_video(struct apple_isp *isp) { struct video_device *vdev = &isp->vdev; @@ -1085,18 +1103,7 @@ int apple_isp_setup_video(struct apple_isp *isp) goto ctrl_free; } - vbq->drv_priv = isp; - vbq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - vbq->io_modes = VB2_MMAP; - vbq->dev = isp->dev; - vbq->ops = &isp_vb2_ops; - vbq->mem_ops = &vb2_dma_sg_memops; - vbq->buf_struct_size = sizeof(struct isp_buffer); - vbq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; - vbq->min_queued_buffers = ISP_MIN_FRAMES; - vbq->lock = &isp->video_lock; - - err = vb2_queue_init(vbq); + err = isp_vb2_queue_init(isp); if (err) { dev_err(isp->dev, "failed to init vb2 queue: %d\n", err); goto v4l2_unregister; @@ -1156,3 +1163,7 @@ void apple_isp_free_video(struct apple_isp *isp) { isp_free_meta_surfaces(isp); } + +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) +#include "isp-v4l2-test.c" +#endif From 6918e26bce6cfa0725be3422e32ba25b6f198b5d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:38:03 +1100 Subject: [PATCH 1056/1579] nvme-apple: Drop the unused stopped-controller handoff Post-M4 ANS2 firmware can only be adopted while it still runs. The driver accepts two hand-over states: a controller the bootloader left enabled and ready, and a stopped controller that the bootloader vouches for with the "apple,rtkit-handoff" = "nvme-disabled-v1" device tree property. No bootloader produces that property. U-Boot, which the T8140 boot chain uses to load Linux, deletes its I/O queues and hands the controller over enabled and ready, and m1n1 does not publish the property either. The second state is therefore unreachable, and its property is device tree ABI that nothing produces. Remove the property parser and the stopped-controller branch of apple_nvme_can_adopt_rtkit(). The live-controller predicate is unchanged: RUN, BOOT_STATUS, CC.EN and CSTS.RDY must be set and CSTS.CFS clear. A device tree without the property, which is every device tree in use, takes exactly the same paths as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 37 ++++++------------------------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index f6c18221ded3e..67b3bd04f2c1a 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -209,7 +209,6 @@ struct apple_nvme { /* Protects reset_ready against RTKit crash callbacks. */ spinlock_t event_lock; bool reset_ready; - bool handoff_disabled; struct list_head quarantine_node; resource_size_t controller_start; bool quarantined; @@ -336,33 +335,15 @@ static bool apple_nvme_can_adopt_rtkit(struct apple_nvme *anv) return false; cc = readl(anv->mmio_nvme + NVME_REG_CC); csts = readl(anv->mmio_nvme + NVME_REG_CSTS); - if (csts & NVME_CSTS_CFS) - return false; /* - * CPU_RUN/BOOT_STATUS alone survive inactive sessions. A disabled - * controller requires the producer's explicit live-session contract. - * Borrowed buffers still require reserved RAM and protected SART ranges. + * CPU_RUN and BOOT_STATUS alone survive a session that has gone + * inactive. Only a controller that the previous owner left enabled + * and ready identifies a live session. Borrowed buffers still require + * reserved RAM and protected SART ranges. */ - if (anv->handoff_disabled) - return !(cc & NVME_CC_ENABLE) && !(csts & NVME_CSTS_RDY); - return (cc & NVME_CC_ENABLE) && (csts & NVME_CSTS_RDY); -} - -static int apple_nvme_read_handoff(struct apple_nvme *anv) -{ - const char *handoff; - int ret; - - ret = of_property_read_string(anv->dev->of_node, "apple,rtkit-handoff", - &handoff); - if (ret == -EINVAL) - return 0; - if (ret || !anv->hw->needs_ioq_registers || - strcmp(handoff, "nvme-disabled-v1")) - return -EINVAL; - anv->handoff_disabled = true; - return apple_nvme_can_adopt_rtkit(anv) ? 0 : -EIO; + return (cc & NVME_CC_ENABLE) && (csts & NVME_CSTS_RDY) && + !(csts & NVME_CSTS_CFS); } static inline struct apple_nvme *queue_to_apple_nvme(struct apple_nvme_queue *q) @@ -2317,12 +2298,6 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) anv->mmio_nvmmu = anv->mmio_nvme; } - ret = apple_nvme_read_handoff(anv); - if (ret) { - dev_err_probe(dev, ret, "Invalid live firmware handoff\n"); - goto out_detach_genpd; - } - if (anv->hw->has_lsq_nvmmu) { anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB; anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB; From 69d06267663dd755c74866c0101bbe91039a832b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:38:20 +1100 Subject: [PATCH 1057/1579] dt-bindings: nvme: apple,nvme-ans: Drop apple,rtkit-handoff The "apple,rtkit-handoff" property was meant to be added at boot by a bootloader that stops the ANS2 NVMe controller but leaves its firmware running. No bootloader does that: U-Boot hands the T8140 controller over enabled and ready, which the driver recognises from the hardware state alone, and the driver no longer parses the property. Remove the property and the condition that limited it to T8140 before it gains a user. No board device tree source carries it. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/nvme/apple,nvme-ans.yaml | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml index 5449db3b25080..fc8a4afd5add4 100644 --- a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml +++ b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml @@ -85,30 +85,7 @@ properties: The SART address filter is documented in iommu/apple,sart.yaml. - apple,rtkit-handoff: - $ref: /schemas/types.yaml#/definitions/string - const: nvme-disabled-v1 - description: | - Added at boot by a bootloader that ran the ANS2 firmware and hands it - over still running instead of shutting it down. It describes the - state that bootloader left behind, not the hardware, and must not be - written into a board device tree source. Its presence states that the - firmware keeps a negotiated RTKit session, that all NVMe controller - DMA has stopped (CC.EN and CSTS.RDY clear), that the SART entries the - bootloader programmed are kept, and that every buffer shared with the - firmware is reserved in the memory map without no-map. - allOf: - - if: - properties: - compatible: - not: - contains: - const: apple,t8140-nvme-ans2 - then: - properties: - apple,rtkit-handoff: false - - if: properties: compatible: From 21db3b50ec38d9f0c6bf2a060408a2ea03db1d0b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:38:45 +1100 Subject: [PATCH 1058/1579] nvme-apple: Name the NVMMU TCB inline encryption flag The TCB flag that enables inline AES for a command is open-coded as BIT(2) where the command is submitted and where its key material is wiped on completion. Define it next to the DMA direction flags of the same field, and fold the key and DUN lookup into the block that programs them. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 67b3bd04f2c1a..12d88a8d75672 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -116,6 +116,7 @@ struct apple_nvmmu_tcb { #define APPLE_ANS_TCB_DMA_FROM_DEVICE BIT(0) #define APPLE_ANS_TCB_DMA_TO_DEVICE BIT(1) +#define APPLE_ANS_TCB_AES BIT(2) u8 dma_flags; u8 command_id; @@ -566,8 +567,6 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, struct request *req) { struct apple_nvme *anv = queue_to_apple_nvme(q); - const u8 *key = NULL; - u64 dun = 0; u32 tag = nvme_tag_from_cid(cmd->common.command_id); struct apple_nvmmu_tcb *tcb = &q->tcbs[tag]; @@ -585,16 +584,10 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE; if (unlikely(req->crypt_ctx)) { - const struct blk_crypto_key *blk_key = req->crypt_ctx->bc_key; + const u8 *key = req->crypt_ctx->bc_key->bytes; + __le64 dun_le = cpu_to_le64(req->crypt_ctx->bc_dun[0]); - key = blk_key->bytes; - dun = req->crypt_ctx->bc_dun[0]; - } - - if (key) { - __le64 dun_le = cpu_to_le64(dun); - - tcb->dma_flags |= BIT(2); + tcb->dma_flags |= APPLE_ANS_TCB_AES; memcpy(tcb->aes_iv, &dun_le, sizeof(dun_le)); memset(tcb->_aes_unk, 0, sizeof(tcb->_aes_unk)); memcpy(tcb->_aes_unk, key, 4); @@ -862,10 +855,10 @@ static __always_inline void apple_nvme_unmap_rq(struct request *req) tag = nvme_tag_from_cid(iod->cmd.common.command_id); tcb = &iod->q->tcbs[tag]; - if (unlikely(tcb->dma_flags & BIT(2))) { + if (unlikely(tcb->dma_flags & APPLE_ANS_TCB_AES)) { memzero_explicit(tcb->aes_iv, sizeof(tcb->aes_iv)); memzero_explicit(tcb->_aes_unk, sizeof(tcb->_aes_unk)); - tcb->dma_flags &= ~BIT(2); + tcb->dma_flags &= ~APPLE_ANS_TCB_AES; } } From e87b71da6428f91b547d7405a82a038aac2e982d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 21:25:06 +1100 Subject: [PATCH 1059/1579] usb: dwc3: Export dwc3_event_buffers_cleanup() The Apple glue calls dwc3_event_buffers_cleanup() when it stops the core, but the function is not exported. With USB_DWC3=y and USB_DWC3_APPLE=m (or both modular) modpost fails: ERROR: modpost: "dwc3_event_buffers_cleanup" [drivers/usb/dwc3/dwc3-apple.ko] undefined! Export it like the other core helpers the glue drivers use (dwc3_enable_susphy(), dwc3_core_exit()). Fixes: f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index c2bbcbcf3797a..498325fa1001d 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -610,6 +610,7 @@ void dwc3_event_buffers_cleanup(struct dwc3 *dwc) reg = dwc3_readl(dwc, DWC3_GEVNTCOUNT(0)); dwc3_writel(dwc, DWC3_GEVNTCOUNT(0), reg); } +EXPORT_SYMBOL_GPL(dwc3_event_buffers_cleanup); static void dwc3_core_num_eps(struct dwc3 *dwc) { From 7156d4c0a7b65e2389a29931d15c082b41c66845 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:25:25 +1100 Subject: [PATCH 1060/1579] phy: apple: atc: Look up the Type-C target mode before the mux update atcphy_mux_set() takes the USB3 root hub of a port behind a fixed hub before atcphy_mux_set_locked() works out which PHY mode the new Type-C state needs. Move that lookup into atcphy_typec_target_mode() and do it first, under the PHY lock, so that the mux update can look at the target before it decides what else it needs. atcphy_mux_set_locked() now takes the PHY and the target mode. The lookup keeps its error messages and still maps the off state of a fixed-hub port to USB2. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 4bb98a02237eb..313cb14ce9783 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -3603,23 +3603,18 @@ static void atcphy_usb3_pm_put(struct atcphy_usb3_pm *pm) #endif } -static int atcphy_mux_set_locked(struct typec_mux_dev *mux, struct typec_mux_state *state) +/* The PHY mode a Type-C mux state asks for; unsupported states turn the PHY off. */ +static enum atcphy_mode atcphy_typec_target_mode(struct apple_atcphy *atcphy, + const struct typec_mux_state *state) { - struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); enum atcphy_mode target_mode; - int ret; - - guard(mutex)(&atcphy->lock); - dev_dbg(atcphy->dev, "mux mode=%lu svid=%04x current=%u fixed=%u dp=%u ss=%u\n", - state->mode, state->alt ? state->alt->svid : 0, atcphy->mode, - atcphy->fixed_usb2, atcphy->dp_tunables, atcphy->ss_tunables); if (state->mode == TYPEC_STATE_SAFE) { target_mode = APPLE_ATCPHY_MODE_OFF; } else if (state->mode == TYPEC_STATE_USB) { target_mode = APPLE_ATCPHY_MODE_USB3; } else if (!state->alt && state->mode == TYPEC_MODE_USB4) { - struct enter_usb_data *data = state->data; + const struct enter_usb_data *data = state->data; u32 eudo_usb_mode = FIELD_GET(EUDO_USB_MODE_MASK, data->eudo); switch (eudo_usb_mode) { @@ -3673,7 +3668,17 @@ static int atcphy_mux_set_locked(struct typec_mux_dev *mux, struct typec_mux_sta if (atcphy->fixed_usb2 && target_mode == APPLE_ATCPHY_MODE_OFF) target_mode = APPLE_ATCPHY_MODE_USB2; - dev_dbg(atcphy->dev, "mux target=%u\n", target_mode); + return target_mode; +} + +static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) +{ + int ret; + + guard(mutex)(&atcphy->lock); + dev_dbg(atcphy->dev, "mux target=%u current=%u dirty=%u\n", target_mode, + atcphy->mode, atcphy->mode_dirty); + if (atcphy->mode == target_mode && !atcphy->mode_dirty) { /* Retry a failed PIPE setup without disturbing the configured lanes. */ if (atcphy->fixed_usb2 && atcphy->host_active && @@ -3757,12 +3762,20 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta { struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); struct atcphy_usb3_pm pm = {}; + enum atcphy_mode target_mode; int ret; + scoped_guard(mutex, &atcphy->lock) { + dev_dbg(atcphy->dev, "mux mode=%lu svid=%04x current=%u fixed=%u dp=%u ss=%u\n", + state->mode, state->alt ? state->alt->svid : 0, atcphy->mode, + atcphy->fixed_usb2, atcphy->dp_tunables, atcphy->ss_tunables); + target_mode = atcphy_typec_target_mode(atcphy, state); + } + ret = atcphy_usb3_pm_get(atcphy, &pm); if (ret) return ret; - ret = atcphy_mux_set_locked(mux, state); + ret = atcphy_mux_set_locked(atcphy, target_mode); atcphy_usb3_pm_put(&pm); return ret; } From b0dee3d08c1afc2d3e76631935619a8192b86335 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:26:25 +1100 Subject: [PATCH 1061/1579] phy: apple: atc: Don't wait for USB enumeration when the mux keeps its mode On a port behind a fixed hub, atcphy_mux_set() takes the device locks of the USB3 root hub and of the device on its port before it looks at the new Type-C state. hub_wq holds the root hub's lock for a whole hub event, and a SuperSpeed enumeration that keeps failing holds it for its full set of retries, each with 5 s command timeouts. The CD321x driver also calls the mux for DisplayPort HPD and status updates that keep the pin assignment. Such an update touches neither the lanes nor the PIPE, but it still waits for hub_wq, and the HPD event it carries reaches the display only afterwards. On a J700 with a USB-C dock (DisplayPort plus USB3), the display came back 27 s after a cable flip, right after hub_wq gave up on the dock's SuperSpeed hub: 22:06:46 usb 2-1: new SuperSpeed Plus Gen 2x1 USB device number 20 using xhci-hcd 22:06:46 usb 2-1: device descriptor read/8, error -71 22:06:51 xhci-hcd xhci-hcd.0.auto: Timeout while waiting for setup device command ... 22:07:08 usb usb2-port1: unable to enumerate USB device 22:07:08 phy-apple-atc 40b000000.phy: Pipehandler lock not acked. 22:07:11 apple-dcp 416e00000.dcp: dcp_hotplug() connected:1 ... Return early, before any USB reference is taken, when the target mode is the current one, no failed transition is pending and the PIPE already sits on the backend that mode needs. atcphy_mux_set_locked() keeps its own check for the case where the state changes in between. A mode change still takes the root hub, as the PIPE has to be handed over while the controller is quiet. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 313cb14ce9783..c8f27bcd3e809 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -3671,6 +3671,22 @@ static enum atcphy_mode atcphy_typec_target_mode(struct apple_atcphy *atcphy, return target_mode; } +/* + * Whether a mux update leaves both the lanes and the PIPE as they are. That is + * the case when the mode does not change and no earlier PIPE handoff to a + * running controller behind a fixed hub is still outstanding. A DisplayPort + * HPD or status update with an unchanged pin assignment is one of these. + */ +static bool atcphy_mux_is_noop(const struct apple_atcphy *atcphy, enum atcphy_mode target_mode) +{ + if (atcphy->mode != target_mode || atcphy->mode_dirty) + return false; + + return !(atcphy->fixed_usb2 && atcphy->host_active && + (atcphy->pipe_state != atcphy_modes[target_mode].pipehandler_state || + atcphy->pipe_pending)); +} + static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) { int ret; @@ -3770,6 +3786,14 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta state->mode, state->alt ? state->alt->svid : 0, atcphy->mode, atcphy->fixed_usb2, atcphy->dp_tunables, atcphy->ss_tunables); target_mode = atcphy_typec_target_mode(atcphy, state); + /* + * Only a lane or PIPE change needs the USB3 root hub below. + * Taking it means waiting for the hub's ongoing work, which + * includes a whole enumeration with its timeouts, and would + * hold up a DisplayPort update that changes neither. + */ + if (atcphy_mux_is_noop(atcphy, target_mode)) + return 0; } ret = atcphy_usb3_pm_get(atcphy, &pm); From 39bb6d308b70024e3a49097462160b877390ab9c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:26:25 +1100 Subject: [PATCH 1062/1579] phy: apple: atc: Add KUnit tests for the Type-C mux decisions Test how atcphy_typec_target_mode() maps Type-C states to PHY modes, on Type-C ports and on ports behind a fixed hub, and when atcphy_mux_is_noop() lets a mux update skip the lane and PIPE work: a DisplayPort update with the same pin assignment does, while a new mode, a failed transition and a PIPE handoff still owed to a running controller do not. The tests are built into the driver, so that they can reach its static functions. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/Kconfig | 14 ++++ drivers/phy/apple/atc-test.c | 132 +++++++++++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 4 ++ 3 files changed, 150 insertions(+) create mode 100644 drivers/phy/apple/atc-test.c diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig index 26cd09c05497c..e67f51241c29b 100644 --- a/drivers/phy/apple/Kconfig +++ b/drivers/phy/apple/Kconfig @@ -13,6 +13,20 @@ config PHY_APPLE_ATC If M is selected the module will be called 'phy-apple-atc'. +config PHY_APPLE_ATC_KUNIT_TEST + bool "KUnit tests for the Apple Type-C PHY" if !KUNIT_ALL_TESTS + depends on PHY_APPLE_ATC && KUNIT + depends on PHY_APPLE_ATC=m || KUNIT=y + default KUNIT_ALL_TESTS + help + Build the KUnit tests for the Type-C mux decisions of the Apple + Type-C PHY driver into the driver. They check how Type-C states + map to PHY modes and which mux updates may skip the lane and + PIPE work. The tests need no hardware and run when the driver + is loaded. + + If unsure, say N. + config PHY_APPLE_DPTX tristate "Apple DPTX PHY" depends on ARCH_APPLE || COMPILE_TEST diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c new file mode 100644 index 0000000000000..e4452d1ab5218 --- /dev/null +++ b/drivers/phy/apple/atc-test.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * KUnit tests for the Type-C mux decisions of the Apple Type-C PHY driver. + * This file is included from atc.c so that it can test static functions. + */ + +#include +#include + +static struct apple_atcphy *atcphy_test_alloc(struct kunit *test, bool fixed_usb2) +{ + struct apple_atcphy *atcphy; + + atcphy = kunit_kzalloc(test, sizeof(*atcphy), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, atcphy); + atcphy->fixed_usb2 = fixed_usb2; + atcphy->mode = APPLE_ATCPHY_MODE_USB3_DP; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; + atcphy->host_active = fixed_usb2; + + return atcphy; +} + +/* The mode lookup reports unsupported states against the PHY device */ +static struct apple_atcphy *atcphy_test_alloc_dev(struct kunit *test, bool fixed_usb2) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc(test, fixed_usb2); + + atcphy->dev = kunit_device_register(test, "atcphy-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, atcphy->dev); + + return atcphy; +} + +static void atcphy_test_target_mode(struct kunit *test) +{ + struct typec_altmode dp = { .svid = USB_TYPEC_DP_SID }; + struct typec_mux_state state = {}; + struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, false); + + state.mode = TYPEC_STATE_SAFE; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), APPLE_ATCPHY_MODE_OFF); + + state.mode = TYPEC_STATE_USB; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), APPLE_ATCPHY_MODE_USB3); + + state.alt = &dp; + state.mode = TYPEC_DP_STATE_C; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), APPLE_ATCPHY_MODE_DP); + state.mode = TYPEC_DP_STATE_D; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), + APPLE_ATCPHY_MODE_USB3_DP); + state.mode = TYPEC_DP_STATE_E; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), APPLE_ATCPHY_MODE_DP); + state.mode = TYPEC_DP_STATE_A; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), APPLE_ATCPHY_MODE_OFF); +} + +/* A port behind a fixed hub keeps USB2 for its hub whenever a mode turns the PHY off */ +static void atcphy_test_target_mode_fixed_hub(struct kunit *test) +{ + struct typec_altmode dp = { .svid = USB_TYPEC_DP_SID }; + struct typec_mux_state state = {}; + struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, true); + + state.mode = TYPEC_STATE_SAFE; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), APPLE_ATCPHY_MODE_USB2); + + state.alt = &dp; + state.mode = TYPEC_DP_STATE_A; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), APPLE_ATCPHY_MODE_USB2); + state.mode = TYPEC_DP_STATE_D; + KUNIT_EXPECT_EQ(test, atcphy_typec_target_mode(atcphy, &state), + APPLE_ATCPHY_MODE_USB3_DP); +} + +static void atcphy_test_noop_same_mode(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc(test, true); + + /* A DisplayPort HPD or status update keeps the pin assignment */ + KUNIT_EXPECT_TRUE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); + + /* The same holds while the controller does not use the USB3 PHY */ + atcphy->host_active = false; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + KUNIT_EXPECT_TRUE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); + + /* Ports without a fixed hub never route the PIPE from the mux */ + atcphy = atcphy_test_alloc(test, false); + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + atcphy->pipe_pending = true; + KUNIT_EXPECT_TRUE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); +} + +static void atcphy_test_noop_needs_work(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc(test, true); + + /* A new pin assignment or the safe state rebuild the lanes */ + KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_DP)); + KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB2)); + + /* So does a failed transition or an orientation change */ + atcphy->mode_dirty = true; + KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); + atcphy->mode_dirty = false; + + /* A running controller is still owed its USB3 backend */ + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); + + /* or a handoff that has not completed */ + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; + atcphy->pipe_pending = true; + KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); +} + +static struct kunit_case atcphy_mux_test_cases[] = { + KUNIT_CASE(atcphy_test_target_mode), + KUNIT_CASE(atcphy_test_target_mode_fixed_hub), + KUNIT_CASE(atcphy_test_noop_same_mode), + KUNIT_CASE(atcphy_test_noop_needs_work), + {} +}; + +static struct kunit_suite atcphy_mux_test_suite = { + .name = "phy-apple-atc-mux", + .test_cases = atcphy_mux_test_cases, +}; + +kunit_test_suite(atcphy_mux_test_suite); diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index c8f27bcd3e809..a1b4a691e57e6 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -4165,3 +4165,7 @@ module_platform_driver(atcphy_driver); MODULE_AUTHOR("Sven Peter "); MODULE_DESCRIPTION("Apple Type-C PHY driver"); MODULE_LICENSE("GPL"); + +#if IS_ENABLED(CONFIG_PHY_APPLE_ATC_KUNIT_TEST) +#include "atc-test.c" +#endif From 27da3a5d53ca11f2aeff4916d25f40308c378a07 Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Tue, 13 Oct 2026 14:20:26 +0200 Subject: [PATCH 1063/1579] arm64: dts: apple: t8140-j700: give the boot framebuffer its panel simpledrm reads the panel's physical size through the framebuffer's panel phandle. Without it the boot display reports 96 dpi, and compositors pick a scale of 1 for the 219 ppi panel. Link it as the other Apple laptops do. Co-Authored-By: Claude Opus 5.5 --- arch/arm64/boot/dts/apple/t8140-j700.dts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 942e360f57c4e..affdd0149be64 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -52,6 +52,8 @@ power-domains = <&ps_disp_cpu>; /* Format properties will be added by loader */ status = "disabled"; + panel = <&panel>; + post-init-providers = <&panel>; }; }; From 64ef3b3a84f0a3faf5d4c5f333cb52284f5075fa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:37:41 +1100 Subject: [PATCH 1064/1579] phy: apple: atc: Split the mux and switch callbacks from their PHY work Move the bodies of atcphy_mux_set() and atcphy_sw_set() into atcphy_mux_update() and atcphy_set_orientation(), which take the PHY rather than the Type-C mux or switch device. The callbacks only look up the PHY and call them. This lets the driver's KUnit tests run a whole mux update and an orientation change on a PHY without registering Type-C devices. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index a1b4a691e57e6..e8f3b64b2a349 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -3350,9 +3350,8 @@ static int atcphy_probe_rcdev(struct apple_atcphy *atcphy) return devm_reset_controller_register(atcphy->dev, &atcphy->rcdev); } -static int atcphy_sw_set(struct typec_switch_dev *sw, enum typec_orientation orientation) +static int atcphy_set_orientation(struct apple_atcphy *atcphy, enum typec_orientation orientation) { - struct apple_atcphy *atcphy = typec_switch_get_drvdata(sw); bool swap_lanes; guard(mutex)(&atcphy->lock); @@ -3381,6 +3380,11 @@ static int atcphy_sw_set(struct typec_switch_dev *sw, enum typec_orientation ori return 0; } +static int atcphy_sw_set(struct typec_switch_dev *sw, enum typec_orientation orientation) +{ + return atcphy_set_orientation(typec_switch_get_drvdata(sw), orientation); +} + static void atcphy_typec_switch_unregister(void *data) { typec_switch_unregister(data); @@ -3774,9 +3778,8 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t return atcphy_configure(atcphy, target_mode); } -static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux_state *state) { - struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); struct atcphy_usb3_pm pm = {}; enum atcphy_mode target_mode; int ret; @@ -3804,6 +3807,11 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta return ret; } +static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + return atcphy_mux_update(typec_mux_get_drvdata(mux), state); +} + static void atcphy_typec_mux_unregister(void *data) { typec_mux_unregister(data); From 0aeb0acde3e1962a1d0e617d8d06da7741bfe78e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:38:03 +1100 Subject: [PATCH 1065/1579] phy: apple: atc: Test that a mux update keeping its mode skips the USB3 root hub The existing tests check atcphy_mux_is_noop() on its own, so they still pass if atcphy_mux_update() stops returning early and takes the USB3 root hub again for every DisplayPort HPD or status update. Run whole mux updates on a PHY behind a fixed hub that has no device tree node. Taking the root hub then fails with -ENODEV at the controller lookup, before any register access. An update with the same pin assignment and orientation must return 0. A new pin assignment, a pending PIPE handoff, a PIPE away from the USB3 backend of a running host, and a cable flip reported through atcphy_set_orientation() must all reach the root hub. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 68 ++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index e4452d1ab5218..a8c0c57d09859 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -116,11 +116,79 @@ static void atcphy_test_noop_needs_work(struct kunit *test) KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); } +/* + * A J700-style PHY behind a fixed hub, in DisplayPort + USB3 with the PIPE on + * USB3 for a running host, but without a device tree node. A mux update that + * goes for the USB3 root hub then fails with -ENODEV at the controller lookup, + * before any register access, which tells the two paths apart. + */ +static struct apple_atcphy *atcphy_test_alloc_fixed_hub(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, true); + + atcphy->hw = &atcphy_hw_t8140; + mutex_init(&atcphy->lock); + + return atcphy; +} + +static const struct typec_altmode atcphy_test_dp = { .svid = USB_TYPEC_DP_SID }; + +static int atcphy_test_update_dp(struct apple_atcphy *atcphy, unsigned long pin) +{ + struct typec_mux_state state = { + .alt = (struct typec_altmode *)&atcphy_test_dp, + .mode = pin, + }; + + return atcphy_mux_update(atcphy, &state); +} + +static void atcphy_test_update_skips_usb(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + + /* An HPD or status update with the same pin assignment */ + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB3_DP); + KUNIT_EXPECT_FALSE(test, atcphy->mode_dirty); + + /* The orientation the lanes already use */ + KUNIT_EXPECT_EQ(test, atcphy_set_orientation(atcphy, TYPEC_ORIENTATION_NORMAL), 0); + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); +} + +static void atcphy_test_update_takes_usb(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + + /* A new pin assignment */ + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -ENODEV); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB3_DP); + + /* A PIPE handoff that has not completed */ + atcphy->pipe_pending = true; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -ENODEV); + atcphy->pipe_pending = false; + + /* A running host whose PIPE is not on the USB3 backend */ + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -ENODEV); + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; + + /* A cable flip moves the USB3 and DisplayPort lanes */ + KUNIT_EXPECT_EQ(test, atcphy_set_orientation(atcphy, TYPEC_ORIENTATION_REVERSE), 0); + KUNIT_EXPECT_TRUE(test, atcphy->mode_dirty); + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -ENODEV); +} + static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_target_mode), KUNIT_CASE(atcphy_test_target_mode_fixed_hub), KUNIT_CASE(atcphy_test_noop_same_mode), KUNIT_CASE(atcphy_test_noop_needs_work), + KUNIT_CASE(atcphy_test_update_skips_usb), + KUNIT_CASE(atcphy_test_update_takes_usb), {} }; From 5680427fe7f9aacc8f32146855845ae10650356b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:21:43 +1100 Subject: [PATCH 1066/1579] drm/apple: remove duplicate AFK service test registration When DRM_APPLE_KUNIT_TEST=m, afk-service-test.o is added to modules.order both by drivers/gpu/drm/Makefile and by the Apple driver Makefile. The duplicate entries cause modules_check to fail with a module name conflict. Keep the registration in drivers/gpu/drm/apple/Makefile next to the driver and remove the redundant top-level entry. Fixes: 18d69dc80648 ("drm/apple: close external admission and recovery races") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index 1ee47178266e1..a0c684efd523d 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -229,7 +229,6 @@ obj-$(CONFIG_DRM_PANTHOR) += panthor/ obj-$(CONFIG_DRM_TYR) += tyr/ obj-$(CONFIG_DRM_APPLE) += apple/ obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple/iomfb-state-test.o -obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple/afk-service-test.o obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/ obj-$(CONFIG_DRM_MCDE) += mcde/ obj-$(CONFIG_DRM_TIDSS) += tidss/ From 43363f1eb7ac812adbbfa56ee4c5ee0198a90306 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:24:02 +1100 Subject: [PATCH 1067/1579] wifi: mt7932: drop received data the hardware did not decrypt On a WPA2 connection the driver hands every header-translated data frame from the associated peer to the network stack. Nothing checks that the hardware decrypted it: a frame that arrived in plaintext, or that was decrypted with a cipher or key length other than the one the key expects, is delivered like any other. The receive descriptor reports both: word1 carries the cipher the frame was decrypted with (zero when it was not) and the cipher and cipher-length mismatch flags, in the same layout as the connac2 devices that mt76 handles. Only a cipher mismatch on a translated frame was rejected; the cipher-length mismatch flag was ignored. Keep the cipher and both mismatch flags in the parsed frame and, on a protected connection, deliver only frames that were decrypted without a mismatch. The firmware runs the WPA2 handshake itself, so the host never needs to accept an unprotected EAPOL frame. An open connection is unchanged. Add KUnit tests with synthetic descriptors: decrypted, plaintext, cipher-length and cipher mismatch, integrity errors, every truncation of the descriptor and its Ethernet header, and a length beyond the buffer. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/Kconfig | 10 ++ drivers/net/wireless/mediatek/mt7932/Makefile | 1 + drivers/net/wireless/mediatek/mt7932/data.c | 7 + .../net/wireless/mediatek/mt7932/input-test.c | 133 ++++++++++++++++++ drivers/net/wireless/mediatek/mt7932/packet.h | 29 +++- 5 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 drivers/net/wireless/mediatek/mt7932/input-test.c diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index c6120372d1f3b..f09fb1f32012b 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -31,3 +31,13 @@ config MT7932_RF_KUNIT_TEST admission paths with synthetic devices, without radio hardware. Check association wakeups, retained peer ownership and refusal to reopen RF admission after a terminal calibration error. + +config MT7932_KUNIT_TEST + bool "KUnit tests for MT7932 firmware input validation" if !KUNIT_ALL_TESTS + depends on KUNIT=y && MT7932_FULLMAC + default KUNIT_ALL_TESTS + help + Exercise the validation of what the firmware and the device hand + to the driver, with synthetic, truncated and corrupted input and + without radio hardware: receive descriptors, including frames the + hardware did not decrypt, and command replies. diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index f854d21e5db81..97b1b760372c1 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o +mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += input-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 99ae52977b582..ba06db134414e 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -234,6 +234,13 @@ void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) m->netdev->stats.rx_dropped++; goto unlock; } + if (!mt7932_rx_protection_ok(frame, m->connect_open)) { + m->netdev->stats.rx_dropped++; + dev_dbg_ratelimited(&m->pdev->dev, + "RX_UNPROTECTED_DROPPED: cipher=%u mismatch=%u\n", + frame->sec_mode, frame->cipher_mismatch); + goto unlock; + } skb = netdev_alloc_skb_ip_align(m->netdev, frame->length); if (!skb) { m->netdev->stats.rx_dropped++; diff --git a/drivers/net/wireless/mediatek/mt7932/input-test.c b/drivers/net/wireless/mediatek/mt7932/input-test.c new file mode 100644 index 0000000000000..409103ed18740 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/input-test.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the validation of firmware input: RX descriptors and + * command replies. The firmware and the device are treated as untrusted. + */ +#include + +#include "mt7932.h" + +#define RX_HEADER 24 +#define RX_ETHERNET 14 +#define RX_PAYLOAD 40 +#define RX_SIZE (RX_HEADER + RX_ETHERNET + RX_PAYLOAD) + +/* A translated (Ethernet) unicast data frame on channel 36 for WTBL 5. */ +static void rx_build(u8 *packet, u32 extra_w1) +{ + memset(packet, 0, RX_SIZE); + put_unaligned_le32(RX_SIZE | (2U << 27), packet); + put_unaligned_le32(5 | extra_w1, packet + 4); + put_unaligned_le32(BIT(13), packet + 8); + put_unaligned_le32((36U << 8) | (1U << 16), packet + 12); + memset(packet + RX_HEADER, 0xa5, RX_ETHERNET + RX_PAYLOAD); + put_unaligned_be16(ETH_P_IP, packet + RX_HEADER + 12); +} + +static u32 rx_cipher(unsigned int mode) +{ + return mode << MT7932_RXD1_SEC_MODE_SHIFT; +} + +static void mt7932_rx_decrypted_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE]; + + rx_build(packet, rx_cipher(4)); + KUNIT_ASSERT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), 0); + KUNIT_EXPECT_TRUE(test, frame.translated); + KUNIT_EXPECT_EQ(test, frame.sec_mode, 4); + KUNIT_EXPECT_EQ(test, frame.wtbl, 5); + KUNIT_EXPECT_EQ(test, frame.length, (size_t)(RX_ETHERNET + RX_PAYLOAD)); + KUNIT_EXPECT_TRUE(test, mt7932_rx_protection_ok(&frame, false)); + KUNIT_EXPECT_TRUE(test, mt7932_rx_protection_ok(&frame, true)); +} + +/* A plaintext frame on a protected connection is never delivered. */ +static void mt7932_rx_plaintext_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE]; + + rx_build(packet, 0); + KUNIT_ASSERT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), 0); + KUNIT_EXPECT_EQ(test, frame.sec_mode, 0); + KUNIT_EXPECT_FALSE(test, mt7932_rx_protection_ok(&frame, false)); + /* An open connection has nothing to decrypt. */ + KUNIT_EXPECT_TRUE(test, mt7932_rx_protection_ok(&frame, true)); +} + +/* Decrypted with the wrong cipher or key length: refused on WPA2. */ +static void mt7932_rx_mismatch_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE]; + + rx_build(packet, rx_cipher(4) | MT7932_RXD1_CIPHER_LEN_MISMATCH); + KUNIT_ASSERT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), 0); + KUNIT_EXPECT_TRUE(test, frame.cipher_mismatch); + KUNIT_EXPECT_FALSE(test, mt7932_rx_protection_ok(&frame, false)); + /* A translated frame with a cipher mismatch is rejected by the parser. */ + rx_build(packet, rx_cipher(4) | MT7932_RXD1_CIPHER_MISMATCH); + KUNIT_EXPECT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), -EBADMSG); +} + +/* ICV, TKIP MIC and FCS errors are rejected before any field is used. */ +static void mt7932_rx_integrity_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE]; + unsigned int bit; + + for (bit = 25; bit <= 27; bit++) { + rx_build(packet, rx_cipher(4) | BIT(bit)); + KUNIT_EXPECT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), + -EBADMSG); + } +} + +/* Short buffers and length fields beyond the buffer never parse. */ +static void mt7932_rx_short_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE], *copy; + size_t cut; + + rx_build(packet, rx_cipher(4)); + for (cut = 0; cut < RX_HEADER + RX_ETHERNET; cut++) { + /* An exact-size copy lets KASAN catch any read past the end. */ + copy = kmemdup(packet, cut, GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, copy || !cut); + if (cut >= 4) + put_unaligned_le32(cut | (2U << 27), copy); + KUNIT_EXPECT_LT_MSG(test, mt7932_rx_frame(copy, cut, &frame), 0, + "length %zu", cut); + kfree(copy); + } + /* The descriptor claims more than the DMA buffer holds. */ + copy = kmemdup(packet, RX_SIZE - 1, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, copy); + KUNIT_EXPECT_EQ(test, mt7932_rx_frame(copy, RX_SIZE - 1, &frame), -EMSGSIZE); + kfree(copy); + /* Optional groups (3 and 5) that do not fit. */ + rx_build(packet, rx_cipher(4) | BIT(13) | BIT(15)); + put_unaligned_le32((RX_HEADER + 20) | (2U << 27), packet); + KUNIT_EXPECT_EQ(test, mt7932_rx_frame(packet, RX_HEADER + 20, &frame), -EMSGSIZE); +} + +static struct kunit_case mt7932_input_test_cases[] = { + KUNIT_CASE(mt7932_rx_decrypted_test), + KUNIT_CASE(mt7932_rx_plaintext_test), + KUNIT_CASE(mt7932_rx_mismatch_test), + KUNIT_CASE(mt7932_rx_integrity_test), + KUNIT_CASE(mt7932_rx_short_test), + {} +}; + +static struct kunit_suite mt7932_input_test_suite = { + .name = "mt7932-input", + .test_cases = mt7932_input_test_cases, +}; + +kunit_test_suite(mt7932_input_test_suite); diff --git a/drivers/net/wireless/mediatek/mt7932/packet.h b/drivers/net/wireless/mediatek/mt7932/packet.h index a93d04d91fd41..d7480c81872b2 100644 --- a/drivers/net/wireless/mediatek/mt7932/packet.h +++ b/drivers/net/wireless/mediatek/mt7932/packet.h @@ -2,6 +2,8 @@ #ifndef MT7932_PACKET_H #define MT7932_PACKET_H +#include + #include "protocol.h" #include "channels.h" @@ -10,10 +12,20 @@ struct mt7932_rx_frame { size_t length; u32 timestamp, rcpi, rxv; u16 frame_control, sequence_control, qos_control, wtbl; - u8 channel, amsdu; + u8 channel, amsdu, sec_mode; bool translated, group4, signal_valid, rate_valid, unicast; + bool cipher_mismatch; }; +/* RX descriptor word1: the cipher the hardware decrypted the frame with (0 + * when it did not decrypt it), and the cipher and cipher-length mismatch + * flags. The layout is the connac2 one. + */ +#define MT7932_RXD1_SEC_MODE_SHIFT 16 +#define MT7932_RXD1_SEC_MODE_MASK 0x1f +#define MT7932_RXD1_CIPHER_MISMATCH BIT(23) +#define MT7932_RXD1_CIPHER_LEN_MISMATCH BIT(24) + /* Strict non-monitor subset of the RX descriptor; its groups are ordered * 4, 1, 2, 3, 5, not by numerical group identifier. */ @@ -54,6 +66,9 @@ static inline int mt7932_rx_frame(const u8 *packet, size_t size, struct mt7932_r out->wtbl = w1 & 0x3ff; out->unicast = ((w3 >> 16) & 3) == 1; out->amsdu = w4 & 3; + out->sec_mode = (w1 >> MT7932_RXD1_SEC_MODE_SHIFT) & MT7932_RXD1_SEC_MODE_MASK; + out->cipher_mismatch = w1 & (MT7932_RXD1_CIPHER_MISMATCH | + MT7932_RXD1_CIPHER_LEN_MISMATCH); if (!mt7932_channel_band(out->channel)) return -EOPNOTSUPP; if ((out->translated && (w2 & (1U << 27))) || (!out->translated && out->amsdu)) @@ -103,6 +118,18 @@ static inline int mt7932_rx_frame(const u8 *packet, size_t size, struct mt7932_r return 0; } +/* + * On a protected connection, deliver only frames the hardware decrypted with + * the cipher and key length it expected. The firmware runs the WPA2 handshake + * itself, so the host has no unprotected EAPOL to accept. Nothing is + * decrypted on an open connection. + */ +static inline bool mt7932_rx_protection_ok(const struct mt7932_rx_frame *frame, + bool open) +{ + return open || (frame->sec_mode && !frame->cipher_mismatch); +} + static inline int mt7932_rx_signal(const struct mt7932_rx_frame *frame, u8 antenna_mask) { int signal = -128; From 43731176c824f8f00ad227cf7769ae9e7264a09a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:11:50 +1100 Subject: [PATCH 1068/1579] soc: apple: sep: Name the failed step of a Touch ID sensor registration When the enclave did not register the sensor, the log said so and nothing else. Eight steps can fail on the way: an enclave refusal of REGISTER_SENSOR, REGISTER_SENSOR_SERIAL, the session share or its commit, and a failed serial read, nonce read or session exchange on the sensor side. Most of them returned false without a word, so a failure could not be told apart from any other. Name the step that failed with the enclave status or the error, and how long after the sensor was powered on. Then report what the sensor shows afterwards: its state, the patch acknowledgment, whether it presents an identifier and how long a frame it holds. The sensor data itself is never logged. The sensor_trace parameter also logs each step as it completes, with its time since power-on. The host test that runs wake_sensor() gets the new fields in its mock. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 137 ++++++++++++++++----- drivers/soc/apple/sensor.rs | 8 ++ drivers/soc/apple/sep.rs | 9 ++ tools/aurora-sep/tests/test_sensor_wake.py | 15 ++- 4 files changed, 134 insertions(+), 35 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index ba21814a32a50..d60cec976fa26 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1701,6 +1701,64 @@ impl SepData { } } + fn sensor_ms(&self) -> u64 { + shim::boottime_ns().saturating_sub(self.sensor_power_ns.load(Relaxed)) / 1_000_000 + } + + fn sensor_trace(&self, step: &CStr) { + if *module_parameters::sensor_trace.value() != 0 { + dev_info!(self.dev, "sensor: {} done at {} ms\n", step, self.sensor_ms()); + } + } + + /// Reports a failed registration step with what the sensor shows + /// afterwards: its state, patch acknowledgment, whether it presents an + /// identifier and the length of any frame it holds. Never the + /// identifier, the serial or any key material. + fn sensor_step_failed(&self, step: &CStr) { + let ms = self.sensor_ms(); + match sensor::status() { + Ok(st) => dev_warn!( + self.dev, + "sensor: {} failed {} ms after power-on; sensor now state {} ack {:#04x} identifier {} holding {} bytes{}\n", + step, + ms, + st.state, + st.patch_ack(), + st.identifier_to_register().is_some(), + st.available().unwrap_or(0), + if st.is_silent() { c" (silent)" } else { c"" } + ), + Err(e) => dev_warn!( + self.dev, + "sensor: {} failed {} ms after power-on; its status read failed too ({:?})\n", + step, + ms, + e + ), + } + } + + /// One enclave request of the registration. A refusal is named. + fn sensor_enclave_step(&self, op: &crate::sbio::SbioOp) -> Option> { + match self.sbio_transfer(op) { + Ok(done) if done.status.is_ok() => { + self.sensor_trace(op.name()); + Some(done.payload) + } + Ok(done) => { + dev_warn!(self.dev, "sensor: {} refused with status {}\n", op.name(), done.status); + self.sensor_step_failed(op.name()); + None + } + Err(e) => { + dev_warn!(self.dev, "sensor: {} transfer failed: {:?}\n", op.name(), e); + self.sensor_step_failed(op.name()); + None + } + } + } + fn register_sensor(&self, id: &sensor::Identifier) -> bool { let stage = self.bringup.load(Relaxed); dev_info!(self.dev, "sbio: registering sensor at bring-up stage {}\n", stage); @@ -1714,9 +1772,12 @@ impl SepData { return false; } }; - if !self.sensor_session_request(0x48, c"PLATFORM_IDENTITY", &platform) - || self.sbio_expect_ok(&crate::sbio::sbio_register_sensor(id)).is_none() - { + if !self.sensor_session_request(0x48, c"PLATFORM_IDENTITY", &platform) { + self.sensor_step_failed(c"PLATFORM_IDENTITY"); + return false; + } + self.sensor_trace(c"PLATFORM_IDENTITY"); + if self.sensor_enclave_step(&crate::sbio::sbio_register_sensor(id)).is_none() { return false; } } @@ -1724,7 +1785,7 @@ impl SepData { if stage >= BRINGUP_ESTABLISHED { let _ = self.log_identity_count(c"before CLEAR_STATE for sensor re-registration"); if self - .sbio_expect_ok(&crate::sbio::sbio_clear_state()) + .sensor_enclave_step(&crate::sbio::sbio_clear_state()) .is_none() { return false; @@ -1752,23 +1813,24 @@ impl SepData { let serial = match sensor::read_sensor_serial() { Ok(serial) => serial, - Err(_) => { + Err(e) => { + dev_warn!(self.dev, "sensor: serial read failed: {:?}\n", e); + self.sensor_step_failed(c"serial read"); return false; } }; + self.sensor_trace(c"serial read"); let op = crate::sbio::sbio_register_sensor_serial(&serial); - match self.sbio_call(&op) { - SbioOutcome::Ok(_payload) => { - if self.profile.nonce_session && !self.authenticate_sensor_session() { - return false; - } - self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); - let _ = self.log_identity_count(c"after sensor serial registration"); - true - } - _ => false, + if self.sensor_enclave_step(&op).is_none() { + return false; + } + if self.profile.nonce_session && !self.authenticate_sensor_session() { + return false; } + self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); + let _ = self.log_identity_count(c"after sensor serial registration"); + true } fn sensor_platform_identity() -> Result<[u8; 12]> { @@ -1802,12 +1864,16 @@ impl SepData { Ok(nonce) => nonce, Err(error) => { dev_err!(self.dev, "sensor: nonce read failed: {:?}\n", error); + self.sensor_step_failed(c"nonce read"); return false; } }; + self.sensor_trace(c"nonce read"); if !self.sensor_session_request(0x81, c"SENSOR_NONCE", &nonce) { + self.sensor_step_failed(c"SENSOR_NONCE"); return false; } + self.sensor_trace(c"SENSOR_NONCE"); if !self.sensor_key_loaded.load(Relaxed) { let firmware = match kernel::firmware::Firmware::request( @@ -1826,9 +1892,11 @@ impl SepData { if firmware.data().is_empty() || !self.sensor_session_request(0x86, c"SENSOR_ROOT_KEY", firmware.data()) { + self.sensor_step_failed(c"SENSOR_ROOT_KEY"); return false; } self.sensor_key_loaded.store(true, Relaxed); + self.sensor_trace(c"SENSOR_ROOT_KEY"); } // Each identification starts a new sensor-side exchange, even though // the signed key remains loaded for the lifetime of the enclave. @@ -1837,6 +1905,7 @@ impl SepData { fn wake_sensor(&self) -> Option { let _source = sensor::power_source(); + self.sensor_power_ns.store(shim::boottime_ns(), Relaxed); let cycled = sensor::power_cycle(); if !sensor::cs_timing().is_hardware() { dev_warn!( @@ -1877,6 +1946,13 @@ impl SepData { answered_without_identifier = true; continue; }; + if *module_parameters::sensor_trace.value() != 0 { + dev_info!( + self.dev, + "sensor: identifier present {} ms after power-on\n", + self.sensor_ms() + ); + } if !self.register_sensor(&id) { dev_warn!(self.dev, "sensor: the enclave did not register the sensor\n"); return None; @@ -2090,43 +2166,37 @@ impl SepData { } fn establish_session(&self) -> bool { - let share = match self.sbio_call(&crate::sbio::sbio_request_session_share()) { - SbioOutcome::Ok(sh) => sh, - SbioOutcome::Status16 => { - return false; - } - SbioOutcome::PrerequisiteMissing => { - return false; - } - SbioOutcome::Other => { - return false; - } + let Some(share) = self.sensor_enclave_step(&crate::sbio::sbio_request_session_share()) + else { + return false; }; // 0x30 reserved for this reply; length is a floor, take the first 40 if share.len() < sensor::SESSION_SHARE_LEN { + dev_warn!(self.dev, "sensor: short session share ({} bytes)\n", share.len()); return false; } let mut out = [0u8; sensor::SESSION_SHARE_LEN]; out.copy_from_slice(&share[..sensor::SESSION_SHARE_LEN]); // relay it: class 0x72, no CRC, no padding - if sensor::send_session_share(&out).is_err() { + if let Err(e) = sensor::send_session_share(&out) { + dev_warn!(self.dev, "sensor: sending the session share failed: {:?}\n", e); + self.sensor_step_failed(c"session share send"); return false; } let reply = match sensor::read_session_reply() { Ok(r) => r, - Err(_) => { + Err(e) => { + dev_warn!(self.dev, "sensor: session reply read failed: {:?}\n", e); + self.sensor_step_failed(c"session reply read"); return false; } }; - match self.sbio_transfer(&crate::sbio::sbio_commit_session_share(&reply)) { - Ok(done) if done.status.is_ok() => true, - Ok(_) => false, - Err(_) => false, - } + self.sensor_enclave_step(&crate::sbio::sbio_commit_session_share(&reply)) + .is_some() } // Each failure names its step: a report that only says "sequence counter @@ -2180,6 +2250,7 @@ impl SepData { Ok(done) if done.status.is_ok() => true, Ok(done) => { dev_warn!(self.dev, "sequence counter: the enclave refused the sensor's reply, status {}\n", done.status); + self.sensor_step_failed(c"challenge commit"); false } Err(e) => { diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 3fcba35e39428..dc3df5c6b1d34 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -635,6 +635,14 @@ impl Identifier { } impl Status { + /// The length of the frame the sensor offers, if it offers one. + pub(crate) fn available(&self) -> Option { + match self.offset12() { + Offset12::Available(n) => Some(n), + _ => None, + } + } + pub(crate) fn offset12(&self) -> Offset12 { match self.state { STATE_IDLE => Offset12::Identifier(u16::from_le_bytes([ diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 0c9bf75a1a843..fd961cfc4d7ee 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -627,6 +627,10 @@ struct SepData { last_capture_end_ns: Atomic, + // Boot-time clock at the start of the last sensor power cycle, for + // reporting how far into a bring-up a step failed. + sensor_power_ns: Atomic, + device_view_synced: Atomic, keybag_designated: Atomic, @@ -851,6 +855,7 @@ impl SepData { enrol_material <- new_mutex!(None), enrol_identity_candidates <- new_mutex!(KVec::new()), last_capture_end_ns: Atomic::new(0), + sensor_power_ns: Atomic::new(0), device_view_synced: Atomic::new(false), phase: Atomic::new(PHASE_ATTACH), ool_xarm <- new_mutex!(Some(ool_xarm)), @@ -2471,6 +2476,10 @@ module! { default: 0, description: "Low 64 bits of an explicit xART OS UUID", }, + sensor_trace: u8 { + default: 0, + description: "Trace Touch ID sensor registration steps and timing, never sensor data", + }, xarm_trace: u8 { default: 0, description: "Opt-in xART opcode/status trace without payloads or key material", diff --git a/tools/aurora-sep/tests/test_sensor_wake.py b/tools/aurora-sep/tests/test_sensor_wake.py index c9bb8c6af7579..cd1c9c74a793e 100644 --- a/tools/aurora-sep/tests/test_sensor_wake.py +++ b/tools/aurora-sep/tests/test_sensor_wake.py @@ -22,6 +22,15 @@ def test_post_registration_status_drives_patch_readiness(self): #![allow(dead_code, unused_variables)] use std::cell::RefCell; macro_rules! dev_warn { ($($args:tt)*) => {} } +macro_rules! dev_info { ($($args:tt)*) => {} } +use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; +mod shim { pub fn boottime_ns()->u64 {0} } +mod module_parameters { + pub struct Param(u8); + impl Param { pub fn value(&self)->&u8 {&self.0} } + #[allow(non_upper_case_globals)] pub static sensor_trace: Param = Param(0); +} +struct Profile { nonce_session: bool } macro_rules! dev_err { ($($args:tt)*) => {} } mod kernel { pub mod time { pub struct Delta; @@ -60,8 +69,10 @@ def test_post_registration_status_drives_patch_readiness(self): pub fn patch_ack(&self)->u8 {self.ack} } } -struct Driver; +struct Driver { profile: Profile, sensor_power_ns: AtomicU64 } impl Driver { + fn new()->Self { Driver { profile: Profile { nonce_session: false }, sensor_power_ns: AtomicU64::new(0) } } + fn sensor_ms(&self)->u64 {0} fn register_sensor(&self,_:&sensor::Identifier)->bool { MOCK.with(|m| {let mut m=m.borrow_mut();assert_eq!(m.reads,1);m.registered=true;}); true } @@ -80,7 +91,7 @@ def test_post_registration_status_drives_patch_readiness(self): (Some(sensor::Status{state:0,ack:0,silent:false}),true,Some(0)), ] { MOCK.with(|m| *m.borrow_mut()=Mock{next,..Mock::default()}); - assert_eq!(Driver.wake_sensor().is_some(),ready); + assert_eq!(Driver::new().wake_sensor().is_some(),ready); MOCK.with(|m| {let m=m.borrow();assert_eq!(m.reads,2);assert!(m.registered);assert_eq!(m.patch,patch);}); } } From ef320e3fffa34b00bddb40d3122513eab3d10b71 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:29:43 +1100 Subject: [PATCH 1069/1579] soc: apple: sep: Clear a partial sensor registration before retrying it On a profile with a nonce session, every registration identifies the sensor first: PLATFORM_IDENTITY, then REGISTER_SENSOR. The enclave state is only cleared once the bring-up stage reads ESTABLISHED. If a registration fails after REGISTER_SENSOR was accepted, the stage stays FRESH while the enclave holds the partial registration. Each retry then identified the sensor again without clearing it. On boot d1962b3a, all 42 retries over 800 s failed that way, and Touch ID stayed unusable until a reboot. Count the enclave as holding the sensor once REGISTER_SENSOR has been submitted, answered or not, since a lost or late reply does not mean the enclave did not act on it. Clear its sensor state before registering again whenever it may hold one. Restarting the bring-up from FRESH does not change that, because only CLEAR_STATE empties the enclave. Profiles without a nonce session keep their flow. The decision is a plain function, with KUnit tests for the first registration, the retry after a partial one, re-registration and the other profiles. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/registration.rs | 116 ++++++++++++++++++++++++++++++ drivers/soc/apple/sbio.rs | 26 +++++-- drivers/soc/apple/sep.rs | 11 ++- 3 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 drivers/soc/apple/registration.rs diff --git a/drivers/soc/apple/registration.rs b/drivers/soc/apple/registration.rs new file mode 100644 index 0000000000000..95edd6a0b65b5 --- /dev/null +++ b/drivers/soc/apple/registration.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Ryan Murray + +//! What a Touch ID sensor registration has to send to the enclave. +//! +//! A registration identifies the sensor (REGISTER_SENSOR), registers its +//! serial and, on profiles with a nonce session, authenticates it. Any of +//! those steps can fail after the enclave has already accepted an earlier +//! one, and then the enclave holds a partial registration. On boot +//! d1962b3a every retry that registered the sensor again without clearing +//! it failed, 42 of them over 800 s. Re-registration that identifies the +//! sensor and then clears the enclave's sensor state, the order every +//! verify uses, has not failed on an idle machine. So the next attempt +//! clears whenever the enclave may hold any registration, complete or not. + +/// Nothing is registered with the enclave yet. +pub(crate) const FRESH: u32 = 0; +/// The sensor is identified; profiles without a nonce session register the +/// serial next. +pub(crate) const IDENTIFIED: u32 = 1; +/// The registration completed. +pub(crate) const ESTABLISHED: u32 = 2; + +/// The enclave requests one registration attempt makes, in order. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Plan { + /// PLATFORM_IDENTITY and REGISTER_SENSOR, before anything else. A + /// nonce-session profile identifies the sensor on every reset. + pub(crate) identify_first: bool, + /// CLEAR_STATE, after any identification. + pub(crate) clear_state: bool, + /// REGISTER_SENSOR after the clear, on a profile without a nonce session. + pub(crate) register_sensor: bool, +} + +/// Plans one registration attempt. +/// +/// `stage` is the bring-up stage and `holds_sensor` whether an +/// identification of the sensor was submitted to the enclave since it +/// booted, answered or not: a lost reply does not mean the enclave did not +/// act on it. `sepos13` says whether the profile starts from the serial, as +/// the 13.5 key store does. +pub(crate) fn plan(stage: u32, holds_sensor: bool, nonce_session: bool, sepos13: bool) -> Plan { + Plan { + identify_first: nonce_session, + clear_state: stage >= ESTABLISHED || (nonce_session && holds_sensor), + register_sensor: stage == FRESH && !nonce_session && !sepos13, + } +} + +#[kernel::macros::kunit_tests(apple_sep_registration)] +mod tests { + use super::*; + + #[test] + fn first_registration_after_boot_does_not_clear() { + assert_eq!( + plan(FRESH, false, true, false), + Plan { + identify_first: true, + clear_state: false, + register_sensor: false + } + ); + } + + #[test] + fn retry_after_a_partial_registration_clears() { + // REGISTER_SENSOR was accepted, then the serial, nonce or session + // step failed: the bring-up stage is still FRESH. + let retry = plan(FRESH, true, true, false); + assert!(retry.identify_first); + assert!(retry.clear_state); + } + + #[test] + fn re_registration_after_a_complete_one_clears() { + assert!(plan(ESTABLISHED, true, true, false).clear_state); + assert!(plan(ESTABLISHED, false, false, false).clear_state); + } + + #[test] + fn a_fresh_restart_still_clears_what_the_enclave_holds() { + // Restarting the bring-up from FRESH does not erase the enclave's + // registration; only CLEAR_STATE does. + assert!(plan(FRESH, true, true, false).clear_state); + } + + #[test] + fn profiles_without_a_nonce_session_are_unchanged() { + assert_eq!( + plan(FRESH, true, false, false), + Plan { + identify_first: false, + clear_state: false, + register_sensor: true + } + ); + assert_eq!( + plan(IDENTIFIED, true, false, false), + Plan { + identify_first: false, + clear_state: false, + register_sensor: false + } + ); + assert_eq!( + plan(FRESH, false, false, true), + Plan { + identify_first: false, + clear_state: false, + register_sensor: false + } + ); + } +} diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index d60cec976fa26..e15af135453ac 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1761,9 +1761,24 @@ impl SepData { fn register_sensor(&self, id: &sensor::Identifier) -> bool { let stage = self.bringup.load(Relaxed); - dev_info!(self.dev, "sbio: registering sensor at bring-up stage {}\n", stage); + let plan = registration::plan( + stage, + self.sensor_held.load(Relaxed), + self.profile.nonce_session, + matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }), + ); + dev_info!( + self.dev, + "sbio: registering sensor at bring-up stage {}{}\n", + stage, + if plan.clear_state && stage < BRINGUP_ESTABLISHED { + c", clearing the enclave's partial registration first" + } else { + c"" + } + ); - if self.profile.nonce_session { + if plan.identify_first { // J700 needs identification before CLEAR_STATE on every reset. let platform = match Self::sensor_platform_identity() { Ok(platform) => platform, @@ -1777,12 +1792,15 @@ impl SepData { return false; } self.sensor_trace(c"PLATFORM_IDENTITY"); + // Counted as held from submission on: a reply that is lost or + // late does not mean the enclave did not act on the request. + self.sensor_held.store(true, Relaxed); if self.sensor_enclave_step(&crate::sbio::sbio_register_sensor(id)).is_none() { return false; } } - if stage >= BRINGUP_ESTABLISHED { + if plan.clear_state { let _ = self.log_identity_count(c"before CLEAR_STATE for sensor re-registration"); if self .sensor_enclave_step(&crate::sbio::sbio_clear_state()) @@ -1799,7 +1817,7 @@ impl SepData { // The 13.5 J313 flow begins with REGISTER_SENSOR_SERIAL (0x17). // The newer REGISTER_SENSOR (0x80) returned 0x16 on this firmware. self.bringup.store(BRINGUP_IDENTIFIED, Relaxed); - } else if stage == BRINGUP_FRESH && !self.profile.nonce_session { + } else if plan.register_sensor { let op = crate::sbio::sbio_register_sensor(id); match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index fd961cfc4d7ee..ee3a7a708bf29 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -25,6 +25,7 @@ mod profile; mod proto; mod refkey; mod refkey_seal; +mod registration; mod rxring; mod sbio; mod scrd; @@ -251,9 +252,9 @@ enum SbioOutcome { Other, } -const BRINGUP_FRESH: u32 = 0; -const BRINGUP_IDENTIFIED: u32 = 1; -const BRINGUP_ESTABLISHED: u32 = 2; +const BRINGUP_FRESH: u32 = registration::FRESH; +const BRINGUP_IDENTIFIED: u32 = registration::IDENTIFIED; +const BRINGUP_ESTABLISHED: u32 = registration::ESTABLISHED; pub(crate) struct PatchLoaded(()); @@ -640,6 +641,9 @@ struct SepData { touchid_failed: Atomic, bringup: Atomic, + // The enclave accepted an identification of the sensor since it booted, + // even if the registration it began then failed. + sensor_held: Atomic, #[pin] ool_sks: Mutex>, @@ -841,6 +845,7 @@ impl SepData { sks_initialized: Atomic::new(false), sks_wedged: Atomic::new(0), bringup: Atomic::new(BRINGUP_FRESH), + sensor_held: Atomic::new(false), keybag_designated: Atomic::new(false), bringup_started: Atomic::new(false), touchid_started: Atomic::new(false), From c61197b7d5e1c9e845f5c207f4ed9259ff975950 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:30:12 +1100 Subject: [PATCH 1070/1579] soc: apple: sep: Re-read a sensor frame that is still on offer A read of a sensor frame can fail on the bus in two ways. The bytes come back garbled, or the sensor does not take the frame. Either way it keeps offering the frame at the same length. Boot 0874c215 showed both: the nonce read failed its CRC, and afterwards the sensor still offered the complete 43-byte nonce frame with a valid CRC. On boot 7f0c01be the serial read succeeded but the serial frame was not taken. The nonce command that followed then waited behind it until it timed out. Check the CRC of the serial, nonce, session and challenge replies. The session and challenge replies were relayed unchecked, but they carry the same CRC; it held on every boot measured. The calibration module reply keeps its own handling. When a reply fails its CRC and the sensor still offers a frame of that length, read it again, at most twice. Before a command that expects a reply, take any stale frame the sensor still offers, so the reply is not queued behind it. Two such frames at most: if one is still on offer after that, fail the step rather than send the command, because a read does not prove the sensor took the frame. Report both, since either means an exchange went wrong on the bus. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 15 +++++ drivers/soc/apple/sensor.rs | 130 +++++++++++++++++++++++++++--------- 2 files changed, 112 insertions(+), 33 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index e15af135453ac..3a6c054f49a6e 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1711,11 +1711,25 @@ impl SepData { } } + /// Reports frames the sensor exchange had to read again or discard. + fn note_sensor_recoveries(&self) { + let (reread, drained) = sensor::take_recoveries(); + if reread != 0 || drained != 0 { + dev_warn!( + self.dev, + "sensor: re-read {} frame(s) the sensor still offered after a garbled read, discarded {} stale frame(s)\n", + reread, + drained + ); + } + } + /// Reports a failed registration step with what the sensor shows /// afterwards: its state, patch acknowledgment, whether it presents an /// identifier and the length of any frame it holds. Never the /// identifier, the serial or any key material. fn sensor_step_failed(&self, step: &CStr) { + self.note_sensor_recoveries(); let ms = self.sensor_ms(); match sensor::status() { Ok(st) => dev_warn!( @@ -1847,6 +1861,7 @@ impl SepData { return false; } self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); + self.note_sensor_recoveries(); let _ = self.log_identity_count(c"after sensor serial registration"); true } diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index dc3df5c6b1d34..c0f3a6159f74c 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -5,6 +5,7 @@ //! The fingerprint sensor over SPI. use kernel::prelude::*; +use kernel::sync::atomic::{Atomic, Relaxed}; use kernel::time::delay::fsleep; use kernel::time::Delta; @@ -118,6 +119,7 @@ const CMD_START_CAPTURE: [u8; CMD_LEN] = [0x80, 0x40, 0x00, 0x07, 0x00, 0x00, 0x // Precedes the patch blob. const CMD_SETUP_PATCH_ENABLE: [u8; CMD_LEN] = [0x80, 0x60, 0x00, 0x07, 0x00, 0x00, 0x00]; const CMD_GET_SENSOR_SERIAL: [u8; CMD_LEN] = [0x80, 0x70, 0x00, 0x07, 0x00, 0x00, 0x00]; +const CMD_GET_NONCE: [u8; CMD_LEN] = [0x80, 0x35, 0x00, 0x07, 0x00, 0x00, 0x00]; const READ_CMD_LEN: usize = 11; const CMD_READ_PREFIX: [u8; CMD_LEN] = [0x80, 0x13, 0x00, 0x0b, 0x00, 0x00, 0x00]; @@ -475,26 +477,18 @@ fn poll_until_ready_any(accepted: &[usize]) -> Result { } pub(crate) fn send_session_share(share: &[u8; SESSION_SHARE_LEN]) -> Result<()> { + drain_stale_frames()?; send_framed(TYPE_SESSION, share) } pub(crate) fn send_challenge(challenge: &[u8; CHALLENGE_LEN]) -> Result<()> { + drain_stale_frames()?; send_framed(TYPE_SEQUENCE, challenge) } static_assert!(FRAME_HEADER + SESSION_SHARE_LEN == 0x2f); static_assert!(FRAME_HEADER + CHALLENGE_LEN == 0x47); -fn read_framed(total: usize, at: usize, out: &mut [u8]) -> Result<()> { - let mut buf = [0xffu8; MAX_REPLY]; - read_reply(total, &mut buf)?; - if at + out.len() > total { - return Err(EINVAL); - } - out.copy_from_slice(&buf[at..at + out.len()]); - Ok(()) -} - fn poll_until_ready(expected: usize) -> Result<()> { for _ in 0..READ_POLL_ATTEMPTS { let st = status()?; @@ -511,11 +505,6 @@ fn poll_until_ready(expected: usize) -> Result<()> { const READ_POLL_MS: i64 = 5; const READ_POLL_ATTEMPTS: usize = 200; -fn read_reply(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { - poll_until_ready(total)?; - read_reply_ready(total, buf) -} - fn read_reply_ready(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { if total > MAX_REPLY { return Err(EINVAL); @@ -538,22 +527,99 @@ fn read_reply_ready(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { }) } -fn read_framed_verified(total: usize, at: usize, out: &mut [u8]) -> Result<()> { - if total < CRC_LEN || at + out.len() > total - CRC_LEN { - return Err(EINVAL); +/// Frames the sensor still offered after the read that should have taken +/// them, and stale frames found on offer before a command. Both mean an +/// exchange went wrong on the bus; the caller reports them. +static REREADS: Atomic = Atomic::new(0); +static DRAINED: Atomic = Atomic::new(0); + +/// Takes the counts of re-read and drained frames since the last call. +pub(crate) fn take_recoveries() -> (u32, u32) { + (REREADS.xchg(0, Relaxed), DRAINED.xchg(0, Relaxed)) +} + +// A garbled read the sensor did not take is read once more, twice at most. +const FRAME_READS: usize = 3; +// Stale frames taken before a command; the sensor queues at most a couple. +const DRAIN_FRAMES: usize = 2; + +/// A reply buffer, wiped however the read that fills it ends. +struct FrameBuf([u8; MAX_REPLY]); + +impl FrameBuf { + fn new() -> Self { + FrameBuf([0xff; MAX_REPLY]) } - let mut buf = [0xffu8; MAX_REPLY]; - read_reply(total, &mut buf)?; +} - let split = total - CRC_LEN; - let expected = u16::from_le_bytes([buf[split], buf[split + 1]]); - let computed = crc16_ansi(&buf[..split]); - if expected != computed { - return Err(EIO); +impl Drop for FrameBuf { + fn drop(&mut self) { + for b in self.0.iter_mut() { + // SAFETY: a valid, uniquely borrowed byte. + unsafe { core::ptr::write_volatile(b, 0) }; + } + } +} + +fn frame_crc_valid(frame: &[u8]) -> bool { + let split = frame.len() - CRC_LEN; + u16::from_le_bytes([frame[split], frame[split + 1]]) == crc16_ansi(&frame[..split]) +} + +/// Takes any frame the sensor still offers before a new command, so the +/// reply to that command is not mistaken for, or queued behind, a stale one. +/// Fails if the sensor still offers a frame once the budget is spent: a +/// read is no proof the sensor took the frame. +fn drain_stale_frames() -> Result<()> { + for _ in 0..DRAIN_FRAMES { + let Some(count) = status()?.available().filter(|n| *n != 0) else { + return Ok(()); + }; + let count = count as usize; + if count > MAX_REPLY { + return Err(EIO); + } + let mut buf = FrameBuf::new(); + read_reply_ready(count, &mut buf.0)?; + DRAINED.fetch_add(1, Relaxed); + } + match status()?.available() { + None | Some(0) => Ok(()), + Some(_) => Err(EBUSY), + } +} + +/// Sends `cmd` and reads its `total`-byte reply, checking the reply's CRC. +fn command_frame(cmd: &[u8; CMD_LEN], total: usize, at: usize, out: &mut [u8]) -> Result<()> { + drain_stale_frames()?; + command(cmd)?; + read_frame(total, at, out) +} + +/// Reads a `total`-byte reply once the sensor offers it and checks its CRC. +/// A read the sensor did not take leaves the frame on offer at the same +/// length: only then is it read again. +fn read_frame(total: usize, at: usize, out: &mut [u8]) -> Result<()> { + if total < CRC_LEN || total > MAX_REPLY || at + out.len() > total - CRC_LEN { + return Err(EINVAL); } + poll_until_ready(total)?; - out.copy_from_slice(&buf[at..at + out.len()]); - Ok(()) + let mut buf = FrameBuf::new(); + for attempt in 0..FRAME_READS { + if attempt > 0 { + if status()?.available() != Some(total as u32) { + break; + } + REREADS.fetch_add(1, Relaxed); + } + read_reply_ready(total, &mut buf.0)?; + if frame_crc_valid(&buf.0[..total]) { + out.copy_from_slice(&buf.0[at..at + out.len()]); + return Ok(()); + } + } + Err(EIO) } const MAX_REPLY: usize = 0x4f; @@ -576,17 +642,15 @@ impl SensorSerial { } pub(crate) fn read_sensor_serial() -> Result { - command(&CMD_GET_SENSOR_SERIAL)?; let mut serial = [0u8; SENSOR_SERIAL_LEN]; - read_framed_verified(SENSOR_SERIAL_REPLY_LEN, SENSOR_SERIAL_AT, &mut serial)?; + command_frame(&CMD_GET_SENSOR_SERIAL, SENSOR_SERIAL_REPLY_LEN, SENSOR_SERIAL_AT, &mut serial)?; Ok(SensorSerial(serial)) } /// Read the nonce as one frame; a separate acknowledgment would consume it. pub(crate) fn session_nonce() -> Result<[u8; 32]> { - command(&[0x80, 0x35, 0, 7, 0, 0, 0])?; let mut value = [0; 32]; - read_framed_verified(9 + value.len() + CRC_LEN, 9, &mut value)?; + command_frame(&CMD_GET_NONCE, 9 + value.len() + CRC_LEN, 9, &mut value)?; if value.iter().all(|byte| *byte == 0) { return Err(EIO); } @@ -595,13 +659,13 @@ pub(crate) fn session_nonce() -> Result<[u8; 32]> { pub(crate) fn read_session_reply() -> Result<[u8; SESSION_SHARE_LEN]> { let mut share = [0u8; SESSION_SHARE_LEN]; - read_framed(SESSION_REPLY_LEN, SESSION_REPLY_AT, &mut share)?; + read_frame(SESSION_REPLY_LEN, SESSION_REPLY_AT, &mut share)?; Ok(share) } pub(crate) fn read_challenge_reply() -> Result<[u8; CHALLENGE_LEN]> { let mut reply = [0u8; CHALLENGE_LEN]; - read_framed(CHALLENGE_REPLY_LEN, CHALLENGE_REPLY_AT, &mut reply)?; + read_frame(CHALLENGE_REPLY_LEN, CHALLENGE_REPLY_AT, &mut reply)?; Ok(reply) } From 6c394131cf7e66c132be30a2f0ebd0562ca478dd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:30:52 +1100 Subject: [PATCH 1071/1579] soc: apple: sep: Retry a failed attach-time sensor bring-up When the bring-up at attach failed, nothing retried it until the first verify, minutes later. That verify was then refused before it touched the sensor, and gdm offered no fingerprint at login. A bring-up that fails on the bus leaves nothing a new attempt cannot undo. Each attempt power-cycles the sensor and, after a failed one, clears the enclave partial registration first. That is the same path every verify takes, and it has never failed on an idle machine. Try up to three times at attach, logging each failed attempt. A power cycle that fails ends the attempt before registration: a sensor that was not reset may still hold the session of the attempt before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 3a6c054f49a6e..2ee4674a6457b 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1080,12 +1080,28 @@ impl SepData { fn attach_bringup(&self) { if self.profile.nonce_session { - if !self.bring_sensor_online() { - dev_warn!( - self.dev, - "sensor: authenticated attach incomplete; capture unavailable\n" - ); + // A bring-up that fails on the bus leaves nothing a new attempt + // cannot undo: each attempt power-cycles the sensor and, after a + // failed one, clears the enclave's partial registration first, + // as every verify does. + const ATTEMPTS: u32 = 3; + for attempt in 1..=ATTEMPTS { + if self.bring_sensor_online() { + return; + } + if attempt < ATTEMPTS { + dev_warn!( + self.dev, + "sensor: attach-time bring-up attempt {} of {} failed; starting again\n", + attempt, + ATTEMPTS + ); + } } + dev_warn!( + self.dev, + "sensor: authenticated attach incomplete; capture unavailable\n" + ); return; } if !sensor::is_bound() { @@ -1940,6 +1956,12 @@ impl SepData { let _source = sensor::power_source(); self.sensor_power_ns.store(shim::boottime_ns(), Relaxed); let cycled = sensor::power_cycle(); + if !cycled && self.profile.nonce_session { + // A sensor that was not reset may still hold the session of an + // earlier attempt; registering it again cannot be trusted. + dev_err!(self.dev, "sensor: the power cycle failed; not registering the sensor\n"); + return None; + } if !sensor::cs_timing().is_hardware() { dev_warn!( self.dev, From e4321646f86a1db4eea5e91e11424ac4df2304dd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:17:40 +1100 Subject: [PATCH 1072/1579] soc: apple: sep: Drop an SBIO reply that belongs to an earlier exchange An SBIO reply is matched to the waiting exchange by opcode alone, and an error marker is matched before the opcode is checked. The receive work reads a reply before it takes the reassembly lock. If the requester times out in between and a retry begins the same opcode, the late reply completes the retry with the earlier attempt result. Bring-up now retries the registration, so the window is real. Number the exchanges. The receive work samples the number before it reads a reply and passes it with the chunk; a chunk sampled before the current exchange began is dropped. Replies to the current exchange can only arrive after it began, since its request is sent after that. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 5 ++- drivers/soc/apple/transfer.rs | 83 ++++++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 2ee4674a6457b..45c290f5523e5 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2617,6 +2617,9 @@ impl SepData { if !self.sbio_ready.load(Relaxed) { return; } + // Before reading the reply: a requester that times out meanwhile and + // begins a new exchange must not receive it. + let generation = self.sbio_rx.lock().generation(); // notification and the enclave's DMA are not ordered; buffer may still be poison if !self.ool_await_written(&self.ool_sbio, 0, transfer::HEADER_LEN) { @@ -2655,7 +2658,7 @@ impl SepData { } }; - let progress = self.sbio_rx.lock().on_chunk(marker, &packet, &payload); + let progress = self.sbio_rx.lock().on_chunk(generation, marker, &packet, &payload); match progress { transfer::Progress::NeedMore(cont) => { diff --git a/drivers/soc/apple/transfer.rs b/drivers/soc/apple/transfer.rs index 1f77f45162576..a4f5f0df3e308 100644 --- a/drivers/soc/apple/transfer.rs +++ b/drivers/soc/apple/transfer.rs @@ -177,6 +177,10 @@ pub(crate) struct Reassembly { done: Option, sending: Option, grants: u32, + // Counts exchanges begun. A reply is matched only by its opcode, so a + // late reply to an exchange that timed out would otherwise complete the + // next exchange with the same opcode. + generation: u64, } impl Reassembly { @@ -186,14 +190,22 @@ impl Reassembly { done: None, sending: None, grants: 0, + generation: 0, } } + /// The exchange a message arriving now can belong to. Sampled before the + /// message is read and passed back to [`Self::on_chunk`]. + pub(crate) fn generation(&self) -> u64 { + self.generation + } + pub(crate) fn begin(&mut self, opcode: u32) -> Result<()> { if self.active.is_some() { return Err(EBUSY); } self.done = None; + self.generation = self.generation.wrapping_add(1); self.active = Some(Active { opcode, total: 0, @@ -242,7 +254,16 @@ impl Reassembly { self.abort(); } - pub(crate) fn on_chunk(&mut self, marker: u8, packet: &Packet, payload: &[u8]) -> Progress { + /// Takes one chunk of a reply. `generation` is [`Self::generation`] as + /// sampled when the message arrived: a chunk that arrived before the + /// current exchange began belongs to an earlier one and is dropped. + pub(crate) fn on_chunk( + &mut self, + generation: u64, + marker: u8, + packet: &Packet, + payload: &[u8], + ) -> Progress { if marker < MARKER_FIRST { return Progress::Notification; } @@ -262,6 +283,9 @@ impl Reassembly { let Some(active_opcode) = self.active.as_ref().map(|a| a.opcode) else { return Progress::Ignored; }; + if generation != self.generation { + return Progress::Ignored; + } // Checked before the opcode test below, deliberately. if marker == MARKER_ERROR { @@ -371,3 +395,60 @@ impl Reassembly { self.done = Some(Completed { status, payload }); } } + +#[kernel::macros::kunit_tests(apple_sep_transfer)] +mod tests { + use super::*; + + const OPCODE: u32 = 0x48; + + fn reply(err: u32) -> Packet { + Packet { + version: VERSION, + total: 0, + offset: 0, + flags: 0, + err, + opcode: OPCODE, + chunk: 0, + } + } + + #[test] + fn a_reply_to_the_current_exchange_completes_it() { + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + let generation = rx.generation(); + assert!(matches!( + rx.on_chunk(generation, MARKER_FIRST, &reply(0), &[]), + Progress::Complete + )); + assert!(rx.take_done().is_some_and(|done| done.status.is_ok())); + } + + #[test] + fn a_late_reply_does_not_complete_the_next_exchange() { + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + // The reply to the first exchange is being read when it times out. + let late = rx.generation(); + rx.abort(); + assert!(rx.take_done().is_some()); + // A retry begins the same opcode before the late reply is delivered. + assert!(rx.begin(OPCODE).is_ok()); + assert!(matches!( + rx.on_chunk(late, MARKER_FIRST, &reply(0), &[]), + Progress::Ignored + )); + assert!(matches!( + rx.on_chunk(late, MARKER_ERROR, &reply(0x16), &[]), + Progress::Ignored + )); + assert!(rx.take_done().is_none()); + let current = rx.generation(); + assert!(matches!( + rx.on_chunk(current, MARKER_FIRST, &reply(0), &[]), + Progress::Complete + )); + } +} From 47dbbd82a926e90b6884c029ca9e343a8ce51ca4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:24:57 +1100 Subject: [PATCH 1073/1579] wifi: mt7932: complete a command only with the event that answers it A command that waits for a reply is completed by the first event, not otherwise dispatched, that carries its sequence number. Nothing checks that the event answers that command. A delayed event after the 8-bit sequence has wrapped, or a malformed one, can complete a different command; most callers then check the reply, but the RTS setup accepts any matching sequence and goes on to publish data readiness. Record the event that answers the command being sent and require it as well as the sequence. Extended commands answer with the extended event (0xed), the runtime capability query (0x8a) with 0xec, other runtime queries with an event of their own ID, the boot ROM's register read with 0x02, its patch semaphore with 0x04 and its status commands with 0x01, the event numbers other connac devices use. These are also the events the callers already check for, and the RTS reply is the extended event. The ROM's OTP read keeps sequence-only matching for now: no public source documents its reply event. Log the event it receives at debug level so that it can be measured and pinned. Add KUnit tests for the matcher and the reply table, including same-sequence events with the wrong event ID. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") Assisted-by: LLM Signed-off-by: Ryan Murray --- .../net/wireless/mediatek/mt7932/input-test.c | 57 +++++++++++++++++++ drivers/net/wireless/mediatek/mt7932/mt7932.c | 5 +- drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 + .../net/wireless/mediatek/mt7932/protocol.h | 32 ++++++++++- 4 files changed, 93 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/input-test.c b/drivers/net/wireless/mediatek/mt7932/input-test.c index 409103ed18740..9ee9253f34a30 100644 --- a/drivers/net/wireless/mediatek/mt7932/input-test.c +++ b/drivers/net/wireless/mediatek/mt7932/input-test.c @@ -116,12 +116,69 @@ static void mt7932_rx_short_test(struct kunit *test) KUNIT_EXPECT_EQ(test, mt7932_rx_frame(packet, RX_HEADER + 20, &frame), -EMSGSIZE); } +static void event_build(u8 *packet, u8 eid, u8 seq, u8 option) +{ + memset(packet, 0, 64); + put_unaligned_le32(64 | (7U << 27), packet); + put_unaligned_le16(40, packet + 24); + packet[28] = eid; + packet[29] = seq; + packet[30] = option; +} + +/* Only the event that answers a command completes it. */ +static void mt7932_event_reply_test(struct kunit *test) +{ + struct mt7932_event event; + u8 packet[64]; + int eid = mt7932_reply_eid(0xed, true); + + event_build(packet, 0xed, 9, 0); + KUNIT_ASSERT_EQ(test, mt7932_event_parse(packet, sizeof(packet), &event), 0); + KUNIT_EXPECT_TRUE(test, mt7932_event_matches(&event, 9, eid)); + /* Another sequence, the no-reply option bit, or nothing waiting. */ + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 10, eid)); + event.option = 4; + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, eid)); + event.option = 0; + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 0, eid)); + /* An unrelated event that happens to carry the waiting sequence. */ + event_build(packet, 0x99, 9, 0); + KUNIT_ASSERT_EQ(test, mt7932_event_parse(packet, sizeof(packet), &event), 0); + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, eid)); + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x8a, true))); + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x03, false))); + /* The ROM's patch semaphore answers with its own event only. */ + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x10, false))); + event_build(packet, 0x04, 9, 0); + KUNIT_ASSERT_EQ(test, mt7932_event_parse(packet, sizeof(packet), &event), 0); + KUNIT_EXPECT_TRUE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x10, false))); + event_build(packet, 0x01, 9, 0); + KUNIT_ASSERT_EQ(test, mt7932_event_parse(packet, sizeof(packet), &event), 0); + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x10, false))); +} + +static void mt7932_event_reply_eid_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0xed, false), 0xed); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0xed, true), 0xed); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x8a, true), 0xec); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0xbf, true), 0xbf); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x03, false), 0x02); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x01, false), 0x01); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x05, false), 0x01); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x10, false), 0x04); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x50, false), -1); +} + static struct kunit_case mt7932_input_test_cases[] = { KUNIT_CASE(mt7932_rx_decrypted_test), KUNIT_CASE(mt7932_rx_plaintext_test), KUNIT_CASE(mt7932_rx_mismatch_test), KUNIT_CASE(mt7932_rx_integrity_test), KUNIT_CASE(mt7932_rx_short_test), + KUNIT_CASE(mt7932_event_reply_test), + KUNIT_CASE(mt7932_event_reply_eid_test), {} }; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 41d34e9c6b8c7..e879b46356994 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -205,7 +205,7 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) } goto unlock; } - if (mt7932_event_matches(&event, m->waiting) && !m->reply_length) { + if (mt7932_event_matches(&event, m->waiting, m->waiting_eid) && !m->reply_length) { memcpy(m->reply, packet, event.length); m->reply_length = event.length; complete(&m->response); @@ -377,6 +377,7 @@ int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, boo reinit_completion(&m->response); spin_lock_irqsave(&m->response_lock, flags); m->waiting = wait ? m->seq : 0; + m->waiting_eid = mt7932_reply_eid(cid, runtime); m->reply_length = 0; spin_unlock_irqrestore(&m->response_lock, flags); ret = mt_submit(m, q, ret); @@ -679,6 +680,8 @@ static int mt_otp_byte(struct mt7932 *m, u32 address, u8 *value) if (m->reply_length < 40) return -EPROTO; *value = m->reply[36]; + dev_dbg(&m->pdev->dev, "OTP_REPLY: eid=%02x word=%08x address=%08x length=%zu\n", + m->reply[28], get_unaligned_le32(m->reply + 32), address, m->reply_length); return 0; } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 6994f0c208105..37c34872fe66f 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -75,6 +75,8 @@ struct mt7932 { unsigned int cal_request_head, cal_request_count; spinlock_t response_lock; u8 seq, waiting; + /* Event that answers the waiting command; see mt7932_reply_eid(). */ + int waiting_eid; u8 mac[ETH_ALEN]; u8 reply[MT7932_RX_CAPACITY]; size_t reply_length; diff --git a/drivers/net/wireless/mediatek/mt7932/protocol.h b/drivers/net/wireless/mediatek/mt7932/protocol.h index 00751e130ef44..e7f8820a9ff8c 100644 --- a/drivers/net/wireless/mediatek/mt7932/protocol.h +++ b/drivers/net/wireless/mediatek/mt7932/protocol.h @@ -65,9 +65,37 @@ static inline int mt7932_event_parse(const u8 *packet, size_t length, return 0; } -static inline bool mt7932_event_matches(const struct mt7932_event *e, u8 seq) +/* + * The event that answers a command waiting for a reply, or -1 for any event. + * Extended commands answer with the extended event, the capability query + * with its own event, and the boot ROM's status commands with the generic + * one. The ROM's register read answers with the access-register event and + * its patch semaphore with the patch-semaphore event, as on other connac + * devices. Other runtime commands answer with an event of their own ID. + * The ROM's OTP read alone still matches on the sequence: its reply event + * is not documented, and mt_otp_byte() records the one it receives. + */ +static inline int mt7932_reply_eid(u8 cid, bool runtime) { - return seq && e->seq == seq && !(e->option & 4); + if (cid == 0xed) + return 0xed; + if (runtime) + return cid == 0x8a ? 0xec : cid; + switch (cid) { + case 0x03: + return 0x02; + case 0x10: + return 0x04; + case 0x50: + return -1; + default: + return 0x01; + } +} + +static inline bool mt7932_event_matches(const struct mt7932_event *e, u8 seq, int eid) +{ + return seq && e->seq == seq && !(e->option & 4) && (eid < 0 || e->eid == eid); } #endif From b5c3a48a07eb169431f557f2a00df3b9f8e72320 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:25:22 +1100 Subject: [PATCH 1074/1579] wifi: mt7932: refuse hibernation The driver supports only suspend-to-idle and refuses other suspend states in its .suspend callback. Hibernation does not go through .suspend: its freeze phase calls .freeze, which the driver does not provide, so the PCI core lets it pass with the firmware still running. The firmware keeps writing to the receive rings while the memory image is saved, and after a restore the kernel resumes with ring state that the firmware started by the booting kernel does not have. Refuse the freeze and poweroff phases. The PM core then aborts hibernation before the device changes state, so there is nothing to undo. Supporting hibernation needs DMA retirement on freeze and a full reinitialisation, including the regulatory pass, on restore. Fixes: b4d89a96d6b0 ("wifi: mt7932: keep the function running across suspend-to-idle") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index e879b46356994..fb07f9fcb790f 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1259,8 +1259,24 @@ static int mt_suspend(struct device *dev) return 0; } +/* + * Hibernation is refused. Its freeze phase does not call .suspend, so without + * this the firmware would keep writing to memory while the image is saved, + * and the restored kernel would resume with ring state the firmware of the + * booting kernel does not have. Supporting it needs DMA retirement and a full + * reinitialisation on restore. Returning an error here has no side effect to + * undo: the PM core aborts hibernation before this device changes state. + */ +static int mt_freeze(struct device *dev) +{ + dev_err(dev, "hibernation is not supported\n"); + return -EOPNOTSUPP; +} + static const struct dev_pm_ops mt_pm_ops = { .suspend = mt_suspend, + .freeze = mt_freeze, + .poweroff = mt_freeze, }; static const struct pci_device_id mt_ids[] = { From cd6b06ffbc497cd783cf07147e61a897ab6757dc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:34:11 +1100 Subject: [PATCH 1075/1579] wifi: mt7932: let a cancelled scan batch retire before a disconnect gives up The disconnect worker quiesces scanning before it retires the connection. mt_scan_quiesce() waits for the current batch to finish, aborts the scan and returns -EBUSY if a cancelled batch has not yet been reported finished. The disconnect worker treats that as fatal and sets link_failed, which nothing clears: the radio stays unusable until the next boot. A multi-batch scan makes this reachable without any firmware fault. When the awaited batch finishes, the scan completion work may submit the next batch before the abort is seen. The abort then cancels that batch, and its end is reported by the firmware a little later, after the quiesce has already returned -EBUSY. cfg80211 disconnects on every suspend, so a background scan at suspend time is enough. Wait for the cancelled batch to retire, bounded to two seconds, as mt_scan_retire_wait() already does for an address change. Only a scan that the firmware never ends still fails the link. Add a KUnit test that drives mt_scan_quiesce() with a synthetic scan: the completion work has already submitted the next batch when the abort arrives, and the firmware reports its end 300 ms later (success) or never (-EBUSY after the two-second bound). Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") Assisted-by: LLM Signed-off-by: Ryan Murray --- .../net/wireless/mediatek/mt7932/frontend.c | 26 +++--- .../net/wireless/mediatek/mt7932/rf-test.c | 91 +++++++++++++++++++ 2 files changed, 105 insertions(+), 12 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 8d873a4379bfa..84170c8f61326 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -123,18 +123,6 @@ void mt_abort_scan(struct wiphy *wiphy, struct wireless_dev *wdev) spin_unlock_irqrestore(&m->response_lock, flags); } -/* No command/response lock held. The caller closes new scan admission first. - * Natural EID0d completion fences the scan; a local cancel alone does not. - */ -int mt_scan_quiesce(struct mt7932 *m) -{ - if (READ_ONCE(m->scan_request) || READ_ONCE(m->retired_scan_seq)) - wait_for_completion_timeout(&m->scan_done, msecs_to_jiffies(5000)); - mt_abort_scan(m->wiphy, &m->wdev); - flush_work(&m->scan_finish_work); - return READ_ONCE(m->retired_scan_seq) ? -EBUSY : 0; -} - /* Wait at most @timeout in total for a cancelled scan to retire. Callers * may hold RTNL, so this never waits longer than asked. A stopping driver * completes scan_done for good, which would turn the wait into a spin. @@ -153,6 +141,20 @@ static void mt_scan_retire_wait(struct mt7932 *m, unsigned long timeout) } } +/* No command/response lock held. The caller closes new scan admission first. + * Natural EID0d completion fences the scan; a local cancel alone does not. + * The batch awaited here may already have been followed by the next one, + * which the abort then cancels: give the firmware time to end that one too, + * rather than declaring the link unrecoverable while its cancel is in flight. + */ +int mt_scan_quiesce(struct mt7932 *m) +{ + if (READ_ONCE(m->scan_request) || READ_ONCE(m->retired_scan_seq)) + wait_for_completion_timeout(&m->scan_done, msecs_to_jiffies(5000)); + mt_scan_retire_wait(m, msecs_to_jiffies(2000)); + return READ_ONCE(m->retired_scan_seq) ? -EBUSY : 0; +} + /* command_mutex held: one RF-band/type batch at a time, with matching EID0d * completion before the next batch. A request without SSIDs is passive; * active requests must also listen passively on NO_IR channels. diff --git a/drivers/net/wireless/mediatek/mt7932/rf-test.c b/drivers/net/wireless/mediatek/mt7932/rf-test.c index da078caa2444a..c95f4057a2305 100644 --- a/drivers/net/wireless/mediatek/mt7932/rf-test.c +++ b/drivers/net/wireless/mediatek/mt7932/rf-test.c @@ -298,6 +298,96 @@ static void mt_startup_band1_queue_drain_test(struct kunit *test) m->cal_test_context = NULL; } +struct mt_scan_retire_test_context { + struct delayed_work retire; + struct mt7932 *m; + bool late; +}; + +static struct mt_scan_retire_test_context *mt_scan_retire_ctx; + +/* The firmware reports the end of the cancelled batch a little later. */ +static void mt_scan_retire_test_late(struct work_struct *work) +{ + struct mt_scan_retire_test_context *ctx = + container_of(to_delayed_work(work), struct mt_scan_retire_test_context, retire); + struct mt7932 *m = ctx->m; + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + m->retired_scan_seq = 0; + complete(&m->scan_done); + spin_unlock_irqrestore(&m->response_lock, flags); +} + +/* + * The scan completion work had already submitted the next batch when the + * abort arrived: the abort cancels it, and its end is still to come. + */ +static void mt_scan_retire_test_finish(struct work_struct *work) +{ + struct mt_scan_retire_test_context *ctx = mt_scan_retire_ctx; + struct mt7932 *m = ctx->m; + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + if (m->scan_request && m->scan_aborted) { + m->scan_request = NULL; + m->retired_scan_seq = 8; + } + spin_unlock_irqrestore(&m->response_lock, flags); + if (ctx->late) + schedule_delayed_work(&ctx->retire, msecs_to_jiffies(300)); +} + +static void mt_scan_quiesce_retirement_test(struct kunit *test) +{ + static const struct cfg80211_ops ops = {}; + struct mt_scan_retire_test_context ctx = {}; + struct cfg80211_scan_request *scan; + struct mt7932 *m = test->priv; + unsigned long start; + unsigned int round; + + scan = kunit_kzalloc(test, sizeof(*scan), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, scan); + m->wiphy = wiphy_new(&ops, sizeof(m)); + KUNIT_ASSERT_NOT_NULL(test, m->wiphy); + *(struct mt7932 **)wiphy_priv(m->wiphy) = m; + init_completion(&m->scan_done); + INIT_WORK(&m->scan_finish_work, mt_scan_retire_test_finish); + INIT_DELAYED_WORK(&ctx.retire, mt_scan_retire_test_late); + ctx.m = m; + mt_scan_retire_ctx = &ctx; + for (round = 0; round < 2; round++) { + ctx.late = !round; + reinit_completion(&m->scan_done); + m->scan_request = scan; + scan->wdev = &m->wdev; + m->scan_aborted = false; + m->retired_scan_seq = 0; + /* The batch the quiesce waits for has just finished. */ + complete(&m->scan_done); + start = jiffies; + if (ctx.late) { + /* A late but genuine end retires the scan: no failure. */ + KUNIT_EXPECT_EQ(test, mt_scan_quiesce(m), 0); + KUNIT_EXPECT_EQ(test, m->retired_scan_seq, 0); + } else { + /* A batch the firmware never ends still fails, after the bound. */ + KUNIT_EXPECT_EQ(test, mt_scan_quiesce(m), -EBUSY); + KUNIT_EXPECT_EQ(test, m->retired_scan_seq, 8); + KUNIT_EXPECT_GE(test, jiffies - start, msecs_to_jiffies(1900)); + KUNIT_EXPECT_LE(test, jiffies - start, msecs_to_jiffies(3000)); + } + cancel_delayed_work_sync(&ctx.retire); + flush_work(&m->scan_finish_work); + } + mt_scan_retire_ctx = NULL; + wiphy_free(m->wiphy); + m->wiphy = NULL; +} + static struct kunit_case mt_rf_test_cases[] = { KUNIT_CASE(mt_startup_band1_queue_drain_test), KUNIT_CASE(mt_missing_policy_admission_retry_test), @@ -305,6 +395,7 @@ static struct kunit_case mt_rf_test_cases[] = { KUNIT_CASE(mt_rf_failure_wakes_association_test), KUNIT_CASE(mt_rf_failure_blocks_startup_test), KUNIT_CASE(mt_rf_failure_preserves_first_error_test), + KUNIT_CASE(mt_scan_quiesce_retirement_test), {} }; From f69dfc9ee6a4bf3bd9fe83fc06cd09b68dfa5474 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:34:59 +1100 Subject: [PATCH 1076/1579] wifi: mt7932: report the connection outcome outside the response lock The connect worker publishes its outcome with response_lock held and interrupts disabled, so that a late firmware success cannot raise the carrier after a cancellation. Some of those cfg80211 calls take bottom-half spinlocks: - a connect cancelled while it was in progress, which happens whenever the system suspends during a connect, drops the BSS reference with cfg80211_put_bss(), which takes rdev->bss_lock with spin_lock_bh(); - cfg80211_connect_done() looks the BSS up again, under the same lock, when it has dropped off the scan list, and drops the reference when it cannot allocate its event. Re-enabling bottom halves with interrupts disabled is not allowed: lockdep reports it, and pending softirqs would then run with response_lock held. Keep deciding the outcome, the connection flags and the carrier under the lock, then make the cfg80211 calls after dropping it, with GFP_KERNEL. The ordering against cancellation still holds: - mt_disconnect() and a firmware disconnect event both leave the report to the disconnect worker, which retires the connection under command_mutex first, and this worker holds command_mutex until its own report is queued; - an RF failure reports a lost connection directly. While the worker is publishing, the failure now marks the connection lost instead, and the worker reports it once its own result has been queued, so cfg80211 never sees the disconnect before the connect. A connect cancelled by userspace or by suspend is not a driver failure: log it at debug level instead of as an error. Add a KUnit test for an RF failure during publication: the failure marks the connection lost and leaves the report to the worker. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/link.c | 72 +++++++++++++------ drivers/net/wireless/mediatek/mt7932/mt7932.h | 4 ++ .../net/wireless/mediatek/mt7932/rf-test.c | 25 +++++++ 3 files changed, 78 insertions(+), 23 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index a1c345ec1f64e..4e9ff20623da0 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -71,11 +71,17 @@ void mt_rf_fail_locked(struct mt7932 *m, int error) complete_all(&m->assoc_done); complete_all(&m->discovery_done); } else if (m->connected) { - /* Hardware ownership is uncertain: report failure, retain the peer. */ + /* Hardware ownership is uncertain: report failure, retain the peer. + * A connect worker still publishing the connection reports this + * after it, so that cfg80211 sees the two in order. + */ m->connected = false; m->disconnecting = true; - cfg80211_disconnected(m->netdev, WLAN_REASON_UNSPECIFIED, - NULL, 0, true, GFP_ATOMIC); + if (m->connect_publishing) + m->connect_lost = true; + else + cfg80211_disconnected(m->netdev, WLAN_REASON_UNSPECIFIED, + NULL, 0, true, GFP_ATOMIC); } if (m->scan_request) { m->scan_aborted = true; @@ -388,7 +394,8 @@ void mt_connect_work(struct work_struct *work) struct cfg80211_connect_resp_params response = {}; u8 *body, mac[8] = {}, queue[32]; unsigned long flags; - bool submitted = false, peer_owned = false; + bool submitted = false, peer_owned = false, cancelled; + u16 reason; int ret; mutex_lock(&m->command_mutex); @@ -504,7 +511,10 @@ void mt_connect_work(struct work_struct *work) } /* Serialize the final outcome with cancellation. In particular, a late * firmware success must not publish carrier/authorization after .deauth. - * cfg80211 queues these notifications; GFP_ATOMIC keeps this IRQ-safe. + * The cfg80211 calls may take bottom-half locks, so they are made after + * response_lock is dropped. command_mutex keeps the disconnect worker, + * which retires the connection before reporting its end, behind them; + * an RF failure in the meantime leaves its report to this worker. */ spin_lock_irqsave(&m->response_lock, flags); if (!ret && (m->connect_error || !mt_rf_allowed(m))) { @@ -541,33 +551,49 @@ void mt_connect_work(struct work_struct *work) } m->connected = !ret; m->disconnecting = false; - if (m->connect_cancelled) { - if (response.links[0].bss) - cfg80211_put_bss(m->wiphy, response.links[0].bss); - cfg80211_disconnected(m->netdev, m->disconnect_reason, NULL, 0, true, GFP_ATOMIC); - } else { - cfg80211_connect_done(m->netdev, &response, GFP_ATOMIC); - } + cancelled = m->connect_cancelled; + reason = m->disconnect_reason; + m->connect_publishing = true; if (!ret) { schedule_work(&m->power_work); if (READ_ONCE(m->data_ready)) { netif_carrier_on(m->netdev); netif_wake_queue(m->netdev); } - if (m->connect_open) { - dev_dbg(&m->pdev->dev, - "OPEN_CONNECTED: owned peer, firmware completion; Ethernet=%u\n", - m->data_ready); - } else { - cfg80211_port_authorized(m->netdev, m->connect_bssid, NULL, 0, GFP_ATOMIC); - dev_dbg(&m->pdev->dev, - "WPA2_AUTHORIZED: firmware handshake complete; Ethernet=%u\n", - m->data_ready); - } + } + spin_unlock_irqrestore(&m->response_lock, flags); + + if (cancelled) { + if (response.links[0].bss) + cfg80211_put_bss(m->wiphy, response.links[0].bss); + cfg80211_disconnected(m->netdev, reason, NULL, 0, true, GFP_KERNEL); } else { - dev_err(&m->pdev->dev, "CONNECT_FAILED: %d; recovery-required=%u\n", ret, m->link_failed); + cfg80211_connect_done(m->netdev, &response, GFP_KERNEL); + } + if (!ret && !m->connect_open) + cfg80211_port_authorized(m->netdev, m->connect_bssid, NULL, 0, GFP_KERNEL); + + spin_lock_irqsave(&m->response_lock, flags); + m->connect_publishing = false; + if (m->connect_lost) { + m->connect_lost = false; + cfg80211_disconnected(m->netdev, WLAN_REASON_UNSPECIFIED, + NULL, 0, true, GFP_ATOMIC); } spin_unlock_irqrestore(&m->response_lock, flags); + if (!ret && m->connect_open) + dev_dbg(&m->pdev->dev, + "OPEN_CONNECTED: owned peer, firmware completion; Ethernet=%u\n", + READ_ONCE(m->data_ready)); + else if (!ret) + dev_dbg(&m->pdev->dev, + "WPA2_AUTHORIZED: firmware handshake complete; Ethernet=%u\n", + READ_ONCE(m->data_ready)); + else if (cancelled) + dev_dbg(&m->pdev->dev, "CONNECT_CANCELLED: %d\n", ret); + else + dev_err(&m->pdev->dev, "CONNECT_FAILED: %d; recovery-required=%u\n", + ret, READ_ONCE(m->link_failed)); mutex_unlock(&m->command_mutex); } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 37c34872fe66f..0460b23e81645 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -147,6 +147,10 @@ struct mt7932 { u8 connect_ssid_length, connect_channel, peer_wtbl; bool connecting, connected, peer_valid, link_failed, connect_open; bool disconnecting, disconnect_seen, disconnect_event_received, disconnect_local; + /* The connect worker is reporting its outcome; an RF failure meanwhile + * sets connect_lost for the worker to report. Under response_lock. + */ + bool connect_publishing, connect_lost; u16 disconnect_reason; int connect_error; u32 event_cipher; diff --git a/drivers/net/wireless/mediatek/mt7932/rf-test.c b/drivers/net/wireless/mediatek/mt7932/rf-test.c index c95f4057a2305..8d12c9a9ee697 100644 --- a/drivers/net/wireless/mediatek/mt7932/rf-test.c +++ b/drivers/net/wireless/mediatek/mt7932/rf-test.c @@ -388,6 +388,30 @@ static void mt_scan_quiesce_retirement_test(struct kunit *test) m->wiphy = NULL; } +/* + * An RF failure while the connect worker is publishing a new connection + * must not report the disconnect ahead of the connect: it leaves the report + * to the worker. (cfg80211 is not called here: the synthetic netdev has no + * wireless device, so a direct report would crash the test.) + */ +static void mt_rf_failure_during_publication_test(struct kunit *test) +{ + struct mt7932 *m = test->priv; + + m->connecting = false; + m->connected = true; + m->connect_publishing = true; + mt_rf_test_fail(m, -EIO); + KUNIT_EXPECT_TRUE(test, m->connect_lost); + KUNIT_EXPECT_FALSE(test, m->connected); + KUNIT_EXPECT_TRUE(test, m->disconnecting); + KUNIT_EXPECT_FALSE(test, netif_carrier_ok(m->netdev)); + /* Ownership is uncertain after an RF failure: keep the peer. */ + KUNIT_EXPECT_TRUE(test, m->peer_valid); + m->connect_lost = false; + m->connect_publishing = false; +} + static struct kunit_case mt_rf_test_cases[] = { KUNIT_CASE(mt_startup_band1_queue_drain_test), KUNIT_CASE(mt_missing_policy_admission_retry_test), @@ -396,6 +420,7 @@ static struct kunit_case mt_rf_test_cases[] = { KUNIT_CASE(mt_rf_failure_blocks_startup_test), KUNIT_CASE(mt_rf_failure_preserves_first_error_test), KUNIT_CASE(mt_scan_quiesce_retirement_test), + KUNIT_CASE(mt_rf_failure_during_publication_test), {} }; From 4e1a415fc82544593b57dffd7cc26e621b505ecd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:27:46 +1100 Subject: [PATCH 1077/1579] wifi: mt7932: keep the link when a beacon hint only lifts NO_IR In the world regulatory domain, cfg80211 lifts NO_IR on a channel once it has heard a beacon there, and calls the notifier because the driver asks for channel-change callbacks. The changed channel flags make a new snapshot, which starts a full regulatory pass, and the pass ends any connection before it programs the firmware. So the first beacon heard on each new channel during a background scan drops a working connection, only to permit more than the firmware currently allows. While associated, keep the stricter domain the firmware already has when the new snapshot differs from the applied one only by NO_IR lifted on some channels: same country, firmware domain, channel list and limits. Record it as deferred and run the pass when the connection has ended, before the disconnect worker lets a new connect start. Any other change, including a restriction, is applied at once as before and replaces a deferred relaxation. Until the deferred pass runs, a scan may probe on such a channel, which cfg80211 now permits, while the firmware domain still marks it passive. Add KUnit tests for the relaxation check. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/Kconfig | 3 +- drivers/net/wireless/mediatek/mt7932/Makefile | 1 + drivers/net/wireless/mediatek/mt7932/link.c | 1 + drivers/net/wireless/mediatek/mt7932/mt7932.h | 6 + .../mediatek/mt7932/regulatory-test.c | 103 ++++++++++++++++++ .../net/wireless/mediatek/mt7932/regulatory.h | 32 ++++++ .../net/wireless/mediatek/mt7932/startup.c | 32 ++++++ 7 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 drivers/net/wireless/mediatek/mt7932/regulatory-test.c diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index f09fb1f32012b..49a6fbad0e67e 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -40,4 +40,5 @@ config MT7932_KUNIT_TEST Exercise the validation of what the firmware and the device hand to the driver, with synthetic, truncated and corrupted input and without radio hardware: receive descriptors, including frames the - hardware did not decrypt, and command replies. + hardware did not decrypt, command replies, regulatory snapshots + and country power packages. diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index 97b1b760372c1..781d4530fbee1 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -3,3 +3,4 @@ obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += input-test.o +mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += regulatory-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 4e9ff20623da0..ccfd4d4124fce 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -710,6 +710,7 @@ void mt_disconnect_work(struct work_struct *work) spin_lock_irqsave(&m->response_lock, flags); m->disconnecting = false; cfg80211_disconnected(m->netdev, reason, NULL, 0, local, GFP_ATOMIC); + mt_reg_apply_deferred_locked(m); spin_unlock_irqrestore(&m->response_lock, flags); dev_dbg(&m->pdev->dev, "DISCONNECTED: teardown=%d local=%u\n", ret, local); } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 0460b23e81645..aae173b97cc13 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -114,6 +114,11 @@ struct mt7932 { bool interface_registered, interface_up; u32 reg_generation, reg_attempted; struct mt7932_reg_snapshot reg_desired; + /* A relaxation kept from the firmware while associated; applied by the + * next pass once the connection has ended. Under response_lock. + */ + struct mt7932_reg_snapshot reg_deferred; + bool reg_deferral; /* Channels the applied country package leaves out; under RTNL. */ DECLARE_BITMAP(policy_disabled, MT7932_CHANNELS); struct net_device *netdev; @@ -187,6 +192,7 @@ struct mt7932 { bool mt_rf_allowed(struct mt7932 *m); void mt_retry_missing_policy(struct mt7932 *m); +void mt_reg_apply_deferred_locked(struct mt7932 *m); int mt_net_open(struct net_device *netdev); int mt_request_input(struct mt7932 *m, const struct firmware **fw, const char *name); /* response_lock held; closes RF admission and reports terminal host state. */ diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c new file mode 100644 index 0000000000000..1292c450ff54b --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the regulatory snapshot logic and, as local files the + * driver must not trust, the country power packages. + */ +#include + +#include "mt7932.h" + +/* A world-domain snapshot: channels 1, 6, 36 and 149, the 5 GHz ones NO_IR. */ +static void reg_build(struct mt7932_reg_snapshot *reg) +{ + static const u16 channels[] = {1, 6, 36, 149}; + unsigned int i; + + memset(reg, 0, sizeof(*reg)); + memcpy(reg->alpha2, "00", 2); + memcpy(reg->domain, "XZ", 2); + reg->domain[4] = 1; + reg->domain[5] = 2; + reg->domain[8] = 2; + reg->domain[9] = 2; + for (i = 0; i < ARRAY_SIZE(channels); i++) { + put_unaligned_le16(channels[i], reg->domain + 12 + i * 8); + put_unaligned_le32(channels[i] > 14 ? IEEE80211_CHAN_NO_IR : 0, + reg->domain + 16 + i * 8); + reg->power[i] = 20; + } + reg->length = 12 + ARRAY_SIZE(channels) * 8; +} + +static void reg_set_flags(struct mt7932_reg_snapshot *reg, unsigned int entry, u32 flags) +{ + put_unaligned_le32(flags, reg->domain + 16 + entry * 8); +} + +/* A beacon hint lifting NO_IR only relaxes; nothing else does. */ +static void mt7932_reg_relaxation_test(struct kunit *test) +{ + struct mt7932_reg_snapshot *prev, *next; + + prev = kunit_kzalloc(test, sizeof(*prev), GFP_KERNEL); + next = kunit_kzalloc(test, sizeof(*next), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, prev); + KUNIT_ASSERT_NOT_NULL(test, next); + reg_build(prev); + + reg_build(next); + reg_set_flags(next, 2, 0); + KUNIT_EXPECT_TRUE(test, mt7932_reg_only_relaxes(prev, next)); + /* An identical snapshot is not treated as a change by the caller, + * but it is trivially within the applied domain. + */ + KUNIT_EXPECT_TRUE(test, mt7932_reg_only_relaxes(prev, prev)); + + /* NO_IR added: a restriction. */ + reg_build(next); + reg_set_flags(next, 0, IEEE80211_CHAN_NO_IR); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + /* Radar or a bandwidth limit appears, or another flag goes away. */ + reg_build(next); + reg_set_flags(next, 2, IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_RADAR); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_set_flags(prev, 3, IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_NO_HT40); + reg_build(next); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(prev); + /* A new country, power limit, channel or channel order. */ + reg_build(next); + memcpy(next->alpha2, "AU", 2); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(next); + next->power[1] = 19; + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(next); + put_unaligned_le16(40, next->domain + 12 + 3 * 8); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(next); + put_unaligned_le16(44, next->domain + 12 + 4 * 8); + next->domain[9] = 3; + next->length += 8; + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + /* A failed snapshot never relaxes, and a corrupt length is refused. */ + reg_build(next); + next->error = -EOPNOTSUPP; + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(next); + prev->length = sizeof(prev->domain) + 8; + next->length = prev->length; + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); +} + +static struct kunit_case mt7932_policy_test_cases[] = { + KUNIT_CASE(mt7932_reg_relaxation_test), + {} +}; + +static struct kunit_suite mt7932_policy_test_suite = { + .name = "mt7932-regulatory", + .test_cases = mt7932_policy_test_cases, +}; + +kunit_test_suite(mt7932_policy_test_suite); diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory.h b/drivers/net/wireless/mediatek/mt7932/regulatory.h index fd65e07d9506a..3a8c1ffc74015 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory.h +++ b/drivers/net/wireless/mediatek/mt7932/regulatory.h @@ -5,6 +5,8 @@ /* Private, versioned packaging of original-derived country policy. This is * not firmware code and does not replace cfg80211's regulatory database. */ +#include + #include "channels.h" #define MT7932_POLICY_TABLES 13 @@ -102,6 +104,36 @@ static inline int mt7932_policy_parse(struct mt7932_policy *out, return at == size ? 0 : -EINVAL; } +/* + * Whether @next differs from the applied @prev only by NO_IR lifted on some + * channels, as a beacon hint does in the world domain: same country, firmware + * domain header, channel list and limits, and every channel's flags equal to + * the applied ones less, at most, NO_IR. Such a change only permits more; the + * firmware may keep the stricter domain until it is idle. + */ +static inline bool mt7932_reg_only_relaxes(const struct mt7932_reg_snapshot *prev, + const struct mt7932_reg_snapshot *next) +{ + unsigned int i, count; + + if (prev->error || next->error || prev->length != next->length || + prev->length < 12 || prev->length > sizeof(prev->domain) || + memcmp(prev->alpha2, next->alpha2, 2) || + memcmp(prev->domain, next->domain, 12) || + memcmp(prev->power, next->power, sizeof(prev->power))) + return false; + count = (prev->length - 12) / 8; + for (i = 0; i < count; i++) { + const u8 *a = prev->domain + 12 + i * 8, *b = next->domain + 12 + i * 8; + u32 was = get_unaligned_le32(a + 4), now = get_unaligned_le32(b + 4); + + if (memcmp(a, b, 4) || (now & ~was) || + ((was & ~now) & ~IEEE80211_CHAN_NO_IR)) + return false; + } + return true; +} + /* Every entry of a validated package starts with its channel number and is * followed by per-rate power limits. A country that does not permit a * channel sets all of that channel's limits to this marker. diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 051ab3793d965..679c9193f85bc 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -279,6 +279,20 @@ static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *poli * Consume the flag under the same lock as the worker publication, so repeated * userspace requests cannot continually invalidate an in-flight attempt. */ +/* response_lock held: start the pass a deferred relaxation is waiting for. */ +void mt_reg_apply_deferred_locked(struct mt7932 *m) +{ + lockdep_assert_held(&m->response_lock); + if (!m->reg_deferral || m->stopping) + return; + m->reg_deferral = false; + m->reg_desired = m->reg_deferred; + m->reg_generation++; + WRITE_ONCE(m->reg_pending, true); + if (READ_ONCE(m->interface_registered)) + schedule_work(&m->startup_work); +} + void mt_retry_missing_policy(struct mt7932 *m) { unsigned long flags; @@ -462,7 +476,24 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request spin_unlock_irqrestore(&m->response_lock, irqflags); return; } + if (!m->reg_pending && m->connected && !m->disconnecting && + mt7932_reg_only_relaxes(&m->reg_desired, ®) && + memcmp(®, &m->reg_desired, sizeof(reg))) { + /* A pass would end the connection only to permit more than the + * firmware now allows. Keep the stricter applied domain and run + * the pass once the connection has ended. + */ + m->reg_deferred = reg; + m->reg_deferral = true; + spin_unlock_irqrestore(&m->response_lock, irqflags); + for_each_set_bit(i, m->policy_disabled, MT7932_CHANNELS) + mt_channel(m, i)->flags |= IEEE80211_CHAN_DISABLED; + dev_dbg(&m->pdev->dev, "REGULATORY_RELAXATION_DEFERRED: %c%c until disconnected\n", + reg.alpha2[0], reg.alpha2[1]); + return; + } if (!memcmp(®, &m->reg_desired, sizeof(reg)) && !m->reg_pending) { + m->reg_deferral = false; spin_unlock_irqrestore(&m->response_lock, irqflags); /* The applied package still holds, but cfg80211 has just * rebuilt the channel flags without its disables. @@ -472,6 +503,7 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request return; } m->reg_desired = reg; + m->reg_deferral = false; m->reg_generation++; /* The next package pass records its own set. */ bitmap_zero(m->policy_disabled, MT7932_CHANNELS); From 5cbf717b9d47f3717b4d30403e325ff59d47eecb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:50:13 +1100 Subject: [PATCH 1078/1579] wifi: mt7932: test the power package parser The power packages are files from the local system, so the parser and the channel filter treat them as untrusted input. Add KUnit tests with synthetic country and world packages: valid packages, a package that names another country, every truncation, trailing data, header, mode, table length, entry count and table type bounds, the world package's 2.4 GHz channel order, the not-permitted marker, the channel filter and 2,000 random corruptions. No driver behaviour changes. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../mediatek/mt7932/regulatory-test.c | 330 +++++++++++++++++- 1 file changed, 328 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c index 1292c450ff54b..225f3b8f35a0c 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c +++ b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c @@ -1,9 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * KUnit tests for the regulatory snapshot logic and, as local files the - * driver must not trust, the country power packages. + * KUnit tests for the regulatory snapshot logic and the power package + * parser and channel filter. The packages are local files, so + * the parser treats them as untrusted. */ #include +#include #include "mt7932.h" @@ -90,8 +92,332 @@ static void mt7932_reg_relaxation_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); } +#define PKG_MODE_SIZE 328 +#define PKG_ENTRY_SIZE 122 + +/* Channels of the synthetic 2.4 and 5 GHz tables 1..9, in table order. */ +static const u8 pkg_channels[] = { + 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, + 36, 40, 44, 48, 52, 56, 60, 64, + 100, 104, 108, 112, 116, 120, 124, 128, + 132, 136, 140, 144, 149, 153, 157, 161, + 38, 46, 54, 62, 102, 110, 118, 126, + 134, 142, 151, 159, 42, 58, 106, 122, + 138, 155, 50, 114, 130, 146, 163, 167, + 165, 169, 171, 173, 175, +}; + +static unsigned int pkg_count(unsigned int table, bool world) +{ + if (table == 2) + return world ? 6 : 5; + return table == 9 ? 5 : 8; +} + +static size_t pkg_size(bool world) +{ + static const u16 others[] = {532, 144, 58, 265}; + size_t size = 16 + (world ? 2 : 3) * PKG_MODE_SIZE; + unsigned int i; + + for (i = 1; i <= 9; i++) + size += 44 + pkg_count(i, world) * PKG_ENTRY_SIZE; + for (i = 0; i < ARRAY_SIZE(others); i++) + size += others[i]; + return size; +} + +/* Build a structurally valid package. Channel 14 is only in the world one. */ +static u8 *pkg_build(struct kunit *test, const char *alpha2, size_t *size) +{ + static const u16 lengths[] = {532, 144, 58, 265}; + static const u8 types[] = {5, 1, 4, 2}; + bool world = !memcmp(alpha2, "XZ", 2); + unsigned int i, j, modes = world ? 2 : 3, channel = 0; + u8 *data, *p; + + *size = pkg_size(world); + data = kunit_kzalloc(test, *size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, data); + memcpy(data, "J7RP", 4); + put_unaligned_le16(1, data + 4); + memcpy(data + 6, alpha2, 2); + data[8] = modes; + data[9] = MT7932_POLICY_TABLES; + put_unaligned_le32(*size, data + 12); + p = data + 16; + for (i = 0; i < modes; i++, p += PKG_MODE_SIZE) + put_unaligned_le32(17 + i, p + 4); + for (i = 0; i < MT7932_POLICY_TABLES; i++) { + if (i >= 1 && i <= 9) { + unsigned int count = pkg_count(i, world); + + p[0] = 3; + p[1] = 0x0a; + put_unaligned_le16(44 + count * PKG_ENTRY_SIZE, p + 2); + p[4] = count; + p[5] = i <= 2 ? 1 : 2; + p[6] = i == 2 || i == 9; + memcpy(p + 8, alpha2, 2); + for (j = 0; j < count; j++) { + u8 *entry = p + 44 + j * PKG_ENTRY_SIZE; + + /* A country package has no channel 14 entry. */ + if (!world && pkg_channels[channel] == 14) + channel++; + entry[0] = pkg_channels[channel++]; + memset(entry + 1, 30, PKG_ENTRY_SIZE - 1); + } + p += 44 + count * PKG_ENTRY_SIZE; + } else { + unsigned int k = i ? i - 9 : 0; + + p[1] = 2; + put_unaligned_le16(lengths[k], p + 2); + p[7] = types[k]; + if (i == 10) + memcpy(p + 8, alpha2, 2); + p += lengths[k]; + } + } + KUNIT_ASSERT_PTR_EQ(test, p, data + *size); + return data; +} + +/* The entry of @channel in a parsed package, or NULL. */ +static u8 *pkg_entry(const struct mt7932_policy *policy, unsigned int channel) +{ + unsigned int i, j; + + for (i = 1; i <= 9; i++) + for (j = 0; j < policy->table[i][4]; j++) + if (policy->table[i][44 + j * PKG_ENTRY_SIZE] == channel) + return (u8 *)policy->table[i] + 44 + j * PKG_ENTRY_SIZE; + return NULL; +} + +static void mt7932_policy_valid_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data; + + data = pkg_build(test, "AU", &size); + KUNIT_EXPECT_EQ(test, size, 10447); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "AU"), 0); + KUNIT_EXPECT_EQ(test, policy.modes, 3); + KUNIT_EXPECT_PTR_EQ(test, policy.table[0], data + 16 + 3 * PKG_MODE_SIZE); + KUNIT_EXPECT_EQ(test, policy.length[12], 265); + KUNIT_EXPECT_TRUE(test, mt7932_policy_permits(&policy, 36)); + KUNIT_EXPECT_FALSE(test, mt7932_policy_permits(&policy, 14)); + + data = pkg_build(test, "XZ", &size); + KUNIT_EXPECT_EQ(test, size, 10241); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), 0); + KUNIT_EXPECT_EQ(test, policy.modes, 2); + KUNIT_EXPECT_TRUE(test, mt7932_policy_permits(&policy, 14)); +} + +/* A package serves only the country it names, in the header and table 10. */ +static void mt7932_policy_country_mismatch_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data; + + data = pkg_build(test, "AU", &size); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), -EINVAL); + /* A country package renamed to the world one is not the world one. */ + memcpy(data + 6, "XZ", 2); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), -EINVAL); +} + +/* Every prefix of a valid package, with a consistent header, is rejected. */ +static void mt7932_policy_truncation_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size, cut; + u8 *data, *copy; + + data = pkg_build(test, "PL", &size); + for (cut = 0; cut < size; cut++) { + /* An exact-size copy lets KASAN catch any read past the end. */ + copy = kmemdup(data, cut, GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, copy || !cut); + if (cut >= 16) + put_unaligned_le32(cut, copy + 12); + KUNIT_EXPECT_EQ_MSG(test, mt7932_policy_parse(&policy, copy, cut, "PL"), + -EINVAL, "length %zu", cut); + kfree(copy); + } +} + +static void mt7932_policy_header_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data, *longer; + + data = pkg_build(test, "PL", &size); + /* Trailing bytes after the last table. */ + longer = kunit_kzalloc(test, size + 4, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, longer); + memcpy(longer, data, size); + put_unaligned_le32(size + 4, longer + 12); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, longer, size + 4, "PL"), -EINVAL); + + data[4] = 2; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + data[4] = 1; + data[8] = 4; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + data[8] = 3; + data[9] = 12; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + data[9] = MT7932_POLICY_TABLES; + put_unaligned_le32(size - 1, data + 12); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + put_unaligned_le32(size, data + 12); + /* Mode command lengths are bounded by the 320-byte body. */ + put_unaligned_le32(321, data + 16 + 4); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + put_unaligned_le32(0, data + 16 + 4); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + put_unaligned_le32(17, data + 16 + 4); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); +} + +/* Table lengths and entry counts must agree with the table type. */ +static void mt7932_policy_table_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data, *table; + + data = pkg_build(test, "PL", &size); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); + table = (u8 *)policy.table[3]; + /* An entry count that would run past the table. */ + table[4] = 9; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + table[4] = 8; + /* A table length that claims the rest of the file. */ + put_unaligned_le16(0xffff, table + 2); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + put_unaligned_le16(44 + 8 * PKG_ENTRY_SIZE, table + 2); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); + table = (u8 *)policy.table[10]; + table[7] = 2; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); +} + +/* The world package's 2.4 GHz tables must list channels 1..14 in order. */ +static void mt7932_policy_world_order_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data; + + data = pkg_build(test, "XZ", &size); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), 0); + pkg_entry(&policy, 9)[0] = 10; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), -EINVAL); +} + +static void mt7932_policy_permits_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data, *entry; + + data = pkg_build(test, "PL", &size); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); + entry = pkg_entry(&policy, 144); + KUNIT_ASSERT_NOT_NULL(test, entry); + memset(entry + 1, MT7932_POLICY_NOT_PERMITTED, PKG_ENTRY_SIZE - 1); + KUNIT_EXPECT_FALSE(test, mt7932_policy_permits(&policy, 144)); + /* One rate with a limit is enough to permit the channel. */ + entry[PKG_ENTRY_SIZE - 1] = 8; + KUNIT_EXPECT_TRUE(test, mt7932_policy_permits(&policy, 144)); + /* A channel the package does not list is not permitted. */ + KUNIT_EXPECT_FALSE(test, mt7932_policy_permits(&policy, 177)); +} + +static void mt7932_policy_filter_test(struct kunit *test) +{ + static const u16 channels[] = {1, 6, 13, 36, 144, 149, 165}; + struct mt7932_reg_snapshot *reg; + struct mt7932_policy policy; + unsigned int i; + size_t size; + u8 *data; + + reg = kunit_kzalloc(test, sizeof(*reg), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, reg); + data = pkg_build(test, "PL", &size); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); + memset(pkg_entry(&policy, 144) + 1, MT7932_POLICY_NOT_PERMITTED, + PKG_ENTRY_SIZE - 1); + memset(pkg_entry(&policy, 6) + 1, MT7932_POLICY_NOT_PERMITTED, + PKG_ENTRY_SIZE - 1); + memcpy(reg->domain, "PL", 2); + for (i = 0; i < ARRAY_SIZE(channels); i++) { + put_unaligned_le16(channels[i], reg->domain + 12 + i * 8); + put_unaligned_le32(i, reg->domain + 16 + i * 8); + } + reg->domain[8] = 3; + reg->domain[9] = 4; + reg->length = 12 + ARRAY_SIZE(channels) * 8; + + mt7932_policy_filter(reg, &policy); + KUNIT_EXPECT_EQ(test, reg->length, 12 + 5 * 8); + KUNIT_EXPECT_EQ(test, reg->domain[8], 2); + KUNIT_EXPECT_EQ(test, reg->domain[9], 3); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(reg->domain + 12 + 8), 13); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(reg->domain + 12 + 3 * 8), 149); + /* Each entry keeps its own channel flags. */ + KUNIT_EXPECT_EQ(test, get_unaligned_le32(reg->domain + 16 + 3 * 8), 5); + /* The vacated tail is cleared. */ + KUNIT_EXPECT_TRUE(test, !memchr_inv(reg->domain + 12 + 5 * 8, 0, 2 * 8)); +} + +/* Random corruption must never be accepted in a way that reads out of bounds. */ +static void mt7932_policy_corruption_test(struct kunit *test) +{ + struct rnd_state rnd; + struct mt7932_policy policy; + unsigned int round, flips, channel; + size_t size; + u8 *data, *copy; + + prandom_seed_state(&rnd, 7932); + data = pkg_build(test, "AU", &size); + copy = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, copy); + for (round = 0; round < 2000; round++) { + memcpy(copy, data, size); + for (flips = 1 + prandom_u32_state(&rnd) % 4; flips; flips--) + copy[prandom_u32_state(&rnd) % size] ^= 1 << (prandom_u32_state(&rnd) % 8); + if (mt7932_policy_parse(&policy, copy, size, "AU")) + continue; + for (channel = 0; channel < 256; channel++) + mt7932_policy_permits(&policy, channel); + } +} + static struct kunit_case mt7932_policy_test_cases[] = { KUNIT_CASE(mt7932_reg_relaxation_test), + KUNIT_CASE(mt7932_policy_valid_test), + KUNIT_CASE(mt7932_policy_country_mismatch_test), + KUNIT_CASE(mt7932_policy_truncation_test), + KUNIT_CASE(mt7932_policy_header_test), + KUNIT_CASE(mt7932_policy_table_test), + KUNIT_CASE(mt7932_policy_world_order_test), + KUNIT_CASE(mt7932_policy_permits_test), + KUNIT_CASE(mt7932_policy_filter_test), + KUNIT_CASE(mt7932_policy_corruption_test), {} }; From 672846413c5933fc4254ad3daf2279d6bd588f35 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 04:18:30 +1100 Subject: [PATCH 1079/1579] drm/apple: Treat a missing legacy EDID service as optional Legacy DCP firmware can start the DPAVSERV endpoint without announcing an EDID service. Stock accepts expiry of the advisory discovery wait, while Aurora reports an endpoint failure even though the endpoint remains live. Keep the discovery window so an early announcement can precede get_modes, and return success when it expires. Only the announcement callback may publish a service. Log expiry and legacy publication at debug level so a late announcement can be distinguished from continued absence without a warning for an optional service. Leave AFK startup errors and H17P's asynchronous admission unchanged. Extend the production-function host tests to check expiry and publication diagnostics as well as optional discovery and genuine startup failures. Fixes: 014aed99ea14 ("drm/apple: retrieve external H17P EDID through bounded AFKV2 inline calls") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/epic/dpavservep.c | 18 ++-- .../selftests/drivers/gpu/apple-dcp/edid.py | 99 ++++++++++++++++++- 2 files changed, 109 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 2476209719c08..39b808306f624 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -138,6 +138,9 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, dev_dbg(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", service->channel); dcpavserv_notify_edid(service); + } else { + /* Also report announcements arriving after the discovery window. */ + dev_dbg(dcp->dev, "DCPAVSERV: legacy EDID service published\n"); } } @@ -488,12 +491,15 @@ int dpavservep_init(struct apple_dcp *dcp) if (ep->wire_v2) return 0; - ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, - msecs_to_jiffies(1000)); - if (!ret) { - dev_warn(dcp->dev, "DCPAVSERV: timed out waiting for EDID service announcement\n"); - return -ETIMEDOUT; - } + /* + * Legacy firmware need not announce an EDID service. Give it the same + * discovery window so an early announcement can precede get_modes, + * but absence is not an endpoint failure. + * Only dcpavserv_init() may publish a service that actually arrived. + */ + if (!wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, + msecs_to_jiffies(1000))) + dev_dbg(dcp->dev, "DCPAVSERV: no legacy EDID service within 1 s\n"); return 0; } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py index 69ebe810f1c2d..e247ce3a5489e 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -45,9 +45,10 @@ def function(file,signature): #define cpu_to_le32(x) (x) #define dev_warn(...) ((void)0) #define dev_err(...) do { diagnostic_errors++; } while (0) +#define dev_dbg(dev,fmt,...) do { if(!strcmp(fmt,"DCPAVSERV: legacy EDID service published\n")) legacy_publishes++; } while (0) #define dev_info(...) do { diagnostic_information++; } while (0) -static unsigned diagnostic_errors,diagnostic_information; +static unsigned diagnostic_errors,diagnostic_information,legacy_publishes; #define DCP_FIRMWARE_H17P 17 #define EPIC_TYPE_NOTIFY 0 #define EPIC_CAT_REPORT 0 @@ -339,7 +340,8 @@ def function(file,signature): dpavservep_detach(&d);afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); assert(!d.dcpavserv.service && legacy->torndown); legacy=prepare(13,0,name,native_class);assert(legacy); - d.typec_generation++;afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); + legacy_publishes=0;d.typec_generation++;afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); + assert(legacy_publishes==1); assert(d.dcpavserv.service==legacy && legacy->users==1);dpavservep_detach(&d); assert(dcpdpserv_prepare(legacy,"dcpdp-service-epic",native_class,0)==0); ep.wire_v2=true;ep.endpoint=0x2a;d.external_physically_absent=false; @@ -573,6 +575,7 @@ def function(file, signature): #define scoped_guard(k,p) for(bool *_lock __attribute__((cleanup(release)))=acquire(p); _lock; _lock=once(_lock)) static void mutex_lock(bool *p) { (void)acquire(p); } static void mutex_unlock(bool *p) { (void)once(p); } +#define dev_dbg(...) ((void)0) #define dev_info(...) ((void)0) #define dev_err(...) ((void)0) #define trace_dcpavserv_init(...) ((void)0) @@ -932,3 +935,95 @@ def function(text, signature): } ''' return prefix + '\n'.join(parts) + suffix + + +def startup(root): + source = (root / 'drivers/gpu/drm/apple/epic/dpavservep.c').read_text() + return r''' +#include +#include +#include +#include +#include +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define DPAVSERV_ENDPOINT 0x28 +#define DPAVSERV_ENDPOINT_V2 0x2a +#define IS_ERR(p) ((uintptr_t)(p) > (uintptr_t)-4096) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define ERR_PTR(e) ((void *)(intptr_t)(e)) +#define dev_warn(...) do { warnings++; } while (0) +#define dev_dbg(...) do { debug++; } while (0) +struct apple_dcp; +struct apple_dcp_afkep { + struct apple_dcp *dcp; + unsigned endpoint; + bool wire_v2,inline_v2,match_epic_name; +}; +struct apple_dcp { + void *dev; + struct { int iomfb_method_profile; } hw; + struct { unsigned enable_completion; bool enabled; } dcpavserv; + struct apple_dcp_afkep *dcpavservep; +}; +static const unsigned dpavservep_ops[] = {0}; +static struct apple_dcp_afkep endpoint; +static int init_error,start_error; +static unsigned warnings,waits,starts,shutdowns,detaches,debug; +static bool announce; +static void init_completion(unsigned *c) { *c=0; } +static unsigned msecs_to_jiffies(unsigned ms) { return ms; } +static unsigned wait_for_completion_timeout(unsigned *c,unsigned timeout) +{ assert(timeout==1000);waits++;return *c; } +static struct apple_dcp_afkep *afk_init(struct apple_dcp *d,unsigned ep,const void *ops) +{ + assert(ops==dpavservep_ops); + endpoint=(struct apple_dcp_afkep){.dcp=d,.endpoint=ep}; + return init_error ? ERR_PTR(init_error) : &endpoint; +} +static int afk_start(struct apple_dcp_afkep *ep) +{ + starts++; + assert(ep==&endpoint&&ep->match_epic_name); + if(announce&&!start_error) { + ep->dcp->dcpavserv.enabled=true; + ep->dcp->dcpavserv.enable_completion=1; + } + return start_error; +} +static void afk_shutdown(struct apple_dcp_afkep *ep) +{ assert(ep==&endpoint);shutdowns++; } +static void dpavservep_detach(struct apple_dcp *d) +{ detaches++;d->dcpavserv.enabled=false; } +''' + block(source, 'int dpavservep_init(') + r''' +int main(void) +{ + for(unsigned v2=0;v2<2;v2++) { + for(unsigned early=0;early<2;early++) { + struct apple_dcp d={.hw={.iomfb_method_profile=v2?2:0}}; + announce=early;waits=warnings=starts=shutdowns=detaches=debug=0; + assert(dpavservep_init(&d)==0); + assert(starts==1&&waits==!v2&&!warnings&&!shutdowns&&!detaches); + assert(d.dcpavservep==&endpoint&&endpoint.wire_v2==v2&&!endpoint.inline_v2); + assert(endpoint.endpoint==(v2?DPAVSERV_ENDPOINT_V2:DPAVSERV_ENDPOINT)); + /* A missing optional service must not be fabricated as enabled. */ + assert(d.dcpavserv.enabled==early); + assert(debug==(!v2&&!early)); + const int errors[]={-ENOMEM,-ETIMEDOUT}; + for(unsigned i=0;i Date: Fri, 16 Oct 2026 04:31:43 +1100 Subject: [PATCH 1080/1579] usb: typec: tipd: Apply SAFE after ordinary USB role shutdown Ordinary Apple USB ports must stop DWC3 before the mux changes the shared PHY lanes. Early SAFE during removal or probe rollback instead reaches ATC with a live PIPE. Prepare display shutdown first and hold the SAFE transaction across role NONE, applying it after successful role shutdown. Let mux providers declare this ordering requirement and use one predicate for disconnect and removal. ATC declares it only for ordinary ports, so a fixed host still applies SAFE while its backend clock is available, even if its firmware graph exposes a no-op role switch. Other providers retain the existing early order. This keeps the fixed-host divergence from Asahi because its shared PIPE must be parked before the host clock disappears. An unregistered role switch returns -EOPNOTSUPP after controller teardown; accept that shutdown result only when it also reports role NONE. Keep unsupported requests on a live role as errors. On other errors abort without applying SAFE and retain recovery ownership. Reject an ordinary-port SAFE against a live PIPE with -EBUSY so the next probe cannot warn or change live lanes. The removal transaction remains open until generic disconnect finishes; the prepared flag prevents an early second application. Cover the ordering, fixed-host no-op switch, absent switch, errors and re-probe in host tests, and the live-PIPE guard in KUnit without MMIO. Fixes: 6f004361c61b ("usb: typec: serialize display teardown and preserve recovery ownership") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 39 ++++++++++ drivers/phy/apple/atc.c | 11 +++ drivers/usb/typec/mux.c | 18 +++++ drivers/usb/typec/mux.h | 1 + drivers/usb/typec/tipd/core.c | 40 +++++++--- drivers/usb/typec/tipd/tps6598x.h | 4 +- include/linux/usb/typec_mux.h | 8 ++ .../selftests/drivers/gpu/apple-dcp/Makefile | 2 +- .../drivers/gpu/apple-dcp/quiesce.py | 78 +++++++++++++++++-- .../selftests/drivers/gpu/apple-dcp/run.py | 3 +- .../drivers/gpu/apple-dcp/transaction.py | 6 +- 11 files changed, 188 insertions(+), 22 deletions(-) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index a8c0c57d09859..d04d8031b9c09 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -182,6 +182,44 @@ static void atcphy_test_update_takes_usb(struct kunit *test) KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -ENODEV); } +struct atcphy_test_configure { + unsigned int calls; + unsigned int lane_requests; +}; + +/* Count lane requests instead of touching MMIO, including stale USB3 requests. */ +static int atcphy_test_configure_stub(struct apple_atcphy *atcphy, enum atcphy_mode mode) +{ + struct kunit *test = kunit_get_current_test(); + struct atcphy_test_configure *config = test->priv; + + lockdep_assert_held(&atcphy->lock); + config->calls++; + if (mode != APPLE_ATCPHY_MODE_OFF) { + config->lane_requests++; + return -EINVAL; + } + return 0; +} + +/* Rebinding after a failed role shutdown must not touch a live PIPE. */ +static void atcphy_test_safe_live_pipe(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, false); + struct atcphy_test_configure config = {}; + + test->priv = &config; + kunit_activate_static_stub(test, atcphy_configure, atcphy_test_configure_stub); + atcphy->hw = &atcphy_hw_t8103; + atcphy->typec_mode = APPLE_ATCPHY_MODE_USB3; + mutex_init(&atcphy->lock); + KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_OFF), -EBUSY); + KUNIT_EXPECT_EQ(test, config.calls, 0U); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB3_DP); + KUNIT_EXPECT_EQ(test, atcphy->typec_mode, APPLE_ATCPHY_MODE_USB3); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_USB3); +} + static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_target_mode), KUNIT_CASE(atcphy_test_target_mode_fixed_hub), @@ -189,6 +227,7 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_noop_needs_work), KUNIT_CASE(atcphy_test_update_skips_usb), KUNIT_CASE(atcphy_test_update_takes_usb), + KUNIT_CASE(atcphy_test_safe_live_pipe), {} }; diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index e8f3b64b2a349..8c6967c2e0b62 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -22,6 +22,8 @@ * Author: Sven Peter */ +#include + #include #include #include @@ -2787,6 +2789,9 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) int ret = 0; u32 reg; + if (IS_ENABLED(CONFIG_PHY_APPLE_ATC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(atcphy_configure, atcphy, mode); + lockdep_assert_held(&atcphy->lock); if (atcphy->pipe_pending) return -EBUSY; @@ -3714,6 +3719,11 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t return 0; } + /* A re-probe after failed role shutdown must leave the live PIPE alone. */ + if (!atcphy->fixed_usb2 && target_mode == APPLE_ATCPHY_MODE_OFF && + atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) + return -EBUSY; + switch (target_mode) { case APPLE_ATCPHY_MODE_TBT: case APPLE_ATCPHY_MODE_USB4: @@ -3824,6 +3834,7 @@ static int atcphy_probe_mux(struct apple_atcphy *atcphy) .drvdata = atcphy, .fwnode = atcphy->dev->fwnode, .set = atcphy_mux_set, + .safe_after_role = !atcphy->fixed_usb2, }; mux = typec_mux_register(atcphy->dev, &mux_desc); diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index a6dd8284ff1b7..b3ae7830f1ed4 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -315,6 +315,23 @@ static void typec_mux_context_put(struct typec_mux_context *context) kfree(context); } +/* Immutable provider policy; the handle pins each provider until put. */ +bool typec_mux_safe_after_role(struct typec_mux *mux) +{ + unsigned int i; + + if (IS_ERR_OR_NULL(mux)) + return false; + + for (i = 0; i < mux->num_mux_devs; i++) { + if (mux->mux_devs[i]->safe_after_role) + return true; + } + + return false; +} +EXPORT_SYMBOL_GPL(typec_mux_safe_after_role); + static int mux_fwnode_match(struct device *dev, const void *fwnode) { if (!is_typec_mux_dev(dev)) @@ -636,6 +653,7 @@ typec_mux_register(struct device *parent, const struct typec_mux_desc *desc) mux_dev->finish = desc->finish; mux_dev->commit = desc->commit; mux_dev->abort = desc->abort; + mux_dev->safe_after_role = desc->safe_after_role; device_initialize(&mux_dev->dev); mux_dev->dev.parent = parent; diff --git a/drivers/usb/typec/mux.h b/drivers/usb/typec/mux.h index 42e44211fd553..823c5529ca675 100644 --- a/drivers/usb/typec/mux.h +++ b/drivers/usb/typec/mux.h @@ -20,6 +20,7 @@ struct typec_mux_dev { typec_mux_notify_fn_t commit; typec_mux_notify_fn_t abort; bool removed; + bool safe_after_role; }; #define to_typec_switch_dev(_dev_) container_of(_dev_, struct typec_switch_dev, dev) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index ca6cc92c8b425..38d13adc35233 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -226,7 +226,9 @@ static int tps6598x_set_data_role(struct tps6598x *tps, role_val = USB_ROLE_NONE; ret = usb_role_switch_set_role(tps->role_sw, role_val); - if (ret) + /* An unregistered role switch has already stopped its controller. */ + if (ret && (connected || ret != -EOPNOTSUPP || + usb_role_switch_get_role(tps->role_sw) != USB_ROLE_NONE)) return ret; typec_set_data_role(tps->port, role); return 0; @@ -288,6 +290,12 @@ static void tps6598x_drop_partner(struct tps6598x *tps, u32 status) typec_set_vconn_role(tps->port, TPS_STATUS_TO_TYPEC_VCONN(status)); } +/* The mux provider knows whether this is a fixed host or a switched PHY. */ +static bool tps6598x_safe_after_role(struct tps6598x *tps) +{ + return tps->role_sw && typec_mux_safe_after_role(tps->mux); +} + static int tps6598x_disconnect(struct tps6598x *tps, u32 status) { struct typec_mux_state safe = { .mode = TYPEC_STATE_SAFE }; @@ -301,13 +309,16 @@ static int tps6598x_disconnect(struct tps6598x *tps, u32 status) ret = typec_mux_begin(tps->mux, &safe); if (ret) return ret; - /* Park while the host still supplies the PIPE backend clock. */ - ret = typec_mux_apply(tps->mux); - if (ret) - return ret; + /* Leave shared PHY shutdown to its USB role consumer. */ + if (!tps6598x_safe_after_role(tps)) { + ret = typec_mux_apply(tps->mux); + if (ret) + return ret; + } } ret = tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), false); if (ret) { + /* Keep recovery ownership; a live PHY must not receive SAFE. */ typec_mux_abort(tps->mux); return ret; } @@ -2046,12 +2057,21 @@ static int cd321x_remove(struct tps6598x *tps) ret = typec_mux_begin(tps->mux, &safe); if (ret) return ret; - /* Park while the host still supplies the PIPE backend clock. */ - ret = typec_mux_apply(tps->mux); - if (ret) - return ret; + /* + * Ordinary PHYs need role shutdown before SAFE. A fixed host must park + * now while its controller supplies the PIPE backend clock, even when + * the firmware graph exposes a no-op role switch for that host. + */ + if (!tps6598x_safe_after_role(tps)) { + ret = typec_mux_apply(tps->mux); + if (ret) + return ret; + } ret = usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); - if (ret) { + /* An unregistered switch means the controller is already stopped. */ + if (ret && (ret != -EOPNOTSUPP || + usb_role_switch_get_role(tps->role_sw) != USB_ROLE_NONE)) { + /* Keep recovery ownership; a live PHY must not receive SAFE. */ typec_mux_abort(tps->mux); return ret; } diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 744efb2bdbdef..2d3bcb513b4a4 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -330,8 +330,8 @@ struct tps6598x { struct typec_capability typec_cap; struct typec_mux *mux; /* - * Set by ->remove() once it has parked the mux in SAFE, so that - * tps6598x_disconnect() only ends that mux transaction. + * Set by ->remove() while its SAFE transaction is still open, so that + * tps6598x_disconnect() ends it after shutting down the USB role. */ bool remove_safe_prepared; diff --git a/include/linux/usb/typec_mux.h b/include/linux/usb/typec_mux.h index a18e4e0141adf..185da86c43d25 100644 --- a/include/linux/usb/typec_mux.h +++ b/include/linux/usb/typec_mux.h @@ -103,6 +103,8 @@ struct typec_mux_desc { typec_mux_set_fn_t finish; typec_mux_notify_fn_t commit; typec_mux_notify_fn_t abort; + /* SAFE needs the USB role consumer to stop the shared PHY first. */ + bool safe_after_role; }; #if IS_ENABLED(CONFIG_TYPEC) @@ -114,6 +116,7 @@ void typec_mux_put(struct typec_mux *mux); int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state *state); /* Apply the providers while retaining the transaction and ownership. */ int typec_mux_apply(struct typec_mux *mux); +bool typec_mux_safe_after_role(struct typec_mux *mux); int typec_mux_end(struct typec_mux *mux); void typec_mux_abort(struct typec_mux *mux); int typec_mux_prepare(struct typec_mux *mux, struct typec_mux_state *state); @@ -142,6 +145,11 @@ static inline int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state static inline int typec_mux_apply(struct typec_mux *mux) { return 0; } +static inline bool typec_mux_safe_after_role(struct typec_mux *mux) +{ + return false; +} + static inline int typec_mux_end(struct typec_mux *mux) { return 0; } static inline void typec_mux_abort(struct typec_mux *mux) {} diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile index bbb39cada55bf..619da7189c114 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT TEST_PROGS := run.sh -TEST_FILES := run.py common.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py boot.py dwc3 README +TEST_FILES := run.py common.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py boot.py quiesce.py transaction.py accounting.py dwc3 README include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py index 6e7b9f69b2a25..e422e9acfa0ae 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -12,6 +12,7 @@ def make_unit(source, parent=False, reference=None): #include #include #include +#include #include #include typedef unsigned u32; @@ -25,6 +26,7 @@ def make_unit(source, parent=False, reference=None): #define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) #define to_platform_device(p) (p) #define typec_mux_get_drvdata(m) ((m)->data) +#define dev_dbg(...) ((void)0) #define dev_warn(...) ((void)0) #define dev_info(...) ((void)0) #define dev_name(p) "device" @@ -38,7 +40,7 @@ def make_unit(source, parent=False, reference=None): #define TPS_STATUS_TO_TYPEC_VCONN(s) (s) #define TPS_STATUS_TO_TYPEC_DATAROLE(s) (s) struct typec_mux_state { void *alt; unsigned long mode; void *data; }; -struct typec_mux_dev { bool removed; void (*commit)(struct typec_mux_dev *,struct typec_mux_state *); void (*abort)(struct typec_mux_dev *,struct typec_mux_state *); int (*finish)(struct typec_mux_dev *,struct typec_mux_state *); void *data; int (*set)(struct typec_mux_dev *,struct typec_mux_state *); int (*prepare)(struct typec_mux_dev *,struct typec_mux_state *); }; +struct typec_mux_dev { bool removed, safe_after_role; void (*commit)(struct typec_mux_dev *,struct typec_mux_state *); void (*abort)(struct typec_mux_dev *,struct typec_mux_state *); int (*finish)(struct typec_mux_dev *,struct typec_mux_state *); void *data; int (*set)(struct typec_mux_dev *,struct typec_mux_state *); int (*prepare)(struct typec_mux_dev *,struct typec_mux_state *); }; struct task_struct { int unused; }; static _Thread_local struct task_struct this_task; #define current (&this_task) @@ -65,6 +67,8 @@ def make_unit(source, parent=False, reference=None): static struct apple_dcp_typec_port *sole_port(void) { return &port; } static char events[100];static unsigned count; static int power_error,phy_error,tbt_error,role_error,last_error,phy_calls,reset_calls,gpio_calls,role_calls,stops,unregisters; +static bool ordinary_port,controller_active,phy_off,safe_noop; + static void event(char c) { assert(count+1external; } static int dcp_dptx_disconnect_oob(void *dev,u32 index) { (void)dev;assert(index==0);event('Q');stops++;if(power_error){display.external_error=power_error;return power_error;}display.external_cable=false;display.external_stopped=true;display.dptxport[0].connected=false;display.typec_cable_connected=false;connector.connected=false;return 0; } @@ -85,7 +89,8 @@ def make_unit(source, parent=False, reference=None): struct typec_thunderbolt_switch_data { int state; }; static void cd321x_stop_work(struct cd321x *c) { (void)c;event('W'); } static int typec_thunderbolt_switch_set(void *p,void *s) { (void)p;(void)s;event('B');return tbt_error; } -static int usb_role_switch_set_role(void *p,int r) { (void)p;assert(r==USB_ROLE_NONE);event('R');role_calls++;return role_error; } +static int usb_role_switch_set_role(void *p,int r) { (void)p;assert(r==USB_ROLE_NONE);if(ordinary_port&&!role_error){controller_active=false;phy_off=true;}event('R');role_calls++;return role_error; } +static enum usb_role __attribute__((unused)) usb_role_switch_get_role(void *p) { (void)p;return controller_active?USB_ROLE_HOST:USB_ROLE_NONE; } static void typec_unregister_partner(void *p) { (void)p;event('P'); } #define typec_set_pwr_opmode(...) ((void)0) #define typec_set_pwr_role(...) ((void)0) @@ -102,9 +107,12 @@ def make_unit(source, parent=False, reference=None): static void gpiod_set_value_cansleep(void *p,int v) { (void)p;assert(v==1);gpio_calls++;event('G'); } '''.replace('REMOVE_RETURN','void' if parent else 'int') - functions='\n'.join(block(mux, marker) for marker in ('static void typec_mux_unlock(', 'void typec_mux_abort(', 'int typec_mux_begin(', 'int typec_mux_apply(', 'int typec_mux_end(', 'int typec_mux_prepare(')) if not parent else block((reference/'drivers/usb/typec/mux.c').read_text(),'int typec_mux_prepare(')+'\n' + policy=block(mux,'bool typec_mux_safe_after_role(')+'\n' if 'bool typec_mux_safe_after_role(' in mux else '' + functions=policy+'\n'.join(block(mux, marker) for marker in ('static void typec_mux_unlock(', 'void typec_mux_abort(', 'int typec_mux_begin(', 'int typec_mux_apply(', 'int typec_mux_end(', 'int typec_mux_prepare(')) if not parent else block((reference/'drivers/usb/typec/mux.c').read_text(),'int typec_mux_prepare(')+'\n' functions+=block(mux,'int typec_mux_set(')+'\n'+block(dcp,'static int dcp_typec_route_prepare(')+'\n'+block(dcp,'static int dcp_typec_route_finish(')+'\n'+block(dcp,'static void dcp_typec_route_commit(')+'\n' functions+=block(dcp,'static struct apple_dcp_typec_route *\ndcp_typec_rebalance_locked(')+'\n' + if 'static bool tps6598x_safe_after_role(' in core: + functions+=block(core,'static bool tps6598x_safe_after_role(')+'\n' if not parent:functions+=block(core,'static void tps6598x_drop_partner(')+'\n'+block(core,'static int tps6598x_set_data_role(')+'\n' functions+=block(core,'static '+('void' if parent else 'int')+' tps6598x_disconnect(')+'\n'+block(core,'static '+('void' if parent else 'int')+' cd321x_remove(')+'\n'+block(core,'void tipd_remove(')+'\n' controls=r''' @@ -113,12 +121,13 @@ def make_unit(source, parent=False, reference=None): static struct typec_port class_port; static struct tipd_data data; -static int phy_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;assert(s->mode==TYPEC_STATE_SAFE);event('L');phy_calls++;return phy_error; } +static int phy_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;assert(s->mode==TYPEC_STATE_SAFE);event('L');phy_calls++;if(ordinary_port&&controller_active)return -EBUSY;if(ordinary_port){assert(phy_off);safe_noop=true;}return phy_error; } static int display_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;event('S');if(port.owner&&display.external&&connector.connected) {int r=dcp_dptx_disconnect_oob(display.dev,0);if(r)return r;}if(port.safe_pending)return 0;port.owner=NULL;return 0; } static void unregister_port(struct tps6598x *t) { (void)t;unregisters++;event('U'); } static int reset(struct tps6598x *t) { (void)t;reset_calls++;event('X');return 0; } static void initial(struct cd321x *c) { count=0;events[0]=0;power_error=phy_error=tbt_error=role_error=last_error=phy_calls=reset_calls=gpio_calls=role_calls=stops=unregisters=deactivate_calls=0; + ordinary_port=false;controller_active=true;phy_off=safe_noop=false; connector=(struct connector){.connected=true};display=(struct apple_dcp){.external=true,.external_cable=true,.dptxport={{.connected=true}},.typec_cable_connected=true,.typec_connector=&connector}; port=(struct apple_dcp_typec_port){.owner=&route,.applied_valid=true,.dp_wanted=true,.dp_hpd=true,.hpd=true};route=(struct apple_dcp_typec_route){.port=&port,.dcp=&display,.selected=true}; phy_provider=(struct typec_mux_dev){.set=phy_set};display_provider=(struct typec_mux_dev){.data=&route,.set=display_set,.prepare=dcp_typec_route_prepare,.finish=dcp_typec_route_finish,.commit=dcp_typec_route_commit}; @@ -130,7 +139,62 @@ def make_unit(source, parent=False, reference=None): int main(int argc,char **argv) { (void)argc;struct cd321x c;struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE}; initial(&c); - if(argv[1][0]=='o') {tipd_remove(&c.tps);fprintf(stderr,"ordered remove: %s\n",events);assert(position('Q')>=0&&position('Q')=0&&position('Q')=0&&position('Q')=0&&position('Q')tps.irq=1;data.register_port=register_port;data.read_data_status=read_data_status;data.connect=connect; + initial(c);ordinary_port=true;phy_provider.safe_after_role=true;c->tps.irq=1;data.register_port=register_port;data.read_data_status=read_data_status;data.connect=connect; connect_error=irq_error=register_error=node_puts=mux_get_error=0; } int main(void) { @@ -337,7 +337,7 @@ def probe(root): probe_initial(&c);irq_error=-ENOSPC;phy_error=-ETIMEDOUT; assert(tipd_init(&c.tps)==-ENOSPC&&reset_calls==0&&gpio_calls==0&&port.owner==&route&&port.safe_pending&&unregisters==1&&node_puts==1); probe_initial(&c);connect_error=-EAGAIN;role_error=-EBUSY; - assert(tipd_init(&c.tps)==-EAGAIN&&reset_calls==0&&gpio_calls==0&&phy_calls==1&&port.owner==&route&&port.safe_pending); + assert(tipd_init(&c.tps)==-EAGAIN&&reset_calls==0&&gpio_calls==0&&phy_calls==0&&port.owner==&route&&port.safe_pending); probe_initial(&c);irq_error=-ENOSPC; assert(tipd_init(&c.tps)==-ENOSPC&&reset_calls==1&&!port.owner&&!port.safe_pending&&unregisters==1); probe_initial(&c);mux_get_error=-ENODEV; @@ -392,7 +392,7 @@ def clock_order(root): unit = unit.replace(block(unit, 'int main('), block(unit, 'int main(').replace('int main(', 'int __attribute__((unused)) prior_main(', 1), 1) unit = unit.replace('static int power_error,', 'static bool backend_clock=true,parked;\nstatic int power_error,') unit = unit.replace("event('R');role_calls++;return role_error;", "event('R');role_calls++;if(!role_error)backend_clock=false;return role_error;", 1) - unit = unit.replace("event('L');phy_calls++;return phy_error;", "event('L');phy_calls++;if(parked)return 0;if(!backend_clock)return -ENODEV;if(!phy_error)parked=true;return phy_error;", 1) + unit = unit.replace("event('L');phy_calls++;", "event('L');phy_calls++;if(parked)return 0;if(!backend_clock)return -ENODEV;if(!phy_error)parked=true;", 1) unit += r""" int main(void) { (void)current;struct cd321x c;initial(&c); From 2757f43e143ed2851d01cde331df9631e15d4329 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 04:31:43 +1100 Subject: [PATCH 1081/1579] phy: apple: atc: Remember mux requests that keep the lane mode USB role shutdown can power the PHY off while its saved Type-C request is still USB3. A subsequent SAFE request needs no lane work, but both mux fast paths return without saving it. A later USB2 PHY initialization can then restore the old USB3 request instead of respecting SAFE. Record every accepted request whose lanes already match, both before acquiring USB runtime ownership and in the locked recheck. Preserve the capability checks, PIPE retries and fixed-hub no-op bypass. Test SAFE after independent shutdown on both paths and call the real USB2 initializer with a KUnit configuration stub that counts lane requests. Extend the host role mock through the production PHY power-off, mux recheck and USB2 init bodies, leaving only MMIO operations mocked. The host cases now verify that role NONE powers the PHY OFF, SAFE requires no lane work, and USB2 init cannot restore stale USB3 after SAFE. Fixes: 48836ec5a5b6 ("phy: apple: atc: Keep USB2 up when the port sits behind a fixed hub") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 56 +++++++++++++++ drivers/phy/apple/atc.c | 6 +- .../drivers/gpu/apple-dcp/quiesce.py | 71 ++++++++++++++++--- 3 files changed, 124 insertions(+), 9 deletions(-) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index d04d8031b9c09..d3a760a7d62fd 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -182,6 +182,39 @@ static void atcphy_test_update_takes_usb(struct kunit *test) KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -ENODEV); } +/* USB shutdown can turn the lanes off before the SAFE mux request arrives. */ +static struct apple_atcphy *atcphy_test_alloc_powered_off(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, false); + + atcphy->hw = &atcphy_hw_t8103; + atcphy->mode = APPLE_ATCPHY_MODE_OFF; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + atcphy->typec_mode = APPLE_ATCPHY_MODE_USB3; + mutex_init(&atcphy->lock); + + return atcphy; +} + +static void atcphy_test_safe_after_poweroff(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_powered_off(test); + struct typec_mux_state safe = { .mode = TYPEC_STATE_SAFE }; + + KUNIT_EXPECT_EQ(test, atcphy_mux_update(atcphy, &safe), 0); + KUNIT_EXPECT_EQ(test, atcphy->typec_mode, APPLE_ATCPHY_MODE_OFF); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); +} + +/* The locked recheck must also record SAFE if shutdown raced the mode lookup. */ +static void atcphy_test_locked_safe_after_poweroff(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_powered_off(test); + + KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_OFF), 0); + KUNIT_EXPECT_EQ(test, atcphy->typec_mode, APPLE_ATCPHY_MODE_OFF); +} + struct atcphy_test_configure { unsigned int calls; unsigned int lane_requests; @@ -220,6 +253,26 @@ static void atcphy_test_safe_live_pipe(struct kunit *test) KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_USB3); } +static void atcphy_test_usb2_init_after_safe(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_powered_off(test); + struct typec_mux_state safe = { .mode = TYPEC_STATE_SAFE }; + struct atcphy_test_configure config = {}; + struct phy *phy; + + phy = kunit_kzalloc(test, sizeof(*phy), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, phy); + phy_set_drvdata(phy, atcphy); + test->priv = &config; + kunit_activate_static_stub(test, atcphy_configure, atcphy_test_configure_stub); + + KUNIT_ASSERT_EQ(test, atcphy_mux_update(atcphy, &safe), 0); + KUNIT_EXPECT_EQ(test, atcphy_usb2_init(phy), 0); + KUNIT_EXPECT_EQ(test, config.calls, 1U); + KUNIT_EXPECT_EQ(test, config.lane_requests, 0U); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_OFF); +} + static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_target_mode), KUNIT_CASE(atcphy_test_target_mode_fixed_hub), @@ -227,7 +280,10 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_noop_needs_work), KUNIT_CASE(atcphy_test_update_skips_usb), KUNIT_CASE(atcphy_test_update_takes_usb), + KUNIT_CASE(atcphy_test_safe_after_poweroff), + KUNIT_CASE(atcphy_test_locked_safe_after_poweroff), KUNIT_CASE(atcphy_test_safe_live_pipe), + KUNIT_CASE(atcphy_test_usb2_init_after_safe), {} }; diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 8c6967c2e0b62..4da347276a231 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -3705,6 +3705,7 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t atcphy->mode, atcphy->mode_dirty); if (atcphy->mode == target_mode && !atcphy->mode_dirty) { + atcphy->typec_mode = target_mode; /* Retry a failed PIPE setup without disturbing the configured lanes. */ if (atcphy->fixed_usb2 && atcphy->host_active && (atcphy->pipe_state != atcphy_modes[target_mode].pipehandler_state || @@ -3805,8 +3806,11 @@ static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux * includes a whole enumeration with its timeouts, and would * hold up a DisplayPort update that changes neither. */ - if (atcphy_mux_is_noop(atcphy, target_mode)) + if (atcphy_mux_is_noop(atcphy, target_mode)) { + /* Record matching requests for the next PHY init to restore. */ + atcphy->typec_mode = target_mode; return 0; + } } ret = atcphy_usb3_pm_get(atcphy, &pm); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py index e422e9acfa0ae..fdab2d0b2cdac 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -6,6 +6,7 @@ def make_unit(source, parent=False, reference=None): reference=source if reference is None else reference core=(source/'drivers/usb/typec/tipd/core.c').read_text();mux=(source/'drivers/usb/typec/mux.c').read_text() + atc=(source/'drivers/phy/apple/atc.c').read_text() dcp=(reference/'drivers/gpu/drm/apple/dcp.c').read_text() prefix=r''' #include @@ -68,7 +69,10 @@ def make_unit(source, parent=False, reference=None): static char events[100];static unsigned count; static int power_error,phy_error,tbt_error,role_error,last_error,phy_calls,reset_calls,gpio_calls,role_calls,stops,unregisters; static bool ordinary_port,controller_active,phy_off,safe_noop; - +static int ordinary_power_off(void); +static int ordinary_safe(void); +static void ordinary_initial(void); +static void ordinary_usb2_check(void); static void event(char c) { assert(count+1external; } static int dcp_dptx_disconnect_oob(void *dev,u32 index) { (void)dev;assert(index==0);event('Q');stops++;if(power_error){display.external_error=power_error;return power_error;}display.external_cable=false;display.external_stopped=true;display.dptxport[0].connected=false;display.typec_cable_connected=false;connector.connected=false;return 0; } @@ -89,7 +93,7 @@ def make_unit(source, parent=False, reference=None): struct typec_thunderbolt_switch_data { int state; }; static void cd321x_stop_work(struct cd321x *c) { (void)c;event('W'); } static int typec_thunderbolt_switch_set(void *p,void *s) { (void)p;(void)s;event('B');return tbt_error; } -static int usb_role_switch_set_role(void *p,int r) { (void)p;assert(r==USB_ROLE_NONE);if(ordinary_port&&!role_error){controller_active=false;phy_off=true;}event('R');role_calls++;return role_error; } +static int usb_role_switch_set_role(void *p,int r) { (void)p;assert(r==USB_ROLE_NONE);if(ordinary_port&&!role_error){int ret=ordinary_power_off();if(ret)return ret;controller_active=false;phy_off=true;}event('R');role_calls++;return role_error; } static enum usb_role __attribute__((unused)) usb_role_switch_get_role(void *p) { (void)p;return controller_active?USB_ROLE_HOST:USB_ROLE_NONE; } static void typec_unregister_partner(void *p) { (void)p;event('P'); } #define typec_set_pwr_opmode(...) ((void)0) @@ -115,19 +119,70 @@ def make_unit(source, parent=False, reference=None): functions+=block(core,'static bool tps6598x_safe_after_role(')+'\n' if not parent:functions+=block(core,'static void tps6598x_drop_partner(')+'\n'+block(core,'static int tps6598x_set_data_role(')+'\n' functions+=block(core,'static '+('void' if parent else 'int')+' tps6598x_disconnect(')+'\n'+block(core,'static '+('void' if parent else 'int')+' cd321x_remove(')+'\n'+block(core,'void tipd_remove(')+'\n' + ordinary_phy=r''' +/* Actual ordinary-port ATC power-off, mux recheck and USB2 init; MMIO leaves. */ +#undef WARN_ON_ONCE +#define WARN_ON_ONCE(x) ({ bool hit=(x);phy_warns+=hit;hit; }) +#define fallthrough __attribute__((fallthrough)) +enum atcphy_mode { APPLE_ATCPHY_MODE_OFF, APPLE_ATCPHY_MODE_USB2, + APPLE_ATCPHY_MODE_USB3, APPLE_ATCPHY_MODE_USB3_DP, APPLE_ATCPHY_MODE_DP, + APPLE_ATCPHY_MODE_TBT, APPLE_ATCPHY_MODE_USB4 }; +enum atcphy_pipehandler_state { ATCPHY_PIPEHANDLER_STATE_DUMMY, ATCPHY_PIPEHANDLER_STATE_USB3 }; +struct apple_atcphy { + void *dev; int lock; + const struct { bool has_usb4,has_t8140_dp; } *hw; + struct { void *usb2phy; } regs; + bool fixed_usb2,host_active,mode_dirty,pipe_pending,usb_off_pending,ss_tunables,dp_tunables; + enum atcphy_mode mode,typec_mode; + enum atcphy_pipehandler_state pipe_state; +}; +struct phy { struct apple_atcphy *data; }; +#define phy_get_drvdata(p) ((p)->data) +#define USB2PHY_SIG 0 +#define USB2PHY_SIG_HOST 1 +static struct { enum atcphy_pipehandler_state pipehandler_state; bool enable_dp_aux; } atcphy_modes[7]; +static struct apple_atcphy ordinary_phy; +static struct phy usb_phy={.data=&ordinary_phy}; +static unsigned phy_warns,configure_calls,lane_requests; +static int atcphy_park_pipehandler(struct apple_atcphy *p) { p->pipe_state=ATCPHY_PIPEHANDLER_STATE_DUMMY;return 0; } +static int atcphy_configure_pipehandler(struct apple_atcphy *p,bool on) { (void)p;(void)on;assert(false);return -EINVAL; } +static int atcphy_configure(struct apple_atcphy *p,enum atcphy_mode mode) { configure_calls++;lane_requests+=mode!=APPLE_ATCPHY_MODE_OFF;p->mode=mode;p->mode_dirty=false;return 0; } +static void set32(void *p,unsigned v) { (void)p;(void)v;assert(false); } +static void atcphy_usb2_power_on(struct apple_atcphy *p) { (void)p;assert(false); } +''' + '\n'.join(block(atc, marker) for marker in ( + 'static int atcphy_usb3_power_off(', 'static int atcphy_mux_set_locked(', + 'static int atcphy_usb2_init(')) + r''' +static int ordinary_power_off(void) { return atcphy_usb3_power_off(&usb_phy); } +static void ordinary_initial(void) { + ordinary_phy=(struct apple_atcphy){.mode=APPLE_ATCPHY_MODE_USB3,.typec_mode=APPLE_ATCPHY_MODE_USB3,.pipe_state=ATCPHY_PIPEHANDLER_STATE_USB3}; + phy_warns=configure_calls=lane_requests=0; +} +static int ordinary_safe(void) { + unsigned before=configure_calls; + int ret=atcphy_mux_set_locked(&ordinary_phy,APPLE_ATCPHY_MODE_OFF); + assert(!phy_warns); + if(!ret){assert(ordinary_phy.mode==APPLE_ATCPHY_MODE_OFF);assert(configure_calls==before);} + return ret; +} +static void ordinary_usb2_check(void) { + unsigned before=lane_requests; + assert(atcphy_usb2_init(&usb_phy)==0); + assert(ordinary_phy.typec_mode==APPLE_ATCPHY_MODE_OFF&&lane_requests==before); +} +''' controls=r''' static struct typec_mux_dev phy_provider,display_provider; static struct typec_mux mux; static struct typec_port class_port; static struct tipd_data data; -static int phy_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;assert(s->mode==TYPEC_STATE_SAFE);event('L');phy_calls++;if(ordinary_port&&controller_active)return -EBUSY;if(ordinary_port){assert(phy_off);safe_noop=true;}return phy_error; } +static int phy_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;assert(s->mode==TYPEC_STATE_SAFE);event('L');phy_calls++;if(ordinary_port){int ret=ordinary_safe();if(ret)return ret;assert(phy_off);safe_noop=true;}return phy_error; } static int display_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;event('S');if(port.owner&&display.external&&connector.connected) {int r=dcp_dptx_disconnect_oob(display.dev,0);if(r)return r;}if(port.safe_pending)return 0;port.owner=NULL;return 0; } static void unregister_port(struct tps6598x *t) { (void)t;unregisters++;event('U'); } static int reset(struct tps6598x *t) { (void)t;reset_calls++;event('X');return 0; } static void initial(struct cd321x *c) { count=0;events[0]=0;power_error=phy_error=tbt_error=role_error=last_error=phy_calls=reset_calls=gpio_calls=role_calls=stops=unregisters=deactivate_calls=0; - ordinary_port=false;controller_active=true;phy_off=safe_noop=false; + ordinary_port=false;controller_active=true;phy_off=safe_noop=false;ordinary_initial(); connector=(struct connector){.connected=true};display=(struct apple_dcp){.external=true,.external_cable=true,.dptxport={{.connected=true}},.typec_cable_connected=true,.typec_connector=&connector}; port=(struct apple_dcp_typec_port){.owner=&route,.applied_valid=true,.dp_wanted=true,.dp_hpd=true,.hpd=true};route=(struct apple_dcp_typec_route){.port=&port,.dcp=&display,.selected=true}; phy_provider=(struct typec_mux_dev){.set=phy_set};display_provider=(struct typec_mux_dev){.data=&route,.set=display_set,.prepare=dcp_typec_route_prepare,.finish=dcp_typec_route_finish,.commit=dcp_typec_route_commit}; @@ -153,7 +208,7 @@ def make_unit(source, parent=False, reference=None): if(!strcmp(argv[1],"ordinary-role-reprobe"))role_error=-EBUSY; if(!strncmp(argv[1],"ordinary-unsupported-",21))role_error=-EOPNOTSUPP; if(!strncmp(argv[1],"ordinary-unregistered-",22)) { - role_error=-EOPNOTSUPP;controller_active=false;phy_off=true; + role_error=-EOPNOTSUPP;assert(ordinary_power_off()==0);controller_active=false;phy_off=true; } if(!strcmp(argv[1],"ordinary-disconnect")||!strcmp(argv[1],"ordinary-unregistered-disconnect")||!strcmp(argv[1],"ordinary-unsupported-disconnect")) { int ret=tps6598x_disconnect(&c.tps,0); @@ -163,7 +218,7 @@ def make_unit(source, parent=False, reference=None): } else { assert(ret==0); assert(position('Q')>=0&&position('Q')=0&&position('Q') Date: Fri, 16 Oct 2026 06:50:57 +1100 Subject: [PATCH 1082/1579] drm/apple: afk: Export afk_service_retry() to the AFK service test afk_service_put() is an inline helper in afk.h that ends by calling afk_service_retry(), which lives in afk.c inside appledrm. The AFK service lifetime test is built as its own module and uses afk_service_put(), so with DRM_APPLE=m and DRM_APPLE_KUNIT_TEST=m (as KUNIT_ALL_TESTS=m selects) modpost fails: ERROR: modpost: "afk_service_retry" [drivers/gpu/drm/apple/afk-service-test.ko] undefined! Export the function to the KUnit test namespace and import that namespace in the test, as other drivers do for tests that live in their own module. The export is compiled out without CONFIG_KUNIT. Fixes: 7357e51053e4 ("drm/apple: scope deferred DPAV recovery to current interfaces") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk-service-test.c | 1 + drivers/gpu/drm/apple/afk.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/drivers/gpu/drm/apple/afk-service-test.c b/drivers/gpu/drm/apple/afk-service-test.c index a86df389deb17..1cf589ae7a417 100644 --- a/drivers/gpu/drm/apple/afk-service-test.c +++ b/drivers/gpu/drm/apple/afk-service-test.c @@ -238,4 +238,5 @@ static struct kunit_suite afk_service_suite = { }; kunit_test_suite(afk_service_suite); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 0d598980c96e4..6be30efbe8b35 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -13,6 +13,8 @@ #include #include +#include + #include "afk.h" #include "afkv2-codec.h" #include "trace.h" @@ -66,6 +68,7 @@ void afk_service_retry(struct apple_dcp_afkep *ep) queue_work(ep->wq, &ep->service_work); spin_unlock_irqrestore(&ep->lock, flags); } +EXPORT_SYMBOL_IF_KUNIT(afk_service_retry); static void afk_send(struct apple_dcp_afkep *ep, u64 message) { From 3bd76754f7836c7e927fcd9295d408f3ec358e2a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 07:27:23 +1100 Subject: [PATCH 1083/1579] soc: apple: sep: Refuse a damaged host-state store instead of erasing it Store::open() initialises the host-state store whenever the file is shorter than the store or its header does not match: it zeroes all 72 blocks and writes an empty superblock. That was by design while the store held records that a seed import restored after a reset. Since the SEP's anti-replay records moved to the shared xART gigalocker, though, the file holds Linux state that nothing regenerates: the Touch ID identity index and the lockout state. A truncated file, a damaged header or a store in a format this driver does not know is now erased on the next probe, together with the evidence needed to recover it. Initialise the store only when this open created the file and it is empty, and refuse an existing file that is short or has an unknown header without writing to it. That includes what an interrupted creation leaves: in general the driver cannot tell it from a damaged store, so the decision is the administrator's. The log line says when the file is empty or all zero: that is all an interrupted creation leaves, and such a file holds nothing to keep. Creating a new store is logged too: on a machine that has used Touch ID it means the old store was removed. Opening also has two ways to lose or leak a record of a store with a valid header. A pending write intent that names a used block makes it free that block, discarding the committed record, and a record length beyond its 32 KiB block makes read() allocate up to 4 GiB and return the following blocks. No writer of the format produces either; refuse them as well instead of acting on them. The on-disk format does not change. Fixes: 7165ae9d98bf ("soc: apple: use shared xART gigalocker") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/shim.h | 1 + drivers/soc/apple/shim.rs | 16 ++++++- drivers/soc/apple/store.rs | 84 ++++++++++++++++++++++++++++++++-- drivers/soc/apple/store_shim.c | 17 +++++++ 4 files changed, 113 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index f43e038fe4205..83839b9353691 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -42,6 +42,7 @@ void sep_hwrng_unregister(void *mem); /* -- store_shim.c ------------------------------------------------------- */ void *sep_store_open(const char *path); +void *sep_store_open_host(const char *path, bool *created); void *sep_store_open_trunc(const char *path); void *sep_store_open_ro(const char *path); void *sep_store_open_block(const char *path, int writable); diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index 4e25060d15d25..7e7f489e59481 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -7,6 +7,7 @@ use kernel::prelude::*; extern "C" { fn sep_store_open(path: *const c_char) -> *mut c_void; + fn sep_store_open_host(path: *const c_char, created: *mut bool) -> *mut c_void; fn sep_store_open_trunc(path: *const c_char) -> *mut c_void; fn sep_store_open_ro(path: *const c_char) -> *mut c_void; fn sep_store_open_block(path: *const c_char, writable: c_int) -> *mut c_void; @@ -29,7 +30,7 @@ pub(crate) fn random_bytes(buf: &mut [u8]) -> Result<()> { /// Backing-store file handle. /// /// # Invariants -/// `handle` is a live `struct file *` from `sep_store_open`. +/// `handle` is a live `struct file *` from one of the store-open shims. pub(crate) struct StoreFile { handle: *mut c_void, } @@ -56,6 +57,19 @@ impl StoreFile { Ok(StoreFile { handle }) } + /// Opens the file like [`StoreFile::open`], and also reports whether this + /// call created it. + pub(crate) fn open_host(path: &CStr) -> Result<(StoreFile, bool)> { + let mut created = false; + // SAFETY: `path` is NUL-terminated and `created` is valid for writes + // for the duration of the call. The shim returns a live file or an + // error pointer. + let handle = unsafe { + kernel::error::from_err_ptr(sep_store_open_host(path.as_char_ptr(), &mut created))? + }; + Ok((StoreFile { handle }, created)) + } + pub(crate) fn open_trunc(path: &CStr) -> Result { // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. let handle = unsafe { sep_store_open_trunc(path.as_char_ptr()) }; diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index dc37025ce3b8a..03bd6f1bf0a01 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -73,6 +73,20 @@ const SLOT_ENTRY_SIZE: usize = 24; const INTENT_NONE: u8 = 0; const INTENT_WRITE: u8 = 1; +/// Why an existing file is refused. It is logged, so it never carries record +/// contents. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Damage { + /// The file is shorter than the store. + Size, + /// The magic, version or geometry is not this format's. + Header, + /// The slot's record is longer than its block. + Slot(usize), + /// The pending write intent names a block that holds a committed record. + Intent, +} + // Not internally locked; the caller holds a mutex around all access. pub(crate) struct Store { file: shim::StoreFile, @@ -92,9 +106,34 @@ fn le64(buf: &[u8], off: usize) -> u64 { u64::from_le_bytes(v) } +/// Describes an existing file that holds no store as "empty" or "all zero". +/// That is all an interrupted creation leaves, and such a file holds nothing +/// to keep; anything else may be what is left of a store. Reads at most a +/// store's worth of the file. +fn describe(file: &shim::StoreFile, size: u64) -> Option<&'static str> { + if size == 0 { + return Some("empty"); + } + if size > STORE_SIZE as u64 { + return None; + } + let mut buf = KVec::new(); + buf.resize(BLOCK_SIZE, 0, GFP_KERNEL).ok()?; + let mut off = 0; + while off < size { + let len = (size - off).min(BLOCK_SIZE as u64) as usize; + file.read_exact(off, &mut buf[..len]).ok()?; + if buf[..len].iter().any(|&b| b != 0) { + return None; + } + off += len as u64; + } + Some("all zero") +} + impl Store { pub(crate) fn open() -> Result { - let file = shim::StoreFile::open(STORE_PATH)?; + let (file, created) = shim::StoreFile::open_host(STORE_PATH)?; let mut store = Store { file, @@ -104,11 +143,18 @@ impl Store { fresh: false, }; + // Only a new, empty file is initialised. An existing file may hold the + // only copy of the host state, so one that this driver cannot use is + // refused and left as it is. let size = store.file.size()?; - if size < STORE_SIZE as u64 { + if created && size == 0 { store.create()?; + pr_info!("created a new host-state store\n"); return Ok(store); } + if size < STORE_SIZE as u64 { + return Err(store.refuse(Damage::Size, size)); + } let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; @@ -119,8 +165,7 @@ impl Store { || le32(&sb, SB_SLOT_COUNT) as usize != SLOT_COUNT || le32(&sb, SB_BLOCK_SIZE) as usize != BLOCK_SIZE { - store.create()?; - return Ok(store); + return Err(store.refuse(Damage::Header, size)); } store.generation = le64(&sb, SB_GENERATION); @@ -134,6 +179,10 @@ impl Store { uuid, len: le32(&sb, off + 18), }; + // A longer record would be read from the blocks of the next slots. + if store.slots[i].used && store.slots[i].len as usize > MAX_VALUE { + return Err(store.refuse(Damage::Slot(i), size)); + } } // Crash recovery: writes are copy-on-write, so discard the intent, never complete it. @@ -142,6 +191,12 @@ impl Store { store.recovered = true; let slot = u16::from_le_bytes([sb[SB_INTENT_SLOT], sb[SB_INTENT_SLOT + 1]]) as usize; if intent_kind == INTENT_WRITE && slot < SLOT_COUNT { + // The intent names the free block that was being filled. One + // that names a committed record is damage, and freeing that + // block would discard the record. + if store.slots[slot].used { + return Err(store.refuse(Damage::Intent, size)); + } store.slots[slot] = Slot::FREE; } store.commit()?; @@ -150,6 +205,27 @@ impl Store { Ok(store) } + /// Logs why the existing file is refused, and returns the error. Nothing + /// is written to the file. + fn refuse(&self, damage: Damage, size: u64) -> Error { + let what = match damage { + Damage::Size | Damage::Header => describe(&self.file, size), + _ => None, + }; + match what { + Some(what) => pr_err!( + "host-state store failed validation ({:?}, {}), refusing it\n", + damage, + what + ), + None => pr_err!( + "host-state store failed validation ({:?}), refusing it\n", + damage + ), + } + EIO + } + fn create(&mut self) -> Result<()> { self.slots = [Slot::FREE; SLOT_COUNT]; self.generation = 1; diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c index bdfd5dfbb6391..4c89f92eb5f13 100644 --- a/drivers/soc/apple/store_shim.c +++ b/drivers/soc/apple/store_shim.c @@ -51,6 +51,23 @@ void *sep_store_open(const char *path) return IS_ERR(f) ? NULL : f; } +/* + * As sep_store_open(), but reports whether this call created the file, so that + * only a new file is ever initialised. Returns an error pointer on failure. + */ +void *sep_store_open_host(const char *path, bool *created) +{ + struct file *f; + + *created = false; + f = open_as_kernel(path, O_RDWR | O_CREAT | O_EXCL | O_LARGEFILE, 0600); + if (PTR_ERR_OR_ZERO(f) == -EEXIST) + return open_as_kernel(path, O_RDWR | O_LARGEFILE, 0); + if (!IS_ERR(f)) + *created = true; + return f; +} + /* * Opens truncated to empty, for callers that rewrite the whole file each time, * so a shorter record cannot leave stale trailing bytes from a larger one. From 7fd578b198e007701218f87f34a562545e54be29 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 11:53:45 +1100 Subject: [PATCH 1084/1579] usb: typec: tipd: Stop waiting for a USB4 router that never binds On the M1 and M2 machines every CD321x Type-C connector is linked through its port@2 endpoint to the USB4 router serving that port, and cd321x_register_port() looks up the router's Thunderbolt switch with fwnode_typec_thunderbolt_switch_get(). The lookup returns -EPROBE_DEFER until the router driver (USB4_APPLE_SOC) has registered the switch. So when that driver is not built, or the router fails to probe, every linked PD controller stays deferred for good: no Type-C port, no partners, no USB role switching and no DisplayPort alternate mode, although the router only takes part in USB4 and Thunderbolt links. Kernels configured for the M1 Pro and M2 from configurations that predate the router driver show exactly that: only the connector-less MagSafe controller binds, and the others stay in devices_deferred with no recorded reason. Handle the switch like other providers that may never get a driver: keep deferring while the driver core still waits for missing dependencies, and once driver_deferred_probe_check_state() gives up, register the port without a switch, as for a connector with no router link. Ports whose router registers its switch in time behave as before. Fixes: fc76d0f20c29 ("usb: typec: tipd: Hook up tbt switch for Apple platforms") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/core.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index ca6cc92c8b425..e17d27392a424 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -1509,6 +1509,27 @@ static void cd321x_stop_work(struct cd321x *cd321x) cancel_delayed_work_sync(&cd321x->update_work); } +/* + * The USB4/Thunderbolt router is only needed once a USB4 or Thunderbolt link + * comes up. If its driver is not built or never binds, do not leave power + * delivery, USB and DisplayPort on this port without a driver as well: stop + * waiting for the router when the driver core stops waiting for missing + * dependencies. + */ +static struct typec_thunderbolt_switch * +cd321x_tbt_switch_get(struct device *dev, struct fwnode_handle *fwnode) +{ + struct typec_thunderbolt_switch *sw; + + sw = fwnode_typec_thunderbolt_switch_get(fwnode); + if (sw != ERR_PTR(-EPROBE_DEFER) || + driver_deferred_probe_check_state(dev) == -EPROBE_DEFER) + return sw; + + dev_info(dev, "USB4/Thunderbolt router unavailable, continuing without it\n"); + return NULL; +} + static int cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) { @@ -1538,7 +1559,7 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) if (IS_ERR(cd321x->connector_fwnode)) cd321x->connector_fwnode = NULL; - cd321x->tbt_switch = fwnode_typec_thunderbolt_switch_get(fwnode); + cd321x->tbt_switch = cd321x_tbt_switch_get(tps->dev, fwnode); if (IS_ERR(cd321x->tbt_switch)) { ret = PTR_ERR(cd321x->tbt_switch); goto err_put_mux; From 6b59e8a697dcb31b4d7ff91354d127f48ed4f0b4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 11:54:02 +1100 Subject: [PATCH 1085/1579] usb: typec: tipd: Add KUnit tests for the USB4 router lookup Test cd321x_tbt_switch_get() on a software node graph shaped like the M1 and M2 device trees: a connector whose port@2 is linked to a node with the thunderbolt-switch property. A connector without that link, and a router that has registered its switch, must not consult the driver core. A router that has not registered its switch must defer the port while the driver core keeps waiting, and must no longer hold the port back once it gives up. Move the driver core query into cd321x_tbt_switch_keep_waiting() so that the tests can redirect it with a static stub. Without KUnit both helpers stay static and the stub compiles away. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/Kconfig | 4 +- drivers/usb/typec/tipd/Makefile | 2 +- drivers/usb/typec/tipd/cd321x-tbt-test.c | 236 +++++++++++++++++++++++ drivers/usb/typec/tipd/core.c | 17 +- drivers/usb/typec/tipd/tps6598x.h | 6 + 5 files changed, 260 insertions(+), 5 deletions(-) create mode 100644 drivers/usb/typec/tipd/cd321x-tbt-test.c diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index a447a4622ea76..aae3e81cca848 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -29,11 +29,13 @@ config TYPEC_SN201202X module will be called sn201202x.ko. config TYPEC_TIPD_KUNIT_TEST - tristate "KUnit tests for CD321x resume state" if !KUNIT_ALL_TESTS + tristate "KUnit tests for CD321x" if !KUNIT_ALL_TESTS depends on KUNIT && TYPEC_TPS6598X_CORE default KUNIT_ALL_TESTS help Test suspend gating, fresh cable-state revalidation, bounded retries and removal during resume. The tests exercise the state transitions shared with the driver, including aborted and repeated sleep cycles. + Also test that a USB4/Thunderbolt router which never registers its + switch does not keep the port from probing. These tests run without accessing a physical Type-C controller. diff --git a/drivers/usb/typec/tipd/Makefile b/drivers/usb/typec/tipd/Makefile index 1a7605e7e57ec..198d57654b577 100644 --- a/drivers/usb/typec/tipd/Makefile +++ b/drivers/usb/typec/tipd/Makefile @@ -12,4 +12,4 @@ tps6598x-y := i2c.o obj-$(CONFIG_TYPEC_SN201202X) += sn201202x.o sn201202x-y := spmi.o -obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o +obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o cd321x-tbt-test.o diff --git a/drivers/usb/typec/tipd/cd321x-tbt-test.c b/drivers/usb/typec/tipd/cd321x-tbt-test.c new file mode 100644 index 0000000000000..63858db80d509 --- /dev/null +++ b/drivers/usb/typec/tipd/cd321x-tbt-test.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the CD321x USB4/Thunderbolt router lookup + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "tps6598x.h" + +/* + * A Type-C connector whose port@2 is linked to a USB4 router, as on the M1 + * and M2 machines, and a connector without such a link. + */ +static const struct software_node cd321x_test_router_ep; +static const struct software_node cd321x_test_connector_ep; + +static const struct property_entry cd321x_test_router_props[] = { + PROPERTY_ENTRY_BOOL("thunderbolt-switch"), + { } +}; + +static const struct software_node cd321x_test_router = { + .name = "cd321x-test-router", + .properties = cd321x_test_router_props, +}; + +static const struct software_node cd321x_test_router_port = { + .name = "port@1", + .parent = &cd321x_test_router, +}; + +static const struct software_node_ref_args cd321x_test_router_refs[] = { + SOFTWARE_NODE_REFERENCE(&cd321x_test_connector_ep), +}; + +static const struct property_entry cd321x_test_router_ep_props[] = { + PROPERTY_ENTRY_REF_ARRAY("remote-endpoint", cd321x_test_router_refs), + { } +}; + +static const struct software_node cd321x_test_router_ep = { + .name = "endpoint", + .parent = &cd321x_test_router_port, + .properties = cd321x_test_router_ep_props, +}; + +static const struct software_node cd321x_test_connector = { + .name = "cd321x-test-connector", +}; + +static const struct software_node cd321x_test_connector_port = { + .name = "port@2", + .parent = &cd321x_test_connector, +}; + +static const struct software_node_ref_args cd321x_test_connector_refs[] = { + SOFTWARE_NODE_REFERENCE(&cd321x_test_router_ep), +}; + +static const struct property_entry cd321x_test_connector_ep_props[] = { + PROPERTY_ENTRY_REF_ARRAY("remote-endpoint", cd321x_test_connector_refs), + { } +}; + +static const struct software_node cd321x_test_connector_ep = { + .name = "endpoint", + .parent = &cd321x_test_connector_port, + .properties = cd321x_test_connector_ep_props, +}; + +static const struct software_node cd321x_test_plain_connector = { + .name = "cd321x-test-plain-connector", +}; + +static const struct software_node * const cd321x_test_nodes[] = { + &cd321x_test_router, + &cd321x_test_router_port, + &cd321x_test_router_ep, + &cd321x_test_connector, + &cd321x_test_connector_port, + &cd321x_test_connector_ep, + &cd321x_test_plain_connector, + NULL +}; + +struct cd321x_tbt_test { + struct device *dev; + bool keep_waiting; + unsigned int keep_waiting_calls; +}; + +/* Stands in for the driver core's answer on whether to keep waiting. */ +static bool cd321x_tbt_test_keep_waiting(struct device *dev) +{ + struct cd321x_tbt_test *ctx = kunit_get_current_test()->priv; + + ctx->keep_waiting_calls++; + return ctx->keep_waiting; +} + +static void cd321x_tbt_test_unregister_nodes(void *unused) +{ + software_node_unregister_node_group(cd321x_test_nodes); +} + +static int cd321x_tbt_test_init(struct kunit *test) +{ + struct cd321x_tbt_test *ctx; + int ret; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + test->priv = ctx; + + ret = software_node_register_node_group(cd321x_test_nodes); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, cd321x_tbt_test_unregister_nodes, + NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + + ctx->dev = kunit_device_register(test, "cd321x"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->dev); + + kunit_activate_static_stub(test, cd321x_tbt_switch_keep_waiting, + cd321x_tbt_test_keep_waiting); + return 0; +} + +static int cd321x_tbt_test_set(struct typec_thunderbolt_switch_dev *sw, + const struct typec_thunderbolt_switch_data *data) +{ + return 0; +} + +static void cd321x_tbt_test_unregister_switch(void *sw) +{ + typec_thunderbolt_switch_unregister(sw); +} + +/* A connector without a router link never waits. */ +static void cd321x_tbt_no_router_test(struct kunit *test) +{ + struct cd321x_tbt_test *ctx = test->priv; + struct typec_thunderbolt_switch *sw; + + sw = cd321x_tbt_switch_get(ctx->dev, + software_node_fwnode(&cd321x_test_plain_connector)); + KUNIT_EXPECT_NULL(test, sw); + KUNIT_EXPECT_EQ(test, ctx->keep_waiting_calls, 0); +} + +/* A router whose driver has registered its switch is used. */ +static void cd321x_tbt_router_bound_test(struct kunit *test) +{ + struct cd321x_tbt_test *ctx = test->priv; + struct typec_thunderbolt_switch_desc desc = { + .fwnode = software_node_fwnode(&cd321x_test_router), + .set = cd321x_tbt_test_set, + }; + struct typec_thunderbolt_switch_dev *sw_dev; + struct typec_thunderbolt_switch *sw; + struct device *router; + int ret; + + router = kunit_device_register(test, "router"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, router); + /* The lookup takes a reference on the router driver's module. */ + KUNIT_ASSERT_NOT_NULL(test, router->driver); + + sw_dev = typec_thunderbolt_switch_register(router, &desc); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sw_dev); + ret = kunit_add_action_or_reset(test, cd321x_tbt_test_unregister_switch, + sw_dev); + KUNIT_ASSERT_EQ(test, ret, 0); + + sw = cd321x_tbt_switch_get(ctx->dev, + software_node_fwnode(&cd321x_test_connector)); + KUNIT_EXPECT_NOT_ERR_OR_NULL(test, sw); + KUNIT_EXPECT_EQ(test, ctx->keep_waiting_calls, 0); + typec_thunderbolt_switch_put(sw); +} + +/* A router that has not registered its switch yet defers the port. */ +static void cd321x_tbt_router_pending_test(struct kunit *test) +{ + struct cd321x_tbt_test *ctx = test->priv; + struct typec_thunderbolt_switch *sw; + + ctx->keep_waiting = true; + sw = cd321x_tbt_switch_get(ctx->dev, + software_node_fwnode(&cd321x_test_connector)); + KUNIT_EXPECT_PTR_EQ(test, sw, ERR_PTR(-EPROBE_DEFER)); + KUNIT_EXPECT_EQ(test, ctx->keep_waiting_calls, 1); +} + +/* + * Once the driver core stops waiting for missing dependencies, a router that + * never registered its switch, for example because its driver is not built, + * no longer keeps the port from probing. + */ +static void cd321x_tbt_router_never_binds_test(struct kunit *test) +{ + struct cd321x_tbt_test *ctx = test->priv; + struct typec_thunderbolt_switch *sw; + + ctx->keep_waiting = false; + sw = cd321x_tbt_switch_get(ctx->dev, + software_node_fwnode(&cd321x_test_connector)); + KUNIT_EXPECT_NULL(test, sw); + KUNIT_EXPECT_EQ(test, ctx->keep_waiting_calls, 1); +} + +static struct kunit_case cd321x_tbt_cases[] = { + KUNIT_CASE(cd321x_tbt_no_router_test), + KUNIT_CASE(cd321x_tbt_router_bound_test), + KUNIT_CASE(cd321x_tbt_router_pending_test), + KUNIT_CASE(cd321x_tbt_router_never_binds_test), + {} +}; + +static struct kunit_suite cd321x_tbt_suite = { + .name = "cd321x-tbt", + .init = cd321x_tbt_test_init, + .test_cases = cd321x_tbt_cases, +}; + +kunit_test_suite(cd321x_tbt_suite); +MODULE_DESCRIPTION("KUnit tests for the CD321x USB4/Thunderbolt router lookup"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index e17d27392a424..ddfd89c8415e3 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -6,6 +6,8 @@ * Author: Heikki Krogerus */ +#include +#include #include #include #include @@ -1509,6 +1511,15 @@ static void cd321x_stop_work(struct cd321x *cd321x) cancel_delayed_work_sync(&cd321x->update_work); } +/* Separate so that the KUnit tests can choose the driver core's answer. */ +VISIBLE_IF_KUNIT bool cd321x_tbt_switch_keep_waiting(struct device *dev) +{ + KUNIT_STATIC_STUB_REDIRECT(cd321x_tbt_switch_keep_waiting, dev); + + return driver_deferred_probe_check_state(dev) == -EPROBE_DEFER; +} +EXPORT_SYMBOL_IF_KUNIT(cd321x_tbt_switch_keep_waiting); + /* * The USB4/Thunderbolt router is only needed once a USB4 or Thunderbolt link * comes up. If its driver is not built or never binds, do not leave power @@ -1516,19 +1527,19 @@ static void cd321x_stop_work(struct cd321x *cd321x) * waiting for the router when the driver core stops waiting for missing * dependencies. */ -static struct typec_thunderbolt_switch * +VISIBLE_IF_KUNIT struct typec_thunderbolt_switch * cd321x_tbt_switch_get(struct device *dev, struct fwnode_handle *fwnode) { struct typec_thunderbolt_switch *sw; sw = fwnode_typec_thunderbolt_switch_get(fwnode); - if (sw != ERR_PTR(-EPROBE_DEFER) || - driver_deferred_probe_check_state(dev) == -EPROBE_DEFER) + if (sw != ERR_PTR(-EPROBE_DEFER) || cd321x_tbt_switch_keep_waiting(dev)) return sw; dev_info(dev, "USB4/Thunderbolt router unavailable, continuing without it\n"); return NULL; } +EXPORT_SYMBOL_IF_KUNIT(cd321x_tbt_switch_get); static int cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 744efb2bdbdef..503757b035e0a 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -456,4 +456,10 @@ static inline void tipd_debugfs_unregister(struct tps6598x *tps) { } static inline void tipd_debugfs_exit(void) { } #endif +#if IS_ENABLED(CONFIG_KUNIT) +bool cd321x_tbt_switch_keep_waiting(struct device *dev); +struct typec_thunderbolt_switch * +cd321x_tbt_switch_get(struct device *dev, struct fwnode_handle *fwnode); +#endif + #endif /* __TPS6598X_H__ */ From 88837bda2a82ece2f9b930ff926983fe4711af00 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 18:15:49 +1100 Subject: [PATCH 1086/1579] drm/apple: Require a complete reply to a DPTX HPD request dptxport_set_hpd_timeout() takes an HPD request as acknowledged when the last word of the firmware's 20-byte reply echoes the HPD state it was sent. It reads the reply with afk_service_call_timeout(), which copies only as much of the reply as its header declares and zero-fills the rest of the output. A reply declaring a shorter body therefore reads as an echo of 0 and acknowledges HPD low, and a reply declaring more than its buffer holds is clipped and accepted for either state. Teardown depends on that acknowledgment: the external H17P DCP releases the display, and with it the Type-C route, only once the firmware has acknowledged HPD low. Make afk_service_call_timeout_len() available. It checks the length a reply declares against what arrived and against the response space the caller reserved, which now includes the caller's output padding, and it reports how much of the output the reply filled. Use it for HPD requests, and refuse a reply that does not fill the whole response before comparing the echo. This asks no more of the firmware for HPD low than every DCP already answers for HPD high, whose echo of 1 cannot come from zero fill. The only other user of the length check, the H17P EDID read, reserves no output padding and is unchanged. The apple-afkv2 service-call test extracts the now global function. Fixes: 5c29016e370b ("drm: apple: recover Type-C displays across link interruptions") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 21 +++++++------------ drivers/gpu/drm/apple/afk.h | 10 +++++++++ drivers/gpu/drm/apple/dptxep.c | 13 ++++++++++-- .../gpu/apple-afkv2/extract-service.py | 3 ++- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 0d598980c96e4..93530e77da7ba 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1808,12 +1808,6 @@ int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, output_pad, MSEC_PER_SEC); } -static int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, - u32 command, const void *data, size_t data_len, - size_t data_pad, void *output, size_t output_len, - size_t output_pad, unsigned int timeout_ms, - size_t *received); - int afk_service_call_len(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, size_t output_len, @@ -1834,11 +1828,11 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, timeout_ms, NULL); } -static int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, - u32 command, const void *data, size_t data_len, - size_t data_pad, void *output, size_t output_len, - size_t output_pad, unsigned int timeout_ms, - size_t *received) +int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, unsigned int timeout_ms, + size_t *received) { struct epic_service_call *call; size_t wire_received = 0, result_size; @@ -1891,11 +1885,12 @@ static int afk_service_call_timeout_len(struct apple_epic_service *service, u16 goto out; } if (received) { + /* The reply may use the padding reserved after the output. */ ret = afkv2_service_reply_size(bfr, wire_received, group, command, - output_len, &result_size); + output_len + output_pad, &result_size); if (ret) goto out; - *received = result_size; + *received = min(result_size, output_len); } if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || le16_to_cpu(call->group) != group || diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 0304410bd6e82..0a12ecda982df 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -419,5 +419,15 @@ int afk_service_call_len(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, size_t output_len, size_t output_pad, size_t *received); +/* + * As afk_service_call_timeout(), but the length the reply declares is checked + * against what arrived and against @output_len + @output_pad, and @received + * is set to the number of bytes of @output that the reply filled. + */ +int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, unsigned int timeout_ms, + size_t *received); #endif diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index b9d4729407ed6..2f0b4fad04642 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -235,6 +235,7 @@ int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, unsigned int timeout_ms) { struct dcpdptx_hotplug_cmd cmd, resp; + size_t received; int ret; memset(&cmd, 0, sizeof(cmd)); @@ -242,10 +243,18 @@ int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, if (hpd) cmd.unk = cpu_to_le32(1); - ret = afk_service_call_timeout(service, 8, 8, &cmd, sizeof(cmd), 12, - &resp, sizeof(resp), 12, timeout_ms); + ret = afk_service_call_timeout_len(service, 8, 8, &cmd, sizeof(cmd), 12, + &resp, sizeof(resp), 12, timeout_ms, + &received); if (ret) return ret; + /* Past the end of a short reply there is only zero fill, not an echo. */ + if (received < sizeof(resp)) { + dev_warn(service->ep->dcp->dev, + "set_hpd: short reply, %zu of %zu bytes (sent hpd=%d)\n", + received, sizeof(resp), hpd); + return -EPROTO; + } if (le32_to_cpu(resp.unk) != hpd) { dev_warn(service->ep->dcp->dev, "set_hpd: unk reply 0x%x (sent hpd=%d)\n", diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py index ab95f5c422710..19381f919d7e8 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py @@ -6,7 +6,7 @@ import sys source = pathlib.Path(sys.argv[1]).read_text() -match = re.search(r"static int afk_service_call_timeout_len\([^;]*?\)\n\{", source) +match = re.search(r"^int afk_service_call_timeout_len\([^;]*?\)\n\{", source, re.M) if not match: raise RuntimeError("production service-call definition not found") start = match.start() @@ -25,6 +25,7 @@ #define kzalloc(n,f) calloc(1,n) #define kfree(p) free(p) #define max(a,b) ((a)>(b)?(a):(b)) +#define min(a,b) ((a)<(b)?(a):(b)) #define cpu_to_le16(x) (x) #define cpu_to_le32(x) (x) #define le16_to_cpu(x) (x) From d1cadb0ca747ae36337d1250379430f920adb0a6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 18:15:49 +1100 Subject: [PATCH 1087/1579] selftests: apple-afkv2: Check the reply length of a DPTX HPD request Extract dptxport_set_hpd_timeout() with the actual service-call body and its timeout wrapper, and answer the HPD request with matched, successful replies in legacy and AFK-v2 framing. For both HPD states, a reply is refused when its declared body stops before the HPD echo (0 to 16 bytes), when it declares more than its buffer holds (36 and 65535 bytes), when it was cut short in transit, or when the echo is wrong. Complete (20 bytes) and padded (32 bytes) replies are accepted. Without "drm/apple: Require a complete reply to a DPTX HPD request", 18 of these checks fail: every short reply acknowledges HPD low and every oversized one is accepted. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../drivers/gpu/apple-afkv2/Makefile | 2 +- .../selftests/drivers/gpu/apple-afkv2/README | 5 + .../drivers/gpu/apple-afkv2/extract-hpd.py | 150 ++++++++++++++++++ .../gpu/apple-afkv2/run-afkv2-tests.sh | 10 ++ 4 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-hpd.py diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile index df1b033d9cb32..a792549ee9f14 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile @@ -4,6 +4,6 @@ export HOSTCC export AFKV2_SOURCE_ROOT := $(abspath ../../../../../..) TEST_PROGS := run-afkv2-tests.sh -TEST_FILES := codec.c owned-prefix.c owned-suffix.c extract-owned.py inline-edid.c extract-service.py extract-send.py extract-edid.py extract-ring.py extract-notify.py host-include +TEST_FILES := codec.c owned-prefix.c owned-suffix.c extract-owned.py inline-edid.c extract-service.py extract-hpd.py extract-send.py extract-edid.py extract-ring.py extract-notify.py host-include include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/README b/tools/testing/selftests/drivers/gpu/apple-afkv2/README index 064cdcf018595..bffcd55fba642 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/README +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/README @@ -28,6 +28,11 @@ EDID header/checksum/block limits, and continued rejection of inline responses by the OOB matcher. The actual service-call body is extracted for short reply, error status and timeout checks; its wire sender is mocked. +The DPTX HPD request is extracted with that service-call body and its timeout +wrapper. With legacy and AFK-v2 framing, a reply whose declared body stops +before the HPD echo, or runs past the bytes that arrived, is refused for both +HPD states; complete and padded replies are accepted and a wrong echo refused. + The real sender is also extracted to test a 128-byte mocked command payload with a 32 KiB response capacity, OOB behavior, send failures and timeout retention. The two production EDID readers are tested independently to diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-hpd.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-hpd.py new file mode 100644 index 0000000000000..6957d92f390e2 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-hpd.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production DPTX HPD request with mocked service-call replies.""" +import pathlib +import re +import sys + + +def function(source, name): + match = re.search(r"^(?:static )?int " + name + r"\([^;]*?\)\n\{", source, re.M) + if not match: + raise RuntimeError("production definition of " + name + " not found") + end, depth = match.end(), 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + # Older trees keep the call body static; the unit declares it below. + return re.sub(r"^static ", "", source[match.start():end]) + + +afk = pathlib.Path(sys.argv[1]).read_text() +dptx = pathlib.Path(sys.argv[2]).read_text() +command = re.search(r"^struct dcpdptx_hotplug_cmd \{.*?\} __attribute__\(\(packed\)\);", + dptx, re.M | re.S) +if not command: + raise RuntimeError("production HPD request layout not found") +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include "afkv2-codec.h" +typedef u32 __le32; +#define GFP_KERNEL 0 +#define kzalloc(n,f) calloc(1,n) +#define kfree(p) free(p) +#define max(a,b) ((a)>(b)?(a):(b)) +#define min(a,b) ((a)<(b)?(a):(b)) +#define cpu_to_le16(x) (x) +#define cpu_to_le32(x) (x) +#define le16_to_cpu(x) (x) +#define le32_to_cpu(x) (x) +#define EPIC_SERVICE_CALL_MAGIC 0x69706378 +#define EPIC_SUBTYPE_STD_SERVICE 0xc0 +#define dev_warn(...) do {} while (0) +#define dev_dbg(...) do {} while (0) +struct epic_service_call { + u8 pad0[2]; u16 group; u32 command, data_len, magic; u8 pad1[48]; +}; +struct dcp { void *dev; }; +struct ops { const char *name; }; +struct ep { struct dcp *dcp; int endpoint; bool inline_v2; }; +struct apple_epic_service { struct ep *ep; struct ops *ops; u32 channel; }; +int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, size_t data_pad, void *output, + size_t output_len, size_t output_pad, unsigned int timeout_ms, size_t *received); +int afk_service_call_timeout(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, size_t data_pad, void *output, + size_t output_len, size_t output_pad, unsigned int timeout_ms); +int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, + unsigned int timeout_ms); +/* + * A matched, successful reply in the request's buffer: a 64-byte header, then + * the body the firmware declares, of which the echo is the word at 16. An + * AFK-v2 transport reports how much of the buffer arrived; a legacy one fills + * it. Without a length query AFK-v2 refuses anything short of the buffer. + */ +static bool wire_v2; +static size_t declared, arrived; +static u32 echo_flip; +static int afk_send_command_timeout_len(struct apple_epic_service *s, u8 type, + const void *payload, size_t payload_len, void *output, size_t output_len, + u32 *retcode, unsigned int timeout_ms, size_t *received) +{ + struct epic_service_call *call = output; + u8 *body = (u8 *)output + sizeof(*call); + u32 echo; + + (void)s; (void)payload; (void)timeout_ms; + assert(type == EPIC_SUBTYPE_STD_SERVICE && payload_len == output_len); + assert(output_len == sizeof(*call) + 32 && call->data_len == 32); + assert(call->group == 8 && call->command == 8); + if (wire_v2 && !received && arrived < output_len - sizeof(*call)) + return -EMSGSIZE; + memcpy(&echo, body + 16, sizeof(echo)); + echo ^= echo_flip; + memcpy(body + 16, &echo, sizeof(echo)); + call->data_len = declared; + *retcode = 0; + if (received) + *received = sizeof(*call) + (wire_v2 ? arrived : output_len - sizeof(*call)); + return 0; +} +''' +suffix = r''' +static int failures; +static void expect(bool ok, const char *what, bool hpd, int ret) +{ + printf("%s %s: %s hpd=%d declared=%zu arrived=%zu ret=%d\n", ok ? "ok" : "FAIL", + wire_v2 ? "afk-v2" : "legacy", what, hpd, declared, arrived, ret); + failures += !ok; +} +int main(void) +{ + static const size_t lengths[] = { 0, 4, 8, 12, 16, 20, 32, 36, 65535 }; + struct dcp dcp = { 0 }; + struct ops ops = { "AppleDCPDPTXRemotePort" }; + struct ep ep = { .dcp = &dcp }; + struct apple_epic_service service = { .ep = &ep, .ops = &ops, .channel = 1 }; + + for (int v2 = 0; v2 < 2; v2++) { + wire_v2 = v2; + for (size_t i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) { + for (int hpd = 0; hpd < 2; hpd++) { + int ret; + + declared = lengths[i]; arrived = 32; echo_flip = 0; + ret = dptxport_set_hpd_timeout(&service, hpd, 1000); + if (declared < sizeof(struct dcpdptx_hotplug_cmd)) + expect(ret < 0, "short reply refused", hpd, ret); + else if (declared > 32) + expect(ret < 0, "reply longer than its buffer refused", hpd, ret); + else + expect(ret == 0, "complete reply accepted", hpd, ret); + } + } + for (int hpd = 0; hpd < 2; hpd++) { + int ret; + + declared = 20; arrived = 32; echo_flip = 1; + ret = dptxport_set_hpd_timeout(&service, hpd, 1000); + expect(ret == -EINVAL, "wrong echo refused", hpd, ret); + if (!wire_v2) + continue; + declared = 20; arrived = 16; echo_flip = 0; + ret = dptxport_set_hpd_timeout(&service, hpd, 1000); + expect(ret < 0, "reply cut short in transit refused", hpd, ret); + } + } + printf("%d HPD reply checks failed\n", failures); + if (failures) + return 1; + puts("PASS: extracted production DPTX HPD request: short, oversized, wrong-echo and complete replies, legacy and AFK-v2"); + return 0; +} +''' +unit = (prefix + command.group(0) + "\n" + function(afk, "afk_service_call_timeout_len") + "\n" + + function(afk, "afk_service_call_timeout") + "\n" + + function(dptx, "dptxport_set_hpd_timeout") + suffix) +pathlib.Path(sys.argv[3]).write_text(unit) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh index 4348eebc78246..5351b80edbcc5 100755 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh @@ -53,6 +53,16 @@ $afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ "$afkv2_tmp/service.c" -o "$afkv2_tmp/service" "$afkv2_tmp/service" +# The DPTX HPD request: only a complete reply carries the HPD echo. +"$afkv2_python" "$afkv2_dir/extract-hpd.py" \ + "$afkv2_root/drivers/gpu/drm/apple/afk.c" \ + "$afkv2_root/drivers/gpu/drm/apple/dptxep.c" "$afkv2_tmp/hpd.c" +# shellcheck disable=SC2086 +$afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/hpd.c" -o "$afkv2_tmp/hpd" +"$afkv2_tmp/hpd" + "$afkv2_python" "$afkv2_dir/extract-send.py" \ "$afkv2_root/drivers/gpu/drm/apple/afk.c" "$afkv2_tmp/send.c" # shellcheck disable=SC2086 From 566ff3adbc7c873817bdd6d9968939e5e099bbe1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 15:09:06 +1100 Subject: [PATCH 1088/1579] drm/apple: Withdraw a Type-C DPTX link that never comes up When the external H17P DCP connects a Type-C DisplayPort sink and no link configuration arrives within two seconds, dcp_dptx_connect_raw() releases the display and returns -ETIMEDOUT. dcp_external_link_locked() then latches that error in external_error, which keeps the external session blocked: every later connect, stop, route deactivation and SAFE transition of the Type-C route returns it until the driver is rebound. The latch exists so that a firmware request whose reply may still arrive is never raced by a retry. A missing link configuration is not such a request: every request of the attempt has been answered, and the sink went away while the firmware trained it, as when a dock is pulled or bounces while it is being plugged in: DPTXPort: firmware reports link fault 22 on target 0:0 (x5) dcp_dptx_connect: timed out waiting for port 0 link configuration sn201202x 1-0c: failed to place Type-C mux in safe mode: -110 (every later cable event, until reboot) Withdraw such an attempt the way the external teardown does: drop HPD, then release the display. When the firmware acknowledges both with complete replies, nothing of the attempt is outstanding, so mark the port released, report the attempt as retryable and do not latch it; the next cable or HPD event connects again. If either request fails, the firmware state is unknown: the port stays marked as holding the display and the error is latched as before. Other DCPs keep their behaviour: they only release the display, and their reconnect worker retries as it did. Log the outcome either way, so that a capture shows whether the withdrawal ran and what the firmware answered. Fixes: 35f4ccf26053 ("drm/apple: add bounded external stop qualification path") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 58 ++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8f601bbb91cbe..bbc7329f591be 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2488,13 +2488,19 @@ bool dcp_has_typec_routes(struct platform_device *pdev) #define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) #define DPTX_RECONNECT_RETRIES 5 -static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_service) +/* + * Connect DPTX port @port. On failure, @retryable (if given) tells whether + * the attempt left nothing outstanding in the firmware, so that a later + * connection may simply try again: the service was not published yet, or a + * link that never came up was withdrawn and the firmware acknowledged that. + */ +static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *retryable) { unsigned long timeout; - int ret = 0; + int ret = 0, err; - if (waiting_service) - *waiting_service = false; + if (retryable) + *retryable = false; if (!dcp->phy) { dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); return -ENODEV; @@ -2511,8 +2517,8 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_s dev_warn(dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); ret = -ENODEV; /* No firmware request has been made; the first publication can retry. */ - if (waiting_service && !READ_ONCE(dcp->dptxport[port].closed)) { - *waiting_service = true; + if (retryable && !READ_ONCE(dcp->dptxport[port].closed)) { + *retryable = true; ret = -EAGAIN; } goto out_unlock; @@ -2577,7 +2583,7 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_s "dcp_dptx_connect: timed out waiting for port %u link configuration\n", port); ret = -ETIMEDOUT; - goto out_disconnect; + goto out_withdraw; } dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", @@ -2593,9 +2599,39 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_s return 0; -out_disconnect: +out_withdraw: mutex_lock(&dcp->hpd_mutex); + if (!dcp_is_external_h17p(dcp)) { + dcp->dptxport[port].connected = false; + goto out_release; + } + /* + * No link configuration arrived, typically because the sink went away + * while the firmware trained it. Withdraw the request the way a + * disconnect does, HPD first. Only once the firmware has acknowledged + * both is nothing of this attempt outstanding; until then the port + * stays held. + */ + err = -ENODEV; + if (dptx_service_available(&dcp->dptxport[port])) { + err = dcp_is_typec_output(dcp) ? + dptxport_set_hpd(dcp->dptxport[port].service, false) : 0; + if (!err) + err = dptxport_release_display(dcp->dptxport[port].service); + } + if (err) { + dev_err(dcp->dev, + "dcp_dptx_connect: port %u not withdrawn, still held: %d\n", + port, err); + goto out_unlock; + } + dev_info(dcp->dev, "dcp_dptx_connect: port %u withdrawn, the next connection retries\n", + port); dcp->dptxport[port].connected = false; + if (retryable) + *retryable = true; + goto out_unlock; + out_release: dptxport_release_display(dcp->dptxport[port].service); @@ -2631,7 +2667,7 @@ static void dcp_external_publish_generation(struct apple_dcp *dcp) /* external_lock is never taken by the serialized IOMFB receive callbacks. */ static int dcp_external_link_locked(struct apple_dcp *dcp) { - bool waiting_service; + bool retryable; int ret; lockdep_assert_held(&dcp->external_lock); @@ -2641,8 +2677,8 @@ static int dcp_external_link_locked(struct apple_dcp *dcp) return dcp->external_error; if (!dcp->external_cable) return -ENODEV; - ret = dcp_dptx_connect_raw(dcp, 0, &waiting_service); - if (ret && !waiting_service) + ret = dcp_dptx_connect_raw(dcp, 0, &retryable); + if (ret && !retryable) dcp->external_error = ret; return ret; } From 97f64ff057dcd611611eab761979090670610079 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 18:18:56 +1100 Subject: [PATCH 1089/1579] selftests: apple-dcp: Replay a fast dock replug that loses the DPTX link Add the sequence of a dock pulled while its display link trains, with the actual external-session functions of drm/apple and a firmware model whose link configuration never arrives while the sink is gone, and whose requests can fail, be refused or go unanswered: - withdrawn-link: the route comes back after the plug-time bounce, the link times out and is withdrawn, the next connect tries again, the unplug tears the session down and the next plug connects. - unacknowledged-withdrawal: HPD low or the release times out, or HPD low gets an incomplete reply. The session stays held and latched. - unanswered-request: validation, connection, the display request or HPD assertion times out, with or without a failed cleanup, on a session whose pipe never powered or was stopped before. The port no longer reads connected, yet the failure stays latched across the unplug and the next plug is refused: the firmware may still carry the request out. - refused-request: a display request or HPD assertion the firmware refuses stays latched, as before. - live-failure: a powered pipe whose power-off failed holds everything. quiesce-before-safe gains withdrawn-still-latched: with a failure latched and the port reading released, a SAFE transition is still refused and never reaches the Type-C PHY. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/README | 4 +- .../drivers/gpu/apple-dcp/lifecycle.py | 169 ++++++++++++++++++ .../drivers/gpu/apple-dcp/quiesce.py | 2 + .../selftests/drivers/gpu/apple-dcp/run.py | 4 +- 4 files changed, 177 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index b04b6b9934193..1f5e3cc8cd68d 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -29,7 +29,9 @@ The modules contain public synthetic host boundaries and retained assertions: faulted ACK/BIST cleanup, dirty-route retry, OFF/new-calibration epochs, warm reset and legacy T8103/T8122/T8140/T8142 route controls. * lifecycle: actual physical-disconnect authority and ownership despite failed - teardown; DWC3/core/PHY PM and reset-reference recovery; exact firmware + teardown; a fast dock replug whose DPTX link never configures (withdrawn, + unacknowledged, unanswered, refused and live failures); DWC3/core/PHY PM + and reset-reference recovery; exact firmware tuple/resource admission, unsupported VRR rejection and power reply bounds; 15,552 D134 dispatcher scope/length/context and legacy reply cases. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index d5b56fe489490..e21ca96e971df 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -366,6 +366,175 @@ def function(file, name): source = prefix + pm + shim + funcs + tests return source +def bounce(root): + """Fast dock replug: the sink vanishes while the DPTX link trains. + + The physical-disconnect fixture with the actual dcp_dptx_connect_raw() + and a firmware model whose link configuration may never arrive and + whose requests may fail, be refused or go unanswered. + """ + unit = disconnect(root) + dcp = (root / 'drivers/gpu/drm/apple/dcp.c').read_text() + m = re.search(r'^static int dcp_dptx_connect_raw\(', dcp, re.M) + assert m, 'dcp_dptx_connect_raw' + raw = block(dcp, m.start()) + + def swap(old, new): + nonlocal unit + assert unit.count(old) == 1, old + unit = unit.replace(old, new) + + swap(' struct { bool connected; void *service; } dptxport[1];', + ' struct { bool connected, closed; void *service, *atcphy; int linkcfg_completion; } dptxport[1];\n' + ' void *phy, *dev, *active_typec_route;\n' + ' int dptx_die, dptx_phy, dptx_dfp_port, connector_type;') + old_raw = unit[unit.index('static int dcp_dptx_connect_raw('):] + old_raw = old_raw[:old_raw.index('return raw_error; }') + len('return raw_error; }')] + swap(old_raw, 'static int dcp_dptx_connect_raw(struct apple_dcp *, u32, bool *);') + swap('static int dptxport_set_hpd(void *p, bool on)\n{ (void)p; assert(!on); return hpd_error; }', + 'static bool fw_hpd, fw_display, link_trains;\n' + 'static int hpd_drops, connects, release_error_once, assert_error;\n' + '/* A request that times out may still be carried out; a refused one is not. */\n' + 'static int dptxport_set_hpd(void *p, bool on)\n' + '{ (void)p; if (on) { fw_hpd |= assert_error != -EINVAL; return assert_error; }\n' + ' hpd_drops++; if (hpd_error) return hpd_error; fw_hpd = false; return 0; }') + swap('static int dptxport_release_display(void *p)\n{ (void)p; releases++; return release_error; }', + 'static int dptxport_release_display(void *p)\n' + '{ (void)p; releases++; if (release_error) return release_error;\n' + ' if (release_error_once) { int r = release_error_once; release_error_once = 0; return r; }\n' + ' fw_display = false; return 0; }') + swap('int main(void)\n{', 'static int __attribute__((unused)) prior_main(void)\n{') + firmware = r''' +typedef uint8_t u8; +#define dev_dbg(...) ((void)0) +#define dev_info(...) ((void)0) +#define DRM_MODE_CONNECTOR_DisplayPort 10 +#define DPTX_CONNECT_TIMEOUT 2000 +#define DPTX_TUNNEL_CONNECT_TIMEOUT 8000 +static void mutex_lock(int *m) { (void)m; } +static void mutex_unlock(int *m) { (void)m; } +static bool dcp_is_usb4_output(struct apple_dcp *d) { (void)d; return false; } +static bool apple_dp_tunnel_t602x(void) { return false; } +static void reinit_completion(int *c) { *c = 0; } +/* The firmware configures the link only while a sink answers it. */ +static unsigned long wait_for_completion_timeout(int *c, unsigned long t) +{ if (link_trains) *c = 1; return *c ? t : 0; } +static unsigned jiffies_to_msecs(unsigned long t) { return t; } +static void usleep_range(int a, int b) { (void)a; (void)b; } +static void av_service_connect(struct apple_dcp *d) { (void)d; } +static int validate_error, connect_error, request_error; +static int dptxport_validate_connection(void *s, int core, int atc, int die) +{ (void)s; (void)core; (void)atc; (void)die; return validate_error; } +static int dptxport_connect(void *s, int core, int atc, int die, bool typec) +{ (void)s; (void)core; (void)atc; (void)die; (void)typec; connects++; return connect_error; } +static int dptxport_request_display(void *s) +{ (void)s; fw_display |= request_error != -EINVAL; return request_error; } +static int dptxport_set_hpd_timeout(void *s, bool on, int t) { (void)t; return dptxport_set_hpd(s, on); } +''' + tests = r''' +static struct platform_device device = { .data = &display }; +/* The session the dock leaves behind after its plug-time route bounce. */ +static void bounced(void) +{ + init(); display.phy = (void *)1; fw_hpd = fw_display = true; + release_error_once = 0; link_trains = true; + CHECK(dcp_external_disconnect(&display, true) == 0); + CHECK(display.external_stopped && !display.dptxport[0].connected && !display.external_error); + connects = hpd_drops = 0; + validate_error = connect_error = request_error = assert_error = 0; +} +/* The failure stays latched across the unplug, and the next plug is refused. */ +static void stays_latched(int error) +{ + int before; + + validate_error = connect_error = request_error = assert_error = 0; + hpd_error = release_error = 0; + CHECK(display.external_error == error); + CHECK(dcp_external_disconnect(&display, true) == error); + CHECK(display.external_error == error); + before = connects; link_trains = true; + CHECK(dcp_dptx_connect_oob(&device, 0) == error && connects == before); +} +int main(void) +{ + /* The route comes back while the sink no longer answers: withdrawn, not latched. */ + bounced(); link_trains = false; + CHECK(dcp_dptx_connect_oob(&device, 0) == -ETIMEDOUT); + CHECK(!display.external_error && !display.dptxport[0].connected); + CHECK(!fw_hpd && !fw_display && hpd_drops == 1); + /* tipd's own OOB connect after the mux tries again rather than refusing. */ + CHECK(dcp_dptx_connect_oob(&device, 0) == -ETIMEDOUT && connects == 2); + /* The unplug's teardown (OOB disconnect, then SAFE) succeeds. */ + CHECK(dcp_external_disconnect(&display, true) == 0 && !display.external_error); + /* The next plug connects: the display recovers without a reboot. */ + link_trains = true; + CHECK(dcp_dptx_connect_oob(&device, 0) == 0); + CHECK(display.dptxport[0].connected && fw_hpd && fw_display && connects == 3); + printf("case withdrawn-link complete\n"); + + /* + * The firmware does not acknowledge the withdrawal: HPD low or the release + * times out, or HPD low gets an incomplete reply. The session stays held. + */ + for (int which = 0; which < 3; which++) { + bounced(); link_trains = false; + if (which == 0) hpd_error = -ETIMEDOUT; + if (which == 1) release_error_once = -ETIMEDOUT; + if (which == 2) hpd_error = -EPROTO; + CHECK(dcp_dptx_connect_oob(&device, 0) == -ETIMEDOUT); + CHECK(display.external_error == -ETIMEDOUT && display.dptxport[0].connected); + stays_latched(-ETIMEDOUT); + } + printf("case unacknowledged-withdrawal complete\n"); + + /* + * A request goes unanswered, so the firmware may still carry it out, with + * its cleanup failing as well or not, on a session that never powered its + * pipe or that stopped it before. A port no longer marked connected is no + * proof that the firmware let go of it: the failure stays latched. + */ + for (int fault = 0; fault < 4; fault++) + for (int cleanup = 0; cleanup < 2; cleanup++) + for (int stopped = 0; stopped < 2; stopped++) { + bounced(); + if (!stopped) + display.external_power_attempted = display.external_stopped = false; + if (fault == 0) validate_error = -ETIMEDOUT; + if (fault == 1) connect_error = -ETIMEDOUT; + if (fault == 2) request_error = -ETIMEDOUT; + if (fault == 3) assert_error = -ETIMEDOUT; + if (cleanup) release_error = -ETIMEDOUT; + CHECK(dcp_dptx_connect_oob(&device, 0) == -ETIMEDOUT); + stays_latched(-ETIMEDOUT); + } + printf("case unanswered-request complete\n"); + + /* A request the firmware refuses is latched as before, released display or not. */ + for (int fault = 0; fault < 2; fault++) { + bounced(); + if (fault == 0) request_error = -EINVAL; + if (fault == 1) assert_error = -EINVAL; + CHECK(dcp_dptx_connect_oob(&device, 0) == -EINVAL); + CHECK(!display.dptxport[0].connected); + stays_latched(-EINVAL); + } + printf("case refused-request complete\n"); + + /* A live failure (powered pipe never acknowledged off) still holds everything. */ + init(); display.phy = (void *)1; power_error = -ETIMEDOUT; connects = 0; + CHECK(dcp_external_disconnect(&display, true) == -ETIMEDOUT); + power_error = 0; + CHECK(dcp_external_disconnect(&display, true) == -ETIMEDOUT); + CHECK(display.external_error == -ETIMEDOUT && display.pipe_enabled_h17p); + CHECK(dcp_dptx_connect_oob(&device, 0) == -ETIMEDOUT && connects == 0); + printf("case live-failure complete\n"); + + printf("%d assertions, %d failures\n", checks, failures); return !!failures; +} +''' + return unit + firmware + raw + '\n' + tests + def teardown(root): files = {n: (root / f).read_text() for n, f in { diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py index 6e7b9f69b2a25..4bf0fe7db09b4 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -138,6 +138,8 @@ def make_unit(source, parent=False, reference=None): if(argv[1][0]=='m') {power_error=-EIO;assert(typec_mux_set(&mux,&safe)==-EIO&&phy_calls==0&&port.owner==&route);} if(argv[1][0]=='d') {assert(typec_mux_prepare(&mux,&safe)==0&&stops==1&&phy_calls==0&&port.owner==&route);assert(typec_mux_prepare(&mux,&safe)==0&&stops==1&&port.owner==&route);} if(argv[1][0]=='e') {display.external_error=-EIO;port.hpd=false;display.typec_cable_connected=false;connector.connected=false;assert(typec_mux_set(&mux,&safe)==-EIO&&phy_calls==0&&port.owner==&route);} + /* A latched failure is no less live once the port reads released: SAFE stays refused. */ + if(argv[1][0]=='w') {display.external_error=-ETIMEDOUT;display.dptxport[0].connected=false;display.external_stopped=true;port.hpd=false;assert(typec_mux_set(&mux,&safe)==-ETIMEDOUT&&phy_calls==0&&port.owner==&route&&deactivate_calls==0);} if(argv[1][0]=='l') {display.external=false;assert(typec_mux_prepare(&mux,&safe)==0&&stops==1&&port.safe_pending&&port.owner==&route);} if(argv[1][0]=='f') {assert(typec_mux_prepare(&mux,&safe)==0&&port.safe_pending);dcp_typec_rebalance_locked(NULL,0);assert(port.owner==&route&&route.selected&&deactivate_calls==0);port.safe_pending=false;dcp_typec_rebalance_locked(NULL,0);assert(port.owner==NULL&&deactivate_calls==1);} if(argv[1][0]=='t') {route.tunnel=true;assert(typec_mux_prepare(&mux,&safe)==0&&stops==0&&port.dp_wanted);} diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index e64cc11a1eda4..775d67ecf4337 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -65,7 +65,7 @@ def main(): ('phy', 'usb-unreachable', lambda r: pm.ownership(r, True), ((),), ()), ('lifecycle', 'cpuidle-s2idle', idle.s2idle, ((),), ('-Wno-unused-parameter', '-Wno-unused-variable')), ('lifecycle', 'typec-probe', pm.probe, ((),), ()), - ('lifecycle', 'quiesce-before-safe', quiesce.make_unit, tuple((c,) for c in ('ordered','power','ack','role','bolt','mux','duplicate','error-latched','legacy','fabric','tunnel')), ()), + ('lifecycle', 'quiesce-before-safe', quiesce.make_unit, tuple((c,) for c in ('ordered','power','ack','role','bolt','mux','duplicate','error-latched','withdrawn-still-latched','legacy','fabric','tunnel')), ()), ('lifecycle', 'probe-mux-pm-dispatch', pm.probe_dispatch, ((),), ()), ('phy', 'parking', lambda r: parking_unit(r), parking_cases, ()), ('phy', 'adverse-retry', phy.adverse, tuple((c,) for c in ('bist', 'init', 'mux', 'gate', 'dp-cache', 'off', 'success')), ()), @@ -77,6 +77,8 @@ def main(): ('phy', 'aux', phy.aux, ((),), ()), ('lifecycle', 'physical-disconnect', lifecycle.disconnect, ((),), ('-Wno-unused-variable', '-Wno-unused-function', '-Wno-unused-but-set-variable')), + ('lifecycle', 'fast-replug-bounce', lifecycle.bounce, ((),), + ('-Wno-unused-variable', '-Wno-unused-function', '-Wno-unused-but-set-variable')), ('lifecycle', 'dwc3-teardown', lifecycle.teardown, (('all',),), ()), ('lifecycle', 'resource-profile', lifecycle.profile, ((),), ('-Wno-sign-compare',)), ('lifecycle', 'firmware-provider', lifecycle.firmware, ((),), ()), From 8f2ae6f50b5cf1af90e13d8f7dfe584edecd7361 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 18:36:04 +1100 Subject: [PATCH 1090/1579] drm/apple: Say which step of an external display stop failed dcp_external_stop_locked() keeps a session whose teardown failed blocked and owned, and every later SAFE transition of its Type-C route then returns the stored error. The failure itself is silent: the first line in the log is tipd's "failed to place Type-C mux in safe mode", which does not tell a queue that never settled from a pipe power-off or a DPTX withdrawal the firmware did not acknowledge. Name the step that failed, once, when the session is latched. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index bbc7329f591be..a34665f20d169 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2718,6 +2718,7 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) /* The generation token belongs to the CRTC that was actually enabled. */ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) { + const char *step; int ret; lockdep_assert_held(&dcp->external_lock); @@ -2732,11 +2733,16 @@ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) /* Route setup can call this before IOMFB or its shared memory exists. */ if (dcp->external_stopped || !dcp->external_power_attempted) goto release_link; + step = "settling the queue"; ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); - if (!ret) + if (!ret) { + step = "powering the pipe off"; ret = iomfb_external_power_h17p(dcp, false); - if (!ret) + } + if (!ret) { + step = "settling the queue after power-off"; ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); + } if (ret) goto failed; /* This is a transport acknowledgment, not permission to free scanout. */ @@ -2744,6 +2750,7 @@ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) release_link: scoped_guard(mutex, &dcp->hpd_mutex) { if (dcp->dptxport[0].connected) { + step = "withdrawing the DPTX link"; if (!dptx_service_available(&dcp->dptxport[0])) { ret = -ENODEV; goto failed; @@ -2763,6 +2770,7 @@ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) return 0; failed: /* A retry could race a late reply. Keep this session blocked and owned. */ + dev_err(dcp->dev, "external display stop failed %s, session held: %d\n", step, ret); dcp->external_error = ret; return ret; } From d6935850b08e560d9f782d6ee6146ef9d56042e3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:09:43 +1100 Subject: [PATCH 1091/1579] drm/apple: Record an external display announced while its pipe is off While its pipe is stopped, an external H17P display passes no hotplug to DRM: dcp_external_hotplug() parks the callback until the pipe starts again, so that the unplug a DPMS-off stop causes does not tear down the connector. It parks the announcement the firmware sends once it has read a returning display as well, before dcp_firmware_metadata_hotplug() can record that announcement. That is the case on resume from system sleep. Suspend stops the pipe and advances the cable generation, and the firmware announces the display again after resume while the CRTC is still off. The modes that need an announcement for the current generation, such as 1080p at 180 Hz on a two-lane HBR3 link, therefore stay unavailable until userspace enables the CRTC at some other mode and the firmware announces the display a second time. Record an announcement while the pipe is stopped, without notifying the connector, as for every hotplug parked there. Accept it only once the link of the current cable generation has been requested again: a callback that arrives before that belongs to a link already released, whatever the generation counters say by then. An unplug is still not recorded, so a DPMS-off stop keeps the link proof of the mode that DPMS on restores. The apple-dcp lifecycle selftest copies dcp_external_hotplug() into its model of the driver, so give that model the new call as well. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 46 +++++- .../gpu/drm/apple/dcp_external_resume_test.c | 141 ++++++++++++++++++ .../drivers/gpu/apple-dcp/lifecycle.py | 1 + 5 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp_external_resume_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 10fcd21cee723..462221be6efa8 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -13,6 +13,7 @@ appledrm-y += iomfb_v13_3.o appledrm-y += iomfb_h17p.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple_resume_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_resume_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 169b2d17af867..0756e7a5d9d2f 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -502,6 +502,7 @@ struct apple_dcp { void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected); void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected); +void dcp_firmware_metadata_announced(struct apple_dcp *dcp); void dcp_firmware_metadata_begin(struct apple_dcp *dcp); void dcp_firmware_metadata_attributes(struct apple_dcp *dcp, const struct dcp_display_metadata *attributes); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a34665f20d169..feb2bfec2d969 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1963,6 +1963,41 @@ void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, boo spin_unlock_irqrestore(&dcp->metadata.lock, flags); } +static void dcp_firmware_metadata_present_locked(struct apple_dcp *dcp) +{ + lockdep_assert_held(&dcp->metadata.lock); + dcp->metadata.hpd_seen = true; + /* + * Fresh proof may precede HPD assertion after a same-cable loss. + * Keep invalidated firmware metadata and old receipt tickets revoked. + */ + if (READ_ONCE(dcp->typec_cable_connected) && + dcp->metadata.epoch.generation == READ_ONCE(dcp->typec_generation)) + dcp->metadata.epoch.connected = true; +} + +/* + * While its pipe is stopped an external display reports no hotplug to DRM + * (see dcp_external_hotplug()). When the firmware announces the display + * there, as when the link returns from system sleep with the CRTC off, the + * announcement still holds for the modes that need it, so record it. The + * connector is not notified, just as for an announcement on DPMS wake. + * + * Only a link requested for the current cable generation can be announced: + * a callback that arrives before that belongs to a link already released. + */ +void dcp_firmware_metadata_announced(struct apple_dcp *dcp) +{ + unsigned long flags; + + if (!dcp_firmware_metadata_enabled(dcp) || + !READ_ONCE(dcp->dptxport[0].connected)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp_firmware_metadata_present_locked(dcp); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); +} + void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) { struct apple_connector *con = READ_ONCE(dcp->connector); @@ -1972,14 +2007,7 @@ void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) return; spin_lock_irqsave(&dcp->metadata.lock, flags); if (connected) { - dcp->metadata.hpd_seen = true; - /* - * Fresh proof may precede HPD assertion after a same-cable loss. - * Keep invalidated firmware metadata and old receipt tickets revoked. - */ - if (READ_ONCE(dcp->typec_cable_connected) && - dcp->metadata.epoch.generation == READ_ONCE(dcp->typec_generation)) - dcp->metadata.epoch.connected = true; + dcp_firmware_metadata_present_locked(dcp); } else if (dcp->metadata.hpd_seen) { dcp->metadata.hpd_seen = false; dcp->rate_proof.connection_sequence++; @@ -2834,6 +2862,8 @@ bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) if (dcp->iomfb.presentation_paused) { dcp->external_hotplug_pending = true; dcp->external_hotplug_connected = connected; + if (connected) + dcp_firmware_metadata_announced(dcp); /* Publish initial/new-cable discovery, never same-cable DPMS wake. */ if (connected && READ_ONCE(dcp->external_cable) && !READ_ONCE(dcp->external_error) && diff --git a/drivers/gpu/drm/apple/dcp_external_resume_test.c b/drivers/gpu/drm/apple/dcp_external_resume_test.c new file mode 100644 index 0000000000000..409a8cd8ffd8b --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_external_resume_test.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include + +#include "dcp-internal.h" + +/* + * An external H17P DCP back from system sleep on the same cable: suspend + * stopped its pipe and advanced the cable generation, and the CRTC is off. + */ +struct external_resume_test { + struct apple_dcp *dcp; +}; + +/* What the firmware hotplug callback does, under the same lock. */ +static bool external_resume_hotplug(struct apple_dcp *dcp, bool connected) +{ + guard(mutex)(&dcp->iomfb.lock); + return dcp_external_hotplug(dcp, connected); +} + +static struct external_resume_test *external_resume_init(struct kunit *test) +{ + struct external_resume_test *priv; + struct apple_dcp *dcp; + struct device *dev; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, priv); + dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dcp); + dev = kunit_device_register(test, "dcp-external-resume"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + dcp->dev = dev; + dcp->hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL; + spin_lock_init(&dcp->metadata.lock); + mutex_init(&dcp->iomfb.lock); + + /* Connected and announced before the sleep. */ + dcp->typec_cable_connected = true; + dcp->typec_generation = 1; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); + dcp_firmware_metadata_hotplug(dcp, true); + dcp->external_cable = true; + dcp->external_power_attempted = true; + dcp->external_generation = 1; + dcp->external_rx_generation = 1; + dcp->external_crtc_generation = 1; + + /* Suspend: the pipe stopped, then the cable generation advanced. */ + dcp->external_stopped = true; + dcp->iomfb.presentation_paused = true; + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + /* Resume: the reconnect work requested the link again. */ + dcp->dptxport[0].connected = true; + + priv->dcp = dcp; + + return priv; +} + +/* The announcement after the link returns must count with the pipe stopped. */ +static void external_paused_announce_recorded_test(struct kunit *test) +{ + struct external_resume_test *priv = external_resume_init(test); + struct apple_dcp *dcp = priv->dcp; + + KUNIT_ASSERT_FALSE(test, dcp->metadata.epoch.connected); + /* A stopped pipe consumes the callback. */ + KUNIT_EXPECT_TRUE(test, external_resume_hotplug(dcp, true)); + + KUNIT_EXPECT_TRUE(test, dcp->metadata.epoch.connected); + KUNIT_EXPECT_EQ(test, dcp->metadata.epoch.generation, dcp->typec_generation); + /* It is replayed to DRM only when the pipe starts again. */ + KUNIT_EXPECT_TRUE(test, dcp->external_hotplug_pending); + KUNIT_EXPECT_TRUE(test, dcp->external_hotplug_connected); +} + +/* The unplug a DPMS-off stop causes must not revoke the link proof. */ +static void external_paused_unplug_ignored_test(struct kunit *test) +{ + struct external_resume_test *priv = external_resume_init(test); + struct apple_dcp *dcp = priv->dcp; + u64 connection_sequence; + + KUNIT_EXPECT_TRUE(test, external_resume_hotplug(dcp, true)); + connection_sequence = dcp->rate_proof.connection_sequence; + KUNIT_EXPECT_TRUE(test, external_resume_hotplug(dcp, false)); + + KUNIT_EXPECT_TRUE(test, dcp->metadata.epoch.connected); + KUNIT_EXPECT_EQ(test, dcp->rate_proof.connection_sequence, connection_sequence); +} + +/* An announcement on an older cable generation is not current. */ +static void external_paused_stale_announce_test(struct kunit *test) +{ + struct external_resume_test *priv = external_resume_init(test); + struct apple_dcp *dcp = priv->dcp; + + dcp->typec_generation++; + KUNIT_EXPECT_TRUE(test, external_resume_hotplug(dcp, true)); + + KUNIT_EXPECT_FALSE(test, dcp->metadata.epoch.connected); +} + +/* + * A callback delayed past a release of the link is not an announcement of + * the next one, even once software has moved both generations on. + */ +static void external_paused_released_link_test(struct kunit *test) +{ + struct external_resume_test *priv = external_resume_init(test); + struct apple_dcp *dcp = priv->dcp; + + dcp->dptxport[0].connected = false; + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + KUNIT_EXPECT_TRUE(test, external_resume_hotplug(dcp, true)); + + KUNIT_EXPECT_FALSE(test, dcp->metadata.epoch.connected); +} + +static struct kunit_case external_resume_cases[] = { + KUNIT_CASE(external_paused_announce_recorded_test), + KUNIT_CASE(external_paused_unplug_ignored_test), + KUNIT_CASE(external_paused_stale_announce_test), + KUNIT_CASE(external_paused_released_link_test), + {} +}; + +static struct kunit_suite external_resume_suite = { + .name = "apple-dcp-external-resume", + .test_cases = external_resume_cases, +}; + +kunit_test_suite(external_resume_suite); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index e21ca96e971df..030c0c34ff575 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -139,6 +139,7 @@ def function(file, name): static void dcp_firmware_metadata_hotplug(struct apple_dcp *d, bool c) { (void)d; (void)c; } static void dcp_firmware_metadata_connection(struct apple_dcp *d, u64 g, bool c) { (void)d; (void)g; (void)c; } +static void dcp_firmware_metadata_announced(struct apple_dcp *d) { (void)d; } static int dcp_external_disconnect(struct apple_dcp *, bool); static struct apple_dcp display; static void drm_connector_oob_hotplug_event(void *node, int status) From 722fe8472504934b140d98a20a077130bd94cfef Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:31:23 +1100 Subject: [PATCH 1092/1579] drm/apple: Tell when an external display's modes are in for its cable The modes of an external H17P display reach the driver in pieces after its link comes up: the TimingElements catalogue, the AV service its EDID is read from, the firmware's announcement of the display and, for the 1080p 180 Hz exception mode, the link rate and lane count DCP configured. They arrive in no fixed order. A probe made before the last of them lists an incomplete set of modes, and userspace picks a fallback mode from it. Nothing can tell yet whether they are in for the current cable. In particular a disconnect revokes the link and announcement facts but keeps the catalogue, which only the next TimingElements replaces, so a non-empty catalogue can still be the previous connection's. Record the cable generation and connection sequence of the receipt that filled the catalogue, as the exception mode's own ticket already does, and add dcp_external_modes_ready(): the catalogue is the current connection's, its EDID service is up, and an exception mode in it passes every check of dcp_180hz_mode_current() except the EDID, which only the probe reads. That needs the existing check split into its link and its EDID parts; the 180 Hz admission itself is unchanged. Callers follow in later patches. The KUnit suite feeds the facts in each order, replugs with the previous catalogue retained, advances the generation as system sleep does and drops the downstream link, and checks that the 180 Hz mode is admitted exactly when the predicate says the connection is ready. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-180hz.c | 103 ++++- drivers/gpu/drm/apple/dcp-180hz.h | 1 + drivers/gpu/drm/apple/dcp-internal.h | 3 + .../gpu/drm/apple/dcp_external_ready_test.c | 431 ++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 6 + 6 files changed, 526 insertions(+), 19 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp_external_ready_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 462221be6efa8..c44fee6b4943c 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -14,6 +14,7 @@ appledrm-y += iomfb_h17p.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_resume_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_ready_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/dcp-180hz.c b/drivers/gpu/drm/apple/dcp-180hz.c index 8eb4166aa6dbb..09bbc5bd20ad7 100644 --- a/drivers/gpu/drm/apple/dcp-180hz.c +++ b/drivers/gpu/drm/apple/dcp-180hz.c @@ -146,34 +146,49 @@ void dcp_180hz_notify(struct apple_dcp *dcp) schedule_work(&con->hotplug_wq); } -bool dcp_180hz_mode_current(struct apple_dcp *dcp, const struct dcp_display_mode *mode) +/* + * Everything the exception mode needs except the EDID, which only the next + * probe reads: a catalogue received for the current cable and connection, + * the AV service of that receipt, an announced display and the configured + * link rate and lane count. + */ +static bool dcp_180hz_link_current(struct apple_dcp *dcp, const struct dcp_display_mode *mode) { struct apple_connector *con = dcp->connector; - const struct edid *raw; lockdep_assert_held(&dcp->hpd_mutex); lockdep_assert_held(&dcp->dcpavserv.lock); lockdep_assert_held(&dcp->metadata.lock); - if (!con || dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL || - !dcp_180hz_direct_port(dcp->dptxport[0].service) || - !mode->unsafe_rgb8_ticket.associated || - mode->unsafe_rgb8_ticket.generation != dcp->typec_generation || - mode->automatic_connection_sequence != dcp->rate_proof.connection_sequence || - mode->automatic_catalog_sequence != dcp->rate_proof.catalog_sequence || - !dcp->metadata.epoch.connected || !dcp->typec_cable_connected || dcp->quiescing || - READ_ONCE(con->dcp) != to_platform_device(dcp->dev) || + return con && dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && + dcp_180hz_direct_port(dcp->dptxport[0].service) && + mode->unsafe_rgb8_ticket.associated && + mode->unsafe_rgb8_ticket.generation == dcp->typec_generation && + mode->automatic_connection_sequence == dcp->rate_proof.connection_sequence && + mode->automatic_catalog_sequence == dcp->rate_proof.catalog_sequence && + dcp->metadata.epoch.connected && dcp->typec_cable_connected && !dcp->quiescing && + READ_ONCE(con->dcp) == to_platform_device(dcp->dev) && + dcp_180hz_timing_epoch(mode, dcp->dcpavserv.epoch) && + dcp->dptxport[0].connected && dcp->dptxport[0].enabled && !dcp->dptxport[0].closed && + dcp_180hz_fact_current(dcp, &dcp->rate_proof.rate) && + dcp_180hz_fact_current(dcp, &dcp->rate_proof.lanes) && + dcp_180hz_payload(dcp->rate_proof.rate.value, dcp->rate_proof.lanes.value, + mode->mode.clock) && + mode->sdr_rgb.depth == 8 && mode->sdr_rgb.format == DCP_COLOR_FORMAT_RGB && + mode->sdr_rgb.range == DCP_COLOR_YCBCR_RANGE_FULL && + mode->sdr_rgb.eotf == DCP_EOTF_SDR_GAMMA && + mode->sdr_rgb.colorimetry == DCP_COLORIMETRY_SRGB; +} + +bool dcp_180hz_mode_current(struct apple_dcp *dcp, const struct dcp_display_mode *mode) +{ + struct apple_connector *con = dcp->connector; + const struct edid *raw; + + if (!dcp_180hz_link_current(dcp, mode) || !dcpavserv_edid_service_current(dcp, con->drm_edid_service_epoch) || - !dcp_180hz_timing_epoch(mode, dcp->dcpavserv.epoch) || !con->drm_edid || con->drm_edid_firmware_derived || READ_ONCE(con->edid_invalidation_pending) || - con->drm_edid_generation != dcp->typec_generation || - !dcp->dptxport[0].connected || !dcp->dptxport[0].enabled || dcp->dptxport[0].closed || - !dcp_180hz_fact_current(dcp, &dcp->rate_proof.rate) || - !dcp_180hz_fact_current(dcp, &dcp->rate_proof.lanes) || - !dcp_180hz_payload(dcp->rate_proof.rate.value, dcp->rate_proof.lanes.value, mode->mode.clock) || - mode->sdr_rgb.depth != 8 || mode->sdr_rgb.format != DCP_COLOR_FORMAT_RGB || - mode->sdr_rgb.range != DCP_COLOR_YCBCR_RANGE_FULL || - mode->sdr_rgb.eotf != DCP_EOTF_SDR_GAMMA || mode->sdr_rgb.colorimetry != DCP_COLORIMETRY_SRGB) + con->drm_edid_generation != dcp->typec_generation) return false; raw = drm_edid_raw(con->drm_edid); /* The transport validates extent/checksums; this quirk accepts only its 256-byte form. */ @@ -181,6 +196,56 @@ bool dcp_180hz_mode_current(struct apple_dcp *dcp, const struct dcp_display_mode dcp_180hz_edid((const u8 *)raw, 256, &mode->mode); } +/* Caller holds metadata.lock. */ +static bool dcp_catalog_current(struct apple_dcp *dcp) +{ + const struct dcp_metadata_ticket *ticket = &dcp->metadata.catalog_ticket; + + return ticket->associated && dcp->nr_modes && + ticket->generation == dcp->typec_generation && + ticket->generation == dcp->metadata.epoch.generation && + dcp->metadata.catalog_connection_sequence == dcp->rate_proof.connection_sequence; +} + +/** + * dcp_external_modes_ready - whether a probe would list the current modes + * @dcp: an external H17P DCP + * + * The modes of an external display arrive in pieces, in no fixed order: the + * TimingElements catalogue, the AV service the EDID is read from, the + * firmware's announcement and the link configuration. A connection is + * ready once the catalogue is the current cable's, its EDID service is up + * and, if the catalogue holds an exception mode, that mode passes every + * check of dcp_180hz_mode_current() but the EDID the probe will read. A + * catalogue retained from an earlier connection never counts. + * + * Caller holds hpd_mutex. + */ +bool dcp_external_modes_ready(struct apple_dcp *dcp) +{ + bool ready, exception = false, proven = false; + unsigned long flags; + unsigned int i; + + lockdep_assert_held(&dcp->hpd_mutex); + guard(mutex)(&dcp->dcpavserv.lock); + if (!dcpavserv_edid_current(dcp, dcp->typec_generation) || + !dcpavserv_edid_service_current(dcp, dcp->dcpavserv.epoch)) + return false; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + ready = dcp_catalog_current(dcp); + for (i = 0; ready && i < dcp->nr_modes; i++) { + if (!dcp->modes[i].automatic_rgb8_quirk) + continue; + exception = true; + proven |= dcp_180hz_link_current(dcp, &dcp->modes[i]); + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + + return ready && (!exception || proven); +} + bool dcp_180hz_pending(struct apple_dcp *dcp, const struct dcp_display_mode *mode) { struct apple_connector *con = dcp->connector; diff --git a/drivers/gpu/drm/apple/dcp-180hz.h b/drivers/gpu/drm/apple/dcp-180hz.h index 4995dae4cf648..87e9258a6782b 100644 --- a/drivers/gpu/drm/apple/dcp-180hz.h +++ b/drivers/gpu/drm/apple/dcp-180hz.h @@ -28,6 +28,7 @@ void dcp_180hz_invalidate(struct apple_dcp *dcp); void dcp_180hz_notify(struct apple_dcp *dcp); void dcp_180hz_catalog_changed(struct apple_dcp *dcp); bool dcp_180hz_mode_current(struct apple_dcp *dcp, const struct dcp_display_mode *mode); +bool dcp_external_modes_ready(struct apple_dcp *dcp); bool dcp_180hz_pending(struct apple_dcp *dcp, const struct dcp_display_mode *mode); void dcp_180hz_retry(struct apple_dcp *dcp); int dcp_180hz_init(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 0756e7a5d9d2f..03d47c0de6019 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -309,6 +309,9 @@ struct apple_dcp { struct dcp_display_metadata attributes; struct drm_display_mode preferred; struct dcp_metadata_ticket attributes_ticket, timing_ticket; + /* The connection whose TimingElements filled the mode catalogue. */ + struct dcp_metadata_ticket catalog_ticket; + u64 catalog_connection_sequence; bool attributes_valid, timing_valid, hpd_seen; } metadata; diff --git a/drivers/gpu/drm/apple/dcp_external_ready_test.c b/drivers/gpu/drm/apple/dcp_external_ready_test.c new file mode 100644 index 0000000000000..80685ef9e0ad8 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_external_ready_test.c @@ -0,0 +1,431 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include +#include + +#include + +#include "afk.h" +#include "connector.h" +#include "dcp-internal.h" +#include "parser.h" + +/* + * An external H17P DCP driving the QG270 S3 at 1920x1080 and 180 Hz over a + * direct two-lane HBR3 link: the facts its first probe needs arrive one by + * one, in any order. + */ +struct ready_test { + struct apple_dcp *dcp; + struct apple_connector *con; + struct apple_dcp_afkep dptx_ep; + struct apple_epic_service dptx; + struct apple_epic_service av; + struct dcp_display_mode modes[2]; +}; + +static const struct drm_display_mode ready_mode_180 = { + .clock = 414090, + .hdisplay = 1920, .hsync_start = 1968, .hsync_end = 2000, .htotal = 2080, + .vdisplay = 1080, .vsync_start = 1083, .vsync_end = 1088, .vtotal = 1106, + .flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC, +}; + +static const struct drm_display_mode ready_mode_60 = { + .clock = 148500, + .hdisplay = 1920, .hsync_start = 2008, .hsync_end = 2052, .htotal = 2200, + .vdisplay = 1080, .vsync_start = 1084, .vsync_end = 1089, .vtotal = 1125, + .flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC, +}; + +static void ready_checksum(u8 *block) +{ + u8 sum = 0; + int i; + + for (i = 0; i < 127; i++) + sum += block[i]; + block[127] = -sum; +} + +/* A 256-byte EDID that admits ready_mode_180. */ +static const struct drm_edid *ready_edid(struct kunit *test) +{ + static const u8 header[] = { 0, 255, 255, 255, 255, 255, 255, 0 }; + static const u8 dtd[] = { + 0xc1, 0xa1, 0x80, 0xa0, 0x70, 0x38, 0x1a, 0x40, 0x30, + 0x20, 0x35, 0x00, 0, 0, 0, 0, 0, 0x1a, + }; + static const u8 name[] = { + 0, 0, 0, 0xfc, 0, 'Q', 'G', '2', '7', '0', ' ', 'S', '3', '\n', + ' ', ' ', ' ', ' ', + }; + static const u8 hf_vsdb[] = { 0x67, 0xd8, 0x5d, 0xc4, 1, 0x54, 0x80, 0 }; + const struct drm_edid *edid; + u8 *p; + + p = kunit_kzalloc(test, 256, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, p); + memcpy(p, header, sizeof(header)); + p[8] = 0x04; + p[9] = 0x72; + p[10] = 0x4f; + p[11] = 0x0b; + p[18] = 1; + p[19] = 4; + p[20] = 0x80; + memcpy(p + 54, dtd, sizeof(dtd)); + memcpy(p + 72, name, sizeof(name)); + p[126] = 1; + ready_checksum(p); + p[128] = 2; + p[129] = 3; + p[130] = 4 + sizeof(hf_vsdb); + memcpy(p + 132, hf_vsdb, sizeof(hf_vsdb)); + ready_checksum(p + 128); + + edid = drm_edid_alloc(p, 256); + KUNIT_ASSERT_NOT_NULL(test, edid); + return edid; +} + +static void ready_free_edid(void *data) +{ + drm_edid_free(data); +} + +static struct ready_test *ready_init(struct kunit *test) +{ + struct platform_device *pdev; + struct ready_test *t; + struct apple_dcp *dcp; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dcp); + t->con = kunit_kzalloc(test, sizeof(*t->con), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t->con); + pdev = kunit_platform_device_alloc(test, "dcp-external-ready", PLATFORM_DEVID_AUTO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev); + KUNIT_ASSERT_EQ(test, kunit_platform_device_add(test, pdev), 0); + + t->dcp = dcp; + dcp->dev = &pdev->dev; + dcp->hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL; + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->dcpavserv.lock); + spin_lock_init(&dcp->metadata.lock); + dcp->connector = t->con; + t->con->dcp = pdev; + t->con->connected = true; + + /* A direct route: the PHY and route are only compared, never used. */ + dcp->phy = kunit_kzalloc(test, 1, GFP_KERNEL); + dcp->active_typec_route = kunit_kzalloc(test, sizeof(*dcp->active_typec_route), + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dcp->phy); + KUNIT_ASSERT_NOT_NULL(test, dcp->active_typec_route); + t->dptx_ep.dcp = dcp; + t->dptx_ep.endpoint = DPTX_ENDPOINT_V2; + t->dptx_ep.wire_v2 = true; + t->dptx.ep = &t->dptx_ep; + t->dptx.cookie = &dcp->dptxport[0]; + t->dptx.enabled = true; + dcp->dptxport[0].service = &t->dptx; + dcp->dptxport[0].enabled = true; + dcp->dptxport[0].connected = true; + dcp->dptxport[0].atcphy = dcp->phy; + + t->modes[0].mode = ready_mode_60; + t->modes[1].mode = ready_mode_180; + t->modes[1].automatic_rgb8_quirk = true; + t->modes[1].sdr_rgb = (struct dcp_color_mode) { + .depth = 8, + .format = DCP_COLOR_FORMAT_RGB, + .range = DCP_COLOR_YCBCR_RANGE_FULL, + .eotf = DCP_EOTF_SDR_GAMMA, + .colorimetry = DCP_COLORIMETRY_SRGB, + }; + + /* A new cable. */ + dcp->typec_cable_connected = true; + dcp->typec_generation = 1; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); + + return t; +} + +/* What dcpep_process_chunks() publishes for a complete TimingElements. */ +static void ready_catalogue(struct ready_test *t, unsigned int nr_modes) +{ + struct apple_dcp *dcp = t->dcp; + struct dcp_metadata_ticket ticket; + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + ticket = dcp_metadata_begin(&dcp->metadata.epoch); + ticket.associated = dcp->metadata.epoch.generation == dcp->typec_generation && + dcp->typec_cable_connected; + dcp->rate_proof.catalog_sequence++; + for (i = 0; i < nr_modes; i++) { + if (!t->modes[i].automatic_rgb8_quirk) + continue; + t->modes[i].unsafe_rgb8_ticket = ticket; + t->modes[i].unsafe_rgb8_service_epoch = dcp->dcpavserv.epoch; + t->modes[i].automatic_service_pending = !dcp->dcpavserv.enabled && + !dcp->dcpavserv.service; + t->modes[i].automatic_connection_sequence = dcp->rate_proof.connection_sequence; + t->modes[i].automatic_catalog_sequence = dcp->rate_proof.catalog_sequence; + } + dcp->modes = t->modes; + dcp->nr_modes = nr_modes; + dcp->metadata.catalog_ticket = ticket; + dcp->metadata.catalog_ticket.associated &= nr_modes > 0; + dcp->metadata.catalog_connection_sequence = dcp->rate_proof.connection_sequence; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); +} + +/* The AV service the EDID is read from comes up (dcpavserv_init()). */ +static void ready_service(struct ready_test *t) +{ + struct apple_dcp *dcp = t->dcp; + + guard(mutex)(&dcp->dcpavserv.lock); + dcp->dcpavserv.service = &t->av; + dcp->dcpavserv.enabled = true; + dcp->dcpavserv.epoch++; +} + +/* The firmware announces the display (dcpep_cb_hotplug()). */ +static void ready_announce(struct ready_test *t) +{ + dcp_firmware_metadata_hotplug(t->dcp, true); +} + +/* The DPTX endpoint reports the link rate and lane count it configured. */ +static void ready_link(struct ready_test *t, u32 rate, u32 lanes) +{ + struct dcp_180hz_token token; + + token = dcp_180hz_config_begin(&t->dptx, false); + dcp_180hz_config_complete(&t->dptx, false, token, rate, true); + token = dcp_180hz_config_begin(&t->dptx, true); + dcp_180hz_config_complete(&t->dptx, true, token, lanes, true); +} + +static bool ready(struct ready_test *t) +{ + guard(mutex)(&t->dcp->hpd_mutex); + return dcp_external_modes_ready(t->dcp); +} + +/* What dcp_get_modes() decides for the 180 Hz mode once it has read the EDID. */ +static bool ready_probe_lists_180(struct kunit *test, struct ready_test *t) +{ + struct apple_dcp *dcp = t->dcp; + const struct drm_edid *edid = ready_edid(test); + unsigned long flags; + bool listed; + + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, ready_free_edid, (void *)edid), 0); + guard(mutex)(&dcp->hpd_mutex); + guard(mutex)(&dcp->dcpavserv.lock); + t->con->drm_edid = edid; + t->con->drm_edid_generation = dcp->typec_generation; + t->con->drm_edid_service_epoch = dcp->dcpavserv.epoch; + spin_lock_irqsave(&dcp->metadata.lock, flags); + listed = dcp_180hz_mode_current(dcp, &t->modes[1]); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + t->con->drm_edid = NULL; + return listed; +} + +static void ready_complete_test(struct kunit *test) +{ + struct ready_test *t = ready_init(test); + + ready_catalogue(t, 2); + ready_service(t); + ready_announce(t); + ready_link(t, 0x1e, 2); + + KUNIT_EXPECT_TRUE(test, ready(t)); + KUNIT_EXPECT_TRUE(test, ready_probe_lists_180(test, t)); +} + +enum ready_fact { READY_CATALOGUE, READY_SERVICE, READY_ANNOUNCE, READY_LINK, READY_FACTS }; + +static void ready_fact(struct ready_test *t, enum ready_fact fact) +{ + switch (fact) { + case READY_CATALOGUE: + ready_catalogue(t, 2); + break; + case READY_SERVICE: + ready_service(t); + break; + case READY_ANNOUNCE: + ready_announce(t); + break; + default: + ready_link(t, 0x1e, 2); + break; + } +} + +/* + * Back from system sleep on the same cable: suspend advanced the generation + * without a new connection, so the firmware has to announce the display + * again. (A new cable counts as announced from dcp_dptx_connect_oob().) + * Each fact can come last; until it has, a probe would miss the 180 Hz mode. + */ +static void ready_any_order_test(struct kunit *test) +{ + enum ready_fact last, fact; + + for (last = 0; last < READY_FACTS; last++) { + struct ready_test *t = ready_init(test); + + t->dcp->typec_generation++; + dcp_firmware_metadata_connection(t->dcp, t->dcp->typec_generation, false); + + for (fact = 0; fact < READY_FACTS; fact++) + if (fact != last) + ready_fact(t, fact); + KUNIT_EXPECT_FALSE_MSG(test, ready(t), "fact %d missing", last); + KUNIT_EXPECT_FALSE_MSG(test, ready_probe_lists_180(test, t), + "fact %d missing", last); + + ready_fact(t, last); + KUNIT_EXPECT_TRUE_MSG(test, ready(t), "fact %d last", last); + KUNIT_EXPECT_TRUE_MSG(test, ready_probe_lists_180(test, t), + "fact %d last", last); + } +} + +/* + * A replug keeps the previous connection's catalogue until TimingElements + * replaces it: that catalogue and a new EDID service must not count. + */ +static void ready_retained_catalogue_test(struct kunit *test) +{ + struct ready_test *t = ready_init(test); + struct apple_dcp *dcp = t->dcp; + + ready_catalogue(t, 2); + ready_service(t); + ready_announce(t); + ready_link(t, 0x1e, 2); + KUNIT_ASSERT_TRUE(test, ready(t)); + + /* Unplug (dcp_external_disconnect()), then a new cable. */ + dcp->typec_cable_connected = false; + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + dcp->dcpavserv.enabled = false; + dcp->dcpavserv.service = NULL; + dcp->dcpavserv.epoch++; + dcp->typec_cable_connected = true; + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); + + ready_service(t); + ready_announce(t); + ready_link(t, 0x1e, 2); + KUNIT_EXPECT_EQ(test, dcp->nr_modes, 2); + KUNIT_EXPECT_FALSE(test, ready(t)); + + ready_catalogue(t, 2); + KUNIT_EXPECT_TRUE(test, ready(t)); +} + +/* + * System sleep advances the generation of a cable that stays attached. + * Facts from before it, and a callback delivered after it, prove nothing. + */ +static void ready_generation_advance_test(struct kunit *test) +{ + struct ready_test *t = ready_init(test); + struct apple_dcp *dcp = t->dcp; + + ready_catalogue(t, 2); + ready_service(t); + ready_announce(t); + ready_link(t, 0x1e, 2); + KUNIT_ASSERT_TRUE(test, ready(t)); + + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + KUNIT_EXPECT_FALSE(test, ready(t)); + + /* A late announcement of the old link, with the new generation set. */ + ready_announce(t); + KUNIT_EXPECT_FALSE(test, ready(t)); + KUNIT_EXPECT_FALSE(test, ready_probe_lists_180(test, t)); + + ready_catalogue(t, 2); + ready_link(t, 0x1e, 2); + KUNIT_EXPECT_TRUE(test, ready(t)); +} + +/* The display behind a dock that stays attached drops its link. */ +static void ready_downstream_loss_test(struct kunit *test) +{ + struct ready_test *t = ready_init(test); + + ready_catalogue(t, 2); + ready_service(t); + ready_announce(t); + ready_link(t, 0x1e, 2); + KUNIT_ASSERT_TRUE(test, ready(t)); + + dcp_firmware_metadata_hotplug(t->dcp, false); + KUNIT_EXPECT_FALSE(test, ready(t)); +} + +/* Without an exception mode, the catalogue and the EDID service suffice. */ +static void ready_ordinary_catalogue_test(struct kunit *test) +{ + struct ready_test *t = ready_init(test); + + ready_catalogue(t, 1); + KUNIT_EXPECT_FALSE(test, ready(t)); + ready_service(t); + KUNIT_EXPECT_TRUE(test, ready(t)); +} + +/* A link configured at another rate never proves the 180 Hz mode. */ +static void ready_other_link_test(struct kunit *test) +{ + struct ready_test *t = ready_init(test); + + ready_catalogue(t, 2); + ready_service(t); + ready_announce(t); + ready_link(t, 0x14, 2); + KUNIT_EXPECT_FALSE(test, ready(t)); + KUNIT_EXPECT_FALSE(test, ready_probe_lists_180(test, t)); +} + +static struct kunit_case ready_cases[] = { + KUNIT_CASE(ready_complete_test), + KUNIT_CASE(ready_any_order_test), + KUNIT_CASE(ready_retained_catalogue_test), + KUNIT_CASE(ready_generation_advance_test), + KUNIT_CASE(ready_downstream_loss_test), + KUNIT_CASE(ready_ordinary_catalogue_test), + KUNIT_CASE(ready_other_link_test), + {} +}; + +static struct kunit_suite ready_suite = { + .name = "apple-dcp-external-ready", + .test_cases = ready_cases, +}; + +kunit_test_suite(ready_suite); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index fd985debcdbe5..2193417455131 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1021,6 +1021,7 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, spin_lock_irqsave(&dcp->metadata.lock, flags); dcp->rate_proof.catalog_sequence++; + dcp->metadata.catalog_ticket.associated = false; spin_unlock_irqrestore(&dcp->metadata.lock, flags); dcp_180hz_catalog_changed(dcp); } @@ -1069,6 +1070,11 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, modes[i].automatic_catalog_sequence = dcp->rate_proof.catalog_sequence; dcp->modes = modes; dcp->nr_modes = nr_modes; + /* Only a complete receipt for a current cable is that cable's catalogue. */ + dcp->metadata.catalog_ticket = rate_ticket; + dcp->metadata.catalog_ticket.associated &= nr_modes > 0; + dcp->metadata.catalog_connection_sequence = + dcp->chunks.automatic_connection_sequence; spin_unlock_irqrestore(&dcp->metadata.lock, flags); /* Notify removals as well as newly eligible modes, outside the lock. */ dcp_180hz_catalog_changed(dcp); From 28758de011dde69d28d19168eb7adcf08d7e7aa9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:31:23 +1100 Subject: [PATCH 1093/1579] drm/apple: Add a state machine for announcing an external display Userspace that probes an external display before its modes are in picks a mode from an incomplete list and switches again once the rest arrives. That costs a second mode set, which the monitor has to resynchronize to, both when a display is plugged in and when its CRTC comes back from system sleep disabled and userspace sets its mode again. Add the policy for when DRM reports such a display and when it sends the hotplug event for it, as a pure helper that a later patch wires up: - A new connection is reported once it is ready, or after 1 s without that. - A reported connection stays reported until the display or the cable goes away, so a link retrained on the same cable does not withdraw it. A cable known to be gone wins at once. - A reported connection can hold its next hotplug event, never the connection itself, until it is ready again, or for 2 s without that. - The timeout of a hold runs from when the caller starts it, which it does only once the link can come back, and only for the cable generation it was started in. A hold that outlives its generation, as across another system sleep, waits for the next one from the start, even if its old deadline is still to come. A hold whose link failed ends at once. - A hold that ends with the connection reported owes userspace one hotplug event, as does a reported connection that ends, until the hotplug worker hands it out. Whoever ends a hold can tell from that whether it has to queue the worker, and evaluating the connection again does not owe a second event. The state also carries the flag with which a later patch stops the announcement for teardown and system sleep. The KUnit suite covers both kinds of hold, their timeouts, a jiffies wrap, the generation rule, failure, unplug during a hold, the event a release owes and a replug the hotplug worker only sees once the new link is up. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 3 +- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-announce-test.c | 404 ++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-announce.h | 151 ++++++++ 4 files changed, 558 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/apple/dcp-announce-test.c create mode 100644 drivers/gpu/drm/apple/dcp-announce.h diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 9c1ce96bc0dbb..3d2a83b1369c1 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -42,4 +42,5 @@ config DRM_APPLE_KUNIT_TEST larger rings, as well as malformed ring size combinations. This option is intended for developer test kernels. Also test the Type-C modeset, hotplug and service ownership state, - and the system resume of a CRTC whose mode is no longer admissible. + the system resume of a CRTC whose mode is no longer admissible, and + when an external display is ready to be reported to userspace. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index c44fee6b4943c..99e79dfbcc0a4 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -24,5 +24,6 @@ appledrm-$(CONFIG_TRACING) += trace.o obj-$(CONFIG_DRM_APPLE) += appledrm.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk_test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_backlight_test.o +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp-announce-test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk-service-test.o diff --git a/drivers/gpu/drm/apple/dcp-announce-test.c b/drivers/gpu/drm/apple/dcp-announce-test.c new file mode 100644 index 0000000000000..bdfd9ef4db774 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-announce-test.c @@ -0,0 +1,404 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include + +#include "dcp-announce.h" + +#define TIMEOUT msecs_to_jiffies(DCP_ANNOUNCE_TIMEOUT_MS) +#define RESUME_TIMEOUT msecs_to_jiffies(DCP_ANNOUNCE_RESUME_TIMEOUT_MS) + +static const struct dcp_announce_facts link_only = { + .generation = 1, .connected = true, .present = true, +}; + +static const struct dcp_announce_facts link_ready = { + .generation = 1, .connected = true, .present = true, .ready = true, +}; + +static const struct dcp_announce_facts unplugged = { + .generation = 2, +}; + +static void announce_ready_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_ready, 100), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, a.announced); + KUNIT_EXPECT_FALSE(test, a.holding); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); +} + +static void announce_waits_for_ready_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 100), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_TRUE(test, a.holding); + KUNIT_EXPECT_FALSE(test, a.announced); + KUNIT_EXPECT_FALSE(test, a.timed); + KUNIT_EXPECT_FALSE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_HOLD)); + dcp_announce_start(&a, 100); + KUNIT_EXPECT_EQ(test, a.deadline, 100 + TIMEOUT); + + /* Neither re-evaluation nor starting again extends the deadline. */ + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 100 + TIMEOUT - 1), + DCP_ANNOUNCE_HOLD); + dcp_announce_start(&a, 100 + TIMEOUT - 1); + KUNIT_EXPECT_EQ(test, a.deadline, 100 + TIMEOUT); + + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_ready, 100 + TIMEOUT - 1), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_FALSE(test, a.holding); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); +} + +static void announce_timeout_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_only, 100); + dcp_announce_start(&a, 100); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 1, 100 + TIMEOUT - 1)); + KUNIT_EXPECT_TRUE(test, dcp_announce_expired(&a, 1, 100 + TIMEOUT)); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 100 + TIMEOUT), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, a.announced); + KUNIT_EXPECT_FALSE(test, a.holding); + /* A timer that fires after the hold ended finds nothing to release. */ + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 1, 100 + 2 * TIMEOUT)); +} + +static void announce_timeout_wraps_test(struct kunit *test) +{ + unsigned long start = ULONG_MAX - TIMEOUT / 2; + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_only, start); + dcp_announce_start(&a, start); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, start + TIMEOUT - 1), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, start + TIMEOUT), + DCP_ANNOUNCE_CONNECTED); +} + +/* A failed link will not get ready; report it without waiting. */ +static void announce_failed_test(struct kunit *test) +{ + struct dcp_announce_facts f = link_only; + struct dcp_announce a = { }; + + f.failed = true; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, 0), DCP_ANNOUNCE_CONNECTED); +} + +/* A link retrained on the same cable must not withdraw the connection. */ +static void announce_keeps_reported_test(struct kunit *test) +{ + struct dcp_announce_facts f = link_only; + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_ready, 0); + f.generation = 2; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, 10 * TIMEOUT), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_FALSE(test, a.holding); +} + +/* A missing cable wins over a link DCP has not dropped yet. */ +static void announce_absent_wins_test(struct kunit *test) +{ + struct dcp_announce_facts f = link_ready; + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_ready, 0); + f.present = false; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, 1), DCP_ANNOUNCE_DISCONNECTED); + KUNIT_EXPECT_FALSE(test, a.announced); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_DISCONNECTED)); +} + +static void announce_disconnect_during_hold_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_only, 0); + dcp_announce_start(&a, 0); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &unplugged, 1), + DCP_ANNOUNCE_DISCONNECTED); + KUNIT_EXPECT_FALSE(test, a.holding); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 1, TIMEOUT)); + + /* The next connection gets a full timeout of its own. */ + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 5 * TIMEOUT), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_FALSE(test, a.timed); + dcp_announce_start(&a, 5 * TIMEOUT); + KUNIT_EXPECT_EQ(test, a.deadline, 5 * TIMEOUT + TIMEOUT); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 1, TIMEOUT)); +} + +/* + * A replug that the hotplug worker only sees once the new link is up: the + * old connection still needs its event while the new one is held. + */ +static void announce_replug_test(struct kunit *test) +{ + struct dcp_announce_facts f = link_only; + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_ready, 0); + dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED); + dcp_announce_end(&a); + f.generation = 2; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, 1), DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_HOLD)); + KUNIT_EXPECT_FALSE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_HOLD)); + f.ready = true; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, 2), DCP_ANNOUNCE_CONNECTED); +} + +/* Ending a connection that was never reported owes userspace nothing. */ +static void announce_end_unreported_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_only, 0); + dcp_announce_end(&a); + KUNIT_EXPECT_FALSE(test, a.withdrawn); + KUNIT_EXPECT_FALSE(test, a.holding); +} + +/* Only a reported connection has an event to hold. */ +static void hold_unreported_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + KUNIT_EXPECT_FALSE(test, dcp_announce_hold(&a, 1)); + KUNIT_EXPECT_FALSE(test, a.holding); +} + +static struct dcp_announce held(u64 generation) +{ + struct dcp_announce a = { .announced = true }; + + dcp_announce_hold(&a, generation); + return a; +} + +/* + * A held event keeps the connection reported, and its timeout runs only once + * started: the link cannot come back before then. + */ +static void hold_reported_test(struct kunit *test) +{ + struct dcp_announce a = held(1); + + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 0), DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_TRUE(test, a.announced); + KUNIT_EXPECT_FALSE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_HOLD)); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 100 * RESUME_TIMEOUT), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 1, 100 * RESUME_TIMEOUT)); + + dcp_announce_start(&a, 1000); + KUNIT_EXPECT_EQ(test, a.deadline, 1000 + RESUME_TIMEOUT); + /* Starting again does not move the deadline. */ + dcp_announce_start(&a, 2000); + KUNIT_EXPECT_EQ(test, a.deadline, 1000 + RESUME_TIMEOUT); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 1000 + RESUME_TIMEOUT - 1), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_TRUE(test, dcp_announce_expired(&a, 1, 1000 + RESUME_TIMEOUT)); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 1000 + RESUME_TIMEOUT), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); + KUNIT_EXPECT_FALSE(test, a.holding); +} + +static void hold_ready_test(struct kunit *test) +{ + struct dcp_announce a = held(1); + + dcp_announce_start(&a, 0); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 1), DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_ready, 2), DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 1, RESUME_TIMEOUT)); +} + +static void hold_failed_test(struct kunit *test) +{ + struct dcp_announce_facts f = link_only; + struct dcp_announce a = held(1); + + f.failed = true; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, 0), DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, a.announced); +} + +/* An unplug while the event is held is reported at once. */ +static void hold_unplug_test(struct kunit *test) +{ + struct dcp_announce a = held(1); + + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &unplugged, 0), + DCP_ANNOUNCE_DISCONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_DISCONNECTED)); + KUNIT_EXPECT_FALSE(test, a.holding); +} + +/* + * A hold that outlives its cable generation, as across another sleep, waits + * for the next one from the start: the old deadline is not carried over. + */ +static void hold_next_generation_test(struct kunit *test) +{ + struct dcp_announce_facts f = link_only; + struct dcp_announce a = held(1); + + dcp_announce_start(&a, 0); + f.generation = 2; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, RESUME_TIMEOUT), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_EQ(test, a.generation, 2); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 2, 10 * RESUME_TIMEOUT)); + dcp_announce_start(&a, 10 * RESUME_TIMEOUT); + KUNIT_EXPECT_EQ(test, a.deadline, 11 * RESUME_TIMEOUT); +} + +/* + * The deadline of one generation never ends the hold of the next, even + * before an update has moved the hold on. + */ +static void hold_deadline_generation_test(struct kunit *test) +{ + struct dcp_announce a = held(1); + + dcp_announce_start(&a, 0); + KUNIT_EXPECT_TRUE(test, dcp_announce_expired(&a, 1, RESUME_TIMEOUT)); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 2, RESUME_TIMEOUT)); +} + +/* A new connection seen first in a later generation restarts its timeout. */ +static void announce_next_generation_test(struct kunit *test) +{ + struct dcp_announce_facts f = link_only; + struct dcp_announce a = { }; + + dcp_announce_update(&a, &f, 0); + dcp_announce_start(&a, 0); + f.generation = 2; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &f, TIMEOUT), DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_FALSE(test, a.timed); + dcp_announce_start(&a, 3 * TIMEOUT); + KUNIT_EXPECT_EQ(test, a.deadline, 4 * TIMEOUT); +} + +/* Until its timeout is started, a hold waits however long it takes. */ +static void announce_untimed_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_only, 0); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 100 * TIMEOUT), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_FALSE(test, dcp_announce_expired(&a, 1, 100 * TIMEOUT)); +} + +/* + * However a new connection is released, and whoever evaluates it, its event + * is owed until it is handed out, and it is handed out once. + */ +static void announce_release_owed_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + dcp_announce_update(&a, &link_only, 0); + dcp_announce_start(&a, 0); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, TIMEOUT), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_owed(&a)); + /* Evaluating it again does not hand the event out. */ + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, TIMEOUT + 1), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_owed(&a)); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); +} + +/* An evaluation after the hold ended and its event went out owes nothing. */ +static void hold_release_once_test(struct kunit *test) +{ + struct dcp_announce a = held(1); + + dcp_announce_start(&a, 0); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, RESUME_TIMEOUT), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_owed(&a)); + dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 2 * RESUME_TIMEOUT), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); +} + +/* + * A new hold before the event of the last one went out holds that event + * too: it goes out once, when the new hold ends. + */ +static void hold_again_test(struct kunit *test) +{ + struct dcp_announce a = held(1); + + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_ready, 0), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_hold(&a, 2)); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); + KUNIT_EXPECT_FALSE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_HOLD)); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_ready, 1), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_owed(&a)); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); +} + +static struct kunit_case announce_cases[] = { + KUNIT_CASE(announce_ready_test), + KUNIT_CASE(announce_waits_for_ready_test), + KUNIT_CASE(announce_timeout_test), + KUNIT_CASE(announce_timeout_wraps_test), + KUNIT_CASE(announce_failed_test), + KUNIT_CASE(announce_keeps_reported_test), + KUNIT_CASE(announce_absent_wins_test), + KUNIT_CASE(announce_disconnect_during_hold_test), + KUNIT_CASE(announce_replug_test), + KUNIT_CASE(announce_end_unreported_test), + KUNIT_CASE(announce_next_generation_test), + KUNIT_CASE(announce_untimed_test), + KUNIT_CASE(announce_release_owed_test), + KUNIT_CASE(hold_unreported_test), + KUNIT_CASE(hold_reported_test), + KUNIT_CASE(hold_ready_test), + KUNIT_CASE(hold_failed_test), + KUNIT_CASE(hold_unplug_test), + KUNIT_CASE(hold_next_generation_test), + KUNIT_CASE(hold_deadline_generation_test), + KUNIT_CASE(hold_release_once_test), + KUNIT_CASE(hold_again_test), + {} +}; + +static struct kunit_suite announce_suite = { + .name = "apple-dcp-announce", + .test_cases = announce_cases, +}; + +kunit_test_suite(announce_suite); + +MODULE_DESCRIPTION("KUnit tests for the Apple DCP external connection announcement"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/dcp-announce.h b/drivers/gpu/drm/apple/dcp-announce.h new file mode 100644 index 0000000000000..cb360e48eb316 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-announce.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __APPLE_DCP_ANNOUNCE_H__ +#define __APPLE_DCP_ANNOUNCE_H__ + +#include +#include + +/* + * The modes of an external display arrive in pieces after its link comes up + * (see dcp_external_modes_ready()). Userspace that probes before they are + * all in picks a mode from an incomplete list, then switches again once the + * rest arrives: a second mode set the monitor has to resynchronize to. + * + * - A new connection is reported to DRM once it is ready, or once + * DCP_ANNOUNCE_TIMEOUT_MS have passed without that. + * - Once reported, a connection stays reported until the display or the + * cable goes away: a link retrained on the same cable, for instance after + * system sleep, does not withdraw it. A cable that is gone is reported as + * such at once, even while DCP still holds the old link. + * - A reported connection can hold its next hotplug event, never the + * connection itself, until it is ready again, or until + * DCP_ANNOUNCE_RESUME_TIMEOUT_MS have passed without that. That is the + * event that has userspace set the mode of a CRTC resumed disabled. + * + * The timeout of a hold runs from dcp_announce_start(), and only for the + * cable generation it was started in. A hold that outlives its generation, + * as across another system sleep, waits for the next one from the start. + * + * A hold that ends with the connection reported owes userspace a hotplug + * event, as does a reported connection that ends, until + * dcp_announce_notify() hands that event out. + */ +#define DCP_ANNOUNCE_TIMEOUT_MS 1000 +#define DCP_ANNOUNCE_RESUME_TIMEOUT_MS 2000 + +enum dcp_announce_status { + DCP_ANNOUNCE_DISCONNECTED, + DCP_ANNOUNCE_HOLD, /* the hotplug event is held */ + DCP_ANNOUNCE_CONNECTED, +}; + +struct dcp_announce { + bool announced; /* DRM reports the connection */ + bool withdrawn; /* a reported connection ended, no event sent yet */ + bool released; /* a hold ended, no event sent yet */ + bool holding; /* waiting for the connection to be ready */ + bool timed; /* the hold of @generation ends at @deadline */ + bool stopped; /* nothing may be queued for the announcement */ + u64 generation; /* the cable generation of the hold */ + unsigned long deadline; +}; + +struct dcp_announce_facts { + u64 generation; /* the current cable generation */ + bool connected; /* DCP reports the display */ + bool present; /* the cable is not known to be gone */ + bool ready; /* a probe would list the modes of this connection */ + bool failed; /* the link failed; it will not get ready */ +}; + +/* The connection ended, whether DRM was told about it or not. */ +static inline void dcp_announce_end(struct dcp_announce *a) +{ + a->withdrawn |= a->announced; + a->announced = false; + a->holding = false; +} + +/* + * Hold the next hotplug event of a reported connection until it is ready on + * cable generation @generation. + */ +static inline bool dcp_announce_hold(struct dcp_announce *a, u64 generation) +{ + if (!a->announced) + return false; + a->holding = true; + a->timed = false; + a->generation = generation; + return true; +} + +/* Start the timeout of a hold that has none for its generation yet. */ +static inline void dcp_announce_start(struct dcp_announce *a, unsigned long now) +{ + if (!a->holding || a->timed) + return; + a->timed = true; + a->deadline = now + msecs_to_jiffies(a->announced ? DCP_ANNOUNCE_RESUME_TIMEOUT_MS : + DCP_ANNOUNCE_TIMEOUT_MS); +} + +static inline bool dcp_announce_expired(const struct dcp_announce *a, u64 generation, + unsigned long now) +{ + return a->holding && a->timed && a->generation == generation && + !time_before(now, a->deadline); +} + +/* Whether a hotplug event is owed that dcp_announce_notify() has not handed out. */ +static inline bool dcp_announce_owed(const struct dcp_announce *a) +{ + return a->withdrawn || (a->released && !a->holding); +} + +static inline enum dcp_announce_status +dcp_announce_update(struct dcp_announce *a, const struct dcp_announce_facts *f, + unsigned long now) +{ + if (!f->connected || !f->present) { + dcp_announce_end(a); + return DCP_ANNOUNCE_DISCONNECTED; + } + if (!a->announced && !a->holding) { + a->holding = true; + a->timed = false; + a->generation = f->generation; + } + if (a->holding && a->generation != f->generation) { + a->generation = f->generation; + a->timed = false; + } + if (a->holding && !f->ready && !f->failed && + !dcp_announce_expired(a, f->generation, now)) + return DCP_ANNOUNCE_HOLD; + if (a->holding) { + a->holding = false; + a->released = true; + } + a->announced = true; + return DCP_ANNOUNCE_CONNECTED; +} + +/* + * Whether a hotplug event is due, which hands out any event owed. A held + * connection changes nothing userspace can see, unless a reported one ended + * before it. + */ +static inline bool dcp_announce_notify(struct dcp_announce *a, + enum dcp_announce_status status) +{ + bool due = status != DCP_ANNOUNCE_HOLD || a->withdrawn; + + if (due) { + a->withdrawn = false; + a->released = false; + } + return due; +} + +#endif /* __APPLE_DCP_ANNOUNCE_H__ */ From 12c1046b9e313d0620570dfd427fbae75085b072 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:31:24 +1100 Subject: [PATCH 1094/1579] drm/apple: Report an external display connected once its modes are in A physical disconnect of an external H17P display first marks the cable absent, advances its generation and retires the AV service the EDID is read from, and only then stops the link and clears the connector's connected bit. A hotplug event in that window has userspace probe a connector that is still connected but has no EDID. GNOME then rejects the stored configuration of an "unknown unknown" monitor. The reverse happens on plug: DCP can report the link before the EDID service, the catalogue of the new connection or the link configuration the 180 Hz mode needs, and userspace picks a fallback mode from what it finds. Wire up the announcement state with dcp_external_modes_ready() as its readiness, for the external H17P pipeline only. detect() evaluates it and reports what it says; it never waits. The hotplug worker sends no event while a connection is held: the last missing fact queues it again, through the paths that already do that, or the timeout work does. A physical disconnect and a new cable end the connection. The timeout is a delayed work item on the freezable workqueue, which follows the Type-C work across bind, unbind and system sleep. It starts the timeout of a hold, so a hold cannot run out while the link cannot come back, and evaluates the hold itself under hpd_mutex, reconciling it with the current cable generation before any deadline counts. Queueing that work counts a deadline only for the generation it was set in, so a hold that crossed another sleep is reconciled as soon as the work is enabled again, not at the deadline of the last generation. Whoever ends a hold, the timeout work or a probe, queues the hotplug worker for the event it owes under that lock. The event of a new connection is therefore not lost to a probe that reports it first, and a timeout work that loses that race adds no second event. Teardown and system sleep stop the announcement under hpd_mutex before they drain the timeout work, and nothing queues the timeout work or the hotplug worker for a stopped announcement. The teardown that keeps the DMA resources of a live DCP then drains the hotplug worker as before, so neither is left queued for the quiesced pipeline, even when the worker was evaluating a hold while it ran. An event that comes due while the announcement is stopped is sent once its work is enabled again. On every other pipeline detect() still returns the connector's connected bit as before, and teardown is unchanged. The KUnit suite runs detect(), the timeout work, the hotplug worker, the sleep and teardown paths: a probe that releases a new connection before its timeout work runs, a probe that races a running timeout work, the real hotplug worker running during unbind and during shutdown, a probe after the DRM device outlived the teardown, and holds that cross another sleep, whose timeout work has to run as soon as it is enabled again. A hook, built only with the KUnit option, pauses the timeout work and the teardown at the points where they race. The apple-dcp lifecycle selftest copies the external disconnect and connect paths into its model of the driver, so give that model the announcement they now end. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/apple_drv.c | 4 +- drivers/gpu/drm/apple/connector.h | 1 + drivers/gpu/drm/apple/dcp-internal.h | 34 ++ drivers/gpu/drm/apple/dcp.c | 38 +- .../drm/apple/dcp_external_announce_test.c | 384 ++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 138 ++++++- .../drivers/gpu/apple-dcp/lifecycle.py | 3 + 8 files changed, 596 insertions(+), 7 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp_external_announce_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 99e79dfbcc0a4..abaa889bfaef5 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -15,6 +15,7 @@ appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_ready_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_announce_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 48167f356890c..472132637d711 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -122,8 +122,8 @@ apple_connector_detect(struct drm_connector *connector, bool force) { struct apple_connector *apple_connector = to_apple_connector(connector); - return apple_connector->connected ? connector_status_connected : - connector_status_disconnected; + return dcp_connector_connected(apple_connector) ? + connector_status_connected : connector_status_disconnected; } static void apple_connector_oob_hotplug(struct drm_connector *connector, diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index e04bc2e119a5e..38772d6731c2c 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -15,6 +15,7 @@ struct apple_connector; #include "dcp-internal.h" void dcp_hotplug(struct work_struct *work); +bool dcp_connector_connected(struct apple_connector *connector); void dcp_retrain_oob(struct apple_connector *connector); /* diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 03d47c0de6019..470089c23e2f4 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -18,6 +18,7 @@ #include "dptxep.h" #include "dcp_backlight.h" +#include "dcp-announce.h" #include "dcp-display-metadata.h" #include "dcp-metadata-epoch.h" #include "dcp-180hz.h" @@ -211,6 +212,11 @@ struct apple_dcp_hw_data { u32 firmware_request; }; +enum dcp_test_point { + DCP_TEST_ANNOUNCE_TIMER_DONE, /* past hpd_mutex in the timeout work */ + DCP_TEST_UNBIND_ANNOUNCE_STOPPED, /* unbind drained the timeout work */ +}; + /* TODO: move IOMFB members to its own struct */ struct apple_dcp { /* An adopted session has no confirmed firmware DMA stop boundary. */ @@ -447,6 +453,13 @@ struct apple_dcp { struct completion typec_iomfb_hpd_ready; struct delayed_work typec_reconnect_wq; u32 typec_reconnect_tries; + /* hpd_mutex: what DRM reports for the external display, and when */ + struct dcp_announce announce; + struct delayed_work announce_wq; +#if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) + /* Lets a test stop work where it races, see dcp_test_point(). */ + void (*test_point)(struct apple_dcp *dcp, enum dcp_test_point point); +#endif struct gpio_desc *hdmi_hpd; struct gpio_desc *hdmi_pwren; @@ -539,6 +552,27 @@ void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id); void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected); +void dcp_announce_arm(struct apple_dcp *dcp); +void dcp_announce_work(struct work_struct *work); + +/* A KUnit test can wait here for the work it races with. */ +static inline void dcp_test_point(struct apple_dcp *dcp, enum dcp_test_point point) +{ +#if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) + void (*test_point)(struct apple_dcp *dcp, enum dcp_test_point point); + + test_point = READ_ONCE(dcp->test_point); + if (test_point) + test_point(dcp, point); +#endif +} + +#if IS_ENABLED(CONFIG_KUNIT) +void dcp_comp_unbind(struct device *dev, struct device *main, void *data); +void dcp_platform_shutdown(struct platform_device *pdev); +void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable); +void dcp_enable_typec_work(struct apple_dcp *dcp); +#endif int dcp_backlight_init(struct apple_dcp *dcp, struct device *dev); void dcp_backlight_stop(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index feb2bfec2d969..24d7256698a50 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ +#include #include #include #include @@ -2817,6 +2818,7 @@ static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + dcp_announce_end(&dcp->announce); } /* HPD is fenced; retire the AV owner after releasing hpd_mutex. */ dpavservep_detach(dcp); @@ -3046,6 +3048,7 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) if (new_cable) { dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); + dcp_announce_end(&dcp->announce); } dcp->placeholder_retried = false; } @@ -4033,29 +4036,45 @@ static int dcp_connector_type_from_dt(struct device_node *np) return DRM_MODE_CONNECTOR_Unknown; } -static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) +VISIBLE_IF_KUNIT void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) { scoped_guard(mutex, &dcp->hpd_mutex) { if (release_cable) WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + /* Nothing arms the timeout work past its drain below. */ + dcp->announce.stopped = true; } if (release_cable && dcp_is_external_h17p(dcp)) dpavservep_detach(dcp); /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); disable_delayed_work_sync(&dcp->placeholder_edid_wq); + disable_delayed_work_sync(&dcp->announce_wq); cancel_work_sync(&dcp->external_stop_work); } -static void dcp_enable_typec_work(struct apple_dcp *dcp) +VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) { enable_delayed_work(&dcp->typec_reconnect_wq); enable_delayed_work(&dcp->placeholder_edid_wq); + enable_delayed_work(&dcp->announce_wq); /* A cable can be routed before the DRM component binds. */ if (READ_ONCE(dcp->typec_cable_connected)) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); + if (dcp_is_external_h17p(dcp)) { + struct apple_connector *con = READ_ONCE(dcp->connector); + + guard(mutex)(&dcp->hpd_mutex); + dcp->announce.stopped = false; + /* A hold that crossed system sleep waits for the new link from now. */ + dcp_announce_arm(dcp); + /* Send an event that came due while stopped. */ + if (dcp_announce_owed(&dcp->announce) && con && + READ_ONCE(con->dcp) == to_platform_device(dcp->dev)) + schedule_work(&con->hotplug_wq); + } } static bool dcp_external_is_idle(struct apple_dcp *dcp) @@ -4273,7 +4292,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) * We need to shutdown DCP before tearing down the display subsystem. Otherwise * the DCP will crash and briefly flash a green screen of death. */ -static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) +VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, void *data) { struct apple_dcp *dcp = dev_get_drvdata(dev); @@ -4303,6 +4322,14 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) cancel_work_sync(&dcp->bl_update_wq); } cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + /* + * Stop the announcement before draining the hotplug work: that + * work arms the timeout work, which in turn queues it. + */ + scoped_guard(mutex, &dcp->hpd_mutex) + dcp->announce.stopped = true; + disable_delayed_work_sync(&dcp->announce_wq); + dcp_test_point(dcp, DCP_TEST_UNBIND_ANNOUNCE_STOPPED); cancel_work_sync(&dcp->vblank_wq); if (dcp->connector) cancel_work_sync(&dcp->connector->hotplug_wq); @@ -4467,10 +4494,13 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp_typec_reconnect_work); INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, dcp_placeholder_edid_work); + INIT_DELAYED_WORK(&dcp->announce_wq, dcp_announce_work); INIT_DELAYED_WORK(&dcp->bl_fallback_wq, dcp_work_backlight_fallback); /* Balanced by enable at successful component bind. */ disable_delayed_work(&dcp->typec_reconnect_wq); disable_delayed_work(&dcp->placeholder_edid_wq); + disable_delayed_work(&dcp->announce_wq); + dcp->announce.stopped = true; platform_set_drvdata(pdev, dcp); @@ -4630,7 +4660,7 @@ static void dcp_platform_remove(struct platform_device *pdev) component_del(&pdev->dev, &dcp_comp_ops); } -static void dcp_platform_shutdown(struct platform_device *pdev) +VISIBLE_IF_KUNIT void dcp_platform_shutdown(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); diff --git a/drivers/gpu/drm/apple/dcp_external_announce_test.c b/drivers/gpu/drm/apple/dcp_external_announce_test.c new file mode 100644 index 0000000000000..58a0c1247acd8 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_external_announce_test.c @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "connector.h" +#include "dcp.h" +#include "dcp-internal.h" + +/* + * The announcement of an external H17P display, through the detect(), the + * hotplug worker, the timeout work and the teardown the driver runs. A + * stub stands in for the hotplug worker and counts the events it would + * send, except where the real one has to race with teardown. + */ +struct announce_test { + struct apple_dcp dcp; + struct platform_device *pdev; + struct apple_connector con; + struct drm_device drm; + atomic_t events; + + /* dcp_test_point() to pause at once, or -1 */ + int pause_at; + struct completion paused, resume; + + struct completion hotplug_entered, hotplug_release, hotplug_done; + struct task_struct *teardown; + struct completion torn_down; +}; + +static void announce_test_noop(struct work_struct *work) +{ +} + +static void announce_test_event(struct work_struct *work) +{ + struct announce_test *t = container_of(work, struct announce_test, con.hotplug_wq); + + atomic_inc(&t->events); +} + +/* The real hotplug worker, held back until the test lets it run. */ +static void announce_test_running_hotplug(struct work_struct *work) +{ + struct announce_test *t = container_of(work, struct announce_test, con.hotplug_wq); + + complete(&t->hotplug_entered); + wait_for_completion(&t->hotplug_release); + dcp_hotplug(work); + complete(&t->hotplug_done); +} + +static void announce_test_point(struct apple_dcp *dcp, enum dcp_test_point point) +{ + struct announce_test *t = container_of(dcp, struct announce_test, dcp); + + if (cmpxchg(&t->pause_at, point, -1) != point) + return; + complete(&t->paused); + wait_for_completion(&t->resume); +} + +static void announce_test_exit(void *data) +{ + struct announce_test *t = data; + + WRITE_ONCE(t->pause_at, -1); + complete_all(&t->resume); + complete_all(&t->hotplug_release); + if (t->teardown) + wait_for_completion_timeout(&t->torn_down, 10 * HZ); + disable_delayed_work_sync(&t->dcp.announce_wq); + cancel_work_sync(&t->con.hotplug_wq); +} + +/* + * A display on cable generation 2, connected, whose modes are not in: no + * catalogue has arrived for this connection. + */ +static struct announce_test *announce_test_init(struct kunit *test) +{ + struct announce_test *t; + struct apple_dcp *dcp; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + dcp = &t->dcp; + t->pdev = kunit_platform_device_alloc(test, "dcp-announce", PLATFORM_DEVID_AUTO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->pdev); + KUNIT_ASSERT_EQ(test, kunit_platform_device_add(test, t->pdev), 0); + platform_set_drvdata(t->pdev, dcp); + + dcp->dev = &t->pdev->dev; + dcp->hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL; + iomfb_queue_init(dcp); + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->dcpavserv.lock); + spin_lock_init(&dcp->metadata.lock); + INIT_DELAYED_WORK(&dcp->announce_wq, dcp_announce_work); + INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, announce_test_noop); + INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, announce_test_noop); + INIT_DELAYED_WORK(&dcp->bl_fallback_wq, announce_test_noop); + INIT_DELAYED_WORK(&dcp->backlight.fade_work, announce_test_noop); + INIT_WORK(&dcp->external_stop_work, announce_test_noop); + INIT_WORK(&dcp->vblank_wq, announce_test_noop); + INIT_WORK(&t->con.hotplug_wq, announce_test_event); + mutex_init(&t->drm.mode_config.mutex); + + t->pause_at = -1; + init_completion(&t->paused); + init_completion(&t->resume); + init_completion(&t->hotplug_entered); + init_completion(&t->hotplug_release); + init_completion(&t->hotplug_done); + init_completion(&t->torn_down); + dcp->test_point = announce_test_point; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, announce_test_exit, t), 0); + + t->con.base.dev = &t->drm; + t->con.dcp = t->pdev; + t->con.connected = true; + dcp->connector = &t->con; + dcp->typec_cable_connected = true; + dcp->typec_generation = 2; + dcp_firmware_metadata_connection(dcp, 2, false); + + return t; +} + +/* A hold of @generation whose timeout ends at @deadline. */ +static void announce_test_hold(struct announce_test *t, bool announced, u64 generation, + unsigned long deadline) +{ + struct dcp_announce *a = &t->dcp.announce; + + a->announced = announced; + a->holding = true; + a->timed = true; + a->generation = generation; + a->deadline = deadline; +} + +static bool announce_test_detect(struct announce_test *t) +{ + bool connected = dcp_connector_connected(&t->con); + + flush_work(&t->con.hotplug_wq); + return connected; +} + +/* + * A new connection whose timeout expired, probed before the timeout work + * runs: the probe reports it and stops the timer, so it has to queue the + * hotplug event the timer would have. Nothing else will. + */ +static void announce_cold_timeout_probe_test(struct kunit *test) +{ + struct announce_test *t = announce_test_init(test); + + announce_test_hold(t, false, 2, jiffies - 1); + mod_delayed_work(system_freezable_wq, &t->dcp.announce_wq, 60 * HZ); + + KUNIT_ASSERT_TRUE(test, announce_test_detect(t)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->dcp.announce_wq)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 1); +} + +/* + * A held event whose timeout expired, released by a probe while the + * timeout work is still running: the event goes out once. + */ +static void announce_timer_probe_once_test(struct kunit *test) +{ + struct announce_test *t = announce_test_init(test); + + announce_test_hold(t, true, 2, jiffies - 1); + t->pause_at = DCP_TEST_ANNOUNCE_TIMER_DONE; + mod_delayed_work(system_freezable_wq, &t->dcp.announce_wq, 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->paused, HZ), 0); + + KUNIT_EXPECT_TRUE(test, announce_test_detect(t)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 1); + KUNIT_EXPECT_FALSE(test, t->dcp.announce.holding); + + complete_all(&t->resume); + flush_delayed_work(&t->dcp.announce_wq); + flush_work(&t->con.hotplug_wq); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 1); +} + +/* + * The timeout work of a hold that outlived its cable generation, as when + * work is enabled after another sleep: it starts a full timeout for the + * current generation rather than wait out the deadline of the last one. + * Nothing is ready, so no event goes out yet. + */ +static void announce_timer_next_generation_test(struct kunit *test) +{ + struct announce_test *t = announce_test_init(test); + struct dcp_announce *a = &t->dcp.announce; + unsigned long old_deadline = jiffies + HZ, before = jiffies; + + announce_test_hold(t, true, 1, old_deadline); + dcp_announce_work(&t->dcp.announce_wq.work); + flush_work(&t->con.hotplug_wq); + + KUNIT_EXPECT_TRUE(test, a->holding); + KUNIT_EXPECT_EQ(test, a->generation, t->dcp.typec_generation); + KUNIT_EXPECT_TRUE(test, a->timed); + KUNIT_EXPECT_NE(test, a->deadline, old_deadline); + KUNIT_EXPECT_TRUE(test, time_after_eq(a->deadline, before + + msecs_to_jiffies(DCP_ANNOUNCE_RESUME_TIMEOUT_MS))); + KUNIT_EXPECT_TRUE(test, delayed_work_pending(&t->dcp.announce_wq)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 0); +} + +static int announce_test_unbind(void *data) +{ + struct announce_test *t = data; + + dcp_comp_unbind(t->dcp.dev, NULL, NULL); + complete(&t->torn_down); + return 0; +} + +static int announce_test_shutdown(void *data) +{ + struct announce_test *t = data; + + dcp_platform_shutdown(t->pdev); + complete(&t->torn_down); + return 0; +} + +/* + * The real hotplug worker evaluates a held connection after a teardown that + * retains the DMA resources stopped the timeout work: neither may be left + * queued once the teardown returns. + */ +static void announce_teardown_running_hotplug(struct kunit *test, + int (*teardown)(void *data)) +{ + struct announce_test *t = announce_test_init(test); + struct apple_dcp *dcp = &t->dcp; + + dcp->retain_dma = true; + announce_test_hold(t, true, 2, jiffies + 60 * HZ); + mod_delayed_work(system_freezable_wq, &dcp->announce_wq, 60 * HZ); + INIT_WORK(&t->con.hotplug_wq, announce_test_running_hotplug); + schedule_work(&t->con.hotplug_wq); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->hotplug_entered, HZ), 0); + + t->pause_at = DCP_TEST_UNBIND_ANNOUNCE_STOPPED; + t->teardown = kthread_run(teardown, t, "dcp-announce-teardown"); + KUNIT_ASSERT_FALSE(test, IS_ERR(t->teardown)); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->paused, HZ), 0); + complete(&t->hotplug_release); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->hotplug_done, HZ), 0); + complete_all(&t->resume); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->torn_down, HZ), 0); + + KUNIT_EXPECT_TRUE(test, dcp->quiescing); + KUNIT_EXPECT_TRUE(test, dcp->announce.holding); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&dcp->announce_wq)); + KUNIT_EXPECT_FALSE(test, work_pending(&t->con.hotplug_wq)); +} + +static void announce_unbind_running_hotplug_test(struct kunit *test) +{ + announce_teardown_running_hotplug(test, announce_test_unbind); +} + +static void announce_shutdown_running_hotplug_test(struct kunit *test) +{ + announce_teardown_running_hotplug(test, announce_test_shutdown); +} + +/* + * The DRM device outlives a teardown that retains the DMA resources, so + * userspace can still probe the connector. A probe that ends an expired + * hold queues no work for the stopped pipeline. + */ +static void announce_probe_after_unbind_test(struct kunit *test) +{ + struct announce_test *t = announce_test_init(test); + + t->dcp.retain_dma = true; + announce_test_hold(t, true, 2, jiffies - 1); + dcp_comp_unbind(t->dcp.dev, NULL, NULL); + + KUNIT_EXPECT_TRUE(test, announce_test_detect(t)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 0); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->dcp.announce_wq)); +} + +/* + * A new connection released by a probe while its work is disabled for + * sleep: its event goes out once the work is enabled again. + */ +static void announce_release_while_stopped_test(struct kunit *test) +{ + struct announce_test *t = announce_test_init(test); + + announce_test_hold(t, false, 2, jiffies + 60 * HZ); + dcp_disable_typec_work(&t->dcp, false); + t->dcp.external_error = -ETIMEDOUT; + + KUNIT_EXPECT_TRUE(test, announce_test_detect(t)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 0); + + dcp_enable_typec_work(&t->dcp); + flush_work(&t->con.hotplug_wq); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 1); +} + +/* + * A hold whose timeout had just started when the system slept again. The + * timeout work that enabling queues runs at once, not at the deadline of + * the last cable generation, and waits a full timeout for the new one, as + * for a hold that crossed a single sleep. Nothing flushes that work, as + * a flush runs delayed work at once. + */ +static void announce_test_sleep_again(struct kunit *test, bool announced, + unsigned int timeout_ms) +{ + struct announce_test *t = announce_test_init(test); + struct dcp_announce *a = &t->dcp.announce; + unsigned long timeout = msecs_to_jiffies(timeout_ms), before; + + announce_test_hold(t, announced, 2, jiffies + timeout); + mod_delayed_work(system_freezable_wq, &t->dcp.announce_wq, timeout); + dcp_disable_typec_work(&t->dcp, false); + t->pause_at = DCP_TEST_ANNOUNCE_TIMER_DONE; + before = jiffies; + dcp_enable_typec_work(&t->dcp); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->paused, timeout / 2), 0); + + KUNIT_EXPECT_TRUE(test, a->holding); + KUNIT_EXPECT_EQ(test, a->generation, 3); + KUNIT_EXPECT_TRUE(test, a->timed); + KUNIT_EXPECT_TRUE(test, time_after_eq(a->deadline, before + timeout)); + KUNIT_EXPECT_TRUE(test, time_before_eq(a->deadline, jiffies + timeout)); + KUNIT_EXPECT_TRUE(test, delayed_work_pending(&t->dcp.announce_wq)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 0); + complete_all(&t->resume); +} + +static void announce_resume_hold_sleep_again_test(struct kunit *test) +{ + announce_test_sleep_again(test, true, DCP_ANNOUNCE_RESUME_TIMEOUT_MS); +} + +static void announce_new_connection_sleep_again_test(struct kunit *test) +{ + announce_test_sleep_again(test, false, DCP_ANNOUNCE_TIMEOUT_MS); +} + +static struct kunit_case external_announce_cases[] = { + KUNIT_CASE(announce_cold_timeout_probe_test), + KUNIT_CASE(announce_timer_probe_once_test), + KUNIT_CASE(announce_timer_next_generation_test), + KUNIT_CASE(announce_unbind_running_hotplug_test), + KUNIT_CASE(announce_shutdown_running_hotplug_test), + KUNIT_CASE(announce_probe_after_unbind_test), + KUNIT_CASE(announce_release_while_stopped_test), + KUNIT_CASE(announce_resume_hold_sleep_again_test), + KUNIT_CASE(announce_new_connection_sleep_again_test), + {} +}; + +static struct kunit_suite external_announce_suite = { + .name = "apple-dcp-external-announce", + .test_cases = external_announce_cases, +}; + +kunit_test_suite(external_announce_suite); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 1c97639bb6b6c..a5de59da93f97 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -794,11 +794,131 @@ static void dcp_consume_edid_invalidation(struct apple_connector *connector) dcp_clear_connector_edid(connector); } +/* + * The external H17P pipeline learns the modes of each cable connection from + * firmware messages that can lag the link; see dcp-announce.h. + */ +static bool dcp_announce_gated(struct apple_dcp *dcp, + struct apple_connector *connector) +{ + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && + READ_ONCE(dcp->connector) == connector; +} + +/* + * Queue the timeout work of a held connection: at once for a hold whose + * timeout has not started for the current cable generation, as only that + * work starts it. + */ +void dcp_announce_arm(struct apple_dcp *dcp) +{ + struct dcp_announce *a = &dcp->announce; + unsigned long now = jiffies, delay = 0; + + lockdep_assert_held(&dcp->hpd_mutex); + if (a->stopped || !a->holding) + return; + if (a->timed && a->generation == dcp->typec_generation && + time_before(now, a->deadline)) + delay = a->deadline - now; + mod_delayed_work(system_freezable_wq, &dcp->announce_wq, delay); +} + +/* + * Evaluate what DRM is to report for the connection. @start starts the + * timeout of a held connection; only the timeout work does that. + */ +static enum dcp_announce_status +dcp_announce_eval(struct apple_dcp *dcp, struct apple_connector *connector, + bool start) +{ + struct dcp_announce *a = &dcp->announce; + struct dcp_announce_facts facts = { + .generation = dcp->typec_generation, + .connected = READ_ONCE(connector->connected), + .present = !READ_ONCE(dcp->external_physically_absent), + .failed = READ_ONCE(dcp->external_error) || READ_ONCE(dcp->crashed), + }; + enum dcp_announce_status status; + unsigned long now = jiffies; + bool held = a->holding, announced = a->announced; + + lockdep_assert_held(&dcp->hpd_mutex); + if (facts.connected && facts.present && !facts.failed) + facts.ready = dcp_external_modes_ready(dcp); + status = dcp_announce_update(a, &facts, now); + if (status == DCP_ANNOUNCE_HOLD) { + if (start) + dcp_announce_start(a, now); + dcp_announce_arm(dcp); + } else if (held) { + /* The hold ended: its timeout work has nothing left to do. */ + cancel_delayed_work(&dcp->announce_wq); + if (status == DCP_ANNOUNCE_CONNECTED && !facts.ready && !facts.failed) + dev_info(dcp->dev, "display modes not ready after %u ms\n", + announced ? DCP_ANNOUNCE_RESUME_TIMEOUT_MS : + DCP_ANNOUNCE_TIMEOUT_MS); + } + return status; +} + +/* + * Evaluate the connection outside the hotplug worker. An event that this + * makes owed is the worker's to send, so queue it, unless the announcement + * is stopped: dcp_enable_typec_work() queues it then. + */ +static void dcp_announce_poll(struct apple_dcp *dcp, struct apple_connector *connector, + bool start) +{ + bool owed = dcp_announce_owed(&dcp->announce); + + dcp_announce_eval(dcp, connector, start); + if (!owed && dcp_announce_owed(&dcp->announce) && !dcp->announce.stopped) + schedule_work(&connector->hotplug_wq); +} + +/* + * The timeout of a held connection. It runs on the freezable workqueue, as + * the link work does, so the timeout of a hold starts here, once the link + * can come back for the hold's generation. Everything it does is under + * hpd_mutex: a hold that ended meanwhile, or an announcement stopped for + * teardown or sleep, gets nothing more from here. + */ +void dcp_announce_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + announce_wq); + struct apple_connector *con = READ_ONCE(dcp->connector); + + scoped_guard(mutex, &dcp->hpd_mutex) { + if (dcp->announce.holding && !dcp->announce.stopped && con && + dcp_announce_gated(dcp, con) && + READ_ONCE(con->dcp) == to_platform_device(dcp->dev)) + dcp_announce_poll(dcp, con, true); + } + dcp_test_point(dcp, DCP_TEST_ANNOUNCE_TIMER_DONE); +} + +/* The connection state DRM reports for @connector. */ +bool dcp_connector_connected(struct apple_connector *connector) +{ + struct platform_device *pdev = READ_ONCE(connector->dcp); + struct apple_dcp *dcp = pdev ? platform_get_drvdata(pdev) : NULL; + + if (!dcp || !dcp_announce_gated(dcp, connector)) + return READ_ONCE(connector->connected); + + guard(mutex)(&dcp->hpd_mutex); + dcp_announce_poll(dcp, connector, false); + return dcp->announce.announced; +} + void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; struct platform_device *pdev; struct apple_dcp *dcp; + bool connected; int ret; connector = container_of(work, struct apple_connector, hotplug_wq); @@ -827,12 +947,28 @@ void dcp_hotplug(struct work_struct *work) dcp_clear_connector_edid(connector); } + if (dcp_announce_gated(dcp, connector)) { + enum dcp_announce_status status; + bool notify; + + scoped_guard(mutex, &dcp->hpd_mutex) { + status = dcp_announce_eval(dcp, connector, false); + notify = dcp_announce_notify(&dcp->announce, status); + connected = dcp->announce.announced; + } + /* The last missing fact or the timeout queues this work again. */ + if (!notify) + return; + } else { + connected = READ_ONCE(connector->connected); + } + /* * DCP defers link training until we set a display mode. But we set * display modes from atomic_flush, so userspace needs to trigger a * flush, or the CRTC gets no signal. */ - if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connector->connected && + if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connected && !(dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x())) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 030c0c34ff575..b580b1a978e3a 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -92,6 +92,7 @@ def function(file, name): static void mod_delayed_work(int q, struct delayed_work *w, int t) { (void)q; (void)t; w->work.pending = true; } struct apple_connector { bool connected, edid_invalidation_pending; }; +struct dcp_announce { int ends; }; struct apple_dcp { int external_lock, hpd_mutex; struct { int lock; bool presentation_paused; } iomfb; @@ -107,6 +108,7 @@ def function(file, name): u64 external_generation, external_rx_generation, external_crtc_generation; u64 external_stop_generation, typec_generation; int external_error, mode_state; void *avep; + struct dcp_announce announce; }; struct platform_device { struct apple_dcp *data; }; static struct apple_dcp *platform_get_drvdata(struct platform_device *p) { return p->data; } @@ -140,6 +142,7 @@ def function(file, name): static void dcp_firmware_metadata_connection(struct apple_dcp *d, u64 g, bool c) { (void)d; (void)g; (void)c; } static void dcp_firmware_metadata_announced(struct apple_dcp *d) { (void)d; } +static void dcp_announce_end(struct dcp_announce *a) { a->ends++; } static int dcp_external_disconnect(struct apple_dcp *, bool); static struct apple_dcp display; static void drm_connector_oob_hotplug_event(void *node, int status) From 921e1d336c99e7a928f59aa0b5c7b6f4eca419d5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:31:25 +1100 Subject: [PATCH 1095/1579] drm/apple: Hold the resume hotplug of an external display until it is ready System sleep releases the link of an external H17P display, and with it the proof of a mode such as 1080p at 180 Hz on a two-lane HBR3 link. A CRTC at that mode is resumed disabled with its link marked bad, and a hotplug event has userspace set its mode again. That event goes out from the resume handler, before the link work has even run. Userspace probes at once, finds the 180 Hz mode missing and sets a 60 Hz mode, then sets 180 Hz again once the firmware has announced the display and the mode is admitted, about 0.7 s after thaw. The monitor has to lock onto two links in a row. Have the DCP of such a CRTC send that event itself, once the display's modes are in for the new cable generation, including every check of the 180 Hz admission except the EDID the probe reads. If they are not in 2 s after the freezable link work can run, as with a monitor switched off during sleep, it sends the event anyway. A probe in the meantime finds the display connected with the modes it has, and nothing waits under a lock. Unplugging the display, a new cable or a failed link releases the event at once. Other CRTCs still get their event from the resume handler. The hold arms the timeout work as any other does, so a DCP whose work is still stopped for sleep arms it when that work is enabled. The firmware announces the returning display while the pipe is still stopped, and a failed reconnect sets no fact the worker otherwise hears about, so both now queue a re-evaluation of a held display, unless the announcement is stopped. apple_drm_drop_stale_crtcs() returns the mask of the CRTCs it disabled so that the resume handler knows whose events to hand over. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 23 ++- drivers/gpu/drm/apple/apple_resume.c | 10 +- drivers/gpu/drm/apple/apple_resume.h | 2 +- drivers/gpu/drm/apple/apple_resume_test.c | 10 +- drivers/gpu/drm/apple/dcp.c | 56 ++++++ drivers/gpu/drm/apple/dcp.h | 1 + .../gpu/drm/apple/dcp_external_resume_test.c | 159 ++++++++++++++++++ 7 files changed, 248 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 472132637d711..2eda90e88d4d5 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -928,16 +928,35 @@ static int apple_platform_suspend(struct device *dev) return 0; } +/* + * The DCP of an external display sends the hotplug event for its CRTC + * itself, once the modes of the new link are in; see dcp_hold_resume_event(). + * Returns false if any of @crtcs needs the event now. + */ +static bool apple_drm_hold_resume_events(struct apple_drm_private *apple, u32 crtcs) +{ + struct drm_crtc *crtc; + bool held = true; + + drm_for_each_crtc(crtc, &apple->drm) { + if (crtcs & drm_crtc_mask(crtc)) + held &= dcp_hold_resume_event(to_apple_crtc(crtc)->dcp); + } + return held; +} + static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - bool dropped = false; + u32 dropped = 0; if (apple) { if (apple->drm.mode_config.suspend_state) dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); - drm_mode_config_helper_resume(&apple->drm); /* Have userspace set the mode of a CRTC resumed disabled again. */ + if (dropped && apple_drm_hold_resume_events(apple, dropped)) + dropped = 0; + drm_mode_config_helper_resume(&apple->drm); if (dropped) drm_kms_helper_hotplug_event(&apple->drm); } diff --git a/drivers/gpu/drm/apple/apple_resume.c b/drivers/gpu/drm/apple/apple_resume.c index 52dabd11d3e51..7d2197e0ca8af 100644 --- a/drivers/gpu/drm/apple/apple_resume.c +++ b/drivers/gpu/drm/apple/apple_resume.c @@ -27,9 +27,9 @@ * The suspend state holds every CRTC, plane and connector already, so it is * edited directly rather than through helpers that need the modeset locks. * - * Returns true if a CRTC is to be resumed disabled. + * Returns the mask of the CRTCs to be resumed disabled. */ -bool apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) +u32 apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) { const struct drm_crtc_helper_funcs *funcs; struct drm_connector_state *conn_state; @@ -38,7 +38,7 @@ bool apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) struct drm_connector *connector; struct drm_plane *plane; struct drm_crtc *crtc; - bool dropped = false; + u32 dropped = 0; int i, j; for_each_new_crtc_in_state(state, crtc, crtc_state, i) { @@ -64,7 +64,7 @@ bool apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) continue; /* Detaching from a CRTC already in the state cannot fail. */ if (WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) - return true; + return dropped | drm_crtc_mask(crtc); /* * Nor does a connector without a CRTC keep an encoder: * the resume check warns about one left routed. @@ -76,7 +76,7 @@ bool apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) conn_state->link_status = DRM_LINK_STATUS_BAD; } crtc_state->active = false; - dropped = true; + dropped |= drm_crtc_mask(crtc); if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) break; } diff --git a/drivers/gpu/drm/apple/apple_resume.h b/drivers/gpu/drm/apple/apple_resume.h index de0dffe2dc8bd..56db4bbb8470a 100644 --- a/drivers/gpu/drm/apple/apple_resume.h +++ b/drivers/gpu/drm/apple/apple_resume.h @@ -8,6 +8,6 @@ struct drm_atomic_state; -bool apple_drm_drop_stale_crtcs(struct drm_atomic_state *state); +u32 apple_drm_drop_stale_crtcs(struct drm_atomic_state *state); #endif /* __APPLE_RESUME_H__ */ diff --git a/drivers/gpu/drm/apple/apple_resume_test.c b/drivers/gpu/drm/apple/apple_resume_test.c index 7f89c07c15294..a4870081a8ac9 100644 --- a/drivers/gpu/drm/apple/apple_resume_test.c +++ b/drivers/gpu/drm/apple/apple_resume_test.c @@ -187,7 +187,7 @@ static void resume_stale_crtc_disabled_test(struct kunit *test) priv->admissible = false; state = priv->drm.mode_config.suspend_state; /* The caller tells userspace about the CRTC it had and lost. */ - KUNIT_EXPECT_TRUE(test, apple_drm_drop_stale_crtcs(state)); + KUNIT_EXPECT_EQ(test, apple_drm_drop_stale_crtcs(state), drm_crtc_mask(priv->crtc)); /* The resume check warns about an encoder routed without a CRTC. */ conn_state = drm_atomic_get_new_connector_state(state, &priv->connector); @@ -218,8 +218,8 @@ static void resume_admissible_crtc_restored_test(struct kunit *test) struct resume_test *priv = resume_test_init(test); KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); - KUNIT_EXPECT_FALSE(test, - apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state)); + KUNIT_EXPECT_EQ(test, apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state), + 0U); KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); KUNIT_EXPECT_TRUE(test, priv->crtc->state->enable); @@ -256,8 +256,8 @@ static void resume_stale_dpms_off_crtc_disabled_test(struct kunit *test) KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); priv->admissible = false; - KUNIT_EXPECT_TRUE(test, - apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state)); + KUNIT_EXPECT_EQ(test, apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state), + drm_crtc_mask(priv->crtc)); KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); KUNIT_EXPECT_NULL(test, priv->connector.state->crtc); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 24d7256698a50..8fd4f2140ac3d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1977,6 +1977,21 @@ static void dcp_firmware_metadata_present_locked(struct apple_dcp *dcp) dcp->metadata.epoch.connected = true; } +/* + * A fact a held connection waits for has changed: evaluate it again. The + * unlocked checks may miss a hold that has just started; such a hold ends at + * its timeout instead. A stopped announcement is evaluated again when its + * work is enabled; teardown quiesces the callers before draining the worker. + */ +static void dcp_announce_kick(struct apple_dcp *dcp) +{ + struct apple_connector *con = READ_ONCE(dcp->connector); + + if (READ_ONCE(dcp->announce.holding) && !READ_ONCE(dcp->announce.stopped) && con && + READ_ONCE(con->dcp) == to_platform_device(dcp->dev)) + schedule_work(&con->hotplug_wq); +} + /* * While its pipe is stopped an external display reports no hotplug to DRM * (see dcp_external_hotplug()). When the firmware announces the display @@ -1997,6 +2012,7 @@ void dcp_firmware_metadata_announced(struct apple_dcp *dcp) spin_lock_irqsave(&dcp->metadata.lock, flags); dcp_firmware_metadata_present_locked(dcp); spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp_announce_kick(dcp); } void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) @@ -2984,6 +3000,9 @@ static void dcp_typec_reconnect_work(struct work_struct *work) dcp->typec_reconnect_tries = 0; return; } + /* A display held for this link will not get ready. */ + if (READ_ONCE(dcp->external_error)) + dcp_announce_kick(dcp); if (++dcp->typec_reconnect_tries < (dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x() ? @@ -2997,6 +3016,43 @@ static void dcp_typec_reconnect_work(struct work_struct *work) dcp->typec_reconnect_tries, ret); } +/** + * dcp_hold_resume_event - hold the resume hotplug event of an external display + * @pdev: the DCP of a CRTC resumed disabled + * + * System sleep releases the link of an external H17P display, and with it + * the proof of modes such as 1080p at 180 Hz, so apple_drm_drop_stale_crtcs() + * resumes the CRTC of such a mode disabled with its link marked bad. A + * hotplug event then has userspace set the mode again. Sent at once, it has + * userspace probe before the link is back, pick a fallback mode, and set its + * own mode again once that is offered: two mode sets the monitor has to + * resynchronize to. + * + * Hold that event, not the connection, until the display's modes are in for + * the new cable generation, or for DCP_ANNOUNCE_RESUME_TIMEOUT_MS after the + * link work can run again, then send it once from the hotplug worker. No + * lock is held while waiting; a probe in the meantime reports the display + * connected with the modes it has. + * + * Returns false if the caller must send the event itself. + */ +bool dcp_hold_resume_event(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct apple_connector *con = READ_ONCE(dcp->connector); + + if (!dcp_is_external_h17p(dcp) || !con || READ_ONCE(con->dcp) != pdev) + return false; + + guard(mutex)(&dcp->hpd_mutex); + if (!READ_ONCE(con->connected) || !dcp->typec_cable_connected || + !dcp_announce_hold(&dcp->announce, dcp->typec_generation)) + return false; + /* Freezable, like the link work: the timeout starts after thaw. */ + dcp_announce_arm(dcp); + return true; +} + static void disconnected_hpd_event(struct apple_connector *con) { if (!con) diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 9678701b8613f..9a778d1a01301 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -84,6 +84,7 @@ void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); +bool dcp_hold_resume_event(struct platform_device *pdev); struct apple_dcp; struct phy; diff --git a/drivers/gpu/drm/apple/dcp_external_resume_test.c b/drivers/gpu/drm/apple/dcp_external_resume_test.c index 409a8cd8ffd8b..cf3db9324d3ec 100644 --- a/drivers/gpu/drm/apple/dcp_external_resume_test.c +++ b/drivers/gpu/drm/apple/dcp_external_resume_test.c @@ -2,9 +2,13 @@ /* Copyright 2026 Ryan Murray */ #include +#include #include #include +#include +#include "connector.h" +#include "dcp.h" #include "dcp-internal.h" /* @@ -125,11 +129,166 @@ static void external_paused_released_link_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp->metadata.epoch.connected); } +/* + * The same display as apple_platform_resume() finds it: reported before the + * sleep, its CRTC just resumed disabled, the link not back yet. + */ +struct resume_hold_test { + struct platform_device *pdev; + struct apple_dcp *dcp; + struct apple_connector *con; + struct apple_connector connector; + atomic_t events; + atomic_t timer_runs; +}; + +static void resume_hold_event(struct work_struct *work) +{ + struct resume_hold_test *t = container_of(work, struct resume_hold_test, + connector.hotplug_wq); + + atomic_inc(&t->events); +} + +static void resume_hold_timer(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + announce_wq); + struct resume_hold_test *t = container_of(dcp->connector, struct resume_hold_test, + connector); + + atomic_inc(&t->timer_runs); +} + +static void resume_hold_exit(void *data) +{ + struct resume_hold_test *t = data; + + cancel_delayed_work_sync(&t->dcp->announce_wq); + cancel_work_sync(&t->con->hotplug_wq); +} + +static struct resume_hold_test *resume_hold_init(struct kunit *test) +{ + struct resume_hold_test *t; + struct apple_dcp *dcp; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dcp); + t->dcp = dcp; + t->con = &t->connector; + t->pdev = kunit_platform_device_alloc(test, "dcp-resume-hold", PLATFORM_DEVID_AUTO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->pdev); + KUNIT_ASSERT_EQ(test, kunit_platform_device_add(test, t->pdev), 0); + platform_set_drvdata(t->pdev, dcp); + + dcp->dev = &t->pdev->dev; + dcp->hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL; + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->dcpavserv.lock); + spin_lock_init(&dcp->metadata.lock); + INIT_DELAYED_WORK(&dcp->announce_wq, resume_hold_timer); + INIT_WORK(&t->con->hotplug_wq, resume_hold_event); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, resume_hold_exit, t), 0); + + t->con->dcp = t->pdev; + t->con->connected = true; + dcp->connector = t->con; + dcp->announce.announced = true; + dcp->typec_cable_connected = true; + dcp->typec_generation = 2; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + + return t; +} + +static bool resume_hold_detect(struct resume_hold_test *t) +{ + bool connected = dcp_connector_connected(t->con); + + flush_work(&t->con->hotplug_wq); + return connected; +} + +/* The event is held, never the connection: a probe still finds it. */ +static void resume_hold_armed_test(struct kunit *test) +{ + struct resume_hold_test *t = resume_hold_init(test); + + KUNIT_EXPECT_TRUE(test, dcp_hold_resume_event(t->pdev)); + KUNIT_EXPECT_TRUE(test, t->dcp->announce.holding); + KUNIT_EXPECT_EQ(test, t->dcp->announce.generation, 2); + /* + * The timer work starts the timeout once it can run, so it is queued + * at once. It may have run already: count its runs after a flush. + */ + KUNIT_EXPECT_FALSE(test, t->dcp->announce.timed); + flush_delayed_work(&t->dcp->announce_wq); + KUNIT_EXPECT_EQ(test, atomic_read(&t->timer_runs), 1); + + KUNIT_EXPECT_TRUE(test, resume_hold_detect(t)); + KUNIT_EXPECT_TRUE(test, t->dcp->announce.holding); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 0); +} + +/* A connection never reported has no resume event to hold. */ +static void resume_hold_unreported_test(struct kunit *test) +{ + struct resume_hold_test *t = resume_hold_init(test); + + t->dcp->announce.announced = false; + KUNIT_EXPECT_FALSE(test, dcp_hold_resume_event(t->pdev)); + KUNIT_EXPECT_FALSE(test, t->dcp->announce.holding); +} + +/* Other pipelines keep sending the event at once. */ +static void resume_hold_internal_test(struct kunit *test) +{ + struct resume_hold_test *t = resume_hold_init(test); + + t->dcp->hw.iomfb_method_profile = DCP_IOMFB_METHODS_DEFAULT; + KUNIT_EXPECT_FALSE(test, dcp_hold_resume_event(t->pdev)); + KUNIT_EXPECT_FALSE(test, t->dcp->announce.holding); +} + +/* A cable unplugged during the sleep is reported at once, with one event. */ +static void resume_hold_unplug_test(struct kunit *test) +{ + struct resume_hold_test *t = resume_hold_init(test); + + KUNIT_ASSERT_TRUE(test, dcp_hold_resume_event(t->pdev)); + t->dcp->external_physically_absent = true; + + KUNIT_EXPECT_FALSE(test, resume_hold_detect(t)); + KUNIT_EXPECT_FALSE(test, t->dcp->announce.holding); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 1); +} + +/* A link that failed will not get ready: the event is not held for it. */ +static void resume_hold_failed_link_test(struct kunit *test) +{ + struct resume_hold_test *t = resume_hold_init(test); + + KUNIT_ASSERT_TRUE(test, dcp_hold_resume_event(t->pdev)); + t->dcp->external_error = -ETIMEDOUT; + + KUNIT_EXPECT_TRUE(test, resume_hold_detect(t)); + KUNIT_EXPECT_FALSE(test, t->dcp->announce.holding); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), 1); +} + static struct kunit_case external_resume_cases[] = { KUNIT_CASE(external_paused_announce_recorded_test), KUNIT_CASE(external_paused_unplug_ignored_test), KUNIT_CASE(external_paused_stale_announce_test), KUNIT_CASE(external_paused_released_link_test), + KUNIT_CASE(resume_hold_armed_test), + KUNIT_CASE(resume_hold_unreported_test), + KUNIT_CASE(resume_hold_internal_test), + KUNIT_CASE(resume_hold_unplug_test), + KUNIT_CASE(resume_hold_failed_link_test), {} }; From 1e54dce092ecae6d341040d8e79f376ff0bde6b8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 15:13:55 +1100 Subject: [PATCH 1096/1579] drm/apple: Let an external display announcement lose its connection An external display can come back on a new link while userspace still scans out to it through an active CRTC. A Type-C route bounce, for instance, tears the DisplayPort link down and brings it back within a second or two, quicker than a compositor reconfigures. If the driver then cannot set the mode of that CRTC again, the display stays dark while DRM still reports it connected, and a compositor that finds the display it had on its next probe has no reason to set a mode. Teach the announcement state machine a lost connection: one whose link could not be set up again on its cable generation. It is reported gone, which owes a hotplug event, until userspace releases the CRTC (dcp_announce_retry()) or the cable generation changes. It is then a new connection, held until its modes are in like any other. The loss is reported with one event. A connection that stays gone sends no more, whatever facts arrive meanwhile: each repeated event would only restart a compositor's wait for the output to stay disconnected (Mutter debounces a disconnect for three seconds and restarts that wait on every event). Nothing loses a connection yet. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-announce-test.c | 89 +++++++++++++++++++++++ drivers/gpu/drm/apple/dcp-announce.h | 57 ++++++++++++++- 2 files changed, 142 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-announce-test.c b/drivers/gpu/drm/apple/dcp-announce-test.c index bdfd9ef4db774..ec5d7dd250cc3 100644 --- a/drivers/gpu/drm/apple/dcp-announce-test.c +++ b/drivers/gpu/drm/apple/dcp-announce-test.c @@ -367,6 +367,91 @@ static void hold_again_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); } +/* A reported connection whose link was not restored is withdrawn once. */ +static void lost_withdraws_test(struct kunit *test) +{ + struct dcp_announce_facts lost = link_ready; + struct dcp_announce a = { }; + + KUNIT_ASSERT_EQ(test, dcp_announce_update(&a, &link_ready, 1), + DCP_ANNOUNCE_CONNECTED); + KUNIT_ASSERT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); + + dcp_announce_lose(&a, 1); + KUNIT_EXPECT_TRUE(test, dcp_announce_lost(&a, 1)); + lost.lost = true; + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &lost, 2), + DCP_ANNOUNCE_DISCONNECTED); + KUNIT_EXPECT_FALSE(test, a.announced); + KUNIT_EXPECT_TRUE(test, dcp_announce_owed(&a)); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_DISCONNECTED)); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); + + /* Modes still in for the lost generation do not bring it back. */ + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &lost, 3), + DCP_ANNOUNCE_DISCONNECTED); + KUNIT_EXPECT_FALSE(test, dcp_announce_owed(&a)); + /* Nor does an unchanged loss send its event again. */ + KUNIT_EXPECT_FALSE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_DISCONNECTED)); +} + +/* A loss belongs to its cable generation: the next cable is not lost. */ +static void lost_generation_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + dcp_announce_lose(&a, 1); + KUNIT_EXPECT_TRUE(test, dcp_announce_lost(&a, 1)); + KUNIT_EXPECT_FALSE(test, dcp_announce_lost(&a, 2)); + + /* Releasing a CRTC on the next cable retries nothing, and forgets the loss. */ + KUNIT_EXPECT_FALSE(test, dcp_announce_retry(&a, 2)); + KUNIT_EXPECT_FALSE(test, dcp_announce_lost(&a, 1)); +} + +/* A lost connection whose CRTC was released is a new connection. */ +static void lost_retry_test(struct kunit *test) +{ + struct dcp_announce_facts lost = link_only; + struct dcp_announce a = { }; + + KUNIT_ASSERT_EQ(test, dcp_announce_update(&a, &link_ready, 1), + DCP_ANNOUNCE_CONNECTED); + dcp_announce_lose(&a, 1); + lost.lost = true; + KUNIT_ASSERT_EQ(test, dcp_announce_update(&a, &lost, 2), + DCP_ANNOUNCE_DISCONNECTED); + KUNIT_ASSERT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_DISCONNECTED)); + + KUNIT_EXPECT_TRUE(test, dcp_announce_retry(&a, 1)); + KUNIT_EXPECT_FALSE(test, dcp_announce_lost(&a, 1)); + KUNIT_EXPECT_FALSE(test, dcp_announce_retry(&a, 1)); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_only, 3), + DCP_ANNOUNCE_HOLD); + KUNIT_EXPECT_FALSE(test, a.announced); + KUNIT_EXPECT_EQ(test, dcp_announce_update(&a, &link_ready, 4), + DCP_ANNOUNCE_CONNECTED); + KUNIT_EXPECT_TRUE(test, dcp_announce_owed(&a)); + KUNIT_EXPECT_TRUE(test, dcp_announce_notify(&a, DCP_ANNOUNCE_CONNECTED)); +} + +/* A connection lost again after its retry waits for the next cable. */ +static void lost_retry_once_test(struct kunit *test) +{ + struct dcp_announce a = { }; + + dcp_announce_lose(&a, 1); + KUNIT_EXPECT_TRUE(test, dcp_announce_retry(&a, 1)); + dcp_announce_lose(&a, 1); + KUNIT_EXPECT_FALSE(test, dcp_announce_retry(&a, 1)); + KUNIT_EXPECT_TRUE(test, dcp_announce_lost(&a, 1)); + + /* The next cable starts afresh. */ + dcp_announce_lose(&a, 2); + KUNIT_EXPECT_TRUE(test, dcp_announce_retry(&a, 2)); + KUNIT_EXPECT_FALSE(test, dcp_announce_lost(&a, 2)); +} + static struct kunit_case announce_cases[] = { KUNIT_CASE(announce_ready_test), KUNIT_CASE(announce_waits_for_ready_test), @@ -390,6 +475,10 @@ static struct kunit_case announce_cases[] = { KUNIT_CASE(hold_deadline_generation_test), KUNIT_CASE(hold_release_once_test), KUNIT_CASE(hold_again_test), + KUNIT_CASE(lost_withdraws_test), + KUNIT_CASE(lost_generation_test), + KUNIT_CASE(lost_retry_test), + KUNIT_CASE(lost_retry_once_test), {} }; diff --git a/drivers/gpu/drm/apple/dcp-announce.h b/drivers/gpu/drm/apple/dcp-announce.h index cb360e48eb316..8a954b016af52 100644 --- a/drivers/gpu/drm/apple/dcp-announce.h +++ b/drivers/gpu/drm/apple/dcp-announce.h @@ -21,6 +21,12 @@ * connection itself, until it is ready again, or until * DCP_ANNOUNCE_RESUME_TIMEOUT_MS have passed without that. That is the * event that has userspace set the mode of a CRTC resumed disabled. + * - A reported connection whose link came back under an active CRTC, but + * could not be set up again, is lost: it is reported gone until userspace + * releases that CRTC or the cable generation changes, and is then a new + * connection. Releasing the CRTC brings it back once per cable + * generation. Userspace may keep driving an output it already uses + * whatever its link status says, but not a display that went away. * * The timeout of a hold runs from dcp_announce_start(), and only for the * cable generation it was started in. A hold that outlives its generation, @@ -46,7 +52,11 @@ struct dcp_announce { bool holding; /* waiting for the connection to be ready */ bool timed; /* the hold of @generation ends at @deadline */ bool stopped; /* nothing may be queued for the announcement */ + bool lost; /* the connection of @lost_generation is lost */ + bool retried; /* a loss of @retried_generation was retried */ u64 generation; /* the cable generation of the hold */ + u64 lost_generation; + u64 retried_generation; unsigned long deadline; }; @@ -56,6 +66,7 @@ struct dcp_announce_facts { bool present; /* the cable is not known to be gone */ bool ready; /* a probe would list the modes of this connection */ bool failed; /* the link failed; it will not get ready */ + bool lost; /* see dcp_announce_lost() */ }; /* The connection ended, whether DRM was told about it or not. */ @@ -97,6 +108,43 @@ static inline bool dcp_announce_expired(const struct dcp_announce *a, u64 genera !time_before(now, a->deadline); } +/* + * The link of the active CRTC of the connection on cable generation + * @generation could not be set up again. + */ +static inline void dcp_announce_lose(struct dcp_announce *a, u64 generation) +{ + a->lost = true; + a->lost_generation = generation; +} + +static inline bool dcp_announce_lost(const struct dcp_announce *a, u64 generation) +{ + return a->lost && a->lost_generation == generation; +} + +/* + * Userspace released the CRTC of the connection on cable generation + * @generation. A lost connection is a new connection from now on, once per + * cable generation: one lost again after that stays lost until the cable + * generation changes, so that a link that cannot come up does not have + * userspace switch between the display and none for as long as it is + * attached. Returns whether the connection is to be announced again. + */ +static inline bool dcp_announce_retry(struct dcp_announce *a, u64 generation) +{ + if (!dcp_announce_lost(a, generation)) { + a->lost = false; + return false; + } + if (a->retried && a->retried_generation == generation) + return false; + a->lost = false; + a->retried = true; + a->retried_generation = generation; + return true; +} + /* Whether a hotplug event is owed that dcp_announce_notify() has not handed out. */ static inline bool dcp_announce_owed(const struct dcp_announce *a) { @@ -107,7 +155,7 @@ static inline enum dcp_announce_status dcp_announce_update(struct dcp_announce *a, const struct dcp_announce_facts *f, unsigned long now) { - if (!f->connected || !f->present) { + if (!f->connected || !f->present || f->lost) { dcp_announce_end(a); return DCP_ANNOUNCE_DISCONNECTED; } @@ -133,13 +181,14 @@ dcp_announce_update(struct dcp_announce *a, const struct dcp_announce_facts *f, /* * Whether a hotplug event is due, which hands out any event owed. A held - * connection changes nothing userspace can see, unless a reported one ended - * before it. + * connection, or one that stays gone, changes nothing userspace can see, + * unless a reported one ended before it. Repeating the event of a lost + * connection would only restart a compositor's wait for it to stay gone. */ static inline bool dcp_announce_notify(struct dcp_announce *a, enum dcp_announce_status status) { - bool due = status != DCP_ANNOUNCE_HOLD || a->withdrawn; + bool due = status == DCP_ANNOUNCE_CONNECTED || a->withdrawn; if (due) { a->withdrawn = false; From 10bab06f30a5272c7ab64653869ef1b3af3bc5f2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 15:13:55 +1100 Subject: [PATCH 1097/1579] drm/apple: Set the mode of an external display again when its link returns When the DisplayPort route of an external H17P display bounces under an active CRTC, the hotplug worker finds the display back without a valid DCP mode and retrains the CRTC with drm_atomic_helper_reset_crtc(). For the 180 Hz exception mode that commit fails its mode fixup: the mode is admitted only against the EDID a probe read for the current cable generation, and nothing has probed the new link yet. The retrain fails with -EINVAL, the link status is left bad and nothing tries again. A compositor that keeps driving an output whatever its link status says, and that finds the same display and modes on its next probe, then keeps the CRTC on a dead link and the monitor stays dark. On the external H17P pipeline, make that recovery end either in a picture or in a disconnect: - If userspace keeps a CRTC active on the connector while its link is down, probe the connector first, as userspace would, so the mode admission sees the EDID of the current link. Then set the mode of the CRTC again, with its link status good. - If that fails, or leaves the link down, mark the link bad and lose the connection. The display is reported disconnected, with a hotplug event, until userspace switches the CRTC off (dcp_poweroff() calls dcp_external_crtc_released()) or the cable generation changes. It is then announced again as a new connection. Both are decided under the modeset locks of the mode set, so a CRTC that userspace switches off meanwhile is either off before the attempt, and left alone, or found lost when it goes off. A route that bounces again during the attempt leaves the link to the newer connection. A mode set of the external display that fails, including one refused because its pipe did not start, now also queues the hotplug worker: atomic_enable cannot report the failure, and the CRTC would otherwise stay active without a link. A pipe that did not start also takes its old mode with it, so that the worker does not take the previous mode for a link; a pipe whose presentation is only paused keeps its mode. A lost connection comes back once per cable generation when its CRTC is switched off, so a link that cannot come up does not have userspace switch the display on and off while it stays attached. The recovery never writes external_error and does not wait for it to clear: a latched failure only makes the mode set fail like any other, and the physical unplug that releases the latch also starts a new cable generation. The worker gets here only when it sends its event, so after the hold of a new connection has ended, and the 180 Hz admission rules are unchanged. Other DCP pipelines, the M1 and M2 ones included, keep the existing retraining. A KUnit suite runs the real hotplug worker, detect() and timeout work against a DRM KUnit device whose CRTC admits its mode the way the exception mode is admitted. It covers a bounce under an active CRTC, the mode gone on the new link, a mode set that leaves the link down, a bounce after a loss, a bounce during the probe, an inactive CRTC, a connector without state yet, userspace switching the CRTC off during the mode set, a link that never comes up, and a DCP of another kind. A new test point counts the hotplug events. Four cases run the real dcp_crtc_atomic_modeset() and H17P mode set path, with a test hook answering IOMFB method calls in place of the firmware: the bounce, a pipe that was stopped while its loss was buffered and then fails to activate, a paused presentation, and unchanged facts after a reported loss, which must not send more events. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-internal.h | 6 + drivers/gpu/drm/apple/dcp.c | 1 + .../gpu/drm/apple/dcp_external_relink_test.c | 849 ++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 201 ++++- 5 files changed, 1053 insertions(+), 5 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp_external_relink_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index abaa889bfaef5..03b08252b99a7 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -16,6 +16,7 @@ appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_ready_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_announce_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_relink_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 470089c23e2f4..697d6f9e3c03e 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -34,6 +34,7 @@ struct apple_dcp; struct apple_dcp_afkep; +struct dcp_method_entry; /* Snapshot of a completed present, independent of the latest DRM state. */ struct iomfb_scanout_h17p { @@ -215,6 +216,7 @@ struct apple_dcp_hw_data { enum dcp_test_point { DCP_TEST_ANNOUNCE_TIMER_DONE, /* past hpd_mutex in the timeout work */ DCP_TEST_UNBIND_ANNOUNCE_STOPPED, /* unbind drained the timeout work */ + DCP_TEST_HOTPLUG_EVENT, /* the hotplug worker sends its event */ }; /* TODO: move IOMFB members to its own struct */ @@ -459,6 +461,9 @@ struct apple_dcp { #if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) /* Lets a test stop work where it races, see dcp_test_point(). */ void (*test_point)(struct apple_dcp *dcp, enum dcp_test_point point); + /* Lets a test answer IOMFB method calls in place of the firmware. */ + void (*test_push)(struct apple_dcp *dcp, const struct dcp_method_entry *call, + dcp_callback_t cb, void *cookie); #endif struct gpio_desc *hdmi_hpd; @@ -554,6 +559,7 @@ void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected); void dcp_announce_arm(struct apple_dcp *dcp); void dcp_announce_work(struct work_struct *work); +void dcp_external_crtc_released(struct apple_dcp *dcp); /* A KUnit test can wait here for the work it races with. */ static inline void dcp_test_point(struct apple_dcp *dcp, enum dcp_test_point point) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8fd4f2140ac3d..13f540e914fa6 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3550,6 +3550,7 @@ void dcp_poweroff(struct platform_device *pdev) ret = dcp_external_stop_locked(dcp, dcp->external_crtc_generation); if (ret) dev_err(dcp->dev, "external display stop failed: %d\n", ret); + dcp_external_crtc_released(dcp); return; } diff --git a/drivers/gpu/drm/apple/dcp_external_relink_test.c b/drivers/gpu/drm/apple/dcp_external_relink_test.c new file mode 100644 index 0000000000000..15c22218127f7 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_external_relink_test.c @@ -0,0 +1,849 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "connector.h" +#include "dcp.h" +#include "dcp-internal.h" +#include "iomfb.h" +#include "parser.h" + +/* + * An external H17P display whose DisplayPort route goes away and comes back + * while userspace keeps its CRTC active, through the real hotplug worker, + * detect(), timeout work and atomic commits. The CRTC stands in for the DCP + * pipe: as for the 180 Hz exception mode, its mode is admitted only once a + * probe has read the EDID of the current cable generation, and setting it + * brings the link up unless the test says otherwise. + */ +struct relink_test { + struct drm_device drm; + struct apple_dcp dcp; + struct apple_connector con; + struct platform_device *pdev; + struct drm_plane *plane; + struct drm_crtc *crtc; + struct drm_encoder encoder; + struct drm_framebuffer fb; + struct drm_display_mode *mode; + + u64 probed_generation; /* the cable generation of the last probe */ + bool offered; /* the display offers the mode */ + bool trains; /* setting the mode brings the link up */ + bool bounce_on_probe; /* the route bounces again during a probe */ + bool real_modeset; /* atomic_enable is the real dcp_crtc_atomic_modeset() */ + bool force_modeset; /* a userspace commit changes the connectors */ + u32 reply; /* what the firmware answers a call with */ + int modeset_ret; /* the result of the real mode set */ + struct dcp_display_mode catalog; + bool hold_modeset; /* a mode set waits for @modeset_release */ + struct completion modeset_entered, modeset_release, userspace_done; + int userspace_ret; + atomic_t probes, checks, modesets, events; +}; + +static struct relink_test *to_relink_test(struct drm_device *drm) +{ + return container_of(drm, struct relink_test, drm); +} + +static void relink_test_noop(struct work_struct *work) +{ +} + +static void relink_test_point(struct apple_dcp *dcp, enum dcp_test_point point) +{ + struct relink_test *t = container_of(dcp, struct relink_test, dcp); + + if (point == DCP_TEST_HOTPLUG_EVENT) + atomic_inc(&t->events); +} + +static enum drm_connector_status relink_test_detect(struct drm_connector *connector, bool force) +{ + return dcp_connector_connected(to_apple_connector(connector)) ? + connector_status_connected : connector_status_disconnected; +} + +/* As dcp_get_modes(): what a probe reads belongs to the current cable. */ +static int relink_test_get_modes(struct drm_connector *connector) +{ + struct relink_test *t = to_relink_test(connector->dev); + struct drm_display_mode *mode; + + atomic_inc(&t->probes); + scoped_guard(mutex, &t->dcp.hpd_mutex) { + t->probed_generation = t->dcp.typec_generation; + if (t->bounce_on_probe) + t->dcp.typec_generation++; + } + if (!t->offered) + return 0; + mode = drm_mode_duplicate(connector->dev, t->mode); + if (!mode) + return 0; + drm_mode_probed_add(connector, mode); + return 1; +} + +/* As lookup_mode() for the exception mode, under the same lock. */ +static bool relink_test_mode_fixup(struct drm_crtc *crtc, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) +{ + struct relink_test *t = to_relink_test(crtc->dev); + + atomic_inc(&t->checks); + guard(mutex)(&t->dcp.hpd_mutex); + return t->offered && t->probed_generation == t->dcp.typec_generation; +} + +/* + * As dcp_crtc_atomic_modeset(): the mode set trains the link, or fails and + * has the hotplug worker recover the CRTC. + */ +/* The firmware for the real mode set path: it answers every method call. */ +static void relink_test_push(struct apple_dcp *dcp, const struct dcp_method_entry *call, + dcp_callback_t cb, void *cookie) +{ + struct relink_test *t = container_of(dcp, struct relink_test, dcp); + + cb(dcp, &t->reply, cookie); +} + +static void relink_test_crtc_enable(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct relink_test *t = to_relink_test(crtc->dev); + unsigned int action; + + atomic_inc(&t->modesets); + if (t->hold_modeset) { + complete(&t->modeset_entered); + wait_for_completion(&t->modeset_release); + } + if (t->real_modeset) { + t->modeset_ret = dcp_crtc_atomic_modeset(crtc, state); + return; + } + dcp_mode_begin(&t->dcp.mode_state); + action = dcp_mode_finish(&t->dcp.mode_state, t->trains, &t->con.connected); + if (!t->trains) + action |= DCP_HOTPLUG_NOTIFY; + dcp_handle_hotplug_actions(&t->dcp, action); +} + +/* As dcp_poweroff() for the external display. */ +static void relink_test_crtc_disable(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct relink_test *t = to_relink_test(crtc->dev); + + if (drm_atomic_get_new_crtc_state(state, crtc)->active) + return; + guard(mutex)(&t->dcp.external_lock); + dcp_mode_invalidate(&t->dcp.mode_state); + dcp_external_crtc_released(&t->dcp); +} + +static const struct drm_crtc_funcs relink_test_crtc_funcs = { + .reset = drm_atomic_helper_crtc_reset, + .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, +}; + +static const struct drm_crtc_helper_funcs relink_test_crtc_helpers = { + .mode_fixup = relink_test_mode_fixup, + .atomic_enable = relink_test_crtc_enable, + .atomic_disable = relink_test_crtc_disable, +}; + +/* Scanout itself is not under test; the plane helpers require an update. */ +static void relink_test_plane_update(struct drm_plane *plane, struct drm_atomic_state *state) +{ +} + +static const struct drm_plane_helper_funcs relink_test_plane_helpers = { + .atomic_update = relink_test_plane_update, +}; + +static const struct drm_connector_helper_funcs relink_test_connector_helpers = { + .get_modes = relink_test_get_modes, +}; + +static const struct drm_connector_funcs relink_test_connector_funcs = { + .fill_modes = drm_helper_probe_single_connector_modes, + .detect = relink_test_detect, + .reset = drm_atomic_helper_connector_reset, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, +}; + +static void relink_test_fb_destroy(struct drm_framebuffer *fb) +{ + drm_framebuffer_cleanup(fb); +} + +static const struct drm_framebuffer_funcs relink_test_fb_funcs = { + .destroy = relink_test_fb_destroy, +}; + +static void relink_test_exit(void *data) +{ + struct relink_test *t = data; + + complete_all(&t->modeset_release); + wait_for_completion_timeout(&t->userspace_done, 10 * HZ); + disable_delayed_work_sync(&t->dcp.announce_wq); + disable_work_sync(&t->con.hotplug_wq); + drm_atomic_helper_shutdown(&t->drm); + drm_framebuffer_put(&t->fb); +} + +/* Commit @active for the CRTC, as userspace would. */ +static int relink_test_commit(struct relink_test *t, bool active, bool init) +{ + struct drm_connector_state *conn_state; + struct drm_plane_state *plane_state; + struct drm_modeset_acquire_ctx ctx; + struct drm_crtc_state *crtc_state; + struct drm_atomic_state *state; + int ret; + + drm_modeset_acquire_init(&ctx, 0); +retry: + state = drm_atomic_state_alloc(&t->drm); + if (!state) { + ret = -ENOMEM; + goto out; + } + state->acquire_ctx = &ctx; + crtc_state = drm_atomic_get_crtc_state(state, t->crtc); + if (IS_ERR(crtc_state)) { + ret = PTR_ERR(crtc_state); + goto put; + } + crtc_state->active = active; + if (t->force_modeset) + crtc_state->connectors_changed = true; + if (init) { + conn_state = drm_atomic_get_connector_state(state, &t->con.base); + if (IS_ERR(conn_state)) { + ret = PTR_ERR(conn_state); + goto put; + } + plane_state = drm_atomic_get_plane_state(state, t->plane); + if (IS_ERR(plane_state)) { + ret = PTR_ERR(plane_state); + goto put; + } + ret = drm_atomic_set_crtc_for_connector(conn_state, t->crtc) ?: + drm_atomic_set_mode_for_crtc(crtc_state, t->mode) ?: + drm_atomic_set_crtc_for_plane(plane_state, t->crtc); + if (ret) + goto put; + drm_atomic_set_fb_for_plane(plane_state, &t->fb); + plane_state->crtc_w = t->mode->hdisplay; + plane_state->crtc_h = t->mode->vdisplay; + plane_state->src_w = t->mode->hdisplay << 16; + plane_state->src_h = t->mode->vdisplay << 16; + } + ret = drm_atomic_commit(state); +put: + drm_atomic_state_put(state); + if (ret == -EDEADLK) { + drm_modeset_backoff(&ctx); + goto retry; + } +out: + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + return ret; +} + +/* + * Let the hotplug worker run until it is idle. A held connection keeps its + * timeout work queued, which is flushed too but may stay queued. + */ +static void relink_test_settle(struct kunit *test, struct relink_test *t) +{ + int i; + + for (i = 0; i < 8; i++) { + flush_delayed_work(&t->dcp.announce_wq); + flush_work(&t->con.hotplug_wq); + if (!work_pending(&t->con.hotplug_wq) && + (t->dcp.announce.holding || !delayed_work_pending(&t->dcp.announce_wq))) + return; + } + KUNIT_FAIL(test, "the hotplug worker did not settle"); +} + +static void relink_test_hotplug(struct relink_test *t) +{ + schedule_work(&t->con.hotplug_wq); + flush_work(&t->con.hotplug_wq); +} + +/* + * End the hold of a new connection by its timeout: the modes of this + * fixture never get ready. + */ +static void relink_test_expire(struct kunit *test, struct relink_test *t) +{ + scoped_guard(mutex, &t->dcp.hpd_mutex) { + t->dcp.announce.timed = true; + t->dcp.announce.deadline = jiffies - 1; + } + mod_delayed_work(system_freezable_wq, &t->dcp.announce_wq, 0); + relink_test_settle(test, t); +} + +/* As dcp_external_disconnect() for a route that went away. */ +static void relink_test_tear(struct relink_test *t) +{ + struct apple_dcp *dcp = &t->dcp; + + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, false); + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + dcp_announce_end(&dcp->announce); + } + dcp_mode_invalidate(&dcp->mode_state); + WRITE_ONCE(t->con.connected, false); + relink_test_hotplug(t); +} + +/* + * As dcp_dptx_connect_oob() for the route coming back, then the firmware's + * connected callback, then the end of the new connection's hold. + */ +static void relink_test_reconnect(struct kunit *test, struct relink_test *t) +{ + struct apple_dcp *dcp = &t->dcp; + + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, true); + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); + dcp_announce_end(&dcp->announce); + } + WRITE_ONCE(t->con.connected, true); + relink_test_hotplug(t); + KUNIT_ASSERT_TRUE(test, dcp->announce.holding); + relink_test_expire(test, t); +} + +static bool relink_test_connected(struct relink_test *t) +{ + return relink_test_detect(&t->con.base, false) == connector_status_connected; +} + +static bool relink_test_link_up(struct relink_test *t) +{ + return READ_ONCE(t->dcp.mode_state.valid); +} + +/* + * A display on cable generation 2, reported connected, with userspace + * scanning out to it at a mode admitted for that generation. + */ +static struct relink_test *relink_test_init(struct kunit *test) +{ + struct apple_crtc *apple_crtc; + struct relink_test *t; + struct apple_dcp *dcp; + struct drm_device *drm; + struct device *dev; + int ret; + + dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + t = drm_kunit_helper_alloc_drm_device(test, dev, struct relink_test, drm, + DRIVER_MODESET | DRIVER_ATOMIC); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t); + drm = &t->drm; + dcp = &t->dcp; + + t->pdev = kunit_platform_device_alloc(test, "dcp-relink", PLATFORM_DEVID_AUTO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->pdev); + KUNIT_ASSERT_EQ(test, kunit_platform_device_add(test, t->pdev), 0); + platform_set_drvdata(t->pdev, dcp); + dcp->dev = &t->pdev->dev; + dcp->hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL; + iomfb_queue_init(dcp); + mutex_init(&dcp->external_lock); + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->dcpavserv.lock); + spin_lock_init(&dcp->metadata.lock); + spin_lock_init(&dcp->mode_state.lock); + INIT_DELAYED_WORK(&dcp->announce_wq, dcp_announce_work); + INIT_WORK(&dcp->vblank_wq, relink_test_noop); + INIT_WORK(&t->con.hotplug_wq, dcp_hotplug); + dcp->test_point = relink_test_point; + dcp->typec_cable_connected = true; + dcp->typec_generation = 2; + dcp_firmware_metadata_connection(dcp, 2, true); + dcp->connector = &t->con; + t->con.dcp = t->pdev; + t->con.connected = true; + t->offered = true; + t->trains = true; + t->probed_generation = 2; + init_completion(&t->modeset_entered); + init_completion(&t->modeset_release); + init_completion(&t->userspace_done); + complete_all(&t->userspace_done); + + t->plane = drm_kunit_helper_create_primary_plane(test, drm, NULL, + &relink_test_plane_helpers, + NULL, 0, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->plane); + /* An apple_crtc, so that the real mode set path finds its DCP. */ + apple_crtc = drmm_kzalloc(drm, sizeof(*apple_crtc), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, apple_crtc); + apple_crtc->dcp = t->pdev; + t->crtc = &apple_crtc->base; + ret = drmm_crtc_init_with_planes(drm, t->crtc, t->plane, NULL, + &relink_test_crtc_funcs, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + drm_crtc_helper_add(t->crtc, &relink_test_crtc_helpers); + ret = drmm_encoder_init(drm, &t->encoder, NULL, DRM_MODE_ENCODER_TMDS, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + t->encoder.possible_crtcs = drm_crtc_mask(t->crtc); + ret = drmm_connector_init(drm, &t->con.base, &relink_test_connector_funcs, + DRM_MODE_CONNECTOR_USB, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + drm_connector_helper_add(&t->con.base, &relink_test_connector_helpers); + KUNIT_ASSERT_EQ(test, drm_connector_attach_encoder(&t->con.base, &t->encoder), 0); + drm_mode_config_reset(drm); + /* The device is never registered, so it has nobody to send uevents to. */ + dev_set_uevent_suppress(drm->primary->kdev, true); + + t->mode = drm_kunit_display_mode_from_cea_vic(test, drm, 16); + KUNIT_ASSERT_NOT_NULL(test, t->mode); + t->fb.dev = drm; + t->fb.format = drm_format_info(DRM_FORMAT_XRGB8888); + t->fb.width = t->mode->hdisplay; + t->fb.height = t->mode->vdisplay; + t->fb.pitches[0] = t->mode->hdisplay * 4; + t->fb.modifier = DRM_FORMAT_MOD_LINEAR; + KUNIT_ASSERT_EQ(test, drm_framebuffer_init(drm, &t->fb, &relink_test_fb_funcs), 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, relink_test_exit, t), 0); + + KUNIT_ASSERT_EQ(test, relink_test_commit(t, true, true), 0); + KUNIT_ASSERT_TRUE(test, t->crtc->state->active); + KUNIT_ASSERT_TRUE(test, relink_test_link_up(t)); + KUNIT_ASSERT_EQ(test, atomic_read(&t->modesets), 1); + dcp->announce.announced = true; + + return t; +} + +/* + * The Neo sequence: the route of an active CRTC goes away and comes back, + * with no valid mode, and before userspace has probed the new cable. The + * mode is set again, and the link is good. + */ +static void relink_bounce_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + int events; + + relink_test_tear(t); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + /* Userspace has not reacted: its CRTC is still on. */ + KUNIT_ASSERT_TRUE(test, t->crtc->state->active); + + events = atomic_read(&t->events); + relink_test_reconnect(test, t); + + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 2); + KUNIT_EXPECT_TRUE(test, relink_test_link_up(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_GOOD); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); + KUNIT_EXPECT_GT(test, atomic_read(&t->events), events); +} + +/* + * The display on the new link no longer offers the mode: the link is bad + * and the display gone until userspace switches the CRTC off, and then it + * is a new connection. + */ +static void relink_mode_gone_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + int events, checks; + + relink_test_tear(t); + t->offered = false; + events = atomic_read(&t->events); + relink_test_reconnect(test, t); + + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 1); + KUNIT_EXPECT_FALSE(test, relink_test_link_up(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_BAD); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + KUNIT_EXPECT_GT(test, atomic_read(&t->events), events); + + /* Later firmware facts neither bring it back nor try again. */ + checks = atomic_read(&t->checks); + relink_test_hotplug(t); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->checks), checks); + + /* Userspace takes the display away, so it is new when it is back. */ + KUNIT_ASSERT_EQ(test, relink_test_commit(t, false, false), 0); + relink_test_settle(test, t); + KUNIT_EXPECT_TRUE(test, t->dcp.announce.holding); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + events = atomic_read(&t->events); + relink_test_expire(test, t); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); + KUNIT_EXPECT_GT(test, atomic_read(&t->events), events); + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 1); +} + +/* The mode is set again but the link does not come up: as above. */ +static void relink_link_fails_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + int events; + + relink_test_tear(t); + t->trains = false; + events = atomic_read(&t->events); + relink_test_reconnect(test, t); + + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 2); + KUNIT_EXPECT_FALSE(test, relink_test_link_up(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_BAD); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + KUNIT_EXPECT_GT(test, atomic_read(&t->events), events); +} + +/* A loss belongs to its cable: the next one is set up again. */ +static void relink_next_cable_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + relink_test_tear(t); + t->offered = false; + relink_test_reconnect(test, t); + KUNIT_ASSERT_FALSE(test, relink_test_connected(t)); + + relink_test_tear(t); + t->offered = true; + relink_test_reconnect(test, t); + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 2); + KUNIT_EXPECT_TRUE(test, relink_test_link_up(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_GOOD); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); +} + +/* A CRTC userspace switched off is left to userspace. */ +static void relink_inactive_crtc_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + KUNIT_ASSERT_EQ(test, relink_test_commit(t, false, false), 0); + relink_test_settle(test, t); + relink_test_tear(t); + relink_test_reconnect(test, t); + + KUNIT_EXPECT_EQ(test, atomic_read(&t->probes), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 1); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); +} + +/* + * Other DCPs keep their retraining: the CRTC is reset with its link marked + * bad, without a probe. + */ +static void relink_other_profile_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + t->dcp.hw.iomfb_method_profile = DCP_IOMFB_METHODS_DEFAULT; + dcp_mode_invalidate(&t->dcp.mode_state); + relink_test_hotplug(t); + + KUNIT_EXPECT_EQ(test, atomic_read(&t->probes), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 2); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_BAD); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); +} + +/* A route that bounces again during the probe leaves it to the new cable. */ +static void relink_newer_cable_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + relink_test_tear(t); + t->bounce_on_probe = true; + relink_test_reconnect(test, t); + + KUNIT_EXPECT_EQ(test, atomic_read(&t->probes), 1); + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 1); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_GOOD); +} + +/* Before drm_mode_config_reset() the connector has no state to recover. */ +static void relink_no_state_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + struct drm_connector_state *state = t->con.base.state; + + t->con.base.state = NULL; + dcp_mode_invalidate(&t->dcp.mode_state); + relink_test_hotplug(t); + t->con.base.state = state; + + KUNIT_EXPECT_EQ(test, atomic_read(&t->probes), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), 1); +} + +static int relink_test_userspace_off(void *data) +{ + struct relink_test *t = data; + + t->userspace_ret = relink_test_commit(t, false, false); + complete(&t->userspace_done); + return 0; +} + +/* + * Userspace switches the CRTC off while the mode set that fails to bring + * the link up is in flight. Whichever goes first, the display is not left + * lost with its CRTC off: it comes back as a new connection. + */ +static void relink_userspace_off_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + struct task_struct *task; + + relink_test_tear(t); + t->trains = false; + t->hold_modeset = true; + reinit_completion(&t->userspace_done); + + /* The reconnect, up to the mode set of the end of its hold. */ + scoped_guard(mutex, &t->dcp.hpd_mutex) { + WRITE_ONCE(t->dcp.typec_cable_connected, true); + t->dcp.typec_generation++; + dcp_firmware_metadata_connection(&t->dcp, t->dcp.typec_generation, true); + dcp_announce_end(&t->dcp.announce); + } + WRITE_ONCE(t->con.connected, true); + relink_test_hotplug(t); + scoped_guard(mutex, &t->dcp.hpd_mutex) { + t->dcp.announce.timed = true; + t->dcp.announce.deadline = jiffies - 1; + } + mod_delayed_work(system_freezable_wq, &t->dcp.announce_wq, 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->modeset_entered, 5 * HZ), 0); + + task = kthread_run(relink_test_userspace_off, t, "relink-userspace"); + if (IS_ERR(task)) + complete(&t->userspace_done); + KUNIT_ASSERT_FALSE(test, IS_ERR(task)); + msleep(50); + t->hold_modeset = false; + complete_all(&t->modeset_release); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&t->userspace_done, 5 * HZ), 0); + KUNIT_ASSERT_EQ(test, t->userspace_ret, 0); + relink_test_settle(test, t); + + KUNIT_EXPECT_FALSE(test, t->crtc->state->active); + relink_test_expire(test, t); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); +} + +/* + * A link that never comes up: lost, back once after userspace switches the + * CRTC off, then lost for good when the mode set userspace makes on it + * fails too, until the next cable. + */ +static void relink_retry_once_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + int modesets; + + relink_test_tear(t); + t->trains = false; + relink_test_reconnect(test, t); + KUNIT_ASSERT_FALSE(test, relink_test_connected(t)); + + /* Userspace switches the CRTC off: the display comes back once. */ + KUNIT_ASSERT_EQ(test, relink_test_commit(t, false, false), 0); + relink_test_settle(test, t); + relink_test_expire(test, t); + KUNIT_ASSERT_TRUE(test, relink_test_connected(t)); + + /* Its mode set fails: lost again, without a mode set of the kernel's own. */ + modesets = atomic_read(&t->modesets); + KUNIT_ASSERT_EQ(test, relink_test_commit(t, true, false), 0); + relink_test_settle(test, t); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_BAD); + KUNIT_EXPECT_EQ(test, atomic_read(&t->modesets), modesets + 2); + + /* Switched off again, it stays gone on this cable. */ + KUNIT_ASSERT_EQ(test, relink_test_commit(t, false, false), 0); + relink_test_settle(test, t); + KUNIT_EXPECT_FALSE(test, t->dcp.announce.holding); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + + /* The next cable is announced again. */ + relink_test_tear(t); + t->trains = true; + relink_test_reconnect(test, t); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); +} + +/* + * Mode sets go through the real dcp_crtc_atomic_modeset() and the H17P mode + * set path, with the test answering for the firmware. + */ +static void relink_test_real(struct relink_test *t) +{ + t->real_modeset = true; + t->dcp.external_running = true; + t->dcp.fw_compat = DCP_FIRMWARE_H17P; + t->catalog.mode = *t->mode; + t->dcp.modes = &t->catalog; + t->dcp.nr_modes = 1; + t->dcp.test_push = relink_test_push; +} + +/* The bounce, through the real mode set: the link comes back. */ +static void relink_real_modeset_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + relink_test_real(t); + relink_test_tear(t); + relink_test_reconnect(test, t); + KUNIT_EXPECT_EQ(test, t->modeset_ret, 0); + KUNIT_EXPECT_TRUE(test, relink_test_link_up(t)); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_GOOD); +} + +/* + * The pipe was stopped while its loss was still buffered by the paused + * hotplug callback, and the next activation fails before its mode set + * begins: the old mode must not pass for a link. + */ +static void relink_early_epipe_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + t->dcp.external_running = true; + t->dcp.external_cable = true; + t->dcp.external_power_attempted = true; + t->dcp.fw_compat = DCP_FIRMWARE_H17P; + scoped_guard(mutex, &t->dcp.iomfb.lock) { + t->dcp.iomfb.presentation_paused = true; + KUNIT_EXPECT_TRUE(test, dcp_external_hotplug(&t->dcp, false)); + } + KUNIT_ASSERT_TRUE(test, relink_test_link_up(t)); + KUNIT_ASSERT_TRUE(test, t->dcp.external_hotplug_pending); + t->dcp.external_running = false; + t->dcp.external_stopped = true; + t->real_modeset = true; + t->force_modeset = true; + KUNIT_ASSERT_EQ(test, relink_test_commit(t, true, false), 0); + KUNIT_ASSERT_EQ(test, t->modeset_ret, -EPIPE); + relink_test_settle(test, t); + KUNIT_EXPECT_FALSE(test, relink_test_link_up(t)); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_BAD); +} + +/* A presentation paused on purpose is not a failed link. */ +static void relink_paused_kept_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + t->dcp.external_running = true; + t->dcp.fw_compat = DCP_FIRMWARE_H17P; + scoped_guard(mutex, &t->dcp.iomfb.lock) + t->dcp.iomfb.presentation_paused = true; + t->real_modeset = true; + t->force_modeset = true; + KUNIT_ASSERT_EQ(test, relink_test_commit(t, true, false), 0); + KUNIT_EXPECT_EQ(test, t->modeset_ret, -EPIPE); + relink_test_settle(test, t); + KUNIT_EXPECT_TRUE(test, relink_test_link_up(t)); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); + scoped_guard(mutex, &t->dcp.iomfb.lock) + t->dcp.iomfb.presentation_paused = false; +} + +/* + * Once a lost connection was reported, unchanged facts send no more events: + * each would restart a compositor's wait for the output to stay gone. + */ +static void relink_lost_quiet_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + int events, i; + + relink_test_tear(t); + t->trains = false; + relink_test_reconnect(test, t); + KUNIT_ASSERT_FALSE(test, relink_test_connected(t)); + relink_test_settle(test, t); + events = atomic_read(&t->events); + for (i = 0; i < 5; i++) + relink_test_hotplug(t); + KUNIT_EXPECT_EQ(test, atomic_read(&t->events), events); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); +} + +static struct kunit_case relink_cases[] = { + KUNIT_CASE(relink_bounce_test), + KUNIT_CASE(relink_mode_gone_test), + KUNIT_CASE(relink_link_fails_test), + KUNIT_CASE(relink_next_cable_test), + KUNIT_CASE(relink_inactive_crtc_test), + KUNIT_CASE(relink_other_profile_test), + KUNIT_CASE(relink_newer_cable_test), + KUNIT_CASE(relink_no_state_test), + KUNIT_CASE(relink_userspace_off_test), + KUNIT_CASE(relink_retry_once_test), + KUNIT_CASE(relink_real_modeset_test), + KUNIT_CASE(relink_early_epipe_test), + KUNIT_CASE(relink_paused_kept_test), + KUNIT_CASE(relink_lost_quiet_test), + {} +}; + +static struct kunit_suite relink_suite = { + .name = "apple-dcp-external-relink", + .test_cases = relink_cases, +}; + +kunit_test_suite(relink_suite); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index a5de59da93f97..dc96c67735489 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -623,6 +624,14 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca { struct dcp_method_entry resolved = *call; +#if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) + /* A KUnit test answers for the firmware. */ + if (dcp->test_push) { + dcp->test_push(dcp, call, cb, cookie); + return; + } +#endif + /* Reply chains stay on the serialized receiver; callers enqueue copies. */ if (iomfb_uses_queue(dcp) && READ_ONCE(dcp->iomfb.owner) != current) { if (iomfb_queue_command(dcp, oob, call, in_len, out_len, @@ -838,6 +847,7 @@ dcp_announce_eval(struct apple_dcp *dcp, struct apple_connector *connector, .connected = READ_ONCE(connector->connected), .present = !READ_ONCE(dcp->external_physically_absent), .failed = READ_ONCE(dcp->external_error) || READ_ONCE(dcp->crashed), + .lost = dcp_announce_lost(a, dcp->typec_generation), }; enum dcp_announce_status status; unsigned long now = jiffies; @@ -913,12 +923,174 @@ bool dcp_connector_connected(struct apple_connector *connector) return dcp->announce.announced; } +/* The CRTC userspace keeps active on @connector while its link is down. */ +static struct drm_crtc *dcp_external_unlinked_crtc(struct apple_dcp *dcp, + struct apple_connector *connector) +{ + struct drm_connector_state *conn_state = connector->base.state; + struct drm_crtc *crtc = conn_state ? conn_state->crtc : NULL; + + drm_modeset_lock_assert_held(&connector->base.dev->mode_config.connection_mutex); + if (!crtc || !crtc->state || !crtc->state->active || + READ_ONCE(dcp->mode_state.valid)) + return NULL; + return crtc; +} + +static bool dcp_external_needs_relink(struct apple_dcp *dcp, + struct apple_connector *connector) +{ + struct drm_device *dev = connector->base.dev; + struct drm_modeset_acquire_ctx ctx; + bool unlinked = false; + int ret; + + DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, ret); + unlinked = !!dcp_external_unlinked_crtc(dcp, connector); + DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); + + return unlinked; +} + +/* Commit @crtc with its mode set again and the link of @connector good. */ +static int dcp_external_commit_relink(struct apple_connector *connector, + struct drm_crtc *crtc, + struct drm_modeset_acquire_ctx *ctx) +{ + struct drm_connector_state *conn_state; + struct drm_crtc_state *crtc_state; + struct drm_atomic_state *state; + int ret; + + state = drm_atomic_state_alloc(crtc->dev); + if (!state) + return -ENOMEM; + state->acquire_ctx = ctx; + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(crtc_state)) { + ret = PTR_ERR(crtc_state); + goto out; + } + conn_state = drm_atomic_get_connector_state(state, &connector->base); + if (IS_ERR(conn_state)) { + ret = PTR_ERR(conn_state); + goto out; + } + crtc_state->connectors_changed = true; + conn_state->link_status = DRM_LINK_STATUS_GOOD; + ret = drm_atomic_commit(state); +out: + drm_atomic_state_put(state); + + return ret; +} + +/* + * Set the mode of the CRTC userspace keeps active on @connector again. If + * that fails, or leaves the link down, mark the link bad and lose the + * connection of cable generation @generation. Both are decided under the + * modeset locks, so a CRTC that userspace switches off is either switched + * off first and left alone, or found lost by dcp_external_crtc_released(). + * + * Returns the error that lost the connection, or 0. + */ +static int dcp_external_reset_crtc(struct apple_dcp *dcp, + struct apple_connector *connector, u64 generation) +{ + struct drm_device *dev = connector->base.dev; + struct drm_modeset_acquire_ctx ctx; + struct drm_crtc *crtc; + int ret, err = 0; + + DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, ret); + + err = 0; + crtc = dcp_external_unlinked_crtc(dcp, connector); + if (!crtc) + goto unlock; + ret = dcp_external_commit_relink(connector, crtc, &ctx); + if (ret == -EDEADLK) + goto unlock; + if (!ret && READ_ONCE(dcp->mode_state.valid)) + goto unlock; + + err = ret ?: -EIO; + ret = 0; + scoped_guard(mutex, &dcp->hpd_mutex) { + if (generation == dcp->typec_generation) { + connector->base.state->link_status = DRM_LINK_STATUS_BAD; + dcp_announce_lose(&dcp->announce, generation); + /* The caller's event is the one this owes. */ + dcp_announce_notify(&dcp->announce, + dcp_announce_eval(dcp, connector, false)); + } else { + /* A newer connection gets a link of its own. */ + err = 0; + } + } +unlock: + DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); + + return ret ?: err; +} + +/* + * The link of an external display came back, or was disturbed, under a CRTC + * that userspace keeps active. Userspace sees the display it had and need + * not set its mode again, so set it here. A mode that needs its link + * proven, such as the 180 Hz exception mode, is admitted only against the + * EDID a probe read for the current cable generation (see + * dcp_180hz_mode_current()), so probe first, as userspace would. + * + * A link that cannot be set up again is marked bad, and the connection is + * lost: reported gone until userspace releases the CRTC, see + * dcp-announce.h. The caller sends the hotplug event. + */ +static void dcp_external_relink(struct apple_dcp *dcp, + struct apple_connector *connector) +{ + struct drm_device *dev = connector->base.dev; + u64 generation; + int ret; + + if (!dcp_external_needs_relink(dcp, connector)) + return; + + scoped_guard(mutex, &dcp->hpd_mutex) + generation = dcp->typec_generation; + scoped_guard(mutex, &dev->mode_config.mutex) + connector->base.funcs->fill_modes(&connector->base, + dev->mode_config.max_width, + dev->mode_config.max_height); + + ret = dcp_external_reset_crtc(dcp, connector, generation); + if (ret) + dev_warn(dcp->dev, "display link not restored (%d), reporting it disconnected\n", + ret); +} + +/* + * The CRTC of the external display was switched off. If its connection was + * lost, report the display again, as a new connection, once per cable + * generation (see dcp_announce_retry()). + */ +void dcp_external_crtc_released(struct apple_dcp *dcp) +{ + struct apple_connector *con = READ_ONCE(dcp->connector); + + guard(mutex)(&dcp->hpd_mutex); + if (dcp_announce_retry(&dcp->announce, dcp->typec_generation) && + !dcp->announce.stopped && con && + READ_ONCE(con->dcp) == to_platform_device(dcp->dev)) + schedule_work(&con->hotplug_wq); +} + void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; struct platform_device *pdev; struct apple_dcp *dcp; - bool connected; + bool connected, gated; int ret; connector = container_of(work, struct apple_connector, hotplug_wq); @@ -947,7 +1119,8 @@ void dcp_hotplug(struct work_struct *work) dcp_clear_connector_edid(connector); } - if (dcp_announce_gated(dcp, connector)) { + gated = dcp_announce_gated(dcp, connector); + if (gated) { enum dcp_announce_status status; bool notify; @@ -968,8 +1141,11 @@ void dcp_hotplug(struct work_struct *work) * display modes from atomic_flush, so userspace needs to trigger a * flush, or the CRTC gets no signal. */ - if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connected && - !(dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x())) { + if (gated) { + if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connected) + dcp_external_relink(dcp, connector); + } else if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connected && + !(dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x())) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); @@ -987,6 +1163,7 @@ void dcp_hotplug(struct work_struct *work) ret); } + dcp_test_point(dcp, DCP_TEST_HOTPLUG_EVENT); drm_kms_helper_connector_hotplug_event(&connector->base); } @@ -1499,8 +1676,19 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, guard(mutex)(&dcp->external_lock); if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && (!dcp->external_running || dcp->external_error || READ_ONCE(dcp->quiescing) || - READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.presentation_paused))) + READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.presentation_paused))) { + /* + * The pipe of this CRTC did not start, so it has no link and its + * old mode is gone with it: the hotplug worker sets it up again + * or reports the display lost. A pipe that runs with its + * presentation paused on purpose keeps its mode. + */ + if (!dcp->external_running || dcp->external_error) { + dcp_mode_invalidate(&dcp->mode_state); + dcp_handle_hotplug_actions(dcp, DCP_HOTPLUG_NOTIFY); + } return -EPIPE; + } crtc_state = drm_atomic_get_new_crtc_state(state, crtc); if (!crtc_state) @@ -1535,6 +1723,9 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, action = dcp_mode_finish(&dcp->mode_state, !ret, dcp->connector ? &dcp->connector->connected : NULL); + /* An external mode set that failed leaves the CRTC without a link too. */ + if (ret && dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) + action |= DCP_HOTPLUG_NOTIFY; dcp_handle_hotplug_actions(dcp, action); return ret; From eda0a777d655a7920560cf1e3905c246df697275 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 20:10:52 +1100 Subject: [PATCH 1098/1579] tools: aurora-sep: label /dev/sep-bio for fprintd under SELinux Fedora enforces its targeted SELinux policy by default. The policy has no rule for /dev/sep-bio, so the node gets the generic device_t label, which fprintd may not open. fprintd then lists no fingerprint reader although the driver is up and the enclave is ready. The MacBook Neo runs with SELinux disabled, which is why this was not seen there; Fedora Asahi Remix on an M1 Pro and an M2 with the default policy both hit it. Add a policy module that gives the node its own type, sep_bio_device_t, and lets fprintd read and write it. Ship it in a new aurora-sep-selinux subpackage the way Fedora packages independent policy: the module is built with selinux-policy-devel, installed at priority 200 by the selinux-policy macros and removed with the subpackage. aurora-sep requires it wherever the targeted policy is installed. udev labels the node when the driver creates it, and the scriptlets relabel a node that already exists. Where SELinux is disabled, the module is installed but not loaded and nothing is relabelled. Arch Linux does not ship SELinux. The Arch package only picks up the README change, and install-aurora-sep.sh, which installs on Omarchy, needs no change. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 16 +++++- tools/aurora-sep/aurora_sep.fc | 1 + tools/aurora-sep/aurora_sep.te | 24 +++++++++ .../packaging/arch/aurora-sep/PKGBUILD | 4 +- .../packaging/fedora/aurora-sep.spec | 52 ++++++++++++++++++- tools/aurora-sep/tests/test_packaging.py | 8 +++ 6 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 tools/aurora-sep/aurora_sep.fc create mode 100644 tools/aurora-sep/aurora_sep.te diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index ea9d2704e9e80..02bbdc1e8ed85 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -125,6 +125,16 @@ systemctl daemon-reload systemctl restart fprintd.service ``` +Where SELinux is enforcing, as on Fedora, fprintd may not open `/dev/sep-bio` +until it is labelled. Build and load the policy module (Fedora's +`selinux-policy-devel` provides the Makefile), then label the device: + +```sh +make -f /usr/share/selinux/devel/Makefile aurora_sep.pp +semodule -i aurora_sep.pp +restorecon /dev/sep-bio +``` + On Omarchy, run `omarchy-apply-lock` once after enrollment. The Quickshell lock screen then selects `omarchy-lock-fingerprint` automatically and accepts Touch ID through fprintd. Hyprlock's separate fingerprint switch is not used @@ -145,9 +155,13 @@ mkdir rpms podman run --rm -v "$PWD":/src:ro,z -v "$PWD/rpms":/out:z \ registry.fedoraproject.org/fedora:45 \ /src/tools/aurora-sep/packaging/fedora/build.sh /out -sudo dnf install rpms/aurora-sep-1-*.noarch.rpm +sudo dnf install rpms/aurora-sep-1-*.noarch.rpm rpms/aurora-sep-selinux-1-*.noarch.rpm ``` +`aurora-sep-selinux` holds the policy module above. `aurora-sep` requires it +wherever the targeted policy is installed. Where SELinux is disabled, the +module is installed but not loaded, and nothing is relabelled. + On Arch Linux and Arch Linux ARM, build `aurora-sep`: ```sh diff --git a/tools/aurora-sep/aurora_sep.fc b/tools/aurora-sep/aurora_sep.fc new file mode 100644 index 0000000000000..c0ef756351d51 --- /dev/null +++ b/tools/aurora-sep/aurora_sep.fc @@ -0,0 +1 @@ +/dev/sep-bio -c gen_context(system_u:object_r:sep_bio_device_t,s0) diff --git a/tools/aurora-sep/aurora_sep.te b/tools/aurora-sep/aurora_sep.te new file mode 100644 index 0000000000000..fd501d6be0fc4 --- /dev/null +++ b/tools/aurora-sep/aurora_sep.te @@ -0,0 +1,24 @@ +policy_module(aurora_sep, 1.0.0) + +######################################## +# +# Declarations +# + +# /dev/sep-bio, the apple_sep driver's Touch ID device. +type sep_bio_device_t; +dev_node(sep_bio_device_t) + +######################################## +# +# Local policy +# + +# fprintd, through libfprint's aurora driver, is the device's only user. +optional_policy(` + gen_require(` + type fprintd_t; + ') + + allow fprintd_t sep_bio_device_t:chr_file rw_chr_file_perms; +') diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 5c0f665f9dcfe..54b734fb27181 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -7,7 +7,7 @@ pkgname=aurora-sep pkgver=1 -pkgrel=5 +pkgrel=6 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -40,7 +40,7 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' - 'ed507587d3356ff2713e988e024b183772bab14061bf787bcaef28691867af0d' + '9a780d186919595de71a52bed574c1a9a8e40d114c8083783f770e8d760fe3d1' '3d92b3be06bc1b670666dec3d0032893fe8c7be428a4f1200d7c9803980c769a') # A unit and the helper it runs ($1, installed as $3), pointed at the diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index b4dd9ca4391b6..bd89ae45d81f7 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -3,9 +3,12 @@ # Built from tools/aurora-sep: rpmbuild --define "_sourcedir " # (build.sh does this). +%global selinuxtype targeted +%global modulename aurora_sep + Name: aurora-sep Version: 1 -Release: 3%{?dist} +Release: 4%{?dist} Summary: Loader and desktop integration for the Apple SEP fingerprint driver License: GPL-2.0-only @@ -21,6 +24,8 @@ Source7: fprintd-no-timeout.conf.in Source8: logind-aurora-sep.conf Source9: aurora-sep-greeter-fingerprint Source10: aurora-sep-greeter-fingerprint.service +Source11: aurora_sep.te +Source12: aurora_sep.fc BuildArch: noarch BuildRequires: systemd-rpm-macros @@ -30,6 +35,7 @@ Requires: coreutils Requires: fprintd Requires: keyutils Requires: kmod +Requires: (%{name}-selinux = %{version}-%{release} if selinux-policy-%{selinuxtype}) # Only for aurora-sep-greeter-fingerprint.service, which is off by default. Suggests: /usr/bin/gresource %{?systemd_requires} @@ -44,11 +50,25 @@ libfprint needs the aurora driver, which this package does not provide; it is packaged in https://github.com/aurora-silicon/aurora-sep-userspace. Read XART-SAFETY.md before installing it on a machine that also runs macOS. +%package selinux +Summary: SELinux policy module for the Apple SEP fingerprint device +BuildRequires: selinux-policy-devel +Requires: selinux-policy-%{selinuxtype} +Requires(post): selinux-policy-%{selinuxtype} +%{?selinux_requires} + +%description selinux +Labels /dev/sep-bio, the Apple SEP driver's Touch ID device, and lets fprintd +open it. The targeted policy otherwise leaves the device as device_t, which +fprintd may not open, so fprintd lists no fingerprint reader. + %prep %setup -q -c -T -cp -p %{SOURCE3} %{SOURCE4} . +cp -p %{SOURCE3} %{SOURCE4} %{SOURCE11} %{SOURCE12} . %build +make -f %{_datadir}/selinux/devel/Makefile %{modulename}.pp +bzip2 -9 %{modulename}.pp %install # A unit and the helper it runs ($1, installed as $3), pointed at the @@ -73,6 +93,8 @@ install -Dpm 0644 %{SOURCE8} \ # Load the driver at boot, so the enclave is up before the login screen asks. install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset +install -Dpm 0644 %{modulename}.pp.bz2 \ + %{buildroot}%{_datadir}/selinux/packages/%{selinuxtype}/%{modulename}.pp.bz2 %post %systemd_post aurora-sep.service aurora-sep-ready.service aurora-sep-greeter-fingerprint.service @@ -84,6 +106,25 @@ echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset # Never restart them: the driver attaches to the enclave once per boot. %systemd_postun aurora-sep.service aurora-sep-ready.service aurora-sep-greeter-fingerprint.service +%pre selinux +%selinux_relabel_pre -s %{selinuxtype} + +%post selinux +%selinux_modules_install -s %{selinuxtype} %{_datadir}/selinux/packages/%{selinuxtype}/%{modulename}.pp.bz2 + +%postun selinux +%selinux_modules_uninstall -s %{selinuxtype} %{modulename} +if [ $1 -eq 0 ] && selinuxenabled 2>/dev/null && [ -c /dev/sep-bio ]; then + restorecon /dev/sep-bio || : +fi + +%posttrans selinux +%selinux_relabel_post -s %{selinuxtype} +# udev labels the device when the driver creates it; label one that exists. +if selinuxenabled 2>/dev/null && [ -c /dev/sep-bio ]; then + restorecon /dev/sep-bio || : +fi + %files %doc README.md XART-SAFETY.md %{_libexecdir}/aurora-sep-load @@ -98,7 +139,14 @@ echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %{_prefix}/lib/systemd/logind.conf.d/50-aurora-sep.conf %{_presetdir}/80-aurora-sep.preset +%files selinux +%{_datadir}/selinux/packages/%{selinuxtype}/%{modulename}.pp.bz2 +%ghost %verify(not md5 size mode mtime) %{_sharedstatedir}/selinux/%{selinuxtype}/active/modules/200/%{modulename} + %changelog +* Fri Oct 16 2026 Ryan Murray - 1-4 +- Add aurora-sep-selinux, which labels /dev/sep-bio for fprintd + * Sun Oct 11 2026 Ryan Murray - 1-3 - libfprint with the aurora driver is now built in aurora-sep-userspace diff --git a/tools/aurora-sep/tests/test_packaging.py b/tools/aurora-sep/tests/test_packaging.py index 961985758ba1e..75755096cd16f 100644 --- a/tools/aurora-sep/tests/test_packaging.py +++ b/tools/aurora-sep/tests/test_packaging.py @@ -53,6 +53,14 @@ def test_sources_are_in_tree(self): for source in sources: self.assertTrue((SEP / source).is_file(), source) + def test_selinux_module_labels_the_loader_device(self): + path, kind, context = (SEP / "aurora_sep.fc").read_text().split() + self.assertEqual(kind, "-c") + self.assertIn(path, (SEP / "load-driver").read_text()) + label = re.search(r":(\w+_t),", context).group(1) + te = (SEP / "aurora_sep.te").read_text() + self.assertRegex(te, r"(?m)^type %s;$" % label) + def test_build_script_builds_only_aurora_sep(self): text = (FEDORA / "build.sh").read_text() self.assertIn("aurora-sep.spec", text) From 2c252de70f4cd7b1d85d5f19689620d55709e04c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 20:26:20 +1100 Subject: [PATCH 1099/1579] tools: aurora-sep: note the cold-boot keybag write in XART-SAFETY.md On an M1 Pro (T6000, J316s), where the driver cold-boots the SEP, creating the Linux identity keybag makes the enclave rewrite a session record that existed before Linux ran. The record is one of the four the enclave reads at every attach, and it is not tied to the OS identity, so Linux's own identity does not keep the write away from what macOS reads. Warm-attach Macs do not do this: on an M2 (T8112, J413) keybag creation made no xART writes. Record this in XART-SAFETY.md, and say that macOS Touch ID on a T600x Mac where Linux has created its keybag is unverified until it has been checked there. Update the Arch package's checksum for the file. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/XART-SAFETY.md | 27 +++++++++++++++++++ .../packaging/arch/aurora-sep/PKGBUILD | 4 +-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md index 5dd70e8b26a80..3ca3b9a5cf3c4 100644 --- a/tools/aurora-sep/XART-SAFETY.md +++ b/tools/aurora-sep/XART-SAFETY.md @@ -10,6 +10,33 @@ identity if missing and recover an existing identity without replacing it. The historical evidence below records the earlier opt-in deployments and does not establish qualification on every supported Mac or APFS layout. +## Cold-boot SEP Macs rewrite a pre-Linux record (2026-10-16) + +On Macs where the driver cold-boots the SEP, creating Linux's identity +keybag reaches beyond Linux's own records. This was observed on an M1 Pro +(T6000, J316s), with the macOS 13.5 sepOS loaded by the driver. + +Creating the keybag makes the enclave do three things: + +- write one new session record; +- rewrite a shared root record; +- rewrite one session record that existed before Linux ever ran. + +The rewritten record is one of the four session records the enclave reads at +every attach, and it grows by 71 bytes per keybag created. It is not tied to +the OS identity, so giving Linux its own identity does not keep this write +away from records macOS also reads. A driver that refuses to rewrite records +Linux did not create cannot create a keybag on these Macs. + +Warm-attach Macs behave differently. On an M2 (T8112, J413), keybag creation +made no xART writes at all. On the J700, the macOS-era records stayed +byte-identical through Linux enrolment. + +On a T600x Mac where Linux has created its keybag, treat macOS Touch ID as +unverified until macOS Touch ID unlock and keychain access have been checked +there. The other cold-boot SoCs (T8103, T6001, T6002) probably behave the +same way; that is untested. + ## Earlier deployment policy The iBoot APFS container and its XART `.gl` file are shared with macOS. The diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 54b734fb27181..c2c508bce9673 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -7,7 +7,7 @@ pkgname=aurora-sep pkgver=1 -pkgrel=6 +pkgrel=7 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -41,7 +41,7 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' '9a780d186919595de71a52bed574c1a9a8e40d114c8083783f770e8d760fe3d1' - '3d92b3be06bc1b670666dec3d0032893fe8c7be428a4f1200d7c9803980c769a') + '756ef505520d192ca06e0e79738258a29f1b0f13fb081e7d5992405cd6ecc19f') # A unit and the helper it runs ($1, installed as $3), pointed at the # packaged helper. From 2c672ce772ff574fe8884b5ad27d99cf5cb64f92 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 20:33:39 +1100 Subject: [PATCH 1100/1579] tools: aurora-sep: say how to turn on fingerprint PAM under authselect Fedora's authselect profiles leave pam_fprintd out until their with-fingerprint feature is selected, and authselect also locks org.gnome.login-screen enable-fingerprint-authentication to false. So with Touch ID up and a finger enrolled, sudo, polkit and the lock screen still never ask for it, and GNOME Settings shows no Fingerprint Login row. Nothing in the logs says why. Which services take a fingerprint is the user's choice, so the packages should not change PAM. Leave it alone and say what to run instead. When Touch ID comes up on a system whose authselect configuration lacks the feature, aurora-sep-ready logs one line with the command: sudo authselect enable-feature with-fingerprint The README's Fedora section gives the same step and how to undo it. Systems without an authselect configuration, such as Arch, get no line. On Omarchy, aurora-touchid-setup already makes the PAM changes after an enrolment. Also correct the README's claim that enable-fingerprint-authentication is true by default, which does not hold under authselect. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 23 ++++++- tools/aurora-sep/aurora-sep-ready | 12 ++++ .../packaging/arch/aurora-sep/PKGBUILD | 6 +- .../packaging/fedora/aurora-sep.spec | 5 +- tools/aurora-sep/tests/test_ready_pam_hint.py | 64 +++++++++++++++++++ 5 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 tools/aurora-sep/tests/test_ready_pam_hint.py diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 02bbdc1e8ed85..30fc5bb0dd327 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -162,6 +162,24 @@ sudo dnf install rpms/aurora-sep-1-*.noarch.rpm rpms/aurora-sep-selinux-1-*.noar wherever the targeted policy is installed. Where SELinux is disabled, the module is installed but not loaded, and nothing is relabelled. +**On Fedora, turn on fingerprint authentication yourself.** PAM decides what +asks for a fingerprint, and the packages leave it alone. Fedora's authselect +profiles keep fingerprints off until their `with-fingerprint` feature is +selected. Until then sudo, polkit and the lock screen never ask, and GNOME +Settings shows no Fingerprint Login row, even with a finger enrolled. Once a +finger is enrolled: + +```sh +sudo authselect enable-feature with-fingerprint +``` + +This adds `pam_fprintd` to the system and fingerprint PAM stacks. It also sets +`org.gnome.login-screen enable-fingerprint-authentication`, which authselect +otherwise locks to false. `sudo authselect disable-feature with-fingerprint` +undoes both. While the feature is off, `aurora-sep-ready` logs this command +once per boot. On Omarchy, `aurora-touchid-setup` makes the PAM changes after +an enrolment. + On Arch Linux and Arch Linux ARM, build `aurora-sep`: ```sh @@ -230,8 +248,9 @@ install -Dm644 logind-aurora-sep.conf /etc/systemd/logind.conf.d/aurora-sep.conf GNOME Shell 51 offers fingerprint only on the lock screen, never on the login screen, and `org.gnome.login-screen enable-fingerprint-authentication` -(true by default) cannot change that: the greeter connects its fingerprint -reader only when it re-authenticates a running session. That is GNOME's +(true in GNOME's defaults, false under authselect until `with-fingerprint` is +on) cannot change that: the greeter connects its fingerprint reader only +when it re-authenticates a running session. That is GNOME's choice, because a fingerprint cannot unlock the login keyring: after a fingerprint login, the first application that wants a stored secret asks for the password. `aurora-sep-greeter-fingerprint.service` lifts the diff --git a/tools/aurora-sep/aurora-sep-ready b/tools/aurora-sep/aurora-sep-ready index 0363a3a298ecb..ed214f0d1fb43 100644 --- a/tools/aurora-sep/aurora-sep-ready +++ b/tools/aurora-sep/aurora-sep-ready @@ -19,6 +19,17 @@ uptime_s() { echo "${up%%.*}" } +# fprintd answers only what PAM asks it. Where authselect manages PAM (Fedora), +# fingerprints stay unused until its with-fingerprint feature is on; that is +# the user's choice, so only say how to make it. +pam_hint() { + features=$(authselect current --raw 2>/dev/null) || return 0 + case " $features " in + *" with-fingerprint "*) ;; + *) log "PAM does not use fingerprints; for Touch ID at sudo, polkit and the lock screen, run: sudo authselect enable-feature with-fingerprint" ;; + esac +} + if [ ! -c "$dev" ]; then log "$dev is not there; Touch ID is not available this boot" exit 0 @@ -30,6 +41,7 @@ while :; do # would create a file if the device went away. if err=$(LC_ALL=C sh -c 'exec 3<"$1"' sh "$dev" 2>&1); then log "Touch ID is ready after $(($(uptime_s) - start)) s" + pam_hint exit 0 fi case "$err" in diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index c2c508bce9673..6b431109496c2 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -7,7 +7,7 @@ pkgname=aurora-sep pkgver=1 -pkgrel=7 +pkgrel=8 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -34,13 +34,13 @@ source=(load-driver sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '447f99cc3c7e683134f31b993aa825fadde24e4854954a3035c96e6631d4d4d0' 'f8d0c4b9385e6716d5ac5dd9941a57459fbc551a8238383bcde36cd929d0523f' - '2db07d3a0191e8d14581a3056505a8c4225b2fc8e6ea4a1956ae5493abcbb0ae' + '7d516d61edee50e066ab4e7bf6eba1ade2810ef4968e69590744750b03062aa9' '5a5e5c7e93d1462d9d1023446fdb297926aeb93d8988867b515aca7adafe4487' '8d8df0ba2a6861b2edb2110e606906d84ca4848617621ad3cb48493a1522971d' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' - '9a780d186919595de71a52bed574c1a9a8e40d114c8083783f770e8d760fe3d1' + '30e131dfd8d5fd5be3d7e7712292645e2c6b47eeddd5bafe304b119b30ba6067' '756ef505520d192ca06e0e79738258a29f1b0f13fb081e7d5992405cd6ecc19f') # A unit and the helper it runs ($1, installed as $3), pointed at the diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index bd89ae45d81f7..1636a34daa455 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -8,7 +8,7 @@ Name: aurora-sep Version: 1 -Release: 4%{?dist} +Release: 5%{?dist} Summary: Loader and desktop integration for the Apple SEP fingerprint driver License: GPL-2.0-only @@ -144,6 +144,9 @@ fi %ghost %verify(not md5 size mode mtime) %{_sharedstatedir}/selinux/%{selinuxtype}/active/modules/200/%{modulename} %changelog +* Fri Oct 16 2026 Ryan Murray - 1-5 +- Log the authselect command when PAM does not use fingerprints + * Fri Oct 16 2026 Ryan Murray - 1-4 - Add aurora-sep-selinux, which labels /dev/sep-bio for fprintd diff --git a/tools/aurora-sep/tests/test_ready_pam_hint.py b/tools/aurora-sep/tests/test_ready_pam_hint.py new file mode 100644 index 0000000000000..f66df6c5de0da --- /dev/null +++ b/tools/aurora-sep/tests/test_ready_pam_hint.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""aurora-sep-ready names the authselect step only when fingerprints are off.""" + +import os +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + +READY = Path(__file__).resolve().parents[1] / "aurora-sep-ready" +COMMAND = "sudo authselect enable-feature with-fingerprint" + + +def function(source, name): + match = re.search(r"^%s\(\) \{\n.*?^\}\n" % name, source, re.M | re.S) + if match is None: + raise AssertionError("no %s() in aurora-sep-ready" % name) + return match.group(0) + + +class PamHintTest(unittest.TestCase): + def hint(self, authselect): + """Runs pam_hint() with `authselect` as the stub's body, or none.""" + source = READY.read_text() + script = function(source, "log") + function(source, "pam_hint") + "pam_hint\n" + with tempfile.TemporaryDirectory() as bindir: + if authselect is not None: + stub = Path(bindir) / "authselect" + stub.write_text("#!/bin/sh\n" + authselect) + stub.chmod(0o755) + env = dict(os.environ, PATH=bindir + ":/usr/bin:/bin") + if authselect is None: + env["PATH"] = bindir + result = subprocess.run( + ["/bin/sh", "-c", script], env=env, capture_output=True, text=True + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + return result.stderr + + def test_off_names_the_command_once(self): + lines = self.hint('echo "local with-silent-lastlog with-mdns4"\n').splitlines() + self.assertEqual(len(lines), 1) + self.assertTrue(lines[0].startswith("aurora-sep-ready: ")) + self.assertTrue(lines[0].endswith(COMMAND)) + + def test_on_is_silent(self): + self.assertEqual(self.hint('echo "local with-fingerprint with-mdns4"\n'), "") + self.assertEqual(self.hint('echo "sssd with-mdns4 with-fingerprint"\n'), "") + + def test_a_similar_feature_is_not_mistaken_for_it(self): + self.assertIn(COMMAND, self.hint('echo "local with-fingerprint-x"\n')) + + def test_no_authselect_configuration_is_silent(self): + self.assertEqual(self.hint('echo "No existing configuration detected." >&2\nexit 2\n'), "") + + def test_no_authselect_is_silent(self): + self.assertEqual(self.hint(None), "") + + +if __name__ == "__main__": + unittest.main() From 8f7e2263a66adfc129eaec9f74b05b783f1d5d60 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 19:46:08 +1100 Subject: [PATCH 1101/1579] drm/apple: Retire an H17P present that the firmware dropped An external H17P display can lose its sink during its first mode set, for instance when a dock is pulled while the display comes up. The firmware fails the mode set after about two seconds, then accepts the next present and drops it, because its pipe is not enabled. It never completes that present, and it reports no abort for it either: DPTXPort: firmware reports link fault 22 on target 0:0 set_digital_out_mode finished:1820 swap not completed, retraining the display IOMFB transaction timed out: cmd=0 oob=0 present=1 accepted=1 swap=162 The swap watchdog already completes the flip for userspace and retrains the display one second after such a present was accepted. Since the H17P commands and presents were serialized, however, a present that was accepted and never completed also holds every command queued behind it: the queue waits until the ten-second transaction timeout declares the coprocessor crashed, and a teardown that waits for the queue to settle in the meantime fails. When the watchdog gives up on such a present, also retire it from the present state, so that the queue moves on. Nothing from the firmware ended the present, so unlike an aborted one it might still complete, and it might still be read. Until the firmware completes a later present, which shows that it is past every earlier one, a completion of any retired present is therefore ignored rather than taken for a protocol error, and the framebuffers of the retired presents, and those they would have displaced, stay referenced. A mode set has to succeed before another present is sent, so this holds at most one present per failed plug; past eight in a row without a completed present, the queue is left to its timeout as before. A watchdog run that finds its present already finished, or a newer present that re-armed it, now does nothing. Other firmware and the H17G method profile have no present state, and their watchdog is unchanged. Add the KUnit suite apple-dcp-dropped-swap, which replays the sequence through the real queue, watchdog and completion callback. Fixes: 5f9574c081d5 ("drm: apple: serialize H17P command and present transactions") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp-internal.h | 12 + drivers/gpu/drm/apple/dcp.c | 13 +- drivers/gpu/drm/apple/dcp_dropped_swap_test.c | 481 ++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 74 ++- drivers/gpu/drm/apple/iomfb_h17p.c | 18 + drivers/gpu/drm/apple/iomfb_h17p.h | 43 ++ drivers/gpu/drm/apple/iomfb_h17p_test.c | 61 +++ drivers/gpu/drm/apple/iomfb_template.c | 12 + 9 files changed, 707 insertions(+), 8 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp_dropped_swap_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 03b08252b99a7..be9ada873650c 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -17,6 +17,7 @@ appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_ready_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_announce_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_relink_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_dropped_swap_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 697d6f9e3c03e..a530ea9173f00 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -40,6 +40,8 @@ struct dcp_method_entry; struct iomfb_scanout_h17p { struct dcp_swap_submit_req_h17p request; struct drm_framebuffer *fb[SWAP_SURFACES]; + /* On iomfb.dropped while the firmware may still read a dropped present. */ + struct list_head link; }; /* One operation, including all of its nested replies and present completion. */ @@ -386,6 +388,8 @@ struct apple_dcp { /* Completes a Type-C swap that DCP dropped, and recovers the pipe. */ struct delayed_work swap_watchdog_wq; unsigned int swap_watchdog_retrains; + /* When the latest arm expires, in jiffies. */ + unsigned long swap_watchdog_deadline; /* List of referenced framebuffers pending a completed replacement swap. * The commit and RTKit work queues share it under this lock. @@ -518,6 +522,11 @@ struct apple_dcp { atomic_t opaque_x_state; struct iomfb_scanout_h17p *scanout; struct iomfb_scanout_h17p *next_scanout; + /* + * Presents the firmware dropped, until it completes a later one: + * at most one per present retired in present_state_h17p. + */ + struct list_head dropped; } iomfb; }; @@ -536,6 +545,7 @@ void iomfb_present_backlight_h17p(struct apple_dcp *dcp); bool iomfb_present_brightness_only_h17p(struct apple_dcp *dcp); bool iomfb_present_complete_h17p(struct apple_dcp *dcp); void iomfb_present_failed_h17p(struct apple_dcp *dcp); +bool iomfb_retire_dropped_present_h17p(struct apple_dcp *dcp); bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, const struct dcp_swap_submit_req_h17p *request, struct dcp_present_h17p *wire); @@ -578,6 +588,8 @@ void dcp_comp_unbind(struct device *dev, struct device *main, void *data); void dcp_platform_shutdown(struct platform_device *pdev); void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable); void dcp_enable_typec_work(struct apple_dcp *dcp); +void dcp_swap_watchdog(struct work_struct *work); +void iomfb_scanout_put_h17p(struct apple_dcp *dcp, struct iomfb_scanout_h17p *scanout); #endif int dcp_backlight_init(struct apple_dcp *dcp, struct device *dev); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 13f540e914fa6..19de6ad77c26f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2277,14 +2277,18 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) * not asserted HPD yet. Userspace would then wait for the flip until the * commit times out, stalling every output it drives. Complete the flip, * mark the mode invalid so later commits do not wait for DCP, and let the - * hotplug worker re-apply the active CRTC. + * hotplug worker re-apply the active CRTC. On H17P the dropped present is + * also retired, so that the commands queued behind it can run. */ -static void dcp_swap_watchdog(struct work_struct *work) +VISIBLE_IF_KUNIT void dcp_swap_watchdog(struct work_struct *work) { struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, swap_watchdog_wq); + if (!iomfb_retire_dropped_present_h17p(dcp)) + return; + dev_warn(dcp->dev, "swap not completed, retraining the display\n"); dcp_mode_invalidate(&dcp->mode_state); dcp_drm_crtc_vblank(dcp->crtc); @@ -2295,9 +2299,12 @@ static void dcp_swap_watchdog(struct work_struct *work) void dcp_swap_watchdog_arm(struct apple_dcp *dcp) { - if (dcp_is_typec_output(dcp)) + if (dcp_is_typec_output(dcp)) { + dcp->swap_watchdog_deadline = jiffies + + msecs_to_jiffies(DCP_SWAP_WATCHDOG_MS); mod_delayed_work(system_wq, &dcp->swap_watchdog_wq, msecs_to_jiffies(DCP_SWAP_WATCHDOG_MS)); + } } void dcp_swap_watchdog_complete(struct apple_dcp *dcp) diff --git a/drivers/gpu/drm/apple/dcp_dropped_swap_test.c b/drivers/gpu/drm/apple/dcp_dropped_swap_test.c new file mode 100644 index 0000000000000..8a16ef2efd9b8 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_dropped_swap_test.c @@ -0,0 +1,481 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "dcp.h" +#include "dcp-internal.h" + +/* + * A dock leaves during the first mode set of an external H17P display. The + * firmware fails the mode set, accepts the next present and drops it without + * a completion or an abort. The teardown that follows waits for the IOMFB + * queue to settle. This replays that sequence through the real queue and + * swap watchdog; the firmware is the test, which answers the present as the + * receive callbacks would and never completes it. + */ +#define DROPPED_SWAP_ID 162 + +struct dropped_test; + +struct dropped_present { + struct iomfb_transaction transaction; + struct dropped_test *t; + struct iomfb_scanout_h17p *scanout; + u32 swap_id; + bool started, released; +}; + +struct dropped_test { + struct apple_dcp dcp; + /* Only its event lock: the watchdog completes the flip. */ + struct drm_device drm; + struct apple_crtc crtc; + struct platform_device *pdev; + /* fb[0] is on screen, and present i shows fb[i + 1]. */ + struct drm_framebuffer fb[4]; + struct dropped_present present[3]; + unsigned int freed; +}; + +static void dropped_test_noop(struct work_struct *work) +{ +} + +static void dropped_test_noop_delayed(struct work_struct *work) +{ +} + +static void dropped_test_fb_free(struct kref *kref) +{ + struct drm_framebuffer *fb = container_of(kref, struct drm_framebuffer, + base.refcount); + struct dropped_test *t = fb->dev ? container_of(fb->dev, struct dropped_test, drm) : NULL; + + if (t) + t->freed++; +} + +static unsigned int dropped_test_refs(struct drm_framebuffer *fb) +{ + return kref_read(&fb->base.refcount); +} + +/* As iomfb_atomic_start() and the accepted submit reply of dcp_swapped(). */ +static void dropped_test_start(struct apple_dcp *dcp, struct iomfb_transaction *transaction) +{ + struct dropped_present *p = container_of(transaction, struct dropped_present, transaction); + + dcp->iomfb.next_scanout = p->scanout; + p->started = dcp_present_begin_h17p(&dcp->present_state_h17p, p->swap_id) && + dcp_present_submit_h17p(&dcp->present_state_h17p, p->swap_id, true); + dcp_swap_watchdog_arm(dcp); +} + +/* As iomfb_atomic_release(). */ +static void dropped_test_release(struct iomfb_transaction *transaction) +{ + struct dropped_present *p = container_of(transaction, struct dropped_present, transaction); + + iomfb_scanout_put_h17p(&p->t->dcp, p->scanout); + p->released = true; +} + +static struct iomfb_scanout_h17p * +dropped_test_scanout(struct kunit *test, struct drm_framebuffer *fb) +{ + struct iomfb_scanout_h17p *scanout = kzalloc_obj(*scanout); + + KUNIT_ASSERT_NOT_NULL(test, scanout); + INIT_LIST_HEAD(&scanout->link); + scanout->fb[0] = fb; + drm_framebuffer_get(fb); + return scanout; +} + +static void dropped_test_scanout_free(struct iomfb_scanout_h17p *scanout) +{ + unsigned int i; + + for (i = 0; i < SWAP_SURFACES; i++) + if (scanout->fb[i]) + drm_framebuffer_put(scanout->fb[i]); + kfree(scanout); +} + +static void dropped_test_exit(void *data) +{ + struct dropped_test *t = data; + struct apple_dcp *dcp = &t->dcp; + struct iomfb_scanout_h17p *scanout, *next; + + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + iomfb_queue_stop(dcp); + guard(mutex)(&dcp->iomfb.lock); + list_for_each_entry_safe(scanout, next, &dcp->iomfb.dropped, link) { + list_del(&scanout->link); + dropped_test_scanout_free(scanout); + } + if (dcp->iomfb.next_scanout && dcp->iomfb.next_scanout != dcp->iomfb.scanout) + dropped_test_scanout_free(dcp->iomfb.next_scanout); + if (dcp->iomfb.scanout) + dropped_test_scanout_free(dcp->iomfb.scanout); +} + +static struct dropped_test *dropped_test_init(struct kunit *test, + enum dcp_firmware_version fw, int profile) +{ + struct dropped_test *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + struct apple_dcp *dcp; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, t); + dcp = &t->dcp; + t->pdev = kunit_platform_device_alloc(test, "dcp-dropped", PLATFORM_DEVID_AUTO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->pdev); + KUNIT_ASSERT_EQ(test, kunit_platform_device_add(test, t->pdev), 0); + platform_set_drvdata(t->pdev, dcp); + dcp->dev = &t->pdev->dev; + dcp->fw_compat = fw; + dcp->hw.iomfb_method_profile = profile; + dcp->fixed_connector_type = DRM_MODE_CONNECTOR_USB; + + iomfb_queue_init(dcp); + mutex_init(&dcp->external_lock); + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->swapped_out_fbs_lock); + INIT_LIST_HEAD(&dcp->swapped_out_fbs); + spin_lock_init(&dcp->mode_state.lock); + spin_lock_init(&dcp->backlight.lock); + INIT_WORK(&dcp->vblank_wq, dropped_test_noop); + INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); + INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, dropped_test_noop_delayed); + INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, dropped_test_noop_delayed); + + spin_lock_init(&t->drm.event_lock); + t->crtc.base.dev = &t->drm; + dcp->crtc = &t->crtc; + dcp->mode_state.valid = true; + + for (i = 0; i < ARRAY_SIZE(t->fb); i++) { + t->fb[i].dev = &t->drm; + kref_init(&t->fb[i].base.refcount); + t->fb[i].base.free_cb = dropped_test_fb_free; + } + /* The display shows fb[0] from a present completed before the plug. */ + dcp->iomfb.scanout = dropped_test_scanout(test, &t->fb[0]); + for (i = 0; i < ARRAY_SIZE(t->present); i++) { + t->present[i].t = t; + t->present[i].swap_id = DROPPED_SWAP_ID + i; + t->present[i].transaction.start = dropped_test_start; + t->present[i].transaction.release = dropped_test_release; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dropped_test_exit, t), 0); + return t; +} + +static struct dropped_test *dropped_test_external(struct kunit *test) +{ + return dropped_test_init(test, DCP_FIRMWARE_H17P, DCP_IOMFB_METHODS_H17P_EXTERNAL); +} + +/* Queue present @i and let the queue send it; the firmware accepts it. */ +static void dropped_test_present(struct kunit *test, struct dropped_test *t, unsigned int i, + struct drm_framebuffer *fb) +{ + struct dropped_present *p = &t->present[i]; + + p->scanout = dropped_test_scanout(test, fb); + KUNIT_ASSERT_EQ(test, iomfb_queue(&t->dcp, &p->transaction), 0); + flush_work(&t->dcp.iomfb.work); + KUNIT_ASSERT_TRUE(test, p->started); +} + +/* As dcpep_cb_swap_complete() for @swap_id on the receive path. */ +static bool dropped_test_complete(struct dropped_test *t, u32 swap_id) +{ + struct apple_dcp *dcp = &t->dcp; + + guard(mutex)(&dcp->iomfb.lock); + if (!dcp_present_complete_h17p(&dcp->present_state_h17p, swap_id)) + return false; + iomfb_present_complete_h17p(dcp); + dcp_swap_watchdog_complete(dcp); + return true; +} + +/* The firmware completes @swap_id: the real receive callback handles it. */ +static int dropped_test_deliver(struct dropped_test *t, u32 swap_id) +{ + guard(mutex)(&t->dcp.iomfb.lock); + return iomfb_h17p_deliver_swap_complete(&t->dcp, swap_id); +} + +/* Present @i is dropped: the watchdog retires it and retrains the display. */ +static void dropped_test_retire(struct kunit *test, struct dropped_test *t, unsigned int i) +{ + dropped_test_present(test, t, i, &t->fb[i + 1]); + KUNIT_ASSERT_EQ(test, iomfb_queue_settle(&t->dcp, msecs_to_jiffies(2000)), 0); + /* The queue settles before the watchdog retrains, outside the lock. */ + flush_delayed_work(&t->dcp.swap_watchdog_wq); + KUNIT_ASSERT_TRUE(test, dcp_present_was_retired_h17p(&t->dcp.present_state_h17p, + t->present[i].swap_id)); +} + +/* + * The captured sequence: the teardown's queue settle times out on the dropped + * present. The swap watchdog retires it a second after it was accepted, so a + * retry settles, the queue moves on and the coprocessor is not declared + * crashed. + */ +static void dropped_present_retired_test(struct kunit *test) +{ + struct dropped_test *t = dropped_test_external(test); + struct apple_dcp *dcp = &t->dcp; + + dropped_test_present(test, t, 0, &t->fb[1]); + KUNIT_EXPECT_EQ(test, iomfb_queue_settle(dcp, msecs_to_jiffies(500)), -ETIMEDOUT); + + KUNIT_EXPECT_EQ(test, iomfb_queue_settle(dcp, msecs_to_jiffies(2000)), 0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->crashed)); + KUNIT_EXPECT_TRUE(test, t->present[0].released); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&dcp->iomfb.timeout)); + KUNIT_EXPECT_FALSE(test, dcp->present_state_h17p.pending); + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&dcp->present_state_h17p, + DROPPED_SWAP_ID)); + /* The firmware did not abort it. */ + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&dcp->present_state_h17p, + DROPPED_SWAP_ID)); + /* + * The display was retrained as before. The queue settles as soon as + * the present is retired; the watchdog retrains after that, outside + * the IOMFB lock. + */ + flush_delayed_work(&dcp->swap_watchdog_wq); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->mode_state.valid)); +} + +/* + * The firmware might still read a dropped present: its framebuffers, and the + * one it would have replaced, stay referenced until a later present + * completes. A late completion of the dropped present releases nothing. + */ +static void dropped_present_buffers_test(struct kunit *test) +{ + struct dropped_test *t = dropped_test_external(test); + struct apple_dcp *dcp = &t->dcp; + + dropped_test_present(test, t, 0, &t->fb[1]); + KUNIT_ASSERT_EQ(test, iomfb_queue_settle(dcp, msecs_to_jiffies(2000)), 0); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[1]), 2); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[0]), 2); + KUNIT_EXPECT_FALSE(test, list_empty(&dcp->iomfb.dropped)); + + KUNIT_EXPECT_FALSE(test, dropped_test_complete(t, DROPPED_SWAP_ID)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&dcp->present_state_h17p, + DROPPED_SWAP_ID)); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[1]), 2); + + /* The next mode set's present is sent and completes. */ + dropped_test_present(test, t, 1, &t->fb[2]); + KUNIT_EXPECT_TRUE(test, dropped_test_complete(t, DROPPED_SWAP_ID + 1)); + KUNIT_EXPECT_TRUE(test, list_empty(&dcp->iomfb.dropped)); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[1]), 1); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[0]), 1); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[2]), 2); + KUNIT_EXPECT_PTR_EQ(test, dcp->iomfb.scanout, t->present[1].scanout); + KUNIT_EXPECT_EQ(test, t->freed, 0); +} + +/* + * A watchdog that was already running when its present completed must not + * retire the next present, which was accepted just now. + */ +static void completed_present_kept_test(struct kunit *test) +{ + struct dropped_test *t = dropped_test_external(test); + struct apple_dcp *dcp = &t->dcp; + + dropped_test_present(test, t, 0, &t->fb[1]); + KUNIT_ASSERT_TRUE(test, dropped_test_complete(t, DROPPED_SWAP_ID)); + /* As iomfb_queue_advance() after the completion. */ + scoped_guard(mutex, &dcp->iomfb.lock) { + dcp->iomfb.active = NULL; + cancel_delayed_work(&dcp->iomfb.timeout); + t->present[0].transaction.release(&t->present[0].transaction); + } + dropped_test_present(test, t, 1, &t->fb[2]); + + dcp_swap_watchdog(&dcp->swap_watchdog_wq.work); + KUNIT_EXPECT_TRUE(test, dcp->present_state_h17p.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&dcp->present_state_h17p, + DROPPED_SWAP_ID + 1)); + KUNIT_EXPECT_FALSE(test, t->present[1].released); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dcp->mode_state.valid)); +} + +/* + * Without a completed present for IOMFB_DROPPED_MAX dropped ones, the next one + * is not retired: the display is retrained as before and the queue is left to + * its timeout. + */ +static void dropped_present_bounded_test(struct kunit *test) +{ + struct dropped_test *t = dropped_test_external(test); + struct apple_dcp *dcp = &t->dcp; + struct dcp_present_state_h17p *state = &dcp->present_state_h17p; + unsigned int i; + + /* As IOMFB_DROPPED_MAX retirements leave the present state and list. */ + for (i = 0; i < IOMFB_DROPPED_MAX; i++) { + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(state, 100 + i)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(state, 100 + i, true)); + KUNIT_ASSERT_TRUE(test, dcp_present_retire_h17p(state, 100 + i)); + list_add_tail(&dropped_test_scanout(test, &t->fb[0])->link, &dcp->iomfb.dropped); + } + dropped_test_present(test, t, 0, &t->fb[1]); + KUNIT_EXPECT_EQ(test, iomfb_queue_settle(dcp, msecs_to_jiffies(1500)), -ETIMEDOUT); + /* The watchdog has given up on the present; let it finish retraining. */ + flush_delayed_work(&dcp->swap_watchdog_wq); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->mode_state.valid)); + KUNIT_EXPECT_TRUE(test, state->pending); + KUNIT_EXPECT_FALSE(test, dcp_present_was_retired_h17p(state, DROPPED_SWAP_ID)); + KUNIT_EXPECT_FALSE(test, t->present[0].released); + KUNIT_EXPECT_EQ(test, list_count_nodes(&dcp->iomfb.dropped), IOMFB_DROPPED_MAX); +} + +/* Other firmware keeps the watchdog it had: no present state is touched. */ +static void dcp_dropped_legacy_watchdog(struct kunit *test, enum dcp_firmware_version fw, + int profile) +{ + struct dropped_test *t = dropped_test_init(test, fw, profile); + struct apple_dcp *dcp = &t->dcp; + + dcp->present_state_h17p.pending = true; + dcp->present_state_h17p.accepted = true; + dcp->swap_watchdog_deadline = jiffies + HZ; + dcp_swap_watchdog(&dcp->swap_watchdog_wq.work); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->mode_state.valid)); + KUNIT_EXPECT_TRUE(test, dcp->present_state_h17p.pending); + KUNIT_EXPECT_FALSE(test, dcp->present_state_h17p.has_last_aborted); + KUNIT_EXPECT_EQ(test, dcp->present_state_h17p.nr_retired, 0U); + dcp->present_state_h17p.pending = false; +} + +static void legacy_watchdog_test(struct kunit *test) +{ + dcp_dropped_legacy_watchdog(test, DCP_FIRMWARE_V_13_5, 0); +} + +/* H17P firmware with the H17G method profile keeps its transport too. */ +static void h17g_watchdog_test(struct kunit *test) +{ + dcp_dropped_legacy_watchdog(test, DCP_FIRMWARE_H17P, DCP_IOMFB_METHODS_H17G); +} + +/* + * Two plugs in a row drop their present: the watchdog retires 162, a mode set + * succeeds, and it retires 163. Both scanouts are kept. The firmware then + * completes 162 after all: a late completion, not an unexpected one. The + * coprocessor is not declared crashed, nothing is released, and the next stop + * finds the queue settled. + */ +static void older_retired_completion_test(struct kunit *test) +{ + struct dropped_test *t = dropped_test_external(test); + struct apple_dcp *dcp = &t->dcp; + + dropped_test_retire(test, t, 0); + /* The recovery mode set succeeds. */ + WRITE_ONCE(dcp->mode_state.valid, true); + dropped_test_retire(test, t, 1); + KUNIT_ASSERT_EQ(test, list_count_nodes(&dcp->iomfb.dropped), 2); + KUNIT_ASSERT_FALSE(test, READ_ONCE(dcp->crashed)); + + KUNIT_ASSERT_EQ(test, dropped_test_deliver(t, DROPPED_SWAP_ID), 0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->crashed)); + KUNIT_EXPECT_EQ(test, list_count_nodes(&dcp->iomfb.dropped), 2); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[0]), 2); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[1]), 2); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[2]), 2); + KUNIT_ASSERT_EQ(test, dropped_test_deliver(t, DROPPED_SWAP_ID + 1), 0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->crashed)); + KUNIT_EXPECT_EQ(test, iomfb_queue_settle(dcp, msecs_to_jiffies(100)), 0); + + /* A completion of a present that was never sent is still unexpected. */ + KUNIT_ASSERT_EQ(test, dropped_test_deliver(t, DROPPED_SWAP_ID - 1), 0); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dcp->crashed)); +} + +/* + * As above, with present 164 of the next mode set in flight when 162 + * completes: that completion neither completes 164 nor stops its watchdog, + * and releases nothing. The completion of 164 shows that the firmware is past + * 162 and 163: their scanouts are released and they are forgotten, so a + * completion of 162 after that is unexpected again. + */ +static void retired_completion_in_flight_test(struct kunit *test) +{ + struct dropped_test *t = dropped_test_external(test); + struct apple_dcp *dcp = &t->dcp; + struct dcp_present_state_h17p *state = &dcp->present_state_h17p; + + dropped_test_retire(test, t, 0); + dropped_test_retire(test, t, 1); + dropped_test_present(test, t, 2, &t->fb[3]); + + KUNIT_ASSERT_EQ(test, dropped_test_deliver(t, DROPPED_SWAP_ID), 0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->crashed)); + KUNIT_EXPECT_TRUE(test, state->pending); + KUNIT_EXPECT_EQ(test, state->swap_id, DROPPED_SWAP_ID + 2); + KUNIT_EXPECT_PTR_EQ(test, dcp->iomfb.active, &t->present[2].transaction); + KUNIT_EXPECT_FALSE(test, t->present[2].transaction.completed); + KUNIT_EXPECT_TRUE(test, delayed_work_pending(&dcp->swap_watchdog_wq)); + KUNIT_EXPECT_EQ(test, list_count_nodes(&dcp->iomfb.dropped), 2); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[1]), 2); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[2]), 2); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[0]), 2); + + KUNIT_ASSERT_EQ(test, dropped_test_deliver(t, DROPPED_SWAP_ID + 2), 0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->crashed)); + KUNIT_EXPECT_FALSE(test, state->pending); + KUNIT_EXPECT_TRUE(test, t->present[2].transaction.completed); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&dcp->swap_watchdog_wq)); + KUNIT_EXPECT_TRUE(test, list_empty(&dcp->iomfb.dropped)); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[1]), 1); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[2]), 1); + KUNIT_EXPECT_EQ(test, dropped_test_refs(&t->fb[0]), 1); + KUNIT_EXPECT_PTR_EQ(test, dcp->iomfb.scanout, t->present[2].scanout); + KUNIT_EXPECT_FALSE(test, dcp_present_was_retired_h17p(state, DROPPED_SWAP_ID)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_retired_h17p(state, DROPPED_SWAP_ID + 1)); + + KUNIT_ASSERT_EQ(test, dropped_test_deliver(t, DROPPED_SWAP_ID), 0); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dcp->crashed)); +} + +static struct kunit_case dropped_swap_cases[] = { + KUNIT_CASE(dropped_present_retired_test), + KUNIT_CASE(dropped_present_buffers_test), + KUNIT_CASE(completed_present_kept_test), + KUNIT_CASE(dropped_present_bounded_test), + KUNIT_CASE(legacy_watchdog_test), + KUNIT_CASE(h17g_watchdog_test), + KUNIT_CASE(older_retired_completion_test), + KUNIT_CASE(retired_completion_in_flight_test), + {} +}; + +static struct kunit_suite dropped_swap_suite = { + .name = "apple-dcp-dropped-swap", + .test_cases = dropped_swap_cases, +}; + +kunit_test_suite(dropped_swap_suite); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index dc96c67735489..3ab3a23fb6684 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ +#include #include #include #include @@ -385,6 +386,7 @@ void iomfb_queue_init(struct apple_dcp *dcp) { mutex_init(&dcp->iomfb.lock); INIT_LIST_HEAD(&dcp->iomfb.pending); + INIT_LIST_HEAD(&dcp->iomfb.dropped); INIT_WORK(&dcp->iomfb.work, iomfb_queue_work); INIT_DELAYED_WORK(&dcp->iomfb.timeout, iomfb_queue_timeout); INIT_DELAYED_WORK(&dcp->iomfb.backlight_retry, iomfb_backlight_retry); @@ -1770,6 +1772,7 @@ iomfb_scanout_prepare_h17p(struct apple_dcp *dcp, struct drm_crtc *crtc, scanout = kzalloc_obj(*scanout); if (!scanout) return NULL; + INIT_LIST_HEAD(&scanout->link); if (dcp->iomfb.scanout) { scanout->request = dcp->iomfb.scanout->request; for (i = 0; i < SWAP_SURFACES; i++) { @@ -1897,8 +1900,14 @@ void iomfb_present_failed_h17p(struct apple_dcp *dcp) bool iomfb_present_complete_h17p(struct apple_dcp *dcp) { struct iomfb_transaction *transaction = dcp->iomfb.active; + struct iomfb_scanout_h17p *dropped, *next; lockdep_assert_held(&dcp->iomfb.lock); + /* The firmware is past every present before this one. */ + list_for_each_entry_safe(dropped, next, &dcp->iomfb.dropped, link) { + list_del(&dropped->link); + iomfb_scanout_release_h17p(dropped); + } if (transaction) { transaction->completed = true; if (transaction->backlight_reserved) { @@ -1914,6 +1923,65 @@ bool iomfb_present_complete_h17p(struct apple_dcp *dcp) return true; } +/* + * The firmware accepts presents while an external pipe is not enabled, for + * instance after a mode set that failed because the sink went away, and then + * drops them: neither a completion nor an abort follows. Such a present + * would hold the queue until its timeout declares the coprocessor crashed, + * and a teardown waiting for the queue to settle would fail. + * + * Called by the swap watchdog: retire the present. The firmware might still + * complete it or read it: until the firmware completes a later present, a + * completion of it is ignored, and the framebuffers of the present and those + * it displaced stay referenced. Without a completed present for + * IOMFB_DROPPED_MAX retired ones, the queue is left to its timeout. Returns + * false if the present ended or a newer one re-armed the watchdog in the + * meantime. Other firmware has no present state. + */ +bool iomfb_retire_dropped_present_h17p(struct apple_dcp *dcp) +{ + struct dcp_present_state_h17p *state = &dcp->present_state_h17p; + u32 swap_id; + + if (!iomfb_uses_queue(dcp)) + return true; + + guard(mutex)(&dcp->iomfb.lock); + swap_id = state->swap_id; + if (time_before(jiffies, dcp->swap_watchdog_deadline) || + !state->pending || !state->accepted) + return false; + if (!dcp_present_retire_h17p(state, swap_id)) + return true; + + iomfb_present_failed_h17p(dcp); + if (!iomfb_present_brightness_only_h17p(dcp)) { + dcp_unarm_retained_framebuffers(dcp, swap_id); + if (dcp->iomfb.next_scanout) { + list_add_tail(&dcp->iomfb.next_scanout->link, + &dcp->iomfb.dropped); + dcp->iomfb.next_scanout = NULL; + } + } + iomfb_queue_advance(dcp); + return true; +} + +/* Release a present's scanout unless it is displayed or was dropped. */ +VISIBLE_IF_KUNIT void iomfb_scanout_put_h17p(struct apple_dcp *dcp, + struct iomfb_scanout_h17p *scanout) +{ + /* A transaction that never started is released without the lock. */ + if (!scanout) + return; + lockdep_assert_held(&dcp->iomfb.lock); + if (scanout == dcp->iomfb.scanout || !list_empty(&scanout->link)) + return; + if (dcp->iomfb.next_scanout == scanout) + dcp->iomfb.next_scanout = NULL; + iomfb_scanout_release_h17p(scanout); +} + static void iomfb_backlight_start(struct apple_dcp *dcp, struct iomfb_transaction *transaction) { @@ -1996,11 +2064,7 @@ static void iomfb_atomic_release(struct iomfb_transaction *transaction) struct iomfb_atomic_transaction *atomic = container_of(transaction, struct iomfb_atomic_transaction, transaction); - if (atomic->scanout && atomic->dcp->iomfb.scanout != atomic->scanout) { - if (atomic->dcp->iomfb.next_scanout == atomic->scanout) - atomic->dcp->iomfb.next_scanout = NULL; - iomfb_scanout_release_h17p(atomic->scanout); - } + iomfb_scanout_put_h17p(atomic->dcp, atomic->scanout); drm_atomic_state_put(atomic->state); kfree(atomic); } diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 6c0b780674206..d0195f2e97d94 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -816,6 +816,24 @@ int iomfb_h17p_first_unbounded_callback(void) return tag; return -1; } + +/* + * Hands a completion of @swap_id to the swap_complete_ap_gated callback, as + * the firmware would. The caller holds the IOMFB lock, as the receiver does. + */ +int iomfb_h17p_deliver_swap_complete(struct apple_dcp *dcp, u32 swap_id) +{ + struct dc_swap_complete_resp_h17p *resp; + + lockdep_assert_held(&dcp->iomfb.lock); + resp = kzalloc_obj(*resp); + if (!resp) + return -ENOMEM; + resp->swap_id = swap_id; + cb_handlers[590](dcp, 590, NULL, resp); + kfree(resp); + return 0; +} #endif #undef DCP_FW_VER diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 75856d1921562..9880e82078178 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -16,6 +16,9 @@ static_assert(sizeof(struct dcp_surface_h17p) == 0x22c); static_assert(sizeof(struct dcp_swap_submit_req_h17p) == 0xe9c); static_assert(sizeof(struct dcp_swap_submit_resp_h17p) == 0x0c); +/* Retired presents kept, with their scanouts, until a later one completes. */ +#define IOMFB_DROPPED_MAX 8 + /* Owned by the RTKit receive thread, independently of DRM state lifetime. */ struct dcp_present_state_h17p { u32 swap_id; @@ -26,6 +29,12 @@ struct dcp_present_state_h17p { /* The most recent aborted present, whose late completion is ignored. */ bool has_last_aborted; u32 last_aborted; + /* + * Presents retired without a completion or an abort, until the firmware + * completes a later one: a late completion of any of them is ignored. + */ + unsigned int nr_retired; + u32 retired[IOMFB_DROPPED_MAX]; }; static inline bool @@ -80,6 +89,37 @@ dcp_present_was_aborted_h17p(const struct dcp_present_state_h17p *state, return state->has_last_aborted && state->last_aborted == swap_id; } +/* + * The swap watchdog retires an accepted present that neither completed nor + * was aborted in time. Nothing from the firmware ended it, so unlike an + * aborted one it may still complete: it is kept until a later present + * completes. Without room for it, the present stays in flight. + */ +static inline bool +dcp_present_retire_h17p(struct dcp_present_state_h17p *state, u32 swap_id) +{ + if (!state->pending || !state->accepted || state->swap_id != swap_id || + state->nr_retired >= IOMFB_DROPPED_MAX) + return false; + state->pending = false; + state->accepted = false; + state->retired[state->nr_retired++] = swap_id; + return true; +} + +/* Whether @swap_id was retired; a completion for it is late, not unexpected. */ +static inline bool +dcp_present_was_retired_h17p(const struct dcp_present_state_h17p *state, + u32 swap_id) +{ + unsigned int i; + + for (i = 0; i < state->nr_retired; i++) + if (state->retired[i] == swap_id) + return true; + return false; +} + static inline bool dcp_present_complete_h17p(struct dcp_present_state_h17p *state, u32 swap_id) { @@ -87,6 +127,8 @@ dcp_present_complete_h17p(struct dcp_present_state_h17p *state, u32 swap_id) return false; state->pending = false; state->accepted = false; + /* The firmware is past every present before this one. */ + state->nr_retired = 0; return true; } @@ -124,6 +166,7 @@ void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, #if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) int iomfb_h17p_first_unbounded_callback(void); +int iomfb_h17p_deliver_swap_complete(struct apple_dcp *dcp, u32 swap_id); #endif #undef DCP_FW_VER diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c index c5955836700ca..8ad8815e98b0c 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p_test.c +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -144,11 +144,72 @@ static void h17p_present_abort_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&state, 9)); } +static void h17p_present_retire_test(struct kunit *test) +{ + struct dcp_present_state_h17p state = {}; + u32 id; + + /* Only an accepted present in flight is retired. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, dcp_present_retire_h17p(&state, 7)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 7, true)); + KUNIT_EXPECT_FALSE(test, dcp_present_retire_h17p(&state, 6)); + KUNIT_EXPECT_TRUE(test, dcp_present_retire_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, state.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 7)); + /* A late completion for it is recognisable; the firmware did not abort it. */ + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&state, 7)); + + /* A second retirement keeps the first. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 8)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 8, true)); + KUNIT_EXPECT_TRUE(test, dcp_present_retire_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&state, 7)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&state, 8)); + + /* An abort is recorded apart from them. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 9)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 9, true)); + KUNIT_EXPECT_TRUE(test, dcp_present_abort_h17p(&state, 9)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_aborted_h17p(&state, 9)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_retired_h17p(&state, 9)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&state, 7)); + + /* A late completion of 7 does not complete present 10 in flight. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 10)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 10, true)); + KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 7)); + KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&state, 7)); + + /* Completing 10 shows the firmware is past 7 and 8; the abort is kept. */ + KUNIT_EXPECT_TRUE(test, dcp_present_complete_h17p(&state, 10)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_retired_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_retired_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_aborted_h17p(&state, 9)); + + /* At most IOMFB_DROPPED_MAX are kept: the next present stays in flight. */ + for (id = 100; id < 100 + IOMFB_DROPPED_MAX; id++) { + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, id)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, id, true)); + KUNIT_ASSERT_TRUE(test, dcp_present_retire_h17p(&state, id)); + } + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, id)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, id, true)); + KUNIT_EXPECT_FALSE(test, dcp_present_retire_h17p(&state, id)); + KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_TRUE(test, dcp_present_was_retired_h17p(&state, 100)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_retired_h17p(&state, id)); + KUNIT_EXPECT_TRUE(test, dcp_present_complete_h17p(&state, id)); +} + static struct kunit_case h17p_present_cases[] = { KUNIT_CASE(h17p_surface_wire_test), KUNIT_CASE(h17p_opaque_x_parameter_test), KUNIT_CASE(h17p_callback_bounds_test), KUNIT_CASE(h17p_present_abort_test), + KUNIT_CASE(h17p_present_retire_test), {} }; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 2193417455131..9a2c626f1b7a6 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -265,6 +265,18 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, resp->swap_id); return; } + /* + * The swap watchdog retired it: its framebuffers stay + * referenced until a later present completes, and a present + * in flight now is a different one. + */ + if (dcp_present_was_retired_h17p(&dcp->present_state_h17p, + resp->swap_id)) { + dev_warn_ratelimited(dcp->dev, + "completion for retired present %u ignored\n", + resp->swap_id); + return; + } dev_err(dcp->dev, "unexpected present completion %u\n", resp->swap_id); dcp->crashed = true; return; From 9a56c642b83f09006ddada51da317f38692bbc8c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 19:48:59 +1100 Subject: [PATCH 1102/1579] drm/apple: Retry an external display stop whose queue did not settle dcp_external_stop_locked() waits up to 500 ms for the IOMFB queue to settle before it asks the firmware to power the external pipe off. If the queue does not settle in time, the error is latched in external_error, which keeps the session blocked: every later SAFE transition of the Type-C route, route deactivation and connection returns it until the driver is rebound. The latch exists so that a retry never races a late reply to a request that timed out. When the queue does not settle, the stop has not sent any request yet, so there is no such reply. This happens when a dock leaves during the first mode set: a present that the firmware dropped keeps the queue busy until the swap watchdog retires it, and the SAFE transition that tipd makes meanwhile latches the stop: set_digital_out_mode finished:1820 sn201202x 1-0c: failed to place Type-C mux in safe mode: -110 external display activation failed: -110 sn201202x 1-0c: failed to place Type-C mux in safe mode: -110 (every later cable event, until reboot) Return such a timeout without latching it. The session stays owned: presentation stays paused, the pipe stays marked enabled and the DPTX port stays held, so the route cannot be deactivated and the Type-C PHY is not touched until a stop succeeds. Then stop again: - a SAFE transition stops the session again, which tipd's next revalidation of the cable does, or the next cable event once those retries are spent; - nothing retries a CRTC that is switched off, so the stop retries itself a second later, after the swap watchdog, at most three times; - an enable of the CRTC on the same cable resumes the deferred session instead of powering a pipe on that was never powered off. A crashed or stopped queue, a failed power-off and anything after the power-off request are still latched. The apple-dcp host selftests that extract dcp_external_stop_locked() get the new fields in their fixture. Fixes: 35f4ccf26053 ("drm/apple: add bounded external stop qualification path") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 4 ++ drivers/gpu/drm/apple/dcp.c | 43 ++++++++++++++++++- drivers/gpu/drm/apple/dcp_dropped_swap_test.c | 43 +++++++++++++++++-- .../drm/apple/dcp_external_announce_test.c | 1 + .../drivers/gpu/apple-dcp/lifecycle.py | 6 ++- 5 files changed, 92 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index a530ea9173f00..313c6688c2c9d 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -255,8 +255,12 @@ struct apple_dcp { bool external_hotplug_pending; bool external_hotplug_connected; bool external_stopped; + /* The last stop sent nothing and stays owned; see external_stop_retry_wq. */ + bool external_stop_deferred; + unsigned int external_stop_retries; int external_error; struct work_struct external_stop_work; + struct delayed_work external_stop_retry_wq; /* firmware version and compatible firmware version */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 19de6ad77c26f..a1bd8825cabf6 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2743,6 +2743,14 @@ static int dcp_external_start_locked(struct apple_dcp *dcp) ret = dcp_external_link_locked(dcp); if (ret || dcp->external_running) return ret; + /* A deferred stop sent nothing: the pipe is still powered. */ + if (dcp->external_stop_deferred) { + dcp->external_stop_deferred = false; + dcp->external_stop_retries = 0; + dcp->external_running = true; + iomfb_pause_presentation(dcp, false); + return 0; + } iomfb_pause_presentation(dcp, true); /* Even a timed-out first activation can have changed firmware state. */ dcp->external_power_attempted = true; @@ -2767,6 +2775,8 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) return dcp_external_link_locked(dcp); } +#define DCP_EXTERNAL_STOP_RETRIES 3 + /* The generation token belongs to the CRTC that was actually enabled. */ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) { @@ -2787,6 +2797,21 @@ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) goto release_link; step = "settling the queue"; ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); + if (ret == -ETIMEDOUT) { + /* + * Nothing was sent, so no late reply can race a retry. The + * session stays owned, but not latched: a SAFE transition, the + * retry work or the next enable of the CRTC takes it from here. + */ + dev_warn(dcp->dev, "external display stop deferred: IOMFB queue busy\n"); + dcp->external_stop_deferred = true; + if (dcp->external_stop_retries++ < DCP_EXTERNAL_STOP_RETRIES) + mod_delayed_work(system_wq, &dcp->external_stop_retry_wq, + msecs_to_jiffies(DCP_SWAP_WATCHDOG_MS)); + return ret; + } + dcp->external_stop_deferred = false; + dcp->external_stop_retries = 0; if (!ret) { step = "powering the pipe off"; ret = iomfb_external_power_h17p(dcp, false); @@ -2857,6 +2882,17 @@ static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) return ret; } +/* Stop a deferred session again once the swap watchdog had its turn. */ +static void dcp_external_stop_retry(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + external_stop_retry_wq); + + guard(mutex)(&dcp->external_lock); + if (dcp->external_stop_deferred) + dcp_external_stop_locked(dcp, dcp->external_generation); +} + static void dcp_external_stop_work(struct work_struct *work) { struct apple_dcp *dcp = container_of(work, struct apple_dcp, external_stop_work); @@ -3099,6 +3135,7 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_delayed_work_sync(&dcp->placeholder_edid_wq); cancel_work_sync(&dcp->external_stop_work); + cancel_delayed_work_sync(&dcp->external_stop_retry_wq); guard(mutex)(&dcp->external_lock); new_cable = !dcp->external_cable; if (new_cable) @@ -3453,6 +3490,7 @@ void dcp_poweron(struct platform_device *pdev) cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_delayed_work_sync(&dcp->placeholder_edid_wq); cancel_work_sync(&dcp->external_stop_work); + cancel_delayed_work_sync(&dcp->external_stop_retry_wq); guard(mutex)(&dcp->external_lock); ret = dcp_external_start_locked(dcp); if (!ret) @@ -3555,7 +3593,7 @@ void dcp_poweroff(struct platform_device *pdev) cancel_delayed_work_sync(&dcp->placeholder_edid_wq); guard(mutex)(&dcp->external_lock); ret = dcp_external_stop_locked(dcp, dcp->external_crtc_generation); - if (ret) + if (ret && !dcp->external_stop_deferred) dev_err(dcp->dev, "external display stop failed: %d\n", ret); dcp_external_crtc_released(dcp); return; @@ -4117,6 +4155,7 @@ VISIBLE_IF_KUNIT void dcp_disable_typec_work(struct apple_dcp *dcp, bool release disable_delayed_work_sync(&dcp->placeholder_edid_wq); disable_delayed_work_sync(&dcp->announce_wq); cancel_work_sync(&dcp->external_stop_work); + cancel_delayed_work_sync(&dcp->external_stop_retry_wq); } VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) @@ -4368,6 +4407,7 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v scoped_guard(mutex, &dcp->iomfb.lock) WRITE_ONCE(dcp->quiescing, true); cancel_work_sync(&dcp->external_stop_work); + cancel_delayed_work_sync(&dcp->external_stop_retry_wq); if (dcp->crtc && !dcp->drm_retained) { drm_dev_get(dcp->crtc->base.dev); dcp->drm_retained = true; @@ -4523,6 +4563,7 @@ static int dcp_platform_probe(struct platform_device *pdev) mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->external_lock); INIT_WORK(&dcp->external_stop_work, dcp_external_stop_work); + INIT_DELAYED_WORK(&dcp->external_stop_retry_wq, dcp_external_stop_retry); mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->metadata.lock); diff --git a/drivers/gpu/drm/apple/dcp_dropped_swap_test.c b/drivers/gpu/drm/apple/dcp_dropped_swap_test.c index 8a16ef2efd9b8..b45010d38fac1 100644 --- a/drivers/gpu/drm/apple/dcp_dropped_swap_test.c +++ b/drivers/gpu/drm/apple/dcp_dropped_swap_test.c @@ -18,9 +18,9 @@ * A dock leaves during the first mode set of an external H17P display. The * firmware fails the mode set, accepts the next present and drops it without * a completion or an abort. The teardown that follows waits for the IOMFB - * queue to settle. This replays that sequence through the real queue and - * swap watchdog; the firmware is the test, which answers the present as the - * receive callbacks would and never completes it. + * queue to settle. This replays that sequence through the real queue, swap + * watchdog and external stop; the firmware is the test, which answers the + * present as the receive callbacks would and never completes it. */ #define DROPPED_SWAP_ID 162 @@ -118,6 +118,7 @@ static void dropped_test_exit(void *data) struct iomfb_scanout_h17p *scanout, *next; cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + cancel_delayed_work_sync(&dcp->external_stop_retry_wq); iomfb_queue_stop(dcp); guard(mutex)(&dcp->iomfb.lock); list_for_each_entry_safe(scanout, next, &dcp->iomfb.dropped, link) { @@ -159,6 +160,8 @@ static struct dropped_test *dropped_test_init(struct kunit *test, INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, dropped_test_noop_delayed); INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, dropped_test_noop_delayed); + /* A retried stop would send the power-off, which this test cannot answer. */ + INIT_DELAYED_WORK(&dcp->external_stop_retry_wq, dropped_test_noop_delayed); spin_lock_init(&t->drm.event_lock); t->crtc.base.dev = &t->drm; @@ -322,6 +325,39 @@ static void completed_present_kept_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, READ_ONCE(dcp->mode_state.valid)); } +/* + * The teardown of the captured sequence: a queue that does not settle before + * the power-off is sent leaves no firmware request outstanding. The stop is + * not latched and keeps everything owned; once the watchdog has retired the + * dropped present, a retry finds the queue settled. + */ +static void stop_deferred_test(struct kunit *test) +{ + struct dropped_test *t = dropped_test_external(test); + struct apple_dcp *dcp = &t->dcp; + + dcp->external_prepared = true; + dcp->external_cable = true; + dcp->external_power_attempted = true; + dcp->external_running = true; + dcp->dptxport[0].connected = true; + dcp->pipe_enabled_h17p = true; + dropped_test_present(test, t, 0, &t->fb[1]); + + dcp_poweroff(t->pdev); + KUNIT_EXPECT_EQ(test, dcp->external_error, 0); + KUNIT_EXPECT_FALSE(test, dcp->external_stopped); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dcp->pipe_enabled_h17p)); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dcp->dptxport[0].connected)); + KUNIT_EXPECT_TRUE(test, dcp->iomfb.presentation_paused); + /* Nothing else retries a CRTC that is off: the stop retries itself. */ + KUNIT_EXPECT_TRUE(test, dcp->external_stop_deferred); + KUNIT_EXPECT_TRUE(test, delayed_work_pending(&dcp->external_stop_retry_wq)); + + KUNIT_EXPECT_EQ(test, iomfb_queue_settle(dcp, msecs_to_jiffies(2000)), 0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->crashed)); +} + /* * Without a completed present for IOMFB_DROPPED_MAX dropped ones, the next one * is not retired: the display is retrained as before and the queue is left to @@ -465,6 +501,7 @@ static struct kunit_case dropped_swap_cases[] = { KUNIT_CASE(dropped_present_retired_test), KUNIT_CASE(dropped_present_buffers_test), KUNIT_CASE(completed_present_kept_test), + KUNIT_CASE(stop_deferred_test), KUNIT_CASE(dropped_present_bounded_test), KUNIT_CASE(legacy_watchdog_test), KUNIT_CASE(h17g_watchdog_test), diff --git a/drivers/gpu/drm/apple/dcp_external_announce_test.c b/drivers/gpu/drm/apple/dcp_external_announce_test.c index 58a0c1247acd8..6dd9792541af2 100644 --- a/drivers/gpu/drm/apple/dcp_external_announce_test.c +++ b/drivers/gpu/drm/apple/dcp_external_announce_test.c @@ -110,6 +110,7 @@ static struct announce_test *announce_test_init(struct kunit *test) INIT_DELAYED_WORK(&dcp->bl_fallback_wq, announce_test_noop); INIT_DELAYED_WORK(&dcp->backlight.fade_work, announce_test_noop); INIT_WORK(&dcp->external_stop_work, announce_test_noop); + INIT_DELAYED_WORK(&dcp->external_stop_retry_wq, announce_test_noop); INIT_WORK(&dcp->vblank_wq, announce_test_noop); INIT_WORK(&t->con.hotplug_wq, announce_test_event); mutex_init(&t->drm.mode_config.mutex); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index b580b1a978e3a..608e7c1872202 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -84,7 +84,9 @@ def function(file, name): #define DCP_HOTPLUG_NOTIFY 1 struct work_struct { bool pending; }; struct delayed_work { struct work_struct work; }; -static int system_freezable_wq; +static int system_freezable_wq, system_wq; +#define DCP_SWAP_WATCHDOG_MS 1000 +#define DCP_EXTERNAL_STOP_RETRIES 3 static int schedule_work(struct work_struct *w) { w->pending = true; return 1; } static void cancel_work_sync(struct work_struct *w) { w->pending = false; } static void cancel_delayed_work_sync(struct delayed_work *w) { w->work.pending = false; } @@ -101,6 +103,8 @@ def function(file, name): struct delayed_work typec_reconnect_wq, placeholder_edid_wq; struct work_struct external_stop_work; bool external_running, external_stopped, external_power_attempted; + bool external_stop_deferred; unsigned int external_stop_retries; + struct delayed_work external_stop_retry_wq; bool external_cable, typec_cable_connected, typec_crtc_off, pipe_enabled_h17p; bool external_physically_absent, external_prepared, placeholder_retried; int typec_reconnect_tries; From 96adf952fcc9dfb4014c899e94ea26685fb629d4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 19:48:59 +1100 Subject: [PATCH 1103/1579] selftests: apple-dcp: Replay a dock that leaves during the first mode set Add dropped-swap-teardown, with tipd's actual update and retry work and the actual external-session functions of drm/apple, against an IOMFB queue that a present the firmware dropped keeps busy until the swap watchdog retires it, or until the transaction timeout declares the coprocessor crashed: - captured: tipd's out-of-band disconnect and its SAFE find the queue busy. The stop is deferred, not latched, and the session stays owned with nothing sent. The next revalidation stops the session, SAFE reaches the PHY and the next plug connects. - retired-during-retry: the watchdog retires the present while the retry waits for the queue. - never-retired: nothing retires the present. The session stays owned through all of tipd's retries. Once the coprocessor is declared crashed, the stop latches and the next plug is refused. - after-power-off: a settle that times out after the power-off request still latches and holds the session. - crashed-queue: a crashed queue still latches. - crtc-off: the same deferral for a CRTC switched off on the same cable. - stop-retry: the deferred stop retries itself, and stops retrying after three deferrals; the session stays owned. - enable-after-deferral: enabling the CRTC on the same cable resumes the session without a second power-on; with the cable gone it is refused. The SAFE prepare of the DCP's Type-C route is modelled by its H17P branch; quiesce-before-safe runs the real one. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/README | 5 +- .../drivers/gpu/apple-dcp/lifecycle.py | 190 ++++++++++++++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 2 + 3 files changed, 196 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index 1f5e3cc8cd68d..33373ea702c30 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -30,7 +30,10 @@ The modules contain public synthetic host boundaries and retained assertions: warm reset and legacy T8103/T8122/T8140/T8142 route controls. * lifecycle: actual physical-disconnect authority and ownership despite failed teardown; a fast dock replug whose DPTX link never configures (withdrawn, - unacknowledged, unanswered, refused and live failures); DWC3/core/PHY PM + unacknowledged, unanswered, refused and live failures); a dock that leaves + during the first mode set, whose dropped present keeps the IOMFB queue busy + (deferred and retried stop, never-retired, post-power-off and crashed-queue + latches); DWC3/core/PHY PM and reset-reference recovery; exact firmware tuple/resource admission, unsupported VRR rejection and power reply bounds; 15,552 D134 dispatcher scope/length/context and legacy reply cases. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 608e7c1872202..db72bc324ee11 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -543,6 +543,196 @@ def swap(old, new): ''' return unit + firmware + raw + '\n' + tests +def dropped(root): + """A dock leaves during the first mode set, and the next swap is dropped. + + The physical-disconnect fixture with tipd's actual update and retry work + and the actual external stop, against an IOMFB queue that a dropped + present keeps busy until the swap watchdog retires it, or until the + transaction timeout declares the coprocessor crashed. Time is the + queue's own clock, advanced by the waits of the stop and of tipd. + """ + unit = disconnect(root) + dcp = (root / 'drivers/gpu/drm/apple/dcp.c').read_text() + extra = block(dcp, dcp.index('static int dcp_external_start_locked(')) + # Without a retry of its own, a deferred stop of a CRTC that is off waits. + retry = 'static void dcp_external_stop_retry(' + extra += '\n' + (block(dcp, dcp.index(retry)) if retry in dcp else + retry + 'struct work_struct *work) { (void)work; }') + + def swap(old, new): + nonlocal unit + assert unit.count(old) == 1, old + unit = unit.replace(old, new) + + swap('static int iomfb_queue_settle(struct apple_dcp *d, int t)\n' + '{ (void)d; (void)t; settles++; return settle_error; }', + '/* Milliseconds since the firmware accepted the dropped present. */\n' + 'static int now, retire_at, crash_at, after_power_error;\n' + 'static bool pending, crashed, powered;\n' + 'static int iomfb_queue_settle(struct apple_dcp *d, int t)\n' + '{ (void)d; settles++; if (settle_error) return settle_error;\n' + ' if (powered) { powered = false; if (after_power_error) return after_power_error; }\n' + ' if (pending && retire_at >= 0 && now + t >= retire_at) {\n' + ' if (now < retire_at) now = retire_at; pending = false; }\n' + ' if (pending && now + t >= crash_at) { if (now < crash_at) now = crash_at; crashed = true; }\n' + ' if (crashed) return -EIO;\n' + ' if (pending) { now += t; return -ETIMEDOUT; }\n' + ' return 0; }') + swap('static int iomfb_external_power_h17p(struct apple_dcp *d, bool on)\n' + '{ (void)d; assert(!on); powers++; return power_error; }', + 'static int iomfb_external_power_h17p(struct apple_dcp *d, bool on)\n' + '{ (void)d; if (on) { power_ons++; return 0; }\n' + ' assert(!pending); powers++; powered = !power_error; return power_error; }') + swap('static int settle_error,', 'static int power_ons, settle_error,') + # The H17P branch of dcp_typec_route_prepare(), which quiesce-before-safe + # runs for real inside typec_mux_set(). + swap('static int typec_set_mode(void *p, int s)\n' + '{ (void)p; assert(s == TYPEC_STATE_SAFE); safes++; return safe_error; }', + 'static int phy_safes;\n' + 'static int typec_set_mode(void *p, int s)\n' + '{ int r; (void)p; assert(s == TYPEC_STATE_SAFE); safes++;\n' + ' if (display.external_error) return display.external_error;\n' + ' if (display.external_cable || !display.external_stopped || display.dptxport[0].connected) {\n' + ' r = dcp_external_disconnect(&display, true); if (r) return r; }\n' + ' phy_safes++; return 0; }') + swap('int main(void)\n{', 'static int __attribute__((unused)) prior_main(void)\n{') + tests = r''' +static struct platform_device device = { .data = &display }; +/* The dock has left; the firmware accepted a present it will not complete. */ +static void left(int retire) +{ + init(); now = 0; retire_at = retire; crash_at = 10000; after_power_error = 0; + pending = true; crashed = powered = false; phy_safes = power_ons = 0; +} +/* tipd's revalidation of the cable after its debounce, as resume_work. */ +static bool revalidate(void) +{ + now += CD321X_DEBOUNCE_DELAY_MS; + if (!cd321x_pm_begin_read(&port.pm)) return false; + cd321x_pm_snapshot_ready(&port.pm); cd321x_update_work(&port.update_work.work); + return true; +} +/* Nothing of the session has been released, and nothing was sent. */ +static void owned(void) +{ + CHECK(!display.external_stopped && display.dptxport[0].connected); + CHECK(display.pipe_enabled_h17p && !display.typec_crtc_off); + CHECK(display.iomfb.presentation_paused && !display.external_running); + CHECK(powers == 0 && releases == 0 && phy_safes == 0); +} +int main(void) +{ + /* + * The capture: tipd's out-of-band disconnect and then its SAFE stop the + * session while the dropped present is in flight. Each settle times out + * after 500 ms, before the power-off is sent: not latched. The watchdog + * retires the present, and tipd's next revalidation stops the session and + * reaches the PHY. + */ + left(1500); + cd321x_update_work(&port.update_work.work); + CHECK(safes == 1 && settles == 2 && now == 1000); + CHECK(!display.external_error && !display.external_cable && !sink.connected); + owned(); + CHECK(port.pm.phase == CD321X_PM_REVALIDATE && !port.state_valid); + CHECK(revalidate()); + CHECK(safes == 2 && phy_safes == 1 && powers == 1 && releases == 1); + CHECK(!display.external_error && display.external_stopped); + CHECK(!display.dptxport[0].connected && !display.pipe_enabled_h17p && display.typec_crtc_off); + CHECK(port.state_valid && port.pm.phase == CD321X_PM_RUNNING); + /* The next plug connects. */ + CHECK(dcp_dptx_connect_oob(&device, 0) == 0); + CHECK(display.dptxport[0].connected && display.external_cable && !display.external_error); + printf("case captured complete\n"); + + /* The watchdog retires the present during the retry's settle. */ + left(1900); + cd321x_update_work(&port.update_work.work); + CHECK(!display.external_error); owned(); + CHECK(revalidate() && phy_safes == 1 && display.external_stopped && now == 1900); + printf("case retired-during-retry complete\n"); + + /* + * Nothing retires the present. The stop is retried within tipd's budget + * and the session stays owned throughout; once the transaction timeout + * declares the coprocessor crashed, the stop latches and the next plug is + * refused: the protected case. + */ + left(-1); + cd321x_update_work(&port.update_work.work); + while (revalidate()) + ; + CHECK(safes == 1 + CD321X_RESUME_ATTEMPTS && port.pm.phase == CD321X_PM_STALE); + CHECK(!display.external_error); owned(); + now = 12000; + CHECK(cd321x_pm_event(&port.pm) && revalidate()); + CHECK(display.external_error == -EIO); owned(); + CHECK(dcp_dptx_connect_oob(&device, 0) == -EIO && raw_links == 0); + printf("case never-retired complete\n"); + + /* A settle after the power-off was sent still latches and holds. */ + left(0); pending = false; after_power_error = -ETIMEDOUT; + cd321x_update_work(&port.update_work.work); + CHECK(powers == 1 && display.external_error == -ETIMEDOUT); + CHECK(display.dptxport[0].connected && display.pipe_enabled_h17p && !display.external_stopped); + CHECK(revalidate() && phy_safes == 0 && powers == 1 && releases == 0); + printf("case after-power-off complete\n"); + + /* A crashed or stopped queue is no proof that DMA stopped: latched. */ + left(1000); settle_error = -EIO; + cd321x_update_work(&port.update_work.work); + CHECK(display.external_error == -EIO); settle_error = 0; + CHECK(revalidate() && phy_safes == 0 && powers == 0); + printf("case crashed-queue complete\n"); + + /* The same deferral for a CRTC switched off on the same cable. */ + left(1000); + CHECK(dcp_external_disconnect(&display, false) == -ETIMEDOUT); + CHECK(!display.external_error && display.external_cable); owned(); + now = 1000; + CHECK(dcp_external_disconnect(&display, false) == 0 && display.external_stopped && powers == 1); + printf("case crtc-off complete\n"); + + /* Nothing else retries a CRTC that is off: the stop retries itself, bounded. */ + left(1000); + CHECK(dcp_external_disconnect(&display, false) == -ETIMEDOUT); + CHECK(display.external_stop_deferred && display.external_stop_retry_wq.work.pending); + now = 1000; display.external_stop_retry_wq.work.pending = false; + dcp_external_stop_retry(&display.external_stop_retry_wq.work); + CHECK(display.external_stopped && !display.external_stop_deferred && powers == 1); + CHECK(!display.external_stop_retry_wq.work.pending && !display.external_stop_retries); + left(-1); crash_at = 1 << 30; + for (int i = 0; i <= DCP_EXTERNAL_STOP_RETRIES; i++) { + display.external_stop_retry_wq.work.pending = false; + dcp_external_stop_retry(&display.external_stop_retry_wq.work); + if (!i) CHECK(dcp_external_disconnect(&display, false) == -ETIMEDOUT); + } + CHECK(!display.external_stop_retry_wq.work.pending && !display.external_error); owned(); + printf("case stop-retry complete\n"); + + /* + * The CRTC is enabled again on the same cable before any stop went + * through: the pipe was never powered off, so it is resumed, not powered + * on a second time. + */ + left(1000); + CHECK(dcp_external_disconnect(&display, false) == -ETIMEDOUT); owned(); + CHECK(dcp_external_start_locked(&display) == 0); + CHECK(power_ons == 0 && powers == 0 && display.external_running); + CHECK(!display.iomfb.presentation_paused && !display.external_stop_deferred); + CHECK(display.pipe_enabled_h17p && display.dptxport[0].connected); + /* Once the cable is gone, the deferred session is not resumed. */ + left(1000); + CHECK(dcp_external_disconnect(&display, true) == -ETIMEDOUT); + CHECK(dcp_external_start_locked(&display) == -ENODEV && power_ons == 0); owned(); + printf("case enable-after-deferral complete\n"); + + printf("%d assertions, %d failures\n", checks, failures); return !!failures; +} +''' + return unit + extra + '\n' + tests + def teardown(root): files = {n: (root / f).read_text() for n, f in { diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index 775d67ecf4337..cffd27caa688f 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -79,6 +79,8 @@ def main(): ('-Wno-unused-variable', '-Wno-unused-function', '-Wno-unused-but-set-variable')), ('lifecycle', 'fast-replug-bounce', lifecycle.bounce, ((),), ('-Wno-unused-variable', '-Wno-unused-function', '-Wno-unused-but-set-variable')), + ('lifecycle', 'dropped-swap-teardown', lifecycle.dropped, ((),), + ('-Wno-unused-variable', '-Wno-unused-function', '-Wno-unused-but-set-variable')), ('lifecycle', 'dwc3-teardown', lifecycle.teardown, (('all',),), ()), ('lifecycle', 'resource-profile', lifecycle.profile, ((),), ('-Wno-sign-compare',)), ('lifecycle', 'firmware-provider', lifecycle.firmware, ((),), ()), From 90324ce24af9d4974e68b62ed0ad2e0fc465ee08 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 20:09:20 +1100 Subject: [PATCH 1104/1579] drm/apple: Drop the mode of an external display whose link faults The DPTX endpoint of the external H17P display reports a sink that stops answering, a busy timeout or a link that did not start through its DEVICE_NOT_RESPONDING, DEVICE_BUSY_TIMEOUT and DEVICE_NOT_STARTED callbacks. The driver only revoked the 180 Hz rate proof there. The mode stayed valid, so a link that faulted after its mode set, without an HPD change, stayed reported connected with link-status GOOD and nothing set its mode again; and a fault during the mode set that recovers the link was undone when the mode set completed. On the external H17P display, treat such a fault as the end of the mode: invalidate it, which a mode set in flight then reports as failed when it finishes, and queue the hotplug worker when a set mode was lost, so that it sets the mode of the active CRTC again or reports the display lost. Before any mode was set there is nothing to recover and nothing is queued. dcp_mode_invalidate() now says whether the mode was valid. The 180 Hz admission conditions are unchanged. Other profiles keep their behaviour. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 16 +++++ .../gpu/drm/apple/dcp_external_relink_test.c | 66 +++++++++++++++++++ drivers/gpu/drm/apple/dptxep.c | 2 +- drivers/gpu/drm/apple/iomfb-state.h | 9 ++- 5 files changed, 92 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 313c6688c2c9d..7916e39b564b4 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -574,6 +574,7 @@ bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected); void dcp_announce_arm(struct apple_dcp *dcp); void dcp_announce_work(struct work_struct *work); void dcp_external_crtc_released(struct apple_dcp *dcp); +void dcp_external_link_fault(struct apple_dcp *dcp); /* A KUnit test can wait here for the work it races with. */ static inline void dcp_test_point(struct apple_dcp *dcp, enum dcp_test_point point) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a1bd8825cabf6..e4d727d02c880 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2313,6 +2313,22 @@ void dcp_swap_watchdog_complete(struct apple_dcp *dcp) dcp->swap_watchdog_retrains = 0; } +/* + * The firmware reports that the DisplayPort sink stopped answering, or that + * the link did not start. On the external H17P display, the mode set + * before, or being set, has no link any more: drop it, also across a mode + * set in flight, and have the hotplug worker set the mode of the CRTC again + * or report the display lost (dcp_external_relink()). A mode set still in + * flight reports this itself when it finishes, and before any mode was set + * there is nothing to recover. + */ +void dcp_external_link_fault(struct apple_dcp *dcp) +{ + dcp_180hz_invalidate(dcp); + if (dcp_is_external_h17p(dcp) && dcp_mode_invalidate(&dcp->mode_state)) + dcp_handle_hotplug_actions(dcp, DCP_HOTPLUG_NOTIFY); +} + static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) { struct apple_dcp *dcp = cookie; diff --git a/drivers/gpu/drm/apple/dcp_external_relink_test.c b/drivers/gpu/drm/apple/dcp_external_relink_test.c index 15c22218127f7..5691404f417b3 100644 --- a/drivers/gpu/drm/apple/dcp_external_relink_test.c +++ b/drivers/gpu/drm/apple/dcp_external_relink_test.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -55,6 +56,7 @@ struct relink_test { bool bounce_on_probe; /* the route bounces again during a probe */ bool real_modeset; /* atomic_enable is the real dcp_crtc_atomic_modeset() */ bool force_modeset; /* a userspace commit changes the connectors */ + bool fault_in_call; /* the link faults before the firmware answers */ u32 reply; /* what the firmware answers a call with */ int modeset_ret; /* the result of the real mode set */ struct dcp_display_mode catalog; @@ -129,7 +131,10 @@ static void relink_test_push(struct apple_dcp *dcp, const struct dcp_method_entr dcp_callback_t cb, void *cookie) { struct relink_test *t = container_of(dcp, struct relink_test, dcp); + bool modeset = !strcmp(call->name, "dcpep_set_digital_out_mode"); + if (modeset && t->fault_in_call) + dcp_external_link_fault(dcp); cb(dcp, &t->reply, cookie); } @@ -823,6 +828,64 @@ static void relink_lost_quiet_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); } +/* + * The link faults while the recovery mode set waits for the firmware, which + * then answers: the mode set did not bring the link up. + */ +static void relink_fault_during_modeset_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + relink_test_real(t); + t->fault_in_call = true; + relink_test_tear(t); + relink_test_reconnect(test, t); + KUNIT_EXPECT_FALSE(test, relink_test_link_up(t)); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_BAD); +} + +/* + * The link faults after a completed mode set, with no hotplug: the mode is + * gone at once, and the worker sets it again. + */ +static void relink_fault_after_modeset_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + int modesets; + + relink_test_real(t); + relink_test_tear(t); + relink_test_reconnect(test, t); + KUNIT_ASSERT_EQ(test, t->modeset_ret, 0); + KUNIT_ASSERT_TRUE(test, relink_test_link_up(t)); + /* See the fault's effect before the worker it queues can run. */ + disable_work_sync(&t->con.hotplug_wq); + modesets = atomic_read(&t->modesets); + dcp_external_link_fault(&t->dcp); + KUNIT_EXPECT_FALSE(test, relink_test_link_up(t)); + KUNIT_EXPECT_TRUE(test, t->dcp.mode_state.invalidated); + enable_work(&t->con.hotplug_wq); + relink_test_hotplug(t); + relink_test_settle(test, t); + KUNIT_EXPECT_GT(test, atomic_read(&t->modesets), modesets); + KUNIT_EXPECT_TRUE(test, relink_test_link_up(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_GOOD); +} + +/* A fault with no mode set queues nothing: there is nothing to recover. */ +static void relink_fault_inactive_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + KUNIT_ASSERT_EQ(test, relink_test_commit(t, false, false), 0); + relink_test_settle(test, t); + disable_work_sync(&t->con.hotplug_wq); + dcp_external_link_fault(&t->dcp); + KUNIT_EXPECT_FALSE(test, work_pending(&t->con.hotplug_wq)); + enable_work(&t->con.hotplug_wq); +} + static struct kunit_case relink_cases[] = { KUNIT_CASE(relink_bounce_test), KUNIT_CASE(relink_mode_gone_test), @@ -838,6 +901,9 @@ static struct kunit_case relink_cases[] = { KUNIT_CASE(relink_early_epipe_test), KUNIT_CASE(relink_paused_kept_test), KUNIT_CASE(relink_lost_quiet_test), + KUNIT_CASE(relink_fault_during_modeset_test), + KUNIT_CASE(relink_fault_after_modeset_test), + KUNIT_CASE(relink_fault_inactive_test), {} }; diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 2f0b4fad04642..d0cc7bdab714d 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -813,7 +813,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_DEVICE_NOT_RESPONDING: case DPTX_APCALL_DEVICE_BUSY_TIMEOUT: case DPTX_APCALL_DEVICE_NOT_STARTED: - dcp_180hz_invalidate(service->ep->dcp); + dcp_external_link_fault(service->ep->dcp); dev_warn(service->ep->dcp->dev, "DPTXPort: firmware reports link fault %u on target %u:%u\n", idx, service->ep->dcp->dptx_die, diff --git a/drivers/gpu/drm/apple/iomfb-state.h b/drivers/gpu/drm/apple/iomfb-state.h index a9ada00a8528b..4cf48977ffd74 100644 --- a/drivers/gpu/drm/apple/iomfb-state.h +++ b/drivers/gpu/drm/apple/iomfb-state.h @@ -32,14 +32,21 @@ static inline void dcp_mode_begin(struct dcp_mode_state *state) spin_unlock_irqrestore(&state->lock, flags); } -static inline void dcp_mode_invalidate(struct dcp_mode_state *state) +/* + * The mode is gone, also for a mode set in flight, which then does not + * complete it. Returns whether a set mode was valid until now. + */ +static inline bool dcp_mode_invalidate(struct dcp_mode_state *state) { unsigned long flags; + bool valid; spin_lock_irqsave(&state->lock, flags); + valid = state->valid; state->valid = false; state->invalidated = true; spin_unlock_irqrestore(&state->lock, flags); + return valid; } /* The caller holds state->lock. */ From f2e5e5330dee2f64e13d3ebbdfbf44fbf48e2d6e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 20:09:20 +1100 Subject: [PATCH 1105/1579] drm/apple: Fail an external mode set that gets no reply The completion of set_digital_out_mode ignored its reply, so a mode set that the transport completed without one counted as a success. On the external H17P display, fail such a mode set with -EIO. The 32-bit result itself is kept and logged when it is nonzero, but not acted on: its meaning is not established for H17P. The firmware's own log shows a nonzero result for a mode set that failed after the sink went away, and on other firmware within 0.1 ms for internal-panel requests that repeat the previous one or come first after boot. A link that does not come up is reported through the DPTX fault callbacks instead. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../gpu/drm/apple/dcp_external_relink_test.c | 37 ++++++++++++++++++- drivers/gpu/drm/apple/iomfb_template.c | 22 +++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp_external_relink_test.c b/drivers/gpu/drm/apple/dcp_external_relink_test.c index 5691404f417b3..02d6d7580e57e 100644 --- a/drivers/gpu/drm/apple/dcp_external_relink_test.c +++ b/drivers/gpu/drm/apple/dcp_external_relink_test.c @@ -57,6 +57,7 @@ struct relink_test { bool real_modeset; /* atomic_enable is the real dcp_crtc_atomic_modeset() */ bool force_modeset; /* a userspace commit changes the connectors */ bool fault_in_call; /* the link faults before the firmware answers */ + bool no_reply; /* the firmware answers a call with no reply */ u32 reply; /* what the firmware answers a call with */ int modeset_ret; /* the result of the real mode set */ struct dcp_display_mode catalog; @@ -135,7 +136,7 @@ static void relink_test_push(struct apple_dcp *dcp, const struct dcp_method_entr if (modeset && t->fault_in_call) dcp_external_link_fault(dcp); - cb(dcp, &t->reply, cookie); + cb(dcp, modeset && t->no_reply ? NULL : &t->reply, cookie); } static void relink_test_crtc_enable(struct drm_crtc *crtc, struct drm_atomic_state *state) @@ -886,6 +887,38 @@ static void relink_fault_inactive_test(struct kunit *test) enable_work(&t->con.hotplug_wq); } +/* A mode set the firmware gives no reply to has failed. */ +static void relink_no_reply_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + relink_test_real(t); + t->no_reply = true; + relink_test_tear(t); + relink_test_reconnect(test, t); + KUNIT_EXPECT_LT(test, t->modeset_ret, 0); + KUNIT_EXPECT_FALSE(test, relink_test_link_up(t)); + KUNIT_EXPECT_FALSE(test, relink_test_connected(t)); + KUNIT_EXPECT_EQ(test, t->con.base.state->link_status, DRM_LINK_STATUS_BAD); +} + +/* + * A nonzero result alone is not a failure: its meaning is not established, + * and it is also seen for a mode set that repeats the current mode. + */ +static void relink_nonzero_reply_test(struct kunit *test) +{ + struct relink_test *t = relink_test_init(test); + + relink_test_real(t); + t->reply = 1; + relink_test_tear(t); + relink_test_reconnect(test, t); + KUNIT_EXPECT_EQ(test, t->modeset_ret, 0); + KUNIT_EXPECT_TRUE(test, relink_test_link_up(t)); + KUNIT_EXPECT_TRUE(test, relink_test_connected(t)); +} + static struct kunit_case relink_cases[] = { KUNIT_CASE(relink_bounce_test), KUNIT_CASE(relink_mode_gone_test), @@ -904,6 +937,8 @@ static struct kunit_case relink_cases[] = { KUNIT_CASE(relink_fault_during_modeset_test), KUNIT_CASE(relink_fault_after_modeset_test), KUNIT_CASE(relink_fault_inactive_test), + KUNIT_CASE(relink_no_reply_test), + KUNIT_CASE(relink_nonzero_reply_test), {} }; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 9a2c626f1b7a6..af1f612dbe702 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -44,6 +44,7 @@ struct dcp_wait_cookie { struct kref refcount; struct completion done; u32 status; + bool replied; }; static void release_wait_cookie(struct kref *ref) @@ -2225,6 +2226,10 @@ static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, struct dcp_wait_cookie *wait = cookie; if (wait) { + if (data) { + WRITE_ONCE(wait->status, *(u32 *)data); + WRITE_ONCE(wait->replied, true); + } complete(&wait->done); kref_put(&wait->refcount, release_wait_cookie); } @@ -2353,6 +2358,23 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, "set_digital_out_mode finished with %d to spare\n", jiffies_to_msecs(ret)); } + /* + * The external H17P display reports a link that does not come up + * through its DPTX fault callbacks. What the 32-bit result of the + * mode set means is not established: a nonzero one is seen both for + * a mode set that failed and for one repeating the current mode. Log + * it, and fail only a mode set that got no reply at all. + */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { + if (!READ_ONCE(cookie->replied)) { + dev_err(dcp->dev, "set_digital_out_mode got no reply\n"); + kref_put(&cookie->refcount, release_wait_cookie); + return -EIO; + } + if (READ_ONCE(cookie->status)) + dev_info(dcp->dev, "set_digital_out_mode returned %#x\n", + READ_ONCE(cookie->status)); + } kref_put(&cookie->refcount, release_wait_cookie); dcp->vrr_enabled = mode->vrr && crtc_state->vrr_enabled; From 3f7970de6b41e28e17c38e356cdc6f6e5fd36ad2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 22:59:46 +1100 Subject: [PATCH 1106/1579] soc: apple: sep: Say why the sensor status after registration was unusable After a sensor registration the status is read again before the patch acknowledgment is trusted. When that read is unusable the bring-up stops with "status unavailable after registration", which covers two different failures: the SPI transfer failed, or the sensor answered with an all-zero (silent) status. They point at different causes, the bus or the sensor, and the line does not say which, nor how long after power-on it happened. Name which one it was and the time since the sensor was powered on, as the other registration steps already do. The start of the message is unchanged, so existing log searches still match. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 45c290f5523e5..95d4adbbc96ce 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2017,8 +2017,21 @@ impl SepData { // acknowledgement sampled before CLEAR_STATE and authentication. let st = match sensor::status() { Ok(st) if !st.is_silent() => st, - _ => { - dev_warn!(self.dev, "sensor: status unavailable after registration\n"); + Ok(_) => { + dev_warn!( + self.dev, + "sensor: status unavailable after registration: silent {} ms after power-on\n", + self.sensor_ms() + ); + return None; + } + Err(e) => { + dev_warn!( + self.dev, + "sensor: status unavailable after registration: read failed ({:?}) {} ms after power-on\n", + e, + self.sensor_ms() + ); return None; } }; From af1c264b6816b07cce2943a99bc6f91d0f10b37f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 23:01:33 +1100 Subject: [PATCH 1107/1579] soc: apple: sep: Retry a failed sensor bring-up before a verify Every verify powers the sensor up and registers it with the enclave again before it captures. The first verify after a system suspend on a MacBook Neo failed that way: the re-registration completed, the status read straight after it was unusable, and the verify ended with "the sensor did not come back online". libfprint reports that as an error, so fprintd ends the verify and the lock screen drops its fingerprint prompt although the sensor works. The first registration after boot fails in the same ways now and then, and commit 6c394131cf7e ("soc: apple: sep: Retry a failed attach-time sensor bring-up") recovers it by starting again from a power-cycled sensor, which clears the enclave's registration before the new one. The verify path never got that retry, and the first verify after a suspend is where a failed re-registration has been seen: in the kernel logs on record, re-registrations that did not follow a suspend have not failed, and one of the four that first followed one failed like this. Before commit 4db9a3896199 ("soc: apple: recheck sensor status after SEP registration") the same verify fell to state 9 instead. Move attach's bounded retry into a helper and use it for a verify too, on the same nonce-session profiles. A verify asks before each new attempt whether it is still wanted, so a cancel, a close or a suspend does not wait for further attempts. Other profiles keep their single attempt. Attempts cost tens of milliseconds each and are logged. Authentication is unchanged: each attempt still needs the enclave to accept the session commit, and the match itself is the same. tools/aurora-sep/tests/test_verify_bringup.py runs the verify and attach code with mocks; it fails without this change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 71 +++++-- tools/aurora-sep/tests/test_verify_bringup.py | 196 ++++++++++++++++++ 2 files changed, 246 insertions(+), 21 deletions(-) create mode 100644 tools/aurora-sep/tests/test_verify_bringup.py diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 95d4adbbc96ce..92d7fb73f65b8 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1078,30 +1078,40 @@ impl SepData { Ok(()) } - fn attach_bringup(&self) { - if self.profile.nonce_session { - // A bring-up that fails on the bus leaves nothing a new attempt - // cannot undo: each attempt power-cycles the sensor and, after a - // failed one, clears the enclave's partial registration first, - // as every verify does. - const ATTEMPTS: u32 = 3; - for attempt in 1..=ATTEMPTS { - if self.bring_sensor_online() { - return; - } - if attempt < ATTEMPTS { - dev_warn!( - self.dev, - "sensor: attach-time bring-up attempt {} of {} failed; starting again\n", - attempt, - ATTEMPTS - ); - } + /// Brings the sensor up, starting again after a failed attempt for as + /// long as `wanted` says the result is still needed. + /// + /// A bring-up that fails on the bus leaves nothing a new attempt cannot + /// undo: each attempt power-cycles the sensor and, after a failed one, + /// clears the enclave's partial registration first, as every verify does. + fn bring_sensor_online_retrying(&self, purpose: &CStr, wanted: impl Fn() -> bool) -> bool { + const ATTEMPTS: u32 = 3; + for attempt in 1..=ATTEMPTS { + if self.bring_sensor_online() { + return true; + } + if attempt == ATTEMPTS || !wanted() { + break; } dev_warn!( self.dev, - "sensor: authenticated attach incomplete; capture unavailable\n" + "sensor: {} bring-up attempt {} of {} failed; starting again\n", + purpose, + attempt, + ATTEMPTS ); + } + false + } + + fn attach_bringup(&self) { + if self.profile.nonce_session { + if !self.bring_sensor_online_retrying(c"attach-time", || true) { + dev_warn!( + self.dev, + "sensor: authenticated attach incomplete; capture unavailable\n" + ); + } return; } if !sensor::is_bound() { @@ -1375,6 +1385,25 @@ impl SepData { self.refresh_match_credential(); } + /// Whether the verify this work item runs is still wanted: not cancelled, + /// not ended by a suspend and not closed. + fn verify_wanted(&self) -> bool { + !self.shutting_down.load(Relaxed) + && bio::verify_is_live(&self.bio_session.lock(), self.capture_epoch.load(Relaxed)) + } + + // Every verify registers the sensor again. On a profile with a nonce + // session that registration can fail as the attach-time one does, most + // often the first one after a system suspend, and a new attempt recovers + // it in the same way. Without one, the failure ends the user's + // authentication even though the sensor works. + fn bring_sensor_online_for_verify(&self) -> bool { + if !self.profile.nonce_session { + return self.bring_sensor_online(); + } + self.bring_sensor_online_retrying(c"verify", || self.verify_wanted()) + } + pub(crate) fn run_verify(&self) { self.retry_restore_proof(); if !self.templates_restored.load(Relaxed) { @@ -1401,7 +1430,7 @@ impl SepData { self.settle_before_capture(); // The enclave refuses a capture from an uncalibrated sensor (0x65 answers 1). - if !self.bring_sensor_online() { + if !self.bring_sensor_online_for_verify() { dev_err!(self.dev, "verify: the sensor did not come back online\n"); self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); let _ = sensor::idle(); diff --git a/tools/aurora-sep/tests/test_verify_bringup.py b/tools/aurora-sep/tests/test_verify_bringup.py new file mode 100644 index 0000000000000..a4e3f4572d0f1 --- /dev/null +++ b/tools/aurora-sep/tests/test_verify_bringup.py @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: MIT +"""Execute the actual verify and attach sensor bring-up code with mocks. + +The first verify after a system suspend re-registers the sensor, and on a +nonce-session profile that registration can fail in a way a new attempt from +a power-cycled sensor recovers. Attach makes up to three attempts; a verify +must too, but stop as soon as the verify is no longer wanted. +""" +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + +APPLE = Path(__file__).resolve().parents[3] / 'drivers/soc/apple' + + +def method(source, name): + """The text of method `name` in `source`, or '' if there is none.""" + match = re.search(r'\n (pub\(crate\) )?fn ' + name + r'\b', source) + if not match: + return '' + opening = source.index('{', match.end()) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[match.start():end] + + +PRELUDE = r''' +#![allow(dead_code, unused_variables, unused_imports)] +use std::cell::RefCell; +use std::ffi::CStr; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering::Relaxed}; +macro_rules! dev_err { ($($args:tt)*) => {} } +macro_rules! dev_warn { ($($args:tt)*) => {} } +macro_rules! dev_info { ($($args:tt)*) => {} } + +const ENROL_STATUS_SENSOR: u32 = 1; +const ENROL_STATUS_ENCLAVE: u32 = 2; +const SBIO_PROBE_USER_ID: u32 = 1; + +#[derive(Default)] +struct Mock { + /// Outcomes of successive bring-up attempts; missing ones fail. + script: Vec, + attempts: usize, + /// The verify stops being wanted once this many attempts have run. + cancel_after: Option, + captured: bool, + finished: Option, +} +thread_local! { static MOCK: RefCell = RefCell::new(Mock::default()); } + +mod bio { + pub const TOKEN_LEN: usize = 4; + #[derive(Clone, Copy, Debug, PartialEq)] + pub enum VerifyOutcome { Matched(u8), NoMatch, Failed(u32) } + pub struct Session; + pub fn verify_is_live(_: &Session, _: u64) -> bool { + super::MOCK.with(|m| { + let m = m.borrow(); + m.cancel_after.map_or(true, |n| m.attempts < n) + }) + } +} +mod sbio { + pub struct UserId; + impl UserId { pub fn new(_: u32) -> Option { Some(UserId) } } +} +mod sensor { + pub fn idle() -> Result<(), ()> { Ok(()) } + pub fn power(_: bool) -> bool { true } + pub fn is_bound() -> bool { true } +} +mod diag { + pub fn sensor_bind() -> u32 { 0 } + pub fn sensor_brought_up(_: u32, _: bool) {} +} +struct PatchLoaded(()); +struct Profile { nonce_session: bool } +struct SessionLock(bio::Session); +impl SessionLock { fn lock(&self) -> &bio::Session { &self.0 } } + +struct Driver { + dev: (), + profile: Profile, + templates_restored: AtomicBool, + shutting_down: AtomicBool, + capture_epoch: AtomicU64, + bio_session: SessionLock, +} + +impl Driver { + fn new(nonce_session: bool) -> Self { + Driver { + dev: (), + profile: Profile { nonce_session }, + templates_restored: AtomicBool::new(true), + shutting_down: AtomicBool::new(false), + capture_epoch: AtomicU64::new(1), + bio_session: SessionLock(bio::Session), + } + } + fn bring_sensor_online(&self) -> bool { + MOCK.with(|m| { + let mut m = m.borrow_mut(); + let ok = m.script.get(m.attempts).copied().unwrap_or(false); + m.attempts += 1; + ok + }) + } + fn retry_restore_proof(&self) {} + fn mint_token_bytes(&self) -> Option<[u8; bio::TOKEN_LEN]> { Some([1; bio::TOKEN_LEN]) } + fn settle_before_capture(&self) {} + fn verify_one_image(&self) -> bio::VerifyOutcome { + MOCK.with(|m| m.borrow_mut().captured = true); + bio::VerifyOutcome::Matched(1) + } + fn note_capture_end(&self) {} + fn save_after_match(&self, _: sbio::UserId) -> bool { true } + fn finish_verify(&self, outcome: bio::VerifyOutcome, _: [u8; bio::TOKEN_LEN]) { + MOCK.with(|m| { + let mut m = m.borrow_mut(); + assert!(m.finished.is_none(), "the verify reported twice"); + m.finished = Some(outcome); + }); + } + fn wake_sensor(&self) -> Option { Some(PatchLoaded(())) } + fn complete_bringup(&self, _: PatchLoaded) -> bool { true } +''' + +MAIN = r''' +} + +fn run(script: &[bool], cancel_after: Option, nonce: bool, verify: bool) -> Mock { + MOCK.with(|m| *m.borrow_mut() = Mock { script: script.to_vec(), cancel_after, ..Mock::default() }); + let driver = Driver::new(nonce); + if verify { driver.run_verify() } else { driver.attach_bringup() } + MOCK.with(|m| std::mem::take(&mut *m.borrow_mut())) +} + +fn main() { + use bio::VerifyOutcome::*; + + // The case seen after a suspend: the first attempt fails, the next works. + let m = run(&[false, true], None, true, true); + assert!(m.captured && m.finished == Some(Matched(1)) && m.attempts == 2, + "a verify gave up after one failed sensor bring-up (attempts {}, finished {:?})", + m.attempts, m.finished); + + // A sensor that never comes up: three attempts, then a sensor failure. + let m = run(&[], None, true, true); + assert_eq!((m.attempts, m.captured, m.finished), (3, false, Some(Failed(ENROL_STATUS_SENSOR)))); + + // Cancelled or suspending during the first attempt: no second one. + let m = run(&[false, true], Some(1), true, true); + assert_eq!((m.attempts, m.captured, m.finished), (1, false, Some(Failed(ENROL_STATUS_SENSOR)))); + + // A first attempt that works is the only one. + let m = run(&[true], None, true, true); + assert_eq!((m.attempts, m.captured, m.finished), (1, true, Some(Matched(1)))); + + // Profiles without a nonce session keep their single attempt. + let m = run(&[false, true], None, false, true); + assert_eq!((m.attempts, m.captured, m.finished), (1, false, Some(Failed(ENROL_STATUS_SENSOR)))); + + // Attach keeps its three attempts. + assert_eq!(run(&[false, false, true], None, true, false).attempts, 3); + assert_eq!(run(&[], None, true, false).attempts, 3); + assert_eq!(run(&[true], None, true, false).attempts, 1); + assert_eq!(run(&[], None, false, false).attempts, 0); +} +''' + + +class VerifyBringupTest(unittest.TestCase): + def test_verify_retries_a_failed_bring_up_like_attach(self): + source = (APPLE / 'sbio.rs').read_text() + methods = [method(source, name) for name in ( + 'run_verify', 'attach_bringup', + 'bring_sensor_online_retrying', 'bring_sensor_online_for_verify', + 'verify_wanted')] + self.assertTrue(methods[0] and methods[1]) + program = PRELUDE + '\n'.join(m for m in methods if m) + MAIN + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'test.rs').write_text(program) + subprocess.run(['rustc', '--edition=2021', str(root / 'test.rs'), + '-o', str(root / 'test')], check=True) + subprocess.run([str(root / 'test')], check=True) + + +if __name__ == '__main__': + unittest.main() From 7e97f2f5ee4553790b00af41e36a9aa4c480ffe4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 21:09:11 +1100 Subject: [PATCH 1108/1579] usb: typec: tipd: cd321x: Never leave interrupt events unacknowledged The CD321x and SN201202x controllers latch their events in INT_EVENT1 until the host writes them to INT_CLEAR1, and raise their interrupt only when an event is latched while none were pending. When cd321x_interrupt() fails to read the latched events it returns without acknowledging them, so the controller never interrupts again, and the port ignores every later plug, power and Alternate Mode change until the driver is rebound. A failed write to INT_CLEAR1 does the same, and a failed status read after the acknowledgment drops the change behind the event. On the J700 the rear-port SN201202x hits this after s2idle: its event handler runs while the controller is still in SPMI sleep, the register select times out, "cd321x_interrupt: failed to read events" is logged, and the port stays dead for the rest of the boot. The resume revalidation reads a fresh snapshot but leaves the latched events alone, so it does not recover the port either. Acknowledge latched events at the start of every snapshot read, which covers them, and have an interrupt that could not read, acknowledge or apply its events schedule such a read with a full retry budget. A port whose earlier recovery gave up is included, since no further interrupt can arrive to reopen it. Failures during suspend or probe need nothing extra, because both end with a snapshot read. A failed read says nothing about the cable, so it does not force a reconnect. Keep the acknowledgment apart from the bounded budget for reading and applying the cable state. While the events stay latched, no interrupt can restart a recovery that gave up, so a snapshot read that fails to acknowledge them is not charged to that budget and never gives up: it is retried after a delay that doubles from 500 ms to at most 30 s. Sleep cancels the retry and the resume read takes it over, and removal ends it. Only the first failure of a series is logged. Add KUnit tests that model the controller's event latch behind a regmap and drive the interrupt handler and the snapshot worker. They check that the next event still raises an interrupt after a failed event read, clear or status read, after a failed read on a port that had given up, and after events latched across a sleep cycle; also when the event reads or clears keep failing long after the cable-state budget has run out, including across a sleep cycle, before the bus recovers. They also check that removal stops the retries. Without this change, all of them fail except the one where the interrupt succeeds. State machine tests check the retry delay and that the acknowledgment leaves the cable-state budget alone. Fixes: 45188f27b3d0 ("usb: typec: tipd: Add support for Apple CD321X") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/Kconfig | 4 + drivers/usb/typec/tipd/Makefile | 3 +- drivers/usb/typec/tipd/cd321x-irq-test.c | 349 +++++++++++++++++++++++ drivers/usb/typec/tipd/cd321x-pm-test.c | 108 +++++++ drivers/usb/typec/tipd/cd321x-pm.h | 47 +++ drivers/usb/typec/tipd/core.c | 73 ++++- drivers/usb/typec/tipd/tps6598x.h | 1 + 7 files changed, 575 insertions(+), 10 deletions(-) create mode 100644 drivers/usb/typec/tipd/cd321x-irq-test.c diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index aae3e81cca848..7c2527913111d 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -31,11 +31,15 @@ config TYPEC_SN201202X config TYPEC_TIPD_KUNIT_TEST tristate "KUnit tests for CD321x" if !KUNIT_ALL_TESTS depends on KUNIT && TYPEC_TPS6598X_CORE + select REGMAP default KUNIT_ALL_TESTS help Test suspend gating, fresh cable-state revalidation, bounded retries and removal during resume. The tests exercise the state transitions shared with the driver, including aborted and repeated sleep cycles. + Also test that interrupt events which could not be read or cleared + are still acknowledged, however long that keeps failing, so that the + controller keeps raising its interrupt. Also test that a USB4/Thunderbolt router which never registers its switch does not keep the port from probing. These tests run without accessing a physical Type-C controller. diff --git a/drivers/usb/typec/tipd/Makefile b/drivers/usb/typec/tipd/Makefile index 198d57654b577..e2e3d9271dd26 100644 --- a/drivers/usb/typec/tipd/Makefile +++ b/drivers/usb/typec/tipd/Makefile @@ -12,4 +12,5 @@ tps6598x-y := i2c.o obj-$(CONFIG_TYPEC_SN201202X) += sn201202x.o sn201202x-y := spmi.o -obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o cd321x-tbt-test.o +obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o cd321x-tbt-test.o \ + cd321x-irq-test.o diff --git a/drivers/usb/typec/tipd/cd321x-irq-test.c b/drivers/usb/typec/tipd/cd321x-irq-test.c new file mode 100644 index 0000000000000..e3df8bc1f4bf6 --- /dev/null +++ b/drivers/usb/typec/tipd/cd321x-irq-test.c @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for CD321x interrupt event acknowledgment + */ + +#include +#include +#include +#include + +#include "tps6598x.h" + +#define TEST_REG_INT_EVENT1 0x14 +#define TEST_REG_INT_CLEAR1 0x18 +#define TEST_REG_STATUS 0x1a + +/* + * The controller latches events in INT_EVENT1 until they are written to + * INT_CLEAR1, and raises its interrupt only when an event is latched while + * none were pending. + */ +struct cd321x_test_ctrl { + u64 latched; + unsigned int interrupts; + unsigned int failing_event_reads; + unsigned int failing_clear_writes; + unsigned int failing_status_reads; +}; + +struct cd321x_test { + struct cd321x cd; + struct cd321x_test_ctrl ctrl; +}; + +static void cd321x_test_raise(struct cd321x_test_ctrl *ctrl, u64 event) +{ + if (!ctrl->latched) + ctrl->interrupts++; + ctrl->latched |= event; +} + +static int cd321x_test_read(void *context, const void *reg, size_t reg_size, + void *val, size_t val_size) +{ + struct cd321x_test_ctrl *ctrl = context; + + memset(val, 0, val_size); + switch (*(const u8 *)reg) { + case TEST_REG_INT_EVENT1: + if (ctrl->failing_event_reads) { + ctrl->failing_event_reads--; + return -ETIMEDOUT; + } + memcpy(val, &ctrl->latched, min(val_size, sizeof(ctrl->latched))); + break; + case TEST_REG_STATUS: + if (ctrl->failing_status_reads) { + ctrl->failing_status_reads--; + return -ETIMEDOUT; + } + break; + } + + return 0; +} + +static int cd321x_test_write(void *context, const void *data, size_t count) +{ + struct cd321x_test_ctrl *ctrl = context; + const u8 *buf = data; + u64 clear = 0; + + if (buf[0] == TEST_REG_INT_CLEAR1) { + if (ctrl->failing_clear_writes) { + ctrl->failing_clear_writes--; + return -ETIMEDOUT; + } + memcpy(&clear, buf + 1, min(count - 1, sizeof(clear))); + ctrl->latched &= ~clear; + } + + return 0; +} + +static const struct regmap_bus cd321x_test_bus = { + .read = cd321x_test_read, + .write = cd321x_test_write, + .reg_format_endian_default = REGMAP_ENDIAN_NATIVE, + .val_format_endian_default = REGMAP_ENDIAN_NATIVE, +}; + +static void cd321x_test_stop(void *data) +{ + struct cd321x *cd = data; + + scoped_guard(mutex, &cd->tps.lock) + cd321x_pm_remove(&cd->pm); + cancel_delayed_work_sync(&cd->resume_work); + cancel_delayed_work_sync(&cd->update_work); +} + +static struct cd321x_test *cd321x_test_port(struct kunit *test) +{ + struct cd321x_test *t; + struct device *dev; + + dev = kunit_device_register(test, "cd321x-irq-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + + mutex_init(&t->cd.tps.lock); + t->cd.tps.dev = dev; + t->cd.tps.data = &tipd_cd321x_data; + t->cd.tps.regmap = devm_regmap_init(dev, &cd321x_test_bus, &t->ctrl, + &tps6598x_regmap_config); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->cd.tps.regmap); + dev_set_drvdata(dev, &t->cd.tps); + + cd321x_init_work(&t->cd); + t->cd.pm.phase = CD321X_PM_RUNNING; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, cd321x_test_stop, &t->cd), 0); + + return t; +} + +static irqreturn_t cd321x_test_interrupt(struct cd321x_test *t) +{ + return t->cd.tps.data->irq_handler(0, &t->cd.tps); +} + +/* The read of a snapshot that was scheduled to recover the events. */ +static void cd321x_test_recover(struct kunit *test, struct cd321x_test *t) +{ + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + flush_delayed_work(&t->cd.resume_work); + scoped_guard(mutex, &t->cd.tps.lock) + KUNIT_EXPECT_TRUE(test, cd321x_pm_can_update(&t->cd.pm)); +} + +/* The time until the next recovery read, which must be scheduled. */ +static unsigned long cd321x_test_retry_delay(struct kunit *test, + struct cd321x_test *t) +{ + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + return t->cd.resume_work.timer.expires - jiffies; +} + +/* + * Keep the events from being acknowledged, by failing every access counted by + * @failures, for much longer than the cable-state budget lasts. Once the bus + * works again, the events must still be acknowledged, so that the next cable + * event raises an interrupt. + */ +static void cd321x_test_ack_outage(struct kunit *test, struct cd321x_test *t, + unsigned int *failures) +{ + unsigned long first, delay = 0; + unsigned int i; + + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + *failures = UINT_MAX; + cd321x_test_interrupt(t); + first = cd321x_test_retry_delay(test, t); + + for (i = 0; i < 4 * CD321X_RESUME_ATTEMPTS; i++) { + flush_delayed_work(&t->cd.resume_work); + KUNIT_EXPECT_NE(test, t->cd.pm.phase, CD321X_PM_STALE); + delay = cd321x_test_retry_delay(test, t); + } + /* Retried more and more slowly, but still at least once a minute. */ + KUNIT_EXPECT_GE(test, delay, 8 * first); + KUNIT_EXPECT_LE(test, delay, msecs_to_jiffies(60 * MSEC_PER_SEC)); + KUNIT_EXPECT_EQ(test, t->ctrl.interrupts, 1U); + + *failures = 0; + cd321x_test_recover(test, t); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + KUNIT_EXPECT_EQ(test, t->ctrl.interrupts, 2U); + KUNIT_EXPECT_EQ(test, cd321x_test_interrupt(t), IRQ_HANDLED); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); +} + +static void cd321x_irq_acknowledges_events_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + KUNIT_EXPECT_EQ(test, cd321x_test_interrupt(t), IRQ_HANDLED); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); +} + +static void cd321x_failed_event_read_recovers_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + t->ctrl.failing_event_reads = 1; + cd321x_test_interrupt(t); + KUNIT_ASSERT_NE(test, t->ctrl.latched, 0); + + cd321x_test_recover(test, t); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); + /* The next cable event must reach the host again. */ + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + KUNIT_EXPECT_EQ(test, t->ctrl.interrupts, 2U); +} + +static void cd321x_failed_status_read_recovers_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + t->ctrl.failing_status_reads = 1; + cd321x_test_interrupt(t); + /* Acknowledged, but the cable state behind the event was not read. */ + KUNIT_ASSERT_EQ(test, t->ctrl.latched, 0); + + cd321x_test_recover(test, t); +} + +static void cd321x_failed_read_after_giving_up_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + t->cd.pm.phase = CD321X_PM_STALE; + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + t->ctrl.failing_event_reads = 1; + cd321x_test_interrupt(t); + + cd321x_test_recover(test, t); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); +} + +static void cd321x_failed_clear_recovers_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + t->ctrl.failing_clear_writes = 1; + cd321x_test_interrupt(t); + KUNIT_ASSERT_NE(test, t->ctrl.latched, 0); + + cd321x_test_recover(test, t); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + KUNIT_EXPECT_EQ(test, t->ctrl.interrupts, 2U); +} + +static void cd321x_event_reads_failing_for_long_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + cd321x_test_ack_outage(test, t, &t->ctrl.failing_event_reads); +} + +static void cd321x_clear_writes_failing_for_long_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + cd321x_test_ack_outage(test, t, &t->ctrl.failing_clear_writes); +} + +static void cd321x_ack_retry_across_sleep_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + struct device *dev = t->cd.tps.dev; + unsigned int i; + + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + t->ctrl.failing_event_reads = UINT_MAX; + cd321x_test_interrupt(t); + flush_delayed_work(&t->cd.resume_work); + + /* Sleep cancels the retry, and the resume read takes it over. */ + KUNIT_ASSERT_EQ(test, tipd_prepare(dev), 0); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + tipd_complete(dev); + for (i = 0; i < 2 * CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + flush_delayed_work(&t->cd.resume_work); + } + + t->ctrl.failing_event_reads = 0; + cd321x_test_recover(test, t); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + KUNIT_EXPECT_EQ(test, t->ctrl.interrupts, 2U); +} + +static void cd321x_ack_retry_ends_on_removal_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + t->ctrl.failing_event_reads = UINT_MAX; + cd321x_test_interrupt(t); + flush_delayed_work(&t->cd.resume_work); + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + + scoped_guard(mutex, &t->cd.tps.lock) + cd321x_pm_remove(&t->cd.pm); + /* A retry that was already queued neither reads nor queues another. */ + flush_delayed_work(&t->cd.resume_work); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_EQ(test, t->ctrl.failing_event_reads, UINT_MAX - 2); +} + +static void cd321x_resume_acknowledges_events_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + struct device *dev = t->cd.tps.dev; + + KUNIT_ASSERT_EQ(test, tipd_prepare(dev), 0); + /* Latched while the interrupt was off, and never delivered. */ + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_POWER_STATUS_UPDATE); + tipd_complete(dev); + + cd321x_test_recover(test, t); + KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); + KUNIT_EXPECT_EQ(test, t->ctrl.interrupts, 2U); +} + +static struct kunit_case cd321x_irq_cases[] = { + KUNIT_CASE(cd321x_irq_acknowledges_events_test), + KUNIT_CASE(cd321x_failed_event_read_recovers_test), + KUNIT_CASE(cd321x_failed_status_read_recovers_test), + KUNIT_CASE(cd321x_failed_read_after_giving_up_test), + KUNIT_CASE(cd321x_failed_clear_recovers_test), + KUNIT_CASE(cd321x_event_reads_failing_for_long_test), + KUNIT_CASE(cd321x_clear_writes_failing_for_long_test), + KUNIT_CASE(cd321x_ack_retry_across_sleep_test), + KUNIT_CASE(cd321x_ack_retry_ends_on_removal_test), + KUNIT_CASE(cd321x_resume_acknowledges_events_test), + {} +}; + +static struct kunit_suite cd321x_irq_suite = { + .name = "cd321x-irq", + .test_cases = cd321x_irq_cases, +}; + +kunit_test_suite(cd321x_irq_suite); +MODULE_DESCRIPTION("KUnit tests for CD321x interrupt event acknowledgment"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index f94f4ad57cb75..977df770dcff5 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -417,6 +417,109 @@ static void cd321x_busy_provider_keeps_real_failure_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, pm.force_reconnect); } +static void cd321x_lost_event_rereads_test(struct kunit *test) +{ + static const enum cd321x_pm_phase phases[] = { + CD321X_PM_RUNNING, CD321X_PM_REVALIDATE, + CD321X_PM_APPLY, CD321X_PM_STALE, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(phases); i++) { + struct cd321x_pm_state pm = { .phase = phases[i] }; + + /* Even an exhausted recovery must read the latched events. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_event_lost(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_REVALIDATE); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); + KUNIT_EXPECT_TRUE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_EQ(test, pm.attempts_left, CD321X_RESUME_ATTEMPTS - 1U); + } +} + +static void cd321x_lost_event_left_to_resume_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_init(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_event_lost(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_INITIALIZING); + KUNIT_ASSERT_TRUE(test, cd321x_pm_ready(&pm)); + + /* A read failing during suspend is redone by the resume read. */ + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_event_lost(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_PREPARED); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_EXPECT_TRUE(test, cd321x_pm_begin_read(&pm)); + + cd321x_pm_remove(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_event_lost(&pm)); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_REMOVED); +} + +static void cd321x_lost_event_keeps_reconnect_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + KUNIT_ASSERT_TRUE(test, cd321x_pm_retry(&pm, true)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_event_lost(&pm)); + /* A failed read is no evidence either way about the cable. */ + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); +} + +static void cd321x_unacked_events_outlast_budget_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int i, delay = 0; + + KUNIT_ASSERT_TRUE(test, cd321x_pm_event_lost(&pm)); + for (i = 0; i < 4 * CD321X_RESUME_ATTEMPTS; i++) { + unsigned int next; + + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + next = cd321x_pm_ack_failed(&pm); + /* Never given up, and not charged to the cable-state budget... */ + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_REVALIDATE); + KUNIT_EXPECT_EQ(test, pm.attempts_left, (unsigned int)CD321X_RESUME_ATTEMPTS); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + /* ...only retried less and less often, down to a limit. */ + KUNIT_EXPECT_EQ(test, next, i ? min(2 * delay, CD321X_ACK_RETRY_MAX_MS) : + CD321X_ACK_RETRY_MS); + delay = next; + } + KUNIT_EXPECT_EQ(test, delay, CD321X_ACK_RETRY_MAX_MS); + + /* Once acknowledged, the cable state has its bounded budget again. */ + KUNIT_EXPECT_TRUE(test, cd321x_pm_acked(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_acked(&pm)); + for (i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm, true), + i + 1 < CD321X_RESUME_ATTEMPTS); + } + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); +} + +static void cd321x_unacked_events_across_sleep_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + unsigned int delay; + + KUNIT_ASSERT_TRUE(test, cd321x_pm_event_lost(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + delay = cd321x_pm_ack_failed(&pm); + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + /* The resume read takes over the retries, without starting afresh. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_GT(test, cd321x_pm_ack_failed(&pm), delay); + cd321x_pm_remove(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); +} + /* Only a pipeline owned by another port skips the retry. */ static void cd321x_display_taken_test(struct kunit *test) { @@ -451,6 +554,11 @@ static struct kunit_case cd321x_pm_cases[] = { KUNIT_CASE(cd321x_read_and_apply_share_budget_test), KUNIT_CASE(cd321x_status_event_after_exhaustion_test), KUNIT_CASE(cd321x_runtime_failure_revalidates_test), + KUNIT_CASE(cd321x_lost_event_rereads_test), + KUNIT_CASE(cd321x_lost_event_left_to_resume_test), + KUNIT_CASE(cd321x_lost_event_keeps_reconnect_test), + KUNIT_CASE(cd321x_unacked_events_outlast_budget_test), + KUNIT_CASE(cd321x_unacked_events_across_sleep_test), KUNIT_CASE(cd321x_display_taken_test), {} }; diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h index 08421d54c570b..3879c8a660dcc 100644 --- a/drivers/usb/typec/tipd/cd321x-pm.h +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -4,10 +4,13 @@ #include +#include #include #define CD321X_RESUME_ATTEMPTS 3 #define CD321X_LINK_EVENTS 3 +#define CD321X_ACK_RETRY_MS 500 +#define CD321X_ACK_RETRY_MAX_MS 30000 enum cd321x_pm_phase { CD321X_PM_RUNNING, @@ -24,6 +27,8 @@ struct cd321x_pm_state { enum cd321x_pm_phase phase; unsigned int attempts_left; unsigned int link_events_left; + /* Non-zero while events may be left unacknowledged: the retry delay. */ + unsigned int ack_retry_ms; bool force_reconnect; }; @@ -127,6 +132,48 @@ static inline bool cd321x_pm_event(struct cd321x_pm_state *pm) return true; } +/* + * Events that could not be read and acknowledged stay latched in the + * controller, which then sends no further interrupt. Only a fresh read can + * acknowledge them, so request one with a full budget even if an earlier + * recovery gave up. Sleep and probe already end with such a read. + */ +static inline bool cd321x_pm_event_lost(struct cd321x_pm_state *pm) +{ + if (pm->phase == CD321X_PM_INITIALIZING || + pm->phase == CD321X_PM_PREPARED || pm->phase == CD321X_PM_REMOVED) + return false; + pm->phase = CD321X_PM_REVALIDATE; + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + return true; +} + +/* + * Until the latched events are acknowledged, the controller raises no + * interrupt that could restart a recovery that gave up. So a snapshot read + * that fails to acknowledge them does not count against the cable-state + * budget, and is never given up: it gives back the attempt taken by + * cd321x_pm_begin_read() and is retried after a delay that doubles up to + * CD321X_ACK_RETRY_MAX_MS. Sleep cancels the retry and the resume read takes + * it over; removal ends it. Returns the delay in milliseconds. + */ +static inline unsigned int cd321x_pm_ack_failed(struct cd321x_pm_state *pm) +{ + pm->attempts_left++; + pm->ack_retry_ms = clamp(2 * pm->ack_retry_ms, CD321X_ACK_RETRY_MS, + CD321X_ACK_RETRY_MAX_MS); + return pm->ack_retry_ms; +} + +/* Returns true if acknowledging the events had failed before. */ +static inline bool cd321x_pm_acked(struct cd321x_pm_state *pm) +{ + bool retried = pm->ack_retry_ms; + + pm->ack_retry_ms = 0; + return retried; +} + static inline void cd321x_pm_complete(struct cd321x_pm_state *pm) { if (pm->phase == CD321X_PM_APPLY) { diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index ddfd89c8415e3..567961dad186a 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -992,6 +992,33 @@ static int cd321x_connect(struct tps6598x *tps, u32 status) return 0; } +/* + * The controller raises its interrupt only when events are latched while none + * were pending, so latched events that are never acknowledged silence the port + * for good. Acknowledge them before a snapshot read: the snapshot covers them. + */ +static int cd321x_ack_events(struct tps6598x *tps) +{ + u64 event; + int ret; + + ret = tps6598x_read64(tps, TPS_REG_INT_EVENT1, &event); + if (ret || !event) + return ret; + trace_cd321x_irq(event); + + return tps6598x_write64(tps, TPS_REG_INT_CLEAR1, event); +} + +/* Recover events that an interrupt could not read, acknowledge or apply. */ +static void cd321x_event_lost(struct cd321x *cd321x) +{ + lockdep_assert_held(&cd321x->tps.lock); + if (cd321x_pm_event_lost(&cd321x->pm)) + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, + msecs_to_jiffies(CD321X_DEBOUNCE_DELAY_MS)); +} + /* * An attached cable need not generate an IRQ after sleep. Read a fresh * snapshot and reapply it with the mode cache invalidated, allowing the @@ -1004,6 +1031,7 @@ static void cd321x_resume_work(struct work_struct *work) struct cd321x, resume_work); struct tps6598x *tps = &cd321x->tps; u32 status; + int ret; guard(mutex)(&tps->lock); if (cd321x->pm.phase == CD321X_PM_INITIALIZING || @@ -1023,6 +1051,18 @@ static void cd321x_resume_work(struct work_struct *work) if (!cd321x_pm_begin_read(&cd321x->pm)) return; + ret = cd321x_ack_events(tps); + if (ret) { + /* Retried until it succeeds, so report only the first failure. */ + if (!cd321x->pm.ack_retry_ms) + dev_err(tps->dev, "failed to acknowledge events: %d\n", ret); + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, + msecs_to_jiffies(cd321x_pm_ack_failed(&cd321x->pm))); + return; + } + if (cd321x_pm_acked(&cd321x->pm)) + dev_info(tps->dev, "acknowledged events after retrying\n"); + if (!tps6598x_read_status(tps, &status)) goto retry; @@ -1070,6 +1110,7 @@ static void cd321x_complete(struct tps6598x *tps) static irqreturn_t cd321x_interrupt(int irq, void *data) { struct tps6598x *tps = data; + struct cd321x *cd321x = container_of(tps, struct cd321x, tps); u64 event = 0; u32 status; int ret; @@ -1079,21 +1120,25 @@ static irqreturn_t cd321x_interrupt(int irq, void *data) ret = tps6598x_read64(tps, TPS_REG_INT_EVENT1, &event); if (ret) { dev_err(tps->dev, "%s: failed to read events\n", __func__); - goto err_unlock; + goto err_lost; } trace_cd321x_irq(event); if (!event) goto err_unlock; - tps6598x_write64(tps, TPS_REG_INT_CLEAR1, event); + ret = tps6598x_write64(tps, TPS_REG_INT_CLEAR1, event); + if (ret) { + dev_err(tps->dev, "%s: failed to clear events\n", __func__); + goto err_lost; + } if (!tps6598x_read_status(tps, &status)) - goto err_unlock; + goto err_lost; if (event & APPLE_CD_REG_INT_POWER_STATUS_UPDATE) { if (!tps6598x_read_power_status(tps)) - goto err_unlock; + goto err_lost; if (TPS_POWER_STATUS_PWROPMODE(tps->pwr_status) == TYPEC_PWR_MODE_PD) { if (tps6598x_read_partner_identity(tps)) { dev_err(tps->dev, "failed to read partner identity\n"); @@ -1104,11 +1149,14 @@ static irqreturn_t cd321x_interrupt(int irq, void *data) if (event & APPLE_CD_REG_INT_DATA_STATUS_UPDATE) if (!tps->data->read_data_status(tps)) - goto err_unlock; + goto err_lost; /* Can be called uncondtionally since it will check for any changes itself */ cd321x_connect(tps, status); + goto err_unlock; +err_lost: + cd321x_event_lost(cd321x); err_unlock: mutex_unlock(&tps->lock); @@ -1541,16 +1589,23 @@ cd321x_tbt_switch_get(struct device *dev, struct fwnode_handle *fwnode) } EXPORT_SYMBOL_IF_KUNIT(cd321x_tbt_switch_get); +/* Separate so that the KUnit tests can drive the workers without a port. */ +VISIBLE_IF_KUNIT void cd321x_init_work(struct cd321x *cd321x) +{ + INIT_DELAYED_WORK(&cd321x->update_work, cd321x_update_work); + INIT_DELAYED_WORK(&cd321x->resume_work, cd321x_resume_work); + atomic_set(&cd321x->link_event, 0); + cd321x_pm_init(&cd321x->pm); +} +EXPORT_SYMBOL_IF_KUNIT(cd321x_init_work); + static int cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); int ret; - INIT_DELAYED_WORK(&cd321x->update_work, cd321x_update_work); - INIT_DELAYED_WORK(&cd321x->resume_work, cd321x_resume_work); - atomic_set(&cd321x->link_event, 0); - cd321x_pm_init(&cd321x->pm); + cd321x_init_work(cd321x); ret = tps6598x_register_port(tps, fwnode); if (ret) diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 503757b035e0a..e24813dbda6df 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -457,6 +457,7 @@ static inline void tipd_debugfs_exit(void) { } #endif #if IS_ENABLED(CONFIG_KUNIT) +void cd321x_init_work(struct cd321x *cd321x); bool cd321x_tbt_switch_keep_waiting(struct device *dev); struct typec_thunderbolt_switch * cd321x_tbt_switch_get(struct device *dev, struct fwnode_handle *fwnode); From 6d51a7a3cf1b4be7fdbdd18699b633db078881eb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 23:06:16 +1100 Subject: [PATCH 1109/1579] usb: typec: tipd: cd321x: Recover from a failed partner identity read When cd321x_interrupt() fails to read the identity of a PD partner after a power status change, it logs the error, clears the identity and goes on to queue the cable update. The event has been acknowledged by then, and nothing reads the identity again. The update then registers a new partner with an all-zero identity, or, for a partner that is already registered, takes the changed identity for a different partner and tears down a working connection to register it again. Handle the failure like the other failed reads in the interrupt: leave the queued state alone and schedule a fresh snapshot read, which reads the identity together with the rest of the cable state, retrying within the usual budget. Add a KUnit test whose identity read fails once: the identity in use must be kept until the fresh read has succeeded, and then be replaced. Without this change it fails. Fixes: 36c791c1996d ("usb: typec: tipd: Update partner identity when power status was updated") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-irq-test.c | 45 ++++++++++++++++++++++++ drivers/usb/typec/tipd/core.c | 2 +- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/drivers/usb/typec/tipd/cd321x-irq-test.c b/drivers/usb/typec/tipd/cd321x-irq-test.c index e3df8bc1f4bf6..2d34af081f473 100644 --- a/drivers/usb/typec/tipd/cd321x-irq-test.c +++ b/drivers/usb/typec/tipd/cd321x-irq-test.c @@ -13,6 +13,8 @@ #define TEST_REG_INT_EVENT1 0x14 #define TEST_REG_INT_CLEAR1 0x18 #define TEST_REG_STATUS 0x1a +#define TEST_REG_POWER_STATUS 0x3f +#define TEST_REG_RX_IDENTITY_SOP 0x48 /* * The controller latches events in INT_EVENT1 until they are written to @@ -25,6 +27,10 @@ struct cd321x_test_ctrl { unsigned int failing_event_reads; unsigned int failing_clear_writes; unsigned int failing_status_reads; + unsigned int failing_identity_reads; + u32 status; + u16 pwr_status; + struct usb_pd_identity identity; }; struct cd321x_test { @@ -58,6 +64,19 @@ static int cd321x_test_read(void *context, const void *reg, size_t reg_size, ctrl->failing_status_reads--; return -ETIMEDOUT; } + memcpy(val, &ctrl->status, min(val_size, sizeof(ctrl->status))); + break; + case TEST_REG_POWER_STATUS: + memcpy(val, &ctrl->pwr_status, min(val_size, sizeof(ctrl->pwr_status))); + break; + case TEST_REG_RX_IDENTITY_SOP: + if (ctrl->failing_identity_reads) { + ctrl->failing_identity_reads--; + return -ETIMEDOUT; + } + /* The identity follows a status byte. */ + memcpy(val + 1, &ctrl->identity, + min(val_size - 1, sizeof(ctrl->identity))); break; } @@ -234,6 +253,31 @@ static void cd321x_failed_read_after_giving_up_test(struct kunit *test) KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); } +static void cd321x_failed_identity_read_recovers_test(struct kunit *test) +{ + struct cd321x_test *t = cd321x_test_port(test); + const struct usb_pd_identity old = { .id_header = 0x11 }; + + t->cd.tps.partner_identity = old; + t->cd.update_status.partner_identity = old; + t->ctrl.status = TPS_STATUS_PLUG_PRESENT; + t->ctrl.pwr_status = FIELD_PREP(TPS_POWER_STATUS_TYPEC_CURRENT_MASK, + TYPEC_PWR_MODE_PD); + t->ctrl.identity.id_header = 0x22; + t->ctrl.failing_identity_reads = 1; + cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_POWER_STATUS_UPDATE); + cd321x_test_interrupt(t); + KUNIT_ASSERT_EQ(test, t->ctrl.latched, 0); + + /* An incomplete read must not replace the identity in use... */ + KUNIT_EXPECT_EQ(test, t->cd.update_status.partner_identity.id_header, + old.id_header); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.update_work)); + /* ...but be completed by a fresh one. */ + cd321x_test_recover(test, t); + KUNIT_EXPECT_EQ(test, t->cd.update_status.partner_identity.id_header, 0x22U); +} + static void cd321x_failed_clear_recovers_test(struct kunit *test) { struct cd321x_test *t = cd321x_test_port(test); @@ -329,6 +373,7 @@ static struct kunit_case cd321x_irq_cases[] = { KUNIT_CASE(cd321x_failed_event_read_recovers_test), KUNIT_CASE(cd321x_failed_status_read_recovers_test), KUNIT_CASE(cd321x_failed_read_after_giving_up_test), + KUNIT_CASE(cd321x_failed_identity_read_recovers_test), KUNIT_CASE(cd321x_failed_clear_recovers_test), KUNIT_CASE(cd321x_event_reads_failing_for_long_test), KUNIT_CASE(cd321x_clear_writes_failing_for_long_test), diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 567961dad186a..395e990943220 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -1142,7 +1142,7 @@ static irqreturn_t cd321x_interrupt(int irq, void *data) if (TPS_POWER_STATUS_PWROPMODE(tps->pwr_status) == TYPEC_PWR_MODE_PD) { if (tps6598x_read_partner_identity(tps)) { dev_err(tps->dev, "failed to read partner identity\n"); - tps->partner_identity = (struct usb_pd_identity) {0}; + goto err_lost; } } } From c2dd5d18206eb916b45ba66a6a17e856546abb2e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 21:09:11 +1100 Subject: [PATCH 1110/1579] usb: typec: tipd: sn201202x: Keep events unhandled while the controller sleeps sn201202x_suspend() puts the controller into SPMI sleep, and sn201202x_resume() wakes it again, but its event interrupt stays enabled for the whole time. An event latched around suspend is handled while the controller sleeps: during suspend until the interrupt is suspended, and during resume as soon as the interrupt is replayed after the noirq phase, before the wakeup command has been sent. The register select then times out and the event read fails. On the J700 this shows up after s2idle as "sn201202x 1-0c: cd321x_interrupt: failed to read events", logged during device resume, with no further rear-port event for the rest of the boot. Disable the event interrupt before the controller is put to sleep, and enable it again once the controller is awake, or when suspend fails. An interrupt arriving meanwhile is replayed when it is enabled, and the resume revalidation acknowledges anything latched during sleep. A resume whose wakeup fails still enables the interrupt, so that a later event can be handled once the controller answers. Polled controllers have no event interrupt and are unaffected. Fixes: 419cafb82d4c ("usb: typec: tipd: Add sn201202x support") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/spmi.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 3ff9f602c23b6..1b83308a606d1 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -289,9 +289,12 @@ static int __maybe_unused sn201202x_resume(struct device *dev) int err; err = sn201202x_wake(sn); - if (err) - return err; - return tipd_resume(tps); + if (!err) + err = tipd_resume(tps); + /* The resume revalidation acknowledges events latched while asleep. */ + if (tps->irq) + enable_irq(tps->irq); + return err; } static int __maybe_unused sn201202x_suspend(struct device *dev) @@ -300,9 +303,16 @@ static int __maybe_unused sn201202x_suspend(struct device *dev) struct sn201202x *sn = tps_to_sn(tps); int err; + /* + * A sleeping controller cannot be read, so its events must not be + * handled until it is awake again. An interrupt arriving meanwhile is + * replayed when it is enabled. + */ + if (tps->irq) + disable_irq(tps->irq); err = tipd_suspend(tps); if (err) - return err; + goto out_enable; reinit_completion(&sn->sleep_completion); err = spmi_command_sleep(sn->sdev); if (err) @@ -316,6 +326,9 @@ static int __maybe_unused sn201202x_suspend(struct device *dev) out_resume: tipd_resume(tps); +out_enable: + if (tps->irq) + enable_irq(tps->irq); return err; } From 0a77048229afe1488764ebe47272cf5e36174e25 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 23:11:20 +1100 Subject: [PATCH 1111/1579] usb: typec: tipd: sn201202x: Wake the controller when suspend fails sn201202x_suspend() fails the suspend when the controller does not confirm the SLEEP command within 100 ms, and rolls back with tipd_resume(). The controller accepted the command, though, so it may well be asleep with only the confirmation missing. The PM core does not resume a device whose suspend failed, so nothing wakes the controller: the rollback reads its registers, its event interrupt is enabled again, and every event read then fails until the next system sleep. Send WAKEUP before rolling back once SLEEP was accepted. A failed wakeup is logged, and the suspend still fails with its own error. A SLEEP command that was rejected leaves the controller awake and needs no wakeup, as before. Failing the suspend, where the Asahi driver only logs the missing confirmation, is kept, so that the system does not sleep with the controller in an unknown state. Make the system sleep callbacks visible to KUnit and test them on a simulated SPMI bus: a sleep cycle keeps the event interrupt off while the controller sleeps, a failed wakeup at resume still enables it, a rejected SLEEP is not followed by a wakeup, and a SLEEP that is never confirmed is followed by a wakeup before the event interrupt is enabled again, also when that wakeup fails. Without this change the last two fail. Fixes: 38c14f6107df ("drm: apple: support ProMotion variable refresh rate") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/Kconfig | 11 + drivers/usb/typec/tipd/Makefile | 1 + drivers/usb/typec/tipd/sn201202x-pm-test.c | 240 +++++++++++++++++++++ drivers/usb/typec/tipd/spmi.c | 16 +- drivers/usb/typec/tipd/tps6598x.h | 2 + 5 files changed, 266 insertions(+), 4 deletions(-) create mode 100644 drivers/usb/typec/tipd/sn201202x-pm-test.c diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index 7c2527913111d..b8a904446109a 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -43,3 +43,14 @@ config TYPEC_TIPD_KUNIT_TEST Also test that a USB4/Thunderbolt router which never registers its switch does not keep the port from probing. These tests run without accessing a physical Type-C controller. + +config TYPEC_SN201202X_KUNIT_TEST + tristate "KUnit tests for SN201202x" if !KUNIT_ALL_TESTS + depends on KUNIT && TYPEC_SN201202X + select REGMAP + default KUNIT_ALL_TESTS + help + Test that the SN201202x events are not handled while the controller + sleeps, and that a system suspend that fails leaves the controller + awake. These tests run on a simulated SPMI bus, without accessing a + physical Type-C controller. diff --git a/drivers/usb/typec/tipd/Makefile b/drivers/usb/typec/tipd/Makefile index e2e3d9271dd26..fcb10573a05b1 100644 --- a/drivers/usb/typec/tipd/Makefile +++ b/drivers/usb/typec/tipd/Makefile @@ -14,3 +14,4 @@ sn201202x-y := spmi.o obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o cd321x-tbt-test.o \ cd321x-irq-test.o +obj-$(CONFIG_TYPEC_SN201202X_KUNIT_TEST) += sn201202x-pm-test.o diff --git a/drivers/usb/typec/tipd/sn201202x-pm-test.c b/drivers/usb/typec/tipd/sn201202x-pm-test.c new file mode 100644 index 0000000000000..4ddb040df0b3a --- /dev/null +++ b/drivers/usb/typec/tipd/sn201202x-pm-test.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for SN201202x system sleep + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "tps6598x.h" + +/* + * A controller on a simulated SPMI bus. It confirms SLEEP and WAKEUP with the + * interrupts the driver waits for, unless the SLEEP confirmation gets lost, + * and it does not answer register accesses while it is asleep. + */ +struct sn201202x_test { + struct sn201202x sn; + struct device *dev; + int irq; + bool asleep; + bool sleep_unconfirmed; + int sleep_error; + int wakeup_error; + unsigned int wakeups; + bool irq_enabled_at_wakeup; +}; + +static bool sn201202x_test_irq_enabled(struct sn201202x_test *t) +{ + return !irqd_irq_disabled(irq_get_irq_data(t->irq)); +} + +static int sn201202x_test_cmd(struct spmi_controller *ctrl, u8 opcode, u8 sid) +{ + struct sn201202x_test *t = spmi_controller_get_drvdata(ctrl); + + switch (opcode) { + case SPMI_CMD_SLEEP: + if (t->sleep_error) + return t->sleep_error; + t->asleep = true; + if (!t->sleep_unconfirmed) + complete(&t->sn.sleep_completion); + return 0; + case SPMI_CMD_WAKEUP: + t->wakeups++; + t->irq_enabled_at_wakeup = sn201202x_test_irq_enabled(t); + if (t->wakeup_error) + return t->wakeup_error; + t->asleep = false; + complete(&t->sn.wake_completion); + return 0; + } + + return -EOPNOTSUPP; +} + +static int sn201202x_test_read(void *context, const void *reg, size_t reg_size, + void *val, size_t val_size) +{ + struct sn201202x_test *t = context; + + /* A sleeping controller never confirms the register selection. */ + if (t->asleep) + return -ETIMEDOUT; + memset(val, 0, val_size); + if (*(const u8 *)reg == TPS_REG_MODE) + memcpy(val, "APP ", min(val_size, 4)); + + return 0; +} + +static int sn201202x_test_write(void *context, const void *data, size_t count) +{ + struct sn201202x_test *t = context; + + return t->asleep ? -ETIMEDOUT : 0; +} + +static const struct regmap_bus sn201202x_test_bus = { + .read = sn201202x_test_read, + .write = sn201202x_test_write, + .reg_format_endian_default = REGMAP_ENDIAN_NATIVE, + .val_format_endian_default = REGMAP_ENDIAN_NATIVE, +}; + +static irqreturn_t sn201202x_test_irq(int irq, void *data) +{ + return IRQ_HANDLED; +} + +static void sn201202x_test_put_device(void *data) +{ + spmi_device_put(data); +} + +static void sn201202x_test_free_irq_desc(void *data) +{ + irq_free_desc((unsigned long)data); +} + +static void sn201202x_test_free_irq(void *data) +{ + struct sn201202x_test *t = data; + + free_irq(t->irq, t); +} + +static struct sn201202x_test *sn201202x_test_port(struct kunit *test) +{ + struct spmi_controller *ctrl; + struct sn201202x_test *t; + struct tps6598x *tps; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + t->dev = kunit_device_register(test, "sn201202x-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->dev); + + ctrl = devm_spmi_controller_alloc(t->dev, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctrl); + spmi_controller_set_drvdata(ctrl, t); + ctrl->cmd = sn201202x_test_cmd; + t->sn.sdev = spmi_device_alloc(ctrl); + KUNIT_ASSERT_NOT_NULL(test, t->sn.sdev); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, sn201202x_test_put_device, + t->sn.sdev), 0); + init_completion(&t->sn.select_completion); + init_completion(&t->sn.sleep_completion); + init_completion(&t->sn.wake_completion); + + tps = &t->sn.cd.tps; + tps->dev = t->dev; + tps->data = &tipd_sn201202x_data; + tps->regmap = devm_regmap_init(t->dev, &sn201202x_test_bus, t, + &tps6598x_regmap_config); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, tps->regmap); + dev_set_drvdata(t->dev, tps); + + /* The event interrupt, which is never raised. */ + t->irq = irq_alloc_desc(NUMA_NO_NODE); + KUNIT_ASSERT_GE(test, t->irq, 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, sn201202x_test_free_irq_desc, + (void *)(unsigned long)t->irq), 0); + irq_set_chip_and_handler(t->irq, &dummy_irq_chip, handle_simple_irq); + irq_clear_status_flags(t->irq, IRQ_NOREQUEST); + KUNIT_ASSERT_EQ(test, request_irq(t->irq, sn201202x_test_irq, 0, + "sn201202x-test", t), 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, sn201202x_test_free_irq, t), 0); + tps->irq = t->irq; + + return t; +} + +static void sn201202x_sleep_cycle_test(struct kunit *test) +{ + struct sn201202x_test *t = sn201202x_test_port(test); + + KUNIT_ASSERT_EQ(test, sn201202x_suspend(t->dev), 0); + KUNIT_EXPECT_TRUE(test, t->asleep); + /* No event may be handled while the controller cannot be read. */ + KUNIT_EXPECT_FALSE(test, sn201202x_test_irq_enabled(t)); + + KUNIT_EXPECT_EQ(test, sn201202x_resume(t->dev), 0); + KUNIT_EXPECT_EQ(test, t->wakeups, 1U); + KUNIT_EXPECT_FALSE(test, t->irq_enabled_at_wakeup); + KUNIT_EXPECT_TRUE(test, sn201202x_test_irq_enabled(t)); +} + +static void sn201202x_failed_wakeup_test(struct kunit *test) +{ + struct sn201202x_test *t = sn201202x_test_port(test); + + KUNIT_ASSERT_EQ(test, sn201202x_suspend(t->dev), 0); + t->wakeup_error = -EIO; + KUNIT_EXPECT_EQ(test, sn201202x_resume(t->dev), -EIO); + /* Events are handled again once the controller answers. */ + KUNIT_EXPECT_TRUE(test, sn201202x_test_irq_enabled(t)); +} + +static void sn201202x_rejected_sleep_test(struct kunit *test) +{ + struct sn201202x_test *t = sn201202x_test_port(test); + + t->sleep_error = -EIO; + KUNIT_EXPECT_EQ(test, sn201202x_suspend(t->dev), -EIO); + KUNIT_EXPECT_FALSE(test, t->asleep); + KUNIT_EXPECT_EQ(test, t->wakeups, 0U); + KUNIT_EXPECT_TRUE(test, sn201202x_test_irq_enabled(t)); +} + +static void sn201202x_unconfirmed_sleep_test(struct kunit *test) +{ + struct sn201202x_test *t = sn201202x_test_port(test); + + t->sleep_unconfirmed = true; + KUNIT_EXPECT_EQ(test, sn201202x_suspend(t->dev), -ETIMEDOUT); + /* No resume follows a failed suspend, so the rollback must wake it... */ + KUNIT_EXPECT_EQ(test, t->wakeups, 1U); + KUNIT_EXPECT_FALSE(test, t->asleep); + /* ...before its events are handled again. */ + KUNIT_EXPECT_FALSE(test, t->irq_enabled_at_wakeup); + KUNIT_EXPECT_TRUE(test, sn201202x_test_irq_enabled(t)); +} + +static void sn201202x_unconfirmed_sleep_failed_wakeup_test(struct kunit *test) +{ + struct sn201202x_test *t = sn201202x_test_port(test); + + t->sleep_unconfirmed = true; + t->wakeup_error = -EIO; + /* The suspend error is reported, not the one from the rollback. */ + KUNIT_EXPECT_EQ(test, sn201202x_suspend(t->dev), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, t->wakeups, 1U); + KUNIT_EXPECT_TRUE(test, sn201202x_test_irq_enabled(t)); +} + +static struct kunit_case sn201202x_pm_cases[] = { + KUNIT_CASE(sn201202x_sleep_cycle_test), + KUNIT_CASE(sn201202x_failed_wakeup_test), + KUNIT_CASE(sn201202x_rejected_sleep_test), + KUNIT_CASE(sn201202x_unconfirmed_sleep_test), + KUNIT_CASE(sn201202x_unconfirmed_sleep_failed_wakeup_test), + {} +}; + +static struct kunit_suite sn201202x_pm_suite = { + .name = "sn201202x-pm", + .test_cases = sn201202x_pm_cases, +}; + +kunit_test_suite(sn201202x_pm_suite); +MODULE_DESCRIPTION("KUnit tests for SN201202x system sleep"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 1b83308a606d1..1b9cf970cb95a 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include @@ -282,7 +283,7 @@ static void sn201202x_remove(struct spmi_device *device) spmi_command_sleep(sn->sdev); } -static int __maybe_unused sn201202x_resume(struct device *dev) +VISIBLE_IF_KUNIT int __maybe_unused sn201202x_resume(struct device *dev) { struct tps6598x *tps = dev_get_drvdata(dev); struct sn201202x *sn = tps_to_sn(tps); @@ -296,12 +297,13 @@ static int __maybe_unused sn201202x_resume(struct device *dev) enable_irq(tps->irq); return err; } +EXPORT_SYMBOL_IF_KUNIT(sn201202x_resume); -static int __maybe_unused sn201202x_suspend(struct device *dev) +VISIBLE_IF_KUNIT int __maybe_unused sn201202x_suspend(struct device *dev) { struct tps6598x *tps = dev_get_drvdata(dev); struct sn201202x *sn = tps_to_sn(tps); - int err; + int err, ret; /* * A sleeping controller cannot be read, so its events must not be @@ -320,10 +322,15 @@ static int __maybe_unused sn201202x_suspend(struct device *dev) if (!sn->polled && !wait_for_completion_timeout(&sn->sleep_completion, msecs_to_jiffies(100))) { err = -ETIMEDOUT; - goto out_resume; + goto out_wake; } return 0; +out_wake: + /* It accepted SLEEP, so it may be asleep without having confirmed it. */ + ret = sn201202x_wake(sn); + if (ret) + dev_err(dev, "failed to wake up after failed suspend: %d\n", ret); out_resume: tipd_resume(tps); out_enable: @@ -331,6 +338,7 @@ static int __maybe_unused sn201202x_suspend(struct device *dev) enable_irq(tps->irq); return err; } +EXPORT_SYMBOL_IF_KUNIT(sn201202x_suspend); MODULE_DEVICE_TABLE(of, sn201202x_of_match); diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index e24813dbda6df..c919dad813a58 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -461,6 +461,8 @@ void cd321x_init_work(struct cd321x *cd321x); bool cd321x_tbt_switch_keep_waiting(struct device *dev); struct typec_thunderbolt_switch * cd321x_tbt_switch_get(struct device *dev, struct fwnode_handle *fwnode); +int sn201202x_suspend(struct device *dev); +int sn201202x_resume(struct device *dev); #endif #endif /* __TPS6598X_H__ */ From 600b095055972ae25308f5f63d29adb31468a6e9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 23:15:03 +1100 Subject: [PATCH 1112/1579] tools: aurora: Notify about unsupported front-port displays A display adapter on the J700's USB 2.0 front connector can enumerate a Billboard device while leaving the display dark. Explain the connector's limitation and direct the user to the rear USB-C port. Mark front-branch Billboards with udev and verify the board's fixed hub and controller topology in an unprivileged user service. Send a normal libnotify notification only for an active local graphical session. Keep per-plug delivery state across service restarts and rate-limit attempts without changing display configuration or notification preferences. Keep this board-specific hint in userspace: missing partner altmodes in the firmware-managed Type-C driver do not establish a lack of DP support. Document the kernel/userspace split, public implementation survey, installation, rollback and attended acceptance checks. Add synthetic host tests and a private-session-bus libnotify integration test. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 5 + .../70-aurora-display-notifier.rules | 7 + tools/aurora-display-notifier/Makefile | 22 ++ tools/aurora-display-notifier/README.md | 263 +++++++++++++++ .../aurora-display-notifier.service.in | 19 ++ tools/aurora-display-notifier/notifier.py | 236 ++++++++++++++ .../org.aurorasilicon.DisplayHint.desktop | 9 + .../tests/integration_dbus.py | 111 +++++++ .../tests/test_notifier.py | 306 ++++++++++++++++++ 9 files changed, 978 insertions(+) create mode 100644 tools/aurora-display-notifier/70-aurora-display-notifier.rules create mode 100644 tools/aurora-display-notifier/Makefile create mode 100644 tools/aurora-display-notifier/README.md create mode 100644 tools/aurora-display-notifier/aurora-display-notifier.service.in create mode 100644 tools/aurora-display-notifier/notifier.py create mode 100644 tools/aurora-display-notifier/org.aurorasilicon.DisplayHint.desktop create mode 100644 tools/aurora-display-notifier/tests/integration_dbus.py create mode 100644 tools/aurora-display-notifier/tests/test_notifier.py diff --git a/MAINTAINERS b/MAINTAINERS index aafd54ac18b65..f02c49da5d11b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4341,6 +4341,11 @@ F: kernel/audit* F: lib/*audit.c K: \baudit_[a-z_0-9]\+\b +AURORA DISPLAY NOTIFIER +M: Ryan Murray +S: Maintained +F: tools/aurora-display-notifier/ + AUTOFDO BUILD M: Rong Xu M: Han Shen diff --git a/tools/aurora-display-notifier/70-aurora-display-notifier.rules b/tools/aurora-display-notifier/70-aurora-display-notifier.rules new file mode 100644 index 0000000000000..abb7f8fdb0a05 --- /dev/null +++ b/tools/aurora-display-notifier/70-aurora-display-notifier.rules @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +# The notifier additionally checks apple,j700 and the fixed hub/controller. +# Never launch a desktop process or guess a user's session bus from udev. +ACTION=="add|change", SUBSYSTEM=="usb", ENV{DEVTYPE}=="usb_device", \ + ATTR{bDeviceClass}=="11", ATTR{devpath}=="1.1|1.1.*", \ + ATTRS{idVendor}=="2109", ATTRS{idProduct}=="2122", \ + ENV{AURORA_DISPLAY_UNSUPPORTED}="1" diff --git a/tools/aurora-display-notifier/Makefile b/tools/aurora-display-notifier/Makefile new file mode 100644 index 0000000000000..7ce15c76936c6 --- /dev/null +++ b/tools/aurora-display-notifier/Makefile @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0-only +PREFIX ?= /usr +LIBEXECDIR ?= $(PREFIX)/libexec +UDEVDIR ?= $(PREFIX)/lib/udev +USERUNITDIR ?= $(PREFIX)/lib/systemd/user +DATADIR ?= $(PREFIX)/share +PYTHON ?= python3 + +.PHONY: test install +test: + PYTHONDONTWRITEBYTECODE=1 $(PYTHON) -m unittest discover -s tests -v + +install: + install -Dm755 notifier.py $(DESTDIR)$(LIBEXECDIR)/aurora-display-notifier + install -Dm644 70-aurora-display-notifier.rules \ + $(DESTDIR)$(UDEVDIR)/rules.d/70-aurora-display-notifier.rules + install -d $(DESTDIR)$(USERUNITDIR) + sed 's|@LIBEXECDIR@|$(LIBEXECDIR)|g' aurora-display-notifier.service.in \ + > $(DESTDIR)$(USERUNITDIR)/aurora-display-notifier.service + chmod 644 $(DESTDIR)$(USERUNITDIR)/aurora-display-notifier.service + install -Dm644 org.aurorasilicon.DisplayHint.desktop \ + $(DESTDIR)$(DATADIR)/applications/org.aurorasilicon.DisplayHint.desktop diff --git a/tools/aurora-display-notifier/README.md b/tools/aurora-display-notifier/README.md new file mode 100644 index 0000000000000..624d5362ffadb --- /dev/null +++ b/tools/aurora-display-notifier/README.md @@ -0,0 +1,263 @@ + +# J700 front USB-C display hint + +The MacBook Neo (J700) front USB-C connector carries USB 2.0 and power, +but has no DisplayPort lanes. A display adapter can enumerate a USB +Billboard device when its alternate mode cannot be entered. This tool +shows an informational desktop notification: + +> This port doesn't support displays. +> To connect a display, use the rear USB-C port. + +It changes no display, USB authorization, power, notification, or saved +device settings. The desktop's notification preferences continue to apply. + +## Recommended split and upstream prospects + +Ship the hint in userspace. The udev rule only marks device-class `0x11` +Billboards on the fixed hub's front branch. An unprivileged user service +verifies the J700 identity and controller topology, then calls libnotify's +`notify-send` through its inherited session bus. There is no root-to-user +bus discovery, udev `RUN`, system service, or desktop-specific extension. +This works with GNOME Shell and with a notification server such as mako. + +No kernel change is included. At the `a837bd5f9a21160256580aeb0829f6930440950c` +base, the public [J700 DT](https://github.com/aurora-silicon/linux/blob/a837bd5f9a21160256580aeb0829f6930440950c/arch/arm64/boot/dts/apple/t8140-j700.dts) +already describes the fixed VIA `2109:2122` hub and its connector graph: +USB downstream path `1.1` is front, `1.2` is rear. Only rear has the +SuperSpeed/DP PHY connection. The controller is `40a280000.usb`. USB bus +numbers, device addresses, and Linux `port0`/`port1` registration order +are deliberately not used to identify the connector. DT source labels +such as `typec1` are also independent of Linux Type-C class numbering. + +The present firmware-managed +[tipd driver](https://github.com/aurora-silicon/linux/blob/a837bd5f9a21160256580aeb0829f6930440950c/drivers/usb/typec/tipd/core.c) +does not register discovered partner altmodes. Absence of those objects +therefore cannot establish whether a port or partner supports DP. The +notifier uses the board's explicit wiring, rather than inferring a missing +capability from missing sysfs objects. + +A generic kernel improvement to expose firmware-discovered partner +capabilities could be upstream-plausible, separately: obtain actual +Discover SVIDs **and Discover Modes VDOs**, register/unregister partner +altmodes on connection changes, and report active state while preserving +firmware ownership of mode entry. A SVID alone is insufficient to invent +a mode descriptor. This needs public controller documentation and tests +for the specific controller variant, discovery failure, disconnect races, +and suspend. It is unnecessary for this hint. A `dev_info` on every PD +connection cannot distinguish a charger from a display adapter; a generic +tipd message naming J700's front/rear ports would mix board UI into a +controller driver. Neither is recommended here. + +Upstream placement for a generic notifier would be a desktop/session +component backed by reliable connector capability and USB-to-Type-C +association data. This implementation is a small Aurora userspace feature +staged under `tools/`, not a proposed Linux kernel interface. Its J700 +mapping belongs in a board support package until a generic interface can +replace it. Upstream acceptance has not been requested or established. + +## Existing public implementations checked + +Source review on 2026-10-16 was limited to public implementation files; +upstream agent/AI instruction files were not read. No implementation was +copied from proprietary sources. + +* [Asahi tipd at 94fb23346d52](https://github.com/AsahiLinux/linux/blob/94fb23346d522edf53722357c426a3e58030beea/drivers/usb/typec/tipd/core.c) + registers port DP/TBT altmodes and partner identity, but has no + `typec_partner_register_altmode` call or Billboard desktop hint in that + driver. This is a bounded driver inspection, not a claim about every + Asahi package. +* [Linux Type-C class API](https://docs.kernel.org/driver-api/usb/typec.html) + represents partner and port modes separately. Exposing real discovered + modes is a driver concern; displaying a desktop notification is a + session concern. The [upstream class implementation at 90201351a862](https://github.com/torvalds/linux/blob/90201351a862182bdabfc1ad1c97b1e1b6ed5931/drivers/usb/typec/class.c) + recognizes the PD identity's Billboard capability as USB 2.0 capability; + that handling does not send a desktop hint. +* [GNOME settings daemon at 4a75955f7481](https://github.com/GNOME/gnome-settings-daemon/tree/4a75955f7481e2bb19fa73fa67456e83d67e8563/plugins) + had no Billboard/alternate-mode handler in the plugin C/header files + searched. Its [USB protection notifications](https://github.com/GNOME/gnome-settings-daemon/blob/4a75955f7481e2bb19fa73fa67456e83d67e8563/plugins/usb-protection/gsd-usb-protection-manager.c) + concern USBGuard authorization, not a connector's DP capability. + This feature does not change those policies. +* [fwupd Analogix at 50181eeefa62](https://github.com/fwupd/fwupd/blob/50181eeefa622a68b123c2994dd4385fe1430765/plugins/analogix/README.md) + uses selected Billboard devices for firmware updates. Its plugin is + not a generic wrong-port display notifier. This tool never claims USB + interfaces, reads BOS through libusb, or sends USB control requests. +* [systemd hwdb at f9d375fc3daf](https://github.com/systemd/systemd/blob/f9d375fc3daff274f7113d757e0cba59e7a332d4/hwdb.d/20-usb-classes.hwdb) + names class `11` as Billboard. The standard rules import that descriptive + metadata; they do not attach a front/rear display hint to it. The + [udev contract](https://github.com/systemd/systemd/blob/f9d375fc3daff274f7113d757e0cba59e7a332d4/man/udev.xml) + also excludes long-running desktop processes from rules. + +## Matching and delivery + +The rule matches `usb_device`, `bDeviceClass=11`, downstream device paths +`1.1` or `1.1.*`, and a common ancestor with USB ID `2109:2122`. The two +`ATTRS` matches refer to the same ancestor. The user process additionally +requires the exact `apple,j700` compatible token, a fixed hub at downstream +path `1`, the direct front child at `1.1`, and controller `40a280000.usb`. +Other machines and rear-port Billboards fail these checks. The rule's +property is a candidate marker, not an authorization or a generic DP ABI. + +The service subscribes to processed udev events before enumerating, and +reconciles every two seconds. It also handles an adapter already attached +at login. All Billboard functions on the same front device instance share +one notification. Disappearance of just the Billboard function does not +reset deduplication while its parent dock stays attached. + +State in `$XDG_RUNTIME_DIR/aurora-display-notifier/` records successful +delivery per USB instance (sysfs path, address, inode, udev initialization +timestamp), plus the last attempt time. It survives service restarts during +the login, contains no peripheral names/serials, and expires with the user +runtime directory. A lock prevents multiple instances for the same user. +Delivery attempts are at least 30 seconds apart. A rapid replug or a +temporarily missing notification server is retried while still attached; +unplugged pending events are discarded. There is a small unavoidable +crash window between notification acceptance and saving success. + +Before sending, logind must report an active local graphical session +owned by the service's UID and attached to a seat. SSH, TTY, inactive +sessions, and logind failures cannot trigger delivery. If the service is +started before the desktop, it waits until that condition holds. It never +selects another user's bus. The user manager/session bus is per UID; +simultaneous graphical logins of the *same UID* share that bus and its +notification server, as they do for other user services. + +Notifications have normal urgency and the server's default expiry. The +[standard notification protocol](https://specifications.freedesktop.org/notification/latest/protocol.html) +is used without actions or settings changes. Users can disable or mask +the service, or use GNOME/mako notification preferences normally. + +Billboard means an alternate-mode failure, not proof of a connected DP +display. Thus the body is conditional on connecting a display. An adapter +advertising another failed alternate mode can also receive this truthful +port-capability hint. A display adapter that never enumerates a Billboard, +or only uses an interface-class Billboard with another device class, is +not detected. USB graphics such as DisplayLink are outside this feature. + +## Install and enable + +Runtime requirements: Python 3.9+, pyudev, libudev/udev, systemd with +logind and a user manager, libnotify's `notify-send`, and a session +notification server. For Fedora these are typically `python3-pyudev` and +`libnotify`; for Arch/Omarchy, `python-pyudev` and `libnotify` (plus mako or +another notification server). The staging Makefile expects GNU install. + +Packagers can stage all files without running hooks: + +```sh +make DESTDIR="$pkgdir" install +``` + +`PREFIX`, `LIBEXECDIR`, `UDEVDIR`, `USERUNITDIR`, and `DATADIR` are +overridable. No service is enabled by the Makefile. After an authorized +installation, reload udev rules with `sudo udevadm control --reload-rules` +and replug the adapter. Existing devices need a new processed add/change +event to obtain the marker; a rules reload alone does not retroactively +mark them. Avoid a global USB trigger. + +For the desktop user: + +```sh +systemctl --user daemon-reload +systemctl --user enable --now aurora-display-notifier.service +``` + +GNOME and [UWSM-managed Hyprland](https://wiki.hypr.land/Useful-Utilities/Systemd-start/) +supply `graphical-session.target`. A Hyprland session that does not manage +that target can execute the following commands from its startup and +shutdown hooks, respectively (use the syntax for the installed compositor +version): + +```sh +systemctl --user start aurora-display-notifier.service +systemctl --user stop aurora-display-notifier.service +``` + +The user service still requires its inherited session bus and logind's +local graphical session. It requires neither DISPLAY/WAYLAND_DISPLAY +imports nor any guessed bus address. +Launching a compositor inside a logind session still classified as `tty` +does not satisfy the graphical-session check; use a properly registered +graphical login for delivery. + +## Host verification + +```sh +make test +dbus-run-session -- env AURORA_TEST_PRIVATE_BUS=1 PYTHONDONTWRITEBYTECODE=1 \ + python3 tests/integration_dbus.py +``` + +The unit tests use synthetic sysfs and udev objects. The second test needs +PyGObject/Gio and `notify-send`; it sends through a fresh private bus to a +mock notification server, never the desktop's real notification server. +It verifies the libnotify message, default expiration, and persistent +deduplication. Neither command requires J700 access or USB writes. + +Validate a staged install on Linux with `udevadm verify` on the rules, +`systemd-analyze --user verify` on the generated service (with its staged +ExecStart path substituted for validation), and `desktop-file-validate`. +There is no kernel/DT change, so kernel builds, KUnit and dtbs_check do +not validate this userspace-only addition. + +## Attended on-Neo acceptance protocol (not executed here) + +The integration operator must obtain controller policy approval and a +session/resource lease explicitly identifying the J700 before any target +inspection, install, service start, or USB replug. Use that controller +session for every target command. Agree the plug sequence with the person +at the Neo. Do not infer that this document grants hardware access. + +1. Record the candidate commit, installed paths and package version. Save + the existing enable/mask/active status of this service and any previous + versions of these four installed files for rollback. Confirm a J700 + identity, a local GNOME or Hyprland graphical login, logind `Active=yes`, + `Remote=no`, matching `User`, and `Type=wayland` or `x11`. Confirm a + notification server owns `org.freedesktop.Notifications` on that user's + session bus. Preserve existing notification preferences; if Do Not + Disturb hides banners, record that and check the notification history. +2. Install this userspace candidate and dependencies with the approved + integration method, reload only these udev rules, then enable/start + the user unit as above. Record `systemctl --user status + aurora-display-notifier.service`. Do not change kernel/DT or display + settings. Start `udevadm monitor --udev --property --subsystem-match=usb` + and `journalctl --user -fu aurora-display-notifier.service` in the + approved session, saving their output as private qualification evidence. +3. With the dock/display disconnected, plug it into **front**. Within a + few seconds of Billboard enumeration, expect exactly one hint with the + text above. Check the Billboard device's `bDeviceClass` is `11` and + `udevadm info --query=property --path=` contains + `AURORA_DISPLAY_UNSUPPORTED=1`. Verify ancestry is the fixed hub's + `1.1` branch under `40a280000.usb`. Do not assume bus number `1` or + Billboard address `1-1.1.5` stays constant. Leave it connected for at + least 35 seconds: no second hint. Restart the user service: no duplicate. +4. Unplug/replug the whole dock in **front** to establish a fresh hint + (the previous step waited over 30 seconds). Immediately unplug/replug + again and keep it connected: at most one new hint, delayed until the + 30-second limit expires. Repeat the rapid unplug/replug cycle, but + remove the dock during that delay: no stale hint after removal. Wait + over 30 seconds from the last hint, then replug and keep it attached: + one new hint. +5. Unplug and connect the same dock/display to **rear**. Expect no new + notification from this service, even if a Billboard enumerates because + of another failure. Verify rear USB 2.0 ancestry uses `1.2`, and check + the external display works with the existing driver. A front-port hint + already in desktop history is not a new rear-port event. +6. Test a plain USB data device and a charger in front: no hint. With no + local active graphical session, test front Billboard enumeration: no + delivery to SSH/TTY or an inactive user's session. On returning to the + local graphical session, a still-connected eligible dock may produce + the pending hint. Confirm user preferences and display configuration + were not changed. Repeat steps 3 and 5 in GNOME and Hyprland/mako. +7. Save pass/fail evidence including failed attempts and the final user + unit journal. Raw USB/session metadata remains private. Stop/disable + the service if it was enabled only for qualification, restore its + original enable/mask/active state and the previous files (or remove + just the four newly installed files), reload rules and user units. + Restart the restored service only if it was previously running. No + kernel rollback is needed. Runtime state can be removed after stopping + all instances if the operator wants to reset the hint for another test. + +Acceptance requires both the visible front-port hint and absence of new +rear-port hints in each desktop. Host tests alone do not prove those +attended results or validate the physical topology on the running DT. diff --git a/tools/aurora-display-notifier/aurora-display-notifier.service.in b/tools/aurora-display-notifier/aurora-display-notifier.service.in new file mode 100644 index 0000000000000..d167a18cb5d79 --- /dev/null +++ b/tools/aurora-display-notifier/aurora-display-notifier.service.in @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only +[Unit] +Description=Explain USB-C display limitations +PartOf=graphical-session.target +After=graphical-session.target +ConditionPathExists=/sys/firmware/devicetree/base/compatible + +[Service] +ExecStart=@LIBEXECDIR@/aurora-display-notifier +Restart=on-failure +RestartSec=10 +RuntimeDirectory=aurora-display-notifier +RuntimeDirectoryMode=0700 +RuntimeDirectoryPreserve=yes +UMask=0077 +NoNewPrivileges=yes + +[Install] +WantedBy=graphical-session.target diff --git a/tools/aurora-display-notifier/notifier.py b/tools/aurora-display-notifier/notifier.py new file mode 100644 index 0000000000000..f2358073ba001 --- /dev/null +++ b/tools/aurora-display-notifier/notifier.py @@ -0,0 +1,236 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Explain the J700 front USB-C port's display limitation in the user's session.""" + +import fcntl +import json +import logging +import os +from pathlib import Path +import subprocess +import time + + +SUMMARY = "This port doesn't support displays." +BODY = "To connect a display, use the rear USB-C port." +MARKER = "AURORA_DISPLAY_UNSUPPORTED" +INTERVAL = 30.0 + + +def read_attr(path, name): + try: + return (path / name).read_text(encoding="ascii").strip() + except (OSError, UnicodeError): + return "" + + +def is_j700(sysfs): + try: + compatible = (sysfs / "firmware/devicetree/base/compatible").read_bytes() + except OSError: + return False + return b"apple,j700" in compatible.split(b"\0") + + +def front_device(path, sysfs): + """Match the fixed hub's front branch, independently of USB bus numbers.""" + path = path.resolve() + if not path.is_relative_to((sysfs / "devices").resolve()): + return None + for front in (path, *path.parents): + if read_attr(front, "devpath") != "1.1": + continue + hub = front.parent + if (read_attr(hub, "devpath") == "1" + and read_attr(hub, "idVendor") == "2109" + and read_attr(hub, "idProduct") == "2122" + and any(p.name == "40a280000.usb" for p in hub.parents)): + return front + return None + + +def candidate(device, sysfs): + if (device.device_type != "usb_device" + or device.properties.get(MARKER) != "1"): + return None + path = Path(device.sys_path) + if read_attr(path, "bDeviceClass") != "11": + return None + return front_device(path, sysfs) + + +def plug_id(front, initialized): + """A new USB device instance on the front branch starts a new plug.""" + number = read_attr(front, "devnum") + try: + inode = front.stat().st_ino + except OSError: + return None + if not number.isdecimal(): + return None + return f"{front}:{number}:{inode}:{initialized}" + + +def command(args): + return subprocess.run(args, check=True, capture_output=True, text=True, timeout=5).stdout + + +def active_desktop(uid): + """The user manager may outlive the local graphical login (lingering/SSH).""" + try: + sessions = command(["loginctl", "show-user", str(uid), "-p", "Sessions", "--value"]) + for session in sessions.split(): + data = command([ + "loginctl", "show-session", session, "-p", "User", "-p", "Active", + "-p", "Remote", "-p", "Type", "-p", "Seat", + ]) + props = dict(line.split("=", 1) for line in data.splitlines() if "=" in line) + if (props.get("User") == str(uid) and props.get("Active") == "yes" + and props.get("Remote") == "no" and props.get("Seat") + and props.get("Type") in ("wayland", "x11")): + return True + except (OSError, subprocess.SubprocessError): + pass + return False + + +def notify(): + try: + command([ + "notify-send", "--app-name=Aurora Display Hint", "--urgency=normal", + "--icon=video-display-symbolic", + "--hint=string:desktop-entry:org.aurorasilicon.DisplayHint", + "--", SUMMARY, BODY, + ]) + except (OSError, subprocess.SubprocessError): + logging.warning("Could not deliver the display hint; retrying in 30 seconds") + return False + logging.info("Sent the front USB-C display hint") + return True + + +class Notifier: + def __init__(self, state_path, send=notify, active=None, clock=time.monotonic): + self.path = state_path + self.send = send + self.active = active if active is not None else lambda: active_desktop(os.getuid()) + self.clock = clock + self.shown = set() + self.last_attempt = None + try: + data = json.loads(self.path.read_text(encoding="utf-8")) + shown = data["shown"] + stamp = data["last_attempt"] + if (isinstance(shown, list) and all(isinstance(x, str) for x in shown) + and (stamp is None or isinstance(stamp, (int, float)))): + self.shown = set(shown) + self.last_attempt = stamp + except (OSError, ValueError, KeyError, TypeError): + pass + + def save(self): + temporary = self.path.with_suffix(".tmp") + temporary.write_text(json.dumps({ + "shown": sorted(self.shown), "last_attempt": self.last_attempt, + }) + "\n", encoding="utf-8") + temporary.replace(self.path) + + def reconcile(self, plugs): + # Keep successful delivery across restarts, but forget disconnected plugs. + current = self.shown & plugs + if current != self.shown: + self.shown = current + self.save() + if not plugs - self.shown: + return + now = self.clock() + if self.last_attempt is not None and now - self.last_attempt < INTERVAL: + return + if not self.active(): + return + self.last_attempt = self.clock() + self.save() + if self.send(): + self.shown.update(plugs) + self.save() + + +def lookup_device(context, path): + import pyudev + + try: + return pyudev.Devices.from_sys_path(context, path) + except pyudev.DeviceNotFoundError: + return None + + +def collect(context, sysfs, device_from_path): + # Live front devices retain deduplication even if the Billboard function + # disappears and reappears while the dock stays connected. + live = set() + eligible = set() + for device in context.list_devices(subsystem="usb", DEVTYPE="usb_device"): + path = Path(device.sys_path) + front = front_device(path, sysfs) + if front is None: + continue + try: + parent = device_from_path(context, str(front)) + if parent is None: + continue + initialized = parent.properties.get("USEC_INITIALIZED", "") + if not initialized: + continue + identity = plug_id(front, initialized) + except (OSError, ValueError): + continue + if identity is None: + continue + live.add(identity) + if candidate(device, sysfs) is not None: + eligible.add(identity) + return live, eligible + + +def run(context, monitor, sysfs, state, device_from_path): + # Subscribe before enumeration so a plug during startup cannot be lost. + monitor.start() + while True: + live, eligible = collect(context, sysfs, device_from_path) + # Previously shown plugs remain tracked after their Billboard disappears. + state.reconcile(eligible | (live & state.shown)) + # Periodic reconciliation also handles login, failed delivery and a + # dropped uevent. Bound work during a burst of USB enumeration events. + deadline = time.monotonic() + 2 + while time.monotonic() < deadline: + monitor.poll(timeout=max(0, deadline - time.monotonic())) + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + sysfs = Path("/sys") + if not is_j700(sysfs): + return + if os.getuid() == 0: + raise SystemExit("Run this notifier in a regular user's graphical session") + if not os.environ.get("DBUS_SESSION_BUS_ADDRESS"): + raise SystemExit("The user session bus is unavailable") + runtime = Path(os.environ["XDG_RUNTIME_DIR"]) / "aurora-display-notifier" + runtime.mkdir(mode=0o700, exist_ok=True) + # One user manager, one session bus, one notifier, including manual starts. + with (runtime / "lock").open("w", encoding="ascii") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return + import pyudev + + context = pyudev.Context() + monitor = pyudev.Monitor.from_netlink(context, source="udev") + monitor.filter_by(subsystem="usb", device_type="usb_device") + state = Notifier(runtime / "state.json") + run(context, monitor, sysfs, state, lookup_device) + + +if __name__ == "__main__": + main() diff --git a/tools/aurora-display-notifier/org.aurorasilicon.DisplayHint.desktop b/tools/aurora-display-notifier/org.aurorasilicon.DisplayHint.desktop new file mode 100644 index 0000000000000..4197a66afdb24 --- /dev/null +++ b/tools/aurora-display-notifier/org.aurorasilicon.DisplayHint.desktop @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +[Desktop Entry] +Type=Application +Name=Aurora Display Hint +Comment=Explain the front USB-C port's display limitation +Exec=systemctl --user start aurora-display-notifier.service +Icon=video-display-symbolic +NoDisplay=true +X-GNOME-UsesNotifications=true diff --git a/tools/aurora-display-notifier/tests/integration_dbus.py b/tools/aurora-display-notifier/tests/integration_dbus.py new file mode 100644 index 0000000000000..8a7d9020a0410 --- /dev/null +++ b/tools/aurora-display-notifier/tests/integration_dbus.py @@ -0,0 +1,111 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise notify-send against a mock server in a PRIVATE dbus-run-session.""" +import os +from pathlib import Path +import threading + +from gi.repository import Gio, GLib + +from test_notifier import Topology, notifier + + +XML = """ + + + + + + + + + + + + + + +""" + + +def main(): + if os.environ.get("AURORA_TEST_PRIVATE_BUS") != "1": + raise SystemExit("Use dbus-run-session with AURORA_TEST_PRIVATE_BUS=1") + bus = Gio.bus_get_sync(Gio.BusType.SESSION, None) + result = bus.call_sync( + "org.freedesktop.DBus", "/org/freedesktop/DBus", "org.freedesktop.DBus", + "RequestName", GLib.Variant("(su)", ("org.freedesktop.Notifications", 4)), + GLib.VariantType.new("(u)"), Gio.DBusCallFlags.NONE, 5000, None, + ) + assert result.unpack() == (1,), "The private test bus must have no notification server" + calls = [] + + def method(connection, sender, path, interface, name, params, invocation): + if name == "GetCapabilities": + reply = GLib.Variant("(as)", (["body"],)) + elif name == "GetServerInformation": + reply = GLib.Variant("(ssss)", ("Test server", "Aurora", "1", "1.2")) + else: + assert name == "Notify" + calls.append(params.unpack()) + reply = GLib.Variant("(u)", (42,)) + invocation.return_value(reply) + + info = Gio.DBusNodeInfo.new_for_xml(XML).interfaces[0] + registration = bus.register_object("/org/freedesktop/Notifications", info, method, None, None) + loop = GLib.MainLoop() + errors = [] + + def exercise(): + fixture = Topology() + fixture.setUp() + try: + assert notifier.is_j700(fixture.sysfs) + events = [fixture.event(fixture.front, ""), fixture.event(fixture.billboard)] + context = type("Context", (), {"list_devices": lambda *args, **kwargs: events})() + parent = type("Device", (), {"properties": {"USEC_INITIALIZED": "1000"}})() + live, eligible = notifier.collect(context, fixture.sysfs, lambda *args: parent) + state = notifier.Notifier(Path(fixture.tmp.name) / "state.json", active=lambda: True) + state.reconcile(eligible) + state.reconcile(eligible) + # A service restart must retain deduplication. + restarted = notifier.Notifier(state.path, active=lambda: True) + restarted.reconcile(eligible) + # Remove the Billboard function; the dock's instance remains live. + events.pop() + live, eligible = notifier.collect(context, fixture.sysfs, lambda *args: parent) + restarted.reconcile(eligible | (live & restarted.shown)) + assert restarted.shown == state.shown + assert len(calls) == 1, calls + app, replacement, icon, summary, body, actions, hints, expiry = calls[0] + assert app == "Aurora Display Hint" + assert replacement == 0 + # libnotify versions may send the icon in app_icon or image-path. + assert (icon == "video-display-symbolic" + or hints.get("image-path") == "video-display-symbolic"), calls[0] + assert (summary, body) == (notifier.SUMMARY, notifier.BODY) + assert actions == [] + assert hints["urgency"] == 1 + assert hints["desktop-entry"] == "org.aurorasilicon.DisplayHint" + assert expiry == -1 # Respect the notification server's expiration policy. + except BaseException as error: + errors.append(error) + finally: + fixture.doCleanups() + GLib.idle_add(loop.quit) + + worker = threading.Thread(target=exercise, daemon=True) + worker.start() + timeout = GLib.timeout_add_seconds(15, loop.quit) + loop.run() + GLib.source_remove(timeout) + worker.join(timeout=1) + bus.unregister_object(registration) + assert not worker.is_alive(), "Timed out waiting for the notifier" + if errors: + raise errors[0] + print("PASS: synthetic front plug -> libnotify -> private D-Bus server; one notification") + + +if __name__ == "__main__": + main() diff --git a/tools/aurora-display-notifier/tests/test_notifier.py b/tools/aurora-display-notifier/tests/test_notifier.py new file mode 100644 index 0000000000000..e59d2d2b61b42 --- /dev/null +++ b/tools/aurora-display-notifier/tests/test_notifier.py @@ -0,0 +1,306 @@ +# SPDX-License-Identifier: GPL-2.0-only +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + + +spec = importlib.util.spec_from_file_location("notifier", Path(__file__).parents[1] / "notifier.py") +notifier = importlib.util.module_from_spec(spec) +spec.loader.exec_module(notifier) + + +class Topology(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.sysfs = Path(self.tmp.name).resolve() + compatible = self.sysfs / "firmware/devicetree/base/compatible" + compatible.parent.mkdir(parents=True) + compatible.write_bytes(b"apple,j700\0apple,t8140\0apple,arm-platform\0") + # Bus 7 deliberately differs from the observed bus 1. + root = self.sysfs / "devices/platform/soc/40a280000.usb/xhci-hcd.9.auto/usb7" + self.hub = self.device(root / "7-1", devpath="1", idVendor="2109", idProduct="2122") + self.front = self.device(self.hub / "7-1.1", devpath="1.1", devnum="4") + self.billboard = self.device(self.front / "7-1.1.5", devpath="1.1.5", bDeviceClass="11") + + def device(self, path, **attributes): + path.mkdir(parents=True, exist_ok=True) + for name, value in attributes.items(): + (path / name).write_text(value + "\n", encoding="ascii") + return path + + def event(self, path, marker="1", kind="usb_device"): + return SimpleNamespace(sys_path=str(path), device_type=kind, + properties={notifier.MARKER: marker}) + + def test_machine_requires_exact_compatible_token(self): + self.assertTrue(notifier.is_j700(self.sysfs)) + (self.sysfs / "firmware/devicetree/base/compatible").write_bytes(b"apple,j700-other\0") + self.assertFalse(notifier.is_j700(self.sysfs)) + + def test_machine_missing(self): + self.assertFalse(notifier.is_j700(self.sysfs / "missing")) + + def test_front_nested_billboard(self): + self.assertEqual(notifier.candidate(self.event(self.billboard), self.sysfs), self.front) + + def test_direct_front_billboard(self): + (self.front / "bDeviceClass").write_text("11\n") + self.assertEqual(notifier.candidate(self.event(self.front), self.sysfs), self.front) + + def test_rear_billboard_is_ignored_even_with_marker(self): + rear = self.device(self.hub / "7-1.2", devpath="1.2", bDeviceClass="11") + nested = self.device(rear / "7-1.2.5", devpath="1.2.5", bDeviceClass="11") + for path in (rear, nested): + self.assertIsNone(notifier.candidate(self.event(path), self.sysfs)) + + def test_requires_marker_and_device_class(self): + for marker, kind in (("", "usb_device"), ("1", "usb_interface")): + self.assertIsNone(notifier.candidate(self.event(self.billboard, marker, kind), self.sysfs)) + (self.billboard / "bDeviceClass").write_text("09\n") + self.assertIsNone(notifier.candidate(self.event(self.billboard), self.sysfs)) + + def test_wrong_fixed_hub_is_ignored(self): + (self.hub / "idProduct").write_text("0817\n") + self.assertIsNone(notifier.candidate(self.event(self.billboard), self.sysfs)) + + def test_wrong_controller_is_ignored(self): + controller = self.hub.parents[2] + controller.rename(controller.with_name("other.usb")) + path = controller.with_name("other.usb") / self.billboard.relative_to(controller) + self.assertIsNone(notifier.candidate(self.event(path), self.sysfs)) + + def test_removed_device_and_outside_sysfs_are_ignored(self): + self.assertIsNone(notifier.candidate(self.event(self.billboard / "gone"), self.sysfs)) + self.assertIsNone(notifier.front_device(Path("/tmp"), self.sysfs)) + + def test_plug_identity_changes_with_device_number(self): + first = notifier.plug_id(self.front, "1000") + (self.front / "devnum").write_text("5\n") + self.assertNotEqual(first, notifier.plug_id(self.front, "1001")) + (self.front / "devnum").unlink() + self.assertIsNone(notifier.plug_id(self.front, "1001")) + + def test_collect_coalesces_and_retains_live_front(self): + second = self.device(self.front / "7-1.1.6", devpath="1.1.6", bDeviceClass="11") + events = [self.event(self.front, ""), self.event(self.billboard), self.event(second)] + context = SimpleNamespace(list_devices=lambda **kwargs: events) + parent = SimpleNamespace(properties={"USEC_INITIALIZED": "1000"}) + live, eligible = notifier.collect(context, self.sysfs, lambda *args: parent) + self.assertEqual(live, eligible) + self.assertEqual(len(eligible), 1) + # Billboard removal does not end the dock's plug. + del events[1:] + live2, eligible2 = notifier.collect(context, self.sysfs, lambda *args: parent) + self.assertEqual(live, live2) + self.assertEqual(eligible2, set()) + + def test_collect_waits_for_front_initialization(self): + context = SimpleNamespace(list_devices=lambda **kwargs: [self.event(self.billboard)]) + parent = SimpleNamespace(properties={}) + self.assertEqual(notifier.collect(context, self.sysfs, lambda *args: parent), (set(), set())) + + def test_collect_tolerates_unplug_race(self): + context = SimpleNamespace(list_devices=lambda **kwargs: [self.event(self.billboard)]) + + def disappeared(*args): + raise OSError("device disappeared") + + self.assertEqual(notifier.collect(context, self.sysfs, disappeared), (set(), set())) + self.assertEqual(notifier.collect(context, self.sysfs, lambda *args: None), (set(), set())) + + def test_lookup_handles_library_unplug_error(self): + class DeviceGone(Exception): + pass + + def disappeared(*args): + raise DeviceGone() + + library = SimpleNamespace(DeviceNotFoundError=DeviceGone, + Devices=SimpleNamespace(from_sys_path=disappeared)) + with patch.dict(sys.modules, {"pyudev": library}): + self.assertIsNone(notifier.lookup_device(None, str(self.front))) + + +class Delivery(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.path = Path(self.tmp.name) / "state.json" + self.now = 100.0 + self.sent = 0 + self.success = True + self.active = True + self.state = self.new_state() + + def new_state(self): + return notifier.Notifier(self.path, self.send, lambda: self.active, lambda: self.now) + + def send(self): + self.sent += 1 + return self.success + + def test_once_per_plug_including_restarts(self): + self.state.reconcile({"plug-a"}) + self.now += 60 + self.state.reconcile({"plug-a"}) + self.new_state().reconcile({"plug-a"}) + self.assertEqual(self.sent, 1) + + def test_replug_waits_for_rate_limit(self): + self.state.reconcile({"plug-a"}) + self.state.reconcile(set()) + self.now += 1 + self.state.reconcile({"plug-b"}) + self.assertEqual(self.sent, 1) + self.now += 29 + self.state.reconcile({"plug-b"}) + self.assertEqual(self.sent, 2) + + def test_slow_session_check_does_not_shorten_rate_limit(self): + sends = [] + checks = [] + + def active(): + checks.append(self.now) + if len(checks) == 1: + self.now += 8 + return True + + def send(): + sends.append(self.now) + # The dispatch timestamp must already be persisted for restarts. + self.assertEqual(json.loads(self.path.read_text())["last_attempt"], self.now) + return True + + state = notifier.Notifier(self.path, send, active, lambda: self.now) + state.reconcile({"plug-a"}) + state.reconcile(set()) + state = notifier.Notifier(self.path, send, active, lambda: self.now) + self.now = 130.0 + state.reconcile({"plug-b"}) + self.assertEqual(sends, [108.0]) + self.assertEqual(checks, [100.0]) + self.now = 138.0 + state.reconcile({"plug-b"}) + self.assertEqual(sends, [108.0, 138.0]) + self.assertGreaterEqual(sends[1] - sends[0], notifier.INTERVAL) + + def test_disconnected_pending_plug_is_never_delivered(self): + self.state.reconcile({"plug-a"}) + self.now += 1 + self.state.reconcile({"plug-b"}) + self.state.reconcile(set()) + self.now += 60 + self.state.reconcile(set()) + self.assertEqual(self.sent, 1) + + def test_inactive_session_defers_until_active(self): + self.active = False + self.state.reconcile({"plug-a"}) + self.assertEqual(self.sent, 0) + self.active = True + self.state.reconcile({"plug-a"}) + self.assertEqual(self.sent, 1) + + def test_delivery_failure_retries_with_rate_limit(self): + self.success = False + self.state.reconcile({"plug-a"}) + self.assertEqual(self.state.shown, set()) + self.now += 29 + self.new_state().reconcile({"plug-a"}) + self.assertEqual(self.sent, 1) + self.success = True + self.now += 1 + self.state.reconcile({"plug-a"}) + self.assertEqual(self.sent, 2) + self.assertEqual(self.state.shown, {"plug-a"}) + + def test_corrupt_state_is_recoverable(self): + for data in ("invalid", "[]", '{"shown": false, "last_attempt": []}'): + self.path.write_text(data) + state = self.new_state() + self.assertEqual(state.shown, set()) + self.assertIsNone(state.last_attempt) + + def test_state_write_failure_prevents_untracked_send(self): + with patch.object(self.state, "save", side_effect=OSError): + with self.assertRaises(OSError): + self.state.reconcile({"plug-a"}) + self.assertEqual(self.sent, 0) + + def test_empty_plugs_never_checks_session_or_notifies(self): + with patch.object(self.state, "active", side_effect=AssertionError): + self.state.reconcile(set()) + self.assertFalse(self.path.exists()) + + def test_state_contains_no_peripheral_names(self): + self.state.reconcile({"sysfs-path:4:123:1000"}) + data = json.loads(self.path.read_text()) + self.assertEqual(set(data), {"shown", "last_attempt"}) + + +class SessionAndTransport(unittest.TestCase): + def test_other_machine_never_checks_session(self): + with patch.object(notifier, "is_j700", return_value=False), \ + patch.object(notifier.os, "getuid", side_effect=AssertionError): + notifier.main() + + def test_root_and_missing_bus_never_start_notifier(self): + with patch.object(notifier, "is_j700", return_value=True), \ + patch.object(notifier.os, "getuid", return_value=0): + with self.assertRaises(SystemExit): + notifier.main() + with patch.object(notifier, "is_j700", return_value=True), \ + patch.object(notifier.os, "getuid", return_value=1000), \ + patch.dict(os.environ, {}, clear=True): + with self.assertRaises(SystemExit): + notifier.main() + + def test_session_gate(self): + valid = "User=1000\nActive=yes\nRemote=no\nType=wayland\nSeat=seat0\n" + cases = [(valid, True)] + [ + (valid.replace(old, new), False) for old, new in ( + ("User=1000", "User=1001"), ("Active=yes", "Active=no"), + ("Remote=no", "Remote=yes"), ("Type=wayland", "Type=tty"), + ("Seat=seat0", "Seat="), + ) + ] + cases.append((valid.replace("wayland", "x11"), True)) + for reply, expected in cases: + with self.subTest(reply=reply), patch.object(notifier, "command", side_effect=["3\n", reply]): + self.assertEqual(notifier.active_desktop(1000), expected) + + def test_logind_errors_fail_closed(self): + for error in (FileNotFoundError(), subprocess.TimeoutExpired("loginctl", 5)): + with patch.object(notifier, "command", side_effect=error): + self.assertFalse(notifier.active_desktop(os.getuid())) + + def test_notification_is_plain_libnotify_on_inherited_bus(self): + with patch.object(notifier, "command", return_value="") as call: + self.assertTrue(notifier.notify()) + args = call.call_args.args[0] + self.assertEqual(args[-3:], ["--", notifier.SUMMARY, notifier.BODY]) + self.assertIn("--urgency=normal", args) + self.assertNotIn("--expire-time", " ".join(args)) + + def test_notification_transport_failure(self): + with patch.object(notifier, "command", side_effect=subprocess.TimeoutExpired("notify-send", 5)): + self.assertFalse(notifier.notify()) + + def test_subprocess_is_bounded_without_shell(self): + with patch.object(subprocess, "run", return_value=SimpleNamespace(stdout="ok")) as run: + self.assertEqual(notifier.command(["example", "literal"]), "ok") + self.assertEqual(run.call_args.kwargs["timeout"], 5) + self.assertNotIn("shell", run.call_args.kwargs) + + +if __name__ == "__main__": + unittest.main() From c6466de02fc51a0de273dddba59af7df9a9c46f6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 02:05:56 +1100 Subject: [PATCH 1113/1579] iommu: apple-dart: Drain runtime PM before injecting test errors The gated-IRQ test injects a runtime-PM error after a successful IRQ, whose pm_runtime_put() queues asynchronous idle work. A pending PM callback can overwrite runtime_error after the injection, allowing the next IRQ to run its handler and making the test fail intermittently. Holding power.lock during injection does not wait for that callback. Drain runtime-PM work before injecting the error so that the failed-resume case reliably exercises the error path. Keep the IRQ, register and usage count assertions. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 86ed79c8d0c1d..91c72a618a579 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -397,6 +397,9 @@ static void apple_dart_test_gated_irq(struct kunit *test) KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 0)), 0x1234U); KUNIT_EXPECT_EQ(test, atomic_read(&dart->dev->power.usage_count), 0); + /* Drain the work queued by the successful IRQ's pm_runtime_put(). */ + pm_runtime_barrier(dart->dev); + /* Failed runtime resume must not run the handler or leak its reference. */ writel(0, dart->regs + DART_TCR(dart, 0)); scoped_guard(spinlock_irqsave, &dart->dev->power.lock) From 82494bf5c6b07739782eeac837b9f6a96c70ca36 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:10:49 +1100 Subject: [PATCH 1114/1579] iommu/apple-dart: Do not reset a DART that kept its state across sleep apple_dart_resume() resets every DART on resume, clearing its page table pointers and disabling translation before restoring the saved values. That is only correct when the DART lost power. A DART whose power domain stayed on still has its state, and may still be translating for a consumer that kept running through the sleep: dwc3-apple keeps its core up across system sleep as a syscore device, which holds the shared power domain on, and with xHCI now left running in suspend-to-idle the controller can write events or touch its scratchpad while the DART is resumed. A DMA that hits the reset window faults, and xHCI halts on a Host System Error. Compare the translation registers with the values saved at suspend. When they still match and at least one page table is valid, the DART did not lose power: leave it alone and only invalidate its TLBs. Otherwise reset and restore it as before. A powered-off DART reads back with no valid page table, so it always takes the reset path. The TLBs are flushed with one command per stream rather than one command for a map of all streams. Where the streams of a map are remapped onto its first stream, a map flush only reaches that first stream, which would leave stale entries of every other stream behind. Two consequences beyond the USB case: - The suspend and resume callbacks are shared by runtime PM and system sleep. Every runtime resume of an unlocked DART whose power domain stayed on now also skips the reset and only invalidates the TLBs. This applies to all DARTs the driver handles, on other Apple SoCs as well, including the t8110 ones. - "Retained" is judged only by TCR and TTBR matching the values saved at suspend. On that path the error status and the stream enables are no longer reset; they are left as the DART kept them. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 91 +++++++++++++++++++++++++++++++++ drivers/iommu/apple-dart.c | 56 ++++++++++++++++++++ 2 files changed, 147 insertions(+) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 91c72a618a579..7dfb81afe8366 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -411,6 +411,93 @@ static void apple_dart_test_gated_irq(struct kunit *test) pm_runtime_disable(dart->dev); } +static struct apple_dart *apple_dart_test_sleeping(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + unsigned int sid; + + dart->tunneled = false; + dart->hw = &apple_dart_hw_t8103; + dart->num_streams = 16; + for (sid = 0; sid < dart->num_streams; sid++) + writel(0x80 + sid, dart->regs + DART_TCR(dart, sid)); + writel(dart->hw->ttbr_valid | 0x1000, dart->regs + DART_TTBR(dart, 2, 0)); + KUNIT_ASSERT_EQ(test, apple_dart_suspend(dart->dev), 0); + return dart; +} + +static void apple_dart_test_resume_retained(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_sleeping(test); + u32 enable = dart->hw->enable_streams; + + /* The domain stayed on and a consumer may still be doing DMA: no reset. */ + writel(0, dart->regs + enable); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + enable), 0U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 2)), 0x82U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 2, 0)), + dart->hw->ttbr_valid | 0x1000); +} + +static void apple_dart_test_resume_lost(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_sleeping(test); + + /* Power was cut: reset and restore the saved translations. */ + memset_io(dart->regs, 0, SZ_16K); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + dart->hw->enable_streams), U32_MAX); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 2)), 0x82U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 2, 0)), + dart->hw->ttbr_valid | 0x1000); +} + +static void apple_dart_test_resume_unused(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + + /* Matching registers without any page table prove nothing: reset. */ + dart->tunneled = false; + dart->hw = &apple_dart_hw_t8103; + dart->num_streams = 16; + KUNIT_ASSERT_EQ(test, apple_dart_suspend(dart->dev), 0); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + dart->hw->enable_streams), U32_MAX); +} + +/* + * A DART that kept its state is flushed stream by stream. Where the streams of + * a map are remapped onto its first stream, as on DARTs that program only that + * stream, a single flush of all streams would reach stream 0 and nothing else. + */ +static void apple_dart_test_resume_retained_each_stream(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + unsigned int sid; + u32 cmd; + + dart->tunneled = false; + dart->hw = &apple_dart_hw_t8110; + dart->num_streams = 16; + for (sid = 0; sid < dart->num_streams; sid++) + writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid)); + writel(dart->hw->ttbr_valid | 0x1000, dart->regs + DART_TTBR(dart, 0, 0)); + writel(dart->hw->ttbr_valid | 0x2000, dart->regs + DART_TTBR(dart, 5, 0)); + KUNIT_ASSERT_EQ(test, apple_dart_suspend(dart->dev), 0); + + writel(0, dart->regs + DART_T8110_TLB_CMD); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + /* Commands go out in stream order: the last one names the last stream. */ + cmd = readl(dart->regs + DART_T8110_TLB_CMD); + KUNIT_EXPECT_EQ(test, FIELD_GET(DART_T8110_TLB_CMD_STREAM, cmd), dart->num_streams - 1); + KUNIT_EXPECT_EQ(test, FIELD_GET(DART_T8110_TLB_CMD_OP, cmd), + (u32)DART_T8110_TLB_CMD_OP_FLUSH_SID); + /* Nothing was reset. */ + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 5, 0)), + dart->hw->ttbr_valid | 0x2000); +} + static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_before_attach), KUNIT_CASE(apple_dart_test_locked_handoff), @@ -423,6 +510,10 @@ static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_tunnel_reset_failure), KUNIT_CASE(apple_dart_test_failed_host_pm), KUNIT_CASE(apple_dart_test_gated_irq), + KUNIT_CASE(apple_dart_test_resume_retained), + KUNIT_CASE(apple_dart_test_resume_lost), + KUNIT_CASE(apple_dart_test_resume_unused), + KUNIT_CASE(apple_dart_test_resume_retained_each_stream), {} }; diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 6a32a91f61d09..d790f9318a835 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -3008,6 +3008,58 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) return 0; } +/* + * Whether the DART still holds the translation state that apple_dart_suspend() + * saved, because its power domain stayed on across system sleep. That is the + * case while a consumer that keeps running in suspend-to-idle, such as a DWC3 + * controller in host mode, holds the domain on. Such a consumer may still be + * doing DMA, so the DART must not be reset under it. + */ +static bool apple_dart_state_retained(struct apple_dart *dart) +{ + bool translating = false; + unsigned int sid, idx; + + for (sid = 0; sid < dart->num_streams; sid++) { + if (apple_dart_readl(dart, DART_TCR(dart, sid)) != dart->save_tcr[sid]) + return false; + + for (idx = 0; idx < dart->hw->ttbr_count; idx++) { + u32 ttbr = apple_dart_readl(dart, DART_TTBR(dart, sid, idx)); + + if (ttbr != dart->save_ttbr[sid][idx]) + return false; + if (ttbr & dart->hw->ttbr_valid) + translating = true; + } + } + + /* A DART that lost power reads back as reset, without valid page tables */ + return translating; +} + +/* + * Flush the TLBs of every stream with a command of its own. A stream map flush + * is not enough for the whole DART: where the streams of a map are remapped + * onto its first stream, it only reaches that first stream. + */ +static int apple_dart_invalidate_each_stream(struct apple_dart *dart) +{ + unsigned int sid; + int ret; + + for (sid = 0; sid < dart->num_streams; sid++) { + struct apple_dart_stream_map stream_map = { .dart = dart }; + + __set_bit(sid, stream_map.sidmap); + ret = dart->hw->invalidate_tlb(&stream_map); + if (ret) + return ret; + } + + return 0; +} + static __maybe_unused int apple_dart_resume(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); @@ -3032,6 +3084,10 @@ static __maybe_unused int apple_dart_resume(struct device *dev) if (dart->locked) return 0; + /* Nothing was lost, only drop what the TLBs may still hold. */ + if (apple_dart_state_retained(dart)) + return apple_dart_invalidate_each_stream(dart); + ret = apple_dart_hw_reset(dart); if (ret) { dev_err(dev, "Failed to reset DART on resume\n"); From 39bc76211b30ca23b29eb740ffdf0ae8cf2ed5d3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:11:32 +1100 Subject: [PATCH 1115/1579] iommu/apple-dart: Keep a retained DART in bypass from being reset The resume path only treats a DART as having kept its state when a stream has a valid page table. A stream of an identity domain bypasses the DART without one, for example with iommu.passthrough=1 or where the default domain type is identity. Such a DART reads back exactly as it was saved, its consumer may still be doing DMA, and it was reset all the same, which disables DMA on every stream before the saved state is restored. Count a stream whose TCR still holds the bypass setting of its generation as live as well, on DARTs that support bypass. A DART that lost power reads back with neither, so it still takes the reset path. This completes "iommu/apple-dart: Do not reset a DART that kept its state across sleep". Test it for a T8103 and a T8110 DART; both tests fail without this change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 36 +++++++++++++++++++++++++++++++++ drivers/iommu/apple-dart.c | 18 ++++++++++++----- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 7dfb81afe8366..b5d4918b395b9 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -498,6 +498,40 @@ static void apple_dart_test_resume_retained_each_stream(struct kunit *test) dart->hw->ttbr_valid | 0x2000); } +/* + * A stream of an identity domain bypasses the DART without a page table. Its + * consumer may still be doing DMA, so a DART that kept that state is not reset. + */ +static void apple_dart_test_resume_retained_bypass_hw(struct kunit *test, + const struct apple_dart_hw *hw) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + u32 enable; + + dart->tunneled = false; + dart->hw = hw; + dart->num_streams = 16; + dart->supports_bypass = true; + enable = dart->hw->enable_streams; + writel(dart->hw->tcr_bypass, dart->regs + DART_TCR(dart, 3)); + KUNIT_ASSERT_EQ(test, apple_dart_suspend(dart->dev), 0); + + writel(0, dart->regs + enable); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + enable), 0U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 3)), dart->hw->tcr_bypass); +} + +static void apple_dart_test_resume_retained_bypass_t8103(struct kunit *test) +{ + apple_dart_test_resume_retained_bypass_hw(test, &apple_dart_hw_t8103); +} + +static void apple_dart_test_resume_retained_bypass_t8110(struct kunit *test) +{ + apple_dart_test_resume_retained_bypass_hw(test, &apple_dart_hw_t8110); +} + static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_before_attach), KUNIT_CASE(apple_dart_test_locked_handoff), @@ -514,6 +548,8 @@ static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_resume_lost), KUNIT_CASE(apple_dart_test_resume_unused), KUNIT_CASE(apple_dart_test_resume_retained_each_stream), + KUNIT_CASE(apple_dart_test_resume_retained_bypass_t8103), + KUNIT_CASE(apple_dart_test_resume_retained_bypass_t8110), {} }; diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index d790f9318a835..60499b636337d 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -3017,12 +3017,17 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) */ static bool apple_dart_state_retained(struct apple_dart *dart) { - bool translating = false; + bool live = false; unsigned int sid, idx; for (sid = 0; sid < dart->num_streams; sid++) { - if (apple_dart_readl(dart, DART_TCR(dart, sid)) != dart->save_tcr[sid]) + u32 tcr = apple_dart_readl(dart, DART_TCR(dart, sid)); + + if (tcr != dart->save_tcr[sid]) return false; + /* An identity domain bypasses the DART without any page table. */ + if (dart->supports_bypass && dart->hw->tcr_bypass && tcr == dart->hw->tcr_bypass) + live = true; for (idx = 0; idx < dart->hw->ttbr_count; idx++) { u32 ttbr = apple_dart_readl(dart, DART_TTBR(dart, sid, idx)); @@ -3030,12 +3035,15 @@ static bool apple_dart_state_retained(struct apple_dart *dart) if (ttbr != dart->save_ttbr[sid][idx]) return false; if (ttbr & dart->hw->ttbr_valid) - translating = true; + live = true; } } - /* A DART that lost power reads back as reset, without valid page tables */ - return translating; + /* + * A DART that lost power reads back as reset: no stream bypasses it and + * none has a valid page table. + */ + return live; } /* From d9fc214474191909db140a5ba565a014819bd6de Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 08:55:11 +1100 Subject: [PATCH 1116/1579] iommu/apple-dart: Restore the translations when the reset on resume fails apple_dart_resume() returns as soon as apple_dart_hw_reset() fails. By then the reset has disabled translation on every stream and cleared the page table pointers, so the DART is left without any translation and every DMA of its consumers faults until something resets it again. On the J700 this happened to the DART of the PCIe radio: "TLB command 0x1 for stream 0 timed out" on resume, after which the Bluetooth function could not restore its transport and Wi-Fi lost its DMA until a reboot. The reset can only fail in its last steps, after the translation registers are cleared: the unlock of a T8110 v2.2 or later DART that locks on errors, and the TLB flush. Move those steps, with the error acknowledgment and unmasking around the unlock, into apple_dart_hw_finish_reset(). It leaves the translation registers alone and reports the step that failed. On resume, write the saved page table pointers and translation controls back whatever the reset returned. They still describe the page tables in use, and this path is only taken when the DART did not keep its state, so its TLBs hold nothing from before the sleep. Then, if the reset failed, repeat all of its last steps once, not only the flush. When the unlock fails, the reset returns before it unmasks the error interrupt and flushes the TLBs, and a flush that completes later says nothing about either. If they fail again, resume returns the error. After a failed system resume the PM core leaves the DART runtime suspended, with the restored registers matching the saved ones, so its next runtime resume would take the path for a DART that kept its state and only flush the TLBs. Have that resume repeat the last steps of the reset instead. The tests cover an unlock that times out while the command engine is busy, followed by stream flushes that complete, through a KUnit stub of the unlock, and an unlock that never gets through. They size the fake register space for the T8110 layout, whose exception records start at 16 KiB, and put a guard page after it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 181 +++++++++++++++++++++++++++++++- drivers/iommu/apple-dart.c | 94 ++++++++++++----- 2 files changed, 246 insertions(+), 29 deletions(-) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index b5d4918b395b9..c953f1da07bb3 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only /* Included by apple-dart.c to exercise domain and command lifecycles. */ #include +#include #include +#include static void apple_dart_test_before_attach(struct kunit *test) { @@ -246,16 +248,28 @@ static void apple_dart_test_resume_commands_failure(struct kunit *test) pm_runtime_disable(dart->dev); } +/* + * Large enough for the T8110 layout, whose exception records reach up to SID + * 255 below 0xc000. The page after it is vmalloc's guard, so an access beyond + * any layout faults instead of reaching other memory. + */ +#define APPLE_DART_TEST_REGS_SIZE SZ_64K + +KUNIT_DEFINE_ACTION_WRAPPER(apple_dart_test_vfree, vfree, const void *); + static struct apple_dart *apple_dart_test_tunnel(struct kunit *test) { struct apple_dart *dart; + void *regs; dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, dart); dart->dev = kunit_device_register(test, "apple-dart-tunnel-test"); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); - dart->regs = (__force void __iomem *)kunit_kzalloc(test, SZ_16K, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, dart->regs); + regs = vzalloc(APPLE_DART_TEST_REGS_SIZE); + KUNIT_ASSERT_NOT_NULL(test, regs); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, apple_dart_test_vfree, regs), 0); + dart->regs = (__force void __iomem *)regs; dart->hw = &apple_dart_hw_t8103_usb4; dart->num_streams = dart->hw->max_sid_count; dart->tunneled = true; @@ -279,7 +293,7 @@ static void apple_dart_test_tunnel_restore(struct kunit *test) dart->regs + DART_TTBR(dart, sid, idx)); } KUNIT_ASSERT_EQ(test, apple_dart_save_tunnel_state(dart->dev), 0); - memset_io(dart->regs, 0, SZ_16K); + memset_io(dart->regs, 0, APPLE_DART_TEST_REGS_SIZE); KUNIT_ASSERT_EQ(test, apple_dart_restore_tunnel_state(dart->dev), 0); for (sid = 0; sid < dart->num_streams; sid++) { KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, sid)), @@ -445,7 +459,7 @@ static void apple_dart_test_resume_lost(struct kunit *test) struct apple_dart *dart = apple_dart_test_sleeping(test); /* Power was cut: reset and restore the saved translations. */ - memset_io(dart->regs, 0, SZ_16K); + memset_io(dart->regs, 0, APPLE_DART_TEST_REGS_SIZE); KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); KUNIT_EXPECT_EQ(test, readl(dart->regs + dart->hw->enable_streams), U32_MAX); KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 2)), 0x82U); @@ -532,6 +546,161 @@ static void apple_dart_test_resume_retained_bypass_t8110(struct kunit *test) apple_dart_test_resume_retained_bypass_hw(test, &apple_dart_hw_t8110); } +/* + * The PCIe DART of the J700 radio: two streams translating for a function + * that keeps running in suspend-to-idle. + */ +static struct apple_dart *apple_dart_test_radio(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_tunnel(test); + unsigned int sid; + + dart->tunneled = false; + dart->hw = &apple_dart_hw_t8110; + dart->num_streams = 16; + dart->version = 0x0202; + for (sid = 1; sid <= 2; sid++) { + writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid)); + writel(dart->hw->ttbr_valid | (0x1000 * sid), + dart->regs + DART_TTBR(dart, sid, 0)); + } + KUNIT_ASSERT_EQ(test, apple_dart_suspend(dart->dev), 0); + return dart; +} + +static void apple_dart_test_radio_flush_timeout(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_radio(test); + u32 busy = DART_T8110_TLB_CMD_BUSY | + FIELD_PREP(DART_T8110_TLB_CMD_OP, DART_T8110_TLB_CMD_OP_FLUSH_SID); + + /* A flush that never completes must not cost the streams their tables. */ + writel(busy, dart->regs + DART_T8110_TLB_CMD); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 1)), dart->hw->tcr_enabled); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 2, 0)), + dart->hw->ttbr_valid | 0x2000); + + /* The same after a power loss, where the DART is reset first. */ + memset_io(dart->regs, 0, APPLE_DART_TEST_REGS_SIZE); + writel(busy, dart->regs + DART_T8110_TLB_CMD); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 2)), dart->hw->tcr_enabled); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 1, 0)), + dart->hw->ttbr_valid | 0x1000); +} + +/* The radio DART after a power loss, set to lock on errors. */ +static struct apple_dart *apple_dart_test_radio_lost(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_radio(test); + + memset_io(dart->regs, 0, APPLE_DART_TEST_REGS_SIZE); + writel(DART_T8110_DIAG_LOCK_ON_ERR, dart->regs + DART_T8110_DIAG_LOCK); + /* The reset unmasks the error interrupt only after the unlock. */ + writel(0x55, dart->regs + DART_T8110_ERROR_MASK); + return dart; +} + +/* The last command flushed the last stream, and nothing is pending. */ +static void apple_dart_test_expect_flushed(struct kunit *test, struct apple_dart *dart) +{ + u32 cmd = readl(dart->regs + DART_T8110_TLB_CMD); + + KUNIT_EXPECT_FALSE(test, cmd & DART_T8110_TLB_CMD_BUSY); + KUNIT_EXPECT_EQ(test, FIELD_GET(DART_T8110_TLB_CMD_OP, cmd), + (u32)DART_T8110_TLB_CMD_OP_FLUSH_SID); + KUNIT_EXPECT_EQ(test, FIELD_GET(DART_T8110_TLB_CMD_STREAM, cmd), dart->num_streams - 1); +} + +static int apple_dart_test_unlocks; + +/* + * The command engine is still busy when the reset unlocks the DART, and goes + * idle just after the unlock gave up waiting for it. + */ +static int apple_dart_test_unlock_late(struct apple_dart *dart) +{ + struct kunit *test = kunit_get_current_test(); + bool first = !apple_dart_test_unlocks++; + int ret; + + if (first) + writel(DART_T8110_TLB_CMD_BUSY, dart->regs + DART_T8110_TLB_CMD); + kunit_deactivate_static_stub(test, apple_dart_t8110_v2_unlock); + ret = apple_dart_t8110_v2_unlock(dart); + kunit_activate_static_stub(test, apple_dart_t8110_v2_unlock, + apple_dart_test_unlock_late); + if (first) + writel(0, dart->regs + DART_T8110_TLB_CMD); + return ret; +} + +static void apple_dart_test_radio_unlock_timeout(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_radio_lost(test); + + apple_dart_test_unlocks = 0; + kunit_activate_static_stub(test, apple_dart_t8110_v2_unlock, + apple_dart_test_unlock_late); + + /* The stream flushes after the failed unlock complete... */ + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + apple_dart_test_expect_flushed(test, dart); + /* ...but the reset only ends once the unlock and the error mask are done. */ + KUNIT_EXPECT_EQ(test, apple_dart_test_unlocks, 2); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_ERROR_MASK), 0U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 1)), dart->hw->tcr_enabled); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 2, 0)), + dart->hw->ttbr_valid | 0x2000); +} + +static void apple_dart_test_radio_unlock_stuck(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_radio_lost(test); + + /* An unlock that never gets through fails resume, with the tables back. */ + writel(DART_T8110_TLB_CMD_BUSY, dart->regs + DART_T8110_TLB_CMD); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_ERROR_MASK), 0x55U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 2)), dart->hw->tcr_enabled); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 1, 0)), + dart->hw->ttbr_valid | 0x1000); + + /* + * The next resume finds the tables as they were saved, and must still + * unlock the DART and unmask its errors, not only flush the TLBs. + */ + writel(0, dart->regs + DART_T8110_TLB_CMD); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + apple_dart_test_expect_flushed(test, dart); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_ERROR_MASK), 0U); +} + +static int apple_dart_test_flush_calls; + +static int apple_dart_test_flush_once_late(struct apple_dart_stream_map *stream_map) +{ + return apple_dart_test_flush_calls++ ? 0 : -ETIMEDOUT; +} + +static void apple_dart_test_resume_reset_retry(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_sleeping(test); + struct apple_dart_hw hw = apple_dart_hw_t8103; + + /* The reset's flush times out once; the second flush completes. */ + hw.invalidate_tlb = apple_dart_test_flush_once_late; + dart->hw = &hw; + apple_dart_test_flush_calls = 0; + memset_io(dart->regs, 0, APPLE_DART_TEST_REGS_SIZE); + KUNIT_EXPECT_EQ(test, apple_dart_resume(dart->dev), 0); + KUNIT_EXPECT_GT(test, apple_dart_test_flush_calls, 1); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TCR(dart, 2)), 0x82U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_TTBR(dart, 2, 0)), + dart->hw->ttbr_valid | 0x1000); +} + static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_before_attach), KUNIT_CASE(apple_dart_test_locked_handoff), @@ -548,6 +717,10 @@ static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_resume_lost), KUNIT_CASE(apple_dart_test_resume_unused), KUNIT_CASE(apple_dart_test_resume_retained_each_stream), + KUNIT_CASE(apple_dart_test_radio_flush_timeout), + KUNIT_CASE(apple_dart_test_radio_unlock_timeout), + KUNIT_CASE(apple_dart_test_radio_unlock_stuck), + KUNIT_CASE(apple_dart_test_resume_reset_retry), KUNIT_CASE(apple_dart_test_resume_retained_bypass_t8103), KUNIT_CASE(apple_dart_test_resume_retained_bypass_t8110), {} diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 60499b636337d..97f73ed930258 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -11,6 +11,7 @@ * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. */ +#include #include #include #include @@ -280,6 +281,8 @@ struct apple_dart { u32 tunnel_state_saved : 1; u32 tunnel_state_restored : 1; bool commands_gated; + /* Resume restored the translations, but the last steps of its reset failed. */ + bool reset_incomplete; struct apple_tunable *tunables; u16 version; @@ -1157,6 +1160,8 @@ static int apple_dart_t8110_v2_unlock(struct apple_dart *dart) { struct apple_dart_stream_map stream_map = { .dart = dart }; + KUNIT_STATIC_STUB_REDIRECT(apple_dart_t8110_v2_unlock, dart); + if (!(readl(dart->regs + DART_T8110_DIAG_LOCK) & DART_T8110_DIAG_LOCK_ON_ERR)) return 0; @@ -1209,6 +1214,41 @@ static void apple_dart_hw_apply_remaps(struct apple_dart *dart) dart->regs + DART_TCR(dart, sid)); } +/* + * The last steps of a reset, which leave the translation registers alone: + * acknowledge the errors latched so far, unlock a DART that locks on errors, + * unmask the error interrupt and flush the TLBs. + */ +static int apple_dart_hw_finish_reset(struct apple_dart *dart) +{ + struct apple_dart_stream_map stream_map = { .dart = dart }; + int ret; + + /* Newer revisions expose per-stream exception records. */ + if (dart->hw->type == DART_T8110 && dart->version >= 0x0202) { + apple_dart_drain_stale_exceptions(dart); + ret = apple_dart_t8110_v2_unlock(dart); + if (ret) { + dev_err(dart->dev, "failed to unlock after reset: %d\n", ret); + return ret; + } + } else { + /* Clear pending errors before the interrupt is unmasked. */ + apple_dart_writel(dart, apple_dart_readl(dart, dart->hw->error), + dart->hw->error); + } + + if (dart->hw->type == DART_T8110) + apple_dart_writel(dart, 0, DART_T8110_ERROR_MASK); + + bitmap_set(stream_map.sidmap, 0, dart->num_streams); + ret = dart->hw->invalidate_tlb(&stream_map); + if (ret) + dev_err(dart->dev, "failed to flush the TLBs after reset: %d\n", ret); + + return ret; +} + static int apple_dart_hw_reset(struct apple_dart *dart) { struct apple_dart_stream_map stream_map; @@ -1227,24 +1267,7 @@ static int apple_dart_hw_reset(struct apple_dart *dart) apple_dart_writel(dart, U32_MAX, dart->hw->enable_streams + 4 * i); - /* Newer revisions expose per-stream exception records. */ - if (dart->hw->type == DART_T8110 && dart->version >= 0x0202) { - int ret; - - apple_dart_drain_stale_exceptions(dart); - ret = apple_dart_t8110_v2_unlock(dart); - if (ret) - return ret; - } else { - /* Clear pending errors before the interrupt is unmasked. */ - apple_dart_writel(dart, apple_dart_readl(dart, dart->hw->error), - dart->hw->error); - } - - if (dart->hw->type == DART_T8110) - apple_dart_writel(dart, 0, DART_T8110_ERROR_MASK); - - return dart->hw->invalidate_tlb(&stream_map); + return apple_dart_hw_finish_reset(dart); } static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, @@ -3092,16 +3115,25 @@ static __maybe_unused int apple_dart_resume(struct device *dev) if (dart->locked) return 0; - /* Nothing was lost, only drop what the TLBs may still hold. */ - if (apple_dart_state_retained(dart)) - return apple_dart_invalidate_each_stream(dart); + if (apple_dart_state_retained(dart)) { + /* Nothing was lost, only drop what the TLBs may still hold. */ + if (!dart->reset_incomplete) + return apple_dart_invalidate_each_stream(dart); - ret = apple_dart_hw_reset(dart); - if (ret) { - dev_err(dev, "Failed to reset DART on resume\n"); + /* A failed resume restored the translations: finish its reset. */ + ret = apple_dart_hw_finish_reset(dart); + dart->reset_incomplete = ret; return ret; } + ret = apple_dart_hw_reset(dart); + + /* + * Restore the translations even if the reset failed. It can only fail + * in its last steps, after clearing them, and the saved registers still + * point to the page tables in use. Leaving them cleared would fault + * every DMA of the consumers until the next reset. + */ for (sid = 0; sid < dart->num_streams; sid++) { for (idx = 0; idx < dart->hw->ttbr_count; idx++) apple_dart_writel(dart, dart->save_ttbr[sid][idx], @@ -3110,7 +3142,19 @@ static __maybe_unused int apple_dart_resume(struct device *dev) DART_TCR(dart, sid)); } - return 0; + /* + * Then repeat all of those last steps once. A TLB flush that completes + * now does not show that the DART was unlocked and its error interrupt + * unmasked. + */ + if (ret) { + dev_warn(dev, "reset on resume failed (%d), retrying\n", ret); + ret = apple_dart_hw_finish_reset(dart); + } + + /* Should they fail again, the next resume repeats them. */ + dart->reset_incomplete = ret; + return ret; } static DEFINE_RUNTIME_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume, NULL); From 43c7fc8f82a65f2e3807f41f03421df2b27ed3de Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 10:33:13 +1100 Subject: [PATCH 1117/1579] irqchip/apple-aic: Allow interrupts to wake the system Neither the AIC nor the AIC2/AIC3 irq_chip implements .irq_set_wake or sets IRQCHIP_SKIP_SET_WAKE, so irq_set_irq_wake() fails with -ENXIO for every hardware interrupt on Apple silicon. Drivers that arm their interrupt for wakeup, directly or through dev_pm_set_wake_irq(), are silently left without a wake interrupt: dev_pm_arm_wake_irq() ignores the error, suspend_device_irqs() then disables the line like any other, and an event on it cannot end suspend-to-idle. Only interrupts requested with IRQF_NO_SUSPEND, such as the Apple mailbox interrupts that carry the SMC lid and power button events, are still delivered, which is why those are the only things that wake these machines today. Suspend-to-idle is the only system sleep state on these SoCs. The AIC stays powered in it and delivers unmasked interrupts to the CPUs idling in WFI as usual, so there is no wake logic to program. Set IRQCHIP_SKIP_SET_WAKE on both chips, like the GICv3 driver does, so that enable_irq_wake() succeeds and the IRQ core keeps wake-armed interrupts enabled and turns their next event into a system wakeup. IRQCHIP_MASK_ON_SUSPEND is deliberately not set. The IRQ core already disables the non-wake interrupts lazily, and masks and records any that fire during suspend so they are replayed on resume. Masking them in hardware up front gains nothing on a controller that has no separate wake path, and would risk losing events from the edge-like MSIs that the PCIe controller delivers through the AIC. The FIQ chip is left alone: it only carries per-CPU timer, PMU and fast IPI interrupts, none of which are system wake sources. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/irqchip/irq-apple-aic.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index eb3a648f2f533..3214501a10635 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -484,6 +484,13 @@ static int aic_irq_set_type(struct irq_data *d, unsigned int type) return (type == IRQ_TYPE_LEVEL_HIGH || type == IRQ_TYPE_EDGE_RISING) ? 0 : -EINVAL; } +/* + * The only system sleep state on these SoCs is suspend-to-idle, in which the + * AIC stays powered and keeps delivering every unmasked interrupt to the CPUs + * idling in WFI. There is no separate wake logic to configure, so any IRQ may + * be a wakeup source: let enable_irq_wake() succeed and leave it to the IRQ + * core to keep wake-armed interrupts enabled across suspend_device_irqs(). + */ static struct irq_chip aic_chip = { .name = "AIC", .irq_mask = aic_irq_mask, @@ -491,6 +498,7 @@ static struct irq_chip aic_chip = { .irq_eoi = aic_irq_eoi, .irq_set_affinity = aic_irq_set_affinity, .irq_set_type = aic_irq_set_type, + .flags = IRQCHIP_SKIP_SET_WAKE, }; static struct irq_chip aic2_chip = { @@ -499,6 +507,7 @@ static struct irq_chip aic2_chip = { .irq_unmask = aic_irq_unmask, .irq_eoi = aic_irq_eoi, .irq_set_type = aic_irq_set_type, + .flags = IRQCHIP_SKIP_SET_WAKE, }; /* From 13e221102941d2404d6c55254869e1328f4a8ddf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 10:34:12 +1100 Subject: [PATCH 1118/1579] usb: dwc3: apple: Arm the core interrupt for system wakeup in host mode USB devices cannot wake an Apple silicon machine from suspend-to-idle. The glue device is never made wakeup capable, so dwc3_core_probe() records sys_wakeup = false, dwc3_host_init() leaves the xHCI platform device with wakeup disabled, and xhci_suspend() clears the root port wake bits. Nothing arms the controller interrupt for wakeup either, so suspend_device_irqs() disables it like any other device interrupt. The core itself is already kept running across system sleep whenever it is up (it is a syscore device, see dwc3_apple_init()), so the controller is in a state to report port events. What is missing is the wakeup policy and the wake interrupt: - Initialise the glue device as a wakeup source, enabled by default as usb_hcd_pci_probe() does for PCI host controllers. This happens before the first dwc3_core_probe(), which then passes the setting on to the xHCI device as on other dwc3 platforms, so xhci_suspend() keeps the root port wake-on-connect/disconnect/overcurrent bits and the root hubs stay armed for remote wakeup. Userspace can still turn it off through the power/wakeup attribute of the glue device. - While the controller is in host mode, register the core interrupt, which xHCI shares, as the wake interrupt of the glue device, and drop it again when host mode is torn down. The PM core arms it in the noirq phase when wakeup is enabled. Device mode gets no wake interrupt: a gadget is resumed by its host, and arming bus activity there would only cause spurious wakeups. This needs the AIC to accept enable_irq_wake(), which "irqchip/apple-aic: Allow interrupts to wake the system" provides; before it, the interrupt could not be armed at all. Not yet verified on hardware: whether this DWC3 integration raises its interrupt for a port status change after xhci_suspend() has halted the controller (the xHCI specification leaves wake signalling from the halted state to the platform). If it does not, the wake interrupt is armed but never fires, behaviour is otherwise unchanged, and a follow-up has to find the platform wake path instead. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index e1d1104db98a3..1e2ee195a82f8 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include #include @@ -78,7 +80,7 @@ * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. * * Whenever the core is up, for either kind of port, it is kept running across system sleep, see - * dwc3_apple_init(). + * dwc3_apple_init(). In host mode its interrupt is then also armed as the wake interrupt. * * enum dwc3_apple_state is defined in dwc3-apple-state.h. */ @@ -87,6 +89,7 @@ * struct dwc3_apple - Apple-specific DWC3 USB controller * @dwc: Core DWC3 structure * @dev: Pointer to the device structure + * @irq: Core interrupt, shared with xHCI in host mode and used as its wake interrupt * @mmio_resource: Resource to be passed to dwc3_core_probe * @apple_regs: Apple-specific DWC3 registers * @reset: Reset control @@ -107,6 +110,7 @@ struct dwc3_apple { struct dwc3 dwc; struct device *dev; + int irq; struct resource *mmio_resource; void __iomem *apple_regs; @@ -438,6 +442,18 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st goto core_exit; } + /* + * xHCI shares the core interrupt and is suspended with its root port wake bits set when + * this device may wake the system (dwc3_host_init() passes that on). Arm the interrupt + * for wakeup while in host mode, so that a port event during suspend-to-idle ends it. + * Device mode has no wake source: a gadget is woken by its host. + */ + if (state == DWC3_APPLE_HOST) { + ret = dev_pm_set_wake_irq(appledwc->dev, appledwc->irq); + if (ret) + dev_warn(appledwc->dev, "Failed to set up the wake IRQ, err=%d\n", ret); + } + appledwc->state = state; /* * A connected cable leaves the Apple DWC3 core and its PHY as one live, @@ -474,6 +490,7 @@ static int dwc3_apple_exit(struct dwc3_apple *appledwc) appledwc->state = DWC3_APPLE_NO_CABLE; break; case DWC3_APPLE_HOST: + dev_pm_clear_wake_irq(appledwc->dev); dwc3_host_exit(&appledwc->dwc); appledwc->state = DWC3_APPLE_NO_CABLE; break; @@ -622,6 +639,18 @@ static int dwc3_apple_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(appledwc->apple_regs), "Failed to map Apple-specific MMIO\n"); + appledwc->irq = platform_get_irq(pdev, 0); + if (appledwc->irq < 0) + return appledwc->irq; + + /* + * Like a PCI xHCI controller, allow USB devices to wake the system by default. This has + * to be set before dwc3_core_probe(), which hands it on to the xHCI device. + */ + ret = devm_device_init_wakeup(dev); + if (ret) + return dev_err_probe(dev, ret, "Failed to set up wakeup\n"); + /* * On this platform, DWC3 can only be brought up after parts of the PHY have been * initialized with knowledge of the target mode and cable orientation from typec_set_mux. From 54f3e0228a22e1a00ebeff349a47d3316c8ce23b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 10:36:40 +1100 Subject: [PATCH 1119/1579] Bluetooth: mt7932: Report the wakeup setting of the PCI function The driver sets no hdev->wakeup callback, so hci_register_dev() leaves HCI_CONN_FLAG_REMOTE_WAKEUP out of the supported connection flags. bluetoothd marks paired HID devices as wake-allowed and is refused ("Bad flag given (0x1) vs supported (0xe)") for each of them on every boot, and hci_suspend_sync() always takes the disconnect-only path. Return device_may_wakeup() of the PCI function, as hci_bcm4377 does for the Bluetooth function of the other Apple PCIe combo chips. The flag is then supported and the setting is honest: wakeup on the PCI function is off by default, so by default suspend behaves exactly as before. When userspace enables it, hci_suspend_sync() additionally leaves page scan and LE passive scan set up for the devices flagged for remote wakeup before bt7932_suspend() quiesces the transport. This does not make Bluetooth wake the system yet. What is still missing, none of which can be verified today: - bt7932_suspend() quiesces the firmware with the quiesce/restore handshake. It is not known whether the firmware keeps scanning in that state and signals the host when a wake-enabled device connects. - Nothing arms the MSI vector for wakeup, and it could not be armed: the pcie-apple MSI chips implement no irq_set_wake, so enable_irq_wake() fails on the MSI as it did on every AIC interrupt. - pcie-apple has no WAKE# handling, and whether a PME from the function reaches a host interrupt in suspend-to-idle has not been tested. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 19d2d714cdd24..ff90ee6d43c75 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1012,6 +1012,19 @@ static void bt7932_hw_error(struct hci_dev *hdev, u8 code) bt_dev_err(hdev, "hardware error 0x%2.2x", code); } +/* + * Follows the wakeup setting of the PCI function, which is off by default. + * Only while it is on does hci_suspend_sync() leave page scan and LE passive + * scan set up for the devices flagged for remote wakeup. The transport is + * quiesced for system sleep either way, and no interrupt is armed for wakeup. + */ +static bool bt7932_wakeup(struct hci_dev *hdev) +{ + struct bt7932 *bt = hci_get_drvdata(hdev); + + return device_may_wakeup(&bt->pdev->dev); +} + static int bt7932_iommu(struct bt7932 *bt) { struct device *dev = &bt->pdev->dev; @@ -1213,6 +1226,7 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) bt->hdev->setup = bt7932_setup; bt->hdev->reset = bt7932_reset; bt->hdev->hw_error = bt7932_hw_error; + bt->hdev->wakeup = bt7932_wakeup; hci_set_quirk(bt->hdev, HCI_QUIRK_RESET_ON_CLOSE); hci_set_quirk(bt->hdev, HCI_QUIRK_NON_PERSISTENT_SETUP); /* System sleep suspends HCI from bt7932_suspend(), after the freeze. */ From 8d33f6497da5a1cbe0d212bbe378ebade7e365f2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 12:06:26 +1100 Subject: [PATCH 1120/1579] usb: hub: keep the VL122 awake instead of ignoring its remote wakeup The VIA Labs VL122 USB 2.0 hub (2109:2122) on the J700 never signals a downstream connect while it is runtime suspended, so it was given USB_QUIRK_IGNORE_REMOTE_WAKEUP. That kept it out of runtime suspend, but it also leaves the hub without a wakeup setting at all, so it is never armed for remote wakeup in system sleep. Every USB 2.0 keyboard and mouse on the machine sits behind this hub, so none of them could wake it. The problem is runtime PM, not remote wakeup: mark the hub with HUB_QUIRK_DISABLE_AUTOSUSPEND, as is done for other hubs with broken runtime suspend, and drop the device quirk so its wakeup setting is available again for system sleep. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/hub.c | 7 +++++++ drivers/usb/core/quirks.c | 4 ---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 24960ba9caa91..7674b700ce33a 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -48,6 +48,8 @@ #define USB_PRODUCT_USB5534B 0x5534 #define USB_VENDOR_CYPRESS 0x04b4 #define USB_PRODUCT_CY7C65632 0x6570 +#define USB_VENDOR_VIA_LABS 0x2109 +#define USB_PRODUCT_VL122 0x2122 #define USB_VENDOR_TEXAS_INSTRUMENTS 0x0451 #define USB_PRODUCT_TUSB8041_USB3 0x8140 #define USB_PRODUCT_TUSB8041_USB2 0x8142 @@ -6025,6 +6027,11 @@ static const struct usb_device_id hub_id_table[] = { | USB_DEVICE_ID_MATCH_PRODUCT, .idVendor = USB_VENDOR_TEXAS_INSTRUMENTS, .idProduct = USB_PRODUCT_TUSB8041_USB3, + .driver_info = HUB_QUIRK_DISABLE_AUTOSUSPEND}, + { .match_flags = USB_DEVICE_ID_MATCH_VENDOR + | USB_DEVICE_ID_MATCH_PRODUCT, + .idVendor = USB_VENDOR_VIA_LABS, + .idProduct = USB_PRODUCT_VL122, .driver_info = HUB_QUIRK_DISABLE_AUTOSUSPEND}, { .match_flags = USB_DEVICE_ID_MATCH_VENDOR | USB_DEVICE_ID_MATCH_PRODUCT, diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index ae941ec487e44..b5b577f0b931c 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -583,10 +583,6 @@ static const struct usb_device_id usb_quirk_list[] = { /* VLI disk */ { USB_DEVICE(0x2109, 0x0711), .driver_info = USB_QUIRK_NO_LPM }, - /* VIA Labs, Inc. VL122 USB2.0 Hub: claims remote wakeup, never sends it */ - { USB_DEVICE(0x2109, 0x2122), .driver_info = - USB_QUIRK_IGNORE_REMOTE_WAKEUP }, - /* VIA Labs, Inc. USB2.0 Hub */ { USB_DEVICE(0x2109, 0x2817), .driver_info = USB_QUIRK_NO_LPM }, From 4d9fbe0bed99fff546a1ccd4221a9be20fa15b3d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 12:44:20 +1100 Subject: [PATCH 1121/1579] usb: xhci: plat: Allow the controller to keep running in suspend-to-idle xhci_suspend() halts the controller (clears USBCMD.R/S) and saves its state. A halted xHC writes no events to its event ring, so it raises no interrupt either: wake events from a halted controller are a separate signal, which on PCI is PME# and on other platforms a PHY or glue wake interrupt (dwc3-qcom, dwc3-imx8mp, xhci-mtk, xhci-tegra all have one). A platform without such a signal cannot be woken by USB devices from system sleep at all, even when it keeps the controller powered and its interrupt armed for wakeup. For suspend-to-idle there is a simple way out: do not halt the controller. The root hubs are still bus suspended with their wake bits set, so the ports are in U3 and downstream devices are armed for remote wakeup, but a remote wakeup or port change now raises a Port Status Change Event and with it the interrupt, which wakes the system once the platform arms it with enable_irq_wake(). Add a property, "xhci-run-in-s2idle", for glue drivers to request this. xhci-plat then skips xhci_suspend() and xhci_resume() across a system suspend when the target state is suspend-to-idle and the controller may wake the system with a wakeup source below it, the same way it already leaves the controller alone while a sideband instance is active. In all other cases, including hibernation and when wakeup is disabled through power/wakeup, the controller is halted as before. Event handling needs nothing new: until the interrupt is suspended an event goes through xhci_irq(), whose usb_hcd_resume_root_hub() reports a wakeup event and aborts the suspend; once it is armed for wakeup, the first interrupt ends suspend-to-idle and is handled when device interrupts are resumed. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci-plat.c | 37 ++++++++++++++++++++++++++++++++++++ drivers/usb/host/xhci-plat.h | 1 + drivers/usb/host/xhci.h | 1 + 3 files changed, 39 insertions(+) diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 074d9c731639f..7f874e848f954 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "xhci.h" @@ -277,6 +278,9 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s if (device_property_read_bool(tmpdev, "xhci-skip-phy-init-quirk")) xhci->quirks |= XHCI_SKIP_PHY_INIT; + if (device_property_read_bool(tmpdev, "xhci-run-in-s2idle")) + xhci->quirks |= XHCI_RUN_IN_S2IDLE; + device_property_read_u32(tmpdev, "imod-interval-ns", &xhci->imod_interval); device_property_read_u16(tmpdev, "num-hc-interrupters", @@ -494,6 +498,24 @@ static int xhci_plat_suspend_common(struct device *dev) return 0; } +/* + * Some platforms have no wake signal for a halted xHC: a halted controller writes no events, so + * its interrupt cannot fire, and on PCI the remote wakeup or port change would be signalled as a + * PME instead. Where the platform keeps the controller powered and clocked in suspend-to-idle, + * leave it running instead, with the root hubs suspended and armed for wakeup as usual. A remote + * wakeup or port change then raises a Port Status Change Event, and the interrupt, armed for + * wakeup by the platform, ends suspend-to-idle. + */ +static bool xhci_plat_run_in_s2idle(struct device *dev) +{ + struct usb_hcd *hcd = dev_get_drvdata(dev); + struct xhci_hcd *xhci = hcd_to_xhci(hcd); + + return (xhci->quirks & XHCI_RUN_IN_S2IDLE) && + pm_suspend_target_state == PM_SUSPEND_TO_IDLE && + device_may_wakeup(dev) && device_wakeup_path(dev); +} + static int xhci_plat_suspend(struct device *dev) { struct usb_hcd *hcd = dev_get_drvdata(dev); @@ -505,6 +527,15 @@ static int xhci_plat_suspend(struct device *dev) return 0; } + if (xhci_plat_run_in_s2idle(dev)) { + /* A runtime suspended controller is halted, start it again. */ + if (pm_runtime_suspended(dev)) + pm_runtime_resume(dev); + priv->running_in_s2idle = 1; + dev_dbg(dev, "wakeup enabled, keep running in suspend-to-idle.\n"); + return 0; + } + return xhci_plat_suspend_common(dev); } @@ -570,6 +601,12 @@ static int xhci_plat_resume(struct device *dev) return 0; } + if (priv->running_in_s2idle) { + priv->running_in_s2idle = 0; + dev_dbg(dev, "kept running in suspend-to-idle, skip resume.\n"); + return 0; + } + return xhci_plat_resume_common(dev, false); } diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h index 00751d8518310..eaaa9bc3e843f 100644 --- a/drivers/usb/host/xhci-plat.h +++ b/drivers/usb/host/xhci-plat.h @@ -17,6 +17,7 @@ struct xhci_plat_priv { unsigned long long quirks; bool power_lost; unsigned sideband_at_suspend:1; + unsigned running_in_s2idle:1; void (*plat_start)(struct usb_hcd *); int (*init_quirk)(struct usb_hcd *); int (*suspend_quirk)(struct usb_hcd *); diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index a23f9e1ef7123..7cd09324b4088 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1647,6 +1647,7 @@ struct xhci_hcd { #define XHCI_ETRON_HOST BIT_ULL(49) #define XHCI_LIMIT_ENDPOINT_INTERVAL_9 BIT_ULL(50) #define XHCI_ASMEDIA_FW_QUIRK BIT_ULL(51) +#define XHCI_RUN_IN_S2IDLE BIT_ULL(52) unsigned int num_active_eps; unsigned int limit_active_eps; From f52428867a055b05c8b20e172e471b5d19221b54 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 12:44:20 +1100 Subject: [PATCH 1122/1579] usb: dwc3: Let glue drivers keep xHCI running in suspend-to-idle Add a dwc3 software property for glue drivers whose controller stays powered in suspend-to-idle but has no wake signal besides the core interrupt, and pass it on to the xHCI platform device as "xhci-run-in-s2idle". The property array now holds up to seven entries, so make room for them and for the terminating empty entry. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/core.c | 3 +++ drivers/usb/dwc3/core.h | 3 +++ drivers/usb/dwc3/glue.h | 4 ++++ drivers/usb/dwc3/host.c | 5 ++++- 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index 498325fa1001d..0741b84ab7824 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -1705,6 +1705,9 @@ static void dwc3_get_software_properties(struct dwc3 *dwc, if (properties->needs_full_reinit) dwc->needs_full_reinit = true; + if (properties->xhci_run_in_s2idle) + dwc->xhci_run_in_s2idle = true; + dwc->gsbuscfg0_reqinfo = DWC3_GSBUSCFG0_REQINFO_UNSPECIFIED; if (properties->gsbuscfg0_reqinfo != diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index e0dee9d287401..6abc51d17bc46 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -1122,6 +1122,8 @@ struct dwc3_glue_ops { * @usb2_gadget_lpm_disable: set to disable usb2 lpm for gadget * @needs_full_reinit: set to indicate the core may lose power and need full * initialization during system pm + * @xhci_run_in_s2idle: set to keep xHCI running in suspend-to-idle when it may + * wake the system * @disable_scramble_quirk: set if we enable the disable scramble quirk * @u2exit_lfps_quirk: set if we enable u2exit lfps quirk * @u2ss_inp3_quirk: set if we enable P3 OK for U2/SS Inactive quirk @@ -1379,6 +1381,7 @@ struct dwc3 { unsigned usb2_lpm_disable:1; unsigned usb2_gadget_lpm_disable:1; unsigned needs_full_reinit:1; + unsigned xhci_run_in_s2idle:1; unsigned disable_scramble_quirk:1; unsigned u2exit_lfps_quirk:1; diff --git a/drivers/usb/dwc3/glue.h b/drivers/usb/dwc3/glue.h index db5f32df712bf..0737f533a8cd0 100644 --- a/drivers/usb/dwc3/glue.h +++ b/drivers/usb/dwc3/glue.h @@ -14,10 +14,14 @@ * @gsbuscfg0_reqinfo: Value to be programmed in the GSBUSCFG0.REQINFO field * @needs_full_reinit: indicate the controller may not remain power during system * pm and need full initialization + * @xhci_run_in_s2idle: the controller stays powered in suspend-to-idle and has no + * wake signal of its own, so xHCI has to keep running for + * USB devices to wake the system */ struct dwc3_properties { u32 gsbuscfg0_reqinfo; unsigned needs_full_reinit:1; + unsigned xhci_run_in_s2idle:1; }; #define DWC3_DEFAULT_PROPERTIES ((struct dwc3_properties){ \ diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c index 96b588bd08cdc..8cbefa118b8a2 100644 --- a/drivers/usb/dwc3/host.c +++ b/drivers/usb/dwc3/host.c @@ -129,7 +129,7 @@ static int dwc3_host_get_irq(struct dwc3 *dwc) int dwc3_host_init(struct dwc3 *dwc) { - struct property_entry props[6]; + struct property_entry props[8]; struct platform_device *xhci; int ret, irq; int prop_idx = 0; @@ -188,6 +188,9 @@ int dwc3_host_init(struct dwc3 *dwc) props[prop_idx++] = PROPERTY_ENTRY_U16("num-hc-interrupters", dwc->num_hc_interrupters); + if (dwc->xhci_run_in_s2idle) + props[prop_idx++] = PROPERTY_ENTRY_BOOL("xhci-run-in-s2idle"); + if (prop_idx) { ret = device_create_managed_software_node(&xhci->dev, props, NULL); if (ret) { From 825856a1ff4c8e8432ab3a26b3d616f11306f3f3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 12:44:21 +1100 Subject: [PATCH 1123/1579] usb: dwc3: apple: Keep xHCI running in suspend-to-idle With the core interrupt armed for wakeup, USB devices still cannot wake the J700 (T8140) from suspend-to-idle. Tested with a USB keyboard behind the on-board VL122 hub and an external hub, with the whole chain from the root hub to the keyboard set to power/wakeup=enabled: a keypress lights the keyboard up again, so it signals remote wakeup and its hubs pass it on, but the xHCI interrupt never fires during the sleep. The reason is that xhci_suspend() halts the controller. A halted xHC writes no events and raises no interrupt; a remote wakeup is then only visible as a platform wake signal, and this controller has none: the DWC3 node has a single interrupt, and the Type-C PHY has none. The core and its power domain are kept up across system sleep anyway (it is a syscore device while it is up), so ask xHCI to keep running in suspend-to-idle when it may wake the system. The remote wakeup then raises a Port Status Change Event on the running controller, and the core interrupt, already armed as the wake interrupt in host mode, ends suspend-to-idle. Not yet verified on hardware with this change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 1e2ee195a82f8..b24789b574561 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -80,7 +80,8 @@ * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. * * Whenever the core is up, for either kind of port, it is kept running across system sleep, see - * dwc3_apple_init(). In host mode its interrupt is then also armed as the wake interrupt. + * dwc3_apple_init(). In host mode its interrupt is then also armed as the wake interrupt, and + * xHCI keeps running in suspend-to-idle so that it can raise it. * * enum dwc3_apple_state is defined in dwc3-apple-state.h. */ @@ -212,6 +213,11 @@ static int dwc3_apple_core_probe(struct dwc3_apple *appledwc) probe_data.ignore_clocks_and_resets = true; probe_data.skip_core_init_mode = true; probe_data.properties = DWC3_DEFAULT_PROPERTIES; + /* + * There is no wake interrupt besides the core interrupt, and a halted xHC raises none, + * see dwc3_apple_init(). + */ + probe_data.properties.xhci_run_in_s2idle = true; /* * This runs from the role switch callback, so a failure leaves the glue @@ -443,9 +449,12 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st } /* - * xHCI shares the core interrupt and is suspended with its root port wake bits set when - * this device may wake the system (dwc3_host_init() passes that on). Arm the interrupt - * for wakeup while in host mode, so that a port event during suspend-to-idle ends it. + * xHCI shares the core interrupt and suspends its root hubs with their wake bits set when + * this device may wake the system (dwc3_host_init() passes that on). The controller has no + * separate wake signal, and a halted xHC writes no events and so raises no interrupt, so + * xHCI is told to keep running in suspend-to-idle instead of halting (the core stays + * powered, see below). Arm the interrupt for wakeup while in host mode, so that the Port + * Status Change Event of a remote wakeup or port change ends suspend-to-idle. * Device mode has no wake source: a gadget is woken by its host. */ if (state == DWC3_APPLE_HOST) { From dbc72dfdaf1a65b80c8555ee48c01c3374e56f93 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:26:15 +1100 Subject: [PATCH 1124/1579] usb: xhci: plat: Add KUnit tests for property quirks and suspend-to-idle Move the mapping of boolean device properties to quirks, and the decision whether to keep the controller running in suspend-to-idle, into helpers that do not need a probed controller, and test them: every property maps to its quirk, "xhci-run-in-s2idle" included, and the controller keeps running only for suspend-to-idle with the quirk set, wakeup enabled and a wakeup source below it. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/Kconfig | 12 +++ drivers/usb/host/Makefile | 1 + drivers/usb/host/xhci-plat-test.c | 117 ++++++++++++++++++++++++++++++ drivers/usb/host/xhci-plat.c | 72 +++++++++++------- drivers/usb/host/xhci-plat.h | 8 ++ 5 files changed, 183 insertions(+), 27 deletions(-) create mode 100644 drivers/usb/host/xhci-plat-test.c diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig index b879b438c7e71..93a5145b3e773 100644 --- a/drivers/usb/host/Kconfig +++ b/drivers/usb/host/Kconfig @@ -70,6 +70,18 @@ config USB_XHCI_PLATFORM If unsure, say N. +config USB_XHCI_PLATFORM_KUNIT_TEST + tristate "KUnit tests for the generic xHCI platform driver" if !KUNIT_ALL_TESTS + depends on USB_XHCI_PLATFORM && KUNIT + default KUNIT_ALL_TESTS + help + Test how the generic xHCI platform driver turns device properties + into quirks, and when it keeps the controller running across + suspend-to-idle instead of halting it. The tests need no xHCI + hardware. + + If unsure, say N. + config USB_XHCI_HISTB tristate "xHCI support for HiSilicon STB SoCs" depends on USB_XHCI_PLATFORM && (ARCH_HISI || COMPILE_TEST) diff --git a/drivers/usb/host/Makefile b/drivers/usb/host/Makefile index 09b4f6c60e2d2..7fa99b5f85404 100644 --- a/drivers/usb/host/Makefile +++ b/drivers/usb/host/Makefile @@ -75,6 +75,7 @@ xhci-pci-y += xhci-pci-core.o xhci-pci-$(CONFIG_USB_XHCI_PCI_ASMEDIA) += xhci-pci-asmedia.o obj-$(CONFIG_USB_XHCI_PCI_RENESAS) += xhci-pci-renesas.o obj-$(CONFIG_USB_XHCI_PLATFORM) += xhci-plat-hcd.o +obj-$(CONFIG_USB_XHCI_PLATFORM_KUNIT_TEST) += xhci-plat-test.o obj-$(CONFIG_USB_XHCI_HISTB) += xhci-histb.o obj-$(CONFIG_USB_XHCI_RCAR) += xhci-rcar-hcd.o obj-$(CONFIG_USB_XHCI_MTK) += xhci-mtk-hcd.o diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c new file mode 100644 index 0000000000000..410cf99da08d6 --- /dev/null +++ b/drivers/usb/host/xhci-plat-test.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for the generic xHCI platform driver + */ + +#include +#include +#include +#include +#include +#include + +#include "xhci.h" +#include "xhci-plat.h" + +/* Only suspend-to-idle with the quirk, wakeup enabled and a wakeup path keeps it running. */ +static void xhci_plat_keep_running_test(struct kunit *test) +{ + static const suspend_state_t targets[] = { + PM_SUSPEND_ON, PM_SUSPEND_TO_IDLE, PM_SUSPEND_STANDBY, PM_SUSPEND_MEM, + }; + unsigned int i, combo; + + for (i = 0; i < ARRAY_SIZE(targets); i++) { + for (combo = 0; combo < 8; combo++) { + u64 quirks = (combo & 1) ? XHCI_RUN_IN_S2IDLE : 0; + bool may_wakeup = combo & 2, wakeup_path = combo & 4; + bool expected = combo == 7 && targets[i] == PM_SUSPEND_TO_IDLE; + + bool kept = xhci_plat_keep_running(quirks, targets[i], may_wakeup, + wakeup_path); + + KUNIT_EXPECT_EQ_MSG(test, kept, expected, + "target %d quirk %d may_wakeup %d wakeup_path %d", + targets[i], combo & 1, may_wakeup, wakeup_path); + } + } +} + +/* Other quirks do not stand in for the suspend-to-idle one. */ +static void xhci_plat_keep_running_other_quirks_test(struct kunit *test) +{ + KUNIT_EXPECT_FALSE(test, xhci_plat_keep_running(~XHCI_RUN_IN_S2IDLE, PM_SUSPEND_TO_IDLE, + true, true)); + KUNIT_EXPECT_TRUE(test, xhci_plat_keep_running(~0ULL, PM_SUSPEND_TO_IDLE, true, true)); +} + +static u64 xhci_plat_test_quirks(struct kunit *test, const struct property_entry *props) +{ + struct device *dev; + + dev = kunit_device_register(test, "xhci-plat-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + KUNIT_ASSERT_EQ(test, device_create_managed_software_node(dev, props, NULL), 0); + + return xhci_plat_property_quirks(dev); +} + +/* A device without properties gets no quirks. */ +static void xhci_plat_no_property_quirks_test(struct kunit *test) +{ + static const struct property_entry props[] = { {} }; + + KUNIT_EXPECT_EQ(test, xhci_plat_test_quirks(test, props), 0ULL); +} + +/* The property a glue driver sets for suspend-to-idle maps to its quirk, and only to it. */ +static void xhci_plat_run_in_s2idle_property_test(struct kunit *test) +{ + static const struct property_entry props[] = { + PROPERTY_ENTRY_BOOL("xhci-run-in-s2idle"), + {} + }; + + KUNIT_EXPECT_EQ(test, xhci_plat_test_quirks(test, props), XHCI_RUN_IN_S2IDLE); +} + +/* Every boolean property maps to its quirk. */ +static void xhci_plat_all_property_quirks_test(struct kunit *test) +{ + static const struct property_entry props[] = { + PROPERTY_ENTRY_BOOL("usb2-lpm-disable"), + PROPERTY_ENTRY_BOOL("usb3-lpm-capable"), + PROPERTY_ENTRY_BOOL("quirk-broken-port-ped"), + PROPERTY_ENTRY_BOOL("xhci-sg-trb-cache-size-quirk"), + PROPERTY_ENTRY_BOOL("write-64-hi-lo-quirk"), + PROPERTY_ENTRY_BOOL("xhci-missing-cas-quirk"), + PROPERTY_ENTRY_BOOL("xhci-skip-phy-init-quirk"), + PROPERTY_ENTRY_BOOL("xhci-run-in-s2idle"), + {} + }; + + KUNIT_EXPECT_EQ(test, xhci_plat_test_quirks(test, props), + XHCI_HW_LPM_DISABLE | XHCI_LPM_SUPPORT | XHCI_BROKEN_PORT_PED | + XHCI_SG_TRB_CACHE_SIZE_QUIRK | XHCI_WRITE_64_HI_LO | XHCI_MISSING_CAS | + XHCI_SKIP_PHY_INIT | XHCI_RUN_IN_S2IDLE); +} + +static struct kunit_case xhci_plat_test_cases[] = { + KUNIT_CASE(xhci_plat_keep_running_test), + KUNIT_CASE(xhci_plat_keep_running_other_quirks_test), + KUNIT_CASE(xhci_plat_no_property_quirks_test), + KUNIT_CASE(xhci_plat_run_in_s2idle_property_test), + KUNIT_CASE(xhci_plat_all_property_quirks_test), + {} +}; + +static struct kunit_suite xhci_plat_test_suite = { + .name = "xhci_plat", + .test_cases = xhci_plat_test_cases, +}; + +kunit_test_suite(xhci_plat_test_suite); + +MODULE_DESCRIPTION("KUnit tests for the generic xHCI platform driver"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 7f874e848f954..8a4413f817e72 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -8,6 +8,7 @@ * A lot of code borrowed from the Linux xHCI driver. */ +#include #include #include #include @@ -154,6 +155,39 @@ static const struct of_device_id usb_xhci_of_match[] = { MODULE_DEVICE_TABLE(of, usb_xhci_of_match); #endif +/* The quirks that the boolean properties of a device, or of one of its parents, ask for */ +VISIBLE_IF_KUNIT u64 xhci_plat_property_quirks(struct device *dev) +{ + u64 quirks = 0; + + if (device_property_read_bool(dev, "usb2-lpm-disable")) + quirks |= XHCI_HW_LPM_DISABLE; + + if (device_property_read_bool(dev, "usb3-lpm-capable")) + quirks |= XHCI_LPM_SUPPORT; + + if (device_property_read_bool(dev, "quirk-broken-port-ped")) + quirks |= XHCI_BROKEN_PORT_PED; + + if (device_property_read_bool(dev, "xhci-sg-trb-cache-size-quirk")) + quirks |= XHCI_SG_TRB_CACHE_SIZE_QUIRK; + + if (device_property_read_bool(dev, "write-64-hi-lo-quirk")) + quirks |= XHCI_WRITE_64_HI_LO; + + if (device_property_read_bool(dev, "xhci-missing-cas-quirk")) + quirks |= XHCI_MISSING_CAS; + + if (device_property_read_bool(dev, "xhci-skip-phy-init-quirk")) + quirks |= XHCI_SKIP_PHY_INIT; + + if (device_property_read_bool(dev, "xhci-run-in-s2idle")) + quirks |= XHCI_RUN_IN_S2IDLE; + + return quirks; +} +EXPORT_SYMBOL_IF_KUNIT(xhci_plat_property_quirks); + int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const struct xhci_plat_priv *priv_match) { const struct hc_driver *driver; @@ -256,30 +290,7 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s /* Iterate over all parent nodes for finding quirks */ for (tmpdev = &pdev->dev; tmpdev; tmpdev = tmpdev->parent) { - - if (device_property_read_bool(tmpdev, "usb2-lpm-disable")) - xhci->quirks |= XHCI_HW_LPM_DISABLE; - - if (device_property_read_bool(tmpdev, "usb3-lpm-capable")) - xhci->quirks |= XHCI_LPM_SUPPORT; - - if (device_property_read_bool(tmpdev, "quirk-broken-port-ped")) - xhci->quirks |= XHCI_BROKEN_PORT_PED; - - if (device_property_read_bool(tmpdev, "xhci-sg-trb-cache-size-quirk")) - xhci->quirks |= XHCI_SG_TRB_CACHE_SIZE_QUIRK; - - if (device_property_read_bool(tmpdev, "write-64-hi-lo-quirk")) - xhci->quirks |= XHCI_WRITE_64_HI_LO; - - if (device_property_read_bool(tmpdev, "xhci-missing-cas-quirk")) - xhci->quirks |= XHCI_MISSING_CAS; - - if (device_property_read_bool(tmpdev, "xhci-skip-phy-init-quirk")) - xhci->quirks |= XHCI_SKIP_PHY_INIT; - - if (device_property_read_bool(tmpdev, "xhci-run-in-s2idle")) - xhci->quirks |= XHCI_RUN_IN_S2IDLE; + xhci->quirks |= xhci_plat_property_quirks(tmpdev); device_property_read_u32(tmpdev, "imod-interval-ns", &xhci->imod_interval); @@ -506,14 +517,21 @@ static int xhci_plat_suspend_common(struct device *dev) * wakeup or port change then raises a Port Status Change Event, and the interrupt, armed for * wakeup by the platform, ends suspend-to-idle. */ +VISIBLE_IF_KUNIT bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, + bool may_wakeup, bool wakeup_path) +{ + return (quirks & XHCI_RUN_IN_S2IDLE) && target == PM_SUSPEND_TO_IDLE && + may_wakeup && wakeup_path; +} +EXPORT_SYMBOL_IF_KUNIT(xhci_plat_keep_running); + static bool xhci_plat_run_in_s2idle(struct device *dev) { struct usb_hcd *hcd = dev_get_drvdata(dev); struct xhci_hcd *xhci = hcd_to_xhci(hcd); - return (xhci->quirks & XHCI_RUN_IN_S2IDLE) && - pm_suspend_target_state == PM_SUSPEND_TO_IDLE && - device_may_wakeup(dev) && device_wakeup_path(dev); + return xhci_plat_keep_running(xhci->quirks, pm_suspend_target_state, + device_may_wakeup(dev), device_wakeup_path(dev)); } static int xhci_plat_suspend(struct device *dev) diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h index eaaa9bc3e843f..5a43502ca27a9 100644 --- a/drivers/usb/host/xhci-plat.h +++ b/drivers/usb/host/xhci-plat.h @@ -34,4 +34,12 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, void xhci_plat_remove(struct platform_device *dev); extern const struct dev_pm_ops xhci_plat_pm_ops; +#if IS_ENABLED(CONFIG_KUNIT) +#include + +u64 xhci_plat_property_quirks(struct device *dev); +bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, bool may_wakeup, + bool wakeup_path); +#endif + #endif /* _XHCI_PLAT_H */ From 0612d22d1796d962799c67af3b5da1fca9a32544 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:26:16 +1100 Subject: [PATCH 1125/1579] usb: dwc3: Add KUnit tests for the xHCI device properties Move the filling of the property array for the xHCI platform device into dwc3_host_xhci_properties() and test it: "xhci-run-in-s2idle" is passed on only when the glue asks for it, and with every optional property set the array still ends in an empty entry. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/Kconfig | 10 +++ drivers/usb/dwc3/Makefile | 1 + drivers/usb/dwc3/core.h | 6 ++ drivers/usb/dwc3/dwc3-host-test.c | 115 ++++++++++++++++++++++++++++++ drivers/usb/dwc3/host.c | 79 ++++++++++++-------- 5 files changed, 181 insertions(+), 30 deletions(-) create mode 100644 drivers/usb/dwc3/dwc3-host-test.c diff --git a/drivers/usb/dwc3/Kconfig b/drivers/usb/dwc3/Kconfig index 3e033ddc86d3d..c862e55ae3a37 100644 --- a/drivers/usb/dwc3/Kconfig +++ b/drivers/usb/dwc3/Kconfig @@ -51,6 +51,16 @@ config USB_DWC3_DUAL_ROLE endchoice +config USB_DWC3_HOST_KUNIT_TEST + tristate "KUnit tests for the DWC3 host glue" if !KUNIT_ALL_TESTS + depends on KUNIT && (USB_DWC3_HOST || USB_DWC3_DUAL_ROLE) + default KUNIT_ALL_TESTS + help + Test the properties with which the DWC3 driver configures its + xHCI platform device. The tests need no DWC3 hardware. + + If unsure, say N. + comment "Platform Glue Driver Support" config USB_DWC3_OMAP diff --git a/drivers/usb/dwc3/Makefile b/drivers/usb/dwc3/Makefile index 76578fc4a5da4..57e0475016b6c 100644 --- a/drivers/usb/dwc3/Makefile +++ b/drivers/usb/dwc3/Makefile @@ -44,6 +44,7 @@ endif obj-$(CONFIG_USB_DWC3_AM62) += dwc3-am62.o obj-$(CONFIG_USB_DWC3_APPLE) += dwc3-apple.o +obj-$(CONFIG_USB_DWC3_HOST_KUNIT_TEST) += dwc3-host-test.o obj-$(CONFIG_USB_DWC3_APPLE_KUNIT_TEST) += dwc3-apple-state-test.o obj-$(CONFIG_USB_DWC3_OMAP) += dwc3-omap.o obj-$(CONFIG_USB_DWC3_EXYNOS) += dwc3-exynos.o diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 6abc51d17bc46..1dc04f40416b7 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -1656,6 +1656,12 @@ static inline void dwc3_pre_run_stop(struct dwc3 *dwc, bool is_on) #if IS_ENABLED(CONFIG_USB_DWC3_HOST) || IS_ENABLED(CONFIG_USB_DWC3_DUAL_ROLE) int dwc3_host_init(struct dwc3 *dwc); void dwc3_host_exit(struct dwc3 *dwc); +/* Most properties dwc3_host_xhci_properties() fills in, not counting the empty entry */ +#define DWC3_XHCI_MAX_PROPERTIES 7 +#if IS_ENABLED(CONFIG_KUNIT) +struct property_entry; +unsigned int dwc3_host_xhci_properties(const struct dwc3 *dwc, struct property_entry *props); +#endif #else static inline int dwc3_host_init(struct dwc3 *dwc) { return 0; } diff --git a/drivers/usb/dwc3/dwc3-host-test.c b/drivers/usb/dwc3/dwc3-host-test.c new file mode 100644 index 0000000000000..6c50ea82bbff9 --- /dev/null +++ b/drivers/usb/dwc3/dwc3-host-test.c @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for the properties dwc3 gives its xHCI device + */ + +#include +#include +#include +#include +#include +#include + +#include "core.h" + +struct dwc3_host_test { + struct dwc3 *dwc; + struct property_entry props[DWC3_XHCI_MAX_PROPERTIES + 1]; +}; + +static int dwc3_host_test_init(struct kunit *test) +{ + struct dwc3_host_test *t; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + t->dwc = kunit_kzalloc(test, sizeof(*t->dwc), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t->dwc); + /* Fill with garbage: the helper has to clear what it does not set. */ + memset(t->props, 0xa5, sizeof(t->props)); + test->priv = t; + + return 0; +} + +static const struct property_entry *dwc3_host_test_find(const struct property_entry *props, + const char *name) +{ + for (; props->name; props++) + if (!strcmp(props->name, name)) + return props; + + return NULL; +} + +/* Without the glue asking for it, xHCI is not told to keep running in suspend-to-idle. */ +static void dwc3_host_no_run_in_s2idle_test(struct kunit *test) +{ + struct dwc3_host_test *t = test->priv; + unsigned int n; + + n = dwc3_host_xhci_properties(t->dwc, t->props); + + KUNIT_EXPECT_LE(test, n, DWC3_XHCI_MAX_PROPERTIES); + KUNIT_EXPECT_NULL(test, t->props[n].name); + KUNIT_EXPECT_NULL(test, dwc3_host_test_find(t->props, "xhci-run-in-s2idle")); + KUNIT_EXPECT_NOT_NULL(test, dwc3_host_test_find(t->props, "xhci-sg-trb-cache-size-quirk")); +} + +/* The glue property reaches xHCI as a boolean property. */ +static void dwc3_host_run_in_s2idle_test(struct kunit *test) +{ + struct dwc3_host_test *t = test->priv; + const struct property_entry *prop; + unsigned int n; + + t->dwc->xhci_run_in_s2idle = true; + n = dwc3_host_xhci_properties(t->dwc, t->props); + + KUNIT_EXPECT_NULL(test, t->props[n].name); + prop = dwc3_host_test_find(t->props, "xhci-run-in-s2idle"); + KUNIT_ASSERT_NOT_NULL(test, prop); + /* A boolean property has no value. */ + KUNIT_EXPECT_EQ(test, prop->length, 0); +} + +/* With every optional property set, the array still ends in an empty entry. */ +static void dwc3_host_all_properties_test(struct kunit *test) +{ + struct dwc3_host_test *t = test->priv; + unsigned int n; + + t->dwc->ip = DWC3_IP; + t->dwc->revision = DWC3_REVISION_290A; + t->dwc->usb3_lpm_capable = true; + t->dwc->usb2_lpm_disable = true; + t->dwc->xhci_run_in_s2idle = true; + n = dwc3_host_xhci_properties(t->dwc, t->props); + + KUNIT_EXPECT_EQ(test, n, DWC3_XHCI_MAX_PROPERTIES); + KUNIT_EXPECT_NULL(test, t->props[n].name); + KUNIT_EXPECT_NOT_NULL(test, dwc3_host_test_find(t->props, "usb3-lpm-capable")); + KUNIT_EXPECT_NOT_NULL(test, dwc3_host_test_find(t->props, "usb2-lpm-disable")); + KUNIT_EXPECT_NOT_NULL(test, dwc3_host_test_find(t->props, "quirk-broken-port-ped")); + KUNIT_EXPECT_NOT_NULL(test, dwc3_host_test_find(t->props, "num-hc-interrupters")); + KUNIT_EXPECT_NOT_NULL(test, dwc3_host_test_find(t->props, "xhci-run-in-s2idle")); +} + +static struct kunit_case dwc3_host_test_cases[] = { + KUNIT_CASE(dwc3_host_no_run_in_s2idle_test), + KUNIT_CASE(dwc3_host_run_in_s2idle_test), + KUNIT_CASE(dwc3_host_all_properties_test), + {} +}; + +static struct kunit_suite dwc3_host_test_suite = { + .name = "dwc3_host", + .init = dwc3_host_test_init, + .test_cases = dwc3_host_test_cases, +}; + +kunit_test_suite(dwc3_host_test_suite); + +MODULE_DESCRIPTION("KUnit tests for the dwc3 xHCI device properties"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c index 8cbefa118b8a2..a138a5f34ebd7 100644 --- a/drivers/usb/dwc3/host.c +++ b/drivers/usb/dwc3/host.c @@ -7,6 +7,7 @@ * Authors: Felipe Balbi , */ +#include #include #include #include @@ -127,9 +128,55 @@ static int dwc3_host_get_irq(struct dwc3 *dwc) return irq; } +/** + * dwc3_host_xhci_properties - Fill in the properties of the xHCI device + * @dwc: Pointer to our controller context structure + * @props: Array of DWC3_XHCI_MAX_PROPERTIES + 1 entries + * + * Return: the number of properties, which are followed by an empty entry. + */ +VISIBLE_IF_KUNIT unsigned int dwc3_host_xhci_properties(const struct dwc3 *dwc, + struct property_entry *props) +{ + unsigned int prop_idx = 0; + + memset(props, 0, sizeof(*props) * (DWC3_XHCI_MAX_PROPERTIES + 1)); + + props[prop_idx++] = PROPERTY_ENTRY_BOOL("xhci-sg-trb-cache-size-quirk"); + + props[prop_idx++] = PROPERTY_ENTRY_BOOL("write-64-hi-lo-quirk"); + + if (dwc->usb3_lpm_capable) + props[prop_idx++] = PROPERTY_ENTRY_BOOL("usb3-lpm-capable"); + + if (dwc->usb2_lpm_disable) + props[prop_idx++] = PROPERTY_ENTRY_BOOL("usb2-lpm-disable"); + + /** + * WORKAROUND: dwc3 revisions <=3.00a have a limitation + * where Port Disable command doesn't work. + * + * The suggested workaround is that we avoid Port Disable + * completely. + * + * This following flag tells XHCI to do just that. + */ + if (DWC3_VER_IS_WITHIN(DWC3, ANY, 300A)) + props[prop_idx++] = PROPERTY_ENTRY_BOOL("quirk-broken-port-ped"); + + props[prop_idx++] = PROPERTY_ENTRY_U16("num-hc-interrupters", + dwc->num_hc_interrupters); + + if (dwc->xhci_run_in_s2idle) + props[prop_idx++] = PROPERTY_ENTRY_BOOL("xhci-run-in-s2idle"); + + return prop_idx; +} +EXPORT_SYMBOL_IF_KUNIT(dwc3_host_xhci_properties); + int dwc3_host_init(struct dwc3 *dwc) { - struct property_entry props[8]; + struct property_entry props[DWC3_XHCI_MAX_PROPERTIES + 1]; struct platform_device *xhci; int ret, irq; int prop_idx = 0; @@ -161,35 +208,7 @@ int dwc3_host_init(struct dwc3 *dwc) goto err; } - memset(props, 0, sizeof(struct property_entry) * ARRAY_SIZE(props)); - - props[prop_idx++] = PROPERTY_ENTRY_BOOL("xhci-sg-trb-cache-size-quirk"); - - props[prop_idx++] = PROPERTY_ENTRY_BOOL("write-64-hi-lo-quirk"); - - if (dwc->usb3_lpm_capable) - props[prop_idx++] = PROPERTY_ENTRY_BOOL("usb3-lpm-capable"); - - if (dwc->usb2_lpm_disable) - props[prop_idx++] = PROPERTY_ENTRY_BOOL("usb2-lpm-disable"); - - /** - * WORKAROUND: dwc3 revisions <=3.00a have a limitation - * where Port Disable command doesn't work. - * - * The suggested workaround is that we avoid Port Disable - * completely. - * - * This following flag tells XHCI to do just that. - */ - if (DWC3_VER_IS_WITHIN(DWC3, ANY, 300A)) - props[prop_idx++] = PROPERTY_ENTRY_BOOL("quirk-broken-port-ped"); - - props[prop_idx++] = PROPERTY_ENTRY_U16("num-hc-interrupters", - dwc->num_hc_interrupters); - - if (dwc->xhci_run_in_s2idle) - props[prop_idx++] = PROPERTY_ENTRY_BOOL("xhci-run-in-s2idle"); + prop_idx = dwc3_host_xhci_properties(dwc, props); if (prop_idx) { ret = device_create_managed_software_node(&xhci->dev, props, NULL); From 0ca3772f141f4b95ca9cb77b2068c5b16c7bfd22 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:26:16 +1100 Subject: [PATCH 1126/1579] usb: core: Add KUnit tests for the VIA Labs VL122 hub quirks Test that the VL122 hub (2109:2122) no longer has USB_QUIRK_IGNORE_REMOTE_WAKEUP and gets HUB_QUIRK_DISABLE_AUTOSUSPEND instead, and that neighbouring entries still match as before. Move the HUB_QUIRK_* flags to hub.h and add small accessors for the device quirk list and the hub id table that the test can call. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/Kconfig | 11 ++++ drivers/usb/core/Makefile | 1 + drivers/usb/core/hub.c | 15 ++++-- drivers/usb/core/hub.h | 9 ++++ drivers/usb/core/quirks.c | 10 +++- drivers/usb/core/usb-quirks-test.c | 87 ++++++++++++++++++++++++++++++ drivers/usb/core/usb.h | 3 ++ 7 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 drivers/usb/core/usb-quirks-test.c diff --git a/drivers/usb/core/Kconfig b/drivers/usb/core/Kconfig index 58e3ca7e47939..fbeab9ee46202 100644 --- a/drivers/usb/core/Kconfig +++ b/drivers/usb/core/Kconfig @@ -97,6 +97,17 @@ config USB_OTG_FSM Implements OTG Finite State Machine as specified in On-The-Go and Embedded Host Supplement to the USB Revision 2.0 Specification. +config USB_QUIRKS_KUNIT_TEST + tristate "KUnit tests for USB device and hub quirk matching" if !KUNIT_ALL_TESTS + depends on USB && KUNIT + default KUNIT_ALL_TESTS + help + Test that the USB core matches devices and hubs against its quirk + tables as intended for the entries these tests cover. The tests + need no USB hardware. + + If unsure, say N. + config USB_LEDS_TRIGGER_USBPORT tristate "USB port LED trigger" depends on USB && LEDS_TRIGGERS diff --git a/drivers/usb/core/Makefile b/drivers/usb/core/Makefile index 60ea76160122b..bf228db5e15f9 100644 --- a/drivers/usb/core/Makefile +++ b/drivers/usb/core/Makefile @@ -21,5 +21,6 @@ usbcore-y += ../misc/onboard_usb_dev_pdevs.o endif obj-$(CONFIG_USB) += usbcore.o +obj-$(CONFIG_USB_QUIRKS_KUNIT_TEST) += usb-quirks-test.o obj-$(CONFIG_USB_LEDS_TRIGGER_USBPORT) += ledtrig-usbport.o diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 7674b700ce33a..3e853c43390dd 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -10,6 +10,7 @@ * Released under the GPLv2 only. */ +#include #include #include #include @@ -57,9 +58,6 @@ #define USB_PRODUCT_USB4913 0x4913 #define USB_PRODUCT_USB4914 0x4914 #define USB_PRODUCT_USB4915 0x4915 -#define HUB_QUIRK_CHECK_PORT_AUTOSUSPEND BIT(0) -#define HUB_QUIRK_DISABLE_AUTOSUSPEND BIT(1) -#define HUB_QUIRK_REDUCE_FRAME_INTR_BINTERVAL BIT(2) #define USB_TP_TRANSMISSION_DELAY 40 /* ns */ #define USB_TP_TRANSMISSION_DELAY_MAX 65535 /* ns */ @@ -6055,6 +6053,17 @@ static const struct usb_device_id hub_id_table[] = { { } /* Terminating entry */ }; +#if IS_ENABLED(CONFIG_KUNIT) +/* The HUB_QUIRK_* flags that hub_id_table gives a hub interface */ +unsigned long usb_hub_id_quirks(struct usb_interface *intf) +{ + const struct usb_device_id *id = usb_match_id(intf, hub_id_table); + + return id ? id->driver_info : 0; +} +EXPORT_SYMBOL_IF_KUNIT(usb_hub_id_quirks); +#endif + MODULE_DEVICE_TABLE(usb, hub_id_table); static struct usb_driver hub_driver = { diff --git a/drivers/usb/core/hub.h b/drivers/usb/core/hub.h index 9ebc5ef54a325..da3235930b0c8 100644 --- a/drivers/usb/core/hub.h +++ b/drivers/usb/core/hub.h @@ -17,6 +17,11 @@ #include #include "usb.h" +/* Quirks that hub_id_table assigns through driver_info */ +#define HUB_QUIRK_CHECK_PORT_AUTOSUSPEND BIT(0) +#define HUB_QUIRK_DISABLE_AUTOSUSPEND BIT(1) +#define HUB_QUIRK_REDUCE_FRAME_INTR_BINTERVAL BIT(2) + struct usb_hub { struct device *intfdev; /* the "interface" device */ struct usb_device *hdev; @@ -183,3 +188,7 @@ static inline int hub_port_debounce_be_stable(struct usb_hub *hub, { return hub_port_debounce(hub, port1, false); } + +#if IS_ENABLED(CONFIG_KUNIT) +unsigned long usb_hub_id_quirks(struct usb_interface *intf); +#endif diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index b5b577f0b931c..94a8f711da9fc 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -6,6 +6,7 @@ * Copyright (c) 2007 Greg Kroah-Hartman */ +#include #include #include #include @@ -767,9 +768,16 @@ static u32 usb_detect_dynamic_quirks(struct usb_device *udev) /* * Detect any quirks the device has, and do any housekeeping for it if needed. */ +/* The quirks that usb_quirk_list gives a device */ +VISIBLE_IF_KUNIT u32 usb_detect_device_list_quirks(struct usb_device *udev) +{ + return usb_detect_static_quirks(udev, usb_quirk_list); +} +EXPORT_SYMBOL_IF_KUNIT(usb_detect_device_list_quirks); + void usb_detect_quirks(struct usb_device *udev) { - udev->quirks = usb_detect_static_quirks(udev, usb_quirk_list); + udev->quirks = usb_detect_device_list_quirks(udev); /* * Pixart-based mice would trigger remote wakeup issue on AMD diff --git a/drivers/usb/core/usb-quirks-test.c b/drivers/usb/core/usb-quirks-test.c new file mode 100644 index 0000000000000..6426f7951ff0e --- /dev/null +++ b/drivers/usb/core/usb-quirks-test.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for USB device and hub quirk matching + */ + +#include +#include +#include +#include +#include + +/* hub.h includes usb.h, which has no include guard */ +#include "hub.h" + +struct usb_quirks_test { + struct usb_device udev; + struct usb_interface intf; + struct usb_host_interface alt; +}; + +static struct usb_quirks_test *usb_quirks_test_hub(struct kunit *test, u16 vid, u16 pid) +{ + struct usb_quirks_test *t; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + + t->udev.descriptor.idVendor = cpu_to_le16(vid); + t->udev.descriptor.idProduct = cpu_to_le16(pid); + t->udev.descriptor.bDeviceClass = USB_CLASS_HUB; + t->alt.desc.bInterfaceClass = USB_CLASS_HUB; + t->intf.altsetting = &t->alt; + t->intf.cur_altsetting = &t->alt; + t->intf.num_altsetting = 1; + t->intf.dev.parent = &t->udev.dev; + + return t; +} + +/* + * The VIA Labs VL122 hub on the Apple J700 misbehaves in runtime suspend. It stays + * out of autosuspend, but keeps its wakeup setting for system sleep. + */ +static void usb_quirks_vl122_test(struct kunit *test) +{ + struct usb_quirks_test *t = usb_quirks_test_hub(test, 0x2109, 0x2122); + + KUNIT_EXPECT_FALSE(test, usb_detect_device_list_quirks(&t->udev) & + USB_QUIRK_IGNORE_REMOTE_WAKEUP); + KUNIT_EXPECT_TRUE(test, usb_hub_id_quirks(&t->intf) & HUB_QUIRK_DISABLE_AUTOSUSPEND); +} + +/* Another VIA Labs hub keeps its own device quirk and no hub quirk. */ +static void usb_quirks_other_via_hub_test(struct kunit *test) +{ + struct usb_quirks_test *t = usb_quirks_test_hub(test, 0x2109, 0x2817); + + KUNIT_EXPECT_EQ(test, usb_detect_device_list_quirks(&t->udev), USB_QUIRK_NO_LPM); + KUNIT_EXPECT_EQ(test, usb_hub_id_quirks(&t->intf), 0UL); +} + +/* A hub that no table names gets no quirks. */ +static void usb_quirks_generic_hub_test(struct kunit *test) +{ + struct usb_quirks_test *t = usb_quirks_test_hub(test, 0x1d6b, 0x0002); + + KUNIT_EXPECT_EQ(test, usb_detect_device_list_quirks(&t->udev), 0U); + KUNIT_EXPECT_EQ(test, usb_hub_id_quirks(&t->intf), 0UL); +} + +static struct kunit_case usb_quirks_test_cases[] = { + KUNIT_CASE(usb_quirks_vl122_test), + KUNIT_CASE(usb_quirks_other_via_hub_test), + KUNIT_CASE(usb_quirks_generic_hub_test), + {} +}; + +static struct kunit_suite usb_quirks_test_suite = { + .name = "usb_quirks", + .test_cases = usb_quirks_test_cases, +}; + +kunit_test_suite(usb_quirks_test_suite); + +MODULE_DESCRIPTION("KUnit tests for USB device and hub quirk matching"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/core/usb.h b/drivers/usb/core/usb.h index a9b37aeb515be..c8880c20f1563 100644 --- a/drivers/usb/core/usb.h +++ b/drivers/usb/core/usb.h @@ -36,6 +36,9 @@ extern int usb_authorize_device(struct usb_device *); extern void usb_deauthorize_interface(struct usb_interface *); extern void usb_authorize_interface(struct usb_interface *); extern void usb_detect_quirks(struct usb_device *udev); +#if IS_ENABLED(CONFIG_KUNIT) +u32 usb_detect_device_list_quirks(struct usb_device *udev); +#endif extern void usb_detect_interface_quirks(struct usb_device *udev); extern void usb_release_quirk_list(void); extern bool usb_endpoint_is_ignored(struct usb_device *udev, From 8700f11cd53554f065b4b1ffb53ad7f0af66f4c3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:26:17 +1100 Subject: [PATCH 1127/1579] tools: aurora-wake: Add an unattended suspend-to-idle regression test Suspend several times with an RTC alarm (rtc1, the PMU RTC, on the J700) and check after each resume that the sleep lasted until the alarm, that suspend_stats reports no failures, that every USB device is back, that the power/wakeup settings of USB devices, xHCI and DWC3 are unchanged, and that the log has no xHCI resume errors, warnings or oopses. Needs no one at the machine. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-wake/s2idle-rtc-regression.sh | 144 +++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100755 tools/aurora-wake/s2idle-rtc-regression.sh diff --git a/tools/aurora-wake/s2idle-rtc-regression.sh b/tools/aurora-wake/s2idle-rtc-regression.sh new file mode 100755 index 0000000000000..d6eee0910010d --- /dev/null +++ b/tools/aurora-wake/s2idle-rtc-regression.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Unattended suspend-to-idle regression test for Apple silicon (J700 and others). +# +# Suspends with an RTC alarm (rtc1, apple-pmu-rtc, by default) several times and +# checks after every resume that: +# - the sleep lasted until the alarm (no spurious wakeup, e.g. from a running xHC), +# - suspend_stats reports no new failures, +# - every USB device present before is back, +# - the power/wakeup settings of USB devices, xHCI and DWC3 are unchanged, +# - the kernel log has no xHCI resume errors, warnings or oopses. +# +# Usage (as root): +# s2idle-rtc-regression.sh [-c cycles] [-s seconds] [-r rtc] [-o outdir] [-i ignore-regex] +# -i takes an extended regex of known, unrelated log lines to leave out of the +# error check, e.g. -i dcp for a display driver failing its own resume. +# Exit status: 0 when every cycle passed, 1 otherwise, 2 for a setup error. + +set -u + +cycles=3 +secs=20 +rtc=rtc1 +out=/var/tmp/s2idle-rtc-regression-$(date +%Y%m%d-%H%M%S) +ignore= + +while getopts "c:s:r:o:i:" opt; do + case $opt in + c) cycles=$OPTARG ;; + s) secs=$OPTARG ;; + r) rtc=$OPTARG ;; + o) out=$OPTARG ;; + i) ignore=$OPTARG ;; + *) echo "usage: $0 [-c cycles] [-s secs] [-r rtc] [-o outdir] [-i regex]" >&2; exit 2 ;; + esac +done + +[ "$(id -u)" = 0 ] || { echo "must run as root" >&2; exit 2; } +command -v rtcwake > /dev/null || { echo "rtcwake not found" >&2; exit 2; } +[ -e "/sys/class/rtc/$rtc/wakealarm" ] || { echo "$rtc has no wakealarm" >&2; exit 2; } +grep -qw s2idle /sys/power/mem_sleep 2> /dev/null || grep -qw freeze /sys/power/state || + { echo "suspend-to-idle not available" >&2; exit 2; } + +mkdir -p "$out" + +# Errors that must never appear in the log of a suspend/resume cycle. +bad='Host halt failed|xHC restore state timeout|xHC error in resume' +bad+='|Controller not ready at resume|HC died|Clearing Run/Stop bit failed' +bad+='|Controller Save State failed|PM: Some devices failed' +bad+='|PM: .*failed to suspend|PM: .*failed to resume|Freezing .* failed' +bad+='|unable to enumerate USB device|WARNING:|BUG:|Oops|Call trace:' + +usb_devices() { + local d + for d in /sys/bus/usb/devices/*; do + [ -f "$d/idVendor" ] || continue + echo "$(basename "$d") $(cat "$d/idVendor"):$(cat "$d/idProduct")" + done | sort +} + +wakeup_state() { + local d + for d in /sys/bus/usb/devices/* /sys/bus/platform/devices/{*.usb,xhci-hcd.*}; do + [ -f "$d/power/wakeup" ] && echo "$(basename "$d") $(cat "$d/power/wakeup")" + done | sort +} + +# The lines that differ between two files, on one line +changes() { + diff "$1" "$2" | grep '^[<>]' | tr '\n' ' ' +} + +# Drop the log lines the caller declared as known and unrelated +known_filter() { + if [ -n "$ignore" ]; then grep -Ev "$ignore"; else cat; fi +} + +stat_fail() { + cat /sys/power/suspend_stats/fail 2> /dev/null || echo 0 +} + +usb_devices > "$out/usb-before.txt" +wakeup_state > "$out/wakeup-before.txt" +uname -a > "$out/uname.txt" +echo "cycles=$cycles secs=$secs rtc=$rtc ignore=$ignore" > "$out/params.txt" + +failed=0 +for i in $(seq 1 "$cycles"); do + c="$out/cycle-$i" + mkdir -p "$c" + reasons=() + fail_before=$(stat_fail) + dmesg -C + sync + + t0=$(date +%s.%N) + rtcwake -d "$rtc" -m freeze -s "$secs" > "$c/rtcwake.txt" 2>&1 + rc=$? + t1=$(date +%s.%N) + slept=$(awk -v a="$t0" -v b="$t1" 'BEGIN { printf "%.1f", b - a }') + + [ $rc = 0 ] || reasons+=("rtcwake exited $rc") + if awk -v s="$slept" -v w="$secs" 'BEGIN { exit !(s + 1.5 < w) }'; then + wirq=$(cat /sys/power/pm_wakeup_irq 2>&1) + reasons+=("woke early after ${slept}s of ${secs}s (pm_wakeup_irq $wirq)") + fi + [ "$(stat_fail)" = "$fail_before" ] || reasons+=("suspend_stats fail count rose") + + # Give USB up to 15 s to come back. + for _ in $(seq 1 30); do + usb_devices > "$c/usb-after.txt" + cmp -s "$out/usb-before.txt" "$c/usb-after.txt" && break + sleep 0.5 + done + cmp -s "$out/usb-before.txt" "$c/usb-after.txt" || + reasons+=("USB devices differ: $(changes "$out/usb-before.txt" \ + "$c/usb-after.txt")") + + wakeup_state > "$c/wakeup-after.txt" + cmp -s "$out/wakeup-before.txt" "$c/wakeup-after.txt" || + reasons+=("power/wakeup changed: $(changes "$out/wakeup-before.txt" \ + "$c/wakeup-after.txt")") + + dmesg > "$c/dmesg.txt" + grep -E "$bad" "$c/dmesg.txt" | known_filter > "$c/bad-lines.txt" + [ -s "$c/bad-lines.txt" ] && + reasons+=("kernel log errors: $(head -3 "$c/bad-lines.txt" | tr '\n' ' ')") + + if [ ${#reasons[@]} = 0 ]; then + echo "cycle $i: PASS (slept ${slept}s)" | tee -a "$out/summary.txt" + else + failed=1 + echo "cycle $i: FAIL (slept ${slept}s)" | tee -a "$out/summary.txt" + printf ' - %s\n' "${reasons[@]}" | tee -a "$out/summary.txt" + fi +done + +if [ $failed = 0 ]; then + echo "RESULT: PASS ($cycles cycles) evidence: $out" | tee -a "$out/summary.txt" +else + echo "RESULT: FAIL evidence: $out" | tee -a "$out/summary.txt" +fi +exit $failed From a28673c796d5475d6aeb0f05052a28f233a09031 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 18:20:43 +1100 Subject: [PATCH 1128/1579] usb: core: Keep a hub awake in suspend-to-idle when it cannot pass on wakeup The VIA Labs VL122 USB 2.0 hub on the Apple J700 does not pass a remote wakeup from a downstream device upstream while it is suspended itself, the same limitation that makes it miss downstream connects in runtime suspend. With the whole chain armed and the xHC running in suspend-to-idle, a keypress on a keyboard behind it never reaches the root port: PORTSC stays in U3 without a link state change, with the keyboard behind a dock and behind a plain USB 2.0 hub alike. The hub does see the resume while it is awake: an active hub completes a downstream resume on its port and reports it through its status change endpoint. Keep such a hub awake in suspend-to-idle when a device below it may wake the system, as the USB offload support already does for devices that stay in use. Every device between it and the root hub, the root hub included, stays awake too, so that the root port stays in U0 and the running controller keeps polling the hub's interrupt endpoint. The devices below the hub are suspended as before. A remote wakeup then completes a transfer on the hub's status endpoint, and the controller interrupt, armed for wakeup, ends suspend-to-idle. This needs a controller that keeps running in suspend-to-idle; xhci-plat says so through a new runs_in_s2idle flag of struct usb_hcd. Mark the VL122 with a new USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP; it keeps HUB_QUIRK_DISABLE_AUTOSUSPEND for runtime PM. The cost is an active hub and an active root port while a wake source sits below the hub. Tested on the J700 with a USB keyboard behind the VL122 and a dock, lid open and closed: a keypress ends suspend-to-idle through the xHCI interrupt, 3.7 s from entering suspend when pressed right away. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/driver.c | 56 +++++++++++++++++++++++++----- drivers/usb/core/quirks.c | 4 +++ drivers/usb/core/usb-quirks-test.c | 10 +++--- drivers/usb/host/xhci-plat.c | 9 +++++ include/linux/usb.h | 1 + include/linux/usb/hcd.h | 2 ++ include/linux/usb/quirks.h | 7 ++++ 7 files changed, 76 insertions(+), 13 deletions(-) diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c index f63004417058e..293a6b5facb46 100644 --- a/drivers/usb/core/driver.c +++ b/drivers/usb/core/driver.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -1383,6 +1384,37 @@ static int usb_resume_interface(struct usb_device *udev, return status; } +/* + * Whether to keep a device awake in suspend-to-idle so that a remote wakeup + * from below can reach the host controller: a hub that does not pass one on + * while it is suspended itself, and every device between such a hub and the + * root hub, root hub included. This only works while the host controller keeps + * running, so its interrupt reports the hub's status change. + */ +static bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_message_t msg) +{ + struct usb_hcd *hcd = bus_to_hcd(udev->bus); + struct usb_device *child; + int port1; + + if (msg.event != PM_EVENT_SUSPEND || !hcd->runs_in_s2idle || + pm_suspend_target_state != PM_SUSPEND_TO_IDLE || + !device_may_wakeup(hcd->self.controller)) + return false; + + if ((udev->quirks & USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP) && + udev->do_remote_wakeup && device_wakeup_path(&udev->dev)) + return true; + + /* Children are done with their suspend before their parent starts */ + usb_hub_for_each_child(udev, port1, child) { + if (child->awake_in_sleep) + return true; + } + + return false; +} + /** * usb_suspend_both - suspend a USB device and its interfaces * @udev: the usb_device to suspend @@ -1426,6 +1458,9 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) dev_dbg(&udev->dev, "device offloaded, skip suspend.\n"); offload_active = true; } + udev->awake_in_sleep = usb_stay_awake_for_wakeup(udev, msg); + if (udev->awake_in_sleep) + dev_dbg(&udev->dev, "kept awake to pass on remote wakeup\n"); /* Suspend all the interfaces and then udev itself */ if (udev->actconfig) { @@ -1438,9 +1473,9 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) * interrupt urbs, allowing interrupt events to be * handled during system suspend. */ - if (offload_active && intf->needs_remote_wakeup) { - dev_dbg(&intf->dev, - "device offloaded, skip suspend.\n"); + if ((offload_active || udev->awake_in_sleep) && + intf->needs_remote_wakeup) { + dev_dbg(&intf->dev, "interface kept active, skip suspend.\n"); continue; } status = usb_suspend_interface(udev, intf, msg); @@ -1453,7 +1488,7 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) } } if (status == 0) { - if (!offload_active) + if (!offload_active && !udev->awake_in_sleep) status = usb_suspend_device(udev, msg); /* @@ -1497,7 +1532,7 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) /* If the suspend succeeded then prevent any more URB submissions * and flush any outstanding URBs. */ - } else { + } else if (!udev->awake_in_sleep) { udev->can_submit = 0; if (!offload_active) { for (i = 0; i < 16; ++i) { @@ -1508,8 +1543,10 @@ static int usb_suspend_both(struct usb_device *udev, pm_message_t msg) } done: - if (status != 0) + if (status != 0) { + udev->awake_in_sleep = 0; usb_offload_set_pm_locked(udev, false); + } dev_vdbg(&udev->dev, "%s: status %d\n", __func__, status); return status; } @@ -1540,7 +1577,9 @@ static int usb_resume_both(struct usb_device *udev, pm_message_t msg) int i; struct usb_interface *intf; bool offload_active = false; + bool awake = udev->awake_in_sleep; + udev->awake_in_sleep = 0; if (udev->state == USB_STATE_NOTATTACHED) { status = -ENODEV; goto done; @@ -1568,9 +1607,8 @@ static int usb_resume_both(struct usb_device *udev, pm_message_t msg) * pending interrupt urbs, allowing interrupt events * to be handled during system suspend. */ - if (offload_active && intf->needs_remote_wakeup) { - dev_dbg(&intf->dev, - "device offloaded, skip resume.\n"); + if ((offload_active || awake) && intf->needs_remote_wakeup) { + dev_dbg(&intf->dev, "interface kept active, skip resume.\n"); continue; } usb_resume_interface(udev, intf, msg, diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index 94a8f711da9fc..bf38174963e5c 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -584,6 +584,10 @@ static const struct usb_device_id usb_quirk_list[] = { /* VLI disk */ { USB_DEVICE(0x2109, 0x0711), .driver_info = USB_QUIRK_NO_LPM }, + /* VIA Labs, Inc. VL122 USB2.0 Hub */ + { USB_DEVICE(0x2109, 0x2122), .driver_info = + USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP }, + /* VIA Labs, Inc. USB2.0 Hub */ { USB_DEVICE(0x2109, 0x2817), .driver_info = USB_QUIRK_NO_LPM }, diff --git a/drivers/usb/core/usb-quirks-test.c b/drivers/usb/core/usb-quirks-test.c index 6426f7951ff0e..81ebc58238b0d 100644 --- a/drivers/usb/core/usb-quirks-test.c +++ b/drivers/usb/core/usb-quirks-test.c @@ -38,15 +38,17 @@ static struct usb_quirks_test *usb_quirks_test_hub(struct kunit *test, u16 vid, } /* - * The VIA Labs VL122 hub on the Apple J700 misbehaves in runtime suspend. It stays - * out of autosuspend, but keeps its wakeup setting for system sleep. + * The VIA Labs VL122 hub on the Apple J700 does not pass on remote wakeup while it + * is suspended. It stays out of autosuspend and keeps its wakeup setting, and is + * kept awake in suspend-to-idle instead. */ static void usb_quirks_vl122_test(struct kunit *test) { struct usb_quirks_test *t = usb_quirks_test_hub(test, 0x2109, 0x2122); + u32 quirks = usb_detect_device_list_quirks(&t->udev); - KUNIT_EXPECT_FALSE(test, usb_detect_device_list_quirks(&t->udev) & - USB_QUIRK_IGNORE_REMOTE_WAKEUP); + KUNIT_EXPECT_FALSE(test, quirks & USB_QUIRK_IGNORE_REMOTE_WAKEUP); + KUNIT_EXPECT_TRUE(test, quirks & USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP); KUNIT_EXPECT_TRUE(test, usb_hub_id_quirks(&t->intf) & HUB_QUIRK_DISABLE_AUTOSUSPEND); } diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 8a4413f817e72..6d2aa0d8f7492 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -367,6 +367,15 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s goto put_usb3_hcd; } + /* + * Let the USB core keep hubs awake that need it for wakeup, see + * XHCI_RUN_IN_S2IDLE. Only now are all quirk sources merged, the xhci_hcd + * quirks module parameter included. + */ + hcd->runs_in_s2idle = !!(xhci->quirks & XHCI_RUN_IN_S2IDLE); + if (xhci->shared_hcd) + xhci->shared_hcd->runs_in_s2idle = hcd->runs_in_s2idle; + device_enable_async_suspend(&pdev->dev); pm_runtime_put_noidle(&pdev->dev); diff --git a/include/linux/usb.h b/include/linux/usb.h index 25a203ac7a7ee..e8c5cfe9e0591 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -708,6 +708,7 @@ struct usb_device { unsigned usb2_hw_lpm_allowed:1; unsigned usb3_lpm_u1_enabled:1; unsigned usb3_lpm_u2_enabled:1; + unsigned awake_in_sleep:1; int string_langid; /* static strings from the device */ diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index 181db044c4d2e..f4520d9922a2b 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -160,6 +160,8 @@ struct usb_hcd { unsigned tpl_support:1; /* OTG & EH TPL support */ unsigned cant_recv_wakeups:1; /* wakeup requests from downstream aren't received */ + unsigned runs_in_s2idle:1; + /* HC keeps running in suspend-to-idle with wakeup on */ unsigned int irq; /* irq allocated */ void __iomem *regs; /* device memory/io */ diff --git a/include/linux/usb/quirks.h b/include/linux/usb/quirks.h index a4043b33c2c2a..8bcceb0d02bd8 100644 --- a/include/linux/usb/quirks.h +++ b/include/linux/usb/quirks.h @@ -84,4 +84,11 @@ /* Use a 255 bytes config descriptor request mirroring windows behavior */ #define USB_QUIRK_WINDOWS_CONFIG_REQ_SIZE BIT(19) +/* + * Hub does not pass a downstream remote wakeup upstream while it is suspended + * itself. Keep it awake in suspend-to-idle when a device below may wake the + * system and the host controller keeps running. + */ +#define USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP BIT(20) + #endif /* __LINUX_USB_QUIRKS_H */ From 6ad2760be257077ee50e6279a94b02aea153a772 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:31:23 +1100 Subject: [PATCH 1129/1579] usb: xhci: plat: Report a controller that stopped in suspend-to-idle as dead A controller left running in suspend-to-idle is expected to come back still running. If it stopped all the same, because power was cut or it hit a Host System Error or Host Controller Error, resume skipped xhci_resume() and handed the root hubs a dead controller. It is not brought back here. Reinitialising it, as xhci_resume() does when told power was lost, is only safe on a quiesced controller: one whose devices were suspended and whose URBs were flushed. In suspend-to-idle devices may have been kept awake, with URBs queued and their interrupt and hub retry paths live, and a reset under them frees state those paths still use. Report the controller dead instead, the path for a controller that stops responding: first mask its interrupts and halt it with xhci_halt(), as the other error paths do, unless its registers no longer answer, since the USB core ignores the interrupts of a dead HCD and a pending one would never be acknowledged; then xhci_hc_died() gives back every pending URB and lets the USB core disconnect the devices. USB then needs the controller to be bound again. The new xhci_quiesce_dying() and xhci_hc_died() are exported for this. The check for a stopped controller and the quiescing are tested with KUnit; the quiescing test fails when the interrupts are left enabled. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci-plat-test.c | 48 +++++++++++++++++++++++++++++++ drivers/usb/host/xhci-plat.c | 31 +++++++++++++++++++- drivers/usb/host/xhci-plat.h | 1 + drivers/usb/host/xhci-ring.c | 1 + drivers/usb/host/xhci.c | 12 ++++++++ drivers/usb/host/xhci.h | 1 + 6 files changed, 93 insertions(+), 1 deletion(-) diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c index 410cf99da08d6..844e71d7ecdb8 100644 --- a/drivers/usb/host/xhci-plat-test.c +++ b/drivers/usb/host/xhci-plat-test.c @@ -45,6 +45,52 @@ static void xhci_plat_keep_running_other_quirks_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, xhci_plat_keep_running(~0ULL, PM_SUSPEND_TO_IDLE, true, true)); } +/* A running controller is left alone; a halted, failed or unpowered one is not. */ +static void xhci_plat_stopped_test(struct kunit *test) +{ + /* Running, with an event pending and a port change */ + KUNIT_EXPECT_FALSE(test, xhci_plat_stopped(0)); + KUNIT_EXPECT_FALSE(test, xhci_plat_stopped(STS_EINT | STS_PORT)); + /* Halted, Host System Error, Host Controller Error */ + KUNIT_EXPECT_TRUE(test, xhci_plat_stopped(STS_HALT)); + KUNIT_EXPECT_TRUE(test, xhci_plat_stopped(STS_FATAL | STS_HALT)); + KUNIT_EXPECT_TRUE(test, xhci_plat_stopped(STS_HCE)); + /* Register reads of a powered-off block */ + KUNIT_EXPECT_TRUE(test, xhci_plat_stopped(~0U)); +} + +/* + * A controller reported dead is halted with its interrupts masked first: + * the USB core ignores the interrupts of a dead HCD, so one left enabled + * and pending would never be acknowledged. + */ +static void xhci_plat_quiesce_dying_test(struct kunit *test) +{ + struct xhci_op_regs *op; + struct xhci_hcd *xhci; + + xhci = kunit_kzalloc(test, sizeof(*xhci), GFP_KERNEL); + op = kunit_kzalloc(test, sizeof(*op), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, xhci); + KUNIT_ASSERT_NOT_NULL(test, op); + xhci->op_regs = (__force struct xhci_op_regs __iomem *)op; + + /* A Host System Error with an event pending, interrupts enabled */ + writel(CMD_RUN | XHCI_IRQS, &xhci->op_regs->command); + writel(STS_HALT | STS_FATAL | STS_EINT, &xhci->op_regs->status); + xhci_quiesce_dying(xhci); + KUNIT_EXPECT_EQ(test, readl(&xhci->op_regs->command) & XHCI_IRQS, 0U); + KUNIT_EXPECT_TRUE(test, xhci->xhc_state & XHCI_STATE_HALTED); + + /* Registers that no longer answer are left alone. */ + xhci->xhc_state = 0; + writel(~0U, &xhci->op_regs->status); + writel(XHCI_IRQS, &xhci->op_regs->command); + xhci_quiesce_dying(xhci); + KUNIT_EXPECT_EQ(test, readl(&xhci->op_regs->command), (u32)XHCI_IRQS); + KUNIT_EXPECT_FALSE(test, xhci->xhc_state & XHCI_STATE_HALTED); +} + static u64 xhci_plat_test_quirks(struct kunit *test, const struct property_entry *props) { struct device *dev; @@ -99,6 +145,8 @@ static void xhci_plat_all_property_quirks_test(struct kunit *test) static struct kunit_case xhci_plat_test_cases[] = { KUNIT_CASE(xhci_plat_keep_running_test), KUNIT_CASE(xhci_plat_keep_running_other_quirks_test), + KUNIT_CASE(xhci_plat_stopped_test), + KUNIT_CASE(xhci_plat_quiesce_dying_test), KUNIT_CASE(xhci_plat_no_property_quirks_test), KUNIT_CASE(xhci_plat_run_in_s2idle_property_test), KUNIT_CASE(xhci_plat_all_property_quirks_test), diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 6d2aa0d8f7492..3f11e5609f613 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -534,6 +534,13 @@ VISIBLE_IF_KUNIT bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, } EXPORT_SYMBOL_IF_KUNIT(xhci_plat_keep_running); +/* Whether a controller that was left running reads back as stopped or failed */ +VISIBLE_IF_KUNIT bool xhci_plat_stopped(u32 usbsts) +{ + return usbsts == ~(u32)0 || (usbsts & (STS_HALT | STS_FATAL | STS_HCE)); +} +EXPORT_SYMBOL_IF_KUNIT(xhci_plat_stopped); + static bool xhci_plat_run_in_s2idle(struct device *dev) { struct usb_hcd *hcd = dev_get_drvdata(dev); @@ -629,8 +636,30 @@ static int xhci_plat_resume(struct device *dev) } if (priv->running_in_s2idle) { + struct xhci_hcd *xhci = hcd_to_xhci(hcd); + u32 status = readl(&xhci->op_regs->status); + priv->running_in_s2idle = 0; - dev_dbg(dev, "kept running in suspend-to-idle, skip resume.\n"); + if (!xhci_plat_stopped(status)) { + dev_dbg(dev, "kept running in suspend-to-idle, skip resume.\n"); + return 0; + } + + /* + * The controller was left running on the platform's promise that it + * stays powered, and stopped all the same: power was cut, or it hit an + * error. It is not brought back here. Devices may have been kept awake + * through the sleep, with URBs queued and their interrupt and retry + * paths live, and a reset under them is not safe. Mask its interrupts and + * halt it, and report it dead, which gives back every pending URB and + * lets the USB core disconnect the devices. USB then needs the controller + * bound again. + */ + dev_err(dev, "stopped in suspend-to-idle (USBSTS %#010x)\n", status); + spin_lock_irq(&xhci->lock); + xhci_quiesce_dying(xhci); + xhci_hc_died(xhci); + spin_unlock_irq(&xhci->lock); return 0; } diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h index 5a43502ca27a9..1956db312b8aa 100644 --- a/drivers/usb/host/xhci-plat.h +++ b/drivers/usb/host/xhci-plat.h @@ -40,6 +40,7 @@ extern const struct dev_pm_ops xhci_plat_pm_ops; u64 xhci_plat_property_quirks(struct device *dev); bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, bool may_wakeup, bool wakeup_path); +bool xhci_plat_stopped(u32 usbsts); #endif #endif /* _XHCI_PLAT_H */ diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index e47e644b296ea..2ec1323998889 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -1405,6 +1405,7 @@ void xhci_hc_died(struct xhci_hcd *xhci) if (notify) usb_hc_died(xhci_to_hcd(xhci)); } +EXPORT_SYMBOL_GPL(xhci_hc_died); /* * When we get a completion for a Set Transfer Ring Dequeue Pointer command, diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 5b15eae233d3f..89490f0770c0c 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -145,6 +145,18 @@ int xhci_halt(struct xhci_hcd *xhci) return ret; } +/* + * Stop a controller that is about to be reported dead: mask its interrupts and + * halt it, as the other error paths do before xhci_hc_died(). A controller whose + * registers no longer answer is left alone. Called with xhci->lock held. + */ +void xhci_quiesce_dying(struct xhci_hcd *xhci) +{ + if (readl(&xhci->op_regs->status) != ~(u32)0) + xhci_halt(xhci); +} +EXPORT_SYMBOL_GPL(xhci_quiesce_dying); + /* * Set the run bit and wait for the host to be running. */ diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index 7cd09324b4088..0f988654e2ab1 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1983,6 +1983,7 @@ struct xhci_hub *xhci_get_rhub(struct usb_hcd *hcd); enum usb_link_tunnel_mode xhci_port_is_tunneled(struct xhci_hcd *xhci, struct xhci_port *port); void xhci_hc_died(struct xhci_hcd *xhci); +void xhci_quiesce_dying(struct xhci_hcd *xhci); #ifdef CONFIG_PM int xhci_bus_suspend(struct usb_hcd *hcd); From 7dfc99ae6302ec23f0e9fcefb349fe6d94ff0a02 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:32:07 +1100 Subject: [PATCH 1130/1579] usb: xhci: plat: Fail system suspend when the controller does not resume Before a system suspend, a runtime suspended controller is resumed: it is halted, and both the halting path and the path that keeps it running in suspend-to-idle need it running. The result of pm_runtime_resume() was ignored. When it failed, the keep-running path marked the controller as running and reported success, so the system went to sleep without the wake source it was promised, and the halting path went on to suspend a controller that was not running. Fail the system suspend with the error instead, before anything is marked. The runtime resume is a helper with KUnit tests, built with CONFIG_PM only; the error test fails when the result is ignored. Fixes: 4d9fbe0bed99 ("usb: xhci: plat: Allow the controller to keep running in suspend-to-idle") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci-plat-test.c | 53 +++++++++++++++++++++++++++++++ drivers/usb/host/xhci-plat.c | 25 ++++++++++++--- drivers/usb/host/xhci-plat.h | 1 + 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c index 844e71d7ecdb8..00646f7ba2f5a 100644 --- a/drivers/usb/host/xhci-plat-test.c +++ b/drivers/usb/host/xhci-plat-test.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "xhci.h" @@ -91,6 +92,54 @@ static void xhci_plat_quiesce_dying_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, xhci->xhc_state & XHCI_STATE_HALTED); } +#ifdef CONFIG_PM +static int xhci_plat_test_rpm_ret; + +static int xhci_plat_test_runtime_resume(struct device *dev) +{ + return xhci_plat_test_rpm_ret; +} + +static struct dev_pm_domain xhci_plat_test_pm_domain = { + .ops = { .runtime_resume = xhci_plat_test_runtime_resume }, +}; + +static struct device *xhci_plat_test_rpm_device(struct kunit *test, int rpm_ret) +{ + struct device *dev = kunit_device_register(test, "xhci-plat-rpm-test"); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + xhci_plat_test_rpm_ret = rpm_ret; + dev->pm_domain = &xhci_plat_test_pm_domain; + pm_runtime_set_suspended(dev); + pm_runtime_enable(dev); + return dev; +} + +/* A runtime resume that fails before system sleep fails the system suspend. */ +static void xhci_plat_resume_from_runtime_error_test(struct kunit *test) +{ + struct device *dev = xhci_plat_test_rpm_device(test, -ETIMEDOUT); + + KUNIT_EXPECT_EQ(test, xhci_plat_resume_from_runtime(dev), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + pm_runtime_disable(dev); + dev->pm_domain = NULL; +} + +/* A successful one, and an active controller, let it go ahead. */ +static void xhci_plat_resume_from_runtime_ok_test(struct kunit *test) +{ + struct device *dev = xhci_plat_test_rpm_device(test, 0); + + KUNIT_EXPECT_EQ(test, xhci_plat_resume_from_runtime(dev), 0); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + KUNIT_EXPECT_EQ(test, xhci_plat_resume_from_runtime(dev), 0); + pm_runtime_disable(dev); + dev->pm_domain = NULL; +} +#endif /* CONFIG_PM */ + static u64 xhci_plat_test_quirks(struct kunit *test, const struct property_entry *props) { struct device *dev; @@ -147,6 +196,10 @@ static struct kunit_case xhci_plat_test_cases[] = { KUNIT_CASE(xhci_plat_keep_running_other_quirks_test), KUNIT_CASE(xhci_plat_stopped_test), KUNIT_CASE(xhci_plat_quiesce_dying_test), +#ifdef CONFIG_PM + KUNIT_CASE(xhci_plat_resume_from_runtime_error_test), + KUNIT_CASE(xhci_plat_resume_from_runtime_ok_test), +#endif KUNIT_CASE(xhci_plat_no_property_quirks_test), KUNIT_CASE(xhci_plat_run_in_s2idle_property_test), KUNIT_CASE(xhci_plat_all_property_quirks_test), diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 3f11e5609f613..aec29654680e3 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -490,14 +490,28 @@ void xhci_plat_remove(struct platform_device *dev) } EXPORT_SYMBOL_GPL(xhci_plat_remove); +/* A runtime suspended controller is halted: bring it back before system sleep. */ +VISIBLE_IF_KUNIT int xhci_plat_resume_from_runtime(struct device *dev) +{ + int ret; + + if (!pm_runtime_suspended(dev)) + return 0; + + ret = pm_runtime_resume(dev); + return ret < 0 ? ret : 0; +} +EXPORT_SYMBOL_IF_KUNIT(xhci_plat_resume_from_runtime); + static int xhci_plat_suspend_common(struct device *dev) { struct usb_hcd *hcd = dev_get_drvdata(dev); struct xhci_hcd *xhci = hcd_to_xhci(hcd); int ret; - if (pm_runtime_suspended(dev)) - pm_runtime_resume(dev); + ret = xhci_plat_resume_from_runtime(dev); + if (ret) + return ret; ret = xhci_priv_suspend_quirk(hcd); if (ret) @@ -562,9 +576,10 @@ static int xhci_plat_suspend(struct device *dev) } if (xhci_plat_run_in_s2idle(dev)) { - /* A runtime suspended controller is halted, start it again. */ - if (pm_runtime_suspended(dev)) - pm_runtime_resume(dev); + int ret = xhci_plat_resume_from_runtime(dev); + + if (ret) + return ret; priv->running_in_s2idle = 1; dev_dbg(dev, "wakeup enabled, keep running in suspend-to-idle.\n"); return 0; diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h index 1956db312b8aa..da55117cbf3f3 100644 --- a/drivers/usb/host/xhci-plat.h +++ b/drivers/usb/host/xhci-plat.h @@ -41,6 +41,7 @@ u64 xhci_plat_property_quirks(struct device *dev); bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, bool may_wakeup, bool wakeup_path); bool xhci_plat_stopped(u32 usbsts); +int xhci_plat_resume_from_runtime(struct device *dev); #endif #endif /* _XHCI_PLAT_H */ From 8d9758ed9430358118fb0065b2a1f9801309ca5f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:23:56 +1100 Subject: [PATCH 1131/1579] tools: aurora-wake: Harden the suspend-to-idle regression script The regression script cleared the kernel log before every cycle. That threw away the log from before the test, and every message logged between one cycle's snapshot and the next clear, so a late USB recovery failure could vanish from both the saved log and the error check. Save the log before the first cycle instead, check each cycle's part of it from a mark without clearing anything, and check what was logged after the last cycle as well. The mark moves past exactly what was saved, from the same read, so a message logged between two reads lands in the next collection. A log that wrapped meanwhile is checked whole. A cycle count of 0, a negative one or a non-number ran no cycle and still reported a pass. Accept only positive counts and a sleep of at least 5 s, refuse other arguments with exit status 2, and pass only when every requested cycle ran. It ran as root and wrote its evidence to a predictable path under /var/tmp, creating it with mkdir -p, so a symlink planted there could redirect what it writes. Create a fresh private directory with mktemp -d by default, and accept an explicit -o only as an absolute path that does not exist yet, created with mode 0700. The early-wake check measured the whole rtcwake command, so suspend and resume work could hide a wake long before the alarm. Measure the time actually spent suspended as the growth of CLOCK_BOOTTIME over CLOCK_MONOTONIC, and report the whole command time beside it. Some failed checks passed quietly. A kernel log that cannot be read, a suspend failure count that cannot be read, and an -i expression grep rejects, which made the filter hide every error, now fail the cycle or the run instead. Add -n to check the arguments without suspending, and a hardware-free test of the argument handling, the output directory rules and the kernel log collection. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-wake/s2idle-rtc-regression.sh | 135 +++++++++++++++--- .../aurora-wake/test-s2idle-rtc-regression.sh | 79 ++++++++++ 2 files changed, 197 insertions(+), 17 deletions(-) create mode 100755 tools/aurora-wake/test-s2idle-rtc-regression.sh diff --git a/tools/aurora-wake/s2idle-rtc-regression.sh b/tools/aurora-wake/s2idle-rtc-regression.sh index d6eee0910010d..89a131aca0b59 100755 --- a/tools/aurora-wake/s2idle-rtc-regression.sh +++ b/tools/aurora-wake/s2idle-rtc-regression.sh @@ -5,44 +5,88 @@ # # Suspends with an RTC alarm (rtc1, apple-pmu-rtc, by default) several times and # checks after every resume that: -# - the sleep lasted until the alarm (no spurious wakeup, e.g. from a running xHC), +# - the time actually spent suspended (CLOCK_BOOTTIME minus CLOCK_MONOTONIC) +# lasted until the alarm: no spurious wakeup, e.g. from a running xHC, # - suspend_stats reports no new failures, # - every USB device present before is back, # - the power/wakeup settings of USB devices, xHCI and DWC3 are unchanged, # - the kernel log has no xHCI resume errors, warnings or oopses. # # Usage (as root): -# s2idle-rtc-regression.sh [-c cycles] [-s seconds] [-r rtc] [-o outdir] [-i ignore-regex] +# s2idle-rtc-regression.sh [-n] [-c cycles] [-s seconds] [-r rtc] [-o new-dir] [-i ignore-regex] # -i takes an extended regex of known, unrelated log lines to leave out of the # error check, e.g. -i dcp for a display driver failing its own resume. -# Exit status: 0 when every cycle passed, 1 otherwise, 2 for a setup error. +# -n only checks the arguments and prints what would run. +# The kernel log is never cleared: it is saved before the first cycle, and each +# cycle checks what was logged since the previous one. +# Exit status: 0 when every requested cycle ran and passed, 1 otherwise, 2 for +# a usage or setup error. set -u cycles=3 secs=20 rtc=rtc1 -out=/var/tmp/s2idle-rtc-regression-$(date +%Y%m%d-%H%M%S) +out= ignore= +dry_run=0 -while getopts "c:s:r:o:i:" opt; do +usage() { + echo "usage: $0 [-n] [-c cycles] [-s secs] [-r rtc] [-o outdir] [-i regex]" >&2 + exit 2 +} + +while getopts "nc:s:r:o:i:" opt; do case $opt in + n) dry_run=1 ;; c) cycles=$OPTARG ;; s) secs=$OPTARG ;; r) rtc=$OPTARG ;; o) out=$OPTARG ;; i) ignore=$OPTARG ;; - *) echo "usage: $0 [-c cycles] [-s secs] [-r rtc] [-o outdir] [-i regex]" >&2; exit 2 ;; + *) usage ;; esac done +[ $OPTIND -gt $# ] || usage + +# A count that runs no cycle must not report a pass. +[[ $cycles =~ ^[1-9][0-9]{0,3}$ ]] || { echo "cycles must be 1-9999: $cycles" >&2; usage; } +[[ $secs =~ ^[1-9][0-9]{0,4}$ ]] && [ "$secs" -ge 5 ] || + { echo "seconds must be 5-99999: $secs" >&2; usage; } +# An invalid expression would make the filter fail and hide every error. +if [ -n "$ignore" ]; then + grep -Eq -- "$ignore" < /dev/null + [ $? -le 1 ] || { echo "invalid -i expression: $ignore" >&2; usage; } +fi + +# The evidence goes to a directory this script creates for itself, with private +# permissions: a fresh one under /var/tmp by default, or the absolute path given +# with -o, which must not exist yet. It never writes through an existing path, +# so a planted symlink cannot redirect what root writes. +if [ -n "$out" ]; then + [[ $out == /* ]] || { echo "the output directory must be an absolute path" >&2; usage; } + [ ! -e "$out" ] && [ ! -L "$out" ] || + { echo "the output directory must not exist yet: $out" >&2; usage; } +fi + +if [ $dry_run = 1 ]; then + echo "would run $cycles cycles of $secs s on $rtc into ${out:-a new /var/tmp directory}" + exit 0 +fi [ "$(id -u)" = 0 ] || { echo "must run as root" >&2; exit 2; } command -v rtcwake > /dev/null || { echo "rtcwake not found" >&2; exit 2; } +command -v python3 > /dev/null || { echo "python3 not found" >&2; exit 2; } [ -e "/sys/class/rtc/$rtc/wakealarm" ] || { echo "$rtc has no wakealarm" >&2; exit 2; } grep -qw s2idle /sys/power/mem_sleep 2> /dev/null || grep -qw freeze /sys/power/state || { echo "suspend-to-idle not available" >&2; exit 2; } -mkdir -p "$out" +if [ -n "$out" ]; then + mkdir -m 700 -- "$out" || { echo "cannot create $out" >&2; exit 2; } +else + out=$(mktemp -d /var/tmp/s2idle-rtc-regression.XXXXXXXX) || + { echo "cannot create an output directory" >&2; exit 2; } +fi # Errors that must never appear in the log of a suspend/resume cycle. bad='Host halt failed|xHC restore state timeout|xHC error in resume' @@ -76,36 +120,75 @@ known_filter() { if [ -n "$ignore" ]; then grep -Ev "$ignore"; else cat; fi } +# The suspend failure count, or "unavailable": a missing count is not zero. stat_fail() { - cat /sys/power/suspend_stats/fail 2> /dev/null || echo 0 + local n + + n=$(cat /sys/power/suspend_stats/fail 2> /dev/null) + [[ $n =~ ^[0-9]+$ ]] && echo "$n" || echo unavailable } +# CLOCK_BOOTTIME and CLOCK_MONOTONIC: only the first one runs while suspended. +clocks() { + python3 -c 'import time; print(time.clock_gettime(time.CLOCK_BOOTTIME), + time.clock_gettime(time.CLOCK_MONOTONIC))' +} + +# --- kernel log collection (also run by test-s2idle-rtc-regression.sh) --- +# Save the kernel log since $mark (a line count) into $1 without clearing it, and +# move $mark past exactly what was saved, from the same read, so that nothing +# logged in between is skipped. A log that wrapped meanwhile no longer starts +# with $log_first: then all of it is saved. A failed read fails. +log_take() { + dmesg > "$out/.dmesg" || return 1 + if [ "$(head -n 1 "$out/.dmesg")" = "$log_first" ]; then + tail -n +"$((mark + 1))" "$out/.dmesg" > "$1" || return 1 + else + { echo "(kernel log wrapped; the whole log follows)"; cat "$out/.dmesg"; } > "$1" || + return 1 + fi + mark=$(wc -l < "$out/.dmesg") +} +# --- end of kernel log collection --- + usb_devices > "$out/usb-before.txt" wakeup_state > "$out/wakeup-before.txt" uname -a > "$out/uname.txt" echo "cycles=$cycles secs=$secs rtc=$rtc ignore=$ignore" > "$out/params.txt" +dmesg > "$out/dmesg-before.txt" || { echo "cannot read the kernel log" >&2; exit 2; } +log_first=$(head -n 1 "$out/dmesg-before.txt") +mark=$(wc -l < "$out/dmesg-before.txt") failed=0 +completed=0 for i in $(seq 1 "$cycles"); do c="$out/cycle-$i" mkdir -p "$c" reasons=() fail_before=$(stat_fail) - dmesg -C sync - t0=$(date +%s.%N) + read -r b0 m0 < <(clocks) rtcwake -d "$rtc" -m freeze -s "$secs" > "$c/rtcwake.txt" 2>&1 rc=$? - t1=$(date +%s.%N) - slept=$(awk -v a="$t0" -v b="$t1" 'BEGIN { printf "%.1f", b - a }') + read -r b1 m1 < <(clocks) + # Time spent suspended, and the whole command with entry and resume + slept=$(awk -v b="$b1" -v a="$b0" -v n="$m1" -v m="$m0" \ + 'BEGIN { printf "%.1f", (b - a) - (n - m) }') + took=$(awk -v b="$b1" -v a="$b0" 'BEGIN { printf "%.1f", b - a }') [ $rc = 0 ] || reasons+=("rtcwake exited $rc") - if awk -v s="$slept" -v w="$secs" 'BEGIN { exit !(s + 1.5 < w) }'; then + # The alarm counts from before suspend entry: allow 3 s for entry. + if awk -v s="$slept" -v w="$secs" 'BEGIN { exit !(s + 3 < w) }'; then wirq=$(cat /sys/power/pm_wakeup_irq 2>&1) reasons+=("woke early after ${slept}s of ${secs}s (pm_wakeup_irq $wirq)") fi - [ "$(stat_fail)" = "$fail_before" ] || reasons+=("suspend_stats fail count rose") + fail_after=$(stat_fail) + if [ "$fail_before" = unavailable ] || [ "$fail_after" = unavailable ]; then + reasons+=("suspend_stats fail count unavailable") + elif [ "$fail_after" != "$fail_before" ]; then + reasons+=("suspend_stats fail count rose") + fi # Give USB up to 15 s to come back. for _ in $(seq 1 30); do @@ -122,20 +205,38 @@ for i in $(seq 1 "$cycles"); do reasons+=("power/wakeup changed: $(changes "$out/wakeup-before.txt" \ "$c/wakeup-after.txt")") - dmesg > "$c/dmesg.txt" + log_take "$c/dmesg.txt" || reasons+=("kernel log unavailable") grep -E "$bad" "$c/dmesg.txt" | known_filter > "$c/bad-lines.txt" [ -s "$c/bad-lines.txt" ] && reasons+=("kernel log errors: $(head -3 "$c/bad-lines.txt" | tr '\n' ' ')") + times="suspended ${slept}s, ${took}s in all" if [ ${#reasons[@]} = 0 ]; then - echo "cycle $i: PASS (slept ${slept}s)" | tee -a "$out/summary.txt" + echo "cycle $i: PASS ($times)" | tee -a "$out/summary.txt" else failed=1 - echo "cycle $i: FAIL (slept ${slept}s)" | tee -a "$out/summary.txt" + echo "cycle $i: FAIL ($times)" | tee -a "$out/summary.txt" printf ' - %s\n' "${reasons[@]}" | tee -a "$out/summary.txt" fi + completed=$((completed + 1)) done +# Whatever was logged after the last check +if ! log_take "$out/dmesg-after.txt"; then + failed=1 + echo "kernel log unavailable after the last cycle" | tee -a "$out/summary.txt" +elif grep -E "$bad" "$out/dmesg-after.txt" | known_filter | grep -q .; then + failed=1 + echo "kernel log errors after the last cycle, see dmesg-after.txt" | + tee -a "$out/summary.txt" +fi +rm -f "$out/.dmesg" + +if [ "$completed" != "$cycles" ]; then + failed=1 + echo "only $completed of $cycles cycles ran" | tee -a "$out/summary.txt" +fi + if [ $failed = 0 ]; then echo "RESULT: PASS ($cycles cycles) evidence: $out" | tee -a "$out/summary.txt" else diff --git a/tools/aurora-wake/test-s2idle-rtc-regression.sh b/tools/aurora-wake/test-s2idle-rtc-regression.sh new file mode 100755 index 0000000000000..32a2b196b6b14 --- /dev/null +++ b/tools/aurora-wake/test-s2idle-rtc-regression.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Hardware-free checks of the argument handling of s2idle-rtc-regression.sh: a +# count that would run no cycle must be refused, not reported as a pass, and the +# output must go to a fresh directory of its own. + +script="$(dirname "$0")/s2idle-rtc-regression.sh" +fails=0 + +expect() { + local want=$1 got + shift + bash "$script" "$@" > /dev/null 2>&1 + got=$? + if [ "$got" = "$want" ]; then + echo "ok: $* -> $got" + else + echo "FAIL: $* -> $got, expected $want" + fails=1 + fi +} + +expect 2 -n -c 0 +expect 2 -n -c -1 +expect 2 -n -c abc +expect 2 -n -c '' +expect 2 -n -s 0 +expect 2 -n -s 2 +expect 2 -n -s x +expect 2 -n extra-argument +expect 0 -n -c 3 -s 20 +expect 0 -n +# An invalid filter would hide every kernel log error. +expect 2 -n -i '[' +expect 0 -n -i 'dcp|DCP' + +# Output only goes to a directory the script creates itself, never to an +# existing path that may be a symlink planted by someone else. +existing=$(mktemp -d) +ln -s "$existing" "$existing.link" +expect 2 -n -o "$existing" +expect 2 -n -o "$existing.link" +expect 2 -n -o relative-dir +expect 0 -n -o "$existing/new" +rm -rf "$existing" "$existing.link" + +# Kernel log collection. A message logged right after a read must land in the +# next collection, and a failed read must fail, not pass as an empty log. +tmp=$(mktemp -d) +sed -n '/^# --- kernel log collection/,/^# --- end of kernel log collection/p' \ + "$script" > "$tmp/lib.sh" +fakelog=$tmp/kmsg +late=0 +dmesg() { + cat "$fakelog" || return 1 + late=$((late + 1)) + echo "WARNING: late $late" >> "$fakelog" +} +printf 'first line\nsecond line\n' > "$fakelog" +# shellcheck source=/dev/null +. "$tmp/lib.sh" +out=$tmp log_first='first line' mark=2 +if log_take "$tmp/c1" && log_take "$tmp/c2" && grep -q 'late 1' "$tmp/c2"; then + echo "ok: a message logged right after a read is in the next collection" +else + echo "FAIL: a message logged between collections was skipped" + fails=1 +fi +fakelog=$tmp/missing +if log_take "$tmp/c3" 2> /dev/null; then + echo "FAIL: a failed kernel log read passed" + fails=1 +else + echo "ok: a failed kernel log read fails" +fi +rm -rf "$tmp" + +exit $fails From 63ad4f5ad9c252d1bd8c86cf27118fc56f34736a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:29:34 +1100 Subject: [PATCH 1132/1579] selftests: apple-dcp: Model the DWC3 wake interrupt in the lifecycle tests The controller-lifecycle and dwc3-teardown tests build the actual DWC3 glue teardown into host units. Since the glue arms the core interrupt for wakeup in host mode, it calls dev_pm_set_wake_irq() and dev_pm_clear_wake_irq() and keeps the interrupt number in struct dwc3_apple, none of which the units provided, so neither compiled and none of their cases ran. Model the wake interrupt in both units, and check in the controller-lifecycle test that host teardown drops it again. Fixes: 13e221102941 ("usb: dwc3: apple: Arm the core interrupt for system wakeup in host mode") Assisted-by: LLM Signed-off-by: Ryan Murray --- .../drivers/gpu/apple-dcp/dwc3/mocks.c | 17 ++++++++++++++++- .../drivers/gpu/apple-dcp/teardown_pm.py | 6 +++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c index e54e73146cbad..0373f28f3b165 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -30,7 +30,7 @@ struct dwc3 { struct phy *usb2_generic_phy[3], *usb3_generic_phy[3]; int num_usb2_ports, num_usb3_ports, usb2_phy, usb3_phy; struct reset_controller_dev *reset; int dev, usb_psy, vbus_draw_work; }; -struct dwc3_apple { struct dwc3 dwc; enum dwc3_apple_state state; +struct dwc3_apple { struct dwc3 dwc; enum dwc3_apple_state state; int irq; struct reset_controller_dev *reset; int lock; void *dev; struct phy *usb2_phy, *usb3_phy; bool fixed_hub, core_active, core_exit_pending, reset_pending; @@ -90,6 +90,21 @@ static void dwc3_clk_disable(struct dwc3 *d) { (void)d; clocks_disabled++; } static void dwc3_host_exit(struct dwc3 *d); static void dwc3_gadget_exit(struct dwc3 *d) { (void)d; gadget_exits++; } static void dev_pm_syscore_device(void *d, bool s) { (void)d; syscore = s; } +static bool wake_irq_set; + +static int dev_pm_set_wake_irq(void *d, int irq) +{ + (void)d; + (void)irq; + wake_irq_set = true; + return 0; +} + +static void dev_pm_clear_wake_irq(void *d) +{ + (void)d; + wake_irq_set = 0; +} static void dwc3_enable_susphy(struct dwc3 *d, bool s) { (void)d; (void)s; } static int atcphy_dwc3_reset_assert(struct reset_controller_dev *, unsigned long); static int reset_control_assert(struct reset_controller_dev *r) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py index 1f17261136a9c..6321c337529cb 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py @@ -80,6 +80,8 @@ def unit_for(source): static void *platform_get_drvdata(struct platform_device *d) { return d==&controller?(void *)&apple_context.dwc:(void *)&hcd; } static void dwc3_enable_susphy(struct dwc3 *d,bool v) { (void)d;assert(v);susphy_calls++;observed_controller_write(); } static void device_init_wakeup(struct device *d,bool b) { (void)d;(void)b; } +static bool wake_irq_set; +static void dev_pm_clear_wake_irq(struct device *d) { assert(d==&controller.dev);wake_irq_set=false; } static void platform_device_unregister(struct platform_device *d); static void dwc3_gadget_exit(struct dwc3 *d) { (void)d;assert(0); } static int dwc3_apple_core_stop(struct dwc3_apple *d) { (void)d;return 0; } @@ -140,7 +142,7 @@ def unit_for(source): xhci_context=(struct xhci_hcd){0}; inject_remove=true;removal_started=blocked_attempt=release_remove=false; atomic_store(&early_write,false);atomic_store(&handoff_active,true); - susphy_calls=host_remove_calls=0; + susphy_calls=host_remove_calls=0;wake_irq_set=true; int ret=safe(&p); atomic_store(&handoff_active,false); pthread_mutex_lock(&schedule_mutex);release_remove=true;pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_mutex); @@ -148,6 +150,8 @@ def unit_for(source): fprintf(stderr,"connected teardown target=%d ret=%d early_write=%d susphy=%d host_remove=%d\n",remove_target,ret,atomic_load(&early_write),susphy_calls,host_remove_calls); assert(ret==0&&!atomic_load(&early_write)&&host_remove_calls==1); assert(susphy_calls==(remove_target?0:1));balanced(&p); + /* Host teardown drops the wake interrupt it armed. */ + assert(remove_target||!wake_irq_set); } puts("PASS actual PM wrapper connected to driver-core/DWC3/xHCI removal; synchronous teardown waits until owned transaction ends"); return 0; From 5261b17aa6855626f86c9c35844921beaa279c8f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 16:55:52 +1100 Subject: [PATCH 1133/1579] usb: xhci: Add KUnit static stubs for PORTSC accesses The change bits in PORTSC are cleared by writing 1, and a port posts a Port Status Change Event only for a change made while none of them is set (xHCI 1.2, section 4.19.2). KUnit tests of the event handling fake the registers in memory, where writing 1 to a change bit leaves it set, so they can show neither. Nor can they let the port change between two accesses of the driver, as the controller may. Let a test stand in for xhci_portsc_writel() and xhci_portsc_readl(), so that it can apply each write the way the controller does, and change the port just before a chosen read. Without CONFIG_KUNIT the redirections compile away, and with it, they only look for a stub while a KUnit test is running. The next change uses them. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 89490f0770c0c..ab62d660f1ffd 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -8,6 +8,7 @@ * Some code borrowed from the Linux EHCI driver. */ +#include #include #include #include @@ -43,6 +44,8 @@ MODULE_PARM_DESC(quirks, "Bit flags for quirks to be enabled as default"); void xhci_portsc_writel(struct xhci_port *port, u32 val) { + KUNIT_STATIC_STUB_REDIRECT(xhci_portsc_writel, port, val); + trace_xhci_portsc_writel(port, val); writel(val, &port->port_reg->portsc); } @@ -50,6 +53,8 @@ EXPORT_SYMBOL_GPL(xhci_portsc_writel); u32 xhci_portsc_readl(struct xhci_port *port) { + KUNIT_STATIC_STUB_REDIRECT(xhci_portsc_readl, port); + return readl(&port->port_reg->portsc); } EXPORT_SYMBOL_GPL(xhci_portsc_readl); From e9573a16e76c5f0f19efaae643fa6c395f258d7b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 14:30:31 +1100 Subject: [PATCH 1134/1579] usb: xhci: Wake from suspend-to-idle only for wakeup events A controller that is kept running in suspend-to-idle (XHCI_RUN_IN_S2IDLE) posts a Port Status Change Event for every change on every root port, and with its interrupt armed for wakeup any of them ends suspend-to-idle. On a J700 the SuperSpeed link of the rear port, behind a debug cable, drops to SS.Inactive every few seconds. Once a wake-armed keyboard keeps the controller running, that link change wakes the system within two seconds, although its root hub is suspended with wakeup disabled: usb usb2: bus suspend, wakeup 0 xhci-hcd xhci-hcd.0.auto: wakeup enabled, keep running in suspend-to-idle. PM: Triggering wakeup from IRQ 93 xhci-hcd xhci-hcd.0.auto: Get port status 2-1 read: 0x4002c0, return 0x4002c0 A halted controller signals only some events as a wakeup. The wake bits in PORTSC, which xhci_bus_suspend() sets only when the root hub may wake the system, select connect, disconnect and over-current changes, and a device signals resume only when the USB core enabled its remote wakeup. While the controller runs those bits select nothing, so make the same choice in software instead: when the controller is running in suspend-to-idle, report a hard wakeup event for exactly those port changes, for a transfer completed by a hub that the USB core kept awake to pass on a remote wakeup (the only devices with transfers in flight), for a function wake notification, and for a controller that stopped or reports a Host Controller or Host System Error, so that resume finds it. Such an error wakes the system even if xhci_halt() then times out, as a failing controller may never halt. This is how macsmc-input decides on wakeups from its suspend-to-idle interrupt. Do not resume or poll the root hub for any other port change, so no wakeup event is reported and no hub status URB completes while the system sleeps. Do not leave the change set in PORTSC either: a port posts a Port Status Change Event only for a change made while none of its change bits is set (xHCI 1.2, section 4.19.2), so a later change of that port that should wake the system, such as a connect, would post none. Clear the change bits and hold them in struct xhci_port instead, where GetPortStatus and the root hub status data report them to the hub driver as before, until it clears them. Clearing a change bit is part of the decision, and the event handling clears none before it, not even the link change of a USB 2 port, which it clears at once when the system is awake. The port may change between a read of PORTSC and the write that clears its change bits: - A change latched in between, while other change bits were still set, posts no event, so PORTSC is read again after each write, and judged. - A device that signals resume while PLC is still set, for example from its U3 entry, sets no new PLC, and the write clears the only change bit of that remote wakeup. Only the link state, now Resume, shows it. The event handling then carries on as if PLC were still set, so that it starts the resume timing of a USB 2 port, or moves a SuperSpeed link to U0. When running in suspend-to-idle begins, the root hubs stop being polled until resume, as for a halted controller, and ports whose change bits are still set, for example for a hub driver that froze before it cleared them, are handled as if they had just posted an event. On resume, the root hubs of ports with held changes are resumed, and all root hubs are polled again, as xhci_resume() does. The interface to the platform glue is xhci_run_in_s2idle_begin() and xhci_run_in_s2idle_end(), and the running state moves from the glue into struct xhci_hcd so the event handling can see it. Add KUnit cases for the decision, and for the event path through xhci_irq() on a controller faked in memory, whose PORTSC takes writes and posts events as the controller does. Before this change, the link change resumes and polls the root hub there, neither a remote wakeup nor a controller error reports a wakeup event, and a port whose link change is left set posts no event for a later connect, disconnect, over-current or remote wakeup. Further cases let the device below a USB 2 or SuperSpeed port signal resume while a change is decided on, just before each PORTSC read in turn and just before the write that clears PLC, both on an event and when running in suspend-to-idle begins: the system wakes, and the resume is taken on. This only decides between events: the controller interrupt is still armed for wakeup, so the IRQ core still ends suspend-to-idle before the handler runs. The following changes keep it enabled instead. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/Kconfig | 5 +- drivers/usb/host/xhci-hub.c | 8 +- drivers/usb/host/xhci-plat-test.c | 948 ++++++++++++++++++++++++++++++ drivers/usb/host/xhci-plat.c | 9 +- drivers/usb/host/xhci-plat.h | 1 - drivers/usb/host/xhci-ring.c | 173 +++++- drivers/usb/host/xhci.c | 83 ++- drivers/usb/host/xhci.h | 12 + 8 files changed, 1224 insertions(+), 15 deletions(-) diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig index 93a5145b3e773..521c1ef0ec948 100644 --- a/drivers/usb/host/Kconfig +++ b/drivers/usb/host/Kconfig @@ -76,8 +76,9 @@ config USB_XHCI_PLATFORM_KUNIT_TEST default KUNIT_ALL_TESTS help Test how the generic xHCI platform driver turns device properties - into quirks, and when it keeps the controller running across - suspend-to-idle instead of halting it. The tests need no xHCI + into quirks, when it keeps the controller running across + suspend-to-idle instead of halting it, and which events of a + controller kept running wake the system. The tests need no xHCI hardware. If unsure, say N. diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c index bacd0ddd0d09b..ff107cbf4b979 100644 --- a/drivers/usb/host/xhci-hub.c +++ b/drivers/usb/host/xhci-hub.c @@ -620,7 +620,8 @@ static void xhci_clear_port_change_bit(struct xhci_hcd *xhci, u16 wValue, struct /* Should never happen */ return; } - /* Change bits are all write 1 to clear */ + /* Change bits are all write 1 to clear, also when held in software */ + port->held_changes &= ~status; xhci_portsc_writel(port, portsc | status); portsc = xhci_portsc_readl(port); @@ -1145,6 +1146,9 @@ static u32 xhci_get_port_status(struct usb_hcd *hcd, struct xhci_bus_state *bus_ rhub = xhci_get_rhub(hcd); port = rhub->ports[portnum]; + /* Changes held while running in suspend-to-idle count as set */ + portsc |= port->held_changes; + /* common wPortChange bits */ if (portsc & PORT_CSC) status |= USB_PORT_STAT_C_CONNECTION << 16; @@ -1661,6 +1665,8 @@ int xhci_hub_status_data(struct usb_hcd *hcd, char *buf) break; } trace_xhci_hub_status_data(ports[i], temp); + /* Changes held while running in suspend-to-idle count as set */ + temp |= ports[i]->held_changes & mask; if ((temp & mask) != 0 || (bus_state->port_c_suspend & 1 << i) || diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c index 00646f7ba2f5a..e781f5662564b 100644 --- a/drivers/usb/host/xhci-plat-test.c +++ b/drivers/usb/host/xhci-plat-test.c @@ -4,12 +4,15 @@ */ #include +#include #include #include #include #include #include #include +#include +#include #include "xhci.h" #include "xhci-plat.h" @@ -140,6 +143,88 @@ static void xhci_plat_resume_from_runtime_ok_test(struct kunit *test) } #endif /* CONFIG_PM */ +/* + * Port changes seen while kept running in suspend-to-idle that are not + * wakeups. The first is from a J700 whose SuperSpeed link on the rear port + * kept dropping to SS.Inactive while its root hub was suspended with wakeup + * disabled: PLC, SS.Inactive, powered, no wake bits. + */ +static void xhci_s2idle_port_no_wakeup_test(struct kunit *test) +{ + static const u32 armed_bits = PORT_WKCONN_E | PORT_WKDISC_E | PORT_WKOC_E; + static const u32 changes[] = { + PORT_PLC | XDEV_INACTIVE | PORT_POWER, + PORT_PLC | XDEV_POLLING | PORT_POWER, + PORT_PLC | XDEV_RXDETECT | PORT_POWER, + PORT_PLC | XDEV_COMP_MODE | PORT_POWER, + /* A device that was resumed and is back in U0 or in U3 */ + PORT_PLC | XDEV_U0 | PORT_POWER | PORT_PE | PORT_CONNECT, + PORT_PLC | XDEV_U3 | PORT_POWER | PORT_PE | PORT_CONNECT, + /* Warm and port reset, enable and config error changes */ + PORT_WRC | PORT_RC | XDEV_U0 | PORT_POWER | PORT_PE | PORT_CONNECT, + PORT_PEC | XDEV_DISABLED | PORT_POWER | PORT_CONNECT, + PORT_CEC | XDEV_U0 | PORT_POWER | PORT_PE | PORT_CONNECT, + /* Resume state, but the link change was not what was reported */ + PORT_RC | XDEV_RESUME | PORT_POWER | PORT_PE | PORT_CONNECT, + }; + unsigned int i; + + KUNIT_EXPECT_FALSE(test, xhci_s2idle_port_wakeup(0x4002c0)); + + for (i = 0; i < ARRAY_SIZE(changes); i++) { + /* Not even with the root hub armed for wakeup */ + KUNIT_EXPECT_FALSE_MSG(test, xhci_s2idle_port_wakeup(changes[i]), + "portsc %#x", changes[i]); + KUNIT_EXPECT_FALSE_MSG(test, xhci_s2idle_port_wakeup(changes[i] | armed_bits), + "portsc %#x", changes[i] | armed_bits); + } +} + +/* Remote wakeup signalled by a device below the port wakes the system. */ +static void xhci_s2idle_port_remote_wakeup_test(struct kunit *test) +{ + u32 resume = PORT_PLC | XDEV_RESUME | PORT_POWER | PORT_PE | PORT_CONNECT; + + KUNIT_EXPECT_TRUE(test, xhci_s2idle_port_wakeup(resume)); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_port_wakeup(resume | PORT_WKDISC_E | PORT_WKOC_E)); +} + +/* Connect, disconnect and over-current wake it only if the port was armed for them. */ +static void xhci_s2idle_port_armed_change_test(struct kunit *test) +{ + u32 connect = PORT_CSC | PORT_CONNECT | XDEV_POLLING | PORT_POWER; + u32 disconnect = PORT_CSC | XDEV_RXDETECT | PORT_POWER; + u32 overcurrent = PORT_OCC | PORT_OC | XDEV_DISABLED | PORT_POWER; + + /* As xhci_bus_suspend() arms an empty and a connected port */ + KUNIT_EXPECT_TRUE(test, xhci_s2idle_port_wakeup(connect | PORT_WKCONN_E | PORT_WKOC_E)); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_port_wakeup(disconnect | PORT_WKDISC_E | PORT_WKOC_E)); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_port_wakeup(overcurrent | PORT_WKOC_E)); + + /* Root hub wakeup disabled: nothing armed */ + KUNIT_EXPECT_FALSE(test, xhci_s2idle_port_wakeup(connect)); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_port_wakeup(disconnect)); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_port_wakeup(overcurrent)); + + /* A wake bit only stands for its own change */ + KUNIT_EXPECT_FALSE(test, xhci_s2idle_port_wakeup(overcurrent | PORT_WKCONN_E | + PORT_WKDISC_E)); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_port_wakeup(PORT_PLC | XDEV_INACTIVE | PORT_POWER | + PORT_WKOC_E)); +} + +/* Only a hub kept awake for wakeup completing a transfer wakes the system. */ +static void xhci_s2idle_transfer_wakeup_test(struct kunit *test) +{ + struct usb_device *udev = kunit_kzalloc(test, sizeof(*udev), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, udev); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_transfer_wakeup(NULL)); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_transfer_wakeup(udev)); + udev->awake_in_sleep = 1; + KUNIT_EXPECT_TRUE(test, xhci_s2idle_transfer_wakeup(udev)); +} + static u64 xhci_plat_test_quirks(struct kunit *test, const struct property_entry *props) { struct device *dev; @@ -203,6 +288,10 @@ static struct kunit_case xhci_plat_test_cases[] = { KUNIT_CASE(xhci_plat_no_property_quirks_test), KUNIT_CASE(xhci_plat_run_in_s2idle_property_test), KUNIT_CASE(xhci_plat_all_property_quirks_test), + KUNIT_CASE(xhci_s2idle_port_no_wakeup_test), + KUNIT_CASE(xhci_s2idle_port_remote_wakeup_test), + KUNIT_CASE(xhci_s2idle_port_armed_change_test), + KUNIT_CASE(xhci_s2idle_transfer_wakeup_test), {} }; @@ -211,7 +300,866 @@ static struct kunit_suite xhci_plat_test_suite = { .test_cases = xhci_plat_test_cases, }; +#ifdef CONFIG_PM_SLEEP +/* + * The event handling of a controller with one SuperSpeed root port, faked in + * memory, whose root hub is suspended. PORTSC writes and port changes go + * through xhci_s2idle_test_portsc_writel() and xhci_s2idle_test_port_change(), + * which act as the controller does. + */ +struct xhci_s2idle_test { + struct usb_hcd *hcd; + struct xhci_hcd *xhci; + struct xhci_op_regs *op; + struct xhci_port_regs *port_regs; + __le32 __iomem *portsc; + union xhci_trb *trbs; + unsigned int events; + /* A change that the controller latches just before the next PORTSC write */ + u32 set_before_write; + /* The PORTSC read just before which the device below signals resume, from 1 */ + unsigned int resume_at_read; + unsigned int reads; + /* The generic xHCI root hub methods, as the USB core calls them */ + struct hc_driver driver; + struct device *dev; + char name[32]; +}; + +static void xhci_s2idle_test_rh_resume(struct work_struct *work) +{ +} + +static void xhci_s2idle_test_rh_timer(struct timer_list *timer) +{ +} + +KUNIT_DEFINE_ACTION_WRAPPER(xhci_s2idle_test_rh_timer_delete, timer_delete_sync, + struct timer_list *); + +/* + * A PORTSC write as the controller takes it: the change bits and PED are + * cleared by writing 1, the link state is only written with LWS, port power + * and the wake bits take the value written, and the read-only bits keep + * theirs. + */ +static void xhci_s2idle_test_portsc_writel(struct xhci_port *port, u32 val) +{ + struct xhci_s2idle_test *t = kunit_get_current_test()->priv; + u32 w1c = PORT_PE | PORT_CHANGE_MASK; + u32 rw = PORT_POWER | PORT_WKCONN_E | PORT_WKDISC_E | PORT_WKOC_E; + u32 old = readl(&port->port_reg->portsc) | t->set_before_write; + u32 now = (old & ~(w1c | rw)) | (old & w1c & ~val) | (val & rw); + + if (val & PORT_LINK_STROBE) + now = (now & ~PORT_PLS_MASK) | (val & PORT_PLS_MASK); + t->set_before_write = 0; + writel(now, &port->port_reg->portsc); +} + +/* The controller posts a Port Status Change Event for root port 1 */ +static void xhci_s2idle_test_post_event(struct xhci_s2idle_test *t) +{ + union xhci_trb *trb = &t->trbs[t->events++]; + + trb->generic.field[0] = cpu_to_le32(1 << 24); + trb->generic.field[2] = cpu_to_le32(COMP_SUCCESS << 24); + trb->generic.field[3] = cpu_to_le32(TRB_TYPE(TRB_PORT_STATUS) | TRB_CYCLE); + writel(readl(&t->xhci->op_regs->status) | STS_EINT, &t->xhci->op_regs->status); +} + +/* + * The controller changes the port. It posts an event only for a change made + * while no change bit is set, see xHCI 1.2 section 4.19.2, and returns whether + * it did. + */ +static bool xhci_s2idle_test_port_change(struct xhci_s2idle_test *t, u32 clear, u32 set) +{ + u32 old = readl(t->portsc); + u32 now = (old & ~clear) | set; + + writel(now, t->portsc); + if ((old & PORT_CHANGE_MASK) || !(now & PORT_CHANGE_MASK)) + return false; + xhci_s2idle_test_post_event(t); + return true; +} + +/* The J700 rear port: its SuperSpeed link drops to SS.Inactive, as in 0x4002c0 */ +static bool xhci_s2idle_test_link_inactive(struct xhci_s2idle_test *t) +{ + return xhci_s2idle_test_port_change(t, PORT_PLS_MASK, XDEV_INACTIVE | PORT_PLC); +} + +/* A device is attached, and its link trains to U0 */ +static bool xhci_s2idle_test_connect(struct xhci_s2idle_test *t) +{ + return xhci_s2idle_test_port_change(t, PORT_PLS_MASK, + XDEV_U0 | PORT_CSC | PORT_CONNECT | PORT_PE); +} + +/* The device below the port is detached */ +static bool xhci_s2idle_test_disconnect(struct xhci_s2idle_test *t) +{ + u32 gone = PORT_PLS_MASK | DEV_SPEED_MASK | PORT_PE | PORT_CONNECT; + + return xhci_s2idle_test_port_change(t, gone, XDEV_RXDETECT | PORT_CSC); +} + +/* The port detects an over-current, and is disabled */ +static bool xhci_s2idle_test_overcurrent(struct xhci_s2idle_test *t) +{ + return xhci_s2idle_test_port_change(t, PORT_PLS_MASK, XDEV_DISABLED | PORT_OCC | PORT_OC); +} + +/* The device below the port signals resume */ +static bool xhci_s2idle_test_remote_wakeup(struct xhci_s2idle_test *t) +{ + return xhci_s2idle_test_port_change(t, PORT_PLS_MASK, XDEV_RESUME | PORT_PLC); +} + +/* A PORTSC read, with the device below signalling resume just before the one chosen */ +static u32 xhci_s2idle_test_portsc_readl(struct xhci_port *port) +{ + struct xhci_s2idle_test *t = kunit_get_current_test()->priv; + + if (++t->reads == t->resume_at_read) + xhci_s2idle_test_remote_wakeup(t); + return readl(&port->port_reg->portsc); +} + +static unsigned int xhci_s2idle_test_count; + +/* The port at @portsc, with an event posted for its change bits if any are set */ +static struct xhci_s2idle_test *xhci_s2idle_test_init(struct kunit *test, u32 usbsts, u32 portsc) +{ + static const struct xhci_driver_overrides no_overrides; + struct xhci_s2idle_test *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + struct xhci_interrupter *ir, **irs; + struct xhci_intr_reg *ir_set; + struct xhci_segment *seg; + struct xhci_ring *ring; + struct xhci_port *port, **ports; + union xhci_trb *trbs; + struct xhci_hcd *xhci; + + KUNIT_ASSERT_NOT_NULL(test, t); + t->hcd = kunit_kzalloc(test, sizeof(*t->hcd) + sizeof(*xhci), GFP_KERNEL); + t->op = kunit_kzalloc(test, sizeof(*t->op), GFP_KERNEL); + t->port_regs = kunit_kzalloc(test, sizeof(*t->port_regs), GFP_KERNEL); + ir = kunit_kzalloc(test, sizeof(*ir), GFP_KERNEL); + irs = kunit_kzalloc(test, sizeof(*irs), GFP_KERNEL); + ring = kunit_kzalloc(test, sizeof(*ring), GFP_KERNEL); + seg = kunit_kzalloc(test, sizeof(*seg), GFP_KERNEL); + trbs = kunit_kcalloc(test, TRBS_PER_SEGMENT, sizeof(*trbs), GFP_KERNEL); + port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL); + ports = kunit_kzalloc(test, sizeof(*ports), GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, t->hcd && t->op && t->port_regs && ir && irs && ring && seg && + trbs && port && ports); + ir_set = kunit_kzalloc(test, sizeof(*ir_set), GFP_KERNEL); + t->hcd->self.root_hub = kunit_kzalloc(test, sizeof(*t->hcd->self.root_hub), GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, ir_set && t->hcd->self.root_hub); + ir->ir_set = (__force struct xhci_intr_reg __iomem *)ir_set; + + test->priv = t; + kunit_activate_static_stub(test, xhci_portsc_writel, xhci_s2idle_test_portsc_writel); + + /* + * The controller device may wake the system, as for running in s2idle. + * A test may set up more than one controller, so number them. + */ + snprintf(t->name, sizeof(t->name), "xhci-s2idle-test.%u", xhci_s2idle_test_count++); + t->dev = kunit_device_register(test, t->name); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->dev); + KUNIT_ASSERT_EQ(test, devm_device_init_wakeup(t->dev), 0); + + xhci = hcd_to_xhci(t->hcd); + t->xhci = xhci; + spin_lock_init(&xhci->lock); + xhci->main_hcd = t->hcd; + xhci->shared_hcd = t->hcd; + xhci->op_regs = (__force struct xhci_op_regs __iomem *)t->op; + xhci_init_driver(&t->driver, &no_overrides); + t->hcd->driver = &t->driver; + t->hcd->self.controller = t->dev; + t->hcd->speed = HCD_USB3; + t->hcd->state = HC_STATE_SUSPENDED; + /* A registered and pollable root hub, as for usb_hcd_resume_root_hub() */ + t->hcd->rh_registered = 1; + t->hcd->rh_pollable = 1; + t->hcd->uses_new_polling = 1; + spin_lock_init(&t->hcd->self.root_hub->dev.power.lock); + INIT_WORK(&t->hcd->wakeup_work, xhci_s2idle_test_rh_resume); + timer_setup(&t->hcd->rh_timer, xhci_s2idle_test_rh_timer, 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, xhci_s2idle_test_rh_timer_delete, + &t->hcd->rh_timer), 0); + + seg->trbs = trbs; + seg->dma = 0x10000; + seg->next = seg; + ring->type = TYPE_EVENT; + ring->first_seg = seg; + ring->deq_seg = seg; + ring->dequeue = trbs; + ring->cycle_state = 1; + ir->event_ring = ring; + irs[0] = ir; + xhci->interrupters = irs; + t->trbs = trbs; + + port->port_reg = (__force struct xhci_port_regs __iomem *)t->port_regs; + port->rhub = &xhci->usb3_rhub; + init_completion(&port->rexit_done); + init_completion(&port->u3exit_done); + ports[0] = port; + xhci->hw_ports = port; + xhci->max_ports = 1; + xhci->usb3_rhub.ports = ports; + xhci->usb3_rhub.num_ports = 1; + xhci->usb3_rhub.hcd = t->hcd; + t->portsc = &port->port_reg->portsc; + + writel(CMD_RUN | CMD_EIE, &xhci->op_regs->command); + writel(usbsts, &xhci->op_regs->status); + writel(portsc, t->portsc); + if (portsc & PORT_CHANGE_MASK) + xhci_s2idle_test_post_event(t); + + return t; +} + +/* Make the port a USB 2 one, on the root hub of a USB 2 bus */ +static void xhci_s2idle_test_usb2(struct xhci_s2idle_test *t) +{ + struct xhci_hcd *xhci = t->xhci; + + xhci->usb2_rhub = xhci->usb3_rhub; + memset(&xhci->usb3_rhub, 0, sizeof(xhci->usb3_rhub)); + xhci->hw_ports->rhub = &xhci->usb2_rhub; + t->hcd->speed = HCD_USB2; +} + +static unsigned long xhci_s2idle_test_wakeups(struct xhci_s2idle_test *t) +{ + return t->dev->power.wakeup->event_count; +} + +/* + * Whether the event handling took on a remote wakeup: the resume timing of a + * USB 2 port runs, a SuperSpeed link was moved to U0. + */ +static bool xhci_s2idle_test_resuming(struct xhci_s2idle_test *t) +{ + if (t->hcd->speed < HCD_USB3) + return t->xhci->hw_ports->resume_timestamp && + test_bit(0, &t->xhci->usb2_rhub.bus_state.resuming_ports); + return (readl(t->portsc) & PORT_PLS_MASK) == XDEV_U0; +} + +/* Whether the root hub was resumed or polled for the event */ +static bool xhci_s2idle_test_rh_resumed(struct xhci_s2idle_test *t) +{ + flush_work(&t->hcd->wakeup_work); + return test_bit(HCD_FLAG_WAKEUP_PENDING, &t->hcd->flags); +} + +static bool xhci_s2idle_test_rh_polled(struct xhci_s2idle_test *t) +{ + return test_bit(HCD_FLAG_POLL_RH, &t->hcd->flags); +} + +/* Whether the root hub reports a change of the port to the hub driver */ +static bool xhci_s2idle_test_rh_change(struct xhci_s2idle_test *t) +{ + char buf[2] = {}; + + return t->driver.hub_status_data(t->hcd, buf) > 0 && (buf[0] & BIT(1)); +} + +/* The port status and changes that the hub driver reads */ +static u32 xhci_s2idle_test_port_status(struct kunit *test, struct xhci_s2idle_test *t) +{ + __le32 status; + + KUNIT_ASSERT_EQ(test, t->driver.hub_control(t->hcd, GetPortStatus, 0, 1, (char *)&status, + sizeof(status)), 0); + return le32_to_cpu(status); +} + +static void xhci_s2idle_test_clear_change(struct kunit *test, struct xhci_s2idle_test *t, + u16 feature) +{ + KUNIT_ASSERT_EQ(test, t->driver.hub_control(t->hcd, ClearPortFeature, feature, 1, NULL, 0), + 0); +} + +static irqreturn_t xhci_s2idle_test_irq(struct xhci_s2idle_test *t) +{ + irqreturn_t ret; + + local_irq_disable(); + ret = xhci_irq(t->hcd); + local_irq_enable(); + return ret; +} + +/* A SuperSpeed port in link training, with nothing connected */ +#define XHCI_S2IDLE_TEST_EMPTY (XDEV_POLLING | PORT_POWER) +/* A SuperSpeed port with a device suspended below it */ +#define XHCI_S2IDLE_TEST_SUSPENDED (XDEV_U3 | XDEV_SS | PORT_POWER | PORT_PE | PORT_CONNECT) +/* A USB 2 port with a high-speed device suspended below it */ +#define XHCI_S2IDLE_TEST_USB2_SUSPENDED (XDEV_U3 | XDEV_HS | PORT_POWER | PORT_PE | PORT_CONNECT) + +/* + * The J700 failure: a SuperSpeed link drops to SS.Inactive while running in + * suspend-to-idle. No wakeup, and the root hub is neither resumed nor polled. + * The change is cleared in PORTSC, so that the port posts an event for its + * next one, and held for the hub driver. + */ +static void xhci_s2idle_link_change_held_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_link_inactive(t)); + KUNIT_ASSERT_EQ(test, readl(t->portsc), 0x4002c0U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_irq(t), IRQ_HANDLED); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_resumed(t)); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_polled(t)); + KUNIT_EXPECT_EQ(test, readl(t->portsc) & PORT_CHANGE_MASK, 0U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_port_status(test, t) >> 16, + (u32)USB_PORT_STAT_C_LINK_STATE); + xhci_run_in_s2idle_end(t->xhci); +} + +/* + * On resume the root hub is resumed and polled again, and the hub driver reads + * the held change as if it had stayed in PORTSC, until it clears it. + */ +static void xhci_s2idle_link_change_resume_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY); + u32 status; + + xhci_run_in_s2idle_begin(t->xhci); + xhci_s2idle_test_link_inactive(t); + xhci_s2idle_test_irq(t); + xhci_run_in_s2idle_end(t->xhci); + KUNIT_EXPECT_FALSE(test, t->xhci->running_in_s2idle); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_resumed(t)); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_polled(t)); + /* The root hub is suspended: the poll keeps the change for its status URB */ + KUNIT_EXPECT_TRUE(test, test_bit(HCD_FLAG_POLL_PENDING, &t->hcd->flags)); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_change(t)); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + + status = xhci_s2idle_test_port_status(test, t); + KUNIT_EXPECT_EQ(test, status >> 16, (u32)USB_PORT_STAT_C_LINK_STATE); + KUNIT_EXPECT_EQ(test, status & USB_PORT_STAT_LINK_STATE, (u32)USB_SS_PORT_LS_SS_INACTIVE); + + xhci_s2idle_test_clear_change(test, t, USB_PORT_FEAT_C_PORT_LINK_STATE); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_port_status(test, t) >> 16, 0U); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_change(t)); +} + +/* Outside suspend-to-idle the same change goes to the USB core at once, as before. */ +static void xhci_s2idle_link_change_awake_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, STS_EINT, 0x4002c0); + + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_irq(t), IRQ_HANDLED); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_resumed(t)); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_polled(t)); + KUNIT_EXPECT_TRUE(test, readl(t->portsc) & PORT_PLC); +} + +/* Remote wakeup from a SuperSpeed device below the port wakes the system. */ +static void xhci_s2idle_remote_wakeup_event_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_SUSPENDED); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_remote_wakeup(t)); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_irq(t), IRQ_HANDLED); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_resumed(t)); + /* The event handling took the remote wakeup on and moved the link to U0 */ + KUNIT_EXPECT_EQ(test, readl(t->portsc) & PORT_PLS_MASK, (u32)XDEV_U0); + xhci_run_in_s2idle_end(t->xhci); +} + +/* + * A held change does not keep the port from posting an event for the next + * one: a device attached to a port armed for connect wakes the system. + */ +static void xhci_s2idle_connect_after_held_change_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY | + PORT_WKCONN_E | PORT_WKOC_E); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_link_inactive(t)); + xhci_s2idle_test_irq(t); + KUNIT_ASSERT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_connect(t)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + xhci_run_in_s2idle_end(t->xhci); +} + +/* Nor an over-current on a port armed for it */ +static void xhci_s2idle_overcurrent_after_held_change_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY | + PORT_WKCONN_E | PORT_WKOC_E); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_link_inactive(t)); + xhci_s2idle_test_irq(t); + KUNIT_ASSERT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_overcurrent(t)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + xhci_run_in_s2idle_end(t->xhci); +} + +/* Nor the disconnect of a device whose link failed, on a port armed for it */ +static void xhci_s2idle_disconnect_after_held_change_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_SUSPENDED | + PORT_WKDISC_E | PORT_WKOC_E); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_link_inactive(t)); + xhci_s2idle_test_irq(t); + KUNIT_ASSERT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_disconnect(t)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + xhci_run_in_s2idle_end(t->xhci); +} + +/* + * Nor a remote wakeup: the over-current change of a port whose root hub may + * not wake the system is held, and the suspended device below it then signals + * resume. + */ +static void xhci_s2idle_remote_wakeup_after_held_change_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_SUSPENDED); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_port_change(t, 0, PORT_OCC)); + xhci_s2idle_test_irq(t); + KUNIT_ASSERT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_remote_wakeup(t)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + KUNIT_EXPECT_EQ(test, readl(t->portsc) & PORT_PLS_MASK, (u32)XDEV_U0); + xhci_run_in_s2idle_end(t->xhci); + KUNIT_EXPECT_TRUE(test, (xhci_s2idle_test_port_status(test, t) >> 16) & + USB_PORT_STAT_C_OVERCURRENT); +} + +/* + * A connect that the controller latches while the link change is cleared + * posts no event, as the port still had a change bit set. The event handling + * reads PORTSC again after the write, and wakes the system for it. + */ +static void xhci_s2idle_connect_during_hold_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY | + PORT_WKCONN_E | PORT_WKOC_E); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_link_inactive(t)); + t->set_before_write = PORT_CSC | PORT_CONNECT; + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, t->set_before_write, 0U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + /* Left in PORTSC, for the hub driver after the wakeup */ + KUNIT_EXPECT_TRUE(test, readl(t->portsc) & PORT_CSC); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_resumed(t)); + xhci_run_in_s2idle_end(t->xhci); +} + +/* + * One that does not wake the system is held as well, so the port is left with + * no change bit set, and the hub driver reads both changes on resume. + */ +static void xhci_s2idle_change_during_hold_test(struct kunit *test) +{ + /* The root hub may not wake the system, so the port is not armed */ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY); + u32 status; + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_link_inactive(t)); + t->set_before_write = PORT_CSC | PORT_CONNECT; + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, t->set_before_write, 0U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + KUNIT_EXPECT_EQ(test, readl(t->portsc) & PORT_CHANGE_MASK, 0U); + xhci_run_in_s2idle_end(t->xhci); + + status = xhci_s2idle_test_port_status(test, t); + KUNIT_EXPECT_EQ(test, status >> 16, + (u32)(USB_PORT_STAT_C_CONNECTION | USB_PORT_STAT_C_LINK_STATE)); + KUNIT_EXPECT_TRUE(test, status & USB_PORT_STAT_CONNECTION); +} + +/* + * A change that the port latched before, and that the hub driver had no + * chance to clear, is held once running in suspend-to-idle begins: the port + * then posts an event for the remote wakeup of the device below it. + */ +static void xhci_s2idle_change_before_begin_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_SUSPENDED); + + /* Its event comes in before, and the change is left for the hub driver */ + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_port_change(t, 0, PORT_OCC)); + xhci_s2idle_test_irq(t); + KUNIT_ASSERT_TRUE(test, readl(t->portsc) & PORT_OCC); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + KUNIT_EXPECT_EQ(test, readl(t->portsc) & PORT_CHANGE_MASK, 0U); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_remote_wakeup(t)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + xhci_run_in_s2idle_end(t->xhci); + KUNIT_EXPECT_TRUE(test, (xhci_s2idle_test_port_status(test, t) >> 16) & + USB_PORT_STAT_C_OVERCURRENT); +} + +/* + * One that wakes the system wakes it when running in suspend-to-idle begins, + * and is left in PORTSC. + */ +static void xhci_s2idle_wakeup_before_begin_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY | + PORT_WKCONN_E | PORT_WKOC_E); + + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_connect(t)); + xhci_s2idle_test_irq(t); + KUNIT_ASSERT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + KUNIT_EXPECT_TRUE(test, readl(t->portsc) & PORT_CSC); + xhci_run_in_s2idle_end(t->xhci); +} + +/* A USB 2 remote wakeup wakes the system and starts the resume timing. */ +static void xhci_s2idle_usb2_remote_wakeup_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, + XHCI_S2IDLE_TEST_USB2_SUSPENDED); + + xhci_s2idle_test_usb2(t); + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_remote_wakeup(t)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_resuming(t)); + xhci_run_in_s2idle_end(t->xhci); +} + +/* + * A remote wakeup that starts while PLC is set, as after U3 entry, sets no new + * PLC and so posts no event, and clearing the old PLC removes its only change + * bit. Its link state still shows it: the system wakes, and the event handling + * takes the resume on. + */ +static void xhci_s2idle_resume_during_plc_clear(struct kunit *test, u32 portsc, bool usb2) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, portsc); + + if (usb2) + xhci_s2idle_test_usb2(t); + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_port_change(t, 0, PORT_PLC)); + t->set_before_write = PORT_PLC | XDEV_RESUME; + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, t->set_before_write, 0U); + KUNIT_EXPECT_EQ(test, t->events, 1U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_resuming(t)); + xhci_run_in_s2idle_end(t->xhci); +} + +static void xhci_s2idle_resume_during_plc_clear_test(struct kunit *test) +{ + xhci_s2idle_resume_during_plc_clear(test, XHCI_S2IDLE_TEST_SUSPENDED, false); +} + +static void xhci_s2idle_usb2_resume_during_plc_clear_test(struct kunit *test) +{ + xhci_s2idle_resume_during_plc_clear(test, XHCI_S2IDLE_TEST_USB2_SUSPENDED, true); +} + +/* So it does for a PLC still set when running in suspend-to-idle begins. */ +static void xhci_s2idle_resume_during_begin_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_SUSPENDED); + + writel(readl(t->portsc) | PORT_PLC, t->portsc); + t->set_before_write = PORT_PLC | XDEV_RESUME; + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_EXPECT_EQ(test, t->set_before_write, 0U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_resuming(t)); + xhci_run_in_s2idle_end(t->xhci); +} + +/* + * A remote wakeup at any point of the decision on an over-current change that + * does not wake the system: the device signals resume just before each PORTSC + * read in turn. While the change is set, its PLC posts no event of its own. The + * system wakes all the same, and the resume is taken on. When the decision is + * over before the chosen read, the change is held and nothing wakes. + */ +static void xhci_s2idle_resume_during_hold(struct kunit *test, u32 portsc, bool usb2, + bool latched) +{ + unsigned int read; + + for (read = 1; ; read++) { + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, portsc); + + if (usb2) + xhci_s2idle_test_usb2(t); + kunit_activate_static_stub(test, xhci_portsc_readl, + xhci_s2idle_test_portsc_readl); + if (latched) { + /* Left for the hub driver before running in suspend-to-idle */ + writel(readl(t->portsc) | PORT_OCC, t->portsc); + t->resume_at_read = read; + xhci_run_in_s2idle_begin(t->xhci); + } else { + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_port_change(t, 0, PORT_OCC)); + t->reads = 0; + t->resume_at_read = read; + } + xhci_s2idle_test_irq(t); + + if (t->reads < read) { + KUNIT_EXPECT_GE(test, read, 3U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + KUNIT_EXPECT_EQ(test, t->xhci->hw_ports->held_changes, (u32)PORT_OCC); + xhci_run_in_s2idle_end(t->xhci); + break; + } + KUNIT_EXPECT_GE_MSG(test, xhci_s2idle_test_wakeups(t), 1UL, + "resume before read %u", read); + KUNIT_EXPECT_TRUE_MSG(test, xhci_s2idle_test_resuming(t), + "resume before read %u", read); + xhci_run_in_s2idle_end(t->xhci); + } +} + +static void xhci_s2idle_resume_during_hold_test(struct kunit *test) +{ + xhci_s2idle_resume_during_hold(test, XHCI_S2IDLE_TEST_SUSPENDED, false, false); +} + +static void xhci_s2idle_usb2_resume_during_hold_test(struct kunit *test) +{ + xhci_s2idle_resume_during_hold(test, XHCI_S2IDLE_TEST_USB2_SUSPENDED, true, false); +} + +static void xhci_s2idle_resume_during_begin_hold_test(struct kunit *test) +{ + xhci_s2idle_resume_during_hold(test, XHCI_S2IDLE_TEST_SUSPENDED, false, true); +} + +static void xhci_s2idle_usb2_resume_during_begin_hold_test(struct kunit *test) +{ + xhci_s2idle_resume_during_hold(test, XHCI_S2IDLE_TEST_USB2_SUSPENDED, true, true); +} + +/* + * Every change the hub driver reads is held across the sleep, and each is + * cleared on its own by ClearPortFeature after resume. None comes back in the + * next sleep. + */ +static void xhci_s2idle_held_clear_test(struct kunit *test) +{ + static const struct { + u32 hw, usb; + u16 feature; + } changes[] = { + { PORT_CSC, USB_PORT_STAT_C_CONNECTION, USB_PORT_FEAT_C_CONNECTION }, + { PORT_OCC, USB_PORT_STAT_C_OVERCURRENT, USB_PORT_FEAT_C_OVER_CURRENT }, + { PORT_RC, USB_PORT_STAT_C_RESET, USB_PORT_FEAT_C_RESET }, + { PORT_WRC, USB_PORT_STAT_C_BH_RESET, USB_PORT_FEAT_C_BH_PORT_RESET }, + { PORT_PLC, USB_PORT_STAT_C_LINK_STATE, USB_PORT_FEAT_C_PORT_LINK_STATE }, + { PORT_CEC, USB_PORT_STAT_C_CONFIG_ERROR, USB_PORT_FEAT_C_PORT_CONFIG_ERROR }, + }; + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_SUSPENDED); + u32 hw = 0, usb = 0; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(changes); i++) { + hw |= changes[i].hw; + usb |= changes[i].usb; + } + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_port_change(t, 0, hw)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + KUNIT_EXPECT_EQ(test, readl(t->portsc) & PORT_CHANGE_MASK, 0U); + xhci_run_in_s2idle_end(t->xhci); + + for (i = 0; i < ARRAY_SIZE(changes); i++) { + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_port_status(test, t) >> 16, usb); + xhci_s2idle_test_clear_change(test, t, changes[i].feature); + usb &= ~changes[i].usb; + } + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_port_status(test, t) >> 16, 0U); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_change(t)); + + xhci_run_in_s2idle_begin(t->xhci); + xhci_run_in_s2idle_end(t->xhci); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_port_status(test, t) >> 16, 0U); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_change(t)); +} + +/* + * A USB 2 link change is cleared but not held: the hub driver never reads one, + * it reads the end of a resume from port_c_suspend instead. + */ +static void xhci_s2idle_usb2_plc_not_held_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, + XHCI_S2IDLE_TEST_USB2_SUSPENDED); + + xhci_s2idle_test_usb2(t); + writel(readl(t->portsc) | PORT_PLC | PORT_OCC, t->portsc); + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 0UL); + KUNIT_EXPECT_EQ(test, t->xhci->hw_ports->held_changes, (u32)PORT_OCC); + KUNIT_EXPECT_EQ(test, readl(t->portsc) & PORT_CHANGE_MASK, 0U); + xhci_run_in_s2idle_end(t->xhci); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_port_status(test, t) >> 16, + (u32)USB_PORT_STAT_C_OVERCURRENT); + xhci_s2idle_test_clear_change(test, t, USB_PORT_FEAT_C_OVER_CURRENT); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_change(t)); +} + +/* + * Root hub polling stops until resume, as for a halted controller: the event + * handling decides on every change, and a poll while the system sleeps would + * only pass held changes to the hub driver of a root hub kept awake. + */ +static void xhci_s2idle_rh_polling_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, 0, XHCI_S2IDLE_TEST_EMPTY); + + /* Polling, as for a change shortly before the system went to sleep */ + set_bit(HCD_FLAG_POLL_RH, &t->hcd->flags); + mod_timer(&t->hcd->rh_timer, jiffies + HZ); + + xhci_run_in_s2idle_begin(t->xhci); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_polled(t)); + KUNIT_EXPECT_FALSE(test, timer_pending(&t->hcd->rh_timer)); + KUNIT_ASSERT_TRUE(test, xhci_s2idle_test_link_inactive(t)); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_FALSE(test, xhci_s2idle_test_rh_polled(t)); + KUNIT_EXPECT_FALSE(test, timer_pending(&t->hcd->rh_timer)); + + xhci_run_in_s2idle_end(t->xhci); + KUNIT_EXPECT_TRUE(test, xhci_s2idle_test_rh_polled(t)); + KUNIT_EXPECT_TRUE(test, timer_pending(&t->hcd->rh_timer)); +} + +/* A controller error halts it and wakes the system, so that resume deals with it. */ +static void xhci_s2idle_controller_error_test(struct kunit *test) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, STS_EINT | STS_HCE | STS_HALT, + 0); + + xhci_run_in_s2idle_begin(t->xhci); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_TRUE(test, t->xhci->xhc_state & XHCI_STATE_HALTED); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + xhci_run_in_s2idle_end(t->xhci); +} + +/* + * So it does if the failing controller never halts, with its interrupts + * masked all the same, for a Host Controller and a Host System Error. + */ +static void xhci_s2idle_error_no_halt(struct kunit *test, u32 error) +{ + struct xhci_s2idle_test *t = xhci_s2idle_test_init(test, STS_EINT | error, 0); + + xhci_run_in_s2idle_begin(t->xhci); + xhci_s2idle_test_irq(t); + KUNIT_EXPECT_FALSE(test, t->xhci->xhc_state & XHCI_STATE_HALTED); + KUNIT_EXPECT_EQ(test, readl(&t->xhci->op_regs->command) & (CMD_RUN | XHCI_IRQS), 0U); + KUNIT_EXPECT_EQ(test, xhci_s2idle_test_wakeups(t), 1UL); + xhci_run_in_s2idle_end(t->xhci); +} + +static void xhci_s2idle_controller_error_no_halt_test(struct kunit *test) +{ + xhci_s2idle_error_no_halt(test, STS_HCE); +} + +static void xhci_s2idle_system_error_no_halt_test(struct kunit *test) +{ + xhci_s2idle_error_no_halt(test, STS_FATAL); +} + +static struct kunit_case xhci_s2idle_test_cases[] = { + KUNIT_CASE(xhci_s2idle_link_change_held_test), + KUNIT_CASE(xhci_s2idle_link_change_resume_test), + KUNIT_CASE(xhci_s2idle_link_change_awake_test), + KUNIT_CASE(xhci_s2idle_remote_wakeup_event_test), + KUNIT_CASE(xhci_s2idle_connect_after_held_change_test), + KUNIT_CASE(xhci_s2idle_overcurrent_after_held_change_test), + KUNIT_CASE(xhci_s2idle_disconnect_after_held_change_test), + KUNIT_CASE(xhci_s2idle_remote_wakeup_after_held_change_test), + KUNIT_CASE(xhci_s2idle_connect_during_hold_test), + KUNIT_CASE(xhci_s2idle_change_during_hold_test), + KUNIT_CASE(xhci_s2idle_change_before_begin_test), + KUNIT_CASE(xhci_s2idle_wakeup_before_begin_test), + KUNIT_CASE(xhci_s2idle_usb2_remote_wakeup_test), + KUNIT_CASE(xhci_s2idle_resume_during_plc_clear_test), + KUNIT_CASE(xhci_s2idle_usb2_resume_during_plc_clear_test), + KUNIT_CASE(xhci_s2idle_resume_during_begin_test), + KUNIT_CASE(xhci_s2idle_resume_during_hold_test), + KUNIT_CASE(xhci_s2idle_usb2_resume_during_hold_test), + KUNIT_CASE(xhci_s2idle_resume_during_begin_hold_test), + KUNIT_CASE(xhci_s2idle_usb2_resume_during_begin_hold_test), + KUNIT_CASE(xhci_s2idle_held_clear_test), + KUNIT_CASE(xhci_s2idle_usb2_plc_not_held_test), + KUNIT_CASE(xhci_s2idle_rh_polling_test), + KUNIT_CASE(xhci_s2idle_controller_error_test), + KUNIT_CASE(xhci_s2idle_controller_error_no_halt_test), + KUNIT_CASE(xhci_s2idle_system_error_no_halt_test), + {} +}; + +static struct kunit_suite xhci_s2idle_test_suite = { + .name = "xhci_s2idle", + .test_cases = xhci_s2idle_test_cases, +}; + +kunit_test_suites(&xhci_plat_test_suite, &xhci_s2idle_test_suite); +#else kunit_test_suite(xhci_plat_test_suite); +#endif /* CONFIG_PM_SLEEP */ MODULE_DESCRIPTION("KUnit tests for the generic xHCI platform driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index aec29654680e3..d38d96ec37a56 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -580,7 +580,7 @@ static int xhci_plat_suspend(struct device *dev) if (ret) return ret; - priv->running_in_s2idle = 1; + xhci_run_in_s2idle_begin(hcd_to_xhci(hcd)); dev_dbg(dev, "wakeup enabled, keep running in suspend-to-idle.\n"); return 0; } @@ -643,6 +643,7 @@ static int xhci_plat_resume(struct device *dev) { struct usb_hcd *hcd = dev_get_drvdata(dev); struct xhci_plat_priv *priv = hcd_to_xhci_priv(hcd); + struct xhci_hcd *xhci = hcd_to_xhci(hcd); if (priv->sideband_at_suspend) { priv->sideband_at_suspend = 0; @@ -650,12 +651,11 @@ static int xhci_plat_resume(struct device *dev) return 0; } - if (priv->running_in_s2idle) { - struct xhci_hcd *xhci = hcd_to_xhci(hcd); + if (xhci->running_in_s2idle) { u32 status = readl(&xhci->op_regs->status); - priv->running_in_s2idle = 0; if (!xhci_plat_stopped(status)) { + xhci_run_in_s2idle_end(xhci); dev_dbg(dev, "kept running in suspend-to-idle, skip resume.\n"); return 0; } @@ -675,6 +675,7 @@ static int xhci_plat_resume(struct device *dev) xhci_quiesce_dying(xhci); xhci_hc_died(xhci); spin_unlock_irq(&xhci->lock); + xhci_run_in_s2idle_end(xhci); return 0; } diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h index da55117cbf3f3..d002aac38ec20 100644 --- a/drivers/usb/host/xhci-plat.h +++ b/drivers/usb/host/xhci-plat.h @@ -17,7 +17,6 @@ struct xhci_plat_priv { unsigned long long quirks; bool power_lost; unsigned sideband_at_suspend:1; - unsigned running_in_s2idle:1; void (*plat_start)(struct usb_hcd *); int (*init_quirk)(struct usb_hcd *); int (*suspend_quirk)(struct usb_hcd *); diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index 2ec1323998889..e2ff3e14b139a 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -52,6 +52,7 @@ * endpoint rings; it generates events on the event ring for these. */ +#include #include #include #include @@ -1941,6 +1942,113 @@ static void handle_vendor_event(struct xhci_hcd *xhci, handle_cmd_completion(xhci, &event->event_cmd); } +/* + * A controller kept running in suspend-to-idle interrupts for every event, see + * xhci_run_in_s2idle_begin(). The event handling decides which of them wake + * the system, as the platform has no other wake logic that would. + */ +void xhci_s2idle_wakeup(struct xhci_hcd *xhci) +{ + pm_wakeup_hard_event(xhci_to_hcd(xhci)->self.controller); +} + +/* + * Whether a port change is one that a halted controller would wake the system + * on: resume signalling, which a device only sends once the USB core enabled + * its remote wakeup, or a connect, disconnect or over-current change that + * xhci_bus_suspend() armed the port for, which it does only when the root hub + * may wake the system. Any other change, such as a SuperSpeed link going to + * SS.Inactive, does not wake it. + */ +VISIBLE_IF_KUNIT bool xhci_s2idle_port_wakeup(u32 portsc) +{ + if ((portsc & PORT_PLC) && (portsc & PORT_PLS_MASK) == XDEV_RESUME) + return true; + + if ((portsc & PORT_CSC) && (portsc & (PORT_WKCONN_E | PORT_WKDISC_E))) + return true; + + return (portsc & PORT_OCC) && (portsc & PORT_WKOC_E); +} +EXPORT_SYMBOL_IF_KUNIT(xhci_s2idle_port_wakeup); + +/** + * xhci_s2idle_port_change - Decide on a port change in suspend-to-idle + * @xhci: The controller, running in suspend-to-idle, with its lock held + * @port: The port + * @portsc: PORTSC as the event handling read it, updated as it is read again + * + * A port posts a Port Status Change Event only for a change made while none of + * its change bits is set, see xHCI 1.2 section 4.19.2. Left set in PORTSC until + * the hub driver clears it after resume, a change that does not wake the system + * would keep the port from posting one for a later change that does, such as a + * connect. Clear the change bits and hold them in software instead, where + * xhci_get_port_status() and xhci_hub_status_data() report them as before. + * + * Clearing them is part of the decision. A change that the port latched in the + * meantime, while other change bits were still set, posted no event, so PORTSC + * is read and judged again after each write. And a remote wakeup that started + * while PLC was set sets no new PLC, so clearing PLC removes its only change + * bit: if the link is then in Resume, PLC is put back into @portsc for the + * event handling, as if it had stayed set. + * + * Return: true if a change of the port wakes the system, or if the port keeps + * changing. The event handling then carries on with @portsc. + */ +static bool xhci_s2idle_port_change(struct xhci_hcd *xhci, struct xhci_port *port, + u32 *portsc) +{ + u32 reported = PORT_CHANGE_MASK; + int retries = 4; + + /* The hub driver sees no USB 2 link state change, the event handling deals with it */ + if (port->rhub == &xhci->usb2_rhub) + reported &= ~PORT_PLC; + + while (!xhci_s2idle_port_wakeup(*portsc)) { + u32 changes = *portsc & PORT_CHANGE_MASK; + + if (!changes) + return false; + if (retries-- == 0) + return true; + + xhci_dbg(xhci, "port %d-%d: no s2idle wakeup, change left for resume\n", + port->rhub->hcd->self.busnum, port->hcd_portnum + 1); + port->held_changes |= changes & reported; + xhci_portsc_writel(port, xhci_port_state_to_neutral(*portsc) | changes); + *portsc = xhci_portsc_readl(port); + + if ((changes & PORT_PLC) && (*portsc & PORT_PLS_MASK) == XDEV_RESUME) + *portsc |= PORT_PLC; + } + + return true; +} + +/* + * Only hubs that the USB core kept awake to pass on a remote wakeup from below + * have transfers in flight, see usb_suspend_both(): a completed one reports + * a change on one of their ports. + */ +VISIBLE_IF_KUNIT bool xhci_s2idle_transfer_wakeup(const struct usb_device *udev) +{ + return udev && udev->awake_in_sleep; +} +EXPORT_SYMBOL_IF_KUNIT(xhci_s2idle_transfer_wakeup); + +static void xhci_s2idle_transfer_event(struct xhci_hcd *xhci, + struct xhci_transfer_event *event) +{ + unsigned int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags)); + struct xhci_virt_device *vdev = xhci->devs[slot_id]; + + if (vdev && xhci_s2idle_transfer_wakeup(vdev->udev)) { + xhci_dbg(xhci, "slot %u transfer: s2idle wakeup\n", slot_id); + xhci_s2idle_wakeup(xhci); + } +} + static void handle_device_notification(struct xhci_hcd *xhci, union xhci_trb *event) { @@ -1956,6 +2064,8 @@ static void handle_device_notification(struct xhci_hcd *xhci, xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n", slot_id); + if (xhci->running_in_s2idle) + xhci_s2idle_wakeup(xhci); udev = xhci->devs[slot_id]->udev; if (udev && udev->parent) usb_wakeup_notification(udev->parent, udev->portnum); @@ -2042,11 +2152,6 @@ static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) trace_xhci_handle_port_status(port, portsc); - if (hcd->state == HC_STATE_SUSPENDED) { - xhci_dbg(xhci, "resume root hub\n"); - usb_hcd_resume_root_hub(hcd); - } - if (vdev && (portsc & PORT_PLS_MASK) == XDEV_INACTIVE) { if (!(portsc & PORT_RESET)) vdev->flags |= VDEV_PORT_ERROR; @@ -2054,6 +2159,26 @@ static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) vdev->flags &= ~VDEV_PORT_ERROR; } + /* + * Running in suspend-to-idle, wake the system only as a halted + * controller would. Hold any other change for the USB core until + * resume, and do not resume or poll the root hub for it now. Nothing + * below may clear a change bit before this decision, see + * xhci_s2idle_port_change(). + */ + if (xhci->running_in_s2idle) { + if (!xhci_s2idle_port_change(xhci, port, &portsc)) + return; + xhci_dbg(xhci, "port %d-%d: s2idle wakeup\n", + hcd->self.busnum, hcd_portnum + 1); + xhci_s2idle_wakeup(xhci); + } + + if (hcd->state == HC_STATE_SUSPENDED) { + xhci_dbg(xhci, "resume root hub\n"); + usb_hcd_resume_root_hub(hcd); + } + if ((portsc & PORT_PLC) && (portsc & PORT_PLS_MASK) == XDEV_RESUME) { xhci_dbg(xhci, "port resume event for port %d\n", port_id); @@ -2164,6 +2289,33 @@ static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) spin_lock(&xhci->lock); } +/** + * xhci_s2idle_latched_changes - Decide on the changes latched before suspend-to-idle + * @xhci: The controller, just set running in suspend-to-idle, with its lock held + * + * A port whose change bits are still set, such as one whose changes the hub + * driver had no chance to clear before it froze, posts no event for its next + * change. Handle each such port as if it had just posted an event, so that its + * changes are held, or wake the system, as for any other event. + */ +void xhci_s2idle_latched_changes(struct xhci_hcd *xhci) +{ + union xhci_trb event = {}; + unsigned int i; + + event.generic.field[2] = cpu_to_le32(COMP_SUCCESS << 24); + for (i = 0; i < xhci->max_ports; i++) { + struct xhci_port *port = &xhci->hw_ports[i]; + + if (!port->rhub || !port->rhub->hcd || + port->hcd_portnum == DUPLICATE_ENTRY || + !(xhci_portsc_readl(port) & PORT_CHANGE_MASK)) + continue; + event.generic.field[0] = cpu_to_le32((i + 1) << 24); + handle_port_status(xhci, &event); + } +} + static void xhci_clear_hub_tt_buffer(struct xhci_hcd *xhci, struct xhci_td *td, struct xhci_virt_ep *ep) { @@ -3009,6 +3161,8 @@ static int xhci_handle_event_trb(struct xhci_hcd *xhci, struct xhci_interrupter handle_port_status(xhci, event); break; case TRB_TRANSFER: + if (xhci->running_in_s2idle) + xhci_s2idle_transfer_event(xhci, &event->trans_event); handle_tx_event(xhci, ir, &event->trans_event); break; case TRB_DEV_NOTE: @@ -3216,6 +3370,15 @@ irqreturn_t xhci_irq(struct usb_hcd *hcd) /* This is the handler of the primary interrupter */ xhci_handle_events(xhci, xhci->interrupters[0], false); out: + /* + * A controller that stopped or failed in suspend-to-idle is dealt with + * on resume. A Host Controller or Host System Error wakes the system + * even if xhci_halt() timed out: a failing controller may never halt. + */ + if (xhci->running_in_s2idle && + ((status & (STS_HCE | STS_FATAL)) || + (xhci->xhc_state & (XHCI_STATE_DYING | XHCI_STATE_HALTED)))) + xhci_s2idle_wakeup(xhci); spin_unlock(&xhci->lock); return ret; diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index ab62d660f1ffd..fa5dfa171f638 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -951,7 +951,7 @@ static bool xhci_pending_portevent(struct xhci_hcd *xhci) ports = xhci->usb2_rhub.ports; while (port_index--) { portsc = xhci_portsc_readl(ports[port_index]); - if (portsc & PORT_CHANGE_MASK || + if (portsc & PORT_CHANGE_MASK || ports[port_index]->held_changes || (portsc & PORT_PLS_MASK) == XDEV_RESUME) return true; } @@ -959,7 +959,7 @@ static bool xhci_pending_portevent(struct xhci_hcd *xhci) ports = xhci->usb3_rhub.ports; while (port_index--) { portsc = xhci_portsc_readl(ports[port_index]); - if (portsc & (PORT_CHANGE_MASK | PORT_CAS) || + if (portsc & (PORT_CHANGE_MASK | PORT_CAS) || ports[port_index]->held_changes || (portsc & PORT_PLS_MASK) == XDEV_RESUME) return true; } @@ -1308,6 +1308,85 @@ int xhci_resume(struct xhci_hcd *xhci, bool power_lost, bool is_auto_resume) return retval; } EXPORT_SYMBOL_GPL(xhci_resume); + +/** + * xhci_run_in_s2idle_begin - Leave the controller running in suspend-to-idle + * @xhci: The controller, with its root hubs suspended or kept awake for wakeup + * + * For platforms without a wake signal for a halted controller. A running + * controller interrupts for every event, not just for the ones that a halted + * controller would signal as a wakeup, so with the interrupt kept enabled the + * event handling makes that decision instead: it wakes the system for those + * events and holds any other port change for the USB core until resume, see + * xhci_s2idle_port_wakeup() and xhci_s2idle_port_change(). + * + * The root hubs are not polled until resume, as for a halted controller in + * xhci_suspend(). Changes that the ports latched before, and that the hub + * driver has not cleared yet, are then decided on as if the ports had just + * posted an event for them, so that each port posts an event for its next + * change. + */ +void xhci_run_in_s2idle_begin(struct xhci_hcd *xhci) +{ + struct usb_hcd *hcds[] = { xhci->usb2_rhub.hcd, xhci->usb3_rhub.hcd }; + unsigned int i; + + spin_lock_irq(&xhci->lock); + xhci->running_in_s2idle = 1; + spin_unlock_irq(&xhci->lock); + + for (i = 0; i < ARRAY_SIZE(hcds); i++) { + if (!hcds[i]) + continue; + clear_bit(HCD_FLAG_POLL_RH, &hcds[i]->flags); + timer_delete_sync(&hcds[i]->rh_timer); + } + + spin_lock_irq(&xhci->lock); + xhci_s2idle_latched_changes(xhci); + spin_unlock_irq(&xhci->lock); +} +EXPORT_SYMBOL_GPL(xhci_run_in_s2idle_begin); + +/** + * xhci_run_in_s2idle_end - Hand the controller back after suspend-to-idle + * @xhci: The controller, still running + * + * Passes the port changes that were held during suspend-to-idle up to the USB + * core and polls the root hubs again, as xhci_resume() does for a halted + * controller, unless the controller has been reported dead in the meantime. + */ +void xhci_run_in_s2idle_end(struct xhci_hcd *xhci) +{ + struct xhci_hub *rhubs[] = { &xhci->usb2_rhub, &xhci->usb3_rhub }; + bool held[ARRAY_SIZE(rhubs)] = {}; + unsigned int i, j; + bool dying; + + spin_lock_irq(&xhci->lock); + xhci->running_in_s2idle = 0; + dying = xhci->xhc_state & XHCI_STATE_DYING; + for (i = 0; i < ARRAY_SIZE(rhubs); i++) { + for (j = 0; j < rhubs[i]->num_ports; j++) + held[i] |= !!rhubs[i]->ports[j]->held_changes; + } + spin_unlock_irq(&xhci->lock); + + if (dying) + return; + + for (i = 0; i < ARRAY_SIZE(rhubs); i++) { + struct usb_hcd *hcd = rhubs[i]->hcd; + + if (!hcd) + continue; + if (held[i] && hcd->state == HC_STATE_SUSPENDED) + usb_hcd_resume_root_hub(hcd); + set_bit(HCD_FLAG_POLL_RH, &hcd->flags); + usb_hcd_poll_rh_status(hcd); + } +} +EXPORT_SYMBOL_GPL(xhci_run_in_s2idle_end); #endif /* CONFIG_PM */ /*-------------------------------------------------------------------------*/ diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index 0f988654e2ab1..38654b4fed110 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1481,6 +1481,8 @@ struct xhci_port { unsigned int lpm_incapable:1; unsigned long resume_timestamp; bool rexit_active; + /* Change bits held for the hub driver, see xhci_s2idle_port_change() */ + u32 held_changes; /* Slot ID is the index of the device directly connected to the port */ int slot_id; struct completion rexit_done; @@ -1660,6 +1662,8 @@ struct xhci_hcd { unsigned broken_suspend:1; /* Indicates that omitting hcd is supported if root hub has no ports */ unsigned allow_single_roothub:1; + /* Kept running in suspend-to-idle, see xhci_run_in_s2idle_begin() */ + unsigned running_in_s2idle:1; /* cached extended protocol port capabilities */ struct xhci_port_cap *port_caps; unsigned int num_port_caps; @@ -1900,6 +1904,8 @@ int xhci_ext_cap_init(struct xhci_hcd *xhci); int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup); int xhci_resume(struct xhci_hcd *xhci, bool power_lost, bool is_auto_resume); +void xhci_run_in_s2idle_begin(struct xhci_hcd *xhci); +void xhci_run_in_s2idle_end(struct xhci_hcd *xhci); irqreturn_t xhci_irq(struct usb_hcd *hcd); irqreturn_t xhci_msi_irq(int irq, void *hcd); @@ -1984,6 +1990,12 @@ enum usb_link_tunnel_mode xhci_port_is_tunneled(struct xhci_hcd *xhci, struct xhci_port *port); void xhci_hc_died(struct xhci_hcd *xhci); void xhci_quiesce_dying(struct xhci_hcd *xhci); +void xhci_s2idle_wakeup(struct xhci_hcd *xhci); +void xhci_s2idle_latched_changes(struct xhci_hcd *xhci); +#if IS_ENABLED(CONFIG_KUNIT) +bool xhci_s2idle_port_wakeup(u32 portsc); +bool xhci_s2idle_transfer_wakeup(const struct usb_device *udev); +#endif #ifdef CONFIG_PM int xhci_bus_suspend(struct usb_hcd *hcd); From acf1861397d812b05b671a2c75fb92f6e615ac28 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 14:31:42 +1100 Subject: [PATCH 1135/1579] usb: xhci: plat: Keep the interrupt enabled when running in suspend-to-idle The event handling of a controller kept running in suspend-to-idle now decides which events wake the system, but its handler never runs then. The glue arms the interrupt for wakeup, and the IRQ core disables an armed interrupt on its first trigger and ends suspend-to-idle without calling the handler. On the J700 that is still IRQ 93 waking the system for a link change on a root port with wakeup disabled. Request the interrupt with IRQF_NO_SUSPEND when the platform asks for XHCI_RUN_IN_S2IDLE, so that suspend_device_irqs() leaves it enabled and xhci_irq() runs while the system sleeps. As the suspend and interrupts documentation describes, the handler of an IRQF_NO_SUSPEND interrupt runs in suspend-to-idle and the system only wakes if it reports a wakeup, which the event handling now does with pm_wakeup_hard_event(). The Apple mailbox interrupts that carry the SMC lid and power button events to macsmc-input work the same way. Interrupts that are no wakeup now cost a pass through the handler, after which suspend-to-idle carries on. A halted controller is unaffected: it writes no events, so it raises no interrupt, and with its hardware marked inaccessible usb_hcd_irq() would not touch it. Only the primary HCD requests the interrupt, and the DWC3 core requests nothing on it in host mode, so it has no other user that would also be kept enabled. IRQF_SHARED is kept as before. Kept enabled, the interrupt is no longer armed by the wake interrupt that the glue sets up for it; the next change removes that. The flags are fixed when the interrupt is requested, so drop XHCI_RUN_IN_S2IDLE again if it only arrives later, from the xhci_hcd quirks module parameter: running without the interrupt could never wake the system. Add a KUnit case for the flags. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci-plat-test.c | 11 ++++++++++ drivers/usb/host/xhci-plat.c | 35 ++++++++++++++++++++++++------- drivers/usb/host/xhci-plat.h | 1 + 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c index e781f5662564b..b07b3f985ea26 100644 --- a/drivers/usb/host/xhci-plat-test.c +++ b/drivers/usb/host/xhci-plat-test.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -49,6 +50,15 @@ static void xhci_plat_keep_running_other_quirks_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, xhci_plat_keep_running(~0ULL, PM_SUSPEND_TO_IDLE, true, true)); } +/* Only a controller kept running in suspend-to-idle keeps its interrupt enabled across it. */ +static void xhci_plat_irq_flags_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, xhci_plat_irq_flags(0), (unsigned long)IRQF_SHARED); + KUNIT_EXPECT_EQ(test, xhci_plat_irq_flags(~XHCI_RUN_IN_S2IDLE), (unsigned long)IRQF_SHARED); + KUNIT_EXPECT_EQ(test, xhci_plat_irq_flags(XHCI_RUN_IN_S2IDLE), + (unsigned long)(IRQF_SHARED | IRQF_NO_SUSPEND)); +} + /* A running controller is left alone; a halted, failed or unpowered one is not. */ static void xhci_plat_stopped_test(struct kunit *test) { @@ -279,6 +289,7 @@ static void xhci_plat_all_property_quirks_test(struct kunit *test) static struct kunit_case xhci_plat_test_cases[] = { KUNIT_CASE(xhci_plat_keep_running_test), KUNIT_CASE(xhci_plat_keep_running_other_quirks_test), + KUNIT_CASE(xhci_plat_irq_flags_test), KUNIT_CASE(xhci_plat_stopped_test), KUNIT_CASE(xhci_plat_quiesce_dying_test), #ifdef CONFIG_PM diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index d38d96ec37a56..ad0e2807a862b 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -188,6 +188,17 @@ VISIBLE_IF_KUNIT u64 xhci_plat_property_quirks(struct device *dev) } EXPORT_SYMBOL_IF_KUNIT(xhci_plat_property_quirks); +/* + * A controller kept running in suspend-to-idle decides in its event handling which events wake the + * system, see xhci_run_in_s2idle_begin(). Its interrupt handler has to run while the system + * sleeps, so keep the interrupt enabled across suspend instead of arming it for wakeup. + */ +VISIBLE_IF_KUNIT unsigned long xhci_plat_irq_flags(u64 quirks) +{ + return IRQF_SHARED | ((quirks & XHCI_RUN_IN_S2IDLE) ? IRQF_NO_SUSPEND : 0); +} +EXPORT_SYMBOL_IF_KUNIT(xhci_plat_irq_flags); + int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const struct xhci_plat_priv *priv_match) { const struct hc_driver *driver; @@ -195,6 +206,7 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s struct xhci_hcd *xhci; struct resource *res; struct usb_hcd *hcd, *usb3_hcd; + unsigned long irqflags; int ret; int irq; struct xhci_plat_priv *priv = NULL; @@ -326,7 +338,8 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s if (priv && (priv->quirks & XHCI_SG_TRB_CACHE_SIZE_QUIRK)) xhci->quirks |= XHCI_SG_TRB_CACHE_SIZE_QUIRK; - ret = usb_add_hcd(hcd, irq, IRQF_SHARED); + irqflags = xhci_plat_irq_flags(xhci->quirks); + ret = usb_add_hcd(hcd, irq, irqflags); if (ret) goto disable_usb_phy; @@ -362,15 +375,22 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s if (xhci->shared_hcd) { xhci->shared_hcd->rsrc_start = hcd->rsrc_start; xhci->shared_hcd->rsrc_len = hcd->rsrc_len; - ret = usb_add_hcd(xhci->shared_hcd, irq, IRQF_SHARED); + ret = usb_add_hcd(xhci->shared_hcd, irq, irqflags); if (ret) goto put_usb3_hcd; } + /* + * Running in suspend-to-idle needs the interrupt requested above for it, + * so a later quirk source, such as the xhci_hcd quirks module parameter, + * cannot turn it on. + */ + if (!(irqflags & IRQF_NO_SUSPEND)) + xhci->quirks &= ~XHCI_RUN_IN_S2IDLE; + /* * Let the USB core keep hubs awake that need it for wakeup, see - * XHCI_RUN_IN_S2IDLE. Only now are all quirk sources merged, the xhci_hcd - * quirks module parameter included. + * XHCI_RUN_IN_S2IDLE. */ hcd->runs_in_s2idle = !!(xhci->quirks & XHCI_RUN_IN_S2IDLE); if (xhci->shared_hcd) @@ -536,9 +556,10 @@ static int xhci_plat_suspend_common(struct device *dev) * Some platforms have no wake signal for a halted xHC: a halted controller writes no events, so * its interrupt cannot fire, and on PCI the remote wakeup or port change would be signalled as a * PME instead. Where the platform keeps the controller powered and clocked in suspend-to-idle, - * leave it running instead, with the root hubs suspended and armed for wakeup as usual. A remote - * wakeup or port change then raises a Port Status Change Event, and the interrupt, armed for - * wakeup by the platform, ends suspend-to-idle. + * leave it running instead, with the root hubs suspended and armed for wakeup as usual. The + * controller then interrupts for every event, its interrupt stays enabled (see + * xhci_plat_irq_flags()), and the event handling wakes the system for the events that a halted + * controller would signal. */ VISIBLE_IF_KUNIT bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, bool may_wakeup, bool wakeup_path) diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h index d002aac38ec20..97d7b8475b9be 100644 --- a/drivers/usb/host/xhci-plat.h +++ b/drivers/usb/host/xhci-plat.h @@ -37,6 +37,7 @@ extern const struct dev_pm_ops xhci_plat_pm_ops; #include u64 xhci_plat_property_quirks(struct device *dev); +unsigned long xhci_plat_irq_flags(u64 quirks); bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, bool may_wakeup, bool wakeup_path); bool xhci_plat_stopped(u32 usbsts); From dc024263b82f53638559d5c40ca17f0c924eb625 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 14:33:20 +1100 Subject: [PATCH 1136/1579] usb: dwc3: apple: Do not arm the core interrupt for wakeup In host mode the core interrupt was set up as the wake interrupt of the glue, so that the Port Status Change Event of a running xHCI would end suspend-to-idle. Armed for wakeup, it ended suspend-to-idle for every event of the controller, which is what made a link change on a root port with wakeup disabled wake the J700. xHCI now keeps its interrupt enabled across suspend-to-idle when it keeps running there and wakes the system itself, for the events a halted controller would signal. A wake interrupt on the same line is never armed then, as the interrupt is no longer suspended, and the PM documentation advises against combining the two on one interrupt. For a halted controller it never served a purpose: a halted xHC writes no events, so the interrupt cannot fire. Drop the wake interrupt and the interrupt number that was kept only for it. The glue stays wakeup capable and enabled by default, which dwc3_host_init() passes on to the xHCI device and which decides whether xHCI keeps running. Drop the wake interrupt from the models of the glue in the DCP lifecycle selftests again as well. Fixes: 13e221102941 ("usb: dwc3: apple: Arm the core interrupt for system wakeup in host mode") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple.c | 32 +++---------------- .../drivers/gpu/apple-dcp/dwc3/mocks.c | 17 +--------- .../drivers/gpu/apple-dcp/teardown_pm.py | 6 +--- 3 files changed, 7 insertions(+), 48 deletions(-) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index b24789b574561..49953a518eec9 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -80,8 +79,8 @@ * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. * * Whenever the core is up, for either kind of port, it is kept running across system sleep, see - * dwc3_apple_init(). In host mode its interrupt is then also armed as the wake interrupt, and - * xHCI keeps running in suspend-to-idle so that it can raise it. + * dwc3_apple_init(). In host mode xHCI then keeps running in suspend-to-idle when a USB device + * may wake the system, as a halted xHC could not signal the wakeup. * * enum dwc3_apple_state is defined in dwc3-apple-state.h. */ @@ -90,7 +89,6 @@ * struct dwc3_apple - Apple-specific DWC3 USB controller * @dwc: Core DWC3 structure * @dev: Pointer to the device structure - * @irq: Core interrupt, shared with xHCI in host mode and used as its wake interrupt * @mmio_resource: Resource to be passed to dwc3_core_probe * @apple_regs: Apple-specific DWC3 registers * @reset: Reset control @@ -111,7 +109,6 @@ struct dwc3_apple { struct dwc3 dwc; struct device *dev; - int irq; struct resource *mmio_resource; void __iomem *apple_regs; @@ -214,8 +211,9 @@ static int dwc3_apple_core_probe(struct dwc3_apple *appledwc) probe_data.skip_core_init_mode = true; probe_data.properties = DWC3_DEFAULT_PROPERTIES; /* - * There is no wake interrupt besides the core interrupt, and a halted xHC raises none, - * see dwc3_apple_init(). + * The controller has no wake signal of its own, and a halted xHC writes no events, so its + * interrupt cannot fire either. xHCI keeps running instead and decides which of its events + * wake the system; the core stays powered across system sleep, see dwc3_apple_init(). */ probe_data.properties.xhci_run_in_s2idle = true; @@ -448,21 +446,6 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st goto core_exit; } - /* - * xHCI shares the core interrupt and suspends its root hubs with their wake bits set when - * this device may wake the system (dwc3_host_init() passes that on). The controller has no - * separate wake signal, and a halted xHC writes no events and so raises no interrupt, so - * xHCI is told to keep running in suspend-to-idle instead of halting (the core stays - * powered, see below). Arm the interrupt for wakeup while in host mode, so that the Port - * Status Change Event of a remote wakeup or port change ends suspend-to-idle. - * Device mode has no wake source: a gadget is woken by its host. - */ - if (state == DWC3_APPLE_HOST) { - ret = dev_pm_set_wake_irq(appledwc->dev, appledwc->irq); - if (ret) - dev_warn(appledwc->dev, "Failed to set up the wake IRQ, err=%d\n", ret); - } - appledwc->state = state; /* * A connected cable leaves the Apple DWC3 core and its PHY as one live, @@ -499,7 +482,6 @@ static int dwc3_apple_exit(struct dwc3_apple *appledwc) appledwc->state = DWC3_APPLE_NO_CABLE; break; case DWC3_APPLE_HOST: - dev_pm_clear_wake_irq(appledwc->dev); dwc3_host_exit(&appledwc->dwc); appledwc->state = DWC3_APPLE_NO_CABLE; break; @@ -648,10 +630,6 @@ static int dwc3_apple_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(appledwc->apple_regs), "Failed to map Apple-specific MMIO\n"); - appledwc->irq = platform_get_irq(pdev, 0); - if (appledwc->irq < 0) - return appledwc->irq; - /* * Like a PCI xHCI controller, allow USB devices to wake the system by default. This has * to be set before dwc3_core_probe(), which hands it on to the xHCI device. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c index 0373f28f3b165..e54e73146cbad 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -30,7 +30,7 @@ struct dwc3 { struct phy *usb2_generic_phy[3], *usb3_generic_phy[3]; int num_usb2_ports, num_usb3_ports, usb2_phy, usb3_phy; struct reset_controller_dev *reset; int dev, usb_psy, vbus_draw_work; }; -struct dwc3_apple { struct dwc3 dwc; enum dwc3_apple_state state; int irq; +struct dwc3_apple { struct dwc3 dwc; enum dwc3_apple_state state; struct reset_controller_dev *reset; int lock; void *dev; struct phy *usb2_phy, *usb3_phy; bool fixed_hub, core_active, core_exit_pending, reset_pending; @@ -90,21 +90,6 @@ static void dwc3_clk_disable(struct dwc3 *d) { (void)d; clocks_disabled++; } static void dwc3_host_exit(struct dwc3 *d); static void dwc3_gadget_exit(struct dwc3 *d) { (void)d; gadget_exits++; } static void dev_pm_syscore_device(void *d, bool s) { (void)d; syscore = s; } -static bool wake_irq_set; - -static int dev_pm_set_wake_irq(void *d, int irq) -{ - (void)d; - (void)irq; - wake_irq_set = true; - return 0; -} - -static void dev_pm_clear_wake_irq(void *d) -{ - (void)d; - wake_irq_set = 0; -} static void dwc3_enable_susphy(struct dwc3 *d, bool s) { (void)d; (void)s; } static int atcphy_dwc3_reset_assert(struct reset_controller_dev *, unsigned long); static int reset_control_assert(struct reset_controller_dev *r) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py index 6321c337529cb..1f17261136a9c 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py @@ -80,8 +80,6 @@ def unit_for(source): static void *platform_get_drvdata(struct platform_device *d) { return d==&controller?(void *)&apple_context.dwc:(void *)&hcd; } static void dwc3_enable_susphy(struct dwc3 *d,bool v) { (void)d;assert(v);susphy_calls++;observed_controller_write(); } static void device_init_wakeup(struct device *d,bool b) { (void)d;(void)b; } -static bool wake_irq_set; -static void dev_pm_clear_wake_irq(struct device *d) { assert(d==&controller.dev);wake_irq_set=false; } static void platform_device_unregister(struct platform_device *d); static void dwc3_gadget_exit(struct dwc3 *d) { (void)d;assert(0); } static int dwc3_apple_core_stop(struct dwc3_apple *d) { (void)d;return 0; } @@ -142,7 +140,7 @@ def unit_for(source): xhci_context=(struct xhci_hcd){0}; inject_remove=true;removal_started=blocked_attempt=release_remove=false; atomic_store(&early_write,false);atomic_store(&handoff_active,true); - susphy_calls=host_remove_calls=0;wake_irq_set=true; + susphy_calls=host_remove_calls=0; int ret=safe(&p); atomic_store(&handoff_active,false); pthread_mutex_lock(&schedule_mutex);release_remove=true;pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_mutex); @@ -150,8 +148,6 @@ def unit_for(source): fprintf(stderr,"connected teardown target=%d ret=%d early_write=%d susphy=%d host_remove=%d\n",remove_target,ret,atomic_load(&early_write),susphy_calls,host_remove_calls); assert(ret==0&&!atomic_load(&early_write)&&host_remove_calls==1); assert(susphy_calls==(remove_target?0:1));balanced(&p); - /* Host teardown drops the wake interrupt it armed. */ - assert(remove_target||!wake_irq_set); } puts("PASS actual PM wrapper connected to driver-core/DWC3/xHCI removal; synchronous teardown waits until owned transaction ends"); return 0; From 69b23ab68ffa1fde47aa1fe140beb757e144bea7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 21:50:08 +1100 Subject: [PATCH 1137/1579] usb: core: Tell whether a remote wakeup from below reaches a hub A host controller that has to keep running in suspend-to-idle for USB devices to wake the system should do so only when one of them can. The power/wakeup setting of a device does not tell: whether its remote wakeup gets through also depends on the hubs on its way, and the USB core decides on that as it suspends each device. Add usb_wakeup_reaches(), which tells, once the devices below a hub are suspended, whether one of them can wake it. A device signals a remote wakeup only if the USB core enabled that when it suspended it. A hub passes one on from below whatever its own wakeup setting, which only arms it for connect, disconnect and over-current changes (USB 2.0, section 7.1.7.7, and commit 7aba8d014341 ("USB: don't enable remote wakeup by default")). The exception is a hub with USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP: while it is suspended, it does not pass one on from below, so the devices below it count only while it is kept awake. The quirk is about passing a wakeup on, so such a hub armed for its own changes still counts. Add KUnit tests for it on device trees built in memory, among them the one behind the front port of a J700. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/Kconfig | 11 ++ drivers/usb/core/Makefile | 1 + drivers/usb/core/hub.c | 36 +++++++ drivers/usb/core/usb-wakeup-test.c | 161 +++++++++++++++++++++++++++++ include/linux/usb/hcd.h | 6 ++ 5 files changed, 215 insertions(+) create mode 100644 drivers/usb/core/usb-wakeup-test.c diff --git a/drivers/usb/core/Kconfig b/drivers/usb/core/Kconfig index fbeab9ee46202..1251c543f6cb0 100644 --- a/drivers/usb/core/Kconfig +++ b/drivers/usb/core/Kconfig @@ -108,6 +108,17 @@ config USB_QUIRKS_KUNIT_TEST If unsure, say N. +config USB_WAKEUP_KUNIT_TEST + tristate "KUnit tests for USB wakeup in system sleep" if !KUNIT_ALL_TESTS + depends on USB && PM && KUNIT + default KUNIT_ALL_TESTS + help + Test which USB devices the USB core finds able to wake the system + once it has suspended them, on device trees built in memory. The + tests need no USB hardware. + + If unsure, say N. + config USB_LEDS_TRIGGER_USBPORT tristate "USB port LED trigger" depends on USB && LEDS_TRIGGERS diff --git a/drivers/usb/core/Makefile b/drivers/usb/core/Makefile index bf228db5e15f9..64468ad4a6d51 100644 --- a/drivers/usb/core/Makefile +++ b/drivers/usb/core/Makefile @@ -22,5 +22,6 @@ endif obj-$(CONFIG_USB) += usbcore.o obj-$(CONFIG_USB_QUIRKS_KUNIT_TEST) += usb-quirks-test.o +obj-$(CONFIG_USB_WAKEUP_KUNIT_TEST) += usb-wakeup-test.o obj-$(CONFIG_USB_LEDS_TRIGGER_USBPORT) += ledtrig-usbport.o diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 3e853c43390dd..09da85ed0bbed 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -3450,6 +3450,42 @@ unsigned usb_wakeup_enabled_descendants(struct usb_device *udev) } EXPORT_SYMBOL_GPL(usb_wakeup_enabled_descendants); +/** + * usb_wakeup_reaches - Whether a remote wakeup from below reaches a hub + * @hdev: The hub, with the devices below it suspended for system sleep + * + * A device signals a remote wakeup only if the USB core enabled that when it + * suspended it: resume signalling, or for a hub, a connect, disconnect or + * over-current change on one of its ports. A hub passes on the remote wakeup + * of a device below it whatever its own wakeup setting (USB 2.0, section + * 7.1.7.7). A hub with USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP does not while it is + * suspended, but still signals its own; a hub kept awake does both. + * + * Context: System sleep, once the devices below @hdev are suspended, so that + * neither they nor their wakeup settings change. Takes no locks. + * + * Return: true if a device below @hdev can wake it up. + */ +bool usb_wakeup_reaches(struct usb_device *hdev) +{ + struct usb_device *child; + int port1; + + usb_hub_for_each_child(hdev, port1, child) { + /* A hub kept awake reports any change on its status endpoint */ + u32 quirks = child->awake_in_sleep ? 0 : child->quirks; + + if (child->do_remote_wakeup) + return true; + if (!(quirks & USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP) && + usb_wakeup_reaches(child)) + return true; + } + + return false; +} +EXPORT_SYMBOL_GPL(usb_wakeup_reaches); + /* * usb_port_suspend - suspend a usb device's upstream port * @udev: device that's no longer in active use, not a root hub diff --git a/drivers/usb/core/usb-wakeup-test.c b/drivers/usb/core/usb-wakeup-test.c new file mode 100644 index 0000000000000..4e8658d316dab --- /dev/null +++ b/drivers/usb/core/usb-wakeup-test.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for the USB core's wakeup decisions in system sleep + */ + +#include +#include +#include +#include +#include + +/* hub.h includes usb.h, which has no include guard */ +#include "hub.h" + +#define USB_WAKEUP_TEST_PORTS 4U + +/* A suspended USB device, with the hub driver state of a hub */ +struct usb_wakeup_test_dev { + struct usb_device udev; + struct usb_host_config config; + struct usb_interface intf; + struct usb_hub hub; + struct usb_port ports[USB_WAKEUP_TEST_PORTS]; + struct usb_port *port_list[USB_WAKEUP_TEST_PORTS]; +}; + +/* A hub or another device, attached to the first free port of @parent if any */ +static struct usb_wakeup_test_dev *usb_wakeup_test_add(struct kunit *test, + struct usb_wakeup_test_dev *parent, + bool hub) +{ + struct usb_wakeup_test_dev *d = kunit_kzalloc(test, sizeof(*d), GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, d); + if (hub) { + for (i = 0; i < USB_WAKEUP_TEST_PORTS; i++) + d->port_list[i] = &d->ports[i]; + d->hub.hdev = &d->udev; + d->hub.ports = d->port_list; + d->config.interface[0] = &d->intf; + usb_set_intfdata(&d->intf, &d->hub); + d->udev.actconfig = &d->config; + d->udev.maxchild = USB_WAKEUP_TEST_PORTS; + } + if (parent) { + for (i = 0; i < USB_WAKEUP_TEST_PORTS && parent->ports[i].child; i++) + ; + KUNIT_ASSERT_LT(test, i, USB_WAKEUP_TEST_PORTS); + parent->ports[i].child = &d->udev; + d->udev.parent = &parent->udev; + } + + return d; +} + +/* + * The defaults behind the front port of an Apple J700: a keyboard with remote + * wakeup enabled, below a hub, below the VIA Labs VL122 hub on the root port, + * every hub with wakeup disabled. The VL122 does not pass a remote wakeup on + * while it is suspended, so the keyboard's does not reach the root hub, unless + * the VL122 is kept awake. + */ +static void usb_wakeup_reaches_vl122_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_add(test, NULL, true); + struct usb_wakeup_test_dev *vl122 = usb_wakeup_test_add(test, root, true); + struct usb_wakeup_test_dev *hub = usb_wakeup_test_add(test, vl122, true); + struct usb_wakeup_test_dev *keyboard = usb_wakeup_test_add(test, hub, false); + + vl122->udev.quirks = USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP; + keyboard->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&hub->udev)); + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&vl122->udev)); + KUNIT_EXPECT_FALSE(test, usb_wakeup_reaches(&root->udev)); + + vl122->udev.awake_in_sleep = 1; + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); +} + +/* Any other hub passes it on, whatever its own wakeup setting. */ +static void usb_wakeup_reaches_hub_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_add(test, NULL, true); + struct usb_wakeup_test_dev *hub = usb_wakeup_test_add(test, root, true); + struct usb_wakeup_test_dev *keyboard = usb_wakeup_test_add(test, hub, false); + + usb_wakeup_test_add(test, hub, false); + usb_wakeup_test_add(test, root, false); + KUNIT_EXPECT_FALSE(test, usb_wakeup_reaches(&root->udev)); + + keyboard->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); +} + +/* + * A hub armed for wakeup signals connect, disconnect and over-current changes + * on its ports, also if it does not pass wakeups on from below while suspended. + */ +static void usb_wakeup_reaches_armed_hub_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_add(test, NULL, true); + struct usb_wakeup_test_dev *hub = usb_wakeup_test_add(test, root, true); + + hub->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); + + hub->udev.quirks = USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP; + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); +} + +/* + * A hub that does not pass wakeups on from below while it is suspended, armed + * for wakeup, with wakeup disabled for the root hub and the device below it: + * its own changes reach the root hub. A remote wakeup from below it does not. + */ +static void usb_wakeup_reaches_armed_quirky_hub_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_add(test, NULL, true); + struct usb_wakeup_test_dev *hub = usb_wakeup_test_add(test, root, true); + struct usb_wakeup_test_dev *keyboard = usb_wakeup_test_add(test, hub, false); + + hub->udev.quirks = USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP; + hub->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); + + hub->udev.do_remote_wakeup = 0; + keyboard->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_FALSE(test, usb_wakeup_reaches(&root->udev)); +} + +/* A root hub with nothing below, and a device that is no hub, reach nothing. */ +static void usb_wakeup_reaches_nothing_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_add(test, NULL, true); + struct usb_wakeup_test_dev *keyboard; + + KUNIT_EXPECT_FALSE(test, usb_wakeup_reaches(&root->udev)); + keyboard = usb_wakeup_test_add(test, root, false); + keyboard->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_FALSE(test, usb_wakeup_reaches(&keyboard->udev)); +} + +static struct kunit_case usb_wakeup_test_cases[] = { + KUNIT_CASE(usb_wakeup_reaches_vl122_test), + KUNIT_CASE(usb_wakeup_reaches_hub_test), + KUNIT_CASE(usb_wakeup_reaches_armed_hub_test), + KUNIT_CASE(usb_wakeup_reaches_armed_quirky_hub_test), + KUNIT_CASE(usb_wakeup_reaches_nothing_test), + {} +}; + +static struct kunit_suite usb_wakeup_test_suite = { + .name = "usb_wakeup", + .test_cases = usb_wakeup_test_cases, +}; + +kunit_test_suite(usb_wakeup_test_suite); + +MODULE_DESCRIPTION("KUnit tests for the USB core's wakeup decisions in system sleep"); +MODULE_LICENSE("GPL"); diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index f4520d9922a2b..604b9dda94317 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -692,6 +692,7 @@ extern wait_queue_head_t usb_kill_urb_queue; #ifdef CONFIG_PM extern unsigned usb_wakeup_enabled_descendants(struct usb_device *udev); +bool usb_wakeup_reaches(struct usb_device *hdev); extern void usb_root_hub_lost_power(struct usb_device *rhdev); extern int hcd_bus_suspend(struct usb_device *rhdev, pm_message_t msg); extern int hcd_bus_resume(struct usb_device *rhdev, pm_message_t msg); @@ -701,6 +702,11 @@ static inline unsigned usb_wakeup_enabled_descendants(struct usb_device *udev) { return 0; } + +static inline bool usb_wakeup_reaches(struct usb_device *hdev) +{ + return false; +} static inline void usb_hcd_resume_root_hub(struct usb_hcd *hcd) { return; From cdb68b78f648455b8be5a841a419c8e0ff856c6b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 00:40:27 +1100 Subject: [PATCH 1138/1579] usb: core: Keep a hub awake in suspend-to-idle for its own wakeup While it is suspended, the VIA Labs VL122 hub on the Apple J700 passes no remote wakeup on from below (USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP), and it does not signal one for a connect, disconnect or over-current change on its own ports either. That is what made it miss downstream connects in runtime suspend: with the hub suspended and its root port armed for wakeup, a dock attached to either connector left the root port in U3, without a link state change, until something resumed the hub from the host side, see commit 601d47365fcf ("usb: core: ignore remote wakeup on the VIA VL122 hub"). So in suspend-to-idle, a VL122 with its wakeup enabled and nothing below it that can wake is suspended like any other hub armed for those changes, and none of them wakes the system. Add USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP for that, and set it for the VL122. Keep a hub with it awake in suspend-to-idle when its wakeup is enabled and the host controller keeps running, as is done for a device below a hub that does not pass wakeups on, so that its status endpoint reports those changes. usb_wakeup_reaches() counts its own wakeup only while it is kept awake. The cost is an active hub and root port while wakeup is enabled for the hub, which it is not by default. Add KUnit tests that the VL122 armed for wakeup, with nothing below it that can wake, is kept awake, and so is the root hub above it, and that a hub with only the other quirk is not, but still counts as able to wake. Fixes: 8d33f6497da5 ("usb: hub: keep the VL122 awake instead of ignoring its remote wakeup") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/driver.c | 18 +++-- drivers/usb/core/hub.c | 8 ++- drivers/usb/core/quirks.c | 3 +- drivers/usb/core/usb-quirks-test.c | 7 +- drivers/usb/core/usb-wakeup-test.c | 105 ++++++++++++++++++++++++++++- drivers/usb/core/usb.h | 1 + drivers/usb/host/xhci-ring.c | 6 +- include/linux/usb/quirks.h | 8 +++ 8 files changed, 140 insertions(+), 16 deletions(-) diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c index 293a6b5facb46..56fa3d9b4a2ba 100644 --- a/drivers/usb/core/driver.c +++ b/drivers/usb/core/driver.c @@ -25,6 +25,7 @@ * */ +#include #include #include #include @@ -1386,12 +1387,14 @@ static int usb_resume_interface(struct usb_device *udev, /* * Whether to keep a device awake in suspend-to-idle so that a remote wakeup - * from below can reach the host controller: a hub that does not pass one on - * while it is suspended itself, and every device between such a hub and the - * root hub, root hub included. This only works while the host controller keeps - * running, so its interrupt reports the hub's status change. + * can reach the host controller: a hub that does not pass one on from below + * while it is suspended itself, a hub with wakeup enabled that does not signal + * the changes of its own ports while it is suspended, and every device between + * such a hub and the root hub, root hub included. This only works while the + * host controller keeps running, so its interrupt reports the hub's status + * change. */ -static bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_message_t msg) +VISIBLE_IF_KUNIT bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_message_t msg) { struct usb_hcd *hcd = bus_to_hcd(udev->bus); struct usb_device *child; @@ -1402,6 +1405,10 @@ static bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_message_t msg) !device_may_wakeup(hcd->self.controller)) return false; + if ((udev->quirks & USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP) && + udev->do_remote_wakeup) + return true; + if ((udev->quirks & USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP) && udev->do_remote_wakeup && device_wakeup_path(&udev->dev)) return true; @@ -1414,6 +1421,7 @@ static bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_message_t msg) return false; } +EXPORT_SYMBOL_IF_KUNIT(usb_stay_awake_for_wakeup); /** * usb_suspend_both - suspend a USB device and its interfaces diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 09da85ed0bbed..d3ce25f24c047 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -3458,8 +3458,9 @@ EXPORT_SYMBOL_GPL(usb_wakeup_enabled_descendants); * suspended it: resume signalling, or for a hub, a connect, disconnect or * over-current change on one of its ports. A hub passes on the remote wakeup * of a device below it whatever its own wakeup setting (USB 2.0, section - * 7.1.7.7). A hub with USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP does not while it is - * suspended, but still signals its own; a hub kept awake does both. + * 7.1.7.7). While it is suspended, a hub with USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP + * does not, and one with USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP does not signal + * its own; a hub kept awake does both. * * Context: System sleep, once the devices below @hdev are suspended, so that * neither they nor their wakeup settings change. Takes no locks. @@ -3475,7 +3476,8 @@ bool usb_wakeup_reaches(struct usb_device *hdev) /* A hub kept awake reports any change on its status endpoint */ u32 quirks = child->awake_in_sleep ? 0 : child->quirks; - if (child->do_remote_wakeup) + if (child->do_remote_wakeup && + !(quirks & USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP)) return true; if (!(quirks & USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP) && usb_wakeup_reaches(child)) diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index bf38174963e5c..50c564c46b48a 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -586,7 +586,8 @@ static const struct usb_device_id usb_quirk_list[] = { /* VIA Labs, Inc. VL122 USB2.0 Hub */ { USB_DEVICE(0x2109, 0x2122), .driver_info = - USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP }, + USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP | + USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP }, /* VIA Labs, Inc. USB2.0 Hub */ { USB_DEVICE(0x2109, 0x2817), .driver_info = USB_QUIRK_NO_LPM }, diff --git a/drivers/usb/core/usb-quirks-test.c b/drivers/usb/core/usb-quirks-test.c index 81ebc58238b0d..5587e1d4cfa0c 100644 --- a/drivers/usb/core/usb-quirks-test.c +++ b/drivers/usb/core/usb-quirks-test.c @@ -38,9 +38,9 @@ static struct usb_quirks_test *usb_quirks_test_hub(struct kunit *test, u16 vid, } /* - * The VIA Labs VL122 hub on the Apple J700 does not pass on remote wakeup while it - * is suspended. It stays out of autosuspend and keeps its wakeup setting, and is - * kept awake in suspend-to-idle instead. + * The VIA Labs VL122 hub on the Apple J700 signals no remote wakeup while it is + * suspended, neither for its own ports nor from below. It stays out of autosuspend + * and keeps its wakeup setting, and is kept awake in suspend-to-idle instead. */ static void usb_quirks_vl122_test(struct kunit *test) { @@ -49,6 +49,7 @@ static void usb_quirks_vl122_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, quirks & USB_QUIRK_IGNORE_REMOTE_WAKEUP); KUNIT_EXPECT_TRUE(test, quirks & USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP); + KUNIT_EXPECT_TRUE(test, quirks & USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP); KUNIT_EXPECT_TRUE(test, usb_hub_id_quirks(&t->intf) & HUB_QUIRK_DISABLE_AUTOSUSPEND); } diff --git a/drivers/usb/core/usb-wakeup-test.c b/drivers/usb/core/usb-wakeup-test.c index 4e8658d316dab..d39c71c9c459e 100644 --- a/drivers/usb/core/usb-wakeup-test.c +++ b/drivers/usb/core/usb-wakeup-test.c @@ -3,8 +3,11 @@ * KUnit tests for the USB core's wakeup decisions in system sleep */ +#include #include +#include #include +#include #include #include #include @@ -14,6 +17,10 @@ #define USB_WAKEUP_TEST_PORTS 4U +/* The device quirks of the VIA Labs VL122 hub */ +#define USB_WAKEUP_TEST_VL122 (USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP | \ + USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP) + /* A suspended USB device, with the hub driver state of a hub */ struct usb_wakeup_test_dev { struct usb_device udev; @@ -49,6 +56,7 @@ static struct usb_wakeup_test_dev *usb_wakeup_test_add(struct kunit *test, KUNIT_ASSERT_LT(test, i, USB_WAKEUP_TEST_PORTS); parent->ports[i].child = &d->udev; d->udev.parent = &parent->udev; + d->udev.bus = parent->udev.bus; } return d; @@ -68,7 +76,7 @@ static void usb_wakeup_reaches_vl122_test(struct kunit *test) struct usb_wakeup_test_dev *hub = usb_wakeup_test_add(test, vl122, true); struct usb_wakeup_test_dev *keyboard = usb_wakeup_test_add(test, hub, false); - vl122->udev.quirks = USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP; + vl122->udev.quirks = USB_WAKEUP_TEST_VL122; keyboard->udev.do_remote_wakeup = 1; KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&hub->udev)); KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&vl122->udev)); @@ -141,12 +149,106 @@ static void usb_wakeup_reaches_nothing_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, usb_wakeup_reaches(&keyboard->udev)); } +/* + * The VL122 does not signal the changes of its own ports while it is suspended + * either: armed for wakeup, it reaches the root hub only while it is kept awake. + */ +static void usb_wakeup_reaches_armed_vl122_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_add(test, NULL, true); + struct usb_wakeup_test_dev *vl122 = usb_wakeup_test_add(test, root, true); + + vl122->udev.quirks = USB_WAKEUP_TEST_VL122; + vl122->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_FALSE(test, usb_wakeup_reaches(&root->udev)); + + vl122->udev.awake_in_sleep = 1; + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); +} + +static void usb_wakeup_test_restore_target(void *state) +{ + pm_suspend_target_state = *(suspend_state_t *)state; +} + +/* + * A root hub on a host controller that keeps running in suspend-to-idle, with + * wakeup enabled, as the system goes to suspend-to-idle. + */ +static struct usb_wakeup_test_dev *usb_wakeup_test_s2idle_root(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_add(test, NULL, true); + struct usb_hcd *hcd = kunit_kzalloc(test, sizeof(*hcd), GFP_KERNEL); + struct device *dev = kunit_device_register(test, "usb-wakeup-test"); + suspend_state_t *target = kunit_kzalloc(test, sizeof(*target), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, hcd); + KUNIT_ASSERT_NOT_NULL(test, target); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + KUNIT_ASSERT_EQ(test, devm_device_init_wakeup(dev), 0); + hcd->self.controller = dev; + hcd->runs_in_s2idle = 1; + root->udev.bus = &hcd->self; + + *target = pm_suspend_target_state; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, usb_wakeup_test_restore_target, + target), 0); + pm_suspend_target_state = PM_SUSPEND_TO_IDLE; + + return root; +} + +/* + * The VL122 armed for wakeup, with nothing below it that can wake: it is kept + * awake in suspend-to-idle for the changes of its own ports, and so is the root + * hub above it, but not for other system sleep transitions, or on a controller + * that halts or may not wake the system. A hub that signals those changes while + * it is suspended is not kept awake for them. + */ +static void usb_wakeup_stay_awake_armed_vl122_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_s2idle_root(test); + struct usb_wakeup_test_dev *vl122 = usb_wakeup_test_add(test, root, true); + struct usb_hcd *hcd = bus_to_hcd(root->udev.bus); + + usb_wakeup_test_add(test, vl122, false); + vl122->udev.quirks = USB_WAKEUP_TEST_VL122; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + + vl122->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_TRUE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_FREEZE)); + + pm_suspend_target_state = PM_SUSPEND_MEM; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + pm_suspend_target_state = PM_SUSPEND_TO_IDLE; + + hcd->runs_in_s2idle = 0; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + hcd->runs_in_s2idle = 1; + + device_set_wakeup_enable(hcd->self.controller, false); + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + device_set_wakeup_enable(hcd->self.controller, true); + + vl122->udev.awake_in_sleep = 1; + KUNIT_EXPECT_TRUE(test, usb_stay_awake_for_wakeup(&root->udev, PMSG_SUSPEND)); + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); + + vl122->udev.awake_in_sleep = 0; + vl122->udev.quirks = USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); +} + static struct kunit_case usb_wakeup_test_cases[] = { KUNIT_CASE(usb_wakeup_reaches_vl122_test), KUNIT_CASE(usb_wakeup_reaches_hub_test), KUNIT_CASE(usb_wakeup_reaches_armed_hub_test), KUNIT_CASE(usb_wakeup_reaches_armed_quirky_hub_test), KUNIT_CASE(usb_wakeup_reaches_nothing_test), + KUNIT_CASE(usb_wakeup_reaches_armed_vl122_test), + KUNIT_CASE(usb_wakeup_stay_awake_armed_vl122_test), {} }; @@ -159,3 +261,4 @@ kunit_test_suite(usb_wakeup_test_suite); MODULE_DESCRIPTION("KUnit tests for the USB core's wakeup decisions in system sleep"); MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/core/usb.h b/drivers/usb/core/usb.h index c8880c20f1563..771902dcb3d1b 100644 --- a/drivers/usb/core/usb.h +++ b/drivers/usb/core/usb.h @@ -38,6 +38,7 @@ extern void usb_authorize_interface(struct usb_interface *); extern void usb_detect_quirks(struct usb_device *udev); #if IS_ENABLED(CONFIG_KUNIT) u32 usb_detect_device_list_quirks(struct usb_device *udev); +bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_message_t msg); #endif extern void usb_detect_interface_quirks(struct usb_device *udev); extern void usb_release_quirk_list(void); diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index e2ff3e14b139a..3dddc6a4da7dd 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -2027,9 +2027,9 @@ static bool xhci_s2idle_port_change(struct xhci_hcd *xhci, struct xhci_port *por } /* - * Only hubs that the USB core kept awake to pass on a remote wakeup from below - * have transfers in flight, see usb_suspend_both(): a completed one reports - * a change on one of their ports. + * Only hubs that the USB core kept awake for a remote wakeup have transfers in + * flight, see usb_suspend_both(): a completed one reports a change on one of + * their ports. */ VISIBLE_IF_KUNIT bool xhci_s2idle_transfer_wakeup(const struct usb_device *udev) { diff --git a/include/linux/usb/quirks.h b/include/linux/usb/quirks.h index 8bcceb0d02bd8..a77d3eaf8dc4c 100644 --- a/include/linux/usb/quirks.h +++ b/include/linux/usb/quirks.h @@ -91,4 +91,12 @@ */ #define USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP BIT(20) +/* + * Hub does not signal a remote wakeup for a connect, disconnect or over-current + * change on its own ports while it is suspended. Keep it awake in + * suspend-to-idle when its wakeup is enabled and the host controller keeps + * running. + */ +#define USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP BIT(21) + #endif /* __LINUX_USB_QUIRKS_H */ From ca00d3fc3aad0f4978458be7f9fc8ec5c899a487 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 21:50:09 +1100 Subject: [PATCH 1139/1579] usb: xhci: plat: Keep running in suspend-to-idle only when USB can wake XHCI_RUN_IN_S2IDLE keeps the controller running in suspend-to-idle when wakeup is enabled for it and for a device below it, so that the device can wake the system. That costs power, and a device with wakeup enabled does not mean that its remote wakeup gets through. On a J700 at its defaults, wakeup is enabled for the controller and, by usbhid, for the USB keyboards, and disabled for the hubs and root hubs. The VIA Labs VL122 hub between the keyboards and the root hub passes no remote wakeup on while it is suspended, and the USB core keeps it awake only when its own wakeup is enabled. So the controller was kept running, and a key press did not wake the system: usb usb1: bus suspend, wakeup 0 usb usb2: bus suspend, wakeup 0 xhci-hcd xhci-hcd.0.auto: wakeup enabled, keep running in suspend-to-idle. Keep it running only when a USB device can wake the system through one of its root hubs: the root hub is armed for connect, disconnect and over-current changes, or kept awake for a hub below it, or the remote wakeup of a device below reaches it, see usb_wakeup_reaches(). Halt it as before otherwise. Add a KUnit test of the decision, with the J700 defaults among its cases. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci-plat-test.c | 38 +++++++++++++++++++++++++++---- drivers/usb/host/xhci-plat.c | 27 ++++++++++++++++++---- drivers/usb/host/xhci-plat.h | 3 ++- 3 files changed, 57 insertions(+), 11 deletions(-) diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c index b07b3f985ea26..49cbd15e4ed34 100644 --- a/drivers/usb/host/xhci-plat-test.c +++ b/drivers/usb/host/xhci-plat-test.c @@ -18,7 +18,7 @@ #include "xhci.h" #include "xhci-plat.h" -/* Only suspend-to-idle with the quirk, wakeup enabled and a wakeup path keeps it running. */ +/* Only suspend-to-idle with the quirk, wakeup enabled and a USB wakeup keeps it running. */ static void xhci_plat_keep_running_test(struct kunit *test) { static const suspend_state_t targets[] = { @@ -29,19 +29,46 @@ static void xhci_plat_keep_running_test(struct kunit *test) for (i = 0; i < ARRAY_SIZE(targets); i++) { for (combo = 0; combo < 8; combo++) { u64 quirks = (combo & 1) ? XHCI_RUN_IN_S2IDLE : 0; - bool may_wakeup = combo & 2, wakeup_path = combo & 4; + bool may_wakeup = combo & 2, usb_wakeup = combo & 4; bool expected = combo == 7 && targets[i] == PM_SUSPEND_TO_IDLE; bool kept = xhci_plat_keep_running(quirks, targets[i], may_wakeup, - wakeup_path); + usb_wakeup); KUNIT_EXPECT_EQ_MSG(test, kept, expected, - "target %d quirk %d may_wakeup %d wakeup_path %d", - targets[i], combo & 1, may_wakeup, wakeup_path); + "target %d quirk %d may_wakeup %d usb_wakeup %d", + targets[i], combo & 1, may_wakeup, usb_wakeup); } } } +/* + * Something below a root hub has to be able to wake the system, which the USB core decided as it + * suspended the devices. The J700 defaults are the first case: wakeup is enabled for a keyboard, + * but disabled for the VL122 hub between it and the root hub, which does not pass the remote + * wakeup on while it is suspended, and for the root hub. Nothing can wake the system through it. + */ +static void xhci_plat_usb_wakeup_test(struct kunit *test) +{ + struct usb_hcd *hcd = kunit_kzalloc(test, sizeof(*hcd), GFP_KERNEL); + struct usb_device *rhdev = kunit_kzalloc(test, sizeof(*rhdev), GFP_KERNEL); + + KUNIT_ASSERT_TRUE(test, hcd && rhdev); + KUNIT_EXPECT_FALSE(test, xhci_plat_usb_wakeup(NULL)); + KUNIT_EXPECT_FALSE(test, xhci_plat_usb_wakeup(hcd)); + hcd->self.root_hub = rhdev; + KUNIT_EXPECT_FALSE(test, xhci_plat_usb_wakeup(hcd)); + + /* Armed for connect, disconnect and over-current changes */ + rhdev->do_remote_wakeup = 1; + KUNIT_EXPECT_TRUE(test, xhci_plat_usb_wakeup(hcd)); + rhdev->do_remote_wakeup = 0; + + /* Kept awake for a hub below, see usb_suspend_both() */ + rhdev->awake_in_sleep = 1; + KUNIT_EXPECT_TRUE(test, xhci_plat_usb_wakeup(hcd)); +} + /* Other quirks do not stand in for the suspend-to-idle one. */ static void xhci_plat_keep_running_other_quirks_test(struct kunit *test) { @@ -289,6 +316,7 @@ static void xhci_plat_all_property_quirks_test(struct kunit *test) static struct kunit_case xhci_plat_test_cases[] = { KUNIT_CASE(xhci_plat_keep_running_test), KUNIT_CASE(xhci_plat_keep_running_other_quirks_test), + KUNIT_CASE(xhci_plat_usb_wakeup_test), KUNIT_CASE(xhci_plat_irq_flags_test), KUNIT_CASE(xhci_plat_stopped_test), KUNIT_CASE(xhci_plat_quiesce_dying_test), diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index ad0e2807a862b..b82ed7dfdc864 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -559,16 +559,32 @@ static int xhci_plat_suspend_common(struct device *dev) * leave it running instead, with the root hubs suspended and armed for wakeup as usual. The * controller then interrupts for every event, its interrupt stays enabled (see * xhci_plat_irq_flags()), and the event handling wakes the system for the events that a halted - * controller would signal. + * controller would signal. That costs power, so do it only when a USB device can wake the system + * through one of its root hubs, see xhci_plat_usb_wakeup(). */ VISIBLE_IF_KUNIT bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, - bool may_wakeup, bool wakeup_path) + bool may_wakeup, bool usb_wakeup) { return (quirks & XHCI_RUN_IN_S2IDLE) && target == PM_SUSPEND_TO_IDLE && - may_wakeup && wakeup_path; + may_wakeup && usb_wakeup; } EXPORT_SYMBOL_IF_KUNIT(xhci_plat_keep_running); +/* + * Whether a USB device can wake the system through the root hub of @hcd, once the USB core has + * suspended them all: the root hub is armed for connect, disconnect and over-current changes, or + * kept awake for a hub below it, or the remote wakeup of a device below reaches it. A device with + * wakeup enabled does not count if a hub on its way does not pass the wakeup on. + */ +VISIBLE_IF_KUNIT bool xhci_plat_usb_wakeup(struct usb_hcd *hcd) +{ + struct usb_device *rhdev = hcd ? hcd->self.root_hub : NULL; + + return rhdev && (rhdev->do_remote_wakeup || rhdev->awake_in_sleep || + usb_wakeup_reaches(rhdev)); +} +EXPORT_SYMBOL_IF_KUNIT(xhci_plat_usb_wakeup); + /* Whether a controller that was left running reads back as stopped or failed */ VISIBLE_IF_KUNIT bool xhci_plat_stopped(u32 usbsts) { @@ -581,8 +597,9 @@ static bool xhci_plat_run_in_s2idle(struct device *dev) struct usb_hcd *hcd = dev_get_drvdata(dev); struct xhci_hcd *xhci = hcd_to_xhci(hcd); - return xhci_plat_keep_running(xhci->quirks, pm_suspend_target_state, - device_may_wakeup(dev), device_wakeup_path(dev)); + return xhci_plat_keep_running(xhci->quirks, pm_suspend_target_state, device_may_wakeup(dev), + xhci_plat_usb_wakeup(xhci->main_hcd) || + xhci_plat_usb_wakeup(xhci->shared_hcd)); } static int xhci_plat_suspend(struct device *dev) diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h index 97d7b8475b9be..0c4affdb5236d 100644 --- a/drivers/usb/host/xhci-plat.h +++ b/drivers/usb/host/xhci-plat.h @@ -39,7 +39,8 @@ extern const struct dev_pm_ops xhci_plat_pm_ops; u64 xhci_plat_property_quirks(struct device *dev); unsigned long xhci_plat_irq_flags(u64 quirks); bool xhci_plat_keep_running(u64 quirks, suspend_state_t target, bool may_wakeup, - bool wakeup_path); + bool usb_wakeup); +bool xhci_plat_usb_wakeup(struct usb_hcd *hcd); bool xhci_plat_stopped(u32 usbsts); int xhci_plat_resume_from_runtime(struct device *dev); #endif From 859fce0b1b565a85ef48668c688e70c0faec9cc4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 21:50:09 +1100 Subject: [PATCH 1140/1579] usb: core: Keep a hub awake for any device below that can wake the system The VIA Labs VL122 hub (USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP) passes no remote wakeup from below on while it is suspended, so in suspend-to-idle the USB core keeps it awake, with its status endpoint running, for a device below that may wake the system. It does so only when wakeup is enabled for the hub itself, though. Hubs have it disabled by default, while usbhid enables it for keyboards, so at its defaults a J700 never keeps the VL122 awake, and a USB keyboard behind it cannot wake the system. The hub's own setting is the wrong condition for a wakeup from below. It already keeps the VL122 awake for the changes of its own ports (USB_QUIRK_HUB_NO_SUSPENDED_PORT_WAKEUP). Any other hub passes on a remote wakeup from below whatever its own wakeup setting, which only arms it for connect, disconnect and over-current changes (USB 2.0, section 7.1.7.7, and commit 7aba8d014341 ("USB: don't enable remote wakeup by default")). Keep the VL122 awake whenever the remote wakeup of a device below can reach it, see usb_wakeup_reaches(), so that it behaves as such a hub does, and the wakeup setting of the keyboard decides, as it does behind any other hub. This costs power at the defaults: with a USB keyboard below the VL122, the hub and the host controller stay up in every suspend-to-idle. To avoid that, disable wakeup for the keyboard, or for the host controller. As the hub reports any change of its ports on its status endpoint, a connect or disconnect on one of them also wakes the system while it is kept awake. Add KUnit tests that the VL122 is kept awake at the J700 defaults, and not outside suspend-to-idle, on a controller that halts or may not wake the system, or without a device below that can wake it. Fixes: a28673c796d5 ("usb: core: Keep a hub awake in suspend-to-idle when it cannot pass on wakeup") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/driver.c | 7 +++- drivers/usb/core/usb-wakeup-test.c | 57 ++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c index 56fa3d9b4a2ba..a63358753e935 100644 --- a/drivers/usb/core/driver.c +++ b/drivers/usb/core/driver.c @@ -1393,6 +1393,11 @@ static int usb_resume_interface(struct usb_device *udev, * such a hub and the root hub, root hub included. This only works while the * host controller keeps running, so its interrupt reports the hub's status * change. + * + * Other hubs pass on the remote wakeup of a device below them whatever their + * own wakeup setting, which only arms them for connect, disconnect and + * over-current changes (USB 2.0, sections 7.1.7.7 and 11.4.4). So does a hub + * kept awake here: it is kept awake for any device below that can wake it. */ VISIBLE_IF_KUNIT bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_message_t msg) { @@ -1410,7 +1415,7 @@ VISIBLE_IF_KUNIT bool usb_stay_awake_for_wakeup(struct usb_device *udev, pm_mess return true; if ((udev->quirks & USB_QUIRK_HUB_NO_SUSPENDED_WAKEUP) && - udev->do_remote_wakeup && device_wakeup_path(&udev->dev)) + usb_wakeup_reaches(udev)) return true; /* Children are done with their suspend before their parent starts */ diff --git a/drivers/usb/core/usb-wakeup-test.c b/drivers/usb/core/usb-wakeup-test.c index d39c71c9c459e..e563dd7822886 100644 --- a/drivers/usb/core/usb-wakeup-test.c +++ b/drivers/usb/core/usb-wakeup-test.c @@ -241,6 +241,61 @@ static void usb_wakeup_stay_awake_armed_vl122_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, usb_wakeup_reaches(&root->udev)); } +/* + * The defaults behind the front port of a J700: the VL122 hub, which does not + * pass a remote wakeup on while it is suspended, has wakeup disabled, as hubs + * have by default, and a keyboard below it has it enabled. The VL122 is kept + * awake for the keyboard, and so is the root hub above it. + */ +static void usb_wakeup_stay_awake_vl122_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_s2idle_root(test); + struct usb_wakeup_test_dev *vl122 = usb_wakeup_test_add(test, root, true); + struct usb_wakeup_test_dev *hub = usb_wakeup_test_add(test, vl122, true); + struct usb_wakeup_test_dev *keyboard = usb_wakeup_test_add(test, hub, false); + + vl122->udev.quirks = USB_WAKEUP_TEST_VL122; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + + keyboard->udev.do_remote_wakeup = 1; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&keyboard->udev, PMSG_SUSPEND)); + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&hub->udev, PMSG_SUSPEND)); + KUNIT_EXPECT_TRUE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + + vl122->udev.awake_in_sleep = 1; + KUNIT_EXPECT_TRUE(test, usb_stay_awake_for_wakeup(&root->udev, PMSG_SUSPEND)); +} + +/* + * Nothing is kept awake for other system sleep transitions, for a controller + * that halts or may not wake the system, or with no device below that can wake. + */ +static void usb_wakeup_stay_awake_not_test(struct kunit *test) +{ + struct usb_wakeup_test_dev *root = usb_wakeup_test_s2idle_root(test); + struct usb_wakeup_test_dev *vl122 = usb_wakeup_test_add(test, root, true); + struct usb_wakeup_test_dev *keyboard = usb_wakeup_test_add(test, vl122, false); + struct usb_hcd *hcd = bus_to_hcd(root->udev.bus); + + vl122->udev.quirks = USB_WAKEUP_TEST_VL122; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + keyboard->udev.do_remote_wakeup = 1; + KUNIT_ASSERT_TRUE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_FREEZE)); + + pm_suspend_target_state = PM_SUSPEND_MEM; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + pm_suspend_target_state = PM_SUSPEND_TO_IDLE; + + hcd->runs_in_s2idle = 0; + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); + hcd->runs_in_s2idle = 1; + + device_set_wakeup_enable(hcd->self.controller, false); + KUNIT_EXPECT_FALSE(test, usb_stay_awake_for_wakeup(&vl122->udev, PMSG_SUSPEND)); +} + static struct kunit_case usb_wakeup_test_cases[] = { KUNIT_CASE(usb_wakeup_reaches_vl122_test), KUNIT_CASE(usb_wakeup_reaches_hub_test), @@ -249,6 +304,8 @@ static struct kunit_case usb_wakeup_test_cases[] = { KUNIT_CASE(usb_wakeup_reaches_nothing_test), KUNIT_CASE(usb_wakeup_reaches_armed_vl122_test), KUNIT_CASE(usb_wakeup_stay_awake_armed_vl122_test), + KUNIT_CASE(usb_wakeup_stay_awake_vl122_test), + KUNIT_CASE(usb_wakeup_stay_awake_not_test), {} }; From 9ccab018d67f2aaacd7ca9a70e35c7eceb6b0da4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:24:02 +1100 Subject: [PATCH 1141/1579] PCI: apple: Let MSIs wake the system enable_irq_wake() fails with -ENXIO for every MSI of a function behind this host bridge. irq_set_irq_wake() only looks at the top-level chip, which for a PCI function is the per-device PCI/MSI chip built from the generic template; msi_lib_init_dev_msi_info() gives it neither an .irq_set_wake callback nor IRQCHIP_SKIP_SET_WAKE, and the MSI bottom chip of this driver does not forward the request either. A function driver that arms its interrupt for wakeup, and the PCIe port PME service, are therefore left without a wake interrupt, suspend_device_irqs() disables the vector, and the function cannot end suspend-to-idle. Each vector is an AIC interrupt of its own (see apple_msi_domain_alloc()), so it can wake the system exactly when that AIC line can. Forward the wake setting down the hierarchy: give the bottom chip irq_chip_set_wake_parent(), and set the same callback on the per-device chip from an init_dev_msi_info() wrapper, as the GICv3 ITS does to adjust its device domains. With the AIC accepting wake requests ("irqchip/ apple-aic: Allow interrupts to wake the system"), enable_irq_wake() on a PCI MSI then succeeds, the vector stays enabled across suspend-to-idle and its next message wakes the system. Nothing is armed by this change alone: it only lets drivers that ask for a wake interrupt get one. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index bd065eaaeff4e..f21b225632cb7 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -472,6 +472,7 @@ static struct irq_chip apple_msi_bottom_chip = { .irq_eoi = irq_chip_eoi_parent, .irq_set_affinity = irq_chip_set_affinity_parent, .irq_set_type = irq_chip_set_type_parent, + .irq_set_wake = irq_chip_set_wake_parent, .irq_compose_msi_msg = apple_msi_compose_msg, }; @@ -1772,6 +1773,26 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, return ret; } +/* + * Each MSI vector is an AIC interrupt of its own, so a vector can wake the + * system exactly when the AIC line behind it can. Forward the wake setting + * from the per-device PCI/MSI chip down to the AIC, which otherwise fails + * enable_irq_wake() for every MSI and leaves PCIe functions without a way + * to end suspend-to-idle. + */ +static bool apple_msi_init_dev_msi_info(struct device *dev, + struct irq_domain *domain, + struct irq_domain *real_parent, + struct msi_domain_info *info) +{ + if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info)) + return false; + + if (!info->chip->irq_set_wake) + info->chip->irq_set_wake = irq_chip_set_wake_parent; + return true; +} + static const struct msi_parent_ops apple_msi_parent_ops = { .supported_flags = (MSI_GENERIC_FLAGS_MASK | MSI_FLAG_PCI_MSIX | @@ -1781,7 +1802,7 @@ static const struct msi_parent_ops apple_msi_parent_ops = { MSI_FLAG_PCI_MSI_MASK_PARENT), .chip_flags = MSI_CHIP_FLAG_SET_EOI, .bus_select_token = DOMAIN_BUS_PCI_MSI, - .init_dev_msi_info = msi_lib_init_dev_msi_info, + .init_dev_msi_info = apple_msi_init_dev_msi_info, }; static int apple_msi_init(struct apple_pcie *pcie) From 7d9f156ec2dd1ce4489a98d805ae37a3e37e18c4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:12:53 +1100 Subject: [PATCH 1142/1579] Bluetooth: mt7932: Keep the controller running for wakeup in suspend-to-idle A paired keyboard or mouse cannot wake the machine. When wakeup is enabled on the PCI function, hci_suspend_dev() sets up page scan and LE passive scan for the devices flagged for remote wakeup, but bt7932_suspend() then quiesces the firmware with the quiesce handshake and lets the PCI core put the function into D3hot, and nothing arms the interrupt. No connection request or advertising report from a wake device can reach the host. In suspend-to-idle with wakeup enabled and the controller up, leave the transport running and wake through its MSI instead: - Track wake reports from the start of the suspend callback, before waiting for queued HCI work and before hci_suspend_dev(). An event that the HCI core records as a remote wake reason (a BR/EDR connection request or completion, an LE advertising report while no discovery runs) is reported with pm_wakeup_dev_event(), so a report that arrives before the interrupt is suspended aborts the suspend rather than being consumed unnoticed. A Hardware Error does the same, so that HCI recovery runs on an awake system. Command replies and vendor events never count, so the suspend setup cannot abort itself. - Arm the MSI with enable_irq_wake(). If it cannot be armed, quiesce as before. - Block transmission until resume. HCI recovery, whose work is not frozen for system sleep and which would need to transmit, waits in the open callback until resume instead of failing its setup. - Save the PCI state in suspend_noirq, which keeps the PCI core from moving the function out of D0, and fail the suspend there if the transport faulted after it was armed. - Hold the interrupt off from resume_noirq until resume, after the PCI core has restored that state. - Whatever raises a terminal fault while that saved state is outstanding saves it again after clearing bus mastering, so that the PCI core never restores bus mastering for a failed transport. A suspend aborted before suspend_noirq has no saved state to restore. With wakeup disabled (the default), with the controller down, or for a user channel, suspend is unchanged. The firmware does not raise interrupts while idle (none in 60 s on an idle J700), so an armed MSI only fires for traffic the wake setup asked for. The wake signal is the function's MSI only; endpoint PME is not enabled on this path. Why the quiesce exists, and why skipping it here holds: "Bluetooth: mt7932: support modules and system suspend" quiesces the firmware so that the PCI power transitions of system sleep never meet a running firmware. This path makes no PCI power transition: the saved state leaves the function in D0 with bus mastering and the DMA arena untouched, the arrangement that "wifi: mt7932: keep the function running across suspend-to-idle" qualified for the Wi-Fi function after finding that clearing bus mastering under running firmware hangs the SoC. The PCIe link and host bridge stay up for that function anyway. The quiesce still runs whenever wakeup is off. ASPM and clock PM stay disabled on the link as the transport was admitted, which also keeps the link in L0 for the MSI. HCI is still suspended only from bt7932_suspend(), after the queued HCI work ("Bluetooth: mt7932: don't cancel HCI commands at suspend entry"). KUnit covers which received events end the sleep. This needs "PCI: apple: Let MSIs wake the system" for enable_irq_wake() to succeed. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/Kconfig | 5 +- drivers/bluetooth/mt7932_bt_pcie.c | 257 ++++++++++++++++++- drivers/bluetooth/mt7932_bt_pcie_wake_test.c | 81 ++++++ 3 files changed, 336 insertions(+), 7 deletions(-) create mode 100644 drivers/bluetooth/mt7932_bt_pcie_wake_test.c diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 2280f206d0b70..3a5769fe363e6 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -50,9 +50,10 @@ config BT_MTK7932_PCIE_KUNIT_TEST help Build KUnit tests into the MT7932 Bluetooth transport. They check, without radio hardware, that HCI packets whose length does not - match their header are rejected in both directions, and that a + match their header are rejected in both directions, that a board file with a bad layout or checksum is never sent to the - controller. + controller, and which received events end a system sleep with + Bluetooth wakeup enabled. If unsure, say N. diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index ff90ee6d43c75..43542ebfc115c 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -25,8 +25,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -139,6 +141,15 @@ struct bt7932 { bool enabled, regions, vectors, irq_requested, retained; bool ipc_ready, opened, awake, waking, fault, registered; bool stopping, runtime_forbidden; + /* + * Suspend-to-idle with the transport left running for wakeup, see + * bt7932_suspend(). Written under lock except irq_held. + */ + bool wake_tracking; /* wake reports become PM wakeup events */ + bool wake_armed; /* the MSI is armed for wakeup */ + bool tx_blocked; /* nothing is sent until resume */ + bool irq_held; /* IRQ off until the PCI state is restored */ + bool pci_saved; /* saved PCI state not yet restored */ enum bt7932_recovery recovery; enum bt7932_pm_state pm_state; }; @@ -181,6 +192,12 @@ static void bt7932_fault_locked(struct bt7932 *bt, const char *reason) bt7932_counter(bt, bt->host_transfer, channel), bt7932_counter(bt, bt->fw_transfer, channel), bt->rings[channel].previous); pci_clear_master(bt->pdev); + /* + * Whatever raised the fault, a saved state that the PCI core has yet + * to restore must not bring bus mastering back: save it again. + */ + if (bt->pci_saved) + pci_save_state(bt->pdev); if (bt->irq_requested) disable_irq_nosync(bt->irq); wake_up_all(&bt->wake_wait); @@ -222,6 +239,64 @@ static bool bt7932_hci_length(u8 packet, const u8 *data, size_t len) return len == expected; } +enum bt7932_wake { + BT7932_WAKE_NONE, + /* A remote device asks for the host. */ + BT7932_WAKE_REMOTE, + /* An LE advertising report: remote, unless discovery is running. */ + BT7932_WAKE_ADVERTISING, + /* The controller lost its state; HCI recovery is coming. */ + BT7932_WAKE_ERROR, +}; + +/* + * Classifies a received event for a system sleep with wakeup armed. The + * remote events are those that the HCI core records as a remote wake reason + * while suspended (see hci_store_wake_reason()): a BR/EDR connection request + * or completion, or an LE advertising report. A Hardware Error must end the + * sleep too, so that recovery runs on an awake system. Command Complete and + * Status, vendor events and the replies to the suspend setup never count. + * The caller has checked the length against the event header. + */ +static enum bt7932_wake bt7932_wake_event(const u8 *data, size_t len) +{ + switch (data[0]) { + case HCI_EV_CONN_COMPLETE: + case HCI_EV_CONN_REQUEST: + case HCI_EV_SYNC_CONN_COMPLETE: + return BT7932_WAKE_REMOTE; + case HCI_EV_HARDWARE_ERROR: + return BT7932_WAKE_ERROR; + case HCI_EV_LE_META: + if (len < HCI_EVENT_HDR_SIZE + 1) + return BT7932_WAKE_NONE; + switch (data[HCI_EVENT_HDR_SIZE]) { + case HCI_EV_LE_ADVERTISING_REPORT: + case HCI_EV_LE_DIRECT_ADV_REPORT: + case HCI_EV_LE_EXT_ADV_REPORT: + return BT7932_WAKE_ADVERTISING; + } + return BT7932_WAKE_NONE; + default: + return BT7932_WAKE_NONE; + } +} + +/* Caller owns lock and has found wake tracking on. */ +static bool bt7932_wake_report(struct bt7932 *bt, const u8 *data, size_t len) +{ + switch (bt7932_wake_event(data, len)) { + case BT7932_WAKE_REMOTE: + case BT7932_WAKE_ERROR: + return true; + case BT7932_WAKE_ADVERTISING: + /* Discovery is paused once HCI suspends; until then it is not a wake. */ + return READ_ONCE(bt->hdev->discovery.state) == DISCOVERY_STOPPED; + default: + return false; + } +} + static int bt7932_validate_ptx(const u8 *data, size_t size) { static const u32 types[] = { 0x101, 0x201, 0x301, 0x401 }; @@ -485,7 +560,7 @@ static int bt7932_wake_and_lock(struct bt7932 *bt, bool require_open) for (;;) { mutex_lock(&bt->lock); if (bt->fault || bt->stopping || bt->pm_state != BT7932_RUNNING || - (require_open && !bt->opened)) { + bt->tx_blocked || (require_open && !bt->opened)) { mutex_unlock(&bt->lock); return -EHOSTDOWN; } @@ -627,6 +702,15 @@ static int bt7932_rx_locked(struct bt7932 *bt, unsigned int channel) hci_skb_pkt_type(skb) = geo->packet; skb_put_data(skb, descriptor + 16, length); bt->hdev->stat.byte_rx += length; + /* + * From the start of bt7932_suspend() until resume, a + * wake report ends the system sleep that is under way: + * before the interrupt is suspended, nothing else would + * remember that it arrived. + */ + if (geo->packet == HCI_EVENT_PKT && bt->wake_tracking && + bt7932_wake_report(bt, descriptor + 16, length)) + pm_wakeup_dev_event(&bt->pdev->dev, 0, true); hci_recv_frame(bt->hdev, skb); } slot = (slot + 1) & BT7932_MASK; @@ -829,6 +913,14 @@ static int bt7932_open(struct hci_dev *hdev) struct bt7932 *bt = hci_get_drvdata(hdev); int ret = 0; + /* + * The HCI core's recovery work is not frozen for system sleep. While + * the transport sleeps with wakeup armed nothing may be sent, so a + * recovery that reopens the device waits here for resume rather than + * failing its setup and faulting the transport. bt7932_resume() + * unblocks it before it resumes HCI, which waits for this open. + */ + wait_event(bt->wake_wait, !READ_ONCE(bt->tx_blocked)); mutex_lock(&bt->lock); if (bt->fault || !bt->ipc_ready || bt->stopping || bt->pm_state != BT7932_RUNNING) @@ -868,7 +960,8 @@ static int bt7932_send(struct hci_dev *hdev, struct sk_buff *skb) int ret; if (READ_ONCE(bt->fault) || !READ_ONCE(bt->opened) || - READ_ONCE(bt->stopping) || READ_ONCE(bt->pm_state) != BT7932_RUNNING) + READ_ONCE(bt->stopping) || READ_ONCE(bt->tx_blocked) || + READ_ONCE(bt->pm_state) != BT7932_RUNNING) return -EHOSTDOWN; if (channel < 0 || skb->len > geometry[channel].payload) return -EMSGSIZE; @@ -1015,8 +1108,8 @@ static void bt7932_hw_error(struct hci_dev *hdev, u8 code) /* * Follows the wakeup setting of the PCI function, which is off by default. * Only while it is on does hci_suspend_sync() leave page scan and LE passive - * scan set up for the devices flagged for remote wakeup. The transport is - * quiesced for system sleep either way, and no interrupt is armed for wakeup. + * scan set up for the devices flagged for remote wakeup, and only then does + * bt7932_suspend() keep the transport running with its interrupt armed. */ static bool bt7932_wakeup(struct hci_dev *hdev) { @@ -1368,20 +1461,133 @@ static bool bt7932_hci_pm(struct bt7932 *bt) return !hci_dev_test_flag(bt->hdev, HCI_USER_CHANNEL); } +/* + * Suspend-to-idle can keep the transport running so that the controller can + * wake the system: with wakeup enabled on the PCI function, the controller + * up and HCI suspended by this driver. + */ +static bool bt7932_may_wake(struct bt7932 *bt) +{ + return device_may_wakeup(&bt->pdev->dev) && pm_suspend_no_platform() && + bt7932_hci_pm(bt) && test_bit(HCI_UP, &bt->hdev->flags); +} + +static void bt7932_track_wake(struct bt7932 *bt, bool on) +{ + mutex_lock(&bt->lock); + bt->wake_tracking = on; + mutex_unlock(&bt->lock); +} + +/* + * Leave the firmware running with its MSI armed. It keeps the scans that + * hci_suspend_dev() set up and reports a match like any other event. Returns + * 0 when armed, -EAGAIN when the transport should be quiesced instead, or + * -EIO when it failed. + */ +static int bt7932_arm_wakeup(struct bt7932 *bt) +{ + struct device *dev = &bt->pdev->dev; + int ret; + + ret = enable_irq_wake(bt->irq); + if (ret) { + dev_warn(dev, "cannot arm the interrupt for wakeup: %d\n", ret); + return -EAGAIN; + } + + mutex_lock(&bt->lock); + ret = bt->fault ? -EIO : 0; + if (!ret) { + bt->wake_armed = true; + /* A failed wake handshake would fault the transport. */ + bt->tx_blocked = true; + } + mutex_unlock(&bt->lock); + if (ret) { + disable_irq_wake(bt->irq); + return ret; + } + /* Drop what HCI queued after its suspend, as the quiesce does. */ + cancel_work_sync(&bt->tx_work); + skb_queue_purge(&bt->tx_queue); + return 0; +} + +/* + * Ends the armed state on resume, or after a suspend that was aborted. The + * interrupt is still disabled if bt7932_resume_noirq() held it. + */ +static void bt7932_disarm_wakeup(struct bt7932 *bt) +{ + disable_irq_wake(bt->irq); + mutex_lock(&bt->lock); + bt->wake_armed = false; + bt->tx_blocked = false; + bt->wake_tracking = false; + bt->pci_saved = false; + mutex_unlock(&bt->lock); + wake_up_all(&bt->wake_wait); + if (bt->irq_held) { + bt->irq_held = false; + enable_irq(bt->irq); + } +} + +/* + * With wakeup enabled, the transport stays up through suspend-to-idle + * instead of being quiesced: + * + * - Wake tracking starts before anything else, so that a wake report or a + * Hardware Error is never consumed unnoticed (bt7932_rx_locked()). + * - bt7932_arm_wakeup() arms the MSI and blocks TX. HCI recovery, which + * would need TX, waits in bt7932_open() until resume. + * - bt7932_suspend_noirq() saves the PCI state, which keeps the PCI core + * from moving the function out of D0, once the handler can no longer + * run, and fails the suspend if the transport faulted in the meantime. + * - bt7932_resume_noirq() holds the interrupt off until the PCI core has + * restored that state and called bt7932_resume(). + * - A terminal fault, from whatever source, saves the state again with + * bus mastering clear while it is outstanding, so restoring it never + * brings bus mastering back for a failed transport. + * + * The PCIe link and host bridge stay up in suspend-to-idle anyway, as the + * Wi-Fi function on the same link is never stopped, and ASPM stays off on + * this link as the transport was admitted. + */ static int bt7932_suspend(struct device *dev) { struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); + bool wake; int ret; if (READ_ONCE(bt->fault)) return -EIO; if (!bt->registered) return 0; + wake = bt7932_may_wake(bt); + if (wake) + bt7932_track_wake(bt, true); if (bt7932_hci_pm(bt)) { bt7932_wait_hci_idle(bt); ret = hci_suspend_dev(bt->hdev); - if (ret) + if (ret) { + bt7932_track_wake(bt, false); + return ret; + } + } + if (wake) { + ret = bt7932_arm_wakeup(bt); + if (!ret) { + dev_dbg(dev, "system sleep with wakeup armed\n"); + return 0; + } + bt7932_track_wake(bt, false); + if (ret != -EAGAIN) { + dev_err(dev, "transport failed while arming wakeup\n"); + hci_resume_dev(bt->hdev); return ret; + } } ret = bt7932_quiesce(bt); if (!ret) { @@ -1397,11 +1603,49 @@ static int bt7932_suspend(struct device *dev) return ret; } +static int bt7932_suspend_noirq(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct bt7932 *bt = pci_get_drvdata(pdev); + int ret = 0; + + if (!bt->wake_armed) + return 0; + mutex_lock(&bt->lock); + if (bt->fault) { + ret = -EIO; + } else { + pci_save_state(pdev); + bt->pci_saved = true; + } + mutex_unlock(&bt->lock); + if (ret) + dev_err(dev, "transport failed while suspending with wakeup armed\n"); + return ret; +} + +static int bt7932_resume_noirq(struct device *dev) +{ + struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); + + if (bt->wake_armed) { + disable_irq(bt->irq); + bt->irq_held = true; + } + return 0; +} + static int bt7932_resume(struct device *dev) { struct bt7932 *bt = pci_get_drvdata(to_pci_dev(dev)); int ret; + if (bt->wake_armed) { + bt7932_disarm_wakeup(bt); + if (READ_ONCE(bt->fault)) + return -EIO; + return bt7932_hci_pm(bt) ? hci_resume_dev(bt->hdev) : 0; + } if (READ_ONCE(bt->fault)) return -EIO; if (!bt->registered) @@ -1425,6 +1669,8 @@ static int bt7932_freeze(struct device *dev) static const struct dev_pm_ops bt7932_pm_ops = { .suspend = bt7932_suspend, + .suspend_noirq = bt7932_suspend_noirq, + .resume_noirq = bt7932_resume_noirq, .resume = bt7932_resume, .freeze = bt7932_freeze, .thaw = bt7932_resume, @@ -1495,4 +1741,5 @@ MODULE_FIRMWARE(BT7932_ADDR); #if IS_ENABLED(CONFIG_BT_MTK7932_PCIE_KUNIT_TEST) #include "mt7932_bt_pcie_test.c" +#include "mt7932_bt_pcie_wake_test.c" #endif diff --git a/drivers/bluetooth/mt7932_bt_pcie_wake_test.c b/drivers/bluetooth/mt7932_bt_pcie_wake_test.c new file mode 100644 index 0000000000000..61e93ce1f31db --- /dev/null +++ b/drivers/bluetooth/mt7932_bt_pcie_wake_test.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for how mt7932_bt_pcie classifies received events while the + * system suspends with wakeup armed. Built as part of mt7932_bt_pcie.c, + * so the tests reach its static helpers. + */ +#include + +static void bt7932_wake_event_bredr_test(struct kunit *test) +{ + /* Connection Request: bdaddr, class of device, link type. */ + static const u8 request[] = { HCI_EV_CONN_REQUEST, 10, + 1, 2, 3, 4, 5, 6, 0x40, 0x25, 0x00, 0x01 }; + static const u8 complete[] = { HCI_EV_CONN_COMPLETE, 11, 0x00, 0x01, 0x00, + 1, 2, 3, 4, 5, 6, 0x01, 0x00 }; + static const u8 sync[] = { HCI_EV_SYNC_CONN_COMPLETE, 1, 0x00 }; + + KUNIT_EXPECT_EQ(test, bt7932_wake_event(request, sizeof(request)), BT7932_WAKE_REMOTE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(complete, sizeof(complete)), BT7932_WAKE_REMOTE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(sync, sizeof(sync)), BT7932_WAKE_REMOTE); +} + +static void bt7932_wake_event_le_test(struct kunit *test) +{ + static const u8 adv[] = { HCI_EV_LE_META, 2, HCI_EV_LE_ADVERTISING_REPORT, 1 }; + static const u8 direct[] = { HCI_EV_LE_META, 1, HCI_EV_LE_DIRECT_ADV_REPORT }; + static const u8 ext[] = { HCI_EV_LE_META, 1, HCI_EV_LE_EXT_ADV_REPORT }; + /* LE Connection Complete and Read Remote Features Complete. */ + static const u8 conn[] = { HCI_EV_LE_META, 1, HCI_EV_LE_CONN_COMPLETE }; + static const u8 features[] = { HCI_EV_LE_META, 1, 0x04 }; + static const u8 no_subevent[] = { HCI_EV_LE_META, 0 }; + + KUNIT_EXPECT_EQ(test, bt7932_wake_event(adv, sizeof(adv)), BT7932_WAKE_ADVERTISING); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(direct, sizeof(direct)), BT7932_WAKE_ADVERTISING); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(ext, sizeof(ext)), BT7932_WAKE_ADVERTISING); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(conn, sizeof(conn)), BT7932_WAKE_NONE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(features, sizeof(features)), BT7932_WAKE_NONE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(no_subevent, sizeof(no_subevent)), + BT7932_WAKE_NONE); +} + +/* The replies to the commands that set up suspend must never end it. */ +static void bt7932_wake_event_replies_test(struct kunit *test) +{ + /* Command Complete for LE Set Scan Enable, Command Status. */ + static const u8 cc[] = { HCI_EV_CMD_COMPLETE, 4, 0x01, 0x0c, 0x20, 0x00 }; + static const u8 cs[] = { HCI_EV_CMD_STATUS, 4, 0x00, 0x01, 0x0d, 0x20 }; + static const u8 disconn[] = { HCI_EV_DISCONN_COMPLETE, 4, 0x00, 0x01, 0x00, 0x15 }; + /* Vendor events, including the MediaTek vendor channel reply. */ + static const u8 vendor[] = { HCI_EV_VENDOR, 1, 0x00 }; + static const u8 mtk[] = { 0xe4, 5, 0x02, 0x01, 0x01, 0x00, 0x00 }; + + KUNIT_EXPECT_EQ(test, bt7932_wake_event(cc, sizeof(cc)), BT7932_WAKE_NONE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(cs, sizeof(cs)), BT7932_WAKE_NONE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(disconn, sizeof(disconn)), BT7932_WAKE_NONE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(vendor, sizeof(vendor)), BT7932_WAKE_NONE); + KUNIT_EXPECT_EQ(test, bt7932_wake_event(mtk, sizeof(mtk)), BT7932_WAKE_NONE); +} + +/* A Hardware Error ends the sleep so that recovery runs awake. */ +static void bt7932_wake_event_error_test(struct kunit *test) +{ + static const u8 error[] = { HCI_EV_HARDWARE_ERROR, 1, 0x00 }; + + KUNIT_EXPECT_EQ(test, bt7932_wake_event(error, sizeof(error)), BT7932_WAKE_ERROR); +} + +static struct kunit_case bt7932_wake_test_cases[] = { + KUNIT_CASE(bt7932_wake_event_bredr_test), + KUNIT_CASE(bt7932_wake_event_le_test), + KUNIT_CASE(bt7932_wake_event_replies_test), + KUNIT_CASE(bt7932_wake_event_error_test), + {} +}; + +static struct kunit_suite bt7932_wake_test_suite = { + .name = "mt7932-bt-wake", + .test_cases = bt7932_wake_test_cases, +}; + +kunit_test_suite(bt7932_wake_test_suite); From 7bfd6f96aa12422914c204b342b569d42fa4eedf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:12:55 +1100 Subject: [PATCH 1143/1579] Bluetooth: mt7932: Tell the controller that the host sleeps When it suspends the system, macOS sends the controller one vendor command, opcode 0xfc6f with seven 0xff parameter bytes, before the PCI function changes power state, and sends nothing when the system wakes (observed in our research of the J700 macOS Bluetooth transport; no code was taken from it). 0xfc6f is the MediaTek vendor channel that btmtk, btmtksdio and btmtkuart also use; those drivers wait for a vendor event in reply, so it is not answered by Command Complete. Send the same command when the transport is left running for wakeup, after hci_suspend_dev() has set up the wake scans and before the interrupt is armed. Place it on the command ring directly, outside the HCI command flow, and wait up to 500 ms until the controller has taken it from the ring. The ring publication is factored out of the TX work for this, so that the command has a result: - Nothing was published (the controller could not be reached without a fault, or the ring was full): wakeup is not armed and the transport is quiesced as without wakeup. - Published but not taken in time: the command may still take effect, so the quiesce cannot be relied on. The suspend fails, HCI is resumed and the controller is reset through HCI recovery. - The transport faulted: the suspend fails. The response contract is not established: any reply is a vendor event, which is never treated as a wake report, but one that arrived after the interrupt is suspended would wake the system. The quiesced path is left alone. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 141 +++++++++++++++++++++++------ 1 file changed, 113 insertions(+), 28 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 43542ebfc115c..725833f8ecc51 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -62,6 +62,9 @@ #define BT7932_CONTROL_POOL_SIZE (BT7932_DEPTH * BT7932_CONTROL_SIZE) #define BT7932_COMPLETION_SIZE (BT7932_DEPTH * 0x18) #define BT7932_TX_QUEUE_MAX 128 +/* Vendor command telling the controller that the host goes to sleep. */ +#define BT7932_OP_HOST_SLEEP 0xfc6f +#define BT7932_HOST_SLEEP_PLEN 7 #define BT7932_FW_B0 "mediatek/MT7932B0_OS_TypeB_0.1.44.0_241001003711.bin" #define BT7932_FW_B1 "mediatek/MT7932B1_OS_TypeB_2.0.177.0_260706180356.bin" @@ -616,51 +619,64 @@ static int bt7932_tx_progress(struct bt7932 *bt, unsigned int channel) return 0; } +/* + * Caller owns lock with the firmware awake. Returns -ENOSPC while the ring is + * full, or -EPROTO after faulting the transport. + */ +static int bt7932_publish_locked(struct bt7932 *bt, unsigned int channel, + const u8 *data, size_t len) +{ + struct bt7932_ring *ring = &bt->rings[channel]; + u8 *descriptor; + + if (bt7932_tx_progress(bt, channel)) { + bt7932_fault_locked(bt, "TX consumer exceeded outstanding descriptors"); + return -EPROTO; + } + if (((ring->host + 1) & BT7932_MASK) == ring->previous) + return -ENOSPC; + descriptor = bt->arena + ring->offset + ring->host * geometry[channel].stride; + memset(descriptor, 0, geometry[channel].stride); + descriptor[0] = 2; + descriptor[1] = len; + descriptor[2] = len >> 8; + descriptor[3] = len >> 16; + put_unaligned_le16(ring->host, descriptor + 12); + descriptor[15] = 2; + memcpy(descriptor + 16, data, len); + ring->host = (ring->host + 1) & BT7932_MASK; + bt7932_publish_counter(bt, bt->host_transfer, channel, ring->host); + bt7932_ring_doorbell(bt, geometry[channel].doorbell); + bt->hdev->stat.byte_tx += len; + return 0; +} + static void bt7932_tx_work(struct work_struct *work) { struct bt7932 *bt = container_of(work, struct bt7932, tx_work); struct sk_buff *skb; + int ret; while ((skb = skb_dequeue(&bt->tx_queue))) { int channel = bt7932_tx_channel(hci_skb_pkt_type(skb)); - struct bt7932_ring *ring; - u16 consumer; - u8 *descriptor; if (bt7932_wake_and_lock(bt, true)) { kfree_skb(skb); skb_queue_purge(&bt->tx_queue); return; } - ring = &bt->rings[channel]; - if (bt7932_tx_progress(bt, channel)) { - bt7932_fault_locked(bt, "TX consumer exceeded outstanding descriptors"); - mutex_unlock(&bt->lock); - kfree_skb(skb); - skb_queue_purge(&bt->tx_queue); - return; - } - consumer = ring->previous; - if (((ring->host + 1) & BT7932_MASK) == consumer) { + ret = bt7932_publish_locked(bt, channel, skb->data, skb->len); + if (ret == -ENOSPC) { skb_queue_head(&bt->tx_queue, skb); mutex_unlock(&bt->lock); return; } - descriptor = bt->arena + ring->offset + ring->host * geometry[channel].stride; - memset(descriptor, 0, geometry[channel].stride); - descriptor[0] = 2; - descriptor[1] = skb->len; - descriptor[2] = skb->len >> 8; - descriptor[3] = skb->len >> 16; - put_unaligned_le16(ring->host, descriptor + 12); - descriptor[15] = 2; - memcpy(descriptor + 16, skb->data, skb->len); - ring->host = (ring->host + 1) & BT7932_MASK; - bt7932_publish_counter(bt, bt->host_transfer, channel, ring->host); - bt7932_ring_doorbell(bt, geometry[channel].doorbell); - bt->hdev->stat.byte_tx += skb->len; mutex_unlock(&bt->lock); kfree_skb(skb); + if (ret) { + skb_queue_purge(&bt->tx_queue); + return; + } } } @@ -1479,11 +1495,62 @@ static void bt7932_track_wake(struct bt7932 *bt, bool on) mutex_unlock(&bt->lock); } +/* + * Before system sleep, the host tells the controller that it is going to + * sleep with a vendor command whose parameters are all ones; nothing is sent + * when the host is back. It is answered by a vendor event, if at all, so + * place it on the command ring directly rather than through the HCI command + * flow, and wait until the controller has taken it. + * + * Returns -EAGAIN if nothing was published (the controller could not be + * reached without a fault, or the ring was full), -EBUSY if the command was + * published but not taken in time, so that it may still take effect, or + * -EIO if the transport faulted. + */ +static int bt7932_host_sleep(struct bt7932 *bt) +{ + u8 cmd[HCI_COMMAND_HDR_SIZE + BT7932_HOST_SLEEP_PLEN]; + unsigned long deadline; + u16 target; + int ret; + + put_unaligned_le16(BT7932_OP_HOST_SLEEP, cmd); + cmd[2] = BT7932_HOST_SLEEP_PLEN; + memset(cmd + HCI_COMMAND_HDR_SIZE, 0xff, BT7932_HOST_SLEEP_PLEN); + + ret = bt7932_wake_and_lock(bt, true); + if (ret) + return READ_ONCE(bt->fault) ? -EIO : -EAGAIN; + ret = bt7932_publish_locked(bt, 1, cmd, sizeof(cmd)); + target = bt->rings[1].host; + mutex_unlock(&bt->lock); + if (ret == -ENOSPC) + return -EAGAIN; + if (ret) + return -EIO; + + deadline = jiffies + msecs_to_jiffies(500); + for (;;) { + mutex_lock(&bt->lock); + ret = bt->fault ? -EIO : 0; + if (!ret && bt7932_counter(bt, bt->fw_transfer, 1) == target) { + mutex_unlock(&bt->lock); + return 0; + } + mutex_unlock(&bt->lock); + if (ret) + return ret; + if (time_after(jiffies, deadline)) + return -EBUSY; + usleep_range(1000, 2000); + } +} + /* * Leave the firmware running with its MSI armed. It keeps the scans that * hci_suspend_dev() set up and reports a match like any other event. Returns - * 0 when armed, -EAGAIN when the transport should be quiesced instead, or - * -EIO when it failed. + * 0 when armed, -EAGAIN when the transport should be quiesced instead, + * -EBUSY when the controller state is uncertain, or -EIO when it failed. */ static int bt7932_arm_wakeup(struct bt7932 *bt) { @@ -1496,6 +1563,13 @@ static int bt7932_arm_wakeup(struct bt7932 *bt) return -EAGAIN; } + ret = bt7932_host_sleep(bt); + if (ret) { + disable_irq_wake(bt->irq); + dev_warn(dev, "host sleep command failed: %d\n", ret); + return ret; + } + mutex_lock(&bt->lock); ret = bt->fault ? -EIO : 0; if (!ret) { @@ -1583,6 +1657,17 @@ static int bt7932_suspend(struct device *dev) return 0; } bt7932_track_wake(bt, false); + if (ret == -EBUSY) { + /* + * The host sleep command may still take effect, so the + * quiesce cannot be relied on. Fail this suspend and + * reset the controller through HCI recovery. + */ + dev_err(dev, "host sleep command not taken; resetting the controller\n"); + hci_resume_dev(bt->hdev); + bt7932_reset(bt->hdev); + return ret; + } if (ret != -EAGAIN) { dev_err(dev, "transport failed while arming wakeup\n"); hci_resume_dev(bt->hdev); From d53c1cd749777fd26e052a34298bacf77a182689 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:13:09 +1100 Subject: [PATCH 1144/1579] tools: aurora-wake: Add Bluetooth wake tests Tests for Bluetooth wake from suspend-to-idle on the J700: - bt-wake-cycle.sh: one RTC-backstopped sleep with wakeup of the Bluetooth PCI function enabled or disabled; checks that the driver armed (or did not arm) its interrupt, that the sleep was ended by the Bluetooth MSI (or by the RTC with wakeup disabled), that the controller is up and a given device is connected again after resume, and the kernel log. - attended-bt-wake.sh: the same with a real keyboard and a person pressing a key, about two minutes. - bt-wake-power.sh: alternating sleeps with wakeup disabled and enabled, recording Bluetooth and total interrupts, s2idle loop re-entries and battery energy. - hog-keyboard.py: an emulated BLE HID-over-GATT keyboard on another BlueZ host that advertises only when told to "press a key", like an idle keyboard, so that the wake test can run unattended. - bt-wake-emulated.sh: driver for the unattended run: pairs the emulator with the system under test, runs wake-enabled and wake-disabled cycles with an emulated key press during each sleep, and removes the pairing. Hosts and Bluetooth addresses come from a local configuration file outside the tree. Not for upstream. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-wake/attended-bt-wake.sh | 44 ++++ tools/aurora-wake/bt-wake-cycle.sh | 155 ++++++++++++ tools/aurora-wake/bt-wake-emulated.sh | 138 +++++++++++ tools/aurora-wake/bt-wake-power.sh | 57 +++++ tools/aurora-wake/hog-keyboard.py | 332 ++++++++++++++++++++++++++ 5 files changed, 726 insertions(+) create mode 100755 tools/aurora-wake/attended-bt-wake.sh create mode 100755 tools/aurora-wake/bt-wake-cycle.sh create mode 100755 tools/aurora-wake/bt-wake-emulated.sh create mode 100755 tools/aurora-wake/bt-wake-power.sh create mode 100755 tools/aurora-wake/hog-keyboard.py diff --git a/tools/aurora-wake/attended-bt-wake.sh b/tools/aurora-wake/attended-bt-wake.sh new file mode 100755 index 0000000000000..f8afc8ac87269 --- /dev/null +++ b/tools/aurora-wake/attended-bt-wake.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Attended Bluetooth keyboard wake check (J700), about two minutes, as root. +# +# Usage: attended-bt-wake.sh +# is a paired keyboard with WakeAllowed set (bluetoothctl wake on). +# +# 1. Press a key on the keyboard so that it connects; the script waits for it. +# 2. At SUSPENDING, wait for the screen to go dark, count to ten, and press one +# key on the keyboard. The RTC ends the sleep after 60 s otherwise. +set -u + +addr=${1:-} +[[ $addr =~ ^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$ ]] || + { echo "usage: $0 " >&2; exit 2; } +here=$(dirname "$(readlink -f "$0")") +out=/var/tmp/bt-wake/attended-$(date +%H%M%S) +mkdir -p "$out" + +bluetoothctl info "$addr" > "$out/device.txt" 2>&1 || + { echo "$addr is not known to BlueZ" >&2; exit 2; } +grep -E "Name|Paired|WakeAllowed|Connected" "$out/device.txt" +grep -q "WakeAllowed: yes" "$out/device.txt" || + { echo "wake is not allowed: bluetoothctl wake $addr on" >&2; exit 2; } + +echo ">>> Press a key on the Bluetooth keyboard so that it connects (60 s)." +for _ in $(seq 1 120); do + bluetoothctl info "$addr" | grep -q "Connected: yes" && break + sleep 0.5 +done +bluetoothctl info "$addr" | grep -q "Connected: yes" || + { echo "the keyboard did not connect" >&2; exit 2; } +echo "connected." + +echo ">>> SUSPENDING. When the screen is dark, count to ten and press ONE key." +"$here/bt-wake-cycle.sh" -w on -s 60 -a "$addr" -o "$out/cycle" +rc=$? +if [ $rc = 0 ]; then + echo "VERDICT: PASS (the Bluetooth keyboard woke the system)" +else + echo "VERDICT: FAIL (see $out/cycle/verdict.txt)" +fi +exit $rc diff --git a/tools/aurora-wake/bt-wake-cycle.sh b/tools/aurora-wake/bt-wake-cycle.sh new file mode 100755 index 0000000000000..3c10162256e97 --- /dev/null +++ b/tools/aurora-wake/bt-wake-cycle.sh @@ -0,0 +1,155 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# One suspend-to-idle cycle for the MT7932 Bluetooth wake test (J700), as root. +# +# Sets power/wakeup of the Bluetooth PCI function, suspends with an RTC alarm as +# a backstop and records what ended the sleep. Something else (a person, or an +# emulated keyboard on another machine) is expected to make a wake-enabled +# Bluetooth HID device reconnect while the system sleeps. +# +# Usage: bt-wake-cycle.sh -w on|off [-s backstop-secs] [-r rtc] [-a bdaddr] [-o outdir] +# -w on expect a Bluetooth wake well before the backstop +# -w off expect no wake: the RTC ends the sleep +# -a Bluetooth address that must be connected again after resume +# The file /suspending appears right before the system suspends. +# Exit status: 0 PASS, 1 FAIL, 2 setup error. + +set -u + +mode= +secs=120 +rtc=rtc1 +addr= +out=/var/tmp/bt-wake/cycle-$(date +%Y%m%d-%H%M%S) +bdf=0000:01:00.1 + +while getopts "w:s:r:a:o:" opt; do + case $opt in + w) mode=$OPTARG ;; + s) secs=$OPTARG ;; + r) rtc=$OPTARG ;; + a) addr=$OPTARG ;; + o) out=$OPTARG ;; + *) exit 2 ;; + esac +done +case $mode in +on|off) ;; +*) echo "usage: $0 -w on|off [-s secs] [-r rtc] [-a bdaddr] [-o outdir]" >&2; exit 2 ;; +esac +[ "$(id -u)" = 0 ] || { echo "must run as root" >&2; exit 2; } +dev=/sys/bus/pci/devices/$bdf +[ -f "$dev/power/wakeup" ] || { echo "$bdf has no power/wakeup" >&2; exit 2; } + +mkdir -p "$out" +irq=$(awk -F: '/mt7932-bt/ { gsub(/ /, "", $1); print $1; exit }' /proc/interrupts) +[ -n "$irq" ] || { echo "no mt7932-bt interrupt" >&2; exit 2; } +# Recorded only: a PME would arrive on the root port's PME interrupt. +pme=$(awk -F: '/PCI-MSI-0000:00:00.0.*PME/ { gsub(/ /, "", $1); print $1; exit }' /proc/interrupts) + +bt_irqs() { + awk -v i="$irq" '$1 == i ":" { + for (k = 2; k <= NF; k++) if ($k ~ /^[0-9]+$/) s += $k; else break + } END { print s + 0 }' /proc/interrupts +} + +orig_wakeup=$(cat "$dev/power/wakeup") +restore() { echo "$orig_wakeup" > "$dev/power/wakeup"; } +trap restore EXIT +[ "$mode" = on ] && echo enabled > "$dev/power/wakeup" || echo disabled > "$dev/power/wakeup" + +echo 'module mt7932_bt_pcie +p' > /sys/kernel/debug/dynamic_debug/control 2> /dev/null +echo 1 > /sys/power/pm_debug_messages + +{ + echo "mode=$mode secs=$secs rtc=$rtc addr=$addr irq=$irq" + uname -r + echo "wakeup=$(cat "$dev/power/wakeup")" \ + "wakeup_count=$(cat "$dev/power/wakeup_count" 2>/dev/null)" + echo "irq_wakeup=$(cat /sys/kernel/irq/"$irq"/wakeup 2>/dev/null)" + echo "bt_irqs=$(bt_irqs)" + bluetoothctl show | grep -E "Powered|PowerState" + [ -n "$addr" ] && bluetoothctl info "$addr" | grep -E "Name|Connected|WakeAllowed" +} > "$out/before.txt" 2>&1 +fail_before=$(cat /sys/power/suspend_stats/fail) +dmesg -C +# HCI trace of the whole cycle: what the host sent at suspend, what woke it. +btmon -w "$out/hci.btsnoop" > /dev/null 2>&1 & +btmon_pid=$! +sleep 1 +sync + +date +%s.%N > "$out/suspending" +t0=$(date +%s.%N) +rtcwake -d "$rtc" -m freeze -s "$secs" > "$out/rtcwake.txt" 2>&1 +rc=$? +t1=$(date +%s.%N) +slept=$(awk -v a="$t0" -v b="$t1" 'BEGIN { printf "%.1f", b - a }') +wirq=$(cat /sys/power/pm_wakeup_irq 2>&1) +irqs_after=$(bt_irqs) +wsrc=other +[ "$wirq" = "$irq" ] && wsrc=bt-msi +[ -n "$pme" ] && [ "$wirq" = "$pme" ] && wsrc=root-port-pme + +# Bluetooth must come back: the controller up and, if given, the device connected. +reconnect=n/a +if [ -n "$addr" ]; then + reconnect=timeout + for i in $(seq 1 60); do + if bluetoothctl info "$addr" 2> /dev/null | grep -q "Connected: yes"; then + reconnect="${i}x0.5s" + break + fi + sleep 0.5 + done +fi +powered=$(bluetoothctl show | awk '/Powered:/ { print $2 }') +kill "$btmon_pid" 2> /dev/null + +dmesg > "$out/dmesg.txt" +{ + echo "rc=$rc slept=${slept}s pm_wakeup_irq=$wirq ($wsrc) bt_irq=$irq pme_irq=$pme" + echo "bt_irqs_after=$irqs_after wakeup_count=$(cat "$dev/power/wakeup_count" 2>/dev/null)" + echo "irq_wakeup_after=$(cat /sys/kernel/irq/"$irq"/wakeup 2>/dev/null)" + echo "reconnect=$reconnect powered=$powered" + grep -E "mt7932|Bluetooth|hci0" "$out/dmesg.txt" | grep -vE "Timekeeping|s2idle" | tail -20 +} > "$out/after.txt" + +reasons=() +[ $rc = 0 ] || reasons+=("rtcwake exited $rc") +[ "$(cat /sys/power/suspend_stats/fail)" = "$fail_before" ] || + reasons+=("suspend_stats fail count rose") +grep -q "system sleep with wakeup armed" "$out/dmesg.txt" +armed=$? +if [ "$mode" = on ]; then + [ $armed = 0 ] || reasons+=("driver did not report wakeup armed") + # The driver wakes through the function's MSI only; a root port PME is + # recorded but is not the designed path. + [ "$wsrc" = bt-msi ] || + reasons+=("pm_wakeup_irq $wirq ($wsrc) is not the Bluetooth MSI $irq") + awk -v s="$slept" -v w="$secs" 'BEGIN { exit !(s < w - 10) }' || + reasons+=("slept ${slept}s of ${secs}s") +else + [ $armed != 0 ] || reasons+=("wakeup armed although disabled") + awk -v s="$slept" -v w="$secs" 'BEGIN { exit !(s + 1.5 >= w) }' || + reasons+=("woke early after ${slept}s (pm_wakeup_irq $wirq)") +fi +[ "$powered" = yes ] || reasons+=("controller not powered after resume") +[ "$reconnect" != timeout ] || reasons+=("$addr did not reconnect within 30 s") +bad='WARNING:|BUG:|Oops|Call trace:|terminal transport fault|failed to (suspend|resume)' +grep -E "$bad" "$out/dmesg.txt" > "$out/bad-lines.txt" +[ -s "$out/bad-lines.txt" ] && + reasons+=("kernel log errors: $(head -2 "$out/bad-lines.txt" | tr '\n' ' ')") + +if [ ${#reasons[@]} = 0 ]; then + echo "VERDICT wake=$mode: PASS slept=${slept}s pm_wakeup_irq=$wirq ($wsrc)" \ + "reconnect=$reconnect" | tee "$out/verdict.txt" + status=0 +else + echo "VERDICT wake=$mode: FAIL slept=${slept}s pm_wakeup_irq=$wirq" | tee "$out/verdict.txt" + printf ' - %s\n' "${reasons[@]}" | tee -a "$out/verdict.txt" + status=1 +fi +echo "evidence: $out" | tee -a "$out/verdict.txt" +exit $status diff --git a/tools/aurora-wake/bt-wake-emulated.sh b/tools/aurora-wake/bt-wake-emulated.sh new file mode 100755 index 0000000000000..64689be5bd715 --- /dev/null +++ b/tools/aurora-wake/bt-wake-emulated.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Unattended Bluetooth wake test with an emulated BLE HID keyboard +# (hog-keyboard.py) on a second BlueZ host. Runs on a third machine that can +# reach both over SSH. +# +# bt-wake-emulated.sh pair pair the emulator with the target +# bt-wake-emulated.sh run [N] N wake-enabled cycles (default 5), then two +# wake-disabled cycles +# bt-wake-emulated.sh unpair remove that pairing on both sides +# +# The lab is described by a local configuration file that is not part of the +# tree, ${BT_WAKE_CONF:-~/.config/aurora/bt-wake.conf}, a shell fragment with: +# TARGET_SSH ssh destination (an ssh_config alias) of the system under test +# EMU_SSH ssh destination of the BlueZ host running the emulator +# EMU_ADDR Bluetooth address of the emulator host's adapter +# TARGET_ADDR Bluetooth address of the target's adapter +# EMU_TOOLS directory holding hog-keyboard.py on the emulator host +# TARGET_SSH_ARGS optional extra ssh/scp options for the target, e.g. -i +# Only the emulator is ever paired or removed; the paired lists of both sides +# are recorded before and after. +set -u + +conf=${BT_WAKE_CONF:-$HOME/.config/aurora/bt-wake.conf} +[ -r "$conf" ] || { echo "no lab configuration: $conf" >&2; exit 2; } +# shellcheck source=/dev/null +. "$conf" +mac='^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$' +for v in TARGET_SSH EMU_SSH EMU_ADDR TARGET_ADDR EMU_TOOLS; do + [ -n "${!v:-}" ] || { echo "$conf does not set $v" >&2; exit 2; } +done +[[ $EMU_ADDR =~ $mac && $TARGET_ADDR =~ $mac ]] || + { echo "EMU_ADDR and TARGET_ADDR must be Bluetooth addresses" >&2; exit 2; } + +ssh_opts=(-o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2) +read -r -a target_args <<< "${TARGET_SSH_ARGS:-}" +tgt() { ssh "${ssh_opts[@]}" "${target_args[@]}" "$TARGET_SSH" "$@"; } +box() { ssh "${ssh_opts[@]}" "$EMU_SSH" "$@"; } +CTL=/run/hogkb/ctl +TGT_DIR=/var/tmp/bt-wake +BACKSTOP=120 +TRIGGER_AFTER=20 +out=${OUT:-./bt-wake-run-$(date +%Y%m%d-%H%M%S)} +mkdir -p "$out" + +say() { echo "$(date +%H:%M:%S) $*" | tee -a "$out/run.log"; } +emu() { box "echo '$*' | sudo -n tee $CTL > /dev/null"; } + +emu_start() { + box "pgrep -f '[h]og-keyboard.py' > /dev/null || { + sudo -n mkdir -p /run/hogkb + sudo -n setsid nohup $EMU_TOOLS/hog-keyboard.py > /dev/null 2>&1 < /dev/null & + sleep 3 + }; sudo -n tail -2 /run/hogkb/log" +} + +paired_lists() { + tgt "bluetoothctl devices Paired" > "$out/target-paired-$1.txt" + box "bluetoothctl devices Paired" > "$out/emulator-paired-$1.txt" + say "paired ($1): target $(wc -l < "$out/target-paired-$1.txt")," \ + "emulator host $(wc -l < "$out/emulator-paired-$1.txt")" +} + +case "${1:-}" in +pair) + paired_lists before-pair + emu_start + emu pairable on + sleep 2 + # Target side: discover, pair (Just Works), trust, allow wake. + tgt "{ echo 'agent NoInputNoOutput'; echo default-agent; echo 'scan le'; sleep 12; + echo 'scan off'; echo 'pair $EMU_ADDR'; sleep 15; echo 'trust $EMU_ADDR'; + sleep 1; echo 'connect $EMU_ADDR'; sleep 8; echo 'wake $EMU_ADDR on'; sleep 1; + echo 'info $EMU_ADDR'; sleep 1; } | bluetoothctl" > "$out/pair-target.txt" 2>&1 + emu pairable off + grep -E "Paired|Bonded|Trusted|Connected|WakeAllowed|UUID: Human" \ + "$out/pair-target.txt" | tail -8 | tee -a "$out/run.log" + paired_lists after-pair + ;; +run) + n=${2:-5} + emu_start + tgt "mkdir -p $TGT_DIR" && + scp -q "${target_args[@]}" "$(dirname "$0")/bt-wake-cycle.sh" \ + "$TARGET_SSH:$TGT_DIR/" + pass=0 + plan=() + for _ in $(seq 1 "$n"); do plan+=(on); done + plan+=(off off) + k=0 + for mode in "${plan[@]}"; do + k=$((k + 1)) + d=$TGT_DIR/cycle-$k-$mode + # Start each cycle with the keyboard connected, as in real use. + emu key 0x2c 30 + sleep 8 + say "cycle $k wake=$mode: suspending the target (backstop ${BACKSTOP}s)" + tgt "rm -rf $d; setsid nohup $TGT_DIR/bt-wake-cycle.sh -w $mode -s $BACKSTOP \ + -a $EMU_ADDR -o $d > $d.log 2>&1 < /dev/null &" + for _ in $(seq 1 30); do + tgt "test -f $d/suspending" 2> /dev/null && break + sleep 1 + done + sleep "$TRIGGER_AFTER" + say "cycle $k: emulated key press" + # Keep advertising past the backstop, so that the keyboard also + # comes back after an RTC wake. + emu key 0x2c $((BACKSTOP + 60)) + for _ in $(seq 1 $((BACKSTOP + 120))); do + tgt "test -f $d/verdict.txt" 2> /dev/null && break + sleep 1 + done + tgt "cat $d/verdict.txt $d/after.txt" > "$out/cycle-$k-$mode.txt" 2>&1 + head -4 "$out/cycle-$k-$mode.txt" | tee -a "$out/run.log" + grep -q "PASS" "$out/cycle-$k-$mode.txt" && pass=$((pass + 1)) + box "sudo -n tail -8 /run/hogkb/log" >> "$out/emulator.log" + sleep 10 + done + say "RESULT: $pass of ${#plan[@]} cycles passed (evidence $out, target $TGT_DIR)" + tgt "tar -C $TGT_DIR -czf - ." > "$out/target-evidence.tgz" + [ "$pass" = "${#plan[@]}" ] + ;; +unpair) + paired_lists before-unpair + if grep -qi "$EMU_ADDR" "$out/target-paired-before-unpair.txt"; then + tgt "bluetoothctl remove $EMU_ADDR" | tee -a "$out/run.log" + fi + box "bluetoothctl devices Paired | grep -qi $TARGET_ADDR && + sudo -n bluetoothctl remove $TARGET_ADDR" | tee -a "$out/run.log" + emu quit + paired_lists after-unpair + ;; +*) + echo "usage: $0 pair | run [cycles] | unpair" >&2 + exit 2 + ;; +esac diff --git a/tools/aurora-wake/bt-wake-power.sh b/tools/aurora-wake/bt-wake-power.sh new file mode 100755 index 0000000000000..af5d49dadb216 --- /dev/null +++ b/tools/aurora-wake/bt-wake-power.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Cost of Bluetooth wake in suspend-to-idle on the J700, as root: alternates +# RTC-timed sleeps with the Bluetooth PCI function's wakeup disabled and +# enabled (no Bluetooth activity expected) and records, per sleep: +# - Bluetooth MSIs and all interrupts taken across the sleep, +# - how often the s2idle loop was re-entered (pm_debug "Timekeeping suspended"), +# - the battery energy used, when running on battery (energy_now). +# Usage: bt-wake-power.sh [-s secs] [-n pairs] [-o outdir] +set -u +secs=300 +pairs=2 +out=/var/tmp/bt-wake/power-$(date +%Y%m%d-%H%M%S) +while getopts "s:n:o:" opt; do + case $opt in s) secs=$OPTARG ;; n) pairs=$OPTARG ;; o) out=$OPTARG ;; *) exit 2 ;; esac +done +[ "$(id -u)" = 0 ] || { echo "must run as root" >&2; exit 2; } +dev=/sys/bus/pci/devices/0000:01:00.1 +bat=/sys/class/power_supply/macsmc-battery +irq=$(awk -F: '/mt7932-bt/ { gsub(/ /, "", $1); print $1; exit }' /proc/interrupts) +mkdir -p "$out" +orig=$(cat "$dev/power/wakeup") +trap 'echo "$orig" > "$dev/power/wakeup"' EXIT +echo 1 > /sys/power/pm_debug_messages +echo 'module mt7932_bt_pcie +p' > /sys/kernel/debug/dynamic_debug/control 2> /dev/null + +irq_sum() { # $1: irq number or "all" + awk -v i="$1" '$1 ~ /^[0-9]+:$/ && (i == "all" || $1 == i ":") { + for (k = 2; k <= NF; k++) if ($k ~ /^[0-9]+$/) s += $k; else break + } END { print s + 0 }' /proc/interrupts +} + +echo "mode secs slept bt_msis all_irqs s2idle_entries energy_mWh status armed" | + tee "$out/results.txt" +for p in $(seq 1 "$pairs"); do + for mode in disabled enabled; do + echo "$mode" > "$dev/power/wakeup" + dmesg -C + b0=$(irq_sum "$irq") + a0=$(irq_sum all) + e0=$(cat "$bat/energy_now") + st=$(cat "$bat/status") + t0=$(date +%s.%N) + rtcwake -d rtc1 -m freeze -s "$secs" > /dev/null 2>&1 + t1=$(date +%s.%N) + b1=$(irq_sum "$irq"); a1=$(irq_sum all); e1=$(cat "$bat/energy_now") + dmesg > "$out/dmesg-$p-$mode.txt" + entries=$(grep -c "Timekeeping suspended" "$out/dmesg-$p-$mode.txt") + armed=$(grep -c "system sleep with wakeup armed" "$out/dmesg-$p-$mode.txt") + slept=$(awk -v a="$t0" -v b="$t1" 'BEGIN { printf "%.1f", b - a }') + echo "$mode $secs $slept $((b1 - b0)) $((a1 - a0)) $entries" \ + "$(((e0 - e1) / 1000)) $st $armed" | tee -a "$out/results.txt" + sleep 20 + done +done +echo "evidence: $out" diff --git a/tools/aurora-wake/hog-keyboard.py b/tools/aurora-wake/hog-keyboard.py new file mode 100755 index 0000000000000..d21f9e09d69bf --- /dev/null +++ b/tools/aurora-wake/hog-keyboard.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Emulated BLE HID-over-GATT keyboard on a BlueZ host, for unattended +Bluetooth wake tests of another machine. + +It behaves like a real BLE keyboard that went idle: it advertises only when a +key is "pressed" (a command on the control FIFO), and stops advertising as soon +as a central connects, so a central that disconnects it for suspend is not +woken again until the next key. + +Run as root: hog-keyboard.py [--ctl /run/hogkb/ctl] [--log /run/hogkb/log] +Commands (one per line, written to the FIFO): + pairable on|off adapter discoverable + pairable, advertise for pairing + key [usage [secs]] advertise if needed, wait up to secs (60) for the + central to connect and subscribe, then press+release + adv on|off advertise / stop (debug) + status log connection state + quit +""" +import argparse +import os +import subprocess +import sys +import threading +import time + +import dbus +import dbus.exceptions +import dbus.mainloop.glib +import dbus.service +from gi.repository import GLib + +BLUEZ = "org.bluez" +OM = "org.freedesktop.DBus.ObjectManager" +PROPS = "org.freedesktop.DBus.Properties" +GATT_MGR = "org.bluez.GattManager1" +SVC = "org.bluez.GattService1" +CHR = "org.bluez.GattCharacteristic1" +DSC = "org.bluez.GattDescriptor1" +AGENT = "org.bluez.Agent1" +ADV_INSTANCE = "1" + +# Boot-compatible keyboard with report ID 1: modifiers, reserved, 6 keys. +REPORT_MAP = bytes([ + 0x05, 0x01, 0x09, 0x06, 0xA1, 0x01, 0x85, 0x01, + 0x05, 0x07, 0x19, 0xE0, 0x29, 0xE7, 0x15, 0x00, 0x25, 0x01, + 0x75, 0x01, 0x95, 0x08, 0x81, 0x02, + 0x95, 0x01, 0x75, 0x08, 0x81, 0x01, + 0x95, 0x06, 0x75, 0x08, 0x15, 0x00, 0x25, 0x65, + 0x05, 0x07, 0x19, 0x00, 0x29, 0x65, 0x81, 0x00, + 0xC0, +]) + +LOG = None + + +def log(msg): + line = "%s %s" % (time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()) + + ".%03d" % (int(time.time() * 1000) % 1000), msg) + print(line, flush=True) + if LOG: + with open(LOG, "a") as f: + f.write(line + "\n") + + +def ay(data): + return dbus.Array([dbus.Byte(b) for b in data], signature="y") + + +class Application(dbus.service.Object): + def __init__(self, bus): + self.path = "/org/aurora/hogkb" + self.services = [] + super().__init__(bus, self.path) + + @dbus.service.method(OM, out_signature="a{oa{sa{sv}}}") + def GetManagedObjects(self): + out = {} + for s in self.services: + out[s.path] = s.props() + for c in s.chars: + out[c.path] = c.props() + for d in c.descs: + out[d.path] = d.props() + return out + + +class Service(dbus.service.Object): + def __init__(self, bus, app, index, uuid): + self.path = "%s/service%d" % (app.path, index) + self.uuid = uuid + self.chars = [] + super().__init__(bus, self.path) + app.services.append(self) + + def props(self): + return {SVC: {"UUID": self.uuid, "Primary": True, + "Characteristics": dbus.Array([c.path for c in self.chars], signature="o")}} + + +class Characteristic(dbus.service.Object): + def __init__(self, bus, svc, index, uuid, flags, value=b""): + self.path = "%s/char%d" % (svc.path, index) + self.svc = svc + self.uuid = uuid + self.flags = flags + self.value = bytes(value) + self.descs = [] + self.notifying = False + super().__init__(bus, self.path) + svc.chars.append(self) + + def props(self): + return {CHR: {"Service": dbus.ObjectPath(self.svc.path), "UUID": self.uuid, + "Flags": self.flags, + "Descriptors": dbus.Array([d.path for d in self.descs], signature="o")}} + + @dbus.service.method(CHR, in_signature="a{sv}", out_signature="ay") + def ReadValue(self, options): + return ay(self.value) + + @dbus.service.method(CHR, in_signature="aya{sv}") + def WriteValue(self, value, options): + self.value = bytes(value) + + @dbus.service.method(CHR) + def StartNotify(self): + self.notifying = True + log("notify on %s" % self.uuid) + + @dbus.service.method(CHR) + def StopNotify(self): + self.notifying = False + log("notify off %s" % self.uuid) + + @dbus.service.signal(PROPS, signature="sa{sv}as") + def PropertiesChanged(self, interface, changed, invalidated): + pass + + def notify(self, value): + self.value = bytes(value) + self.PropertiesChanged(CHR, {"Value": ay(value)}, []) + + +class Descriptor(dbus.service.Object): + def __init__(self, bus, chrc, index, uuid, flags, value): + self.path = "%s/desc%d" % (chrc.path, index) + self.chrc = chrc + self.uuid = uuid + self.flags = flags + self.value = bytes(value) + super().__init__(bus, self.path) + chrc.descs.append(self) + + def props(self): + return {DSC: {"Characteristic": dbus.ObjectPath(self.chrc.path), "UUID": self.uuid, + "Flags": self.flags}} + + @dbus.service.method(DSC, in_signature="a{sv}", out_signature="ay") + def ReadValue(self, options): + return ay(self.value) + + +class Agent(dbus.service.Object): + """Just Works: accept everything (test peripheral only).""" + + @dbus.service.method(AGENT) + def Release(self): + pass + + @dbus.service.method(AGENT, in_signature="os") + def AuthorizeService(self, device, uuid): + log("authorize service %s %s" % (device, uuid)) + + @dbus.service.method(AGENT, in_signature="ou") + def RequestConfirmation(self, device, passkey): + log("confirm %s %06d" % (device, passkey)) + + @dbus.service.method(AGENT, in_signature="o") + def RequestAuthorization(self, device): + log("authorize %s" % device) + + @dbus.service.method(AGENT) + def Cancel(self): + log("agent cancel") + + +class Keyboard: + def __init__(self, bus, adapter): + self.bus = bus + self.adapter_path = adapter + self.adapter = dbus.Interface(bus.get_object(BLUEZ, adapter), PROPS) + self.app = Application(bus) + hid = Service(bus, self.app, 0, "1812") + Characteristic(bus, hid, 0, "2a4a", ["read"], [0x11, 0x01, 0x00, 0x02]) + Characteristic(bus, hid, 1, "2a4b", ["encrypt-read"], REPORT_MAP) + Characteristic(bus, hid, 2, "2a4c", ["write-without-response"], [0]) + Characteristic(bus, hid, 3, "2a4e", ["read", "write-without-response"], [1]) + self.report = Characteristic(bus, hid, 4, "2a4d", ["encrypt-read", "notify"], [0] * 8) + Descriptor(bus, self.report, 0, "2908", ["read"], [0x01, 0x01]) + bat = Service(bus, self.app, 1, "180f") + Characteristic(bus, bat, 0, "2a19", ["read", "notify"], [100]) + dis = Service(bus, self.app, 2, "180a") + # PnP ID: USB vendor source, Linux Foundation, test product. + Characteristic(bus, dis, 0, "2a50", ["read"], [0x02, 0x6b, 0x1d, 0x46, 0x02, 0x01, 0x00]) + self.index = adapter.rsplit("hci", 1)[1] + self.adv = False + self.connected = set() + mgr = dbus.Interface(bus.get_object(BLUEZ, adapter), GATT_MGR) + mgr.RegisterApplication(self.app.path, {}, + reply_handler=lambda: log("GATT application registered"), + error_handler=lambda e: log("GATT register failed: %s" % e)) + self.agent = Agent(bus, "/org/aurora/hogkb/agent") + am = dbus.Interface(bus.get_object(BLUEZ, "/org/bluez"), "org.bluez.AgentManager1") + am.RegisterAgent(self.agent._object_path, "NoInputNoOutput") + am.RequestDefaultAgent(self.agent._object_path) + bus.add_signal_receiver(self.props_changed, dbus_interface=PROPS, + signal_name="PropertiesChanged", path_keyword="path") + + def props_changed(self, interface, changed, invalidated, path=None): + if interface != "org.bluez.Device1" or "Connected" not in changed: + return + if changed["Connected"]: + self.connected.add(path) + log("connected %s" % path) + # A real keyboard stops advertising once its host is back. + self.set_adv(False) + else: + self.connected.discard(path) + log("disconnected %s" % path) + + def set_adv(self, on, discoverable=False): + """ + Legacy MGMT advertising through btmgmt: on some kernels bluetoothd's + extended advertising path rejects the data (Add Extended Advertising + Data: Invalid Parameters), while Add Advertising works. Instance 1, + connectable, HID service UUID; appearance and name in scan response. + """ + if on and not self.adv: + scan_rsp = "0319c103" + "0d09" + b"AuroraTestKB".hex() + cmd = ["btmgmt", "--index", self.index, "add-adv", "-c", "-u", "1812", + "-s", scan_rsp] + if discoverable: + cmd.append("-g") + ret = subprocess.run(cmd + [ADV_INSTANCE], capture_output=True, text=True, + timeout=10, stdin=subprocess.PIPE) + self.adv = "Instance added" in ret.stdout + log("advertising" if self.adv else "advertise failed: %s" % ret.stdout.strip()) + elif not on and self.adv: + subprocess.run(["btmgmt", "--index", self.index, "rm-adv", ADV_INSTANCE], + capture_output=True, timeout=10, stdin=subprocess.PIPE) + self.adv = False + log("advertising stopped") + + def pairable(self, on): + self.adapter.Set("org.bluez.Adapter1", "Pairable", dbus.Boolean(on)) + self.adapter.Set("org.bluez.Adapter1", "Discoverable", dbus.Boolean(on)) + self.set_adv(False) + if on: + self.set_adv(True, discoverable=True) + log("pairable %s" % on) + + def key(self, usage, deadline): + """Advertise until a central is connected and subscribed, then type.""" + if not (self.connected and self.report.notifying): + if not self.adv: + self.set_adv(True) + if time.monotonic() < deadline: + GLib.timeout_add(50, self.key, usage, deadline) + else: + log("key %#x: no central subscribed before the deadline" % usage) + self.set_adv(False) + return False + self.report.notify([0, 0, usage, 0, 0, 0, 0, 0]) + GLib.timeout_add(30, lambda: (self.report.notify([0] * 8), False)[1]) + log("key %#x sent" % usage) + return False + + +def fifo_reader(path, kb): + while True: + with open(path) as f: + for line in f: + words = line.split() + if not words: + continue + GLib.idle_add(command, kb, words) + + +def command(kb, words): + cmd = words[0] + log("command: %s" % " ".join(words)) + if cmd == "pairable": + kb.pairable(words[1:] == ["on"]) + elif cmd == "adv": + kb.set_adv(words[1:] == ["on"]) + elif cmd == "key": + usage = int(words[1], 0) if len(words) > 1 else 0x2c # space + wait = float(words[2]) if len(words) > 2 else 60 + kb.key(usage, time.monotonic() + wait) + elif cmd == "status": + log("connected=%s notifying=%s advertising=%s" % + (sorted(kb.connected), kb.report.notifying, kb.adv)) + elif cmd == "quit": + kb.set_adv(False) + loop.quit() + return False + + +def main(): + global LOG, loop + ap = argparse.ArgumentParser() + ap.add_argument("--ctl", default="/run/hogkb/ctl") + ap.add_argument("--log", default="/run/hogkb/log") + ap.add_argument("--adapter", default="/org/bluez/hci0") + args = ap.parse_args() + os.makedirs(os.path.dirname(args.ctl), exist_ok=True) + if not os.path.exists(args.ctl): + os.mkfifo(args.ctl, 0o600) + LOG = args.log + dbus.mainloop.glib.DBusGMainLoop(set_as_default=True) + bus = dbus.SystemBus() + kb = Keyboard(bus, args.adapter) + threading.Thread(target=fifo_reader, args=(args.ctl, kb), daemon=True).start() + loop = GLib.MainLoop() + log("ready, control FIFO %s" % args.ctl) + loop.run() + + +if __name__ == "__main__": + sys.exit(main()) From f22b055cef58cc509dacbba2d95e265ce69097c2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 21:44:22 +1100 Subject: [PATCH 1145/1579] Bluetooth: mt7932: Enable wakeup by default A paired Bluetooth keyboard or mouse can wake the system only while wakeup is enabled on the PCI function of the controller, see bt7932_wakeup() and bt7932_may_wake(). The PCI core leaves it disabled, so at its defaults a J700 is not woken by a Bluetooth keyboard: the tests that qualified Bluetooth wakeup enabled it from userspace. Enable it once the controller is registered, as btmtksdio does for MediaTek SDIO controllers and hci_bcm for Broadcom UART ones. Which paired device may wake the system stays the user's choice, through the WakeAllowed property that BlueZ keeps per device, and power/wakeup still turns wakeup off for the whole controller. This costs power in suspend-to-idle while Bluetooth is up: the controller keeps running with its interrupt armed, scanning for the devices allowed to wake the system. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 725833f8ecc51..8197fa04fc4ea 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1122,7 +1122,7 @@ static void bt7932_hw_error(struct hci_dev *hdev, u8 code) } /* - * Follows the wakeup setting of the PCI function, which is off by default. + * Follows the wakeup setting of the PCI function, see bt7932_probe(). * Only while it is on does hci_suspend_sync() leave page scan and LE passive * scan set up for the devices flagged for remote wakeup, and only then does * bt7932_suspend() keep the transport running with its interrupt armed. @@ -1370,6 +1370,12 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret < 0) goto retained_fault; bt->registered = true; + /* + * Let paired devices wake the system by default, as btmtksdio does + * for MediaTek SDIO controllers: BlueZ still decides per device + * (WakeAllowed), and power/wakeup turns it off for the controller. + */ + device_set_wakeup_enable(&pdev->dev, true); dev_dbg(&pdev->dev, "BT_HCI_REGISTERED; setup/discovery/connection remain separate gates\n"); return 0; retained_fault: From 1dbd5f2b44d3fe9bf0430e5a3a66429506bd5fbe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 12:29:57 +1100 Subject: [PATCH 1146/1579] selftests: apple-dcp: Refresh extraction for current host regressions The host runner fails 16 of 55 cases on the current drivers because its extraction fixtures still expect the old ATC, CD321x, DPTX and RTKit boundaries. Share a lane-only adapter around the actual Type-C mode mapper and locked mux body. Extract orientation and the complete PM update path separately, model scoped locking, and disable KUnit redirects in host fixtures. Check that clean same-mode updates bypass PM while dirty or pending work still needs ownership. Extract the optional-router lookup for CD321x probe, model its nullable boundary, and check both deferred probing and router fallback. Refresh the DPTX callback name and catalogue/announcement state, and follow RTKit's debug notification handling for opaque OSLOG types 2 through 5. Retain the strict compiler and sanitizer settings and all 55 runner cases. The full buildbox run improves from 39 pass / 16 fail to 55 pass / 0 fail. No production driver changes are needed. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/README | 12 ++++- .../selftests/drivers/gpu/apple-dcp/boot.py | 21 ++++---- .../selftests/drivers/gpu/apple-dcp/common.py | 16 ++++++ .../drivers/gpu/apple-dcp/dwc3/prefix.c | 3 ++ .../drivers/gpu/apple-dcp/lifecycle.py | 6 +-- .../selftests/drivers/gpu/apple-dcp/phy.py | 22 +++++--- .../selftests/drivers/gpu/apple-dcp/pm.py | 51 +++++++++++++------ .../selftests/drivers/gpu/apple-dcp/rate.py | 37 +++++++++----- 8 files changed, 117 insertions(+), 51 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index 33373ea702c30..ad39902e5e259 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -75,8 +75,18 @@ or admit a hardware trial. Token/sequence semantics remain a separate protocol fact to establish; the tests do not manufacture a firmware echo guarantee. The boot group executes the actual OSLOG dispatcher with synthetic fixed-width -words. Known opaque event types remain visible, unknown types retain warnings, +words. Opaque event types 2-5 use debug logging, unknown types retain warnings, and only the existing buffer-request path can allocate/ACK buffers. The EDID service-rebind group also distinguishes intentional optional EOPNOTSUPP from required admission failures and genuine optional-service errors, and proves refused services never OPEN or publish. No captured firmware values are used. + +The ATC lane/PIPE and DWC3 teardown fixtures use a shared host adapter around +both the extracted Type-C target-mode mapper and the extracted locked mux body. +They omit USB PM at that boundary. The PM fixtures replace the adapter with the +actual mux update and callback, including the scoped no-op check: a clean +same-mode request needs neither root-hub lookup nor PM, while dirty lanes, +pending PIPE work and a mismatched PIPE still require ownership. KUnit static +redirects are disabled in host fixtures so the production bodies execute. +CD321x probe uses the actual optional-router lookup with a modeled deferred +probe decision; both supplier waiting and optional-router fallback are checked. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py index bf95df845443d..1f6fc49e90a68 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py @@ -19,37 +19,36 @@ def oslog(root): #define APPLE_RTKIT_EP_OSLOG 8 struct buffer { unsigned sentinel; }; struct apple_rtkit { void *dev; struct buffer oslog_buffer; }; -static unsigned warnings, information, requests; +static unsigned warnings, notifications, requests; static u64 observed; -static u8 observed_type; #define dev_warn(dev,fmt,msg) do { (void)(dev); (void)(fmt); warnings++; observed=(msg); } while (0) -#define dev_info(dev,fmt,type,msg) do { (void)(dev); (void)(fmt); information++; observed_type=(type); observed=(msg); } while (0) +#define dev_dbg(dev,fmt,msg) do { (void)(dev); (void)(fmt); notifications++; observed=(msg); } while (0) static void apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, struct buffer *b, u8 ep, u64 msg) { assert(b==&rtk->oslog_buffer && ep==8); requests++; observed=msg; } ''' + mask + '\n' + block(source, 'static void apple_rtkit_oslog_rx(') + r''' -static void reset(void) { warnings=information=requests=0; observed=0; observed_type=0; } +static void reset(void) { warnings=notifications=requests=0; observed=0; } int main(void) { struct apple_rtkit rtk={.oslog_buffer={.sentinel=0x12345678}}; - for (u8 type=3; type<=5; type++) { + for (u8 type=2; type<=5; type++) { for (unsigned edge=0; edge<2; edge++) { u64 msg=((u64)type<<56) | (edge ? ((1ULL<<56)-1) : 0); reset(); apple_rtkit_oslog_rx(&rtk,msg); - assert(!warnings && information==1 && !requests); - assert(observed==msg && observed_type==type); + assert(!warnings && notifications==1 && !requests); + assert(observed==msg); assert(rtk.oslog_buffer.sentinel==0x12345678); } } puts("PASS recognized opaque types preserve both payload bounds without buffer/ACK operations"); - const u8 unknown[]={0,2,6,0xff}; + const u8 unknown[]={0,6,0xff}; for (unsigned i=0;im); } +#define CONFIG_PHY_APPLE_ATC_KUNIT_TEST 0 +#define IS_ENABLED(option) (option) +#define KUNIT_STATIC_STUB_REDIRECT(...) ((void)0) #define guard(kind) struct held held_guard __attribute__((cleanup(release))) = hold enum phy_mode { PHY_MODE_USB_HOST, PHY_MODE_USB_DEVICE }; enum { APPLE_ATCPHY_LANE_0, APPLE_ATCPHY_LANE_1 }; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index db72bc324ee11..7756046d7c07a 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -3,7 +3,7 @@ """Actual disconnect authority, DWC3 teardown and firmware resource admission.""" from pathlib import Path import re -from common import block, source_defines +from common import block, atc_mux_without_pm, source_defines def disconnect(root): # Copyright 2026 Ryan Murray @@ -749,11 +749,11 @@ def fn(n, name): s = files[n] normalized = n == 'atc' and name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in s if normalized: - name = 'atcphy_mux_set_locked' + return atc_mux_without_pm(s) m = re.search(r'^(?:static [^\n]*\b|(?:int|void) )' + name + r'\(', s, re.M) assert m, (n, name) result = block(s, m.start()) - return result.replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) if normalized else result + return result decls = [] for name in ('atcphy_generation', 'atcphy_lane_mode', 'atcphy_pipehandler_state', 'atcphy_mode', 'atcphy_dp_link_rate'): diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py index fbb1951c0f0b1..e29f767e80a29 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py @@ -3,7 +3,7 @@ """Actual PHY parking, failure/retry, masked context and OFF controls.""" from pathlib import Path import re -from common import block, source_defines +from common import block, source_defines, atc_mux_without_pm def parking(root): @@ -13,8 +13,7 @@ def parking(root): def function(source, name): if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: - marker = re.search(r'^static [^\n]+\batcphy_mux_set_locked\(', source, re.M).group() - return block(source, marker).replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) + return atc_mux_without_pm(source) return block(source, re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) @@ -133,7 +132,7 @@ def function(source, name): parts.append(dp) for name in ('atcphy_configure_lanes', 'j700_aux_enable', 'j700_aux_disable', 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', - 'atcphy_power_on', 'atcphy_configure', 'atcphy_sw_set', 'atcphy_mux_set', + 'atcphy_power_on', 'atcphy_configure', 'atcphy_set_orientation', 'atcphy_sw_set', 'atcphy_mux_set', 'atcphy_usb3_set_mode', 'atcphy_usb3_power_off'): parts.append(function(atc, name)) production = '\n'.join(parts) @@ -163,6 +162,9 @@ def function(source, name): #define dev_err report #define lockdep_assert_held(...) ((void)0) #define WARN_ON_ONCE(v) assert(!(v)) +#define CONFIG_PHY_APPLE_ATC_KUNIT_TEST 0 +#define IS_ENABLED(option) (option) +#define KUNIT_STATIC_STUB_REDIRECT(...) ((void)0) #define guard(kind) (void) #define fallthrough __attribute__((fallthrough)) #define ndelay(n) ((void)(n)) @@ -1672,8 +1674,7 @@ def block(marker): def function(name): if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: - marker = re.search(r'^static [^\n]+\batcphy_mux_set_locked\(', source, re.M).group() - return block(marker).replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) + return atc_mux_without_pm(source) return block(re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) @@ -1773,6 +1774,9 @@ def function(name): #define dev_err(...) ((void)0) #define lockdep_assert_held(...) ((void)0) #define WARN_ON_ONCE(v) assert(!(v)) +#define CONFIG_PHY_APPLE_ATC_KUNIT_TEST 0 +#define IS_ENABLED(option) (option) +#define KUNIT_STATIC_STUB_REDIRECT(...) ((void)0) #define guard(kind) (void) #define fallthrough __attribute__((fallthrough)) #define usleep_range(...) ((void)0) @@ -2023,8 +2027,7 @@ def unwind(root): def function(source, name): if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: - marker = re.search(r'^static [^\n]+\batcphy_mux_set_locked\(', source, re.M).group() - return block(source, marker).replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) + return atc_mux_without_pm(source) return block(source, re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) @@ -2131,6 +2134,9 @@ def function(source, name): #define dev_err report #define lockdep_assert_held(...) ((void)0) #define WARN_ON_ONCE(v) assert(!(v)) +#define CONFIG_PHY_APPLE_ATC_KUNIT_TEST 0 +#define IS_ENABLED(option) (option) +#define KUNIT_STATIC_STUB_REDIRECT(...) ((void)0) #define guard(kind) (void) #define fallthrough __attribute__((fallthrough)) #define ndelay(n) ((void)(n)) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 6125dbf312b8f..27b207fd7b8f8 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -10,7 +10,8 @@ def ownership(root, usb_unreachable=False): ns=phy.parking(root) s=ns['atc'];has_pm='static int atcphy_usb3_pm_get(' in s prod=ns['production'].replace('struct apple_atcphy {','struct apple_atcphy {\n struct device_node *np;',1) - if has_pm: prod=prod.replace('static int atcphy_mux_set(', 'static int atcphy_mux_set_locked(',1) + if has_pm: + prod = prod.replace(block(prod, 'static int atcphy_mux_set('), '') # Non-mux fixture entry points continue to exercise actual source. Only the new # wrapper owns PM; the fixture's original helpers call it through this declaration. prefix=ns['prefix']+'\nstruct device_node;\nstatic int atcphy_mux_set(struct typec_mux_dev *, struct typec_mux_state *);\n' @@ -35,8 +36,9 @@ def ownership(root, usb_unreachable=False): prefix=prefix.replace('#define guard(kind) (void)',r''' struct lock_guard { int *lock; }; static struct lock_guard lock_take(int *lock) { assert(!*lock); *lock=1; return (struct lock_guard){lock}; } -static void lock_release(struct lock_guard *g) { assert(*g->lock==1); *g->lock=0; } +static void lock_release(struct lock_guard *g) { if(g->lock) { assert(*g->lock==1); *g->lock=0; } } #define guard(kind) struct lock_guard __attribute__((cleanup(lock_release))) _guard = lock_take +#define scoped_guard(kind, ptr) for (struct lock_guard __attribute__((cleanup(lock_release))) _scope = lock_take(ptr); _scope.lock; lock_release(&_scope), _scope.lock=NULL) ''') # Basic USB/device APIs are modeled, but get/put/wrapper and all parking/ # restoration bodies are extracted verbatim from the chosen public source. @@ -240,13 +242,21 @@ def ownership(root, usb_unreachable=False): assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==-ETIMEDOUT&&p.mode_dirty&&p.pipe_pending);balanced(&p); bist_status_stuck=false;recover();assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0);balanced(&p); } - /* Scope and clean same-mode preserve MMIO guards; fixed no-op briefly owns PM. */ + /* Clean same-mode updates bypass PM; dirty lanes or PIPE retry still need it. */ struct typec_altmode alt={.svid=USB_TYPEC_DP_SID}; struct typec_mux_dev mux={&p};struct typec_mux_state same={.mode=TYPEC_DP_STATE_D,.alt=&alt}; prepare(&p,false,true,false);before=writes;assert(atcphy_mux_set(&mux,&same)==0&&resumes==0&&writes==before);balanced(&p); prepare(&p,true,false,false);before=writes;assert(atcphy_mux_set(&mux,&same)==0&&resumes==0&&writes==before);balanced(&p); prepare(&p,true,true,false);before=writes; - assert(atcphy_mux_set(&mux,&same)==0&&writes==before&&resumes==1);balanced(&p); + root_missing=true;resume_error=-EIO; + assert(atcphy_mux_set(&mux,&same)==0&&writes==before&&resumes==0);balanced(&p); + for(unsigned int retry=0;retry<3;retry++){ + prepare(&p,true,true,false);before=writes;root_missing=true; + if(retry==0)p.mode_dirty=true; + if(retry==1)p.pipe_pending=true; + if(retry==2)p.pipe_state=ATCPHY_PIPEHANDLER_STATE_DUMMY; + assert(atcphy_mux_set(&mux,&same)==-EAGAIN&&writes==before&&resumes==0);balanced(&p); + } /* Binding/identity failure must not reach USB PM or PHY programming. */ for(unsigned int missing=0;missing<3;missing++){ prepare(&p,true,true,false);before=writes; @@ -302,7 +312,8 @@ def ownership(root, usb_unreachable=False): helpers+=ns['function'](s,'atcphy_usb3_link_get')+'\n'+ns['function'](s,'atcphy_usb3_link_put')+'\n' helpers+=ns['function'](s,'atcphy_usb3_pm_find')+'\n#endif\n' helpers+=ns['function'](s,'atcphy_usb3_pm_get')+'\n'+ns['function'](s,'atcphy_usb3_pm_put')+'\n' - # The fixture extractor adapts the locked body; extract the wrapper directly. + helpers+=ns['function'](s,'atcphy_mux_is_noop')+'\n'+ns['function'](s,'atcphy_mux_update')+'\n' + # Lane fixtures provide the target mapping and locked body; use the real callback here. marker=re.search(r'^static int atcphy_mux_set\(',s,re.M).group() helpers+=ns['block'](s,marker)+'\n' else: @@ -346,6 +357,8 @@ def function(text, name): #include #define EPROBE_DEFER 517 #define IS_ERR(p) ((intptr_t)(p)<0) +#define ERR_PTR(e) ((void *)(intptr_t)(e)) +#define dev_info(...) ((void)0) #define PTR_ERR(p) ((int)(intptr_t)(p)) #define IS_ERR_OR_NULL(p) (!(p)||IS_ERR(p)) #define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) @@ -356,7 +369,8 @@ def function(text, name): struct work { bool pending; }; struct fwnode_handle { int refs; }; struct typec_mux_state { void *alt,*data; unsigned long mode; }; -struct tps6598x { int lock; void *port; }; +struct device; +struct tps6598x { int lock; void *port; struct device *dev; }; struct notifier_block { int (*notifier_call)(struct notifier_block *,unsigned long,void *); }; ''' pm = source('drivers/usb/typec/tipd/cd321x-pm.h').replace('#include ', '') @@ -375,7 +389,9 @@ def function(text, name): static struct fwnode_handle connector; static int mux_error, notifier_error, switch_error; static int ports,altmodes,muxes,switches,notifiers,stop_calls; -static bool displayport=true; +struct typec_thunderbolt_switch; +static bool keep_waiting=true; +static bool cd321x_tbt_switch_keep_waiting(struct device *d) { (void)d;return keep_waiting; } static int tps6598x_register_port(struct tps6598x *t,struct fwnode_handle *f) { (void)f;t->port=t;ports++;return 0; } @@ -383,7 +399,6 @@ def function(text, name): c->port_altmode_dp=c;c->port_altmode_tbt=c;altmodes+=2;return 0; } static void *fwnode_typec_mux_get(struct fwnode_handle *f) { (void)f;muxes++;return &muxes; } -static bool fwnode_property_present(struct fwnode_handle *f,const char *s) { (void)f;(void)s;return displayport; } static struct fwnode_handle *fwnode_find_reference(struct fwnode_handle *f,const char *s,int n) { (void)f;(void)s;(void)n;connector.refs++;return &connector; } @@ -396,23 +411,25 @@ def function(text, name): static int typec_set_mode(void *p,unsigned long m) { (void)p;assert(m==TYPEC_STATE_SAFE);return mux_error; } static int cd321x_tbt_notify(struct notifier_block *n,unsigned long e,void *d) { (void)n;(void)e;(void)d;return 0; } static int typec_thunderbolt_switch_register_notifier(void *s,struct notifier_block *n) { - (void)s;(void)n;if(notifier_error)return notifier_error;notifiers++;return 0; + (void)n;if(!s)return 0;if(notifier_error)return notifier_error;notifiers++;return 0; } -static void typec_thunderbolt_switch_put(void *s) { assert(s&&switches>0);switches--; } +static void typec_thunderbolt_switch_put(void *s) { if(s){assert(switches>0);switches--;} } static void typec_mux_put(void *m) { assert(m&&muxes>0);muxes--; } static void typec_unregister_altmode(void *a) { if(a){assert(altmodes>0);altmodes--;} } static void typec_unregister_port(void *p) { assert(p&&ports>0);ports--; } static void cd321x_stop_work(struct cd321x *c) { - stop_calls++;if(c->tbt_notifier_registered){notifiers--;c->tbt_notifier_registered=false;} + stop_calls++;if(c->tbt_notifier_registered){if(c->tbt_switch)notifiers--;c->tbt_notifier_registered=false;} cd321x_pm_remove(&c->pm);c->resume_work.pending=c->update_work.pending=false; } ''' core = source('drivers/usb/typec/tipd/core.c') - production = function(core, 'cd321x_register_port') + '\n' + function(core, 'cd321x_unregister_port') + switch_get = block(core, 'VISIBLE_IF_KUNIT struct typec_thunderbolt_switch *\ncd321x_tbt_switch_get(') + production = switch_get.replace('VISIBLE_IF_KUNIT', 'static', 1) + '\n' + production += function(core, 'cd321x_register_port') + '\n' + function(core, 'cd321x_unregister_port') controls = r''' static void reset(struct cd321x *c) { memset(c,0,sizeof(*c));connector.refs=0; - mux_error=notifier_error=switch_error=0;stop_calls=0; + mux_error=notifier_error=switch_error=0;stop_calls=0;keep_waiting=true; assert(!ports&&!altmodes&&!muxes&&!switches&&!notifiers); } static void balanced(void) { @@ -422,6 +439,7 @@ def function(text, name): struct cd321x c; struct cd321x_pm_state budget; cd321x_pm_init(&budget);assert(cd321x_pm_ready(&budget)); + assert(cd321x_display_taken(-ENOSPC)&&!cd321x_display_taken(-EBUSY)&&!cd321x_display_taken(0)); for(unsigned int i=0;ilock);*g->lock=false; } +#define guard(kind) struct rate_guard __attribute__((cleanup(rate_unlock))) _guard = rate_lock static bool dcp_is_usb4_output(struct apple_dcp *d) { return d->usb4; } static void schedule_work(struct work_struct *w) { w->scheduled++; } static const struct edid *drm_edid_raw(const struct drm_edid *e) { return (const void *)e->data; } +static bool dcpavserv_edid_current(struct apple_dcp *d, u64 generation) { + assert(d->hpd_mutex);return d->typec_cable_connected&&generation==d->typec_generation&&!d->quiescing; +} static bool dcpavserv_edid_service_current(struct apple_dcp *d, u64 e) { assert(d->dcpavserv.lock); return e==d->dcpavserv.epoch && d->dcpavserv.enabled && d->dcpavserv.service; } @@ -423,8 +436,8 @@ def case(value, automatic=True, internal=False, external=True, admitted=True): static void qualify(struct apple_dcp *d, struct apple_epic_service *s) { struct dptxport_apcall_link_rate rate={.link_rate=0x1e}, rate_reply={}; struct dptxport_apcall_set_active_lane_count lanes={.lane_count=2}, lanes_reply={}; - assert(!dptxport_call_qualified_config(s,false,&rate,sizeof(rate),&rate_reply,sizeof(rate_reply))); - assert(!dptxport_call_qualified_config(s,true,&lanes,sizeof(lanes),&lanes_reply,sizeof(lanes_reply))); + assert(!dptxport_call_link_config(s,false,&rate,sizeof(rate),&rate_reply,sizeof(rate_reply))); + assert(!dptxport_call_link_config(s,true,&lanes,sizeof(lanes),&lanes_reply,sizeof(lanes_reply))); (void)d; } static bool current(struct apple_dcp *d,const struct dcp_display_mode *m) { @@ -496,9 +509,9 @@ def case(value, automatic=True, internal=False, external=True, admitted=True): DENY(m.sdr_rgb.depth,10);DENY(m.sdr_rgb.colorimetry,1);DENY(m.sdr_rgb.range,1);DENY(m.mode.clock,414091); struct dptxport_apcall_link_rate rate={.link_rate=0x1e}, response={}; configure_error=-EIO; - assert(dptxport_call_qualified_config(&s,false,&rate,sizeof(rate),&response,sizeof(response))==-EIO); + assert(dptxport_call_link_config(&s,false,&rate,sizeof(rate),&response,sizeof(response))==-EIO); assert(!current(&d,&m));configure_error=0;qualify(&d,&s); - assert(dptxport_call_qualified_config(&s,false,&rate,sizeof(rate)-1,&response,sizeof(response))==-EINVAL); + assert(dptxport_call_link_config(&s,false,&rate,sizeof(rate)-1,&response,sizeof(response))==-EINVAL); assert(!current(&d,&m));qualify(&d,&s); dcp_firmware_metadata_connection(&d,7,false);dcp_firmware_metadata_connection(&d,7,true); assert(!current(&d,&m));qualify(&d,&s);assert(!current(&d,&m)); @@ -543,7 +556,6 @@ def publication(root): };''') state = state.replace('struct { bool lock, enabled; void *service; u64 epoch; } dcpavserv;', 'struct { bool lock, enabled; void *service; u64 epoch; int enable_completion; } dcpavserv;') state = state.replace('u64 typec_generation; unsigned int nr_modes;', '''u64 typec_generation; unsigned int nr_modes; - struct dcp_display_mode *modes; struct apple_epic *dcpavservep; int fw_compat,width_mm,height_mm,fixed_connector_type; unsigned notch_height; struct { size_t length,received; void *data; bool contiguous,payload_complete; @@ -551,7 +563,7 @@ def publication(root): struct dcp_metadata_ticket metadata_ticket,mode_ticket; } chunks;''') - start = state.index('static bool dcpavserv_edid_service_current(') + start = state.index('static bool dcpavserv_edid_current(') state = state[:start] stubs = r''' @@ -604,7 +616,6 @@ def publication(root): } static const struct drm_edid *dcp_firmware_edid(struct apple_dcp *d,u64 *generation) { (void)d; (void)generation; return ERR_PTR(-ENODATA); } static bool dcp_firmware_metadata_current(struct apple_dcp *d,u64 generation) { (void)d; (void)generation; return false; } -static bool dcp_has_panel(struct apple_dcp *d) { return d->fixed_connector_type==DRM_CONNECTOR_eDP; } static struct drm_display_mode *drm_mode_duplicate(struct drm_device *dev,const struct drm_display_mode *m) { assert(dev->mode_config.mutex);struct drm_display_mode *copy=malloc(sizeof(*copy));assert(copy);*copy=*m; if(invalidate_during_duplicate) { invalidate_during_duplicate=false;dcp_180hz_invalidate(probing); } @@ -691,9 +702,9 @@ def publication(root): static void timings(void) { start_receipt();finish_receipt(); } static void callbacks(bool lanes) { if(lanes) { struct dptxport_apcall_set_active_lane_count a={.lane_count=2},b={}; - assert(!dptxport_call_qualified_config(&s,true,&a,sizeof(a),&b,sizeof(b))); } + assert(!dptxport_call_link_config(&s,true,&a,sizeof(a),&b,sizeof(b))); } else { struct dptxport_apcall_link_rate a={.link_rate=0x1e},b={}; - assert(!dptxport_call_qualified_config(&s,false,&a,sizeof(a),&b,sizeof(b))); } + assert(!dptxport_call_link_config(&s,false,&a,sizeof(a),&b,sizeof(b))); } } static void probe(bool expected) { drm.mode_config.mutex=true;unsigned before=published; @@ -753,7 +764,7 @@ def publication(root): reset();callbacks(false);callbacks(true);timings();admit(0);fail_update=true;probe(false);assert(!con.drm_edid);cleanup(); reset();callbacks(false);callbacks(true);timings();admit(0);transport_edid.data[10]^=1;checksum(transport_edid.data);probe(false);cleanup();transport_edid=raw; reset();callbacks(false);callbacks(true);timings();admit(0);configure_error=-EIO; - struct dptxport_apcall_link_rate req={.link_rate=0x1e},resp={};assert(dptxport_call_qualified_config(&s,false,&req,sizeof(req),&resp,sizeof(resp))==-EIO); + struct dptxport_apcall_link_rate req={.link_rate=0x1e},resp={};assert(dptxport_call_link_config(&s,false,&req,sizeof(req),&resp,sizeof(resp))==-EIO); configure_error=0;probe(false);cleanup(); puts("PASS: 24 callback/timing/AV arrival permutations and real get_modes/lookup publication; early configuration needs no second callback; CLOSE, physical reconnect, receipt epochs and publication races deny stale proof"); } From c7c9a17ef9980368d3cf928bdc400ce7235c544a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:33:44 +1100 Subject: [PATCH 1147/1579] drm/apple: afk: Do not warn when a retired channel is closed afk_recv_handle_teardown() warns when the co-processor closes a channel that has no live service. On M1 and M2 that should not happen, because the driver never gives up a service on its own. Since the H17P reconnect support, it gives up services in two places: - dpavservep_detach() retires the DCPAVSERV EDID service when the external display goes away, before the firmware closes its channel; - afk_service_discard() drops a service whose preparation declined it, such as the optional AppleDCPDP2HDMI service on its second channel. The firmware then closes both channels on every unplug and the driver prints two "teardown for disabled channel" warnings, although it has nothing left to do. Log a teardown at debug level when the slot that last served the channel is torn down, and keep the warning for a channel no slot knows. A slot that is reused for another channel forgets the old one, so an unexpected close is still reported. Rename that warning to say "unknown", which is the only case left. Add KUnit cases for a retired, a declined, a reused and a live slot. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk-service-test.c | 33 ++++++++++++++++++++++++ drivers/gpu/drm/apple/afk.c | 19 ++++++++++++-- drivers/gpu/drm/apple/afk.h | 16 ++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/afk-service-test.c b/drivers/gpu/drm/apple/afk-service-test.c index 1cf589ae7a417..a64064758a906 100644 --- a/drivers/gpu/drm/apple/afk-service-test.c +++ b/drivers/gpu/drm/apple/afk-service-test.c @@ -218,6 +218,37 @@ static void dptx_service_duplicate_close_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, port.enabled); } +static void afk_service_retired_channel_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + /* A slot that never served the channel does not know it. */ + KUNIT_EXPECT_FALSE(test, afk_service_retired(service, 1)); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 1)); + /* A live service is found by afk_service_matches(), not here. */ + KUNIT_EXPECT_FALSE(test, afk_service_retired(service, 1)); + /* Retired locally (display unplugged) before the co-processor closes it. */ + afk_test_retire(service); + KUNIT_EXPECT_FALSE(test, afk_service_matches(service, 1)); + KUNIT_EXPECT_TRUE(test, afk_service_retired(service, 1)); + KUNIT_EXPECT_FALSE(test, afk_service_retired(service, 2)); + /* Reusing the slot forgets the old channel. */ + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + KUNIT_EXPECT_FALSE(test, afk_service_retired(service, 1)); + KUNIT_EXPECT_FALSE(test, afk_service_retired(service, 2)); +} + +static void afk_service_declined_channel_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + /* An announcement whose preparation we declined is closed later too. */ + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &prepared_ops, 3)); + afk_service_discard(service); + KUNIT_EXPECT_FALSE(test, afk_service_matches(service, 3)); + KUNIT_EXPECT_TRUE(test, afk_service_retired(service, 3)); +} + static struct kunit_case afk_service_cases[] = { KUNIT_CASE(afk_service_admission_test), KUNIT_CASE(dptx_service_identity_test), @@ -228,6 +259,8 @@ static struct kunit_case afk_service_cases[] = { KUNIT_CASE(afk_service_completed_reader_test), KUNIT_CASE(afk_service_tag_sequence_test), KUNIT_CASE(afk_service_unconverted_owner_test), + KUNIT_CASE(afk_service_retired_channel_test), + KUNIT_CASE(afk_service_declined_channel_test), { } }; diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 6ca27849346f8..adf1a755e4602 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -344,6 +344,17 @@ static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep * return NULL; } +static bool afk_epic_channel_retired(struct apple_dcp_afkep *ep, u32 channel) +{ + u32 i; + + for (i = 0; i < ep->num_channels; i++) + if (afk_service_retired(&ep->services[i], channel)) + return true; + + return false; +} + static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, enum epic_type etype, enum epic_category ecat, u8 stype, const void *payload, size_t payload_len, u64 token); @@ -579,8 +590,12 @@ static void afk_recv_handle_teardown(struct apple_dcp_afkep *ep, u32 channel) service = afk_epic_find_service(ep, channel); if (!service) { - dev_warn(ep->dcp->dev, "AFK[ep:%02x]: teardown for disabled channel %u\n", - ep->endpoint, channel); + if (afk_epic_channel_retired(ep, channel)) + dev_dbg(ep->dcp->dev, "AFK[ep:%02x]: teardown for retired channel %u\n", + ep->endpoint, channel); + else + dev_warn(ep->dcp->dev, "AFK[ep:%02x]: teardown for unknown channel %u\n", + ep->endpoint, channel); return; } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 0a12ecda982df..1b605ed80797e 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -191,6 +191,22 @@ static inline bool afk_service_matches(struct apple_epic_service *service, return found; } +/* + * True if this slot last served @channel and was retired or discarded on our + * side: the co-processor may still close that channel afterwards. + */ +static inline bool afk_service_retired(struct apple_epic_service *service, + u32 channel) +{ + unsigned long flags; + bool retired; + + spin_lock_irqsave(&service->lock, flags); + retired = service->channel == channel && service->torndown; + spin_unlock_irqrestore(&service->lock, flags); + return retired; +} + /* The caller holds service->lock. */ static inline bool afk_service_reclaimable(struct apple_epic_service *service) { From bcd2e166aadb9457956435a57ad9a61b92d8467f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:33:44 +1100 Subject: [PATCH 1148/1579] drm/apple: iomfb: Log the J700 power-LUT fallback at info level The H17P DCP asks for five power-LUT tables through the provider property callback. Commit 22778725c286 ("drm: apple: retain J700 provider replies with older loaders") answers with the J700 tables when the device tree lacks them, because the loader in use predates those properties, and warns once that it does so. No released loader adds these properties yet, so the warning appears on every J700 boot. The reply carries the same bytes as the disp0 entries of the Apple Device Tree, and both displays work with it, so there is no fault to report. Log the fallback at info level instead. The message still names the missing property, and properties that are present but malformed are still errors. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index d0195f2e97d94..b9870ae288675 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -452,7 +452,7 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, memset(reply, 0, sizeof(*reply)); memcpy(reply->data, property->legacy_data, property->length); reply->length = cpu_to_le32(property->length); - dev_warn_once(dcp->dev, + dev_info_once(dcp->dev, "missing provider property %s (%u bytes); using the J700 defaults\n", property->dt_name, property->length); return true; From e6dc18912d8788972ee18ce8f038745d2571ac32 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:35:15 +1100 Subject: [PATCH 1149/1579] rust: types: keep the OwnableMut safety section in its doc comment A blank source line separates the OwnableMut description from its safety section. Mark it as a documentation paragraph so Clippy does not reject the kernel crate with empty_line_after_doc_comments before checking Rust drivers. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 58b5080efc8ae..c57ddb08d127c 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -448,7 +448,7 @@ pub unsafe trait Ownable { /// /// When this type is implemented for an Ownable type, it allows Owned to be dereferenced /// into a &mut T. - +/// /// # Safety /// /// Implementers must ensure that the kernel never mutates the underlying type while From 748b2a25f2ca960017eef868e2323a27e07f0fe2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:38:15 +1100 Subject: [PATCH 1150/1579] drm/asahi: clean up the statistics Rust style Use infallible conversions for the sysfs enable flag and firmware utilization, and let field access dereference the DRM reference. Document why the registration helpers may pass the driver's device and optional snapshot to C. This clears the Clippy findings in the statistics code without changing its accounting, ownership or teardown order. Apply rustfmt to the statistics registration code in the driver and its sysfs wrapper. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/channel.rs | 12 +++++++----- drivers/gpu/drm/asahi/driver.rs | 20 ++++++++++++++------ drivers/gpu/drm/asahi/sysfs_exports.rs | 15 +++++++++++++-- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index a1a9773da5aeb..4f205e21ce331 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -672,11 +672,13 @@ impl StatsChannel::ver { // rem * util * 1e9 / (100 * ts_hz), which // cannot overflow u64 (rem < ts_hz <= 24e6 // on supported SoCs). - let util = [*util1, *util2, *util3, *util4] - .into_iter() - .max() - .unwrap_or(0) - .min(100) as u64; + let util = u64::from( + [*util1, *util2, *util3, *util4] + .into_iter() + .max() + .unwrap_or(0) + .min(100), + ); let ts = timestamp.0; if self.last_util_ts != 0 && ts > self.last_util_ts { let delta = ts - self.last_util_ts; diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index ffa01c6bb60d6..b2a03e394be64 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -258,7 +258,7 @@ impl platform::Driver for AsahiDriver { // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); - let stats = (*drm).gpu.stats_snapshot(); + let stats = drm.gpu.stats_snapshot(); let snapshot = stats.as_ref().map_or(core::ptr::null(), Arc::as_ptr); let raw_stats = crate::sysfs_exports::register( raw_dev, @@ -266,7 +266,12 @@ impl platform::Driver for AsahiDriver { stats.is_some() && *module_parameters::stats_export.value() != 0, )?; - Ok(Self { drm, raw_dev, raw_stats, stats }) + Ok(Self { + drm, + raw_dev, + raw_stats, + stats, + }) } } @@ -296,16 +301,19 @@ impl AsahiDriver { } // G17's firmware channel format has no supported stats decoder yet. let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = match crate::sysfs_exports::register( - raw_dev, core::ptr::null(), false, - ) { + let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { Ok(handle) => handle, Err(error) => { gpu.shutdown(); return Err(error); } }; - Ok(Self { drm, raw_dev, raw_stats, stats: None }) + Ok(Self { + drm, + raw_dev, + raw_stats, + stats: None, + }) } } diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index 1b0eb7c6ea50f..079623d1ef210 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -3,8 +3,8 @@ //! Per-device sysfs registration. The caller retains the snapshot Arc until //! unregister has drained active readers and freed the C attribute wrapper. -use core::ffi::{c_int, c_void}; use crate::stats::StatsSnapshot; +use core::ffi::{c_int, c_void}; extern "C" { fn asahi_sysfs_register( @@ -23,7 +23,16 @@ pub(crate) fn register( export_enabled: bool, ) -> kernel::error::Result<*mut c_void> { let mut handle = core::ptr::null_mut(); - let ret = unsafe { asahi_sysfs_register(dev.cast(), snapshot, export_enabled as c_int, &mut handle) }; + // SAFETY: The bound driver keeps the device and optional snapshot alive until + // unregister drains readers. `handle` is writable for the duration of the call. + let ret = unsafe { + asahi_sysfs_register( + dev.cast(), + snapshot, + c_int::from(export_enabled), + &mut handle, + ) + }; if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { @@ -33,5 +42,7 @@ pub(crate) fn register( /// Drain readers before the caller drops its snapshot Arc. pub(crate) fn unregister(dev: *mut kernel::bindings::device, handle: *mut c_void) { + // SAFETY: `handle` was returned by register for this device. The driver still + // owns the device and optional snapshot while the C helper drains readers. unsafe { asahi_sysfs_unregister(dev.cast(), handle) }; } From d4f2ff81de0c83043c151a71a58840db7a916753 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:22:46 +1100 Subject: [PATCH 1151/1579] rust: refresh examples for the Asahi kernel APIs Kernel doctests still use standard-library vectors and strings, the old resource-based IoMem constructor, and a fallible memset call. Adapt the examples to KVec, IoRequest, pinned devres initialization and the current whole-map memset API so they compile and run in the kernel test harness. Mark the declaration example for the crate-private flags macro as an internal illustration. External doctest crates cannot import that macro; keep its visibility and the runtime APIs unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/alloc/kvec.rs | 24 ++++++++++++++---------- rust/kernel/alloc/kvec/drain.rs | 11 +++++++---- rust/kernel/io.rs | 12 ++++++------ rust/kernel/iosys_map.rs | 11 ++--------- rust/kernel/types.rs | 5 ++++- 5 files changed, 33 insertions(+), 30 deletions(-) diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index da9d95be1fc81..29b30930cc08a 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -774,14 +774,14 @@ where /// # Examples /// /// ``` - /// let mut v = vec![1, 2, 3]; - /// let u: Vec<_> = v.drain(1..).collect(); + /// let mut v = kernel::kvec![1, 2, 3]?; + /// assert!(v.drain(1..).eq([2, 3])); /// assert_eq!(v, &[1]); - /// assert_eq!(u, &[2, 3]); /// /// // A full range clears the vector, like `clear()` does /// v.drain(..); /// assert_eq!(v, &[]); + /// # Ok::<(), Error>(()) /// ``` pub fn drain(&mut self, range: R) -> Drain<'_, T, A> where @@ -1710,25 +1710,29 @@ impl TryFrom> for [T; N] { /// # Examples /// /// ``` - /// assert_eq!(vec![1, 2, 3].try_into(), Ok([1, 2, 3])); - /// assert_eq!(>::new().try_into(), Ok([])); + /// assert_eq!(kernel::kvec![1, 2, 3]?.try_into(), Ok([1, 2, 3])); + /// assert_eq!(KVec::::new().try_into(), Ok([])); + /// # Ok::<(), Error>(()) /// ``` /// /// If the length doesn't match, the input comes back in `Err`: /// ``` - /// let r: Result<[i32; 4], _> = (0..10).collect::>().try_into(); - /// assert_eq!(r, Err(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9])); + /// let r: Result<[i32; 4], _> = kernel::kvec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]?.try_into(); + /// assert_eq!(r, Err(kernel::kvec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]?)); + /// # Ok::<(), Error>(()) /// ``` /// /// If you're fine with just getting a prefix of the `Vec`, /// you can call [`.truncate(N)`](Vec::truncate) first. /// ``` - /// let mut v = String::from("hello world").into_bytes(); - /// v.sort(); + /// let mut v = KVec::new(); + /// v.extend_from_slice(b"hello world", GFP_KERNEL)?; + /// v.sort_unstable(); /// v.truncate(2); - /// let [a, b]: [_; 2] = v.try_into().unwrap(); + /// let [a, b]: [_; 2] = v.try_into().map_err(|_| EINVAL)?; /// assert_eq!(a, b' '); /// assert_eq!(b, b'd'); + /// # Ok::<(), Error>(()) /// ``` fn try_from(mut vec: Vec) -> Result<[T; N], Vec> { if vec.len() != N { diff --git a/rust/kernel/alloc/kvec/drain.rs b/rust/kernel/alloc/kvec/drain.rs index 035878fd11284..381c07882c935 100644 --- a/rust/kernel/alloc/kvec/drain.rs +++ b/rust/kernel/alloc/kvec/drain.rs @@ -22,8 +22,10 @@ use super::{Allocator, Vec}; /// # Example /// /// ``` -/// let mut v = vec![0, 1, 2]; -/// let iter: std::vec::Drain<'_, _> = v.drain(..); +/// let mut v = kernel::kvec![0, 1, 2]?; +/// let iter = v.drain(..); +/// assert_eq!(iter.as_slice(), &[0, 1, 2]); +/// # Ok::<(), Error>(()) /// ``` // #[stable(feature = "drain", since = "1.6.0")] pub struct Drain< @@ -54,11 +56,12 @@ impl<'a, T, A: Allocator> Drain<'a, T, A> { /// # Examples /// /// ``` - /// let mut vec = vec!['a', 'b', 'c']; + /// let mut vec = kernel::kvec!['a', 'b', 'c']?; /// let mut drain = vec.drain(..); /// assert_eq!(drain.as_slice(), &['a', 'b', 'c']); - /// let _ = drain.next().unwrap(); + /// assert_eq!(drain.next(), Some('a')); /// assert_eq!(drain.as_slice(), &['b', 'c']); + /// # Ok::<(), Error>(()) /// ``` #[must_use] // #[stable(feature = "vec_drain_as_slice", since = "1.46.0")] diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 1856232ef4ace..ceaf4b90b9c53 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -275,14 +275,14 @@ pub trait Io { /// /// # Examples /// ``` - /// use kernel::io::mem::IoMem; - /// use kernel::io::mem::Resource; + /// use kernel::{device::Core, io::{mem::IoMem, Io}, platform, prelude::*}; /// - /// fn test(device: &Device, res: Resource) -> Result { + /// fn test(device: &platform::Device) -> Result { /// // Create an i/o memory block of at least 100 bytes. - /// let devres_mem = IoMem::<100>::new(res, device)?; - /// // aquire access to memory block - /// let mem = devres_mem.try_access()?; + /// let request = device.io_request_by_index(0).ok_or(ENODEV)?; + /// let devres_mem = KBox::pin_init(IoMem::<100>::new(request), GFP_KERNEL)?; + /// // Acquire access to the memory block. + /// let mem = devres_mem.access(device.as_ref())?; /// /// let mut buffer: [u8; 32] = [0; 32]; /// diff --git a/rust/kernel/iosys_map.rs b/rust/kernel/iosys_map.rs index ef31660eefaca..b657455f91bae 100644 --- a/rust/kernel/iosys_map.rs +++ b/rust/kernel/iosys_map.rs @@ -186,14 +186,7 @@ impl<'a, T: AsBytes + FromBytes> IoSysMapRef<'a, T> { Ok(()) } - /// Memset the region starting from `offset`. - /// - /// `offset` and `len` are in units of `T`, not the number of bytes. - /// - /// This function can return the following errors: - /// - /// * [`EOVERFLOW`] if calculating the length of the slice results in an overflow. - /// * [`EINVAL`] if the slice would go out of bounds of the memory region. + /// Fill the entire mapped region with `value`. /// /// # Examples /// @@ -204,7 +197,7 @@ impl<'a, T: AsBytes + FromBytes> IoSysMapRef<'a, T> { /// # let mut map = tests::VecIoSysMap::new(&[0u8; 3])?; /// # { /// # let mut map = map.get(); - /// map.memset(7)?; // (now [7, 7, 7]) + /// map.memset(7); // (now [7, 7, 7]) /// # } /// # /// # map.assert_eq(&[7, 7, 7]); diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index c57ddb08d127c..05210d05dbf24 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -559,7 +559,10 @@ pub const NotThreadSafe: NotThreadSafe = PhantomData; /// /// # Examples /// -/// ``` +/// This macro is crate-private, so these declarations are only available within +/// the `kernel` crate. +/// +/// ```ignore /// declare_flags_type! { /// /// Flags to be used for foo. /// pub struct FooFlags(u32); From caa00345cc1c397e2e09c47569b2335eb9543618 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:31:14 +1100 Subject: [PATCH 1152/1579] ASoC: apple: t8140-aop-audio: use infallible conversions in tests The volume table contains u32 gains. Widen them with From when checking the speaker ceiling and volume steps so the KUnit build passes Clippy's cast_lossless lint. Keep the test values and assertions unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index b63c6c8baa084..6a40c8baf5877 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -2892,7 +2892,7 @@ mod tests { #[test] fn the_top_of_the_volume_range_is_the_amplifier_full_scale() { assert_eq!( - SPKR_VOLUME_Q31[SPKR_VOLUME_MAX as usize] as i64, + i64::from(SPKR_VOLUME_Q31[SPKR_VOLUME_MAX as usize]), SPKR_CEILING_Q31 ); assert_eq!(SPKR_CEILING_Q31, 1 << 30); @@ -2903,7 +2903,10 @@ mod tests { fn the_volume_table_falls_in_half_db_steps_to_mute() { assert_eq!(SPKR_VOLUME_Q31[0], 0); for i in 2..=SPKR_VOLUME_MAX as usize { - let (lo, hi) = (SPKR_VOLUME_Q31[i - 1] as u128, SPKR_VOLUME_Q31[i] as u128); + let (lo, hi) = ( + u128::from(SPKR_VOLUME_Q31[i - 1]), + u128::from(SPKR_VOLUME_Q31[i]), + ); assert!(hi > lo); // hi / lo == 10^(0.5/20) to within the table's rounding let expect = lo * HALF_DB_PPB / 1_000_000_000; @@ -2913,7 +2916,7 @@ mod tests { #[test] fn the_locked_level_is_20_db_below_full_scale() { - let locked = SPKR_VOLUME_Q31[LOCKED_VOLUME] as i64; + let locked = i64::from(SPKR_VOLUME_Q31[LOCKED_VOLUME]); // 10 * locked == ceiling to within the table's rounding (0.1 ppm) assert!((10 * locked - SPKR_CEILING_Q31).abs() <= 10); } From bd7739d6232e818ef7cf30acbcc50d24723e061c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:06:03 +1100 Subject: [PATCH 1153/1579] mfd: macsmc: Reject short fixed-width replies The raw SMC interface reports how many bytes firmware returned. Copying the requested width instead can expose stale SRAM data, while fixed-width readers may consume bytes that firmware never supplied. Copy only returned bytes and reject oversized replies. Make every fixed-width reader reject short replies with -EIO and clear its value before returning. Keep capacity-based identity string reads unchanged. Fixes: e038d985c982 ("mfd: Add Apple Silicon System Management Controller") Cc: stable@vger.kernel.org Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/mfd/macsmc.c | 15 ++++++++++----- drivers/power/supply/macsmc-power.c | 4 ++++ drivers/rtc/rtc-macsmc.c | 8 ++++++-- include/linux/mfd/macsmc.h | 16 ++++++++++++++-- 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 8cdc59a7dd88c..e7ad580f58fca 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -157,6 +157,9 @@ static int apple_smc_rw_locked(struct apple_smc *smc, smc_key key, if (ret < 0) return ret; + if (rsize && ret > rsize) + return -EIO; + if (rsize) { /* * Small data <= 4 bytes is returned as part of the reply @@ -165,9 +168,9 @@ static int apple_smc_rw_locked(struct apple_smc *smc, smc_key key, * Device memory. */ if (rsize <= 4) - memcpy(rbuf, &rdata, rsize); + memcpy(rbuf, &rdata, ret); else - memcpy_fromio(rbuf, smc->shmem.iomem, rsize); + memcpy_fromio(rbuf, smc->shmem.iomem, ret); } return ret; @@ -415,7 +418,7 @@ static int apple_smc_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct device_node *np; struct apple_smc *smc; - __be32 count; + __be32 count = 0; int ret; smc = devm_kzalloc(dev, sizeof(*smc), GFP_KERNEL); @@ -467,8 +470,10 @@ static int apple_smc_probe(struct platform_device *pdev) BLOCKING_INIT_NOTIFIER_HEAD(&smc->event_handlers); ret = apple_smc_read(smc, SMC_KEY(#KEY), &count, sizeof(count)); - if (ret >= 0 && ret != sizeof(count)) - ret = -EINVAL; + if (ret >= 0 && ret != sizeof(count)) { + count = 0; + ret = -EIO; + } if (ret < 0) return dev_err_probe(smc->dev, ret, "Failed to get key count"); smc->key_count = be32_to_cpu(count); diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index 7996b8d9cebd4..a7fae0bfff4cc 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -334,6 +334,10 @@ static int macsmc_battery_get_charge_behaviour(struct macsmc_power *power) ret = apple_smc_read(power->smc, SMC_KEY(CHTE), chte_buf, 4); if (ret < 0) return ret; + if (ret != sizeof(chte_buf)) { + memset(chte_buf, 0, sizeof(chte_buf)); + return -EIO; + } if (chte_buf[0] == 0x01) return POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE; diff --git a/drivers/rtc/rtc-macsmc.c b/drivers/rtc/rtc-macsmc.c index 1c1c529df8296..a28dd7c577bde 100644 --- a/drivers/rtc/rtc-macsmc.c +++ b/drivers/rtc/rtc-macsmc.c @@ -74,8 +74,10 @@ static int macsmc_rtc_get_time(struct device *dev, struct rtc_time *tm) ret = apple_smc_read(rtc->smc, SMC_KEY(CLKM), &ctr, RTC_BYTES); if (ret < 0) return ret; - if (ret != RTC_BYTES) + if (ret != RTC_BYTES) { + ctr = 0; return -EIO; + } if (rtc->offset_in_seconds) return macsmc_rtc_get_time_secs(rtc, ctr, tm); @@ -107,8 +109,10 @@ static int macsmc_rtc_set_time(struct device *dev, struct rtc_time *tm) ret = apple_smc_read(rtc->smc, SMC_KEY(CLKM), &ctr, RTC_BYTES); if (ret < 0) return ret; - if (ret != RTC_BYTES) + if (ret != RTC_BYTES) { + ctr = 0; return -EIO; + } if (rtc->offset_in_seconds) { __le64 off_secs; diff --git a/include/linux/mfd/macsmc.h b/include/linux/mfd/macsmc.h index cc09ecce0df74..fbc047b00afd8 100644 --- a/include/linux/mfd/macsmc.h +++ b/include/linux/mfd/macsmc.h @@ -229,7 +229,13 @@ static inline bool apple_smc_key_exists(struct apple_smc *smc, smc_key key) static inline int apple_smc_read_##type(struct apple_smc *smc, smc_key key, type *p) \ { \ int ret = apple_smc_read(smc, key, p, sizeof(*p)); \ - return (ret < 0) ? ret : ((ret != sizeof(*p)) ? -EINVAL : 0); \ + if (ret < 0) \ + return ret; \ + if (ret != sizeof(*p)) { \ + *p = 0; \ + return -EIO; \ + } \ + return 0; \ } \ static inline int apple_smc_write_##type(struct apple_smc *smc, smc_key key, type p) \ { \ @@ -243,7 +249,13 @@ static inline bool apple_smc_key_exists(struct apple_smc *smc, smc_key key) type w, type *r) \ { \ int ret = apple_smc_rw(smc, key, &w, sizeof(w), r, sizeof(*r)); \ - return (ret < 0) ? ret : ((ret != sizeof(*r)) ? -EINVAL : 0); \ + if (ret < 0) \ + return ret; \ + if (ret != sizeof(*r)) { \ + *r = 0; \ + return -EIO; \ + } \ + return 0; \ } APPLE_SMC_TYPE_OPS(u64) From 5b62d0c81387e8d0d68f2a88244eeb7a618f9ec1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:06:33 +1100 Subject: [PATCH 1154/1579] mfd: macsmc: Preserve the key output on lookup failure A failed key lookup may leave its output unwritten, but the unconditional byte swap reads and changes that output anyway. This can read an uninitialized caller value and changes an existing key on error. Receive the wire key in a temporary and publish it only after a successful command. Fixes: e038d985c982 ("mfd: Add Apple Silicon System Management Controller") Cc: stable@vger.kernel.org Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/mfd/macsmc.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index e7ad580f58fca..92f1b987bf888 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -203,11 +203,14 @@ EXPORT_SYMBOL(apple_smc_rw); int apple_smc_get_key_by_index(struct apple_smc *smc, int index, smc_key *key) { + u32 data; int ret; - ret = apple_smc_cmd(smc, SMC_MSG_GET_KEY_BY_INDEX, index, 0, 0, key); + ret = apple_smc_cmd(smc, SMC_MSG_GET_KEY_BY_INDEX, index, 0, 0, &data); + if (ret < 0) + return ret; - *key = swab32(*key); + *key = swab32(data); return ret; } EXPORT_SYMBOL(apple_smc_get_key_by_index); From ec34fb1f2835b291f978bd091ed73347cba7ceae Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:06:49 +1100 Subject: [PATCH 1155/1579] mfd: macsmc: Reject out-of-range key indices The GPIO key-range lookup asks for key_count - 1. A malformed zero key count therefore reaches this interface as -1 and is converted to an unsigned mailbox argument. Indices at or above the advertised key count can likewise be sent to the coprocessor. Validate the index against the key count before issuing the command. Fixes: e038d985c982 ("mfd: Add Apple Silicon System Management Controller") Cc: stable@vger.kernel.org Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/mfd/macsmc.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 92f1b987bf888..5dd8f8bc21956 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -206,6 +206,9 @@ int apple_smc_get_key_by_index(struct apple_smc *smc, int index, smc_key *key) u32 data; int ret; + if (index < 0 || index >= smc->key_count) + return -EINVAL; + ret = apple_smc_cmd(smc, SMC_MSG_GET_KEY_BY_INDEX, index, 0, 0, &data); if (ret < 0) return ret; From a494d618f5a9af40ae466166176ae36476b4638a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:06:59 +1100 Subject: [PATCH 1156/1579] mfd: macsmc: Hold the mutex while decoding key metadata apple_smc_cmd() releases the command mutex before key metadata is copied from shared SRAM. Another key transaction can overwrite that SRAM between the reply and the copy, returning metadata for a different key. Hold the mutex through the command and metadata copy. Fixes: e038d985c982 ("mfd: Add Apple Silicon System Management Controller") Cc: stable@vger.kernel.org Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/mfd/macsmc.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 5dd8f8bc21956..cf25327a7bd8b 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -223,7 +223,9 @@ int apple_smc_get_key_info(struct apple_smc *smc, smc_key key, struct apple_smc_ u8 key_info[6]; int ret; - ret = apple_smc_cmd(smc, SMC_MSG_GET_KEY_INFO, key, 0, 0, NULL); + guard(mutex)(&smc->mutex); + + ret = apple_smc_cmd_locked(smc, SMC_MSG_GET_KEY_INFO, key, 0, 0, NULL); if (ret >= 0 && info) { memcpy_fromio(key_info, smc->shmem.iomem, sizeof(key_info)); info->size = key_info[0]; From 87314d48cc080bcf535291dc636f1ec6164a06f9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:41:32 +1100 Subject: [PATCH 1157/1579] drm/apple: Test firmware message bounds with KUnit Exercise AFKv2 frame and command lengths, owned DMA spans, callback capacities and reply correlation, plus short EPIC announcements and replies and invalid command slots. Keep the existing ring payload predicate in a helper so it can be tested without a live coprocessor. Test the H17P IOMFB receive dispatcher with short records, invalid contexts, excessive nesting, malformed tags and overflowing lengths. Include a valid callback to check that accepted messages still reach their handler. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 10 ++ drivers/gpu/drm/apple/Makefile | 2 + drivers/gpu/drm/apple/afk-parser-test.c | 106 ++++++++++++ drivers/gpu/drm/apple/afk.c | 8 +- drivers/gpu/drm/apple/afk.h | 5 + drivers/gpu/drm/apple/afkv2-test.c | 195 ++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb-parser-test.c | 154 +++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 4 + 8 files changed, 482 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/apple/afk-parser-test.c create mode 100644 drivers/gpu/drm/apple/afkv2-test.c create mode 100644 drivers/gpu/drm/apple/iomfb-parser-test.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 3d2a83b1369c1..8445d82991a93 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -44,3 +44,13 @@ config DRM_APPLE_KUNIT_TEST Also test the Type-C modeset, hotplug and service ownership state, the system resume of a CRTC whose mode is no longer admissible, and when an external display is ready to be reported to userspace. + +config DRM_APPLE_PARSER_KUNIT_TEST + bool "KUnit tests for Apple firmware message parsers" if !KUNIT_ALL_TESTS + depends on DRM_APPLE && KUNIT=y + default KUNIT_ALL_TESTS + help + Build tests for AFKv2 lengths, DMA ownership and reply correlation, + legacy EPIC replies, and H17P IOMFB callback envelopes and sizes. + The tests use allocated buffers and fake devices and do not require + an Apple display controller or any firmware. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index be9ada873650c..254afcbdfde90 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -30,3 +30,5 @@ obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_backlight_test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp-announce-test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk-service-test.o + +obj-$(CONFIG_DRM_APPLE_PARSER_KUNIT_TEST) += afkv2-test.o diff --git a/drivers/gpu/drm/apple/afk-parser-test.c b/drivers/gpu/drm/apple/afk-parser-test.c new file mode 100644 index 0000000000000..fd8c5cb0c26e5 --- /dev/null +++ b/drivers/gpu/drm/apple/afk-parser-test.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Included by afk.c to exercise the EPIC receive handlers. */ +#include +#include + +static int afk_parser_test_init(struct kunit *test) +{ + struct apple_dcp_afkep *ep; + struct apple_dcp *dcp; + + ep = kunit_kzalloc(test, sizeof(*ep), GFP_KERNEL); + dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + if (!ep || !dcp) + return -ENOMEM; + dcp->dev = kunit_device_register(test, "afk-parser"); + if (IS_ERR(dcp->dev)) + return PTR_ERR(dcp->dev); + dcp->fw_compat = DCP_FIRMWARE_H17P; + ep->dcp = dcp; + ep->num_channels = 1; + ep->services[0].channel = 1; + ep->services[0].enabled = true; + spin_lock_init(&ep->services[0].lock); + test->priv = ep; + return 0; +} + +static void afk_compact_header_test(struct kunit *test) +{ + u8 record[sizeof(struct epic_hdr) + sizeof(struct epic_sub_hdr_h17p)] = {}; + unsigned int len; + + for (len = 0; len < sizeof(record); len++) + KUNIT_EXPECT_FALSE(test, afk_validate_h17p_header(NULL, len)); + put_unaligned_le32(sizeof(record) - 8, record + 4); + KUNIT_EXPECT_TRUE(test, afk_validate_h17p_header(record, sizeof(record))); + KUNIT_EXPECT_FALSE(test, afk_validate_h17p_header(record, sizeof(record) + 1)); + put_unaligned_le32(U32_MAX, record + 4); + KUNIT_EXPECT_FALSE(test, afk_validate_h17p_header(record, sizeof(record))); +} + +static void afk_short_epic_test(struct kunit *test) +{ + struct apple_dcp_afkep *ep = test->priv; + unsigned int len; + + for (len = 0; len < sizeof(struct epic_hdr) + sizeof(struct epic_sub_hdr_h17p); len++) + afk_recv_handle(ep, 0, EPIC_TYPE_NOTIFY, NULL, len); + for (len = 0; len < 32; len++) + afk_recv_handle_init(ep, 2, NULL, len); + KUNIT_EXPECT_EQ(test, ep->num_channels, 1U); +} + +static void afk_short_reply_test(struct kunit *test) +{ + struct apple_dcp_afkep *ep = test->priv; + struct epic_cmd_info *pending = &ep->services[0].cmds[0]; + unsigned int len; + + set_bit(0, ep->services[0].cmd_map); + pending->tag = 0x200; + for (len = 0; len < sizeof(struct epic_cmd); len++) { + afk_recv_handle_reply(ep, 1, 0x200, NULL, len); + KUNIT_EXPECT_FALSE(test, pending->done); + } +} + +static void afk_reply_index_test(struct kunit *test) +{ + struct apple_dcp_afkep *ep = test->priv; + struct epic_cmd_info *pending = &ep->services[0].cmds[0]; + struct epic_cmd reply = { .retcode = cpu_to_le32(7) }; + + set_bit(0, ep->services[0].cmd_map); + pending->tag = 0x200; + afk_recv_handle_reply(ep, 1, MAX_PENDING_CMDS, &reply, sizeof(reply)); + KUNIT_EXPECT_FALSE(test, pending->done); + afk_recv_handle_reply(ep, 1, 0x2ff, &reply, sizeof(reply)); + KUNIT_EXPECT_FALSE(test, pending->done); + afk_recv_handle_reply(ep, 1, 0x300, &reply, sizeof(reply)); + KUNIT_EXPECT_FALSE(test, pending->done); + afk_recv_handle_reply(ep, 2, 0x200, NULL, sizeof(reply)); + KUNIT_EXPECT_FALSE(test, pending->done); + afk_recv_handle_reply(ep, 1, 0x200, &reply, sizeof(reply)); + KUNIT_EXPECT_TRUE(test, pending->done); + KUNIT_EXPECT_EQ(test, pending->retcode, 7U); + reply.retcode = cpu_to_le32(9); + afk_recv_handle_reply(ep, 1, 0x200, &reply, sizeof(reply)); + KUNIT_EXPECT_EQ(test, pending->retcode, 7U); +} + +static struct kunit_case afk_parser_cases[] = { + KUNIT_CASE(afk_compact_header_test), + KUNIT_CASE(afk_short_epic_test), + KUNIT_CASE(afk_short_reply_test), + KUNIT_CASE(afk_reply_index_test), + {} +}; + +static struct kunit_suite afk_parser_suite = { + .name = "apple-epic-parser", + .init = afk_parser_test_init, + .test_cases = afk_parser_cases, +}; + +kunit_test_suite(afk_parser_suite); diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 6ca27849346f8..9de9e61b3f69b 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1220,7 +1220,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) * the current dummy queue entry and we have to advance to the next one * which will contain the real data. */ - if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { + if (!afk_ring_payload_fits(rptr, size, ep->rxbfr.bufsz)) { rptr = 0; hdr = ep->rxbfr.buf + rptr; magic = le32_to_cpu(hdr->magic); @@ -1235,7 +1235,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) } } - if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { + if (!afk_ring_payload_fits(rptr, size, ep->rxbfr.bufsz)) { dev_warn(ep->dcp->dev, "AFK[ep:%02x]: queue entry out of bounds: 0x%lx > 0x%lx\n", ep->endpoint, rptr + size + sizeof(*hdr), ep->rxbfr.bufsz); @@ -2068,3 +2068,7 @@ static void afk_remove_service_debugfs(struct apple_epic_service *srv) } #endif + +#if IS_ENABLED(CONFIG_DRM_APPLE_PARSER_KUNIT_TEST) +#include "afk-parser-test.c" +#endif diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 0a12ecda982df..b9cbe586370a9 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -266,6 +266,11 @@ struct afk_qe { u8 data[]; }; +static inline bool afk_ring_payload_fits(u32 ptr, u32 bytes, size_t bufsz) +{ + return ptr + bytes + sizeof(struct afk_qe) <= bufsz; +} + struct epic_hdr { u8 version; __le16 seq; diff --git a/drivers/gpu/drm/apple/afkv2-test.c b/drivers/gpu/drm/apple/afkv2-test.c new file mode 100644 index 0000000000000..646e0fd3c5471 --- /dev/null +++ b/drivers/gpu/drm/apple/afkv2-test.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "afk.h" +#include "afkv2-codec.h" + +static void afkv2_frame_bounds_test(struct kunit *test) +{ + u8 record[AFKV2_HEADER_SIZE + 8] = {}; + struct afkv2_frame frame; + size_t size; + + for (size = 0; size < AFKV2_HEADER_SIZE; size++) + KUNIT_EXPECT_EQ(test, afkv2_decode_frame(record, size, false, &frame), + -EINVAL); + KUNIT_EXPECT_EQ(test, afkv2_decode_frame(NULL, sizeof(record), false, &frame), + -EINVAL); + KUNIT_EXPECT_EQ(test, afkv2_decode_frame(record, SIZE_MAX, false, &frame), + -E2BIG); + afkv2_put32(record + 4, U32_MAX); + KUNIT_EXPECT_EQ(test, afkv2_decode_frame(record, sizeof(record), false, &frame), + -EINVAL); + afkv2_put32(record + 4, 15); + KUNIT_EXPECT_EQ(test, afkv2_decode_frame(record, sizeof(record), true, &frame), + -EINVAL); + afkv2_put32(record + 4, 16); + KUNIT_EXPECT_EQ(test, afkv2_decode_frame(record, sizeof(record), false, &frame), + -EINVAL); + KUNIT_ASSERT_EQ(test, afkv2_decode_frame(record, sizeof(record), true, &frame), 0); + KUNIT_EXPECT_EQ(test, frame.payload_size, 0U); + KUNIT_EXPECT_EQ(test, frame.padding_size, 8U); + afkv2_put32(record + 4, sizeof(record) - 8); + KUNIT_ASSERT_EQ(test, afkv2_decode_frame(record, sizeof(record), false, &frame), 0); + KUNIT_EXPECT_EQ(test, frame.payload_size, 8U); + KUNIT_EXPECT_PTR_EQ(test, frame.payload, record + AFKV2_HEADER_SIZE); + record[17] = AFKV2_RESPONSE + 1; + KUNIT_EXPECT_EQ(test, afkv2_decode_frame(record, sizeof(record), false, &frame), + -EPROTO); +} + +static void afkv2_command_bounds_test(struct kunit *test) +{ + u8 payload[AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE] = { 1 }; + struct afkv2_command command; + size_t size; + + for (size = 0; size < sizeof(payload); size++) + KUNIT_EXPECT_EQ(test, afkv2_decode_command(payload, size, &command), -EINVAL); + KUNIT_ASSERT_EQ(test, afkv2_decode_command(payload, sizeof(payload), &command), 0); + afkv2_put64(payload + 8, U64_MAX); + afkv2_put32(payload + 24, 2); + KUNIT_EXPECT_EQ(test, afkv2_decode_command(payload, sizeof(payload), &command), + -EINVAL); + afkv2_put32(payload + 24, 1); + KUNIT_EXPECT_EQ(test, afkv2_decode_command(payload, sizeof(payload), &command), 0); + afkv2_put64(payload + 8, 0); + KUNIT_EXPECT_EQ(test, afkv2_decode_command(payload, sizeof(payload), &command), + -EINVAL); + afkv2_put64(payload + 8, 0x1000); + afkv2_put32(payload + 24, AFKV2_MAX_PAYLOAD + 1); + KUNIT_EXPECT_EQ(test, afkv2_decode_command(payload, sizeof(payload), &command), + -EINVAL); + payload[0] = 2; + KUNIT_EXPECT_EQ(test, afkv2_decode_command(payload, sizeof(payload), &command), + -EINVAL); + payload[0] = 0; + payload[2] = 1; + KUNIT_EXPECT_EQ(test, afkv2_decode_command(payload, sizeof(payload), &command), + -EINVAL); +} + +static void afkv2_owned_span_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, afkv2_owned_span(0x1000, 0x100, 0x1000, 0x100)); + KUNIT_EXPECT_TRUE(test, afkv2_owned_span(0x1000, 0x100, 0x10ff, 1)); + KUNIT_EXPECT_FALSE(test, afkv2_owned_span(0x1000, 0x100, 0xfff, 1)); + KUNIT_EXPECT_FALSE(test, afkv2_owned_span(0x1000, 0x100, 0x10ff, 2)); + KUNIT_EXPECT_FALSE(test, afkv2_owned_span(0x1000, 0x100, 0x1000, SIZE_MAX)); + KUNIT_EXPECT_FALSE(test, afkv2_owned_span(U64_MAX, 2, U64_MAX, 1)); + KUNIT_EXPECT_TRUE(test, afkv2_owned_span(U64_MAX, 1, U64_MAX, 1)); +} + +static void afkv2_service_reply_bounds_test(struct kunit *test) +{ + u8 reply[68] = {}; + size_t size = SIZE_MAX; + unsigned int len; + + afkv2_put16(reply + 2, 3); + afkv2_put32(reply + 4, 8); + afkv2_put32(reply + 8, 4); + afkv2_put32(reply + 12, 0x69706378); + for (len = 0; len < 64; len++) + KUNIT_EXPECT_EQ(test, afkv2_service_reply_size(reply, len, 3, 8, 4, &size), + -EPROTO); + KUNIT_ASSERT_EQ(test, afkv2_service_reply_size(reply, sizeof(reply), 3, 8, 4, + &size), 0); + KUNIT_EXPECT_EQ(test, size, 4U); + KUNIT_EXPECT_EQ(test, afkv2_service_reply_size(reply, sizeof(reply), 3, 8, 3, + &size), -EMSGSIZE); + afkv2_put32(reply + 8, U32_MAX); + KUNIT_EXPECT_EQ(test, afkv2_service_reply_size(reply, sizeof(reply), 3, 8, + SIZE_MAX, &size), -EMSGSIZE); +} + +static void afkv2_response_bounds_test(struct kunit *test) +{ + struct afkv2_frame frame = { .category = AFKV2_RESPONSE, .interface = 4, .type = 9 }; + struct afkv2_command command = { + .id = 0xf1, .oob = true, .rx_address = 0x1000, .rx_size = 64, + .tx_address = 0x2000, .tx_size = 32, + }; + + KUNIT_EXPECT_EQ(test, afkv2_check_response(&frame, &command, 4, 9, 0xf1, + 0x1000, 64, 0x2000, 32), 0); + command.rx_size++; + KUNIT_EXPECT_EQ(test, afkv2_check_response(&frame, &command, 4, 9, 0xf1, + 0x1000, 64, 0x2000, 32), -EPROTO); + command.rx_size--; + command.id = 0x01; + KUNIT_EXPECT_EQ(test, afkv2_check_response(&frame, &command, 4, 9, 0xf1, + 0x1000, 64, 0x2000, 32), -EPROTO); + command.id = 0xf1; + command.rx_address++; + KUNIT_EXPECT_EQ(test, afkv2_check_response(&frame, &command, 4, 9, 0xf1, + 0x1000, 64, 0x2000, 32), -EPROTO); +} + +static void afkv2_callback_bounds_test(struct kunit *test) +{ + struct afkv2_command command = { + .oob = true, .rx_size = 64, .tx_size = 32, .value = 64, + }; + size_t request = SIZE_MAX, response = SIZE_MAX; + + KUNIT_ASSERT_EQ(test, afkv2_callback_lengths(&command, 32, 256, &request, &response), 0); + KUNIT_EXPECT_EQ(test, request, 32U); + KUNIT_EXPECT_EQ(test, response, 64U); + command.tx_size = 31; + KUNIT_EXPECT_EQ(test, afkv2_callback_lengths(&command, 32, 256, &request, &response), + -EINVAL); + command.tx_size = 32; + command.rx_size = 65; + KUNIT_EXPECT_EQ(test, afkv2_callback_lengths(&command, 32, 256, &request, &response), + -EINVAL); + command.value = U32_MAX; + command.rx_size = 257; + KUNIT_EXPECT_EQ(test, afkv2_callback_lengths(&command, 32, 256, &request, &response), + -EINVAL); +} + +static void afkv2_inline_response_test(struct kunit *test) +{ + struct afkv2_frame frame = { .category = AFKV2_RESPONSE, .interface = 4, .type = 9 }; + struct afkv2_command command = { .id = 3, .inline_size = 64 }; + + KUNIT_EXPECT_EQ(test, afkv2_check_inline_response(&frame, &command, 4, 9, 3, 64), 0); + command.inline_size++; + KUNIT_EXPECT_EQ(test, afkv2_check_inline_response(&frame, &command, 4, 9, 3, 64), + -EPROTO); + command.inline_size--; + frame.token = 1; + KUNIT_EXPECT_EQ(test, afkv2_check_inline_response(&frame, &command, 4, 9, 3, 64), + -EPROTO); +} + +static void afk_ring_payload_bounds_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, afk_ring_payload_fits(0x40, 0x100, 0x1000)); + KUNIT_EXPECT_TRUE(test, afk_ring_payload_fits(0xfc0, 0x30, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0xfc0, 0x31, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0x1000, 0, 0x1000)); +} + +static struct kunit_case afkv2_parser_cases[] = { + KUNIT_CASE(afk_ring_payload_bounds_test), + KUNIT_CASE(afkv2_frame_bounds_test), + KUNIT_CASE(afkv2_command_bounds_test), + KUNIT_CASE(afkv2_owned_span_test), + KUNIT_CASE(afkv2_service_reply_bounds_test), + KUNIT_CASE(afkv2_response_bounds_test), + KUNIT_CASE(afkv2_callback_bounds_test), + KUNIT_CASE(afkv2_inline_response_test), + {} +}; + +static struct kunit_suite afkv2_parser_suite = { + .name = "apple-afkv2-parser", + .test_cases = afkv2_parser_cases, +}; + +kunit_test_suite(afkv2_parser_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/iomfb-parser-test.c b/drivers/gpu/drm/apple/iomfb-parser-test.c new file mode 100644 index 0000000000000..e829ff43bda4a --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-parser-test.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Included by iomfb.c to exercise the receive dispatcher itself. */ +#include +#include + +static int iomfb_parser_test_init(struct kunit *test) +{ + struct apple_dcp *dcp; + + dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + if (!dcp) + return -ENOMEM; + dcp->dev = kunit_device_register(test, "iomfb-parser"); + if (IS_ERR(dcp->dev)) + return PTR_ERR(dcp->dev); + dcp->fw_compat = DCP_FIRMWARE_H17P; + test->priv = dcp; + return 0; +} + +static void iomfb_tag_test(struct kunit *test) +{ + char tag[] = "411D"; + unsigned int i; + + KUNIT_EXPECT_EQ(test, dcp_parse_tag(tag), 114); + for (i = 0; i < sizeof(tag) - 1; i++) { + char saved = tag[i]; + + tag[i] = 0xff; + KUNIT_EXPECT_EQ(test, dcp_parse_tag(tag), -EINVAL); + tag[i] = saved; + } + tag[0] = '/'; + KUNIT_EXPECT_EQ(test, dcp_parse_tag(tag), -EINVAL); + tag[0] = ':'; + KUNIT_EXPECT_EQ(test, dcp_parse_tag(tag), -EINVAL); +} + +static void iomfb_short_callback_test(struct kunit *test) +{ + struct apple_dcp *dcp = test->priv; + unsigned int len; + + /* A NULL record must never be read before the envelope is checked. */ + for (len = 0; len < sizeof(struct dcp_packet_header); len++) { + dcp->crashed = false; + dcpep_handle_cb(dcp, DCP_CONTEXT_CB, NULL, len, 0); + KUNIT_EXPECT_TRUE(test, dcp->crashed); + KUNIT_EXPECT_EQ(test, dcp->ch_cb.depth, 0); + } +} + +static void iomfb_callback_envelope_test(struct kunit *test) +{ + struct apple_dcp *dcp = test->priv; + static const struct { + u32 length; + u16 offset; + u8 depth; + } cases[] = { + { sizeof(struct dcp_packet_header), 0x8000, 0 }, + { sizeof(struct dcp_packet_header), 0x7ff8, 0 }, + { U32_MAX, 0, 0 }, + { sizeof(struct dcp_packet_header), 0, DCP_MAX_CALL_DEPTH }, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cases); i++) { + dcp->crashed = false; + dcp->ch_cb.depth = cases[i].depth; + dcpep_handle_cb(dcp, DCP_CONTEXT_CB, NULL, cases[i].length, cases[i].offset); + KUNIT_EXPECT_TRUE(test, dcp->crashed); + KUNIT_EXPECT_EQ(test, dcp->ch_cb.depth, cases[i].depth); + } + dcp->crashed = false; + dcpep_handle_cb(dcp, 1, NULL, sizeof(struct dcp_packet_header), 0); + KUNIT_EXPECT_TRUE(test, dcp->crashed); +} + +static void iomfb_callback_lengths_test(struct kunit *test) +{ + struct apple_dcp *dcp = test->priv; + struct dcp_packet_header record = { .tag = { '4', '1', '1', 'D' } }; + static const u32 lengths[][2] = { + { U32_MAX, 1 }, { 1, U32_MAX }, { 1, 0 }, { 0, 1 }, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(lengths); i++) { + record.in_len = lengths[i][0]; + record.out_len = lengths[i][1]; + dcp->crashed = false; + dcpep_handle_cb(dcp, DCP_CONTEXT_CB, &record, sizeof(record), 0); + KUNIT_EXPECT_TRUE(test, dcp->crashed); + KUNIT_EXPECT_EQ(test, dcp->ch_cb.depth, 0); + } + memcpy(record.tag, "/11D", 4); + record.in_len = 0; + record.out_len = 0; + dcp->crashed = false; + dcpep_handle_cb(dcp, DCP_CONTEXT_CB, &record, sizeof(record), 0); + KUNIT_EXPECT_TRUE(test, dcp->crashed); +} + +static bool iomfb_parser_callback(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + put_unaligned_le32(0x12345678, out); + return false; +} + +static void iomfb_callback_size_test(struct kunit *test) +{ + struct apple_dcp *dcp = test->priv; + iomfb_cb_handler *handlers; + struct dcp_packet_header *record; + size_t size = sizeof(*record) + 0x1044 + 0x1004; + + handlers = kunit_kcalloc(test, IOMFB_MAX_CB, sizeof(*handlers), GFP_KERNEL); + record = kunit_kzalloc(test, size + 4, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, handlers); + KUNIT_ASSERT_NOT_NULL(test, record); + handlers[114] = iomfb_parser_callback; + dcp->cb_handlers = handlers; + memcpy(record->tag, "411D", 4); + record->in_len = 0x1044; + record->out_len = 0x1008; + dcpep_handle_cb(dcp, DCP_CONTEXT_CB, record, size + 4, 0); + KUNIT_EXPECT_TRUE(test, dcp->crashed); + KUNIT_EXPECT_EQ(test, dcp->ch_cb.depth, 0); + dcp->crashed = false; + record->out_len = 0x1004; + dcpep_handle_cb(dcp, DCP_CONTEXT_CB, record, size, 0); + KUNIT_EXPECT_FALSE(test, dcp->crashed); + KUNIT_EXPECT_EQ(test, dcp->ch_cb.depth, 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)(record + 1) + 0x1044), 0x12345678U); +} + +static struct kunit_case iomfb_parser_cases[] = { + KUNIT_CASE(iomfb_tag_test), + KUNIT_CASE(iomfb_short_callback_test), + KUNIT_CASE(iomfb_callback_envelope_test), + KUNIT_CASE(iomfb_callback_lengths_test), + KUNIT_CASE(iomfb_callback_size_test), + {} +}; + +static struct kunit_suite iomfb_parser_suite = { + .name = "apple-iomfb-parser", + .init = iomfb_parser_test_init, + .test_cases = iomfb_parser_cases, +}; + +kunit_test_suite(iomfb_parser_suite); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 3ab3a23fb6684..afddbc56a3a65 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -2285,3 +2285,7 @@ void iomfb_shutdown(struct apple_dcp *dcp) break; } } + +#if IS_ENABLED(CONFIG_DRM_APPLE_PARSER_KUNIT_TEST) +#include "iomfb-parser-test.c" +#endif From 1df5bd529669987e8804b6ab38dbd1d5175f508d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:41:34 +1100 Subject: [PATCH 1158/1579] HID: dockchannel-hid: Test MTP message bounds with KUnit Exercise the receive handlers with short subheaders, oversized bodies, invalid interface numbers and mismatched acknowledgment identifiers. Check packet checksums and response truncation with canaries, and accept a valid descriptor. Include the tests in the driver to use the actual receive handlers without exporting private implementation details. The fixtures have no transport or HID device and cannot communicate with a coprocessor. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/Kconfig | 10 + .../dockchannel-hid-parser-test.c | 204 ++++++++++++++++++ drivers/hid/dockchannel-hid/dockchannel-hid.c | 16 +- 3 files changed, 226 insertions(+), 4 deletions(-) create mode 100644 drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig index a19b543c066cb..fe2f5b96aa749 100644 --- a/drivers/hid/dockchannel-hid/Kconfig +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -27,3 +27,13 @@ config HID_DOCKCHANNEL_KUNIT_TEST If unsure, say N. endmenu + +config HID_DOCKCHANNEL_PARSER_KUNIT_TEST + bool "KUnit tests for DockChannel HID message parsing" if !KUNIT_ALL_TESTS + depends on HID_DOCKCHANNEL && KUNIT=y + default KUNIT_ALL_TESTS + help + Build tests that pass malformed MTP frames to the initialization, + input-report and acknowledgment handlers. Cover short headers, + oversized lengths, interface indices and acknowledgment identity. + The tests run without a DockChannel or HID device. diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c b/drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c new file mode 100644 index 0000000000000..ef0e8869bd99f --- /dev/null +++ b/drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Included by dockchannel-hid.c to exercise the MTP receive handlers. */ +#include +#include + +static int dchid_parser_test_init(struct kunit *test) +{ + struct dockchannel_hid *dchid; + struct dchid_iface *iface; + + dchid = kunit_kzalloc(test, sizeof(*dchid), GFP_KERNEL); + iface = kunit_kzalloc(test, sizeof(*iface), GFP_KERNEL); + if (!dchid || !iface) + return -ENOMEM; + dchid->dev = kunit_device_register(test, "dchid-parser"); + if (IS_ERR(dchid->dev)) + return PTR_ERR(dchid->dev); + iface->dchid = dchid; + iface->name = "test"; + iface->out_report = 0x10; + iface->out_flags = 0x80; + iface->tx_seq = 7; + init_completion(&iface->out_complete); + dchid->ifaces[1] = iface; + test->priv = iface; + return 0; +} + +static void dchid_short_ack_test(struct kunit *test) +{ + struct dchid_iface *iface = test->priv; + struct dchid_hdr hdr = {}; + + for (hdr.length = 0; hdr.length < sizeof(struct dchid_subhdr); hdr.length++) { + dchid_handle_ack(iface, &hdr, NULL); + KUNIT_EXPECT_FALSE(test, completion_done(&iface->out_complete)); + KUNIT_EXPECT_EQ(test, iface->out_report, 0x10); + } +} + +static void dchid_ack_length_test(struct kunit *test) +{ + struct dchid_iface *iface = test->priv; + struct dchid_hdr hdr = { .length = sizeof(struct dchid_subhdr), .seq = 7 }; + struct dchid_subhdr sub = { .flags = 0x80 }; + + /* A report ID is required even when no response body was requested. */ + dchid_handle_ack(iface, &hdr, &sub); + KUNIT_EXPECT_FALSE(test, completion_done(&iface->out_complete)); + sub.length = U16_MAX; + dchid_handle_ack(iface, &hdr, &sub); + KUNIT_EXPECT_FALSE(test, completion_done(&iface->out_complete)); + KUNIT_EXPECT_EQ(test, iface->out_report, 0x10); +} + +static void dchid_ack_identity_test(struct kunit *test) +{ + struct dchid_iface *iface = test->priv; + struct dchid_hdr hdr = { .length = sizeof(struct dchid_subhdr) + 1, .seq = 7 }; + struct { + struct dchid_subhdr sub; + u8 id; + } __packed ack = { .sub = { .flags = 0x80, .length = 1 }, .id = 0x11 }; + + dchid_handle_ack(iface, &hdr, &ack); + KUNIT_EXPECT_FALSE(test, completion_done(&iface->out_complete)); + ack.id = 0x10; + hdr.seq++; + dchid_handle_ack(iface, &hdr, &ack); + KUNIT_EXPECT_FALSE(test, completion_done(&iface->out_complete)); + hdr.seq--; + ack.sub.flags++; + dchid_handle_ack(iface, &hdr, &ack); + KUNIT_EXPECT_FALSE(test, completion_done(&iface->out_complete)); + ack.sub.flags--; + dchid_handle_ack(iface, &hdr, &ack); + KUNIT_EXPECT_TRUE(test, completion_done(&iface->out_complete)); + KUNIT_EXPECT_EQ(test, iface->out_report, -1); + KUNIT_EXPECT_EQ(test, iface->resp_size, 1U); +} + +static void dchid_ack_copy_test(struct kunit *test) +{ + struct dchid_iface *iface = test->priv; + struct dchid_hdr hdr = { .length = sizeof(struct dchid_subhdr) + 5, .seq = 7 }; + struct { + struct dchid_subhdr sub; + u8 data[5]; + } __packed ack = { .sub = { .flags = 0x80, .length = 5 }, + .data = { 0x10, 1, 2, 3, 4 } }; + u8 response[4] = { 0xa5, 0xa5, 0xa5, 0xa5 }; + + iface->resp_buf = response + 1; + iface->resp_size = 2; + dchid_handle_ack(iface, &hdr, &ack); + KUNIT_EXPECT_TRUE(test, completion_done(&iface->out_complete)); + KUNIT_EXPECT_EQ(test, iface->resp_size, 3U); + KUNIT_EXPECT_MEMEQ(test, response, ((u8[]){ 0xa5, 1, 2, 0xa5 }), sizeof(response)); +} + +static void dchid_init_bounds_test(struct kunit *test) +{ + struct dchid_iface *iface = test->priv; + struct dockchannel_hid *dchid = iface->dchid; + struct { + struct dchid_init_hdr hdr; + struct dchid_init_block_hdr block; + } __packed msg = { .hdr = { .iface = 1, .more_packets = 1 } }; + unsigned int len; + + for (len = 0; len < sizeof(msg.hdr); len++) + dchid_handle_init(dchid, NULL, len); + msg.hdr.iface = MAX_INTERFACES; + dchid_handle_init(dchid, &msg, sizeof(msg)); + msg.hdr.iface = U8_MAX; + dchid_handle_init(dchid, &msg, sizeof(msg)); + msg.hdr.iface = 1; + msg.block.type = INIT_HID_DESCRIPTOR; + msg.block.length = U16_MAX; + dchid_handle_init(dchid, &msg, sizeof(msg)); + KUNIT_EXPECT_PTR_EQ(test, iface->hid_desc, NULL); + msg.block.type = INIT_GPIO_REQUEST; + msg.block.length = 0; + dchid_handle_init(dchid, &msg, sizeof(msg)); + KUNIT_EXPECT_FALSE(test, iface->gpio_announced); + KUNIT_EXPECT_PTR_EQ(test, dchid->ifaces[1], iface); +} + +static void dchid_short_input_test(struct kunit *test) +{ + struct dchid_iface *iface = test->priv; + struct dchid_work *work; + unsigned int len; + + /* Any attempted HID delivery would dereference the absent HID device. */ + iface->open = true; + for (len = 0; len < sizeof(struct dchid_subhdr); len++) { + work = kzalloc(struct_size(work, data, len), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, work); + work->iface = iface; + work->hdr.iface = 1; + work->hdr.length = len; + dchid_packet_work(&work->work); + } + work = kzalloc(struct_size(work, data, sizeof(struct dchid_subhdr)), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, work); + work->iface = iface; + work->hdr.iface = 1; + work->hdr.length = sizeof(struct dchid_subhdr); + ((struct dchid_subhdr *)work->data)->length = U16_MAX; + dchid_packet_work(&work->work); +} + +static void dchid_init_descriptor_test(struct kunit *test) +{ + struct dchid_iface *iface = test->priv; + struct { + struct dchid_init_hdr hdr; + struct dchid_init_block_hdr block; + u8 descriptor[4]; + } __packed msg = { + .hdr = { .iface = 1, .more_packets = 1 }, + .block = { .type = INIT_HID_DESCRIPTOR, .length = 4 }, + .descriptor = { 0x05, 0x01, 0x09, 0x06 }, + }; + + dchid_handle_init(iface->dchid, &msg, sizeof(msg)); + KUNIT_ASSERT_NOT_NULL(test, iface->hid_desc); + KUNIT_EXPECT_EQ(test, iface->hid_desc_len, sizeof(msg.descriptor)); + KUNIT_EXPECT_MEMEQ(test, iface->hid_desc, msg.descriptor, sizeof(msg.descriptor)); +} + +static void dchid_packet_checksum_test(struct kunit *test) +{ + struct dchid_hdr hdr = { .hdr_len = sizeof(hdr), .length = 8 }; + u8 body[12] = {}; + u32 checksum; + + put_unaligned_le32(U32_MAX - dchid_checksum(&hdr, sizeof(hdr)), body + 8); + KUNIT_EXPECT_TRUE(test, dchid_verify_checksum(&hdr, body, &checksum)); + KUNIT_EXPECT_EQ(test, checksum, U32_MAX); + body[0] ^= 1; + KUNIT_EXPECT_FALSE(test, dchid_verify_checksum(&hdr, body, &checksum)); +} + +static struct kunit_case dchid_parser_cases[] = { + KUNIT_CASE(dchid_short_ack_test), + KUNIT_CASE(dchid_ack_length_test), + KUNIT_CASE(dchid_ack_identity_test), + KUNIT_CASE(dchid_ack_copy_test), + KUNIT_CASE(dchid_init_bounds_test), + KUNIT_CASE(dchid_short_input_test), + KUNIT_CASE(dchid_init_descriptor_test), + KUNIT_CASE(dchid_packet_checksum_test), + {} +}; + +static struct kunit_suite dchid_parser_suite = { + .name = "dockchannel-hid-parser", + .init = dchid_parser_test_init, + .test_cases = dchid_parser_cases, +}; + +kunit_test_suite(dchid_parser_suite); diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 6a5e412887082..46f943efc622b 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -303,6 +303,13 @@ static u32 dchid_checksum(void *p, size_t length) return sum; } +static bool dchid_verify_checksum(struct dchid_hdr *hdr, void *data, u32 *checksum) +{ + *checksum = dchid_checksum(hdr, sizeof(*hdr)); + *checksum += dchid_checksum(data, hdr->length + sizeof(u32)); + return *checksum == U32_MAX; +} + static int dchid_send(struct dchid_iface *iface, u32 flags, void *msg, size_t size) { u32 checksum = 0xffffffff; @@ -1199,10 +1206,7 @@ static void dchid_handle_packet(void *cookie, size_t avail) goto done; } - checksum = dchid_checksum(&hdr, sizeof(hdr)); - checksum += dchid_checksum(dchid->pkt_buf, hdr.length + 4); - - if (checksum != 0xffffffff) { + if (!dchid_verify_checksum(&hdr, dchid->pkt_buf, &checksum)) { dev_err(dchid->dev, "Checksum mismatch (iface %d): 0x%08x != 0xffffffff\n", hdr.iface, checksum); goto done; @@ -1452,3 +1456,7 @@ module_platform_driver(dockchannel_hid_driver); MODULE_DESCRIPTION("Apple DockChannel HID transport driver"); MODULE_AUTHOR("Hector Martin "); MODULE_LICENSE("Dual MIT/GPL"); + +#if IS_ENABLED(CONFIG_HID_DOCKCHANNEL_PARSER_KUNIT_TEST) +#include "dockchannel-hid-parser-test.c" +#endif From 3006ca1a383525a092ad1cbe9c268786292a32de Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:41:37 +1100 Subject: [PATCH 1159/1579] mfd: macsmc: Test key and fixed-width readers with KUnit Use KUnit static stubs for the command, SRAM, and battery read paths to exercise the real readers without a mailbox. Check request bounds, inline and SRAM values, short and oversized replies, key lookup errors, index validation, metadata locking, and the CHTE charge-behaviour path. The short CHTE cases poison the unread buffer while returning zero or two bytes. They fail if the caller inspects data from a short reply. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/mfd/Kconfig | 10 + drivers/mfd/macsmc-test.c | 221 +++++++++++++++++++++++ drivers/mfd/macsmc.c | 20 +- drivers/power/supply/Kconfig | 10 + drivers/power/supply/macsmc-power-test.c | 58 ++++++ drivers/power/supply/macsmc-power.c | 16 +- 6 files changed, 332 insertions(+), 3 deletions(-) create mode 100644 drivers/mfd/macsmc-test.c create mode 100644 drivers/power/supply/macsmc-power-test.c diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index 7192c9d1d268e..cca50c0cb7319 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -343,6 +343,16 @@ config MFD_MACSMC various features. The MFD core device handles this protocol and exposes it to the sub-devices. +config MFD_MACSMC_KUNIT_TEST + bool "KUnit tests for Apple SMC key readers" if !KUNIT_ALL_TESTS + depends on MFD_MACSMC && KUNIT=y + default KUNIT_ALL_TESTS + help + Build tests for SMC key/value readers, request limits and metadata + decoding. Static stubs replace mailbox commands and SRAM access, + allowing malformed replies and output buffer limits to be checked + without a running SMC. + config MFD_MADERA tristate "Cirrus Logic Madera codecs" select MFD_CORE diff --git a/drivers/mfd/macsmc-test.c b/drivers/mfd/macsmc-test.c new file mode 100644 index 0000000000000..c4f9329dbc01c --- /dev/null +++ b/drivers/mfd/macsmc-test.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Included by macsmc.c; the command and SRAM transports are stubbed. */ +#include + +struct macsmc_test_context { + struct apple_smc smc; + u8 shmem[SMC_MAX_SIZE]; + u32 data; + int result; + unsigned int calls; + bool require_sram_lock; +}; + +static int macsmc_test_command(struct apple_smc *smc, u64 cmd, u64 arg, + u64 size, u64 wsize, u32 *ret_data) +{ + struct macsmc_test_context *ctx = container_of(smc, struct macsmc_test_context, smc); + + KUNIT_EXPECT_TRUE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); + ctx->calls++; + if (ret_data && ctx->result >= 0) + *ret_data = ctx->data; + return ctx->result; +} + +static void macsmc_test_read_sram(struct apple_smc *smc, void *buf, size_t size) +{ + struct macsmc_test_context *ctx = container_of(smc, struct macsmc_test_context, smc); + + if (ctx->require_sram_lock) + KUNIT_EXPECT_TRUE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); + memcpy(buf, ctx->shmem, size); +} + +static int macsmc_test_init(struct kunit *test) +{ + struct macsmc_test_context *ctx; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + mutex_init(&ctx->smc.mutex); + ctx->smc.shmem.iomem = (void __iomem *)ctx->shmem; + ctx->smc.key_count = 3; + test->priv = ctx; + kunit_activate_static_stub(test, apple_smc_cmd_locked, macsmc_test_command); + kunit_activate_static_stub(test, apple_smc_read_sram, macsmc_test_read_sram); + return 0; +} + +static void macsmc_value_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + u8 byte = 0; + u32 word = 0; + + ctx->data = 0x12345678; + ctx->result = sizeof(word); + KUNIT_EXPECT_EQ(test, apple_smc_read_u32(&ctx->smc, SMC_KEY(TEST), &word), 0); + KUNIT_EXPECT_EQ(test, word, ctx->data); + ctx->result = sizeof(byte); + KUNIT_EXPECT_EQ(test, apple_smc_read_u8(&ctx->smc, SMC_KEY(TEST), &byte), 0); + KUNIT_EXPECT_EQ(test, byte, 0x78); + ctx->result = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, apple_smc_read_u8(&ctx->smc, SMC_KEY(TEST), &byte), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, byte, 0x78); +} + +static void macsmc_fixed_value_short_reply_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + u32 word = 0xa5a5a5a5; + u8 byte = 0xa5; + + ctx->data = 0x12345678; + ctx->result = 2; + KUNIT_EXPECT_EQ(test, apple_smc_read_u32(&ctx->smc, SMC_KEY(TEST), &word), -EIO); + KUNIT_EXPECT_EQ(test, word, 0U); + word = 0xa5a5a5a5; + KUNIT_EXPECT_EQ(test, + apple_smc_rw_u32(&ctx->smc, SMC_KEY(TEST), 0, &word), -EIO); + KUNIT_EXPECT_EQ(test, word, 0U); + ctx->result = 0; + KUNIT_EXPECT_EQ(test, apple_smc_read_u8(&ctx->smc, SMC_KEY(TEST), &byte), -EIO); + KUNIT_EXPECT_EQ(test, byte, 0); +} + +static void macsmc_request_bounds_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + u8 byte; + + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), &byte, + SMC_MAX_SIZE + 1), -EINVAL); + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), &byte, SIZE_MAX), + -EINVAL); + KUNIT_EXPECT_EQ(test, apple_smc_write(&ctx->smc, SMC_KEY(TEST), &byte, SIZE_MAX), + -EINVAL); + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), &byte, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, ctx->calls, 0U); +} + +static void macsmc_sram_value_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + u8 value[SMC_MAX_SIZE + 2]; + unsigned int i; + + for (i = 0; i < sizeof(ctx->shmem); i++) + ctx->shmem[i] = i; + memset(value, 0xa5, sizeof(value)); + ctx->result = SMC_MAX_SIZE; + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), value + 1, + SMC_MAX_SIZE), SMC_MAX_SIZE); + KUNIT_EXPECT_MEMEQ(test, value + 1, ctx->shmem, SMC_MAX_SIZE); + KUNIT_EXPECT_EQ(test, value[0], 0xa5); + KUNIT_EXPECT_EQ(test, value[SMC_MAX_SIZE + 1], 0xa5); +} + +static void macsmc_key_info_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + struct apple_smc_key_info info = {}; + const u8 wire[] = { 255, 'u', 'i', '3', '2', APPLE_SMC_READABLE }; + + memcpy(ctx->shmem, wire, sizeof(wire)); + KUNIT_EXPECT_EQ(test, apple_smc_get_key_info(&ctx->smc, SMC_KEY(TEST), &info), 0); + KUNIT_EXPECT_EQ(test, info.size, 255); + KUNIT_EXPECT_EQ(test, info.type_code, SMC_KEY(ui32)); + KUNIT_EXPECT_EQ(test, info.flags, APPLE_SMC_READABLE); + ctx->result = -EIO; + memset(ctx->shmem, 0, sizeof(wire)); + KUNIT_EXPECT_EQ(test, apple_smc_get_key_info(&ctx->smc, SMC_KEY(TEST), &info), -EIO); + KUNIT_EXPECT_EQ(test, info.size, 255); +} + +static void macsmc_short_value_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + u8 value[4]; + + memset(value, 0xa5, sizeof(value)); + ctx->data = 0x12345678; + ctx->result = 0; + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), value, 4), 0); + KUNIT_EXPECT_MEMEQ(test, value, ((u8[]){ 0xa5, 0xa5, 0xa5, 0xa5 }), 4); + ctx->result = 2; + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), value, 4), 2); + KUNIT_EXPECT_MEMEQ(test, value, ((u8[]){ 0x78, 0x56, 0xa5, 0xa5 }), 4); + memset(value, 0xa5, sizeof(value)); + ctx->result = 5; + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), value, 4), -EIO); + KUNIT_EXPECT_MEMEQ(test, value, ((u8[]){ 0xa5, 0xa5, 0xa5, 0xa5 }), 4); + ctx->result = 7; + memset(ctx->shmem, 0x3c, sizeof(ctx->shmem)); + memset(ctx->shmem + 16, 0xa5, 8); + KUNIT_EXPECT_EQ(test, apple_smc_read(&ctx->smc, SMC_KEY(TEST), ctx->shmem + 16, 8), 7); + KUNIT_EXPECT_EQ(test, ctx->shmem[23], 0xa5); +} + +static void macsmc_key_error_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + smc_key key = SMC_KEY(TEST); + + ctx->result = -EIO; + KUNIT_EXPECT_EQ(test, apple_smc_get_key_by_index(&ctx->smc, 0, &key), -EIO); + KUNIT_EXPECT_EQ(test, key, SMC_KEY(TEST)); + ctx->result = 0; + ctx->data = swab32(SMC_KEY(TEST)); + KUNIT_EXPECT_EQ(test, apple_smc_get_key_by_index(&ctx->smc, 2, &key), 0); + KUNIT_EXPECT_EQ(test, key, SMC_KEY(TEST)); +} + +static void macsmc_key_index_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + smc_key key = SMC_KEY(TEST); + int index = -1; + + KUNIT_EXPECT_EQ(test, + apple_smc_get_key_by_index(&ctx->smc, READ_ONCE(index), &key), + -EINVAL); + KUNIT_EXPECT_EQ(test, apple_smc_get_key_by_index(&ctx->smc, 3, &key), -EINVAL); + KUNIT_EXPECT_EQ(test, apple_smc_get_key_by_index(&ctx->smc, INT_MAX, &key), -EINVAL); + KUNIT_EXPECT_EQ(test, ctx->calls, 0U); + KUNIT_EXPECT_EQ(test, key, SMC_KEY(TEST)); +} + +static void macsmc_key_info_lock_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + struct apple_smc_key_info info; + + ctx->require_sram_lock = true; + KUNIT_EXPECT_EQ(test, apple_smc_get_key_info(&ctx->smc, SMC_KEY(TEST), &info), 0); + KUNIT_EXPECT_FALSE(test, mutex_is_locked(&ctx->smc.mutex)); + KUNIT_EXPECT_EQ(test, apple_smc_get_key_info(&ctx->smc, SMC_KEY(TEST), NULL), 0); + KUNIT_EXPECT_FALSE(test, mutex_is_locked(&ctx->smc.mutex)); +} + +static struct kunit_case macsmc_parser_cases[] = { + KUNIT_CASE(macsmc_key_info_lock_test), + KUNIT_CASE(macsmc_key_index_test), + KUNIT_CASE(macsmc_key_error_test), + KUNIT_CASE(macsmc_short_value_test), + KUNIT_CASE(macsmc_fixed_value_short_reply_test), + KUNIT_CASE(macsmc_value_test), + KUNIT_CASE(macsmc_request_bounds_test), + KUNIT_CASE(macsmc_sram_value_test), + KUNIT_CASE(macsmc_key_info_test), + {} +}; + +static struct kunit_suite macsmc_parser_suite = { + .name = "macsmc-parser", + .init = macsmc_test_init, + .test_cases = macsmc_parser_cases, +}; + +kunit_test_suite(macsmc_parser_suite); diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index cf25327a7bd8b..55d52c684d38b 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -5,6 +5,8 @@ * Copyright The Asahi Linux Contributors */ +#include + #include #include #include @@ -57,6 +59,13 @@ static const struct mfd_cell apple_smc_devs[] = { static const struct mfd_cell apple_smc_low_gpio_dev = MFD_CELL_OF("macsmc-low-gpio", NULL, NULL, 0, 0, "apple,smc-low-gpio"); +static void apple_smc_read_sram(struct apple_smc *smc, void *buf, size_t size) +{ + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_read_sram, smc, buf, size); + memcpy_fromio(buf, smc->shmem.iomem, size); +} + static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, u64 size, u64 wsize, u32 *ret_data) { @@ -65,6 +74,9 @@ static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, u64 msg; lockdep_assert_held(&smc->mutex); + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_cmd_locked, smc, cmd, arg, size, wsize, + ret_data); if (smc->boot_stage != APPLE_SMC_INITIALIZED) return -EIO; @@ -170,7 +182,7 @@ static int apple_smc_rw_locked(struct apple_smc *smc, smc_key key, if (rsize <= 4) memcpy(rbuf, &rdata, ret); else - memcpy_fromio(rbuf, smc->shmem.iomem, ret); + apple_smc_read_sram(smc, rbuf, ret); } return ret; @@ -227,7 +239,7 @@ int apple_smc_get_key_info(struct apple_smc *smc, smc_key key, struct apple_smc_ ret = apple_smc_cmd_locked(smc, SMC_MSG_GET_KEY_INFO, key, 0, 0, NULL); if (ret >= 0 && info) { - memcpy_fromio(key_info, smc->shmem.iomem, sizeof(key_info)); + apple_smc_read_sram(smc, key_info, sizeof(key_info)); info->size = key_info[0]; info->type_code = get_unaligned_be32(&key_info[1]); info->flags = key_info[5]; @@ -536,3 +548,7 @@ MODULE_AUTHOR("Hector Martin "); MODULE_AUTHOR("Sven Peter "); MODULE_LICENSE("Dual MIT/GPL"); MODULE_DESCRIPTION("Apple SMC driver"); + +#if IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST) +#include "macsmc-test.c" +#endif diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig index 83392ed6a8da9..1f1b36ffa835a 100644 --- a/drivers/power/supply/Kconfig +++ b/drivers/power/supply/Kconfig @@ -1165,4 +1165,14 @@ config MACSMC_POWER Say Y or M here if you have an Apple Silicon based Mac. +config MACSMC_POWER_KUNIT_TEST + bool "KUnit tests for Apple SMC battery replies" if !KUNIT_ALL_TESTS + depends on MACSMC_POWER && KUNIT=y + default KUNIT_ALL_TESTS + help + Build tests for malformed fixed-width battery replies. The tests + exercise zero-length and short CHTE replies and check that the + charge-behaviour reader rejects them. Static stubs replace SMC + reads, so the tests need no Apple hardware. + endif # POWER_SUPPLY diff --git a/drivers/power/supply/macsmc-power-test.c b/drivers/power/supply/macsmc-power-test.c new file mode 100644 index 0000000000000..78f7f1b877beb --- /dev/null +++ b/drivers/power/supply/macsmc-power-test.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Included by macsmc-power.c; the charge-behaviour reader is stubbed. */ +#include + +struct macsmc_power_test_context { + struct macsmc_power power; + int result; +}; + +static int macsmc_power_test_read_chte(struct macsmc_power *power, u8 *buf) +{ + struct macsmc_power_test_context *ctx = + container_of(power, struct macsmc_power_test_context, power); + + memset(buf, 0xa5, 4); + buf[0] = 1; + return ctx->result; +} + +static int macsmc_power_test_init(struct kunit *test) +{ + struct macsmc_power_test_context *ctx; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + ctx->power.has_chte = true; + test->priv = ctx; + kunit_activate_static_stub(test, macsmc_power_read_chte, + macsmc_power_test_read_chte); + return 0; +} + +static void macsmc_power_short_chte_test(struct kunit *test) +{ + struct macsmc_power_test_context *ctx = test->priv; + + ctx->result = 0; + KUNIT_EXPECT_EQ(test, macsmc_battery_get_charge_behaviour(&ctx->power), -EIO); + ctx->result = 2; + KUNIT_EXPECT_EQ(test, macsmc_battery_get_charge_behaviour(&ctx->power), -EIO); + ctx->result = 4; + KUNIT_EXPECT_EQ(test, macsmc_battery_get_charge_behaviour(&ctx->power), + POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE); +} + +static struct kunit_case macsmc_power_test_cases[] = { + KUNIT_CASE(macsmc_power_short_chte_test), + {} +}; + +static struct kunit_suite macsmc_power_test_suite = { + .name = "macsmc-power", + .init = macsmc_power_test_init, + .test_cases = macsmc_power_test_cases, +}; + +kunit_test_suite(macsmc_power_test_suite); diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index a7fae0bfff4cc..9acc3ce4d095d 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -22,6 +22,8 @@ #include #include +#include + #define MAX_STRING_LENGTH 256 /* @@ -316,6 +318,14 @@ static int macsmc_battery_get_status(struct macsmc_power *power) return macsmc_battery_charging_status(power); } +static int macsmc_power_read_chte(struct macsmc_power *power, u8 *buf) +{ + if (IS_ENABLED(CONFIG_MACSMC_POWER_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(macsmc_power_read_chte, power, buf); + + return apple_smc_read(power->smc, SMC_KEY(CHTE), buf, 4); +} + static int macsmc_battery_get_charge_behaviour(struct macsmc_power *power) { int ret; @@ -331,7 +341,7 @@ static int macsmc_battery_get_charge_behaviour(struct macsmc_power *power) } if (power->has_chte) { - ret = apple_smc_read(power->smc, SMC_KEY(CHTE), chte_buf, 4); + ret = macsmc_power_read_chte(power, chte_buf); if (ret < 0) return ret; if (ret != sizeof(chte_buf)) { @@ -1123,3 +1133,7 @@ MODULE_LICENSE("Dual MIT/GPL"); MODULE_DESCRIPTION("Apple SMC battery and power management driver"); MODULE_AUTHOR("Hector Martin "); MODULE_AUTHOR("Michael Reeves "); + +#if IS_ENABLED(CONFIG_MACSMC_POWER_KUNIT_TEST) +#include "macsmc-power-test.c" +#endif From 910327707f852c060b5ada81e64a2a4563cb64b4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:41:39 +1100 Subject: [PATCH 1160/1579] ASoC: apple: Test T8140 AOP producer reports with KUnit Separate producer position decoding and cursor validation from stream handling so that malformed reports can be checked without an audio device. Keep the existing counter checks and publish report readiness before cursor validation as before. Cover short fields and reports, unaligned little-endian values, addition and multiplication overflow, and cursors outside the owned ring. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 142 +++++++++++++++++++++++++---- 1 file changed, 123 insertions(+), 19 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index b63c6c8baa084..bdfd56f8295d2 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -1072,33 +1072,54 @@ fn copy_span( Ok(dst) } +#[derive(Debug, PartialEq, Eq)] +enum ProducerReportError { + Short, + Overflow, +} + +fn producer_position(report: &[u8]) -> core::result::Result<(u64, u64), ProducerReportError> { + if report.len() < REPORT_MIN_LEN { + return Err(ProducerReportError::Short); + } + let count = le_u64(report, 0x40).map_err(|_| ProducerReportError::Short)?; + let absolute = count + .checked_add(1) + .and_then(|frames| frames.checked_mul(LPAI_FRAME_BYTES as u64)) + .ok_or(ProducerReportError::Overflow)?; + let cursor = le_u64(report, 0x60).map_err(|_| ProducerReportError::Short)?; + Ok((absolute, cursor)) +} + +fn producer_cursor_matches(absolute: u64, cursor: u64, ring_bytes: usize) -> bool { + ring_bytes != 0 && cursor == absolute % ring_bytes as u64 +} + impl ReportListener for SndSocT8140AopData { fn process_report(&self, _subtype: u16, report: &[u8]) -> Result<()> { - if report.len() < REPORT_MIN_LEN { - if !self.report_warnings[0].xchg(true, Relaxed) { - dev_warn!( - self.dev, - "short lpai producer report ({} bytes)\n", - report.len() - ); + let (absolute, cursor) = match producer_position(report) { + Ok(position) => position, + Err(ProducerReportError::Short) => { + if !self.report_warnings[0].xchg(true, Relaxed) { + dev_warn!( + self.dev, + "short lpai producer report ({} bytes)\n", + report.len() + ); + } + return Ok(()); } - return Ok(()); - } - let count = le_u64(report, 0x40)?; - let Some(absolute) = count - .checked_add(1) - .and_then(|frames| frames.checked_mul(LPAI_FRAME_BYTES as u64)) - else { - if !self.report_warnings[1].xchg(true, Relaxed) { - dev_warn!(self.dev, "overflowing lpai producer counter\n"); + Err(ProducerReportError::Overflow) => { + if !self.report_warnings[1].xchg(true, Relaxed) { + dev_warn!(self.dev, "overflowing lpai producer counter\n"); + } + return Ok(()); } - return Ok(()); }; let frames = absolute / LPAI_FRAME_BYTES as u64; self.reported.store(true, Relaxed); - let cursor = le_u64(report, 0x60)?; let ring_bytes = self.ring_bytes.load(Relaxed) as usize; - if ring_bytes == 0 || cursor != absolute % ring_bytes as u64 { + if !producer_cursor_matches(absolute, cursor, ring_bytes) { if !self.report_warnings[2].xchg(true, Relaxed) { dev_warn!( self.dev, @@ -2960,3 +2981,86 @@ mod tests { assert_eq!(f32_bits_of_q31(i32::MIN), 0xbf80_0000); // -1.0 } } + +#[kunit_tests(snd_soc_t8140_aop_audio_parser)] +mod parser_tests { + use super::*; + + #[test] + fn short_reply_fields_are_rejected() { + let data = [0u8; REPORT_MIN_LEN]; + for len in 0..4 { + assert_eq!(le_u32(&data[..len], 0), Err(EINVAL)); + } + for len in 0..8 { + assert_eq!(le_u64(&data[..len], 0), Err(EINVAL)); + } + for len in 0..REPORT_MIN_LEN { + assert_eq!(le_u64(&data[..len], 0x60), Err(EINVAL)); + } + } + + #[test] + fn unaligned_fields_at_the_end_of_a_reply_are_decoded() { + let data = [0xff, 1, 2, 3, 4, 5, 6, 7, 8]; + assert_eq!(le_u32(&data, 5), Ok(0x0807_0605)); + assert_eq!(le_u64(&data, 1), Ok(0x0807_0605_0403_0201)); + assert_eq!(le_u32(&data, 6), Err(EINVAL)); + assert_eq!(le_u64(&data, 2), Err(EINVAL)); + } + + fn producer_report(count: u64, cursor: u64) -> [u8; REPORT_MIN_LEN] { + let mut report = [0u8; REPORT_MIN_LEN]; + report[0x40..0x48].copy_from_slice(&count.to_le_bytes()); + report[0x60..0x68].copy_from_slice(&cursor.to_le_bytes()); + report + } + + #[test] + fn short_producer_reports_are_rejected() { + let report = producer_report(3199, LPAI_PERIOD_BYTES as u64); + for len in 0..REPORT_MIN_LEN { + assert_eq!( + producer_position(&report[..len]), + Err(ProducerReportError::Short) + ); + } + assert_eq!( + producer_position(&report), + Ok((LPAI_PERIOD_BYTES as u64, LPAI_PERIOD_BYTES as u64)) + ); + } + + #[test] + fn overflowing_producer_positions_are_rejected() { + for count in [u64::MAX, u64::MAX / LPAI_FRAME_BYTES as u64] { + assert_eq!( + producer_position(&producer_report(count, 0)), + Err(ProducerReportError::Overflow) + ); + } + let count = u64::MAX / LPAI_FRAME_BYTES as u64 - 1; + assert_eq!( + producer_position(&producer_report(count, 0)), + Ok(((count + 1) * LPAI_FRAME_BYTES as u64, 0)) + ); + } + + #[test] + fn producer_cursors_must_match_the_owned_ring() { + assert!(producer_cursor_matches(RING_BYTES as u64, 0, RING_BYTES)); + assert!(producer_cursor_matches( + RING_BYTES as u64 + 8, + 8, + RING_BYTES + )); + assert!(!producer_cursor_matches(8, 8, 0)); + assert!(!producer_cursor_matches( + 8, + RING_BYTES as u64 + 8, + RING_BYTES + )); + assert!(!producer_cursor_matches(8, u64::MAX, RING_BYTES)); + assert!(!producer_cursor_matches(8, 7, RING_BYTES)); + } +} From a239464d233d2e94b89dea159f298b70e015b6a6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:41:40 +1100 Subject: [PATCH 1161/1579] kunit: Add an Apple firmware parser test configuration Provide an arm64 configuration for the Apple AFK, IOMFB, DockChannel HID, SMC and T8140 AOP parser suites. The drivers can be built into a QEMU test kernel without attaching any Apple hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../kunit/configs/apple_firmware.config | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tools/testing/kunit/configs/apple_firmware.config diff --git a/tools/testing/kunit/configs/apple_firmware.config b/tools/testing/kunit/configs/apple_firmware.config new file mode 100644 index 0000000000000..61eeab466b809 --- /dev/null +++ b/tools/testing/kunit/configs/apple_firmware.config @@ -0,0 +1,32 @@ +CONFIG_KUNIT=y +CONFIG_ARCH_APPLE=y +CONFIG_PM=y +CONFIG_APPLE_MAILBOX=y +CONFIG_APPLE_RTKIT=y +CONFIG_DRM=y +CONFIG_BACKLIGHT_CLASS_DEVICE=y +CONFIG_DRM_APPLE=y +CONFIG_DRM_APPLE_PARSER_KUNIT_TEST=y +CONFIG_HID=y +CONFIG_INPUT=y +CONFIG_APPLE_DOCKCHANNEL=y +CONFIG_HID_DOCKCHANNEL=y +CONFIG_HID_DOCKCHANNEL_PARSER_KUNIT_TEST=y +CONFIG_MFD_MACSMC=y +CONFIG_MFD_MACSMC_KUNIT_TEST=y +CONFIG_POWER_SUPPLY=y +CONFIG_MACSMC_POWER=y +CONFIG_MACSMC_POWER_KUNIT_TEST=y +CONFIG_RUST=y +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_SND_SOC=y +CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_FW_LOADER=y +CONFIG_DMADEVICES=y +CONFIG_DMA_ENGINE=y +CONFIG_PM_GENERIC_DOMAINS=y +CONFIG_RUST_OVERFLOW_CHECKS=y +# CONFIG_LOCALVERSION_AUTO is not set From 0943ae25f9bd82917a7a4813962d624d22056fd3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:44:44 +1100 Subject: [PATCH 1162/1579] HID: dockchannel-hid: Reject unaligned packet lengths before checksumming The receive checksum sums whole words and warns about any trailing bytes. A peer can provide an unaligned packet length and a checksum that matches the truncated sum. Such a packet both triggers a warning and passes the checksum test despite bytes being left unchecked. Reject lengths that are not word aligned before computing the checksum. The packet body has already been consumed, so this keeps the next packet at its original boundary. Test all three unaligned lengths using packets whose truncated checksums would otherwise match. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../dockchannel-hid/dockchannel-hid-parser-test.c | 14 ++++++++++++++ drivers/hid/dockchannel-hid/dockchannel-hid.c | 5 +++++ 2 files changed, 19 insertions(+) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c b/drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c index ef0e8869bd99f..56efa764aaec6 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c @@ -183,7 +183,21 @@ static void dchid_packet_checksum_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, dchid_verify_checksum(&hdr, body, &checksum)); } +static void dchid_unaligned_checksum_test(struct kunit *test) +{ + struct dchid_hdr hdr = { .hdr_len = sizeof(hdr) }; + u8 body[16] = {}; + u32 checksum; + + for (hdr.length = 9; hdr.length < 12; hdr.length++) { + put_unaligned_le32(U32_MAX - dchid_checksum(&hdr, sizeof(hdr)), body + 8); + memset(body + 12, 0xa5, 4); + KUNIT_EXPECT_FALSE(test, dchid_verify_checksum(&hdr, body, &checksum)); + } +} + static struct kunit_case dchid_parser_cases[] = { + KUNIT_CASE(dchid_unaligned_checksum_test), KUNIT_CASE(dchid_short_ack_test), KUNIT_CASE(dchid_ack_length_test), KUNIT_CASE(dchid_ack_identity_test), diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 46f943efc622b..e6e809e9401bc 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -305,6 +305,11 @@ static u32 dchid_checksum(void *p, size_t length) static bool dchid_verify_checksum(struct dchid_hdr *hdr, void *data, u32 *checksum) { + if (!IS_ALIGNED(hdr->length, sizeof(u32))) { + *checksum = 0; + return false; + } + *checksum = dchid_checksum(hdr, sizeof(*hdr)); *checksum += dchid_checksum(data, hdr->length + sizeof(u32)); return *checksum == U32_MAX; From ee6f48ce60bdf68d596c4bedd31e35f55cd6bb03 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:00:58 +1100 Subject: [PATCH 1163/1579] MAINTAINERS: Cover Apple firmware parser tests Include the SMC test alongside the existing SMC driver in the Apple machine support entry. Add the DockChannel parser test and shared KUnit configuration to the existing Apple Silicon entry so changes to them also reach their driver maintainers. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index f02c49da5d11b..1ea81e2a135c3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2004,6 +2004,7 @@ F: arch/arm64/boot/dts/apple/t8140* F: arch/arm64/configs/j700*.config F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S +F: drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ F: drivers/phy/apple/atc-j700-dp.h @@ -2017,6 +2018,7 @@ F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs F: tools/testing/apple/ +F: tools/testing/kunit/configs/apple_firmware.config F: tools/testing/selftests/drivers/gpu/apple-dcp/ APPLETALK NETWORK LAYER @@ -2633,7 +2635,7 @@ F: drivers/iommu/apple-dart-test.c F: drivers/iommu/apple-dart.c F: drivers/iommu/io-pgtable-dart.c F: drivers/irqchip/irq-apple-aic.c -F: drivers/mfd/macsmc.c +F: drivers/mfd/macsmc*.c F: drivers/nvme/host/apple.c F: drivers/nvmem/apple-efuses.c F: drivers/nvmem/apple-spmi-nvmem.c @@ -2641,7 +2643,7 @@ F: drivers/phy/apple/ F: drivers/pinctrl/pinctrl-apple-gpio.c F: drivers/pmdomain/apple/ F: drivers/power/reset/macsmc-reboot.c -F: drivers/power/supply/macsmc-power.c +F: drivers/power/supply/macsmc-power*.c F: drivers/pwm/pwm-apple.c F: drivers/reset/reset-apple-cio.c F: drivers/rtc/rtc-macsmc.c From 232faf36664c7e23aa9950bc3019dbf6ba534ee8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:09:12 +1100 Subject: [PATCH 1164/1579] HID: dockchannel-hid: Keep parser tests in the subsystem menu The parser KUnit option appears after the DockChannel HID menu closes, so menuconfig presents it outside the subsystem it tests. Move the option above endmenu without changing its dependencies. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/Kconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig index fe2f5b96aa749..5847ce4a485bf 100644 --- a/drivers/hid/dockchannel-hid/Kconfig +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -26,8 +26,6 @@ config HID_DOCKCHANNEL_KUNIT_TEST If unsure, say N. -endmenu - config HID_DOCKCHANNEL_PARSER_KUNIT_TEST bool "KUnit tests for DockChannel HID message parsing" if !KUNIT_ALL_TESTS depends on HID_DOCKCHANNEL && KUNIT=y @@ -37,3 +35,5 @@ config HID_DOCKCHANNEL_PARSER_KUNIT_TEST input-report and acknowledgment handlers. Cover short headers, oversized lengths, interface indices and acknowledgment identity. The tests run without a DockChannel or HID device. + +endmenu From e30058de77e3fd2dc2e3bf6ddbf0d2e4cb577093 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:31:34 +1100 Subject: [PATCH 1165/1579] spi: apple: Program the hardware CS delays for every message apple_spi_set_cs_timing() writes the CS delay registers and the CS override bit once, when the SPI core calls it from spi_setup(). The controller uses auto_runtime_pm without an autosuspend delay, so it is runtime suspended right after spi_setup() and after every message, and its power domain can then be gated. The block comes back with its registers at their reset values, which disables the hardware delays. Because the controller provides set_cs_timing, the SPI core no longer inserts the software CS setup and hold delays either. The fingerprint sensor on this bus therefore runs without any CS setup or hold time once the domain has been gated. The registers are also shared by all devices on the bus, so the last device to call spi_setup() would set the timing for all of them. Keep the delay calculation and validation in set_cs_timing, but write the registers from prepare_message, where the controller is powered and the device of the message is known. The hardware delays exist on purpose for the Mesa sensor and are only installed on its bus, see commit 1d0fbf391a2a ("spi: apple: install hardware CS timing only on the Mesa sensor's bus"). That scope and the delay values are unchanged; only the point where the registers are written moves. Fixes: 4a356a39160b ("soc: apple: add production SEP FileVault integration") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple.c | 88 ++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 31 deletions(-) diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c index f18cc6e1b8d0e..74f3c1f09d739 100644 --- a/drivers/spi/spi-apple.c +++ b/drivers/spi/spi-apple.c @@ -190,22 +190,6 @@ static void apple_spi_init(struct apple_spi *spi) reg_write(spi, APPLE_SPI_DELAY_POST_CYCLES, 0); } -static int apple_spi_prepare_message(struct spi_controller *ctlr, struct spi_message *msg) -{ - struct apple_spi *spi = spi_controller_get_devdata(ctlr); - struct spi_device *device = msg->spi; - - u32 cfg = ((device->mode & SPI_CPHA ? APPLE_SPI_CFG_CPHA : 0) | - (device->mode & SPI_CPOL ? APPLE_SPI_CFG_CPOL : 0) | - (device->mode & SPI_LSB_FIRST ? APPLE_SPI_CFG_LSB_FIRST : 0)); - - /* Update core config */ - reg_mask(spi, APPLE_SPI_CFG, - APPLE_SPI_CFG_CPHA | APPLE_SPI_CFG_CPOL | APPLE_SPI_CFG_LSB_FIRST, cfg); - - return 0; -} - static void apple_spi_set_cs(struct spi_device *device, bool is_high) { struct apple_spi *spi = spi_controller_get_devdata(device->controller); @@ -232,30 +216,32 @@ static int apple_spi_delay_cycles(struct apple_spi *spi, struct spi_delay *delay return 0; } -static int apple_spi_set_cs_timing(struct spi_device *device) +static int apple_spi_cs_timing_cycles(struct apple_spi *spi, + struct spi_device *device, + u32 *setup_cycles, u32 *hold_cycles) { - struct apple_spi *spi = spi_controller_get_devdata(device->controller); - u32 setup_cycles, hold_cycles; int ret; /* This controller path has no hardware representation for inactive time. */ if (device->cs_inactive.value) return -EOPNOTSUPP; - ret = apple_spi_delay_cycles(spi, &device->cs_setup, &setup_cycles); - if (ret) - return ret; - ret = apple_spi_delay_cycles(spi, &device->cs_hold, &hold_cycles); + ret = apple_spi_delay_cycles(spi, &device->cs_setup, setup_cycles); if (ret) return ret; + return apple_spi_delay_cycles(spi, &device->cs_hold, hold_cycles); +} +static void apple_spi_write_cs_timing(struct apple_spi *spi, u32 setup_cycles, + u32 hold_cycles) +{ if (!setup_cycles && !hold_cycles) { reg_mask(spi, APPLE_SPI_SHIFTCFG, APPLE_SPI_SHIFTCFG_OVERRIDE_CS, 0); reg_write(spi, APPLE_SPI_DELAY_PRE, 0); reg_write(spi, APPLE_SPI_DELAY_POST, 0); reg_write(spi, APPLE_SPI_DELAY_POST_CYCLES, 0); - return 0; + return; } reg_write(spi, APPLE_SPI_DELAY_PRE, @@ -267,14 +253,54 @@ static int apple_spi_set_cs_timing(struct spi_device *device) APPLE_SPI_DELAY_POST_FLAGS); reg_mask(spi, APPLE_SPI_SHIFTCFG, 0, APPLE_SPI_SHIFTCFG_OVERRIDE_CS); +} + +/* + * Only validate the delays here. The registers are programmed for each + * message in apple_spi_prepare_message(): their contents are lost when + * runtime PM gates the controller's power domain between messages, and + * they are shared by all devices on the bus. + */ +static int apple_spi_set_cs_timing(struct spi_device *device) +{ + struct apple_spi *spi = spi_controller_get_devdata(device->controller); + u32 setup_cycles, hold_cycles; + int ret; + + ret = apple_spi_cs_timing_cycles(spi, device, &setup_cycles, + &hold_cycles); + if (ret) + return ret; + + dev_dbg(&device->dev, "hardware CS setup=%u hold=%u cycles\n", + setup_cycles, hold_cycles); + return 0; +} + +static int apple_spi_prepare_message(struct spi_controller *ctlr, struct spi_message *msg) +{ + struct apple_spi *spi = spi_controller_get_devdata(ctlr); + struct spi_device *device = msg->spi; + + u32 cfg = ((device->mode & SPI_CPHA ? APPLE_SPI_CFG_CPHA : 0) | + (device->mode & SPI_CPOL ? APPLE_SPI_CFG_CPOL : 0) | + (device->mode & SPI_LSB_FIRST ? APPLE_SPI_CFG_LSB_FIRST : 0)); + + /* Update core config */ + reg_mask(spi, APPLE_SPI_CFG, + APPLE_SPI_CFG_CPHA | APPLE_SPI_CFG_CPOL | APPLE_SPI_CFG_LSB_FIRST, cfg); + + if (ctlr->set_cs_timing) { + u32 setup_cycles, hold_cycles; + int ret; + + ret = apple_spi_cs_timing_cycles(spi, device, &setup_cycles, + &hold_cycles); + if (ret) + return ret; + apple_spi_write_cs_timing(spi, setup_cycles, hold_cycles); + } - dev_dbg(&device->dev, - "hardware CS setup=%u hold=%u cycles: pre=%#08x post=%#08x post_cycles=%#08x shiftcfg=%#08x\n", - setup_cycles, hold_cycles, - reg_read(spi, APPLE_SPI_DELAY_PRE), - reg_read(spi, APPLE_SPI_DELAY_POST), - reg_read(spi, APPLE_SPI_DELAY_POST_CYCLES), - reg_read(spi, APPLE_SPI_SHIFTCFG)); return 0; } From d6799a78db13ad06a2f26b66aad97c7e7626714d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:42:23 +1100 Subject: [PATCH 1166/1579] drm/apple: dcp: Initialise work items at probe The vblank, swap watchdog and backlight work items are initialised in dcp_comp_bind(), but system sleep and shutdown cancel them on any probed DCP: dcp_platform_suspend() cancels swap_watchdog_wq unconditionally, and dcp_platform_shutdown() cancels bl_register_wq and bl_update_wq for the internal H17P DCP. A DCP whose DRM component never bound, or whose bind failed before reaching those INIT_WORK() calls (for instance on -EPROBE_DEFER from the PIODMA domain), therefore hands an all-zero work item to cancel_work_sync(). __flush_work() then warns about the missing function on every suspend or reboot. Initialise the work items and the backlight registration mutex once at probe, next to the other work items the platform device owns. Binding again no longer re-initialises them either, which was only harmless because unbind had cancelled them first. Fixes: be6e4f675c23 ("drm/apple: recover Type-C outputs whose swaps DCP drops") Fixes: 43039912593a ("drm: apple: retain adopted H17P DMA resources through reboot") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e4d727d02c880..bddcd92976d24 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4296,9 +4296,6 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) of_node_put(panel_np); dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; dcp->connector_type = DRM_MODE_CONNECTOR_eDP; - INIT_WORK(&dcp->bl_register_wq, dcp_work_register_backlight); - mutex_init(&dcp->bl_register_mutex); - INIT_WORK(&dcp->bl_update_wq, dcp_work_update_backlight); } ret = dcp_create_piodma_iommu_dev(dcp); @@ -4343,9 +4340,6 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry set_bit(0, dcp->memdesc_map); - INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); - INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { /* Pin before the first write or mailbox wake can expose DMA. */ @@ -4617,6 +4611,12 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp_placeholder_edid_work); INIT_DELAYED_WORK(&dcp->announce_wq, dcp_announce_work); INIT_DELAYED_WORK(&dcp->bl_fallback_wq, dcp_work_backlight_fallback); + /* System sleep and shutdown cancel these even if no bind succeeded. */ + INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); + INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); + INIT_WORK(&dcp->bl_register_wq, dcp_work_register_backlight); + INIT_WORK(&dcp->bl_update_wq, dcp_work_update_backlight); + mutex_init(&dcp->bl_register_mutex); /* Balanced by enable at successful component bind. */ disable_delayed_work(&dcp->typec_reconnect_wq); disable_delayed_work(&dcp->placeholder_edid_wq); From 32418c60d4cdd6b37e2e9c7e88c7e299d473cf19 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:42:25 +1100 Subject: [PATCH 1167/1579] drm/apple: dcp: Do not hold the mailbox of an unbound DCP for sleep dcp_hold_for_sleep() holds the RTKit mailbox of an internal H17P DCP whose panel uses soft DPMS. That state is decided from the connector type, which dcp_comp_bind() sets to eDP as soon as it finds the panel node, before RTKit is initialised. If the bind then fails, dcp->rtk is left NULL (or an error pointer when apple_rtkit_init() itself failed), the IOMFB queue was never stopped and so reads as idle, and the next system suspend calls apple_rtkit_hold_rx() on it and dereferences rtk->mbox. Skip the hold when there is no RTKit instance. There is nothing to hold then, and dcp_release_after_sleep() only releases a hold that was taken. Fixes: 1efb7996cdcf ("drm/apple: Hold internal DCP messages during system sleep") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index bddcd92976d24..0dda3ffa0328a 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4867,7 +4867,9 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) */ static void dcp_hold_for_sleep(struct apple_dcp *dcp) { - if (!dcp_uses_soft_dpms(dcp) || dcp->fw_compat != DCP_FIRMWARE_H17P || + /* A failed component bind leaves no coprocessor session to hold. */ + if (IS_ERR_OR_NULL(dcp->rtk) || + !dcp_uses_soft_dpms(dcp) || dcp->fw_compat != DCP_FIRMWARE_H17P || dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || READ_ONCE(dcp->crashed)) return; From 1f6c1fbb7057db9206dce7187cfd3ec73233cd67 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:42:27 +1100 Subject: [PATCH 1168/1579] drm/apple: dcp: Release the fixed display route with the device A DCP with a fixed HDMI/DP output selects its crossbar state and takes a Type-C mux reference in probe, but neither is ever released for the device: - mux_control_select() keeps the mux control's lock until mux_control_deselect(), and devm_mux_control_get() only drops the reference. dcp_register_typec_routes() runs after the selection, and its devm_phy_get() and devm_mux_control_get() can return -EPROBE_DEFER on a pipeline that also has Type-C routes (dcpext0 on the M2 Pro/Max MacBook Pro). The next probe attempt then blocks in mux_control_select() forever, as does a probe after unbinding the driver. - The Type-C mux reference is dropped in dcp_comp_unbind() instead, so it leaks when probe fails, and every later bind/unbind cycle of the DRM device drops it once more. Release both from a devres action registered right after the crossbar control is obtained. It runs after the Type-C routes are unregistered, which may have handed the pipeline back to its fixed output, so the selection is dropped in whatever state the routes left it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0dda3ffa0328a..6cf6d75bff59a 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4154,6 +4154,23 @@ static int dcp_connector_type_from_dt(struct device_node *np) return DRM_MODE_CONNECTOR_Unknown; } +/* + * The fixed route's crossbar selection and Type-C mux are taken at probe and + * must be released with the device: a crossbar left selected blocks the next + * probe's mux_control_select() for good. + */ +static void dcp_release_fixed_route(void *data) +{ + struct apple_dcp *dcp = data; + + if (dcp->fixed_route_selected) { + mux_control_deselect(dcp->xbar); + dcp->fixed_route_selected = false; + } + typec_mux_put(dcp->typec_mux); + dcp->typec_mux = NULL; +} + VISIBLE_IF_KUNIT void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) { scoped_guard(mutex, &dcp->hpd_mutex) { @@ -4457,7 +4474,6 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v disable_irq(dcp->hdmi_hpd_irq); dcp_disable_typec_work(dcp, true); - typec_mux_put(dcp->typec_mux); if (dcp->avep) { av_service_disconnect(dcp); @@ -4723,6 +4739,9 @@ static int dcp_platform_probe(struct platform_device *pdev) dev_err(dev, "Failed to get dp-xbar: %ld\n", PTR_ERR(dcp->xbar)); return PTR_ERR(dcp->xbar); } + ret = devm_add_action_or_reset(dev, dcp_release_fixed_route, dcp); + if (ret) + return ret; ret = mux_control_select(dcp->xbar, mux_index); if (ret) dev_warn(dev, "mux_control_select failed: %d\n", ret); From c0e5730f8058f14c8fdeb5cd96213e57822f7bbb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:43:10 +1100 Subject: [PATCH 1169/1579] drm/apple: dcp: Tidy RTKit start-up logging Two lines are logged at info level on every boot of a machine with an H17P DCP, describing normal start-up rather than anything that needs attention: the retained DMA resources and the RTKit negotiation with a running coprocessor. Log them at debug level. The first was added by commit 97c4777bcb11 ("drm: apple: retain live H17P resources across component teardown") to say that rebinding is unsupported. That is still reported where it matters: probe and component bind refuse a retained session with an error saying a reboot is required. Nothing parses either line. dcp_start() also checks for a missing RTKit instance twice, the second time with an error message for a case the first check already returned on. Keep one check, covering the error pointer left by a failed apple_rtkit_init() as well; apple_drm_init_dcp() already reports the failed start. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 6cf6d75bff59a..a392e68f3086b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -105,7 +105,7 @@ static int dcp_pin_live_session(struct apple_dcp *dcp) mutex_lock(&dcp_retained_sessions_lock); list_add_tail(&session->link, &dcp_retained_sessions); mutex_unlock(&dcp_retained_sessions_lock); - dev_info(dcp->dev, "live H17P DMA resources retained until reboot; rebind unsupported\n"); + dev_dbg(dcp->dev, "live H17P DMA resources retained until reboot\n"); return 0; } @@ -3305,23 +3305,12 @@ int dcp_start(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; - if (!dcp->rtk) + /* A failed component bind leaves no RTKit instance to start. */ + if (IS_ERR_OR_NULL(dcp->rtk)) return -ENODEV; init_completion(&dcp->start_done); - /* - * The coprocessor start can fail (or be recovered from) without tearing - * the DRM side down, so this can be reached with no RTKit instance at - * all. Starting an endpoint then dereferences NULL inside - * apple_rtkit_start_ep() and takes the machine with it. - */ - if (!dcp->rtk || IS_ERR(dcp->rtk)) { - dev_err(dcp->dev, - "cannot start endpoints: RTKit is not initialised\n"); - return -ENODEV; - } - /* Make the H17P allocator available before external service discovery. */ if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); @@ -4398,7 +4387,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) goto err_piodma; if (dcp->hw.adopt_live_session) - dev_info(dev, "negotiating RTKit with the running DCP\n"); + dev_dbg(dev, "negotiating RTKit with the running DCP\n"); /* Announce host readiness before awaiting external IOP readiness. */ if (dcp->hw.iomfb_method_profile == From 765577e012da1ee83e6eb56029bdbb9f1f799dce Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:43:12 +1100 Subject: [PATCH 1170/1579] drm/apple: dptxep: Remove call counters and tidy tunnel comments The validate, connect, request and release display calls keep per-boot counters only to log them with the caller's return address at debug level. The trace events already record these calls, so drop the counters and their log lines. Several comments in the Thunderbolt DP tunnel paths retell how the code was ported and debugged rather than what it does. Reduce them to the facts a reader of this file needs. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 111 +++++++-------------------------- drivers/gpu/drm/apple/dptxep.h | 7 --- 2 files changed, 21 insertions(+), 97 deletions(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index d0cc7bdab714d..275d64d98d037 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -84,10 +84,9 @@ struct dptxport_apcall_set_tiled { }; /* - * Ported from aurora-silicon/linux#8: a Thunderbolt DP tunnel uses the same - * plain CORE|ATC|DIE|CONNECTED target as a direct alt-mode PHY. CORE is the - * DFP port (0 = dpphy, 1/2 = dpin0/dpin1, dcp->dptx_dfp_port), ATC is the - * route's own ATC index -- no separate DPIN field. + * A Thunderbolt DP tunnel uses the same CORE|ATC|DIE|CONNECTED target as a + * direct alt-mode PHY: CORE is the DFP port (0 = dpphy, 1/2 = dpin0/dpin1) + * and ATC the route's own ATC index. */ static u32 dptxport_remote_target(struct apple_dcp *dcp, u8 core, u8 atc, u8 die) @@ -107,29 +106,13 @@ int dptxport_validate_connection(struct apple_epic_service *service, u8 core, u32 target = dptxport_remote_target(service->ep->dcp, core, atc, die); /* - * attributes: role (0 = direct PHY, 1 = Thunderbolt/USB4 DP IN) | - * supportsHPD << 8. A real, hardware-validated reference - * implementation of the equivalent Thunderbolt DP tunnel mechanism - * on a different SoC (aurora-silicon/linux#8) adds this exact role - * bit for its tunnel routes; our own analog-DPIN path has never set - * it, always sending a plain direct-PHY attributes value even - * though this is a genuinely USB4-tunneled connection. DCP firmware - * appears to treat a failed AUX probe as fatal for role=0 but keeps - * training regardless for role=1 (a Thunderbolt tunnel's AUX - * proxying may not respond as fast as a direct PHY's): repeated - * test runs with substantially different Activate/crossbar ordering - * all reached the identical INACTIVE_SINK_DETECTED/DPRX-timeout - * outcome, which points at a signal we are not sending at all - * rather than a timing issue fixable by reordering. + * Attributes: role (0 = direct PHY, 1 = Thunderbolt/USB4 DP IN) | + * supports HPD << 8. Without the DP IN role DCP gives up on a tunnel + * whose AUX channel is slower to answer than a direct PHY's. */ u32 attrs = 0x100 | (dcp_is_usb4_output(service->ep->dcp) ? 1 : 0); trace_dptxport_validate_connection(dptx, core, atc, die); - dptx->validate_calls++; - dev_dbg(service->ep->dcp->dev, - "DPTX validate: call #%u this boot target=0x%x core=%u atc=%u die=%u attrs=0x%x caller=%pS\n", - dptx->validate_calls, target, core, atc, die, - attrs, __builtin_return_address(0)); cmd.target = cpu_to_le32(target); cmd.unk = cpu_to_le32(attrs); @@ -172,11 +155,6 @@ int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, u32 target = dptxport_remote_target(service->ep->dcp, core, atc, die); trace_dptxport_connect(dptx, core, atc, die); - dptx->connect_calls++; - dev_dbg(service->ep->dcp->dev, - "DPTX connect: call #%u this boot target=0x%x unk=0x%x caller=%pS\n", - dptx->connect_calls, target, unk_field, - __builtin_return_address(0)); cmd.target = cpu_to_le32(target); cmd.unk = cpu_to_le32(unk_field); @@ -200,35 +178,13 @@ int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, int dptxport_request_display(struct apple_epic_service *service) { - struct dptx_port *dptx = service->cookie; - int ret; - - dptx->request_calls++; - dev_dbg(service->ep->dcp->dev, - "DPTX request_display: call #%u this boot caller=%pS\n", - dptx->request_calls, __builtin_return_address(0)); - ret = afk_service_call(service, 0, 6, NULL, 0, 16, NULL, 0, 16); - dev_dbg(service->ep->dcp->dev, - "DPTX request_display: call #%u result=%d\n", - dptx->request_calls, ret); - return ret; + return afk_service_call(service, 0, 6, NULL, 0, 16, NULL, 0, 16); } int dptxport_release_display(struct apple_epic_service *service) { - struct dptx_port *dptx = service->cookie; - int ret; - dcp_180hz_invalidate(service->ep->dcp); - dptx->release_calls++; - dev_dbg(service->ep->dcp->dev, - "DPTX release_display: call #%u this boot caller=%pS\n", - dptx->release_calls, __builtin_return_address(0)); - ret = afk_service_call(service, 0, 7, NULL, 0, 16, NULL, 0, 16); - dev_dbg(service->ep->dcp->dev, - "DPTX release_display: call #%u result=%d\n", - dptx->release_calls, ret); - return ret; + return afk_service_call(service, 0, 7, NULL, 0, 16, NULL, 0, 16); } int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, @@ -467,23 +423,8 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic dev_dbg(dcp->dev, "USB4/DPTX: SET_ACTIVE_LANE_COUNT %llu\n", lane_count); /* - * dcp_usb4_drm_allowed() (usb4_force_dptx) used to gate this - * for the old manual-training sysfs knob (module_param_cb - * usb4_dptx_train), removed wholesale in 0dc9f50 when - * apple_dcp_tb_dp_tunnel() replaced it with automatic tunnel - * detection -- but usb4_force_dptx itself, and this gate, - * were left behind with no way left to ever set it true. - * dcp_dptx_connect()'s single wait_for_completion_timeout() - * on linkcfg_completion is now shared by both the alt-mode - * and USB4-tunnel paths (also part of 0dc9f50's unification), - * so gating it here meant every tunneled connect timed out - * after a fully successful DPRX/AUX handshake and lane - * negotiation -- firmware does not send FORCE_HOTPLUG_DETECT - * (the only other completion site) for a tunnel. The - * reference (aurora-silicon/linux#8) completes - * linkcfg_completion here unconditionally; usb4_lane_completion - * has no consumer anywhere in this tree, so this drops both the - * dead gate and the dead completion and matches the reference. + * dcp_dptx_connect() waits for this on alt-mode and tunnel + * routes alike: DCP sends no FORCE_HOTPLUG_DETECT for a tunnel. */ complete(&dptx->linkcfg_completion); } @@ -523,11 +464,8 @@ static int dptxport_call_did_change_link_config(struct apple_epic_service *service) { /* - * Ported from aurora-silicon/linux#8: bringing the tunnel crossbar - * connection up (dcp_tunnel_crossbar_up()) now happens in - * dptxport_call(), after this succeeds, gated on - * dptx_tunnel && link_rate -- not inside this handler, and its - * result is never propagated back to DCP as a call failure. + * dptxport_call() brings a tunnel's crossbar connection up after + * this; its result is never reported to DCP as a call failure. */ /* assume the link config did change and wait a little bit */ @@ -589,11 +527,10 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, if (phy_set_rate) { /* - * Ported from aurora-silicon/linux#8: a Thunderbolt DP tunnel - * starts/stops its own pixel clock instead of configuring the - * PHY directly, and never fails the apcall over it -- a - * failed clock just keeps the crossbar connection down - * (dcp->tb_clock_ok), which dcp_tunnel_crossbar_up() checks. + * A Thunderbolt DP tunnel starts and stops its own pixel clock + * instead of configuring the PHY, and never fails the call over + * it: a failed clock keeps the crossbar connection down, see + * dcp_tunnel_crossbar_up(). */ if (dptx->atcphy && service->ep->dcp->dptx_tunnel) { dcp_tunnel_set_rate(service->ep->dcp, dptx->atcphy, @@ -673,12 +610,10 @@ dptxport_call_activate(struct apple_epic_service *service, struct apple_dcp *dcp = service->ep->dcp; /* - * Ported from aurora-silicon/linux#8: no crossbar, and no PHY mode - * change for a tunnel (dptx->atcphy must stay in USB4/TBT mode the - * whole connection, see dcp_tunnel_set_rate()). The DP IN adapter is - * only woken (DPTX_INACTIVE=0) here, via dcp_tunnel_dpin_activate(); - * waking it earlier hangs the machine. Activate always replies - * success to DCP. + * No PHY mode change for a tunnel: the ATC PHY stays in USB4/TBT + * mode for the whole connection. The DP IN adapter may only be woken + * (DPTX_INACTIVE=0) from here; waking it earlier hangs the machine. + * Activate always replies success to DCP. */ if (dptx->atcphy && !dcp->phy_managed_by_typec) phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); @@ -764,11 +699,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, switch (idx) { case DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG: - /* - * Ported from aurora-silicon/linux#8: a re-link on an - * established tunnel takes the crossbar connection down - * first. - */ + /* A re-link takes an established tunnel's crossbar down first. */ if (service->ep->dcp->dptx_tunnel && dptx->link_rate) dcp_tunnel_crossbar_down(service->ep->dcp); return dptxport_call_will_change_link_config(service); diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index e731604fb6768..f5096c59e8df8 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -63,13 +63,6 @@ struct dptx_port { u32 link_rate, pending_link_rate; u32 drive_settings[2]; u32 trained_voltage[4], trained_pre[4]; - /* - * Per-boot invocation counters for the validate/connect/request/ - * release AFK calls, used together with the caller return address - * logged alongside them to correlate call ordering when diagnosing - * link-training sequencing issues. - */ - u32 validate_calls, connect_calls, request_calls, release_calls; }; int dptxport_validate_connection(struct apple_epic_service *service, u8 core, From 63d3b0f419399d027c865d4ccffa12e3617a73b8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:27:44 +1100 Subject: [PATCH 1171/1579] drm: apple: afk: factor the receive bounds check into a helper afk_recv() open-codes the same "does this queue entry fit before the end of the ring" test twice, once to detect the wrap-around dummy entry and once to reject an entry that still does not fit. Move it into afk_ring_payload_fits() so the predicate can be tested without a live co-processor. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 4 ++-- drivers/gpu/drm/apple/afk.h | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 6ca27849346f8..85ed81c7d2974 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1220,7 +1220,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) * the current dummy queue entry and we have to advance to the next one * which will contain the real data. */ - if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { + if (!afk_ring_payload_fits(rptr, size, ep->rxbfr.bufsz)) { rptr = 0; hdr = ep->rxbfr.buf + rptr; magic = le32_to_cpu(hdr->magic); @@ -1235,7 +1235,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) } } - if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { + if (!afk_ring_payload_fits(rptr, size, ep->rxbfr.bufsz)) { dev_warn(ep->dcp->dev, "AFK[ep:%02x]: queue entry out of bounds: 0x%lx > 0x%lx\n", ep->endpoint, rptr + size + sizeof(*hdr), ep->rxbfr.bufsz); diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 0a12ecda982df..b9cbe586370a9 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -266,6 +266,11 @@ struct afk_qe { u8 data[]; }; +static inline bool afk_ring_payload_fits(u32 ptr, u32 bytes, size_t bufsz) +{ + return ptr + bytes + sizeof(struct afk_qe) <= bufsz; +} + struct epic_hdr { u8 version; __le16 seq; From 9142d428cb6956aeb36ffebabe42e1a5c2396c2c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:28:00 +1100 Subject: [PATCH 1172/1579] drm: apple: afk: bound received queue entries without 32-bit wrap afk_recv() checks that a receive queue entry fits in the ring with "rptr + size + sizeof(*hdr) > bufsz". rptr and size are both u32 read from memory the co-processor writes, so their sum is computed in 32 bits and wraps before it is widened. An entry at offset 0x40 that claims 0xfffffff8 bytes passes both checks, and the entry is then copied into the receive scratch buffer, which is only bufsz bytes long. Since the scratch buffer was introduced this is a heap overflow driven by a single bad length; before it, the handlers read far past the ring. Do the check in afk_ring_payload_fits() with subtractions that cannot wrap, report the entry size and offset instead of a derived end, and add KUnit cases for the exact end, offsets past the ring and wrapping sums. Fixes: 63861816dc0c ("drm/apple: preallocate the AFK receive dispatch buffer") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 4 ++-- drivers/gpu/drm/apple/afk.h | 3 ++- drivers/gpu/drm/apple/afk_test.c | 21 +++++++++++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 85ed81c7d2974..5b69cb14e70e3 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1237,8 +1237,8 @@ static bool afk_recv(struct apple_dcp_afkep *ep) if (!afk_ring_payload_fits(rptr, size, ep->rxbfr.bufsz)) { dev_warn(ep->dcp->dev, - "AFK[ep:%02x]: queue entry out of bounds: 0x%lx > 0x%lx\n", - ep->endpoint, rptr + size + sizeof(*hdr), ep->rxbfr.bufsz); + "AFK[ep:%02x]: queue entry of 0x%x bytes at 0x%x out of bounds\n", + ep->endpoint, size, rptr); return false; } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index b9cbe586370a9..47d140441a15e 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -268,7 +268,8 @@ struct afk_qe { static inline bool afk_ring_payload_fits(u32 ptr, u32 bytes, size_t bufsz) { - return ptr + bytes + sizeof(struct afk_qe) <= bufsz; + return ptr <= bufsz && sizeof(struct afk_qe) <= bufsz - ptr && + bytes <= bufsz - ptr - sizeof(struct afk_qe); } struct epic_hdr { diff --git a/drivers/gpu/drm/apple/afk_test.c b/drivers/gpu/drm/apple/afk_test.c index e58a952642045..c3130086c78e1 100644 --- a/drivers/gpu/drm/apple/afk_test.c +++ b/drivers/gpu/drm/apple/afk_test.c @@ -29,8 +29,29 @@ static void afk_ring_stride_test(struct kunit *test) KUNIT_EXPECT_EQ(test, afk_ring_advance(0xfc0, 0x40, 0x1000, stride), 0U); } +static void afk_ring_payload_fits_test(struct kunit *test) +{ + const size_t hdr = sizeof(struct afk_qe); + + /* An entry ending exactly at the end of the ring fits. */ + KUNIT_EXPECT_TRUE(test, afk_ring_payload_fits(0, 0x1000 - hdr, 0x1000)); + KUNIT_EXPECT_TRUE(test, afk_ring_payload_fits(0xfc0, 0x30, 0x1000)); + KUNIT_EXPECT_TRUE(test, afk_ring_payload_fits(0x1000 - hdr, 0, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0xfc0, 0x31, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0xff8, 0, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0x1000, 0, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0x1040, 0, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0, 0, hdr - 1)); + /* Sizes that wrap a 32-bit offset sum must not pass for a fit. */ + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0x40, 0xfffffff8, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0x40, U32_MAX - 0x3f, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0x100, U32_MAX - 0xff, 0x1000)); + KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0, U32_MAX, 0x1000)); +} + static struct kunit_case afk_ring_cases[] = { KUNIT_CASE(afk_ring_stride_test), + KUNIT_CASE(afk_ring_payload_fits_test), {} }; From 70d9b6f64ee76e9da96b752ee3933e2d0a66af0b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:45:28 +1100 Subject: [PATCH 1173/1579] drm: apple: free property dictionaries the connector does not keep dcp_connector_update_dict() keeps the ColorElements, TimingElements, DisplayAttributes and Transport dictionaries for debugfs and clears the caller's pointer for every key, so the buffer of any other key is lost. That used to be harmless because the chunk buffer was device-managed. Since it became a plain kzalloc() allocation, every property transfer with another key leaks up to the 1 MiB the firmware may announce, for as long as the system runs. The switch away from devm is deliberate and stays: a Type-C port can move to another DCP before its dictionaries are replaced, and freeing against the connector's current DCP then fails devres_destroy(). Free the buffer with kfree() when no dictionary takes ownership of it. Fixes: 38c14f6107df ("drm: apple: support ProMotion variable refresh rate") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/connector.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/apple/connector.c b/drivers/gpu/drm/apple/connector.c index c7a2d97087499..27082dc21038e 100644 --- a/drivers/gpu/drm/apple/connector.c +++ b/drivers/gpu/drm/apple/connector.c @@ -148,6 +148,8 @@ void dcp_connector_update_dict(struct apple_connector *connector, const char *ke dcp_connector_set_dict(connector, &connector->display_attributes, chunks); else if (!strcmp(key, "Transport")) dcp_connector_set_dict(connector, &connector->transport, chunks); + else + kfree(chunks->data); chunks->data = NULL; chunks->length = 0; From 543ac70dce46b749f1a446cb80d4e63cf08949cd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:45:30 +1100 Subject: [PATCH 1174/1579] drm: apple: parser: add KUnit tests for malformed dictionaries The serialized dictionaries come from the DCP firmware, which the driver does not trust. Add a suite that feeds the parser hand-built blobs: a valid EPIC service announcement, the H17P spelling of its keys with and without the H17P flag, strings and dictionaries running past the end of the blob, nesting deeper than the skip limit, a bad header, and the extent check used for delivered property payloads. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 2 +- drivers/gpu/drm/apple/parser_test.c | 220 ++++++++++++++++++++++++++++ 2 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/apple/parser_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index be9ada873650c..befd8765cff40 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -11,7 +11,7 @@ appledrm-y += ibootep.o appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o appledrm-y += iomfb_h17p.o -appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o parser_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_resume_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_ready_test.o diff --git a/drivers/gpu/drm/apple/parser_test.c b/drivers/gpu/drm/apple/parser_test.c new file mode 100644 index 0000000000000..7dd8ca6eac189 --- /dev/null +++ b/drivers/gpu/drm/apple/parser_test.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include +#include +#include + +#include "parser.h" + +/* Serialized dictionary tags: size in bits 0-23, type in 24-28, last in 31. */ +#define T_DICT 1 +#define T_ARRAY 2 +#define T_INT64 4 +#define T_STRING 9 +#define T_BOOL 11 + +struct blob { + u8 data[512]; + size_t len; +}; + +static void blob_align(struct blob *b) +{ + b->len = round_up(b->len, 4); +} + +static void blob_tag(struct blob *b, u32 type, u32 size) +{ + blob_align(b); + put_unaligned_le32(size | type << 24, b->data + b->len); + b->len += 4; +} + +static void blob_init(struct blob *b) +{ + memset(b, 0, sizeof(*b)); + put_unaligned_le32(0xd3, b->data); + b->len = 4; +} + +static void blob_string(struct blob *b, const char *s) +{ + size_t n = strlen(s); + + blob_tag(b, T_STRING, n); + memcpy(b->data + b->len, s, n); + b->len += n; +} + +static void blob_int(struct blob *b, s64 value) +{ + blob_tag(b, T_INT64, 64); + put_unaligned_le64(value, b->data + b->len); + b->len += 8; +} + +static int parse_service(struct kunit *test, const struct blob *b, bool h17p, + const char **name, const char **class, s64 *unit) +{ + struct dcp_parse_ctx ctx; + + KUNIT_ASSERT_EQ(test, parse(b->data, b->len, &ctx), 0); + return parse_epic_service_init(&ctx, name, class, unit, h17p); +} + +static void parser_service_init_test(struct kunit *test) +{ + const char *name, *class; + struct blob b; + s64 unit = -1; + + blob_init(&b); + blob_tag(&b, T_DICT, 4); + blob_string(&b, "EPICName"); + blob_string(&b, "dcpav-service-epic"); + blob_string(&b, "Other"); + blob_tag(&b, T_BOOL, 1); + blob_string(&b, "EPICProviderClass"); + blob_string(&b, "AppleDCPAVService"); + blob_string(&b, "EPICUnit"); + blob_int(&b, 2); + + KUNIT_ASSERT_EQ(test, parse_service(test, &b, false, &name, &class, &unit), 0); + KUNIT_EXPECT_STREQ(test, name, "dcpav-service-epic"); + KUNIT_EXPECT_STREQ(test, class, "AppleDCPAVService"); + KUNIT_EXPECT_EQ(test, unit, 2); + kfree(name); + kfree(class); +} + +static void parser_service_init_h17p_keys_test(struct kunit *test) +{ + const char *name, *class; + struct blob b; + s64 unit = -1; + + blob_init(&b); + blob_tag(&b, T_DICT, 3); + blob_string(&b, "compartment"); + blob_string(&b, "system"); + blob_string(&b, "name"); + blob_string(&b, "system"); + blob_string(&b, "interface-id"); + blob_int(&b, 1); + + KUNIT_EXPECT_EQ(test, parse_service(test, &b, false, &name, &class, &unit), + -ENOENT); + KUNIT_EXPECT_TRUE(test, IS_ERR(name)); + KUNIT_EXPECT_TRUE(test, IS_ERR(class)); + + KUNIT_ASSERT_EQ(test, parse_service(test, &b, true, &name, &class, &unit), 0); + KUNIT_EXPECT_STREQ(test, name, "system"); + KUNIT_EXPECT_STREQ(test, class, "system"); + KUNIT_EXPECT_EQ(test, unit, 1); + kfree(name); + kfree(class); +} + +static void parser_service_init_truncated_test(struct kunit *test) +{ + const char *name, *class; + struct blob b; + s64 unit; + + /* A string length running past the end of the blob. */ + blob_init(&b); + blob_tag(&b, T_DICT, 3); + blob_string(&b, "EPICName"); + blob_tag(&b, T_STRING, 0x100); + memcpy(b.data + b.len, "dcp", 3); + b.len += 3; + + KUNIT_EXPECT_LT(test, parse_service(test, &b, false, &name, &class, &unit), 0); + KUNIT_EXPECT_TRUE(test, IS_ERR(name)); + KUNIT_EXPECT_TRUE(test, IS_ERR(class)); + + /* A dictionary announcing more pairs than the blob holds. */ + blob_init(&b); + blob_tag(&b, T_DICT, 0xffffff); + blob_string(&b, "EPICName"); + blob_string(&b, "dcp"); + + KUNIT_EXPECT_LT(test, parse_service(test, &b, false, &name, &class, &unit), 0); + KUNIT_EXPECT_TRUE(test, IS_ERR(name)); + KUNIT_EXPECT_TRUE(test, IS_ERR(class)); +} + +static void parser_nesting_test(struct kunit *test) +{ + const char *name, *class; + struct blob b; + unsigned int i; + s64 unit; + + /* An ignored value nested far deeper than any real dictionary. */ + blob_init(&b); + blob_tag(&b, T_DICT, 4); + blob_string(&b, "EPICName"); + blob_string(&b, "dcp"); + blob_string(&b, "EPICProviderClass"); + blob_string(&b, "dcp"); + blob_string(&b, "EPICUnit"); + blob_int(&b, 0); + blob_string(&b, "Ignored"); + for (i = 0; i < 64; i++) + blob_tag(&b, T_ARRAY, 1); + blob_tag(&b, T_BOOL, 0); + + KUNIT_EXPECT_LT(test, parse_service(test, &b, false, &name, &class, &unit), 0); + KUNIT_EXPECT_TRUE(test, IS_ERR(name)); + KUNIT_EXPECT_TRUE(test, IS_ERR(class)); +} + +static void parser_bad_header_test(struct kunit *test) +{ + struct dcp_parse_ctx ctx; + struct blob b; + + blob_init(&b); + KUNIT_EXPECT_EQ(test, parse(b.data, 3, &ctx), -EINVAL); + put_unaligned_le32(0xd4, b.data); + KUNIT_EXPECT_EQ(test, parse(b.data, b.len, &ctx), -EINVAL); +} + +static void parser_property_complete_test(struct kunit *test) +{ + struct blob b; + + blob_init(&b); + blob_tag(&b, T_DICT, 1); + blob_string(&b, "Key"); + blob_int(&b, 7); + + KUNIT_EXPECT_TRUE(test, dcp_property_complete(b.data, b.len, b.len)); + /* One unsent byte after the aligned object is tolerated. */ + KUNIT_EXPECT_TRUE(test, dcp_property_complete(b.data, b.len, b.len + 1)); + KUNIT_EXPECT_FALSE(test, dcp_property_complete(b.data, b.len, b.len + 4)); + KUNIT_EXPECT_FALSE(test, dcp_property_complete(b.data, b.len - 4, b.len)); + KUNIT_EXPECT_FALSE(test, dcp_property_complete(b.data, b.len + 4, b.len + 4)); + KUNIT_EXPECT_FALSE(test, dcp_property_complete(NULL, b.len, b.len)); +} + +static struct kunit_case parser_cases[] = { + KUNIT_CASE(parser_service_init_test), + KUNIT_CASE(parser_service_init_h17p_keys_test), + KUNIT_CASE(parser_service_init_truncated_test), + KUNIT_CASE(parser_nesting_test), + KUNIT_CASE(parser_bad_header_test), + KUNIT_CASE(parser_property_complete_test), + {} +}; + +static struct kunit_suite parser_suite = { + .name = "apple-dcp-parser", + .test_cases = parser_cases, +}; + +kunit_test_suite(parser_suite); From 3d0c9c8899198a8ad6d0d0241283ba04d2286fd7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:45:33 +1100 Subject: [PATCH 1175/1579] drm: apple: parser: bound the timing catalog allocation enumerate_modes() allocates one display mode for every element the firmware announces in the TimingElements array before looking at any of them. The count is a 24-bit field, so a malformed catalog asks kcalloc() for gigabytes, which fails with an allocation warning instead of being rejected as malformed input. Every element takes at least a four-byte tag, so reject counts larger than the rest of the blob can hold, and pass __GFP_NOWARN so a catalog that is merely too large to allocate fails quietly like any other malformed one. Add a KUnit case for both an impossible count and a catalog whose entries are present but invalid. Fixes: 38c14f6107df ("drm: apple: support ProMotion variable refresh rate") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 10 +++++++-- drivers/gpu/drm/apple/parser_test.c | 32 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 0ff3c2d79e805..d67eaae16d0f5 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -1018,8 +1018,14 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, if (ret) return ERR_PTR(ret); - /* Start with a worst case allocation. */ - modes = kcalloc(it.len, sizeof(*modes), GFP_KERNEL); + /* + * Start with a worst case allocation. The count comes from firmware: + * every entry needs at least a tag in what is left of the blob, and a + * catalog too large to allocate fails like any other malformed one. + */ + if (it.len > (handle->len - handle->pos) / sizeof(struct dcp_parse_tag)) + return ERR_PTR(-EINVAL); + modes = kcalloc(it.len, sizeof(*modes), GFP_KERNEL | __GFP_NOWARN); *count = 0; if (!modes) diff --git a/drivers/gpu/drm/apple/parser_test.c b/drivers/gpu/drm/apple/parser_test.c index 7dd8ca6eac189..6795fcdf18864 100644 --- a/drivers/gpu/drm/apple/parser_test.c +++ b/drivers/gpu/drm/apple/parser_test.c @@ -184,6 +184,37 @@ static void parser_bad_header_test(struct kunit *test) KUNIT_EXPECT_EQ(test, parse(b.data, b.len, &ctx), -EINVAL); } +static void parser_mode_count_test(struct kunit *test) +{ + struct dcp_display_mode *modes; + struct dcp_parse_ctx ctx; + unsigned int count = 1; + struct blob b; + + /* An element count that cannot be backed by the bytes present. */ + blob_init(&b); + blob_tag(&b, T_ARRAY, 0xffffff); + blob_tag(&b, T_DICT, 0); + + KUNIT_ASSERT_EQ(test, parse(b.data, b.len, &ctx), 0); + modes = enumerate_modes(&ctx, &count, 0, 0, 0, false, false); + KUNIT_EXPECT_PTR_EQ(test, modes, ERR_PTR(-EINVAL)); + + /* Entries that are present but invalid are skipped, not fatal. */ + blob_init(&b); + blob_tag(&b, T_ARRAY, 2); + blob_tag(&b, T_DICT, 0); + blob_tag(&b, T_DICT, 1); + blob_string(&b, "ID"); + blob_int(&b, 1); + + KUNIT_ASSERT_EQ(test, parse(b.data, b.len, &ctx), 0); + modes = enumerate_modes(&ctx, &count, 0, 0, 0, false, false); + KUNIT_ASSERT_FALSE(test, IS_ERR(modes)); + KUNIT_EXPECT_EQ(test, count, 0); + kfree(modes); +} + static void parser_property_complete_test(struct kunit *test) { struct blob b; @@ -208,6 +239,7 @@ static struct kunit_case parser_cases[] = { KUNIT_CASE(parser_service_init_truncated_test), KUNIT_CASE(parser_nesting_test), KUNIT_CASE(parser_bad_header_test), + KUNIT_CASE(parser_mode_count_test), KUNIT_CASE(parser_property_complete_test), {} }; From ba69d4dde9751976c323055ef683d76febf82a2b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:29:16 +1100 Subject: [PATCH 1176/1579] drm: apple: afk: validate the receive write pointer and bound a drain afk_recv() checks the read pointer against the ring, but uses the write pointer, which the co-processor owns, only to decide when the ring is empty. The receive worker calls afk_recv() until it reports an empty ring. Since the consumer advances rptr modulo the ring in whole blocks, a write pointer at or past the end of the ring, or one that is not block aligned, is never reached: with every slot holding a valid entry the worker dispatches the same ring contents forever and holds the ordered endpoint queue. A producer that keeps moving a valid wptr ahead of rptr has the same effect. Check both pointers on every read, before any entry is touched: rptr must leave room for a queue entry header, wptr must lie inside the ring, and both must be block aligned, which is how the consumer advances and therefore what the producer has to write for the ring to drain. Treat a violation like the other invalid-entry cases and stop draining. Also bound one RECV notification to as many entries as the ring has blocks. A well-behaved producer cannot have more than that queued when the notification arrives; entries it writes during the pass come with their own RECV notification, which drains them. The pointer and budget checks are pure helpers with KUnit cases for an end-of-ring and an all-ones write pointer, misaligned pointers and a write pointer that keeps moving. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 32 +++++++++++++++----- drivers/gpu/drm/apple/afk.h | 35 ++++++++++++++++++++++ drivers/gpu/drm/apple/afk_test.c | 51 ++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 5b69cb14e70e3..c0ccd3676cd56 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1192,15 +1192,15 @@ static bool afk_recv(struct apple_dcp_afkep *ep) wptr = le32_to_cpu(*ep->rxbfr.wptr); trace_afk_recv_rwptr_pre(ep, rptr, wptr); - if (rptr == wptr) + if (!afk_ring_ptrs_valid(rptr, wptr, ep->rxbfr.bufsz, ep->rxbfr.stride)) { + dev_warn_ratelimited(ep->dcp->dev, + "AFK[ep:%02x]: invalid receive ring pointers: rptr 0x%x wptr 0x%x\n", + ep->endpoint, rptr, wptr); return false; + } - if (rptr > (ep->rxbfr.bufsz - sizeof(*hdr))) { - dev_warn(ep->dcp->dev, - "AFK[ep:%02x]: rptr out of bounds: 0x%x > 0x%lx\n", - ep->endpoint, rptr, ep->rxbfr.bufsz - sizeof(*hdr)); + if (rptr == wptr) return false; - } dma_rmb(); @@ -1264,6 +1264,23 @@ static bool afk_recv(struct apple_dcp_afkep *ep) return true; } +static bool afk_recv_entry(void *ep) +{ + return afk_recv(ep); +} + +static void afk_recv_drain(struct apple_dcp_afkep *ep) +{ + unsigned int budget = afk_ring_drain_budget(ep->rxbfr.bufsz, + ep->rxbfr.stride); + + if (afk_ring_drain(afk_recv_entry, ep, budget) == budget && + ep->rxbfr.ready && le32_to_cpu(*ep->rxbfr.rptr) != le32_to_cpu(*ep->rxbfr.wptr)) + dev_warn_ratelimited(ep->dcp->dev, + "AFK[ep:%02x]: receive ring did not drain in one pass\n", + ep->endpoint); +} + static void afk_receive_message_worker(struct work_struct *work_) { struct afk_receive_message_work *work; @@ -1298,8 +1315,7 @@ static void afk_receive_message_worker(struct work_struct *work_) break; case RBEP_RECV: - while (afk_recv(work->ep)) - ; + afk_recv_drain(work->ep); break; default: diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 47d140441a15e..a85ed77597755 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -272,6 +272,41 @@ static inline bool afk_ring_payload_fits(u32 ptr, u32 bytes, size_t bufsz) bytes <= bufsz - ptr - sizeof(struct afk_qe); } +/* + * Check the receive ring pointers before reading an entry. The consumer + * only advances rptr in whole blocks of @stride bytes and stops when it + * meets wptr, so the producer has to keep wptr block aligned and inside + * the ring as well; wptr == bufsz is never written, it wraps to 0. + */ +static inline bool afk_ring_ptrs_valid(u32 rptr, u32 wptr, size_t bufsz, + u32 stride) +{ + return stride && bufsz >= sizeof(struct afk_qe) && + rptr <= bufsz - sizeof(struct afk_qe) && wptr < bufsz && + !(rptr % stride) && !(wptr % stride); +} + +/* + * One RECV notification drains at most as many entries as the ring has + * blocks, so a producer that keeps moving wptr cannot hold the ordered + * receive worker. Entries written after this pass come with their own + * RECV notification. + */ +static inline unsigned int afk_ring_drain_budget(size_t bufsz, u32 stride) +{ + return stride ? bufsz / stride : 1; +} + +static inline unsigned int afk_ring_drain(bool (*recv)(void *ctx), void *ctx, + unsigned int budget) +{ + unsigned int n = 0; + + while (n < budget && recv(ctx)) + n++; + return n; +} + struct epic_hdr { u8 version; __le16 seq; diff --git a/drivers/gpu/drm/apple/afk_test.c b/drivers/gpu/drm/apple/afk_test.c index c3130086c78e1..0621eebe914df 100644 --- a/drivers/gpu/drm/apple/afk_test.c +++ b/drivers/gpu/drm/apple/afk_test.c @@ -49,7 +49,58 @@ static void afk_ring_payload_fits_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, afk_ring_payload_fits(0, U32_MAX, 0x1000)); } +static void afk_ring_ptrs_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, afk_ring_ptrs_valid(0, 0, 0x1000, 0x40)); + KUNIT_EXPECT_TRUE(test, afk_ring_ptrs_valid(0xfc0, 0x40, 0x1000, 0x40)); + KUNIT_EXPECT_TRUE(test, afk_ring_ptrs_valid(0x80, 0xf80, 0x1000, 0x80)); + /* The write pointer must lie inside the ring. */ + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0, 0x1000, 0x1000, 0x40)); + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0, U32_MAX, 0x1000, 0x40)); + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0x1000, 0, 0x1000, 0x40)); + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(U32_MAX, 0, 0x1000, 0x40)); + /* Both pointers must be block aligned. */ + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0, 0x20, 0x1000, 0x40)); + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0x10, 0, 0x1000, 0x40)); + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0x40, 0x80, 0x1000, 0x80)); + /* An uninitialised ring has no valid pointers. */ + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0, 0, 0x1000, 0)); + KUNIT_EXPECT_FALSE(test, afk_ring_ptrs_valid(0, 0, 0, 0x40)); +} + +struct afk_drain_ctx { + unsigned int calls, entries; +}; + +/* A producer that publishes @entries entries, or one that never stops. */ +static bool afk_drain_recv(void *data) +{ + struct afk_drain_ctx *ctx = data; + + return ++ctx->calls <= ctx->entries; +} + +static void afk_ring_drain_test(struct kunit *test) +{ + struct afk_drain_ctx ctx = { .entries = 3 }; + unsigned int budget = afk_ring_drain_budget(0x1000, 0x40); + + KUNIT_EXPECT_EQ(test, budget, 64U); + KUNIT_EXPECT_EQ(test, afk_ring_drain(afk_drain_recv, &ctx, budget), 3U); + KUNIT_EXPECT_EQ(test, ctx.calls, 4U); + + /* A write pointer that keeps moving ahead exhausts the budget. */ + ctx = (struct afk_drain_ctx){ .entries = UINT_MAX }; + KUNIT_EXPECT_EQ(test, afk_ring_drain(afk_drain_recv, &ctx, budget), budget); + KUNIT_EXPECT_EQ(test, ctx.calls, budget); + + /* Without a ring the single pass reports the missing setup. */ + KUNIT_EXPECT_EQ(test, afk_ring_drain_budget(0, 0), 1U); +} + static struct kunit_case afk_ring_cases[] = { + KUNIT_CASE(afk_ring_ptrs_test), + KUNIT_CASE(afk_ring_drain_test), KUNIT_CASE(afk_ring_stride_test), KUNIT_CASE(afk_ring_payload_fits_test), {} From 033943b2b6edb176aaebffa917e2273c3bf8f625 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:30:04 +1100 Subject: [PATCH 1177/1579] drm: apple: parser: cap the number of timing modes enumerate_modes() allocates one struct dcp_display_mode (320 bytes) for every element of the firmware's TimingElements array. The previous fix only rejects counts the blob cannot hold, so a 1 MiB property transfer of empty dictionaries can still request about 80 MiB. The published catalog is deliberately never freed while mode readers may hold pointers into it (see dcp_modes_snapshot()), so repeated deliveries retain each oversized array. Reject catalogs with more than DCP_MAX_TIMING_MODES (1024) elements before allocating. DCP reports one element per timing it derived from the sink's EDID and DisplayID; every catalog in the J700 boot and hotplug logs collected so far has at most 14 entries (1, 6 or 14 are the only non-zero counts seen). 1024 is far above anything an EDID with extension blocks and DisplayID can describe in practice, and bounds the array at 320 KiB. The retention itself is left as it is. Add KUnit cases for a catalog at the maximum, which is accepted, and one element above it, which is rejected. Fixes: 38c14f6107df ("drm: apple: support ProMotion variable refresh rate") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 8 +++++-- drivers/gpu/drm/apple/parser.h | 8 +++++++ drivers/gpu/drm/apple/parser_test.c | 36 +++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index d67eaae16d0f5..74e7e7d362dab 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -1020,11 +1020,15 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, /* * Start with a worst case allocation. The count comes from firmware: - * every entry needs at least a tag in what is left of the blob, and a - * catalog too large to allocate fails like any other malformed one. + * every entry needs at least a tag in what is left of the blob, and no + * display has more timings than DCP_MAX_TIMING_MODES. The catalog is + * retained once published, so reject anything larger before + * allocating for it. */ if (it.len > (handle->len - handle->pos) / sizeof(struct dcp_parse_tag)) return ERR_PTR(-EINVAL); + if (it.len > DCP_MAX_TIMING_MODES) + return ERR_PTR(-E2BIG); modes = kcalloc(it.len, sizeof(*modes), GFP_KERNEL | __GFP_NOWARN); *count = 0; diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 3b21263154754..75c866518ed52 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -117,6 +117,14 @@ struct dimension { s64 pixel_repetition; }; +/* + * Upper bound on the TimingElements catalog. DCP reports one element per + * timing it derived from the sink's EDID and DisplayID, and every catalog + * logged on J700 so far holds at most 14. The cap leaves ample headroom + * while keeping the retained array (320 bytes per entry) below 1 MiB. + */ +#define DCP_MAX_TIMING_MODES 1024 + int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); bool dcp_property_complete(const void *blob, size_t received, size_t advertised); struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, diff --git a/drivers/gpu/drm/apple/parser_test.c b/drivers/gpu/drm/apple/parser_test.c index 6795fcdf18864..7fc03175a2324 100644 --- a/drivers/gpu/drm/apple/parser_test.c +++ b/drivers/gpu/drm/apple/parser_test.c @@ -215,6 +215,41 @@ static void parser_mode_count_test(struct kunit *test) kfree(modes); } +/* Build an array of @count empty dictionaries in a test allocation. */ +static struct dcp_display_mode *parse_empty_modes(struct kunit *test, + unsigned int count, + unsigned int *found) +{ + size_t len = 8 + 4 * count; + struct dcp_parse_ctx ctx; + unsigned int i; + u8 *data; + + data = kunit_kzalloc(test, len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, data); + put_unaligned_le32(0xd3, data); + put_unaligned_le32(count | T_ARRAY << 24, data + 4); + for (i = 0; i < count; i++) + put_unaligned_le32(T_DICT << 24, data + 8 + 4 * i); + + KUNIT_ASSERT_EQ(test, parse(data, len, &ctx), 0); + return enumerate_modes(&ctx, found, 0, 0, 0, false, false); +} + +static void parser_mode_cap_test(struct kunit *test) +{ + struct dcp_display_mode *modes; + unsigned int count = 1; + + modes = parse_empty_modes(test, DCP_MAX_TIMING_MODES, &count); + KUNIT_ASSERT_FALSE(test, IS_ERR(modes)); + KUNIT_EXPECT_EQ(test, count, 0); + kfree(modes); + + modes = parse_empty_modes(test, DCP_MAX_TIMING_MODES + 1, &count); + KUNIT_EXPECT_PTR_EQ(test, modes, ERR_PTR(-E2BIG)); +} + static void parser_property_complete_test(struct kunit *test) { struct blob b; @@ -240,6 +275,7 @@ static struct kunit_case parser_cases[] = { KUNIT_CASE(parser_nesting_test), KUNIT_CASE(parser_bad_header_test), KUNIT_CASE(parser_mode_count_test), + KUNIT_CASE(parser_mode_cap_test), KUNIT_CASE(parser_property_complete_test), {} }; From b8f892b79561ea085c0ab2e44368f873f15b9183 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:46:27 +1100 Subject: [PATCH 1178/1579] soc: apple: aop: Decode ring entry headers without pointer casts The ring entry header is read by casting a byte array to a QEHeader reference and written by viewing the struct as a byte slice, both in unsafe blocks with no safety comment, and clippy flags the casts. Encode and decode the header field by field as little-endian words instead, as the driver already does for the EPIC headers, and hoist the duplicated QEH_SIZE constant. While at it, address the remaining clippy findings in the driver: add the missing safety comments, use as_char_ptr() and ptr::from_ref() instead of pointer casts when filling the platform_device_info, and drop a no-op u64 cast in patch_bootargs(). No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 52 +++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index cf79092c1fa7b..f01c510dba88b 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -209,6 +209,7 @@ fn le_u64(b: &[u8], off: usize) -> u64 { #[inline(always)] fn mem_sync() { + // SAFETY: a full-system barrier touches no memory or registers of ours. unsafe { asm!("dsb sy"); } @@ -223,6 +224,29 @@ struct QEHeader { ty: u32, } +/// The size of a ring entry header. +const QEH_SIZE: usize = mem::size_of::(); + +impl QEHeader { + fn from_bytes(b: &[u8; QEH_SIZE]) -> QEHeader { + QEHeader { + magic: le_u32(b, 0), + size: le_u32(b, 4), + channel: le_u32(b, 8), + ty: le_u32(b, 12), + } + } + + fn to_bytes(self) -> [u8; QEH_SIZE] { + let mut b = [0; QEH_SIZE]; + b[0..4].copy_from_slice(&{ self.magic }.to_le_bytes()); + b[4..8].copy_from_slice(&{ self.size }.to_le_bytes()); + b[8..12].copy_from_slice(&{ self.channel }.to_le_bytes()); + b[12..16].copy_from_slice(&{ self.ty }.to_le_bytes()); + b + } +} + #[repr(C, packed)] #[derive(Clone, Copy, Default)] struct EPICHeader { @@ -597,11 +621,10 @@ impl AFKEndpoint { mem_sync(); let base = buf_offset + block_size * 3; let mut msg_buf = KVec::new(); - const QEH_SIZE: usize = mem::size_of::(); while wptr as usize != rptr { let mut qeh_bytes = [0; QEH_SIZE]; self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; - let mut qeh = unsafe { &*(qeh_bytes.as_ptr() as *const QEHeader) }; + let mut qeh = QEHeader::from_bytes(&qeh_bytes); if qeh.magic != QE_MAGIC1 && qeh.magic != QE_MAGIC2 { let magic = qeh.magic; dev_err!( @@ -616,7 +639,7 @@ impl AFKEndpoint { if qeh.size as usize > room { rptr = 0; self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; - qeh = unsafe { &*(qeh_bytes.as_ptr() as *const QEHeader) }; + qeh = QEHeader::from_bytes(&qeh_bytes); if qeh.magic != QE_MAGIC1 && qeh.magic != QE_MAGIC2 { let magic = qeh.magic; @@ -639,7 +662,7 @@ impl AFKEndpoint { self.drop_message(&client.dev, fmt!("short message ({} bytes)", msg_buf.len())) } Some((header, msg)) => { - if let Err(e) = self.handle_ipc(client, qeh, &header, msg) { + if let Err(e) = self.handle_ipc(client, &qeh, &header, msg) { let channel = qeh.channel; self.drop_message( &client.dev, @@ -785,7 +808,6 @@ impl AFKEndpoint { mem_sync(); let rptr = self.iomem_read32(buf_offset + block_size)? as usize; let mut wptr = self.iomem_read32(buf_offset + block_size * 2)? as usize; - const QEH_SIZE: usize = mem::size_of::(); if wptr < rptr && wptr + QEH_SIZE >= rptr { dev_err!(client.dev, "Tx buffer full at endpoint {}", self.index); return Err(EIO); @@ -797,16 +819,11 @@ impl AFKEndpoint { channel, ty, }; - let qeh_bytes = unsafe { - slice::from_raw_parts( - &qeh as *const QEHeader as *const u8, - mem::size_of::(), - ) - }; - self.memcpy_to_iomem(base + wptr, qeh_bytes)?; + let qeh_bytes = qeh.to_bytes(); + self.memcpy_to_iomem(base + wptr, &qeh_bytes)?; if payload_len > buf_size.checked_sub(wptr + QEH_SIZE).ok_or(EIO)? { wptr = 0; - self.memcpy_to_iomem(base + wptr, qeh_bytes)?; + self.memcpy_to_iomem(base + wptr, &qeh_bytes)?; } self.memcpy_to_iomem(base + wptr + QEH_SIZE, header)?; self.memcpy_to_iomem(base + wptr + QEH_SIZE + header.len(), data)?; @@ -1231,11 +1248,11 @@ impl WorkItem for AopServiceRegisterWork { .and_then(|x| x.get_child_by_name(this.name)); let info = bindings::platform_device_info { parent: this.data.dev.as_raw(), - name: this.name.as_ptr() as *const _, + name: this.name.as_char_ptr(), id: bindings::PLATFORM_DEVID_AUTO, res: ptr::null_mut(), num_res: 0, - data: &this.service as *const EPICService as *const _, + data: ptr::from_ref(&this.service).cast(), size_data: mem::size_of::(), dma_mask: 0, fwnode: fwnode @@ -1479,7 +1496,7 @@ impl AopData { if *k != key { continue; } - arg_bytes[idx..idx + size].copy_from_slice(&(*v as u64).to_le_bytes()[..size]); + arg_bytes[idx..idx + size].copy_from_slice(&v.to_le_bytes()[..size]); break; } idx += size; @@ -2296,6 +2313,7 @@ impl platform::Driver for AopDriver { ); return Err(ENODEV); } + // SAFETY: nothing of this device has DMA in flight before probe. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::new::<42>())? }; let aop_req = pdev.io_request_by_index(0).ok_or(EINVAL)?; let aop_mmio = KBox::pin_init(aop_req.iomap_sized::(), GFP_KERNEL)?; @@ -2357,6 +2375,8 @@ impl Drop for AopDriver { } } +// SAFETY: the wrapped `Arc` is `Send + Sync` (the trait requires +// both), so the handle may move to the thread that unbinds the device. unsafe impl Send for AopDriver {} module_platform_driver! { From 7a6e327251cc51d0a483483f0b81dda99a811791 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 06:59:14 +1100 Subject: [PATCH 1179/1579] soc: apple: aop: Validate the receive ring before taking entries recv_rb() takes the read and write pointers of a receive ring from the shared buffer, which the firmware can write, and checks neither against the ring: - The read pointer is used for a header copy before anything establishes that it lies in the ring; only the check against the whole allocation stops the copy, so it can read from the other ring or the ring header blocks. - The write pointer is only the loop terminator, re-read after every entry. A value the read pointer cannot reach, at or past the end of the ring or off the block grid, keeps the loop dispatching entries forever and holds the RTKit receive worker. - When an entry does not fit before the end of the ring, its header is read again from the top, and only the magic of that second header is checked. Its size sizes the message buffer and the copy, so a bad value asks for an allocation of up to 4 GiB or copies past the ring. - An entry that ends where it started, or a wrapped entry that ends on the read pointer it came from, leaves the read pointer where it was, and the same entry is dispatched again. Move the ring arithmetic into an aop_ring module of pure functions and check, before the first entry and after every re-read of the write pointer, that both pointers are block boundaries inside the ring with room for an entry header. An entry, wrapped or not, must fit in the ring and must move the read pointer. Refuse the ring with EIO otherwise, as for a bad magic, and log the pointers or the header that failed. Also refuse a ring whose size is not a whole number of blocks, which the entry alignment relies on. Bound one pass to the number of blocks in the ring, the most entries it can hold, so that a producer that keeps moving the write pointer cannot hold the worker either; entries beyond that are taken on the doorbell that announces them. The checks have a KUnit suite, apple_aop_ring. Fixes: 5ba018e083e7 ("soc: apple: aop: Bounds-check ring accesses and access pointers volatile") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 77 +++++++----- drivers/soc/apple/aop_ring.rs | 221 ++++++++++++++++++++++++++++++++++ 2 files changed, 270 insertions(+), 28 deletions(-) create mode 100644 drivers/soc/apple/aop_ring.rs diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index f01c510dba88b..d0cfcb84bb1f2 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -184,6 +184,8 @@ fn align_up(v: usize, a: usize) -> usize { (v + a - 1) & !(a - 1) } +mod aop_ring; + /// Index into the AFK endpoint tables for RTKit endpoint `ep`, if it is one /// of the endpoints this driver drives. fn afk_endpoint_index(ep: u8) -> Option { @@ -617,43 +619,62 @@ impl AFKEndpoint { } }; let mut rptr = self.iomem_read32(buf_offset + block_size)? as usize; - let mut wptr = self.iomem_read32(buf_offset + block_size * 2)?; + let mut wptr = self.iomem_read32(buf_offset + block_size * 2)? as usize; mem_sync(); let base = buf_offset + block_size * 3; let mut msg_buf = KVec::new(); - while wptr as usize != rptr { - let mut qeh_bytes = [0; QEH_SIZE]; - self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; - let mut qeh = QEHeader::from_bytes(&qeh_bytes); - if qeh.magic != QE_MAGIC1 && qeh.magic != QE_MAGIC2 { - let magic = qeh.magic; + let ring = match aop_ring::Ring::new(buf_size, block_size) { + Ok(ring) => ring, + Err(e) => { + dev_err!( + client.dev, + "Invalid ring on ep {}: size {:#x} block {:#x}", + self.index, + buf_size, + block_size + ); + return Err(e); + } + }; + // Entries the producer adds while this pass runs beyond one ring's + // worth are left for the doorbell that announces them. + let mut budget = ring.capacity(); + loop { + if ring.check_ptr(rptr).is_err() || ring.check_ptr(wptr).is_err() { dev_err!( client.dev, - "Invalid magic on ep {}, got {:x}", + "Ring pointers out of range on ep {}: rptr {:#x} wptr {:#x}", self.index, - magic + rptr, + wptr ); return Err(EIO); } - let room = buf_size.checked_sub(rptr + QEH_SIZE).ok_or(EIO)?; - if qeh.size as usize > room { - rptr = 0; - self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; - qeh = QEHeader::from_bytes(&qeh_bytes); - - if qeh.magic != QE_MAGIC1 && qeh.magic != QE_MAGIC2 { - let magic = qeh.magic; - dev_err!( - client.dev, - "Invalid magic on ep {}, got {:x}", - self.index, - magic - ); - return Err(EIO); - } + if wptr == rptr || budget == 0 { + break; } + budget -= 1; + let mut last = QEHeader::default(); + let read = ring.read_entry(rptr, |ptr| { + let mut qeh_bytes = [0; QEH_SIZE]; + self.memcpy_from_iomem(base + ptr, &mut qeh_bytes)?; + last = QEHeader::from_bytes(&qeh_bytes); + Ok(last) + }); + let Ok((qeh, entry, next)) = read else { + let (magic, size) = (last.magic, last.size); + dev_err!( + client.dev, + "Invalid entry on ep {} at {:#x}: magic {:x} size {:#x}", + self.index, + rptr, + magic, + size + ); + return Err(EIO); + }; msg_buf.resize(qeh.size as usize, 0, GFP_KERNEL)?; - self.memcpy_from_iomem(base + rptr + QEH_SIZE, &mut msg_buf)?; + self.memcpy_from_iomem(base + entry + QEH_SIZE, &mut msg_buf)?; // A message the endpoint cannot handle is skipped. The ring // position is still good, and not advancing past the entry would // read it again on every doorbell and stall the endpoint for good. @@ -678,10 +699,10 @@ impl AFKEndpoint { } } } - rptr = align_up(rptr + QEH_SIZE + qeh.size as usize, block_size) % buf_size; + rptr = next; mem_sync(); self.iomem_write32(buf_offset + block_size, rptr as u32)?; - wptr = self.iomem_read32(buf_offset + block_size * 2)?; + wptr = self.iomem_read32(buf_offset + block_size * 2)? as usize; mem_sync(); } Ok(()) diff --git a/drivers/soc/apple/aop_ring.rs b/drivers/soc/apple/aop_ring.rs new file mode 100644 index 0000000000000..d854cc221091b --- /dev/null +++ b/drivers/soc/apple/aop_ring.rs @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounds for the entries of an AFK ring. +//! +//! The ring pointers and the entry headers live in memory the firmware can +//! write. These checks keep every entry inside its ring, independent of the +//! shared allocation that also holds the other ring and the header blocks. + +use super::{QEHeader, QEH_SIZE, QE_MAGIC1, QE_MAGIC2}; +use kernel::prelude::*; + +/// A ring of `size` bytes of entries made of `block`-byte blocks. Entries +/// start on a block boundary. +pub(super) struct Ring { + size: usize, + block: usize, +} + +impl Ring { + pub(super) fn new(size: usize, block: usize) -> Result { + if size < QEH_SIZE || !block.is_power_of_two() || size % block != 0 { + return Err(EIO); + } + Ok(Self { size, block }) + } + + /// The most entries the ring can hold, one per block. One receive pass + /// takes at most this many, so that a producer that keeps moving the + /// write pointer cannot hold the receive worker. + pub(super) fn capacity(&self) -> usize { + self.size / self.block + } + + /// Checks a read or write pointer: it must be a block boundary inside + /// the ring with room for an entry header. + pub(super) fn check_ptr(&self, ptr: usize) -> Result { + if ptr >= self.size || ptr % self.block != 0 || self.size - ptr < QEH_SIZE { + return Err(EIO); + } + Ok(()) + } + + /// Returns the pointer after an entry of `len` bytes at `ptr`, if the + /// entry fits in the ring. + fn end(&self, ptr: usize, len: usize) -> Result { + self.check_ptr(ptr)?; + let end = ptr + .checked_add(QEH_SIZE) + .and_then(|v| v.checked_add(len)) + .ok_or(EIO)?; + let aligned = end.checked_add(self.block - 1).ok_or(EIO)? & !(self.block - 1); + if aligned > self.size { + return Err(EIO); + } + Ok(aligned % self.size) + } + + /// Finds the entry at read pointer `ptr`, reading entry headers with + /// `read`. An entry that does not fit before the end of the ring starts + /// again at the top, where its header is read a second time. Returns the + /// header, where the entry is and the read pointer after it. + pub(super) fn read_entry( + &self, + mut ptr: usize, + mut read: impl FnMut(usize) -> Result, + ) -> Result<(QEHeader, usize, usize)> { + self.check_ptr(ptr)?; + let previous = ptr; + let mut header = read(ptr)?; + Self::check_magic(&header)?; + if header.size as usize > self.size - ptr - QEH_SIZE { + ptr = 0; + header = read(ptr)?; + Self::check_magic(&header)?; + } + // The header at the top is firmware-written too: it must fit in the + // ring before it sizes an allocation or a copy. + let next = self.end(ptr, header.size as usize)?; + // An entry that leaves the read pointer where it was would be taken + // again and again. + if next == ptr || next == previous { + return Err(EIO); + } + Ok((header, ptr, next)) + } + + fn check_magic(header: &QEHeader) -> Result { + if header.magic != QE_MAGIC1 && header.magic != QE_MAGIC2 { + return Err(EIO); + } + Ok(()) + } +} + +#[kunit_tests(apple_aop_ring)] +mod tests { + use super::*; + + const BUF: usize = 0x1000; + const BLOCK: usize = 0x40; + + fn header(size: u32) -> QEHeader { + QEHeader { + magic: QE_MAGIC1, + size, + ..QEHeader::default() + } + } + + #[test] + fn invalid_layout_and_alignment_overflow_are_rejected() -> Result { + assert!(Ring::new(0, 64).is_err()); + assert!(Ring::new(256, 0).is_err()); + assert!(Ring::new(256, 3).is_err()); + assert!(Ring::new(255, 64).is_err()); + // no room for an entry header + assert!(Ring::new(QEH_SIZE - 1, 1).is_err()); + let ring = Ring::new(usize::MAX - 63, 64)?; + assert!(ring.end(0, usize::MAX - QEH_SIZE).is_err()); + Ok(()) + } + + #[test] + fn pointers_inside_the_ring_are_accepted() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + for ptr in [0, 0x40, 0xfc0] { + assert!(ring.check_ptr(ptr).is_ok()); + } + Ok(()) + } + + #[test] + fn pointers_at_or_past_the_end_are_refused() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + for ptr in [BUF, u32::MAX as usize, 0x10_0000, usize::MAX] { + assert!(ring.check_ptr(ptr).is_err()); + } + Ok(()) + } + + #[test] + fn misaligned_pointers_are_refused() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + for ptr in [0x10, 0x44, 0x7f, 0xbf] { + assert!(ring.check_ptr(ptr).is_err()); + } + Ok(()) + } + + #[test] + fn invalid_pointers_are_rejected_before_reading() -> Result { + let ring = Ring::new(256, 64)?; + for ptr in [256, 320, usize::MAX, 1, 255] { + let mut reads = 0; + assert!(ring + .read_entry(ptr, |_| { + reads += 1; + Ok(header(0)) + }) + .is_err()); + assert_eq!(reads, 0); + } + Ok(()) + } + + #[test] + fn oversized_wrapped_header_is_rejected() -> Result { + let ring = Ring::new(256, 64)?; + let mut reads = 0; + let result = ring.read_entry(192, |ptr| { + reads += 1; + Ok(header(if ptr == 0 { 256 } else { 64 })) + }); + assert!(result.is_err()); + assert_eq!(reads, 2); + assert!(ring + .read_entry(192, |ptr| Ok(header(if ptr == 0 { u32::MAX } else { 64 }))) + .is_err()); + Ok(()) + } + + #[test] + fn wrapped_entry_stays_in_ring() -> Result { + let ring = Ring::new(256, 64)?; + let (_, ptr, next) = + ring.read_entry(192, |ptr| Ok(header(if ptr == 0 { 48 } else { 64 })))?; + assert_eq!((ptr, next), (0, 64)); + assert_eq!(ring.read_entry(192, |_| Ok(header(48)))?.2, 0); + Ok(()) + } + + #[test] + fn receive_entry_must_advance_the_read_pointer() -> Result { + let ring = Ring::new(256, 64)?; + assert!(ring.read_entry(0, |_| Ok(header(240))).is_err()); + assert!(ring + .read_entry(192, |ptr| Ok(header(if ptr == 0 { 176 } else { 64 }))) + .is_err()); + Ok(()) + } + + #[test] + fn one_pass_takes_at_most_a_ring_of_entries() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + assert_eq!(ring.capacity(), 64); + // A producer that moves the write pointer one block ahead of every + // entry taken never lets the pointers meet; the pass still ends. The + // test itself gives up after two rings' worth. + let (mut rptr, mut wptr, mut taken) = (0, BLOCK, 0); + let mut budget = ring.capacity(); + while wptr != rptr && budget != 0 && taken < 2 * BUF / BLOCK { + budget -= 1; + let (_, _, next) = ring.read_entry(rptr, |_| Ok(header(16)))?; + taken += 1; + rptr = next; + wptr = (wptr + BLOCK) % BUF; + } + assert_eq!(taken, 64); + Ok(()) + } +} From 01c8fe5e98f4ab8805957ca3206d6d46a65b62e7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 06:59:56 +1100 Subject: [PATCH 1180/1579] soc: apple: aop: Do not overrun unread entries in the transmit ring write_entry() refuses an entry only when the write pointer sits less than one header below the read pointer. An entry that does not fit before the end of the ring is moved to the top without looking at the read pointer, and one placed below the read pointer is not checked against it either. Either way the host overwrites entries the firmware has not read yet, and an entry that ends exactly on the read pointer makes a full ring read as empty. The pointers read back from the shared buffer are not validated, and an entry larger than the ring is written past it. Work out where the entry goes, and whether it fits, from both pointers: at the write pointer if it fits before the end of the ring, at the top of the ring otherwise, and in either case only if it ends before the read pointer. Validate the pointers first, as the receive path does. A ring without room fails the call with EIO and logs "Tx buffer full" as before. Order the entry before the write pointer that publishes it. Add KUnit cases for the placement to apple_aop_ring. Fixes: 5ba018e083e7 ("soc: apple: aop: Bounds-check ring accesses and access pointers volatile") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 40 ++++++++------ drivers/soc/apple/aop_ring.rs | 98 +++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 17 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index d0cfcb84bb1f2..910c84e8934ef 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -180,10 +180,6 @@ const ALS_CALIBRATION_LEN: usize = 80; const ALS_CALIBRATION_OPERATION: u64 = 0x0746_b8d6_6515_2e31; const ALS_CALIBRATION_BODY_LEN: u64 = 56; -fn align_up(v: usize, a: usize) -> usize { - (v + a - 1) & !(a - 1) -} - mod aop_ring; /// Index into the AFK endpoint tables for RTKit endpoint `ep`, if it is one @@ -828,29 +824,39 @@ impl AFKEndpoint { let base = buf_offset + block_size * 3; mem_sync(); let rptr = self.iomem_read32(buf_offset + block_size)? as usize; - let mut wptr = self.iomem_read32(buf_offset + block_size * 2)? as usize; - if wptr < rptr && wptr + QEH_SIZE >= rptr { - dev_err!(client.dev, "Tx buffer full at endpoint {}", self.index); + let wptr = self.iomem_read32(buf_offset + block_size * 2)? as usize; + let ring = aop_ring::Ring::new(buf_size, block_size)?; + if ring.check_ptr(rptr).is_err() || ring.check_ptr(wptr).is_err() { + dev_err!( + client.dev, + "Tx ring pointers out of range at endpoint {}: rptr {:#x} wptr {:#x}", + self.index, + rptr, + wptr + ); return Err(EIO); } - let payload_len = header.len() + data.len(); + let payload_len = header.len().checked_add(data.len()).ok_or(EIO)?; + let Ok((entry, next)) = ring.write_entry(rptr, wptr, payload_len) else { + dev_err!(client.dev, "Tx buffer full at endpoint {}", self.index); + return Err(EIO); + }; let qeh = QEHeader { magic: QE_MAGIC1, - size: payload_len as u32, + size: u32::try_from(payload_len).map_err(|_| EIO)?, channel, ty, }; let qeh_bytes = qeh.to_bytes(); - self.memcpy_to_iomem(base + wptr, &qeh_bytes)?; - if payload_len > buf_size.checked_sub(wptr + QEH_SIZE).ok_or(EIO)? { - wptr = 0; + if entry != wptr { self.memcpy_to_iomem(base + wptr, &qeh_bytes)?; } - self.memcpy_to_iomem(base + wptr + QEH_SIZE, header)?; - self.memcpy_to_iomem(base + wptr + QEH_SIZE + header.len(), data)?; - wptr = align_up(wptr + QEH_SIZE + payload_len, block_size) % buf_size; - self.iomem_write32(buf_offset + block_size * 2, wptr as u32)?; - Ok(wptr as u64 | (AFK_OPC_SEND << 48)) + self.memcpy_to_iomem(base + entry, &qeh_bytes)?; + self.memcpy_to_iomem(base + entry + QEH_SIZE, header)?; + self.memcpy_to_iomem(base + entry + QEH_SIZE + header.len(), data)?; + mem_sync(); + self.iomem_write32(buf_offset + block_size * 2, next as u32)?; + Ok(next as u64 | (AFK_OPC_SEND << 48)) } fn epic_notify( &mut self, diff --git a/drivers/soc/apple/aop_ring.rs b/drivers/soc/apple/aop_ring.rs index d854cc221091b..ccbb661331e2a 100644 --- a/drivers/soc/apple/aop_ring.rs +++ b/drivers/soc/apple/aop_ring.rs @@ -90,6 +90,43 @@ impl Ring { } Ok(()) } + + /// Places an entry of `len` bytes after its header, given the read and + /// write pointers. It goes at `wptr`, or at the top of the ring when it + /// does not fit before the end (its header at `wptr` then tells the + /// consumer to wrap). It never runs into the unread entries from `rptr` + /// on, and never ends on `rptr`, where a full ring would read as empty. + /// Returns where the entry goes and the write pointer after it. + pub(super) fn write_entry( + &self, + rptr: usize, + wptr: usize, + len: usize, + ) -> Result<(usize, usize)> { + self.check_ptr(rptr)?; + self.check_ptr(wptr)?; + // Reject an entry the whole ring cannot take before even writing + // the wrap header. + let head_next = self.end(0, len)?; + let ptr = if len > self.size - wptr - QEH_SIZE { + 0 + } else { + wptr + }; + let next = if ptr == 0 { + head_next + } else { + self.end(ptr, len)? + }; + if head_next == 0 + || next == rptr + || (wptr < rptr && (ptr == 0 || next == 0 || next > rptr)) + || (ptr < wptr && (rptr == 0 || next > rptr)) + { + return Err(ENOSPC); + } + Ok((ptr, next)) + } } #[kunit_tests(apple_aop_ring)] @@ -159,6 +196,8 @@ mod tests { }) .is_err()); assert_eq!(reads, 0); + assert!(ring.write_entry(ptr, 0, 1).is_err()); + assert!(ring.write_entry(0, ptr, 1).is_err()); } Ok(()) } @@ -218,4 +257,63 @@ mod tests { assert_eq!(taken, 64); Ok(()) } + + #[test] + fn transmit_wrap_checks_whole_entry_and_free_space() -> Result { + let ring = Ring::new(256, 64)?; + assert!(ring.write_entry(128, 192, 256).is_err()); + assert!(ring.write_entry(128, 192, usize::MAX).is_err()); + assert_eq!(ring.write_entry(192, 192, 64)?, (0, 128)); + Ok(()) + } + + #[test] + fn transmit_cannot_catch_read_pointer() -> Result { + let ring = Ring::new(256, 64)?; + assert!(ring.write_entry(64, 192, 64).is_err()); + assert!(ring.write_entry(128, 64, 64).is_err()); + assert!(ring.write_entry(0, 192, 48).is_err()); + assert!(ring.write_entry(192, 64, 176).is_err()); + assert!(ring.write_entry(192, 192, 240).is_err()); + assert_eq!(ring.write_entry(192, 64, 48)?, (64, 128)); + Ok(()) + } + + #[test] + fn an_entry_that_fits_goes_at_the_write_pointer() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + assert_eq!(ring.write_entry(0, 0, 0x100 - QEH_SIZE)?.0, 0); + assert_eq!(ring.write_entry(0x200, 0x100, 0x80 - QEH_SIZE)?.0, 0x100); + assert_eq!(ring.write_entry(0x40, 0xf00, 0x100 - QEH_SIZE)?, (0xf00, 0)); + Ok(()) + } + + #[test] + fn a_wrapping_entry_goes_to_the_top_when_there_is_room() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + assert_eq!( + ring.write_entry(0x200, 0xfc0, 0x100 - QEH_SIZE)?, + (0, 0x100) + ); + Ok(()) + } + + #[test] + fn a_wrapping_entry_never_overruns_unread_entries() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + // the unread entries start at 0x80; the wrapped entry would cover them + assert!(ring.write_entry(0x80, 0xfc0, 0x100 - QEH_SIZE).is_err()); + assert!(ring.write_entry(0, 0xfc0, 0x100 - QEH_SIZE).is_err()); + Ok(()) + } + + #[test] + fn an_entry_never_reaches_the_read_pointer() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + // ending on rptr would make a full ring read as empty + assert!(ring.write_entry(0x200, 0x100, 0x100 - QEH_SIZE).is_err()); + assert!(ring.write_entry(0x200, 0x100, 0x110 - QEH_SIZE).is_err()); + assert!(ring.write_entry(0, 0xf00, 0x100 - QEH_SIZE).is_err()); + Ok(()) + } } From 2d5c7b6bc705f0ba8eef8f3d1ea23a53d4248eba Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:46:24 +1100 Subject: [PATCH 1181/1579] ASoC: apple: t8140-aop-audio: Refuse card names that do not fit The card's id, short and long names are built from the apple,chassis-name and apple,machine-kind properties and copied into ALSA's fixed-size fields with a slice copy. A source longer than its field (the id has room for 15 characters, so any chassis name over seven characters) indexes out of range, which is a Rust panic and an oops in probe. Make copy_str() return EINVAL when the string does not fit and fail the probe with an error message instead. The names on the J700 fit, so nothing changes there. Add a KUnit suite for the copy: a name that fits is copied with its NUL, and one longer than the field is refused and leaves the field untouched. Fixes: eeb112648f2a ("ASoC: apple: Add T8140 AOP audio driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 58 +++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index b63c6c8baa084..c2c80da11539b 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -1550,8 +1550,14 @@ const PCM_PERIOD_BYTES_MAX: usize = 0x10000; /// 16384 frames (341 ms), while still allowing the largest PipeWire quantum. const SPKR_BUFFER_MAX: usize = 128 * 1024; -fn copy_str(target: &mut [u8], source: &[u8]) { - target[..source.len()].copy_from_slice(source) +/// Copies the NUL-terminated `source` into the fixed-size C string field +/// `target`; EINVAL if it does not fit. +fn copy_str(target: &mut [u8], source: &[u8]) -> Result { + target + .get_mut(..source.len()) + .ok_or(EINVAL)? + .copy_from_slice(source); + Ok(()) } // ---- ASoC component: DPCM front-end PCMs on the ADMAC channels, back-end @@ -2604,16 +2610,19 @@ impl SndSocT8140AopDriver { let id_str = CString::try_from_fmt(fmt!("Apple{}AOP", *chassis))?; let shortname = CString::try_from_fmt(fmt!("{} {} AOP", *machine_kind, *chassis))?; let longname = CString::try_from_fmt(fmt!("{} {} AOP Audio", *machine_kind, *chassis))?; - // SAFETY: `card` was created by `snd_card_new` above; the fixed-size name fields are filled - // with NUL-terminated strings that fit. - unsafe { - copy_str( - &mut (*this.card).driver, - c"t8140-aop-audio".to_bytes_with_nul(), + // SAFETY: `card` was created by `snd_card_new` above and is not + // registered yet, so nothing else accesses it. + let card = unsafe { &mut *this.card }; + let named = copy_str(&mut card.driver, c"t8140-aop-audio".to_bytes_with_nul()) + .and_then(|()| copy_str(&mut card.id, id_str.to_bytes_with_nul())) + .and_then(|()| copy_str(&mut card.shortname, shortname.to_bytes_with_nul())) + .and_then(|()| copy_str(&mut card.longname, longname.to_bytes_with_nul())); + if let Err(e) = named { + dev_err!( + data.dev, + "apple,chassis-name or apple,machine-kind too long for the card names\n" ); - copy_str(&mut (*this.card).id, id_str.to_bytes_with_nul()); - copy_str(&mut (*this.card).shortname, shortname.to_bytes_with_nul()); - copy_str(&mut (*this.card).longname, longname.to_bytes_with_nul()); + return Err(e); } this.new_pcm(0, c"lpai", c"Low-Power Microphone", &Self::LPAI_OPS)?; @@ -2664,14 +2673,15 @@ impl SndSocT8140AopDriver { return Err(Error::from_errno(ret)); } // SAFETY: `pcm` was created by `snd_pcm_new` above; the ops are static and the private - // data is an `Arc` released by `pcm_free_private`; `name` fits the fixed-size field. + // data is an `Arc` released by `pcm_free_private`. unsafe { bindings::snd_pcm_set_ops(pcm, bindings::SNDRV_PCM_STREAM_CAPTURE as i32, ops); (*pcm).private_data = self.data.clone().into_foreign().cast(); (*pcm).private_free = Some(pcm_free_private); (*pcm).info_flags = 0; - copy_str(&mut (*pcm).name, name.to_bytes_with_nul()); } + // SAFETY: as above. + copy_str(unsafe { &mut (*pcm).name }, name.to_bytes_with_nul())?; if device == 0 { // The low-power microphone's buffer is filled by the driver from // the source ring; the array's is set up on its DMA device at @@ -2960,3 +2970,25 @@ mod tests { assert_eq!(f32_bits_of_q31(i32::MIN), 0xbf80_0000); // -1.0 } } + +/// The card and PCM names are copied into ALSA's fixed-size fields from +/// device-tree strings. +#[kunit_tests(snd_soc_t8140_aop_audio_names)] +mod name_tests { + use super::*; + + #[test] + fn a_name_that_fits_is_copied_with_its_nul() { + let mut field = [0xffu8; 16]; + assert!(copy_str(&mut field, c"AppleJ700AOP".to_bytes_with_nul()).is_ok()); + assert_eq!(&field[..13], b"AppleJ700AOP\0"); + } + + #[test] + fn a_name_longer_than_its_field_is_refused() { + let mut field = [0u8; 16]; + let long = c"AppleJ700LongerChassisAOP".to_bytes_with_nul(); + assert!(copy_str(&mut field, long).is_err()); + assert_eq!(field, [0u8; 16]); + } +} From e65283c426783b12298f72896ef6f0dc33c4222c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:46:25 +1100 Subject: [PATCH 1182/1579] ASoC: apple: t8140-aop-audio: Let power request sequences wrap The power request sequence numbers are taken with fetch_add(1) + 1. The atomic add wraps, but the "+ 1" on its result does not: with CONFIG_RUST_OVERFLOW_CHECKS=y the request after u32::MAX of them panics instead of sending sequence 0. Use wrapping_add() so the counter wraps like the firmware's u32 field. This takes billions of requests, so it is not a practical problem, but the panic serves no purpose. Fixes: eeb112648f2a ("ASoC: apple: Add T8140 AOP audio driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index c2c80da11539b..1fe19b7d38b22 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -664,7 +664,7 @@ impl SndSocT8140AopData { } fn set_power(&self, state: u32) -> Result<()> { - let sequence = self.sequence.fetch_add(1, Relaxed) + 1; + let sequence = self.sequence.fetch_add(1, Relaxed).wrapping_add(1); let req = PowerRequest::new(0, sequence, state, 1)?; let ret = self.epic_wrapped_call(&AudioSetDeviceProp::new( AUDIO_DEV_LPAI, @@ -732,7 +732,7 @@ impl SndSocT8140AopData { /// Property 202 with the service's sequence and a 24 MHz timestamp; /// `pw0 ` is confirmed through property 200 like the native host does. fn service_set_power(&self, svc: &Service, state: u32) -> Result<()> { - let sequence = svc.sequence.fetch_add(1, Relaxed) + 1; + let sequence = svc.sequence.fetch_add(1, Relaxed).wrapping_add(1); let req = PowerRequest::new(svc.dev_id, sequence, state, 0)?; let ret = self.epic_wrapped_call(&AudioSetDeviceProp::new( svc.dev_id, From d90daf04de8431f70eed8b590a532f38a6249095 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:16:02 +1100 Subject: [PATCH 1183/1579] media: apple: isp: Do not trust the batch count when a batch fails isp_submit_buffers() builds the buffer batch in the command area, which lies in the IPC surface the firmware reads and writes. When the firmware does not acknowledge the batch, the error path walks the batch again to put its buffers back on the pending list, and takes the number of entries from the batch header in that shared memory. Firmware that wrote to the area in the meantime, or that misbehaves, can make the driver walk far past the 40 entries the area holds, reading beyond the surface. Count the entries in a local variable while building the batch, write the count to the header once, and use the local count in the error path. The batch the firmware sees is unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 4cc79ee8a2307..3f4d63e2516b0 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -153,7 +153,7 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) struct isp_channel *chan = isp->chan_bh; struct isp_message *req = &chan->req; struct isp_buffer *buf, *tmp; - unsigned int num_rendered = 0; + unsigned int num_rendered = 0, num_buffers = 0; unsigned long flags; size_t offset; int err; @@ -180,7 +180,7 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) bufd->iovas[0] = meta->iova; bufd->flags[0] = 0x40000000; bufd++; - bl->num_buffers++; + num_buffers++; meta->submitted = true; } @@ -199,7 +199,7 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) bufd->iovas[0] = meta->iova; bufd->flags[0] = 0x40000000; bufd++; - bl->num_buffers++; + num_buffers++; meta->submitted = true; } @@ -222,7 +222,7 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) buf->surfs[0].iova, buf->surfs[0].iova + buf->surfs[0].size); */ bufd++; - bl->num_buffers++; + num_buffers++; num_rendered++; /* @@ -236,6 +236,12 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) send: spin_unlock_irqrestore(&isp->buf_lock, flags); + /* + * The firmware can write to the command area, so the error path below + * goes by the count kept here rather than the one in the batch. + */ + bl->num_buffers = num_buffers; + req->arg0 = isp->cmd_iova; req->arg1 = max_t(u64, ISP_IPC_BUFEXC_STAT_SIZE, ((uintptr_t)bufd - (uintptr_t)bl)); @@ -255,7 +261,7 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) spin_lock_irqsave(&isp->buf_lock, flags); bufd = &bl->buffers[0]; - for (int i = 0; i < bl->num_buffers; i++, bufd++) { + for (unsigned int i = 0; i < num_buffers; i++, bufd++) { list_for_each_entry_safe_reverse( buf, tmp, &isp->bufs_submitted, link) { if (bufd->iovas[0] == buf->surfs[0].iova) { From d9aface7ea8533965e33476631c8dbc44dfeb439 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:26:25 +1100 Subject: [PATCH 1184/1579] HID: dockchannel: Build the protocol KUnit test only with the driver The protocol test lives in drivers/hid/dockchannel-hid/, which the HID Makefile enters only for CONFIG_HID_DOCKCHANNEL, but the test depends only on APPLE_DOCKCHANNEL. With the transport driver disabled the test option can be set, or turned on by KUNIT_ALL_TESTS, and the suite is silently not built: a KUnit run reports zero tests for it. Make the test depend on HID_DOCKCHANNEL. Fixes: 1f5c943c4327 ("HID: dockchannel: Support MTP power method 2") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig index a19b543c066cb..5f1da27d445a8 100644 --- a/drivers/hid/dockchannel-hid/Kconfig +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -16,7 +16,7 @@ config HID_DOCKCHANNEL config HID_DOCKCHANNEL_KUNIT_TEST tristate "KUnit tests for the DockChannel HID protocol helpers" if !KUNIT_ALL_TESTS - depends on KUNIT && APPLE_DOCKCHANNEL + depends on KUNIT && HID_DOCKCHANNEL default KUNIT_ALL_TESTS help Build the KUnit tests for the DockChannel HID protocol helpers, From a27007e6ace3815d11c340c25a3ee90cfb2981b0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:08 +1100 Subject: [PATCH 1185/1579] HID: dockchannel: Log routine interface set-up at debug level Every boot logs a run of info lines while the MTP coprocessor announces its interfaces: the announcement and product name of each interface, and the start, firmware upload, reset and GPIO request of the multi-touch interface. They describe the normal sequence and say nothing actionable, so log them at debug level. Failures keep their warnings and errors. The one line per interface that reports it ready stays at info level: it is the only kernel-log evidence that the multi-touch firmware came up, and boot checks rely on it. A DT that sets apple,firmware-managed-reset on an interface that cannot use it made the driver ignore that interface without saying why; report it. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 6a5e412887082..dd1af665ebc26 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -245,8 +245,11 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) firmware_managed_reset = of_property_read_bool(of_node, "apple,firmware-managed-reset"); if (firmware_managed_reset && - (strcmp(name, "multi-touch") || power_method != DCHID_POWER_METHOD_2)) + (strcmp(name, "multi-touch") || power_method != DCHID_POWER_METHOD_2)) { + dev_err(dchid->dev, "Firmware-managed reset is not supported for %s\n", + name); goto err_put_node; + } if (power_method != DCHID_POWER_METHOD_1 && power_method != DCHID_POWER_METHOD_2) { dev_err(dchid->dev, "Unsupported power method %u for %s\n", @@ -548,8 +551,8 @@ static int dchid_request_gpio(struct dchid_iface *iface) if (iface->gpio) return 0; - dev_info(iface->dchid->dev, "Requesting GPIO %s#%d: %s\n", - iface->name, iface->gpio_id, iface->gpio_name); + dev_dbg(iface->dchid->dev, "Requesting GPIO %s#%d: %s\n", + iface->name, iface->gpio_id, iface->gpio_name); /* A power method 2 interface holds its reset line from probe on. */ if (iface->power_method == DCHID_POWER_METHOD_2 && @@ -584,7 +587,7 @@ static int dchid_start_interface(struct dchid_iface *iface) return -EINPROGRESS; } - dev_info(iface->dchid->dev, "Starting interface %s\n", iface->name); + dev_dbg(iface->dchid->dev, "Starting interface %s\n", iface->name); iface->starting = true; @@ -604,7 +607,7 @@ static int dchid_start_interface(struct dchid_iface *iface) if (fw && size) { /* Send firmware to the device */ - dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); + dev_dbg(iface->dchid->dev, "Sending firmware for %s\n", iface->name); ret = dchid_send_firmware(iface, fw, size); if (ret < 0) { dev_err(iface->dchid->dev, "Failed to send %s firmware\n", iface->name); @@ -612,7 +615,7 @@ static int dchid_start_interface(struct dchid_iface *iface) } /* After loading firmware, multi-touch needs a reset */ - dev_info(iface->dchid->dev, "Resetting %s\n", iface->name); + dev_dbg(iface->dchid->dev, "Resetting %s\n", iface->name); if (iface->power_method == DCHID_POWER_METHOD_2) { ret = dchid_pm2_set_power(iface, DCHID_POWER_STATE_OFF); if (ret < 0) @@ -749,7 +752,7 @@ static void dchid_create_interface_work(struct work_struct *ws) return; } - dev_info(dchid->dev, "New interface %s\n", iface->name); + dev_dbg(dchid->dev, "New interface %s\n", iface->name); /* Start the interface. This is not the entire init process, as firmware is loaded later on device open. */ ret = dchid_enable_interface(iface); @@ -951,8 +954,8 @@ static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t dev_warn(dchid->dev, "Unterminated product name for %s\n", iface->name); } else { - dev_info(dchid->dev, "Product name for %s: %s\n", - iface->name, product); + dev_dbg(dchid->dev, "Product name for %s: %s\n", + iface->name, product); } break; } @@ -1011,7 +1014,7 @@ static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t goto err; } - dev_info(dchid->dev, "GPIO command: %s#%d: %d\n", iface->name, cmd->gpio, cmd->cmd); + dev_dbg(dchid->dev, "GPIO command: %s#%d: %d\n", iface->name, cmd->gpio, cmd->cmd); switch (cmd->cmd) { case 3: @@ -1393,7 +1396,7 @@ static int dockchannel_hid_probe(struct platform_device *pdev) return -EIO; } - dev_info(dchid->dev, "Initialized, awaiting packets\n"); + dev_dbg(dchid->dev, "Initialized, awaiting packets\n"); platform_set_drvdata(pdev, dchid); dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); From d6094473be4ceae69e8385ffb03891a22c60c3e4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:09 +1100 Subject: [PATCH 1186/1579] soc: apple: rtkit-helper: Terminate log messages Six messages lack a trailing newline, and the RTKit initialisation error misspells "initialize". No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit-helper.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index 3dd941241893b..5f03bed772f26 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -51,7 +51,7 @@ static int apple_rtkit_helper_shmem_setup(void *cookie, struct apple_rtkit_shmem if (!helper->sram) { dev_err(helper->dev, - "RTKit buffer request with no SRAM region: %pR", &res); + "RTKit buffer request with no SRAM region: %pR\n", &res); return -EFAULT; } @@ -59,7 +59,7 @@ static int apple_rtkit_helper_shmem_setup(void *cookie, struct apple_rtkit_shmem if (res.end < res.start || !resource_contains(helper->sram, &res)) { dev_err(helper->dev, - "RTKit buffer request outside SRAM region: %pR", &res); + "RTKit buffer request outside SRAM region: %pR\n", &res); return -EFAULT; } @@ -139,7 +139,7 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) helper->sram_base = devm_ioremap_resource(dev, helper->sram); if (IS_ERR(helper->sram_base)) return dev_err_probe(dev, PTR_ERR(helper->sram_base), - "Failed to map SRAM region"); + "Failed to map SRAM region\n"); } if (apple_rtkit_helper_live(helper)) { @@ -147,7 +147,7 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) &apple_rtkit_helper_ops); if (IS_ERR(helper->rtk)) return dev_err_probe(dev, PTR_ERR(helper->rtk), - "Failed to adopt RTKit"); + "Failed to adopt RTKit\n"); ret = devm_add_action_or_reset(dev, apple_rtkit_helper_release, helper->rtk); @@ -163,7 +163,7 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) devm_apple_rtkit_init(dev, helper, NULL, 0, &apple_rtkit_helper_ops); if (IS_ERR(helper->rtk)) return dev_err_probe(dev, PTR_ERR(helper->rtk), - "Failed to intialize RTKit"); + "Failed to initialize RTKit\n"); writel_relaxed(APPLE_ASC_CPU_CONTROL_RUN, helper->asc_base + APPLE_ASC_CPU_CONTROL); @@ -174,7 +174,7 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) else ret = apple_rtkit_wake(helper->rtk); if (ret != 0) - return dev_err_probe(dev, ret, "Failed to wake up coprocessor"); + return dev_err_probe(dev, ret, "Failed to wake up coprocessor\n"); return 0; } From 9779b2457e8f13b0f257964570c9883f079e65f6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:44 +1100 Subject: [PATCH 1187/1579] pmdomain: apple: pmc: keep the DCS limit state when the rail enable fails apple_pmc_dcs_set_cur_state() records the new cooling state before it rewrites the DCS rail enable word, and keeps it when the write does not stick. Two things then go wrong: - Lifting a limit: the rail still honours only the AP agent, but the AP vote has already been put back to its boot value, the polling work is cancelled and dcs_state reads 0. The PMP and GPU votes are ignored with nothing left voting on their behalf, and since a request for state 0 now matches dcs_state, no later call retries the write. - Setting a limit: dcs_state reports a limit the PMC does not apply. Put the limit change into a helper and, when the enable write fails, return to the previous state with the same helper, so the rail, the AP vote, the polling work and dcs_state agree again and the next request retries. On success the sequence of writes is unchanged. The failure needs a rail enable word that does not take the write, which memory standing in for the registers cannot produce, so there is no KUnit case for it; the existing apple-pmc suite covers the successful limit cycle. Fixes: cf8edfc522ce ("pmdomain: apple: pmc: limit the DCS rail as a cooling device") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmc.c | 50 +++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index d1d37a9917ff4..c50e5fe2a92ec 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -386,36 +386,56 @@ static int apple_pmc_dcs_get_cur_state(struct thermal_cooling_device *cdev, return 0; } +/* + * Caller holds pmc->lock. Vote the capped demand before the other agents + * stop being honoured, and drop that vote only once they are honoured + * again, so that the rail never dips below either. + */ +static int apple_pmc_dcs_limit(struct apple_pmc *pmc, unsigned long state) +{ + unsigned int rail = pmc->hw->dcs_rail; + int ret; + + WRITE_ONCE(pmc->dcs_state, state); + if (state) { + apple_pmc_dcs_vote(pmc); + return apple_pmc_enable_write(pmc, rail); + } + + ret = apple_pmc_enable_write(pmc, rail); + if (!ret) + apple_pmc_dcs_vote(pmc); + + return ret; +} + static int apple_pmc_dcs_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state) { struct apple_pmc *pmc = cdev->devdata; - unsigned int rail = pmc->hw->dcs_rail; - unsigned long flags; + unsigned long flags, old; + bool limited; int ret = 0; if (state > pmc->hw->dcs_max) return -EINVAL; raw_spin_lock_irqsave(&pmc->lock, flags); - if (state != pmc->dcs_state) { + old = pmc->dcs_state; + if (state != old) { + ret = apple_pmc_dcs_limit(pmc, state); /* - * Vote the capped demand before the other agents stop being - * honoured, and drop that vote only once they are honoured - * again, so that the rail never dips below either. + * The rail enable did not change, so neither did the limit: + * go back to the old state, which the next call then retries + * to leave. */ - WRITE_ONCE(pmc->dcs_state, state); - if (state) { - apple_pmc_dcs_vote(pmc); - ret = apple_pmc_enable_write(pmc, rail); - } else { - ret = apple_pmc_enable_write(pmc, rail); - apple_pmc_dcs_vote(pmc); - } + if (ret) + apple_pmc_dcs_limit(pmc, old); } + limited = pmc->dcs_state; raw_spin_unlock_irqrestore(&pmc->lock, flags); - if (state) + if (limited) queue_delayed_work(system_freezable_power_efficient_wq, &pmc->dcs_work, msecs_to_jiffies(APPLE_PMC_DCS_POLL_MS)); From fdd138bbc7228b8b1a232268e2a02cc126b24d95 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:47 +1100 Subject: [PATCH 1188/1579] hwmon: macsmc: move the module help text back to its option The KUnit option was inserted between the help paragraphs of SENSORS_MACSMC_HWMON, so the sentence naming the macsmc-hwmon module now ends the help of SENSORS_MACSMC_HWMON_KUNIT_TEST, a bool. Move it back. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hwmon/Kconfig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index 59e379f867d71..d00243bbf544e 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -1174,6 +1174,9 @@ config SENSORS_MACSMC_HWMON on Apple Silicon devices. Say Y here if you have an Apple Silicon device. + This driver can also be built as a module. If so, the module will + be called macsmc-hwmon. + config SENSORS_MACSMC_HWMON_KUNIT_TEST bool "KUnit tests for the Apple SMC hwmon driver" if !KUNIT_ALL_TESTS depends on SENSORS_MACSMC_HWMON && KUNIT=y @@ -1184,9 +1187,6 @@ config SENSORS_MACSMC_HWMON_KUNIT_TEST that cannot be set up must keep its channel; the tests check that. No SMC is needed. Only useful for kernel developers. - This driver can also be built as a module. If so, the module will - be called macsmc-hwmon. - config SENSORS_MAX1111 tristate "Maxim MAX1111 Serial 8-bit ADC chip and compatibles" depends on SPI_MASTER From 9523159a72886c5169e7bfd566de9c0aa545ba85 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:49 +1100 Subject: [PATCH 1189/1579] soc: apple: clpc: drop the syscon select and the GPU from Kconfig The driver no longer reads the GPU power state through the PMGR syscon, so it does not need MFD_SYSCON, and the help text should not promise a GPU input. No functional change for configurations that enable it: the PMGR power-state driver selects MFD_SYSCON itself. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 639435f966c3f..4c3086213a47d 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -169,12 +169,11 @@ config APPLE_CLPC tristate "Apple host performance controller floors" depends on ARCH_APPLE || COMPILE_TEST depends on APPLE_PMC && CPU_FREQ - select MFD_SYSCON default ARCH_APPLE help Raises the DRAM controller and SoC fabric performance floors of the - host's PMC voter agent while CPU clusters, and optionally the GPU, - are busy, and releases them at idle. + host's PMC voter agent while CPU clusters are busy, and releases + them at idle. Say 'y' or 'm' here if you have an Apple SoC with a PMC. From 13857139ed4643ef532281808bc8035ad64c9a10 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:51 +1100 Subject: [PATCH 1190/1579] soc: apple: mcc: print the boot DMA device before dropping its node apple_mcc_wait_boot_dma() drops the reference the phandle iterator holds on the node and then prints the node with %pOF. Print it first. The node is part of the base device tree, so this is a correctness cleanup rather than a reachable use after free. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mcc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c index 7b8c2e2220c8b..352ef030270a7 100644 --- a/drivers/soc/apple/mcc.c +++ b/drivers/soc/apple/mcc.c @@ -48,9 +48,10 @@ static int apple_mcc_wait_boot_dma(struct device *dev) if (pdev) put_device(&pdev->dev); if (!bound) { + ret = dev_err_probe(dev, -EPROBE_DEFER, "waiting for %pOF to bind\n", + it.node); of_node_put(it.node); - return dev_err_probe(dev, -EPROBE_DEFER, "waiting for %pOF to bind\n", - it.node); + return ret; } } From 0b65493ccd0f4a779e592258032cb40ce03d1a1c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:53 +1100 Subject: [PATCH 1191/1579] pmdomain: apple: pmgr-pwrstate: tidy the rate-limited error messages Align the continuation lines of the messages that became rate-limited, and make the reset/disable timeout report the value it polled rather than the register value read before the write, which is what "now" claims to show. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmgr-pwrstate.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 053c087c37184..d0b75e0b4b671 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -68,7 +68,7 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a /* Resets are synchronous, and only work if the device is powered and clocked. */ if (reg & APPLE_PMGR_RESET && pstate != APPLE_PMGR_PS_ACTIVE) dev_err_ratelimited(ps->dev, "PS %s: powering off with RESET active\n", - genpd->name); + genpd->name); if (pstate != APPLE_PMGR_PS_ACTIVE && (ps->force_disable || ps->force_reset)) { u32 reg_pre = reg & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | @@ -88,8 +88,9 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a APPLE_PMGR_PS_SET_TIMEOUT); if (ret < 0) - dev_err_ratelimited(ps->dev, "PS %s: Failed to set reset/disable bits (now: 0x%x)\n", - genpd->name, reg); + dev_err_ratelimited(ps->dev, + "PS %s: Failed to set reset/disable bits (now: 0x%x)\n", + genpd->name, cur); } /* Do not replay the hardware-owned BUSY bit in this whole-word write. */ @@ -119,8 +120,9 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a } if (ret < 0) - dev_err_ratelimited(ps->dev, "PS %s: Failed to reach power state 0x%x (requested 0x%x, now: 0x%x)\n", - genpd->name, pstate, reg, cur); + dev_err_ratelimited(ps->dev, + "PS %s: Failed to reach power state 0x%x (requested 0x%x, now: 0x%x)\n", + genpd->name, pstate, reg, cur); if (auto_enable) { /* Not all devices implement this; this is a no-op where not implemented. */ @@ -222,7 +224,8 @@ static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned if (ps->genpd.status == GENPD_STATE_OFF) dev_err_ratelimited(ps->dev, - "PS 0x%x: asserting RESET while powered down\n", ps->offset); + "PS 0x%x: asserting RESET while powered down\n", + ps->offset); dev_dbg(ps->dev, "PS 0x%x: assert reset\n", ps->offset); /* Quiesce device before asserting reset */ @@ -252,7 +255,8 @@ static int apple_pmgr_reset_deassert(struct reset_controller_dev *rcdev, unsigne if (ps->genpd.status == GENPD_STATE_OFF) dev_err_ratelimited(ps->dev, - "PS 0x%x: RESET was deasserted while powered down\n", ps->offset); + "PS 0x%x: RESET was deasserted while powered down\n", + ps->offset); spin_unlock_irqrestore(&ps->genpd.slock, flags); From 712530b9753fd8692b31382b3e9c6aaaa720ffe0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:54 +1100 Subject: [PATCH 1192/1579] gpio: macsmc: tidy the J700 pcIO helpers Align the parameter list and the condition of macsmc_gpio_is_j700_pcio() with the opening parenthesis, and build the pcIO level bit without OR-ing in a logical negation, which sparse reports as dubious. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpio/gpio-macsmc.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpio/gpio-macsmc.c b/drivers/gpio/gpio-macsmc.c index 471bce2e54b4f..424f19b1c4a5d 100644 --- a/drivers/gpio/gpio-macsmc.c +++ b/drivers/gpio/gpio-macsmc.c @@ -99,11 +99,11 @@ struct macsmc_gpio { }; static bool macsmc_gpio_is_j700_pcio(struct macsmc_gpio *smcgp, - unsigned int offset) + unsigned int offset) { return smcgp->j700_pcio && - (offset == MACSMC_J700_AFE_RESET_GPIO || - offset == MACSMC_J700_RADIO_REG_ON_GPIO); + (offset == MACSMC_J700_AFE_RESET_GPIO || + offset == MACSMC_J700_RADIO_REG_ON_GPIO); } static int macsmc_gpio_nr(smc_key key) @@ -236,7 +236,7 @@ static int macsmc_gpio_set(struct gpio_chip *gc, unsigned int offset, int value) lo = MACSMC_J700_AFE_RESET_LO; hi = MACSMC_J700_AFE_RESET_HI; } - payload = ((u64)hi << 32) | lo | !!value; + payload = ((u64)hi << 32) | lo | (value ? 1 : 0); ret = apple_smc_write_u64(smcgp->smc, SMC_KEY(pcIO), payload); if (ret < 0) { From 1c797d3eb5cb79f7e80696b4d7c93800172771c0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:47:55 +1100 Subject: [PATCH 1193/1579] cpufreq: apple-soc: say why a cluster with verified transitions fails init When the OPP levels are outside the SoC P-state range, the DVFS command register stays busy, or the inherited P-state has no OPP, ->init() returns an error without a message, and the cpufreq core only reports the failure at debug level. The cluster is then left without cpufreq and nothing in the log says why. Report each case once. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 057954affc4fb..24a4e4105dbc0 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -845,6 +845,8 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) dev_pm_opp_put(opp); if (freq_table[i].driver_data < info->min_pstate || freq_table[i].driver_data > info->max_pstate) { + dev_err(cpu_dev, "OPP level %u is not a valid P-state\n", + freq_table[i].driver_data); ret = -EINVAL; goto out_free_cpufreq_table; } @@ -858,13 +860,16 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) ret = readq_poll_timeout_atomic(reg_base + APPLE_DVFS_CMD, cmd, !(cmd & APPLE_DVFS_CMD_BUSY), 2, apple_soc_cpufreq_timeout(info)); - if (ret) + if (ret) { + dev_err(cpu_dev, "DVFS busy at init, command=%#llx\n", cmd); goto out_free_cpufreq_table; + } state = FIELD_GET(APPLE_DVFS_CMD_PS1, cmd); for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) if (freq_table[i].driver_data == state) break; if (freq_table[i].frequency == CPUFREQ_TABLE_END) { + dev_err(cpu_dev, "current P-state %u has no OPP\n", state); ret = -ERANGE; goto out_free_cpufreq_table; } From df5213734e31c372fc36a2375539e4d69b5f815a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:30:07 +1100 Subject: [PATCH 1194/1579] soc: apple: rtkit: Only parse an initial crash log in inherited sessions The first message on the crashlog endpoint is normally the buffer request, and type 1 is used both for that request and for a crash report. Since commit c07d044a0311 ("soc: apple: rtkit: recognise an initial crash log in adopted sessions") the handler reads the header of a freshly set up buffer and reports a crash when it finds a valid crash log there. That check is only needed for a session inherited from the bootloader, which has already negotiated its buffer and can only be reporting a crash. For a session that went through the handshake it is wrong: a co-processor that places its own buffers (DCP, the T8140 GPU) hands over memory that is not cleared by Linux. If that memory still holds the log of an earlier crash, for example after apple_rtkit_reinit() recovered from one, the buffer request is reported as a new crash, the instance is marked crashed and the client starts recovery again. Return after setting up the buffer unless the session is inherited, as the handler did before the inherited case was added. That commit relied on a fresh session's buffer being zeroed, which holds for Linux's own coherent allocations but not for firmware-placed ones. The inherited-session handling it added is kept unchanged. Fixes: c07d044a0311 ("soc: apple: rtkit: recognise an initial crash log in adopted sessions") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 62e7a9add49ff..cc9671c2534c1 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -422,6 +422,14 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) if (apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, APPLE_RTKIT_EP_CRASHLOG, msg)) goto unmapped; + /* + * In a session that went through the handshake, the first + * message on this endpoint is the buffer request. Firmware may + * place that buffer in memory which still holds the log of an + * earlier crash, so only look for one in an inherited session. + */ + if (!rtk->crashlog_inherited) + return; if (rtk->crashlog_buffer.size < sizeof(header)) goto unmapped; From a516c3f7c3f85aeb05efe5e3aff5b7890c7d022c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:36:03 +1100 Subject: [PATCH 1195/1579] iommu: apple-dart: Invalidate every DART even if a TLB sync step fails apple_dart_domain_flush_tlb_range() publishes the host page table into locked roots and then invalidates the TLB of each DART of the domain. If publishing fails for a locked stream, it skips the invalidation of that DART, and if powering a DART or publishing fails it returns before looking at the remaining DARTs of the domain. This function also backs iotlb_sync(). After an unmap the DMA layer frees the pages as soon as it returns, so a skipped invalidation leaves translations to freed memory in the TLB, and the device can then read or write pages that have been reused. Publishing fails, for example, when a firmware-owned root slot changed under a locked stream, which is independent of the unmap that has to be flushed. Always invalidate a DART that could be powered, whatever the outcome of publishing, go on with the remaining DARTs, and return the first error. The map path still sees the error and fails the mapping. Leaving firmware-owned root slots out of range invalidates is deliberate, see commit dd5ac6fdffb0 ("iommu: apple-dart: leave live firmware translations out of invalidates"), and still holds: a failed publication does not change which slots apple_dart_fw_invalidate() skips, it only no longer suppresses the flush of the others. Neither the skip nor the early return was explained when it was added. Add a KUnit case with a two-DART domain whose locked stream fails to publish, and check that both DARTs receive a TLB command. Fixes: 79379798f12f ("iommu/apple-dart: Support T8140 DARTs and firmware-owned stream roots") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 71 +++++++++++++++++++++++++++++++++ drivers/iommu/apple-dart.c | 42 ++++++++++--------- 2 files changed, 95 insertions(+), 18 deletions(-) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index c953f1da07bb3..f7afabf12e3ad 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -167,6 +167,76 @@ static void apple_dart_test_firmware_roots(struct kunit *test) KUNIT_EXPECT_EQ(test, live[1], 0x4001ULL); } +static struct apple_dart *apple_dart_test_t8110(struct kunit *test, const char *name) +{ + struct apple_dart *dart; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->dev = kunit_device_register(test, name); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); + dart->regs = (__force void __iomem *)kunit_kzalloc(test, SZ_16K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart->regs); + dart->hw = &apple_dart_hw_t8110; + dart->version = 0x0202; + dart->num_streams = 1; + spin_lock_init(&dart->lock); + pm_runtime_set_active(dart->dev); + pm_runtime_enable(dart->dev); + return dart; +} + +static void apple_dart_test_unmap_flush_after_publish_failure(struct kunit *test) +{ + struct iommu_iotlb_gather gather = { .start = 0, .end = SZ_16K - 1 }; + struct apple_dart_stream_map stream = {}; + struct apple_dart_domain *domain; + struct apple_dart *locked, *other; + struct io_pgtable *iop; + u64 live[] = { 0x1001, 0, 0, 0 }; + u64 owned[ARRAY_SIZE(live)] = {}; + u64 ours[] = { 0x3001, 0, 0, 0 }; + u32 sentinel = 0x40; + + /* A locked stream whose root slot firmware holds: publishing fails. */ + locked = apple_dart_test_t8110(test, "apple-dart-flush-locked"); + locked->locked = true; + locked->pgsize = sizeof(live); + locked->locked_ttbr[0][0] = live; + locked->locked_owned[0][0] = owned; + other = apple_dart_test_t8110(test, "apple-dart-flush-other"); + other->pgsize = SZ_16K; + + iop = kunit_kzalloc(test, sizeof(*iop), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, iop); + iop->cfg.apple_dart_cfg.n_ttbrs = 1; + iop->cfg.apple_dart_cfg.ttbr[0] = ours; + domain = kunit_kzalloc(test, sizeof(*domain), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, domain); + domain->pgtbl_ops = &iop->ops; + domain->stream_maps[0].dart = locked; + atomic_long_set(&domain->stream_maps[0].sidmap[0], 1); + domain->stream_maps[1].dart = other; + atomic_long_set(&domain->stream_maps[1].sidmap[0], 1); + + writel(sentinel, locked->regs + DART_T8110_TLB_CMD); + writel(sentinel, other->regs + DART_T8110_TLB_CMD); + stream.dart = locked; + __set_bit(0, stream.sidmap); + KUNIT_ASSERT_EQ(test, apple_dart_hw_sync_locked(&iop->cfg, &stream, true), + -EBUSY); + + /* The pages are freed after this: both DARTs must drop them. */ + apple_dart_iotlb_sync(&domain->domain, &gather); + KUNIT_EXPECT_NE(test, readl(locked->regs + DART_T8110_TLB_CMD), sentinel); + KUNIT_EXPECT_NE(test, readl(other->regs + DART_T8110_TLB_CMD), sentinel); + KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); + KUNIT_EXPECT_EQ(test, atomic_read(&locked->dev->power.usage_count), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&other->dev->power.usage_count), 0); + pm_runtime_disable(locked->dev); + pm_runtime_disable(other->dev); +} + static void apple_dart_test_gated_commands(struct kunit *test) { struct apple_dart_stream_map stream = {}; @@ -706,6 +776,7 @@ static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_locked_handoff), KUNIT_CASE(apple_dart_test_sync_unknown_roots), KUNIT_CASE(apple_dart_test_firmware_roots), + KUNIT_CASE(apple_dart_test_unmap_flush_after_publish_failure), KUNIT_CASE(apple_dart_test_gated_commands), KUNIT_CASE(apple_dart_test_resume_commands_failure), KUNIT_CASE(apple_dart_test_tunnel_restore), diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 97f73ed930258..1545ca41dd307 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -1276,8 +1276,15 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, struct apple_dart_atomic_stream_map *map; struct apple_dart_stream_map stream; struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; - int i, j, ret; + const u32 op = DART_T8110_TLB_CMD_OP_FLUSH_SID; + int i, j, ret, err = 0; + u64 start, end; + /* + * After an unmap, the caller frees the pages once this returns. Every + * DART that can be reached must therefore be invalidated, even if + * publishing the page table or powering another DART failed. + */ for_each_stream_map(i, domain, map) { stream.dart = map->dart; for (j = 0; j < BITS_TO_LONGS(stream.dart->num_streams); j++) @@ -1288,31 +1295,30 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, if (ret < 0) { dev_err_ratelimited(stream.dart->dev, "failed to power DART for TLB synchronization: %d\n", ret); - return ret; + err = err ?: ret; + continue; } ret = stream.dart->locked ? apple_dart_hw_sync_locked(cfg, &stream, true) : 0; - if (ret) + if (ret) { dev_err_ratelimited(stream.dart->dev, "failed to publish host page-table entries: %d\n", ret); - else { - /* The range registers take the DVA the device issues. */ - if (range && stream.dart->fw_mirror && stream.dart->version >= 0x0202) - ret = apple_dart_fw_invalidate(&stream, first + domain->dma_offset, - last + domain->dma_offset); - else if (range && stream.dart->locked && stream.dart->version >= 0x0202) - ret = apple_dart_t8110_hw_tlb_command_range(&stream, - DART_T8110_TLB_CMD_OP_FLUSH_SID, true, - first + domain->dma_offset, - last + domain->dma_offset); - else - ret = stream.dart->hw->invalidate_tlb(&stream); + err = err ?: ret; } + /* The range registers take the DVA the device issues. */ + start = first + domain->dma_offset; + end = last + domain->dma_offset; + if (range && stream.dart->fw_mirror && stream.dart->version >= 0x0202) + ret = apple_dart_fw_invalidate(&stream, start, end); + else if (range && stream.dart->locked && stream.dart->version >= 0x0202) + ret = apple_dart_t8110_hw_tlb_command_range(&stream, op, true, + start, end); + else + ret = stream.dart->hw->invalidate_tlb(&stream); pm_runtime_put(stream.dart->dev); - if (ret) - return ret; + err = err ?: ret; } - return 0; + return err; } static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) From ed0925cdf091083ba1efee97bbb288e9c331f812 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:33:08 +1100 Subject: [PATCH 1196/1579] spi: apple-qspi: Restore the controller configuration after system sleep The controller is reset and configured once, in probe: interrupt enables, FIFO thresholds and the fixed timing registers written by apple_qspi_init_config(). The driver has no PM callbacks, so when the QSPI power domain is gated during system sleep these registers come back with their reset values and are never written again. Only the clock divider and the mode are programmed for each operation. The first flash access after resume, typically an NVRAM read or write, then runs with an unconfigured controller. If it fails to complete it costs a full transfer timeout before the timeout path resets the block. Add system sleep callbacks that stop the controller across suspend and run the same reset and configuration as probe on resume. Fixes: 9057400a21c4 ("spi: apple: add T8140 QSPIMC controller") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index e0d8653e51984..2142b14e167bf 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -593,6 +594,35 @@ static int apple_qspi_probe(struct platform_device *pdev) return 0; } +static int apple_qspi_suspend(struct device *dev) +{ + struct spi_controller *ctlr = dev_get_drvdata(dev); + + return spi_controller_suspend(ctlr); +} + +/* + * The controller loses its configuration when its power domain is gated + * during system sleep. Restore it before the first operation after resume. + */ +static int apple_qspi_resume(struct device *dev) +{ + struct spi_controller *ctlr = dev_get_drvdata(dev); + struct apple_qspi *qspi = spi_controller_get_devdata(ctlr); + int ret; + + ret = apple_qspi_hw_reset(qspi); + if (ret) { + dev_err(dev, "controller reset failed on resume: %d\n", ret); + return ret; + } + + return spi_controller_resume(ctlr); +} + +static DEFINE_SIMPLE_DEV_PM_OPS(apple_qspi_pm_ops, apple_qspi_suspend, + apple_qspi_resume); + static const struct of_device_id apple_qspi_of_match[] = { { .compatible = "apple,t8140-qspi" }, {} @@ -604,6 +634,7 @@ static struct platform_driver apple_qspi_driver = { .driver = { .name = "apple-qspi", .of_match_table = apple_qspi_of_match, + .pm = pm_sleep_ptr(&apple_qspi_pm_ops), }, }; module_platform_driver(apple_qspi_driver); From ff3ebdbad58c1c50dadbf9cecd7f67aada516dc1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:34:16 +1100 Subject: [PATCH 1197/1579] nvme-apple: Reword two post-M4 handoff comments Describe what the inherited-buffer check and the restart of a stopped post-M4 coprocessor rely on, without the references to particular boot stages and their test history. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 12d88a8d75672..aabaa652505dd 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -410,9 +410,8 @@ static bool apple_nvme_in_inherited_rtkit_range(struct apple_nvme *anv, /* * A range property or a partial System RAM intersection does not prove - * ownership. Every page must be reserved and present in the linear map. - * Native m1n1 reserves the predecessor's inherited allocation prefix. - * RAM excluded by a resident EL2 stage needs a separate cache contract. + * ownership. Every page must be reserved and present in the linear map, + * as the bootloader leaves the buffers it hands over. */ if (region_intersects(bfr->iova, bfr->size, IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE) != REGION_INTERSECTS) @@ -1640,8 +1639,8 @@ static void apple_nvme_reset_work(struct work_struct *work) * Post-M4 ANS cannot be reset through PMGR: the reset * assert raises an SError. A previous boot stage that * stopped the coprocessor cleanly leaves its firmware - * resident, so mirror the qualified U-Boot handoff: - * set RUN and run a fresh RTKit boot handshake. + * resident, so set RUN and run a fresh RTKit boot + * handshake. */ dev_dbg(anv->dev, "post-M4 firmware handoff: restarting stopped ANS without reset\n"); From 0afefd9910288c1783a2756c4240dfc80ee8470f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:03:04 +1100 Subject: [PATCH 1198/1579] iommu: apple-dart: Return IOMEM_ERR_PTR() from the tunneled MMIO helper apple_dart_platform_ioremap_np_resource() returns a void __iomem pointer but builds its error values with ERR_PTR(), which sparse reports as an address space mismatch. Use IOMEM_ERR_PTR(). No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 1545ca41dd307..6ee58ade4cd70 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -2476,20 +2476,20 @@ apple_dart_platform_ioremap_np_resource(struct platform_device *pdev, res = platform_get_resource(pdev, IORESOURCE_MEM, index); if (!res) - return ERR_PTR(-EINVAL); + return IOMEM_ERR_PTR(-EINVAL); if (!devm_request_mem_region(dev, res->start, resource_size(res), dev_name(dev))) - return ERR_PTR(-EBUSY); + return IOMEM_ERR_PTR(-EBUSY); regs = ioremap_np(res->start, resource_size(res)); if (!regs) - return ERR_PTR(-ENOMEM); + return IOMEM_ERR_PTR(-ENOMEM); ret = devm_add_action_or_reset(dev, apple_dart_iounmap_np, (__force void *)regs); if (ret) - return ERR_PTR(ret); + return IOMEM_ERR_PTR(ret); *res_out = res; return regs; From 7543b0f32df84cff9478d643ab5fb5366ae7f217 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 13:25:59 +1100 Subject: [PATCH 1199/1579] selftests: apple-dcp: Follow the retired-channel teardown and LUT log level The service-rebind group extracts afk_recv_handle_teardown(), which now asks afk_epic_channel_retired() whether a teardown names a channel that was retired before it logs. Extract that helper and the afk_service_retired() check it uses, so the group keeps compiling the production teardown path. The firmware-provider group compiles trampoline_provider_property_h17p(), which now reports the J700 power-LUT fallback with dev_info_once(). Give its host prelude a no-op dev_info_once() next to dev_warn_once(). No expectation changes; all 55 host tests pass. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/testing/selftests/drivers/gpu/apple-dcp/edid.py | 1 + tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py index e247ce3a5489e..fc86c7f8be7df 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -158,6 +158,7 @@ def function(file,signature): function('epic/dpavservep.c','static void dcpavserv_teardown('),function('epic/dpavservep.c','void dpavservep_detach('), function('afk.c','static const struct apple_epic_service_ops *\nafk_match_service('), function('afk.c','static struct apple_epic_service *afk_epic_find_service('), + function('afk.h','static inline bool afk_service_retired('),function('afk.c','static bool afk_epic_channel_retired('), function('afk.c','static void afk_discard_pending_open('),function('afk.c','static void afk_retry_service_init('),function('afk.c','static void afk_service_worker('),function('afk.c','static void afk_recv_handle_init('), function('epic/dpavservep.c','static int dcpdpserv_prepare('), function('epic/dpavservep.c','static void dcpdpserv_teardown('),function('afk.c','static void afk_recv_handle_teardown('), diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 7756046d7c07a..aa8e136e86aba 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -974,6 +974,7 @@ def firmware(root): #define trace_iomfb_callback(d, t, n) ((void)(d), (void)(t)) #define dev_err(...) ((void)0) #define dev_warn_once(...) ((void)0) +#define dev_info_once(...) ((void)0) struct device_node { bool internal, external, present; int length; }; struct device { struct device_node *of_node; }; struct apple_dcp { struct { int iomfb_method_profile; } hw; struct device *dev; bool crashed; }; From 3132675b256e07ad782280c42c853c63f6d18ef1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 13:28:49 +1100 Subject: [PATCH 1200/1579] selftests: apple-dcp: Run the queue recovery control through the RX drain afk_receive_message_worker() now hands RBEP_RECV to afk_recv_drain(), which takes at most one ring's worth of entries per pass through afk_ring_drain(). The actual-queue-quiesce control extracts the worker, so it no longer compiled. Extract afk_recv_entry(), afk_recv_drain() and the afk.h budget helpers alongside the worker, and give the fixture endpoint a receive ring geometry so the budget covers the four entries the control replays. The fixture's afk_init_rxtx() mock takes the ring as an opaque pointer now that the endpoint's rxbfr is a struct. The ordered-recovery, ordered-pending-closed and quiesce-pending cases keep their assertions; all 55 host tests pass. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/recovery.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py b/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py index 310af408b0f96..7d548c29a42ed 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py @@ -294,7 +294,7 @@ def run(name,text,cases): #define complete_all(p) complete(p) ''') queued=queued.replace('struct apple_dcp_afkep { u8 *command_tags;bool lock,stopping;', - 'struct apple_dcp_afkep { u8 *command_tags;int started,stopped,txbfr,rxbfr;void *debugfs_entry;bool lock,stopping;') + 'struct apple_dcp_afkep { u8 *command_tags;int started,stopped,txbfr;struct { u32 bufsz,stride,*rptr,*wptr;bool ready; } rxbfr;void *debugfs_entry;bool lock,stopping;') queued=queued.replace('bool external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;', 'bool retain_dma,external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;') queued=queued.replace('static void run_retries(void) { assert(!ep.lock);if(ep.service_work.pending) { ep.service_work.pending=false;afk_service_worker(&ep.service_work); } }', @@ -305,8 +305,9 @@ def run(name,text,cases): queued+=r''' struct afk_receive_message_work { struct apple_dcp_afkep *ep;u64 message;struct work_struct work; }; static void afk_getbuf(struct apple_dcp_afkep *e,u64 m) { (void)e;(void)m;assert(!"unexpected GETBUF"); } - static void afk_init_rxtx(struct apple_dcp_afkep *e,u64 m,int *r) { (void)e;(void)m;(void)r;assert(!"unexpected INIT"); } + static void afk_init_rxtx(struct apple_dcp_afkep *e,u64 m,void *r) { (void)e;(void)m;(void)r;assert(!"unexpected INIT"); } static unsigned int rx_step;static bool close_pending; + #define dev_warn_ratelimited(...) ((void)0) static bool afk_recv(struct apple_dcp_afkep *e) { assert(e==&ep);rx_running=true; if(rx_step==0) closed(13,0x6a01,0); @@ -318,11 +319,16 @@ def run(name,text,cases): } ''' af=(root/'drivers/gpu/drm/apple/afk.c').read_text() - for signature in ['static void afk_receive_message_worker(', 'int afk_receive_message(', 'void afk_quiesce(', 'static void afk_release_context(']: + ah=(root/'drivers/gpu/drm/apple/afk.h').read_text() + # RECV drains the ring through the production budget helpers. + for signature in ['static inline unsigned int afk_ring_drain_budget(', 'static inline unsigned int afk_ring_drain(']: + queued+=extract(ah,signature)+'\n' + for signature in ['static bool afk_recv_entry(', 'static void afk_recv_drain(', 'static void afk_receive_message_worker(', 'int afk_receive_message(', 'void afk_quiesce(', 'static void afk_release_context(']: queued+=extract(af,signature)+'\n' queued+=r''' int main(int argc,char **argv) { assert(argc==2);setup();reuse_dma=true;INIT_WORK(&ep.service_work,afk_service_worker); + ep.rxbfr.bufsz=0x1000;ep.rxbfr.stride=0x40; announce(13,name,native_class,13);struct apple_epic_service *old=d.dcpavserv.service;assert(old); assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack();run_retries(); unsigned int before=opens; From 2e36dd688f13ab22023fc9fcbf0ebf32dba61c8a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 13:33:57 +1100 Subject: [PATCH 1201/1579] usb: typec: tipd: cd321x: Don't race the recovery read in the IRQ test cd321x_test_recover() asserted that the recovery read was still pending before flushing it. The driver queues that read with no delay, so on an SMP test machine a worker can run it before the assertion, and the test fails although the driver behaved correctly. With four CPUs, cd321x_resume_acknowledges_events_test failed 1 time in 5. Check the read's effect instead. After the flush the port must accept updates again, which it only does once the recovery read has run. Removing the resume path's mod_delayed_work() still makes the test fail. Fixes: 7e97f2f5ee45 ("usb: typec: tipd: cd321x: Never leave interrupt events unacknowledged") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-irq-test.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/usb/typec/tipd/cd321x-irq-test.c b/drivers/usb/typec/tipd/cd321x-irq-test.c index 2d34af081f473..17ef778f96826 100644 --- a/drivers/usb/typec/tipd/cd321x-irq-test.c +++ b/drivers/usb/typec/tipd/cd321x-irq-test.c @@ -148,10 +148,13 @@ static irqreturn_t cd321x_test_interrupt(struct cd321x_test *t) return t->cd.tps.data->irq_handler(0, &t->cd.tps); } -/* The read of a snapshot that was scheduled to recover the events. */ +/* + * Run the snapshot read that was scheduled to recover the events. It is queued + * without delay, so on SMP a worker may already have run it: check its effect, + * not whether it is still pending. + */ static void cd321x_test_recover(struct kunit *test, struct cd321x_test *t) { - KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); flush_delayed_work(&t->cd.resume_work); scoped_guard(mutex, &t->cd.tps.lock) KUNIT_EXPECT_TRUE(test, cd321x_pm_can_update(&t->cd.pm)); From 2911b586c885463279d04d560137c147c05ca099 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 14:36:08 +1100 Subject: [PATCH 1202/1579] usb: xhci: plat: Hold a runtime PM reference in the resume-from-runtime tests xhci_plat_resume_from_runtime_ok_test resumes a mock device with no runtime PM reference held. The PM core then queues an idle check, which tries a runtime suspend. The mock has no suspend callback, so the attempt fails and the device returns to active, but for a moment its status is RPM_SUSPENDING. A test that reads pm_runtime_active() then sees false. Under lockdep with two CPUs this failed 1 run in 10. The real caller cannot see this: system suspend calls the suspend callbacks with the reference that device_prepare() takes. Hold one in the tests too, and drop it when they release the device. Fixes: 7dfc99ae6302 ("usb: xhci: plat: Fail system suspend when the controller does not resume") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/host/xhci-plat-test.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c index 49cbd15e4ed34..ca4814898714e 100644 --- a/drivers/usb/host/xhci-plat-test.c +++ b/drivers/usb/host/xhci-plat-test.c @@ -152,10 +152,24 @@ static struct device *xhci_plat_test_rpm_device(struct kunit *test, int rpm_ret) xhci_plat_test_rpm_ret = rpm_ret; dev->pm_domain = &xhci_plat_test_pm_domain; pm_runtime_set_suspended(dev); + /* + * System suspend runs the suspend callbacks with a runtime PM reference + * held (device_prepare()). Hold one here too, or the idle check queued + * after a resume starts a runtime suspend while the test looks at the + * state. + */ + pm_runtime_get_noresume(dev); pm_runtime_enable(dev); return dev; } +static void xhci_plat_test_rpm_release(struct device *dev) +{ + pm_runtime_disable(dev); + pm_runtime_put_noidle(dev); + dev->pm_domain = NULL; +} + /* A runtime resume that fails before system sleep fails the system suspend. */ static void xhci_plat_resume_from_runtime_error_test(struct kunit *test) { @@ -163,8 +177,7 @@ static void xhci_plat_resume_from_runtime_error_test(struct kunit *test) KUNIT_EXPECT_EQ(test, xhci_plat_resume_from_runtime(dev), -ETIMEDOUT); KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); - pm_runtime_disable(dev); - dev->pm_domain = NULL; + xhci_plat_test_rpm_release(dev); } /* A successful one, and an active controller, let it go ahead. */ @@ -175,8 +188,7 @@ static void xhci_plat_resume_from_runtime_ok_test(struct kunit *test) KUNIT_EXPECT_EQ(test, xhci_plat_resume_from_runtime(dev), 0); KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); KUNIT_EXPECT_EQ(test, xhci_plat_resume_from_runtime(dev), 0); - pm_runtime_disable(dev); - dev->pm_domain = NULL; + xhci_plat_test_rpm_release(dev); } #endif /* CONFIG_PM */ From befa72f4f47d58327130236376ceeab26b45ee5c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 14:46:43 +1100 Subject: [PATCH 1203/1579] selftests: apple-dcp: Follow the tipd lost-IRQ recovery helpers The host runner fails 5 of 55 cases since the tipd lost-IRQ recovery because the typec-probe, probe-mux-pm-dispatch, physical-disconnect, fast-replug-bounce and dropped-swap-teardown units no longer compile. They inline cd321x-pm.h, which now includes for the acknowledgment retry backoff, and the probe units do not extract the new cd321x_init_work() called by cd321x_register_port(). Inline the header through one shared helper that provides a host clamp() in place of minmax.h, and extract the actual cd321x_init_work() for both probe units. Check in typec-probe that a lost event reopens even a stale port with a full budget, that a failed acknowledgment keeps the budget with a backoff capped at CD321X_ACK_RETRY_MAX_MS, and that sleep and removal leave lost events to the resume read. The dispatch unit drops these helpers like the other ones it does not exercise. Retain the strict compiler and sanitizer settings and all 55 runner cases. The full buildbox run improves from 50 pass / 5 fail to 55 pass / 0 fail. No production driver changes are needed. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/common.py | 9 +++++++ .../drivers/gpu/apple-dcp/lifecycle.py | 5 ++-- .../selftests/drivers/gpu/apple-dcp/pm.py | 24 ++++++++++++++++--- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py index db10625064000..89c2602ddb271 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -25,6 +25,15 @@ def source_defines(source): definitions.append('\n'.join(definition)) return definitions +def tipd_pm_header(root): + """CD321x PM state machine for host units, with a host clamp() for minmax.h.""" + header = (root / 'drivers/usb/typec/tipd/cd321x-pm.h').read_text() + header = header.replace('#include ', '') + # Single line: probe-mux-pm-dispatch filters duplicate definitions per line. + return header.replace('#include ', + '#define clamp(val, lo, hi) ({ __auto_type _v = (val); ' + '__typeof__(_v) _lo = (lo), _hi = (hi); _v < _lo ? _lo : _v > _hi ? _hi : _v; })') + def run_unit(name, unit, output, cases=((),), includes=(), extra=()): work = output / name work.mkdir(parents=True, exist_ok=True) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 7756046d7c07a..d8cdfe59e53d9 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -3,7 +3,7 @@ """Actual disconnect authority, DWC3 teardown and firmware resource admission.""" from pathlib import Path import re -from common import block, atc_mux_without_pm, source_defines +from common import block, atc_mux_without_pm, source_defines, tipd_pm_header def disconnect(root): # Copyright 2026 Ryan Murray @@ -21,8 +21,7 @@ def function(file, name): pos += 1 return s[m.start():pos] - pm = (r / 'drivers/usb/typec/tipd/cd321x-pm.h').read_text() - pm = pm.replace('#include ', '') + pm = tipd_pm_header(r) core = 'drivers/usb/typec/tipd/core.c' dcp = 'drivers/gpu/drm/apple/dcp.c' funcs = '\n'.join(function(f, n) for f, n in ( diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 27b207fd7b8f8..ac1249af085ca 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -4,7 +4,7 @@ from pathlib import Path import os, re, subprocess import phy -from common import block +from common import block, tipd_pm_header def ownership(root, usb_unreachable=False): ns=phy.parking(root) @@ -373,7 +373,7 @@ def function(text, name): struct tps6598x { int lock; void *port; struct device *dev; }; struct notifier_block { int (*notifier_call)(struct notifier_block *,unsigned long,void *); }; ''' - pm = source('drivers/usb/typec/tipd/cd321x-pm.h').replace('#include ', '') + pm = tipd_pm_header(root) mocks = r''' struct cd321x { struct tps6598x tps; @@ -425,6 +425,8 @@ def function(text, name): core = source('drivers/usb/typec/tipd/core.c') switch_get = block(core, 'VISIBLE_IF_KUNIT struct typec_thunderbolt_switch *\ncd321x_tbt_switch_get(') production = switch_get.replace('VISIBLE_IF_KUNIT', 'static', 1) + '\n' + init_work = block(core, 'VISIBLE_IF_KUNIT void cd321x_init_work(') + production += init_work.replace('VISIBLE_IF_KUNIT', 'static', 1) + '\n' production += function(core, 'cd321x_register_port') + '\n' + function(core, 'cd321x_unregister_port') controls = r''' static void reset(struct cd321x *c) { @@ -453,6 +455,22 @@ def function(text, name): assert(!cd321x_pm_can_update(&budget)&&cd321x_pm_resume(&budget)); assert(cd321x_pm_link_event(&budget)); puts("PASS actual retry budget becomes stale without a fresh event"); + struct cd321x_pm_state lost={0}; + cd321x_pm_init(&lost);assert(!cd321x_pm_event_lost(&lost)&&cd321x_pm_ready(&lost)); + for(unsigned int i=0;i Date: Sat, 17 Oct 2026 14:55:54 +1100 Subject: [PATCH 1204/1579] soc: apple: aop: Accept a transmit entry at a write pointer of zero write_entry() takes an entry placed at 0 to be one that wrapped to the top of the ring. An entry that fits at the write pointer is placed at 0 as well when the write pointer itself is 0, which it is whenever the previous entry ended exactly at the end of the ring. Until the firmware has read that entry, the read pointer is above the write pointer, and the check for a wrapped entry below the read pointer refuses every entry although [0, rptr) is free. The call fails with "Tx buffer full" and EIO until the firmware catches up. Decide whether the entry wraps from its length and the write pointer, as the placement already does, and check that instead of the placement. Add KUnit cases for an entry at a write pointer of 0 with an unread entry above it. Fixes: 01c8fe5e98f4 ("soc: apple: aop: Do not overrun unread entries in the transmit ring") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop_ring.rs | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/drivers/soc/apple/aop_ring.rs b/drivers/soc/apple/aop_ring.rs index ccbb661331e2a..41608ba7699b5 100644 --- a/drivers/soc/apple/aop_ring.rs +++ b/drivers/soc/apple/aop_ring.rs @@ -108,20 +108,20 @@ impl Ring { // Reject an entry the whole ring cannot take before even writing // the wrap header. let head_next = self.end(0, len)?; - let ptr = if len > self.size - wptr - QEH_SIZE { - 0 - } else { - wptr - }; - let next = if ptr == 0 { + // The entry wraps when it does not fit before the end of the ring. + // It then goes at 0, but so does an entry that fits at a write + // pointer of 0: where it goes does not tell whether it wrapped. + let wrapped = len > self.size - wptr - QEH_SIZE; + let ptr = if wrapped { 0 } else { wptr }; + let next = if wrapped { head_next } else { self.end(ptr, len)? }; if head_next == 0 || next == rptr - || (wptr < rptr && (ptr == 0 || next == 0 || next > rptr)) - || (ptr < wptr && (rptr == 0 || next > rptr)) + || (wptr < rptr && (wrapped || next == 0 || next > rptr)) + || (wrapped && (rptr == 0 || next > rptr)) { return Err(ENOSPC); } @@ -307,6 +307,19 @@ mod tests { Ok(()) } + #[test] + fn an_entry_fits_at_a_write_pointer_of_zero() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + // The last entry ended at the end of the ring and is still unread, + // so the write pointer is 0 below the read pointer: [0, rptr) is free. + assert_eq!(ring.write_entry(0xf80, 0, 0x40 - QEH_SIZE)?, (0, 0x40)); + assert_eq!(ring.write_entry(0x80, 0, 0x40 - QEH_SIZE)?, (0, 0x40)); + // It still never reaches the read pointer. + assert!(ring.write_entry(0x40, 0, 0x40 - QEH_SIZE).is_err()); + assert!(ring.write_entry(0x80, 0, 0x80 - QEH_SIZE).is_err()); + Ok(()) + } + #[test] fn an_entry_never_reaches_the_read_pointer() -> Result { let ring = Ring::new(BUF, BLOCK)?; From 02694d65655baae0536d47beef4f0356145069f4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 14:55:54 +1100 Subject: [PATCH 1205/1579] spi: apple-qspi: Resume the controller even if its reset fails apple_qspi_resume() returns before spi_controller_resume() when the controller reset times out. The controller then stays marked suspended and every later spi_sync() on it fails with -ESHUTDOWN until reboot, while spi-mem operations, which do not look at that flag, go on using it anyway. Log the failure and resume the controller regardless. A transfer that then times out resets the controller again, as it does outside of system sleep. Fixes: ed0925cdf091 ("spi: apple-qspi: Restore the controller configuration after system sleep") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index 2142b14e167bf..a7edd0ff4e8d5 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -611,11 +611,14 @@ static int apple_qspi_resume(struct device *dev) struct apple_qspi *qspi = spi_controller_get_devdata(ctlr); int ret; + /* + * Reopen the controller even if the reset failed. Left suspended, it + * would fail every transfer until reboot; a transfer that times out + * resets it again instead. + */ ret = apple_qspi_hw_reset(qspi); - if (ret) { + if (ret) dev_err(dev, "controller reset failed on resume: %d\n", ret); - return ret; - } return spi_controller_resume(ctlr); } From b1f7d09dcf33e3bb06e81d846783b698d604be5b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 14:55:54 +1100 Subject: [PATCH 1206/1579] drm/apple: dcp: Initialise the start completion before checking RTKit dcp_start() returns for a missing or failed RTKit instance before it initialises start_done, which dcp_wait_ready() then waits on. No caller reaches dcp_start() after a failed component bind today, but nothing keeps it that way. Initialise the completion first, so a wait after a failed start times out like any other. Fixes: c0e5730f8058 ("drm/apple: dcp: Tidy RTKit start-up logging") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a392e68f3086b..9a5f7ba943e85 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3305,12 +3305,12 @@ int dcp_start(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; - /* A failed component bind leaves no RTKit instance to start. */ + /* Even if nothing starts, dcp_wait_ready() must find it initialised. */ + init_completion(&dcp->start_done); + if (IS_ERR_OR_NULL(dcp->rtk)) return -ENODEV; - init_completion(&dcp->start_done); - /* Make the H17P allocator available before external service discovery. */ if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); From 35d6b26e5c9d8bf22ce6284bf0594084ffd2dcdf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 15:30:05 +1100 Subject: [PATCH 1207/1579] soc: apple: aop: Test that a wrapped entry is refused below the read pointer When the write pointer is below the read pointer, the unread entries include [0, wptr), so a transmit entry must never wrap to the top of the ring there, even when it would end before the read pointer. write_entry() refuses that through its `wptr < rptr && wrapped` term, which commit 033fa5e3c095 ("soc: apple: aop: Accept a transmit entry at a write pointer of zero") rewrote, but no test pinned it: removing the term left every case passing. Add a case with two such entries. Neither ends on or past the read pointer, so only that term refuses them. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop_ring.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/soc/apple/aop_ring.rs b/drivers/soc/apple/aop_ring.rs index 41608ba7699b5..b07dede89fe23 100644 --- a/drivers/soc/apple/aop_ring.rs +++ b/drivers/soc/apple/aop_ring.rs @@ -320,6 +320,18 @@ mod tests { Ok(()) } + #[test] + fn a_wrapped_entry_is_refused_below_the_read_pointer() -> Result { + let ring = Ring::new(BUF, BLOCK)?; + // wptr < rptr: [0, wptr) holds unread entries, so an entry must + // not wrap even when it would end before the read pointer. + assert!(ring.write_entry(0xfc0, 0xf80, 0x80).is_err()); + assert!(ring + .write_entry(0xf80, 0xf00, 0x100 - QEH_SIZE + 1) + .is_err()); + Ok(()) + } + #[test] fn an_entry_never_reaches_the_read_pointer() -> Result { let ring = Ring::new(BUF, BLOCK)?; From 2719756a77ecd0fc030a4116ee2b74c48df020d2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 16:56:00 +1100 Subject: [PATCH 1208/1579] KEYS: trusted: Test trusted-key sources registered at runtime Add KUnit tests for register_trusted_key_source() and unregister_trusted_key_source(), with a synthetic source that touches no hardware and no real key material. They cover dispatch to the source and back to the absent stubs when it withdraws, the migration policy that each registration brings (a key update included), a second source being refused and unable to withdraw the first, a failing init(), and incomplete sources. Built in, KUnit runs before mark_readonly(), so one case, which registers with read-only-after-init data protected, skips there and says so. Loaded as a module after boot, as apple_sep is, the whole suite runs that way. [ryan: on 7.1 'migratable' is still writable, so registration after mark_readonly() cannot fault here. The case guards against a backport of the v7.2 change that makes it read-only after init.] Assisted-by: LLM Signed-off-by: Ryan Murray --- security/keys/trusted-keys/Kconfig | 20 + security/keys/trusted-keys/Makefile | 2 + .../trusted-keys/tests/trusted_source_kunit.c | 342 ++++++++++++++++++ 3 files changed, 364 insertions(+) create mode 100644 security/keys/trusted-keys/tests/trusted_source_kunit.c diff --git a/security/keys/trusted-keys/Kconfig b/security/keys/trusted-keys/Kconfig index 9e00482d886a5..b5b52b3d13725 100644 --- a/security/keys/trusted-keys/Kconfig +++ b/security/keys/trusted-keys/Kconfig @@ -57,3 +57,23 @@ config TRUSTED_KEYS_PKWM if !HAVE_TRUSTED_KEYS comment "No trust source selected!" endif + +config TRUSTED_SOURCE_KUNIT_TEST + tristate "KUnit tests for trusted-key sources registered at runtime" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds unit tests for register_trusted_key_source() and + unregister_trusted_key_source(), using a synthetic source that + touches no hardware and no real key material. + + Built in, they run before mark_readonly(), while read-only-after-init + data can still be written. Build them as a module and load it after + boot to test registration the way a source module such as apple_sep + does it. They skip when another source holds the slot; a source + module that registers while they run gets -EBUSY. + + For more information on KUnit and unit tests in general, + please refer to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. diff --git a/security/keys/trusted-keys/Makefile b/security/keys/trusted-keys/Makefile index 5fc053a21dadf..a940ac3f0d666 100644 --- a/security/keys/trusted-keys/Makefile +++ b/security/keys/trusted-keys/Makefile @@ -18,3 +18,5 @@ trusted-$(CONFIG_TRUSTED_KEYS_CAAM) += trusted_caam.o trusted-$(CONFIG_TRUSTED_KEYS_DCP) += trusted_dcp.o trusted-$(CONFIG_TRUSTED_KEYS_PKWM) += trusted_pkwm.o + +obj-$(CONFIG_TRUSTED_SOURCE_KUNIT_TEST) += tests/trusted_source_kunit.o diff --git a/security/keys/trusted-keys/tests/trusted_source_kunit.c b/security/keys/trusted-keys/tests/trusted_source_kunit.c new file mode 100644 index 0000000000000..f0491743f8d5a --- /dev/null +++ b/security/keys/trusted-keys/tests/trusted_source_kunit.c @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for trusted-key sources registered at runtime, through + * register_trusted_key_source() and unregister_trusted_key_source(). + * + * The source under test is synthetic: it touches no hardware and seals + * nothing for real. A test is skipped when another source holds the slot. + * + * Copyright 2026 Ryan Murray + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define FAKE_NAME "trusted_source_kunit" + +struct trusted_source_fake { + struct trusted_key_ops ops; + struct trusted_key_source src; + int init_ret; + int inits, exits, seals, unseals; +}; + +/* The ops have no context argument; the tests run one at a time. */ +static struct trusted_source_fake *fake; + +static int fake_init(void) +{ + fake->inits++; + return fake->init_ret; +} + +static void fake_exit(void) +{ + fake->exits++; +} + +/* Not a cipher: an invertible stand-in, so that unseal can be checked. */ +static int fake_seal(struct trusted_key_payload *p, char *datablob) +{ + unsigned int i; + + for (i = 0; i < p->key_len; i++) + p->blob[i] = p->key[i] ^ 0x5a; + p->blob_len = p->key_len; + fake->seals++; + return 0; +} + +static int fake_unseal(struct trusted_key_payload *p, char *datablob) +{ + unsigned int i; + + if (p->blob_len < MIN_KEY_SIZE || p->blob_len > MAX_KEY_SIZE) + return -EINVAL; + for (i = 0; i < p->blob_len; i++) + p->key[i] = p->blob[i] ^ 0x5a; + p->key_len = p->blob_len; + fake->unseals++; + return 0; +} + +static int fake_get_random(unsigned char *key, size_t key_len) +{ + size_t i; + + for (i = 0; i < key_len; i++) + key[i] = i; + return key_len; +} + +KUNIT_DEFINE_ACTION_WRAPPER(trusted_source_withdraw, + unregister_trusted_key_source, + struct trusted_key_source *); +KUNIT_DEFINE_ACTION_WRAPPER(trusted_source_key_put, key_put, struct key *); + +/* Register the fake, or skip if another source owns or claims the slot. */ +static void fake_register(struct kunit *test) +{ + int ret = register_trusted_key_source(&fake->src); + + if (ret == -EBUSY || ret == -ENODEV) + kunit_skip(test, "another trusted-key source holds the slot (%d)", + ret); + KUNIT_ASSERT_EQ(test, ret, 0); +} + +/* Instantiate a trusted key from @cmd; it is put when the test ends. */ +static struct key *fake_key(struct kunit *test, const char *cmd) +{ + struct key *key; + int ret; + + key = key_alloc(&key_type_trusted, FAKE_NAME, GLOBAL_ROOT_UID, + GLOBAL_ROOT_GID, current_cred(), KEY_POS_ALL, + KEY_ALLOC_NOT_IN_QUOTA, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, key); + + ret = key_instantiate_and_link(key, cmd, strlen(cmd), NULL, NULL); + if (ret) { + key_put(key); + return ERR_PTR(ret); + } + ret = kunit_add_action_or_reset(test, trusted_source_key_put, key); + KUNIT_ASSERT_EQ(test, ret, 0); + return key; +} + +static unsigned char fake_key_migratable(struct key *key) +{ + struct trusted_key_payload *p; + unsigned char migratable; + + down_read(&key->sem); + p = dereference_key_locked(key); + migratable = p->migratable; + up_read(&key->sem); + return migratable; +} + +/* + * Built in, KUnit runs before mark_readonly(), while read-only-after-init + * data can still be written. Loaded as a module after boot, as apple_sep + * is, every registration in this suite runs with that protection on. + */ +static void trusted_source_after_mark_readonly(struct kunit *test) +{ + int i; + + if (system_state < SYSTEM_RUNNING) + kunit_skip(test, "built in: runs before mark_readonly()"); + + for (i = 0; i < 2; i++) { + fake_register(test); + KUNIT_EXPECT_FALSE(test, IS_ERR(fake_key(test, "new 32"))); + unregister_trusted_key_source(&fake->src); + } + KUNIT_EXPECT_EQ(test, fake->inits, 2); + KUNIT_EXPECT_EQ(test, fake->exits, 2); + KUNIT_EXPECT_EQ(test, fake->seals, 2); +} + +/* Keys are sealed and unsealed by the source until it withdraws. */ +static void trusted_source_dispatch(struct kunit *test) +{ + struct trusted_key_payload *p; + unsigned char *plain; + unsigned int len; + struct key *key; + char *load; + + plain = kunit_kzalloc(test, MAX_KEY_SIZE, GFP_KERNEL); + load = kunit_kzalloc(test, sizeof("load ") + 2 * MAX_BLOB_SIZE, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, plain); + KUNIT_ASSERT_NOT_NULL(test, load); + + fake_register(test); + KUNIT_EXPECT_EQ(test, fake->inits, 1); + + key = fake_key(test, "new 32"); + KUNIT_ASSERT_FALSE(test, IS_ERR(key)); + KUNIT_EXPECT_EQ(test, fake->seals, 1); + + down_read(&key->sem); + p = dereference_key_locked(key); + len = min_t(unsigned int, p->key_len, MAX_KEY_SIZE); + memcpy(plain, p->key, len); + memcpy(load, "load ", 5); + bin2hex(load + 5, p->blob, + min_t(unsigned int, p->blob_len, MAX_BLOB_SIZE)); + up_read(&key->sem); + KUNIT_EXPECT_EQ(test, len, 32U); + + key = fake_key(test, load); + KUNIT_ASSERT_FALSE(test, IS_ERR(key)); + KUNIT_EXPECT_EQ(test, fake->unseals, 1); + + down_read(&key->sem); + p = dereference_key_locked(key); + KUNIT_EXPECT_EQ(test, p->key_len, len); + KUNIT_EXPECT_MEMEQ(test, p->key, plain, len); + up_read(&key->sem); + + unregister_trusted_key_source(&fake->src); + KUNIT_EXPECT_EQ(test, fake->exits, 1); + + /* Dispatch is back on the absent stubs, not on the departed source. */ + KUNIT_EXPECT_EQ(test, PTR_ERR(fake_key(test, "new 32")), -ENODEV); + KUNIT_EXPECT_EQ(test, PTR_ERR(fake_key(test, load)), -ENODEV); + KUNIT_EXPECT_EQ(test, fake->seals, 1); + KUNIT_EXPECT_EQ(test, fake->unseals, 1); +} + +/* Each registration brings its own policy; a withdrawn one leaves none. */ +static void trusted_source_migratable(struct kunit *test) +{ + static const unsigned char policy[] = { 1, 0, 1 }; + struct key *key; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(policy); i++) { + fake->ops.migratable = policy[i]; + fake_register(test); + + key = fake_key(test, "new 32"); + KUNIT_ASSERT_FALSE(test, IS_ERR(key)); + KUNIT_EXPECT_EQ(test, fake_key_migratable(key), policy[i]); + + /* Only a migratable key may be resealed. */ + KUNIT_EXPECT_EQ(test, key_update(make_key_ref(key, true), + "update", 6), + policy[i] ? 0 : -EPERM); + KUNIT_EXPECT_EQ(test, fake_key_migratable(key), policy[i]); + + unregister_trusted_key_source(&fake->src); + } + KUNIT_EXPECT_EQ(test, fake->seals, 3 + 2); +} + +/* A second source is refused, and cannot withdraw the first. */ +static void trusted_source_exclusive(struct kunit *test) +{ + struct trusted_key_source other = { + .name = FAKE_NAME, + .ops = &fake->ops, + }; + + fake_register(test); + KUNIT_EXPECT_EQ(test, register_trusted_key_source(&other), -EBUSY); + KUNIT_EXPECT_EQ(test, register_trusted_key_source(&fake->src), -EBUSY); + KUNIT_EXPECT_EQ(test, fake->inits, 1); + + unregister_trusted_key_source(&other); + KUNIT_EXPECT_EQ(test, fake->exits, 0); + KUNIT_EXPECT_FALSE(test, IS_ERR(fake_key(test, "new 32"))); + KUNIT_EXPECT_EQ(test, fake->seals, 1); + + unregister_trusted_key_source(&fake->src); + KUNIT_EXPECT_EQ(test, fake->exits, 1); +} + +/* A failed init() leaves no source behind, and registration can retry. */ +static void trusted_source_init_failure(struct kunit *test) +{ + int ret; + + fake->init_ret = -EIO; + ret = register_trusted_key_source(&fake->src); + if (ret == -EBUSY || ret == -ENODEV) + kunit_skip(test, "another trusted-key source holds the slot (%d)", + ret); + KUNIT_EXPECT_EQ(test, ret, -EIO); + KUNIT_EXPECT_EQ(test, fake->inits, 1); + KUNIT_EXPECT_EQ(test, fake->exits, 0); + KUNIT_EXPECT_EQ(test, PTR_ERR(fake_key(test, "new 32")), -ENODEV); + + fake->init_ret = 0; + fake_register(test); + KUNIT_EXPECT_FALSE(test, IS_ERR(fake_key(test, "new 32"))); + unregister_trusted_key_source(&fake->src); + KUNIT_EXPECT_EQ(test, fake->inits, 2); + KUNIT_EXPECT_EQ(test, fake->exits, 1); +} + +static void trusted_source_expect_invalid(struct kunit *test, + struct trusted_key_source *src) +{ + int ret = register_trusted_key_source(src); + + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + if (!ret) + unregister_trusted_key_source(src); +} + +/* Incomplete sources are refused before anything is called. */ +static void trusted_source_invalid(struct kunit *test) +{ + struct trusted_key_ops no_seal = fake->ops; + struct trusted_key_source no_name = { .ops = &fake->ops }; + struct trusted_key_source no_ops = { .name = FAKE_NAME }; + struct trusted_key_source bad_ops = { + .name = FAKE_NAME, + .ops = &no_seal, + }; + + no_seal.seal = NULL; + trusted_source_expect_invalid(test, NULL); + trusted_source_expect_invalid(test, &no_name); + trusted_source_expect_invalid(test, &no_ops); + trusted_source_expect_invalid(test, &bad_ops); + KUNIT_EXPECT_EQ(test, fake->inits, 0); +} + +static int trusted_source_init(struct kunit *test) +{ + fake = kunit_kzalloc(test, sizeof(*fake), GFP_KERNEL); + if (!fake) + return -ENOMEM; + + fake->ops.init = fake_init; + fake->ops.seal = fake_seal; + fake->ops.unseal = fake_unseal; + fake->ops.get_random = fake_get_random; + fake->ops.exit = fake_exit; + fake->src.name = FAKE_NAME; + fake->src.ops = &fake->ops; + + /* However the test ends; a no-op unless the fake is registered. */ + return kunit_add_action_or_reset(test, trusted_source_withdraw, + &fake->src); +} + +static struct kunit_case trusted_source_cases[] = { + KUNIT_CASE(trusted_source_after_mark_readonly), + KUNIT_CASE(trusted_source_dispatch), + KUNIT_CASE(trusted_source_migratable), + KUNIT_CASE(trusted_source_exclusive), + KUNIT_CASE(trusted_source_init_failure), + KUNIT_CASE(trusted_source_invalid), + {} +}; + +static struct kunit_suite trusted_source_suite = { + .name = "trusted_source", + .init = trusted_source_init, + .test_cases = trusted_source_cases, +}; + +kunit_test_suite(trusted_source_suite); + +MODULE_DESCRIPTION("KUnit tests for trusted-key sources registered at runtime"); +MODULE_LICENSE("GPL"); From 4f15946bdd603c77e344d922e39fd1949466fdb2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 16:57:00 +1100 Subject: [PATCH 1209/1579] KEYS: trusted: Keep a source that registers before init_trusted() A trusted-key source module that is built in registers from its probe: apple_sep with CONFIG_APPLE_SEP=y probes at device_initcall, before init_trusted() runs at late_initcall. init_trusted() then looks for a built-in source, finds none, and points the dispatch at the absent stubs, without taking the source mutex or checking for a registered source. That source still holds the slot, so nothing can take it again. Every new trusted key now fails with -ENODEV after registration was logged as a success, and the key type stays registered. A modular apple_sep loads after init_trusted() and is not affected. Mainline chooses the source once, in init_trusted(). Keep it that way, and have init_trusted() defer: under the source mutex, it leaves the slot and the dispatch to a source that is already active, and otherwise chooses as before. Whichever source comes up first keeps the slot, and a later one still gets -EBUSY; trusted.source= and trusted.rng= are checked on both paths, as before. The choice moves into trusted_key_core_init(), which the KUnit tests call with synthetic sources, in both orders: a source that registers before the core's init keeps the slot and its migration policy, and one that registers after it replaces the absent stubs. Fixes: bac0846c9087 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") Assisted-by: LLM Signed-off-by: Ryan Murray --- include/keys/trusted-type.h | 5 ++ security/keys/trusted-keys/Kconfig | 5 +- .../trusted-keys/tests/trusted_source_kunit.c | 73 ++++++++++++++++++- security/keys/trusted-keys/trusted_core.c | 70 ++++++++++++++---- 4 files changed, 134 insertions(+), 19 deletions(-) diff --git a/include/keys/trusted-type.h b/include/keys/trusted-type.h index bd9d21f55f6cb..e4a360f9e1670 100644 --- a/include/keys/trusted-type.h +++ b/include/keys/trusted-type.h @@ -86,6 +86,11 @@ extern struct key_type key_type_trusted; int register_trusted_key_source(struct trusted_key_source *src); void unregister_trusted_key_source(struct trusted_key_source *src); +#if IS_ENABLED(CONFIG_KUNIT) +/* For the KUnit tests of the trusted-key core only. */ +int trusted_key_core_init(const struct trusted_key_source *sources, size_t n); +#endif + #define TRUSTED_DEBUG 0 #if TRUSTED_DEBUG diff --git a/security/keys/trusted-keys/Kconfig b/security/keys/trusted-keys/Kconfig index b5b52b3d13725..0cd6e17af1e52 100644 --- a/security/keys/trusted-keys/Kconfig +++ b/security/keys/trusted-keys/Kconfig @@ -64,8 +64,9 @@ config TRUSTED_SOURCE_KUNIT_TEST default KUNIT_ALL_TESTS help This builds unit tests for register_trusted_key_source() and - unregister_trusted_key_source(), using a synthetic source that - touches no hardware and no real key material. + unregister_trusted_key_source(), called before and after the + core's own init, using synthetic sources that touch no hardware + and no real key material. Built in, they run before mark_readonly(), while read-only-after-init data can still be written. Build them as a module and load it after diff --git a/security/keys/trusted-keys/tests/trusted_source_kunit.c b/security/keys/trusted-keys/tests/trusted_source_kunit.c index f0491743f8d5a..6a5eb3e79d756 100644 --- a/security/keys/trusted-keys/tests/trusted_source_kunit.c +++ b/security/keys/trusted-keys/tests/trusted_source_kunit.c @@ -1,7 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only /* * KUnit tests for trusted-key sources registered at runtime, through - * register_trusted_key_source() and unregister_trusted_key_source(). + * register_trusted_key_source() and unregister_trusted_key_source(), before + * and after the core's own init. * * The source under test is synthetic: it touches no hardware and seals * nothing for real. A test is skipped when another source holds the slot. @@ -22,12 +23,17 @@ #include #define FAKE_NAME "trusted_source_kunit" +#define FAKE_BUILTIN_NAME "trusted_source_kunit_builtin" struct trusted_source_fake { struct trusted_key_ops ops; struct trusted_key_source src; int init_ret; int inits, exits, seals, unseals; + /* A built-in source for trusted_key_core_init(). */ + struct trusted_key_ops builtin_ops; + struct trusted_key_source builtin; + int builtin_inits; }; /* The ops have no context argument; the tests run one at a time. */ @@ -44,6 +50,13 @@ static void fake_exit(void) fake->exits++; } +/* A built-in source that finds no device, as each of them does on Apple. */ +static int fake_builtin_init(void) +{ + fake->builtin_inits++; + return -ENODEV; +} + /* Not a cipher: an invertible stand-in, so that unseal can be checked. */ static int fake_seal(struct trusted_key_payload *p, char *datablob) { @@ -149,6 +162,56 @@ static void trusted_source_after_mark_readonly(struct kunit *test) KUNIT_EXPECT_EQ(test, fake->seals, 2); } +/* + * A source module that is built in, as apple_sep can be, registers from its + * probe at device_initcall, before init_trusted() runs at late_initcall. + * The core then leaves it the slot and the dispatch. + */ +static void trusted_source_before_core_init(struct kunit *test) +{ + struct key *key; + + fake->ops.migratable = 1; + fake_register(test); + + KUNIT_EXPECT_EQ(test, trusted_key_core_init(&fake->builtin, 1), 0); + KUNIT_EXPECT_EQ(test, fake->builtin_inits, 0); + + key = fake_key(test, "new 32"); + KUNIT_ASSERT_FALSE(test, IS_ERR(key)); + KUNIT_EXPECT_EQ(test, fake->seals, 1); + KUNIT_EXPECT_EQ(test, fake_key_migratable(key), 1); + + /* The source still holds the slot, so it can withdraw. */ + unregister_trusted_key_source(&fake->src); + KUNIT_EXPECT_EQ(test, fake->exits, 1); + KUNIT_EXPECT_EQ(test, PTR_ERR(fake_key(test, "new 32")), -ENODEV); + KUNIT_EXPECT_EQ(test, fake->seals, 1); +} + +/* + * A source module registers after the core's init found no built-in + * source, as a modular apple_sep does. + */ +static void trusted_source_after_core_init(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, trusted_key_core_init(&fake->builtin, 1), 0); + if (!fake->builtin_inits) + kunit_skip(test, "another trusted-key source holds the slot"); + + /* With no source, the absent stubs answer. */ + KUNIT_EXPECT_EQ(test, PTR_ERR(fake_key(test, "new 32")), -ENODEV); + + fake_register(test); + KUNIT_EXPECT_FALSE(test, IS_ERR(fake_key(test, "new 32"))); + KUNIT_EXPECT_EQ(test, fake->seals, 1); + + unregister_trusted_key_source(&fake->src); + KUNIT_EXPECT_EQ(test, fake->inits, 1); + KUNIT_EXPECT_EQ(test, fake->exits, 1); + KUNIT_EXPECT_EQ(test, fake->builtin_inits, 1); +} + /* Keys are sealed and unsealed by the source until it withdraws. */ static void trusted_source_dispatch(struct kunit *test) { @@ -315,6 +378,11 @@ static int trusted_source_init(struct kunit *test) fake->src.name = FAKE_NAME; fake->src.ops = &fake->ops; + fake->builtin_ops = fake->ops; + fake->builtin_ops.init = fake_builtin_init; + fake->builtin.name = FAKE_BUILTIN_NAME; + fake->builtin.ops = &fake->builtin_ops; + /* However the test ends; a no-op unless the fake is registered. */ return kunit_add_action_or_reset(test, trusted_source_withdraw, &fake->src); @@ -322,6 +390,8 @@ static int trusted_source_init(struct kunit *test) static struct kunit_case trusted_source_cases[] = { KUNIT_CASE(trusted_source_after_mark_readonly), + KUNIT_CASE(trusted_source_before_core_init), + KUNIT_CASE(trusted_source_after_core_init), KUNIT_CASE(trusted_source_dispatch), KUNIT_CASE(trusted_source_migratable), KUNIT_CASE(trusted_source_exclusive), @@ -338,5 +408,6 @@ static struct kunit_suite trusted_source_suite = { kunit_test_suite(trusted_source_suite); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); MODULE_DESCRIPTION("KUnit tests for trusted-key sources registered at runtime"); MODULE_LICENSE("GPL"); diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c index 143fc687880ab..718a480484330 100644 --- a/security/keys/trusted-keys/trusted_core.c +++ b/security/keys/trusted-keys/trusted_core.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -473,18 +474,45 @@ static int kernel_get_random(unsigned char *key, size_t key_len) return get_random_bytes_wait(key, key_len) ?: key_len; } -static int __init init_trusted(void) +/** + * trusted_key_core_init - give the source slot to a built-in source + * @sources: the built-in sources, in order of preference + * @n: the number of @sources + * + * A source module that is built in, such as apple_sep, can register from + * its probe at device_initcall, before init_trusted() runs at + * late_initcall. Such a source keeps the slot: the built-in sources are + * not tried, and the dispatch stays with it. Otherwise the first built-in + * source to initialise takes the slot, and with none the dispatch goes to + * the absent stubs, so that a source module can register later. + * + * init_trusted() calls this once, with trusted_key_sources. The KUnit tests + * call it with synthetic sources of their own. + * + * Return: 0, or the error of trusted.rng or of a built-in source's init(). + */ +VISIBLE_IF_KUNIT int +trusted_key_core_init(const struct trusted_key_source *sources, size_t n) { int (*get_random)(unsigned char *key, size_t key_len); /* -ENODEV when the built-in array is empty or nothing matches, so a * later module source can still register instead of hitting -EBUSY. */ - int i, ret = -ENODEV; + int ret = -ENODEV; + size_t i; + + mutex_lock(&trusted_key_source_lock); + + /* A source module that is built in may have registered first. */ + if (trusted_key_source_active) { + ret = 0; + goto out; + } - for (i = 0; i < ARRAY_SIZE(trusted_key_sources); i++) { + for (i = 0; i < n; i++) { if (trusted_key_source && - strncmp(trusted_key_source, trusted_key_sources[i].name, - strlen(trusted_key_sources[i].name))) + strncmp(trusted_key_source, sources[i].name, + strlen(sources[i].name))) continue; /* @@ -492,31 +520,32 @@ static int __init init_trusted(void) * well as trusted.rng=$trusted.source if the trust source * defines its own get_random callback. */ - get_random = trusted_key_sources[i].ops->get_random; + get_random = sources[i].ops->get_random; if (trusted_rng && strcmp(trusted_rng, "default")) { if (!strcmp(trusted_rng, "kernel")) { get_random = kernel_get_random; - } else if (strcmp(trusted_rng, trusted_key_sources[i].name) || + } else if (strcmp(trusted_rng, sources[i].name) || !get_random) { pr_warn("Unsupported RNG. Supported: kernel"); if (get_random) - pr_cont(", %s", trusted_key_sources[i].name); + pr_cont(", %s", sources[i].name); pr_cont(", default\n"); - return -EINVAL; + ret = -EINVAL; + goto out; } } if (!get_random) get_random = kernel_get_random; - ret = trusted_key_sources[i].ops->init(); + ret = sources[i].ops->init(); if (!ret) { - static_call_update(trusted_key_seal, trusted_key_sources[i].ops->seal); - static_call_update(trusted_key_unseal, trusted_key_sources[i].ops->unseal); + static_call_update(trusted_key_seal, sources[i].ops->seal); + static_call_update(trusted_key_unseal, sources[i].ops->unseal); static_call_update(trusted_key_get_random, get_random); - trusted_key_exit = trusted_key_sources[i].ops->exit; - migratable = trusted_key_sources[i].ops->migratable; + trusted_key_exit = sources[i].ops->exit; + migratable = sources[i].ops->migratable; } if (!ret || ret != -ENODEV) @@ -537,14 +566,23 @@ static int __init init_trusted(void) static_call_update(trusted_key_unseal, trusted_key_absent_unseal); static_call_update(trusted_key_get_random, trusted_key_absent_get_random); - return 0; + ret = 0; + goto out; } if (!ret) trusted_key_source_active = true; - +out: + mutex_unlock(&trusted_key_source_lock); return ret; } +EXPORT_SYMBOL_IF_KUNIT(trusted_key_core_init); + +static int __init init_trusted(void) +{ + return trusted_key_core_init(trusted_key_sources, + ARRAY_SIZE(trusted_key_sources)); +} static void __exit cleanup_trusted(void) { From 0c44f4c9bd8571cf59f13d06bd0cfd0b873c193a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 17:19:43 +1100 Subject: [PATCH 1210/1579] usb: xhci: plat: Call the interrupt handler through the HC driver in tests The suspend-to-idle tests call xhci_irq() directly, but that function is not exported. With the xHCI core built in and the tests built as a module, as with KUNIT_ALL_TESTS=m on a distribution config, modpost fails: ERROR: modpost: "xhci_irq" [drivers/usb/host/xhci-plat-test.ko] undefined! The tests already set up the generic xHCI hc_driver with xhci_init_driver() and call its hub_control and hub_status_data methods. Call the interrupt handler through its irq method as well, which is xhci_irq() and is how the USB core invokes it from usb_hcd_irq(). Fixes: e9573a16e76c ("usb: xhci: Wake from suspend-to-idle only for wakeup events") Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/usb/host/xhci-plat-test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/usb/host/xhci-plat-test.c b/drivers/usb/host/xhci-plat-test.c index ca4814898714e..8583b63e5fe76 100644 --- a/drivers/usb/host/xhci-plat-test.c +++ b/drivers/usb/host/xhci-plat-test.c @@ -371,7 +371,7 @@ struct xhci_s2idle_test { /* The PORTSC read just before which the device below signals resume, from 1 */ unsigned int resume_at_read; unsigned int reads; - /* The generic xHCI root hub methods, as the USB core calls them */ + /* The generic xHCI root hub and interrupt methods, as the USB core calls them */ struct hc_driver driver; struct device *dev; char name[32]; @@ -649,7 +649,7 @@ static irqreturn_t xhci_s2idle_test_irq(struct xhci_s2idle_test *t) irqreturn_t ret; local_irq_disable(); - ret = xhci_irq(t->hcd); + ret = t->driver.irq(t->hcd); local_irq_enable(); return ret; } From 67a37af9b8c7fabb66b2ab3e54caa23db6d59aed Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 17:19:44 +1100 Subject: [PATCH 1211/1579] usb: typec: tipd: cd321x: Add a module description to the PM tests modpost warns when the CD321x PM tests are built as a module: WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/usb/typec/tipd/cd321x-pm-test.o Add one, as the other tipd test modules have. Fixes: c2514e492d2e ("thunderbolt: restore M1 tunneled PCIe state and recover late link loss") Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/usb/typec/tipd/cd321x-pm-test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index 977df770dcff5..08adc096cbb7c 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -569,4 +569,5 @@ static struct kunit_suite cd321x_pm_suite = { }; kunit_test_suite(cd321x_pm_suite); +MODULE_DESCRIPTION("KUnit tests for CD321x system sleep and cable-state revalidation"); MODULE_LICENSE("GPL"); From ee77f893e79f0ee43e11e685dd81e38832bbb0b4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 17:21:46 +1100 Subject: [PATCH 1212/1579] drm/apple: Add module descriptions to the KUnit test modules modpost warns about four of the Apple display KUnit test modules when DRM_APPLE_KUNIT_TEST is built as a module, as KUNIT_ALL_TESTS=m does on a distribution config: WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/gpu/drm/apple/afk_test.o WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/gpu/drm/apple/dcp_backlight_test.o WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/gpu/drm/apple/afk-service-test.o WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/gpu/drm/apple/iomfb-state-test.o Add one to each, as dcp-announce-test has. Fixes: 1ff8bb4e3bd8 ("drm: apple: afk: derive the ring header stride from INITRB") Fixes: 0da3b48fc8f7 ("drm: apple: test H17P backlight takeover and DPMS policy") Fixes: 5837706b10e3 ("drm/apple: pin DP service owners before recycling retired AFK slots") Fixes: fb40feb283b6 ("drm/apple: preserve mode invalidation across hotplug and modeset") Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/apple/afk-service-test.c | 1 + drivers/gpu/drm/apple/afk_test.c | 1 + drivers/gpu/drm/apple/dcp_backlight_test.c | 1 + drivers/gpu/drm/apple/iomfb-state-test.c | 1 + 4 files changed, 4 insertions(+) diff --git a/drivers/gpu/drm/apple/afk-service-test.c b/drivers/gpu/drm/apple/afk-service-test.c index a64064758a906..358d0edf34519 100644 --- a/drivers/gpu/drm/apple/afk-service-test.c +++ b/drivers/gpu/drm/apple/afk-service-test.c @@ -271,5 +271,6 @@ static struct kunit_suite afk_service_suite = { }; kunit_test_suite(afk_service_suite); +MODULE_DESCRIPTION("KUnit tests for the Apple AFK EPIC service lifetime"); MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/afk_test.c b/drivers/gpu/drm/apple/afk_test.c index 0621eebe914df..3bbed1096144d 100644 --- a/drivers/gpu/drm/apple/afk_test.c +++ b/drivers/gpu/drm/apple/afk_test.c @@ -113,4 +113,5 @@ static struct kunit_suite afk_ring_suite = { kunit_test_suite(afk_ring_suite); +MODULE_DESCRIPTION("KUnit tests for the Apple AFK ring buffers"); MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index a6fb45958b168..70379571a5dd3 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -528,4 +528,5 @@ static struct kunit_suite backlight_suite = { kunit_test_suite(backlight_suite); +MODULE_DESCRIPTION("KUnit tests for the Apple DCP backlight"); MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/iomfb-state-test.c b/drivers/gpu/drm/apple/iomfb-state-test.c index d751625a94058..97ba9924d454f 100644 --- a/drivers/gpu/drm/apple/iomfb-state-test.c +++ b/drivers/gpu/drm/apple/iomfb-state-test.c @@ -124,4 +124,5 @@ static struct kunit_suite dcp_mode_suite = { }; kunit_test_suite(dcp_mode_suite); +MODULE_DESCRIPTION("KUnit tests for the Apple DCP mode and connection state"); MODULE_LICENSE("Dual MIT/GPL"); From 7ec5a7211cda47308f4eb0c6eda4ee240f17a801 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 18:10:53 +1100 Subject: [PATCH 1213/1579] usb: typec: tipd: cd321x: Don't race the resume read in the IRQ test cd321x_ack_retry_across_sleep_test() asserted that the resume read was pending right after tipd_complete(). The driver queues that read with no delay, so on an SMP test machine a worker can already be running it, with the pending bit clear, and the test fails although the driver behaved correctly. With two CPUs it failed in 1 of 20 runs. Wait for the resume read with flush_work() instead. Unlike flush_delayed_work(), it does not run early the retry that the read schedules, so the loop that follows still sees exactly one retry pending after each read. Check that the resume read tried to read the events. The other pending and timer checks in the test look either at work that nothing can queue at that point, or at a retry scheduled at least 500 ms ahead. Not queueing the resume read, or giving up the retry after the cable-state budget, still makes the test fail, and now so does queueing the resume read late. Fixes: 7e97f2f5ee45 ("usb: typec: tipd: cd321x: Never leave interrupt events unacknowledged") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-irq-test.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/usb/typec/tipd/cd321x-irq-test.c b/drivers/usb/typec/tipd/cd321x-irq-test.c index 17ef778f96826..0c5067e9ab024 100644 --- a/drivers/usb/typec/tipd/cd321x-irq-test.c +++ b/drivers/usb/typec/tipd/cd321x-irq-test.c @@ -314,7 +314,7 @@ static void cd321x_ack_retry_across_sleep_test(struct kunit *test) { struct cd321x_test *t = cd321x_test_port(test); struct device *dev = t->cd.tps.dev; - unsigned int i; + unsigned int i, left; cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); t->ctrl.failing_event_reads = UINT_MAX; @@ -324,7 +324,15 @@ static void cd321x_ack_retry_across_sleep_test(struct kunit *test) /* Sleep cancels the retry, and the resume read takes it over. */ KUNIT_ASSERT_EQ(test, tipd_prepare(dev), 0); KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + left = t->ctrl.failing_event_reads; tipd_complete(dev); + /* + * The resume read is queued without delay, so on SMP a worker may be + * running it already. Wait for it, without running the retry it + * schedules early, and check that it tried to read the events. + */ + flush_work(&t->cd.resume_work.work); + KUNIT_EXPECT_EQ(test, t->ctrl.failing_event_reads, left - 1); for (i = 0; i < 2 * CD321X_RESUME_ATTEMPTS; i++) { KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); flush_delayed_work(&t->cd.resume_work); From d70b99b2ed147b35eca50b85cdea4c09aade0441 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 18:29:41 +1100 Subject: [PATCH 1214/1579] soc: apple: sep: Take an SBIO reply header only once it has landed The receive work waits for an SBIO reply header to leave the poison fill, but takes it as soon as any one byte has changed. The mailbox notification is not ordered against the enclave's DMA, so the header can be only partly visible. With chunk still poison, the payload read fails its bounds check; with version still poison, the reassembly rejects the reply. Both end the exchange with the NotAnswered sentinel, so the log says that no message arrived at all when one did. An M1 Pro logged exactly that for INIT_SBIO_COMMUNICATION about 3 ms after powering the sensor. That is when its reply normally arrives, and far short of the 5 s SBIO timeout, which returns ETIMEDOUT instead. Wait until every header word that taking the message reads has left the fill, within the existing 200 x 1 ms poll budget: all but flags for a data chunk, and err for an error report. No valid reply holds the fill in those words. A grant or a notification reads no header, so it is no longer held up by the buffer. Before the first reply of a boot, a grant used to wait 200 ms and then end the exchange that it was granting. An error report has no payload, so none is read. Report a reply whose header never fully lands, or fails a check, as Malformed. A header that stays entirely poison is still reported as BufferNeverWritten, and NotAnswered is kept for host-side failures. This is host-side only. No message to the enclave is added, removed or reordered, and no xART path is touched. KUnit tests (apple_sep_transfer) cover a torn header, a header that lands late, one that never lands, one left partly written, poison in flags, an error report, a grant and a reply failing its checks. Fixes: bac0846c9087 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 54 +++--- drivers/soc/apple/sep.rs | 20 +++ drivers/soc/apple/transfer.rs | 305 +++++++++++++++++++++++++++++++++- 3 files changed, 355 insertions(+), 24 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 92d7fb73f65b8..d0a731e5aa11b 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2664,31 +2664,31 @@ impl SepData { let generation = self.sbio_rx.lock().generation(); // notification and the enclave's DMA are not ordered; buffer may still be poison - if !self.ool_await_written(&self.ool_sbio, 0, transfer::HEADER_LEN) { - self.sbio_fail_unwritten(); - return; - } - - let header = match self.ool_read(&self.ool_sbio, 0, transfer::HEADER_LEN) { - Ok(h) => h, - Err(_) => { - self.sbio_fail(); - return; - } - }; - let packet = match transfer::Packet::decode(&header) { + let packet = match transfer::take_header( + &SbioHeader(self), + marker, + OOL_POISON_OUTBOUND, + OOL_WRITE_POLL_ATTEMPTS, + ) { Ok(p) => p, - Err(_) => { - self.sbio_fail(); + Err(status) => { + self.sbio_fail_with(status); return; } }; - let chunk = packet.chunk as usize; + let chunk = if transfer::carries_payload(marker) { + packet.chunk as usize + } else { + 0 + }; let payload = if chunk == 0 { KVec::new() + } else if chunk > OOL_SIZE_SBIO - transfer::HEADER_LEN { + self.sbio_fail_with(transfer::DeviceStatus::Malformed); + return; } else if !self.ool_await_written(&self.ool_sbio, transfer::HEADER_LEN, chunk) { - self.sbio_fail_unwritten(); + self.sbio_fail_with(transfer::DeviceStatus::BufferNeverWritten); return; } else { match self.ool_read(&self.ool_sbio, transfer::HEADER_LEN, chunk) { @@ -2741,10 +2741,8 @@ impl SepData { self.sbio_wq.notify_all(); } - fn sbio_fail_unwritten(&self) { - self.sbio_rx - .lock() - .abort_with(transfer::DeviceStatus::BufferNeverWritten); + fn sbio_fail_with(&self, status: transfer::DeviceStatus) { + self.sbio_rx.lock().abort_with(status); self.sbio_wq.notify_all(); } @@ -3460,6 +3458,20 @@ impl SepData { } } +/// The SBIO reply header, as [`transfer::take_header`] samples it. +struct SbioHeader<'a>(&'a SepData); + +impl transfer::HeaderSlot for SbioHeader<'_> { + fn sample(&self, header: &mut [u8; transfer::HEADER_LEN]) -> bool { + self.0.ool_peek(&self.0.ool_sbio, 0, header) + } + + fn pause(&self) { + let poll = kernel::time::Delta::from_millis(i64::from(OOL_WRITE_POLL_MS)); + kernel::time::delay::fsleep(poll); + } +} + pub(crate) struct SbioOp { opcode: u16, payload: [u8; SBIO_MAX_PAYLOAD], diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index ee3a7a708bf29..1b7e455073e2b 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -169,6 +169,10 @@ const OOL_WRITE_POLL_ATTEMPTS: u32 = 200; const OOL_POISON_INBOUND: u8 = 0xA5; const OOL_POISON_OUTBOUND: u8 = 0x5A; +// transfer::take_header() tells a landed SBIO header from the fill by its +// words, which a valid reply cannot hold as four equal nonzero bytes. +static_assert!(OOL_POISON_OUTBOUND != 0); + const PROTECTED_DATA_AVAILABLE: bool = true; const RNG_MAX_WORDS_PER_READ: usize = 16; @@ -1481,6 +1485,22 @@ impl SepData { Ok(payload) } + /// Copies `out.len()` bytes at `off` of the outbound buffer into `out`; + /// `false` once the buffers are gone or the range is outside them. + fn ool_peek(&self, slot: &Mutex>, off: usize, out: &mut [u8]) -> bool { + let guard = slot.lock(); + let Some(buffers) = guard.as_ref() else { + return false; + }; + if off + out.len() > buffers.allocated { + return false; + } + // SAFETY: the region is within the allocation as checked above, and + // the driver is the only host-side accessor. + out.copy_from_slice(unsafe { &buffers.outbound.as_ref()[off..off + out.len()] }); + true + } + fn ool_await_written(&self, slot: &Mutex>, off: usize, len: usize) -> bool { for _ in 0..OOL_WRITE_POLL_ATTEMPTS { { diff --git a/drivers/soc/apple/transfer.rs b/drivers/soc/apple/transfer.rs index a4f5f0df3e308..d82ca1a394eb1 100644 --- a/drivers/soc/apple/transfer.rs +++ b/drivers/soc/apple/transfer.rs @@ -20,6 +20,8 @@ pub(crate) enum DeviceStatus { ReportedWithoutStatus, NotAnswered, BufferNeverWritten, + /// A reply arrived, but its header never fully landed or failed a check. + Malformed, } impl DeviceStatus { @@ -28,7 +30,8 @@ impl DeviceStatus { DeviceStatus::Answered(err) => Some(*err), DeviceStatus::ReportedWithoutStatus | DeviceStatus::NotAnswered - | DeviceStatus::BufferNeverWritten => None, + | DeviceStatus::BufferNeverWritten + | DeviceStatus::Malformed => None, } } @@ -53,6 +56,10 @@ impl kernel::fmt::Display for DeviceStatus { f, "NONE — a payload notification arrived but the outbound buffer was never written (a race with the enclave's DMA, not a silent enclave)" ), + DeviceStatus::Malformed => write!( + f, + "NONE — a reply arrived but its header was left partly written or failed a check (host sentinel, not an enclave value)" + ), } } } @@ -80,7 +87,102 @@ fn le32(buf: &[u8], word: usize) -> u32 { u32::from_le_bytes([buf[o], buf[o + 1], buf[o + 2], buf[o + 3]]) } +// The header words, numbered as in `Packet::decode`, that taking a message +// reads. A data chunk reads all but flags; an error report reads only err. +// None of them can hold a fill of four equal nonzero bytes in a valid reply: +// version is VERSION, opcode is an echoed 16-bit opcode, and total, offset +// and chunk are at most MAX_TRANSACTION. err is the enclave's status. A +// poison err taken as landed would be reported as a refusal the enclave did +// not make. If the enclave ever answered with that value, waiting costs the +// poll budget and the exchange fails either way. +const DATA_WORDS: [usize; 6] = [0, 1, 2, 4, 5, 6]; +const ERROR_WORDS: [usize; 1] = [4]; + +static_assert!(MAX_TRANSACTION < 0x0101_0101); + +fn header_words_read(marker: u8) -> &'static [usize] { + match marker { + MARKER_FIRST | MARKER_NEXT => &DATA_WORDS, + MARKER_ERROR => &ERROR_WORDS, + // A grant or a notification: `Reassembly::on_chunk` reads no header. + _ => &[], + } +} + +/// Whether a message with `marker` has a payload after its header. +pub(crate) fn carries_payload(marker: u8) -> bool { + matches!(marker, MARKER_FIRST | MARKER_NEXT) +} + +fn header_landed(header: &[u8; HEADER_LEN], words: &[usize], poison: u8) -> bool { + let poison = u32::from_le_bytes([poison; 4]); + words.iter().all(|&word| le32(header, word) != poison) +} + +/// The host's view of the header at the start of an endpoint's outbound +/// buffer, which the enclave fills by DMA. +pub(crate) trait HeaderSlot { + /// Copies the header out; `false` once the buffers are gone. + fn sample(&self, header: &mut [u8; HEADER_LEN]) -> bool; + /// Waits before the next sample. + fn pause(&self); +} + +/// Takes the header of a message with `marker` from `slot`, sampling it up to +/// `attempts` times until every word that taking the message reads has left +/// the `poison` fill, a nonzero byte. The mailbox notification is not ordered +/// against the enclave's DMA, so the header can be unwritten, or only partly +/// written, when the message arrives: one changed byte is not enough. A +/// message whose handling reads no header is taken without looking at the +/// buffer. +pub(crate) fn take_header( + slot: &impl HeaderSlot, + marker: u8, + poison: u8, + attempts: u32, +) -> Result { + let words = header_words_read(marker); + if words.is_empty() { + return Ok(Packet::NONE); + } + + let mut header = [poison; HEADER_LEN]; + let mut landed = false; + for attempt in 0..attempts { + if attempt > 0 { + slot.pause(); + } + if !slot.sample(&mut header) { + return Err(DeviceStatus::NotAnswered); + } + if header_landed(&header, words, poison) { + landed = true; + break; + } + } + + if !landed { + return Err(if header.iter().all(|&b| b == poison) { + DeviceStatus::BufferNeverWritten + } else { + DeviceStatus::Malformed + }); + } + Packet::decode(&header).map_err(|_| DeviceStatus::Malformed) +} + impl Packet { + /// Stands in for the header of a message whose handling reads none of it. + pub(crate) const NONE: Packet = Packet { + version: 0, + total: 0, + offset: 0, + flags: 0, + err: 0, + opcode: 0, + chunk: 0, + }; + pub(crate) fn decode(buf: &[u8]) -> Result { if buf.len() < HEADER_LEN { return Err(EINVAL); @@ -251,7 +353,7 @@ impl Reassembly { } fn fail(&mut self) { - self.abort(); + self.abort_with(DeviceStatus::Malformed); } /// Takes one chunk of a reply. `generation` is [`Self::generation`] as @@ -368,7 +470,7 @@ impl Reassembly { .extend_from_slice(payload, GFP_KERNEL) .is_err() { - self.fail(); + self.abort(); return Progress::Failed; } @@ -399,8 +501,11 @@ impl Reassembly { #[kernel::macros::kunit_tests(apple_sep_transfer)] mod tests { use super::*; + use core::cell::Cell; const OPCODE: u32 = 0x48; + const POISON: u8 = 0x5A; + const ATTEMPTS: u32 = 200; fn reply(err: u32) -> Packet { Packet { @@ -414,6 +519,79 @@ mod tests { } } + fn data(total: u32, offset: u32, chunk: u32) -> Packet { + Packet { + total, + offset, + chunk, + ..reply(0) + } + } + + /// An outbound buffer header that the enclave's DMA fills word by word, + /// from word 0 up, while the host pauses between samples. + struct Dma { + header: Cell<[u8; HEADER_LEN]>, + incoming: Cell<[u8; HEADER_LEN]>, + landed: Cell, + per_pause: Cell, + samples: Cell, + } + + impl Dma { + fn new() -> Dma { + Dma { + header: Cell::new([POISON; HEADER_LEN]), + incoming: Cell::new([POISON; HEADER_LEN]), + landed: Cell::new(0), + per_pause: Cell::new(0), + samples: Cell::new(0), + } + } + + /// The enclave starts writing `packet`: `now` words are visible at + /// once, and `per_pause` more after each pause. + fn deliver(&self, packet: &Packet, now: usize, per_pause: usize) { + let mut bytes = [0u8; HEADER_LEN]; + let _ = packet.encode(&mut bytes); + self.incoming.set(bytes); + self.landed.set(0); + self.per_pause.set(per_pause); + self.land(now); + } + + fn land(&self, words: usize) { + let landed = core::cmp::min(self.landed.get() + words, HEADER_WORDS); + let mut header = self.header.get(); + header[..landed * 4].copy_from_slice(&self.incoming.get()[..landed * 4]); + self.header.set(header); + self.landed.set(landed); + } + + /// One word becomes visible on its own. + fn show(&self, word: usize, value: u32) { + let mut header = self.header.get(); + header[word * 4..word * 4 + 4].copy_from_slice(&value.to_le_bytes()); + self.header.set(header); + } + } + + impl HeaderSlot for Dma { + fn sample(&self, header: &mut [u8; HEADER_LEN]) -> bool { + self.samples.set(self.samples.get() + 1); + *header = self.header.get(); + true + } + + fn pause(&self) { + self.land(self.per_pause.get()); + } + } + + fn take(dma: &Dma, marker: u8) -> Result { + take_header(dma, marker, POISON, ATTEMPTS) + } + #[test] fn a_reply_to_the_current_exchange_completes_it() { let mut rx = Reassembly::new(); @@ -451,4 +629,125 @@ mod tests { Progress::Complete )); } + + #[test] + fn a_torn_header_is_taken_once_it_has_landed() { + let dma = Dma::new(); + // version..flags are visible; err, opcode and chunk are still poison. + dma.deliver(&data(8, 0, 8), 4, 1); + let taken = take(&dma, MARKER_FIRST); + assert!(taken.is_ok()); + let Ok(packet) = taken else { + return; + }; + assert_eq!(packet.chunk, 8); + assert_eq!(packet.err, 0); + assert_eq!(packet.opcode, OPCODE); + + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + let generation = rx.generation(); + assert!(matches!( + rx.on_chunk(generation, MARKER_FIRST, &packet, &[0u8; 8]), + Progress::Complete + )); + assert!(rx.take_done().is_some_and(|done| done.status.is_ok())); + } + + #[test] + fn a_late_header_is_taken_whole() { + let dma = Dma::new(); + // Nothing is visible when the message arrives; then a word per pause. + dma.deliver(&data(8, 0, 8), 0, 1); + let taken = take(&dma, MARKER_FIRST); + assert!(taken.is_ok()); + let Ok(packet) = taken else { + return; + }; + assert_eq!(packet.total, 8); + assert_eq!(packet.chunk, 8); + assert_eq!(packet.opcode, OPCODE); + assert_eq!(dma.samples.get(), HEADER_WORDS as u32 + 1); + } + + #[test] + fn a_header_that_never_lands_is_reported_as_never_written() { + let dma = Dma::new(); + assert!(matches!( + take(&dma, MARKER_FIRST), + Err(DeviceStatus::BufferNeverWritten) + )); + assert_eq!(dma.samples.get(), ATTEMPTS); + } + + #[test] + fn a_header_left_partly_written_is_reported_as_malformed() { + let dma = Dma::new(); + dma.deliver(&data(8, 0, 8), 4, 0); + assert!(matches!( + take(&dma, MARKER_FIRST), + Err(DeviceStatus::Malformed) + )); + } + + #[test] + fn flags_may_hold_the_poison() { + let dma = Dma::new(); + let packet = Packet { + flags: u32::from_le_bytes([POISON; 4]), + ..data(8, 0, 8) + }; + dma.deliver(&packet, HEADER_WORDS, 0); + assert!(take(&dma, MARKER_FIRST).is_ok()); + assert_eq!(dma.samples.get(), 1); + } + + #[test] + fn an_error_report_reads_only_its_status() { + let dma = Dma::new(); + dma.show(4, 0x16); + let taken = take(&dma, MARKER_ERROR); + assert!(taken.is_ok()); + let Ok(packet) = taken else { + return; + }; + assert_eq!(packet.err, 0x16); + + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + let generation = rx.generation(); + assert!(matches!( + rx.on_chunk(generation, MARKER_ERROR, &packet, &[]), + Progress::Complete + )); + assert!(rx + .take_done() + .is_some_and(|done| done.status.answered() == Some(0x16))); + } + + #[test] + fn a_grant_or_a_notification_reads_no_header() { + let dma = Dma::new(); + assert!(take(&dma, MARKER_REQUEST).is_ok()); + assert!(take(&dma, MARKER_FIRST - 1).is_ok()); + assert_eq!(dma.samples.get(), 0); + } + + #[test] + fn a_reply_failing_a_check_is_reported_as_malformed() { + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + let generation = rx.generation(); + let bad = Packet { + version: VERSION + 1, + ..reply(0) + }; + assert!(matches!( + rx.on_chunk(generation, MARKER_FIRST, &bad, &[]), + Progress::Failed + )); + assert!(rx + .take_done() + .is_some_and(|done| matches!(done.status, DeviceStatus::Malformed))); + } } From f66113f26bcbafd5ebb00bf5f619406c6d27788c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 18:29:41 +1100 Subject: [PATCH 1215/1579] soc: apple: sep: Poison an SBIO reply header again once it is read Waiting for the reply header to leave the poison fill only works while the header region holds the fill, and it does so only until the first reply of a boot. Every later header lands on top of the one before it. If its notification overtakes the DMA, the receive work finds the previous header complete and takes it for the new one. A reply to the same opcode then completes the exchange with the previous answer, and a continuation chunk fails its offset check. Fill the header with the poison again as soon as it has been looked at, before the message is processed. That is before any continuation is sent, and before the requester wakes to send its next request. The receive work is the only reader of the header, so this cannot race a read. A grant or a notification does not read the header and leaves it alone. This is host-side only. Nothing sent to the enclave changes. KUnit tests (apple_sep_transfer) cover the next exchange's reply, a continuation chunk and the refill itself. Fixes: bac0846c9087 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 5 +++ drivers/soc/apple/sep.rs | 18 ++++++++ drivers/soc/apple/transfer.rs | 82 +++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index d0a731e5aa11b..7a9afe8fbebf2 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -3466,6 +3466,11 @@ impl transfer::HeaderSlot for SbioHeader<'_> { self.0.ool_peek(&self.0.ool_sbio, 0, header) } + fn repoison(&self) { + self.0 + .ool_repoison(&self.0.ool_sbio, 0, transfer::HEADER_LEN); + } + fn pause(&self) { let poll = kernel::time::Delta::from_millis(i64::from(OOL_WRITE_POLL_MS)); kernel::time::delay::fsleep(poll); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 1b7e455073e2b..b738f553b3030 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1501,6 +1501,24 @@ impl SepData { true } + /// Fills `len` bytes at `off` of the outbound buffer with the poison + /// again, once the message they held has been read. + fn ool_repoison(&self, slot: &Mutex>, off: usize, len: usize) { + let guard = slot.lock(); + let Some(buffers) = guard.as_ref() else { + return; + }; + if off + len > buffers.allocated { + return; + } + // SAFETY: the region is within the allocation as checked above, and + // the driver is the only host-side accessor. The enclave writes the + // region only to deliver a message, and the one it held has been + // looked at. If part of it was still landing, that part may land over + // the fill afterwards, as a header that has not landed yet does. + unsafe { buffers.outbound.as_mut()[off..off + len].fill(OOL_POISON_OUTBOUND) }; + } + fn ool_await_written(&self, slot: &Mutex>, off: usize, len: usize) -> bool { for _ in 0..OOL_WRITE_POLL_ATTEMPTS { { diff --git a/drivers/soc/apple/transfer.rs b/drivers/soc/apple/transfer.rs index d82ca1a394eb1..3323e7e2cfc09 100644 --- a/drivers/soc/apple/transfer.rs +++ b/drivers/soc/apple/transfer.rs @@ -124,6 +124,8 @@ fn header_landed(header: &[u8; HEADER_LEN], words: &[usize], poison: u8) -> bool pub(crate) trait HeaderSlot { /// Copies the header out; `false` once the buffers are gone. fn sample(&self, header: &mut [u8; HEADER_LEN]) -> bool; + /// Fills the header with the poison again. + fn repoison(&self); /// Waits before the next sample. fn pause(&self); } @@ -135,6 +137,9 @@ pub(crate) trait HeaderSlot { /// written, when the message arrives: one changed byte is not enough. A /// message whose handling reads no header is taken without looking at the /// buffer. +/// +/// Once looked at, the header is poisoned again whatever the outcome, so the +/// next message cannot take it for its own. pub(crate) fn take_header( slot: &impl HeaderSlot, marker: u8, @@ -160,6 +165,7 @@ pub(crate) fn take_header( break; } } + slot.repoison(); if !landed { return Err(if header.iter().all(|&b| b == poison) { @@ -574,6 +580,10 @@ mod tests { header[word * 4..word * 4 + 4].copy_from_slice(&value.to_le_bytes()); self.header.set(header); } + + fn poisoned(&self) -> bool { + self.header.get().iter().all(|&b| b == POISON) + } } impl HeaderSlot for Dma { @@ -583,6 +593,10 @@ mod tests { true } + fn repoison(&self) { + self.header.set([POISON; HEADER_LEN]); + } + fn pause(&self) { self.land(self.per_pause.get()); } @@ -750,4 +764,72 @@ mod tests { .take_done() .is_some_and(|done| matches!(done.status, DeviceStatus::Malformed))); } + + #[test] + fn the_previous_replys_header_is_not_taken_for_the_next() { + let dma = Dma::new(); + dma.deliver(&data(8, 0, 8), HEADER_WORDS, 0); + assert!(take(&dma, MARKER_FIRST).is_ok_and(|p| p.chunk == 8)); + + // The next exchange's message arrives before any of its header. + dma.deliver(&data(4, 0, 4), 0, HEADER_WORDS); + let taken = take(&dma, MARKER_FIRST); + assert!(taken.is_ok()); + let Ok(packet) = taken else { + return; + }; + assert_eq!(packet.chunk, 4); + assert_eq!(packet.total, 4); + } + + #[test] + fn a_continuation_does_not_take_the_first_chunks_header() { + let dma = Dma::new(); + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + let generation = rx.generation(); + + dma.deliver(&data(16, 0, 8), HEADER_WORDS, 0); + let taken = take(&dma, MARKER_FIRST); + assert!(taken.is_ok()); + let Ok(first) = taken else { + return; + }; + assert!(matches!( + rx.on_chunk(generation, MARKER_FIRST, &first, &[0u8; 8]), + Progress::NeedMore(_) + )); + + // The second chunk's message arrives before its header lands. + dma.deliver(&data(16, 8, 8), 0, 2); + let taken = take(&dma, MARKER_NEXT); + assert!(taken.is_ok()); + let Ok(next) = taken else { + return; + }; + assert!(matches!( + rx.on_chunk(generation, MARKER_NEXT, &next, &[0u8; 8]), + Progress::Complete + )); + assert!(rx + .take_done() + .is_some_and(|done| done.status.is_ok() && done.payload.len() == 16)); + } + + #[test] + fn a_header_is_poisoned_again_once_looked_at() { + let dma = Dma::new(); + dma.deliver(&data(8, 0, 8), HEADER_WORDS, 0); + assert!(take(&dma, MARKER_FIRST).is_ok()); + assert!(dma.poisoned()); + + dma.deliver(&data(8, 0, 8), 4, 0); + assert!(take(&dma, MARKER_FIRST).is_err()); + assert!(dma.poisoned()); + + // A grant leaves the buffer alone. + dma.show(4, 0x16); + assert!(take(&dma, MARKER_REQUEST).is_ok()); + assert!(!dma.poisoned()); + } } From 17224f4570a4d4f2dd2d5ae34f584455efd217e6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 05:51:41 +1100 Subject: [PATCH 1216/1579] usb: hub: Add helpers to power a port off and on for a lane change A Type-C PHY that switches the PIPE interface of a SuperSpeed root port between its USB3 lanes and a dummy backend needs the controller to have stopped using that interface first. A dual-role port gets that from a role change, which takes the whole controller down. On Apple's T8140 (J700) the controller's USB2 port serves a fixed hub on the board, behind which sit both USB-C connectors and whatever keyboard is plugged in, so the controller has to stay up and only its SuperSpeed root port can be taken down. The port's "disable" attribute already does that for userspace: it disconnects the device on the port and switches the port power off, recorded in power_bits. Give a driver that holds the hub the same, and a counterpart that powers the port on again and has hub_wq look at it. hub_wq warm resets a SuperSpeed port before that, as toggling the port power can leave the two ends of a link out of sync, see usb_port_runtime_resume(). The driver relies on the port staying off until it powers it on, but the hub lock it holds does not keep everyone else away from the port power: - Port runtime PM switches it without the hub lock. Once userspace lets a port power off, its runtime suspend and resume can run at any time, and the resume of its USB2 peer resumes it too. Power-off therefore resumes the port and keeps a runtime PM reference until power-on. Resuming first also tells a port that runtime PM had switched off, which is then held off for the caller, from one that someone switched off on purpose, which is reported and left alone. Taking the hub lock in the runtime PM callbacks instead would deadlock, as hub_event() waits for port runtime PM with that lock held. A port that was switched off through "disable" is recorded as such, so that runtime PM suspending it later does not make it the driver's, and a port whose runtime PM failed earlier is held as it is. - A connect change queued for the port, say by a failed reset of the device just disconnected, made hub_port_connect() power it on again. Power-off drops the connect change and warm reset requests still queued for the port once it is off, and hub_port_connect() no longer switches on a port whose power_bits bit is clear, as hub_power_on() already does not. That changes hub_port_connect() for every hub: a port switched off through the "disable" attribute now stays off through such a change as well. - Writing 0 to the "disable" attribute powers the port on. For a port that a driver holds off, that write now fails with -EBUSY. Writing 1 takes the port over from the driver: the hold ends, and power-on then reports that the port is no longer held and leaves it off. The ABI description of the attribute says so. Only hubs that switch the power of each port on its own are supported, so that nothing but the given port is touched. On an xHCI root hub with port power control that is the port's PORTSC power bit, which the hub driver already toggles for its "attempt power cycle" recovery. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/ABI/testing/sysfs-bus-usb | 5 + drivers/usb/core/hub.c | 174 +++++++++++++++++++++++- drivers/usb/core/hub.h | 5 + drivers/usb/core/port.c | 21 ++- include/linux/usb.h | 4 + 5 files changed, 204 insertions(+), 5 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-bus-usb b/Documentation/ABI/testing/sysfs-bus-usb index af9b653422f16..a556843b7eb76 100644 --- a/Documentation/ABI/testing/sysfs-bus-usb +++ b/Documentation/ABI/testing/sysfs-bus-usb @@ -247,6 +247,11 @@ Description: attached to the port will not be detected, initialized, or enumerated. + A driver that moves the data lanes behind a port can hold + the port off for a while, see usb_hub_port_power_off(). + Writing 0 then fails with -EBUSY. Writing 1 takes the port + over from the driver, which leaves it off from then on. + What: /sys/bus/usb/devices/...//port/early_stop Date: Sep 2022 Contact: Ray Chi diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index d3ce25f24c047..191ff9e21dac4 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -2120,6 +2120,170 @@ bool usb_device_is_owned(struct usb_device *udev) return !!hub->ports[udev->portnum - 1]->port_owner; } +/** + * usb_hub_port_power_off - disconnect a hub port and hold its power off + * @hdev: USB device belonging to the usb hub + * @port1: port number + * + * For a driver that has to move the data lanes behind a port while the hub + * keeps running, such as a Type-C PHY that switches the PIPE interface of a + * SuperSpeed root port to another backend. The device on the port is + * disconnected and the port is powered off, which takes its link down, as + * the port's "disable" attribute does. Connect changes still queued for the + * port are dropped. Only hubs that switch the power of each port on its own + * are supported. + * + * The port is then held off until usb_hub_port_power_on(): the hub driver + * leaves it off, and the port is kept runtime-active, so that neither its + * own runtime resume nor that of its USB2 peer switches it on in between. + * Writing 0 to its "disable" attribute fails with -EBUSY; writing 1 takes the + * port over, after which usb_hub_port_power_on() leaves it off. + * + * A port that is off for someone else, say through its "disable" attribute, + * is left as it is, and the caller can tell from the return value that it is + * not held for it. A port that runtime PM had switched off is not such a + * port: it is resumed, then held off for the caller. + * + * Context: process context; the caller holds @hdev's device lock. + * + * Return: 0 if the port is held off for the caller, also if it was already, + * 1 if it is off for someone else, -EOPNOTSUPP if the hub cannot switch the + * power of a single port, or another negative error code. + */ +int usb_hub_port_power_off(struct usb_device *hdev, int port1) +{ + struct usb_hub *hub = usb_hub_to_struct_hub(hdev); + struct usb_interface *intf; + struct usb_port *port_dev; + int ret; + + device_lock_assert(&hdev->dev); + if (!hub || hub->disconnected) + return -ENODEV; + if (port1 < 1 || port1 > hdev->maxchild) + return -EINVAL; + if ((le16_to_cpu(hub->descriptor->wHubCharacteristics) & HUB_CHAR_LPSM) != + HUB_CHAR_INDV_PORT_LPSM) + return -EOPNOTSUPP; + + port_dev = hub->ports[port1 - 1]; + if (port_dev->held_off) + return 0; + /* Runtime PM may have suspended it since, which does not make it ours. */ + if (port_dev->disabled) + return 1; + + intf = to_usb_interface(hub->intfdev); + ret = usb_autopm_get_interface(intf); + if (ret < 0) + return ret; + + /* + * The port's runtime resume switches its power on, and does not take + * the hub lock, as hub_event() waits for it under that lock. Keep port + * runtime PM out instead, after waiting for any of it in progress. + */ + ret = pm_runtime_resume_and_get(&port_dev->dev); + if (ret < 0) { + /* After a runtime PM error, nothing resumes the port any more. */ + if (!pm_runtime_active(&port_dev->dev)) + goto out; + pm_runtime_get_noresume(&port_dev->dev); + } + + if (!port_dev->child && !test_bit(port1, hub->power_bits)) { + ret = 1; + goto put_port; + } + + if (port_dev->child) + usb_disconnect(&port_dev->child); + + ret = usb_hub_set_port_power(hdev, hub, port1, false); + if (ret) { + /* The link may still be up: have hub_wq look for its device. */ + set_bit(port1, hub->change_bits); + kick_hub_wq(hub); + goto put_port; + } + + /* The runtime PM reference is dropped once the hold ends. */ + port_dev->held_off = true; + /* Nothing hub_wq still has queued for the port applies to it off. */ + clear_bit(port1, hub->change_bits); + clear_bit(port1, hub->warm_reset_bits); + msleep(2 * hub_power_on_good_delay(hub)); + /* Losing the connection with the power is not an event. */ + usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_CONNECTION); + if (!port_dev->is_superspeed) + usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_ENABLE); + usb_autopm_put_interface(intf); + return 0; + +put_port: + pm_runtime_put(&port_dev->dev); +out: + usb_autopm_put_interface(intf); + return ret; +} +EXPORT_SYMBOL_GPL(usb_hub_port_power_off); + +/** + * usb_hub_port_power_on - switch on a port held off by usb_hub_port_power_off() + * @hdev: USB device belonging to the usb hub + * @port1: port number + * + * hub_wq then checks the port for a connection. It warm resets a SuperSpeed + * port first: toggling the port power can leave the two ends of the link + * out of sync (see usb_port_runtime_resume()), and the warm reset starts + * both from a reset. The port is under its usual runtime PM policy again. + * + * A port that is not held off, because its "disable" attribute took it over + * for example, is left alone. + * + * Context: process context; the caller holds @hdev's device lock. + * + * Return: 0 if the port was switched on, 1 if it was not held off, or a + * negative error code, in which case the port stays held off. + */ +int usb_hub_port_power_on(struct usb_device *hdev, int port1) +{ + struct usb_hub *hub = usb_hub_to_struct_hub(hdev); + struct usb_interface *intf; + struct usb_port *port_dev; + int ret; + + device_lock_assert(&hdev->dev); + if (!hub || hub->disconnected) + return -ENODEV; + if (port1 < 1 || port1 > hdev->maxchild) + return -EINVAL; + + port_dev = hub->ports[port1 - 1]; + if (!port_dev->held_off) + return 1; + + intf = to_usb_interface(hub->intfdev); + ret = usb_autopm_get_interface(intf); + if (ret < 0) + return ret; + + ret = usb_hub_set_port_power(hdev, hub, port1, true); + if (!ret) { + msleep(hub_power_on_good_delay(hub)); + if (hub_is_superspeed(hdev)) + set_bit(port1, hub->warm_reset_bits); + set_bit(port1, hub->change_bits); + kick_hub_wq(hub); + port_dev->held_off = false; + pm_runtime_put(&port_dev->dev); + } + + usb_autopm_put_interface(intf); + return ret; +} +EXPORT_SYMBOL_GPL(usb_hub_port_power_on); + static void update_port_device_state(struct usb_device *udev) { struct usb_hub *hub; @@ -5475,11 +5639,13 @@ static void hub_port_connect(struct usb_hub *hub, int port1, u16 portstatus, /* * maybe switch power back on (e.g. root hub was reset) - * but only if the port isn't owned by someone else. + * but only if the port isn't owned by someone else, and + * was not switched off on purpose, see hub_power_on(). */ - if (hub_is_port_power_switchable(hub) - && !usb_port_is_power_on(hub, portstatus) - && !port_dev->port_owner) + if (hub_is_port_power_switchable(hub) && + !usb_port_is_power_on(hub, portstatus) && + !port_dev->port_owner && + test_bit(port1, hub->power_bits)) set_port_feature(hdev, port1, USB_PORT_FEAT_POWER); if (portstatus & USB_PORT_STAT_ENABLE) diff --git a/drivers/usb/core/hub.h b/drivers/usb/core/hub.h index da3235930b0c8..3c4465e378ef4 100644 --- a/drivers/usb/core/hub.h +++ b/drivers/usb/core/hub.h @@ -102,6 +102,9 @@ struct usb_hub { * @usb3_lpm_u2_permit: whether USB3 U2 LPM is permitted. * @early_stop: whether port initialization will be stopped earlier. * @ignore_event: whether events of the port are ignored. + * @disabled: switched off through the "disable" attribute + * @held_off: switched off by usb_hub_port_power_off(), which holds the port + * runtime-active until usb_hub_port_power_on() */ struct usb_port { struct usb_device *child; @@ -123,6 +126,8 @@ struct usb_port { unsigned int is_superspeed:1; unsigned int usb3_lpm_u1_permit:1; unsigned int usb3_lpm_u2_permit:1; + bool disabled; + bool held_off; }; #define to_usb_port(_dev) \ diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c index b4452b665f591..89066ef6ffdd0 100644 --- a/drivers/usb/core/port.c +++ b/drivers/usb/core/port.c @@ -151,6 +151,21 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, goto out_hdev_lock; } + /* + * A driver holds the port off while it moves the lanes behind it, see + * usb_hub_port_power_off(). Powering it on would run its link on lanes + * in transition. Keeping it off takes the port over from the driver, + * which then leaves it off. + */ + if (port_dev->held_off) { + if (!disabled) { + rc = -EBUSY; + goto out_hdev_lock; + } + port_dev->held_off = false; + pm_runtime_put(&port_dev->dev); + } + if (disabled && port_dev->child) usb_disconnect(&port_dev->child); @@ -163,8 +178,10 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_ENABLE); } - if (!rc) + if (!rc) { + port_dev->disabled = disabled; rc = count; + } out_hdev_lock: usb_unlock_device(hdev); @@ -860,6 +877,8 @@ void usb_hub_remove_port_device(struct usb_hub *hub, int port1) peer = port_dev->peer; if (peer) unlink_peers(port_dev, peer); + if (port_dev->held_off) + pm_runtime_put_noidle(&port_dev->dev); component_del(&port_dev->dev, &connector_ops); sysfs_put(port_dev->state_kn); device_unregister(&port_dev->dev); diff --git a/include/linux/usb.h b/include/linux/usb.h index e8c5cfe9e0591..359259fffbd79 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -955,6 +955,10 @@ int usb_hub_claim_port(struct usb_device *hdev, unsigned port1, int usb_hub_release_port(struct usb_device *hdev, unsigned port1, struct usb_dev_state *owner); +/* port power switching for a driver that moves the lanes behind a port */ +int usb_hub_port_power_off(struct usb_device *hdev, int port1); +int usb_hub_port_power_on(struct usb_device *hdev, int port1); + /** * usb_make_path - returns stable device path in the usb tree * @dev: the device whose path is being constructed From 24b1c5eb3cb6e0677845c2ad3e203b7d3bf25a72 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 07:27:28 +1100 Subject: [PATCH 1217/1579] usb: core: Add KUnit tests for the port power helpers usb_hub_port_power_off() promises that a port stays off until usb_hub_port_power_on(), whatever its runtime PM, the runtime PM of its USB2 peer and its "disable" attribute do in between. A mocked hub cannot show that. Run the real hub driver instead, on the two root hubs of a host controller made of software: a USB2 and a SuperSpeed root hub with one port each, peered as on an xHCI. The controller keeps the port status and nothing else, and no device ever connects. The tests write port attributes through sysfs, as userspace does, and check that: - the port is held off, and only the given port is switched; - with port power-off allowed, neither the port's runtime suspend and resume nor the resume of its USB2 peer power it on while it is held; - a port that runtime PM had switched off is held for the caller, not reported as someone else's; - the "disable" attribute cannot power a held port on, and can take it over, after which power-on leaves it off. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/Kconfig | 13 + drivers/usb/core/Makefile | 1 + drivers/usb/core/port-power-test.c | 558 +++++++++++++++++++++++++++++ 3 files changed, 572 insertions(+) create mode 100644 drivers/usb/core/port-power-test.c diff --git a/drivers/usb/core/Kconfig b/drivers/usb/core/Kconfig index 1251c543f6cb0..869acdf0c3a36 100644 --- a/drivers/usb/core/Kconfig +++ b/drivers/usb/core/Kconfig @@ -165,3 +165,16 @@ config USB_DEFAULT_AUTHORIZATION_MODE ACPI selecting value 2 is analogous to selecting value 0. If unsure, keep the default value. + +config USB_PORT_POWER_KUNIT_TEST + tristate "KUnit tests for switching hub port power" if !KUNIT_ALL_TESTS + depends on USB && KUNIT + default KUNIT_ALL_TESTS + help + Test that a port powered off with usb_hub_port_power_off() stays + off until usb_hub_port_power_on(), whatever port runtime PM and + the port's "disable" attribute do in between. The tests run the + hub driver on the root hubs of a software host controller and + need no USB hardware. + + If unsure, say N. diff --git a/drivers/usb/core/Makefile b/drivers/usb/core/Makefile index 64468ad4a6d51..68101ef9880d8 100644 --- a/drivers/usb/core/Makefile +++ b/drivers/usb/core/Makefile @@ -25,3 +25,4 @@ obj-$(CONFIG_USB_QUIRKS_KUNIT_TEST) += usb-quirks-test.o obj-$(CONFIG_USB_WAKEUP_KUNIT_TEST) += usb-wakeup-test.o obj-$(CONFIG_USB_LEDS_TRIGGER_USBPORT) += ledtrig-usbport.o +obj-$(CONFIG_USB_PORT_POWER_KUNIT_TEST) += port-power-test.o diff --git a/drivers/usb/core/port-power-test.c b/drivers/usb/core/port-power-test.c new file mode 100644 index 0000000000000..2a5fee7fe6854 --- /dev/null +++ b/drivers/usb/core/port-power-test.c @@ -0,0 +1,558 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for usb_hub_port_power_off() and usb_hub_port_power_on(). + * + * The tests run on a host controller made of software: a USB2 and a + * SuperSpeed root hub with one port each, peered as on an xHCI, so that the + * hub driver, the port devices and their runtime PM are the real ones. The + * controller only keeps the status of its ports, and no device ever connects. + * Port attributes are written through sysfs, as userspace writes them. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct port_power_test_hc { + spinlock_t lock; /* protects the port status, as hub_wq reads it too */ + u16 status; + u16 change; + unsigned int power_ons; +}; + +struct port_power_test { + struct usb_hcd *hcd; + struct usb_hcd *shared_hcd; + struct usb_device *hs_root; + struct usb_device *ss_root; + struct device *hs_port; + struct device *ss_port; + struct vfsmount *sysfs; +}; + +static const struct { + struct usb_bos_descriptor bos; + struct usb_ss_cap_descriptor ss_cap; +} __packed port_power_test_bos = { + .bos = { + .bLength = USB_DT_BOS_SIZE, + .bDescriptorType = USB_DT_BOS, + .wTotalLength = cpu_to_le16(sizeof(port_power_test_bos)), + .bNumDeviceCaps = 1, + }, + .ss_cap = { + .bLength = USB_DT_USB_SS_CAP_SIZE, + .bDescriptorType = USB_DT_DEVICE_CAPABILITY, + .bDevCapabilityType = USB_SS_CAP_TYPE, + .wSpeedSupported = cpu_to_le16(USB_5GBPS_OPERATION), + .bFunctionalitySupport = USB_LOW_SPEED_OPERATION, + }, +}; + +static struct port_power_test_hc *port_power_test_hc(struct usb_hcd *hcd) +{ + return (struct port_power_test_hc *)hcd->hcd_priv; +} + +static u16 port_power_test_power_bit(struct usb_hcd *hcd) +{ + return hcd->speed >= HCD_USB3 ? USB_SS_PORT_STAT_POWER : USB_PORT_STAT_POWER; +} + +static void port_power_test_set_power(struct usb_hcd *hcd, bool on) +{ + struct port_power_test_hc *hc = port_power_test_hc(hcd); + + if (on) { + hc->status |= port_power_test_power_bit(hcd); + hc->power_ons++; + } else { + hc->status &= ~port_power_test_power_bit(hcd); + } + if (hcd->speed >= HCD_USB3) { + hc->status &= ~USB_PORT_STAT_LINK_STATE; + hc->status |= on ? USB_SS_PORT_LS_RX_DETECT : USB_SS_PORT_LS_SS_DISABLED; + } +} + +static bool port_power_test_powered(struct usb_hcd *hcd) +{ + struct port_power_test_hc *hc = port_power_test_hc(hcd); + + guard(spinlock_irqsave)(&hc->lock); + return hc->status & port_power_test_power_bit(hcd); +} + +static unsigned int port_power_test_power_ons(struct usb_hcd *hcd) +{ + struct port_power_test_hc *hc = port_power_test_hc(hcd); + + guard(spinlock_irqsave)(&hc->lock); + return hc->power_ons; +} + +static void port_power_test_hub_descriptor(struct usb_hcd *hcd, struct usb_hub_descriptor *desc) +{ + memset(desc, 0, sizeof(*desc)); + desc->bNbrPorts = 1; + desc->wHubCharacteristics = cpu_to_le16(HUB_CHAR_INDV_PORT_LPSM | HUB_CHAR_NO_OCPM); + desc->bPwrOn2PwrGood = 1; + if (hcd->speed >= HCD_USB3) { + desc->bDescriptorType = USB_DT_SS_HUB; + desc->bDescLength = USB_DT_SS_HUB_SIZE; + } else { + desc->bDescriptorType = USB_DT_HUB; + desc->bDescLength = USB_DT_HUB_NONVAR_SIZE + 2; + desc->u.hs.DeviceRemovable[1] = 0xff; + } +} + +static u16 port_power_test_change_bit(u16 feature) +{ + switch (feature) { + case USB_PORT_FEAT_C_CONNECTION: + return USB_PORT_STAT_C_CONNECTION; + case USB_PORT_FEAT_C_ENABLE: + return USB_PORT_STAT_C_ENABLE; + case USB_PORT_FEAT_C_RESET: + return USB_PORT_STAT_C_RESET; + case USB_PORT_FEAT_C_BH_PORT_RESET: + return USB_PORT_STAT_C_BH_RESET; + case USB_PORT_FEAT_C_PORT_LINK_STATE: + return USB_PORT_STAT_C_LINK_STATE; + } + return 0; +} + +static int port_power_test_hub_control(struct usb_hcd *hcd, u16 type_req, u16 value, u16 index, + char *buf, u16 length) +{ + struct port_power_test_hc *hc = port_power_test_hc(hcd); + + switch (type_req) { + case DeviceRequest | USB_REQ_GET_DESCRIPTOR: + if (hcd->speed < HCD_USB3 || value >> 8 != USB_DT_BOS) + return -EPIPE; + length = min_t(u16, length, sizeof(port_power_test_bos)); + memcpy(buf, &port_power_test_bos, length); + return length; + case GetHubDescriptor: + port_power_test_hub_descriptor(hcd, (struct usb_hub_descriptor *)buf); + return 0; + case GetHubStatus: + memset(buf, 0, 4); + return 0; + } + + if ((index & 0xff) != 1) + return -EPIPE; + + guard(spinlock_irqsave)(&hc->lock); + switch (type_req) { + case GetPortStatus: + put_unaligned_le16(hc->status, buf); + put_unaligned_le16(hc->change, buf + 2); + return 0; + case SetPortFeature: + if (value == USB_PORT_FEAT_POWER) + port_power_test_set_power(hcd, true); + else if (value == USB_PORT_FEAT_RESET) + hc->change |= USB_PORT_STAT_C_RESET; + else if (value == USB_PORT_FEAT_BH_PORT_RESET) + hc->change |= USB_PORT_STAT_C_RESET | USB_PORT_STAT_C_BH_RESET; + return 0; + case ClearPortFeature: + if (value == USB_PORT_FEAT_POWER) + port_power_test_set_power(hcd, false); + else + hc->change &= ~port_power_test_change_bit(value); + return 0; + } + return -EPIPE; +} + +static int port_power_test_reset(struct usb_hcd *hcd) +{ + struct port_power_test_hc *hc = port_power_test_hc(hcd); + + /* As on an xHCI, the primary HCD has the USB2 root hub */ + if (usb_hcd_is_primary_hcd(hcd)) { + hcd->speed = HCD_USB2; + hcd->self.root_hub->speed = USB_SPEED_HIGH; + } + spin_lock_init(&hc->lock); + port_power_test_set_power(hcd, false); + hc->power_ons = 0; + + return 0; +} + +static int port_power_test_start(struct usb_hcd *hcd) +{ + hcd->uses_new_polling = 1; + return 0; +} + +static void port_power_test_stop(struct usb_hcd *hcd) +{ +} + +static int port_power_test_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags) +{ + return -ENODEV; +} + +static int port_power_test_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status) +{ + return -EINVAL; +} + +static int port_power_test_get_frame_number(struct usb_hcd *hcd) +{ + return 0; +} + +static int port_power_test_hub_status_data(struct usb_hcd *hcd, char *buf) +{ + return 0; +} + +static const struct hc_driver port_power_test_hc_driver = { + .description = "usb-port-power-test", + .product_desc = "USB port power test controller", + .hcd_priv_size = sizeof(struct port_power_test_hc), + .flags = HCD_USB3 | HCD_SHARED, + .reset = port_power_test_reset, + .start = port_power_test_start, + .stop = port_power_test_stop, + .urb_enqueue = port_power_test_urb_enqueue, + .urb_dequeue = port_power_test_urb_dequeue, + .get_frame_number = port_power_test_get_frame_number, + .hub_status_data = port_power_test_hub_status_data, + .hub_control = port_power_test_hub_control, +}; + +static void port_power_test_put_hcds(void *data) +{ + struct port_power_test *t = data; + + usb_put_hcd(t->shared_hcd); + usb_put_hcd(t->hcd); +} + +static void port_power_test_remove_hcds(void *data) +{ + struct port_power_test *t = data; + + put_device(t->ss_port); + put_device(t->hs_port); + usb_remove_hcd(t->shared_hcd); + usb_remove_hcd(t->hcd); +} + +static void port_power_test_unmount(void *data) +{ + kern_unmount(data); +} + +/* The hub driver has set up a root hub once it waits for status changes. */ +static void port_power_test_wait_hub(struct kunit *test, struct usb_hcd *hcd) +{ + unsigned int ms; + + for (ms = 0; ms < 2000 && !READ_ONCE(hcd->status_urb); ms += 20) + msleep(20); + KUNIT_ASSERT_NOT_NULL(test, READ_ONCE(hcd->status_urb)); +} + +static struct device *port_power_test_port(struct kunit *test, struct usb_device *root) +{ + char *name = kasprintf(GFP_KERNEL, "%s-port1", dev_name(&root->dev)); + struct device *port; + + KUNIT_ASSERT_NOT_NULL(test, name); + port = device_find_child_by_name(&root->actconfig->interface[0]->dev, name); + kfree(name); + KUNIT_ASSERT_NOT_NULL(test, port); + + return port; +} + +static int port_power_test_init(struct kunit *test) +{ + struct file_system_type *sysfs_type; + struct port_power_test *t; + struct device *dev; + int ret; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + test->priv = t; + + dev = kunit_device_register(test, "usb-port-power-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + t->hcd = usb_create_hcd(&port_power_test_hc_driver, dev, dev_name(dev)); + KUNIT_ASSERT_NOT_NULL(test, t->hcd); + t->hcd->skip_phy_initialization = 1; + t->shared_hcd = usb_create_shared_hcd(&port_power_test_hc_driver, dev, dev_name(dev), + t->hcd); + if (!t->shared_hcd) + usb_put_hcd(t->hcd); + KUNIT_ASSERT_NOT_NULL(test, t->shared_hcd); + ret = kunit_add_action_or_reset(test, port_power_test_put_hcds, t); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = usb_add_hcd(t->hcd, 0, 0); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = usb_add_hcd(t->shared_hcd, 0, 0); + if (ret) + usb_remove_hcd(t->hcd); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, port_power_test_remove_hcds, t); + KUNIT_ASSERT_EQ(test, ret, 0); + + t->hs_root = t->hcd->self.root_hub; + t->ss_root = t->shared_hcd->self.root_hub; + port_power_test_wait_hub(test, t->hcd); + port_power_test_wait_hub(test, t->shared_hcd); + t->hs_port = port_power_test_port(test, t->hs_root); + t->ss_port = port_power_test_port(test, t->ss_root); + KUNIT_ASSERT_TRUE(test, port_power_test_powered(t->hcd)); + KUNIT_ASSERT_TRUE(test, port_power_test_powered(t->shared_hcd)); + + /* Root hub runtime PM is not what these tests are about. */ + usb_disable_autosuspend(t->hs_root); + usb_disable_autosuspend(t->ss_root); + + /* sysfs is built in, so its type holds no module reference. */ + sysfs_type = get_fs_type("sysfs"); + KUNIT_ASSERT_NOT_NULL(test, sysfs_type); + t->sysfs = kern_mount(sysfs_type); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->sysfs); + ret = kunit_add_action_or_reset(test, port_power_test_unmount, t->sysfs); + KUNIT_ASSERT_EQ(test, ret, 0); + + return 0; +} + +/* Write @val to the attribute @attr of @dev, as userspace does. */ +static int port_power_test_write(struct kunit *test, struct device *dev, const char *attr, + const char *val) +{ + struct port_power_test *t = test->priv; + struct path root = { .mnt = t->sysfs, .dentry = t->sysfs->mnt_root }; + char *kpath = kobject_get_path(&dev->kobj, GFP_KERNEL); + struct file *file; + loff_t pos = 0; + ssize_t ret; + char *path; + + KUNIT_ASSERT_NOT_NULL(test, kpath); + path = kasprintf(GFP_KERNEL, "%s/%s", kpath + 1, attr); + kfree(kpath); + KUNIT_ASSERT_NOT_NULL(test, path); + + file = file_open_root(&root, path, O_WRONLY, 0); + kfree(path); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = kernel_write(file, val, strlen(val), &pos); + fput(file); + + return ret < 0 ? ret : 0; +} + +/* Let userspace allow runtime PM to switch off both ports. */ +static void port_power_test_allow_power_off(struct kunit *test) +{ + struct port_power_test *t = test->priv; + + if (!IS_ENABLED(CONFIG_PM)) + kunit_skip(test, "needs CONFIG_PM"); + + KUNIT_ASSERT_EQ(test, port_power_test_write(test, t->hs_port, "power/pm_qos_no_power_off", + "0"), 0); + KUNIT_ASSERT_EQ(test, port_power_test_write(test, t->ss_port, "power/pm_qos_no_power_off", + "0"), 0); +} + +static int port_power_test_usage(struct device *dev) +{ + return atomic_read(&dev->power.usage_count); +} + +/* The port stays off until powered on, and only the given port is switched. */ +static void port_power_test_hold(struct kunit *test) +{ + struct port_power_test *t = test->priv; + unsigned int hs_power_ons = port_power_test_power_ons(t->hcd); + int usage = port_power_test_usage(t->ss_port); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + /* A second change of the lanes finds the port held for it */ + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage); + + /* Nothing is held any more */ + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 1); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 2), -EINVAL); + usb_unlock_device(t->ss_root); + + /* The USB2 root port was never switched */ + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->hcd)); + KUNIT_EXPECT_EQ(test, port_power_test_power_ons(t->hcd), hs_power_ons); +} + +/* + * With port power-off allowed, port runtime PM runs while the caller holds + * the hub: it can neither suspend the held port nor resume it, which would + * switch its power on, and neither can the resume of its USB2 peer. + */ +static void port_power_test_runtime_pm(struct kunit *test) +{ + struct port_power_test *t = test->priv; + + port_power_test_allow_power_off(test); + /* The USB2 port, while active, keeps its SuperSpeed peer active */ + KUNIT_ASSERT_EQ(test, pm_runtime_resume_and_get(t->hs_port), 0); + KUNIT_ASSERT_TRUE(test, pm_runtime_active(t->ss_port)); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + + /* The USB2 port suspends, which lets its peer suspend */ + pm_runtime_put_sync(t->hs_port); + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(t->hs_port)); + KUNIT_EXPECT_LT(test, pm_runtime_suspend(t->ss_port), 0); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(t->ss_port)); + + /* The USB2 port resumes, and resumes its peer first */ + KUNIT_ASSERT_EQ(test, pm_runtime_resume_and_get(t->hs_port), 0); + KUNIT_EXPECT_GE(test, pm_runtime_resume(t->ss_port), 0); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(t->ss_port)); + usb_unlock_device(t->ss_root); + pm_runtime_put(t->hs_port); +} + +/* A port that runtime PM switched off is the caller's to hold, not someone else's */ +static void port_power_test_runtime_suspended(struct kunit *test) +{ + struct port_power_test *t = test->priv; + + port_power_test_allow_power_off(test); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->hs_port), 0); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->ss_port), 0); + KUNIT_ASSERT_TRUE(test, pm_runtime_suspended(t->ss_port)); + KUNIT_ASSERT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(t->ss_port)); + + /* The resume of the USB2 peer leaves it off */ + KUNIT_ASSERT_EQ(test, pm_runtime_resume_and_get(t->hs_port), 0); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + /* and only the caller switches it on */ + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); + usb_unlock_device(t->ss_root); + pm_runtime_put(t->hs_port); +} + +/* The "disable" attribute can keep a held port off, and then owns it */ +static void port_power_test_disable(struct kunit *test) +{ + struct port_power_test *t = test->priv; + int usage = port_power_test_usage(t->ss_port); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + usb_unlock_device(t->ss_root); + + /* Not powered on behind the lanes' back */ + KUNIT_EXPECT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "0"), -EBUSY); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + /* Kept off: from now on it is userspace's port */ + KUNIT_EXPECT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "1"), 0); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 1); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + usb_unlock_device(t->ss_root); + + KUNIT_EXPECT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "0"), 0); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); +} + +/* A disabled port stays userspace's after runtime PM has suspended it */ +static void port_power_test_disabled_suspended(struct kunit *test) +{ + struct port_power_test *t = test->priv; + + KUNIT_ASSERT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "1"), 0); + port_power_test_allow_power_off(test); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->hs_port), 0); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->ss_port), 0); + KUNIT_ASSERT_TRUE(test, pm_runtime_suspended(t->ss_port)); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 1); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + usb_unlock_device(t->ss_root); +} + +/* The host controller can go away while a port is held off */ +static void port_power_test_removed_held(struct kunit *test) +{ + struct port_power_test *t = test->priv; + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + usb_unlock_device(t->ss_root); +} + +static struct kunit_case port_power_test_cases[] = { + KUNIT_CASE(port_power_test_hold), + KUNIT_CASE(port_power_test_runtime_pm), + KUNIT_CASE(port_power_test_runtime_suspended), + KUNIT_CASE(port_power_test_disable), + KUNIT_CASE(port_power_test_disabled_suspended), + KUNIT_CASE(port_power_test_removed_held), + {} +}; + +static struct kunit_suite port_power_test_suite = { + .name = "usb-hub-port-power", + .init = port_power_test_init, + .test_cases = port_power_test_cases, +}; + +kunit_test_suite(port_power_test_suite); + +MODULE_DESCRIPTION("KUnit tests for USB hub port power switching"); +MODULE_LICENSE("GPL"); From cbb9188fef4aca19c88d7e47b81eb29acd4d315b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 05:51:41 +1100 Subject: [PATCH 1218/1579] phy: apple: atc: Move a fixed-hub PIPE with the SuperSpeed port off On a port behind a fixed hub the controller stays in host mode, so the Type-C mux callback parks the PIPE, rebuilds the lanes and routes the PIPE back to the USB3 PHY underneath a running controller. dwc3-apple.c documents the order the hardware expects instead: the controller stops using the PIPE, then the PHY switches it to the dummy backend, then the lanes change, and in reverse on the way up. Get it wrong and the port "is just broken until a phy+dwc3 reset". On a J700 the PIPE lock is not acknowledged while the controller's SuperSpeed port still has a device, so unplugging a USB-C dock with DisplayPort and USB3 failed to park the PIPE until hub_wq had processed the SuperSpeed disconnect, seconds later: 22:05:35 usb 1-1.2: USB disconnect, device number 17 22:05:36 phy-apple-atc 40b000000.phy: Pipehandler lock not acked. 22:05:36 sn201202x 1-0c: failed to place Type-C mux in safe mode: -110 ... 22:05:39 sn201202x 1-0c: cable revalidation failed after 3 attempts 22:05:48 usb 2-1: USB disconnect, device number 9 After a run of these, the dock's SuperSpeed hub trained at 10 Gb/s on every following plug but never enumerated, until a reboot. What kept it there is not established, but it followed PIPE moves under a live link. Stopping the controller is no option here: its USB2 port serves the hub on the board, and with it both USB-C connectors and the keyboard. Follow the order for the SuperSpeed side only. Before a mux update touches the PIPE or the lanes, have the USB core disconnect the device on the controller's SuperSpeed root port and power the port off. That is the port power the hub driver toggles for its "attempt power cycle" recovery, which the same logs show leaving the USB2 devices alone. Power the port on again only once the PIPE is on the USB3 backend of the running host with no change left unfinished, and let hub_wq warm reset it before enumerating. After a failed change, or in a mode without USB3, the port stays off, so that the controller does not train a link against a half-configured PHY. A mode that the PHY cannot run, such as USB4 on a T8140, is turned down before the port goes down for it, so that a working USB3 link is not disconnected for nothing. When the port cannot be restored after a failed change, that error is returned rather than -EOPNOTSUPP, which the Type-C core takes for success. A port that someone else switched off, through its "disable" attribute, stays theirs: the USB core reports it when the PHY takes the port down or brings it up, and the PHY then owes it nothing. A mux update that keeps the mode still skips all of this, unless an earlier update left the port off; then it only powers the port on, and a USB failure on the way does not fail what may be a DisplayPort update. A host that starts up powers its ports itself, so starting one clears the record of a port left off. The device on the port used to be resumed and held across the change instead. It is now disconnected, as it would be anyway once its lanes change. Ports without a fixed hub, which includes every M1 and M2 port, take no root hub and are unchanged. Fixes: 70f7c21e7d32 ("phy: apple: atc: Route the PIPE from the Type-C mux on a fixed-hub port") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/Kconfig | 7 +- drivers/phy/apple/atc-test.c | 384 ++++++++++++++++++++++++++++++++++- drivers/phy/apple/atc.c | 309 ++++++++++++++++++---------- 3 files changed, 593 insertions(+), 107 deletions(-) diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig index e67f51241c29b..1fb7d1368fa21 100644 --- a/drivers/phy/apple/Kconfig +++ b/drivers/phy/apple/Kconfig @@ -21,9 +21,10 @@ config PHY_APPLE_ATC_KUNIT_TEST help Build the KUnit tests for the Type-C mux decisions of the Apple Type-C PHY driver into the driver. They check how Type-C states - map to PHY modes and which mux updates may skip the lane and - PIPE work. The tests need no hardware and run when the driver - is loaded. + map to PHY modes, which mux updates may skip the lane and PIPE + work, and that a port behind a fixed hub has the controller's + SuperSpeed port off while its PIPE moves. The tests need no + hardware and run when the driver is loaded. If unsure, say N. diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index d3a760a7d62fd..6cc291bc052ce 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -5,6 +5,7 @@ */ #include +#include #include static struct apple_atcphy *atcphy_test_alloc(struct kunit *test, bool fixed_usb2) @@ -91,6 +92,13 @@ static void atcphy_test_noop_same_mode(struct kunit *test) atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; atcphy->pipe_pending = true; KUNIT_EXPECT_TRUE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); + + /* A mode without USB3 leaves the SuperSpeed port off */ + atcphy = atcphy_test_alloc(test, true); + atcphy->mode = APPLE_ATCPHY_MODE_DP; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + atcphy->usb3_port_off = true; + KUNIT_EXPECT_TRUE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_DP)); } static void atcphy_test_noop_needs_work(struct kunit *test) @@ -114,6 +122,39 @@ static void atcphy_test_noop_needs_work(struct kunit *test) atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; atcphy->pipe_pending = true; KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); + atcphy->pipe_pending = false; + + /* or its SuperSpeed port, still off since the last handoff */ + atcphy->usb3_port_off = true; + KUNIT_EXPECT_FALSE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); +} + +/* The controller's SuperSpeed port only runs on a completed USB3 route */ +static void atcphy_test_usb3_port_ready(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc(test, true); + + KUNIT_EXPECT_TRUE(test, atcphy_usb3_port_ready(atcphy)); + + atcphy->mode_dirty = true; + KUNIT_EXPECT_FALSE(test, atcphy_usb3_port_ready(atcphy)); + atcphy->mode_dirty = false; + + atcphy->pipe_pending = true; + KUNIT_EXPECT_FALSE(test, atcphy_usb3_port_ready(atcphy)); + atcphy->pipe_pending = false; + + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + KUNIT_EXPECT_FALSE(test, atcphy_usb3_port_ready(atcphy)); + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; + + atcphy->host_active = false; + KUNIT_EXPECT_FALSE(test, atcphy_usb3_port_ready(atcphy)); + + /* A Type-C port's controller is never handled here */ + atcphy = atcphy_test_alloc(test, false); + atcphy->host_active = true; + KUNIT_EXPECT_FALSE(test, atcphy_usb3_port_ready(atcphy)); } /* @@ -127,6 +168,9 @@ static struct apple_atcphy *atcphy_test_alloc_fixed_hub(struct kunit *test) struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, true); atcphy->hw = &atcphy_hw_t8140; + /* The lane and DisplayPort tunables a J700 has */ + atcphy->ss_tunables = true; + atcphy->dp_tunables = true; mutex_init(&atcphy->lock); return atcphy; @@ -182,6 +226,328 @@ static void atcphy_test_update_takes_usb(struct kunit *test) KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -ENODEV); } +/* + * The steps of a mux update on a fixed-hub port, recorded by static stubs + * for the USB side and for the locked mode change: g(et the root hub), + * 0 (SuperSpeed port off), m(ode change) and 1 (SuperSpeed port on). + */ +struct atcphy_test_steps { + char log[16]; + int pm_get_ret; + int port_off_ret; + int port_on_ret; + int mode_ret; + bool mode_port_on; +}; + +static struct atcphy_test_steps *atcphy_test_steps(void) +{ + return kunit_get_current_test()->priv; +} + +static void atcphy_test_step(char step) +{ + struct atcphy_test_steps *steps = atcphy_test_steps(); + size_t len = strlen(steps->log); + + if (len < sizeof(steps->log) - 1) + steps->log[len] = step; +} + +static int atcphy_test_pm_get(struct apple_atcphy *atcphy, struct atcphy_usb3_pm *pm) +{ + atcphy_test_step('g'); + return atcphy_test_steps()->pm_get_ret; +} + +static int atcphy_test_port_off(struct apple_atcphy *atcphy, struct atcphy_usb3_pm *pm) +{ + atcphy_test_step('0'); + return atcphy_test_steps()->port_off_ret; +} + +static int atcphy_test_port_on(struct apple_atcphy *atcphy, struct atcphy_usb3_pm *pm) +{ + atcphy_test_step('1'); + return atcphy_test_steps()->port_on_ret; +} + +static int atcphy_test_mode(struct apple_atcphy *atcphy, enum atcphy_mode target_mode, + bool *port_on) +{ + atcphy_test_step('m'); + *port_on = atcphy_test_steps()->mode_port_on; + return atcphy_test_steps()->mode_ret; +} + +static struct atcphy_test_steps *atcphy_test_stub_steps(struct kunit *test) +{ + struct atcphy_test_steps *steps = kunit_kzalloc(test, sizeof(*steps), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, steps); + test->priv = steps; + kunit_activate_static_stub(test, atcphy_usb3_pm_get, atcphy_test_pm_get); + kunit_activate_static_stub(test, atcphy_usb3_port_off, atcphy_test_port_off); + kunit_activate_static_stub(test, atcphy_usb3_port_on, atcphy_test_port_on); + kunit_activate_static_stub(test, atcphy_mux_set_locked, atcphy_test_mode); + + return steps; +} + +/* The SuperSpeed port goes down before the PHY changes and comes back after it */ +static void atcphy_test_update_port_order(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + + /* A new pin assignment with USB3 routed again */ + steps->mode_port_on = true; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + + /* An update keeping the mode touches neither */ + memset(steps->log, 0, sizeof(steps->log)); + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, ""); + + /* A mode without USB3: the port stays down */ + steps->mode_port_on = false; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m"); + KUNIT_EXPECT_TRUE(test, atcphy->usb3_port_off); +} + +/* No PIPE or lane work unless the port went down, and no port up after a failure */ +static void atcphy_test_update_port_failure(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + + steps->port_off_ret = -EPIPE; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -EPIPE); + KUNIT_EXPECT_STREQ(test, steps->log, "g0"); + + memset(steps->log, 0, sizeof(steps->log)); + steps->port_off_ret = 0; + steps->mode_ret = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -ETIMEDOUT); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m"); + + /* A change refused before it touched the USB3 route brings the port back */ + memset(steps->log, 0, sizeof(steps->log)); + steps->mode_ret = -EOPNOTSUPP; + steps->mode_port_on = true; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -EOPNOTSUPP); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1"); + + /* and if that fails, the Type-C core must not take the refusal for success */ + memset(steps->log, 0, sizeof(steps->log)); + steps->port_on_ret = -ENOMEM; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -ENOMEM); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1"); + steps->port_on_ret = 0; + + /* A hub that cannot switch the port is no success the Type-C core skips */ + memset(steps->log, 0, sizeof(steps->log)); + steps->port_off_ret = -EOPNOTSUPP; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -ENODEV); + KUNIT_EXPECT_STREQ(test, steps->log, "g0"); +} + +/* A port owed only its power is brought up, and USB alone never fails the display */ +static void atcphy_test_update_port_owed(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + + atcphy->usb3_port_off = true; + + /* An update keeping the pin assignment */ + steps->mode_port_on = true; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + atcphy->usb3_port_off = true; + + /* The same, with the root hub gone */ + memset(steps->log, 0, sizeof(steps->log)); + steps->pm_get_ret = -ENODEV; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g"); + + /* or failing to take the port down */ + memset(steps->log, 0, sizeof(steps->log)); + steps->pm_get_ret = 0; + steps->port_off_ret = -EPIPE; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g0"); + + /* A new pin assignment still needs the USB side */ + memset(steps->log, 0, sizeof(steps->log)); + steps->pm_get_ret = -ENODEV; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -ENODEV); + KUNIT_EXPECT_STREQ(test, steps->log, "g"); + + /* A failed mode change is no USB failure */ + memset(steps->log, 0, sizeof(steps->log)); + steps->pm_get_ret = 0; + steps->port_off_ret = 0; + steps->mode_ret = -EIO; + steps->mode_port_on = false; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -EIO); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m"); +} + +/* A mode the PHY cannot run is turned down before USB3 goes down for it */ +static void atcphy_test_update_unsupported(struct kunit *test) +{ + struct typec_altmode tbt = { .svid = USB_TYPEC_TBT_SID }; + struct typec_mux_state state = { .alt = &tbt, .mode = TYPEC_STATE_MODAL }; + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + + /* Thunderbolt, which this PHY does not have */ + KUNIT_EXPECT_EQ(test, atcphy_mux_update(atcphy, &state), -EOPNOTSUPP); + KUNIT_EXPECT_STREQ(test, steps->log, ""); + + /* DisplayPort without its tunables */ + atcphy->dp_tunables = false; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -EOPNOTSUPP); + KUNIT_EXPECT_STREQ(test, steps->log, ""); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); +} + +/* The locked mode change as it reports the port: up once the PHY took it down */ +static int atcphy_test_mode_ready(struct apple_atcphy *atcphy, enum atcphy_mode target_mode, + bool *port_on) +{ + atcphy_test_step('m'); + *port_on = atcphy->usb3_port_off && atcphy_test_steps()->mode_port_on; + return atcphy_test_steps()->mode_ret; +} + +/* A port that userspace switched off while the PHY had it down stays off */ +static void atcphy_test_update_port_taken(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + + kunit_activate_static_stub(test, atcphy_mux_set_locked, atcphy_test_mode_ready); + steps->mode_port_on = true; + + /* Unplugged, with the port down for the PHY, then disabled through sysfs */ + atcphy->mode = APPLE_ATCPHY_MODE_USB2; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + atcphy->usb3_port_off = true; + + /* The replug finds the port off for someone else, and leaves it to them */ + steps->port_off_ret = 1; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + + /* Disabled while the PHY moved the lanes: nothing is owed afterwards */ + memset(steps->log, 0, sizeof(steps->log)); + steps->port_off_ret = 0; + steps->port_on_ret = 1; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); +} + +/* + * Zeroed memory in place of the core, PIPE handler and USB2 PHY registers: + * enough to park the PIPE and power the PHY off, with a PIPE lock request + * that is never acknowledged. + */ +static void atcphy_test_fake_regs(struct kunit *test, struct apple_atcphy *atcphy) +{ + void *core = kunit_kzalloc(test, ATCPHY_MISC + 4, GFP_KERNEL); + void *pipehandler = kunit_kzalloc(test, PIPEHANDLER_NONSELECTED_OVERRIDE + 4, GFP_KERNEL); + void *usb2phy = kunit_kzalloc(test, USB2PHY_MISCTUNE + 4, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, core); + KUNIT_ASSERT_NOT_NULL(test, pipehandler); + KUNIT_ASSERT_NOT_NULL(test, usb2phy); + atcphy->regs.core = (__force void __iomem *)core; + atcphy->regs.pipehandler = (__force void __iomem *)pipehandler; + atcphy->regs.usb2phy = (__force void __iomem *)usb2phy; +} + +/* The locked mode change reports a port that can come up again */ +static void atcphy_test_set_port_on(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + bool port_on = true; + + /* The port was not taken down */ + KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_USB3_DP, &port_on), + 0); + KUNIT_EXPECT_FALSE(test, port_on); + + /* The port is down and the PIPE already on the USB3 backend */ + atcphy->usb3_port_off = true; + KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_USB3_DP, &port_on), + 0); + KUNIT_EXPECT_TRUE(test, port_on); + + /* A change that failed, here an unplug whose PIPE lock was refused */ + atcphy_test_fake_regs(test, atcphy); + KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_USB2, &port_on), + -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, atcphy->mode_dirty); + KUNIT_EXPECT_FALSE(test, port_on); +} + +/* A host that starts up powers its SuperSpeed port on itself */ +static void atcphy_test_host_start(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct phy *phy = kunit_kzalloc(test, sizeof(*phy), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, phy); + phy_set_drvdata(phy, atcphy); + atcphy->usb3_port_off = true; + + KUNIT_EXPECT_EQ(test, atcphy_usb3_set_mode(phy, PHY_MODE_USB_HOST, 0), 0); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_TRUE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); +} + +#if IS_REACHABLE(CONFIG_USB) +/* Only the controller's SuperSpeed root hub is ever handed the port changes */ +static void atcphy_test_usb3_root(struct kunit *test) +{ + struct device *controller = kunit_kzalloc(test, 2 * sizeof(*controller), GFP_KERNEL); + struct usb_bus *bus = kunit_kzalloc(test, 2 * sizeof(*bus), GFP_KERNEL); + struct usb_device *udev = kunit_kzalloc(test, 2 * sizeof(*udev), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, controller); + KUNIT_ASSERT_NOT_NULL(test, bus); + KUNIT_ASSERT_NOT_NULL(test, udev); + bus[0].sysdev = &controller[0]; + bus[1].sysdev = &controller[1]; + + udev[0].bus = &bus[0]; + udev[0].speed = USB_SPEED_SUPER_PLUS; + KUNIT_EXPECT_TRUE(test, atcphy_usb3_is_root(&udev[0], &controller[0])); + + /* The USB2 root hub, with the fixed hub on it */ + udev[1].bus = &bus[0]; + udev[1].speed = USB_SPEED_HIGH; + KUNIT_EXPECT_FALSE(test, atcphy_usb3_is_root(&udev[1], &controller[0])); + + /* A device on the SuperSpeed root port */ + udev[1].speed = USB_SPEED_SUPER; + udev[1].parent = &udev[0]; + KUNIT_EXPECT_FALSE(test, atcphy_usb3_is_root(&udev[1], &controller[0])); + + /* The SuperSpeed root hub of another controller */ + KUNIT_EXPECT_FALSE(test, atcphy_usb3_is_root(&udev[0], &controller[1])); +} +#endif + /* USB shutdown can turn the lanes off before the SAFE mux request arrives. */ static struct apple_atcphy *atcphy_test_alloc_powered_off(struct kunit *test) { @@ -210,8 +576,9 @@ static void atcphy_test_safe_after_poweroff(struct kunit *test) static void atcphy_test_locked_safe_after_poweroff(struct kunit *test) { struct apple_atcphy *atcphy = atcphy_test_alloc_powered_off(test); + bool port_on; - KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_OFF), 0); + KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_OFF, &port_on), 0); KUNIT_EXPECT_EQ(test, atcphy->typec_mode, APPLE_ATCPHY_MODE_OFF); } @@ -240,13 +607,15 @@ static void atcphy_test_safe_live_pipe(struct kunit *test) { struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, false); struct atcphy_test_configure config = {}; + bool port_on; test->priv = &config; kunit_activate_static_stub(test, atcphy_configure, atcphy_test_configure_stub); atcphy->hw = &atcphy_hw_t8103; atcphy->typec_mode = APPLE_ATCPHY_MODE_USB3; mutex_init(&atcphy->lock); - KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_OFF), -EBUSY); + KUNIT_EXPECT_EQ(test, atcphy_mux_set_locked(atcphy, APPLE_ATCPHY_MODE_OFF, &port_on), + -EBUSY); KUNIT_EXPECT_EQ(test, config.calls, 0U); KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB3_DP); KUNIT_EXPECT_EQ(test, atcphy->typec_mode, APPLE_ATCPHY_MODE_USB3); @@ -278,8 +647,19 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_target_mode_fixed_hub), KUNIT_CASE(atcphy_test_noop_same_mode), KUNIT_CASE(atcphy_test_noop_needs_work), + KUNIT_CASE(atcphy_test_usb3_port_ready), KUNIT_CASE(atcphy_test_update_skips_usb), KUNIT_CASE(atcphy_test_update_takes_usb), + KUNIT_CASE(atcphy_test_update_port_order), + KUNIT_CASE(atcphy_test_update_port_failure), + KUNIT_CASE(atcphy_test_update_port_owed), + KUNIT_CASE(atcphy_test_update_unsupported), + KUNIT_CASE(atcphy_test_update_port_taken), + KUNIT_CASE(atcphy_test_set_port_on), + KUNIT_CASE(atcphy_test_host_start), +#if IS_REACHABLE(CONFIG_USB) + KUNIT_CASE(atcphy_test_usb3_root), +#endif KUNIT_CASE(atcphy_test_safe_after_poweroff), KUNIT_CASE(atcphy_test_locked_safe_after_poweroff), KUNIT_CASE(atcphy_test_safe_live_pipe), diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 4da347276a231..0d720e4707b68 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -22,9 +22,8 @@ * Author: Sven Peter */ -#include - #include +#include #include #include #include @@ -678,6 +677,8 @@ struct atcphy_hw { * it when the PHY is brought back up after a power-off * @host_active: dwc3 has selected host mode on the USB3 PHY; a fixed-hub port * then routes the PIPE itself on later Type-C mode changes + * @usb3_port_off: The SuperSpeed root port of a fixed-hub controller was powered + * off for a PIPE or lane change and is not powered on again yet * @mode: Current PHY operating mode * @swap_lanes: True if lanes must be swapped due to cable orientation * @dp_link_rate: DisplayPort link rate @@ -743,6 +744,7 @@ struct apple_atcphy { bool fixed_usb2; enum atcphy_mode typec_mode; bool host_active; + bool usb3_port_off; enum atcphy_mode mode; int dp_link_rate; bool swap_lanes; @@ -3043,6 +3045,8 @@ static int atcphy_usb3_power_off(struct phy *phy) return ret; } atcphy->host_active = false; + /* The next host powers its SuperSpeed port on as it starts. */ + atcphy->usb3_port_off = false; return 0; } @@ -3059,6 +3063,8 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode switch (mode) { case PHY_MODE_USB_HOST: atcphy->host_active = true; + /* A host that starts up powers the ports of its root hubs itself. */ + atcphy->usb3_port_off = false; break; case PHY_MODE_USB_DEVICE: atcphy->host_active = false; @@ -3314,6 +3320,7 @@ static int atcphy_dwc3_reset_assert(struct reset_controller_dev *rcdev, unsigned _atcphy_dwc3_reset_assert(atcphy); atcphy->host_active = false; + atcphy->usb3_port_off = false; atcphy_usb2_power_off(atcphy); return 0; @@ -3412,71 +3419,31 @@ static int atcphy_probe_switch(struct apple_atcphy *atcphy) } /* - * The fixed USB2 hub keeps the host controller active, but its independent - * SuperSpeed root hub may autosuspend while PIPE still needs it awake. - * Own a bounded root-hub resume reference across a Type-C handoff. Resume - * precedes the ATC lock: USB PM can call back into the PHY on recovery. + * The controller of a fixed-hub port stays up across Type-C mode changes, so + * the PHY moves the PIPE while its SuperSpeed root port is powered off, see + * atcphy_mux_update(). Hold the controller, its xHCI device and the awake + * SuperSpeed root hub across the change: hub_wq then cannot use the port in + * between, and none of them can go away. They are taken before the ATC lock, + * as USB PM can call back into the PHY on recovery. */ struct atcphy_usb3_pm { struct platform_device *controller; struct device *host; struct usb_device *root; - struct usb_device *link; }; #if IS_REACHABLE(CONFIG_USB) -/* The root's runtime state does not cover selective suspend of its port. */ -static int atcphy_usb3_link_get(struct atcphy_usb3_pm *pm) -{ - struct usb_device *link; - int ret; - - /* The graph identifies the controller, not a port in a multiport root. */ - if (pm->root->maxchild != 1) - return pm->root->maxchild ? -ENODEV : -EAGAIN; - link = usb_hub_find_child(pm->root, 1); - if (!link) - return 0; - pm->link = usb_get_dev(link); - usb_lock_device(pm->link); - if (READ_ONCE(pm->link->state) == USB_STATE_NOTATTACHED) { - ret = 0; - goto unlock_link; - } - ret = pm_runtime_resume_and_get(&pm->link->dev); - if (ret < 0) - goto unlock_link; - if (READ_ONCE(pm->link->state) == USB_STATE_NOTATTACHED) { - pm_runtime_put_noidle(&pm->link->dev); - ret = -ENODEV; - goto unlock_link; - } - return 0; - -unlock_link: - usb_unlock_device(pm->link); - usb_put_dev(pm->link); - pm->link = NULL; - return ret; -} - -static void atcphy_usb3_link_put(struct atcphy_usb3_pm *pm) +/* The SuperSpeed root hub of the controller; its USB2 root hub never matches */ +static bool atcphy_usb3_is_root(const struct usb_device *udev, const struct device *controller) { - if (!pm->link) - return; - usb_mark_last_busy(pm->link); - pm_runtime_put_autosuspend(&pm->link->dev); - usb_unlock_device(pm->link); - usb_put_dev(pm->link); - pm->link = NULL; + return !udev->parent && udev->speed >= USB_SPEED_SUPER && udev->bus->sysdev == controller; } static int atcphy_usb3_pm_find(struct usb_device *udev, void *data) { struct atcphy_usb3_pm *pm = data; - if (udev->parent || udev->speed < USB_SPEED_SUPER || - udev->bus->sysdev != &pm->controller->dev) + if (!atcphy_usb3_is_root(udev, &pm->controller->dev)) return 0; pm->root = usb_get_dev(udev); @@ -3493,6 +3460,8 @@ static int atcphy_usb3_pm_get(struct apple_atcphy *atcphy, struct atcphy_usb3_pm int ret; #endif + KUNIT_STATIC_STUB_REDIRECT(atcphy_usb3_pm_get, atcphy, pm); + if (!atcphy->fixed_usb2 || !atcphy->hw->has_t8140_dp) return 0; @@ -3543,26 +3512,17 @@ static int atcphy_usb3_pm_get(struct apple_atcphy *atcphy, struct atcphy_usb3_pm ret = -ENODEV; goto unlock_root; } - ret = pm_runtime_resume_and_get(&pm->root->dev); - if (ret < 0) - goto unlock_root; - if (READ_ONCE(pm->root->state) == USB_STATE_NOTATTACHED || - HCD_DEAD(hcd) || !HCD_HW_ACCESSIBLE(hcd)) { - pm_runtime_put_noidle(&pm->root->dev); - ret = -ENODEV; + /* The graph identifies the controller, not a port in a multiport root. */ + if (pm->root->maxchild != 1) { + ret = pm->root->maxchild ? -ENODEV : -EAGAIN; goto unlock_root; } - /* Resume the attached upstream link before taking the ATC mutex. */ - ret = atcphy_usb3_link_get(pm); - if (ret) { - usb_mark_last_busy(pm->root); - pm_runtime_put_autosuspend(&pm->root->dev); + ret = pm_runtime_resume_and_get(&pm->root->dev); + if (ret < 0) goto unlock_root; - } if (READ_ONCE(pm->root->state) == USB_STATE_NOTATTACHED || READ_ONCE(hcd->state) != HC_STATE_RUNNING || HCD_DEAD(hcd) || !HCD_HW_ACCESSIBLE(hcd)) { - atcphy_usb3_link_put(pm); pm_runtime_put_noidle(&pm->root->dev); ret = -ENODEV; goto unlock_root; @@ -3594,9 +3554,6 @@ static void atcphy_usb3_pm_put(struct atcphy_usb3_pm *pm) if (!pm->root) return; - /* Release child ownership before the root: keep the lock order hierarchical. */ - atcphy_usb3_link_put(pm); - /* Restore normal runtime policy after both success and failed parking. */ usb_mark_last_busy(pm->root); pm_runtime_put_autosuspend(&pm->root->dev); @@ -3612,6 +3569,64 @@ static void atcphy_usb3_pm_put(struct atcphy_usb3_pm *pm) #endif } +/* + * Whether the controller of a fixed-hub port may run its SuperSpeed port: the + * PIPE is on the USB3 backend for its host, and no lane or PIPE change is left + * unfinished. + */ +static bool atcphy_usb3_port_ready(const struct apple_atcphy *atcphy) +{ + return atcphy->fixed_usb2 && atcphy->host_active && !atcphy->mode_dirty && + !atcphy->pipe_pending && atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3; +} + +/* + * Take the controller's SuperSpeed port down before the PIPE moves: the device + * on it is disconnected and the port held powered off, so that nothing drives + * the PIPE when it is handed over. Nothing else of the controller stops, its + * USB2 root port and the fixed hub on it included. Returns 1 when there is no + * such port to take down, and for a port that someone else switched off: that + * stays theirs and is not powered on again. + */ +static int atcphy_usb3_port_off(struct apple_atcphy *atcphy, struct atcphy_usb3_pm *pm) +{ + int ret = 1; + + KUNIT_STATIC_STUB_REDIRECT(atcphy_usb3_port_off, atcphy, pm); + +#if IS_REACHABLE(CONFIG_USB) + if (!pm->root) + return 1; + + ret = usb_hub_port_power_off(pm->root, 1); + if (ret < 0) + dev_warn(atcphy->dev, "Failed to power the SuperSpeed port off: %d\n", ret); +#endif + return ret; +} + +/* + * Bring the SuperSpeed port up again once the PIPE is back on the USB3 backend. + * Returns 1 for a port that someone else has taken over since it went down, + * through its "disable" attribute: it stays off. + */ +static int atcphy_usb3_port_on(struct apple_atcphy *atcphy, struct atcphy_usb3_pm *pm) +{ + int ret = 1; + + KUNIT_STATIC_STUB_REDIRECT(atcphy_usb3_port_on, atcphy, pm); + +#if IS_REACHABLE(CONFIG_USB) + if (!pm->root) + return 1; + + ret = usb_hub_port_power_on(pm->root, 1); + if (ret < 0) + dev_warn(atcphy->dev, "Failed to power the SuperSpeed port on: %d\n", ret); +#endif + return ret; +} + /* The PHY mode a Type-C mux state asks for; unsupported states turn the PHY off. */ static enum atcphy_mode atcphy_typec_target_mode(struct apple_atcphy *atcphy, const struct typec_mux_state *state) @@ -3681,12 +3696,11 @@ static enum atcphy_mode atcphy_typec_target_mode(struct apple_atcphy *atcphy, } /* - * Whether a mux update leaves both the lanes and the PIPE as they are. That is - * the case when the mode does not change and no earlier PIPE handoff to a - * running controller behind a fixed hub is still outstanding. A DisplayPort - * HPD or status update with an unchanged pin assignment is one of these. + * Whether the lanes and the PIPE are as @target_mode needs them: the mode does + * not change, and no earlier PIPE handoff to a running controller behind a + * fixed hub is still outstanding. */ -static bool atcphy_mux_is_noop(const struct apple_atcphy *atcphy, enum atcphy_mode target_mode) +static bool atcphy_mux_in_mode(const struct apple_atcphy *atcphy, enum atcphy_mode target_mode) { if (atcphy->mode != target_mode || atcphy->mode_dirty) return false; @@ -3696,11 +3710,49 @@ static bool atcphy_mux_is_noop(const struct apple_atcphy *atcphy, enum atcphy_mo atcphy->pipe_pending)); } -static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) +/* + * Whether a mux update leaves both the lanes and the PIPE as they are, and + * owes no SuperSpeed port its power either. A DisplayPort HPD or status + * update with an unchanged pin assignment is one of these. + */ +static bool atcphy_mux_is_noop(const struct apple_atcphy *atcphy, enum atcphy_mode target_mode) +{ + return atcphy_mux_in_mode(atcphy, target_mode) && + !(atcphy->usb3_port_off && atcphy_usb3_port_ready(atcphy)); +} + +/* Whether the PHY can run @mode at all, which needs no lane or PIPE work to tell */ +static int atcphy_mux_check(const struct apple_atcphy *atcphy, enum atcphy_mode mode) +{ + switch (mode) { + case APPLE_ATCPHY_MODE_TBT: + case APPLE_ATCPHY_MODE_USB4: + if (!atcphy->hw->has_usb4) + return -EOPNOTSUPP; + fallthrough; + case APPLE_ATCPHY_MODE_USB3: + case APPLE_ATCPHY_MODE_USB3_DP: + case APPLE_ATCPHY_MODE_DP: + if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && + !atcphy->dp_tunables) + return -EOPNOTSUPP; + /* Without the lane tunables the SuperSpeed lanes are not calibrated */ + if (!atcphy->ss_tunables) + return -EOPNOTSUPP; + break; + case APPLE_ATCPHY_MODE_OFF: + case APPLE_ATCPHY_MODE_USB2: + break; + } + + return 0; +} + +static int atcphy_mux_switch(struct apple_atcphy *atcphy, enum atcphy_mode target_mode) { int ret; - guard(mutex)(&atcphy->lock); + lockdep_assert_held(&atcphy->lock); dev_dbg(atcphy->dev, "mux target=%u current=%u dirty=%u\n", target_mode, atcphy->mode, atcphy->mode_dirty); @@ -3725,26 +3777,9 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) return -EBUSY; - switch (target_mode) { - case APPLE_ATCPHY_MODE_TBT: - case APPLE_ATCPHY_MODE_USB4: - if (!atcphy->hw->has_usb4) - return -EOPNOTSUPP; - fallthrough; - case APPLE_ATCPHY_MODE_USB3: - case APPLE_ATCPHY_MODE_USB3_DP: - case APPLE_ATCPHY_MODE_DP: - if (atcphy->hw->has_t8140_dp && atcphy_modes[target_mode].enable_dp_aux && - !atcphy->dp_tunables) - return -EOPNOTSUPP; - /* Without the lane tunables the SuperSpeed lanes are not calibrated */ - if (!atcphy->ss_tunables) - return -EOPNOTSUPP; - break; - case APPLE_ATCPHY_MODE_OFF: - case APPLE_ATCPHY_MODE_USB2: - break; - } + ret = atcphy_mux_check(atcphy, target_mode); + if (ret) + return ret; atcphy->typec_mode = target_mode; if (atcphy->fixed_usb2) { @@ -3757,6 +3792,8 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t * from a later set_mode call. Even another USB3 mode can move the * backend to the other PMA when orientation changes. Park before * rebuilding the lanes, then reconnect while dwc3 is in host mode. + * The controller's SuperSpeed port is off by now, see + * atcphy_mux_update(). */ ret = atcphy_park_pipehandler(atcphy); if (ret) @@ -3789,10 +3826,28 @@ static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode t return atcphy_configure(atcphy, target_mode); } +/* + * Change the mode under the PHY lock. @port_on tells whether the SuperSpeed + * port of a fixed-hub controller, taken down for the change, can come up again. + */ +static int atcphy_mux_set_locked(struct apple_atcphy *atcphy, enum atcphy_mode target_mode, + bool *port_on) +{ + int ret; + + KUNIT_STATIC_STUB_REDIRECT(atcphy_mux_set_locked, atcphy, target_mode, port_on); + + guard(mutex)(&atcphy->lock); + ret = atcphy_mux_switch(atcphy, target_mode); + *port_on = atcphy->usb3_port_off && atcphy_usb3_port_ready(atcphy); + return ret; +} + static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux_state *state) { struct atcphy_usb3_pm pm = {}; enum atcphy_mode target_mode; + bool in_mode, port_on, usb_err = true; int ret; scoped_guard(mutex, &atcphy->lock) { @@ -3801,23 +3856,73 @@ static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux atcphy->fixed_usb2, atcphy->dp_tunables, atcphy->ss_tunables); target_mode = atcphy_typec_target_mode(atcphy, state); /* - * Only a lane or PIPE change needs the USB3 root hub below. - * Taking it means waiting for the hub's ongoing work, which - * includes a whole enumeration with its timeouts, and would - * hold up a DisplayPort update that changes neither. + * Only a lane or PIPE change, or a SuperSpeed port still owed + * its power, needs the USB3 root hub below. Taking it means + * waiting for the hub's ongoing work, which includes a whole + * enumeration with its timeouts, and would hold up a + * DisplayPort update that changes neither. */ if (atcphy_mux_is_noop(atcphy, target_mode)) { /* Record matching requests for the next PHY init to restore. */ atcphy->typec_mode = target_mode; return 0; } + /* Turn down a mode the PHY cannot run before USB3 goes down for it. */ + ret = atcphy_mux_check(atcphy, target_mode); + if (ret) + return ret; + in_mode = atcphy_mux_in_mode(atcphy, target_mode); } ret = atcphy_usb3_pm_get(atcphy, &pm); if (ret) - return ret; - ret = atcphy_mux_set_locked(atcphy, target_mode); + goto out; + + /* + * The order in which dwc3 is taken down and brought up around a PHY + * change (see dwc3-apple.c), for the SuperSpeed side only: the + * controller's SuperSpeed port goes down before the PIPE is parked and + * the lanes are rebuilt, and comes up only once the PIPE is on the USB3 + * backend again. It stays down after a failed change, and in a mode + * without USB3. + */ + ret = atcphy_usb3_port_off(atcphy, &pm); + if (ret < 0) { + /* typec_mux_set() skips EOPNOTSUPP, which would report a false success. */ + if (ret == -EOPNOTSUPP) + ret = -ENODEV; + goto put; + } + /* A port that someone else switched off is no longer the PHY's to restore. */ + scoped_guard(mutex, &atcphy->lock) + atcphy->usb3_port_off = !ret; + + ret = atcphy_mux_set_locked(atcphy, target_mode, &port_on); + usb_err = false; + if (port_on) { + int err = atcphy_usb3_port_on(atcphy, &pm); + + if (err >= 0) { + scoped_guard(mutex, &atcphy->lock) + atcphy->usb3_port_off = false; + } else if (!ret || ret == -EOPNOTSUPP) { + /* A port left off must not hide behind a code typec_mux_set() skips */ + ret = err; + usb_err = true; + } + } +put: atcphy_usb3_pm_put(&pm); +out: + /* + * With the lanes and the PIPE already as the mode needs them, only + * USB3 is missing, and that must not fail what may be a DisplayPort + * update. The port stays off until the next update. + */ + if (ret && usb_err && in_mode) { + dev_dbg(atcphy->dev, "SuperSpeed port left off: %d\n", ret); + return 0; + } return ret; } From 50ff54154f2cfe87131d22b9c7444cea91bc389a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 05:51:41 +1100 Subject: [PATCH 1219/1579] phy: apple: atc: Park the PIPE of a stopped fixed-hub controller A PIPE handoff that did not complete leaves pipe_pending set, and the dummy handoff then insists on the PIPE lock being acknowledged before it moves the PIPE, as it must while a host is running. dwc3 powers the USB3 PHY off and asserts its reset only after its xHCI has gone, though. On a port behind a fixed hub a missing acknowledgment then fails both; the glue keeps the power-off and the reset pending, and its next start retries the same park before anything else. Unbinding and binding dwc3-apple, the way to restart a fixed-hub controller by hand, could then leave the board without USB until a reboot. Clear host_active as the USB3 PHY is powered off or the reset asserted, before the PIPE is parked, and treat the PIPE of a fixed-hub controller as driven only while that controller is in host mode, pending handoff or not. A stopped controller's park then warns about the missing acknowledgment and goes on, as the park does on other ports once no handoff is pending. A running host still has to acknowledge. Ports without a fixed hub keep their check. The new tests run the power-off and the reset on registers that never acknowledge; both failed with -ETIMEDOUT before. Fixes: f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 53 ++++++++++++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 37 +++++++++++++++++++------ 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index 6cc291bc052ce..994f758da0680 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -515,6 +515,57 @@ static void atcphy_test_host_start(struct kunit *test) KUNIT_EXPECT_TRUE(test, atcphy_mux_is_noop(atcphy, APPLE_ATCPHY_MODE_USB3_DP)); } +/* A fixed-hub PHY whose USB3 handoff failed while the host was up */ +static struct apple_atcphy *atcphy_test_alloc_failed_handoff(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + + atcphy_test_fake_regs(test, atcphy); + atcphy->pipe_pending = true; + atcphy->usb3_port_off = true; + + /* The running host has to acknowledge the PIPE lock */ + scoped_guard(mutex, &atcphy->lock) + KUNIT_EXPECT_EQ(test, atcphy_park_pipehandler(atcphy), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_USB3); + KUNIT_EXPECT_TRUE(test, atcphy->pipe_pending); + + return atcphy; +} + +/* Once dwc3 has stopped, its USB3 PHY power-off parks the PIPE without one */ +static void atcphy_test_stopped_power_off(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_failed_handoff(test); + struct phy *phy = kunit_kzalloc(test, sizeof(*phy), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, phy); + phy_set_drvdata(phy, atcphy); + + KUNIT_EXPECT_EQ(test, atcphy_usb3_power_off(phy), 0); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_OFF); + KUNIT_EXPECT_FALSE(test, atcphy->host_active); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); +} + +/* and so does the reset that dwc3 asserts once it has stopped */ +static void atcphy_test_stopped_reset(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_failed_handoff(test); + u32 aon; + + KUNIT_EXPECT_EQ(test, atcphy_dwc3_reset_assert(&atcphy->rcdev, 0), 0); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); + KUNIT_EXPECT_FALSE(test, atcphy->host_active); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + aon = readl(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN); + KUNIT_EXPECT_TRUE(test, aon & PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + KUNIT_EXPECT_FALSE(test, aon & PIPEHANDLER_AON_GEN_DWC3_RESET_N); +} + #if IS_REACHABLE(CONFIG_USB) /* Only the controller's SuperSpeed root hub is ever handed the port changes */ static void atcphy_test_usb3_root(struct kunit *test) @@ -657,6 +708,8 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_update_port_taken), KUNIT_CASE(atcphy_test_set_port_on), KUNIT_CASE(atcphy_test_host_start), + KUNIT_CASE(atcphy_test_stopped_power_off), + KUNIT_CASE(atcphy_test_stopped_reset), #if IS_REACHABLE(CONFIG_USB) KUNIT_CASE(atcphy_test_usb3_root), #endif diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 0d720e4707b68..366bef334b970 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1462,11 +1462,26 @@ static void atcphy_enable_dummy_phy(struct apple_atcphy *atcphy) PIPEHANDLER_DUMMY_PHY_EN); } +/* + * Whether a controller may still be driving the PIPE, which then has to + * acknowledge the lock before the PIPE moves. A handoff that did not complete + * leaves the PIPE in an unknown state. A controller behind a fixed hub only + * drives it in host mode, which it has left by the time it powers the USB3 PHY + * off or asserts its reset. + */ +static bool atcphy_pipe_live(const struct apple_atcphy *atcphy) +{ + if (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2) + return atcphy->host_active && + (atcphy->pipe_pending || + atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3); + + return atcphy->pipe_pending; +} + static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy) { - bool live_usb3 = atcphy->pipe_pending || - (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2 && atcphy->host_active && - atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3); + bool live_usb3 = atcphy_pipe_live(atcphy); int ret; ret = atcphy_pipehandler_check(atcphy); @@ -3036,6 +3051,13 @@ static int atcphy_usb3_power_off(struct phy *phy) guard(mutex)(&atcphy->lock); + /* + * dwc3 has left host mode: nothing drives the PIPE any more, and the + * next host powers its SuperSpeed port on as it starts. + */ + atcphy->host_active = false; + atcphy->usb3_port_off = false; + ret = atcphy_park_pipehandler(atcphy); if (ret) return ret; @@ -3044,9 +3066,6 @@ static int atcphy_usb3_power_off(struct phy *phy) if (ret) return ret; } - atcphy->host_active = false; - /* The next host powers its SuperSpeed port on as it starts. */ - atcphy->usb3_port_off = false; return 0; } @@ -3313,14 +3332,16 @@ static int atcphy_dwc3_reset_assert(struct reset_controller_dev *rcdev, unsigned guard(mutex)(&atcphy->lock); + /* The controller is only reset once it has stopped. */ + atcphy->host_active = false; + atcphy->usb3_port_off = false; + /* Do not reset the controller or USB2 PHY if PIPE parking fails. */ ret = atcphy_park_pipehandler(atcphy); if (ret) return ret; _atcphy_dwc3_reset_assert(atcphy); - atcphy->host_active = false; - atcphy->usb3_port_off = false; atcphy_usb2_power_off(atcphy); return 0; From 29dbc02ce89c73e39faebd969c3eca585226e795 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 05:51:41 +1100 Subject: [PATCH 1220/1579] phy: apple: atc: Don't let USB3 PHY trouble stop a fixed-hub host dwc3 calls phy_set_mode(PHY_MODE_USB_HOST) on the USB3 PHY as it brings its host up, and usb_add_hcd() calls it again for the SuperSpeed root hub of the xHCI it registers. The PHY routes the PIPE to the USB3 backend there. It fails with -EBUSY while a lane change is unfinished, or with the error of a PIPE lock or BIST that did not complete. A failure in usb_add_hcd() fails the whole xHCI registration: xhci-plat removes its USB2 HCD again as well. On a port behind a fixed hub that takes down the USB2 root hub, the hub on it and everything behind both USB-C connectors. A cable flip that arrives while the host starts, which only marks the lanes for a rebuild, is enough. For a fixed-hub host, a PIPE that cannot be routed no longer fails the call: park it if that works, and return 0. The Type-C mux update that follows a flip, or retries a failed change, restores the lanes and routes the PIPE with the controller's SuperSpeed port off. If the PIPE cannot even be parked, say so: the SuperSpeed port then faces a PIPE in an unknown state until that update. Other ports keep the error: their controller only starts after the mux has configured the lanes. Fixes: f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 35 +++++++++++++++++++++ drivers/phy/apple/atc.c | 61 +++++++++++++++++++++++++----------- 2 files changed, 78 insertions(+), 18 deletions(-) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index 994f758da0680..aef8ab6009e81 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -566,6 +566,40 @@ static void atcphy_test_stopped_reset(struct kunit *test) KUNIT_EXPECT_FALSE(test, aon & PIPEHANDLER_AON_GEN_DWC3_RESET_N); } +/* A SuperSpeed problem does not keep the host behind a fixed hub from starting */ +static void atcphy_test_host_start_without_usb3(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct phy *phy = kunit_kzalloc(test, sizeof(*phy), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, phy); + phy_set_drvdata(phy, atcphy); + atcphy_test_fake_regs(test, atcphy); + atcphy->host_active = false; + + /* An unfinished lane change: the PIPE stays parked for the next mux update */ + atcphy->mode_dirty = true; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + KUNIT_EXPECT_EQ(test, atcphy_usb3_set_mode(phy, PHY_MODE_USB_HOST, 0), 0); + KUNIT_EXPECT_TRUE(test, atcphy->host_active); + KUNIT_EXPECT_TRUE(test, atcphy->mode_dirty); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + + /* A handoff that cannot finish, as the PIPE lock is refused */ + atcphy->mode_dirty = false; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; + atcphy->pipe_pending = true; + KUNIT_EXPECT_EQ(test, atcphy_usb3_set_mode(phy, PHY_MODE_USB_HOST, 0), 0); + KUNIT_EXPECT_TRUE(test, atcphy->pipe_pending); + + /* A Type-C port's controller still waits for the lanes */ + atcphy = atcphy_test_alloc(test, false); + mutex_init(&atcphy->lock); + phy_set_drvdata(phy, atcphy); + atcphy->mode_dirty = true; + KUNIT_EXPECT_EQ(test, atcphy_usb3_set_mode(phy, PHY_MODE_USB_HOST, 0), -EBUSY); +} + #if IS_REACHABLE(CONFIG_USB) /* Only the controller's SuperSpeed root hub is ever handed the port changes */ static void atcphy_test_usb3_root(struct kunit *test) @@ -710,6 +744,7 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_host_start), KUNIT_CASE(atcphy_test_stopped_power_off), KUNIT_CASE(atcphy_test_stopped_reset), + KUNIT_CASE(atcphy_test_host_start_without_usb3), #if IS_REACHABLE(CONFIG_USB) KUNIT_CASE(atcphy_test_usb3_root), #endif diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 366bef334b970..3f7920c715bef 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -3070,9 +3070,38 @@ static int atcphy_usb3_power_off(struct phy *phy) return 0; } +/* Route the PIPE to the backend of the current mode for the role dwc3 selected. */ +static int atcphy_usb3_route(struct apple_atcphy *atcphy, bool host) +{ + int ret; + + lockdep_assert_held(&atcphy->lock); + + /* + * We may get multiple calls to set_mode (for host mode e.g. at least one from the dwc3 glue + * driver and then another one from the generic xhci code) but must only configure the + * PIPE handler once. Nothing needs to be done either when the PIPE is already routed to + * the backend the current mode uses, which includes the dummy backend of the USB2 modes. + */ + /* A failed lane/domain transition must be restored before routing PIPE. */ + if (atcphy->mode_dirty) + return -EBUSY; + if (atcphy->pipe_pending) { + ret = atcphy_park_pipehandler(atcphy); + if (ret) + return ret; + } + if (atcphy->pipe_state == atcphy_modes[atcphy->mode].pipehandler_state && + !atcphy->pipe_pending) + return 0; + + return atcphy_configure_pipehandler(atcphy, host); +} + static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode) { struct apple_atcphy *atcphy = phy_get_drvdata(phy); + int ret; guard(mutex)(&atcphy->lock); @@ -3092,26 +3121,22 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode return -EINVAL; } + ret = atcphy_usb3_route(atcphy, mode == PHY_MODE_USB_HOST); + if (!ret || !atcphy->fixed_usb2 || mode != PHY_MODE_USB_HOST) + return ret; + /* - * We may get multiple calls to set_mode (for host mode e.g. at least one from the dwc3 glue - * driver and then another one from the generic xhci code) but must only configure the - * PIPE handler once. Nothing needs to be done either when the PIPE is already routed to - * the backend the current mode uses, which includes the dummy backend of the USB2 modes. + * The host behind a fixed hub serves that hub as well, and when this + * fails, xHCI registers neither of its root hubs. Let it start, with + * the PIPE parked if that works: the Type-C mux update that restores + * the lanes or retries the handoff routes the PIPE, with the + * SuperSpeed port off. */ - /* A failed lane/domain transition must be restored before routing PIPE. */ - if (atcphy->mode_dirty) - return -EBUSY; - if (atcphy->pipe_pending) { - int ret = atcphy_park_pipehandler(atcphy); - - if (ret) - return ret; - } - if (atcphy->pipe_state == atcphy_modes[atcphy->mode].pipehandler_state && - !atcphy->pipe_pending) - return 0; - - return atcphy_configure_pipehandler(atcphy, mode == PHY_MODE_USB_HOST); + if (atcphy_park_pipehandler(atcphy)) + dev_warn(atcphy->dev, "Host starts with its PIPE not parked\n"); + else + dev_dbg(atcphy->dev, "Host starts without USB3: %d\n", ret); + return 0; } static const struct phy_ops apple_atc_usb3_phy_ops = { From 77ebd433838e09c662238c18ca51ff7fc5a0ab02 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 07:27:28 +1100 Subject: [PATCH 1221/1579] phy: apple: atc: Retry a SuperSpeed port power-on that a mux update owes A mux update that keeps the mode, but still owes the SuperSpeed root port of a fixed-hub controller its power, does not fail on the USB side, so that the display it may carry stays up. The Type-C driver then records the mode as applied, and nothing brings the port up later: with a USB3 dock that sends no further event, USB3 stays off although the host and the PHY are ready for it. The request to the root hub can fail for want of memory, which leaves the controller running. Retry the power-on from delayed work, five times over about three seconds. Each try takes the locks of a mux update, looks up the current root hub, and powers the port on only while it is still owed and the PIPE is on the USB3 backend of a running host; a mux update, a host stop or a new host in between ends the retries. A mode change that fails only in that last power-on still returns the error, and its power-on is retried the same way. The work is freezable, so it waits across system sleep, and it is cancelled when the PHY goes away. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 80 ++++++++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 88 ++++++++++++++++++++++++++++++++++-- 2 files changed, 164 insertions(+), 4 deletions(-) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index aef8ab6009e81..65e82a6d6edf0 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -157,6 +157,13 @@ static void atcphy_test_usb3_port_ready(struct kunit *test) KUNIT_EXPECT_FALSE(test, atcphy_usb3_port_ready(atcphy)); } +static void atcphy_test_cancel_port_work(void *data) +{ + struct apple_atcphy *atcphy = data; + + cancel_delayed_work_sync(&atcphy->usb3_port_work); +} + /* * A J700-style PHY behind a fixed hub, in DisplayPort + USB3 with the PIPE on * USB3 for a running host, but without a device tree node. A mux update that @@ -166,6 +173,11 @@ static void atcphy_test_usb3_port_ready(struct kunit *test) static struct apple_atcphy *atcphy_test_alloc_fixed_hub(struct kunit *test) { struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, true); + int ret; + + INIT_DELAYED_WORK(&atcphy->usb3_port_work, atcphy_usb3_port_work); + ret = kunit_add_action_or_reset(test, atcphy_test_cancel_port_work, atcphy); + KUNIT_ASSERT_EQ(test, ret, 0); atcphy->hw = &atcphy_hw_t8140; /* The lane and DisplayPort tunables a J700 has */ @@ -280,6 +292,20 @@ static int atcphy_test_mode(struct apple_atcphy *atcphy, enum atcphy_mode target return atcphy_test_steps()->mode_ret; } +/* Retries stay queued until a test runs them, in its own context. */ +static unsigned long atcphy_test_retry_delay(unsigned int retry) +{ + return MAX_JIFFY_OFFSET; +} + +static bool atcphy_test_run_retry(struct apple_atcphy *atcphy) +{ + if (!cancel_delayed_work_sync(&atcphy->usb3_port_work)) + return false; + atcphy_usb3_port_work(&atcphy->usb3_port_work.work); + return true; +} + static struct atcphy_test_steps *atcphy_test_stub_steps(struct kunit *test) { struct atcphy_test_steps *steps = kunit_kzalloc(test, sizeof(*steps), GFP_KERNEL); @@ -290,6 +316,7 @@ static struct atcphy_test_steps *atcphy_test_stub_steps(struct kunit *test) kunit_activate_static_stub(test, atcphy_usb3_port_off, atcphy_test_port_off); kunit_activate_static_stub(test, atcphy_usb3_port_on, atcphy_test_port_on); kunit_activate_static_stub(test, atcphy_mux_set_locked, atcphy_test_mode); + kunit_activate_static_stub(test, atcphy_usb3_port_retry_delay, atcphy_test_retry_delay); return steps; } @@ -399,6 +426,58 @@ static void atcphy_test_update_port_owed(struct kunit *test) KUNIT_EXPECT_STREQ(test, steps->log, "g0m"); } +/* A power-on owed after a USB failure is retried, while it is owed, a few times */ +static void atcphy_test_update_port_retry(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_steps *steps = atcphy_test_stub_steps(test); + unsigned int i; + + atcphy->usb3_port_off = true; + steps->mode_port_on = true; + + /* Out of memory for the hub request: the update keeps the display */ + steps->port_on_ret = -ENOMEM; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1"); + + /* and with no further Type-C event, the retry brings the port up */ + memset(steps->log, 0, sizeof(steps->log)); + steps->port_on_ret = 0; + KUNIT_EXPECT_TRUE(test, atcphy_test_run_retry(atcphy)); + KUNIT_EXPECT_STREQ(test, steps->log, "g1"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_FALSE(test, atcphy_test_run_retry(atcphy)); + + /* A retry finding the host gone leaves the port to the next host */ + atcphy->usb3_port_off = true; + steps->port_on_ret = -ENOMEM; + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + atcphy->host_active = false; + memset(steps->log, 0, sizeof(steps->log)); + KUNIT_EXPECT_TRUE(test, atcphy_test_run_retry(atcphy)); + KUNIT_EXPECT_STREQ(test, steps->log, "g"); + KUNIT_EXPECT_FALSE(test, atcphy_test_run_retry(atcphy)); + atcphy->host_active = true; + + /* A port that keeps failing is given up after a few retries */ + memset(steps->log, 0, sizeof(steps->log)); + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + for (i = 0; i < ATCPHY_USB3_PORT_RETRIES; i++) + KUNIT_EXPECT_TRUE(test, atcphy_test_run_retry(atcphy)); + KUNIT_EXPECT_FALSE(test, atcphy_test_run_retry(atcphy)); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1g1g1g1g1g1"); + KUNIT_EXPECT_TRUE(test, atcphy->usb3_port_off); + + /* A mode change fails on the last step, and is retried from there too */ + memset(steps->log, 0, sizeof(steps->log)); + KUNIT_EXPECT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_C), -ENOMEM); + steps->port_on_ret = 0; + KUNIT_EXPECT_TRUE(test, atcphy_test_run_retry(atcphy)); + KUNIT_EXPECT_STREQ(test, steps->log, "g0m1g1"); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); +} + /* A mode the PHY cannot run is turned down before USB3 goes down for it */ static void atcphy_test_update_unsupported(struct kunit *test) { @@ -738,6 +817,7 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_update_port_order), KUNIT_CASE(atcphy_test_update_port_failure), KUNIT_CASE(atcphy_test_update_port_owed), + KUNIT_CASE(atcphy_test_update_port_retry), KUNIT_CASE(atcphy_test_update_unsupported), KUNIT_CASE(atcphy_test_update_port_taken), KUNIT_CASE(atcphy_test_set_port_on), diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 3f7920c715bef..b6fdf6b496b4c 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -679,6 +680,8 @@ struct atcphy_hw { * then routes the PIPE itself on later Type-C mode changes * @usb3_port_off: The SuperSpeed root port of a fixed-hub controller was powered * off for a PIPE or lane change and is not powered on again yet + * @usb3_port_work: Retries the power-on of @usb3_port_off after a USB failure + * @usb3_port_retries: Retries left for @usb3_port_work * @mode: Current PHY operating mode * @swap_lanes: True if lanes must be swapped due to cable orientation * @dp_link_rate: DisplayPort link rate @@ -745,6 +748,8 @@ struct apple_atcphy { enum atcphy_mode typec_mode; bool host_active; bool usb3_port_off; + struct delayed_work usb3_port_work; + unsigned int usb3_port_retries; enum atcphy_mode mode; int dp_link_rate; bool swap_lanes; @@ -3673,6 +3678,77 @@ static int atcphy_usb3_port_on(struct apple_atcphy *atcphy, struct atcphy_usb3_p return ret; } +/* + * A mux update that keeps the mode does not fail on the USB side, so that the + * display it may carry stays up, and no Type-C event need follow any update. + * A SuperSpeed port power-on that an update still owes after a USB failure is + * retried from here, a few times, with the locks of a mux update, the current + * root hub, and only while the PIPE is on the USB3 backend of a running host. + * The work is freezable, so it waits across system sleep, and is cancelled on + * removal. + */ +#define ATCPHY_USB3_PORT_RETRIES 5 +#define ATCPHY_USB3_PORT_RETRY_MS 100 + +static unsigned long atcphy_usb3_port_retry_delay(unsigned int retry) +{ + KUNIT_STATIC_STUB_REDIRECT(atcphy_usb3_port_retry_delay, retry); + + return msecs_to_jiffies(ATCPHY_USB3_PORT_RETRY_MS << retry); +} + +static void atcphy_usb3_port_retry(struct apple_atcphy *atcphy, bool restart, int err) +{ + unsigned int retry = 0; + + scoped_guard(mutex, &atcphy->lock) { + if (restart) + atcphy->usb3_port_retries = ATCPHY_USB3_PORT_RETRIES; + if (!atcphy->usb3_port_retries) { + dev_warn(atcphy->dev, "SuperSpeed port left off: %d\n", err); + return; + } + retry = ATCPHY_USB3_PORT_RETRIES - atcphy->usb3_port_retries--; + } + dev_dbg(atcphy->dev, "SuperSpeed port power-on deferred: %d\n", err); + mod_delayed_work(system_freezable_wq, &atcphy->usb3_port_work, + atcphy_usb3_port_retry_delay(retry)); +} + +/* Power the SuperSpeed port on if it is still owed and the route still ready */ +static int atcphy_usb3_port_restore(struct apple_atcphy *atcphy, struct atcphy_usb3_pm *pm) +{ + int ret; + + scoped_guard(mutex, &atcphy->lock) { + if (!atcphy->usb3_port_off || !atcphy_usb3_port_ready(atcphy)) + return 0; + } + ret = atcphy_usb3_port_on(atcphy, pm); + if (ret < 0) + return ret; + scoped_guard(mutex, &atcphy->lock) + atcphy->usb3_port_off = false; + + return 0; +} + +static void atcphy_usb3_port_work(struct work_struct *work) +{ + struct apple_atcphy *atcphy = container_of(to_delayed_work(work), struct apple_atcphy, + usb3_port_work); + struct atcphy_usb3_pm pm = {}; + int ret; + + ret = atcphy_usb3_pm_get(atcphy, &pm); + if (!ret) { + ret = atcphy_usb3_port_restore(atcphy, &pm); + atcphy_usb3_pm_put(&pm); + } + if (ret) + atcphy_usb3_port_retry(atcphy, false, ret); +} + /* The PHY mode a Type-C mux state asks for; unsupported states turn the PHY off. */ static enum atcphy_mode atcphy_typec_target_mode(struct apple_atcphy *atcphy, const struct typec_mux_state *state) @@ -3963,11 +4039,12 @@ static int atcphy_mux_update(struct apple_atcphy *atcphy, const struct typec_mux /* * With the lanes and the PIPE already as the mode needs them, only * USB3 is missing, and that must not fail what may be a DisplayPort - * update. The port stays off until the next update. + * update. Either way, retry the port from here. */ - if (ret && usb_err && in_mode) { - dev_dbg(atcphy->dev, "SuperSpeed port left off: %d\n", ret); - return 0; + if (ret && usb_err) { + atcphy_usb3_port_retry(atcphy, true, ret); + if (in_mode) + return 0; } return ret; } @@ -4264,6 +4341,9 @@ static int atcphy_probe(struct platform_device *pdev) atcphy->np = dev->of_node; mutex_init(&atcphy->lock); platform_set_drvdata(pdev, atcphy); + ret = devm_delayed_work_autocancel(dev, &atcphy->usb3_port_work, atcphy_usb3_port_work); + if (ret) + return ret; ret = atcphy_map_resources(pdev, atcphy); if (ret) From 29431eb20dbd81c256b921f70d6f31015d49198d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 05:51:41 +1100 Subject: [PATCH 1222/1579] usb: dwc3: apple: Document the fixed-hub SuperSpeed handover The comment at the top of the driver lays down the order in which dwc3 and its USB3 PHY have to be brought down and up around a PHY change, and that a controller behind a fixed hub never sees a role change. It does not say how such a controller keeps to that order when the SuperSpeed lanes change underneath it. Add that the USB core holds the controller's SuperSpeed root port powered off for the PHY, port runtime PM included, from before the PIPE moves until it is back on the USB3 PHY, while the USB2 side stays up. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/dwc3-apple.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 49953a518eec9..464e5d9e691fe 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -76,7 +76,11 @@ * the board, whose downstream ports serve the Type-C connectors (Apple J700). Such a controller is * a host whatever happens at the connectors and never sees a role change: it is brought up in * DWC3_APPLE_HOST at probe and stays there. The Type-C port controllers only drive the PHY's - * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. + * Type-C mux for the SuperSpeed lanes, and the PHY follows those mode changes on its own. It keeps + * to the order above for the SuperSpeed side: the USB core holds the controller's SuperSpeed root + * port powered off for the PHY, port runtime PM included, from before the PIPE goes to the dummy + * backend until the PIPE is back on the USB3 PHY. The USB2 root port and the hub on it stay up + * throughout. * * Whenever the core is up, for either kind of port, it is kept running across system sleep, see * dwc3_apple_init(). In host mode xHCI then keeps running in suspend-to-idle when a USB device From 5b70b099a3003e104471dfa240515dac59b171a6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 23:16:00 +1100 Subject: [PATCH 1223/1579] usb: core: Preserve the port hold when administrative disable fails A disable=1 write can take over a port held off for a PHY lane change. Dropping the hold before ClearPortFeature(POWER) succeeds loses both the runtime PM exclusion and the obligation to restore power if that request fails. A later PHY update then treats the unpowered port as someone else's. Publish administrative ownership and release the hold only after the power request succeeds. Keep the hold and its PM reference on failure, and retain the rejection of disable=0 while the PHY owns the port. Add a software-HCD regression test with an injected failed power request. Fixes: 8ccb06392f04 ("usb: hub: Add helpers to power a port off and on for a lane change") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/port-power-test.c | 28 ++++++++++++++++++++++++++++ drivers/usb/core/port.c | 14 +++++++------- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/drivers/usb/core/port-power-test.c b/drivers/usb/core/port-power-test.c index 2a5fee7fe6854..d4a0460c7a08b 100644 --- a/drivers/usb/core/port-power-test.c +++ b/drivers/usb/core/port-power-test.c @@ -26,6 +26,7 @@ struct port_power_test_hc { u16 status; u16 change; unsigned int power_ons; + bool fail_power_off; }; struct port_power_test { @@ -170,6 +171,10 @@ static int port_power_test_hub_control(struct usb_hcd *hcd, u16 type_req, u16 va hc->change |= USB_PORT_STAT_C_RESET | USB_PORT_STAT_C_BH_RESET; return 0; case ClearPortFeature: + if (value == USB_PORT_FEAT_POWER && hc->fail_power_off) { + hc->fail_power_off = false; + return -ENOMEM; + } if (value == USB_PORT_FEAT_POWER) port_power_test_set_power(hcd, false); else @@ -536,6 +541,28 @@ static void port_power_test_removed_held(struct kunit *test) usb_unlock_device(t->ss_root); } +/* Review: a failed takeover request must not consume the driver's PM hold. */ +static void port_power_review_failed_takeover(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_hc(t->shared_hcd); + int usage = port_power_test_usage(t->ss_port); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + usb_unlock_device(t->ss_root); + scoped_guard(spinlock_irqsave, &hc->lock) + hc->fail_power_off = true; + KUNIT_EXPECT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "1"), -ENOMEM); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage + 1); + usb_lock_device(t->ss_root); + /* The next PHY update must still own its restoration. */ + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); + usb_unlock_device(t->ss_root); +} + static struct kunit_case port_power_test_cases[] = { KUNIT_CASE(port_power_test_hold), KUNIT_CASE(port_power_test_runtime_pm), @@ -543,6 +570,7 @@ static struct kunit_case port_power_test_cases[] = { KUNIT_CASE(port_power_test_disable), KUNIT_CASE(port_power_test_disabled_suspended), KUNIT_CASE(port_power_test_removed_held), + KUNIT_CASE(port_power_review_failed_takeover), {} }; diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c index 89066ef6ffdd0..721757e6242dc 100644 --- a/drivers/usb/core/port.c +++ b/drivers/usb/core/port.c @@ -157,13 +157,9 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, * in transition. Keeping it off takes the port over from the driver, * which then leaves it off. */ - if (port_dev->held_off) { - if (!disabled) { - rc = -EBUSY; - goto out_hdev_lock; - } - port_dev->held_off = false; - pm_runtime_put(&port_dev->dev); + if (port_dev->held_off && !disabled) { + rc = -EBUSY; + goto out_hdev_lock; } if (disabled && port_dev->child) @@ -180,6 +176,10 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, if (!rc) { port_dev->disabled = disabled; + if (port_dev->held_off) { + port_dev->held_off = false; + pm_runtime_put(&port_dev->dev); + } rc = count; } From fff435f15b8a259ece48395518cc5cf6d5545e26 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 23:17:23 +1100 Subject: [PATCH 1224/1579] usb: core: Keep administratively disabled ports off during runtime resume A PHY may move lanes after usb_hub_port_power_off() returns a nonnegative result. An administrative disable used to return early without excluding the port runtime-resume writer. Resuming its USB2 peer could then power the SuperSpeed port while the PHY changed lanes, even under the root-hub lock. Serialize disable and the runtime-resume power writer with a per-port mutex. Once disable succeeds, runtime resume leaves the port off, including the hub-reset path that records the desired power state. Take this mutex after resuming the peer and hub interface, never the hub device lock: the hub worker waits for port PM while holding that lock. Reassert power-off for administrative ports before allowing the lane change, since a disabled flag alone does not establish the physical state. Leave their ownership with sysfs and propagate any failed power request. Every other port now acquires a PHY hold, even if already unpowered, so there is no other successful path without runtime-PM exclusion. Retain the peer-resume counterexample and test a previously repowered administrative port, failed reassertion, retry, and explicit enable. Fixes: 8ccb06392f04 ("usb: hub: Add helpers to power a port off and on for a lane change") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/hub.c | 51 +++++++++-------- drivers/usb/core/hub.h | 2 + drivers/usb/core/port-power-test.c | 89 ++++++++++++++++++++++++++++++ drivers/usb/core/port.c | 24 ++++++-- 4 files changed, 136 insertions(+), 30 deletions(-) diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 191ff9e21dac4..638178f0f6482 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -2139,15 +2139,17 @@ bool usb_device_is_owned(struct usb_device *udev) * Writing 0 to its "disable" attribute fails with -EBUSY; writing 1 takes the * port over, after which usb_hub_port_power_on() leaves it off. * - * A port that is off for someone else, say through its "disable" attribute, - * is left as it is, and the caller can tell from the return value that it is - * not held for it. A port that runtime PM had switched off is not such a - * port: it is resumed, then held off for the caller. + * An administratively disabled port is powered off again, in case it was + * repowered before the handover. It stays under administrative ownership: + * runtime resume leaves it off, and the caller's hub lock prevents a write + * to "disable" from enabling it during the lane change. The return value + * distinguishes this from a port held for the caller. Other ports, including + * those switched off by runtime PM, are resumed and held off for the caller. * * Context: process context; the caller holds @hdev's device lock. * * Return: 0 if the port is held off for the caller, also if it was already, - * 1 if it is off for someone else, -EOPNOTSUPP if the hub cannot switch the + * 1 if it is administratively disabled, -EOPNOTSUPP if the hub cannot switch the * power of a single port, or another negative error code. */ int usb_hub_port_power_off(struct usb_device *hdev, int port1) @@ -2155,6 +2157,7 @@ int usb_hub_port_power_off(struct usb_device *hdev, int port1) struct usb_hub *hub = usb_hub_to_struct_hub(hdev); struct usb_interface *intf; struct usb_port *port_dev; + bool disabled; int ret; device_lock_assert(&hdev->dev); @@ -2169,9 +2172,7 @@ int usb_hub_port_power_off(struct usb_device *hdev, int port1) port_dev = hub->ports[port1 - 1]; if (port_dev->held_off) return 0; - /* Runtime PM may have suspended it since, which does not make it ours. */ - if (port_dev->disabled) - return 1; + disabled = port_dev->disabled; intf = to_usb_interface(hub->intfdev); ret = usb_autopm_get_interface(intf); @@ -2179,21 +2180,18 @@ int usb_hub_port_power_off(struct usb_device *hdev, int port1) return ret; /* - * The port's runtime resume switches its power on, and does not take - * the hub lock, as hub_event() waits for it under that lock. Keep port - * runtime PM out instead, after waiting for any of it in progress. + * Administrative disable excludes the runtime-resume power writer under + * power_lock. For other ports, wait for any port PM and keep it out + * until power-on. Neither path takes the hub lock in the PM callback. */ - ret = pm_runtime_resume_and_get(&port_dev->dev); - if (ret < 0) { - /* After a runtime PM error, nothing resumes the port any more. */ - if (!pm_runtime_active(&port_dev->dev)) - goto out; - pm_runtime_get_noresume(&port_dev->dev); - } - - if (!port_dev->child && !test_bit(port1, hub->power_bits)) { - ret = 1; - goto put_port; + if (!disabled) { + ret = pm_runtime_resume_and_get(&port_dev->dev); + if (ret < 0) { + /* After a runtime PM error, nothing resumes the port any more. */ + if (!pm_runtime_active(&port_dev->dev)) + goto out; + pm_runtime_get_noresume(&port_dev->dev); + } } if (port_dev->child) @@ -2208,7 +2206,7 @@ int usb_hub_port_power_off(struct usb_device *hdev, int port1) } /* The runtime PM reference is dropped once the hold ends. */ - port_dev->held_off = true; + port_dev->held_off = !disabled; /* Nothing hub_wq still has queued for the port applies to it off. */ clear_bit(port1, hub->change_bits); clear_bit(port1, hub->warm_reset_bits); @@ -2217,11 +2215,12 @@ int usb_hub_port_power_off(struct usb_device *hdev, int port1) usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_CONNECTION); if (!port_dev->is_superspeed) usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_ENABLE); - usb_autopm_put_interface(intf); - return 0; + ret = disabled; + goto out; put_port: - pm_runtime_put(&port_dev->dev); + if (!disabled) + pm_runtime_put(&port_dev->dev); out: usb_autopm_put_interface(intf); return ret; diff --git a/drivers/usb/core/hub.h b/drivers/usb/core/hub.h index 3c4465e378ef4..aabaf445b012e 100644 --- a/drivers/usb/core/hub.h +++ b/drivers/usb/core/hub.h @@ -96,6 +96,7 @@ struct usb_hub { * @state_kn: kernfs_node of the sysfs attribute that accesses @state * @location: opaque representation of platform connector location * @status_lock: synchronize port_event() vs usb_port_{suspend|resume} + * @power_lock: serialize administrative disable with runtime resume * @portnum: port index num based one * @is_superspeed cache super-speed status * @usb3_lpm_u1_permit: whether USB3 U1 LPM is permitted. @@ -118,6 +119,7 @@ struct usb_port { struct kernfs_node *state_kn; usb_port_location_t location; struct mutex status_lock; + struct mutex power_lock; /* administrative disable vs runtime resume */ u32 over_current_count; u8 portnum; u32 quirks; diff --git a/drivers/usb/core/port-power-test.c b/drivers/usb/core/port-power-test.c index d4a0460c7a08b..9fe6ef0eb6d83 100644 --- a/drivers/usb/core/port-power-test.c +++ b/drivers/usb/core/port-power-test.c @@ -541,6 +541,29 @@ static void port_power_test_removed_held(struct kunit *test) usb_unlock_device(t->ss_root); } +/* Review: admin-disabled SS port must stay off while the mux changes lanes. */ +static void port_power_review_disabled_peer_resume(struct kunit *test) +{ + struct port_power_test *t = test->priv; + + KUNIT_ASSERT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "1"), 0); + port_power_test_allow_power_off(test); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->hs_port), 0); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->ss_port), 0); + KUNIT_ASSERT_TRUE(test, pm_runtime_suspended(t->ss_port)); + KUNIT_ASSERT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + /* The same root-hub lock the mux holds over the PIPE/lane change. */ + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + /* A peer resume is not excluded by that root-hub lock. */ + KUNIT_ASSERT_GE(test, pm_runtime_resume_and_get(t->hs_port), 0); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + usb_unlock_device(t->ss_root); + pm_runtime_put(t->hs_port); +} + /* Review: a failed takeover request must not consume the driver's PM hold. */ static void port_power_review_failed_takeover(struct kunit *test) { @@ -563,6 +586,69 @@ static void port_power_review_failed_takeover(struct kunit *test) usb_unlock_device(t->ss_root); } +/* A previously repowered administrative port must be off before moving lanes. */ +static void port_power_test_disabled_repowered(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_hc(t->shared_hcd); + int usage = port_power_test_usage(t->ss_port); + + KUNIT_ASSERT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "1"), 0); + /* Model power restored behind usbcore's back, with disable still set. */ + scoped_guard(spinlock_irqsave, &hc->lock) + port_power_test_set_power(t->shared_hcd, true); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + + /* A failed reassertion must not permit the lane change. */ + scoped_guard(spinlock_irqsave, &hc->lock) { + port_power_test_set_power(t->shared_hcd, true); + hc->fail_power_off = true; + } + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), -ENOMEM); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + usb_unlock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage); + + /* Ending the handover leaves ownership with sysfs. */ + KUNIT_EXPECT_EQ(test, port_power_test_write(test, t->ss_port, "disable", "0"), 0); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); +} + +/* An unpowered port without an administrative owner still needs a PM hold. */ +static void port_power_test_unowned_off(struct kunit *test) +{ + struct port_power_test *t = test->priv; + int usage; + + port_power_test_allow_power_off(test); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->hs_port), 0); + KUNIT_ASSERT_GE(test, pm_runtime_suspend(t->ss_port), 0); + KUNIT_ASSERT_FALSE(test, port_power_test_powered(t->shared_hcd)); + /* Model an active port whose PM is unavailable and PP is still off. */ + pm_runtime_disable(t->ss_port); + KUNIT_EXPECT_EQ(test, pm_runtime_set_active(t->ss_port), 0); + usage = port_power_test_usage(t->ss_port); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage + 1); + KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); + KUNIT_EXPECT_TRUE(test, port_power_test_powered(t->shared_hcd)); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage); + usb_unlock_device(t->ss_root); + pm_runtime_enable(t->ss_port); +} + static struct kunit_case port_power_test_cases[] = { KUNIT_CASE(port_power_test_hold), KUNIT_CASE(port_power_test_runtime_pm), @@ -570,6 +656,9 @@ static struct kunit_case port_power_test_cases[] = { KUNIT_CASE(port_power_test_disable), KUNIT_CASE(port_power_test_disabled_suspended), KUNIT_CASE(port_power_test_removed_held), + KUNIT_CASE(port_power_review_disabled_peer_resume), + KUNIT_CASE(port_power_test_disabled_repowered), + KUNIT_CASE(port_power_test_unowned_off), KUNIT_CASE(port_power_review_failed_takeover), {} }; diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c index 721757e6242dc..7f0d8960a2e2a 100644 --- a/drivers/usb/core/port.c +++ b/drivers/usb/core/port.c @@ -165,7 +165,12 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, if (disabled && port_dev->child) usb_disconnect(&port_dev->child); - rc = usb_hub_set_port_power(hdev, hub, port1, !disabled); + /* Runtime resume must not repower the port after a successful disable. */ + scoped_guard(mutex, &port_dev->power_lock) { + rc = usb_hub_set_port_power(hdev, hub, port1, !disabled); + if (!rc) + port_dev->disabled = disabled; + } msleep(2 * hub_power_on_good_delay(hub)); if (disabled) { @@ -175,7 +180,6 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, } if (!rc) { - port_dev->disabled = disabled; if (port_dev->held_off) { port_dev->held_off = false; pm_runtime_put(&port_dev->dev); @@ -394,7 +398,10 @@ static int usb_port_runtime_resume(struct device *dev) if (!hub) return -EINVAL; if (hub->in_reset) { - set_bit(port1, hub->power_bits); + guard(mutex)(&port_dev->power_lock); + + if (!port_dev->disabled) + set_bit(port1, hub->power_bits); return 0; } @@ -409,7 +416,14 @@ static int usb_port_runtime_resume(struct device *dev) if (retval < 0) return retval; - retval = usb_hub_set_port_power(hdev, hub, port1, true); + /* hub_event() waits for port PM under the hub lock; do not take it here. */ + scoped_guard(mutex, &port_dev->power_lock) { + if (port_dev->disabled) { + retval = 0; + goto out; + } + retval = usb_hub_set_port_power(hdev, hub, port1, true); + } msleep(hub_power_on_good_delay(hub)); if (udev && !retval) { /* @@ -434,6 +448,7 @@ static int usb_port_runtime_resume(struct device *dev) } } +out: usb_autopm_put_interface(intf); return retval; @@ -800,6 +815,7 @@ int usb_hub_create_port_device(struct usb_hub *hub, int port1) dev_set_name(&port_dev->dev, "%s-port%d", dev_name(&hub->hdev->dev), port1); mutex_init(&port_dev->status_lock); + mutex_init(&port_dev->power_lock); retval = device_register(&port_dev->dev); if (retval) { put_device(&port_dev->dev); From f771659aba5e9ab34342df9a90499e0bb71a76c8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 17:43:29 +1100 Subject: [PATCH 1225/1579] soc: apple: sep: Recover the entropy exchange after a request times out Every GET_ENTROPY request carries the same reserved tag, and the exchange ends as soon as its waiter gives up. A reply that comes after its request timed out is then taken as the answer to the next request, and from there on every answer is one request late. Nothing stops the hwrng fill thread from asking again either. Each retry is one more message the enclave is not reading, in the inbox it shares with the key store, Touch ID and the xART service, and once that is full every send to the enclave waits 500 ms for room and fails. Count the requests that the mailbox took and the enclave has not answered. The enclave answers in order, so only the reply that settles the newest one is its answer, and earlier replies are dropped. After a request goes unanswered, or the mailbox has no room for it, hold further requests back with EAGAIN for 1 s, doubling with each failure up to 60 s, and resume as soon as a late reply shows that the enclave is answering again. A request made once the wait is over counts what is still owed as lost, so a reply that never comes cannot stop the channel for good. A held-back hwrng read returns -EAGAIN and is not logged as a failure; the core tries again 10 s later. A held-back survey attempt is not counted against the five the survey gets. KUnit tests (apple_sep_control) cover a reply in time, a late reply, a reply that comes after the retry, a lost reply, a full mailbox, a failed send, the back-off and a request abandoned for removal. Fixes: bac0846c9087 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/control.rs | 247 ++++++++++++++++++++++++++++++++++- drivers/soc/apple/sep.rs | 37 +++++- 2 files changed, 272 insertions(+), 12 deletions(-) diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index beb6b608415de..aeda23f5ef42e 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -23,13 +23,44 @@ impl Slot { }; } +/// After an entropy request goes unanswered, the next one waits this long, +/// doubling with each further failure up to [`ENTROPY_RETRY_MAX_MS`]. +pub(crate) const ENTROPY_RETRY_MIN_MS: u64 = 1000; +pub(crate) const ENTROPY_RETRY_MAX_MS: u64 = 60_000; + struct EntropySink { value: Option, busy: bool, + // Requests made and not answered yet, oldest first. A request that timed + // out may still be answered, and every reply carries the reserved tag, + // so only their order tells them apart. + owed: u32, + // A request went unanswered: none is sent before this time, in ms. + retry_at: Option, + retry_ms: u64, +} + +/// How an entropy exchange ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum EntropyEnd { + /// The reply was taken. + Answered, + /// No reply in time. It may still come. + TimedOut, + /// The mailbox did not take the request; `full` if it ran out of time + /// waiting for room, so the enclave is not reading it either. + NotSent { full: bool }, + /// Given up for a removal. The reply may still come. + Abandoned, } pub(crate) enum Delivery { Entropy, + /// The answer to an entropy request that had timed out. `recovered` if + /// requests were being held back until the enclave answered again. + EntropyLate { + recovered: bool, + }, Matched, Unmatched, } @@ -52,6 +83,9 @@ impl ControlState { entropy: EntropySink { value: None, busy: false, + owed: 0, + retry_at: None, + retry_ms: ENTROPY_RETRY_MIN_MS, }, } } @@ -125,8 +159,7 @@ impl ControlState { pub(crate) fn deliver(&mut self, reply: proto::ControlReply) -> Delivery { // The reserved entropy tag is claimed first, outstanding request or not. if reply.tag == proto::TAG_ENTROPY { - self.entropy.value = Some(reply.data_lo); - return Delivery::Entropy; + return self.deliver_entropy(reply.data_lo); } if let Some(slot) = self @@ -146,12 +179,51 @@ impl ControlState { Delivery::Unmatched } - pub(crate) fn entropy_begin(&mut self) -> Result<()> { + // The enclave answers in order, so a reply belongs to the oldest request + // still owed. Only the one that settles the newest request is its answer; + // any before it answer requests that timed out, and are dropped. + fn deliver_entropy(&mut self, value: u32) -> Delivery { + let entropy = &mut self.entropy; + if entropy.owed == 0 { + // Answers a request already written off as lost. + return Delivery::Unmatched; + } + entropy.owed -= 1; + if entropy.owed == 0 && entropy.busy { + entropy.value = Some(value); + return Delivery::Entropy; + } + // The enclave is answering again, so stop holding requests back. + let recovered = !entropy.busy && entropy.retry_at.take().is_some(); + if recovered { + entropy.retry_ms = ENTROPY_RETRY_MIN_MS; + } + Delivery::EntropyLate { recovered } + } + + /// Claims the entropy exchange for a request to be sent at `now_ms`. + /// + /// Refused with EAGAIN after a request the enclave did not answer, until + /// it answers or the retry time comes. A request that fails then doubles + /// the wait. Each claim counts as owed until [`Self::entropy_end`], + /// because the reply can arrive before the sender returns from the + /// mailbox. + pub(crate) fn entropy_begin(&mut self, now_ms: u64) -> Result<()> { if self.entropy.busy { return Err(EBUSY); } + if let Some(at) = self.entropy.retry_at { + if now_ms < at { + return Err(EAGAIN); + } + // Nothing came back in all that time. Count those replies as + // lost, or the answer to this request would be taken for one of + // theirs and no request would be answered again. + self.entropy.owed = 0; + } self.entropy.busy = true; self.entropy.value = None; + self.entropy.owed = self.entropy.owed.saturating_add(1); Ok(()) } @@ -159,7 +231,172 @@ impl ControlState { self.entropy.value.take() } - pub(crate) fn entropy_end(&mut self) { - self.entropy.busy = false; + /// Ends the exchange claimed by [`Self::entropy_begin`], at `now_ms`. + pub(crate) fn entropy_end(&mut self, how: EntropyEnd, now_ms: u64) { + let entropy = &mut self.entropy; + entropy.busy = false; + let failed = match how { + EntropyEnd::Answered => { + entropy.retry_at = None; + entropy.retry_ms = ENTROPY_RETRY_MIN_MS; + false + } + EntropyEnd::TimedOut => true, + EntropyEnd::NotSent { full } => { + entropy.owed = entropy.owed.saturating_sub(1); + full + } + EntropyEnd::Abandoned => false, + }; + if failed { + entropy.retry_at = Some(now_ms.saturating_add(entropy.retry_ms)); + entropy.retry_ms = + core::cmp::min(entropy.retry_ms.saturating_mul(2), ENTROPY_RETRY_MAX_MS); + } + } +} + +#[kernel::macros::kunit_tests(apple_sep_control)] +mod tests { + use super::*; + + fn entropy(value: u32) -> proto::ControlReply { + proto::ControlReply { + tag: proto::TAG_ENTROPY, + data_lo: value, + } + } + + // One request answered in time, as the hwrng read makes it. + fn draw(state: &mut ControlState, now_ms: u64, value: u32) -> Option { + state.entropy_begin(now_ms).ok()?; + let delivered = matches!(state.deliver(entropy(value)), Delivery::Entropy); + let taken = state.entropy_take(); + state.entropy_end( + if taken.is_some() { + EntropyEnd::Answered + } else { + EntropyEnd::TimedOut + }, + now_ms, + ); + if delivered { + taken + } else { + None + } + } + + #[test] + fn a_request_is_answered() { + let mut state = ControlState::new(); + assert_eq!(draw(&mut state, 0, 0x1234_5678), Some(0x1234_5678)); + assert_eq!(draw(&mut state, 1, 0x9abc_def0), Some(0x9abc_def0)); + } + + #[test] + fn a_late_reply_is_not_the_next_requests_answer() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::TimedOut, 0); + // The retry is held back until the enclave answers. + assert_eq!(state.entropy_begin(1), Err(EAGAIN)); + // It answers the request that timed out: dropped, and reads resume. + assert!(matches!( + state.deliver(entropy(1)), + Delivery::EntropyLate { recovered: true } + )); + assert_eq!(draw(&mut state, 2, 2), Some(2)); + } + + #[test] + fn a_reply_later_than_the_retry_shifts_nothing_after_it() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::TimedOut, 0); + // The retry goes out once its time comes, the first reply written off. + let at = ENTROPY_RETRY_MIN_MS; + assert!(state.entropy_begin(at).is_ok()); + // That reply comes after all. The retry takes it: it is enclave + // output nobody has used. + assert!(matches!(state.deliver(entropy(1)), Delivery::Entropy)); + assert_eq!(state.entropy_take(), Some(1)); + state.entropy_end(EntropyEnd::Answered, at); + // The retry's own reply is then owed to nobody. + assert!(matches!(state.deliver(entropy(2)), Delivery::Unmatched)); + assert_eq!(draw(&mut state, at + 1, 3), Some(3)); + } + + #[test] + fn a_request_abandoned_for_removal_does_not_shift_the_answers() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::Abandoned, 0); + assert!(state.entropy_begin(0).is_ok()); + assert!(matches!( + state.deliver(entropy(1)), + Delivery::EntropyLate { recovered: false } + )); + assert_eq!(state.entropy_take(), None); + assert!(matches!(state.deliver(entropy(2)), Delivery::Entropy)); + assert_eq!(state.entropy_take(), Some(2)); + state.entropy_end(EntropyEnd::Answered, 0); + } + + #[test] + fn a_lost_reply_does_not_stop_the_channel_for_good() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::TimedOut, 0); + // That reply never comes. The retry must still get its own answer. + let at = ENTROPY_RETRY_MIN_MS; + assert_eq!(draw(&mut state, at, 9), Some(9)); + assert_eq!(draw(&mut state, at + 1, 10), Some(10)); + } + + #[test] + fn a_full_mailbox_holds_requests_back_too() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::NotSent { full: true }, 0); + assert_eq!(state.entropy_begin(ENTROPY_RETRY_MIN_MS - 1), Err(EAGAIN)); + // Nothing was owed: the next request's reply is its own. + assert_eq!(draw(&mut state, ENTROPY_RETRY_MIN_MS, 4), Some(4)); + } + + #[test] + fn a_failed_send_owes_nothing() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::NotSent { full: false }, 0); + assert_eq!(draw(&mut state, 0, 8), Some(8)); + } + + #[test] + fn retries_back_off_up_to_a_limit() { + let mut state = ControlState::new(); + let mut now = 0; + let mut wait = ENTROPY_RETRY_MIN_MS; + for _ in 0..12 { + assert!(state.entropy_begin(now).is_ok()); + state.entropy_end(EntropyEnd::TimedOut, now); + assert_eq!(state.entropy_begin(now + wait - 1), Err(EAGAIN)); + now += wait; + wait = core::cmp::min(wait * 2, ENTROPY_RETRY_MAX_MS); + } + assert_eq!(wait, ENTROPY_RETRY_MAX_MS); + // An answer resets the wait. + assert_eq!(draw(&mut state, now, 1), Some(1)); + assert!(state.entropy_begin(now).is_ok()); + state.entropy_end(EntropyEnd::TimedOut, now); + assert!(state.entropy_begin(now + ENTROPY_RETRY_MIN_MS).is_ok()); + state.entropy_end(EntropyEnd::Abandoned, now); + } + + #[test] + fn a_reply_nobody_is_owed_is_dropped() { + let mut state = ControlState::new(); + assert!(matches!(state.deliver(entropy(1)), Delivery::Unmatched)); + assert_eq!(draw(&mut state, 0, 2), Some(2)); } } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index ee3a7a708bf29..612330f8f9431 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -207,6 +207,10 @@ fn sks_declared_sizes() -> (usize, usize) { (0x8000, 0x4000) } +fn boottime_ms() -> u64 { + shim::boottime_ns() / 1_000_000 +} + struct Hex<'a>(&'a [u8]); impl kernel::fmt::Display for Hex<'_> { @@ -1160,25 +1164,28 @@ impl SepData { return Err(EINVAL); } - self.control.lock().entropy_begin()?; + self.control.lock().entropy_begin(boottime_ms())?; let msg = proto::encode_control(&op, proto::TAG_ENTROPY); if let Err(e) = self.send(msg) { - self.control.lock().entropy_end(); + let full = e == ETIMEDOUT; + self.control + .lock() + .entropy_end(control::EntropyEnd::NotSent { full }, boottime_ms()); return Err(e); } let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); let mut guard = self.control.lock(); - let result = loop { + let (result, end) = loop { if let Some(value) = guard.entropy_take() { - break Ok(value); + break (Ok(value), control::EntropyEnd::Answered); } if self.rng_shutdown.load(Relaxed) { - break Err(ECANCELED); + break (Err(ECANCELED), control::EntropyEnd::Abandoned); } if remaining == 0 { - break Err(ETIMEDOUT); + break (Err(ETIMEDOUT), control::EntropyEnd::TimedOut); } match self .control_wq @@ -1189,7 +1196,7 @@ impl SepData { CondVarTimeoutResult::Timeout => remaining = 0, } }; - guard.entropy_end(); + guard.entropy_end(end, boottime_ms()); result } @@ -1228,6 +1235,12 @@ impl SepData { this.rng_survey_attempts.store(attempt, Relaxed); match this.survey_hwrng() { Ok(()) => dev_info!(this.dev, "hwrng: registered after ready-phase survey (attempt {})\n", attempt), + // Holding back after a request that went unanswered: this + // attempt did not ask the enclave anything. + Err(e) if e == EAGAIN => { + this.rng_survey_attempts.store(attempt - 1, Relaxed); + Self::arm_settle(this); + } Err(e) if attempt < RNG_SURVEY_ATTEMPTS => { dev_warn!(this.dev, "hwrng: survey attempt {} failed ({:?}); retrying\n", attempt, e); Self::arm_settle(this); @@ -1834,6 +1847,11 @@ impl SepData { // must return a negative errno, not 0: a blocking reader would spin on 0 let e = failure.unwrap_or(EIO); + // Holding back after a request that went unanswered. Nothing failed + // in this read; the core tries again later. + if e == EAGAIN { + return e.to_errno(); + } let n = self.rng_failures.load(Relaxed).wrapping_add(1); self.rng_failures.store(n, Relaxed); if n == 1 { @@ -2081,6 +2099,11 @@ impl SepData { control::Delivery::Entropy => { self.control_wq.notify_all(); } + control::Delivery::EntropyLate { recovered: true } => dev_info!( + self.dev, + "hwrng: the enclave answered a request that had timed out; reads resume\n" + ), + control::Delivery::EntropyLate { recovered: false } => {} control::Delivery::Matched => self.control_wq.notify_all(), control::Delivery::Unmatched => {} } From 1b0451249175e77ba9775e276d75a1d6e2310bf1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 17:43:29 +1100 Subject: [PATCH 1226/1579] soc: apple: sep: Hold entropy requests while the system sleeps The hwrng fill thread is not frozen for system sleep, and nothing in the driver held it back, so it could ask the enclave for entropy while devices were suspended. On an M1 Pro (T6000) one such request, sent during the 5 s pm_test=platform hold, was never answered. After the resume the enclave sent nothing more and did not read its mailbox again: every send timed out waiting for room, so /dev/hwrng stayed dead until reboot, and no other enclave service could be reached either. With the hwrng source deselected over three suspend cycles the enclave kept working, so it survives the sleep itself; it does not survive a request in that window. Hold entropy requests from the start of a suspend or hibernation until its end is reported, on the PM notifier that already holds Touch ID captures. Its window contains the whole of device suspend. A request already sent is first given time to be answered, which the enclave can still do then. A read in the window returns -EAGAIN, which the core treats as a source with nothing to give, and it tries again after the resume. KUnit tests cover a read while the system sleeps and a request in flight when the sleep begins. Fixes: bac0846c9087 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/control.rs | 57 ++++++++++++++++++++++++++++++++---- drivers/soc/apple/pm.rs | 9 ++++-- drivers/soc/apple/sep.rs | 41 +++++++++++++++++++++++--- 3 files changed, 96 insertions(+), 11 deletions(-) diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index aeda23f5ef42e..5eb9d51c9f421 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -35,6 +35,8 @@ struct EntropySink { // out may still be answered, and every reply carries the reserved tag, // so only their order tells them apart. owed: u32, + // The system is going to sleep: no request may be sent. + asleep: bool, // A request went unanswered: none is sent before this time, in ms. retry_at: Option, retry_ms: u64, @@ -84,6 +86,7 @@ impl ControlState { value: None, busy: false, owed: 0, + asleep: false, retry_at: None, retry_ms: ENTROPY_RETRY_MIN_MS, }, @@ -203,15 +206,18 @@ impl ControlState { /// Claims the entropy exchange for a request to be sent at `now_ms`. /// - /// Refused with EAGAIN after a request the enclave did not answer, until - /// it answers or the retry time comes. A request that fails then doubles - /// the wait. Each claim counts as owed until [`Self::entropy_end`], - /// because the reply can arrive before the sender returns from the - /// mailbox. + /// Refused with EAGAIN while the system sleeps, and after a request the + /// enclave did not answer, until it answers or the retry time comes. A + /// request that fails then doubles the wait. Each claim counts as owed + /// until [`Self::entropy_end`], because the reply can arrive before the + /// sender returns from the mailbox. pub(crate) fn entropy_begin(&mut self, now_ms: u64) -> Result<()> { if self.entropy.busy { return Err(EBUSY); } + if self.entropy.asleep { + return Err(EAGAIN); + } if let Some(at) = self.entropy.retry_at { if now_ms < at { return Err(EAGAIN); @@ -254,6 +260,21 @@ impl ControlState { core::cmp::min(entropy.retry_ms.saturating_mul(2), ENTROPY_RETRY_MAX_MS); } } + + /// The system is going to sleep: refuse requests until + /// [`Self::entropy_wake`]. Returns whether one is still in flight. + pub(crate) fn entropy_sleep(&mut self) -> bool { + self.entropy.asleep = true; + self.entropy.busy + } + + pub(crate) fn entropy_busy(&self) -> bool { + self.entropy.busy + } + + pub(crate) fn entropy_wake(&mut self) { + self.entropy.asleep = false; + } } #[kernel::macros::kunit_tests(apple_sep_control)] @@ -294,6 +315,32 @@ mod tests { assert_eq!(draw(&mut state, 1, 0x9abc_def0), Some(0x9abc_def0)); } + #[test] + fn no_request_is_sent_while_the_system_sleeps() { + let mut state = ControlState::new(); + assert!(!state.entropy_sleep()); + assert_eq!(state.entropy_begin(0), Err(EAGAIN)); + assert_eq!(state.entropy_begin(ENTROPY_RETRY_MAX_MS * 10), Err(EAGAIN)); + state.entropy_wake(); + assert_eq!(draw(&mut state, 1, 7), Some(7)); + } + + #[test] + fn a_request_in_flight_finishes_before_sleep() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + // The sleep has to wait for this one. + assert!(state.entropy_sleep()); + assert!(matches!(state.deliver(entropy(5)), Delivery::Entropy)); + assert_eq!(state.entropy_take(), Some(5)); + state.entropy_end(EntropyEnd::Answered, 0); + assert!(!state.entropy_busy()); + // And nothing new starts until the system is awake. + assert_eq!(state.entropy_begin(0), Err(EAGAIN)); + state.entropy_wake(); + assert_eq!(draw(&mut state, 0, 6), Some(6)); + } + #[test] fn a_late_reply_is_not_the_next_requests_answer() { let mut state = ControlState::new(); diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index a517e6e448732..b276890c27c78 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -4,8 +4,10 @@ //! talking to the sensor over SPI, so one left running into a suspend is still //! active while devices go down. A J313 that suspended with the lock screen's //! capture in flight did not wake. Before the system suspends, end any capture -//! in progress and hold new ones until it has resumed. The notifier itself -//! is in `pm_shim.c`; the capture handling is on [`SepData`]. +//! in progress and hold new ones until it has resumed. The hwrng fill thread +//! is not frozen either, so entropy requests are held over the same window. +//! The notifier itself is in `pm_shim.c`; the capture and entropy handling is +//! on [`SepData`]. use core::sync::atomic::{AtomicPtr, Ordering}; @@ -51,13 +53,16 @@ unsafe extern "C" fn pm_event(entering: bool) -> c_int { // notifier call, so it is live for this borrow. let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; if entering { + sep.rng_sleep_prepare(); if let Err(error) = sep.sleep_prepare() { // A failing notifier is not included in the PM chain's rollback. sep.sleep_finished(); + sep.rng_sleep_finished(); return error.to_errno(); } } else { sep.sleep_finished(); + sep.rng_sleep_finished(); } 0 } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 612330f8f9431..abfe68fb6acf4 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -173,6 +173,10 @@ const PROTECTED_DATA_AVAILABLE: bool = true; const RNG_MAX_WORDS_PER_READ: usize = 16; +// How long going to sleep waits for an entropy request in flight: the +// mailbox's 500 ms wait for room, then the wait for the reply. +const RNG_SLEEP_DRAIN_MS: u32 = proto::CONTROL_TIMEOUT_MS + 1000; + extern "C" { fn sep_cancel_work_sync(work: *mut c_void); fn sep_cancel_delayed_work_sync(work: *mut c_void); @@ -1200,6 +1204,35 @@ impl SepData { result } + /// The system is about to suspend: send no entropy request until it has + /// resumed, and let one already sent be answered while the enclave can + /// still answer it. On a T6000 a request sent while devices were + /// suspended was never answered, and the enclave did not read its mailbox + /// again after the resume. + pub(crate) fn rng_sleep_prepare(&self) { + if !self.control.lock().entropy_sleep() { + return; + } + let mut waited: u32 = 0; + while self.control.lock().entropy_busy() { + if waited >= RNG_SLEEP_DRAIN_MS { + dev_warn!( + self.dev, + "hwrng: a request was still in flight after {} ms; suspending anyway\n", + RNG_SLEEP_DRAIN_MS + ); + return; + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(10)); + waited += 10; + } + } + + /// The system has resumed, or the suspend was aborted. + pub(crate) fn rng_sleep_finished(&self) { + self.control.lock().entropy_wake(); + } + fn control_survey(&self) { for param in [0x00u8, 0x01, 0xff] { let _ = self.control_request(&proto::op_nop(param)); @@ -1235,8 +1268,8 @@ impl SepData { this.rng_survey_attempts.store(attempt, Relaxed); match this.survey_hwrng() { Ok(()) => dev_info!(this.dev, "hwrng: registered after ready-phase survey (attempt {})\n", attempt), - // Holding back after a request that went unanswered: this - // attempt did not ask the enclave anything. + // Asleep, or holding back after a request that went unanswered: + // this attempt did not ask the enclave anything. Err(e) if e == EAGAIN => { this.rng_survey_attempts.store(attempt - 1, Relaxed); Self::arm_settle(this); @@ -1847,8 +1880,8 @@ impl SepData { // must return a negative errno, not 0: a blocking reader would spin on 0 let e = failure.unwrap_or(EIO); - // Holding back after a request that went unanswered. Nothing failed - // in this read; the core tries again later. + // Asleep, or holding back after a request that went unanswered. + // Nothing failed in this read; the core tries again later. if e == EAGAIN { return e.to_errno(); } From d6b7ba26ef7fb1cde5a17ea4702e1ae690d6a48e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 18:59:35 +1100 Subject: [PATCH 1227/1579] soc: apple: sep: Poison the SBIO reply header before each host send A grant leaves the outbound header alone, because the reply may already be landing behind it. The enclave's grant may itself have written a header there, as the host's own grant writes a request-next header in the inbound buffer, and nothing reads or refills it. After a multi-chunk send, such as a Touch ID capture relay, the reply could take that header once its own has not landed yet: a data reply would fail its offset check as malformed, and an error report would read err 0 and hide the enclave's status. Poison the header again just before each chunk of a request is sent, the first and every later one, through a new send_for_reply(). The enclave is waiting for the host there, so no header can be landing. A continuation is sent straight after its chunk's header was taken and refilled, so it needs no change. A grant still leaves the buffer alone. Add KUnit cases for a grant's header followed by a data reply and by an error report, and for the refill coming before the send. The first two fail with the refill taken out of send_for_reply(). Fixes: bac0846c9087 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 4 +- drivers/soc/apple/transfer.rs | 83 ++++++++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 7a9afe8fbebf2..e3f6b613cc605 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2859,7 +2859,9 @@ impl SepData { crate::sbio::encode_sbio_next(opcode, seq) } .ok_or(EPERM)?; - self.send(msg)?; + // The last grant's header may sit in the outbound buffer; the + // reply to this chunk must not take it. + transfer::send_for_reply(&SbioHeader(self), || self.send(msg))?; offset += chunk; first = false; diff --git a/drivers/soc/apple/transfer.rs b/drivers/soc/apple/transfer.rs index 3323e7e2cfc09..082baf3ff1bf5 100644 --- a/drivers/soc/apple/transfer.rs +++ b/drivers/soc/apple/transfer.rs @@ -139,7 +139,9 @@ pub(crate) trait HeaderSlot { /// buffer. /// /// Once looked at, the header is poisoned again whatever the outcome, so the -/// next message cannot take it for its own. +/// next message cannot take it for its own. A message whose header is not +/// looked at, such as a grant, leaves the buffer alone: the reply may already +/// be landing behind it. [`send_for_reply`] poisons that header instead. pub(crate) fn take_header( slot: &impl HeaderSlot, marker: u8, @@ -177,6 +179,19 @@ pub(crate) fn take_header( Packet::decode(&header).map_err(|_| DeviceStatus::Malformed) } +/// Poisons the header in `slot` again, then calls `send` for a host message +/// that asks the enclave for its next one: a request, or the next chunk of +/// one. The enclave may have written a header there that nothing read, such +/// as a grant's, and the next reply must not take it for its own. Until +/// `send`, the enclave is waiting for the host, so no header can be landing. +pub(crate) fn send_for_reply( + slot: &impl HeaderSlot, + send: impl FnOnce() -> Result, +) -> Result { + slot.repoison(); + send() +} + impl Packet { /// Stands in for the header of a message whose handling reads none of it. pub(crate) const NONE: Packet = Packet { @@ -827,9 +842,73 @@ mod tests { assert!(take(&dma, MARKER_FIRST).is_err()); assert!(dma.poisoned()); - // A grant leaves the buffer alone. + // A grant leaves the buffer alone; the host's next send poisons it. dma.show(4, 0x16); assert!(take(&dma, MARKER_REQUEST).is_ok()); assert!(!dma.poisoned()); + assert!(send_for_reply(&dma, || Ok(())).is_ok()); + assert!(dma.poisoned()); + } + + /// The enclave's grant writes a request-next header, as the host's own + /// grant does, and the host sends its next chunk. + fn grant_then_send(dma: &Dma) { + dma.deliver( + &Packet::request_next(OPCODE, 0x3fe4, 0x8000), + HEADER_WORDS, + 0, + ); + assert!(take(dma, MARKER_REQUEST).is_ok()); + assert!(send_for_reply(dma, || Ok(())).is_ok()); + } + + #[test] + fn a_grants_header_is_not_taken_for_the_reply() { + let dma = Dma::new(); + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + rx.begin_send(OPCODE); + let generation = rx.generation(); + grant_then_send(&dma); + assert!(matches!( + rx.on_chunk(generation, MARKER_REQUEST, &Packet::NONE, &[]), + Progress::Grant + )); + rx.end_send(); + + // The reply arrives before any of its header. + dma.deliver(&data(8, 0, 8), 0, HEADER_WORDS); + let taken = take(&dma, MARKER_FIRST); + assert!(taken.is_ok()); + let Ok(packet) = taken else { + return; + }; + assert_eq!(packet.offset, 0); + assert_eq!(packet.chunk, 8); + assert!(matches!( + rx.on_chunk(generation, MARKER_FIRST, &packet, &[0u8; 8]), + Progress::Complete + )); + assert!(rx.take_done().is_some_and(|done| done.status.is_ok())); + } + + #[test] + fn a_grants_header_does_not_hide_an_error_status() { + let dma = Dma::new(); + grant_then_send(&dma); + dma.deliver(&reply(0x16), 0, HEADER_WORDS); + assert!(take(&dma, MARKER_ERROR).is_ok_and(|p| p.err == 0x16)); + } + + #[test] + fn the_header_is_poisoned_before_the_host_sends() { + let dma = Dma::new(); + dma.deliver(&data(8, 0, 8), HEADER_WORDS, 0); + let sent = send_for_reply(&dma, || { + assert!(dma.poisoned()); + Ok(7u32) + }); + assert!(sent.is_ok_and(|v| v == 7)); + assert!(send_for_reply(&dma, || -> Result<()> { Err(EIO) }).is_err()); } } From db757e1f92b02bbf265b936b34865c12eb54be60 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 19:43:03 +1100 Subject: [PATCH 1228/1579] selftests: apple-dcp: Follow the fixed-hub SuperSpeed handover The host runner fails 31 of 55 cases on the fixed-hub SuperSpeed handover and the PIPE park retry, because their units no longer compile. The ATC mux body moved to atcphy_mux_switch(), with atcphy_mux_check(), atcphy_mux_park() and atcphy_usb3_port_ready() split out, and atcphy_mux_set_locked() takes a third argument. The dummy handoff and the USB3 set_mode now call atcphy_pipe_live() and atcphy_usb3_route(). The PHY state has usb3_port_off and park_owed, and the mux update powers the SuperSpeed root port off and on and queues retries. Extract the split helpers wherever the fixtures extract their callers, and add the new state to the fixtures' copies of the PHY. The lane adapter now runs the actual atcphy_mux_switch() under atcphy_mux_set_locked(), as the ordinary-port quiesce unit does. Both check that no SuperSpeed port comes up, since they take none down. The PM fixtures extract the actual port power steps, retries and atcphy_mux_change(). They model the root port's power: powering it off disconnects the device on it, and the PIPE mux may not be written while the held root's port is up. A power-on asserts that the PIPE is on the USB3 backend of a clean running host. Three behaviours change on purpose, and the assertions that check them follow the source: - The link on the port is disconnected rather than resumed. The selective-suspend link checks still run for sources that resume the link. With the handover, the same scenarios check instead that SAFE takes the port down and keeps it down, and that a USB3 mode brings it back. They also check that a port that fails to go down stops the change before any PHY write, that a port someone else switched off stays theirs, and that a failed power-on fails the change, queues a retry and is completed by the same mode. The root state changes that were injected at the child lookup are injected at the root resume. That resume also starts the controller-lifecycle removal race. - dwc3 has left host mode once a USB3 power-off or controller reset fails, so the parking checks expect host_active cleared, and no lane rebuild from a stopped fixed-hub controller. - A fixed-hub host whose PIPE cannot be routed still starts. The orientation and warm-reset checks expect 0 there. The retry check expects a parked PIPE that the next mux update routes, or, if the lock stays held, the existing pending/acknowledged-retry sequence. Keep the existing assertions: for sources without the handover the expectations are the same as before. The new runner passes 55 of 55 on the drivers of 3abf4a56b59e, on the handover alone and with the park retry. Reverting each of five handover behaviours in atc.c fails it. No production driver changes are needed. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/apple-dcp/README | 12 ++ .../selftests/drivers/gpu/apple-dcp/common.py | 33 ++++- .../drivers/gpu/apple-dcp/dwc3/mocks.c | 1 + .../drivers/gpu/apple-dcp/lifecycle.py | 4 +- .../selftests/drivers/gpu/apple-dcp/phy.py | 71 ++++++++--- .../selftests/drivers/gpu/apple-dcp/pm.py | 119 +++++++++++++++++- .../drivers/gpu/apple-dcp/quiesce.py | 13 +- .../drivers/gpu/apple-dcp/teardown_pm.py | 3 +- 8 files changed, 228 insertions(+), 28 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index ad39902e5e259..906e56b7bf613 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -90,3 +90,15 @@ pending PIPE work and a mismatched PIPE still require ownership. KUnit static redirects are disabled in host fixtures so the production bodies execute. CD321x probe uses the actual optional-router lookup with a modeled deferred probe decision; both supplier waiting and optional-router fallback are checked. + +With the fixed-hub SuperSpeed handover the locked mux body is +atcphy_mux_switch(), and the PM fixtures run the actual USB side around it: +the SuperSpeed root port goes down before the PIPE moves, the device on it +with it, and comes up only once the PIPE is back on the USB3 backend of the +running host. A modeled port refuses PIPE moves while it is up, and its +power-off and power-on failures, a port someone else switched off and the +retries the update queues are checked. The adapter of the lane fixtures takes +no port down and checks that none comes up. A failed USB3 power-off or reset +leaves dwc3 out of host mode, and a fixed-hub host whose PIPE cannot be routed +still starts. Each fixture detects these from the source, so the expectations +for sources without the handover remain. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py index 89c2602ddb271..be3c888577dfd 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -54,6 +54,20 @@ def run_unit(name, unit, output, cases=((),), includes=(), extra=()): return hashlib.sha256(unit.encode()).hexdigest() +# Helpers that the fixed-hub SuperSpeed handover split out of extracted functions +ATC_SPLIT = ('atcphy_pipe_live', 'atcphy_usb3_route') + + +def atc_function(source, name): + """The actual ATC function @name, or nothing for a source that does not have it. + + The fixed-hub SuperSpeed handover splits helpers out of the functions that the + fixtures extract; older sources keep that code inline. + """ + marker = re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M) + return block(source, marker.start()) + '\n' if marker else '' + + def atc_mux_without_pm(source): """Lane/teardown fixtures omit USB PM, keeping mode mapping and lane work real. @@ -61,10 +75,27 @@ def atc_mux_without_pm(source): """ mapping = block(source, 'static enum atcphy_mode atcphy_typec_target_mode(') locked = block(source, 'static int atcphy_mux_set_locked(') - return mapping + '\n' + locked + r''' + if 'static int atcphy_mux_switch(' not in source: + return mapping + '\n' + locked + r''' static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) { struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); return atcphy_mux_set_locked(atcphy, atcphy_typec_target_mode(atcphy, state)); } +''' + # The locked body is atcphy_mux_switch(); atcphy_mux_set_locked() takes the lock + # and tells whether a SuperSpeed port that the USB side took down may come up. + helpers = ''.join(atc_function(source, name) for name in + ('atcphy_usb3_port_ready', 'atcphy_mux_check', 'atcphy_mux_park')) + return mapping + '\n' + helpers + block(source, 'static int atcphy_mux_switch(') + '\n' + locked + r''' +static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); + bool port_on; + int ret = atcphy_mux_set_locked(atcphy, atcphy_typec_target_mode(atcphy, state), &port_on); + + /* Without the USB side no SuperSpeed port went down, so none may come up. */ + assert(!port_on); + return ret; +} ''' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c index e54e73146cbad..940255a98a2f2 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -11,6 +11,7 @@ struct apple_atcphy { unsigned int dp_usb_context_valid, dp_usb_context_dirty; u32 dp_usb_context[2][27]; bool pipe_pending, usb_off_pending, usb_off_drained; + bool usb3_port_off, park_owed; enum atcphy_mode mode, typec_mode; enum atcphy_pipehandler_state pipe_state; }; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 0ac5c3bbb2f0f..4b47239655bfc 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -774,12 +774,12 @@ def fn(n, name): 'atcphy_configure_pipehandler_usb3_host_t8103', 'atcphy_configure_pipehandler_usb3_host_t8122', 'atcphy_configure_pipehandler_usb3', 'atcphy_configure_pipehandler_usb4', - 'atcphy_enable_dummy_phy', 'atcphy_configure_pipehandler_dummy', + 'atcphy_enable_dummy_phy', 'atcphy_pipe_live', 'atcphy_configure_pipehandler_dummy', 'atcphy_configure_pipehandler', 'atcphy_park_pipehandler', 'atcphy_get_mode_config', 'atcphy_dp_configure', 'j700_field_update', 'j700_aux_disable', 'atcphy_disable_dp_aux', 'atcphy_power_off', 'atcphy_power_on', 'atcphy_configure', 'atcphy_usb2_init', 'atcphy_usb3_power_off', - 'atcphy_usb3_set_mode', 'atcphy_usb2_power_off', + 'atcphy_usb3_route', 'atcphy_usb3_set_mode', 'atcphy_usb2_power_off', '_atcphy_dwc3_reset_assert', 'atcphy_dwc3_reset_assert', 'atcphy_mux_set'), 'phy': ('phy_init', 'phy_exit', 'phy_power_on', 'phy_power_off'), 'core': ('dwc3_phy_power_on', 'dwc3_phy_power_off', 'dwc3_phy_exit', 'dwc3_core_exit'), diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py index e29f767e80a29..e424a1b9dcb9e 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py @@ -3,7 +3,7 @@ """Actual PHY parking, failure/retry, masked context and OFF controls.""" from pathlib import Path import re -from common import block, source_defines, atc_mux_without_pm +from common import block, source_defines, atc_mux_without_pm, atc_function, ATC_SPLIT def parking(root): @@ -14,6 +14,8 @@ def parking(root): def function(source, name): if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: return atc_mux_without_pm(source) + if name in ATC_SPLIT: + return atc_function(source, name) return block(source, re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) @@ -39,6 +41,7 @@ def function(source, name): unsigned int dp_usb_context_valid, dp_usb_context_dirty; u32 dp_usb_context[2][27]; bool pipe_pending, usb_off_pending, usb_off_drained; + bool usb3_port_off, park_owed; enum atcphy_mode mode, typec_mode; enum atcphy_pipehandler_state pipe_state; }; @@ -123,7 +126,7 @@ def function(source, name): 'atcphy_configure_pipehandler_usb3_host_t8122', 'atcphy_configure_pipehandler_usb3_host_t8103', 'atcphy_configure_pipehandler_usb3', 'atcphy_configure_pipehandler_usb4', - 'atcphy_enable_dummy_phy', 'atcphy_configure_pipehandler_dummy', + 'atcphy_enable_dummy_phy', 'atcphy_pipe_live', 'atcphy_configure_pipehandler_dummy', 'atcphy_configure_pipehandler', 'atcphy_park_pipehandler', 'atcphy_usb2_power_off', '_atcphy_dwc3_reset_assert', 'atcphy_dwc3_reset_assert'): parts.append(function(atc, name)) @@ -133,11 +136,17 @@ def function(source, name): for name in ('atcphy_configure_lanes', 'j700_aux_enable', 'j700_aux_disable', 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', 'atcphy_power_on', 'atcphy_configure', 'atcphy_set_orientation', 'atcphy_sw_set', 'atcphy_mux_set', - 'atcphy_usb3_set_mode', 'atcphy_usb3_power_off'): + 'atcphy_usb3_route', 'atcphy_usb3_set_mode', 'atcphy_usb3_power_off'): parts.append(function(atc, name)) production = '\n'.join(parts) if 'static int j700_usb_capture(' in dp: production = '#define HAS_USB_CONTEXT\n' + production + # With the fixed-hub SuperSpeed handover, dwc3 has left host mode once it + # powers the USB3 PHY off or asserts its reset, and a fixed-hub host starts + # even if its PIPE cannot be routed. + production = ('#define STOPPED_HOST_PARKS ' + str(int('static bool atcphy_pipe_live(' in atc)) + + '\n#define FIXED_HOST_STARTS ' + str(int('static int atcphy_usb3_route(' in atc)) + + '\n' + production) prefix = r''' #include #include @@ -272,6 +281,13 @@ def function(source, name): }; return atcphy_mux_set(&mux, &state); } +/* + * Whether a USB3 power-off or controller reset that failed leaves dwc3 a host. + * With the fixed-hub SuperSpeed handover it has left host mode either way. + */ +#define FAILED_STOP_HOST (!STOPPED_HOST_PARKS) +/* A host start whose PIPE cannot be routed fails, except on a fixed-hub port with the handover. */ +#define HOST_START_ERROR(p, err) (FIXED_HOST_STARTS && (p)->fixed_usb2 ? 0 : (err)) static void unchanged(struct apple_atcphy *p, bool dirty, bool host) { bool expected_dirty = dirty || (p->hw->has_t8140_dp && p->fixed_usb2 && host); @@ -344,10 +360,10 @@ def function(source, name): int ret = atcphy_usb3_power_off(&phy); observed("power-off parking failure", ret); assert(ret == -ETIMEDOUT); - unchanged(&p, dirty, true); + unchanged(&p, dirty, FAILED_STOP_HOST); if (!fault) { assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); - unchanged(&p, dirty, true); + unchanged(&p, dirty, FAILED_STOP_HOST); } recover(); assert(atcphy_usb3_power_off(&phy) == 0); @@ -373,15 +389,15 @@ def function(source, name): int ret = atcphy_dwc3_reset_assert(&p.rcdev, 0); observed("reset parking failure", ret); assert(ret == -ETIMEDOUT); - unchanged(&p, dirty, true); + unchanged(&p, dirty, FAILED_STOP_HOST); if (!fault) { assert(atcphy_dwc3_reset_assert(&p.rcdev, 0) == -ETIMEDOUT); - unchanged(&p, dirty, true); + unchanged(&p, dirty, FAILED_STOP_HOST); } recover(); assert(atcphy_dwc3_reset_assert(&p.rcdev, 0) == 0); assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && !p.host_active); - assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.mode_dirty == (dirty || fixed)); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.mode_dirty == (dirty || (fixed && FAILED_STOP_HOST))); assert(p.dp_owned_pmas == 3 && p.dp_idle_pmas == (dirty ? 1U : 0U)); assert(p.aux_power_requested && writes == 0); assert(reset_writes == 2 && usb2_writes > 0); @@ -403,7 +419,7 @@ def function(source, name): fail_poll = 1; /* The first DP stop ACK cannot complete. */ int ret = atcphy_usb3_power_off(&phy); observed("power-off DP cleanup failure", ret); - assert(ret == -ETIMEDOUT && p.mode_dirty && p.host_active); + assert(ret == -ETIMEDOUT && p.mode_dirty && p.host_active == FAILED_STOP_HOST); assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.dp_owned_pmas == 3); assert(p.aux_power_requested && usb2_writes == 0 && reset_writes == 0); struct phy_configure_opts_dp training = { @@ -420,7 +436,7 @@ def function(source, name): p.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; fail_poll = 1; assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); - assert(p.mode_dirty && p.host_active && p.dp_owned_pmas == 3); + assert(p.mode_dirty && p.host_active == FAILED_STOP_HOST && p.dp_owned_pmas == 3); recover(); assert(atcphy_usb3_power_off(&phy) == 0); assert(p.mode == APPLE_ATCPHY_MODE_OFF && !p.mode_dirty && !p.host_active); @@ -433,7 +449,7 @@ def function(source, name): p.aux_power_requested = false; fail_poll = fault; /* BIG/SMALL sleep ACK. */ assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); - assert(p.mode == APPLE_ATCPHY_MODE_USB2 && p.mode_dirty && p.host_active); + assert(p.mode == APPLE_ATCPHY_MODE_USB2 && p.mode_dirty && p.host_active == FAILED_STOP_HOST); recover(); assert(atcphy_usb3_power_off(&phy) == 0 && !p.mode_dirty && !p.host_active); } @@ -447,7 +463,7 @@ def function(source, name): p.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; fail_poll = fault; assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); - assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.mode_dirty && p.host_active); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.mode_dirty && p.host_active == FAILED_STOP_HOST); assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); assert(usb2_writes == 0 && reset_writes == 0); unsigned int owned = p.dp_owned_pmas, idle = p.dp_idle_pmas; @@ -557,7 +573,7 @@ def function(source, name): assert(p.mode_dirty); struct phy phy = { &p }; unsigned int before = writes; - assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == -EBUSY); + assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == HOST_START_ERROR(&p, -EBUSY)); assert(writes == before); } require_parked = true; @@ -998,6 +1014,23 @@ def adverse(root): struct phy phy={&p}; if (unlock) hold_unlock_ack=true; else poll_errors[2]=-EIO; +#if FIXED_HOST_STARTS + if (fixed) { + /* A fixed-hub host starts, with its PIPE parked if the lock lets go. */ + assert(atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0)==0); + assert(p.mode_dirty && p.pipe_pending==(bool)unlock); + if (!unlock) { + /* Parked: the next mux update restores the lanes and routes the PIPE. */ + assert(p.pipe_state==ATCPHY_PIPEHANDLER_STATE_DUMMY); + memset(poll_errors,0,sizeof(poll_errors)); + assert((mux ? select_mode(&p,APPLE_ATCPHY_MODE_USB3) : atcphy_usb2_init(&phy))==0); + assert(!p.mode_dirty && !p.pipe_pending); + assert(atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0)==0); + assert(p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3 && !p.pipe_pending); + continue; + } + } else +#endif assert(atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0)== (unlock ? -ETIMEDOUT : -EIO)); assert(p.mode_dirty && p.pipe_pending); @@ -1346,7 +1379,7 @@ def warm_reset(root): assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); struct phy phy = { &p }; unsigned int before = writes; - assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == -EBUSY); + assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == HOST_START_ERROR(&p, -EBUSY)); assert(j700_dp_configure(&p, &rate) == -EBUSY && writes == before); fail_caller = NULL; assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); @@ -1675,6 +1708,8 @@ def block(marker): def function(name): if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: return atc_mux_without_pm(source) + if name in ATC_SPLIT: + return atc_function(source, name) return block(re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) @@ -1694,6 +1729,7 @@ def function(name): unsigned int dp_usb_context_valid, dp_usb_context_dirty; u32 dp_usb_context[2][27]; bool pipe_pending, usb_off_pending, usb_off_drained; + bool usb3_port_off, park_owed; bool swap_lanes; enum atcphy_mode mode, typec_mode; enum atcphy_pipehandler_state pipe_state; @@ -1749,9 +1785,9 @@ def function(name): 'atcphy_configure_pipehandler_usb3_host_t8122', 'atcphy_configure_pipehandler_usb3_host_t8103', 'atcphy_configure_pipehandler_usb3', 'atcphy_configure_pipehandler_usb4', - 'atcphy_enable_dummy_phy', 'atcphy_configure_pipehandler_dummy', + 'atcphy_enable_dummy_phy', 'atcphy_pipe_live', 'atcphy_configure_pipehandler_dummy', 'atcphy_configure_pipehandler', 'atcphy_park_pipehandler', - 'atcphy_usb3_set_mode', 'atcphy_mux_set'): + 'atcphy_usb3_route', 'atcphy_usb3_set_mode', 'atcphy_mux_set'): parts.append(function(name)) production = '\n'.join(parts) prefix = r''' @@ -2028,6 +2064,8 @@ def unwind(root): def function(source, name): if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: return atc_mux_without_pm(source) + if name in ATC_SPLIT: + return atc_function(source, name) return block(source, re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) @@ -2052,6 +2090,7 @@ def function(source, name): unsigned int dp_usb_context_valid, dp_usb_context_dirty; u32 dp_usb_context[2][27]; bool pipe_pending, usb_off_pending, usb_off_drained; + bool usb3_port_off, park_owed; enum atcphy_mode mode, typec_mode; enum atcphy_pipehandler_state pipe_state; }; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index ac1249af085ca..be73d45fadcb2 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -4,12 +4,30 @@ from pathlib import Path import os, re, subprocess import phy -from common import block, tipd_pm_header +from common import block, tipd_pm_header, atc_function def ownership(root, usb_unreachable=False): ns=phy.parking(root) s=ns['atc'];has_pm='static int atcphy_usb3_pm_get(' in s - prod=ns['production'].replace('struct apple_atcphy {','struct apple_atcphy {\n struct device_node *np;',1) + # The fixed-hub SuperSpeed handover powers the SuperSpeed root port off around + # a change, where the PM wrapper used to resume the link on that port. + port_power='static int atcphy_usb3_port_off(' in s + link_resume='static int atcphy_usb3_link_get(' in s + fields='\n struct device_node *np;' + if port_power: + fields+='\n struct delayed_work usb3_port_work, park_work;\n unsigned int usb3_port_retries, park_retries;' + prod=ns['production'].replace('struct apple_atcphy {','struct apple_atcphy {'+fields,1) + if port_power: + # The PIPE may only move while the SuperSpeed port of the held root is down. + mux_write='if (p == (void *)&pipe_regs[PIPEHANDLER_MUX_CTRL / 4]) mux_writes++;' + assert mux_write in prod + prod=prod.replace(mux_write,'if (p == (void *)&pipe_regs[PIPEHANDLER_MUX_CTRL / 4]) { mux_writes++; assert(!pipe_change_window() || usb3_port_down); }',1) + prod=r''' +struct work_struct { int unused; }; +struct delayed_work { struct work_struct work; unsigned int queued; unsigned long delay; }; +static bool usb3_port_down; +static bool pipe_change_window(void); +'''+prod if has_pm: prod = prod.replace(block(prod, 'static int atcphy_mux_set('), '') # Non-mux fixture entry points continue to exercise actual source. Only the new @@ -106,10 +124,16 @@ def ownership(root, usb_unreachable=False): static struct usb_device *usb_get_dev(struct usb_device *u) { u->refs++;return u; } static void usb_put_dev(struct usb_device *u) { assert(u->refs>0);u->refs--; } static int usb_for_each_dev(void *data,int (*fn)(struct usb_device *,void *)) { if(root_missing)return 0;for(unsigned int i=0;i<4;i++){int r=fn(&roots[i],data);if(r)return r;}return 0; } -static struct usb_device *usb_hub_find_child(struct usb_device *u, int port) { assert(u==&roots[3]&&root_locked&&port==1); +/* Root state changes while the root is held: at the lookup of its child or, without one, at its resume. */ +static void root_held_events(struct usb_device *u) { if(detach_at_lookup)usb_set_device_state(u,USB_STATE_NOTATTACHED); if(quiesce_at_lookup)hcd.state=HC_STATE_QUIESCING; +} +#if LINK_RESUME +static struct usb_device *usb_hub_find_child(struct usb_device *u, int port) { assert(u==&roots[3]&&root_locked&&port==1); + root_held_events(u); return link_present?&link_device:NULL; } +#endif static void usb_lock_device(struct usb_device *u) { if(u==&roots[3]){assert(!root_locked);if(REQUIRES_PLATFORM_LOCKS)assert(controller_locked&&host_locked);root_locked=true;if(detach_on_lock)u->state=0;} else {assert(u==&link_device&&root_locked&&!link_locked);link_locked=true;} @@ -134,6 +158,7 @@ def ownership(root, usb_unreachable=False): struct lock_guard g=lock_take(&pm_phy->lock);lock_release(&g); if(resume_error)return resume_error; pm_refs++;stuck_ack=link_present&&link_device.state!=USB_STATE_NOTATTACHED&&link_suspended;if(detach_on_resume)roots[3].state=0; + if(!LINK_RESUME)root_held_events(&roots[3]); if(inaccessible_on_resume)clear_bit(HCD_FLAG_HW_ACCESSIBLE,&hcd.flags); if(dead_on_resume)set_bit(HCD_FLAG_DEAD,&hcd.flags); return 0; @@ -151,6 +176,43 @@ def ownership(root, usb_unreachable=False): if(d==&link_device.dev){assert(link_locked&&link_pm_refs==1);link_pm_refs--;} else {assert(d==&roots[3].dev&&!link_locked&&pm_refs==1);pm_refs--;} } +#define dev_info(...) ((void)0) +#if PORT_POWER +#define to_delayed_work(w) container_of(w, struct delayed_work, work) +#define msecs_to_jiffies(ms) ((unsigned long)(ms)) +struct workqueue_struct { int unused; }; +static struct workqueue_struct freezable_wq; +#define system_freezable_wq (&freezable_wq) +static unsigned int port_offs, port_ons; +static int port_off_error, port_on_error; +static bool port_disabled; +static bool mod_delayed_work(struct workqueue_struct *wq, struct delayed_work *w, unsigned long delay) +{ assert(wq==system_freezable_wq&&!pm_phy->lock);w->queued++;w->delay=delay;return true; } +static bool pipe_change_window(void) { return root_locked; } +/* Port power of the held root; the device on it goes, and with it its hold on the PIPE. */ +static int usb_hub_port_power_off(struct usb_device *u, int port) +{ + assert(u==&roots[3]&&port==1&&root_locked&&pm_refs==1&&!pm_phy->lock); + port_offs++; + if(port_off_error)return port_off_error; + if(port_disabled)return 1; + if(link_present&&link_device.state!=USB_STATE_NOTATTACHED){link_device.state=USB_STATE_NOTATTACHED;stuck_ack=false;} + usb3_port_down=true; + return 0; +} +/* Only once the PIPE is back on the USB3 backend of the running host. */ +static int usb_hub_port_power_on(struct usb_device *u, int port) +{ + assert(u==&roots[3]&&port==1&&root_locked&&pm_refs==1&&!pm_phy->lock); + assert(usb3_port_down&&pm_phy->host_active&&!pm_phy->mode_dirty&&!pm_phy->pipe_pending); + assert(pm_phy->pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3); + port_ons++; + if(port_on_error)return port_on_error; + if(port_disabled)return 1; + usb3_port_down=false; + return 0; +} +#endif ''' suffix=ns['suffix'].replace('int main(', 'int __attribute__((unused)) fixture_main(',1) controls=r''' @@ -175,6 +237,9 @@ def ownership(root, usb_unreachable=False): link_present=link_suspended=link_locked=false; link_pm_refs=link_resumes=link_error=link_detach_on_resume=link_dead_on_resume=0; detach_at_lookup=quiesce_at_lookup=0; +#if PORT_POWER + usb3_port_down=port_disabled=false;port_offs=port_ons=0;port_off_error=port_on_error=0; +#endif } static int safe(struct apple_atcphy *p) { @@ -272,6 +337,7 @@ def ownership(root, usb_unreachable=False): int ret=safe(&p);fprintf(stderr,"post-child admission event %u returned %d writes %u\n",event,ret,writes-before); assert(ret==-ENODEV&&writes==before);balanced(&p); } +#if LINK_RESUME /* Selectively suspended upstream link is not resumed by root ownership alone. */ for(unsigned int swap=0;swap<2;swap++){ prepare(&p,true,true,swap);link_present=link_suspended=stuck_ack=true; @@ -291,6 +357,38 @@ def ownership(root, usb_unreachable=False): prepare(&p,true,true,false);link_present=link_suspended=true;fail_caller="atcphy_pipehandler_lock";before=writes; assert(safe(&p)==-ETIMEDOUT&&writes==before&&p.mode_dirty&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3);balanced(&p); fail_caller=NULL;assert(safe(&p)==0);balanced(&p); +#else + /* The device on the SuperSpeed port goes with the port, before the PIPE moves. */ + for(unsigned int swap=0;swap<2;swap++){ + prepare(&p,true,true,swap);link_present=link_suspended=stuck_ack=true; + int link_ret=safe(&p);fprintf(stderr,"port-off SAFE returned %d, root resumes %d, port offs %u\n",link_ret,resumes,port_offs); + assert(link_ret==0&&resumes==1&&link_resumes==0&&link_device.state==USB_STATE_NOTATTACHED); + /* SAFE has no USB3: the port stays off until a USB3 mode has the PIPE back. */ + assert(port_offs==1&&!port_ons&&usb3_port_down&&p.usb3_port_off);balanced(&p); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3); + assert(port_offs==2&&port_ons==1&&!usb3_port_down&&!p.usb3_port_off);balanced(&p); + } + /* A port that does not go down stops the change before any PHY programming. */ + prepare(&p,true,true,false);link_present=link_suspended=true;port_off_error=-EIO;before=writes; + unsigned int retained=p.dp_usb_context_valid,owners=p.dp_owned_pmas; + assert(safe(&p)==-EIO&&writes==before&&p.dp_usb_context_valid==retained&&p.dp_owned_pmas==owners); + assert(!port_ons&&!usb3_port_down&&!p.usb3_port_off);balanced(&p); + port_off_error=0;assert(safe(&p)==0&&usb3_port_down);balanced(&p); + /* A port that someone else switched off stays theirs. */ + prepare(&p,true,true,false);port_disabled=usb3_port_down=true; + assert(safe(&p)==0&&port_offs==1&&!p.usb3_port_off);balanced(&p); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0&&!port_ons&&usb3_port_down);balanced(&p); + /* A port that does not come back fails the change; the same mode then only powers it on. */ + prepare(&p,true,true,false);assert(safe(&p)==0);port_on_error=-EIO; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==-EIO&&port_ons==1&&p.usb3_port_off&&p.usb3_port_work.queued==1); + balanced(&p);port_on_error=0;before=writes; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0&&port_ons==2&&!p.usb3_port_off&&writes==before);balanced(&p); + /* PM resume is NOT proof of ACK; the port is down and stays down. */ + prepare(&p,true,true,false);link_present=link_suspended=true;fail_caller="atcphy_pipehandler_lock";before=writes; + assert(safe(&p)==-ETIMEDOUT&&writes==before&&p.mode_dirty&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3); + assert(usb3_port_down&&p.usb3_port_off&&!port_ons);balanced(&p); + fail_caller=NULL;assert(safe(&p)==0);balanced(&p); +#endif for(unsigned int count=0;count<2;count++){ prepare(&p,true,true,false);roots[3].maxchild=count?2:0;before=writes; assert(safe(&p)==(count?-ENODEV:-EAGAIN)&&writes==before);balanced(&p); @@ -308,11 +406,20 @@ def ownership(root, usb_unreachable=False): if has_pm: helpers=ns['block'](s,'struct atcphy_usb3_pm {')+';\n' helpers+='#if IS_REACHABLE(CONFIG_USB)\n' - if 'static int atcphy_usb3_link_get(' in s: + if link_resume: helpers+=ns['function'](s,'atcphy_usb3_link_get')+'\n'+ns['function'](s,'atcphy_usb3_link_put')+'\n' + helpers+=atc_function(s,'atcphy_usb3_is_root') helpers+=ns['function'](s,'atcphy_usb3_pm_find')+'\n#endif\n' helpers+=ns['function'](s,'atcphy_usb3_pm_get')+'\n'+ns['function'](s,'atcphy_usb3_pm_put')+'\n' - helpers+=ns['function'](s,'atcphy_mux_is_noop')+'\n'+ns['function'](s,'atcphy_mux_update')+'\n' + # The port power steps and the retries that the mux update queues; the + # delayed work functions themselves are left to the KUnit tests. + helpers+=''.join(atc_function(s,name) for name in + ('atcphy_usb3_port_off','atcphy_usb3_port_on','atcphy_usb3_port_retry_delay', + 'atcphy_usb3_port_retry','atcphy_mux_in_mode')) + helpers+=ns['function'](s,'atcphy_mux_is_noop')+'\n' + helpers+=''.join(atc_function(s,name) for name in + ('atcphy_park_retry_delay','atcphy_park_retry','atcphy_park_due','atcphy_mux_change')) + helpers+=ns['function'](s,'atcphy_mux_update')+'\n' # Lane fixtures provide the target mapping and locked body; use the real callback here. marker=re.search(r'^static int atcphy_mux_set\(',s,re.M).group() helpers+=ns['block'](s,marker)+'\n' @@ -326,7 +433,7 @@ def ownership(root, usb_unreachable=False): ('static void typec_mux_unlock(', 'void typec_mux_abort(', 'int typec_mux_begin(', 'int typec_mux_apply(', 'int typec_mux_end(', 'int typec_mux_set('))+'\n' hcd_source=(root/'drivers/usb/core/hcd.c').read_text() hcd_death=ns['block'](hcd_source,'void usb_hc_died (')+'\n' - text=prefix+prod+'\n#define REQUIRES_PLATFORM_LOCKS '+str(int('device_lock(&pm->controller->dev)' in s))+'\n#define USB_REACHABLE '+str(int(not usb_unreachable))+'\n#define HCD_DEAD_ONLY '+str(int(False))+'\n'+pmfixture+helpers+typec_caller+hcd_death+suffix+controls + text=prefix+prod+'\n#define REQUIRES_PLATFORM_LOCKS '+str(int('device_lock(&pm->controller->dev)' in s))+'\n#define PORT_POWER '+str(int(port_power))+'\n#define LINK_RESUME '+str(int(link_resume))+'\n#define USB_REACHABLE '+str(int(not usb_unreachable))+'\n#define HCD_DEAD_ONLY '+str(int(False))+'\n'+pmfixture+helpers+typec_caller+hcd_death+suffix+controls return text def probe(root): diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py index af56c57419131..09557684964bf 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only # Copyright 2026 Ryan Murray """Actual mux prepare/set and TPS removal ordering; firmware/PHY leaves are mocks.""" -from common import block +from common import block, atc_function def make_unit(source, parent=False, reference=None): reference=source if reference is None else reference @@ -118,12 +118,16 @@ def make_unit(source, parent=False, reference=None): if 'static bool tps6598x_safe_after_role(' in core: functions+=block(core,'static bool tps6598x_safe_after_role(')+'\n' if not parent:functions+=block(core,'static void tps6598x_drop_partner(')+'\n'+block(core,'static int tps6598x_set_data_role(')+'\n' + # The fixed-hub SuperSpeed handover moved the locked mux body to atcphy_mux_switch(); + # atcphy_mux_set_locked() also reports whether a SuperSpeed port may come back up. + split=[atc_function(atc,name) for name in ('atcphy_usb3_port_ready','atcphy_mux_check','atcphy_mux_park','atcphy_mux_switch')] functions+=block(core,'static '+('void' if parent else 'int')+' tps6598x_disconnect(')+'\n'+block(core,'static '+('void' if parent else 'int')+' cd321x_remove(')+'\n'+block(core,'void tipd_remove(')+'\n' ordinary_phy=r''' /* Actual ordinary-port ATC power-off, mux recheck and USB2 init; MMIO leaves. */ #undef WARN_ON_ONCE #define WARN_ON_ONCE(x) ({ bool hit=(x);phy_warns+=hit;hit; }) #define fallthrough __attribute__((fallthrough)) +#define KUNIT_STATIC_STUB_REDIRECT(...) ((void)0) enum atcphy_mode { APPLE_ATCPHY_MODE_OFF, APPLE_ATCPHY_MODE_USB2, APPLE_ATCPHY_MODE_USB3, APPLE_ATCPHY_MODE_USB3_DP, APPLE_ATCPHY_MODE_DP, APPLE_ATCPHY_MODE_TBT, APPLE_ATCPHY_MODE_USB4 }; @@ -133,6 +137,7 @@ def make_unit(source, parent=False, reference=None): const struct { bool has_usb4,has_t8140_dp; } *hw; struct { void *usb2phy; } regs; bool fixed_usb2,host_active,mode_dirty,pipe_pending,usb_off_pending,ss_tunables,dp_tunables; + bool usb3_port_off,park_owed; enum atcphy_mode mode,typec_mode; enum atcphy_pipehandler_state pipe_state; }; @@ -149,7 +154,7 @@ def make_unit(source, parent=False, reference=None): static int atcphy_configure(struct apple_atcphy *p,enum atcphy_mode mode) { configure_calls++;lane_requests+=mode!=APPLE_ATCPHY_MODE_OFF;p->mode=mode;p->mode_dirty=false;return 0; } static void set32(void *p,unsigned v) { (void)p;(void)v;assert(false); } static void atcphy_usb2_power_on(struct apple_atcphy *p) { (void)p;assert(false); } -''' + '\n'.join(block(atc, marker) for marker in ( +''' + ''.join(split) + '\n'.join(block(atc, marker) for marker in ( 'static int atcphy_usb3_power_off(', 'static int atcphy_mux_set_locked(', 'static int atcphy_usb2_init(')) + r''' static int ordinary_power_off(void) { return atcphy_usb3_power_off(&usb_phy); } @@ -170,6 +175,10 @@ def make_unit(source, parent=False, reference=None): assert(ordinary_phy.typec_mode==APPLE_ATCPHY_MODE_OFF&&lane_requests==before); } ''' + if split[-1]: + # An ordinary port takes no SuperSpeed port down, so none may come up. + ordinary_phy=ordinary_phy.replace(' int ret=atcphy_mux_set_locked(&ordinary_phy,APPLE_ATCPHY_MODE_OFF);', + ' bool port_on;\n int ret=atcphy_mux_set_locked(&ordinary_phy,APPLE_ATCPHY_MODE_OFF,&port_on);\n assert(!port_on);',1) controls=r''' static struct typec_mux_dev phy_provider,display_provider; static struct typec_mux mux; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py index 1f17261136a9c..822ba20aa87b0 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py @@ -58,7 +58,8 @@ def unit_for(source): else {assert(d==&controller.dev&&controller_locked);if(!remover)atomic_store(&handoff_active,false);controller_locked=false;pthread_mutex_unlock(&controller_mutex);} }''' unit=unit.replace(old,new) - # The scheduler injection is a child-lookup interleaving, as in the review. + # The scheduler injection is a child-lookup interleaving, as in the review, or + # the root resume for a source that takes the SuperSpeed port down instead. unit=unit.replace('if(detach_at_lookup)usb_set_device_state(u,USB_STATE_NOTATTACHED);','if(inject_remove){inject_remove=false;start_remove();}\n if(detach_at_lookup)usb_set_device_state(u,USB_STATE_NOTATTACHED);',1) unit=unit.replace('static void pm_runtime_put_noidle(struct device *d) {','static void pm_runtime_put_noidle(struct device *d) {\n if(remover&&d==&host_device)return;',1) # Old source lacks platform-lock helpers. Keep fixture-only API definitions From 373c3271edb7da0489b5ab4ec63e1e199ade74f8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 19:34:47 +1100 Subject: [PATCH 1229/1579] usb: typec: mux: Let a provider declare its timeouts transient A mux provider can time out on a change that its hardware holds up for a while, and that goes through later unchanged. On a J700 the ATC PHY cannot park the PIPE of the controller behind the fixed hub while that controller still drives it, which a partner renegotiating power can keep up for several seconds. Such a timeout says nothing wrong about the state asked for, and a Type-C driver that gives the state up after a few quick retries leaves the port in the wrong mode until the next cable event. Let a provider declare that -ETIMEDOUT from its set callback is such a hold-up. That is a property of the provider, not of a mux handle: one handle can hold several providers, and on the rear connector of a J700 it holds the PHY and a DCP display route, whose own timeouts can be permanent. So note, for each request, whether the provider that failed it timed out having declared its timeouts transient, and let the consumer ask with typec_mux_timeout_transient() after a request that timed out. Nothing declares it yet, so no consumer changes. The apple-dcp host selftests extract the transaction code; their mux shims gain the two new fields. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/mux.c | 29 +++++++++++++++++++ drivers/usb/typec/mux.h | 1 + include/linux/usb/typec_mux.h | 13 +++++++++ .../selftests/drivers/gpu/apple-dcp/pm.py | 4 +-- .../drivers/gpu/apple-dcp/quiesce.py | 4 +-- 5 files changed, 47 insertions(+), 4 deletions(-) diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index b3ae7830f1ed4..836e7cb40a7b5 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -282,6 +282,8 @@ struct typec_mux { int srcu_index; unsigned int prepared; bool applied; + /* The last request timed out in a provider with transient timeouts. */ + bool timeout_transient; }; static struct typec_mux_context *typec_mux_context_get(struct fwnode_handle *fwnode) @@ -332,6 +334,28 @@ bool typec_mux_safe_after_role(struct typec_mux *mux) } EXPORT_SYMBOL_GPL(typec_mux_safe_after_role); +/** + * typec_mux_timeout_transient - Check whether a mux request timed out for now + * @mux: Mux handle + * + * For use after a request on @mux, by typec_mux_set() or by typec_mux_begin() + * and the calls that follow it, failed with -ETIMEDOUT. A handle can hold more + * than one provider, and only the one that timed out is asked: whether it + * declared its timeouts transient (&typec_mux_desc.transient_timeouts). Its + * hardware then only held the change up, and the same state can be applied + * again later. A timeout of any other provider can be permanent. + * + * The answer is about the last request on @mux: no other request on the same + * handle may run in between. + * + * Return: true if the provider that timed out declared its timeouts transient. + */ +bool typec_mux_timeout_transient(struct typec_mux *mux) +{ + return !IS_ERR_OR_NULL(mux) && mux->timeout_transient; +} +EXPORT_SYMBOL_GPL(typec_mux_timeout_transient); + static int mux_fwnode_match(struct device *dev, const void *fwnode) { if (!is_typec_mux_dev(dev)) @@ -508,6 +532,7 @@ int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state *state) mux->srcu_index = index; mux->owner = current; mux->transaction = *state; + mux->timeout_transient = false; for (i = 0; i < mux->num_mux_devs; i++) { struct typec_mux_dev *dev = mux->mux_devs[i]; @@ -549,6 +574,9 @@ int typec_mux_apply(struct typec_mux *mux) ret = dev->set(dev, &mux->transaction); if (ret && ret != -EOPNOTSUPP) { + /* Only the provider that failed knows what its timeout means. */ + mux->timeout_transient = ret == -ETIMEDOUT && + dev->transient_timeouts; typec_mux_abort(mux); return ret; } @@ -654,6 +682,7 @@ typec_mux_register(struct device *parent, const struct typec_mux_desc *desc) mux_dev->commit = desc->commit; mux_dev->abort = desc->abort; mux_dev->safe_after_role = desc->safe_after_role; + mux_dev->transient_timeouts = desc->transient_timeouts; device_initialize(&mux_dev->dev); mux_dev->dev.parent = parent; diff --git a/drivers/usb/typec/mux.h b/drivers/usb/typec/mux.h index 823c5529ca675..89c6061e0863f 100644 --- a/drivers/usb/typec/mux.h +++ b/drivers/usb/typec/mux.h @@ -21,6 +21,7 @@ struct typec_mux_dev { typec_mux_notify_fn_t abort; bool removed; bool safe_after_role; + bool transient_timeouts; }; #define to_typec_switch_dev(_dev_) container_of(_dev_, struct typec_switch_dev, dev) diff --git a/include/linux/usb/typec_mux.h b/include/linux/usb/typec_mux.h index 185da86c43d25..f853143048ded 100644 --- a/include/linux/usb/typec_mux.h +++ b/include/linux/usb/typec_mux.h @@ -105,6 +105,13 @@ struct typec_mux_desc { typec_mux_notify_fn_t abort; /* SAFE needs the USB role consumer to stop the shared PHY first. */ bool safe_after_role; + /* + * -ETIMEDOUT from @set only means that the hardware held the change up + * for now: the same state can be applied again later, and is worth + * applying again for longer than after other failures. Consumers learn + * it from typec_mux_timeout_transient(). + */ + bool transient_timeouts; }; #if IS_ENABLED(CONFIG_TYPEC) @@ -117,6 +124,7 @@ int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state *state); /* Apply the providers while retaining the transaction and ownership. */ int typec_mux_apply(struct typec_mux *mux); bool typec_mux_safe_after_role(struct typec_mux *mux); +bool typec_mux_timeout_transient(struct typec_mux *mux); int typec_mux_end(struct typec_mux *mux); void typec_mux_abort(struct typec_mux *mux); int typec_mux_prepare(struct typec_mux *mux, struct typec_mux_state *state); @@ -150,6 +158,11 @@ static inline bool typec_mux_safe_after_role(struct typec_mux *mux) return false; } +static inline bool typec_mux_timeout_transient(struct typec_mux *mux) +{ + return false; +} + static inline int typec_mux_end(struct typec_mux *mux) { return 0; } static inline void typec_mux_abort(struct typec_mux *mux) {} diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index be73d45fadcb2..c3dbcdac5a3a7 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -39,7 +39,7 @@ def ownership(root, usb_unreachable=False): int (*prepare)(struct typec_mux_dev *, struct typec_mux_state *); int (*finish)(struct typec_mux_dev *, struct typec_mux_state *); void (*commit)(struct typec_mux_dev *, struct typec_mux_state *); - void (*abort)(struct typec_mux_dev *, struct typec_mux_state *);bool removed; }; + void (*abort)(struct typec_mux_dev *, struct typec_mux_state *);bool removed, transient_timeouts; }; struct typec_mux_context { struct { bool held; } lock; }; struct task_struct { int unused; };static struct task_struct test_task; #define current (&test_task) @@ -48,7 +48,7 @@ def ownership(root, usb_unreachable=False): static void srcu_read_unlock(int *s,int i) { (void)s;(void)i; } #define mutex_trylock(m) ((m)->held ? 0 : ((m)->held=true)) #define mutex_unlock(m) ((m)->held=false) - struct typec_mux { unsigned int num_mux_devs; struct typec_mux_dev **mux_devs; struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied; }; + struct typec_mux { unsigned int num_mux_devs; struct typec_mux_dev **mux_devs; struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied, timeout_transient; }; #define IS_ERR_OR_NULL(p) (!(p))''') prefix=prefix.replace('#define WARN_ON_ONCE(v) assert(!(v))', 'static bool checked_warn(bool v) { assert(!v);return v; }\n#define WARN_ON_ONCE(v) checked_warn(v)') prefix=prefix.replace('#define guard(kind) (void)',r''' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py index 09557684964bf..88e07fb5bbfa5 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -41,7 +41,7 @@ def make_unit(source, parent=False, reference=None): #define TPS_STATUS_TO_TYPEC_VCONN(s) (s) #define TPS_STATUS_TO_TYPEC_DATAROLE(s) (s) struct typec_mux_state { void *alt; unsigned long mode; void *data; }; -struct typec_mux_dev { bool removed, safe_after_role; void (*commit)(struct typec_mux_dev *,struct typec_mux_state *); void (*abort)(struct typec_mux_dev *,struct typec_mux_state *); int (*finish)(struct typec_mux_dev *,struct typec_mux_state *); void *data; int (*set)(struct typec_mux_dev *,struct typec_mux_state *); int (*prepare)(struct typec_mux_dev *,struct typec_mux_state *); }; +struct typec_mux_dev { bool removed, safe_after_role; void (*commit)(struct typec_mux_dev *,struct typec_mux_state *); void (*abort)(struct typec_mux_dev *,struct typec_mux_state *); int (*finish)(struct typec_mux_dev *,struct typec_mux_state *); void *data; int (*set)(struct typec_mux_dev *,struct typec_mux_state *); int (*prepare)(struct typec_mux_dev *,struct typec_mux_state *); bool transient_timeouts; }; struct task_struct { int unused; }; static _Thread_local struct task_struct this_task; #define current (&this_task) @@ -53,7 +53,7 @@ def make_unit(source, parent=False, reference=None): static void srcu_read_unlock(int *s,int i) { (void)s;(void)i; } static int mutex_trylock(struct mutex *m) { return pthread_mutex_trylock(&m->value)==0; } static void mutex_unlock(struct mutex *m) { assert(!pthread_mutex_unlock(&m->value)); } -struct typec_mux { struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied; unsigned num_mux_devs; struct typec_mux_dev *mux_devs[3]; }; +struct typec_mux { struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied; unsigned num_mux_devs; struct typec_mux_dev *mux_devs[3]; bool timeout_transient; }; struct connector { bool connected; }; struct apple_dcp { int external_lock,external_error; bool external_cable,external_stopped; struct { bool connected; } dptxport[1]; void *hdmi_hpd; bool active; bool external,typec_cable_connected; void *dev; struct connector *typec_connector; }; struct apple_dcp_typec_route; From 81903767d050f2d39d2baf402539ce0fb831e2cd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 19:45:51 +1100 Subject: [PATCH 1230/1579] phy: apple: atc: Declare the J700 fixed-hub mux timeouts transient On a J700 the rear port's controller keeps running across Type-C mode changes, so the PHY parks its PIPE itself, after taking the controller's SuperSpeed root port down. The PIPE lock is only acknowledged once the controller lets go of the PIPE. With a Mac on the rear port that did not happen within the lock timeout twice in one day, both times in a burst of SMC power source elections, 11 and 18 "Elec Cause" lines over a few seconds, through which the Mac took the device role: phy-apple-atc 40b000000.phy: Pipehandler lock not acked. phy-apple-atc 40b000000.phy: Failed to switch PIPE to dummy: -110 sn201202x 1-0c: failed to place Type-C mux in safe mode: -110 The Mac then stayed on USB2 behind the fixed hub for the rest of the boot. A refused change leaves the lanes dirty and the SuperSpeed port down, and the next mux update that gets through puts both right: the safe state parks the PIPE, and the USB state routes it to USB3 and powers the port on. But the Type-C driver tries its whole state only a few times, about a second apart, which such a burst outlasts, and then waits for the next cable event, which need not come. Declare the mux timeouts of a J700 fixed-hub port transient, so that the Type-C driver keeps applying its whole state until it goes through. Ports without a fixed hub, every M1 and M2 port among them, and fixed-hub ports on other PHYs do not declare it. Only the PHY's own timeouts count: the DCP display route on the same connector declares nothing, and its timeouts keep the bounded retries. The declaration covers the PHY's other timeouts too, those of its power and PLL polls, which retrying does not clear. Telling them apart from a refused lock would mean changing what the shared poll helpers return to every other caller, and the Type-C driver stops retrying after about two minutes, which bounds them as well. The new tests run the safe state of a data role swap, and the USB state after it, against a PIPE lock that is refused until the elections settle, with the real mode change between stubbed USB steps. However often the lock was refused, a refusal leaves the port down with nothing queued, and the safe state and then the USB state applied again bring SuperSpeed back. They also check which PHYs declare the timeouts transient. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 173 +++++++++++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 38 +++++++- 2 files changed, 210 insertions(+), 1 deletion(-) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index 65e82a6d6edf0..6efddf059e18e 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -679,6 +679,176 @@ static void atcphy_test_host_start_without_usb3(struct kunit *test) KUNIT_EXPECT_EQ(test, atcphy_usb3_set_mode(phy, PHY_MODE_USB_HOST, 0), -EBUSY); } +/* + * The controller as the PIPE lock sees it: while @held, as through the PD + * elections of a Mac taking another data role, it does not acknowledge the + * lock. The USB side stubs share @steps. + */ +struct atcphy_test_election { + struct atcphy_test_steps steps; + bool held; +}; + +static int atcphy_test_lock(struct apple_atcphy *atcphy) +{ + struct atcphy_test_election *election = kunit_get_current_test()->priv; + + return election->held ? -ETIMEDOUT : 0; +} + +/* The host BIST of a USB3 handoff, which zeroed registers cannot run */ +static int atcphy_test_bist(struct apple_atcphy *atcphy) +{ + return 0; +} + +/* The lane change, as far as the PIPE handling depends on it */ +static int atcphy_test_lanes(struct apple_atcphy *atcphy, enum atcphy_mode mode) +{ + lockdep_assert_held(&atcphy->lock); + atcphy->mode = mode; + atcphy->mode_dirty = false; + return 0; +} + +/* + * The rear port of a J700 with a Mac on it: USB3, with the PIPE on the USB3 + * backend of the running host. The real mode change runs between the recorded + * USB side steps of atcphy_test_stub_steps(). + */ +static struct apple_atcphy *atcphy_test_alloc_election(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_fixed_hub(test); + struct atcphy_test_election *election; + + election = kunit_kzalloc(test, sizeof(*election), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, election); + test->priv = election; + kunit_activate_static_stub(test, atcphy_pipehandler_lock, atcphy_test_lock); + kunit_activate_static_stub(test, atcphy_configure_pipehandler_usb3_host_t8122, + atcphy_test_bist); + kunit_activate_static_stub(test, atcphy_configure, atcphy_test_lanes); + kunit_activate_static_stub(test, atcphy_usb3_pm_get, atcphy_test_pm_get); + kunit_activate_static_stub(test, atcphy_usb3_port_off, atcphy_test_port_off); + kunit_activate_static_stub(test, atcphy_usb3_port_on, atcphy_test_port_on); + kunit_activate_static_stub(test, atcphy_usb3_port_retry_delay, atcphy_test_retry_delay); + atcphy_test_fake_regs(test, atcphy); + atcphy->mode = APPLE_ATCPHY_MODE_USB3; + atcphy->typec_mode = APPLE_ATCPHY_MODE_USB3; + + return atcphy; +} + +static const struct typec_mux_state atcphy_test_safe = { .mode = TYPEC_STATE_SAFE }; +static const struct typec_mux_state atcphy_test_usb = { .mode = TYPEC_STATE_USB }; + +/* A refused PIPE change leaves the port down, and nothing queued */ +static void atcphy_test_election_refused(struct kunit *test, struct apple_atcphy *atcphy, + const struct typec_mux_state *state) +{ + struct atcphy_test_election *election = test->priv; + + memset(election->steps.log, 0, sizeof(election->steps.log)); + KUNIT_EXPECT_EQ(test, atcphy_mux_update(atcphy, state), -ETIMEDOUT); + KUNIT_EXPECT_STREQ(test, election->steps.log, "g0"); + KUNIT_EXPECT_TRUE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_FALSE(test, atcphy_usb3_port_ready(atcphy)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&atcphy->usb3_port_work)); +} + +/* The safe state and then the USB state, as the Type-C driver applies them again */ +static void atcphy_test_election_redrive(struct kunit *test, struct apple_atcphy *atcphy) +{ + struct atcphy_test_election *election = test->priv; + char *log = election->steps.log; + + memset(log, 0, sizeof(election->steps.log)); + KUNIT_EXPECT_EQ(test, atcphy_mux_update(atcphy, &atcphy_test_safe), 0); + KUNIT_EXPECT_STREQ(test, log, "g0"); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB2); + KUNIT_EXPECT_FALSE(test, atcphy->mode_dirty); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + KUNIT_EXPECT_TRUE(test, atcphy->usb3_port_off); + + memset(log, 0, sizeof(election->steps.log)); + KUNIT_EXPECT_EQ(test, atcphy_mux_update(atcphy, &atcphy_test_usb), 0); + KUNIT_EXPECT_STREQ(test, log, "g01"); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB3); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_USB3); + KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_TRUE(test, atcphy_usb3_port_ready(atcphy)); +} + +/* + * SuperSpeed comes back after a data role swap whose safe state found the PIPE + * held, however many times it was refused, once the Type-C driver applies its + * whole state again after the elections. + */ +static void atcphy_test_election_held_safe(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_election(test); + struct atcphy_test_election *election = test->priv; + unsigned int i; + + election->held = true; + /* Many more times than the Type-C driver used to retry */ + for (i = 0; i < 8; i++) + atcphy_test_election_refused(test, atcphy, &atcphy_test_safe); + /* The PIPE could not be parked, so the lanes were left alone */ + KUNIT_EXPECT_TRUE(test, atcphy->mode_dirty); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_USB3); + + election->held = false; + atcphy_test_election_redrive(test, atcphy); +} + +/* The same if the elections go on into the USB state that follows the safe one */ +static void atcphy_test_election_held_usb(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_election(test); + struct atcphy_test_election *election = test->priv; + + election->held = true; + atcphy_test_election_refused(test, atcphy, &atcphy_test_safe); + election->held = false; + KUNIT_EXPECT_EQ(test, atcphy_mux_update(atcphy, &atcphy_test_safe), 0); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + + /* The lanes are rebuilt for USB3, but the PIPE cannot follow */ + election->held = true; + atcphy_test_election_refused(test, atcphy, &atcphy_test_usb); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB3); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + + election->held = false; + atcphy_test_election_redrive(test, atcphy); +} + +/* + * Only the fixed-hub ports of a J700 have the Type-C driver retry a timeout + * until it passes; every M1 and M2 port, and fixed-hub ports on other PHYs, + * keep its bounded retries. + */ +static void atcphy_test_transient_timeouts(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc(test, true); + + atcphy->hw = &atcphy_hw_t8140; + KUNIT_EXPECT_TRUE(test, atcphy_mux_transient_timeouts(atcphy)); + atcphy->fixed_usb2 = false; + KUNIT_EXPECT_FALSE(test, atcphy_mux_transient_timeouts(atcphy)); + + atcphy->hw = &atcphy_hw_t8103; + KUNIT_EXPECT_FALSE(test, atcphy_mux_transient_timeouts(atcphy)); + atcphy->hw = &atcphy_hw_t8122; + KUNIT_EXPECT_FALSE(test, atcphy_mux_transient_timeouts(atcphy)); + atcphy->fixed_usb2 = true; + KUNIT_EXPECT_FALSE(test, atcphy_mux_transient_timeouts(atcphy)); + atcphy->hw = &atcphy_hw_t8142; + KUNIT_EXPECT_FALSE(test, atcphy_mux_transient_timeouts(atcphy)); +} + #if IS_REACHABLE(CONFIG_USB) /* Only the controller's SuperSpeed root hub is ever handed the port changes */ static void atcphy_test_usb3_root(struct kunit *test) @@ -825,6 +995,9 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_stopped_power_off), KUNIT_CASE(atcphy_test_stopped_reset), KUNIT_CASE(atcphy_test_host_start_without_usb3), + KUNIT_CASE(atcphy_test_election_held_safe), + KUNIT_CASE(atcphy_test_election_held_usb), + KUNIT_CASE(atcphy_test_transient_timeouts), #if IS_REACHABLE(CONFIG_USB) KUNIT_CASE(atcphy_test_usb3_root), #endif diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index b6fdf6b496b4c..a53eeb5f9f5a7 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1112,8 +1112,13 @@ static int atcphy_pipehandler_lock(struct apple_atcphy *atcphy) { int ret; u32 reg; - u32 requested = readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ); + u32 requested; + /* The tests stand in for the controller that acknowledges the lock */ + if (IS_ENABLED(CONFIG_PHY_APPLE_ATC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(atcphy_pipehandler_lock, atcphy); + + requested = readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ); dev_dbg(atcphy->dev, "PIPE request: req=%#x mode=%u pipe=%u host=%u dirty=%u\n", requested, atcphy->mode, atcphy->pipe_state, atcphy->host_active, atcphy->mode_dirty); @@ -1198,6 +1203,9 @@ static int atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *atc int ret, error = 0; u32 reg; + if (IS_ENABLED(CONFIG_PHY_APPLE_ATC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(atcphy_configure_pipehandler_usb3_host_t8122, atcphy); + core_set32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_CLOCK_EN); /* This should be a mask, but unsure what bits should be cleared */ writel(ACIOPHY_BIST_MAC_USBPHY_CFG_OV1_LN0_PIPE_RESET_N_OV | FIELD_PREP(ACIOPHY_BIST_MAC_USBPHY_CFG_OV1_LN0_PWR_DOWN_OV, 2), @@ -4059,6 +4067,33 @@ static void atcphy_typec_mux_unregister(void *data) typec_mux_unregister(data); } +/* + * A controller behind a fixed hub keeps running across mode changes, so the PHY + * parks its PIPE with the controller's SuperSpeed port taken down. The PIPE + * lock is still only acknowledged once the controller lets go of the PIPE, and + * on a J700 that need not happen within the lock timeout while the partner + * renegotiates. A Mac on the port did so through a burst of power elections + * lasting several seconds, and the safe state of its data role swap then found + * the PIPE held: + * + * phy-apple-atc 40b000000.phy: Pipehandler lock not acked. + * phy-apple-atc 40b000000.phy: Failed to switch PIPE to dummy: -110 + * sn201202x 1-0c: failed to place Type-C mux in safe mode: -110 + * + * The failed change leaves the lanes dirty and the SuperSpeed port down, and + * the next mux update that gets through puts both right. Such a timeout says + * nothing about the state asked for, so tell the Type-C driver to keep applying + * its whole state, the safe state and then the USB one, until it goes through. + * Only a timeout of this PHY counts, not one of the display route that shares + * the connector. The PHY's other timeouts, of its power and PLL polls, are + * faults that retrying does not clear, but the Type-C driver stops retrying + * after about two minutes, which bounds those as well. + */ +static bool atcphy_mux_transient_timeouts(const struct apple_atcphy *atcphy) +{ + return atcphy->fixed_usb2 && atcphy->hw->has_t8140_dp; +} + static int atcphy_probe_mux(struct apple_atcphy *atcphy) { struct typec_mux_dev *mux; @@ -4067,6 +4102,7 @@ static int atcphy_probe_mux(struct apple_atcphy *atcphy) .fwnode = atcphy->dev->fwnode, .set = atcphy_mux_set, .safe_after_role = !atcphy->fixed_usb2, + .transient_timeouts = atcphy_mux_transient_timeouts(atcphy), }; mux = typec_mux_register(atcphy->dev, &mux_desc); From ed289aabb019c8dfb1ee8575ba55430d8f43089c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 19:45:51 +1100 Subject: [PATCH 1231/1579] usb: typec: tipd: Keep applying a Type-C state that a busy mux holds up When a mux update fails, the CD321x update work hands the port to the cable-state recovery: a fresh read and a new attempt at the whole state, three times, then nothing until the next controller interrupt. Each attempt takes about a second, the resume read and the update being debounced by half a second each. On the rear port of a J700 that is not enough. The PHY cannot park its PIPE while the controller behind the fixed hub still drives it, and a Mac renegotiating power, in bursts of up to about 5.5 s, kept that up past the last attempt. Its safe state failed with -ETIMEDOUT each time, so the USB state of the new data role was never asked for, and with no further interrupt the Mac stayed on USB2 for the rest of the boot. Such a failure says nothing about the cable state, which was read fine. When the mux provider that failed a change timed out having declared its timeouts transient, do as for events that could not be acknowledged: do not charge the attempt to the cable-state budget. Apply the whole state again from a fresh read, starting with the safe state, after a delay that doubles from 500 ms up to 10 s. A controller event during the backoff brings the retry forward to the usual debounce, so that its state is applied promptly and the retry that was due later finds nothing left to do. Sleep cancels the retry and the resume read takes it over; removal, or any failure that is not such a hold-up, ends it, the latter going back to the bounded retries with no backoff left over. Only the first failure of such a series is reported, and the change that finally goes through is logged. The provider cannot tell a held-up lock from its other timeouts, so a PHY that stays stuck is not retried for good: after 16 retries, some two minutes and over twenty times the longest burst seen, the port gives up with one warning, as the budget does, and the next controller event starts over. The rear connector's mux handle holds the DCP display route as well as the PHY, and the route declares nothing, so whether a change was held up is asked of the provider that failed it, right after each mux request. The safe state of the update work now goes through the driver's own handle too, the one it asks, rather than the port's: both are the connector's, with the same providers, and the state is the same. Only the PHY of a J700 fixed-hub port declares its timeouts transient. For every other provider, any M1 or M2 port's among them and the DCP display routes, and for any other error, the driver behaves as before. The new tests drive the interrupt, resume read and update work of a port whose mux is refused for longer than the budget lasts, and expect the safe state and then the USB state once it lets go, with no further event, also on a J700 rear connector, where the display route follows the PHY on the handle. They also cover the backoff and its limit, an event that takes over from the backoff, sleep and removal during the retries, giving up on a PHY that stays stuck until the next event, a series that another error ends, and that other muxes, the display route on the J700 connector, and other errors keep the bounded retries. The apple-dcp host selftests follow: the units that extract the update work extract the new helpers with it, against a mux that does not declare its timeouts transient, and the probe unit runs the new state transitions. Fixes: 70f7c21e7d32 ("phy: apple: atc: Route the PIPE from the Type-C mux on a fixed-hub port") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/Kconfig | 5 + drivers/usb/typec/tipd/Makefile | 2 +- drivers/usb/typec/tipd/cd321x-mux-test.c | 812 ++++++++++++++++++ drivers/usb/typec/tipd/cd321x-pm-test.c | 137 +++ drivers/usb/typec/tipd/cd321x-pm.h | 74 ++ drivers/usb/typec/tipd/core.c | 90 +- .../drivers/gpu/apple-dcp/lifecycle.py | 22 +- .../selftests/drivers/gpu/apple-dcp/pm.py | 42 +- 8 files changed, 1167 insertions(+), 17 deletions(-) create mode 100644 drivers/usb/typec/tipd/cd321x-mux-test.c diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index b8a904446109a..e0ee8fa1f55b6 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -42,6 +42,11 @@ config TYPEC_TIPD_KUNIT_TEST controller keeps raising its interrupt. Also test that a USB4/Thunderbolt router which never registers its switch does not keep the port from probing. + Also test that a Type-C state which a mux provider with transient + timeouts holds up is applied again until it goes through, for up to + about two minutes, also next to a display route that declares + nothing, and that other providers and errors keep the bounded + retries. These tests run without accessing a physical Type-C controller. config TYPEC_SN201202X_KUNIT_TEST diff --git a/drivers/usb/typec/tipd/Makefile b/drivers/usb/typec/tipd/Makefile index fcb10573a05b1..0138a734b7a2e 100644 --- a/drivers/usb/typec/tipd/Makefile +++ b/drivers/usb/typec/tipd/Makefile @@ -13,5 +13,5 @@ obj-$(CONFIG_TYPEC_SN201202X) += sn201202x.o sn201202x-y := spmi.o obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o cd321x-tbt-test.o \ - cd321x-irq-test.o + cd321x-irq-test.o cd321x-mux-test.o obj-$(CONFIG_TYPEC_SN201202X_KUNIT_TEST) += sn201202x-pm-test.o diff --git a/drivers/usb/typec/tipd/cd321x-mux-test.c b/drivers/usb/typec/tipd/cd321x-mux-test.c new file mode 100644 index 0000000000000..539ee405dc924 --- /dev/null +++ b/drivers/usb/typec/tipd/cd321x-mux-test.c @@ -0,0 +1,812 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for CD321x Type-C states that the mux holds up + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tps6598x.h" + +#define TEST_REG_INT_EVENT1 0x14 +#define TEST_REG_INT_CLEAR1 0x18 +#define TEST_REG_STATUS 0x1a +#define TEST_REG_DATA_STATUS 0x5f + +/* Longer than the bursts of power elections seen so far, up to ~5.5 s */ +#define TEST_ELECTIONS_MS 8000 + +/* A connector whose mode switch is the PHY, as on Apple machines */ +static const struct property_entry cd321x_mux_test_switch_props[] = { + PROPERTY_ENTRY_BOOL("mode-switch"), + { } +}; + +static const struct software_node cd321x_mux_test_phy_node = { + .name = "cd321x-mux-test-phy", + .properties = cd321x_mux_test_switch_props, +}; + +/* The DCP display route, the second mode switch of a J700's rear connector */ +static const struct software_node cd321x_mux_test_display_node = { + .name = "cd321x-mux-test-display", + .properties = cd321x_mux_test_switch_props, +}; + +static const struct software_node_ref_args cd321x_mux_test_refs[] = { + SOFTWARE_NODE_REFERENCE(&cd321x_mux_test_phy_node), +}; + +static const struct property_entry cd321x_mux_test_connector_props[] = { + PROPERTY_ENTRY_REF_ARRAY("mode-switch", cd321x_mux_test_refs), + { } +}; + +static const struct software_node cd321x_mux_test_connector = { + .name = "cd321x-mux-test-connector", + .properties = cd321x_mux_test_connector_props, +}; + +/* The rear connector of a J700, as in t8140-j700.dts */ +static const struct software_node_ref_args cd321x_mux_test_j700_refs[] = { + SOFTWARE_NODE_REFERENCE(&cd321x_mux_test_phy_node), + SOFTWARE_NODE_REFERENCE(&cd321x_mux_test_display_node), +}; + +static const struct property_entry cd321x_mux_test_j700_connector_props[] = { + PROPERTY_ENTRY_REF_ARRAY("mode-switch", cd321x_mux_test_j700_refs), + { } +}; + +static const struct software_node cd321x_mux_test_j700_connector = { + .name = "cd321x-mux-test-j700-connector", + .properties = cd321x_mux_test_j700_connector_props, +}; + +static const struct software_node * const cd321x_mux_test_nodes[] = { + &cd321x_mux_test_phy_node, + &cd321x_mux_test_display_node, + &cd321x_mux_test_connector, + &cd321x_mux_test_j700_connector, + NULL +}; + +struct cd321x_mux_test; + +/* + * A mode switch on the connector. While @held, as the rear port PHY of a J700 + * is through the power elections of a Mac taking another data role, when it + * cannot park the PIPE, it fails every change with @held_ret. @log records the + * states asked for: s(afe), u(sb) or o(ther), in capitals when refused. The + * display route also fails to quiesce, before any provider changes, while + * @prepare_held. + */ +struct cd321x_mux_test_phy { + struct cd321x_mux_test *test; + bool held; + bool prepare_held; + int held_ret; + char log[64]; + unsigned int last_refused_ms; +}; + +struct cd321x_mux_test { + struct cd321x cd; + u64 latched; + u32 status; + u32 data_status; + struct cd321x_mux_test_phy phy; + /* Only on a J700's rear connector, where it does not declare anything */ + struct cd321x_mux_test_phy display; + /* The time the scheduled work has been run at */ + unsigned int now_ms; +}; + +static int cd321x_mux_test_read(void *context, const void *reg, size_t reg_size, + void *val, size_t val_size) +{ + struct cd321x_mux_test *t = context; + + memset(val, 0, val_size); + switch (*(const u8 *)reg) { + case TEST_REG_INT_EVENT1: + memcpy(val, &t->latched, min(val_size, sizeof(t->latched))); + break; + case TEST_REG_STATUS: + memcpy(val, &t->status, min(val_size, sizeof(t->status))); + break; + case TEST_REG_DATA_STATUS: + memcpy(val, &t->data_status, min(val_size, sizeof(t->data_status))); + break; + } + + return 0; +} + +static int cd321x_mux_test_write(void *context, const void *data, size_t count) +{ + struct cd321x_mux_test *t = context; + const u8 *buf = data; + u64 clear = 0; + + if (buf[0] == TEST_REG_INT_CLEAR1) { + memcpy(&clear, buf + 1, min(count - 1, sizeof(clear))); + t->latched &= ~clear; + } + + return 0; +} + +static const struct regmap_bus cd321x_mux_test_bus = { + .read = cd321x_mux_test_read, + .write = cd321x_mux_test_write, + .reg_format_endian_default = REGMAP_ENDIAN_NATIVE, + .val_format_endian_default = REGMAP_ENDIAN_NATIVE, +}; + +static int cd321x_mux_test_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + struct cd321x_mux_test_phy *phy = typec_mux_get_drvdata(mux); + size_t len = strlen(phy->log); + char step; + + if (state->mode == TYPEC_STATE_SAFE) + step = 's'; + else if (state->mode == TYPEC_STATE_USB && !state->alt) + step = 'u'; + else + step = 'o'; + if (phy->held) { + step = toupper(step); + phy->last_refused_ms = phy->test->now_ms; + } + if (len < sizeof(phy->log) - 1) + phy->log[len] = step; + + return phy->held ? phy->held_ret : 0; +} + +static int cd321x_mux_test_prepare(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + struct cd321x_mux_test_phy *phy = typec_mux_get_drvdata(mux); + + return phy->prepare_held ? phy->held_ret : 0; +} + +static int cd321x_mux_test_psy_get(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + val->intval = 0; + return 0; +} + +static const enum power_supply_property cd321x_mux_test_psy_props[] = { + POWER_SUPPLY_PROP_ONLINE, +}; + +static const struct power_supply_desc cd321x_mux_test_psy_desc = { + .name = "cd321x-mux-test", + .type = POWER_SUPPLY_TYPE_USB, + .properties = cd321x_mux_test_psy_props, + .num_properties = ARRAY_SIZE(cd321x_mux_test_psy_props), + .get_property = cd321x_mux_test_psy_get, +}; + +static void cd321x_mux_test_unregister_nodes(void *unused) +{ + software_node_unregister_node_group(cd321x_mux_test_nodes); +} + +static void cd321x_mux_test_unregister_mux(void *mux) +{ + typec_mux_unregister(mux); +} + +static void cd321x_mux_test_unregister_port(void *port) +{ + typec_unregister_port(port); +} + +static void cd321x_mux_test_put_mux(void *mux) +{ + typec_mux_put(mux); +} + +static void cd321x_mux_test_unregister_psy(void *psy) +{ + power_supply_unregister(psy); +} + +static void cd321x_mux_test_stop(void *data) +{ + struct cd321x *cd = data; + + scoped_guard(mutex, &cd->tps.lock) + cd321x_pm_remove(&cd->pm); + cancel_delayed_work_sync(&cd->resume_work); + cancel_delayed_work_sync(&cd->update_work); + if (!IS_ERR_OR_NULL(cd->tps.partner)) + typec_unregister_partner(cd->tps.partner); + cd->tps.partner = NULL; +} + +/* + * A running port with nothing attached, whose mux is the test PHY, and on a + * @j700 rear connector the display route after it. Only a PHY with @transient + * timeouts, the rear port PHY of a J700, has the driver retry its timeouts + * until they pass. + */ +static struct cd321x_mux_test *cd321x_mux_test_setup(struct kunit *test, bool transient, + bool j700) +{ + struct typec_mux_desc mux_desc = { + .set = cd321x_mux_test_set, + .transient_timeouts = transient, + }; + struct typec_mux_desc display_desc = { + .set = cd321x_mux_test_set, + .prepare = cd321x_mux_test_prepare, + .name = "cd321x-mux-test-display", + }; + struct typec_capability cap = { + .type = TYPEC_PORT_DRP, + .data = TYPEC_PORT_DRD, + .revision = USB_TYPEC_REV_1_2, + .prefer_role = TYPEC_NO_PREFERRED_ROLE, + }; + struct power_supply_config psy_cfg = {}; + struct fwnode_handle *connector; + struct typec_mux_dev *mux_dev; + struct cd321x_mux_test *t; + struct device *dev; + int ret; + + ret = software_node_register_node_group(cd321x_mux_test_nodes); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, cd321x_mux_test_unregister_nodes, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + connector = software_node_fwnode(j700 ? &cd321x_mux_test_j700_connector : + &cd321x_mux_test_connector); + mux_desc.fwnode = software_node_fwnode(&cd321x_mux_test_phy_node); + display_desc.fwnode = software_node_fwnode(&cd321x_mux_test_display_node); + + dev = kunit_device_register(test, "cd321x-mux-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + t->phy.test = t; + t->phy.held_ret = -ETIMEDOUT; + t->display.test = t; + t->display.held_ret = -ETIMEDOUT; + + mux_desc.drvdata = &t->phy; + mux_dev = typec_mux_register(dev, &mux_desc); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mux_dev); + ret = kunit_add_action_or_reset(test, cd321x_mux_test_unregister_mux, mux_dev); + KUNIT_ASSERT_EQ(test, ret, 0); + if (j700) { + display_desc.drvdata = &t->display; + mux_dev = typec_mux_register(dev, &display_desc); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mux_dev); + ret = kunit_add_action_or_reset(test, cd321x_mux_test_unregister_mux, mux_dev); + KUNIT_ASSERT_EQ(test, ret, 0); + } + + mutex_init(&t->cd.tps.lock); + t->cd.tps.dev = dev; + t->cd.tps.data = &tipd_cd321x_data; + t->cd.tps.regmap = devm_regmap_init(dev, &cd321x_mux_test_bus, t, + &tps6598x_regmap_config); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->cd.tps.regmap); + dev_set_drvdata(dev, &t->cd.tps); + + cap.fwnode = connector; + cap.driver_data = &t->cd.tps; + t->cd.tps.port = typec_register_port(dev, &cap); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->cd.tps.port); + ret = kunit_add_action_or_reset(test, cd321x_mux_test_unregister_port, t->cd.tps.port); + KUNIT_ASSERT_EQ(test, ret, 0); + + t->cd.mux = fwnode_typec_mux_get(connector); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->cd.mux); + ret = kunit_add_action_or_reset(test, cd321x_mux_test_put_mux, t->cd.mux); + KUNIT_ASSERT_EQ(test, ret, 0); + + psy_cfg.drv_data = &t->cd.tps; + t->cd.tps.psy = power_supply_register(dev, &cd321x_mux_test_psy_desc, &psy_cfg); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->cd.tps.psy); + ret = kunit_add_action_or_reset(test, cd321x_mux_test_unregister_psy, t->cd.tps.psy); + KUNIT_ASSERT_EQ(test, ret, 0); + + cd321x_init_work(&t->cd); + t->cd.pm.phase = CD321X_PM_RUNNING; + ret = kunit_add_action_or_reset(test, cd321x_mux_test_stop, &t->cd); + KUNIT_ASSERT_EQ(test, ret, 0); + + return t; +} + +static struct cd321x_mux_test *cd321x_mux_test_port(struct kunit *test, bool transient) +{ + return cd321x_mux_test_setup(test, transient, false); +} + +/* The rear port of a J700: its PHY declares its timeouts transient. */ +static struct cd321x_mux_test *cd321x_mux_test_j700_port(struct kunit *test) +{ + return cd321x_mux_test_setup(test, true, true); +} + +/* A controller event, delivered as the interrupt handler would get it */ +static void cd321x_mux_test_event(struct cd321x_mux_test *t, u64 event) +{ + t->latched |= event; + t->cd.tps.data->irq_handler(0, &t->cd.tps); +} + +/* A Mac attaches, the port being the USB host */ +static void cd321x_mux_test_attach(struct cd321x_mux_test *t) +{ + t->status = TPS_STATUS_PLUG_PRESENT; + t->data_status = TPS_DATA_STATUS_DATA_CONNECTION | TPS_DATA_STATUS_USB2_CONNECTION | + TPS_DATA_STATUS_USB3_CONNECTION; + cd321x_mux_test_event(t, APPLE_CD_REG_INT_PLUG_EVENT | + APPLE_CD_REG_INT_DATA_STATUS_UPDATE); +} + +/* The delay the work was scheduled with, from now */ +static unsigned int cd321x_mux_test_due_ms(struct delayed_work *dwork) +{ + long left = (long)(dwork->timer.expires - jiffies); + + return left > 0 ? jiffies_to_msecs(left) : 0; +} + +/* + * Run the next work of the port, the one due first, as if its delay had passed. + * Returns false if none is scheduled. + */ +static bool cd321x_mux_test_next(struct cd321x_mux_test *t) +{ + struct delayed_work *resume = &t->cd.resume_work, *update = &t->cd.update_work; + struct delayed_work *next; + + /* A work whose delay ran out meanwhile may be running already. */ + if (!delayed_work_pending(resume) && !delayed_work_pending(update)) { + flush_delayed_work(resume); + flush_delayed_work(update); + } + if (delayed_work_pending(resume) && + (!delayed_work_pending(update) || + time_before_eq(resume->timer.expires, update->timer.expires))) + next = resume; + else if (delayed_work_pending(update)) + next = update; + else + return false; + + t->now_ms += cd321x_mux_test_due_ms(next); + flush_delayed_work(next); + return true; +} + +/* Run the work of the port until none is left, which must be soon. */ +static void cd321x_mux_test_settle(struct kunit *test, struct cd321x_mux_test *t) +{ + unsigned int i; + + for (i = 0; i < 16 && cd321x_mux_test_next(t); i++) + ; + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.update_work)); +} + +static enum cd321x_pm_phase cd321x_mux_test_phase(struct cd321x_mux_test *t) +{ + guard(mutex)(&t->cd.tps.lock); + return t->cd.pm.phase; +} + +/* + * The delay of the retry of a held mux. Unlike what is left of it, this does + * not shrink as real time passes while the tests run. + */ +static unsigned int cd321x_mux_test_backoff_ms(struct cd321x_mux_test *t) +{ + guard(mutex)(&t->cd.tps.lock); + return t->cd.pm.mux_retry_ms; +} + +/* + * The resume read is queued without delay, so with no timer, rather than after + * a backoff. A worker may run it already. + */ +static void cd321x_mux_test_expect_read_now(struct kunit *test, struct cd321x_mux_test *t) +{ + struct delayed_work *resume = &t->cd.resume_work; + + KUNIT_EXPECT_FALSE(test, timer_pending(&resume->timer)); + flush_delayed_work(resume); +} + +/* The state is applied: the mux set to USB, and nothing retried any more */ +static void cd321x_mux_test_expect_usb(struct kunit *test, struct cd321x_mux_test *t) +{ + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_RUNNING); + KUNIT_EXPECT_TRUE(test, t->cd.state_valid); + KUNIT_EXPECT_EQ(test, t->cd.state.mode, TYPEC_STATE_USB); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); + KUNIT_EXPECT_NOT_NULL(test, t->cd.tps.partner); +} + +/* + * Through the elections the PHY refuses the safe state of the new connection + * for longer than the three cable-state attempts last. A mux request for USB + * must still follow once they settle, with no further controller event. + */ +static void cd321x_mux_held_past_budget_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + + t->phy.held = true; + cd321x_mux_test_attach(t); + while (t->now_ms < TEST_ELECTIONS_MS) { + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + KUNIT_ASSERT_NE(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + } + /* Refused well past the third attempt, never asked for USB */ + KUNIT_EXPECT_GT(test, t->phy.last_refused_ms, 3000U); + KUNIT_EXPECT_NULL(test, strpbrk(t->phy.log, "uU")); + KUNIT_EXPECT_GE(test, strlen(t->phy.log), (size_t)CD321X_RESUME_ATTEMPTS + 1); + + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* Retried more and more slowly while held, but at least every ten seconds */ +static void cd321x_mux_held_backs_off_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + unsigned int i, first = 0, delay = 0; + + t->phy.held = true; + cd321x_mux_test_attach(t); + for (i = 0; i < 8 * CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + if (!delayed_work_pending(&t->cd.resume_work)) + continue; + delay = cd321x_mux_test_due_ms(&t->cd.resume_work); + if (!first) + first = delay; + KUNIT_EXPECT_LE(test, delay, (unsigned int)CD321X_MUX_RETRY_MAX_MS); + KUNIT_EXPECT_NE(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_GT(test, t->cd.pm.attempts_left, 0); + } + KUNIT_EXPECT_GE(test, delay, 8 * first); + KUNIT_EXPECT_GT(test, t->now_ms, 30000U); + + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* + * A controller event during the backoff takes over from the retry: its state + * is applied after the usual debounce, and the retry that was due later steps + * aside rather than repeating the change. + */ +static void cd321x_mux_held_event_takes_over_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + + t->phy.held = true; + cd321x_mux_test_attach(t); + while (!delayed_work_pending(&t->cd.resume_work) || + cd321x_mux_test_backoff_ms(t) < 4000) + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + + /* The elections end with the Mac taking the device role */ + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + t->data_status |= TPS_DATA_STATUS_USB_DATA_ROLE; + cd321x_mux_test_event(t, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_LE(test, cd321x_mux_test_due_ms(&t->cd.resume_work), 500U); + + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + cd321x_mux_test_expect_usb(test, t); + KUNIT_EXPECT_LT(test, t->now_ms, TEST_ELECTIONS_MS); +} + +/* Sleep cancels the retry, and the resume read takes it over */ +static void cd321x_mux_held_across_sleep_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + struct device *dev = t->cd.tps.dev; + unsigned int i; + + t->phy.held = true; + cd321x_mux_test_attach(t); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + + KUNIT_ASSERT_EQ(test, tipd_prepare(dev), 0); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.update_work)); + tipd_complete(dev); + /* The resume read is queued without delay: a worker may run it already. */ + flush_delayed_work(&t->cd.resume_work); + + /* Still held after resume: the state is retried, not given up */ + memset(t->phy.log, 0, sizeof(t->phy.log)); + for (i = 0; i < 6 * CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + KUNIT_EXPECT_NE(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + } + KUNIT_EXPECT_GT(test, strlen(t->phy.log), (size_t)CD321X_RESUME_ATTEMPTS + 1); + + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* Removal ends the retry */ +static void cd321x_mux_held_ends_on_removal_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + + t->phy.held = true; + cd321x_mux_test_attach(t); + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + + scoped_guard(mutex, &t->cd.tps.lock) + cd321x_pm_remove(&t->cd.pm); + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, ""); +} + +/* + * Every other mux, such as the PHY of any M1 or M2 port, keeps the bounded + * cable-state retries: the driver gives up after them, and the next event + * starts over. + */ +static void cd321x_mux_timeout_bounded_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, false); + + t->phy.held = true; + cd321x_mux_test_attach(t); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "SSSS"); + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); + + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_event(t, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + flush_delayed_work(&t->cd.resume_work); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* and on the J700, any failure but a timeout does too */ +static void cd321x_mux_other_error_bounded_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + + t->phy.held = true; + t->phy.held_ret = -EBUSY; + cd321x_mux_test_attach(t); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "SSSS"); + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); +} + +/* + * The rear connector of a J700 has a second mode switch after the PHY, the DCP + * display route, which does not declare its timeouts transient. A refusal of + * the PHY is still retried past the budget, and once the elections settle + * both providers get the safe state and then the USB one. + */ +static void cd321x_mux_j700_held_past_budget_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_j700_port(test); + + t->phy.held = true; + cd321x_mux_test_attach(t); + while (t->now_ms < TEST_ELECTIONS_MS) { + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + KUNIT_ASSERT_NE(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + } + KUNIT_EXPECT_GT(test, t->phy.last_refused_ms, 3000U); + KUNIT_EXPECT_NULL(test, strpbrk(t->phy.log, "uU")); + KUNIT_EXPECT_GE(test, strlen(t->phy.log), (size_t)CD321X_RESUME_ATTEMPTS + 1); + + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + memset(t->display.log, 0, sizeof(t->display.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + KUNIT_EXPECT_STREQ(test, t->display.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* There a timeout of the display route keeps the bounded retries. */ +static void cd321x_mux_j700_display_timeout_bounded_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_j700_port(test); + + t->display.held = true; + cd321x_mux_test_attach(t); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->display.log, "SSSS"); + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); + + t->display.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + memset(t->display.log, 0, sizeof(t->display.log)); + cd321x_mux_test_event(t, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + flush_delayed_work(&t->cd.resume_work); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + KUNIT_EXPECT_STREQ(test, t->display.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* + * So does a timeout of the display route while it quiesces the display, before + * any provider changes, even right after the PHY held changes up. + */ +static void cd321x_mux_j700_display_prepare_timeout_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_j700_port(test); + + t->phy.held = true; + cd321x_mux_test_attach(t); + while (!delayed_work_pending(&t->cd.resume_work) || + cd321x_mux_test_backoff_ms(t) < 2000) + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + + t->phy.held = false; + t->display.prepare_held = true; + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, ""); + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); + + t->display.prepare_held = false; + memset(t->display.log, 0, sizeof(t->display.log)); + cd321x_mux_test_event(t, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + cd321x_mux_test_expect_read_now(test, t); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + KUNIT_EXPECT_STREQ(test, t->display.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* + * A PHY that stays stuck is taken for a fault after some two minutes of + * retries. The port then waits for the next controller event, which starts + * the retries over. + */ +static void cd321x_mux_held_gives_up_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_j700_port(test); + unsigned int i; + + t->phy.held = true; + cd321x_mux_test_attach(t); + for (i = 0; i < 4 * CD321X_MUX_RETRIES && cd321x_mux_test_next(t); i++) + ; + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.update_work)); + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, strlen(t->phy.log), (size_t)CD321X_MUX_RETRIES + 1); + KUNIT_EXPECT_NULL(test, strpbrk(t->phy.log, "suU")); + KUNIT_EXPECT_GE(test, t->phy.last_refused_ms, 120000U); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); + KUNIT_EXPECT_FALSE(test, t->cd.state_valid); + + /* Still stuck at the next event: the retries start over */ + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_event(t, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + cd321x_mux_test_expect_read_now(test, t); + for (i = 0; i < 8 && !strchr(t->phy.log, 'S'); i++) + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + KUNIT_ASSERT_STREQ(test, t->phy.log, "S"); + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_LE(test, cd321x_mux_test_due_ms(&t->cd.resume_work), + (unsigned int)CD321X_MUX_RETRY_MS); + + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +/* + * A series of retries that ends in another failure leaves no backoff behind: + * the next event is read without delay, and the next hold-up starts over at + * the first delay. + */ +static void cd321x_mux_held_then_other_error_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + unsigned int i; + + t->phy.held = true; + cd321x_mux_test_attach(t); + while (!delayed_work_pending(&t->cd.resume_work) || + cd321x_mux_test_backoff_ms(t) < 4000) + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + + t->phy.held_ret = -EBUSY; + cd321x_mux_test_settle(test, t); + KUNIT_ASSERT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); + + t->phy.held_ret = -ETIMEDOUT; + memset(t->phy.log, 0, sizeof(t->phy.log)); + t->data_status |= TPS_DATA_STATUS_USB_DATA_ROLE; + cd321x_mux_test_event(t, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + cd321x_mux_test_expect_read_now(test, t); + for (i = 0; i < 8 && !strchr(t->phy.log, 'S'); i++) + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + KUNIT_ASSERT_NOT_NULL(test, strchr(t->phy.log, 'S')); + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_LE(test, cd321x_mux_test_due_ms(&t->cd.resume_work), + (unsigned int)CD321X_MUX_RETRY_MS); + + t->phy.held = false; + cd321x_mux_test_settle(test, t); + cd321x_mux_test_expect_usb(test, t); +} + +static struct kunit_case cd321x_mux_cases[] = { + KUNIT_CASE(cd321x_mux_held_past_budget_test), + KUNIT_CASE(cd321x_mux_held_backs_off_test), + KUNIT_CASE(cd321x_mux_held_event_takes_over_test), + KUNIT_CASE(cd321x_mux_held_across_sleep_test), + KUNIT_CASE(cd321x_mux_held_ends_on_removal_test), + KUNIT_CASE(cd321x_mux_timeout_bounded_test), + KUNIT_CASE(cd321x_mux_other_error_bounded_test), + KUNIT_CASE(cd321x_mux_j700_held_past_budget_test), + KUNIT_CASE(cd321x_mux_j700_display_timeout_bounded_test), + KUNIT_CASE(cd321x_mux_j700_display_prepare_timeout_test), + KUNIT_CASE(cd321x_mux_held_gives_up_test), + KUNIT_CASE(cd321x_mux_held_then_other_error_test), + {} +}; + +static struct kunit_suite cd321x_mux_suite = { + .name = "cd321x-mux", + .test_cases = cd321x_mux_cases, +}; + +kunit_test_suite(cd321x_mux_suite); +MODULE_DESCRIPTION("KUnit tests for CD321x Type-C states that the mux holds up"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index 977df770dcff5..377ece4f41f89 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -520,6 +520,139 @@ static void cd321x_unacked_events_across_sleep_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); } +static void cd321x_held_mux_outlasts_budget_test(struct kunit *test) +{ + struct cd321x_pm_state pm = { .phase = CD321X_PM_RUNNING }; + unsigned int i, delay = 0; + + for (i = 0; i < 4 * CD321X_RESUME_ATTEMPTS; i++) { + unsigned int next = cd321x_pm_mux_held(&pm); + + /* Never given up, and not charged to the cable-state budget... */ + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_REVALIDATE); + KUNIT_EXPECT_EQ(test, pm.attempts_left, (unsigned int)CD321X_RESUME_ATTEMPTS); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + /* ...but applied afresh from SAFE, less and less often, down to a limit. */ + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + KUNIT_EXPECT_EQ(test, next, i ? min(2 * delay, CD321X_MUX_RETRY_MAX_MS) : + CD321X_MUX_RETRY_MS); + delay = next; + /* An event brings the retry forward, but not a read already under way. */ + KUNIT_EXPECT_TRUE(test, cd321x_pm_mux_event(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); + } + KUNIT_EXPECT_EQ(test, delay, CD321X_MUX_RETRY_MAX_MS); + + /* Once through, the next failure has the bounded budget again. */ + KUNIT_EXPECT_TRUE(test, cd321x_pm_mux_done(&pm)); + cd321x_pm_complete(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_done(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); + for (i = 0; i <= CD321X_RESUME_ATTEMPTS; i++) { + KUNIT_EXPECT_EQ(test, cd321x_pm_retry(&pm, true), i < CD321X_RESUME_ATTEMPTS); + if (i == CD321X_RESUME_ATTEMPTS) + break; + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + } + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); +} + +static void cd321x_held_mux_across_sleep_test(struct kunit *test) +{ + struct cd321x_pm_state pm = { .phase = CD321X_PM_RUNNING }; + + cd321x_pm_mux_held(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), 2 * CD321X_MUX_RETRY_MS); + + /* The resume read takes the retry over, and a new hold-up starts afresh. */ + cd321x_pm_prepare(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_resume(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), CD321X_MUX_RETRY_MS); + + /* Removal ends it. */ + cd321x_pm_remove(&pm); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); +} + +static void cd321x_held_mux_gives_up_test(struct kunit *test) +{ + struct cd321x_pm_state pm = { .phase = CD321X_PM_RUNNING }; + unsigned int i, total = 0; + + for (i = 0; i < CD321X_MUX_RETRIES; i++) { + unsigned int delay = cd321x_pm_mux_held(&pm); + + KUNIT_EXPECT_GT(test, delay, 0); + total += delay; + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + } + /* Some two minutes on, a hold-up is taken for a fault... */ + KUNIT_EXPECT_GE(test, total, 120000U); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), 0); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, pm.mux_retry_ms, 0); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + + /* ...until a later event applies the whole state again, afresh. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_event(&pm)); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), CD321X_MUX_RETRY_MS); + KUNIT_EXPECT_EQ(test, pm.mux_retries, 1); +} + +static void cd321x_held_mux_other_failure_test(struct kunit *test) +{ + struct cd321x_pm_state pm = { .phase = CD321X_PM_RUNNING }; + unsigned int i; + + for (i = 0; i < 4; i++) { + cd321x_pm_mux_held(&pm); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + } + KUNIT_ASSERT_EQ(test, pm.mux_retry_ms, 8 * CD321X_MUX_RETRY_MS); + + /* + * A budgeted failure ends the hold-up, and what is left of the budget, + * all but the read under way, runs out as usual. + */ + for (i = 0; cd321x_pm_retry(&pm, true); i++) { + KUNIT_EXPECT_EQ(test, pm.mux_retry_ms, 0); + KUNIT_EXPECT_EQ(test, pm.mux_retries, 0); + KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + } + KUNIT_EXPECT_EQ(test, i, CD321X_RESUME_ATTEMPTS - 1); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, pm.mux_retry_ms, 0); + + /* The next hold-up starts over, and removal ends it. */ + KUNIT_ASSERT_TRUE(test, cd321x_pm_event(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), CD321X_MUX_RETRY_MS); + cd321x_pm_remove(&pm); + KUNIT_EXPECT_EQ(test, pm.mux_retry_ms, 0); + KUNIT_EXPECT_EQ(test, pm.mux_retries, 0); +} + /* Only a pipeline owned by another port skips the retry. */ static void cd321x_display_taken_test(struct kunit *test) { @@ -559,6 +692,10 @@ static struct kunit_case cd321x_pm_cases[] = { KUNIT_CASE(cd321x_lost_event_keeps_reconnect_test), KUNIT_CASE(cd321x_unacked_events_outlast_budget_test), KUNIT_CASE(cd321x_unacked_events_across_sleep_test), + KUNIT_CASE(cd321x_held_mux_outlasts_budget_test), + KUNIT_CASE(cd321x_held_mux_across_sleep_test), + KUNIT_CASE(cd321x_held_mux_gives_up_test), + KUNIT_CASE(cd321x_held_mux_other_failure_test), KUNIT_CASE(cd321x_display_taken_test), {} }; diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h index 3879c8a660dcc..1a3b5a4b659e3 100644 --- a/drivers/usb/typec/tipd/cd321x-pm.h +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -11,6 +11,10 @@ #define CD321X_LINK_EVENTS 3 #define CD321X_ACK_RETRY_MS 500 #define CD321X_ACK_RETRY_MAX_MS 30000 +#define CD321X_MUX_RETRY_MS 500 +#define CD321X_MUX_RETRY_MAX_MS 10000 +/* About two minutes of backoff, twenty times the longest hold-up seen */ +#define CD321X_MUX_RETRIES 16 enum cd321x_pm_phase { CD321X_PM_RUNNING, @@ -29,6 +33,10 @@ struct cd321x_pm_state { unsigned int link_events_left; /* Non-zero while events may be left unacknowledged: the retry delay. */ unsigned int ack_retry_ms; + /* Non-zero while a mux that held a change up is retried: the retry delay. */ + unsigned int mux_retry_ms; + /* The retries of that change so far, up to CD321X_MUX_RETRIES. */ + unsigned int mux_retries; bool force_reconnect; }; @@ -57,11 +65,26 @@ static inline bool cd321x_pm_can_update(const struct cd321x_pm_state *pm) return pm->phase == CD321X_PM_RUNNING || pm->phase == CD321X_PM_APPLY; } +/* + * Ends the retries of a change that a mux held up, see cd321x_pm_mux_held(). + * Returns true if there were any. + */ +static inline bool cd321x_pm_mux_done(struct cd321x_pm_state *pm) +{ + bool retried = pm->mux_retry_ms; + + pm->mux_retry_ms = 0; + pm->mux_retries = 0; + return retried; +} + static inline void cd321x_pm_prepare(struct cd321x_pm_state *pm) { if (pm->phase != CD321X_PM_REMOVED) pm->phase = CD321X_PM_PREPARED; pm->attempts_left = 0; + /* The resume read applies the state afresh. */ + cd321x_pm_mux_done(pm); } static inline bool cd321x_pm_resume(struct cd321x_pm_state *pm) @@ -104,6 +127,8 @@ static inline void cd321x_pm_snapshot_ready(struct cd321x_pm_state *pm) static inline bool cd321x_pm_retry(struct cd321x_pm_state *pm, bool reconnect) { + /* Any other failure ends a hold-up: this one is budgeted. */ + cd321x_pm_mux_done(pm); /* Runtime cable updates need the same bounded fresh-read recovery. */ if (pm->phase == CD321X_PM_RUNNING) { pm->force_reconnect |= reconnect; @@ -174,6 +199,54 @@ static inline bool cd321x_pm_acked(struct cd321x_pm_state *pm) return retried; } +/* + * The mux provider that failed a change timed out because its hardware held + * the change up (typec_mux_timeout_transient()). On a J700 the PHY cannot park + * the PIPE while the controller behind the fixed hub still drives it, and a + * partner renegotiating power can keep that up for longer than the cable-state + * budget lasts, after which nothing would apply the state again. The cable + * state itself was read fine, so such a failure does not count against the + * budget: it gives back the attempt taken by cd321x_pm_begin_read(), and the + * whole state is applied again from a fresh read, starting with SAFE, after a + * delay that doubles up to CD321X_MUX_RETRY_MAX_MS. A later event brings the + * retry forward, see cd321x_pm_mux_event(); sleep cancels it and the resume + * read takes it over; removal or any other failure ends it. + * + * A hold-up that outlasts CD321X_MUX_RETRIES retries is taken for a fault, as + * the provider cannot tell its timeouts apart: the retries end as the budget + * does, and only a later event starts over. Returns the delay in milliseconds, + * or 0 once the retries have ended. + */ +static inline unsigned int cd321x_pm_mux_held(struct cd321x_pm_state *pm) +{ + pm->force_reconnect = true; + if (pm->mux_retries >= CD321X_MUX_RETRIES) { + cd321x_pm_mux_done(pm); + pm->phase = CD321X_PM_STALE; + pm->attempts_left = 0; + return 0; + } + if (pm->phase == CD321X_PM_RUNNING) + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + else + pm->attempts_left++; + pm->phase = CD321X_PM_REVALIDATE; + pm->mux_retries++; + pm->mux_retry_ms = clamp(2 * pm->mux_retry_ms, CD321X_MUX_RETRY_MS, + CD321X_MUX_RETRY_MAX_MS); + return pm->mux_retry_ms; +} + +/* + * Whether a controller event should bring forward the retry of a held mux: the + * event's state is then read and applied after the usual debounce, instead of + * waiting out the backoff. + */ +static inline bool cd321x_pm_mux_event(const struct cd321x_pm_state *pm) +{ + return pm->mux_retry_ms && pm->phase == CD321X_PM_REVALIDATE; +} + static inline void cd321x_pm_complete(struct cd321x_pm_state *pm) { if (pm->phase == CD321X_PM_APPLY) { @@ -187,6 +260,7 @@ static inline void cd321x_pm_remove(struct cd321x_pm_state *pm) { pm->phase = CD321X_PM_REMOVED; pm->attempts_left = 0; + cd321x_pm_mux_done(pm); } /* diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 35f164d4eb2f2..bc94b08615458 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -579,8 +579,30 @@ static void tps6598x_handle_plug_event(struct tps6598x *tps, u32 status) } } +/* The mux held the change up, see cd321x_pm_mux_held(). */ +static bool cd321x_mux_held(struct cd321x *cd321x, int ret) +{ + return ret == -ETIMEDOUT && typec_mux_timeout_transient(cd321x->mux); +} + +/* Set the mux to the mode state, telling whether a provider held it up */ +static int cd321x_mux_set(struct cd321x *cd321x, bool *held) +{ + int ret = typec_mux_set(cd321x->mux, &cd321x->state); + + *held = cd321x_mux_held(cd321x, ret); + return ret; +} + +/* A held mux is retried: report only the first failure of the series. */ +static bool cd321x_mux_still_held(struct cd321x *cd321x, bool held) +{ + return held && cd321x->pm.mux_retry_ms; +} + static int cd321x_typec_update_mode(struct tps6598x *tps, - struct cd321x_status *st, bool *display_busy) + struct cd321x_status *st, bool *display_busy, + bool *held) { struct cd321x *cd321x = container_of(tps, struct cd321x, tps); struct typec_mux_state old_state = cd321x->state; @@ -590,6 +612,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, int ret = 0; *display_busy = false; + *held = false; if (!(st->data_status & TPS_DATA_STATUS_DATA_CONNECTION)) { if (cd321x->state_valid && cd321x->state.mode == TYPEC_STATE_SAFE) return 0; @@ -599,7 +622,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); if (ret) goto out; - ret = typec_mux_set(cd321x->mux, &cd321x->state); + ret = cd321x_mux_set(cd321x, held); } else if (st->data_status & CD321X_DATA_STATUS_USB4_CONNECTION) { struct enter_usb_data eusb_data; @@ -614,7 +637,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, cd321x->state.alt = NULL; cd321x->state.data = &eusb_data; cd321x->state.mode = TYPEC_MODE_USB4; - ret = typec_mux_set(cd321x->mux, &cd321x->state); + ret = cd321x_mux_set(cd321x, held); if (ret) goto out; @@ -647,7 +670,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, cd321x->state.alt = cd321x->port_altmode_tbt; cd321x->state.mode = TYPEC_TBT_MODE; cd321x->state.data = &tbt_data; - ret = typec_mux_set(cd321x->mux, &cd321x->state); + ret = cd321x_mux_set(cd321x, held); if (ret) goto out; @@ -704,7 +727,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); if (ret) goto out; - ret = typec_mux_set(cd321x->mux, &cd321x->state); + ret = cd321x_mux_set(cd321x, held); *display_busy = cd321x_display_taken(ret); if (!ret) { cd321x->dp_status = dp_data.status; @@ -720,7 +743,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); if (ret) goto out; - ret = typec_mux_set(cd321x->mux, &cd321x->state); + ret = cd321x_mux_set(cd321x, held); } out: @@ -733,6 +756,8 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, /* A provider that is not ready yet asks for a retry. */ if (ret == -EAGAIN) dev_dbg(tps->dev, "Type-C connection deferred\n"); + else if (cd321x_mux_still_held(cd321x, *held)) + dev_dbg(tps->dev, "Type-C connection still held up: %d\n", ret); else dev_warn(tps->dev, "failed to configure Type-C connection: %d\n", ret); } else if (cd321x->state.alt != cd321x->port_altmode_dp) { @@ -761,6 +786,31 @@ static void cd321x_retry_revalidation(struct cd321x *cd321x, bool reconnect) CD321X_RESUME_ATTEMPTS); } +static void cd321x_retry_held_mux(struct cd321x *cd321x) +{ + unsigned int delay; + + lockdep_assert_held(&cd321x->tps.lock); + cd321x->state_valid = false; + if (!cd321x->pm.mux_retry_ms) + dev_info(cd321x->tps.dev, "Type-C mux held up, retrying\n"); + delay = cd321x_pm_mux_held(&cd321x->pm); + if (delay) + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, + msecs_to_jiffies(delay)); + else + dev_warn(cd321x->tps.dev, + "Type-C mux still held up after %u retries, waiting for the next event\n", + CD321X_MUX_RETRIES); +} + +static void cd321x_update_done(struct cd321x *cd321x) +{ + if (cd321x_pm_mux_done(&cd321x->pm)) + dev_info(cd321x->tps.dev, "Type-C mux change went through after retrying\n"); + cd321x_pm_complete(&cd321x->pm); +} + static void cd321x_update_work(struct work_struct *work) { struct cd321x *cd321x = container_of(to_delayed_work(work), @@ -770,7 +820,7 @@ static void cd321x_update_work(struct work_struct *work) struct typec_thunderbolt_switch_data tbt_switch_data = { .state = TYPEC_THUNDERBOLT_SWITCH_OFF, }; - bool display_busy; + bool display_busy, held = false; int ret; guard(mutex)(&tps->lock); @@ -878,12 +928,16 @@ static void cd321x_update_work(struct work_struct *work) cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; cd321x->state.data = NULL; - ret = typec_set_mode(tps->port, TYPEC_STATE_SAFE); + /* Through cd321x->mux, which can then tell if SAFE was held up */ + ret = cd321x_mux_set(cd321x, &held); cd321x->state_valid = !ret; if (ret) { - dev_warn(tps->dev, - "failed to place Type-C mux in safe mode: %d\n", - ret); + if (cd321x_mux_still_held(cd321x, held)) + dev_dbg(tps->dev, "Type-C mux still held up: %d\n", ret); + else + dev_warn(tps->dev, + "failed to place Type-C mux in safe mode: %d\n", + ret); cd321x->state = old_state; goto retry; } @@ -900,7 +954,7 @@ static void cd321x_update_work(struct work_struct *work) /* If the plug is disconnected, we are done */ if (!new_connected) { - cd321x_pm_complete(&cd321x->pm); + cd321x_update_done(cd321x); return; } @@ -936,7 +990,7 @@ static void cd321x_update_work(struct work_struct *work) } /* Update the TypeC MUX/PHY state */ - ret = cd321x_typec_update_mode(tps, &st, &display_busy); + ret = cd321x_typec_update_mode(tps, &st, &display_busy, &held); if (ret && !display_busy) goto retry; /* A shared display route can be busy while the dock's USB path is usable. */ @@ -953,10 +1007,14 @@ static void cd321x_update_work(struct work_struct *work) connector_status_connected); power_supply_changed(tps->psy); - cd321x_pm_complete(&cd321x->pm); + cd321x_update_done(cd321x); return; retry: + if (held) { + cd321x_retry_held_mux(cd321x); + return; + } /* A deferred PCIe activation is not evidence of a failed cable. */ cd321x_retry_revalidation(cd321x, ret != -EAGAIN); } @@ -988,6 +1046,10 @@ static int cd321x_connect(struct tps6598x *tps, u32 status) /* A plug or power IRQ alone does not refresh the cached cable metadata. */ if (cd321x_pm_event(&cd321x->pm)) mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); + /* A held mux waits out its backoff only while nothing else happens. */ + if (cd321x_pm_mux_event(&cd321x->pm)) + mod_delayed_work(system_freezable_wq, &cd321x->resume_work, + msecs_to_jiffies(CD321X_DEBOUNCE_DELAY_MS)); if (!cd321x_pm_can_update(&cd321x->pm)) return 0; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 4b47239655bfc..5b825b97eaee4 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -24,6 +24,10 @@ def function(file, name): pm = tipd_pm_header(r) core = 'drivers/usb/typec/tipd/core.c' dcp = 'drivers/gpu/drm/apple/dcp.c' + # The retry of a change that a mux with transient timeouts held up + held = [n for n in ('cd321x_mux_held', 'cd321x_mux_set', 'cd321x_mux_still_held', + 'cd321x_retry_held_mux', 'cd321x_update_done') + if re.search(r'^static \w+ ' + n + r'\(', (r / core).read_text(), re.M)] funcs = '\n'.join(function(f, n) for f, n in ( ('drivers/gpu/drm/apple/iomfb.c', 'iomfb_pause_presentation'), (dcp, 'dcp_external_report_disconnected'), @@ -35,6 +39,7 @@ def function(file, name): (dcp, 'dcp_external_link_locked'), (dcp, 'dcp_dptx_connect_oob'), (core, 'cd321x_retry_revalidation'), + *((core, n) for n in held), (core, 'cd321x_update_work'))) prefix = r''' #include @@ -167,7 +172,11 @@ def function(file, name): struct cd321x { struct tps6598x tps; struct cd321x_status update_status; struct cd321x_pm_state pm; struct delayed_work update_work, resume_work; struct typec_mux_state state; bool state_valid, display_route_active, cur_partner_is_pd; - struct usb_pd_identity cur_partner_identity; void *connector_fwnode, *tbt_switch; }; + struct usb_pd_identity cur_partner_identity; void *connector_fwnode, *tbt_switch, *mux; }; +/* No mux here declares its timeouts transient: every failure is budgeted. */ +static bool typec_mux_timeout_transient(void *m) { (void)m; return false; } +#define dev_dbg(...) ((void)0) +#define dev_info(...) ((void)0) static int tbt_error, role_error, safe_error, mode_error; static int tbts, roles, safes; static enum usb_role usb_role_switch_get_role(enum usb_role *r) { return *r; } @@ -179,6 +188,9 @@ def function(file, name): { (void)p; assert(s == TYPEC_STATE_SAFE); safes++; return safe_error; } static int cd321x_typec_update_mode(struct tps6598x *t, struct cd321x_status *s, bool *busy) { (void)t; (void)s; *busy = mode_error == -EBUSY; return mode_error; } +/* SAFE through the driver's own mux handle is the same SAFE request. */ +static int __attribute__((unused)) typec_mux_set(void *m, struct typec_mux_state *s) +{ (void)m; return typec_set_mode(NULL, s->mode); } static void typec_unregister_partner(void *p) { (void)p; } static void *typec_register_partner(void *p, struct typec_partner_desc *d) { (void)p; (void)d; return (void *)1; } @@ -370,6 +382,14 @@ def function(file, name): printf("%d assertions, %d failures\n", checks, failures); return !!failures; } ''' + # The mode update also tells whether a mux held it up. + if re.search(r'bool \*display_busy,\s*bool \*held\)', (r / core).read_text()): + old = ('static int cd321x_typec_update_mode(struct tps6598x *t, struct cd321x_status *s, bool *busy)\n' + '{ (void)t; (void)s; *busy = mode_error == -EBUSY; return mode_error; }') + assert shim.count(old) == 1 + shim = shim.replace(old, 'static int cd321x_typec_update_mode(struct tps6598x *t, struct cd321x_status *s, bool *busy,\n' + ' bool *held)\n' + '{ (void)t; (void)s; *busy = mode_error == -EBUSY; *held = false; return mode_error; }') source = prefix + pm + shim + funcs + tests return source diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index c3dbcdac5a3a7..557162b627907 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -611,6 +611,44 @@ def function(text, name): } ''' + # A mux that holds a change up is retried outside the cable-state budget. + if 'cd321x_pm_mux_held(' in pm: + lost = ' puts("PASS actual lost events reopen a stale port; unacknowledged events keep their budget with capped backoff");\n' + assert controls.count(lost) == 1 + controls = controls.replace(lost, lost + r''' struct cd321x_pm_state held={0}; + held.phase=CD321X_PM_RUNNING; + for(unsigned int i=0,delay=0;i<4*CD321X_RESUME_ATTEMPTS;i++){ + unsigned int expected=i?(2*delay Date: Fri, 16 Oct 2026 02:32:14 +1100 Subject: [PATCH 1232/1579] ASoC: apple: t8140-aop-audio: Publish the LPAI pointer after its frames The low-power microphone's report path copies a span into the ALSA buffer and then advances hw_ptr_frames; the .pointer callback returns that value. ALSA calls .pointer from a status or HWSYNC request on another CPU, with the PCM stream lock held but not the driver's stream state lock, so the copy and the pointer are only ordered by the atomic itself. Both sides used relaxed accesses, which order nothing else: on arm64 a reader can see the new pointer before the frames behind it, and hand userspace stale buffer contents as captured audio. Store the pointer with release after the copy, and load it with acquire in .pointer. The resets at open and prepare publish a pointer too and use release as well. This is the message-passing pattern of the Linux kernel memory model's MP+pooncerelease+poacquireonce test (forbidden), where the relaxed form, MP+poonceonces, is allowed. Fixes: eeb112648f2a ("ASoC: apple: Add T8140 AOP audio driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 000a548ba37b5..8ecc439986db5 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -68,9 +68,11 @@ use kernel::{ sync::{ aref::ARef, atomic::{ + Acquire, Atomic, AtomicFlag, - Relaxed, // + Relaxed, + Release, // }, Arc, ArcBorrow, @@ -457,6 +459,10 @@ struct SndSocT8140AopData { /// The low-power microphone stream is triggered (set and cleared under /// the ALSA stream lock, read by the report path). running: AtomicFlag, + /// The low-power microphone's hardware pointer. Stored with release + /// after the frames before it are in the ALSA buffer, loaded with acquire + /// by the .pointer callback, which ALSA calls without the stream state's + /// lock (a status or HWSYNC on another CPU). hw_ptr_frames: Atomic, /// One warning per producer-report fault class and driver instance. report_warnings: [AtomicFlag; 5], @@ -1180,7 +1186,7 @@ impl ReportListener for SndSocT8140AopData { st.copied = Some(absolute); st.period_acc += delta; self.hw_ptr_frames - .store((new_ptr / LPAI_FRAME_BYTES) as u32, Relaxed); + .store((new_ptr / LPAI_FRAME_BYTES) as u32, Release); if st.period_acc >= LPAI_PERIOD_BYTES { st.period_acc %= LPAI_PERIOD_BYTES; // SAFETY: the substream is valid while it is registered in @@ -1244,7 +1250,7 @@ unsafe extern "C" fn lpai_pcm_open(substream: *mut bindings::snd_pcm_substream) st.copied = None; st.hw_ptr_bytes = 0; st.period_acc = 0; - data.hw_ptr_frames.store(0, Relaxed); + data.hw_ptr_frames.store(0, Release); 0 } @@ -1285,7 +1291,7 @@ unsafe extern "C" fn lpai_pcm_prepare(substream: *mut bindings::snd_pcm_substrea st.hw_ptr_bytes = 0; st.period_acc = 0; st.copied = None; - data.hw_ptr_frames.store(0, Relaxed); + data.hw_ptr_frames.store(0, Release); } if !data.powered.load(Relaxed) { if let Err(e) = data.lpai_run_get() { @@ -1350,7 +1356,7 @@ unsafe extern "C" fn lpai_pcm_pointer( ) -> bindings::snd_pcm_uframes_t { // SAFETY: ALSA calls this op with a live substream of the LPAI PCM. let data = unsafe { pcm_data(substream) }; - data.hw_ptr_frames.load(Relaxed) as bindings::snd_pcm_uframes_t + data.hw_ptr_frames.load(Acquire) as bindings::snd_pcm_uframes_t } /// The microphone array's capture: the LEAP produces float32 into the ADMAC From b43c54275343c833707f42e80daad6821edd0333 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:44:53 +1100 Subject: [PATCH 1233/1579] rust: helpers: Add spin_lock_irqsave() and spin_unlock_irqrestore() Rust code that has to take a spinlock owned by C code, one that the C side takes with interrupts saved, has no way to do so: the helpers only cover spin_lock() and spin_unlock(). An input handler resynchronizing with its device's switch state is one such user: the input core delivers events with dev->event_lock held and interrupts off, and a handler reading the device's state outside of an event must hold the same lock. Add the irqsave/irqrestore pair, returning the flags to the caller. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/helpers/spinlock.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c index 4d13062cf253e..6a79c7fe2d359 100644 --- a/rust/helpers/spinlock.c +++ b/rust/helpers/spinlock.c @@ -27,6 +27,21 @@ __rust_helper void rust_helper_spin_unlock(spinlock_t *lock) spin_unlock(lock); } +__rust_helper unsigned long rust_helper_spin_lock_irqsave(spinlock_t *lock) +{ + unsigned long flags; + + spin_lock_irqsave(lock, flags); + + return flags; +} + +__rust_helper void rust_helper_spin_unlock_irqrestore(spinlock_t *lock, + unsigned long flags) +{ + spin_unlock_irqrestore(lock, flags); +} + __rust_helper int rust_helper_spin_trylock(spinlock_t *lock) { return spin_trylock(lock); From 05902062e7cef8dcdd353f3e9f22382e69675318 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:32:16 +1100 Subject: [PATCH 1234/1579] ASoC: apple: t8140-aop-audio: Report the mics unplugged with the lid closed Closing the lid of the MacBook Neo (J700) disconnects its internal microphones in hardware. Nothing tells the AP: the array then reads back as digital silence and the low-power microphone as a constant near negative full scale, and userspace keeps offering both as working microphones. A meeting app recording in clamshell mode with an external display records nothing, and never falls back to a microphone that works. The lid switch is already an input device ("Apple SMC power/lid events", SW_LID, from macsmc-input). Follow it with an input handler, as hid-appletb-kbd and the input-events LED trigger follow other input devices, and report the result as a microphone jack on the card, "Internal Mic Jack": unplugged while any lid switch reads closed. The handler is a passive observer, so it does not open the device on its own; macsmc-input reports lid changes whether or not it is open. Every update of a switch's recorded state happens under that device's event lock: the input core delivers events with it held, and the handler reads the device's switch state with it held when it connects (after opening the handle, so no event falls in between) and from its start() callback, which the input core calls when a client that grabbed the device releases it. A reading can therefore never overwrite a newer event, and events a grabbing client kept to itself are caught up with. Readings only count once the handle is open: the input core calls start() while registering the handle, before the handler knows whether opening it succeeds, and a handle that fails to open is freed without a disconnect. A lid switch that goes away no longer counts as closed. The count of closed switches is the only state shared between switches. snd_jack_report() takes mutexes and is not ordered against other callers, so a single work item makes every report, including the initial one, which probe queues and flushes before registering the card: reports reach ALSA in the order of the states they read. The jack has no input device of its own: the lid it follows already is one, and the control is what userspace reads. With UCM naming the jack as the microphone's JackControl, PipeWire and PulseAudio mark the microphone port unavailable, switch the card's profile off and fall back to another source; when the lid opens the profile, the node and the user's default source come back. Without such a JackControl nothing changes. [ryan: rebased onto aurora-wip 0daed2db + ATC PIPE park v3. #345 (harden-all) added the parser_tests and name_tests KUnit modules at the end of t8140_aop_audio.rs, where this patch adds lid_tests: kept both, lid_tests last. The added lines are those of the reviewed patch.] Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/bindings/bindings_helper.h | 2 + sound/soc/apple/Kconfig | 3 +- sound/soc/apple/t8140_aop_audio.rs | 531 +++++++++++++++++++++++++++++ 3 files changed, 535 insertions(+), 1 deletion(-) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index d819681a1e9c6..5491f167438f2 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -70,6 +70,7 @@ #include #include #include +#include #include #include #include @@ -111,6 +112,7 @@ #include #include #include +#include #include #include #include diff --git a/sound/soc/apple/Kconfig b/sound/soc/apple/Kconfig index a57eadd07a1ba..a67dcdf353f59 100644 --- a/sound/soc/apple/Kconfig +++ b/sound/soc/apple/Kconfig @@ -15,6 +15,7 @@ config SND_SOC_APPLE_AOP_AUDIO config SND_SOC_APPLE_T8140_AOP_AUDIO tristate "Apple T8140 AOP audio driver" depends on ARCH_APPLE || COMPILE_TEST + depends on INPUT depends on RUST select APPLE_AOP select SND_DMAENGINE_PCM @@ -23,7 +24,7 @@ config SND_SOC_APPLE_T8140_AOP_AUDIO co-processor on Apple T8140 SoCs: a card with the low-power and the high-quality microphones, and ASoC back-end DAIs for the headphone jack and the internal speaker, which the macaudio driver links into - its card. + its card. The microphones read as unplugged while the lid is closed. config SND_SOC_APPLE_MCA tristate "Apple Silicon MCA driver" diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 8ecc439986db5..c8a83c859ec5f 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -31,6 +31,13 @@ //! the "Speaker Playback Volume" control, which the card's volume lock holds //! 20 dB down whenever no speaker-protection daemon owns it. //! +//! Closing the lid disconnects both internal microphones in hardware. The +//! array then reads back as digital silence, but the low-power microphone's +//! decimator, fed a PDM line held low, reads back as a constant close to +//! negative full scale. The driver follows the lid switch through the input +//! core: while a lid is closed its card's "Internal Mic" jack reads +//! unplugged, so userspace stops offering the microphones. +//! //! Copyright (C) The Asahi Linux Contributors //! Copyright (C) 2026 The Aurora Silicon Contributors @@ -444,6 +451,106 @@ struct LpaiRun { running: bool, } +/// The lid switches seen through the input core. The internal microphones +/// are disconnected while any of them reads closed. +/// +/// Each switch's flag (in its [`LidHandle`]) and the count change together: +/// every writer of one switch runs under that input device's event lock (the +/// event callback, and the resynchronizations in `lid_resync`), or after the +/// device stopped delivering events to it (`lid_disconnect`). The count is +/// the only state shared between switches, and it is a single atomic word. +struct LidState { + /// Lid switches currently closed. + closed: Atomic, +} + +impl LidState { + const fn new() -> Self { + LidState { + closed: Atomic::new(0), + } + } + + /// Records that the switch whose state is `switch` now reads `closed`. + /// Returns whether that changed anything; a repeated state does not. + fn set(&self, switch: &AtomicFlag, closed: bool) -> bool { + if switch.xchg(closed, Relaxed) == closed { + return false; + } + if closed { + self.closed.add(1, Relaxed); + } else { + let _ = self.closed.fetch_sub(1, Relaxed); + } + true + } + + fn mic_cut(&self) -> bool { + self.closed.load(Relaxed) != 0 + } +} + +/// The "Internal Mic" jack status for the lid state. +fn mic_jack_status(mic_cut: bool) -> i32 { + if mic_cut { + 0 + } else { + bindings::snd_jack_types_SND_JACK_MICROPHONE as i32 + } +} + +/// The input handler that follows the lid switches. Only its address is +/// shared: the input core owns its contents once it is registered. +struct LidHandler(Opaque); + +// SAFETY: the handler is filled in before registration and only the input +// core, which serializes its own access, touches it after that. +unsafe impl Send for LidHandler {} +// SAFETY: as for `Send`. +unsafe impl Sync for LidHandler {} + +/// One lid switch the handler is connected to. `handle` comes first so the +/// input core's handle pointer is also a pointer to this. +#[repr(C)] +struct LidHandle { + handle: Opaque, + /// The switch reads closed, as recorded in [`LidState`]. + closed: AtomicFlag, + /// Readings are recorded: the handle is open. + open: AtomicFlag, +} + +impl LidHandle { + /// Records a reading of this switch, once the handle is open. + /// + /// `input_register_handle()` calls `lid_start()` before `lid_connect()` + /// knows whether opening the handle succeeds, and a handle that fails to + /// open is freed without `lid_disconnect()`: a reading recorded then + /// could never be taken back. `lid_connect()` reads the switch again + /// once the handle is open. + fn record(&self, lid: &LidState, closed: bool) -> bool { + self.open.load(Relaxed) && lid.set(&self.closed, closed) + } + + /// Records the switch as read under its device's event lock. With + /// `activate`, readings are enabled first: `lid_connect()` enables and + /// reads the switch in one step after opening the handle, so an event + /// delivered between the open and this reading, which `record()` ignored, + /// is caught up with. + fn sync(&self, lid: &LidState, closed: bool, activate: bool) -> bool { + if activate { + self.open.store(true, Relaxed); + } + self.record(lid, closed) + } + + /// The switch goes away: it no longer disconnects the microphones. + fn forget(&self, lid: &LidState) -> bool { + self.open.store(false, Relaxed); + lid.set(&self.closed, false) + } +} + #[pin_data] struct SndSocT8140AopData { dev: ARef, @@ -492,6 +599,14 @@ struct SndSocT8140AopData { spkr_start_work: Work, #[pin] hpai_start_work: Work, + /// The lid switches, and the card's "Internal Mic" jack that follows + /// them (null outside the card's lifetime), reported from `jack_work` + /// because the input core delivers switch events in atomic context. + lid: LidState, + lid_handler: LidHandler, + mic_jack: Atomic<*mut bindings::snd_jack>, + #[pin] + jack_work: Work, #[pin] stream: SpinLock, /// Every lpai power request goes through here, see @@ -502,10 +617,21 @@ struct SndSocT8140AopData { const SPKR_START_WORK: u64 = 0; const HPAI_START_WORK: u64 = 1; +const JACK_WORK: u64 = 2; impl_has_work! { impl HasWork for SndSocT8140AopData { self.spkr_start_work } impl HasWork for SndSocT8140AopData { self.hpai_start_work } + impl HasWork for SndSocT8140AopData { self.jack_work } +} + +/// The "Internal Mic" jack report after a lid change: it takes mutexes. +impl WorkItem for SndSocT8140AopData { + type Pointer = Arc; + + fn run(this: Arc) { + this.report_mic_jack(); + } } /// The speaker stream start (pwrd + unmute), queued by the back-end DAI's @@ -963,6 +1089,46 @@ impl SndSocT8140AopData { self.service_shutdown(&self.hpai, &self.hpai_domains()) } + // ---- the lid: both microphones are disconnected in hardware while it + // is closed ---- + + /// A lid switch now reads `closed`: queue the jack report on a change. + fn lid_set(self: ArcBorrow<'_, Self>, switch: &LidHandle, closed: bool) { + if switch.record(&self.lid, closed) { + self.queue_jack_report(); + } + } + + /// A lid switch went away. + fn lid_forget(self: ArcBorrow<'_, Self>, switch: &LidHandle) { + if switch.forget(&self.lid) { + self.queue_jack_report(); + } + } + + /// The jack work is the only reporter, so reports reach ALSA in the order + /// of the states they read: a queued or running report that read an + /// older state is followed by another run. + fn queue_jack_report(self: ArcBorrow<'_, Self>) { + // Err = already queued: the pending report reads the new state. + let _ = workqueue::system().enqueue::<_, JACK_WORK>(Arc::from(self)); + } + + fn report_mic_jack(&self) { + let jack = self.mic_jack.load(Relaxed); + if !jack.is_null() { + // SAFETY: the jack belongs to the card, which teardown frees only + // after clearing `mic_jack` and flushing this work. + unsafe { bindings::snd_jack_report(jack, mic_jack_status(self.lid.mic_cut())) }; + } + } + + fn flush_jack_report(&self) { + // SAFETY: the work item lives in `self` for the driver's lifetime; + // flushing waits for a queued report to finish. + unsafe { bindings::flush_work(Work::raw_get(&self.jack_work)) }; + } + // ---- internal speakers: the spkr service on LEAP TX0 (leap-ns ADMAC // channel 0); the leaves are tx0 (-> leap_c -> leap_a -> audio_fr -> // audio_a), leap_mca (-> audio_p) and the MCA0 serializer the LEAP @@ -1197,6 +1363,166 @@ impl ReportListener for SndSocT8140AopData { } } +// ---- the lid switch handler: an input handler on every device with +// SW_LID, observing the events of the lid switch userspace already +// listens to ---- + +/// Devices with a lid switch, and the terminating entry. +static LID_IDS: [bindings::input_device_id; 2] = { + // SAFETY: the ID is plain integers, for which zero is valid. + let mut lid: bindings::input_device_id = unsafe { mem::zeroed() }; + lid.flags = (bindings::INPUT_DEVICE_ID_MATCH_EVBIT | bindings::INPUT_DEVICE_ID_MATCH_SWBIT) + as bindings::kernel_ulong_t; + lid.evbit[0] = 1 << bindings::EV_SW; + lid.swbit[0] = 1 << bindings::SW_LID; + // SAFETY: as above. + [lid, unsafe { mem::zeroed() }] +}; + +/// The driver data behind an input handle of the lid handler. +/// +/// # Safety +/// +/// `handle` must be a handle `lid_connect` registered and the handler must +/// still be registered, so the `Arc` leaked into its private data is live. +unsafe fn lid_data<'a>(handle: *mut bindings::input_handle) -> ArcBorrow<'a, SndSocT8140AopData> { + // SAFETY: by the caller's contract. + unsafe { Arc::::borrow((*(*handle).handler).private.cast()) } +} + +/// Reads the switch from the device's state and records it, under the +/// device's event lock: events are delivered under that lock after the input +/// core updated the state, so no event can be overtaken by an older reading. +/// `activate` enables readings first (see [`LidHandle::sync`]). +/// +/// # Safety +/// +/// `handle` must be a handle `lid_connect` registered, with the handler +/// still registered. +unsafe fn lid_resync(handle: *mut bindings::input_handle, activate: bool) { + // SAFETY: the handle and its device are live while it is registered. + let dev = unsafe { (*handle).dev }; + // SAFETY: `dev` is live; its event lock is a spinlock taken with + // interrupts saved, as the input core takes it. + let lock = unsafe { &raw mut (*dev).event_lock }; + // SAFETY: as above. + let flags = unsafe { bindings::spin_lock_irqsave(lock) }; + // SAFETY: the switch bitmap is written under the event lock, held here. + let closed = unsafe { (*dev).sw[0] } & (1 << bindings::SW_LID) != 0; + // SAFETY: the handle is the first field of a `LidHandle`, live while it + // is registered, and the handler is registered. + let (lid, data) = unsafe { (&*handle.cast::(), lid_data(handle)) }; + if lid.sync(&data.lid, closed, activate) { + data.queue_jack_report(); + } + // SAFETY: taken above with these flags. + unsafe { bindings::spin_unlock_irqrestore(lock, flags) }; +} + +/// Called after the handle is registered, and again when a client that +/// grabbed the device releases it: events delivered only to the grabbing +/// client in between never reached `lid_event`. +/// +/// # Safety +/// +/// Called by the input core with a handle `lid_connect` registered. +unsafe extern "C" fn lid_start(handle: *mut bindings::input_handle) { + // SAFETY: by the caller's contract; the input core calls this without + // the event lock. + unsafe { lid_resync(handle, false) }; +} + +/// # Safety +/// +/// Called by the input core, in atomic context with the device's event lock +/// held, with a handle `lid_connect` registered. +unsafe extern "C" fn lid_event( + handle: *mut bindings::input_handle, + type_: u32, + code: u32, + value: i32, +) { + if type_ != bindings::EV_SW || code != bindings::SW_LID { + return; + } + // SAFETY: the handle is the first field of a `LidHandle`, live until + // `lid_disconnect`, and the handler is registered. + let (lid, data) = unsafe { (&*handle.cast::(), lid_data(handle)) }; + data.lid_set(lid, value != 0); +} + +/// # Safety +/// +/// Called by the input core with the registered lid handler and a device +/// that matched `LID_IDS`. +unsafe extern "C" fn lid_connect( + handler: *mut bindings::input_handler, + dev: *mut bindings::input_dev, + _id: *const bindings::input_device_id, +) -> i32 { + let lid = match KBox::new( + LidHandle { + handle: Opaque::zeroed(), + closed: AtomicFlag::new(false), + open: AtomicFlag::new(false), + }, + GFP_KERNEL, + ) { + Ok(lid) => KBox::into_raw(lid), + Err(_) => return ENOMEM.to_errno(), + }; + // SAFETY: `lid` was allocated above; `dev` and `handler` are live for + // the handle's lifetime, which ends in `lid_disconnect`. + let handle = unsafe { (*lid).handle.get() }; + // SAFETY: as above; the handle is not registered yet. + unsafe { + (*handle).dev = dev; + (*handle).handler = handler; + (*handle).name = c"t8140-aop-audio".as_char_ptr(); + } + // SAFETY: the handle is filled in and stays allocated until it is + // unregistered. + let ret = unsafe { bindings::input_register_handle(handle) }; + if ret < 0 { + // SAFETY: allocated above and never registered. + drop(unsafe { KBox::from_raw(lid) }); + return ret; + } + // SAFETY: the handle is registered. + let ret = unsafe { bindings::input_open_device(handle) }; + if ret < 0 { + // Nothing was recorded for the handle (`LidHandle::record`). + // SAFETY: registered above, not open; nothing else refers to it. + unsafe { + bindings::input_unregister_handle(handle); + drop(KBox::from_raw(lid)); + } + return ret; + } + // Events reach the handle from here on: enable readings and read the + // switch as it is now, in one step under the event lock. + // SAFETY: the handle is registered and open, and so is the handler. + unsafe { lid_resync(handle, true) }; + 0 +} + +/// # Safety +/// +/// Called by the input core with a handle `lid_connect` registered, while +/// the handler is still registered. +unsafe extern "C" fn lid_disconnect(handle: *mut bindings::input_handle) { + // SAFETY: the handle is registered and open; after this no event + // reaches it. + unsafe { + bindings::input_close_device(handle); + bindings::input_unregister_handle(handle); + } + // SAFETY: the handle is the first field of the `LidHandle` allocated by + // `lid_connect`, and the handler is still registered. + let (lid, data) = unsafe { (KBox::from_raw(handle.cast::()), lid_data(handle)) }; + data.lid_forget(&lid); +} + // ---- the driver's own card: the low-power microphone (PCM 0) and the // microphone array (PCM 1) ---- @@ -2545,6 +2871,7 @@ struct SndSocT8140AopDriver { data: Arc, asoc: KBox, asoc_registered: bool, + lid_registered: bool, closed: AtomicFlag, } @@ -2600,6 +2927,7 @@ impl SndSocT8140AopDriver { asoc: AsocContext::new(data.clone())?, data: data.clone(), asoc_registered: false, + lid_registered: false, closed: AtomicFlag::new(false), }; @@ -2655,6 +2983,32 @@ impl SndSocT8140AopDriver { this.new_pcm(0, c"lpai", c"Low-Power Microphone", &Self::LPAI_OPS)?; this.new_pcm(1, c"hpai", c"High-Quality Microphone", &Self::HPAI_OPS)?; + // Both microphones' jack: unplugged while a lid is closed. + let mut jack = ptr::null_mut(); + // SAFETY: `card` was created by `snd_card_new` above and is not registered yet; the jack + // becomes one of its devices and is freed with it. + let ret = unsafe { + bindings::snd_jack_new( + this.card, + c"Internal Mic".as_char_ptr(), + bindings::snd_jack_types_SND_JACK_MICROPHONE as i32, + &mut jack, + true, + // No input device of its own: the lid it follows already + // is one, and the control is what userspace reads. + true, + ) + }; + if ret < 0 { + dev_err!(data.dev, "unable to create the microphone jack\n"); + return Err(Error::from_errno(ret)); + } + data.mic_jack.store(jack, Relaxed); + this.register_lid_handler()?; + // The state as the card is registered, from the only reporter. + data.as_arc_borrow().queue_jack_report(); + data.flush_jack_report(); + // SAFETY: `card` was created by `snd_card_new` above and is not registered yet. let ret = unsafe { bindings::snd_card_register(this.card) }; if ret < 0 { @@ -2682,6 +3036,36 @@ impl SndSocT8140AopDriver { Ok(this) } + /// Follow the lid switches: the input core connects the handler to every + /// device with one, now and as they come. The handler's private data is + /// an `Arc` of the driver data, reclaimed in `teardown`. + fn register_lid_handler(&mut self) -> Result<()> { + let handler = self.data.lid_handler.0.get(); + // SAFETY: the handler lives in the driver data, which this driver + // keeps allocated until after unregistering it, and is not + // registered yet. + unsafe { + (*handler).private = self.data.clone().into_foreign().cast(); + (*handler).event = Some(lid_event); + (*handler).connect = Some(lid_connect); + (*handler).disconnect = Some(lid_disconnect); + (*handler).start = Some(lid_start); + (*handler).passive_observer = true; + (*handler).name = c"t8140-aop-audio".as_char_ptr(); + (*handler).id_table = LID_IDS.as_ptr(); + } + // SAFETY: the handler is filled in above. + let ret = unsafe { bindings::input_register_handler(handler) }; + if ret < 0 { + // SAFETY: the `Arc` leaked above; the handler never registered. + drop(unsafe { Arc::::from_foreign((*handler).private.cast()) }); + dev_err!(self.data.dev, "unable to follow the lid switch: {}\n", ret); + return Err(Error::from_errno(ret)); + } + self.lid_registered = true; + Ok(()) + } + /// One capture PCM of the card, with the driver data as its private /// data. fn new_pcm( @@ -2737,6 +3121,18 @@ impl SndSocT8140AopDriver { return; } self.data.running.store(false, Relaxed); + if self.lid_registered { + let handler = self.data.lid_handler.0.get(); + // SAFETY: registered in `register_lid_handler`; unregistering + // disconnects every handle, after which no callback runs. + unsafe { bindings::input_unregister_handler(handler) }; + // SAFETY: the `Arc` leaked into the handler, unused from here on. + drop(unsafe { Arc::::from_foreign((*handler).private.cast()) }); + } + // No lid change can queue a jack report any more: let the last one + // finish before the card, and the jack with it, goes away. + self.data.flush_jack_report(); + self.data.mic_jack.store(ptr::null_mut(), Relaxed); // Stop reports before ALSA destroys any substream or module callback. self.data .adata @@ -2882,6 +3278,10 @@ impl platform::Driver for SndSocT8140AopDriver { spkr_volume: Atomic::new(SPKR_VOLUME_MAX), spkr_start_work <- new_work!("SndSocT8140AopData::spkr_start_work"), hpai_start_work <- new_work!("SndSocT8140AopData::hpai_start_work"), + lid: LidState::new(), + lid_handler: LidHandler(Opaque::zeroed()), + mic_jack: Atomic::new(ptr::null_mut()), + jack_work <- new_work!("SndSocT8140AopData::jack_work"), stream <- new_spinlock!(StreamState { substream: ptr::null_mut(), copied: None, @@ -3105,3 +3505,134 @@ mod name_tests { assert_eq!(field, [0u8; 16]); } } + +/// The lid bookkeeping behind the microphone jack. +#[kunit_tests(snd_soc_t8140_aop_audio_lid)] +mod lid_tests { + use super::*; + + fn handle() -> LidHandle { + LidHandle { + handle: Opaque::zeroed(), + closed: AtomicFlag::new(false), + open: AtomicFlag::new(false), + } + } + + fn opened() -> LidHandle { + let h = handle(); + h.open.store(true, Relaxed); + h + } + + #[test] + fn a_closed_lid_unplugs_the_microphone_jack() { + let lid = LidState::new(); + let switch = opened(); + assert!(!lid.mic_cut()); + assert_ne!(mic_jack_status(false), 0); + + assert!(switch.record(&lid, true)); + assert!(lid.mic_cut()); + assert_eq!(mic_jack_status(lid.mic_cut()), 0); + + assert!(switch.record(&lid, false)); + assert!(!lid.mic_cut()); + } + + #[test] + fn a_repeated_switch_state_changes_nothing() { + let lid = LidState::new(); + let switch = opened(); + // the reading at connect, then the same state again from an event + assert!(!switch.record(&lid, false)); + assert!(switch.record(&lid, true)); + assert!(!switch.record(&lid, true)); + assert!(lid.mic_cut()); + assert!(switch.record(&lid, false)); + assert!(!switch.record(&lid, false)); + assert!(!lid.mic_cut()); + } + + #[test] + fn any_closed_lid_switch_cuts_the_microphones() { + let lid = LidState::new(); + let (a, b) = (opened(), opened()); + a.record(&lid, true); + b.record(&lid, true); + a.record(&lid, false); + assert!(lid.mic_cut()); + b.record(&lid, false); + assert!(!lid.mic_cut()); + } + + #[test] + fn a_lid_switch_that_goes_away_closed_restores_the_microphones() { + let lid = LidState::new(); + let switch = opened(); + switch.record(&lid, true); + assert!(switch.forget(&lid)); + assert!(!lid.mic_cut()); + assert_eq!(lid.closed.load(Relaxed), 0); + // nothing reaches a forgotten handle any more + assert!(!switch.record(&lid, true)); + assert!(!lid.mic_cut()); + } + + /// Re-review finding 2: `lid_start()` runs from `input_register_handle()` + /// before the open; a handle whose open then fails is freed without + /// `lid_disconnect()`, so its closed reading must never have counted. A + /// later handle for the same device that opens counts normally. + #[test] + fn a_failed_open_leaves_no_closed_switch_behind() { + let lid = LidState::new(); + let failed = handle(); + // lid_start() during input_register_handle(), switch closed + assert!(!failed.record(&lid, true)); + // input_open_device() fails: unregistered and freed, no disconnect + drop(failed); + assert!(!lid.mic_cut()); + assert_eq!(lid.closed.load(Relaxed), 0); + + let attached = handle(); + // lid_start() during registration, before the open + assert!(!attached.sync(&lid, true, false)); + // opened: lid_connect() enables and reads the switch in one step + assert!(attached.sync(&lid, true, true)); + assert!(lid.mic_cut()); + assert!(attached.forget(&lid)); + assert!(!lid.mic_cut()); + } + + /// Review of v3, minor 3: an initially closed switch must count once the + /// handle is open. `lid_connect()` enables readings and reads the switch + /// in the same step, in that order; reading first would drop the reading + /// and leave the jack plugged until the next lid event. + #[test] + fn a_connect_enables_readings_before_reading_the_switch() { + let lid = LidState::new(); + let switch = handle(); + assert!(switch.sync(&lid, true, true)); + assert!(lid.mic_cut()); + // the start() from a grab release reads without enabling anything + let other = handle(); + assert!(!other.sync(&lid, true, false)); + assert_eq!(lid.closed.load(Relaxed), 1); + } + + /// Events delivered only to a client that grabbed the device never reach + /// the handler; the resynchronization on release reads the device's + /// switch and repairs the state, in both directions. + #[test] + fn a_resync_after_missed_events_repairs_the_state() { + let lid = LidState::new(); + let switch = opened(); + switch.record(&lid, true); + // grabbed: the lid opens unseen; on release the bitmap reads open + assert!(switch.record(&lid, false)); + assert!(!lid.mic_cut()); + // grabbed: the lid closes unseen; on release the bitmap reads closed + assert!(switch.record(&lid, true)); + assert!(lid.mic_cut()); + } +} From 44e53339f6c5147001f4869e52ebb4c7fbf993d2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 00:19:47 +1100 Subject: [PATCH 1235/1579] usb: typec: tipd: Factor out the CD321x DisplayPort hotplug decisions cd321x_update_work() decides inline whether a cable update first reports the external display as disconnected, and whether it reports the display as connected once the mux is set. The conditions mix Type-C session state with the DisplayPort HPD and pin assignment bits of DATA_STATUS, and so far they could only be exercised with real cables. Move them into cd321x-dp.h as pure functions of the update's DATA_STATUS, the bits that changed since the previous update and whether a display route was up, and add KUnit tests that pin down the current behaviour. No functional change. [ryan: rebased onto aurora-wip 0daed2db + ATC PIPE park v3. Kconfig and Makefile: the tests' help text and objects follow those of the interrupt (#338) and held-mux (PIPE park v3) tests. core.c: cd321x_dp_notify() follows v3's cd321x_retry_held_mux() and cd321x_update_done(), and the connected event is sent before v3's cd321x_update_done(). No other change.] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/Kconfig | 2 + drivers/usb/typec/tipd/Makefile | 2 +- drivers/usb/typec/tipd/cd321x-dp-test.c | 148 ++++++++++++++++++++++++ drivers/usb/typec/tipd/cd321x-dp.h | 62 ++++++++++ drivers/usb/typec/tipd/core.c | 37 +++--- 5 files changed, 233 insertions(+), 18 deletions(-) create mode 100644 drivers/usb/typec/tipd/cd321x-dp-test.c create mode 100644 drivers/usb/typec/tipd/cd321x-dp.h diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index e0ee8fa1f55b6..86c84b161aa27 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -47,6 +47,8 @@ config TYPEC_TIPD_KUNIT_TEST about two minutes, also next to a display route that declares nothing, and that other providers and errors keep the bounded retries. + They also cover which DisplayPort hotplug event a cable update sends + to the display connector. These tests run without accessing a physical Type-C controller. config TYPEC_SN201202X_KUNIT_TEST diff --git a/drivers/usb/typec/tipd/Makefile b/drivers/usb/typec/tipd/Makefile index 0138a734b7a2e..d8b3b0f4ec5ec 100644 --- a/drivers/usb/typec/tipd/Makefile +++ b/drivers/usb/typec/tipd/Makefile @@ -13,5 +13,5 @@ obj-$(CONFIG_TYPEC_SN201202X) += sn201202x.o sn201202x-y := spmi.o obj-$(CONFIG_TYPEC_TIPD_KUNIT_TEST) += cd321x-pm-test.o cd321x-tbt-test.o \ - cd321x-irq-test.o cd321x-mux-test.o + cd321x-irq-test.o cd321x-mux-test.o cd321x-dp-test.o obj-$(CONFIG_TYPEC_SN201202X_KUNIT_TEST) += sn201202x-pm-test.o diff --git a/drivers/usb/typec/tipd/cd321x-dp-test.c b/drivers/usb/typec/tipd/cd321x-dp-test.c new file mode 100644 index 0000000000000..d19881bf17774 --- /dev/null +++ b/drivers/usb/typec/tipd/cd321x-dp-test.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "cd321x-dp.h" + +/* A dock in pin assignment D: DisplayPort plus USB 3. */ +#define DP_D (TPS_DATA_STATUS_DATA_CONNECTION | TPS_DATA_STATUS_USB2_CONNECTION | \ + TPS_DATA_STATUS_USB3_CONNECTION | TPS_DATA_STATUS_DP_CONNECTION | \ + FIELD_PREP(TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK, 1)) +#define HPD CD321X_DATA_STATUS_HPD_LEVEL + +static void cd321x_dp_first_hpd_connects_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = DP_D | HPD, + .changed = DP_D | HPD, + }; + + KUNIT_EXPECT_FALSE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); +} + +/* Entering DP without HPD sets up the route but announces nothing. */ +static void cd321x_dp_route_without_hpd_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = DP_D, + .changed = DP_D, + }; + + KUNIT_EXPECT_FALSE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_NONE); +} + +static void cd321x_dp_unplug_tears_down_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .changed = DP_D | HPD, + .link_reset = true, + .route_active = true, + }; + + KUNIT_EXPECT_TRUE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, false), CD321X_DP_EVENT_NONE); +} + +/* A flip or a new partner rebuilds the route even with HPD still high. */ +static void cd321x_dp_link_reset_reconnects_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = DP_D | HPD, + .link_reset = true, + .route_active = true, + }; + + KUNIT_EXPECT_TRUE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); +} + +static void cd321x_dp_hpd_low_tears_down_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = DP_D, + .changed = HPD, + .route_active = true, + }; + + KUNIT_EXPECT_TRUE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_NONE); +} + +static void cd321x_dp_exit_tears_down_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = TPS_DATA_STATUS_DATA_CONNECTION | + TPS_DATA_STATUS_USB2_CONNECTION | + TPS_DATA_STATUS_USB3_CONNECTION, + .changed = TPS_DATA_STATUS_DP_CONNECTION | HPD | + TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK, + .route_active = true, + }; + + KUNIT_EXPECT_TRUE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, false), CD321X_DP_EVENT_NONE); +} + +/* A new pin assignment rebuilds the lanes under the stream. */ +static void cd321x_dp_pin_change_reconnects_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = DP_D | HPD, + .changed = TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK, + .route_active = true, + }; + + KUNIT_EXPECT_TRUE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); +} + +/* A power or USB-only update leaves a live display alone. */ +static void cd321x_dp_unrelated_update_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = DP_D | HPD, + .changed = TPS_DATA_STATUS_USB_DATA_ROLE, + .route_active = true, + }; + + KUNIT_EXPECT_FALSE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); +} + +/* A port that does not own the display never reports for it. */ +static void cd321x_dp_display_taken_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = DP_D | HPD, + .changed = DP_D | HPD, + }; + + KUNIT_EXPECT_FALSE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, false), CD321X_DP_EVENT_NONE); + u.data_status = 0; + KUNIT_EXPECT_FALSE(test, cd321x_dp_teardown(&u)); +} + +static struct kunit_case cd321x_dp_cases[] = { + KUNIT_CASE(cd321x_dp_first_hpd_connects_test), + KUNIT_CASE(cd321x_dp_route_without_hpd_test), + KUNIT_CASE(cd321x_dp_unplug_tears_down_test), + KUNIT_CASE(cd321x_dp_link_reset_reconnects_test), + KUNIT_CASE(cd321x_dp_hpd_low_tears_down_test), + KUNIT_CASE(cd321x_dp_exit_tears_down_test), + KUNIT_CASE(cd321x_dp_pin_change_reconnects_test), + KUNIT_CASE(cd321x_dp_unrelated_update_test), + KUNIT_CASE(cd321x_dp_display_taken_test), + {} +}; + +static struct kunit_suite cd321x_dp_suite = { + .name = "cd321x-dp", + .test_cases = cd321x_dp_cases, +}; + +kunit_test_suite(cd321x_dp_suite); +MODULE_DESCRIPTION("KUnit tests for the CD321x DisplayPort hotplug decisions"); +MODULE_LICENSE("GPL"); diff --git a/drivers/usb/typec/tipd/cd321x-dp.h b/drivers/usb/typec/tipd/cd321x-dp.h new file mode 100644 index 0000000000000..ae043a13aa383 --- /dev/null +++ b/drivers/usb/typec/tipd/cd321x-dp.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __CD321X_DP_H__ +#define __CD321X_DP_H__ + +#include + +#include "tps6598x.h" + +/* DATA_STATUS bits whose change rebuilds the DisplayPort lanes. */ +#define CD321X_DP_MODE_BITS \ + (TPS_DATA_STATUS_DP_CONNECTION | TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK) + +/* What the display connector is told once a cable update has been applied. */ +enum cd321x_dp_event { + CD321X_DP_EVENT_NONE, + CD321X_DP_EVENT_CONNECTED, +}; + +/** + * struct cd321x_dp_update - DisplayPort view of one cable update + * @data_status: DATA_STATUS of this update + * @changed: DATA_STATUS bits that changed since the previous update + * @link_reset: the Type-C session is torn down: unplug, flip, partner + * change, Thunderbolt/USB4 entry or exit, or a forced reconnect + * @route_active: the previous update left a display route up + */ +struct cd321x_dp_update { + u32 data_status; + u32 changed; + bool link_reset; + bool route_active; +}; + +static inline bool cd321x_dp_mode_changed(u32 changed) +{ + return changed & CD321X_DP_MODE_BITS; +} + +/* Does the display connector lose its sink before this update is applied? */ +static inline bool cd321x_dp_teardown(const struct cd321x_dp_update *u) +{ + if (!u->route_active) + return false; + + return u->link_reset || + !(u->data_status & TPS_DATA_STATUS_DP_CONNECTION) || + !(u->data_status & CD321X_DATA_STATUS_HPD_LEVEL) || + (u->changed & CD321X_DATA_STATUS_HPD_LEVEL) || + cd321x_dp_mode_changed(u->changed); +} + +/* @route_up: the update left this port's display route up. */ +static inline enum cd321x_dp_event +cd321x_dp_event(const struct cd321x_dp_update *u, bool route_up) +{ + if (!route_up || !(u->data_status & CD321X_DATA_STATUS_HPD_LEVEL)) + return CD321X_DP_EVENT_NONE; + + return CD321X_DP_EVENT_CONNECTED; +} + +#endif /* __CD321X_DP_H__ */ diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index bc94b08615458..945965e6644aa 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -24,6 +24,7 @@ #include #include +#include "cd321x-dp.h" #include "tps6598x.h" #include "trace.h" @@ -811,6 +812,13 @@ static void cd321x_update_done(struct cd321x *cd321x) cd321x_pm_complete(&cd321x->pm); } +static void cd321x_dp_notify(struct cd321x *cd321x, + enum drm_connector_status status) +{ + if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode) + drm_connector_oob_hotplug_event(cd321x->connector_fwnode, status); +} + static void cd321x_update_work(struct work_struct *work) { struct cd321x *cd321x = container_of(to_delayed_work(work), @@ -840,14 +848,13 @@ static void cd321x_update_work(struct work_struct *work) bool usb_connection = st.data_status & (TPS_DATA_STATUS_USB2_CONNECTION | TPS_DATA_STATUS_USB3_CONNECTION); - bool dp_mode_changed = st.data_status_changed & - (TPS_DATA_STATUS_DP_CONNECTION | - TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK); - bool dp_route_was_active = cd321x->display_route_active; + bool dp_mode_changed = cd321x_dp_mode_changed(st.data_status_changed); bool dp_connected = st.data_status & TPS_DATA_STATUS_DP_CONNECTION; - - bool dp_hpd = st.data_status & CD321X_DATA_STATUS_HPD_LEVEL; - bool dp_hpd_changed = st.data_status_changed & CD321X_DATA_STATUS_HPD_LEVEL; + struct cd321x_dp_update dp = { + .data_status = st.data_status, + .changed = st.data_status_changed, + .route_active = cd321x->display_route_active, + }; enum usb_role old_role = usb_role_switch_get_role(tps->role_sw); enum usb_role new_role = USB_ROLE_NONE; @@ -884,12 +891,9 @@ static void cd321x_update_work(struct work_struct *work) * sink can enter Alt Mode while its mux request fails with -ENOSPC, and * must not tear down the display carried by the other port. */ - if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && - dp_route_was_active && - (!new_connected || was_disconnected || !dp_connected || !dp_hpd || - dp_hpd_changed || dp_mode_changed)) - drm_connector_oob_hotplug_event(cd321x->connector_fwnode, - connector_status_disconnected); + dp.link_reset = !new_connected || was_disconnected; + if (cd321x_dp_teardown(&dp)) + cd321x_dp_notify(cd321x, connector_status_disconnected); /* * ACIO carries the tunneled PCIe reset handshake over the still-live @@ -1001,10 +1005,9 @@ static void cd321x_update_work(struct work_struct *work) if (ret) goto retry; - if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && - cd321x->display_route_active && dp_hpd) - drm_connector_oob_hotplug_event(cd321x->connector_fwnode, - connector_status_connected); + if (cd321x_dp_event(&dp, cd321x->display_route_active) == + CD321X_DP_EVENT_CONNECTED) + cd321x_dp_notify(cd321x, connector_status_connected); power_supply_changed(tps->psy); cd321x_update_done(cd321x); From 71729d27c6f5108c837a1c67a228726ee9c6e707 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:26:16 +1100 Subject: [PATCH 1236/1579] usb: typec: tipd: Trace the CD321x DisplayPort hotplug events A CD321x cable update can tell the display connector that its sink has gone, or that it is present again. The data status trace shows each register read, but not the debounced update that produced the event, so a display that drops out after a plug cannot be traced back to the HPD or pin assignment transition that caused it. Add a cd321x_dp_hotplug tracepoint where the event is sent. It records the connector status, whether a display route was up, the HPD level, the pin assignment and the DATA_STATUS bits that changed since the previous update. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/core.c | 12 +++++++---- drivers/usb/typec/tipd/trace.h | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 945965e6644aa..40cd1fc0089f0 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -813,10 +813,14 @@ static void cd321x_update_done(struct cd321x *cd321x) } static void cd321x_dp_notify(struct cd321x *cd321x, + const struct cd321x_dp_update *u, enum drm_connector_status status) { - if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode) - drm_connector_oob_hotplug_event(cd321x->connector_fwnode, status); + if (!IS_REACHABLE(CONFIG_DRM) || !cd321x->connector_fwnode) + return; + + trace_cd321x_dp_hotplug(u, status); + drm_connector_oob_hotplug_event(cd321x->connector_fwnode, status); } static void cd321x_update_work(struct work_struct *work) @@ -893,7 +897,7 @@ static void cd321x_update_work(struct work_struct *work) */ dp.link_reset = !new_connected || was_disconnected; if (cd321x_dp_teardown(&dp)) - cd321x_dp_notify(cd321x, connector_status_disconnected); + cd321x_dp_notify(cd321x, &dp, connector_status_disconnected); /* * ACIO carries the tunneled PCIe reset handshake over the still-live @@ -1007,7 +1011,7 @@ static void cd321x_update_work(struct work_struct *work) if (cd321x_dp_event(&dp, cd321x->display_route_active) == CD321X_DP_EVENT_CONNECTED) - cd321x_dp_notify(cd321x, connector_status_connected); + cd321x_dp_notify(cd321x, &dp, connector_status_connected); power_supply_changed(tps->psy); cd321x_update_done(cd321x); diff --git a/drivers/usb/typec/tipd/trace.h b/drivers/usb/typec/tipd/trace.h index e9e40425138a0..bbd963b7890b6 100644 --- a/drivers/usb/typec/tipd/trace.h +++ b/drivers/usb/typec/tipd/trace.h @@ -12,8 +12,10 @@ #if !defined(_TPS6598X_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ) #define _TPS6598X_TRACE_H_ +#include "cd321x-dp.h" #include "tps6598x.h" +#include #include #include #include @@ -427,6 +429,42 @@ TRACE_EVENT(cd321x_data_status, ) ); +TRACE_DEFINE_ENUM(connector_status_connected); +TRACE_DEFINE_ENUM(connector_status_disconnected); +TRACE_DEFINE_ENUM(connector_status_unknown); + +#define show_dp_hotplug_status(status) \ + __print_symbolic(status, \ + { connector_status_connected, "connected" }, \ + { connector_status_disconnected, "disconnected" }, \ + { connector_status_unknown, "unknown" }) + +TRACE_EVENT(cd321x_dp_hotplug, + TP_PROTO(const struct cd321x_dp_update *u, + enum drm_connector_status status), + TP_ARGS(u, status), + + TP_STRUCT__entry(__field(u32, data_status) + __field(u32, changed) + __field(bool, route_active) + __field(int, status)), + + TP_fast_assign(__entry->data_status = u->data_status; + __entry->changed = u->changed; + __entry->route_active = u->route_active; + __entry->status = status;), + + TP_printk("%s: route %d, HPD %d%s%s, changed %s%s", + show_dp_hotplug_status(__entry->status), + __entry->route_active, + !!(__entry->data_status & CD321X_DATA_STATUS_HPD_LEVEL), + __entry->data_status & TPS_DATA_STATUS_DP_CONNECTION ? ", DP pinout " : "", + maybe_show_data_status_dp_pin_assignment(__entry->data_status), + show_cd321x_data_status_flags(__entry->changed), + __entry->changed & TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK ? + "|DP_PIN_ASSIGNMENT" : "") +); + #endif /* _TPS6598X_TRACE_H_ */ /* This part must be outside protection */ From 2e1645484a7295e595fd8360048dbeebd899d440 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:27:18 +1100 Subject: [PATCH 1237/1579] usb: typec: tipd: Retrain the display link on an HPD pulse A CD321x cable update reports the external display as disconnected and then as connected again whenever the HPD level changed since the previous update. Updates run CD321X_DEBOUNCE_DELAY_MS after the last interrupt and accumulate the DATA_STATUS bits that changed in between, so a sink that drops HPD for less than the debounce and raises it again looks the same to this check as a sink that was unplugged. The display connector then tears down the pipeline and rebuilds it, and the picture drops although the sink never left. The DisplayPort Alt Mode driver (altmodes/displayport.c) forwards the HPD level as it is reported and handles IRQ_HPD as a separate attention event; it never reports a disconnect while HPD is high. Do the same. When the partner, the alternate mode and the pin assignment are unchanged and HPD was high before the update and is high after it, keep the sink and report connector_status_unknown, which asks the connector to recheck and retrain its existing link. Handle a newly set HPD_IRQ with HPD held high the same way. "HPD was high" has to mean that the connector was told about the sink. The cached mux status cannot answer that: an update that reported the sink gone and then failed with -EAGAIN keeps the route marked up and the old status, and its retry would retrain a link the connector no longer has. Record the last event sent to the connector instead. C and D, and E and F, differ only in USB3_CONNECTION, and a change between them changes the DisplayPort lane count. A pulse that comes with such a change still reconnects the display. Every other case is unchanged. An unplug, flip, new partner, DisplayPort exit, pin assignment change or HPD low still tears the display down, and HPD rising after a reported low still announces it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-dp-test.c | 128 ++++++++++++++++++++++++ drivers/usb/typec/tipd/cd321x-dp.h | 40 +++++++- drivers/usb/typec/tipd/core.c | 14 ++- drivers/usb/typec/tipd/tps6598x.h | 1 + drivers/usb/typec/tipd/trace.h | 6 +- 5 files changed, 180 insertions(+), 9 deletions(-) diff --git a/drivers/usb/typec/tipd/cd321x-dp-test.c b/drivers/usb/typec/tipd/cd321x-dp-test.c index d19881bf17774..8d52aac57ee6c 100644 --- a/drivers/usb/typec/tipd/cd321x-dp-test.c +++ b/drivers/usb/typec/tipd/cd321x-dp-test.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only #include +#include #include #include "cd321x-dp.h" @@ -125,6 +126,126 @@ static void cd321x_dp_display_taken_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, cd321x_dp_teardown(&u)); } +/* + * HPD went low and back within the debounce on the same dock and lanes: the + * route is kept and its link rechecked, so the picture does not drop. + */ +static void cd321x_dp_hpd_pulse_retrains_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = DP_D | HPD, + .changed = HPD, + .route_active = true, + .sink_reported = true, + }; + + KUNIT_EXPECT_FALSE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_RETRAIN); +} + +/* An IRQ_HPD attention with HPD held high asks for a link check. */ +static void cd321x_dp_irq_hpd_retrains_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ, + .changed = CD321X_DATA_STATUS_HPD_IRQ, + .route_active = true, + .sink_reported = true, + }; + + KUNIT_EXPECT_FALSE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_RETRAIN); + + /* The attention bit clearing again is not a new request. */ + u.data_status &= ~CD321X_DATA_STATUS_HPD_IRQ; + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); +} + +/* HPD rising after a reported low is a connect, as before. */ +static void cd321x_dp_hpd_rise_connects_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = DP_D | HPD, + .changed = HPD, + .route_active = true, + }; + + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); + /* Attention is no reason to skip announcing a sink that just came up. */ + u.data_status |= CD321X_DATA_STATUS_HPD_IRQ; + u.changed |= CD321X_DATA_STATUS_HPD_IRQ; + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); +} + +/* C <-> D changes the DisplayPort lane count: no pulse, a reconnect. */ +static void cd321x_dp_pulse_with_lane_change_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = DP_D | HPD, + .changed = HPD | TPS_DATA_STATUS_USB3_CONNECTION, + .route_active = true, + .sink_reported = true, + }; + + KUNIT_EXPECT_TRUE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); + + u.changed = HPD | TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK; + KUNIT_EXPECT_TRUE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); +} + +/* A replug, flip or new dock inside the debounce is still a reconnect. */ +static void cd321x_dp_pulse_with_link_reset_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ, + .changed = HPD | CD321X_DATA_STATUS_HPD_IRQ, + .link_reset = true, + .route_active = true, + .sink_reported = true, + }; + + KUNIT_EXPECT_TRUE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); +} + +/* A route without a delivered HPD has nothing to retrain yet. */ +static void cd321x_dp_retrain_needs_live_sink_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ, + .changed = CD321X_DATA_STATUS_HPD_IRQ, + .route_active = true, + }; + + KUNIT_EXPECT_FALSE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); + + /* A port that just entered DP connects rather than retrains. */ + u.route_active = false; + u.sink_reported = true; + u.changed = HPD; + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); +} + +/* + * A pass that reported the sink gone and then failed leaves the route + * marked up. When the retry finds HPD high again the sink must be + * announced, not retrained: the connector has no link left to retrain. + */ +static void cd321x_dp_retry_after_teardown_test(struct kunit *test) +{ + struct cd321x_dp_update u = { + .data_status = DP_D | HPD, + .changed = HPD, + .route_active = true, + }; + + KUNIT_EXPECT_TRUE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); +} + static struct kunit_case cd321x_dp_cases[] = { KUNIT_CASE(cd321x_dp_first_hpd_connects_test), KUNIT_CASE(cd321x_dp_route_without_hpd_test), @@ -135,6 +256,13 @@ static struct kunit_case cd321x_dp_cases[] = { KUNIT_CASE(cd321x_dp_pin_change_reconnects_test), KUNIT_CASE(cd321x_dp_unrelated_update_test), KUNIT_CASE(cd321x_dp_display_taken_test), + KUNIT_CASE(cd321x_dp_hpd_pulse_retrains_test), + KUNIT_CASE(cd321x_dp_irq_hpd_retrains_test), + KUNIT_CASE(cd321x_dp_hpd_rise_connects_test), + KUNIT_CASE(cd321x_dp_pulse_with_lane_change_test), + KUNIT_CASE(cd321x_dp_pulse_with_link_reset_test), + KUNIT_CASE(cd321x_dp_retrain_needs_live_sink_test), + KUNIT_CASE(cd321x_dp_retry_after_teardown_test), {} }; diff --git a/drivers/usb/typec/tipd/cd321x-dp.h b/drivers/usb/typec/tipd/cd321x-dp.h index ae043a13aa383..2f88f5ae5e761 100644 --- a/drivers/usb/typec/tipd/cd321x-dp.h +++ b/drivers/usb/typec/tipd/cd321x-dp.h @@ -14,6 +14,7 @@ enum cd321x_dp_event { CD321X_DP_EVENT_NONE, CD321X_DP_EVENT_CONNECTED, + CD321X_DP_EVENT_RETRAIN, }; /** @@ -23,12 +24,14 @@ enum cd321x_dp_event { * @link_reset: the Type-C session is torn down: unplug, flip, partner * change, Thunderbolt/USB4 entry or exit, or a forced reconnect * @route_active: the previous update left a display route up + * @sink_reported: the connector was last told that a sink is present */ struct cd321x_dp_update { u32 data_status; u32 changed; bool link_reset; bool route_active; + bool sink_reported; }; static inline bool cd321x_dp_mode_changed(u32 changed) @@ -36,17 +39,37 @@ static inline bool cd321x_dp_mode_changed(u32 changed) return changed & CD321X_DP_MODE_BITS; } +/* + * HPD dropped for less than the debounce delay and came back high while the + * partner, the alternate mode and the pin assignment stayed the same. The + * sink is still there, so check and retrain its link rather than report an + * unplug. USB3_CONNECTION is part of the pin assignment (C/D, E/F), so a + * pulse that comes with a lane count change is not one of these. + */ +static inline bool cd321x_dp_hpd_pulse(const struct cd321x_dp_update *u) +{ + return u->route_active && !u->link_reset && u->sink_reported && + (u->data_status & TPS_DATA_STATUS_DP_CONNECTION) && + (u->data_status & CD321X_DATA_STATUS_HPD_LEVEL) && + (u->changed & CD321X_DATA_STATUS_HPD_LEVEL) && + !cd321x_dp_mode_changed(u->changed) && + !(u->changed & TPS_DATA_STATUS_USB3_CONNECTION); +} + /* Does the display connector lose its sink before this update is applied? */ static inline bool cd321x_dp_teardown(const struct cd321x_dp_update *u) { if (!u->route_active) return false; - return u->link_reset || - !(u->data_status & TPS_DATA_STATUS_DP_CONNECTION) || - !(u->data_status & CD321X_DATA_STATUS_HPD_LEVEL) || - (u->changed & CD321X_DATA_STATUS_HPD_LEVEL) || - cd321x_dp_mode_changed(u->changed); + if (u->link_reset || + !(u->data_status & TPS_DATA_STATUS_DP_CONNECTION) || + !(u->data_status & CD321X_DATA_STATUS_HPD_LEVEL) || + cd321x_dp_mode_changed(u->changed)) + return true; + + return (u->changed & CD321X_DATA_STATUS_HPD_LEVEL) && + !cd321x_dp_hpd_pulse(u); } /* @route_up: the update left this port's display route up. */ @@ -56,6 +79,13 @@ cd321x_dp_event(const struct cd321x_dp_update *u, bool route_up) if (!route_up || !(u->data_status & CD321X_DATA_STATUS_HPD_LEVEL)) return CD321X_DP_EVENT_NONE; + /* The sink stays connected: have its link checked, not re-announced. */ + if (cd321x_dp_hpd_pulse(u)) + return CD321X_DP_EVENT_RETRAIN; + if (u->route_active && u->sink_reported && !cd321x_dp_teardown(u) && + (u->data_status & u->changed & CD321X_DATA_STATUS_HPD_IRQ)) + return CD321X_DP_EVENT_RETRAIN; + return CD321X_DP_EVENT_CONNECTED; } diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 40cd1fc0089f0..d42750f6e465e 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -820,6 +820,7 @@ static void cd321x_dp_notify(struct cd321x *cd321x, return; trace_cd321x_dp_hotplug(u, status); + cd321x->dp_sink_reported = status != connector_status_disconnected; drm_connector_oob_hotplug_event(cd321x->connector_fwnode, status); } @@ -858,6 +859,7 @@ static void cd321x_update_work(struct work_struct *work) .data_status = st.data_status, .changed = st.data_status_changed, .route_active = cd321x->display_route_active, + .sink_reported = cd321x->dp_sink_reported, }; enum usb_role old_role = usb_role_switch_get_role(tps->role_sw); @@ -1009,9 +1011,17 @@ static void cd321x_update_work(struct work_struct *work) if (ret) goto retry; - if (cd321x_dp_event(&dp, cd321x->display_route_active) == - CD321X_DP_EVENT_CONNECTED) + switch (cd321x_dp_event(&dp, cd321x->display_route_active)) { + case CD321X_DP_EVENT_NONE: + break; + case CD321X_DP_EVENT_CONNECTED: cd321x_dp_notify(cd321x, &dp, connector_status_connected); + break; + case CD321X_DP_EVENT_RETRAIN: + /* An unknown status asks the connector to recheck its link. */ + cd321x_dp_notify(cd321x, &dp, connector_status_unknown); + break; + } power_supply_changed(tps->psy); cd321x_update_done(cd321x); diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 3e1f190b44289..29fcc9247e1f9 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -406,6 +406,7 @@ struct cd321x { bool tbt_notifier_registered; atomic_t link_event; bool display_route_active; + bool dp_sink_reported; struct cd321x_status update_status; struct delayed_work update_work; diff --git a/drivers/usb/typec/tipd/trace.h b/drivers/usb/typec/tipd/trace.h index bbd963b7890b6..cabbc0f9670ea 100644 --- a/drivers/usb/typec/tipd/trace.h +++ b/drivers/usb/typec/tipd/trace.h @@ -447,16 +447,18 @@ TRACE_EVENT(cd321x_dp_hotplug, TP_STRUCT__entry(__field(u32, data_status) __field(u32, changed) __field(bool, route_active) + __field(bool, sink_reported) __field(int, status)), TP_fast_assign(__entry->data_status = u->data_status; __entry->changed = u->changed; __entry->route_active = u->route_active; + __entry->sink_reported = u->sink_reported; __entry->status = status;), - TP_printk("%s: route %d, HPD %d%s%s, changed %s%s", + TP_printk("%s: route %d, sink %d, HPD %d%s%s, changed %s%s", show_dp_hotplug_status(__entry->status), - __entry->route_active, + __entry->route_active, __entry->sink_reported, !!(__entry->data_status & CD321X_DATA_STATUS_HPD_LEVEL), __entry->data_status & TPS_DATA_STATUS_DP_CONNECTION ? ", DP pinout " : "", maybe_show_data_status_dp_pin_assignment(__entry->data_status), From ffdebdac1ec0da1908a00193a7ee59fda4b14bb1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:27:08 +1100 Subject: [PATCH 1238/1579] usb: typec: tipd: Rebuild the CD321x display on a C/D or E/F change The DisplayPort pin assignment that cd321x_typec_update_mode() programs into the mux is decoded from two DATA_STATUS fields: the two-bit DP_PIN_ASSIGNMENT field selects A, B, C/D or E/F, and USB3_CONNECTION picks D over C and F over E. The cable update only looks at DP_CONNECTION and DP_PIN_ASSIGNMENT to decide whether the DisplayPort lanes change. A partner that moves from D to C, or from F to E, while HPD stays high changes USB3_CONNECTION alone. The update then neither reports the display as disconnected nor drops the USB role before the mux is reprogrammed, and the lanes are reallocated from two to four (or back) under a running display stream. Treat USB3_CONNECTION as part of the pin assignment while DisplayPort is connected, in both decisions. Without DisplayPort it remains a plain USB change and still leaves the role alone. The check came with the CD321x DisplayPort hotplug support in the initial import of this tree. Fixes: 38c14f6107df ("drm: apple: support ProMotion variable refresh rate") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-dp-test.c | 48 +++++++++++++++++++++++++ drivers/usb/typec/tipd/cd321x-dp.h | 22 ++++++++---- drivers/usb/typec/tipd/core.c | 3 +- 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/drivers/usb/typec/tipd/cd321x-dp-test.c b/drivers/usb/typec/tipd/cd321x-dp-test.c index 8d52aac57ee6c..fff902169feb5 100644 --- a/drivers/usb/typec/tipd/cd321x-dp-test.c +++ b/drivers/usb/typec/tipd/cd321x-dp-test.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only #include +#include #include #include @@ -10,6 +11,11 @@ TPS_DATA_STATUS_USB3_CONNECTION | TPS_DATA_STATUS_DP_CONNECTION | \ FIELD_PREP(TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK, 1)) #define HPD CD321X_DATA_STATUS_HPD_LEVEL +/* The same dock giving the USB 3 lanes to DisplayPort: pin assignment C. */ +#define DP_C (DP_D & ~TPS_DATA_STATUS_USB3_CONNECTION) +/* A USB-C to DisplayPort cable: pin assignment E, or F with USB 3. */ +#define DP_E (TPS_DATA_STATUS_DATA_CONNECTION | TPS_DATA_STATUS_DP_CONNECTION) +#define DP_F (DP_E | TPS_DATA_STATUS_USB2_CONNECTION | TPS_DATA_STATUS_USB3_CONNECTION) static void cd321x_dp_first_hpd_connects_test(struct kunit *test) { @@ -99,6 +105,46 @@ static void cd321x_dp_pin_change_reconnects_test(struct kunit *test) KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); } +/* + * C <-> D and E <-> F change only USB3_CONNECTION, but they change the + * DisplayPort lane count, so the lanes are rebuilt with HPD held high. + */ +static void cd321x_dp_usb3_pin_change_reconnects_test(struct kunit *test) +{ + const u32 pins[][2] = { + { DP_D, DP_C }, { DP_C, DP_D }, { DP_F, DP_E }, { DP_E, DP_F }, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(pins); i++) { + struct cd321x_dp_update u = { + .data_status = pins[i][1] | HPD, + .changed = pins[i][0] ^ pins[i][1], + .route_active = true, + .sink_reported = true, + }; + + /* Neither DP_CONNECTION nor the pin assignment field moves. */ + KUNIT_EXPECT_FALSE(test, u.changed & CD321X_DP_MODE_BITS); + KUNIT_EXPECT_TRUE(test, cd321x_dp_mode_changed(u.data_status, u.changed)); + KUNIT_EXPECT_TRUE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); + } +} + +/* Without DisplayPort, USB 3 coming or going is not a lane change. */ +static void cd321x_dp_usb3_change_without_dp_test(struct kunit *test) +{ + u32 usb2 = TPS_DATA_STATUS_DATA_CONNECTION | TPS_DATA_STATUS_USB2_CONNECTION; + u32 usb3 = TPS_DATA_STATUS_USB3_CONNECTION; + u32 dp = TPS_DATA_STATUS_DP_CONNECTION; + + KUNIT_EXPECT_FALSE(test, cd321x_dp_mode_changed(usb2, usb3)); + KUNIT_EXPECT_FALSE(test, cd321x_dp_mode_changed(usb2 | usb3, usb3)); + /* Leaving DisplayPort is a mode change whatever happens to USB 3. */ + KUNIT_EXPECT_TRUE(test, cd321x_dp_mode_changed(usb2, dp | usb3)); +} + /* A power or USB-only update leaves a live display alone. */ static void cd321x_dp_unrelated_update_test(struct kunit *test) { @@ -254,6 +300,8 @@ static struct kunit_case cd321x_dp_cases[] = { KUNIT_CASE(cd321x_dp_hpd_low_tears_down_test), KUNIT_CASE(cd321x_dp_exit_tears_down_test), KUNIT_CASE(cd321x_dp_pin_change_reconnects_test), + KUNIT_CASE(cd321x_dp_usb3_pin_change_reconnects_test), + KUNIT_CASE(cd321x_dp_usb3_change_without_dp_test), KUNIT_CASE(cd321x_dp_unrelated_update_test), KUNIT_CASE(cd321x_dp_display_taken_test), KUNIT_CASE(cd321x_dp_hpd_pulse_retrains_test), diff --git a/drivers/usb/typec/tipd/cd321x-dp.h b/drivers/usb/typec/tipd/cd321x-dp.h index 2f88f5ae5e761..19143509a7efa 100644 --- a/drivers/usb/typec/tipd/cd321x-dp.h +++ b/drivers/usb/typec/tipd/cd321x-dp.h @@ -34,17 +34,26 @@ struct cd321x_dp_update { bool sink_reported; }; -static inline bool cd321x_dp_mode_changed(u32 changed) +/* + * USB3_CONNECTION is part of the pin assignment while DisplayPort is up: C + * and D, and E and F, differ only in it, and moving between them changes + * the DisplayPort lane count. Without DisplayPort it is a plain USB change. + */ +static inline bool cd321x_dp_mode_changed(u32 data_status, u32 changed) { - return changed & CD321X_DP_MODE_BITS; + u32 bits = CD321X_DP_MODE_BITS; + + if (data_status & TPS_DATA_STATUS_DP_CONNECTION) + bits |= TPS_DATA_STATUS_USB3_CONNECTION; + + return changed & bits; } /* * HPD dropped for less than the debounce delay and came back high while the * partner, the alternate mode and the pin assignment stayed the same. The * sink is still there, so check and retrain its link rather than report an - * unplug. USB3_CONNECTION is part of the pin assignment (C/D, E/F), so a - * pulse that comes with a lane count change is not one of these. + * unplug. */ static inline bool cd321x_dp_hpd_pulse(const struct cd321x_dp_update *u) { @@ -52,8 +61,7 @@ static inline bool cd321x_dp_hpd_pulse(const struct cd321x_dp_update *u) (u->data_status & TPS_DATA_STATUS_DP_CONNECTION) && (u->data_status & CD321X_DATA_STATUS_HPD_LEVEL) && (u->changed & CD321X_DATA_STATUS_HPD_LEVEL) && - !cd321x_dp_mode_changed(u->changed) && - !(u->changed & TPS_DATA_STATUS_USB3_CONNECTION); + !cd321x_dp_mode_changed(u->data_status, u->changed); } /* Does the display connector lose its sink before this update is applied? */ @@ -65,7 +73,7 @@ static inline bool cd321x_dp_teardown(const struct cd321x_dp_update *u) if (u->link_reset || !(u->data_status & TPS_DATA_STATUS_DP_CONNECTION) || !(u->data_status & CD321X_DATA_STATUS_HPD_LEVEL) || - cd321x_dp_mode_changed(u->changed)) + cd321x_dp_mode_changed(u->data_status, u->changed)) return true; return (u->changed & CD321X_DATA_STATUS_HPD_LEVEL) && diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index d42750f6e465e..afdcf01379969 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -853,7 +853,8 @@ static void cd321x_update_work(struct work_struct *work) bool usb_connection = st.data_status & (TPS_DATA_STATUS_USB2_CONNECTION | TPS_DATA_STATUS_USB3_CONNECTION); - bool dp_mode_changed = cd321x_dp_mode_changed(st.data_status_changed); + bool dp_mode_changed = cd321x_dp_mode_changed(st.data_status, + st.data_status_changed); bool dp_connected = st.data_status & TPS_DATA_STATUS_DP_CONNECTION; struct cd321x_dp_update dp = { .data_status = st.data_status, From 5ebc05befce78d9f9635156069a27162b71bc924 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:31:52 +1100 Subject: [PATCH 1239/1579] usb: typec: tipd: Reconnect the display on every CD321x HPD level change Commit bd00b3c0aa40 ("usb: typec: tipd: Retrain the display link on an HPD pulse") keeps the external display connected when HPD was high before a debounced cable update and is high after it, with the same partner and pin assignment. It asks the connector to retrain instead. The accumulated DATA_STATUS change mask records only that the HPD level changed, not for how long it was low. A monitor unplugged behind a dock and another one plugged in, or a KVM switching sinks, within CD321X_DEBOUNCE_DELAY_MS looks exactly the same. A retrain does not retire the old sink's EDID, so the connector can go on offering the old sink's modes for the new one. DisplayPort Alt Mode already separates the two cases. The partner reports the sink's short IRQ_HPD pulse as an attention with IRQ_HPD set and HPD_State still high, and the CD321x reflects that in HPD_IRQ while HPD_LEVEL stays high. HPD_State goes low only for a long low, which DisplayPort treats as an unplug. The DisplayPort Alt Mode driver (altmodes/displayport.c) likewise forwards every HPD level change it receives and counts IRQ_HPD separately. So report any sampled HPD level change as a disconnect followed by a connect again, as before that commit. Ask for a retrain only for an IRQ_HPD on a sink whose HPD level stayed high. When retraining, first report the sink as connected, as every other update with HPD high does. A connector that already has the sink ignores that, and one that lost it in the meantime connects it again rather than dropping the retrain. Fixes: bd00b3c0aa40 ("usb: typec: tipd: Retrain the display link on an HPD pulse") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-dp-test.c | 33 +++++++++++-------- drivers/usb/typec/tipd/cd321x-dp.h | 43 ++++++++++--------------- drivers/usb/typec/tipd/core.c | 7 +++- 3 files changed, 43 insertions(+), 40 deletions(-) diff --git a/drivers/usb/typec/tipd/cd321x-dp-test.c b/drivers/usb/typec/tipd/cd321x-dp-test.c index fff902169feb5..45848326843de 100644 --- a/drivers/usb/typec/tipd/cd321x-dp-test.c +++ b/drivers/usb/typec/tipd/cd321x-dp-test.c @@ -173,10 +173,11 @@ static void cd321x_dp_display_taken_test(struct kunit *test) } /* - * HPD went low and back within the debounce on the same dock and lanes: the - * route is kept and its link rechecked, so the picture does not drop. + * HPD went low and back within the debounce on the same dock and lanes. + * That is not an IRQ_HPD: the sink may have been replaced behind the dock, + * so it is reported gone and connected again. */ -static void cd321x_dp_hpd_pulse_retrains_test(struct kunit *test) +static void cd321x_dp_hpd_bounce_reconnects_test(struct kunit *test) { struct cd321x_dp_update u = { .data_status = DP_D | HPD, @@ -185,8 +186,14 @@ static void cd321x_dp_hpd_pulse_retrains_test(struct kunit *test) .sink_reported = true, }; - KUNIT_EXPECT_FALSE(test, cd321x_dp_teardown(&u)); - KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_RETRAIN); + KUNIT_EXPECT_TRUE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); + + /* An IRQ_HPD in the same window does not make the bounce harmless. */ + u.data_status |= CD321X_DATA_STATUS_HPD_IRQ; + u.changed |= CD321X_DATA_STATUS_HPD_IRQ; + KUNIT_EXPECT_TRUE(test, cd321x_dp_teardown(&u)); + KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); } /* An IRQ_HPD attention with HPD held high asks for a link check. */ @@ -223,8 +230,8 @@ static void cd321x_dp_hpd_rise_connects_test(struct kunit *test) KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); } -/* C <-> D changes the DisplayPort lane count: no pulse, a reconnect. */ -static void cd321x_dp_pulse_with_lane_change_test(struct kunit *test) +/* An HPD bounce with a C <-> D lane change: a reconnect either way. */ +static void cd321x_dp_bounce_with_lane_change_test(struct kunit *test) { struct cd321x_dp_update u = { .data_status = DP_D | HPD, @@ -241,12 +248,12 @@ static void cd321x_dp_pulse_with_lane_change_test(struct kunit *test) KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); } -/* A replug, flip or new dock inside the debounce is still a reconnect. */ -static void cd321x_dp_pulse_with_link_reset_test(struct kunit *test) +/* A replug, flip or new dock with an IRQ_HPD is still a reconnect. */ +static void cd321x_dp_irq_hpd_with_link_reset_test(struct kunit *test) { struct cd321x_dp_update u = { .data_status = DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ, - .changed = HPD | CD321X_DATA_STATUS_HPD_IRQ, + .changed = CD321X_DATA_STATUS_HPD_IRQ, .link_reset = true, .route_active = true, .sink_reported = true, @@ -304,11 +311,11 @@ static struct kunit_case cd321x_dp_cases[] = { KUNIT_CASE(cd321x_dp_usb3_change_without_dp_test), KUNIT_CASE(cd321x_dp_unrelated_update_test), KUNIT_CASE(cd321x_dp_display_taken_test), - KUNIT_CASE(cd321x_dp_hpd_pulse_retrains_test), + KUNIT_CASE(cd321x_dp_hpd_bounce_reconnects_test), KUNIT_CASE(cd321x_dp_irq_hpd_retrains_test), KUNIT_CASE(cd321x_dp_hpd_rise_connects_test), - KUNIT_CASE(cd321x_dp_pulse_with_lane_change_test), - KUNIT_CASE(cd321x_dp_pulse_with_link_reset_test), + KUNIT_CASE(cd321x_dp_bounce_with_lane_change_test), + KUNIT_CASE(cd321x_dp_irq_hpd_with_link_reset_test), KUNIT_CASE(cd321x_dp_retrain_needs_live_sink_test), KUNIT_CASE(cd321x_dp_retry_after_teardown_test), {} diff --git a/drivers/usb/typec/tipd/cd321x-dp.h b/drivers/usb/typec/tipd/cd321x-dp.h index 19143509a7efa..36d94864fc548 100644 --- a/drivers/usb/typec/tipd/cd321x-dp.h +++ b/drivers/usb/typec/tipd/cd321x-dp.h @@ -10,7 +10,11 @@ #define CD321X_DP_MODE_BITS \ (TPS_DATA_STATUS_DP_CONNECTION | TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK) -/* What the display connector is told once a cable update has been applied. */ +/* + * What the display connector is told once a cable update has been applied: + * nothing, that the sink is present, or that the sink is present and asked + * for its link to be checked. + */ enum cd321x_dp_event { CD321X_DP_EVENT_NONE, CD321X_DP_EVENT_CONNECTED, @@ -50,34 +54,23 @@ static inline bool cd321x_dp_mode_changed(u32 data_status, u32 changed) } /* - * HPD dropped for less than the debounce delay and came back high while the - * partner, the alternate mode and the pin assignment stayed the same. The - * sink is still there, so check and retrain its link rather than report an - * unplug. + * Does the display connector lose its sink before this update is applied? + * + * Any HPD level change counts, however short. The partner reports the + * sink's short IRQ_HPD pulse as an attention with the HPD level held high. + * A low HPD level is how DisplayPort signals an unplug, and the sink seen + * again within the debounce may be a different one behind the same dock. */ -static inline bool cd321x_dp_hpd_pulse(const struct cd321x_dp_update *u) -{ - return u->route_active && !u->link_reset && u->sink_reported && - (u->data_status & TPS_DATA_STATUS_DP_CONNECTION) && - (u->data_status & CD321X_DATA_STATUS_HPD_LEVEL) && - (u->changed & CD321X_DATA_STATUS_HPD_LEVEL) && - !cd321x_dp_mode_changed(u->data_status, u->changed); -} - -/* Does the display connector lose its sink before this update is applied? */ static inline bool cd321x_dp_teardown(const struct cd321x_dp_update *u) { if (!u->route_active) return false; - if (u->link_reset || - !(u->data_status & TPS_DATA_STATUS_DP_CONNECTION) || - !(u->data_status & CD321X_DATA_STATUS_HPD_LEVEL) || - cd321x_dp_mode_changed(u->data_status, u->changed)) - return true; - - return (u->changed & CD321X_DATA_STATUS_HPD_LEVEL) && - !cd321x_dp_hpd_pulse(u); + return u->link_reset || + !(u->data_status & TPS_DATA_STATUS_DP_CONNECTION) || + !(u->data_status & CD321X_DATA_STATUS_HPD_LEVEL) || + (u->changed & CD321X_DATA_STATUS_HPD_LEVEL) || + cd321x_dp_mode_changed(u->data_status, u->changed); } /* @route_up: the update left this port's display route up. */ @@ -87,9 +80,7 @@ cd321x_dp_event(const struct cd321x_dp_update *u, bool route_up) if (!route_up || !(u->data_status & CD321X_DATA_STATUS_HPD_LEVEL)) return CD321X_DP_EVENT_NONE; - /* The sink stays connected: have its link checked, not re-announced. */ - if (cd321x_dp_hpd_pulse(u)) - return CD321X_DP_EVENT_RETRAIN; + /* IRQ_HPD on a sink that stays connected: have its link checked. */ if (u->route_active && u->sink_reported && !cd321x_dp_teardown(u) && (u->data_status & u->changed & CD321X_DATA_STATUS_HPD_IRQ)) return CD321X_DP_EVENT_RETRAIN; diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index afdcf01379969..6e483bd176cc7 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -1019,7 +1019,12 @@ static void cd321x_update_work(struct work_struct *work) cd321x_dp_notify(cd321x, &dp, connector_status_connected); break; case CD321X_DP_EVENT_RETRAIN: - /* An unknown status asks the connector to recheck its link. */ + /* + * Report the HPD level as for any other update, so a connector + * that has lost the sink meanwhile connects it again. Then an + * unknown status asks the connector to recheck its link. + */ + cd321x_dp_notify(cd321x, &dp, connector_status_connected); cd321x_dp_notify(cd321x, &dp, connector_status_unknown); break; } From 1a47b24109395e005e2b4ce1b2985c8809bb496b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:31:53 +1100 Subject: [PATCH 1240/1579] usb: typec: tipd: Keep a CD321x IRQ_HPD pending until it is delivered A debounced cable update takes the DATA_STATUS bits that changed since the previous update and clears them before it programs the mux and the USB role. If either of those fails, the update is retried from a fresh read; with -EAGAIN, for instance while the PHY's USB consumer is not ready yet, the retry keeps the session and does not tear the display down. The fresh read then shows the same DATA_STATUS as before, with no changed bits, so an IRQ_HPD that the failed pass was about to deliver turns into a plain connected event and the retrain it asked for is lost. Moreover, IRQ_HPD is taken from the change mask across the debounced update, so an attention that sets HPD_IRQ and clears it again within CD321X_DEBOUNCE_DELAY_MS is missed. Track the attention on every interrupt instead. TI documents the HPD IRQ bit of DATA_STATUS as sticky, and the partner's DisplayPort status keeps IRQ_HPD until its next status message, so either becoming set while HPD is high is a new IRQ_HPD. Keep it pending until an update has been applied and its event sent. A connect overtakes it, and HPD going low or the partner leaving drops it. Fixes: bd00b3c0aa40 ("usb: typec: tipd: Retrain the display link on an HPD pulse") [ryan: rebased onto aurora-wip 0daed2db + ATC PIPE park v3. A disconnected plug now ends the update with v3's cd321x_update_done() instead of cd321x_pm_complete(); the pending IRQ_HPD is dropped before it, as reviewed. A change that a held mux retries keeps the IRQ_HPD pending, as the bounded retries do.] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-dp-test.c | 48 +++++++++++++++++++++++-- drivers/usb/typec/tipd/cd321x-dp.h | 35 ++++++++++++++++-- drivers/usb/typec/tipd/core.c | 10 ++++++ drivers/usb/typec/tipd/tps6598x.h | 1 + 4 files changed, 90 insertions(+), 4 deletions(-) diff --git a/drivers/usb/typec/tipd/cd321x-dp-test.c b/drivers/usb/typec/tipd/cd321x-dp-test.c index 45848326843de..39eb2a9110309 100644 --- a/drivers/usb/typec/tipd/cd321x-dp-test.c +++ b/drivers/usb/typec/tipd/cd321x-dp-test.c @@ -192,6 +192,7 @@ static void cd321x_dp_hpd_bounce_reconnects_test(struct kunit *test) /* An IRQ_HPD in the same window does not make the bounce harmless. */ u.data_status |= CD321X_DATA_STATUS_HPD_IRQ; u.changed |= CD321X_DATA_STATUS_HPD_IRQ; + u.irq_hpd = true; KUNIT_EXPECT_TRUE(test, cd321x_dp_teardown(&u)); KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); } @@ -202,6 +203,7 @@ static void cd321x_dp_irq_hpd_retrains_test(struct kunit *test) struct cd321x_dp_update u = { .data_status = DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ, .changed = CD321X_DATA_STATUS_HPD_IRQ, + .irq_hpd = true, .route_active = true, .sink_reported = true, }; @@ -209,11 +211,48 @@ static void cd321x_dp_irq_hpd_retrains_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, cd321x_dp_teardown(&u)); KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_RETRAIN); - /* The attention bit clearing again is not a new request. */ - u.data_status &= ~CD321X_DATA_STATUS_HPD_IRQ; + /* The sticky bit seen again, with nothing pending, is no new request. */ + u.irq_hpd = false; KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); } +/* Either IRQ_HPD indication becoming set is an attention. */ +static void cd321x_dp_irq_hpd_sources_test(struct kunit *test) +{ + u32 hpd = DP_D | HPD, irq = hpd | CD321X_DATA_STATUS_HPD_IRQ; + + KUNIT_EXPECT_TRUE(test, cd321x_dp_irq_hpd(false, hpd, irq, 0, 0)); + KUNIT_EXPECT_TRUE(test, cd321x_dp_irq_hpd(false, hpd, hpd, DP_STATUS_HPD_STATE, + DP_STATUS_HPD_STATE | DP_STATUS_IRQ_HPD)); + /* Still set from an earlier attention, or clearing: nothing new. */ + KUNIT_EXPECT_FALSE(test, cd321x_dp_irq_hpd(false, irq, irq, DP_STATUS_IRQ_HPD, + DP_STATUS_IRQ_HPD)); + KUNIT_EXPECT_FALSE(test, cd321x_dp_irq_hpd(false, irq, hpd, DP_STATUS_IRQ_HPD, 0)); + /* IRQ_HPD is only valid with HPD high. */ + KUNIT_EXPECT_FALSE(test, cd321x_dp_irq_hpd(false, DP_D, DP_D | CD321X_DATA_STATUS_HPD_IRQ, + 0, DP_STATUS_IRQ_HPD)); +} + +/* A pending attention survives later interrupts until HPD drops. */ +static void cd321x_dp_irq_hpd_pending_test(struct kunit *test) +{ + u32 hpd = DP_D | HPD, irq = hpd | CD321X_DATA_STATUS_HPD_IRQ; + bool pending; + + /* Set and cleared again within one debounce window. */ + pending = cd321x_dp_irq_hpd(false, hpd, irq, 0, 0); + pending = cd321x_dp_irq_hpd(pending, irq, hpd, 0, 0); + KUNIT_EXPECT_TRUE(test, pending); + /* An unrelated interrupt rereads the same status. */ + pending = cd321x_dp_irq_hpd(pending, hpd, hpd, 0, 0); + KUNIT_EXPECT_TRUE(test, pending); + /* HPD low: the sink is reconnected, nothing is left to retrain. */ + pending = cd321x_dp_irq_hpd(pending, hpd, DP_D, 0, 0); + KUNIT_EXPECT_FALSE(test, pending); + pending = cd321x_dp_irq_hpd(pending, DP_D, hpd, 0, 0); + KUNIT_EXPECT_FALSE(test, pending); +} + /* HPD rising after a reported low is a connect, as before. */ static void cd321x_dp_hpd_rise_connects_test(struct kunit *test) { @@ -227,6 +266,7 @@ static void cd321x_dp_hpd_rise_connects_test(struct kunit *test) /* Attention is no reason to skip announcing a sink that just came up. */ u.data_status |= CD321X_DATA_STATUS_HPD_IRQ; u.changed |= CD321X_DATA_STATUS_HPD_IRQ; + u.irq_hpd = true; KUNIT_EXPECT_EQ(test, cd321x_dp_event(&u, true), CD321X_DP_EVENT_CONNECTED); } @@ -255,6 +295,7 @@ static void cd321x_dp_irq_hpd_with_link_reset_test(struct kunit *test) .data_status = DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ, .changed = CD321X_DATA_STATUS_HPD_IRQ, .link_reset = true, + .irq_hpd = true, .route_active = true, .sink_reported = true, }; @@ -269,6 +310,7 @@ static void cd321x_dp_retrain_needs_live_sink_test(struct kunit *test) struct cd321x_dp_update u = { .data_status = DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ, .changed = CD321X_DATA_STATUS_HPD_IRQ, + .irq_hpd = true, .route_active = true, }; @@ -313,6 +355,8 @@ static struct kunit_case cd321x_dp_cases[] = { KUNIT_CASE(cd321x_dp_display_taken_test), KUNIT_CASE(cd321x_dp_hpd_bounce_reconnects_test), KUNIT_CASE(cd321x_dp_irq_hpd_retrains_test), + KUNIT_CASE(cd321x_dp_irq_hpd_sources_test), + KUNIT_CASE(cd321x_dp_irq_hpd_pending_test), KUNIT_CASE(cd321x_dp_hpd_rise_connects_test), KUNIT_CASE(cd321x_dp_bounce_with_lane_change_test), KUNIT_CASE(cd321x_dp_irq_hpd_with_link_reset_test), diff --git a/drivers/usb/typec/tipd/cd321x-dp.h b/drivers/usb/typec/tipd/cd321x-dp.h index 36d94864fc548..53346b34f52fe 100644 --- a/drivers/usb/typec/tipd/cd321x-dp.h +++ b/drivers/usb/typec/tipd/cd321x-dp.h @@ -3,6 +3,7 @@ #define __CD321X_DP_H__ #include +#include #include "tps6598x.h" @@ -27,6 +28,7 @@ enum cd321x_dp_event { * @changed: DATA_STATUS bits that changed since the previous update * @link_reset: the Type-C session is torn down: unplug, flip, partner * change, Thunderbolt/USB4 entry or exit, or a forced reconnect + * @irq_hpd: the sink signalled IRQ_HPD and that has not been delivered * @route_active: the previous update left a display route up * @sink_reported: the connector was last told that a sink is present */ @@ -34,6 +36,7 @@ struct cd321x_dp_update { u32 data_status; u32 changed; bool link_reset; + bool irq_hpd; bool route_active; bool sink_reported; }; @@ -53,6 +56,34 @@ static inline bool cd321x_dp_mode_changed(u32 data_status, u32 changed) return changed & bits; } +/** + * cd321x_dp_irq_hpd() - Track the sink's IRQ_HPD across interrupts + * @pending: an IRQ_HPD is waiting to be delivered + * @old: DATA_STATUS read at the previous interrupt + * @new: DATA_STATUS read at this interrupt + * @old_rx: DisplayPort status last received from the partner, as of @old + * @new_rx: the same, as of @new + * + * TI documents the HPD IRQ bit of DATA_STATUS as sticky, and the partner's + * DisplayPort status keeps IRQ_HPD until its next status message, so either + * bit becoming set is a new attention. It is looked at on every interrupt, + * not across the debounced update, so an attention that comes and goes + * within the debounce is not lost. It stays pending until it has been + * delivered, including through updates that have to be retried. HPD going + * low drops it: the sink is then reported gone and connected again anyway. + * + * Return: whether an IRQ_HPD is pending after this interrupt. + */ +static inline bool cd321x_dp_irq_hpd(bool pending, u32 old, u32 new, + u32 old_rx, u32 new_rx) +{ + if (!(new & CD321X_DATA_STATUS_HPD_LEVEL)) + return false; + + return pending || (~old & new & CD321X_DATA_STATUS_HPD_IRQ) || + (~old_rx & new_rx & DP_STATUS_IRQ_HPD); +} + /* * Does the display connector lose its sink before this update is applied? * @@ -81,8 +112,8 @@ cd321x_dp_event(const struct cd321x_dp_update *u, bool route_up) return CD321X_DP_EVENT_NONE; /* IRQ_HPD on a sink that stays connected: have its link checked. */ - if (u->route_active && u->sink_reported && !cd321x_dp_teardown(u) && - (u->data_status & u->changed & CD321X_DATA_STATUS_HPD_IRQ)) + if (u->irq_hpd && u->route_active && u->sink_reported && + !cd321x_dp_teardown(u)) return CD321X_DP_EVENT_RETRAIN; return CD321X_DP_EVENT_CONNECTED; diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 6e483bd176cc7..fa2c831c59ff9 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -859,6 +859,7 @@ static void cd321x_update_work(struct work_struct *work) struct cd321x_dp_update dp = { .data_status = st.data_status, .changed = st.data_status_changed, + .irq_hpd = cd321x->dp_irq_hpd, .route_active = cd321x->display_route_active, .sink_reported = cd321x->dp_sink_reported, }; @@ -965,6 +966,7 @@ static void cd321x_update_work(struct work_struct *work) /* If the plug is disconnected, we are done */ if (!new_connected) { + cd321x->dp_irq_hpd = false; cd321x_update_done(cd321x); return; } @@ -1012,6 +1014,8 @@ static void cd321x_update_work(struct work_struct *work) if (ret) goto retry; + /* An IRQ_HPD is delivered now, or overtaken by a connect. */ + cd321x->dp_irq_hpd = false; switch (cd321x_dp_event(&dp, cd321x->display_route_active)) { case CD321X_DP_EVENT_NONE: break; @@ -1044,6 +1048,12 @@ static void cd321x_update_work(struct work_struct *work) static void cd321x_queue_status(struct cd321x *cd321x) { + cd321x->dp_irq_hpd = + cd321x_dp_irq_hpd(cd321x->dp_irq_hpd, + cd321x->update_status.data_status, + cd321x->tps.data_status, + le32_to_cpu(cd321x->update_status.dp_sid_status.status_rx), + le32_to_cpu(cd321x->dp_sid_status.status_rx)); cd321x->update_status.status_changed |= cd321x->update_status.status ^ cd321x->tps.status; cd321x->update_status.data_status_changed |= cd321x->update_status.data_status ^ cd321x->tps.data_status; diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 29fcc9247e1f9..35d186d9c5656 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -407,6 +407,7 @@ struct cd321x { atomic_t link_event; bool display_route_active; bool dp_sink_reported; + bool dp_irq_hpd; struct cd321x_status update_status; struct delayed_work update_work; From a495a26c2f9e2f1c243751bb019d9029fd263815 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:31:54 +1100 Subject: [PATCH 1241/1579] usb: typec: tipd: Trace the port and reset cause of CD321x DP events The cd321x_dp_hotplug tracepoint records the HPD level and the DATA_STATUS bits that changed, but not why the Type-C session was reset. A plug removed and inserted again within the debounce, a flip, a different partner or a forced reconnect after a failed update can end with the same DATA_STATUS as before. Those updates correctly report the display as gone, but their trace shows HPD high and nothing relevant changed, which looks like a spurious disconnect. The tracepoint does not identify the controller either, so events of the two ports of a machine cannot be told apart. Record the reasons for a session reset as flags in the update instead of a single bool, and trace them together with the device name and whether an IRQ_HPD is pending. The reset decision itself is unchanged: the flags are set from the same conditions that set was_disconnected, plus the partner leaving. Fixes: 33af24cea854 ("usb: typec: tipd: Trace the CD321x DisplayPort hotplug events") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-dp-test.c | 6 ++--- drivers/usb/typec/tipd/cd321x-dp.h | 15 +++++++++--- drivers/usb/typec/tipd/core.c | 31 ++++++++++++++++++------- drivers/usb/typec/tipd/trace.h | 30 +++++++++++++++++++----- 4 files changed, 62 insertions(+), 20 deletions(-) diff --git a/drivers/usb/typec/tipd/cd321x-dp-test.c b/drivers/usb/typec/tipd/cd321x-dp-test.c index 39eb2a9110309..fd9b07d3b9601 100644 --- a/drivers/usb/typec/tipd/cd321x-dp-test.c +++ b/drivers/usb/typec/tipd/cd321x-dp-test.c @@ -44,7 +44,7 @@ static void cd321x_dp_unplug_tears_down_test(struct kunit *test) { struct cd321x_dp_update u = { .changed = DP_D | HPD, - .link_reset = true, + .link_reset = CD321X_DP_RESET_UNPLUG | CD321X_DP_RESET_PLUG, .route_active = true, }; @@ -57,7 +57,7 @@ static void cd321x_dp_link_reset_reconnects_test(struct kunit *test) { struct cd321x_dp_update u = { .data_status = DP_D | HPD, - .link_reset = true, + .link_reset = CD321X_DP_RESET_FLIP, .route_active = true, }; @@ -294,7 +294,7 @@ static void cd321x_dp_irq_hpd_with_link_reset_test(struct kunit *test) struct cd321x_dp_update u = { .data_status = DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ, .changed = CD321X_DATA_STATUS_HPD_IRQ, - .link_reset = true, + .link_reset = CD321X_DP_RESET_PARTNER, .irq_hpd = true, .route_active = true, .sink_reported = true, diff --git a/drivers/usb/typec/tipd/cd321x-dp.h b/drivers/usb/typec/tipd/cd321x-dp.h index 53346b34f52fe..85b024ddc8190 100644 --- a/drivers/usb/typec/tipd/cd321x-dp.h +++ b/drivers/usb/typec/tipd/cd321x-dp.h @@ -2,6 +2,7 @@ #ifndef __CD321X_DP_H__ #define __CD321X_DP_H__ +#include #include #include @@ -11,6 +12,14 @@ #define CD321X_DP_MODE_BITS \ (TPS_DATA_STATUS_DP_CONNECTION | TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK) +/* Why a cable update resets the Type-C session: cd321x_dp_update.link_reset */ +#define CD321X_DP_RESET_UNPLUG BIT(0) /* the partner is gone */ +#define CD321X_DP_RESET_PLUG BIT(1) /* the plug was removed or inserted */ +#define CD321X_DP_RESET_FLIP BIT(2) /* the plug orientation changed */ +#define CD321X_DP_RESET_TUNNEL BIT(3) /* Thunderbolt or USB4 came or went */ +#define CD321X_DP_RESET_PARTNER BIT(4) /* a different partner is attached */ +#define CD321X_DP_RESET_FORCED BIT(5) /* recovery after a failed update */ + /* * What the display connector is told once a cable update has been applied: * nothing, that the sink is present, or that the sink is present and asked @@ -26,8 +35,8 @@ enum cd321x_dp_event { * struct cd321x_dp_update - DisplayPort view of one cable update * @data_status: DATA_STATUS of this update * @changed: DATA_STATUS bits that changed since the previous update - * @link_reset: the Type-C session is torn down: unplug, flip, partner - * change, Thunderbolt/USB4 entry or exit, or a forced reconnect + * @link_reset: why the Type-C session is torn down (CD321X_DP_RESET_*), + * or 0 if it is kept * @irq_hpd: the sink signalled IRQ_HPD and that has not been delivered * @route_active: the previous update left a display route up * @sink_reported: the connector was last told that a sink is present @@ -35,7 +44,7 @@ enum cd321x_dp_event { struct cd321x_dp_update { u32 data_status; u32 changed; - bool link_reset; + u32 link_reset; bool irq_hpd; bool route_active; bool sink_reported; diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index fa2c831c59ff9..022cfc732f9af 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -819,7 +819,7 @@ static void cd321x_dp_notify(struct cd321x *cd321x, if (!IS_REACHABLE(CONFIG_DRM) || !cd321x->connector_fwnode) return; - trace_cd321x_dp_hotplug(u, status); + trace_cd321x_dp_hotplug(cd321x->tps.dev, u, status); cd321x->dp_sink_reported = status != connector_status_disconnected; drm_connector_oob_hotplug_event(cd321x->connector_fwnode, status); } @@ -846,10 +846,19 @@ static void cd321x_update_work(struct work_struct *work) bool old_connected = !!tps->partner; bool new_connected = st.status & TPS_STATUS_PLUG_PRESENT; - bool was_disconnected = cd321x->pm.force_reconnect || - (st.status_changed & (TPS_STATUS_PLUG_PRESENT | TPS_STATUS_PLUG_UPSIDE_DOWN)) || - (st.data_status_changed & (TPS_DATA_STATUS_TBT_CONNECTION | - CD321X_DATA_STATUS_USB4_CONNECTION)); + u32 reset = 0; + + if (cd321x->pm.force_reconnect) + reset |= CD321X_DP_RESET_FORCED; + if (st.status_changed & TPS_STATUS_PLUG_PRESENT) + reset |= CD321X_DP_RESET_PLUG; + if (st.status_changed & TPS_STATUS_PLUG_UPSIDE_DOWN) + reset |= CD321X_DP_RESET_FLIP; + if (st.data_status_changed & (TPS_DATA_STATUS_TBT_CONNECTION | + CD321X_DATA_STATUS_USB4_CONNECTION)) + reset |= CD321X_DP_RESET_TUNNEL; + + bool was_disconnected = reset; bool usb_connection = st.data_status & (TPS_DATA_STATUS_USB2_CONNECTION | TPS_DATA_STATUS_USB3_CONNECTION); @@ -883,13 +892,19 @@ static void cd321x_update_work(struct work_struct *work) } bool is_pd = pwr_opmode == TYPEC_PWR_MODE_PD; - bool partner_changed = old_connected && new_connected && - (was_disconnected || is_pd != cd321x->cur_partner_is_pd || + bool new_partner = old_connected && new_connected && + (is_pd != cd321x->cur_partner_is_pd || (is_pd && memcmp(&st.partner_identity, &cd321x->cur_partner_identity, sizeof(struct usb_pd_identity)))); + bool partner_changed = new_partner || + (old_connected && new_connected && was_disconnected); /* A different dock must release the old routing before reconfiguration. */ was_disconnected |= partner_changed; + if (new_partner) + reset |= CD321X_DP_RESET_PARTNER; + if (!new_connected) + reset |= CD321X_DP_RESET_UNPLUG; /* * Report the sink as gone before any hardware transition that can fail: @@ -899,7 +914,7 @@ static void cd321x_update_work(struct work_struct *work) * sink can enter Alt Mode while its mux request fails with -ENOSPC, and * must not tear down the display carried by the other port. */ - dp.link_reset = !new_connected || was_disconnected; + dp.link_reset = reset; if (cd321x_dp_teardown(&dp)) cd321x_dp_notify(cd321x, &dp, connector_status_disconnected); diff --git a/drivers/usb/typec/tipd/trace.h b/drivers/usb/typec/tipd/trace.h index cabbc0f9670ea..5bd8f91ab275d 100644 --- a/drivers/usb/typec/tipd/trace.h +++ b/drivers/usb/typec/tipd/trace.h @@ -16,6 +16,7 @@ #include "tps6598x.h" #include +#include #include #include #include @@ -439,29 +440,46 @@ TRACE_DEFINE_ENUM(connector_status_unknown); { connector_status_disconnected, "disconnected" }, \ { connector_status_unknown, "unknown" }) +#define show_dp_reset_flags(flags) \ + __print_flags(flags, "|", \ + { CD321X_DP_RESET_UNPLUG, "UNPLUG" }, \ + { CD321X_DP_RESET_PLUG, "PLUG" }, \ + { CD321X_DP_RESET_FLIP, "FLIP" }, \ + { CD321X_DP_RESET_TUNNEL, "TUNNEL" }, \ + { CD321X_DP_RESET_PARTNER, "PARTNER" }, \ + { CD321X_DP_RESET_FORCED, "FORCED" }) + TRACE_EVENT(cd321x_dp_hotplug, - TP_PROTO(const struct cd321x_dp_update *u, + TP_PROTO(const struct device *dev, const struct cd321x_dp_update *u, enum drm_connector_status status), - TP_ARGS(u, status), + TP_ARGS(dev, u, status), - TP_STRUCT__entry(__field(u32, data_status) + TP_STRUCT__entry(__string(dev, dev_name(dev)) + __field(u32, data_status) __field(u32, changed) + __field(u32, link_reset) + __field(bool, irq_hpd) __field(bool, route_active) __field(bool, sink_reported) __field(int, status)), - TP_fast_assign(__entry->data_status = u->data_status; + TP_fast_assign(__assign_str(dev); + __entry->data_status = u->data_status; __entry->changed = u->changed; + __entry->link_reset = u->link_reset; + __entry->irq_hpd = u->irq_hpd; __entry->route_active = u->route_active; __entry->sink_reported = u->sink_reported; __entry->status = status;), - TP_printk("%s: route %d, sink %d, HPD %d%s%s, changed %s%s", - show_dp_hotplug_status(__entry->status), + TP_printk("%s: %s: route %d, sink %d, HPD %d%s%s, IRQ_HPD %d, reset %s, changed %s%s", + __get_str(dev), show_dp_hotplug_status(__entry->status), __entry->route_active, __entry->sink_reported, !!(__entry->data_status & CD321X_DATA_STATUS_HPD_LEVEL), __entry->data_status & TPS_DATA_STATUS_DP_CONNECTION ? ", DP pinout " : "", maybe_show_data_status_dp_pin_assignment(__entry->data_status), + __entry->irq_hpd, + __entry->link_reset ? show_dp_reset_flags(__entry->link_reset) : "none", show_cd321x_data_status_flags(__entry->changed), __entry->changed & TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK ? "|DP_PIN_ASSIGNMENT" : "") From 3a8397344bbc4fd07d8f37f06f6687acc235e406 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 03:31:55 +1100 Subject: [PATCH 1242/1579] usb: typec: tipd: Test CD321x DisplayPort hotplug sequences The cd321x-dp tests check one debounced update at a time. The display bugs fixed recently needed several interrupts and updates to show: samples that are folded into one update, an IRQ_HPD that has to outlive a deferred update, and a lane change that only one sample reveals. Add a second suite that models a port the way cd321x_queue_status() and cd321x_update_work() drive it. Interrupts add samples through cd321x_dp_irq_hpd() and the change masks, and debounced updates make the display decisions around a mux update that can fail and be retried through the cd321x-pm helpers. Each case checks the exact order of connector events it sends. The cases cover a Type-C replug, a monitor replaced behind a dock, IRQ_HPD from either source, an IRQ_HPD across a deferred update, HPD low, a C/D change with and without an HPD bounce, and a dock removed while an IRQ_HPD is pending. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/cd321x-dp-test.c | 274 +++++++++++++++++++++++- 1 file changed, 273 insertions(+), 1 deletion(-) diff --git a/drivers/usb/typec/tipd/cd321x-dp-test.c b/drivers/usb/typec/tipd/cd321x-dp-test.c index fd9b07d3b9601..1aed96431f018 100644 --- a/drivers/usb/typec/tipd/cd321x-dp-test.c +++ b/drivers/usb/typec/tipd/cd321x-dp-test.c @@ -3,8 +3,10 @@ #include #include #include +#include #include "cd321x-dp.h" +#include "cd321x-pm.h" /* A dock in pin assignment D: DisplayPort plus USB 3. */ #define DP_D (TPS_DATA_STATUS_DATA_CONNECTION | TPS_DATA_STATUS_USB2_CONNECTION | \ @@ -370,6 +372,276 @@ static struct kunit_suite cd321x_dp_suite = { .test_cases = cd321x_dp_cases, }; -kunit_test_suite(cd321x_dp_suite); +/* + * One port as cd321x_queue_status() and cd321x_update_work() drive it: + * every interrupt adds a sample, and a debounced update reports the display + * around a mux update that can fail. @log records the connector events in + * order: D(isconnected), C(onnected) and U(nknown, i.e. retrain). + */ +struct cd321x_dp_port { + struct cd321x_pm_state pm; + u32 status, data_status, rx; + u32 status_changed, data_changed; + bool irq_hpd, route_active, sink_reported; + char log[16]; +}; + +static void port_notify(struct cd321x_dp_port *p, char event) +{ + size_t n = strlen(p->log); + + if (n + 1 < sizeof(p->log)) + p->log[n] = event; + p->sink_reported = event != 'D'; +} + +/* An interrupt that reads @status, @data_status and the DP status @rx. */ +static void port_irq(struct cd321x_dp_port *p, u32 status, u32 data_status, u32 rx) +{ + p->irq_hpd = cd321x_dp_irq_hpd(p->irq_hpd, p->data_status, data_status, p->rx, rx); + p->status_changed |= p->status ^ status; + p->data_changed |= p->data_status ^ data_status; + p->status = status; + p->data_status = data_status; + p->rx = rx; +} + +/* A debounced update whose mux update returns @ret. */ +static void port_update(struct cd321x_dp_port *p, int ret) +{ + bool connected = p->status & TPS_STATUS_PLUG_PRESENT; + struct cd321x_dp_update u = { + .data_status = p->data_status, + .changed = p->data_changed, + .irq_hpd = p->irq_hpd, + .route_active = p->route_active, + .sink_reported = p->sink_reported, + }; + + if (p->pm.force_reconnect) + u.link_reset |= CD321X_DP_RESET_FORCED; + if (p->status_changed & TPS_STATUS_PLUG_PRESENT) + u.link_reset |= CD321X_DP_RESET_PLUG; + if (!connected) + u.link_reset |= CD321X_DP_RESET_UNPLUG; + p->status_changed = 0; + p->data_changed = 0; + + if (cd321x_dp_teardown(&u)) + port_notify(p, 'D'); + if (u.link_reset) + p->route_active = false; + if (!connected) { + p->irq_hpd = false; + cd321x_pm_complete(&p->pm); + return; + } + if (ret) { + cd321x_pm_retry(&p->pm, ret != -EAGAIN); + return; + } + + p->route_active = u.data_status & TPS_DATA_STATUS_DP_CONNECTION; + p->irq_hpd = false; + switch (cd321x_dp_event(&u, p->route_active)) { + case CD321X_DP_EVENT_NONE: + break; + case CD321X_DP_EVENT_CONNECTED: + port_notify(p, 'C'); + break; + case CD321X_DP_EVENT_RETRAIN: + port_notify(p, 'C'); + port_notify(p, 'U'); + break; + } + cd321x_pm_complete(&p->pm); +} + +/* The fresh read that a retried update starts from. */ +static void port_reread(struct kunit *test, struct cd321x_dp_port *p) +{ + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&p->pm)); + cd321x_pm_snapshot_ready(&p->pm); + port_irq(p, p->status, p->data_status, p->rx); +} + +#define PLUG TPS_STATUS_PLUG_PRESENT +#define HPD_RX DP_STATUS_HPD_STATE + +/* A dock in pin assignment D with its sink announced and nothing pending. */ +static void port_docked(struct kunit *test, struct cd321x_dp_port *p) +{ + *p = (struct cd321x_dp_port){ .pm.phase = CD321X_PM_RUNNING }; + port_irq(p, PLUG, DP_D | HPD, HPD_RX); + port_update(p, 0); + KUNIT_ASSERT_STREQ(test, p->log, "C"); + memset(p->log, 0, sizeof(p->log)); +} + +/* + * The cases below are the nine sequence invariants of the tipd HPD pulse + * review, in its order, run through the helpers the driver uses. + */ + +/* 1, 2: a Type-C unplug and replug within the debounce. */ +static void cd321x_dp_seq_typec_replug_test(struct kunit *test) +{ + struct cd321x_dp_port p; + + port_docked(test, &p); + port_irq(&p, 0, 0, HPD_RX); + port_irq(&p, PLUG, DP_D | HPD, HPD_RX); + port_update(&p, 0); + KUNIT_EXPECT_STREQ(test, p.log, "DC"); +} + +/* 3: the monitor behind the dock is replaced; the dock stays. */ +static void cd321x_dp_seq_downstream_replug_test(struct kunit *test) +{ + struct cd321x_dp_port p; + + port_docked(test, &p); + port_irq(&p, PLUG, DP_D, 0); + port_irq(&p, PLUG, DP_D | HPD, HPD_RX); + port_update(&p, 0); + KUNIT_EXPECT_STREQ(test, p.log, "DC"); +} + +/* 4: the sink's IRQ_HPD retrains the link and keeps the sink. */ +static void cd321x_dp_seq_irq_hpd_retrains_test(struct kunit *test) +{ + struct cd321x_dp_port p; + + port_docked(test, &p); + port_irq(&p, PLUG, DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ, HPD_RX); + port_update(&p, 0); + KUNIT_EXPECT_STREQ(test, p.log, "CU"); + + /* Reported by the partner's DP status alone, set and cleared again. */ + memset(p.log, 0, sizeof(p.log)); + port_irq(&p, PLUG, DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ, HPD_RX | DP_STATUS_IRQ_HPD); + port_irq(&p, PLUG, DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ, HPD_RX); + port_update(&p, 0); + KUNIT_EXPECT_STREQ(test, p.log, "CU"); +} + +/* 5: -EAGAIN retries from a fresh read and keeps the session. */ +static void cd321x_dp_seq_eagain_keeps_session_test(struct kunit *test) +{ + struct cd321x_pm_state pm = { .phase = CD321X_PM_RUNNING }; + + KUNIT_EXPECT_TRUE(test, cd321x_pm_retry(&pm, false)); + KUNIT_EXPECT_FALSE(test, pm.force_reconnect); + KUNIT_EXPECT_TRUE(test, cd321x_pm_begin_read(&pm)); +} + +/* 6: an IRQ_HPD whose update is deferred is still delivered, once. */ +static void cd321x_dp_seq_irq_hpd_after_eagain_test(struct kunit *test) +{ + struct cd321x_dp_port p; + + port_docked(test, &p); + port_irq(&p, PLUG, DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ, HPD_RX); + port_update(&p, -EAGAIN); + KUNIT_EXPECT_STREQ(test, p.log, ""); + port_reread(test, &p); + port_update(&p, 0); + KUNIT_EXPECT_STREQ(test, p.log, "CU"); + + /* Delivered: the sticky bit seen again is not another request. */ + port_irq(&p, PLUG, DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ | + TPS_DATA_STATUS_USB_DATA_ROLE, HPD_RX); + port_update(&p, 0); + KUNIT_EXPECT_STREQ(test, p.log, "CUC"); +} + +/* 7: HPD low at the end of the debounce. */ +static void cd321x_dp_seq_hpd_low_test(struct kunit *test) +{ + struct cd321x_dp_port p; + + port_docked(test, &p); + port_irq(&p, PLUG, DP_D, 0); + port_update(&p, 0); + KUNIT_EXPECT_STREQ(test, p.log, "D"); +} + +/* 8: D to C with HPD held high: the lane count changes. */ +static void cd321x_dp_seq_usb3_pin_change_test(struct kunit *test) +{ + struct cd321x_dp_port p; + + port_docked(test, &p); + port_irq(&p, PLUG, DP_C | HPD, HPD_RX); + port_update(&p, 0); + KUNIT_EXPECT_STREQ(test, p.log, "DC"); +} + +/* 9: D to C with an HPD bounce. */ +static void cd321x_dp_seq_pin_change_with_bounce_test(struct kunit *test) +{ + struct cd321x_dp_port p; + + port_docked(test, &p); + port_irq(&p, PLUG, DP_D, 0); + port_irq(&p, PLUG, DP_C | HPD, HPD_RX); + port_update(&p, 0); + KUNIT_EXPECT_STREQ(test, p.log, "DC"); +} + +/* A bounce whose reconnect is deferred announces the sink on the retry. */ +static void cd321x_dp_seq_bounce_after_eagain_test(struct kunit *test) +{ + struct cd321x_dp_port p; + + port_docked(test, &p); + port_irq(&p, PLUG, DP_D, 0); + port_irq(&p, PLUG, DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ, HPD_RX); + port_update(&p, -EAGAIN); + KUNIT_EXPECT_STREQ(test, p.log, "D"); + port_reread(test, &p); + port_update(&p, 0); + KUNIT_EXPECT_STREQ(test, p.log, "DC"); +} + +/* An IRQ_HPD still pending when the dock leaves is dropped with it. */ +static void cd321x_dp_seq_irq_hpd_then_unplug_test(struct kunit *test) +{ + struct cd321x_dp_port p; + + port_docked(test, &p); + port_irq(&p, PLUG, DP_D | HPD | CD321X_DATA_STATUS_HPD_IRQ, HPD_RX); + port_update(&p, -EAGAIN); + port_irq(&p, 0, 0, HPD_RX); + port_reread(test, &p); + port_update(&p, 0); + KUNIT_EXPECT_STREQ(test, p.log, "D"); + KUNIT_EXPECT_FALSE(test, p.irq_hpd); + + port_irq(&p, PLUG, DP_D | HPD, HPD_RX); + port_update(&p, 0); + KUNIT_EXPECT_STREQ(test, p.log, "DC"); +} + +static struct kunit_case cd321x_dp_seq_cases[] = { + KUNIT_CASE(cd321x_dp_seq_typec_replug_test), + KUNIT_CASE(cd321x_dp_seq_downstream_replug_test), + KUNIT_CASE(cd321x_dp_seq_irq_hpd_retrains_test), + KUNIT_CASE(cd321x_dp_seq_eagain_keeps_session_test), + KUNIT_CASE(cd321x_dp_seq_irq_hpd_after_eagain_test), + KUNIT_CASE(cd321x_dp_seq_hpd_low_test), + KUNIT_CASE(cd321x_dp_seq_usb3_pin_change_test), + KUNIT_CASE(cd321x_dp_seq_pin_change_with_bounce_test), + KUNIT_CASE(cd321x_dp_seq_bounce_after_eagain_test), + KUNIT_CASE(cd321x_dp_seq_irq_hpd_then_unplug_test), + {} +}; + +static struct kunit_suite cd321x_dp_seq_suite = { + .name = "cd321x-dp-sequence", + .test_cases = cd321x_dp_seq_cases, +}; + +kunit_test_suites(&cd321x_dp_suite, &cd321x_dp_seq_suite); MODULE_DESCRIPTION("KUnit tests for the CD321x DisplayPort hotplug decisions"); MODULE_LICENSE("GPL"); From bd98d3083bfab0b040d59beceeb6439dcf0807b2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 13:31:41 +1100 Subject: [PATCH 1243/1579] selftests: apple-dcp: Compile the CD321x update with its DisplayPort helpers The physical-disconnect fixture, which fast-replug-bounce and dropped-swap-teardown build on, extracts cd321x_update_work(). The hotplug decisions it takes are now the inline helpers in cd321x-dp.h, reported through cd321x_dp_notify(), and the port keeps a pending IRQ_HPD and whether a sink was reported. When cd321x-dp.h is present, inline it (without its kernel includes) and extract cd321x_dp_notify() with the update work. Give the fixture's port the two new flags, turn its connector status values into the enum the helpers name (same values: the OOB mock still tears the display down on disconnected only, and an unknown status is a retrain, not a teardown), and supply DP_STATUS_IRQ_HPD and a no-op trace point. The fixtures keep their assertions; all 55 host tests pass. [ryan: rebased onto aurora-wip 0daed2db + ATC PIPE park v3, whose commit "usb: typec: tipd: Keep applying a Type-C state that a busy mux holds up" changed this fixture too: it extracts v3's held-mux helpers, adds the mux handle and its shims, and gives the stubbed cd321x_typec_update_mode() its held out-parameter. Kept all of that; cd321x-dp.h is inlined and cd321x_dp_notify() extracted as reviewed, the two port flags join v3's struct, and the source takes the header after v3's shim.] Assisted-by: LLM Signed-off-by: Ryan Murray --- .../drivers/gpu/apple-dcp/lifecycle.py | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 5b825b97eaee4..1d9313a06ce12 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -28,7 +28,22 @@ def function(file, name): held = [n for n in ('cd321x_mux_held', 'cd321x_mux_set', 'cd321x_mux_still_held', 'cd321x_retry_held_mux', 'cd321x_update_done') if re.search(r'^static \w+ ' + n + r'\(', (r / core).read_text(), re.M)] - funcs = '\n'.join(function(f, n) for f, n in ( + # The DisplayPort hotplug decisions live in cd321x-dp.h when present. + dp_header = r / 'drivers/usb/typec/tipd/cd321x-dp.h' + dp = '' + if dp_header.exists(): + dp = '\n'.join(l for l in dp_header.read_text().splitlines() + if not l.startswith('#include')) + dp = r''' +#ifndef BIT +#define BIT(n) (1U << (n)) +#endif +#define CD321X_DATA_STATUS_HPD_IRQ (1U << 9) +#define DP_STATUS_IRQ_HPD BIT(8) +#define trace_cd321x_dp_hotplug(...) ((void)0) +''' + dp + '\n' + notify = (('drivers/usb/typec/tipd/core.c', 'cd321x_dp_notify'),) if dp else () + funcs = '\n'.join(function(f, n) for f, n in notify + ( ('drivers/gpu/drm/apple/iomfb.c', 'iomfb_pause_presentation'), (dcp, 'dcp_external_report_disconnected'), (dcp, 'dcp_external_publish_generation'), @@ -155,8 +170,8 @@ def function(file, name): static struct apple_dcp display; static void drm_connector_oob_hotplug_event(void *node, int status) { assert(node); if (!status) dcp_external_disconnect(&display, true); } -#define connector_status_disconnected 0 -#define connector_status_connected 1 +enum drm_connector_status { connector_status_disconnected, connector_status_connected, + connector_status_unknown }; ''' shim = r''' enum usb_role { USB_ROLE_NONE, USB_ROLE_HOST, USB_ROLE_DEVICE }; @@ -172,6 +187,7 @@ def function(file, name): struct cd321x { struct tps6598x tps; struct cd321x_status update_status; struct cd321x_pm_state pm; struct delayed_work update_work, resume_work; struct typec_mux_state state; bool state_valid, display_route_active, cur_partner_is_pd; + bool dp_irq_hpd, dp_sink_reported; struct usb_pd_identity cur_partner_identity; void *connector_fwnode, *tbt_switch, *mux; }; /* No mux here declares its timeouts transient: every failure is budgeted. */ static bool typec_mux_timeout_transient(void *m) { (void)m; return false; } @@ -390,7 +406,7 @@ def function(file, name): shim = shim.replace(old, 'static int cd321x_typec_update_mode(struct tps6598x *t, struct cd321x_status *s, bool *busy,\n' ' bool *held)\n' '{ (void)t; (void)s; *busy = mode_error == -EBUSY; *held = false; return mode_error; }') - source = prefix + pm + shim + funcs + tests + source = prefix + pm + shim + dp + funcs + tests return source def bounce(root): From 802b8045fd44242e6b979a4c52aa87efaa99ea7c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:35:05 +1100 Subject: [PATCH 1244/1579] rust: dma_buf: gate the GEM stub on built-in DRM The DMA-BUF stub was introduced for GEM export callbacks. Its crate-private conversion helpers are used only by the DRM abstraction, which is built only with CONFIG_DRM=y. Building Rust without that abstraction leaves the helpers unused and fails with CONFIG_WERROR=y. Give the stub the same configuration guard as its consumer. Gating on CONFIG_DMA_SHARED_BUFFER alone would still leave the helpers unused when another subsystem selects DMA-BUF support without built-in DRM. Fixes: 177c185048df ("rust: Add dma_buf stub bindings") Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index a40585a222baa..20f05cb1afa29 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -80,6 +80,7 @@ pub mod device; pub mod device_id; pub mod devres; pub mod dma; +#[cfg(CONFIG_DRM = "y")] pub mod dma_buf; #[cfg(CONFIG_DMA_SHARED_BUFFER)] pub mod dma_fence; From 9ecb37ad37681712a032575a838ec3b9c87cff1f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:35:05 +1100 Subject: [PATCH 1245/1579] rust: xarray: allocate KUnit values before taking xa_lock The xarray tests create KBox values with GFP_KERNEL after acquiring the XArray spinlock. DEBUG_ATOMIC_SLEEP reports a sleeping allocation in rust_xarray_kunit, although the test assertions themselves pass. Allocate every test value before acquiring the guard and pass the values into the insertion checks. Keep GFP_KERNEL for the XArray operations: the C __xa_* helpers release and reacquire xa_lock when they need a blocking allocation. KBox::new cannot release that lock. Fixes: 977de23ccc68 ("rust: xarray: add `insert` and `reserve`") Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/xarray.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/rust/kernel/xarray.rs b/rust/kernel/xarray.rs index f2489e4e55dd4..098c8244b4455 100644 --- a/rust/kernel/xarray.rs +++ b/rust/kernel/xarray.rs @@ -655,13 +655,14 @@ mod tests { fn test_alloc_kind(kind: AllocKind, expected_index: usize) -> Result { stack_pin_init!(let xa = XArray::new(kind)); + let value = new_kbox(0x1337)?; let mut guard = xa.lock(); let reservation = guard.reserve_limit(.., GFP_KERNEL)?; assert_eq!(reservation.index(), expected_index); reservation.release_locked(&mut guard)?; - let insertion = guard.insert_limit(.., new_kbox(0x1337)?, GFP_KERNEL); + let insertion = guard.insert_limit(.., value, GFP_KERNEL); assert!(insertion.is_ok()); let insertion_index = insertion.unwrap(); assert_eq!(insertion_index, expected_index); @@ -680,10 +681,9 @@ mod tests { } #[track_caller] - fn check_not_vacant<'a>(guard: &mut Guard<'a, KBox>) -> Result { + fn check_not_vacant<'a>(guard: &mut Guard<'a, KBox>, beef: KBox) -> Result { // Insertion fails. { - let beef = new_kbox(0xbeef)?; let ret = insert(guard, beef); assert!(ret.is_err()); let StoreError { error, value } = ret.unwrap_err(); @@ -704,6 +704,12 @@ mod tests { #[test] fn test_insert_and_reserve_interaction() -> Result { stack_pin_init!(let xa = XArray::new(Default::default())); + // Allocate the test values before taking the XArray spinlock. + let busy_reserved = new_kbox(0xbeef)?; + let dead = new_kbox(0xdead)?; + let busy_inserted = new_kbox(0xbeef)?; + let beef = new_kbox(0xbeef)?; + let busy_filled = new_kbox(0xbeef)?; let mut guard = xa.lock(); // Vacant. @@ -719,7 +725,7 @@ mod tests { // Reserved presents as vacant. assert_eq!(guard.get(IDX), None); - check_not_vacant(&mut guard)?; + check_not_vacant(&mut guard, busy_reserved)?; // Release reservation. { @@ -733,20 +739,18 @@ mod tests { // Insert succeeds. { - let dead = new_kbox(0xdead)?; let ret = insert(&mut guard, dead); assert!(ret.is_ok()); let () = ret.unwrap(); } - check_not_vacant(&mut guard)?; + check_not_vacant(&mut guard, busy_inserted)?; // Remove. assert_eq!(guard.remove(IDX).as_deref(), Some(&0xdead)); // Reserve and fill. { - let beef = new_kbox(0xbeef)?; let ret = reserve(&mut guard); assert!(ret.is_ok()); let reservation = ret.unwrap(); @@ -755,7 +759,7 @@ mod tests { let () = ret.unwrap(); }; - check_not_vacant(&mut guard)?; + check_not_vacant(&mut guard, busy_filled)?; // Remove. assert_eq!(guard.remove(IDX).as_deref(), Some(&0xbeef)); From 9e04a4f5f16cf71ee7a554ec830732368161a3d4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:35:33 +1100 Subject: [PATCH 1246/1579] rust: sync: barrier: Add dma_wmb() and dma_rmb() A driver that fills a coherent DMA buffer and then rings the device with a relaxed MMIO write has to order the two itself, as C drivers do with dma_wmb(). On receive it is the other way round, with dma_rmb(). The barrier module has only the smp_*() barriers. Those order accesses between CPUs, not against a device, and compile away on !SMP. Add both, through helpers as for the others. They do not depend on SMP. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/helpers/barrier.c | 10 ++++++++++ rust/kernel/sync/barrier.rs | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/rust/helpers/barrier.c b/rust/helpers/barrier.c index fed8853745c83..7d681b1c4ab60 100644 --- a/rust/helpers/barrier.c +++ b/rust/helpers/barrier.c @@ -16,3 +16,13 @@ __rust_helper void rust_helper_smp_rmb(void) { smp_rmb(); } + +__rust_helper void rust_helper_dma_wmb(void) +{ + dma_wmb(); +} + +__rust_helper void rust_helper_dma_rmb(void) +{ + dma_rmb(); +} diff --git a/rust/kernel/sync/barrier.rs b/rust/kernel/sync/barrier.rs index 8f2d435fcd944..b1c9b8526a5fc 100644 --- a/rust/kernel/sync/barrier.rs +++ b/rust/kernel/sync/barrier.rs @@ -59,3 +59,23 @@ pub fn smp_rmb() { barrier(); } } + +/// A write-write memory barrier for memory shared with a device. +/// +/// Orders the CPU's writes to memory a device reads, such as a coherent DMA buffer, before its +/// later writes, including a relaxed MMIO write that tells the device to read that memory. +#[inline(always)] +pub fn dma_wmb() { + // SAFETY: `dma_wmb()` is safe to call. + unsafe { bindings::dma_wmb() }; +} + +/// A read-read memory barrier for memory shared with a device. +/// +/// Orders the CPU's reads of memory a device writes, such as a coherent DMA buffer, after its +/// earlier reads, including a relaxed MMIO read that told it the device had written that memory. +#[inline(always)] +pub fn dma_rmb() { + // SAFETY: `dma_rmb()` is safe to call. + unsafe { bindings::dma_rmb() }; +} From 44710408a8cf73b76901de1cb53a865d09c7955d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:36:29 +1100 Subject: [PATCH 1247/1579] soc: apple: sep: Order the shared buffers against the mailbox The host writes a buffer and then sends the message that points the enclave at it: a request payload, the poison that refills an SBIO reply header, the shared memory a registration hands over. The buffers are coherent allocations. The SEP node is not dma-coherent, so on arm64 they are mapped Normal-NC, and the mailbox write is writeq_relaxed() to Device memory. Nothing orders the two, so the enclave can be told to look before the bytes are there. It would then read a stale request, or write its reply's header and have the poison refill land over it. Receiving is the same the other way round: the message comes in through readq_relaxed(), and nothing keeps the reads of the buffer it points to from going ahead of it. RTKit has the same contract with the same mailbox. It puts dma_wmb() before apple_mbox_send() and dma_rmb() after a message is read. The SEP driver calls the mailbox directly and has neither. Add them: dma_wmb() before every send, in the one place sends reach the mailbox, and dma_rmb() in the receive callback, before the message is queued for the work item that reads the buffers. dma_wmb() is the right barrier for this mapping. A coherent buffer has no dma_sync_*_for_device() call, which would order it as part of the cache maintenance a streaming mapping needs. Here the barrier has to be explicit. Normal-NC needs no cache maintenance either, only the ordering of the CPU's stores to the buffer before its store to the mailbox, as the enclave sees them. On arm64 dma_wmb() is dmb(oshst), the barrier writel() itself puts before the MMIO store (__io_bw()). So dma_wmb() followed by the relaxed doorbell is what writeq() would do, with no wmb()'s full dsb. dma_rmb() is dmb(oshld), as in readq(). It also orders the MMIO read before the stores that queue the message, so the work item's later reads are ordered after it. Fixes: bac0846c9087 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sep.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index f4648fb4a2717..0562752f927bb 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -61,6 +61,10 @@ use kernel::{ Atomic, Relaxed, // }, + barrier::{ + dma_rmb, + dma_wmb, // + }, new_condvar, Arc, CondVar, @@ -956,7 +960,7 @@ impl SepData { } self.registered.store(true, Relaxed); - self.mbox.lock().as_ref().ok_or(EINVAL)?.send(msg, false)?; + Self::ring(self.mbox.lock().as_ref().ok_or(EINVAL)?, msg)?; // no ack on 0xFE; success is the discovery burst on 0xFD Ok(()) @@ -1109,7 +1113,20 @@ impl SepData { if self.shutting_down.load(Relaxed) { return Err(ENODEV); } - self.mbox.lock().as_ref().ok_or(ENODEV)?.send(msg, false) + Self::ring(self.mbox.lock().as_ref().ok_or(ENODEV)?, msg) + } + + /// Puts `msg` in the mailbox once the enclave can see what the CPU wrote + /// to the buffers before it: a request payload, an SBIO header's poison, + /// the shared memory a registration hands over. Those are coherent + /// allocations, which have no sync call to order them, and the mailbox + /// write is relaxed MMIO, which is not ordered after them either. + /// Without the barrier the enclave could read old bytes, or have its + /// reply's header overwritten by a refill that lands late. dma_wmb() is + /// what writel() would put there, and what RTKit puts before its sends. + fn ring(mbox: &Mailbox, msg: Message) -> Result<()> { + dma_wmb(); + mbox.send(msg, false) } @@ -2360,6 +2377,9 @@ impl MailCallback for SepData { if data.shutting_down.load(Relaxed) { return; } + // The message was read from the mailbox with relaxed MMIO. Read the + // buffers the enclave wrote before it only after that, as RTKit does. + dma_rmb(); if !data.rx.push(msg) && data.rx.dropped() == 1 { dev_err!( data.dev, From 101931a15c0ea701d7c9f0d1ceb9118110baf0be Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:38:32 +1100 Subject: [PATCH 1248/1579] soc: apple: sep: Test the hwrng and sleep glue KUnit covers ControlState, the entropy exchange's bookkeeping. The code that drives it from the hwrng core and the PM notifier has no test: - the drain on the way to sleep, and its limit; - the order of the sleep holds, and their release when Touch ID refuses the suspend; - the mapping of a mailbox timeout to a full mailbox, which holds the next request back; - the hwrng read's result when requests are held back, which is not logged as a failure; - the ready-phase survey, which does not count an attempt that asked the enclave nothing. Move each decision into a small function the driver calls, with the locking, sleeping and logging left where they are. Test them there: the drain, the read's result, the survey and the mapping in control.rs, and the sleep event in pm.rs, through a trait that SepData implements and the test records calls on. Nothing changes in what the driver does. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/control.rs | 194 +++++++++++++++++++++++++++++++++++ drivers/soc/apple/pm.rs | 149 ++++++++++++++++++++++++--- drivers/soc/apple/sep.rs | 74 ++++++------- 3 files changed, 369 insertions(+), 48 deletions(-) diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index 5eb9d51c9f421..3c6bb59149824 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -56,6 +56,92 @@ pub(crate) enum EntropyEnd { Abandoned, } +impl EntropyEnd { + /// How a request ended that the mailbox refused with `e`. ETIMEDOUT is + /// its wait for room running out. + pub(crate) fn not_sent(e: Error) -> Self { + EntropyEnd::NotSent { + full: e == ETIMEDOUT, + } + } +} + +/// How a hwrng read ends. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ReadEnd { + /// It took this many bytes. + Bytes(usize), + /// None, as requests are being held back: nothing failed, and the core + /// tries again later. + Held(Error), + /// None, as the request failed. + Failed(Error), +} + +/// How a hwrng read ends that took `written` bytes and then stopped for +/// `failure`, if anything stopped it. +pub(crate) fn read_end(written: usize, failure: Option) -> ReadEnd { + if written > 0 { + return ReadEnd::Bytes(written); + } + // must return a negative errno, not 0: a blocking reader would spin on 0 + match failure.unwrap_or(EIO) { + e if e == EAGAIN => ReadEnd::Held(e), + e => ReadEnd::Failed(e), + } +} + +/// What came of an attempt at the ready-phase hwrng survey. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Survey { + Registered, + /// Requests were held back, so the enclave was not asked: try again, + /// without counting the attempt. + Held, + /// It failed, and attempts are left. + Retry(Error), + /// It failed, and that was the last attempt. + GiveUp(Error), +} + +/// Runs `survey` once if fewer than `max` attempts have been `used`. +/// Returns the attempts used after it, and what came of it. +pub(crate) fn survey_step( + used: u32, + max: u32, + survey: impl FnOnce() -> Result<()>, +) -> Option<(u32, Survey)> { + let attempt = used + 1; + if attempt > max { + return None; + } + Some(match survey() { + Ok(()) => (attempt, Survey::Registered), + Err(e) if e == EAGAIN => (used, Survey::Held), + Err(e) if attempt < max => (attempt, Survey::Retry(e)), + Err(e) => (attempt, Survey::GiveUp(e)), + }) +} + +/// Waits for `busy()` to clear, sleeping `step_ms` at a time through +/// `sleep()`, for at most `limit_ms`. Returns whether it cleared. +pub(crate) fn wait_idle( + limit_ms: u32, + step_ms: u32, + mut busy: impl FnMut() -> bool, + mut sleep: impl FnMut(u32), +) -> bool { + let mut waited: u32 = 0; + while busy() { + if waited >= limit_ms { + return false; + } + sleep(step_ms); + waited = waited.saturating_add(step_ms); + } + true +} + pub(crate) enum Delivery { Entropy, /// The answer to an entropy request that had timed out. `recovered` if @@ -280,6 +366,7 @@ impl ControlState { #[kernel::macros::kunit_tests(apple_sep_control)] mod tests { use super::*; + use core::cell::Cell; fn entropy(value: u32) -> proto::ControlReply { proto::ControlReply { @@ -446,4 +533,111 @@ mod tests { assert!(matches!(state.deliver(entropy(1)), Delivery::Unmatched)); assert_eq!(draw(&mut state, 0, 2), Some(2)); } + + #[test] + fn a_full_mailbox_is_told_from_other_send_failures() { + assert_eq!( + EntropyEnd::not_sent(ETIMEDOUT), + EntropyEnd::NotSent { full: true } + ); + for e in [ENODEV, EINTR, EIO] { + assert_eq!(EntropyEnd::not_sent(e), EntropyEnd::NotSent { full: false }); + } + // Only a full mailbox holds the next request back. + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::not_sent(ENODEV), 0); + assert_eq!(draw(&mut state, 0, 1), Some(1)); + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::not_sent(ETIMEDOUT), 0); + assert_eq!(state.entropy_begin(0), Err(EAGAIN)); + } + + #[test] + fn a_read_returns_what_it_took_or_why_not() { + assert_eq!(read_end(4, Some(ETIMEDOUT)), ReadEnd::Bytes(4)); + assert_eq!(read_end(64, None), ReadEnd::Bytes(64)); + assert_eq!(read_end(0, Some(ETIMEDOUT)), ReadEnd::Failed(ETIMEDOUT)); + // Never 0, which a blocking reader would spin on. + assert_eq!(read_end(0, None), ReadEnd::Failed(EIO)); + } + + #[test] + fn a_held_read_is_not_a_failure() { + assert_eq!(read_end(0, Some(EAGAIN)), ReadEnd::Held(EAGAIN)); + assert_eq!(read_end(8, Some(EAGAIN)), ReadEnd::Bytes(8)); + } + + #[test] + fn a_held_survey_attempt_is_not_counted() { + let max = 5; + assert_eq!(survey_step(0, max, || Err(EAGAIN)), Some((0, Survey::Held))); + assert_eq!(survey_step(3, max, || Err(EAGAIN)), Some((3, Survey::Held))); + // Not even when it would have been the last. + assert_eq!( + survey_step(max - 1, max, || Err(EAGAIN)), + Some((max - 1, Survey::Held)) + ); + } + + #[test] + fn the_survey_gives_up_after_its_attempts() { + let max = 5; + let mut used = 0; + for _ in 1..max { + assert_eq!( + survey_step(used, max, || Err(EIO)), + Some((used + 1, Survey::Retry(EIO))) + ); + used += 1; + } + assert_eq!( + survey_step(used, max, || Err(EIO)), + Some((max, Survey::GiveUp(EIO))) + ); + let ran = Cell::new(false); + let run = || { + ran.set(true); + Ok(()) + }; + assert_eq!(survey_step(max, max, run), None); + assert!(!ran.get()); + assert_eq!( + survey_step(2, max, || Ok(())), + Some((3, Survey::Registered)) + ); + } + + #[test] + fn the_drain_waits_for_the_request_in_flight() { + let slept = Cell::new(0u32); + let polls = Cell::new(0u32); + let busy = || { + polls.set(polls.get() + 1); + polls.get() <= 3 + }; + assert!(wait_idle(3000, 10, busy, |ms| slept.set(slept.get() + ms))); + assert_eq!(slept.get(), 30); + // Nothing in flight: no wait at all. + slept.set(0); + assert!(wait_idle( + 3000, + 10, + || false, + |ms| slept.set(slept.get() + ms) + )); + assert_eq!(slept.get(), 0); + } + + #[test] + fn the_drain_gives_up_at_its_limit() { + let slept = Cell::new(0u32); + assert!(!wait_idle( + 3000, + 10, + || true, + |ms| slept.set(slept.get() + ms) + )); + assert_eq!(slept.get(), 3000); + } } diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index b276890c27c78..2d918ec1fcf2a 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -41,6 +41,49 @@ pub(crate) fn keep_domains(dev: &kernel::device::Device) -> Result<()> { Ok(()) } +/// The holds a sleep event takes and releases, on [`SepData`]. +trait SleepHolds { + /// Holds entropy requests, once any in flight is answered. + fn hold_entropy(&self); + /// Ends any Touch ID capture and holds new ones. + fn hold_captures(&self) -> Result<()>; + fn release_captures(&self); + fn release_entropy(&self); +} + +impl SleepHolds for SepData { + fn hold_entropy(&self) { + self.rng_sleep_prepare(); + } + fn hold_captures(&self) -> Result<()> { + self.sleep_prepare() + } + fn release_captures(&self) { + self.sleep_finished(); + } + fn release_entropy(&self) { + self.rng_sleep_finished(); + } +} + +/// `entering`: the system is about to sleep; otherwise it has resumed, or +/// the suspend was aborted. +fn sleep_event(sep: &impl SleepHolds, entering: bool) -> Result<()> { + if entering { + sep.hold_entropy(); + if let Err(error) = sep.hold_captures() { + // A failing notifier is not included in the PM chain's rollback. + sep.release_captures(); + sep.release_entropy(); + return Err(error); + } + } else { + sep.release_captures(); + sep.release_entropy(); + } + Ok(()) +} + /// # Safety /// Called by `pm_shim.c` from the PM notifier chain, in process context. unsafe extern "C" fn pm_event(entering: bool) -> c_int { @@ -52,19 +95,10 @@ unsafe extern "C" fn pm_event(entering: bool) -> c_int { // `unregister` only after `sep_pm_unregister` has waited out any running // notifier call, so it is live for this borrow. let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; - if entering { - sep.rng_sleep_prepare(); - if let Err(error) = sep.sleep_prepare() { - // A failing notifier is not included in the PM chain's rollback. - sep.sleep_finished(); - sep.rng_sleep_finished(); - return error.to_errno(); - } - } else { - sep.sleep_finished(); - sep.rng_sleep_finished(); + match sleep_event(&*sep, entering) { + Ok(()) => 0, + Err(error) => error.to_errno(), } - 0 } pub(crate) fn register(sep: Arc) -> Result<()> { @@ -114,3 +148,94 @@ pub(crate) fn unregister() { drop(unsafe { as ForeignOwnable>::from_foreign(ptr) }); } } + +#[kernel::macros::kunit_tests(apple_sep_pm)] +mod tests { + use super::*; + use core::cell::{Cell, RefCell}; + + const HOLD_ENTROPY: u8 = 1; + const HOLD_CAPTURES: u8 = 2; + const RELEASE_CAPTURES: u8 = 3; + const RELEASE_ENTROPY: u8 = 4; + + struct Calls { + made: RefCell<[u8; 8]>, + count: Cell, + refuse: Option, + } + + impl Calls { + fn new(refuse: Option) -> Self { + Calls { + made: RefCell::new([0; 8]), + count: Cell::new(0), + refuse, + } + } + fn call(&self, which: u8) { + let n = self.count.get(); + self.made.borrow_mut()[n] = which; + self.count.set(n + 1); + } + fn made(&self) -> [u8; 8] { + *self.made.borrow() + } + } + + impl SleepHolds for Calls { + fn hold_entropy(&self) { + self.call(HOLD_ENTROPY); + } + fn hold_captures(&self) -> Result<()> { + self.call(HOLD_CAPTURES); + self.refuse.map_or(Ok(()), Err) + } + fn release_captures(&self) { + self.call(RELEASE_CAPTURES); + } + fn release_entropy(&self) { + self.call(RELEASE_ENTROPY); + } + } + + #[test] + fn going_to_sleep_holds_entropy_then_captures() { + let calls = Calls::new(None); + assert_eq!(sleep_event(&calls, true), Ok(())); + assert_eq!( + calls.made(), + [HOLD_ENTROPY, HOLD_CAPTURES, 0, 0, 0, 0, 0, 0] + ); + } + + #[test] + fn a_refused_sleep_releases_what_it_held() { + // The chain sends no POST event to the notifier that failed. + let calls = Calls::new(Some(EBUSY)); + assert_eq!(sleep_event(&calls, true), Err(EBUSY)); + assert_eq!( + calls.made(), + [ + HOLD_ENTROPY, + HOLD_CAPTURES, + RELEASE_CAPTURES, + RELEASE_ENTROPY, + 0, + 0, + 0, + 0 + ] + ); + } + + #[test] + fn waking_releases_captures_then_entropy() { + let calls = Calls::new(None); + assert_eq!(sleep_event(&calls, false), Ok(())); + assert_eq!( + calls.made(), + [RELEASE_CAPTURES, RELEASE_ENTROPY, 0, 0, 0, 0, 0, 0] + ); + } +} diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 0562752f927bb..3ae2bef3f4694 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1193,10 +1193,9 @@ impl SepData { let msg = proto::encode_control(&op, proto::TAG_ENTROPY); if let Err(e) = self.send(msg) { - let full = e == ETIMEDOUT; self.control .lock() - .entropy_end(control::EntropyEnd::NotSent { full }, boottime_ms()); + .entropy_end(control::EntropyEnd::not_sent(e), boottime_ms()); return Err(e); } @@ -1234,18 +1233,18 @@ impl SepData { if !self.control.lock().entropy_sleep() { return; } - let mut waited: u32 = 0; - while self.control.lock().entropy_busy() { - if waited >= RNG_SLEEP_DRAIN_MS { - dev_warn!( - self.dev, - "hwrng: a request was still in flight after {} ms; suspending anyway\n", - RNG_SLEEP_DRAIN_MS - ); - return; - } - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(10)); - waited += 10; + let idle = control::wait_idle( + RNG_SLEEP_DRAIN_MS, + 10, + || self.control.lock().entropy_busy(), + |ms| kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ms))), + ); + if !idle { + dev_warn!( + self.dev, + "hwrng: a request was still in flight after {} ms; suspending anyway\n", + RNG_SLEEP_DRAIN_MS + ); } } @@ -1282,24 +1281,32 @@ impl SepData { if this.rng.lock().is_some() { return; } - let attempt = this.rng_survey_attempts.load(Relaxed) + 1; - if attempt > RNG_SURVEY_ATTEMPTS { + let used = this.rng_survey_attempts.load(Relaxed); + let Some((attempt, next)) = + control::survey_step(used, RNG_SURVEY_ATTEMPTS, || this.survey_hwrng()) + else { return; - } + }; this.rng_survey_attempts.store(attempt, Relaxed); - match this.survey_hwrng() { - Ok(()) => dev_info!(this.dev, "hwrng: registered after ready-phase survey (attempt {})\n", attempt), + match next { + control::Survey::Registered => dev_info!( + this.dev, + "hwrng: registered after ready-phase survey (attempt {})\n", + attempt + ), // Asleep, or holding back after a request that went unanswered: // this attempt did not ask the enclave anything. - Err(e) if e == EAGAIN => { - this.rng_survey_attempts.store(attempt - 1, Relaxed); - Self::arm_settle(this); - } - Err(e) if attempt < RNG_SURVEY_ATTEMPTS => { + control::Survey::Held => Self::arm_settle(this), + control::Survey::Retry(e) => { dev_warn!(this.dev, "hwrng: survey attempt {} failed ({:?}); retrying\n", attempt, e); Self::arm_settle(this); } - Err(e) => dev_err!(this.dev, "hwrng: survey failed after {} attempts ({:?})\n", attempt, e), + control::Survey::GiveUp(e) => dev_err!( + this.dev, + "hwrng: survey failed after {} attempts ({:?})\n", + attempt, + e + ), } } @@ -1929,17 +1936,12 @@ impl SepData { } } - if written > 0 { - return written as c_int; - } - - // must return a negative errno, not 0: a blocking reader would spin on 0 - let e = failure.unwrap_or(EIO); - // Asleep, or holding back after a request that went unanswered. - // Nothing failed in this read; the core tries again later. - if e == EAGAIN { - return e.to_errno(); - } + let e = match control::read_end(written, failure) { + control::ReadEnd::Bytes(n) => return n as c_int, + // Asleep, or holding back after a request that went unanswered. + control::ReadEnd::Held(e) => return e.to_errno(), + control::ReadEnd::Failed(e) => e, + }; let n = self.rng_failures.load(Relaxed).wrapping_add(1); self.rng_failures.store(n, Relaxed); if n == 1 { From d6bdc9d4ba38487b9fd41101b3f175015655c45b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:39:47 +1100 Subject: [PATCH 1249/1579] soc: apple: sep: Write no entropy request once the system starts to sleep Going to sleep sets the hold and then waits for a request in flight, for up to 3 s. That was meant to cover the mailbox's 500 ms wait for room and then the 2 s wait for the reply. But the hold is checked when the exchange is claimed, before the sender takes the mailbox lock. A sender claimed in time can then queue for the lock behind other senders, each of which may wait 500 ms for room. One of them can be a Touch ID capture, which is ended only after the entropy drain. So the 3 s bound did not hold. Once it ran out, the drain warned and the suspend went on, and the sender could write its request later in the window. That is what the hold exists to stop: on a T6000 a request sent while devices were suspended was never answered. Check the hold again with the mailbox lock held, just before the request is written. A request claimed before the sleep began but not yet written ends there as not sent, owing nothing and holding nothing back after the wake. Only a request already being written can still go out. Its 500 ms wait for room and 2 s wait for the reply are within the drain's 3 s, so the bound now holds. The drain still only warns when it runs out, as there is then nothing left that it could stop. Add a KUnit case for a request that sleep overtakes before it is written. It fails when the new check always allows the send. Fixes: 1b0451249175 ("soc: apple: sep: Hold entropy requests while the system sleeps") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/control.rs | 28 ++++++++++++++++++++++++++++ drivers/soc/apple/sep.rs | 24 ++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index 3c6bb59149824..9140989d37305 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -358,6 +358,13 @@ impl ControlState { self.entropy.busy } + /// Whether a request claimed by [`Self::entropy_begin`] may still be + /// written: not once the system has begun going to sleep. Such a + /// request ends as [`EntropyEnd::NotSent`], not full. + pub(crate) fn entropy_may_send(&self) -> bool { + !self.entropy.asleep + } + pub(crate) fn entropy_wake(&mut self) { self.entropy.asleep = false; } @@ -428,6 +435,27 @@ mod tests { assert_eq!(draw(&mut state, 0, 6), Some(6)); } + #[test] + fn a_request_not_yet_written_when_sleep_begins_is_not_sent() { + let mut state = ControlState::new(); + assert!(state.entropy_begin(0).is_ok()); + assert!(state.entropy_may_send()); + // The sleep begins while the request waits for the mailbox. + assert!(state.entropy_sleep()); + assert!(!state.entropy_may_send()); + state.entropy_end(EntropyEnd::not_sent(EAGAIN), 0); + // That ends the drain, and the request is not held against the + // enclave: nothing is owed, and nothing is held back after the wake. + assert!(!state.entropy_busy()); + assert_eq!(state.entropy_begin(0), Err(EAGAIN)); + state.entropy_wake(); + assert!(state.entropy_begin(0).is_ok()); + assert!(state.entropy_may_send()); + assert!(matches!(state.deliver(entropy(3)), Delivery::Entropy)); + assert_eq!(state.entropy_take(), Some(3)); + state.entropy_end(EntropyEnd::Answered, 0); + } + #[test] fn a_late_reply_is_not_the_next_requests_answer() { let mut state = ControlState::new(); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 3ae2bef3f4694..07f733921e23b 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -182,7 +182,9 @@ const PROTECTED_DATA_AVAILABLE: bool = true; const RNG_MAX_WORDS_PER_READ: usize = 16; // How long going to sleep waits for an entropy request in flight: the -// mailbox's 500 ms wait for room, then the wait for the reply. +// mailbox's 500 ms wait for room, then the wait for the reply. Only a request +// already being written can still go out once the sleep has begun; see +// `send_entropy_request()`. const RNG_SLEEP_DRAIN_MS: u32 = proto::CONTROL_TIMEOUT_MS + 1000; extern "C" { @@ -1192,7 +1194,7 @@ impl SepData { self.control.lock().entropy_begin(boottime_ms())?; let msg = proto::encode_control(&op, proto::TAG_ENTROPY); - if let Err(e) = self.send(msg) { + if let Err(e) = self.send_entropy_request(msg) { self.control .lock() .entropy_end(control::EntropyEnd::not_sent(e), boottime_ms()); @@ -1224,6 +1226,24 @@ impl SepData { result } + /// Sends an entropy request claimed by `entropy_begin()`, or EAGAIN if + /// the system has begun going to sleep since. The check is made with the + /// mailbox lock held. A sender can wait for that lock behind others, such + /// as a Touch ID capture still running, for longer than the drain on the + /// way to sleep waits for it. It must not then write its request into + /// the suspend. Lock order: mailbox, then control. + fn send_entropy_request(&self, msg: Message) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + let mbox = self.mbox.lock(); + let mbox = mbox.as_ref().ok_or(ENODEV)?; + if !self.control.lock().entropy_may_send() { + return Err(EAGAIN); + } + Self::ring(mbox, msg) + } + /// The system is about to suspend: send no entropy request until it has /// resumed, and let one already sent be answered while the enclave can /// still answer it. On a T6000 a request sent while devices were From 4bba4c5c965970d0c752e245e703163f107d5dac Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:40:39 +1100 Subject: [PATCH 1250/1579] soc: apple: sep: Let a blocking hwrng read wait while requests are held While entropy requests are held back, after one went unanswered or while the system goes to sleep, the read refuses at once with EAGAIN. The hwrng core passes that straight to a reader of /dev/hwrng, even one that blocks. Before the hold, such a reader blocked for the 2 s request timeout and then got ETIMEDOUT. Now it gets EAGAIN from a blocking file descriptor, which is unusual. A reader that tries again on EAGAIN spins for the whole backoff, up to 60 s. The fill thread is not affected: it sleeps 10 s after any read that returns nothing. hwrng_fill() cannot tell the fill thread from a user reader, but both read with wait set. So when a blocking read has nothing yet, let it wait for the hold to end, for at most the time a request waits for its reply. That is until the retry time, the wake, or the enclave answering again, each of which now wakes it. If the hold ends, the read asks then. If it does not, the read fails with ETIMEDOUT, as a request that went unanswered does, and that is not logged as a failure. A signal ends the wait with ERESTARTSYS, so a reader being frozen for the suspend restarts its read after the resume. The enclave is still not asked while requests are held. A read that already has bytes still returns them at once. The wait holds the hwrng core's reading_mutex for up to 2 s. That is no longer than a request that timed out held it before the hold existed. Add KUnit cases for how long a held read waits and for what it returns when the hold outlasts it. The second fails with the old EAGAIN mapping. Fixes: f771659aba5e ("soc: apple: sep: Recover the entropy exchange after a request times out") Fixes: 1b0451249175 ("soc: apple: sep: Hold entropy requests while the system sleeps") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/control.rs | 66 +++++++++++++++++++++++++++++++++--- drivers/soc/apple/sep.rs | 64 ++++++++++++++++++++++++++++++---- 2 files changed, 120 insertions(+), 10 deletions(-) diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs index 9140989d37305..946c95d7a46d3 100644 --- a/drivers/soc/apple/control.rs +++ b/drivers/soc/apple/control.rs @@ -71,8 +71,8 @@ impl EntropyEnd { pub(crate) enum ReadEnd { /// It took this many bytes. Bytes(usize), - /// None, as requests are being held back: nothing failed, and the core - /// tries again later. + /// None, but nothing failed: requests are being held back, or a signal + /// ended the wait for that to end. The core tries again later. Held(Error), /// None, as the request failed. Failed(Error), @@ -86,7 +86,12 @@ pub(crate) fn read_end(written: usize, failure: Option) -> ReadEnd { } // must return a negative errno, not 0: a blocking reader would spin on 0 match failure.unwrap_or(EIO) { - e if e == EAGAIN => ReadEnd::Held(e), + // The reader blocked while requests were held back, as long as a + // request waits for its reply. Tell it that it timed out, as an + // unanswered request would. EAGAIN would invite it to read again + // at once. + e if e == EAGAIN => ReadEnd::Held(ETIMEDOUT), + e if e == ERESTARTSYS => ReadEnd::Held(e), e => ReadEnd::Failed(e), } } @@ -358,6 +363,18 @@ impl ControlState { self.entropy.busy } + /// How long from `now_ms` requests are held back, up to `limit_ms`, or + /// `None` if [`Self::entropy_begin`] would not refuse one for that. + pub(crate) fn entropy_held_ms(&self, now_ms: u64, limit_ms: u64) -> Option { + if self.entropy.asleep { + return Some(limit_ms); + } + match self.entropy.retry_at { + Some(at) if now_ms < at => Some(core::cmp::min(at - now_ms, limit_ms)), + _ => None, + } + } + /// Whether a request claimed by [`Self::entropy_begin`] may still be /// written: not once the system has begun going to sleep. Such a /// request ends as [`EntropyEnd::NotSent`], not full. @@ -592,10 +609,51 @@ mod tests { #[test] fn a_held_read_is_not_a_failure() { - assert_eq!(read_end(0, Some(EAGAIN)), ReadEnd::Held(EAGAIN)); + assert_eq!(read_end(0, Some(ERESTARTSYS)), ReadEnd::Held(ERESTARTSYS)); assert_eq!(read_end(8, Some(EAGAIN)), ReadEnd::Bytes(8)); } + #[test] + fn a_read_held_until_it_gives_up_is_told_it_timed_out() { + // Not EAGAIN: a blocking reader that retries on it would spin. + assert_eq!(read_end(0, Some(EAGAIN)), ReadEnd::Held(ETIMEDOUT)); + } + + #[test] + fn a_held_read_knows_how_long_to_wait() { + let limit = u64::from(proto::CONTROL_TIMEOUT_MS); + let mut state = ControlState::new(); + assert_eq!(state.entropy_held_ms(0, limit), None); + // Held back after a request went unanswered: until the retry time, + // or the reader's limit if that comes first. + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::TimedOut, 0); + let at = ENTROPY_RETRY_MIN_MS; + assert_eq!( + state.entropy_held_ms(0, limit), + Some(core::cmp::min(at, limit)) + ); + assert_eq!(state.entropy_held_ms(at - 10, limit), Some(10)); + assert_eq!(state.entropy_held_ms(at - 10, 0), Some(0)); + assert_eq!(state.entropy_held_ms(at, limit), None); + // The hold ends early when the enclave answers again. + assert!(matches!( + state.deliver(entropy(1)), + Delivery::EntropyLate { recovered: true } + )); + assert_eq!(state.entropy_held_ms(0, limit), None); + // Asleep: for as long as the reader may wait. + assert!(!state.entropy_sleep()); + assert_eq!(state.entropy_held_ms(0, limit), Some(limit)); + state.entropy_wake(); + assert_eq!(state.entropy_held_ms(0, limit), None); + // Whenever begin refuses with EAGAIN, the reader has a hold to wait on. + assert!(state.entropy_begin(0).is_ok()); + state.entropy_end(EntropyEnd::TimedOut, 0); + assert_eq!(state.entropy_begin(1), Err(EAGAIN)); + assert!(state.entropy_held_ms(1, limit).is_some()); + } + #[test] fn a_held_survey_attempt_is_not_counted() { let max = 5; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 07f733921e23b..ba16606f0dfed 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1226,6 +1226,46 @@ impl SepData { result } + /// As [`Self::get_entropy_word`], for a reader that blocks. While requests + /// are held back, wait for that to end, for as long as a request waits + /// for its reply, rather than fail at once. EAGAIN if it does not end + /// in that time, ERESTARTSYS if a signal comes. + fn get_entropy_word_blocking(&self) -> Result { + let limit = u64::from(proto::CONTROL_TIMEOUT_MS); + let deadline = boottime_ms().saturating_add(limit); + loop { + // As `hwrng_fill()` checks before each word: a removal wakes the + // wait below, and no request may go out after it. + if self.rng_shutdown.load(Relaxed) { + return Err(ENODEV); + } + match self.get_entropy_word() { + Err(e) if e == EAGAIN => {} + done => return done, + } + let mut guard = self.control.lock(); + let now = boottime_ms(); + let left = deadline.saturating_sub(now); + match guard.entropy_held_ms(now, left) { + // It ended since; ask now. + None if left > 0 => continue, + None | Some(0) => return Err(EAGAIN), + Some(ms) => { + if self.rng_shutdown.load(Relaxed) { + return Err(ENODEV); + } + let ms = u32::try_from(ms).unwrap_or(proto::CONTROL_TIMEOUT_MS); + if let CondVarTimeoutResult::Signal { .. } = self + .control_wq + .wait_interruptible_timeout(&mut guard, time::msecs_to_jiffies(ms)) + { + return Err(ERESTARTSYS); + } + } + } + } + } + /// Sends an entropy request claimed by `entropy_begin()`, or EAGAIN if /// the system has begun going to sleep since. The check is made with the /// mailbox lock held. A sender can wait for that lock behind others, such @@ -1271,6 +1311,8 @@ impl SepData { /// The system has resumed, or the suspend was aborted. pub(crate) fn rng_sleep_finished(&self) { self.control.lock().entropy_wake(); + // A blocking hwrng reader may be waiting for this. + self.control_wq.notify_all(); } fn control_survey(&self) { @@ -1942,7 +1984,13 @@ impl SepData { failure = Some(ENODEV); break; } - match self.get_entropy_word() { + // Only a read that has nothing yet waits while requests are held. + let word = if written == 0 { + self.get_entropy_word_blocking() + } else { + self.get_entropy_word() + }; + match word { Ok(value) => { // SAFETY: the core guarantees `buf` is valid for `max` // bytes and aligned for any type, and `i < max / 4`. @@ -1958,7 +2006,7 @@ impl SepData { let e = match control::read_end(written, failure) { control::ReadEnd::Bytes(n) => return n as c_int, - // Asleep, or holding back after a request that went unanswered. + // Held back for as long as the read could wait, or interrupted. control::ReadEnd::Held(e) => return e.to_errno(), control::ReadEnd::Failed(e) => e, }; @@ -2209,10 +2257,14 @@ impl SepData { control::Delivery::Entropy => { self.control_wq.notify_all(); } - control::Delivery::EntropyLate { recovered: true } => dev_info!( - self.dev, - "hwrng: the enclave answered a request that had timed out; reads resume\n" - ), + control::Delivery::EntropyLate { recovered: true } => { + dev_info!( + self.dev, + "hwrng: the enclave answered a request that had timed out; reads resume\n" + ); + // A blocking hwrng reader may be waiting for this. + self.control_wq.notify_all(); + } control::Delivery::EntropyLate { recovered: false } => {} control::Delivery::Matched => self.control_wq.notify_all(), control::Delivery::Unmatched => {} From 39b1fdcee0d32127f3be04c578353da9a2783cd6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:41:54 +1100 Subject: [PATCH 1251/1579] KEYS: trusted: Skip after_core_init for the reason it cannot run trusted_source_after_core_init expects the core's init to succeed and then skips when the built-in fake was not tried, blaming another source for holding the slot. With trusted.rng=trusted_source_kunit the init fails with -EINVAL, as it should, since that RNG is no built-in source's. The expectation fails, and the skip that follows overwrites the failure, so the case is reported as skipped for a slot nobody holds. With trusted.source= naming another source, the fake is not tried either, and the skip blames another source too. Look at the result first, and skip with the reason the fake was not tried: trusted.rng= rules it out, trusted.source= rules it out, or a source holds the slot. Which of the last two applies is found by trying to register the runtime fake, which trusted.source= refuses with -ENODEV and a held slot with -EBUSY. Any other result of the init is still a failure. Let the built-in fake's init() result be set by a test, for the next change. It stays -ENODEV here. Fixes: 4f15946bdd60 ("KEYS: trusted: Keep a source that registers before init_trusted()") Assisted-by: LLM Signed-off-by: Ryan Murray --- .../trusted-keys/tests/trusted_source_kunit.c | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/security/keys/trusted-keys/tests/trusted_source_kunit.c b/security/keys/trusted-keys/tests/trusted_source_kunit.c index 6a5eb3e79d756..3892df41078ff 100644 --- a/security/keys/trusted-keys/tests/trusted_source_kunit.c +++ b/security/keys/trusted-keys/tests/trusted_source_kunit.c @@ -33,6 +33,7 @@ struct trusted_source_fake { /* A built-in source for trusted_key_core_init(). */ struct trusted_key_ops builtin_ops; struct trusted_key_source builtin; + int builtin_ret; int builtin_inits; }; @@ -54,7 +55,7 @@ static void fake_exit(void) static int fake_builtin_init(void) { fake->builtin_inits++; - return -ENODEV; + return fake->builtin_ret; } /* Not a cipher: an invertible stand-in, so that unseal can be checked. */ @@ -107,6 +108,32 @@ static void fake_register(struct kunit *test) KUNIT_ASSERT_EQ(test, ret, 0); } +/* + * Run the core's init with the built-in fake, or skip, giving the reason, if + * the fake is not tried: a source holds the slot, or trusted.source= or + * trusted.rng= rules the fake out. + */ +static int fake_core_init(struct kunit *test) +{ + int ret = trusted_key_core_init(&fake->builtin, 1); + + if (fake->builtin_inits) + return ret; + + /* The only error before a source's init(). */ + if (ret == -EINVAL) + kunit_skip(test, "trusted.rng= rules out the built-in source"); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = register_trusted_key_source(&fake->src); + if (!ret) + unregister_trusted_key_source(&fake->src); + if (ret == -EBUSY) + kunit_skip(test, "another trusted-key source holds the slot"); + kunit_skip(test, "trusted.source= rules out the built-in source"); + return 0; +} + /* Instantiate a trusted key from @cmd; it is put when the test ends. */ static struct key *fake_key(struct kunit *test, const char *cmd) { @@ -195,9 +222,7 @@ static void trusted_source_before_core_init(struct kunit *test) */ static void trusted_source_after_core_init(struct kunit *test) { - KUNIT_EXPECT_EQ(test, trusted_key_core_init(&fake->builtin, 1), 0); - if (!fake->builtin_inits) - kunit_skip(test, "another trusted-key source holds the slot"); + KUNIT_EXPECT_EQ(test, fake_core_init(test), 0); /* With no source, the absent stubs answer. */ KUNIT_EXPECT_EQ(test, PTR_ERR(fake_key(test, "new 32")), -ENODEV); @@ -380,6 +405,7 @@ static int trusted_source_init(struct kunit *test) fake->builtin_ops = fake->ops; fake->builtin_ops.init = fake_builtin_init; + fake->builtin_ret = -ENODEV; fake->builtin.name = FAKE_BUILTIN_NAME; fake->builtin.ops = &fake->builtin_ops; From 75890a6821030364000ad84e49b58fdb2e2332e7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 23:42:37 +1100 Subject: [PATCH 1252/1579] KEYS: trusted: Give the slot to a built-in source only from init_trusted() trusted_key_core_init() is exported for KUnit so that the tests can run the core's init again, after init_trusted(), with synthetic built-in sources. When such a source's init() succeeds, the core gives it the slot as init_trusted() would. But a source that takes the slot this way is never withdrawn: unregister_trusted_key_source() withdraws only a registered source, and this one is not. A test's source is freed when the test ends, so the dispatch would be left on it, and every later seal would call into freed memory. Today's tests pass only a built-in source that fails with -ENODEV, so nothing has hit this. Let only the first call, init_trusted()'s, give the slot to a built-in source. A later call whose source initialises calls the source's exit() and returns -EBUSY, leaving the slot as it was. init_trusted() runs once, at late_initcall, and built-in KUnit runs after the initcalls, so no real source is affected. Nothing changes without KUnit. Add a KUnit case for a built-in source that initialises after the core's init. It fails without the check. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../trusted-keys/tests/trusted_source_kunit.c | 24 +++++++++++++++++++ security/keys/trusted-keys/trusted_core.c | 17 +++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/security/keys/trusted-keys/tests/trusted_source_kunit.c b/security/keys/trusted-keys/tests/trusted_source_kunit.c index 3892df41078ff..3ef2e348f17a0 100644 --- a/security/keys/trusted-keys/tests/trusted_source_kunit.c +++ b/security/keys/trusted-keys/tests/trusted_source_kunit.c @@ -237,6 +237,28 @@ static void trusted_source_after_core_init(struct kunit *test) KUNIT_EXPECT_EQ(test, fake->builtin_inits, 1); } +/* + * After the core's init, a built-in source is not given the slot, even one + * that initialises: nothing could withdraw it, and this one is gone once the + * test ends. The slot stays free for a source module. + */ +static void trusted_source_builtin_after_core_init(struct kunit *test) +{ + fake->builtin_ret = 0; + KUNIT_EXPECT_EQ(test, fake_core_init(test), -EBUSY); + KUNIT_EXPECT_EQ(test, fake->builtin_inits, 1); + KUNIT_EXPECT_EQ(test, fake->exits, 1); + KUNIT_EXPECT_EQ(test, PTR_ERR(fake_key(test, "new 32")), -ENODEV); + KUNIT_EXPECT_EQ(test, fake->seals, 0); + + /* The fake was tried, so no source held the slot: no skip here. */ + KUNIT_ASSERT_EQ(test, register_trusted_key_source(&fake->src), 0); + KUNIT_EXPECT_FALSE(test, IS_ERR(fake_key(test, "new 32"))); + KUNIT_EXPECT_EQ(test, fake->seals, 1); + unregister_trusted_key_source(&fake->src); + KUNIT_EXPECT_EQ(test, fake->exits, 2); +} + /* Keys are sealed and unsealed by the source until it withdraws. */ static void trusted_source_dispatch(struct kunit *test) { @@ -423,6 +445,8 @@ static struct kunit_case trusted_source_cases[] = { KUNIT_CASE(trusted_source_exclusive), KUNIT_CASE(trusted_source_init_failure), KUNIT_CASE(trusted_source_invalid), + /* Last, as a core without the check would leave the fake in the slot. */ + KUNIT_CASE(trusted_source_builtin_after_core_init), {} }; diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c index 718a480484330..18937a1665848 100644 --- a/security/keys/trusted-keys/trusted_core.c +++ b/security/keys/trusted-keys/trusted_core.c @@ -65,6 +65,8 @@ static unsigned char migratable; static DEFINE_MUTEX(trusted_key_source_lock); static bool trusted_key_source_active; static struct trusted_key_source *trusted_key_source_registered; +/* trusted_key_core_init() has run once, from init_trusted(). */ +static bool trusted_key_core_done; /* Defined below; registration needs it before its definition. */ static int kernel_get_random(unsigned char *key, size_t key_len); @@ -487,9 +489,13 @@ static int kernel_get_random(unsigned char *key, size_t key_len) * the absent stubs, so that a source module can register later. * * init_trusted() calls this once, with trusted_key_sources. The KUnit tests - * call it with synthetic sources of their own. + * call it later, with synthetic sources of their own. Only the first call + * gives the slot to a built-in source: one that takes it is never withdrawn, + * as unregister_trusted_key_source() withdraws only a registered source, + * and a test's source is gone once the test ends. * - * Return: 0, or the error of trusted.rng or of a built-in source's init(). + * Return: 0, the error of trusted.rng or of a built-in source's init(), or + * -EBUSY for a built-in source that initialises after the first call. */ VISIBLE_IF_KUNIT int trusted_key_core_init(const struct trusted_key_source *sources, size_t n) @@ -539,6 +545,12 @@ trusted_key_core_init(const struct trusted_key_source *sources, size_t n) get_random = kernel_get_random; ret = sources[i].ops->init(); + if (!ret && trusted_key_core_done) { + if (sources[i].ops->exit) + sources[i].ops->exit(); + ret = -EBUSY; + goto out; + } if (!ret) { static_call_update(trusted_key_seal, sources[i].ops->seal); static_call_update(trusted_key_unseal, sources[i].ops->unseal); @@ -573,6 +585,7 @@ trusted_key_core_init(const struct trusted_key_source *sources, size_t n) if (!ret) trusted_key_source_active = true; out: + trusted_key_core_done = true; mutex_unlock(&trusted_key_source_lock); return ret; } From 9e5bef87c68aca2b876dc9adb8f85d6354b1a9ec Mon Sep 17 00:00:00 2001 From: "Mukesh Kumar Chaurasiya (IBM)" Date: Wed, 8 Jul 2026 13:54:49 +0530 Subject: [PATCH 1253/1579] dma-resv: Fix undefined symbol when CONFIG_DMA_SHARED_BUFFER is disabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When building with LLVM=1 for architectures like powerpc where CONFIG_DMA_SHARED_BUFFER is not enabled, the build fails with: ld.lld: error: undefined symbol: dma_resv_reset_max_fences >>> referenced by helpers.c >>> rust/helpers/helpers.o:(rust_helper_dma_resv_unlock) The issue occurs because: 1. CONFIG_DEBUG_MUTEXES=y is enabled 2. CONFIG_DMA_SHARED_BUFFER is not enabled 3. dma_resv_reset_max_fences() is declared in the header when CONFIG_DEBUG_MUTEXES is set 4. But the function is only compiled in drivers/dma-buf/dma-resv.c, which is only built when CONFIG_DMA_SHARED_BUFFER is enabled 5. Rust helpers call dma_resv_unlock() which calls dma_resv_reset_max_fences(), causing an undefined symbol Fix this by compiling `dma-resv.c` file only when CONFIG_DMA_SHARED_BUFFER is enabled. (cherry picked from commit be809b60cbb61aab96179f44ac3670242ae72996 ("dma-resv: Fix undefined symbol when CONFIG_DMA_SHARED_BUFFER is disabled")) Fixes: 9b836641d3bf ("rust: helpers: Add bindings/wrappers for dma_resv_lock") Reviewed-by: Christian König Reviewed-by: Gary Guo Signed-off-by: Mukesh Kumar Chaurasiya (IBM) Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260708082454.1254320-3-mkchauras@gmail.com Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/helpers/helpers.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 8f6a7c7a63a0d..66df433146cba 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -60,7 +60,9 @@ #include "dma.c" #include "dma-fence.c" #include "dma-mapping.c" +#ifdef CONFIG_DMA_SHARED_BUFFER #include "dma-resv.c" +#endif #include "drm.c" #include "drm_gpuvm.c" #include "drm_syncobj.c" From 6f715d4dc5ac7fb4e165720f65f43a97d0a57d43 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 17 Oct 2026 22:40:45 +1100 Subject: [PATCH 1254/1579] usb: typec: tipd: Limit the retries of a held mux by time, not count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A mux that holds a change up is retried until it goes through, and a PHY that stays stuck is given up after 16 retries, said to be some two minutes. That holds only while nothing else happens. A controller event during the backoff brings the retry forward to the usual debounce, and that retry counted against the 16 too, without making the hold-up any shorter. With an event in every backoff, as a partner that keeps renegotiating raises, the retries come about a second apart and the port gave up after some 16.5 s. If the PHY then let go after the last event, nothing applied the state again until a later event. Limit a series by time instead: note when its first failure was, and give up at the first failure two minutes or more after it. The start is noted whenever a failure finds no series under way, so a series that ended in any way, each of which clears its backoff, never lends its start to the next. The state machine keeps to milliseconds, as for its delays, and the driver passes jiffies_to_msecs(jiffies); the difference is taken so that the clock may wrap. Keep the count only as a bound on the warnings of a series. Ordinary controller events bring retries about a second apart, debouncing both the read and the update. Tunnel notifications skip the read debounce, but the 500 ms update debounce alone keeps the limit of 240 from ending a series before two minutes. Counting only the retries that the timer runs would bound nothing that the time limit does not. The warning now gives the retries and the time. Correct the texts that promised about two minutes for a count. Also make typec_mux_timeout_transient()'s kernel-doc say that it describes the last request that failed with -ETIMEDOUT: one that fails with -EBUSY before it owns the providers leaves the answer as it was. A consumer asks only right after -ETIMEDOUT, so say so rather than change the core. The query keeps its form, a flag on the handle; upstream would rather have typec_mux_set() return the answer. The new tests inject an event into every backoff while the PHY holds the change up, and expect the port to give up only two minutes after the first refusal, after some 120 retries rather than 16. With an event every 1.5 s for well over a minute, after which the PHY lets go, the next retry of the backoff must apply the safe and USB states with no further event. The fixture runs works before their delay, so it makes a series as old on the driver's clock as it is in the time of the tests. The state machine tests cover the limit, a clock that wraps, retries a second apart, the count and the fresh start after an event, and the backoff test reads the backoff rather than the time left on its timer. The apple-dcp host selftests pass the time of each failure and check that retries a second apart do not end a series sooner. Repair two existing test races exposed while qualifying this change on ed289aabb019. The IRQ test joins the immediate resume read and checks that retries attempt acknowledgment, rather than requiring work to stay pending. The host teardown fixture ends its observation at the lock that fences the selected removal: direct xHCI teardown may proceed once the host mutex is dropped, while the controller mutex is still held. Force that interleaving and check that the PHY and root accesses have finished before removal can proceed. [ryan: rebased onto #355 — kept HPD v2 and #349 test changes; combined the ceiling retry race checks, clock shims and new cases.] Fixes: 70f7c21e7d32 ("phy: apple: atc: Route the PIPE from the Type-C mux on a fixed-hub port") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 2 +- drivers/usb/typec/mux.c | 6 +- drivers/usb/typec/tipd/cd321x-irq-test.c | 17 ++- drivers/usb/typec/tipd/cd321x-mux-test.c | 139 ++++++++++++++++-- drivers/usb/typec/tipd/cd321x-pm-test.c | 87 ++++++++--- drivers/usb/typec/tipd/cd321x-pm.h | 33 ++++- drivers/usb/typec/tipd/core.c | 12 +- .../drivers/gpu/apple-dcp/lifecycle.py | 8 + .../selftests/drivers/gpu/apple-dcp/pm.py | 31 +++- .../drivers/gpu/apple-dcp/teardown_pm.py | 24 ++- 10 files changed, 301 insertions(+), 58 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index a53eeb5f9f5a7..77f9bc5d953dd 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -4087,7 +4087,7 @@ static void atcphy_typec_mux_unregister(void *data) * Only a timeout of this PHY counts, not one of the display route that shares * the connector. The PHY's other timeouts, of its power and PLL polls, are * faults that retrying does not clear, but the Type-C driver stops retrying - * after about two minutes, which bounds those as well. + * once a hold-up has lasted two minutes, which bounds those as well. */ static bool atcphy_mux_transient_timeouts(const struct apple_atcphy *atcphy) { diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index 836e7cb40a7b5..8a6a231e6b976 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -345,8 +345,10 @@ EXPORT_SYMBOL_GPL(typec_mux_safe_after_role); * hardware then only held the change up, and the same state can be applied * again later. A timeout of any other provider can be permanent. * - * The answer is about the last request on @mux: no other request on the same - * handle may run in between. + * The answer describes the last request on @mux that failed with -ETIMEDOUT, + * so ask before any other request on the same handle runs. One that fails + * before it owns the providers, such as with -EBUSY, leaves the answer as it + * was; any other request changes it. * * Return: true if the provider that timed out declared its timeouts transient. */ diff --git a/drivers/usb/typec/tipd/cd321x-irq-test.c b/drivers/usb/typec/tipd/cd321x-irq-test.c index 0c5067e9ab024..9ba1af1904f42 100644 --- a/drivers/usb/typec/tipd/cd321x-irq-test.c +++ b/drivers/usb/typec/tipd/cd321x-irq-test.c @@ -315,6 +315,7 @@ static void cd321x_ack_retry_across_sleep_test(struct kunit *test) struct cd321x_test *t = cd321x_test_port(test); struct device *dev = t->cd.tps.dev; unsigned int i, left; + unsigned int failures; cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); t->ctrl.failing_event_reads = UINT_MAX; @@ -332,13 +333,23 @@ static void cd321x_ack_retry_across_sleep_test(struct kunit *test) * schedules early, and check that it tried to read the events. */ flush_work(&t->cd.resume_work.work); - KUNIT_EXPECT_EQ(test, t->ctrl.failing_event_reads, left - 1); + scoped_guard(mutex, &t->cd.tps.lock) { + KUNIT_EXPECT_EQ(test, t->ctrl.failing_event_reads, left - 1); + KUNIT_EXPECT_LT(test, t->ctrl.failing_event_reads, left); + } for (i = 0; i < 2 * CD321X_RESUME_ATTEMPTS; i++) { - KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + scoped_guard(mutex, &t->cd.tps.lock) + failures = t->ctrl.failing_event_reads; flush_delayed_work(&t->cd.resume_work); + scoped_guard(mutex, &t->cd.tps.lock) { + KUNIT_EXPECT_LT(test, t->ctrl.failing_event_reads, failures); + KUNIT_EXPECT_NE(test, t->cd.pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_GT(test, t->cd.pm.ack_retry_ms, 0U); + } } - t->ctrl.failing_event_reads = 0; + scoped_guard(mutex, &t->cd.tps.lock) + t->ctrl.failing_event_reads = 0; cd321x_test_recover(test, t); KUNIT_EXPECT_EQ(test, t->ctrl.latched, 0); cd321x_test_raise(&t->ctrl, APPLE_CD_REG_INT_PLUG_EVENT); diff --git a/drivers/usb/typec/tipd/cd321x-mux-test.c b/drivers/usb/typec/tipd/cd321x-mux-test.c index 539ee405dc924..23a6c740b2364 100644 --- a/drivers/usb/typec/tipd/cd321x-mux-test.c +++ b/drivers/usb/typec/tipd/cd321x-mux-test.c @@ -84,9 +84,9 @@ struct cd321x_mux_test; * A mode switch on the connector. While @held, as the rear port PHY of a J700 * is through the power elections of a Mac taking another data role, when it * cannot park the PIPE, it fails every change with @held_ret. @log records the - * states asked for: s(afe), u(sb) or o(ther), in capitals when refused. The - * display route also fails to quiesce, before any provider changes, while - * @prepare_held. + * states asked for: s(afe), u(sb) or o(ther), in capitals when refused, and + * @refused counts the refusals. The display route also fails to quiesce, + * before any provider changes, while @prepare_held. */ struct cd321x_mux_test_phy { struct cd321x_mux_test *test; @@ -94,6 +94,7 @@ struct cd321x_mux_test_phy { bool prepare_held; int held_ret; char log[64]; + unsigned int refused; unsigned int last_refused_ms; }; @@ -107,6 +108,8 @@ struct cd321x_mux_test { struct cd321x_mux_test_phy display; /* The time the scheduled work has been run at */ unsigned int now_ms; + /* The time the driver's latest series of retries of a held mux began */ + unsigned int series_ms; }; static int cd321x_mux_test_read(void *context, const void *reg, size_t reg_size, @@ -165,6 +168,7 @@ static int cd321x_mux_test_set(struct typec_mux_dev *mux, struct typec_mux_state step = 'o'; if (phy->held) { step = toupper(step); + phy->refused++; phy->last_refused_ms = phy->test->now_ms; } if (len < sizeof(phy->log) - 1) @@ -370,6 +374,29 @@ static unsigned int cd321x_mux_test_due_ms(struct delayed_work *dwork) return left > 0 ? jiffies_to_msecs(left) : 0; } +/* + * Run @dwork now, rather than after its delay. The driver also measures how + * long it has retried a held mux, on its own clock, so a series that is under + * way is made as old there as it is in the time of the tests. + */ +static void cd321x_mux_test_run(struct cd321x_mux_test *t, struct delayed_work *dwork) +{ + struct cd321x_pm_state *pm = &t->cd.pm; + bool retrying; + + scoped_guard(mutex, &t->cd.tps.lock) { + retrying = pm->mux_retry_ms; + if (retrying) + pm->mux_since_ms = jiffies_to_msecs(jiffies) - + (t->now_ms - t->series_ms); + } + flush_delayed_work(dwork); + scoped_guard(mutex, &t->cd.tps.lock) { + if (!retrying && pm->mux_retry_ms) + t->series_ms = t->now_ms; + } +} + /* * Run the next work of the port, the one due first, as if its delay had passed. * Returns false if none is scheduled. @@ -394,7 +421,7 @@ static bool cd321x_mux_test_next(struct cd321x_mux_test *t) return false; t->now_ms += cd321x_mux_test_due_ms(next); - flush_delayed_work(next); + cd321x_mux_test_run(t, next); return true; } @@ -486,7 +513,7 @@ static void cd321x_mux_held_backs_off_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); if (!delayed_work_pending(&t->cd.resume_work)) continue; - delay = cd321x_mux_test_due_ms(&t->cd.resume_work); + delay = cd321x_mux_test_backoff_ms(t); if (!first) first = delay; KUNIT_EXPECT_LE(test, delay, (unsigned int)CD321X_MUX_RETRY_MAX_MS); @@ -707,9 +734,9 @@ static void cd321x_mux_j700_display_prepare_timeout_test(struct kunit *test) } /* - * A PHY that stays stuck is taken for a fault after some two minutes of - * retries. The port then waits for the next controller event, which starts - * the retries over. + * A PHY that stays stuck is taken for a fault two minutes after its first + * refusal, at the first retry from then on. The port then waits for the next + * controller event, which starts the retries over. */ static void cd321x_mux_held_gives_up_test(struct kunit *test) { @@ -718,14 +745,16 @@ static void cd321x_mux_held_gives_up_test(struct kunit *test) t->phy.held = true; cd321x_mux_test_attach(t); - for (i = 0; i < 4 * CD321X_MUX_RETRIES && cd321x_mux_test_next(t); i++) + while (t->now_ms < 2 * CD321X_MUX_HELD_MS && cd321x_mux_test_next(t)) ; KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.update_work)); KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); - KUNIT_EXPECT_EQ(test, strlen(t->phy.log), (size_t)CD321X_MUX_RETRIES + 1); + KUNIT_EXPECT_EQ(test, strlen(t->phy.log), (size_t)t->phy.refused); KUNIT_EXPECT_NULL(test, strpbrk(t->phy.log, "suU")); - KUNIT_EXPECT_GE(test, t->phy.last_refused_ms, 120000U); + KUNIT_EXPECT_GE(test, t->phy.last_refused_ms, t->series_ms + CD321X_MUX_HELD_MS); + KUNIT_EXPECT_LE(test, t->phy.last_refused_ms, + t->series_ms + CD321X_MUX_HELD_MS + CD321X_MUX_RETRY_MAX_MS + 1000); KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); KUNIT_EXPECT_FALSE(test, t->cd.state_valid); @@ -747,6 +776,92 @@ static void cd321x_mux_held_gives_up_test(struct kunit *test) cd321x_mux_test_expect_usb(test, t); } +/* + * Wait for the next refusal of the PHY, which needs a retry to be scheduled, + * injecting a controller event, as a partner that keeps renegotiating raises, + * into each backoff that started at least @gap_ms after the last event. + * Returns the number of events. + */ +static unsigned int cd321x_mux_test_refusal_events(struct kunit *test, + struct cd321x_mux_test *t, + unsigned int gap_ms, + unsigned int *last_event_ms) +{ + unsigned int refused = t->phy.refused, events = 0; + + while (t->phy.refused == refused) { + if (t->now_ms - *last_event_ms >= gap_ms && cd321x_mux_test_backoff_ms(t) && + cd321x_mux_test_phase(t) == CD321X_PM_REVALIDATE && + delayed_work_pending(&t->cd.resume_work)) { + cd321x_mux_test_event(t, APPLE_CD_REG_INT_DATA_STATUS_UPDATE); + *last_event_ms = t->now_ms; + events++; + } + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + } + return events; +} + +/* + * An event in every backoff brings each retry forward, to about a second after + * the last, but does not make the hold-up any shorter: the port still gives + * up only two minutes after the first refusal, not after so many retries. + */ +static void cd321x_mux_held_dense_events_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_j700_port(test); + unsigned int events = 0, last_event_ms = 0; + + t->phy.held = true; + cd321x_mux_test_attach(t); + cd321x_mux_test_refusal_events(test, t, UINT_MAX, &last_event_ms); + KUNIT_ASSERT_EQ(test, t->phy.refused, 1); + while (cd321x_mux_test_phase(t) != CD321X_PM_STALE) { + KUNIT_ASSERT_LT(test, t->now_ms, t->series_ms + 2 * CD321X_MUX_HELD_MS); + events += cd321x_mux_test_refusal_events(test, t, 0, &last_event_ms); + } + KUNIT_EXPECT_GE(test, t->phy.last_refused_ms, t->series_ms + CD321X_MUX_HELD_MS); + KUNIT_EXPECT_LE(test, t->phy.last_refused_ms, t->series_ms + CD321X_MUX_HELD_MS + 1000); + /* So the retries came about a second apart, each one brought forward. */ + KUNIT_EXPECT_GE(test, t->phy.refused, CD321X_MUX_HELD_MS / 1100); + KUNIT_EXPECT_EQ(test, events, t->phy.refused - 1); + KUNIT_EXPECT_NULL(test, strpbrk(t->phy.log, "suU")); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.update_work)); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0); +} + +/* + * When the PHY lets go after the last of such events, the next retry of the + * backoff applies the state, with no further event, provided the hold-up was + * shorter than two minutes, however many retries events brought forward. + */ +static void cd321x_mux_held_release_after_events_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_j700_port(test); + unsigned int last_event_ms = 0, released_ms; + + t->phy.held = true; + cd321x_mux_test_attach(t); + cd321x_mux_test_refusal_events(test, t, UINT_MAX, &last_event_ms); + while (t->now_ms < t->series_ms + CD321X_MUX_HELD_MS - 2 * CD321X_MUX_RETRY_MAX_MS) { + cd321x_mux_test_refusal_events(test, t, 1500, &last_event_ms); + KUNIT_ASSERT_NE(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + } + KUNIT_ASSERT_TRUE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_ASSERT_GT(test, t->phy.refused, 16); + + t->phy.held = false; + released_ms = t->now_ms; + memset(t->phy.log, 0, sizeof(t->phy.log)); + memset(t->display.log, 0, sizeof(t->display.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + KUNIT_EXPECT_STREQ(test, t->display.log, "su"); + cd321x_mux_test_expect_usb(test, t); + KUNIT_EXPECT_LE(test, t->now_ms, released_ms + CD321X_MUX_RETRY_MAX_MS + 1000); +} + /* * A series of retries that ends in another failure leaves no backoff behind: * the next event is read without delay, and the next hold-up starts over at @@ -797,6 +912,8 @@ static struct kunit_case cd321x_mux_cases[] = { KUNIT_CASE(cd321x_mux_j700_display_timeout_bounded_test), KUNIT_CASE(cd321x_mux_j700_display_prepare_timeout_test), KUNIT_CASE(cd321x_mux_held_gives_up_test), + KUNIT_CASE(cd321x_mux_held_dense_events_test), + KUNIT_CASE(cd321x_mux_held_release_after_events_test), KUNIT_CASE(cd321x_mux_held_then_other_error_test), {} }; diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index 2196ec4642721..fb8190341295f 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -520,13 +520,16 @@ static void cd321x_unacked_events_across_sleep_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); } +/* A millisecond clock that wraps half way through the time a hold-up is retried */ +#define TEST_WRAP_MS (0U - CD321X_MUX_HELD_MS / 2) + static void cd321x_held_mux_outlasts_budget_test(struct kunit *test) { struct cd321x_pm_state pm = { .phase = CD321X_PM_RUNNING }; - unsigned int i, delay = 0; + unsigned int i, delay = 0, now = TEST_WRAP_MS; for (i = 0; i < 4 * CD321X_RESUME_ATTEMPTS; i++) { - unsigned int next = cd321x_pm_mux_held(&pm); + unsigned int next = cd321x_pm_mux_held(&pm, now); /* Never given up, and not charged to the cable-state budget... */ KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_REVALIDATE); @@ -537,6 +540,7 @@ static void cd321x_held_mux_outlasts_budget_test(struct kunit *test) KUNIT_EXPECT_EQ(test, next, i ? min(2 * delay, CD321X_MUX_RETRY_MAX_MS) : CD321X_MUX_RETRY_MS); delay = next; + now += delay; /* An event brings the retry forward, but not a read already under way. */ KUNIT_EXPECT_TRUE(test, cd321x_pm_mux_event(&pm)); KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); @@ -565,10 +569,11 @@ static void cd321x_held_mux_across_sleep_test(struct kunit *test) { struct cd321x_pm_state pm = { .phase = CD321X_PM_RUNNING }; - cd321x_pm_mux_held(&pm); + cd321x_pm_mux_held(&pm, 0); KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); cd321x_pm_snapshot_ready(&pm); - KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), 2 * CD321X_MUX_RETRY_MS); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm, CD321X_MUX_RETRY_MS), + 2 * CD321X_MUX_RETRY_MS); /* The resume read takes the retry over, and a new hold-up starts afresh. */ cd321x_pm_prepare(&pm); @@ -578,7 +583,8 @@ static void cd321x_held_mux_across_sleep_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); cd321x_pm_snapshot_ready(&pm); KUNIT_EXPECT_TRUE(test, pm.force_reconnect); - KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), CD321X_MUX_RETRY_MS); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm, CD321X_MUX_HELD_MS), + CD321X_MUX_RETRY_MS); /* Removal ends it. */ cd321x_pm_remove(&pm); @@ -586,24 +592,33 @@ static void cd321x_held_mux_across_sleep_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); } +/* A failure of a held mux, which must be retried after a delay */ +static void cd321x_held_mux_retried(struct kunit *test, struct cd321x_pm_state *pm, + unsigned int now) +{ + KUNIT_EXPECT_GT(test, cd321x_pm_mux_held(pm, now), 0); + KUNIT_EXPECT_EQ(test, pm->phase, CD321X_PM_REVALIDATE); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(pm)); + cd321x_pm_snapshot_ready(pm); +} + +/* A hold-up is retried for CD321X_MUX_HELD_MS from its first failure... */ static void cd321x_held_mux_gives_up_test(struct kunit *test) { struct cd321x_pm_state pm = { .phase = CD321X_PM_RUNNING }; - unsigned int i, total = 0; + unsigned int now; - for (i = 0; i < CD321X_MUX_RETRIES; i++) { - unsigned int delay = cd321x_pm_mux_held(&pm); + for (now = TEST_WRAP_MS; now - TEST_WRAP_MS < CD321X_MUX_HELD_MS; + now += CD321X_MUX_RETRY_MAX_MS) + cd321x_held_mux_retried(test, &pm, now); + cd321x_held_mux_retried(test, &pm, TEST_WRAP_MS + CD321X_MUX_HELD_MS - 1); + KUNIT_EXPECT_LT(test, pm.mux_retries, CD321X_MUX_RETRIES); - KUNIT_EXPECT_GT(test, delay, 0); - total += delay; - KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); - cd321x_pm_snapshot_ready(&pm); - } - /* Some two minutes on, a hold-up is taken for a fault... */ - KUNIT_EXPECT_GE(test, total, 120000U); - KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), 0); + /* ...and is then taken for a fault... */ + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm, TEST_WRAP_MS + CD321X_MUX_HELD_MS), 0); KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); KUNIT_EXPECT_EQ(test, pm.mux_retry_ms, 0); + KUNIT_EXPECT_EQ(test, pm.mux_retries, 0); KUNIT_EXPECT_FALSE(test, cd321x_pm_mux_event(&pm)); KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); @@ -612,8 +627,40 @@ static void cd321x_held_mux_gives_up_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, pm.force_reconnect); KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); cd321x_pm_snapshot_ready(&pm); - KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), CD321X_MUX_RETRY_MS); + now = TEST_WRAP_MS + 2 * CD321X_MUX_HELD_MS; + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm, now), CD321X_MUX_RETRY_MS); KUNIT_EXPECT_EQ(test, pm.mux_retries, 1); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + cd321x_held_mux_retried(test, &pm, now + CD321X_MUX_HELD_MS - 1); +} + +/* + * ...however many retries events bring forward: they come no closer together + * than the two debounces, about a second. + */ +static void cd321x_held_mux_brought_forward_test(struct kunit *test) +{ + struct cd321x_pm_state pm = { .phase = CD321X_PM_RUNNING }; + unsigned int i; + + for (i = 0; i < CD321X_MUX_HELD_MS / 1000; i++) + cd321x_held_mux_retried(test, &pm, TEST_WRAP_MS + i * 1000); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm, TEST_WRAP_MS + i * 1000), 0); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); +} + +/* Should retries ever come faster, their number still bounds the warnings. */ +static void cd321x_held_mux_retries_bounded_test(struct kunit *test) +{ + struct cd321x_pm_state pm = { .phase = CD321X_PM_RUNNING }; + unsigned int i; + + for (i = 0; i < CD321X_MUX_RETRIES; i++) + cd321x_held_mux_retried(test, &pm, TEST_WRAP_MS); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm, TEST_WRAP_MS), 0); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, pm.mux_retries, 0); } static void cd321x_held_mux_other_failure_test(struct kunit *test) @@ -622,7 +669,7 @@ static void cd321x_held_mux_other_failure_test(struct kunit *test) unsigned int i; for (i = 0; i < 4; i++) { - cd321x_pm_mux_held(&pm); + cd321x_pm_mux_held(&pm, i * CD321X_MUX_RETRY_MS); KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); cd321x_pm_snapshot_ready(&pm); } @@ -647,7 +694,7 @@ static void cd321x_held_mux_other_failure_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, cd321x_pm_event(&pm)); KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); cd321x_pm_snapshot_ready(&pm); - KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm), CD321X_MUX_RETRY_MS); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm, CD321X_MUX_HELD_MS), CD321X_MUX_RETRY_MS); cd321x_pm_remove(&pm); KUNIT_EXPECT_EQ(test, pm.mux_retry_ms, 0); KUNIT_EXPECT_EQ(test, pm.mux_retries, 0); @@ -695,6 +742,8 @@ static struct kunit_case cd321x_pm_cases[] = { KUNIT_CASE(cd321x_held_mux_outlasts_budget_test), KUNIT_CASE(cd321x_held_mux_across_sleep_test), KUNIT_CASE(cd321x_held_mux_gives_up_test), + KUNIT_CASE(cd321x_held_mux_brought_forward_test), + KUNIT_CASE(cd321x_held_mux_retries_bounded_test), KUNIT_CASE(cd321x_held_mux_other_failure_test), KUNIT_CASE(cd321x_display_taken_test), {} diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h index 1a3b5a4b659e3..a178c315869d4 100644 --- a/drivers/usb/typec/tipd/cd321x-pm.h +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -13,8 +13,15 @@ #define CD321X_ACK_RETRY_MAX_MS 30000 #define CD321X_MUX_RETRY_MS 500 #define CD321X_MUX_RETRY_MAX_MS 10000 -/* About two minutes of backoff, twenty times the longest hold-up seen */ -#define CD321X_MUX_RETRIES 16 +/* Two minutes, over twenty times the longest hold-up seen */ +#define CD321X_MUX_HELD_MS 120000 +/* + * Ordinary controller events bring retries about a second apart, with a + * debounce for the read and another for the update. Tunnel notifications skip + * the read debounce, but the 500 ms update debounce alone keeps this count + * from ending a series before CD321X_MUX_HELD_MS. It only bounds its warnings. + */ +#define CD321X_MUX_RETRIES 240 enum cd321x_pm_phase { CD321X_PM_RUNNING, @@ -37,6 +44,8 @@ struct cd321x_pm_state { unsigned int mux_retry_ms; /* The retries of that change so far, up to CD321X_MUX_RETRIES. */ unsigned int mux_retries; + /* When the change was first held up, in milliseconds. */ + unsigned int mux_since_ms; bool force_reconnect; }; @@ -75,6 +84,7 @@ static inline bool cd321x_pm_mux_done(struct cd321x_pm_state *pm) pm->mux_retry_ms = 0; pm->mux_retries = 0; + pm->mux_since_ms = 0; return retried; } @@ -212,15 +222,22 @@ static inline bool cd321x_pm_acked(struct cd321x_pm_state *pm) * retry forward, see cd321x_pm_mux_event(); sleep cancels it and the resume * read takes it over; removal or any other failure ends it. * - * A hold-up that outlasts CD321X_MUX_RETRIES retries is taken for a fault, as - * the provider cannot tell its timeouts apart: the retries end as the budget - * does, and only a later event starts over. Returns the delay in milliseconds, - * or 0 once the retries have ended. + * A hold-up that outlasts CD321X_MUX_HELD_MS is taken for a fault, as the + * provider cannot tell its timeouts apart: the retries end as the budget does, + * and only a later event starts over. That limit is on time, counted from the + * first failure of the series, as events that bring retries forward do not + * make the hold-up any shorter. @now_ms is the time of this failure, in + * milliseconds on a clock that may wrap. Returns the delay in milliseconds, or + * 0 once the retries have ended. */ -static inline unsigned int cd321x_pm_mux_held(struct cd321x_pm_state *pm) +static inline unsigned int cd321x_pm_mux_held(struct cd321x_pm_state *pm, + unsigned int now_ms) { pm->force_reconnect = true; - if (pm->mux_retries >= CD321X_MUX_RETRIES) { + if (!pm->mux_retry_ms) + pm->mux_since_ms = now_ms; + if (now_ms - pm->mux_since_ms >= CD321X_MUX_HELD_MS || + pm->mux_retries >= CD321X_MUX_RETRIES) { cd321x_pm_mux_done(pm); pm->phase = CD321X_PM_STALE; pm->attempts_left = 0; diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 022cfc732f9af..aeedb8f4656d9 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -789,20 +789,24 @@ static void cd321x_retry_revalidation(struct cd321x *cd321x, bool reconnect) static void cd321x_retry_held_mux(struct cd321x *cd321x) { - unsigned int delay; + unsigned int now_ms = jiffies_to_msecs(jiffies); + unsigned int retries, held_ms, delay; lockdep_assert_held(&cd321x->tps.lock); + /* For the warning, should the retries end here */ + retries = cd321x->pm.mux_retries; + held_ms = now_ms - cd321x->pm.mux_since_ms; cd321x->state_valid = false; if (!cd321x->pm.mux_retry_ms) dev_info(cd321x->tps.dev, "Type-C mux held up, retrying\n"); - delay = cd321x_pm_mux_held(&cd321x->pm); + delay = cd321x_pm_mux_held(&cd321x->pm, now_ms); if (delay) mod_delayed_work(system_freezable_wq, &cd321x->resume_work, msecs_to_jiffies(delay)); else dev_warn(cd321x->tps.dev, - "Type-C mux still held up after %u retries, waiting for the next event\n", - CD321X_MUX_RETRIES); + "Type-C mux still held up after %u retries in %u s, waiting for the next event\n", + retries, held_ms / 1000); } static void cd321x_update_done(struct cd321x *cd321x) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 1d9313a06ce12..74156455728b4 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -406,6 +406,10 @@ def function(file, name): shim = shim.replace(old, 'static int cd321x_typec_update_mode(struct tps6598x *t, struct cd321x_status *s, bool *busy,\n' ' bool *held)\n' '{ (void)t; (void)s; *busy = mode_error == -EBUSY; *held = false; return mode_error; }') + # How long a held mux has been retried is measured on the driver's clock. + if 'jiffies_to_msecs(jiffies)' in funcs: + shim += ('\nstatic unsigned long jiffies;\n' + 'static unsigned jiffies_to_msecs(unsigned long t) { return t; }\n') source = prefix + pm + shim + dp + funcs + tests return source @@ -576,6 +580,10 @@ def swap(old, new): printf("%d assertions, %d failures\n", checks, failures); return !!failures; } ''' + # unless the update work already needed the clock + clock = 'static unsigned jiffies_to_msecs(unsigned long t) { return t; }\n' + if clock in unit: + firmware = firmware.replace(clock, '') return unit + firmware + raw + '\n' + tests def dropped(root): diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 557162b627907..f704af0a4cc28 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -615,11 +615,17 @@ def function(text, name): if 'cd321x_pm_mux_held(' in pm: lost = ' puts("PASS actual lost events reopen a stale port; unacknowledged events keep their budget with capped backoff");\n' assert controls.count(lost) == 1 - controls = controls.replace(lost, lost + r''' struct cd321x_pm_state held={0}; + controls = controls.replace(lost, lost + r'''/* The time of the failure, once the limit of the retries is on time */ +#ifdef CD321X_MUX_HELD_MS +#define MUX_HELD(pm,ms) cd321x_pm_mux_held(pm,ms) +#else +#define MUX_HELD(pm,ms) ((void)(ms),cd321x_pm_mux_held(pm)) +#endif + struct cd321x_pm_state held={0}; held.phase=CD321X_PM_RUNNING; - for(unsigned int i=0,delay=0;i<4*CD321X_RESUME_ATTEMPTS;i++){ + for(unsigned int i=0,delay=0,now=0;i<4*CD321X_RESUME_ATTEMPTS;i++,now+=delay){ unsigned int expected=i?(2*delaylock&&!pm_refs&&!link_pm_refs); + atomic_store(&handoff_active,false); + } + host_locked=false;pthread_mutex_unlock(&host_mutex); + if(!remover&&remove_target){ + /* Force teardown into the gap before the controller lock is released. */ + pthread_mutex_lock(&schedule_mutex); + while(!removal_started)pthread_cond_wait(&schedule_cond,&schedule_mutex); + pthread_mutex_unlock(&schedule_mutex); + } + }else{ + assert(d==&controller.dev&&controller_locked); + if(!remover&&!remove_target){ + assert(!root_locked&&!link_locked&&!pm_phy->lock&&!pm_refs&&!link_pm_refs); + atomic_store(&handoff_active,false); + } + controller_locked=false;pthread_mutex_unlock(&controller_mutex); + } }''' unit=unit.replace(old,new) # The scheduler injection is a child-lookup interleaving, as in the review, or From fb88c6bad70389dd7c1024b4cd012f69dd24bf63 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 16:28:22 +1100 Subject: [PATCH 1255/1579] soc: apple: sep: Add an opt-in guard for other OSes' xART records The xART gigalocker on the iBoot system container is shared by every OS installed on the Mac. The enclave keeps each OS's anti-replay state in session records keyed by a random session id, and asks the host to write and delete them. Nothing in a record says which OS wrote it, so the driver serves any request the enclave makes, including one that rewrites or deletes a record another OS created. That happened on an M1 Pro (J316s) that booted with no OS identity: the gigalocker went from 13 to 14 records, but two session records carried new revisions, so at least one record that predated the boot was rewritten or deleted. Which OS owned it cannot be told from the record. Add xart_guard. With it set, the driver lists every session record the enclave creates while Linux runs in /var/lib/aurora-sep-xart-owned.bin, and refuses to rewrite or delete any session record that was in the gigalocker when the driver opened it and is not on that list. Level 2 also refuses writes to the two shared root records. A refused request fails the same way a read-only gigalocker does. The list entry is made durable before the record is written, so a crash can leave only an entry for a record that was never written. The default stays off. Machines that already hold Linux records from before the list existed would otherwise lose write access to their own records. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sep.rs | 11 +- drivers/soc/apple/xart_guard.rs | 272 ++++++++++++++++++++++++++++++++ drivers/soc/apple/xart_store.rs | 29 +++- 3 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 drivers/soc/apple/xart_guard.rs diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index ba16606f0dfed..6a5c54c0495bb 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -38,6 +38,7 @@ mod transfer; mod trusted; mod xarm; mod xart_apfs; +mod xart_guard; mod xart_store; use kernel::{ @@ -804,8 +805,12 @@ impl SepData { let xart_writes = *module_parameters::xart_writes.value() != 0; let xart_start = *module_parameters::xart_start_sector.value(); + let xart_guard = *module_parameters::xart_guard.value(); let store = match xart_store::Store::open_owner(xart_writes, xart_start) { - Ok(store) => { + Ok(mut store) => { + if xart_guard != 0 && store.writes_enabled() { + store.set_guard(xart_guard)?; + } let (slots, records, revision, malformed, duplicates, repaired, writable) = store.summary(); let base = store.base(); @@ -2661,6 +2666,10 @@ module! { default: 1, description: "Create the Linux identity keybag when none exists; set to 0 to disable first-boot provisioning; requires xart_writes=1", }, + xart_guard: u8 { + default: 0, + description: "1: refuse the enclave's writes and deletes of xART session records this Linux install did not create; 2: also refuse writes to the shared root records", + }, os_uuid_hi: u64 { default: 0, description: "High 64 bits of an explicit xART OS UUID", diff --git a/drivers/soc/apple/xart_guard.rs b/drivers/soc/apple/xart_guard.rs new file mode 100644 index 0000000000000..7cf3ea5f52d06 --- /dev/null +++ b/drivers/soc/apple/xart_guard.rs @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Ryan Murray + +//! Keeps the enclave's xART writes to the records this Linux install created. +//! +//! The gigalocker is shared by every OS installed on the Mac. Each OS keeps +//! its anti-replay state in session records keyed by a random session id. +//! Nothing in a record says which OS wrote it, so the driver cannot tell +//! another OS's record from its own by looking at it. +//! +//! With `xart_guard` set, the driver keeps a list of the session records the +//! enclave created while Linux was running, in a host file. It then refuses +//! to rewrite or delete any session record that was in the gigalocker when the +//! driver opened it and is not on that list. With `xart_guard=2` it also +//! refuses writes to the two shared root records. A refused write fails the +//! enclave's request, the same way a read-only gigalocker does: Touch ID may +//! then be unavailable, but no other OS's record changes. +//! +//! The list entry is made durable before the enclave's write reaches the +//! gigalocker. A crash between the two leaves an entry for a record that was +//! never written, which is harmless. It can never leave a Linux record that +//! the next boot treats as another OS's. + +use crate::shim::StoreFile; +use crate::xart_store::{crc32_ieee, Key}; +use kernel::prelude::*; + +/// Host file listing the session records Linux created. +pub(crate) const OWNED_PATH: &CStr = c"/var/lib/aurora-sep-xart-owned.bin"; + +const ENTRY_SIZE: usize = 32; +const ENTRY_MAGIC: &[u8; 4] = b"AXGO"; +/// Bounds the file the driver reads at probe. The gigalocker has at most a +/// few hundred slots, so a longer list is damaged. +const MAX_ENTRIES: usize = 4096; +/// Refusals are logged individually up to this count, then only counted. +const LOGGED_REFUSALS: usize = 8; + +/// What the guard does with one enclave write or delete. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Verdict { + /// The record is Linux's own, or a shared root at level 1. + Allow, + /// A session record that did not exist when the gigalocker was opened: + /// list it as Linux's before writing it. + Adopt, + /// A record that predates this boot and that Linux did not create. + Refuse, +} + +/// Decides one request. `level` is the `xart_guard` parameter, `existed` +/// whether the record was in the gigalocker when the driver opened it, and +/// `owned` whether it is on Linux's list. +pub(crate) fn decide(level: u8, key: &Key, existed: bool, owned: bool) -> Verdict { + if key.kind <= 2 { + return if level >= 2 { Verdict::Refuse } else { Verdict::Allow }; + } + if owned { + Verdict::Allow + } else if existed { + Verdict::Refuse + } else { + Verdict::Adopt + } +} + +/// Encodes one list entry. +pub(crate) fn encode(key: &Key) -> [u8; ENTRY_SIZE] { + let mut entry = [0u8; ENTRY_SIZE]; + entry[..4].copy_from_slice(ENTRY_MAGIC); + entry[4] = key.kind; + entry[8..24].copy_from_slice(&key.uuid); + let crc = crc32_ieee(&entry[..24]); + entry[24..28].copy_from_slice(&crc.to_le_bytes()); + entry +} + +/// Decodes one list entry; `None` for a torn or foreign one. +pub(crate) fn decode(entry: &[u8]) -> Option { + if entry.len() != ENTRY_SIZE || &entry[..4] != ENTRY_MAGIC || entry[5..8] != [0; 3] { + return None; + } + let crc = u32::from_le_bytes([entry[24], entry[25], entry[26], entry[27]]); + if crc != crc32_ieee(&entry[..24]) || !(3..=4).contains(&entry[4]) { + return None; + } + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&entry[8..24]); + Some(Key::new(entry[4], uuid)) +} + +pub(crate) struct Guard { + level: u8, + /// `None` when the list could not be opened: nothing can be adopted, so + /// every session write that is not already listed is refused. + file: Option, + /// Byte offset of the next entry. + end: u64, + owned: KVec, + existed: KVec, + refused: usize, +} + +impl Guard { + /// Opens Linux's list and records which keys `existing` holds now. Never + /// fails: a list that cannot be read protects every existing record. + pub(crate) fn open(level: u8, existing: KVec) -> Guard { + let mut guard = Guard { + level, + file: None, + end: 0, + owned: KVec::new(), + existed: existing, + refused: 0, + }; + match StoreFile::open_host(OWNED_PATH) { + Ok((file, _created)) => { + if let Err(e) = guard.load(&file) { + pr_warn!( + "xART guard: the list of Linux's records is unreadable ({:?}); every existing session record is protected and none can be adopted\n", + e + ); + } else { + guard.file = Some(file); + } + } + Err(e) => pr_warn!( + "xART guard: the list of Linux's records could not be opened ({:?}); every existing session record is protected and none can be adopted\n", + e + ), + } + let sessions = guard.existed.iter().filter(|k| k.kind > 2).count(); + let own = guard + .existed + .iter() + .filter(|k| k.kind > 2 && guard.owned.contains(k)) + .count(); + let roots = if level >= 2 { + ", and so are the shared roots" + } else { + "" + }; + pr_info!( + "xART guard level {}: {} session records exist, {} created by Linux; the other {} are protected{}\n", + level, + sessions, + own, + sessions - own, + roots + ); + guard + } + + fn load(&mut self, file: &StoreFile) -> Result<()> { + let size = file.size()?; + let whole = (size as usize / ENTRY_SIZE).min(MAX_ENTRIES + 1); + if whole > MAX_ENTRIES { + return Err(EFBIG); + } + let mut torn = 0usize; + let mut entry = [0u8; ENTRY_SIZE]; + for i in 0..whole { + file.read_exact((i * ENTRY_SIZE) as u64, &mut entry)?; + match decode(&entry) { + Some(key) if !self.owned.contains(&key) => self.owned.push(key, GFP_KERNEL)?, + Some(_) => {} + None => torn += 1, + } + } + if torn != 0 { + pr_warn!("xART guard: ignored {} unreadable list entries\n", torn); + } + // A torn tail entry is overwritten by the next adoption. + self.end = (whole * ENTRY_SIZE) as u64; + Ok(()) + } + + fn adopt(&mut self, key: &Key) -> Result<()> { + let file = self.file.as_ref().ok_or(ENOENT)?; + if self.owned.len() >= MAX_ENTRIES { + return Err(ENOSPC); + } + file.write_all(self.end, &encode(key))?; + file.sync()?; + self.end += ENTRY_SIZE as u64; + self.owned.push(*key, GFP_KERNEL)?; + Ok(()) + } + + /// Checks one enclave write (`write`) or delete before it reaches the + /// gigalocker. An error refuses the request. + pub(crate) fn check(&mut self, key: &Key, write: bool) -> Result<()> { + let existed = self.existed.contains(key); + let owned = self.owned.contains(key); + match decide(self.level, key, existed, owned) { + Verdict::Allow => Ok(()), + Verdict::Adopt if !write => Ok(()), + Verdict::Adopt => self.adopt(key).inspect_err(|e| { + pr_warn!( + "xART guard: could not list a new kind-{} record as Linux's ({:?}); refusing to write it\n", + key.kind, + e + ) + }), + Verdict::Refuse => { + self.refused += 1; + if self.refused <= LOGGED_REFUSALS { + let action = if write { "rewrite" } else { "delete" }; + pr_warn!( + "xART guard: refused to {} a kind-{} record this Linux install did not create\n", + action, + key.kind + ); + } + Err(EPERM) + } + } + } +} + +#[kernel::macros::kunit_tests(apple_sep_xart_guard)] +mod tests { + use super::*; + + fn session(byte: u8) -> Key { + Key::new(4, [byte; 16]) + } + + #[test] + fn another_os_record_is_refused() { + assert_eq!(decide(1, &session(1), true, false), Verdict::Refuse); + assert_eq!(decide(2, &Key::new(3, [1; 16]), true, false), Verdict::Refuse); + } + + #[test] + fn linux_records_stay_writable() { + assert_eq!(decide(1, &session(2), true, true), Verdict::Allow); + assert_eq!(decide(1, &session(2), false, true), Verdict::Allow); + } + + #[test] + fn a_new_record_is_adopted() { + assert_eq!(decide(1, &session(3), false, false), Verdict::Adopt); + } + + #[test] + fn roots_follow_the_level() { + assert_eq!(decide(1, &Key::root(1), true, false), Verdict::Allow); + assert_eq!(decide(1, &Key::root(2), true, false), Verdict::Allow); + assert_eq!(decide(2, &Key::root(1), true, false), Verdict::Refuse); + assert_eq!(decide(2, &Key::root(2), false, false), Verdict::Refuse); + } + + #[test] + fn entries_round_trip() { + let key = Key::new(3, [0x5a; 16]); + assert_eq!(decode(&encode(&key)), Some(key)); + } + + #[test] + fn damaged_entries_are_ignored() { + let key = session(7); + let mut entry = encode(&key); + entry[9] ^= 1; + assert_eq!(decode(&entry), None); + let mut root = encode(&Key::root(1)); + let crc = crc32_ieee(&root[..24]); + root[24..28].copy_from_slice(&crc.to_le_bytes()); + assert_eq!(decode(&root), None); + assert_eq!(decode(&encode(&key)[..31]), None); + } +} diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs index e47275ccb62d5..f083c42a5881b 100644 --- a/drivers/soc/apple/xart_store.rs +++ b/drivers/soc/apple/xart_store.rs @@ -33,7 +33,7 @@ const CRC: usize = 0x16; const REVISION: usize = 0x1a; const PAYLOAD: usize = HEADER_SIZE; -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Key { pub(crate) kind: u8, pub(crate) uuid: [u8; 16], @@ -86,6 +86,8 @@ pub(crate) struct Store { malformed_records: usize, duplicate_records: usize, repaired_records: usize, + /// Set with `xart_guard`: refuses writes to records Linux did not create. + guard: Option, } fn le32(bytes: &[u8], off: usize) -> u32 { @@ -180,6 +182,7 @@ impl Store { malformed_records: 0, duplicate_records: 0, repaired_records: 0, + guard: None, }; store.scan()?; // Serving an empty or unrelated 6 MiB mapping is the failure mode that @@ -348,6 +351,10 @@ impl Store { return Err(EINVAL); } + if let Some(guard) = self.guard.as_mut() { + guard.check(key, true)?; + } + let old = self.find(key); // A new key skips the first free slot; a replacement reuses it. let fresh = self.find_free(usize::from(old.is_none())).ok_or(ENOSPC)?; @@ -395,6 +402,9 @@ impl Store { if !self.writes_enabled { return Err(EROFS); } + if let Some(guard) = self.guard.as_mut() { + guard.check(key, false)?; + } self.delete_slot(idx)?; self.slots[idx] = Slot::FREE; self.valid_records = self.slots.iter().filter(|slot| slot.used).count(); @@ -407,6 +417,23 @@ impl Store { self.writes_enabled } + /// Every live record's key, for the guard's view of what predates it. + pub(crate) fn live_keys(&self) -> Result> { + let mut keys = KVec::new(); + for slot in self.slots.iter().filter(|slot| slot.used) { + keys.push(slot.key, GFP_KERNEL)?; + } + Ok(keys) + } + + /// From now on, refuses the enclave's writes to records Linux did not + /// create. See [`crate::xart_guard`]. + pub(crate) fn set_guard(&mut self, level: u8) -> Result<()> { + let keys = self.live_keys()?; + self.guard = Some(crate::xart_guard::Guard::open(level, keys)); + Ok(()) + } + /// Byte offset of the APFS-resolved gigalocker extent in the container. pub(crate) fn base(&self) -> u64 { self.base From c4f2cd9eb04f80e296f3626c6bc3e666ef3be238 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 16:47:16 +1100 Subject: [PATCH 1256/1579] soc: apple: sep: Take the SEP DART DMA window from the platform profile On warm-attach platforms the driver enables the SEP DART at module load and, when the device tree does not describe the DMA window, programs one itself: IOVAs 0..4 GiB offset by BIT(40). That high alias is right for T6020 and T8140, but the T8112 SEP DART has a 32-bit input aperture (0x4000 + 0xffff0000, as the T8112 profile already records) and no high alias, so the fallback programs a window the hardware cannot reach. Move the window into the platform profile. T8112 gets no offset and the 0x4000..4 GiB range; every other profile keeps the BIT(40) window. A window the device tree gives is still used unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/dt.rs | 18 ++++++++---------- drivers/soc/apple/profile.rs | 32 ++++++++++++++++++++++++++++++++ drivers/soc/apple/sep.rs | 2 +- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/drivers/soc/apple/dt.rs b/drivers/soc/apple/dt.rs index 51c83d1249cdf..0eebd735f6685 100644 --- a/drivers/soc/apple/dt.rs +++ b/drivers/soc/apple/dt.rs @@ -15,13 +15,10 @@ const IOMMUS_PROP: &CStr = c"iommus"; const STATUS_PROP: &CStr = c"status"; const STATUS_OKAY: &CStr = c"okay"; -// SEP DMA is offset BIT(40) above the DART input; one 64-bit value = two cells. +// The SEP DART's DMA window; each 64-bit value is two cells. The values +// come from the platform profile when the device tree does not give them. const DMA_OFFSET_PROP: &CStr = c"apple,dma-offset"; -// [0x100, 0] = BIT(40); [0x1, 0] would be BIT(32) and miss the page tables. -const DMA_OFFSET_CELLS: [u32; 2] = [0x100, 0x0]; - const DMA_RANGE_PROP: &CStr = c"apple,dma-range"; -const DMA_RANGE_CELLS: [u32; 4] = [0, 0, 1, 0]; const REGISTERED_PROP: &CStr = c"apple,sep-shmem-registered-iova"; const CHOSEN_PATH: &CStr = c"/chosen"; @@ -188,7 +185,7 @@ pub(crate) fn preboot_uuid() -> Option<[u8; 16]> { Some(uuid.b) } -pub(crate) fn enable_sep_and_dart() -> Result<()> { +pub(crate) fn enable_sep_and_dart(window: &crate::profile::DartWindow) -> Result<()> { let sep = sep_node().ok_or_else(|| { pr_err!( "apple_sep: no device-tree node with compatible '{}'\n", @@ -218,7 +215,7 @@ pub(crate) fn enable_sep_and_dart() -> Result<()> { if dart_available && (need_offset || need_range) { pr_err!( - "apple_sep: SEP DART already enabled but missing apple,dma-offset/apple,dma-range; the BIT(40) offset applies only at DART probe. Reboot and load this module first.\n" + "apple_sep: SEP DART already enabled but missing apple,dma-offset/apple,dma-range; the DMA window applies only at DART probe. Reboot and load this module first.\n" ); return Err(EBUSY); } @@ -228,11 +225,11 @@ pub(crate) fn enable_sep_and_dart() -> Result<()> { let mut queued = false; // Properties before status: the DART probes inside of_changeset_apply(). if need_offset { - queue_u32_array(cs, &dart, DMA_OFFSET_PROP, &DMA_OFFSET_CELLS)?; + queue_u32_array(cs, &dart, DMA_OFFSET_PROP, &window.offset)?; queued = true; } if need_range { - queue_u32_array(cs, &dart, DMA_RANGE_PROP, &DMA_RANGE_CELLS)?; + queue_u32_array(cs, &dart, DMA_RANGE_PROP, &window.range)?; queued = true; } if !dart_available { @@ -242,7 +239,8 @@ pub(crate) fn enable_sep_and_dart() -> Result<()> { Ok(queued) })?; pr_info!( - "apple_sep: SEP DART changeset applied (offset BIT(40), range 0..4GiB, status okay)\n" + "apple_sep: SEP DART changeset applied ({}, status okay)\n", + window.name ); } else { pr_info!("apple_sep: SEP DART already enabled and configured\n"); diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 85ed9c4b03cec..13bc03c557412 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -122,6 +122,31 @@ pub(crate) enum KeyStore { Variant5, } +/// The SEP DART DMA window a warm-attach profile programs when the device +/// tree does not describe it, as `apple,dma-offset` and `apple,dma-range` +/// cells. +pub(crate) struct DartWindow { + pub(crate) name: &'static str, + pub(crate) offset: [u32; 2], + pub(crate) range: [u32; 4], +} + +/// IOVAs 0..4 GiB, offset by BIT(40): `[0x100, 0]`. `[0x1, 0]` would be +/// BIT(32) and miss the page tables. +const DART_HIGH_ALIAS: DartWindow = DartWindow { + name: "offset BIT(40), range 0..4 GiB", + offset: [0x100, 0x0], + range: [0, 0, 1, 0], +}; + +/// The T8112 SEP DART has a 32-bit input aperture, 0x4000 + 0xffff0000, and +/// no high alias. +const DART_T8112: DartWindow = DartWindow { + name: "no offset, range 0x4000..4 GiB", + offset: [0x0, 0x0], + range: [0x0, 0x4000, 0x0, 0xffff_0000], +}; + pub(crate) struct PlatformProfile { pub(crate) name: &'static str, /// Capacity of the boot shared-memory window. The layout is still checked @@ -138,6 +163,8 @@ pub(crate) struct PlatformProfile { pub(crate) dart_range_required: bool, /// Whether the SEP DART IOVA window needs a DMA mask wider than 32 bits. pub(crate) wide_dma_mask: bool, + /// The SEP DART DMA window programmed on warm attach (see [`DartWindow`]). + pub(crate) dart_window: &'static DartWindow, /// Reserved-memory region holding the SEP firmware image (cold-boot path). pub(crate) firmware_region: &'static CStr, /// Identity keybag CREATE_KEYBAG field encoding (per-SoC; see [`KeybagCreate`]). @@ -168,6 +195,7 @@ const T8103: PlatformProfile = PlatformProfile { }, dart_range_required: false, wide_dma_mask: false, + dart_window: &DART_HIGH_ALIAS, firmware_region: c"sepfw", // The 13.5 key store creates an identity bag with // version 2, type 0x400000, parent -1 (it accepts only -1 or <= -10). @@ -207,6 +235,7 @@ const T6000: PlatformProfile = PlatformProfile { }, dart_range_required: false, wide_dma_mask: false, + dart_window: &DART_HIGH_ALIAS, firmware_region: c"sepfw", keybag_create: KeybagCreate { variant: T8103.keybag_create.variant, @@ -239,6 +268,7 @@ const T6020: PlatformProfile = PlatformProfile { }, dart_range_required: true, wide_dma_mask: false, + dart_window: &DART_HIGH_ALIAS, firmware_region: c"sepfw", // Proven encoding: the lenient enclave takes the variant in the first word. keybag_create: KeybagCreate { @@ -274,6 +304,7 @@ const T8140: PlatformProfile = PlatformProfile { }, dart_range_required: true, wide_dma_mask: true, + dart_window: &DART_HIGH_ALIAS, firmware_region: c"sepfw", // J700 hardware refused the strict encoding and accepted this T6020 form // (first verified on hardware 2026-09-19). @@ -313,6 +344,7 @@ const T8112: PlatformProfile = PlatformProfile { }, dart_range_required: true, wide_dma_mask: false, + dart_window: &DART_T8112, firmware_region: c"sepfw", // J415 accepted the 13.5 GET_CAPABILITIES/SET_ENV init but refused the // strict CREATE_KEYBAG (mailbox status -1, empty reply), as J700 did; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 6a5c54c0495bb..4cd52b18594af 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2628,7 +2628,7 @@ impl kernel::InPlaceModule for SepModule { match profile::detect() { Ok(p) => { if matches!(p.bootstrap, profile::Bootstrap::WarmRegister) { - dt::enable_sep_and_dart()?; + dt::enable_sep_and_dart(p.dart_window)?; } } // Without this the module loads, probes nothing and logs From d31f63179638b0d7279e801b9f617ce90cbc542f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Thu, 8 Oct 2026 00:55:45 -0700 Subject: [PATCH 1257/1579] arm64: dts: apple: bound T8112 SEP DMA below 4 GiB Describe the SEP DART zero-alias window in the shared T8112 device tree. Keep both devices disabled until SEP module initialization and preserve firmware-supplied DMA properties. This prevents the high-alias fallback from exceeding the 32-bit DART input aperture on J413 and J415. Add compiled board and production DART admission controls for malformed ranges, high aliases and low IOVA mappings to high physical pages. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> [ryan: DT portion only; dropped tools/aurora-sep/tests/test_t8112_sep_dma.py, which checks DART admission code this tree does not carry] (cherry picked from commit 3bc511372704583c03dbbc0f651c96cb2d3e2935) Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8112.dtsi | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index c61e78b0b360b..e81b83bdde991 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -1680,7 +1680,8 @@ /* * iBoot hands over a running SEP. Keep Linux off its DART at boot; - * apple_sep sets the DMA range and enables it, as on t602x. + * apple_sep enables it at module initialization. This DART has a + * 32-bit input aperture and does not use a high DMA alias. */ sep_dart: iommu@25d2c0000 { compatible = "apple,t8112-dart", "apple,t8110-dart"; @@ -1688,6 +1689,8 @@ #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; + apple,dma-offset = <0x0 0x0>; + apple,dma-range = <0x0 0x4000 0x0 0xffff0000>; status = "disabled"; }; From d29bc17e14fc33e36cefde25118c97866a4f4061 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 18 Oct 2026 11:03:30 +1100 Subject: [PATCH 1258/1579] soc: apple: sep: Test the xART guard's list of Linux's records The xART guard's KUnit suite covers its decisions and the entry format, but not the list itself: that an adopted record is listed before it is written, that the next boot reads it back and keeps the record writable, that a refusal lists nothing, that a torn last entry is overwritten, and that without a readable list nothing new can be adopted. Let the guard keep its list in anything that implements a small ListFile trait, the host StoreFile in the driver, and test those cases against a list held in memory. The driver's behaviour does not change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/xart_guard.rs | 179 ++++++++++++++++++++++++++++++-- 1 file changed, 172 insertions(+), 7 deletions(-) diff --git a/drivers/soc/apple/xart_guard.rs b/drivers/soc/apple/xart_guard.rs index 7cf3ea5f52d06..e3154bfa31be4 100644 --- a/drivers/soc/apple/xart_guard.rs +++ b/drivers/soc/apple/xart_guard.rs @@ -89,11 +89,37 @@ pub(crate) fn decode(entry: &[u8]) -> Option { Some(Key::new(entry[4], uuid)) } -pub(crate) struct Guard { +/// The file that holds Linux's list: the host file, or memory in the tests. +pub(crate) trait ListFile { + fn size(&self) -> Result; + fn read_exact(&self, off: u64, buf: &mut [u8]) -> Result<()>; + fn write_all(&self, off: u64, buf: &[u8]) -> Result<()>; + fn sync(&self) -> Result<()>; +} + +impl ListFile for StoreFile { + fn size(&self) -> Result { + StoreFile::size(self) + } + + fn read_exact(&self, off: u64, buf: &mut [u8]) -> Result<()> { + StoreFile::read_exact(self, off, buf) + } + + fn write_all(&self, off: u64, buf: &[u8]) -> Result<()> { + StoreFile::write_all(self, off, buf) + } + + fn sync(&self) -> Result<()> { + StoreFile::sync(self) + } +} + +pub(crate) struct Guard { level: u8, /// `None` when the list could not be opened: nothing can be adopted, so /// every session write that is not already listed is refused. - file: Option, + file: Option, /// Byte offset of the next entry. end: u64, owned: KVec, @@ -102,9 +128,21 @@ pub(crate) struct Guard { } impl Guard { - /// Opens Linux's list and records which keys `existing` holds now. Never - /// fails: a list that cannot be read protects every existing record. + /// Opens Linux's list in the host file and records which keys `existing` + /// holds now. Never fails: a list that cannot be read protects every + /// existing record. pub(crate) fn open(level: u8, existing: KVec) -> Guard { + Guard::with_file( + level, + existing, + StoreFile::open_host(OWNED_PATH).map(|(file, _created)| file), + ) + } +} + +impl Guard { + /// As [`Guard::open`], with the list in `file`. + pub(crate) fn with_file(level: u8, existing: KVec, file: Result) -> Guard { let mut guard = Guard { level, file: None, @@ -113,8 +151,8 @@ impl Guard { existed: existing, refused: 0, }; - match StoreFile::open_host(OWNED_PATH) { - Ok((file, _created)) => { + match file { + Ok(file) => { if let Err(e) = guard.load(&file) { pr_warn!( "xART guard: the list of Linux's records is unreadable ({:?}); every existing session record is protected and none can be adopted\n", @@ -151,7 +189,7 @@ impl Guard { guard } - fn load(&mut self, file: &StoreFile) -> Result<()> { + fn load(&mut self, file: &F) -> Result<()> { let size = file.size()?; let whole = (size as usize / ENTRY_SIZE).min(MAX_ENTRIES + 1); if whole > MAX_ENTRIES { @@ -221,6 +259,7 @@ impl Guard { #[kernel::macros::kunit_tests(apple_sep_xart_guard)] mod tests { use super::*; + use core::cell::RefCell; fn session(byte: u8) -> Key { Key::new(4, [byte; 16]) @@ -269,4 +308,130 @@ mod tests { assert_eq!(decode(&root), None); assert_eq!(decode(&encode(&key)[..31]), None); } + + /// A list file in memory, kept across the guards of successive boots. + struct MemFile<'a>(&'a RefCell>); + + impl ListFile for MemFile<'_> { + fn size(&self) -> Result { + Ok(self.0.borrow().len() as u64) + } + + fn read_exact(&self, off: u64, buf: &mut [u8]) -> Result<()> { + let bytes = self.0.borrow(); + let off = usize::try_from(off).map_err(|_| EINVAL)?; + let end = off.checked_add(buf.len()).ok_or(EINVAL)?; + buf.copy_from_slice(bytes.get(off..end).ok_or(EIO)?); + Ok(()) + } + + fn write_all(&self, off: u64, buf: &[u8]) -> Result<()> { + let mut bytes = self.0.borrow_mut(); + let off = usize::try_from(off).map_err(|_| EINVAL)?; + let end = off.checked_add(buf.len()).ok_or(EINVAL)?; + while bytes.len() < end { + bytes.push(0, GFP_KERNEL)?; + } + bytes[off..end].copy_from_slice(buf); + Ok(()) + } + + fn sync(&self) -> Result<()> { + Ok(()) + } + } + + fn keys(list: &[Key]) -> Result> { + let mut keys = KVec::new(); + for key in list { + keys.push(*key, GFP_KERNEL)?; + } + Ok(keys) + } + + fn listed(bytes: &RefCell>) -> KVec> { + let mut entries = KVec::new(); + for entry in bytes.borrow().chunks(ENTRY_SIZE) { + if entries.push(decode(entry), GFP_KERNEL).is_err() { + break; + } + } + entries + } + + #[test] + fn a_guard_refuses_another_os_record() -> Result<()> { + let bytes = RefCell::new(KVec::new()); + let foreign = session(1); + let mut guard = Guard::with_file(1, keys(&[foreign])?, Ok(MemFile(&bytes))); + assert_eq!(guard.check(&foreign, true), Err(EPERM)); + assert_eq!(guard.check(&foreign, false), Err(EPERM)); + // A refusal lists nothing. + assert_eq!(bytes.borrow().len(), 0); + Ok(()) + } + + #[test] + fn a_guard_allows_a_listed_record() -> Result<()> { + let own = session(2); + let mut list = KVec::new(); + list.extend_from_slice(&encode(&own), GFP_KERNEL)?; + let bytes = RefCell::new(list); + let mut guard = Guard::with_file(1, keys(&[own])?, Ok(MemFile(&bytes))); + assert_eq!(guard.check(&own, true), Ok(())); + assert_eq!(guard.check(&own, false), Ok(())); + // An entry that is already listed is not written again. + assert_eq!(bytes.borrow().len(), ENTRY_SIZE); + Ok(()) + } + + #[test] + fn the_owned_list_survives_a_reboot() -> Result<()> { + let bytes = RefCell::new(KVec::new()); + let foreign = session(1); + let created = session(3); + + // First boot: the enclave creates a record, which is listed first. + let mut guard = Guard::with_file(1, keys(&[foreign])?, Ok(MemFile(&bytes))); + assert_eq!(guard.check(&created, true), Ok(())); + assert_eq!(listed(&bytes).as_slice(), &[Some(created)]); + drop(guard); + + // Next boot: both records predate the guard; only Linux's is writable. + let mut guard = Guard::with_file(1, keys(&[foreign, created])?, Ok(MemFile(&bytes))); + assert_eq!(guard.check(&created, true), Ok(())); + assert_eq!(guard.check(&created, false), Ok(())); + assert_eq!(guard.check(&foreign, true), Err(EPERM)); + assert_eq!(listed(&bytes).as_slice(), &[Some(created)]); + Ok(()) + } + + #[test] + fn a_torn_tail_entry_is_overwritten() -> Result<()> { + let first = session(4); + let second = session(5); + let mut list = KVec::new(); + list.extend_from_slice(&encode(&first), GFP_KERNEL)?; + // A crash during the next append left part of an entry behind. + list.extend_from_slice(&encode(&second)[..10], GFP_KERNEL)?; + let bytes = RefCell::new(list); + + let mut guard = Guard::with_file(1, keys(&[first])?, Ok(MemFile(&bytes))); + assert_eq!(guard.check(&first, true), Ok(())); + assert_eq!(guard.check(&second, true), Ok(())); + assert_eq!(listed(&bytes).as_slice(), &[Some(first), Some(second)]); + Ok(()) + } + + #[test] + fn without_the_list_nothing_is_adopted() -> Result<()> { + let foreign = session(1); + let mut guard = Guard::>::with_file(1, keys(&[foreign])?, Err(ENOENT)); + assert_eq!(guard.check(&foreign, true), Err(EPERM)); + assert!(guard.check(&session(6), true).is_err()); + // Deleting a record that does not exist changes nothing. + assert_eq!(guard.check(&session(6), false), Ok(())); + assert_eq!(guard.check(&Key::root(1), true), Ok(())); + Ok(()) + } } From d190da621ce9937f36e917cc90b1eac2b6986fb8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 18 Oct 2026 11:05:19 +1100 Subject: [PATCH 1259/1579] soc: apple: sep: Turn the xART guard on by default on M1 and M2 Macs The xART guard was opt-in. On an M2 (J413), whose enclave Linux shares with a macOS install, it ran from Linux's first SEP boot: the identity keybag was created without any xART write, and through enrolment and a reboot no pre-existing session record changed. On an M1 Pro (J316s) it refused the one write that reached a record from before Linux, which is what it is for. Give each platform profile a default level, and make the parameter's default, -1, select it. M1 (T8103, T6000/T6001) and M2 (T8112, T6020/T6021) Macs default to 1. The MacBook Neo (T8140) stays at 0: it already holds Linux records from before the list existed, which a guarded boot would refuse, and Linux has left its macOS records unchanged. An explicit xart_guard=0, 1 or 2 still wins. With the guard off and writes enabled, the driver now says so at probe. On M1 and M2 Macs this changes two things. On a cold-boot M1-family Mac, creating the identity keybag also rewrites a session record from before Linux; the guard refuses that, so the keybag can only be created there with xart_guard=0. And records Linux created while the guard was off are not on the list, so a guarded boot refuses them too; such a Mac needs xart_guard=0 to keep using them. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/profile.rs | 26 +++++++++++++++++----- drivers/soc/apple/sep.rs | 18 ++++++++++------ drivers/soc/apple/xart_guard.rs | 38 +++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 13bc03c557412..8338fe336e4fd 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -175,9 +175,12 @@ pub(crate) struct PlatformProfile { pub(crate) key_store: KeyStore, /// Platforms with reboot-tested owner export and user-before-master saves. pub(crate) persistent_enrol: bool, + /// The `xart_guard` level when the parameter is left at its default (see + /// [`crate::xart_guard`]). + pub(crate) xart_guard: u8, } -const T8103: PlatformProfile = PlatformProfile { +pub(crate) const T8103: PlatformProfile = PlatformProfile { name: "T8103/J313", shmem_capacity: 0x3_0000, shmem_first_item: b"CNIP", @@ -210,6 +213,8 @@ const T8103: PlatformProfile = PlatformProfile { }, persistent_enrol: true, nonce_session: false, + // macOS shares this enclave: keep Linux to its own xART records. + xart_guard: 1, }; /// M1 Pro (J316s). The M1 family cold-boots its SEP like T8103, so the boot @@ -217,7 +222,7 @@ const T8103: PlatformProfile = PlatformProfile { /// SPI2 at the T6020 address, in the mode T6020 verified: the J313, J316s and /// J414s platform device trees describe the same sensor (id 0x3352) /// identically. Enrolment persistence is reboot-tested on J314s. -const T6000: PlatformProfile = PlatformProfile { +pub(crate) const T6000: PlatformProfile = PlatformProfile { name: "T6000/J316s", shmem_capacity: 0x3_0000, shmem_first_item: b"CNIP", @@ -248,9 +253,13 @@ const T6000: PlatformProfile = PlatformProfile { }, persistent_enrol: true, nonce_session: false, + // As T8103. On J316s, creating the identity keybag also rewrites a + // session record that predates Linux. The guard refuses that, so the + // keybag can only be created there with xart_guard=0. + xart_guard: 1, }; -const T6020: PlatformProfile = PlatformProfile { +pub(crate) const T6020: PlatformProfile = PlatformProfile { name: "T6020/J414s", shmem_capacity: 0x4_0000, shmem_first_item: b"CINP", @@ -279,12 +288,14 @@ const T6020: PlatformProfile = PlatformProfile { key_store: KeyStore::Variant5, persistent_enrol: true, nonce_session: false, + // A warm-attach enclave shared with macOS, as on T8112. + xart_guard: 1, }; /// MacBook Neo. The J700 ADT records a pre-booted SEP, a spi2 Mesa sensor /// (0x3356), and a SEP DART aperture above 4 GiB. These values describe that /// board; device-tree nodes remain authoritative for addresses and resources. -const T8140: PlatformProfile = PlatformProfile { +pub(crate) const T8140: PlatformProfile = PlatformProfile { key_store: KeyStore::Variant5, persistent_enrol: true, name: "T8140/J700", @@ -314,6 +325,9 @@ const T8140: PlatformProfile = PlatformProfile { arg: -1, }, nonce_session: true, + // Linux records from before the guard's list exist here, and Linux has + // left the macOS records unchanged on this platform. + xart_guard: 0, }; /// MacBook Air M2 (J413, J415). iBoot leaves the SEP running and the Mesa @@ -326,7 +340,7 @@ const T8140: PlatformProfile = PlatformProfile { /// /// The shared-memory geometry and key-store dialect follow the other /// warm-registration platforms (T6020, T8140). -const T8112: PlatformProfile = PlatformProfile { +pub(crate) const T8112: PlatformProfile = PlatformProfile { name: "T8112/J415", shmem_capacity: 0x4_0000, shmem_first_item: b"CINP", @@ -357,6 +371,8 @@ const T8112: PlatformProfile = PlatformProfile { key_store: KeyStore::Variant5, persistent_enrol: true, nonce_session: false, + // Guarded from its first Linux boot (2026-10-16): no record refused. + xart_guard: 1, }; static_assert!(T8103.shmem_capacity == 0x30000); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 4cd52b18594af..c251dcbe857fe 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -805,11 +805,17 @@ impl SepData { let xart_writes = *module_parameters::xart_writes.value() != 0; let xart_start = *module_parameters::xart_start_sector.value(); - let xart_guard = *module_parameters::xart_guard.value(); + let xart_guard = + xart_guard::level(*module_parameters::xart_guard.value(), profile.xart_guard); let store = match xart_store::Store::open_owner(xart_writes, xart_start) { Ok(mut store) => { - if xart_guard != 0 && store.writes_enabled() { - store.set_guard(xart_guard)?; + match xart_guard { + Some(level) if store.writes_enabled() => store.set_guard(level)?, + None if store.writes_enabled() => dev_info!( + dev, + "xART guard off: the enclave may rewrite any session record\n" + ), + _ => {} } let (slots, records, revision, malformed, duplicates, repaired, writable) = store.summary(); @@ -2666,9 +2672,9 @@ module! { default: 1, description: "Create the Linux identity keybag when none exists; set to 0 to disable first-boot provisioning; requires xart_writes=1", }, - xart_guard: u8 { - default: 0, - description: "1: refuse the enclave's writes and deletes of xART session records this Linux install did not create; 2: also refuse writes to the shared root records", + xart_guard: i8 { + default: -1, + description: "-1: the platform's level, 1 on M1 and M2 Macs and 0 on the MacBook Neo; 0: off; 1: refuse the enclave's writes and deletes of xART session records this Linux install did not create; 2: also refuse writes to the shared root records", }, os_uuid_hi: u64 { default: 0, diff --git a/drivers/soc/apple/xart_guard.rs b/drivers/soc/apple/xart_guard.rs index e3154bfa31be4..8807ab32b074b 100644 --- a/drivers/soc/apple/xart_guard.rs +++ b/drivers/soc/apple/xart_guard.rs @@ -16,6 +16,11 @@ //! enclave's request, the same way a read-only gigalocker does: Touch ID may //! then be unavailable, but no other OS's record changes. //! +//! The parameter's default, -1, takes the platform profile's level: 1 on the +//! M1 and M2 Macs, whose enclave Linux shares with a macOS install, and 0 on +//! the MacBook Neo, which already holds Linux records from before the list +//! existed and would otherwise lose write access to them. +//! //! The list entry is made durable before the enclave's write reaches the //! gigalocker. A crash between the two leaves an entry for a record that was //! never written, which is harmless. It can never leave a Linux record that @@ -64,6 +69,17 @@ pub(crate) fn decide(level: u8, key: &Key, existed: bool, owned: bool) -> Verdic } } +/// The guard level for `xart_guard=param` on a platform whose profile +/// defaults to `platform`, or `None` for no guard. A negative value takes the +/// platform's level; a level above 2 is treated as 2, the strictest. +pub(crate) fn level(param: i8, platform: u8) -> Option { + let level = u8::try_from(param).unwrap_or(platform); + match level { + 0 => None, + level => Some(level.min(2)), + } +} + /// Encodes one list entry. pub(crate) fn encode(key: &Key) -> [u8; ENTRY_SIZE] { let mut entry = [0u8; ENTRY_SIZE]; @@ -434,4 +450,26 @@ mod tests { assert_eq!(guard.check(&Key::root(1), true), Ok(())); Ok(()) } + + #[test] + fn the_default_follows_the_platform() { + use crate::profile; + + // On where Linux shares the enclave with macOS. + assert_eq!(level(-1, profile::T8103.xart_guard), Some(1)); + assert_eq!(level(-1, profile::T6000.xart_guard), Some(1)); + assert_eq!(level(-1, profile::T6020.xart_guard), Some(1)); + assert_eq!(level(-1, profile::T8112.xart_guard), Some(1)); + // Off on the Neo, whose Linux records predate the list. + assert_eq!(level(-1, profile::T8140.xart_guard), None); + assert_eq!(level(i8::MIN, 1), Some(1)); + } + + #[test] + fn an_explicit_level_overrides_the_platform() { + assert_eq!(level(0, 1), None); + assert_eq!(level(1, 0), Some(1)); + assert_eq!(level(2, 1), Some(2)); + assert_eq!(level(i8::MAX, 0), Some(2)); + } } From 795c664f7cdfaef010f404f52ee27aa61844d8a4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 18 Oct 2026 11:05:55 +1100 Subject: [PATCH 1260/1579] tools: aurora-sep: document the xART guard and its default The driver now defaults to xart_guard=1 on M1 and M2 Macs and to 0 on the MacBook Neo. Say in XART-SAFETY.md what the guard refuses, where it keeps its list, which platform gets which default, and the two cases on an M1 or M2 Mac that need xart_guard=0: creating the keybag on a cold-boot M1-family Mac, and records Linux created with the guard off. Arch PKGBUILD: the XART-SAFETY.md checksum, and pkgrel 9. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/XART-SAFETY.md | 33 +++++++++++++++++++ .../packaging/arch/aurora-sep/PKGBUILD | 4 +-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md index 3ca3b9a5cf3c4..388dede8c7515 100644 --- a/tools/aurora-sep/XART-SAFETY.md +++ b/tools/aurora-sep/XART-SAFETY.md @@ -10,6 +10,39 @@ identity if missing and recover an existing identity without replacing it. The historical evidence below records the earlier opt-in deployments and does not establish qualification on every supported Mac or APFS layout. +## Other OSes' records: `xart_guard` + +The gigalocker is shared by every OS on the Mac, and nothing in a session +record says which OS wrote it. With `xart_guard=1` the driver lists every +session record the enclave creates while Linux runs, in +`/var/lib/aurora-sep-xart-owned.bin`, before the record is written. It then +refuses to rewrite or delete any session record that was already in the +gigalocker and is not on that list. `xart_guard=2` also refuses writes to the +two shared root records. A refused write fails the enclave's request the way +a read-only gigalocker does: Touch ID may then be unavailable, but no other +OS's record changes. + +The default, `xart_guard=-1`, takes the platform's level: + +| Platform | Default | +| --- | --- | +| M1 (T8103), M1 Pro/Max (T6000/T6001) | 1 | +| M2 (T8112), M2 Pro/Max (T6020/T6021) | 1 | +| MacBook Neo (T8140) | 0 | + +The Neo stays at 0 because it already holds Linux records from before the +list existed, and a guarded boot would treat those as another OS's. + +Two cases need a decision on an M1 or M2 Mac: + +- On a cold-boot M1-family Mac, creating the identity keybag also rewrites a + record from before Linux (next section). The guard refuses that, so Linux + cannot create its keybag there unless `options apple_sep xart_guard=0` is + set. Set it only once the rewrite has been accepted for that Mac. +- Records Linux created while the guard was off are not on the list, so a + guarded boot refuses them too. A Mac that already used Touch ID with the + guard off needs `xart_guard=0` to keep using those records. + ## Cold-boot SEP Macs rewrite a pre-Linux record (2026-10-16) On Macs where the driver cold-boots the SEP, creating Linux's identity diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 6b431109496c2..e3574e49c31ec 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -7,7 +7,7 @@ pkgname=aurora-sep pkgver=1 -pkgrel=8 +pkgrel=9 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -41,7 +41,7 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' '30e131dfd8d5fd5be3d7e7712292645e2c6b47eeddd5bafe304b119b30ba6067' - '756ef505520d192ca06e0e79738258a29f1b0f13fb081e7d5992405cd6ecc19f') + 'cc5eb4b19e3a4445405e834192c29ff7b90458757a0cfb2e12599a51775ec28b') # A unit and the helper it runs ($1, installed as $3), pointed at the # packaged helper. From 0c7b5074ac461ce03fa605bd4c11abf4b28b0f41 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 18 Oct 2026 12:38:23 +1100 Subject: [PATCH 1261/1579] soc: apple: sep: Test the SEP DART window of each platform profile The T8112 SEP DART has a 32-bit input aperture and no high alias. A BIT(40) DMA offset there makes the DART probe fail ("Invalid DMA offset for ias=32"), and the SEP never attaches. The window now comes from the platform profile; pin it with a KUnit suite: T8112 has no offset and the 0x4000 + 0xffff0000 range, inside 4 GiB, while every other profile keeps the BIT(40) offset and the 0..4 GiB range. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/profile.rs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 8338fe336e4fd..e755c83b5d5ae 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -454,3 +454,36 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { } Err(ENODEV) } + +#[kernel::macros::kunit_tests(apple_sep_dart_window)] +mod dart_window_tests { + use super::*; + + /// The 64-bit value of `apple,dma-offset`. + fn offset(window: &DartWindow) -> u64 { + (u64::from(window.offset[0]) << 32) | u64::from(window.offset[1]) + } + + /// `apple,dma-range` as the IOVAs it starts and ends at. + fn range(window: &DartWindow) -> (u64, u64) { + let start = (u64::from(window.range[0]) << 32) | u64::from(window.range[1]); + let size = (u64::from(window.range[2]) << 32) | u64::from(window.range[3]); + (start, start + size) + } + + #[test] + fn t8112_has_no_offset_and_stays_below_4_gib() { + assert_eq!(offset(T8112.dart_window), 0); + assert_eq!(range(T8112.dart_window), (0x4000, 0x4000 + 0xffff_0000)); + // The whole window fits the 32-bit DART input aperture. + assert!(range(T8112.dart_window).1 <= 1 << 32); + } + + #[test] + fn the_other_profiles_keep_the_bit40_window() { + for profile in [&T8103, &T6000, &T6020, &T8140] { + assert_eq!(offset(profile.dart_window), 1 << 40); + assert_eq!(range(profile.dart_window), (0, 1 << 32)); + } + } +} From 8533f072dc30e076206631f68f41fb4f8d7512a7 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 19 Aug 2026 10:10:38 +0200 Subject: [PATCH 1262/1579] iommu: apple-dart: Support specifying the DMA aperture in the DT Apple DARTs are often connected directly to devices that expect only a portion of their address space to be used for DMA (for example, because other ranges are mapped directly to something else). Devices can specify the start and end of the aperture via "iommus" args if the DART device node specifies "#iommu-cells = <5>". Aperture start and length are 64-bit values as apertures above and larger than 4GB are used. Limit compile testing to 64-bit architectures to avoid warnings in the calculation of 64-bit dma_addr_t values. This range *can* be outside of the DART's IAS. In that case, it is assumed that the hardware truncates addresses and the page tables will only map the lower bits of the address. However, the specified range cannot straddle an IAS boundary (you cannot cover more than IAS worth of address space nor wrap). This corresponds to the vm-base and vm-size properties on the Apple device tree side of things. [ryan: backported to 7.1 - preserved the existing legacy-window, DMA alias, locked-DART handoff, retained-state, and range-invalidation paths; factored window parsing for focused KUnit coverage] (cherry picked from commit 3658355cfaba9e212ce2c31be6ccc8a998bde66e) Co-developed-by: Hector Martin Signed-off-by: Hector Martin Signed-off-by: Janne Grunau Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart.c | 77 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 6ee58ade4cd70..4bb2f9ff15123 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -425,6 +425,13 @@ struct apple_dart_master_cfg { u32 supports_bypass : 1; u32 locked : 1; + /* + * DMA aperture start and end to be overridden by "iommus"' phandle + * args. By default determined by DART's ias but may be outside of it. + */ + dma_addr_t dma_min; + dma_addr_t dma_max; + struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; }; @@ -1604,8 +1611,8 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, { struct apple_dart *dart = cfg->stream_maps[0].dart; struct io_pgtable_cfg pgtbl_cfg; - dma_addr_t dma_min = dart_domain->has_dma_range ? dart_domain->dma_min : dart->dma_min; - dma_addr_t dma_max = dart_domain->has_dma_range ? dart_domain->dma_max : dart->dma_max; + dma_addr_t dma_min = dart_domain->has_dma_range ? dart_domain->dma_min : cfg->dma_min; + dma_addr_t dma_max = dart_domain->has_dma_range ? dart_domain->dma_max : cfg->dma_max; u64 translated_max; u32 ias; int ret = 0; @@ -1981,6 +1988,52 @@ static void apple_dart_domain_free(struct iommu_domain *domain) kfree(dart_domain); } +static int apple_dart_parse_dma_window(struct device *dev, + const struct apple_dart *dart, + const struct of_phandle_args *args, + dma_addr_t *dma_min, + dma_addr_t *dma_max) +{ + dma_addr_t length; + + *dma_min = dart->dma_min; + *dma_max = dart->dma_max; + + if (args->args_count == 1) + return 0; + if (args->args_count != 5) + return -EINVAL; + + length = ((dma_addr_t)args->args[3] << 32) | args->args[4]; + if (!length) + return -EINVAL; + + if (*dma_min != 0 || *dma_max != DMA_BIT_MASK(dart->ias)) + dev_err(dev, "still using legacy \"apple,dma-range\" property\n"); + + *dma_min = ((dma_addr_t)args->args[1] << 32) | args->args[2]; + + if (!IS_ALIGNED(*dma_min, dart->pgsize) || + !IS_ALIGNED(length, dart->pgsize)) { + dev_err(dev, "Unaligned DMA window %pad, %pad (0x%x)\n", + dma_min, &length, dart->pgsize); + return -EINVAL; + } + if (check_add_overflow(*dma_min, length - 1, dma_max)) { + dev_err(dev, "DMA window length (%pad) overflows range for start %pad\n", + &length, dma_min); + return -EINVAL; + } + + /* Ensure that the DMA window does not exceed the DART's ias. */ + if ((*dma_min ^ *dma_max) & ~DMA_BIT_MASK(dart->ias)) { + dev_err(dev, "Invalid DMA window for ias=%d\n", dart->ias); + return -EINVAL; + } + + return 0; +} + static int apple_dart_of_xlate(struct device *dev, const struct of_phandle_args *args) { @@ -1988,12 +2041,16 @@ static int apple_dart_of_xlate(struct device *dev, struct platform_device *iommu_pdev = of_find_device_by_node(args->np); struct apple_dart *dart = platform_get_drvdata(iommu_pdev); struct apple_dart *cfg_dart; + dma_addr_t dma_max; + dma_addr_t dma_min; int i, sid; put_device(&iommu_pdev->dev); - if (args->args_count != 1) - return -EINVAL; + i = apple_dart_parse_dma_window(dev, dart, args, &dma_min, &dma_max); + if (i) + return i; + sid = args->args[0]; if (!cfg) { @@ -2004,6 +2061,18 @@ static int apple_dart_of_xlate(struct device *dev, cfg->supports_bypass = true; /* Will be ORed with DART capabilities*/ cfg->locked = false; + /* Will be merged with other DARTs to the common range. */ + cfg->dma_min = dma_min; + cfg->dma_max = dma_max; + } else { + if (dma_min >= cfg->dma_max || cfg->dma_min >= dma_max) { + dev_err(dev, "non-overlapping DMA windows: %pad..%pad, %pad..%pad\n", + &dma_min, &dma_max, + &cfg->dma_min, &cfg->dma_max); + return -EINVAL; + } + cfg->dma_min = max(dma_min, cfg->dma_min); + cfg->dma_max = min(dma_max, cfg->dma_max); } dev_iommu_priv_set(dev, cfg); From 6f0e0c7df865ba2bef67ae54277c0917a35e38e1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 18 Oct 2026 16:09:08 +1100 Subject: [PATCH 1263/1579] iommu: apple-dart: Test legacy and 5-cell DT windows Exercise the one-cell form using the DART-node aperture and the five-cell form using the equivalent consumer aperture. Cover representative display DART windows from T6000, T6020, and T8112 device trees so either DTB generation remains accepted by the parser. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 68 +++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index f7afabf12e3ad..41430d44702d0 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -5,6 +5,72 @@ #include #include +struct apple_dart_test_window { + const char *soc; + u32 ias; + u32 sid; + u64 base; + u64 size; +}; + +static void apple_dart_test_iommu_specifiers(struct kunit *test) +{ + static const struct apple_dart_test_window windows[] = { + { "t6000", 32, 0, 0, 0xfc000000 }, + { "t6020", 42, 5, 0x10000000000, 0x1000000000 }, + { "t8112", 36, 5, 0x800000000, 0x7ffff0000 }, + }; + struct device *dev; + int i; + + dev = kunit_device_register(test, "apple-dart-window-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + for (i = 0; i < ARRAY_SIZE(windows); i++) { + const struct apple_dart_test_window *window = &windows[i]; + struct of_phandle_args args = { .args_count = 1 }; + struct apple_dart *dart; + dma_addr_t dma_min; + dma_addr_t dma_max; + int ret; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->ias = window->ias; + dart->pgsize = SZ_16K; + dart->dma_min = window->base; + dart->dma_max = window->base + window->size - 1; + + args.args[0] = window->sid; + ret = apple_dart_parse_dma_window(dev, dart, &args, + &dma_min, &dma_max); + KUNIT_ASSERT_EQ_MSG(test, ret, 0, "%s legacy specifier", + window->soc); + KUNIT_EXPECT_EQ_MSG(test, dma_min, (dma_addr_t)window->base, + "%s legacy window start", window->soc); + KUNIT_EXPECT_EQ_MSG(test, dma_max, + (dma_addr_t)(window->base + window->size - 1), + "%s legacy window end", window->soc); + + dart->dma_min = 0; + dart->dma_max = DMA_BIT_MASK(dart->ias); + args.args_count = 5; + args.args[1] = upper_32_bits(window->base); + args.args[2] = lower_32_bits(window->base); + args.args[3] = upper_32_bits(window->size); + args.args[4] = lower_32_bits(window->size); + ret = apple_dart_parse_dma_window(dev, dart, &args, + &dma_min, &dma_max); + KUNIT_ASSERT_EQ_MSG(test, ret, 0, "%s 5-cell specifier", + window->soc); + KUNIT_EXPECT_EQ_MSG(test, dma_min, (dma_addr_t)window->base, + "%s 5-cell window start", window->soc); + KUNIT_EXPECT_EQ_MSG(test, dma_max, + (dma_addr_t)(window->base + window->size - 1), + "%s 5-cell window end", window->soc); + } +} + static void apple_dart_test_before_attach(struct kunit *test) { struct apple_dart_master_cfg master = {}; @@ -31,6 +97,7 @@ static void apple_dart_test_before_attach(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, dart->regs); spin_lock_init(&dart->lock); mutex_init(&domain->init_lock); + master.dma_max = DMA_BIT_MASK(dart->ias); master.stream_maps[0].dart = dart; __set_bit(0, master.stream_maps[0].sidmap); writel(dart->hw->ttbr_valid, dart->regs + DART_TTBR(dart, 0, 0)); @@ -772,6 +839,7 @@ static void apple_dart_test_resume_reset_retry(struct kunit *test) } static struct kunit_case apple_dart_test_cases[] = { + KUNIT_CASE(apple_dart_test_iommu_specifiers), KUNIT_CASE(apple_dart_test_before_attach), KUNIT_CASE(apple_dart_test_locked_handoff), KUNIT_CASE(apple_dart_test_sync_unknown_roots), From d80847118522567a39106da6bc6fb1ba16b57275 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 24 Oct 2026 23:16:40 +1100 Subject: [PATCH 1264/1579] Bluetooth: bcm4377: abort suspend if wake IRQ arming fails With controller wake enabled, bcm4377_suspend() returns success even when enable_irq_wake() fails. The MSI is then not a wake source, although HCI has been suspended for remote wake and the function remains in D0. A fault-injected callback test reproduces a zero return with HCI still suspended and no wake reference after enable_irq_wake() returns -ENXIO. Return the arming error and undo HCI suspension using the existing resume path. Clear the powered-resume epoch before that rollback, since no system sleep completed, and preserve the original error if HCI rollback also fails. Arm the IRQ before saving PCI state, so an aborted suspend does not leave a saved-state marker. The PM core does not invoke resume for the device whose suspend callback failed. The extracted PM callbacks pass host tests with mocked kernel APIs for arming failure, rollback failure, repeated successful wake cycles, wake-disabled quiescence and HCI suspend failure, with recovery support on and off. ARM64 kernel build and hardware wake qualification remain pending; this does not establish the cause of a particular machine hang. Fixes: 0609ca7fa79a ("Bluetooth: hci_bcm4377: Support remote wakeup") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/hci_bcm4377.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 69b87ed4fa6fe..9d3bc53d54607 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -3049,6 +3049,8 @@ static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id) hdev); } +static int bcm4377_resume(struct device *dev); + static int bcm4377_suspend(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); @@ -3071,6 +3073,21 @@ static int bcm4377_suspend(struct device *dev) } if (device_may_wakeup(dev)) { + ret = enable_irq_wake(pdev->irq); + if (ret) { + int resume_ret; + + dev_err(dev, "Failed to arm Bluetooth wake IRQ: %d\n", ret); + /* The PM core will not resume a device that failed suspend. */ + bcm4377->resume_powered = false; + resume_ret = bcm4377_resume(dev); + if (resume_ret) + dev_err(dev, "Failed to undo Bluetooth suspend: %d\n", + resume_ret); + return ret; + } + bcm4377->wake_armed = true; + /* * Remote wakeup: leave the firmware running and the function * in D0, so a connection attempt from a wake-allowed device @@ -3078,7 +3095,6 @@ static int bcm4377_suspend(struct device *dev) * core from putting the function into D3hot. */ pci_save_state(pdev); - bcm4377->wake_armed = !enable_irq_wake(pdev->irq); return 0; } From 1ea4c92a7e4b44c65153b65db82c4ce73632f1c4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 24 Oct 2026 23:18:58 +1100 Subject: [PATCH 1265/1579] docs: explain Apple s2idle wake paths and qualification Distinguish ordinary armed wake IRQs from filtered NO_SUSPEND transport interrupts, and describe how to identify a suspend-entry or resume stall. Cover SMC, dockchannel, USB, Bluetooth and RTC prerequisites without assuming one radio or PMIC across Apple Silicon boards. Document a source-by-source test matrix, wake-policy checks and recovery requirements. Restrict s2idle timed PM tests to checkpoints reached before the idle loop; processors and core can otherwise cause real sleep. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/power/apple-s2idle.rst | 109 +++++++++++++++++++++++++++ Documentation/power/index.rst | 1 + MAINTAINERS | 1 + 3 files changed, 111 insertions(+) create mode 100644 Documentation/power/apple-s2idle.rst diff --git a/Documentation/power/apple-s2idle.rst b/Documentation/power/apple-s2idle.rst new file mode 100644 index 0000000000000..aeceefe3fabff --- /dev/null +++ b/Documentation/power/apple-s2idle.rst @@ -0,0 +1,109 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Apple Silicon suspend-to-idle wake qualification +=============================================== + +A missing wake IRQ number does not establish that an Apple Silicon system +has no wake source. There are two interrupt paths to distinguish: + +* An ordinary wake interrupt is enabled with ``enable_irq_wake()``. The IRQ + core keeps it enabled during device IRQ suspension and reports a system + wakeup when it arrives. AIC and AIC2/AIC3 use ``IRQCHIP_SKIP_SET_WAKE``: + retained-power suspend-to-idle needs normal interrupt delivery rather + than a second hardware wake routing configuration. +* An ``IRQF_NO_SUSPEND`` interrupt continues to run its handler. The handler + or its consumer must identify a genuine wake event and report it to the + PM core. The interrupt alone does not end system suspend. This permits + transports with unrelated traffic to filter their wake events. + +SMC mailbox interrupts use the second path. The SMC input driver makes +power-button presses and lid-open events hard wake events during its sleep +window. The RTKit notification worker must remain runnable. Do not arm the +whole SMC mailbox as an ordinary wake IRQ: unrelated notifications and +command replies are not user wake requests. + +Dockchannel HID also uses the second path. Its RX child interrupt stays +live below a chained AIC parent. The HID transport classifies keyboard +reports and trackpad clicks before reporting a hard wake event; pointer +motion, touch and other interface traffic do not wake the system. On +systems using a receive DMA ring, both the coprocessor and its translations +must remain usable while the system sleeps. + +Apple platform xHCI can keep running during suspend-to-idle and report +filtered USB wake events. Peripheral remote-wake support, hub policy and +controller retention all matter. An unarmed xHCI IRQ by itself does not +prove that this path is broken. + +Bluetooth depends on the fitted radio and its transport driver. In +particular, the MT7932 driver is not used by a board with a Broadcom radio. +Check the bound driver, device wake policy, HCI event filters, PCI/MSI wake +propagation and the return value of wake IRQ arming. A driver must undo its +own suspend state if it returns an error after suspending HCI: the PM core +will not invoke its normal resume callback for that failed suspend. + +An SMC RTC that provides timekeeping is not necessarily an alarm wake +source. Enumerate RTC names, alarm support and wake policy rather than +assuming that ``rtc0`` or a PMU driver enabled in the kernel configuration +provides a working recovery alarm. A PMIC's compatible string, register +layout, offset cell and interrupt wiring must all match its RTC driver. + +Establish the failing phase +-------------------------- + +The ``PM: suspend entry (s2idle)`` message is printed before process +freezing and device suspend. It does not prove entry into the idle loop. +Likewise, loss of SSH can follow network suspension while another device +is still suspending. A black display does not establish whether CPUs have +resumed. + +Capture the console independently of the display and network under test. +Use the existing ``power:suspend_resume`` and +``power:device_pm_callback_start``/``power:device_pm_callback_end`` trace +points, PM debug messages and callback timing. A ``machine_suspend`` entry +for suspend-to-idle establishes that the idle path was reached; callback +start/end pairs locate a device suspend or resume stall. See +:doc:`basic-pm-debugging` for staged PM tests. For suspend-to-idle, use +``freezer``, ``devices`` and ``platform``. The idle loop runs before the +``processors`` and ``core`` test checkpoints, so those levels do not +provide an automatic return from suspend-to-idle. + +Before actual sleep, record: + +* Board compatible strings, kernel release, source commit, configuration + and the DTB used by the boot loader. +* ``/sys/power/mem_sleep`` and ``/sys/power/pm_test``. +* The wake policy of the SMC input, dockchannel HID, Bluetooth PCI function + and USB devices; the bound drivers; relevant IRQ names and counts. +* ``/sys/kernel/debug/wakeup_sources`` and + ``/sys/power/suspend_stats`` where available. +* Each RTC's name, alarm support and device wake policy. + +Use ``/sys/power/pm_wakeup_irq`` for ordinary armed-IRQ attribution. For a +filtered transport, also compare its wakeup-source counters and interrupt +trace. An empty IRQ result is compatible with a valid filtered wake. + +Qualification matrix +-------------------- + +Test one wake stimulus at a time, first after a short sleep and then after +repeated cycles: + +* A short power-button press, and a lid close/open cycle. +* The built-in keyboard and a trackpad click. Separately verify that + motion and touch do not cause an unwanted wake. +* A wake-enabled, paired Bluetooth keyboard with the fitted radio driver. + Repeat with the PCI function's wake policy disabled. +* A directly connected USB keyboard, followed by the intended hub/dock, + with the necessary device and hub wake policies enabled. +* A timed RTC alarm only on a device known to provide that functionality. + +Also test a wake during entry, an aborted suspend, Bluetooth wake IRQ +arming failure, and suspend with Bluetooth wake disabled. A wake-arming +failure must abort suspend and leave the running transport recoverable; +successful cycles must balance wake IRQ enable/disable calls. + +For every cycle record the stimulus, idle entry, wake event, resume +completion and post-resume input, storage and network health separately. +Preserve a failed cycle's console and trace before recovery. Keep a known +bootable fallback and an independently available recovery transport, and +restore any temporary debug settings and wake policies after testing. diff --git a/Documentation/power/index.rst b/Documentation/power/index.rst index b4581e4ae785b..ce6d50693040f 100644 --- a/Documentation/power/index.rst +++ b/Documentation/power/index.rst @@ -8,6 +8,7 @@ Power Management :maxdepth: 1 apm-acpi + apple-s2idle basic-pm-debugging charger-manager drivers-testing diff --git a/MAINTAINERS b/MAINTAINERS index 0a50e30dd7f82..6eb8b5aab2bff 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2740,6 +2740,7 @@ F: Documentation/devicetree/bindings/usb/apple,dwc3.yaml F: Documentation/devicetree/bindings/usb/apple,sn201202x.yaml F: Documentation/devicetree/bindings/watchdog/apple,wdt.yaml F: Documentation/hwmon/macsmc-hwmon.rst +F: Documentation/power/apple-s2idle.rst F: arch/arm64/boot/dts/apple/ F: drivers/bluetooth/hci_bcm4377.c F: drivers/clk/clk-apple-nco.c From d5ee0bd28d7e75edfb2acceef113138c457e5c76 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 25 Oct 2026 04:55:18 +1100 Subject: [PATCH 1266/1579] drm/asahi: keep work-queue firmware structures until the context is invalidated When a queue is destroyed while it still has jobs on the GPU, drm_sched_fini() cancels the scheduler jobs, but the firmware keeps running the commands it already holds. Each work queue then stays alive only through its event slot and is dropped from the completion path as soon as its last stamp is seen. Dropping it frees its queue info (ring state, ring and notifier list), and the last command frees the queue's notifier and threshold. Those come from the shared and private kernel allocators, so the memory can be reused right away. The GPU context that the firmware still associates those structures with is only invalidated at the next alloc(), so until then the firmware can still refer to a work queue whose memory has already been handed out. The context data already keeps the tile vertex buffer alive until it is invalidated. Do the same for the queue structures. A GpuContext now keeps a list of retained objects. A work queue moves its queue info there when it is dropped, and the queue adds its notifier when it is created. The list travels with the context data to the garbage list and is freed once DestroyContext has completed. The queue info points to the context weakly now, since the context owns it from then on. This closes the lifetime window for those two objects. On its own it does not stop the sustained-teardown firmware crash on the M1 Pro (G13X): that crash still reproduces with this change, so at least one other kernel object on the teardown path is read by the firmware after it is freed and reused. Investigation continues. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/fw/workqueue.rs | 7 +- drivers/gpu/drm/asahi/gpu.rs | 14 +- drivers/gpu/drm/asahi/queue/mod.rs | 15 +- drivers/gpu/drm/asahi/workqueue.rs | 202 +++++++++++++++++--------- 4 files changed, 160 insertions(+), 78 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/workqueue.rs b/drivers/gpu/drm/asahi/fw/workqueue.rs index b86bd3ff3757a..78ac6c676a6eb 100644 --- a/drivers/gpu/drm/asahi/fw/workqueue.rs +++ b/drivers/gpu/drm/asahi/fw/workqueue.rs @@ -152,7 +152,7 @@ pub(crate) mod raw { pub(crate) unk_94: u32, pub(crate) pending: AtomicU32, pub(crate) unk_9c: u32, - pub(crate) gpu_context: GpuPointer<'a, super::GpuContextData>, + pub(crate) gpu_context: GpuWeakPointer, pub(crate) unk_a8: U64, #[ver(V >= V13_2 && G < G14X)] pub(crate) unk_b0: u32, @@ -171,6 +171,10 @@ impl GpuStruct for GpuContextData { type Raw<'a> = raw::GpuContextData; } +/// The firmware structures of a work queue. +/// +/// The raw structure points to the queue's GPU context weakly. The `GpuContext` keeps these +/// structures until it has been invalidated, and only then frees its own data. #[versions(AGX)] #[derive(Debug)] pub(crate) struct QueueInfo { @@ -178,7 +182,6 @@ pub(crate) struct QueueInfo { pub(crate) ring: GpuArray, pub(crate) gpu_buf: GpuArray, pub(crate) notifier_list: Arc>, - pub(crate) gpu_context: Arc, } #[versions(AGX)] diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index b68381ce60b30..62c049ff7dddc 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -260,9 +260,8 @@ pub(crate) struct GpuManager { stats: Arc, buffer_mgr: buffer::BufferManager::ver, ids: SequenceIDs, - #[allow(clippy::vec_box)] #[pin] - garbage_contexts: Mutex>>>, + garbage_contexts: Mutex>, } /// Operations the DRM driver performs on a GPU, independent of its firmware interface. @@ -401,8 +400,8 @@ pub(crate) trait GpuManager: Send + Sync { fn get_cfg(&self) -> &'static hw::HwConfig; /// Get the dynamic GPU configuration for this SoC. fn get_dyncfg(&self) -> &hw::DynConfig; - /// Register an unused context as garbage - fn free_context(&self, data: KBox>); + /// Register an unused context, with the objects it retains, for invalidation. + fn free_context(&self, ctx: workqueue::RetiredContext); } /// Private generic trait for functions that don't need to escape this module. @@ -1456,7 +1455,7 @@ impl GpuManager for GpuManager::ver { core::mem::swap(&mut *self.garbage_contexts.lock(), &mut garbage_ctx); for ctx in garbage_ctx { - if self.invalidate_context(&ctx).is_err() { + if self.invalidate_context(&ctx.data).is_err() { dev_err!( self.dev.as_ref(), "GpuContext: Failed to invalidate GPU context!\n" @@ -1465,6 +1464,9 @@ impl GpuManager for GpuManager::ver { panic!("GPU firmware timed out"); } } + // The firmware no longer knows the context's queues, so their structures may go now, + // ahead of the context data they point to. + core::mem::drop(ctx.retained); } let mut guard = self.alloc.lock(); @@ -1733,7 +1735,7 @@ impl GpuManager for GpuManager::ver { &self.dyncfg } - fn free_context(&self, ctx: KBox>) { + fn free_context(&self, ctx: workqueue::RetiredContext) { let mut garbage = self.garbage_contexts.lock(); if garbage.push(ctx, GFP_KERNEL).is_err() { diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 2b32b97b5e169..2d9052c2edb51 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -525,6 +525,15 @@ impl Queue::ver { let buffer = buffer::Buffer::ver::new(gpu, alloc, ualloc.clone(), ualloc_priv, mgr)?; + let gpu_context = Arc::new( + workqueue::GpuContext::new(dev.clone(), alloc, buffer.any_ref())?, + GFP_KERNEL, + )?; + // Every command of this queue points the firmware at the notifier, and the notifier links + // into the notifier list of the work queues. Keep it as long as they are kept, until the + // context is invalidated, instead of freeing it with the last command. + gpu_context.retain(KBox::new(notifier.clone(), GFP_KERNEL)?)?; + let mut ret = Queue::ver { dev: dev.clone(), _sched: sched, @@ -538,11 +547,7 @@ impl Queue::ver { inner: QueueInner::ver { dev: dev.clone(), ualloc, - gpu_context: Arc::new( - workqueue::GpuContext::new(dev.clone(), alloc, buffer.any_ref())?, - GFP_KERNEL, - )?, - + gpu_context, buffer, notifier_list: Arc::new(notifier_list, GFP_KERNEL)?, notifier, diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index 6281ef684e251..d6695a1b3ee96 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -95,10 +95,31 @@ impl From for kernel::error::Error { } } +/// An object the firmware may still reference while its GPU context is live. +pub(crate) type RetainedObject = KBox; + +/// Objects retained by every context: the notifier and the vertex, fragment and compute work +/// queues of the owning queue. +const CONTEXT_RETAINED_OBJECTS: usize = 4; + +/// A dropped GPU context waiting for the firmware to invalidate it. +pub(crate) struct RetiredContext { + /// The context data named in the invalidation request. + pub(crate) data: KBox>, + /// Firmware structures of the context's queues. They must not be freed, and so must not be + /// reused, until the invalidation has completed. + pub(crate) retained: KVec, +} + /// A GPU context tracking structure, which must be explicitly invalidated when dropped. +/// +/// The firmware learns about the work queues of a context as they are run, and only forgets them +/// when the context is invalidated. Their firmware structures are therefore handed to the context +/// when their owner drops them, and freed after the invalidation. pub(crate) struct GpuContext { dev: driver::AsahiDevice, data: Option>>, + retained: Pin>>>, } no_debug!(GpuContext); @@ -118,27 +139,42 @@ impl GpuContext { )?, GFP_KERNEL, )?), + retained: KBox::pin_init( + new_mutex!( + KVec::with_capacity(CONTEXT_RETAINED_OBJECTS, GFP_KERNEL)?, + "GpuContext::retained" + ), + GFP_KERNEL, + )?, }) } - /// Returns the GPU pointer to the inner GPU context data structure. - pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, fw::workqueue::GpuContextData> { - self.data.as_ref().unwrap().gpu_pointer() + /// Returns a weak GPU pointer to the inner GPU context data structure. + pub(crate) fn weak_pointer(&self) -> GpuWeakPointer { + self.data.as_ref().unwrap().weak_pointer() + } + + /// Keep `object` allocated until the firmware has invalidated this context. + pub(crate) fn retain(&self, object: RetainedObject) -> Result { + self.retained.lock().push(object, GFP_KERNEL)?; + Ok(()) } } impl Drop for GpuContext { fn drop(&mut self) { + let data = self.data.take().unwrap(); + let retained = core::mem::take(&mut *self.retained.lock()); + let Some(guard) = self.dev.registration_guard() else { return; }; guard.registration_data_with(|reg_data| { - let data = self.data.take().unwrap(); // A context only exists for the manager whose allocators created it, so the manager is // always present. if let Some(gpu) = reg_data.gpu.manager() { - gpu.free_context(data); + gpu.free_context(RetiredContext { data, retained }); } }) } @@ -237,7 +273,10 @@ impl) + Send + Sync> GenSubm struct WorkQueueInner { dev: driver::AsahiDevice, event_manager: Arc, - info: GpuObject, + /// The firmware structures of this queue. Always `Some` until the queue is dropped, when they + /// move to `gpu_context`. + info: Option>>, + gpu_context: Arc, new: bool, pipe_type: PipeType, size: u32, @@ -265,10 +304,22 @@ pub(crate) struct WorkQueue { #[versions(AGX)] impl WorkQueueInner::ver { + fn info(&self) -> &GpuObject { + self.info + .as_deref() + .expect("WorkQueue: lost its queue info") + } + + fn info_mut(&mut self) -> &mut GpuObject { + self.info + .as_deref_mut() + .expect("WorkQueue: lost its queue info") + } + /// Return the GPU done pointer, representing how many work items have been completed by the /// GPU. fn doneptr(&self) -> u32 { - self.info + self.info() .state .with(|raw, _inner| raw.gpu_doneptr.load(Ordering::Acquire)) } @@ -485,7 +536,7 @@ impl Job::ver { let next_wptr = (wptr + 1) % inner.size; assert!(inner.doneptr() != next_wptr); - inner.info.ring[wptr as usize] = command.inner.gpu_va().get(); + inner.info_mut().ring[wptr as usize] = command.inner.gpu_va().get(); wptr = next_wptr; // Cannot fail, since we did a reserve(1) above @@ -515,7 +566,7 @@ impl<'a> JobSubmission::ver<'a> { core::mem::forget(self); inner - .info + .info() .state .with(|raw, _inner| raw.cpu_wptr.store(wptr, Ordering::Release)); @@ -527,7 +578,7 @@ impl<'a> JobSubmission::ver<'a> { let msg = fw::channels::RunWorkQueueMsg::ver { pipe_type: inner.pipe_type, - work_queue: Some(inner.info.weak_pointer()), + work_queue: Some(inner.info().weak_pointer()), wptr: inner.wptr, event_slot, is_new: inner.new, @@ -617,6 +668,23 @@ impl<'a> Drop for JobSubmission::ver<'a> { } } +#[versions(AGX)] +impl Drop for WorkQueueInner::ver { + fn drop(&mut self) { + // The firmware keeps referring to this queue until its context is invalidated. When the + // queue's owner went away with work in flight, the queue is dropped as soon as that work + // completes, which can be well before the invalidation. + if let Some(info) = self.info.take() { + if self.gpu_context.retain(info).is_err() { + pr_err!( + "WorkQueue({:?}): Failed to keep the queue until its context is invalidated\n", + self.pipe_type + ); + } + } + } +} + #[versions(AGX)] impl WorkQueueInner::ver { /// Return the number of free entries in the workqueue @@ -680,61 +748,65 @@ impl WorkQueue::ver { prio.5 = 1; } + let gpu_context_pointer = gpu_context.weak_pointer(); + + // Use shared (coherent) state with verbose faults so we can dump state correctly + let info = if debug_enabled(DebugFlags::VerboseFaults) { + &mut alloc.shared + } else { + &mut alloc.private + } + .new_init( + try_init!(QueueInfo::ver { + state: { + state.with_mut(|raw, _inner| { + raw.rb_size = size; + }); + state + }, + ring, + gpu_buf, + notifier_list: notifier_list, + }), + |inner, _p| { + try_init!(raw::QueueInfo::ver { + state: inner.state.gpu_pointer(), + ring: inner.ring.gpu_pointer(), + notifier_list: inner.notifier_list.gpu_pointer(), + gpu_buf: inner.gpu_buf.gpu_pointer(), + gpu_rptr1: Default::default(), + gpu_rptr2: Default::default(), + gpu_rptr3: Default::default(), + event_id: AtomicI32::new(-1), + priority: prio, + unk_4c: -1, + uuid: id as u32, + unk_54: -1, + unk_58: Default::default(), + busy: Default::default(), + __pad: Default::default(), + #[ver(V >= V13_2 && G < G14X)] + unk_84_0: 0, + unk_84_state: Default::default(), + error_count: Default::default(), + unk_8c: 0, + unk_90: 0, + unk_94: 0, + pending: Default::default(), + unk_9c: 0, + gpu_context: gpu_context_pointer, + unk_a8: Default::default(), + #[ver(V >= V13_2 && G < G14X)] + unk_b0: 0, + }) + }, + )?; + let inner = WorkQueueInner::ver { dev: dev.into(), event_manager, - // Use shared (coherent) state with verbose faults so we can dump state correctly - info: if debug_enabled(DebugFlags::VerboseFaults) { - &mut alloc.shared - } else { - &mut alloc.private - } - .new_init( - try_init!(QueueInfo::ver { - state: { - state.with_mut(|raw, _inner| { - raw.rb_size = size; - }); - state - }, - ring, - gpu_buf, - notifier_list: notifier_list, - gpu_context: gpu_context, - }), - |inner, _p| { - try_init!(raw::QueueInfo::ver { - state: inner.state.gpu_pointer(), - ring: inner.ring.gpu_pointer(), - notifier_list: inner.notifier_list.gpu_pointer(), - gpu_buf: inner.gpu_buf.gpu_pointer(), - gpu_rptr1: Default::default(), - gpu_rptr2: Default::default(), - gpu_rptr3: Default::default(), - event_id: AtomicI32::new(-1), - priority: prio, - unk_4c: -1, - uuid: id as u32, - unk_54: -1, - unk_58: Default::default(), - busy: Default::default(), - __pad: Default::default(), - #[ver(V >= V13_2 && G < G14X)] - unk_84_0: 0, - unk_84_state: Default::default(), - error_count: Default::default(), - unk_8c: 0, - unk_90: 0, - unk_94: 0, - pending: Default::default(), - unk_9c: 0, - gpu_context: inner.gpu_context.gpu_pointer(), - unk_a8: Default::default(), - #[ver(V >= V13_2 && G < G14X)] - unk_b0: 0, - }) - }, - )?, + info: Some(KBox::new(info, GFP_KERNEL)?), + gpu_context, new: true, pipe_type, size, @@ -751,7 +823,7 @@ impl WorkQueue::ver { last_submitted: None, }; - let info_pointer = inner.info.weak_pointer(); + let info_pointer = inner.info().weak_pointer(); Arc::pin_init( pin_init!(Self { @@ -835,7 +907,7 @@ impl WorkQueue::ver { pub(crate) fn dump_info(&self) { pr_info!("WorkQueue @ {:?}:", self.info_pointer); - self.inner.lock().info.with(|raw, _inner| { + self.inner.lock().info().with(|raw, _inner| { pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr1.load(Ordering::Relaxed)); pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr2.load(Ordering::Relaxed)); pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr3.load(Ordering::Relaxed)); @@ -953,7 +1025,7 @@ impl WorkQueue for WorkQueue::ver { ); inner - .info + .info() .state .with(|raw, _inner| raw.cpu_freeptr.store(last_wptr, Ordering::Release)); From a2a5040d95b26704b9ce51b55301944f99c75e1c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 25 Oct 2026 11:23:00 +1100 Subject: [PATCH 1267/1579] arm64: configs: enable the Asahi default thermal governor's dependencies The shared fragment requests the power allocator as the default thermal governor but does not enable the governor itself. A Fedora base explicitly disables THERMAL_GOV_POWER_ALLOCATOR, so olddefconfig drops the requested default and falls back to step-wise. Defconfig happens to enable it. Enable the governor and its energy-model dependency in the common fragment. This keeps the requested hardware default available on both defconfig and distribution bases without a device-specific configuration. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 3c5b8c02f06f7..7ff013f56825b 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -11,6 +11,8 @@ CONFIG_ARM64_16K_PAGES=y CONFIG_ARM64_MEMORY_MODEL_CONTROL=y CONFIG_ARM_APPLE_CPUIDLE=y CONFIG_ARM_APPLE_SOC_CPUFREQ=m +CONFIG_ENERGY_MODEL=y +CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m From a9e24ae824ec205cf90f3545b0125631adf1326a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 00:42:19 +1100 Subject: [PATCH 1268/1579] merge: retain the 7.3 implementations where older ports duplicate them The first merge gate exposed duplicate functions, fields and superseded signatures from the 7.1 line. Keep the newer 7.3 implementations in those shared paths. Retain all WIP-only files, AVE plumbing, board additions and distinct non-conflicting guards. The initial merge and its failed gate evidence remain preserved. Signed-off-by: Ryan Murray Assisted-by: LLM --- Documentation/PCI/controller/apple-t8140.rst | 2 +- .../devicetree/bindings/gpu/apple,agx.yaml | 2 - drivers/bluetooth/hci_bcm4377.c | 19 - drivers/gpu/drm/apple/audio.c | 26 -- drivers/gpu/drm/apple/dcp.c | 162 --------- drivers/gpu/drm/apple/dptxep.c | 1 - drivers/gpu/drm/apple/iomfb-state.h | 1 - drivers/gpu/drm/apple/iomfb.c | 342 ------------------ drivers/gpu/drm/apple/iomfb_template.h | 18 - drivers/gpu/drm/apple/plane.c | 17 - drivers/gpu/drm/asahi/driver.rs | 7 - drivers/gpu/drm/asahi/file.rs | 32 -- drivers/gpu/drm/asahi/mmu.rs | 105 ------ drivers/gpu/drm/asahi/regs.rs | 177 --------- drivers/gpu/drm/drm_gem_shmem_helper.c | 16 - drivers/iommu/apple-dart-test.c | 1 - drivers/iommu/apple-dart.c | 12 - .../net/wireless/mediatek/mt7932/channels.h | 55 --- drivers/nvme/host/apple.c | 30 +- drivers/pmdomain/apple/Kconfig | 24 -- drivers/pmdomain/apple/pmgr-pwrstate.c | 3 - drivers/soc/apple/pm.rs | 6 +- drivers/soc/apple/pm_shim.c | 9 +- drivers/soc/apple/sbio.rs | 84 +---- drivers/soc/apple/shim.h | 2 +- drivers/usb/dwc3/dwc3-apple.c | 8 - drivers/usb/typec/tipd/cd321x-pm.h | 14 - drivers/usb/typec/tipd/core.c | 6 - drivers/vfio/vfio_iommu_type1.c | 141 ++++---- include/linux/bpf.h | 9 - include/linux/soc/apple/rtkit.h | 8 - io_uring/rsrc.c | 4 - kernel/fork.c | 5 - kernel/futex/core.c | 38 -- mm/filemap.c | 5 - mm/internal.h | 4 - mm/memory.c | 190 +++++----- mm/mempolicy.c | 4 - mm/page_vma_mapped.c | 12 - mm/util.c | 18 +- mm/vma.c | 4 - rust/helpers/barrier.c | 10 - rust/kernel/drm/device.rs | 9 +- rust/kernel/sync/barrier.rs | 20 - security/tomoyo/domain.c | 38 +- 45 files changed, 227 insertions(+), 1473 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 496e9b94d40f6..7d342d1c1b4d1 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -1,7 +1,7 @@ .. SPDX-License-Identifier: GPL-2.0-only T8140/J700 experimental radio PCIe bootstrap -========================================== +============================================ The J700 path trains port 0, validates a bounded PIODMA bootstrap request, and enables native ECAM for the MediaTek Wi-Fi and Bluetooth functions. diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index b65cd174f31f6..d29b1ab0310aa 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -97,8 +97,6 @@ properties: Operating-point table of the GPU's AFR clock domain, one voltage cell per state. - operating-points-v2: true - apple,firmware-abi: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 3 diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 2ca515a292b71..69b87ed4fa6fe 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -628,13 +628,6 @@ struct bcm4377_data { bool quiesced; bool wake_armed; - /* Set once a recovery reprobe has been queued for this binding. */ - unsigned long reset_pending; - - /* System sleep: firmware quiesced, or kept running to wake us. */ - bool quiesced; - bool wake_armed; - struct bcm4377_context *ctx; dma_addr_t ctx_dma; @@ -3089,18 +3082,6 @@ static int bcm4377_suspend(struct device *dev) return 0; } - if (device_may_wakeup(dev)) { - /* - * Remote wakeup: leave the firmware running and the function - * in D0, so a connection attempt from a wake-allowed device - * raises an interrupt. Saving the state here stops the PCI - * core from putting the function into D3hot. - */ - pci_save_state(pdev); - bcm4377->wake_armed = !enable_irq_wake(pdev->irq); - return 0; - } - iowrite32(BCM4377_BAR0_SLEEP_CONTROL_QUIESCE, bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); /* Flush the posted write before the system goes to sleep. */ diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index a1fbb7bb89ccb..cc54ae2a9b587 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -113,32 +113,6 @@ static const struct snd_pcm_hardware dcp_pcm_hw_unplugged = { .periods_max = DCPAUD_PERIODS_MAX, }; -/* - * Without a sink the DCP has no audio elements to derive constraints from. - * Offer a plain stereo format so that the PCM can still be opened and - * configured, as HDMI PCMs on other platforms can. Userspace (PipeWire, - * PulseAudio) probes the PCM once when the card appears and drops the - * output for good if that fails, so a monitor attached later would never - * show up. Starting the stream still fails until a sink is connected. - */ -static const struct snd_pcm_hardware dcp_pcm_hw_unplugged = { - .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | - SNDRV_PCM_INFO_INTERLEAVED, - .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE | - SNDRV_PCM_FMTBIT_S32_LE, - .rates = SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_44100 | - SNDRV_PCM_RATE_48000, - .rate_min = 32000, - .rate_max = 48000, - .channels_min = 2, - .channels_max = 2, - .buffer_bytes_max = SIZE_MAX, - .period_bytes_min = 4096, - .period_bytes_max = SIZE_MAX, - .periods_min = 2, - .periods_max = UINT_MAX, -}; - static int dcpaud_read_remote_info(struct dcp_audio *dcpaud) { int ret; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 82ff25b2b3c0c..1078e5dff3345 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -635,21 +635,6 @@ void dcp_update_connector_dimensions(struct apple_dcp *dcp, connector->display_info.height_mm = height_mm; } -void dcp_update_connector_dimensions(struct apple_dcp *dcp, - struct drm_connector *connector) -{ - int width_mm = READ_ONCE(dcp->width_mm); - int height_mm = READ_ONCE(dcp->height_mm); - - lockdep_assert_held(&connector->dev->mode_config.mutex); - if (!width_mm || !height_mm) { - width_mm = dcp->panel.width_mm; - height_mm = dcp->panel.height_mm; - } - connector->display_info.width_mm = width_mm; - connector->display_info.height_mm = height_mm; -} - bool dcp_has_panel(struct apple_dcp *dcp) { return iomfb_is_m4(dcp) || dcp->panel.width_mm > 0; @@ -1030,12 +1015,6 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi return ret; } - /* External H17P does not program adaptive sync. */ - if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL && - crtc_state->vrr_enabled) - return -EINVAL; - needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !READ_ONCE(dcp->mode_state.valid); if (!needs_modeset && (!dcp->connector || !dcp->connector->connected)) { @@ -2320,32 +2299,6 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) return dcp_external_link_locked(dcp); } - if (dcp_is_external_h17p(dcp)) { - bool new_cable; - - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->placeholder_edid_wq); - cancel_work_sync(&dcp->external_stop_work); - cancel_delayed_work_sync(&dcp->external_stop_retry_wq); - guard(mutex)(&dcp->external_lock); - new_cable = !dcp->external_cable; - if (new_cable) - dcp->external_generation++; - dcp->external_cable = true; - WRITE_ONCE(dcp->external_physically_absent, false); - dcp_external_publish_generation(dcp); - scoped_guard(mutex, &dcp->hpd_mutex) { - WRITE_ONCE(dcp->typec_cable_connected, true); - if (new_cable) { - dcp->typec_generation++; - dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); - dcp_announce_end(&dcp->announce); - } - dcp->placeholder_retried = false; - } - return dcp_external_link_locked(dcp); - } - if (dcp_is_typec_output(dcp)) { cancel_delayed_work_sync(&dcp->placeholder_edid_wq); guard(mutex)(&dcp->hpd_mutex); @@ -2587,16 +2540,6 @@ int dcp_start(struct platform_device *pdev) return dev_err_probe(dcp->dev, ret, "Failed to start remote allocator\n"); } - if (IS_ERR_OR_NULL(dcp->rtk)) - return -ENODEV; - - /* Make the H17P allocator available before external service discovery. */ - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { - ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); - if (ret) - return dev_err_probe(dcp->dev, ret, "Failed to start remote allocator\n"); - } - /* start RTKit endpoints */ ret = systemep_init(dcp); if (ret) @@ -3854,26 +3797,6 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat "DCP was not left running by the bootloader\n"); } - if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL && - !dcp_external_is_idle(dcp)) - return dev_err_probe(dev, -EBUSY, - "External DCP is not in the inherited idle state\n"); - - /* - * Display coprocessors running H17-generation firmware are left running - * by the bootloader with their firmware mapped through a locked DART. - * Linux attaches to that session with a standard RTKit INIT/HELLO - * exchange and must not stop or restart the ASC. - */ - if (dcp->hw.adopt_live_session) { - cpu_ctrl = readl_relaxed(dcp->coproc_reg + - APPLE_DCP_COPROC_CPU_CONTROL); - if (!(cpu_ctrl & APPLE_DCP_COPROC_CPU_CONTROL_RUN)) - return dev_err_probe(dev, -EOPNOTSUPP, - "DCP was not left running by the bootloader\n"); - } - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && !dcp_external_is_idle(dcp)) @@ -4211,11 +4134,6 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v dcp->dptx_controller_ep = NULL; } - if (!IS_ERR_OR_NULL(dcp->dptx_controller_ep)) { - afk_shutdown(dcp->dptx_controller_ep); - dcp->dptx_controller_ep = NULL; - } - if (dcp->ibootep) { afk_shutdown(dcp->ibootep); dcp->ibootep = NULL; @@ -5013,86 +4931,6 @@ static void dcp_platform_complete(struct device *dev) mutex_unlock(&dcp->hpd_mutex); } -/* - * The display stays powered across system sleep, and the blanking present - * queued by the DRM and backlight suspend handlers must reach the firmware - * before its mailbox interrupt is suspended. - */ -static void dcp_drain_for_sleep(struct apple_dcp *dcp) -{ - int pass; - - if (!dcp->hw.adopt_live_session || dcp->fw_compat != DCP_FIRMWARE_H17P || - dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) - return; - - /* - * A completed present can schedule one more brightness present, and a - * rejected one a delayed retry; run a pending retry now instead of - * letting it fire after the mailbox has suspended. Four passes cover - * the first attempt and the policy's three retries. A fade still - * running ends at its target first. - */ - if (dcp_has_panel(dcp)) - dcp_backlight_finish_fade(dcp); - for (pass = 0; pass < 4; pass++) { - flush_delayed_work(&dcp->iomfb.backlight_retry); - if (dcp_has_panel(dcp)) - flush_work(&dcp->bl_update_wq); - if (!iomfb_queue_drain(dcp, msecs_to_jiffies(500))) { - dev_warn(dcp->dev, "display updates still pending at suspend\n"); - return; - } - /* - * A crashed or stopped DCP takes no more presents, so a level - * still pending there can never be sent and is not waited for. - * Checking after the drain, which returns at once for such a - * DCP, also covers a crash during the pass. - */ - if (READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.stopped)) - return; - if (!dcp_backlight_active(dcp) || !dcp_backlight_pending(dcp)) - return; - } - dev_warn(dcp->dev, "brightness change still pending at suspend\n"); -} - -/* - * DPMS off leaves the internal H17P pipe scanning out (see dcp_poweroff()), - * and while it scans out the firmware publishes a property to the AP on every - * frame. The mailbox interrupt is not suspended, so each of those messages - * wakes a CPU out of suspend-to-idle about 60 times a second. Hold the - * messages in the mailbox for system sleep and process them at resume. - * apple-drm is a consumer of this device, so its suspend handler has blanked - * the panel before this runs and its resume handler restores the CRTC after. - */ -static void dcp_hold_for_sleep(struct apple_dcp *dcp) -{ - /* A failed component bind leaves no coprocessor session to hold. */ - if (IS_ERR_OR_NULL(dcp->rtk) || - !dcp_uses_soft_dpms(dcp) || dcp->fw_compat != DCP_FIRMWARE_H17P || - dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || - READ_ONCE(dcp->crashed)) - return; - - iomfb_pause_presentation(dcp, true); - if (iomfb_queue_settle(dcp, msecs_to_jiffies(500))) { - iomfb_pause_presentation(dcp, false); - return; - } - apple_rtkit_hold_rx(dcp->rtk); - dcp->rx_held = true; -} - -static void dcp_release_after_sleep(struct apple_dcp *dcp) -{ - if (!dcp->rx_held) - return; - dcp->rx_held = false; - apple_rtkit_resume_rx(dcp->rtk); - iomfb_pause_presentation(dcp, false); -} - static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 6eb56789f0679..d61b28e04546a 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -1067,7 +1067,6 @@ static const struct apple_epic_service_ops dptxep_ops[] = { .teardown = dptxport_teardown, .init = dptxport_init, .call = dptxport_call, - .call_v2 = dptxport_group_call, }, {} }; diff --git a/drivers/gpu/drm/apple/iomfb-state.h b/drivers/gpu/drm/apple/iomfb-state.h index ce3a15f4f73c1..84d16ca4a9405 100644 --- a/drivers/gpu/drm/apple/iomfb-state.h +++ b/drivers/gpu/drm/apple/iomfb-state.h @@ -36,7 +36,6 @@ static inline void dcp_mode_begin(struct dcp_mode_state *state) static inline void dcp_mode_set_valid(struct dcp_mode_state *state, bool valid) { unsigned long flags; - bool valid; spin_lock_irqsave(&state->lock, flags); state->valid = valid; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 98e31d09eb4e7..d463c85dd885f 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1133,305 +1133,6 @@ void dcp_route_failure_notify(struct apple_connector *connector) schedule_work(&connector->hotplug_wq); } -static void dcp_clear_connector_edid(struct apple_connector *connector) -{ - lockdep_assert_held(&connector->base.dev->mode_config.mutex); - drm_edid_connector_update(&connector->base, NULL); - drm_edid_free(connector->drm_edid); - connector->drm_edid = NULL; - connector->drm_edid_firmware_derived = false; -} - -static void dcp_consume_edid_invalidation(struct apple_connector *connector) -{ - lockdep_assert_held(&connector->base.dev->mode_config.mutex); - /* Preserve a disconnect even if the port reconnected before probing. */ - if (xchg(&connector->edid_invalidation_pending, false)) - dcp_clear_connector_edid(connector); -} - -/* - * The external H17P pipeline learns the modes of each cable connection from - * firmware messages that can lag the link; see dcp-announce.h. - */ -static bool dcp_announce_gated(struct apple_dcp *dcp, - struct apple_connector *connector) -{ - return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && - READ_ONCE(dcp->connector) == connector; -} - -/* - * Queue the timeout work of a held connection: at once for a hold whose - * timeout has not started for the current cable generation, as only that - * work starts it. - */ -void dcp_announce_arm(struct apple_dcp *dcp) -{ - struct dcp_announce *a = &dcp->announce; - unsigned long now = jiffies, delay = 0; - - lockdep_assert_held(&dcp->hpd_mutex); - if (a->stopped || !a->holding) - return; - if (a->timed && a->generation == dcp->typec_generation && - time_before(now, a->deadline)) - delay = a->deadline - now; - mod_delayed_work(system_freezable_wq, &dcp->announce_wq, delay); -} - -/* - * Evaluate what DRM is to report for the connection. @start starts the - * timeout of a held connection; only the timeout work does that. - */ -static enum dcp_announce_status -dcp_announce_eval(struct apple_dcp *dcp, struct apple_connector *connector, - bool start) -{ - struct dcp_announce *a = &dcp->announce; - struct dcp_announce_facts facts = { - .generation = dcp->typec_generation, - .connected = READ_ONCE(connector->connected), - .present = !READ_ONCE(dcp->external_physically_absent), - .failed = READ_ONCE(dcp->external_error) || READ_ONCE(dcp->crashed), - .lost = dcp_announce_lost(a, dcp->typec_generation), - }; - enum dcp_announce_status status; - unsigned long now = jiffies; - bool held = a->holding, announced = a->announced; - - lockdep_assert_held(&dcp->hpd_mutex); - if (facts.connected && facts.present && !facts.failed) - facts.ready = dcp_external_modes_ready(dcp); - status = dcp_announce_update(a, &facts, now); - if (status == DCP_ANNOUNCE_HOLD) { - if (start) - dcp_announce_start(a, now); - dcp_announce_arm(dcp); - } else if (held) { - /* The hold ended: its timeout work has nothing left to do. */ - cancel_delayed_work(&dcp->announce_wq); - if (status == DCP_ANNOUNCE_CONNECTED && !facts.ready && !facts.failed) - dev_info(dcp->dev, "display modes not ready after %u ms\n", - announced ? DCP_ANNOUNCE_RESUME_TIMEOUT_MS : - DCP_ANNOUNCE_TIMEOUT_MS); - } - return status; -} - -/* - * Evaluate the connection outside the hotplug worker. An event that this - * makes owed is the worker's to send, so queue it, unless the announcement - * is stopped: dcp_enable_typec_work() queues it then. - */ -static void dcp_announce_poll(struct apple_dcp *dcp, struct apple_connector *connector, - bool start) -{ - bool owed = dcp_announce_owed(&dcp->announce); - - dcp_announce_eval(dcp, connector, start); - if (!owed && dcp_announce_owed(&dcp->announce) && !dcp->announce.stopped) - schedule_work(&connector->hotplug_wq); -} - -/* - * The timeout of a held connection. It runs on the freezable workqueue, as - * the link work does, so the timeout of a hold starts here, once the link - * can come back for the hold's generation. Everything it does is under - * hpd_mutex: a hold that ended meanwhile, or an announcement stopped for - * teardown or sleep, gets nothing more from here. - */ -void dcp_announce_work(struct work_struct *work) -{ - struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, - announce_wq); - struct apple_connector *con = READ_ONCE(dcp->connector); - - scoped_guard(mutex, &dcp->hpd_mutex) { - if (dcp->announce.holding && !dcp->announce.stopped && con && - dcp_announce_gated(dcp, con) && - READ_ONCE(con->dcp) == to_platform_device(dcp->dev)) - dcp_announce_poll(dcp, con, true); - } - dcp_test_point(dcp, DCP_TEST_ANNOUNCE_TIMER_DONE); -} - -/* The connection state DRM reports for @connector. */ -bool dcp_connector_connected(struct apple_connector *connector) -{ - struct platform_device *pdev = READ_ONCE(connector->dcp); - struct apple_dcp *dcp = pdev ? platform_get_drvdata(pdev) : NULL; - - if (!dcp || !dcp_announce_gated(dcp, connector)) - return READ_ONCE(connector->connected); - - guard(mutex)(&dcp->hpd_mutex); - dcp_announce_poll(dcp, connector, false); - return dcp->announce.announced; -} - -/* The CRTC userspace keeps active on @connector while its link is down. */ -static struct drm_crtc *dcp_external_unlinked_crtc(struct apple_dcp *dcp, - struct apple_connector *connector) -{ - struct drm_connector_state *conn_state = connector->base.state; - struct drm_crtc *crtc = conn_state ? conn_state->crtc : NULL; - - drm_modeset_lock_assert_held(&connector->base.dev->mode_config.connection_mutex); - if (!crtc || !crtc->state || !crtc->state->active || - READ_ONCE(dcp->mode_state.valid)) - return NULL; - return crtc; -} - -static bool dcp_external_needs_relink(struct apple_dcp *dcp, - struct apple_connector *connector) -{ - struct drm_device *dev = connector->base.dev; - struct drm_modeset_acquire_ctx ctx; - bool unlinked = false; - int ret; - - DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, ret); - unlinked = !!dcp_external_unlinked_crtc(dcp, connector); - DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); - - return unlinked; -} - -/* Commit @crtc with its mode set again and the link of @connector good. */ -static int dcp_external_commit_relink(struct apple_connector *connector, - struct drm_crtc *crtc, - struct drm_modeset_acquire_ctx *ctx) -{ - struct drm_connector_state *conn_state; - struct drm_crtc_state *crtc_state; - struct drm_atomic_state *state; - int ret; - - state = drm_atomic_state_alloc(crtc->dev); - if (!state) - return -ENOMEM; - state->acquire_ctx = ctx; - crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(crtc_state)) { - ret = PTR_ERR(crtc_state); - goto out; - } - conn_state = drm_atomic_get_connector_state(state, &connector->base); - if (IS_ERR(conn_state)) { - ret = PTR_ERR(conn_state); - goto out; - } - crtc_state->connectors_changed = true; - conn_state->link_status = DRM_LINK_STATUS_GOOD; - ret = drm_atomic_commit(state); -out: - drm_atomic_state_put(state); - - return ret; -} - -/* - * Set the mode of the CRTC userspace keeps active on @connector again. If - * that fails, or leaves the link down, mark the link bad and lose the - * connection of cable generation @generation. Both are decided under the - * modeset locks, so a CRTC that userspace switches off is either switched - * off first and left alone, or found lost by dcp_external_crtc_released(). - * - * Returns the error that lost the connection, or 0. - */ -static int dcp_external_reset_crtc(struct apple_dcp *dcp, - struct apple_connector *connector, u64 generation) -{ - struct drm_device *dev = connector->base.dev; - struct drm_modeset_acquire_ctx ctx; - struct drm_crtc *crtc; - int ret, err = 0; - - DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, ret); - - err = 0; - crtc = dcp_external_unlinked_crtc(dcp, connector); - if (!crtc) - goto unlock; - ret = dcp_external_commit_relink(connector, crtc, &ctx); - if (ret == -EDEADLK) - goto unlock; - if (!ret && READ_ONCE(dcp->mode_state.valid)) - goto unlock; - - err = ret ?: -EIO; - ret = 0; - scoped_guard(mutex, &dcp->hpd_mutex) { - if (generation == dcp->typec_generation) { - connector->base.state->link_status = DRM_LINK_STATUS_BAD; - dcp_announce_lose(&dcp->announce, generation); - /* The caller's event is the one this owes. */ - dcp_announce_notify(&dcp->announce, - dcp_announce_eval(dcp, connector, false)); - } else { - /* A newer connection gets a link of its own. */ - err = 0; - } - } -unlock: - DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); - - return ret ?: err; -} - -/* - * The link of an external display came back, or was disturbed, under a CRTC - * that userspace keeps active. Userspace sees the display it had and need - * not set its mode again, so set it here. A mode that needs its link - * proven, such as the 180 Hz exception mode, is admitted only against the - * EDID a probe read for the current cable generation (see - * dcp_180hz_mode_current()), so probe first, as userspace would. - * - * A link that cannot be set up again is marked bad, and the connection is - * lost: reported gone until userspace releases the CRTC, see - * dcp-announce.h. The caller sends the hotplug event. - */ -static void dcp_external_relink(struct apple_dcp *dcp, - struct apple_connector *connector) -{ - struct drm_device *dev = connector->base.dev; - u64 generation; - int ret; - - if (!dcp_external_needs_relink(dcp, connector)) - return; - - scoped_guard(mutex, &dcp->hpd_mutex) - generation = dcp->typec_generation; - scoped_guard(mutex, &dev->mode_config.mutex) - connector->base.funcs->fill_modes(&connector->base, - dev->mode_config.max_width, - dev->mode_config.max_height); - - ret = dcp_external_reset_crtc(dcp, connector, generation); - if (ret) - dev_warn(dcp->dev, "display link not restored (%d), reporting it disconnected\n", - ret); -} - -/* - * The CRTC of the external display was switched off. If its connection was - * lost, report the display again, as a new connection, once per cable - * generation (see dcp_announce_retry()). - */ -void dcp_external_crtc_released(struct apple_dcp *dcp) -{ - struct apple_connector *con = READ_ONCE(dcp->connector); - - guard(mutex)(&dcp->hpd_mutex); - if (dcp_announce_retry(&dcp->announce, dcp->typec_generation) && - !dcp->announce.stopped && con && - READ_ONCE(con->dcp) == to_platform_device(dcp->dev)) - schedule_work(&con->hotplug_wq); -} - void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; @@ -1553,23 +1254,6 @@ static bool dcp_external_zero_reply(struct apple_dcp *dcp, int tag, of_device_is_compatible(dcp->dev->of_node, "apple,t8140-dcpext"); } -/* - * J700/25G83 asks D134 after publishing its external video interface. - * The observed async record has no input and a four-byte result. Use the - * existing zero reply for this record only; its feature remains unsupported. - */ -static bool dcp_external_zero_reply(struct apple_dcp *dcp, int tag, - enum dcp_context_id context, - u32 in_len, u32 out_len) -{ - return tag == 134 && context == DCP_CONTEXT_ASYNC && - in_len == 0 && out_len == sizeof(u32) && - dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL && - of_machine_is_compatible("apple,j700") && - of_device_is_compatible(dcp->dev->of_node, "apple,t8140-dcpext"); -} - static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, void *data, u32 length, u16 offset) { @@ -2346,29 +2030,6 @@ static bool dcp_mode_settable(struct apple_dcp *dcp, return true; } -/* - * H17P keeps the mode the bootloader programmed and never sends - * set_digital_out_mode (see DCP_INHERIT_BOOT_MODE in iomfb_template.c), so a - * modeset to any other mode would silently not be applied. The bootloader - * brings the panel up in its native timing, which is the mode the firmware - * scores highest and enumerate_modes() marks preferred; offer only that one. - */ -static bool dcp_mode_settable(struct apple_dcp *dcp, - const struct drm_display_mode *mode) -{ - struct dcp_display_mode *dcp_mode = lookup_mode(dcp, mode); - - if (!dcp_mode) - return false; - - if (dcp->fw_compat == DCP_FIRMWARE_H17P && - dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL) - return dcp_mode->mode.type & DRM_MODE_TYPE_PREFERRED; - - return true; -} - enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, const struct drm_display_mode *mode) { @@ -3018,9 +2679,6 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) case DCP_FIRMWARE_H17P: iomfb_flush_h17p(dcp, crtc, commit); break; - case DCP_FIRMWARE_H17P: - iomfb_flush_h17p(dcp, crtc, state); - break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 8c5b9c833d8d8..b987ddfbd8ce7 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -28,17 +28,6 @@ struct DCP_FW_NAME(dcp_swap) { u8 unk_v14_7[0x48]; #endif -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * H17P inserts 0x48 bytes between flags2 and the swap ID, which moves - * to +0x98. Every following field shifts with it: surf_ids +0x9c, - * src_rect +0xac, surf_flags +0xec, dst_rect +0x10c, - * swap_enabled/completed +0x14c/+0x150, bg_color +0x154. The gap holds - * two small u32 counters (at +0x50 and +0x78) that are not modelled. - */ - u8 h17p_pre_swap_id[0x48]; -#endif - #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) /* * H17P inserts 0x48 bytes between flags2 and the swap ID, which moves @@ -104,13 +93,6 @@ struct DCP_FW_NAME(dcp_swap) { */ u8 h17p_tail[0xda]; #endif -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * The H17P A408 request is 0xe9c bytes; with a 0x22c-byte surface, - * SWAP_SURFACES = 4 and no surf2 block, that leaves this tail. - */ - u8 h17p_tail[0xda]; -#endif } __packed; /* Information describing a surface */ diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index 7503f2e0b9288..31ea943333e17 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -583,23 +583,6 @@ static const u32 dcp_primary_formats_h17p[] = { #endif }; -/* The H17P alpha semantics of 10-bit XRGB are unknown. */ -static const u32 dcp_primary_formats_h17p[] = { - DRM_FORMAT_ARGB2101010, - DRM_FORMAT_XRGB8888, - DRM_FORMAT_ARGB8888, - DRM_FORMAT_XBGR8888, - DRM_FORMAT_ABGR8888, - DRM_FORMAT_NV12, - DRM_FORMAT_NV16, - DRM_FORMAT_NV24, - DRM_FORMAT_P010, - DRM_FORMAT_P210, -#if defined(DRM_FORMAT_P410) - DRM_FORMAT_P410, -#endif -}; - static const u32 dcp_overlay_formats[] = { DRM_FORMAT_ABGR16161616F, DRM_FORMAT_ARGB2101010, diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index c1e178b5ca246..47f6a1cf4a856 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -217,13 +217,6 @@ impl platform::Driver for AsahiPlatformDriver { } } - fn unbind(_pdev: &platform::Device, this: Pin<&Self>) { - if let Some(gpu) = (*this.drm).gpu.as_any().downcast_ref::() { - // Driver data is dropped after devres, so CPU control must happen in unbind. - gpu.shutdown(); - } - } - /// Device probe function. fn probe<'bound>( pdev: &'bound platform::Device>, diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index ecd0748540b1b..d1b8084902bcc 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -376,32 +376,6 @@ impl File { _ => {} } - match data.param_group { - uapi::DRM_ASAHI_PARAM_GROUP_VM_STATUS if gpu.supports_vm_status() => { - return Self::get_vm_status(data, file); - } - uapi::DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR if gpu.supports_vm_status() => { - return Self::register_vm_status_mirror(data, file); - } - uapi::DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS => { - if data.pad != 0 - || data.size < core::mem::size_of::() as u64 - { - return Err(EINVAL); - } - let limits = gpu.queue_limits().ok_or(EINVAL)?; - let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; - UserSlice::new( - UserPtr::from_addr(data.pointer as _), - core::mem::size_of_val(&words), - ) - .writer() - .write(&words)?; - return Ok(0); - } - _ => {} - } - if data.param_group != 0 || data.pad != 0 { cls_pr_debug!(Errors, "get_params: Invalid arguments\n"); return Err(EINVAL); @@ -1455,12 +1429,6 @@ impl File { } } - if ret.is_ok() { - if let Some((util_min, duration_ms)) = gpu.syncobj_wait_hint() { - file.set_syncobj_wait_hint(util_min, duration_ms); - } - } - match ret { Err(ERESTARTSYS) => Err(ERESTARTSYS), Err(e) => { diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 2e1e60c64830b..6a80d1b484b34 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -3397,111 +3397,6 @@ impl Uat { Ok(()) } - /// Returns the address space geometry of this UAT. - pub(crate) fn geometry(&self) -> UatGeometry { - self.geometry - } - - /// Adopts the top-level entries that the primary firmware instance published in the kernel - /// root while booting ([`UatFirmware::PublishedRoots`]). - /// - /// The entries below the kernel window belong to the firmware, and the kernel window entry - /// points at a table that the bootloader reserved, so the driver maps kernel objects beneath - /// it and never frees it. Entries above the kernel window do not belong to anyone and are - /// cleared. Must run after the primary instance booted and before anything is mapped into the - /// kernel `Vm`. - pub(crate) fn adopt_firmware_root_entries(&self) -> Result<[u64; 3]> { - let owned = IOVA_KERN_TOP_LEVEL_INDEX as usize + 1; - let mut snapshot = [0; 3]; - // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives - // this `Uat`. - unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { - for (value, pte) in snapshot.iter_mut().zip(ptes) { - *value = pte.load(Ordering::Relaxed); - } - for pte in &ptes[owned..] { - pte.store(0, Ordering::Relaxed); - } - }) - }?; - - mem::sync(); - mem::tlbi_all(); - mem::sync(); - Ok(snapshot) - } - - /// Checks that allocation under the shared subtree did not replace a firmware-owned entry. - pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { - // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. - let matches = unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { - ptes.iter() - .zip(snapshot) - .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) - }) - }?; - if !matches { - return Err(EIO); - } - Ok(()) - } - - /// Copies the kernel root's top-level entries from the kernel window up into the root of the - /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are - /// private to that instance. - pub(crate) fn mirror_secondary_root(&self) -> Result { - let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; - let count = UAT_PGSZ / size_of::(); - let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; - - // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives - // this `Uat`. - unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { - pgtable::with_root_entries(secondary, count, |secondary| { - for (src, dst) in primary.iter().zip(secondary).skip(first) { - let pte = src.load(Ordering::Relaxed); - if pte != 0 && dst.load(Ordering::Relaxed) != pte { - dst.store(pte, Ordering::Relaxed); - } - } - }) - }) - }??; - - mem::sync(); - mem::tlbi_all(); - mem::sync(); - Ok(()) - } - - /// Activates the device-global render root in context 1 after the boot records retire. - /// Both contexts must still contain their original kernel roots. - pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { - let inner = self.inner.lock(); - inner.handoff().lock(); - let ttbs = inner.ttbs(); - let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { - ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) - && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) - }); - if !ready { - inner.handoff().unlock(); - return Err(EIO); - } - ttbs[KERNEL_ALIAS_CTX] - .ttb0 - .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.handoff().unlock(); - core::mem::drop(inner); - mem::sync(); - mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); - mem::sync(); - Ok(()) - } - /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] fn make_inner( diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 18336602dc5a9..e7f6fd810b4ad 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -291,23 +291,6 @@ impl Resources<'_> { }) } - /// Installs both render queues while the caller holds the device mutex. - /// No other port publisher may interleave between this power witness and - /// the final validity strobe. False means no write was attempted. - pub(crate) fn install_render_pair( - &self, - registration: &crate::g17::fw::kick::RenderRegistration, - ) -> Result { - let sgx = self.sgx.try_access().ok_or(ENODEV)?; - if sgx.relaxed().read32(G17_HOST_IRQ_SUMMARY) & G17_HOST_POWERED == 0 { - return Ok(false); - } - for &(offset, value) in registration.writes() { - sgx.try_write64(value, offset)?; - } - Ok(true) - } - fn sgx_read32(&self) -> u32 { self.sgx.as_view().relaxed().read32(OFF) } @@ -453,166 +436,6 @@ impl Resources<'_> { }) } - /// Get the identification of a G17 GPU from its registers. - pub(crate) fn get_g17_id(&self) -> Result { - let id_version = self.sgx_read32::(); - let id_unk08 = self.sgx_read32::(); - let id_counts_1 = self.sgx_read32::(); - let id_counts_2 = self.sgx_read32::(); - let id_unk18 = self.sgx_read32::(); - let id_clusters = self.sgx_read32::(); - - dev_info!( - self.dev.as_ref(), - "GPU ID registers: {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}\n", - id_version, - id_unk08, - id_counts_1, - id_counts_2, - id_unk18, - id_clusters - ); - - // Later reads can fault while the firmware has gated the cores. Submission uses only - // these saved values and never re-reads the registers. - let perf_control = self.sgx_read32::() & 1; - let perf_map = self.sgx_read32::(); - let mut core_mask_regs = [ - self.sgx_read32::(), - self.sgx_read32::<{ CORE_MASKS_G14X + 4 }>(), - self.sgx_read32::<{ CORE_MASKS_G14X + 8 }>(), - ]; - dev_info!(self.dev.as_ref(), "Core masks: {:#x?}\n", core_mask_regs); - - let num_dies = (id_counts_1 >> 16) & 0xf; - let num_clusters = ((id_counts_1 >> 8) & 0xff) * num_dies; - let num_cores = id_counts_1 & 0xff; - if num_cores == 0 || num_cores > 32 || num_cores * num_clusters > 3 * 32 { - dev_err!( - self.dev.as_ref(), - "Invalid core counts ({} x {})\n", - num_clusters, - num_cores - ); - return Err(ENODEV); - } - let (core_masks, _) = - self.split_core_masks(&mut core_mask_regs, num_clusters, num_cores)?; - let (gpu_rev, _) = self.gpu_revision(id_version)?; - - Ok(G17Id { - family: ((id_version >> 24) & 0xff) as u8, - variant: ((id_version >> 16) & 0xff) as u8, - gpu_rev, - num_dies, - num_clusters, - num_cores, - core_masks, - perf_control, - perf_map: [perf_map & 0xf, (perf_map >> 16) & 0xf], - }) - } - - /// Enable the G17 GPU-to-fabric bridge and check that it took effect. - /// - /// The GPU power domain must be on. The coprocessors cannot reach memory until this is done. - pub(crate) fn enable_g17_bridge(&self) -> Result { - let sgx = self.sgx.try_access().ok_or(ENODEV)?; - for offset in G17_BRIDGE_CONTROL { - let val = sgx.try_read32(offset)?; - sgx.try_write32(val | G17_BRIDGE_ENABLE, offset)?; - if sgx.try_read32(offset)? & G17_BRIDGE_ENABLE == 0 { - dev_err!( - self.dev.as_ref(), - "GPU bridge {:#x} did not enable\n", - offset - ); - return Err(EIO); - } - } - Ok(()) - } - - /// Disable the G17 GPU-to-fabric bridge. Both coprocessors must be stopped. - pub(crate) fn disable_g17_bridge(&self) { - if let Some(sgx) = self.sgx.try_access() { - for offset in G17_BRIDGE_CONTROL { - if let Ok(val) = sgx.try_read32(offset) { - // Cannot fail: the offset was read just before. - let _ = sgx.try_write32(val & !G17_BRIDGE_ENABLE, offset); - } - } - } - } - - /// Power witness for the queue-configuration port, accessible while the cores are gated. - pub(crate) fn g17_configuration_powered(&self) -> bool { - self.sgx_read32::() & 0xf != 0 - } - - /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is - /// initialized. - pub(crate) fn enable_g17_gating(&self) -> Result { - let sgx = self.sgx.try_access().ok_or(ENODEV)?; - let val = sgx.try_read32(G17_GATING_CONTROL)?; - sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) - } - - /// Split the packed per-cluster core masks into one mask per cluster. - /// - /// Returns the masks and the total number of enabled cores. - fn split_core_masks( - &self, - core_mask_regs: &mut [u32], - num_clusters: u32, - num_cores: u32, - ) -> Result<(KVec, u32)> { - let mut core_masks = KVec::new(); - let mut total_active_cores: u32 = 0; - - let max_core_mask = ((1u64 << num_cores) - 1) as u32; - for _ in 0..num_clusters { - let mask = core_mask_regs[0] & max_core_mask; - core_masks.push(mask, GFP_KERNEL)?; - for i in 0..core_mask_regs.len() { - core_mask_regs[i] = core_mask_regs[i].checked_shr(num_cores).unwrap_or(0); - if i < (core_mask_regs.len() - 1) { - core_mask_regs[i] |= core_mask_regs[i + 1] - .checked_shl(32 - num_cores) - .unwrap_or(0); - } - } - total_active_cores += mask.count_ones(); - } - - if core_mask_regs.iter().any(|a| *a != 0) { - dev_err!( - self.dev.as_ref(), - "Leftover core mask: {:#x?}\n", - core_mask_regs - ); - return Err(EIO); - } - - Ok((core_masks, total_active_cores)) - } - - /// Decode the silicon revision from the ID version register. - fn gpu_revision(&self, id_version: u32) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { - Ok(match (id_version >> 8) & 0xff { - 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), - 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), - 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), - 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), - 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), - 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), - a => { - dev_err!(self.dev.as_ref(), "Unknown GPU revision {}\n", a); - return Err(ENODEV); - } - }) - } - /// Get the fault information from the MMU status register, if one occurred. pub(crate) fn get_fault_info(&self, cfg: &'static hw::HwConfig) -> Option { let g14x = cfg.gpu_core as u32 >= hw::GpuCore::G14S as u32; diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 9a4db4fa5e60c..4b613b1cc7f7a 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -1018,22 +1018,6 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct } #endif - /* Revocation zaps the device anon mapping only. */ - if (shmem->reclaimable_cpu_mappings && - (!vma->vm_file || vma->vm_file->f_mapping != obj->dev->anon_inode->i_mapping)) - return -EINVAL; - - /* Validate bytes, including a partial native-page offset from PRIME. */ - { - pgoff_t index = drm_gem_shmem_object_offset(obj, vma->vm_pgoff); - unsigned long offset, length = vma->vm_end - vma->vm_start; - - if (check_mul_overflow(index, PAGE_SIZE, &offset) || - check_add_overflow(offset, (unsigned long)vma_subpage_offset(vma), &offset) || - offset > obj->size || length > obj->size - offset) - return -EINVAL; - } - /* Revocation zaps the device anon mapping only. */ if (shmem->reclaimable_cpu_mappings && (!vma->vm_file || vma->vm_file->f_mapping != obj->dev->anon_inode->i_mapping)) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 626769cc3d61e..51e504321798d 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -97,7 +97,6 @@ static void apple_dart_test_before_attach(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, dart->regs); spin_lock_init(&dart->lock); mutex_init(&domain->init_lock); - master.dma_max = DMA_BIT_MASK(dart->ias); master.stream_maps[0].dart = dart; __set_bit(0, master.stream_maps[0].sidmap); /* of_xlate() starts each master with the window of its DART. */ diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index bddaa5582584a..ab61aa391c55b 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -2275,18 +2275,6 @@ static void apple_dart_set_locked_window(struct apple_dart_domain *domain, WRITE_ONCE(dart->locked_window[sid], start); } -/* Record the DVA window of @domain for invalidates of @stream_map's streams. */ -static void apple_dart_set_locked_window(struct apple_dart_domain *domain, - struct apple_dart_stream_map *stream_map) -{ - struct apple_dart *dart = stream_map->dart; - u64 start = domain->domain.geometry.aperture_start + domain->dma_offset; - int sid; - - for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) - WRITE_ONCE(dart->locked_window[sid], start); -} - static int apple_dart_setup_translation_locked(struct apple_dart_domain *domain, struct apple_dart_stream_map *stream_map) diff --git a/drivers/net/wireless/mediatek/mt7932/channels.h b/drivers/net/wireless/mediatek/mt7932/channels.h index 76fa712d19cd5..a1323f729757e 100644 --- a/drivers/net/wireless/mediatek/mt7932/channels.h +++ b/drivers/net/wireless/mediatek/mt7932/channels.h @@ -131,59 +131,4 @@ static inline unsigned int mt7932_channel_cal_centre(unsigned int primary) return primary == 165 ? 167 : mt7932_channel_ht40_centre(primary); } -/* The nth 5 GHz primary, in the order the wiphy advertises them. */ -static inline unsigned int mt7932_channel_5g(unsigned int index) -{ - if (index < 8) - return 36 + 4 * index; - if (index < 20) - return 100 + 4 * (index - 8); - return 149 + 4 * (index - 20); -} - -/* Offset of a 5 GHz primary within its 80 MHz block numbering. */ -static inline unsigned int mt7932_channel_slot(unsigned int primary, unsigned int *base) -{ - *base = primary >= 149 ? 149 : primary >= 100 ? 100 : 36; - return (primary - *base) / 4; -} - -/* 40 MHz pair centre of a 5 GHz primary; 0 where no pair exists (165). */ -static inline unsigned int mt7932_channel_ht40_centre(unsigned int primary) -{ - unsigned int base, slot; - - if (mt7932_channel_band(primary) != 2 || primary == 165) - return 0; - slot = mt7932_channel_slot(primary, &base); - return base + 4 * (slot & ~1U) + 2; -} - -/* 80 MHz block centre of a 5 GHz primary; 0 where no block exists (165). */ -static inline unsigned int mt7932_channel_vht80_centre(unsigned int primary) -{ - unsigned int base, slot; - - if (mt7932_channel_band(primary) != 2 || primary == 165) - return 0; - slot = mt7932_channel_slot(primary, &base); - return base + 4 * (slot & ~3U) + 6; -} - -/* A 5 GHz primary or a 40/80 MHz centre that calibration can be asked for. */ -static inline bool mt7932_channel_5g_cal(unsigned int channel) -{ - if ((channel >= 36 && channel <= 64) || (channel >= 100 && channel <= 144)) - return !(channel % 2); - return channel >= 149 && channel <= 167 && (channel % 2); -} - -/* Calibration centre for a 5 GHz primary. 165 has no 40 MHz pair; its own - * calibration package carries the record for the 165/169 centre, 167. - */ -static inline unsigned int mt7932_channel_cal_centre(unsigned int primary) -{ - return primary == 165 ? 167 : mt7932_channel_ht40_centre(primary); -} - #endif diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 84b92bd79d116..1722096ec2177 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -218,6 +218,7 @@ struct apple_nvme { /* Protects reset_ready against RTKit crash callbacks. */ spinlock_t event_lock; bool reset_ready; + bool handoff_disabled; struct list_head quarantine_node; resource_size_t controller_start; bool quarantined; @@ -344,14 +345,33 @@ static bool apple_nvme_can_adopt_rtkit(struct apple_nvme *anv) return false; cc = readl(anv->mmio_nvme + NVME_REG_CC); csts = readl(anv->mmio_nvme + NVME_REG_CSTS); + if (csts & NVME_CSTS_CFS) + return false; /* * CPU_RUN and BOOT_STATUS alone survive inactive sessions. A disabled * controller requires the producer's explicit live-session contract. * Borrowed buffers still require reserved RAM and protected SART ranges. */ - return (cc & NVME_CC_ENABLE) && (csts & NVME_CSTS_RDY) && - !(csts & NVME_CSTS_CFS); + if (anv->handoff_disabled) + return !(cc & NVME_CC_ENABLE) && !(csts & NVME_CSTS_RDY); + return (cc & NVME_CC_ENABLE) && (csts & NVME_CSTS_RDY); +} + +static int apple_nvme_read_handoff(struct apple_nvme *anv) +{ + const char *handoff; + int ret; + + ret = of_property_read_string(anv->dev->of_node, "apple,rtkit-handoff", + &handoff); + if (ret == -EINVAL) + return 0; + if (ret || !anv->hw->needs_ioq_registers || + strcmp(handoff, "nvme-disabled-v1")) + return -EINVAL; + anv->handoff_disabled = true; + return apple_nvme_can_adopt_rtkit(anv) ? 0 : -EIO; } static inline struct apple_nvme *queue_to_apple_nvme(struct apple_nvme_queue *q) @@ -2300,6 +2320,12 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) anv->mmio_nvmmu = anv->mmio_nvme; } + ret = apple_nvme_read_handoff(anv); + if (ret) { + dev_err_probe(dev, ret, "Invalid live firmware handoff\n"); + goto out_detach_genpd; + } + if (anv->hw->has_lsq_nvmmu) { void __iomem *lsq = anv->hw->lsq_in_nvmmu ? anv->mmio_nvmmu : anv->mmio_nvme; diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index cae4c76df6e3a..b2f9b5113782f 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -77,30 +77,6 @@ config APPLE_PMP_REPORT_START_KUNIT_TEST start or a consumer waits. The report region is simulated in memory. Only useful for kernel developers. -config APPLE_PMC - bool "Apple PMC voter interface" - depends on OF - depends on PM - select PM_GENERIC_DOMAINS - default ARCH_APPLE - help - The PMC in some Apple SoCs arbitrates the performance states of - shared rails, such as the DRAM controller and the SoC fabric, - between voter agents. This driver enables the static agents and - provides power domains that enable a dynamic agent, such as the - GPU, while its device is powered on. It can also limit the DRAM - controller rail as a thermal cooling device. - -config APPLE_PMC_KUNIT_TEST - bool "KUnit tests for the Apple PMC voter interface" if !KUNIT_ALL_TESTS - depends on APPLE_PMC && KUNIT=y - default KUNIT_ALL_TESTS - help - Unit tests for the Apple PMC driver. They check that enabling and - disabling a voter agent only changes that agent on each rail, and - that lifting a DCS limit gives the AP agent its own vote back. The - registers are simulated in memory. Only useful for kernel developers. - endif config APPLE_PMP_REPORT_KUNIT_TEST diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index e5410c80b6520..5dc828584ee43 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -271,9 +271,6 @@ static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned if (ps->firmware_owned || ps->inherited_on) return -EPERM; - if (ps->firmware_owned) - return -EPERM; - spin_lock_irqsave(&ps->genpd.slock, flags); if (ps->genpd.status == GENPD_STATE_OFF) diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index 5095635948472..3701f08c18487 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -23,7 +23,7 @@ use crate::SepData; static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); extern "C" { - fn sep_pm_register(event: unsafe extern "C" fn(bool) -> c_int) -> c_int; + fn sep_pm_register(event: unsafe extern "C" fn(bool)) -> c_int; fn sep_pm_unregister(); fn sep_pm_keep_domains(dev: *mut c_void) -> c_int; } @@ -80,10 +80,10 @@ fn sleep_event(sep: &impl SleepHolds, entering: bool) { /// # Safety /// Called by `pm_shim.c` from the PM notifier chain, in process context. -unsafe extern "C" fn pm_event(entering: bool) -> c_int { +unsafe extern "C" fn pm_event(entering: bool) { let ptr = SEP.load(Ordering::Acquire); if ptr.is_null() { - return 0; + return; } // SAFETY: produced by `into_foreign` in `register`, and reclaimed by // `unregister` only after `sep_pm_unregister` has waited out any running diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index 2ef96c5fd96eb..884ecbb354a06 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -14,12 +14,12 @@ #include "shim.h" -static int (*sep_pm_event)(bool entering); +static void (*sep_pm_event)(bool entering); static int sep_pm_notify(struct notifier_block *nb, unsigned long action, void *data) { - int (*event)(bool entering) = READ_ONCE(sep_pm_event); + void (*event)(bool entering) = READ_ONCE(sep_pm_event); if (!event) return NOTIFY_DONE; @@ -28,7 +28,8 @@ static int sep_pm_notify(struct notifier_block *nb, unsigned long action, case PM_SUSPEND_PREPARE: case PM_HIBERNATION_PREPARE: case PM_RESTORE_PREPARE: - return notifier_from_errno(event(true)); + event(true); + return NOTIFY_OK; case PM_POST_SUSPEND: case PM_POST_HIBERNATION: case PM_POST_RESTORE: @@ -43,7 +44,7 @@ static struct notifier_block sep_pm_nb = { .notifier_call = sep_pm_notify, }; -int sep_pm_register(int (*event)(bool entering)) +int sep_pm_register(void (*event)(bool entering)) { int ret; diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index cb360b7ae4394..88be672817b1c 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2163,83 +2163,6 @@ impl SepData { // Each step has said why it failed. Err(_) => false, } - if self.profile.nonce_session && !self.authenticate_sensor_session() { - return false; - } - self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); - self.note_sensor_recoveries(); - let _ = self.log_identity_count(c"after sensor serial registration"); - true - } - - fn sensor_platform_identity() -> Result<[u8; 12]> { - let node = kernel::of::root().ok_or(ENODEV)?; - let property: KVec = node.get_property(c"serial-number")?; - let length = property.iter().position(|byte| *byte == 0).ok_or(EINVAL)?; - let mut identity = [0; 12]; - if length == 0 || length > identity.len() || property.len() != length + 1 { - return Err(EINVAL); - } - identity[..length].copy_from_slice(&property[..length]); - Ok(identity) - } - - fn sensor_session_request(&self, opcode: u16, name: &'static CStr, bytes: &[u8]) -> bool { - match self.sbio_transfer_raw(opcode, name, bytes) { - Ok(reply) if reply.status.is_ok() => true, - Ok(reply) => { - dev_err!(self.dev, "sensor: {} refused with status {}\n", name, reply.status); - false - } - Err(error) => { - dev_err!(self.dev, "sensor: {} transfer failed: {:?}\n", name, error); - false - } - } - } - - fn authenticate_sensor_session(&self) -> bool { - let nonce = match sensor::session_nonce() { - Ok(nonce) => nonce, - Err(error) => { - dev_err!(self.dev, "sensor: nonce read failed: {:?}\n", error); - self.sensor_step_failed(c"nonce read"); - return false; - } - }; - self.sensor_trace(c"nonce read"); - if !self.sensor_session_request(0x81, c"SENSOR_NONCE", &nonce) { - self.sensor_step_failed(c"SENSOR_NONCE"); - return false; - } - self.sensor_trace(c"SENSOR_NONCE"); - - if !self.sensor_key_loaded.load(Relaxed) { - let firmware = match kernel::firmware::Firmware::request( - c"apple/msrk-j700.bin", &self.dev, - ) { - Ok(firmware) => firmware, - Err(error) => { - dev_err!( - self.dev, - "sensor: signed root-key container unavailable: {:?}\n", - error - ); - return false; - } - }; - if firmware.data().is_empty() - || !self.sensor_session_request(0x86, c"SENSOR_ROOT_KEY", firmware.data()) - { - self.sensor_step_failed(c"SENSOR_ROOT_KEY"); - return false; - } - self.sensor_key_loaded.store(true, Relaxed); - self.sensor_trace(c"SENSOR_ROOT_KEY"); - } - // Each identification starts a new sensor-side exchange, even though - // the signed key remains loaded for the lifetime of the enclave. - self.establish_session() } /// SENSOR_NONCE, the signed root key once a boot, and the session share. @@ -3373,7 +3296,7 @@ impl SepData { /// The system is about to suspend. End any capture in progress, hold new /// ones, and wait for the enrol or verify work to stop touching the sensor /// before devices go down. - pub(crate) fn sleep_prepare(&self) -> Result<()> { + pub(crate) fn sleep_prepare(&self) { let woke = { let mut session = self.bio_session.lock(); self.suspending.store(true, Relaxed); @@ -3396,15 +3319,14 @@ impl SepData { // the sensor again. dev_warn!( self.dev, - "Touch ID: capture did not drain within {} ms; aborting suspend\n", + "Touch ID: a capture was still running {} ms after the system started to sleep; continuing\n", SLEEP_DRAIN_MS ); - return Err(EBUSY); + return; } kernel::time::delay::fsleep(kernel::time::Delta::from_millis(10)); waited += 10; } - Ok(()) } /// The system has resumed, or the suspend was aborted. diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 3aaa9eedbc0c2..0c3d680185685 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -25,7 +25,7 @@ void sep_cancel_delayed_work_sync(void *work); /* -- pm_shim.c ---------------------------------------------------------- */ -int sep_pm_register(int (*event)(bool entering)); +int sep_pm_register(void (*event)(bool entering)); void sep_pm_unregister(void); int sep_pm_keep_domains(void *dev); diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 468c43b1ad434..998b6cffe6297 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -898,14 +898,6 @@ static int dwc3_apple_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "Failed to set up wakeup\n"); - /* - * Like a PCI xHCI controller, allow USB devices to wake the system by default. This has - * to be set before dwc3_core_probe(), which hands it on to the xHCI device. - */ - ret = devm_device_init_wakeup(dev); - if (ret) - return dev_err_probe(dev, ret, "Failed to set up wakeup\n"); - /* * On this platform, DWC3 can only be brought up after parts of the PHY have been * initialized with knowledge of the target mode and cable orientation from typec_set_mux. diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h index 5be24272ca059..8260e559b0c87 100644 --- a/drivers/usb/typec/tipd/cd321x-pm.h +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -96,20 +96,6 @@ static inline bool cd321x_pm_mux_done(struct cd321x_pm_state *pm) return retried; } -/* - * Ends the retries of a change that a mux held up, see cd321x_pm_mux_held(). - * Returns true if there were any. - */ -static inline bool cd321x_pm_mux_done(struct cd321x_pm_state *pm) -{ - bool retried = pm->mux_retry_ms; - - pm->mux_retry_ms = 0; - pm->mux_retries = 0; - pm->mux_since_ms = 0; - return retried; -} - static inline void cd321x_pm_prepare(struct cd321x_pm_state *pm) { if (pm->phase != CD321X_PM_REMOVED) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 0584e05874f4d..aa630b95dafe9 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -3054,12 +3054,6 @@ static void __exit tipd_core_exit(void) } module_exit(tipd_core_exit); -static void __exit tipd_core_exit(void) -{ - tipd_debugfs_exit(); -} -module_exit(tipd_core_exit); - MODULE_AUTHOR("Heikki Krogerus "); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("TI TPS6598x USB Power Delivery Controller Core Functions"); diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 166bc13621957..9da7326f684d2 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -1121,41 +1121,56 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, { struct vfio_iommu *iommu = iommu_data; struct vfio_iommu_group *group; - unsigned long pinned = 0; + int i, j, ret; + unsigned long remote_vaddr; + struct vfio_dma *dma; bool do_accounting; dma_addr_t iova_end; size_t iova_size; - int i, ret; if (!iommu || !pages || npage <= 0) return -EINVAL; - /* Callers recover the original byte offset from the returned page. */ - user_iova = ALIGN_DOWN(user_iova, PAGE_SIZE); + + /* Supported for v2 version only */ if (!iommu->v2) return -EACCES; + if (check_mul_overflow(npage, PAGE_SIZE, &iova_size) || check_add_overflow(user_iova, iova_size - 1, &iova_end)) return -EOVERFLOW; mutex_lock(&iommu->lock); + if (WARN_ONCE(iommu->vaddr_invalid_count, "vfio_pin_pages not allowed with VFIO_UPDATE_VADDR\n")) { ret = -EBUSY; goto pin_done; } + + /* Fail if no dma_umap notifier is registered */ if (list_empty(&iommu->device_list)) { ret = -EINVAL; goto pin_done; } + + /* + * If iommu capable domain exist in the container then all pages are + * already pinned and accounted. Accounting should be done if there is no + * iommu capable domain in the container. + */ do_accounting = list_empty(&iommu->domain_list); + for (i = 0; i < npage; i++) { - unsigned long done = 0; + unsigned long phys_pfn; + dma_addr_t iova; + struct vfio_pfn *vpfn; - pages[i] = NULL; - while (done < PAGE_SIZE) { - dma_addr_t iova = user_iova + PAGE_SIZE * i + done; - struct vfio_dma *dma = vfio_find_dma(iommu, iova, 1); - unsigned long length, pfn; + iova = user_iova + PAGE_SIZE * i; + dma = vfio_find_dma(iommu, iova, PAGE_SIZE); + if (!dma) { + ret = -EINVAL; + goto pin_unwind; + } if ((dma->prot & prot) != prot) { ret = -EPERM; @@ -1199,17 +1214,26 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, (iova - dma->iova) >> pgshift, 1); } } - ret = npage; + ret = i; + group = vfio_iommu_find_iommu_group(iommu, iommu_group); if (!group->pinned_page_dirty_scope) { group->pinned_page_dirty_scope = true; iommu->num_non_pinned_groups--; } + goto pin_done; + pin_unwind: - vfio_unpin_external_range(iommu, user_iova, pinned, do_accounting); - while (i >= 0) - pages[i--] = NULL; + pages[i] = NULL; + for (j = 0; j < i; j++) { + dma_addr_t iova; + + iova = user_iova + PAGE_SIZE * j; + dma = vfio_find_dma(iommu, iova, PAGE_SIZE); + vfio_unpin_page_external(dma, iova, do_accounting); + pages[j] = NULL; + } pin_done: mutex_unlock(&iommu->lock); return ret; @@ -1219,21 +1243,39 @@ static void vfio_iommu_type1_unpin_pages(void *iommu_data, dma_addr_t user_iova, int npage) { struct vfio_iommu *iommu = iommu_data; + bool do_accounting; dma_addr_t iova_end; size_t iova_size; - unsigned long unpinned; + int i; - if (WARN_ON(!iommu->v2) || WARN_ON(npage <= 0)) + /* Supported for v2 version only */ + if (WARN_ON(!iommu->v2)) return; - user_iova = ALIGN_DOWN(user_iova, PAGE_SIZE); + + if (WARN_ON(npage <= 0)) + return; + if (WARN_ON(check_mul_overflow(npage, PAGE_SIZE, &iova_size) || check_add_overflow(user_iova, iova_size - 1, &iova_end))) return; + mutex_lock(&iommu->lock); - unpinned = vfio_unpin_external_range(iommu, user_iova, iova_size, - list_empty(&iommu->domain_list)); + + do_accounting = list_empty(&iommu->domain_list); + for (i = 0; i < npage; i++) { + dma_addr_t iova = user_iova + PAGE_SIZE * i; + struct vfio_dma *dma; + + dma = vfio_find_dma(iommu, iova, PAGE_SIZE); + if (!dma) + break; + + vfio_unpin_page_external(dma, iova, do_accounting); + } + mutex_unlock(&iommu->lock); - WARN_ON(unpinned != iova_size); + + WARN_ON(i != npage); } #endif @@ -1913,27 +1955,25 @@ static int vfio_change_dma_owner(struct vfio_iommu *iommu, struct vfio_dma *dma, static int vfio_change_dma_owner(struct vfio_iommu *iommu, struct vfio_dma *dma, unsigned long vaddr) { - long old_pages = dma->locked_vm; + struct task_struct *task = current->group_leader; + struct mm_struct *mm = current->mm; + long npage = dma->locked_vm; + bool lock_cap; int ret; - if (mm == dma->mm) { - if (pages == old_pages) - return 0; - ret = mm_lock_acct(task, mm, lock_cap, (long)pages - old_pages); - if (!ret) - dma->locked_vm = pages; - return ret; - } - ret = mm_lock_acct(task, mm, lock_cap, pages); + if (mm == dma->mm) + return 0; + + lock_cap = capable(CAP_IPC_LOCK); + ret = mm_lock_acct(task, mm, lock_cap, npage); if (ret) return ret; - /* Metadata adoption can deduplicate charges, but not original PIN refs. */ - ret = mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -old_pages); - if (ret) { - mm_lock_acct(task, mm, lock_cap, -(long)pages); - return ret; + if (mmget_not_zero(dma->mm)) { + mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -npage); + mmput(dma->mm); } + if (dma->task != task) { put_task_struct(dma->task); dma->task = get_task_struct(task); @@ -1942,41 +1982,10 @@ static int vfio_change_dma_owner(struct vfio_iommu *iommu, struct vfio_dma *dma, dma->mm = mm; mmgrab(dma->mm); dma->lock_cap = lock_cap; - dma->locked_vm = pages; return 0; } #endif -static int vfio_change_dma_mm(struct vfio_dma *dma, struct task_struct *task, - struct mm_struct *mm, bool lock_cap) -{ - return vfio_change_dma_mm_pages(dma, task, mm, lock_cap, dma->locked_vm); -} - -static int vfio_change_dma_owner(struct vfio_iommu *iommu, struct vfio_dma *dma, - unsigned long vaddr) -{ - struct vfio_fragment_provider *provider; - int ret; - - if (dma->fragments || (mm_page_size(current->mm) == PAGE_SIZE && - IS_ALIGNED(vaddr, PAGE_SIZE))) - return vfio_change_dma_mm(dma, current->group_leader, current->mm, - capable(CAP_IPC_LOCK)); - - /* Prepare only metadata: failure must preserve every legacy pin/charge. */ - provider = vfio_fragments_prepare_native(iommu, dma); - if (IS_ERR(provider)) - return PTR_ERR(provider); - ret = vfio_change_dma_mm_pages(dma, current->group_leader, current->mm, - capable(CAP_IPC_LOCK), vfio_fragments_native_pages(provider)); - if (ret) - vfio_fragments_abort_native(provider); - else - vfio_fragments_commit_native(dma, provider); - return ret; -} - static int vfio_dma_do_map(struct vfio_iommu *iommu, struct vfio_iommu_type1_dma_map *map) { diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 1888699c67b5c..5f4415432aecf 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -610,15 +610,6 @@ static inline u32 bpf_obj_uptr_extra_size(const struct btf_record *rec) #endif } -static inline u32 bpf_obj_uptr_extra_size(const struct btf_record *rec) -{ -#ifdef CONFIG_MM_SUBPAGE - return IS_ERR_OR_NULL(rec) ? 0 : rec->uptr_extra_size; -#else - return 0; -#endif -} - static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) { unsigned long *src_uptr, *dst_uptr; diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 7ecafabcf58ad..85a4a10f0dbb3 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -57,14 +57,6 @@ struct apple_rtkit_inherited_buffer { /* This client requires endpoint 7 setup before IOP power can be acknowledged. */ #define APPLE_RTKIT_NEEDS_COREANALYTICS BIT(1) -/* - * The co-processor places its shared buffers itself. A buffer request that - * carries an IOVA is set up with shmem_setup and acknowledged by sending the - * request back unchanged. The crashlog endpoint is started while the endpoint - * map is handled, before the map is acknowledged. - */ -#define APPLE_RTKIT_COPROC_PLACES_BUFFERS BIT(0) - /* * Struct to represent implementation-specific RTKit operations. * diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 98a603b72b7c0..ebfbe3288749a 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -81,10 +81,6 @@ static bool hpage_acct_unref(struct io_ring_ctx *ctx, struct page *hpage) static void io_fragment_account_put(struct io_fragment_account *account); #endif -#ifdef CONFIG_MM_SUBPAGE -static void io_fragment_account_put(struct io_fragment_account *account); -#endif - /* only define max */ #define IORING_MAX_FIXED_FILES (1U << 20) #define IORING_MAX_REG_BUFFERS (1U << 14) diff --git a/kernel/fork.c b/kernel/fork.c index 9ef443277fce9..4971fc539da83 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1194,11 +1194,6 @@ static struct mm_struct *__mm_alloc(bool for_exec, unsigned int page_shift) return mm_init(mm, current); } -struct mm_struct *mm_alloc(void) -{ - return __mm_alloc(false, 0); -} - struct mm_struct *mm_alloc(void) { return __mm_alloc(false, 0); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index c07a917f6d7be..6e24eef5e995b 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -523,44 +523,6 @@ static void futex_put_page_ref(struct futex_page_ref *ref) put_page(ref->page); } -/* Keep a typed slot reference alive until the mapping identity is sampled. */ -struct futex_page_ref { - struct page *page; -#ifdef CONFIG_MM_SUBPAGE - struct user_page_fragment fragment; -#endif -}; - -static int futex_get_page_ref(struct mm_struct *mm, unsigned long address, - unsigned long uaddr, unsigned int size, unsigned int flags, - struct futex_page_ref *ref) -{ -#ifdef CONFIG_MM_SUBPAGE - if (mm_page_size(mm) != PAGE_SIZE) { - int ret = get_user_fragments_remote(mm, uaddr, size, flags, - &ref->fragment, 1); - - if (ret != 1) - return ret ?: -EFAULT; - ref->page = folio_page(ref->fragment.folio, - ref->fragment.offset >> PAGE_SHIFT); - return 1; - } -#endif - return get_user_pages_fast(address, 1, flags, &ref->page); -} - -static void futex_put_page_ref(struct futex_page_ref *ref) -{ -#ifdef CONFIG_MM_SUBPAGE - if (ref->fragment.folio) { - release_user_fragments(&ref->fragment, 1, false); - return; - } -#endif - put_page(ref->page); -} - /** * get_futex_key() - Get parameters which are the keys for a futex * @uaddr: virtual address of the futex diff --git a/mm/filemap.c b/mm/filemap.c index 4708139ae8318..ec459766620f4 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3972,11 +3972,6 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, unsigned long rss = 0; unsigned int nr_pages = 0, folio_type; -#ifdef CONFIG_MM_SUBPAGE - if (mm_page_size(vma->vm_mm) != PAGE_SIZE) - return filemap_map_user_pages(vmf, start_pgoff, end_pgoff); -#endif - #ifdef CONFIG_MM_SUBPAGE if (mm_page_size(vma->vm_mm) != PAGE_SIZE) return filemap_map_user_pages(vmf, start_pgoff, end_pgoff); diff --git a/mm/internal.h b/mm/internal.h index b52f8249a1011..4eccdbab5c9b5 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -534,10 +534,6 @@ static inline pte_t pte_move_swp_offset(pte_t pte, long delta) if (pte_swp_uffd(pte)) new = pte_swp_mkuffd(new); -#ifdef CONFIG_MM_SUBPAGE - new = pte_swp_set_subpage_offset(new, pte_swp_subpage_offset(pte)); -#endif - #ifdef CONFIG_MM_SUBPAGE new = pte_swp_set_subpage_offset(new, pte_swp_subpage_offset(pte)); #endif diff --git a/mm/memory.c b/mm/memory.c index 26d4167cf1e93..ef71f8b2e406e 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2987,10 +2987,7 @@ static int insert_pages(struct vm_area_struct *vma, unsigned long addr, static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte, unsigned long addr, struct page *page, pgprot_t prot) { - struct mm_struct *mm = vma->vm_mm; - unsigned long remaining = *length; - unsigned long size = mm_page_size(mm); - int ret = 0; + int err; err = validate_page_before_insert(vma, page); if (err) @@ -2998,58 +2995,67 @@ static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte, return insert_page_into_pte_locked(vma, pte, addr, page, prot, false, 0); } +/* insert_pages() amortizes the cost of spinlock operations + * when inserting pages in a loop. + */ static int insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num, pgprot_t prot) { - unsigned long length; + pmd_t *pmd = NULL; + pte_t *start_pte, *pte; + spinlock_t *pte_lock; + struct mm_struct *const mm = vma->vm_mm; + unsigned long curr_page_idx = 0; + unsigned long remaining_pages_total = *num; + unsigned long pages_to_write_in_pmd; int ret; +more: + ret = -EFAULT; + pmd = walk_to_pmd(mm, addr); + if (!pmd) + goto out; - if (check_mul_overflow(*num, PAGE_SIZE, &length)) - return -EOVERFLOW; - ret = insert_page_range(vma, addr, pages, 0, &length, prot); - /* A partially inserted native page is still an incomplete input page. */ - *num = DIV_ROUND_UP(length, PAGE_SIZE); - return ret; -} -#endif + pages_to_write_in_pmd = min_t(unsigned long, + remaining_pages_total, PTRS_PER_PTE - pte_index(addr)); -/** - * vm_insert_pages_range - map a byte range from a native-page array - * @vma: target VMA, held under the mmap write lock - * @addr: first user address - * @pages: native backing pages - * @nr_pages: size of @pages - * @offset: byte offset within the backing array - * @length: bytes to map - * - * Address, offset and length must match the target mm's granule. A failed - * insertion can leave a prefix mapped, as with vm_insert_pages(); the caller - * must unmap that prefix when abandoning the mapping. Physical ownership - * remains in native pages. Each installed user leaf acquires its own ref. - */ -int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, - struct page **pages, unsigned long nr_pages, - unsigned long offset, unsigned long length) -{ - unsigned long bytes, end, size = mm_page_size(vma->vm_mm); + /* Allocate the PTE if necessary; takes PMD lock once only. */ + ret = -ENOMEM; + if (pte_alloc(mm, pmd)) + goto out; - if (!IS_ALIGNED(addr | offset | length, size) || - check_mul_overflow(nr_pages, PAGE_SIZE, &bytes) || - offset > bytes || length > bytes - offset || - check_add_overflow(addr, length, &end) || - !range_in_vma(vma, addr, end)) - return -EINVAL; - if (!length) - return 0; - if (vma->vm_flags & VM_PFNMAP) - return -EINVAL; - if (!(vma->vm_flags & VM_MIXEDMAP)) { - mmap_assert_write_locked(vma->vm_mm); - vm_flags_set(vma, VM_MIXEDMAP); + while (pages_to_write_in_pmd) { + int pte_idx = 0; + const int batch_size = min_t(int, pages_to_write_in_pmd, 8); + + start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock); + if (!start_pte) { + ret = -EFAULT; + goto out; + } + for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) { + int err = insert_page_in_batch_locked(vma, pte, + addr, pages[curr_page_idx], prot); + if (unlikely(err)) { + pte_unmap_unlock(start_pte, pte_lock); + ret = err; + remaining_pages_total -= pte_idx; + goto out; + } + addr += PAGE_SIZE; + ++curr_page_idx; + } + pte_unmap_unlock(start_pte, pte_lock); + pages_to_write_in_pmd -= batch_size; + remaining_pages_total -= batch_size; } - return insert_page_range(vma, addr, pages, offset, &length, vma->vm_page_prot); + if (remaining_pages_total) + goto more; + ret = 0; +out: + *num = remaining_pages_total; + return ret; } -EXPORT_SYMBOL_GPL(vm_insert_pages_range); +#endif /** * vm_insert_pages_range - map a byte range from a native-page array @@ -3119,10 +3125,7 @@ int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, #else const unsigned long end = addr + PAGE_SIZE * nr_pages; - if (!IS_ALIGNED(addr, mm_page_size(vma->vm_mm)) || - check_mul_overflow(nr_pages, PAGE_SIZE, &length) || - check_add_overflow(addr, length, &end) || - !range_in_vma(vma, addr, end)) + if (!range_in_vma(vma, addr, end)) return -EFAULT; #endif if (!(vma->vm_flags & VM_MIXEDMAP)) { @@ -3280,14 +3283,12 @@ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long count = vma_pages(vma); unsigned long uaddr = vma->vm_start; - if (offset >= num || - check_mul_overflow(num - offset, PAGE_SIZE, &available) || - subpage_offset >= PAGE_SIZE || - check_mul_overflow(offset, PAGE_SIZE, &bytes) || - check_add_overflow(bytes, (unsigned long)subpage_offset, &bytes)) + /* Fail if the user requested offset is beyond the end of the object */ + if (offset >= num) return -ENXIO; - if (vma->vm_end - vma->vm_start > - available - subpage_offset) + + /* Fail if the user requested size exceeds available object size */ + if (count > num - offset) return -ENXIO; return vm_insert_pages(vma, uaddr, pages + offset, &count); @@ -3465,9 +3466,9 @@ static vm_fault_t __vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, * Context: Process context. May allocate using %GFP_KERNEL. * Return: vm_fault_t value. */ -static vm_fault_t __vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, +vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot, - bool mkwrite, unsigned int offset) + bool mkwrite) { return __vmf_insert_pfn_prot_mkwrite(vma, addr, pfn, pgprot, mkwrite, 0); } @@ -3988,37 +3989,35 @@ static int __simple_ioremap_prep(struct mm_struct *mm, unsigned long vm_len, #else static int __simple_ioremap_prep(unsigned long vm_len, pgoff_t vm_pgoff, phys_addr_t start_phys, unsigned long size, - struct vm_page_offset *physical) + unsigned long *pfnp) { - unsigned long granule = mm_page_size(mm); - unsigned long offset, bias = start_phys & (granule - 1); + unsigned long pfn, pages; /* Check that the physical memory area passed in looks valid */ if (start_phys + size < start_phys) return -EINVAL; /* - * Preserve the historical allowance for unaligned I/O resources, using - * the target MM's granule for the prefix and final partial mapping. + * You *really* shouldn't map things that aren't page-aligned, + * but we've historically allowed it because IO memory might + * just have smaller alignment. */ - if (check_add_overflow(size, bias, &size) || size > ULONG_MAX - (granule - 1)) - return -EINVAL; - size = ALIGN(size, granule); - start_phys -= bias; - if (start_phys + size < start_phys) - return -EINVAL; - if (pos.offset >= PAGE_SIZE || !IS_ALIGNED(pos.offset, granule) || - check_mul_overflow(pos.index, PAGE_SIZE, &offset) || - check_add_overflow(offset, (unsigned long)pos.offset, &offset)) + size += start_phys & ~PAGE_MASK; + pfn = start_phys >> PAGE_SHIFT; + pages = (size + ~PAGE_MASK) >> PAGE_SHIFT; + if (pfn + pages < pfn) return -EINVAL; - if (offset > size || vm_len > size - offset) + /* We start the mapping 'vm_pgoff' pages into the area */ + if (vm_pgoff > pages) return -EINVAL; + pfn += vm_pgoff; + pages -= vm_pgoff; - start_phys += offset; - physical->index = PHYS_PFN(start_phys); - physical->offset = offset_in_page(start_phys); - if (PFN_PHYS(physical->index) + physical->offset != start_phys) + /* Can we fit all of the mapping? */ + if ((vm_len >> PAGE_SHIFT) > pages) return -EINVAL; + + *pfnp = pfn; return 0; } #endif @@ -4045,11 +4044,8 @@ int simple_ioremap_prepare(struct vm_area_desc *desc) unsigned long pfn; int err; -#ifdef CONFIG_MM_SUBPAGE - pos.offset = desc->subpage_offset; -#endif - err = __simple_ioremap_prep(desc->mm, vma_desc_size(desc), pos, - start, size, &physical); + err = __simple_ioremap_prep(vma_desc_size(desc), desc->pgoff, + start, size, &pfn); if (err) return err; @@ -4095,8 +4091,7 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long unsigned long pfn; int err; - err = __simple_ioremap_prep(vma->vm_mm, vm_len, vma_page_offset_at(vma, vm_start), - start, len, &physical); + err = __simple_ioremap_prep(vm_len, vma->vm_pgoff, start, len, &pfn); if (err) return err; @@ -5386,29 +5381,6 @@ static inline void unmap_mapping_range_tree(struct address_space *mapping, } #endif -#ifdef CONFIG_MM_SUBPAGE - if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { - unsigned long end = vma->vm_end; - - start = vma_address(vma, first_index, last_index - first_index + 1); - /* The interval tree guarantees a nonempty intersection. */ - VM_BUG_ON_VMA(IS_ERR_VALUE(start), vma); - if (last_index < vma_last_pgoff(vma)) - end = vma_address_at_offset(vma, (struct vm_page_offset) { - .index = last_index + 1, - }); - size = end - start; - } else if (mm_page_size(vma->vm_mm) > PAGE_SIZE) { - unsigned long granule = mm_page_size(vma->vm_mm); - unsigned long end = min(start + size, vma->vm_end); - - /* A native-page invalidation removes every overlapping leaf. */ - start = max(round_down(start, granule), vma->vm_start); - end = min(round_up(end, granule), vma->vm_end); - size = end - start; - } -#endif - tlb_gather_mmu(&tlb, vma->vm_mm); zap_vma_range_batched(&tlb, vma, start, size, details); tlb_finish_mmu(&tlb); diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 2dfe3eb48d864..a5e6b403474a2 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -3278,10 +3278,6 @@ int mpol_set_shared_policy(struct shared_policy *sp, struct sp_node *new = NULL; int err; - /* Never truncate a partial cache-page policy update to zero or a neighbour. */ - if (vma_subpage_offset(vma) || ((vma->vm_end - vma->vm_start) & ~PAGE_MASK)) - return -EINVAL; - /* Never truncate a partial cache-page policy update to zero or a neighbour. */ if (vma_subpage_offset(vma) || ((vma->vm_end - vma->vm_start) & ~PAGE_MASK)) return -EINVAL; diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index 97be489bce65b..d045c04fdf1fb 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -115,18 +115,6 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr) else ptent = ptep_get(pvmw->pte); -#ifdef CONFIG_MM_SUBPAGE - if (pvmw->flags & PVMW_SUBPAGE) { - if (!(pvmw->flags & PVMW_MIGRATION)) - return pte_present(ptent) && - pte_phys_mm(pvmw->vma->vm_mm, ptent) == - PFN_PHYS(pvmw->pfn) + pvmw->subpage_offset; - if (pte_swp_subpage_offset(ptent) != pvmw->subpage_offset) - return false; - /* The common migration check below validates the native PFN. */ - } -#endif - #ifdef CONFIG_MM_SUBPAGE if (pvmw->flags & PVMW_SUBPAGE) { if (!(pvmw->flags & PVMW_MIGRATION)) diff --git a/mm/util.c b/mm/util.c index 6c8ee0f59e645..2c123a9aaac87 100644 --- a/mm/util.c +++ b/mm/util.c @@ -565,35 +565,27 @@ int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, const struct task_struct *task, bool bypass_rlim) { unsigned long locked_vm, limit; - unsigned int shift = PAGE_SHIFT - MM_ACCOUNT_SHIFT; int ret = 0; mmap_assert_write_locked(mm); - /* This exported API receives native physical-page counts. */ - if (pages > ULONG_MAX >> shift) - return -ENOMEM; - pages <<= shift; locked_vm = mm->locked_vm; if (inc) { if (!bypass_rlim) { - limit = task_rlimit(task, RLIMIT_MEMLOCK) >> MM_ACCOUNT_SHIFT; - if (pages > limit || locked_vm > limit - pages) + limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT; + if (locked_vm + pages > limit) ret = -ENOMEM; } - if (!ret && pages > ULONG_MAX - locked_vm) - ret = -ENOMEM; if (!ret) mm->locked_vm = locked_vm + pages; } else { - if (WARN_ON_ONCE(pages > locked_vm)) - return -EINVAL; + WARN_ON_ONCE(pages > locked_vm); mm->locked_vm = locked_vm - pages; } pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid, - (void *)_RET_IP_, (inc) ? '+' : '-', pages << MM_ACCOUNT_SHIFT, - locked_vm << MM_ACCOUNT_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK), + (void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT, + locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK), ret ? " - exceeded" : ""); return ret; diff --git a/mm/vma.c b/mm/vma.c index 00d76c63dfa9f..383802d275249 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -3087,10 +3087,6 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, .index = addr >> PAGE_SHIFT, .offset = addr & ~PAGE_MASK, }); - vmg_set_page_offset(&vmg, (struct vm_page_offset) { - .index = addr >> PAGE_SHIFT, .offset = addr & ~PAGE_MASK, - }); - vmg.prev = vma; /* vmi is positioned at prev, which this mode expects. */ vmg.just_expand = true; diff --git a/rust/helpers/barrier.c b/rust/helpers/barrier.c index f3dd9c742291b..dbc7a3017c780 100644 --- a/rust/helpers/barrier.c +++ b/rust/helpers/barrier.c @@ -46,13 +46,3 @@ __rust_helper void rust_helper_smp_rmb(void) { smp_rmb(); } - -__rust_helper void rust_helper_dma_wmb(void) -{ - dma_wmb(); -} - -__rust_helper void rust_helper_dma_rmb(void) -{ - dma_rmb(); -} diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 62796350f7666..3b76d329688db 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -46,7 +46,6 @@ use core::{ self, NonNull, // }, - sync::atomic::{AtomicBool, Ordering}, }; #[cfg(CONFIG_DRM_LEGACY)] @@ -509,10 +508,7 @@ impl Deref for Device { type Target = T::Data; fn deref(&self) -> &Self::Target { - debug_assert!(self.data_initialized.load(Ordering::Acquire)); - // SAFETY: Safe constructors initialize data; new_uninit() requires callers - // to prevent data access until init_data() succeeds. - unsafe { &*self.data.get().cast::() } + &self.data } } @@ -627,8 +623,7 @@ where unsafe fn work_container_of(ptr: *mut Work) -> *mut Self { // SAFETY: The caller promises that `ptr` points at a `Work` field in // `T::Data`. - let data_ptr = unsafe { T::Data::work_container_of(ptr) } - .cast::>>(); + let data_ptr = unsafe { T::Data::work_container_of(ptr) }; // SAFETY: `T::Data` is stored as the `data` field in `Device`. unsafe { crate::container_of!(data_ptr, Self, data) } diff --git a/rust/kernel/sync/barrier.rs b/rust/kernel/sync/barrier.rs index 46ae6ae0d6ad2..1180695d533ad 100644 --- a/rust/kernel/sync/barrier.rs +++ b/rust/kernel/sync/barrier.rs @@ -130,23 +130,3 @@ pub fn smp_mb>(_: T) { pub fn dma_mb>(_: T) { T::run() } - -/// A write-write memory barrier for memory shared with a device. -/// -/// Orders the CPU's writes to memory a device reads, such as a coherent DMA buffer, before its -/// later writes, including a relaxed MMIO write that tells the device to read that memory. -#[inline(always)] -pub fn dma_wmb() { - // SAFETY: `dma_wmb()` is safe to call. - unsafe { bindings::dma_wmb() }; -} - -/// A read-read memory barrier for memory shared with a device. -/// -/// Orders the CPU's reads of memory a device writes, such as a coherent DMA buffer, after its -/// earlier reads, including a relaxed MMIO read that told it the device had written that memory. -#[inline(always)] -pub fn dma_rmb() { - // SAFETY: `dma_rmb()` is safe to call. - unsafe { bindings::dma_rmb() }; -} diff --git a/security/tomoyo/domain.c b/security/tomoyo/domain.c index 5aa3fac838fef..c0bb9305d05d5 100644 --- a/security/tomoyo/domain.c +++ b/security/tomoyo/domain.c @@ -945,38 +945,46 @@ bool tomoyo_dump_page(struct linux_binprm *bprm, unsigned long pos, bool tomoyo_dump_page(struct linux_binprm *bprm, unsigned long pos, struct tomoyo_page_dump *dump) { - const unsigned int offset = offset_in_page(pos); - const unsigned int length = PAGE_SIZE - offset; + struct page *page; +#ifdef CONFIG_MMU + int ret; +#endif /* dump->data is released by tomoyo_find_next_domain(). */ if (!dump->data) { dump->data = kzalloc(PAGE_SIZE, GFP_NOFS); if (!dump->data) return false; - dump->pos = ULONG_MAX; } - if (pos >= dump->pos && (pos & PAGE_MASK) == (dump->pos & PAGE_MASK)) - return true; - /* A failed partial copy must not leave the previous cache valid. */ - dump->pos = ULONG_MAX; + /* Same with get_arg_page(bprm, pos, 0) in fs/exec.c */ #ifdef CONFIG_MMU /* - * Inspect the nascent MM's virtual bytes. Smaller userspace mappings - * can use several independently retained fragments within this range. + * This is called at execve() time in order to dig around + * in the argv/environment of the new process + * (represented by bprm). */ - if (access_remote_vm(bprm->mm, pos, dump->data + offset, length, - FOLL_FORCE) != length) + mmap_read_lock(bprm->mm); + ret = get_user_pages_remote(bprm->mm, pos, 1, + FOLL_FORCE, &page, NULL); + mmap_read_unlock(bprm->mm); + if (ret <= 0) return false; #else - { - struct page *page = bprm->page[pos / PAGE_SIZE]; + page = bprm->page[pos / PAGE_SIZE]; +#endif + if (page != dump->page) { + const unsigned int offset = pos % PAGE_SIZE; char *kaddr = kmap_local_page(page); - memcpy(dump->data + offset, kaddr + offset, length); + dump->page = page; + memcpy(dump->data + offset, kaddr + offset, + PAGE_SIZE - offset); kunmap_local(kaddr); } + /* Same with put_arg_page(page) in fs/exec.c */ +#ifdef CONFIG_MMU + put_page(page); #endif - dump->pos = pos; return true; } #endif From b405438339a0f0e71c4fb2bf380b6cbf2bb8ef5f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 00:44:57 +1100 Subject: [PATCH 1269/1579] selftests: align host fixtures with the current 7.3 driver interfaces Model DCP dispatch and the DART synchronization argument, current G17 state and queue-ID fields, RTKit coreanalytics cleanup, and DWC3 runtime-PM ownership. Keep production bodies under test and retain the existing assertions. Signed-off-by: Ryan Murray Assisted-by: LLM --- tools/asahi/apple-dart-range-tests.py | 10 +++++++--- tools/asahi/g17-admission-deadline-tests.py | 4 ++-- tools/asahi/g17-owner-release-order-tests.py | 4 ++-- tools/asahi/g17-resource-tests.py | 2 +- tools/asahi/rtkit-oslog-tests.py | 6 +++--- .../selftests/drivers/gpu/apple-dcp/dwc3/mocks.c | 16 ++++++++++++++++ 6 files changed, 31 insertions(+), 11 deletions(-) diff --git a/tools/asahi/apple-dart-range-tests.py b/tools/asahi/apple-dart-range-tests.py index 6fa006381918c..d389d95be5920 100644 --- a/tools/asahi/apple-dart-range-tests.py +++ b/tools/asahi/apple-dart-range-tests.py @@ -31,6 +31,7 @@ def main(): #include #include typedef uint64_t u64; +typedef uint32_t u32; #define BITS_TO_LONGS(n) 1 #define atomic_long_read(x) (*(x)) #define bitmap_empty(x, n) (!(x)[0]) @@ -45,7 +46,7 @@ def main(): struct apple_dart { int num_streams, locked, version; void *dev; struct apple_dart_hw *hw; }; struct apple_dart_atomic_stream_map { struct apple_dart *dart; unsigned long sidmap[1]; }; struct apple_dart_domain { - void *pgtbl_ops; struct apple_dart_atomic_stream_map streams[2]; int count; + void *pgtbl_ops, *dcp; struct apple_dart_atomic_stream_map streams[2]; int count; u64 dma_offset, mask; }; #define for_each_stream_map(i,d,m) for ((i)=0; (i)<(d)->count && ((m)=&(d)->streams[i],1); (i)++) @@ -53,8 +54,11 @@ def main(): static u64 seen_first, seen_last; static int pm_runtime_resume_and_get(void *dev) { if (power_error) return power_error; powered++; return 0; } static void pm_runtime_put(void *dev) { assert(powered>0); powered--; } -static int apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, struct apple_dart_stream_map *stream) { - syncs++; return sync_error; +static int apple_dart_dcp_domain_flush(struct apple_dart_domain *domain) { + (void)domain; assert(!"unexpected DCP dispatch in range fixture"); return -EINVAL; +} +static int apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, struct apple_dart_stream_map *stream, bool sync) { + assert(sync); syncs++; return sync_error; } static int apple_dart_t8110_hw_tlb_command_range(struct apple_dart_stream_map *stream, int command, bool range, u64 first, u64 last) { diff --git a/tools/asahi/g17-admission-deadline-tests.py b/tools/asahi/g17-admission-deadline-tests.py index 5002f70997307..ac92be72e65ff 100644 --- a/tools/asahi/g17-admission-deadline-tests.py +++ b/tools/asahi/g17-admission-deadline-tests.py @@ -40,7 +40,7 @@ def fixture(): const QUARANTINE_WAIT_MS:u32=1000; #[derive(Clone,Copy,PartialEq)] enum Plan {Any,Conflict} #[derive(Clone,Copy)] struct Request<'a>(&'a ()); -struct State {owners:[u64;2],quarantined:u64,next_ticket:u64,waiters:usize,conflict:bool,ready:bool} +struct State {owners:[u64;2],quarantined:u64,spared:u64,next_ticket:u64,waiters:usize,conflict:bool,ready:bool} impl State { fn exhausted(&self)->bool {false} fn refusal(&self,_:u64)->Result {Ok(())} @@ -69,7 +69,7 @@ def fixture(): ''' + item(source, 'fn wait_slice_ms(') + '\nimpl Pool {\n' + item(source, 'pub(crate) fn acquire_render(') + r''' } fn new_pool(conflict:bool,quarantined:u64)->Arc { - Arc::new(Pool{state:Lock(Mutex::new(State{owners:[0,1],quarantined,next_ticket:1,waiters:0,conflict,ready:false})),released:CondVar}) + Arc::new(Pool{state:Lock(Mutex::new(State{owners:[0,1],quarantined,spared:0,next_ticket:1,waiters:0,conflict,ready:false})),released:CondVar}) } fn reset(mode:usize){NOW.store(0,Ordering::SeqCst);WAITS.store(0,Ordering::SeqCst);MODE.store(mode,Ordering::SeqCst);} fn main(){ diff --git a/tools/asahi/g17-owner-release-order-tests.py b/tools/asahi/g17-owner-release-order-tests.py index 201f525718950..fd0e2c4a80936 100755 --- a/tools/asahi/g17-owner-release-order-tests.py +++ b/tools/asahi/g17-owner-release-order-tests.py @@ -118,7 +118,7 @@ def fixture(): struct Ticket { ordinal: u64 } struct Active { ticket: Ticket } struct Packet { context: Arc } -struct Pair { executed: u64, switch: Option<(u64, u64)>, replay: Option<()>, active: Vec, idle: bool } +struct Pair { executed: u64, switch: Option<(u64, u64)>, replay: Vec<()>, active: Vec, idle: bool } impl Pair { SWITCH /// Host stand-in for a successful publication of `ordinal` (stand-in, not production). @@ -177,7 +177,7 @@ def fixture(): assert!(!table.releasing(&quiet)); // Switch tracking across an owner change of one pair. - let mut pair = Pair { executed: 0, switch: None, replay: None, active: Vec::new(), idle: true }; + let mut pair = Pair { executed: 0, switch: None, replay: Vec::new(), active: Vec::new(), idle: true }; assert_eq!(pair.owner_switch_from(&c), None, "a fresh graph has no previous scheduler"); pair.publish(&Packet { context: d.clone() }, 0); pair.active.clear(); diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py index 35f970799b086..37f74d724c602 100644 --- a/tools/asahi/g17-resource-tests.py +++ b/tools/asahi/g17-resource-tests.py @@ -150,7 +150,7 @@ def item(s,start): #[derive(Clone,Copy)] struct Identity {id:Id,published:bool} '''+constants+'\n'+item(s,'pub(crate) struct QueueIds')+'\nimpl QueueIds {\n'+''.join(item(s,'pub(crate) fn '+name+'(')+'\n' for name in ['reserve_initial_render','reserve','publish','can_cancel_unpublished','cancel_unpublished'])+'}\n' test=''' - fn fresh()->QueueIds {QueueIds {entries:[None;QID_COUNT],initial_render:[None;2],compute_publication:0,leased:0}} + fn fresh()->QueueIds {QueueIds {entries:[None;QID_COUNT],initial_render:[None;2],publication:0,leased:0,floor:0}} fn main(){ for render in [0,2,16,32,112,126] { let mut q=fresh(); diff --git a/tools/asahi/rtkit-oslog-tests.py b/tools/asahi/rtkit-oslog-tests.py index 5f8eb290ebde5..9d8fcf3c9bd83 100644 --- a/tools/asahi/rtkit-oslog-tests.py +++ b/tools/asahi/rtkit-oslog-tests.py @@ -45,14 +45,14 @@ def function(source, name): void *dev, *wq; struct mailbox *mbox; struct apple_rtkit_shmem oslog_buffer, syslog_buffer, ioreport_buffer, crashlog_buffer; - struct apple_rtkit_shmem kdebug_buffer0, kdebug_buffer1; + struct apple_rtkit_shmem kdebug_buffer0, kdebug_buffer1, coreanalytics_buffer; int sync_ack_fifo; bool sync_syslog_ack, oslog_inherited, syslog_inherited, crashlog_inherited, crashed, shutting_down; unsigned iop_power_state, ap_power_state; unsigned long endpoints[4]; char *syslog_msg_buffer; size_t syslog_n_entries, syslog_msg_size; - int epmap_completion, iop_pwr_ack_completion, ap_pwr_ack_completion; + int epmap_completion, iop_pwr_ack_completion, ap_pwr_ack_completion, coreanalytics_completion; }; static unsigned sent, warnings, errors, requests, detachments, flushes, freed, checks; static int send_error, setup_error, start_error; @@ -125,7 +125,7 @@ def function(source, name): detachments=flushes=freed=0;start_error=fail?-5:0; assert(apple_rtkit_reinit(&r)==start_error); assert(!r.oslog_inherited && !r.syslog_inherited && !r.crashlog_inherited); - assert(!r.oslog_buffer.size && !r.crashed && !r.syslog_msg_size && freed==6); + assert(!r.oslog_buffer.size && !r.crashed && !r.syslog_msg_size && freed==7); assert(r.endpoints[0]==(1UL< Date: Mon, 26 Oct 2026 01:22:41 +1100 Subject: [PATCH 1270/1579] merge: adapt the retained store guard and DWC3 fixture to 7.3 Keep the WIP store-size guard with the current error API. Describe host-only DWC3 state and the register-tuning boundary in the existing teardown fixture. Signed-off-by: Ryan Murray Assisted-by: LLM --- drivers/soc/apple/store.rs | 2 +- tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index 9401404918a37..1d04917229779 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -264,7 +264,7 @@ impl Store { return Ok(store); } if size < STORE_SIZE as u64 { - return Err(store.refuse(Damage::Size, size)); + return Err(store.refuse(Damage::Size)); } let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c index 25b9f768aaaca..3e9900e11a00d 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -69,6 +69,7 @@ static void regulator_disable(int p) assert(!p); } struct dwc3 { + unsigned int current_dr_role; struct phy *usb2_generic_phy[3], *usb3_generic_phy[3]; int num_usb2_ports, num_usb3_ports, usb2_phy, usb3_phy; struct reset_controller_dev *reset; @@ -83,7 +84,7 @@ struct dwc3_apple { void *dev; struct phy *usb2_phy, *usb3_phy; bool fixed_hub, core_active, core_exit_pending, reset_pending, shutting_down; - bool rpm_held; + bool rpm_held, host_only; unsigned long usb2_power_off_pending, usb3_power_off_pending; }; static unsigned int polls, fail_poll, reset_calls, reset_writes, core_writes; @@ -314,6 +315,9 @@ static int dwc3_apple_usb2_mode_before_reset(enum dwc3_apple_state s, *m = s == DWC3_APPLE_HOST ? PHY_MODE_INVALID : PHY_MODE_USB_DEVICE; return 0; } +/* Register tuning is outside this teardown fixture's simulated device. */ +static void dwc3_apple_apply_tunables(struct dwc3_apple *a) { (void)a; } +#define dev_info_ratelimited(...) ((void)0) static int dwc3_apple_set_role(struct dwc3_apple *a, enum dwc3_apple_state s); static int dwc3_host_init(struct dwc3 *d) { From 6bcd1504aed9621e2ee0661147dcf3d97ae32cca Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 02:40:02 +1100 Subject: [PATCH 1271/1579] selftests: align M4 host models with the public 7.3 interfaces Extend only the UAPI declaration snapshot from the public C header for the optional capacity tail. Preserve independent encoder byte fixtures and record the reference. Extract current UAT quirk wrappers and use scalar module-parameter values while retaining existing fault/admission assertions. Signed-off-by: Ryan Murray Assisted-by: LLM --- .../gpu/asahi-m4/fixtures/ABI-EXTENSIONS.md | 11 +++++++++++ .../drivers/gpu/asahi-m4/fixtures/manifest.json | 15 +++++++++++++-- .../drivers/gpu/asahi-m4/fixtures/public-uapi.rs | 4 ++++ .../drivers/gpu/asahi-m4/test-mmu-retention.py | 14 ++++++++++---- .../drivers/gpu/asahi-m4/test-vm-registry.py | 2 +- 5 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 tools/testing/selftests/drivers/gpu/asahi-m4/fixtures/ABI-EXTENSIONS.md diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/fixtures/ABI-EXTENSIONS.md b/tools/testing/selftests/drivers/gpu/asahi-m4/fixtures/ABI-EXTENSIONS.md new file mode 100644 index 0000000000000..f61ded1b37ddc --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/fixtures/ABI-EXTENSIONS.md @@ -0,0 +1,11 @@ +SPDX-License-Identifier: GPL-2.0-only OR MIT + +The public-uapi.rs declaration snapshot includes the optional memory_limit and +client_memory_limit tail fields from the public MIT C UAPI at kernel commit +a1b269f79e35844660892b9779adb2f68abc38b7 (include/uapi/drm/asahi_drm.h). +The preceding layout remains the legacy 600-byte reply prefix; the full struct +is 616 bytes. test-uapi.py compiles the C and Rust declarations and compares +their sizes, offsets and constants, including the paired Mesa header. + +The independent C-generated encoder and VM byte fixtures were not regenerated +or changed. Their original provenance and hashes remain in manifest.json. diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/fixtures/manifest.json b/tools/testing/selftests/drivers/gpu/asahi-m4/fixtures/manifest.json index f33da77aa1795..9ad503ae03eee 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/fixtures/manifest.json +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/fixtures/manifest.json @@ -136,8 +136,19 @@ "sha256_before_spdx_notice": "e3ba6fa572ed16636c9b6d7428e921cf1d81930429e4610738f5a86e906bc7fe" }, "public-uapi.rs": { - "sha256": "1cb62b5c57600855b38ffff960d59b35cd5f306d35cfe77a47ed3d60ee29e601", - "sha256_before_spdx_notice": "e9f1ebf957d800bd1ed6fb372a4f8890fb1f1d8ff30e2fd436134de8c998cc76" + "sha256": "33eb92306ec7b088f14847c2cd974eebad88e5b75ac37fcc18a523c171edd0c6", + "sha256_before_spdx_notice": "e9f1ebf957d800bd1ed6fb372a4f8890fb1f1d8ff30e2fd436134de8c998cc76", + "abi_tail_extension": { + "reference_kernel_commit": "a1b269f79e35844660892b9779adb2f68abc38b7", + "reference_header_blob": "66396f89cc33393b774b99ace0945794215e2131", + "reference_header_sha256": "697f494b0e4c4719959bc73b4b4731c540b9b058b694ec8ccd7abd24b1b3dd21", + "previous_bindings_sha256": "1cb62b5c57600855b38ffff960d59b35cd5f306d35cfe77a47ed3d60ee29e601", + "fields": [ + "memory_limit", + "client_memory_limit" + ], + "reference": "Public MIT C UAPI declarations; checked by compiled C/Rust layout comparison. Encoder byte fixtures are unchanged." + } }, "vm-case-0.bin.gz": { "sha256": "2758f69e5813fa2f7f08f2008217de05c2482af4ff706125025d39b42c40be36", diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/fixtures/public-uapi.rs b/tools/testing/selftests/drivers/gpu/asahi-m4/fixtures/public-uapi.rs index 786446c4f7206..8dbdd1451fb7f 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/fixtures/public-uapi.rs +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/fixtures/public-uapi.rs @@ -77,6 +77,10 @@ pub struct drm_asahi_params_global { pub usc_generation: __u32, #[doc = " @gpu_hal_generation: Apple GPU HAL/command-stream generation.\n\n This is DRM_ASAHI_GPU_HAL_LEGACY for G13-G15,\n DRM_ASAHI_GPU_HAL_200 for G16 and G17P/G17A, and\n DRM_ASAHI_GPU_HAL_300 for G17G/G17S/G17C."] pub gpu_hal_generation: __u32, + /// Optional device-wide committed/resident-byte capacity; zero means no limit. + pub memory_limit: __u64, + /// Optional per-DRM-file committed-byte capacity; zero means no limit. + pub client_memory_limit: __u64, } impl Default for drm_asahi_params_global { fn default() -> Self { diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py index 03184a58211cf..92ed7670812a3 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py @@ -19,7 +19,7 @@ def body(source,marker): mm=(ROOT/'drivers/gpu/drm/asahi/mmu.rs').read_text();pt=(ROOT/'drivers/gpu/drm/asahi/pgtable.rs').read_text() pre=r''' #![allow(dead_code,unused_variables)] -use std::{cell::RefCell,rc::Rc,mem::ManuallyDrop,ops::{Deref,DerefMut,Range},num::NonZeroUsize,sync::{Mutex as StdMutex,atomic::{AtomicBool as StdAtomicBool,AtomicU8,AtomicU32,AtomicU64,AtomicUsize,Ordering,fence}}}; +use std::{cell::RefCell,rc::Rc,mem::{ManuallyDrop,size_of},ops::{Deref,DerefMut,Range},num::NonZeroUsize,sync::{Mutex as StdMutex,atomic::{AtomicBool as StdAtomicBool,AtomicU8,AtomicU32,AtomicU64,AtomicUsize,Ordering,fence}}}; type Result = std::result::Result;const EIO:i32=5;const ETIMEDOUT:i32=110;const EOVERFLOW:i32=75; static TRIP_LOAD:AtomicUsize=AtomicUsize::new(0); struct AtomicBool(StdAtomicBool);impl AtomicBool{fn new(v:bool)->Self{Self(StdAtomicBool::new(v))}fn store(&self,v:bool,o:Ordering){self.0.store(v,o);}fn load(&self,o:Ordering)->bool{let n=TRIP_LOAD.load(Ordering::Relaxed);if n!=0&&TRIP_LOAD.fetch_sub(1,Ordering::Relaxed)==1{self.0.store(true,Ordering::Release);}self.0.load(o)}} @@ -36,17 +36,23 @@ def body(source,marker): handoff='\n'.join(body(mm,m) for m in ['fn lock(&self) {\n self.lock_ap', 'fn unlock(&self) {\n self.turn', 'fn current_slot(']) mid=r''' } -struct SlotTTBS{ttb0:AtomicU64,ttb1:AtomicU64}struct UatShared{handoff:Handoff,ttbs:[SlotTTBS;64],map_kernel_to_user:bool,kernel_ttb1:u64,binding_generations:[u64;64],binding_published:[bool;64],owned_user_roots:[u64;64]} +struct SlotTTBS{ttb0:AtomicU64,ttb1:AtomicU64}struct UatShared{quirks:UatQuirks,handoff:Handoff,ttbs:[SlotTTBS;64],map_kernel_to_user:bool,kernel_ttb1:u64,binding_generations:[u64;64],binding_published:[bool;64],owned_user_roots:[u64;64]} impl UatShared{fn handoff(&self)->&Handoff{&self.handoff}fn ttbs(&self)->&[SlotTTBS;64]{&self.ttbs}} struct UatInner{firmware:UatFirmware,m4_adopted:bool,failed:AtomicBool,shared:Mutex} ''' +# Read quirk policy and wrappers from production rather than copying their behavior. +quirks=body(mm,'pub(crate) struct UatQuirks {')+'\n'+body(mm,'impl UatQuirks {') +pre=pre.replace('struct Handoff{',quirks+'\nstruct Handoff{',1) +pre=pre.replace('mod mem{', 'mod pgtable{pub fn clean_poc(_p:*const u8,_n:usize){panic!("cache-clean profile is outside this fixture");}pub fn poc_barrier(){panic!("cache-clean profile is outside this fixture");}}\nmod mem{',1) +shared_methods='\n'.join(body(mm,m) for m in ['fn handoff_lock(', 'fn handoff_unlock(', 'fn publish_slot(']) +mid=mid.replace('impl UatShared{','impl UatShared{'+shared_methods+'\n',1) raii='#[path="'+str(ROOT/'drivers/gpu/drm/asahi/mmu/ppl.rs')+'"]mod ppl;\n'+'\n'.join(body(mm,m) for m in [ 'struct M4Transport<', 'impl ppl::Memory for M4Transport', 'enum HandoffGuard<', 'impl HandoffGuard<', 'impl Drop for HandoffGuard<' ]) inner='impl UatInner{\n'+'\n'.join(body(mm,m) for m in ['fn lock(&self) -> Guard', 'fn check_m4(', 'fn lock_handoff(', 'fn lock_m4_upper('])+'\n}\n' models=r''' -fn uat(m4:bool,firmware:UatFirmware)->Rc{Rc::new(UatInner{firmware,m4_adopted:m4,failed:AtomicBool::new(false),shared:Mutex::new(UatShared{handoff:Handoff::new(),ttbs:std::array::from_fn(|_|SlotTTBS{ttb0:AtomicU64::new(0),ttb1:AtomicU64::new(0)}),map_kernel_to_user:!m4,kernel_ttb1:0x800000000,binding_generations:[0;64],binding_published:[false;64],owned_user_roots:[0;64]})})} +fn uat(m4:bool,firmware:UatFirmware)->Rc{Rc::new(UatInner{firmware,m4_adopted:m4,failed:AtomicBool::new(false),shared:Mutex::new(UatShared{quirks:UatQuirks::DEFAULT,handoff:Handoff::new(),ttbs:std::array::from_fn(|_|SlotTTBS{ttb0:AtomicU64::new(0),ttb1:AtomicU64::new(0)}),map_kernel_to_user:!m4,kernel_ttb1:0x800000000,binding_generations:[0;64],binding_published:[false;64],owned_user_roots:[0;64]})})} #[derive(Clone,Copy)]struct Prot{cached:bool}impl Prot{fn is_cached_noncoherent(&self)->bool{self.cached}} struct Mutation;impl Drop for Mutation{fn drop(&mut self){MUTATIONS.fetch_sub(1,Ordering::Relaxed);}} struct MapTable{fail:bool}impl MapTable{fn unmap_pages(&mut self,_r:Range)->Result{LEAF_CLEARS.fetch_add(1,Ordering::Relaxed);if self.fail{Err(EIO)}else{Ok(())}}} @@ -72,7 +78,7 @@ def body(source,marker): #[derive(Clone)]struct Vm{id:u64,binding:Rc>} struct VmBind(Vm,u32); struct Uat{inner:Rc,slots:Allocator} -mod module_parameters{pub struct Param;impl Param{pub fn value(&self)->&u32{&0}}pub static robust_isolation:Param=Param;} +mod module_parameters{pub struct Param;impl Param{pub fn value(&self)->u32{0}}pub static robust_isolation:Param=Param;} ''' helpers='\n'.join(body(mm,m) for m in ['const fn binding_wire_generation(', 'const fn application_ttb1(']) bind='impl Uat{'+body(mm,'pub(crate) fn bind(&self, vm: &Vm)')+'}\n' diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-registry.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-registry.py index d7750aab9e147..add96b48bb335 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-registry.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-vm-registry.py @@ -20,7 +20,7 @@ def item(marker): implstd::ops::Deref for KVec{type Target=Vec;fn deref(&self)->&Vec{&self.0}} implstd::ops::DerefMut for KVec{fn deref_mut(&mut self)->&mut Vec{&mut self.0}} mod hw{pub mod sizes{pub const MAX_VMS:u32=63;}} -mod module_parameters{pub struct Param;impl Param{pub fn value(&self)->&u32{&0}}pub static robust_isolation:Param=Param;} +mod module_parameters{pub struct Param;impl Param{pub fn value(&self)->u32{0}}pub static robust_isolation:Param=Param;} mod types{pub struct ScopeGuard(Option);implScopeGuard{pub fn new(f:F)->Self{Self(Some(f))}pub fn dismiss(mut self){self.0.take();}}implDrop for ScopeGuard{fn drop(&mut self){if let Some(f)=self.0.take(){f()}}}} struct Handle;impl Handle{fn is_closed(&self)->bool{false}} struct VmState{id:u64,vm:Handle,drops:Arc}impl Drop for VmState{fn drop(&mut self){self.drops.fetch_add(1,Ordering::AcqRel);}} From 69d4217668ce0dfc7852c6aaa0dbe46e06b68735 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 03:49:54 +1100 Subject: [PATCH 1272/1579] selftests: follow the shared firmware identity declaration in M4 goldens Compile the actual FirmwareBuild declaration from the shared production module after the G16 identity refactor. Preserve all independent C byte fixtures and existing packet assertions, and record the declaration source hash separately from encoder coverage. Signed-off-by: Ryan Murray Assisted-by: LLM --- .../drivers/gpu/asahi-m4/check-channels.rs | 3 +++ .../drivers/gpu/asahi-m4/run-golden-tests.py | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/check-channels.rs b/tools/testing/selftests/drivers/gpu/asahi-m4/check-channels.rs index 52461a122b89d..fccab550c6f29 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/check-channels.rs +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/check-channels.rs @@ -4,6 +4,9 @@ mod channels; #[path = "@KERNEL@/drivers/gpu/drm/asahi/g16/fw/graph.rs"] mod graph; +mod fwid { +@FIRMWARE_BUILD@ +} mod hw {pub enum GpuVariant {G}} struct Config { chip_id:u32,gpu_variant:hw::GpuVariant,gpu_revision:u32,uat_ias:u32,uat_oas:u32, diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/run-golden-tests.py b/tools/testing/selftests/drivers/gpu/asahi-m4/run-golden-tests.py index 7d3a06dac82c0..d751e9c93f255 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/run-golden-tests.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/run-golden-tests.py @@ -15,10 +15,21 @@ def main(): manifest = unpack_fixtures(golden) (out / 'public-uapi.rs').write_bytes(frozen_bindings().read_bytes()) backend = root / 'drivers/gpu/drm/asahi/g16' + # The hardware catalog now imports its data-only declaration from shared + # fwid. Compile that exact production declaration; firmware measurement is + # outside this wire-encoder harness. Independent byte fixtures stay frozen. + fwid = root / 'drivers/gpu/drm/asahi/fwid.rs' + declaration = '' + if fwid.exists(): + identity = fwid.read_text() + begin = identity.index('pub(crate) struct FirmwareBuild {') + end = identity.index('\n}', begin) + 2 + declaration = identity[begin:end] logs = [] for name in ['initdata', 'compute', 'render', 'channels', 'initdata-bounds']: source = (HERE / f'check-{name}.rs').read_text() source = source.replace('@KERNEL@', str(root)).replace('@OUTPUT@', str(out)) + source = source.replace('@FIRMWARE_BUILD@', declaration) test = out / f'check-{name}.rs' test.write_text(source) binary = out / f'check-{name}' @@ -33,7 +44,9 @@ def main(): report = {'passed': True, 'scope': 'Production Rust encoder output versus frozen independent C fixtures; CPU-only, no hardware acceptance.', 'reference_kernel_commit': manifest['reference_kernel_commit'], 'fixture_manifest_sha256': sha(HERE / 'fixtures/manifest.json'), 'seconds': round(time.monotonic()-start, 3), 'compared_encoder_bytes': 3736988, - 'source_sha256': {str(p.relative_to(root)): sha(p) for p in sorted(backend.rglob('*.rs'))}} + 'source_sha256': {str(p.relative_to(root)): sha(p) for p in sorted(backend.rglob('*.rs'))}, + 'firmware_declaration_source_sha256': sha(fwid) if fwid.exists() else None, + 'firmware_declaration_scope': 'Shared FirmwareBuild data declaration only; identity measurement is not executed by this harness.'} (out / 'golden-tests.json').write_text(json.dumps(report, indent=2)+'\n') return 0 From d85084b2a6cdc7d0e85a4a2f478a82f125488afd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 05:44:35 +1100 Subject: [PATCH 1273/1579] drm/apple: initialize compact endpoint state in lifetime fixtures The shared AFK quiesce path now cancels compact commands on every endpoint. Fake endpoints bypass afk_init, so initialize the same empty command list and transmit mutex before exercising real devres cleanup. Preserve all lifetime ordering, successor and drain assertions. Signed-off-by: Ryan Murray Assisted-by: LLM --- drivers/gpu/drm/apple/av-lifetime-test.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/apple/av-lifetime-test.c b/drivers/gpu/drm/apple/av-lifetime-test.c index 0232dfb670bbd..7485436cbf595 100644 --- a/drivers/gpu/drm/apple/av-lifetime-test.c +++ b/drivers/gpu/drm/apple/av-lifetime-test.c @@ -155,6 +155,8 @@ static struct av_life_test *av_life_setup(struct kunit *test, work_func_t owner_ t->dcp->dev = t->dev; dev_set_drvdata(t->dev, t->dcp); spin_lock_init(&t->ep->lock); + INIT_LIST_HEAD(&t->ep->compact_cmds); + mutex_init(&t->ep->compact_tx_lock); init_rwsem(&t->asrv->srv_rwsem); INIT_WORK(&t->asrv->start_av_service_wq, owner_fn); INIT_WORK(&t->asrv->terminate_wq, owner_fn); @@ -484,6 +486,8 @@ static struct av_life_test *av_life_group_setup(struct kunit *test, t->ep = devm_kzalloc(t->dev, sizeof(*t->ep), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, t->ep); spin_lock_init(&t->ep->lock); + INIT_LIST_HEAD(&t->ep->compact_cmds); + mutex_init(&t->ep->compact_tx_lock); t->ep->dcp = t->dcp; t->wq = alloc_ordered_workqueue("av-lifetime-devres", 0); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->wq); From 51aafb566d64906eb1c655f9932b3c006e62f088 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 10:36:46 +1100 Subject: [PATCH 1274/1579] arm64: dts: apple: repair the J616s PCI Wi-Fi schema description Name the BCM4388 PCI function while retaining the family fallback used by the driver for country and MAC properties. Allow that pair in the binding and omit the redundant root-port compatible. This is independent of GPU enablement, split from the J616s bring-up to keep that review focused. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/net/wireless/brcm,bcm4329-fmac.yaml | 1 + arch/arm64/boot/dts/apple/t6040-j616s.dts | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/net/wireless/brcm,bcm4329-fmac.yaml b/Documentation/devicetree/bindings/net/wireless/brcm,bcm4329-fmac.yaml index 81fd3e37452a6..8f935a2d2b320 100644 --- a/Documentation/devicetree/bindings/net/wireless/brcm,bcm4329-fmac.yaml +++ b/Documentation/devicetree/bindings/net/wireless/brcm,bcm4329-fmac.yaml @@ -46,6 +46,7 @@ properties: - cypress,cyw4373-fmac - cypress,cyw43012-fmac - infineon,cyw43439-fmac + - pci14e4,4434 # BCM4388 - const: brcm,bcm4329-fmac - enum: - brcm,bcm4329-fmac diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index a40a3fa1e1dc0..c62fee61be44a 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -569,8 +569,7 @@ device_type = "pci"; port00: pci@0,0 { - compatible = "pciclass,0604"; - reg = <0 0 0 0 0>; + reg = <0 0 0 0 0>; ranges; interrupt-map = <0 0 0 1 &port00 0 0 0 0 0 0 0 2 &port00 0 0 0 1 0 0 0 3 &port00 0 0 0 2 0 0 0 4 &port00 0 0 0 3>; interrupt-map-mask = <0 0 0 7>; @@ -592,7 +591,7 @@ }; wifi@0,0 { - compatible = "brcm,bcm4329-fmac"; + compatible = "pci14e4,4434", "brcm,bcm4329-fmac"; reg = <0 0 0 0 0>; brcm,ccode-map-trivial; brcm,board-type = "apple,hostomel"; From bbcf21a8b416f71a6eedfbb383b38b12b89b617a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 10:40:12 +1100 Subject: [PATCH 1275/1579] arm64: dts: apple: keep the root-port property indentation Preserve the existing indent after removing the redundant compatible. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index c62fee61be44a..65ee34b8b362f 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -569,7 +569,7 @@ device_type = "pci"; port00: pci@0,0 { - reg = <0 0 0 0 0>; + reg = <0 0 0 0 0>; ranges; interrupt-map = <0 0 0 1 &port00 0 0 0 0 0 0 0 2 &port00 0 0 0 1 0 0 0 3 &port00 0 0 0 2 0 0 0 4 &port00 0 0 0 3>; interrupt-map-mask = <0 0 0 7>; From e4a518ae0e28c8a227397485fe19b573bdc3a013 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 14:13:12 +1100 Subject: [PATCH 1276/1579] arm64: configs: trim redundant Asahi requests and enable shared backends Resolve each candidate removal over both arm64 defconfig and the Fedora Asahi base. The 36 removed requests cause no resolved configuration changes on either recipe: the bases or the remaining selections retain those drivers and capabilities. Keep the default thermal governor available over the Fedora base by enabling ENERGY_MODEL and THERMAL_GOV_POWER_ALLOCATOR, as identified in #438. Request 48-bit VA and mixed-granule userspace as in #394, and build the native G15 backend alongside the other supported GPUs. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 41 ++++----------------------------- 1 file changed, 5 insertions(+), 36 deletions(-) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 783e41230238c..d07b70afd9294 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -3,40 +3,28 @@ # olddefconfig and verify that the requested settings survive resolution. # Device-specific boot, debug and distribution policy stays in the base. CONFIG_RUST=y -CONFIG_ARM64_ACTLR_STATE=y -CONFIG_ARCH_APPLE=y -# CONFIG_ARM64_4K_PAGES is not set CONFIG_ARM64_16K_PAGES=y -# CONFIG_ARM64_64K_PAGES is not set +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y CONFIG_ARM64_MEMORY_MODEL_CONTROL=y CONFIG_ARM_APPLE_CPUIDLE=y -CONFIG_ARM_APPLE_SOC_CPUFREQ=m +CONFIG_ENERGY_MODEL=y +CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m CONFIG_BT_MTK7932_PCIE=m CONFIG_PCIE_APPLE=y -CONFIG_NVME_APPLE=m CONFIG_MMC_SDHCI_PCI=m -CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y CONFIG_PCIE_APPLE_PIODMA_DIAG=y CONFIG_PCIEASPM=y CONFIG_NET_VENDOR_INTEL=y -CONFIG_IGB=m CONFIG_IGC=m CONFIG_MT7932_FULLMAC=m -CONFIG_TOUCHSCREEN_APPLE_Z2=m CONFIG_INPUT_MACSMC_INPUT=m -CONFIG_I2C_APPLE=m -CONFIG_SPI_APPLE=m CONFIG_SPI_APPLE_QSPI=m -CONFIG_SPMI_APPLE=m -CONFIG_PINCTRL_APPLE_GPIO=m -CONFIG_GPIO_MACSMC=m -CONFIG_POWER_RESET_MACSMC=m CONFIG_MACSMC_POWER=m -CONFIG_SENSORS_MACSMC_HWMON=m CONFIG_SENSORS_APPLE_PMP_J613=m CONFIG_THERMAL_NETLINK=y CONFIG_NVME_HWMON=y @@ -45,6 +33,7 @@ CONFIG_VIDEO_APPLE_AVD=m CONFIG_VIDEO_APPLE_ISP=m CONFIG_DRM=y CONFIG_DRM_ASAHI=m +CONFIG_DRM_ASAHI_G15=y CONFIG_DRM_ASAHI_G16=y CONFIG_DRM_ADP=m CONFIG_DRM_APPLE=m @@ -56,7 +45,6 @@ CONFIG_DRM_SIMPLEDRM=y CONFIG_SYSFB_SIMPLEFB=y CONFIG_SND_SOC_APPLE_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m -CONFIG_SND_SOC_APPLE_MCA=m CONFIG_SND_SOC_APPLE_MACAUDIO=m CONFIG_SND_SOC_CS42L83=m CONFIG_SND_SOC_CS42L84=m @@ -66,24 +54,16 @@ CONFIG_SND_SOC_TAS2770=m CONFIG_HID_APPLE=m CONFIG_HID_MAGICMOUSE=m CONFIG_HID_APPLE_MTP_HAPTIC=m -CONFIG_SERIAL_SAMSUNG=y -CONFIG_SERIAL_SAMSUNG_CONSOLE=y CONFIG_HID_DOCKCHANNEL=m CONFIG_SPI_HID_APPLE_OF=m -CONFIG_SPI_HID_APPLE_CORE=m CONFIG_USB_DWC3_APPLE=m -CONFIG_TYPEC=m CONFIG_USB4=m CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_TYPEC_SN201202X=m -CONFIG_RTC_DRV_MACSMC=m CONFIG_RTC_DRV_APPLE_PMU=m -CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m CONFIG_APPLE_SIO_T6030=y -CONFIG_MFD_MACSMC=m -CONFIG_COMMON_CLK_APPLE_NCO=m CONFIG_APPLE_DART=y CONFIG_APPLE_DART_J713_DCP=y CONFIG_APPLE_DOCKCHANNEL=m @@ -93,10 +73,7 @@ CONFIG_APPLE_MAILBOX=y CONFIG_APPLE_PMGR_MISC=y CONFIG_APPLE_RTKIT=y CONFIG_APPLE_RTKIT_HELPER=m -CONFIG_APPLE_SART=m -CONFIG_RUST_APPLE_RTKIT=y CONFIG_APPLE_AOP=m -CONFIG_HW_RANDOM=y CONFIG_KEYS=y CONFIG_TRUSTED_KEYS=y # CONFIG_TRUSTED_KEYS_TPM is not set @@ -104,28 +81,20 @@ CONFIG_TRUSTED_KEYS=y CONFIG_APPLE_SEP=m CONFIG_APPLE_PMGR_PWRSTATE=y CONFIG_APPLE_PMP=m -CONFIG_APPLE_PMP_REPORT=y CONFIG_APPLE_PMP_THERMAL=m CONFIG_APPLE_T8132_PLATFORM_HANDOFF=y CONFIG_IIO_AOP_SENSOR_LAS=m CONFIG_IIO_AOP_SENSOR_ALS=m CONFIG_RUST_FW_LOADER_ABSTRACTIONS=y -CONFIG_PWM_APPLE=m -CONFIG_APPLE_AIC=y CONFIG_PHY_APPLE_ATC=m CONFIG_PHY_APPLE_T6040_USB2=m CONFIG_PHY_APPLE_DPTX=m -CONFIG_APPLE_M1_CPU_PMU=y -CONFIG_NVMEM_APPLE_EFUSES=m -CONFIG_NVMEM_APPLE_SPMI=m CONFIG_MUX_APPLE_DPXBAR=m -CONFIG_TYPEC_TPS6598X=m CONFIG_RESET_APPLE_CIO=y CONFIG_INFINIBAND=m CONFIG_INFINIBAND_USER_ACCESS=m CONFIG_INFINIBAND_USER_MAD=m CONFIG_RDMA_RXE=m CONFIG_RDMA_SIW=m -CONFIG_DMABUF_HEAPS=y CONFIG_DMABUF_HEAPS_SYSTEM=y CONFIG_UDMABUF=y From eb4b4269f3c8b1c8c2c070d83f22b36a9b0b7059 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 14:13:12 +1100 Subject: [PATCH 1277/1579] selftests: apple-dcp: initialise the probe retry budget fixture The probe fixture calls cd321x_pm_init() on an uninitialised stack object. That helper sets the initial phase and link-event budget but expects the other fields to have the zeroed allocation used by the real driver. The fixture can read arbitrary setup_pending values and fail its retry assertions under UBSan. Start it from the same zeroed state as the other fixtures. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/testing/selftests/drivers/gpu/apple-dcp/pm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index d5b3890d30861..664ba046dad35 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -547,7 +547,7 @@ def function(text, name): int main(void) { (void)cd321x_pm_notifications; struct cd321x c; - struct cd321x_pm_state budget; + struct cd321x_pm_state budget={0}; cd321x_pm_init(&budget);assert(cd321x_pm_ready(&budget)); assert(cd321x_display_taken(-ENOSPC,true)&&!cd321x_display_taken(-EBUSY,true)&&!cd321x_display_taken(0,true)&&cd321x_display_taken(-EBUSY,false)&&!cd321x_display_taken(-ENOSPC,false)&&!cd321x_display_taken(0,false)); for(unsigned int i=0;i Date: Mon, 26 Oct 2026 14:13:12 +1100 Subject: [PATCH 1278/1579] arm64: configs: trim redundant Asahi requests and enable shared backends Resolve each candidate removal over both arm64 defconfig and the Fedora Asahi base. The 36 removed requests cause no resolved configuration changes on either recipe: the bases or the remaining selections retain those drivers and capabilities. Keep the default thermal governor available over the Fedora base by enabling ENERGY_MODEL and THERMAL_GOV_POWER_ALLOCATOR, as identified in #438. Request 48-bit VA and mixed-granule userspace as in #394, and build the native G15 backend alongside the other supported GPUs. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 41 ++++----------------------------- 1 file changed, 5 insertions(+), 36 deletions(-) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 783e41230238c..d07b70afd9294 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -3,40 +3,28 @@ # olddefconfig and verify that the requested settings survive resolution. # Device-specific boot, debug and distribution policy stays in the base. CONFIG_RUST=y -CONFIG_ARM64_ACTLR_STATE=y -CONFIG_ARCH_APPLE=y -# CONFIG_ARM64_4K_PAGES is not set CONFIG_ARM64_16K_PAGES=y -# CONFIG_ARM64_64K_PAGES is not set +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y CONFIG_ARM64_MEMORY_MODEL_CONTROL=y CONFIG_ARM_APPLE_CPUIDLE=y -CONFIG_ARM_APPLE_SOC_CPUFREQ=m +CONFIG_ENERGY_MODEL=y +CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m CONFIG_BT_MTK7932_PCIE=m CONFIG_PCIE_APPLE=y -CONFIG_NVME_APPLE=m CONFIG_MMC_SDHCI_PCI=m -CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y CONFIG_PCIE_APPLE_PIODMA_DIAG=y CONFIG_PCIEASPM=y CONFIG_NET_VENDOR_INTEL=y -CONFIG_IGB=m CONFIG_IGC=m CONFIG_MT7932_FULLMAC=m -CONFIG_TOUCHSCREEN_APPLE_Z2=m CONFIG_INPUT_MACSMC_INPUT=m -CONFIG_I2C_APPLE=m -CONFIG_SPI_APPLE=m CONFIG_SPI_APPLE_QSPI=m -CONFIG_SPMI_APPLE=m -CONFIG_PINCTRL_APPLE_GPIO=m -CONFIG_GPIO_MACSMC=m -CONFIG_POWER_RESET_MACSMC=m CONFIG_MACSMC_POWER=m -CONFIG_SENSORS_MACSMC_HWMON=m CONFIG_SENSORS_APPLE_PMP_J613=m CONFIG_THERMAL_NETLINK=y CONFIG_NVME_HWMON=y @@ -45,6 +33,7 @@ CONFIG_VIDEO_APPLE_AVD=m CONFIG_VIDEO_APPLE_ISP=m CONFIG_DRM=y CONFIG_DRM_ASAHI=m +CONFIG_DRM_ASAHI_G15=y CONFIG_DRM_ASAHI_G16=y CONFIG_DRM_ADP=m CONFIG_DRM_APPLE=m @@ -56,7 +45,6 @@ CONFIG_DRM_SIMPLEDRM=y CONFIG_SYSFB_SIMPLEFB=y CONFIG_SND_SOC_APPLE_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m -CONFIG_SND_SOC_APPLE_MCA=m CONFIG_SND_SOC_APPLE_MACAUDIO=m CONFIG_SND_SOC_CS42L83=m CONFIG_SND_SOC_CS42L84=m @@ -66,24 +54,16 @@ CONFIG_SND_SOC_TAS2770=m CONFIG_HID_APPLE=m CONFIG_HID_MAGICMOUSE=m CONFIG_HID_APPLE_MTP_HAPTIC=m -CONFIG_SERIAL_SAMSUNG=y -CONFIG_SERIAL_SAMSUNG_CONSOLE=y CONFIG_HID_DOCKCHANNEL=m CONFIG_SPI_HID_APPLE_OF=m -CONFIG_SPI_HID_APPLE_CORE=m CONFIG_USB_DWC3_APPLE=m -CONFIG_TYPEC=m CONFIG_USB4=m CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_TYPEC_SN201202X=m -CONFIG_RTC_DRV_MACSMC=m CONFIG_RTC_DRV_APPLE_PMU=m -CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m CONFIG_APPLE_SIO_T6030=y -CONFIG_MFD_MACSMC=m -CONFIG_COMMON_CLK_APPLE_NCO=m CONFIG_APPLE_DART=y CONFIG_APPLE_DART_J713_DCP=y CONFIG_APPLE_DOCKCHANNEL=m @@ -93,10 +73,7 @@ CONFIG_APPLE_MAILBOX=y CONFIG_APPLE_PMGR_MISC=y CONFIG_APPLE_RTKIT=y CONFIG_APPLE_RTKIT_HELPER=m -CONFIG_APPLE_SART=m -CONFIG_RUST_APPLE_RTKIT=y CONFIG_APPLE_AOP=m -CONFIG_HW_RANDOM=y CONFIG_KEYS=y CONFIG_TRUSTED_KEYS=y # CONFIG_TRUSTED_KEYS_TPM is not set @@ -104,28 +81,20 @@ CONFIG_TRUSTED_KEYS=y CONFIG_APPLE_SEP=m CONFIG_APPLE_PMGR_PWRSTATE=y CONFIG_APPLE_PMP=m -CONFIG_APPLE_PMP_REPORT=y CONFIG_APPLE_PMP_THERMAL=m CONFIG_APPLE_T8132_PLATFORM_HANDOFF=y CONFIG_IIO_AOP_SENSOR_LAS=m CONFIG_IIO_AOP_SENSOR_ALS=m CONFIG_RUST_FW_LOADER_ABSTRACTIONS=y -CONFIG_PWM_APPLE=m -CONFIG_APPLE_AIC=y CONFIG_PHY_APPLE_ATC=m CONFIG_PHY_APPLE_T6040_USB2=m CONFIG_PHY_APPLE_DPTX=m -CONFIG_APPLE_M1_CPU_PMU=y -CONFIG_NVMEM_APPLE_EFUSES=m -CONFIG_NVMEM_APPLE_SPMI=m CONFIG_MUX_APPLE_DPXBAR=m -CONFIG_TYPEC_TPS6598X=m CONFIG_RESET_APPLE_CIO=y CONFIG_INFINIBAND=m CONFIG_INFINIBAND_USER_ACCESS=m CONFIG_INFINIBAND_USER_MAD=m CONFIG_RDMA_RXE=m CONFIG_RDMA_SIW=m -CONFIG_DMABUF_HEAPS=y CONFIG_DMABUF_HEAPS_SYSTEM=y CONFIG_UDMABUF=y From 79c848b22719fe24ade98cf3091685a4a83e8c25 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 14:13:12 +1100 Subject: [PATCH 1279/1579] selftests: apple-dcp: initialise the probe retry budget fixture The probe fixture calls cd321x_pm_init() on an uninitialised stack object. That helper sets the initial phase and link-event budget but expects the other fields to have the zeroed allocation used by the real driver. The fixture can read arbitrary setup_pending values and fail its retry assertions under UBSan. Start it from the same zeroed state as the other fixtures. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/testing/selftests/drivers/gpu/apple-dcp/pm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 75fe10221fcc9..af3a590b375a4 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -547,7 +547,7 @@ def function(text, name): int main(void) { (void)cd321x_pm_notifications; struct cd321x c; - struct cd321x_pm_state budget; + struct cd321x_pm_state budget={0}; cd321x_pm_init(&budget);assert(cd321x_pm_ready(&budget)); assert(cd321x_display_taken(-ENOSPC,true)&&!cd321x_display_taken(-EBUSY,true)&&!cd321x_display_taken(0,true)&&cd321x_display_taken(-EBUSY,false)&&!cd321x_display_taken(-ENOSPC,false)&&!cd321x_display_taken(0,false)); for(unsigned int i=0;i Date: Mon, 26 Oct 2026 14:14:30 +1100 Subject: [PATCH 1280/1579] drm/apple: drop an unused probe switch and the M3 notch alias dcpext_probe has no consumer; the external display paths already use their compatible and firmware admission rules. Remove the unused registration. The generic show_notch preference already applies to the 14.x and 26.6 internal displays. Use it directly and remove t6030_show_notch, which only duplicated that choice for those firmware variants. Document the command-line migration while retaining the common notch preference and the matching m1n1 framebuffer preference. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/gpu/apple-drm-parameters.rst | 12 +++++++++++ Documentation/gpu/drivers.rst | 1 + drivers/gpu/drm/apple/dcp.c | 25 +--------------------- drivers/gpu/drm/apple/iomfb_v26_6.c | 2 +- 4 files changed, 15 insertions(+), 25 deletions(-) create mode 100644 Documentation/gpu/apple-drm-parameters.rst diff --git a/Documentation/gpu/apple-drm-parameters.rst b/Documentation/gpu/apple-drm-parameters.rst new file mode 100644 index 0000000000000..b1e8bcaa4d163 --- /dev/null +++ b/Documentation/gpu/apple-drm-parameters.rst @@ -0,0 +1,12 @@ +Apple display module parameters +=============================== + +Use ``appledrm.show_notch=1`` to include the notch rows in the panel mode. +This common preference also applies to the M3 internal display firmware; +it replaces the platform-specific ``appledrm.t6030_show_notch`` alias. +When the boot framebuffer should cover those rows too, keep the useful +``chosen.asahi,show-notch=1`` m1n1 display preference alongside it. + +The old ``appledrm.dcpext_probe`` parameter had no consumer and has been +removed. External display support follows the device tree and the driver's +compatible and firmware admission checks. diff --git a/Documentation/gpu/drivers.rst b/Documentation/gpu/drivers.rst index 8351e25ba8ef1..ced2721b96c75 100644 --- a/Documentation/gpu/drivers.rst +++ b/Documentation/gpu/drivers.rst @@ -7,6 +7,7 @@ GPU Driver Documentation amdgpu/index apple-dcp-backlight + apple-drm-parameters asahi-reclaim asahi-m4-bringup apple-dcp-m4 diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8a2e2c5bcbf10..eb5108fe05a04 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -198,15 +198,6 @@ static bool show_notch; module_param(show_notch, bool, 0644); MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); -static bool dcpext_probe; -module_param(dcpext_probe, bool, 0444); -MODULE_PARM_DESC(dcpext_probe, "Opt into experimental T6030 external DCP probe (may hang)"); - -static bool t6030_show_notch; -module_param(t6030_show_notch, bool, 0444); -MODULE_PARM_DESC(t6030_show_notch, - "M3 internal display (14.x and 26.6 IOMFB): use the full height and show the notch"); - bool hdmi_audio; module_param(hdmi_audio, bool, 0644); MODULE_PARM_DESC(hdmi_audio, "Enable unstable HDMI audio support"); @@ -4510,20 +4501,6 @@ static bool dcp_external_is_idle(struct apple_dcp *dcp) (status & APPLE_DCP_COPROC_CPU_STATUS_IDLE); } -/* - * The M3 internal displays (14.x and 26.6 IOMFB) also accept - * appledrm.t6030_show_notch=1, the switch existing J613/T6030 installs pass, - * so the panel comes up at its full height (2560x1664 on the J613) instead - * of the module refusing an unknown parameter. - */ -static bool dcp_show_notch(struct apple_dcp *dcp) -{ - if (dcp->fw_compat != DCP_FIRMWARE_V_14_7 && - dcp->fw_compat != DCP_FIRMWARE_V_26_6) - return show_notch; - return show_notch || t6030_show_notch; -} - /* * T8122 dcpext (DCP_FIRMWARE_V_14_7_TEMPLATE): if the PMP runs, the display * gate added its DISPEXT request and pointed the dcpext at it @@ -4670,7 +4647,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); - if (!dcp_show_notch(dcp)) + if (!show_notch) ret = of_property_read_u32(dev->of_node, "apple,notch-height", &dcp->notch_height); diff --git a/drivers/gpu/drm/apple/iomfb_v26_6.c b/drivers/gpu/drm/apple/iomfb_v26_6.c index 3a33ae65d2018..bf2f0c4f610c7 100644 --- a/drivers/gpu/drm/apple/iomfb_v26_6.c +++ b/drivers/gpu/drm/apple/iomfb_v26_6.c @@ -28,7 +28,7 @@ * Scanout is one plane, XRGB8888 or ARGB8888, linear, exactly the mode size. * Each flush is one swap, completed when the firmware reports that swap id, * and then the page-flip event is sent. The notch rows (apple,notch-height, - * unless appledrm.t6030_show_notch=1) are outside the mode and stay black. + * unless appledrm.show_notch=1) are outside the mode and stay black. */ #include From c3a5cff12d7f6915c4c6fd119a77d3f1331b71e8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 14:21:51 +1100 Subject: [PATCH 1281/1579] docs: gpu: identify the Apple parameter migration license Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/gpu/apple-drm-parameters.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/gpu/apple-drm-parameters.rst b/Documentation/gpu/apple-drm-parameters.rst index b1e8bcaa4d163..599cf050f7c81 100644 --- a/Documentation/gpu/apple-drm-parameters.rst +++ b/Documentation/gpu/apple-drm-parameters.rst @@ -1,3 +1,5 @@ +.. SPDX-License-Identifier: GPL-2.0-only + Apple display module parameters =============================== From e3653e6e1a563902ef2d32e25ac1d6e3b9f69a7d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 14:18:53 +1100 Subject: [PATCH 1282/1579] dmaengine: apple-sio: select firmware behavior from compatible data The T6030 backend is already enabled in the shared kernel fragment, but it has a separate build option and repeated conditional code. Compile it with the common SIO driver and select its boot-table validator, FIFO threshold and descriptor handling through the existing apple,t6030-sio compatible. The generic apple,sio profile retains the M1/M2 behavior. Keep the protocol-10 validation and refuse incompatible tables before DMA or hardware setup. Remove APPLE_SIO_T6030 and its shared fragment request. Update the binding description, synthetic DT overlay and KUnit probe cases to cover the specific match, missing/older tables and an unknown compatible. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/dma/apple,sio.yaml | 4 +- arch/arm64/configs/asahi.config | 1 - drivers/dma/Kconfig | 20 +---- drivers/dma/apple-sio-probe-test.c | 46 +++++++++-- drivers/dma/apple-sio-probe-test.dtso | 10 +++ drivers/dma/apple-sio.c | 79 +++++++++---------- 6 files changed, 93 insertions(+), 67 deletions(-) diff --git a/Documentation/devicetree/bindings/dma/apple,sio.yaml b/Documentation/devicetree/bindings/dma/apple,sio.yaml index dbaaa2d3d1a1a..708d7bb8bed96 100644 --- a/Documentation/devicetree/bindings/dma/apple,sio.yaml +++ b/Documentation/devicetree/bindings/dma/apple,sio.yaml @@ -9,7 +9,9 @@ title: Apple SIO Coprocessor description: SIO is a coprocessor on Apple M1 and later chips (and maybe also on earlier chips). Its role is to offload SPI, UART and DisplayPort audio transfers, - being a pretend DMA controller. + being a pretend DMA controller. The SoC-specific compatible selects the + firmware protocol and channel configuration. In particular, apple,t6030-sio + requires protocol-10 boot tables before the coprocessor can be started. maintainers: - Martin Povišer diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index d07b70afd9294..590df15222c8b 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -63,7 +63,6 @@ CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_TYPEC_SN201202X=m CONFIG_RTC_DRV_APPLE_PMU=m CONFIG_APPLE_SIO=m -CONFIG_APPLE_SIO_T6030=y CONFIG_APPLE_DART=y CONFIG_APPLE_DART_J713_DCP=y CONFIG_APPLE_DOCKCHANNEL=m diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 94f9c2e2860d9..6ba3a178c636b 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -111,26 +111,12 @@ config APPLE_SIO_KUNIT_TEST default KUNIT_ALL_TESTS help Check which SIO device-tree nodes the probe goes on with: the - generic SIO of the M1 and M2 in every configuration, and a T6030 - SIO only with APPLE_SIO_T6030 and protocol-10 boot tables. Run it - both with and without APPLE_SIO_T6030. - -config APPLE_SIO_T6030 - bool "Apple SIO on the M3 Pro (T6030)" - depends on APPLE_SIO - help - Configure the SIO channels of the T6030 (apple,t6030-sio) the way - its firmware expects them: the DisplayPort audio channels refill - their FIFO at the threshold the system's device tree records for - them (0x600 bytes, not 0x800), and every period descriptor is - published whole. The firmware's boot tables come from the boot - loader, as on the other SoCs; the SIO is only started when they - are protocol-10 tables, which a boot loader without T6030 support - does not build. Without this option a T6030 SIO is left stopped. + generic SIO of the M1 and M2, and a T6030 SIO only with validated + protocol-10 boot tables. The compatible selects the firmware layout. config APPLE_SIO_T6030_KUNIT_TEST bool "KUnit tests for the T6030 SIO boot-table check" if !KUNIT_ALL_TESTS - depends on APPLE_SIO=y && APPLE_SIO_T6030 && KUNIT=y + depends on APPLE_SIO=y && KUNIT=y depends on OF_OVERLAY default KUNIT_ALL_TESTS help diff --git a/drivers/dma/apple-sio-probe-test.c b/drivers/dma/apple-sio-probe-test.c index 7049c8acb532a..c486c5072f07c 100644 --- a/drivers/dma/apple-sio-probe-test.c +++ b/drivers/dma/apple-sio-probe-test.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * Included by apple-sio.c: which SIO nodes the probe goes on with, with and - * without APPLE_SIO_T6030. The test devices have no register range, so a probe + * Included by apple-sio.c: the compatible selects which firmware tables a + * node needs. The test devices have no register range, so a probe * that goes on toward the hardware stops there with -EINVAL; a refused node * returns -ENODEV first. */ @@ -13,16 +13,17 @@ OF_OVERLAY_DECLARE(apple_sio_probe_test); static void sio_probe_nodes_test(struct kunit *test) { - const bool t6030 = IS_ENABLED(CONFIG_APPLE_SIO_T6030); const struct { const char *path; int err; } cases[] = { - /* M1/M2: the generic SIO, either way */ + /* M1/M2: the generic SIO */ { "/sio-probe-test/sio-probe-test-t8103", -EINVAL }, - /* T6030: only with its support built, and with protocol-10 tables */ - { "/sio-probe-test/sio-probe-test-t6030-v10", t6030 ? -EINVAL : -ENODEV }, + /* T6030: protocol-10 tables select the admitted path */ + { "/sio-probe-test/sio-probe-test-t6030-v10", -EINVAL }, { "/sio-probe-test/sio-probe-test-t6030-earlier", -ENODEV }, + { "/sio-probe-test/sio-probe-test-t6030-missing", -ENODEV }, + { "/sio-probe-test/sio-probe-test-unknown", -ENODEV }, }; unsigned int i; @@ -38,15 +39,44 @@ static void sio_probe_nodes_test(struct kunit *test) KUNIT_ASSERT_NOT_NULL_MSG(test, pdev->dev.of_node, "%s", cases[i].path); mask = pdev->dev.coherent_dma_mask; KUNIT_EXPECT_EQ_MSG(test, sio_probe(pdev), cases[i].err, "%s", cases[i].path); - /* Without its support, a T6030 is refused before its DMA is set up */ - if (!t6030 && cases[i].err == -ENODEV) + /* Bad protocol tables are refused before DMA setup. */ + if (cases[i].err == -ENODEV) KUNIT_EXPECT_EQ_MSG(test, pdev->dev.coherent_dma_mask, mask, "%s", cases[i].path); } } +static void sio_match_data_test(struct kunit *test) +{ + const struct of_device_id *match; + struct device_node *np; + const struct sio_hw_data *hw; + + KUNIT_ASSERT_EQ(test, 0, of_overlay_apply_kunit(test, apple_sio_probe_test)); + np = of_find_node_by_path("/sio-probe-test/sio-probe-test-t6030-v10"); + KUNIT_ASSERT_NOT_NULL(test, np); + match = of_match_node(sio_of_match, np); + of_node_put(np); + KUNIT_ASSERT_NOT_NULL(test, match); + hw = match->data; + KUNIT_EXPECT_EQ(test, hw->fifo_threshold, 0x600U); + KUNIT_EXPECT_TRUE(test, hw->clear_period_desc); + KUNIT_EXPECT_TRUE(test, hw->check_boot_tables == sio_t6030_check_boot_tables); + + np = of_find_node_by_path("/sio-probe-test/sio-probe-test-t8103"); + KUNIT_ASSERT_NOT_NULL(test, np); + match = of_match_node(sio_of_match, np); + of_node_put(np); + KUNIT_ASSERT_NOT_NULL(test, match); + hw = match->data; + KUNIT_EXPECT_EQ(test, hw->fifo_threshold, 0x800U); + KUNIT_EXPECT_FALSE(test, hw->clear_period_desc); + KUNIT_EXPECT_FALSE(test, hw->check_boot_tables); +} + static struct kunit_case sio_probe_test_cases[] = { KUNIT_CASE(sio_probe_nodes_test), + KUNIT_CASE(sio_match_data_test), {} }; diff --git a/drivers/dma/apple-sio-probe-test.dtso b/drivers/dma/apple-sio-probe-test.dtso index 79f3346beaad8..93816e9f6b202 100644 --- a/drivers/dma/apple-sio-probe-test.dtso +++ b/drivers/dma/apple-sio-probe-test.dtso @@ -17,6 +17,16 @@ dma-channels = <8>; }; + sio-probe-test-t6030-missing { + compatible = "apple,t6030-sio", "apple,sio"; + dma-channels = <8>; + }; + + sio-probe-test-unknown { + compatible = "apple,sio-probe-test-unknown"; + dma-channels = <8>; + }; + v10_w: v10-w { iommu-addresses = <&v10 0x0 0x30000 0x0 0x4000>, <&v10 0x0 0x34000 0x0 0x20000>, <&v10 0x0 0x54000 0x0 0x4000>, <&v10 0x0 0x58000 0x0 0x4000>; diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c index 0c06e0264b7b6..65ba4511501c7 100644 --- a/drivers/dma/apple-sio.c +++ b/drivers/dma/apple-sio.c @@ -108,7 +108,14 @@ struct sio_chan { typedef void (*sio_ack_callback)(struct sio_chan *, void *, bool); +struct sio_hw_data { + u32 fifo_threshold; + bool clear_period_desc; + int (*check_boot_tables)(struct device *dev); +}; + struct sio_data { + const struct sio_hw_data *hw; void __iomem *base; struct dma_device dma; struct device *dev; @@ -130,9 +137,6 @@ struct sio_data { } tags; int nchannels; -#if IS_ENABLED(CONFIG_APPLE_SIO_T6030) - bool t6030; -#endif struct sio_chan channels[]; }; @@ -155,16 +159,6 @@ static int sio_send_siomsg_atomic(struct sio_data *sio, u64 msg, void *cookie); static int sio_call(struct sio_data *sio, u64 msg); -static inline bool sio_is_t6030(struct sio_data *sio) -{ -#if IS_ENABLED(CONFIG_APPLE_SIO_T6030) - return sio->t6030; -#else - return false; -#endif -} - -#if IS_ENABLED(CONFIG_APPLE_SIO_T6030) /* * Boot tables of the T6030 SIO firmware, protocol 10. The boot loader builds them from the ADT * and passes each one as two apple,sio-firmware-params: the table's IOVA page under its key and @@ -320,7 +314,23 @@ static int sio_t6030_check_boot_tables(struct device *dev) } return 0; } -#endif + +static const struct sio_hw_data sio_hw_generic = { + .fifo_threshold = 0x800, +}; + +static const struct sio_hw_data sio_hw_t6030 = { + .fifo_threshold = 0x600, + .clear_period_desc = true, + .check_boot_tables = sio_t6030_check_boot_tables, +}; + +static const struct of_device_id sio_of_match[] = { + { .compatible = "apple,t6030-sio", .data = &sio_hw_t6030 }, + { .compatible = "apple,sio", .data = &sio_hw_generic }, + { } +}; +MODULE_DEVICE_TABLE(of, sio_of_match); static struct sio_chan *to_sio_chan(struct dma_chan *chan) { @@ -459,7 +469,7 @@ static struct dma_async_tx_descriptor *sio_prep_dma_cyclic( * T6030: publish the whole descriptor, its other fields zero; * a slot may come back from an earlier transfer. */ - if (sio_is_t6030(siochan->host)) + if (siochan->host->hw->clear_period_desc) memset(d, 0, sizeof(*d)); d->flag = 1; /* not sure what's up with this */ d->iova = buf_addr + period_len * i; @@ -839,7 +849,7 @@ static int sio_device_config(struct dma_chan *chan, * dp-audioN dma-channels: channel, data shape, timeout, FIFO size, * threshold). T6030's DisplayPort audio channels refill at 0x600. */ - cfg.threshold = sio_is_t6030(sio) ? 0x600 : 0x800; + cfg.threshold = sio->hw->fifo_threshold; /* * Dmaengine prescribes we ought to apply the new configuration only @@ -967,20 +977,23 @@ static int sio_send_dt_params(struct sio_data *sio) static int sio_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; + const struct of_device_id *match; + const struct sio_hw_data *hw; struct sio_data *sio; struct dma_device *dma; int nchannels; int err, i; - /* - * Only APPLE_SIO_T6030 checks a T6030's boot tables and configures its - * channels: without it, leave the SIO stopped rather than start it as - * the generic one. - */ - if (!IS_ENABLED(CONFIG_APPLE_SIO_T6030) && - of_device_is_compatible(np, "apple,t6030-sio")) - return dev_err_probe(&pdev->dev, -ENODEV, - "T6030 SIO support (APPLE_SIO_T6030) not built; leaving the SIO stopped\n"); + match = of_match_node(sio_of_match, np); + if (!match) + return -ENODEV; + hw = match->data; + /* Refuse incompatible firmware tables before DMA or hardware setup. */ + if (hw->check_boot_tables) { + err = hw->check_boot_tables(&pdev->dev); + if (err) + return err; + } err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); if (err) @@ -998,15 +1011,7 @@ static int sio_probe(struct platform_device *pdev) platform_set_drvdata(pdev, sio); sio->dev = &pdev->dev; sio->nchannels = nchannels; -#if IS_ENABLED(CONFIG_APPLE_SIO_T6030) - sio->t6030 = of_device_is_compatible(np, "apple,t6030-sio"); - /* Before any register, power or firmware access */ - if (sio->t6030) { - err = sio_t6030_check_boot_tables(&pdev->dev); - if (err) - return err; - } -#endif + sio->hw = hw; sio->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(sio->base)) @@ -1107,12 +1112,6 @@ static void sio_remove(struct platform_device *pdev) dma_async_device_unregister(&sio->dma); } -static const struct of_device_id sio_of_match[] = { - { .compatible = "apple,sio", }, - { } -}; -MODULE_DEVICE_TABLE(of, sio_of_match); - static __maybe_unused int sio_suspend(struct device *dev) { /* From c7aed9d6140f061b4af2e437c3c4a8008aaa81f2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 14:48:02 +1100 Subject: [PATCH 1283/1579] arm64: configs: retire the superseded J700 fragments on aurora-wip Integration #388 already replaced these four files with the shared asahi.config recipe. Remove the stale copies from aurora-wip after carrying the common configuration and compatible-data cleanup, so the branch has the same hardware recipe as integration. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/aurora_j700_defconfig | 2895 ------------------ arch/arm64/configs/j700-device-parity.config | 1053 ------- arch/arm64/configs/j700-radio.config | 20 - arch/arm64/configs/j700.config | 113 - 4 files changed, 4081 deletions(-) delete mode 100644 arch/arm64/configs/aurora_j700_defconfig delete mode 100644 arch/arm64/configs/j700-device-parity.config delete mode 100644 arch/arm64/configs/j700-radio.config delete mode 100644 arch/arm64/configs/j700.config diff --git a/arch/arm64/configs/aurora_j700_defconfig b/arch/arm64/configs/aurora_j700_defconfig deleted file mode 100644 index 49e5c03d106ea..0000000000000 --- a/arch/arm64/configs/aurora_j700_defconfig +++ /dev/null @@ -1,2895 +0,0 @@ -CONFIG_WERROR=y -CONFIG_LOCALVERSION="-aurora" -# CONFIG_LOCALVERSION_AUTO is not set -CONFIG_SYSVIPC=y -CONFIG_POSIX_MQUEUE=y -CONFIG_AUDIT=y -CONFIG_NO_HZ_IDLE=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_BPF_SYSCALL=y -CONFIG_BPF_JIT=y -CONFIG_PREEMPT=y -CONFIG_IRQ_TIME_ACCOUNTING=y -CONFIG_BSD_PROCESS_ACCT=y -CONFIG_BSD_PROCESS_ACCT_V3=y -CONFIG_TASKSTATS=y -CONFIG_TASK_XACCT=y -CONFIG_TASK_IO_ACCOUNTING=y -CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y -CONFIG_UCLAMP_TASK=y -CONFIG_NUMA_BALANCING=y -CONFIG_MEMCG=y -CONFIG_BLK_CGROUP=y -CONFIG_UCLAMP_TASK_GROUP=y -CONFIG_CGROUP_PIDS=y -CONFIG_CGROUP_FREEZER=y -CONFIG_CGROUP_HUGETLB=y -CONFIG_CPUSETS=y -CONFIG_CGROUP_DEVICE=y -CONFIG_CGROUP_CPUACCT=y -CONFIG_CGROUP_PERF=y -CONFIG_CGROUP_BPF=y -CONFIG_CGROUP_MISC=y -CONFIG_USER_NS=y -CONFIG_CHECKPOINT_RESTORE=y -CONFIG_SCHED_AUTOGROUP=y -CONFIG_BLK_DEV_INITRD=y -CONFIG_KALLSYMS_ALL=y -CONFIG_PROFILING=y -CONFIG_RUST=y -CONFIG_ARCH_ACTIONS=y -CONFIG_ARCH_AIROHA=y -CONFIG_ARCH_SUNXI=y -CONFIG_ARCH_ALPINE=y -CONFIG_ARCH_APPLE=y -CONFIG_ARCH_ARTPEC=y -CONFIG_ARCH_AXIADO=y -CONFIG_ARCH_BCM=y -CONFIG_ARCH_BCM2835=y -CONFIG_ARCH_BCM_IPROC=y -CONFIG_ARCH_BCMBCA=y -CONFIG_ARCH_BRCMSTB=y -CONFIG_ARCH_BERLIN=y -CONFIG_ARCH_BLAIZE=y -CONFIG_ARCH_BST=y -CONFIG_ARCH_CIX=y -CONFIG_ARCH_EXYNOS=y -CONFIG_ARCH_K3=y -CONFIG_ARCH_LG1K=y -CONFIG_ARCH_HISI=y -CONFIG_ARCH_KEEMBAY=y -CONFIG_ARCH_MEDIATEK=y -CONFIG_ARCH_MESON=y -CONFIG_ARCH_MICROCHIP=y -CONFIG_ARCH_SPARX5=y -CONFIG_ARCH_MVEBU=y -CONFIG_ARCH_NXP=y -CONFIG_ARCH_LAYERSCAPE=y -CONFIG_ARCH_MXC=y -CONFIG_ARCH_S32=y -CONFIG_ARCH_MA35=y -CONFIG_ARCH_NPCM=y -CONFIG_ARCH_QCOM=y -CONFIG_ARCH_REALTEK=y -CONFIG_ARCH_RENESAS=y -CONFIG_ARCH_ROCKCHIP=y -CONFIG_ARCH_SEATTLE=y -CONFIG_ARCH_INTEL_SOCFPGA=y -CONFIG_ARCH_SOPHGO=y -CONFIG_ARCH_STM32=y -CONFIG_ARCH_SYNQUACER=y -CONFIG_ARCH_TEGRA=y -CONFIG_ARCH_TESLA_FSD=y -CONFIG_ARCH_SPRD=y -CONFIG_ARCH_THUNDER=y -CONFIG_ARCH_THUNDER2=y -CONFIG_ARCH_UNIPHIER=y -CONFIG_ARCH_VEXPRESS=y -CONFIG_ARCH_VISCONTI=y -CONFIG_ARCH_XGENE=y -CONFIG_ARCH_ZYNQMP=y -CONFIG_ARM64_16K_PAGES=y -CONFIG_ARM64_VA_BITS_48=y -CONFIG_NUMA=y -CONFIG_XEN=y -CONFIG_ARM64_MEMORY_MODEL_CONTROL=y -CONFIG_HIBERNATION=y -CONFIG_PM_DEBUG=y -CONFIG_PM_ADVANCED_DEBUG=y -CONFIG_WQ_POWER_EFFICIENT_DEFAULT=y -CONFIG_ENERGY_MODEL=y -CONFIG_ARM_PSCI_CPUIDLE=y -CONFIG_CPU_FREQ=y -CONFIG_CPU_FREQ_STAT=y -CONFIG_CPU_FREQ_GOV_POWERSAVE=m -CONFIG_CPU_FREQ_GOV_USERSPACE=y -CONFIG_CPU_FREQ_GOV_ONDEMAND=y -CONFIG_CPU_FREQ_GOV_CONSERVATIVE=m -CONFIG_CPUFREQ_DT=y -CONFIG_ARM_ALLWINNER_SUN50I_CPUFREQ_NVMEM=m -CONFIG_ARM_APPLE_SOC_CPUFREQ=y -CONFIG_ARM_ARMADA_37XX_CPUFREQ=y -CONFIG_ARM_SCPI_CPUFREQ=y -CONFIG_ARM_IMX_CPUFREQ_DT=m -CONFIG_ARM_MEDIATEK_CPUFREQ=y -CONFIG_ARM_QCOM_CPUFREQ_NVMEM=y -CONFIG_ARM_QCOM_CPUFREQ_HW=y -CONFIG_ARM_RASPBERRYPI_CPUFREQ=m -CONFIG_ARM_SCMI_CPUFREQ=y -CONFIG_ARM_TEGRA186_CPUFREQ=y -CONFIG_QORIQ_CPUFREQ=y -CONFIG_ACPI_CPPC_CPUFREQ=m -CONFIG_ACPI=y -CONFIG_ACPI_HOTPLUG_MEMORY=y -CONFIG_ACPI_HMAT=y -CONFIG_ACPI_APEI=y -CONFIG_ACPI_APEI_GHES=y -CONFIG_ACPI_APEI_PCIEAER=y -CONFIG_ACPI_APEI_MEMORY_FAILURE=y -CONFIG_ACPI_APEI_EINJ=y -CONFIG_VIRTUALIZATION=y -CONFIG_KVM=y -CONFIG_COMPAT_32BIT_TIME=y -# CONFIG_RELR is not set -CONFIG_MODULES=y -CONFIG_MODULE_UNLOAD=y -CONFIG_PARTITION_ADVANCED=y -CONFIG_AIX_PARTITION=y -CONFIG_OSF_PARTITION=y -CONFIG_MAC_PARTITION=y -CONFIG_BSD_DISKLABEL=y -CONFIG_MINIX_SUBPARTITION=y -CONFIG_SOLARIS_X86_PARTITION=y -CONFIG_UNIXWARE_DISKLABEL=y -CONFIG_LDM_PARTITION=y -CONFIG_SGI_PARTITION=y -CONFIG_SUN_PARTITION=y -# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set -CONFIG_BINFMT_MISC=m -# CONFIG_COMPAT_BRK is not set -CONFIG_MEMORY_HOTPLUG=y -CONFIG_MEMORY_HOTREMOVE=y -CONFIG_KSM=y -CONFIG_MEMORY_FAILURE=y -CONFIG_TRANSPARENT_HUGEPAGE=y -CONFIG_PAGE_BLOCK_MAX_ORDER=10 -CONFIG_USERFAULTFD=y -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_PACKET_DIAG=m -CONFIG_UNIX_DIAG=m -CONFIG_TLS=m -CONFIG_XFRM_USER=m -CONFIG_XFRM_INTERFACE=m -CONFIG_XFRM_STATISTICS=y -CONFIG_NET_KEY=m -CONFIG_NET_KEY_MIGRATE=y -CONFIG_IP_MULTICAST=y -CONFIG_IP_ADVANCED_ROUTER=y -CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IP_ROUTE_MULTIPATH=y -CONFIG_IP_PNP=y -CONFIG_IP_PNP_DHCP=y -CONFIG_IP_PNP_BOOTP=y -CONFIG_NET_IPIP=m -CONFIG_NET_IPGRE_DEMUX=m -CONFIG_NET_IPGRE=m -CONFIG_NET_IPGRE_BROADCAST=y -CONFIG_NET_IPVTI=m -CONFIG_NET_FOU_IP_TUNNELS=y -CONFIG_INET_AH=m -CONFIG_INET_ESP=m -CONFIG_INET_ESP_OFFLOAD=m -CONFIG_INET_ESPINTCP=y -CONFIG_INET_IPCOMP=m -CONFIG_INET_UDP_DIAG=m -CONFIG_INET_RAW_DIAG=m -CONFIG_INET_DIAG_DESTROY=y -CONFIG_TCP_CONG_ADVANCED=y -CONFIG_TCP_CONG_HSTCP=m -CONFIG_TCP_CONG_HYBLA=m -CONFIG_TCP_CONG_NV=m -CONFIG_TCP_CONG_SCALABLE=m -CONFIG_TCP_CONG_LP=m -CONFIG_TCP_CONG_VENO=m -CONFIG_TCP_CONG_YEAH=m -CONFIG_TCP_CONG_ILLINOIS=m -CONFIG_TCP_CONG_DCTCP=m -CONFIG_TCP_CONG_CDG=m -CONFIG_TCP_CONG_BBR=m -CONFIG_INET6_AH=m -CONFIG_INET6_ESP=m -CONFIG_INET6_ESP_OFFLOAD=m -CONFIG_INET6_ESPINTCP=y -CONFIG_INET6_IPCOMP=m -CONFIG_IPV6_VTI=m -CONFIG_IPV6_GRE=m -CONFIG_IPV6_MULTIPLE_TABLES=y -CONFIG_NETFILTER=y -CONFIG_BRIDGE_NETFILTER=m -CONFIG_NETFILTER_NETLINK_HOOK=m -CONFIG_NF_CONNTRACK=m -CONFIG_NF_CONNTRACK_ZONES=y -CONFIG_NF_CONNTRACK_PROCFS=y -CONFIG_NF_CONNTRACK_EVENTS=y -CONFIG_NF_CONNTRACK_TIMEOUT=y -CONFIG_NF_CONNTRACK_TIMESTAMP=y -CONFIG_NF_CONNTRACK_AMANDA=m -CONFIG_NF_CONNTRACK_FTP=m -CONFIG_NF_CONNTRACK_H323=m -CONFIG_NF_CONNTRACK_IRC=m -CONFIG_NF_CONNTRACK_NETBIOS_NS=m -CONFIG_NF_CONNTRACK_SNMP=m -CONFIG_NF_CONNTRACK_PPTP=m -CONFIG_NF_CONNTRACK_SANE=m -CONFIG_NF_CONNTRACK_SIP=m -CONFIG_NF_CONNTRACK_TFTP=m -CONFIG_NF_CT_NETLINK=m -CONFIG_NF_CT_NETLINK_TIMEOUT=m -CONFIG_NF_CT_NETLINK_HELPER=m -CONFIG_NETFILTER_NETLINK_GLUE_CT=y -CONFIG_NF_TABLES=m -CONFIG_NF_TABLES_INET=y -CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_NUMGEN=m -CONFIG_NFT_CT=m -CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_CONNLIMIT=m -CONFIG_NFT_LOG=m -CONFIG_NFT_LIMIT=m -CONFIG_NFT_MASQ=m -CONFIG_NFT_REDIR=m -CONFIG_NFT_NAT=m -CONFIG_NFT_TUNNEL=m -CONFIG_NFT_QUEUE=m -CONFIG_NFT_QUOTA=m -CONFIG_NFT_REJECT=m -CONFIG_NFT_COMPAT=m -CONFIG_NFT_HASH=m -CONFIG_NFT_FIB_INET=m -CONFIG_NFT_XFRM=m -CONFIG_NFT_SOCKET=m -CONFIG_NFT_OSF=m -CONFIG_NFT_TPROXY=m -CONFIG_NFT_DUP_NETDEV=m -CONFIG_NFT_FWD_NETDEV=m -CONFIG_NFT_FIB_NETDEV=m -CONFIG_NFT_REJECT_NETDEV=m -CONFIG_NF_FLOW_TABLE_INET=m -CONFIG_NF_FLOW_TABLE=m -CONFIG_NF_FLOW_TABLE_PROCFS=y -CONFIG_NETFILTER_XTABLES_LEGACY=y -CONFIG_NETFILTER_XT_SET=m -CONFIG_NETFILTER_XT_TARGET_AUDIT=m -CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m -CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m -CONFIG_NETFILTER_XT_TARGET_CONNMARK=m -CONFIG_NETFILTER_XT_TARGET_DSCP=m -CONFIG_NETFILTER_XT_TARGET_HMARK=m -CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m -CONFIG_NETFILTER_XT_TARGET_LED=m -CONFIG_NETFILTER_XT_TARGET_LOG=m -CONFIG_NETFILTER_XT_TARGET_MARK=m -CONFIG_NETFILTER_XT_TARGET_NFLOG=m -CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m -CONFIG_NETFILTER_XT_TARGET_NOTRACK=m -CONFIG_NETFILTER_XT_TARGET_TEE=m -CONFIG_NETFILTER_XT_TARGET_TPROXY=m -CONFIG_NETFILTER_XT_TARGET_TRACE=m -CONFIG_NETFILTER_XT_TARGET_TCPMSS=m -CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m -CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m -CONFIG_NETFILTER_XT_MATCH_BPF=m -CONFIG_NETFILTER_XT_MATCH_CGROUP=m -CONFIG_NETFILTER_XT_MATCH_CLUSTER=m -CONFIG_NETFILTER_XT_MATCH_COMMENT=m -CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m -CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m -CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m -CONFIG_NETFILTER_XT_MATCH_CONNMARK=m -CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m -CONFIG_NETFILTER_XT_MATCH_CPU=m -CONFIG_NETFILTER_XT_MATCH_DCCP=m -CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m -CONFIG_NETFILTER_XT_MATCH_DSCP=m -CONFIG_NETFILTER_XT_MATCH_ESP=m -CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m -CONFIG_NETFILTER_XT_MATCH_HELPER=m -CONFIG_NETFILTER_XT_MATCH_IPCOMP=m -CONFIG_NETFILTER_XT_MATCH_IPRANGE=m -CONFIG_NETFILTER_XT_MATCH_IPVS=m -CONFIG_NETFILTER_XT_MATCH_LENGTH=m -CONFIG_NETFILTER_XT_MATCH_LIMIT=m -CONFIG_NETFILTER_XT_MATCH_MAC=m -CONFIG_NETFILTER_XT_MATCH_MARK=m -CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m -CONFIG_NETFILTER_XT_MATCH_NFACCT=m -CONFIG_NETFILTER_XT_MATCH_OSF=m -CONFIG_NETFILTER_XT_MATCH_OWNER=m -CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m -CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m -CONFIG_NETFILTER_XT_MATCH_QUOTA=m -CONFIG_NETFILTER_XT_MATCH_RATEEST=m -CONFIG_NETFILTER_XT_MATCH_REALM=m -CONFIG_NETFILTER_XT_MATCH_RECENT=m -CONFIG_NETFILTER_XT_MATCH_SCTP=m -CONFIG_NETFILTER_XT_MATCH_SOCKET=m -CONFIG_NETFILTER_XT_MATCH_STATE=m -CONFIG_NETFILTER_XT_MATCH_STATISTIC=m -CONFIG_NETFILTER_XT_MATCH_STRING=m -CONFIG_NETFILTER_XT_MATCH_TCPMSS=m -CONFIG_NETFILTER_XT_MATCH_TIME=m -CONFIG_NETFILTER_XT_MATCH_U32=m -CONFIG_IP_SET=m -CONFIG_IP_SET_BITMAP_IP=m -CONFIG_IP_SET_BITMAP_IPMAC=m -CONFIG_IP_SET_BITMAP_PORT=m -CONFIG_IP_SET_HASH_IP=m -CONFIG_IP_SET_HASH_IPMARK=m -CONFIG_IP_SET_HASH_IPPORT=m -CONFIG_IP_SET_HASH_IPPORTIP=m -CONFIG_IP_SET_HASH_IPPORTNET=m -CONFIG_IP_SET_HASH_IPMAC=m -CONFIG_IP_SET_HASH_MAC=m -CONFIG_IP_SET_HASH_NETPORTNET=m -CONFIG_IP_SET_HASH_NET=m -CONFIG_IP_SET_HASH_NETNET=m -CONFIG_IP_SET_HASH_NETPORT=m -CONFIG_IP_SET_HASH_NETIFACE=m -CONFIG_IP_SET_LIST_SET=m -CONFIG_IP_VS=m -CONFIG_NFT_DUP_IPV4=m -CONFIG_NFT_FIB_IPV4=m -CONFIG_NF_TABLES_ARP=y -CONFIG_NF_LOG_ARP=m -CONFIG_NF_LOG_IPV4=m -CONFIG_IP_NF_IPTABLES=m -CONFIG_IP_NF_MATCH_AH=m -CONFIG_IP_NF_MATCH_ECN=m -CONFIG_IP_NF_MATCH_RPFILTER=m -CONFIG_IP_NF_MATCH_TTL=m -CONFIG_IP_NF_TARGET_REJECT=m -CONFIG_IP_NF_NAT=m -CONFIG_IP_NF_TARGET_MASQUERADE=m -CONFIG_IP_NF_TARGET_NETMAP=m -CONFIG_IP_NF_TARGET_REDIRECT=m -CONFIG_IP_NF_TARGET_ECN=m -CONFIG_IP_NF_TARGET_TTL=m -CONFIG_IP_NF_RAW=m -CONFIG_IP_NF_ARPFILTER=m -CONFIG_IP_NF_ARP_MANGLE=m -CONFIG_NFT_DUP_IPV6=m -CONFIG_NFT_FIB_IPV6=m -CONFIG_IP6_NF_IPTABLES=m -CONFIG_IP6_NF_MATCH_AH=m -CONFIG_IP6_NF_MATCH_EUI64=m -CONFIG_IP6_NF_MATCH_FRAG=m -CONFIG_IP6_NF_MATCH_OPTS=m -CONFIG_IP6_NF_MATCH_HL=m -CONFIG_IP6_NF_MATCH_IPV6HEADER=m -CONFIG_IP6_NF_MATCH_MH=m -CONFIG_IP6_NF_MATCH_RPFILTER=m -CONFIG_IP6_NF_MATCH_RT=m -CONFIG_IP6_NF_MATCH_SRH=m -CONFIG_IP6_NF_TARGET_HL=m -CONFIG_IP6_NF_TARGET_REJECT=m -CONFIG_IP6_NF_RAW=m -CONFIG_IP6_NF_NAT=m -CONFIG_IP6_NF_TARGET_MASQUERADE=m -CONFIG_IP6_NF_TARGET_NPT=m -CONFIG_NF_TABLES_BRIDGE=m -CONFIG_NFT_BRIDGE_META=m -CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NF_CONNTRACK_BRIDGE=m -CONFIG_BRIDGE_NF_EBTABLES_LEGACY=m -CONFIG_BRIDGE_NF_EBTABLES=m -CONFIG_BRIDGE_EBT_BROUTE=m -CONFIG_BRIDGE_EBT_T_FILTER=m -CONFIG_BRIDGE_EBT_T_NAT=m -CONFIG_BRIDGE_EBT_802_3=m -CONFIG_BRIDGE_EBT_AMONG=m -CONFIG_BRIDGE_EBT_ARP=m -CONFIG_BRIDGE_EBT_IP=m -CONFIG_BRIDGE_EBT_IP6=m -CONFIG_BRIDGE_EBT_LIMIT=m -CONFIG_BRIDGE_EBT_MARK=m -CONFIG_BRIDGE_EBT_PKTTYPE=m -CONFIG_BRIDGE_EBT_STP=m -CONFIG_BRIDGE_EBT_VLAN=m -CONFIG_BRIDGE_EBT_ARPREPLY=m -CONFIG_BRIDGE_EBT_DNAT=m -CONFIG_BRIDGE_EBT_MARK_T=m -CONFIG_BRIDGE_EBT_REDIRECT=m -CONFIG_BRIDGE_EBT_SNAT=m -CONFIG_BRIDGE_EBT_LOG=m -CONFIG_BRIDGE_EBT_NFLOG=m -CONFIG_L2TP=m -CONFIG_L2TP_V3=y -CONFIG_L2TP_IP=m -CONFIG_L2TP_ETH=m -CONFIG_BRIDGE=m -CONFIG_BRIDGE_VLAN_FILTERING=y -CONFIG_NET_DSA=m -CONFIG_VLAN_8021Q=m -CONFIG_VLAN_8021Q_GVRP=y -CONFIG_VLAN_8021Q_MVRP=y -CONFIG_NET_SCHED=y -CONFIG_NET_SCH_HTB=m -CONFIG_NET_SCH_HFSC=m -CONFIG_NET_SCH_PRIO=m -CONFIG_NET_SCH_MULTIQ=m -CONFIG_NET_SCH_RED=m -CONFIG_NET_SCH_SFB=m -CONFIG_NET_SCH_SFQ=m -CONFIG_NET_SCH_TEQL=m -CONFIG_NET_SCH_TBF=m -CONFIG_NET_SCH_CBS=m -CONFIG_NET_SCH_ETF=m -CONFIG_NET_SCH_TAPRIO=m -CONFIG_NET_SCH_GRED=m -CONFIG_NET_SCH_NETEM=m -CONFIG_NET_SCH_DRR=m -CONFIG_NET_SCH_MQPRIO=m -CONFIG_NET_SCH_CHOKE=m -CONFIG_NET_SCH_QFQ=m -CONFIG_NET_SCH_CODEL=m -CONFIG_NET_SCH_FQ_CODEL=m -CONFIG_NET_SCH_CAKE=m -CONFIG_NET_SCH_FQ=m -CONFIG_NET_SCH_HHF=m -CONFIG_NET_SCH_PIE=m -CONFIG_NET_SCH_FQ_PIE=m -CONFIG_NET_SCH_INGRESS=m -CONFIG_NET_SCH_PLUG=m -CONFIG_NET_SCH_ETS=m -CONFIG_NET_SCH_DUALPI2=m -CONFIG_NET_CLS_BASIC=m -CONFIG_NET_CLS_ROUTE4=m -CONFIG_NET_CLS_FW=m -CONFIG_NET_CLS_U32=m -CONFIG_CLS_U32_PERF=y -CONFIG_CLS_U32_MARK=y -CONFIG_NET_CLS_FLOW=m -CONFIG_NET_CLS_BPF=m -CONFIG_NET_CLS_FLOWER=m -CONFIG_NET_CLS_MATCHALL=m -CONFIG_NET_EMATCH=y -CONFIG_NET_EMATCH_CMP=m -CONFIG_NET_EMATCH_NBYTE=m -CONFIG_NET_EMATCH_U32=m -CONFIG_NET_EMATCH_META=m -CONFIG_NET_EMATCH_TEXT=m -CONFIG_NET_EMATCH_IPSET=m -CONFIG_NET_EMATCH_IPT=m -CONFIG_NET_CLS_ACT=y -CONFIG_NET_ACT_POLICE=m -CONFIG_NET_ACT_GACT=m -CONFIG_GACT_PROB=y -CONFIG_NET_ACT_MIRRED=m -CONFIG_NET_ACT_SAMPLE=m -CONFIG_NET_ACT_NAT=m -CONFIG_NET_ACT_PEDIT=m -CONFIG_NET_ACT_SIMP=m -CONFIG_NET_ACT_SKBEDIT=m -CONFIG_NET_ACT_CSUM=m -CONFIG_NET_ACT_MPLS=m -CONFIG_NET_ACT_VLAN=m -CONFIG_NET_ACT_BPF=m -CONFIG_NET_ACT_CONNMARK=m -CONFIG_NET_ACT_CTINFO=m -CONFIG_NET_ACT_SKBMOD=m -CONFIG_NET_ACT_TUNNEL_KEY=m -CONFIG_NET_ACT_CT=m -CONFIG_NET_ACT_GATE=m -CONFIG_VSOCKETS=m -CONFIG_NETLINK_DIAG=m -CONFIG_HSR=m -CONFIG_QRTR_SMD=m -CONFIG_QRTR_TUN=m -CONFIG_CAN=m -CONFIG_CAN_J1939=m -CONFIG_CAN_ISOTP=m -CONFIG_BT=m -CONFIG_BT_RFCOMM=m -CONFIG_BT_RFCOMM_TTY=y -CONFIG_BT_BNEP=m -CONFIG_BT_BNEP_MC_FILTER=y -CONFIG_BT_BNEP_PROTO_FILTER=y -CONFIG_BT_HIDP=m -CONFIG_BT_LEDS=y -CONFIG_BT_MSFTEXT=y -CONFIG_BT_AOSPEXT=y -CONFIG_BT_BRCMEXT=y -# CONFIG_BT_DEBUGFS is not set -CONFIG_BT_MTK7932_PCIE=m -CONFIG_BT_HCIBTUSB=m -CONFIG_BT_HCIBTUSB_AUTOSUSPEND=y -CONFIG_BT_HCIBTUSB_MTK=y -CONFIG_BT_HCIUART=m -CONFIG_BT_HCIUART_LL=y -CONFIG_BT_HCIUART_BCM=y -CONFIG_BT_HCIUART_QCA=y -CONFIG_BT_HCIUART_MRVL=y -CONFIG_BT_HCIBCM203X=m -CONFIG_BT_HCIBCM4377=m -CONFIG_BT_HCIBPA10X=m -CONFIG_BT_HCIBFUSB=m -CONFIG_BT_HCIVHCI=m -CONFIG_BT_MRVL=m -CONFIG_BT_MRVL_SDIO=m -CONFIG_BT_ATH3K=m -CONFIG_BT_QCOMSMD=m -CONFIG_BT_NXPUART=m -CONFIG_CFG80211=m -CONFIG_MAC80211=m -CONFIG_RFKILL=y -CONFIG_RFKILL_GPIO=m -CONFIG_NET_9P=y -CONFIG_NET_9P_VIRTIO=y -CONFIG_NFC=m -CONFIG_NFC_NCI=m -CONFIG_NFC_NXP_NCI=m -CONFIG_NFC_NXP_NCI_I2C=m -CONFIG_NFC_S3FWRN5_I2C=m -CONFIG_PCI=y -CONFIG_PCIEPORTBUS=y -CONFIG_PCIEAER=y -CONFIG_PCI_IOV=y -CONFIG_PCI_PASID=y -CONFIG_HOTPLUG_PCI=y -CONFIG_HOTPLUG_PCI_ACPI=y -CONFIG_PCI_AARDVARK=y -CONFIG_PCIE_ALTERA=y -CONFIG_PCIE_ALTERA_MSI=y -CONFIG_PCIE_APPLE=y -CONFIG_PCIE_APPLE_PIODMA_DIAG=y -CONFIG_PCIE_BRCMSTB=m -CONFIG_PCI_HOST_THUNDER_PEM=y -CONFIG_PCI_HOST_THUNDER_ECAM=y -CONFIG_PCI_HOST_GENERIC=y -CONFIG_PCIE_MEDIATEK_GEN3=m -CONFIG_PCI_TEGRA=y -CONFIG_PCIE_RCAR_HOST=y -CONFIG_PCIE_RCAR_EP=y -CONFIG_PCIE_RENESAS_RZG3S_HOST=y -CONFIG_PCIE_ROCKCHIP_HOST=m -CONFIG_PCI_XGENE=y -CONFIG_PCIE_XILINX=y -CONFIG_PCIE_XILINX_DMA_PL=y -CONFIG_PCIE_XILINX_NWL=y -CONFIG_PCIE_XILINX_CPM=y -CONFIG_PCI_J721E_HOST=m -CONFIG_PCI_IMX6_HOST=y -CONFIG_PCI_LAYERSCAPE=y -CONFIG_PCI_HISI=y -CONFIG_PCIE_KIRIN=y -CONFIG_PCIE_HISI_STB=y -CONFIG_PCIE_ARMADA_8K=y -CONFIG_PCIE_TEGRA194_HOST=m -CONFIG_PCIE_TEGRA194_EP=m -CONFIG_PCIE_QCOM=y -CONFIG_PCIE_RCAR_GEN4_HOST=m -CONFIG_PCIE_RCAR_GEN4_EP=m -CONFIG_PCIE_ROCKCHIP_DW_HOST=y -CONFIG_PCIE_VISCONTI_HOST=y -CONFIG_PCIE_LAYERSCAPE_GEN4=y -CONFIG_PCI_ENDPOINT=y -CONFIG_PCI_ENDPOINT_CONFIGFS=y -CONFIG_PCI_EPF_TEST=m -CONFIG_DEVTMPFS=y -CONFIG_DEVTMPFS_MOUNT=y -CONFIG_FW_LOADER_USER_HELPER=y -CONFIG_FW_LOADER_COMPRESS=y -CONFIG_FW_LOADER_COMPRESS_ZSTD=y -CONFIG_HISILICON_LPC=y -CONFIG_IMX_AIPSTZ=m -CONFIG_TEGRA_ACONNECT=m -CONFIG_MHI_BUS_PCI_GENERIC=m -CONFIG_ARM_SCPI_PROTOCOL=y -CONFIG_RASPBERRYPI_FIRMWARE=y -CONFIG_INTEL_STRATIX10_SERVICE=y -CONFIG_INTEL_STRATIX10_RSU=m -CONFIG_MTK_ADSP_IPC=m -CONFIG_SYSFB_SIMPLEFB=y -CONFIG_GOOGLE_FIRMWARE=y -CONFIG_GOOGLE_CBMEM=m -CONFIG_GOOGLE_COREBOOT_TABLE=m -CONFIG_EFI_CAPSULE_LOADER=y -CONFIG_IMX_SCU=y -CONFIG_QCOM_TZMEM_MODE_SHMBRIDGE=y -CONFIG_QCOM_QSEECOM=y -CONFIG_QCOM_QSEECOM_UEFISECAPP=y -CONFIG_EXYNOS_ACPM_PROTOCOL=m -CONFIG_TEGRA_BPMP=y -CONFIG_ZYNQMP_FIRMWARE_DEBUG=y -CONFIG_GNSS=m -CONFIG_GNSS_MTK_SERIAL=m -CONFIG_GNSS_USB=m -CONFIG_MTD=y -CONFIG_MTD_BLOCK=y -CONFIG_MTD_CFI=y -CONFIG_MTD_CFI_ADV_OPTIONS=y -CONFIG_MTD_CFI_INTELEXT=y -CONFIG_MTD_CFI_AMDSTD=y -CONFIG_MTD_CFI_STAA=y -CONFIG_MTD_PHYSMAP=y -CONFIG_MTD_PHYSMAP_OF=y -CONFIG_MTD_DATAFLASH=y -CONFIG_MTD_SST25L=y -CONFIG_MTD_RAW_NAND=y -CONFIG_MTD_NAND_DENALI_DT=y -CONFIG_MTD_NAND_MARVELL=y -CONFIG_MTD_NAND_BRCMNAND=m -CONFIG_MTD_NAND_FSL_IFC=y -CONFIG_MTD_NAND_QCOM=y -CONFIG_MTD_SPI_NAND=m -CONFIG_MTD_SPI_NOR=y -CONFIG_MTD_UBI=m -CONFIG_MTD_HYPERBUS=m -CONFIG_HBMC_AM654=m -CONFIG_OF_OVERLAY=y -CONFIG_ZRAM=y -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_NBD=m -CONFIG_VIRTIO_BLK=y -CONFIG_BLK_DEV_NVME=y -CONFIG_NVME_HWMON=y -CONFIG_NVME_APPLE=y -CONFIG_QCOM_COINCELL=m -CONFIG_QCOM_FASTRPC=m -CONFIG_SRAM=y -CONFIG_PCI_ENDPOINT_TEST=m -CONFIG_XILINX_SDFEC=m -CONFIG_NTSYNC=m -CONFIG_EEPROM_AT24=m -CONFIG_EEPROM_AT25=m -CONFIG_MISC_RTSX_USB=m -CONFIG_UACCE=m -CONFIG_MISC_RP1=m -# CONFIG_SCSI_PROC_FS is not set -CONFIG_BLK_DEV_SD=y -CONFIG_BLK_DEV_SR=m -CONFIG_CHR_DEV_SG=m -CONFIG_SCSI_SAS_ATA=y -CONFIG_SCSI_HISI_SAS=y -CONFIG_SCSI_HISI_SAS_PCI=y -CONFIG_MEGARAID_SAS=y -CONFIG_SCSI_MPT3SAS=m -CONFIG_ATA=y -CONFIG_SATA_AHCI=y -CONFIG_SATA_AHCI_PLATFORM=y -CONFIG_AHCI_BRCM=m -CONFIG_AHCI_DWC=m -CONFIG_AHCI_CEVA=y -CONFIG_AHCI_MVEBU=y -CONFIG_AHCI_XGENE=y -CONFIG_AHCI_QORIQ=y -CONFIG_SATA_SIL24=y -CONFIG_SATA_RCAR=y -CONFIG_PATA_OF_PLATFORM=y -CONFIG_MD=y -CONFIG_MD_LINEAR=m -CONFIG_BLK_DEV_DM=m -CONFIG_DM_CRYPT=m -CONFIG_DM_SNAPSHOT=m -CONFIG_DM_THIN_PROVISIONING=m -CONFIG_DM_CACHE=m -CONFIG_DM_WRITECACHE=m -CONFIG_DM_MIRROR=m -CONFIG_DM_RAID=m -CONFIG_DM_ZERO=m -CONFIG_DM_UEVENT=y -CONFIG_DM_VERITY=m -CONFIG_DM_VERITY_FEC=y -CONFIG_DM_INTEGRITY=m -CONFIG_NETDEVICES=y -CONFIG_BONDING=m -CONFIG_DUMMY=m -CONFIG_WIREGUARD=m -CONFIG_OVPN=m -CONFIG_IFB=m -CONFIG_NET_TEAM=m -CONFIG_NET_TEAM_MODE_BROADCAST=m -CONFIG_NET_TEAM_MODE_ROUNDROBIN=m -CONFIG_NET_TEAM_MODE_RANDOM=m -CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m -CONFIG_NET_TEAM_MODE_LOADBALANCE=m -CONFIG_MACVLAN=m -CONFIG_MACVTAP=m -CONFIG_VXLAN=m -CONFIG_GENEVE=m -CONFIG_TUN=y -CONFIG_VETH=m -CONFIG_VIRTIO_NET=y -CONFIG_MHI_NET=m -CONFIG_NET_DSA_BCM_SF2=m -CONFIG_NET_DSA_MSCC_FELIX=m -CONFIG_ENA_ETHERNET=m -CONFIG_AMD_XGBE=y -CONFIG_NET_XGENE=y -CONFIG_ATL1C=m -CONFIG_BCMGENET=m -CONFIG_BNX2X=m -CONFIG_SYSTEMPORT=m -CONFIG_MACB=y -CONFIG_THUNDER_NIC_PF=y -CONFIG_FEC=y -CONFIG_FSL_FMAN=y -CONFIG_FSL_DPAA_ETH=y -CONFIG_FSL_DPAA2_ETH=y -CONFIG_FSL_ENETC=y -CONFIG_NXP_ENETC4=m -CONFIG_FSL_ENETC_VF=y -CONFIG_FSL_ENETC_QOS=y -CONFIG_NXP_NETC_BLK_CTRL=m -CONFIG_HIX5HD2_GMAC=y -CONFIG_HNS_DSAF=y -CONFIG_HNS_ENET=y -CONFIG_HNS3=y -CONFIG_HNS3_HCLGE=y -CONFIG_HNS3_ENET=y -CONFIG_E1000=y -CONFIG_E1000E=y -CONFIG_IGB=m -CONFIG_IGBVF=y -CONFIG_IGC=m -CONFIG_MVNETA=y -CONFIG_MVPP2=y -CONFIG_SKY2=y -CONFIG_NET_VENDOR_MEDIATEK=y -CONFIG_NET_MEDIATEK_STAR_EMAC=m -CONFIG_MLX4_EN=m -CONFIG_MLX5_CORE=m -CONFIG_MLX5_CORE_EN=y -CONFIG_QCOM_EMAC=m -CONFIG_RMNET=m -CONFIG_R8169=m -CONFIG_SH_ETH=y -CONFIG_RAVB=y -CONFIG_RENESAS_ETHER_SWITCH=y -CONFIG_RTSN=y -CONFIG_SMC91X=y -CONFIG_SMSC911X=y -CONFIG_SNI_AVE=y -CONFIG_SNI_NETSEC=y -CONFIG_STMMAC_ETH=m -CONFIG_DWMAC_MEDIATEK=m -CONFIG_DWMAC_TEGRA=m -CONFIG_TI_K3_AM65_CPSW_NUSS=y -CONFIG_TI_ICSSG_PRUETH=m -CONFIG_XILINX_AXI_EMAC=m -CONFIG_QCOM_IPA=m -CONFIG_MESON_GXL_PHY=m -CONFIG_AQUANTIA_PHY=y -CONFIG_BCM54140_PHY=m -CONFIG_MARVELL_PHY=m -CONFIG_MARVELL_10G_PHY=y -CONFIG_MARVELL_88Q2XXX_PHY=y -CONFIG_MICREL_PHY=y -CONFIG_MICROSEMI_PHY=y -CONFIG_AT803X_PHY=y -CONFIG_QCA808X_PHY=m -CONFIG_REALTEK_PHY=y -CONFIG_ROCKCHIP_PHY=y -CONFIG_DP83867_PHY=y -CONFIG_DP83869_PHY=m -CONFIG_DP83TD510_PHY=y -CONFIG_DP83TG720_PHY=m -CONFIG_VITESSE_PHY=y -CONFIG_XILINX_GMII2RGMII=m -CONFIG_CAN_FLEXCAN=m -CONFIG_CAN_XILINXCAN=m -CONFIG_CAN_M_CAN=m -CONFIG_CAN_M_CAN_PLATFORM=m -CONFIG_CAN_RCAR=m -CONFIG_CAN_RCAR_CANFD=m -CONFIG_CAN_MCP251XFD=m -CONFIG_CAN_8DEV_USB=m -CONFIG_CAN_EMS_USB=m -CONFIG_CAN_ESD_USB=m -CONFIG_CAN_F81604=m -CONFIG_CAN_GS_USB=m -CONFIG_CAN_KVASER_USB=m -CONFIG_CAN_MCBA_USB=m -CONFIG_CAN_PEAK_USB=m -CONFIG_MDIO_GPIO=y -CONFIG_MDIO_BUS_MUX_MULTIPLEXER=y -CONFIG_MDIO_BUS_MUX_MMIOREG=y -CONFIG_PPP=m -CONFIG_PPP_BSDCOMP=m -CONFIG_PPP_DEFLATE=m -CONFIG_PPP_FILTER=y -CONFIG_PPP_MPPE=m -CONFIG_PPP_MULTILINK=y -CONFIG_PPPOE=m -CONFIG_PPTP=m -CONFIG_PPPOL2TP=m -CONFIG_PPP_ASYNC=m -CONFIG_PPP_SYNC_TTY=m -CONFIG_USB_CATC=m -CONFIG_USB_KAWETH=m -CONFIG_USB_PEGASUS=m -CONFIG_USB_RTL8150=m -CONFIG_USB_RTL8152=y -CONFIG_USB_LAN78XX=m -CONFIG_USB_USBNET=y -CONFIG_USB_NET_CDC_EEM=m -CONFIG_USB_NET_HUAWEI_CDC_NCM=m -CONFIG_USB_NET_CDC_MBIM=m -CONFIG_USB_NET_DM9601=m -CONFIG_USB_NET_SR9700=m -CONFIG_USB_NET_SR9800=m -CONFIG_USB_NET_SMSC75XX=m -CONFIG_USB_NET_SMSC95XX=m -CONFIG_USB_NET_GL620A=m -CONFIG_USB_NET_NET1080=m -CONFIG_USB_NET_PLUSB=m -CONFIG_USB_NET_MCS7830=m -CONFIG_USB_NET_RNDIS_HOST=m -CONFIG_USB_NET_CDC_SUBSET=m -CONFIG_USB_ALI_M5632=y -CONFIG_USB_AN2720=y -CONFIG_USB_EPSON2888=y -CONFIG_USB_KC2190=y -CONFIG_USB_NET_ZAURUS=m -CONFIG_USB_NET_CX82310_ETH=m -CONFIG_USB_NET_KALMIA=m -CONFIG_USB_NET_QMI_WWAN=m -CONFIG_USB_HSO=m -CONFIG_USB_NET_INT51X1=m -CONFIG_USB_IPHETH=m -CONFIG_USB_SIERRA_NET=m -CONFIG_USB_VL600=m -CONFIG_USB_NET_CH9200=m -CONFIG_USB_NET_AQC111=m -CONFIG_ATH9K_HTC=m -CONFIG_CARL9170=m -CONFIG_ATH6KL=m -CONFIG_ATH6KL_USB=m -CONFIG_AR5523=m -CONFIG_ATH10K=m -CONFIG_ATH10K_PCI=m -CONFIG_ATH10K_SDIO=m -CONFIG_ATH10K_USB=m -CONFIG_ATH10K_SNOC=m -CONFIG_WCN36XX=m -CONFIG_ATH11K=m -CONFIG_ATH11K_AHB=m -CONFIG_ATH11K_PCI=m -CONFIG_ATH12K=m -CONFIG_BRCMFMAC=m -CONFIG_BRCMFMAC_USB=y -CONFIG_BRCMFMAC_PCIE=y -CONFIG_IWLWIFI=m -CONFIG_IWLDVM=m -CONFIG_IWLMVM=m -CONFIG_LIBERTAS_THINFIRM=m -CONFIG_LIBERTAS_THINFIRM_USB=m -CONFIG_MWIFIEX=m -CONFIG_MWIFIEX_SDIO=m -CONFIG_MWIFIEX_PCIE=m -CONFIG_MWIFIEX_USB=m -CONFIG_MT7601U=m -CONFIG_MT76x0U=m -CONFIG_MT76x2U=m -CONFIG_MT7663U=m -CONFIG_MT7921E=m -CONFIG_MT7921U=m -CONFIG_MT7925U=m -CONFIG_MT7932_FULLMAC=m -CONFIG_RT2X00=m -CONFIG_RT2500USB=m -CONFIG_RT73USB=m -CONFIG_RT2800USB=m -CONFIG_RT2800USB_RT3573=y -CONFIG_RT2800USB_RT53XX=y -CONFIG_RT2800USB_RT55XX=y -CONFIG_RT2800USB_UNKNOWN=y -CONFIG_RTL8187=m -CONFIG_RTL8192DU=m -CONFIG_RTL8XXXU=m -CONFIG_RTW88=m -CONFIG_RTW88_8822BU=m -CONFIG_RTW88_8822CU=m -CONFIG_RTW88_8723DU=m -CONFIG_RTW88_8821CU=m -CONFIG_RTW88_8821AU=m -CONFIG_RTW88_8812AU=m -CONFIG_RTW88_8814AU=m -CONFIG_RTW89=m -CONFIG_RTW89_8851BU=m -CONFIG_RTW89_8852AU=m -CONFIG_RTW89_8852BU=m -CONFIG_RTW89_8852CU=m -CONFIG_RSI_91X=m -CONFIG_WL18XX=m -CONFIG_WLCORE_SDIO=m -CONFIG_ZD1211RW=m -CONFIG_WWAN=m -CONFIG_MHI_WWAN_CTRL=m -CONFIG_MHI_WWAN_MBIM=m -CONFIG_INPUT_JOYDEV=m -CONFIG_INPUT_EVDEV=y -CONFIG_KEYBOARD_ADC=m -CONFIG_KEYBOARD_GPIO=y -CONFIG_KEYBOARD_GPIO_POLLED=m -CONFIG_KEYBOARD_SNVS_PWRKEY=m -CONFIG_KEYBOARD_IMX_SC_KEY=m -CONFIG_KEYBOARD_CROS_EC=y -CONFIG_KEYBOARD_MTK_PMIC=m -CONFIG_MOUSE_ELAN_I2C=m -CONFIG_MOUSE_SYNAPTICS_USB=m -CONFIG_INPUT_JOYSTICK=y -CONFIG_JOYSTICK_IFORCE=m -CONFIG_JOYSTICK_IFORCE_USB=m -CONFIG_JOYSTICK_XPAD=m -CONFIG_JOYSTICK_XPAD_FF=y -CONFIG_JOYSTICK_XPAD_LEDS=y -CONFIG_JOYSTICK_PXRC=m -CONFIG_INPUT_TABLET=y -CONFIG_TABLET_USB_ACECAD=m -CONFIG_TABLET_USB_AIPTEK=m -CONFIG_TABLET_USB_HANWANG=m -CONFIG_TABLET_USB_KBTAB=m -CONFIG_TABLET_USB_PEGASUS=m -CONFIG_INPUT_TOUCHSCREEN=y -CONFIG_TOUCHSCREEN_APPLE_Z2=m -CONFIG_TOUCHSCREEN_ATMEL_MXT=m -CONFIG_TOUCHSCREEN_GOODIX=m -CONFIG_TOUCHSCREEN_GOODIX_BERLIN_SPI=m -CONFIG_TOUCHSCREEN_ILI210X=m -CONFIG_TOUCHSCREEN_ELAN=m -CONFIG_TOUCHSCREEN_EDT_FT5X06=m -CONFIG_TOUCHSCREEN_USB_COMPOSITE=m -CONFIG_TOUCHSCREEN_HIMAX_HX83112B=m -CONFIG_INPUT_MISC=y -CONFIG_INPUT_AW86927=m -CONFIG_INPUT_BBNSM_PWRKEY=m -CONFIG_INPUT_PM8941_PWRKEY=y -CONFIG_INPUT_PM8XXX_VIBRATOR=m -CONFIG_INPUT_TPS65219_PWRBUTTON=m -CONFIG_INPUT_TPS6594_PWRBUTTON=m -CONFIG_INPUT_UINPUT=m -CONFIG_INPUT_PWM_BEEPER=m -CONFIG_INPUT_PWM_VIBRA=m -CONFIG_INPUT_RK805_PWRKEY=m -CONFIG_INPUT_DA9063_ONKEY=m -CONFIG_INPUT_HISI_POWERKEY=y -CONFIG_INPUT_MACSMC_INPUT=m -CONFIG_RMI4_F1A=y -CONFIG_RMI4_F21=y -CONFIG_RMI4_F34=y -CONFIG_RMI4_F55=y -# CONFIG_SERIO_SERPORT is not set -CONFIG_SERIO_AMBAKMI=y -CONFIG_LEGACY_PTY_COUNT=16 -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_SERIAL_8250_NR_UARTS=8 -CONFIG_SERIAL_8250_EXTENDED=y -CONFIG_SERIAL_8250_SHARE_IRQ=y -CONFIG_SERIAL_8250_BCM2835AUX=y -CONFIG_SERIAL_8250_DW=y -CONFIG_SERIAL_8250_EM=y -CONFIG_SERIAL_8250_OMAP=y -CONFIG_SERIAL_8250_MT6577=y -CONFIG_SERIAL_8250_UNIPHIER=y -CONFIG_SERIAL_OF_PLATFORM=y -CONFIG_SERIAL_AMBA_PL011=y -CONFIG_SERIAL_AMBA_PL011_CONSOLE=y -CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y -CONFIG_SERIAL_APPLE_DOCKCHANNEL=y -CONFIG_SERIAL_MESON=y -CONFIG_SERIAL_MESON_CONSOLE=y -CONFIG_SERIAL_SAMSUNG=y -CONFIG_SERIAL_SAMSUNG_CONSOLE=y -CONFIG_SERIAL_TEGRA=y -CONFIG_SERIAL_TEGRA_TCU=y -CONFIG_SERIAL_IMX=y -CONFIG_SERIAL_IMX_CONSOLE=y -CONFIG_SERIAL_SH_SCI=y -CONFIG_SERIAL_RSCI=y -CONFIG_SERIAL_MSM=y -CONFIG_SERIAL_MSM_CONSOLE=y -CONFIG_SERIAL_QCOM_GENI=y -CONFIG_SERIAL_QCOM_GENI_CONSOLE=y -CONFIG_SERIAL_XILINX_PS_UART=y -CONFIG_SERIAL_XILINX_PS_UART_CONSOLE=y -CONFIG_SERIAL_FSL_LPUART=y -CONFIG_SERIAL_FSL_LPUART_CONSOLE=y -CONFIG_SERIAL_FSL_LINFLEXUART=y -CONFIG_SERIAL_FSL_LINFLEXUART_CONSOLE=y -CONFIG_SERIAL_STM32=y -CONFIG_SERIAL_STM32_CONSOLE=y -CONFIG_SERIAL_MVEBU_UART=y -CONFIG_SERIAL_OWL=y -CONFIG_SERIAL_DEV_BUS=y -CONFIG_VIRTIO_CONSOLE=y -CONFIG_IPMI_HANDLER=m -CONFIG_IPMI_DEVICE_INTERFACE=m -CONFIG_IPMI_SI=m -CONFIG_HW_RANDOM=y -CONFIG_HW_RANDOM_VIRTIO=y -CONFIG_TCG_TPM=y -CONFIG_TCG_TIS=m -CONFIG_TCG_TIS_SPI=m -CONFIG_TCG_TIS_SPI_CR50=y -CONFIG_TCG_TIS_I2C_CR50=m -CONFIG_TCG_TIS_I2C_INFINEON=y -CONFIG_XILLYUSB=m -CONFIG_I2C_CHARDEV=y -CONFIG_I2C_MUX=y -CONFIG_I2C_MUX_PCA954x=y -CONFIG_I2C_MUX_PINCTRL=m -CONFIG_I2C_BCM2835=m -CONFIG_I2C_CADENCE=m -CONFIG_I2C_DESIGNWARE_CORE=y -CONFIG_I2C_GPIO=m -CONFIG_I2C_IMX=y -CONFIG_I2C_IMX_LPI2C=y -CONFIG_I2C_MESON=y -CONFIG_I2C_MT65XX=y -CONFIG_I2C_MV64XXX=y -CONFIG_I2C_OMAP=y -CONFIG_I2C_OWL=y -CONFIG_I2C_APPLE=y -CONFIG_I2C_PXA=y -CONFIG_I2C_QCOM_CCI=m -CONFIG_I2C_QCOM_GENI=m -CONFIG_I2C_QUP=y -CONFIG_I2C_RIIC=y -CONFIG_I2C_RK3X=y -CONFIG_I2C_RZV2M=m -CONFIG_I2C_S3C2410=y -CONFIG_I2C_SH_MOBILE=y -CONFIG_I2C_TEGRA=y -CONFIG_I2C_UNIPHIER_F=y -CONFIG_I2C_XILINX=m -CONFIG_I2C_RCAR=y -CONFIG_I2C_DIOLAN_U2C=m -CONFIG_I2C_DLN2=m -CONFIG_I2C_CP2615=m -CONFIG_I2C_TINY_USB=m -CONFIG_I2C_CROS_EC_TUNNEL=y -CONFIG_SPI=y -CONFIG_SPI_APPLE=m -CONFIG_SPI_APPLE_QSPI=m -CONFIG_SPI_ARMADA_3700=y -CONFIG_SPI_BCM2835=m -CONFIG_SPI_BCM2835AUX=m -CONFIG_SPI_CADENCE_QUADSPI=y -CONFIG_SPI_CH341=m -CONFIG_SPI_DESIGNWARE=m -CONFIG_SPI_DW_DMA=y -CONFIG_SPI_DW_MMIO=m -CONFIG_SPI_DLN2=m -CONFIG_SPI_FSL_LPSPI=y -CONFIG_SPI_FSL_QUADSPI=y -CONFIG_SPI_NXP_FLEXSPI=y -CONFIG_SPI_IMX=m -CONFIG_SPI_FSL_DSPI=y -CONFIG_SPI_MESON_SPICC=m -CONFIG_SPI_MESON_SPIFC=m -CONFIG_SPI_MT65XX=y -CONFIG_SPI_MTK_NOR=m -CONFIG_SPI_OMAP24XX=m -CONFIG_SPI_ORION=y -CONFIG_SPI_PL022=y -CONFIG_SPI_ROCKCHIP=y -CONFIG_SPI_ROCKCHIP_SFC=m -CONFIG_SPI_RPCIF=m -CONFIG_SPI_RSPI=m -CONFIG_SPI_RZV2H_RSPI=m -CONFIG_SPI_RZV2M_CSI=m -CONFIG_SPI_QCOM_QSPI=m -CONFIG_SPI_QUP=y -CONFIG_SPI_QCOM_GENI=m -CONFIG_SPI_S3C64XX=y -CONFIG_SPI_SH_MSIOF=m -CONFIG_SPI_STM32_OSPI=m -CONFIG_SPI_SUN6I=y -CONFIG_SPI_TEGRA210_QUAD=m -CONFIG_SPI_TEGRA114=m -CONFIG_SPI_XILINX=m -CONFIG_SPI_ZYNQMP_GQSPI=m -CONFIG_SPI_SPIDEV=m -CONFIG_SPMI=y -CONFIG_SPMI_APPLE=y -CONFIG_SPMI_MTK_PMIF=m -CONFIG_PINCTRL_APPLE_GPIO=y -CONFIG_PINCTRL_DA9062=m -CONFIG_PINCTRL_MAX77620=y -CONFIG_PINCTRL_RK805=m -CONFIG_PINCTRL_SINGLE=y -CONFIG_PINCTRL_SX150X=m -CONFIG_PINCTRL_OWL=y -CONFIG_PINCTRL_S700=y -CONFIG_PINCTRL_S900=y -CONFIG_PINCTRL_BRCMSTB=y -CONFIG_PINCTRL_BCM2712=y -CONFIG_PINCTRL_IMX_SCMI=y -CONFIG_PINCTRL_IMX91=y -CONFIG_PINCTRL_MSM=y -CONFIG_PINCTRL_ELIZA=y -CONFIG_PINCTRL_GLYMUR=y -CONFIG_PINCTRL_IPQ5018=y -CONFIG_PINCTRL_IPQ5210=y -CONFIG_PINCTRL_IPQ5332=y -CONFIG_PINCTRL_IPQ5424=y -CONFIG_PINCTRL_IPQ8074=y -CONFIG_PINCTRL_IPQ6018=y -CONFIG_PINCTRL_IPQ9574=y -CONFIG_PINCTRL_KAANAPALI=y -CONFIG_PINCTRL_MSM8916=y -CONFIG_PINCTRL_MSM8953=y -CONFIG_PINCTRL_MSM8976=y -CONFIG_PINCTRL_MSM8994=y -CONFIG_PINCTRL_MSM8996=y -CONFIG_PINCTRL_MSM8998=y -CONFIG_PINCTRL_QCM2290=y -CONFIG_PINCTRL_QCS404=y -CONFIG_PINCTRL_QCS615=y -CONFIG_PINCTRL_QCS8300=y -CONFIG_PINCTRL_QDF2XXX=y -CONFIG_PINCTRL_QDU1000=y -CONFIG_PINCTRL_SA8775P=y -CONFIG_PINCTRL_SC7180=y -CONFIG_PINCTRL_SC7280=y -CONFIG_PINCTRL_SC8180X=y -CONFIG_PINCTRL_SC8280XP=y -CONFIG_PINCTRL_SDM660=y -CONFIG_PINCTRL_SDM670=y -CONFIG_PINCTRL_SDM845=y -CONFIG_PINCTRL_SDX75=y -CONFIG_PINCTRL_SM4450=y -CONFIG_PINCTRL_SM6115=y -CONFIG_PINCTRL_SM6125=y -CONFIG_PINCTRL_SM6350=y -CONFIG_PINCTRL_SM6375=y -CONFIG_PINCTRL_MILOS=y -CONFIG_PINCTRL_SM8150=y -CONFIG_PINCTRL_SM8250=y -CONFIG_PINCTRL_SM8350=y -CONFIG_PINCTRL_SM8450=y -CONFIG_PINCTRL_SM8550=y -CONFIG_PINCTRL_SM8650=y -CONFIG_PINCTRL_SM8750=y -CONFIG_PINCTRL_X1E80100=y -CONFIG_PINCTRL_QCOM_SPMI_PMIC=y -CONFIG_PINCTRL_LPASS_LPI=m -CONFIG_PINCTRL_MILOS_LPASS_LPI=m -CONFIG_PINCTRL_SC7280_LPASS_LPI=m -CONFIG_PINCTRL_SM6115_LPASS_LPI=m -CONFIG_PINCTRL_SM8250_LPASS_LPI=m -CONFIG_PINCTRL_SM8450_LPASS_LPI=m -CONFIG_PINCTRL_SC8280XP_LPASS_LPI=m -CONFIG_PINCTRL_SM8550_LPASS_LPI=m -CONFIG_PINCTRL_SM8650_LPASS_LPI=m -CONFIG_PINCTRL_SOPHGO_SG2000=y -CONFIG_GPIO_ALTERA=m -CONFIG_GPIO_DAVINCI=y -CONFIG_GPIO_DWAPB=y -CONFIG_GPIO_MB86S7X=y -CONFIG_GPIO_MPC8XXX=y -CONFIG_GPIO_MXC=y -CONFIG_GPIO_PL061=y -CONFIG_GPIO_RCAR=y -CONFIG_GPIO_SYSCON=y -CONFIG_GPIO_UNIPHIER=y -CONFIG_GPIO_VF610=y -CONFIG_GPIO_VISCONTI=y -CONFIG_GPIO_WCD934X=m -CONFIG_GPIO_XGENE=y -CONFIG_GPIO_XGENE_SB=y -CONFIG_GPIO_XILINX=m -CONFIG_GPIO_ZYNQ=m -CONFIG_GPIO_MAX732X=y -CONFIG_GPIO_PCA953X=y -CONFIG_GPIO_PCA953X_IRQ=y -CONFIG_GPIO_PCF857X=m -CONFIG_GPIO_TPIC2810=m -CONFIG_GPIO_ADP5585=m -CONFIG_GPIO_BD9571MWV=m -CONFIG_GPIO_DLN2=m -CONFIG_GPIO_MACSMC=y -CONFIG_GPIO_MAX77620=y -CONFIG_GPIO_SL28CPLD=m -CONFIG_GPIO_MPSSE=m -CONFIG_GPIO_AGGREGATOR=m -CONFIG_W1=m -CONFIG_W1_MASTER_DS2490=m -CONFIG_W1_SLAVE_THERM=m -CONFIG_W1_SLAVE_SMEM=m -CONFIG_W1_SLAVE_DS2405=m -CONFIG_W1_SLAVE_DS2408=m -CONFIG_W1_SLAVE_DS2413=m -CONFIG_W1_SLAVE_DS2406=m -CONFIG_W1_SLAVE_DS2423=m -CONFIG_W1_SLAVE_DS2805=m -CONFIG_W1_SLAVE_DS2430=m -CONFIG_W1_SLAVE_DS2431=m -CONFIG_W1_SLAVE_DS2433=m -CONFIG_W1_SLAVE_DS2433_CRC=y -CONFIG_W1_SLAVE_DS2438=m -CONFIG_W1_SLAVE_DS2780=m -CONFIG_W1_SLAVE_DS2781=m -CONFIG_W1_SLAVE_DS28E04=m -CONFIG_POWER_RESET_GPIO=y -CONFIG_POWER_RESET_GPIO_RESTART=y -CONFIG_POWER_RESET_MACSMC=y -CONFIG_POWER_RESET_MSM=y -CONFIG_POWER_RESET_QCOM_PON=m -CONFIG_POWER_RESET_TORADEX_EC=m -CONFIG_POWER_RESET_XGENE=y -CONFIG_POWER_RESET_SYSCON=y -CONFIG_POWER_RESET_SYSCON_POWEROFF=y -CONFIG_SYSCON_REBOOT_MODE=y -CONFIG_NVMEM_REBOOT_MODE=m -CONFIG_POWER_SEQUENCING_PCIE_M2=m -CONFIG_BATTERY_QCOM_BATTMGR=m -CONFIG_BATTERY_SBS=m -CONFIG_BATTERY_BQ27XXX=y -CONFIG_BATTERY_MAX17042=m -CONFIG_CHARGER_MT6360=m -CONFIG_CHARGER_BQ25890=m -CONFIG_CHARGER_BQ25980=m -CONFIG_CHARGER_RK817=m -CONFIG_MACSMC_POWER=m -CONFIG_SENSORS_AQUACOMPUTER_D5NEXT=m -CONFIG_SENSORS_ASUS_ROG_RYUJIN=m -CONFIG_SENSORS_ARM_SCMI=y -CONFIG_SENSORS_ARM_SCPI=y -CONFIG_SENSORS_CORSAIR_CPRO=m -CONFIG_SENSORS_CORSAIR_PSU=m -CONFIG_SENSORS_GIGABYTE_WATERFORCE=m -CONFIG_SENSORS_GPIO_FAN=m -CONFIG_SENSORS_JC42=m -CONFIG_SENSORS_POWERZ=m -CONFIG_SENSORS_MACSMC_HWMON=y -CONFIG_SENSORS_LM75=m -CONFIG_SENSORS_LM90=m -CONFIG_SENSORS_NZXT_KRAKEN2=m -CONFIG_SENSORS_NZXT_KRAKEN3=m -CONFIG_SENSORS_NZXT_SMART2=m -CONFIG_SENSORS_PWM_FAN=m -CONFIG_SENSORS_RASPBERRYPI_HWMON=m -CONFIG_SENSORS_SL28CPLD=m -CONFIG_SENSORS_AMC6821=m -CONFIG_SENSORS_INA2XX=m -CONFIG_SENSORS_INA3221=m -CONFIG_SENSORS_TMP102=m -CONFIG_THERMAL_NETLINK=y -CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y -CONFIG_THERMAL_GOV_STEP_WISE=y -CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y -CONFIG_CPU_THERMAL=y -CONFIG_DEVFREQ_THERMAL=y -CONFIG_THERMAL_EMULATION=y -CONFIG_IMX_SC_THERMAL=m -CONFIG_IMX8MM_THERMAL=m -CONFIG_K3_THERMAL=m -CONFIG_QORIQ_THERMAL=m -CONFIG_SUN8I_THERMAL=y -CONFIG_ROCKCHIP_THERMAL=m -CONFIG_ARMADA_THERMAL=y -CONFIG_MTK_THERMAL=m -CONFIG_MTK_LVTS_THERMAL=m -CONFIG_BCM2711_THERMAL=m -CONFIG_BCM2835_THERMAL=m -CONFIG_BRCMSTB_THERMAL=m -CONFIG_EXYNOS_THERMAL=y -CONFIG_RCAR_THERMAL=y -CONFIG_RCAR_GEN3_THERMAL=y -CONFIG_RZG2L_THERMAL=y -CONFIG_RZG3E_THERMAL=y -CONFIG_RZG3S_THERMAL=m -CONFIG_TEGRA_SOCTHERM=m -CONFIG_TEGRA_BPMP_THERMAL=m -CONFIG_GENERIC_ADC_THERMAL=m -CONFIG_QCOM_TSENS=y -CONFIG_QCOM_SPMI_ADC_TM5=m -CONFIG_QCOM_SPMI_TEMP_ALARM=m -CONFIG_QCOM_LMH=m -CONFIG_UNIPHIER_THERMAL=y -CONFIG_KHADAS_MCU_FAN_THERMAL=m -CONFIG_WATCHDOG=y -CONFIG_XILINX_WATCHDOG=m -CONFIG_XILINX_WINDOW_WATCHDOG=m -CONFIG_SL28CPLD_WATCHDOG=m -CONFIG_ARM_SP805_WATCHDOG=y -CONFIG_ARM_SBSA_WATCHDOG=y -CONFIG_S3C2410_WATCHDOG=y -CONFIG_DW_WATCHDOG=y -CONFIG_K3_RTI_WATCHDOG=m -CONFIG_SUNXI_WATCHDOG=m -CONFIG_NPCM7XX_WATCHDOG=y -CONFIG_IMX2_WDT=y -CONFIG_IMX_SC_WDT=m -CONFIG_IMX7ULP_WDT=m -CONFIG_QCOM_WDT=m -CONFIG_MESON_GXBB_WATCHDOG=m -CONFIG_MESON_WATCHDOG=m -CONFIG_ARM_SMC_WATCHDOG=y -CONFIG_RENESAS_WDT=y -CONFIG_RENESAS_RZG2LWDT=y -CONFIG_RENESAS_RZV2HWDT=y -CONFIG_UNIPHIER_WATCHDOG=y -CONFIG_PM8916_WATCHDOG=m -CONFIG_APPLE_WATCHDOG=y -CONFIG_BCM2835_WDT=y -CONFIG_BCM7038_WDT=m -CONFIG_USBPCWATCHDOG=m -CONFIG_MFD_ADP5585=m -CONFIG_MFD_ALTERA_SYSMGR=y -CONFIG_MFD_BD9571MWV=y -CONFIG_MFD_AXP20X_I2C=y -CONFIG_MFD_AXP20X_RSB=y -CONFIG_MFD_MACSMC=y -CONFIG_MFD_DA9062=m -CONFIG_MFD_DLN2=m -CONFIG_MFD_EXYNOS_LPASS=m -CONFIG_MFD_HI6421_PMIC=y -CONFIG_MFD_HI655X_PMIC=y -CONFIG_MFD_MAX77620=y -CONFIG_MFD_MAX77759=m -CONFIG_MFD_MT6360=y -CONFIG_MFD_MT6397=y -CONFIG_MFD_SPMI_PMIC=y -CONFIG_MFD_RK8XX_I2C=y -CONFIG_MFD_RK8XX_SPI=y -CONFIG_MFD_SEC_ACPM=m -CONFIG_MFD_SEC_I2C=y -CONFIG_MFD_SL28CPLD=y -CONFIG_RZ_MTU3=y -CONFIG_MFD_TI_AM335X_TSCADC=m -CONFIG_MFD_TI_LP873X=m -CONFIG_MFD_TPS65219=y -CONFIG_MFD_TPS6594_I2C=m -CONFIG_MFD_ROHM_BD718XX=y -CONFIG_MFD_STM32_LPTIMER=m -CONFIG_MFD_STM32_TIMERS=m -CONFIG_MFD_WCD934X=m -CONFIG_MFD_KHADAS_MCU=m -CONFIG_MFD_QCOM_PM8008=m -CONFIG_REGULATOR_FIXED_VOLTAGE=y -CONFIG_REGULATOR_AXP20X=y -CONFIG_REGULATOR_BD718XX=y -CONFIG_REGULATOR_BD9571MWV=y -CONFIG_REGULATOR_CROS_EC=y -CONFIG_REGULATOR_DA9211=m -CONFIG_REGULATOR_FAN53555=y -CONFIG_REGULATOR_GPIO=y -CONFIG_REGULATOR_HI6421V530=y -CONFIG_REGULATOR_HI655X=y -CONFIG_REGULATOR_LP873X=m -CONFIG_REGULATOR_MAX77620=y -CONFIG_REGULATOR_MAX8973=y -CONFIG_REGULATOR_MAX20411=m -CONFIG_REGULATOR_MP8859=y -CONFIG_REGULATOR_MT6315=m -CONFIG_REGULATOR_MT6357=y -CONFIG_REGULATOR_MT6358=y -CONFIG_REGULATOR_MT6359=y -CONFIG_REGULATOR_MT6360=y -CONFIG_REGULATOR_MT6397=y -CONFIG_REGULATOR_PCA9450=y -CONFIG_REGULATOR_PF8X00=y -CONFIG_REGULATOR_PFUZE100=y -CONFIG_REGULATOR_PWM=y -CONFIG_REGULATOR_QCOM_PM8008=m -CONFIG_REGULATOR_QCOM_REFGEN=m -CONFIG_REGULATOR_QCOM_RPMH=y -CONFIG_REGULATOR_QCOM_SMD_RPM=y -CONFIG_REGULATOR_QCOM_SPMI=y -CONFIG_REGULATOR_QCOM_USB_VBUS=m -CONFIG_REGULATOR_RAA215300=y -CONFIG_REGULATOR_RK808=y -CONFIG_REGULATOR_S2MPS11=y -CONFIG_REGULATOR_TPS65132=m -CONFIG_REGULATOR_TPS65219=y -CONFIG_REGULATOR_VCTRL=m -CONFIG_RC_CORE=m -CONFIG_RC_DECODERS=y -CONFIG_RC_DEVICES=y -CONFIG_IR_GPIO_CIR=m -CONFIG_IR_MESON=m -CONFIG_IR_SUNXI=m -CONFIG_MEDIA_CEC_SUPPORT=y -CONFIG_CEC_MESON_G12A_AO=m -CONFIG_MEDIA_SUPPORT=m -CONFIG_MEDIA_CAMERA_SUPPORT=y -CONFIG_MEDIA_ANALOG_TV_SUPPORT=y -CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y -CONFIG_MEDIA_SDR_SUPPORT=y -CONFIG_MEDIA_PLATFORM_SUPPORT=y -# CONFIG_DVB_NET is not set -CONFIG_MEDIA_USB_SUPPORT=y -CONFIG_USB_GSPCA=m -CONFIG_USB_GSPCA_BENQ=m -CONFIG_USB_GSPCA_CONEX=m -CONFIG_USB_GSPCA_CPIA1=m -CONFIG_USB_GSPCA_DTCS033=m -CONFIG_USB_GSPCA_ETOMS=m -CONFIG_USB_GSPCA_FINEPIX=m -CONFIG_USB_GSPCA_JEILINJ=m -CONFIG_USB_GSPCA_JL2005BCD=m -CONFIG_USB_GSPCA_KINECT=m -CONFIG_USB_GSPCA_KONICA=m -CONFIG_USB_GSPCA_MARS=m -CONFIG_USB_GSPCA_MR97310A=m -CONFIG_USB_GSPCA_NW80X=m -CONFIG_USB_GSPCA_OV519=m -CONFIG_USB_GSPCA_OV534=m -CONFIG_USB_GSPCA_OV534_9=m -CONFIG_USB_GSPCA_PAC207=m -CONFIG_USB_GSPCA_PAC7302=m -CONFIG_USB_GSPCA_PAC7311=m -CONFIG_USB_GSPCA_SE401=m -CONFIG_USB_GSPCA_SN9C2028=m -CONFIG_USB_GSPCA_SN9C20X=m -CONFIG_USB_GSPCA_SONIXB=m -CONFIG_USB_GSPCA_SONIXJ=m -CONFIG_USB_GSPCA_SPCA1528=m -CONFIG_USB_GSPCA_SPCA500=m -CONFIG_USB_GSPCA_SPCA501=m -CONFIG_USB_GSPCA_SPCA505=m -CONFIG_USB_GSPCA_SPCA506=m -CONFIG_USB_GSPCA_SPCA508=m -CONFIG_USB_GSPCA_SPCA561=m -CONFIG_USB_GSPCA_SQ905=m -CONFIG_USB_GSPCA_SQ905C=m -CONFIG_USB_GSPCA_SQ930X=m -CONFIG_USB_GSPCA_STK014=m -CONFIG_USB_GSPCA_STK1135=m -CONFIG_USB_GSPCA_STV0680=m -CONFIG_USB_GSPCA_SUNPLUS=m -CONFIG_USB_GSPCA_T613=m -CONFIG_USB_GSPCA_TOPRO=m -CONFIG_USB_GSPCA_TOUPTEK=m -CONFIG_USB_GSPCA_TV8532=m -CONFIG_USB_GSPCA_VC032X=m -CONFIG_USB_GSPCA_VICAM=m -CONFIG_USB_GSPCA_XIRLINK_CIT=m -CONFIG_USB_GSPCA_ZC3XX=m -CONFIG_USB_GL860=m -CONFIG_USB_M5602=m -CONFIG_USB_STV06XX=m -CONFIG_USB_PWC=m -CONFIG_USB_VIDEO_CLASS=m -CONFIG_V4L_PLATFORM_DRIVERS=y -CONFIG_SDR_PLATFORM_DRIVERS=y -CONFIG_V4L_MEM2MEM_DRIVERS=y -CONFIG_VIDEO_AMPHION_VPU=m -CONFIG_VIDEO_APPLE_ISP=m -CONFIG_VIDEO_APPLE_AVD=m -CONFIG_VIDEO_CADENCE_CSI2RX=m -CONFIG_VIDEO_WAVE_VPU=m -CONFIG_VIDEO_E5010_JPEG_ENC=m -CONFIG_VIDEO_MEDIATEK_JPEG=m -CONFIG_VIDEO_MEDIATEK_VCODEC=m -CONFIG_VIDEO_MEDIATEK_MDP3=m -CONFIG_VIDEO_IMX7_CSI=m -CONFIG_VIDEO_IMX_MIPI_CSIS=m -CONFIG_VIDEO_IMX8_ISI=m -CONFIG_VIDEO_IMX8_ISI_M2M=y -CONFIG_VIDEO_IMX8_JPEG=m -CONFIG_VIDEO_QCOM_CAMSS=m -CONFIG_VIDEO_QCOM_IRIS=m -CONFIG_VIDEO_QCOM_VENUS=m -CONFIG_VIDEO_RCAR_CSI2=m -CONFIG_VIDEO_RCAR_ISP=m -CONFIG_VIDEO_RCAR_VIN=m -CONFIG_VIDEO_RZG2L_CSI2=m -CONFIG_VIDEO_RZG2L_CRU=m -CONFIG_VIDEO_RENESAS_FCP=m -CONFIG_VIDEO_RENESAS_FDP1=m -CONFIG_VIDEO_RENESAS_VSP1=m -CONFIG_VIDEO_RCAR_DRIF=m -CONFIG_VIDEO_ROCKCHIP_RGA=m -CONFIG_VIDEO_ROCKCHIP_CIF=m -CONFIG_VIDEO_SAMSUNG_EXYNOS_GSC=m -CONFIG_VIDEO_SAMSUNG_S5P_JPEG=m -CONFIG_VIDEO_SAMSUNG_S5P_MFC=m -CONFIG_VIDEO_SUN6I_CSI=m -CONFIG_VIDEO_SYNOPSYS_HDMIRX=m -CONFIG_VIDEO_SYNOPSYS_HDMIRX_LOAD_DEFAULT_EDID=y -CONFIG_VIDEO_TI_J721E_CSI2RX=m -CONFIG_VIDEO_HANTRO=m -CONFIG_VIDEO_IMX219=m -CONFIG_VIDEO_IMX412=m -CONFIG_VIDEO_OV5640=m -CONFIG_VIDEO_OV5645=m -CONFIG_VIDEO_S5KJN1=m -CONFIG_DRM=y -CONFIG_DRM_ADP=m -CONFIG_DRM_APPLE=m -CONFIG_DRM_HDLCD=m -CONFIG_DRM_MALI_DISPLAY=m -CONFIG_DRM_KOMEDA=m -CONFIG_DRM_ASAHI=m -CONFIG_DRM_FSL_LDB=m -CONFIG_DRM_I2C_NXP_TDA998X=m -CONFIG_DRM_ITE_IT6263=m -CONFIG_DRM_LONTIUM_LT8912B=m -CONFIG_DRM_LONTIUM_LT9611=m -CONFIG_DRM_LONTIUM_LT9611UXC=m -CONFIG_DRM_LONTIUM_LT8713SX=m -CONFIG_DRM_ITE_IT66121=m -CONFIG_DRM_NWL_MIPI_DSI=m -CONFIG_DRM_PARADE_PS8640=m -CONFIG_DRM_SII902X=m -CONFIG_DRM_SIMPLE_BRIDGE=m -CONFIG_DRM_THINE_THC63LVD1024=m -CONFIG_DRM_TOSHIBA_TC358767=m -CONFIG_DRM_TOSHIBA_TC358768=m -CONFIG_DRM_TI_TFP410=m -CONFIG_DRM_TI_SN65DSI83=m -CONFIG_DRM_TI_SN65DSI86=m -CONFIG_DRM_ANALOGIX_ANX7625=m -CONFIG_DRM_I2C_ADV7511=m -CONFIG_DRM_I2C_ADV7511_AUDIO=y -CONFIG_DRM_CDNS_DSI=m -CONFIG_DRM_CDNS_MHDP8546=m -CONFIG_DRM_IMX8MP_DW_HDMI_BRIDGE=m -CONFIG_DRM_DW_HDMI_AHB_AUDIO=m -CONFIG_DRM_DW_HDMI_CEC=m -CONFIG_DRM_DW_HDMI_QP_CEC=y -CONFIG_DRM_ETNAVIV=m -CONFIG_DRM_EXYNOS=m -CONFIG_DRM_EXYNOS5433_DECON=y -CONFIG_DRM_EXYNOS7_DECON=y -CONFIG_DRM_EXYNOS_DSI=y -# CONFIG_DRM_EXYNOS_DP is not set -CONFIG_DRM_EXYNOS_HDMI=y -CONFIG_DRM_EXYNOS_MIC=y -CONFIG_DRM_GUD=m -CONFIG_DRM_HISI_HIBMC=m -CONFIG_DRM_HISI_KIRIN=m -CONFIG_DRM_POWERVR=m -CONFIG_DRM_IMX_DCSS=m -CONFIG_DRM_LIMA=m -CONFIG_DRM_MEDIATEK=m -CONFIG_DRM_MEDIATEK_DP=m -CONFIG_DRM_MEDIATEK_HDMI=m -CONFIG_DRM_MEDIATEK_HDMI_V2=m -CONFIG_DRM_MESON=m -CONFIG_DRM_MSM=m -CONFIG_DRM_MXSFB=m -CONFIG_DRM_IMX_LCDIF=m -CONFIG_DRM_NOUVEAU=m -CONFIG_DRM_PANEL_BOE_TV101WUM_NL6=m -CONFIG_DRM_PANEL_LVDS=m -CONFIG_DRM_PANEL_HIMAX_HX8279=m -CONFIG_DRM_PANEL_HIMAX_HX83112A=m -CONFIG_DRM_PANEL_HIMAX_HX83112B=m -CONFIG_DRM_PANEL_ILITEK_ILI9882T=m -CONFIG_DRM_PANEL_KHADAS_TS050=m -CONFIG_DRM_PANEL_MANTIX_MLAF057WE51=m -CONFIG_DRM_PANEL_NOVATEK_NT36672A=m -CONFIG_DRM_PANEL_NOVATEK_NT36672E=m -CONFIG_DRM_PANEL_NOVATEK_NT37801=m -CONFIG_DRM_PANEL_RAYDIUM_RM67191=m -CONFIG_DRM_PANEL_RAYDIUM_RM692E5=m -CONFIG_DRM_PANEL_SAMSUNG_ATNA33XC20=m -CONFIG_DRM_PANEL_SITRONIX_ST7703=m -CONFIG_DRM_PANEL_STARTEK_KD070FHFID015=m -CONFIG_DRM_PANEL_EDP=m -CONFIG_DRM_PANEL_SIMPLE=m -CONFIG_DRM_PANEL_SUMMIT=m -CONFIG_DRM_PANEL_TRULY_NT35597_WQXGA=m -CONFIG_DRM_PANEL_VISIONOX_VTDR6130=m -CONFIG_DRM_PANFROST=m -CONFIG_DRM_PANTHOR=m -CONFIG_DRM_PL111=m -CONFIG_DRM_RCAR_DU=m -CONFIG_DRM_RCAR_DW_HDMI=m -CONFIG_DRM_RZG2L_DU=m -CONFIG_DRM_ROCKCHIP=m -CONFIG_ROCKCHIP_VOP2=y -CONFIG_ROCKCHIP_ANALOGIX_DP=y -CONFIG_ROCKCHIP_CDN_DP=y -CONFIG_ROCKCHIP_DW_DP=y -CONFIG_ROCKCHIP_DW_HDMI=y -CONFIG_ROCKCHIP_DW_HDMI_QP=y -CONFIG_ROCKCHIP_DW_MIPI_DSI=y -CONFIG_ROCKCHIP_INNO_HDMI=y -CONFIG_ROCKCHIP_LVDS=y -CONFIG_DRM_STM=m -CONFIG_DRM_STM_LVDS=m -CONFIG_DRM_SUN4I=m -CONFIG_DRM_SIMPLEDRM=y -CONFIG_DRM_TEGRA=m -CONFIG_DRM_TIDSS=m -CONFIG_DRM_GM12U320=m -CONFIG_DRM_UDL=m -CONFIG_DRM_V3D=m -CONFIG_DRM_VC4=m -CONFIG_DRM_ZYNQMP_DPSUB=m -CONFIG_DRM_ZYNQMP_DPSUB_AUDIO=y -CONFIG_FB=y -CONFIG_FB_EFI=y -CONFIG_BACKLIGHT_CLASS_DEVICE=y -CONFIG_BACKLIGHT_PWM=m -CONFIG_BACKLIGHT_APPLE_DWI=m -CONFIG_BACKLIGHT_QCOM_WLED=m -CONFIG_BACKLIGHT_LP855X=m -CONFIG_BACKLIGHT_GPIO=m -CONFIG_LOGO=y -CONFIG_LOGO_LINUX_CLUT224_FILE="drivers/video/logo/logo_linux_clut224.ppm" -CONFIG_DRM_ACCEL=y -CONFIG_DRM_ACCEL_ANE=m -CONFIG_SOUND=m -CONFIG_SND=m -CONFIG_SND_UMP_LEGACY_RAWMIDI=y -CONFIG_SND_HRTIMER=m -CONFIG_SND_UTIMER=y -CONFIG_SND_SEQUENCER=m -CONFIG_SND_SEQ_DUMMY=m -CONFIG_SND_DUMMY=m -CONFIG_SND_ALOOP=m -CONFIG_SND_VIRMIDI=m -CONFIG_SND_HDA_TEGRA=m -CONFIG_SND_HDA_CODEC_HDMI=m -CONFIG_SND_USB_AUDIO=m -CONFIG_SND_USB_AUDIO_MIDI_V2=y -CONFIG_SND_USB_UA101=m -CONFIG_SND_USB_CAIAQ=m -CONFIG_SND_USB_CAIAQ_INPUT=y -CONFIG_SND_USB_6FIRE=m -CONFIG_SND_USB_HIFACE=m -CONFIG_SND_BCD2000=m -CONFIG_SND_USB_AUDIO_QMI=m -CONFIG_SND_USB_POD=m -CONFIG_SND_USB_PODHD=m -CONFIG_SND_USB_TONEPORT=m -CONFIG_SND_USB_VARIAX=m -CONFIG_SND_SOC=m -CONFIG_SND_SOC_USB=m -CONFIG_SND_SOC_APPLE_AOP_AUDIO=m -CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m -CONFIG_SND_SOC_APPLE_MACAUDIO=m -CONFIG_SND_BCM2835_SOC_I2S=m -CONFIG_SND_SOC_FSL_ASRC=m -CONFIG_SND_SOC_FSL_MICFIL=m -CONFIG_SND_SOC_FSL_EASRC=m -CONFIG_SND_IMX_SOC=m -CONFIG_SND_SOC_IMX_SGTL5000=m -CONFIG_SND_SOC_FSL_ASOC_CARD=m -CONFIG_SND_SOC_IMX_AUDMIX=m -CONFIG_SND_SOC_MT8183=m -CONFIG_SND_SOC_MT8183_MT6358_TS3A227E_MAX98357A=m -CONFIG_SND_SOC_MT8183_DA7219_MAX98357A=m -CONFIG_SND_SOC_MT8188=m -CONFIG_SND_SOC_MT8188_MT6359=m -CONFIG_SND_SOC_MT8192=m -CONFIG_SND_SOC_MT8192_MT6359_RT1015_RT5682=m -CONFIG_SND_SOC_MT8195=m -CONFIG_SND_SOC_MT8195_MT6359=m -CONFIG_SND_SOC_MT8365=m -CONFIG_SND_SOC_MT8365_MT6357=m -CONFIG_SND_MESON_AXG_SOUND_CARD=m -CONFIG_SND_MESON_GX_SOUND_CARD=m -CONFIG_SND_SOC_QCOM=m -CONFIG_SND_SOC_APQ8016_SBC=m -CONFIG_SND_SOC_QDSP6_USB=m -CONFIG_SND_SOC_MSM8996=m -CONFIG_SND_SOC_SDM845=m -CONFIG_SND_SOC_SM8250=m -CONFIG_SND_SOC_SC8280XP=m -CONFIG_SND_SOC_SC7180=m -CONFIG_SND_SOC_SC7280=m -CONFIG_SND_SOC_X1E80100=m -CONFIG_SND_SOC_RCAR=m -CONFIG_SND_SOC_MSIOF=m -CONFIG_SND_SOC_RZ=m -CONFIG_SND_SOC_ROCKCHIP_I2S_TDM=m -CONFIG_SND_SOC_ROCKCHIP_SAI=m -CONFIG_SND_SOC_ROCKCHIP_SPDIF=m -CONFIG_SND_SOC_ROCKCHIP_RT5645=m -CONFIG_SND_SOC_RK3399_GRU_SOUND=m -CONFIG_SND_SOC_SAMSUNG=m -CONFIG_SND_SUN8I_CODEC=m -CONFIG_SND_SUN8I_CODEC_ANALOG=m -CONFIG_SND_SUN50I_CODEC_ANALOG=m -CONFIG_SND_SUN4I_I2S=m -CONFIG_SND_SUN4I_SPDIF=m -CONFIG_SND_SOC_TEGRA=m -CONFIG_SND_SOC_TEGRA210_AHUB=m -CONFIG_SND_SOC_TEGRA210_DMIC=m -CONFIG_SND_SOC_TEGRA210_I2S=m -CONFIG_SND_SOC_TEGRA210_OPE=m -CONFIG_SND_SOC_TEGRA186_ASRC=m -CONFIG_SND_SOC_TEGRA186_DSPK=m -CONFIG_SND_SOC_TEGRA210_ADMAIF=m -CONFIG_SND_SOC_TEGRA210_MVC=m -CONFIG_SND_SOC_TEGRA210_SFC=m -CONFIG_SND_SOC_TEGRA210_AMX=m -CONFIG_SND_SOC_TEGRA210_ADX=m -CONFIG_SND_SOC_TEGRA210_MIXER=m -CONFIG_SND_SOC_TEGRA_AUDIO_GRAPH_CARD=m -CONFIG_SND_SOC_J721E_EVM=m -CONFIG_SND_SOC_XILINX_I2S=m -CONFIG_SND_SOC_XILINX_AUDIO_FORMATTER=m -CONFIG_SND_SOC_XILINX_SPDIF=m -CONFIG_SND_SOC_SOF_TOPLEVEL=y -CONFIG_SND_SOC_SOF_OF=m -CONFIG_SND_SOC_SOF_MTK_TOPLEVEL=y -CONFIG_SND_SOC_SOF_MT8186=m -CONFIG_SND_SOC_SOF_MT8195=m -CONFIG_SND_SOC_AK4613=m -CONFIG_SND_SOC_AK4619=m -CONFIG_SND_SOC_DA7213=m -CONFIG_SND_SOC_ES7134=m -CONFIG_SND_SOC_ES7241=m -CONFIG_SND_SOC_ES8316=m -CONFIG_SND_SOC_ES8328_I2C=m -CONFIG_SND_SOC_GTM601=m -CONFIG_SND_SOC_MAX98090=m -CONFIG_SND_SOC_MSM8916_WCD_ANALOG=m -CONFIG_SND_SOC_MSM8916_WCD_DIGITAL=m -CONFIG_SND_SOC_RK3308=m -CONFIG_SND_SOC_RK817=m -CONFIG_SND_SOC_RT5640=m -CONFIG_SND_SOC_RT5659=m -CONFIG_SND_SOC_SIMPLE_AMPLIFIER=m -CONFIG_SND_SOC_SIMPLE_MUX=m -CONFIG_SND_SOC_TAS2552=m -CONFIG_SND_SOC_TAS571X=m -CONFIG_SND_SOC_TLV320AIC32X4_I2C=m -CONFIG_SND_SOC_TLV320AIC3X_I2C=m -CONFIG_SND_SOC_WCD9335=m -CONFIG_SND_SOC_WCD934X=m -CONFIG_SND_SOC_WCD937X_SDW=m -CONFIG_SND_SOC_WCD939X_SDW=m -CONFIG_SND_SOC_WM8524=m -CONFIG_SND_SOC_WM8904=m -CONFIG_SND_SOC_WM8960=m -CONFIG_SND_SOC_WM8962=m -CONFIG_SND_SOC_WM8978=m -CONFIG_SND_SOC_WSA881X=m -CONFIG_SND_SOC_WSA883X=m -CONFIG_SND_SOC_WSA884X=m -CONFIG_SND_SOC_NAU8822=m -CONFIG_SND_SOC_LPASS_WSA_MACRO=m -CONFIG_SND_SOC_LPASS_VA_MACRO=m -CONFIG_SND_SIMPLE_CARD=m -CONFIG_SND_AUDIO_GRAPH_CARD=m -CONFIG_SND_AUDIO_GRAPH_CARD2=m -CONFIG_HID_BATTERY_STRENGTH=y -CONFIG_HIDRAW=y -CONFIG_UHID=y -CONFIG_HID_A4TECH=m -CONFIG_HID_ACCUTOUCH=m -CONFIG_HID_ACRUX=m -CONFIG_HID_ACRUX_FF=y -CONFIG_HID_APPLE=y -CONFIG_HID_APPLEIR=m -CONFIG_HID_AUREAL=m -CONFIG_HID_BELKIN=m -CONFIG_HID_BETOP_FF=m -CONFIG_HID_BIGBEN_FF=m -CONFIG_HID_CHERRY=m -CONFIG_HID_CHICONY=m -CONFIG_HID_CORSAIR=m -CONFIG_HID_COUGAR=m -CONFIG_HID_MACALLY=m -CONFIG_HID_PRODIKEYS=m -CONFIG_HID_CMEDIA=m -CONFIG_HID_CP2112=m -CONFIG_HID_CREATIVE_SB0540=m -CONFIG_HID_CYPRESS=m -CONFIG_HID_DRAGONRISE=m -CONFIG_DRAGONRISE_FF=y -CONFIG_HID_EMS_FF=m -CONFIG_HID_ELAN=m -CONFIG_HID_ELECOM=m -CONFIG_HID_ELO=m -CONFIG_HID_EVISION=m -CONFIG_HID_EZKEY=m -CONFIG_HID_FT260=m -CONFIG_HID_GEMBIRD=m -CONFIG_HID_GFRM=m -CONFIG_HID_GLORIOUS=m -CONFIG_HID_HOLTEK=m -CONFIG_HOLTEK_FF=y -CONFIG_HID_GOOGLE_STADIA_FF=m -CONFIG_HID_VIVALDI=m -CONFIG_HID_GT683R=m -CONFIG_HID_KEYTOUCH=m -CONFIG_HID_KYE=m -CONFIG_HID_KYSONA=m -CONFIG_HID_UCLOGIC=m -CONFIG_HID_WALTOP=m -CONFIG_HID_VIEWSONIC=m -CONFIG_HID_XIAOMI=m -CONFIG_HID_GYRATION=m -CONFIG_HID_ICADE=m -CONFIG_HID_ITE=m -CONFIG_HID_JABRA=m -CONFIG_HID_TWINHAN=m -CONFIG_HID_KENSINGTON=m -CONFIG_HID_LCPOWER=m -CONFIG_HID_LENOVO=m -CONFIG_HID_LENOVO_GO=m -CONFIG_HID_LENOVO_GO_S=m -CONFIG_HID_LETSKETCH=m -CONFIG_HID_LOGITECH=m -CONFIG_HID_LOGITECH_DJ=m -CONFIG_LOGITECH_FF=y -CONFIG_LOGIRUMBLEPAD2_FF=y -CONFIG_LOGIG940_FF=y -CONFIG_HID_MAGICMOUSE=y -CONFIG_HID_MALTRON=m -CONFIG_HID_MAYFLASH=m -CONFIG_HID_MEGAWORLD_FF=m -CONFIG_HID_MICROSOFT=m -CONFIG_HID_MONTEREY=m -CONFIG_HID_MULTITOUCH=m -CONFIG_HID_NINTENDO=m -CONFIG_NINTENDO_FF=y -CONFIG_HID_NTI=m -CONFIG_HID_NTRIG=m -CONFIG_HID_NVIDIA_SHIELD=m -CONFIG_NVIDIA_SHIELD_FF=y -CONFIG_HID_ORTEK=m -CONFIG_HID_PANTHERLORD=m -CONFIG_PANTHERLORD_FF=y -CONFIG_HID_PENMOUNT=m -CONFIG_HID_PETALYNX=m -CONFIG_HID_PICOLCD=m -CONFIG_HID_PLANTRONICS=m -CONFIG_HID_PLAYSTATION=m -CONFIG_PLAYSTATION_FF=y -CONFIG_HID_PXRC=m -CONFIG_HID_RAPOO=m -CONFIG_HID_RAZER=m -CONFIG_HID_PRIMAX=m -CONFIG_HID_RETRODE=m -CONFIG_HID_ROCCAT=m -CONFIG_HID_SAITEK=m -CONFIG_HID_SAMSUNG=m -CONFIG_HID_SEMITEK=m -CONFIG_HID_SIGMAMICRO=m -CONFIG_HID_SONY=m -CONFIG_SONY_FF=y -CONFIG_HID_SPEEDLINK=m -CONFIG_HID_STEAM=m -CONFIG_STEAM_FF=y -CONFIG_HID_STEELSERIES=m -CONFIG_HID_SUNPLUS=m -CONFIG_HID_RMI=m -CONFIG_HID_GREENASIA=m -CONFIG_GREENASIA_FF=y -CONFIG_HID_SMARTJOYPLUS=m -CONFIG_SMARTJOYPLUS_FF=y -CONFIG_HID_TIVO=m -CONFIG_HID_TOPSEED=m -CONFIG_HID_TOPRE=m -CONFIG_HID_THINGM=m -CONFIG_HID_THRUSTMASTER=m -CONFIG_THRUSTMASTER_FF=y -CONFIG_HID_UDRAW_PS3=m -CONFIG_HID_U2FZERO=m -CONFIG_HID_UNIVERSAL_PIDFF=m -CONFIG_HID_WACOM=m -CONFIG_HID_WIIMOTE=m -CONFIG_HID_WINWING=m -CONFIG_HID_XINMO=m -CONFIG_HID_ZEROPLUS=m -CONFIG_ZEROPLUS_FF=y -CONFIG_HID_ZYDACRON=m -CONFIG_HID_ALPS=m -CONFIG_HID_MCP2221=m -CONFIG_HID_HUAWEI=m -CONFIG_I2C_HID_ACPI=m -CONFIG_I2C_HID_OF=m -CONFIG_I2C_HID_OF_ELAN=m -CONFIG_HID_PID=y -CONFIG_USB_HIDDEV=y -CONFIG_SPI_HID_APPLE_OF=m -CONFIG_HID_DOCKCHANNEL=y -CONFIG_USB=y -CONFIG_USB_OTG=y -CONFIG_USB_MON=m -CONFIG_USB_XHCI_HCD=y -CONFIG_USB_XHCI_PCI_RENESAS=m -CONFIG_USB_XHCI_RZV2M=y -CONFIG_USB_XHCI_SIDEBAND=y -CONFIG_USB_XHCI_TEGRA=y -CONFIG_USB_BRCMSTB=m -CONFIG_USB_EHCI_HCD=y -CONFIG_USB_EHCI_EXYNOS=y -CONFIG_USB_EHCI_HCD_PLATFORM=y -CONFIG_USB_OHCI_HCD=y -CONFIG_USB_OHCI_EXYNOS=y -CONFIG_USB_OHCI_HCD_PLATFORM=y -CONFIG_USB_RENESAS_USBHS_HCD=m -CONFIG_USB_RENESAS_USBHS=m -CONFIG_USB_PRINTER=m -CONFIG_USB_TMC=m -CONFIG_USB_STORAGE=y -CONFIG_USB_STORAGE_REALTEK=m -CONFIG_USB_STORAGE_DATAFAB=m -CONFIG_USB_STORAGE_FREECOM=m -CONFIG_USB_STORAGE_ISD200=m -CONFIG_USB_STORAGE_USBAT=m -CONFIG_USB_STORAGE_SDDR09=m -CONFIG_USB_STORAGE_SDDR55=m -CONFIG_USB_STORAGE_JUMPSHOT=m -CONFIG_USB_STORAGE_ALAUDA=m -CONFIG_USB_STORAGE_ONETOUCH=m -CONFIG_USB_STORAGE_KARMA=m -CONFIG_USB_STORAGE_CYPRESS_ATACB=m -CONFIG_USB_STORAGE_ENE_UB6250=m -CONFIG_USB_UAS=m -CONFIG_USB_MDC800=m -CONFIG_USB_MICROTEK=m -CONFIG_USBIP_CORE=m -CONFIG_USBIP_VHCI_HCD=m -CONFIG_USBIP_HOST=m -CONFIG_USB_CDNS_SUPPORT=m -CONFIG_USB_CDNS3=m -CONFIG_USB_CDNS3_GADGET=y -CONFIG_USB_CDNS3_HOST=y -CONFIG_USB_MTU3=y -CONFIG_USB_MUSB_HDRC=y -CONFIG_USB_MUSB_SUNXI=y -CONFIG_USB_DWC3=y -CONFIG_USB_DWC2=y -CONFIG_USB_CHIPIDEA=y -CONFIG_USB_CHIPIDEA_UDC=y -CONFIG_USB_CHIPIDEA_HOST=y -CONFIG_USB_ISP1760=y -CONFIG_USB_SERIAL=m -CONFIG_USB_SERIAL_GENERIC=y -CONFIG_USB_SERIAL_SIMPLE=m -CONFIG_USB_SERIAL_AIRCABLE=m -CONFIG_USB_SERIAL_ARK3116=m -CONFIG_USB_SERIAL_BELKIN=m -CONFIG_USB_SERIAL_CH341=m -CONFIG_USB_SERIAL_WHITEHEAT=m -CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m -CONFIG_USB_SERIAL_CP210X=m -CONFIG_USB_SERIAL_CYPRESS_M8=m -CONFIG_USB_SERIAL_EMPEG=m -CONFIG_USB_SERIAL_FTDI_SIO=m -CONFIG_USB_SERIAL_VISOR=m -CONFIG_USB_SERIAL_IPAQ=m -CONFIG_USB_SERIAL_IR=m -CONFIG_USB_SERIAL_EDGEPORT=m -CONFIG_USB_SERIAL_EDGEPORT_TI=m -CONFIG_USB_SERIAL_F81232=m -CONFIG_USB_SERIAL_F8153X=m -CONFIG_USB_SERIAL_GARMIN=m -CONFIG_USB_SERIAL_IPW=m -CONFIG_USB_SERIAL_IUU=m -CONFIG_USB_SERIAL_KEYSPAN_PDA=m -CONFIG_USB_SERIAL_KEYSPAN=m -CONFIG_USB_SERIAL_KLSI=m -CONFIG_USB_SERIAL_KOBIL_SCT=m -CONFIG_USB_SERIAL_MCT_U232=m -CONFIG_USB_SERIAL_MOS7720=m -CONFIG_USB_SERIAL_MOS7840=m -CONFIG_USB_SERIAL_NAVMAN=m -CONFIG_USB_SERIAL_PL2303=m -CONFIG_USB_SERIAL_OTI6858=m -CONFIG_USB_SERIAL_QCAUX=m -CONFIG_USB_SERIAL_QUALCOMM=m -CONFIG_USB_SERIAL_SPCP8X5=m -CONFIG_USB_SERIAL_SAFE=m -CONFIG_USB_SERIAL_SAFE_PADDED=y -CONFIG_USB_SERIAL_SIERRAWIRELESS=m -CONFIG_USB_SERIAL_SYMBOL=m -CONFIG_USB_SERIAL_TI=m -CONFIG_USB_SERIAL_CYBERJACK=m -CONFIG_USB_SERIAL_OPTION=m -CONFIG_USB_SERIAL_OMNINET=m -CONFIG_USB_SERIAL_OPTICON=m -CONFIG_USB_SERIAL_XSENS_MT=m -CONFIG_USB_SERIAL_SSU100=m -CONFIG_USB_SERIAL_QT2=m -CONFIG_USB_SERIAL_UPD78F0730=m -CONFIG_USB_SERIAL_XR=m -CONFIG_USB_EMI62=m -CONFIG_USB_EMI26=m -CONFIG_USB_ADUTUX=m -CONFIG_USB_SEVSEG=m -CONFIG_USB_LEGOTOWER=m -CONFIG_USB_LCD=m -CONFIG_USB_IDMOUSE=m -CONFIG_USB_APPLEDISPLAY=m -CONFIG_USB_QCOM_EUD=m -CONFIG_APPLE_MFI_FASTCHARGE=m -CONFIG_USB_SISUSBVGA=m -CONFIG_USB_LD=m -CONFIG_USB_TRANCEVIBRATOR=m -CONFIG_USB_IOWARRIOR=m -CONFIG_USB_ISIGHTFW=m -CONFIG_USB_YUREX=m -CONFIG_USB_HSIC_USB3503=y -CONFIG_USB_CHAOSKEY=m -CONFIG_USB_ONBOARD_DEV=m -CONFIG_USB_ONBOARD_DEV_USB5744=y -CONFIG_NOP_USB_XCEIV=y -CONFIG_USB_MXS_PHY=m -CONFIG_USB_GADGET=y -CONFIG_USB_RENESAS_USBHS_UDC=m -CONFIG_USB_RZV2M_USB3DRD=y -CONFIG_USB_RENESAS_USB3=m -CONFIG_USB_TEGRA_XUDC=m -CONFIG_USB_CONFIGFS=m -CONFIG_USB_CONFIGFS_SERIAL=y -CONFIG_USB_CONFIGFS_ACM=y -CONFIG_USB_CONFIGFS_OBEX=y -CONFIG_USB_CONFIGFS_NCM=y -CONFIG_USB_CONFIGFS_ECM=y -CONFIG_USB_CONFIGFS_ECM_SUBSET=y -CONFIG_USB_CONFIGFS_RNDIS=y -CONFIG_USB_CONFIGFS_EEM=y -CONFIG_USB_CONFIGFS_MASS_STORAGE=y -CONFIG_USB_CONFIGFS_F_FS=y -CONFIG_USB_MASS_STORAGE=m -CONFIG_TYPEC=y -CONFIG_TYPEC_TCPM=m -CONFIG_TYPEC_TCPCI=m -CONFIG_TYPEC_RT1711H=m -CONFIG_TYPEC_MT6360=m -CONFIG_TYPEC_TCPCI_MAXIM=m -CONFIG_TYPEC_FUSB302=m -CONFIG_TYPEC_QCOM_PMIC=m -CONFIG_TYPEC_UCSI=m -CONFIG_UCSI_CCG=m -CONFIG_UCSI_PMIC_GLINK=m -CONFIG_TYPEC_TPS6598X=m -CONFIG_TYPEC_SN201202X=y -CONFIG_TYPEC_HD3SS3220=m -CONFIG_TYPEC_MUX_FSA4480=m -CONFIG_TYPEC_MUX_GPIO_SBU=m -CONFIG_TYPEC_MUX_IT5205=m -CONFIG_TYPEC_MUX_NB7VPQ904M=m -CONFIG_TYPEC_MUX_PS883X=m -CONFIG_TYPEC_MUX_PTN36502=m -CONFIG_TYPEC_MUX_WCD939X_USBSS=m -CONFIG_TYPEC_DP_ALTMODE=m -CONFIG_MMC=y -CONFIG_MMC_BLOCK_MINORS=32 -CONFIG_MMC_ARMMMCI=y -CONFIG_MMC_SDHCI=y -CONFIG_MMC_SDHCI_ACPI=y -CONFIG_MMC_SDHCI_PLTFM=y -CONFIG_MMC_SDHCI_OF_ARASAN=y -CONFIG_MMC_SDHCI_OF_ESDHC=y -CONFIG_MMC_SDHCI_OF_DWCMSHC=y -CONFIG_MMC_SDHCI_OF_SPARX5=y -CONFIG_MMC_SDHCI_CADENCE=y -CONFIG_MMC_SDHCI_ESDHC_IMX=y -CONFIG_MMC_SDHCI_TEGRA=y -CONFIG_MMC_SDHCI_F_SDH30=y -CONFIG_MMC_MESON_GX=y -CONFIG_MMC_SDHCI_MSM=y -CONFIG_MMC_SPI=y -CONFIG_MMC_SDHI=y -CONFIG_MMC_UNIPHIER=y -CONFIG_MMC_DW=y -CONFIG_MMC_DW_EXYNOS=y -CONFIG_MMC_DW_HI3798CV200=y -CONFIG_MMC_DW_K3=y -CONFIG_MMC_DW_ROCKCHIP=y -CONFIG_MMC_VUB300=m -CONFIG_MMC_USHC=m -CONFIG_MMC_REALTEK_USB=m -CONFIG_MMC_SUNXI=y -CONFIG_MMC_BCM2835=y -CONFIG_MMC_MTK=y -CONFIG_MMC_SDHCI_XENON=y -CONFIG_MMC_SDHCI_AM654=y -CONFIG_MMC_OWL=y -CONFIG_SCSI_UFSHCD=y -CONFIG_SCSI_UFS_BSG=y -CONFIG_SCSI_UFS_CRYPTO=y -CONFIG_SCSI_UFSHCD_PLATFORM=y -CONFIG_SCSI_UFS_CDNS_PLATFORM=m -CONFIG_SCSI_UFS_QCOM=m -CONFIG_SCSI_UFS_MEDIATEK=y -CONFIG_SCSI_UFS_HISI=y -CONFIG_SCSI_UFS_RENESAS=m -CONFIG_SCSI_UFS_TI_J721E=m -CONFIG_SCSI_UFS_EXYNOS=y -CONFIG_SCSI_UFS_ROCKCHIP=y -CONFIG_MEMSTICK=m -CONFIG_MSPRO_BLOCK=m -CONFIG_MEMSTICK_REALTEK_USB=m -CONFIG_LEDS_CLASS=y -CONFIG_LEDS_CLASS_FLASH=m -CONFIG_LEDS_CLASS_MULTICOLOR=m -CONFIG_LEDS_LM3692X=m -CONFIG_LEDS_PCA9532=m -CONFIG_LEDS_GPIO=y -CONFIG_LEDS_PWM=y -CONFIG_LEDS_SYSCON=y -CONFIG_LEDS_QCOM_FLASH=m -CONFIG_LEDS_QCOM_LPG=m -CONFIG_LEDS_TRIGGER_TIMER=y -CONFIG_LEDS_TRIGGER_DISK=y -CONFIG_LEDS_TRIGGER_HEARTBEAT=y -CONFIG_LEDS_TRIGGER_CPU=y -CONFIG_LEDS_TRIGGER_DEFAULT_ON=y -CONFIG_LEDS_TRIGGER_PANIC=y -CONFIG_ACCESSIBILITY=y -CONFIG_SPEAKUP=m -CONFIG_SPEAKUP_SYNTH_ACNTSA=m -CONFIG_SPEAKUP_SYNTH_APOLLO=m -CONFIG_SPEAKUP_SYNTH_AUDPTR=m -CONFIG_SPEAKUP_SYNTH_BNS=m -CONFIG_SPEAKUP_SYNTH_DECTLK=m -CONFIG_SPEAKUP_SYNTH_LTLK=m -CONFIG_SPEAKUP_SYNTH_SOFT=m -CONFIG_SPEAKUP_SYNTH_SPKOUT=m -CONFIG_SPEAKUP_SYNTH_TXPRT=m -CONFIG_INFINIBAND=m -CONFIG_INFINIBAND_USER_MAD=m -CONFIG_INFINIBAND_USER_ACCESS=m -CONFIG_RDMA_RXE=m -CONFIG_RDMA_SIW=m -CONFIG_EDAC=y -CONFIG_EDAC_GHES=y -CONFIG_EDAC_LAYERSCAPE=m -CONFIG_EDAC_ZYNQMP=m -CONFIG_EDAC_VERSAL=m -CONFIG_RTC_CLASS=y -CONFIG_RTC_DRV_DS1307=m -CONFIG_RTC_DRV_HYM8563=m -CONFIG_RTC_DRV_MAX77686=y -CONFIG_RTC_DRV_NVIDIA_VRS10=m -CONFIG_RTC_DRV_RK808=m -CONFIG_RTC_DRV_ISL1208=m -CONFIG_RTC_DRV_PCF85363=m -CONFIG_RTC_DRV_PCF8563=m -CONFIG_RTC_DRV_M41T80=m -CONFIG_RTC_DRV_BQ32K=m -CONFIG_RTC_DRV_RX8581=m -CONFIG_RTC_DRV_RV3028=m -CONFIG_RTC_DRV_RV8803=m -CONFIG_RTC_DRV_S5M=y -CONFIG_RTC_DRV_DS3232=y -CONFIG_RTC_DRV_PCF2127=m -CONFIG_RTC_DRV_PCF85063=m -CONFIG_RTC_DRV_DA9063=m -CONFIG_RTC_DRV_EFI=y -CONFIG_RTC_DRV_ZYNQMP=m -CONFIG_RTC_DRV_CROS_EC=y -CONFIG_RTC_DRV_FSL_FTM_ALARM=m -CONFIG_RTC_DRV_S3C=y -CONFIG_RTC_DRV_PL031=y -CONFIG_RTC_DRV_SUN6I=y -CONFIG_RTC_DRV_ARMADA38X=y -CONFIG_RTC_DRV_PM8XXX=m -CONFIG_RTC_DRV_TEGRA=y -CONFIG_RTC_DRV_SNVS=m -CONFIG_RTC_DRV_BBNSM=m -CONFIG_RTC_DRV_IMX_SC=m -CONFIG_RTC_DRV_MT6397=m -CONFIG_RTC_DRV_XGENE=y -CONFIG_RTC_DRV_TI_K3=m -CONFIG_RTC_DRV_RENESAS_RTCA3=m -CONFIG_RTC_DRV_APPLE_PMU=m -CONFIG_RTC_DRV_MACSMC=m -CONFIG_RTC_DRV_S32G=m -CONFIG_DMADEVICES=y -CONFIG_APPLE_SIO=m -CONFIG_DMA_BCM2835=y -CONFIG_DMA_SUN6I=m -CONFIG_FSL_EDMA=y -CONFIG_IMX_SDMA=m -CONFIG_K3_DMA=y -CONFIG_MV_XOR=y -CONFIG_MV_XOR_V2=y -CONFIG_OWL_DMA=y -CONFIG_PL330_DMA=y -CONFIG_TEGRA186_GPC_DMA=y -CONFIG_TEGRA20_APB_DMA=y -CONFIG_TEGRA210_ADMA=m -CONFIG_XILINX_DMA=m -CONFIG_XILINX_ZYNQMP_DMA=m -CONFIG_XILINX_ZYNQMP_DPDMA=m -CONFIG_MTK_UART_APDMA=m -CONFIG_QCOM_BAM_DMA=y -CONFIG_QCOM_GPI_DMA=m -CONFIG_QCOM_HIDMA_MGMT=y -CONFIG_QCOM_HIDMA=y -CONFIG_DW_EDMA=m -CONFIG_RCAR_DMAC=y -CONFIG_RENESAS_USB_DMAC=m -CONFIG_RZ_DMAC=y -CONFIG_TI_K3_UDMA=y -CONFIG_TI_K3_UDMA_GLUE_LAYER=y -CONFIG_STM32_DMA3=m -CONFIG_UDMABUF=y -CONFIG_DMABUF_HEAPS=y -CONFIG_DMABUF_HEAPS_SYSTEM=y -CONFIG_VFIO=y -CONFIG_VFIO_PCI=y -CONFIG_VIRTIO_PCI=y -CONFIG_VIRTIO_BALLOON=y -CONFIG_VIRTIO_MMIO=y -CONFIG_VHOST_NET=m -CONFIG_VHOST_VSOCK=m -CONFIG_XEN_GNTDEV=y -CONFIG_XEN_GRANT_DEV_ALLOC=y -CONFIG_GREYBUS=m -CONFIG_GREYBUS_BEAGLEPLAY=m -CONFIG_GPIB=m -CONFIG_GPIB_AGILENT_82357A=m -CONFIG_GPIB_NI_USB=m -CONFIG_GPIB_LPVO=m -CONFIG_STAGING=y -CONFIG_STAGING_MEDIA=y -CONFIG_VIDEO_MAX96712=m -CONFIG_VIDEO_MESON_VDEC=m -CONFIG_SND_BCM2835=m -CONFIG_CHROME_PLATFORMS=y -CONFIG_CROS_EC=y -CONFIG_CROS_EC_I2C=y -CONFIG_CROS_EC_RPMSG=m -CONFIG_CROS_EC_SPI=y -CONFIG_CROS_KBD_LED_BACKLIGHT=m -CONFIG_CROS_EC_CHARDEV=m -CONFIG_CROS_EC_TYPEC=m -CONFIG_CROS_TYPEC_SWITCH=m -CONFIG_EC_ACER_ASPIRE1=m -CONFIG_EC_HUAWEI_GAOKUN=m -CONFIG_EC_LENOVO_YOGA_C630=m -CONFIG_EC_LENOVO_THINKPAD_T14S=m -CONFIG_COMMON_CLK_RK808=y -CONFIG_COMMON_CLK_SCPI=y -CONFIG_COMMON_CLK_CS2000_CP=y -CONFIG_COMMON_CLK_FSL_SAI=y -CONFIG_COMMON_CLK_S2MPS11=y -CONFIG_COMMON_CLK_PWM=y -CONFIG_COMMON_CLK_RS9_PCIE=y -CONFIG_COMMON_CLK_VC3=y -CONFIG_COMMON_CLK_VC5=y -CONFIG_COMMON_CLK_BD718XX=m -CONFIG_CLK_RASPBERRYPI=y -CONFIG_CLK_IMX8MM=y -CONFIG_CLK_IMX8MN=y -CONFIG_CLK_IMX8MP=y -CONFIG_CLK_IMX8MQ=y -CONFIG_CLK_IMX8QXP=y -CONFIG_CLK_IMX8ULP=y -CONFIG_CLK_IMX93=y -CONFIG_CLK_IMX95_BLK_CTL=y -CONFIG_TI_SCI_CLK=y -CONFIG_COMMON_CLK_QCOM=y -CONFIG_CLK_ELIZA_DISPCC=m -CONFIG_CLK_ELIZA_GCC=y -CONFIG_CLK_ELIZA_TCSRCC=m -CONFIG_CLK_GLYMUR_DISPCC=m -CONFIG_CLK_GLYMUR_GCC=y -CONFIG_CLK_GLYMUR_TCSRCC=m -CONFIG_CLK_KAANAPALI_CAMCC=m -CONFIG_CLK_KAANAPALI_DISPCC=m -CONFIG_CLK_KAANAPALI_GCC=y -CONFIG_CLK_KAANAPALI_GPUCC=m -CONFIG_CLK_KAANAPALI_TCSRCC=m -CONFIG_CLK_KAANAPALI_VIDEOCC=m -CONFIG_CLK_X1E80100_CAMCC=m -CONFIG_CLK_X1E80100_DISPCC=m -CONFIG_CLK_X1E80100_GCC=y -CONFIG_CLK_X1E80100_GPUCC=m -CONFIG_CLK_X1E80100_TCSRCC=y -CONFIG_CLK_X1P42100_GPUCC=m -CONFIG_CLK_QCM2290_GPUCC=m -CONFIG_QCOM_A53PLL=y -CONFIG_QCOM_CLK_APCS_MSM8916=y -CONFIG_QCOM_CLK_APCC_MSM8996=y -CONFIG_QCOM_CLK_SMD_RPM=y -CONFIG_QCOM_CLK_RPMH=y -CONFIG_IPQ_APSS_6018=y -CONFIG_IPQ_CMN_PLL=m -CONFIG_IPQ_GCC_5018=y -CONFIG_IPQ_GCC_5210=y -CONFIG_IPQ_GCC_5332=y -CONFIG_IPQ_GCC_5424=y -CONFIG_IPQ_GCC_6018=y -CONFIG_IPQ_GCC_8074=y -CONFIG_IPQ_GCC_9574=y -CONFIG_IPQ_NSSCC_5424=m -CONFIG_IPQ_NSSCC_9574=m -CONFIG_MSM_GCC_8916=y -CONFIG_MSM_GCC_8953=y -CONFIG_MSM_MMCC_8994=m -CONFIG_MSM_GCC_8994=y -CONFIG_MSM_GCC_8996=y -CONFIG_MSM_MMCC_8996=m -CONFIG_MSM_GCC_8998=y -CONFIG_MSM_MMCC_8998=m -CONFIG_QCM_GCC_2290=y -CONFIG_QCM_DISPCC_2290=m -CONFIG_QCS_DISPCC_615=m -CONFIG_QCS_CAMCC_615=m -CONFIG_QCS_GCC_404=y -CONFIG_SA_CAMCC_8775P=m -CONFIG_QCS_GCC_8300=y -CONFIG_QCS_GCC_615=y -CONFIG_QCS_GPUCC_615=m -CONFIG_QCS_VIDEOCC_615=m -CONFIG_SC_CAMCC_7280=m -CONFIG_SC_CAMCC_8280XP=m -CONFIG_SA_DISPCC_8775P=m -CONFIG_SC_DISPCC_7280=m -CONFIG_SC_DISPCC_8280XP=m -CONFIG_SA_GCC_8775P=y -CONFIG_SA_GPUCC_8775P=m -CONFIG_SC_GCC_7180=y -CONFIG_SC_GCC_7280=y -CONFIG_SC_GCC_8180X=y -CONFIG_SC_GCC_8280XP=y -CONFIG_SC_GPUCC_7280=m -CONFIG_SC_GPUCC_8280XP=m -CONFIG_SC_LPASSCC_8280XP=m -CONFIG_SC_LPASS_CORECC_7280=m -CONFIG_SC_VIDEOCC_7280=m -CONFIG_SDM_CAMCC_845=m -CONFIG_QDU_GCC_1000=y -CONFIG_SDM_GPUCC_845=y -CONFIG_SDM_VIDEOCC_845=y -CONFIG_SDM_DISPCC_845=y -CONFIG_SDM_LPASSCC_845=m -CONFIG_SDX_GCC_75=y -CONFIG_SM_CAMCC_6350=m -CONFIG_SM_CAMCC_MILOS=m -CONFIG_SM_CAMCC_8250=m -CONFIG_SM_CAMCC_8550=m -CONFIG_SM_CAMCC_8650=m -CONFIG_SM_CAMCC_8750=m -CONFIG_SM_DISPCC_6115=m -CONFIG_SM_DISPCC_8250=y -CONFIG_SM_DISPCC_6350=m -CONFIG_SM_DISPCC_MILOS=m -CONFIG_SM_DISPCC_8450=m -CONFIG_SM_DISPCC_8550=m -CONFIG_SM_DISPCC_8750=m -CONFIG_SM_GCC_4450=y -CONFIG_SM_GCC_6115=y -CONFIG_SM_GCC_6350=y -CONFIG_SM_GCC_MILOS=y -CONFIG_SM_GCC_8350=y -CONFIG_SM_GCC_8450=y -CONFIG_SM_GCC_8550=y -CONFIG_SM_GCC_8650=y -CONFIG_SM_GCC_8750=y -CONFIG_SM_GPUCC_6115=m -CONFIG_SM_GPUCC_6350=m -CONFIG_SM_GPUCC_MILOS=m -CONFIG_SM_GPUCC_8150=y -CONFIG_SM_GPUCC_8250=y -CONFIG_SM_GPUCC_8350=m -CONFIG_SM_GPUCC_8450=m -CONFIG_SM_GPUCC_8550=m -CONFIG_SM_GPUCC_8650=m -CONFIG_SM_GPUCC_8750=m -CONFIG_SM_TCSRCC_8550=y -CONFIG_SM_TCSRCC_8650=y -CONFIG_SM_TCSRCC_8750=m -CONFIG_SA_VIDEOCC_8775P=m -CONFIG_SM_VIDEOCC_6350=m -CONFIG_SM_VIDEOCC_MILOS=m -CONFIG_SM_VIDEOCC_8250=y -CONFIG_SM_VIDEOCC_8550=m -CONFIG_SM_VIDEOCC_8750=m -CONFIG_QCOM_HFPLL=y -CONFIG_CLK_GFM_LPASS_SM8250=m -CONFIG_SM_VIDEOCC_8450=m -CONFIG_CLK_RCAR_USB2_CLOCK_SEL=y -CONFIG_CLK_RENESAS_VBATTB=m -CONFIG_EXYNOS_ACPM_CLK=m -CONFIG_CLK_SOPHGO_CV1800=y -CONFIG_XILINX_VCU=m -CONFIG_COMMON_CLK_XLNX_CLKWZRD=m -CONFIG_HWSPINLOCK=y -CONFIG_HWSPINLOCK_OMAP=m -CONFIG_HWSPINLOCK_QCOM=y -CONFIG_TEGRA186_TIMER=y -CONFIG_CLKSRC_STM32_LP=y -CONFIG_RENESAS_OSTM=y -CONFIG_ARM_MHU=y -CONFIG_EXYNOS_MBOX=m -CONFIG_IMX_MBOX=y -CONFIG_PLATFORM_MHU=y -CONFIG_OMAP2PLUS_MBOX=m -CONFIG_BCM2835_MBOX=y -CONFIG_QCOM_APCS_IPC=y -CONFIG_TEGRA_HSP_MBOX=y -CONFIG_MTK_ADSP_MBOX=m -CONFIG_QCOM_CPUCP_MBOX=m -CONFIG_QCOM_IPCC=y -CONFIG_CIX_MBOX=y -CONFIG_ARM_SMMU=y -CONFIG_ARM_SMMU_V3=y -CONFIG_QCOM_IOMMU=y -CONFIG_ROCKCHIP_IOMMU=y -CONFIG_TEGRA_IOMMU_SMMU=y -CONFIG_APPLE_DART=y -CONFIG_MTK_IOMMU=y -CONFIG_REMOTEPROC=y -CONFIG_IMX_REMOTEPROC=y -CONFIG_MTK_SCP=m -CONFIG_QCOM_Q6V5_ADSP=m -CONFIG_QCOM_Q6V5_MSS=m -CONFIG_QCOM_Q6V5_PAS=m -CONFIG_QCOM_SYSMON=m -CONFIG_QCOM_WCNSS_PIL=m -CONFIG_TI_K3_DSP_REMOTEPROC=m -CONFIG_TI_K3_M4_REMOTEPROC=m -CONFIG_TI_K3_R5_REMOTEPROC=m -CONFIG_RPMSG_CHAR=m -CONFIG_RPMSG_CTRL=m -CONFIG_RPMSG_QCOM_GLINK_RPM=y -CONFIG_RPMSG_QCOM_GLINK_SMEM=m -CONFIG_RPMSG_QCOM_SMD=y -CONFIG_RPMSG_VIRTIO=y -CONFIG_SOUNDWIRE=m -CONFIG_SOUNDWIRE_QCOM=m -CONFIG_APPLE_DOCKCHANNEL=y -CONFIG_APPLE_PMGR_MISC=y -CONFIG_APPLE_RTKIT_HELPER=y -CONFIG_APPLE_SART=y -CONFIG_APPLE_CLPC=m -CONFIG_FSL_DPAA=y -CONFIG_FSL_MC_DPIO=y -CONFIG_FSL_RCPM=y -CONFIG_MTK_CMDQ=m -CONFIG_MTK_DEVAPC=m -CONFIG_MTK_PMIC_WRAP=y -CONFIG_MTK_SVS=m -CONFIG_QCOM_AOSS_QMP=y -CONFIG_QCOM_COMMAND_DB=y -CONFIG_QCOM_GENI_SE=y -CONFIG_QCOM_LLCC=m -CONFIG_QCOM_OCMEM=m -CONFIG_QCOM_PMIC_GLINK=m -CONFIG_QCOM_RMTFS_MEM=m -CONFIG_QCOM_RPMH=y -CONFIG_QCOM_SMEM=y -CONFIG_QCOM_SMD_RPM=y -CONFIG_QCOM_SMP2P=y -CONFIG_QCOM_SMSM=y -CONFIG_QCOM_SOCINFO=m -CONFIG_QCOM_SPM=m -CONFIG_QCOM_STATS=m -CONFIG_QCOM_WCNSS_CTRL=m -CONFIG_QCOM_APR=m -CONFIG_QCOM_ICC_BWMON=m -CONFIG_QCOM_PBS=m -CONFIG_ROCKCHIP_IODOMAIN=y -CONFIG_TI_PRUSS=m -CONFIG_OWL_PM_DOMAINS=y -CONFIG_RASPBERRYPI_POWER=y -CONFIG_IMX_SCU_PD=y -CONFIG_QCOM_CPR=y -CONFIG_QCOM_RPMHPD=y -CONFIG_QCOM_RPMPD=y -CONFIG_TI_SCI_PM_DOMAINS=y -CONFIG_ARM_IMX_BUS_DEVFREQ=y -CONFIG_ARM_IMX8M_DDRC_DEVFREQ=m -CONFIG_ARM_MEDIATEK_CCI_DEVFREQ=m -CONFIG_PM_DEVFREQ_EVENT=y -CONFIG_DEVFREQ_EVENT_ROCKCHIP_DFI=m -CONFIG_EXTCON_PTN5150=m -CONFIG_EXTCON_USB_GPIO=y -CONFIG_EXTCON_USBC_CROS_EC=y -CONFIG_FSL_IFC=y -CONFIG_RENESAS_RPCIF=m -CONFIG_STM32_OMM=m -CONFIG_DLN2_ADC=m -CONFIG_EXYNOS_ADC=m -CONFIG_IMX8QXP_ADC=m -CONFIG_IMX93_ADC=m -CONFIG_MAX9611=m -CONFIG_MEDIATEK_MT6359_AUXADC=m -CONFIG_MEDIATEK_MT6577_AUXADC=m -CONFIG_QCOM_SPMI_VADC=m -CONFIG_QCOM_SPMI_ADC5=m -CONFIG_ROCKCHIP_SARADC=m -CONFIG_RZG2L_ADC=m -CONFIG_RZT2H_ADC=m -CONFIG_SOPHGO_CV1800B_ADC=m -CONFIG_TI_ADS1015=m -CONFIG_TI_AM335X_ADC=m -CONFIG_XILINX_XADC=m -CONFIG_XILINX_AMS=m -CONFIG_IIO_AOP_SENSOR_ALS=m -CONFIG_IIO_CROS_EC_SENSORS_CORE=m -CONFIG_IIO_CROS_EC_SENSORS=m -CONFIG_IIO_ST_LSM6DSX=m -CONFIG_IIO_CROS_EC_LIGHT_PROX=m -CONFIG_SENSORS_ISL29018=m -CONFIG_VCNL4000=m -CONFIG_IIO_ST_MAGN_3AXIS=m -CONFIG_IIO_STM32_TIMER_TRIGGER=m -CONFIG_IIO_CROS_EC_BARO=m -CONFIG_MPL3115=m -CONFIG_PWM=y -CONFIG_PWM_ADP5585=m -CONFIG_PWM_APPLE=m -CONFIG_PWM_BCM2835=m -CONFIG_PWM_BRCMSTB=m -CONFIG_PWM_CROS_EC=m -CONFIG_PWM_IMX27=m -CONFIG_PWM_MEDIATEK=m -CONFIG_PWM_MESON=m -CONFIG_PWM_MTK_DISP=m -CONFIG_PWM_RENESAS_RCAR=m -CONFIG_PWM_RENESAS_RZG2L_GPT=m -CONFIG_PWM_RENESAS_RZ_MTU3=m -CONFIG_PWM_RENESAS_TPU=m -CONFIG_PWM_ROCKCHIP=y -CONFIG_PWM_SAMSUNG=y -CONFIG_PWM_SL28CPLD=m -CONFIG_PWM_STM32=m -CONFIG_PWM_SUN4I=m -CONFIG_PWM_TEGRA=m -CONFIG_PWM_TIECAP=m -CONFIG_PWM_TIEHRPWM=m -CONFIG_PWM_VISCONTI=m -CONFIG_PWM_XILINX=m -CONFIG_SL28CPLD_INTC=y -CONFIG_XILINX_INTC=y -CONFIG_QCOM_PDC=y -CONFIG_QCOM_MPM=y -CONFIG_TI_SCI_INTR_IRQCHIP=y -CONFIG_TI_SCI_INTA_IRQCHIP=y -CONFIG_RESET_GPIO=m -CONFIG_RESET_IMX7=y -CONFIG_RESET_QCOM_AOSS=y -CONFIG_RESET_QCOM_PDC=m -CONFIG_RESET_RZG2L_USBPHY_CTRL=y -CONFIG_RESET_RZV2H_USB2PHY=m -CONFIG_RESET_TI_SCI=y -CONFIG_PHY_CAN_TRANSCEIVER=m -CONFIG_PHY_NXP_PTN3222=m -CONFIG_PHY_SNPS_EUSB2=m -CONFIG_PHY_XGENE=y -CONFIG_PHY_SUN4I_USB=y -CONFIG_PHY_APPLE_ATC=y -CONFIG_PHY_APPLE_DPTX=m -CONFIG_PHY_CADENCE_TORRENT=m -CONFIG_PHY_CADENCE_DPHY=m -CONFIG_PHY_CADENCE_DPHY_RX=m -CONFIG_PHY_CADENCE_SIERRA=m -CONFIG_PHY_CADENCE_SALVO=m -CONFIG_PHY_FSL_IMX8MQ_USB=m -CONFIG_PHY_MIXEL_MIPI_DPHY=m -CONFIG_PHY_FSL_IMX8M_PCIE=y -CONFIG_PHY_HI6220_USB=y -CONFIG_PHY_HISTB_COMBPHY=y -CONFIG_PHY_HISI_INNO_USB2=y -CONFIG_PHY_MVEBU_CP110_COMPHY=y -CONFIG_PHY_MTK_PCIE=m -CONFIG_PHY_MTK_TPHY=y -CONFIG_PHY_MTK_HDMI=m -CONFIG_PHY_MTK_MIPI_DSI=m -CONFIG_PHY_MTK_DP=m -CONFIG_PHY_QCOM_EDP=m -CONFIG_PHY_QCOM_PCIE2=m -CONFIG_PHY_QCOM_QMP=m -CONFIG_PHY_QCOM_QUSB2=m -CONFIG_PHY_QCOM_EUSB2_REPEATER=m -CONFIG_PHY_QCOM_M31_USB=m -CONFIG_PHY_QCOM_M31_EUSB=m -CONFIG_PHY_QCOM_USB_HS=m -CONFIG_PHY_QCOM_USB_SNPS_FEMTO_V2=m -CONFIG_PHY_QCOM_USB_HS_28NM=m -CONFIG_PHY_QCOM_USB_SS=m -CONFIG_PHY_QCOM_SGMII_ETH=m -CONFIG_PHY_R8A779F0_ETHERNET_SERDES=y -CONFIG_PHY_RCAR_GEN3_PCIE=y -CONFIG_PHY_RCAR_GEN3_USB2=y -CONFIG_PHY_RCAR_GEN3_USB3=m -CONFIG_PHY_RZ_G3E_USB3=m -CONFIG_PHY_ROCKCHIP_EMMC=y -CONFIG_PHY_ROCKCHIP_INNO_HDMI=m -CONFIG_PHY_ROCKCHIP_INNO_USB2=y -CONFIG_PHY_ROCKCHIP_INNO_DSIDPHY=m -CONFIG_PHY_ROCKCHIP_NANENG_COMBO_PHY=m -CONFIG_PHY_ROCKCHIP_PCIE=m -CONFIG_PHY_ROCKCHIP_SAMSUNG_HDPTX=m -CONFIG_PHY_ROCKCHIP_SNPS_PCIE3=y -CONFIG_PHY_ROCKCHIP_TYPEC=y -CONFIG_PHY_ROCKCHIP_USBDP=m -CONFIG_PHY_SAMSUNG_UFS=y -CONFIG_PHY_EXYNOS5_USBDRD=m -CONFIG_PHY_UNIPHIER_USB2=y -CONFIG_PHY_UNIPHIER_USB3=y -CONFIG_PHY_TEGRA_XUSB=y -CONFIG_PHY_AM654_SERDES=m -CONFIG_PHY_J721E_WIZ=m -CONFIG_OMAP_USB2=m -CONFIG_PHY_XILINX_ZYNQMP=m -CONFIG_ARM_CCI_PMU=m -CONFIG_ARM_CCN=m -CONFIG_ARM_CMN=m -CONFIG_ARM_SMMU_V3_PMU=m -CONFIG_ARM_DSU_PMU=m -CONFIG_FSL_IMX8_DDR_PMU=m -CONFIG_QCOM_L2_PMU=y -CONFIG_QCOM_L3_PMU=y -CONFIG_ARM_SPE_PMU=m -CONFIG_ARM_DMC620_PMU=m -CONFIG_APPLE_M1_CPU_PMU=y -CONFIG_HISI_PMU=y -CONFIG_ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU=m -CONFIG_NVIDIA_CORESIGHT_PMU_ARCH_SYSTEM_PMU=m -CONFIG_MESON_DDR_PMU=m -CONFIG_NVMEM_LAYOUT_SL28_VPD=m -CONFIG_NVMEM_APPLE_EFUSES=m -CONFIG_NVMEM_APPLE_SPMI=y -CONFIG_NVMEM_IMX_OCOTP=y -CONFIG_NVMEM_IMX_OCOTP_ELE=m -CONFIG_NVMEM_IMX_OCOTP_SCU=y -CONFIG_NVMEM_LAYERSCAPE_SFP=m -CONFIG_NVMEM_MTK_EFUSE=y -CONFIG_NVMEM_QCOM_QFPROM=y -CONFIG_NVMEM_QCOM_SEC_QFPROM=m -CONFIG_NVMEM_RMEM=m -CONFIG_NVMEM_ROCKCHIP_EFUSE=y -CONFIG_NVMEM_ROCKCHIP_OTP=y -CONFIG_NVMEM_SNVS_LPGPR=y -CONFIG_NVMEM_SPMI_SDAM=m -CONFIG_NVMEM_SUNXI_SID=y -CONFIG_NVMEM_UNIPHIER_EFUSE=y -CONFIG_NVMEM_ZYNQMP=m -CONFIG_FPGA=y -CONFIG_FPGA_MGR_ALTERA_CVP=m -CONFIG_FPGA_MGR_STRATIX10_SOC=m -CONFIG_FPGA_BRIDGE=m -CONFIG_ALTERA_FREEZE_BRIDGE=m -CONFIG_XILINX_PR_DECOUPLER=m -CONFIG_FPGA_REGION=m -CONFIG_OF_FPGA_REGION=m -CONFIG_FPGA_MGR_ZYNQMP_FPGA=m -CONFIG_FPGA_MGR_VERSAL_FPGA=m -CONFIG_TEE=y -CONFIG_OPTEE=y -CONFIG_QCOMTEE=m -CONFIG_MUX_APPLE_DPXBAR=y -CONFIG_MUX_GPIO=m -CONFIG_MUX_MMIO=y -CONFIG_SLIM_QCOM_NGD_CTRL=m -CONFIG_INTERCONNECT_IMX=y -CONFIG_INTERCONNECT_IMX8MM=m -CONFIG_INTERCONNECT_IMX8MN=m -CONFIG_INTERCONNECT_IMX8MQ=m -CONFIG_INTERCONNECT_IMX8MP=y -CONFIG_INTERCONNECT_QCOM=y -CONFIG_INTERCONNECT_QCOM_ELIZA=y -CONFIG_INTERCONNECT_QCOM_GLYMUR=y -CONFIG_INTERCONNECT_QCOM_KAANAPALI=y -CONFIG_INTERCONNECT_QCOM_MSM8916=m -CONFIG_INTERCONNECT_QCOM_MSM8953=y -CONFIG_INTERCONNECT_QCOM_MSM8996=y -CONFIG_INTERCONNECT_QCOM_OSM_L3=m -CONFIG_INTERCONNECT_QCOM_QCM2290=y -CONFIG_INTERCONNECT_QCOM_QCS404=m -CONFIG_INTERCONNECT_QCOM_QCS615=y -CONFIG_INTERCONNECT_QCOM_QCS8300=y -CONFIG_INTERCONNECT_QCOM_QDU1000=y -CONFIG_INTERCONNECT_QCOM_SA8775P=y -CONFIG_INTERCONNECT_QCOM_SC7180=y -CONFIG_INTERCONNECT_QCOM_SC7280=y -CONFIG_INTERCONNECT_QCOM_SC8180X=y -CONFIG_INTERCONNECT_QCOM_SC8280XP=y -CONFIG_INTERCONNECT_QCOM_SDM845=y -CONFIG_INTERCONNECT_QCOM_SDX75=y -CONFIG_INTERCONNECT_QCOM_SM6115=y -CONFIG_INTERCONNECT_QCOM_SM6350=y -CONFIG_INTERCONNECT_QCOM_MILOS=y -CONFIG_INTERCONNECT_QCOM_SM8150=y -CONFIG_INTERCONNECT_QCOM_SM8250=y -CONFIG_INTERCONNECT_QCOM_SM8350=y -CONFIG_INTERCONNECT_QCOM_SM8450=y -CONFIG_INTERCONNECT_QCOM_SM8550=y -CONFIG_INTERCONNECT_QCOM_SM8650=y -CONFIG_INTERCONNECT_QCOM_SM8750=y -CONFIG_INTERCONNECT_QCOM_X1E80100=y -CONFIG_COUNTER=m -CONFIG_RZ_MTU3_CNT=m -CONFIG_STM32_TIMER_CNT=m -CONFIG_TI_EQEP=m -CONFIG_HTE=y -CONFIG_HTE_TEGRA194=y -CONFIG_HTE_TEGRA194_TEST=m -CONFIG_EXT4_FS=y -CONFIG_EXT4_FS_POSIX_ACL=y -CONFIG_EXT4_FS_SECURITY=y -CONFIG_XFS_FS=m -CONFIG_XFS_SUPPORT_V4=y -CONFIG_XFS_SUPPORT_ASCII_CI=y -CONFIG_XFS_QUOTA=y -CONFIG_XFS_POSIX_ACL=y -CONFIG_XFS_RT=y -CONFIG_BTRFS_FS=y -CONFIG_BTRFS_FS_POSIX_ACL=y -CONFIG_F2FS_FS=m -CONFIG_F2FS_FS_SECURITY=y -CONFIG_F2FS_FS_COMPRESSION=y -CONFIG_F2FS_UNFAIR_RWSEM=y -CONFIG_FANOTIFY=y -CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y -CONFIG_QUOTA=y -CONFIG_AUTOFS_FS=y -CONFIG_FUSE_FS=m -CONFIG_CUSE=m -CONFIG_OVERLAY_FS=m -CONFIG_ISO9660_FS=m -CONFIG_JOLIET=y -CONFIG_ZISOFS=y -CONFIG_UDF_FS=m -CONFIG_MSDOS_FS=m -CONFIG_VFAT_FS=y -CONFIG_EXFAT_FS=m -CONFIG_NTFS3_FS=m -CONFIG_NTFS3_LZX_XPRESS=y -CONFIG_NTFS3_FS_POSIX_ACL=y -CONFIG_TMPFS_POSIX_ACL=y -CONFIG_HUGETLBFS=y -CONFIG_EFIVAR_FS=y -CONFIG_HFS_FS=m -CONFIG_HFSPLUS_FS=m -CONFIG_UBIFS_FS=m -CONFIG_SQUASHFS=y -CONFIG_SQUASHFS_XATTR=y -CONFIG_SQUASHFS_LZ4=y -CONFIG_SQUASHFS_LZO=y -CONFIG_SQUASHFS_XZ=y -CONFIG_SQUASHFS_ZSTD=y -CONFIG_PSTORE_RAM=m -CONFIG_EROFS_FS=m -CONFIG_EROFS_FS_ZIP_DEFLATE=y -CONFIG_EROFS_FS_ZIP_ZSTD=y -CONFIG_EROFS_FS_ZIP_ACCEL=y -CONFIG_NFS_FS=y -CONFIG_NFS_V4=y -CONFIG_NFS_V4_2=y -CONFIG_ROOT_NFS=y -CONFIG_CIFS=m -CONFIG_CIFS_UPCALL=y -CONFIG_CIFS_XATTR=y -CONFIG_CIFS_POSIX=y -CONFIG_CIFS_DFS_UPCALL=y -CONFIG_CIFS_SWN_UPCALL=y -CONFIG_9P_FS=y -CONFIG_NLS_CODEPAGE_437=y -CONFIG_NLS_CODEPAGE_737=m -CONFIG_NLS_CODEPAGE_775=m -CONFIG_NLS_CODEPAGE_850=m -CONFIG_NLS_CODEPAGE_852=m -CONFIG_NLS_CODEPAGE_855=m -CONFIG_NLS_CODEPAGE_857=m -CONFIG_NLS_CODEPAGE_860=m -CONFIG_NLS_CODEPAGE_861=m -CONFIG_NLS_CODEPAGE_862=m -CONFIG_NLS_CODEPAGE_863=m -CONFIG_NLS_CODEPAGE_864=m -CONFIG_NLS_CODEPAGE_865=m -CONFIG_NLS_CODEPAGE_866=m -CONFIG_NLS_CODEPAGE_869=m -CONFIG_NLS_CODEPAGE_936=m -CONFIG_NLS_CODEPAGE_950=m -CONFIG_NLS_CODEPAGE_932=m -CONFIG_NLS_CODEPAGE_949=m -CONFIG_NLS_CODEPAGE_874=m -CONFIG_NLS_ISO8859_8=m -CONFIG_NLS_CODEPAGE_1250=m -CONFIG_NLS_CODEPAGE_1251=m -CONFIG_NLS_ASCII=m -CONFIG_NLS_ISO8859_1=y -CONFIG_NLS_ISO8859_2=m -CONFIG_NLS_ISO8859_3=m -CONFIG_NLS_ISO8859_4=m -CONFIG_NLS_ISO8859_5=m -CONFIG_NLS_ISO8859_6=m -CONFIG_NLS_ISO8859_7=m -CONFIG_NLS_ISO8859_9=m -CONFIG_NLS_ISO8859_13=m -CONFIG_NLS_ISO8859_14=m -CONFIG_NLS_ISO8859_15=m -CONFIG_NLS_KOI8_R=m -CONFIG_NLS_KOI8_U=m -CONFIG_NLS_MAC_ROMAN=m -CONFIG_NLS_MAC_CELTIC=m -CONFIG_NLS_MAC_CENTEURO=m -CONFIG_NLS_MAC_CROATIAN=m -CONFIG_NLS_MAC_CYRILLIC=m -CONFIG_NLS_MAC_GAELIC=m -CONFIG_NLS_MAC_GREEK=m -CONFIG_NLS_MAC_ICELAND=m -CONFIG_NLS_MAC_INUIT=m -CONFIG_NLS_MAC_ROMANIAN=m -CONFIG_NLS_MAC_TURKISH=m -CONFIG_PERSISTENT_KEYRINGS=y -CONFIG_TRUSTED_KEYS=y -# CONFIG_TRUSTED_KEYS_TPM is not set -# CONFIG_TRUSTED_KEYS_TEE is not set -CONFIG_SECURITY=y -CONFIG_SECURITY_YAMA=y -CONFIG_SECURITY_LANDLOCK=y -CONFIG_CRYPTO_USER=y -CONFIG_CRYPTO_NULL=m -CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_BENCHMARK=m -CONFIG_CRYPTO_ECDH=y -CONFIG_CRYPTO_AES=y -CONFIG_CRYPTO_BLOWFISH=m -CONFIG_CRYPTO_CAMELLIA=m -CONFIG_CRYPTO_CAST5=m -CONFIG_CRYPTO_CAST6=m -CONFIG_CRYPTO_SERPENT=m -CONFIG_CRYPTO_TWOFISH=m -CONFIG_CRYPTO_ADIANTUM=m -CONFIG_CRYPTO_CTS=m -CONFIG_CRYPTO_HCTR2=m -CONFIG_CRYPTO_LRW=m -CONFIG_CRYPTO_PCBC=m -CONFIG_CRYPTO_AEGIS128=m -CONFIG_CRYPTO_CCM=y -CONFIG_CRYPTO_GCM=y -CONFIG_CRYPTO_ECHAINIV=y -CONFIG_CRYPTO_BLAKE2B=m -CONFIG_CRYPTO_CMAC=y -CONFIG_CRYPTO_HMAC=y -CONFIG_CRYPTO_MD4=m -CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_SHA256=y -CONFIG_CRYPTO_STREEBOG=m -CONFIG_CRYPTO_WP512=m -CONFIG_CRYPTO_XCBC=m -CONFIG_CRYPTO_XXHASH=m -CONFIG_CRYPTO_CRC32C=m -CONFIG_CRYPTO_CRC32=m -CONFIG_CRYPTO_LZ4HC=m -CONFIG_CRYPTO_USER_API_HASH=m -CONFIG_CRYPTO_USER_API_SKCIPHER=m -CONFIG_CRYPTO_USER_API_RNG=m -CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_GHASH_ARM64_CE=y -CONFIG_CRYPTO_AES_ARM64_BS=m -CONFIG_CRYPTO_AES_ARM64_CE_CCM=y -CONFIG_CRYPTO_DEV_SUN8I_CE=m -CONFIG_CRYPTO_DEV_FSL_CAAM=m -CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM=m -CONFIG_CRYPTO_DEV_QCE=m -CONFIG_CRYPTO_DEV_QCOM_RNG=m -CONFIG_CRYPTO_DEV_TEGRA=m -CONFIG_CRYPTO_DEV_XILINX_TRNG=m -CONFIG_CRYPTO_DEV_ZYNQMP_AES=m -CONFIG_CRYPTO_DEV_ZYNQMP_SHA3=m -CONFIG_CRYPTO_DEV_CCREE=m -CONFIG_CRYPTO_DEV_HISI_SEC2=m -CONFIG_CRYPTO_DEV_HISI_ZIP=m -CONFIG_CRYPTO_DEV_HISI_HPRE=m -CONFIG_CRYPTO_DEV_HISI_TRNG=m -CONFIG_CRYPTO_DEV_SA2UL=m -CONFIG_DMA_RESTRICTED_POOL=y -CONFIG_CMA_SIZE_MBYTES=32 -CONFIG_PRINTK_TIME=y -CONFIG_DYNAMIC_DEBUG=y -CONFIG_DEBUG_KERNEL=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y -CONFIG_DEBUG_INFO_REDUCED=y -CONFIG_MAGIC_SYSRQ=y -CONFIG_DEBUG_FS=y -CONFIG_ENABLE_DEFAULT_TRACERS=y -CONFIG_CORESIGHT=m -CONFIG_CORESIGHT_LINK_AND_SINK_TMC=m -CONFIG_CORESIGHT_CATU=m -CONFIG_CORESIGHT_SINK_TPIU=m -CONFIG_CORESIGHT_SINK_ETBV10=m -CONFIG_CORESIGHT_STM=m -CONFIG_CORESIGHT_CPU_DEBUG=m -CONFIG_CORESIGHT_CTI=m -CONFIG_ARM64_USER4K_EXPERIMENTAL=y -CONFIG_MEMTEST=y diff --git a/arch/arm64/configs/j700-device-parity.config b/arch/arm64/configs/j700-device-parity.config deleted file mode 100644 index 55bce7a9ac0af..0000000000000 --- a/arch/arm64/configs/j700-device-parity.config +++ /dev/null @@ -1,1053 +0,0 @@ -# J700 (MacBook Neo, T8140) device-parity fragment. -# -# Brings the J700 configuration to device and feature parity with a general-purpose -# distribution kernel (reference: Fedora Asahi Remix 44, 7.1.13-402.asahi.fc44.aarch64+16k): -# drivers for devices that can attach over USB or Bluetooth, and common protocols, -# filesystems and features. Most entries are modules. Nothing here changes an option -# that defconfig, asahi.config, j700-radio.config or j700.config set. -# -# Merge after j700.config; see that file for the full sequence. - -# Prerequisites of the entries below: nftables core, hidraw, uinput and the ALSA sequencer -CONFIG_NF_TABLES=m -CONFIG_NF_TABLES_INET=y -CONFIG_NF_TABLES_IPV4=y -CONFIG_NF_TABLES_IPV6=y -CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_CT=m -CONFIG_NFT_LOG=m -CONFIG_NFT_LIMIT=m -CONFIG_NFT_MASQ=m -CONFIG_NFT_NAT=m -CONFIG_NFT_REJECT=m -CONFIG_NFT_REJECT_INET=m -CONFIG_NFT_FIB_INET=m -CONFIG_NFT_FIB_IPV4=m -CONFIG_NFT_FIB_IPV6=m -CONFIG_HIDRAW=y -CONFIG_INPUT_UINPUT=m -CONFIG_SND_SEQUENCER=m -CONFIG_SND_HRTIMER=m -CONFIG_SND_SEQ_DUMMY=m -CONFIG_SND_SEQ_HRTIMER_DEFAULT=y - -# HID / input / game controllers (USB + Bluetooth HID devices, xpad, joydev, USB tablets/touchscreens) -CONFIG_DRAGONRISE_FF=y -CONFIG_GREENASIA_FF=y -CONFIG_HID_A4TECH=m -CONFIG_HID_ACCUTOUCH=m -CONFIG_HID_ACRUX=m -CONFIG_HID_ACRUX_FF=y -CONFIG_HID_ALPS=m -CONFIG_HID_APPLEIR=m -CONFIG_HID_AUREAL=m -CONFIG_HID_BATTERY_STRENGTH=y -CONFIG_HID_BELKIN=m -CONFIG_HID_BETOP_FF=m -CONFIG_HID_BIGBEN_FF=m -CONFIG_HID_CHERRY=m -CONFIG_HID_CHICONY=m -CONFIG_HID_CMEDIA=m -CONFIG_HID_CORSAIR=m -CONFIG_HID_COUGAR=m -CONFIG_HID_CP2112=m -CONFIG_HID_CREATIVE_SB0540=m -CONFIG_HID_CYPRESS=m -CONFIG_HID_DRAGONRISE=m -CONFIG_HID_ELAN=m -CONFIG_HID_ELECOM=m -CONFIG_HID_ELO=m -CONFIG_HID_EMS_FF=m -CONFIG_HID_EVISION=m -CONFIG_HID_EZKEY=m -CONFIG_HID_FT260=m -CONFIG_HID_GEMBIRD=m -CONFIG_HID_GFRM=m -CONFIG_HID_GLORIOUS=m -CONFIG_HID_GOOGLE_STADIA_FF=m -CONFIG_HID_GREENASIA=m -CONFIG_HID_GT683R=m -CONFIG_HID_GYRATION=m -CONFIG_HID_HOLTEK=m -CONFIG_HID_HUAWEI=m -CONFIG_HID_ICADE=m -CONFIG_HID_ITE=m -CONFIG_HID_JABRA=m -CONFIG_HID_KENSINGTON=m -CONFIG_HID_KEYTOUCH=m -CONFIG_HID_KYE=m -CONFIG_HID_KYSONA=m -CONFIG_HID_LCPOWER=m -CONFIG_HID_LED=m -CONFIG_HID_LENOVO=m -CONFIG_HID_LENOVO_GO=m -CONFIG_HID_LENOVO_GO_S=m -CONFIG_HID_LETSKETCH=m -CONFIG_HID_LOGITECH=m -CONFIG_HID_LOGITECH_DJ=m -CONFIG_HID_LOGITECH_HIDPP=m -CONFIG_HID_MACALLY=m -CONFIG_HID_MALTRON=m -CONFIG_HID_MAYFLASH=m -CONFIG_HID_MCP2221=m -CONFIG_HID_MEGAWORLD_FF=m -CONFIG_HID_MICROSOFT=m -CONFIG_HID_MONTEREY=m -CONFIG_HID_NINTENDO=m -CONFIG_HID_NTI=m -CONFIG_HID_NTRIG=m -CONFIG_HID_NVIDIA_SHIELD=m -CONFIG_HID_ORTEK=m -CONFIG_HID_PANTHERLORD=m -CONFIG_HID_PENMOUNT=m -CONFIG_HID_PETALYNX=m -CONFIG_HID_PICOLCD=m -CONFIG_HID_PICOLCD_BACKLIGHT=y -CONFIG_HID_PICOLCD_FB=y -CONFIG_HID_PICOLCD_LEDS=y -CONFIG_HID_PID=y -CONFIG_HID_PLANTRONICS=m -CONFIG_HID_PLAYSTATION=m -CONFIG_HID_PRIMAX=m -CONFIG_HID_PRODIKEYS=m -CONFIG_HID_PXRC=m -CONFIG_HID_RAPOO=m -CONFIG_HID_RAZER=m -CONFIG_HID_RETRODE=m -CONFIG_HID_RMI=m -CONFIG_HID_ROCCAT=m -CONFIG_HID_SAITEK=m -CONFIG_HID_SAMSUNG=m -CONFIG_HID_SEMITEK=m -CONFIG_HID_SIGMAMICRO=m -CONFIG_HID_SMARTJOYPLUS=m -CONFIG_HID_SONY=m -CONFIG_HID_SPEEDLINK=m -CONFIG_HID_STEAM=m -CONFIG_HID_STEELSERIES=m -CONFIG_HID_SUNPLUS=m -CONFIG_HID_THINGM=m -CONFIG_HID_THRUSTMASTER=m -CONFIG_HID_TIVO=m -CONFIG_HID_TOPRE=m -CONFIG_HID_TOPSEED=m -CONFIG_HID_TWINHAN=m -CONFIG_HID_U2FZERO=m -CONFIG_HID_UCLOGIC=m -CONFIG_HID_UDRAW_PS3=m -CONFIG_HID_UNIVERSAL_PIDFF=m -CONFIG_HID_VIEWSONIC=m -CONFIG_HID_VIVALDI=m -CONFIG_HID_WACOM=m -CONFIG_HID_WALTOP=m -CONFIG_HID_WIIMOTE=m -CONFIG_HID_WINWING=m -CONFIG_HID_XIAOMI=m -CONFIG_HID_XINMO=m -CONFIG_HID_ZEROPLUS=m -CONFIG_HID_ZYDACRON=m -CONFIG_HOLTEK_FF=y -CONFIG_INPUT_JOYDEV=m -CONFIG_INPUT_JOYSTICK=y -CONFIG_INPUT_TABLET=y -CONFIG_JOYSTICK_IFORCE=m -CONFIG_JOYSTICK_IFORCE_USB=m -CONFIG_JOYSTICK_PXRC=m -CONFIG_JOYSTICK_XPAD=m -CONFIG_JOYSTICK_XPAD_FF=y -CONFIG_JOYSTICK_XPAD_LEDS=y -CONFIG_LOGIG940_FF=y -CONFIG_LOGIRUMBLEPAD2_FF=y -CONFIG_LOGITECH_FF=y -CONFIG_LOGIWHEELS_FF=y -CONFIG_MOUSE_SYNAPTICS_USB=m -CONFIG_NINTENDO_FF=y -CONFIG_NVIDIA_SHIELD_FF=y -CONFIG_PANTHERLORD_FF=y -CONFIG_PLAYSTATION_FF=y -CONFIG_RMI4_CORE=m -CONFIG_RMI4_F03=y -CONFIG_RMI4_F11=y -CONFIG_RMI4_F12=y -CONFIG_RMI4_F1A=y -CONFIG_RMI4_F21=y -CONFIG_RMI4_F30=y -CONFIG_RMI4_F34=y -CONFIG_RMI4_F3A=y -CONFIG_RMI4_F55=y -CONFIG_SMARTJOYPLUS_FF=y -CONFIG_SONY_FF=y -CONFIG_STEAM_FF=y -CONFIG_TABLET_USB_ACECAD=m -CONFIG_TABLET_USB_AIPTEK=m -CONFIG_TABLET_USB_HANWANG=m -CONFIG_TABLET_USB_KBTAB=m -CONFIG_TABLET_USB_PEGASUS=m -CONFIG_THRUSTMASTER_FF=y -CONFIG_TOUCHSCREEN_USB_3M=y -CONFIG_TOUCHSCREEN_USB_COMPOSITE=m -CONFIG_TOUCHSCREEN_USB_DMC_TSC10=y -CONFIG_TOUCHSCREEN_USB_E2I=y -CONFIG_TOUCHSCREEN_USB_EASYTOUCH=y -CONFIG_TOUCHSCREEN_USB_EGALAX=y -CONFIG_TOUCHSCREEN_USB_ELO=y -CONFIG_TOUCHSCREEN_USB_ETT_TC45USB=y -CONFIG_TOUCHSCREEN_USB_ETURBO=y -CONFIG_TOUCHSCREEN_USB_GENERAL_TOUCH=y -CONFIG_TOUCHSCREEN_USB_GOTOP=y -CONFIG_TOUCHSCREEN_USB_GUNZE=y -CONFIG_TOUCHSCREEN_USB_IDEALTEK=y -CONFIG_TOUCHSCREEN_USB_IRTOUCH=y -CONFIG_TOUCHSCREEN_USB_ITM=y -CONFIG_TOUCHSCREEN_USB_JASTEC=y -CONFIG_TOUCHSCREEN_USB_NEXIO=y -CONFIG_TOUCHSCREEN_USB_PANJIT=y -CONFIG_TOUCHSCREEN_USB_ZYTRONIC=y -CONFIG_USB_HIDDEV=y -CONFIG_ZEROPLUS_FF=y - -# USB device classes: serial adapters, storage sub-drivers, printers/TMC/WDM, misc USB gadgets, usbmon, USB/IP, card readers, maker adapters -CONFIG_APPLE_MFI_FASTCHARGE=m -CONFIG_DLN2_ADC=m -CONFIG_GNSS_USB=m -CONFIG_GPIB=m -CONFIG_GPIB_AGILENT_82357A=m -CONFIG_GPIB_COMMON=m -CONFIG_GPIB_LPVO=m -CONFIG_GPIB_NI_USB=m -CONFIG_GPIO_DLN2=m -CONFIG_GPIO_MPSSE=m -CONFIG_I2C_CP2615=m -CONFIG_I2C_DIOLAN_U2C=m -CONFIG_I2C_DLN2=m -CONFIG_I2C_TINY_USB=m -CONFIG_MEMSTICK=m -CONFIG_MEMSTICK_REALTEK_USB=m -CONFIG_MFD_DLN2=m -CONFIG_MISC_RTSX_USB=m -CONFIG_MMC_REALTEK_USB=m -CONFIG_MMC_USHC=m -CONFIG_MMC_VUB300=m -CONFIG_MSPRO_BLOCK=m -CONFIG_REALTEK_AUTOPM=y -CONFIG_SENSORS_AQUACOMPUTER_D5NEXT=m -CONFIG_SENSORS_ASUS_ROG_RYUJIN=m -CONFIG_SENSORS_CORSAIR_CPRO=m -CONFIG_SENSORS_CORSAIR_PSU=m -CONFIG_SENSORS_GIGABYTE_WATERFORCE=m -CONFIG_SENSORS_NZXT_KRAKEN2=m -CONFIG_SENSORS_NZXT_KRAKEN3=m -CONFIG_SENSORS_NZXT_SMART2=m -CONFIG_SENSORS_POWERZ=m -CONFIG_SPI_CH341=m -CONFIG_SPI_DLN2=m -CONFIG_USBIP_CORE=m -CONFIG_USBIP_HOST=m -CONFIG_USBIP_VHCI_HCD=m -CONFIG_USBPCWATCHDOG=m -CONFIG_USB_ADUTUX=m -CONFIG_USB_APPLEDISPLAY=m -CONFIG_USB_CHAOSKEY=m -CONFIG_USB_EMI26=m -CONFIG_USB_EMI62=m -CONFIG_USB_EZUSB_FX2=m -CONFIG_USB_IDMOUSE=m -CONFIG_USB_IOWARRIOR=m -CONFIG_USB_ISIGHTFW=m -CONFIG_USB_LCD=m -CONFIG_USB_LD=m -CONFIG_USB_LEGOTOWER=m -CONFIG_USB_MDC800=m -CONFIG_USB_MICROTEK=m -CONFIG_USB_MON=m -CONFIG_USB_PRINTER=m -CONFIG_USB_SERIAL_AIRCABLE=m -CONFIG_USB_SERIAL_ARK3116=m -CONFIG_USB_SERIAL_BELKIN=m -CONFIG_USB_SERIAL_CH341=m -CONFIG_USB_SERIAL_CYBERJACK=m -CONFIG_USB_SERIAL_CYPRESS_M8=m -CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m -CONFIG_USB_SERIAL_EDGEPORT=m -CONFIG_USB_SERIAL_EDGEPORT_TI=m -CONFIG_USB_SERIAL_EMPEG=m -CONFIG_USB_SERIAL_F81232=m -CONFIG_USB_SERIAL_F8153X=m -CONFIG_USB_SERIAL_GARMIN=m -CONFIG_USB_SERIAL_GENERIC=y -CONFIG_USB_SERIAL_IPAQ=m -CONFIG_USB_SERIAL_IPW=m -CONFIG_USB_SERIAL_IR=m -CONFIG_USB_SERIAL_IUU=m -CONFIG_USB_SERIAL_KEYSPAN=m -CONFIG_USB_SERIAL_KEYSPAN_PDA=m -CONFIG_USB_SERIAL_KLSI=m -CONFIG_USB_SERIAL_KOBIL_SCT=m -CONFIG_USB_SERIAL_MCT_U232=m -CONFIG_USB_SERIAL_MOS7720=m -CONFIG_USB_SERIAL_MOS7840=m -CONFIG_USB_SERIAL_NAVMAN=m -CONFIG_USB_SERIAL_OMNINET=m -CONFIG_USB_SERIAL_OPTICON=m -CONFIG_USB_SERIAL_OTI6858=m -CONFIG_USB_SERIAL_PL2303=m -CONFIG_USB_SERIAL_QCAUX=m -CONFIG_USB_SERIAL_QT2=m -CONFIG_USB_SERIAL_QUALCOMM=m -CONFIG_USB_SERIAL_SAFE=m -CONFIG_USB_SERIAL_SAFE_PADDED=y -CONFIG_USB_SERIAL_SIERRAWIRELESS=m -CONFIG_USB_SERIAL_SIMPLE=m -CONFIG_USB_SERIAL_SPCP8X5=m -CONFIG_USB_SERIAL_SSU100=m -CONFIG_USB_SERIAL_SYMBOL=m -CONFIG_USB_SERIAL_TI=m -CONFIG_USB_SERIAL_UPD78F0730=m -CONFIG_USB_SERIAL_VISOR=m -CONFIG_USB_SERIAL_WHITEHEAT=m -CONFIG_USB_SERIAL_XR=m -CONFIG_USB_SERIAL_XSENS_MT=m -CONFIG_USB_SEVSEG=m -CONFIG_USB_SISUSBVGA=m -CONFIG_USB_STORAGE_ALAUDA=m -CONFIG_USB_STORAGE_CYPRESS_ATACB=m -CONFIG_USB_STORAGE_DATAFAB=m -CONFIG_USB_STORAGE_ENE_UB6250=m -CONFIG_USB_STORAGE_FREECOM=m -CONFIG_USB_STORAGE_ISD200=m -CONFIG_USB_STORAGE_JUMPSHOT=m -CONFIG_USB_STORAGE_KARMA=m -CONFIG_USB_STORAGE_ONETOUCH=m -CONFIG_USB_STORAGE_REALTEK=m -CONFIG_USB_STORAGE_SDDR09=m -CONFIG_USB_STORAGE_SDDR55=m -CONFIG_USB_STORAGE_USBAT=m -CONFIG_USB_TMC=m -CONFIG_USB_TRANCEVIBRATOR=m -CONFIG_USB_WDM=m -CONFIG_USB_YUREX=m -CONFIG_W1=m -CONFIG_W1_MASTER_DS2490=m -CONFIG_W1_SLAVE_DS2405=m -CONFIG_W1_SLAVE_DS2406=m -CONFIG_W1_SLAVE_DS2408=m -CONFIG_W1_SLAVE_DS2413=m -CONFIG_W1_SLAVE_DS2423=m -CONFIG_W1_SLAVE_DS2430=m -CONFIG_W1_SLAVE_DS2431=m -CONFIG_W1_SLAVE_DS2433=m -CONFIG_W1_SLAVE_DS2433_CRC=y -CONFIG_W1_SLAVE_DS2438=m -CONFIG_W1_SLAVE_DS2780=m -CONFIG_W1_SLAVE_DS2781=m -CONFIG_W1_SLAVE_DS2805=m -CONFIG_W1_SLAVE_DS28E04=m -CONFIG_W1_SLAVE_SMEM=m -CONFIG_W1_SLAVE_THERM=m -CONFIG_XILLYUSB=m - -# USB network adapters: USB Ethernet/tethering/mobile broadband, USB Wi-Fi dongles, USB CAN adapters -CONFIG_AR5523=m -CONFIG_ATH10K_USB=m -CONFIG_ATH6KL=m -CONFIG_ATH6KL_USB=m -CONFIG_ATH9K_HTC=m -CONFIG_BRCMFMAC_USB=y -CONFIG_CAN_8DEV_USB=m -CONFIG_CAN_EMS_USB=m -CONFIG_CAN_ESD_USB=m -CONFIG_CAN_F81604=m -CONFIG_CAN_GS_USB=m -CONFIG_CAN_ISOTP=m -CONFIG_CAN_J1939=m -CONFIG_CAN_KVASER_USB=m -CONFIG_CAN_MCBA_USB=m -CONFIG_CAN_PEAK_USB=m -CONFIG_CARL9170=m -CONFIG_CARL9170_LEDS=y -CONFIG_LIBERTAS_THINFIRM=m -CONFIG_LIBERTAS_THINFIRM_USB=m -CONFIG_MT7601U=m -CONFIG_MT7663U=m -CONFIG_MT76x0U=m -CONFIG_MT76x2U=m -CONFIG_MT7921U=m -CONFIG_MT7925U=m -CONFIG_RT2500USB=m -CONFIG_RT2800USB=m -CONFIG_RT2800USB_RT33XX=y -CONFIG_RT2800USB_RT3573=y -CONFIG_RT2800USB_RT35XX=y -CONFIG_RT2800USB_RT53XX=y -CONFIG_RT2800USB_RT55XX=y -CONFIG_RT2800USB_UNKNOWN=y -CONFIG_RT2X00=m -CONFIG_RT73USB=m -CONFIG_RTL8187=m -CONFIG_RTL8192DU=m -CONFIG_RTL8XXXU=m -CONFIG_RTW88=m -CONFIG_RTW88_8723DU=m -CONFIG_RTW88_8812AU=m -CONFIG_RTW88_8814AU=m -CONFIG_RTW88_8821AU=m -CONFIG_RTW88_8821CU=m -CONFIG_RTW88_8822BU=m -CONFIG_RTW88_8822CU=m -CONFIG_RTW89=m -CONFIG_RTW89_8851BU=m -CONFIG_RTW89_8852AU=m -CONFIG_RTW89_8852BU=m -CONFIG_RTW89_8852CU=m -CONFIG_USB_ALI_M5632=y -CONFIG_USB_AN2720=y -CONFIG_USB_CATC=m -CONFIG_USB_EPSON2888=y -CONFIG_USB_HSO=m -CONFIG_USB_IPHETH=m -CONFIG_USB_KAWETH=m -CONFIG_USB_KC2190=y -CONFIG_USB_NET_AQC111=m -CONFIG_USB_NET_CDC_EEM=m -CONFIG_USB_NET_CDC_MBIM=m -CONFIG_USB_NET_CH9200=m -CONFIG_USB_NET_CX82310_ETH=m -CONFIG_USB_NET_GL620A=m -CONFIG_USB_NET_HUAWEI_CDC_NCM=m -CONFIG_USB_NET_INT51X1=m -CONFIG_USB_NET_KALMIA=m -CONFIG_USB_NET_QMI_WWAN=m -CONFIG_USB_NET_RNDIS_HOST=m -CONFIG_USB_NET_SR9700=m -CONFIG_USB_SIERRA_NET=m -CONFIG_USB_VL600=m -CONFIG_ZD1211RW=m - -# Storage: optical/SCSI generic, device-mapper (LUKS, LVM, verity, integrity), MD RAID, partition tables -CONFIG_AIX_PARTITION=y -CONFIG_BLK_DEV_SR=m -CONFIG_BSD_DISKLABEL=y -CONFIG_CHR_DEV_SG=m -CONFIG_DM_CACHE=m -CONFIG_DM_CACHE_SMQ=m -CONFIG_DM_CRYPT=m -CONFIG_DM_INTEGRITY=m -CONFIG_DM_RAID=m -CONFIG_DM_SNAPSHOT=m -CONFIG_DM_THIN_PROVISIONING=m -CONFIG_DM_UEVENT=y -CONFIG_DM_VERITY=m -CONFIG_DM_VERITY_FEC=y -CONFIG_DM_WRITECACHE=m -CONFIG_LDM_PARTITION=y -CONFIG_MAC_PARTITION=y -CONFIG_MD_LINEAR=m -CONFIG_MD_RAID0=m -CONFIG_MD_RAID1=m -CONFIG_MD_RAID10=m -CONFIG_MD_RAID456=m -CONFIG_MINIX_SUBPARTITION=y -CONFIG_OSF_PARTITION=y -CONFIG_PARTITION_ADVANCED=y -CONFIG_SGI_PARTITION=y -CONFIG_SOLARIS_X86_PARTITION=y -CONFIG_SUN_PARTITION=y -CONFIG_UNIXWARE_DISKLABEL=y - -# Filesystems for removable media and images, binfmt_misc, SMB client -CONFIG_BINFMT_MISC=m -CONFIG_CIFS=m -CONFIG_CIFS_ALLOW_INSECURE_LEGACY=y -CONFIG_CIFS_DFS_UPCALL=y -CONFIG_CIFS_POSIX=y -CONFIG_CIFS_SWN_UPCALL=y -CONFIG_CIFS_UPCALL=y -CONFIG_CIFS_XATTR=y -CONFIG_EROFS_FS=m -CONFIG_EROFS_FS_BACKED_BY_FILE=y -CONFIG_EROFS_FS_POSIX_ACL=y -CONFIG_EROFS_FS_SECURITY=y -CONFIG_EROFS_FS_XATTR=y -CONFIG_EROFS_FS_ZIP=y -CONFIG_EROFS_FS_ZIP_ACCEL=y -CONFIG_EROFS_FS_ZIP_DEFLATE=y -CONFIG_EROFS_FS_ZIP_LZMA=y -CONFIG_EROFS_FS_ZIP_ZSTD=y -CONFIG_EXFAT_FS=m -CONFIG_F2FS_FS=m -CONFIG_F2FS_FS_COMPRESSION=y -CONFIG_F2FS_FS_LZ4=y -CONFIG_F2FS_FS_LZ4HC=y -CONFIG_F2FS_FS_LZO=y -CONFIG_F2FS_FS_LZORLE=y -CONFIG_F2FS_FS_POSIX_ACL=y -CONFIG_F2FS_FS_SECURITY=y -CONFIG_F2FS_FS_XATTR=y -CONFIG_F2FS_FS_ZSTD=y -CONFIG_F2FS_IOSTAT=y -CONFIG_F2FS_STAT_FS=y -CONFIG_F2FS_UNFAIR_RWSEM=y -CONFIG_HFSPLUS_FS=m -CONFIG_HFS_FS=m -CONFIG_ISO9660_FS=m -CONFIG_JOLIET=y -CONFIG_MSDOS_FS=m -CONFIG_NLS_ASCII=m -CONFIG_NLS_CODEPAGE_1250=m -CONFIG_NLS_CODEPAGE_1251=m -CONFIG_NLS_CODEPAGE_737=m -CONFIG_NLS_CODEPAGE_775=m -CONFIG_NLS_CODEPAGE_850=m -CONFIG_NLS_CODEPAGE_852=m -CONFIG_NLS_CODEPAGE_855=m -CONFIG_NLS_CODEPAGE_857=m -CONFIG_NLS_CODEPAGE_860=m -CONFIG_NLS_CODEPAGE_861=m -CONFIG_NLS_CODEPAGE_862=m -CONFIG_NLS_CODEPAGE_863=m -CONFIG_NLS_CODEPAGE_864=m -CONFIG_NLS_CODEPAGE_865=m -CONFIG_NLS_CODEPAGE_866=m -CONFIG_NLS_CODEPAGE_869=m -CONFIG_NLS_CODEPAGE_874=m -CONFIG_NLS_CODEPAGE_932=m -CONFIG_NLS_CODEPAGE_936=m -CONFIG_NLS_CODEPAGE_949=m -CONFIG_NLS_CODEPAGE_950=m -CONFIG_NLS_ISO8859_13=m -CONFIG_NLS_ISO8859_14=m -CONFIG_NLS_ISO8859_15=m -CONFIG_NLS_ISO8859_2=m -CONFIG_NLS_ISO8859_3=m -CONFIG_NLS_ISO8859_4=m -CONFIG_NLS_ISO8859_5=m -CONFIG_NLS_ISO8859_6=m -CONFIG_NLS_ISO8859_7=m -CONFIG_NLS_ISO8859_8=m -CONFIG_NLS_ISO8859_9=m -CONFIG_NLS_KOI8_R=m -CONFIG_NLS_KOI8_U=m -CONFIG_NLS_MAC_CELTIC=m -CONFIG_NLS_MAC_CENTEURO=m -CONFIG_NLS_MAC_CROATIAN=m -CONFIG_NLS_MAC_CYRILLIC=m -CONFIG_NLS_MAC_GAELIC=m -CONFIG_NLS_MAC_GREEK=m -CONFIG_NLS_MAC_ICELAND=m -CONFIG_NLS_MAC_INUIT=m -CONFIG_NLS_MAC_ROMAN=m -CONFIG_NLS_MAC_ROMANIAN=m -CONFIG_NLS_MAC_TURKISH=m -CONFIG_NLS_UTF8=m -CONFIG_NTFS3_FS=m -CONFIG_NTFS3_FS_POSIX_ACL=y -CONFIG_NTFS3_LZX_XPRESS=y -CONFIG_SQUASHFS_LZ4=y -CONFIG_SQUASHFS_LZO=y -CONFIG_SQUASHFS_XATTR=y -CONFIG_SQUASHFS_XZ=y -CONFIG_SQUASHFS_ZSTD=y -CONFIG_UDF_FS=m -CONFIG_XFS_FS=m -CONFIG_XFS_ONLINE_REPAIR=y -CONFIG_XFS_ONLINE_SCRUB=y -CONFIG_XFS_ONLINE_SCRUB_STATS=y -CONFIG_XFS_POSIX_ACL=y -CONFIG_XFS_QUOTA=y -CONFIG_XFS_RT=y -CONFIG_XFS_SUPPORT_ASCII_CI=y -CONFIG_XFS_SUPPORT_V4=y -CONFIG_ZISOFS=y - -# Network protocols: WireGuard/OpenVPN-DCO/L2TP/PPP/IPsec/kTLS, tunnels, policy routing, nftables+xtables+ipset+ebtables, tc, TCP CC, ss diag, virtual links -CONFIG_BONDING=m -CONFIG_BRIDGE_EBT_802_3=m -CONFIG_BRIDGE_EBT_AMONG=m -CONFIG_BRIDGE_EBT_ARP=m -CONFIG_BRIDGE_EBT_ARPREPLY=m -CONFIG_BRIDGE_EBT_BROUTE=m -CONFIG_BRIDGE_EBT_DNAT=m -CONFIG_BRIDGE_EBT_IP=m -CONFIG_BRIDGE_EBT_IP6=m -CONFIG_BRIDGE_EBT_LIMIT=m -CONFIG_BRIDGE_EBT_LOG=m -CONFIG_BRIDGE_EBT_MARK=m -CONFIG_BRIDGE_EBT_MARK_T=m -CONFIG_BRIDGE_EBT_NFLOG=m -CONFIG_BRIDGE_EBT_PKTTYPE=m -CONFIG_BRIDGE_EBT_REDIRECT=m -CONFIG_BRIDGE_EBT_SNAT=m -CONFIG_BRIDGE_EBT_STP=m -CONFIG_BRIDGE_EBT_T_FILTER=m -CONFIG_BRIDGE_EBT_T_NAT=m -CONFIG_BRIDGE_EBT_VLAN=m -CONFIG_BRIDGE_NF_EBTABLES=m -CONFIG_BRIDGE_NF_EBTABLES_LEGACY=m -CONFIG_CLS_U32_MARK=y -CONFIG_CLS_U32_PERF=y -CONFIG_DEFAULT_CUBIC=y -CONFIG_DUMMY=m -CONFIG_GACT_PROB=y -CONFIG_GENEVE=m -CONFIG_IFB=m -CONFIG_INET6_AH=m -CONFIG_INET6_ESP=m -CONFIG_INET6_ESPINTCP=y -CONFIG_INET6_ESP_OFFLOAD=m -CONFIG_INET6_IPCOMP=m -CONFIG_INET_AH=m -CONFIG_INET_DIAG_DESTROY=y -CONFIG_INET_ESP=m -CONFIG_INET_ESPINTCP=y -CONFIG_INET_ESP_OFFLOAD=m -CONFIG_INET_IPCOMP=m -CONFIG_INET_RAW_DIAG=m -CONFIG_INET_UDP_DIAG=m -CONFIG_IP6_NF_FILTER=m -CONFIG_IP6_NF_IPTABLES_LEGACY=m -CONFIG_IP6_NF_MANGLE=m -CONFIG_IP6_NF_MATCH_AH=m -CONFIG_IP6_NF_MATCH_EUI64=m -CONFIG_IP6_NF_MATCH_FRAG=m -CONFIG_IP6_NF_MATCH_HL=m -CONFIG_IP6_NF_MATCH_IPV6HEADER=m -CONFIG_IP6_NF_MATCH_MH=m -CONFIG_IP6_NF_MATCH_OPTS=m -CONFIG_IP6_NF_MATCH_RPFILTER=m -CONFIG_IP6_NF_MATCH_RT=m -CONFIG_IP6_NF_MATCH_SRH=m -CONFIG_IP6_NF_NAT=m -CONFIG_IP6_NF_RAW=m -CONFIG_IP6_NF_TARGET_HL=m -CONFIG_IP6_NF_TARGET_MASQUERADE=m -CONFIG_IP6_NF_TARGET_NPT=m -CONFIG_IP6_NF_TARGET_REJECT=m -CONFIG_IPV6_GRE=m -CONFIG_IPV6_MULTIPLE_TABLES=y -CONFIG_IPV6_TUNNEL=m -CONFIG_IPV6_VTI=m -CONFIG_IP_ADVANCED_ROUTER=y -CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IP_NF_ARPFILTER=m -CONFIG_IP_NF_ARPTABLES=m -CONFIG_IP_NF_ARP_MANGLE=m -CONFIG_IP_NF_FILTER=m -CONFIG_IP_NF_IPTABLES_LEGACY=m -CONFIG_IP_NF_MANGLE=m -CONFIG_IP_NF_MATCH_AH=m -CONFIG_IP_NF_MATCH_ECN=m -CONFIG_IP_NF_MATCH_RPFILTER=m -CONFIG_IP_NF_MATCH_TTL=m -CONFIG_IP_NF_NAT=m -CONFIG_IP_NF_RAW=m -CONFIG_IP_NF_TARGET_ECN=m -CONFIG_IP_NF_TARGET_MASQUERADE=m -CONFIG_IP_NF_TARGET_NETMAP=m -CONFIG_IP_NF_TARGET_REDIRECT=m -CONFIG_IP_NF_TARGET_REJECT=m -CONFIG_IP_NF_TARGET_TTL=m -CONFIG_IP_ROUTE_MULTIPATH=y -CONFIG_IP_SET=m -CONFIG_IP_SET_BITMAP_IP=m -CONFIG_IP_SET_BITMAP_IPMAC=m -CONFIG_IP_SET_BITMAP_PORT=m -CONFIG_IP_SET_HASH_IP=m -CONFIG_IP_SET_HASH_IPMAC=m -CONFIG_IP_SET_HASH_IPMARK=m -CONFIG_IP_SET_HASH_IPPORT=m -CONFIG_IP_SET_HASH_IPPORTIP=m -CONFIG_IP_SET_HASH_IPPORTNET=m -CONFIG_IP_SET_HASH_MAC=m -CONFIG_IP_SET_HASH_NET=m -CONFIG_IP_SET_HASH_NETIFACE=m -CONFIG_IP_SET_HASH_NETNET=m -CONFIG_IP_SET_HASH_NETPORT=m -CONFIG_IP_SET_HASH_NETPORTNET=m -CONFIG_IP_SET_LIST_SET=m -CONFIG_L2TP=m -CONFIG_L2TP_ETH=m -CONFIG_L2TP_IP=m -CONFIG_L2TP_V3=y -CONFIG_NETFILTER_NETLINK_ACCT=m -CONFIG_NETFILTER_NETLINK_GLUE_CT=y -CONFIG_NETFILTER_NETLINK_HOOK=m -CONFIG_NETFILTER_NETLINK_LOG=m -CONFIG_NETFILTER_NETLINK_OSF=m -CONFIG_NETFILTER_NETLINK_QUEUE=m -CONFIG_NETFILTER_XTABLES_LEGACY=y -CONFIG_NETFILTER_XT_CONNMARK=m -CONFIG_NETFILTER_XT_MATCH_BPF=m -CONFIG_NETFILTER_XT_MATCH_CGROUP=m -CONFIG_NETFILTER_XT_MATCH_CLUSTER=m -CONFIG_NETFILTER_XT_MATCH_COMMENT=m -CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m -CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m -CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m -CONFIG_NETFILTER_XT_MATCH_CONNMARK=m -CONFIG_NETFILTER_XT_MATCH_CPU=m -CONFIG_NETFILTER_XT_MATCH_DCCP=m -CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m -CONFIG_NETFILTER_XT_MATCH_DSCP=m -CONFIG_NETFILTER_XT_MATCH_ECN=m -CONFIG_NETFILTER_XT_MATCH_ESP=m -CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m -CONFIG_NETFILTER_XT_MATCH_HELPER=m -CONFIG_NETFILTER_XT_MATCH_HL=m -CONFIG_NETFILTER_XT_MATCH_IPCOMP=m -CONFIG_NETFILTER_XT_MATCH_IPRANGE=m -CONFIG_NETFILTER_XT_MATCH_L2TP=m -CONFIG_NETFILTER_XT_MATCH_LENGTH=m -CONFIG_NETFILTER_XT_MATCH_LIMIT=m -CONFIG_NETFILTER_XT_MATCH_MAC=m -CONFIG_NETFILTER_XT_MATCH_MARK=m -CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m -CONFIG_NETFILTER_XT_MATCH_NFACCT=m -CONFIG_NETFILTER_XT_MATCH_OSF=m -CONFIG_NETFILTER_XT_MATCH_OWNER=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m -CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m -CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_QUOTA=m -CONFIG_NETFILTER_XT_MATCH_RATEEST=m -CONFIG_NETFILTER_XT_MATCH_REALM=m -CONFIG_NETFILTER_XT_MATCH_RECENT=m -CONFIG_NETFILTER_XT_MATCH_SCTP=m -CONFIG_NETFILTER_XT_MATCH_SOCKET=m -CONFIG_NETFILTER_XT_MATCH_STATE=m -CONFIG_NETFILTER_XT_MATCH_STATISTIC=m -CONFIG_NETFILTER_XT_MATCH_STRING=m -CONFIG_NETFILTER_XT_MATCH_TCPMSS=m -CONFIG_NETFILTER_XT_MATCH_TIME=m -CONFIG_NETFILTER_XT_MATCH_U32=m -CONFIG_NETFILTER_XT_NAT=m -CONFIG_NETFILTER_XT_SET=m -CONFIG_NETFILTER_XT_TARGET_AUDIT=m -CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m -CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m -CONFIG_NETFILTER_XT_TARGET_CONNMARK=m -CONFIG_NETFILTER_XT_TARGET_CT=m -CONFIG_NETFILTER_XT_TARGET_DSCP=m -CONFIG_NETFILTER_XT_TARGET_HL=m -CONFIG_NETFILTER_XT_TARGET_HMARK=m -CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m -CONFIG_NETFILTER_XT_TARGET_LED=m -CONFIG_NETFILTER_XT_TARGET_MARK=m -CONFIG_NETFILTER_XT_TARGET_MASQUERADE=m -CONFIG_NETFILTER_XT_TARGET_NETMAP=m -CONFIG_NETFILTER_XT_TARGET_NFLOG=m -CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m -CONFIG_NETFILTER_XT_TARGET_NOTRACK=m -CONFIG_NETFILTER_XT_TARGET_RATEEST=m -CONFIG_NETFILTER_XT_TARGET_REDIRECT=m -CONFIG_NETFILTER_XT_TARGET_TCPMSS=m -CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m -CONFIG_NETFILTER_XT_TARGET_TEE=m -CONFIG_NETFILTER_XT_TARGET_TPROXY=m -CONFIG_NETFILTER_XT_TARGET_TRACE=m -CONFIG_NETLINK_DIAG=m -CONFIG_NET_ACT_BPF=m -CONFIG_NET_ACT_CONNMARK=m -CONFIG_NET_ACT_CSUM=m -CONFIG_NET_ACT_CT=m -CONFIG_NET_ACT_CTINFO=m -CONFIG_NET_ACT_MPLS=m -CONFIG_NET_ACT_NAT=m -CONFIG_NET_ACT_PEDIT=m -CONFIG_NET_ACT_POLICE=m -CONFIG_NET_ACT_SAMPLE=m -CONFIG_NET_ACT_SIMP=m -CONFIG_NET_ACT_SKBEDIT=m -CONFIG_NET_ACT_SKBMOD=m -CONFIG_NET_ACT_TUNNEL_KEY=m -CONFIG_NET_ACT_VLAN=m -CONFIG_NET_CLS_BPF=m -CONFIG_NET_CLS_FLOW=m -CONFIG_NET_CLS_FW=m -CONFIG_NET_CLS_MATCHALL=m -CONFIG_NET_CLS_ROUTE4=m -CONFIG_NET_CLS_U32=m -CONFIG_NET_EMATCH=y -CONFIG_NET_EMATCH_CMP=m -CONFIG_NET_EMATCH_IPSET=m -CONFIG_NET_EMATCH_IPT=m -CONFIG_NET_EMATCH_META=m -CONFIG_NET_EMATCH_NBYTE=m -CONFIG_NET_EMATCH_TEXT=m -CONFIG_NET_EMATCH_U32=m -CONFIG_NET_FOU=m -CONFIG_NET_FOU_IP_TUNNELS=y -CONFIG_NET_IPGRE=m -CONFIG_NET_IPGRE_BROADCAST=y -CONFIG_NET_IPGRE_DEMUX=m -CONFIG_NET_IPIP=m -CONFIG_NET_IPVTI=m -CONFIG_NET_KEY=m -CONFIG_NET_KEY_MIGRATE=y -CONFIG_NET_SCH_CAKE=m -CONFIG_NET_SCH_CHOKE=m -CONFIG_NET_SCH_CODEL=m -CONFIG_NET_SCH_DRR=m -CONFIG_NET_SCH_DUALPI2=m -CONFIG_NET_SCH_ETS=m -CONFIG_NET_SCH_FQ=m -CONFIG_NET_SCH_FQ_CODEL=m -CONFIG_NET_SCH_FQ_PIE=m -CONFIG_NET_SCH_GRED=m -CONFIG_NET_SCH_HFSC=m -CONFIG_NET_SCH_HHF=m -CONFIG_NET_SCH_HTB=m -CONFIG_NET_SCH_MULTIQ=m -CONFIG_NET_SCH_NETEM=m -CONFIG_NET_SCH_PIE=m -CONFIG_NET_SCH_PLUG=m -CONFIG_NET_SCH_PRIO=m -CONFIG_NET_SCH_QFQ=m -CONFIG_NET_SCH_RED=m -CONFIG_NET_SCH_SFB=m -CONFIG_NET_SCH_SFQ=m -CONFIG_NET_SCH_TBF=m -CONFIG_NET_SCH_TEQL=m -CONFIG_NET_TEAM=m -CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m -CONFIG_NET_TEAM_MODE_BROADCAST=m -CONFIG_NET_TEAM_MODE_LOADBALANCE=m -CONFIG_NET_TEAM_MODE_RANDOM=m -CONFIG_NET_TEAM_MODE_ROUNDROBIN=m -CONFIG_NFT_BRIDGE_META=m -CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NFT_COMPAT=m -CONFIG_NFT_CONNLIMIT=m -CONFIG_NFT_DUP_IPV4=m -CONFIG_NFT_DUP_IPV6=m -CONFIG_NFT_DUP_NETDEV=m -CONFIG_NFT_FIB_NETDEV=m -CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_FWD_NETDEV=m -CONFIG_NFT_HASH=m -CONFIG_NFT_NUMGEN=m -CONFIG_NFT_OSF=m -CONFIG_NFT_QUEUE=m -CONFIG_NFT_QUOTA=m -CONFIG_NFT_REDIR=m -CONFIG_NFT_REJECT_NETDEV=m -CONFIG_NFT_SOCKET=m -CONFIG_NFT_TPROXY=m -CONFIG_NFT_TUNNEL=m -CONFIG_NFT_XFRM=m -CONFIG_NF_CONNTRACK_AMANDA=m -CONFIG_NF_CONNTRACK_BRIDGE=m -CONFIG_NF_CONNTRACK_FTP=m -CONFIG_NF_CONNTRACK_H323=m -CONFIG_NF_CONNTRACK_IRC=m -CONFIG_NF_CONNTRACK_LABELS=y -CONFIG_NF_CONNTRACK_MARK=y -CONFIG_NF_CONNTRACK_NETBIOS_NS=m -CONFIG_NF_CONNTRACK_PPTP=m -CONFIG_NF_CONNTRACK_PROCFS=y -CONFIG_NF_CONNTRACK_SANE=m -CONFIG_NF_CONNTRACK_SIP=m -CONFIG_NF_CONNTRACK_SNMP=m -CONFIG_NF_CONNTRACK_TFTP=m -CONFIG_NF_CONNTRACK_TIMEOUT=y -CONFIG_NF_CONNTRACK_TIMESTAMP=y -CONFIG_NF_CONNTRACK_ZONES=y -CONFIG_NF_CT_NETLINK=m -CONFIG_NF_CT_NETLINK_HELPER=m -CONFIG_NF_CT_NETLINK_TIMEOUT=m -CONFIG_NF_DUP_IPV4=m -CONFIG_NF_DUP_IPV6=m -CONFIG_NF_DUP_NETDEV=m -CONFIG_NF_FLOW_TABLE=m -CONFIG_NF_FLOW_TABLE_INET=m -CONFIG_NF_FLOW_TABLE_PROCFS=y -CONFIG_NF_LOG_ARP=m -CONFIG_NF_LOG_IPV4=m -CONFIG_NF_NAT_SNMP_BASIC=m -CONFIG_NF_SOCKET_IPV4=m -CONFIG_NF_SOCKET_IPV6=m -CONFIG_NF_TABLES_ARP=y -CONFIG_NF_TABLES_BRIDGE=m -CONFIG_NF_TPROXY_IPV4=m -CONFIG_NF_TPROXY_IPV6=m -CONFIG_OVPN=m -CONFIG_PACKET_DIAG=m -CONFIG_PPP=m -CONFIG_PPPOE=m -CONFIG_PPPOE_HASH_BITS_4=y -CONFIG_PPPOL2TP=m -CONFIG_PPP_ASYNC=m -CONFIG_PPP_BSDCOMP=m -CONFIG_PPP_DEFLATE=m -CONFIG_PPP_FILTER=y -CONFIG_PPP_MPPE=m -CONFIG_PPP_MULTILINK=y -CONFIG_PPP_SYNC_TTY=m -CONFIG_PPTP=m -CONFIG_PSAMPLE=m -CONFIG_TCP_CONG_ADVANCED=y -CONFIG_TCP_CONG_BBR=m -CONFIG_TCP_CONG_BIC=m -CONFIG_TCP_CONG_CDG=m -CONFIG_TCP_CONG_DCTCP=m -CONFIG_TCP_CONG_HSTCP=m -CONFIG_TCP_CONG_HTCP=m -CONFIG_TCP_CONG_HYBLA=m -CONFIG_TCP_CONG_ILLINOIS=m -CONFIG_TCP_CONG_LP=m -CONFIG_TCP_CONG_NV=m -CONFIG_TCP_CONG_SCALABLE=m -CONFIG_TCP_CONG_VEGAS=m -CONFIG_TCP_CONG_VENO=m -CONFIG_TCP_CONG_WESTWOOD=m -CONFIG_TCP_CONG_YEAH=m -CONFIG_TLS=m -CONFIG_UNIX_DIAG=m -CONFIG_VXLAN=m -CONFIG_WIREGUARD=m -CONFIG_XFRM_INTERFACE=m -CONFIG_XFRM_MIGRATE=y -CONFIG_XFRM_STATISTICS=y -CONFIG_XFRM_USER=m - -# Bluetooth: USB dongle drivers, vhci, MSFT/AOSP vendor extensions (USB controllers only) -CONFIG_BT_AOSPEXT=y -CONFIG_BT_ATH3K=m -CONFIG_BT_HCIBCM203X=m -CONFIG_BT_HCIBFUSB=m -CONFIG_BT_HCIBPA10X=m -CONFIG_BT_HCIBTUSB_AUTOSUSPEND=y -CONFIG_BT_HCIVHCI=m -CONFIG_BT_MSFTEXT=y - -# Containers / virtualisation: CRIU/checkpoint, userfaultfd, misc cgroup, KVM host vhost-net/vhost-vsock/vsock -CONFIG_CGROUP_MISC=y -CONFIG_CHECKPOINT_RESTORE=y -CONFIG_USERFAULTFD=y -CONFIG_VHOST_NET=m -CONFIG_VHOST_VSOCK=m -CONFIG_VSOCKETS=m -CONFIG_VSOCKETS_DIAG=m -CONFIG_VSOCKETS_LOOPBACK=m - -# Crypto: AF_ALG user API (cryptsetup/iwd), dm-crypt/fscrypt modes, VeraCrypt/TrueCrypt ciphers+hashes, IPsec AEADs -CONFIG_CRYPTO_ADIANTUM=m -CONFIG_CRYPTO_AEGIS128=m -CONFIG_CRYPTO_AEGIS128_SIMD=y -CONFIG_CRYPTO_BLAKE2B=m -CONFIG_CRYPTO_BLOWFISH=m -CONFIG_CRYPTO_CAMELLIA=m -CONFIG_CRYPTO_CAST5=m -CONFIG_CRYPTO_CAST6=m -CONFIG_CRYPTO_CHACHA20POLY1305=m -CONFIG_CRYPTO_CRC32=m -CONFIG_CRYPTO_CRC32C=m -CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_CTS=m -CONFIG_CRYPTO_ESSIV=m -CONFIG_CRYPTO_HCTR2=m -CONFIG_CRYPTO_LRW=m -CONFIG_CRYPTO_LZ4HC=m -CONFIG_CRYPTO_MD4=m -CONFIG_CRYPTO_NULL=m -CONFIG_CRYPTO_PCBC=m -CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_SEQIV=m -CONFIG_CRYPTO_SERPENT=m -CONFIG_CRYPTO_STREEBOG=m -CONFIG_CRYPTO_TWOFISH=m -CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_USER_API_HASH=m -CONFIG_CRYPTO_USER_API_SKCIPHER=m -CONFIG_CRYPTO_WP512=m -CONFIG_CRYPTO_XCBC=m -CONFIG_CRYPTO_XXHASH=m - -# Sound: USB audio/MIDI device drivers, MIDI 2.0, virtual MIDI, dummy card, userspace timers -CONFIG_SND_BCD2000=m -CONFIG_SND_DUMMY=m -CONFIG_SND_SEQ_UMP=y -CONFIG_SND_UMP_LEGACY_RAWMIDI=y -CONFIG_SND_USB_6FIRE=m -CONFIG_SND_USB_AUDIO_MIDI_V2=y -CONFIG_SND_USB_CAIAQ=m -CONFIG_SND_USB_CAIAQ_INPUT=y -CONFIG_SND_USB_HIFACE=m -CONFIG_SND_USB_POD=m -CONFIG_SND_USB_PODHD=m -CONFIG_SND_USB_TONEPORT=m -CONFIG_SND_USB_UA101=m -CONFIG_SND_USB_VARIAX=m -CONFIG_SND_UTIMER=y -CONFIG_SND_VIRMIDI=m - -# Video: non-UVC USB webcams (gspca family, Philips pwc), USB displays (DisplayLink USB2 udl, GUD, GM12U320) -CONFIG_DRM_GM12U320=m -CONFIG_DRM_GUD=m -CONFIG_DRM_UDL=m -CONFIG_USB_GL860=m -CONFIG_USB_GSPCA=m -CONFIG_USB_GSPCA_BENQ=m -CONFIG_USB_GSPCA_CONEX=m -CONFIG_USB_GSPCA_CPIA1=m -CONFIG_USB_GSPCA_DTCS033=m -CONFIG_USB_GSPCA_ETOMS=m -CONFIG_USB_GSPCA_FINEPIX=m -CONFIG_USB_GSPCA_JEILINJ=m -CONFIG_USB_GSPCA_JL2005BCD=m -CONFIG_USB_GSPCA_KINECT=m -CONFIG_USB_GSPCA_KONICA=m -CONFIG_USB_GSPCA_MARS=m -CONFIG_USB_GSPCA_MR97310A=m -CONFIG_USB_GSPCA_NW80X=m -CONFIG_USB_GSPCA_OV519=m -CONFIG_USB_GSPCA_OV534=m -CONFIG_USB_GSPCA_OV534_9=m -CONFIG_USB_GSPCA_PAC207=m -CONFIG_USB_GSPCA_PAC7302=m -CONFIG_USB_GSPCA_PAC7311=m -CONFIG_USB_GSPCA_SE401=m -CONFIG_USB_GSPCA_SN9C2028=m -CONFIG_USB_GSPCA_SN9C20X=m -CONFIG_USB_GSPCA_SONIXB=m -CONFIG_USB_GSPCA_SONIXJ=m -CONFIG_USB_GSPCA_SPCA1528=m -CONFIG_USB_GSPCA_SPCA500=m -CONFIG_USB_GSPCA_SPCA501=m -CONFIG_USB_GSPCA_SPCA505=m -CONFIG_USB_GSPCA_SPCA506=m -CONFIG_USB_GSPCA_SPCA508=m -CONFIG_USB_GSPCA_SPCA561=m -CONFIG_USB_GSPCA_SQ905=m -CONFIG_USB_GSPCA_SQ905C=m -CONFIG_USB_GSPCA_SQ930X=m -CONFIG_USB_GSPCA_STK014=m -CONFIG_USB_GSPCA_STK1135=m -CONFIG_USB_GSPCA_STV0680=m -CONFIG_USB_GSPCA_SUNPLUS=m -CONFIG_USB_GSPCA_T613=m -CONFIG_USB_GSPCA_TOPRO=m -CONFIG_USB_GSPCA_TOUPTEK=m -CONFIG_USB_GSPCA_TV8532=m -CONFIG_USB_GSPCA_VC032X=m -CONFIG_USB_GSPCA_VICAM=m -CONFIG_USB_GSPCA_XIRLINK_CIT=m -CONFIG_USB_GSPCA_ZC3XX=m -CONFIG_USB_M5602=m -CONFIG_USB_PWC=m -CONFIG_USB_PWC_INPUT_EVDEV=y -CONFIG_USB_STV06XX=m - -# Other user-facing: Kerberos persistent keyrings, Speakup screen reader, Wine/Proton ntsync -CONFIG_ACCESSIBILITY=y -CONFIG_NTSYNC=m -CONFIG_PERSISTENT_KEYRINGS=y -CONFIG_SPEAKUP=m -CONFIG_SPEAKUP_SYNTH_ACNTSA=m -CONFIG_SPEAKUP_SYNTH_APOLLO=m -CONFIG_SPEAKUP_SYNTH_AUDPTR=m -CONFIG_SPEAKUP_SYNTH_BNS=m -CONFIG_SPEAKUP_SYNTH_DECTLK=m -CONFIG_SPEAKUP_SYNTH_LTLK=m -CONFIG_SPEAKUP_SYNTH_SOFT=m -CONFIG_SPEAKUP_SYNTH_SPKOUT=m -CONFIG_SPEAKUP_SYNTH_TXPRT=m diff --git a/arch/arm64/configs/j700-radio.config b/arch/arm64/configs/j700-radio.config deleted file mode 100644 index f5c5d3c3ec0bf..0000000000000 --- a/arch/arm64/configs/j700-radio.config +++ /dev/null @@ -1,20 +0,0 @@ -# J700 retained radio bootstrap; merge after defconfig and asahi.config. -# System sleep is retained; active radio firmware cannot cross kexec. -# Firmware/calibration and runtime admission follow the MT7932 documentation. -# CONFIG_KEXEC is not set -# CONFIG_KEXEC_FILE is not set -# CONFIG_KEXEC_HANDOVER is not set -CONFIG_PCI=y -CONFIG_PCIE_APPLE=y -CONFIG_APPLE_DART=y -CONFIG_PCIE_APPLE_PIODMA_DIAG=y -CONFIG_CFG80211=m -CONFIG_WLAN=y -CONFIG_WLAN_VENDOR_MEDIATEK=y -CONFIG_MT7932_FULLMAC=m -CONFIG_BT=m -CONFIG_BT_BREDR=y -CONFIG_BT_LE=y -CONFIG_BT_MTK7932_PCIE=m -CONFIG_CRYPTO_AES=y -CONFIG_CRYPTO_CMAC=y diff --git a/arch/arm64/configs/j700.config b/arch/arm64/configs/j700.config deleted file mode 100644 index 976fd65b7161c..0000000000000 --- a/arch/arm64/configs/j700.config +++ /dev/null @@ -1,113 +0,0 @@ -# J700 (MacBook Neo, T8140) base fragment. -# -# Merge after defconfig, asahi.config and j700-radio.config, then add the device-parity fragment: -# make defconfig -# scripts/kconfig/merge_config.sh -m .config arch/arm64/configs/asahi.config arch/arm64/configs/j700-radio.config -# make olddefconfig -# scripts/kconfig/merge_config.sh -m .config arch/arm64/configs/j700.config arch/arm64/configs/j700-device-parity.config -# make olddefconfig -# The result is the configuration the J700 kernel is tested with. - -# Boot NOR (nvram) access for boot-volume switching, and the PMU RTC wake alarm -CONFIG_SPI_APPLE_QSPI=m -CONFIG_RTC_DRV_APPLE_PMU=m - -# Classic Bluetooth profiles (headsets, tethering) -CONFIG_BT_RFCOMM=m -CONFIG_BT_RFCOMM_TTY=y -CONFIG_BT_BNEP=m -CONFIG_BT_BNEP_MC_FILTER=y -CONFIG_BT_BNEP_PROTO_FILTER=y - -# Distribution firmware files are compressed -CONFIG_FW_LOADER_COMPRESS=y -CONFIG_FW_LOADER_COMPRESS_ZSTD=y - -# 48-bit virtual addresses; KASLR is untested on this machine -CONFIG_ARM64_VA_BITS_48=y -# CONFIG_RANDOMIZE_BASE is not set - -# Boot path and platform built in, so storage, keyboard, SMC and USB do not depend on the initramfs -CONFIG_BLK_DEV_NVME=y -CONFIG_NVME_APPLE=y -CONFIG_APPLE_SART=y -CONFIG_APPLE_RTKIT_HELPER=y -CONFIG_BTRFS_FS=y -CONFIG_APPLE_DOCKCHANNEL=y -CONFIG_HID_DOCKCHANNEL=y -CONFIG_HID_APPLE=y -CONFIG_HID_MAGICMOUSE=y -CONFIG_SPMI_APPLE=y -CONFIG_NVMEM_APPLE_SPMI=y -CONFIG_MFD_MACSMC=y -CONFIG_GPIO_MACSMC=y -CONFIG_POWER_RESET_MACSMC=y -CONFIG_SENSORS_MACSMC_HWMON=y -CONFIG_I2C_APPLE=y -CONFIG_PINCTRL_APPLE_GPIO=y -CONFIG_APPLE_WATCHDOG=y -CONFIG_ARM_APPLE_SOC_CPUFREQ=y -CONFIG_PHY_APPLE_ATC=y -CONFIG_USB_DWC3_APPLE=y -CONFIG_TYPEC=y -CONFIG_TYPEC_SN201202X=y -CONFIG_MUX_APPLE_DPXBAR=y -CONFIG_BACKLIGHT_CLASS_DEVICE=y -CONFIG_SYSFB_SIMPLEFB=y -CONFIG_ZRAM=y -CONFIG_UHID=y -CONFIG_RFKILL=y -CONFIG_THERMAL_NETLINK=y -CONFIG_NVME_HWMON=y - -# J700 driver choices -CONFIG_APPLE_CLPC=m -CONFIG_APPLE_PMP=y -CONFIG_DRM_ACCEL=y -CONFIG_DRM_ACCEL_ANE=m -# CONFIG_HID_APPLE_MTP_HAPTIC is not set -CONFIG_IIO=m -# CONFIG_IIO_AOP_SENSOR_LAS is not set -# CONFIG_USB4 is not set - -# Per-process 4K user pages on the 16K kernel (x86 emulation) -CONFIG_ARM64_USER4K_EXPERIMENTAL=y - -# USB Ethernet built in, for a wired link before modules load -CONFIG_USB_USBNET=y -CONFIG_USB_NET_AX8817X=y -CONFIG_USB_NET_AX88179_178A=y -CONFIG_USB_NET_CDC_NCM=y -CONFIG_USB_RTL8152=y -CONFIG_USB_UAS=m - -# Crypto that Bluetooth and Wi-Fi need early -CONFIG_CRYPTO_CCM=y -CONFIG_CRYPTO_ECDH=y -CONFIG_CRYPTO_GCM=y -CONFIG_CRYPTO_HMAC=y -CONFIG_CRYPTO_SHA256=y - -# Utilisation clamping -CONFIG_UCLAMP_TASK=y -CONFIG_UCLAMP_TASK_GROUP=y - -# LSMs named in CONFIG_LSM -CONFIG_SECURITY_LANDLOCK=y -CONFIG_SECURITY_YAMA=y - -# dma-buf heaps and udmabuf (camera and GPU buffer sharing) -CONFIG_DMABUF_HEAPS=y -CONFIG_DMABUF_HEAPS_SYSTEM=y -CONFIG_UDMABUF=y - -# Debugging aids kept in the tested configuration -CONFIG_DYNAMIC_DEBUG=y -CONFIG_FTRACE=y -CONFIG_ENABLE_DEFAULT_TRACERS=y -CONFIG_PM_DEBUG=y -CONFIG_PM_ADVANCED_DEBUG=y - -# Match the tested kernel image: no relative relocation packing, no SCM version suffix -# CONFIG_RELR is not set -# CONFIG_LOCALVERSION_AUTO is not set From 0ad03dc7fb76a31cd5aa11d7fc695ba4c17d98ff Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 15:08:26 +1100 Subject: [PATCH 1284/1579] drm/asahi: keep G15 compatible with the shared G16 interfaces Building G15 and G16 together exposes stale initializers and calls in the G15 adapter. Initialise the inactive G16 platform owner and preserve the RTKit default protocol negotiation and legacy shmem callback. Resolve timestamp handles through Buffer::firmware_mapping(), retaining the firmware mapping and rejecting CPU-only destinations. Pass no reserved region map when walking G15 host-allocated RAM page tables, matching the same shared API contract already used by G16. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/driver.rs | 2 ++ drivers/gpu/drm/asahi/g15/boot.rs | 2 ++ drivers/gpu/drm/asahi/g15/command.rs | 1 + drivers/gpu/drm/asahi/g15/uat.rs | 2 +- 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index addabd04fe701..ab68573881659 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -346,6 +346,8 @@ impl AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, raw_dev: core::ptr::null_mut(), raw_stats: core::ptr::null_mut(), stats: None, diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index ddcf5efef4c3d..7a06aa64ffbc3 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -306,10 +306,12 @@ unsafe extern "C" fn rtkit_shmem_destroy( /// The GPU's RTKit callbacks. static RTKIT_OPS: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { flags: 0, + protocol_version: 0, crashed: Some(rtkit_crashed), recv_message: Some(rtkit_recv), recv_message_early: Some(rtkit_recv_early), shmem_setup: Some(rtkit_shmem_setup), + shmem_setup_ep: None, shmem_destroy: Some(rtkit_shmem_destroy), syslog_is_debug: None, }; diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index 45b65d1ddc026..6fc1e03fb637c 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -361,6 +361,7 @@ impl Timestamps<'_> { .ok_or(ENOENT)? .clone(); let file::Object::TimestampBuffer(mapping) = object; + let mapping = mapping.firmware_mapping()?; let offset = timestamp.offset as usize; if offset % 8 != 0 || offset >= mapping.size() || mapping.size() - offset < 8 { return Err(EINVAL); diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 854ff34e729a3..d6971489ee80f 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -96,7 +96,7 @@ fn with_table( // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on // unmap, and these VMs are retained until reboot). - unsafe { pgtable::with_root_entries(pa, count, f) } + unsafe { pgtable::with_root_entries(None, pa, count, f) } } /// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of From a2b3221b38cd211d4dc9e67487575e2897ad9e31 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 15:08:26 +1100 Subject: [PATCH 1285/1579] drm/asahi: keep G15 compatible with the shared G16 interfaces Building G15 and G16 together exposes stale initializers and calls in the G15 adapter. Initialise the inactive G16 platform owner and preserve the RTKit default protocol negotiation and legacy shmem callback. Resolve timestamp handles through Buffer::firmware_mapping(), retaining the firmware mapping and rejecting CPU-only destinations. Pass no reserved region map when walking G15 host-allocated RAM page tables, matching the same shared API contract already used by G16. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/driver.rs | 2 ++ drivers/gpu/drm/asahi/g15/boot.rs | 2 ++ drivers/gpu/drm/asahi/g15/command.rs | 1 + drivers/gpu/drm/asahi/g15/uat.rs | 2 +- 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index addabd04fe701..ab68573881659 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -346,6 +346,8 @@ impl AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, raw_dev: core::ptr::null_mut(), raw_stats: core::ptr::null_mut(), stats: None, diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index ddcf5efef4c3d..7a06aa64ffbc3 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -306,10 +306,12 @@ unsafe extern "C" fn rtkit_shmem_destroy( /// The GPU's RTKit callbacks. static RTKIT_OPS: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { flags: 0, + protocol_version: 0, crashed: Some(rtkit_crashed), recv_message: Some(rtkit_recv), recv_message_early: Some(rtkit_recv_early), shmem_setup: Some(rtkit_shmem_setup), + shmem_setup_ep: None, shmem_destroy: Some(rtkit_shmem_destroy), syslog_is_debug: None, }; diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index 45b65d1ddc026..6fc1e03fb637c 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -361,6 +361,7 @@ impl Timestamps<'_> { .ok_or(ENOENT)? .clone(); let file::Object::TimestampBuffer(mapping) = object; + let mapping = mapping.firmware_mapping()?; let offset = timestamp.offset as usize; if offset % 8 != 0 || offset >= mapping.size() || mapping.size() - offset < 8 { return Err(EINVAL); diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 854ff34e729a3..d6971489ee80f 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -96,7 +96,7 @@ fn with_table( // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on // unmap, and these VMs are retained until reboot). - unsafe { pgtable::with_root_entries(pa, count, f) } + unsafe { pgtable::with_root_entries(None, pa, count, f) } } /// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of From c9648e3322d53a3b546b8b2e037fd0cb50184b27 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 15:50:54 +1100 Subject: [PATCH 1286/1579] arm64: configs: drop helpers selected by the shared Apple drivers The shared drivers already select keys, AOP, the MAX98357A codec, Rust firmware loading, the Apple mailbox and C RTKit support. Removing these six requests cumulatively changes no resolved symbol over either ARM64 defconfig or the Fedora Asahi base, with WERROR enabled. Keep the drivers and their selections as the source of those dependencies. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 6 ------ 1 file changed, 6 deletions(-) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 590df15222c8b..59d8572309847 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -48,7 +48,6 @@ CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_MACAUDIO=m CONFIG_SND_SOC_CS42L83=m CONFIG_SND_SOC_CS42L84=m -CONFIG_SND_SOC_MAX98357A=m CONFIG_SND_SOC_TAS2764=m CONFIG_SND_SOC_TAS2770=m CONFIG_HID_APPLE=m @@ -68,12 +67,8 @@ CONFIG_APPLE_DART_J713_DCP=y CONFIG_APPLE_DOCKCHANNEL=m CONFIG_SERIAL_APPLE_DOCKCHANNEL=y CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y -CONFIG_APPLE_MAILBOX=y CONFIG_APPLE_PMGR_MISC=y -CONFIG_APPLE_RTKIT=y CONFIG_APPLE_RTKIT_HELPER=m -CONFIG_APPLE_AOP=m -CONFIG_KEYS=y CONFIG_TRUSTED_KEYS=y # CONFIG_TRUSTED_KEYS_TPM is not set # CONFIG_TRUSTED_KEYS_TEE is not set @@ -84,7 +79,6 @@ CONFIG_APPLE_PMP_THERMAL=m CONFIG_APPLE_T8132_PLATFORM_HANDOFF=y CONFIG_IIO_AOP_SENSOR_LAS=m CONFIG_IIO_AOP_SENSOR_ALS=m -CONFIG_RUST_FW_LOADER_ABSTRACTIONS=y CONFIG_PHY_APPLE_ATC=m CONFIG_PHY_APPLE_T6040_USB2=m CONFIG_PHY_APPLE_DPTX=m From 22e984f1e1b166b7ef2a45a3442f53d70f414bfc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 17:49:55 +1100 Subject: [PATCH 1287/1579] selftests: apple-dcp: keep WIP host fixtures portable Carry the existing integration errno and GNU make selection fixes onto the WIP consistency port. Extracted PM helpers use the host errno header, the synthetic EREMOTEIO status has a portable definition, and Kconfig producer checks honour MAKE or find gmake. The initial WIP Darwin run exposed these fixture-only failures; no driver or test timeout changes. Based on the public integration/aurora fixes at 9cbdd0734d9e. Assisted-by: LLM --- tools/testing/selftests/drivers/gpu/apple-dcp/common.py | 2 ++ tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py | 4 ++++ tools/testing/selftests/drivers/gpu/apple-dcp/pm.py | 5 +++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py index cdf20256dfa4f..918a9d91fa30f 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -47,6 +47,8 @@ def source_defines(source): def tipd_pm_header(root): """CD321x PM state machine for host units, with a host clamp() for minmax.h.""" header = (root / 'drivers/usb/typec/tipd/cd321x-pm.h').read_text() + # Use the host errno namespace, not the AFK Linux-numbered shim. + header = header.replace('#include ', '#include ') # Host integer types; a repeated identical typedef is valid C11. header = header.replace('#include ', '#include \ntypedef uint32_t u32;') diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index d7485a6681747..21a962fb10cb3 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -1770,6 +1770,10 @@ def power_reply(root): #include #include #include +/* Darwin has no EREMOTEIO; this synthetic status is never passed to libc. */ +#ifndef EREMOTEIO +#define EREMOTEIO 121 +#endif typedef uint32_t u32; typedef uint64_t u64; #define WRITE_ONCE(p,v) ((p)=(v)) #define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index af3a590b375a4..664ba046dad35 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -2,7 +2,7 @@ # Copyright 2026 Ryan Murray """Actual PM/HCD/mux/probe boundaries using the shared PHY extraction fixture.""" from pathlib import Path -import os, re, subprocess +import os, re, shlex, shutil, subprocess import phy from common import block, tipd_pm_header, atc_function @@ -681,7 +681,8 @@ def kconfig(root, output): 'config PHY_APPLE_ATC\n\ttristate "PHY"\n\t' + dependency + '\n') compiler = Path(os.environ.get('KCONFIG_CONF', root / 'scripts/kconfig/conf')) if not compiler.is_file(): - subprocess.run(['make', '-C', str(root), 'O=' + str(directory / 'producer'), + make = shlex.split(os.environ.get('MAKE', shutil.which('gmake') or 'make')) + subprocess.run(make + ['-C', str(root), 'O=' + str(directory / 'producer'), 'ARCH=arm64', 'LLVM=1', 'allnoconfig'], check=True) compiler = directory / 'producer/scripts/kconfig/conf' for usb, phy_setting in [('n', 'y'), ('y', 'y'), ('m', 'y'), ('m', 'm')]: From c01ad1ba4f1a558fe281dd123423032d7aa54172 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 17:23:00 +1100 Subject: [PATCH 1288/1579] drm/asahi: g17: build test-only helpers only with built-in KUnit Four G17 helpers exist only for the KUnit suites: the compute QID floor, the growth gate and admission pool constructors and the runtime registry constructor. They are gated on cfg(CONFIG_KUNIT), which is also set when KUnit is a module. #[kunit_tests] compiles its suites only for CONFIG_KUNIT=y, so with CONFIG_KUNIT=m (the Fedora-based release configuration) nothing uses the helpers and rustc warns about dead code: warning: constant `COMPUTE_FLOOR` is never used warning: associated function `new` is never used (x3) With CONFIG_WERROR=y, KBUILD_RUSTFLAGS gains -Dwarnings and the asahi module no longer builds. Gate the helpers on CONFIG_KUNIT="y", the condition the suites use. Every caller is inside a #[kunit_tests] module, so built-in KUnit runs are unchanged. Fixes: a36e352d8564 ("drm/asahi: size the G17 compute QID budget by open logical queues") Fixes: 014605a36795 ("drm/asahi: add G17 resource wait domains and use them for queue creation") Fixes: 64f717dcfd1e ("drm/asahi: wait for G17 compute capacity on the release that frees it") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g17/kick.rs | 2 +- drivers/gpu/drm/asahi/g17/preparation.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/admission.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index a1f8f70f597d4..4adf2c01ac4d5 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -43,7 +43,7 @@ const RENDER_HEADROOM: usize = 24; /// The render reserve with every execution context open. const FULL_RENDER_RESERVE: usize = render_reserve(crate::mmu::MAX_EXECUTION_CONTEXTS as usize); /// Compute channels that stay reachable with every execution context open. -#[cfg(CONFIG_KUNIT)] +#[cfg(CONFIG_KUNIT = "y")] pub(super) const COMPUTE_FLOOR: usize = QID_COUNT - FULL_RENDER_RESERVE; /// Upper bound on distinct physical compute channels named by one kick. pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT; diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index eb2366fa3451b..2aa3213115076 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -60,7 +60,7 @@ pub(super) struct Gate { } impl Gate { - #[cfg(CONFIG_KUNIT)] + #[cfg(CONFIG_KUNIT = "y")] pub(super) fn new() -> Result> { Self::with_waiters(None) } diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index e1e7552c243f2..5d770ca1f64b0 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -406,7 +406,7 @@ pub(crate) struct Pool { renders: Option>, } impl Pool { - #[cfg(CONFIG_KUNIT)] + #[cfg(CONFIG_KUNIT = "y")] pub(crate) fn new(max_queues: u32) -> Result> { Self::with_waiters(max_queues, None, None) } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index e04295d576606..9fe17db8a2809 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -186,7 +186,7 @@ pub(super) struct Registry { impl Registry { /// Initialize inside the firmware allocation instead of staging the whole /// registry on the probe stack. No additional allocation is required. - #[cfg(CONFIG_KUNIT)] + #[cfg(CONFIG_KUNIT = "y")] pub(super) fn new( clusters: u32, deflake: crate::g17::fw::render::Deflake, From d9e7c8dbf9a28e34c61f9f7faf509d75d2adf9fe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 18:09:02 +1100 Subject: [PATCH 1289/1579] arm64: configs: inherit the energy model from both bases The universal and Fedora bases already enable ENERGY_MODEL. Drop the redundant fragment request, retaining the power allocator governor and default. Both complete integration resolutions have zero delta from this removal. Assisted-by: LLM --- arch/arm64/configs/asahi.config | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 59d8572309847..1ca03ba6bdbdc 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -8,7 +8,6 @@ CONFIG_ARM64_VA_BITS_48=y CONFIG_ARM64_USER4K_EXPERIMENTAL=y CONFIG_ARM64_MEMORY_MODEL_CONTROL=y CONFIG_ARM_APPLE_CPUIDLE=y -CONFIG_ENERGY_MODEL=y CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y CONFIG_BT_BRCMEXT=y From e7843a83510dfb1c3c7fd0a3462379c98decc7c5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 18:19:49 +1100 Subject: [PATCH 1290/1579] rust: device: drop the stale private-parent lint expectation Device::parent is a public API in the carried Asahi abstractions, and AOP service lookup uses it with AUXILIARY_BUS disabled. The inherited expectation described the upstream crate-private helper and becomes unfulfilled in the built-in Asahi/firmware KUnit recipe. WERROR correctly rejects it. Remove the obsolete dead_code expectation from the public method. Keep warning enforcement and the minimal test config; do not enable an unrelated bus to hide the warning. Production behavior is unchanged. Assisted-by: LLM --- rust/kernel/device.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index e54bf419c7a66..f047b90e7b04a 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -268,7 +268,6 @@ impl Device { } /// Returns a reference to the parent device, if any. - #[cfg_attr(not(CONFIG_AUXILIARY_BUS), expect(dead_code))] pub fn parent(&self) -> Option<&Device> { // SAFETY: // - By the type invariant `self.as_raw()` is always valid. From 27060370b2013a1f673cbf9aabd61d5934809c1f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 18:19:49 +1100 Subject: [PATCH 1291/1579] rust: device: drop the stale private-parent lint expectation Device::parent is a public API in the carried Asahi abstractions, and AOP service lookup uses it with AUXILIARY_BUS disabled. The inherited expectation described the upstream crate-private helper and becomes unfulfilled in the built-in Asahi/firmware KUnit recipe. WERROR correctly rejects it. Remove the obsolete dead_code expectation from the public method. Keep warning enforcement and the minimal test config; do not enable an unrelated bus to hide the warning. Production behavior is unchanged. Assisted-by: LLM (cherry picked from commit e7843a83510dfb1c3c7fd0a3462379c98decc7c5) --- rust/kernel/device.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index e54bf419c7a66..f047b90e7b04a 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -268,7 +268,6 @@ impl Device { } /// Returns a reference to the parent device, if any. - #[cfg_attr(not(CONFIG_AUXILIARY_BUS), expect(dead_code))] pub fn parent(&self) -> Option<&Device> { // SAFETY: // - By the type invariant `self.as_raw()` is always valid. From 7b563df1ba87e6f22d54be1186dce7024eab5934 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 20:06:25 +1100 Subject: [PATCH 1292/1579] rust: require checked binding layouts for every kernel build With SCHEDSTATS disabled, stock bindgen loses the alignment of the empty sched_statistics member and shifts task_struct.mm. A current-MM read can then pass a scheduler-list pointer to the RSS helpers. Keep unconditional size, alignment and offset checks in production builds and document the corrected generator and command-line BINDGEN selection. --- Documentation/rust/bindgen-layout-checks.rst | 57 ++++++++++---------- rust/Makefile | 15 ++---- tools/aurora-bindgen/README.rst | 20 +++---- tools/aurora-bindgen/build.py | 2 +- 4 files changed, 43 insertions(+), 51 deletions(-) diff --git a/Documentation/rust/bindgen-layout-checks.rst b/Documentation/rust/bindgen-layout-checks.rst index aaac5b24fb742..5af9951a64338 100644 --- a/Documentation/rust/bindgen-layout-checks.rst +++ b/Documentation/rust/bindgen-layout-checks.rst @@ -3,22 +3,21 @@ Checking generated binding layouts ================================== -bindgen can emit compile-time assertions that compare the size, alignment and -field offsets of every generated Rust type with the values Clang computes for -the C type. As upstream, the kernel build passes ``--no-layout-tests`` to -bindgen, so the generated bindings do not contain these assertions and any -supported bindgen version can build the kernel. - -To check the layouts of a configuration, build with ``BINDGEN_LAYOUT_TESTS=1``:: - - make LLVM=1 BINDGEN_LAYOUT_TESTS=1 prepare - -The bindings are then generated with the assertions, which are checked while -the ``bindings`` and ``uapi`` crates are compiled; nothing is executed. A -mismatch stops the build with an ``E0080`` error naming the type and the -property (``Size of``, ``Alignment of`` or ``Offset of``). Changing the -variable regenerates the bindings. The check covers only the configuration -being built. +Every Rust build retains bindgen's unconditional compile-time assertions for +size, alignment and field offsets. A mismatch stops the bindings crate build +with an ``E0080`` error naming the C type and property. These assertions are +required even when KUnit is disabled; ``BINDGEN_LAYOUT_TESTS=0`` cannot disable +them. Build the complete Image and all modules with the same generator. + +A successful C build does not validate Rust field offsets. In particular, +``CONFIG_SCHEDSTATS=n`` leaves a zero-sized, cacheline-aligned +``sched_statistics`` in ``task_struct``. Stock bindgen 0.73.2 loses its alignment +and can shift subsequent fields, including ``mm``. The ARM64 defconfig + +Asahi configuration exposed a 40-byte difference: Rust read a scheduler list +pointer as the current mm and Asahi's RSS charge faulted in +``percpu_counter_add_batch``. The C RSS helpers and allocator agreed on the +counter layout; the pointer supplied to them was wrong. Enabling SCHEDSTATS +only masks the generator defect and is not a requirement of the driver. Known generator limitations --------------------------- @@ -37,16 +36,11 @@ Unmodified bindgen 0.73 does not reproduce every C layout used by the kernel: * It can drop the explicit alignment of a defined zero-sized C struct, such as ``struct empty {} __attribute__((aligned(64)));``. -With the assertions enabled and stock bindgen 0.73.2, an arm64 Apple -silicon configuration of this tree fails on the size of ``ns_common`` and of -the namespace structures that embed it; bindgen 0.73.1 also fails on -``task_struct``, ``zone`` and ``pglist_data``. In those builds every offset -and alignment assertion passes: bindgen pads each field to the offset Clang -reports, so only the end of the Rust type is short. Rust code that reaches -such a type through a pointer and reads its fields gets the right data, but -must not rely on ``size_of`` for it, copy it by value or index an array of -it. The affected types depend on the bindgen version and the configuration, -which is what the check is for. +The earlier Fedora configuration exposed only trailing size differences for +some namespace types. That observation does not extend to defconfig: an +aligned empty member can also shift field offsets. Neither size nor offset +mismatches are safe to ignore. The affected types depend on the generator +version and the resolved kernel configuration, so validate each build. Corrected generator ------------------- @@ -56,7 +50,7 @@ repair both cases, a private build recipe and compile-only tests; see ``tools/aurora-bindgen/README.rst``. It installs nothing system-wide. To check the layouts with it:: - make LLVM=1 BINDGEN=/path/to/bindgen BINDGEN_LAYOUT_TESTS=1 ... + make LLVM=1 BINDGEN=/path/to/bindgen ... The generated bindings are consumed by the Rust kernel crate and by several drivers. After changing the bindings generator, rebuild the kernel Image and @@ -64,5 +58,10 @@ all modules together. The assertions need bindgen 0.71.1 or later (the kernel minimum) together with the configured ``--rust-target 1.85``: that selects unconditional constant -assertions rather than ``#[test]`` functions, so a build with -``BINDGEN_LAYOUT_TESTS=1`` checks them without a Rust test configuration. +assertions rather than ``#[test]`` functions. They are checked without a +Rust test configuration. + +Pass ``BINDGEN=/absolute/path/to/bindgen`` as a make command-line argument. +Exporting it in the environment is insufficient: the top-level Makefile's +``BINDGEN = bindgen`` assignment takes precedence over an environment value. +Check the recorded bindings command, not just the toolchain version banner. diff --git a/rust/Makefile b/rust/Makefile index f9b22e936fa40..b5ba0c3176bce 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -513,23 +513,16 @@ endif # architecture instead of generating `usize`. bindgen_c_flags_final = $(bindgen_c_flags_lto) -fno-builtin -D__BINDGEN__ -# Layout assertions are off by default, as upstream: stock bindgen 0.73 -# computes a different total size for a few kernel types, so the bindings -# would not build with it. `make BINDGEN_LAYOUT_TESTS=1` keeps them, for -# checking a bindings generator; see -# Documentation/rust/bindgen-layout-checks.rst. -ifeq ($(BINDGEN_LAYOUT_TESTS),1) -bindgen_layout_flags := -else -bindgen_layout_flags := --no-layout-tests -endif +# Keep bindgen's unconditional layout assertions in every build. An empty, +# cacheline-aligned sched_statistics (CONFIG_SCHEDSTATS=n) exposes a generator +# bug that shifts task_struct.mm, so even pointer field access can be unsafe. +# See Documentation/rust/bindgen-layout-checks.rst for the corrected generator. # `--rust-target` points to our minimum supported Rust version. quiet_cmd_bindgen = BINDGEN $@ cmd_bindgen = \ $(BINDGEN) $< $(bindgen_target_flags) --rust-target 1.85 \ --use-core --with-derive-default --ctypes-prefix ffi \ - $(bindgen_layout_flags) \ --no-debug '.*' --enable-function-attribute-detection \ -o $@ -- $(bindgen_c_flags_final) -DMODULE \ $(bindgen_target_cflags) $(bindgen_target_extra) diff --git a/tools/aurora-bindgen/README.rst b/tools/aurora-bindgen/README.rst index 75d4c3a2480e7..35e4b6eb8a274 100644 --- a/tools/aurora-bindgen/README.rst +++ b/tools/aurora-bindgen/README.rst @@ -25,10 +25,10 @@ locked dependency graph is used, with at most two build jobs. ``--archive`` can supply an already downloaded source archive; its digest is still checked. The script builds and runs compile-only valid-input checks, then prints the exact ``BINDGEN=/absolute/path/to/bindgen`` make argument. It installs nothing globally. -The kernel build passes ``--no-layout-tests`` to bindgen unless -``BINDGEN_LAYOUT_TESTS=1`` is given, so add that variable to check layouts -with the private tool, and build the complete kernel Image and matching -modules with the same tool. +The kernel always retains layout assertions. Pass the printed ``BINDGEN=`` +argument on the make command line; an exported environment variable is +overridden by the top-level Makefile. Build the complete kernel Image and +matching modules with the same tool. Both scripts reject optimized Python (``-O``, ``-OO`` or ``PYTHONOPTIMIZE``) before creating output, since their validation assertions must remain enabled. @@ -70,14 +70,14 @@ generator, and its changes and both tool digests belong in the build record. Repository guard and supported tool versions -------------------------------------------- -By default the kernel passes ``--no-layout-tests``, as upstream, so stock -bindgen builds it; ``make BINDGEN_LAYOUT_TESTS=1`` keeps the layout checks -(see ``Documentation/rust/bindgen-layout-checks.rst``). Its minimum bindgen is +Every kernel build keeps the layout checks (see +``Documentation/rust/bindgen-layout-checks.rst``). Stock bindgen can fail these +checks for valid configurations; use the corrected generator rather than +disabling the assertions or changing configuration to hide a layout bug. Its minimum bindgen is 0.71.1, and its fixed ``--rust-target 1.85`` enables the ``offset_of`` feature (available from Rust 1.77). In both official 0.71.1 and the pinned 0.73.2 generator, that feature selects unconditional const size/alignment/offset -assertions rather than ``#[test]`` functions. A bindings crate compile with -``BINDGEN_LAYOUT_TESTS=1`` therefore checks layouts without requiring a Rust +assertions rather than ``#[test]`` functions. A bindings crate compile therefore checks layouts without requiring a Rust unit-test configuration. This is source-backed for the accepted minimum and pinned tool; future generator changes must retain that capability. @@ -87,6 +87,6 @@ Primary sources: * https://github.com/rust-lang/rust-bindgen/blob/v0.71.1/bindgen/codegen/mod.rs * https://github.com/rust-lang/rust-bindgen/tree/v0.73.2 -A complete kernel build with ``BINDGEN_LAYOUT_TESTS=1`` remains the +A complete kernel build remains the configuration-specific layout check; these small generator fixtures do not replace it or establish runtime GPU stability. diff --git a/tools/aurora-bindgen/build.py b/tools/aurora-bindgen/build.py index 7aa222cc028bb..fa87873098480 100644 --- a/tools/aurora-bindgen/build.py +++ b/tools/aurora-bindgen/build.py @@ -22,4 +22,4 @@ cmd=['cargo','build','--locked','--release','-p','bindgen-cli','-j'+str(a.jobs)] with (O/'build.log').open('w') as log:subprocess.run(cmd,cwd=S,stdout=log,stderr=subprocess.STDOUT,check=True) binary=S/'target/release/bindgen';subprocess.run([sys.executable,str(P/'test.py'),'--bindgen',str(binary),'--output-dir',str(O/'tests')],check=True) -r={'status':'BUILT_AND_COMPILE_TESTED','binary':str(binary),'binary_sha256':sha(binary),'source_manifest_sha256':sha(P/'sources.json'),'build_command':cmd,'rustc_version':subprocess.check_output(['rustc','-vV'],text=True),'cargo_version':subprocess.check_output(['cargo','-vV'],text=True),'tests_manifest_sha256':sha(O/'tests/result.json'),'installed_globally':False};(O/'result.json').write_text(json.dumps(r,indent=2)+'\n');print(json.dumps(r,indent=2));print('Use make BINDGEN='+str(binary)+' BINDGEN_LAYOUT_TESTS=1 ... to build the kernel with layout assertions.') +r={'status':'BUILT_AND_COMPILE_TESTED','binary':str(binary),'binary_sha256':sha(binary),'source_manifest_sha256':sha(P/'sources.json'),'build_command':cmd,'rustc_version':subprocess.check_output(['rustc','-vV'],text=True),'cargo_version':subprocess.check_output(['cargo','-vV'],text=True),'tests_manifest_sha256':sha(O/'tests/result.json'),'installed_globally':False};(O/'result.json').write_text(json.dumps(r,indent=2)+'\n');print(json.dumps(r,indent=2));print('Use make BINDGEN='+str(binary)+' ... to build the kernel with layout assertions.') From 551e4e5d6dc51116269ff925c4bc41c8f9c3c092 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 20:06:26 +1100 Subject: [PATCH 1293/1579] arm64: restore common distro and peripheral requirements in asahi.config Keep defconfig plus the single shared fragment while restoring SELinux and its LSM policy order, nftables and NAT, container and systemd requirements, filesystems, hardening, desktop compatibility and attached USB peripherals. Do not enable scheduler statistics to hide an invalid Rust binding layout. --- arch/arm64/configs/asahi.config | 1265 ++++++++++++++++++++++++++++++- 1 file changed, 1262 insertions(+), 3 deletions(-) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index d07b70afd9294..60d8a679594ef 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -1,7 +1,7 @@ -# Shared Apple Silicon hardware defaults for M1 and later, including Neo. -# Merge over ARM64 defconfig or a distribution base config, then run +# Shared Apple Silicon hardware and distribution defaults for M1 and later. +# Merge over ARM64 defconfig, then run # olddefconfig and verify that the requested settings survive resolution. -# Device-specific boot, debug and distribution policy stays in the base. +# One fragment serves every device; distribution policy is selected at runtime. CONFIG_RUST=y CONFIG_ARM64_16K_PAGES=y CONFIG_ARM64_VA_BITS_48=y @@ -98,3 +98,1262 @@ CONFIG_RDMA_RXE=m CONFIG_RDMA_SIW=m CONFIG_DMABUF_HEAPS_SYSTEM=y CONFIG_UDMABUF=y + +# Distribution requirements for the universal defconfig recipe. + +# Security policy, audit and labelled filesystems for the supported distributions. +CONFIG_AUDIT=y +CONFIG_AUDITSYSCALL=y +CONFIG_DEFAULT_SECURITY_SELINUX=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_KEYS=y +CONFIG_KEYS_REQUEST_CACHE=y +CONFIG_LOAD_UEFI_KEYS=y +CONFIG_LSM="lockdown,yama,integrity,selinux,bpf,landlock,ipe" +CONFIG_SECONDARY_TRUSTED_KEYRING=y +CONFIG_SECURITY=y +CONFIG_SECURITYFS=y +CONFIG_SECURITY_DMESG_RESTRICT=y +CONFIG_SECURITY_INFINIBAND=y +CONFIG_SECURITY_IPE=y +CONFIG_SECURITY_LANDLOCK=y +CONFIG_SECURITY_LANDLOCK_LOG=y +CONFIG_SECURITY_LOCKDOWN_LSM=y +CONFIG_SECURITY_LOCKDOWN_LSM_EARLY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_NETWORK_XFRM=y +CONFIG_SECURITY_PATH=y +CONFIG_SECURITY_SELINUX=y +CONFIG_SECURITY_SELINUX_AVC_STATS=y +CONFIG_SECURITY_SELINUX_BOOTPARAM=y +CONFIG_SECURITY_SELINUX_DEVELOP=y +CONFIG_SECURITY_YAMA=y +CONFIG_SYSTEM_BLACKLIST_KEYRING=y +CONFIG_SYSTEM_TRUSTED_KEYRING=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_TMPFS_XATTR=y +CONFIG_XFRM=y +CONFIG_XFRM_USER=y + +# Integrity appraisal and verified storage; distribution policy controls enforcement. +CONFIG_EVM=y +CONFIG_EVM_ATTR_FSUUID=y +CONFIG_FS_ENCRYPTION=y +CONFIG_FS_ENCRYPTION_ALGS=y +CONFIG_FS_ENCRYPTION_INLINE_CRYPT=y +CONFIG_FS_VERITY=y +CONFIG_IMA=y +CONFIG_IMA_APPRAISE=y +CONFIG_IMA_APPRAISE_BOOTPARAM=y +CONFIG_IMA_APPRAISE_MODSIG=y +CONFIG_IMA_ARCH_POLICY=y +CONFIG_IMA_DEFAULT_HASH="sha256" +CONFIG_IMA_DEFAULT_HASH_SHA256=y +CONFIG_IMA_DEFAULT_TEMPLATE="ima-ng" +CONFIG_IMA_KEXEC=y +CONFIG_IMA_KEXEC_EXTRA_MEMORY_KB=0 +CONFIG_IMA_KEYRINGS_PERMIT_SIGNED_BY_BUILTIN_OR_SECONDARY=y +CONFIG_IMA_LSM_RULES=y +CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS=y +CONFIG_IMA_MEASURE_PCR_IDX=10 +CONFIG_IMA_NG_TEMPLATE=y +CONFIG_IMA_QUEUE_EARLY_BOOT_KEYS=y +CONFIG_IMA_READ_POLICY=y +CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT=y +CONFIG_IMA_WRITE_POLICY=y +CONFIG_INTEGRITY=y +CONFIG_INTEGRITY_ASYMMETRIC_KEYS=y +CONFIG_INTEGRITY_AUDIT=y +CONFIG_INTEGRITY_CA_MACHINE_KEYRING=y +CONFIG_INTEGRITY_CA_MACHINE_KEYRING_MAX=y +CONFIG_INTEGRITY_MACHINE_KEYRING=y +CONFIG_INTEGRITY_PLATFORM_KEYRING=y +CONFIG_INTEGRITY_SIGNATURE=y +CONFIG_INTEGRITY_TRUSTED_KEYRING=y + +# Firewalld nftables, IPv4/IPv6/bridge filtering, NAT, sets and iptables compatibility. +CONFIG_BRIDGE=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_IP6=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_NETFILTER=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_NF_EBTABLES_LEGACY=m +CONFIG_IP6_NF_FILTER=m +CONFIG_IP6_NF_IPTABLES=m +CONFIG_IP6_NF_IPTABLES_LEGACY=m +CONFIG_IP6_NF_MANGLE=m +CONFIG_IP6_NF_MATCH_AH=m +CONFIG_IP6_NF_MATCH_EUI64=m +CONFIG_IP6_NF_MATCH_FRAG=m +CONFIG_IP6_NF_MATCH_HL=m +CONFIG_IP6_NF_MATCH_IPV6HEADER=m +CONFIG_IP6_NF_MATCH_MH=m +CONFIG_IP6_NF_MATCH_OPTS=m +CONFIG_IP6_NF_MATCH_RPFILTER=m +CONFIG_IP6_NF_MATCH_RT=m +CONFIG_IP6_NF_MATCH_SRH=m +CONFIG_IP6_NF_NAT=m +CONFIG_IP6_NF_RAW=m +CONFIG_IP6_NF_SECURITY=m +CONFIG_IP6_NF_TARGET_HL=m +CONFIG_IP6_NF_TARGET_MASQUERADE=m +CONFIG_IP6_NF_TARGET_NPT=m +CONFIG_IP6_NF_TARGET_REJECT=m +CONFIG_IP6_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_IPTABLES_LEGACY=m +CONFIG_IP_NF_MANGLE=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_SECURITY=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMAC=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_IP_SET_MAX=256 +CONFIG_NETFILTER=y +CONFIG_NETFILTER_ADVANCED=y +CONFIG_NETFILTER_BPF_LINK=y +CONFIG_NETFILTER_CONNCOUNT=m +CONFIG_NETFILTER_EGRESS=y +CONFIG_NETFILTER_FAMILY_ARP=y +CONFIG_NETFILTER_FAMILY_BRIDGE=y +CONFIG_NETFILTER_INGRESS=y +CONFIG_NETFILTER_NETLINK=y +CONFIG_NETFILTER_NETLINK_ACCT=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NETFILTER_NETLINK_HOOK=m +CONFIG_NETFILTER_NETLINK_LOG=m +CONFIG_NETFILTER_NETLINK_OSF=m +CONFIG_NETFILTER_NETLINK_QUEUE=m +CONFIG_NETFILTER_SKIP_EGRESS=y +CONFIG_NETFILTER_SYNPROXY=m +CONFIG_NETFILTER_XTABLES=y +CONFIG_NETFILTER_XTABLES_LEGACY=y +CONFIG_NETFILTER_XT_CONNMARK=m +CONFIG_NETFILTER_XT_MARK=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DCCP=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ECN=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_HL=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_L2TP=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SCTP=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_NETFILTER_XT_NAT=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_AUDIT=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=m +CONFIG_NETFILTER_XT_TARGET_CT=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HL=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LED=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_MASQUERADE=m +CONFIG_NETFILTER_XT_TARGET_NETMAP=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_NOTRACK=m +CONFIG_NETFILTER_XT_TARGET_RATEEST=m +CONFIG_NETFILTER_XT_TARGET_REDIRECT=m +CONFIG_NETFILTER_XT_TARGET_SECMARK=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NFT_BRIDGE_META=m +CONFIG_NFT_BRIDGE_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_COMPAT_ARP=m +CONFIG_NFT_CONNLIMIT=m +CONFIG_NFT_CT=m +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_DUP_IPV6=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FIB=m +CONFIG_NFT_FIB_INET=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NFT_FIB_IPV6=m +CONFIG_NFT_FIB_NETDEV=m +CONFIG_NFT_FLOW_OFFLOAD=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NFT_HASH=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_NAT=m +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_OSF=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_REJECT_INET=m +CONFIG_NFT_REJECT_IPV4=m +CONFIG_NFT_REJECT_IPV6=m +CONFIG_NFT_REJECT_NETDEV=m +CONFIG_NFT_SOCKET=m +CONFIG_NFT_SYNPROXY=m +CONFIG_NFT_TPROXY=m +CONFIG_NFT_TUNNEL=m +CONFIG_NFT_XFRM=m +CONFIG_NF_CONNTRACK=m +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_BRIDGE=m +CONFIG_NF_CONNTRACK_BROADCAST=m +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_LABELS=y +CONFIG_NF_CONNTRACK_MARK=y +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_OVS=y +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_PROCFS=y +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_CT_PROTO_GRE=y +CONFIG_NF_CT_PROTO_SCTP=y +CONFIG_NF_DEFRAG_IPV4=m +CONFIG_NF_DEFRAG_IPV6=m +CONFIG_NF_DUP_IPV4=m +CONFIG_NF_DUP_IPV6=m +CONFIG_NF_DUP_NETDEV=m +CONFIG_NF_FLOW_TABLE=m +CONFIG_NF_FLOW_TABLE_INET=m +CONFIG_NF_FLOW_TABLE_PROCFS=y +CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m +CONFIG_NF_LOG_IPV6=m +CONFIG_NF_LOG_SYSLOG=m +CONFIG_NF_NAT=m +CONFIG_NF_NAT_AMANDA=m +CONFIG_NF_NAT_FTP=m +CONFIG_NF_NAT_H323=m +CONFIG_NF_NAT_IRC=m +CONFIG_NF_NAT_MASQUERADE=y +CONFIG_NF_NAT_OVS=y +CONFIG_NF_NAT_PPTP=m +CONFIG_NF_NAT_REDIRECT=y +CONFIG_NF_NAT_SIP=m +CONFIG_NF_NAT_SNMP_BASIC=m +CONFIG_NF_NAT_TFTP=m +CONFIG_NF_REJECT_IPV4=m +CONFIG_NF_REJECT_IPV6=m +CONFIG_NF_SOCKET_IPV4=m +CONFIG_NF_SOCKET_IPV6=m +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_TABLES_BRIDGE=m +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_IPV4=y +CONFIG_NF_TABLES_IPV6=y +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NF_TPROXY_IPV4=m +CONFIG_NF_TPROXY_IPV6=m + +# systemd and rootless containers: namespaces, cgroup v2 controllers, seccomp, BPF and pressure reporting. +CONFIG_BLK_CGROUP=y +CONFIG_BPF=y +CONFIG_BPF_EVENTS=y +CONFIG_BPF_JIT=y +CONFIG_BPF_JIT_ALWAYS_ON=y +CONFIG_BPF_LSM=y +CONFIG_BPF_SYSCALL=y +CONFIG_CFS_BANDWIDTH=y +CONFIG_CGROUPS=y +CONFIG_CGROUP_BPF=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_DMEM=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_MISC=y +CONFIG_CGROUP_NET_CLASSID=y +CONFIG_CGROUP_NET_PRIO=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_RDMA=y +CONFIG_CGROUP_SCHED=y +CONFIG_CHECKPOINT_RESTORE=y +CONFIG_CPUSETS=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_EFIVAR_FS=y +CONFIG_EFI_PARTITION=y +CONFIG_EPOLL=y +CONFIG_FAIR_GROUP_SCHED=y +CONFIG_FANOTIFY=y +CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y +CONFIG_FHANDLE=y +CONFIG_INOTIFY_USER=y +CONFIG_IPC_NS=y +CONFIG_KCMP=y +CONFIG_MEMCG=y +CONFIG_NAMESPACES=y +CONFIG_NET_NS=y +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_FQ_CODEL=y +CONFIG_PID_NS=y +CONFIG_POSIX_MQUEUE=y +CONFIG_PROC_FS=y +CONFIG_PSI=y +CONFIG_SECCOMP=y +CONFIG_SECCOMP_FILTER=y +CONFIG_SHMEM=y +CONFIG_SIGNALFD=y +CONFIG_SYSFS=y +CONFIG_TIMERFD=y +CONFIG_TIME_NS=y +CONFIG_TMPFS=y +CONFIG_UNIX=y +CONFIG_USER_NS=y +CONFIG_UTS_NS=y + +# Container networking, VPNs and the t-3155 distribution compatibility gate. +CONFIG_APPLE_PMC=y +CONFIG_BINFMT_MISC=m +CONFIG_BRCMFMAC=m +CONFIG_GENEVE=m +CONFIG_HIDRAW=y +CONFIG_INPUT_UINPUT=m +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_IPVLAN=m +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_MACVLAN=m +CONFIG_MACVTAP=m +CONFIG_NET_CLS_CGROUP=y +CONFIG_NTSYNC=m +CONFIG_OPENVSWITCH=m +CONFIG_TUN=m +CONFIG_UCLAMP_TASK=y +CONFIG_UCLAMP_TASK_GROUP=y +CONFIG_USERFAULTFD=y +CONFIG_VETH=m +CONFIG_VHOST_NET=m +CONFIG_VHOST_VSOCK=m +CONFIG_VIRTIO_VSOCKETS=m +CONFIG_VSOCKETS=m +CONFIG_VXLAN=m +CONFIG_WIREGUARD=m +CONFIG_ZRAM=m + +# Filesystems, quotas and network shares used by distribution installations and removable media. +CONFIG_AUTOFS_FS=y +CONFIG_BLK_DEV_ZONED=y +CONFIG_BTRFS_FS=y +CONFIG_BTRFS_FS_POSIX_ACL=y +CONFIG_CACHEFILES=m +CONFIG_CIFS=m +CONFIG_CIFS_DFS_UPCALL=y +CONFIG_CIFS_FSCACHE=y +CONFIG_CIFS_POSIX=y +CONFIG_CIFS_SWN_UPCALL=y +CONFIG_CIFS_UPCALL=y +CONFIG_CIFS_XATTR=y +CONFIG_CUSE=m +CONFIG_DAX=y +CONFIG_EXFAT_DEFAULT_IOCHARSET="utf8" +CONFIG_EXFAT_FS=m +CONFIG_EXT4_FS=y +CONFIG_EXT4_USE_FOR_EXT2=y +CONFIG_F2FS_FS=m +CONFIG_F2FS_FS_COMPRESSION=y +CONFIG_F2FS_FS_LZ4=y +CONFIG_F2FS_FS_LZ4HC=y +CONFIG_F2FS_FS_LZO=y +CONFIG_F2FS_FS_LZORLE=y +CONFIG_F2FS_FS_POSIX_ACL=y +CONFIG_F2FS_FS_SECURITY=y +CONFIG_F2FS_FS_XATTR=y +CONFIG_F2FS_FS_ZSTD=y +CONFIG_F2FS_IOSTAT=y +CONFIG_F2FS_STAT_FS=y +CONFIG_F2FS_UNFAIR_RWSEM=y +CONFIG_FAT_FS=m +CONFIG_FSCACHE=y +CONFIG_FSCACHE_STATS=y +CONFIG_FS_DAX=y +CONFIG_FUSE_DAX=y +CONFIG_FUSE_FS=m +CONFIG_FUSE_IO_URING=y +CONFIG_FUSE_PASSTHROUGH=y +CONFIG_LCD_CLASS_DEVICE=m +CONFIG_NFSD=m +CONFIG_NFSD_BLOCKLAYOUT=y +CONFIG_NFSD_FLEXFILELAYOUT=y +CONFIG_NFSD_PNFS=y +CONFIG_NFSD_SCSILAYOUT=y +CONFIG_NFSD_V3_ACL=y +CONFIG_NFSD_V4=y +CONFIG_NFSD_V4_2_INTER_SSC=y +CONFIG_NFSD_V4_POSIX_ACLS=y +CONFIG_NFSD_V4_SECURITY_LABEL=y +CONFIG_NFS_ACL_SUPPORT=m +CONFIG_NFS_COMMON=y +CONFIG_NFS_COMMON_LOCALIO_SUPPORT=m +CONFIG_NFS_DISABLE_UDP_SUPPORT=y +CONFIG_NFS_FS=m +CONFIG_NFS_FSCACHE=y +CONFIG_NFS_LOCALIO=y +CONFIG_NFS_SWAP=y +CONFIG_NFS_USE_KERNEL_DNS=y +CONFIG_NFS_V3=m +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=m +CONFIG_NFS_V4_0=y +CONFIG_NFS_V4_1_IMPLEMENTATION_ID_DOMAIN="kernel.org" +CONFIG_NFS_V4_2=y +CONFIG_NFS_V4_2_SSC_HELPER=y +CONFIG_NFS_V4_SECURITY_LABEL=y +CONFIG_NLS_ASCII=y +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_936=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_950=m +CONFIG_NLS_DEFAULT="utf8" +CONFIG_NLS_ISO8859_1=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_NLS_UCS2_UTILS=m +CONFIG_NLS_UTF8=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_FS_POSIX_ACL=y +CONFIG_NTFS3_LZX_XPRESS=y +CONFIG_OVERLAY_FS=m +CONFIG_OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW=y +CONFIG_QFMT_V2=y +CONFIG_QUOTA=y +CONFIG_QUOTACTL=y +CONFIG_QUOTA_NETLINK_INTERFACE=y +CONFIG_QUOTA_TREE=y +CONFIG_TMPFS_INODE64=y +CONFIG_TMPFS_QUOTA=y +CONFIG_VFAT_FS=m +CONFIG_VIRTIO_FS=m +CONFIG_XFS_BTREE_IN_MEM=y +CONFIG_XFS_DRAIN_INTENTS=y +CONFIG_XFS_FS=m +CONFIG_XFS_LIVE_HOOKS=y +CONFIG_XFS_MEMORY_BUFS=y +CONFIG_XFS_ONLINE_REPAIR=y +CONFIG_XFS_ONLINE_SCRUB=y +CONFIG_XFS_ONLINE_SCRUB_STATS=y +CONFIG_XFS_POSIX_ACL=y +CONFIG_XFS_QUOTA=y +CONFIG_XFS_RT=y +CONFIG_XFS_SUPPORT_ASCII_CI=y +CONFIG_XFS_SUPPORT_V4=y +CONFIG_ZONE_DEVICE=y + +# Encrypted/root storage, software RAID, compression and zram backends. +CONFIG_ASYMMETRIC_KEY_TYPE=y +CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=y +CONFIG_BLK_DEV_DM=y +CONFIG_BLK_DEV_DM_BUILTIN=y +CONFIG_BLK_DEV_MD=y +CONFIG_CRYPTO_842=y +CONFIG_CRYPTO_ACOMP2=y +CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_AEAD=y +CONFIG_CRYPTO_AEAD2=y +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128_SIMD=y +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_AES_ARM64_BS=m +CONFIG_CRYPTO_AES_ARM64_CE_BLK=y +CONFIG_CRYPTO_AES_ARM64_CE_CCM=y +CONFIG_CRYPTO_AES_ARM64_NEON_BLK=y +CONFIG_CRYPTO_AKCIPHER=y +CONFIG_CRYPTO_AKCIPHER2=y +CONFIG_CRYPTO_ALGAPI=y +CONFIG_CRYPTO_ALGAPI2=y +CONFIG_CRYPTO_AUTHENC=y +CONFIG_CRYPTO_BENCHMARK=m +CONFIG_CRYPTO_BLAKE2B=y +CONFIG_CRYPTO_BLOWFISH=m +CONFIG_CRYPTO_BLOWFISH_COMMON=m +CONFIG_CRYPTO_CAMELLIA=m +CONFIG_CRYPTO_CAST5=m +CONFIG_CRYPTO_CAST6=m +CONFIG_CRYPTO_CAST_COMMON=m +CONFIG_CRYPTO_CBC=y +CONFIG_CRYPTO_CCM=y +CONFIG_CRYPTO_CHACHA20=m +CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_CMAC=y +CONFIG_CRYPTO_CRC32=m +CONFIG_CRYPTO_CRC32C=y +CONFIG_CRYPTO_CRYPTD=y +CONFIG_CRYPTO_CTR=y +CONFIG_CRYPTO_CTS=y +CONFIG_CRYPTO_DEFLATE=y +CONFIG_CRYPTO_DES=m +CONFIG_CRYPTO_DH=y +CONFIG_CRYPTO_DH_RFC7919_GROUPS=y +CONFIG_CRYPTO_DRBG=y +CONFIG_CRYPTO_ECB=y +CONFIG_CRYPTO_ECC=y +CONFIG_CRYPTO_ECDH=y +CONFIG_CRYPTO_ECDSA=y +CONFIG_CRYPTO_ECHAINIV=m +CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_ESSIV=m +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_GENIV=y +CONFIG_CRYPTO_GHASH_ARM64_CE=m +CONFIG_CRYPTO_HASH=y +CONFIG_CRYPTO_HASH2=y +CONFIG_CRYPTO_HASH_INFO=y +CONFIG_CRYPTO_HCTR2=m +CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_HW=y +CONFIG_CRYPTO_JITTERENTROPY=y +CONFIG_CRYPTO_JITTERENTROPY_MEMORY_BLOCKS=64 +CONFIG_CRYPTO_JITTERENTROPY_MEMORY_BLOCKSIZE=32 +CONFIG_CRYPTO_JITTERENTROPY_OSR=1 +CONFIG_CRYPTO_KPP=y +CONFIG_CRYPTO_KPP2=y +CONFIG_CRYPTO_KRB5=m +CONFIG_CRYPTO_KRB5ENC=m +CONFIG_CRYPTO_LIB_AES=y +CONFIG_CRYPTO_LIB_AES_ARCH=y +CONFIG_CRYPTO_LIB_AES_CBC=y +CONFIG_CRYPTO_LIB_AES_CBC_MACS=y +CONFIG_CRYPTO_LIB_AES_CCM=y +CONFIG_CRYPTO_LIB_AES_CTR=y +CONFIG_CRYPTO_LIB_AES_ECB=y +CONFIG_CRYPTO_LIB_AES_GCM=y +CONFIG_CRYPTO_LIB_AES_XTS=y +CONFIG_CRYPTO_LIB_ARC4=m +CONFIG_CRYPTO_LIB_BLAKE2B=y +CONFIG_CRYPTO_LIB_CHACHA_ARCH=y +CONFIG_CRYPTO_LIB_CURVE25519=m +CONFIG_CRYPTO_LIB_CURVE25519_GENERIC=y +CONFIG_CRYPTO_LIB_DES=m +CONFIG_CRYPTO_LIB_GF128HASH=y +CONFIG_CRYPTO_LIB_GF128HASH_ARCH=y +CONFIG_CRYPTO_LIB_GF128MUL=m +CONFIG_CRYPTO_LIB_MD5=y +CONFIG_CRYPTO_LIB_NH=m +CONFIG_CRYPTO_LIB_NH_ARCH=y +CONFIG_CRYPTO_LIB_POLY1305_ARCH=y +CONFIG_CRYPTO_LIB_POLY1305_GENERIC=y +CONFIG_CRYPTO_LIB_POLY1305_RSIZE=9 +CONFIG_CRYPTO_LIB_SHA1=y +CONFIG_CRYPTO_LIB_SHA1_ARCH=y +CONFIG_CRYPTO_LIB_SHA256=y +CONFIG_CRYPTO_LIB_SHA256_ARCH=y +CONFIG_CRYPTO_LIB_SHA3=y +CONFIG_CRYPTO_LIB_SHA3_ARCH=y +CONFIG_CRYPTO_LIB_SHA512=y +CONFIG_CRYPTO_LIB_SHA512_ARCH=y +CONFIG_CRYPTO_LIB_UTILS=y +CONFIG_CRYPTO_LRW=m +CONFIG_CRYPTO_LZ4=y +CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_LZO=y +CONFIG_CRYPTO_MANAGER=y +CONFIG_CRYPTO_MANAGER2=y +CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=y +CONFIG_CRYPTO_NULL=y +CONFIG_CRYPTO_PCRYPT=m +CONFIG_CRYPTO_RMD160=m +CONFIG_CRYPTO_RNG=y +CONFIG_CRYPTO_RNG2=y +CONFIG_CRYPTO_RSA=y +CONFIG_CRYPTO_SELFTESTS=y +CONFIG_CRYPTO_SEQIV=y +CONFIG_CRYPTO_SERPENT=m +CONFIG_CRYPTO_SHA1=y +CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_SHA3=y +CONFIG_CRYPTO_SHA512=y +CONFIG_CRYPTO_SIG=y +CONFIG_CRYPTO_SIG2=y +CONFIG_CRYPTO_SKCIPHER=y +CONFIG_CRYPTO_SKCIPHER2=y +CONFIG_CRYPTO_STREEBOG=m +CONFIG_CRYPTO_TWOFISH=m +CONFIG_CRYPTO_TWOFISH_COMMON=m +CONFIG_CRYPTO_USER=m +CONFIG_CRYPTO_USER_API=y +CONFIG_CRYPTO_USER_API_AEAD=y +CONFIG_CRYPTO_USER_API_HASH=y +CONFIG_CRYPTO_USER_API_RNG=y +CONFIG_CRYPTO_USER_API_SKCIPHER=y +CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_XCBC=m +CONFIG_CRYPTO_XCTR=m +CONFIG_CRYPTO_XTS=y +CONFIG_CRYPTO_XXHASH=y +CONFIG_CRYPTO_ZSTD=m +CONFIG_DM_AUDIT=y +CONFIG_DM_BIO_PRISON=m +CONFIG_DM_BUFIO=y +CONFIG_DM_CACHE=m +CONFIG_DM_CACHE_SMQ=m +CONFIG_DM_CLONE=m +CONFIG_DM_CRYPT=m +CONFIG_DM_DELAY=m +CONFIG_DM_DUST=m +CONFIG_DM_EBS=m +CONFIG_DM_ERA=m +CONFIG_DM_FLAKEY=m +CONFIG_DM_INIT=y +CONFIG_DM_INTEGRITY=m +CONFIG_DM_LOG_USERSPACE=m +CONFIG_DM_LOG_WRITES=m +CONFIG_DM_MIRROR=y +CONFIG_DM_MULTIPATH=m +CONFIG_DM_MULTIPATH_HST=m +CONFIG_DM_MULTIPATH_IOA=m +CONFIG_DM_MULTIPATH_QL=m +CONFIG_DM_MULTIPATH_ST=m +CONFIG_DM_PERSISTENT_DATA=m +CONFIG_DM_RAID=m +CONFIG_DM_SNAPSHOT=y +CONFIG_DM_SWITCH=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_UEVENT=y +CONFIG_DM_UNSTRIPED=m +CONFIG_DM_VDO=m +CONFIG_DM_VERITY=m +CONFIG_DM_VERITY_FEC=y +CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG=y +CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG_PLATFORM_KEYRING=y +CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG_SECONDARY_KEYRING=y +CONFIG_DM_WRITECACHE=m +CONFIG_DM_ZERO=y +CONFIG_DM_ZONED=m +CONFIG_MD_AUTODETECT=y +CONFIG_MD_BITMAP=y +CONFIG_MD_BITMAP_FILE=y +CONFIG_MD_LINEAR=m +CONFIG_MD_RAID0=m +CONFIG_MD_RAID1=m +CONFIG_MD_RAID10=m +CONFIG_MD_RAID456=m +CONFIG_SYSTEM_DATA_VERIFICATION=y +CONFIG_ZRAM_BACKEND_842=y +CONFIG_ZRAM_BACKEND_DEFLATE=y +CONFIG_ZRAM_BACKEND_LZ4=y +CONFIG_ZRAM_BACKEND_LZ4HC=y +CONFIG_ZRAM_BACKEND_LZO=y +CONFIG_ZRAM_BACKEND_ZSTD=y +CONFIG_ZRAM_DEF_COMP="lzo-rle" +CONFIG_ZRAM_DEF_COMP_LZORLE=y +CONFIG_ZRAM_MULTI_COMP=y +CONFIG_ZRAM_WRITEBACK=y +CONFIG_ZSMALLOC_CHAIN_SIZE=8 + +# Attached USB networking, serial adapters, HID, audio and storage peripherals. +CONFIG_DRAGONRISE_FF=y +CONFIG_GREENASIA_FF=y +CONFIG_HID=y +CONFIG_HID_A4TECH=m +CONFIG_HID_ACCUTOUCH=m +CONFIG_HID_ACRUX=m +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_ALPS=m +CONFIG_HID_APPLEIR=m +CONFIG_HID_AUREAL=m +CONFIG_HID_BATTERY_STRENGTH=y +CONFIG_HID_BELKIN=m +CONFIG_HID_BETOP_FF=m +CONFIG_HID_BIGBEN_FF=m +CONFIG_HID_BPF=y +CONFIG_HID_CHERRY=m +CONFIG_HID_CHICONY=m +CONFIG_HID_CMEDIA=m +CONFIG_HID_CORSAIR=m +CONFIG_HID_COUGAR=m +CONFIG_HID_CP2112=m +CONFIG_HID_CREATIVE_SB0540=m +CONFIG_HID_CYPRESS=m +CONFIG_HID_DRAGONRISE=m +CONFIG_HID_ELAN=m +CONFIG_HID_ELECOM=m +CONFIG_HID_ELO=m +CONFIG_HID_EMS_FF=m +CONFIG_HID_EVISION=m +CONFIG_HID_EZKEY=m +CONFIG_HID_FT260=m +CONFIG_HID_GEMBIRD=m +CONFIG_HID_GENERIC=y +CONFIG_HID_GFRM=m +CONFIG_HID_GLORIOUS=m +CONFIG_HID_GOODIX_SPI=m +CONFIG_HID_GOOGLE_STADIA_FF=m +CONFIG_HID_GREENASIA=m +CONFIG_HID_GT683R=m +CONFIG_HID_GYRATION=m +CONFIG_HID_HAPTIC=y +CONFIG_HID_HOLTEK=m +CONFIG_HID_ICADE=m +CONFIG_HID_ITE=m +CONFIG_HID_JABRA=m +CONFIG_HID_KENSINGTON=m +CONFIG_HID_KEYTOUCH=m +CONFIG_HID_KYE=m +CONFIG_HID_KYSONA=m +CONFIG_HID_LCPOWER=m +CONFIG_HID_LED=m +CONFIG_HID_LENOVO=m +CONFIG_HID_LETSKETCH=m +CONFIG_HID_LOGITECH=m +CONFIG_HID_LOGITECH_DJ=m +CONFIG_HID_LOGITECH_HIDPP=m +CONFIG_HID_MACALLY=m +CONFIG_HID_MALTRON=m +CONFIG_HID_MAYFLASH=m +CONFIG_HID_MCP2221=m +CONFIG_HID_MEGAWORLD_FF=m +CONFIG_HID_MICROSOFT=m +CONFIG_HID_MONTEREY=m +CONFIG_HID_MULTITOUCH=m +CONFIG_HID_NINTENDO=m +CONFIG_HID_NTI=m +CONFIG_HID_NTRIG=y +CONFIG_HID_NVIDIA_SHIELD=m +CONFIG_HID_ORTEK=m +CONFIG_HID_PANTHERLORD=m +CONFIG_HID_PENMOUNT=m +CONFIG_HID_PETALYNX=m +CONFIG_HID_PICOLCD=m +CONFIG_HID_PICOLCD_BACKLIGHT=y +CONFIG_HID_PICOLCD_FB=y +CONFIG_HID_PICOLCD_LCD=y +CONFIG_HID_PICOLCD_LEDS=y +CONFIG_HID_PID=y +CONFIG_HID_PLANTRONICS=m +CONFIG_HID_PLAYSTATION=m +CONFIG_HID_PRIMAX=m +CONFIG_HID_PRODIKEYS=m +CONFIG_HID_PXRC=m +CONFIG_HID_RAPOO=m +CONFIG_HID_RAZER=m +CONFIG_HID_RETRODE=m +CONFIG_HID_RMI=m +CONFIG_HID_ROCCAT=m +CONFIG_HID_SAITEK=m +CONFIG_HID_SAMSUNG=m +CONFIG_HID_SEMITEK=m +CONFIG_HID_SENSOR_HUB=m +CONFIG_HID_SIGMAMICRO=m +CONFIG_HID_SMARTJOYPLUS=m +CONFIG_HID_SONY=m +CONFIG_HID_SPEEDLINK=m +CONFIG_HID_STEAM=m +CONFIG_HID_STEELSERIES=m +CONFIG_HID_SUNPLUS=m +CONFIG_HID_SUPPORT=y +CONFIG_HID_THINGM=m +CONFIG_HID_THRUSTMASTER=m +CONFIG_HID_TIVO=m +CONFIG_HID_TOPRE=m +CONFIG_HID_TOPSEED=m +CONFIG_HID_TWINHAN=m +CONFIG_HID_U2FZERO=m +CONFIG_HID_UCLOGIC=m +CONFIG_HID_UDRAW_PS3=m +CONFIG_HID_UNIVERSAL_PIDFF=m +CONFIG_HID_VIEWSONIC=m +CONFIG_HID_VIVALDI=m +CONFIG_HID_VIVALDI_COMMON=m +CONFIG_HID_WACOM=m +CONFIG_HID_WALTOP=m +CONFIG_HID_WIIMOTE=m +CONFIG_HID_WINWING=m +CONFIG_HID_XIAOMI=m +CONFIG_HID_XINMO=m +CONFIG_HID_ZEROPLUS=m +CONFIG_HID_ZYDACRON=m +CONFIG_HOLTEK_FF=y +CONFIG_I2C_HID=y +CONFIG_I2C_HID_ACPI=m +CONFIG_I2C_HID_CORE=y +CONFIG_I2C_HID_OF=y +CONFIG_I2C_HID_OF_ELAN=m +CONFIG_I2C_HID_OF_GOODIX=m +CONFIG_LOGIG940_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGITECH_FF=y +CONFIG_LOGIWHEELS_FF=y +CONFIG_NINTENDO_FF=y +CONFIG_NVIDIA_SHIELD_FF=y +CONFIG_PANTHERLORD_FF=y +CONFIG_PLAYSTATION_FF=y +CONFIG_REALTEK_AUTOPM=y +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_SND_BCD2000=m +CONFIG_SND_USB=y +CONFIG_SND_USB_6FIRE=m +CONFIG_SND_USB_AUDIO=m +CONFIG_SND_USB_AUDIO_MIDI_V2=y +CONFIG_SND_USB_AUDIO_QMI=m +CONFIG_SND_USB_AUDIO_USE_MEDIA_CONTROLLER=y +CONFIG_SND_USB_CAIAQ=m +CONFIG_SND_USB_CAIAQ_INPUT=y +CONFIG_SND_USB_HIFACE=m +CONFIG_SND_USB_LINE6=m +CONFIG_SND_USB_POD=m +CONFIG_SND_USB_PODHD=m +CONFIG_SND_USB_TONEPORT=m +CONFIG_SND_USB_UA101=m +CONFIG_SND_USB_VARIAX=m +CONFIG_SONY_FF=y +CONFIG_SPI_HID_APPLE_CORE=m +CONFIG_STEAM_FF=y +CONFIG_THRUSTMASTER_FF=y +CONFIG_UHID=m +CONFIG_USB_ALI_M5632=y +CONFIG_USB_AN2720=y +CONFIG_USB_ARMLINUX=y +CONFIG_USB_BELKIN=y +CONFIG_USB_CATC=m +CONFIG_USB_EPSON2888=y +CONFIG_USB_HID=y +CONFIG_USB_HIDDEV=y +CONFIG_USB_HSO=m +CONFIG_USB_IPHETH=m +CONFIG_USB_KAWETH=m +CONFIG_USB_KC2190=y +CONFIG_USB_LAN78XX=m +CONFIG_USB_NET_AQC111=m +CONFIG_USB_NET_AX88179_178A=m +CONFIG_USB_NET_AX8817X=m +CONFIG_USB_NET_CDCETHER=m +CONFIG_USB_NET_CDC_EEM=m +CONFIG_USB_NET_CDC_MBIM=m +CONFIG_USB_NET_CDC_NCM=m +CONFIG_USB_NET_CDC_SUBSET=m +CONFIG_USB_NET_CDC_SUBSET_ENABLE=m +CONFIG_USB_NET_CH9200=m +CONFIG_USB_NET_CX82310_ETH=m +CONFIG_USB_NET_DM9601=m +CONFIG_USB_NET_DRIVERS=y +CONFIG_USB_NET_GL620A=m +CONFIG_USB_NET_HUAWEI_CDC_NCM=m +CONFIG_USB_NET_INT51X1=m +CONFIG_USB_NET_KALMIA=m +CONFIG_USB_NET_MCS7830=m +CONFIG_USB_NET_NET1080=m +CONFIG_USB_NET_PLUSB=m +CONFIG_USB_NET_QMI_WWAN=m +CONFIG_USB_NET_RNDIS_HOST=m +CONFIG_USB_NET_SMSC75XX=m +CONFIG_USB_NET_SMSC95XX=m +CONFIG_USB_NET_SR9700=m +CONFIG_USB_NET_ZAURUS=m +CONFIG_USB_PEGASUS=m +CONFIG_USB_RTL8150=m +CONFIG_USB_RTL8152=m +CONFIG_USB_RTL8153_ECM=m +CONFIG_USB_SERIAL=y +CONFIG_USB_SERIAL_AIRCABLE=m +CONFIG_USB_SERIAL_ARK3116=m +CONFIG_USB_SERIAL_BELKIN=m +CONFIG_USB_SERIAL_CH341=m +CONFIG_USB_SERIAL_CONSOLE=y +CONFIG_USB_SERIAL_CP210X=m +CONFIG_USB_SERIAL_CYBERJACK=m +CONFIG_USB_SERIAL_CYPRESS_M8=m +CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m +CONFIG_USB_SERIAL_EDGEPORT=m +CONFIG_USB_SERIAL_EDGEPORT_TI=m +CONFIG_USB_SERIAL_EMPEG=m +CONFIG_USB_SERIAL_F81232=m +CONFIG_USB_SERIAL_F8153X=m +CONFIG_USB_SERIAL_FTDI_SIO=m +CONFIG_USB_SERIAL_GARMIN=m +CONFIG_USB_SERIAL_GENERIC=y +CONFIG_USB_SERIAL_IPAQ=m +CONFIG_USB_SERIAL_IPW=m +CONFIG_USB_SERIAL_IR=m +CONFIG_USB_SERIAL_IUU=m +CONFIG_USB_SERIAL_KEYSPAN=m +CONFIG_USB_SERIAL_KEYSPAN_PDA=m +CONFIG_USB_SERIAL_KLSI=m +CONFIG_USB_SERIAL_KOBIL_SCT=m +CONFIG_USB_SERIAL_MCT_U232=m +CONFIG_USB_SERIAL_MOS7720=m +CONFIG_USB_SERIAL_MOS7840=m +CONFIG_USB_SERIAL_NAVMAN=m +CONFIG_USB_SERIAL_OMNINET=m +CONFIG_USB_SERIAL_OPTICON=m +CONFIG_USB_SERIAL_OPTION=m +CONFIG_USB_SERIAL_OTI6858=m +CONFIG_USB_SERIAL_PL2303=m +CONFIG_USB_SERIAL_QCAUX=m +CONFIG_USB_SERIAL_QT2=m +CONFIG_USB_SERIAL_QUALCOMM=m +CONFIG_USB_SERIAL_SAFE=m +CONFIG_USB_SERIAL_SAFE_PADDED=y +CONFIG_USB_SERIAL_SIERRAWIRELESS=m +CONFIG_USB_SERIAL_SIMPLE=m +CONFIG_USB_SERIAL_SPCP8X5=m +CONFIG_USB_SERIAL_SSU100=m +CONFIG_USB_SERIAL_SYMBOL=m +CONFIG_USB_SERIAL_TI=m +CONFIG_USB_SERIAL_UPD78F0730=m +CONFIG_USB_SERIAL_VISOR=m +CONFIG_USB_SERIAL_WHITEHEAT=m +CONFIG_USB_SERIAL_WWAN=m +CONFIG_USB_SERIAL_XR=m +CONFIG_USB_SERIAL_XSENS_MT=m +CONFIG_USB_SIERRA_NET=m +CONFIG_USB_STORAGE=m +CONFIG_USB_STORAGE_ALAUDA=m +CONFIG_USB_STORAGE_CYPRESS_ATACB=m +CONFIG_USB_STORAGE_DATAFAB=m +CONFIG_USB_STORAGE_ENE_UB6250=m +CONFIG_USB_STORAGE_FREECOM=m +CONFIG_USB_STORAGE_ISD200=m +CONFIG_USB_STORAGE_JUMPSHOT=m +CONFIG_USB_STORAGE_KARMA=m +CONFIG_USB_STORAGE_ONETOUCH=m +CONFIG_USB_STORAGE_REALTEK=m +CONFIG_USB_STORAGE_SDDR09=m +CONFIG_USB_STORAGE_SDDR55=m +CONFIG_USB_STORAGE_USBAT=m +CONFIG_USB_UAS=m +CONFIG_USB_USBNET=m +CONFIG_USB_VL600=m +CONFIG_ZEROPLUS_FF=y + +# Desktop compatibility and distribution diagnostics; these do not select a device. +CONFIG_COMPAT=y +CONFIG_COMPAT_VDSO=y +CONFIG_DEBUG_INFO_BTF=y +CONFIG_DEBUG_INFO_BTF_MODULES=y +CONFIG_DEBUG_INFO_DWARF5=y +CONFIG_DEBUG_INFO_REDUCED=n +CONFIG_DYNAMIC_DEBUG=y +CONFIG_DYNAMIC_FTRACE=y +CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS=y +CONFIG_EXPERT=y +CONFIG_FTRACE=y +CONFIG_FUNCTION_TRACER=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_KALLSYMS=y +CONFIG_KALLSYMS_ALL=y +CONFIG_KPROBES=y +CONFIG_PERF_EVENTS=y +CONFIG_UPROBES=y + +# Memory and userspace hardening retained from the distribution kernel. +CONFIG_BIG_KEYS=y +CONFIG_BUG_ON_DATA_CORRUPTION=y +CONFIG_CC_HAS_AUTO_VAR_INIT_PATTERN=y +CONFIG_CC_HAS_AUTO_VAR_INIT_ZERO=y +CONFIG_CC_HAS_AUTO_VAR_INIT_ZERO_BARE=y +CONFIG_CC_HAS_RANDSTRUCT=y +CONFIG_CC_HAS_SANCOV_STACK_DEPTH_CALLBACK=y +CONFIG_CC_HAS_ZERO_CALL_USED_REGS=y +CONFIG_ENCRYPTED_KEYS=y +CONFIG_FORTIFY_SOURCE=y +CONFIG_HARDENED_USERCOPY=y +CONFIG_HARDENED_USERCOPY_DEFAULT_ON=y +CONFIG_INIT_ON_ALLOC_DEFAULT_ON=y +CONFIG_INIT_STACK_ALL_ZERO=y +CONFIG_LIST_HARDENED=y +CONFIG_NETLABEL=y +CONFIG_NETWORK_SECMARK=y +CONFIG_PERSISTENT_KEYRINGS=y +CONFIG_RANDSTRUCT_NONE=y +CONFIG_SYN_COOKIES=y + +# Bluetooth protocols and common removable/immutable filesystem formats. +CONFIG_BT_AOSPEXT=y +CONFIG_BT_BNEP=m +CONFIG_BT_BNEP_MC_FILTER=y +CONFIG_BT_BNEP_PROTO_FILTER=y +CONFIG_BT_LE=y +CONFIG_BT_LEDS=y +CONFIG_BT_LE_L2CAP_ECRED=y +CONFIG_BT_MSFTEXT=y +CONFIG_BT_RFCOMM=m +CONFIG_BT_RFCOMM_TTY=y +CONFIG_EROFS_FS=m +CONFIG_EROFS_FS_BACKED_BY_FILE=y +CONFIG_EROFS_FS_POSIX_ACL=y +CONFIG_EROFS_FS_SECURITY=y +CONFIG_EROFS_FS_XATTR=y +CONFIG_EROFS_FS_ZIP=y +CONFIG_EROFS_FS_ZIP_ACCEL=y +CONFIG_EROFS_FS_ZIP_DEFLATE=y +CONFIG_EROFS_FS_ZIP_LZMA=y +CONFIG_EROFS_FS_ZIP_LZMA_DEFAULT_MAX_STREAMS=16 +CONFIG_EROFS_FS_ZIP_ZSTD=y +CONFIG_HFSPLUS_FS=m +CONFIG_HFS_FS=m +CONFIG_ISO9660_FS=m +CONFIG_JOLIET=y +CONFIG_MSDOS_FS=m +CONFIG_SQUASHFS=m +CONFIG_SQUASHFS_COMPILE_DECOMP_MULTI_PERCPU=y +CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU=y +CONFIG_SQUASHFS_FILE_DIRECT=y +CONFIG_SQUASHFS_FRAGMENT_CACHE_SIZE=3 +CONFIG_SQUASHFS_LZ4=y +CONFIG_SQUASHFS_LZO=y +CONFIG_SQUASHFS_XATTR=y +CONFIG_SQUASHFS_XZ=y +CONFIG_SQUASHFS_ZLIB=y +CONFIG_SQUASHFS_ZSTD=y +CONFIG_UDF_FS=m +CONFIG_ZISOFS=y + +# IPsec, socket diagnostics and traffic control for VPNs and container networking. +CONFIG_CLS_U32_MARK=y +CONFIG_CLS_U32_PERF=y +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_ESPINTCP=y +CONFIG_INET6_ESP_OFFLOAD=m +CONFIG_INET6_IPCOMP=m +CONFIG_INET6_XFRM_TUNNEL=m +CONFIG_INET_AH=m +CONFIG_INET_ESP=m +CONFIG_INET_ESPINTCP=y +CONFIG_INET_ESP_OFFLOAD=m +CONFIG_INET_IPCOMP=m +CONFIG_INET_RAW_DIAG=y +CONFIG_INET_UDP_DIAG=y +CONFIG_INET_XFRM_TUNNEL=m +CONFIG_IPV6_VTI=m +CONFIG_NETLINK_DIAG=y +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_CONNMARK=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_CT=m +CONFIG_NET_ACT_CTINFO=m +CONFIG_NET_ACT_GACT=m +CONFIG_NET_ACT_GATE=m +CONFIG_NET_ACT_IFE=m +CONFIG_NET_ACT_MIRRED=m +CONFIG_NET_ACT_MPLS=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_CLS_ACT=y +CONFIG_NET_CLS_BASIC=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_U32=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CANID=m +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_IPSET=m +CONFIG_NET_EMATCH_IPT=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_STACK=32 +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_IFE=m +CONFIG_NET_IFE_SKBMARK=m +CONFIG_NET_IFE_SKBPRIO=m +CONFIG_NET_IFE_SKBTCINDEX=m +CONFIG_NET_IPVTI=m +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_NET_SCH_BPF=y +CONFIG_NET_SCH_CAKE=m +CONFIG_NET_SCH_CBS=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_DUALPI2=m +CONFIG_NET_SCH_ETF=m +CONFIG_NET_SCH_ETS=m +CONFIG_NET_SCH_FIFO=y +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_FQ_PIE=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_INGRESS=m +CONFIG_NET_SCH_MQPRIO=m +CONFIG_NET_SCH_MQPRIO_LIB=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TAPRIO=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_TEQL=m +CONFIG_PACKET_DIAG=y +CONFIG_PSAMPLE=m +CONFIG_UNIX_DIAG=y +CONFIG_XFRM_AH=m +CONFIG_XFRM_ALGO=y +CONFIG_XFRM_ESP=m +CONFIG_XFRM_ESPINTCP=y +CONFIG_XFRM_INTERFACE=m +CONFIG_XFRM_IPCOMP=m +CONFIG_XFRM_IPTFS=m +CONFIG_XFRM_MIGRATE=y +CONFIG_XFRM_OFFLOAD=y +CONFIG_XFRM_STATISTICS=y +CONFIG_XFRM_SUB_POLICY=y From d282380c3baa7a596846df8670e8d993deec39bf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 20:29:57 +1100 Subject: [PATCH 1294/1579] arm64: retain the distro mmap floors and key notification support --- arch/arm64/configs/asahi.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 60d8a679594ef..ec2215594d521 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -109,6 +109,8 @@ CONFIG_EXT4_FS_POSIX_ACL=y CONFIG_EXT4_FS_SECURITY=y CONFIG_KEYS=y CONFIG_KEYS_REQUEST_CACHE=y +CONFIG_KEY_DH_OPERATIONS=y +CONFIG_KEY_NOTIFICATIONS=y CONFIG_LOAD_UEFI_KEYS=y CONFIG_LSM="lockdown,yama,integrity,selinux,bpf,landlock,ipe" CONFIG_SECONDARY_TRUSTED_KEYRING=y @@ -133,6 +135,7 @@ CONFIG_SYSTEM_BLACKLIST_KEYRING=y CONFIG_SYSTEM_TRUSTED_KEYRING=y CONFIG_TMPFS_POSIX_ACL=y CONFIG_TMPFS_XATTR=y +CONFIG_WATCH_QUEUE=y CONFIG_XFRM=y CONFIG_XFRM_USER=y @@ -1194,6 +1197,7 @@ CONFIG_CC_HAS_AUTO_VAR_INIT_ZERO_BARE=y CONFIG_CC_HAS_RANDSTRUCT=y CONFIG_CC_HAS_SANCOV_STACK_DEPTH_CALLBACK=y CONFIG_CC_HAS_ZERO_CALL_USED_REGS=y +CONFIG_DEFAULT_MMAP_MIN_ADDR=65536 CONFIG_ENCRYPTED_KEYS=y CONFIG_FORTIFY_SOURCE=y CONFIG_HARDENED_USERCOPY=y @@ -1201,6 +1205,7 @@ CONFIG_HARDENED_USERCOPY_DEFAULT_ON=y CONFIG_INIT_ON_ALLOC_DEFAULT_ON=y CONFIG_INIT_STACK_ALL_ZERO=y CONFIG_LIST_HARDENED=y +CONFIG_LSM_MMAP_MIN_ADDR=65535 CONFIG_NETLABEL=y CONFIG_NETWORK_SECMARK=y CONFIG_PERSISTENT_KEYRINGS=y From b18158c5ab592eebef397e455f7c28fb377a9fec Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 21:41:02 +1100 Subject: [PATCH 1295/1579] rust: select the prepared corrected bindgen by default Prepare the pinned generator at a stable source-tree entry point and prefer it for in-tree and O= builds. Keep every generated layout assertion mandatory and retain stock-tool rejection coverage. Document upstream status and the scope of aligned empty records. Signed-off-by: Ryan Murray Assisted-by: LLM --- Documentation/rust/bindgen-layout-checks.rst | 22 +++++--- Makefile | 5 +- tools/aurora-bindgen/.gitignore | 1 + tools/aurora-bindgen/README.rst | 58 ++++++++++++++++---- tools/aurora-bindgen/build.py | 7 ++- 5 files changed, 72 insertions(+), 21 deletions(-) create mode 100644 tools/aurora-bindgen/.gitignore diff --git a/Documentation/rust/bindgen-layout-checks.rst b/Documentation/rust/bindgen-layout-checks.rst index 5af9951a64338..4e0a996bd701d 100644 --- a/Documentation/rust/bindgen-layout-checks.rst +++ b/Documentation/rust/bindgen-layout-checks.rst @@ -46,11 +46,17 @@ Corrected generator ------------------- ``tools/aurora-bindgen/`` contains pinned patches for bindgen 0.73.2 that -repair both cases, a private build recipe and compile-only tests; see -``tools/aurora-bindgen/README.rst``. It installs nothing system-wide. To check -the layouts with it:: +repair both cases, a local build recipe and compile-only tests; see +``tools/aurora-bindgen/README.rst``. Prepare it once before configuration:: - make LLVM=1 BINDGEN=/path/to/bindgen ... + python3 tools/aurora-bindgen/build.py + make LLVM=1 ... + +The top-level Makefile automatically selects +``tools/aurora-bindgen/build/bin/bindgen`` from the source tree, also for ``O=`` +builds. Packagers prepare this entry point in their build roots. Make does not +fetch tools or install anything system-wide. If the entry point is absent, +Kbuild uses ``bindgen`` from PATH and the same mandatory assertions apply. The generated bindings are consumed by the Rust kernel crate and by several drivers. After changing the bindings generator, rebuild the kernel Image and @@ -61,7 +67,9 @@ the configured ``--rust-target 1.85``: that selects unconditional constant assertions rather than ``#[test]`` functions. They are checked without a Rust test configuration. -Pass ``BINDGEN=/absolute/path/to/bindgen`` as a make command-line argument. -Exporting it in the environment is insufficient: the top-level Makefile's -``BINDGEN = bindgen`` assignment takes precedence over an environment value. Check the recorded bindings command, not just the toolchain version banner. +Command-line overrides remain available for generator regression tests. +The defect can affect any generated aligned empty record and containing types +under other configurations; all emitted size/alignment/offset checks remain +mandatory. They do not cover C-only types or opaque internal fields. See the +generator README for the upstream status and related empty-struct issues. diff --git a/Makefile b/Makefile index c61ea07ee5051..956665f4965fe 100644 --- a/Makefile +++ b/Makefile @@ -550,7 +550,10 @@ RUSTC = rustc RUSTDOC = rustdoc RUSTFMT = rustfmt CLIPPY_DRIVER = clippy-driver -BINDGEN = bindgen +# Prefer the locally prepared, layout-corrected generator, including O= builds. +# Keep command-line overrides for toolchain testing; layouts are always checked. +aurora_bindgen := $(abs_srctree)/tools/aurora-bindgen/build/bin/bindgen +BINDGEN = $(if $(wildcard $(aurora_bindgen)),$(aurora_bindgen),bindgen) PAHOLE = pahole RESOLVE_BTFIDS = $(objtree)/tools/bpf/resolve_btfids/resolve_btfids LEX = flex diff --git a/tools/aurora-bindgen/.gitignore b/tools/aurora-bindgen/.gitignore new file mode 100644 index 0000000000000..84c048a73cc2e --- /dev/null +++ b/tools/aurora-bindgen/.gitignore @@ -0,0 +1 @@ +/build/ diff --git a/tools/aurora-bindgen/README.rst b/tools/aurora-bindgen/README.rst index 35e4b6eb8a274..9399e5da50c03 100644 --- a/tools/aurora-bindgen/README.rst +++ b/tools/aurora-bindgen/README.rst @@ -1,6 +1,6 @@ .. SPDX-License-Identifier: GPL-2.0-only -Private bindgen with checked C/Rust layouts +Default bindgen with checked C/Rust layouts ========================================== This package builds bindgen 0.73.2 with two generic binding-generation repairs: @@ -16,19 +16,30 @@ compiler or build path produces identical bytes. The manifest identifies when that digest predates comment-only patch changes; each rebuild must record its own binary digest. -Build in a fresh private directory:: +Prepare the default generator once before configuring or building the kernel:: - python3 tools/aurora-bindgen/build.py --output-dir /absolute/new/tool-build + python3 tools/aurora-bindgen/build.py + make LLVM=1 ... Python 3.12 or newer, Cargo/Rust, Clang/libclang and patch are required. Cargo's -locked dependency graph is used, with at most two build jobs. ``--archive`` can -supply an already downloaded source archive; its digest is still checked. The -script builds and runs compile-only valid-input checks, then prints the exact -``BINDGEN=/absolute/path/to/bindgen`` make argument. It installs nothing globally. -The kernel always retains layout assertions. Pass the printed ``BINDGEN=`` -argument on the make command line; an exported environment variable is -overridden by the top-level Makefile. Build the complete kernel Image and -matching modules with the same tool. +locked dependency graph is used, with at most two build jobs. The script builds +in ``tools/aurora-bindgen/build/`` and publishes ``build/bin/bindgen`` only after +its compile-only checks pass. The top-level Makefile selects that entry point +automatically for both in-tree and ``O=`` builds. No make variable or environment +setting is needed. It installs nothing globally and make never downloads or +builds the generator implicitly. Preserve the build log and ``build/result.json``. + +``--archive`` can supply an already downloaded source archive; its digest is +still checked. ``--output-dir`` supports separate tool experiments; a nondefault +location is not automatically selected. Packagers may instead place their +independently built and tested pinned tool at the same ``build/bin/bindgen`` +entry point (a symlink is sufficient). Remove or move a previous build directory +before rebuilding; the script refuses to overwrite it. + +If the prepared entry point is absent, Kbuild falls back to ``bindgen`` on PATH. +The mandatory layout assertions still reject incorrect bindings. A command-line +override remains available for generator regression testing. + Both scripts reject optimized Python (``-O``, ``-OO`` or ``PYTHONOPTIMIZE``) before creating output, since their validation assertions must remain enabled. @@ -90,3 +101,28 @@ Primary sources: A complete kernel build remains the configuration-specific layout check; these small generator fixtures do not replace it or establish runtime GPU stability. + +Upstream status and affected configurations +------------------------------------------ + +As checked on 2026-10-26, the latest upstream release is 0.73.2 and Linux's +minimum is 0.71.1. Both versions, and bindgen main, skip explicit alignment +handling for zero-sized non-opaque structs. Raising the minimum therefore does +not fix this defect. Upstream issue #551 and fix #552 concern C empty-struct +size, while #1683 concerns the Windows ABI; neither fixes this aligned-empty +case. No exact upstream issue or merged fix was found in the issue/PR search. + +The defect is generic: any explicitly aligned empty C record can have the wrong +Rust alignment, and embedding it can shift later fields or change enclosing +size/alignment. Configurations that empty conditional records, change cacheline +alignment or change preceding fields can expose it; it is not specific to +Asahi or ARM64. SCHEDSTATS=y masks this particular scheduler case. Every emitted +layout assertion is compiled regardless of KUnit, but this only covers types +and fields emitted by bindgen, not arbitrary C-only or opaque internals. + +* https://github.com/rust-lang/rust-bindgen/releases/tag/v0.73.2 +* https://github.com/torvalds/linux/blob/master/scripts/min-tool-version.sh +* https://github.com/rust-lang/rust-bindgen/blob/main/bindgen/codegen/mod.rs +* https://github.com/rust-lang/rust-bindgen/issues/551 +* https://github.com/rust-lang/rust-bindgen/pull/552 +* https://github.com/rust-lang/rust-bindgen/issues/1683 diff --git a/tools/aurora-bindgen/build.py b/tools/aurora-bindgen/build.py index fa87873098480..3ae28e18625ea 100644 --- a/tools/aurora-bindgen/build.py +++ b/tools/aurora-bindgen/build.py @@ -6,7 +6,7 @@ import argparse,hashlib,json,subprocess,tarfile,urllib.request,shutil,os,sys from pathlib import Path -P=Path(__file__).resolve().parent;ap=argparse.ArgumentParser();ap.add_argument('--output-dir',required=True,type=Path);ap.add_argument('--archive',type=Path);ap.add_argument('--jobs',type=int,default=2);a=ap.parse_args();assert 1<=a.jobs<=2 +P=Path(__file__).resolve().parent;ap=argparse.ArgumentParser();ap.add_argument('--output-dir',default=P/'build',type=Path);ap.add_argument('--archive',type=Path);ap.add_argument('--jobs',type=int,default=2);a=ap.parse_args();assert 1<=a.jobs<=2 O=a.output_dir.resolve();assert not O.exists(),'refusing to overwrite an existing source or tool';O.mkdir(parents=True) m=json.loads((P/'sources.json').read_text());sha=lambda p:hashlib.sha256(p.read_bytes()).hexdigest();archive=O/'source.tar.gz' if a.archive:shutil.copyfile(a.archive,archive) @@ -22,4 +22,7 @@ cmd=['cargo','build','--locked','--release','-p','bindgen-cli','-j'+str(a.jobs)] with (O/'build.log').open('w') as log:subprocess.run(cmd,cwd=S,stdout=log,stderr=subprocess.STDOUT,check=True) binary=S/'target/release/bindgen';subprocess.run([sys.executable,str(P/'test.py'),'--bindgen',str(binary),'--output-dir',str(O/'tests')],check=True) -r={'status':'BUILT_AND_COMPILE_TESTED','binary':str(binary),'binary_sha256':sha(binary),'source_manifest_sha256':sha(P/'sources.json'),'build_command':cmd,'rustc_version':subprocess.check_output(['rustc','-vV'],text=True),'cargo_version':subprocess.check_output(['cargo','-vV'],text=True),'tests_manifest_sha256':sha(O/'tests/result.json'),'installed_globally':False};(O/'result.json').write_text(json.dumps(r,indent=2)+'\n');print(json.dumps(r,indent=2));print('Use make BINDGEN='+str(binary)+' ... to build the kernel with layout assertions.') +r={'status':'BUILT_AND_COMPILE_TESTED','binary':str(binary),'binary_sha256':sha(binary),'source_manifest_sha256':sha(P/'sources.json'),'build_command':cmd,'rustc_version':subprocess.check_output(['rustc','-vV'],text=True),'cargo_version':subprocess.check_output(['cargo','-vV'],text=True),'tests_manifest_sha256':sha(O/'tests/result.json'),'installed_globally':False};(O/'result.json').write_text(json.dumps(r,indent=2)+'\n') +# Publish the stable entry point only after every compile-only check passed. +(O/'bin').mkdir();(O/'bin/bindgen').symlink_to(binary.relative_to(O/'bin',walk_up=True)) +print(json.dumps(r,indent=2));print('Prepared '+str(O/'bin/bindgen')+'. The default output is selected automatically by Kbuild.') From 7448de6cec9d39d206174b692820c5d472013f6a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 22:00:16 +1100 Subject: [PATCH 1296/1579] arm64: keep the default ESP charset available in initramfs Asahi dracut loads vfat and mounts the system ESP without charset options before unpacking vendor firmware. Match Fedora codepage 437 and ASCII defaults with built-in converters so the mount does not depend on a modular ISO-8859-1 converter absent from initramfs. Retain XZ and Zstd firmware loading for distribution firmware packages. Signed-off-by: Ryan Murray Assisted-by: LLM --- arch/arm64/configs/asahi.config | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index ec2215594d521..8980f8f408d75 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -542,6 +542,19 @@ CONFIG_VXLAN=m CONFIG_WIREGUARD=m CONFIG_ZRAM=m +# Asahi dracut mounts the ESP with default VFAT options before the rootfs exists. +# Match Fedora's 437/ASCII defaults and keep both converters built in: a modular +# default charset may be absent from the early vendor-firmware initramfs. +CONFIG_FAT_DEFAULT_CODEPAGE=437 +CONFIG_FAT_DEFAULT_IOCHARSET="ascii" +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ASCII=y + +# Distribution firmware packages can contain .xz and .zst files after extraction. +CONFIG_FW_LOADER_COMPRESS=y +CONFIG_FW_LOADER_COMPRESS_XZ=y +CONFIG_FW_LOADER_COMPRESS_ZSTD=y + # Filesystems, quotas and network shares used by distribution installations and removable media. CONFIG_AUTOFS_FS=y CONFIG_BLK_DEV_ZONED=y @@ -610,10 +623,8 @@ CONFIG_NFS_V4_1_IMPLEMENTATION_ID_DOMAIN="kernel.org" CONFIG_NFS_V4_2=y CONFIG_NFS_V4_2_SSC_HELPER=y CONFIG_NFS_V4_SECURITY_LABEL=y -CONFIG_NLS_ASCII=y CONFIG_NLS_CODEPAGE_1250=m CONFIG_NLS_CODEPAGE_1251=m -CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_737=m CONFIG_NLS_CODEPAGE_775=m CONFIG_NLS_CODEPAGE_850=m From f0f840c6cef1c7eb8a3f8b3f962d8540f4b846f2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 17:48:38 +1100 Subject: [PATCH 1297/1579] ASoC: apple: refuse speaker backends without enforced protection Do not let the legacy unsafe parameter admit an unsupported backend, skip safe-volume setup failures or expose fixed protection controls. Generic apple,macaudio and SSM3515 backends remain disabled. Require a supported amplifier, speaker layout and negative safe volume before adding a Speaker link. Keep the parameter accepted for compatibility, without unsafe overrides. Add a hardware-free KUnit regression covering generic fallback, unsupported and missing-protection configurations with parameter 2 and other values, plus the supported TAS and software-gain paths. Signed-off-by: Ryan Murray --- sound/soc/apple/macaudio-admission-test.h | 41 +++++++++++++++++++++++ sound/soc/apple/macaudio.c | 30 +++++++++++------ 2 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 sound/soc/apple/macaudio-admission-test.h diff --git a/sound/soc/apple/macaudio-admission-test.h b/sound/soc/apple/macaudio-admission-test.h new file mode 100644 index 0000000000000..d0465e887d734 --- /dev/null +++ b/sound/soc/apple/macaudio-admission-test.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Exercise the same admission predicate used for Speaker/Speakers DT links. */ +#include + +static void macaudio_admission_unsafe_parameter_cannot_admit_speakers(struct kunit *test) +{ + struct macaudio_platform_cfg cfg = macaudio_j413_cfg; + int saved = please_blow_up_my_speakers; + int values[] = { 0, 1, 2, 3, -1 }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(values); i++) { + please_blow_up_my_speakers = values[i]; + /* Generic apple,macaudio plus a DT Speaker backend. */ + KUNIT_EXPECT_FALSE(test, macaudio_speakers_allowed(&macaudio_fallback_cfg)); + KUNIT_EXPECT_FALSE(test, macaudio_speakers_allowed(&macaudio_j45x_cfg)); + KUNIT_EXPECT_TRUE(test, macaudio_speakers_allowed(&macaudio_j413_cfg)); + KUNIT_EXPECT_TRUE(test, macaudio_speakers_allowed(&macaudio_j700_cfg)); + cfg = macaudio_j413_cfg; + cfg.safe_vol = 0; + KUNIT_EXPECT_FALSE(test, macaudio_speakers_allowed(&cfg)); + cfg = macaudio_j413_cfg; + cfg.speakers = SPKR_NONE; + KUNIT_EXPECT_FALSE(test, macaudio_speakers_allowed(&cfg)); + cfg = macaudio_j413_cfg; + cfg.amp = AMP_NONE; + KUNIT_EXPECT_FALSE(test, macaudio_speakers_allowed(&cfg)); + } + please_blow_up_my_speakers = saved; +} + +static struct kunit_case macaudio_admission_cases[] = { + KUNIT_CASE(macaudio_admission_unsafe_parameter_cannot_admit_speakers), + {} +}; + +static struct kunit_suite macaudio_admission_suite = { + .name = "apple-speaker-admission", + .test_cases = macaudio_admission_cases, +}; +kunit_test_suite(macaudio_admission_suite); diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index cc1123e10f398..d40dfb20d9675 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -222,7 +222,7 @@ struct macaudio_snd_data { static int please_blow_up_my_speakers; module_param(please_blow_up_my_speakers, int, 0644); -MODULE_PARM_DESC(please_blow_up_my_speakers, "Allow unsafe or untested operating configurations"); +MODULE_PARM_DESC(please_blow_up_my_speakers, "Deprecated: cannot bypass speaker admission or protection"); SND_SOC_DAILINK_DEFS(primary, DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-0")), // CPU @@ -686,6 +686,16 @@ static int macaudio_copy_link(struct device *dev, struct snd_soc_dai_link *targe return 0; } +/* A Speaker DT link alone does not establish a protected backend. */ +static bool macaudio_speakers_allowed(const struct macaudio_platform_cfg *cfg) +{ + /* Every admitted backend must have a configured, enforced safe level. */ + return cfg->enable_speakers && cfg->safe_vol < 0 && + cfg->speakers != SPKR_NONE && + (cfg->amp == AMP_TAS5770 || cfg->amp == AMP_SN012776 || + cfg->amp == AMP_MAX98360A); +} + static int macaudio_parse_of_component(struct device_node *node, int index, struct snd_soc_dai_link_component *comp) { @@ -882,7 +892,7 @@ static int macaudio_parse_of(struct macaudio_snd_data *ma) speakers = !strcmp(link_name, "Speaker") || !strcmp(link_name, "Speakers"); if (speakers) { - if (!ma->cfg->enable_speakers && !please_blow_up_my_speakers) { + if (!macaudio_speakers_allowed(ma->cfg)) { dev_err(card->dev, "driver can't assure safety on this model, disabling speakers\n"); continue; } @@ -1603,8 +1613,7 @@ static int macaudio_card_remove(struct snd_soc_card *card) if (err < 0) { \ dev_err(card->dev, "%s on '%s': %d\n", #call, pattern, \ ret); \ - if (please_blow_up_my_speakers < 2) \ - return err; \ + return err; \ } else { \ dev_dbg(card->dev, "%s on '%s': %d hits\n", #call, \ pattern, ret); \ @@ -1644,8 +1653,7 @@ static int macaudio_set_speaker(struct snd_soc_card *card, const char *prefix, b CHECK_CONCAT(snd_soc_set_enum_kctl, "HPF Corner Frequency", tweeter ? "800 Hz" : "2 Hz"); - if (please_blow_up_my_speakers < 2) - CHECK_CONCAT(snd_soc_deactivate_kctl, "HPF Corner Frequency", 0); + CHECK_CONCAT(snd_soc_deactivate_kctl, "HPF Corner Frequency", 0); CHECK_CONCAT(snd_soc_set_enum_kctl, "OCE Handling", "Retry"); CHECK_CONCAT(snd_soc_deactivate_kctl, "OCE Handling", 0); @@ -1654,8 +1662,7 @@ static int macaudio_set_speaker(struct snd_soc_card *card, const char *prefix, b /* TODO: check */ CHECK_CONCAT(snd_soc_set_enum_kctl, "DAC Analog Gain Select", "8.4 V Span"); - if (please_blow_up_my_speakers < 2) - CHECK_CONCAT(snd_soc_deactivate_kctl, "DAC Analog Gain Select", 0); + CHECK_CONCAT(snd_soc_deactivate_kctl, "DAC Analog Gain Select", 0); /* TODO: HPF, needs new call to set */ break; @@ -1695,8 +1702,7 @@ static int macaudio_fixup_controls(struct snd_soc_card *card) */ switch(ma->cfg->speakers) { case SPKR_NONE: - WARN_ON(please_blow_up_my_speakers < 2); - return please_blow_up_my_speakers >= 2 ? 0 : -EINVAL; + return -EINVAL; case SPKR_1W: /* only 1W stereo system (J313) is uses cs42l83 */ if (ma->cfg->stereo) { @@ -2251,3 +2257,7 @@ module_platform_driver(macaudio_snd_driver); MODULE_AUTHOR("Martin Povišer "); MODULE_DESCRIPTION("Apple Silicon Macs machine-level sound driver"); MODULE_LICENSE("GPL"); + +#if IS_ENABLED(CONFIG_KUNIT) && !defined(MODULE) +#include "macaudio-admission-test.h" +#endif From 9a9ba9ba3cead0c0e3bb56d370dce474f943ee07 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 23:30:09 +1100 Subject: [PATCH 1298/1579] dt-bindings: spi: apple: Add the T6041 SPI controller The M4 Pro (T6040/T6041) SPI controllers are register compatible with T8103, like the T6020 and T8140 ones. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/spi/apple,spi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/spi/apple,spi.yaml b/Documentation/devicetree/bindings/spi/apple,spi.yaml index 4b30265686fa1..be47c5f9ff075 100644 --- a/Documentation/devicetree/bindings/spi/apple,spi.yaml +++ b/Documentation/devicetree/bindings/spi/apple,spi.yaml @@ -18,6 +18,7 @@ properties: - items: - enum: - apple,t6020-spi + - apple,t6041-spi - apple,t8140-spi - const: apple,t8103-spi - items: From d5235866cdcd53ba6a5f5ca4f4bc0cd8230adb7c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 23:30:09 +1100 Subject: [PATCH 1299/1579] dt-bindings: input: apple,mesa-fingerprint: Name the signed root key Sensors whose enclave session needs a signed root key (the J700 and J616s part, 0x3356) take it as per-device factory data, like the calibration. Let firmware-name carry it as an optional second entry so the board description names it. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/input/apple,mesa-fingerprint.yaml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml index 53b304bcec7ff..631ba47a5a3ca 100644 --- a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml +++ b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml @@ -57,11 +57,17 @@ properties: description: Rail selector words, in power-on order. firmware-name: - maxItems: 1 + minItems: 1 + items: + - description: + Calibration blob for this sensor. + - description: + Signed root-key container, for sensors whose enclave session needs + one. description: - Calibration blob for this sensor. It is per-device factory data written - at manufacture, so it is neither redistributable nor derivable from the - compatible; the board description has to name it. + Per-device factory data written at manufacture, so neither + redistributable nor derivable from the compatible; the board + description has to name it. spi-cpha: true From c737a230556b23e043a26100e3b785db28f99e2b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 23:30:10 +1100 Subject: [PATCH 1300/1579] soc: apple: sep: Take the sensor root key name from the device tree The signed root-key container is per-device data, but its name was fixed to J700's. Read it from the sensor node's second firmware-name entry and keep J700's name when the board gives only the calibration, so the J700 device tree and its installed file keep working unchanged. Name the file in the error when it cannot be loaded. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 9 +++++---- drivers/soc/apple/sensor.rs | 15 +++++++++++++-- drivers/soc/apple/sensor_shim.c | 24 ++++++++++++++---------- drivers/soc/apple/sep.rs | 3 +++ drivers/soc/apple/shim.h | 2 +- 5 files changed, 36 insertions(+), 17 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 88be672817b1c..5a81a370dda38 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2183,14 +2183,15 @@ impl SepData { self.sensor_trace(c"SENSOR_NONCE"); if !self.sensor_key_loaded.load(Relaxed) { - let firmware = match kernel::firmware::Firmware::request( - c"apple/msrk-j700.bin", &self.dev, - ) { + // Per-device data, so the board names it; J700 predates that. + let name = sensor::root_key_firmware_name().unwrap_or(ROOT_KEY_FIRMWARE); + let firmware = match kernel::firmware::Firmware::request(name, &self.dev) { Ok(firmware) => firmware, Err(error) => { dev_err!( self.dev, - "sensor: signed root-key container unavailable: {:?}\n", + "sensor: signed root-key container {} unavailable: {:?}\n", + name, error ); return false; diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 63a4ae28fbc6d..13bb470c89b3b 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -25,15 +25,26 @@ extern "C" { fn sep_sensor_irq_available() -> c_int; fn sep_sensor_irq_arm(); fn sep_sensor_irq_wait(timeout_ms: c_uint) -> c_int; - fn sep_sensor_firmware_name() -> *const c_char; + fn sep_sensor_firmware_name(index: c_uint) -> *const c_char; } /// The calibration blob's name from the board description, or `None` when the /// sensor node has no `firmware-name` property (the caller then uses a default). pub(crate) fn firmware_name() -> Option<&'static CStr> { + firmware_name_at(0) +} + +/// The signed root-key container's name from the board description: the +/// second `firmware-name` entry, or `None` when the board gives only the +/// calibration (the caller then uses a default). +pub(crate) fn root_key_firmware_name() -> Option<&'static CStr> { + firmware_name_at(1) +} + +fn firmware_name_at(index: c_uint) -> Option<&'static CStr> { // SAFETY: the shim returns NULL or a pointer to a NUL-terminated string // owned by the device property and valid for the device's lifetime. - let ptr = unsafe { sep_sensor_firmware_name() }; + let ptr = unsafe { sep_sensor_firmware_name(index) }; if ptr.is_null() { return None; } diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index 68405e0a74507..bed228b4d1ca3 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -568,19 +568,23 @@ static int sep_sensor_power_line_locked(void) } /* - * The per-device calibration blob name, as the board description gives it in - * the sensor node's "firmware-name" property. NULL when absent, so the caller - * can fall back to a default. + * A per-device blob name, as the board description gives it in the sensor + * node's "firmware-name" list: index 0 is the calibration, index 1 the signed + * root key. NULL when that entry is absent, so the caller can fall back to a + * default. */ -static const char *sep_sensor_firmware_name_locked(void) +static const char *sep_sensor_firmware_name_locked(unsigned int index) { - const char *name = NULL; + const char *names[2] = { NULL, NULL }; + int count; - if (!sep_spi) + if (!sep_spi || index >= ARRAY_SIZE(names)) return NULL; - if (device_property_read_string(&sep_spi->dev, "firmware-name", &name)) + count = device_property_read_string_array(&sep_spi->dev, "firmware-name", + names, ARRAY_SIZE(names)); + if (count <= (int)index) return NULL; - return name; + return names[index]; } /* Powers the sensor on/off, holding the hardware settling delay. */ @@ -871,14 +875,14 @@ int sep_sensor_power_line(void) return result; } -const char *sep_sensor_firmware_name(void) +const char *sep_sensor_firmware_name(unsigned int index) { struct sep_sensor_lifetime *sensor = sep_lock_sensor(); const char *result; if (!sensor) return NULL; - result = sep_sensor_firmware_name_locked(); + result = sep_sensor_firmware_name_locked(index); mutex_unlock(&sensor->lock); return result; } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index df71496a564fe..7d9f4f2af1869 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -449,6 +449,9 @@ const PATCH_POLL_MS: u32 = 20; const PATCH_POLL_ATTEMPTS: u32 = 250; const CALIBRATION_FIRMWARE: &CStr = c"apple/mesa_calibration.bin"; +/// The signed root-key container on boards whose sensor node names only the +/// calibration (J700). +const ROOT_KEY_FIRMWARE: &CStr = c"apple/msrk-j700.bin"; const SBIO_PROBE_USER_ID: i32 = 1000; static_assert!(SBIO_PROBE_USER_ID >= crate::sks::SKS_DESIGNATE_USER_MIN); diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 0425bf2cdaa6e..99d2661e2cb1c 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -132,7 +132,7 @@ int sep_sensor_cs_timing_mode(void); int sep_sensor_power_cycle(void); int sep_sensor_power_source(void); int sep_sensor_power(int on); -const char *sep_sensor_firmware_name(void); +const char *sep_sensor_firmware_name(unsigned int index); int sep_sensor_xfer(const void *tx, void *rx, size_t len); int sep_sensor_xfer_tx(const void *tx, size_t len); int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len); From 20cf8265749b808dfb9302d060458991a6887519 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 23:30:10 +1100 Subject: [PATCH 1301/1579] soc: apple: sep: Add the T6040 (J616s) platform profile The J616s ADT (26.6.2) describes a SEP that iBoot boots before the AP OS, an iop-sep,ascwrap-v6 whose DART window starts at the BIT(40) alias, and the Mesa sensor J700 carries (0x3356) on spi2 at 0x429108000, 8 MHz, chip select 0. So this is a warm-registration platform with T8140's authenticated sensor session, and the xART guard on from the first boot because macOS shares the enclave. Only J616s is matched; the other T6040 Macs' sensor wiring is unchecked. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/profile.rs | 40 ++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 0ba0eb510122b..0e632c65da81f 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -9,7 +9,7 @@ //! here once. The driver reads addresses from the device tree, never scans for //! them, and never patches properties at runtime. //! -//! Five bring-up targets are modelled: +//! These bring-up targets are modelled: //! //! * `T8103` / J313 (MacBook Air, M1): the host boots the SEP with the boot //! endpoint handshake over a 0x30000 shared-memory window. @@ -24,6 +24,8 @@ //! a powered sensor, so the driver uses the warm registration path. //! * `T8122` / J613, J615 (MacBook Air, M3): the T8112 warm registration with //! the T8140 SEP DART aperture above 4 GiB. +//! * `T6040` / J616s (MacBook Pro 16", M4 Pro): iBoot boots sepOS, so the +//! driver uses the warm registration path; the sensor is the J700 part. // The boot handshake, identity source, sensor and DART fields are consumed by // the boot-endpoint, identity and sensor-transport paths. @@ -426,6 +428,28 @@ pub(crate) const T8122: PlatformProfile = PlatformProfile { ..T8112 }; +/// MacBook Pro 16" M4 Pro (J616s). The J616s ADT (26.6.2, 25G83) records a +/// pre-booted SEP (`sepfw-load-at-boot`), an `iop-sep,ascwrap-v6` SEP whose +/// DART aperture starts at the BIT(40) alias (`vm-base` 0x10000004000), and +/// the Mesa sensor J700 carries (`sensor-id` 0x3356) on spi2 at +/// `0x429108000`, 8 MHz, chip select 0. Its power is NUB GPIO 53 and its +/// data-ready line AP GPIO 116. `/defaults` carries `cpx-encryption-mode = 2`. +/// +/// A warm-registration platform like T8140, with T8140's authenticated +/// sensor session: the enclave serves this sensor only after the nonce and +/// the signed root key. Device-tree nodes remain authoritative for addresses. +pub(crate) const T6040: PlatformProfile = PlatformProfile { + name: "T6040/J616s", + sensor: SensorProfile { + controller_base: 0x4_2910_8000, + ..T8140.sensor + }, + // macOS shares this enclave and Linux has no records here yet: keep + // Linux to its own xART records from the first boot. + xart_guard: 1, + ..T8140 +}; + static_assert!(T8103.shmem_capacity == 0x30000); static_assert!(T6020.shmem_capacity == 0x40000); static_assert!(T8140.shmem_capacity == 0x40000); @@ -441,6 +465,8 @@ static_assert!(T8112.sensor.controller_base == T8103.sensor.controller_base); static_assert!(T8122.shmem_capacity == T8112.shmem_capacity); static_assert!(T8122.sensor.controller_base == 0x2a1108000); static_assert!(T8122.wide_dma_mask); +static_assert!(T6040.sensor.controller_base == 0x429108000); +static_assert!(T6040.sensor.expected_id == T8140.sensor.expected_id); /// Whether the machine root declares `compatible`. /// @@ -587,6 +613,11 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { if machine_has(b"apple,t8122") && (machine_has(b"apple,j613") || machine_has(b"apple,j615")) { return Ok(&T8122); } + // Only the board whose ADT was read: J616s. The other T6040 Macs share + // the SoC but their sensor wiring is unchecked. + if machine_has(b"apple,t6040") && machine_has(b"apple,j616s") { + return Ok(&T6040); + } Err(ENODEV) } @@ -637,6 +668,11 @@ mod tests { assert!(T8122.wide_dma_mask); assert!(T8122.shmem_capacity == T8112.shmem_capacity); assert!(T8122.sensor.controller_base == 0x2_a110_8000); + assert!(T6040.nonce_session); + assert!(T6040.wide_dma_mask); + assert!(T6040.bootstrap == Bootstrap::WarmRegister); + assert!(T6040.identity == IdentitySource::Chosen); + assert!(T6040.xart_guard == 1); } } @@ -667,7 +703,7 @@ mod dart_window_tests { #[test] fn the_other_profiles_keep_the_bit40_window() { - for profile in [&T8103, &T6000, &T6020, &T8140, &T6020_SEPOS13, &T8122] { + for profile in [&T8103, &T6000, &T6020, &T8140, &T6020_SEPOS13, &T8122, &T6040] { assert_eq!(offset(profile.dart_window), 1 << 40); assert_eq!(range(profile.dart_window), (0, 1 << 32)); } From a35d348cbe97f390cc075d70124f9d8fa3f853d5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 23:30:10 +1100 Subject: [PATCH 1302/1579] arm64: dts: apple: t6040: Add the SEP, its DART and mailbox, and spi2 From the J616s ADT: the SEP at 0x406600000 with its mailbox at +0x8000 (AIC 666-669), its DART at 0x404c80000 (AIC 676) with the window iBoot programmed (vm-base 0x10000004000, vm-size 0x3ff00000), and spi2 at 0x429108000 (AIC 1569) with the Mesa sensor child. Add the sep, spi_p and spi2 power states from the ADT pmgr devices. The SEP and its DART stay disabled; apple_sep enables them when it loads. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-pmgr.dtsi | 28 ++++++++++ arch/arm64/boot/dts/apple/t6040.dtsi | 66 +++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi index 8c05fb4d4d5d3..bf357745d581d 100644 --- a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi @@ -140,6 +140,24 @@ power-domains = <&ps_sio>; }; + ps_spi_p: power-controller@198 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x198 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + power-domains = <&ps_sio>; + }; + + ps_spi2: power-controller@1c8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi2"; + power-domains = <&ps_spi_p>; + }; + ps_ans: power-controller@370 { compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x370 4>; @@ -178,6 +196,16 @@ power-domains = <&ps_ans>, <&ps_apcie_st1>; apple,always-on; }; + + ps_sep: power-controller@c00 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc00 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sep"; + power-domains = <&ps_fab3_soc>; + apple,always-on; + }; }; /* diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 01486e0f5ac65..c054f5accf531 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -702,6 +702,48 @@ status = "disabled"; }; + /* + * iBoot hands over a running SEP. Keep Linux off its DART at boot; + * apple_sep enables it when the driver binds. The SEP only accepts + * IOVAs in the window iBoot programmed (ADT vm-base 0x10000004000, + * vm-size 0x3ff00000), which carries the BIT(40) alias the DART + * translates. + */ + sep_dart: iommu@404c80000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x4 0x04c80000 0x0 0x20000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = /bits/ 64 <0x10000004000 0x3ff00000>; + apple,dma-offset = /bits/ 64 <0x0>; + status = "disabled"; + }; + + sep: sep@406600000 { + compatible = "apple,sep"; + reg = <0x4 0x06600000 0x0 0x88000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + power-domains = <&ps_sep>; + status = "disabled"; + }; + + sep_mbox: mailbox@406608000 { + compatible = "apple,t6041-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x06608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + ans_mbox: mailbox@409608000 { compatible = "apple,t6041-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x4 0x09608000 0x0 0x4000>; @@ -827,6 +869,30 @@ status = "disabled"; }; + /* + * Touch ID. iBoot leaves the pads muxed and the controller uses its + * native chip select, so there is no pinctrl here. + */ + spi2: spi@429108000 { + compatible = "apple,t6041-spi", "apple,t8103-spi"; + reg = <0x4 0x29108000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clkref>; + power-domains = <&ps_spi2>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + mesa: fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpol; + status = "disabled"; + }; + }; + nvme: nvme@44dcc0000 { compatible = "apple,t6041-nvme-ans2"; reg = <0x4 0x4dcc0000 0x0 0x10000>, From a1153e73f1002a56bca81d564b10af77b1a280ba Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 23:30:10 +1100 Subject: [PATCH 1303/1579] arm64: dts: apple: t6040-j616s: Enable Touch ID Give the boot loader the sep alias it reserves the SEP firmware through, enable the SEP mailbox and spi2, and describe the sensor: NUB GPIO 53 powers it, AP GPIO 116 is its data-ready line, and its sealed calibration and root key come from this Mac's own iBoot system container. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index c47e71ed1e258..500d9fe370ea9 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -29,6 +29,8 @@ disp0-piodma = &disp0_piodma; /* the boot loader finds the GPU node through this alias */ gpu = &gpu; + /* the boot loader reserves the SEP firmware through this alias */ + sep = &sep; }; chosen { @@ -1350,6 +1352,33 @@ status = "okay"; }; +/* apple_sep enables the SEP and its DART when it loads. */ +&sep_mbox { + status = "okay"; +}; + +/* + * Touch ID sensor (Mesa 0x3356). Its power is NUB GPIO 53 and its + * data-ready line is AP GPIO 116, as the J616s ADT describes them. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_nub 53 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl>; + interrupts = <116 IRQ_TYPE_EDGE_RISING>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + /* + * Per-device factory data from this Mac's own iBoot system container: + * the sealed calibration and the sealed root key. + */ + firmware-name = "apple/mesacal-j616s.bin", "apple/msrk-j616s.bin"; +}; + /* The Type-C retimers (ADT atcrt at 0x18/0x19/0x1a) sit on this bus; no driver yet. */ &i2c6 { status = "okay"; From 65f748be0452ad6d80a41d1fc063eeac9867db32 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:02:20 +1100 Subject: [PATCH 1304/1579] arm64: dts: apple: describe the remaining T6040 PMGR power states Describe the power-state registers and their parent graph. Leave providers without a qualified Linux consumer disabled, since registering an unused domain may power down hardware still owned by firmware. Keep the existing tested providers unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-pmgr.dtsi | 1762 +++++++++++++++++++++ arch/arm64/boot/dts/apple/t6040.dtsi | 21 + 2 files changed, 1783 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi index bf357745d581d..18ec5df7d33bf 100644 --- a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi @@ -506,3 +506,1765 @@ power-domains = <&ps_atc2_phymxwrap>; }; }; + +/* + * Remaining ADT power states. Keep providers without a qualified consumer + * disabled: registering an unused genpd can power down firmware-owned blocks. + * Enable the provider and its parents together with the consumer. + */ +&pmgr { + ps_apcie_gp: power-controller@100 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_gp"; + status = "disabled"; + }; + + ps_fab4_soc: power-controller@128 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fab4_soc"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_sio_cpu: power-controller@130 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x130 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sio_cpu"; + power-domains = <&ps_sio>; + apple,always-on; + status = "disabled"; + }; + + ps_fpwm1: power-controller@140 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm1"; + power-domains = <&ps_sio>; + status = "disabled"; + }; + + ps_fpwm2: power-controller@148 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm2"; + power-domains = <&ps_sio>; + status = "disabled"; + }; + + ps_i2c0: power-controller@150 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c0"; + power-domains = <&ps_sio>; + status = "disabled"; + }; + + ps_i2c5: power-controller@178 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x178 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c5"; + power-domains = <&ps_sio>; + status = "disabled"; + }; + + ps_i2c7: power-controller@188 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x188 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c7"; + power-domains = <&ps_sio>; + status = "disabled"; + }; + + ps_spi_p: power-controller@198 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x198 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + power-domains = <&ps_sio>; + status = "disabled"; + }; + + ps_uart_p: power-controller@1a0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_p"; + power-domains = <&ps_sio>; + status = "disabled"; + }; + + ps_dpa_p: power-controller@1a8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa_p"; + power-domains = <&ps_sio>; + apple,always-on; + status = "disabled"; + }; + + ps_aes: power-controller@1b0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aes"; + power-domains = <&ps_sio>; + apple,always-on; + status = "disabled"; + }; + + ps_spi0: power-controller@1b8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi0"; + power-domains = <&ps_spi_p>; + status = "disabled"; + }; + + ps_spi1: power-controller@1c0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi1"; + power-domains = <&ps_spi_p>; + status = "disabled"; + }; + + ps_spi2: power-controller@1c8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi2"; + power-domains = <&ps_spi_p>; + status = "disabled"; + }; + + ps_spi3: power-controller@1d0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi3"; + power-domains = <&ps_spi_p>; + status = "disabled"; + }; + + ps_spi4: power-controller@1d8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi4"; + power-domains = <&ps_spi_p>; + status = "disabled"; + }; + + ps_spi5: power-controller@1e0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi5"; + power-domains = <&ps_spi_p>; + status = "disabled"; + }; + + ps_qspi: power-controller@1e8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "qspi"; + power-domains = <&ps_spi_p>; + status = "disabled"; + }; + + ps_uart_n: power-controller@1f0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_n"; + power-domains = <&ps_uart_p>; + status = "disabled"; + }; + + ps_uart0: power-controller@1f8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart0"; + power-domains = <&ps_uart_p>; + status = "disabled"; + }; + + ps_uart1: power-controller@200 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x200 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart1"; + power-domains = <&ps_uart_p>; + status = "disabled"; + }; + + ps_uart2: power-controller@208 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x208 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart2"; + power-domains = <&ps_uart_p>; + status = "disabled"; + }; + + ps_uart3: power-controller@210 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x210 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart3"; + power-domains = <&ps_uart_p>; + status = "disabled"; + }; + + ps_uart4: power-controller@218 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x218 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart4"; + power-domains = <&ps_uart_p>; + status = "disabled"; + }; + + ps_uart5: power-controller@220 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x220 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart5"; + power-domains = <&ps_uart_p>; + status = "disabled"; + }; + + ps_uart6: power-controller@228 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x228 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart6"; + power-domains = <&ps_uart_p>; + status = "disabled"; + }; + + ps_dpa0: power-controller@230 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x230 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa0"; + power-domains = <&ps_dpa_p>; + status = "disabled"; + }; + + ps_dpa1: power-controller@238 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x238 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa1"; + power-domains = <&ps_dpa_p>; + status = "disabled"; + }; + + ps_dpa2: power-controller@240 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x240 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa2"; + power-domains = <&ps_dpa_p>; + status = "disabled"; + }; + + ps_dpa3: power-controller@248 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x248 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa3"; + power-domains = <&ps_dpa_p>; + status = "disabled"; + }; + + ps_dpa4: power-controller@250 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x250 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa4"; + power-domains = <&ps_dpa_p>; + status = "disabled"; + }; + + ps_disp_sys: power-controller@258 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x258 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_sys"; + power-domains = <&ps_fab6_soc>; + status = "disabled"; + }; + + ps_amcc4: power-controller@260 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x260 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc4"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_amcc5: power-controller@268 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x268 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc5"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_amcc6: power-controller@270 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x270 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc6"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_amcc7: power-controller@278 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x278 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc7"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_amcc12: power-controller@280 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x280 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc12"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_amcc13: power-controller@288 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc13"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_amcc14: power-controller@290 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x290 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc14"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_amcc15: power-controller@298 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x298 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc15"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_04: power-controller@2e0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_04"; + power-domains = <&ps_amcc4>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_05: power-controller@2e8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_05"; + power-domains = <&ps_amcc5>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_06: power-controller@2f0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_06"; + power-domains = <&ps_amcc6>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_07: power-controller@2f8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_07"; + power-domains = <&ps_amcc7>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_12: power-controller@300 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_12"; + power-domains = <&ps_amcc12>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_13: power-controller@308 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x308 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_13"; + power-domains = <&ps_amcc13>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_14: power-controller@310 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x310 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_14"; + power-domains = <&ps_amcc14>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_15: power-controller@318 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x318 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_15"; + power-domains = <&ps_amcc15>; + apple,always-on; + status = "disabled"; + }; + + ps_disp_fe: power-controller@368 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x368 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_fe"; + power-domains = <&ps_disp_sys>; + status = "disabled"; + }; + + ps_apcie_sys_gp: power-controller@390 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_gp"; + power-domains = <&ps_fab3_soc>, <&ps_apcie_gp>; + status = "disabled"; + }; + + ps_apcie_phy_sw: power-controller@398 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x398 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_phy_sw"; + power-domains = <&ps_apcie_sys_st0>, <&ps_apcie_sys_st1>; + status = "disabled"; + }; + + ps_jpg: power-controller@3a0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "jpg"; + power-domains = <&ps_fab4_soc>; + status = "disabled"; + }; + + ps_ane_sys: power-controller@3c0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_sys"; + power-domains = <&ps_fab5_soc>, <&ps_fab4_soc>; + status = "disabled"; + }; + + ps_sep: power-controller@c00 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc00 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sep"; + power-domains = <&ps_fab3_soc>; + apple,always-on; + status = "disabled"; + }; +}; + +&pmgr_aux { + ps_ecpu_0: power-controller@0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ecpu_0"; + status = "disabled"; + }; + + ps_ecpu_1: power-controller@8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ecpu_1"; + status = "disabled"; + }; + + ps_ecpu_2: power-controller@10 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x10 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ecpu_2"; + status = "disabled"; + }; + + ps_ecpu_3: power-controller@18 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x18 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ecpu_3"; + status = "disabled"; + }; + + ps_pcpu0_0: power-controller@20 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x20 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpu0_0"; + status = "disabled"; + }; + + ps_pcpu0_1: power-controller@28 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x28 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpu0_1"; + status = "disabled"; + }; + + ps_pcpu0_2: power-controller@30 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x30 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpu0_2"; + status = "disabled"; + }; + + ps_pcpu0_3: power-controller@38 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x38 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpu0_3"; + status = "disabled"; + }; + + ps_pcpu0_4: power-controller@40 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpu0_4"; + status = "disabled"; + }; + + ps_pcpu0_5: power-controller@48 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x48 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpu0_5"; + status = "disabled"; + }; + + ps_pcpu1_0: power-controller@50 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x50 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpu1_0"; + status = "disabled"; + }; + + ps_pcpu1_1: power-controller@58 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x58 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpu1_1"; + status = "disabled"; + }; + + ps_pcpu1_2: power-controller@60 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpu1_2"; + status = "disabled"; + }; + + ps_pcpu1_3: power-controller@68 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x68 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpu1_3"; + status = "disabled"; + }; + + ps_pcpu1_4: power-controller@70 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x70 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpu1_4"; + status = "disabled"; + }; + + ps_pcpu1_5: power-controller@78 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x78 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpu1_5"; + status = "disabled"; + }; + + ps_ecpm: power-controller@80 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x80 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ecpm"; + status = "disabled"; + }; + + ps_pcpm0: power-controller@88 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x88 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpm0"; + status = "disabled"; + }; + + ps_pcpm1: power-controller@90 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x90 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpm1"; + status = "disabled"; + }; + + ps_dwi: power-controller@108 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dwi"; + status = "disabled"; + }; + + ps_aic: power-controller@118 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aic"; + power-domains = <&ps_pms>; + status = "disabled"; + }; + + ps_pmc: power-controller@120 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmc"; + power-domains = <&ps_pms>; + apple,always-on; + status = "disabled"; + }; + + ps_pms_c1ppt: power-controller@128 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_c1ppt"; + power-domains = <&ps_pms>; + apple,always-on; + status = "disabled"; + }; + + ps_soc_dpe: power-controller@130 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x130 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_dpe"; + power-domains = <&ps_pms>; + apple,always-on; + status = "disabled"; + }; + + ps_pmgr_soc_ocla: power-controller@138 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x138 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmgr_soc_ocla"; + status = "disabled"; + }; + + ps_pms_fpwm0: power-controller@140 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm0"; + power-domains = <&ps_pms>; + apple,always-on; + status = "disabled"; + }; + + ps_pms_fpwm1: power-controller@148 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm1"; + power-domains = <&ps_pms>; + apple,always-on; + status = "disabled"; + }; + + ps_pms_fpwm2: power-controller@150 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm2"; + power-domains = <&ps_pms>; + apple,always-on; + status = "disabled"; + }; + + ps_pms_fpwm3: power-controller@158 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm3"; + power-domains = <&ps_pms>; + apple,always-on; + status = "disabled"; + }; + + ps_pms_fpwm4: power-controller@160 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x160 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm4"; + power-domains = <&ps_pms>; + apple,always-on; + status = "disabled"; + }; + + ps_fab0_soc: power-controller@168 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fab0_soc"; + power-domains = <&ps_afi>; + apple,always-on; + status = "disabled"; + }; + + ps_amcc0: power-controller@180 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x180 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc0"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_00: power-controller@188 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x188 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_00"; + power-domains = <&ps_amcc0>; + apple,always-on; + status = "disabled"; + }; + + ps_rom: power-controller@190 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "rom"; + power-domains = <&ps_amcc0>; + status = "disabled"; + }; + + ps_amcc1: power-controller@198 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x198 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc1"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_01: power-controller@1a0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_01"; + power-domains = <&ps_amcc1>; + apple,always-on; + status = "disabled"; + }; + + ps_amcc2: power-controller@1a8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc2"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_02: power-controller@1b0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_02"; + power-domains = <&ps_amcc2>; + apple,always-on; + status = "disabled"; + }; + + ps_amcc3: power-controller@1b8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc3"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_03: power-controller@1c0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_03"; + power-domains = <&ps_amcc3>; + apple,always-on; + status = "disabled"; + }; + + ps_amcc8: power-controller@1c8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc8"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_08: power-controller@1d0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_08"; + power-domains = <&ps_amcc8>; + apple,always-on; + status = "disabled"; + }; + + ps_amcc9: power-controller@1d8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc9"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_09: power-controller@1e0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_09"; + power-domains = <&ps_amcc9>; + apple,always-on; + status = "disabled"; + }; + + ps_amcc10: power-controller@1e8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc10"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_10: power-controller@1f0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_10"; + power-domains = <&ps_amcc10>; + apple,always-on; + status = "disabled"; + }; + + ps_amcc11: power-controller@1f8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "amcc11"; + power-domains = <&ps_afc>; + apple,always-on; + status = "disabled"; + }; + + ps_dcs_11: power-controller@200 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x200 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs_11"; + power-domains = <&ps_amcc11>; + apple,always-on; + status = "disabled"; + }; + + ps_venc0_sys: power-controller@278 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x278 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_sys"; + power-domains = <&ps_fab0_soc>; + status = "disabled"; + }; + + ps_fab_gw_0: power-controller@288 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fab_gw_0"; + apple,always-on; + status = "disabled"; + }; + + ps_fab_gw_1: power-controller@290 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x290 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fab_gw_1"; + apple,always-on; + status = "disabled"; + }; + + ps_fab_afr: power-controller@298 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x298 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fab_afr"; + apple,always-on; + status = "disabled"; + }; + + ps_fab1_soc: power-controller@2d0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fab1_soc"; + power-domains = <&ps_afi>; + apple,always-on; + status = "disabled"; + }; + + ps_scodec: power-controller@328 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x328 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "scodec"; + power-domains = <&ps_fab1_soc>; + status = "disabled"; + }; + + ps_scodec_streaming: power-controller@330 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "scodec_streaming"; + power-domains = <&ps_scodec>; + status = "disabled"; + }; + + ps_fab2_soc: power-controller@338 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x338 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fab2_soc"; + power-domains = <&ps_afi>; + apple,always-on; + status = "disabled"; + }; + + ps_dispext0_sys: power-controller@340 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x340 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_sys"; + power-domains = <&ps_fab2_soc>; + status = "disabled"; + }; + + ps_dispext0_fe: power-controller@348 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x348 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_fe"; + power-domains = <&ps_dispext0_sys>; + status = "disabled"; + }; + + ps_dispext0_cpu: power-controller@350 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x350 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_cpu"; + power-domains = <&ps_dispext0_fe>; + status = "disabled"; + }; + + ps_atc0_cio: power-controller@398 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x398 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_cio"; + power-domains = <&ps_atc0_common>; + status = "disabled"; + }; + + ps_atc0_pcie: power-controller@3a0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_pcie"; + power-domains = <&ps_atc0_common>; + status = "disabled"; + }; + + ps_atc0_cio_pcie: power-controller@3a8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_cio_pcie"; + power-domains = <&ps_atc0_cio>; + status = "disabled"; + }; + + ps_atc0_cio_usb: power-controller@3b0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_cio_usb"; + power-domains = <&ps_atc0_cio>; + status = "disabled"; + }; + + ps_atc1_cio: power-controller@3c0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_cio"; + power-domains = <&ps_atc1_common>; + status = "disabled"; + }; + + ps_atc1_pcie: power-controller@3c8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_pcie"; + power-domains = <&ps_atc1_common>; + status = "disabled"; + }; + + ps_atc1_cio_pcie: power-controller@3d0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_cio_pcie"; + power-domains = <&ps_atc1_cio>; + status = "disabled"; + }; + + ps_atc1_cio_usb: power-controller@3d8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_cio_usb"; + power-domains = <&ps_atc1_cio>; + status = "disabled"; + }; + + ps_atc2_cio: power-controller@3e8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc2_cio"; + power-domains = <&ps_atc2_common>; + status = "disabled"; + }; + + ps_atc2_pcie: power-controller@3f0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc2_pcie"; + power-domains = <&ps_atc2_common>; + status = "disabled"; + }; + + ps_atc2_cio_pcie: power-controller@3f8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc2_cio_pcie"; + power-domains = <&ps_atc2_cio>; + status = "disabled"; + }; + + ps_atc2_cio_usb: power-controller@400 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x400 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc2_cio_usb"; + power-domains = <&ps_atc2_cio>; + status = "disabled"; + }; + + ps_atc3_common: power-controller@408 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x408 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_common"; + power-domains = <&ps_fab2_soc>; + status = "disabled"; + }; + + ps_atc3_cio: power-controller@420 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x420 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_cio"; + power-domains = <&ps_atc3_common>; + status = "disabled"; + }; + + ps_atc3_pcie: power-controller@428 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x428 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_pcie"; + power-domains = <&ps_atc3_common>; + status = "disabled"; + }; + + ps_atc3_cio_pcie: power-controller@430 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x430 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_cio_pcie"; + power-domains = <&ps_atc3_cio>; + status = "disabled"; + }; + + ps_atc3_cio_usb: power-controller@438 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x438 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_cio_usb"; + power-domains = <&ps_atc3_cio>; + status = "disabled"; + }; + + ps_fab5_soc: power-controller@440 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x440 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fab5_soc"; + power-domains = <&ps_afi>; + apple,always-on; + status = "disabled"; + }; + + ps_sbr: power-controller@448 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x448 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sbr"; + power-domains = <&ps_fab5_soc>; + apple,always-on; + status = "disabled"; + }; + + ps_gpio: power-controller@450 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x450 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gpio"; + power-domains = <&ps_sbr>; + status = "disabled"; + }; + + ps_msg: power-controller@458 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x458 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msg"; + power-domains = <&ps_sbr>; + status = "disabled"; + }; + + ps_ap_tmm: power-controller@468 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x468 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ap_tmm"; + power-domains = <&ps_sbr>; + status = "disabled"; + }; + + ps_msr0: power-controller@470 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x470 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr0"; + power-domains = <&ps_fab5_soc>; + status = "disabled"; + }; + + ps_msr0_ase_core: power-controller@478 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x478 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr0_ase_core"; + power-domains = <&ps_msr0>; + status = "disabled"; + }; + + ps_fab7_soc: power-controller@488 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x488 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fab7_soc"; + power-domains = <&ps_afi>; + apple,always-on; + status = "disabled"; + }; + + ps_dispext1_sys: power-controller@490 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x490 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext1_sys"; + power-domains = <&ps_fab7_soc>; + status = "disabled"; + }; + + ps_dispext1_fe: power-controller@498 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x498 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext1_fe"; + power-domains = <&ps_dispext1_sys>; + status = "disabled"; + }; + + ps_dispext1_cpu: power-controller@4a0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext1_cpu"; + power-domains = <&ps_dispext1_fe>; + status = "disabled"; + }; +}; + +&pmgr_ane { + ps_ane_mpm: power-controller@0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_mpm"; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + ps_ane_cpu: power-controller@8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_cpu"; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + ps_ane_td: power-controller@10 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x10 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_td"; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + ps_ane_base: power-controller@18 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x18 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_base"; + power-domains = <&ps_ane_td>; + status = "disabled"; + }; +}; + +&pmgr_mini { + ps_debug_gated: power-controller@0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_gated"; + apple,always-on; + status = "disabled"; + }; + + ps_nub_spmi_ahb: power-controller@58 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x58 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi_ahb"; + apple,always-on; + status = "disabled"; + }; + + ps_nub_spmi0: power-controller@60 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi0"; + power-domains = <&ps_nub_spmi_ahb>; + apple,always-on; + status = "disabled"; + }; + + ps_nub_spmi_a2: power-controller@78 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x78 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi_a2"; + apple,always-on; + status = "disabled"; + }; + + ps_nub_spi0: power-controller@80 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x80 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spi0"; + apple,always-on; + status = "disabled"; + }; + + ps_nub_aon: power-controller@88 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x88 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_aon"; + apple,always-on; + status = "disabled"; + }; + + ps_nub_ocla: power-controller@98 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x98 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_ocla"; + apple,always-on; + status = "disabled"; + }; + + ps_atc3_common_dp: power-controller@b8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_common_dp"; + status = "disabled"; + }; + + ps_nub_spmi1: power-controller@c0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi1"; + power-domains = <&ps_nub_spmi_ahb>; + apple,always-on; + status = "disabled"; + }; + + ps_atc3_usb_aon: power-controller@e0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xe0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_usb_aon"; + status = "disabled"; + }; + + ps_nub_spmi2: power-controller@e8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xe8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi2"; + power-domains = <&ps_nub_spmi_ahb>; + apple,always-on; + status = "disabled"; + }; + + ps_nub_spmi3: power-controller@f0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xf0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi3"; + power-domains = <&ps_nub_spmi_ahb>; + apple,always-on; + status = "disabled"; + }; + + ps_nub_spmi4: power-controller@f8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xf8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi4"; + power-domains = <&ps_nub_spmi_ahb>; + apple,always-on; + status = "disabled"; + }; + + ps_nub_fabric: power-controller@100 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_fabric"; + apple,always-on; + status = "disabled"; + }; + + ps_nub_sram: power-controller@108 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_sram"; + power-domains = <&ps_nub_fabric>; + apple,always-on; + status = "disabled"; + }; + + ps_debug_switch: power-controller@110 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_switch"; + apple,always-on; + status = "disabled"; + }; + + ps_atc3_usb: power-controller@148 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_usb"; + power-domains = <&ps_atc3_common>, <&ps_atc3_usb_aon>; + status = "disabled"; + }; + + ps_atc3_phymxwrap: power-controller@168 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_phymxwrap"; + power-domains = <&ps_atc3_common>, <&ps_atc3_usb_aon>; + status = "disabled"; + }; +}; + +&pmgr_disp { + ps_disp_cpu: power-controller@0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_cpu"; + power-domains = <&ps_disp_fe>; + status = "disabled"; + }; +}; + +&pmgr_gfx { + ps_gpx: power-controller@0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gpx"; + apple,always-on; + status = "disabled"; + }; +}; + +&pmgr_venc { + ps_venc0_dma: power-controller@0 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_dma"; + power-domains = <&ps_venc0_sys>; + status = "disabled"; + }; + + ps_venc0_pipe4: power-controller@8 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_pipe4"; + power-domains = <&ps_venc0_dma>; + status = "disabled"; + }; + + ps_venc0_pipe5: power-controller@10 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x10 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_pipe5"; + power-domains = <&ps_venc0_dma>; + status = "disabled"; + }; + + ps_venc0_me0: power-controller@18 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x18 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_me0"; + power-domains = <&ps_venc0_dma>; + status = "disabled"; + }; + + ps_venc0_me1: power-controller@20 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x20 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc0_me1"; + power-domains = <&ps_venc0_me0>; + status = "disabled"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index c054f5accf531..ba79030262ad6 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -1030,6 +1030,27 @@ /* child nodes are added in t6040-pmgr.dtsi */ }; + pmgr_ane: power-management@50228c000 { + compatible = "apple,t6041-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <0x5 0x0228c000 0x0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + }; + + pmgr_disp: power-management@502290000 { + compatible = "apple,t6041-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <0x5 0x02290000 0x0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + }; + + pmgr_venc: power-management@502808000 { + compatible = "apple,t6041-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <0x5 0x02808000 0x0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + }; + pmgr_isp: power-management@502284000 { compatible = "apple,t6041-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; #address-cells = <1>; From 327ef44386db8a77ce53edce35b6317ef1cb3259 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:04:10 +1100 Subject: [PATCH 1305/1579] arm64: dts: apple: describe the remaining T6040 DART banks Describe the separate ANE and AVE translation banks, PCIe port 1, SIO, external displays, scaler, APR and JPEG IOMMUs. Keep each disabled until its DMA consumer owns its power and streams. Shared DAPF and remap apertures are not independent IOMMU instances. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040.dtsi | 173 +++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index ba79030262ad6..fb4cde463cc2d 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -1808,6 +1808,179 @@ phy-names = "usb2-phy", "usb3-phy"; status = "disabled"; }; + /* Additional DARTs; enabled together with their DMA consumers. */ + ave0_dart1: iommu@283080000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x2 0x83080000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_venc0_sys>; + status = "disabled"; + }; + + ave0_dart0: iommu@2830a0000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x2 0x830a0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_venc0_sys>; + status = "disabled"; + }; + + apr0_dart: iommu@301020000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x3 0x01020000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_scodec>; + status = "disabled"; + }; + + apciec0_dart: iommu@384000000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x3 0x84000000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + apciec1_dart: iommu@38c000000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x3 0x8c000000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + apciec2_dart: iommu@394000000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x3 0x94000000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + dispext0_dart: iommu@3a2300000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x3 0xa2300000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + dcpext0_dart: iommu@3a2340000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x3 0xa2340000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + apcie1_dart: iommu@411000000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x4 0x11000000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + sio_dart: iommu@429060000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x4 0x29060000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_sio>; + status = "disabled"; + }; + + ane0_dart0: iommu@485800000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x4 0x85800000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + ane0_dart1: iommu@485820000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x4 0x85820000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + ane0_dart2: iommu@485840000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x4 0x85840000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + jpeg0_dart: iommu@487020000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x4 0x87020000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + + jpeg1_dart: iommu@487080000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x4 0x87080000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + + scaler0_dart: iommu@521260000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x5 0x21260000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_msr0>; + status = "disabled"; + }; + + dispext1_dart: iommu@602300000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x6 0x02300000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + dcpext1_dart: iommu@602340000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x6 0x02340000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + }; #include "t6040-pmgr.dtsi" From c5da358f20e09da54d058cf08c9451dc64ba464e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:06:22 +1100 Subject: [PATCH 1306/1579] arm64: dts: apple: describe the remaining T6040 GPIO and SPMI controllers Add the AOP and SMC pin banks with the ADT pin counts and interrupt groups, and the four remaining NUB SPMI buses plus the AOP SPMI bus. Leave coprocessor-owned banks disabled until their ownership can be transferred. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040.dtsi | 89 ++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index fb4cde463cc2d..0a2d7104a103b 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -1981,6 +1981,95 @@ status = "disabled"; }; + pinctrl_aop: pinctrl@510824000 { + compatible = "apple,t6041-pinctrl", "apple,t8103-pinctrl"; + reg = <0x5 0x10824000 0x0 0x4000>; + clocks = <&clkref>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_aop 0 0 85>; + apple,npins = <85>; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + /* Coprocessor pins and interrupt groups need an ownership handoff. */ + status = "disabled"; + }; + + pinctrl_smc: pinctrl@50c824000 { + compatible = "apple,t6041-pinctrl", "apple,t8103-pinctrl"; + reg = <0x5 0x0c824000 0x0 0x4000>; + clocks = <&clkref>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_smc 0 0 24>; + apple,npins = <24>; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + /* Coprocessor pins and interrupt groups need an ownership handoff. */ + status = "disabled"; + }; + + nub_spmi1: spmi@509054000 { + compatible = "apple,t6041-spmi", "apple,t8103-spmi"; + reg = <0x5 0x09054000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + power-domains = <&ps_nub_spmi1>; + status = "disabled"; + }; + + nub_spmi2: spmi@509094000 { + compatible = "apple,t6041-spmi", "apple,t8103-spmi"; + reg = <0x5 0x09094000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + power-domains = <&ps_nub_spmi2>; + status = "disabled"; + }; + + nub_spmi3: spmi@5090d4000 { + compatible = "apple,t6041-spmi", "apple,t8103-spmi"; + reg = <0x5 0x090d4000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + power-domains = <&ps_nub_spmi3>; + status = "disabled"; + }; + + nub_spmi4: spmi@509114000 { + compatible = "apple,t6041-spmi", "apple,t8103-spmi"; + reg = <0x5 0x09114000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + power-domains = <&ps_nub_spmi4>; + status = "disabled"; + }; + + aop_spmi0: spmi@510914000 { + compatible = "apple,t6041-spmi", "apple,t8103-spmi"; + reg = <0x5 0x10914000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + /* Shared with AOP firmware; no AP bus ownership handoff yet. */ + status = "disabled"; + }; + }; #include "t6040-pmgr.dtsi" From 90e3fa40ecf297a0ecefd74f9a3a57428bb2f55e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:06:33 +1100 Subject: [PATCH 1307/1579] arm64: dts: apple: describe the J616s additional Abbey PMICs Expose the Abbey PMICs at USID 14 on NUB SPMI buses 1 and 2 through the existing NVMEM binding. Enable their bus power states and shared parent. Use polled bus access as on NUB SPMI0 so firmware-owned peripheral interrupts remain untouched. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 31 +++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 500d9fe370ea9..8b5156308068d 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -4360,3 +4360,34 @@ }; #include "t6040-j616s-usb3.dtsi" + +/* Additional Abbey PMICs on the two independent NUB SPMI buses. */ +&nub_spmi1 { + status = "okay"; + + pmic@e { + compatible = "apple,abbey-pmic", "apple,spmi-nvmem"; + reg = <0xe SPMI_USID>; + }; +}; + +&nub_spmi2 { + status = "okay"; + + pmic@e { + compatible = "apple,abbey-pmic", "apple,spmi-nvmem"; + reg = <0xe SPMI_USID>; + }; +}; + +&ps_nub_spmi_ahb { + status = "okay"; +}; + +&ps_nub_spmi1 { + status = "okay"; +}; + +&ps_nub_spmi2 { + status = "okay"; +}; From 706908f638494abe6d378f96db2b8a0716738e0a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:09:30 +1100 Subject: [PATCH 1308/1579] arm64: dts: apple: describe T6040 SIO, panel SPI and QSPIMC Describe the SIO DMA coprocessor and its mailbox, panel SPI controller and firmware NOR controller with their register, interrupt, clock and power resources. Extend the SIO and QSPIMC bindings for T6041. Keep them disabled pending firmware-table, panel-ownership and flash-write-policy qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/dma/apple,sio.yaml | 1 + .../devicetree/bindings/spi/apple,qspi.yaml | 1 + arch/arm64/boot/dts/apple/t6040.dtsi | 61 +++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/Documentation/devicetree/bindings/dma/apple,sio.yaml b/Documentation/devicetree/bindings/dma/apple,sio.yaml index dbaaa2d3d1a1a..ab6bdf3686971 100644 --- a/Documentation/devicetree/bindings/dma/apple,sio.yaml +++ b/Documentation/devicetree/bindings/dma/apple,sio.yaml @@ -23,6 +23,7 @@ properties: - enum: - apple,t6000-sio - apple,t6030-sio + - apple,t6041-sio - apple,t8103-sio - const: apple,sio diff --git a/Documentation/devicetree/bindings/spi/apple,qspi.yaml b/Documentation/devicetree/bindings/spi/apple,qspi.yaml index 39924afc9a8b6..0b98af1ad68d2 100644 --- a/Documentation/devicetree/bindings/spi/apple,qspi.yaml +++ b/Documentation/devicetree/bindings/spi/apple,qspi.yaml @@ -22,6 +22,7 @@ properties: enum: - apple,t8122-qspi - apple,t8132-qspi + - apple,t6041-qspi - apple,t8140-qspi reg: diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 0a2d7104a103b..1be0417d0a8b8 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -2070,6 +2070,67 @@ status = "disabled"; }; + /* The QSPIMC input is ADT clock 434 (200 MHz). */ + clk_200m: clock-200m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <200000000>; + clock-output-names = "clk_200m"; + }; + + spi4: spi@429110000 { + compatible = "apple,t6041-spi", "apple,t8103-spi"; + reg = <0x4 0x29110000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clkref>; + power-domains = <&ps_spi4>; + #address-cells = <1>; + #size-cells = <0>; + /* Panel/TCON control belongs to the display firmware. */ + status = "disabled"; + }; + + qspi: spi@429118000 { + compatible = "apple,t6041-qspi"; + reg = <0x4 0x29118000 0x0 0x4800>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clk_200m>; + power-domains = <&ps_qspi>; + #address-cells = <1>; + #size-cells = <0>; + /* The firmware NOR needs a qualified partition/write policy. */ + status = "disabled"; + }; + + /* The loader supplies the SIO firmware regions and protocol tables. */ + sio: dma-controller@429e00000 { + compatible = "apple,t6041-sio", "apple,sio"; + reg = <0x4 0x29e00000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&sio_mbox>; + iommus = <&sio_dart 0>; + power-domains = <&ps_sio_cpu>; + status = "disabled"; + }; + + sio_mbox: mailbox@429e08000 { + compatible = "apple,t6041-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x29e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_sio_cpu>; + status = "disabled"; + }; + }; #include "t6040-pmgr.dtsi" From 1821a91f9e88472630ec351d46b8066b02d08891 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:10:43 +1100 Subject: [PATCH 1309/1579] arm64: dts: apple: describe the T6040 external display processors Describe both external display coprocessors and mailboxes, their PIODMA streams, all four display crossbars and the fixed HDMI PHY with its mailbox. Keep them disabled until the loader can supply a qualified live display handoff. Admit the existing T6041 external DCP driver compatible in the binding and require its bandwidth handoff when enabled. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/display/apple,dcp.yaml | 18 ++- arch/arm64/boot/dts/apple/t6040.dtsi | 137 ++++++++++++++++++ 2 files changed, 151 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index d79ad457f7703..e9adc197d0dea 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -34,6 +34,7 @@ properties: - apple,t6000-dcpext - apple,t6020-dcpext - const: apple,dcpext + - const: apple,t6041-dcpext - items: - const: apple,t8140-dcpext - const: apple,dcp @@ -372,10 +373,19 @@ allOf: pattern: '^(dp-xbar|typec[0-9]+)$' - if: not: - properties: - compatible: - contains: - const: apple,t8132-dcp + anyOf: + - properties: + compatible: + contains: + const: apple,t8132-dcp + - properties: + compatible: + contains: + const: apple,t6041-dcpext + status: + const: disabled + required: + - status then: required: - apple,bw-scratch diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 1be0417d0a8b8..49e76b24ed648 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -2131,6 +2131,143 @@ status = "disabled"; }; + dcpext0: dcp@3a2e00000 { + compatible = "apple,t6041-dcpext"; + reg = <0x3 0xa2e00000 0x0 0x4000>, + <0x3 0xa0000000 0x0 0x4000000>, + <0x3 0xa2240000 0x0 0x10000>, + <0x3 0xa23a8000 0x0 0x4000>, + <0x3 0xa2800000 0x0 0x800000>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + mboxes = <&dcpext0_mbox>; + mbox-names = "mbox"; + iommus = <&dcpext0_dart 23>; + power-domains = <&ps_dispext0_cpu>; + /* The loader must supply the bandwidth and live display handoff. */ + status = "disabled"; + + dispext0_piodma: piodma { + iommus = <&dispext0_dart 16>; + }; + }; + + dcpext0_mbox: mailbox@3a2e08000 { + compatible = "apple,t6041-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0xa2e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_dispext0_cpu>; + status = "disabled"; + }; + + dcpext1: dcp@602e00000 { + compatible = "apple,t6041-dcpext"; + reg = <0x6 0x02e00000 0x0 0x4000>, + <0x6 0x00000000 0x0 0x4000000>, + <0x6 0x02240000 0x0 0x10000>, + <0x6 0x023a8000 0x0 0x4000>, + <0x6 0x02800000 0x0 0x800000>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + mboxes = <&dcpext1_mbox>; + mbox-names = "mbox"; + iommus = <&dcpext1_dart 23>; + power-domains = <&ps_dispext1_cpu>; + /* The loader must supply the bandwidth and live display handoff. */ + status = "disabled"; + + dispext1_piodma: piodma { + iommus = <&dispext1_dart 16>; + }; + }; + + dcpext1_mbox: mailbox@602e08000 { + compatible = "apple,t6041-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x6 0x02e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_dispext1_cpu>; + status = "disabled"; + }; + + atcphy0_xbar: mux-controller@38324c000 { + compatible = "apple,t6041-display-crossbar"; + reg = <0x3 0x8324c000 0x0 0x4000>; + #mux-control-cells = <1>; + status = "disabled"; + }; + + atcphy1_xbar: mux-controller@38b24c000 { + compatible = "apple,t6041-display-crossbar"; + reg = <0x3 0x8b24c000 0x0 0x4000>; + #mux-control-cells = <1>; + status = "disabled"; + }; + + atcphy2_xbar: mux-controller@39324c000 { + compatible = "apple,t6041-display-crossbar"; + reg = <0x3 0x9324c000 0x0 0x4000>; + #mux-control-cells = <1>; + status = "disabled"; + }; + + atcphy3_xbar: mux-controller@39b24c000 { + compatible = "apple,t6041-display-crossbar"; + reg = <0x3 0x9b24c000 0x0 0x4000>; + #mux-control-cells = <1>; + status = "disabled"; + }; + + /* The fourth ATC is a fixed DisplayPort output for the HDMI bridge. */ + atcphy3: phy@39aa90000 { + compatible = "apple,t6040-atcphy"; + reg = <0x3 0x9aa90000 0x0 0x4000>, + <0x3 0x9a800000 0x0 0x4000>, + <0x3 0x9b000000 0x0 0x30000>, + <0x3 0x9b200000 0x0 0x4000>, + <0x3 0x9b140000 0x0 0x4000>, + <0x3 0x9b0c0000 0x0 0x10000>, + <0x3 0x98000000 0x0 0x8000>, + <0x3 0x98020000 0x0 0x4000>, + <0x3 0x9b244000 0x0 0x4000>, + <0x3 0x9b250000 0x0 0x4000>; + reg-names = "bank0", "bank1", "core", "cfg", "cpu", "cpu-data", + "axi2af", "axi2af-lioa", "fabric", "common-cfg"; + #phy-cells = <1>; + #reset-cells = <0>; + power-domains = <&ps_atc3_common_dp>; + mboxes = <&atcphy3_mbox>; + mbox-names = "phy-cpu"; + nonposted-mmio; + status = "disabled"; + }; + + atcphy3_mbox: mailbox@39b148000 { + compatible = "apple,t6041-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x9b148000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_atc3_phymxwrap>; + status = "disabled"; + }; + }; #include "t6040-pmgr.dtsi" From ca1fcc36d96f73986e062912d83dfd7ad672888d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:12:20 +1100 Subject: [PATCH 1310/1579] arm64: dts: apple: make the J616s loader placeholders warning-free Leave GPU reserved-memory reg properties absent until the loader supplies their addresses. The loader sets these properties rather than updating them in place. Give the DCP child address space an explicit identity mapping, as on the other Apple display descriptions. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/display/apple,dcp.yaml | 2 ++ arch/arm64/boot/dts/apple/t6040-j616s.dts | 6 ------ arch/arm64/boot/dts/apple/t6040.dtsi | 1 + 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index e9adc197d0dea..ee14690fb6319 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -277,6 +277,8 @@ properties: '#size-cells': const: 2 + ranges: true + piodma: type: object properties: diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 8b5156308068d..ad27eacdc07dd 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -82,7 +82,6 @@ * segment-ranges[0]); the driver identifies the build from it. */ gpu_fw_text: gpu-fw-text { - reg = <0 0 0 0>; status = "disabled"; no-map; }; @@ -92,7 +91,6 @@ * places its crash log here. */ gpu_fw_data: gpu-fw-data { - reg = <0 0 0 0>; status = "disabled"; no-map; }; @@ -103,25 +101,21 @@ * the boot loader adds them to /memory. */ uat_ttbs: uat-ttbs { - reg = <0 0 0 0>; status = "disabled"; no-map; }; uat_pagetables: uat-pagetables { - reg = <0 0 0 0>; status = "disabled"; no-map; }; uat_l2_pagetables: uat-l2-pagetables { - reg = <0 0 0 0>; status = "disabled"; no-map; }; uat_handoff: uat-handoff { - reg = <0 0 0 0>; status = "disabled"; no-map; }; diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 49e76b24ed648..d35c5f9f03e1e 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -680,6 +680,7 @@ /* the boot loader describes the piodma's regions with 2+2 cells */ #address-cells = <2>; #size-cells = <2>; + ranges; /* enabled once the boot loader has handed the display over */ status = "disabled"; From 4c0bf2ae5c7180b83d2c3bc66422765f88b5990f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:14:05 +1100 Subject: [PATCH 1311/1579] arm64: dts: apple: describe the T6040 Neural Engine Describe the engine, its interrupt and three DART translation banks. Extend the ANE binding for the single-interrupt T6041 firmware interface and leave the engine disabled until that interface has a Linux backend. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/npu/apple,ane.yaml | 27 ++++++++++++++++--- arch/arm64/boot/dts/apple/t6040.dtsi | 12 +++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/npu/apple,ane.yaml b/Documentation/devicetree/bindings/npu/apple,ane.yaml index 79cf942654f7b..feae5b133ac2f 100644 --- a/Documentation/devicetree/bindings/npu/apple,ane.yaml +++ b/Documentation/devicetree/bindings/npu/apple,ane.yaml @@ -15,7 +15,9 @@ description: drives the engine through its register window. On T602x and T8112 a coprocessor runs the ANE firmware and the host talks to it through an ASC mailbox. T8140 has two interrupts and one system power domain; its - coprocessor uses a different firmware interface. + coprocessor uses a different firmware interface. T6041 likewise needs a + generation-specific firmware backend; it has one interrupt and its CPU + power state supplies the engine through the ANE system domain. properties: compatible: @@ -25,6 +27,7 @@ properties: - apple,t6020-ane - apple,t6021-ane - apple,t6022-ane + - apple,t6041-ane - apple,t8112-ane - apple,t8140-ane @@ -87,19 +90,35 @@ required: - power-domains allOf: + - if: + properties: + compatible: + const: apple,t8140-ane + then: + properties: + interrupts: + minItems: 2 + - if: + properties: + compatible: + const: apple,t6041-ane + then: + properties: + interrupts: + maxItems: 1 - if: properties: compatible: contains: - const: apple,t8140-ane + enum: + - apple,t8140-ane + - apple,t6041-ane then: properties: reg: maxItems: 1 reg-names: maxItems: 1 - interrupts: - minItems: 2 interrupt-names: false iommus: minItems: 3 diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index d35c5f9f03e1e..c83c8e8ce8f1f 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -2269,6 +2269,18 @@ status = "disabled"; }; + /* Firmware interface support is required before enabling the ANE. */ + ane: ane@484000000 { + compatible = "apple,t6041-ane"; + reg = <0x4 0x84000000 0x0 0x2000000>; + reg-names = "engine"; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&ane0_dart0 0>, <&ane0_dart1 0>, <&ane0_dart2 0>; + power-domains = <&ps_ane_cpu>; + status = "disabled"; + }; + }; #include "t6040-pmgr.dtsi" From e6f9117e65f4e9eb15c02c5a0b8f06c8c4704a1c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:14:14 +1100 Subject: [PATCH 1312/1579] arm64: dts: apple: describe the T6040 video encoder Describe the codec, mailbox and coprocessor windows, all five interrupts, the five hardware power states and both DART banks using the AVE binding. Keep it disabled until firmware support is available. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/media/apple,t8140-ave.yaml | 8 +++++--- arch/arm64/boot/dts/apple/t6040.dtsi | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml b/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml index 110ebbc3d09e2..61e10feb5df71 100644 --- a/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml +++ b/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml @@ -4,19 +4,21 @@ $id: http://devicetree.org/schemas/media/apple,t8140-ave.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple T8140 video encoder +title: Apple video encoder maintainers: - Miguel Cruz <7362641+Acelogic@users.noreply.github.com> description: | - The T8140 AVE coprocessor encodes video using firmware preloaded by the + The T6041 and T8140 AVE coprocessors encode video using firmware preloaded by the bootloader. Its two DART instances translate the same stream. The bootloader supplies two reserved firmware regions and their IOVA mappings. properties: compatible: - const: apple,t8140-ave + enum: + - apple,t6041-ave + - apple,t8140-ave reg: items: diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index c83c8e8ce8f1f..84e96c3a9135e 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -2281,6 +2281,26 @@ status = "disabled"; }; + /* The video encoder stays disabled until its firmware backend is available. */ + ave: video-codec@283100000 { + compatible = "apple,t6041-ave"; + reg = <0x2 0x83100000 0x0 0x45c000>, + <0x2 0x83050000 0x0 0x4000>, + <0x2 0x83800000 0x0 0x800000>; + reg-names = "codec", "mailbox", "coproc"; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + ; + power-domains = <&ps_venc0_dma>, <&ps_venc0_pipe4>, + <&ps_venc0_pipe5>, <&ps_venc0_me0>, <&ps_venc0_me1>; + power-domain-names = "dma", "pipe4", "pipe5", "me0", "me1"; + iommus = <&ave0_dart0 0>, <&ave0_dart1 0>; + status = "disabled"; + }; + }; #include "t6040-pmgr.dtsi" From d475557caa6287d0eefe05971ca9641bd830366b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:14:43 +1100 Subject: [PATCH 1313/1579] arm64: dts: apple: describe the J616s SD reader PCIe port Describe port 1, its interrupt, reset and clock-request pins and its separate DART mapping. Leave the port disabled because its SMC power switch is not a hexadecimal gPxx GPIO and requires a separate service binding. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index ad27eacdc07dd..132f7f3c38773 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -4385,3 +4385,36 @@ &ps_nub_spmi2 { status = "okay"; }; + +&pinctrl { + sd_clkreq_pins: sd-clkreq-pins { + pinmux = ; + }; +}; + +&pcie0 { + interrupts = , + ; + bus-range = <0 2>; + iommu-map = <0x100 &pcie_dart 1 2>, <0x200 &apcie1_dart 1 1>; + + port01: pci@1,0 { + device_type = "pci"; + reg = <0x800 0 0 0 0>; + reset-gpios = <&pinctrl 5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&sd_clkreq_pins>; + #address-cells = <3>; + #size-cells = <2>; + ranges; + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port01 0 0 0 0>, + <0 0 0 2 &port01 0 0 0 1>, + <0 0 0 3 &port01 0 0 0 2>, + <0 0 0 4 &port01 0 0 0 3>; + /* The SD power switch needs an SMC service binding. */ + status = "disabled"; + }; +}; From edb0d3fb6e3cdf4d462c66ce0711d7ae68102757 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:15:26 +1100 Subject: [PATCH 1314/1579] arm64: dts: apple: describe the J616s console transports and tether variant Move the UART and DockChannel descriptions into the SoC tree, disabled by default, and provide a tether variant that enables its consoles while leaving the host-owned ATC0 port untouched. Give the normal board the PMP alias needed for its firmware handoff as well. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/Makefile | 1 + .../boot/dts/apple/t6040-j616s-tether.dts | 45 +++++++++++++++++++ arch/arm64/boot/dts/apple/t6040-j616s.dts | 1 + arch/arm64/boot/dts/apple/t6040.dtsi | 25 +++++++++++ 4 files changed, 72 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t6040-j616s-tether.dts diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index 007e204fdbc56..0f6e6714f7de7 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -95,6 +95,7 @@ dtb-$(CONFIG_ARCH_APPLE) += t6032-j575d.dtb dtb-$(CONFIG_ARCH_APPLE) += t6034-j514m.dtb dtb-$(CONFIG_ARCH_APPLE) += t6034-j516m.dtb dtb-$(CONFIG_ARCH_APPLE) += t6040-j616s.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6040-j616s-tether.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j413.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j415.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j473.dtb diff --git a/arch/arm64/boot/dts/apple/t6040-j616s-tether.dts b/arch/arm64/boot/dts/apple/t6040-j616s-tether.dts new file mode 100644 index 0000000000000..619e09a88d091 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6040-j616s-tether.dts @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* J616s tether console; the host owns ATC0 during the debug session. */ +#include "t6040-j616s.dts" + +/ { + aliases { + serial0 = &serial0; + }; +}; + +&serial0 { + status = "okay"; +}; + +&ps_uart0 { + status = "okay"; +}; + +&dcuart { + status = "okay"; +}; + +&hpm0 { + status = "disabled"; +}; + +&atcphy0 { + status = "disabled"; +}; + +&atcphy0_mbox { + status = "disabled"; +}; + +&usb_drd0 { + status = "disabled"; +}; + +&usb_drd0_dart_0 { + status = "disabled"; +}; + +&usb_drd0_dart_1 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 132f7f3c38773..deaedfa9b12ef 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -29,6 +29,7 @@ disp0-piodma = &disp0_piodma; /* the boot loader finds the GPU node through this alias */ gpu = &gpu; + pmp = &pmp; /* the boot loader reserves the SEP firmware through this alias */ sep = &sep; }; diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 84e96c3a9135e..0b81ceb697d3a 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -2301,6 +2301,31 @@ status = "disabled"; }; + serial0: serial@429200000 { + compatible = "apple,s5l-uart"; + reg = <0x4 0x29200000 0x0 0x4000>; + reg-io-width = <4>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clkref>, <&clkref>; + clock-names = "uart", "clk_uart_baud0"; + power-domains = <&ps_uart0>; + status = "disabled"; + }; + + dcuart: serial@50883c000 { + compatible = "apple,dockchannel-uart"; + reg = <0x5 0x0883c000 0x0 0x4000>, + <0x5 0x08838000 0x0 0x4000>, + <0x5 0x0880c000 0x0 0x4000>; + reg-names = "data", "config", "irq"; + interrupt-parent = <&aic>; + interrupts = ; + apple,channel = <2>; + apple,sw-drain; + status = "disabled"; + }; + }; #include "t6040-pmgr.dtsi" From 658fbd4f89a35ea9668ec0ec4c8ca95c642db5d2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:19:09 +1100 Subject: [PATCH 1315/1579] arm64: dts: apple: complete J616s console supply and fixed-PHY constraints Enable the UART supply parent with the tether UART and omit USB reset-provider cells on the fixed DisplayPort PHY, as required by its binding. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s-tether.dts | 4 ++++ arch/arm64/boot/dts/apple/t6040.dtsi | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s-tether.dts b/arch/arm64/boot/dts/apple/t6040-j616s-tether.dts index 619e09a88d091..8027360847204 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s-tether.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s-tether.dts @@ -12,6 +12,10 @@ status = "okay"; }; +&ps_uart_p { + status = "okay"; +}; + &ps_uart0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 0b81ceb697d3a..2cf2f626c8d68 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -2246,7 +2246,6 @@ reg-names = "bank0", "bank1", "core", "cfg", "cpu", "cpu-data", "axi2af", "axi2af-lioa", "fabric", "common-cfg"; #phy-cells = <1>; - #reset-cells = <0>; power-domains = <&ps_atc3_common_dp>; mboxes = <&atcphy3_mbox>; mbox-names = "phy-cpu"; From 16b681646cf555d26833fd24464d47caf67dbb09 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:20:29 +1100 Subject: [PATCH 1316/1579] arm64: dts: apple: describe the T6040 ACIO mailboxes and DARTs Describe all three USB4 coprocessor mailboxes and their NHI translation banks with their interrupt and power resources. Keep them disabled until the host-router reset, SRAM and stream handoff is qualified. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040.dtsi | 75 ++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 2cf2f626c8d68..e101bd2596d78 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -2325,6 +2325,81 @@ status = "disabled"; }; + /* ACIO 0 remains disabled until its host-router handoff is supported. */ + usb4_0_mbox: mailbox@381108000 { + compatible = "apple,t6041-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x81108000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_atc0_cio>; + status = "disabled"; + }; + + usb4_0_dart: iommu@381a80000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x3 0x81a80000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + /* ACIO 1 remains disabled until its host-router handoff is supported. */ + usb4_1_mbox: mailbox@389108000 { + compatible = "apple,t6041-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x89108000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_atc1_cio>; + status = "disabled"; + }; + + usb4_1_dart: iommu@389a80000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x3 0x89a80000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + /* ACIO 2 remains disabled until its host-router handoff is supported. */ + usb4_2_mbox: mailbox@391108000 { + compatible = "apple,t6041-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x91108000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_atc2_cio>; + status = "disabled"; + }; + + usb4_2_dart: iommu@391a80000 { + compatible = "apple,t6041-dart", "apple,t8110-dart"; + reg = <0x3 0x91a80000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + }; #include "t6040-pmgr.dtsi" From 82f3d4154f1542d0dd19331950cc803ec489774b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:26:18 +1100 Subject: [PATCH 1317/1579] arm64: dts: apple: complete active T6040 power-state parent links Connect the three ATC common states, AVD, PMP and PMS SRAM to their ADT hardware parents. Enable the two missing fabric providers, both marked critical by the ADT, so active consumers never depend on disabled suppliers. Retain the existing additional PHY wrapper ordering dependencies. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-pmgr.dtsi | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi index 18ec5df7d33bf..eab3825a95c26 100644 --- a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi @@ -297,6 +297,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "avd_sys"; + power-domains = <&ps_fab0_soc>; }; ps_afc: power-controller@178 { @@ -315,6 +316,7 @@ #reset-cells = <0>; label = "pmp"; apple,always-on; + power-domains = <&ps_pms>; }; ps_pms_sram: power-controller@418 { @@ -324,6 +326,7 @@ #reset-cells = <0>; label = "pms_sram"; apple,always-on; + power-domains = <&ps_pms>; }; ps_fab6_soc: power-controller@480 { @@ -358,6 +361,7 @@ #reset-cells = <0>; label = "atc0_common"; apple,always-on; + power-domains = <&ps_fab2_soc>; }; ps_atc1_common: power-controller@3b8 { @@ -367,6 +371,7 @@ #reset-cells = <0>; label = "atc1_common"; apple,always-on; + power-domains = <&ps_fab2_soc>; }; ps_atc2_common: power-controller@3e0 { @@ -376,6 +381,7 @@ #reset-cells = <0>; label = "atc2_common"; apple,always-on; + power-domains = <&ps_fab2_soc>; }; }; @@ -1380,7 +1386,6 @@ label = "fab0_soc"; power-domains = <&ps_afi>; apple,always-on; - status = "disabled"; }; ps_amcc0: power-controller@180 { @@ -1648,7 +1653,6 @@ label = "fab2_soc"; power-domains = <&ps_afi>; apple,always-on; - status = "disabled"; }; ps_dispext0_sys: power-controller@340 { From 40cb9c2f490063278df238b72213d3a2051b6a8f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:56:32 +1100 Subject: [PATCH 1318/1579] arm64: dts: apple: keep T6041 SIO on its own firmware compatible The T6041 firmware protocol has not been qualified against the legacy SIO backend. Describe its resources with a generation-specific compatible, so a loader cannot accidentally bind that backend by enabling the node. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/dma/apple,sio.yaml | 16 +++++++++------- arch/arm64/boot/dts/apple/t6040.dtsi | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Documentation/devicetree/bindings/dma/apple,sio.yaml b/Documentation/devicetree/bindings/dma/apple,sio.yaml index ab6bdf3686971..495adc52c7e6e 100644 --- a/Documentation/devicetree/bindings/dma/apple,sio.yaml +++ b/Documentation/devicetree/bindings/dma/apple,sio.yaml @@ -19,13 +19,15 @@ allOf: properties: compatible: - items: - - enum: - - apple,t6000-sio - - apple,t6030-sio - - apple,t6041-sio - - apple,t8103-sio - - const: apple,sio + oneOf: + - items: + - enum: + - apple,t6000-sio + - apple,t6030-sio + - apple,t8103-sio + - const: apple,sio + - description: T6041 firmware requires a generation-specific backend. + const: apple,t6041-sio reg: maxItems: 1 diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index e101bd2596d78..75c9b3f06ac21 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -2107,7 +2107,7 @@ /* The loader supplies the SIO firmware regions and protocol tables. */ sio: dma-controller@429e00000 { - compatible = "apple,t6041-sio", "apple,sio"; + compatible = "apple,t6041-sio"; reg = <0x4 0x29e00000 0x0 0x8000>; dma-channels = <128>; #dma-cells = <1>; From 9f1ce86eae38113b8bdb147b4daa8adda0d869cc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 16:14:43 +1100 Subject: [PATCH 1319/1579] arm64: dts: apple: retain the active J616s PCIe scan range Keep the existing host bus range while the SD port is disabled. Generic ECAM access does not filter unavailable OF ports, so describing that port must not expand downstream configuration probing before its power handoff is qualified. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index deaedfa9b12ef..e24205e1d40da 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -4396,7 +4396,7 @@ &pcie0 { interrupts = , ; - bus-range = <0 2>; + /* Widen bus-range only with the SD power/port ownership handoff. */ iommu-map = <0x100 &pcie_dart 1 2>, <0x200 &apcie1_dart 1 1>; port01: pci@1,0 { From c744beabc5e71f58b58d1baafb9637307e8580bc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 17:45:07 +1100 Subject: [PATCH 1320/1579] ASoC: apple: macaudio: expose speaker variant straps from the device tree Allow a machine card to describe speaker identity inputs with speaker-id-gpios, least significant bit first. Expose their logical levels in a read-only volatile Speaker Variant ID control. Cards without the property retain their existing controls. Describe the two J616s speaker variant inputs on AP GPIO 52 and 53. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/sound/apple,macaudio.yaml | 9 +++ arch/arm64/boot/dts/apple/t6040-j616s.dts | 2 + sound/soc/apple/macaudio.c | 60 +++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/Documentation/devicetree/bindings/sound/apple,macaudio.yaml b/Documentation/devicetree/bindings/sound/apple,macaudio.yaml index fdf3d1883a99d..755c5d195cf0f 100644 --- a/Documentation/devicetree/bindings/sound/apple,macaudio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,macaudio.yaml @@ -36,6 +36,15 @@ properties: Model name for presentation to users $ref: /schemas/types.yaml#/definitions/string + speaker-id-gpios: + minItems: 1 + maxItems: 8 + description: + Inputs encoding the fitted speaker assembly variant, least significant + bit first. The machine driver exposes their logical levels through the + read-only Speaker Variant ID control. GPIO polarity flags define each + bit's logical level. + patternProperties: "^dai-link(@[0-9a-f]+)?$": description: | diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index e24205e1d40da..f4b0aa85e2758 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -1258,6 +1258,8 @@ sound: sound { compatible = "apple,j616-macaudio", "apple,macaudio"; model = "MacBook Pro J616"; + speaker-id-gpios = <&pinctrl 52 GPIO_ACTIVE_HIGH>, + <&pinctrl 53 GPIO_ACTIVE_HIGH>; #address-cells = <1>; #size-cells = <0>; diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index cc1123e10f398..b5511ed056887 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -62,6 +63,7 @@ #define MACAUDIO_NUM_FE_LINKS 3 /* milliseconds */ #define SPEAKER_LOCK_TIMEOUT 250 +#define MACAUDIO_MAX_SPEAKER_ID_GPIOS 8 enum macaudio_amp_type { AMP_NONE, @@ -166,6 +168,7 @@ struct macaudio_snd_data { bool has_sense; bool has_safety; unsigned int max_channels; + struct gpio_descs *speaker_id_gpios; struct macaudio_link_props { /* frontend props */ @@ -1430,6 +1433,50 @@ static const struct snd_soc_jack_pin macaudio_jack_pins[] = { }, }; +static int macaudio_speaker_id_info(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_info *uinfo) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; + uinfo->count = 1; + uinfo->value.integer.min = 0; + uinfo->value.integer.max = BIT(ma->speaker_id_gpios->ndescs) - 1; + uinfo->value.integer.step = 1; + + return 0; +} + +static int macaudio_speaker_id_get(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + unsigned int value = 0; + unsigned int i; + int ret; + + for (i = 0; i < ma->speaker_id_gpios->ndescs; i++) { + ret = gpiod_get_value_cansleep(ma->speaker_id_gpios->desc[i]); + if (ret < 0) + return ret; + if (ret) + value |= BIT(i); + } + uvalue->value.integer.value[0] = value; + + return 0; +} + +static const struct snd_kcontrol_new macaudio_speaker_id_control = { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .name = "Speaker Variant ID", + .access = SNDRV_CTL_ELEM_ACCESS_READ | SNDRV_CTL_ELEM_ACCESS_VOLATILE, + .info = macaudio_speaker_id_info, + .get = macaudio_speaker_id_get, +}; + static int macaudio_probe(struct snd_soc_card *card) { struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); @@ -1460,6 +1507,9 @@ static int macaudio_probe(struct snd_soc_card *card) snd_jack_set_key(ma->jack.jack, SND_JACK_BTN_2, KEY_VOLUMEDOWN); snd_jack_set_key(ma->jack.jack, SND_JACK_BTN_3, KEY_VOICECOMMAND); + if (ma->speaker_id_gpios) + return snd_soc_add_card_controls(card, &macaudio_speaker_id_control, 1); + return ret; } @@ -2183,6 +2233,16 @@ static int macaudio_snd_platform_probe(struct platform_device *pdev) if (ret) return ret; + data->speaker_id_gpios = devm_gpiod_get_array_optional(dev, "speaker-id", + GPIOD_IN); + if (IS_ERR(data->speaker_id_gpios)) + return dev_err_probe(dev, PTR_ERR(data->speaker_id_gpios), + "Failed to get speaker ID inputs\n"); + if (data->speaker_id_gpios && + (!data->speaker_id_gpios->ndescs || + data->speaker_id_gpios->ndescs > MACAUDIO_MAX_SPEAKER_ID_GPIOS)) + return dev_err_probe(dev, -EINVAL, "Invalid speaker ID input count\n"); + /* * Without sense the PCM2 link is gone, and a card whose front-ends * are not the MCA's (the J616 jack) has no PCM2 RX widget at all. From 4d5617e813a0db14124c9139df6ee7dabc75ec21 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 18:12:58 +1100 Subject: [PATCH 1321/1579] ASoC: apple: macaudio: append variant identity after existing controls Register the optional read-only identity control in late_probe, after the existing card and codec controls have been created. Their numeric identifiers remain stable when a board gains speaker-id-gpios. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/macaudio.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index b5511ed056887..96cd57aae29a8 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -1507,9 +1507,6 @@ static int macaudio_probe(struct snd_soc_card *card) snd_jack_set_key(ma->jack.jack, SND_JACK_BTN_2, KEY_VOLUMEDOWN); snd_jack_set_key(ma->jack.jack, SND_JACK_BTN_3, KEY_VOICECOMMAND); - if (ma->speaker_id_gpios) - return snd_soc_add_card_controls(card, &macaudio_speaker_id_control, 1); - return ret; } @@ -1634,6 +1631,13 @@ static int macaudio_late_probe(struct snd_soc_card *card) mutex_unlock(&ma->volume_lock_mutex); } + /* Append identity metadata without renumbering the existing controls. */ + if (ma->speaker_id_gpios) { + ret = snd_soc_add_card_controls(card, &macaudio_speaker_id_control, 1); + if (ret) + return ret; + } + /* The notifier walks the card's controls: only while the card is bound */ return register_pm_notifier(&ma->pm_nb); } From 3124fffeb69797ef8aba88d49836f97608bc757f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 18:41:55 +1100 Subject: [PATCH 1322/1579] arm64: dts: apple: describe the J616s fixed HDMI route and HPD The J616s routes external display pipeline 0 through ATC3 to the HDMI bridge. Describe the fixed DP PHY, crossbar selection and reciprocal graph, and its active-high hotplug input on AOP GPIO 36, using the existing DCP properties. Permit the fixed DisplayPort PHY graph in the T6040 PHY binding, as well as the Type-C SuperSpeed graph. Leave both the external DCP and ATC3 disabled. The SMC bridge reset and power callbacks have a separate service contract, and the current firmware handoff still needs qualification before enabling the route. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/phy/apple,t6040-atcphy.yaml | 5 +-- arch/arm64/boot/dts/apple/t6040-j616s.dts | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml index 04c8907584946..7f24f79264211 100644 --- a/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml @@ -106,8 +106,8 @@ properties: port: $ref: /schemas/graph.yaml#/properties/port description: > - Link to the SuperSpeed lanes of the Type-C connector. Absent on a fixed - DisplayPort PHY. + Link to the SuperSpeed lanes of the Type-C connector, or to the DCP + DisplayPort output on a fixed DisplayPort PHY. apple,tunable-axi2af: $ref: "#/$defs/apple,tunable" @@ -174,7 +174,6 @@ allOf: "#reset-cells": false orientation-switch: false mode-switch: false - port: false additionalProperties: false diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index f4b0aa85e2758..75e67c49002a4 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -4358,6 +4358,43 @@ #include "t6040-j616s-usb3.dtsi" +/* + * The HDMI bridge is fed by the fourth ATC's fixed DisplayPort output. + * Keep the external DCP disabled until its firmware handoff and the SMC + * bridge-power services are supported; those services are not GPIO outputs. + */ +&dcpext0 { + apple,connector-type = "HDMI-A"; + hdmi-hpd-gpios = <&pinctrl_aop 36 GPIO_ACTIVE_HIGH>; + phys = <&atcphy3 PHY_TYPE_DP>; + phy-names = "dp-phy"; + mux-controls = <&atcphy3_xbar 0>; + mux-control-names = "dp-xbar"; + mux-index = <0>; + apple,dptx-phy = <3>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + dcpext0_dpout: endpoint { + remote-endpoint = <&atcphy3_dp>; + }; + }; + }; +}; + +&atcphy3 { + port { + atcphy3_dp: endpoint { + remote-endpoint = <&dcpext0_dpout>; + }; + }; +}; + /* Additional Abbey PMICs on the two independent NUB SPMI buses. */ &nub_spmi1 { status = "okay"; From 467a513ce08b463af44d4335faafa49534b2d493 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 18:46:20 +1100 Subject: [PATCH 1323/1579] arm64: dts: apple: retain the enabled T6040 SEP power suppliers The complete power-state table and the SEP series both describe SPI_P, SPI2 and SEP. Replaying their individual patches during a rebase leaves duplicate nodes whose later disabled status overrides the enabled suppliers required by the SPI2 fingerprint transport and SEP. Keep one definition of each physical power state and retain the enabled SEP-series definitions, as in the previously integrated tree. This also restores the active-provider invariant checked against the compiled DT. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-pmgr.dtsi | 28 ----------------------- 1 file changed, 28 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi index eab3825a95c26..e0af3e1903cfb 100644 --- a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi @@ -600,15 +600,6 @@ status = "disabled"; }; - ps_spi_p: power-controller@198 { - compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x198 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "spi_p"; - power-domains = <&ps_sio>; - status = "disabled"; - }; ps_uart_p: power-controller@1a0 { compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; @@ -662,15 +653,6 @@ status = "disabled"; }; - ps_spi2: power-controller@1c8 { - compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0x1c8 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "spi2"; - power-domains = <&ps_spi_p>; - status = "disabled"; - }; ps_spi3: power-controller@1d0 { compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; @@ -1078,16 +1060,6 @@ status = "disabled"; }; - ps_sep: power-controller@c00 { - compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; - reg = <0xc00 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "sep"; - power-domains = <&ps_fab3_soc>; - apple,always-on; - status = "disabled"; - }; }; &pmgr_aux { From 0f830f8c816a489f0aef7b6432b3b99ff181fa97 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 18:54:22 +1100 Subject: [PATCH 1324/1579] ASoC: apple: make fixed speaker controls read-only INACTIVE hides a control but does not reject direct ALSA element writes. Set each fixed speaker enum during control fixup, deactivate it, and remove WRITE access from every element before publishing the card. Cover ASI1 selection, HPF corner, OCE handling and the dormant DAC gain setup independently of the deprecated unsafe parameter. Add a no-device host regression composing the actual speaker setup and ALSA matcher, enum setter, activation and element-write functions with in-memory registers. Require EPERM for direct writes to fixed controls, while unrelated jack controls remain writable. Cover both input-validation settings, mono and stereo prefixes, all elements and parameter values. Signed-off-by: Ryan Murray --- sound/soc/apple/macaudio.c | 50 ++-- .../selftests/alsa/macaudio-fixed-controls.py | 216 ++++++++++++++++++ 2 files changed, 250 insertions(+), 16 deletions(-) create mode 100644 tools/testing/selftests/alsa/macaudio-fixed-controls.py diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index d40dfb20d9675..1e70f5d365e23 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -1606,6 +1606,33 @@ static int macaudio_card_remove(struct snd_soc_card *card) return 0; } +/* Called during control fixup, before the card is exposed to userspace. */ +static int macaudio_set_fixed_enum(struct snd_soc_card *card, const char *name, + const char *value) +{ + struct snd_kcontrol *kctl; + unsigned int i; + int ret; + + ret = snd_soc_set_enum_kctl(card, name, value); + if (ret < 0) + return ret; + + ret = snd_soc_deactivate_kctl(card, name, 0); + if (ret < 0) + return ret; + + /* INACTIVE only hides controls; direct ALSA writes require WRITE. */ + list_for_each_entry(kctl, &card->snd_card->controls, list) { + if (!snd_soc_control_matches(kctl, name)) + continue; + for (i = 0; i < kctl->count; i++) + kctl->vd[i].access &= ~SNDRV_CTL_ELEM_ACCESS_WRITE; + } + + return ret; +} + #define CHECK(call, pattern, value, min) \ { \ int ret = call(card, pattern, value); \ @@ -1636,33 +1663,24 @@ static int macaudio_set_speaker(struct snd_soc_card *card, const char *prefix, b switch (ma->cfg->amp) { case AMP_TAS5770: - if (ma->cfg->stereo) { - CHECK_CONCAT(snd_soc_set_enum_kctl, "ASI1 Sel", "Left"); - CHECK_CONCAT(snd_soc_deactivate_kctl, "ASI1 Sel", 0); - } + if (ma->cfg->stereo) + CHECK_CONCAT(macaudio_set_fixed_enum, "ASI1 Sel", "Left"); CHECK_CONCAT(snd_soc_limit_volume, "Amp Gain Volume", ma->cfg->amp_gain); break; case AMP_SN012776: - if (ma->cfg->stereo) { - CHECK_CONCAT(snd_soc_set_enum_kctl, "ASI1 Sel", "Left"); - CHECK_CONCAT(snd_soc_deactivate_kctl, "ASI1 Sel", 0); - } + if (ma->cfg->stereo) + CHECK_CONCAT(macaudio_set_fixed_enum, "ASI1 Sel", "Left"); CHECK_CONCAT(snd_soc_limit_volume, "Amp Gain Volume", ma->cfg->amp_gain); - CHECK_CONCAT(snd_soc_set_enum_kctl, "HPF Corner Frequency", + CHECK_CONCAT(macaudio_set_fixed_enum, "HPF Corner Frequency", tweeter ? "800 Hz" : "2 Hz"); - CHECK_CONCAT(snd_soc_deactivate_kctl, "HPF Corner Frequency", 0); - - CHECK_CONCAT(snd_soc_set_enum_kctl, "OCE Handling", "Retry"); - CHECK_CONCAT(snd_soc_deactivate_kctl, "OCE Handling", 0); + CHECK_CONCAT(macaudio_set_fixed_enum, "OCE Handling", "Retry"); break; case AMP_SSM3515: /* TODO: check */ - CHECK_CONCAT(snd_soc_set_enum_kctl, "DAC Analog Gain Select", "8.4 V Span"); - - CHECK_CONCAT(snd_soc_deactivate_kctl, "DAC Analog Gain Select", 0); + CHECK_CONCAT(macaudio_set_fixed_enum, "DAC Analog Gain Select", "8.4 V Span"); /* TODO: HPF, needs new call to set */ break; diff --git a/tools/testing/selftests/alsa/macaudio-fixed-controls.py b/tools/testing/selftests/alsa/macaudio-fixed-controls.py new file mode 100644 index 0000000000000..12160072b45fa --- /dev/null +++ b/tools/testing/selftests/alsa/macaudio-fixed-controls.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""No-device regression for fixed macaudio controls and direct ALSA writes. + +Run: python3 tools/testing/selftests/alsa/macaudio-fixed-controls.py +Requires a C compiler. Extracts the actual setup, pattern matcher, enum setter, +enum put callback, activation and write functions from the source under test. +Shims model serialized controls and in-memory registers, not devices or ioctls. +Input-validation info/range checking is shimmed for valid enum values. Both +CONFIG_SND_CTL_INPUT_VALIDATION settings exercise the real permission check. +""" +import argparse +import os +from pathlib import Path +import re +import subprocess +import tempfile + + +def function(source, name): + match = re.search(r"^(?:static (?:inline )?)?(?:int|bool) " + name + + r"\(.*?^\}", source, re.M | re.S) + if not match: + raise ValueError("missing source function: " + name) + return match.group() + "\n" + + +PREFIX = r''' +#include +#include +#include +#include +#include +#define SNDRV_CTL_ELEM_ACCESS_WRITE 2 +#define SNDRV_CTL_ELEM_ACCESS_INACTIVE 256 +#define SNDRV_CTL_EVENT_MASK_INFO 2 +#define SNDRV_CTL_EVENT_MASK_VALUE 1 +#define SNDRV_CTL_ELEM_TYPE_ENUMERATED 3 +#define IS_ENABLED(x) VALIDATE +#define unlikely(x) (x) +#define dev_err(...) ((void)0) +#define dev_dbg(...) ((void)0) +struct snd_ctl_file { int unused; }; +struct snd_ctl_elem_id { char name[64]; unsigned int index; }; +struct snd_ctl_elem_value { + struct snd_ctl_elem_id id; + union { struct { unsigned int item[2]; } enumerated; } value; +}; +struct snd_ctl_elem_info { + struct snd_ctl_elem_id id; unsigned int type, count; + union { struct { unsigned int items, item; char name[64]; } enumerated; } value; +}; +struct snd_kcontrol_volatile { unsigned int access; struct snd_ctl_file *owner; }; +struct snd_soc_component { unsigned int reg, writes; }; +struct soc_enum { unsigned int items, shift_l, shift_r, mask, reg; }; +struct snd_kcontrol { + struct snd_ctl_elem_id id; struct snd_kcontrol_volatile vd[2]; unsigned int count; + int (*put)(struct snd_kcontrol *, struct snd_ctl_elem_value *); + int (*info)(struct snd_kcontrol *, struct snd_ctl_elem_info *); + unsigned long private_value; struct snd_soc_component *component; + struct snd_kcontrol *next; const char * const *texts; +}; +struct snd_card { int controls_rwsem; struct snd_kcontrol *controls; }; +enum { AMP_NONE, AMP_TAS5770, AMP_SN012776, AMP_SSM3515, AMP_MAX98360A }; +struct macaudio_platform_cfg { int amp, amp_gain; bool stereo; }; +struct macaudio_snd_data { bool has_speakers; struct macaudio_platform_cfg *cfg; }; +struct snd_soc_card { struct snd_card *snd_card; struct macaudio_snd_data *data; }; +static int please_blow_up_my_speakers; +#define snd_soc_card_get_drvdata(card) ((card)->data) +#define down_write(x) ((void)0) +#define up_write(x) ((void)0) +#define downgrade_write(x) ((void)0) +#define up_read(x) ((void)0) +#define list_for_each_entry(pos, head, member) \ + for (pos = *(head); pos; pos = pos->next) +static struct snd_kcontrol *snd_ctl_find_id(struct snd_card *c, struct snd_ctl_elem_id *id) +{ + struct snd_kcontrol *k; + for (k = c->controls; k; k = k->next) + if (!strcmp(k->id.name, id->name) && id->index < k->count) return k; + return NULL; +} +static unsigned int snd_ctl_get_ioff(struct snd_kcontrol *k, struct snd_ctl_elem_id *id) { return id->index; } +static void snd_ctl_build_ioff(struct snd_ctl_elem_id *id, struct snd_kcontrol *k, unsigned int i) {} +static void snd_ctl_notify_one(struct snd_card *c, int e, struct snd_kcontrol *k, unsigned int i) {} +static int __snd_ctl_elem_info(struct snd_card *c, struct snd_kcontrol *k, struct snd_ctl_elem_info *i, void *p) { return 0; } +static int sanity_check_input_values(struct snd_card *c, struct snd_ctl_elem_value *v, struct snd_ctl_elem_info *i, bool b) { return v->value.enumerated.item[0] < 7 ? 0 : -EINVAL; } +static struct snd_soc_component *snd_kcontrol_chip(struct snd_kcontrol *k) { return k->component; } +static unsigned int snd_soc_enum_item_to_val(struct soc_enum *e, unsigned int item) { return item; } +static int snd_soc_component_update_bits(struct snd_soc_component *c, unsigned int r, unsigned int m, unsigned int v) +{ unsigned int old = c->reg; c->writes++; c->reg = (old & ~m) | (v & m); return old != c->reg; } +/* Amp gain is volume-limited separately, not a fixed enum. */ +static int snd_soc_limit_volume(struct snd_soc_card *c, const char *n, int v) { return 1; } +static int enum_info(struct snd_kcontrol *k, struct snd_ctl_elem_info *i) +{ + struct soc_enum *e = (void *)k->private_value; + i->type = SNDRV_CTL_ELEM_TYPE_ENUMERATED; i->count = 1; + i->value.enumerated.items = e->items; + if (i->value.enumerated.item >= e->items) i->value.enumerated.item = e->items - 1; + snprintf(i->value.enumerated.name, sizeof(i->value.enumerated.name), "%s", k->texts[i->value.enumerated.item]); + return 0; +} +''' + +MAIN = r''' +static int failures, attempts; +static void check(bool ok, const char *what, const char *name) +{ + if (!ok) { failures++; fprintf(stderr, "FAIL validation=%d param=%d %s: %s\n", VALIDATE, please_blow_up_my_speakers, name, what); } +} +static void scenario(int amp, const char *prefix, bool tweeter, bool stereo) +{ + const char * const texts[] = { "Disabled", "Left", "2 Hz", "800 Hz", "Retry", "8.4 V Span", "Other" }; + const char * const suffix[] = { "ASI1 Sel", "HPF Corner Frequency", "OCE Handling", "DAC Analog Gain Select" }; + struct soc_enum e = { .items = 7, .mask = 7 }; + struct snd_kcontrol controls[9] = {0}; + struct snd_soc_component regs[9] = {0}; + struct macaudio_platform_cfg cfg = { .amp = amp, .amp_gain = 10, .stereo = stereo }; + struct macaudio_snd_data ma = { .has_speakers = true, .cfg = &cfg }; + struct snd_card card = { .controls = controls }; + struct snd_soc_card soc = { .snd_card = &card, .data = &ma }; + struct snd_ctl_file ordinary = {0}; + unsigned int i, element; + for (i = 0; i < 9; i++) { + const char *side = i < 4 ? "Left" : "Right"; + const char *kind = tweeter ? "Tweeter" : "Woofer"; + /* Broad 1W prefixes are used with CS42L83, which has no jack HPF. */ + if (i == 8) snprintf(controls[i].id.name, sizeof(controls[i].id.name), "%s", + !strcmp(prefix, "* ") ? "Jack Playback Volume" : "Jack HPF Corner Frequency"); + else if (!strcmp(prefix, "")) snprintf(controls[i].id.name, sizeof(controls[i].id.name), "%s", suffix[i % 4]); + else if (!strcmp(prefix, "* ")) snprintf(controls[i].id.name, sizeof(controls[i].id.name), "%s %s", side, suffix[i % 4]); + else snprintf(controls[i].id.name, sizeof(controls[i].id.name), "%s %s %s", side, kind, suffix[i % 4]); + controls[i].count = 2; controls[i].vd[0].access = controls[i].vd[1].access = 3; + controls[i].put = snd_soc_put_enum_double; controls[i].info = enum_info; + controls[i].private_value = (unsigned long)&e; controls[i].component = ®s[i]; + controls[i].texts = texts; controls[i].next = i < 8 ? &controls[i + 1] : NULL; + regs[i].reg = 6; + } + /* Unprefixed mono controls must have distinct names. */ + if (!strcmp(prefix, "")) { controls[3].next = &controls[8]; } + check(macaudio_set_speaker(&soc, prefix, tweeter) == 0, "setup", prefix); + for (i = 0; i < 9; i++) { + if (!strcmp(prefix, "") && i >= 4 && i < 8) continue; + unsigned int which = i % 4, expected = 6; + bool fixed = false; + if (i != 8) { + if ((amp == AMP_TAS5770 || amp == AMP_SN012776) && stereo && which == 0) { fixed = true; expected = 1; } + if (amp == AMP_SN012776 && which == 1) { fixed = true; expected = tweeter ? 3 : 2; } + if (amp == AMP_SN012776 && which == 2) { fixed = true; expected = 4; } + if (amp == AMP_SSM3515 && which == 3) { fixed = true; expected = 5; } + } + check(regs[i].reg == expected, "configured enum", controls[i].id.name); + for (element = 0; element < controls[i].count; element++) { + struct snd_ctl_elem_value v = { .id = controls[i].id }; + v.id.index = element; + unsigned int before = regs[i].writes; + int rc = snd_ctl_elem_write(&card, &ordinary, &v); + attempts++; + check(fixed ? rc == -EPERM : rc == 0, fixed ? "direct write must return -EPERM" : "unrelated control remains writable", controls[i].id.name); + if (fixed) check(regs[i].writes == before && regs[i].reg == expected, "blocked callback preserves transform", controls[i].id.name); + } + } +} +int main(void) +{ + int values[] = {0, 1, 2, 3, -1}; unsigned int p; + for (p = 0; p < sizeof(values) / sizeof(values[0]); p++) { + please_blow_up_my_speakers = values[p]; + scenario(AMP_SN012776, "* Tweeter ", true, true); + scenario(AMP_SN012776, "* Woofer ", false, true); + scenario(AMP_SN012776, "* ", false, true); + scenario(AMP_SN012776, "", false, false); + scenario(AMP_TAS5770, "* ", false, true); + /* SSM3515 is refused at admission; keep its dormant fixed setup locked too. */ + scenario(AMP_SSM3515, "* ", false, true); + scenario(AMP_MAX98360A, "* ", false, true); + } + printf("validation=%d attempts=%d failures=%d\n", VALIDATE, attempts, failures); + return failures ? 1 : 0; +} +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, default=Path(__file__).resolve().parents[4]) + args = parser.parse_args() + core = (args.source / "sound/core/control.c").read_text() + ops = (args.source / "sound/soc/soc-ops.c").read_text() + ma = (args.source / "sound/soc/apple/macaudio.c").read_text() + parts = [PREFIX, function(ops, "snd_soc_control_matches"), + function(core, "snd_ctl_activate_id"), + function(ops, "snd_soc_put_enum_double"), + core[core.index("static inline int snd_ctl_put("):core.index("\n#endif", core.index("static inline int snd_ctl_put("))], + function(core, "snd_ctl_elem_write"), + function(ops, "snd_soc_deactivate_kctl"), + function(ops, "soc_set_enum_kctl"), function(ops, "snd_soc_set_enum_kctl")] + if "static int macaudio_set_fixed_enum(" in ma: + parts.append(function(ma, "macaudio_set_fixed_enum")) + parts += [ma[ma.index("#define CHECK(call,"):ma.index("static int macaudio_set_speaker(")], + function(ma, "macaudio_set_speaker"), MAIN] + with tempfile.TemporaryDirectory(prefix="macaudio-fixed-") as temp: + source, binary = Path(temp) / "test.c", Path(temp) / "test" + source.write_text("\n".join(parts)) + failures = 0 + for validation in (0, 1): + subprocess.run([os.environ.get("CC", "cc"), "-std=c11", "-O2", + "-DVALIDATE=" + str(validation), str(source), + "-o", str(binary)], check=True) + failures += subprocess.run([str(binary)]).returncode != 0 + if failures: + raise SystemExit(1) + + +if __name__ == "__main__": + main() From d29673ec23c43190000c7f6f51164c563faa5ecf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 19:06:41 +1100 Subject: [PATCH 1325/1579] arm64: dts: apple: describe the T6040 DisplayPort audio interfaces Describe the five SIO-fed DPA interfaces and their power domains. Connect DPA0, DPA1 and DPA2 to their assigned DCP, DCPEXT0 and DCPEXT1 audio ports; DPA3 and DPA4 have no assigned display processor. Extend the existing DPA binding for T6041 and require unconnected interfaces to stay disabled. Keep all five interfaces disabled: the T6041 SIO firmware backend and compatible DCP audio services are not implemented. In particular, adding an audio graph must not pull an unavailable component into the active display pipeline or claim that HDMI audio works. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/sound/apple,t6030-dpaudio.yaml | 43 ++++-- arch/arm64/boot/dts/apple/t6040-dpaudio.dtsi | 132 ++++++++++++++++++ arch/arm64/boot/dts/apple/t6040.dtsi | 1 + 3 files changed, 168 insertions(+), 8 deletions(-) create mode 100644 arch/arm64/boot/dts/apple/t6040-dpaudio.dtsi diff --git a/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml b/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml index 9b4e9328d9020..bb92d6a30c428 100644 --- a/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml @@ -4,29 +4,36 @@ $id: http://devicetree.org/schemas/sound/apple,t6030-dpaudio.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple T6030 DisplayPort audio +title: Apple T6030 and T6041 DisplayPort audio maintainers: - Ryan Murray description: | - The external display processor provides the HDMI port's audio service. - Its DisplayPort audio interface has no application-processor registers. - Samples arrive through SIO, and the graph connects this interface to - port 0 of the display processor that owns the audio service. + Each DisplayPort audio interface has no application-processor registers. + Samples arrive through SIO, and the graph connects the interface to port 0 + of the display processor that owns its audio service. An interface without + an assigned display processor has no graph connection and remains disabled. + + T6041 requires a generation-specific SIO firmware backend and a compatible + display audio service before these interfaces can be enabled. select: properties: compatible: contains: - const: apple,t6030-dpaudio + enum: + - apple,t6030-dpaudio + - apple,t6041-dpaudio required: - compatible properties: compatible: items: - - const: apple,t6030-dpaudio + - enum: + - apple,t6030-dpaudio + - apple,t6041-dpaudio - const: apple,dpaudio dmas: @@ -52,6 +59,26 @@ required: - dmas - dma-names - power-domains - - ports + +allOf: + - if: + properties: + compatible: + contains: + const: apple,t6030-dpaudio + then: + required: + - ports + + - if: + not: + required: + - ports + then: + properties: + status: + const: disabled + required: + - status additionalProperties: false diff --git a/arch/arm64/boot/dts/apple/t6040-dpaudio.dtsi b/arch/arm64/boot/dts/apple/t6040-dpaudio.dtsi new file mode 100644 index 0000000000000..68d346a4e1160 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6040-dpaudio.dtsi @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * DPA interfaces are fed by SIO and have no AP-side MMIO registers. + * Keep them disabled until the T6041 SIO firmware and DCP audio services + * are supported. DPA3/4 have no assigned DCP in this SoC's firmware tree. + */ + +/ { + dpaudio0: audio-controller-0 { + compatible = "apple,t6041-dpaudio", "apple,dpaudio"; + dmas = <&sio 0x64>; + dma-names = "tx"; + power-domains = <&ps_dpa0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + dpaudio0_dcp: endpoint { + remote-endpoint = <&dcp_audio>; + }; + }; + }; + }; + + dpaudio1: audio-controller-1 { + compatible = "apple,t6041-dpaudio", "apple,dpaudio"; + dmas = <&sio 0x66>; + dma-names = "tx"; + power-domains = <&ps_dpa1>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + dpaudio1_dcp: endpoint { + remote-endpoint = <&dcpext0_audio>; + }; + }; + }; + }; + + dpaudio2: audio-controller-2 { + compatible = "apple,t6041-dpaudio", "apple,dpaudio"; + dmas = <&sio 0x68>; + dma-names = "tx"; + power-domains = <&ps_dpa2>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + dpaudio2_dcp: endpoint { + remote-endpoint = <&dcpext1_audio>; + }; + }; + }; + }; + + dpaudio3: audio-controller-3 { + compatible = "apple,t6041-dpaudio", "apple,dpaudio"; + dmas = <&sio 0x6a>; + dma-names = "tx"; + power-domains = <&ps_dpa3>; + status = "disabled"; + }; + + dpaudio4: audio-controller-4 { + compatible = "apple,t6041-dpaudio", "apple,dpaudio"; + dmas = <&sio 0x6c>; + dma-names = "tx"; + power-domains = <&ps_dpa4>; + status = "disabled"; + }; +}; + +&dcp { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + dcp_audio: endpoint { + remote-endpoint = <&dpaudio0_dcp>; + }; + }; + }; +}; + +&dcpext0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + dcpext0_audio: endpoint { + remote-endpoint = <&dpaudio1_dcp>; + }; + }; + }; +}; + +&dcpext1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + dcpext1_audio: endpoint { + remote-endpoint = <&dpaudio2_dcp>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 75c9b3f06ac21..d834d562c65d2 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -2403,3 +2403,4 @@ }; #include "t6040-pmgr.dtsi" +#include "t6040-dpaudio.dtsi" From f70f1acd35233683c65b64a7cfee79418817b9c6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 23:12:35 +1100 Subject: [PATCH 1326/1579] soc: apple: guard SEP enrolment and recover exact identities Validate enrolment metadata and durable Linux bindings before capture. Prepare an empty owner context only after userspace has validated the account. Expose Linux user-context identities and an exact-ID recovery delete, saving owner, user and master Catacombs before reporting success. Report the identity returned by the enclave at match time. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/bio.rs | 132 ++++++++++-- drivers/soc/apple/catacomb.rs | 6 + drivers/soc/apple/sbio.rs | 374 +++++++++++++++++++++++++++++++-- drivers/soc/apple/sep.rs | 4 + drivers/soc/apple/shim.h | 1 + drivers/soc/apple/shim.rs | 10 + drivers/soc/apple/store.rs | 6 + drivers/soc/apple/store_shim.c | 8 + include/uapi/linux/apple-sep.h | 20 +- 9 files changed, 527 insertions(+), 34 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 6b4e1edbe2966..573830b7973d0 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -6,7 +6,7 @@ use kernel::ioctl::{_IO, _IOR, _IOW, _IOWR}; use kernel::prelude::*; use kernel::uaccess::{UserPtr, UserSlice}; -pub(crate) const IFACE_VERSION: u32 = 4; +pub(crate) const IFACE_VERSION: u32 = 5; pub(crate) const ENROL_USER_ID: i32 = 1000; @@ -100,10 +100,35 @@ struct List { #[repr(C)] struct EnrolStart { flags: u32, + account_uid: u32, + finger: u32, reserved: u32, label: [u8; LABEL_LEN], } +#[repr(C)] +pub(crate) struct EnclaveList { + pub(crate) count: u32, + pub(crate) reserved: u32, + pub(crate) uuid: [[u8; UUID_LEN]; MAX_IDENTITIES], +} + +impl EnclaveList { + pub(crate) fn new() -> Self { + Self { count: 0, reserved: 0, uuid: [[0; UUID_LEN]; MAX_IDENTITIES] } + } + + pub(crate) fn push(&mut self, uuid: [u8; UUID_LEN]) -> Result<()> { + let at = self.count as usize; + if at >= MAX_IDENTITIES { + return Err(EOVERFLOW); + } + self.uuid[at] = uuid; + self.count += 1; + Ok(()) + } +} + #[repr(C)] #[derive(Default)] struct EnrolPoll { @@ -136,8 +161,8 @@ struct VerifyPoll { } #[repr(C)] -struct Delete { - uuid: [u8; UUID_LEN], +pub(crate) struct Delete { + pub(crate) uuid: [u8; UUID_LEN], } pub(crate) const ATTEST_CHALLENGE_LEN: usize = 32; @@ -162,7 +187,8 @@ pub(crate) struct Attest { static_assert!(core::mem::size_of::() == 144); static_assert!(core::mem::size_of::() == 32); static_assert!(core::mem::size_of::() == 4616); -static_assert!(core::mem::size_of::() == 136); +static_assert!(core::mem::size_of::() == 144); +static_assert!(core::mem::size_of::() == 520); static_assert!(core::mem::size_of::() == 40); static_assert!(core::mem::size_of::() == 40); static_assert!(core::mem::size_of::() == 72); @@ -182,7 +208,7 @@ macro_rules! ioctl_pod { }; } ioctl_pod! { - to_user: Info, List, EnrolPoll, VerifyPoll, Attest; + to_user: Info, List, EnclaveList, EnrolPoll, VerifyPoll, Attest; from_user: EnrolStart, VerifyStart, Delete, Attest } @@ -198,6 +224,9 @@ const IOC_CANCEL: u32 = _IO(MAGIC, 0x07); const IOC_DELETE: u32 = _IOW::(MAGIC, 0x08); const IOC_DELETE_ALL: u32 = _IO(MAGIC, 0x09); pub(crate) const IOC_ATTEST: u32 = _IOWR::(MAGIC, 0x0a); +pub(crate) const IOC_LIST_ENCLAVE: u32 = _IOR::(MAGIC, 0x0b); +pub(crate) const IOC_DELETE_ORPHAN: u32 = _IOW::(MAGIC, 0x0c); +pub(crate) const IOC_PREPARE_ENROL: u32 = _IOW::(MAGIC, 0x0d); pub(crate) struct MatchEvidence { identity: [u8; UUID_LEN], @@ -338,6 +367,10 @@ impl IdentityIndex { self.save(store) } + pub(crate) fn can_persist_next(&self, store: &Store) -> bool { + store.can_write(&identity_key(), 8 + (self.entries.len() + 1) * (UUID_LEN + LABEL_LEN)) + } + pub(crate) fn identity_v1_for( &self, uuid: &[u8; UUID_LEN], @@ -481,7 +514,7 @@ pub(crate) struct Handled { pub(crate) delete_identities: KVec, } -fn ok() -> Result { +pub(crate) fn ok() -> Result { Ok(Handled { ret: 0, wake: false, @@ -492,7 +525,7 @@ fn ok() -> Result { }) } -fn require_admin() -> Result<()> { +pub(crate) fn require_admin() -> Result<()> { if crate::shim::capable_admin() { Ok(()) } else { @@ -500,6 +533,12 @@ fn require_admin() -> Result<()> { } } +pub(crate) fn orphan_delete(identity: crate::sbio::IdentityV1) -> Result { + let mut handled = ok()?; + handled.delete_identity = Some(identity); + Ok(handled) +} + fn require_sensor(present: bool) -> Result<()> { if present { Ok(()) @@ -567,22 +606,85 @@ fn list(ctx: &mut Context<'_>, user: UserPtr) -> Result { ok() } +fn validate_enrol_request(request: &EnrolStart) -> Result<()> { + // NSS account resolution occurs in libfprint. This ABI requires its + // resolved uid, the named finger and the generated user label. + if request.flags != 0 || request.reserved != 0 + || request.account_uid == u32::MAX || !(1..=10).contains(&request.finger) + || !request.label.starts_with(b"FP1-") + || request.label[12] != b'-' || request.label[14] != b'-' + || request.label[23] != b'-' || request.label[24] == 0 + || request.label[13] != b"123456789A"[request.finger as usize - 1] + || !request.label.contains(&0) + { + return Err(EINVAL); + } + Ok(()) +} + +pub(crate) fn validate_enrol_start_arg(arg: usize) -> Result<()> { + let user = UserPtr::from_addr(arg); + let request: EnrolStart = UserSlice::new(user, core::mem::size_of::()) + .reader().read()?; + validate_enrol_request(&request) +} + +#[cfg(CONFIG_APPLE_SEP_KUNIT_TEST)] +#[::kernel::macros::kunit_tests(apple_sep_enrol_request)] +mod enrol_request_tests { + use super::*; + + fn valid() -> EnrolStart { + let mut request = EnrolStart { + flags: 0, account_uid: 1000, finger: 7, reserved: 0, + label: [0; LABEL_LEN], + }; + request.label[..28].copy_from_slice(b"FP1-20261027-7-12345678-user"); + request + } + + #[test] + fn resolved_account_and_known_finger() { + assert!(validate_enrol_request(&valid()).is_ok()); + } + + #[test] + fn missing_uid_is_refused() { + let mut request = valid(); + request.account_uid = u32::MAX; + assert!(validate_enrol_request(&request).is_err()); + } + + #[test] + fn unknown_or_mismatched_finger_is_refused() { + let mut request = valid(); + request.finger = 0; + assert!(validate_enrol_request(&request).is_err()); + request.finger = 6; + assert!(validate_enrol_request(&request).is_err()); + } + + #[test] + fn missing_username_is_refused() { + let mut request = valid(); + request.label[24] = 0; + assert!(validate_enrol_request(&request).is_err()); + } +} + fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { require_admin()?; let request: EnrolStart = UserSlice::new(user, core::mem::size_of::()) .reader() .read()?; - if request.flags != 0 || request.reserved != 0 { - return Err(EINVAL); - } + validate_enrol_request(&request)?; require_sensor(ctx.sensor_present)?; if ctx.session.worker_epoch.is_some() || !ctx.session.op.may_start() { return Err(EBUSY); } - if matches!(ctx.live_identity_count, Some(count) if count >= SEP_USER_CAPACITY) - || ctx.index.total() >= MAX_IDENTITIES - { + let count = ctx.live_identity_count.ok_or(EIO)?; + if count >= SEP_USER_CAPACITY || ctx.index.total() >= MAX_IDENTITIES { return Err(ENOSPC); } @@ -960,6 +1062,10 @@ pub(crate) fn starts_capture(cmd: u32) -> bool { cmd == IOC_ENROL_START || cmd == IOC_VERIFY_START } +pub(crate) fn idle_for_mutation(session: &Session) -> bool { + session.worker_epoch.is_none() && session.op.may_start() +} + fn delete(ctx: &mut Context<'_>, user: UserPtr) -> Result { require_admin()?; if !ctx.session.op.may_start() { diff --git a/drivers/soc/apple/catacomb.rs b/drivers/soc/apple/catacomb.rs index 8b18efb4ead77..9c60e19b5861d 100644 --- a/drivers/soc/apple/catacomb.rs +++ b/drivers/soc/apple/catacomb.rs @@ -28,6 +28,12 @@ pub(crate) fn is_kind(kind: u8) -> bool { path_for(kind).is_some() } +pub(crate) fn writable(kind: u8) -> bool { + path_for(kind) + .and_then(|path| shim::StoreFile::open_writable_existing(path).ok()) + .is_some() +} + pub(crate) fn write(kind: u8, blob: &[u8]) -> Result<()> { let path = path_for(kind).ok_or(EINVAL)?; if blob.len() > MAX_CATACOMB { diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 88be672817b1c..2f72bfa2ce7cc 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -10,6 +10,128 @@ use crate::sks::SKS_AUTH_TOKEN_LEN; use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; +struct EnrolPrerequisites { + lane: bool, + live_count: Option, + bag_bound: bool, + catacomb_readable: [bool; 3], + catacomb_writable: [bool; 3], + keybag_writable: bool, + host_store_capacity: bool, +} + +fn admit_enrol(p: &EnrolPrerequisites) -> Result<()> { + if !p.lane { return Err(ENXIO); } + let count = p.live_count.ok_or(EIO)?; + if count >= 3 { return Err(ENOSPC); } + if !p.bag_bound || p.catacomb_readable.contains(&false) { + return Err(ENODATA); + } + if p.catacomb_writable.contains(&false) || !p.keybag_writable { + return Err(EROFS); + } + if !p.host_store_capacity { return Err(ENOSPC); } + Ok(()) +} + +fn catacomb_needed_for_enrol(kind: u8, live_count: Option) -> bool { + live_count != Some(0) || kind == PRIVATE_TYPE_CATACOMB_OWNER +} + +fn admissible_orphan_target( + listed_linux: &[[u8; bio::UUID_LEN]], + uuid: &[u8; bio::UUID_LEN], +) -> Result<()> { + if *uuid == [0; bio::UUID_LEN] { return Err(EINVAL); } + if !listed_linux.contains(uuid) { return Err(ENOENT); } + Ok(()) +} + +#[cfg(CONFIG_APPLE_SEP_KUNIT_TEST)] +#[::kernel::macros::kunit_tests(apple_sep_enrol_admission)] +mod admission_tests { + use super::*; + + fn ready() -> EnrolPrerequisites { + EnrolPrerequisites { + lane: true, live_count: Some(1), bag_bound: true, + catacomb_readable: [true; 3], catacomb_writable: [true; 3], + keybag_writable: true, host_store_capacity: true, + } + } + + fn refused(p: &EnrolPrerequisites, expected: Error) { + assert_eq!(admit_enrol(p).unwrap_err().to_errno(), expected.to_errno()); + } + + #[test] + fn missing_os_identity() { + let mut p = ready(); p.lane = false; refused(&p, ENXIO); + } + + #[test] + fn unavailable_enclave_list() { + let mut p = ready(); p.live_count = None; refused(&p, EIO); + } + + #[test] + fn enclave_full() { + let mut p = ready(); p.live_count = Some(3); refused(&p, ENOSPC); + } + + #[test] + fn missing_identity_bag_binding() { + let mut p = ready(); p.bag_bound = false; refused(&p, ENODATA); + } + + #[test] + fn missing_catacomb() { + for i in 0..3 { + let mut p = ready(); p.catacomb_readable[i] = false; refused(&p, ENODATA); + } + } + + #[test] + fn unwritable_catacomb_or_keybag() { + for i in 0..3 { + let mut p = ready(); p.catacomb_writable[i] = false; refused(&p, EROFS); + } + let mut p = ready(); p.keybag_writable = false; refused(&p, EROFS); + } + + #[test] + fn host_store_full() { + let mut p = ready(); p.host_store_capacity = false; refused(&p, ENOSPC); + } + + #[test] + fn ready_binding_is_admitted() { + assert!(admit_enrol(&ready()).is_ok()); + } + + #[test] + fn clean_lane_requires_prepared_owner_before_first_capture() { + assert!(catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_OWNER, Some(0))); + assert!(!catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_USER, Some(0))); + assert!(!catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_MASTER, Some(0))); + assert!(catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_USER, Some(1))); + } + + #[test] + fn orphan_delete_requires_exact_linux_identity() { + let linux = [1u8; bio::UUID_LEN]; + let other_lane = [2u8; bio::UUID_LEN]; + assert!(admissible_orphan_target(&[linux], &linux).is_ok()); + refused_orphan(&[linux], &other_lane, ENOENT); + refused_orphan(&[linux], &[0; bio::UUID_LEN], EINVAL); + } + + fn refused_orphan(list: &[[u8; bio::UUID_LEN]], uuid: &[u8; bio::UUID_LEN], expected: Error) { + assert_eq!(admissible_orphan_target(list, uuid).unwrap_err().to_errno(), + expected.to_errno()); + } +} + impl SepData { fn sbio_enrol_step(&self, op: &crate::sbio::SbioOp) -> Option> { match self.sbio_call(op) { @@ -349,6 +471,12 @@ impl SepData { return self.enrol_into_existing_context(user); }; + // PREPARE_ENROL may already have selected the empty Linux context in + // this boot. A file left from an earlier boot is not that proof. + if self.empty_context_prepared.xchg(false, Relaxed) { + return self.enrol_into_existing_context(user); + } + self.open_fresh_context(user, &proof) } @@ -1777,6 +1905,7 @@ impl SepData { }; if !parsed.matches(user) { + dev_info!(self.dev, "verify: enclave found no identity in the Linux user context\n"); return bio::VerifyOutcome::NoMatch; } @@ -1784,6 +1913,12 @@ impl SepData { let identity = *parsed.identity_uuid(); let known = self.bio_index.lock().contains_uuid(&identity); + dev_info!( + self.dev, + "verify: enclave matched identity {} (host index {})\n", + Hex(&identity), if known { "present" } else { "absent" } + ); + if !known { dev_err!( self.dev, @@ -3403,6 +3538,115 @@ impl SepData { Some(bytes) } + fn require_linux_bio_lane(&self) -> Result<()> { + if !self.touchid_started.load(Relaxed) || self.xarm.lock().os_uuid.is_none() { + return Err(ENXIO); + } + Ok(()) + } + + /// Refuse before ENROL_START schedules work or sends BEGIN_ENROL. A + /// userspace passwd lookup proves the host account; these are the durable + /// Linux-lane bindings that only the kernel can inspect. + fn preflight_enrol(&self, live_count: Option) -> Result<()> { + let lane = self.touchid_started.load(Relaxed) && self.xarm.lock().os_uuid.is_some(); + let bag_bound = self.keybag_designated.load(Relaxed) && self.enrol_material.lock() + .as_ref().is_some_and(|material| material.snapshot_identity.is_some()); + let kinds = [PRIVATE_TYPE_CATACOMB_OWNER, PRIVATE_TYPE_CATACOMB_USER, + PRIVATE_TYPE_CATACOMB_MASTER]; + let readable = kinds.map(|kind| !catacomb_needed_for_enrol(kind, live_count) + || crate::catacomb::read(kind).is_some()); + let writable = kinds.map(|kind| !catacomb_needed_for_enrol(kind, live_count) + || crate::catacomb::writable(kind)); + let index = self.bio_index.lock(); + let preconditions = EnrolPrerequisites { + lane, + live_count, + bag_bound, + catacomb_readable: readable, + catacomb_writable: writable, + keybag_writable: shim::StoreFile::open_writable_existing( + keybag::Slot::Identity.path()).is_ok(), + host_store_capacity: self.with_host_store(|store| index.can_persist_next(store)) + .unwrap_or(false), + }; + let result = admit_enrol(&preconditions); + if let Err(e) = &result { + dev_warn!(self.dev, "enrol: prerequisites failed before enclave capture: {:?}\n", e); + } + result + } + + fn bio_list_enclave(&self, arg: usize) -> Result { + bio::require_admin()?; + self.require_linux_bio_lane()?; + let ids = self.enclave_identities_for(SBIO_PROBE_USER_ID).ok_or(EIO)?; + let mut out = bio::EnclaveList::new(); + for uuid in ids.iter() { + out.push(*uuid)?; + } + kernel::uaccess::UserSlice::new( + kernel::uaccess::UserPtr::from_addr(arg), core::mem::size_of::() + ).writer().write(&out)?; + bio::ok() + } + + fn bio_delete_orphan(&self, arg: usize) -> Result { + bio::require_admin()?; + self.require_linux_bio_lane()?; + if !bio::idle_for_mutation(&self.bio_session.lock()) { return Err(EBUSY); } + let request: bio::Delete = kernel::uaccess::UserSlice::new( + kernel::uaccess::UserPtr::from_addr(arg), core::mem::size_of::() + ).reader().read()?; + // The exact SEP user and built-in selector are checked by the parser. + // The host index may still hold an orphan after fprintd lost its file, + // so that index cannot decide whether this exact-ID recovery is valid. + let ids = self.enclave_identities_for(SBIO_PROBE_USER_ID).ok_or(EIO)?; + admissible_orphan_target(&ids, &request.uuid)?; + bio::orphan_delete(crate::sbio::IdentityV1::from_index( + SBIO_PROBE_USER_ID, request.uuid)) + } + + fn bio_prepare_enrol(&self, arg: usize) -> Result { + bio::require_admin()?; + bio::validate_enrol_start_arg(arg)?; + self.require_linux_bio_lane()?; + if !bio::idle_for_mutation(&self.bio_session.lock()) { return Err(EBUSY); } + if !self.xart_writable() { return Err(EROFS); } + let ids = self.enclave_identities_for(SBIO_PROBE_USER_ID).ok_or(EIO)?; + if !ids.is_empty() { + // An existing identity must already have a durable owner file; + // selecting SYSTEM here could disturb its live context. + return if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_some() { + bio::ok() + } else { Err(ENODATA) }; + } + if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_some() { + return bio::ok(); + } + if !self.keybag_designated.load(Relaxed) + || !self.enrol_material.lock().as_ref() + .is_some_and(|material| material.snapshot_identity.is_some()) { + return Err(ENODATA); + } + let user = crate::sbio::UserId::new(SBIO_PROBE_USER_ID).ok_or(EINVAL)?; + loop { + self.wait_until_awake()?; + let _writer = self.catacomb_writer.lock(); + if !self.capture_begin() { continue; } + let prepared = self.open_enrolment_context(user); + self.capture_end(); + if !prepared { return Err(EIO); } + break; + } + if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() + || !crate::catacomb::writable(PRIVATE_TYPE_CATACOMB_OWNER) { + return Err(EROFS); + } + self.empty_context_prepared.store(true, Relaxed); + bio::ok() + } + pub(crate) fn bio_ioctl(&self, cmd: u32, arg: usize) -> Result { if self.shutting_down.load(Relaxed) { return Err(ENODEV); @@ -3410,6 +3654,12 @@ impl SepData { if cmd == bio::IOC_ATTEST { return self.bio_attest(arg); } + if cmd == bio::IOC_LIST_ENCLAVE { + return self.bio_list_enclave(arg); + } + if cmd == bio::IOC_PREPARE_ENROL { + return self.bio_prepare_enrol(arg); + } // With xART writes off the enclave cannot persist what an enrolment or // a match changes. It saves anti-replay state after every match, and a // refused save has left it unresponsive until reboot. Refuse the @@ -3434,14 +3684,21 @@ impl SepData { let live_identity_count = if (cmd == bio::IOC_GET_INFO || cmd == bio::IOC_ENROL_START) && self.touchid_started.load(Relaxed) { - self.enclave_identity_count() + self.enclave_identities_for(SBIO_PROBE_USER_ID).map(|ids| ids.len()) } else { None }; + if cmd == bio::IOC_ENROL_START { + bio::require_admin()?; + bio::validate_enrol_start_arg(arg)?; + self.preflight_enrol(live_identity_count)?; + } // The SPI probe may have deferred until its SMC power supplier bound. // A saved attach-time snapshot would incorrectly reject later ioctls. let sensor_present = sensor::is_bound(); - let handled = { + let handled = if cmd == bio::IOC_DELETE_ORPHAN { + self.bio_delete_orphan(arg)? + } else { let mut session = self.bio_session.lock(); if bio::starts_capture(cmd) { self.wait_for_worker_exit(&mut session, false)?; @@ -3478,7 +3735,7 @@ impl SepData { if !self.capture_begin() { continue; } - let deleted = self.delete_and_persist(&handled); + let deleted = self.delete_and_persist(&handled, cmd == bio::IOC_DELETE_ORPHAN); self.capture_end(); break deleted; }; @@ -3721,7 +3978,7 @@ impl SepData { /// Called with `catacomb_writer` held. A host index entry goes only once /// its deletion is saved: if the save fails, the entry stays, userspace /// still lists the fingerprint, and a retried deletion saves again. - fn delete_and_persist(&self, handled: &bio::Handled) -> Result<()> { + fn delete_and_persist(&self, handled: &bio::Handled, recovery: bool) -> Result<()> { let retry = self.deletion_unsaved.load(Relaxed); let wanted = Iterator::chain(handled.delete_identity.iter(), handled.delete_identities.iter()); @@ -3729,10 +3986,13 @@ impl SepData { let mut gone: KVec<[u8; bio::UUID_LEN]> = KVec::with_capacity(wanted.clone().count(), GFP_KERNEL)?; let mut delete_err: Option = None; + let mut orphan_removed = false; for identity in wanted { - match self.delete_identity(identity) { + let orphan = recovery || !self.bio_index.lock().contains_uuid(identity.uuid()); + match self.delete_identity(identity, orphan) { Ok(true) => { let _ = gone.push_within_capacity(*identity.uuid()); + orphan_removed |= orphan; } Ok(false) => {} Err(e) => delete_err = delete_err.or(Some(e)), @@ -3740,7 +4000,7 @@ impl SepData { } // Save what was removed even if another deletion failed. if !gone.is_empty() { - match self.persist_deletion(retry) { + match self.persist_deletion(retry, orphan_removed) { Ok(()) => { self.deletion_unsaved.store(false, Relaxed); for uuid in gone.iter() { @@ -3765,7 +4025,7 @@ impl SepData { /// Returns whether the enclave no longer holds the identity and the host /// entry must wait for [`Self::persist_deletion`]. - fn delete_identity(&self, identity: &crate::sbio::IdentityV1) -> Result { + fn delete_identity(&self, identity: &crate::sbio::IdentityV1, orphan: bool) -> Result { if self.enclave_lists(identity.uuid()) == Some(false) { // Unless an earlier deletion's save failed, which leaves the // enclave without an identity the saved Catacomb still holds. @@ -3776,18 +4036,39 @@ impl SepData { return Ok(false); } - // 0x57 cannot be undone. Check first that its deletion can be saved. + // 0x57 cannot be undone. Ensure the host can save the affected user + // and master Catacombs before sending it. For an unindexed orphan the + // owner Catacomb provides the recovery path even when the bag snapshot + // has no binding; export that owner first if this install lacks a file. + for kind in [PRIVATE_TYPE_CATACOMB_USER, PRIVATE_TYPE_CATACOMB_MASTER] { + if crate::catacomb::read(kind).is_none() || !crate::catacomb::writable(kind) { + dev_err!( + self.dev, + "bio: not deleting: user/master Catacomb is unavailable or unwritable\n" + ); + return Err(EIO); + } + } + if orphan && crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() { + let Some(reply) = self.read_component_states() else { return Err(EIO); }; + let state = crate::sbio::ComponentStates::new(&reply) + .state_for(crate::sbio::CatacombUser::OWNER.value()); + if !state.is_some_and(|s| s & crate::sbio::COMPONENT_STATE_ACTIVE != 0) + || !self.save_catacomb(crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"orphan-recovery owner Catacomb") { + dev_err!(self.dev, "bio: not deleting: owner Catacomb export failed\n"); + return Err(EIO); + } + } if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() - || !self - .enrol_material - .lock() - .as_ref() - .is_some_and(|material| material.snapshot_identity.is_some()) + || !crate::catacomb::writable(PRIVATE_TYPE_CATACOMB_OWNER) { - dev_err!( - self.dev, - "bio: not deleting: there is no owner Catacomb file or identity-bag binding to save the deletion with; nothing was changed\n" - ); + return Err(EIO); + } + if !orphan && !self.enrol_material.lock().as_ref() + .is_some_and(|material| material.snapshot_identity.is_some()) { + dev_err!(self.dev, "bio: not deleting: indexed identity has no identity-bag binding\n"); return Err(EIO); } @@ -3813,7 +4094,7 @@ impl SepData { /// identity bag, which must match the Catacombs. Safe to repeat after a /// failure partway: a `retry` accepts a user Catacomb an earlier attempt /// already saved. - fn persist_deletion(&self, retry: bool) -> Result<()> { + fn persist_deletion(&self, retry: bool, orphan: bool) -> Result<()> { let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { return Err(EIO); }; @@ -3828,6 +4109,11 @@ impl SepData { PRIVATE_TYPE_CATACOMB_MASTER, c"master Catacomb after deletion", ), + ( + crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"owner Catacomb after deletion", + ), ] { let Some(reply) = self.read_component_states() else { dev_err!(self.dev, "bio: cannot read component state before saving the {}\n", what); @@ -3843,7 +4129,8 @@ impl SepData { dev_err!(self.dev, "bio: the {} is not active; the deletion cannot be saved\n", what); return Err(EIO); } - if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING != 0 { + if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING != 0 || + (orphan && who.value() == crate::sbio::CatacombUser::OWNER.value()) { if !self.save_catacomb(who, kind, what) { dev_err!(self.dev, "bio: saving the {} failed\n", what); return Err(EIO); @@ -3864,13 +4151,19 @@ impl SepData { dev_err!(self.dev, "bio: lockout persistence failed after the deletion\n"); return Err(EIO); } - if !self.resnapshot_identity_keybag() { + if !orphan && !self.resnapshot_identity_keybag() { dev_err!( self.dev, "bio: Catacombs saved after the deletion but the identity-bag snapshot failed\n" ); return Err(EIO); } + if orphan { + dev_info!( + self.dev, + "bio: orphan deletion saved via Linux user/master/owner Catacombs; no unbound identity bag was snapshotted\n" + ); + } Ok(()) } @@ -4677,6 +4970,47 @@ impl<'a> IdentityRecords<'a> { } } +#[cfg(CONFIG_APPLE_SEP_KUNIT_TEST)] +#[::kernel::macros::kunit_tests(apple_sep_enclave_identity_scope)] +mod identity_scope_tests { + use super::*; + + #[test] + fn list_excludes_other_users_and_selectors() { + let mut reply = [0u8; 3 * IDENTITY_RECORD_LEN]; + for (i, (user, selector, byte)) in [ + (SBIO_PROBE_USER_ID, IR_SELECTOR_BUILTIN, 1u8), + (501, IR_SELECTOR_BUILTIN, 2u8), + (SBIO_PROBE_USER_ID, 2, 3u8), + ].iter().enumerate() { + let at = i * IDENTITY_RECORD_LEN; + reply[at..at + 4].copy_from_slice(&user.to_le_bytes()); + reply[at + 4..at + IDENTITY_V1_LEN].fill(*byte); + reply[at + IR_SELECTOR..at + IR_SELECTOR + 4] + .copy_from_slice(&selector.to_le_bytes()); + } + let records = IdentityRecords::new(&reply).unwrap(); + let mut seen = 0; + records.each_built_in_for(SBIO_PROBE_USER_ID, |identity| { + seen += 1; + assert!(*identity.uuid() == [1u8; IDENTITY_UUID_LEN]); + }); + assert_eq!(seen, 1); + } + + #[test] + fn match_result_carries_the_exact_enclave_identity() { + let mut reply = [0u8; SBIO_MATCH_RESULT_LEN]; + reply[MR_USER_ID..MR_USER_ID + 4] + .copy_from_slice(&SBIO_PROBE_USER_ID.to_le_bytes()); + reply[MR_IDENTITY..MR_IDENTITY + IDENTITY_UUID_LEN].fill(0xa5); + let result = MatchResult::parse(&reply).unwrap(); + let user = UserId::new(SBIO_PROBE_USER_ID).unwrap(); + assert!(result.matches(user)); + assert!(*result.identity_uuid() == [0xa5; IDENTITY_UUID_LEN]); + } +} + const FIC_DEVICE_BUILTIN: u32 = 1; pub(crate) const SBIO_FREE_COUNT_LEN: usize = 4 + 20; static_assert!(SBIO_FREE_COUNT_LEN == SBIO_SAVE_SELECTOR_LEN); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index df71496a564fe..4e8ff6ba2caba 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -781,6 +781,9 @@ struct SepData { // only under `catacomb_writer`. deletion_unsaved: Atomic, + // PREPARE_ENROL selected the empty Linux context during this boot. + empty_context_prepared: Atomic, + #[pin] rx_work: Work, @@ -980,6 +983,7 @@ impl SepData { worker_gone <- new_condvar!("SepData::worker_gone"), catacomb_writer <- new_mutex!(()), deletion_unsaved: Atomic::new(false), + empty_context_prepared: Atomic::new(false), rx_work <- new_work!("SepData::rx_work"), enrol_work <- new_work!("SepData::enrol_work"), verify_work <- new_work!("SepData::verify_work"), diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 0425bf2cdaa6e..69b18c9110c66 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -44,6 +44,7 @@ void sep_hwrng_unregister(void *mem); void *sep_store_open(const char *path); void *sep_store_open_host(const char *path, bool *created); void *sep_store_open_trunc(const char *path); +void *sep_store_open_writable_existing(const char *path); void *sep_store_open_ro(const char *path); void *sep_store_open_block(const char *path, int writable); void sep_store_close(void *handle); diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index 8e30c3f54b817..6d546a5603f37 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -9,6 +9,7 @@ extern "C" { fn sep_store_open(path: *const c_char) -> *mut c_void; fn sep_store_open_host(path: *const c_char, created: *mut bool) -> *mut c_void; fn sep_store_open_trunc(path: *const c_char) -> *mut c_void; + fn sep_store_open_writable_existing(path: *const c_char) -> *mut c_void; fn sep_store_open_ro(path: *const c_char) -> *mut c_void; fn sep_store_open_block(path: *const c_char, writable: c_int) -> *mut c_void; fn sep_store_close(handle: *mut c_void); @@ -80,6 +81,15 @@ impl StoreFile { Ok(StoreFile { handle }) } + pub(crate) fn open_writable_existing(path: &CStr) -> Result { + // SAFETY: `path` is NUL-terminated; this open cannot create or truncate. + let handle = unsafe { sep_store_open_writable_existing(path.as_char_ptr()) }; + if handle.is_null() { + return Err(EROFS); + } + Ok(StoreFile { handle }) + } + pub(crate) fn open_readonly(path: &CStr) -> Result { // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. let handle = unsafe { sep_store_open_ro(path.as_char_ptr()) }; diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index e3abe11c7d5c8..bff68e070db4b 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -359,6 +359,12 @@ impl Store { self.slots.iter().position(|s| !s.used) } + /// Admission check for the copy-on-write slot used by the next write. + /// Even replacing an existing key needs a free slot until commit. + pub(crate) fn can_write(&self, key: &Key, len: usize) -> bool { + valid_kind(key.kind) && len <= MAX_VALUE && self.find_free().is_some() + } + fn block_offset(slot: usize) -> u64 { ((slot + 1) * BLOCK_SIZE) as u64 } diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c index d50270d717be5..77319a4702164 100644 --- a/drivers/soc/apple/store_shim.c +++ b/drivers/soc/apple/store_shim.c @@ -83,6 +83,14 @@ void *sep_store_open_trunc(const char *path) return IS_ERR(f) ? NULL : f; } +/* Admission check only: never create or truncate an existing record. */ +void *sep_store_open_writable_existing(const char *path) +{ + struct file *f = open_as_kernel(path, O_WRONLY | O_LARGEFILE, 0); + + return IS_ERR(f) ? NULL : f; +} + /* * Opens an existing file read-only, NULL if absent. O_RDONLY and no O_CREAT so * the seed cannot be created, truncated or written by mistake. diff --git a/include/uapi/linux/apple-sep.h b/include/uapi/linux/apple-sep.h index 394b497ff7a59..fbae982a4c0ef 100644 --- a/include/uapi/linux/apple-sep.h +++ b/include/uapi/linux/apple-sep.h @@ -14,7 +14,7 @@ #include #include -#define SEP_BIO_IFACE_VERSION 4 +#define SEP_BIO_IFACE_VERSION 5 #define SEP_BIO_UUID_LEN 16 #define SEP_BIO_LABEL_LEN 128 @@ -64,10 +64,22 @@ struct sep_bio_list { struct sep_bio_enrol_start { __u32 flags; + __u32 account_uid; /* resolved passwd uid; UINT32_MAX is invalid */ + __u32 finger; /* FP_FINGER_LEFT_THUMB..FP_FINGER_RIGHT_LITTLE (1..10) */ __u32 reserved; __u8 label[SEP_BIO_LABEL_LEN]; }; +/* Linux's built-in SEP user context only. This does not enumerate other SEP + * users or OS lanes. DELETE_ORPHAN uses an exact UUID; the kernel index may + * still hold it when fprintd has already lost its own record. + */ +struct sep_bio_enclave_list { + __u32 count; + __u32 reserved; + __u8 uuid[SEP_BIO_MAX_IDENTITIES][SEP_BIO_UUID_LEN]; +}; + #define SEP_BIO_GUIDANCE_NONE 0 #define SEP_BIO_GUIDANCE_PLACE 1 #define SEP_BIO_GUIDANCE_LIFT_AND_MOVE 2 @@ -129,5 +141,11 @@ struct sep_bio_attest { #define SEP_BIO_DELETE _IOW(SEP_BIO_IOC_MAGIC, 0x08, struct sep_bio_delete) #define SEP_BIO_DELETE_ALL _IO(SEP_BIO_IOC_MAGIC, 0x09) #define SEP_BIO_ATTEST _IOWR(SEP_BIO_IOC_MAGIC, 0x0a, struct sep_bio_attest) +#define SEP_BIO_LIST_ENCLAVE _IOR(SEP_BIO_IOC_MAGIC, 0x0b, struct sep_bio_enclave_list) +#define SEP_BIO_DELETE_ORPHAN _IOW(SEP_BIO_IOC_MAGIC, 0x0c, struct sep_bio_delete) +/* Creates the empty Linux owner context only when SEP lists zero Linux + * identities. ENROL_START itself never repairs a missing persistence binding. + */ +#define SEP_BIO_PREPARE_ENROL _IOW(SEP_BIO_IOC_MAGIC, 0x0d, struct sep_bio_enrol_start) #endif /* _UAPI_LINUX_APPLE_SEP_H */ From b42591955c08df88bbe7361f005ba768317828f3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 23:18:14 +1100 Subject: [PATCH 1327/1579] tools: aurora-sep: provide exact-ID Linux identity recovery Install a utility that lists the selected Linux SEP user context and deletes one named identity while requiring the operator to name identities that must remain. Document the recovery and admission sequence and test the CLI and package sources. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 20 +++++ .../packaging/arch/aurora-sep/PKGBUILD | 11 ++- .../arch/aurora-sep/sep-bio-identities | 1 + .../packaging/fedora/aurora-sep.spec | 8 +- tools/aurora-sep/sep-bio-identities | 82 +++++++++++++++++++ .../tests/test_sep_bio_identities.py | 44 ++++++++++ 6 files changed, 161 insertions(+), 5 deletions(-) create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/sep-bio-identities create mode 100755 tools/aurora-sep/sep-bio-identities create mode 100644 tools/aurora-sep/tests/test_sep_bio_identities.py diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index a4f96039ca6d3..142594f8218f3 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -102,6 +102,26 @@ and starts a match again by itself. When the kernel ends an enrolment because too many captures in a row were unusable (`SEP_BIO_STATUS_REJECTED`), the enrolment fails with that reason in fprintd's log, and fprintd reports `enroll-unknown-error`. Kernels from before these statuses never send them. + +Interface version 5 requires a resolved Unix uid, a known finger, the selected +Linux SEP OS identity, a live identity count below three, a bound identity bag, +and readable, writable owner/user/master Catacomb files before ENROL_START can +schedule enclave capture. On an empty Linux lane, the driver prepares the owner +context after libfprint validates the account; the user/master Catacombs are +saved at the first completed enrolment. The matching aurora libfprint build +rejects a missing passwd entry before either ioctl. A missing persistence file +for an existing identity is an admission error; repair the binding rather than +retrying a capture against unbound state. + +`sep-bio-identities` is the recovery interface for a print with no fprintd +record. As root, run `sep-bio-identities` to list only the selected Linux OS +identity's built-in SEP user context. Use `sep-bio-identities --delete-orphan +UUID --keep GOOD_UUID` when another identity must remain. The kernel index may +still hold a UUID whose fprintd record was lost, so compare the IDs carefully. +The utility does not expose DELETE_ALL. Record the list and +backup of any known-good print before deletion, and confirm the deleted UUID +stays absent after a reboot. This path must be tested on the affected hardware +before treating a deletion as a successful recovery. The device-specific `apple/mesa_calibration.bin` on m2 was recovered using the bundled [`extract-mesa-calibration.py`](extract-mesa-calibration.py), copied from [Gist revision `eb079a8007985d04e75182f20994f1a1c496f12e`](https://gist.github.com/DjDeveloperr/867a1961b861c570442724f48f770158/eb079a8007985d04e75182f20994f1a1c496f12e), diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index fc44475693abb..1a57d48551d87 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -7,7 +7,7 @@ pkgname=aurora-sep pkgver=1 -pkgrel=13 +pkgrel=14 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -31,7 +31,8 @@ source=(load-driver aurora-sep-greeter-fingerprint.service README.md XART-SAFETY.md - OS-IDENTITY.md) + OS-IDENTITY.md + sep-bio-identities) sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '447f99cc3c7e683134f31b993aa825fadde24e4854954a3035c96e6631d4d4d0' 'f8d0c4b9385e6716d5ac5dd9941a57459fbc551a8238383bcde36cd929d0523f' @@ -41,9 +42,10 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' - 'e5d684962c5323e285486e7504c2dd003ce67ff9c2fda49f9c3ec3b6dfc27563' + 'cf245fbdaa7b3596033d050900fcc94000f620072a25360a898330a8da141974' '4422988e5e7f2417791957d066c501741e929705724083da76cdf1cb70d7a2fc' - '52020e636e62b60f0c3e6251c3b67b8a8c409bff0486bc3463f356fd3073384b') + '52020e636e62b60f0c3e6251c3b67b8a8c409bff0486bc3463f356fd3073384b' + '6209d874a8d2950e5c6a68e2a9c19ffd4ba6e86a27bde3351f37e69290830349') # A unit and the helper it runs ($1, installed as $3), pointed at the # packaged helper. @@ -71,5 +73,6 @@ package() { >"$pkgdir/usr/lib/systemd/system/fprintd.service.d/50-aurora-sep-no-timeout.conf" install -Dm644 logind-aurora-sep.conf \ "$pkgdir/usr/lib/systemd/logind.conf.d/50-aurora-sep.conf" + install -Dm755 sep-bio-identities "$pkgdir/usr/bin/sep-bio-identities" install -Dm644 -t "$pkgdir/usr/share/doc/$pkgname" README.md XART-SAFETY.md OS-IDENTITY.md } diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/sep-bio-identities b/tools/aurora-sep/packaging/arch/aurora-sep/sep-bio-identities new file mode 120000 index 0000000000000..ddd469de602a7 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/sep-bio-identities @@ -0,0 +1 @@ +../../../sep-bio-identities \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index 01e39051e9ec5..b3f0fbc7f2fe7 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -8,7 +8,7 @@ Name: aurora-sep Version: 1 -Release: 8%{?dist} +Release: 9%{?dist} Summary: Loader and desktop integration for the Apple SEP fingerprint driver License: GPL-2.0-only @@ -27,6 +27,7 @@ Source10: aurora-sep-greeter-fingerprint.service Source11: aurora_sep.te Source12: aurora_sep.fc Source13: OS-IDENTITY.md +Source14: sep-bio-identities BuildArch: noarch BuildRequires: systemd-rpm-macros @@ -91,6 +92,7 @@ sed 's|@FPRINTD@|%{_libexecdir}/fprintd|' %{SOURCE7} \ >%{buildroot}%{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf install -Dpm 0644 %{SOURCE8} \ %{buildroot}%{_prefix}/lib/systemd/logind.conf.d/50-aurora-sep.conf +install -Dpm 0755 %{SOURCE14} %{buildroot}%{_bindir}/sep-bio-identities # Load the driver at boot, so the enclave is up before the login screen asks. install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset @@ -131,6 +133,7 @@ fi %{_libexecdir}/aurora-sep-load %{_libexecdir}/aurora-sep-ready %{_libexecdir}/aurora-sep-greeter-fingerprint +%{_bindir}/sep-bio-identities %{_unitdir}/aurora-sep.service %{_unitdir}/aurora-sep-ready.service %{_unitdir}/aurora-sep-greeter-fingerprint.service @@ -145,6 +148,9 @@ fi %ghost %verify(not md5 size mode mtime) %{_sharedstatedir}/selinux/%{selinuxtype}/active/modules/200/%{modulename} %changelog +* Tue Oct 27 2026 Aurora Silicon contributors - 1-9 +- Add the Linux-lane orphan identity list and delete tool + * Mon Oct 19 2026 Ryan Murray - 1-7 - XART-SAFETY.md: what the xART guard allows for restore, use, setup and recovery, per platform, and what a rollback does to its list diff --git a/tools/aurora-sep/sep-bio-identities b/tools/aurora-sep/sep-bio-identities new file mode 100755 index 0000000000000..2231434678645 --- /dev/null +++ b/tools/aurora-sep/sep-bio-identities @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""List Linux SEP identities and remove one fprintd orphan by its UUID. + +The kernel limits both ioctls to the selected Linux SEP OS identity's built-in +user context. The kernel index may still hold a UUID after fprintd lost its +record, so the operator must identify the exact orphan. This utility has no +delete-all operation. +""" + +import argparse +import fcntl +import os +import struct +import sys +import uuid + +MAGIC = 0xB1 +LIST_SIZE = 8 + 32 * 16 +DELETE_SIZE = 16 + + +def ioctl_number(direction, number, size): + return (direction << 30) | (size << 16) | (MAGIC << 8) | number + + +LIST_ENCLAVE = ioctl_number(2, 0x0B, LIST_SIZE) +DELETE_ORPHAN = ioctl_number(1, 0x0C, DELETE_SIZE) + + +def list_identities(fd): + reply = bytearray(LIST_SIZE) + fcntl.ioctl(fd, LIST_ENCLAVE, reply, True) + count, reserved = struct.unpack_from("=II", reply) + if reserved != 0 or count > 32: + raise ValueError("invalid SEP identity-list reply") + return [uuid.UUID(bytes=bytes(reply[8 + 16 * i:24 + 16 * i])) + for i in range(count)] + + +def check_target(before, target, keep): + if target not in before: + raise ValueError("identity is absent from the Linux-lane list") + if len(before) > 1 and not keep: + raise ValueError("name the identity to preserve with --keep before deleting") + if target in keep or any(identity not in before for identity in keep): + raise ValueError("--keep must list a different identity present in the Linux lane") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--delete-orphan", type=uuid.UUID, metavar="UUID", + help="delete exactly this Linux-lane identity with no fprintd record") + parser.add_argument("--keep", type=uuid.UUID, action="append", default=[], metavar="UUID", + help="identity that must remain; required when more than one is listed") + args = parser.parse_args() + + try: + with os.fdopen(os.open("/dev/sep-bio", os.O_RDWR | os.O_CLOEXEC), "rb", closefd=True) as device: + fd = device.fileno() + before = list_identities(fd) + if args.delete_orphan is None: + for identity in before: + print(identity) + return 0 + check_target(before, args.delete_orphan, args.keep) + fcntl.ioctl(fd, DELETE_ORPHAN, args.delete_orphan.bytes) + after = list_identities(fd) + if args.delete_orphan in after: + raise RuntimeError("SEP still lists the identity after deletion") + if any(identity not in after for identity in args.keep): + raise RuntimeError("a protected identity is absent after deletion") + print("deleted orphan", args.delete_orphan) + print("Linux-lane identities:", len(before), "->", len(after)) + return 0 + except (OSError, ValueError, RuntimeError) as error: + print(f"sep-bio-identities: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-sep/tests/test_sep_bio_identities.py b/tools/aurora-sep/tests/test_sep_bio_identities.py new file mode 100644 index 0000000000000..23505157866f5 --- /dev/null +++ b/tools/aurora-sep/tests/test_sep_bio_identities.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0-only +"""Synthetic tests for the exact-ID Linux-lane recovery utility.""" + +import runpy +import struct +import unittest +import uuid +from pathlib import Path +from unittest.mock import patch + + +CLI = runpy.run_path(str(Path(__file__).parents[1] / "sep-bio-identities"), run_name="sep_bio_test") + + +class IdentityToolTests(unittest.TestCase): + def test_list_decodes_only_counted_uuids(self): + a = uuid.UUID(int=1) + b = uuid.UUID(int=2) + + def fake_ioctl(_fd, request, reply, _mutate): + self.assertEqual(request, CLI["LIST_ENCLAVE"]) + struct.pack_into("=II", reply, 0, 2, 0) + reply[8:24] = a.bytes + reply[24:40] = b.bytes + + with patch.object(CLI["fcntl"], "ioctl", fake_ioctl): + self.assertEqual(CLI["list_identities"](7), [a, b]) + + def test_exact_target_and_keep_are_required(self): + a = uuid.UUID(int=1) + b = uuid.UUID(int=2) + self.assertIsNone(CLI["check_target"]([a, b], a, [b])) + for before, target, keep in [ + ([a, b], a, []), + ([a, b], a, [a]), + ([a, b], uuid.UUID(int=3), [b]), + ([a, b], a, [uuid.UUID(int=3)]), + ]: + with self.assertRaises(ValueError): + CLI["check_target"](before, target, keep) + + +if __name__ == "__main__": + unittest.main() From 0cfd2275d43f4d543137ffa976bcc9e9b7651a17 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 23:24:13 +1100 Subject: [PATCH 1328/1579] soc: apple: snapshot a bound bag on exact-ID recovery Use the identity-bag snapshot when recovery still has a verified bag binding. Reserve the owner-Catacomb fallback for an unbound orphan and require the persisted Linux bag file before the irreversible enclave delete. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 2f72bfa2ce7cc..6e2138af3154f 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -4037,7 +4037,7 @@ impl SepData { } // 0x57 cannot be undone. Ensure the host can save the affected user - // and master Catacombs before sending it. For an unindexed orphan the + // and master Catacombs before sending it. For an explicit recovery the // owner Catacomb provides the recovery path even when the bag snapshot // has no binding; export that owner first if this install lacks a file. for kind in [PRIVATE_TYPE_CATACOMB_USER, PRIVATE_TYPE_CATACOMB_MASTER] { @@ -4071,6 +4071,11 @@ impl SepData { dev_err!(self.dev, "bio: not deleting: indexed identity has no identity-bag binding\n"); return Err(EIO); } + if orphan && !matches!(keybag::read(keybag::Slot::Identity), + Ok(keybag::State::Present(_))) { + dev_err!(self.dev, "bio: not deleting: Linux identity bag file is unavailable\n"); + return Err(EIO); + } let op = crate::sbio::sbio_delete_identity(identity); @@ -4091,9 +4096,10 @@ impl SepData { /// the enclave marks save-pending (a SAVE_CATACOMB of one that is not has /// been refused), the user before the master because saving the user can /// change the master, state re-read before each, then the lockout and the - /// identity bag, which must match the Catacombs. Safe to repeat after a - /// failure partway: a `retry` accepts a user Catacomb an earlier attempt - /// already saved. + /// identity bag when a verified binding exists. An exact-ID orphan + /// recovery without that binding saves the Linux owner Catacomb instead. + /// Safe to repeat after a failure partway: a `retry` accepts a user + /// Catacomb an earlier attempt already saved. fn persist_deletion(&self, retry: bool, orphan: bool) -> Result<()> { let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { return Err(EIO); @@ -4151,18 +4157,23 @@ impl SepData { dev_err!(self.dev, "bio: lockout persistence failed after the deletion\n"); return Err(EIO); } - if !orphan && !self.resnapshot_identity_keybag() { - dev_err!( - self.dev, - "bio: Catacombs saved after the deletion but the identity-bag snapshot failed\n" - ); - return Err(EIO); - } - if orphan { + let bag_bound = self.enrol_material.lock().as_ref() + .is_some_and(|material| material.snapshot_identity.is_some()); + if bag_bound { + if !self.resnapshot_identity_keybag() { + dev_err!( + self.dev, + "bio: Catacombs saved after the deletion but the identity-bag snapshot failed\n" + ); + return Err(EIO); + } + } else if orphan { dev_info!( self.dev, - "bio: orphan deletion saved via Linux user/master/owner Catacombs; no unbound identity bag was snapshotted\n" + "bio: orphan deletion saved via Linux user/master/owner Catacombs without a bag snapshot binding\n" ); + } else { + return Err(EIO); } Ok(()) } From 9c755514027857565e57ba7325c5de4cb8786b18 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:34:11 +1100 Subject: [PATCH 1329/1579] soc: apple: require a designated Linux bag for recovery ioctls The scoped list, exact-ID delete and empty-owner preparation require the Linux identity bag to be designated and persisted before issuing SBIO commands. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/soc/apple/sbio.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 6e2138af3154f..6e51b0b081052 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -3542,6 +3542,12 @@ impl SepData { if !self.touchid_started.load(Relaxed) || self.xarm.lock().os_uuid.is_none() { return Err(ENXIO); } + if !self.keybag_designated.load(Relaxed) + || self.enrol_material.lock().is_none() + || !matches!(keybag::read(keybag::Slot::Identity), + Ok(keybag::State::Present(_))) { + return Err(ENODATA); + } Ok(()) } From 54f584f9cec247d60a1255caceac0279bb8cb3f0 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:34:56 +1100 Subject: [PATCH 1330/1579] ci: check SEP ioctl ABI and recovery package sources Run the scoped ioctl size check and existing recovery/package tests on Linux for SEP pull requests. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- .github/workflows/sep-source.yml | 38 ++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/sep-source.yml diff --git a/.github/workflows/sep-source.yml b/.github/workflows/sep-source.yml new file mode 100644 index 0000000000000..bcb8171021b96 --- /dev/null +++ b/.github/workflows/sep-source.yml @@ -0,0 +1,38 @@ +name: SEP source checks + +on: + pull_request: + paths: + - 'drivers/soc/apple/**' + - 'include/uapi/linux/apple-sep.h' + - 'tools/aurora-sep/**' + - '.github/workflows/sep-source.yml' + +jobs: + source: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + filter: blob:none + sparse-checkout-cone-mode: false + sparse-checkout: | + drivers/soc/apple/** + include/uapi/linux/apple-sep.h + tools/aurora-sep/** + - name: Check biometric ioctl ABI sizes + run: | + cat > /tmp/sep-abi.c <<'EOF' + #include + _Static_assert(sizeof(struct sep_bio_enrol_start) == 144, "enrol ABI"); + _Static_assert(sizeof(struct sep_bio_enclave_list) == 520, "list ABI"); + _Static_assert(sizeof(struct sep_bio_delete) == 16, "delete ABI"); + int main(void) { return 0; } + EOF + cc -Wall -Werror -Iinclude/uapi /tmp/sep-abi.c -o /tmp/sep-abi + /tmp/sep-abi + - name: Test recovery utility and package sources + run: | + python3 -m unittest tools/aurora-sep/tests/test_sep_bio_identities.py -v + python3 -m unittest discover -s tools/aurora-sep/tests -p test_packaging.py -v From 3126b0c339e2cad9940d0d69337cfad5ae9e8a96 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:36:21 +1100 Subject: [PATCH 1331/1579] tools: aurora-sep: keep recovery source non-executable The Fedora and Arch packages install the utility with mode 0755; keep the source mode aligned with kernel checkpatch conventions. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- tools/aurora-sep/sep-bio-identities | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 tools/aurora-sep/sep-bio-identities diff --git a/tools/aurora-sep/sep-bio-identities b/tools/aurora-sep/sep-bio-identities old mode 100755 new mode 100644 From eca653976a86572b8c04049e3b2fb2c77bd6b0e0 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:38:14 +1100 Subject: [PATCH 1332/1579] selftests: apple-sep: exercise orphan delete save retry Compile and execute the driver deletion orchestration with scripted enclave and persistence outcomes. Check that an indexed fprintd orphan is removable by exact ID and that failed saves retain the host index for retry. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- .github/workflows/sep-source.yml | 1 + tools/aurora-sep/tests/test_orphan_delete.py | 151 +++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 tools/aurora-sep/tests/test_orphan_delete.py diff --git a/.github/workflows/sep-source.yml b/.github/workflows/sep-source.yml index bcb8171021b96..bb5befe0515bf 100644 --- a/.github/workflows/sep-source.yml +++ b/.github/workflows/sep-source.yml @@ -35,4 +35,5 @@ jobs: - name: Test recovery utility and package sources run: | python3 -m unittest tools/aurora-sep/tests/test_sep_bio_identities.py -v + python3 -m unittest tools/aurora-sep/tests/test_orphan_delete.py -v python3 -m unittest discover -s tools/aurora-sep/tests -p test_packaging.py -v diff --git a/tools/aurora-sep/tests/test_orphan_delete.py b/tools/aurora-sep/tests/test_orphan_delete.py new file mode 100644 index 0000000000000..229f6a2c43e38 --- /dev/null +++ b/tools/aurora-sep/tests/test_orphan_delete.py @@ -0,0 +1,151 @@ +# SPDX-License-Identifier: MIT +"""Run the driver's delete-and-persist method with scripted save failures.""" + +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + + +SOURCE = Path(__file__).resolve().parents[3] / "drivers/soc/apple/sbio.rs" + + +def driver_method(name): + source = SOURCE.read_text() + match = re.search(r"^ fn " + name + r"\(", source, re.M) + if match is None: + raise AssertionError(f"missing driver method {name}") + opening = source.index("{", match.start()) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start():end] + + +HARNESS = r''' +#![allow(dead_code, unused_variables)] +use std::ffi::CStr; +use std::ops::Deref; +use std::sync::atomic::{AtomicBool, Ordering::Relaxed}; +use std::sync::{Mutex as StdMutex, MutexGuard}; + +type Error = i32; +type Result = std::result::Result; +const GFP_KERNEL: u32 = 0; +const EIO: Error = -5; + +macro_rules! dev_err { ($($arg:tt)*) => {}; } + +struct Mutex(StdMutex); +impl Mutex { + fn new(value: T) -> Self { Self(StdMutex::new(value)) } + fn lock(&self) -> MutexGuard<'_, T> { self.0.lock().unwrap() } +} + +struct KVec(Vec); +impl KVec { + fn with_capacity(count: usize, _: u32) -> Result { Ok(Self(Vec::with_capacity(count))) } + fn push_within_capacity(&mut self, value: T) -> Result<()> { self.0.push(value); Ok(()) } + fn is_empty(&self) -> bool { self.0.is_empty() } + fn iter(&self) -> std::slice::Iter<'_, T> { self.0.iter() } +} +impl Deref for KVec { + type Target = [T]; + fn deref(&self) -> &[T] { &self.0 } +} + +mod sbio { + #[derive(Clone)] + pub struct IdentityV1 { pub uuid: [u8; 16] } + impl IdentityV1 { pub fn uuid(&self) -> &[u8; 16] { &self.uuid } } +} +mod bio { + pub const UUID_LEN: usize = 16; + pub struct Handled { + pub delete_identity: Option, + pub delete_identities: Vec, + } +} + +struct Index { held: Vec<[u8; 16]> } +impl Index { + fn contains_uuid(&self, uuid: &[u8; 16]) -> bool { self.held.contains(uuid) } +} + +struct SepData { + dev: (), + deletion_unsaved: AtomicBool, + bio_index: Mutex, + delete_replies: Mutex>>, + save_replies: Mutex>>, + events: Mutex>, +} +impl SepData { + fn new(uuid: [u8; 16], saves: Vec>) -> Self { + Self { + dev: (), deletion_unsaved: AtomicBool::new(false), + bio_index: Mutex::new(Index { held: vec![uuid] }), + delete_replies: Mutex::new(vec![Ok(true), Ok(true)]), + save_replies: Mutex::new(saves), events: Mutex::new(Vec::new()), + } + } + fn delete_identity(&self, _: &sbio::IdentityV1, recovery: bool) -> Result { + self.events.lock().push(format!("delete:{recovery}")); + self.delete_replies.lock().remove(0) + } + fn persist_deletion(&self, retry: bool, recovery: bool) -> Result<()> { + self.events.lock().push(format!("save:{retry}:{recovery}")); + self.save_replies.lock().remove(0) + } + fn forget_identity(&self, uuid: &[u8; 16], _: &CStr) { + self.events.lock().push("forget".into()); + self.bio_index.lock().held.retain(|held| held != uuid); + } +METHOD +} + +fn handled(uuid: [u8; 16]) -> bio::Handled { + bio::Handled { delete_identity: Some(sbio::IdentityV1 { uuid }), delete_identities: vec![] } +} + +fn main() { + let uuid = [0xa5; 16]; + let sep = SepData::new(uuid, vec![Ok(())]); + sep.delete_and_persist(&handled(uuid), true).unwrap(); + assert_eq!(*sep.events.lock(), ["delete:true", "save:false:true", "forget"]); + assert!(!sep.bio_index.lock().contains_uuid(&uuid)); + + let sep = SepData::new(uuid, vec![Err(EIO), Ok(())]); + assert!(sep.delete_and_persist(&handled(uuid), true).is_err()); + assert!(sep.deletion_unsaved.load(Relaxed)); + assert!(sep.bio_index.lock().contains_uuid(&uuid)); + sep.delete_and_persist(&handled(uuid), true).unwrap(); + assert_eq!(*sep.events.lock(), ["delete:true", "save:false:true", + "delete:true", "save:true:true", "forget"]); + assert!(!sep.deletion_unsaved.load(Relaxed)); + assert!(!sep.bio_index.lock().contains_uuid(&uuid)); + + let sep = SepData::new(uuid, vec![]); + *sep.delete_replies.lock() = vec![Err(EIO)]; + assert!(sep.delete_and_persist(&handled(uuid), false).is_err()); + assert_eq!(*sep.events.lock(), ["delete:false"]); + assert!(sep.bio_index.lock().contains_uuid(&uuid)); +} +''' + + +class OrphanDeleteTests(unittest.TestCase): + def test_failure_keeps_index_and_retry_saves(self): + source = HARNESS.replace("METHOD", driver_method("delete_and_persist")) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + (path / "test.rs").write_text(source) + subprocess.run(["rustc", "--edition=2021", "test.rs", "-o", "test"], + cwd=path, check=True, capture_output=True, text=True) + subprocess.run([str(path / "test")], check=True, capture_output=True, text=True) + + +if __name__ == "__main__": + unittest.main() From 3694c1661ceaa16bf22fc5f9dde1672493d0fcc4 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:39:33 +1100 Subject: [PATCH 1333/1579] selftests: apple-sep: execute enrol refusal cases Compile the kernel admission functions in a host harness and exercise missing UID, unknown finger, OS identity, bag, Catacomb and storage-capacity refusals before hardware testing. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- .github/workflows/sep-source.yml | 1 + .../aurora-sep/tests/test_enrol_admission.py | 129 ++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 tools/aurora-sep/tests/test_enrol_admission.py diff --git a/.github/workflows/sep-source.yml b/.github/workflows/sep-source.yml index bb5befe0515bf..cd75b0f266306 100644 --- a/.github/workflows/sep-source.yml +++ b/.github/workflows/sep-source.yml @@ -36,4 +36,5 @@ jobs: run: | python3 -m unittest tools/aurora-sep/tests/test_sep_bio_identities.py -v python3 -m unittest tools/aurora-sep/tests/test_orphan_delete.py -v + python3 -m unittest tools/aurora-sep/tests/test_enrol_admission.py -v python3 -m unittest discover -s tools/aurora-sep/tests -p test_packaging.py -v diff --git a/tools/aurora-sep/tests/test_enrol_admission.py b/tools/aurora-sep/tests/test_enrol_admission.py new file mode 100644 index 0000000000000..5bb688d3b5571 --- /dev/null +++ b/tools/aurora-sep/tests/test_enrol_admission.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: MIT +"""Execute the kernel's enrol request and persistence admission functions.""" + +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + + +APPLE = Path(__file__).resolve().parents[3] / "drivers/soc/apple" + + +def function(file, name): + source = (APPLE / file).read_text() + match = re.search(r"^fn " + name + r"\(", source, re.M) + if match is None: + raise AssertionError(f"missing driver function {name}") + opening = source.index("{", match.start()) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start():end] + + +HARNESS = r''' +#![allow(dead_code)] +type Error = i32; +type Result = std::result::Result; +const EINVAL: Error = -22; +const ENXIO: Error = -6; +const EIO: Error = -5; +const ENOSPC: Error = -28; +const ENODATA: Error = -61; +const EROFS: Error = -30; +const PRIVATE_TYPE_CATACOMB_OWNER: u8 = 0xf2; +const PRIVATE_TYPE_CATACOMB_USER: u8 = 0xf3; +const PRIVATE_TYPE_CATACOMB_MASTER: u8 = 0xf4; + +struct EnrolPrerequisites { + lane: bool, + live_count: Option, + bag_bound: bool, + catacomb_readable: [bool; 3], + catacomb_writable: [bool; 3], + keybag_writable: bool, + host_store_capacity: bool, +} +const LABEL_LEN: usize = 128; +struct EnrolStart { + flags: u32, + account_uid: u32, + finger: u32, + reserved: u32, + label: [u8; LABEL_LEN], +} + +FUNCTIONS + +fn ready() -> EnrolPrerequisites { + EnrolPrerequisites { + lane: true, live_count: Some(1), bag_bound: true, + catacomb_readable: [true; 3], catacomb_writable: [true; 3], + keybag_writable: true, host_store_capacity: true, + } +} +fn request() -> EnrolStart { + let mut r = EnrolStart { + flags: 0, account_uid: 1000, finger: 7, reserved: 0, + label: [0; LABEL_LEN], + }; + r.label[..28].copy_from_slice(b"FP1-20261027-7-12345678-user"); + r +} + +fn main() { + assert!(admit_enrol(&ready()).is_ok()); + let mut p = ready(); p.lane = false; assert_eq!(admit_enrol(&p), Err(ENXIO)); + let mut p = ready(); p.live_count = None; assert_eq!(admit_enrol(&p), Err(EIO)); + let mut p = ready(); p.live_count = Some(3); assert_eq!(admit_enrol(&p), Err(ENOSPC)); + let mut p = ready(); p.bag_bound = false; assert_eq!(admit_enrol(&p), Err(ENODATA)); + for i in 0..3 { + let mut p = ready(); p.catacomb_readable[i] = false; + assert_eq!(admit_enrol(&p), Err(ENODATA)); + let mut p = ready(); p.catacomb_writable[i] = false; + assert_eq!(admit_enrol(&p), Err(EROFS)); + } + let mut p = ready(); p.keybag_writable = false; assert_eq!(admit_enrol(&p), Err(EROFS)); + let mut p = ready(); p.host_store_capacity = false; assert_eq!(admit_enrol(&p), Err(ENOSPC)); + + assert!(catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_OWNER, Some(0))); + assert!(!catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_USER, Some(0))); + assert!(!catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_MASTER, Some(0))); + assert!(catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_USER, Some(1))); + + assert!(validate_enrol_request(&request()).is_ok()); + let mut r = request(); r.account_uid = u32::MAX; + assert_eq!(validate_enrol_request(&r), Err(EINVAL)); + let mut r = request(); r.finger = 0; + assert_eq!(validate_enrol_request(&r), Err(EINVAL)); + let mut r = request(); r.finger = 11; + assert_eq!(validate_enrol_request(&r), Err(EINVAL)); + let mut r = request(); r.finger = 6; + assert_eq!(validate_enrol_request(&r), Err(EINVAL)); + let mut r = request(); r.label[24] = 0; + assert_eq!(validate_enrol_request(&r), Err(EINVAL)); +} +''' + + +class EnrolAdmissionTests(unittest.TestCase): + def test_refused_cases_before_capture(self): + methods = "\n".join([ + function("sbio.rs", "admit_enrol"), + function("sbio.rs", "catacomb_needed_for_enrol"), + function("bio.rs", "validate_enrol_request"), + ]) + source = HARNESS.replace("FUNCTIONS", methods) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + (path / "test.rs").write_text(source) + subprocess.run(["rustc", "--edition=2021", "test.rs", "-o", "test"], + cwd=path, check=True, capture_output=True, text=True) + subprocess.run([str(path / "test")], check=True, capture_output=True, text=True) + + +if __name__ == "__main__": + unittest.main() From 60a47463ecab303d0cc30e7b637704bc8e5aed8b Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:41:54 +1100 Subject: [PATCH 1334/1579] soc: apple: preflight clean owner preparation writes Check sensor availability, identity-bag file writability and host index capacity before PREPARE_ENROL selects or saves an empty owner context. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/soc/apple/sbio.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 6e51b0b081052..726678efef5a2 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -3618,6 +3618,7 @@ impl SepData { bio::validate_enrol_start_arg(arg)?; self.require_linux_bio_lane()?; if !bio::idle_for_mutation(&self.bio_session.lock()) { return Err(EBUSY); } + if !sensor::is_bound() { return Err(ENODEV); } if !self.xart_writable() { return Err(EROFS); } let ids = self.enclave_identities_for(SBIO_PROBE_USER_ID).ok_or(EIO)?; if !ids.is_empty() { @@ -3635,6 +3636,14 @@ impl SepData { .is_some_and(|material| material.snapshot_identity.is_some()) { return Err(ENODATA); } + if shim::StoreFile::open_writable_existing(keybag::Slot::Identity.path()).is_err() { + return Err(EROFS); + } + let index = self.bio_index.lock(); + if !self.with_host_store(|store| index.can_persist_next(store)).unwrap_or(false) { + return Err(ENOSPC); + } + drop(index); let user = crate::sbio::UserId::new(SBIO_PROBE_USER_ID).ok_or(EINVAL)?; loop { self.wait_until_awake()?; From cf588fef6e074e68286dc3e62c0a15ad2c779cf9 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:42:43 +1100 Subject: [PATCH 1335/1579] soc: apple: reserve owner file before fresh SEP context Create and sync the missing Linux owner file before SELECT_CONTEXT writes to the enclave. Refuse an existing invalid file without truncating its failure evidence. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/soc/apple/catacomb.rs | 12 ++++++++++++ drivers/soc/apple/sbio.rs | 3 +++ 2 files changed, 15 insertions(+) diff --git a/drivers/soc/apple/catacomb.rs b/drivers/soc/apple/catacomb.rs index 9c60e19b5861d..d71b415fcb321 100644 --- a/drivers/soc/apple/catacomb.rs +++ b/drivers/soc/apple/catacomb.rs @@ -34,6 +34,18 @@ pub(crate) fn writable(kind: u8) -> bool { .is_some() } +/// Reserve a missing Linux owner file before SELECT_CONTEXT changes SEP state. +/// A corrupt existing file is never truncated here: it is failure evidence. +pub(crate) fn prepare_missing_owner() -> Result<()> { + let path = path_for(crate::PRIVATE_TYPE_CATACOMB_OWNER).ok_or(EINVAL)?; + let (file, created) = shim::StoreFile::open_host(path)?; + if !created { + return Err(ENODATA); + } + file.sync()?; + file.sync_parent() +} + pub(crate) fn write(kind: u8, blob: &[u8]) -> Result<()> { let path = path_for(kind).ok_or(EINVAL)?; if blob.len() > MAX_CATACOMB { diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 726678efef5a2..a71695a23195d 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -3645,6 +3645,9 @@ impl SepData { } drop(index); let user = crate::sbio::UserId::new(SBIO_PROBE_USER_ID).ok_or(EINVAL)?; + // The host must be able to create and sync the owner file before a + // system-context write. A pre-existing invalid file is left intact. + crate::catacomb::prepare_missing_owner()?; loop { self.wait_until_awake()?; let _writer = self.catacomb_writer.lock(); From f7e5b4a2d09597d6a523f8260138436f3fc488f9 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:43:16 +1100 Subject: [PATCH 1336/1579] selftests: apple-sep: check owner-file preparation outcomes Execute the driver preflight function with a scripted file backend and verify that a corrupt existing owner file is not truncated before SEP context writes. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- .github/workflows/sep-source.yml | 1 + .../tests/test_owner_preparation.py | 90 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 tools/aurora-sep/tests/test_owner_preparation.py diff --git a/.github/workflows/sep-source.yml b/.github/workflows/sep-source.yml index cd75b0f266306..aecbf352d3ec3 100644 --- a/.github/workflows/sep-source.yml +++ b/.github/workflows/sep-source.yml @@ -37,4 +37,5 @@ jobs: python3 -m unittest tools/aurora-sep/tests/test_sep_bio_identities.py -v python3 -m unittest tools/aurora-sep/tests/test_orphan_delete.py -v python3 -m unittest tools/aurora-sep/tests/test_enrol_admission.py -v + python3 -m unittest tools/aurora-sep/tests/test_owner_preparation.py -v python3 -m unittest discover -s tools/aurora-sep/tests -p test_packaging.py -v diff --git a/tools/aurora-sep/tests/test_owner_preparation.py b/tools/aurora-sep/tests/test_owner_preparation.py new file mode 100644 index 0000000000000..29c1217d7eb24 --- /dev/null +++ b/tools/aurora-sep/tests/test_owner_preparation.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: MIT +"""Execute the kernel's owner-file preflight against scripted file outcomes.""" + +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + + +SOURCE = Path(__file__).resolve().parents[3] / "drivers/soc/apple/catacomb.rs" + + +def method(): + source = SOURCE.read_text() + match = re.search(r"^pub\(crate\) fn prepare_missing_owner\(", source, re.M) + if match is None: + raise AssertionError("missing owner preparation function") + opening = source.index("{", match.start()) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start():end] + + +HARNESS = r''' +use std::ffi::CStr; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering::Relaxed}; +type Error = i32; +type Result = std::result::Result; +const EINVAL: Error = -22; +const ENODATA: Error = -61; +const EROFS: Error = -30; +const PRIVATE_TYPE_CATACOMB_OWNER: u8 = 0xf2; + +fn path_for(kind: u8) -> Option<&'static CStr> { + (kind == PRIVATE_TYPE_CATACOMB_OWNER).then_some(c"/var/lib/apple-sep-catacomb-owner.bin") +} + +mod shim { + use super::*; + pub static CREATE: AtomicBool = AtomicBool::new(true); + pub static FAIL_OPEN: AtomicBool = AtomicBool::new(false); + pub static SYNC: AtomicU32 = AtomicU32::new(0); + pub static PARENT: AtomicU32 = AtomicU32::new(0); + pub struct StoreFile; + impl StoreFile { + pub fn open_host(_: &CStr) -> Result<(StoreFile, bool)> { + if FAIL_OPEN.load(Relaxed) { Err(EROFS) } + else { Ok((StoreFile, CREATE.load(Relaxed))) } + } + pub fn sync(&self) -> Result<()> { SYNC.fetch_add(1, Relaxed); Ok(()) } + pub fn sync_parent(&self) -> Result<()> { PARENT.fetch_add(1, Relaxed); Ok(()) } + } +} + +METHOD + +fn main() { + assert_eq!(prepare_missing_owner(), Ok(())); + assert_eq!(shim::SYNC.load(Relaxed), 1); + assert_eq!(shim::PARENT.load(Relaxed), 1); + + shim::CREATE.store(false, Relaxed); + assert_eq!(prepare_missing_owner(), Err(ENODATA)); + assert_eq!(shim::SYNC.load(Relaxed), 1); + assert_eq!(shim::PARENT.load(Relaxed), 1); + + shim::FAIL_OPEN.store(true, Relaxed); + assert_eq!(prepare_missing_owner(), Err(EROFS)); + assert_eq!(shim::SYNC.load(Relaxed), 1); + assert_eq!(shim::PARENT.load(Relaxed), 1); +} +''' + + +class OwnerPreparationTests(unittest.TestCase): + def test_invalid_file_is_preserved_before_sep_write(self): + source = HARNESS.replace("METHOD", method()) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + (path / "test.rs").write_text(source) + subprocess.run(["rustc", "--edition=2021", "test.rs", "-o", "test"], + cwd=path, check=True, capture_output=True, text=True) + subprocess.run([str(path / "test")], check=True, capture_output=True, text=True) + + +if __name__ == "__main__": + unittest.main() From 78d4fc29388e2b4f974818702c8ac786ff4fbf6e Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:56:13 +1100 Subject: [PATCH 1337/1579] ci: cross-compile Apple SEP Rust module Add a targeted arm64 module build so the biometric driver changes receive a kernel type-check in addition to the source and synthetic tests. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- .github/workflows/sep-source.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/sep-source.yml b/.github/workflows/sep-source.yml index aecbf352d3ec3..70b8320a4b962 100644 --- a/.github/workflows/sep-source.yml +++ b/.github/workflows/sep-source.yml @@ -39,3 +39,29 @@ jobs: python3 -m unittest tools/aurora-sep/tests/test_enrol_admission.py -v python3 -m unittest tools/aurora-sep/tests/test_owner_preparation.py -v python3 -m unittest discover -s tools/aurora-sep/tests -p test_packaging.py -v + + rust-build: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Install kernel build dependencies + run: | + sudo apt-get update + sudo apt-get install -y clang lld llvm libclang-dev libelf-dev libssl-dev \ + flex bison bc pkg-config make perl + rustup component add rust-src + cargo install bindgen-cli --version 0.71.1 --locked + - name: Compile Apple SEP module for arm64 + env: + LIBCLANG_PATH: /usr/lib/llvm-18/lib + run: | + make LLVM=1 ARCH=arm64 rustavailable + make LLVM=1 ARCH=arm64 defconfig + scripts/kconfig/merge_config.sh -m .config arch/arm64/configs/asahi.config + make LLVM=1 ARCH=arm64 olddefconfig + grep -qx 'CONFIG_APPLE_SEP=m' .config + make -j2 LLVM=1 ARCH=arm64 modules_prepare + make -j2 LLVM=1 ARCH=arm64 drivers/soc/apple/apple-sep.o From b8b8e2693c47cbb94ba181a7a700df81b7d0e6fa Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:59:44 +1100 Subject: [PATCH 1338/1579] ci: type-check Apple SEP KUnit cases in arm64 build Enable KUnit and the SEP test configuration for the targeted Rust module build so the new admission and identity tests are compiled. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- .github/workflows/sep-source.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/sep-source.yml b/.github/workflows/sep-source.yml index 70b8320a4b962..052f282d60061 100644 --- a/.github/workflows/sep-source.yml +++ b/.github/workflows/sep-source.yml @@ -61,7 +61,9 @@ jobs: make LLVM=1 ARCH=arm64 rustavailable make LLVM=1 ARCH=arm64 defconfig scripts/kconfig/merge_config.sh -m .config arch/arm64/configs/asahi.config + scripts/config -e KUNIT -e APPLE_SEP_KUNIT_TEST make LLVM=1 ARCH=arm64 olddefconfig grep -qx 'CONFIG_APPLE_SEP=m' .config + grep -qx 'CONFIG_APPLE_SEP_KUNIT_TEST=y' .config make -j2 LLVM=1 ARCH=arm64 modules_prepare make -j2 LLVM=1 ARCH=arm64 drivers/soc/apple/apple-sep.o From cfbedf606e557cb3a5ba3ba645df0a8272d1029f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 10:42:55 +1100 Subject: [PATCH 1339/1579] wifi: brcmfmac: query assoc_info with a full-sized buffer Since commit cda37449d576 ("wifi: brcmfmac bound association IEs to returned payloads") the "assoc_info" iovar is queried with an 8-byte buffer, the size of the two lengths the driver reads. The firmware fails a get whose buffer is shorter than its whole association record, so every association on the BCM4387 and BCM4388 now logs brcmf_get_assoc_ies: could not get assoc info (-52) and cfg80211 gets no association request or response IEs, and the WMM AC priorities are never read. Saved boot logs show the message on every association after that change and on none before it. Query with WL_ASSOC_INFO_MAX bytes as before, into the buffer the IE queries already use, and keep the bounds checks on what the firmware returns. Fixes: cda37449d576 ("wifi: brcmfmac bound association IEs to returned payloads") Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 1ded4c578b5f5..989546c595291 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6572,30 +6572,37 @@ static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, s32 err; brcmf_clear_assoc_ies(cfg); - err = brcmf_fil_iovar_data_get_len(ifp, "assoc_info", &assoc_info, - sizeof(assoc_info), &received); + buffer = kzalloc(WL_EXTRA_BUF_MAX, GFP_KERNEL); + if (!buffer) + return -ENOMEM; + /* + * The firmware fails the query with a buffer shorter than its whole + * association record, so ask with WL_ASSOC_INFO_MAX bytes and take + * the two lengths that lead the reply. + */ + err = brcmf_fil_iovar_data_get_len(ifp, "assoc_info", buffer, + WL_ASSOC_INFO_MAX, &received); if (err) { bphy_err(drvr, "could not get assoc info (%d)\n", err); - return err; + goto done; } if (received < sizeof(assoc_info)) { bphy_err(drvr, "short assoc info: received %u required %zu\n", received, sizeof(assoc_info)); - return -EBADMSG; + err = -EBADMSG; + goto done; } + memcpy(&assoc_info, buffer, sizeof(assoc_info)); req_len = le32_to_cpu(assoc_info.req_len); resp_len = le32_to_cpu(assoc_info.resp_len); if (req_len > WL_EXTRA_BUF_MAX || resp_len > WL_EXTRA_BUF_MAX) { bphy_err(drvr, "invalid lengths in assoc info: req %u resp %u\n", req_len, resp_len); - return -EINVAL; - } - if (req_len || resp_len) { - buffer = kzalloc(WL_EXTRA_BUF_MAX, GFP_KERNEL); - if (!buffer) - return -ENOMEM; + err = -EINVAL; + goto done; } if (req_len) { + memset(buffer, 0, WL_EXTRA_BUF_MAX); err = brcmf_fil_iovar_data_get_len(ifp, "assoc_req_ies", buffer, max_t(u32, req_len, WL_ASSOC_INFO_MAX), &received); From 4d1241d8fb1d89162ee2dfccec44b21f731d46cc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 10:44:16 +1100 Subject: [PATCH 1340/1579] drm/apple: expose the pixel blend mode of DCP planes Every DCP plane advertises formats with alpha but no "pixel blend mode" property, so drm_mode_config_validate() warns once per plane at boot: [PLANE:35:plane-0] pixel format with alpha exposed but blend mode not setup. Please fix. Without the property userspace must assume pre-multiplied alpha, while the driver leaves is_premultiplied clear for every alpha format and DCP blends those by coverage. Create the property. DCP blends pre-multiplied when the surface's is_premultiplied flag is set and by coverage when it is clear, so expose both, with pre-multiplied as the default the uAPI documents, and set the flag from the plane state. The H17P swap surface has no is_premultiplied byte and its firmware applies alpha as coverage, so H17P planes expose coverage only. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/apple/apple_drv.c | 6 +++++- drivers/gpu/drm/apple/plane.c | 21 +++++++++++++++++++-- drivers/gpu/drm/apple/plane.h | 3 ++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 8fc8cb7f7a6ab..71af3024a0266 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -639,6 +639,7 @@ static int apple_probe_per_dcp(struct device *dev, bool supports_l10r = !dcp_fw_compat_is_12_x(dcp); struct apple_dcp *dcp_data = platform_get_drvdata(dcp); bool supports_xrgb2101010; + bool supports_premultiplied; enum drm_plane_type plane_type; bool interchange; bool supports_interchange; @@ -650,6 +651,8 @@ static int apple_probe_per_dcp(struct device *dev, * (T8140 and T8122): advertise linear framebuffers only there. */ supports_interchange = dcp_data->fw_compat != DCP_FIRMWARE_H17P; + /* The H17P swap surface has no is_premultiplied flag. */ + supports_premultiplied = dcp_data->fw_compat != DCP_FIRMWARE_H17P; if (bitmap_empty(iomfb_surfaces, DCP_MAX_PLANES)) return dev_err_probe(dev, -EINVAL, "No usable display surfaces\n"); @@ -689,7 +692,8 @@ static int apple_probe_per_dcp(struct device *dev, planes[zpos] = apple_plane_init(drm, 1U << num, surf, supports_l10r, supports_xrgb2101010, - plane_type, interchange); + plane_type, interchange, + supports_premultiplied); } if (IS_ERR(planes[zpos])) return PTR_ERR(planes[zpos]); diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index a0343aa5ec5ec..87fecce18ee20 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -12,6 +12,7 @@ #include "iomfb_plane.h" #include #include +#include #include #include #include @@ -429,7 +430,8 @@ static void apple_plane_atomic_update(struct drm_plane *plane, */ if (fmt->format == DRM_FORMAT_XRGB8888 || fmt->format == DRM_FORMAT_XBGR8888 || - fmt->format == DRM_FORMAT_XBGR2101010) + fmt->format == DRM_FORMAT_XBGR2101010 || + base->pixel_blend_mode == DRM_MODE_BLEND_PREMULTI) is_premultiplied = true; new_state->src_rect = drm_to_dcp_rect_fp(&base->src); @@ -742,11 +744,14 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, bool supports_l10r, bool supports_xrgb2101010, enum drm_plane_type type, - bool interchange) + bool interchange, + bool premultiplied) { struct apple_plane *plane; + unsigned int blend_modes; const u32 *fmts; u32 num_fmts; + int ret; switch (type) { case DRM_PLANE_TYPE_PRIMARY: @@ -786,6 +791,18 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, if (IS_ERR(plane)) return ERR_PTR(PTR_ERR(plane)); + /* + * The surface flag is_premultiplied selects pre-multiplied blending; + * without it DCP applies the alpha channel as coverage. Firmware whose + * swap surface has no such flag only blends by coverage. + */ + blend_modes = BIT(DRM_MODE_BLEND_COVERAGE); + if (premultiplied) + blend_modes |= BIT(DRM_MODE_BLEND_PREMULTI); + ret = drm_plane_create_blend_mode_property(&plane->base, blend_modes); + if (ret) + return ERR_PTR(ret); + drm_plane_create_color_properties(&plane->base, (1 << DRM_COLOR_ENCODING_MAX) - 1, (1 << DRM_COLOR_RANGE_MAX) - 1, diff --git a/drivers/gpu/drm/apple/plane.h b/drivers/gpu/drm/apple/plane.h index f7a819c73645a..69229ad6a5775 100644 --- a/drivers/gpu/drm/apple/plane.h +++ b/drivers/gpu/drm/apple/plane.h @@ -38,7 +38,8 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, bool supports_l10r, bool supports_xrgb2101010, enum drm_plane_type type, - bool interchange); + bool interchange, + bool premultiplied); struct drm_plane *apple_plane_init_v14_7(struct drm_device *dev, unsigned long possible_crtcs, u32 iomfb_surf, From 6905adadf7079291cb3a063e830633a31513c5cf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 10:44:51 +1100 Subject: [PATCH 1341/1579] iio: aop_als: report a missing calibration blob once Without apple/aop-als-cal.bin the probe logs two warnings for one fact, the firmware loader's Direct firmware load for apple/aop-als-cal.bin failed with error -2 and the driver's own "no calibration". The driver's message is the one that says what the file is for, so request the blob with request_nowarn(). Correct the comment and the message as well. They say the sensor still reports without the blob, but the VD6286 of the M4 Pro (J616s) never sends a report without it: its channel reads ENODATA for the whole boot and the AOP log shows "Calibration Override:0". Say that the sensor may not report, so the warning points at the file that is missing. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/iio/common/aop_sensors/aop_als.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/iio/common/aop_sensors/aop_als.rs b/drivers/iio/common/aop_sensors/aop_als.rs index 3e262a003c9ea..bec8459aba2c4 100644 --- a/drivers/iio/common/aop_sensors/aop_als.rs +++ b/drivers/iio/common/aop_sensors/aop_als.rs @@ -72,15 +72,18 @@ fn enable_als( offset: usize, ) -> Result<()> { if offset != LUX_OFFSET_CT817 { - // The calibration blob comes from macOS and is not installed on every - // system. Without it the sensor still reports, uncalibrated. - match Firmware::request(c_str!("apple/aop-als-cal.bin"), dev.as_ref()) { + // The installer collects the calibration blob from macOS, so it is + // not present on every system. Without it the VD6286 of the M4 Pro + // never reports and its channel reads ENODATA. The warning below + // reports the missing file once, so the firmware loader is asked not + // to report it too. + match Firmware::request_nowarn(c_str!("apple/aop-als-cal.bin"), dev.as_ref()) { Ok(fw) => { set_als_property(aop, svc, ALS_PROP_CALIBRATION, fw.data())?; } Err(e) => dev_warn!( dev.as_ref(), - "no calibration ({:?}); readings are uncalibrated\n", + "no apple/aop-als-cal.bin ({:?}); the sensor may not report\n", e ), } From ec5c702bb54d3332295e7d03344b9005f5e203a8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 10:46:10 +1100 Subject: [PATCH 1342/1579] Bluetooth: hci_bcm4377: handle function resets through the PCI core Probe resets the function with pci_reset_function_locked() while the driver is bound. The driver has no reset_prepare or reset_done handler, so since commit a3151e6daaec ("PCI: Warn if a running device is unaware of reset") every boot logs hci_bcm4377 0000:01:00.1: resetting hci_bcm4377 0000:01:00.1: reset done on the M1 Pro, the M2 and the M4 Pro. The warning has a point: a reset from the PCI core or from sysfs while the controller runs loses its firmware and rings, and nothing in the driver notices until commands time out. Add the handlers. Probe resets before it sets the driver data, and bcm4377_pci_reset() already quiesces and restarts the controller around its own reset, so both are left alone. Any other reset keeps the interrupt off while it runs and then rebinds the driver, the recovery the driver already uses after a command timeout; that path moves into bcm4377_queue_rebind() so both callers share it. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/bluetooth/hci_bcm4377.c | 62 +++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 9d3bc53d54607..dc24c3e5057a0 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -601,6 +601,8 @@ struct bcm4377_data { const char *board_type; bool needs_reset; + /* Set while bcm4377_pci_reset() resets the function itself. */ + bool own_reset; bool cal_needed; bool cal_loaded; bool setup_retry_used; @@ -1422,7 +1424,9 @@ static int bcm4377_pci_reset(struct bcm4377_data *bcm4377) if (bcm4377->hw->disable_aspm) bcm4377_disable_aspm(bcm4377); + WRITE_ONCE(bcm4377->own_reset, true); ret = pci_reset_function(bcm4377->pdev); + WRITE_ONCE(bcm4377->own_reset, false); if (ret) { dev_err(&bcm4377->pdev->dev, "function level reset failed: %d\n", ret); @@ -2783,14 +2787,12 @@ static void bcm4377_reset_work(struct work_struct *work) * every later command times out until the module is reloaded. Recover the * same way, by rebinding the driver. */ -static void bcm4377_hci_reset(struct hci_dev *hdev) +static void bcm4377_queue_rebind(struct bcm4377_data *bcm4377, + const char *why) { - struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); + struct hci_dev *hdev = bcm4377->hdev; struct bcm4377_reset_work *rw; - /* A timeout while still setting up would only loop probe failures. */ - if (hci_dev_test_flag(hdev, HCI_SETUP)) - return; if (test_and_set_bit(0, &bcm4377->reset_pending)) return; /* @@ -2808,12 +2810,59 @@ static void bcm4377_hci_reset(struct hci_dev *hdev) return; } - bt_dev_err(hdev, "controller not responding, rebinding the driver"); + bt_dev_err(hdev, "%s, rebinding the driver", why); rw->pdev = pci_dev_get(bcm4377->pdev); INIT_WORK(&rw->work, bcm4377_reset_work); queue_work(bcm4377_reset_wq, &rw->work); } +static void bcm4377_hci_reset(struct hci_dev *hdev) +{ + struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev); + + /* A timeout while still setting up would only loop probe failures. */ + if (hci_dev_test_flag(hdev, HCI_SETUP)) + return; + bcm4377_queue_rebind(bcm4377, "controller not responding"); +} + +/* + * A function reset requested by the PCI core or by userspace. Probe resets the + * function before it sets the driver data, and bcm4377_pci_reset() quiesces + * and restarts the controller around its own reset, so neither needs more. + * Any other reset loses the firmware and the rings under a running driver: + * keep the interrupt off across it, then rebind, as for a command timeout. + */ +static void bcm4377_reset_prepare(struct pci_dev *pdev) +{ + struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev); + int irq; + + if (!bcm4377 || READ_ONCE(bcm4377->own_reset)) + return; + irq = pci_irq_vector(pdev, 0); + if (irq > 0) + disable_irq(irq); +} + +static void bcm4377_reset_done(struct pci_dev *pdev) +{ + struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev); + int irq; + + if (!bcm4377 || READ_ONCE(bcm4377->own_reset)) + return; + irq = pci_irq_vector(pdev, 0); + if (irq > 0) + enable_irq(irq); + bcm4377_queue_rebind(bcm4377, "function reset"); +} + +static const struct pci_error_handlers bcm4377_err_handler = { + .reset_prepare = bcm4377_reset_prepare, + .reset_done = bcm4377_reset_done, +}; + static void bcm4377_hci_unregister_dev(void *data) { struct hci_dev *hdev = data; @@ -3250,6 +3299,7 @@ static struct pci_driver bcm4377_pci_driver = { .name = "hci_bcm4377", .id_table = bcm4377_devid_table, .probe = bcm4377_probe, + .err_handler = &bcm4377_err_handler, .driver.pm = &bcm4377_ops, }; From e40dcef15b708111b3eed0b91c141a56beb1ec9a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 11:43:44 +1100 Subject: [PATCH 1343/1579] dt-bindings: input: apple,dockchannel-hid: add the actuator click strength While the trackpad actuator is device-controlled, the coprocessor firmware plays its own click on every press and release. It starts with a weak placeholder strength for both, and nothing in the firmware raises it, so the machine has to say what the click should be. Describe apple,click-strength on the actuator node: the press and the release strength, each from 1 to 255. Without it the firmware keeps its own strength. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- .../bindings/input/apple,dockchannel-hid.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml b/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml index e039ebf276123..19e22d47a2db4 100644 --- a/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml +++ b/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml @@ -188,6 +188,23 @@ properties: additionalProperties: false description: The trackpad haptic actuator. + properties: + apple,click-strength: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + Strengths at which the actuator firmware plays its own click + waveforms while it is device-controlled, first on press and then on + release. The coprocessor firmware starts with a weak placeholder + strength, so the host writes these over it. When absent, the + firmware's strength is kept. + items: + - description: Strength of the press click. + minimum: 1 + maximum: 255 + - description: Strength of the release click. + minimum: 1 + maximum: 255 + tp_accel: type: object additionalProperties: false @@ -275,6 +292,7 @@ examples: }; actuator { + apple,click-strength = <0xc0 0x6e>; }; tp_accel { From 91dcaa520a513df207cc110b86602e761df9da17 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 11:43:48 +1100 Subject: [PATCH 1344/1579] HID: apple-mtp-haptic: set the click strength from the device tree The MTP actuator firmware plays its own press and release clicks from feature reports 0x22 and 0x23. The third byte of each is the strength. The firmware boots with 0x23 for the press and 0x14 for the release on both a J413 and a J616s, and nothing on the Linux side ever changes it. The click on an M2 MacBook Air is noticeably weaker than on an M1 Pro, whose SPI trackpad clicks without any help from the host. When the actuator node has apple,click-strength, read each waveform back, replace only its strength and write it again, keeping the shape of the waveform as the firmware has it. This runs at probe, after the actuator is returned to device-controlled mode. The coprocessor keeps running across suspend-to-idle, so the setting does not need to be restored on resume. Machines without the property are left as they are. The force-click waveforms (0x24 and 0x25) are left alone. The J413 firmware refuses writes to them. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/hid/hid-apple-mtp-haptic.c | 86 ++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/drivers/hid/hid-apple-mtp-haptic.c b/drivers/hid/hid-apple-mtp-haptic.c index 42ab019f7a27a..dd35d72ffe023 100644 --- a/drivers/hid/hid-apple-mtp-haptic.c +++ b/drivers/hid/hid-apple-mtp-haptic.c @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#include #include +#include +#include #include #include "hid-ids.h" @@ -56,6 +59,87 @@ int apple_taptic_switch_modes(struct hid_device *haptic_hdev, bool enable_host_c } EXPORT_SYMBOL_GPL(apple_taptic_switch_modes); +/* + * While the actuator is device-controlled, its firmware plays a click + * waveform of its own on every press and release: feature report 0x22 on + * press and 0x23 on release. Both begin with the report ID, a version byte + * and the strength the waveform is played at. The MTP firmware boots with a + * weak strength there and nothing else sets one, so the click is much + * fainter than the same pad gives under macOS. A machine whose device tree + * gives the actuator a click strength has it written over the firmware's, + * leaving the shape of each waveform as the firmware has it. + */ +#define APPLE_TAPTIC_PRESS_WAVEFORM 0x22 +#define APPLE_TAPTIC_RELEASE_WAVEFORM 0x23 +#define APPLE_TAPTIC_WAVEFORM_STRENGTH 2 +#define APPLE_TAPTIC_WAVEFORM_MAX 64 + +static int apple_taptic_set_strength(struct hid_device *haptic_hdev, u8 report, + u8 strength) +{ + u8 *buf __free(kfree) = kzalloc(APPLE_TAPTIC_WAVEFORM_MAX, GFP_KERNEL); + int len; + + if (!buf) + return -ENOMEM; + + len = hid_hw_raw_request(haptic_hdev, report, buf, + APPLE_TAPTIC_WAVEFORM_MAX, HID_FEATURE_REPORT, + HID_REQ_GET_REPORT); + if (len < 0) + return len; + if (len <= APPLE_TAPTIC_WAVEFORM_STRENGTH || buf[0] != report) + return -EPROTO; + if (buf[APPLE_TAPTIC_WAVEFORM_STRENGTH] == strength) + return 0; + + buf[APPLE_TAPTIC_WAVEFORM_STRENGTH] = strength; + len = hid_hw_raw_request(haptic_hdev, report, buf, len, + HID_FEATURE_REPORT, HID_REQ_SET_REPORT); + + return len < 0 ? len : 0; +} + +static void apple_taptic_set_click_strength(struct hid_device *haptic_hdev) +{ + static const u8 waveforms[] = { + APPLE_TAPTIC_PRESS_WAVEFORM, + APPLE_TAPTIC_RELEASE_WAVEFORM, + }; + struct device_node *np __free(device_node) = NULL; + u32 strength[ARRAY_SIZE(waveforms)]; + unsigned int i; + int ret; + + if (haptic_hdev->dev.parent) + np = of_get_child_by_name(haptic_hdev->dev.parent->of_node, + "actuator"); + if (!np) + return; + + ret = of_property_read_u32_array(np, "apple,click-strength", strength, + ARRAY_SIZE(strength)); + if (ret == -EINVAL) + return; + + for (i = 0; !ret && i < ARRAY_SIZE(strength); i++) + if (!strength[i] || strength[i] > U8_MAX) + ret = -ERANGE; + if (ret) { + hid_warn(haptic_hdev, "invalid apple,click-strength (%d)\n", ret); + return; + } + + for (i = 0; i < ARRAY_SIZE(waveforms); i++) { + ret = apple_taptic_set_strength(haptic_hdev, waveforms[i], + strength[i]); + if (ret) + hid_warn(haptic_hdev, + "cannot set the strength of click waveform 0x%02x (%d)\n", + waveforms[i], ret); + } +} + static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid_device_id *id) { int ret; @@ -87,6 +171,8 @@ static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid "cannot return the actuator to device-controlled mode (%d)\n", ret); + apple_taptic_set_click_strength(haptic_hdev); + return 0; } From 99768d194bf90d13205e6e7de0eb11c7382ae524 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 11:43:51 +1100 Subject: [PATCH 1345/1579] arm64: dts: apple: t8112-j413: set the trackpad click strength The J413 actuator firmware clicks at strength 0x23 on press and 0x14 on release, which feels weak next to an M1 Pro trackpad. Raise both by the same factor, to 0xc0 and 0x6e, so the release keeps its proportion to the press. Tested on a J413 running 7.3. With the property on the live actuator node, a rebind of the driver writes both values. A boot with the same values written at probe shows them in place before the touch controller starts and still there once it is running. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- arch/arm64/boot/dts/apple/t8112-j413.dts | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index eb03ebf173d76..7939c20917155 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -251,6 +251,7 @@ }; actuator { + apple,click-strength = <0xc0 0x6e>; }; tp_accel { From f0fb0a51b1e5c49cb9205046d693d56142ac308d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 12:34:26 +1100 Subject: [PATCH 1346/1579] ASoC: apple: macaudio: no PCM for the J616s route-only front-ends The J616s speaker card has no headphone codec (the jack is a card of its own), and the jack card has no second output. Their CPU DAIs j616-pcm-0 and j616-jack-pcm-1 exist only so the fixed Speaker and Headphone mux routes resolve; the AOP component has no stream for them. The cards still create PCM devices for them, hw:AppleJ616,0 and hw:AppleJack,1. When the PipeWire pro-audio profile probes every PCM of the speaker card at session start, opening hw:AppleJ616,0 finds no back-end and logs at boot Primary: ASoC: no backend DAIs enabled for Primary, possibly missing ALSA mixer-based routing or UCM profile even though UCM routes the speakers through Secondary as it should. Mark those front-ends in the machine data and leave their links unbound. The DAI widgets still exist, since a component creates them for all its DAIs, so the mux routes and controls do not change and UCM sets "Speaker Playback Mux" to Secondary as before. The AOP component uses the link number as the PCM device number, so the speakers stay hw:AppleJ616,1 and the speaker sense hw:AppleJ616,2, which UCM, asahi-audio and the speaker protection daemons open by number. Back-end internal PCMs move above the front-ends. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/aop_spkr.rs | 7 +++++++ sound/soc/apple/macaudio.c | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/sound/soc/apple/aop_spkr.rs b/sound/soc/apple/aop_spkr.rs index b5b2f5a7f6acf..64efaa83ce8d6 100644 --- a/sound/soc/apple/aop_spkr.rs +++ b/sound/soc/apple/aop_spkr.rs @@ -1542,9 +1542,16 @@ fn to_kvec( Ok(v) } +/// The first PCM device number of the back-ends' internal PCMs, above every front-end's. +const BE_PCM_BASE: i32 = 16; + fn component_driver(name: &'static CStr) -> bindings::snd_soc_component_driver { bindings::snd_soc_component_driver { name: name.as_char_ptr(), + // A front-end's PCM device is its link number, so a front-end that macaudio leaves + // unbound (one that only completes the fixed routes) does not renumber the others. + use_dai_pcm_id: true, + be_pcm_base: BE_PCM_BASE, open: Some(pcm_open), close: Some(pcm_close), hw_params: Some(pcm_hw_params), diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 1d1c8924eb2fb..b366282f3386d 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -120,6 +120,14 @@ struct macaudio_platform_cfg { * for its own card). */ bool fe_cpu_is_platform; + /* + * Front-ends (bits by link index) whose CPU DAI only completes the + * fixed DAPM routes and can never carry a stream on this machine. + * Their links are not bound, so the card has no PCM device for them + * that userspace could probe and fail to open. The CPU component + * keeps the other front-ends on their link numbers. + */ + unsigned int route_only_fes; /* * The highest headphone volume, a "Jack DAC Playback Volume" value * (CS42L84_DAC_DB()), 0 for the codec's own maximum. @@ -859,6 +867,8 @@ static int macaudio_parse_of(struct macaudio_snd_data *ma) memcpy(link_props, &macaudio_fe_link_props[i], sizeof(struct macaudio_link_props)); if (ma->cfg->fe_dai_names[i]) link->cpus[0].dai_name = ma->cfg->fe_dai_names[i]; + if (ma->cfg->route_only_fes & BIT(i)) + link->ignore = 1; if (ma->cfg->dai_fmt && !link_props->is_sense) link->dai_fmt = ma->cfg->dai_fmt; if (i == 0 && ma->cfg->primary_bclk_ratio) @@ -2117,6 +2127,8 @@ struct macaudio_platform_cfg macaudio_j616_cfg = { SND_SOC_DAIFMT_GATED | SND_SOC_DAIFMT_NB_IF, .secondary_bclk_ratio = 250, .fe_cpu_is_platform = true, + /* No headphone codec on this card: the jack is a card of its own. */ + .route_only_fes = BIT(0), }; /* @@ -2133,6 +2145,7 @@ struct macaudio_platform_cfg macaudio_j616_jack_cfg = { .fe_dai_names = { "j616-jack-pcm-0", "j616-jack-pcm-1" }, .dai_fmt = SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_CBC_CFC | SND_SOC_DAIFMT_IB_NF, .fe_cpu_is_platform = true, + .route_only_fes = BIT(1), /* * Above 0 dB the DAC only adds digital gain, which clips a full-scale * signal. The 6 dB below that are margin: the analog full scale of From 5449640813e71164280a29c9447a7233833c9950 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 12:47:14 +1100 Subject: [PATCH 1347/1579] ci: build the SEP module with the layout-corrected bindgen aurora-wip keeps bindgen's layout assertions in every Rust build, as Documentation/rust/bindgen-layout-checks.rst requires. Stock bindgen 0.71.1 loses the alignment of the empty, cacheline-aligned sched_statistics and of efi_signature_data_t, so the bindings crate stops with E0080 before the SEP module is reached. The rust-build job has failed this way on every branch that carries that rule since 10-28. Prepare the generator from tools/aurora-bindgen/ instead of installing stock bindgen-cli. The top-level Makefile picks up its build/bin/bindgen by itself, so the compile steps stay as they are and the assertions stay on. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/sep-source.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sep-source.yml b/.github/workflows/sep-source.yml index 052f282d60061..985d0791ed713 100644 --- a/.github/workflows/sep-source.yml +++ b/.github/workflows/sep-source.yml @@ -53,7 +53,10 @@ jobs: sudo apt-get install -y clang lld llvm libclang-dev libelf-dev libssl-dev \ flex bison bc pkg-config make perl rustup component add rust-src - cargo install bindgen-cli --version 0.71.1 --locked + - name: Prepare the layout-corrected bindgen + env: + LIBCLANG_PATH: /usr/lib/llvm-18/lib + run: python3 tools/aurora-bindgen/build.py - name: Compile Apple SEP module for arm64 env: LIBCLANG_PATH: /usr/lib/llvm-18/lib From a6d008b8fc2b9b6bcafa0e65bdb91d0ba4dadd03 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 02:23:21 +0000 Subject: [PATCH 1348/1579] arm64: configs: asahi: Cap the page block order at 2 MiB With 16K pages the page block order follows the PMD order, so a page block is 32 MiB. khugepaged sizes min_free_kbytes from the page block size, eleven blocks per zone, which comes to 360448 kB (352 MiB) on every 8 GB Mac with this config. The low and high watermarks follow at 440 MiB and 528 MiB, and the per-CPU page list limits are derived from the low watermark, so another 250 to 440 MiB of free pages sits on per-CPU lists where MemFree and MemAvailable do not count it. On a MacBook Neo (8 GB) idling at the GNOME desktop this explains about 1.4 GiB of the 2.9 GiB that free and the desktop monitors report as used: 1.2 GiB of watermark deductions in MemAvailable plus 265 MiB on the per-CPU lists. kswapd also keeps about half a gigabyte free that the page cache could use. The M2 MacBook Air shows the same 360448 kB. Cap the page block order at 7, a 2 MiB block, the same size 4K arm64 and x86 kernels use. min_free_kbytes on the Neo drops to about 22 MiB and the high watermark to about 37 MiB. 32 MiB transparent huge pages are still allocated; compaction needs more free blocks to build one, as the Kconfig help notes. THP is madvise only in our configs. CMA areas need 2 MiB alignment instead of 32 MiB. Booted in QEMU (virt, 6 CPUs, 8 GB, 16K pages, THP madvise) with only this option changed: order 11: min_free_kbytes 411824, MemAvailable 7476144 kB order 7: min_free_kbytes 45056, MemAvailable 8026816 kB QEMU has two zones, so its values are twice a single-zone Mac's. A 128 MiB MADV_HUGEPAGE region got 4 of 4 32 MiB THPs with no fallback on both kernels. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- arch/arm64/configs/asahi.config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index a9e8f94368aa8..9580f88a59c34 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -7,6 +7,9 @@ CONFIG_ARM64_16K_PAGES=y CONFIG_ARM64_VA_BITS_48=y CONFIG_ARM64_USER4K_EXPERIMENTAL=y CONFIG_ARM64_MEMORY_MODEL_CONTROL=y +# 16K pages make a page block 32 MiB, and khugepaged then sets min_free_kbytes +# to 352 MiB on an 8 GB Mac. Keep page blocks at 2 MiB as on 4K kernels. +CONFIG_PAGE_BLOCK_MAX_ORDER=7 CONFIG_ARM_APPLE_CPUIDLE=y CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y From 8cd0419030a7b5d28128fc7d298a5efe0231e89d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 14:13:57 +1100 Subject: [PATCH 1349/1579] arm64: configs: asahi: restore the explicit hardware requests b7fa6db51313 dropped 36 requests that defconfig and the Fedora base resolve anyway. The file states every hardware request explicitly on purpose, so the survival check verifies each one after olddefconfig. Go back to the file as integration/aurora carries it. The one difference left is CONFIG_APPLE_SIO_T6030. This branch no longer has that option, since the SIO firmware behaviour comes from compatible data (e3653e6e1a56), and the same line goes in integration/aurora with that change. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- arch/arm64/configs/asahi.config | 39 ++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index fcf0d27611aae..4422527c3ed65 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -3,11 +3,16 @@ # olddefconfig and verify that the requested settings survive resolution. # Device-specific boot, debug and distribution policy stays in the base. CONFIG_RUST=y +CONFIG_ARM64_ACTLR_STATE=y +CONFIG_ARCH_APPLE=y +# CONFIG_ARM64_4K_PAGES is not set CONFIG_ARM64_16K_PAGES=y +# CONFIG_ARM64_64K_PAGES is not set CONFIG_ARM64_VA_BITS_48=y CONFIG_ARM64_USER4K_EXPERIMENTAL=y CONFIG_ARM64_MEMORY_MODEL_CONTROL=y CONFIG_ARM_APPLE_CPUIDLE=y +CONFIG_ARM_APPLE_SOC_CPUFREQ=m CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y # CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE is not set CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y @@ -15,17 +20,27 @@ CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m CONFIG_BT_MTK7932_PCIE=m CONFIG_PCIE_APPLE=y +CONFIG_NVME_APPLE=m CONFIG_MMC_SDHCI_PCI=m +CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y CONFIG_PCIE_APPLE_PIODMA_DIAG=y CONFIG_PCIEASPM=y CONFIG_NET_VENDOR_INTEL=y +CONFIG_IGB=m CONFIG_IGC=m CONFIG_MT7932_FULLMAC=m +CONFIG_TOUCHSCREEN_APPLE_Z2=m CONFIG_INPUT_MACSMC_INPUT=m +CONFIG_I2C_APPLE=m +CONFIG_SPI_APPLE=m CONFIG_SPI_APPLE_QSPI=m CONFIG_SPMI_APPLE=y +CONFIG_PINCTRL_APPLE_GPIO=m +CONFIG_GPIO_MACSMC=m +CONFIG_POWER_RESET_MACSMC=m CONFIG_MACSMC_POWER=m +CONFIG_SENSORS_MACSMC_HWMON=m CONFIG_SENSORS_APPLE_PMP_J613=m CONFIG_THERMAL_NETLINK=y CONFIG_NVME_HWMON=y @@ -37,7 +52,6 @@ CONFIG_VIDEO_APPLE_AVD=m CONFIG_VIDEO_APPLE_ISP=m CONFIG_DRM=y CONFIG_DRM_ASAHI=m -CONFIG_DRM_ASAHI_G15=y CONFIG_DRM_ASAHI_G16=y CONFIG_DRM_ADP=m CONFIG_DRM_APPLE=m @@ -48,53 +62,76 @@ CONFIG_DRM_ACCEL_ANE=m CONFIG_DRM_SIMPLEDRM=y CONFIG_SYSFB_SIMPLEFB=y CONFIG_SND_SOC_APPLE_AOP_AUDIO=m +CONFIG_SND_SOC_APPLE_MCA=m CONFIG_SND_SOC_APPLE_MACAUDIO=m CONFIG_SND_SOC_CS42L83=m CONFIG_SND_SOC_CS42L84=m +CONFIG_SND_SOC_MAX98357A=m CONFIG_SND_SOC_TAS2764=m CONFIG_SND_SOC_TAS2770=m CONFIG_HID_APPLE=m CONFIG_HID_MAGICMOUSE=m CONFIG_HID_APPLE_MTP_HAPTIC=m +CONFIG_SERIAL_SAMSUNG=y +CONFIG_SERIAL_SAMSUNG_CONSOLE=y CONFIG_HID_DOCKCHANNEL=m CONFIG_SPI_HID_APPLE_OF=m +CONFIG_SPI_HID_APPLE_CORE=m CONFIG_USB_DWC3_APPLE=m +CONFIG_TYPEC=m CONFIG_USB4=m CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_TYPEC_SN201202X=m CONFIG_RTC_DRV_MACSMC=y CONFIG_RTC_DRV_APPLE_PMU=m +CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m CONFIG_MFD_MACSMC=y +CONFIG_COMMON_CLK_APPLE_NCO=m CONFIG_APPLE_DART=y CONFIG_APPLE_DART_J713_DCP=y CONFIG_APPLE_DOCKCHANNEL=m CONFIG_SERIAL_APPLE_DOCKCHANNEL=y CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y +CONFIG_APPLE_MAILBOX=y CONFIG_APPLE_PMGR_MISC=y +CONFIG_APPLE_RTKIT=y CONFIG_APPLE_RTKIT_HELPER=m +CONFIG_APPLE_SART=m +CONFIG_RUST_APPLE_RTKIT=y +CONFIG_APPLE_AOP=m +CONFIG_HW_RANDOM=y +CONFIG_KEYS=y CONFIG_TRUSTED_KEYS=y # CONFIG_TRUSTED_KEYS_TPM is not set # CONFIG_TRUSTED_KEYS_TEE is not set CONFIG_APPLE_SEP=m CONFIG_APPLE_PMGR_PWRSTATE=y CONFIG_APPLE_PMP=m +CONFIG_APPLE_PMP_REPORT=y CONFIG_APPLE_PMP_THERMAL=m CONFIG_APPLE_T8132_PLATFORM_HANDOFF=y CONFIG_IIO_AOP_SENSOR_LAS=m CONFIG_IIO_AOP_SENSOR_ALS=m +CONFIG_RUST_FW_LOADER_ABSTRACTIONS=y +CONFIG_PWM_APPLE=m +CONFIG_APPLE_AIC=y CONFIG_PHY_APPLE_ATC=m CONFIG_PHY_APPLE_T6040_USB2=m CONFIG_PHY_APPLE_DPTX=m +CONFIG_APPLE_M1_CPU_PMU=y +CONFIG_NVMEM_APPLE_EFUSES=m CONFIG_NVMEM_APPLE_SPMI=y CONFIG_MUX_APPLE_DPXBAR=m +CONFIG_TYPEC_TPS6598X=m CONFIG_RESET_APPLE_CIO=y CONFIG_INFINIBAND=m CONFIG_INFINIBAND_USER_ACCESS=m CONFIG_INFINIBAND_USER_MAD=m CONFIG_RDMA_RXE=m CONFIG_RDMA_SIW=m +CONFIG_DMABUF_HEAPS=y CONFIG_DMABUF_HEAPS_SYSTEM=y CONFIG_UDMABUF=y CONFIG_DETECT_HUNG_TASK=y From afbf8f192a82f0dea15ed896d1040795c97755a7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 14:13:58 +1100 Subject: [PATCH 1350/1579] drm/asahi: drop the unused crash dumper source crashdump.rs is not referenced by the driver or the Makefile, so it is never compiled. integration/aurora does not carry it. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/crashdump.rs | 307 ----------------------------- 1 file changed, 307 deletions(-) delete mode 100644 drivers/gpu/drm/asahi/crashdump.rs diff --git a/drivers/gpu/drm/asahi/crashdump.rs b/drivers/gpu/drm/asahi/crashdump.rs deleted file mode 100644 index 148cae7aa9b46..0000000000000 --- a/drivers/gpu/drm/asahi/crashdump.rs +++ /dev/null @@ -1,307 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU crash dump formatter -//! -//! Takes a raw dump of firmware/kernel mapped pages from `pgtable` and formats it into -//! an ELF core dump suitable for dumping into userspace. - -use core::mem::size_of; - -use kernel::{ - devcoredump::DevCoreDump, - error::Result, - page::{Page, PAGE_MASK, PAGE_SHIFT, PAGE_SIZE}, - prelude::*, - types::Owned, - uapi, -}; - -use crate::hw; -use crate::pgtable::{self, DumpedPage, Prot, UAT_PGSZ}; -use crate::util::align; - -pub(crate) struct CrashDump { - headers: KVVec, - pages: KVVec>, -} - -const NOTE_NAME_AGX: &str = "AGX"; -const NOTE_AGX_DUMP_INFO: u32 = 1; - -const NOTE_NAME_RTKIT: &str = "RTKIT"; -const NOTE_RTKIT_CRASHLOG: u32 = 1; - -#[repr(C)] -pub(crate) struct AGXDumpInfo { - initdata_address: u64, - chip_id: u32, - gpu_gen: hw::GpuGen, - gpu_variant: hw::GpuVariant, - gpu_rev: hw::GpuRevision, - total_active_cores: u32, - firmware_version: [u32; 6], - reserved: u32, -} - -struct ELFNote { - name: &'static str, - ty: u32, - data: KVVec, -} - -pub(crate) struct CrashDumpBuilder { - page_dump: KVVec, - notes: KVec, -} - -/// Helper to convert ELF headers into byte slices -/// TODO: Hook this up into kernel::AsBytes somehow -/// -/// # Safety -/// -/// Types implementing this trait must have no padding bytes. -unsafe trait AsBytes: Sized { - fn as_bytes(&self) -> &[u8] { - // SAFETY: This trait is only implemented for types with no padding bytes - unsafe { core::slice::from_raw_parts(self as *const _ as *const u8, size_of::()) } - } - fn slice_as_bytes(slice: &[Self]) -> &[u8] { - // SAFETY: This trait is only implemented for types with no padding bytes - unsafe { - core::slice::from_raw_parts(slice.as_ptr() as *const u8, core::mem::size_of_val(slice)) - } - } -} - -// SAFETY: This type has no padding -unsafe impl AsBytes for uapi::Elf64_Ehdr {} -// SAFETY: This type has no padding -unsafe impl AsBytes for uapi::Elf64_Phdr {} -// SAFETY: This type has no padding -unsafe impl AsBytes for uapi::Elf64_Nhdr {} -// SAFETY: This type has no padding -unsafe impl AsBytes for AGXDumpInfo {} - -const FIRMWARE_ENTRYPOINT: u64 = 0xFFFFFF8000000000u64; - -impl CrashDumpBuilder { - pub(crate) fn new(page_dump: KVVec) -> Result { - Ok(CrashDumpBuilder { - page_dump, - notes: KVec::new(), - }) - } - - pub(crate) fn add_agx_info( - &mut self, - cfg: &hw::HwConfig, - dyncfg: &hw::DynConfig, - initdata_address: u64, - ) -> Result { - let mut info = AGXDumpInfo { - chip_id: cfg.chip_id, - gpu_gen: dyncfg.id.gpu_gen, - gpu_variant: dyncfg.id.gpu_variant, - gpu_rev: dyncfg.id.gpu_rev, - total_active_cores: dyncfg.id.total_active_cores, - firmware_version: [0; 6], - reserved: 0, - initdata_address, - }; - info.firmware_version[..dyncfg.firmware_version.len().min(6)] - .copy_from_slice(&dyncfg.firmware_version); - - let mut data = KVVec::new(); - data.extend_from_slice(info.as_bytes(), GFP_KERNEL)?; - - self.notes.push( - ELFNote { - name: NOTE_NAME_AGX, - ty: NOTE_AGX_DUMP_INFO, - data, - }, - GFP_KERNEL, - )?; - Ok(()) - } - - pub(crate) fn add_crashlog(&mut self, crashlog: &[u8]) -> Result { - let mut data = KVVec::new(); - data.extend_from_slice(crashlog, GFP_KERNEL)?; - - self.notes.push( - ELFNote { - name: NOTE_NAME_RTKIT, - ty: NOTE_RTKIT_CRASHLOG, - data, - }, - GFP_KERNEL, - )?; - - Ok(()) - } - - pub(crate) fn finalize(self) -> Result { - let CrashDumpBuilder { page_dump, notes } = self; - - let mut ehdr: uapi::Elf64_Ehdr = Default::default(); - - ehdr.e_ident[uapi::EI_MAG0 as usize..=uapi::EI_MAG3 as usize].copy_from_slice(b"\x7fELF"); - ehdr.e_ident[uapi::EI_CLASS as usize] = uapi::ELFCLASS64 as u8; - ehdr.e_ident[uapi::EI_DATA as usize] = uapi::ELFDATA2LSB as u8; - ehdr.e_ident[uapi::EI_VERSION as usize] = uapi::EV_CURRENT as u8; - ehdr.e_type = uapi::ET_CORE as u16; - ehdr.e_machine = uapi::EM_AARCH64 as u16; - ehdr.e_version = uapi::EV_CURRENT; - ehdr.e_entry = FIRMWARE_ENTRYPOINT; - ehdr.e_ehsize = core::mem::size_of::() as u16; - ehdr.e_phentsize = core::mem::size_of::() as u16; - - let phdr_offset = core::mem::size_of::(); - - // PHDRs come after the ELF header - ehdr.e_phoff = phdr_offset as u64; - - let mut phdrs = KVVec::new(); - - // First PHDR is the NOTE section - phdrs.push( - uapi::Elf64_Phdr { - p_type: uapi::PT_NOTE, - p_flags: uapi::PF_R, - p_align: 1, - ..Default::default() - }, - GFP_KERNEL, - )?; - - // Generate the page phdrs. The offset will be fixed up later. - let mut off: usize = 0; - let mut next = None; - let mut pages: KVVec> = KVVec::new(); - - for mut page in page_dump { - let vaddr = page.iova; - let paddr = page.pte & pgtable::PTE_ADDR_BITS; - let flags = Prot::from_pte(page.pte).elf_flags(); - let valid = page.data.is_some(); - let cur = (vaddr, paddr, flags, valid); - if Some(cur) != next { - phdrs.push( - uapi::Elf64_Phdr { - p_type: uapi::PT_LOAD, - p_offset: if valid { off as u64 } else { 0 }, - p_vaddr: vaddr, - p_paddr: paddr, - p_filesz: if valid { UAT_PGSZ as u64 } else { 0 }, - p_memsz: UAT_PGSZ as u64, - p_flags: flags, - p_align: UAT_PGSZ as u64, - }, - GFP_KERNEL, - )?; - if valid { - off += UAT_PGSZ; - } - } else { - let ph = phdrs.last_mut().unwrap(); - ph.p_memsz += UAT_PGSZ as u64; - if valid { - ph.p_filesz += UAT_PGSZ as u64; - off += UAT_PGSZ; - } - } - if let Some(data_page) = page.data.take() { - pages.push(data_page, GFP_KERNEL)?; - } - next = Some(( - vaddr + UAT_PGSZ as u64, - paddr + UAT_PGSZ as u64, - flags, - valid, - )); - } - - ehdr.e_phnum = phdrs.len() as u16; - - let note_offset = phdr_offset + size_of::() * phdrs.len(); - - let mut note_data: KVVec = KVVec::new(); - - for note in notes { - let hdr = uapi::Elf64_Nhdr { - n_namesz: note.name.len() as u32 + 1, - n_descsz: note.data.len() as u32, - n_type: note.ty, - }; - note_data.extend_from_slice(hdr.as_bytes(), GFP_KERNEL)?; - note_data.extend_from_slice(note.name.as_bytes(), GFP_KERNEL)?; - note_data.push(0, GFP_KERNEL)?; - while note_data.len() & 3 != 0 { - note_data.push(0, GFP_KERNEL)?; - } - note_data.extend_from_slice(¬e.data, GFP_KERNEL)?; - while note_data.len() & 3 != 0 { - note_data.push(0, GFP_KERNEL)?; - } - } - - // NOTE section comes after the PHDRs - phdrs[0].p_offset = note_offset as u64; - phdrs[0].p_filesz = note_data.len() as u64; - - // Align data section to the page size - let data_offset = align(note_offset + note_data.len(), UAT_PGSZ); - - // Fix up data PHDR offsets - for phdr in &mut phdrs[1..] { - phdr.p_offset += data_offset as u64; - } - - // Build ELF header buffer - let mut headers: KVVec = KVVec::from_elem(0, data_offset, GFP_KERNEL)?; - - headers[0..size_of::()].copy_from_slice(ehdr.as_bytes()); - headers[phdr_offset..phdr_offset + phdrs.len() * size_of::()] - .copy_from_slice(AsBytes::slice_as_bytes(&phdrs)); - headers[note_offset..note_offset + note_data.len()].copy_from_slice(¬e_data); - - Ok(CrashDump { headers, pages }) - } -} - -impl DevCoreDump for CrashDump { - fn read(&self, buf: &mut [u8], mut offset: usize) -> Result { - let mut read = 0; - let mut left = buf.len(); - if offset < self.headers.len() { - let block = left.min(self.headers.len() - offset); - buf[..block].copy_from_slice(&self.headers[offset..offset + block]); - read += block; - offset += block; - left -= block; - } - if left == 0 { - return Ok(read); - } - offset -= self.headers.len(); // Offset from the page area - - while left > 0 { - let page_index = offset >> PAGE_SHIFT; - let page_offset = offset & !PAGE_MASK; - let block = left.min(PAGE_SIZE - page_offset); - let Some(page) = self.pages.get(page_index) else { - break; - }; - let slice = &mut buf[read..read + block]; - // SAFETY: We own the page, and the slice guarantees the - // dst length is sufficient. - unsafe { page.read_raw(slice.as_mut_ptr(), page_offset, slice.len())? }; - read += block; - offset += block; - left -= block; - } - - Ok(read) - } -} From f9916109ec8003a60cb8cf93161baf7aeddef23d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 14:13:59 +1100 Subject: [PATCH 1351/1579] arm64: dts: apple: drop the unused T6040 DP audio and DCP binding drafts No DTS includes t6040-dpaudio.dtsi, and integration/aurora carries the live J616s device tree. Return the dpaudio, DCP and T6040 ATC binding text to the integration/aurora version. The DP audio description can come back with the J616s HDMI audio work once the T6041 SIO backend exists. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- .../bindings/display/apple,dcp.yaml | 20 +-- .../bindings/phy/apple,t6040-atcphy.yaml | 5 +- .../bindings/sound/apple,t6030-dpaudio.yaml | 43 ++---- arch/arm64/boot/dts/apple/t6040-dpaudio.dtsi | 132 ------------------ 4 files changed, 15 insertions(+), 185 deletions(-) delete mode 100644 arch/arm64/boot/dts/apple/t6040-dpaudio.dtsi diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index c67a71b7a169b..38be82bc7d1bd 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -35,7 +35,6 @@ properties: - apple,t6020-dcpext - apple,t6041-dcpext - const: apple,dcpext - - const: apple,t6041-dcpext - items: - const: apple,t8140-dcpext - const: apple,dcp @@ -295,8 +294,6 @@ properties: '#size-cells': const: 2 - ranges: true - piodma: type: object properties: @@ -393,19 +390,10 @@ allOf: pattern: '^(dp-xbar|typec[0-9]+)$' - if: not: - anyOf: - - properties: - compatible: - contains: - const: apple,t8132-dcp - - properties: - compatible: - contains: - const: apple,t6041-dcpext - status: - const: disabled - required: - - status + properties: + compatible: + contains: + const: apple,t8132-dcp then: required: - apple,bw-scratch diff --git a/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml index 7f24f79264211..04c8907584946 100644 --- a/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml @@ -106,8 +106,8 @@ properties: port: $ref: /schemas/graph.yaml#/properties/port description: > - Link to the SuperSpeed lanes of the Type-C connector, or to the DCP - DisplayPort output on a fixed DisplayPort PHY. + Link to the SuperSpeed lanes of the Type-C connector. Absent on a fixed + DisplayPort PHY. apple,tunable-axi2af: $ref: "#/$defs/apple,tunable" @@ -174,6 +174,7 @@ allOf: "#reset-cells": false orientation-switch: false mode-switch: false + port: false additionalProperties: false diff --git a/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml b/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml index bb92d6a30c428..9b4e9328d9020 100644 --- a/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml @@ -4,36 +4,29 @@ $id: http://devicetree.org/schemas/sound/apple,t6030-dpaudio.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple T6030 and T6041 DisplayPort audio +title: Apple T6030 DisplayPort audio maintainers: - Ryan Murray description: | - Each DisplayPort audio interface has no application-processor registers. - Samples arrive through SIO, and the graph connects the interface to port 0 - of the display processor that owns its audio service. An interface without - an assigned display processor has no graph connection and remains disabled. - - T6041 requires a generation-specific SIO firmware backend and a compatible - display audio service before these interfaces can be enabled. + The external display processor provides the HDMI port's audio service. + Its DisplayPort audio interface has no application-processor registers. + Samples arrive through SIO, and the graph connects this interface to + port 0 of the display processor that owns the audio service. select: properties: compatible: contains: - enum: - - apple,t6030-dpaudio - - apple,t6041-dpaudio + const: apple,t6030-dpaudio required: - compatible properties: compatible: items: - - enum: - - apple,t6030-dpaudio - - apple,t6041-dpaudio + - const: apple,t6030-dpaudio - const: apple,dpaudio dmas: @@ -59,26 +52,6 @@ required: - dmas - dma-names - power-domains - -allOf: - - if: - properties: - compatible: - contains: - const: apple,t6030-dpaudio - then: - required: - - ports - - - if: - not: - required: - - ports - then: - properties: - status: - const: disabled - required: - - status + - ports additionalProperties: false diff --git a/arch/arm64/boot/dts/apple/t6040-dpaudio.dtsi b/arch/arm64/boot/dts/apple/t6040-dpaudio.dtsi deleted file mode 100644 index 68d346a4e1160..0000000000000 --- a/arch/arm64/boot/dts/apple/t6040-dpaudio.dtsi +++ /dev/null @@ -1,132 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ OR MIT -/* - * DPA interfaces are fed by SIO and have no AP-side MMIO registers. - * Keep them disabled until the T6041 SIO firmware and DCP audio services - * are supported. DPA3/4 have no assigned DCP in this SoC's firmware tree. - */ - -/ { - dpaudio0: audio-controller-0 { - compatible = "apple,t6041-dpaudio", "apple,dpaudio"; - dmas = <&sio 0x64>; - dma-names = "tx"; - power-domains = <&ps_dpa0>; - status = "disabled"; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - - dpaudio0_dcp: endpoint { - remote-endpoint = <&dcp_audio>; - }; - }; - }; - }; - - dpaudio1: audio-controller-1 { - compatible = "apple,t6041-dpaudio", "apple,dpaudio"; - dmas = <&sio 0x66>; - dma-names = "tx"; - power-domains = <&ps_dpa1>; - status = "disabled"; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - - dpaudio1_dcp: endpoint { - remote-endpoint = <&dcpext0_audio>; - }; - }; - }; - }; - - dpaudio2: audio-controller-2 { - compatible = "apple,t6041-dpaudio", "apple,dpaudio"; - dmas = <&sio 0x68>; - dma-names = "tx"; - power-domains = <&ps_dpa2>; - status = "disabled"; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - - dpaudio2_dcp: endpoint { - remote-endpoint = <&dcpext1_audio>; - }; - }; - }; - }; - - dpaudio3: audio-controller-3 { - compatible = "apple,t6041-dpaudio", "apple,dpaudio"; - dmas = <&sio 0x6a>; - dma-names = "tx"; - power-domains = <&ps_dpa3>; - status = "disabled"; - }; - - dpaudio4: audio-controller-4 { - compatible = "apple,t6041-dpaudio", "apple,dpaudio"; - dmas = <&sio 0x6c>; - dma-names = "tx"; - power-domains = <&ps_dpa4>; - status = "disabled"; - }; -}; - -&dcp { - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - - dcp_audio: endpoint { - remote-endpoint = <&dpaudio0_dcp>; - }; - }; - }; -}; - -&dcpext0 { - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - - dcpext0_audio: endpoint { - remote-endpoint = <&dpaudio1_dcp>; - }; - }; - }; -}; - -&dcpext1 { - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - - dcpext1_audio: endpoint { - remote-endpoint = <&dpaudio2_dcp>; - }; - }; - }; -}; From 2b750dd76de6a32d200578afbfcdad78d2e5cfa9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 14:13:59 +1100 Subject: [PATCH 1352/1579] phy: apple: atc: drop the duplicate usb_off_pending check atcphy_usb3_set_mode() already returns -EBUSY while usb_off_pending is set, right after restoring the PHY. The second check further down came from an older composite and can never trigger. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/phy/apple/atc.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 09b2f86cff0a3..84cef035e0865 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -4326,9 +4326,6 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode mode == PHY_MODE_USB_DEVICE) return -EOPNOTSUPP; - if (atcphy->usb_off_pending) - return -EBUSY; - switch (mode) { case PHY_MODE_USB_HOST: atcphy->host_active = true; From cde55660779f30430a263a03ed477b568bd06f69 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 14:14:00 +1100 Subject: [PATCH 1353/1579] selftests: use the integration/aurora host fixtures The fixture alignments made on this branch on 10-26 are older than the ones integration/aurora carries: apple-dart-range-tests.py fails here and passes there. Take the integration/aurora versions of the tools/asahi DART and RTKit fixtures, the asahi-m4 host models and av-lifetime-test. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/apple/av-lifetime-test.c | 2 - tools/asahi/apple-dart-range-tests.py | 47 ++++++++++++++----- tools/asahi/rtkit-oslog-tests.py | 6 +-- .../selftests/drivers/gpu/asahi-m4/abi.py | 4 +- .../drivers/gpu/asahi-m4/check-channels.rs | 2 +- .../gpu/asahi-m4/fixtures/ABI-EXTENSIONS.md | 11 ----- .../gpu/asahi-m4/fixtures/manifest.json | 15 +----- .../gpu/asahi-m4/fixtures/public-uapi.rs | 4 -- .../drivers/gpu/asahi-m4/run-golden-tests.py | 19 ++------ .../gpu/asahi-m4/test-mmu-retention.py | 13 ++--- 10 files changed, 52 insertions(+), 71 deletions(-) delete mode 100644 tools/testing/selftests/drivers/gpu/asahi-m4/fixtures/ABI-EXTENSIONS.md diff --git a/drivers/gpu/drm/apple/av-lifetime-test.c b/drivers/gpu/drm/apple/av-lifetime-test.c index 7485436cbf595..193358ce2bfba 100644 --- a/drivers/gpu/drm/apple/av-lifetime-test.c +++ b/drivers/gpu/drm/apple/av-lifetime-test.c @@ -156,7 +156,6 @@ static struct av_life_test *av_life_setup(struct kunit *test, work_func_t owner_ dev_set_drvdata(t->dev, t->dcp); spin_lock_init(&t->ep->lock); INIT_LIST_HEAD(&t->ep->compact_cmds); - mutex_init(&t->ep->compact_tx_lock); init_rwsem(&t->asrv->srv_rwsem); INIT_WORK(&t->asrv->start_av_service_wq, owner_fn); INIT_WORK(&t->asrv->terminate_wq, owner_fn); @@ -487,7 +486,6 @@ static struct av_life_test *av_life_group_setup(struct kunit *test, KUNIT_ASSERT_NOT_NULL(test, t->ep); spin_lock_init(&t->ep->lock); INIT_LIST_HEAD(&t->ep->compact_cmds); - mutex_init(&t->ep->compact_tx_lock); t->ep->dcp = t->dcp; t->wq = alloc_ordered_workqueue("av-lifetime-devres", 0); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->wq); diff --git a/tools/asahi/apple-dart-range-tests.py b/tools/asahi/apple-dart-range-tests.py index d389d95be5920..5d311fef0813d 100644 --- a/tools/asahi/apple-dart-range-tests.py +++ b/tools/asahi/apple-dart-range-tests.py @@ -43,34 +43,36 @@ def main(): struct apple_dart; struct apple_dart_stream_map { struct apple_dart *dart; unsigned long sidmap[1]; }; struct apple_dart_hw { int (*invalidate_tlb)(struct apple_dart_stream_map *); }; -struct apple_dart { int num_streams, locked, version; void *dev; struct apple_dart_hw *hw; }; +struct apple_dart { int num_streams, locked, version; bool fw_mirror, fw_handoff, managed_sids; void *dev; struct apple_dart_hw *hw; }; struct apple_dart_atomic_stream_map { struct apple_dart *dart; unsigned long sidmap[1]; }; struct apple_dart_domain { - void *pgtbl_ops, *dcp; struct apple_dart_atomic_stream_map streams[2]; int count; + void *pgtbl_ops; bool dcp; struct apple_dart_atomic_stream_map streams[2]; int count; u64 dma_offset, mask; }; #define for_each_stream_map(i,d,m) for ((i)=0; (i)<(d)->count && ((m)=&(d)->streams[i],1); (i)++) -static int powered, power_error, sync_error, range_error, syncs, ranges, fulls; +static int powered, power_error, sync_error, range_error, syncs, ranges, fulls, firmware_flushes, quarantines, dcp_flushes; static u64 seen_first, seen_last; -static int pm_runtime_resume_and_get(void *dev) { if (power_error) return power_error; powered++; return 0; } +static int pm_runtime_resume_and_get(void *dev) { if (power_error && !dev) return power_error; powered++; return 0; } static void pm_runtime_put(void *dev) { assert(powered>0); powered--; } -static int apple_dart_dcp_domain_flush(struct apple_dart_domain *domain) { - (void)domain; assert(!"unexpected DCP dispatch in range fixture"); return -EINVAL; -} -static int apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, struct apple_dart_stream_map *stream, bool sync) { - assert(sync); syncs++; return sync_error; +static int apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, struct apple_dart_stream_map *stream, bool invalidate) { + assert(invalidate); syncs++; return sync_error; } static int apple_dart_t8110_hw_tlb_command_range(struct apple_dart_stream_map *stream, int command, bool range, u64 first, u64 last) { assert(command==1 && range); ranges++; seen_first=first; seen_last=last; return range_error; } +static int apple_dart_dcp_domain_flush(struct apple_dart_domain *domain) { assert(domain->dcp); dcp_flushes++; return -EAGAIN; } +static void apple_dart_managed_quarantine(struct apple_dart *dart) { assert(dart->managed_sids); quarantines++; } +static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream, u64 first, u64 last) { + firmware_flushes++; seen_first=first; seen_last=last; return range_error; +} static int full(struct apple_dart_stream_map *stream) { fulls++; return 0; } ''' source += production source += r''' static void reset(void) { assert(powered==0); power_error=sync_error=range_error=syncs=ranges=fulls=0; - seen_first=seen_last=0; + seen_first=seen_last=0; firmware_flushes=quarantines=dcp_flushes=0; } int main(void) { struct apple_dart_hw hw={.invalidate_tlb=full}; @@ -92,14 +94,35 @@ def main(): assert(fulls==1 && ranges==0 && syncs==0); reset(); dart.locked=1; sync_error=-EBUSY; assert(apple_dart_domain_flush_tlb_range(&domain,true,0x4000,0x7fff)==-EBUSY); - assert(ranges==0 && fulls==0 && powered==0); + assert(ranges==1 && fulls==0 && powered==0); reset(); power_error=-EHOSTDOWN; assert(apple_dart_domain_flush_tlb_range(&domain,true,0x4000,0x7fff)==-EHOSTDOWN); assert(syncs==0 && ranges==0 && fulls==0 && powered==0); reset(); range_error=-ETIMEDOUT; assert(apple_dart_domain_flush_tlb_range(&domain,true,0x4000,0x7fff)==-ETIMEDOUT); assert(ranges==1 && powered==0); - reset(); domain.streams[0].sidmap[0]=0; + reset(); dart.managed_sids=true; range_error=-ETIMEDOUT; + assert(apple_dart_domain_flush_tlb_range(&domain,true,0x4000,0x7fff)==-ETIMEDOUT); + assert(ranges==1 && quarantines==1 && powered==0); + reset(); sync_error=-EBUSY; range_error=-ETIMEDOUT; + assert(apple_dart_domain_flush_tlb_range(&domain,true,0x4000,0x7fff)==-EBUSY); + assert(ranges==1 && quarantines==1 && powered==0); + reset(); dart.fw_mirror=true; + assert(!apple_dart_domain_flush_tlb_range(&domain,true,0x4000,0x7fff)); + assert(firmware_flushes==1 && ranges==0 && syncs==1); + reset(); dart.fw_mirror=false; dart.fw_handoff=true; + assert(!apple_dart_domain_flush_tlb_range(&domain,true,0x4000,0x7fff)); + assert(firmware_flushes==1 && ranges==0); + reset(); dart.fw_handoff=false; domain.dcp=true; + assert(apple_dart_domain_flush_tlb_range(&domain,true,0x4000,0x7fff)==-EAGAIN); + assert(dcp_flushes==1 && ranges==0 && syncs==0); + reset(); domain.dcp=false; + struct apple_dart second=dart; second.dev=(void *)1; + domain.streams[1].dart=&second; domain.streams[1].sidmap[0]=1; domain.count=2; + power_error=-EHOSTDOWN; + assert(apple_dart_domain_flush_tlb_range(&domain,true,0x4000,0x7fff)==-EHOSTDOWN); + assert(ranges==1 && syncs==1 && powered==0); + reset(); domain.count=1; domain.streams[0].sidmap[0]=0; assert(!apple_dart_domain_flush_tlb_range(&domain,true,0x4000,0x7fff)); assert(syncs==0 && ranges==0 && fulls==0 && powered==0); puts("PASS: device DVA, low window, legacy/unlocked fallback, sync/power/command errors and empty stream"); diff --git a/tools/asahi/rtkit-oslog-tests.py b/tools/asahi/rtkit-oslog-tests.py index 9d8fcf3c9bd83..5f8eb290ebde5 100644 --- a/tools/asahi/rtkit-oslog-tests.py +++ b/tools/asahi/rtkit-oslog-tests.py @@ -45,14 +45,14 @@ def function(source, name): void *dev, *wq; struct mailbox *mbox; struct apple_rtkit_shmem oslog_buffer, syslog_buffer, ioreport_buffer, crashlog_buffer; - struct apple_rtkit_shmem kdebug_buffer0, kdebug_buffer1, coreanalytics_buffer; + struct apple_rtkit_shmem kdebug_buffer0, kdebug_buffer1; int sync_ack_fifo; bool sync_syslog_ack, oslog_inherited, syslog_inherited, crashlog_inherited, crashed, shutting_down; unsigned iop_power_state, ap_power_state; unsigned long endpoints[4]; char *syslog_msg_buffer; size_t syslog_n_entries, syslog_msg_size; - int epmap_completion, iop_pwr_ack_completion, ap_pwr_ack_completion, coreanalytics_completion; + int epmap_completion, iop_pwr_ack_completion, ap_pwr_ack_completion; }; static unsigned sent, warnings, errors, requests, detachments, flushes, freed, checks; static int send_error, setup_error, start_error; @@ -125,7 +125,7 @@ def function(source, name): detachments=flushes=freed=0;start_error=fail?-5:0; assert(apple_rtkit_reinit(&r)==start_error); assert(!r.oslog_inherited && !r.syslog_inherited && !r.crashlog_inherited); - assert(!r.oslog_buffer.size && !r.crashed && !r.syslog_msg_size && freed==7); + assert(!r.oslog_buffer.size && !r.crashed && !r.syslog_msg_size && freed==6); assert(r.endpoints[0]==(1UL< Self { diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/run-golden-tests.py b/tools/testing/selftests/drivers/gpu/asahi-m4/run-golden-tests.py index d751e9c93f255..962713941338b 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/run-golden-tests.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/run-golden-tests.py @@ -15,21 +15,14 @@ def main(): manifest = unpack_fixtures(golden) (out / 'public-uapi.rs').write_bytes(frozen_bindings().read_bytes()) backend = root / 'drivers/gpu/drm/asahi/g16' - # The hardware catalog now imports its data-only declaration from shared - # fwid. Compile that exact production declaration; firmware measurement is - # outside this wire-encoder harness. Independent byte fixtures stay frozen. - fwid = root / 'drivers/gpu/drm/asahi/fwid.rs' - declaration = '' - if fwid.exists(): - identity = fwid.read_text() - begin = identity.index('pub(crate) struct FirmwareBuild {') - end = identity.index('\n}', begin) + 2 - declaration = identity[begin:end] + identity = (root / 'drivers/gpu/drm/asahi/fwid.rs').read_text() + first = identity.index('pub(crate) struct FirmwareBuild {') + build_type = identity[first:identity.index('\n}', first) + 2] logs = [] for name in ['initdata', 'compute', 'render', 'channels', 'initdata-bounds']: source = (HERE / f'check-{name}.rs').read_text() source = source.replace('@KERNEL@', str(root)).replace('@OUTPUT@', str(out)) - source = source.replace('@FIRMWARE_BUILD@', declaration) + source = source.replace('@FWID_BUILD@', build_type) test = out / f'check-{name}.rs' test.write_text(source) binary = out / f'check-{name}' @@ -44,9 +37,7 @@ def main(): report = {'passed': True, 'scope': 'Production Rust encoder output versus frozen independent C fixtures; CPU-only, no hardware acceptance.', 'reference_kernel_commit': manifest['reference_kernel_commit'], 'fixture_manifest_sha256': sha(HERE / 'fixtures/manifest.json'), 'seconds': round(time.monotonic()-start, 3), 'compared_encoder_bytes': 3736988, - 'source_sha256': {str(p.relative_to(root)): sha(p) for p in sorted(backend.rglob('*.rs'))}, - 'firmware_declaration_source_sha256': sha(fwid) if fwid.exists() else None, - 'firmware_declaration_scope': 'Shared FirmwareBuild data declaration only; identity measurement is not executed by this harness.'} + 'source_sha256': {str(p.relative_to(root)): sha(p) for p in sorted(backend.rglob('*.rs'))}} (out / 'golden-tests.json').write_text(json.dumps(report, indent=2)+'\n') return 0 diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py index 92ed7670812a3..787992e9b4c24 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py @@ -29,6 +29,7 @@ def body(source,marker): struct MutexBackend;struct Mutex(StdMutex);implMutex{fn new(v:T)->Self{Self(StdMutex::new(v))}fn lock(&self)->Guard<'_,T,MutexBackend>{Guard(self.0.lock().unwrap(),std::marker::PhantomData)}} struct Guard<'a,T,M>(std::sync::MutexGuard<'a,T>,std::marker::PhantomData);implDeref for Guard<'_,T,M>{type Target=T;fn deref(&self)->&T{&self.0}}implDerefMut for Guard<'_,T,M>{fn deref_mut(&mut self)->&mut T{&mut self.0}} struct Monotonic;struct Delta;impl Delta{fn from_millis(n:u64)->u64{n}}struct Instant{n:std::cell::Cell,p:std::marker::PhantomData}implInstant{fn now()->Self{Self{n:std::cell::Cell::new(0),p:std::marker::PhantomData}}fn elapsed(&self)->u64{let n=self.n.get();self.n.set(n+1);n}} +mod pgtable{pub fn clean_poc(_p:*const u8,_n:usize){super::BARRIERS.fetch_add(1,super::Ordering::Relaxed);}pub fn poc_barrier(){super::BARRIERS.fetch_add(1,super::Ordering::Relaxed);}} mod mem{pub fn sync(){super::BARRIERS.fetch_add(1,super::Ordering::Relaxed);}pub fn tlbi_asid(_:u8){}} #[derive(Clone,Copy,PartialEq,Eq)]enum UatFirmware{Handoff,PublishedRoots} struct Handoff{lock_ap:AtomicU8,lock_fw:AtomicU8,turn:AtomicU32,cur_slot:AtomicU32}impl Handoff{fn new()->Self{Self{lock_ap:AtomicU8::new(0),lock_fw:AtomicU8::new(0),turn:AtomicU32::new(0),cur_slot:AtomicU32::new(0xffff)}} @@ -36,23 +37,19 @@ def body(source,marker): handoff='\n'.join(body(mm,m) for m in ['fn lock(&self) {\n self.lock_ap', 'fn unlock(&self) {\n self.turn', 'fn current_slot(']) mid=r''' } -struct SlotTTBS{ttb0:AtomicU64,ttb1:AtomicU64}struct UatShared{quirks:UatQuirks,handoff:Handoff,ttbs:[SlotTTBS;64],map_kernel_to_user:bool,kernel_ttb1:u64,binding_generations:[u64;64],binding_published:[bool;64],owned_user_roots:[u64;64]} +struct Quirks{handoff_lock:bool,clean_tables:bool} +struct SlotTTBS{ttb0:AtomicU64,ttb1:AtomicU64}struct UatShared{handoff:Handoff,ttbs:[SlotTTBS;64],map_kernel_to_user:bool,kernel_ttb1:u64,binding_generations:[u64;64],binding_published:[bool;64],owned_user_roots:[u64;64],quirks:Quirks} impl UatShared{fn handoff(&self)->&Handoff{&self.handoff}fn ttbs(&self)->&[SlotTTBS;64]{&self.ttbs}} struct UatInner{firmware:UatFirmware,m4_adopted:bool,failed:AtomicBool,shared:Mutex} ''' -# Read quirk policy and wrappers from production rather than copying their behavior. -quirks=body(mm,'pub(crate) struct UatQuirks {')+'\n'+body(mm,'impl UatQuirks {') -pre=pre.replace('struct Handoff{',quirks+'\nstruct Handoff{',1) -pre=pre.replace('mod mem{', 'mod pgtable{pub fn clean_poc(_p:*const u8,_n:usize){panic!("cache-clean profile is outside this fixture");}pub fn poc_barrier(){panic!("cache-clean profile is outside this fixture");}}\nmod mem{',1) -shared_methods='\n'.join(body(mm,m) for m in ['fn handoff_lock(', 'fn handoff_unlock(', 'fn publish_slot(']) -mid=mid.replace('impl UatShared{','impl UatShared{'+shared_methods+'\n',1) +mid=mid.replace('impl UatShared{','impl UatShared{'+''.join(body(mm,m) for m in ['fn handoff_lock(', 'fn handoff_unlock(', 'fn publish_slot('])) raii='#[path="'+str(ROOT/'drivers/gpu/drm/asahi/mmu/ppl.rs')+'"]mod ppl;\n'+'\n'.join(body(mm,m) for m in [ 'struct M4Transport<', 'impl ppl::Memory for M4Transport', 'enum HandoffGuard<', 'impl HandoffGuard<', 'impl Drop for HandoffGuard<' ]) inner='impl UatInner{\n'+'\n'.join(body(mm,m) for m in ['fn lock(&self) -> Guard', 'fn check_m4(', 'fn lock_handoff(', 'fn lock_m4_upper('])+'\n}\n' models=r''' -fn uat(m4:bool,firmware:UatFirmware)->Rc{Rc::new(UatInner{firmware,m4_adopted:m4,failed:AtomicBool::new(false),shared:Mutex::new(UatShared{quirks:UatQuirks::DEFAULT,handoff:Handoff::new(),ttbs:std::array::from_fn(|_|SlotTTBS{ttb0:AtomicU64::new(0),ttb1:AtomicU64::new(0)}),map_kernel_to_user:!m4,kernel_ttb1:0x800000000,binding_generations:[0;64],binding_published:[false;64],owned_user_roots:[0;64]})})} +fn uat(m4:bool,firmware:UatFirmware)->Rc{Rc::new(UatInner{firmware,m4_adopted:m4,failed:AtomicBool::new(false),shared:Mutex::new(UatShared{handoff:Handoff::new(),ttbs:std::array::from_fn(|_|SlotTTBS{ttb0:AtomicU64::new(0),ttb1:AtomicU64::new(0)}),map_kernel_to_user:!m4,kernel_ttb1:0x800000000,binding_generations:[0;64],binding_published:[false;64],owned_user_roots:[0;64],quirks:Quirks{handoff_lock:true,clean_tables:false}})})} #[derive(Clone,Copy)]struct Prot{cached:bool}impl Prot{fn is_cached_noncoherent(&self)->bool{self.cached}} struct Mutation;impl Drop for Mutation{fn drop(&mut self){MUTATIONS.fetch_sub(1,Ordering::Relaxed);}} struct MapTable{fail:bool}impl MapTable{fn unmap_pages(&mut self,_r:Range)->Result{LEAF_CLEARS.fetch_add(1,Ordering::Relaxed);if self.fail{Err(EIO)}else{Ok(())}}} From fe9ff6f651d562fe72ee0d4ae7ac14cdd5b5a8e8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 14:14:01 +1100 Subject: [PATCH 1354/1579] MAINTAINERS: drop the stale PMP DVFS doc and the duplicate QSPIMC entry The PMP dvfs debugfs file that t8140-pmp-dvfs.rst describes exists on neither branch, and the ARM/APPLE QSPIMC DRIVER entry repeats the existing QSPI entry. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- Documentation/soc/apple/t8140-pmp-dvfs.rst | 24 ---------------------- MAINTAINERS | 8 -------- 2 files changed, 32 deletions(-) delete mode 100644 Documentation/soc/apple/t8140-pmp-dvfs.rst diff --git a/Documentation/soc/apple/t8140-pmp-dvfs.rst b/Documentation/soc/apple/t8140-pmp-dvfs.rst deleted file mode 100644 index 7ec4ae7366ed9..0000000000000 --- a/Documentation/soc/apple/t8140-pmp-dvfs.rst +++ /dev/null @@ -1,24 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0-only - -T8140 PMP DVFS diagnostics -========================= - -The existing PMP report provider exposes a read-only ``dvfs`` file under -``/sys/kernel/debug/apple-pmp-/`` on T8140. The provider must be -bound; the file is readable by root. Its output lists ADT report entries -8 through 11 and both raw 64-bit words of each 16-byte entry. Only those four entries -are read. The report aperture is not dumped. - -Reading returns ``EAGAIN`` if the PMP running bit is clear before or after -sampling. Entries are sampled individually, rather than as an atomic -firmware snapshot. A running PMP alone does not establish freshness of -each value. - -The field encoding and conversion to clock rates are not yet established. -Values must not be interpreted as Hz, temperatures, or verified hardware -clock states. This interface changes neither firmware requests nor cache -allocation nor thermal policy. - -CPU temperatures remain available through ``macsmc-hwmon``. CPU cooling -and firmware frequency limits remain under their existing drivers. The -DVFS diagnostic file is not a thermal sensor. diff --git a/MAINTAINERS b/MAINTAINERS index 8b2cc48645bf4..d966b42b5cbe6 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2110,7 +2110,6 @@ F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml F: Documentation/gpu/apple-dcp-backlight.rst F: Documentation/hid/apple-j700-multitouch.rst -F: Documentation/soc/apple/t8140-pmp-dvfs.rst F: Documentation/networking/device_drivers/wifi/mt7932-neo.rst F: arch/arm64/boot/dts/apple/t8140* F: arch/arm64/configs/j700*.config @@ -2809,13 +2808,6 @@ S: Maintained F: Documentation/devicetree/bindings/spi/apple,qspi.yaml F: drivers/spi/spi-apple-qspi.c -ARM/APPLE QSPIMC DRIVER -M: Ryan Murray -W: https://aurora-silicon.github.io -S: Maintained -F: Documentation/devicetree/bindings/spi/apple,qspi.yaml -F: drivers/spi/spi-apple-qspi.c - ARM/ARTPEC MACHINE SUPPORT M: Jesper Nilsson M: Lars Persson From dba95779bf549bf5e7f379d1f6b38e41c73b2bb9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 14:14:22 +1100 Subject: [PATCH 1355/1579] tools: aurora-sep: bump the package release for the identity tool integration/aurora already shipped aurora-sep 1-9 (Fedora) and pkgrel 14 (Arch) without sep-bio-identities. This branch added the tool under the same release numbers, so a build from here would collide with the shipped packages. Give it its own release: Fedora 1-10 with its own changelog entry, and Arch pkgrel 15. The sources and their checksums do not change. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD | 2 +- tools/aurora-sep/packaging/fedora/aurora-sep.spec | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 11d3f521bee8b..33a6f7b85502a 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -7,7 +7,7 @@ pkgname=aurora-sep pkgver=1 -pkgrel=14 +pkgrel=15 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index efb0a1afc669f..4690dd111949f 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -8,7 +8,7 @@ Name: aurora-sep Version: 1 -Release: 9%{?dist} +Release: 10%{?dist} Summary: Loader and desktop integration for the Apple SEP fingerprint driver License: GPL-2.0-only @@ -155,6 +155,9 @@ fi %ghost %verify(not md5 size mode mtime) %{_sharedstatedir}/selinux/%{selinuxtype}/active/modules/200/%{modulename} %changelog +* Fri Oct 30 2026 Ryan Murray - 1-10 +- Add the Linux-lane orphan identity list and delete tool + * Wed Oct 28 2026 Ryan Murray - 1-9 - Start the login screen without waiting for Touch ID, and show it again once Touch ID is ready if nobody has logged in yet @@ -163,7 +166,6 @@ fi the record a boot policy change makes the enclave rewrite - aurora-sep-policy-change: back up the gigalocker and allow that one rewrite on the next boot, so Touch ID survives a boot policy change -- Add the Linux-lane orphan identity list and delete tool * Mon Oct 19 2026 Ryan Murray - 1-7 - XART-SAFETY.md: what the xART guard allows for restore, use, setup and From eb86efd6f805027b1a64af509e662c8a1c0d1743 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 13:10:43 +1100 Subject: [PATCH 1356/1579] selftests: apple-dcp: follow the PMP auxiliary buffer in ordered-rx #472 added a T6050 PMP registry-v2 path to apple_rtkit_rx_work(): endpoint 7 now goes to apple_rtkit_pmp_aux_rx() when rtk->pmp_registry_v2 is set. The ordered-rx fixture extracts apple_rtkit_rx_work(), but its struct apple_rtkit has no such field and nothing defines the new handler, so the group stopped compiling and the apple-dcp host suite went from 80/80 to 79/80 on integration/aurora. Give the fixture struct apple_rtkit the field and stub the handler the way the CoreAnalytics one is stubbed, as unreachable in this fixture. No assertion changes. The edid group also still passes on the tree before #472, where the stub is unused. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 14b55156c4d6a64a0db70140ae5852bdd429d907) --- tools/testing/selftests/drivers/gpu/apple-dcp/edid.py | 2 +- tools/testing/selftests/drivers/gpu/apple-dcp/run.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py index 60ed9b365d099..9b949a1a6ff1d 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -855,7 +855,7 @@ def function(text, signature): }; struct afk_receive_message_work { struct apple_dcp_afkep *ep; u64 message; struct work_struct work; }; struct apple_rtkit_ops { void (*recv_message)(void *,u8,u64); }; -struct apple_rtkit { bool shutting_down, sync_syslog_ack; const struct apple_rtkit_ops *ops; void *cookie,*dev; }; +struct apple_rtkit { bool shutting_down, sync_syslog_ack, pmp_registry_v2; const struct apple_rtkit_ops *ops; void *cookie,*dev; }; struct apple_rtkit_rx_work { struct apple_rtkit *rtk; u8 ep; u64 msg; struct work_struct work; }; #define current ((void *)1) static pthread_mutex_t reply_lock=PTHREAD_MUTEX_INITIALIZER; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index efc79d1345f7a..f836743f2fc8d 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -156,6 +156,9 @@ def main(): prefix += ('#define APPLE_RTKIT_EP_COREANALYTICS ' + endpoint + '\n' '#define apple_rtkit_coreanalytics_rx(...) ' 'assert(!"CoreAnalytics is outside this ordered-display fixture")\n') + # The T6050 PMP registry-v2 buffer shares endpoint 7. + prefix += ('#define apple_rtkit_pmp_aux_rx(...) ' + 'assert(!"The PMP auxiliary buffer is outside this ordered-display fixture")\n') unit = prefix + unit if unit is not None: result['translation_unit_sha256'] = run_unit(name, unit, output, cases, includes if name == 'service-rebind' else (), flags) From ce061da9bd77cc581aaaf12115baeabd1bf32943 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 14:19:11 +1100 Subject: [PATCH 1357/1579] tools: asahi: model the T8132 members in the ATCPHY USB3 harness The T8132 support in atc.c added the PIPE lock timeout helper, T8132 lane and PIPE back ends, a USB2-only power restore and new state members. The harness extracts the driver bodies, so it stopped compiling on integration/aurora. Extract atcphy_pipehandler_lock_timeout() with the other driver functions, since the ACK polling moved into it, and point the missing-ack-check mutation there. Add the new members to the modeled structs, and stub the T8132 back ends and the USB2-only power restore with assertions: the modeled machines never select that generation. All 36 cases pass and all 49 mutations are killed. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 97dafebbcceadb931a05f495d4a9724fb83ff855) --- tools/asahi/atc-usb3-tests.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tools/asahi/atc-usb3-tests.py b/tools/asahi/atc-usb3-tests.py index 5e45e11c3c46d..af585383ed33e 100644 --- a/tools/asahi/atc-usb3-tests.py +++ b/tools/asahi/atc-usb3-tests.py @@ -74,7 +74,7 @@ def block(source, marker): STUBS = r''' struct atcphy_hw { enum atcphy_generation gen; - bool has_t8140_dp, dp_t8122, has_usb4, park_pipe_unlocked, park_dummy_phy; }; + bool has_t8140_dp, dp_t8122, has_usb4, park_pipe_unlocked, park_dummy_phy, usb2_only; }; struct device_node { const char *const *compatible; unsigned int count; }; static bool of_device_is_compatible(const struct device_node *root, const char *compatible) { if (!root) return false; @@ -95,6 +95,9 @@ def block(source, marker): enum atcphy_mode typec_mode; bool pipe_pending, t8103_dummy_pending, t8103_usb3_retry, t8103_usb3_workaround, mode_dirty, usb_off_pending; bool host_active, usb3_port_off, dp_only, ss_tunables, fixed_usb2, t8140_dp_tunables, dp_tunables; + /* T8132 state: the modeled machines never select that generation. */ + bool t8132_ss, t8132_common_owned, t8132_failed; + struct { void *usb3_dp; } tunables; }; struct phy { struct apple_atcphy *data; }; struct typec_mux_dev { struct apple_atcphy *data; }; @@ -169,6 +172,16 @@ def block(source, marker): static int atcphy_usb2_power_off(struct apple_atcphy *a) { (void)a; usb2_power_off_calls++; return 0; } +/* T8132 back ends: out of scope, and never reached by the modeled generations. */ +static int atcphy_configure_t8132(struct apple_atcphy *a, enum atcphy_mode mode) { + (void)a; (void)mode; assert(!"T8132 lane setup reached"); return -EINVAL; +} +static int atcphy_configure_pipehandler_usb3_host_t8132(struct apple_atcphy *a) { + (void)a; assert(!"T8132 PIPE setup reached"); return -EINVAL; +} +static void atcphy_usb2_power_on(struct apple_atcphy *a) { + (void)a; assert(!"USB2-only power restore reached"); +} static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) { /* ACTUAL_CONFIGURE_ENTRY */ struct apple_atcphy *a = atcphy; @@ -619,6 +632,7 @@ def harness(source): end = source.index("\n};", source.index("} atcphy_modes[] = {", start)) + 3 table = source[start:end] function_specs = [ + ("int", "atcphy_pipehandler_lock_timeout"), ("int", "atcphy_pipehandler_lock"), ("int", "atcphy_pipehandler_lock_live"), ("int", "atcphy_configure_pipehandler_usb3"), @@ -662,7 +676,7 @@ def harness(source): MUTATIONS = {'no-live-retry': ('atcphy_pipehandler_lock_live', '? 10 : 1;', '? 1 : 1;'), 'retry-later-generations': ('atcphy_pipehandler_lock_live', '? 10 : 1;', '? 10 : 10;'), 'sleep-after-last': ('atcphy_pipehandler_lock_live', 'if (attempt + 1 < attempts)', 'if (true)'), - 'missing-ack-check': ('atcphy_pipehandler_lock', + 'missing-ack-check': ('atcphy_pipehandler_lock_timeout', 'if (requested & PIPEHANDLER_LOCK_EN) {', 'if (requested & PIPEHANDLER_LOCK_EN) return 0;\n' '\tif (requested & PIPEHANDLER_LOCK_EN) {'), From 3b0dd96e31dec3a167ac5e476640dbed4a00ecd8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 16:17:49 +1100 Subject: [PATCH 1358/1579] media: apple: ave: turn the J700 encoder transport into a V4L2 encoder The AVE driver was a root-only debugfs transport that replays captured firmware commands at a fixed 320x240, and it could not be built into a kernel with suspend or kexec. Rework it into a normal V4L2 stateful encoder driver on top of Acelogic's mailbox and ring transport from 4612e5f8a66a ("media: apple: add an opt-in J700 AVE encoding transport"), which stays as the IPC layer with its start-up handshake. - Probe binds by DT only: no enable or snapshot parameters, no SoC checks beyond the compatible. The five encoder power domains are attached as a list and follow the device's runtime PM. - The firmware is booted on first use instead of through debugfs. The driver keeps a copy of the firmware DATA segment taken while it still matches the loader's checksum and puts it back before every boot, so the coprocessor can lose power and start again from the same state. - Commands are built in the driver. Config, Open, Complete, Stop, Close and Halt are complete. Session start and per-frame Process are left returning -EOPNOTSUPP until their layouts are described. - Firmware replies are checked for their status word. - A V4L2 M2M encoder (NV12 in, H.264 or HEVC out) per dev-encoder.rst, with drain, EOS and bitrate, GOP, profile, level and QP controls. - System sleep marks the firmware lost when the coprocessor lost power; the next session boots it again. Runtime suspend keeps power while the firmware runs until its stop sequence is in. Rename the symbol to VIDEO_APPLE_AVE and drop the !SUSPEND, !HIBERNATION and !KEXEC_CORE dependencies. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- Documentation/admin-guide/media/apple-ave.rst | 102 +-- drivers/media/platform/apple/ave/Kconfig | 22 +- drivers/media/platform/apple/ave/Makefile | 3 +- .../media/platform/apple/ave/apple-ave-ipc.h | 446 ---------- drivers/media/platform/apple/ave/apple-ave.c | 532 ----------- drivers/media/platform/apple/ave/ave-cmd.c | 108 +++ drivers/media/platform/apple/ave/ave-drv.c | 178 ++++ drivers/media/platform/apple/ave/ave-fw.c | 359 ++++++++ drivers/media/platform/apple/ave/ave-ipc.c | 384 ++++++++ drivers/media/platform/apple/ave/ave-regs.h | 128 +++ drivers/media/platform/apple/ave/ave-v4l2.c | 831 ++++++++++++++++++ drivers/media/platform/apple/ave/ave.h | 167 ++++ 12 files changed, 2209 insertions(+), 1051 deletions(-) delete mode 100644 drivers/media/platform/apple/ave/apple-ave-ipc.h delete mode 100644 drivers/media/platform/apple/ave/apple-ave.c create mode 100644 drivers/media/platform/apple/ave/ave-cmd.c create mode 100644 drivers/media/platform/apple/ave/ave-drv.c create mode 100644 drivers/media/platform/apple/ave/ave-fw.c create mode 100644 drivers/media/platform/apple/ave/ave-ipc.c create mode 100644 drivers/media/platform/apple/ave/ave-regs.h create mode 100644 drivers/media/platform/apple/ave/ave-v4l2.c create mode 100644 drivers/media/platform/apple/ave/ave.h diff --git a/Documentation/admin-guide/media/apple-ave.rst b/Documentation/admin-guide/media/apple-ave.rst index d7e63d029de15..dabb4250a72a1 100644 --- a/Documentation/admin-guide/media/apple-ave.rst +++ b/Documentation/admin-guide/media/apple-ave.rst @@ -1,63 +1,43 @@ .. SPDX-License-Identifier: GPL-2.0-only -Experimental T8140 video encoding -================================ - -The J700 AVE driver exposes a root-only debugfs firmware transport for -hardware encoding experiments. It is not a V4L2 encoder, has no stable -userspace ABI, and does not make an FFmpeg encoder available. The initial -scope is the existing fixed 320x240 NV12 H.264/HEVC protocol experiment. - -Build ``CONFIG_VIDEO_APPLE_AVE_EXPERIMENTAL=m`` on a 16 KiB-page Apple kernel -with debugfs and translated DMA. Suspend, hibernation and kexec must be off. -Load with ``modprobe apple-ave enable=1``; probing is disabled otherwise. - -Firmware and boot requirements ------------------------------- - -Use the public Aurora m1n1 J700 AVE handoff. It reserves the firmware already -loaded by iBoot, fills the device's ``memory-region`` references and exports -``aurora,ave-text-crc32`` and ``aurora,ave-data-crc32`` strings under -``/chosen``. No firmware file is included or requested by this driver. -The current transport requires the qualified 25G76 two-segment layout: -0x14c000 text bytes and 0x130000 data bytes, mapped starting at IOVA -0x10000000000. Every mapping and both content CRCs are checked before use. -Text CRCs are restricted to the two inspected 9003.78.0 images (0x2ce7359e -and 0x69c54454). These checks detect build drift, not firmware authenticity. -Different firmware is unsupported, even if the device has the same SoC. - -The driver holds the DMA, pipe4, pipe5, me0 and me1 power domains and owns -the IPC, queue, log and firmware-requested heap allocations. Debugfs lives -under ``/sys/kernel/debug/285100000.video-codec``: - -* ``ave-status`` reports mappings, protocol phase and reply counters. -* ``ave-start-once`` accepts one explicit firmware start per boot. -* ``ave-command`` accepts one bounded packet and returns a framed reply. -* ``ave-arena`` stages image/reference/output surfaces within owned memory. -* ``ave-poll`` drains later firmware replies without another submission. - -Writes require ``CAP_SYS_RAWIO``. Command packets must come from a trusted, -matching local protocol experiment with all embedded surface addresses -relocated into this boot's owned arena. Root access does not make arbitrary -firmware packets valid. Binary command captures are supplied separately and -are not included in this contribution. - -Lifecycle and verification --------------------------- - -After the start attempt, the module, DMA buffers and power references remain -pinned until an external hardware reset, including after failure. Do not -force-unload, unbind the DARTs, reuse uncertain buffers or retry a failed -command. Firmware quiescence and normal production teardown are unfinished. - -A transport acknowledgement is not an encoded frame. For physical tests, -record the exact kernel/module, DT and firmware hashes; submit changing -images; validate each codec reply; independently decode the current DMA -output; check every frame's dimensions and quality; then verify Stop/Close, -kernel health and the existing desktop/radio stack. Keep the generated -inputs, bitstreams, decoded pixels, logs and hashes as a repeatable artifact. - -The prior implementation was authored by Ace (Acelogic). This contribution -ports that existing GPL work. Arbitrary resolutions, standard media APIs, -throughput, concurrent clients and other codecs or SoCs are outside this -experiment. +Apple AVE video encoder +======================= + +The AVE block in Apple T8140 (A18 Pro) SoCs, as used in the MacBook Neo, +encodes H.264 and HEVC. ``CONFIG_VIDEO_APPLE_AVE`` builds the ``apple-ave`` +driver, which registers a V4L2 memory-to-memory stateful encoder following +Documentation/userspace-api/media/v4l/dev-encoder.rst. The OUTPUT queue +takes NV12 frames (64-byte aligned line stride), the CAPTURE queue returns +H.264 or HEVC. + +Firmware +-------- + +The encoder runs firmware on its own coprocessor. The bootloader loads it into +two reserved, DART-mapped segments (TEXT and DATA) but does not start it. The +Aurora Stage 2 loader passes both regions in the device's ``memory-region`` +property and records their CRC32 under ``/chosen`` as +``aurora,ave-text-crc32`` and ``aurora,ave-data-crc32``. No firmware file is +requested. + +The driver only binds to firmware builds whose command formats it knows, +identified by the TEXT checksum, and only when DATA still matches the +checksum the loader recorded. It keeps a copy of DATA and restores it before +every firmware boot, so the coprocessor can lose power and start again from +the same state. + +Power management +---------------- + +The firmware boots when the first encoding session starts. Runtime PM uses a +two second autosuspend delay. While the firmware runs the encoder stays +powered; across system sleep the power domains go down, open sessions fail +their next frame, and the next session boots the firmware again. + +Status +------ + +Firmware start-up, the command rings, session open/close and the V4L2 +interface are implemented. Session start and per-frame encoding are not yet +wired up, so ``VIDIOC_STREAMON`` on both queues currently fails with +``EOPNOTSUPP``. diff --git a/drivers/media/platform/apple/ave/Kconfig b/drivers/media/platform/apple/ave/Kconfig index 629ce9244039a..59937167b29ce 100644 --- a/drivers/media/platform/apple/ave/Kconfig +++ b/drivers/media/platform/apple/ave/Kconfig @@ -1,16 +1,16 @@ # SPDX-License-Identifier: GPL-2.0-only -config VIDEO_APPLE_AVE_EXPERIMENTAL - tristate "Apple T8140 experimental video encoder transport" - depends on ARCH_APPLE && ARM64_16K_PAGES - depends on PM_GENERIC_DOMAINS && IOMMU_DMA && DEBUG_FS - depends on !SUSPEND && !HIBERNATION && !KEXEC_CORE +config VIDEO_APPLE_AVE + tristate "Apple AVE video encoder" + depends on ARCH_APPLE || COMPILE_TEST + depends on VIDEO_DEV && IOMMU_DMA && PM_GENERIC_DOMAINS select CRC32 + select V4L2_MEM2MEM_DEV + select VIDEOBUF2_DMA_CONTIG help - Experimental root-only AVE firmware and command transport for the - MacBook Neo. This does not register a V4L2 or FFmpeg encoder. + V4L2 memory-to-memory driver for the AVE H.264/HEVC encoder in + Apple T8140 (A18 Pro) SoCs, as in the MacBook Neo. The encoder + firmware is loaded by the bootloader. - It requires the bootloader's reserved, preloaded firmware and an - explicit enable=1 module parameter. After firmware startup, memory - and power references remain pinned until a hardware reset. Do not - enable this on a general-purpose kernel requiring suspend or kexec. + To compile this driver as a module, choose M here: the module + will be called apple-ave. diff --git a/drivers/media/platform/apple/ave/Makefile b/drivers/media/platform/apple/ave/Makefile index c8e673e68f080..8e0a5519a520b 100644 --- a/drivers/media/platform/apple/ave/Makefile +++ b/drivers/media/platform/apple/ave/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_VIDEO_APPLE_AVE_EXPERIMENTAL) += apple-ave.o +obj-$(CONFIG_VIDEO_APPLE_AVE) += apple-ave.o +apple-ave-y := ave-drv.o ave-fw.o ave-ipc.o ave-cmd.o ave-v4l2.o diff --git a/drivers/media/platform/apple/ave/apple-ave-ipc.h b/drivers/media/platform/apple/ave/apple-ave-ipc.h deleted file mode 100644 index d317af1dad72b..0000000000000 --- a/drivers/media/platform/apple/ave/apple-ave-ipc.h +++ /dev/null @@ -1,446 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Experimental 25G76/64-bit AVE mailbox and bidirectional ring transport. - * Included after private driver types. All callers hold ave->lock. - */ -#define AVE_CHANNEL_OFF 0x4000 -#define AVE_HANDSHAKE_OFF 0x24000 -#define AVE_SHARED_OFF 0x28000 -#define AVE_COMMAND_OFF 0x40000 -#define AVE_COMMAND_BYTES SZ_128K -#define AVE_ARENA_OFF 0x80000 -#define AVE_START_MAGIC 0x08042006 - -static int ave_fw_offset(struct apple_ave *ave, u64 address, size_t bytes, u32 *offset) -{ - u64 delta = address - ave->ipc_fw_base; - - if (address < ave->ipc_fw_base || delta > ave->ipc.size || bytes > ave->ipc.size - delta) - return -ERANGE; - *offset = delta; - return 0; -} - -static void ave_mailbox_send(struct apple_ave *ave, u32 a, u32 b, u32 c, u32 d) -{ - writel(a, ave->mailbox + 0x18); - writel(b, ave->mailbox + 0x1c); - writel(c, ave->mailbox + 0x20); - writel(d, ave->mailbox + 0x24); - wmb(); - writel(1, ave->mailbox + 0x0c); -} - -static int ave_mailbox_receive(struct apple_ave *ave) -{ - u32 status; - int ret = readl_poll_timeout(ave->mailbox + 0x10, status, status & 1, 50, 5000000); - - if (ret) - return ret; - /* Acknowledge the interrupt before reading the four scratch words. */ - writel(1, ave->mailbox + 0x10); - for (unsigned int i = 0; i < 4; i++) - ave->ipc_reply[i] = readl(ave->mailbox + 0x18 + i * 4); - dma_rmb(); - dev_info(ave->dev, "AVE_IPC_MAILBOX phase=%s words=%#x,%#x,%#x,%#x\n", - ave->phase, ave->ipc_reply[0], ave->ipc_reply[1], ave->ipc_reply[2], ave->ipc_reply[3]); - return 0; -} - -static int ave_create_channels(struct apple_ave *ave) -{ - bool found[2] = { false, false }; - - for (unsigned int i = 0; i < 2; i++) { - u8 *desc = ave->ipc.cpu + AVE_CHANNEL_OFF + i * 0x100; - u32 direction = get_unaligned_le32(desc + 0x40); - u32 interrupt = get_unaligned_le32(desc + 0x44); - u32 slots = get_unaligned_le32(desc + 0x48), offset; - u64 address = get_unaligned_le64(desc + 0x4c); - struct ave_channel *c; - unsigned int id; - - if (!memchr(desc, 0, 64)) - return -EPROTO; - if (!strcmp(desc, "IO")) - id = 0; - else if (!strcmp(desc, "IO_T2H")) - id = 1; - else - return -EPROTO; - if (found[id] || direction != id || !interrupt || interrupt >= 32 || - !slots || slots > ave->channel_bytes / 64 || !IS_ALIGNED(address, 64) || - ave_fw_offset(ave, address, (size_t)slots * 64, &offset) || - offset < AVE_CHANNEL_OFF + 0x200 || - offset + slots * 64 > AVE_CHANNEL_OFF + ave->channel_bytes) - return -EPROTO; - found[id] = true; - c = &ave->channel[id]; - strscpy(c->name, desc, sizeof(c->name)); - c->mode = direction ^ 1; - c->interrupt = interrupt; - c->slots = slots; - c->offset = offset; - c->consumer = c->mode ? -1 : 0; - c->producer = c->mode ? 0 : -1; - } - if (!found[0] || !found[1] || - (ave->channel[0].offset < ave->channel[1].offset + ave->channel[1].slots * 64 && - ave->channel[1].offset < ave->channel[0].offset + ave->channel[0].slots * 64)) - return -EPROTO; - /* Initialize only host-originated slots; leave all padding and - * firmware-originated slots unchanged. Ownership is bit zero of address. - */ - for (unsigned int i = 0; i < ave->channel[0].slots; i++) { - __le64 *slot = ave->ipc.cpu + ave->channel[0].offset + i * 64; - slot[0] = cpu_to_le64(1); - slot[1] = 0; - slot[2] = 0; - } - dma_wmb(); - return 0; -} - -static int ave_negotiate_ipc(struct apple_ave *ave) -{ - u64 ticks, time_delta, handshake_address, channel_address; - u32 frequency, status, offset; - u8 *handshake = ave->ipc.cpu + AVE_HANDSHAKE_OFF; - int ret; - - ave->phase = "validate-offer"; - if (ave->hello[0] != 2 || ave->hello[2] != 0x100 || - ave->hello[1] < 0x200 || ave->hello[1] > AVE_HANDSHAKE_OFF - AVE_CHANNEL_OFF || - !ave->hello[3] || ave->hello[3] > 4 * SZ_1M) - return -EPROTO; - ave->channel_bytes = ave->hello[1]; - ret = ave_alloc_buffer(ave, &ave->heap, PAGE_ALIGN(ave->hello[3])); - if (ret) - return ret; - writel(1, ave->mailbox + 0x10); - ticks = readq(ave->coproc + 0x378000); - frequency = readl(ave->coproc + 0x360020); - if (frequency && frequency < 1000000) - return -EPROTO; - ticks = div64_u64(ticks, frequency ? frequency / 1000000 : 24); - time_delta = div_u64(ktime_get_ns(), 1000) - ticks; - writel(lower_32_bits(time_delta), ave->mailbox + 0x28); - writel(upper_32_bits(time_delta), ave->mailbox + 0x2c); - ave->phase = "ipc-base"; - ave_mailbox_send(ave, lower_32_bits(ave->ipc.dma), upper_32_bits(ave->ipc.dma), ave->ipc.size, 0); - ret = ave_mailbox_receive(ave); - if (ret) - return ret; - ave->ipc_fw_base = (u64)ave->ipc_reply[1] << 32 | ave->ipc_reply[0]; - if (!ave->ipc_fw_base || !IS_ALIGNED(ave->ipc_fw_base, PAGE_SIZE) || - ave->ipc_fw_base > U64_MAX - ave->ipc.size) - return -EPROTO; - memset(ave->ipc.cpu + AVE_CHANNEL_OFF, 0, ave->channel_bytes); - memset(handshake, 0, 0x50); - memset(ave->ipc.cpu + AVE_SHARED_OFF, 0, SZ_64K); - channel_address = ave->ipc_fw_base + AVE_CHANNEL_OFF; - handshake_address = ave->ipc_fw_base + AVE_HANDSHAKE_OFF; - put_unaligned_le64(channel_address, handshake + 8); - put_unaligned_le64(ave->ipc_fw_base + AVE_SHARED_OFF, handshake + 0x10); - put_unaligned_le32(SZ_64K, handshake + 0x18); - put_unaligned_le64(ave->heap.dma, handshake + 0x1c); - put_unaligned_le32(ave->hello[3], handshake + 0x24); - put_unaligned_le32(27, handshake + 0x28); - dma_wmb(); - ave->phase = "channel-offer"; - ave_mailbox_send(ave, lower_32_bits(handshake_address), upper_32_bits(handshake_address), 0, 0); - ret = ave_mailbox_receive(ave); - if (ret) - return ret; - ave->extra_bytes = ave->ipc_reply[2]; - if (ave_fw_offset(ave, (u64)ave->ipc_reply[1] << 32 | ave->ipc_reply[0], - ave->channel_bytes, &offset) || offset != AVE_CHANNEL_OFF || - ave->extra_bytes > 0x13c000) - return -EPROTO; - ave->phase = "channel-layout"; - ret = ave_create_channels(ave); - if (ret) - return ret; - ave->phase = "ready-flag"; - writel(AVE_START_MAGIC, ave->mailbox + 0x24); - ret = readl_poll_timeout(ave->mailbox + 0x24, status, !status, 50, 5000000); - if (ret) - return ret; - ave->ipc_ready = true; - ave->phase = "ready"; - return 0; -} - -static int ave_ring_send(struct apple_ave *ave, struct ave_channel *c, u64 addr, u32 len, u32 flags) -{ - __le64 *slot; - int next; - - if (c->producer < 0) - return -ENOSPC; - slot = ave->ipc.cpu + c->offset + c->producer * 64; - WRITE_ONCE(slot[1], cpu_to_le64(len)); - WRITE_ONCE(slot[2], cpu_to_le64(flags)); - dma_wmb(); - WRITE_ONCE(slot[0], cpu_to_le64(addr | (c->mode ^ 1))); - dma_wmb(); - if (c->consumer < 0) - c->consumer = c->producer; - next = (c->producer + 1) % c->slots; - c->producer = next == c->consumer ? -1 : next; - writel(BIT(c->interrupt), ave->mailbox + 0x0c); - return 0; -} - -static int ave_ring_receive(struct apple_ave *ave, struct ave_channel *c, u64 *addr, u32 *len, u32 *flags) -{ - __le64 *slot; - u64 word; - int next; - - if (c->consumer < 0) - return -EAGAIN; - slot = ave->ipc.cpu + c->offset + c->consumer * 64; - word = le64_to_cpu(READ_ONCE(slot[0])); - if ((word & 1) != c->mode) - return -EAGAIN; - dma_rmb(); - *addr = word & ~3ULL; - *len = le64_to_cpu(READ_ONCE(slot[1])); - *flags = le64_to_cpu(READ_ONCE(slot[2])); - if (c->producer < 0) - c->producer = c->consumer; - next = (c->consumer + 1) % c->slots; - c->consumer = next == c->producer ? -1 : next; - return 0; -} - -/* Root-only experimental packet transport. The packet is copied into owned - * coherent memory; timeout pins the command too and forbids reuse this boot. - * Firmware command success is established by decoding replies, not this API. - */ -static ssize_t ave_command_write(struct file *file, const char __user *data, size_t len, loff_t *pos) -{ - struct apple_ave *ave = file->private_data; - u8 *cmd = ave->ipc.cpu + AVE_COMMAND_OFF; - u64 deadline, addr; - u32 bytes, flags, offset; - bool ack = false; - int ret = 0; - - if (!capable(CAP_SYS_RAWIO)) - return -EPERM; - static_assert(AVE_COMMAND_OFF + AVE_COMMAND_BYTES <= AVE_ARENA_OFF); - if (len < 0x48 || len > AVE_COMMAND_BYTES) - return -EINVAL; - mutex_lock(&ave->lock); - if (!ave->ipc_ready || ave->command_failed) { - ret = -EBUSY; - goto out; - } - if (copy_from_user(cmd, data, len)) { - ret = -EFAULT; - goto out; - } - /* The common command timeout is {duration_us, creation_time_us}. - * Stamp it in the same Linux monotonic clock synchronized at startup; - * captured macOS absolute times cannot be replayed on this boot. - */ - put_unaligned_le64(3000000, cmd + 0x30); - put_unaligned_le64(div_u64(ktime_get_ns(), 1000), cmd + 0x38); - ave->response_bytes = 0; - ret = ave_ring_send(ave, &ave->channel[0], ave->ipc_fw_base + AVE_COMMAND_OFF, len, 0); - if (ret) - goto out; - ave->commands++; - deadline = ktime_get_ns() + 5ULL * NSEC_PER_SEC; - do { - u32 pending = readl(ave->mailbox + 0x10); - if (pending) - writel(pending, ave->mailbox + 0x10); - for (unsigned int ch = 0; ch < 2; ch++) { - ret = ave_ring_receive(ave, &ave->channel[ch], &addr, &bytes, &flags); - if (ret == -EAGAIN) - continue; - /* Channel 1 returns the original address with zero payload - * bytes (verified native AVC captures). Only channel 2 is - * copied to response, independent of submitted packet size. - */ - if (ret || (!ch && bytes != 0) || - (ch && bytes > sizeof(ave->response) - 24) || - ave_fw_offset(ave, addr, bytes, &offset)) { - ret = -EPROTO; - goto failed; - } - dev_info(ave->dev, "AVE_COMMAND_REPLY channel=%u address=%#llx bytes=%u flags=%#x\n", - ch + 1, addr, bytes, flags); - if (!ch) { - if (addr != ave->ipc_fw_base + AVE_COMMAND_OFF) { - ret = -EPROTO; - goto failed; - } - ack = true; - ave->transport_acks++; - } else { - put_unaligned_le32(ch + 1, ave->response); - put_unaligned_le32(bytes, ave->response + 4); - put_unaligned_le32(flags, ave->response + 8); - put_unaligned_le32(0, ave->response + 12); - put_unaligned_le64(addr, ave->response + 16); - memcpy(ave->response + 24, ave->ipc.cpu + offset, bytes); - ave->response_bytes = bytes + 24; - ave->replies++; - /* Reply echoes use zero flags on the observed transport. */ - ret = ave_ring_send(ave, &ave->channel[ch], addr, bytes, 0); - if (ret) - goto failed; - } - } - if (ack && ave->response_bytes) { - ret = 0; - goto out; - } - usleep_range(100, 200); - } while (ktime_get_ns() < deadline); - ret = -ETIMEDOUT; -failed: - ave->command_failed = true; -out: - mutex_unlock(&ave->lock); - return ret ?: len; -} - -static ssize_t ave_command_read(struct file *file, char __user *data, size_t len, loff_t *pos) -{ - struct apple_ave *ave = file->private_data; - ssize_t ret; - - mutex_lock(&ave->lock); - ret = simple_read_from_buffer(data, len, pos, ave->response, ave->response_bytes); - mutex_unlock(&ave->lock); - return ret; -} - -static const struct file_operations ave_command_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .write = ave_command_write, - .read = ave_command_read, -}; - -/* Drain later firmware events without publishing another command. Results - * replace the response buffer and consist of one or more 24-byte-header - * records, each followed by its declared payload. Never discard silently. - */ -static ssize_t ave_poll_write(struct file *file, const char __user *data, size_t len, loff_t *pos) -{ - struct apple_ave *ave = file->private_data; - bool requested; - u64 deadline, addr; - u32 bytes, flags, offset; - int ret; - - if (!capable(CAP_SYS_RAWIO)) - return -EPERM; - ret = kstrtobool_from_user(data, len, &requested); - if (ret || !requested) - return ret ?: -EINVAL; - mutex_lock(&ave->lock); - if (!ave->ipc_ready || ave->command_failed) { - ret = -EBUSY; - goto out; - } - ave->response_bytes = 0; - deadline = ktime_get_ns() + NSEC_PER_SEC; - do { - u32 pending = readl(ave->mailbox + 0x10); - u8 *dst; - - if (pending) - writel(pending, ave->mailbox + 0x10); - ret = ave_ring_receive(ave, &ave->channel[1], &addr, &bytes, &flags); - if (ret == -EAGAIN) { - usleep_range(100, 200); - continue; - } - if (ret || ave_fw_offset(ave, addr, bytes, &offset) || - bytes > sizeof(ave->response) - 24 || - ave->response_bytes > sizeof(ave->response) - 24 - bytes) { - ret = -EPROTO; - goto failed; - } - dst = ave->response + ave->response_bytes; - put_unaligned_le32(2, dst); - put_unaligned_le32(bytes, dst + 4); - put_unaligned_le32(flags, dst + 8); - put_unaligned_le32(0, dst + 12); - put_unaligned_le64(addr, dst + 16); - memcpy(dst + 24, ave->ipc.cpu + offset, bytes); - ave->response_bytes += 24 + bytes; - ave->replies++; - ret = ave_ring_send(ave, &ave->channel[1], addr, bytes, 0); - if (ret) - goto failed; - } while (ktime_get_ns() < deadline); - ret = 0; - goto out; -failed: - ave->command_failed = true; -out: - mutex_unlock(&ave->lock); - return ret ?: len; -} - -static const struct file_operations ave_poll_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .write = ave_poll_write, -}; - -/* Bounded staging area for image/reference/output surfaces. Userspace offsets - * start at zero here, beyond all handshake, ring and command allocations. - * The status file supplies both DART and firmware addresses for this arena. - */ -static ssize_t ave_arena_read(struct file *file, char __user *data, size_t len, loff_t *pos) -{ - struct apple_ave *ave = file->private_data; - ssize_t ret; - - mutex_lock(&ave->lock); - dma_rmb(); - ret = simple_read_from_buffer(data, len, pos, ave->ipc.cpu + AVE_ARENA_OFF, - ave->ipc.size - AVE_ARENA_OFF); - mutex_unlock(&ave->lock); - return ret; -} - -static ssize_t ave_arena_write(struct file *file, const char __user *data, size_t len, loff_t *pos) -{ - struct apple_ave *ave = file->private_data; - ssize_t ret; - - if (!capable(CAP_SYS_RAWIO)) - return -EPERM; - mutex_lock(&ave->lock); - if (!ave->ipc_ready || ave->command_failed) - ret = -EBUSY; - else - ret = simple_write_to_buffer(ave->ipc.cpu + AVE_ARENA_OFF, - ave->ipc.size - AVE_ARENA_OFF, pos, data, len); - dma_wmb(); - mutex_unlock(&ave->lock); - return ret; -} - -static loff_t ave_arena_seek(struct file *file, loff_t offset, int whence) -{ - return fixed_size_llseek(file, offset, whence, AVE_IPC_BYTES - AVE_ARENA_OFF); -} - -static const struct file_operations ave_arena_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .read = ave_arena_read, - .write = ave_arena_write, - .llseek = ave_arena_seek, -}; diff --git a/drivers/media/platform/apple/ave/apple-ave.c b/drivers/media/platform/apple/ave/apple-ave.c deleted file mode 100644 index d2cf7abadcadf..0000000000000 --- a/drivers/media/platform/apple/ave/apple-ave.c +++ /dev/null @@ -1,532 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* T8140 AVE bring-up: resources, firmware negotiation and IPC transport. - * Experimental root-only command interface. No registered encoder yet. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define AVE_IPC_BYTES (20 * SZ_1M) -#define AVE_QUEUE_BYTES SZ_64K -#define AVE_FW_IOVA 0x10000000000ULL -#define AVE_LOG_BYTES 0x24000 -#define AVE_LOG_USED 0x20200 - -static bool enable; -module_param(enable, bool, 0400); -MODULE_PARM_DESC(enable, "Opt in to the experimental AVE transport"); - -/* Read the known Uranus registers once after PM resume. The separate hello - * command pins all resources because reset/quiescence is not yet qualified. - */ -static bool snapshot; -module_param(snapshot, bool, 0400); -MODULE_PARM_DESC(snapshot, "Capture powered AVE registers once without writing them"); - -struct ave_firmware { - phys_addr_t physical; - u64 iova; - size_t size; - u32 crc; - unsigned int verified_pages; -}; - -struct ave_buffer { - void *cpu; - dma_addr_t dma; - size_t size; - unsigned int verified_pages; -}; - -struct ave_channel { - char name[64]; - u32 mode, interrupt, slots; - u32 offset; - int consumer, producer; -}; - -struct apple_ave { - struct device *dev; - struct device *power[5]; - unsigned int num_power; - struct iommu_domain *domain; - struct ave_buffer ipc; - struct ave_buffer queue; - struct ave_buffer log; - struct ave_buffer heap; - struct ave_firmware fw[2]; - bool firmware_verified; - bool start_requested; - int start_result; - struct mutex lock; - void __iomem *coproc, *mailbox; - u32 hello[4]; - u64 ipc_fw_base; - u32 ipc_reply[4], channel_bytes, extra_bytes; - const char *phase; - bool ipc_ready, command_failed; - struct ave_channel channel[2]; - u32 commands, replies, transport_acks; - u8 response[SZ_64K]; - size_t response_bytes; - struct debugfs_blob_wrapper ipc_blob; - struct debugfs_blob_wrapper log_blob; - struct debugfs_blob_wrapper config_blob; - u64 vcbar; - u32 core_control, core_status, mailbox_status, scratch[8]; - struct dentry *debugfs; -}; - -static int ave_alloc_buffer(struct apple_ave *ave, struct ave_buffer *buf, size_t size); -#include "apple-ave-ipc.h" - -/* Cold-boot hello only. A successful response still needs the remaining IPC - * negotiation. Keep the module, PM reference and DMA buffers pinned until a - * target reboot, on both success and timeout; no unproven stop/free sequence. - */ -static ssize_t ave_start_write(struct file *file, const char __user *data, - size_t len, loff_t *ppos) -{ - struct apple_ave *ave = file->private_data; - __le32 *cfg = ave->queue.cpu, *log = ave->log.cpu; - bool requested; - u32 status; - int ret; - - if (!capable(CAP_SYS_RAWIO)) - return -EPERM; - ret = kstrtobool_from_user(data, len, &requested); - if (ret || !requested) - return ret ?: -EINVAL; - mutex_lock(&ave->lock); - if (ave->start_requested) { - ret = -EALREADY; - goto out; - } - if (!ave->firmware_verified || readq(ave->coproc + 0x50000) != (AVE_FW_IOVA | 1) || - readl(ave->coproc + 0x600044) || readl(ave->mailbox + 0x10)) { - ret = -EBUSY; - goto out; - } - for (unsigned int i = 0; i < 8; i++) { - if (readl(ave->mailbox + 0x18 + i * 4)) { - ret = -EBUSY; - goto out; - } - } - /* Startup values observed with the qualified 25G76 firmware: device - * enum 29, one active unit/group and sub-ID mask 1. Firmware maps the - * full page-aligned log allocation; the ring header advertises usage. - */ - memset(cfg, 0, 0x40); - cfg[1] = cpu_to_le32(29); - cfg[2] = cpu_to_le32(1); - cfg[3] = cpu_to_le32(1); - cfg[4] = cpu_to_le32(1); - cfg[8] = cpu_to_le32(lower_32_bits(ave->log.dma)); - cfg[9] = cpu_to_le32(upper_32_bits(ave->log.dma)); - cfg[10] = cpu_to_le32(AVE_LOG_BYTES); - cfg[12] = cpu_to_le32(0x77); - /* Log header: 128 KiB ring after its 512-byte header. */ - log[0] = cpu_to_le32(AVE_LOG_USED); - log[1] = cpu_to_le32(0x200); - log[2] = cpu_to_le32(0x20000); - dma_wmb(); - __module_get(THIS_MODULE); - ave->start_requested = true; - ave->phase = "hello"; - ave->start_result = -EINPROGRESS; - dev_info(ave->dev, "AVE_START_ONCE cfg=%pad log=%pad resources_pinned_until_reboot=1\n", - &ave->queue.dma, &ave->log.dma); - writel(0x08042006, ave->mailbox + 0x18); - writel(lower_32_bits(ave->queue.dma), ave->mailbox + 0x1c); - writel(upper_32_bits(ave->queue.dma), ave->mailbox + 0x20); - /* Preserve the protected firmware mappings inherited from iBoot. */ - writel(1, ave->coproc + 0x600808); - writel(0, ave->coproc + 0x600044); - writel(0x10000, ave->coproc + 0x600400); - writel(0x10, ave->coproc + 0x600044); - ret = readl_poll_timeout(ave->mailbox + 0x10, status, status & 1, 50, 5000000); - ave->start_result = ret; - ave->core_control = readl(ave->coproc + 0x600044); - ave->core_status = readl(ave->coproc + 0x600048); - ave->mailbox_status = status; - for (unsigned int i = 0; i < 8; i++) - ave->scratch[i] = readl(ave->mailbox + 0x18 + i * 4); - for (unsigned int i = 0; i < 4; i++) - ave->hello[i] = ave->scratch[i]; - dma_rmb(); - dev_info(ave->dev, "AVE_HELLO_RESULT result=%d status=%#x words=%#x,%#x,%#x,%#x core=%#x\n", - ret, status, ave->hello[0], ave->hello[1], ave->hello[2], ave->hello[3], ave->core_status); - if (!ret) - ret = ave_negotiate_ipc(ave); - ave->start_result = ret; - dev_info(ave->dev, "AVE_IPC_RESULT result=%d phase=%s ready=%u\n", - ret, ave->phase, ave->ipc_ready); -out: - mutex_unlock(&ave->lock); - return ret ?: len; -} - -static const struct file_operations ave_start_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .write = ave_start_write, -}; - -static int ave_verify_firmware(struct apple_ave *ave) -{ - static const char * const crc_names[] = { - "aurora,ave-text-crc32", "aurora,ave-data-crc32", - }; - static const size_t sizes[] = { 0x14c000, 0x130000 }; - struct device_node *chosen, *node; - unsigned int i; - int ret = 0; - - if (of_count_phandle_with_args(ave->dev->of_node, "memory-region", NULL) != 2) - return dev_err_probe(ave->dev, -EINVAL, "Expected two live AVE firmware reservations\n"); - chosen = of_find_node_by_path("/chosen"); - if (!chosen) - return -ENOENT; - for (i = 0; i < ARRAY_SIZE(ave->fw); i++) { - struct ave_firmware *fw = &ave->fw[i]; - struct resource res; - const char *crc_string; - u32 maps[5], expected_crc; - void *cpu; - size_t offset; - - node = of_parse_phandle(ave->dev->of_node, "memory-region", i); - ret = of_address_to_resource(node, 0, &res); - if (!ret) - ret = of_property_read_u32_array(node, "iommu-addresses", maps, ARRAY_SIZE(maps)); - if (!ret && (!of_property_read_bool(node, "no-map") || - !of_device_is_compatible(node, "apple,asc-mem"))) - ret = -EINVAL; - of_node_put(node); - if (ret) - break; - fw->physical = res.start; - fw->size = resource_size(&res); - fw->iova = (u64)maps[1] << 32 | maps[2]; - if (fw->size != sizes[i] || !IS_ALIGNED(fw->physical, PAGE_SIZE) || - maps[0] != ave->dev->of_node->phandle || maps[3] || maps[4] != sizes[i] || - fw->iova != AVE_FW_IOVA + (i ? sizes[0] : 0)) { - ret = -EINVAL; - break; - } - /* The IOMMU core owns these translated reservations across module - * unloads. DMA allocation must never reuse their IOVAs. - */ - for (offset = 0; offset < fw->size; offset += PAGE_SIZE) { - if (iommu_iova_to_phys(ave->domain, fw->iova + offset) != fw->physical + offset) { - dev_err(ave->dev, "Firmware %u mapping mismatch at %#zx\n", i, offset); - ret = -EIO; - break; - } - fw->verified_pages++; - } - if (ret) - break; - ret = of_property_read_string(chosen, crc_names[i], &crc_string); - if (!ret) - ret = kstrtou32(crc_string, 16, &expected_crc); - if (ret) - break; - cpu = memremap(fw->physical, fw->size, MEMREMAP_WB); - if (!cpu) { - ret = -ENOMEM; - break; - } - fw->crc = crc32_le(~0U, cpu, fw->size) ^ ~0U; - memunmap(cpu); - if (fw->crc != expected_crc) { - dev_err(ave->dev, "Firmware %u changed since the boot preflight\n", i); - ret = -EILSEQ; - break; - } - /* Only the two inspected 9003.78.0 text images are supported. - * CRCs detect build drift; they are not firmware authentication. - */ - if (!i && fw->crc != 0x2ce7359e && fw->crc != 0x69c54454) { - dev_err(ave->dev, "Unsupported AVE firmware text CRC %#x\n", fw->crc); - ret = -ENODEV; - break; - } - } - of_node_put(chosen); - if (!ret) - ave->firmware_verified = true; - return ret; -} - -/* Check the real IOMMU page table against the CPU mapping of each allocation. - * This proves mappings, not that firmware has performed a DMA transaction. - */ -static int ave_verify_mapping(struct apple_ave *ave, struct ave_buffer *buf) -{ - size_t offset; - - if (!IS_ALIGNED(buf->dma, PAGE_SIZE) || !IS_ALIGNED(buf->size, PAGE_SIZE)) - return -EINVAL; - for (offset = 0; offset < buf->size; offset += PAGE_SIZE) { - void *cpu = buf->cpu + offset; - struct page *page = is_vmalloc_addr(cpu) ? vmalloc_to_page(cpu) : virt_to_page(cpu); - phys_addr_t physical = iommu_iova_to_phys(ave->domain, buf->dma + offset); - - if (!page || physical != page_to_phys(page)) { - dev_err(ave->dev, "DMA mapping mismatch at offset %#zx\n", offset); - return -EIO; - } - buf->verified_pages++; - } - return 0; -} - -static int ave_alloc_buffer(struct apple_ave *ave, struct ave_buffer *buf, size_t size) -{ - buf->size = size; - buf->cpu = dma_alloc_coherent(ave->dev, size, &buf->dma, GFP_KERNEL); - if (!buf->cpu) - return -ENOMEM; - memset(buf->cpu, 0, size); - return ave_verify_mapping(ave, buf); -} - -static void ave_free_buffer(struct apple_ave *ave, struct ave_buffer *buf) -{ - if (buf->cpu) - dma_free_coherent(ave->dev, buf->size, buf->cpu, buf->dma); - buf->cpu = NULL; -} - -static void ave_power_off(struct apple_ave *ave) -{ - while (ave->num_power) { - struct device *pd = ave->power[--ave->num_power]; - - pm_runtime_put_sync(pd); - dev_pm_domain_detach(pd, true); - } -} - -static int ave_power_on(struct apple_ave *ave) -{ - static const char * const names[] = { "dma", "pipe4", "pipe5", "me0", "me1" }; - int ret; - - for (unsigned int i = 0; i < ARRAY_SIZE(names); i++) { - struct device *pd = dev_pm_domain_attach_by_name(ave->dev, names[i]); - - if (IS_ERR(pd)) - return PTR_ERR(pd); - if (!pd) - return -ENODEV; - ret = pm_runtime_resume_and_get(pd); - if (ret < 0) { - dev_pm_domain_detach(pd, true); - return ret; - } - ave->power[ave->num_power++] = pd; - } - return 0; -} - -static int ave_status_show(struct seq_file *s, void *unused) -{ - struct apple_ave *ave = s->private; - - mutex_lock(&ave->lock); - seq_printf(s, "state=%s\nfirmware_start_requested=%u\nstart_result=%d\n", - ave->start_requested ? (ave->start_result ? "startup-failed" : "ipc-ready") : - "firmware-mapped", ave->start_requested, ave->start_result); - seq_printf(s, "hardware_dma_tested=%u\nencoder_registered=0\nphase=%s\n", - ave->ipc_ready, ave->phase ?: "not-started"); - seq_printf(s, "ipc_ready=%u\nipc_fw_base=%#llx\nchannel_bytes=%#x\nextra_bytes=%#x\n", - ave->ipc_ready, ave->ipc_fw_base, ave->channel_bytes, ave->extra_bytes); - seq_printf(s, "commands=%u\nreplies=%u\ntransport_acks=%u\ncommand_failed=%u\nresponse_bytes=%zu\n", - ave->commands, ave->replies, ave->transport_acks, ave->command_failed, ave->response_bytes); - seq_printf(s, "arena_bytes=%zu\narena_ipc_offset=%#x\narena_dma=%#llx\narena_fw=%#llx\n", - ave->ipc.size - AVE_ARENA_OFF, AVE_ARENA_OFF, - (u64)ave->ipc.dma + AVE_ARENA_OFF, ave->ipc_fw_base + AVE_ARENA_OFF); - for (unsigned int i = 0; i < 2; i++) { - struct ave_channel *c = &ave->channel[i]; - seq_printf(s, "channel%u_name=%s\nchannel%u_mode=%u\nchannel%u_interrupt=%u\n" - "channel%u_slots=%u\nchannel%u_offset=%#x\nchannel%u_consumer=%d\nchannel%u_producer=%d\n", - i, c->name, i, c->mode, i, c->interrupt, i, c->slots, - i, c->offset, i, c->consumer, i, c->producer); - } - seq_printf(s, "page_size=%lu\n", PAGE_SIZE); - seq_printf(s, "powered_domains=%u\n", ave->num_power); - seq_printf(s, "ipc_bytes=%zu\nipc_dma=%pad\nipc_verified_pages=%u\n", - ave->ipc.size, &ave->ipc.dma, ave->ipc.verified_pages); - seq_printf(s, "queue_bytes=%zu\nqueue_dma=%pad\nqueue_verified_pages=%u\n", - ave->queue.size, &ave->queue.dma, ave->queue.verified_pages); - seq_printf(s, "log_bytes=%zu\nlog_dma=%pad\nlog_verified_pages=%u\n", - ave->log.size, &ave->log.dma, ave->log.verified_pages); - seq_printf(s, "heap_bytes=%zu\nheap_dma=%pad\nheap_verified_pages=%u\n", - ave->heap.size, &ave->heap.dma, ave->heap.verified_pages); - for (unsigned int i = 0; i < ARRAY_SIZE(ave->fw); i++) - seq_printf(s, "fw%u_physical=%pa\nfw%u_iova=%#llx\nfw%u_bytes=%zu\n" - "fw%u_verified_pages=%u\nfw%u_crc32=%08x\n", - i, &ave->fw[i].physical, i, ave->fw[i].iova, i, ave->fw[i].size, - i, ave->fw[i].verified_pages, i, ave->fw[i].crc); - seq_printf(s, "register_snapshot=%u\n", snapshot); - if (snapshot) { - seq_printf(s, "vcbar=%#llx\ncore_control=%#x\ncore_status=%#x\nmailbox_status=%#x\n", - ave->vcbar, ave->core_control, ave->core_status, ave->mailbox_status); - for (unsigned int i = 0; i < ARRAY_SIZE(ave->scratch); i++) - seq_printf(s, "scratch%u=%#x\n", i, ave->scratch[i]); - } - seq_printf(s, "resources_pinned_until_reboot=%u\n", ave->start_requested); - mutex_unlock(&ave->lock); - return 0; -} -DEFINE_SHOW_ATTRIBUTE(ave_status); - -static int apple_ave_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct apple_ave *ave; - void __iomem *regs, *mailbox, *coproc; - int ret; - - if (!enable) - return -ENODEV; - if (!of_machine_is_compatible("apple,j700") || PAGE_SIZE != SZ_16K) - return -ENODEV; - ave = devm_kzalloc(dev, sizeof(*ave), GFP_KERNEL); - if (!ave) - return -ENOMEM; - ave->dev = dev; - mutex_init(&ave->lock); - /* Claim only dedicated AVE ranges, never the shared PMGR/SRAM windows. */ - regs = devm_platform_ioremap_resource_byname(pdev, "codec"); - if (IS_ERR(regs)) - return PTR_ERR(regs); - mailbox = devm_platform_ioremap_resource_byname(pdev, "mailbox"); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - coproc = devm_platform_ioremap_resource_byname(pdev, "coproc"); - if (IS_ERR(coproc)) - return PTR_ERR(coproc); - ave->coproc = coproc; - ave->mailbox = mailbox; - ave->domain = iommu_get_domain_for_dev(dev); - if (!ave->domain || !(ave->domain->type & __IOMMU_DOMAIN_PAGING)) - return dev_err_probe(dev, -ENODEV, "A translated IOMMU domain is required\n"); - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); - if (ret) - return ret; - ret = ave_power_on(ave); - if (ret) - goto release_power; - ret = devm_pm_runtime_enable(dev); - if (ret) - goto release_power; - ret = pm_runtime_resume_and_get(dev); - if (ret < 0) - goto release_power; - ret = ave_alloc_buffer(ave, &ave->ipc, AVE_IPC_BYTES); - if (ret) - goto free_buffers; - ret = ave_alloc_buffer(ave, &ave->queue, AVE_QUEUE_BYTES); - if (ret) - goto free_buffers; - ret = ave_alloc_buffer(ave, &ave->log, AVE_LOG_BYTES); - if (ret) - goto free_buffers; - ret = ave_verify_firmware(ave); - if (ret) - goto free_buffers; - if (snapshot) { - ave->vcbar = readq(coproc + 0x50000); - ave->core_control = readl(coproc + 0x600044); - ave->core_status = readl(coproc + 0x600048); - ave->mailbox_status = readl(mailbox + 0x10); - for (unsigned int i = 0; i < ARRAY_SIZE(ave->scratch); i++) - ave->scratch[i] = readl(mailbox + 0x18 + i * 4); - } - platform_set_drvdata(pdev, ave); - ave->debugfs = debugfs_create_dir(dev_name(dev), NULL); - debugfs_create_file("ave-status", 0400, ave->debugfs, ave, &ave_status_fops); - debugfs_create_file("ave-start-once", 0200, ave->debugfs, ave, &ave_start_fops); - debugfs_create_file("ave-command", 0600, ave->debugfs, ave, &ave_command_fops); - debugfs_create_file("ave-poll", 0200, ave->debugfs, ave, &ave_poll_fops); - debugfs_create_file("ave-arena", 0600, ave->debugfs, ave, &ave_arena_fops); - ave->ipc_blob.data = ave->ipc.cpu; - ave->ipc_blob.size = ave->ipc.size; - debugfs_create_blob("ave-ipc-memory", 0400, ave->debugfs, &ave->ipc_blob); - ave->log_blob.data = ave->log.cpu; - ave->log_blob.size = ave->log.size; - debugfs_create_blob("ave-firmware-log", 0400, ave->debugfs, &ave->log_blob); - ave->config_blob.data = ave->queue.cpu; - ave->config_blob.size = 0x40; - debugfs_create_blob("ave-boot-config", 0400, ave->debugfs, &ave->config_blob); - dev_info(dev, "AVE_DMA_READY ipc=%zu queue=%zu verified_pages=%u firmware_start=0\n", - ave->ipc.size, ave->queue.size, - ave->ipc.verified_pages + ave->queue.verified_pages); - dev_info(dev, "AVE_FIRMWARE_MAPPED pages=%u contents_match_boot=1 register_snapshot=%u\n", - ave->fw[0].verified_pages + ave->fw[1].verified_pages, snapshot); - return 0; - -free_buffers: - ave_free_buffer(ave, &ave->heap); - ave_free_buffer(ave, &ave->log); - ave_free_buffer(ave, &ave->queue); - ave_free_buffer(ave, &ave->ipc); - pm_runtime_put_sync(dev); -release_power: - ave_power_off(ave); - return dev_err_probe(dev, ret, "AVE DMA allocation/verification failed\n"); -} - -static void apple_ave_remove(struct platform_device *pdev) -{ - struct apple_ave *ave = platform_get_drvdata(pdev); - - debugfs_remove_recursive(ave->debugfs); - ave_free_buffer(ave, &ave->heap); - ave_free_buffer(ave, &ave->log); - ave_free_buffer(ave, &ave->queue); - ave_free_buffer(ave, &ave->ipc); - pm_runtime_put_sync(ave->dev); - ave_power_off(ave); - dev_info(ave->dev, "AVE_DMA_RELEASED\n"); -} - -static const struct of_device_id apple_ave_match[] = { - { .compatible = "apple,t8140-ave" }, - { } -}; -MODULE_DEVICE_TABLE(of, apple_ave_match); - -static struct platform_driver apple_ave_driver = { - .probe = apple_ave_probe, - .remove = apple_ave_remove, - .driver = { - .name = "apple-ave", - .of_match_table = apple_ave_match, - .suppress_bind_attrs = true, - }, -}; -module_platform_driver(apple_ave_driver); -MODULE_DESCRIPTION("Apple T8140 experimental video encoder transport"); -MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/ave/ave-cmd.c b/drivers/media/platform/apple/ave/ave-cmd.c new file mode 100644 index 0000000000000..6948bc60496a6 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-cmd.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder: firmware commands + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include +#include + +#include + +#include "ave.h" +#include "ave-regs.h" + +static void ave_cmd_header(struct ave_dev *ave, u8 *cmd, enum ave_opcode op, + u32 client, enum ave_codec codec) +{ + put_unaligned_le32(op, cmd + AVE_CMD_OPCODE); + put_unaligned_le64(++ave->next_token, cmd + AVE_CMD_TOKEN); + put_unaligned_le64(client, cmd + AVE_CMD_CLIENT); + put_unaligned_le32(1, cmd + AVE_CMD_UNIT); + put_unaligned_le32(codec, cmd + AVE_CMD_CODEC); + put_unaligned_le32(op, cmd + AVE_CMD_OPCODE_ECHO); +} + +/* Firmware-wide configuration, sent once after every firmware boot. */ +int ave_cmd_config(struct ave_dev *ave) +{ + u8 cmd[AVE_CONFIG_BYTES] = {}; + + put_unaligned_le32(AVE_OP_CONFIG, cmd + AVE_CMD_OPCODE); + put_unaligned_le64(++ave->next_token, cmd + AVE_CMD_TOKEN); + put_unaligned_le32(U32_MAX, cmd + AVE_CONFIG_ALL_UNITS); + cmd[AVE_CONFIG_OPT_A] = 1; + cmd[AVE_CONFIG_OPT_B] = 1; + return ave_ipc_call(ave, cmd, sizeof(cmd)); +} + +/* Firmware-wide stop, sent after every session has closed. */ +int ave_cmd_halt(struct ave_dev *ave) +{ + u8 cmd[AVE_CMD_HDR_BYTES] = {}; + + put_unaligned_le32(AVE_OP_HALT, cmd + AVE_CMD_OPCODE); + put_unaligned_le64(++ave->next_token, cmd + AVE_CMD_TOKEN); + put_unaligned_le32(AVE_HALT_ALL, cmd + AVE_CMD_OPCODE_ECHO); + return ave_ipc_call(ave, cmd, sizeof(cmd)); +} + +static int ave_cmd_simple(struct ave_ctx *ctx, enum ave_opcode op, enum ave_codec codec, + u32 arg0, u32 arg2) +{ + u8 cmd[AVE_CMD_HDR_BYTES] = {}; + + ave_cmd_header(ctx->ave, cmd, op, ctx->client, codec); + put_unaligned_le32(arg0, cmd + AVE_CMD_ARG0); + put_unaligned_le32(AVE_SESSION_ARG1, cmd + AVE_CMD_ARG1); + put_unaligned_le32(arg2, cmd + AVE_CMD_ARG2); + return ave_ipc_call(ctx->ave, cmd, sizeof(cmd)); +} + +int ave_cmd_open(struct ave_ctx *ctx, enum ave_codec codec) +{ + struct ave_dev *ave = ctx->ave; + int ret; + + ctx->client = ++ave->next_client; + ret = ave_cmd_simple(ctx, AVE_OP_OPEN, codec, AVE_SESSION_ARG0, AVE_OPEN_ARG2); + if (!ret) + ctx->session_open = true; + return ret; +} + +/* + * Session parameters. The Start layout (resolution, formats, rate control, + * GOP, the per-session buffer table) is not described yet, so sessions cannot + * start; everything around it is in place. + */ +int ave_cmd_start(struct ave_ctx *ctx, enum ave_codec codec) +{ + return -EOPNOTSUPP; +} + +int ave_cmd_process(struct ave_ctx *ctx, enum ave_codec codec, + struct vb2_v4l2_buffer *src, struct vb2_v4l2_buffer *dst, + u32 *bytesused, bool *keyframe) +{ + return -EOPNOTSUPP; +} + +/* Flush, stop and close the session. */ +int ave_cmd_finish(struct ave_ctx *ctx, enum ave_codec codec) +{ + u32 tail = codec == AVE_CODEC_HEVC ? AVE_FINISH_ARG2_HEVC : AVE_FINISH_ARG2_H264; + int ret, err; + + if (!ctx->session_open) + return 0; + + ret = ave_cmd_simple(ctx, AVE_OP_COMPLETE, codec, 0, tail); + err = ave_cmd_simple(ctx, AVE_OP_STOP, codec, AVE_SESSION_ARG0, tail); + ret = ret ?: err; + err = ave_cmd_simple(ctx, AVE_OP_CLOSE, codec, AVE_SESSION_ARG0, tail); + ctx->session_open = false; + return ret ?: err; +} diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c new file mode 100644 index 0000000000000..edaaf29aa6739 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder + * + * The AVE block runs an encoder firmware on its own coprocessor, which the + * bootloader loads but does not start. This driver starts it on first use, + * talks to it over two shared-memory rings and exposes it as a V4L2 + * stateful encoder. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include +#include +#include +#include +#include + +#include "ave.h" + +#define AVE_AUTOSUSPEND_MS 2000 + +static const char * const ave_pd_names[] = { "dma", "pipe4", "pipe5", "me0", "me1" }; + +static int ave_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct dev_pm_domain_attach_data pd_data = { + .pd_names = ave_pd_names, + .num_pd_names = ARRAY_SIZE(ave_pd_names), + .pd_flags = PD_FLAG_DEV_LINK_ON, + }; + struct ave_dev *ave; + int ret; + + ave = devm_kzalloc(dev, sizeof(*ave), GFP_KERNEL); + if (!ave) + return -ENOMEM; + ave->dev = dev; + mutex_init(&ave->hw_lock); + platform_set_drvdata(pdev, ave); + + ave->codec = devm_platform_ioremap_resource_byname(pdev, "codec"); + if (IS_ERR(ave->codec)) + return PTR_ERR(ave->codec); + ave->mbox = devm_platform_ioremap_resource_byname(pdev, "mailbox"); + if (IS_ERR(ave->mbox)) + return PTR_ERR(ave->mbox); + ave->coproc = devm_platform_ioremap_resource_byname(pdev, "coproc"); + if (IS_ERR(ave->coproc)) + return PTR_ERR(ave->coproc); + + ave->domain = iommu_get_domain_for_dev(dev); + if (!ave->domain || !(ave->domain->type & __IOMMU_DOMAIN_PAGING)) + return dev_err_probe(dev, -ENODEV, "needs a translating IOMMU\n"); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + if (ret) + return ret; + + /* The encoder pipes, the DMA block and the motion engines. */ + ret = devm_pm_domain_attach_list(dev, &pd_data, &ave->pd_list); + if (ret < 0) + return dev_err_probe(dev, ret, "cannot attach power domains\n"); + + pm_runtime_set_autosuspend_delay(dev, AVE_AUTOSUSPEND_MS); + pm_runtime_use_autosuspend(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; + + ret = ave_fw_probe(ave); + if (ret) + return ret; + + ret = ave_v4l2_register(ave); + if (ret) { + ave_fw_remove(ave); + return ret; + } + return 0; +} + +static void ave_remove(struct platform_device *pdev) +{ + struct ave_dev *ave = platform_get_drvdata(pdev); + + ave_v4l2_unregister(ave); + + /* + * Make sure the coprocessor really loses power before its memory is + * freed: drop to runtime suspend synchronously and check. + */ + pm_runtime_dont_use_autosuspend(ave->dev); + if (pm_runtime_suspended(ave->dev) || pm_runtime_status_suspended(ave->dev)) { + mutex_lock(&ave->hw_lock); + ave_fw_lost(ave); + mutex_unlock(&ave->hw_lock); + ave_fw_remove(ave); + } else { + /* The firmware still owns its buffers; leave them allocated. */ + dev_warn(ave->dev, "firmware still running, keeping its memory\n"); + } +} + +static int ave_runtime_suspend(struct device *dev) +{ + struct ave_dev *ave = dev_get_drvdata(dev); + int ret; + + mutex_lock(&ave->hw_lock); + ret = ave_fw_shutdown(ave); + mutex_unlock(&ave->hw_lock); + return ret; +} + +static int ave_runtime_resume(struct device *dev) +{ + return 0; +} + +static int ave_suspend(struct device *dev) +{ + struct ave_dev *ave = dev_get_drvdata(dev); + + /* Let a running frame finish; the m2m core holds back new jobs. */ + v4l2_m2m_suspend(ave->m2m_dev); + + mutex_lock(&ave->hw_lock); + if (ave->fw_state != AVE_FW_OFF && ave_fw_shutdown(ave)) + dev_dbg(dev, "firmware will restart after resume\n"); + mutex_unlock(&ave->hw_lock); + return 0; +} + +static int ave_resume(struct device *dev) +{ + struct ave_dev *ave = dev_get_drvdata(dev); + + /* + * If the power domains went down with the system, the coprocessor is + * back in reset: open sessions fail their next frame and a new session + * boots the firmware again from its saved state. + */ + mutex_lock(&ave->hw_lock); + ave_fw_resume(ave); + mutex_unlock(&ave->hw_lock); + + v4l2_m2m_resume(ave->m2m_dev); + return 0; +} + +static const struct dev_pm_ops ave_pm_ops = { + SYSTEM_SLEEP_PM_OPS(ave_suspend, ave_resume) + RUNTIME_PM_OPS(ave_runtime_suspend, ave_runtime_resume, NULL) +}; + +static const struct of_device_id ave_of_match[] = { + { .compatible = "apple,t8140-ave" }, + { } +}; +MODULE_DEVICE_TABLE(of, ave_of_match); + +static struct platform_driver ave_driver = { + .probe = ave_probe, + .remove = ave_remove, + .driver = { + .name = "apple-ave", + .of_match_table = ave_of_match, + .pm = pm_ptr(&ave_pm_ops), + /* The firmware cannot be stopped yet, so neither can the driver. */ + .suppress_bind_attrs = true, + }, +}; +module_platform_driver(ave_driver); + +MODULE_DESCRIPTION("Apple AVE video encoder"); +MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c new file mode 100644 index 0000000000000..fa2df964d16f4 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder: firmware lifecycle + * + * The bootloader leaves the coprocessor firmware in two reserved, DART-mapped + * segments: TEXT and DATA. Running the firmware changes DATA, so the driver + * keeps a copy of DATA taken before the first boot and puts it back before + * every later boot. That lets the coprocessor lose power (runtime PM, system + * sleep) and start again from the same state as the first time. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ave.h" +#include "ave-regs.h" + +/* + * Firmware builds whose command layouts this driver implements, by the CRC32 + * of their TEXT segment. The command formats change between builds. + */ +static const u32 ave_known_text_crc[] = { + 0x2ce7359e, /* AppleAVE2FW 9003.78.0 */ + 0x69c54454, /* AppleAVE2FW 9003.78.0 (macOS 26.0.x) */ +}; + +static int ave_fw_chosen_crc(struct ave_dev *ave, const char *name, u32 *crc) +{ + struct device_node *chosen = of_find_node_by_path("/chosen"); + const char *str; + int ret; + + if (!chosen) + return -ENOENT; + ret = of_property_read_string(chosen, name, &str); + of_node_put(chosen); + if (ret) + return ret; + return kstrtou32(str, 16, crc); +} + +static int ave_fw_segment(struct ave_dev *ave, unsigned int index, + struct ave_fw_segment *seg) +{ + struct device_node *np; + struct resource res; + const __be32 *prop; + int len, ret; + + np = of_parse_phandle(ave->dev->of_node, "memory-region", index); + if (!np) + return -ENOENT; + ret = of_address_to_resource(np, 0, &res); + if (ret) + goto out; + + /* iommu-addresses = <&ave iova-hi iova-lo size-hi size-lo> */ + prop = of_get_property(np, "iommu-addresses", &len); + if (!prop || len != 5 * sizeof(u32) || + be32_to_cpup(prop) != ave->dev->of_node->phandle) { + ret = -EINVAL; + goto out; + } + seg->phys = res.start; + seg->size = resource_size(&res); + seg->iova = of_read_number(prop + 1, 2); + if (of_read_number(prop + 3, 2) != seg->size || + !IS_ALIGNED(seg->phys, SZ_16K) || !IS_ALIGNED(seg->iova, SZ_16K)) + ret = -EINVAL; +out: + of_node_put(np); + return ret; +} + +static int ave_fw_check_mapping(struct ave_dev *ave, const struct ave_fw_segment *seg) +{ + for (size_t off = 0; off < seg->size; off += PAGE_SIZE) + if (iommu_iova_to_phys(ave->domain, seg->iova + off) != seg->phys + off) + return -EIO; + return 0; +} + +static u32 ave_fw_crc(const struct ave_fw_segment *seg, void **copy) +{ + void *va = memremap(seg->phys, seg->size, MEMREMAP_WB); + u32 crc; + + if (!va) + return 0; + crc = crc32_le(~0U, va, seg->size) ^ ~0U; + if (copy) + memcpy(*copy, va, seg->size); + memunmap(va); + return crc; +} + +/* + * Find the firmware, check that it is a build we speak to and that DATA is + * still as the bootloader left it, and keep a copy of DATA for restarts. + */ +int ave_fw_probe(struct ave_dev *ave) +{ + struct device *dev = ave->dev; + u32 text_crc, data_crc, want; + bool known = false; + int ret; + + if (of_count_phandle_with_args(dev->of_node, "memory-region", NULL) != 2) + return dev_err_probe(dev, -ENODEV, "no preloaded firmware\n"); + + ret = ave_fw_segment(ave, 0, &ave->text); + if (!ret) + ret = ave_fw_segment(ave, 1, &ave->data); + if (ret) + return dev_err_probe(dev, ret, "bad firmware reservation\n"); + + if (ave_fw_check_mapping(ave, &ave->text) || ave_fw_check_mapping(ave, &ave->data)) + return dev_err_probe(dev, -EIO, "firmware is not mapped where it was loaded\n"); + + text_crc = ave_fw_crc(&ave->text, NULL); + for (unsigned int i = 0; i < ARRAY_SIZE(ave_known_text_crc); i++) + known |= text_crc == ave_known_text_crc[i]; + if (!known) + return dev_err_probe(dev, -ENODEV, "unsupported firmware (text CRC %08x)\n", + text_crc); + + ave->data_pristine = vmalloc(ave->data.size); + if (!ave->data_pristine) + return -ENOMEM; + data_crc = ave_fw_crc(&ave->data, &ave->data_pristine); + + /* + * The loader records the DATA checksum before anything runs. If it + * differs, the firmware already ran and there is no clean copy left. + */ + ret = ave_fw_chosen_crc(ave, "aurora,ave-data-crc32", &want); + if (ret || data_crc != want) { + vfree(ave->data_pristine); + ave->data_pristine = NULL; + return dev_err_probe(dev, -ENODEV, + "firmware data is not in its boot state\n"); + } + + ave->fw_state = AVE_FW_OFF; + return 0; +} + +static void ave_fw_restore_data(struct ave_dev *ave) +{ + void *va; + + if (!ave->data_dirty) + return; + va = memremap(ave->data.phys, ave->data.size, MEMREMAP_WC); + if (WARN_ON(!va)) + return; + memcpy(va, ave->data_pristine, ave->data.size); + wmb(); + memunmap(va); + ave->data_dirty = false; +} + +static void ave_free(struct ave_dev *ave, struct ave_buf *b) +{ + if (b->cpu) + dma_free_coherent(ave->dev, b->size, b->cpu, b->dma); + b->cpu = NULL; +} + +static int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size) +{ + if (b->cpu) + return 0; + b->size = size; + b->cpu = dma_alloc_coherent(ave->dev, size, &b->dma, GFP_KERNEL); + if (!b->cpu) + return -ENOMEM; + if (!IS_ALIGNED(b->dma, SZ_16K)) { + ave_free(ave, b); + return -EINVAL; + } + return 0; +} + +void ave_fw_remove(struct ave_dev *ave) +{ + ave_free(ave, &ave->heap); + ave_free(ave, &ave->log); + ave_free(ave, &ave->boot_cfg); + ave_free(ave, &ave->ipc); + vfree(ave->data_pristine); + ave->data_pristine = NULL; +} + +static bool ave_fw_idle(struct ave_dev *ave) +{ + if (readq(ave->coproc + AVE_COPROC_VCBAR) != (ave->text.iova | AVE_COPROC_VCBAR_VALID)) + return false; + if (readl(ave->coproc + AVE_COPROC_CTRL) || readl(ave->mbox + AVE_MBOX_STATUS)) + return false; + for (unsigned int i = 0; i < AVE_MBOX_SCRATCH_COUNT; i++) + if (readl(ave->mbox + AVE_MBOX_SCRATCH(i))) + return false; + return true; +} + +/* Boot the coprocessor and bring up its command rings. Caller holds hw_lock. */ +static int ave_fw_boot(struct ave_dev *ave) +{ + __le32 *cfg, *log; + u32 status, hello[4]; + int ret; + + ret = ave_alloc(ave, &ave->ipc, AVE_IPC_BYTES); + if (!ret) + ret = ave_alloc(ave, &ave->boot_cfg, SZ_16K); + if (!ret) + ret = ave_alloc(ave, &ave->log, AVE_LOG_BYTES); + if (ret) + return ret; + + /* + * The bootloader leaves the coprocessor stopped with its code base + * set. Do not start it from any other state. + */ + if (!ave_fw_idle(ave)) { + dev_err(ave->dev, "coprocessor is not in its reset state (vcbar %#llx ctrl %#x)\n", + readq(ave->coproc + AVE_COPROC_VCBAR), + readl(ave->coproc + AVE_COPROC_CTRL)); + return -EIO; + } + + ave_fw_restore_data(ave); + + cfg = ave->boot_cfg.cpu; + log = ave->log.cpu; + memset(cfg, 0, AVE_BOOT_CFG_BYTES); + memset(log, 0, ave->log.size); + cfg[AVE_BOOT_CFG_DEVICE] = cpu_to_le32(AVE_BOOT_CFG_DEVICE_T8140); + cfg[AVE_BOOT_CFG_UNITS] = cpu_to_le32(1); + cfg[AVE_BOOT_CFG_GROUPS] = cpu_to_le32(1); + cfg[AVE_BOOT_CFG_SUBID_MASK] = cpu_to_le32(1); + cfg[AVE_BOOT_CFG_LOG_LO] = cpu_to_le32(lower_32_bits(ave->log.dma)); + cfg[AVE_BOOT_CFG_LOG_HI] = cpu_to_le32(upper_32_bits(ave->log.dma)); + cfg[AVE_BOOT_CFG_LOG_BYTES] = cpu_to_le32(AVE_LOG_BYTES); + cfg[AVE_BOOT_CFG_OPTIONS] = cpu_to_le32(AVE_BOOT_CFG_OPTIONS_VALUE); + log[0] = cpu_to_le32(AVE_LOG_HDR_USED); + log[1] = cpu_to_le32(AVE_LOG_HDR_BYTES); + log[2] = cpu_to_le32(AVE_LOG_RING_BYTES); + dma_wmb(); + + writel(AVE_BOOT_MAGIC, ave->mbox + AVE_MBOX_SCRATCH(0)); + writel(lower_32_bits(ave->boot_cfg.dma), ave->mbox + AVE_MBOX_SCRATCH(1)); + writel(upper_32_bits(ave->boot_cfg.dma), ave->mbox + AVE_MBOX_SCRATCH(2)); + + ave->data_dirty = true; + writel(1, ave->coproc + AVE_COPROC_KEEP_MAPPINGS); + writel(0, ave->coproc + AVE_COPROC_CTRL); + writel(AVE_COPROC_BOOT_ARG_VALUE, ave->coproc + AVE_COPROC_BOOT_ARG); + writel(AVE_COPROC_CTRL_RUN, ave->coproc + AVE_COPROC_CTRL); + + ret = readl_poll_timeout(ave->mbox + AVE_MBOX_STATUS, status, + status & AVE_MBOX_STATUS_PENDING, 50, AVE_MBOX_TIMEOUT_US); + if (ret) { + dev_err(ave->dev, "firmware did not answer (status %#x)\n", + readl(ave->coproc + AVE_COPROC_STATUS)); + return ret; + } + for (unsigned int i = 0; i < 4; i++) + hello[i] = readl(ave->mbox + AVE_MBOX_SCRATCH(i)); + dma_rmb(); + + ret = ave_ipc_negotiate(ave, hello); + if (ret) { + dev_err(ave->dev, "firmware start-up failed: %d\n", ret); + return ret; + } + + ave->fw_state = AVE_FW_RUNNING; + ret = ave_cmd_config(ave); + if (ret) + return ret; + + dev_dbg(ave->dev, "firmware running, %u/%u ring slots\n", + ave->ring[0].slots, ave->ring[1].slots); + return 0; +} + +/* Caller holds hw_lock. */ +int ave_fw_ensure_running(struct ave_dev *ave) +{ + int ret; + + lockdep_assert_held(&ave->hw_lock); + switch (ave->fw_state) { + case AVE_FW_RUNNING: + return 0; + case AVE_FW_BROKEN: + /* Only a power cycle gets the coprocessor back; see runtime PM. */ + return -EIO; + case AVE_FW_OFF: + break; + } + + ret = ave_fw_boot(ave); + if (ret) + ave->fw_state = AVE_FW_BROKEN; + return ret; +} + +/* + * Stop the firmware before its power goes away. Caller holds hw_lock. + * + * The firmware is told to halt once every session is closed. Stopping the + * coprocessor core itself and checking that the fabric is idle are not + * described yet, so until then a running firmware keeps its power (runtime + * suspend fails with -EBUSY). Across system sleep the domains go down anyway; + * ave_fw_resume() notices and the next session boots the firmware again from + * the saved DATA copy. + */ +int ave_fw_shutdown(struct ave_dev *ave) +{ + lockdep_assert_held(&ave->hw_lock); + if (ave->fw_state == AVE_FW_OFF) + return 0; + return -EBUSY; +} + +/* The coprocessor lost power behind the firmware's back. Caller holds hw_lock. */ +void ave_fw_lost(struct ave_dev *ave) +{ + lockdep_assert_held(&ave->hw_lock); + ave->fw_state = AVE_FW_OFF; + memset(ave->ring, 0, sizeof(ave->ring)); + ave->ipc_fw_base = 0; +} + +/* After system resume: did the coprocessor keep running? Caller holds hw_lock. */ +void ave_fw_resume(struct ave_dev *ave) +{ + lockdep_assert_held(&ave->hw_lock); + if (ave->fw_state == AVE_FW_OFF) + return; + /* A running firmware keeps the device runtime active, so this is powered. */ + if (readl(ave->coproc + AVE_COPROC_CTRL) & AVE_COPROC_CTRL_RUN) + return; + dev_dbg(ave->dev, "coprocessor lost power during sleep\n"); + ave_fw_lost(ave); +} diff --git a/drivers/media/platform/apple/ave/ave-ipc.c b/drivers/media/platform/apple/ave/ave-ipc.c new file mode 100644 index 0000000000000..11f0aceb6fb65 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-ipc.c @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder: coprocessor mailbox and command rings + * + * The mailbox negotiation and the 64-byte ring protocol here come from + * Acelogic's J700 AVE transport (the first Linux code to drive this + * firmware); this file reshapes it into the driver core. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ave.h" +#include "ave-regs.h" + +static int ave_fw_offset(struct ave_dev *ave, u64 addr, size_t len, u32 *offset) +{ + u64 delta = addr - ave->ipc_fw_base; + + if (addr < ave->ipc_fw_base || delta > ave->ipc.size || + len > ave->ipc.size - delta) + return -ERANGE; + *offset = delta; + return 0; +} + +static void ave_mbox_send(struct ave_dev *ave, u32 a, u32 b, u32 c, u32 d) +{ + writel(a, ave->mbox + AVE_MBOX_SCRATCH(0)); + writel(b, ave->mbox + AVE_MBOX_SCRATCH(1)); + writel(c, ave->mbox + AVE_MBOX_SCRATCH(2)); + writel(d, ave->mbox + AVE_MBOX_SCRATCH(3)); + wmb(); + writel(1, ave->mbox + AVE_MBOX_DOORBELL); +} + +static int ave_mbox_recv(struct ave_dev *ave, u32 reply[4]) +{ + u32 status; + int ret; + + ret = readl_poll_timeout(ave->mbox + AVE_MBOX_STATUS, status, + status & AVE_MBOX_STATUS_PENDING, 50, + AVE_MBOX_TIMEOUT_US); + if (ret) + return ret; + + /* Acknowledge before reading the scratch words. */ + writel(AVE_MBOX_STATUS_PENDING, ave->mbox + AVE_MBOX_STATUS); + for (unsigned int i = 0; i < 4; i++) + reply[i] = readl(ave->mbox + AVE_MBOX_SCRATCH(i)); + dma_rmb(); + return 0; +} + +static int ave_ring_setup(struct ave_dev *ave) +{ + bool found[2] = { false, false }; + + for (unsigned int i = 0; i < 2; i++) { + u8 *desc = ave->ipc.cpu + AVE_IPC_CHANNEL_OFF + i * AVE_CHAN_DESC_STRIDE; + u32 dir = get_unaligned_le32(desc + AVE_CHAN_DESC_DIR); + u32 doorbell = get_unaligned_le32(desc + AVE_CHAN_DESC_IRQ); + u32 slots = get_unaligned_le32(desc + AVE_CHAN_DESC_SLOTS); + u64 addr = get_unaligned_le64(desc + AVE_CHAN_DESC_ADDR); + struct ave_ring *r; + unsigned int id; + u32 offset; + + if (!memchr(desc, 0, AVE_CHAN_DESC_NAME_LEN)) + return -EPROTO; + if (!strcmp(desc, "IO")) + id = 0; + else if (!strcmp(desc, "IO_T2H")) + id = 1; + else + return -EPROTO; + + if (found[id] || dir != id || !doorbell || doorbell >= 32 || !slots || + slots > ave->channel_bytes / AVE_RING_SLOT_BYTES || + !IS_ALIGNED(addr, AVE_RING_SLOT_BYTES) || + ave_fw_offset(ave, addr, (size_t)slots * AVE_RING_SLOT_BYTES, &offset) || + offset < AVE_IPC_CHANNEL_OFF + 2 * AVE_CHAN_DESC_STRIDE || + offset + slots * AVE_RING_SLOT_BYTES > AVE_IPC_CHANNEL_OFF + ave->channel_bytes) + return -EPROTO; + + found[id] = true; + r = &ave->ring[id]; + strscpy(r->name, desc, sizeof(r->name)); + /* Host-to-firmware rings start empty, firmware-to-host rings full. */ + r->mode = dir ^ 1; + r->doorbell = doorbell; + r->slots = slots; + r->offset = offset; + r->consumer = r->mode ? -1 : 0; + r->producer = r->mode ? 0 : -1; + } + + if (!found[0] || !found[1]) + return -EPROTO; + if (ave->ring[0].offset < ave->ring[1].offset + ave->ring[1].slots * AVE_RING_SLOT_BYTES && + ave->ring[1].offset < ave->ring[0].offset + ave->ring[0].slots * AVE_RING_SLOT_BYTES) + return -EPROTO; + + /* Hand every host-to-firmware slot to the host; ownership is bit 0. */ + for (unsigned int i = 0; i < ave->ring[0].slots; i++) { + __le64 *slot = ave->ipc.cpu + ave->ring[0].offset + i * AVE_RING_SLOT_BYTES; + + slot[0] = cpu_to_le64(1); + slot[1] = 0; + slot[2] = 0; + } + dma_wmb(); + return 0; +} + +/* + * Finish the coprocessor start-up once the firmware has answered the boot + * request: hand it the heap and the IPC window, then the channel area, and + * read back the two ring descriptors it writes there. + */ +int ave_ipc_negotiate(struct ave_dev *ave, const u32 hello[4]) +{ + u8 *hs = ave->ipc.cpu + AVE_IPC_HANDSHAKE_OFF; + u64 ticks, offset_us; + u32 freq, status, reply[4], off; + int ret; + + if (hello[0] != AVE_HELLO_VERSION || hello[2] != AVE_HELLO_DESC_BYTES || + hello[1] < 2 * AVE_CHAN_DESC_STRIDE || + hello[1] > AVE_IPC_HANDSHAKE_OFF - AVE_IPC_CHANNEL_OFF || + !hello[3] || hello[3] > 4 * SZ_1M) + return -EPROTO; + ave->channel_bytes = hello[1]; + + if (ave->heap.cpu && ave->heap.size != PAGE_ALIGN(hello[3])) + return -EPROTO; + if (!ave->heap.cpu) { + ave->heap.size = PAGE_ALIGN(hello[3]); + ave->heap.cpu = dma_alloc_coherent(ave->dev, ave->heap.size, + &ave->heap.dma, GFP_KERNEL); + if (!ave->heap.cpu) + return -ENOMEM; + } + memset(ave->heap.cpu, 0, ave->heap.size); + writel(AVE_MBOX_STATUS_PENDING, ave->mbox + AVE_MBOX_STATUS); + + /* Tell the firmware how far its timebase is from the host's. */ + ticks = readq(ave->coproc + AVE_COPROC_TIMEBASE); + freq = readl(ave->coproc + AVE_COPROC_TIMEBASE_FREQ); + if (freq && freq < 1000000) + return -EPROTO; + ticks = div64_u64(ticks, freq ? freq / 1000000 : 24); + offset_us = div_u64(ktime_get_ns(), NSEC_PER_USEC) - ticks; + writel(lower_32_bits(offset_us), ave->mbox + AVE_MBOX_TIME_LO); + writel(upper_32_bits(offset_us), ave->mbox + AVE_MBOX_TIME_HI); + + ave_mbox_send(ave, lower_32_bits(ave->ipc.dma), upper_32_bits(ave->ipc.dma), + ave->ipc.size, 0); + ret = ave_mbox_recv(ave, reply); + if (ret) + return ret; + ave->ipc_fw_base = (u64)reply[1] << 32 | reply[0]; + if (!ave->ipc_fw_base || !IS_ALIGNED(ave->ipc_fw_base, SZ_16K) || + ave->ipc_fw_base > U64_MAX - ave->ipc.size) + return -EPROTO; + + memset(ave->ipc.cpu + AVE_IPC_CHANNEL_OFF, 0, ave->channel_bytes); + memset(hs, 0, AVE_HS_BYTES); + memset(ave->ipc.cpu + AVE_IPC_SHARED_OFF, 0, AVE_IPC_SHARED_BYTES); + put_unaligned_le64(ave->ipc_fw_base + AVE_IPC_CHANNEL_OFF, hs + AVE_HS_CHANNELS); + put_unaligned_le64(ave->ipc_fw_base + AVE_IPC_SHARED_OFF, hs + AVE_HS_SHARED); + put_unaligned_le32(AVE_IPC_SHARED_BYTES, hs + AVE_HS_SHARED_BYTES); + put_unaligned_le64(ave->heap.dma, hs + AVE_HS_HEAP); + put_unaligned_le32(hello[3], hs + AVE_HS_HEAP_BYTES); + put_unaligned_le32(AVE_HS_MAGIC_VALUE, hs + AVE_HS_MAGIC); + dma_wmb(); + + ave_mbox_send(ave, lower_32_bits(ave->ipc_fw_base + AVE_IPC_HANDSHAKE_OFF), + upper_32_bits(ave->ipc_fw_base + AVE_IPC_HANDSHAKE_OFF), 0, 0); + ret = ave_mbox_recv(ave, reply); + if (ret) + return ret; + if (ave_fw_offset(ave, (u64)reply[1] << 32 | reply[0], ave->channel_bytes, &off) || + off != AVE_IPC_CHANNEL_OFF || reply[2] > AVE_HELLO_EXTRA_MAX) + return -EPROTO; + + ret = ave_ring_setup(ave); + if (ret) + return ret; + + writel(AVE_BOOT_MAGIC, ave->mbox + AVE_MBOX_READY); + return readl_poll_timeout(ave->mbox + AVE_MBOX_READY, status, !status, 50, + AVE_MBOX_TIMEOUT_US); +} + +static int ave_ring_send(struct ave_dev *ave, struct ave_ring *r, u64 addr, u32 len) +{ + __le64 *slot; + int next; + + if (r->producer < 0) + return -ENOSPC; + + slot = ave->ipc.cpu + r->offset + r->producer * AVE_RING_SLOT_BYTES; + WRITE_ONCE(slot[1], cpu_to_le64(len)); + WRITE_ONCE(slot[2], 0); + dma_wmb(); + WRITE_ONCE(slot[0], cpu_to_le64(addr | (r->mode ^ 1))); + dma_wmb(); + + if (r->consumer < 0) + r->consumer = r->producer; + next = (r->producer + 1) % r->slots; + r->producer = next == r->consumer ? -1 : next; + writel(BIT(r->doorbell), ave->mbox + AVE_MBOX_DOORBELL); + return 0; +} + +static int ave_ring_recv(struct ave_dev *ave, struct ave_ring *r, u64 *addr, u32 *len) +{ + __le64 *slot; + u64 word; + int next; + + if (r->consumer < 0) + return -EAGAIN; + + slot = ave->ipc.cpu + r->offset + r->consumer * AVE_RING_SLOT_BYTES; + word = le64_to_cpu(READ_ONCE(slot[0])); + if ((word & 1) != r->mode) + return -EAGAIN; + dma_rmb(); + *addr = word & ~3ULL; + *len = le64_to_cpu(READ_ONCE(slot[1])); + + if (r->producer < 0) + r->producer = r->consumer; + next = (r->consumer + 1) % r->slots; + r->consumer = next == r->producer ? -1 : next; + return 0; +} + +/* + * Poll both rings once. Returns 1 when the command at @cmd_addr was + * acknowledged, 2 when a firmware message was copied to ave->msg (and its + * buffer handed back), 0 when nothing was pending. + */ +static int ave_ipc_poll_once(struct ave_dev *ave, u64 cmd_addr) +{ + u32 pending, len, off; + u64 addr; + int ret; + + pending = readl(ave->mbox + AVE_MBOX_STATUS); + if (pending) + writel(pending, ave->mbox + AVE_MBOX_STATUS); + + ret = ave_ring_recv(ave, &ave->ring[0], &addr, &len); + if (!ret) { + /* Transport acks return the command address with no payload. */ + if (len || addr != cmd_addr) + return -EPROTO; + return 1; + } + if (ret != -EAGAIN) + return ret; + + ret = ave_ring_recv(ave, &ave->ring[1], &addr, &len); + if (ret == -EAGAIN) + return 0; + if (ret) + return ret; + if (len > sizeof(ave->msg) || ave_fw_offset(ave, addr, len, &off)) + return -EPROTO; + memcpy(ave->msg, ave->ipc.cpu + off, len); + ave->msg_len = len; + /* Return the message buffer to the firmware. */ + ret = ave_ring_send(ave, &ave->ring[1], addr, len); + return ret ?: 2; +} + +/* + * Send one command and wait for its transport ack and the firmware's reply + * message. The reply is left in ave->msg. Caller holds hw_lock. + */ +int ave_ipc_call(struct ave_dev *ave, const void *cmd, size_t len) +{ + u64 cmd_addr = ave->ipc_fw_base + AVE_IPC_CMD_OFF; + u8 *buf = ave->ipc.cpu + AVE_IPC_CMD_OFF; + bool acked = false, replied = false; + ktime_t deadline; + u32 status; + int ret; + + lockdep_assert_held(&ave->hw_lock); + if (ave->fw_state != AVE_FW_RUNNING) + return -EIO; + if (len < AVE_CMD_HDR_BYTES || len > AVE_IPC_CMD_BYTES || !IS_ALIGNED(len, 4)) + return -EINVAL; + + memcpy(buf, cmd, len); + /* The timeout is {duration, creation} in the clock synced at boot. */ + put_unaligned_le64(AVE_CMD_TIMEOUT_US, buf + AVE_CMD_TIMEOUT); + put_unaligned_le64(div_u64(ktime_get_ns(), NSEC_PER_USEC), buf + AVE_CMD_CREATED); + ave->msg_len = 0; + + ret = ave_ring_send(ave, &ave->ring[0], cmd_addr, len); + if (ret) + goto broken; + + deadline = ktime_add_ms(ktime_get(), AVE_CMD_WAIT_MS); + while (!acked || !replied) { + ret = ave_ipc_poll_once(ave, cmd_addr); + if (ret < 0) + goto broken; + if (ret == 1) + acked = true; + else if (ret == 2) + replied = true; + else if (ktime_after(ktime_get(), deadline)) + goto timeout; + else + usleep_range(50, 150); + } + + /* The firmware refused the command: the session fails, the firmware lives on. */ + if (ave->msg_len < AVE_REPLY_MIN_BYTES) + goto broken_len; + status = get_unaligned_le32(ave->msg + AVE_REPLY_STATUS); + if (status) { + dev_err(ave->dev, "command %u refused: status %#x\n", + get_unaligned_le32(cmd), status); + return -EIO; + } + return 0; + +broken_len: + ret = -EPROTO; + goto broken; +timeout: + ret = -ETIMEDOUT; +broken: + dev_err(ave->dev, "command %u failed: %d (acked %d, replied %d)\n", + get_unaligned_le32(cmd), ret, acked, replied); + ave->fw_state = AVE_FW_BROKEN; + return ret; +} + +/* Wait for one more firmware message (frame completions). */ +int ave_ipc_wait_msg(struct ave_dev *ave, unsigned int timeout_ms) +{ + ktime_t deadline = ktime_add_ms(ktime_get(), timeout_ms); + int ret; + + lockdep_assert_held(&ave->hw_lock); + if (ave->fw_state != AVE_FW_RUNNING) + return -EIO; + + for (;;) { + ret = ave_ipc_poll_once(ave, 0); + if (ret == 2) + return 0; + if (ret < 0) + break; + if (ktime_after(ktime_get(), deadline)) { + ret = -ETIMEDOUT; + break; + } + usleep_range(50, 150); + } + dev_err(ave->dev, "no firmware message: %d\n", ret); + ave->fw_state = AVE_FW_BROKEN; + return ret; +} diff --git a/drivers/media/platform/apple/ave/ave-regs.h b/drivers/media/platform/apple/ave/ave-regs.h new file mode 100644 index 0000000000000..91e7c2c403c3e --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-regs.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Apple AVE video encoder: registers and firmware interface constants + * + * Copyright The Aurora Silicon Contributors + */ +#ifndef _APPLE_AVE_REGS_H_ +#define _APPLE_AVE_REGS_H_ + +#include + +/* Mailbox window */ +#define AVE_MBOX_DOORBELL 0x0c +#define AVE_MBOX_STATUS 0x10 +#define AVE_MBOX_STATUS_PENDING BIT(0) +#define AVE_MBOX_SCRATCH(n) (0x18 + (n) * 4) +#define AVE_MBOX_SCRATCH_COUNT 8 +#define AVE_MBOX_READY AVE_MBOX_SCRATCH(3) +#define AVE_MBOX_TIME_LO AVE_MBOX_SCRATCH(4) +#define AVE_MBOX_TIME_HI AVE_MBOX_SCRATCH(5) +#define AVE_MBOX_TIMEOUT_US 5000000 + +/* Coprocessor window */ +#define AVE_COPROC_VCBAR 0x50000 +#define AVE_COPROC_VCBAR_VALID BIT(0) +#define AVE_COPROC_TIMEBASE_FREQ 0x360020 +#define AVE_COPROC_TIMEBASE 0x378000 +#define AVE_COPROC_CTRL 0x600044 +#define AVE_COPROC_CTRL_RUN BIT(4) +#define AVE_COPROC_STATUS 0x600048 +#define AVE_COPROC_BOOT_ARG 0x600400 +#define AVE_COPROC_BOOT_ARG_VALUE 0x10000 +#define AVE_COPROC_KEEP_MAPPINGS 0x600808 + +/* Boot request and the configuration block it points at */ +#define AVE_BOOT_MAGIC 0x08042006 +#define AVE_BOOT_CFG_BYTES 0x40 +#define AVE_BOOT_CFG_DEVICE 1 /* word index */ +#define AVE_BOOT_CFG_DEVICE_T8140 29 +#define AVE_BOOT_CFG_UNITS 2 +#define AVE_BOOT_CFG_GROUPS 3 +#define AVE_BOOT_CFG_SUBID_MASK 4 +#define AVE_BOOT_CFG_LOG_LO 8 +#define AVE_BOOT_CFG_LOG_HI 9 +#define AVE_BOOT_CFG_LOG_BYTES 10 +#define AVE_BOOT_CFG_OPTIONS 12 +#define AVE_BOOT_CFG_OPTIONS_VALUE 0x77 + +/* Firmware log: 512-byte header, then a 128 KiB ring. Mapping is page sized. */ +#define AVE_LOG_BYTES 0x24000 +#define AVE_LOG_HDR_USED 0x20200 +#define AVE_LOG_HDR_BYTES 0x200 +#define AVE_LOG_RING_BYTES 0x20000 + +/* Hello reply: {version, channel area bytes, descriptor bytes, heap bytes} */ +#define AVE_HELLO_VERSION 2 +#define AVE_HELLO_DESC_BYTES 0x100 +#define AVE_HELLO_EXTRA_MAX 0x13c000 + +/* Handshake block in the IPC window */ +#define AVE_HS_BYTES 0x50 +#define AVE_HS_CHANNELS 0x08 +#define AVE_HS_SHARED 0x10 +#define AVE_HS_SHARED_BYTES 0x18 +#define AVE_HS_HEAP 0x1c +#define AVE_HS_HEAP_BYTES 0x24 +#define AVE_HS_MAGIC 0x28 +#define AVE_HS_MAGIC_VALUE 27 + +/* Channel descriptors the firmware writes into the channel area */ +#define AVE_CHAN_DESC_STRIDE 0x100 +#define AVE_CHAN_DESC_NAME_LEN 0x40 +#define AVE_CHAN_DESC_DIR 0x40 +#define AVE_CHAN_DESC_IRQ 0x44 +#define AVE_CHAN_DESC_SLOTS 0x48 +#define AVE_CHAN_DESC_ADDR 0x4c +#define AVE_RING_SLOT_BYTES 64 + +/* Common command header */ +#define AVE_CMD_OPCODE 0x00 +#define AVE_CMD_TOKEN 0x08 +#define AVE_CMD_CLIENT 0x10 +#define AVE_CMD_UNIT 0x18 +#define AVE_CMD_CODEC 0x1c +#define AVE_CMD_OPCODE_ECHO 0x20 +#define AVE_CMD_ARG0 0x24 +#define AVE_CMD_ARG1 0x28 +#define AVE_CMD_ARG2 0x2c +#define AVE_CMD_TIMEOUT 0x30 +#define AVE_CMD_CREATED 0x38 +#define AVE_CMD_HDR_BYTES 0x48 +#define AVE_CMD_TIMEOUT_US 3000000 +#define AVE_CMD_WAIT_MS 5000 + +enum ave_opcode { + AVE_OP_CONFIG = 1, + AVE_OP_HALT = 2, + AVE_OP_OPEN = 3, + AVE_OP_CLOSE = 4, + AVE_OP_START = 6, + AVE_OP_STOP = 7, + AVE_OP_PROCESS = 8, + AVE_OP_COMPLETE = 9, + AVE_OP_PRIORITY = 10, +}; + +/* Firmware-to-host messages */ +#define AVE_REPLY_ID 0x00 +#define AVE_REPLY_CLIENT 0x10 +#define AVE_REPLY_SUBTYPE 0x20 +#define AVE_REPLY_STATUS 0x40 +#define AVE_REPLY_MIN_BYTES 0x48 + +#define AVE_HALT_ALL 0xffffffff + +/* Session command arguments, as the firmware expects them */ +#define AVE_SESSION_ARG0 0xc8 +#define AVE_SESSION_ARG1 0x36 +#define AVE_OPEN_ARG2 0x1e0 +#define AVE_FINISH_ARG2_H264 0x140 +#define AVE_FINISH_ARG2_HEVC 0x20 + +#define AVE_CONFIG_BYTES 0x78 +#define AVE_CONFIG_ALL_UNITS 0x20 +#define AVE_CONFIG_OPT_A 0x49 +#define AVE_CONFIG_OPT_B 0x4a + +#endif diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c new file mode 100644 index 0000000000000..410b86d20b28b --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -0,0 +1,831 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder: V4L2 stateful encoder interface + * + * Follows Documentation/userspace-api/media/v4l/dev-encoder.rst: the CAPTURE + * queue carries the coded stream (H.264 or HEVC), the OUTPUT queue raw NV12. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include + +#include +#include +#include +#include + +#include "ave.h" + +#define AVE_NAME "apple-ave" + +/* + * Frame size limits and input stride alignment. Provisional: conservative + * values the hardware is known to cover (up to the 2408x1506 panel and 1080p + * and 4K class streams); refined once the firmware limits are described. + */ +#define AVE_MIN_W 64 +#define AVE_MIN_H 64 +#define AVE_MAX_W 4096 +#define AVE_MAX_H 4096 +#define AVE_STEP 16 +#define AVE_STRIDE_ALIGN 64 +#define AVE_DEFAULT_W 1920 +#define AVE_DEFAULT_H 1080 + +static const u32 ave_coded_fmts[] = { + V4L2_PIX_FMT_H264, + V4L2_PIX_FMT_HEVC, +}; + +static const u32 ave_raw_fmts[] = { + V4L2_PIX_FMT_NV12, +}; + +static const struct v4l2_event ave_eos_event = { + .type = V4L2_EVENT_EOS, +}; + +enum ave_codec ave_ctx_codec(struct ave_ctx *ctx) +{ + return ctx->dst_fmt.pixelformat == V4L2_PIX_FMT_HEVC ? AVE_CODEC_HEVC : + AVE_CODEC_H264; +} + +static bool ave_fmt_listed(const u32 *list, unsigned int n, u32 fourcc) +{ + for (unsigned int i = 0; i < n; i++) + if (list[i] == fourcc) + return true; + return false; +} + +static void ave_clamp_size(u32 *w, u32 *h) +{ + *w = clamp_t(u32, ALIGN(*w, 2), AVE_MIN_W, AVE_MAX_W); + *h = clamp_t(u32, ALIGN(*h, 2), AVE_MIN_H, AVE_MAX_H); +} + +/* Raw NV12, one buffer: Y plane, then interleaved CbCr at half height. */ +static void ave_fill_raw(struct v4l2_pix_format_mplane *f) +{ + u32 stride, height; + + f->pixelformat = V4L2_PIX_FMT_NV12; + ave_clamp_size(&f->width, &f->height); + stride = ALIGN(max_t(u32, f->width, f->plane_fmt[0].bytesperline), AVE_STRIDE_ALIGN); + height = ALIGN(f->height, AVE_STEP); + f->num_planes = 1; + f->plane_fmt[0].bytesperline = stride; + f->plane_fmt[0].sizeimage = stride * height * 3 / 2; + f->field = V4L2_FIELD_NONE; + memset(f->plane_fmt[0].reserved, 0, sizeof(f->plane_fmt[0].reserved)); + memset(f->reserved, 0, sizeof(f->reserved)); +} + +static void ave_fill_coded(struct v4l2_pix_format_mplane *f) +{ + u32 min; + + if (!ave_fmt_listed(ave_coded_fmts, ARRAY_SIZE(ave_coded_fmts), f->pixelformat)) + f->pixelformat = ave_coded_fmts[0]; + ave_clamp_size(&f->width, &f->height); + /* Room for an intra frame at low compression plus headers. */ + min = max_t(u32, f->width * f->height * 3 / 4, SZ_256K); + f->num_planes = 1; + f->plane_fmt[0].bytesperline = 0; + f->plane_fmt[0].sizeimage = clamp_t(u32, f->plane_fmt[0].sizeimage, min, SZ_32M); + f->field = V4L2_FIELD_NONE; + memset(f->plane_fmt[0].reserved, 0, sizeof(f->plane_fmt[0].reserved)); + memset(f->reserved, 0, sizeof(f->reserved)); +} + +static void ave_reset_fmts(struct ave_ctx *ctx) +{ + struct v4l2_pix_format_mplane *src = &ctx->src_fmt, *dst = &ctx->dst_fmt; + + memset(dst, 0, sizeof(*dst)); + dst->pixelformat = V4L2_PIX_FMT_H264; + dst->width = AVE_DEFAULT_W; + dst->height = AVE_DEFAULT_H; + ave_fill_coded(dst); + + memset(src, 0, sizeof(*src)); + src->width = dst->width; + src->height = dst->height; + ave_fill_raw(src); + src->colorspace = V4L2_COLORSPACE_REC709; + src->xfer_func = V4L2_XFER_FUNC_DEFAULT; + src->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; + src->quantization = V4L2_QUANTIZATION_LIM_RANGE; + dst->colorspace = src->colorspace; + dst->xfer_func = src->xfer_func; + dst->ycbcr_enc = src->ycbcr_enc; + dst->quantization = src->quantization; + + ctx->timeperframe = (struct v4l2_fract){ 1, 30 }; +} + +/* ioctls */ + +static int ave_querycap(struct file *file, void *priv, struct v4l2_capability *cap) +{ + strscpy(cap->driver, AVE_NAME, sizeof(cap->driver)); + strscpy(cap->card, "Apple AVE video encoder", sizeof(cap->card)); + return 0; +} + +static int ave_enum_fmt_cap(struct file *file, void *priv, struct v4l2_fmtdesc *f) +{ + if (f->index >= ARRAY_SIZE(ave_coded_fmts)) + return -EINVAL; + f->pixelformat = ave_coded_fmts[f->index]; + f->flags = V4L2_FMT_FLAG_COMPRESSED; + return 0; +} + +static int ave_enum_fmt_out(struct file *file, void *priv, struct v4l2_fmtdesc *f) +{ + if (f->index >= ARRAY_SIZE(ave_raw_fmts)) + return -EINVAL; + f->pixelformat = ave_raw_fmts[f->index]; + return 0; +} + +static int ave_enum_framesizes(struct file *file, void *priv, struct v4l2_frmsizeenum *fs) +{ + if (fs->index || + !ave_fmt_listed(ave_coded_fmts, ARRAY_SIZE(ave_coded_fmts), fs->pixel_format)) + return -EINVAL; + fs->type = V4L2_FRMSIZE_TYPE_STEPWISE; + fs->stepwise = (struct v4l2_frmsize_stepwise){ + AVE_MIN_W, AVE_MAX_W, 2, AVE_MIN_H, AVE_MAX_H, 2, + }; + return 0; +} + +static int ave_g_fmt_cap(struct file *file, void *priv, struct v4l2_format *f) +{ + f->fmt.pix_mp = ave_file_to_ctx(file)->dst_fmt; + return 0; +} + +static int ave_g_fmt_out(struct file *file, void *priv, struct v4l2_format *f) +{ + f->fmt.pix_mp = ave_file_to_ctx(file)->src_fmt; + return 0; +} + +static int ave_try_fmt_cap(struct file *file, void *priv, struct v4l2_format *f) +{ + ave_fill_coded(&f->fmt.pix_mp); + return 0; +} + +static int ave_try_fmt_out(struct file *file, void *priv, struct v4l2_format *f) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp; + + ave_fill_raw(pix); + if (pix->colorspace == V4L2_COLORSPACE_DEFAULT) { + pix->colorspace = ctx->src_fmt.colorspace; + pix->xfer_func = ctx->src_fmt.xfer_func; + pix->ycbcr_enc = ctx->src_fmt.ycbcr_enc; + pix->quantization = ctx->src_fmt.quantization; + } + return 0; +} + +static int ave_s_fmt_cap(struct file *file, void *priv, struct v4l2_format *f) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + struct vb2_queue *vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type); + struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp; + + if (vb2_is_busy(vq)) + return -EBUSY; + /* The coded format comes first; the raw side follows its size. */ + ave_fill_coded(pix); + pix->colorspace = ctx->src_fmt.colorspace; + pix->xfer_func = ctx->src_fmt.xfer_func; + pix->ycbcr_enc = ctx->src_fmt.ycbcr_enc; + pix->quantization = ctx->src_fmt.quantization; + ctx->dst_fmt = *pix; + return 0; +} + +static int ave_s_fmt_out(struct file *file, void *priv, struct v4l2_format *f) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + struct vb2_queue *vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type); + struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp; + int ret; + + if (vb2_is_busy(vq)) + return -EBUSY; + ret = ave_try_fmt_out(file, priv, f); + if (ret) + return ret; + ctx->src_fmt = *pix; + + /* The coded size follows the raw size (crop is the whole frame). */ + ctx->dst_fmt.width = pix->width; + ctx->dst_fmt.height = pix->height; + ctx->dst_fmt.colorspace = pix->colorspace; + ctx->dst_fmt.xfer_func = pix->xfer_func; + ctx->dst_fmt.ycbcr_enc = pix->ycbcr_enc; + ctx->dst_fmt.quantization = pix->quantization; + ave_fill_coded(&ctx->dst_fmt); + return 0; +} + +static int ave_g_selection(struct file *file, void *priv, struct v4l2_selection *s) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + + if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) + return -EINVAL; + switch (s->target) { + case V4L2_SEL_TGT_CROP: + case V4L2_SEL_TGT_CROP_DEFAULT: + case V4L2_SEL_TGT_CROP_BOUNDS: + s->r = (struct v4l2_rect){ 0, 0, ctx->src_fmt.width, ctx->src_fmt.height }; + return 0; + default: + return -EINVAL; + } +} + +static int ave_s_selection(struct file *file, void *priv, struct v4l2_selection *s) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + + if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT || s->target != V4L2_SEL_TGT_CROP) + return -EINVAL; + /* Only full-frame encoding for now. */ + s->r = (struct v4l2_rect){ 0, 0, ctx->src_fmt.width, ctx->src_fmt.height }; + return 0; +} + +static int ave_g_parm(struct file *file, void *priv, struct v4l2_streamparm *a) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + + if (!V4L2_TYPE_IS_OUTPUT(a->type)) + return -EINVAL; + a->parm.output.capability = V4L2_CAP_TIMEPERFRAME; + a->parm.output.timeperframe = ctx->timeperframe; + return 0; +} + +static int ave_s_parm(struct file *file, void *priv, struct v4l2_streamparm *a) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + struct v4l2_fract *t = &a->parm.output.timeperframe; + + if (!V4L2_TYPE_IS_OUTPUT(a->type)) + return -EINVAL; + if (!t->numerator || !t->denominator) + *t = ctx->timeperframe; + /* 1 to 240 frames per second. */ + if ((u64)t->denominator > (u64)t->numerator * 240) + *t = (struct v4l2_fract){ 1, 240 }; + if (t->numerator > t->denominator) + *t = (struct v4l2_fract){ 1, 1 }; + ctx->timeperframe = *t; + a->parm.output.capability = V4L2_CAP_TIMEPERFRAME; + return 0; +} + +static int ave_encoder_cmd(struct file *file, void *priv, struct v4l2_encoder_cmd *ec) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + int ret; + + ret = v4l2_m2m_ioctl_try_encoder_cmd(file, priv, ec); + if (ret) + return ret; + if (!vb2_is_streaming(v4l2_m2m_get_src_vq(ctx->fh.m2m_ctx)) || + !vb2_is_streaming(v4l2_m2m_get_dst_vq(ctx->fh.m2m_ctx))) + return 0; + + ret = v4l2_m2m_ioctl_encoder_cmd(file, priv, ec); + if (ret) + return ret; + if (ec->cmd == V4L2_ENC_CMD_STOP && v4l2_m2m_has_stopped(ctx->fh.m2m_ctx)) + v4l2_event_queue_fh(&ctx->fh, &ave_eos_event); + if (ec->cmd == V4L2_ENC_CMD_START && v4l2_m2m_has_stopped(ctx->fh.m2m_ctx)) + vb2_clear_last_buffer_dequeued(v4l2_m2m_get_dst_vq(ctx->fh.m2m_ctx)); + return 0; +} + +static int ave_subscribe_event(struct v4l2_fh *fh, const struct v4l2_event_subscription *sub) +{ + switch (sub->type) { + case V4L2_EVENT_EOS: + return v4l2_event_subscribe(fh, sub, 0, NULL); + default: + return v4l2_ctrl_subscribe_event(fh, sub); + } +} + +static const struct v4l2_ioctl_ops ave_ioctl_ops = { + .vidioc_querycap = ave_querycap, + .vidioc_enum_fmt_vid_cap = ave_enum_fmt_cap, + .vidioc_enum_fmt_vid_out = ave_enum_fmt_out, + .vidioc_enum_framesizes = ave_enum_framesizes, + .vidioc_g_fmt_vid_cap_mplane = ave_g_fmt_cap, + .vidioc_g_fmt_vid_out_mplane = ave_g_fmt_out, + .vidioc_try_fmt_vid_cap_mplane = ave_try_fmt_cap, + .vidioc_try_fmt_vid_out_mplane = ave_try_fmt_out, + .vidioc_s_fmt_vid_cap_mplane = ave_s_fmt_cap, + .vidioc_s_fmt_vid_out_mplane = ave_s_fmt_out, + .vidioc_g_selection = ave_g_selection, + .vidioc_s_selection = ave_s_selection, + .vidioc_g_parm = ave_g_parm, + .vidioc_s_parm = ave_s_parm, + + .vidioc_reqbufs = v4l2_m2m_ioctl_reqbufs, + .vidioc_querybuf = v4l2_m2m_ioctl_querybuf, + .vidioc_qbuf = v4l2_m2m_ioctl_qbuf, + .vidioc_dqbuf = v4l2_m2m_ioctl_dqbuf, + .vidioc_prepare_buf = v4l2_m2m_ioctl_prepare_buf, + .vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs, + .vidioc_expbuf = v4l2_m2m_ioctl_expbuf, + .vidioc_streamon = v4l2_m2m_ioctl_streamon, + .vidioc_streamoff = v4l2_m2m_ioctl_streamoff, + + .vidioc_try_encoder_cmd = v4l2_m2m_ioctl_try_encoder_cmd, + .vidioc_encoder_cmd = ave_encoder_cmd, + + .vidioc_subscribe_event = ave_subscribe_event, + .vidioc_unsubscribe_event = v4l2_event_unsubscribe, +}; + +/* Controls */ + +static int ave_s_ctrl(struct v4l2_ctrl *ctrl) +{ + struct ave_ctx *ctx = container_of(ctrl->handler, struct ave_ctx, hdl); + + switch (ctrl->id) { + case V4L2_CID_MPEG_VIDEO_BITRATE_MODE: + ctx->bitrate_mode = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_BITRATE: + ctx->bitrate = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_BITRATE_PEAK: + ctx->bitrate_peak = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_GOP_SIZE: + ctx->gop_size = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME: + ctx->force_key_frame = true; + break; + case V4L2_CID_MPEG_VIDEO_H264_PROFILE: + ctx->h264_profile = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_H264_LEVEL: + ctx->h264_level = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_H264_I_PERIOD: + ctx->h264_i_period = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_H264_MIN_QP: + ctx->h264_min_qp = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_H264_MAX_QP: + ctx->h264_max_qp = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_HEVC_PROFILE: + ctx->hevc_profile = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_HEVC_LEVEL: + ctx->hevc_level = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_HEVC_MIN_QP: + ctx->hevc_min_qp = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_HEVC_MAX_QP: + ctx->hevc_max_qp = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_HEADER_MODE: + case V4L2_CID_MPEG_VIDEO_B_FRAMES: + case V4L2_CID_MIN_BUFFERS_FOR_OUTPUT: + break; + default: + return -EINVAL; + } + return 0; +} + +static const struct v4l2_ctrl_ops ave_ctrl_ops = { + .s_ctrl = ave_s_ctrl, +}; + +static int ave_init_ctrls(struct ave_ctx *ctx) +{ + struct v4l2_ctrl_handler *hdl = &ctx->hdl; + const struct v4l2_ctrl_ops *ops = &ave_ctrl_ops; + struct v4l2_ctrl *c; + + v4l2_ctrl_handler_init(hdl, 18); + + v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_BITRATE_MODE, + V4L2_MPEG_VIDEO_BITRATE_MODE_CQ, + 0, V4L2_MPEG_VIDEO_BITRATE_MODE_VBR); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_BITRATE, 100000, 200000000, 1, + 8000000); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_BITRATE_PEAK, 100000, 400000000, 1, + 16000000); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_GOP_SIZE, 0, 65535, 1, 60); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME, 0, 0, 0, 0); + /* No frame reordering until reference handling is described. */ + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_B_FRAMES, 0, 0, 1, 0); + v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_HEADER_MODE, + V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME, + BIT(V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE), + V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME); + + v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_H264_PROFILE, + V4L2_MPEG_VIDEO_H264_PROFILE_HIGH, + ~(BIT(V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE) | + BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE) | + BIT(V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) | + BIT(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH)), + V4L2_MPEG_VIDEO_H264_PROFILE_HIGH); + v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_H264_LEVEL, + V4L2_MPEG_VIDEO_H264_LEVEL_5_2, 0, + V4L2_MPEG_VIDEO_H264_LEVEL_4_1); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_H264_I_PERIOD, 0, 65535, 1, 0); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_H264_MIN_QP, 0, 51, 1, 0); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_H264_MAX_QP, 0, 51, 1, 51); + + v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_HEVC_PROFILE, + V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10, 0, + V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN); + v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_HEVC_LEVEL, + V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1, 0, + V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_HEVC_MIN_QP, 0, 51, 1, 0); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_HEVC_MAX_QP, 0, 51, 1, 51); + + c = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MIN_BUFFERS_FOR_OUTPUT, 1, 32, 1, 1); + if (c) + c->flags |= V4L2_CTRL_FLAG_READ_ONLY; + + if (hdl->error) { + int ret = hdl->error; + + v4l2_ctrl_handler_free(hdl); + return ret; + } + ctx->fh.ctrl_handler = hdl; + return v4l2_ctrl_handler_setup(hdl); +} + +/* Queues */ + +static int ave_queue_setup(struct vb2_queue *vq, unsigned int *nbufs, unsigned int *nplanes, + unsigned int sizes[], struct device *alloc_devs[]) +{ + struct ave_ctx *ctx = vb2_get_drv_priv(vq); + const struct v4l2_pix_format_mplane *f = + V4L2_TYPE_IS_OUTPUT(vq->type) ? &ctx->src_fmt : &ctx->dst_fmt; + + if (*nplanes) + return (*nplanes != 1 || sizes[0] < f->plane_fmt[0].sizeimage) ? -EINVAL : 0; + *nplanes = 1; + sizes[0] = f->plane_fmt[0].sizeimage; + return 0; +} + +static int ave_buf_prepare(struct vb2_buffer *vb) +{ + struct ave_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + const struct v4l2_pix_format_mplane *f = + V4L2_TYPE_IS_OUTPUT(vb->type) ? &ctx->src_fmt : &ctx->dst_fmt; + + if (vb2_plane_size(vb, 0) < f->plane_fmt[0].sizeimage) + return -EINVAL; + if (V4L2_TYPE_IS_OUTPUT(vb->type)) + vb2_set_plane_payload(vb, 0, f->plane_fmt[0].sizeimage); + return 0; +} + +static void ave_buf_queue(struct vb2_buffer *vb) +{ + struct ave_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb); + + if (V4L2_TYPE_IS_CAPTURE(vb->type) && vb2_is_streaming(vb->vb2_queue) && + v4l2_m2m_dst_buf_is_last(ctx->fh.m2m_ctx)) { + vb2_set_plane_payload(vb, 0, 0); + vbuf->sequence = ctx->dst_sequence++; + v4l2_m2m_last_buffer_done(ctx->fh.m2m_ctx, vbuf); + v4l2_event_queue_fh(&ctx->fh, &ave_eos_event); + return; + } + v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf); +} + +static void ave_return_bufs(struct ave_ctx *ctx, struct vb2_queue *q, + enum vb2_buffer_state state) +{ + struct vb2_v4l2_buffer *vbuf; + + for (;;) { + vbuf = V4L2_TYPE_IS_OUTPUT(q->type) ? + v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx) : + v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + if (!vbuf) + break; + v4l2_m2m_buf_done(vbuf, state); + } +} + +static int ave_session_begin(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + enum ave_codec codec = ave_ctx_codec(ctx); + int ret; + + ret = pm_runtime_resume_and_get(ave->dev); + if (ret) + return ret; + + mutex_lock(&ave->hw_lock); + ret = ave_fw_ensure_running(ave); + if (!ret) + ret = ave_cmd_open(ctx, codec); + if (!ret) + ret = ave_cmd_start(ctx, codec); + if (ret && ctx->session_open) + ave_cmd_finish(ctx, codec); + mutex_unlock(&ave->hw_lock); + + if (ret) { + dev_dbg(ave->dev, "session start failed: %d\n", ret); + pm_runtime_mark_last_busy(ave->dev); + pm_runtime_put_autosuspend(ave->dev); + } + return ret; +} + +static void ave_session_end(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + + cancel_work_sync(&ctx->run_work); + mutex_lock(&ave->hw_lock); + if (ave->fw_state == AVE_FW_RUNNING) + ave_cmd_finish(ctx, ave_ctx_codec(ctx)); + ctx->session_open = false; + mutex_unlock(&ave->hw_lock); + pm_runtime_mark_last_busy(ave->dev); + pm_runtime_put_autosuspend(ave->dev); +} + +static int ave_start_streaming(struct vb2_queue *q, unsigned int count) +{ + struct ave_ctx *ctx = vb2_get_drv_priv(q); + struct vb2_queue *other = V4L2_TYPE_IS_OUTPUT(q->type) ? + v4l2_m2m_get_dst_vq(ctx->fh.m2m_ctx) : + v4l2_m2m_get_src_vq(ctx->fh.m2m_ctx); + int ret = 0; + + v4l2_m2m_update_start_streaming_state(ctx->fh.m2m_ctx, q); + if (V4L2_TYPE_IS_OUTPUT(q->type)) + ctx->src_sequence = 0; + else + ctx->dst_sequence = 0; + + /* The session starts once both queues stream. */ + if (vb2_is_streaming(other) && !ctx->session_open) + ret = ave_session_begin(ctx); + if (ret) + ave_return_bufs(ctx, q, VB2_BUF_STATE_QUEUED); + return ret; +} + +static void ave_stop_streaming(struct vb2_queue *q) +{ + struct ave_ctx *ctx = vb2_get_drv_priv(q); + + if (ctx->session_open) + ave_session_end(ctx); + ave_return_bufs(ctx, q, VB2_BUF_STATE_ERROR); + v4l2_m2m_update_stop_streaming_state(ctx->fh.m2m_ctx, q); + if (V4L2_TYPE_IS_OUTPUT(q->type) && v4l2_m2m_has_stopped(ctx->fh.m2m_ctx)) + v4l2_event_queue_fh(&ctx->fh, &ave_eos_event); +} + +static const struct vb2_ops ave_qops = { + .queue_setup = ave_queue_setup, + .buf_prepare = ave_buf_prepare, + .buf_queue = ave_buf_queue, + .start_streaming = ave_start_streaming, + .stop_streaming = ave_stop_streaming, +}; + +static int ave_queue_init(void *priv, struct vb2_queue *src, struct vb2_queue *dst) +{ + struct ave_ctx *ctx = priv; + int ret; + + src->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + src->io_modes = VB2_MMAP | VB2_DMABUF; + src->drv_priv = ctx; + src->buf_struct_size = sizeof(struct v4l2_m2m_buffer); + src->ops = &ave_qops; + src->mem_ops = &vb2_dma_contig_memops; + src->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + src->lock = &ctx->ave->vdev_lock; + src->dev = ctx->ave->dev; + ret = vb2_queue_init(src); + if (ret) + return ret; + + dst->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + dst->io_modes = VB2_MMAP | VB2_DMABUF; + dst->drv_priv = ctx; + dst->buf_struct_size = sizeof(struct v4l2_m2m_buffer); + dst->ops = &ave_qops; + dst->mem_ops = &vb2_dma_contig_memops; + dst->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + dst->lock = &ctx->ave->vdev_lock; + dst->dev = ctx->ave->dev; + return vb2_queue_init(dst); +} + +/* Jobs: one raw frame in, one coded frame out. */ + +static void ave_run_work(struct work_struct *work) +{ + struct ave_ctx *ctx = container_of(work, struct ave_ctx, run_work); + struct ave_dev *ave = ctx->ave; + struct vb2_v4l2_buffer *src, *dst; + enum vb2_buffer_state state = VB2_BUF_STATE_DONE; + bool keyframe = false; + u32 bytes = 0; + int ret; + + src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + dst = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); + + mutex_lock(&ave->hw_lock); + ret = ave_cmd_process(ctx, ave_ctx_codec(ctx), src, dst, &bytes, &keyframe); + ctx->force_key_frame = false; + mutex_unlock(&ave->hw_lock); + if (ret) { + dev_dbg(ave->dev, "frame %u failed: %d\n", ctx->src_sequence, ret); + state = VB2_BUF_STATE_ERROR; + bytes = 0; + } + + src = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); + dst = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + src->sequence = ctx->src_sequence++; + dst->sequence = ctx->dst_sequence++; + v4l2_m2m_buf_copy_metadata(src, dst); + dst->flags &= ~(V4L2_BUF_FLAG_KEYFRAME | V4L2_BUF_FLAG_PFRAME | V4L2_BUF_FLAG_LAST); + dst->flags |= keyframe ? V4L2_BUF_FLAG_KEYFRAME : V4L2_BUF_FLAG_PFRAME; + vb2_set_plane_payload(&dst->vb2_buf, 0, bytes); + + if (v4l2_m2m_is_last_draining_src_buf(ctx->fh.m2m_ctx, src)) { + dst->flags |= V4L2_BUF_FLAG_LAST; + v4l2_event_queue_fh(&ctx->fh, &ave_eos_event); + v4l2_m2m_mark_stopped(ctx->fh.m2m_ctx); + } + + v4l2_m2m_buf_done(src, state); + v4l2_m2m_buf_done(dst, state); + pm_runtime_mark_last_busy(ave->dev); + v4l2_m2m_job_finish(ave->m2m_dev, ctx->fh.m2m_ctx); +} + +static void ave_device_run(void *priv) +{ + struct ave_ctx *ctx = priv; + + queue_work(ctx->ave->wq, &ctx->run_work); +} + +static const struct v4l2_m2m_ops ave_m2m_ops = { + .device_run = ave_device_run, +}; + +/* File operations */ + +static int ave_open(struct file *file) +{ + struct ave_dev *ave = video_drvdata(file); + struct ave_ctx *ctx; + int ret; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + ctx->ave = ave; + INIT_WORK(&ctx->run_work, ave_run_work); + ave_reset_fmts(ctx); + + v4l2_fh_init(&ctx->fh, video_devdata(file)); + ret = ave_init_ctrls(ctx); + if (ret) + goto err_fh; + + ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(ave->m2m_dev, ctx, ave_queue_init); + if (IS_ERR(ctx->fh.m2m_ctx)) { + ret = PTR_ERR(ctx->fh.m2m_ctx); + goto err_ctrls; + } + v4l2_fh_add(&ctx->fh, file); + return 0; + +err_ctrls: + v4l2_ctrl_handler_free(&ctx->hdl); +err_fh: + v4l2_fh_exit(&ctx->fh); + kfree(ctx); + return ret; +} + +static int ave_release(struct file *file) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + struct ave_dev *ave = ctx->ave; + + mutex_lock(&ave->vdev_lock); + v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); + mutex_unlock(&ave->vdev_lock); + v4l2_ctrl_handler_free(&ctx->hdl); + v4l2_fh_del(&ctx->fh, file); + v4l2_fh_exit(&ctx->fh); + kfree(ctx); + return 0; +} + +static const struct v4l2_file_operations ave_fops = { + .owner = THIS_MODULE, + .open = ave_open, + .release = ave_release, + .poll = v4l2_m2m_fop_poll, + .unlocked_ioctl = video_ioctl2, + .mmap = v4l2_m2m_fop_mmap, +}; + +int ave_v4l2_register(struct ave_dev *ave) +{ + struct video_device *vdev = &ave->vdev; + int ret; + + mutex_init(&ave->vdev_lock); + ave->wq = alloc_ordered_workqueue("apple-ave", WQ_MEM_RECLAIM); + if (!ave->wq) + return -ENOMEM; + + ret = v4l2_device_register(ave->dev, &ave->v4l2_dev); + if (ret) + goto err_wq; + + ave->m2m_dev = v4l2_m2m_init(&ave_m2m_ops); + if (IS_ERR(ave->m2m_dev)) { + ret = PTR_ERR(ave->m2m_dev); + goto err_v4l2; + } + + strscpy(vdev->name, AVE_NAME, sizeof(vdev->name)); + vdev->fops = &ave_fops; + vdev->ioctl_ops = &ave_ioctl_ops; + vdev->release = video_device_release_empty; + vdev->lock = &ave->vdev_lock; + vdev->v4l2_dev = &ave->v4l2_dev; + vdev->vfl_dir = VFL_DIR_M2M; + vdev->device_caps = V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING; + video_set_drvdata(vdev, ave); + + ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1); + if (ret) + goto err_m2m; + return 0; + +err_m2m: + v4l2_m2m_release(ave->m2m_dev); +err_v4l2: + v4l2_device_unregister(&ave->v4l2_dev); +err_wq: + destroy_workqueue(ave->wq); + return ret; +} + +void ave_v4l2_unregister(struct ave_dev *ave) +{ + video_unregister_device(&ave->vdev); + v4l2_m2m_release(ave->m2m_dev); + v4l2_device_unregister(&ave->v4l2_dev); + destroy_workqueue(ave->wq); +} diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h new file mode 100644 index 0000000000000..6421a49e7fdd5 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave.h @@ -0,0 +1,167 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Apple AVE video encoder + * + * Copyright The Aurora Silicon Contributors + */ +#ifndef _APPLE_AVE_H_ +#define _APPLE_AVE_H_ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +struct dev_pm_domain_list; +struct iommu_domain; + +/* Coprocessor memory every firmware boot uses, all inside one allocation. */ +#define AVE_IPC_BYTES (20 * SZ_1M) +#define AVE_IPC_CHANNEL_OFF 0x4000 +#define AVE_IPC_HANDSHAKE_OFF 0x24000 +#define AVE_IPC_SHARED_OFF 0x28000 +#define AVE_IPC_SHARED_BYTES SZ_64K +#define AVE_IPC_CMD_OFF 0x40000 +#define AVE_IPC_CMD_BYTES SZ_128K +#define AVE_IPC_END (AVE_IPC_CMD_OFF + AVE_IPC_CMD_BYTES) + +#define AVE_MSG_MAX SZ_4K + +enum ave_codec { + AVE_CODEC_H264 = 1, + AVE_CODEC_HEVC = 2, +}; + +struct ave_buf { + void *cpu; + dma_addr_t dma; + size_t size; +}; + +struct ave_fw_segment { + phys_addr_t phys; + u64 iova; + size_t size; +}; + +struct ave_ring { + char name[16]; + u32 mode; + u32 doorbell; + u32 slots; + u32 offset; + int consumer; + int producer; +}; + +enum ave_fw_state { + AVE_FW_OFF, + AVE_FW_RUNNING, + AVE_FW_BROKEN, +}; + +struct ave_dev { + struct device *dev; + void __iomem *codec; + void __iomem *mbox; + void __iomem *coproc; + struct dev_pm_domain_list *pd_list; + struct iommu_domain *domain; + + struct ave_fw_segment text; + struct ave_fw_segment data; + void *data_pristine; + bool data_dirty; + + /* Serialises every firmware and ring access. */ + struct mutex hw_lock; + enum ave_fw_state fw_state; + struct ave_buf ipc; + struct ave_buf boot_cfg; + struct ave_buf log; + struct ave_buf heap; + u64 ipc_fw_base; + u32 channel_bytes; + struct ave_ring ring[2]; + u64 next_token; + u32 next_client; + u8 msg[AVE_MSG_MAX]; + u32 msg_len; + + struct v4l2_device v4l2_dev; + struct video_device vdev; + struct v4l2_m2m_dev *m2m_dev; + struct mutex vdev_lock; + struct workqueue_struct *wq; +}; + +struct ave_ctx { + struct v4l2_fh fh; + struct ave_dev *ave; + struct v4l2_ctrl_handler hdl; + struct v4l2_pix_format_mplane src_fmt; + struct v4l2_pix_format_mplane dst_fmt; + struct v4l2_fract timeperframe; + struct work_struct run_work; + + u32 bitrate_mode; + u32 bitrate; + u32 bitrate_peak; + u32 gop_size; + u32 h264_profile; + u32 h264_level; + u32 h264_i_period; + u32 h264_min_qp; + u32 h264_max_qp; + u32 hevc_profile; + u32 hevc_level; + u32 hevc_min_qp; + u32 hevc_max_qp; + bool force_key_frame; + + u32 client; + bool session_open; + u32 src_sequence; + u32 dst_sequence; +}; + +static inline struct ave_ctx *ave_file_to_ctx(struct file *file) +{ + return container_of(file_to_v4l2_fh(file), struct ave_ctx, fh); +} + +/* ave-fw.c */ +int ave_fw_probe(struct ave_dev *ave); +void ave_fw_remove(struct ave_dev *ave); +int ave_fw_ensure_running(struct ave_dev *ave); +int ave_fw_shutdown(struct ave_dev *ave); +void ave_fw_lost(struct ave_dev *ave); +void ave_fw_resume(struct ave_dev *ave); + +/* ave-ipc.c */ +int ave_ipc_negotiate(struct ave_dev *ave, const u32 hello[4]); +int ave_ipc_call(struct ave_dev *ave, const void *cmd, size_t len); +int ave_ipc_wait_msg(struct ave_dev *ave, unsigned int timeout_ms); + +/* ave-cmd.c */ +int ave_cmd_config(struct ave_dev *ave); +int ave_cmd_halt(struct ave_dev *ave); +int ave_cmd_open(struct ave_ctx *ctx, enum ave_codec codec); +int ave_cmd_start(struct ave_ctx *ctx, enum ave_codec codec); +int ave_cmd_process(struct ave_ctx *ctx, enum ave_codec codec, + struct vb2_v4l2_buffer *src, struct vb2_v4l2_buffer *dst, + u32 *bytesused, bool *keyframe); +int ave_cmd_finish(struct ave_ctx *ctx, enum ave_codec codec); + +/* ave-v4l2.c */ +int ave_v4l2_register(struct ave_dev *ave); +void ave_v4l2_unregister(struct ave_dev *ave); +enum ave_codec ave_ctx_codec(struct ave_ctx *ctx); + +#endif From 0210e6b1b935bc8656e45a0d9cb6c69f69130427 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 16:17:49 +1100 Subject: [PATCH 1359/1579] arm64: dts: apple: t8140-j700: enable the video encoder aef49ec18ecf ("arm64: dts: apple: t8140-j700: leave the video encoder disabled") turned the AVE encoder, its two DARTs and the six venc0 power domains off because no driver bound apple,t8140-ave and the domains sat in sync_state forever. The apple-ave driver now binds the node, so enable them again. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- arch/arm64/boot/dts/apple/t8140-j700.dts | 12 ++++++++++++ arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 6 ------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index ece0d9c6bfefb..61427cfe7538f 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -388,6 +388,18 @@ #include "isp-imx558.dtsi" +&ave_dart0 { + status = "okay"; +}; + +&ave_dart1 { + status = "okay"; +}; + +&ave { + status = "okay"; +}; + &serial0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 8a1864e0fabbd..0a479f7e8e8e5 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -69,7 +69,6 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "venc0_sys"; - status = "disabled"; }; ps_apcie_gp: power-controller@258 { @@ -408,7 +407,6 @@ #reset-cells = <0>; label = "venc0_dma"; power-domains = <&ps_venc0_sys>; - status = "disabled"; }; ps_venc0_pipe4: power-controller@8008 { @@ -418,7 +416,6 @@ #reset-cells = <0>; label = "venc0_pipe4"; power-domains = <&ps_venc0_dma>; - status = "disabled"; }; ps_venc0_pipe5: power-controller@8010 { @@ -428,7 +425,6 @@ #reset-cells = <0>; label = "venc0_pipe5"; power-domains = <&ps_venc0_dma>; - status = "disabled"; }; ps_venc0_me0: power-controller@8018 { @@ -438,7 +434,6 @@ #reset-cells = <0>; label = "venc0_me0"; power-domains = <&ps_venc0_dma>; - status = "disabled"; }; ps_venc0_me1: power-controller@8020 { @@ -448,7 +443,6 @@ #reset-cells = <0>; label = "venc0_me1"; power-domains = <&ps_venc0_me0>; - status = "disabled"; }; }; From 91c3b9d88c78373eb143b93c613ef38dcddb5805 Mon Sep 17 00:00:00 2001 From: Ryan <81541646+RyanTheTide@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:31:40 +1100 Subject: [PATCH 1360/1579] arm64: asahi.config: build the Apple video encoder as a module The J700 video encoder (AVE) gets a V4L2 stateful encoder driver in this series. Build it as a module in the universal config, next to the decoder (AVD) and ISP, so the device-tree enable does not leave the encoder and its power domains waiting on a driver that is never built. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- arch/arm64/configs/asahi.config | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index fcf0d27611aae..8d9f76581d226 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -34,6 +34,7 @@ CONFIG_APPLE_WATCHDOG=y CONFIG_MTD=y CONFIG_MTD_PHRAM=m CONFIG_VIDEO_APPLE_AVD=m +CONFIG_VIDEO_APPLE_AVE=m CONFIG_VIDEO_APPLE_ISP=m CONFIG_DRM=y CONFIG_DRM_ASAHI=m From d67d834a872326d95097219b01da7bf8d132babc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:24:15 +1100 Subject: [PATCH 1361/1579] tools/asahi: follow the CLPC dense GPU floor in the demand test The CLPC sampler gained dense light GPU work (264558de4e36) and floor release counters (87e3a4d597af). The host test still extracted the old bodies, so it stopped compiling. Two of its mutations had also gone stale: the streaming and PMP-backoff strings no longer appear in the driver, so those controls tested nothing. Extract the dense and counter helpers, give the fake GPU a submission sequence, and check that dense submissions on T8140 earn the floor, that one sparse or late period ends the streak, and that the release and return counters follow the PMP backoff. Rewrite the stale mutations against the current code, add controls for the new paths, and refuse any mutation that no longer changes the source. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e01cbc2b167bccc5985c7959d809c4c898e23e42) --- tools/asahi/clpc-gpu-demand-tests.py | 58 ++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/tools/asahi/clpc-gpu-demand-tests.py b/tools/asahi/clpc-gpu-demand-tests.py index ecf0fff465711..66f0175802e5a 100644 --- a/tools/asahi/clpc-gpu-demand-tests.py +++ b/tools/asahi/clpc-gpu-demand-tests.py @@ -24,6 +24,7 @@ def main(): source = (ROOT / 'drivers/soc/apple/clpc.c').read_text() header = (ROOT / 'include/linux/soc/apple/pmc.h').read_text() rail_shift = re.search(r'^#define APPLE_PMC_RAIL_SHIFT\(rail\).*$', header, re.M).group(0) + dense_samples = re.search(r'^#define APPLE_CLPC_GPU_DENSE_SAMPLES\s.*$', source, re.M).group(0) types = '\n'.join(item(source, s) + ';' for s in [ 'struct apple_clpc_hw {', 'struct apple_clpc_vote {']) tables = '\n'.join(item(source, 'static const struct apple_clpc_hw ' + n) + ';' @@ -31,7 +32,10 @@ def main(): methods = '\n'.join(item(source, s) for s in [ 'static unsigned int apple_clpc_vote(', 'static u32 apple_clpc_floor(', 'static bool apple_clpc_gpu_streaming(', + 'static bool apple_clpc_gpu_dense_step(', + 'static bool apple_clpc_gpu_dense(', 'static bool apple_clpc_gpu_limited(', + 'static void apple_clpc_gpu_floor_count(', 'static void apple_clpc_sample(']) code = r''' #include @@ -56,7 +60,7 @@ def main(): #define msecs_to_jiffies(ms) (ms) enum apple_clpc_rail { APPLE_CLPC_DCS, APPLE_CLPC_SOC, APPLE_CLPC_RAILS }; ''' + types + r''' -struct apple_clpc_gpu { bool active; }; +struct apple_clpc_gpu { bool active; u64 sequence; }; struct apple_clpc_cluster { unsigned int request[APPLE_CLPC_RAILS]; }; struct apple_clpc { const struct apple_clpc_hw *hw; @@ -66,14 +70,18 @@ def main(): unsigned long samples, next_scan, update_errors; void *covered, *pmc, *gpu_pmgr; u32 gpu_offset, mask, last_floor, last_reason; - bool gpu_on, gpu_limited; + bool gpu_on, gpu_limited, gpu_floor_released; unsigned long gpu_on_since, gpu_sampled, gpu_limited_at; + u64 gpu_sequence; + unsigned int gpu_dense_streak; + unsigned long gpu_floor_releases, gpu_floor_returns; + unsigned long gpu_streaming_samples, gpu_dense_samples; u64 last_vote_ns; struct apple_clpc_gpu *gpu; }; static unsigned long jiffies; static u64 ktime_get_mono_fast_ns(void) { return jiffies * 1000000ull; } -static bool apple_clpc_gpu_active(struct apple_clpc_gpu *g, u64 *u, u64 *s) { *u=jiffies; *s=1; return g->active; } +static bool apple_clpc_gpu_active(struct apple_clpc_gpu *g, u64 *u, u64 *s) { *u=jiffies; *s=g->sequence; return g->active; } static void *cpu_possible_mask; static u32 pmgr_status, pmp_floor, published; static int read_error, vote_error; @@ -91,7 +99,7 @@ def main(): (void)pmc; assert(agent==1 && mask==3); published=floor; return vote_error; } static void apple_clpc_queue(struct apple_clpc *c, bool idle) { (void)c; queued_idle=idle; } -''' + tables + '\n' + methods + r''' +''' + dense_samples + '\n' + tables + '\n' + methods + r''' static void check_t8132(void) { struct apple_clpc c={.hw=&apple_clpc_hw_t8132,.agent=1,.mask=3,.gpu_pmgr=(void *)1}; read_error=vote_error=0; pmgr_status=0; pmp_floor=0; jiffies=0; @@ -118,12 +126,38 @@ def main(): pmp_floor=3; for (jiffies=168; jiffies<=204; jiffies+=4) apple_clpc_sample(&c); assert(published==0 && c.gpu_limited && !queued_idle); + assert(c.gpu_floor_releases==1 && c.gpu_floor_returns==0); pmp_floor=4; for (jiffies=208; jiffies<=304; jiffies+=4) apple_clpc_sample(&c); assert(published==0x34 && c.last_reason==4); + assert(c.gpu_floor_releases==1 && c.gpu_floor_returns==1); + assert(c.gpu_streaming_samples && !c.gpu_dense_samples); jiffies=308; pmgr_status=0; apple_clpc_sample(&c); jiffies=340; apple_clpc_sample(&c); assert(published==0 && queued_idle); } +// Dense light work: graphics powers off between submissions, so it never +// streams, but 2+ submissions in each of 4 periods earn the GPU floor. +static void check_t8140_dense(void) { + struct apple_clpc_gpu gpu={0}; + struct apple_clpc c={.hw=&apple_clpc_hw_t8140,.agent=1,.mask=3,.gpu_pmgr=(void *)1,.gpu=&gpu}; + read_error=vote_error=0; pmp_floor=4; pmgr_status=0; published=~0u; + jiffies=100; apple_clpc_sample(&c); assert(published==0 && queued_idle); + for (jiffies=104; jiffies<=116; jiffies+=4) { + gpu.sequence+=2; apple_clpc_sample(&c); + assert(published==(jiffies<116 ? 0 : 0x34)); + } + assert(c.last_reason==8 && c.gpu_dense_samples==1 && !c.gpu_streaming_samples); + // One sparse period ends the streak; the vote holds, then drops. + jiffies=120; gpu.sequence+=1; apple_clpc_sample(&c); + assert(published==0x34 && c.last_reason==16); + jiffies=148; apple_clpc_sample(&c); assert(published==0 && queued_idle); + // A sample more than one period late does not count as a dense period. + for (jiffies=152; jiffies<=164; jiffies+=4) { gpu.sequence+=2; apple_clpc_sample(&c); } + assert(published==0x34); + jiffies=176; gpu.sequence+=40; apple_clpc_sample(&c); + assert(c.last_reason==16); +} + int main(void) { // T6040's supplier is always powered: it must NEVER substitute for work. struct apple_clpc_gpu gpu={0}; @@ -149,7 +183,8 @@ def main(): jiffies=264; apple_clpc_sample(&c); assert(published==0); check_t8132(); check_t8140(); - puts("PASS: T6040 host demand; T8132 powered floor; T8140 streaming and PMP backoff; idle release"); + check_t8140_dense(); + puts("PASS: T6040 host demand; T8132 powered floor; T8140 streaming, dense work and PMP backoff; idle release"); } ''' code = code.replace('#define APPLE_PMC_RAIL_SHIFT(r) ((r)*8)', rail_shift) @@ -158,12 +193,19 @@ def main(): for name, text in [ ('fixed', code), ('omit-host-demand', code.replace('if (clpc->gpu && !clpc->hw->gpu_demand) {', 'if (false) {')), - ('omit-streaming-demand', code.replace('if (clpc->gpu_pmgr && apple_clpc_gpu_streaming(clpc, now) &&', 'if (false && apple_clpc_gpu_streaming(clpc, now) &&')), - ('omit-pmp-backoff', code.replace('!apple_clpc_gpu_limited(clpc, now))', - '(!apple_clpc_gpu_limited(clpc, now) || true))')), + ('omit-streaming-demand', code.replace('bool streaming = apple_clpc_gpu_streaming(clpc, now);', + 'bool streaming = (apple_clpc_gpu_streaming(clpc, now), false);')), + ('omit-dense-demand', code.replace('if (streaming || dense) {', 'if (streaming || (dense && false)) {')), + ('dense-ignores-late', code.replace('if (!threshold || late || submissions < threshold)', + 'if (!threshold || (late && false) || submissions < threshold)')), + ('uncounted-floor-return', code.replace('clpc->gpu_floor_returns++;', 'clpc->gpu_floor_returns += 0;')), + ('omit-pmp-backoff', code.replace('bool released = apple_clpc_gpu_limited(clpc, now);', + 'bool released = (apple_clpc_gpu_limited(clpc, now), false);')), ('release-too-early', code.replace('msecs_to_jiffies(APPLE_CLPC_HOLD_MS)', '0')), ('retain-at-idle', code.replace('vote->state = 0;', '')), ]: + # A mutation that no longer matches the driver tests nothing. + assert name == 'fixed' or text != code, 'stale mutation: ' + name path = root / (name + '.c') path.write_text(text) binary = path.with_suffix('') From 1f37598237d7aafaa85d8ac0626d2a28bfae96c6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:24:19 +1100 Subject: [PATCH 1362/1579] tools/asahi: fall back to rustc when rustup is absent in G15 tests The PB growth and stats drain runners default to 'rustup run 1.93.1 rustc'. On a host without rustup they stop before testing anything. Keep the pinned toolchain where rustup exists, and otherwise use $RUSTC or rustc, as the other Rust host tests do. The receipt already records the compiler version that ran. Both pass, mutations included, with rustc 1.96. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 (cherry picked from commit ae8725d081e6300958f51c524916e0e7435335dc) --- tools/asahi/g15-pb-growth-tests.py | 12 ++++++++++-- tools/asahi/g15-stats-drain-tests.py | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index f25195017d686..12ee4f16a4955 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -58,6 +58,7 @@ import hashlib import json import os +import shutil from pathlib import Path import re import resource @@ -1362,13 +1363,20 @@ def main(): p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) p.add_argument('--output', type=Path, help='Work directory (default: a temporary one)') p.add_argument('--rustc', default=os.environ.get('HOSTRUSTC'), - help='Host compiler executable (default: rustup run 1.93.1 rustc)') + help='Host compiler executable (default: rustup run 1.93.1 rustc, ' + 'or $RUSTC/rustc where rustup is not installed)') p.add_argument('--ref', help='Read the tested bodies at this git revision') p.add_argument('--mutation', choices=sorted(MUTATIONS) + sorted(CONTROLS)) p.add_argument('--mutations', action='store_true', help='Also require every mutation to fail and every control to pass') a = p.parse_args() - compiler = [a.rustc] if a.rustc else ['rustup', 'run', '1.93.1', 'rustc'] + # The receipt records whichever compiler version ran. + if a.rustc: + compiler = [a.rustc] + elif shutil.which('rustup'): + compiler = ['rustup', 'run', '1.93.1', 'rustc'] + else: + compiler = [os.environ.get('RUSTC', 'rustc')] rustc_version = subprocess.check_output(compiler + ['--version'], text=True).strip() import tempfile out = (a.output or Path(tempfile.mkdtemp(prefix='g15-pb-growth-'))).resolve() diff --git a/tools/asahi/g15-stats-drain-tests.py b/tools/asahi/g15-stats-drain-tests.py index 0db5a798f9253..230a72c260e76 100644 --- a/tools/asahi/g15-stats-drain-tests.py +++ b/tools/asahi/g15-stats-drain-tests.py @@ -27,6 +27,7 @@ import hashlib import json import os +import shutil from pathlib import Path import re import resource @@ -376,12 +377,19 @@ def main(): p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) p.add_argument('--output', type=Path, help='Work directory (default: a temporary one)') p.add_argument('--rustc', default=os.environ.get('HOSTRUSTC'), - help='Host compiler executable (default: rustup run 1.93.1 rustc)') + help='Host compiler executable (default: rustup run 1.93.1 rustc, ' + 'or $RUSTC/rustc where rustup is not installed)') p.add_argument('--ref', help='Read the tested bodies at this git revision') p.add_argument('--mutation', choices=sorted(MUTATIONS)) p.add_argument('--mutations', action='store_true', help='Also require every mutation to fail') a = p.parse_args() - compiler = [a.rustc] if a.rustc else ['rustup', 'run', '1.93.1', 'rustc'] + # The receipt records whichever compiler version ran. + if a.rustc: + compiler = [a.rustc] + elif shutil.which('rustup'): + compiler = ['rustup', 'run', '1.93.1', 'rustc'] + else: + compiler = [os.environ.get('RUSTC', 'rustc')] rustc_version = subprocess.check_output(compiler + ['--version'], text=True).strip() import tempfile out = (a.output or Path(tempfile.mkdtemp(prefix='g15-stats-drain-'))).resolve() From 6c9cd9afcfdea3aeac3fb310ae4f546e453602d2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:24:21 +1100 Subject: [PATCH 1363/1579] tools/asahi: pass the engine to work_state acquire in the G17 test Work-state acquisition now takes the engine (3b5847f4d6b3, a node is kept for the other engine), so the compute deadline fixture no longer compiled. Give the fixture the Engine type, have its acquire check that compute admission asks for a compute node, and add a control that requests a render node instead. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5a12fc06cc805ae7b3f15d773e665ea5742f1ad7) --- tools/asahi/g17-compute-deadline-tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/asahi/g17-compute-deadline-tests.py b/tools/asahi/g17-compute-deadline-tests.py index 88c89f6857799..98c8751773f9b 100644 --- a/tools/asahi/g17-compute-deadline-tests.py +++ b/tools/asahi/g17-compute-deadline-tests.py @@ -70,7 +70,9 @@ def fixture(): struct Scratch;impl Scratch{fn pages(&self)->u16{0}} enum Validated{Compute{scratch:Scratch}} struct Status;impl Status{fn get(&self)->i32{if now_ns()>=STATUS.load(Ordering::SeqCst){-5}else{0}}} -struct Work;impl Work{fn acquire(&self)->Result{if MODE.load(Ordering::SeqCst)==4{Err(EBUSY)}else{Ok(())}}} +mod g17{pub mod job{#[derive(Clone,Copy,Debug,PartialEq)]pub enum Engine{Render,Compute}}} +// A work-state node is taken per engine: compute admission must ask for a compute one. +struct Work;impl Work{fn acquire(&self,engine:g17::job::Engine)->Result{assert_eq!(engine,g17::job::Engine::Compute,"compute admission takes a compute work state");if MODE.load(Ordering::SeqCst)==4{Err(EBUSY)}else{Ok(1)}}} struct Completion;impl Completion{fn status(&self)->Status{Status}fn work_state(&self)->Result{Ok(Work)}} struct Packet{command:Validated,completion:Completion} impl Packet{fn check_dependencies(&self)->Result{Ok(())}fn cancel_requested(&self)->bool{now_ns()>=CANCEL.load(Ordering::SeqCst)}fn is_published(&self)->bool{false}} @@ -159,6 +161,7 @@ def main(): ('fixed', source, ''), ('renew-on-binding-loss', source.replace('let admission = Admission::new(deadline, &check);\n let progress = &self.shared.progress;\n loop {', 'let progress = &self.shared.progress;\n loop {\n let admission = Admission::new(now_ns().saturating_add(WORK_RING_WAIT_NS), &check);'), 'admission budget exceeded'), ('ignore-deadline', source.replace('if now_ns() >= self.deadline', 'if false'), 'admission budget exceeded'), + ('render-work-state', source.replace('acquire(crate::g17::job::Engine::Compute)', 'acquire(crate::g17::job::Engine::Render)'), 'compute admission takes a compute work state'), ('omit-gate-check', source.replace(' admission.check()?;\n match self.try_enter()', ' match self.try_enter()').replace('admission.slice_ms()?', '10'), 'gate wake storm'), ] # Bind the old unbounded mutant with the review's eventual cancellation. From 3c0507dac5a29bf91d23d0d20b7c6830b6da0f94 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:24:24 +1100 Subject: [PATCH 1364/1579] tools/asahi: cover the new RTKit buffers in the OSLog test apple_rtkit_reinit() now also frees the CoreAnalytics and PMP aux buffers and resets the CoreAnalytics completion. The OSLog host test's struct lacked those fields and stopped compiling. Add them, fill every buffer before reinit and check each one is freed, rather than counting six frees. Add a control that keeps the OSLog buffer across reinit, and refuse a mutation that no longer changes the source. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 (cherry picked from commit ac72d4f74c277a40f78f944f166c6b65fc55dd13) --- tools/asahi/rtkit-oslog-tests.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tools/asahi/rtkit-oslog-tests.py b/tools/asahi/rtkit-oslog-tests.py index 5f8eb290ebde5..160c163758d54 100644 --- a/tools/asahi/rtkit-oslog-tests.py +++ b/tools/asahi/rtkit-oslog-tests.py @@ -45,14 +45,14 @@ def function(source, name): void *dev, *wq; struct mailbox *mbox; struct apple_rtkit_shmem oslog_buffer, syslog_buffer, ioreport_buffer, crashlog_buffer; - struct apple_rtkit_shmem kdebug_buffer0, kdebug_buffer1; + struct apple_rtkit_shmem kdebug_buffer0, kdebug_buffer1, coreanalytics_buffer, pmp_aux_buffer; int sync_ack_fifo; bool sync_syslog_ack, oslog_inherited, syslog_inherited, crashlog_inherited, crashed, shutting_down; unsigned iop_power_state, ap_power_state; unsigned long endpoints[4]; char *syslog_msg_buffer; size_t syslog_n_entries, syslog_msg_size; - int epmap_completion, iop_pwr_ack_completion, ap_pwr_ack_completion; + int epmap_completion, iop_pwr_ack_completion, ap_pwr_ack_completion, coreanalytics_completion; }; static unsigned sent, warnings, errors, requests, detachments, flushes, freed, checks; static int send_error, setup_error, start_error; @@ -121,11 +121,16 @@ def function(source, name): for (unsigned fail=0;fail<2;fail++) { memset(&r,0,sizeof(r));r.mbox=&m;apple_rtkit_mark_running(&r); assert(r.oslog_inherited && (r.endpoints[0]&(1UL<size=4096; + r.crashed=true;r.syslog_msg_size=128; detachments=flushes=freed=0;start_error=fail?-5:0; assert(apple_rtkit_reinit(&r)==start_error); assert(!r.oslog_inherited && !r.syslog_inherited && !r.crashlog_inherited); - assert(!r.oslog_buffer.size && !r.crashed && !r.syslog_msg_size && freed==6); + assert(!r.crashed && !r.syslog_msg_size && freed==ARRAY_SIZE(buffers)); + // Every buffer of the old firmware session goes, the OSLog one included. + for (unsigned i=0;isize); assert(r.endpoints[0]==(1UL<oslog_inherited = false;', 'rtk->oslog_inherited = true;') + elif args.mutate == 'buffer': + bodies = bodies.replace('apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer);', '') + # A mutation that no longer matches the driver would test nothing. + assert not args.mutate or bodies != '\n'.join( + function(source, name) for name in ['apple_rtkit_oslog_rx', 'apple_rtkit_mark_running', 'apple_rtkit_reinit']), \ + 'stale mutation: ' + args.mutate code = PREFIX.replace('struct apple_rtkit_shmem', constants + '\nstruct apple_rtkit_shmem', 1) + bodies + MAIN def run(out): From e2fa42d82b11899de12070d2f50f4f5054153d52 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:24:27 +1100 Subject: [PATCH 1365/1579] tools/asahi: model the yielded Type-C port in the tunnel route test dcp_fabric_snapshot_port() now leaves a port that yielded both pipes to an HDMI display unwanted (0d43a7f471ac). The fixture's port had no typec_yielded field, so the test stopped compiling. Add the field and check the snapshot for both values. Also check that a T6030 tunnel on source 1, which has no display-engine clock, gets no crossbar control, since no case covered a blocked source. The runner now builds two mutations, ignoring the clock and ignoring the yield, and requires both to fail. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 (cherry picked from commit ff1234c49ce3913598d5d3b1df3a2b5aafd51cfc) --- tools/asahi/tunnel-route-tests.py | 26 ++++++++++++++++++++------ tools/asahi/tunnel-route/fixture.c | 23 +++++++++++++++++++++-- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/tools/asahi/tunnel-route-tests.py b/tools/asahi/tunnel-route-tests.py index 5be05c91c156f..088ca4df1e1a3 100644 --- a/tools/asahi/tunnel-route-tests.py +++ b/tools/asahi/tunnel-route-tests.py @@ -16,6 +16,11 @@ 'drivers/gpu/drm/apple/dcp-fabric.c': [ 'dcp_fabric_snapshot_port', 'dcp_typec_tunnel_ctl'], } +# Each must make the fixture fail. +MUTATIONS = { + 'clock-ignored': ('& BIT(dpin)))', '& 0))'), + 'snapshot-ignores-yield': ('port->dp_wanted && !port->typec_yielded;', 'port->dp_wanted;'), +} def function(source, name): @@ -45,12 +50,21 @@ def main(): code = (ROOT / 'tools/asahi/tunnel-route/fixture.c').read_text().replace( '/* CORE_HEADER */', header).replace( '/* PRODUCTION_FUNCTIONS */', '\n'.join(bodies)) - source = out / 'production.c' - source.write_text(code) - binary = out / 'production' - subprocess.run(['cc', '-std=gnu11', '-Wall', '-Werror', - str(source), '-o', str(binary)], check=True) - run = subprocess.run([str(binary)], capture_output=True, text=True) + + def build(name, text): + source = out / (name + '.c') + source.write_text(text) + binary = out / name + subprocess.run(['cc', '-std=gnu11', '-Wall', '-Werror', + str(source), '-o', str(binary)], check=True) + return subprocess.run([str(binary)], capture_output=True, text=True) + + for name, (old, new) in MUTATIONS.items(): + assert code.count(old) == 1, 'stale mutation: ' + name + mutant = build(name, code.replace(old, new)) + assert mutant.returncode, 'mutation survived: ' + name + print('PASS negative control:', name) + run = build('production', code) receipt = { 'bodies': {name: hashlib.sha256(body.encode()).hexdigest() for name, body in zip( diff --git a/tools/asahi/tunnel-route/fixture.c b/tools/asahi/tunnel-route/fixture.c index dd63853c3bd36..98133274f9ff8 100644 --- a/tools/asahi/tunnel-route/fixture.c +++ b/tools/asahi/tunnel-route/fixture.c @@ -50,7 +50,7 @@ struct apple_dcp_typec_port { struct apple_dcp_typec_route *routes, *owner, *secondary_owner, *preferred_route; struct apple_connector *connector, *secondary_connector; - bool dp_wanted, dp_hpd; + bool dp_wanted, dp_hpd, typec_yielded; }; static bool t6030; static bool of_machine_is_compatible(const char *compatible) @@ -128,6 +128,25 @@ int main(void) &chip.mux[1]); } } - puts("PASS: DCP snapshot, named/legacy controls, invalid DP IN, stale snapshot"); + /* T6030 has no display-engine clock for a tunnel on source 1. */ + struct apple_dcp_typec_route source1 = { + .dcp = &dcp, + .xbar = &chip.mux[0], + .mux_index = 1, + }; + for (unsigned int soc = 0; soc < 2; soc++) { + t6030 = soc; + for (unsigned int dpin = 0; dpin < 2; dpin++) + assert(dcp_typec_tunnel_ctl(&source1, dpin) == + (soc ? NULL : &chip.mux[1 + dpin])); + } + /* A display that yielded both pipes to an HDMI display wants none. */ + for (unsigned int yielded = 0; yielded < 2; yielded++) { + port.dp_wanted = true; + port.typec_yielded = yielded; + dcp_fabric_snapshot_port(&port, true); + assert(port.core.wanted == !yielded); + } + puts("PASS: DCP snapshot, named/legacy controls, invalid DP IN, stale snapshot, yielded port"); return 0; } From 9b83ffd3639261c53634944c02f2e5c5c7664c05 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:24:29 +1100 Subject: [PATCH 1366/1579] tools/asahi: load mt7932_power by path in the 6 GHz policy tool mt7932-policy-6ghz.py imported its sibling mt7932_power through sys.path. Under 'python3 -I', which our host-test runs use, the script directory is not on sys.path, so the tool and its test failed with ModuleNotFoundError. Load the sibling from the tool's own directory. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 (cherry picked from commit dda23b948860f44b3fe5c54a9a33425debab69c0) --- tools/asahi/mt7932-policy-6ghz.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/asahi/mt7932-policy-6ghz.py b/tools/asahi/mt7932-policy-6ghz.py index 7c0dde2fecb12..a2f8a5e36b9c0 100644 --- a/tools/asahi/mt7932-policy-6ghz.py +++ b/tools/asahi/mt7932-policy-6ghz.py @@ -3,12 +3,24 @@ """Extend J7RP country files from authenticated IZUBA text data, without blobs.""" import argparse import hashlib +import importlib.util import json from pathlib import Path import re import struct +import sys -import mt7932_power as power + +def _sibling(name): + """Load a module from this directory, also under python3 -I or by path.""" + spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(name + '.py')) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +power = _sibling('mt7932_power') SIX_NAME = 'IZUBA_TxPwrLimit6G_MT79x1.dat' DB_NAME = 'IZUBA_db.dat' From 9775d96503c0d0321455cdff7cd61e1c7d897699 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:40:34 +1100 Subject: [PATCH 1367/1579] tools/asahi: follow the variant table and suspend rollback in the bcm4377-resume test The bcm4377-resume host test stopped compiling after two driver changes. Commit 0877cd766126 ("arm64: apple: enable J713 Bluetooth with scoped radio DMA") made the hardware variant table name its context register through BCM4377_BAR2_CONTEXT_ADDR_HI and BCM4388_BAR2_CONTEXT_ADDR_HI. The test compiles that table unchanged but never had those macros. Commit afd51176986a ("Bluetooth: bcm4377: abort suspend if wake IRQ arming fails") made bcm4377_suspend() call bcm4377_resume() to undo a failed arming, through a forward declaration ahead of suspend. The test pastes suspend before resume, so the call became an implicit declaration that clashed with the later static definition. Take every #define the variant table refers to from the driver source, and take the forward declaration of bcm4377_resume() from the driver as well, so both follow the code instead of being restated in the fixture. The controls, assertions and the five mutations are unchanged, and each mutation is still killed by its control. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- tools/asahi/bcm4377-resume-tests.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/asahi/bcm4377-resume-tests.py b/tools/asahi/bcm4377-resume-tests.py index 2eab82a96c1ca..4c7104ff1b458 100755 --- a/tools/asahi/bcm4377-resume-tests.py +++ b/tools/asahi/bcm4377-resume-tests.py @@ -35,6 +35,11 @@ def extract(s,n): hw_enum=re.search(r'^enum bcm4377_chip \{.*?^\};',s,re.M|re.S).group() hw_table=re.search(r'^static const struct bcm4377_hw bcm4377_hw_variants\[\] = \{.*?^\};',s,re.M|re.S).group() fixture=re.sub(r'^struct bcm4377_hw \{.*?\};', 'typedef uint32_t u32;\n'+hw_struct,fixture,count=1,flags=re.M) +# The variant table names register offsets by macro; take their definitions from the driver. +hw_macros='\n'+''.join(m.group()+'\n' for name in sorted(set(re.findall(r'\b[A-Z][A-Z0-9_]+\b',hw_table))) for m in [re.search(r'^#define '+name+r'\b.*$',s,re.M)] if m) +assert 'BAR2_CONTEXT_ADDR_HI' in hw_macros,hw_macros +# bcm4377_suspend() undoes a failed wake arming through bcm4377_resume(), declared ahead of it. +resume_decl=re.search(r'^static int bcm4377_resume\(struct device \*dev\);$',s,re.M).group()+'\n' fixture=fixture.replace('struct bcm4377_hw *hw;', 'const struct bcm4377_hw *hw;') table_callbacks=''' static int send_cal(struct bcm4377_data *); @@ -45,7 +50,7 @@ def extract(s,n): #define bcm4377_send_ptb send_ptb #define bcm4378_send_ptb send_ptb ''' -code=fixture+hw_enum+table_callbacks+hw_table+'\n'.join(bodies.values())+'\n'.join(core_bodies.values())+extract(core,'hci_reset_dev_sync')+(r/'tools/asahi/bcm4377-resume/controls.c').read_text() +code=fixture+hw_enum+hw_macros+table_callbacks+hw_table+resume_decl+'\n'.join(bodies.values())+'\n'.join(core_bodies.values())+extract(core,'hci_reset_dev_sync')+(r/'tools/asahi/bcm4377-resume/controls.c').read_text() (o/'production.c').write_text(code);env=dict(os.environ,TMPDIR=str(o/'tmp')) c=subprocess.run(['cc','-std=gnu11','-pthread','-Wall','-Wextra','-Wno-unused-function','-Wno-unused-parameter','-fsanitize=address,undefined','-fno-pie','-no-pie',str(o/'production.c'),'-o',str(o/'controls')],env=env,capture_output=True,text=True) (o/'compile.log').write_text(c.stdout+c.stderr);assert not c.returncode,c.stderr From c56864aeba7df3b755539cdc1111959a49cb9658 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:42:02 +1100 Subject: [PATCH 1368/1579] tools/asahi: follow the grown variant table in the bcm4377-id test The bcm4377-id host test compiles the gate from before commit 8935b9f50620 ("Bluetooth: bcm4377: match the hardware table ID for J493 recovery") against the current hardware tables, and it required the hardware struct and the variant table to be byte for byte the same in the old and the current driver. Commit 0877cd766126 ("arm64: apple: enable J713 Bluetooth with scoped radio DMA") added a context register field to both, so that check failed with any pre-fix ref. The new table entries also name BCM4377_BAR2_CONTEXT_ADDR_HI and BCM4388_BAR2_CONTEXT_ADDR_HI, which the test never defined, so even the positive build stopped compiling. Keep the exact match for the chip enum, the PCI ID table, the probe selection and the reset registration, which is what the old gate is paired with. For the struct and the table, check what the gate reads instead, the id field and the same chip to ID mapping in both versions. Take every #define the variant table refers to from the driver source. All five builds keep their expected results, and with the pre-fix ref the original, decimal ID, other chip and board bypass gates still abort on the probe registration check. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- tools/asahi/bcm4377-id-tests.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tools/asahi/bcm4377-id-tests.py b/tools/asahi/bcm4377-id-tests.py index 6d5bdb4b68427..167139cfffbc4 100755 --- a/tools/asahi/bcm4377-id-tests.py +++ b/tools/asahi/bcm4377-id-tests.py @@ -97,15 +97,28 @@ def main(): } pieces = {name: take(current, pattern) for name, pattern in patterns.items()} old_pieces = {name: take(old, pattern) for name, pattern in patterns.items()} + # The old gate is compiled against the current tables. Later commits added + # hardware fields and variant settings, so the struct and table may differ, + # but the old gate must still see the same chip IDs, PCI table and probe code. for name in patterns: - if name != 'gate': + if name not in ('gate', 'hw_struct', 'variants'): assert pieces[name] == old_pieces[name], name + for src in (pieces, old_pieces): + assert re.search(r'^\tunsigned int id;$', src['hw_struct'], re.M), src['hw_struct'] + variant_ids = r'^\t\[(BCM\d+)\] = \{\n\t\t\.id = (0x[0-9a-f]+),' + current_ids = re.findall(variant_ids, pieces['variants'], re.M) + assert len(current_ids) == 4, current_ids + assert current_ids == re.findall(variant_ids, old_pieces['variants'], re.M), 'variant ids' defs = '\n'.join(re.findall(r'^#define BCM\d+_DEVICE_ID [^\n]+', current, re.M)) assert defs == '\n'.join(re.findall(r'^#define BCM\d+_DEVICE_ID [^\n]+', old, re.M)) ids = (root / 'include/linux/pci_ids.h').read_text() pci_defs = '\n'.join(re.findall(r'^#define (?:PCI_VENDOR_ID_BROADCOM|PCI_CLASS_NETWORK_OTHER)\s+[^\n]+', ids, re.M)) any_id = take((root / 'include/linux/device-id/pci.h').read_text(), r'^#define PCI_ANY_ID[^\n]+') - source_prefix = PREFIX + defs + '\n' + pci_defs + '\n' + any_id + '\n' + '\n'.join(pieces[n] for n in ('enum', 'hw_struct', 'variants', 'pci_macro_table')) + # Register offsets the variant table names by macro, as defined in the driver. + table_macros = sorted(set(re.findall(r'\b[A-Z][A-Z0-9_]+\b', pieces['variants']))) + hw_defs = '\n'.join(m.group() for name in table_macros for m in [re.search(r'^#define ' + name + r'\b[^\n]*', current, re.M)] if m) + assert 'BAR2_CONTEXT_ADDR_HI' in hw_defs, hw_defs + source_prefix = PREFIX + defs + '\n' + hw_defs + '\n' + pci_defs + '\n' + any_id + '\n' + '\n'.join(pieces[n] for n in ('enum', 'hw_struct', 'variants', 'pci_macro_table')) suffix = SUFFIX.replace('@SELECTION@', pieces['selection']).replace('@REGISTRATION@', pieces['registration']) gates = {'positive': pieces['gate'], 'published-original': old_pieces['gate']} assert pieces['gate'].count('0x4378') == 1 @@ -124,7 +137,7 @@ def no_core(): (args.out / f'{name}.log').write_text(run.stdout + run.stderr) assert run.returncode == (0 if name == 'positive' else -6), (name, run.returncode) results[name] = {'exit': run.returncode, 'source_sha256': sha(source.read_text()), 'stdout': run.stdout.strip()} - receipt = {'source_sha256': sha(current), 'old_source_sha256': sha(old), 'old_ref': args.old_ref, 'actual_inputs': {name: sha(value) for name, value in pieces.items()}, 'device_ids_sha256': sha(defs), 'pci_defs_sha256': sha(pci_defs + any_id), 'results': results, 'scope': 'Actual variant/PCI tables, hardware struct, probe selection statement, reset registration block and admission function. PCI enumeration, OF board matching and unrelated probe operations are controlled; no full probe or hardware execution.'} + receipt = {'source_sha256': sha(current), 'old_source_sha256': sha(old), 'old_ref': args.old_ref, 'actual_inputs': {name: sha(value) for name, value in pieces.items()}, 'device_ids_sha256': sha(defs), 'table_macros_sha256': sha(hw_defs), 'variant_ids': current_ids, 'pci_defs_sha256': sha(pci_defs + any_id), 'results': results, 'scope': 'Actual variant/PCI tables, hardware struct, probe selection statement, reset registration block and admission function. PCI enumeration, OF board matching and unrelated probe operations are controlled; no full probe or hardware execution.'} (args.out / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') print(json.dumps(receipt, indent=2)) From e8cf14bc873206326a1edcf9859f03d02445f067 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:43:01 +1100 Subject: [PATCH 1369/1579] tools/asahi: follow the max3() conversion in the dma-heap-fd test The dma-heap-fd host test compiles the real dma_heap_ioctl() against a small user space harness that defines max() but not max3(). Commit d59676b3ebad ("dma-buf: heaps: use max3 in dma_heap_ioctl") came in with the 7.3 base and changed the ioctl to size its buffer with max3(), so the positive build stopped compiling with an implicit declaration error. Define max3() in the harness the same way the kernel composes it from max(). The bodies under test, the old ref build and all five mutations are unchanged. With the parent of 30d0aff2c65a ("dma-buf: dma-heap: don't publish fd before copy_to_user() succeeds") as the old ref, the positive build passes at both page sizes and the original and every mutation still abort on their assertions. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- tools/asahi/dma-heap-fd-tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/asahi/dma-heap-fd-tests.py b/tools/asahi/dma-heap-fd-tests.py index f40d0709d58a8..4bf6770547efe 100755 --- a/tools/asahi/dma-heap-fd-tests.py +++ b/tools/asahi/dma-heap-fd-tests.py @@ -45,6 +45,7 @@ def function(source, name): #define PAGE_ALIGN(x) (((x)+PAGE_SIZE-1)&~(uint64_t)(PAGE_SIZE-1)) #define ARRAY_SIZE(x) (sizeof(x)/sizeof((x)[0])) #define max(a,b) ((a)>(b)?(a):(b)) +#define max3(a,b,c) max(max(a,b),c) #define IOC_IN (_IOC_WRITE << _IOC_DIRSHIFT) #define IOC_OUT (_IOC_READ << _IOC_DIRSHIFT) #define array_index_nospec(x,n) (x) From b3a3a1b36eb58f8d1e2c1ac7999c568591934e8f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:39:47 +1100 Subject: [PATCH 1370/1579] tools/asahi: follow the closed-client logging in the g15-input-error test The test copies the real VmStatus::report_failure body into a host prelude. Commit f227c7671c0f ("drm/asahi: log the first GPU work failure reported to each G17 VM") made that body use a warned flag, and commit 769e0b0f6214 ("drm/asahi: g17: log a closed client's teardown failures at debug level") made it call unobserved_teardown and mod_pr_debug. The prelude had none of these, so the control no longer built. Give the host VmStatus the client_closed and warned fields, add ENOENT and a mod_pr_debug stub next to pr_warn, and extract the real unobserved_teardown body with the other status methods. The control and all nine mutations run as before, and the removed-fix control still reproduces the lost input error with --ref ffe8011a479c^. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- tools/asahi/g15-input-error-tests.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/asahi/g15-input-error-tests.py b/tools/asahi/g15-input-error-tests.py index 47971fccfa710..809411963fd17 100644 --- a/tools/asahi/g15-input-error-tests.py +++ b/tools/asahi/g15-input-error-tests.py @@ -32,13 +32,14 @@ def extract(text, marker): impl Error { fn from_errno(n:i32)->Self {assert!(n<0);Self(n)} fn to_errno(self)->i32{self.0} } type Result = std::result::Result; const EIO:Error=Error(-5);const ENODEV:Error=Error(-19);const ETIMEDOUT:Error=Error(-110); -const ECANCELED:Error=Error(-125);const EINVAL:Error=Error(-22);const ENOMEM:Error=Error(-12); +const ECANCELED:Error=Error(-125);const EINVAL:Error=Error(-22);const ENOMEM:Error=Error(-12);const ENOENT:Error=Error(-2); macro_rules! pr_warn { ($($arg:tt)*)=>{{ let _=format_args!($($arg)*); }} } +macro_rules! mod_pr_debug { ($($arg:tt)*)=>{{ let _=format_args!($($arg)*); }} } struct Mutex(StdMutex);impl Mutex{fn lock(&self)->MutexGuard<'_,T>{self.0.lock().unwrap()}} struct Mirror{word:*mut i32} -struct VmStatus{error:AtomicI32,reported_error:AtomicI32,mirror:Mutex>} +struct VmStatus{error:AtomicI32,reported_error:AtomicI32,client_closed:AtomicBool,warned:AtomicBool,mirror:Mutex>} impl VmStatus { - fn new(word:*mut i32)->Self{Self{error:AtomicI32::new(0),reported_error:AtomicI32::new(0),mirror:Mutex(StdMutex::new(Some(Mirror{word})))}} + fn new(word:*mut i32)->Self{Self{error:AtomicI32::new(0),reported_error:AtomicI32::new(0),client_closed:AtomicBool::new(false),warned:AtomicBool::new(false),mirror:Mutex(StdMutex::new(Some(Mirror{word})))}} __STATUS__ } mod g17 {use super::*;__POLICY__} @@ -137,7 +138,8 @@ def main(): bodies = "\n".join(methods) code = PRELUDE.replace("__JOB__", bodies).replace("__STATUS__", "\n".join( extract(status, marker) for marker in [" pub(crate) fn get(", " pub(crate) fn record(", - " pub(crate) fn reported_error(", " pub(crate) fn report_failure("])).replace( + " pub(crate) fn reported_error(", " pub(crate) fn report_failure(", + " fn unobserved_teardown("])).replace( "__POLICY__", extract(policy, "pub(crate) fn inherited_error(")) variants = {"control": code} expected = {} From 17756caf4290ae8382248885fce762ae3affe6e8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:40:45 +1100 Subject: [PATCH 1371/1579] tools/asahi: follow the moved queue info setup in the compute-preempt test The test extracts the priority policy from WorkQueue::new as the text between the PRIORITY lookup and the WorkQueueInner literal. Commit ab30807676fb ("drm/asahi: keep work-queue firmware structures until the context is invalidated") moved the QueueInfo allocation out of that literal to just after the policy, so the extracted body pulled in try_init!, alloc, gpu_context and DebugFlags::VerboseFaults and no longer built. End the extracted body at the close of the compute priority override, which is the last statement of the policy in the current driver and in the older ones. All four mutations are unchanged and still fail, and --old-ref c8aed729e9bf^ still fails the same assertions. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- tools/asahi/compute-preempt-tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/asahi/compute-preempt-tests.py b/tools/asahi/compute-preempt-tests.py index fdab1e9e7406b..ed246a3e69060 100755 --- a/tools/asahi/compute-preempt-tests.py +++ b/tools/asahi/compute-preempt-tests.py @@ -29,7 +29,9 @@ def read(path, ref=None): return (root / path).read_text() def policy(text): start = text.index(' let mut prio = *raw::PRIORITY.get(priority as usize).ok_or(EINVAL)?;') - end = text.index(' let inner = WorkQueueInner::ver {', start) + # The policy ends with the compute priority override; queue allocation follows. + tail = ' prio.5 = 1;\n }\n' + end = text.index(tail, start) + len(tail) return text[start:end] queue = read('drivers/gpu/drm/asahi/workqueue.rs', args.ref) macro = read('rust/macros/versions.rs', args.ref) From 68bca8e1a7e484fafe202196ba0cdadb3bd354f3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:40:19 +1100 Subject: [PATCH 1372/1579] tools/asahi: follow the IRQ-safe PMP report entries in the T6030 graph test The PMP report entry probe gained an apple,irq-safe opt-in that sets GENPD_FLAG_IRQ_SAFE (540f52fa22b1). The host fixture had no such flag, so the extracted probe stopped compiling. Define the flag, let the fake OF node carry apple,irq-safe and read it from the overlay alongside the other entry properties. Check that each entry gets the ACTIVE_WAKEUP and IRQ_SAFE flags its overlay node asks for, and add a case where only one entry opts in. Two new mutations drop the flag and set it without the property, and both are killed. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- tools/asahi/t6030-pmp-graph-tests.c | 27 +++++++++++++++++++++++++-- tools/asahi/t6030-pmp-graph-tests.py | 7 +++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/tools/asahi/t6030-pmp-graph-tests.c b/tools/asahi/t6030-pmp-graph-tests.c index be106bf7e3d1f..9f018ee94d628 100644 --- a/tools/asahi/t6030-pmp-graph-tests.c +++ b/tools/asahi/t6030-pmp-graph-tests.c @@ -22,6 +22,7 @@ typedef uint8_t u8; #define PMP_REPORT_READY 1 #define GFP_KERNEL 0 #define GENPD_FLAG_ACTIVE_WAKEUP 1 +#define GENPD_FLAG_IRQ_SAFE 2 #define MAX_PHANDLE_ARGS 4 #define EPROBE_DEFER 517 #define SZ_4K 4096 @@ -44,7 +45,7 @@ static void host_log(const char *format, ...) #define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) struct device_node { const char *name, *compatible; - bool available, always_on, no_ack, deferred, uuid, soc_record; + bool available, always_on, no_ack, deferred, irq_safe, uuid, soc_record; u32 reg; int refs; struct device_node *parent, *child, *sibling, *pmp; @@ -387,6 +388,8 @@ static bool of_property_read_bool(const struct device_node *n, const char *p) return n->no_ack; if (!strcmp(p, "apple,defer-startup-ack")) return n->deferred; + if (!strcmp(p, "apple,irq-safe")) + return n->irq_safe; if (!strcmp(p, "apple,soc-device-record")) return n->soc_record; return false; @@ -620,7 +623,7 @@ static int of_platform_populate(struct device_node *n, void *a, void *b, } struct overlay_entry { u32 reg; - bool always_on, no_ack, deferred; + bool always_on, no_ack, deferred, irq_safe; }; #include "overlay.h" static void cleanup(void) @@ -669,6 +672,7 @@ static struct apple_pmp_report *setup(void) .always_on = o->always_on, .no_ack = o->no_ack, .deferred = o->deferred, + .irq_safe = o->irq_safe, .parent = &report_node, .sibling = i < 3 ? &entries[i + 1] : NULL }; @@ -727,6 +731,14 @@ int main(void) assert(regs[0x1180 / 8] == r->start->seed); assert(((struct apple_pmp_report_entry *)entry_devs[3].dev.data) ->no_ack); + /* genpd flags follow the overlay: always-on wakes, irq-safe opts in. */ + for (unsigned int i = 0; i < 4; i++) { + const struct apple_pmp_report_entry *e = entry_devs[i].dev.data; + assert(!!(e->genpd.flags & GENPD_FLAG_ACTIVE_WAKEUP) == + overlay_entries[i].always_on); + assert(!!(e->genpd.flags & GENPD_FLAG_IRQ_SAFE) == + overlay_entries[i].irq_safe); + } assert(apple_pmp_report_wait_ready(&entries[0], 30000) == 0 && enables == 1 && pmp_node.available); assert(g15_vote() == 0 && commands == 1 && enables == 1); @@ -747,6 +759,17 @@ int main(void) assert(g15_vote() == -EIO && commands == 1); cases++; cleanup(); + /* Only an entry that carries apple,irq-safe becomes an IRQ-safe domain. */ + setup(); + entries[1].irq_safe = true; + assert(apple_pmp_report_probe(&report_dev) == 0); + for (unsigned int i = 0; i < 4; i++) { + const struct apple_pmp_report_entry *e = entry_devs[i].dev.data; + assert(!!(e->genpd.flags & GENPD_FLAG_IRQ_SAFE) == + (i == 1 || overlay_entries[i].irq_safe)); + } + cases++; + cleanup(); /* Missing display acknowledgement cannot release G15, even if bound+READY. */ setup(); assert(apple_pmp_report_probe(&report_dev) == 0); diff --git a/tools/asahi/t6030-pmp-graph-tests.py b/tools/asahi/t6030-pmp-graph-tests.py index af7c849401d3b..6ba26a09c0176 100644 --- a/tools/asahi/t6030-pmp-graph-tests.py +++ b/tools/asahi/t6030-pmp-graph-tests.py @@ -30,9 +30,10 @@ def main(): entries = [] for node in re.findall(r'report@[0-9a-f]+\s*\{(.*?)\n\s*\};', overlay, re.S): reg = int(re.search(r'reg = <(0x[0-9a-f]+|[0-9]+)>', node)[1], 0) - entries.append((reg, 'apple,always-on;' in node, 'apple,no-ack;' in node, 'apple,defer-startup-ack;' in node)) + entries.append((reg, 'apple,always-on;' in node, 'apple,no-ack;' in node, 'apple,defer-startup-ack;' in node, + 'apple,irq-safe;' in node)) assert len(entries) == 4 - entry_data = ',\n'.join('{%d, %d, %d, %d}' % e for e in entries) + entry_data = ',\n'.join('{%d, %d, %d, %d, %d}' % e for e in entries) names = ['static int apple_pmp_report_bit(', 'static int apple_pmp_start_check_layout(', 'static int apple_pmp_start_describe(', 'static void apple_pmp_start_finish(', 'static bool apple_pmp_start_stopping(', 'static void apple_pmp_start_work(', @@ -81,6 +82,8 @@ def main(): ('ignore-startup-ack', '(ack & st->startup_ack) != st->startup_ack', 'false'), ('ignore-entry-ack', 'status == PMP_REPORT_READY && (ack & BIT_ULL(id))', '((void)ack, status == PMP_REPORT_READY)'), ('storage-ack-layout', 'st->ack & ~st->record_seed,', 'st->seed & ~st->record_seed,'), + ('ignore-irq-safe', 'ent->genpd.flags |= GENPD_FLAG_IRQ_SAFE;', '(void)0;'), + ('irq-safe-without-property', 'of_property_read_bool(node, "apple,irq-safe")', 'true'), ('skip-supplier-wait', 'apple_pmp_report_wait_ready(entry, msecs_to_jiffies(timeout_ms))', '((void)timeout_ms, 0)'), ] for name, old, new in controls: From 969fb93c15484af904b602b93b436b3a310a2892 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:43:22 +1100 Subject: [PATCH 1373/1579] tools/asahi: follow the AV1 segmentation map tail in the AVD test The AV1 capture tail gained a per-frame segmentation map below the CDFs (57b0e7c91643), and the preamble now refuses frames larger than the capture format and out-of-range primary references (9fb5f796fed2). The host test did not extract avd_av1_seg_size() and its fixture had no decoded format or s8, so it stopped compiling. Extract avd_av1_seg_size() and avd_av1_set_seg(), give the fixture buffers a DMA address, and give the context a decoded format. The reservation check now counts the map and places it between the compressed data and the CDFs, and the preamble check covers the map address. New cases cover the size and reference index guards, and the map that set_seg zeroes, copies from a reference with the same MiCols and MiRows, or leaves for the AVD, with the CPU and DMA offsets agreeing. Every mutation now names the text it rewrites and is refused if the driver no longer contains it. Four new ones break the map CPU offset, the MiCols match, the reference index bound and the frame width guard, and all six are killed. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- tools/asahi/avd-av1-tail-tests.py | 105 +++++++++++++++++++++++++----- 1 file changed, 88 insertions(+), 17 deletions(-) diff --git a/tools/asahi/avd-av1-tail-tests.py b/tools/asahi/avd-av1-tail-tests.py index 8a36298013af6..d602801b7c0a7 100644 --- a/tools/asahi/avd-av1-tail-tests.py +++ b/tools/asahi/avd-av1-tail-tests.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only OR MIT -"""AV1 auxiliary alignment, probability inheritance and reservation controls.""" +"""AV1 auxiliary alignment, probability and segmentation map inheritance, and reservation controls.""" import argparse import hashlib import json @@ -44,20 +44,22 @@ def structure(source, name): #include #include "controls.h" typedef uint8_t u8; +typedef int8_t s8; typedef uint16_t u16; typedef uint32_t u32; typedef uint64_t dma_addr_t; #define ALIGN(x,a) (((x)+(a)-1)&~((__typeof__(x))(a)-1)) #define ALIGN_DOWN(x,a) ((x)&~((__typeof__(x))(a)-1)) #define WARN_ON(x) (x) +#define DIV_ROUND_UP(n,d) (((n)+(d)-1)/(d)) @ALIGNMENT@ @CDF_HEADER@ @CDF_DEFAULTS@ -struct vb2_buffer {struct {unsigned int length;} planes[1]; unsigned char *memory;}; +struct vb2_buffer {struct {unsigned int length;} planes[1]; unsigned char *memory; dma_addr_t dma;}; struct vb2_v4l2_buffer {struct vb2_buffer vb2_buf;}; struct avd_decoded_buffer { struct {struct vb2_v4l2_buffer vb;} base; - struct {unsigned int color_size;} av1; + struct {unsigned int color_size; unsigned short width,height; size_t seg_size;} av1; }; struct avd_buf {void *cpu; dma_addr_t addr;}; struct avd_run {struct {struct vb2_v4l2_buffer *dst;} bufs; dma_addr_t y_out;}; @@ -65,7 +67,7 @@ def structure(source, name): @CTX_STRUCT@ struct v4l2_ctrl {struct {void *p;} p_cur;}; struct v4l2_pix_format_mplane {unsigned int width,height; struct {unsigned int sizeimage;} plane_fmt[1];}; -struct avd_ctx {void *priv; int ctrl_hdl;}; +struct avd_ctx {void *priv; int ctrl_hdl; struct {struct {struct {unsigned int width,height;} pix_mp;} fmt;} decoded_fmt;}; static struct avd_decoded_buffer *reference; static struct v4l2_ctrl_av1_frame frame; static struct v4l2_ctrl_av1_sequence sequence; @@ -74,6 +76,7 @@ def structure(source, name): static struct v4l2_ctrl controls[4]; static unsigned int lookup_calls, preamble_calls; static void *vb2_plane_vaddr(struct vb2_buffer *b,unsigned int p){assert(p==0);return b->memory;} +static dma_addr_t vb2_dma_contig_plane_dma_addr(struct vb2_buffer *b,unsigned int p){assert(p==0);return b->dma;} static struct avd_decoded_buffer *vb2_to_avd_decoded_buf(struct vb2_buffer *b){return (void *)b;} static struct avd_decoded_buffer *avd_get_ref_buf(struct avd_ctx *ctx,struct vb2_v4l2_buffer *dst,uint64_t timestamp){(void)ctx;(void)dst;assert(timestamp==1000);lookup_calls++;return reference;} static void avd_run_preamble(struct avd_ctx *ctx,struct avd_run *run){(void)ctx;(void)run;preamble_calls++;} @@ -89,11 +92,14 @@ def structure(source, name): TESTS = r''' static size_t aligned_cdf(size_t length,size_t color){return (length&~255UL)-((color+255)&~255UL)-9216;} +/* 3 bits per 4x4 block over 128x128 superblocks, 256-byte aligned. */ +static size_t seg_bytes(size_t w,size_t h){return ALIGN(ALIGN(w,128)*ALIGN(h,128)/128*3,256);} +static void set_frame_ctx(struct avd_ctx *ctx){ctx->decoded_fmt.fmt.pix_mp.width=1920;ctx->decoded_fmt.fmt.pix_mp.height=1080;controls[0].p_cur.p=&sequence;controls[1].p_cur.p=&frame;controls[2].p_cur.p=&tile;controls[3].p_cur.p=&grain;} static void read_exact(const char *path,void *data,size_t n){FILE *f=fopen(path,"rb");assert(f);assert(fread(data,1,n,f)==n);assert(fgetc(f)==EOF);assert(fclose(f)==0);} static void inheritance(unsigned int residue,unsigned int mode,const char *neighbors,bool check_addresses){ const size_t ref_len=6964992+residue,dst_len=7000064+(residue?((residue+79)%256):0); const size_t color=138240,ref_offset=aligned_cdf(ref_len,color),dst_offset=aligned_cdf(dst_len,color); - struct avd_decoded_buffer src={0},dst={0};struct avd_av1_ctx av1={0};struct avd_ctx ctx={.priv=&av1};struct avd_av1_run run={0}; + struct avd_decoded_buffer src={0},dst={0};struct avd_av1_ctx av1={0};struct avd_ctx ctx={.priv=&av1};struct avd_av1_run run={0};set_frame_ctx(&ctx); struct avd_av1_cdfs expected={0};unsigned char probabilities[sizeof(expected)]; src.base.vb.vb2_buf.planes[0].length=ref_len;src.base.vb.vb2_buf.memory=malloc(ref_len);assert(src.base.vb.vb2_buf.memory); dst.base.vb.vb2_buf.planes[0].length=dst_len;dst.base.vb.vb2_buf.memory=malloc(dst_len);assert(dst.base.vb.vb2_buf.memory); @@ -108,7 +114,7 @@ def structure(source, name): controls[0].p_cur.p=&sequence;controls[1].p_cur.p=&frame;controls[2].p_cur.p=&tile;controls[3].p_cur.p=&grain; run.base.bufs.dst=&dst.base.vb;run.base.y_out=0x120000000UL;av1.bufs.probs.cpu=probabilities;lookup_calls=0;preamble_calls=0; assert(avd_av1_run_preamble(&ctx,&run)==0);assert(preamble_calls==1); - if(check_addresses){assert(run.addresses.probs_out==run.base.y_out+dst_offset);assert(run.addresses.color==run.base.y_out+dst_offset+9216);} + if(check_addresses){assert(run.addresses.probs_out==run.base.y_out+dst_offset);assert(run.addresses.color==run.base.y_out+dst_offset+9216);assert(run.addresses.seg==run.base.y_out+dst_offset-seg_bytes(1920,1080));} avd_av1_set_prob(&ctx,&run); assert(lookup_calls==(mode?0:1));assert(memcmp(probabilities,&expected,sizeof(expected))==0); assert(memcmp(dst.base.vb.vb2_buf.memory+dst_offset,&expected,sizeof(expected))==0); @@ -120,18 +126,68 @@ def structure(source, name): struct v4l2_pix_format_mplane pix={.width=width,.height=height};struct avd_ctx ctx={0};pix.plane_fmt[0].sizeimage=compressed; avd_av1_adjust_decoded_fmt(&ctx,&pix); size_t min=pix.plane_fmt[0].sizeimage,color=avd_color_size(width,height),length=min+residue; - assert(min%256==0);assert(min==ALIGN(compressed,256)+9216+ALIGN(color,256)); - size_t cdf=AVD_AV1_CDFS_OFFSET(length,color),co=AVD_AV1_COLOR_OFFSET(length,color); + size_t seg=seg_bytes(width,height); + assert(min%256==0);assert(min==ALIGN(compressed,256)+9216+ALIGN(color,256)+seg); + size_t cdf=AVD_AV1_CDFS_OFFSET(length,color),co=AVD_AV1_COLOR_OFFSET(length,color),sg=AVD_AV1_SEG_OFFSET(length,color,seg); + assert(sg%256==0);assert(sg>=ALIGN(compressed,256));assert(sg+seg==cdf); assert(cdf%256==0&&co%256==0);assert(cdf>=compressed);assert(cdf+sizeof(struct avd_av1_cdfs)<=co);assert(co+color<=length); assert(cdf==aligned_cdf(length,color));assert(co==cdf+9216); } +/* The preamble refuses frames larger than the format and out-of-range reference indices. */ +static int preamble(unsigned int w,unsigned int h,unsigned char primary,signed char idx){ + struct avd_decoded_buffer dst={0};struct avd_av1_ctx av1={0};struct avd_ctx ctx={.priv=&av1};struct avd_av1_run run={0};set_frame_ctx(&ctx); + dst.base.vb.vb2_buf.planes[0].length=7000064;memset(&frame,0,sizeof(frame)); + frame.frame_width_minus_1=w-1;frame.frame_height_minus_1=h-1;frame.primary_ref_frame=primary;if(primary<7)frame.ref_frame_idx[primary]=idx; + run.base.bufs.dst=&dst.base.vb;run.base.y_out=0x120000000UL;return avd_av1_run_preamble(&ctx,&run); +} +static void guards(void){ + assert(preamble(1920,1080,0,0)==0);assert(preamble(1280,720,0,0)==0); + assert(preamble(1921,1080,0,0)==-EINVAL);assert(preamble(1920,1081,0,0)==-EINVAL); + assert(preamble(1920,1080,7,0)==0);assert(preamble(1920,1080,8,0)==-EINVAL);assert(preamble(1920,1080,255,0)==-EINVAL); + assert(preamble(1920,1080,6,7)==0);assert(preamble(1920,1080,6,8)==-EINVAL);assert(preamble(1920,1080,0,-1)==-EINVAL);assert(preamble(1920,1080,3,-128)==-EINVAL); +} +/* Segmentation map kept with each frame: expect 0 zeroes it, 1 copies the reference's, 2 leaves the AVD's map. */ +static void segmap(unsigned char flags,unsigned char primary,unsigned int ref_w,unsigned int ref_h,bool missing,int expect){ + const size_t len=7000064+79,ref_len=6964992+64+(ref_w*ref_h>1920*1080?65536:0),color=138240,seg=seg_bytes(1920,1080),ref_seg=seg_bytes(ref_w,ref_h); + const size_t off=aligned_cdf(len,color)-seg,roff=aligned_cdf(ref_len,color)-ref_seg;const dma_addr_t none=0x140000000UL; + struct avd_decoded_buffer src={0},dst={0};struct avd_av1_ctx av1={0};struct avd_ctx ctx={.priv=&av1};struct avd_av1_run run={0};set_frame_ctx(&ctx); + dst.base.vb.vb2_buf.planes[0].length=len;dst.base.vb.vb2_buf.memory=malloc(len);dst.base.vb.vb2_buf.dma=0x120000000UL;assert(dst.base.vb.vb2_buf.memory); + src.base.vb.vb2_buf.planes[0].length=ref_len;src.base.vb.vb2_buf.memory=malloc(ref_len);src.base.vb.vb2_buf.dma=0x130000000UL;assert(src.base.vb.vb2_buf.memory); + dst.av1.color_size=color;dst.av1.seg_size=seg;src.av1.color_size=color;src.av1.seg_size=ref_seg;src.av1.width=ref_w-1;src.av1.height=ref_h-1; + memset(dst.base.vb.vb2_buf.memory,0xa5,len);memset(src.base.vb.vb2_buf.memory,0x3c,ref_len); + for(size_t i=0;i=roff&&i1&&(strcmp(argv[1],"snapshot")==0||strcmp(argv[1],"snapshot-copy")==0)){assert(argc==3);inheritance(64,0,argv[2],strcmp(argv[1],"snapshot")==0);puts("snapshot inheritance PASS");return 0;} if(argc>1&&strcmp(argv[1],"copy")==0){inheritance(64,0,NULL,false);puts("probability inheritance PASS");return 0;} if(argc>1&&strcmp(argv[1],"seed")==0){inheritance(64,1,NULL,false);puts("default destination seed PASS");return 0;} if(argc>1&&strcmp(argv[1],"aligned")==0){for(unsigned int mode=0;mode<=4;mode++)inheritance(0,mode,NULL,true);reservation(1920,1080,6425600,0);puts("aligned compatibility PASS");return 0;} unsigned int cases=0; + guards();cases++;cases+=segmaps(); for(unsigned int residue=0;residue<256;residue++){ inheritance(residue,0,NULL,true);cases++; for(unsigned int mode=1;mode<=4;mode++){inheritance(residue,mode,NULL,true);cases++;} @@ -148,21 +204,36 @@ def main(): parser.add_argument('--out', type=Path, required=True) parser.add_argument('--neighbors', type=Path) parser.add_argument('--case', choices=['all', 'snapshot', 'snapshot-copy', 'copy', 'seed', 'aligned'], default='all') - parser.add_argument('--mutation', choices=['round-up', 'cpu-unaligned']) + parser.add_argument('--mutation', choices=['round-up', 'cpu-unaligned', 'seg-cpu-unaligned', 'seg-any-size', 'ref-index-upper', 'frame-width-unchecked']) args = parser.parse_args() args.out.mkdir(parents=True, exist_ok=True) paths = {name: args.source / DRIVER / name for name in ['avd-av1.c', 'avd-av1-entropymode.h', 'avd-av1-entropymode.c', 'avd.h']} text = {name: p.read_text() for name, p in paths.items()} src = text['avd-av1.c'] macros = src[src.index('#define AVD_CDFS_SIZE'):src.index('struct avd_av1_run {')] - functions = {n: function(src, n) for n in ['avd_av1_set_prob', 'avd_color_size', 'avd_av1_run_preamble', 'avd_av1_adjust_decoded_fmt']} - if args.mutation == 'round-up': - if 'ALIGN_DOWN((dst), AVD_ALIGN)' not in macros: - parser.error('round-up mutation requires aligned tail macros') - macros = macros.replace('ALIGN_DOWN((dst), AVD_ALIGN)', 'ALIGN((dst), AVD_ALIGN)') - elif args.mutation == 'cpu-unaligned': - functions['avd_av1_set_prob'] = functions['avd_av1_set_prob'].replace('AVD_AV1_CDFS_OFFSET(', 'UNALIGNED_CDFS_OFFSET(') - macros += '\n#define UNALIGNED_CDFS_OFFSET(dst,cl) ((dst)-ALIGN(cl,AVD_ALIGN)-ALIGN(AVD_CDFS_SIZE,AVD_ALIGN))\n' + functions = {n: function(src, n) for n in ['avd_av1_set_prob', 'avd_av1_set_seg', 'avd_av1_seg_size', 'avd_color_size', 'avd_av1_run_preamble', 'avd_av1_adjust_decoded_fmt']} + unaligned = ('\n#define UNALIGNED_CDFS_OFFSET(dst,cl) ((dst)-ALIGN(cl,AVD_ALIGN)-ALIGN(AVD_CDFS_SIZE,AVD_ALIGN))\n' + '#define UNALIGNED_SEG_OFFSET(dst,cl,sl) (UNALIGNED_CDFS_OFFSET(dst,cl)-(sl))\n') + # Each mutation names the extracted text it rewrites; a stale one is refused, never run as a no-op. + mutations = { + 'round-up': (None, 'ALIGN_DOWN((dst), AVD_ALIGN)', 'ALIGN((dst), AVD_ALIGN)'), + 'cpu-unaligned': ('avd_av1_set_prob', 'AVD_AV1_CDFS_OFFSET(', 'UNALIGNED_CDFS_OFFSET('), + 'seg-cpu-unaligned': ('avd_av1_set_seg', 'AVD_AV1_SEG_OFFSET(dst->', 'UNALIGNED_SEG_OFFSET(dst->'), + 'seg-any-size': ('avd_av1_set_seg', 'DIV_ROUND_UP(ref->av1.width + 1, 8)', 'DIV_ROUND_UP(frame->frame_width_minus_1 + 1, 8)'), + 'ref-index-upper': ('avd_av1_run_preamble', 'idx < 0 || idx >= V4L2_AV1_TOTAL_REFS_PER_FRAME', 'idx < 0'), + 'frame-width-unchecked': ('avd_av1_run_preamble', 'run->frame->frame_width_minus_1 + 1 > ctx->decoded_fmt.fmt.pix_mp.width ||', '0 ||'), + } + if args.mutation: + target, old, new = mutations[args.mutation] + body = macros if target is None else functions[target] + if old not in body: + parser.error(args.mutation + ' mutation no longer matches the driver source') + body = body.replace(old, new) + if target is None: + macros = body + else: + functions[target] = body + macros += unaligned header = text['avd-av1-entropymode.h'].replace('#include ', '') defaults = re.sub(r'^#include .*$', '', text['avd-av1-entropymode.c'], flags=re.M) alignment = re.search(r'^#define AVD_ALIGN\s+\d+', text['avd.h'], re.M).group() From 3d70f0e07afa7d3c9e689bf85c8103e4c156e3f6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:44:58 +1100 Subject: [PATCH 1374/1579] tools/asahi: follow the HT capability log in the MT7932 association test The association capability event gained a dynamic debug line for the generated HT capabilities (b23d29d6f661). It looks the element up with cfg80211_find_elem(), which the host fixture did not provide, so the extracted mt_link_event() stopped compiling. Define WLAN_EID_HT_CAPABILITY, give struct element its real id and length header, and add a cfg80211_find_elem() that walks only the IEs it is given. The capability event had no case before, so add one where a generated HT element is exported and the lookup reads the driver's copy, and one where a malformed element fails the attempt without any lookup. The four existing mutations are unchanged and still killed. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- tools/asahi/mt7932-association-events-tests.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/asahi/mt7932-association-events-tests.py b/tools/asahi/mt7932-association-events-tests.py index 3f352376a1c82..436ab4d00b75f 100644 --- a/tools/asahi/mt7932-association-events-tests.py +++ b/tools/asahi/mt7932-association-events-tests.py @@ -52,6 +52,8 @@ def function(source, name): #define NL80211_CHAN_WIDTH_40 1 #define NL80211_CHAN_WIDTH_80 2 #define WLAN_REASON_UNSPECIFIED 1 +#define WLAN_EID_HT_CAPABILITY 45 +struct element {u8 id,datalen,data[];}; #define WLAN_EID_EXTENSION 255 #define WLAN_EID_EXT_HE_CAPABILITY 35 #define dev_info(...) ((void)0) @@ -74,7 +76,6 @@ def function(source, name): static void complete_all(struct completion*c){c->done=UINT_MAX;} struct ieee80211_channel {int center_freq;}; struct cfg80211_chan_def {struct ieee80211_channel *chan;int width,center_freq1;}; -struct element {u8 data[32];}; struct test_band {struct {bool ht_supported;}ht_cap;}; struct test_wiphy {struct test_band *bands[NUM_NL80211_BANDS];}; static struct test_wiphy wiphy; @@ -108,6 +109,13 @@ def function(source, name): static bool cfg80211_chandef_usable(void*w,const struct cfg80211_chan_def*d,int flags){return usable;} static bool mt7932_assoc_phy_valid(const void*b,const void*p,size_t n){return true;} static const struct element *cfg80211_find_ext_elem(int id,const void*p,size_t n){return NULL;} +/* The HT lookup feeds a debug log; it walks only the exported request IEs. */ +static unsigned ht_lookups;static const void *ht_found; +static const struct element *cfg80211_find_elem(int id,const void*p,size_t n){ + const u8*b=p;size_t o=0;assert(id==WLAN_EID_HT_CAPABILITY&&p&&n<=512);ht_lookups++;ht_found=NULL; + while(n-o>=2&&b[o+1]<=n-o-2){if(b[o]==id)return ht_found=b+o;o+=2+b[o+1];} + return NULL; +} static void netif_carrier_off(void*n){} static void netif_stop_queue(void*n){} static void schedule_work(void*w){scheduled++;} @@ -209,6 +217,12 @@ def function(source, name): reset(&m,SUCCESS);u8 rejected[52]={};put_unaligned_le16(1,rejected); receive(&m,0xee,0,rejected,sizeof(rejected),0x40); assert(m.connect_error==-ECONNREFUSED&&m.assoc_start.done&&m.assoc_done.done&&!m.peer_valid);cases++; + // The generated request IEs are exported once; the HT log reads the copy. + reset(&m,SUCCESS);u8 ies[32]={};put_unaligned_le32(28,ies);ies[4]=WLAN_EID_HT_CAPABILITY;ies[5]=26;for(unsigned i=0;i<26;i++)ies[6+i]=i+1;ht_lookups=0; + receive(&m,0xee,0,ies,sizeof(ies),0x5e); + assert(!m.connect_error&&m.assoc_request_seen&&m.assoc_request_ie_len==28&&!memcmp(m.assoc_request_ies,ies+4,28)&&ht_lookups==1&&ht_found==m.assoc_request_ies);cases++; + reset(&m,SUCCESS);ies[5]=25;ht_lookups=0;receive(&m,0xee,0,ies,sizeof(ies),0x5e); + assert(m.connect_error==-EPROTO&&!m.assoc_request_seen&&!ht_lookups&&m.assoc_start.done&&m.assoc_done.done);cases++; printf("PASS %u association event/wait/calibration ownership controls\n",cases); } ''' From 2c242fb405658efe78a8b21fcd25d0fb080a84de Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 17:49:27 +1100 Subject: [PATCH 1375/1579] tools/asahi: follow 6 GHz policy, USB runtime PM and RX wake in the Neo stability test Several driver changes left this harness behind. The MT7932 6 GHz station support (12010e055535) made channels band-qualified, added channels6, PSD and VLP handling to the regulatory notifier and a band argument to mt7932_policy_permits(), so the harness stopped compiling. The DWC3 glue gained a runtime PM reference (c8372a0ec637), a host-only role mark (a50abc4f7b41) and a set_role that can fail (3949e3640d94). DockChannel no longer makes the receive IRQ a wake IRQ and requests it with IRQF_NO_SUSPEND instead (f98122b072d6), so the wake case and its mutation tested behavior the driver no longer has. Give the fixture the nl80211 band, 6 GHz channels with frequency, PSD and VLP flags, a recording cfg80211 rule lookup and the missing helpers. The regulatory case now checks band-qualified permission where channel numbers overlap, that only 6 GHz channels consult the rule, that the VLP-client channel stays out of the domain and the others lose NO_IR, that the policy pass lifts NO_IR only under a rule without it, and that the disable bitmap spans every channel. The USB case checks that the runtime PM reference is dropped exactly once per stop and kept while the reset assert fails, that a refused PHY role stops the core before any host or gadget start, and that only a host-only host start marks the host role before the core comes up. The wake case now checks that the device is a wakeup source without a wake IRQ, and runs the RX IRQ request statement from dockchannel_init() to check its flags. The wake-irq mutation now restores the old wake IRQ, and new rx-suspend, usb-rpm, vlp-client and six-band mutations join it. Every mutation is refused if it no longer matches the driver. All seven mutations and both old-code controls are killed. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- tools/asahi/neo-stability-tests.py | 167 ++++++++++++++++++++++------- 1 file changed, 127 insertions(+), 40 deletions(-) diff --git a/tools/asahi/neo-stability-tests.py b/tools/asahi/neo-stability-tests.py index f65cab986c626..ff02d627e5711 100644 --- a/tools/asahi/neo-stability-tests.py +++ b/tools/asahi/neo-stability-tests.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Host controls for retained domains, wake IRQs, USB roles and country policy.""" +"""Host controls for retained domains, wake IRQs, USB roles and band-qualified country policy.""" import argparse import hashlib import json @@ -34,7 +34,16 @@ def function(source, name): #include #include #define SHA256_DIGEST_SIZE 32 -typedef uint8_t u8; typedef uint16_t u16; typedef uint32_t u32; typedef int32_t s32; +typedef uint8_t u8; typedef uint16_t u16; typedef uint32_t u32; typedef uint64_t u64; typedef int8_t s8; typedef int32_t s32; +enum nl80211_band {NL80211_BAND_2GHZ,NL80211_BAND_5GHZ,NL80211_BAND_60GHZ,NL80211_BAND_6GHZ,NL80211_BAND_S1GHZ,NL80211_BAND_LC,NUM_NL80211_BANDS}; +#define min(a,b) ((a)<(b)?(a):(b)) +#define min_t(t,a,b) ((t)(a)<(t)(b)?(t)(a):(t)(b)) +#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) +#define BIT_ULL(n) (1ULL<<(n)) +#define GENMASK_ULL(h,l) (((~0ULL)>>(63-(h)))&(~0ULL<<(l))) +#define ERR_PTR(e) ((void*)(intptr_t)(e)) +#define IS_ERR(p) ((uintptr_t)(p)>=(uintptr_t)-4095) +#define MHZ_TO_KHZ(f) ((f)*1000) #define READ_ONCE(x) (x) #define WRITE_ONCE(x,v) ((x)=(v)) #define lockdep_assert_held(x) ((void)0) @@ -61,12 +70,24 @@ def function(source, name): static int devm_add_action_or_reset(struct device*d,void(*f)(void*),void*p){if(action_error){f(p);return action_error;}cleanup=f;cleanup_data=p;return 0;} static int devm_device_init_wakeup(struct device*d){wakeup_calls++;if(!wakeup_error)d->wakeup=true;return wakeup_error;} static int devm_pm_set_wake_irq(struct device*d,int i){irq_calls++;if(!irq_error)d->wake_irq=i;return irq_error;} +#define IRQF_NO_SUSPEND 0x4000 +#define IRQF_NO_AUTOEN 0x80000 +static unsigned long rx_flags;static int rx_irq_requested; +static int dockchannel_rx_irq(int i,void*p){return 0;} +static int dockchannel_rx_irq_thread(int i,void*p){return 0;} +static int devm_request_threaded_irq(struct device*d,int irq,int(*h)(int,void*),int(*t)(int,void*),unsigned long f,const char*n,void*p){ + assert(h==dockchannel_rx_irq&&t==dockchannel_rx_irq_thread&&!strcmp(n,"apple-dockchannel-rx"));rx_irq_requested=irq;rx_flags=f;return 0;} enum dwc3_apple_state {DWC3_APPLE_PROBE_PENDING,DWC3_APPLE_NO_CABLE,DWC3_APPLE_HOST,DWC3_APPLE_DEVICE,DWC3_APPLE_SUSPENDED}; -struct dwc3 {int num_usb3_ports,num_usb2_ports;int usb3_generic_phy[2],usb2_generic_phy[2],usb3_phy,usb2_phy;}; -struct dwc3_apple {struct device*dev;struct dwc3 dwc;int lock,reset;enum dwc3_apple_state state;bool fixed_hub,core_active,core_exit_pending,reset_pending;unsigned usb3_power_off_pending,usb2_power_off_pending;}; -static int core_error,host_error,gadget_error,reset_error; -static int dwc3_apple_core_start(struct dwc3_apple*a,enum dwc3_apple_state s){if(core_error)return core_error;a->core_active=true;a->reset_pending=true;return 0;} -static void dwc3_apple_set_role(struct dwc3_apple*a,enum dwc3_apple_state s){} +#define DWC3_GCTL_PRTCAP_HOST 1 +struct dwc3 {int num_usb3_ports,num_usb2_ports;int usb3_generic_phy[2],usb2_generic_phy[2],usb3_phy,usb2_phy;int current_dr_role;}; +struct dwc3_apple {struct device*dev;struct dwc3 dwc;int lock,reset;enum dwc3_apple_state state;bool fixed_hub,host_only,rpm_held,core_active,core_exit_pending,reset_pending;unsigned usb3_power_off_pending,usb2_power_off_pending;}; +static int core_error,host_error,gadget_error,reset_error,role_error,start_role; +static unsigned rpm_puts; +/* A started core holds the glue runtime PM reference until core_stop drops it. */ +static int dwc3_apple_core_start(struct dwc3_apple*a,enum dwc3_apple_state s){start_role=a->dwc.current_dr_role;if(core_error)return core_error;a->core_active=true;a->reset_pending=true;a->rpm_held=true;return 0;} +static int dwc3_apple_set_role(struct dwc3_apple*a,enum dwc3_apple_state s){assert(a->core_active);return role_error;} +static void pm_runtime_mark_last_busy(struct device*d){} +static void pm_runtime_put_autosuspend(struct device*d){rpm_puts++;} static int dwc3_host_init(struct dwc3*d){return host_error;} static int dwc3_gadget_init(struct dwc3*d){return gadget_error;} static void dwc3_host_exit(struct dwc3*d){role_exits++;} @@ -93,16 +114,24 @@ def function(source, name): #define WLAN_REASON_DEAUTH_LEAVING 3 #define __assign_bit(i,b,v) ((b)[(i)/64]=((b)[(i)/64]&~(1ULL<<((i)%64)))|((uint64_t)!!(v)<<((i)%64))) #define for_each_set_bit(i,b,n) for((i)=0;(i)<(n);(i)++) if((b)[(i)/64]&(1ULL<<((i)%64))) -#define bitmap_zero(b,n) memset((b),0,8) -struct ieee80211_channel {unsigned flags,hw_value;int max_power;}; +#define bitmap_zero(b,n) memset((b),0,((n)+63)/64*8) +#define IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT 128 +#define NL80211_RRF_NO_IR (1<<7) +struct ieee80211_channel {unsigned flags,hw_value;int max_power;enum nl80211_band band;int center_freq;s8 psd;}; +static u64 get_unaligned_le64(const u8*p){return get_unaligned_le32(p)|(u64)get_unaligned_le32(p+4)<<32;} +static const void *memchr_inv(const void*p,int c,size_t n){const u8*b=p;for(size_t i=0;i=5955000&&khz<=7115000&&(khz/1000-5955)%20==0);return rule6_absent?ERR_PTR(-ERANGE):&rule6;} static unsigned rtnl_depth,wiphy_depth,schedules; static void rtnl_lock(void){assert(!rtnl_depth++);} static void rtnl_unlock(void){assert(rtnl_depth--==1);} static void wiphy_lock(struct wiphy*w){assert(rtnl_depth&& !wiphy_depth++);} static void wiphy_unlock(struct wiphy*w){assert(wiphy_depth--==1);} -struct mt7932 {struct ieee80211_channel channels[MT7932_CHANNELS_2G],channels5[MT7932_CHANNELS_5G];struct wiphy*wiphy;struct net_device*netdev;struct {struct device dev;}*pdev;struct mt7932_reg_snapshot reg_desired,reg_deferred;unsigned reg_generation,response_lock,startup_work;bool reg_pending,stopping,interface_registered,connecting,connected,disconnecting,reg_deferral,data_ready;int data_lock;int connect_error,assoc_start,assoc_done,discovery_done;uint64_t policy_disabled[1];}; +struct mt7932 {struct ieee80211_channel channels[MT7932_CHANNELS_2G],channels5[MT7932_CHANNELS_5G],channels6[MT7932_CHANNELS_6G];struct wiphy*wiphy;struct net_device*netdev;struct {struct device dev;}*pdev;struct mt7932_reg_snapshot reg_desired,reg_deferred;unsigned reg_generation,response_lock,startup_work;bool reg_pending,stopping,interface_registered,connecting,connected,disconnecting,reg_deferral,data_ready;int data_lock;int connect_error,assoc_start,assoc_done,discovery_done;uint64_t policy_disabled[(MT7932_CHANNELS+63)/64];}; #define spin_lock(l) ((void)(l)) #define spin_unlock(l) ((void)(l)) static void mt_data_kick_locked(struct mt7932*m){assert(0 && "unexpected TX boundary");} @@ -117,9 +146,11 @@ def function(source, name): int main(int argc,char**argv){ assert(argc==2);struct device d={};struct dockchannel dc={.dev=&d,.rx_irq=42}; if(!strcmp(argv[1],"wake")){ - assert(!dockchannel_init_wakeup(&dc)&&d.wakeup&&d.wake_irq==42); - wakeup_error=-ENOMEM;assert(dockchannel_init_wakeup(&dc)==-ENOMEM&&irq_calls==1); - wakeup_error=0;irq_error=-EINVAL;assert(dockchannel_init_wakeup(&dc)==-EINVAL); + /* The FIFO's device is a wakeup source; the receive IRQ is not a wake IRQ. */ + assert(!dockchannel_init_wakeup(&dc)&&d.wakeup&&!d.wake_irq&&!irq_calls&&wakeup_calls==1); + wakeup_error=-ENOMEM;d.wakeup=false;assert(dockchannel_init_wakeup(&dc)==-ENOMEM&&!d.wakeup&&!irq_calls); + /* It keeps receiving through system sleep, so the client filters wake data. */ + assert(!dockchannel_request_rx(&d,&dc)&&rx_irq_requested==42&&rx_flags==(IRQF_NO_AUTOEN|IRQF_NO_SUSPEND)); }else if(!strcmp(argv[1],"sep")){ domain_count=1;assert(!sep_pm_keep_domains(&d)&&!attach_calls); domain_count=2;attach_error=-EAGAIN;assert(sep_pm_keep_domains(&d)==-EAGAIN&&!pd0.syscore); @@ -127,39 +158,78 @@ def function(source, name): cleanup(cleanup_data);assert(!pd0.syscore&&!pd1.syscore); action_error=-ENOMEM;assert(sep_pm_keep_domains(&d)==-ENOMEM&&!pd0.syscore&&!pd1.syscore); }else if(!strcmp(argv[1],"usb")){ - for(unsigned fixed=0;fixed<2;fixed++)for(unsigned role=DWC3_APPLE_HOST;role<=DWC3_APPLE_DEVICE;role++){ - struct dwc3_apple a={.dev=&d,.fixed_hub=fixed,.state=DWC3_APPLE_PROBE_PENDING}; - assert(!dwc3_apple_init(&a,role)&&d.syscore&&a.state==role); - assert(!dwc3_apple_exit(&a)&&!d.syscore&&a.state==DWC3_APPLE_NO_CABLE); - unsigned n=core_exits;assert(!dwc3_apple_exit(&a)&&core_exits==n); - core_error=-EIO;assert(dwc3_apple_init(&a,role)==-EIO&&!d.syscore);core_error=0; - host_error=gadget_error=-ENOMEM;n=core_exits;assert(dwc3_apple_init(&a,role)==-ENOMEM&&!d.syscore&&core_exits==n+1);host_error=gadget_error=0; - assert(!dwc3_apple_init(&a,role)&&d.syscore);reset_error=-EIO;assert(dwc3_apple_exit(&a)==-EIO&&d.syscore&&a.reset_pending); - reset_error=0;n=core_exits;unsigned exits=role_exits; - assert(!dwc3_apple_exit(&a)&&!d.syscore&&!a.reset_pending); + for(unsigned fixed=0;fixed<2;fixed++)for(unsigned host_only=0;host_only<2;host_only++)for(unsigned role=DWC3_APPLE_HOST;role<=DWC3_APPLE_DEVICE;role++){ + struct dwc3_apple a={.dev=&d,.fixed_hub=fixed,.host_only=host_only,.state=DWC3_APPLE_PROBE_PENDING}; + unsigned puts=rpm_puts; + assert(!dwc3_apple_init(&a,role)&&d.syscore&&a.state==role&&a.rpm_held); + /* Only a host-only host start marks the role before the core comes up. */ + assert(start_role==(host_only&&role==DWC3_APPLE_HOST?DWC3_GCTL_PRTCAP_HOST:0)); + assert(!dwc3_apple_exit(&a)&&!d.syscore&&a.state==DWC3_APPLE_NO_CABLE&&!a.rpm_held&&rpm_puts==puts+1); + unsigned n=core_exits;assert(!dwc3_apple_exit(&a)&&core_exits==n&&rpm_puts==puts+1); + core_error=-EIO;assert(dwc3_apple_init(&a,role)==-EIO&&!d.syscore&&!a.rpm_held);core_error=0; + host_error=gadget_error=-ENOMEM;n=core_exits;assert(dwc3_apple_init(&a,role)==-ENOMEM&&!d.syscore&&core_exits==n+1&&!a.rpm_held&&rpm_puts==puts+2);host_error=gadget_error=0; + /* A refused PHY role stops the core again before any host or gadget start. */ + role_error=-EIO;n=core_exits;unsigned exits=role_exits; + assert(dwc3_apple_init(&a,role)==-EIO&&!d.syscore&&!a.core_active&&!a.rpm_held&&core_exits==n+1&&role_exits==exits&&a.state==DWC3_APPLE_NO_CABLE&&rpm_puts==puts+3);role_error=0; + assert(!dwc3_apple_init(&a,role)&&d.syscore);reset_error=-EIO;assert(dwc3_apple_exit(&a)==-EIO&&d.syscore&&a.reset_pending&&a.rpm_held&&rpm_puts==puts+3); + reset_error=0;n=core_exits;exits=role_exits; + assert(!dwc3_apple_exit(&a)&&!d.syscore&&!a.reset_pending&&!a.rpm_held&&rpm_puts==puts+4); assert(core_exits==n&&role_exits==exits); } }else if(!strcmp(argv[1],"regulatory")){ + enum {N25=MT7932_CHANNELS_2G+MT7932_CHANNELS_5G,SIX13=N25+3,VLP=N25+5}; struct mt7932 m={.reg_generation=7};struct wiphy w={.priv=&m};m.wiphy=&w; struct mt7932_policy p={};u8 tables[9][1020]={}; - for(unsigned i=0;i<9;i++){p.table[i+1]=tables[i];tables[i][4]=0;} - for(unsigned i=0;iband=ihw_value=imax_power=20; memset(tables[t]+44+j*122,0xc4,122);tables[t][44+j*122]=c->hw_value; if(i!=10)tables[t][45+j*122]=1; } - assert(!mt7932_policy_permits(&p,11)&&mt7932_policy_permits(&p,10)&&mt7932_policy_permits(&p,165)&&!mt7932_policy_permits(&p,14)); - struct regulatory_request request={.alpha2={'U','S'}};mt_regulatory_notify(&w,&request);m.reg_pending=false; - unsigned generation=m.reg_generation;mt_policy_disable(&m,&p,generation-1);assert(!m.channels[10].flags&&!m.policy_disabled[0]); - mt_policy_disable(&m,&p,generation);assert(m.channels[10].flags==1&&m.policy_disabled[0]==(1ULL<<10)); - struct mt7932_reg_snapshot reg=m.reg_desired;mt7932_policy_filter(®,&p);assert(reg.length==12+(MT7932_CHANNELS-1)*8&®.domain[8]==12&®.domain[9]==MT7932_CHANNELS_5G); - for(unsigned i=0;iband=NL80211_BAND_6GHZ;c->hw_value=mt7932_channel_6g(k);c->center_freq=5950+5*c->hw_value; + c->max_power=14;c->psd=-1;c->flags=IEEE80211_CHAN_NO_IR|IEEE80211_CHAN_PSD|(N25+k==VLP?IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT:0); + } + assert(!mt7932_policy_permits(&p,NL80211_BAND_2GHZ,11)&&mt7932_policy_permits(&p,NL80211_BAND_2GHZ,10)&&mt7932_policy_permits(&p,NL80211_BAND_5GHZ,165)&&!mt7932_policy_permits(&p,NL80211_BAND_2GHZ,14)); + /* Channel numbers mean nothing without their band. */ + assert(mt7932_policy_permits(&p,NL80211_BAND_2GHZ,13)&&!mt7932_policy_permits(&p,NL80211_BAND_6GHZ,13)&&mt7932_policy_permits(&p,NL80211_BAND_6GHZ,1)); + assert(!mt7932_policy_permits(&p,NL80211_BAND_5GHZ,1)&&!mt7932_policy_permits(&p,NL80211_BAND_2GHZ,36)&&!mt7932_policy_permits(&p,NL80211_BAND_6GHZ,3)&&!mt7932_policy_permits(&p,NUM_NL80211_BANDS,1)); + struct regulatory_request request={.alpha2={'U','S'}};rule_lookups=0;mt_regulatory_notify(&w,&request);m.reg_pending=false; + assert(rule_lookups==MT7932_CHANNELS_6G); + /* The VLP-client channel stays out of the firmware domain; the others lose NO_IR there. */ + assert(m.reg_desired.length==12+(MT7932_CHANNELS-1)*8&&m.reg_desired.domain[8]==13&&m.reg_desired.domain[9]==MT7932_CHANNELS_5G&&m.reg_desired.domain[10]==MT7932_CHANNELS_6G-1); + for(unsigned i=0;ihw_value&&!f); + else{unsigned k=i-N25+(i>=VLP);assert(get_unaligned_le16(e)==mt7932_channel_6g(k)&&f==IEEE80211_CHAN_PSD);} + } + for(unsigned k=0;k=N25-1?c!=13:true);} + /* A rule that keeps NO_IR, or no rule at all, leaves the channel passive. */ + m.channels6[0].flags|=IEEE80211_CHAN_NO_IR;m.channels6[1].flags|=IEEE80211_CHAN_NO_IR; + rule6.flags=NL80211_RRF_NO_IR;mt_policy_disable(&m,&p,generation);assert(m.channels6[0].flags&IEEE80211_CHAN_NO_IR); + rule6.flags=0;rule6_absent=true;mt_policy_disable(&m,&p,generation);assert(m.channels6[1].flags&IEEE80211_CHAN_NO_IR); + rule6_absent=false;mt_policy_disable(&m,&p,generation);assert(!(m.channels6[0].flags&IEEE80211_CHAN_NO_IR)&&!(m.channels6[1].flags&IEEE80211_CHAN_NO_IR)); + m.channels[10].flags=0;m.channels6[3].flags=IEEE80211_CHAN_NO_IR|IEEE80211_CHAN_PSD;mt_regulatory_notify(&w,&request); + assert(m.reg_generation==generation&&m.channels[10].flags==1&&(m.channels6[3].flags&IEEE80211_CHAN_DISABLED)); + m.channels[10].flags=0;m.channels6[3].flags=IEEE80211_CHAN_NO_IR|IEEE80211_CHAN_PSD;request.alpha2[0]='C';request.alpha2[1]='A';mt_regulatory_notify(&w,&request); + assert(m.reg_generation==generation+1&&!m.policy_disabled[0]&&!m.policy_disabled[1]); m.reg_pending=false;m.connected=true;unsigned applied=m.reg_generation; put_unaligned_le32(IEEE80211_CHAN_NO_IR,m.reg_desired.domain+16); mt_regulatory_notify(&w,&request); @@ -178,7 +248,7 @@ def main(): parser.add_argument('--output', type=Path, required=True) parser.add_argument('--old-usb', action='store_true') parser.add_argument('--old-regulatory', action='store_true') - parser.add_argument('--mutation', choices=['sep-syscore', 'wake-irq', 'stale-policy']) + parser.add_argument('--mutation', choices=['sep-syscore', 'wake-irq', 'rx-suspend', 'stale-policy', 'usb-rpm', 'vlp-client', 'six-band']) args = parser.parse_args() args.output.mkdir(parents=True, exist_ok=True) files = { @@ -199,16 +269,33 @@ def main(): ROOT/'drivers/net/wireless/mediatek/mt7932/regulatory.h'] header = '\n'.join(re.sub(r'^#include .*\n', '', path.read_text(), flags=re.M) for path in headers) + # The receive IRQ request statement from dockchannel_init(), run against a recording stub. + dock = (ROOT/'drivers/soc/apple/dockchannel.c').read_text() + request = re.search(r'\tret = devm_request_threaded_irq\(dev, dockchannel->rx_irq,.*?\);\n', dock, re.S) + if not request: + raise ValueError('dockchannel RX IRQ request') + bodies['dockchannel_request_rx'] = ('static int dockchannel_request_rx(struct device *dev, struct dockchannel *dockchannel)\n' + '{\n\tint ret;\n\n' + request[0] + '\treturn ret;\n}\n') at = PREFIX.index('struct mt7932 {') code = PREFIX[:at] + header + PREFIX[at:] + '\n'.join(bodies.values()) + MAIN if not args.old_usb: code = code.replace(',DWC3_APPLE_SUSPENDED', '') - if args.mutation == 'sep-syscore': - code = code.replace('dev_pm_syscore_device(list->pd_devs[i], true);', 'dev_pm_syscore_device(list->pd_devs[i], false);') - if args.mutation == 'wake-irq': - code = code.replace('return devm_pm_set_wake_irq(dockchannel->dev, dockchannel->rx_irq);', 'return 0;') - if args.mutation == 'stale-policy': - code = code.replace('generation != READ_ONCE(m->reg_generation)', 'false') + mutations = { + 'sep-syscore': ('dev_pm_syscore_device(list->pd_devs[i], true);', 'dev_pm_syscore_device(list->pd_devs[i], false);'), + 'wake-irq': ('return devm_device_init_wakeup(dockchannel->dev);', + 'return devm_device_init_wakeup(dockchannel->dev) ?: devm_pm_set_wake_irq(dockchannel->dev, dockchannel->rx_irq);'), + 'rx-suspend': ('IRQF_NO_AUTOEN | IRQF_NO_SUSPEND', 'IRQF_NO_AUTOEN'), + 'stale-policy': ('generation != READ_ONCE(m->reg_generation)', 'false'), + 'usb-rpm': ('pm_runtime_put_autosuspend(appledwc->dev);', '(void)0;'), + 'vlp-client': ('if (flags & IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT)', 'if (0)'), + 'six-band': ('(policy->six_channels & BIT_ULL((channel - 1) / 4))', 'true'), + } + if args.mutation: + old, new = mutations[args.mutation] + # A mutation that no longer matches the driver would test nothing. + if code.count(old) != 1: + parser.error(args.mutation + ' mutation no longer matches the driver source') + code = code.replace(old, new) source = args.output/'controls.c'; source.write_text(code) binary = args.output/'controls' subprocess.run(['cc', '-std=gnu11', '-g', '-Wall', '-Werror', '-Wno-unused-function', From 937e9b0336e870bff457eba617d0204f394dd65a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 18:00:01 +1100 Subject: [PATCH 1376/1579] selftests: apple-afkv2: follow compact EPIC and reply lengths in the HPD, sender and EDID tests Since the HPD reply check (e8f48a64e1b1), driver work reached this tree that the extracted units never followed, so three of them stopped compiling. The 14.7 external role closes its AV client on an HPD low (cadaaeb5e040). Compact EPIC on desktop 26.x endpoints (8e82151640fc, 99870822214e) gave the sender a compact path, the HPD request a status word reply and the legacy EDID reader a 2 KiB request. The sender gained a legacy reply-length interface on top of afk_send_command_common() (e6c10627c55e), and the legacy EDID reader now bounds the EDID by the reported reply length (48cbf951457b). HPD: count dcpavserv_v14_close() calls and check that only an HPD low closes the AV client, before the request goes out. On compact EPIC check that a zero status is accepted without an echo and that any nonzero status is refused. Sender: extract afk_send_command_common(), the reply-length wrapper and the compact status allocator. Legacy reply lengths go through the real completer, and the test checks the bounded copy with zero fill, the oversized refusal, the U32 overflow and NULL guards, and that AFK-v2 refuses the interface without sending. Compact requests check the status echo mapping and that no DMA or tag slot is used. EDID: model the reply-length call and the compact buffer size. The legacy reader keeps its checksum tolerance but now refuses a truncated reply, a wrong max_size echo, partial blocks and a short block count. Its compact request asks for 2 KiB and expects that size echoed. Add EOVERFLOW and EOPNOTSUPP to the host errno header. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- .../drivers/gpu/apple-afkv2/extract-edid.py | 46 +++++-- .../drivers/gpu/apple-afkv2/extract-hpd.py | 79 ++++++++++-- .../drivers/gpu/apple-afkv2/extract-send.py | 119 ++++++++++++++---- .../apple-afkv2/host-include/linux/errno.h | 2 + 4 files changed, 202 insertions(+), 44 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py index 0d554d1c82b5d..bcced2f684cfa 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py @@ -29,12 +29,17 @@ def function(name): #define kzalloc(n,f) calloc(1,n) #define cpu_to_le64(x) (x) #define le64_to_cpu(x) (x) +#define READ_ONCE(x) (x) +#define KUNIT_STATIC_STUB_REDIRECT(...) do {} while (0) +#define SZ_2K 2048U +#define EDID_COMPACT_BUF_SIZE (EDID_LEADING_DATA_SIZE + SZ_2K) #define EDID_EXT_BLOCK_COUNT_OFFSET 0x7e #define ERR_PTR(x) ((void*)(intptr_t)(x)) #define IS_ERR(p) ((uintptr_t)(p)>(uintptr_t)-4096) static void cleanup_free(void *p) { free(*(void**)p); } #define __free(x) __attribute__((cleanup(cleanup_free))) -struct apple_epic_service { int unused; }; +struct ep { bool compact; }; +struct apple_epic_service { struct ep *ep; }; struct drm_edid { size_t size; }; struct dpavserv_copy_edid_cmd { u64 max_size;u8 pad1[24];u64 used_size;u8 pad2[8]; }; struct dpavserv_copy_edid_resp { u64 max_size;u8 pad1[24];u64 used_size;u8 pad2[8];u8 data[]; }; @@ -42,26 +47,36 @@ def function(name): static bool truncated; static const struct drm_edid *drm_edid_alloc(const void *p,size_t n) { struct drm_edid *e=malloc(sizeof(*e));(void)p;e->size=n;return e; } -static int afk_service_call(struct apple_epic_service *s,u16 group,u32 command, +/* The request asks for the buffer the transport can carry: 2 KiB on compact EPIC. */ +static size_t request_size(const struct apple_epic_service *s, bool inline_call) +{ return !inline_call && s->ep->compact ? EDID_COMPACT_BUF_SIZE : EDID_BUF_SIZE; } +static int edid_call(struct apple_epic_service *s,bool inline_call,u16 group,u32 command, const void *data,size_t data_len,size_t data_pad,void *output,size_t output_len,size_t output_pad) { const struct dpavserv_copy_edid_cmd *cmd=data; - (void)s;(void)data_pad;(void)output_pad; - assert(group==1 && command==7 && data_len==48 && cmd->max_size==EDID_BUF_SIZE); + size_t want=request_size(s,inline_call); + assert(group==1 && command==7 && data_len==48 && cmd->max_size==want && data_pad==want); + assert(output_len==48+want && output_pad==0); assert(output_len>=sizeof(fixture));memcpy(output,fixture,sizeof(fixture));return 0; } static int afk_service_call_len(struct apple_epic_service *s,u16 group,u32 command, const void *data,size_t data_len,size_t data_pad,void *output,size_t output_len, size_t output_pad,size_t *received) -{ int ret=afk_service_call(s,group,command,data,data_len,data_pad,output,output_len,output_pad); +{ int ret=edid_call(s,true,group,command,data,data_len,data_pad,output,output_len,output_pad); *received=sizeof(fixture)-(truncated?1:0);return ret; } +/* Legacy EPIC: the reply length the firmware reported. */ +static int afk_service_call_with_reply_len(struct apple_epic_service *s,u16 group,u32 command, + const void *data,size_t data_len,size_t data_pad,void *output,size_t output_len, + size_t output_pad,size_t *reply_len) +{ int ret=edid_call(s,false,group,command,data,data_len,data_pad,output,output_len,output_pad); + *reply_len=sizeof(fixture)-(truncated?1:0);return ret; } static void fix_checksum(u8 *p) { unsigned int i;u8 sum=0;p[127]=0;for(i=0;i<127;i++)sum+=p[i];p[127]=-sum; } ''' suffix = r''' int main(void) { - struct apple_epic_service service={0};const struct drm_edid *e; + struct ep ep={0};struct apple_epic_service service={.ep=&ep};const struct drm_edid *e; u8 *base=fixture+56,*extension=base+128; afkv2_put64(fixture,EDID_BUF_SIZE);afkv2_put64(fixture+32,8+256); memset(base+1,255,6);base[126]=1;fix_checksum(base); @@ -72,8 +87,23 @@ def function(name): e=dcpavserv_read_edid_inline(&service);assert(!IS_ERR(e) && e->size==256);free((void*)e); truncated=true; e=dcpavserv_read_edid_inline(&service);assert(IS_ERR(e)); - e=dcpavserv_read_edid_legacy(&service);assert(!IS_ERR(e));free((void*)e); - puts("PASS: actual legacy EDID checksum tolerance preserved; inline checksum/actual-length validation enforced"); + /* The legacy reader now bounds the EDID by the reported reply length too. */ + e=dcpavserv_read_edid_legacy(&service);assert(IS_ERR(e)); + truncated=false; + /* Its max_size echo must match the request, and the EDID must be whole blocks. */ + afkv2_put64(fixture,EDID_BUF_SIZE-1); + e=dcpavserv_read_edid_legacy(&service);assert(IS_ERR(e)); + afkv2_put64(fixture,EDID_BUF_SIZE);afkv2_put64(fixture+32,8+200); + e=dcpavserv_read_edid_legacy(&service);assert(IS_ERR(e)); + afkv2_put64(fixture+32,8+128); + e=dcpavserv_read_edid_legacy(&service);assert(IS_ERR(e)); /* base block announces one extension */ + afkv2_put64(fixture+32,8+256); + /* Compact EPIC asks for 2 KiB and expects that size echoed. */ + ep.compact=true; + e=dcpavserv_read_edid_legacy(&service);assert(IS_ERR(e)); + afkv2_put64(fixture,EDID_COMPACT_BUF_SIZE); + e=dcpavserv_read_edid_legacy(&service);assert(!IS_ERR(e) && e->size==256);free((void*)e); + puts("PASS: actual legacy EDID checksum tolerance preserved with reply-length, echo and block bounds (legacy and compact); inline checksum/actual-length validation enforced"); } ''' pathlib.Path(sys.argv[2]).write_text(prefix + function("dcpavserv_read_edid_legacy") + function("dcpavserv_read_edid_inline") + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-hpd.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-hpd.py index 6957d92f390e2..b21cba6c7665c 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-hpd.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-hpd.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Exercise the production DPTX HPD request with mocked service-call replies.""" +"""Exercise the production DPTX HPD request with mocked service-call replies. + +Covers the legacy, AFK-v2 and compact EPIC replies and the 14.7 AV client close. +""" import pathlib import re import sys @@ -44,12 +47,19 @@ def function(source, name): #define EPIC_SUBTYPE_STD_SERVICE 0xc0 #define dev_warn(...) do {} while (0) #define dev_dbg(...) do {} while (0) +#define READ_ONCE(x) (x) +static u32 get_unaligned_le32(const void *p) +{ + const u8 *b = p; + + return b[0] | b[1] << 8 | b[2] << 16 | (u32)b[3] << 24; +} struct epic_service_call { u8 pad0[2]; u16 group; u32 command, data_len, magic; u8 pad1[48]; }; struct dcp { void *dev; }; struct ops { const char *name; }; -struct ep { struct dcp *dcp; int endpoint; bool inline_v2; }; +struct ep { struct dcp *dcp; int endpoint; bool inline_v2; bool compact; }; struct apple_epic_service { struct ep *ep; struct ops *ops; u32 channel; }; int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, @@ -67,7 +77,16 @@ def function(source, name): */ static bool wire_v2; static size_t declared, arrived; -static u32 echo_flip; +static u32 echo_flip, compact_status; +/* An HPD low closes the 14.7 external AV client first; the stub counts it. */ +static struct dcp *expected_dcp; +static unsigned int v14_closes, closes_at_send; +/* Kept even if a mutant drops the call, so the count check is what fails. */ +__attribute__((unused)) static void dcpavserv_v14_close(struct dcp *dcp) +{ + assert(dcp == expected_dcp); + v14_closes++; +} static int afk_send_command_timeout_len(struct apple_epic_service *s, u8 type, const void *payload, size_t payload_len, void *output, size_t output_len, u32 *retcode, unsigned int timeout_ms, size_t *received) @@ -76,15 +95,21 @@ def function(source, name): u8 *body = (u8 *)output + sizeof(*call); u32 echo; - (void)s; (void)payload; (void)timeout_ms; + (void)payload; (void)timeout_ms; assert(type == EPIC_SUBTYPE_STD_SERVICE && payload_len == output_len); assert(output_len == sizeof(*call) + 32 && call->data_len == 32); assert(call->group == 8 && call->command == 8); + closes_at_send = v14_closes; if (wire_v2 && !received && arrived < output_len - sizeof(*call)) return -EMSGSIZE; memcpy(&echo, body + 16, sizeof(echo)); echo ^= echo_flip; memcpy(body + 16, &echo, sizeof(echo)); + /* Compact EPIC answers with an IOReturn status in the first word. */ + if (s->ep->compact) { + for (int i = 0; i < 4; i++) + body[i] = compact_status >> (8 * i); + } call->data_len = declared; *retcode = 0; if (received) @@ -94,12 +119,24 @@ def function(source, name): ''' suffix = r''' static int failures; +static struct ep *the_ep; static void expect(bool ok, const char *what, bool hpd, int ret) { - printf("%s %s: %s hpd=%d declared=%zu arrived=%zu ret=%d\n", ok ? "ok" : "FAIL", - wire_v2 ? "afk-v2" : "legacy", what, hpd, declared, arrived, ret); + printf("%s %s%s: %s hpd=%d declared=%zu arrived=%zu ret=%d\n", ok ? "ok" : "FAIL", + wire_v2 ? "afk-v2" : "legacy", the_ep->compact ? " compact" : "", what, hpd, + declared, arrived, ret); failures += !ok; } +/* Every request closes the AV client exactly when it lowers HPD, and before it is sent. */ +static int set_hpd(struct apple_epic_service *service, int hpd) +{ + unsigned int before = v14_closes; + int ret = dptxport_set_hpd_timeout(service, hpd, 1000); + + expect(v14_closes == before + !hpd && closes_at_send == before + !hpd, + "AV client closed before an HPD low only", hpd, ret); + return ret; +} int main(void) { static const size_t lengths[] = { 0, 4, 8, 12, 16, 20, 32, 36, 65535 }; @@ -108,6 +145,9 @@ def function(source, name): struct ep ep = { .dcp = &dcp }; struct apple_epic_service service = { .ep = &ep, .ops = &ops, .channel = 1 }; + the_ep = &ep; + expected_dcp = &dcp; + for (int v2 = 0; v2 < 2; v2++) { wire_v2 = v2; for (size_t i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) { @@ -115,7 +155,7 @@ def function(source, name): int ret; declared = lengths[i]; arrived = 32; echo_flip = 0; - ret = dptxport_set_hpd_timeout(&service, hpd, 1000); + ret = set_hpd(&service, hpd); if (declared < sizeof(struct dcpdptx_hotplug_cmd)) expect(ret < 0, "short reply refused", hpd, ret); else if (declared > 32) @@ -128,19 +168,38 @@ def function(source, name): int ret; declared = 20; arrived = 32; echo_flip = 1; - ret = dptxport_set_hpd_timeout(&service, hpd, 1000); + ret = set_hpd(&service, hpd); expect(ret == -EINVAL, "wrong echo refused", hpd, ret); if (!wire_v2) continue; declared = 20; arrived = 16; echo_flip = 0; - ret = dptxport_set_hpd_timeout(&service, hpd, 1000); + ret = set_hpd(&service, hpd); expect(ret < 0, "reply cut short in transit refused", hpd, ret); } + /* Compact EPIC: the status word decides, there is no echo to compare. */ + ep.compact = true; + for (int hpd = 0; hpd < 2; hpd++) { + int ret; + + declared = 20; arrived = 32; echo_flip = 0; compact_status = 0; + ret = set_hpd(&service, hpd); + expect(ret == 0, "zero status accepted", hpd, ret); + echo_flip = 1; + ret = set_hpd(&service, hpd); + expect(ret == 0, "echo word ignored", hpd, ret); + echo_flip = 0; compact_status = 0xe00002c2; + ret = set_hpd(&service, hpd); + expect(ret == -EIO, "firmware error status refused", hpd, ret); + compact_status = 0x00000100; + ret = set_hpd(&service, hpd); + expect(ret == -EIO, "any nonzero status byte refused", hpd, ret); + } + ep.compact = false; compact_status = 0; } printf("%d HPD reply checks failed\n", failures); if (failures) return 1; - puts("PASS: extracted production DPTX HPD request: short, oversized, wrong-echo and complete replies, legacy and AFK-v2"); + puts("PASS: extracted production DPTX HPD request: short, oversized, wrong-echo and complete replies, legacy, AFK-v2 and compact status, AV client close on HPD low"); return 0; } ''' diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py index 70dbd5d3a69f4..eb55dd281456f 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py @@ -1,25 +1,27 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Compile the real command sender against synthetic ring and DMA operations.""" +"""Compile the real command sender against synthetic ring and DMA operations. + +Covers AFK-v2, legacy reply-length and compact EPIC requests. +""" import pathlib import re import sys source = pathlib.Path(sys.argv[1]).read_text() -match = re.search(r"static int afk_send_command_timeout_len\([^;]*?\)\n\{", source) -if not match: - raise RuntimeError("production command sender not found") -start, end, depth = match.start(), match.end(), 1 -while depth: - depth += (source[end] == "{") - (source[end] == "}") - end += 1 -reply_match = re.search(r"static void afk_recv_handle_reply\([^;]*?\)\n\{", source) -if not reply_match: - raise RuntimeError("production reply completer not found") -reply_end, reply_depth = reply_match.end(), 1 -while reply_depth: - reply_depth += (source[reply_end] == "{") - (source[reply_end] == "}") - reply_end += 1 + + +def function(pattern, what): + match = re.search(r"^" + pattern + r"\([^;]*?\)\n\{", source, re.M) + if not match: + raise RuntimeError("production " + what + " not found") + end, depth = match.end(), 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start():end] + "\n" + + prefix = r''' #undef NDEBUG #include @@ -27,7 +29,10 @@ #include #include #include "afkv2-codec.h" +#include "afk_reply.h" #define SZ_1M 1048576U +#define U32_MAX 0xffffffffU +#define MSEC_PER_SEC 1000U #define GFP_KERNEL 0 #define MAX_PENDING_CMDS 16 #define AFKV2_INTERFACES (1U << 16) @@ -55,10 +60,13 @@ struct cmd_info { u16 tag; u8 type; size_t reply_length; void *rxbuf,*txbuf; dma_addr_t rxbuf_dma,txbuf_dma; size_t rxlen,txlen; u32 retcode; + size_t reply_len; bool reply_len_valid; bool done,free_on_ack; int *completion; }; -struct dcp { void *dev; bool quiescing; }; -struct apple_dcp_afkep { struct dcp *dcp; bool wire_v2,inline_v2; int endpoint,lock;u8 *command_tags; }; +struct dcp { void *dev; bool quiescing,external; void *rtk; }; +struct afk_ringbuffer { bool ready; }; +struct apple_dcp_afkep { struct dcp *dcp; bool wire_v2,inline_v2,compact; int endpoint,lock;u8 *command_tags; + struct afk_ringbuffer txbfr,rxbfr; }; struct ops { const char *name; }; struct apple_epic_service { struct apple_dcp_afkep *ep; struct ops *ops; unsigned int channel; unsigned long cmd_map[1]; @@ -68,6 +76,25 @@ static unsigned int frees, sends; static int wire_fail, timeout, reply_status; static size_t owned_reply_size=32888; +/* Legacy replies can go through the real completer, as from the ring. */ +static bool real_reply; +static u32 legacy_rxlen; +static void afk_recv_handle_reply(struct apple_dcp_afkep *ep,u32 channel,u16 tag,void *payload,size_t payload_size); +/* Compact EPIC goes inline through the ring; this mock stands for that transport. */ +static bool compact_echo; +static u32 compact_reply,compact_request; +static size_t compact_copied; +static unsigned int compact_sends; +static int afk_send_compact_command(struct apple_epic_service *s,u8 type,const void *payload,size_t payload_len, + void *output,size_t output_len,u32 request_status,u32 *retcode,size_t *copied,unsigned long timeout) +{ + (void)timeout; + assert(s==active && type==0xc0 && payload_len==112 && ((const u8 *)payload)[0]==0x5a && output_len==32888); + compact_sends++;compact_request=request_status; + *retcode=compact_echo?request_status:compact_reply; + memset(output,0x33,compact_copied);*copied=compact_copied; + return 0; +} static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) { if(!s->enabled) return NULL; s->users++; return s; } static void afk_service_put(struct apple_epic_service *s) { assert(s->users); s->users--; } @@ -100,6 +127,14 @@ } if(wire_fail)return -ENOSPC; if(timeout)return 0; + if(real_reply) { + struct epic_cmd reply={.retcode=reply_status,.rxlen=legacy_rxlen}; + assert(!ep->wire_v2); + memset(c->rxbuf,0x77,legacy_rxlenrxlen?legacy_rxlen:c->rxlen); + afk_recv_handle_reply(ep,channel,tag,&reply,sizeof(reply)); + assert(c->done && c->reply_len_valid && c->reply_len==legacy_rxlen && *c->completion==1); + return 0; + } c->reply_length=ep->inline_v2?4:owned_reply_size;c->retcode=reply_status; memset(c->rxbuf,0x77,c->reply_length);c->done=true;*c->completion=1; return 0; @@ -139,6 +174,39 @@ ep.wire_v2=false;owned_reply_size=4; assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); assert(received==32888 && s.users==1); + /* Legacy EPIC reply lengths: bounded by the buffer, rest of the output zeroed. */ + real_reply=true;legacy_rxlen=4;size_t rlen=99;memset(output,0x19,32888); + assert(!afk_send_command_with_reply_len(&s,0xc0,request,112,output,32888,&status,&rlen)); + assert(rlen==4 && status==0 && output[0]==0x77 && output[3]==0x77 && !output[4] && !output[32887]); + assert(!s.cmd_map[0] && s.users==1); + legacy_rxlen=32889;rlen=99; + assert(afk_send_command_with_reply_len(&s,0xc0,request,112,output,32888,&status,&rlen)==-EMSGSIZE); + assert(rlen==0 && !s.cmd_map[0] && s.users==1); + assert(afk_send_command_with_reply_len(&s,0xc0,request,112,output,32888,&status,NULL)==-EINVAL); + unsigned int before_sends=sends; + assert(afk_send_command_with_reply_len(&s,0xc0,request,(size_t)U32_MAX+1,output,32888,&status,&rlen)==-EOVERFLOW); + /* AFK-v2 reports reply lengths its own way and refuses this interface. */ + ep.wire_v2=true; + assert(afk_send_command_with_reply_len(&s,0xc0,request,112,output,32888,&status,&rlen)==-EOPNOTSUPP); + assert(sends==before_sends && s.users==1 && !s.cmd_map[0]); + real_reply=false; + /* Compact EPIC: success is the reply echoing the request's own status. */ + ep.compact=true;unsigned int before_frees=frees;status=99;received=99; + compact_echo=true;compact_copied=20; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(status==0 && received==20 && output[0]==0x33 && compact_sends==1); + u32 first=compact_request; + compact_echo=false;compact_reply=0xe00002c2; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(status==0xe00002c2 && compact_request==((((first>>8)+1)&0xff)<<8)); + compact_reply=0; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(compact_request && status==~0U); + compact_echo=true;compact_copied=8;rlen=99; + assert(!afk_send_command_with_reply_len(&s,0xc0,request,112,output,32888,&status,&rlen) && rlen==8 && !status); + assert(compact_sends==4 && sends==before_sends && frees==before_frees && s.users==1 && !s.cmd_map[0]); + ep.compact=false; + dcp.external=true;ep.txbfr.ready=ep.rxbfr.ready=true; ep.wire_v2=true;owned_reply_size=32888;timeout=1; assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)==-ETIMEDOUT); assert(s.cmd_map[0] && s.cmds[0].free_on_ack && !s.cmds[0].completion); @@ -155,14 +223,13 @@ s.enabled=true;s.torndown=false;timeout=0; assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); assert(!s.cmd_map[0] && s.users==1);free(output); - puts("PASS: extracted production sender, 128-byte command envelope / 32888-byte reply, short OOB, strict/legacy callers and actual late-ACK ownership"); + puts("PASS: extracted production sender, 128-byte command envelope / 32888-byte reply, short OOB, strict/legacy callers, legacy reply lengths, compact status and actual late-ACK ownership"); } ''' -tag_match = re.search(r"static u16 afk_command_tag\([^;]*?\)\n\{", source) -if not tag_match: - raise RuntimeError("production interface tag allocator not found") -tag_end, tag_depth = tag_match.end(), 1 -while tag_depth: - tag_depth += (source[tag_end] == "{") - (source[tag_end] == "}") - tag_end += 1 -pathlib.Path(sys.argv[2]).write_text(prefix + source[tag_match.start():tag_end] + source[start:end] + source[reply_match.start():reply_end] + suffix) +bodies = [function(r"static u32 afk_compact_next_status", "compact status allocator"), + function(r"static u16 afk_command_tag", "interface tag allocator"), + function(r"static int afk_send_command_common", "command sender"), + function(r"static int afk_send_command_timeout_len", "length-aware sender"), + function(r"int afk_send_command_with_reply_len", "legacy reply-length sender"), + function(r"static void afk_recv_handle_reply", "reply completer")] +pathlib.Path(sys.argv[2]).write_text(prefix + "".join(bodies) + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h index 23e11801f1e76..e7a1050305b68 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h @@ -8,3 +8,5 @@ #define ENODEV 19 #define ETIMEDOUT 110 #define ENOMEM 12 +#define EOVERFLOW 75 +#define EOPNOTSUPP 95 From 76378dd6ec7af7beae4a0ae11e15089703ff9564 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 18:01:17 +1100 Subject: [PATCH 1377/1579] ci: run every host test on each push and pull request The host tests extract production code from this tree and build it on the host. Nothing ran them automatically, so a driver change could break one while the kernel build stayed green, and by 2026-10-30 seventeen of them had stopped passing. Add tools/asahi/run-host-tests.py. It runs every tools/asahi test under python3 -I in its own directory, together with its mutation and old-code controls, plus the Rust and shell tests there and the apple-dcp, apple- afkv2 (gcc and clang, with sanitizers), asahi-g15 and asahi-m4 selftests. A mutation only counts as killed when the built control reports the failure, so a mutant that just stops compiling fails the run. A tools/asahi test that is not registered in the runner also fails it, so new tests cannot be skipped by accident. Add a workflow that runs it on ubuntu-24.04 with pinned Python 3.14, rustc 1.96.0, clang 18 and gcc 13, for every push and pull request on integration/aurora and aurora-wip. The checkout keeps the commit history without trees so the old-code controls can read their revisions. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- .github/workflows/host-tests.yml | 87 +++++++++++ tools/asahi/run-host-tests.py | 250 +++++++++++++++++++++++++++++++ 2 files changed, 337 insertions(+) create mode 100644 .github/workflows/host-tests.yml create mode 100755 tools/asahi/run-host-tests.py diff --git a/.github/workflows/host-tests.yml b/.github/workflows/host-tests.yml new file mode 100644 index 0000000000000..73a3c67c1842b --- /dev/null +++ b/.github/workflows/host-tests.yml @@ -0,0 +1,87 @@ +name: Host tests + +# Every host-side test of the Aurora drivers: tools/asahi (with their mutation +# and old-code controls) and the apple-dcp, apple-afkv2, asahi-g15 and asahi-m4 +# selftests. They extract production code from this tree and build it on the +# host, so a driver change can break them while the kernel build stays green. + +on: + push: + branches: [integration/aurora, aurora-wip] + pull_request: + branches: [integration/aurora, aurora-wip] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: host-tests-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + RUST_VERSION: '1.96.0' + CLANG_VERSION: '18' + GCC_VERSION: '13' + +jobs: + host-tests: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + # Negative controls read older revisions with `git show`, so keep the + # whole history. Trees and blobs of old commits load on demand. + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + filter: tree:0 + + - uses: actions/setup-python@v5 + with: + python-version: '3.14' + + - name: Cache the Rust toolchain + uses: actions/cache@v4 + with: + path: ~/.rustup/toolchains/${{ env.RUST_VERSION }}-x86_64-unknown-linux-gnu + key: rustup-${{ runner.os }}-${{ env.RUST_VERSION }} + + - name: Pin the host toolchains + run: | + rustup toolchain install "$RUST_VERSION" --profile minimal --no-self-update + echo "RUSTUP_TOOLCHAIN=$RUST_VERSION" >> "$GITHUB_ENV" + need= + for p in clang-$CLANG_VERSION gcc-$GCC_VERSION flex bison bc; do + command -v "$p" >/dev/null || need="$need $p" + done + dpkg -s "libclang-rt-$CLANG_VERSION-dev" >/dev/null 2>&1 || need="$need libclang-rt-$CLANG_VERSION-dev" + if [ -n "$need" ]; then + sudo apt-get update -q + sudo apt-get install -y -q $need + fi + # The tests call clang, cc, gcc and rustc by name. + bin="$RUNNER_TEMP/toolchain" + mkdir -p "$bin" + ln -sf "$(command -v clang-$CLANG_VERSION)" "$bin/clang" + ln -sf "$(command -v gcc-$GCC_VERSION)" "$bin/gcc" + ln -sf "$(command -v gcc-$GCC_VERSION)" "$bin/cc" + echo "$bin" >> "$GITHUB_PATH" + + - name: Show the toolchains + run: | + python3 --version + rustc --version + clang --version | head -1 + cc --version | head -1 + git --version + + - name: Run the host tests + run: python3 -I tools/asahi/run-host-tests.py --jobs "$(nproc)" --work "$RUNNER_TEMP/host-tests" + + - name: Keep the logs of a failed run + if: failure() + uses: actions/upload-artifact@v4 + with: + name: host-test-logs + path: ${{ runner.temp }}/host-tests/**/log + retention-days: 14 diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py new file mode 100755 index 0000000000000..bcc42fe042c0f --- /dev/null +++ b/tools/asahi/run-host-tests.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Run every Aurora host test: tools/asahi and the GPU/display selftests. + +Each tools/asahi test runs under `python3 -I` with its own output directory, +once as a control and once per mutation or old-code negative control listed +below. A mutation run is only accepted as a kill when the test reports one +(an assertion, an abort or a panic of the built control), so a mutant that +merely stops compiling is a failure, not a pass. + +Every tools/asahi/*-tests.{py,rs,sh} file must appear here. A new test that +is not registered fails the run, so it cannot be silently skipped. + +Negative controls read older revisions with `git show`, so the checkout needs +the commits (a full or treeless clone, not a depth-1 one). + +Usage: tools/asahi/run-host-tests.py [--jobs N] [--work DIR] [--only SUBSTR] +""" +import argparse +import concurrent.futures +import os +import re +import shlex +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +ASAHI = ROOT / 'tools/asahi' +GPU = ROOT / 'tools/testing/selftests/drivers/gpu' + +PASS = 'pass' +KILL = 'kill' + +# A killed mutant shows the failing control: a C assertion or abort (-6), or +# a Rust panic (exit 101). A Python traceback means the harness itself broke. +KILL_SIGN = re.compile(r'Assertion|": -6\b|"exit": 101\b|panicked at') +BROKEN_SIGN = re.compile(r'Traceback \(most recent call last\)') + + +def out(flag='--out'): + return [flag, '{out}'] + + +def mutations(flag, names, base, expect=KILL): + return [(base + [flag, n], expect) for n in names] + + +# test file -> [(arguments, expectation)]. '{out}' is a fresh directory. +ASAHI_TESTS = { + 'apple-dart-fw-tests.py': [(out(), PASS)], + 'apple-dart-range-tests.py': [([], PASS)], + 'atc-usb3-tests.py': [(out(), PASS), (out() + ['--mutations'], PASS)], + 'avd-av1-tail-tests.py': [(out(), PASS)] + + [(out() + ['--case', c], PASS) for c in ('copy', 'seed', 'aligned')] + + mutations('--mutation', ('round-up', 'cpu-unaligned', 'seg-cpu-unaligned', + 'seg-any-size', 'ref-index-upper', 'frame-width-unchecked'), out()), + 'avd-codec-tail-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], + # The old revision is the one before the J493 hardware-table ID match. + 'bcm4377-id-tests.py': [(out() + ['--old-ref', '87ddae1397f6f70817cb57df76477a86bdf96424'], PASS)], + # These mutations are caught inside the test, which then exits 0. + 'bcm4377-resume-tests.py': [(out(), PASS)] + + mutations('--mutation', ('epoch', 'close', 'producer-fence', 'active-fence', 'board'), out(), PASS), + 'brcmf-assoc-tests.py': [(out('--output'), PASS)] + + mutations('--mutation', ('short-ie', 'header-length', 'skb-capacity', 'cmplt-header'), out('--output')), + 'brcmf-bss-tests.py': [(out(), PASS)], + 'brcmf-pcie-dma-tests.py': [(out('--output'), PASS)] + + mutations('--mutation', ('narrow-stream', 'ignore-aperture', 'revision-wide'), out('--output')), + 'brcmf-roam-tests.py': [(out(), PASS)], + 'brcmfmac-scan-timeout-tests.py': [(out('--output'), PASS)], + 'clpc-gpu-demand-tests.py': [([], PASS)], + # The old policy always disabled compute preemption. + 'compute-preempt-tests.py': [(out(), PASS), + (out() + ['--old-ref', '6dbfbcb21eb09d07ae59037a1ceca8cc3500841f'], PASS)], + 'dcp14-backlight-tests.py': [([], PASS)], + 'dcp14-panel-tests.py': [([], PASS)], + 'devcoredump-init-tests.py': [([], PASS)], + 'display26-tests.py': [([], PASS)], + # The old revision publishes the fd before copy_to_user() succeeds. + 'dma-heap-fd-tests.py': [(out() + ['--old-ref', 'cb732d027aa18e1fcf9d2797f47d20b179ebc59c'], PASS)], + 'g15-input-error-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS), + (out('--output') + ['--ref', 'ede99b3c1990b4a4a8c045fdf627a2bd0f14446a'], PASS)], + 'g15-maintained-handoff-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], + 'g15-pb-growth-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], + 'g15-probe-init-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], + 'g15-queue-credit-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], + 'g15-retention-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], + 'g15-stats-drain-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], + 'g15-zero-words-tests.py': [([], PASS), (['--mutations'], PASS)], + 'g17-admission-deadline-tests.py': [([], PASS)], + 'g17-admission-order-tests.py': [([], PASS)], + 'g17-capacity-fault-tests.py': [([], PASS)], + 'g17-compute-constructor-tests.py': [([], PASS)], + 'g17-compute-deadline-tests.py': [([], PASS)], + 'g17-compute-growth-deadline-tests.py': [([], PASS)], + 'g17-firmware-selection-tests.py': [([], PASS)], + 'g17-growth-gate-tests.py': [([], PASS)], + 'g17-growth-tests.py': [([], PASS)], + 'g17-owner-release-order-tests.py': [([], PASS)], + 'g17-pool-release-tests.py': [([], PASS)], + 'g17-resource-tests.py': [([], PASS)], + 'g17-shared-pool-tests.py': [([], PASS)], + 'g17-uninstalled-cleanup-tests.py': [([], PASS)], + 'g17-unpublished-tests.py': [([], PASS)], + 'mt7932-association-events-tests.py': [(out(), PASS)], + 'mt7932-connect-result-tests.py': [(out('--output'), PASS)], + 'mt7932-owe-retry-tests.py': [(out(), PASS)], + 'mt7932-policy-6ghz-tests.py': [([], PASS)], + 'neo-auxiliary-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutation'], KILL), + (out('--output') + ['--old'], KILL)], + 'neo-stability-tests.py': [(out('--output'), PASS)] + + mutations('--mutation', ('sep-syscore', 'wake-irq', 'rx-suspend', 'stale-policy', + 'usb-rpm', 'vlp-client', 'six-band'), out('--output')) + + [(out('--output') + ['--old-usb'], KILL), (out('--output') + ['--old-regulatory'], KILL)], + 'pmp-j613-hwmon-tests.py': [([], PASS)], + 'qspi-nvram-tests.py': [(out('--output'), PASS)], + 'rtkit-oslog-tests.py': [(out(), PASS)] + + mutations('--mutate', ('admission', 'channel', 'counter', 'reset', 'buffer'), out()), + 't6030-clock-tests.py': [(out(), PASS)], + 't6030-display-admission-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], + 't6030-display-linkage-tests.py': [(out('--output') + ['--conf', '{conf}'], PASS)], + 't6030-pmp-graph-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], + 't6030-pmp-supplier-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], + 't6030-wiring-tests.py': [(out(), PASS), (out() + ['--old-classifier', 'tb'], PASS), + (out() + ['--old-classifier', 'dcp'], PASS)], + 't6040-calibration-tests.py': [([], PASS)], + 'tb-dp-repair-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], + 'tbnet-refill-tests.py': [(out(), PASS)] + + mutations('--mutant', ('refill-count', 'no-retry', 'no-cancel', 'worker-allocation'), out()), + 'tiled-split-tests.py': [(out(), PASS), (out() + ['--mutations'], PASS)], + 'tunnel-route-tests.py': [(out(), PASS)], +} + +# Rust unit-test files built with `rustc --test`, and shell drivers. +ASAHI_RUST = ['m3-batching-tests.rs', 'm3-runtime-tests.rs', 'pmp-bootargs-tests.rs', 'pmp-power-tests.rs'] +ASAHI_SHELL = ['g17-ray-query-timeout-tests.sh'] + +# Python that runs on the host but imports its neighbours, so it keeps the +# script directory on sys.path (-E -s instead of -I). +SUITES = [ + ('apple-dcp', GPU / 'apple-dcp', ['python3', '-E', '-s', '-B', 'run.py', '--output', '{out}']), + ('asahi-g15', GPU / 'asahi-g15', ['python3', '-E', '-s', '-B', 'run.py', '--output', '{out}']), + ('asahi-m4', GPU / 'asahi-m4', ['python3', '-E', '-s', '-B', 'run-host-tests.py']), + ('apple-afkv2-gcc', GPU / 'apple-afkv2', ['sh', 'run-afkv2-tests.sh']), + ('apple-afkv2-clang', GPU / 'apple-afkv2', ['sh', 'run-afkv2-tests.sh']), + ('t8140-external-bw', GPU, ['python3', '-I', '-B', 'test_t8140_external_bw.py']), +] + + +def jobs(work, conf): + found = {p.name for p in ASAHI.iterdir() + if re.search(r'-tests\.(py|rs|sh)$', p.name) and p.name != Path(__file__).name} + known = set(ASAHI_TESTS) | set(ASAHI_RUST) | set(ASAHI_SHELL) + missing = sorted(found - known) + gone = sorted(known - found) + if missing or gone: + for name in missing: + print(f'not registered in {Path(__file__).name}: tools/asahi/{name}', file=sys.stderr) + for name in gone: + print(f'registered but missing: tools/asahi/{name}', file=sys.stderr) + sys.exit(2) + result = [] + # The suites take longest, so start them first. + for name, cwd, cmd in SUITES: + d = work / name + # Only ASan catches an unbounded legacy reply copy, so keep it on. + env = {'AFKV2_SANITIZERS': '1', 'HOSTCC': name.rsplit('-', 1)[1]} if name.startswith('apple-afkv2') else {} + result.append((name, [a.replace('{out}', str(d / 'out')) for a in cmd], cwd, PASS, d, env)) + for name, runs in ASAHI_TESTS.items(): + for index, (args, expect) in enumerate(runs): + label = name.removesuffix('.py') + ('' if not args or args == out() or args == out('--output') + else ' ' + ' '.join(a for a in args if a not in ('{out}', '--out', '--output', '{conf}', '--conf'))) + d = work / f'{name}.{index}' + cmd = ['python3', '-I', '-B', str(ASAHI / name)] + [a.replace('{out}', str(d / 'out')).replace('{conf}', str(conf)) for a in args] + result.append((label, cmd, ASAHI, expect, d, {})) + for name in ASAHI_RUST: + d = work / name + binary = d / 'tests' + cmd = ['sh', '-c', f'{os.environ.get("RUSTC", "rustc")} --edition=2021 --test {shlex.quote(str(ASAHI / name))} ' + f'-o {shlex.quote(str(binary))} && {shlex.quote(str(binary))}'] + result.append((name.removesuffix('.rs'), cmd, ASAHI, PASS, d, {})) + for name in ASAHI_SHELL: + d = work / name + result.append((name.removesuffix('.sh'), ['sh', str(ASAHI / name), str(d / 'out')], ROOT, PASS, d, {})) + return result + + +def run(job, timeout): + label, cmd, cwd, expect, d, extra = job + (d / 'tmp').mkdir(parents=True, exist_ok=True) + env = dict(os.environ, TMPDIR=str(d / 'tmp'), **extra) + start = time.monotonic() + try: + p = subprocess.run(cmd, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, errors='replace', timeout=timeout) + rc, text = p.returncode, p.stdout + except subprocess.TimeoutExpired as e: + rc, text = 'timeout', (e.stdout or b'').decode(errors='replace') if isinstance(e.stdout, bytes) else (e.stdout or '') + took = time.monotonic() - start + (d / 'log').write_text(f'$ {shlex.join(cmd)}\n{text}\nexit: {rc}\n') + if expect == PASS: + ok = rc == 0 + why = '' if ok else f'exit {rc}' + else: + ok = rc not in (0, 'timeout') and KILL_SIGN.search(text) and not BROKEN_SIGN.search(text) + why = '' if ok else (f'mutant survived (exit {rc})' if rc == 0 else f'exit {rc} without a control failure') + return label, bool(ok), why, took, d / 'log' + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split('\n', 1)[0]) + ap.add_argument('--jobs', type=int, default=os.cpu_count() or 2) + ap.add_argument('--work', type=Path, help='keep logs and build products here (default: a temporary directory)') + ap.add_argument('--conf', type=Path, help='a built scripts/kconfig/conf (default: build one)') + ap.add_argument('--only', help='run only jobs whose name contains this text') + ap.add_argument('--timeout', type=int, default=900) + args = ap.parse_args() + + work = args.work or Path(tempfile.mkdtemp(prefix='aurora-host-tests-')) + work = work.resolve() + work.mkdir(parents=True, exist_ok=True) + conf = args.conf + if conf is None: + build = work / 'kconfig' + subprocess.run(['make', '-s', '-C', str(ROOT), f'O={build}', 'ARCH=arm64', 'allnoconfig'], + check=True, stdout=subprocess.DEVNULL) + conf = build / 'scripts/kconfig/conf' + todo = jobs(work, conf.resolve()) + if args.only: + todo = [j for j in todo if args.only in j[0]] + print(f'{len(todo)} runs, {args.jobs} at a time, logs in {work}', flush=True) + failed = [] + start = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool: + for label, ok, why, took, log in pool.map(lambda j: run(j, args.timeout), todo): + print(f'{"ok" if ok else "FAIL"} {label} ({took:.1f}s){"" if ok else ": " + why}', flush=True) + if not ok: + failed.append((label, log)) + print(f'{len(todo) - len(failed)}/{len(todo)} passed in {time.monotonic() - start:.0f}s') + for label, log in failed: + text = log.read_text(errors='replace').splitlines() + print(f'\n=== {label}: last lines of {log}') + print('\n'.join(text[-40:])) + return 1 if failed else 0 + + +if __name__ == '__main__': + sys.exit(main()) From 08c62cc8d71a6b66fb086b46a76dedebf9a96270 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 18:08:58 +1100 Subject: [PATCH 1378/1579] selftests: asahi-m4: find newer UAPI headers in the tree for the layout probe The C layout probe includes the tree's asahi_drm.h, which pulls in drm_mode.h and so . That header only exists in recent kernel UAPI exports. On a host with older ones, such as Ubuntu 24.04's 6.8 linux-libc-dev, the probe stopped compiling and test-uapi failed. Search the tree's include/uapi after the system headers. Headers the host already has still come from the host, and only the missing ones come from the tree. The measured layout is unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- tools/testing/selftests/drivers/gpu/asahi-m4/abi.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/abi.py b/tools/testing/selftests/drivers/gpu/asahi-m4/abi.py index 38c6d7806eabd..7b9bbce635e6e 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/abi.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/abi.py @@ -104,7 +104,9 @@ def c_layout(header, bindings, output, tag): lines += [f'printf("{key}=%llu\\n", (unsigned long long)({expr}));' for key, expr in keys] source.write_text('\n'.join(lines + ['return 0;', '}']) + '\n') return run_measurements(shlex.split(os.environ.get('CC', 'cc')) + - ['-std=gnu11', '-Wall', '-Werror', '-D__user='], + ['-std=gnu11', '-Wall', '-Werror', '-D__user=', + # Fall back to the tree for UAPI headers the host lacks. + '-idirafter', str(header.parents[1])], source, output / tag) From 2c5ba39fa5f1ccc0317169f03e4ee96fab88f235 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 18:08:58 +1100 Subject: [PATCH 1379/1579] ci: host tests: provide ld.lld and use the pinned rustc everywhere The first run on ubuntu-24.04 found two gaps in the toolchain setup. The apple-dcp usb-kconfig case configures the tree with LLVM=1, which needs ld.lld. The g15-pb-growth and g15-stats-drain tests ask rustup for their own toolchain unless HOSTRUSTC names a compiler. Install lld 18 next to clang 18, and point HOSTRUSTC at the pinned rustc. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- .github/workflows/host-tests.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/host-tests.yml b/.github/workflows/host-tests.yml index 73a3c67c1842b..f16fb33953f7c 100644 --- a/.github/workflows/host-tests.yml +++ b/.github/workflows/host-tests.yml @@ -50,19 +50,23 @@ jobs: run: | rustup toolchain install "$RUST_VERSION" --profile minimal --no-self-update echo "RUSTUP_TOOLCHAIN=$RUST_VERSION" >> "$GITHUB_ENV" + # Tests that would pick their own rustup toolchain use this one. + echo "HOSTRUSTC=rustc" >> "$GITHUB_ENV" need= for p in clang-$CLANG_VERSION gcc-$GCC_VERSION flex bison bc; do command -v "$p" >/dev/null || need="$need $p" done + command -v "ld.lld-$CLANG_VERSION" >/dev/null || need="$need lld-$CLANG_VERSION" dpkg -s "libclang-rt-$CLANG_VERSION-dev" >/dev/null 2>&1 || need="$need libclang-rt-$CLANG_VERSION-dev" if [ -n "$need" ]; then sudo apt-get update -q sudo apt-get install -y -q $need fi - # The tests call clang, cc, gcc and rustc by name. + # The tests call clang, ld.lld, cc and gcc by name. bin="$RUNNER_TEMP/toolchain" mkdir -p "$bin" ln -sf "$(command -v clang-$CLANG_VERSION)" "$bin/clang" + ln -sf "$(command -v ld.lld-$CLANG_VERSION)" "$bin/ld.lld" ln -sf "$(command -v gcc-$GCC_VERSION)" "$bin/gcc" ln -sf "$(command -v gcc-$GCC_VERSION)" "$bin/cc" echo "$bin" >> "$GITHUB_PATH" @@ -72,6 +76,7 @@ jobs: python3 --version rustc --version clang --version | head -1 + ld.lld --version cc --version | head -1 git --version From 42660490d8fe92baae3665c6d4c12f570b233c4d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 18:33:19 +1100 Subject: [PATCH 1380/1579] media: apple: ave: start sessions, encode frames and halt the firmware Fill in the parts of the J700 encoder driver that were left returning -EOPNOTSUPP, so a V4L2 client gets H.264 or HEVC out of NV12 frames. Session start builds the Start command from the encoding parameters: picture and coded size, cropping, profile, level (raised to what size, frame rate and bit rate need), VBR with a one-second data rate cap or CBR, QP range, GOP length, the B-frame structure and frame rate. The driver allocates the encoder working buffers with sizes derived from the picture size and lists them in the command. Frames go out as Process commands over a ring of five slots (three without B-frames). The firmware reorders B-frames itself and answers in bitstream order, so CAPTURE buffers come back in that order with the timestamp of the frame they code. The first frame is sent alone and followed by a Complete, as macOS does. Process carries the input planes, frame number, presentation time, the current bit rate (it may change while streaming) and the forced key frame request. HEVC output is assembled from the parameter sets, the slice header and the slice data the firmware leaves in three buffers. For H.264 the length is not known yet, so the coded buffer is read as Annex B up to where the data ends. Odd output is reported once per session with a dump under dynamic debug. Commands no longer wait for their answer one at a time. Each command buffer is tracked until the firmware acknowledges it, answers are matched by command id, and frame answers by session and slot. A short poller collects frame answers while frames are outstanding, since the firmware interrupt is not wired up. Drain sends four Completes, then Priority, Stop and Close, and the firmware is halted after every session. Halt is never answered; after a short wait the coprocessor counts as stopped, runtime suspend may drop the power domains, and the next session restarts the firmware through the boot path from the saved DATA copy. A firmware that stops answering keeps its power and its buffers, and is not used again until reboot. The constant words of Start and Process are not built into the driver. It loads them per firmware build from apple/ave-t8140--{h264,hevc}-{start,process}.bin and overwrites every session and frame field. Without those files STREAMON fails with ENOENT. Field layouts are facts from our private research (macOS traces); no copied code. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- Documentation/admin-guide/media/apple-ave.rst | 60 +- drivers/media/platform/apple/ave/Kconfig | 5 +- drivers/media/platform/apple/ave/Makefile | 2 +- drivers/media/platform/apple/ave/ave-cmd.c | 622 ++++++++++++++++-- drivers/media/platform/apple/ave/ave-drv.c | 51 +- drivers/media/platform/apple/ave/ave-fw.c | 180 ++++- drivers/media/platform/apple/ave/ave-ipc.c | 307 ++++++--- drivers/media/platform/apple/ave/ave-regs.h | 29 +- drivers/media/platform/apple/ave/ave-sess.c | 493 ++++++++++++++ drivers/media/platform/apple/ave/ave-v4l2.c | 519 ++++++++++++--- drivers/media/platform/apple/ave/ave.h | 161 ++++- 11 files changed, 2093 insertions(+), 336 deletions(-) create mode 100644 drivers/media/platform/apple/ave/ave-sess.c diff --git a/Documentation/admin-guide/media/apple-ave.rst b/Documentation/admin-guide/media/apple-ave.rst index dabb4250a72a1..d8f9acabb2117 100644 --- a/Documentation/admin-guide/media/apple-ave.rst +++ b/Documentation/admin-guide/media/apple-ave.rst @@ -8,7 +8,7 @@ encodes H.264 and HEVC. ``CONFIG_VIDEO_APPLE_AVE`` builds the ``apple-ave`` driver, which registers a V4L2 memory-to-memory stateful encoder following Documentation/userspace-api/media/v4l/dev-encoder.rst. The OUTPUT queue takes NV12 frames (64-byte aligned line stride), the CAPTURE queue returns -H.264 or HEVC. +H.264 (Baseline, Main, High) or HEVC (Main) as Annex B. Firmware -------- @@ -17,27 +17,55 @@ The encoder runs firmware on its own coprocessor. The bootloader loads it into two reserved, DART-mapped segments (TEXT and DATA) but does not start it. The Aurora Stage 2 loader passes both regions in the device's ``memory-region`` property and records their CRC32 under ``/chosen`` as -``aurora,ave-text-crc32`` and ``aurora,ave-data-crc32``. No firmware file is -requested. +``aurora,ave-text-crc32`` and ``aurora,ave-data-crc32``. The driver only binds to firmware builds whose command formats it knows, identified by the TEXT checksum, and only when DATA still matches the checksum the loader recorded. It keeps a copy of DATA and restores it before -every firmware boot, so the coprocessor can lose power and start again from -the same state. +every firmware boot, so the firmware can be started again from the same state. -Power management ----------------- +Session images +-------------- -The firmware boots when the first encoding session starts. Runtime PM uses a -two second autosuspend delay. While the firmware runs the encoder stays -powered; across system sleep the power domains go down, open sessions fail -their next frame, and the next session boots the firmware again. +The two large session commands, Start and Process, are mostly constant for a +given firmware build. The driver writes every field that depends on the +session or the frame (sizes, rate control, GOP structure, levels, cropping, +buffer addresses, frame numbers and times) and takes the remaining constant +words from four files per firmware build:: -Status + apple/ave-t8140--h264-start.bin (0x3180 bytes) + apple/ave-t8140--h264-process.bin (0x63d8 bytes) + apple/ave-t8140--hevc-start.bin (0x13f28 bytes) + apple/ave-t8140--hevc-process.bin (0xb1c0 bytes) + +Each file is a complete command image. Address and size fields in it are +ignored. Without these files, ``VIDIOC_STREAMON`` fails with ``ENOENT``. + +Sessions and power management +----------------------------- + +Each encoding session boots the firmware, opens and starts a session, encodes +and, at the end, drains, stops and closes the session and halts the firmware. +The next session restarts the firmware through the boot path. One session +runs at a time; a second one gets ``EBUSY``. + +Up to five frames are inside the firmware at once (three without B-frames). +The firmware reorders B-frames itself, so CAPTURE buffers come back in +bitstream order, each carrying the timestamp of the frame it codes. The driver +asks for at least six OUTPUT buffers. + +Runtime PM uses a two second autosuspend delay. A firmware that stops +answering keeps the encoder powered and is not used again until reboot. +Across system sleep an open session ends; the next one starts afresh. + +Limits ------ -Firmware start-up, the command rings, session open/close and the V4L2 -interface are implemented. Session start and per-frame encoding are not yet -wired up, so ``VIDIOC_STREAMON`` on both queues currently fails with -``EOPNOTSUPP``. +Picture sizes from 320x240 to 2560x1600 (in steps of two), up to 240 frames +per second. Rate control is VBR (with ``BITRATE_PEAK`` as a one-second data +rate cap) or CBR; constant quality is not offered. ``BITRATE`` may change +while streaming and applies from the next frame; ``FORCE_KEY_FRAME`` makes +the next frame an IDR. H.264 uses either three B-frames or none (Baseline and +a GOP size of 1 never use them); HEVC always uses them for now. The level +control is a minimum: the driver raises it to what the size, frame rate and +bit rate need. diff --git a/drivers/media/platform/apple/ave/Kconfig b/drivers/media/platform/apple/ave/Kconfig index 59937167b29ce..2b3f380abf752 100644 --- a/drivers/media/platform/apple/ave/Kconfig +++ b/drivers/media/platform/apple/ave/Kconfig @@ -5,12 +5,15 @@ config VIDEO_APPLE_AVE depends on ARCH_APPLE || COMPILE_TEST depends on VIDEO_DEV && IOMMU_DMA && PM_GENERIC_DOMAINS select CRC32 + select FW_LOADER select V4L2_MEM2MEM_DEV select VIDEOBUF2_DMA_CONTIG help V4L2 memory-to-memory driver for the AVE H.264/HEVC encoder in Apple T8140 (A18 Pro) SoCs, as in the MacBook Neo. The encoder - firmware is loaded by the bootloader. + firmware is loaded by the bootloader; the constant parts of its + session commands come from files under apple/ in the firmware + search path. To compile this driver as a module, choose M here: the module will be called apple-ave. diff --git a/drivers/media/platform/apple/ave/Makefile b/drivers/media/platform/apple/ave/Makefile index 8e0a5519a520b..542a7adeae360 100644 --- a/drivers/media/platform/apple/ave/Makefile +++ b/drivers/media/platform/apple/ave/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_VIDEO_APPLE_AVE) += apple-ave.o -apple-ave-y := ave-drv.o ave-fw.o ave-ipc.o ave-cmd.o ave-v4l2.o +apple-ave-y := ave-drv.o ave-fw.o ave-ipc.o ave-cmd.o ave-sess.o ave-v4l2.o diff --git a/drivers/media/platform/apple/ave/ave-cmd.c b/drivers/media/platform/apple/ave/ave-cmd.c index 6948bc60496a6..0b539fc201c3b 100644 --- a/drivers/media/platform/apple/ave/ave-cmd.c +++ b/drivers/media/platform/apple/ave/ave-cmd.c @@ -2,107 +2,621 @@ /* * Apple AVE video encoder: firmware commands * + * Start and Process are large command images. Most of their words never + * change between sessions of a firmware build; the driver takes those from + * a per-build image file (see ave_fw_templates()) and writes every field + * that depends on the session or the frame itself. + * * Copyright The Aurora Silicon Contributors */ #include +#include +#include #include #include #include +#include #include #include "ave.h" #include "ave-regs.h" -static void ave_cmd_header(struct ave_dev *ave, u8 *cmd, enum ave_opcode op, - u32 client, enum ave_codec codec) +/* + * IEEE 754 double for the positive value @num / @den (0 for 0), built with + * integer arithmetic; the firmware takes rates and limits as doubles. + */ +static u64 ave_f64(u64 num, u64 den) +{ + u64 mant = 0, rem; + int exp = 0; + + if (!num || !den) + return 0; + while (num >= den << 1) { + den <<= 1; + exp++; + } + while (num < den) { + num <<= 1; + exp--; + } + /* 1 <= num/den < 2: take 53 significant bits by long division. */ + rem = num; + for (unsigned int i = 0; i < 53; i++) { + mant <<= 1; + if (rem >= den) { + mant |= 1; + rem -= den; + } + rem <<= 1; + } + return (u64)(exp + 1023) << 52 | (mant & GENMASK_ULL(51, 0)); +} + +#define AVE_F64_NEG BIT_ULL(63) + +static void ave_put32(u8 *cmd, u32 off, u32 val) +{ + put_unaligned_le32(val, cmd + off); +} + +static void ave_put64(u8 *cmd, u32 off, u64 val) +{ + put_unaligned_le64(val, cmd + off); +} + +/* A surface descriptor: {u64 DART address, u32 size, u32 0}. */ +static void ave_put_surface(u8 *cmd, u32 off, dma_addr_t addr, u32 size) +{ + ave_put64(cmd, off, addr); + ave_put32(cmd, off + 8, size); + ave_put32(cmd, off + 12, 0); +} + +static void ave_put_buf(u8 *cmd, u32 off, const struct ave_buf *b, u32 size) +{ + ave_put_surface(cmd, off, b->dma, size); +} + +static u32 ave_session_arg2(enum ave_codec codec) { - put_unaligned_le32(op, cmd + AVE_CMD_OPCODE); - put_unaligned_le64(++ave->next_token, cmd + AVE_CMD_TOKEN); - put_unaligned_le64(client, cmd + AVE_CMD_CLIENT); - put_unaligned_le32(1, cmd + AVE_CMD_UNIT); - put_unaligned_le32(codec, cmd + AVE_CMD_CODEC); - put_unaligned_le32(op, cmd + AVE_CMD_OPCODE_ECHO); + return codec == AVE_CODEC_HEVC ? AVE_SESSION_ARG2_HEVC : AVE_SESSION_ARG2_H264; +} + +static void ave_cmd_header(struct ave_ctx *ctx, u8 *cmd, enum ave_opcode op, u32 echo, + u32 arg0, u32 arg2) +{ + struct ave_dev *ave = ctx->ave; + + ave->next_token += AVE_TOKEN_STEP; + ave_put32(cmd, AVE_CMD_OPCODE, op); + ave_put64(cmd, AVE_CMD_TOKEN, ave->next_token); + ave_put64(cmd, AVE_CMD_CLIENT, ctx->sess->client); + ave_put32(cmd, AVE_CMD_UNIT, AVE_SESSION_UNIT); + ave_put32(cmd, AVE_CMD_CODEC, ctx->sess->p.codec); + ave_put32(cmd, AVE_CMD_OPCODE_ECHO, echo); + ave_put32(cmd, AVE_CMD_ARG0, arg0); + ave_put32(cmd, AVE_CMD_ARG1, AVE_SESSION_ARG1); + ave_put32(cmd, AVE_CMD_ARG2, arg2); } /* Firmware-wide configuration, sent once after every firmware boot. */ int ave_cmd_config(struct ave_dev *ave) { - u8 cmd[AVE_CONFIG_BYTES] = {}; + int buf = ave_ipc_scalar_buf(ave); + u8 *cmd; + int ret; - put_unaligned_le32(AVE_OP_CONFIG, cmd + AVE_CMD_OPCODE); - put_unaligned_le64(++ave->next_token, cmd + AVE_CMD_TOKEN); - put_unaligned_le32(U32_MAX, cmd + AVE_CONFIG_ALL_UNITS); + if (buf < 0) + return buf; + cmd = ave_ipc_cmd(ave, buf, AVE_CONFIG_BYTES); + ave_put32(cmd, AVE_CMD_OPCODE, AVE_OP_CONFIG); + ave_put32(cmd, AVE_CONFIG_ALL_UNITS, U32_MAX); cmd[AVE_CONFIG_OPT_A] = 1; cmd[AVE_CONFIG_OPT_B] = 1; - return ave_ipc_call(ave, cmd, sizeof(cmd)); + ave_put32(cmd, AVE_CONFIG_MCC_DSID, AVE_CONFIG_MCC_DSID_T8140); + ret = ave_ipc_submit(ave, buf, AVE_CONFIG_BYTES, AVE_CMD_GLOBAL_TIMEOUT_US, true); + return ret ?: ave_ipc_settle(ave, AVE_CMD_WAIT_MS); } -/* Firmware-wide stop, sent after every session has closed. */ +/* + * Firmware-wide stop, sent after every session has closed. The firmware + * neither acknowledges nor answers it. + */ int ave_cmd_halt(struct ave_dev *ave) { - u8 cmd[AVE_CMD_HDR_BYTES] = {}; + int buf = ave_ipc_scalar_buf(ave); + u8 *cmd; - put_unaligned_le32(AVE_OP_HALT, cmd + AVE_CMD_OPCODE); - put_unaligned_le64(++ave->next_token, cmd + AVE_CMD_TOKEN); - put_unaligned_le32(AVE_HALT_ALL, cmd + AVE_CMD_OPCODE_ECHO); - return ave_ipc_call(ave, cmd, sizeof(cmd)); + if (buf < 0) + return buf; + cmd = ave_ipc_cmd(ave, buf, AVE_CMD_HDR_BYTES); + ave_put32(cmd, AVE_CMD_OPCODE, AVE_OP_HALT); + ave_put32(cmd, AVE_CMD_OPCODE_ECHO, AVE_HALT_ALL); + return ave_ipc_submit(ave, buf, AVE_CMD_HDR_BYTES, AVE_CMD_GLOBAL_TIMEOUT_US, false); } -static int ave_cmd_simple(struct ave_ctx *ctx, enum ave_opcode op, enum ave_codec codec, - u32 arg0, u32 arg2) +static int ave_cmd_scalar(struct ave_ctx *ctx, enum ave_opcode op, u32 arg0, u32 arg2) { - u8 cmd[AVE_CMD_HDR_BYTES] = {}; + struct ave_dev *ave = ctx->ave; + int buf = ave_ipc_scalar_buf(ave); + u8 *cmd; - ave_cmd_header(ctx->ave, cmd, op, ctx->client, codec); - put_unaligned_le32(arg0, cmd + AVE_CMD_ARG0); - put_unaligned_le32(AVE_SESSION_ARG1, cmd + AVE_CMD_ARG1); - put_unaligned_le32(arg2, cmd + AVE_CMD_ARG2); - return ave_ipc_call(ctx->ave, cmd, sizeof(cmd)); + if (buf < 0) + return buf; + cmd = ave_ipc_cmd(ave, buf, AVE_CMD_HDR_BYTES); + ave_cmd_header(ctx, cmd, op, op, arg0, arg2); + return ave_ipc_submit(ave, buf, AVE_CMD_HDR_BYTES, AVE_CMD_SESSION_TIMEOUT_US, true); } -int ave_cmd_open(struct ave_ctx *ctx, enum ave_codec codec) +int ave_cmd_open(struct ave_ctx *ctx) { struct ave_dev *ave = ctx->ave; int ret; - ctx->client = ++ave->next_client; - ret = ave_cmd_simple(ctx, AVE_OP_OPEN, codec, AVE_SESSION_ARG0, AVE_OPEN_ARG2); + ave->next_client += AVE_CLIENT_STEP; + ctx->sess->client = ave->next_client; + ret = ave_cmd_scalar(ctx, AVE_OP_OPEN, AVE_SESSION_ARG0, AVE_SESSION_ARG2_SETUP); if (!ret) - ctx->session_open = true; + ctx->sess->fw_open = true; + return ret ?: ave_ipc_settle(ave, AVE_CMD_WAIT_MS); +} + +/* Drain barrier: answered once every Process sent before it is answered. */ +int ave_cmd_complete(struct ave_ctx *ctx, unsigned int count) +{ + int ret = 0; + + for (unsigned int i = 0; i < count && !ret; i++) + ret = ave_cmd_scalar(ctx, AVE_OP_COMPLETE, 0, ave_session_arg2(ctx->sess->p.codec)); return ret; } -/* - * Session parameters. The Start layout (resolution, formats, rate control, - * GOP, the per-session buffer table) is not described yet, so sessions cannot - * start; everything around it is in place. - */ -int ave_cmd_start(struct ave_ctx *ctx, enum ave_codec codec) +/* Priority and Stop go out together, Close after Stop is answered. */ +int ave_cmd_close(struct ave_ctx *ctx) +{ + u32 arg2 = ave_session_arg2(ctx->sess->p.codec); + int ret, err; + + ret = ave_cmd_scalar(ctx, AVE_OP_PRIORITY, AVE_SESSION_ARG0, arg2); + if (!ret) + ret = ave_cmd_scalar(ctx, AVE_OP_STOP, AVE_SESSION_ARG0, arg2); + err = ave_ipc_settle(ctx->ave, AVE_CMD_WAIT_MS); + ret = ret ?: err; + if (ctx->ave->fw_state != AVE_FW_RUNNING) + return ret; + err = ave_cmd_scalar(ctx, AVE_OP_CLOSE, AVE_SESSION_ARG0, arg2); + if (!err) + err = ave_ipc_settle(ctx->ave, AVE_CMD_WAIT_MS); + ctx->sess->fw_open = false; + return ret ?: err; +} + +/* Start (id 6) */ + +#define AVE_START_H264_BYTES 0x3180 +#define AVE_START_HEVC_BYTES 0x13f28 + +#define AVE_START_SURFACE 0x48 /* and an unused second one at 0x58 */ +#define AVE_START_SURFACE_BYTES 0x12c000 +#define AVE_START_SURFACE2 0x58 +#define AVE_START_REF_A(i) (0x390 + 0x20 * (i)) +#define AVE_START_REF_B(i) (0x3a0 + 0x20 * (i)) +#define AVE_START_REF_C(i) (0x7d0 + 0x20 * (i)) +#define AVE_START_REF_C_ALIAS(i) (0x7e0 + 0x20 * (i)) +#define AVE_START_MB 0xc10 +#define AVE_START_CODED(s) (0xd50 + 0x10 * (s)) +#define AVE_START_META(s) (0xf50 + 0x10 * (s)) +#define AVE_START_ARRAY_LEN 30 +#define AVE_START_REF_D(i) (0x1130 + 0x10 * (i)) +#define AVE_START_ROW_A 0x1350 +#define AVE_START_ROW_B 0x1390 +#define AVE_START_FIXED 0x13d0 +#define AVE_START_FIXED_BYTES 0x4000 +#define AVE_START_HEVC_EXTRA(i) (0x2520 + 0x10 * (i)) + +static const u16 ave_start_tile_off[AVE_TILE_BUFS] = { + 0x1410, 0x1430, 0x1490, 0x14b0, /* both codecs */ + 0x1510, 0x1530, 0x1590, 0x15b0, /* H.264 only */ +}; + +/* Session configuration words */ +#define AVE_START_CODEC 0xc0 +#define AVE_START_8BIT 0x208 +#define AVE_START_CONST_214 0x214 +#define AVE_START_CONST_218 0x218 +#define AVE_START_FPS 0x220 +#define AVE_START_RC_MODE 0x234 +#define AVE_START_BITRATE 0x238 +#define AVE_START_INIT_QP(i) (0x240 + 4 * (i)) +#define AVE_START_INIT_QP_VALUE 26 +#define AVE_START_LIMITS 0x250 +#define AVE_START_LIMIT_BYTES 0x258 +#define AVE_START_LIMIT_SECS 0x260 +#define AVE_START_F64_288 0x288 +#define AVE_START_F64_290 0x290 +#define AVE_START_MIN_QP 0x298 +#define AVE_START_MAX_QP 0x29c +#define AVE_START_GOP_FLAGS 0x2a0 +#define AVE_START_P_FRAMES 0x2ac +#define AVE_START_MAX_B 0x2b0 +#define AVE_START_REF_DEPTH 0x2b4 +#define AVE_START_IDR_INTERVAL 0x2b8 +#define AVE_START_F64_2C0 0x2c0 +#define AVE_START_F64_2C8 0x2c8 +#define AVE_START_RC_FLAGS 0x300 +#define AVE_START_MIN_QP2 0x308 +#define AVE_START_MAX_QP2 0x30c +#define AVE_START_RC_310 0x310 +#define AVE_START_SIZE_CLASS 0x320 +#define AVE_START_CODED_W 0x368 +#define AVE_START_CODED_H 0x36c +#define AVE_START_CODEC_FLAGS 0x370 +#define AVE_START_H264_380 0x380 +#define AVE_START_H264_384 0x384 +#define AVE_START_H264_HIGH 0x2588 +#define AVE_START_BIT_DEPTH 0x2598 +#define AVE_START_CODED_H2 0x25dc + +/* H.264 parameter-set words */ +#define AVE_START_H264_PROFILE 0x291c +#define AVE_START_H264_LEVEL 0x2938 +#define AVE_START_H264_MAX_REFS 0x2d44 +#define AVE_START_H264_WIDTH_MBS 0x2d4c +#define AVE_START_H264_HEIGHT_MBS 0x2d50 +#define AVE_START_H264_CROP 0x2db4 +#define AVE_START_H264_CROP_LEFT 0x2db8 +#define AVE_START_H264_CROP_RIGHT 0x2dbc +#define AVE_START_H264_CROP_TOP 0x2dc0 +#define AVE_START_H264_CROP_BOTTOM 0x2dc4 +#define AVE_START_H264_CABAC 0x2fd8 +#define AVE_START_H264_8X8 0x3023 /* byte */ + +/* HEVC parameter-set words; most appear twice */ +#define AVE_START_HEVC_PAD 0x26e0 +#define AVE_START_HEVC_PROFILE_A 0x2938 +#define AVE_START_HEVC_COMPAT_A 0x293d /* two bytes */ +#define AVE_START_HEVC_8BIT_A 0x2962 /* byte */ +#define AVE_START_HEVC_LEVEL_A 0x296c +#define AVE_START_HEVC_PROFILE_B 0x5904 +#define AVE_START_HEVC_COMPAT_B 0x5909 +#define AVE_START_HEVC_8BIT_B 0x592e +#define AVE_START_HEVC_LEVEL_B 0x5938 +#define AVE_START_HEVC_WIDTH 0x5ab4 +#define AVE_START_HEVC_HEIGHT 0x5ab8 +#define AVE_START_HEVC_WIDTH_CTB 0x5abc +#define AVE_START_HEVC_HEIGHT_CTB 0x5ac0 +#define AVE_START_HEVC_CONF 0x5ac4 +#define AVE_START_HEVC_CONF_LEFT 0x5ac8 +#define AVE_START_HEVC_CONF_RIGHT 0x5acc +#define AVE_START_HEVC_CONF_TOP 0x5ad0 +#define AVE_START_HEVC_CONF_BOTTOM 0x5ad4 +#define AVE_START_HEVC_DEPTH_LUMA 0x5ad8 +#define AVE_START_HEVC_DEPTH_CHROMA 0x5adc +/* HEVC reference structure with B-frames (reordering on) */ +#define AVE_START_HEVC_REFS 0x259c +#define AVE_START_HEVC_REORDER_COPY 0x2e54 +#define AVE_START_HEVC_DPB_MINUS1 0x5b00 +#define AVE_START_HEVC_REORDER 0x5b04 +#define AVE_START_HEVC_REORDER_ON 0x95fc + +static void ave_start_surfaces(struct ave_session *sess, u8 *cmd) { - return -EOPNOTSUPP; + const struct ave_params *p = &sess->p; + bool hevc = p->codec == AVE_CODEC_HEVC; + unsigned int tiles = hevc ? 4 : AVE_TILE_BUFS; + + /* Clear every descriptor the image may carry, then fill ours. */ + for (unsigned int i = 0; i < AVE_MAX_REFS; i++) { + ave_put_surface(cmd, AVE_START_REF_A(i), 0, 0); + ave_put_surface(cmd, AVE_START_REF_B(i), 0, 0); + ave_put_surface(cmd, AVE_START_REF_C(i), 0, 0); + ave_put_surface(cmd, AVE_START_REF_C_ALIAS(i), 0, 0); + ave_put_surface(cmd, AVE_START_REF_D(i), 0, 0); + } + for (unsigned int s = 0; s < AVE_START_ARRAY_LEN; s++) { + ave_put_surface(cmd, AVE_START_CODED(s), 0, 0); + ave_put_surface(cmd, AVE_START_META(s), 0, 0); + } + + ave_put_buf(cmd, AVE_START_SURFACE, &sess->hdr, AVE_START_SURFACE_BYTES); + ave_put_surface(cmd, AVE_START_SURFACE2, 0, 0); + for (unsigned int i = 0; i < p->refs; i++) { + ave_put_buf(cmd, AVE_START_REF_A(i), &sess->ref_a[i], sess->ref_a[i].size); + ave_put_buf(cmd, AVE_START_REF_B(i), &sess->ref_b[i], sess->ref_b[i].size); + ave_put_buf(cmd, AVE_START_REF_C(i), &sess->ref_c[i], sess->ref_c[i].size); + ave_put_surface(cmd, AVE_START_REF_C_ALIAS(i), sess->ref_c[i].dma, 0); + ave_put_buf(cmd, AVE_START_REF_D(i), &sess->ref_d[i], sess->ref_d[i].size); + } + ave_put_buf(cmd, AVE_START_MB, &sess->mb, sess->mb.size); + for (unsigned int s = 0; s < p->slots; s++) { + ave_put_buf(cmd, AVE_START_CODED(s), &sess->slot[s].coded, p->coded_cap); + ave_put_buf(cmd, AVE_START_META(s), &sess->slot[s].meta, sess->slot[s].meta.size); + } + ave_put_buf(cmd, AVE_START_ROW_A, &sess->row_a, sess->row_a.size); + ave_put_buf(cmd, AVE_START_ROW_B, &sess->row_b, sess->row_b.size); + ave_put_buf(cmd, AVE_START_FIXED, &sess->fixed, AVE_START_FIXED_BYTES); + for (unsigned int i = 0; i < AVE_TILE_BUFS; i++) { + u32 off = ave_start_tile_off[i]; + + if (i < tiles) { + ave_put_buf(cmd, off, &sess->tile[i], sess->tile[i].size); + ave_put_buf(cmd, off + 0x10, &sess->tile_aux[i], sess->tile_aux[i].size); + } else { + ave_put_surface(cmd, off, 0, 0); + ave_put_surface(cmd, off + 0x10, 0, 0); + } + } + if (hevc) + for (unsigned int i = 0; i < 2; i++) + ave_put_buf(cmd, AVE_START_HEVC_EXTRA(i), &sess->hevc_extra[i], + sess->hevc_extra[i].size); } -int ave_cmd_process(struct ave_ctx *ctx, enum ave_codec codec, - struct vb2_v4l2_buffer *src, struct vb2_v4l2_buffer *dst, - u32 *bytesused, bool *keyframe) +static void ave_start_config(const struct ave_params *p, u8 *cmd) { - return -EOPNOTSUPP; + bool hevc = p->codec == AVE_CODEC_HEVC; + u32 crop_r = (p->cw - p->width) / 2, crop_b = (p->ch - p->height) / 2; + bool crop = crop_r || crop_b; + + ave_put32(cmd, AVE_START_CODEC, p->codec); + ave_put32(cmd, AVE_START_8BIT, hevc ? 1 : 0); + ave_put32(cmd, AVE_START_CONST_214, 0x39d); + ave_put32(cmd, AVE_START_CONST_218, 0x18); + ave_put32(cmd, AVE_START_FPS, p->fps_int); + + /* Rate control */ + ave_put32(cmd, AVE_START_RC_MODE, p->rc_mode); + ave_put32(cmd, AVE_START_BITRATE, p->bitrate); + for (unsigned int i = 0; i < 3; i++) + ave_put32(cmd, AVE_START_INIT_QP(i), AVE_START_INIT_QP_VALUE); + ave_put32(cmd, AVE_START_LIMITS, p->limit_bytes ? 1 : 0); + if (p->limit_bytes) { + ave_put64(cmd, AVE_START_LIMIT_BYTES, ave_f64(p->limit_bytes, 1)); + ave_put64(cmd, AVE_START_LIMIT_SECS, ave_f64(1, 1)); + } + ave_put64(cmd, AVE_START_F64_288, ave_f64(1, 1)); + ave_put64(cmd, AVE_START_F64_290, ave_f64(1, 2)); + ave_put64(cmd, AVE_START_F64_2C0, AVE_F64_NEG | ave_f64(1, 1)); + ave_put64(cmd, AVE_START_F64_2C8, AVE_F64_NEG | ave_f64(1, 1)); + ave_put32(cmd, AVE_START_MIN_QP, p->min_qp); + ave_put32(cmd, AVE_START_MAX_QP, p->max_qp); + ave_put32(cmd, AVE_START_MIN_QP2, p->min_qp); + ave_put32(cmd, AVE_START_MAX_QP2, p->max_qp); + ave_put32(cmd, AVE_START_RC_FLAGS, hevc ? 0x736 : 0x8202); + ave_put32(cmd, AVE_START_RC_310, 30); + if (!hevc) + ave_put32(cmd, AVE_START_H264_380, 1); + + /* GOP structure: with B-frames as captured, or P-only, or intra-only. */ + ave_put32(cmd, AVE_START_GOP_FLAGS, p->bframes ? 0x600 : 0x200); + ave_put32(cmd, AVE_START_P_FRAMES, p->intra_only ? 0 : 1); + ave_put32(cmd, AVE_START_MAX_B, p->bframes ? 3 : 0); + ave_put32(cmd, AVE_START_REF_DEPTH, p->bframes ? 3 : 1); + ave_put32(cmd, AVE_START_IDR_INTERVAL, p->gop); + + /* Picture size */ + ave_put32(cmd, AVE_START_SIZE_CLASS, p->width * p->height > 1280 * 720 ? 3 : 1); + ave_put32(cmd, AVE_START_CODED_W, p->cw); + ave_put32(cmd, AVE_START_CODED_H, p->ch); + ave_put32(cmd, AVE_START_CODED_H2, p->ch); + ave_put32(cmd, AVE_START_CODEC_FLAGS, hevc ? 0x2000002 : 2); + ave_put32(cmd, AVE_START_BIT_DEPTH, 8); + + if (!hevc) { + ave_put32(cmd, AVE_START_H264_384, p->width * p->height <= 320 * 240 ? 0x40 : 0x10); + ave_put32(cmd, AVE_START_H264_HIGH, p->high ? 2 : 0); + ave_put32(cmd, AVE_START_H264_PROFILE, p->profile); + ave_put32(cmd, AVE_START_H264_LEVEL, p->level); + ave_put32(cmd, AVE_START_H264_MAX_REFS, p->refs - 1); + ave_put32(cmd, AVE_START_H264_WIDTH_MBS, p->cw / 16 - 1); + ave_put32(cmd, AVE_START_H264_HEIGHT_MBS, p->ch / 16 - 1); + ave_put32(cmd, AVE_START_H264_CROP, crop); + ave_put32(cmd, AVE_START_H264_CROP_LEFT, 0); + ave_put32(cmd, AVE_START_H264_CROP_RIGHT, crop_r); + ave_put32(cmd, AVE_START_H264_CROP_TOP, 0); + ave_put32(cmd, AVE_START_H264_CROP_BOTTOM, crop_b); + ave_put32(cmd, AVE_START_H264_CABAC, p->profile != 2); + cmd[AVE_START_H264_8X8] = p->high; + return; + } + + ave_put32(cmd, AVE_START_HEVC_PAD, 1 | + (p->cw > p->width ? BIT(16) | BIT(18) : 0) | + (p->ch > p->height ? BIT(17) | BIT(19) : 0)); + ave_put32(cmd, AVE_START_HEVC_PROFILE_A, p->profile); + ave_put32(cmd, AVE_START_HEVC_PROFILE_B, p->profile); + cmd[AVE_START_HEVC_COMPAT_A] = 1; + cmd[AVE_START_HEVC_COMPAT_A + 1] = 1; + cmd[AVE_START_HEVC_COMPAT_B] = 1; + cmd[AVE_START_HEVC_COMPAT_B + 1] = 1; + cmd[AVE_START_HEVC_8BIT_A] = 1; + cmd[AVE_START_HEVC_8BIT_B] = 1; + ave_put32(cmd, AVE_START_HEVC_LEVEL_A, p->level); + ave_put32(cmd, AVE_START_HEVC_LEVEL_B, p->level); + ave_put32(cmd, AVE_START_HEVC_WIDTH, p->cw); + ave_put32(cmd, AVE_START_HEVC_HEIGHT, p->ch); + ave_put32(cmd, AVE_START_HEVC_WIDTH_CTB, DIV_ROUND_UP(p->cw, 32)); + ave_put32(cmd, AVE_START_HEVC_HEIGHT_CTB, DIV_ROUND_UP(p->ch, 32)); + ave_put32(cmd, AVE_START_HEVC_CONF, crop); + ave_put32(cmd, AVE_START_HEVC_CONF_LEFT, 0); + ave_put32(cmd, AVE_START_HEVC_CONF_RIGHT, crop_r); + ave_put32(cmd, AVE_START_HEVC_CONF_TOP, 0); + ave_put32(cmd, AVE_START_HEVC_CONF_BOTTOM, crop_b); + ave_put32(cmd, AVE_START_HEVC_DEPTH_LUMA, 0); + ave_put32(cmd, AVE_START_HEVC_DEPTH_CHROMA, 0); + /* Only the B-frame structure is described for HEVC (see ave-sess.c). */ + ave_put32(cmd, AVE_START_HEVC_REFS, 4); + ave_put32(cmd, AVE_START_HEVC_DPB_MINUS1, 4); + ave_put32(cmd, AVE_START_HEVC_REORDER, 2); + ave_put32(cmd, AVE_START_HEVC_REORDER_COPY, 2); + ave_put32(cmd, AVE_START_HEVC_REORDER_ON, 1); } -/* Flush, stop and close the session. */ -int ave_cmd_finish(struct ave_ctx *ctx, enum ave_codec codec) +int ave_cmd_start(struct ave_ctx *ctx) { - u32 tail = codec == AVE_CODEC_HEVC ? AVE_FINISH_ARG2_HEVC : AVE_FINISH_ARG2_H264; - int ret, err; + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + const struct firmware *tmpl = ave->tmpl[sess->p.codec - 1][AVE_TMPL_START]; + u8 *cmd; + int ret; - if (!ctx->session_open) - return 0; + cmd = ave_ipc_cmd(ave, AVE_BUF_CMD, tmpl->size); + if (!cmd) + return -EBUSY; + memcpy(cmd, tmpl->data, tmpl->size); + ave_cmd_header(ctx, cmd, AVE_OP_START, AVE_OP_START, AVE_SESSION_ARG0, + AVE_SESSION_ARG2_SETUP); + ave_start_surfaces(sess, cmd); + ave_start_config(&sess->p, cmd); - ret = ave_cmd_simple(ctx, AVE_OP_COMPLETE, codec, 0, tail); - err = ave_cmd_simple(ctx, AVE_OP_STOP, codec, AVE_SESSION_ARG0, tail); - ret = ret ?: err; - err = ave_cmd_simple(ctx, AVE_OP_CLOSE, codec, AVE_SESSION_ARG0, tail); - ctx->session_open = false; - return ret ?: err; + ret = ave_ipc_submit(ave, AVE_BUF_CMD, tmpl->size, AVE_CMD_SESSION_TIMEOUT_US, true); + return ret ?: ave_ipc_settle(ave, AVE_START_WAIT_MS); +} + +/* Process (id 8) */ + +struct ave_proc_layout { + u32 first_frame; + u32 y, uv; /* {u64 address, u32 size, u32 stride} */ + u32 in_w, in_h; + u32 coded_cap; + u32 fps_int; + u32 bitrate; + u32 limits, limit_bytes, limit_secs; + u32 const5; + u32 fps; /* f64 */ + u32 timescale; /* u64 */ + u32 slot; + u32 coded, meta; /* u64 addresses */ + u32 ordinal; + u32 pts; /* u64 */ + u32 force_key; /* 1 on a forced key frame */ + u32 force_type; /* 2 on a forced key frame */ + u32 frame_buf; /* H.264 only */ + u32 shdr_table; /* HEVC only */ + u32 crop; /* HEVC only */ +}; + +static const struct ave_proc_layout ave_proc_h264 = { + .first_frame = 0x12c0, + .y = 0x5830, .uv = 0x5850, + .in_w = 0x58e8, .in_h = 0x58ec, + .coded_cap = 0x61c8, + .fps_int = 0x1610, + .bitrate = 0x1620, + .limits = 0x1628, .limit_bytes = 0x1630, .limit_secs = 0x1638, + .const5 = 0x6238, + .fps = 0x6268, .timescale = 0x6278, + .slot = 0x61b4, .coded = 0x61b8, .meta = 0x61c0, + .ordinal = 0x6228, .pts = 0x6270, + .force_key = 0x1658, .force_type = 0x29f8, + .frame_buf = 0x58c8, +}; + +static const struct ave_proc_layout ave_proc_hevc = { + .first_frame = 0x5d90, + .y = 0xa300, .uv = 0xa320, + .in_w = 0xa3b8, .in_h = 0xa3bc, + .coded_cap = 0xac98, + .fps_int = 0x60e0, + .bitrate = 0x60f0, + .limits = 0x60f8, .limit_bytes = 0x6100, .limit_secs = 0x6108, + .const5 = 0xad08, + .fps = 0xad38, .timescale = 0xad48, + .slot = 0xac84, .coded = 0xac88, .meta = 0xac90, + .ordinal = 0xacf8, .pts = 0xad40, + .force_key = 0x6128, .force_type = 0x74c8, + .shdr_table = 0xd88, + .crop = 0x74dc, +}; + +#define AVE_PROC_TIMESCALE 14400000ULL +#define AVE_PROC_SHDR_ENTRIES 256 +#define AVE_PROC_SHDR_STRIDE 0x400 +#define AVE_PROC_FORCED_TYPE 2 + +int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *src, + u32 bitrate, bool force_key) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + const struct ave_params *p = &sess->p; + const struct ave_proc_layout *l = p->codec == AVE_CODEC_HEVC ? &ave_proc_hevc : + &ave_proc_h264; + const struct firmware *tmpl = ave->tmpl[p->codec - 1][AVE_TMPL_PROCESS]; + struct ave_slot *s = &sess->slot[slot]; + unsigned int buf = AVE_BUF_PROC(slot); + dma_addr_t y; + u64 pts; + u8 *cmd; + int ret; + + y = vb2_dma_contig_plane_dma_addr(&src->vb2_buf, 0) + src->vb2_buf.planes[0].data_offset; + if (!IS_ALIGNED(y, 64)) + return -EINVAL; + + ret = ave_ipc_wait_ack(ave, buf); + if (ret) + return ret; + cmd = ave_ipc_cmd(ave, buf, tmpl->size); + if (!cmd) + return -EBUSY; + memcpy(cmd, tmpl->data, tmpl->size); + ave_cmd_header(ctx, cmd, AVE_OP_PROCESS, AVE_PROC_SUBTYPE + slot, 0, + ave_session_arg2(p->codec)); + + /* Input picture: NV12, chroma right after p->height luma lines. */ + ave_put64(cmd, l->y, y); + ave_put32(cmd, l->y + 8, p->stride * p->height); + ave_put32(cmd, l->y + 12, p->stride); + ave_put64(cmd, l->uv, y + p->stride * p->height); + ave_put32(cmd, l->uv + 8, p->stride * p->height / 2); + ave_put32(cmd, l->uv + 12, p->stride); + ave_put32(cmd, l->in_w, p->width); + ave_put32(cmd, l->in_h, p->height); + + /* Session values repeated in every frame */ + ave_put32(cmd, l->coded_cap, p->coded_cap); + ave_put32(cmd, l->fps_int, p->fps_int); + /* The bit rate may change while streaming; it takes effect here. */ + ave_put32(cmd, l->bitrate, bitrate); + ave_put32(cmd, l->limits, p->limit_bytes ? 1 : 0); + if (p->limit_bytes) { + ave_put64(cmd, l->limit_bytes, ave_f64(p->limit_bytes, 1)); + ave_put64(cmd, l->limit_secs, ave_f64(1, 1)); + } + ave_put32(cmd, l->const5, 5); + ave_put64(cmd, l->fps, ave_f64(p->fps_num, p->fps_den)); + ave_put64(cmd, l->timescale, AVE_PROC_TIMESCALE); + if (l->crop) + ave_put32(cmd, l->crop, p->cw != p->width || p->ch != p->height); + + /* This frame */ + ave_put32(cmd, l->first_frame, sess->frames == 0); + ave_put32(cmd, l->slot, slot); + ave_put64(cmd, l->coded, s->coded.dma); + ave_put64(cmd, l->meta, s->meta.dma); + if (l->frame_buf) + ave_put64(cmd, l->frame_buf, s->aux.dma); + if (l->shdr_table) + for (unsigned int i = 0; i < AVE_PROC_SHDR_ENTRIES; i++) + ave_put64(cmd, l->shdr_table + 8 * i, + s->aux.dma + i * AVE_PROC_SHDR_STRIDE); + ave_put32(cmd, l->ordinal, sess->frames); + pts = mul_u64_u64_div_u64((u64)sess->frames * p->fps_den, AVE_PROC_TIMESCALE, p->fps_num); + ave_put64(cmd, l->pts, pts); + ave_put32(cmd, l->force_key, force_key); + ave_put32(cmd, l->force_type, force_key ? AVE_PROC_FORCED_TYPE : 0); + + ret = ave_ipc_submit(ave, buf, tmpl->size, AVE_CMD_SESSION_TIMEOUT_US, false); + if (ret) + return ret; + s->src = src; + s->busy = true; + s->status = 0; + sess->busy++; + sess->frames++; + return 0; } diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c index edaaf29aa6739..5ed6cf9176a85 100644 --- a/drivers/media/platform/apple/ave/ave-drv.c +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -88,19 +88,19 @@ static void ave_remove(struct platform_device *pdev) ave_v4l2_unregister(ave); /* - * Make sure the coprocessor really loses power before its memory is - * freed: drop to runtime suspend synchronously and check. + * The coprocessor memory can only go once the firmware never ran or + * was reset by a power cycle. A halted coprocessor may keep power + * (its domain is shared with the DARTs), so its memory stays. */ pm_runtime_dont_use_autosuspend(ave->dev); - if (pm_runtime_suspended(ave->dev) || pm_runtime_status_suspended(ave->dev)) { - mutex_lock(&ave->hw_lock); - ave_fw_lost(ave); + mutex_lock(&ave->hw_lock); + if (ave->fw_state != AVE_FW_OFF) { mutex_unlock(&ave->hw_lock); - ave_fw_remove(ave); - } else { - /* The firmware still owns its buffers; leave them allocated. */ - dev_warn(ave->dev, "firmware still running, keeping its memory\n"); + dev_warn(ave->dev, "firmware was started, keeping its memory\n"); + return; } + mutex_unlock(&ave->hw_lock); + ave_fw_remove(ave); } static int ave_runtime_suspend(struct device *dev) @@ -123,12 +123,21 @@ static int ave_suspend(struct device *dev) { struct ave_dev *ave = dev_get_drvdata(dev); - /* Let a running frame finish; the m2m core holds back new jobs. */ + /* Let a running job finish; the m2m core holds back new ones. */ v4l2_m2m_suspend(ave->m2m_dev); + /* + * An open session cannot survive the power going down: it is ended + * here (its next job fails) and the firmware halted, so the next + * session restarts it from the saved DATA copy. + */ mutex_lock(&ave->hw_lock); - if (ave->fw_state != AVE_FW_OFF && ave_fw_shutdown(ave)) - dev_dbg(dev, "firmware will restart after resume\n"); + if (ave->active) { + ave->active->sess->dead = true; + ave->active = NULL; + } + if (ave_fw_shutdown(ave)) + dev_warn(dev, "firmware not answering, it cannot be stopped for sleep\n"); mutex_unlock(&ave->hw_lock); return 0; } @@ -137,15 +146,7 @@ static int ave_resume(struct device *dev) { struct ave_dev *ave = dev_get_drvdata(dev); - /* - * If the power domains went down with the system, the coprocessor is - * back in reset: open sessions fail their next frame and a new session - * boots the firmware again from its saved state. - */ - mutex_lock(&ave->hw_lock); - ave_fw_resume(ave); - mutex_unlock(&ave->hw_lock); - + /* The next boot checks the coprocessor state itself. */ v4l2_m2m_resume(ave->m2m_dev); return 0; } @@ -168,11 +169,17 @@ static struct platform_driver ave_driver = { .name = "apple-ave", .of_match_table = ave_of_match, .pm = pm_ptr(&ave_pm_ops), - /* The firmware cannot be stopped yet, so neither can the driver. */ + /* A started coprocessor keeps its memory; do not unbind at will. */ .suppress_bind_attrs = true, }, }; module_platform_driver(ave_driver); +/* Constant parts of Start and Process for AppleAVE2FW 9003.78.0 (macOS 26.0.x) */ +MODULE_FIRMWARE("apple/ave-t8140-69c54454-h264-start.bin"); +MODULE_FIRMWARE("apple/ave-t8140-69c54454-h264-process.bin"); +MODULE_FIRMWARE("apple/ave-t8140-69c54454-hevc-start.bin"); +MODULE_FIRMWARE("apple/ave-t8140-69c54454-hevc-process.bin"); + MODULE_DESCRIPTION("Apple AVE video encoder"); MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c index fa2df964d16f4..62eb494c149b7 100644 --- a/drivers/media/platform/apple/ave/ave-fw.c +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,7 @@ #include #include #include +#include #include #include "ave.h" @@ -133,6 +135,8 @@ int ave_fw_probe(struct ave_dev *ave) if (!known) return dev_err_probe(dev, -ENODEV, "unsupported firmware (text CRC %08x)\n", text_crc); + ave->text_crc = text_crc; + mutex_init(&ave->tmpl_lock); ave->data_pristine = vmalloc(ave->data.size); if (!ave->data_pristine) @@ -170,19 +174,24 @@ static void ave_fw_restore_data(struct ave_dev *ave) ave->data_dirty = false; } -static void ave_free(struct ave_dev *ave, struct ave_buf *b) +void ave_free(struct ave_dev *ave, struct ave_buf *b) { if (b->cpu) - dma_free_coherent(ave->dev, b->size, b->cpu, b->dma); + dma_free_attrs(ave->dev, PAGE_ALIGN(b->size), b->cpu, b->dma, b->attrs); b->cpu = NULL; } -static int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size) +/* + * Firmware-visible memory, zeroed and 16 KiB aligned in the encoder's DART. + * @size is what the firmware is told; the allocation is page rounded. + */ +int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, unsigned long attrs) { if (b->cpu) return 0; b->size = size; - b->cpu = dma_alloc_coherent(ave->dev, size, &b->dma, GFP_KERNEL); + b->attrs = attrs; + b->cpu = dma_alloc_attrs(ave->dev, PAGE_ALIGN(size), &b->dma, GFP_KERNEL, attrs); if (!b->cpu) return -ENOMEM; if (!IS_ALIGNED(b->dma, SZ_16K)) { @@ -192,8 +201,61 @@ static int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size) return 0; } +/* + * Start and Process are mostly constant for a firmware build. Those + * constant words come from per-build image files named after the TEXT + * checksum; the driver fills in every session and frame field itself. + */ +static const char * const ave_codec_name[] = { "h264", "hevc" }; +static const char * const ave_tmpl_name[AVE_TMPL_COUNT] = { "start", "process" }; +static const size_t ave_tmpl_size[2][AVE_TMPL_COUNT] = { + { 0x3180, 0x63d8 }, /* H.264 */ + { 0x13f28, 0xb1c0 }, /* HEVC */ +}; + +#define AVE_TMPL_CODEC 0xc0 /* Start: codec word of the session image */ + +int ave_fw_templates(struct ave_dev *ave, enum ave_codec codec) +{ + unsigned int c = codec - 1; + char name[64]; + int ret = 0; + + mutex_lock(&ave->tmpl_lock); + for (unsigned int t = 0; t < AVE_TMPL_COUNT && !ret; t++) { + const struct firmware *fw; + + if (ave->tmpl[c][t]) + continue; + snprintf(name, sizeof(name), "apple/ave-t8140-%08x-%s-%s.bin", ave->text_crc, + ave_codec_name[c], ave_tmpl_name[t]); + ret = request_firmware(&fw, name, ave->dev); + if (ret) { + dev_err(ave->dev, "cannot load %s: %d\n", name, ret); + break; + } + if (fw->size != ave_tmpl_size[c][t] || + (t == AVE_TMPL_START && + get_unaligned_le32(fw->data + AVE_TMPL_CODEC) != codec)) { + dev_err(ave->dev, "%s does not fit this driver (%zu bytes)\n", name, + fw->size); + release_firmware(fw); + ret = -EINVAL; + break; + } + ave->tmpl[c][t] = fw; + } + mutex_unlock(&ave->tmpl_lock); + return ret; +} + void ave_fw_remove(struct ave_dev *ave) { + for (unsigned int c = 0; c < 2; c++) + for (unsigned int t = 0; t < AVE_TMPL_COUNT; t++) { + release_firmware(ave->tmpl[c][t]); + ave->tmpl[c][t] = NULL; + } ave_free(ave, &ave->heap); ave_free(ave, &ave->log); ave_free(ave, &ave->boot_cfg); @@ -214,6 +276,38 @@ static bool ave_fw_idle(struct ave_dev *ave) return true; } +/* + * After a Halt the firmware no longer runs, but the coprocessor is still + * set going if its power stayed on. A restart uses the normal boot path + * (macOS restarts the firmware for every session the same way): clear the + * run bit, as the boot sequence does first anyway, and the mailbox words + * left from the previous run, so the reset-state checks hold again. The + * code base register must still be set: the driver never writes it. + */ +static int ave_fw_prepare_restart(struct ave_dev *ave) +{ + u32 status; + + if (readq(ave->coproc + AVE_COPROC_VCBAR) != (ave->text.iova | AVE_COPROC_VCBAR_VALID)) { + dev_err(ave->dev, "coprocessor lost its code base (vcbar %#llx)\n", + readq(ave->coproc + AVE_COPROC_VCBAR)); + return -EIO; + } + if (ave->fw_state != AVE_FW_HALTED) + return 0; + + if (readl(ave->coproc + AVE_COPROC_CTRL)) { + writel(0, ave->coproc + AVE_COPROC_CTRL); + usleep_range(1000, 2000); + } + status = readl(ave->mbox + AVE_MBOX_STATUS); + if (status) + writel(status, ave->mbox + AVE_MBOX_STATUS); + for (unsigned int i = 0; i < AVE_MBOX_SCRATCH_COUNT; i++) + writel(0, ave->mbox + AVE_MBOX_SCRATCH(i)); + return 0; +} + /* Boot the coprocessor and bring up its command rings. Caller holds hw_lock. */ static int ave_fw_boot(struct ave_dev *ave) { @@ -221,11 +315,20 @@ static int ave_fw_boot(struct ave_dev *ave) u32 status, hello[4]; int ret; - ret = ave_alloc(ave, &ave->ipc, AVE_IPC_BYTES); + ret = ave_alloc(ave, &ave->ipc, AVE_IPC_BYTES, 0); if (!ret) - ret = ave_alloc(ave, &ave->boot_cfg, SZ_16K); + ret = ave_alloc(ave, &ave->boot_cfg, SZ_16K, 0); if (!ret) - ret = ave_alloc(ave, &ave->log, AVE_LOG_BYTES); + ret = ave_alloc(ave, &ave->log, AVE_LOG_BYTES, 0); + if (ret) + return ret; + + if (ave_fw_check_mapping(ave, &ave->text) || ave_fw_check_mapping(ave, &ave->data)) { + dev_err(ave->dev, "firmware segments are no longer mapped\n"); + return -EIO; + } + + ret = ave_fw_prepare_restart(ave); if (ret) return ret; @@ -240,6 +343,13 @@ static int ave_fw_boot(struct ave_dev *ave) return -EIO; } + memset(ave->ring, 0, sizeof(ave->ring)); + ave->unacked = 0; + memset(ave->want, 0, sizeof(ave->want)); + ave->reply_err = 0; + ave->scalar_next = 0; + ave->next_token = AVE_TOKEN_BASE; + ave_fw_restore_data(ave); cfg = ave->boot_cfg.cpu; @@ -306,9 +416,9 @@ int ave_fw_ensure_running(struct ave_dev *ave) case AVE_FW_RUNNING: return 0; case AVE_FW_BROKEN: - /* Only a power cycle gets the coprocessor back; see runtime PM. */ return -EIO; case AVE_FW_OFF: + case AVE_FW_HALTED: break; } @@ -318,22 +428,47 @@ int ave_fw_ensure_running(struct ave_dev *ave) return ret; } +/* + * Tell the firmware to stop, once every session is closed. Halt gets no + * ack and no answer; after a short wait the coprocessor is treated as + * stopped and the next session restarts it through the boot path. + * Caller holds hw_lock. + */ +void ave_fw_halt(struct ave_dev *ave) +{ + lockdep_assert_held(&ave->hw_lock); + if (ave->fw_state != AVE_FW_RUNNING) + return; + if (ave_cmd_halt(ave)) { + ave->fw_state = AVE_FW_BROKEN; + return; + } + msleep(AVE_HALT_SETTLE_MS); + ave->fw_state = AVE_FW_HALTED; + memset(ave->ring, 0, sizeof(ave->ring)); + ave->ipc_fw_base = 0; +} + /* * Stop the firmware before its power goes away. Caller holds hw_lock. * - * The firmware is told to halt once every session is closed. Stopping the - * coprocessor core itself and checking that the fabric is idle are not - * described yet, so until then a running firmware keeps its power (runtime - * suspend fails with -EBUSY). Across system sleep the domains go down anyway; - * ave_fw_resume() notices and the next session boots the firmware again from - * the saved DATA copy. + * Sessions keep the device runtime active, so the firmware has normally + * been halted already. A firmware that stopped answering keeps its power: + * whether it still has transfers in flight cannot be checked, and only a + * reboot brings it back. */ int ave_fw_shutdown(struct ave_dev *ave) { lockdep_assert_held(&ave->hw_lock); - if (ave->fw_state == AVE_FW_OFF) + switch (ave->fw_state) { + case AVE_FW_RUNNING: + ave_fw_halt(ave); + return ave->fw_state == AVE_FW_HALTED ? 0 : -EBUSY; + case AVE_FW_BROKEN: + return -EBUSY; + default: return 0; - return -EBUSY; + } } /* The coprocessor lost power behind the firmware's back. Caller holds hw_lock. */ @@ -344,16 +479,3 @@ void ave_fw_lost(struct ave_dev *ave) memset(ave->ring, 0, sizeof(ave->ring)); ave->ipc_fw_base = 0; } - -/* After system resume: did the coprocessor keep running? Caller holds hw_lock. */ -void ave_fw_resume(struct ave_dev *ave) -{ - lockdep_assert_held(&ave->hw_lock); - if (ave->fw_state == AVE_FW_OFF) - return; - /* A running firmware keeps the device runtime active, so this is powered. */ - if (readl(ave->coproc + AVE_COPROC_CTRL) & AVE_COPROC_CTRL_RUN) - return; - dev_dbg(ave->dev, "coprocessor lost power during sleep\n"); - ave_fw_lost(ave); -} diff --git a/drivers/media/platform/apple/ave/ave-ipc.c b/drivers/media/platform/apple/ave/ave-ipc.c index 11f0aceb6fb65..d316edca01b3c 100644 --- a/drivers/media/platform/apple/ave/ave-ipc.c +++ b/drivers/media/platform/apple/ave/ave-ipc.c @@ -250,16 +250,172 @@ static int ave_ring_recv(struct ave_dev *ave, struct ave_ring *r, u64 *addr, u32 return 0; } +static int ave_ipc_buf_offset(unsigned int buf, size_t *max) +{ + if (buf == AVE_BUF_CMD) { + *max = AVE_IPC_CMD_BYTES; + return AVE_IPC_CMD_OFF; + } + if (buf < AVE_BUF_PROC(0)) { + *max = AVE_IPC_SCALAR_BYTES; + return AVE_IPC_SCALAR_OFF + (buf - AVE_BUF_SCALAR(0)) * AVE_IPC_SCALAR_BYTES; + } + *max = AVE_IPC_PROC_BYTES; + return AVE_IPC_PROC_OFF + (buf - AVE_BUF_PROC(0)) * AVE_IPC_PROC_BYTES; +} + +/* Which command buffer does a transport ack refer to? */ +static int ave_ipc_buf_of(struct ave_dev *ave, u64 addr) +{ + u64 off = addr - ave->ipc_fw_base; + size_t max; + + if (addr < ave->ipc_fw_base || off < AVE_IPC_CMD_OFF || off >= AVE_IPC_END) + return -EPROTO; + for (unsigned int buf = 0; buf < AVE_BUF_COUNT; buf++) + if (ave_ipc_buf_offset(buf, &max) == off) + return buf; + return -EPROTO; +} + +/* + * Get command buffer @buf, cleared for @len bytes, or NULL while the + * firmware still owns it. Caller holds hw_lock. + */ +void *ave_ipc_cmd(struct ave_dev *ave, unsigned int buf, size_t len) +{ + size_t max; + int off; + + if (buf >= AVE_BUF_COUNT || (ave->unacked & BIT(buf))) + return NULL; + off = ave_ipc_buf_offset(buf, &max); + if (len > max) + return NULL; + memset(ave->ipc.cpu + off, 0, len); + return ave->ipc.cpu + off; +} + +static bool ave_ipc_acked(struct ave_dev *ave, void *arg) +{ + return !(ave->unacked & BIT(*(unsigned int *)arg)); +} + +/* Wait until the firmware has taken command buffer @buf. Caller holds hw_lock. */ +int ave_ipc_wait_ack(struct ave_dev *ave, unsigned int buf) +{ + return ave_ipc_pump(ave, AVE_CMD_WAIT_MS, ave_ipc_acked, &buf); +} + +static bool ave_ipc_scalar_free(struct ave_dev *ave, void *arg) +{ + return (ave->unacked & GENMASK(AVE_BUF_PROC(0) - 1, AVE_BUF_SCALAR(0))) != + GENMASK(AVE_BUF_PROC(0) - 1, AVE_BUF_SCALAR(0)); +} + +/* Pick a free buffer for a scalar command. Caller holds hw_lock. */ +int ave_ipc_scalar_buf(struct ave_dev *ave) +{ + int ret; + + ret = ave_ipc_pump(ave, AVE_CMD_WAIT_MS, ave_ipc_scalar_free, NULL); + if (ret) + return ret; + for (unsigned int i = 0; i < AVE_IPC_SCALAR_BUFS; i++) { + unsigned int buf = AVE_BUF_SCALAR((ave->scalar_next + i) % AVE_IPC_SCALAR_BUFS); + + if (!(ave->unacked & BIT(buf))) { + ave->scalar_next = (buf - AVE_BUF_SCALAR(0) + 1) % AVE_IPC_SCALAR_BUFS; + return buf; + } + } + return -EBUSY; +} + +static void ave_ipc_mark_broken(struct ave_dev *ave, int err, const char *what) +{ + if (ave->fw_state == AVE_FW_RUNNING) + dev_err(ave->dev, "firmware stopped answering (%s: %d); it restarts after the next power cycle\n", + what, err); + ave->fw_state = AVE_FW_BROKEN; +} + +/* + * Hand the command in buffer @buf to the firmware. The timeout words are + * {duration, creation} in the host clock the firmware was synced to at boot. + * With @reply, the firmware's answer is counted as expected; Halt and + * Process are answered differently. Caller holds hw_lock. + */ +int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, u64 timeout_us, + bool reply) +{ + size_t max; + int off = ave_ipc_buf_offset(buf, &max); + u8 *cmd = ave->ipc.cpu + off; + u32 op = get_unaligned_le32(cmd + AVE_CMD_OPCODE); + int ret; + + lockdep_assert_held(&ave->hw_lock); + if (ave->fw_state != AVE_FW_RUNNING) + return -EIO; + if (len < AVE_CMD_HDR_BYTES || len > max || !IS_ALIGNED(len, 4) || + (ave->unacked & BIT(buf)) || (reply && op >= ARRAY_SIZE(ave->want))) + return -EINVAL; + + put_unaligned_le64(timeout_us, cmd + AVE_CMD_TIMEOUT); + put_unaligned_le64(div_u64(ktime_get_ns(), NSEC_PER_USEC), cmd + AVE_CMD_CREATED); + + ret = ave_ring_send(ave, &ave->ring[0], ave->ipc_fw_base + off, len); + if (ret) { + ave_ipc_mark_broken(ave, ret, "command ring full"); + return ret; + } + ave->unacked |= BIT(buf); + if (reply) + ave->want[op]++; + return 0; +} + +static void ave_ipc_dispatch(struct ave_dev *ave) +{ + u32 id, status; + + if (ave->msg_len < AVE_REPLY_MIN_BYTES) { + dev_warn(ave->dev, "short firmware message (%u bytes)\n", ave->msg_len); + ave->reply_err = -EPROTO; + return; + } + id = get_unaligned_le16(ave->msg + AVE_REPLY_ID); + status = get_unaligned_le32(ave->msg + AVE_REPLY_STATUS); + + if (id == AVE_OP_PROCESS) { + if (ave->active) + ave_sess_reply(ave->active, ave->msg, ave->msg_len); + else + dev_warn(ave->dev, "frame reply without a session\n"); + return; + } + if (id < ARRAY_SIZE(ave->want) && ave->want[id]) { + ave->want[id]--; + if (status) { + dev_err(ave->dev, "command %u refused: status %#x\n", id, status); + ave->reply_err = -EIO; + } + return; + } + dev_warn_ratelimited(ave->dev, "unexpected firmware message %u (status %#x)\n", + id, status); +} + /* - * Poll both rings once. Returns 1 when the command at @cmd_addr was - * acknowledged, 2 when a firmware message was copied to ave->msg (and its - * buffer handed back), 0 when nothing was pending. + * Poll both rings once. Returns 1 when an event (ack or message) was + * handled, 0 when nothing was pending. */ -static int ave_ipc_poll_once(struct ave_dev *ave, u64 cmd_addr) +static int ave_ipc_poll_once(struct ave_dev *ave) { u32 pending, len, off; u64 addr; - int ret; + int ret, buf; pending = readl(ave->mbox + AVE_MBOX_STATUS); if (pending) @@ -268,8 +424,12 @@ static int ave_ipc_poll_once(struct ave_dev *ave, u64 cmd_addr) ret = ave_ring_recv(ave, &ave->ring[0], &addr, &len); if (!ret) { /* Transport acks return the command address with no payload. */ - if (len || addr != cmd_addr) + buf = ave_ipc_buf_of(ave, addr); + if (len || buf < 0 || !(ave->unacked & BIT(buf))) { + dev_err(ave->dev, "bad command ack %#llx/%u\n", addr, len); return -EPROTO; + } + ave->unacked &= ~BIT(buf); return 1; } if (ret != -EAGAIN) @@ -282,103 +442,80 @@ static int ave_ipc_poll_once(struct ave_dev *ave, u64 cmd_addr) return ret; if (len > sizeof(ave->msg) || ave_fw_offset(ave, addr, len, &off)) return -EPROTO; + /* Messages must not land in the buffers the host writes commands to. */ + if (off < AVE_IPC_END && off + len > AVE_IPC_CMD_OFF) { + dev_err(ave->dev, "firmware message at %#x overlaps the command area\n", off); + return -EPROTO; + } memcpy(ave->msg, ave->ipc.cpu + off, len); ave->msg_len = len; /* Return the message buffer to the firmware. */ ret = ave_ring_send(ave, &ave->ring[1], addr, len); - return ret ?: 2; + if (ret) + return ret; + ave_ipc_dispatch(ave); + return 1; } /* - * Send one command and wait for its transport ack and the firmware's reply - * message. The reply is left in ave->msg. Caller holds hw_lock. + * Handle acks and messages until @cond holds, for at most @wait_ms. With no + * @cond, handle whatever is pending and return. Running out of time means + * the firmware is stuck: it is marked broken. Caller holds hw_lock. */ -int ave_ipc_call(struct ave_dev *ave, const void *cmd, size_t len) +int ave_ipc_pump(struct ave_dev *ave, unsigned int wait_ms, + bool (*cond)(struct ave_dev *ave, void *arg), void *arg) { - u64 cmd_addr = ave->ipc_fw_base + AVE_IPC_CMD_OFF; - u8 *buf = ave->ipc.cpu + AVE_IPC_CMD_OFF; - bool acked = false, replied = false; - ktime_t deadline; - u32 status; + ktime_t deadline = ktime_add_ms(ktime_get(), wait_ms); int ret; lockdep_assert_held(&ave->hw_lock); - if (ave->fw_state != AVE_FW_RUNNING) - return -EIO; - if (len < AVE_CMD_HDR_BYTES || len > AVE_IPC_CMD_BYTES || !IS_ALIGNED(len, 4)) - return -EINVAL; - - memcpy(buf, cmd, len); - /* The timeout is {duration, creation} in the clock synced at boot. */ - put_unaligned_le64(AVE_CMD_TIMEOUT_US, buf + AVE_CMD_TIMEOUT); - put_unaligned_le64(div_u64(ktime_get_ns(), NSEC_PER_USEC), buf + AVE_CMD_CREATED); - ave->msg_len = 0; - - ret = ave_ring_send(ave, &ave->ring[0], cmd_addr, len); - if (ret) - goto broken; - - deadline = ktime_add_ms(ktime_get(), AVE_CMD_WAIT_MS); - while (!acked || !replied) { - ret = ave_ipc_poll_once(ave, cmd_addr); - if (ret < 0) - goto broken; - if (ret == 1) - acked = true; - else if (ret == 2) - replied = true; - else if (ktime_after(ktime_get(), deadline)) - goto timeout; - else - usleep_range(50, 150); - } - - /* The firmware refused the command: the session fails, the firmware lives on. */ - if (ave->msg_len < AVE_REPLY_MIN_BYTES) - goto broken_len; - status = get_unaligned_le32(ave->msg + AVE_REPLY_STATUS); - if (status) { - dev_err(ave->dev, "command %u refused: status %#x\n", - get_unaligned_le32(cmd), status); - return -EIO; + for (;;) { + if (cond && cond(ave, arg)) + return 0; + if (ave->fw_state != AVE_FW_RUNNING) + return -EIO; + ret = ave_ipc_poll_once(ave); + if (ret < 0) { + ave_ipc_mark_broken(ave, ret, "ring error"); + return ret; + } + if (ret) + continue; + if (!cond) + return 0; + if (ktime_after(ktime_get(), deadline)) { + ave_ipc_mark_broken(ave, -ETIMEDOUT, "no answer"); + return -ETIMEDOUT; + } + usleep_range(50, 150); } - return 0; +} -broken_len: - ret = -EPROTO; - goto broken; -timeout: - ret = -ETIMEDOUT; -broken: - dev_err(ave->dev, "command %u failed: %d (acked %d, replied %d)\n", - get_unaligned_le32(cmd), ret, acked, replied); - ave->fw_state = AVE_FW_BROKEN; - return ret; +static bool ave_ipc_answered(struct ave_dev *ave, void *arg) +{ + u32 mask = *(u32 *)arg; + + if (ave->unacked & mask) + return false; + for (unsigned int i = 0; i < ARRAY_SIZE(ave->want); i++) + if (ave->want[i] && i != AVE_OP_HALT) + return false; + return true; } -/* Wait for one more firmware message (frame completions). */ -int ave_ipc_wait_msg(struct ave_dev *ave, unsigned int timeout_ms) +/* + * Wait until every command except Process has been acknowledged and + * answered. Returns the first refusal seen since the last call. + * Caller holds hw_lock. + */ +int ave_ipc_settle(struct ave_dev *ave, unsigned int wait_ms) { - ktime_t deadline = ktime_add_ms(ktime_get(), timeout_ms); + u32 mask = GENMASK(AVE_BUF_PROC(0) - 1, 0); int ret; - lockdep_assert_held(&ave->hw_lock); - if (ave->fw_state != AVE_FW_RUNNING) - return -EIO; - - for (;;) { - ret = ave_ipc_poll_once(ave, 0); - if (ret == 2) - return 0; - if (ret < 0) - break; - if (ktime_after(ktime_get(), deadline)) { - ret = -ETIMEDOUT; - break; - } - usleep_range(50, 150); - } - dev_err(ave->dev, "no firmware message: %d\n", ret); - ave->fw_state = AVE_FW_BROKEN; + ret = ave_ipc_pump(ave, wait_ms, ave_ipc_answered, &mask); + if (!ret) + ret = ave->reply_err; + ave->reply_err = 0; return ret; } diff --git a/drivers/media/platform/apple/ave/ave-regs.h b/drivers/media/platform/apple/ave/ave-regs.h index 91e7c2c403c3e..894d2e3d4ff0a 100644 --- a/drivers/media/platform/apple/ave/ave-regs.h +++ b/drivers/media/platform/apple/ave/ave-regs.h @@ -89,8 +89,15 @@ #define AVE_CMD_TIMEOUT 0x30 #define AVE_CMD_CREATED 0x38 #define AVE_CMD_HDR_BYTES 0x48 -#define AVE_CMD_TIMEOUT_US 3000000 + +/* Timeouts the firmware is given, and how long the host waits */ +#define AVE_CMD_GLOBAL_TIMEOUT_US 3000000ULL +#define AVE_CMD_SESSION_TIMEOUT_US 94400000ULL #define AVE_CMD_WAIT_MS 5000 +#define AVE_START_WAIT_MS 10000 +#define AVE_FRAME_WAIT_MS 5000 +/* Halt is never answered; give the firmware this long before power-off. */ +#define AVE_HALT_SETTLE_MS 20 enum ave_opcode { AVE_OP_CONFIG = 1, @@ -110,19 +117,31 @@ enum ave_opcode { #define AVE_REPLY_SUBTYPE 0x20 #define AVE_REPLY_STATUS 0x40 #define AVE_REPLY_MIN_BYTES 0x48 +#define AVE_REPLY_PROCESS_BYTES 0x50 #define AVE_HALT_ALL 0xffffffff -/* Session command arguments, as the firmware expects them */ +/* Session command header values */ +#define AVE_SESSION_UNIT 1 #define AVE_SESSION_ARG0 0xc8 #define AVE_SESSION_ARG1 0x36 -#define AVE_OPEN_ARG2 0x1e0 -#define AVE_FINISH_ARG2_H264 0x140 -#define AVE_FINISH_ARG2_HEVC 0x20 +#define AVE_SESSION_ARG2_SETUP 0x1e0 /* Open and Start */ +#define AVE_SESSION_ARG2_H264 0x40 /* every other session command */ +#define AVE_SESSION_ARG2_HEVC 0x20 +#define AVE_TOKEN_STEP 100 +#define AVE_TOKEN_BASE 300 /* the first Open carries 400 */ +#define AVE_CLIENT_STEP 10 + +/* Process slots: the command subtype is this plus the slot index */ +#define AVE_PROC_SUBTYPE 0x15 +/* Completes sent to drain a session */ +#define AVE_DRAIN_COMPLETES 4 #define AVE_CONFIG_BYTES 0x78 #define AVE_CONFIG_ALL_UNITS 0x20 #define AVE_CONFIG_OPT_A 0x49 #define AVE_CONFIG_OPT_B 0x4a +#define AVE_CONFIG_MCC_DSID 0x60 +#define AVE_CONFIG_MCC_DSID_T8140 0x12 #endif diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c new file mode 100644 index 0000000000000..4d65dd09d732a --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -0,0 +1,493 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder: encoding sessions + * + * A session owns the encoder's working buffers (sized from the picture + * size), a ring of up to five Process slots and the coded output of each + * slot. The firmware answers frames in bitstream order, so the N-th answer + * is the N-th access unit; with B-frames that is not the input order. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "ave.h" +#include "ave-regs.h" + +#define AVE_RC_VBR 4 +#define AVE_RC_CBR 2 + +#define AVE_H264_BASELINE 2 +#define AVE_H264_MAIN 4 +#define AVE_H264_HIGH 6 +#define AVE_HEVC_MAIN 1 + +#define AVE_META_BYTES 0xc000 +#define AVE_META_HEVC_PAYLOAD 0x120 +#define AVE_META_HEVC_SHDR 0x128 +#define AVE_META_HEVC_PAYLOAD2 0x12c +#define AVE_META_HEVC_PARAMS 0xb2a4 +#define AVE_SHDR_BYTES (256 * 0x400) +#define AVE_SHDR_ENTRY_BYTES 0x400 +#define AVE_CODED_MIN 0x3c000 +#define AVE_HDR_BYTES 0x12c000 + +/* Spacing of the H.264 per-frame buffer seen at 1080p; scaled with area. */ +#define AVE_H264_FRAME_BUF_1080P 0x364000 +#define AVE_MBS_1080P (120 * 68) + +/* Level limits: max frame size, max rate, max bit rate (kbit/s, Main). */ +struct ave_level { + u32 max_fs; + u64 max_rate; + u32 max_kbps; +}; + +/* H.264 Table A-1, in V4L2 menu order (1, 1b, 1.1 ... 5.2); macroblocks. */ +static const struct ave_level ave_h264_levels[] = { + { 99, 1485, 64 }, { 99, 1485, 128 }, { 396, 3000, 192 }, { 396, 6000, 384 }, + { 396, 11880, 768 }, { 396, 11880, 2000 }, { 792, 19800, 4000 }, + { 1620, 20250, 4000 }, { 1620, 40500, 10000 }, { 3600, 108000, 14000 }, + { 5120, 216000, 20000 }, { 8192, 245760, 20000 }, { 8192, 245760, 50000 }, + { 8704, 522240, 50000 }, { 22080, 589824, 135000 }, { 36864, 983040, 240000 }, + { 36864, 2073600, 240000 }, +}; + +/* HEVC Table A-8 (Main tier), in V4L2 menu order (1, 2, 2.1 ... 5.1); samples. */ +static const struct ave_level ave_hevc_levels[] = { + { 36864, 552960, 128 }, { 122880, 3686400, 1500 }, { 245760, 7372800, 3000 }, + { 552960, 16588800, 6000 }, { 983040, 33177600, 10000 }, + { 2228224, 66846720, 12000 }, { 2228224, 133693440, 20000 }, + { 8912896, 267386880, 25000 }, { 8912896, 534773760, 40000 }, +}; + +/* + * The lowest level that fits the picture, rate and bit rate, or the + * requested one if that is higher. Returned as the firmware's 1-based enum. + */ +static int ave_pick_level(const struct ave_params *p, u32 requested) +{ + bool hevc = p->codec == AVE_CODEC_HEVC; + const struct ave_level *t = hevc ? ave_hevc_levels : ave_h264_levels; + unsigned int n = hevc ? ARRAY_SIZE(ave_hevc_levels) : ARRAY_SIZE(ave_h264_levels); + u64 size = hevc ? (u64)p->cw * p->ch : p->mbs; + u64 side = hevc ? max(p->cw, p->ch) : p->cw / 16; + u64 rate = DIV_ROUND_UP_ULL(size * p->fps_num, p->fps_den); + u32 kbps = DIV_ROUND_UP(p->bitrate, 1000); + + for (unsigned int i = 0; i < n; i++) { + u32 max_kbps = t[i].max_kbps; + + if (p->high) + max_kbps += max_kbps / 4; + if (size <= t[i].max_fs && side * side <= 8ULL * t[i].max_fs && + rate <= t[i].max_rate && kbps <= max_kbps) + return max(i, requested) + 1; + } + return -EINVAL; +} + +static int ave_sess_params(struct ave_ctx *ctx, struct ave_params *p) +{ + bool baseline = false; + int level; + + p->codec = ave_ctx_codec(ctx); + p->width = ctx->src_fmt.width; + p->height = ctx->src_fmt.height; + p->cw = ALIGN(p->width, 16); + p->ch = ALIGN(p->height, 16); + p->stride = ctx->src_fmt.plane_fmt[0].bytesperline; + p->mbs = (p->cw / 16) * (p->ch / 16); + p->fps_num = ctx->timeperframe.denominator; + p->fps_den = ctx->timeperframe.numerator; + p->fps_int = max(DIV_ROUND_CLOSEST(p->fps_num, p->fps_den), 1U); + + if (p->codec == AVE_CODEC_H264) { + switch (ctx->h264_profile) { + case V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE: + case V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE: + p->profile = AVE_H264_BASELINE; + baseline = true; + break; + case V4L2_MPEG_VIDEO_H264_PROFILE_HIGH: + p->profile = AVE_H264_HIGH; + p->high = true; + break; + default: + p->profile = AVE_H264_MAIN; + break; + } + p->min_qp = ctx->h264_min_qp; + p->max_qp = ctx->h264_max_qp; + } else { + p->profile = AVE_HEVC_MAIN; + p->min_qp = ctx->hevc_min_qp; + p->max_qp = ctx->hevc_max_qp; + } + if (p->min_qp > p->max_qp) + p->min_qp = p->max_qp; + + /* + * Reference structures as macOS sets them up. H.264 has all three + * (B-frames, P only, intra only). HEVC without B-frames needs a + * reference-set table whose contents are not known yet, so HEVC + * always uses B-frames. + */ + p->intra_only = p->codec == AVE_CODEC_H264 && ctx->gop_size == 1; + p->bframes = p->codec == AVE_CODEC_HEVC || + (!baseline && !p->intra_only && ctx->b_frames); + p->refs = p->intra_only ? 1 : p->bframes ? 5 : 2; + p->slots = p->bframes ? 5 : 3; + p->gop = ctx->gop_size ?: 120; + + p->bitrate = ctx->bitrate; + if (ctx->bitrate_mode == V4L2_MPEG_VIDEO_BITRATE_MODE_CBR) { + p->rc_mode = AVE_RC_CBR; + p->limit_bytes = 0; + } else { + p->rc_mode = AVE_RC_VBR; + p->limit_bytes = max(ctx->bitrate_peak, ctx->bitrate) / 8; + } + + level = ave_pick_level(p, p->codec == AVE_CODEC_H264 ? ctx->h264_level : + ctx->hevc_level); + if (level < 0) + return level; + p->level = level; + + /* Coded-data capacity: 1.5 bytes per (aligned) pixel, 16 KiB pages. */ + p->coded_cap = ALIGN(3 * (p->codec == AVE_CODEC_HEVC ? ALIGN(p->width, 32) : + ALIGN(p->width, 16)) * + ALIGN(p->height, 16) / 2, SZ_16K); + p->coded_cap = max_t(u32, p->coded_cap, AVE_CODED_MIN); + return 0; +} + +/* Working buffer sizes, all from the requested picture size. */ +static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) +{ + const struct ave_params *p = &sess->p; + bool hevc = p->codec == AVE_CODEC_HEVC; + u32 w = p->width, h = p->height; + u32 w16 = DIV_ROUND_UP(w, 16), w32 = DIV_ROUND_UP(w, 32); + u32 h32 = DIV_ROUND_UP(h, 32), w64 = DIV_ROUND_UP(w, 64), h64 = DIV_ROUND_UP(h, 64); + const unsigned long nomap = DMA_ATTR_NO_KERNEL_MAPPING; + size_t ref_a, ref_b, ref_c, ref_d, tile, frame_buf; + int ret = 0; + + ref_a = ALIGN(w, 32) * ALIGN(h + 16, 32); + ref_b = roundup_pow_of_two(w) * roundup_pow_of_two(h) / 32; + ref_c = hevc ? 512 * w64 * h64 : ALIGN(w, 64) * (ALIGN(h, 16) / 2 + 24); + ref_d = ALIGN(hevc ? 64 * w32 * h32 : 128 * p->mbs, SZ_16K); + tile = hevc ? 2304 * w32 * h64 : ALIGN(w, 16) * ALIGN(h, 64); + + ret |= ave_alloc(ave, &sess->hdr, AVE_HDR_BYTES, nomap); + for (unsigned int i = 0; i < p->refs; i++) { + ret |= ave_alloc(ave, &sess->ref_a[i], ref_a, nomap); + ret |= ave_alloc(ave, &sess->ref_b[i], ref_b, nomap); + ret |= ave_alloc(ave, &sess->ref_c[i], ref_c, nomap); + ret |= ave_alloc(ave, &sess->ref_d[i], ref_d, nomap); + } + ret |= ave_alloc(ave, &sess->mb, ALIGN(82 * p->mbs, SZ_16K), nomap); + ret |= ave_alloc(ave, &sess->row_a, hevc ? SZ_16K : ALIGN(256 * w16, SZ_16K), nomap); + ret |= ave_alloc(ave, &sess->row_b, ALIGN(hevc ? 768 * w32 : 1024 * w16, SZ_16K), nomap); + ret |= ave_alloc(ave, &sess->fixed, SZ_16K, nomap); + for (unsigned int i = 0; i < (hevc ? 4 : AVE_TILE_BUFS); i++) { + ret |= ave_alloc(ave, &sess->tile[i], tile, nomap); + ret |= ave_alloc(ave, &sess->tile_aux[i], ALIGN(h, 128), nomap); + } + if (hevc) + for (unsigned int i = 0; i < 2; i++) + ret |= ave_alloc(ave, &sess->hevc_extra[i], ALIGN(w * h32, SZ_16K), nomap); + + /* Per-slot output; the size of the H.264 frame buffer is not known. */ + frame_buf = ALIGN(DIV_ROUND_UP_ULL((u64)AVE_H264_FRAME_BUF_1080P * p->mbs, AVE_MBS_1080P), + SZ_16K); + frame_buf = max_t(size_t, frame_buf, AVE_H264_FRAME_BUF_1080P); + for (unsigned int s = 0; s < p->slots; s++) { + ret |= ave_alloc(ave, &sess->slot[s].coded, p->coded_cap, 0); + ret |= ave_alloc(ave, &sess->slot[s].meta, AVE_META_BYTES, 0); + ret |= ave_alloc(ave, &sess->slot[s].aux, hevc ? AVE_SHDR_BYTES : frame_buf, + hevc ? 0 : nomap); + } + return ret ? -ENOMEM : 0; +} + +static void ave_sess_free_bufs(struct ave_dev *ave, struct ave_session *sess) +{ + ave_free(ave, &sess->hdr); + for (unsigned int i = 0; i < AVE_MAX_REFS; i++) { + ave_free(ave, &sess->ref_a[i]); + ave_free(ave, &sess->ref_b[i]); + ave_free(ave, &sess->ref_c[i]); + ave_free(ave, &sess->ref_d[i]); + } + ave_free(ave, &sess->mb); + ave_free(ave, &sess->row_a); + ave_free(ave, &sess->row_b); + ave_free(ave, &sess->fixed); + for (unsigned int i = 0; i < AVE_TILE_BUFS; i++) { + ave_free(ave, &sess->tile[i]); + ave_free(ave, &sess->tile_aux[i]); + } + for (unsigned int i = 0; i < 2; i++) + ave_free(ave, &sess->hevc_extra[i]); + for (unsigned int s = 0; s < AVE_MAX_SLOTS; s++) { + ave_free(ave, &sess->slot[s].coded); + ave_free(ave, &sess->slot[s].meta); + ave_free(ave, &sess->slot[s].aux); + } +} + +/* Work out the parameters and allocate the buffers; no firmware access. */ +struct ave_session *ave_sess_create(struct ave_ctx *ctx) +{ + struct ave_session *sess; + int ret; + + sess = kzalloc(sizeof(*sess), GFP_KERNEL); + if (!sess) + return ERR_PTR(-ENOMEM); + ret = ave_sess_params(ctx, &sess->p); + if (!ret) + ret = ave_sess_alloc(ctx->ave, sess); + if (ret) { + ave_sess_free_bufs(ctx->ave, sess); + kfree(sess); + return ERR_PTR(ret); + } + for (unsigned int s = 0; s < sess->p.slots; s++) + ave_sess_prepare_slot(sess, s); + dev_dbg(ctx->ave->dev, "session %ux%u codec %u profile %u level %u, %u slots, %u refs\n", + sess->p.width, sess->p.height, sess->p.codec, sess->p.profile, sess->p.level, + sess->p.slots, sess->p.refs); + return sess; +} + +/* + * Free the buffers. A firmware that stopped answering may still write to + * them, and a freed DART range is soon handed out again, so in that case + * they stay allocated (until reboot, like the firmware itself). + */ +void ave_sess_destroy(struct ave_ctx *ctx) +{ + struct ave_session *sess = ctx->sess; + + if (!sess) + return; + if (ctx->ave->fw_state == AVE_FW_BROKEN) + dev_warn(ctx->ave->dev, "keeping the buffers of a failed session\n"); + else + ave_sess_free_bufs(ctx->ave, sess); + kfree(sess); + ctx->sess = NULL; +} + +int ave_sess_free_slot(struct ave_session *sess) +{ + for (unsigned int s = 0; s < sess->p.slots; s++) + if (!sess->slot[s].busy && !sess->slot[s].done) + return s; + return -EBUSY; +} + +/* + * Clear what the firmware writes, so the output can be measured: the whole + * metadata buffer, the first slice header and (H.264) the part of the coded + * buffer the previous frame used. + */ +void ave_sess_prepare_slot(struct ave_session *sess, unsigned int slot) +{ + struct ave_slot *s = &sess->slot[slot]; + + memset(s->meta.cpu, 0, s->meta.size); + if (sess->p.codec == AVE_CODEC_HEVC) { + memset(s->aux.cpu, 0, AVE_SHDR_ENTRY_BYTES); + } else if (s->coded_dirty) { + memset(s->coded.cpu, 0, min(s->coded_dirty, s->coded.size)); + s->coded_dirty = 0; + } +} + +/* A frame reply: +0x10 our client, +0x20 the slot's subtype, +0x40 status. */ +void ave_sess_reply(struct ave_ctx *ctx, const u8 *msg, u32 len) +{ + struct ave_session *sess = ctx->sess; + u64 client = get_unaligned_le64(msg + AVE_REPLY_CLIENT); + u32 subtype = get_unaligned_le32(msg + AVE_REPLY_SUBTYPE); + u32 slot = subtype - AVE_PROC_SUBTYPE; + struct ave_slot *s; + + if (!sess || len < AVE_REPLY_PROCESS_BYTES || client != sess->client || + slot >= sess->p.slots || !sess->slot[slot].busy) { + dev_warn(ctx->ave->dev, "stray frame reply (client %llu subtype %#x len %u)\n", + client, subtype, len); + return; + } + s = &sess->slot[slot]; + s->status = get_unaligned_le32(msg + AVE_REPLY_STATUS); + if (s->status) + dev_dbg(ctx->ave->dev, "frame in slot %u: status %#x\n", slot, s->status); + s->busy = false; + s->done = true; + sess->busy--; + sess->done_fifo[(sess->done_head + sess->done_count) % AVE_MAX_SLOTS] = slot; + sess->done_count++; + atomic_inc(&ctx->ready); +} + +/* + * Copy Annex B data from @src to @dst until the data ends: NAL units never + * hold three zero bytes in a row, and the buffer was cleared before the + * firmware wrote it, so a run of @zero_run zero bytes marks the end. + * Returns the length or -ENOSPC. + */ +static ssize_t ave_copy_annexb(u8 *dst, size_t room, const u8 *src, size_t cap, + unsigned int zero_run) +{ + size_t pos = 0, end = 0; + unsigned int zeros = 0; + + while (pos < cap) { + size_t chunk = min3((size_t)SZ_16K, cap - pos, room - pos); + + if (!chunk) + return -ENOSPC; + memcpy(dst + pos, src + pos, chunk); + for (size_t i = pos; i < pos + chunk; i++) { + if (dst[i]) { + end = i + 1; + zeros = 0; + } else if (end && ++zeros >= zero_run) { + return end; + } + } + pos += chunk; + } + return end; +} + +static bool ave_starts_annexb(const u8 *p, size_t len) +{ + return (len >= 3 && !p[0] && !p[1] && p[2] == 1) || + (len >= 4 && !p[0] && !p[1] && !p[2] && p[3] == 1); +} + +/* NAL unit types in an Annex B buffer: is there a key frame / parameter set? */ +static void ave_scan_nals(const u8 *p, size_t len, bool hevc, bool *key, bool *params) +{ + for (size_t i = 0; i + 3 < len; i++) { + u32 type; + + if (p[i] || p[i + 1] || p[i + 2] != 1) + continue; + type = hevc ? (p[i + 3] >> 1) & 0x3f : p[i + 3] & 0x1f; + if (hevc) { + *key |= type >= 16 && type <= 21; + *params |= type >= 32 && type <= 34; + } else { + *key |= type == 5; + *params |= type == 7; + } + i += 2; + } +} + +static void ave_sess_dump(struct ave_ctx *ctx, struct ave_slot *s, const char *why) +{ + struct ave_session *sess = ctx->sess; + + if (sess->warned) + return; + sess->warned = true; + dev_warn(ctx->ave->dev, "frame %u output: %s (enable dynamic debug for a dump)\n", + ctx->dst_sequence, why); + print_hex_dump_debug("ave coded: ", DUMP_PREFIX_OFFSET, 16, 1, s->coded.cpu, 128, false); + print_hex_dump_debug("ave meta: ", DUMP_PREFIX_OFFSET, 16, 4, s->meta.cpu, 0x200, false); + if (sess->p.codec == AVE_CODEC_HEVC) + print_hex_dump_debug("ave meta params: ", DUMP_PREFIX_OFFSET, 16, 1, + s->meta.cpu + AVE_META_HEVC_PARAMS, 128, false); +} + +/* + * Assemble the access unit of @slot into @dst. + * + * HEVC: the firmware puts the parameter sets in the metadata buffer, the + * slice header in the first slice-header entry and the slice data in the + * coded buffer; metadata words give the two lengths. + * + * H.264: where the length is reported is not known yet, so the coded buffer + * is read as Annex B until the data ends. + */ +int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *dst, + bool *keyframe) +{ + struct ave_session *sess = ctx->sess; + struct ave_slot *s = &sess->slot[slot]; + bool hevc = sess->p.codec == AVE_CODEC_HEVC; + u8 *out = vb2_plane_vaddr(&dst->vb2_buf, 0); + size_t room = vb2_plane_size(&dst->vb2_buf, 0); + bool params = false; + ssize_t len = 0; + + *keyframe = false; + if (!out) + return -EFAULT; + if (s->status) + return -EIO; + + if (hevc) { + const u8 *meta = s->meta.cpu; + u32 shdr = get_unaligned_le32(meta + AVE_META_HEVC_SHDR); + u32 data = get_unaligned_le32(meta + AVE_META_HEVC_PAYLOAD); + ssize_t ps = 0; + + if (ave_starts_annexb(meta + AVE_META_HEVC_PARAMS, + AVE_META_BYTES - AVE_META_HEVC_PARAMS)) { + ps = ave_copy_annexb(out, room, meta + AVE_META_HEVC_PARAMS, + AVE_META_BYTES - AVE_META_HEVC_PARAMS, 16); + if (ps < 0) + return ps; + } + if (shdr > AVE_SHDR_ENTRY_BYTES || data > sess->p.coded_cap || + ps + shdr + data > room) { + dev_err(ctx->ave->dev, "bad HEVC output sizes %u/%u\n", shdr, data); + ave_sess_dump(ctx, s, "bad sizes"); + return -EPROTO; + } + if (get_unaligned_le32(meta + AVE_META_HEVC_PAYLOAD2) != data) + dev_dbg(ctx->ave->dev, "HEVC payload words differ: %u/%u\n", data, + get_unaligned_le32(meta + AVE_META_HEVC_PAYLOAD2)); + memcpy(out + ps, s->aux.cpu, shdr); + memcpy(out + ps + shdr, s->coded.cpu, data); + len = ps + shdr + data; + } else { + len = ave_copy_annexb(out, room, s->coded.cpu, s->coded.size, 256); + if (len < 0) { + dev_err(ctx->ave->dev, "coded frame larger than the CAPTURE buffer\n"); + return len; + } + s->coded_dirty = len + 256; + } + + if (!len) { + ave_sess_dump(ctx, s, "empty"); + return -ENODATA; + } + if (!ave_starts_annexb(out, len)) + ave_sess_dump(ctx, s, "not Annex B"); + ave_scan_nals(out, len, hevc, keyframe, ¶ms); + if (!ctx->dst_sequence && !params) + ave_sess_dump(ctx, s, "first frame has no parameter sets"); + vb2_set_plane_payload(&dst->vb2_buf, 0, len); + return 0; +} diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index 410b86d20b28b..2807d8e3041f1 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -16,18 +16,19 @@ #include #include "ave.h" +#include "ave-regs.h" #define AVE_NAME "apple-ave" /* - * Frame size limits and input stride alignment. Provisional: conservative - * values the hardware is known to cover (up to the 2408x1506 panel and 1080p - * and 4K class streams); refined once the firmware limits are described. + * Frame size limits and input stride alignment: the range the encoder was + * seen working at (320x240 up to the 2408x1506 panel), with a little room. + * Larger sizes need buffer sizes nobody has checked yet. */ -#define AVE_MIN_W 64 -#define AVE_MIN_H 64 -#define AVE_MAX_W 4096 -#define AVE_MAX_H 4096 +#define AVE_MIN_W 320 +#define AVE_MIN_H 240 +#define AVE_MAX_W 2560 +#define AVE_MAX_H 1600 #define AVE_STEP 16 #define AVE_STRIDE_ALIGN 64 #define AVE_DEFAULT_W 1920 @@ -90,8 +91,8 @@ static void ave_fill_coded(struct v4l2_pix_format_mplane *f) if (!ave_fmt_listed(ave_coded_fmts, ARRAY_SIZE(ave_coded_fmts), f->pixelformat)) f->pixelformat = ave_coded_fmts[0]; ave_clamp_size(&f->width, &f->height); - /* Room for an intra frame at low compression plus headers. */ - min = max_t(u32, f->width * f->height * 3 / 4, SZ_256K); + /* The firmware's coded-data capacity plus parameter sets and headers. */ + min = ALIGN(3 * ALIGN(f->width, 32) * ALIGN(f->height, 16) / 2, SZ_16K) + SZ_64K; f->num_planes = 1; f->plane_fmt[0].bytesperline = 0; f->plane_fmt[0].sizeimage = clamp_t(u32, f->plane_fmt[0].sizeimage, min, SZ_32M); @@ -298,25 +299,39 @@ static int ave_s_parm(struct file *file, void *priv, struct v4l2_streamparm *a) return 0; } +/* + * Drain: frames still inside the firmware (B-frames) only come out after a + * flush, so the driver tracks the drain point itself instead of using the + * m2m helpers, which would mark the next CAPTURE buffer last right away. + */ static int ave_encoder_cmd(struct file *file, void *priv, struct v4l2_encoder_cmd *ec) { struct ave_ctx *ctx = ave_file_to_ctx(file); + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; int ret; ret = v4l2_m2m_ioctl_try_encoder_cmd(file, priv, ec); if (ret) return ret; - if (!vb2_is_streaming(v4l2_m2m_get_src_vq(ctx->fh.m2m_ctx)) || - !vb2_is_streaming(v4l2_m2m_get_dst_vq(ctx->fh.m2m_ctx))) + if (!vb2_is_streaming(v4l2_m2m_get_src_vq(m2m)) || + !vb2_is_streaming(v4l2_m2m_get_dst_vq(m2m))) return 0; - ret = v4l2_m2m_ioctl_encoder_cmd(file, priv, ec); - if (ret) - return ret; - if (ec->cmd == V4L2_ENC_CMD_STOP && v4l2_m2m_has_stopped(ctx->fh.m2m_ctx)) - v4l2_event_queue_fh(&ctx->fh, &ave_eos_event); - if (ec->cmd == V4L2_ENC_CMD_START && v4l2_m2m_has_stopped(ctx->fh.m2m_ctx)) - vb2_clear_last_buffer_dequeued(v4l2_m2m_get_dst_vq(ctx->fh.m2m_ctx)); + if (ec->cmd == V4L2_ENC_CMD_STOP) { + if (ctx->draining || ctx->stopped) + return 0; + ctx->drain_src = v4l2_m2m_last_src_buf(m2m); + ctx->draining = true; + if (!ctx->drain_src) + WRITE_ONCE(ctx->flush_pending, true); + } else { + if (ctx->draining && !ctx->stopped) + return -EBUSY; + ctx->draining = false; + ctx->stopped = false; + vb2_clear_last_buffer_dequeued(v4l2_m2m_get_dst_vq(m2m)); + } + v4l2_m2m_try_schedule(m2m); return 0; } @@ -374,7 +389,7 @@ static int ave_s_ctrl(struct v4l2_ctrl *ctrl) ctx->bitrate_mode = ctrl->val; break; case V4L2_CID_MPEG_VIDEO_BITRATE: - ctx->bitrate = ctrl->val; + WRITE_ONCE(ctx->bitrate, ctrl->val); break; case V4L2_CID_MPEG_VIDEO_BITRATE_PEAK: ctx->bitrate_peak = ctrl->val; @@ -383,7 +398,10 @@ static int ave_s_ctrl(struct v4l2_ctrl *ctrl) ctx->gop_size = ctrl->val; break; case V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME: - ctx->force_key_frame = true; + WRITE_ONCE(ctx->force_key_frame, true); + break; + case V4L2_CID_MPEG_VIDEO_B_FRAMES: + ctx->b_frames = ctrl->val; break; case V4L2_CID_MPEG_VIDEO_H264_PROFILE: ctx->h264_profile = ctrl->val; @@ -391,18 +409,12 @@ static int ave_s_ctrl(struct v4l2_ctrl *ctrl) case V4L2_CID_MPEG_VIDEO_H264_LEVEL: ctx->h264_level = ctrl->val; break; - case V4L2_CID_MPEG_VIDEO_H264_I_PERIOD: - ctx->h264_i_period = ctrl->val; - break; case V4L2_CID_MPEG_VIDEO_H264_MIN_QP: ctx->h264_min_qp = ctrl->val; break; case V4L2_CID_MPEG_VIDEO_H264_MAX_QP: ctx->h264_max_qp = ctrl->val; break; - case V4L2_CID_MPEG_VIDEO_HEVC_PROFILE: - ctx->hevc_profile = ctrl->val; - break; case V4L2_CID_MPEG_VIDEO_HEVC_LEVEL: ctx->hevc_level = ctrl->val; break; @@ -413,7 +425,7 @@ static int ave_s_ctrl(struct v4l2_ctrl *ctrl) ctx->hevc_max_qp = ctrl->val; break; case V4L2_CID_MPEG_VIDEO_HEADER_MODE: - case V4L2_CID_MPEG_VIDEO_B_FRAMES: + case V4L2_CID_MPEG_VIDEO_HEVC_PROFILE: case V4L2_CID_MIN_BUFFERS_FOR_OUTPUT: break; default: @@ -432,19 +444,27 @@ static int ave_init_ctrls(struct ave_ctx *ctx) const struct v4l2_ctrl_ops *ops = &ave_ctrl_ops; struct v4l2_ctrl *c; - v4l2_ctrl_handler_init(hdl, 18); + v4l2_ctrl_handler_init(hdl, 17); + /* Constant quality needs a quality-to-QP mapping that is not known yet. */ v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_BITRATE_MODE, - V4L2_MPEG_VIDEO_BITRATE_MODE_CQ, - 0, V4L2_MPEG_VIDEO_BITRATE_MODE_VBR); - v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_BITRATE, 100000, 200000000, 1, + V4L2_MPEG_VIDEO_BITRATE_MODE_CBR, + BIT(V4L2_MPEG_VIDEO_BITRATE_MODE_CQ), + V4L2_MPEG_VIDEO_BITRATE_MODE_VBR); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_BITRATE, 100000, 100000000, 1, + 4000000); + /* In VBR mode the peak caps the data rate over any one second. */ + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_BITRATE_PEAK, 100000, 200000000, 1, 8000000); - v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_BITRATE_PEAK, 100000, 400000000, 1, - 16000000); - v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_GOP_SIZE, 0, 65535, 1, 60); + /* 0 means no limit; the firmware then uses 120 frames. */ + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_GOP_SIZE, 0, 65535, 1, 30); v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME, 0, 0, 0, 0); - /* No frame reordering until reference handling is described. */ - v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_B_FRAMES, 0, 0, 1, 0); + /* + * The firmware either uses up to three B-frames (reordering them + * itself) or none. Baseline and intra-only H.264 never have any; + * HEVC always has them for now. + */ + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_B_FRAMES, 0, 3, 3, 3); v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_HEADER_MODE, V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME, BIT(V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE), @@ -456,24 +476,26 @@ static int ave_init_ctrls(struct ave_ctx *ctx) BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE) | BIT(V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) | BIT(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH)), - V4L2_MPEG_VIDEO_H264_PROFILE_HIGH); + V4L2_MPEG_VIDEO_H264_PROFILE_MAIN); + /* A minimum: the driver raises it to what size, rate and bit rate need. */ v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_H264_LEVEL, V4L2_MPEG_VIDEO_H264_LEVEL_5_2, 0, - V4L2_MPEG_VIDEO_H264_LEVEL_4_1); - v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_H264_I_PERIOD, 0, 65535, 1, 0); + V4L2_MPEG_VIDEO_H264_LEVEL_1_0); v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_H264_MIN_QP, 0, 51, 1, 0); v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_H264_MAX_QP, 0, 51, 1, 51); v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_HEVC_PROFILE, - V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10, 0, + V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN, 0, V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN); v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_HEVC_LEVEL, V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1, 0, - V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1); + V4L2_MPEG_VIDEO_HEVC_LEVEL_1); v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_HEVC_MIN_QP, 0, 51, 1, 0); v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_HEVC_MAX_QP, 0, 51, 1, 51); - c = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MIN_BUFFERS_FOR_OUTPUT, 1, 32, 1, 1); + /* The firmware holds up to five frames; one more keeps it fed. */ + c = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MIN_BUFFERS_FOR_OUTPUT, 1, 32, 1, + AVE_MAX_SLOTS + 1); if (c) c->flags |= V4L2_CTRL_FLAG_READ_ONLY; @@ -496,6 +518,10 @@ static int ave_queue_setup(struct vb2_queue *vq, unsigned int *nbufs, unsigned i const struct v4l2_pix_format_mplane *f = V4L2_TYPE_IS_OUTPUT(vq->type) ? &ctx->src_fmt : &ctx->dst_fmt; + /* Frames stay with the firmware until encoded; keep one spare. */ + if (V4L2_TYPE_IS_OUTPUT(vq->type) && + vb2_get_num_buffers(vq) + *nbufs < AVE_MAX_SLOTS + 1) + *nbufs = AVE_MAX_SLOTS + 1 - vb2_get_num_buffers(vq); if (*nplanes) return (*nplanes != 1 || sizes[0] < f->plane_fmt[0].sizeimage) ? -EINVAL : 0; *nplanes = 1; @@ -519,17 +545,8 @@ static int ave_buf_prepare(struct vb2_buffer *vb) static void ave_buf_queue(struct vb2_buffer *vb) { struct ave_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); - struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb); - if (V4L2_TYPE_IS_CAPTURE(vb->type) && vb2_is_streaming(vb->vb2_queue) && - v4l2_m2m_dst_buf_is_last(ctx->fh.m2m_ctx)) { - vb2_set_plane_payload(vb, 0, 0); - vbuf->sequence = ctx->dst_sequence++; - v4l2_m2m_last_buffer_done(ctx->fh.m2m_ctx, vbuf); - v4l2_event_queue_fh(&ctx->fh, &ave_eos_event); - return; - } - v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf); + v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, to_vb2_v4l2_buffer(vb)); } static void ave_return_bufs(struct ave_ctx *ctx, struct vb2_queue *q, @@ -547,30 +564,95 @@ static void ave_return_bufs(struct ave_ctx *ctx, struct vb2_queue *q, } } +/* Sessions */ + +/* Caller holds hw_lock. */ +static void ave_session_close(struct ave_ctx *ctx, bool flush) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + + lockdep_assert_held(&ave->hw_lock); + if (!sess || ave->active != ctx) + return; + if (sess->fw_open && ave->fw_state == AVE_FW_RUNNING) { + /* Complete is a barrier: answered after every frame before it. */ + if (flush && sess->frames && !ave_cmd_complete(ctx, AVE_DRAIN_COMPLETES)) + ave_ipc_settle(ave, AVE_FRAME_WAIT_MS); + if (ave->fw_state == AVE_FW_RUNNING) + ave_cmd_close(ctx); + } + sess->fw_open = false; + ave->active = NULL; + ave_fw_halt(ave); +} + +/* Return frames the firmware never answered; then drop the session. */ +static void ave_session_free(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + + if (!sess) + return; + for (unsigned int i = 0; i < sess->p.slots; i++) { + struct ave_slot *s = &sess->slot[i]; + + if (s->src) + v4l2_m2m_buf_done(s->src, VB2_BUF_STATE_ERROR); + s->src = NULL; + } + atomic_set(&ctx->ready, 0); + ave_sess_destroy(ctx); + pm_runtime_mark_last_busy(ave->dev); + pm_runtime_put_autosuspend(ave->dev); +} + +/* Boot the firmware if needed, then Open and Start a session. */ static int ave_session_begin(struct ave_ctx *ctx) { struct ave_dev *ave = ctx->ave; - enum ave_codec codec = ave_ctx_codec(ctx); + struct ave_session *sess; int ret; - ret = pm_runtime_resume_and_get(ave->dev); + if (ctx->sess) + return 0; + ret = ave_fw_templates(ave, ave_ctx_codec(ctx)); if (ret) return ret; + if (READ_ONCE(ave->active)) + return -EBUSY; + sess = ave_sess_create(ctx); + if (IS_ERR(sess)) + return PTR_ERR(sess); + + ret = pm_runtime_resume_and_get(ave->dev); + if (ret) { + ctx->sess = sess; + ave_sess_destroy(ctx); + return ret; + } mutex_lock(&ave->hw_lock); + ctx->sess = sess; + if (ave->active) { + ret = -EBUSY; + goto out; + } ret = ave_fw_ensure_running(ave); + if (ret) + goto out; + ave->active = ctx; + ret = ave_cmd_open(ctx); if (!ret) - ret = ave_cmd_open(ctx, codec); - if (!ret) - ret = ave_cmd_start(ctx, codec); - if (ret && ctx->session_open) - ave_cmd_finish(ctx, codec); + ret = ave_cmd_start(ctx); + if (ret) + ave_session_close(ctx, false); +out: mutex_unlock(&ave->hw_lock); - if (ret) { - dev_dbg(ave->dev, "session start failed: %d\n", ret); - pm_runtime_mark_last_busy(ave->dev); - pm_runtime_put_autosuspend(ave->dev); + dev_err(ave->dev, "cannot start an encoding session: %d\n", ret); + ave_session_free(ctx); } return ret; } @@ -579,14 +661,19 @@ static void ave_session_end(struct ave_ctx *ctx) { struct ave_dev *ave = ctx->ave; - cancel_work_sync(&ctx->run_work); + cancel_delayed_work_sync(&ctx->poll_work); mutex_lock(&ave->hw_lock); - if (ave->fw_state == AVE_FW_RUNNING) - ave_cmd_finish(ctx, ave_ctx_codec(ctx)); - ctx->session_open = false; + ave_session_close(ctx, true); mutex_unlock(&ave->hw_lock); - pm_runtime_mark_last_busy(ave->dev); - pm_runtime_put_autosuspend(ave->dev); + ave_session_free(ctx); +} + +static void ave_reset_drain(struct ave_ctx *ctx) +{ + ctx->draining = false; + WRITE_ONCE(ctx->flush_pending, false); + ctx->stopped = false; + ctx->drain_src = NULL; } static int ave_start_streaming(struct vb2_queue *q, unsigned int count) @@ -597,14 +684,14 @@ static int ave_start_streaming(struct vb2_queue *q, unsigned int count) v4l2_m2m_get_src_vq(ctx->fh.m2m_ctx); int ret = 0; - v4l2_m2m_update_start_streaming_state(ctx->fh.m2m_ctx, q); if (V4L2_TYPE_IS_OUTPUT(q->type)) ctx->src_sequence = 0; else ctx->dst_sequence = 0; + ave_reset_drain(ctx); - /* The session starts once both queues stream. */ - if (vb2_is_streaming(other) && !ctx->session_open) + /* The session starts once both queues stream; errors show up here. */ + if (vb2_is_streaming(other)) ret = ave_session_begin(ctx); if (ret) ave_return_bufs(ctx, q, VB2_BUF_STATE_QUEUED); @@ -615,12 +702,9 @@ static void ave_stop_streaming(struct vb2_queue *q) { struct ave_ctx *ctx = vb2_get_drv_priv(q); - if (ctx->session_open) - ave_session_end(ctx); + ave_session_end(ctx); + ave_reset_drain(ctx); ave_return_bufs(ctx, q, VB2_BUF_STATE_ERROR); - v4l2_m2m_update_stop_streaming_state(ctx->fh.m2m_ctx, q); - if (V4L2_TYPE_IS_OUTPUT(q->type) && v4l2_m2m_has_stopped(ctx->fh.m2m_ctx)) - v4l2_event_queue_fh(&ctx->fh, &ave_eos_event); } static const struct vb2_ops ave_qops = { @@ -661,50 +745,266 @@ static int ave_queue_init(void *priv, struct vb2_queue *src, struct vb2_queue *d return vb2_queue_init(dst); } -/* Jobs: one raw frame in, one coded frame out. */ +/* + * Jobs. A job delivers finished frames to CAPTURE buffers and sends at most + * one new frame. Up to five frames are inside the firmware at once; their + * answers are collected by the job and, while frames are outstanding, by a + * short poller, since the firmware interrupt is not wired up. + */ + +#define AVE_POLL_MS 2 + +static int ave_job_ready(void *priv) +{ + struct ave_ctx *ctx = priv; + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + + if (atomic_read(&ctx->ready) || READ_ONCE(ctx->flush_pending)) + return 1; + if (ctx->stopped || (ctx->draining && !ctx->drain_src)) + return 0; + return v4l2_m2m_num_src_bufs_ready(m2m) > 0; +} + +/* + * Hand finished frames, in bitstream order, to CAPTURE buffers. Returns + * true when the frame that ends a drain went out (with the LAST flag). + */ +static bool ave_deliver(struct ave_ctx *ctx) +{ + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + struct ave_session *sess = ctx->sess; + bool last = false; + + while (sess && sess->done_count) { + unsigned int slot = sess->done_fifo[sess->done_head]; + struct ave_slot *s = &sess->slot[slot]; + struct vb2_v4l2_buffer *dst; + enum vb2_buffer_state state = VB2_BUF_STATE_DONE; + bool key = false; + + dst = v4l2_m2m_dst_buf_remove(m2m); + if (!dst) + break; + if (ave_sess_output(ctx, slot, dst, &key)) { + vb2_set_plane_payload(&dst->vb2_buf, 0, 0); + state = VB2_BUF_STATE_ERROR; + } + v4l2_m2m_buf_copy_metadata(s->src, dst); + dst->flags &= ~(V4L2_BUF_FLAG_KEYFRAME | V4L2_BUF_FLAG_PFRAME | + V4L2_BUF_FLAG_BFRAME | V4L2_BUF_FLAG_LAST); + dst->flags |= key ? V4L2_BUF_FLAG_KEYFRAME : V4L2_BUF_FLAG_PFRAME; + dst->sequence = ctx->dst_sequence++; + + sess->done_head = (sess->done_head + 1) % AVE_MAX_SLOTS; + sess->done_count--; + atomic_dec(&ctx->ready); + s->done = false; + ave_sess_prepare_slot(sess, slot); + + /* The last frame of a drain carries the LAST flag. */ + if (READ_ONCE(ctx->flush_pending) && !sess->fw_open && !sess->done_count) { + dst->flags |= V4L2_BUF_FLAG_LAST; + last = true; + } + v4l2_m2m_buf_done(s->src, state); + s->src = NULL; + v4l2_m2m_buf_done(dst, state); + } + return last; +} + +static bool ave_slot_answered(struct ave_dev *ave, void *arg) +{ + struct ave_session *sess = arg; + + return sess->busy < sess->p.slots || sess->done_count; +} + +static bool ave_frame_answered(struct ave_dev *ave, void *arg) +{ + struct ave_slot *s = arg; + + return !s->busy; +} + +/* Send the next OUTPUT frame. Caller holds hw_lock. */ +static int ave_encode_one(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + struct vb2_v4l2_buffer *src; + bool last; + int slot, ret; + + slot = ave_sess_free_slot(sess); + if (slot < 0 && sess->busy == sess->p.slots) { + /* Every slot is inside the firmware: wait for one to come back. */ + ret = ave_ipc_pump(ave, AVE_FRAME_WAIT_MS, ave_slot_answered, sess); + if (ret) + return ret; + ave_deliver(ctx); + slot = ave_sess_free_slot(sess); + } + if (slot < 0) + return 0; /* waiting for CAPTURE buffers */ + + src = v4l2_m2m_src_buf_remove(m2m); + if (!src) + return 0; + last = src == ctx->drain_src; + src->sequence = ctx->src_sequence++; + + ret = ave_cmd_process(ctx, slot, src, READ_ONCE(ctx->bitrate), + xchg(&ctx->force_key_frame, false)); + if (ret) { + v4l2_m2m_buf_done(src, VB2_BUF_STATE_ERROR); + return ret; + } + + /* Like macOS: the first frame goes alone, then a Complete. */ + if (sess->frames == 1) { + ret = ave_ipc_pump(ave, AVE_FRAME_WAIT_MS, ave_frame_answered, &sess->slot[slot]); + if (!ret) + ret = ave_cmd_complete(ctx, 1); + if (!ret) + ret = ave_ipc_settle(ave, AVE_FRAME_WAIT_MS); + if (ret) + return ret; + } else { + ret = ave_ipc_pump(ave, 0, NULL, NULL); + if (ret) + return ret; + } + + if (last) { + ctx->drain_src = NULL; + WRITE_ONCE(ctx->flush_pending, true); + } + return 0; +} + +/* + * End of a drain: flush and close the session, deliver what is left and + * finish with a LAST buffer. Caller holds hw_lock; returns true once the + * LAST buffer is out. + */ +static bool ave_finish_drain(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + struct vb2_v4l2_buffer *dst; + + if (sess && sess->fw_open) + ave_session_close(ctx, true); + if (sess && sess->busy) { + /* The run job fails the session. */ + dev_err(ave->dev, "%u frames never came back from the firmware\n", sess->busy); + return false; + } + if (sess && sess->done_count) + return ave_deliver(ctx); /* false: waiting for CAPTURE buffers */ + + /* Nothing left to deliver: an empty buffer carries the LAST flag. */ + dst = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + if (!dst) + return false; + vb2_set_plane_payload(&dst->vb2_buf, 0, 0); + dst->flags |= V4L2_BUF_FLAG_LAST; + dst->sequence = ctx->dst_sequence++; + v4l2_m2m_buf_done(dst, VB2_BUF_STATE_DONE); + return true; +} + +/* The firmware or the session failed: return everything, fail the queues. */ +static void ave_job_fail(struct ave_ctx *ctx, int err) +{ + struct ave_dev *ave = ctx->ave; + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + + dev_err(ave->dev, "encoding failed: %d\n", err); + mutex_lock(&ave->hw_lock); + ave_session_close(ctx, false); + mutex_unlock(&ave->hw_lock); + ave_session_free(ctx); + ave_return_bufs(ctx, v4l2_m2m_get_src_vq(m2m), VB2_BUF_STATE_ERROR); + ave_return_bufs(ctx, v4l2_m2m_get_dst_vq(m2m), VB2_BUF_STATE_ERROR); + vb2_queue_error(v4l2_m2m_get_src_vq(m2m)); + vb2_queue_error(v4l2_m2m_get_dst_vq(m2m)); + ave_reset_drain(ctx); +} static void ave_run_work(struct work_struct *work) { struct ave_ctx *ctx = container_of(work, struct ave_ctx, run_work); struct ave_dev *ave = ctx->ave; - struct vb2_v4l2_buffer *src, *dst; - enum vb2_buffer_state state = VB2_BUF_STATE_DONE; - bool keyframe = false; - u32 bytes = 0; - int ret; + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + bool eos = false, poll = false; + int ret = 0; - src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); - dst = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); + /* After a drain, the next frame starts a new session. */ + if (!ctx->sess && !READ_ONCE(ctx->flush_pending) && v4l2_m2m_num_src_bufs_ready(m2m)) + ret = ave_session_begin(ctx); mutex_lock(&ave->hw_lock); - ret = ave_cmd_process(ctx, ave_ctx_codec(ctx), src, dst, &bytes, &keyframe); - ctx->force_key_frame = false; + if (!ret && ctx->sess && (ctx->sess->dead || ave->active != ctx)) + ret = -EIO; + if (!ret) { + eos = ave_deliver(ctx); + if (!eos && !READ_ONCE(ctx->flush_pending) && ctx->sess && + v4l2_m2m_num_src_bufs_ready(m2m) && v4l2_m2m_num_dst_bufs_ready(m2m)) + ret = ave_encode_one(ctx); + } + if (!ret && !eos) { + eos = ave_deliver(ctx); + if (!eos && READ_ONCE(ctx->flush_pending)) + eos = ave_finish_drain(ctx); + /* Frames outstanding need a running firmware. */ + if (!eos && ctx->sess && ctx->sess->busy && ave->fw_state != AVE_FW_RUNNING) + ret = -EIO; + poll = !ret && !eos && ctx->sess && ctx->sess->busy; + } mutex_unlock(&ave->hw_lock); + if (ret) { - dev_dbg(ave->dev, "frame %u failed: %d\n", ctx->src_sequence, ret); - state = VB2_BUF_STATE_ERROR; - bytes = 0; + ave_job_fail(ctx, ret); + } else if (eos) { + ave_session_free(ctx); + ctx->draining = false; + WRITE_ONCE(ctx->flush_pending, false); + ctx->stopped = true; + v4l2_event_queue_fh(&ctx->fh, &ave_eos_event); + } else if (poll) { + mod_delayed_work(ave->wq, &ctx->poll_work, msecs_to_jiffies(AVE_POLL_MS)); } + pm_runtime_mark_last_busy(ave->dev); + v4l2_m2m_job_finish(ave->m2m_dev, m2m); +} - src = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); - dst = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); - src->sequence = ctx->src_sequence++; - dst->sequence = ctx->dst_sequence++; - v4l2_m2m_buf_copy_metadata(src, dst); - dst->flags &= ~(V4L2_BUF_FLAG_KEYFRAME | V4L2_BUF_FLAG_PFRAME | V4L2_BUF_FLAG_LAST); - dst->flags |= keyframe ? V4L2_BUF_FLAG_KEYFRAME : V4L2_BUF_FLAG_PFRAME; - vb2_set_plane_payload(&dst->vb2_buf, 0, bytes); +/* Collect frame answers while frames are outstanding and no job runs. */ +static void ave_poll_work(struct work_struct *work) +{ + struct ave_ctx *ctx = container_of(to_delayed_work(work), struct ave_ctx, poll_work); + struct ave_dev *ave = ctx->ave; + bool again = false, fail = false; - if (v4l2_m2m_is_last_draining_src_buf(ctx->fh.m2m_ctx, src)) { - dst->flags |= V4L2_BUF_FLAG_LAST; - v4l2_event_queue_fh(&ctx->fh, &ave_eos_event); - v4l2_m2m_mark_stopped(ctx->fh.m2m_ctx); + mutex_lock(&ave->hw_lock); + if (ctx->sess && ave->active == ctx && !ctx->sess->dead) { + fail = ave_ipc_pump(ave, 0, NULL, NULL) != 0; + again = !fail && ctx->sess->busy; + } else if (ctx->sess && ctx->sess->busy) { + fail = true; } + mutex_unlock(&ave->hw_lock); - v4l2_m2m_buf_done(src, state); - v4l2_m2m_buf_done(dst, state); - pm_runtime_mark_last_busy(ave->dev); - v4l2_m2m_job_finish(ave->m2m_dev, ctx->fh.m2m_ctx); + /* Jobs run on the same ordered queue, so none runs right now. */ + if (fail) + ave_job_fail(ctx, -EIO); + else if (atomic_read(&ctx->ready)) + v4l2_m2m_try_schedule(ctx->fh.m2m_ctx); + if (again) + queue_delayed_work(ave->wq, &ctx->poll_work, msecs_to_jiffies(AVE_POLL_MS)); } static void ave_device_run(void *priv) @@ -716,6 +1016,7 @@ static void ave_device_run(void *priv) static const struct v4l2_m2m_ops ave_m2m_ops = { .device_run = ave_device_run, + .job_ready = ave_job_ready, }; /* File operations */ @@ -731,6 +1032,8 @@ static int ave_open(struct file *file) return -ENOMEM; ctx->ave = ave; INIT_WORK(&ctx->run_work, ave_run_work); + INIT_DELAYED_WORK(&ctx->poll_work, ave_poll_work); + atomic_set(&ctx->ready, 0); ave_reset_fmts(ctx); v4l2_fh_init(&ctx->fh, video_devdata(file)); @@ -743,6 +1046,8 @@ static int ave_open(struct file *file) ret = PTR_ERR(ctx->fh.m2m_ctx); goto err_ctrls; } + /* Finished frames can be delivered with no new OUTPUT buffer queued. */ + v4l2_m2m_set_src_buffered(ctx->fh.m2m_ctx, true); v4l2_fh_add(&ctx->fh, file); return 0; @@ -762,6 +1067,8 @@ static int ave_release(struct file *file) mutex_lock(&ave->vdev_lock); v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); mutex_unlock(&ave->vdev_lock); + cancel_delayed_work_sync(&ctx->poll_work); + cancel_work_sync(&ctx->run_work); v4l2_ctrl_handler_free(&ctx->hdl); v4l2_fh_del(&ctx->fh, file); v4l2_fh_exit(&ctx->fh); diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h index 6421a49e7fdd5..20d0c6831f5ef 100644 --- a/drivers/media/platform/apple/ave/ave.h +++ b/drivers/media/platform/apple/ave/ave.h @@ -7,6 +7,7 @@ #ifndef _APPLE_AVE_H_ #define _APPLE_AVE_H_ +#include #include #include #include @@ -19,17 +20,37 @@ #include struct dev_pm_domain_list; +struct firmware; struct iommu_domain; -/* Coprocessor memory every firmware boot uses, all inside one allocation. */ +/* Frames the firmware can hold at once (Process "slots"). */ +#define AVE_MAX_SLOTS 5 + +/* + * Coprocessor memory every firmware boot uses, all inside one allocation. + * Host commands live in three areas: one large buffer for Start, small + * buffers for the scalar commands and one buffer per Process slot, so a + * command stays untouched until the firmware acknowledges it. + */ #define AVE_IPC_BYTES (20 * SZ_1M) #define AVE_IPC_CHANNEL_OFF 0x4000 #define AVE_IPC_HANDSHAKE_OFF 0x24000 #define AVE_IPC_SHARED_OFF 0x28000 #define AVE_IPC_SHARED_BYTES SZ_64K #define AVE_IPC_CMD_OFF 0x40000 -#define AVE_IPC_CMD_BYTES SZ_128K -#define AVE_IPC_END (AVE_IPC_CMD_OFF + AVE_IPC_CMD_BYTES) +#define AVE_IPC_CMD_BYTES 0x1c000 +#define AVE_IPC_SCALAR_OFF 0x5c000 +#define AVE_IPC_SCALAR_BYTES 0x100 +#define AVE_IPC_SCALAR_BUFS 16 +#define AVE_IPC_PROC_OFF 0x60000 +#define AVE_IPC_PROC_BYTES 0x10000 +#define AVE_IPC_END (AVE_IPC_PROC_OFF + AVE_MAX_SLOTS * AVE_IPC_PROC_BYTES) + +/* Command buffer ids, one bit each in ave_dev.unacked. */ +#define AVE_BUF_CMD 0 +#define AVE_BUF_SCALAR(i) (1 + (i)) +#define AVE_BUF_PROC(s) (1 + AVE_IPC_SCALAR_BUFS + (s)) +#define AVE_BUF_COUNT AVE_BUF_PROC(AVE_MAX_SLOTS) #define AVE_MSG_MAX SZ_4K @@ -42,6 +63,7 @@ struct ave_buf { void *cpu; dma_addr_t dma; size_t size; + unsigned long attrs; }; struct ave_fw_segment { @@ -61,11 +83,21 @@ struct ave_ring { }; enum ave_fw_state { - AVE_FW_OFF, + AVE_FW_OFF, /* coprocessor in reset */ AVE_FW_RUNNING, + AVE_FW_HALTED, /* told to halt; restarts through the boot path */ AVE_FW_BROKEN, }; +/* Session command images that only differ from the listed fields. */ +enum ave_tmpl { + AVE_TMPL_START, + AVE_TMPL_PROCESS, + AVE_TMPL_COUNT, +}; + +struct ave_ctx; + struct ave_dev { struct device *dev; void __iomem *codec; @@ -76,9 +108,14 @@ struct ave_dev { struct ave_fw_segment text; struct ave_fw_segment data; + u32 text_crc; void *data_pristine; bool data_dirty; + /* Protects tmpl[]; the images stay loaded until the driver goes. */ + struct mutex tmpl_lock; + const struct firmware *tmpl[2][AVE_TMPL_COUNT]; + /* Serialises every firmware and ring access. */ struct mutex hw_lock; enum ave_fw_state fw_state; @@ -91,6 +128,11 @@ struct ave_dev { struct ave_ring ring[2]; u64 next_token; u32 next_client; + u32 unacked; + u8 want[16]; + int reply_err; + unsigned int scalar_next; + struct ave_ctx *active; u8 msg[AVE_MSG_MAX]; u32 msg_len; @@ -101,6 +143,69 @@ struct ave_dev { struct workqueue_struct *wq; }; +/* Encoding parameters, fixed for the lifetime of a session. */ +struct ave_params { + enum ave_codec codec; + u32 width, height; /* visible picture */ + u32 cw, ch; /* coded size, multiples of 16 */ + u32 stride; /* NV12 line stride, both planes */ + u32 mbs; /* 16x16 macroblocks */ + u32 fps_num, fps_den; /* frames per second = num / den */ + u32 fps_int; /* the same, rounded */ + u32 slots; + u32 refs; /* max_num_ref_frames + 1 */ + bool bframes; + bool intra_only; + u32 gop; + u32 rc_mode; + u32 bitrate; + u32 limit_bytes; /* 0: no data-rate limit */ + s32 min_qp; + u32 max_qp; + u32 profile; /* H.264: 2, 4, 6; HEVC: 1 */ + u32 level; /* firmware level enum, 1-based */ + bool high; + u32 coded_cap; +}; + +struct ave_slot { + struct ave_buf coded; + struct ave_buf meta; + struct ave_buf aux; /* H.264 per-frame buffer, HEVC slice headers */ + struct vb2_v4l2_buffer *src; + size_t coded_dirty; + u32 status; + bool busy; /* Process sent, reply pending */ + bool done; /* reply in, output not delivered yet */ +}; + +#define AVE_MAX_REFS 5 +#define AVE_TILE_BUFS 8 + +struct ave_session { + struct ave_params p; + struct ave_buf hdr; + struct ave_buf ref_a[AVE_MAX_REFS]; + struct ave_buf ref_b[AVE_MAX_REFS]; + struct ave_buf ref_c[AVE_MAX_REFS]; + struct ave_buf ref_d[AVE_MAX_REFS]; + struct ave_buf mb; + struct ave_buf row_a, row_b, fixed; + struct ave_buf tile[AVE_TILE_BUFS]; + struct ave_buf tile_aux[AVE_TILE_BUFS]; + struct ave_buf hevc_extra[2]; + struct ave_slot slot[AVE_MAX_SLOTS]; + u8 done_fifo[AVE_MAX_SLOTS]; + unsigned int done_head; + unsigned int done_count; + unsigned int busy; + u32 frames; /* Process commands sent */ + u32 client; + bool fw_open; /* Open sent, Close not yet */ + bool dead; /* firmware gone; buffers only */ + bool warned; +}; + struct ave_ctx { struct v4l2_fh fh; struct ave_dev *ave; @@ -109,24 +214,28 @@ struct ave_ctx { struct v4l2_pix_format_mplane dst_fmt; struct v4l2_fract timeperframe; struct work_struct run_work; + struct delayed_work poll_work; u32 bitrate_mode; u32 bitrate; u32 bitrate_peak; u32 gop_size; + u32 b_frames; u32 h264_profile; u32 h264_level; - u32 h264_i_period; u32 h264_min_qp; u32 h264_max_qp; - u32 hevc_profile; u32 hevc_level; u32 hevc_min_qp; u32 hevc_max_qp; bool force_key_frame; - u32 client; - bool session_open; + struct ave_session *sess; + atomic_t ready; /* outputs waiting for a CAPTURE buffer */ + bool draining; + bool flush_pending; + bool stopped; + struct vb2_v4l2_buffer *drain_src; u32 src_sequence; u32 dst_sequence; }; @@ -140,24 +249,42 @@ static inline struct ave_ctx *ave_file_to_ctx(struct file *file) int ave_fw_probe(struct ave_dev *ave); void ave_fw_remove(struct ave_dev *ave); int ave_fw_ensure_running(struct ave_dev *ave); +void ave_fw_halt(struct ave_dev *ave); int ave_fw_shutdown(struct ave_dev *ave); void ave_fw_lost(struct ave_dev *ave); -void ave_fw_resume(struct ave_dev *ave); +int ave_fw_templates(struct ave_dev *ave, enum ave_codec codec); +int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, unsigned long attrs); +void ave_free(struct ave_dev *ave, struct ave_buf *b); /* ave-ipc.c */ int ave_ipc_negotiate(struct ave_dev *ave, const u32 hello[4]); -int ave_ipc_call(struct ave_dev *ave, const void *cmd, size_t len); -int ave_ipc_wait_msg(struct ave_dev *ave, unsigned int timeout_ms); +void *ave_ipc_cmd(struct ave_dev *ave, unsigned int buf, size_t len); +int ave_ipc_scalar_buf(struct ave_dev *ave); +int ave_ipc_wait_ack(struct ave_dev *ave, unsigned int buf); +int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, u64 timeout_us, + bool reply); +int ave_ipc_pump(struct ave_dev *ave, unsigned int wait_ms, + bool (*cond)(struct ave_dev *ave, void *arg), void *arg); +int ave_ipc_settle(struct ave_dev *ave, unsigned int wait_ms); /* ave-cmd.c */ int ave_cmd_config(struct ave_dev *ave); int ave_cmd_halt(struct ave_dev *ave); -int ave_cmd_open(struct ave_ctx *ctx, enum ave_codec codec); -int ave_cmd_start(struct ave_ctx *ctx, enum ave_codec codec); -int ave_cmd_process(struct ave_ctx *ctx, enum ave_codec codec, - struct vb2_v4l2_buffer *src, struct vb2_v4l2_buffer *dst, - u32 *bytesused, bool *keyframe); -int ave_cmd_finish(struct ave_ctx *ctx, enum ave_codec codec); +int ave_cmd_open(struct ave_ctx *ctx); +int ave_cmd_start(struct ave_ctx *ctx); +int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *src, + u32 bitrate, bool force_key); +int ave_cmd_complete(struct ave_ctx *ctx, unsigned int count); +int ave_cmd_close(struct ave_ctx *ctx); + +/* ave-sess.c */ +struct ave_session *ave_sess_create(struct ave_ctx *ctx); +void ave_sess_destroy(struct ave_ctx *ctx); +int ave_sess_free_slot(struct ave_session *sess); +void ave_sess_reply(struct ave_ctx *ctx, const u8 *msg, u32 len); +int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *dst, + bool *keyframe); +void ave_sess_prepare_slot(struct ave_session *sess, unsigned int slot); /* ave-v4l2.c */ int ave_v4l2_register(struct ave_dev *ave); From 731f241335e9fcfbcf26423a877c7e34946d963c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 18:42:36 +1100 Subject: [PATCH 1381/1579] media: apple: ave: take the H.264 length from the metadata, fail stuck frames The first hardware run encoded frames, and showed three problems. H.264 output was guessed from the coded buffer. The firmware reports the coded length at metadata +0x120 (repeated at +0x12c) for H.264 as for HEVC, and the coded buffer holds only the slice NAL units. Use that length for both codecs, and take the parameter sets from the metadata buffer: at +0xb2a4 where HEVC has them, otherwise from the first SPS (H.264) or VPS (HEVC) start code found past the header words. Frames that the firmware stops answering left the client waiting forever: the poller had no time limit. When frames are outstanding that the firmware must answer (any frame without B-frames, four with them) and nothing comes back for 5 s, the firmware is marked stuck and the session fails, which completes every buffer with an error. The run also showed DART write faults right after Start. To find their source, the driver now logs its buffer map under dynamic debug and warns about any word in the Start or Process images that looks like an encoder DART address but is not one of the session's buffers, which would be a stale address left in a session image. The output dump now also covers the metadata around the parameter sets and every nonzero metadata line. Field layouts are facts from our private research (macOS traces) and our own hardware runs; no copied code. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/media/platform/apple/ave/ave-cmd.c | 3 + drivers/media/platform/apple/ave/ave-sess.c | 248 +++++++++++++++----- drivers/media/platform/apple/ave/ave-v4l2.c | 11 + drivers/media/platform/apple/ave/ave.h | 6 +- 4 files changed, 207 insertions(+), 61 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-cmd.c b/drivers/media/platform/apple/ave/ave-cmd.c index 0b539fc201c3b..8514509b13bdf 100644 --- a/drivers/media/platform/apple/ave/ave-cmd.c +++ b/drivers/media/platform/apple/ave/ave-cmd.c @@ -470,6 +470,7 @@ int ave_cmd_start(struct ave_ctx *ctx) AVE_SESSION_ARG2_SETUP); ave_start_surfaces(sess, cmd); ave_start_config(&sess->p, cmd); + ave_sess_check_addrs(ctx, cmd, tmpl->size, AVE_TMPL_START, 0); ret = ave_ipc_submit(ave, AVE_BUF_CMD, tmpl->size, AVE_CMD_SESSION_TIMEOUT_US, true); return ret ?: ave_ipc_settle(ave, AVE_START_WAIT_MS); @@ -609,6 +610,7 @@ int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff ave_put64(cmd, l->pts, pts); ave_put32(cmd, l->force_key, force_key); ave_put32(cmd, l->force_type, force_key ? AVE_PROC_FORCED_TYPE : 0); + ave_sess_check_addrs(ctx, cmd, tmpl->size, AVE_TMPL_PROCESS, y); ret = ave_ipc_submit(ave, buf, tmpl->size, AVE_CMD_SESSION_TIMEOUT_US, false); if (ret) @@ -616,6 +618,7 @@ int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff s->src = src; s->busy = true; s->status = 0; + sess->progress = jiffies; sess->busy++; sess->frames++; return 0; diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index 4d65dd09d732a..aa93d50fd1143 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -10,6 +10,7 @@ * Copyright The Aurora Silicon Contributors */ #include +#include #include #include #include @@ -31,10 +32,12 @@ #define AVE_HEVC_MAIN 1 #define AVE_META_BYTES 0xc000 -#define AVE_META_HEVC_PAYLOAD 0x120 +/* Metadata words: coded bytes (twice), HEVC slice-header bytes, parameter sets */ +#define AVE_META_PAYLOAD 0x120 #define AVE_META_HEVC_SHDR 0x128 -#define AVE_META_HEVC_PAYLOAD2 0x12c -#define AVE_META_HEVC_PARAMS 0xb2a4 +#define AVE_META_PAYLOAD2 0x12c +#define AVE_META_PARAMS 0xb2a4 +#define AVE_META_SCAN_FROM 0x200 #define AVE_SHDR_BYTES (256 * 0x400) #define AVE_SHDR_ENTRY_BYTES 0x400 #define AVE_CODED_MIN 0x3c000 @@ -248,6 +251,39 @@ static void ave_sess_free_bufs(struct ave_dev *ave, struct ave_session *sess) } } +/* Debug map of the session buffers, to match DART faults to a buffer. */ +static void ave_log_buf(struct ave_dev *ave, const char *name, int i, const struct ave_buf *b) +{ + if (b->cpu) + dev_dbg(ave->dev, "buf %s%d %pad +%#zx\n", name, i, &b->dma, b->size); +} + +static void ave_sess_log_map(struct ave_dev *ave, struct ave_session *sess) +{ + ave_log_buf(ave, "hdr", 0, &sess->hdr); + ave_log_buf(ave, "mb", 0, &sess->mb); + ave_log_buf(ave, "row_a", 0, &sess->row_a); + ave_log_buf(ave, "row_b", 0, &sess->row_b); + ave_log_buf(ave, "fixed", 0, &sess->fixed); + for (int i = 0; i < 2; i++) + ave_log_buf(ave, "hevc_extra", i, &sess->hevc_extra[i]); + for (int i = 0; i < AVE_MAX_REFS; i++) { + ave_log_buf(ave, "ref_a", i, &sess->ref_a[i]); + ave_log_buf(ave, "ref_b", i, &sess->ref_b[i]); + ave_log_buf(ave, "ref_c", i, &sess->ref_c[i]); + ave_log_buf(ave, "ref_d", i, &sess->ref_d[i]); + } + for (int i = 0; i < AVE_TILE_BUFS; i++) { + ave_log_buf(ave, "tile", i, &sess->tile[i]); + ave_log_buf(ave, "tile_aux", i, &sess->tile_aux[i]); + } + for (int i = 0; i < AVE_MAX_SLOTS; i++) { + ave_log_buf(ave, "coded", i, &sess->slot[i].coded); + ave_log_buf(ave, "meta", i, &sess->slot[i].meta); + ave_log_buf(ave, "aux", i, &sess->slot[i].aux); + } +} + /* Work out the parameters and allocate the buffers; no firmware access. */ struct ave_session *ave_sess_create(struct ave_ctx *ctx) { @@ -267,6 +303,7 @@ struct ave_session *ave_sess_create(struct ave_ctx *ctx) } for (unsigned int s = 0; s < sess->p.slots; s++) ave_sess_prepare_slot(sess, s); + ave_sess_log_map(ctx->ave, sess); dev_dbg(ctx->ave->dev, "session %ux%u codec %u profile %u level %u, %u slots, %u refs\n", sess->p.width, sess->p.height, sess->p.codec, sess->p.profile, sess->p.level, sess->p.slots, sess->p.refs); @@ -301,21 +338,16 @@ int ave_sess_free_slot(struct ave_session *sess) } /* - * Clear what the firmware writes, so the output can be measured: the whole - * metadata buffer, the first slice header and (H.264) the part of the coded - * buffer the previous frame used. + * Clear what the firmware reports through, so a stale value is never taken + * for a new one: the whole metadata buffer and the first slice header. */ void ave_sess_prepare_slot(struct ave_session *sess, unsigned int slot) { struct ave_slot *s = &sess->slot[slot]; memset(s->meta.cpu, 0, s->meta.size); - if (sess->p.codec == AVE_CODEC_HEVC) { + if (sess->p.codec == AVE_CODEC_HEVC) memset(s->aux.cpu, 0, AVE_SHDR_ENTRY_BYTES); - } else if (s->coded_dirty) { - memset(s->coded.cpu, 0, min(s->coded_dirty, s->coded.size)); - s->coded_dirty = 0; - } } /* A frame reply: +0x10 our client, +0x20 the slot's subtype, +0x40 status. */ @@ -334,6 +366,7 @@ void ave_sess_reply(struct ave_ctx *ctx, const u8 *msg, u32 len) return; } s = &sess->slot[slot]; + sess->progress = jiffies; s->status = get_unaligned_le32(msg + AVE_REPLY_STATUS); if (s->status) dev_dbg(ctx->ave->dev, "frame in slot %u: status %#x\n", slot, s->status); @@ -402,9 +435,29 @@ static void ave_scan_nals(const u8 *p, size_t len, bool hevc, bool *key, bool *p } } +/* Where in the metadata buffer do the parameter sets start, if anywhere? */ +static int ave_find_params(const u8 *meta, bool hevc) +{ + if (ave_starts_annexb(meta + AVE_META_PARAMS, AVE_META_BYTES - AVE_META_PARAMS)) + return AVE_META_PARAMS; + /* Not at the HEVC offset: look for an SPS (H.264) or VPS (HEVC). */ + for (int i = AVE_META_SCAN_FROM; i + 4 < AVE_META_BYTES; i++) { + u8 type; + + if (meta[i] || meta[i + 1] || meta[i + 2] != 1) + continue; + type = hevc ? (meta[i + 3] >> 1) & 0x3f : meta[i + 3] & 0x1f; + if (type == (hevc ? 32 : 7)) + return i > AVE_META_SCAN_FROM && !meta[i - 1] ? i - 1 : i; + } + return -ENOENT; +} + static void ave_sess_dump(struct ave_ctx *ctx, struct ave_slot *s, const char *why) { struct ave_session *sess = ctx->sess; + const u8 *meta = s->meta.cpu; + int ps; if (sess->warned) return; @@ -412,21 +465,27 @@ static void ave_sess_dump(struct ave_ctx *ctx, struct ave_slot *s, const char *w dev_warn(ctx->ave->dev, "frame %u output: %s (enable dynamic debug for a dump)\n", ctx->dst_sequence, why); print_hex_dump_debug("ave coded: ", DUMP_PREFIX_OFFSET, 16, 1, s->coded.cpu, 128, false); - print_hex_dump_debug("ave meta: ", DUMP_PREFIX_OFFSET, 16, 4, s->meta.cpu, 0x200, false); - if (sess->p.codec == AVE_CODEC_HEVC) - print_hex_dump_debug("ave meta params: ", DUMP_PREFIX_OFFSET, 16, 1, - s->meta.cpu + AVE_META_HEVC_PARAMS, 128, false); + print_hex_dump_debug("ave meta: ", DUMP_PREFIX_OFFSET, 16, 4, meta, 0x200, false); + print_hex_dump_debug("ave meta b200: ", DUMP_PREFIX_OFFSET, 16, 1, meta + 0xb200, + 0x200, false); + ps = ave_find_params(meta, sess->p.codec == AVE_CODEC_HEVC); + if (ps >= 0) { + dev_dbg(ctx->ave->dev, "parameter sets at metadata +%#x\n", ps); + print_hex_dump_debug("ave meta params: ", DUMP_PREFIX_OFFSET, 16, 1, meta + ps, + min(128, AVE_META_BYTES - ps), false); + } + /* Anything else the firmware wrote past the header words. */ + for (int i = AVE_META_SCAN_FROM; i < AVE_META_BYTES; i += 16) + if (memchr_inv(meta + i, 0, 16)) + dev_dbg(ctx->ave->dev, "metadata +%#x: %16ph\n", i, meta + i); } /* - * Assemble the access unit of @slot into @dst. - * - * HEVC: the firmware puts the parameter sets in the metadata buffer, the - * slice header in the first slice-header entry and the slice data in the - * coded buffer; metadata words give the two lengths. - * - * H.264: where the length is reported is not known yet, so the coded buffer - * is read as Annex B until the data ends. + * Assemble the access unit of @slot into @dst: the parameter sets the + * firmware leaves in the metadata buffer (when it emits them), for HEVC the + * slice header from the first slice-header entry, then the coded data. + * Metadata +0x120 (repeated at +0x12c) holds the coded length for both + * codecs, +0x128 the HEVC slice-header length. */ int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *dst, bool *keyframe) @@ -436,8 +495,12 @@ int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff bool hevc = sess->p.codec == AVE_CODEC_HEVC; u8 *out = vb2_plane_vaddr(&dst->vb2_buf, 0); size_t room = vb2_plane_size(&dst->vb2_buf, 0); + const u8 *meta = s->meta.cpu; + u32 data, data2, shdr = 0; bool params = false; - ssize_t len = 0; + ssize_t ps = 0; + size_t len; + int at; *keyframe = false; if (!out) @@ -445,44 +508,31 @@ int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff if (s->status) return -EIO; - if (hevc) { - const u8 *meta = s->meta.cpu; - u32 shdr = get_unaligned_le32(meta + AVE_META_HEVC_SHDR); - u32 data = get_unaligned_le32(meta + AVE_META_HEVC_PAYLOAD); - ssize_t ps = 0; - - if (ave_starts_annexb(meta + AVE_META_HEVC_PARAMS, - AVE_META_BYTES - AVE_META_HEVC_PARAMS)) { - ps = ave_copy_annexb(out, room, meta + AVE_META_HEVC_PARAMS, - AVE_META_BYTES - AVE_META_HEVC_PARAMS, 16); - if (ps < 0) - return ps; - } - if (shdr > AVE_SHDR_ENTRY_BYTES || data > sess->p.coded_cap || - ps + shdr + data > room) { - dev_err(ctx->ave->dev, "bad HEVC output sizes %u/%u\n", shdr, data); - ave_sess_dump(ctx, s, "bad sizes"); - return -EPROTO; - } - if (get_unaligned_le32(meta + AVE_META_HEVC_PAYLOAD2) != data) - dev_dbg(ctx->ave->dev, "HEVC payload words differ: %u/%u\n", data, - get_unaligned_le32(meta + AVE_META_HEVC_PAYLOAD2)); - memcpy(out + ps, s->aux.cpu, shdr); - memcpy(out + ps + shdr, s->coded.cpu, data); - len = ps + shdr + data; - } else { - len = ave_copy_annexb(out, room, s->coded.cpu, s->coded.size, 256); - if (len < 0) { - dev_err(ctx->ave->dev, "coded frame larger than the CAPTURE buffer\n"); - return len; - } - s->coded_dirty = len + 256; + data = get_unaligned_le32(meta + AVE_META_PAYLOAD); + data2 = get_unaligned_le32(meta + AVE_META_PAYLOAD2); + if (hevc) + shdr = get_unaligned_le32(meta + AVE_META_HEVC_SHDR); + if (data != data2) + dev_dbg(ctx->ave->dev, "coded length words differ: %u/%u\n", data, data2); + + at = ave_find_params(meta, hevc); + if (at >= 0) { + ps = ave_copy_annexb(out, room, meta + at, AVE_META_BYTES - at, 16); + if (ps < 0) + return ps; } - - if (!len) { - ave_sess_dump(ctx, s, "empty"); - return -ENODATA; + if (!data || data > sess->p.coded_cap || shdr > AVE_SHDR_ENTRY_BYTES || + ps + shdr + data > room) { + dev_err(ctx->ave->dev, "bad output sizes %u/%u (capacity %u, room %zu)\n", + shdr, data, sess->p.coded_cap, room); + ave_sess_dump(ctx, s, "bad sizes"); + return -EPROTO; } + if (hevc) + memcpy(out + ps, s->aux.cpu, shdr); + memcpy(out + ps + shdr, s->coded.cpu, data); + len = ps + shdr + data; + if (!ave_starts_annexb(out, len)) ave_sess_dump(ctx, s, "not Annex B"); ave_scan_nals(out, len, hevc, keyframe, ¶ms); @@ -491,3 +541,81 @@ int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff vb2_set_plane_payload(&dst->vb2_buf, 0, len); return 0; } + +/* + * Every 64-bit word in a command that looks like an encoder DART address + * should be one of ours. One that is not came from the session image (a + * stale macOS address) and makes the firmware write to an unmapped page. + * Reported once per session and command kind. + */ +static bool ave_buf_has(const struct ave_buf *b, u64 addr) +{ + return b->cpu && addr >= b->dma && addr < b->dma + PAGE_ALIGN(b->size); +} + +static bool ave_sess_owns(struct ave_session *sess, u64 addr) +{ + const struct ave_buf *fixed[] = { + &sess->hdr, &sess->mb, &sess->row_a, &sess->row_b, &sess->fixed, + &sess->hevc_extra[0], &sess->hevc_extra[1], + }; + + for (unsigned int i = 0; i < ARRAY_SIZE(fixed); i++) + if (ave_buf_has(fixed[i], addr)) + return true; + for (unsigned int i = 0; i < AVE_MAX_REFS; i++) + if (ave_buf_has(&sess->ref_a[i], addr) || ave_buf_has(&sess->ref_b[i], addr) || + ave_buf_has(&sess->ref_c[i], addr) || ave_buf_has(&sess->ref_d[i], addr)) + return true; + for (unsigned int i = 0; i < AVE_TILE_BUFS; i++) + if (ave_buf_has(&sess->tile[i], addr) || ave_buf_has(&sess->tile_aux[i], addr)) + return true; + for (unsigned int s = 0; s < AVE_MAX_SLOTS; s++) + if (ave_buf_has(&sess->slot[s].coded, addr) || + ave_buf_has(&sess->slot[s].meta, addr) || + ave_buf_has(&sess->slot[s].aux, addr)) + return true; + return false; +} + +void ave_sess_check_addrs(struct ave_ctx *ctx, const u8 *cmd, size_t len, unsigned int kind, + dma_addr_t input) +{ + struct ave_session *sess = ctx->sess; + struct ave_dev *ave = ctx->ave; + unsigned int bad = 0; + + if (sess->checked[kind]) + return; + sess->checked[kind] = true; + for (size_t off = AVE_CMD_HDR_BYTES; off + 8 <= len; off += 4) { + u64 v = get_unaligned_le64(cmd + off); + + /* The encoder's DART window: 0x100_0000_0000 + 256 MiB. */ + if ((v >> 28) != 0x1000) + continue; + if (ave_sess_owns(sess, v) || (input && v >= input && v < input + SZ_32M) || + (v >= ave->ipc.dma && v < ave->ipc.dma + ave->ipc.size)) + continue; + if (bad++ < 16) + dev_warn(ave->dev, "%s +%#zx holds %#llx, not a buffer of this session\n", + kind == AVE_TMPL_START ? "Start" : "Process", off, v); + } + if (bad) + dev_warn(ave->dev, "%u stale addresses in the %s image\n", bad, + kind == AVE_TMPL_START ? "Start" : "Process"); +} + +/* + * Frames are outstanding, nothing came back for a while, and the firmware + * has enough frames that it should have answered: without B-frames every + * frame is answered on its own; with B-frames, once four are inside (one + * reference plus three B) one of them must come out. + */ +bool ave_sess_stalled(struct ave_session *sess) +{ + unsigned int need = sess->p.bframes ? 4 : 1; + + return sess->busy >= need && + time_after(jiffies, sess->progress + msecs_to_jiffies(AVE_FRAME_WAIT_MS)); +} diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index 2807d8e3041f1..36ef6ab0fde77 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -992,6 +992,17 @@ static void ave_poll_work(struct work_struct *work) mutex_lock(&ave->hw_lock); if (ctx->sess && ave->active == ctx && !ctx->sess->dead) { fail = ave_ipc_pump(ave, 0, NULL, NULL) != 0; + if (!fail && ave_sess_stalled(ctx->sess)) { + /* + * The firmware holds frames it should have answered. + * Treat it as stuck, so the session fails now instead + * of leaving the client waiting. + */ + dev_err(ave->dev, "%u frames unanswered for %u ms; firmware is stuck\n", + ctx->sess->busy, AVE_FRAME_WAIT_MS); + ave->fw_state = AVE_FW_BROKEN; + fail = true; + } again = !fail && ctx->sess->busy; } else if (ctx->sess && ctx->sess->busy) { fail = true; diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h index 20d0c6831f5ef..e8d433edcb64f 100644 --- a/drivers/media/platform/apple/ave/ave.h +++ b/drivers/media/platform/apple/ave/ave.h @@ -173,7 +173,6 @@ struct ave_slot { struct ave_buf meta; struct ave_buf aux; /* H.264 per-frame buffer, HEVC slice headers */ struct vb2_v4l2_buffer *src; - size_t coded_dirty; u32 status; bool busy; /* Process sent, reply pending */ bool done; /* reply in, output not delivered yet */ @@ -204,6 +203,8 @@ struct ave_session { bool fw_open; /* Open sent, Close not yet */ bool dead; /* firmware gone; buffers only */ bool warned; + bool checked[2]; /* Start/Process images checked for stale addresses */ + unsigned long progress; /* jiffies of the last frame sent or answered */ }; struct ave_ctx { @@ -285,6 +286,9 @@ void ave_sess_reply(struct ave_ctx *ctx, const u8 *msg, u32 len); int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *dst, bool *keyframe); void ave_sess_prepare_slot(struct ave_session *sess, unsigned int slot); +void ave_sess_check_addrs(struct ave_ctx *ctx, const u8 *cmd, size_t len, unsigned int kind, + dma_addr_t input); +bool ave_sess_stalled(struct ave_session *sess); /* ave-v4l2.c */ int ave_v4l2_register(struct ave_dev *ave); From e4698294d7a69439696aaee2212205ec67197711 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 18:48:35 +1100 Subject: [PATCH 1382/1579] ASoC: apple: macaudio: accept the read-back unlock state as a no-op "Speaker Volume Unlock" reads back the lock state (0 or 1) but only accepts the magic ping value on write and returns -EINVAL for anything else. alsactl saves the read-back value and writes it back at boot. When alsa-state restores the card before speakerguardd has taken the lock on the control, the write reaches the put handler and fails, and alsactl logs an error on every such boot: alsactl: set_control:1525: Cannot write control 'numid=31,iface=MIXER,name='Speaker Volume Unlock'' : Invalid argument The J616s hit this race on 66 of 236 boots over three days, and the M1 Pro hits it too. When the daemon wins the race the core rejects the write with -EPERM because the control is locked, and our alsactl skips locked controls, so the log stays clean. Writing 0 or 1 back cannot change anything: the lease is renewed only by the magic value from the lock owner. Treat those two values as a no-op so that restoring a saved mixer state round-trips cleanly. Other values still fail with -EINVAL, and the lock owner, magic value and lease checks are unchanged, so nothing new can unlock the speaker volumes. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- sound/soc/apple/macaudio.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 1d1c8924eb2fb..4207cf5ca9999 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -1876,8 +1876,18 @@ static int macaudio_slk_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_v { struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + long val = uvalue->value.integer.value[0]; - if (uvalue->value.integer.value[0] != SPEAKER_MAGIC_VALUE) + /* + * Reads return the lock state (0 or 1), so a saved mixer state holds + * one of those. Writing it back changes nothing: only the lock owner's + * magic value renews the lease. Accept it as a no-op so that restoring + * a saved state before the daemon has taken the lock is not an error. + */ + if (val == 0 || val == 1) + return 0; + + if (val != SPEAKER_MAGIC_VALUE) return -EINVAL; mutex_lock(&ma->volume_lock_mutex); From 46af9cb5733dc06b33d1365c107f26a7fca2bd7d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 19:29:35 +1100 Subject: [PATCH 1383/1579] media: apple: ave: map working buffers with room behind them The second 1080p H.264 run (P frames only) still had DART write faults right after the first frame was encoded, from 0x7800 to at least 0xa000 bytes past the end of the first +0x390 buffer, and the firmware stopped answering after that frame. The size the driver puts in Start for that group is the one macOS reports, so the firmware writes beyond the size it is given and macOS must map more than that. Keep the reported sizes, but map the +0x390 group at one and a half times its size and every other working buffer with an eighth (at least 64 KiB) more. Buffers now carry their mapped size separately, and the debug buffer map shows both. The stale address check no longer reports pointers into the firmware image itself (the Start image points at the TEXT segment, which sits at the same address as on macOS). Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/media/platform/apple/ave/ave-fw.c | 15 +++-- drivers/media/platform/apple/ave/ave-ipc.c | 1 + drivers/media/platform/apple/ave/ave-sess.c | 63 +++++++++++++++------ drivers/media/platform/apple/ave/ave.h | 5 +- 4 files changed, 61 insertions(+), 23 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c index 62eb494c149b7..f895b1bd06fd5 100644 --- a/drivers/media/platform/apple/ave/ave-fw.c +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -177,21 +177,23 @@ static void ave_fw_restore_data(struct ave_dev *ave) void ave_free(struct ave_dev *ave, struct ave_buf *b) { if (b->cpu) - dma_free_attrs(ave->dev, PAGE_ALIGN(b->size), b->cpu, b->dma, b->attrs); + dma_free_attrs(ave->dev, b->alloc, b->cpu, b->dma, b->attrs); b->cpu = NULL; } /* * Firmware-visible memory, zeroed and 16 KiB aligned in the encoder's DART. - * @size is what the firmware is told; the allocation is page rounded. + * @size is what the firmware is told; @alloc (at least @size) is mapped. */ -int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, unsigned long attrs) +int ave_alloc_padded(struct ave_dev *ave, struct ave_buf *b, size_t size, size_t alloc, + unsigned long attrs) { if (b->cpu) return 0; b->size = size; + b->alloc = PAGE_ALIGN(max(size, alloc)); b->attrs = attrs; - b->cpu = dma_alloc_attrs(ave->dev, PAGE_ALIGN(size), &b->dma, GFP_KERNEL, attrs); + b->cpu = dma_alloc_attrs(ave->dev, b->alloc, &b->dma, GFP_KERNEL, attrs); if (!b->cpu) return -ENOMEM; if (!IS_ALIGNED(b->dma, SZ_16K)) { @@ -201,6 +203,11 @@ int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, unsigned long return 0; } +int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, unsigned long attrs) +{ + return ave_alloc_padded(ave, b, size, size, attrs); +} + /* * Start and Process are mostly constant for a firmware build. Those * constant words come from per-build image files named after the TEXT diff --git a/drivers/media/platform/apple/ave/ave-ipc.c b/drivers/media/platform/apple/ave/ave-ipc.c index d316edca01b3c..93178824a3604 100644 --- a/drivers/media/platform/apple/ave/ave-ipc.c +++ b/drivers/media/platform/apple/ave/ave-ipc.c @@ -146,6 +146,7 @@ int ave_ipc_negotiate(struct ave_dev *ave, const u32 hello[4]) return -EPROTO; if (!ave->heap.cpu) { ave->heap.size = PAGE_ALIGN(hello[3]); + ave->heap.alloc = ave->heap.size; ave->heap.cpu = dma_alloc_coherent(ave->dev, ave->heap.size, &ave->heap.dma, GFP_KERNEL); if (!ave->heap.cpu) diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index aa93d50fd1143..4b44f0512cb7a 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -175,6 +175,24 @@ static int ave_sess_params(struct ave_ctx *ctx, struct ave_params *p) return 0; } +/* + * The firmware writes past the sizes it is given. The first 1080p run + * wrote 0x7800..0x8a00 bytes (and maybe more; DART faults are rate + * limited) beyond the end of the first +0x390 buffer, so the sizes stay as + * macOS reports them but the mappings get room behind them: that group is + * mapped at 1.5 times its size, every other working buffer with an eighth + * (at least 64 KiB) more. + */ +static size_t ave_pad(size_t size) +{ + return size + max_t(size_t, SZ_64K, ALIGN(size / 8, SZ_16K)); +} + +static int ave_work_buf(struct ave_dev *ave, struct ave_buf *b, size_t size) +{ + return ave_alloc_padded(ave, b, size, ave_pad(size), DMA_ATTR_NO_KERNEL_MAPPING); +} + /* Working buffer sizes, all from the requested picture size. */ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) { @@ -193,34 +211,39 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) ref_d = ALIGN(hevc ? 64 * w32 * h32 : 128 * p->mbs, SZ_16K); tile = hevc ? 2304 * w32 * h64 : ALIGN(w, 16) * ALIGN(h, 64); - ret |= ave_alloc(ave, &sess->hdr, AVE_HDR_BYTES, nomap); + ret |= ave_work_buf(ave, &sess->hdr, AVE_HDR_BYTES); for (unsigned int i = 0; i < p->refs; i++) { - ret |= ave_alloc(ave, &sess->ref_a[i], ref_a, nomap); - ret |= ave_alloc(ave, &sess->ref_b[i], ref_b, nomap); - ret |= ave_alloc(ave, &sess->ref_c[i], ref_c, nomap); - ret |= ave_alloc(ave, &sess->ref_d[i], ref_d, nomap); + ret |= ave_alloc_padded(ave, &sess->ref_a[i], ref_a, + max(ref_a * 3 / 2, ave_pad(ref_a)), nomap); + ret |= ave_work_buf(ave, &sess->ref_b[i], ref_b); + ret |= ave_work_buf(ave, &sess->ref_c[i], ref_c); + ret |= ave_work_buf(ave, &sess->ref_d[i], ref_d); } - ret |= ave_alloc(ave, &sess->mb, ALIGN(82 * p->mbs, SZ_16K), nomap); - ret |= ave_alloc(ave, &sess->row_a, hevc ? SZ_16K : ALIGN(256 * w16, SZ_16K), nomap); - ret |= ave_alloc(ave, &sess->row_b, ALIGN(hevc ? 768 * w32 : 1024 * w16, SZ_16K), nomap); - ret |= ave_alloc(ave, &sess->fixed, SZ_16K, nomap); + ret |= ave_work_buf(ave, &sess->mb, ALIGN(82 * p->mbs, SZ_16K)); + ret |= ave_work_buf(ave, &sess->row_a, hevc ? SZ_16K : ALIGN(256 * w16, SZ_16K)); + ret |= ave_work_buf(ave, &sess->row_b, ALIGN(hevc ? 768 * w32 : 1024 * w16, SZ_16K)); + ret |= ave_work_buf(ave, &sess->fixed, SZ_16K); for (unsigned int i = 0; i < (hevc ? 4 : AVE_TILE_BUFS); i++) { - ret |= ave_alloc(ave, &sess->tile[i], tile, nomap); - ret |= ave_alloc(ave, &sess->tile_aux[i], ALIGN(h, 128), nomap); + ret |= ave_work_buf(ave, &sess->tile[i], tile); + ret |= ave_work_buf(ave, &sess->tile_aux[i], ALIGN(h, 128)); } if (hevc) for (unsigned int i = 0; i < 2; i++) - ret |= ave_alloc(ave, &sess->hevc_extra[i], ALIGN(w * h32, SZ_16K), nomap); + ret |= ave_work_buf(ave, &sess->hevc_extra[i], ALIGN(w * h32, SZ_16K)); /* Per-slot output; the size of the H.264 frame buffer is not known. */ frame_buf = ALIGN(DIV_ROUND_UP_ULL((u64)AVE_H264_FRAME_BUF_1080P * p->mbs, AVE_MBS_1080P), SZ_16K); frame_buf = max_t(size_t, frame_buf, AVE_H264_FRAME_BUF_1080P); for (unsigned int s = 0; s < p->slots; s++) { - ret |= ave_alloc(ave, &sess->slot[s].coded, p->coded_cap, 0); - ret |= ave_alloc(ave, &sess->slot[s].meta, AVE_META_BYTES, 0); - ret |= ave_alloc(ave, &sess->slot[s].aux, hevc ? AVE_SHDR_BYTES : frame_buf, - hevc ? 0 : nomap); + ret |= ave_alloc_padded(ave, &sess->slot[s].coded, p->coded_cap, + ave_pad(p->coded_cap), 0); + ret |= ave_alloc_padded(ave, &sess->slot[s].meta, AVE_META_BYTES, + ave_pad(AVE_META_BYTES), 0); + if (hevc) + ret |= ave_alloc(ave, &sess->slot[s].aux, AVE_SHDR_BYTES, 0); + else + ret |= ave_work_buf(ave, &sess->slot[s].aux, frame_buf); } return ret ? -ENOMEM : 0; } @@ -255,7 +278,8 @@ static void ave_sess_free_bufs(struct ave_dev *ave, struct ave_session *sess) static void ave_log_buf(struct ave_dev *ave, const char *name, int i, const struct ave_buf *b) { if (b->cpu) - dev_dbg(ave->dev, "buf %s%d %pad +%#zx\n", name, i, &b->dma, b->size); + dev_dbg(ave->dev, "buf %s%d %pad +%#zx (mapped %#zx)\n", name, i, &b->dma, + b->size, b->alloc); } static void ave_sess_log_map(struct ave_dev *ave, struct ave_session *sess) @@ -550,7 +574,7 @@ int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff */ static bool ave_buf_has(const struct ave_buf *b, u64 addr) { - return b->cpu && addr >= b->dma && addr < b->dma + PAGE_ALIGN(b->size); + return b->cpu && addr >= b->dma && addr < b->dma + b->alloc; } static bool ave_sess_owns(struct ave_session *sess, u64 addr) @@ -597,6 +621,9 @@ void ave_sess_check_addrs(struct ave_ctx *ctx, const u8 *cmd, size_t len, unsign if (ave_sess_owns(sess, v) || (input && v >= input && v < input + SZ_32M) || (v >= ave->ipc.dma && v < ave->ipc.dma + ave->ipc.size)) continue; + /* Pointers into the firmware image are the same as on macOS. */ + if (v >= ave->text.iova && v < ave->data.iova + ave->data.size) + continue; if (bad++ < 16) dev_warn(ave->dev, "%s +%#zx holds %#llx, not a buffer of this session\n", kind == AVE_TMPL_START ? "Start" : "Process", off, v); diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h index e8d433edcb64f..eac0ba7901f63 100644 --- a/drivers/media/platform/apple/ave/ave.h +++ b/drivers/media/platform/apple/ave/ave.h @@ -62,7 +62,8 @@ enum ave_codec { struct ave_buf { void *cpu; dma_addr_t dma; - size_t size; + size_t size; /* what the firmware is told */ + size_t alloc; /* what is mapped, page rounded */ unsigned long attrs; }; @@ -255,6 +256,8 @@ int ave_fw_shutdown(struct ave_dev *ave); void ave_fw_lost(struct ave_dev *ave); int ave_fw_templates(struct ave_dev *ave, enum ave_codec codec); int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, unsigned long attrs); +int ave_alloc_padded(struct ave_dev *ave, struct ave_buf *b, size_t size, size_t alloc, + unsigned long attrs); void ave_free(struct ave_dev *ave, struct ave_buf *b); /* ave-ipc.c */ From 77d95f77855018abd05e0cdc7344a49d062c74a0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 19:51:26 +1100 Subject: [PATCH 1384/1579] arm64: dts: apple: t6040-j616s: power dcpext1 from DISPEXT1_FE dcpext1, its two DARTs and its mailbox sit in the DISPEXT1 block, as dcpext0 and its DARTs and mailbox sit in DISPEXT0. dcpext0's nodes name ps_dispext0_fe as their power domain, dcpext1's name none. So DISPEXT1 is powered only when iBoot left it on, which it does when an external display is plugged into a Type-C port at boot. genpd then sees DISPEXT1_FE with no consumer and turns it off once it has a provider, under a running dcpext1. Without a display at boot nothing powers it before dcpext1 probes. Give them ps_dispext1_fe, as on DISPEXT0, so they hold DISPEXT1 (and through it the PMP request) while they are bound. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- .../dts/apple/t6040-j616s-extdisp-usbc.dtsi | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-usbc.dtsi b/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-usbc.dtsi index 6fbf8d6a27f89..fb151f79ecd5b 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-usbc.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-usbc.dtsi @@ -152,7 +152,26 @@ }; }; +/* + * As DISPEXT0: dcpext1, its DARTs and its mailbox are in the DISPEXT1 block, + * so they hold DISPEXT1_FE (and with it the PMP request) while they are bound. + * Without it DISPEXT1 is only powered when iBoot left it on for a display + * plugged in at boot, and genpd turns it off as unused under a running dcpext1. + */ +&dcpext1_dart { + power-domains = <&ps_dispext1_fe>; +}; + +&dispext1_dart { + power-domains = <&ps_dispext1_fe>; +}; + +&dcpext1_mbox { + power-domains = <&ps_dispext1_fe>; +}; + &dcpext1 { + power-domains = <&ps_dispext1_fe>; apple,connector-type = "USB-C"; /* as dcpext0: without it apple-dcp's probe fails with -EINVAL (m4.63) */ apple,iomfb-surfaces = <1 1 0 0>; From 3db9b1f2bbb7e8c7d727f45e28486200e512f151 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 19:51:30 +1100 Subject: [PATCH 1385/1579] pmdomain: apple: pmgr-pwrstate: power every parent of a domain left on genpd adds a power state that is on only below parents that are on. For a power state with one parent this always holds: the platform bus attaches the device to its domain and powers it on before probe. With more than one power-domains entry the bus attaches to none, so a domain that the boot loader left on cannot be added below a parent that is off. On the J616s DISPEXT1_SYS has two parents, FAB7_SOC and the PMP DISPEXT1 request. iBoot leaves DISPEXT1_SYS and DISPEXT1_FE on when a display is plugged into a Type-C port at boot, and the PMP request entry starts off (the PMP starts with no requests), so every such boot logged apple-pmgr-pwrstate 502800000.power-management:power-controller@490: failed to add to parent domain: -22 (report -> power-controller) platform 502800000.power-management:power-controller@498: deferred probe pending: ... supplier power-controller@490 not ready DISPEXT0_SYS, left on the same way below its single PMP entry, never failed, because the bus powered that entry on first. When a domain with several parents is on at probe, attach a virtual device to each parent and hold it powered until the domain is added below them all; the subdomain links keep them on from there. The PMP request for DISPEXT1 is then made and handed to the PMP when it runs (requests 0x63c00 instead of 0x23c00). Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/pmdomain/apple/pmgr-pwrstate.c | 60 ++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index bfcfb5ccbf1a4..5ebd7a51172f3 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -370,12 +371,63 @@ static int apple_pmgr_reset_xlate(struct reset_controller_dev *rcdev, return 0; } +/* + * genpd adds a power state that is on only below parents that are on. With a + * single parent the platform bus has already attached this device to it and + * powered it on before probe; with more than one it attaches to none, so a + * domain the boot loader left on (DISPEXT1_SYS on the J616s, below FAB7_SOC + * and its PMP request) could not be added. Power every parent on through a + * virtual device until the subdomain links hold them. + */ +static int apple_pmgr_ps_hold_parents(struct apple_pmgr_ps *ps, bool active, + struct dev_pm_domain_list **parents) +{ + static const struct dev_pm_domain_attach_data data = { + .pd_flags = PD_FLAG_NO_DEV_LINK, + }; + struct dev_pm_domain_list *list = NULL; + unsigned int i; + int ret; + + *parents = NULL; + if (!active || of_count_phandle_with_args(ps->dev->of_node, "power-domains", + "#power-domain-cells") < 2) + return 0; + + ret = dev_pm_domain_attach_list(ps->dev, &data, &list); + if (ret <= 0) + return ret; + for (i = 0; i < list->num_pds; i++) { + ret = pm_runtime_resume_and_get(list->pd_devs[i]); + if (ret < 0) { + while (i--) + pm_runtime_put(list->pd_devs[i]); + dev_pm_domain_detach_list(list); + return dev_err_probe(ps->dev, ret, "cannot power parent domain %u\n", i); + } + } + *parents = list; + return 0; +} + +static void apple_pmgr_ps_release_parents(struct dev_pm_domain_list *parents) +{ + unsigned int i; + + if (!parents) + return; + for (i = 0; i < parents->num_pds; i++) + pm_runtime_put(parents->pd_devs[i]); + dev_pm_domain_detach_list(parents); +} + static int apple_pmgr_ps_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *node = dev->of_node; struct apple_pmgr_ps *ps; struct regmap *regmap; + struct dev_pm_domain_list *parents = NULL; struct of_phandle_iterator it; int ret; const char *name; @@ -528,6 +580,10 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) return ret; } + ret = apple_pmgr_ps_hold_parents(ps, active, &parents); + if (ret < 0) + goto err_remove; + of_for_each_phandle(&it, ret, node, "power-domains", "#power-domain-cells", -1) { struct of_phandle_args parent, child; @@ -548,6 +604,9 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) } } + /* The subdomain links hold the parents now. */ + apple_pmgr_ps_release_parents(parents); + /* * Do not participate in regular PM; parent power domains are handled via the * genpd hierarchy. @@ -567,6 +626,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) return 0; err_remove: + apple_pmgr_ps_release_parents(parents); of_genpd_del_provider(node); pm_genpd_remove(&ps->genpd); return ret; From e84b0b5faedac230434f864a77372ad579625961 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 19:52:14 +1100 Subject: [PATCH 1386/1579] media: apple: isp: let a P010 queue allocate its two buffers VIDIOC_REQBUFS fails with ENOMEM for P010 at any size and count, while NV12 works. The P010 rendered pool has two slots, and queue_setup limits the queue to the pool. vb2 allocates at least min_queued_buffers + 1 buffers, three with ISP_MIN_FRAMES, so __vb2_queue_alloc gets two and vb2_core_reqbufs returns -ENOMEM. P010 has failed this way since it was added; the IOVA allocator and the DART window are not involved (the two buffers map fine and are torn down again). Set the queue minimum from the format whenever S_FMT changes it, which only happens with no buffers: one queued buffer starts a P010 stream, so vb2 needs two. NV12 keeps ISP_MIN_FRAMES. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/media/platform/apple/isp/isp-v4l2.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index ff0133e34dee5..431c85c6695c9 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -24,6 +24,8 @@ #define ISP_BUFFER_TIMEOUT msecs_to_jiffies(1500) #define ISP_STRIDE_ALIGNMENT 64 +static void isp_vb2_set_min_buffers(struct apple_isp *isp); + /* Capture rates offered to userspace, in frames per second */ static const unsigned int isp_frame_rates[] = { 30, 25, 24, 20, 15 }; @@ -835,6 +837,7 @@ static int isp_vidioc_set_format(struct file *file, void *fh, isp_get_sp_pix_format(isp, f, fmt); isp->vbq.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + isp_vb2_set_min_buffers(isp); return 0; } @@ -1025,6 +1028,22 @@ static void isp_free_meta_surfaces(struct apple_isp *isp) } } +/* + * vb2 allocates at least min_queued_buffers + 1 buffers, and the queue may + * hold no more than the rendered pool (isp_vb2_queue_setup()). The P010 pool + * has two slots, so a P010 stream starts with one queued buffer; otherwise + * REQBUFS can never get the three buffers vb2 asks for and fails with ENOMEM. + * Only called while the queue has no buffers. + */ +static void isp_vb2_set_min_buffers(struct apple_isp *isp) +{ + struct vb2_queue *vbq = &isp->vbq; + + vbq->min_queued_buffers = min_t(unsigned int, ISP_MIN_FRAMES, + isp_num_rendered(isp) - 1); + vbq->min_reqbufs_allocation = vbq->min_queued_buffers + 1; +} + static int isp_vb2_queue_init(struct apple_isp *isp) { struct vb2_queue *vbq = &isp->vbq; From 1808be7c2b0ef5c1af13451aa489bf9eacde53b1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 20:17:30 +1100 Subject: [PATCH 1387/1579] media: apple: ave: restart only from reset, flush held frames, free safe buffers The third hardware run encoded H.264 cleanly twice in a row, with a power cycle between the sessions. HEVC then hung its client for 20 minutes and broke every later session. ffmpeg's v4l2m2m encoder sends a frame and waits for its coded output before sending the next. With B-frames the firmware keeps frames until later ones arrive, so both sides waited, and the stuck-frame timeout only counted four or more held frames. Now, when frames are held and no input came for 50 ms, the driver sends a Complete, which makes the firmware code what it holds (macOS sends one after the first frame of every session). If those frames do not come back within 5 s, the session fails as before. The next session started before runtime suspend had powered the encoder down, and restarting the halted coprocessor while it kept power got no answer. A session now runtime suspends the encoder first when the firmware is halted, and a restart is refused while the coprocessor still runs, instead of trying it. A failed firmware boot marked the firmware broken for good and kept the buffers of the session that triggered it, so later sessions could not allocate. A failed boot now leaves the firmware halted, to be retried after a power cycle. Only buffers the firmware was given in a Start are kept when it stops answering. Field layouts are facts from our private research (macOS traces) and our own hardware runs; no copied code. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/media/platform/apple/ave/ave-cmd.c | 7 +++++- drivers/media/platform/apple/ave/ave-fw.c | 28 ++++++++++++++------- drivers/media/platform/apple/ave/ave-sess.c | 26 +++++++++++++++---- drivers/media/platform/apple/ave/ave-v4l2.c | 10 ++++++++ drivers/media/platform/apple/ave/ave.h | 4 +++ 5 files changed, 60 insertions(+), 15 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-cmd.c b/drivers/media/platform/apple/ave/ave-cmd.c index 8514509b13bdf..0aa872b2d720f 100644 --- a/drivers/media/platform/apple/ave/ave-cmd.c +++ b/drivers/media/platform/apple/ave/ave-cmd.c @@ -473,7 +473,10 @@ int ave_cmd_start(struct ave_ctx *ctx) ave_sess_check_addrs(ctx, cmd, tmpl->size, AVE_TMPL_START, 0); ret = ave_ipc_submit(ave, AVE_BUF_CMD, tmpl->size, AVE_CMD_SESSION_TIMEOUT_US, true); - return ret ?: ave_ipc_settle(ave, AVE_START_WAIT_MS); + if (ret) + return ret; + sess->handed = true; + return ave_ipc_settle(ave, AVE_START_WAIT_MS); } /* Process (id 8) */ @@ -619,6 +622,8 @@ int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff s->busy = true; s->status = 0; sess->progress = jiffies; + sess->last_sent = jiffies; + sess->nudged = false; sess->busy++; sess->frames++; return 0; diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c index f895b1bd06fd5..bd434d508f874 100644 --- a/drivers/media/platform/apple/ave/ave-fw.c +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -284,11 +284,11 @@ static bool ave_fw_idle(struct ave_dev *ave) } /* - * After a Halt the firmware no longer runs, but the coprocessor is still - * set going if its power stayed on. A restart uses the normal boot path - * (macOS restarts the firmware for every session the same way): clear the - * run bit, as the boot sequence does first anyway, and the mailbox words - * left from the previous run, so the reset-state checks hold again. The + * After a Halt the firmware is restarted from a power cycle, as macOS does + * for every session: the encoder must have been runtime suspended in + * between, which puts the coprocessor back in reset. Hardware runs showed + * that a restart while it kept power gets no answer, so that is refused + * (-EAGAIN) and the state stays halted until the power really went. The * code base register must still be set: the driver never writes it. */ static int ave_fw_prepare_restart(struct ave_dev *ave) @@ -304,8 +304,8 @@ static int ave_fw_prepare_restart(struct ave_dev *ave) return 0; if (readl(ave->coproc + AVE_COPROC_CTRL)) { - writel(0, ave->coproc + AVE_COPROC_CTRL); - usleep_range(1000, 2000); + dev_warn(ave->dev, "coprocessor kept power after its halt; retry once it suspends\n"); + return -EAGAIN; } status = readl(ave->mbox + AVE_MBOX_STATUS); if (status) @@ -416,6 +416,7 @@ static int ave_fw_boot(struct ave_dev *ave) /* Caller holds hw_lock. */ int ave_fw_ensure_running(struct ave_dev *ave) { + enum ave_fw_state prev; int ret; lockdep_assert_held(&ave->hw_lock); @@ -429,9 +430,18 @@ int ave_fw_ensure_running(struct ave_dev *ave) break; } + /* + * A failed boot has no session buffers in the firmware's hands. If the + * coprocessor was set going, treat it as halted: the next attempt + * then waits for a power cycle. Otherwise nothing changed. + */ + prev = ave->fw_state; ret = ave_fw_boot(ave); - if (ret) - ave->fw_state = AVE_FW_BROKEN; + if (ret) { + ave->fw_state = readl(ave->coproc + AVE_COPROC_CTRL) ? AVE_FW_HALTED : prev; + memset(ave->ring, 0, sizeof(ave->ring)); + ave->ipc_fw_base = 0; + } return ret; } diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index 4b44f0512cb7a..99406f1e39013 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -335,9 +335,10 @@ struct ave_session *ave_sess_create(struct ave_ctx *ctx) } /* - * Free the buffers. A firmware that stopped answering may still write to - * them, and a freed DART range is soon handed out again, so in that case - * they stay allocated (until reboot, like the firmware itself). + * Free the buffers. A firmware that stopped answering in the middle of a + * session may still write to them, and a freed DART range is soon handed + * out again, so those stay allocated (until reboot, like the firmware + * itself). Buffers the firmware never saw (no Start sent) always go. */ void ave_sess_destroy(struct ave_ctx *ctx) { @@ -345,7 +346,7 @@ void ave_sess_destroy(struct ave_ctx *ctx) if (!sess) return; - if (ctx->ave->fw_state == AVE_FW_BROKEN) + if (sess->handed && ctx->ave->fw_state == AVE_FW_BROKEN) dev_warn(ctx->ave->dev, "keeping the buffers of a failed session\n"); else ave_sess_free_bufs(ctx->ave, sess); @@ -643,6 +644,21 @@ bool ave_sess_stalled(struct ave_session *sess) { unsigned int need = sess->p.bframes ? 4 : 1; - return sess->busy >= need && + return sess->busy && (sess->busy >= need || sess->nudged) && time_after(jiffies, sess->progress + msecs_to_jiffies(AVE_FRAME_WAIT_MS)); } + +/* + * With B-frames the firmware keeps frames until later ones arrive. A client + * that waits for each coded frame before sending the next (ffmpeg's + * v4l2m2m encoder does) would wait forever. When frames are held and no + * input came for a while, a Complete makes the firmware code what it holds, + * as macOS does after the first frame. + */ +#define AVE_STARVE_MS 50 + +bool ave_sess_starved(struct ave_session *sess) +{ + return sess->busy && !sess->nudged && sess->handed && + time_after(jiffies, sess->last_sent + msecs_to_jiffies(AVE_STARVE_MS)); +} diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index 36ef6ab0fde77..6fd3224fbbb74 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -617,6 +617,9 @@ static int ave_session_begin(struct ave_ctx *ctx) if (ctx->sess) return 0; + /* A halted firmware restarts only from reset: let the power go first. */ + if (READ_ONCE(ave->fw_state) == AVE_FW_HALTED) + pm_runtime_suspend(ave->dev); ret = ave_fw_templates(ave, ave_ctx_codec(ctx)); if (ret) return ret; @@ -992,6 +995,13 @@ static void ave_poll_work(struct work_struct *work) mutex_lock(&ave->hw_lock); if (ctx->sess && ave->active == ctx && !ctx->sess->dead) { fail = ave_ipc_pump(ave, 0, NULL, NULL) != 0; + if (!fail && ave_sess_starved(ctx->sess) && + !v4l2_m2m_num_src_bufs_ready(ctx->fh.m2m_ctx)) { + dev_dbg(ave->dev, "no input, flushing %u held frames\n", ctx->sess->busy); + ctx->sess->nudged = true; + ctx->sess->progress = jiffies; + fail = ave_cmd_complete(ctx, 1) != 0; + } if (!fail && ave_sess_stalled(ctx->sess)) { /* * The firmware holds frames it should have answered. diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h index eac0ba7901f63..010a8481ae3b3 100644 --- a/drivers/media/platform/apple/ave/ave.h +++ b/drivers/media/platform/apple/ave/ave.h @@ -205,7 +205,10 @@ struct ave_session { bool dead; /* firmware gone; buffers only */ bool warned; bool checked[2]; /* Start/Process images checked for stale addresses */ + bool handed; /* Start sent: the firmware knows the buffers */ + bool nudged; /* Complete sent because input stopped */ unsigned long progress; /* jiffies of the last frame sent or answered */ + unsigned long last_sent; /* jiffies of the last frame sent */ }; struct ave_ctx { @@ -292,6 +295,7 @@ void ave_sess_prepare_slot(struct ave_session *sess, unsigned int slot); void ave_sess_check_addrs(struct ave_ctx *ctx, const u8 *cmd, size_t len, unsigned int kind, dma_addr_t input); bool ave_sess_stalled(struct ave_session *sess); +bool ave_sess_starved(struct ave_session *sess); /* ave-v4l2.c */ int ave_v4l2_register(struct ave_dev *ave); From 593f94d13c246dc2d2bca155434c754484ba4d1e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 20:21:00 +1100 Subject: [PATCH 1388/1579] media: apple: ave: report raw frame sizes and cap the buffer counts v4l2-compliance failed the format tests: a stateful encoder must answer VIDIOC_ENUM_FRAMESIZES for every format it enumerates, and the driver only did for the coded ones. Report the same range for NV12. CREATE_BUFS with the queue maximum failed because 32 buffers of each kind do not fit the encoder's 256 MiB DART window next to a session. Allow 16 OUTPUT and 8 CAPTURE buffers. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/media/platform/apple/ave/ave-v4l2.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index 6fd3224fbbb74..13da5a0e7d6ff 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -156,7 +156,8 @@ static int ave_enum_fmt_out(struct file *file, void *priv, struct v4l2_fmtdesc * static int ave_enum_framesizes(struct file *file, void *priv, struct v4l2_frmsizeenum *fs) { if (fs->index || - !ave_fmt_listed(ave_coded_fmts, ARRAY_SIZE(ave_coded_fmts), fs->pixel_format)) + (!ave_fmt_listed(ave_coded_fmts, ARRAY_SIZE(ave_coded_fmts), fs->pixel_format) && + !ave_fmt_listed(ave_raw_fmts, ARRAY_SIZE(ave_raw_fmts), fs->pixel_format))) return -EINVAL; fs->type = V4L2_FRMSIZE_TYPE_STEPWISE; fs->stepwise = (struct v4l2_frmsize_stepwise){ @@ -732,6 +733,8 @@ static int ave_queue_init(void *priv, struct vb2_queue *src, struct vb2_queue *d src->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; src->lock = &ctx->ave->vdev_lock; src->dev = ctx->ave->dev; + /* A full set of both queues must fit the encoder's 256 MiB DART window. */ + src->max_num_buffers = 16; ret = vb2_queue_init(src); if (ret) return ret; @@ -745,6 +748,7 @@ static int ave_queue_init(void *priv, struct vb2_queue *src, struct vb2_queue *d dst->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; dst->lock = &ctx->ave->vdev_lock; dst->dev = ctx->ave->dev; + dst->max_num_buffers = 8; return vb2_queue_init(dst); } From 01ce954b7f0d7635010c386a62b24c2dc3bb7b98 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 20:44:30 +1100 Subject: [PATCH 1389/1579] media: apple: ave: cap buffer counts in queue_setup The previous commit set max_num_buffers on both queues, which this kernel's vb2_core_queue_init() refuses below VB2_MAX_FRAME, so every open of the device failed. Leave the queue maximum alone and limit the count in queue_setup instead: 16 OUTPUT and 8 CAPTURE buffers, so a full set still fits the encoder's 256 MiB DART window. Fixes: 593f94d13c24 ("media: apple: ave: report raw frame sizes and cap the buffer counts") Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/media/platform/apple/ave/ave-v4l2.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index 13da5a0e7d6ff..c081734e86151 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -33,6 +33,8 @@ #define AVE_STRIDE_ALIGN 64 #define AVE_DEFAULT_W 1920 #define AVE_DEFAULT_H 1080 +#define AVE_MAX_OUTPUT_BUFS 16 +#define AVE_MAX_CAPTURE_BUFS 8 static const u32 ave_coded_fmts[] = { V4L2_PIX_FMT_H264, @@ -518,11 +520,17 @@ static int ave_queue_setup(struct vb2_queue *vq, unsigned int *nbufs, unsigned i struct ave_ctx *ctx = vb2_get_drv_priv(vq); const struct v4l2_pix_format_mplane *f = V4L2_TYPE_IS_OUTPUT(vq->type) ? &ctx->src_fmt : &ctx->dst_fmt; + unsigned int have = vb2_get_num_buffers(vq); + unsigned int max = V4L2_TYPE_IS_OUTPUT(vq->type) ? AVE_MAX_OUTPUT_BUFS : + AVE_MAX_CAPTURE_BUFS; /* Frames stay with the firmware until encoded; keep one spare. */ - if (V4L2_TYPE_IS_OUTPUT(vq->type) && - vb2_get_num_buffers(vq) + *nbufs < AVE_MAX_SLOTS + 1) - *nbufs = AVE_MAX_SLOTS + 1 - vb2_get_num_buffers(vq); + if (V4L2_TYPE_IS_OUTPUT(vq->type) && have + *nbufs < AVE_MAX_SLOTS + 1) + *nbufs = AVE_MAX_SLOTS + 1 - have; + /* A full set of both queues must fit the encoder's 256 MiB DART window. */ + if (have >= max) + return -ENOMEM; + *nbufs = min(*nbufs, max - have); if (*nplanes) return (*nplanes != 1 || sizes[0] < f->plane_fmt[0].sizeimage) ? -EINVAL : 0; *nplanes = 1; @@ -733,8 +741,6 @@ static int ave_queue_init(void *priv, struct vb2_queue *src, struct vb2_queue *d src->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; src->lock = &ctx->ave->vdev_lock; src->dev = ctx->ave->dev; - /* A full set of both queues must fit the encoder's 256 MiB DART window. */ - src->max_num_buffers = 16; ret = vb2_queue_init(src); if (ret) return ret; @@ -748,7 +754,6 @@ static int ave_queue_init(void *priv, struct vb2_queue *src, struct vb2_queue *d dst->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; dst->lock = &ctx->ave->vdev_lock; dst->dev = ctx->ave->dev; - dst->max_num_buffers = 8; return vb2_queue_init(dst); } From 124be686eb7b65a56d77757bb7174cb1d02f448e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 20:53:45 +1100 Subject: [PATCH 1390/1579] media: apple: ave: rate limit, memory budget, low-delay HEVC, restart retry Fixes from the fourth hardware round. Rate control: with only BITRATE set (ffmpeg's case) the one-second data-rate limit equalled the average rate, and the firmware then coded far below the requested rate. macOS always sets the limit to twice the average. BITRATE_PEAK now defaults to 0, which means exactly that; a client value is still used as given (and never below the average). Memory: H.264 at 2408x1506 ran out of room in the encoder's 256 MiB DART window. The DMA layer maps every buffer at a power-of-two alignment, so the size-proportional margins pushed buffers into the next size class. Margins are now 256 KiB on the +0x390 group and 64 KiB elsewhere, and the OUTPUT and CAPTURE queues get a fixed share of the window (64 and 32 MiB, counted at the mapped size, at least 6 OUTPUT and 2 CAPTURE buffers). HEVC with B_FRAMES set to 0, as ffmpeg sets it: the firmware still uses its B-frame structure, so each frame waited for the 50 ms no-input flush. Send a Complete right after every frame in that case instead. Restart: right after a session the coprocessor can still be powered when the next one starts, because the DARTs suspend a little later. Instead of failing with EAGAIN, cycle the power up to ten times, 100 ms apart. Formats: a huge OUTPUT bytesperline wrapped the size computation to 0 (v4l2-compliance). Clamp it to twice the maximum width. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- Documentation/admin-guide/media/apple-ave.rst | 8 ++- drivers/media/platform/apple/ave/ave-sess.c | 29 +++++--- drivers/media/platform/apple/ave/ave-v4l2.c | 67 ++++++++++++++++--- drivers/media/platform/apple/ave/ave.h | 1 + 4 files changed, 81 insertions(+), 24 deletions(-) diff --git a/Documentation/admin-guide/media/apple-ave.rst b/Documentation/admin-guide/media/apple-ave.rst index d8f9acabb2117..8ca9950dbea4b 100644 --- a/Documentation/admin-guide/media/apple-ave.rst +++ b/Documentation/admin-guide/media/apple-ave.rst @@ -52,7 +52,8 @@ runs at a time; a second one gets ``EBUSY``. Up to five frames are inside the firmware at once (three without B-frames). The firmware reorders B-frames itself, so CAPTURE buffers come back in bitstream order, each carrying the timestamp of the frame it codes. The driver -asks for at least six OUTPUT buffers. +asks for at least six OUTPUT buffers, and limits both queues so a full set fits +the encoder's 256 MiB DMA window (fewer buffers at larger sizes). Runtime PM uses a two second autosuspend delay. A firmware that stops answering keeps the encoder powered and is not used again until reboot. @@ -63,9 +64,10 @@ Limits Picture sizes from 320x240 to 2560x1600 (in steps of two), up to 240 frames per second. Rate control is VBR (with ``BITRATE_PEAK`` as a one-second data -rate cap) or CBR; constant quality is not offered. ``BITRATE`` may change +rate cap, twice the average when left at 0) or CBR; constant quality is not offered. ``BITRATE`` may change while streaming and applies from the next frame; ``FORCE_KEY_FRAME`` makes the next frame an IDR. H.264 uses either three B-frames or none (Baseline and -a GOP size of 1 never use them); HEVC always uses them for now. The level +a GOP size of 1 never use them). HEVC always uses them for now; with +``B_FRAMES`` set to 0 every frame is flushed through at once instead. The level control is a minimum: the driver raises it to what the size, frame rate and bit rate need. diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index 99406f1e39013..f20c21570f4b8 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -150,6 +150,8 @@ static int ave_sess_params(struct ave_ctx *ctx, struct ave_params *p) (!baseline && !p->intra_only && ctx->b_frames); p->refs = p->intra_only ? 1 : p->bframes ? 5 : 2; p->slots = p->bframes ? 5 : 3; + /* HEVC asked to go without B-frames: flush every frame instead. */ + p->low_delay = p->codec == AVE_CODEC_HEVC && !ctx->b_frames; p->gop = ctx->gop_size ?: 120; p->bitrate = ctx->bitrate; @@ -157,8 +159,14 @@ static int ave_sess_params(struct ave_ctx *ctx, struct ave_params *p) p->rc_mode = AVE_RC_CBR; p->limit_bytes = 0; } else { + /* + * The data-rate limit macOS sets is twice the average rate over + * one second; a peak equal to the average starved the rate + * control on hardware. A peak of 0 means that default. + */ p->rc_mode = AVE_RC_VBR; - p->limit_bytes = max(ctx->bitrate_peak, ctx->bitrate) / 8; + p->limit_bytes = (ctx->bitrate_peak ? max(ctx->bitrate_peak, ctx->bitrate) : + 2ULL * ctx->bitrate) / 8; } level = ave_pick_level(p, p->codec == AVE_CODEC_H264 ? ctx->h264_level : @@ -176,16 +184,18 @@ static int ave_sess_params(struct ave_ctx *ctx, struct ave_params *p) } /* - * The firmware writes past the sizes it is given. The first 1080p run - * wrote 0x7800..0x8a00 bytes (and maybe more; DART faults are rate - * limited) beyond the end of the first +0x390 buffer, so the sizes stay as - * macOS reports them but the mappings get room behind them: that group is - * mapped at 1.5 times its size, every other working buffer with an eighth - * (at least 64 KiB) more. + * The firmware writes past the sizes it is given. A 1080p run wrote up to + * about 0x8c00 bytes beyond the end of the first +0x390 buffer, so the + * sizes stay as macOS reports them but the mappings get room behind them: + * 256 KiB for that group, 64 KiB for every other working buffer. Larger + * margins cost more than they look: the DMA layer aligns each mapping to a + * power of two in the encoder's 256 MiB window. */ +#define AVE_REF_A_PAD SZ_256K + static size_t ave_pad(size_t size) { - return size + max_t(size_t, SZ_64K, ALIGN(size / 8, SZ_16K)); + return size + SZ_64K; } static int ave_work_buf(struct ave_dev *ave, struct ave_buf *b, size_t size) @@ -213,8 +223,7 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) ret |= ave_work_buf(ave, &sess->hdr, AVE_HDR_BYTES); for (unsigned int i = 0; i < p->refs; i++) { - ret |= ave_alloc_padded(ave, &sess->ref_a[i], ref_a, - max(ref_a * 3 / 2, ave_pad(ref_a)), nomap); + ret |= ave_alloc_padded(ave, &sess->ref_a[i], ref_a, ref_a + AVE_REF_A_PAD, nomap); ret |= ave_work_buf(ave, &sess->ref_b[i], ref_b); ret |= ave_work_buf(ave, &sess->ref_c[i], ref_c); ret |= ave_work_buf(ave, &sess->ref_d[i], ref_d); diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index c081734e86151..0868fc960c8ba 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -7,6 +7,8 @@ * * Copyright The Aurora Silicon Contributors */ +#include +#include #include #include @@ -33,8 +35,16 @@ #define AVE_STRIDE_ALIGN 64 #define AVE_DEFAULT_W 1920 #define AVE_DEFAULT_H 1080 -#define AVE_MAX_OUTPUT_BUFS 16 -#define AVE_MAX_CAPTURE_BUFS 8 +/* + * Buffer counts are limited so a full set fits the encoder's 256 MiB DART + * window next to a session: each queue may use this much, counting every + * buffer at the power-of-two size the DMA layer maps it with. + */ +#define AVE_OUTPUT_BUDGET SZ_64M +#define AVE_CAPTURE_BUDGET SZ_32M +#define AVE_MAX_QUEUE_BUFS 16 +#define AVE_RESTART_TRIES 10 +#define AVE_RESTART_WAIT_MS 100 static const u32 ave_coded_fmts[] = { V4L2_PIX_FMT_H264, @@ -76,7 +86,9 @@ static void ave_fill_raw(struct v4l2_pix_format_mplane *f) f->pixelformat = V4L2_PIX_FMT_NV12; ave_clamp_size(&f->width, &f->height); - stride = ALIGN(max_t(u32, f->width, f->plane_fmt[0].bytesperline), AVE_STRIDE_ALIGN); + /* Keep a client's stride, within reason, rounded to 64 bytes. */ + stride = ALIGN(clamp_t(u32, f->plane_fmt[0].bytesperline, f->width, 2 * AVE_MAX_W), + AVE_STRIDE_ALIGN); height = ALIGN(f->height, AVE_STEP); f->num_planes = 1; f->plane_fmt[0].bytesperline = stride; @@ -456,9 +468,11 @@ static int ave_init_ctrls(struct ave_ctx *ctx) V4L2_MPEG_VIDEO_BITRATE_MODE_VBR); v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_BITRATE, 100000, 100000000, 1, 4000000); - /* In VBR mode the peak caps the data rate over any one second. */ - v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_BITRATE_PEAK, 100000, 200000000, 1, - 8000000); + /* + * In VBR mode the peak caps the data rate over any one second; + * 0 means twice the average, as macOS sets it. + */ + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_BITRATE_PEAK, 0, 200000000, 1, 0); /* 0 means no limit; the firmware then uses 120 frames. */ v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_GOP_SIZE, 0, 65535, 1, 30); v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME, 0, 0, 0, 0); @@ -521,8 +535,12 @@ static int ave_queue_setup(struct vb2_queue *vq, unsigned int *nbufs, unsigned i const struct v4l2_pix_format_mplane *f = V4L2_TYPE_IS_OUTPUT(vq->type) ? &ctx->src_fmt : &ctx->dst_fmt; unsigned int have = vb2_get_num_buffers(vq); - unsigned int max = V4L2_TYPE_IS_OUTPUT(vq->type) ? AVE_MAX_OUTPUT_BUFS : - AVE_MAX_CAPTURE_BUFS; + unsigned long slot = roundup_pow_of_two(PAGE_ALIGN(f->plane_fmt[0].sizeimage)); + unsigned int max = (V4L2_TYPE_IS_OUTPUT(vq->type) ? AVE_OUTPUT_BUDGET : + AVE_CAPTURE_BUDGET) / slot; + + max = clamp(max, V4L2_TYPE_IS_OUTPUT(vq->type) ? AVE_MAX_SLOTS + 1U : 2U, + AVE_MAX_QUEUE_BUFS); /* Frames stay with the firmware until encoded; keep one spare. */ if (V4L2_TYPE_IS_OUTPUT(vq->type) && have + *nbufs < AVE_MAX_SLOTS + 1) @@ -638,20 +656,39 @@ static int ave_session_begin(struct ave_ctx *ctx) if (IS_ERR(sess)) return PTR_ERR(sess); + ctx->sess = sess; ret = pm_runtime_resume_and_get(ave->dev); if (ret) { - ctx->sess = sess; ave_sess_destroy(ctx); return ret; } mutex_lock(&ave->hw_lock); - ctx->sess = sess; if (ave->active) { ret = -EBUSY; goto out; } - ret = ave_fw_ensure_running(ave); + /* + * A halted coprocessor restarts only from reset. Its power can take + * a moment to go after a suspend (the DARTs follow later), so cycle + * it again a few times before giving up. + */ + for (unsigned int i = 0; i < AVE_RESTART_TRIES; i++) { + ret = ave_fw_ensure_running(ave); + if (ret != -EAGAIN) + break; + mutex_unlock(&ave->hw_lock); + pm_runtime_put_sync_suspend(ave->dev); + msleep(AVE_RESTART_WAIT_MS); + ret = pm_runtime_resume_and_get(ave->dev); + mutex_lock(&ave->hw_lock); + if (ret) { + mutex_unlock(&ave->hw_lock); + ave_sess_destroy(ctx); + return ret; + } + ret = -EAGAIN; + } if (ret) goto out; ave->active = ctx; @@ -884,6 +921,14 @@ static int ave_encode_one(struct ave_ctx *ctx) ret = ave_ipc_settle(ave, AVE_FRAME_WAIT_MS); if (ret) return ret; + } else if (sess->p.low_delay) { + /* Make the firmware code this frame now, without lookahead. */ + ret = ave_cmd_complete(ctx, 1); + sess->nudged = true; + if (!ret) + ret = ave_ipc_pump(ave, 0, NULL, NULL); + if (ret) + return ret; } else { ret = ave_ipc_pump(ave, 0, NULL, NULL); if (ret) diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h index 010a8481ae3b3..018795a5ebfd8 100644 --- a/drivers/media/platform/apple/ave/ave.h +++ b/drivers/media/platform/apple/ave/ave.h @@ -157,6 +157,7 @@ struct ave_params { u32 refs; /* max_num_ref_frames + 1 */ bool bframes; bool intra_only; + bool low_delay; /* Complete after every frame */ u32 gop; u32 rc_mode; u32 bitrate; From 44596dca5fdbb353ba1f10403d31a72141291129 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 21:06:11 +1100 Subject: [PATCH 1391/1579] media: apple: ave: map the reconstruction buffers at their real size The +0x390 group holds a 4:2:0 picture and its size field only covers the luma part. macOS leaves 1.5 times that size behind each one (0x318000 behind the 0x20d000 buffer at 1080p), and with only 256 KiB of margin the firmware wrote chroma far into unmapped space (from +0x252000 on, with hundreds of thousands of faults suppressed) and stopped answering mid stream. Map 1.5 times the size plus the usual 64 KiB. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/media/platform/apple/ave/ave-sess.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index f20c21570f4b8..c84aa3bf7cff6 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -184,14 +184,15 @@ static int ave_sess_params(struct ave_ctx *ctx, struct ave_params *p) } /* - * The firmware writes past the sizes it is given. A 1080p run wrote up to - * about 0x8c00 bytes beyond the end of the first +0x390 buffer, so the - * sizes stay as macOS reports them but the mappings get room behind them: - * 256 KiB for that group, 64 KiB for every other working buffer. Larger + * The firmware writes past the sizes it is given, so the sizes stay as + * macOS reports them but the mappings get room behind them. The +0x390 + * group holds a 4:2:0 picture whose size field covers luma only: macOS + * leaves 0x318000 bytes behind the 0x20d000 one at 1080p (1.5 times), and + * the firmware writes chroma there. Every other working buffer gets 64 KiB + * more (macOS leaves 0x28000 behind the 0x20000 +0x3a0 one). Larger * margins cost more than they look: the DMA layer aligns each mapping to a * power of two in the encoder's 256 MiB window. */ -#define AVE_REF_A_PAD SZ_256K static size_t ave_pad(size_t size) { @@ -223,7 +224,7 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) ret |= ave_work_buf(ave, &sess->hdr, AVE_HDR_BYTES); for (unsigned int i = 0; i < p->refs; i++) { - ret |= ave_alloc_padded(ave, &sess->ref_a[i], ref_a, ref_a + AVE_REF_A_PAD, nomap); + ret |= ave_alloc_padded(ave, &sess->ref_a[i], ref_a, ave_pad(ref_a * 3 / 2), nomap); ret |= ave_work_buf(ave, &sess->ref_b[i], ref_b); ret |= ave_work_buf(ave, &sess->ref_c[i], ref_c); ret |= ave_work_buf(ave, &sess->ref_d[i], ref_d); From b36da980a378f575149547b8c5a23ce4de248d3e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 21:11:50 +1100 Subject: [PATCH 1392/1579] media: apple: ave: report buffer overruns, refuse sheared NV12 input Encoded streams look right but drift from the source picture over a GOP, and the firmware is known to write past the sizes it is given. A write that runs into a neighbouring mapped buffer raises no DART fault, so it would go unnoticed. Keep a kernel mapping of the working buffers and, when a session ends normally, report for each one how far past its size the firmware wrote, with a warning when it reached the end of the mapping. This costs a scan of the zeroed margins once per session. At 2408x1506, ffmpeg's v4l2m2m encoder produced a sheared picture: it copies NV12 with its own frame line size and ignores the bytesperline from S_FMT, which differs when the width is not a multiple of 64. The encoder needs the 64-byte aligned stride, so the driver now keeps the client's bytesused on OUTPUT buffers, recognises a frame laid out with a smaller stride from it, and fails the stream with a message naming both strides instead of encoding garbage. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/media/platform/apple/ave/ave-sess.c | 67 +++++++++++++++++++-- drivers/media/platform/apple/ave/ave-v4l2.c | 33 +++++++++- 2 files changed, 94 insertions(+), 6 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index c84aa3bf7cff6..3ea435de52c6a 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -199,9 +199,13 @@ static size_t ave_pad(size_t size) return size + SZ_64K; } +/* + * Working buffers keep a kernel mapping so that, when a session ends, the + * driver can see how far past each reported size the firmware wrote. + */ static int ave_work_buf(struct ave_dev *ave, struct ave_buf *b, size_t size) { - return ave_alloc_padded(ave, b, size, ave_pad(size), DMA_ATTR_NO_KERNEL_MAPPING); + return ave_alloc_padded(ave, b, size, ave_pad(size), 0); } /* Working buffer sizes, all from the requested picture size. */ @@ -212,7 +216,6 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) u32 w = p->width, h = p->height; u32 w16 = DIV_ROUND_UP(w, 16), w32 = DIV_ROUND_UP(w, 32); u32 h32 = DIV_ROUND_UP(h, 32), w64 = DIV_ROUND_UP(w, 64), h64 = DIV_ROUND_UP(h, 64); - const unsigned long nomap = DMA_ATTR_NO_KERNEL_MAPPING; size_t ref_a, ref_b, ref_c, ref_d, tile, frame_buf; int ret = 0; @@ -224,7 +227,7 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) ret |= ave_work_buf(ave, &sess->hdr, AVE_HDR_BYTES); for (unsigned int i = 0; i < p->refs; i++) { - ret |= ave_alloc_padded(ave, &sess->ref_a[i], ref_a, ave_pad(ref_a * 3 / 2), nomap); + ret |= ave_alloc_padded(ave, &sess->ref_a[i], ref_a, ave_pad(ref_a * 3 / 2), 0); ret |= ave_work_buf(ave, &sess->ref_b[i], ref_b); ret |= ave_work_buf(ave, &sess->ref_c[i], ref_c); ret |= ave_work_buf(ave, &sess->ref_d[i], ref_d); @@ -318,6 +321,57 @@ static void ave_sess_log_map(struct ave_dev *ave, struct ave_session *sess) } } +/* + * How far did the firmware write past the size it was given? The margin + * behind each buffer was zero when allocated. A write reaching the end of + * the mapping may have gone on into the next buffer. + */ +static void ave_check_tail(struct ave_dev *ave, const char *name, int i, + const struct ave_buf *b) +{ + const u8 *p = b->cpu; + size_t end; + + if (!p || b->attrs & DMA_ATTR_NO_KERNEL_MAPPING || b->alloc <= b->size) + return; + for (end = b->alloc; end > b->size && !p[end - 1]; end--) + ; + if (end == b->size) + return; + if (end == b->alloc) + dev_warn(ave->dev, "buf %s%d: firmware wrote to the end of its mapping (+%#zx, size %#zx)\n", + name, i, b->alloc, b->size); + else + dev_info(ave->dev, "buf %s%d: firmware wrote %#zx bytes past its size %#zx\n", + name, i, end - b->size, b->size); +} + +static void ave_sess_check_tails(struct ave_dev *ave, struct ave_session *sess) +{ + ave_check_tail(ave, "hdr", 0, &sess->hdr); + ave_check_tail(ave, "mb", 0, &sess->mb); + ave_check_tail(ave, "row_a", 0, &sess->row_a); + ave_check_tail(ave, "row_b", 0, &sess->row_b); + ave_check_tail(ave, "fixed", 0, &sess->fixed); + for (int i = 0; i < 2; i++) + ave_check_tail(ave, "hevc_extra", i, &sess->hevc_extra[i]); + for (int i = 0; i < AVE_MAX_REFS; i++) { + ave_check_tail(ave, "ref_a", i, &sess->ref_a[i]); + ave_check_tail(ave, "ref_b", i, &sess->ref_b[i]); + ave_check_tail(ave, "ref_c", i, &sess->ref_c[i]); + ave_check_tail(ave, "ref_d", i, &sess->ref_d[i]); + } + for (int i = 0; i < AVE_TILE_BUFS; i++) { + ave_check_tail(ave, "tile", i, &sess->tile[i]); + ave_check_tail(ave, "tile_aux", i, &sess->tile_aux[i]); + } + for (int i = 0; i < AVE_MAX_SLOTS; i++) { + ave_check_tail(ave, "coded", i, &sess->slot[i].coded); + ave_check_tail(ave, "meta", i, &sess->slot[i].meta); + ave_check_tail(ave, "aux", i, &sess->slot[i].aux); + } +} + /* Work out the parameters and allocate the buffers; no firmware access. */ struct ave_session *ave_sess_create(struct ave_ctx *ctx) { @@ -356,10 +410,13 @@ void ave_sess_destroy(struct ave_ctx *ctx) if (!sess) return; - if (sess->handed && ctx->ave->fw_state == AVE_FW_BROKEN) + if (sess->handed && ctx->ave->fw_state == AVE_FW_BROKEN) { dev_warn(ctx->ave->dev, "keeping the buffers of a failed session\n"); - else + } else { + if (sess->handed) + ave_sess_check_tails(ctx->ave, sess); ave_sess_free_bufs(ctx->ave, sess); + } kfree(sess); ctx->sess = NULL; } diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index 0868fc960c8ba..07182d92592f4 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -564,7 +564,8 @@ static int ave_buf_prepare(struct vb2_buffer *vb) if (vb2_plane_size(vb, 0) < f->plane_fmt[0].sizeimage) return -EINVAL; - if (V4L2_TYPE_IS_OUTPUT(vb->type)) + /* Keep what the client filled in, so its layout can be checked. */ + if (V4L2_TYPE_IS_OUTPUT(vb->type) && !vb2_get_plane_payload(vb, 0)) vb2_set_plane_payload(vb, 0, f->plane_fmt[0].sizeimage); return 0; } @@ -877,6 +878,30 @@ static bool ave_frame_answered(struct ave_dev *ave, void *arg) return !s->busy; } +/* + * The encoder reads NV12 with the 64-byte aligned stride S_FMT reported. + * ffmpeg's v4l2m2m encoder copies frames with its own line size instead, + * which differs when the width is not a multiple of 64; the result is a + * sheared picture. Spot that from the payload size and refuse it. + */ +static int ave_check_src_layout(struct ave_ctx *ctx, struct vb2_v4l2_buffer *src) +{ + const struct ave_params *p = &ctx->sess->p; + u32 used = vb2_get_plane_payload(&src->vb2_buf, 0); + u32 rows = p->height + DIV_ROUND_UP(p->height, 2); + u32 stride; + + if (!used || used % rows) + return 0; + stride = used / rows; + if (stride >= p->stride || stride < p->width) + return 0; + dev_err(ctx->ave->dev, + "OUTPUT frame laid out with a %u byte stride, not the %u reported by S_FMT\n", + stride, p->stride); + return -EINVAL; +} + /* Send the next OUTPUT frame. Caller holds hw_lock. */ static int ave_encode_one(struct ave_ctx *ctx) { @@ -905,6 +930,12 @@ static int ave_encode_one(struct ave_ctx *ctx) last = src == ctx->drain_src; src->sequence = ctx->src_sequence++; + ret = ave_check_src_layout(ctx, src); + if (ret) { + v4l2_m2m_buf_done(src, VB2_BUF_STATE_ERROR); + return ret; + } + ret = ave_cmd_process(ctx, slot, src, READ_ONCE(ctx->bitrate), xchg(&ctx->force_key_frame, false)); if (ret) { From 502230fb6915bb4d4036559cdaca4cd31dfd32f5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 21:26:00 +1100 Subject: [PATCH 1393/1579] media: apple: ave: size reconstruction buffers from measured writes The overrun report from the previous commit gave the same numbers in every session. At 1080p the firmware wrote 0x313604 bytes into the 0x20d000 +0x390 buffer and 0x26a6a into the 0x20000 +0x3a0 one; at 2408x1506 0x557e06 into 0x390000 and 0x4c77d into 0x40000. No other buffer was written past its size. So the +0x390 group is a 4:2:0 picture whose size field is the luma part: map 1.5 times it. Map the +0x3a0 group at 1.25 times its size (it is written up to 1.2 times, and macOS spaces it that way). Drop the margins on every other buffer, which saves DART window space. The session debug line now also shows the rate-control mode, bit rate, data-rate limit, GOP length and frame rate the firmware gets. Measurements are from our own hardware runs; no copied code. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/media/platform/apple/ave/ave-sess.c | 37 +++++++++------------ 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index 3ea435de52c6a..c4eea08aa437e 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -184,28 +184,22 @@ static int ave_sess_params(struct ave_ctx *ctx, struct ave_params *p) } /* - * The firmware writes past the sizes it is given, so the sizes stay as - * macOS reports them but the mappings get room behind them. The +0x390 - * group holds a 4:2:0 picture whose size field covers luma only: macOS - * leaves 0x318000 bytes behind the 0x20d000 one at 1080p (1.5 times), and - * the firmware writes chroma there. Every other working buffer gets 64 KiB - * more (macOS leaves 0x28000 behind the 0x20000 +0x3a0 one). Larger - * margins cost more than they look: the DMA layer aligns each mapping to a - * power of two in the encoder's 256 MiB window. + * Two groups hold more than their size fields say, measured on hardware + * at 1080p and 2408x1506: the +0x390 group is a 4:2:0 picture whose size + * covers luma only (the firmware writes up to 1.5 times it), and the + * +0x3a0 group is written up to 1.2 times its size (mapped at 1.25 times, + * as macOS spaces them). Every other buffer stays within its size; the + * mapping is only rounded up to whole pages. */ -static size_t ave_pad(size_t size) -{ - return size + SZ_64K; -} - /* * Working buffers keep a kernel mapping so that, when a session ends, the - * driver can see how far past each reported size the firmware wrote. + * driver can see how far past each reported size the firmware wrote (into + * the page rounding, or the room the two groups above get). */ static int ave_work_buf(struct ave_dev *ave, struct ave_buf *b, size_t size) { - return ave_alloc_padded(ave, b, size, ave_pad(size), 0); + return ave_alloc(ave, b, size, 0); } /* Working buffer sizes, all from the requested picture size. */ @@ -227,8 +221,8 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) ret |= ave_work_buf(ave, &sess->hdr, AVE_HDR_BYTES); for (unsigned int i = 0; i < p->refs; i++) { - ret |= ave_alloc_padded(ave, &sess->ref_a[i], ref_a, ave_pad(ref_a * 3 / 2), 0); - ret |= ave_work_buf(ave, &sess->ref_b[i], ref_b); + ret |= ave_alloc_padded(ave, &sess->ref_a[i], ref_a, ref_a * 3 / 2, 0); + ret |= ave_alloc_padded(ave, &sess->ref_b[i], ref_b, ref_b * 5 / 4, 0); ret |= ave_work_buf(ave, &sess->ref_c[i], ref_c); ret |= ave_work_buf(ave, &sess->ref_d[i], ref_d); } @@ -249,10 +243,8 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) SZ_16K); frame_buf = max_t(size_t, frame_buf, AVE_H264_FRAME_BUF_1080P); for (unsigned int s = 0; s < p->slots; s++) { - ret |= ave_alloc_padded(ave, &sess->slot[s].coded, p->coded_cap, - ave_pad(p->coded_cap), 0); - ret |= ave_alloc_padded(ave, &sess->slot[s].meta, AVE_META_BYTES, - ave_pad(AVE_META_BYTES), 0); + ret |= ave_alloc(ave, &sess->slot[s].coded, p->coded_cap, 0); + ret |= ave_alloc(ave, &sess->slot[s].meta, AVE_META_BYTES, 0); if (hevc) ret |= ave_alloc(ave, &sess->slot[s].aux, AVE_SHDR_BYTES, 0); else @@ -395,6 +387,9 @@ struct ave_session *ave_sess_create(struct ave_ctx *ctx) dev_dbg(ctx->ave->dev, "session %ux%u codec %u profile %u level %u, %u slots, %u refs\n", sess->p.width, sess->p.height, sess->p.codec, sess->p.profile, sess->p.level, sess->p.slots, sess->p.refs); + dev_dbg(ctx->ave->dev, "rate control mode %u, %u bit/s, limit %u bytes/s, gop %u, %u/%u fps\n", + sess->p.rc_mode, sess->p.bitrate, sess->p.limit_bytes, sess->p.gop, + sess->p.fps_num, sess->p.fps_den); return sess; } From 7962e096edb6864ce2f27ee1744a2bb91715d654 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 21:30:14 +1100 Subject: [PATCH 1394/1579] media: apple: ave: keep delivering after a drain, say why a stream fails A GStreamer run stopped at frame 240 with only "encoding failed: -5". A job refused to touch a session whose firmware it no longer owned, but a drain closes the session and halts the firmware first and only then hands out the frames still waiting for CAPTURE buffers. When those buffers came in later, the next job saw the closed session and failed the stream without a word. Only a session that is still open must own the firmware. Every other path that ends a stream with -EIO now logs its reason too: a command or wait on a firmware that is no longer running, a firmware that stopped answering in an earlier session, a failed Halt, a session ended by system sleep, frames outstanding without a firmware, and a frame the firmware answered with an error status. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/media/platform/apple/ave/ave-fw.c | 2 ++ drivers/media/platform/apple/ave/ave-ipc.c | 10 ++++++++-- drivers/media/platform/apple/ave/ave-sess.c | 5 ++++- drivers/media/platform/apple/ave/ave-v4l2.c | 18 ++++++++++++++++-- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c index bd434d508f874..e1f9a11118ed0 100644 --- a/drivers/media/platform/apple/ave/ave-fw.c +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -424,6 +424,7 @@ int ave_fw_ensure_running(struct ave_dev *ave) case AVE_FW_RUNNING: return 0; case AVE_FW_BROKEN: + dev_err(ave->dev, "firmware stopped answering earlier; reboot to use the encoder\n"); return -EIO; case AVE_FW_OFF: case AVE_FW_HALTED: @@ -457,6 +458,7 @@ void ave_fw_halt(struct ave_dev *ave) if (ave->fw_state != AVE_FW_RUNNING) return; if (ave_cmd_halt(ave)) { + dev_err(ave->dev, "cannot send Halt; firmware treated as stuck\n"); ave->fw_state = AVE_FW_BROKEN; return; } diff --git a/drivers/media/platform/apple/ave/ave-ipc.c b/drivers/media/platform/apple/ave/ave-ipc.c index 93178824a3604..af067fb024e1e 100644 --- a/drivers/media/platform/apple/ave/ave-ipc.c +++ b/drivers/media/platform/apple/ave/ave-ipc.c @@ -357,8 +357,11 @@ int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, u64 timeou int ret; lockdep_assert_held(&ave->hw_lock); - if (ave->fw_state != AVE_FW_RUNNING) + if (ave->fw_state != AVE_FW_RUNNING) { + dev_err_ratelimited(ave->dev, "command %u not sent: firmware state %d\n", op, + ave->fw_state); return -EIO; + } if (len < AVE_CMD_HDR_BYTES || len > max || !IS_ALIGNED(len, 4) || (ave->unacked & BIT(buf)) || (reply && op >= ARRAY_SIZE(ave->want))) return -EINVAL; @@ -473,8 +476,11 @@ int ave_ipc_pump(struct ave_dev *ave, unsigned int wait_ms, for (;;) { if (cond && cond(ave, arg)) return 0; - if (ave->fw_state != AVE_FW_RUNNING) + if (ave->fw_state != AVE_FW_RUNNING) { + dev_err_ratelimited(ave->dev, "waiting on a firmware in state %d\n", + ave->fw_state); return -EIO; + } ret = ave_ipc_poll_once(ave); if (ret < 0) { ave_ipc_mark_broken(ave, ret, "ring error"); diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index c4eea08aa437e..9f5b46704ac4c 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -592,8 +592,11 @@ int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff *keyframe = false; if (!out) return -EFAULT; - if (s->status) + if (s->status) { + dev_warn_ratelimited(ctx->ave->dev, "frame %u: firmware status %#x\n", + ctx->dst_sequence, s->status); return -EIO; + } data = get_unaligned_le32(meta + AVE_META_PAYLOAD); data2 = get_unaligned_le32(meta + AVE_META_PAYLOAD2); diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index 07182d92592f4..281e9cbad38b0 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -1036,8 +1036,17 @@ static void ave_run_work(struct work_struct *work) ret = ave_session_begin(ctx); mutex_lock(&ave->hw_lock); - if (!ret && ctx->sess && (ctx->sess->dead || ave->active != ctx)) + /* + * A closed session (after a drain) may still hold frames for CAPTURE + * buffers; only an open one must still own the firmware. + */ + if (!ret && ctx->sess && ctx->sess->dead) { + dev_err(ave->dev, "session ended by system sleep\n"); + ret = -EIO; + } else if (!ret && ctx->sess && ctx->sess->fw_open && ave->active != ctx) { + dev_err(ave->dev, "session lost the firmware\n"); ret = -EIO; + } if (!ret) { eos = ave_deliver(ctx); if (!eos && !READ_ONCE(ctx->flush_pending) && ctx->sess && @@ -1049,8 +1058,11 @@ static void ave_run_work(struct work_struct *work) if (!eos && READ_ONCE(ctx->flush_pending)) eos = ave_finish_drain(ctx); /* Frames outstanding need a running firmware. */ - if (!eos && ctx->sess && ctx->sess->busy && ave->fw_state != AVE_FW_RUNNING) + if (!eos && ctx->sess && ctx->sess->busy && ave->fw_state != AVE_FW_RUNNING) { + dev_err(ave->dev, "%u frames outstanding, firmware state %d\n", + ctx->sess->busy, ave->fw_state); ret = -EIO; + } poll = !ret && !eos && ctx->sess && ctx->sess->busy; } mutex_unlock(&ave->hw_lock); @@ -1100,6 +1112,8 @@ static void ave_poll_work(struct work_struct *work) } again = !fail && ctx->sess->busy; } else if (ctx->sess && ctx->sess->busy) { + dev_err(ave->dev, "%u frames outstanding in a session that lost the firmware\n", + ctx->sess->busy); fail = true; } mutex_unlock(&ave->hw_lock); From b2fd1d5d90d6b12c537d23c29d48cae372ea4bb5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 21:54:56 +1100 Subject: [PATCH 1395/1579] media: apple: ave: two streams, survive system sleep, copy odd strides Make the encoder usable as a general V4L2 device: - Two streams at once. One firmware boot serves both, as macOS was seen doing: each session has its own command token counter and Process buffers, replies are routed by session token, header word +0x2c follows the set of open sessions (0x140 on Open and Start and 0 elsewhere while another session is open), and the firmware is halted only when the last session closes. - System sleep. Open sessions are drained, stopped and closed and the firmware halted before the power goes. On resume each session is opened and started again with the same parameters on its next frame, which then starts with an IDR and fresh parameter sets. Clients see no error. - Input with a different line size. ffmpeg's v4l2m2m encoder fills NV12 buffers with its own line size, which differs from the 64-byte aligned stride the encoder needs when the width is not a multiple of 64. Such frames, recognised from the payload size, are now copied to a per-slot buffer in the encoder's layout instead of being refused. - Session images are named after the machine instead of the firmware checksum (apple/ave-j700-{h264,hevc}-{start,process}.bin), since they are extracted from the user's own macOS installation. Errors name the missing or mis-sized file. - Polling backs off to sleeps of up to a millisecond once a wait goes on, so waiting for frames costs next to no CPU time. Which interrupt line signals the rings is still not known. - At the end of an H.264 session the debug log reports how much of the per-frame buffer (Process +0x58c8) the firmware used, to size it later. Also clean up the remaining checkpatch --strict notes. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/media/platform/apple/ave/ave-cmd.c | 56 +++-- drivers/media/platform/apple/ave/ave-drv.c | 21 +- drivers/media/platform/apple/ave/ave-fw.c | 45 +++- drivers/media/platform/apple/ave/ave-ipc.c | 23 +- drivers/media/platform/apple/ave/ave-regs.h | 1 + drivers/media/platform/apple/ave/ave-sess.c | 96 ++++++- drivers/media/platform/apple/ave/ave-v4l2.c | 263 +++++++++++++------- drivers/media/platform/apple/ave/ave.h | 27 +- 8 files changed, 379 insertions(+), 153 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-cmd.c b/drivers/media/platform/apple/ave/ave-cmd.c index 0aa872b2d720f..922db11c98b80 100644 --- a/drivers/media/platform/apple/ave/ave-cmd.c +++ b/drivers/media/platform/apple/ave/ave-cmd.c @@ -16,7 +16,6 @@ #include #include -#include #include #include "ave.h" @@ -79,19 +78,41 @@ static void ave_put_buf(u8 *cmd, u32 off, const struct ave_buf *b, u32 size) ave_put_surface(cmd, off, b->dma, size); } -static u32 ave_session_arg2(enum ave_codec codec) +/* Is another session open in the firmware besides @ctx's? */ +static bool ave_cmd_others_open(struct ave_ctx *ctx) { - return codec == AVE_CODEC_HEVC ? AVE_SESSION_ARG2_HEVC : AVE_SESSION_ARG2_H264; + struct ave_dev *ave = ctx->ave; + + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i] && ave->sessions[i] != ctx && ave->sessions[i]->sess->fw_open) + return true; + return false; +} + +/* + * Header word +0x2c depends on the set of open sessions: 0x1e0 on Open and + * Start (0x140 when another session is open), the codec value on every + * other command of a lone session, and 0 while another one is open. + */ +static u32 ave_session_arg2(struct ave_ctx *ctx) +{ + if (ave_cmd_others_open(ctx)) + return 0; + return ctx->sess->p.codec == AVE_CODEC_HEVC ? AVE_SESSION_ARG2_HEVC : + AVE_SESSION_ARG2_H264; +} + +static u32 ave_session_setup_arg2(struct ave_ctx *ctx) +{ + return ave_cmd_others_open(ctx) ? AVE_SESSION_ARG2_SETUP_SHARED : AVE_SESSION_ARG2_SETUP; } static void ave_cmd_header(struct ave_ctx *ctx, u8 *cmd, enum ave_opcode op, u32 echo, u32 arg0, u32 arg2) { - struct ave_dev *ave = ctx->ave; - - ave->next_token += AVE_TOKEN_STEP; + ctx->sess->token += AVE_TOKEN_STEP; ave_put32(cmd, AVE_CMD_OPCODE, op); - ave_put64(cmd, AVE_CMD_TOKEN, ave->next_token); + ave_put64(cmd, AVE_CMD_TOKEN, ctx->sess->token); ave_put64(cmd, AVE_CMD_CLIENT, ctx->sess->client); ave_put32(cmd, AVE_CMD_UNIT, AVE_SESSION_UNIT); ave_put32(cmd, AVE_CMD_CODEC, ctx->sess->p.codec); @@ -157,7 +178,8 @@ int ave_cmd_open(struct ave_ctx *ctx) ave->next_client += AVE_CLIENT_STEP; ctx->sess->client = ave->next_client; - ret = ave_cmd_scalar(ctx, AVE_OP_OPEN, AVE_SESSION_ARG0, AVE_SESSION_ARG2_SETUP); + ctx->sess->token = AVE_TOKEN_BASE; + ret = ave_cmd_scalar(ctx, AVE_OP_OPEN, AVE_SESSION_ARG0, ave_session_setup_arg2(ctx)); if (!ret) ctx->sess->fw_open = true; return ret ?: ave_ipc_settle(ave, AVE_CMD_WAIT_MS); @@ -169,14 +191,14 @@ int ave_cmd_complete(struct ave_ctx *ctx, unsigned int count) int ret = 0; for (unsigned int i = 0; i < count && !ret; i++) - ret = ave_cmd_scalar(ctx, AVE_OP_COMPLETE, 0, ave_session_arg2(ctx->sess->p.codec)); + ret = ave_cmd_scalar(ctx, AVE_OP_COMPLETE, 0, ave_session_arg2(ctx)); return ret; } /* Priority and Stop go out together, Close after Stop is answered. */ int ave_cmd_close(struct ave_ctx *ctx) { - u32 arg2 = ave_session_arg2(ctx->sess->p.codec); + u32 arg2 = ave_session_arg2(ctx); int ret, err; ret = ave_cmd_scalar(ctx, AVE_OP_PRIORITY, AVE_SESSION_ARG0, arg2); @@ -467,7 +489,7 @@ int ave_cmd_start(struct ave_ctx *ctx) return -EBUSY; memcpy(cmd, tmpl->data, tmpl->size); ave_cmd_header(ctx, cmd, AVE_OP_START, AVE_OP_START, AVE_SESSION_ARG0, - AVE_SESSION_ARG2_SETUP); + ave_session_setup_arg2(ctx)); ave_start_surfaces(sess, cmd); ave_start_config(&sess->p, cmd); ave_sess_check_addrs(ctx, cmd, tmpl->size, AVE_TMPL_START, 0); @@ -542,7 +564,7 @@ static const struct ave_proc_layout ave_proc_hevc = { #define AVE_PROC_FORCED_TYPE 2 int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *src, - u32 bitrate, bool force_key) + dma_addr_t input, u32 bitrate, bool force_key) { struct ave_dev *ave = ctx->ave; struct ave_session *sess = ctx->sess; @@ -551,16 +573,12 @@ int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff &ave_proc_h264; const struct firmware *tmpl = ave->tmpl[p->codec - 1][AVE_TMPL_PROCESS]; struct ave_slot *s = &sess->slot[slot]; - unsigned int buf = AVE_BUF_PROC(slot); - dma_addr_t y; + unsigned int buf = AVE_BUF_PROC(sess->index * AVE_MAX_SLOTS + slot); + dma_addr_t y = input; u64 pts; u8 *cmd; int ret; - y = vb2_dma_contig_plane_dma_addr(&src->vb2_buf, 0) + src->vb2_buf.planes[0].data_offset; - if (!IS_ALIGNED(y, 64)) - return -EINVAL; - ret = ave_ipc_wait_ack(ave, buf); if (ret) return ret; @@ -569,7 +587,7 @@ int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff return -EBUSY; memcpy(cmd, tmpl->data, tmpl->size); ave_cmd_header(ctx, cmd, AVE_OP_PROCESS, AVE_PROC_SUBTYPE + slot, 0, - ave_session_arg2(p->codec)); + ave_session_arg2(ctx)); /* Input picture: NV12, chroma right after p->height luma lines. */ ave_put64(cmd, l->y, y); diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c index 5ed6cf9176a85..6d67ddf3780e1 100644 --- a/drivers/media/platform/apple/ave/ave-drv.c +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -127,15 +127,12 @@ static int ave_suspend(struct device *dev) v4l2_m2m_suspend(ave->m2m_dev); /* - * An open session cannot survive the power going down: it is ended - * here (its next job fails) and the firmware halted, so the next - * session restarts it from the saved DATA copy. + * The firmware does not survive the power going down. Drain and close + * every open session and halt the firmware, as at the end of a stream; + * each session reopens in a cold-started firmware on its next frame. */ mutex_lock(&ave->hw_lock); - if (ave->active) { - ave->active->sess->dead = true; - ave->active = NULL; - } + ave_v4l2_suspend(ave); if (ave_fw_shutdown(ave)) dev_warn(dev, "firmware not answering, it cannot be stopped for sleep\n"); mutex_unlock(&ave->hw_lock); @@ -175,11 +172,11 @@ static struct platform_driver ave_driver = { }; module_platform_driver(ave_driver); -/* Constant parts of Start and Process for AppleAVE2FW 9003.78.0 (macOS 26.0.x) */ -MODULE_FIRMWARE("apple/ave-t8140-69c54454-h264-start.bin"); -MODULE_FIRMWARE("apple/ave-t8140-69c54454-h264-process.bin"); -MODULE_FIRMWARE("apple/ave-t8140-69c54454-hevc-start.bin"); -MODULE_FIRMWARE("apple/ave-t8140-69c54454-hevc-process.bin"); +/* Session images, extracted from the user's macOS installation */ +MODULE_FIRMWARE("apple/ave-j700-h264-start.bin"); +MODULE_FIRMWARE("apple/ave-j700-h264-process.bin"); +MODULE_FIRMWARE("apple/ave-j700-hevc-start.bin"); +MODULE_FIRMWARE("apple/ave-j700-hevc-process.bin"); MODULE_DESCRIPTION("Apple AVE video encoder"); MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c index e1f9a11118ed0..5db179711c0a1 100644 --- a/drivers/media/platform/apple/ave/ave-fw.c +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -106,6 +106,21 @@ static u32 ave_fw_crc(const struct ave_fw_segment *seg, void **copy) return crc; } +/* + * The session images are per machine: "j700" from "apple,j700", the first + * root compatible. + */ +static void ave_fw_board_name(struct ave_dev *ave) +{ + const char *compat = NULL, *comma; + + of_property_read_string_index(of_root, "compatible", 0, &compat); + if (!compat) + compat = "unknown"; + comma = strchr(compat, ','); + strscpy(ave->board, comma ? comma + 1 : compat, sizeof(ave->board)); +} + /* * Find the firmware, check that it is a build we speak to and that DATA is * still as the bootloader left it, and keep a copy of DATA for restarts. @@ -137,6 +152,7 @@ int ave_fw_probe(struct ave_dev *ave) text_crc); ave->text_crc = text_crc; mutex_init(&ave->tmpl_lock); + ave_fw_board_name(ave); ave->data_pristine = vmalloc(ave->data.size); if (!ave->data_pristine) @@ -169,6 +185,7 @@ static void ave_fw_restore_data(struct ave_dev *ave) if (WARN_ON(!va)) return; memcpy(va, ave->data_pristine, ave->data.size); + /* The restored image must be in memory before the coprocessor starts. */ wmb(); memunmap(va); ave->data_dirty = false; @@ -209,9 +226,10 @@ int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, unsigned long } /* - * Start and Process are mostly constant for a firmware build. Those - * constant words come from per-build image files named after the TEXT - * checksum; the driver fills in every session and frame field itself. + * Start and Process are mostly constant for a machine and its firmware. + * Those constant words come from image files named after the machine, + * which the user extracts from their own macOS installation; the driver + * fills in every session and frame field itself. */ static const char * const ave_codec_name[] = { "h264", "hevc" }; static const char * const ave_tmpl_name[AVE_TMPL_COUNT] = { "start", "process" }; @@ -234,18 +252,24 @@ int ave_fw_templates(struct ave_dev *ave, enum ave_codec codec) if (ave->tmpl[c][t]) continue; - snprintf(name, sizeof(name), "apple/ave-t8140-%08x-%s-%s.bin", ave->text_crc, + snprintf(name, sizeof(name), "apple/ave-%s-%s-%s.bin", ave->board, ave_codec_name[c], ave_tmpl_name[t]); ret = request_firmware(&fw, name, ave->dev); if (ret) { - dev_err(ave->dev, "cannot load %s: %d\n", name, ret); + dev_err(ave->dev, "cannot load %s (%d); extract it from macOS first\n", + name, ret); + break; + } + if (fw->size != ave_tmpl_size[c][t]) { + dev_err(ave->dev, "%s is %zu bytes, expected %zu\n", name, fw->size, + ave_tmpl_size[c][t]); + release_firmware(fw); + ret = -EINVAL; break; } - if (fw->size != ave_tmpl_size[c][t] || - (t == AVE_TMPL_START && - get_unaligned_le32(fw->data + AVE_TMPL_CODEC) != codec)) { - dev_err(ave->dev, "%s does not fit this driver (%zu bytes)\n", name, - fw->size); + if (t == AVE_TMPL_START && get_unaligned_le32(fw->data + AVE_TMPL_CODEC) != codec) { + dev_err(ave->dev, "%s is not a %s session image\n", name, + ave_codec_name[c]); release_firmware(fw); ret = -EINVAL; break; @@ -355,7 +379,6 @@ static int ave_fw_boot(struct ave_dev *ave) memset(ave->want, 0, sizeof(ave->want)); ave->reply_err = 0; ave->scalar_next = 0; - ave->next_token = AVE_TOKEN_BASE; ave_fw_restore_data(ave); diff --git a/drivers/media/platform/apple/ave/ave-ipc.c b/drivers/media/platform/apple/ave/ave-ipc.c index af067fb024e1e..974f3b2e8cc90 100644 --- a/drivers/media/platform/apple/ave/ave-ipc.c +++ b/drivers/media/platform/apple/ave/ave-ipc.c @@ -39,6 +39,7 @@ static void ave_mbox_send(struct ave_dev *ave, u32 a, u32 b, u32 c, u32 d) writel(b, ave->mbox + AVE_MBOX_SCRATCH(1)); writel(c, ave->mbox + AVE_MBOX_SCRATCH(2)); writel(d, ave->mbox + AVE_MBOX_SCRATCH(3)); + /* All four words must land before the doorbell. */ wmb(); writel(1, ave->mbox + AVE_MBOX_DOORBELL); } @@ -393,10 +394,7 @@ static void ave_ipc_dispatch(struct ave_dev *ave) status = get_unaligned_le32(ave->msg + AVE_REPLY_STATUS); if (id == AVE_OP_PROCESS) { - if (ave->active) - ave_sess_reply(ave->active, ave->msg, ave->msg_len); - else - dev_warn(ave->dev, "frame reply without a session\n"); + ave_sess_reply(ave, ave->msg, ave->msg_len); return; } if (id < ARRAY_SIZE(ave->want) && ave->want[id]) { @@ -465,11 +463,19 @@ static int ave_ipc_poll_once(struct ave_dev *ave) * Handle acks and messages until @cond holds, for at most @wait_ms. With no * @cond, handle whatever is pending and return. Running out of time means * the firmware is stuck: it is marked broken. Caller holds hw_lock. + * + * Which interrupt line signals the rings is not known, so this polls. It + * sleeps between polls: briefly at first (commands are acknowledged within + * microseconds), then for up to a millisecond, so a long wait for a frame + * costs next to no CPU time. */ +#define AVE_POLL_FAST_ROUNDS 20 + int ave_ipc_pump(struct ave_dev *ave, unsigned int wait_ms, bool (*cond)(struct ave_dev *ave, void *arg), void *arg) { ktime_t deadline = ktime_add_ms(ktime_get(), wait_ms); + unsigned int idle = 0; int ret; lockdep_assert_held(&ave->hw_lock); @@ -486,15 +492,20 @@ int ave_ipc_pump(struct ave_dev *ave, unsigned int wait_ms, ave_ipc_mark_broken(ave, ret, "ring error"); return ret; } - if (ret) + if (ret) { + idle = 0; continue; + } if (!cond) return 0; if (ktime_after(ktime_get(), deadline)) { ave_ipc_mark_broken(ave, -ETIMEDOUT, "no answer"); return -ETIMEDOUT; } - usleep_range(50, 150); + if (idle++ < AVE_POLL_FAST_ROUNDS) + usleep_range(50, 100); + else + usleep_range(500, 1000); } } diff --git a/drivers/media/platform/apple/ave/ave-regs.h b/drivers/media/platform/apple/ave/ave-regs.h index 894d2e3d4ff0a..a85fde26278a6 100644 --- a/drivers/media/platform/apple/ave/ave-regs.h +++ b/drivers/media/platform/apple/ave/ave-regs.h @@ -126,6 +126,7 @@ enum ave_opcode { #define AVE_SESSION_ARG0 0xc8 #define AVE_SESSION_ARG1 0x36 #define AVE_SESSION_ARG2_SETUP 0x1e0 /* Open and Start */ +#define AVE_SESSION_ARG2_SETUP_SHARED 0x140 /* the same, another session open */ #define AVE_SESSION_ARG2_H264 0x40 /* every other session command */ #define AVE_SESSION_ARG2_HEVC 0x20 #define AVE_TOKEN_STEP 100 diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index 9f5b46704ac4c..b96a84ad6d1a8 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -18,6 +18,7 @@ #include #include +#include #include #include "ave.h" @@ -276,9 +277,67 @@ static void ave_sess_free_bufs(struct ave_dev *ave, struct ave_session *sess) ave_free(ave, &sess->slot[s].coded); ave_free(ave, &sess->slot[s].meta); ave_free(ave, &sess->slot[s].aux); + ave_free(ave, &sess->slot[s].bounce); } } +/* + * The input picture for @slot, as a DART address. The encoder reads NV12 + * with the 64-byte aligned stride S_FMT reported and chroma right after + * height luma lines. Some clients (ffmpeg's v4l2m2m encoder) fill the + * buffer with their own, smaller line size instead, which shows in the + * payload size; such frames are copied to a per-slot buffer in the + * encoder's layout. + */ +int ave_sess_input(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *src, + dma_addr_t *input) +{ + struct ave_session *sess = ctx->sess; + const struct ave_params *p = &sess->p; + struct vb2_buffer *vb = &src->vb2_buf; + u32 off = vb->planes[0].data_offset; + u32 used = vb2_get_plane_payload(vb, 0) - min(off, vb2_get_plane_payload(vb, 0)); + u32 crows = DIV_ROUND_UP(p->height, 2), rows = p->height + crows; + u32 width = ALIGN(p->width, 2), stride; + struct ave_buf *b = &sess->slot[slot].bounce; + const u8 *from; + u8 *to; + int ret; + + *input = vb2_dma_contig_plane_dma_addr(vb, 0) + off; + if (!used || used >= p->stride * rows || used % rows || used / rows < width) { + if (IS_ALIGNED(*input, 64)) + return 0; + dev_err(ctx->ave->dev, "OUTPUT plane at %pad is not 64-byte aligned\n", input); + return -EINVAL; + } + stride = used / rows; + + from = vb2_plane_vaddr(vb, 0); + if (!from) { + dev_err(ctx->ave->dev, "cannot map an OUTPUT buffer to copy its %u byte stride\n", + stride); + return -EFAULT; + } + from += off; + ret = ave_alloc(ctx->ave, b, p->stride * rows, 0); + if (ret) { + dev_err(ctx->ave->dev, "no room to copy %u byte stride input\n", stride); + return ret; + } + if (!sess->copy_noted) { + sess->copy_noted = true; + dev_info(ctx->ave->dev, "input has a %u byte stride; copying to %u\n", stride, + p->stride); + } + + to = b->cpu; + for (u32 y = 0; y < rows; y++) + memcpy(to + y * p->stride, from + y * stride, width); + *input = b->dma; + return 0; +} + /* Debug map of the session buffers, to match DART faults to a buffer. */ static void ave_log_buf(struct ave_dev *ave, const char *name, int i, const struct ave_buf *b) { @@ -362,6 +421,20 @@ static void ave_sess_check_tails(struct ave_dev *ave, struct ave_session *sess) ave_check_tail(ave, "meta", i, &sess->slot[i].meta); ave_check_tail(ave, "aux", i, &sess->slot[i].aux); } + + /* + * The size of the H.264 per-frame buffer is not known; it is + * allocated generously. Report how much of it the firmware used. + */ + if (sess->p.codec == AVE_CODEC_H264 && sess->slot[0].aux.cpu) { + const struct ave_buf *b = &sess->slot[0].aux; + const u64 *w = b->cpu; + size_t n = b->size / 8; + + while (n && !w[n - 1]) + n--; + dev_dbg(ave->dev, "per-frame buffer: %#zx of %#zx bytes used\n", n * 8, b->size); + } } /* Work out the parameters and allocate the buffers; no firmware access. */ @@ -370,7 +443,7 @@ struct ave_session *ave_sess_create(struct ave_ctx *ctx) struct ave_session *sess; int ret; - sess = kzalloc(sizeof(*sess), GFP_KERNEL); + sess = kzalloc_obj(*sess); if (!sess) return ERR_PTR(-ENOMEM); ret = ave_sess_params(ctx, &sess->p); @@ -437,18 +510,27 @@ void ave_sess_prepare_slot(struct ave_session *sess, unsigned int slot) memset(s->aux.cpu, 0, AVE_SHDR_ENTRY_BYTES); } -/* A frame reply: +0x10 our client, +0x20 the slot's subtype, +0x40 status. */ -void ave_sess_reply(struct ave_ctx *ctx, const u8 *msg, u32 len) +/* + * A frame reply: +0x10 the session's client token, +0x20 the slot's + * subtype, +0x40 status. Caller holds hw_lock. + */ +void ave_sess_reply(struct ave_dev *ave, const u8 *msg, u32 len) { - struct ave_session *sess = ctx->sess; u64 client = get_unaligned_le64(msg + AVE_REPLY_CLIENT); u32 subtype = get_unaligned_le32(msg + AVE_REPLY_SUBTYPE); u32 slot = subtype - AVE_PROC_SUBTYPE; + struct ave_session *sess = NULL; + struct ave_ctx *ctx = NULL; struct ave_slot *s; - if (!sess || len < AVE_REPLY_PROCESS_BYTES || client != sess->client || - slot >= sess->p.slots || !sess->slot[slot].busy) { - dev_warn(ctx->ave->dev, "stray frame reply (client %llu subtype %#x len %u)\n", + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i] && ave->sessions[i]->sess->client == client) + ctx = ave->sessions[i]; + if (ctx) + sess = ctx->sess; + if (!sess || len < AVE_REPLY_PROCESS_BYTES || slot >= sess->p.slots || + !sess->slot[slot].busy) { + dev_warn(ave->dev, "stray frame reply (client %llu subtype %#x len %u)\n", client, subtype, len); return; } diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index 281e9cbad38b0..caa874ba48de3 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -594,14 +594,35 @@ static void ave_return_bufs(struct ave_ctx *ctx, struct vb2_queue *q, /* Sessions */ -/* Caller holds hw_lock. */ +static int ave_session_index(struct ave_dev *ave, struct ave_ctx *ctx) +{ + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i] == ctx) + return i; + return -1; +} + +static unsigned int ave_session_count(struct ave_dev *ave) +{ + unsigned int n = 0; + + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + n += !!ave->sessions[i]; + return n; +} + +/* + * Drain (with @flush), stop and close the session in the firmware. The + * last session to go halts the firmware. Caller holds hw_lock. + */ static void ave_session_close(struct ave_ctx *ctx, bool flush) { struct ave_dev *ave = ctx->ave; struct ave_session *sess = ctx->sess; + int idx = ave_session_index(ave, ctx); lockdep_assert_held(&ave->hw_lock); - if (!sess || ave->active != ctx) + if (!sess || idx < 0) return; if (sess->fw_open && ave->fw_state == AVE_FW_RUNNING) { /* Complete is a barrier: answered after every frame before it. */ @@ -611,8 +632,81 @@ static void ave_session_close(struct ave_ctx *ctx, bool flush) ave_cmd_close(ctx); } sess->fw_open = false; - ave->active = NULL; - ave_fw_halt(ave); + ave->sessions[idx] = NULL; + if (!ave_session_count(ave)) + ave_fw_halt(ave); +} + +/* + * Open and Start the session in the firmware, booting the firmware first + * if no other session runs. Returns -EAGAIN when a halted coprocessor has + * not lost power yet. Caller holds hw_lock. + */ +static int ave_session_attach(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + int idx = -1, ret; + + lockdep_assert_held(&ave->hw_lock); + for (unsigned int i = 0; i < AVE_MAX_SESSIONS && idx < 0; i++) + if (!ave->sessions[i]) + idx = i; + if (idx < 0) { + dev_err(ave->dev, "already encoding %u streams\n", AVE_MAX_SESSIONS); + return -EBUSY; + } + if (!ave_session_count(ave)) { + ret = ave_fw_ensure_running(ave); + if (ret) + return ret; + } else if (ave->fw_state != AVE_FW_RUNNING) { + dev_err(ave->dev, "firmware state %d, cannot add a session\n", ave->fw_state); + return -EIO; + } + + sess->index = idx; + sess->frames = 0; + sess->nudged = false; + ave->sessions[idx] = ctx; + ret = ave_cmd_open(ctx); + if (!ret) + ret = ave_cmd_start(ctx); + if (ret) + ave_session_close(ctx, false); + return ret; +} + +/* + * Attach with the device runtime active and one PM reference held by the + * caller. A halted coprocessor restarts only from reset, and its power can + * take a moment to go (the DARTs suspend a little later), so drop the + * reference and cycle the power a few times before giving up. On return + * the reference is still held, unless *pm_lost says otherwise. + */ +static int ave_session_attach_cycled(struct ave_ctx *ctx, bool *pm_lost) +{ + struct ave_dev *ave = ctx->ave; + int ret = -EAGAIN; + + *pm_lost = false; + for (unsigned int i = 0; i < AVE_RESTART_TRIES && ret == -EAGAIN; i++) { + if (i) { + pm_runtime_put_sync_suspend(ave->dev); + msleep(AVE_RESTART_WAIT_MS); + ret = pm_runtime_resume_and_get(ave->dev); + if (ret) { + *pm_lost = true; + return ret; + } + } + mutex_lock(&ave->hw_lock); + ret = ave_session_attach(ctx); + mutex_unlock(&ave->hw_lock); + } + if (ret == -EAGAIN) + dev_err(ave->dev, "coprocessor did not power down; cannot restart it\n"); + return ret; } /* Return frames the firmware never answered; then drop the session. */ @@ -641,21 +735,24 @@ static int ave_session_begin(struct ave_ctx *ctx) { struct ave_dev *ave = ctx->ave; struct ave_session *sess; + bool pm_lost; int ret; if (ctx->sess) return 0; - /* A halted firmware restarts only from reset: let the power go first. */ - if (READ_ONCE(ave->fw_state) == AVE_FW_HALTED) - pm_runtime_suspend(ave->dev); ret = ave_fw_templates(ave, ave_ctx_codec(ctx)); if (ret) return ret; - if (READ_ONCE(ave->active)) + if (READ_ONCE(ave->sessions[0]) && READ_ONCE(ave->sessions[1])) return -EBUSY; + /* A halted firmware restarts only from reset: let the power go first. */ + if (READ_ONCE(ave->fw_state) == AVE_FW_HALTED) + pm_runtime_suspend(ave->dev); sess = ave_sess_create(ctx); - if (IS_ERR(sess)) + if (IS_ERR(sess)) { + dev_err(ave->dev, "cannot set up a session: %ld\n", PTR_ERR(sess)); return PTR_ERR(sess); + } ctx->sess = sess; ret = pm_runtime_resume_and_get(ave->dev); @@ -663,50 +760,59 @@ static int ave_session_begin(struct ave_ctx *ctx) ave_sess_destroy(ctx); return ret; } - - mutex_lock(&ave->hw_lock); - if (ave->active) { - ret = -EBUSY; - goto out; - } - /* - * A halted coprocessor restarts only from reset. Its power can take - * a moment to go after a suspend (the DARTs follow later), so cycle - * it again a few times before giving up. - */ - for (unsigned int i = 0; i < AVE_RESTART_TRIES; i++) { - ret = ave_fw_ensure_running(ave); - if (ret != -EAGAIN) - break; - mutex_unlock(&ave->hw_lock); - pm_runtime_put_sync_suspend(ave->dev); - msleep(AVE_RESTART_WAIT_MS); - ret = pm_runtime_resume_and_get(ave->dev); - mutex_lock(&ave->hw_lock); - if (ret) { - mutex_unlock(&ave->hw_lock); - ave_sess_destroy(ctx); - return ret; - } - ret = -EAGAIN; - } - if (ret) - goto out; - ave->active = ctx; - ret = ave_cmd_open(ctx); - if (!ret) - ret = ave_cmd_start(ctx); - if (ret) - ave_session_close(ctx, false); -out: - mutex_unlock(&ave->hw_lock); + ret = ave_session_attach_cycled(ctx, &pm_lost); if (ret) { dev_err(ave->dev, "cannot start an encoding session: %d\n", ret); - ave_session_free(ctx); + if (pm_lost) + ave_sess_destroy(ctx); + else + ave_session_free(ctx); } return ret; } +/* + * After system sleep: the session was closed when the firmware went down. + * Open and Start it again with the same parameters; its next frame is an + * IDR with fresh parameter sets, so the stream stays decodable. + */ +static int ave_session_reopen(struct ave_ctx *ctx) +{ + bool pm_lost; + int ret; + + ret = ave_session_attach_cycled(ctx, &pm_lost); + if (ret) { + if (pm_lost) { + /* The session's PM reference is gone; do not drop it again. */ + pm_runtime_get_noresume(ctx->ave->dev); + } + return ret; + } + ctx->sess->resume = false; + dev_dbg(ctx->ave->dev, "session reopened after system sleep\n"); + return 0; +} + +/* + * System sleep: drain every open session, close it and halt the firmware, + * as at the end of a stream. Sessions reopen on their next frame. + * Caller holds hw_lock. + */ +void ave_v4l2_suspend(struct ave_dev *ave) +{ + lockdep_assert_held(&ave->hw_lock); + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) { + struct ave_ctx *ctx = ave->sessions[i]; + + if (!ctx) + continue; + cancel_delayed_work(&ctx->poll_work); + ave_session_close(ctx, true); + ctx->sess->resume = true; + } +} + static void ave_session_end(struct ave_ctx *ctx) { struct ave_dev *ave = ctx->ave; @@ -878,30 +984,6 @@ static bool ave_frame_answered(struct ave_dev *ave, void *arg) return !s->busy; } -/* - * The encoder reads NV12 with the 64-byte aligned stride S_FMT reported. - * ffmpeg's v4l2m2m encoder copies frames with its own line size instead, - * which differs when the width is not a multiple of 64; the result is a - * sheared picture. Spot that from the payload size and refuse it. - */ -static int ave_check_src_layout(struct ave_ctx *ctx, struct vb2_v4l2_buffer *src) -{ - const struct ave_params *p = &ctx->sess->p; - u32 used = vb2_get_plane_payload(&src->vb2_buf, 0); - u32 rows = p->height + DIV_ROUND_UP(p->height, 2); - u32 stride; - - if (!used || used % rows) - return 0; - stride = used / rows; - if (stride >= p->stride || stride < p->width) - return 0; - dev_err(ctx->ave->dev, - "OUTPUT frame laid out with a %u byte stride, not the %u reported by S_FMT\n", - stride, p->stride); - return -EINVAL; -} - /* Send the next OUTPUT frame. Caller holds hw_lock. */ static int ave_encode_one(struct ave_ctx *ctx) { @@ -909,6 +991,7 @@ static int ave_encode_one(struct ave_ctx *ctx) struct ave_session *sess = ctx->sess; struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; struct vb2_v4l2_buffer *src; + dma_addr_t input; bool last; int slot, ret; @@ -930,14 +1013,10 @@ static int ave_encode_one(struct ave_ctx *ctx) last = src == ctx->drain_src; src->sequence = ctx->src_sequence++; - ret = ave_check_src_layout(ctx, src); - if (ret) { - v4l2_m2m_buf_done(src, VB2_BUF_STATE_ERROR); - return ret; - } - - ret = ave_cmd_process(ctx, slot, src, READ_ONCE(ctx->bitrate), - xchg(&ctx->force_key_frame, false)); + ret = ave_sess_input(ctx, slot, src, &input); + if (!ret) + ret = ave_cmd_process(ctx, slot, src, input, READ_ONCE(ctx->bitrate), + xchg(&ctx->force_key_frame, false)); if (ret) { v4l2_m2m_buf_done(src, VB2_BUF_STATE_ERROR); return ret; @@ -1031,19 +1110,23 @@ static void ave_run_work(struct work_struct *work) bool eos = false, poll = false; int ret = 0; - /* After a drain, the next frame starts a new session. */ - if (!ctx->sess && !READ_ONCE(ctx->flush_pending) && v4l2_m2m_num_src_bufs_ready(m2m)) - ret = ave_session_begin(ctx); + /* + * After a drain the next frame starts a new session; after system + * sleep it reopens the old one in the restarted firmware. + */ + if (!READ_ONCE(ctx->flush_pending) && v4l2_m2m_num_src_bufs_ready(m2m)) { + if (!ctx->sess) + ret = ave_session_begin(ctx); + else if (ctx->sess->resume) + ret = ave_session_reopen(ctx); + } mutex_lock(&ave->hw_lock); /* - * A closed session (after a drain) may still hold frames for CAPTURE - * buffers; only an open one must still own the firmware. + * A closed session (after a drain or for sleep) may still hold frames + * for CAPTURE buffers; only an open one must still own the firmware. */ - if (!ret && ctx->sess && ctx->sess->dead) { - dev_err(ave->dev, "session ended by system sleep\n"); - ret = -EIO; - } else if (!ret && ctx->sess && ctx->sess->fw_open && ave->active != ctx) { + if (!ret && ctx->sess && ctx->sess->fw_open && ave_session_index(ave, ctx) < 0) { dev_err(ave->dev, "session lost the firmware\n"); ret = -EIO; } @@ -1090,7 +1173,7 @@ static void ave_poll_work(struct work_struct *work) bool again = false, fail = false; mutex_lock(&ave->hw_lock); - if (ctx->sess && ave->active == ctx && !ctx->sess->dead) { + if (ctx->sess && ave_session_index(ave, ctx) >= 0) { fail = ave_ipc_pump(ave, 0, NULL, NULL) != 0; if (!fail && ave_sess_starved(ctx->sess) && !v4l2_m2m_num_src_bufs_ready(ctx->fh.m2m_ctx)) { @@ -1147,7 +1230,7 @@ static int ave_open(struct file *file) struct ave_ctx *ctx; int ret; - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + ctx = kzalloc_obj(*ctx); if (!ctx) return -ENOMEM; ctx->ave = ave; diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h index 018795a5ebfd8..c5dce46faa373 100644 --- a/drivers/media/platform/apple/ave/ave.h +++ b/drivers/media/platform/apple/ave/ave.h @@ -23,8 +23,10 @@ struct dev_pm_domain_list; struct firmware; struct iommu_domain; -/* Frames the firmware can hold at once (Process "slots"). */ +/* Frames the firmware can hold at once per session (Process "slots"). */ #define AVE_MAX_SLOTS 5 +/* Sessions one firmware boot serves at once (macOS was seen running two). */ +#define AVE_MAX_SESSIONS 2 /* * Coprocessor memory every firmware boot uses, all inside one allocation. @@ -44,13 +46,14 @@ struct iommu_domain; #define AVE_IPC_SCALAR_BUFS 16 #define AVE_IPC_PROC_OFF 0x60000 #define AVE_IPC_PROC_BYTES 0x10000 -#define AVE_IPC_END (AVE_IPC_PROC_OFF + AVE_MAX_SLOTS * AVE_IPC_PROC_BYTES) +#define AVE_IPC_PROC_BUFS (AVE_MAX_SESSIONS * AVE_MAX_SLOTS) +#define AVE_IPC_END (AVE_IPC_PROC_OFF + AVE_IPC_PROC_BUFS * AVE_IPC_PROC_BYTES) /* Command buffer ids, one bit each in ave_dev.unacked. */ #define AVE_BUF_CMD 0 #define AVE_BUF_SCALAR(i) (1 + (i)) #define AVE_BUF_PROC(s) (1 + AVE_IPC_SCALAR_BUFS + (s)) -#define AVE_BUF_COUNT AVE_BUF_PROC(AVE_MAX_SLOTS) +#define AVE_BUF_COUNT AVE_BUF_PROC(AVE_IPC_PROC_BUFS) #define AVE_MSG_MAX SZ_4K @@ -110,6 +113,7 @@ struct ave_dev { struct ave_fw_segment text; struct ave_fw_segment data; u32 text_crc; + char board[16]; /* from the root compatible, e.g. "j700" */ void *data_pristine; bool data_dirty; @@ -127,19 +131,19 @@ struct ave_dev { u64 ipc_fw_base; u32 channel_bytes; struct ave_ring ring[2]; - u64 next_token; u32 next_client; u32 unacked; u8 want[16]; int reply_err; unsigned int scalar_next; - struct ave_ctx *active; + struct ave_ctx *sessions[AVE_MAX_SESSIONS]; u8 msg[AVE_MSG_MAX]; u32 msg_len; struct v4l2_device v4l2_dev; struct video_device vdev; struct v4l2_m2m_dev *m2m_dev; + /* The video device's ioctl and queue lock. */ struct mutex vdev_lock; struct workqueue_struct *wq; }; @@ -174,6 +178,7 @@ struct ave_slot { struct ave_buf coded; struct ave_buf meta; struct ave_buf aux; /* H.264 per-frame buffer, HEVC slice headers */ + struct ave_buf bounce; /* input copied to the encoder's stride */ struct vb2_v4l2_buffer *src; u32 status; bool busy; /* Process sent, reply pending */ @@ -202,8 +207,11 @@ struct ave_session { unsigned int busy; u32 frames; /* Process commands sent */ u32 client; + u64 token; /* command order token, per session */ + unsigned int index; /* in ave_dev.sessions[] */ bool fw_open; /* Open sent, Close not yet */ - bool dead; /* firmware gone; buffers only */ + bool resume; /* closed for system sleep, reopen on next frame */ + bool copy_noted; bool warned; bool checked[2]; /* Start/Process images checked for stale addresses */ bool handed; /* Start sent: the firmware knows the buffers */ @@ -281,7 +289,7 @@ int ave_cmd_halt(struct ave_dev *ave); int ave_cmd_open(struct ave_ctx *ctx); int ave_cmd_start(struct ave_ctx *ctx); int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *src, - u32 bitrate, bool force_key); + dma_addr_t input, u32 bitrate, bool force_key); int ave_cmd_complete(struct ave_ctx *ctx, unsigned int count); int ave_cmd_close(struct ave_ctx *ctx); @@ -289,7 +297,9 @@ int ave_cmd_close(struct ave_ctx *ctx); struct ave_session *ave_sess_create(struct ave_ctx *ctx); void ave_sess_destroy(struct ave_ctx *ctx); int ave_sess_free_slot(struct ave_session *sess); -void ave_sess_reply(struct ave_ctx *ctx, const u8 *msg, u32 len); +void ave_sess_reply(struct ave_dev *ave, const u8 *msg, u32 len); +int ave_sess_input(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *src, + dma_addr_t *input); int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *dst, bool *keyframe); void ave_sess_prepare_slot(struct ave_session *sess, unsigned int slot); @@ -299,6 +309,7 @@ bool ave_sess_stalled(struct ave_session *sess); bool ave_sess_starved(struct ave_session *sess); /* ave-v4l2.c */ +void ave_v4l2_suspend(struct ave_dev *ave); int ave_v4l2_register(struct ave_dev *ave); void ave_v4l2_unregister(struct ave_dev *ave); enum ave_codec ave_ctx_codec(struct ave_ctx *ctx); From 26f65bc170b0542ea178b632e91846dc639f48c2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 21:54:56 +1100 Subject: [PATCH 1396/1579] media: apple: ave: document usage and session images, add MAINTAINERS Describe how to use the encoder, where the session images come from (the user's own macOS installation, through the Aurora firmware extractor; no Apple data in the tree), how sessions, power and system sleep behave, and the limits. Update the Kconfig help to match, and add a MAINTAINERS entry. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- Documentation/admin-guide/media/apple-ave.rst | 111 +++++++++++------- MAINTAINERS | 8 ++ drivers/media/platform/apple/ave/Kconfig | 10 +- 3 files changed, 84 insertions(+), 45 deletions(-) diff --git a/Documentation/admin-guide/media/apple-ave.rst b/Documentation/admin-guide/media/apple-ave.rst index 8ca9950dbea4b..788c121d8db51 100644 --- a/Documentation/admin-guide/media/apple-ave.rst +++ b/Documentation/admin-guide/media/apple-ave.rst @@ -5,10 +5,26 @@ Apple AVE video encoder The AVE block in Apple T8140 (A18 Pro) SoCs, as used in the MacBook Neo, encodes H.264 and HEVC. ``CONFIG_VIDEO_APPLE_AVE`` builds the ``apple-ave`` -driver, which registers a V4L2 memory-to-memory stateful encoder following +driver, a V4L2 memory-to-memory stateful encoder following Documentation/userspace-api/media/v4l/dev-encoder.rst. The OUTPUT queue -takes NV12 frames (64-byte aligned line stride), the CAPTURE queue returns -H.264 (Baseline, Main, High) or HEVC (Main) as Annex B. +takes NV12 frames, the CAPTURE queue returns H.264 (Baseline, Main, High) or +HEVC (Main) as Annex B. + +Usage +----- + +The encoder shows up as a ``/dev/videoN`` node named "Apple AVE video +encoder". GStreamer's ``v4l2h264enc`` and ``v4l2h265enc`` and ffmpeg's +``h264_v4l2m2m`` and ``hevc_v4l2m2m`` work with it, for example:: + + gst-launch-1.0 -e videotestsrc num-buffers=300 ! \ + video/x-raw,format=NV12,width=1920,height=1080,framerate=30/1 ! \ + v4l2h264enc extra-controls="controls,video_bitrate=8000000" ! \ + h264parse ! filesink location=out.h264 + + ffmpeg -i in.mkv -pix_fmt nv12 -c:v h264_v4l2m2m -b:v 8M -g 30 out.h264 + +Up to two streams can be encoded at the same time. Firmware -------- @@ -17,57 +33,70 @@ The encoder runs firmware on its own coprocessor. The bootloader loads it into two reserved, DART-mapped segments (TEXT and DATA) but does not start it. The Aurora Stage 2 loader passes both regions in the device's ``memory-region`` property and records their CRC32 under ``/chosen`` as -``aurora,ave-text-crc32`` and ``aurora,ave-data-crc32``. - -The driver only binds to firmware builds whose command formats it knows, -identified by the TEXT checksum, and only when DATA still matches the -checksum the loader recorded. It keeps a copy of DATA and restores it before -every firmware boot, so the firmware can be started again from the same state. +``aurora,ave-text-crc32`` and ``aurora,ave-data-crc32``. The driver only binds +to firmware builds whose command formats it knows, identified by the TEXT +checksum, and only while DATA still matches the recorded checksum. It keeps +a copy of DATA and restores it before every firmware boot. Session images -------------- -The two large session commands, Start and Process, are mostly constant for a -given firmware build. The driver writes every field that depends on the -session or the frame (sizes, rate control, GOP structure, levels, cropping, -buffer addresses, frame numbers and times) and takes the remaining constant -words from four files per firmware build:: +Most words of the two large session commands, Start and Process, are +constant for a machine and its firmware. They are not part of the kernel. +The driver loads them from four files named after the machine (the first +root compatible without its vendor prefix, ``j700`` on the MacBook Neo):: - apple/ave-t8140--h264-start.bin (0x3180 bytes) - apple/ave-t8140--h264-process.bin (0x63d8 bytes) - apple/ave-t8140--hevc-start.bin (0x13f28 bytes) - apple/ave-t8140--hevc-process.bin (0xb1c0 bytes) + apple/ave-j700-h264-start.bin (0x3180 bytes) + apple/ave-j700-h264-process.bin (0x63d8 bytes) + apple/ave-j700-hevc-start.bin (0x13f28 bytes) + apple/ave-j700-hevc-process.bin (0xb1c0 bytes) -Each file is a complete command image. Address and size fields in it are -ignored. Without these files, ``VIDIOC_STREAMON`` fails with ``ENOENT``. +The files are extracted from the user's own macOS installation by the +Aurora firmware extractor, in the same way as other firmware the machine +needs, and installed under the firmware search path (usually +``/lib/firmware``). No Apple data is shipped with the kernel. The driver +writes every field that depends on the session or the frame itself, so the +same files serve every size, profile and rate. A missing or wrong-sized +file makes ``VIDIOC_STREAMON`` fail with a message naming it. Sessions and power management ----------------------------- -Each encoding session boots the firmware, opens and starts a session, encodes -and, at the end, drains, stops and closes the session and halts the firmware. -The next session restarts the firmware through the boot path. One session -runs at a time; a second one gets ``EBUSY``. +The firmware boots when the first stream starts and is halted when the last +one ends, as macOS does; the next stream starts it from reset. Runtime PM +powers the encoder down two seconds after the last stream. + +Up to five frames per stream are inside the firmware at once (three without +B-frames). The firmware reorders B-frames itself, so CAPTURE buffers come +back in bitstream order, each with the timestamp of the frame it codes. -Up to five frames are inside the firmware at once (three without B-frames). -The firmware reorders B-frames itself, so CAPTURE buffers come back in -bitstream order, each carrying the timestamp of the frame it codes. The driver -asks for at least six OUTPUT buffers, and limits both queues so a full set fits -the encoder's 256 MiB DMA window (fewer buffers at larger sizes). +Across system sleep, open streams are drained, closed and the firmware is +halted. On resume each stream reopens on its next frame, which starts with +an IDR and fresh parameter sets; clients see no error. -Runtime PM uses a two second autosuspend delay. A firmware that stops -answering keeps the encoder powered and is not used again until reboot. -Across system sleep an open session ends; the next one starts afresh. +Every wait for the firmware is bounded. A command or frame that gets no +answer within 5 seconds (Start: 10 seconds) fails the stream with ``EIO``; +a firmware that stops answering keeps the encoder powered and is not used +again until reboot. The driver polls the firmware rings with sleeping waits. Limits ------ -Picture sizes from 320x240 to 2560x1600 (in steps of two), up to 240 frames -per second. Rate control is VBR (with ``BITRATE_PEAK`` as a one-second data -rate cap, twice the average when left at 0) or CBR; constant quality is not offered. ``BITRATE`` may change -while streaming and applies from the next frame; ``FORCE_KEY_FRAME`` makes -the next frame an IDR. H.264 uses either three B-frames or none (Baseline and -a GOP size of 1 never use them). HEVC always uses them for now; with -``B_FRAMES`` set to 0 every frame is flushed through at once instead. The level -control is a minimum: the driver raises it to what the size, frame rate and -bit rate need. +- Picture sizes from 320x240 to 2560x1600 in steps of two, up to 240 frames + per second. +- The encoder reads NV12 with a 64-byte aligned line stride, which + ``VIDIOC_S_FMT`` reports. Clients that fill buffers with a smaller line + size (ffmpeg does, for widths that are not a multiple of 64) are detected + from the payload size and their frames copied to the encoder's layout. +- Rate control is VBR, with ``BITRATE_PEAK`` as a one-second data-rate cap + (twice the average when left at 0), or CBR. Constant quality is not + offered. ``BITRATE`` may change while streaming and applies from the next + frame; ``FORCE_KEY_FRAME`` makes the next frame an IDR. +- H.264 uses either three B-frames or none (Baseline and a GOP size of 1 never + use them). HEVC always uses them; with ``B_FRAMES`` set to 0 every frame is + flushed through at once instead. +- The level control is a minimum: the driver raises it to what the size, + frame rate and bit rate need. +- Buffers are mapped through the encoder's 256 MiB DMA window, so both queues + are limited to what fits next to the encoder's working memory (fewer + buffers at larger sizes). diff --git a/MAINTAINERS b/MAINTAINERS index 8b2cc48645bf4..7e126d34bed5c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2802,6 +2802,14 @@ F: net/bluetooth/brcm.c F: net/bluetooth/brcm.h F: tools/testing/selftests/drivers/apple/ +ARM/APPLE AVE VIDEO ENCODER DRIVER +M: Ryan Murray +L: linux-media@vger.kernel.org +S: Maintained +W: https://aurora-silicon.github.io +F: Documentation/admin-guide/media/apple-ave.rst +F: drivers/media/platform/apple/ave/ + ARM/APPLE QSPIMC DRIVER M: Ryan Murray W: https://aurora-silicon.github.io diff --git a/drivers/media/platform/apple/ave/Kconfig b/drivers/media/platform/apple/ave/Kconfig index 2b3f380abf752..e1bdc5de54a7f 100644 --- a/drivers/media/platform/apple/ave/Kconfig +++ b/drivers/media/platform/apple/ave/Kconfig @@ -10,10 +10,12 @@ config VIDEO_APPLE_AVE select VIDEOBUF2_DMA_CONTIG help V4L2 memory-to-memory driver for the AVE H.264/HEVC encoder in - Apple T8140 (A18 Pro) SoCs, as in the MacBook Neo. The encoder - firmware is loaded by the bootloader; the constant parts of its - session commands come from files under apple/ in the firmware - search path. + Apple T8140 (A18 Pro) SoCs, as in the MacBook Neo. + + The encoder firmware is loaded by the bootloader. The constant + parts of its session commands are read from apple/ave--*.bin + in the firmware search path, extracted from the user's own macOS + installation. See Documentation/admin-guide/media/apple-ave.rst. To compile this driver as a module, choose M here: the module will be called apple-ave. From 68c514b2264a98ce9a73b54e3e453bc8ca17e365 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 22:03:34 +1100 Subject: [PATCH 1397/1579] media: apple: ave: build the session commands in the driver macOS does not store the Start and Process commands anywhere: its driver builds them from the session settings and its own defaults. Apart from the fields this driver already computes, every captured command holds the same small set of constant words. They are now named tables in ave-image.c, grouped and commented, and each command is built in a zeroed buffer from the header, those words and the computed fields. The four session image files and request_firmware() are gone. The constants were checked on one firmware build only (AppleAVE2FW 9003.78.0, TEXT checksum 69c54454), so streams refuse to start on any other build, with a message naming the checksum. With these facts in place: - HEVC can run without B-frames, with the low-delay P reference structure macOS uses when frame reordering is off (Start +0xe120 table, the reorder and reference counts, Process +0xcf8). The per-frame Complete that stood in for it is gone. - H.264 marks a client's B-frame opt-out in Start byte +0x2585, and HEVC sets +0x2588, as macOS does. - CBR pads with filler data once more than 0.5 s of bit rate has gone unused, the rule seen in macOS streams. If the firmware's own output carries no filler, the driver appends a filler-data NAL unit to the access unit; the debug log says which side pads. debugfs apple-ave/last_start_{h264,hevc} and last_process0_{h264,hevc} hold the last Start and the first Process of the last session, for comparison with reference captures. Field values are facts from our private research (macOS traces); no copied code. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- Documentation/admin-guide/media/apple-ave.rst | 50 ++-- drivers/media/platform/apple/ave/Kconfig | 7 +- drivers/media/platform/apple/ave/Makefile | 2 +- drivers/media/platform/apple/ave/ave-cmd.c | 53 ++-- drivers/media/platform/apple/ave/ave-drv.c | 54 +++- drivers/media/platform/apple/ave/ave-fw.c | 82 +----- drivers/media/platform/apple/ave/ave-image.c | 278 ++++++++++++++++++ drivers/media/platform/apple/ave/ave-sess.c | 154 +++++----- drivers/media/platform/apple/ave/ave-v4l2.c | 13 +- drivers/media/platform/apple/ave/ave.h | 47 ++- 10 files changed, 496 insertions(+), 244 deletions(-) create mode 100644 drivers/media/platform/apple/ave/ave-image.c diff --git a/Documentation/admin-guide/media/apple-ave.rst b/Documentation/admin-guide/media/apple-ave.rst index 788c121d8db51..2a76d4eac23d1 100644 --- a/Documentation/admin-guide/media/apple-ave.rst +++ b/Documentation/admin-guide/media/apple-ave.rst @@ -34,30 +34,21 @@ two reserved, DART-mapped segments (TEXT and DATA) but does not start it. The Aurora Stage 2 loader passes both regions in the device's ``memory-region`` property and records their CRC32 under ``/chosen`` as ``aurora,ave-text-crc32`` and ``aurora,ave-data-crc32``. The driver only binds -to firmware builds whose command formats it knows, identified by the TEXT -checksum, and only while DATA still matches the recorded checksum. It keeps -a copy of DATA and restores it before every firmware boot. - -Session images --------------- - -Most words of the two large session commands, Start and Process, are -constant for a machine and its firmware. They are not part of the kernel. -The driver loads them from four files named after the machine (the first -root compatible without its vendor prefix, ``j700`` on the MacBook Neo):: - - apple/ave-j700-h264-start.bin (0x3180 bytes) - apple/ave-j700-h264-process.bin (0x63d8 bytes) - apple/ave-j700-hevc-start.bin (0x13f28 bytes) - apple/ave-j700-hevc-process.bin (0xb1c0 bytes) - -The files are extracted from the user's own macOS installation by the -Aurora firmware extractor, in the same way as other firmware the machine -needs, and installed under the firmware search path (usually -``/lib/firmware``). No Apple data is shipped with the kernel. The driver -writes every field that depends on the session or the frame itself, so the -same files serve every size, profile and rate. A missing or wrong-sized -file makes ``VIDIOC_STREAMON`` fail with a message naming it. +while DATA still matches the recorded checksum, and keeps a copy of DATA that +it restores before every firmware boot. + +No firmware files are needed. The driver builds every command itself, +including the two large session commands (Start and Process), from the +session settings and a set of constant fields known for one firmware build: +AppleAVE2FW 9003.78.0, TEXT checksum ``69c54454``. With any other firmware +build the device binds, but starting a stream fails with ``ENOTSUPP`` and a +message naming the checksum, until that build's command layout has been +checked. + +With debugfs mounted, ``apple-ave/last_start_h264``, +``last_start_hevc``, ``last_process0_h264`` and ``last_process0_hevc`` hold the +last Start command built and the first Process command of the last session, +for comparison with reference captures. Sessions and power management ----------------------------- @@ -89,12 +80,13 @@ Limits size (ffmpeg does, for widths that are not a multiple of 64) are detected from the payload size and their frames copied to the encoder's layout. - Rate control is VBR, with ``BITRATE_PEAK`` as a one-second data-rate cap - (twice the average when left at 0), or CBR. Constant quality is not - offered. ``BITRATE`` may change while streaming and applies from the next + (twice the average when left at 0), or CBR. In CBR mode, frames are padded + with filler data once more than half a second of bit rate has gone unused, + as macOS does; the driver adds the filler itself if the firmware does not. + Constant quality is not offered. ``BITRATE`` may change while streaming and applies from the next frame; ``FORCE_KEY_FRAME`` makes the next frame an IDR. -- H.264 uses either three B-frames or none (Baseline and a GOP size of 1 never - use them). HEVC always uses them; with ``B_FRAMES`` set to 0 every frame is - flushed through at once instead. +- Either three B-frames or none (``B_FRAMES`` 3 or 0). H.264 Baseline and a + GOP size of 1 never use them. - The level control is a minimum: the driver raises it to what the size, frame rate and bit rate need. - Buffers are mapped through the encoder's 256 MiB DMA window, so both queues diff --git a/drivers/media/platform/apple/ave/Kconfig b/drivers/media/platform/apple/ave/Kconfig index e1bdc5de54a7f..baf6218ecdd5b 100644 --- a/drivers/media/platform/apple/ave/Kconfig +++ b/drivers/media/platform/apple/ave/Kconfig @@ -5,17 +5,14 @@ config VIDEO_APPLE_AVE depends on ARCH_APPLE || COMPILE_TEST depends on VIDEO_DEV && IOMMU_DMA && PM_GENERIC_DOMAINS select CRC32 - select FW_LOADER select V4L2_MEM2MEM_DEV select VIDEOBUF2_DMA_CONTIG help V4L2 memory-to-memory driver for the AVE H.264/HEVC encoder in Apple T8140 (A18 Pro) SoCs, as in the MacBook Neo. - The encoder firmware is loaded by the bootloader. The constant - parts of its session commands are read from apple/ave--*.bin - in the firmware search path, extracted from the user's own macOS - installation. See Documentation/admin-guide/media/apple-ave.rst. + The encoder firmware is loaded by the bootloader; the driver builds + every command itself. See Documentation/admin-guide/media/apple-ave.rst. To compile this driver as a module, choose M here: the module will be called apple-ave. diff --git a/drivers/media/platform/apple/ave/Makefile b/drivers/media/platform/apple/ave/Makefile index 542a7adeae360..53bb6b554f770 100644 --- a/drivers/media/platform/apple/ave/Makefile +++ b/drivers/media/platform/apple/ave/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_VIDEO_APPLE_AVE) += apple-ave.o -apple-ave-y := ave-drv.o ave-fw.o ave-ipc.o ave-cmd.o ave-sess.o ave-v4l2.o +apple-ave-y := ave-drv.o ave-fw.o ave-ipc.o ave-cmd.o ave-image.o ave-sess.o ave-v4l2.o diff --git a/drivers/media/platform/apple/ave/ave-cmd.c b/drivers/media/platform/apple/ave/ave-cmd.c index 922db11c98b80..e453700693f5d 100644 --- a/drivers/media/platform/apple/ave/ave-cmd.c +++ b/drivers/media/platform/apple/ave/ave-cmd.c @@ -2,15 +2,13 @@ /* * Apple AVE video encoder: firmware commands * - * Start and Process are large command images. Most of their words never - * change between sessions of a firmware build; the driver takes those from - * a per-build image file (see ave_fw_templates()) and writes every field - * that depends on the session or the frame itself. + * Start and Process are large command images. Each is built in a zeroed + * buffer: the header, the constant words (ave-image.c), then every field + * that depends on the session or the frame. * * Copyright The Aurora Silicon Contributors */ #include -#include #include #include #include @@ -217,9 +215,6 @@ int ave_cmd_close(struct ave_ctx *ctx) /* Start (id 6) */ -#define AVE_START_H264_BYTES 0x3180 -#define AVE_START_HEVC_BYTES 0x13f28 - #define AVE_START_SURFACE 0x48 /* and an unused second one at 0x58 */ #define AVE_START_SURFACE_BYTES 0x12c000 #define AVE_START_SURFACE2 0x58 @@ -277,7 +272,7 @@ static const u16 ave_start_tile_off[AVE_TILE_BUFS] = { #define AVE_START_CODEC_FLAGS 0x370 #define AVE_START_H264_380 0x380 #define AVE_START_H264_384 0x384 -#define AVE_START_H264_HIGH 0x2588 +#define AVE_START_CODEC_2588 0x2588 /* 2 for H.264 High and HEVC */ #define AVE_START_BIT_DEPTH 0x2598 #define AVE_START_CODED_H2 0x25dc @@ -428,7 +423,7 @@ static void ave_start_config(const struct ave_params *p, u8 *cmd) if (!hevc) { ave_put32(cmd, AVE_START_H264_384, p->width * p->height <= 320 * 240 ? 0x40 : 0x10); - ave_put32(cmd, AVE_START_H264_HIGH, p->high ? 2 : 0); + ave_put32(cmd, AVE_START_CODEC_2588, p->high ? 2 : 0); ave_put32(cmd, AVE_START_H264_PROFILE, p->profile); ave_put32(cmd, AVE_START_H264_LEVEL, p->level); ave_put32(cmd, AVE_START_H264_MAX_REFS, p->refs - 1); @@ -468,33 +463,35 @@ static void ave_start_config(const struct ave_params *p, u8 *cmd) ave_put32(cmd, AVE_START_HEVC_CONF_BOTTOM, crop_b); ave_put32(cmd, AVE_START_HEVC_DEPTH_LUMA, 0); ave_put32(cmd, AVE_START_HEVC_DEPTH_CHROMA, 0); - /* Only the B-frame structure is described for HEVC (see ave-sess.c). */ - ave_put32(cmd, AVE_START_HEVC_REFS, 4); + /* Reference structure: B-frame pyramid or low-delay P (ave-image.c) */ + ave_put32(cmd, AVE_START_HEVC_REFS, p->bframes ? 4 : 1); ave_put32(cmd, AVE_START_HEVC_DPB_MINUS1, 4); - ave_put32(cmd, AVE_START_HEVC_REORDER, 2); - ave_put32(cmd, AVE_START_HEVC_REORDER_COPY, 2); - ave_put32(cmd, AVE_START_HEVC_REORDER_ON, 1); + ave_put32(cmd, AVE_START_HEVC_REORDER, p->bframes ? 2 : 0); + ave_put32(cmd, AVE_START_HEVC_REORDER_COPY, p->bframes ? 2 : 0); + ave_put32(cmd, AVE_START_HEVC_REORDER_ON, p->bframes); } int ave_cmd_start(struct ave_ctx *ctx) { struct ave_dev *ave = ctx->ave; struct ave_session *sess = ctx->sess; - const struct firmware *tmpl = ave->tmpl[sess->p.codec - 1][AVE_TMPL_START]; + const struct ave_params *p = &sess->p; + size_t len = p->codec == AVE_CODEC_HEVC ? AVE_START_HEVC_BYTES : AVE_START_H264_BYTES; u8 *cmd; int ret; - cmd = ave_ipc_cmd(ave, AVE_BUF_CMD, tmpl->size); + cmd = ave_ipc_cmd(ave, AVE_BUF_CMD, len); if (!cmd) return -EBUSY; - memcpy(cmd, tmpl->data, tmpl->size); + ave_image_start(cmd, len, p->codec, p->reorder_off, + p->codec == AVE_CODEC_HEVC && !p->bframes); ave_cmd_header(ctx, cmd, AVE_OP_START, AVE_OP_START, AVE_SESSION_ARG0, ave_session_setup_arg2(ctx)); ave_start_surfaces(sess, cmd); ave_start_config(&sess->p, cmd); - ave_sess_check_addrs(ctx, cmd, tmpl->size, AVE_TMPL_START, 0); + ave_debugfs_save(ave, p->codec, AVE_IMAGE_START, cmd, len); - ret = ave_ipc_submit(ave, AVE_BUF_CMD, tmpl->size, AVE_CMD_SESSION_TIMEOUT_US, true); + ret = ave_ipc_submit(ave, AVE_BUF_CMD, len, AVE_CMD_SESSION_TIMEOUT_US, true); if (ret) return ret; sess->handed = true; @@ -523,6 +520,7 @@ struct ave_proc_layout { u32 frame_buf; /* H.264 only */ u32 shdr_table; /* HEVC only */ u32 crop; /* HEVC only */ + u32 low_delay; /* HEVC only: 3 with reordering off */ }; static const struct ave_proc_layout ave_proc_h264 = { @@ -556,12 +554,14 @@ static const struct ave_proc_layout ave_proc_hevc = { .force_key = 0x6128, .force_type = 0x74c8, .shdr_table = 0xd88, .crop = 0x74dc, + .low_delay = 0xcf8, }; #define AVE_PROC_TIMESCALE 14400000ULL #define AVE_PROC_SHDR_ENTRIES 256 #define AVE_PROC_SHDR_STRIDE 0x400 #define AVE_PROC_FORCED_TYPE 2 +#define AVE_PROC_LOW_DELAY 3 int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *src, dma_addr_t input, u32 bitrate, bool force_key) @@ -571,7 +571,7 @@ int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff const struct ave_params *p = &sess->p; const struct ave_proc_layout *l = p->codec == AVE_CODEC_HEVC ? &ave_proc_hevc : &ave_proc_h264; - const struct firmware *tmpl = ave->tmpl[p->codec - 1][AVE_TMPL_PROCESS]; + size_t len = p->codec == AVE_CODEC_HEVC ? AVE_PROCESS_HEVC_BYTES : AVE_PROCESS_H264_BYTES; struct ave_slot *s = &sess->slot[slot]; unsigned int buf = AVE_BUF_PROC(sess->index * AVE_MAX_SLOTS + slot); dma_addr_t y = input; @@ -582,10 +582,10 @@ int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff ret = ave_ipc_wait_ack(ave, buf); if (ret) return ret; - cmd = ave_ipc_cmd(ave, buf, tmpl->size); + cmd = ave_ipc_cmd(ave, buf, len); if (!cmd) return -EBUSY; - memcpy(cmd, tmpl->data, tmpl->size); + ave_image_process(cmd, len, p->codec); ave_cmd_header(ctx, cmd, AVE_OP_PROCESS, AVE_PROC_SUBTYPE + slot, 0, ave_session_arg2(ctx)); @@ -631,9 +631,12 @@ int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff ave_put64(cmd, l->pts, pts); ave_put32(cmd, l->force_key, force_key); ave_put32(cmd, l->force_type, force_key ? AVE_PROC_FORCED_TYPE : 0); - ave_sess_check_addrs(ctx, cmd, tmpl->size, AVE_TMPL_PROCESS, y); + if (l->low_delay) + ave_put32(cmd, l->low_delay, p->bframes ? 0 : AVE_PROC_LOW_DELAY); + if (!sess->frames) + ave_debugfs_save(ave, p->codec, AVE_IMAGE_PROCESS, cmd, len); - ret = ave_ipc_submit(ave, buf, tmpl->size, AVE_CMD_SESSION_TIMEOUT_US, false); + ret = ave_ipc_submit(ave, buf, len, AVE_CMD_SESSION_TIMEOUT_US, false); if (ret) return ret; s->src = src; diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c index 6d67ddf3780e1..b25ac9df48a67 100644 --- a/drivers/media/platform/apple/ave/ave-drv.c +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -9,6 +9,7 @@ * * Copyright The Aurora Silicon Contributors */ +#include #include #include #include @@ -21,6 +22,51 @@ #define AVE_AUTOSUSPEND_MS 2000 +/* + * debugfs: apple-ave/last_start_{h264,hevc} hold the last Start built and + * last_process0_{h264,hevc} the first Process of the last session, so the + * generated commands can be compared with reference captures. + */ +static const size_t ave_image_bytes[2][AVE_IMAGE_COUNT] = { + { AVE_START_H264_BYTES, AVE_PROCESS_H264_BYTES }, + { AVE_START_HEVC_BYTES, AVE_PROCESS_HEVC_BYTES }, +}; + +void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, enum ave_image kind, + const void *cmd, size_t len) +{ + void *dst = ave->dump[codec - 1][kind]; + + if (dst && len == ave_image_bytes[codec - 1][kind]) + memcpy(dst, cmd, len); +} + +static void ave_debugfs_init(struct ave_dev *ave) +{ + static const char * const names[2][AVE_IMAGE_COUNT] = { + { "last_start_h264", "last_process0_h264" }, + { "last_start_hevc", "last_process0_hevc" }, + }; + + if (!IS_ENABLED(CONFIG_DEBUG_FS)) + return; + ave->debugfs = debugfs_create_dir("apple-ave", NULL); + for (unsigned int c = 0; c < 2; c++) + for (unsigned int k = 0; k < AVE_IMAGE_COUNT; k++) { + struct debugfs_blob_wrapper *blob; + + blob = devm_kzalloc(ave->dev, sizeof(*blob), GFP_KERNEL); + ave->dump[c][k] = devm_kzalloc(ave->dev, ave_image_bytes[c][k], GFP_KERNEL); + if (!blob || !ave->dump[c][k]) { + ave->dump[c][k] = NULL; + continue; + } + blob->data = ave->dump[c][k]; + blob->size = ave_image_bytes[c][k]; + debugfs_create_blob(names[c][k], 0400, ave->debugfs, blob); + } +} + static const char * const ave_pd_names[] = { "dma", "pipe4", "pipe5", "me0", "me1" }; static int ave_probe(struct platform_device *pdev) @@ -78,6 +124,7 @@ static int ave_probe(struct platform_device *pdev) ave_fw_remove(ave); return ret; } + ave_debugfs_init(ave); return 0; } @@ -85,6 +132,7 @@ static void ave_remove(struct platform_device *pdev) { struct ave_dev *ave = platform_get_drvdata(pdev); + debugfs_remove_recursive(ave->debugfs); ave_v4l2_unregister(ave); /* @@ -172,11 +220,5 @@ static struct platform_driver ave_driver = { }; module_platform_driver(ave_driver); -/* Session images, extracted from the user's macOS installation */ -MODULE_FIRMWARE("apple/ave-j700-h264-start.bin"); -MODULE_FIRMWARE("apple/ave-j700-h264-process.bin"); -MODULE_FIRMWARE("apple/ave-j700-hevc-start.bin"); -MODULE_FIRMWARE("apple/ave-j700-hevc-process.bin"); - MODULE_DESCRIPTION("Apple AVE video encoder"); MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c index 5db179711c0a1..4987a32ad5d42 100644 --- a/drivers/media/platform/apple/ave/ave-fw.c +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -21,7 +20,6 @@ #include #include #include -#include #include #include "ave.h" @@ -106,21 +104,6 @@ static u32 ave_fw_crc(const struct ave_fw_segment *seg, void **copy) return crc; } -/* - * The session images are per machine: "j700" from "apple,j700", the first - * root compatible. - */ -static void ave_fw_board_name(struct ave_dev *ave) -{ - const char *compat = NULL, *comma; - - of_property_read_string_index(of_root, "compatible", 0, &compat); - if (!compat) - compat = "unknown"; - comma = strchr(compat, ','); - strscpy(ave->board, comma ? comma + 1 : compat, sizeof(ave->board)); -} - /* * Find the firmware, check that it is a build we speak to and that DATA is * still as the bootloader left it, and keep a copy of DATA for restarts. @@ -151,8 +134,6 @@ int ave_fw_probe(struct ave_dev *ave) return dev_err_probe(dev, -ENODEV, "unsupported firmware (text CRC %08x)\n", text_crc); ave->text_crc = text_crc; - mutex_init(&ave->tmpl_lock); - ave_fw_board_name(ave); ave->data_pristine = vmalloc(ave->data.size); if (!ave->data_pristine) @@ -226,67 +207,20 @@ int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, unsigned long } /* - * Start and Process are mostly constant for a machine and its firmware. - * Those constant words come from image files named after the machine, - * which the user extracts from their own macOS installation; the driver - * fills in every session and frame field itself. + * The driver generates Start and Process for one firmware build; other + * builds may lay them out differently, so they are not driven. */ -static const char * const ave_codec_name[] = { "h264", "hevc" }; -static const char * const ave_tmpl_name[AVE_TMPL_COUNT] = { "start", "process" }; -static const size_t ave_tmpl_size[2][AVE_TMPL_COUNT] = { - { 0x3180, 0x63d8 }, /* H.264 */ - { 0x13f28, 0xb1c0 }, /* HEVC */ -}; - -#define AVE_TMPL_CODEC 0xc0 /* Start: codec word of the session image */ - -int ave_fw_templates(struct ave_dev *ave, enum ave_codec codec) +int ave_fw_check_build(struct ave_dev *ave) { - unsigned int c = codec - 1; - char name[64]; - int ret = 0; - - mutex_lock(&ave->tmpl_lock); - for (unsigned int t = 0; t < AVE_TMPL_COUNT && !ret; t++) { - const struct firmware *fw; - - if (ave->tmpl[c][t]) - continue; - snprintf(name, sizeof(name), "apple/ave-%s-%s-%s.bin", ave->board, - ave_codec_name[c], ave_tmpl_name[t]); - ret = request_firmware(&fw, name, ave->dev); - if (ret) { - dev_err(ave->dev, "cannot load %s (%d); extract it from macOS first\n", - name, ret); - break; - } - if (fw->size != ave_tmpl_size[c][t]) { - dev_err(ave->dev, "%s is %zu bytes, expected %zu\n", name, fw->size, - ave_tmpl_size[c][t]); - release_firmware(fw); - ret = -EINVAL; - break; - } - if (t == AVE_TMPL_START && get_unaligned_le32(fw->data + AVE_TMPL_CODEC) != codec) { - dev_err(ave->dev, "%s is not a %s session image\n", name, - ave_codec_name[c]); - release_firmware(fw); - ret = -EINVAL; - break; - } - ave->tmpl[c][t] = fw; - } - mutex_unlock(&ave->tmpl_lock); - return ret; + if (ave->text_crc == AVE_FW_TEXT_CRC_9003_78_0) + return 0; + dev_err(ave->dev, "encoder firmware %08x is not validated for encoding (only %08x is)\n", + ave->text_crc, AVE_FW_TEXT_CRC_9003_78_0); + return -ENOTSUPP; } void ave_fw_remove(struct ave_dev *ave) { - for (unsigned int c = 0; c < 2; c++) - for (unsigned int t = 0; t < AVE_TMPL_COUNT; t++) { - release_firmware(ave->tmpl[c][t]); - ave->tmpl[c][t] = NULL; - } ave_free(ave, &ave->heap); ave_free(ave, &ave->log); ave_free(ave, &ave->boot_cfg); diff --git a/drivers/media/platform/apple/ave/ave-image.c b/drivers/media/platform/apple/ave/ave-image.c new file mode 100644 index 0000000000000..f2f15f3c3d83a --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-image.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder: constant words of the session commands + * + * macOS builds Start and Process in code, from the session settings and + * its own defaults; nothing is stored. Besides the fields the driver + * computes (sizes, geometry, profile, level, rate control, GOP, buffer + * addresses, per-frame values), every captured command on firmware + * AppleAVE2FW 9003.78.0 holds the same small set of constant words. They + * are listed here as facts, grouped by where they sit; for most of them + * the meaning is not known, only that macOS always sets that value. The + * commands are built in a zeroed buffer: these words first, then the + * computed fields. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include + +#include "ave.h" +#include "ave-regs.h" + +/* @count consecutive u32 words at @off, each set to @val. */ +struct ave_word { + u32 off; + u16 count; + u32 val; +}; + +#define W(o, v) { .off = (o), .count = 1, .val = (v) } +#define RUN(o, n, v) { .off = (o), .count = (n), .val = (v) } + +/* IEEE 754 doubles, high words only (the low words are zero) */ +#define F64_HI_1_0 0x3ff00000 /* 1.0 */ +#define F64_HI_0_5 0x3fe00000 /* 0.5 */ +#define F64_HI_M1_0 0xbff00000 /* -1.0 */ + +/* Session configuration words common to both codecs (Start +0x68..) */ +static const struct ave_word ave_start_common[] = { + /* First surface size: always the 0x12c000 header surface */ + W(0x50, 0x12c000), + W(0x94, 0x36), + W(0x214, 0x39d), + W(0x218, 0x18), + /* Rate-control doubles: 1.0, 0.5 (the 0.5 s VBV buffer), -1, -1 */ + W(0x28c, F64_HI_1_0), + W(0x294, F64_HI_0_5), + W(0x2c4, F64_HI_M1_0), + W(0x2cc, F64_HI_M1_0), + W(0x2d0, 1), + W(0x2e4, 1), + RUN(0x2f4, 3, 1), + RUN(0x338, 2, 0xffffffff), + /* Buffer set: metadata sizes of slots 0..2, +0x13d0 surface size */ + W(0xf58, 0xc000), + W(0xf68, 0xc000), + W(0xf78, 0xc000), + W(0x13d8, 0x4000), +}; + +static const struct ave_word ave_h264_start[] = { + /* Codec (1) and four flags after it */ + RUN(0xc0, 5, 1), + W(0x370, 2), + /* Codec settings block (+0x2580..): see also ave_start_codec_bytes() */ + W(0x258c, 3), + W(0x2594, 1), + W(0x2598, 8), /* bit depth */ + W(0x25a4, 0x100), + W(0x25cc, 1), + W(0x25d4, 1), + W(0x26dc, 4), + W(0x26f8, 0x15), + RUN(0x2704, 2, 1), + W(0x2734, 1), + W(0x2738, 0x10), + W(0x27c0, 8), + /* Sequence and picture parameter block (+0x2900..) */ + W(0x2940, 1), + W(0x2970, 0x100000), + RUN(0x2974, 239, 0x100010), + W(0x2d30, 0x10), + W(0x2d34, 1), + W(0x2d3c, 2), + RUN(0x2d54, 2, 1), + W(0x2d5c, 0x1000000), + W(0x2d60, 5), + W(0x2d68, 1), + RUN(0x2d6c, 3, 2), + W(0x2dc8, 1), + /* Two runs of -13 */ + RUN(0x3004, 7, 0xfffffff3), + RUN(0x302c, 7, 0xfffffff3), + W(0x3048, 1), + /* Execution-unit info */ + W(0x3150, 1), + W(0x3158, 1), +}; + +static const struct ave_word ave_hevc_start[] = { + /* Codec (2) and four flags after it */ + W(0xc0, 2), + RUN(0xc4, 4, 1), + W(0x370, 0x2000002), + W(0x1358, 0x4000), /* +0x1350 surface size, fixed for HEVC */ + /* Codec settings block (+0x2580..): see also ave_start_codec_bytes() */ + W(0x2588, 2), + W(0x2594, 1), + W(0x2598, 8), /* bit depth */ + W(0x25a4, 0x100), + W(0x25cc, 1), + W(0x25d4, 1), + W(0x26dc, 4), + RUN(0x26f8, 2, 0x15), + W(0x2700, 2), + RUN(0x2704, 2, 1), + W(0x2710, 1), + W(0x2734, 1), + W(0x2738, 0x10), + W(0x27c0, 8), + /* Profile, tier and level block, first copy (+0x2920..) */ + W(0x2920, 0x101), + W(0x292c, 1), + W(0x2934, 1), + W(0x295c, 0x1010001), + W(0x2964, 1), + RUN(0x2a8c, 7, 0x1010101), + W(0x2aa8, 0x101), + W(0x2abc, 0x1010101), + W(0x2ac0, 0x101), + RUN(0x2e38, 7, 4), + /* Timing pair 1001 / 24000, also at +0x74f0 */ + W(0x3298, 1001), + W(0x329c, 24000), + /* Sequence parameter block, second copy (+0x58f0..) */ + W(0x58f8, 1), + W(0x5900, 1), + W(0x5928, 0x1010001), + W(0x5930, 1), + W(0x5aac, 1), + W(0x5ae0, 7), /* log2_max_pic_order_cnt_lsb_minus4: 11-bit POC */ + RUN(0x5ae8, 7, 4), /* ends with sps_max_dec_pic_buffering_minus1 */ + W(0x5b40, 2), + W(0x5b48, 3), + W(0x5b4c, 1), + W(0x5b54, 1), + /* Picture parameter and slice defaults (+0x7400..) */ + W(0x7400, 0x100), + RUN(0x7404, 2, 7), + W(0x740c, 1), + W(0x7414, 1), + W(0x749c, 1), + W(0x74b0, 0x100), + W(0x74b4, 5), + W(0x74bc, 1), + RUN(0x74c0, 3, 2), + W(0x74f0, 1001), + W(0x74f4, 24000), + W(0x7500, 0x101), + W(0x7504, 1), + W(0x7508, 3), + W(0x750c, 5), + RUN(0x7510, 3, 0x17), + RUN(0x7524, 2, 0xbebb), + W(0x7558, 0x100), + /* Rate-control block (+0x9600..), with two runs of -13 */ + W(0x960c, 2), + RUN(0x9614, 7, 0xfffffff3), + RUN(0x9634, 7, 0xfffffff3), + W(0x9654, 0x100), + W(0x9e64, 0x10000), + W(0xb734, 2), + W(0xb774, 1), + /* Execution-unit info */ + RUN(0x13ef4, 2, 1), + W(0x13f00, 1), +}; + +/* + * HEVC with frame reordering off: a low-delay P reference structure (the + * reference picture sets hold the delta POCs -1 .. -4). All zero with + * B-frames. + */ +static const struct ave_word ave_hevc_start_low_delay[] = { + W(0xe120, 4), W(0xe154, 4), W(0xe17c, 0x01010101), + W(0xe1dc, 4), W(0xe1e4, 0x01010101), + W(0xe204, -1), W(0xe208, -2), W(0xe20c, -3), W(0xe210, -4), + W(0xe244, 1), W(0xe248, 2), W(0xe24c, 3), W(0xe250, 4), + W(0xe284, 4), W(0xe2b8, 1), W(0xe2e0, 1), W(0xe340, 1), W(0xe348, 1), + W(0xe368, -1), W(0xe3a8, 1), W(0xe3e8, 1), + W(0xe41c, 2), W(0xe444, 0x101), W(0xe4a4, 2), W(0xe4ac, 0x101), + W(0xe4cc, -1), W(0xe4d0, -2), + W(0xe50c, 1), W(0xe510, 2), W(0xe54c, 2), + W(0xe580, 3), W(0xe5a8, 0x10101), W(0xe608, 3), W(0xe610, 0x10101), + W(0xe630, -1), W(0xe634, -2), W(0xe638, -3), + W(0xe670, 1), W(0xe674, 2), W(0xe678, 3), W(0xe6b0, 3), +}; + +/* Process: the first-frame flag and per-frame values are computed. */ +static const struct ave_word ave_h264_process[] = { + /* Codec block */ + W(0x99c, 1), + W(0x9a8, 2), + W(0x9bc, 1), + W(0x9d4, 1), + W(0xeb4, 1), + W(0x12bc, 0xffffffff), + /* Picture-management block */ + W(0x29fc, 0xffffffff), + W(0x6238, 5), + W(0x6278, 14400000), /* presentation timescale, low word */ +}; + +static const struct ave_word ave_hevc_process[] = { + /* Codec block */ + W(0x9a8, 1), + W(0x9b8, 2), + W(0xab8, 0x10100), + W(0xb44, 0x10000), + RUN(0xb4c, 2, 5), + RUN(0xd70, 2, 0xffffffff), + W(0xd7c, 1), + /* Picture-management block */ + W(0x74cc, 0xffffffff), + W(0x74d0, 0x100), + W(0x74e0, 1), + W(0xad08, 5), + W(0xad3c, 0x403e0000), /* frame rate 30.0, rewritten per session */ + W(0xad48, 14400000), /* presentation timescale, low word */ + W(0xad68, 0xffffffff), +}; + +static void ave_put_words(u8 *cmd, size_t len, const struct ave_word *w, size_t n) +{ + for (size_t i = 0; i < n; i++) + for (unsigned int j = 0; j < w[i].count; j++) { + size_t off = w[i].off + 4 * j; + + if (WARN_ON_ONCE(off + 4 > len)) + return; + put_unaligned_le32(w[i].val, cmd + off); + } +} + +/* + * The 8 bytes at Start +0x2580 look like a DART address but are separate + * settings: byte 0 is 4 for HEVC, byte 3 is 1, byte 5 is 1 unless the + * client turned frame reordering off (H.264 only; HEVC keeps 1). + */ +static void ave_start_codec_bytes(u8 *cmd, enum ave_codec codec, bool reorder_off) +{ + cmd[0x2580] = codec == AVE_CODEC_HEVC ? 4 : 0; + cmd[0x2583] = 1; + cmd[0x2585] = codec == AVE_CODEC_H264 && reorder_off ? 0 : 1; +} + +void ave_image_start(u8 *cmd, size_t len, enum ave_codec codec, bool reorder_off, + bool hevc_low_delay) +{ + ave_put_words(cmd, len, ave_start_common, ARRAY_SIZE(ave_start_common)); + if (codec == AVE_CODEC_HEVC) { + ave_put_words(cmd, len, ave_hevc_start, ARRAY_SIZE(ave_hevc_start)); + if (hevc_low_delay) + ave_put_words(cmd, len, ave_hevc_start_low_delay, + ARRAY_SIZE(ave_hevc_start_low_delay)); + } else { + ave_put_words(cmd, len, ave_h264_start, ARRAY_SIZE(ave_h264_start)); + } + ave_start_codec_bytes(cmd, codec, reorder_off); +} + +void ave_image_process(u8 *cmd, size_t len, enum ave_codec codec) +{ + if (codec == AVE_CODEC_HEVC) + ave_put_words(cmd, len, ave_hevc_process, ARRAY_SIZE(ave_hevc_process)); + else + ave_put_words(cmd, len, ave_h264_process, ARRAY_SIZE(ave_h264_process)); +} diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index b96a84ad6d1a8..b3c12bc184dea 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -27,6 +27,10 @@ #define AVE_RC_VBR 4 #define AVE_RC_CBR 2 +/* Filler-data NAL unit types */ +#define AVE_H264_NAL_FILLER 12 +#define AVE_HEVC_NAL_FILLER 38 + #define AVE_H264_BASELINE 2 #define AVE_H264_MAIN 4 #define AVE_H264_HIGH 6 @@ -141,18 +145,14 @@ static int ave_sess_params(struct ave_ctx *ctx, struct ave_params *p) p->min_qp = p->max_qp; /* - * Reference structures as macOS sets them up. H.264 has all three - * (B-frames, P only, intra only). HEVC without B-frames needs a - * reference-set table whose contents are not known yet, so HEVC - * always uses B-frames. + * Reference structures as macOS sets them up: a B-frame pyramid, + * P frames only (low delay), or (H.264) intra only. */ p->intra_only = p->codec == AVE_CODEC_H264 && ctx->gop_size == 1; - p->bframes = p->codec == AVE_CODEC_HEVC || - (!baseline && !p->intra_only && ctx->b_frames); + p->bframes = !baseline && !p->intra_only && ctx->b_frames; + p->reorder_off = !ctx->b_frames; p->refs = p->intra_only ? 1 : p->bframes ? 5 : 2; p->slots = p->bframes ? 5 : 3; - /* HEVC asked to go without B-frames: flush every frame instead. */ - p->low_delay = p->codec == AVE_CODEC_HEVC && !ctx->b_frames; p->gop = ctx->gop_size ?: 120; p->bitrate = ctx->bitrate; @@ -584,8 +584,9 @@ static bool ave_starts_annexb(const u8 *p, size_t len) (len >= 4 && !p[0] && !p[1] && !p[2] && p[3] == 1); } -/* NAL unit types in an Annex B buffer: is there a key frame / parameter set? */ -static void ave_scan_nals(const u8 *p, size_t len, bool hevc, bool *key, bool *params) +/* NAL unit types in an Annex B buffer: key frame, parameter sets, filler? */ +static void ave_scan_nals(const u8 *p, size_t len, bool hevc, bool *key, bool *params, + bool *filler) { for (size_t i = 0; i + 3 < len; i++) { u32 type; @@ -596,9 +597,11 @@ static void ave_scan_nals(const u8 *p, size_t len, bool hevc, bool *key, bool *p if (hevc) { *key |= type >= 16 && type <= 21; *params |= type >= 32 && type <= 34; + *filler |= type == AVE_HEVC_NAL_FILLER; } else { *key |= type == 5; *params |= type == 7; + *filler |= type == AVE_H264_NAL_FILLER; } i += 2; } @@ -649,6 +652,65 @@ static void ave_sess_dump(struct ave_ctx *ctx, struct ave_slot *s, const char *w dev_dbg(ctx->ave->dev, "metadata +%#x: %16ph\n", i, meta + i); } +/* + * CBR padding. macOS streams in CBR mode follow a buffer model: each frame + * has a budget of bitrate / (8 x fps) bytes, and once the bytes left unused + * add up to more than 0.5 s of bit rate (the 0.5 s VBV buffer of Start + * +0x290), every frame is padded with a filler-data NAL unit so the surplus + * stays there. Whether the firmware or macOS user space writes that filler + * is not known; if the firmware's output never carries filler, the driver + * adds it here. + */ +#define AVE_FILLER_OVERHEAD 7 /* start code, NAL header, trailing bits */ + +static size_t ave_sess_cbr_fill(struct ave_ctx *ctx, u8 *out, size_t len, size_t room, + bool fw_filler) +{ + struct ave_session *sess = ctx->sess; + const struct ave_params *p = &sess->p; + bool hevc = p->codec == AVE_CODEC_HEVC; + s64 budget = div_u64((u64)p->bitrate * p->fps_den, 8ULL * p->fps_num); + s64 vbv = p->bitrate / 16; /* 0.5 s of bit rate, in bytes */ + s64 excess, pad; + u8 *f; + + if (p->rc_mode != AVE_RC_CBR) + return len; + if (fw_filler && !sess->fw_fills) { + sess->fw_fills = true; + dev_dbg(ctx->ave->dev, "CBR: the firmware writes filler data\n"); + } + excess = sess->cbr_surplus + budget - (s64)len - vbv; + if (!sess->fw_fills && excess > AVE_FILLER_OVERHEAD) { + pad = min_t(s64, excess, room - len); + if (pad > AVE_FILLER_OVERHEAD) { + f = out + len; + f[0] = 0; + f[1] = 0; + f[2] = 0; + f[3] = 1; + if (hevc) { + /* nal_unit_type 38, layer 0, temporal id 0 + 1 */ + f[4] = AVE_HEVC_NAL_FILLER << 1; + f[5] = 1; + memset(f + 6, 0xff, pad - AVE_FILLER_OVERHEAD); + } else { + f[4] = AVE_H264_NAL_FILLER; + memset(f + 5, 0xff, pad - AVE_FILLER_OVERHEAD + 1); + } + f[pad - 1] = 0x80; /* rbsp_stop_one_bit */ + len += pad; + if (!sess->host_fill_noted) { + sess->host_fill_noted = true; + dev_dbg(ctx->ave->dev, + "CBR: no filler from the firmware, the driver adds it\n"); + } + } + } + sess->cbr_surplus += budget - (s64)len; + return len; +} + /* * Assemble the access unit of @slot into @dst: the parameter sets the * firmware leaves in the metadata buffer (when it emits them), for HEVC the @@ -666,7 +728,7 @@ int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff size_t room = vb2_plane_size(&dst->vb2_buf, 0); const u8 *meta = s->meta.cpu; u32 data, data2, shdr = 0; - bool params = false; + bool params = false, filler = false; ssize_t ps = 0; size_t len; int at; @@ -707,80 +769,14 @@ int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff if (!ave_starts_annexb(out, len)) ave_sess_dump(ctx, s, "not Annex B"); - ave_scan_nals(out, len, hevc, keyframe, ¶ms); + ave_scan_nals(out, len, hevc, keyframe, ¶ms, &filler); if (!ctx->dst_sequence && !params) ave_sess_dump(ctx, s, "first frame has no parameter sets"); + len = ave_sess_cbr_fill(ctx, out, len, room, filler); vb2_set_plane_payload(&dst->vb2_buf, 0, len); return 0; } -/* - * Every 64-bit word in a command that looks like an encoder DART address - * should be one of ours. One that is not came from the session image (a - * stale macOS address) and makes the firmware write to an unmapped page. - * Reported once per session and command kind. - */ -static bool ave_buf_has(const struct ave_buf *b, u64 addr) -{ - return b->cpu && addr >= b->dma && addr < b->dma + b->alloc; -} - -static bool ave_sess_owns(struct ave_session *sess, u64 addr) -{ - const struct ave_buf *fixed[] = { - &sess->hdr, &sess->mb, &sess->row_a, &sess->row_b, &sess->fixed, - &sess->hevc_extra[0], &sess->hevc_extra[1], - }; - - for (unsigned int i = 0; i < ARRAY_SIZE(fixed); i++) - if (ave_buf_has(fixed[i], addr)) - return true; - for (unsigned int i = 0; i < AVE_MAX_REFS; i++) - if (ave_buf_has(&sess->ref_a[i], addr) || ave_buf_has(&sess->ref_b[i], addr) || - ave_buf_has(&sess->ref_c[i], addr) || ave_buf_has(&sess->ref_d[i], addr)) - return true; - for (unsigned int i = 0; i < AVE_TILE_BUFS; i++) - if (ave_buf_has(&sess->tile[i], addr) || ave_buf_has(&sess->tile_aux[i], addr)) - return true; - for (unsigned int s = 0; s < AVE_MAX_SLOTS; s++) - if (ave_buf_has(&sess->slot[s].coded, addr) || - ave_buf_has(&sess->slot[s].meta, addr) || - ave_buf_has(&sess->slot[s].aux, addr)) - return true; - return false; -} - -void ave_sess_check_addrs(struct ave_ctx *ctx, const u8 *cmd, size_t len, unsigned int kind, - dma_addr_t input) -{ - struct ave_session *sess = ctx->sess; - struct ave_dev *ave = ctx->ave; - unsigned int bad = 0; - - if (sess->checked[kind]) - return; - sess->checked[kind] = true; - for (size_t off = AVE_CMD_HDR_BYTES; off + 8 <= len; off += 4) { - u64 v = get_unaligned_le64(cmd + off); - - /* The encoder's DART window: 0x100_0000_0000 + 256 MiB. */ - if ((v >> 28) != 0x1000) - continue; - if (ave_sess_owns(sess, v) || (input && v >= input && v < input + SZ_32M) || - (v >= ave->ipc.dma && v < ave->ipc.dma + ave->ipc.size)) - continue; - /* Pointers into the firmware image are the same as on macOS. */ - if (v >= ave->text.iova && v < ave->data.iova + ave->data.size) - continue; - if (bad++ < 16) - dev_warn(ave->dev, "%s +%#zx holds %#llx, not a buffer of this session\n", - kind == AVE_TMPL_START ? "Start" : "Process", off, v); - } - if (bad) - dev_warn(ave->dev, "%u stale addresses in the %s image\n", bad, - kind == AVE_TMPL_START ? "Start" : "Process"); -} - /* * Frames are outstanding, nothing came back for a while, and the firmware * has enough frames that it should have answered: without B-frames every diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index caa874ba48de3..3d6b5fa2352b4 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -478,8 +478,7 @@ static int ave_init_ctrls(struct ave_ctx *ctx) v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME, 0, 0, 0, 0); /* * The firmware either uses up to three B-frames (reordering them - * itself) or none. Baseline and intra-only H.264 never have any; - * HEVC always has them for now. + * itself) or none. Baseline and intra-only H.264 never have any. */ v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_B_FRAMES, 0, 3, 3, 3); v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_HEADER_MODE, @@ -740,7 +739,7 @@ static int ave_session_begin(struct ave_ctx *ctx) if (ctx->sess) return 0; - ret = ave_fw_templates(ave, ave_ctx_codec(ctx)); + ret = ave_fw_check_build(ave); if (ret) return ret; if (READ_ONCE(ave->sessions[0]) && READ_ONCE(ave->sessions[1])) @@ -1031,14 +1030,6 @@ static int ave_encode_one(struct ave_ctx *ctx) ret = ave_ipc_settle(ave, AVE_FRAME_WAIT_MS); if (ret) return ret; - } else if (sess->p.low_delay) { - /* Make the firmware code this frame now, without lookahead. */ - ret = ave_cmd_complete(ctx, 1); - sess->nudged = true; - if (!ret) - ret = ave_ipc_pump(ave, 0, NULL, NULL); - if (ret) - return ret; } else { ret = ave_ipc_pump(ave, 0, NULL, NULL); if (ret) diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h index c5dce46faa373..0f6e2b38d86ca 100644 --- a/drivers/media/platform/apple/ave/ave.h +++ b/drivers/media/platform/apple/ave/ave.h @@ -19,8 +19,9 @@ #include #include +struct debugfs_blob_wrapper; +struct dentry; struct dev_pm_domain_list; -struct firmware; struct iommu_domain; /* Frames the firmware can hold at once per session (Process "slots"). */ @@ -93,13 +94,21 @@ enum ave_fw_state { AVE_FW_BROKEN, }; -/* Session command images that only differ from the listed fields. */ -enum ave_tmpl { - AVE_TMPL_START, - AVE_TMPL_PROCESS, - AVE_TMPL_COUNT, +/* The two large session commands, and their sizes per codec */ +enum ave_image { + AVE_IMAGE_START, + AVE_IMAGE_PROCESS, + AVE_IMAGE_COUNT, }; +#define AVE_START_H264_BYTES 0x3180 +#define AVE_START_HEVC_BYTES 0x13f28 +#define AVE_PROCESS_H264_BYTES 0x63d8 +#define AVE_PROCESS_HEVC_BYTES 0xb1c0 + +/* The firmware build whose command layouts the driver generates */ +#define AVE_FW_TEXT_CRC_9003_78_0 0x69c54454 + struct ave_ctx; struct ave_dev { @@ -113,13 +122,12 @@ struct ave_dev { struct ave_fw_segment text; struct ave_fw_segment data; u32 text_crc; - char board[16]; /* from the root compatible, e.g. "j700" */ void *data_pristine; bool data_dirty; - /* Protects tmpl[]; the images stay loaded until the driver goes. */ - struct mutex tmpl_lock; - const struct firmware *tmpl[2][AVE_TMPL_COUNT]; + /* debugfs: the last Start and first Process built, per codec */ + struct dentry *debugfs; + void *dump[2][AVE_IMAGE_COUNT]; /* Serialises every firmware and ring access. */ struct mutex hw_lock; @@ -161,7 +169,7 @@ struct ave_params { u32 refs; /* max_num_ref_frames + 1 */ bool bframes; bool intra_only; - bool low_delay; /* Complete after every frame */ + bool reorder_off; /* the client asked for no B-frames */ u32 gop; u32 rc_mode; u32 bitrate; @@ -212,6 +220,10 @@ struct ave_session { bool fw_open; /* Open sent, Close not yet */ bool resume; /* closed for system sleep, reopen on next frame */ bool copy_noted; + /* CBR: bytes under budget so far, and who pads (see ave_sess_cbr_fill()) */ + s64 cbr_surplus; + bool fw_fills; + bool host_fill_noted; bool warned; bool checked[2]; /* Start/Process images checked for stale addresses */ bool handed; /* Start sent: the firmware knows the buffers */ @@ -266,7 +278,7 @@ int ave_fw_ensure_running(struct ave_dev *ave); void ave_fw_halt(struct ave_dev *ave); int ave_fw_shutdown(struct ave_dev *ave); void ave_fw_lost(struct ave_dev *ave); -int ave_fw_templates(struct ave_dev *ave, enum ave_codec codec); +int ave_fw_check_build(struct ave_dev *ave); int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, unsigned long attrs); int ave_alloc_padded(struct ave_dev *ave, struct ave_buf *b, size_t size, size_t alloc, unsigned long attrs); @@ -303,11 +315,18 @@ int ave_sess_input(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffe int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *dst, bool *keyframe); void ave_sess_prepare_slot(struct ave_session *sess, unsigned int slot); -void ave_sess_check_addrs(struct ave_ctx *ctx, const u8 *cmd, size_t len, unsigned int kind, - dma_addr_t input); bool ave_sess_stalled(struct ave_session *sess); bool ave_sess_starved(struct ave_session *sess); +/* ave-image.c */ +void ave_image_start(u8 *cmd, size_t len, enum ave_codec codec, bool reorder_off, + bool hevc_low_delay); +void ave_image_process(u8 *cmd, size_t len, enum ave_codec codec); + +/* ave-drv.c */ +void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, enum ave_image kind, + const void *cmd, size_t len); + /* ave-v4l2.c */ void ave_v4l2_suspend(struct ave_dev *ave); int ave_v4l2_register(struct ave_dev *ave); From 6a99b550f9055eeb643d5ceebfecc5de6b807a6a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 22:16:08 +1100 Subject: [PATCH 1398/1579] media: apple: ave: fit the native panel size in the DMA window At 2408x1506 every H.264 setup and HEVC with B-frames ran out of the encoder's 256 MiB DART window. Three changes make room: - The H.264 per-frame buffer of Process +0x58c8 was sized after the macOS slot spacing (0x364000 at 1080p, scaled with the picture). The firmware never wrote a byte of it in any bit-rate session on hardware, so it is now one page. - Session buffers are mapped largest first. The DMA layer aligns every mapping to a power of two, and the smaller buffers now fill the gaps the large ones leave. The debug log reports how much a session maps, and an allocation failure says how far it got. - ffmpeg's lines are spread out in place when the OUTPUT buffer is one the driver allocated (MMAP), which is always large enough for the encoder's layout. That needs no extra DMA space; imported buffers still get a per-slot copy. The OUTPUT and CAPTURE queues get 48 and 24 MiB. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/media/platform/apple/ave/ave-sess.c | 139 ++++++++++++++------ drivers/media/platform/apple/ave/ave-v4l2.c | 4 +- 2 files changed, 102 insertions(+), 41 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index b3c12bc184dea..27ea45c76080f 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -48,9 +49,8 @@ #define AVE_CODED_MIN 0x3c000 #define AVE_HDR_BYTES 0x12c000 -/* Spacing of the H.264 per-frame buffer seen at 1080p; scaled with area. */ -#define AVE_H264_FRAME_BUF_1080P 0x364000 -#define AVE_MBS_1080P (120 * 68) +/* H.264 per-frame buffer (Process +0x58c8): unused in bit-rate modes */ +#define AVE_H264_FRAME_BUF SZ_16K /* Level limits: max frame size, max rate, max bit rate (kbit/s, Main). */ struct ave_level { @@ -198,12 +198,37 @@ static int ave_sess_params(struct ave_ctx *ctx, struct ave_params *p) * driver can see how far past each reported size the firmware wrote (into * the page rounding, or the room the two groups above get). */ -static int ave_work_buf(struct ave_dev *ave, struct ave_buf *b, size_t size) + +/* One buffer to allocate: what the firmware is told and what is mapped. */ +struct ave_buf_req { + struct ave_buf *b; + size_t size; + size_t alloc; +}; + +#define AVE_MAX_BUF_REQS (1 + 4 * AVE_MAX_REFS + 4 + 2 * AVE_TILE_BUFS + 2 + \ + 3 * AVE_MAX_SLOTS) + +static int ave_buf_req_cmp(const void *a, const void *b) +{ + const struct ave_buf_req *x = a, *y = b; + + return x->alloc < y->alloc ? 1 : x->alloc > y->alloc ? -1 : 0; +} + +static void ave_req(struct ave_buf_req *r, unsigned int *n, struct ave_buf *b, size_t size, + size_t alloc) { - return ave_alloc(ave, b, size, 0); + r[*n] = (struct ave_buf_req){ .b = b, .size = size, .alloc = max(size, alloc) }; + (*n)++; } -/* Working buffer sizes, all from the requested picture size. */ +/* + * Working buffer sizes, all from the requested picture size. The DMA layer + * aligns each mapping to a power of two in the encoder's 256 MiB window, so + * the buffers are mapped largest first: the smaller ones then fill the + * alignment gaps the large ones leave. + */ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) { const struct ave_params *p = &sess->p; @@ -211,47 +236,63 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) u32 w = p->width, h = p->height; u32 w16 = DIV_ROUND_UP(w, 16), w32 = DIV_ROUND_UP(w, 32); u32 h32 = DIV_ROUND_UP(h, 32), w64 = DIV_ROUND_UP(w, 64), h64 = DIV_ROUND_UP(h, 64); - size_t ref_a, ref_b, ref_c, ref_d, tile, frame_buf; + size_t ref_a, ref_b, ref_c, ref_d, tile, mapped = 0; + struct ave_buf_req *r; + unsigned int n = 0; int ret = 0; + r = kcalloc(AVE_MAX_BUF_REQS, sizeof(*r), GFP_KERNEL); + if (!r) + return -ENOMEM; + ref_a = ALIGN(w, 32) * ALIGN(h + 16, 32); ref_b = roundup_pow_of_two(w) * roundup_pow_of_two(h) / 32; ref_c = hevc ? 512 * w64 * h64 : ALIGN(w, 64) * (ALIGN(h, 16) / 2 + 24); ref_d = ALIGN(hevc ? 64 * w32 * h32 : 128 * p->mbs, SZ_16K); tile = hevc ? 2304 * w32 * h64 : ALIGN(w, 16) * ALIGN(h, 64); - ret |= ave_work_buf(ave, &sess->hdr, AVE_HDR_BYTES); + ave_req(r, &n, &sess->hdr, AVE_HDR_BYTES, 0); for (unsigned int i = 0; i < p->refs; i++) { - ret |= ave_alloc_padded(ave, &sess->ref_a[i], ref_a, ref_a * 3 / 2, 0); - ret |= ave_alloc_padded(ave, &sess->ref_b[i], ref_b, ref_b * 5 / 4, 0); - ret |= ave_work_buf(ave, &sess->ref_c[i], ref_c); - ret |= ave_work_buf(ave, &sess->ref_d[i], ref_d); - } - ret |= ave_work_buf(ave, &sess->mb, ALIGN(82 * p->mbs, SZ_16K)); - ret |= ave_work_buf(ave, &sess->row_a, hevc ? SZ_16K : ALIGN(256 * w16, SZ_16K)); - ret |= ave_work_buf(ave, &sess->row_b, ALIGN(hevc ? 768 * w32 : 1024 * w16, SZ_16K)); - ret |= ave_work_buf(ave, &sess->fixed, SZ_16K); + ave_req(r, &n, &sess->ref_a[i], ref_a, ref_a * 3 / 2); + ave_req(r, &n, &sess->ref_b[i], ref_b, ref_b * 5 / 4); + ave_req(r, &n, &sess->ref_c[i], ref_c, 0); + ave_req(r, &n, &sess->ref_d[i], ref_d, 0); + } + ave_req(r, &n, &sess->mb, ALIGN(82 * p->mbs, SZ_16K), 0); + ave_req(r, &n, &sess->row_a, hevc ? SZ_16K : ALIGN(256 * w16, SZ_16K), 0); + ave_req(r, &n, &sess->row_b, ALIGN(hevc ? 768 * w32 : 1024 * w16, SZ_16K), 0); + ave_req(r, &n, &sess->fixed, SZ_16K, 0); for (unsigned int i = 0; i < (hevc ? 4 : AVE_TILE_BUFS); i++) { - ret |= ave_work_buf(ave, &sess->tile[i], tile); - ret |= ave_work_buf(ave, &sess->tile_aux[i], ALIGN(h, 128)); + ave_req(r, &n, &sess->tile[i], tile, 0); + ave_req(r, &n, &sess->tile_aux[i], ALIGN(h, 128), 0); } if (hevc) for (unsigned int i = 0; i < 2; i++) - ret |= ave_work_buf(ave, &sess->hevc_extra[i], ALIGN(w * h32, SZ_16K)); - - /* Per-slot output; the size of the H.264 frame buffer is not known. */ - frame_buf = ALIGN(DIV_ROUND_UP_ULL((u64)AVE_H264_FRAME_BUF_1080P * p->mbs, AVE_MBS_1080P), - SZ_16K); - frame_buf = max_t(size_t, frame_buf, AVE_H264_FRAME_BUF_1080P); + ave_req(r, &n, &sess->hevc_extra[i], ALIGN(w * h32, SZ_16K), 0); for (unsigned int s = 0; s < p->slots; s++) { - ret |= ave_alloc(ave, &sess->slot[s].coded, p->coded_cap, 0); - ret |= ave_alloc(ave, &sess->slot[s].meta, AVE_META_BYTES, 0); - if (hevc) - ret |= ave_alloc(ave, &sess->slot[s].aux, AVE_SHDR_BYTES, 0); - else - ret |= ave_work_buf(ave, &sess->slot[s].aux, frame_buf); - } - return ret ? -ENOMEM : 0; + ave_req(r, &n, &sess->slot[s].coded, p->coded_cap, 0); + ave_req(r, &n, &sess->slot[s].meta, AVE_META_BYTES, 0); + /* + * HEVC: slice headers. H.264: the per-frame buffer of Process + * +0x58c8, which the firmware never wrote in any bit-rate + * mode on hardware; one page. + */ + ave_req(r, &n, &sess->slot[s].aux, hevc ? AVE_SHDR_BYTES : AVE_H264_FRAME_BUF, 0); + } + + sort(r, n, sizeof(*r), ave_buf_req_cmp, NULL); + for (unsigned int i = 0; i < n && !ret; i++) { + ret = ave_alloc_padded(ave, r[i].b, r[i].size, r[i].alloc, 0); + mapped += r[i].b->alloc; + } + kfree(r); + if (ret) { + dev_err(ave->dev, "out of encoder DMA space after %zu MiB of session buffers\n", + mapped >> 20); + return -ENOMEM; + } + dev_dbg(ave->dev, "session buffers: %u, %zu KiB mapped\n", n, mapped >> 10); + return 0; } static void ave_sess_free_bufs(struct ave_dev *ave, struct ave_session *sess) @@ -281,6 +322,15 @@ static void ave_sess_free_bufs(struct ave_dev *ave, struct ave_session *sess) } } +static void ave_note_copy(struct ave_ctx *ctx, u32 stride, const char *how) +{ + if (ctx->sess->copy_noted) + return; + ctx->sess->copy_noted = true; + dev_info(ctx->ave->dev, "input has a %u byte stride; moving lines to %u %s\n", stride, + ctx->sess->p.stride, how); +} + /* * The input picture for @slot, as a DART address. The encoder reads NV12 * with the 64-byte aligned stride S_FMT reported and chroma right after @@ -320,20 +370,31 @@ int ave_sess_input(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffe return -EFAULT; } from += off; + + /* + * Buffers the driver allocated (MMAP) are large enough for the + * encoder's layout: spread the lines out in place, last line first, + * so no extra DMA space is needed. Imported buffers may be shared + * with other users and get a copy instead. + */ + if (vb->memory == VB2_MEMORY_MMAP && + vb2_plane_size(vb, 0) >= off + (size_t)p->stride * rows) { + to = (u8 *)from; + for (u32 y = rows; y-- > 0;) + memmove(to + y * p->stride, from + y * stride, width); + ave_note_copy(ctx, stride, "in place"); + return 0; + } + ret = ave_alloc(ctx->ave, b, p->stride * rows, 0); if (ret) { dev_err(ctx->ave->dev, "no room to copy %u byte stride input\n", stride); return ret; } - if (!sess->copy_noted) { - sess->copy_noted = true; - dev_info(ctx->ave->dev, "input has a %u byte stride; copying to %u\n", stride, - p->stride); - } - to = b->cpu; for (u32 y = 0; y < rows; y++) memcpy(to + y * p->stride, from + y * stride, width); + ave_note_copy(ctx, stride, "to a copy"); *input = b->dma; return 0; } diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index 3d6b5fa2352b4..b316489b7294d 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -40,8 +40,8 @@ * window next to a session: each queue may use this much, counting every * buffer at the power-of-two size the DMA layer maps it with. */ -#define AVE_OUTPUT_BUDGET SZ_64M -#define AVE_CAPTURE_BUDGET SZ_32M +#define AVE_OUTPUT_BUDGET (48 * SZ_1M) +#define AVE_CAPTURE_BUDGET (24 * SZ_1M) #define AVE_MAX_QUEUE_BUFS 16 #define AVE_RESTART_TRIES 10 #define AVE_RESTART_WAIT_MS 100 From ec7c3a7f7577ef3b564bffd1cfc3a44e5afa14ff Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 22:24:17 +1100 Subject: [PATCH 1399/1579] media: apple: ave: match macOS Start byte for byte, give GStreamer its buffers A byte comparison of the generated commands against macOS for the same settings (1080p30 Main, VBR 4 Mb/s, GOP 30, B-frames), with addresses, tokens and timestamps masked, found both Process commands identical and five Start words different: - H.264 +0x3020: byte +0x3020 is 1. The word was not in the constant list because its byte +0x3023 (8x8 transform) varies. - HEVC +0x2960 and +0x592c: the bytes around the computed max-8-bit flags are 1 (+0x2960, +0x2961, +0x2963 and +0x592f). - HEVC +0x270c = 2 and +0x9608 = 0x10000: the rate-control words that only change in constant-quality mode, at their bit-rate mode values. GStreamer stopped after five frames at 2408x1506: the v14 queue limits left it 6 OUTPUT and 3 CAPTURE buffers there. Sessions map much less now, so the queues get 80 and 40 MiB of the DMA window again (10 and 5 buffers at that size), and a trimmed or refused buffer request is logged at debug level. Field values are facts from our private research (macOS traces) and our own hardware runs; no copied code. Assisted-by: LLM Signed-off-by: Ryan Murray Co-Authored-By: Claude Opus 5.5 --- drivers/media/platform/apple/ave/ave-cmd.c | 7 +++++++ drivers/media/platform/apple/ave/ave-image.c | 6 ++++++ drivers/media/platform/apple/ave/ave-v4l2.c | 12 +++++++++--- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-cmd.c b/drivers/media/platform/apple/ave/ave-cmd.c index e453700693f5d..7a03ed53b457b 100644 --- a/drivers/media/platform/apple/ave/ave-cmd.c +++ b/drivers/media/platform/apple/ave/ave-cmd.c @@ -292,6 +292,9 @@ static const u16 ave_start_tile_off[AVE_TILE_BUFS] = { /* HEVC parameter-set words; most appear twice */ #define AVE_START_HEVC_PAD 0x26e0 +#define AVE_START_HEVC_RC_270C 0x270c /* 2 in bit-rate modes, 5 in CQ */ +#define AVE_START_HEVC_RC_9604 0x9604 /* 0 in bit-rate modes, 2 in CQ */ +#define AVE_START_HEVC_RC_9608 0x9608 /* 0x10000 in bit-rate modes, 0 in CQ */ #define AVE_START_HEVC_PROFILE_A 0x2938 #define AVE_START_HEVC_COMPAT_A 0x293d /* two bytes */ #define AVE_START_HEVC_8BIT_A 0x2962 /* byte */ @@ -450,6 +453,10 @@ static void ave_start_config(const struct ave_params *p, u8 *cmd) cmd[AVE_START_HEVC_COMPAT_B + 1] = 1; cmd[AVE_START_HEVC_8BIT_A] = 1; cmd[AVE_START_HEVC_8BIT_B] = 1; + /* Rate-control words that only change in constant-quality mode */ + ave_put32(cmd, AVE_START_HEVC_RC_270C, 2); + ave_put32(cmd, AVE_START_HEVC_RC_9604, 0); + ave_put32(cmd, AVE_START_HEVC_RC_9608, 0x10000); ave_put32(cmd, AVE_START_HEVC_LEVEL_A, p->level); ave_put32(cmd, AVE_START_HEVC_LEVEL_B, p->level); ave_put32(cmd, AVE_START_HEVC_WIDTH, p->cw); diff --git a/drivers/media/platform/apple/ave/ave-image.c b/drivers/media/platform/apple/ave/ave-image.c index f2f15f3c3d83a..273dd2a69b013 100644 --- a/drivers/media/platform/apple/ave/ave-image.c +++ b/drivers/media/platform/apple/ave/ave-image.c @@ -88,6 +88,8 @@ static const struct ave_word ave_h264_start[] = { W(0x2d68, 1), RUN(0x2d6c, 3, 2), W(0x2dc8, 1), + /* Picture parameters: byte +0x3020 is 1; +0x3023 (8x8 transform) is computed */ + W(0x3020, 1), /* Two runs of -13 */ RUN(0x3004, 7, 0xfffffff3), RUN(0x302c, 7, 0xfffffff3), @@ -123,6 +125,8 @@ static const struct ave_word ave_hevc_start[] = { W(0x292c, 1), W(0x2934, 1), W(0x295c, 0x1010001), + /* Constraint flag bytes; +0x2962 (max 8 bit) is computed */ + W(0x2960, 0x01000101), W(0x2964, 1), RUN(0x2a8c, 7, 0x1010101), W(0x2aa8, 0x101), @@ -136,6 +140,8 @@ static const struct ave_word ave_hevc_start[] = { W(0x58f8, 1), W(0x5900, 1), W(0x5928, 0x1010001), + /* Constraint flag bytes, second copy; +0x592e (max 8 bit) is computed */ + W(0x592c, 0x01000000), W(0x5930, 1), W(0x5aac, 1), W(0x5ae0, 7), /* log2_max_pic_order_cnt_lsb_minus4: 11-bit POC */ diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index b316489b7294d..0adbaaf8513d5 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -40,8 +40,8 @@ * window next to a session: each queue may use this much, counting every * buffer at the power-of-two size the DMA layer maps it with. */ -#define AVE_OUTPUT_BUDGET (48 * SZ_1M) -#define AVE_CAPTURE_BUDGET (24 * SZ_1M) +#define AVE_OUTPUT_BUDGET (80 * SZ_1M) +#define AVE_CAPTURE_BUDGET (40 * SZ_1M) #define AVE_MAX_QUEUE_BUFS 16 #define AVE_RESTART_TRIES 10 #define AVE_RESTART_WAIT_MS 100 @@ -545,8 +545,14 @@ static int ave_queue_setup(struct vb2_queue *vq, unsigned int *nbufs, unsigned i if (V4L2_TYPE_IS_OUTPUT(vq->type) && have + *nbufs < AVE_MAX_SLOTS + 1) *nbufs = AVE_MAX_SLOTS + 1 - have; /* A full set of both queues must fit the encoder's 256 MiB DART window. */ - if (have >= max) + if (have >= max) { + dev_dbg(ctx->ave->dev, "%s queue already has its %u buffers\n", + V4L2_TYPE_IS_OUTPUT(vq->type) ? "OUTPUT" : "CAPTURE", max); return -ENOMEM; + } + if (*nbufs > max - have) + dev_dbg(ctx->ave->dev, "%s queue: %u buffers asked, %u given\n", + V4L2_TYPE_IS_OUTPUT(vq->type) ? "OUTPUT" : "CAPTURE", *nbufs, max - have); *nbufs = min(*nbufs, max - have); if (*nplanes) return (*nplanes != 1 || sizes[0] < f->plane_fmt[0].sizeimage) ? -EINVAL : 0; From a149ba924449885e01d540ddcf9da5633b7b6f64 Mon Sep 17 00:00:00 2001 From: Ryan <81541646+RyanTheTide@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:42:23 +1100 Subject: [PATCH 1400/1579] soc: apple: mailbox: Do not sleep past free FIFO space (#630) A sleeping sender that finds the send FIFO full waits on tx_empty for the send-empty interrupt, and gives up after 500 ms. Two things make it miss free space and time out with room in the FIFO: - The interrupt reports an empty FIFO. A firmware that takes some of the queued messages but not all of them frees space without raising it. - Two senders can wait at once, for example a driver starting its endpoints while the RTKit work item acknowledges a syslog message. The handler masks the interrupt and calls complete(), which wakes only one of them. The other sleeps on with the interrupt masked. A sender that timed out also masked it, stranding any sender still waiting. On the J616s (M4 Pro) this delayed the AOP start on some warm boots. One endpoint start failed 504 ms after it was issued, while the AOP firmware was logging over syslog: 2.891389 aop: ep 0x28 init sent 3.395337 aop: Failed to send management message type 0x05: -110 3.395341 aop: Retrying unsent endpoint start 0x29 3.395354 aop: ep 0x2b init sent 3.402107 aop: syslog message: ... AOP IS READY The retry and the next three requests went out within 20 us, so the FIFO had room when the first send gave up, and the firmware reported ready 7 ms later. In an earlier boot the same stall made an AFK init time out after the firmware had reported ready, and the probe failed. Wake every waiter with complete_all(), and check the FIFO again every 2 ms while waiting, until the same 500 ms deadline. A woken sender that still finds the FIFO full rearms the interrupt before it waits, as before. A sender that times out now leaves the interrupt unmasked for the others. tx_irq_unmasked still records the state, so the enable stays balanced: the handler masks it when it fires, and provider removal masks it otherwise. Atomic senders, which poll, are unchanged. Rebased onto aurora-wip, which has the TX wait failed report. That report now prints once, at the deadline of an interrupt-driven wait. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mailbox.c | 58 ++++++++++++++++++------------- include/linux/soc/apple/mailbox.h | 4 +-- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 902f872ff994e..2591560a9b716 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -119,6 +119,11 @@ static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == #define APPLE_MBOX_MSG1_MSG GENMASK(31, 0) #define APPLE_MBOX_TX_TIMEOUT 500 +/* + * The send-empty interrupt reports an empty FIFO, not free space. A sender + * that waits for it also checks the FIFO this often. + */ +#define APPLE_MBOX_TX_RECHECK_MS 2 #define APPLE_G17_MBOX_RX_BUDGET 64 #define APPLE_G17_MBOX_ATOMIC_TX_US 1000 @@ -330,7 +335,16 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, while (mbox_ctrl & mbox->hw->control_full) { now = jiffies; if (time_after_eq(now, deadline)) { + u32 i2a = readl_relaxed(mbox->regs + mbox->hw->i2a_control); + spin_unlock_irqrestore(&mbox->tx_lock, flags); + if (mbox->irq_send_empty >= 0) + dev_err(mbox->dev, + "TX wait failed a2i=%#x i2a=%#x full=%#x send_irq=%d recv_irq=%d msg=%#llx/%#x\n", + mbox_ctrl, i2a, mbox->hw->control_full, + mbox->irq_send_empty, + mbox->irq_recv_not_empty, + msg.msg0, msg.msg1); return -ETIMEDOUT; } remaining = deadline - now; @@ -383,31 +397,19 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, } spin_unlock_irqrestore(&mbox->tx_lock, flags); + /* + * The firmware can take messages without emptying the FIFO, + * which raises no interrupt, so check it again after a short + * wait; the deadline above ends the loop. Other senders may be + * waiting on the same interrupt: a sender that gives up leaves + * it unmasked for them. The handler masks it when it fires, + * and removal masks it otherwise. + */ t = wait_for_completion_interruptible_timeout( - &mbox->tx_empty, remaining); - if (t <= 0) { - u32 a2i = 0, i2a = 0; - bool removing; - - /* A late IRQ may already have masked this under tx_lock. */ - spin_lock_irqsave(&mbox->tx_lock, flags); - removing = READ_ONCE(mbox->removing); - if (!removing) { - a2i = readl_relaxed(mbox->regs + mbox->hw->a2i_control); - i2a = readl_relaxed(mbox->regs + mbox->hw->i2a_control); - } - if (mbox->tx_irq_unmasked) { - disable_irq_nosync(mbox->irq_send_empty); - mbox->tx_irq_unmasked = false; - } - spin_unlock_irqrestore(&mbox->tx_lock, flags); - dev_err(mbox->dev, - "TX wait failed ret=%ld a2i=%#x i2a=%#x full=%#x send_irq=%d recv_irq=%d removing=%d msg=%#llx/%#x\n", - t, a2i, i2a, mbox->hw->control_full, - mbox->irq_send_empty, mbox->irq_recv_not_empty, - removing, msg.msg0, msg.msg1); - return t < 0 ? t : -ETIMEDOUT; - } + &mbox->tx_empty, + min(remaining, msecs_to_jiffies(APPLE_MBOX_TX_RECHECK_MS))); + if (t < 0) + return t; spin_lock_irqsave(&mbox->tx_lock, flags); if (unlikely(READ_ONCE(mbox->removing))) { @@ -443,7 +445,13 @@ static irqreturn_t apple_mbox_send_empty_irq(int irq, void *data) disable_irq_nosync(mbox->irq_send_empty); mbox->tx_irq_unmasked = false; } - complete(&mbox->tx_empty); + /* + * Wake every waiting sender, not just one. The interrupt is now + * masked, so a sender left asleep would only wake at its timeout, + * with room in the FIFO. A woken sender that still finds the FIFO + * full rearms the interrupt before it waits again. + */ + complete_all(&mbox->tx_empty); spin_unlock(&mbox->tx_lock); return IRQ_HANDLED; diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index 694d538d8c70b..4c27621f22e0a 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -87,8 +87,8 @@ struct apple_mbox { /* * Protected by tx_lock: set while irq_send_empty is unmasked, so that - * the IRQ handler, a failed send and provider removal mask it exactly - * once between them. + * the IRQ handler and provider removal mask it exactly once between + * them. */ bool tx_irq_unmasked; }; From f494c8495ea4cc14e528d963b83623ecbb9923d0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 23:11:47 +1100 Subject: [PATCH 1401/1579] media: apple: ave: drop the bring-up diagnostics The buffer sizes are settled on hardware, so remove what was only there to measure them: the scan of every working buffer's margin at the end of a session, which logged at info level each time, and the usage scan of the H.264 per-frame buffer. Working buffers no longer need a kernel mapping. The stride fix-up note and the retried restart of a still powered coprocessor are normal operation and move to debug level. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/ave-fw.c | 2 +- drivers/media/platform/apple/ave/ave-sess.c | 125 +++++--------------- 2 files changed, 29 insertions(+), 98 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c index 4987a32ad5d42..9b4d3c3f67662 100644 --- a/drivers/media/platform/apple/ave/ave-fw.c +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -262,7 +262,7 @@ static int ave_fw_prepare_restart(struct ave_dev *ave) return 0; if (readl(ave->coproc + AVE_COPROC_CTRL)) { - dev_warn(ave->dev, "coprocessor kept power after its halt; retry once it suspends\n"); + dev_dbg(ave->dev, "coprocessor kept power after its halt; retry once it suspends\n"); return -EAGAIN; } status = readl(ave->mbox + AVE_MBOX_STATUS); diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index 27ea45c76080f..e58d485eb61c1 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -193,17 +193,12 @@ static int ave_sess_params(struct ave_ctx *ctx, struct ave_params *p) * mapping is only rounded up to whole pages. */ -/* - * Working buffers keep a kernel mapping so that, when a session ends, the - * driver can see how far past each reported size the firmware wrote (into - * the page rounding, or the room the two groups above get). - */ - /* One buffer to allocate: what the firmware is told and what is mapped. */ struct ave_buf_req { struct ave_buf *b; size_t size; size_t alloc; + unsigned long attrs; }; #define AVE_MAX_BUF_REQS (1 + 4 * AVE_MAX_REFS + 4 + 2 * AVE_TILE_BUFS + 2 + \ @@ -217,9 +212,11 @@ static int ave_buf_req_cmp(const void *a, const void *b) } static void ave_req(struct ave_buf_req *r, unsigned int *n, struct ave_buf *b, size_t size, - size_t alloc) + size_t alloc, unsigned long attrs) { - r[*n] = (struct ave_buf_req){ .b = b, .size = size, .alloc = max(size, alloc) }; + r[*n] = (struct ave_buf_req){ + .b = b, .size = size, .alloc = max(size, alloc), .attrs = attrs, + }; (*n)++; } @@ -236,6 +233,7 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) u32 w = p->width, h = p->height; u32 w16 = DIV_ROUND_UP(w, 16), w32 = DIV_ROUND_UP(w, 32); u32 h32 = DIV_ROUND_UP(h, 32), w64 = DIV_ROUND_UP(w, 64), h64 = DIV_ROUND_UP(h, 64); + const unsigned long nomap = DMA_ATTR_NO_KERNEL_MAPPING; size_t ref_a, ref_b, ref_c, ref_d, tile, mapped = 0; struct ave_buf_req *r; unsigned int n = 0; @@ -251,38 +249,39 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) ref_d = ALIGN(hevc ? 64 * w32 * h32 : 128 * p->mbs, SZ_16K); tile = hevc ? 2304 * w32 * h64 : ALIGN(w, 16) * ALIGN(h, 64); - ave_req(r, &n, &sess->hdr, AVE_HDR_BYTES, 0); + ave_req(r, &n, &sess->hdr, AVE_HDR_BYTES, 0, nomap); for (unsigned int i = 0; i < p->refs; i++) { - ave_req(r, &n, &sess->ref_a[i], ref_a, ref_a * 3 / 2); - ave_req(r, &n, &sess->ref_b[i], ref_b, ref_b * 5 / 4); - ave_req(r, &n, &sess->ref_c[i], ref_c, 0); - ave_req(r, &n, &sess->ref_d[i], ref_d, 0); - } - ave_req(r, &n, &sess->mb, ALIGN(82 * p->mbs, SZ_16K), 0); - ave_req(r, &n, &sess->row_a, hevc ? SZ_16K : ALIGN(256 * w16, SZ_16K), 0); - ave_req(r, &n, &sess->row_b, ALIGN(hevc ? 768 * w32 : 1024 * w16, SZ_16K), 0); - ave_req(r, &n, &sess->fixed, SZ_16K, 0); + ave_req(r, &n, &sess->ref_a[i], ref_a, ref_a * 3 / 2, nomap); + ave_req(r, &n, &sess->ref_b[i], ref_b, ref_b * 5 / 4, nomap); + ave_req(r, &n, &sess->ref_c[i], ref_c, 0, nomap); + ave_req(r, &n, &sess->ref_d[i], ref_d, 0, nomap); + } + ave_req(r, &n, &sess->mb, ALIGN(82 * p->mbs, SZ_16K), 0, nomap); + ave_req(r, &n, &sess->row_a, hevc ? SZ_16K : ALIGN(256 * w16, SZ_16K), 0, nomap); + ave_req(r, &n, &sess->row_b, ALIGN(hevc ? 768 * w32 : 1024 * w16, SZ_16K), 0, nomap); + ave_req(r, &n, &sess->fixed, SZ_16K, 0, nomap); for (unsigned int i = 0; i < (hevc ? 4 : AVE_TILE_BUFS); i++) { - ave_req(r, &n, &sess->tile[i], tile, 0); - ave_req(r, &n, &sess->tile_aux[i], ALIGN(h, 128), 0); + ave_req(r, &n, &sess->tile[i], tile, 0, nomap); + ave_req(r, &n, &sess->tile_aux[i], ALIGN(h, 128), 0, nomap); } if (hevc) for (unsigned int i = 0; i < 2; i++) - ave_req(r, &n, &sess->hevc_extra[i], ALIGN(w * h32, SZ_16K), 0); + ave_req(r, &n, &sess->hevc_extra[i], ALIGN(w * h32, SZ_16K), 0, nomap); for (unsigned int s = 0; s < p->slots; s++) { - ave_req(r, &n, &sess->slot[s].coded, p->coded_cap, 0); - ave_req(r, &n, &sess->slot[s].meta, AVE_META_BYTES, 0); + ave_req(r, &n, &sess->slot[s].coded, p->coded_cap, 0, 0); + ave_req(r, &n, &sess->slot[s].meta, AVE_META_BYTES, 0, 0); /* * HEVC: slice headers. H.264: the per-frame buffer of Process * +0x58c8, which the firmware never wrote in any bit-rate * mode on hardware; one page. */ - ave_req(r, &n, &sess->slot[s].aux, hevc ? AVE_SHDR_BYTES : AVE_H264_FRAME_BUF, 0); + ave_req(r, &n, &sess->slot[s].aux, hevc ? AVE_SHDR_BYTES : AVE_H264_FRAME_BUF, 0, + hevc ? 0 : nomap); } sort(r, n, sizeof(*r), ave_buf_req_cmp, NULL); for (unsigned int i = 0; i < n && !ret; i++) { - ret = ave_alloc_padded(ave, r[i].b, r[i].size, r[i].alloc, 0); + ret = ave_alloc_padded(ave, r[i].b, r[i].size, r[i].alloc, r[i].attrs); mapped += r[i].b->alloc; } kfree(r); @@ -327,8 +326,8 @@ static void ave_note_copy(struct ave_ctx *ctx, u32 stride, const char *how) if (ctx->sess->copy_noted) return; ctx->sess->copy_noted = true; - dev_info(ctx->ave->dev, "input has a %u byte stride; moving lines to %u %s\n", stride, - ctx->sess->p.stride, how); + dev_dbg(ctx->ave->dev, "input has a %u byte stride; moving lines to %u %s\n", stride, + ctx->sess->p.stride, how); } /* @@ -433,71 +432,6 @@ static void ave_sess_log_map(struct ave_dev *ave, struct ave_session *sess) } } -/* - * How far did the firmware write past the size it was given? The margin - * behind each buffer was zero when allocated. A write reaching the end of - * the mapping may have gone on into the next buffer. - */ -static void ave_check_tail(struct ave_dev *ave, const char *name, int i, - const struct ave_buf *b) -{ - const u8 *p = b->cpu; - size_t end; - - if (!p || b->attrs & DMA_ATTR_NO_KERNEL_MAPPING || b->alloc <= b->size) - return; - for (end = b->alloc; end > b->size && !p[end - 1]; end--) - ; - if (end == b->size) - return; - if (end == b->alloc) - dev_warn(ave->dev, "buf %s%d: firmware wrote to the end of its mapping (+%#zx, size %#zx)\n", - name, i, b->alloc, b->size); - else - dev_info(ave->dev, "buf %s%d: firmware wrote %#zx bytes past its size %#zx\n", - name, i, end - b->size, b->size); -} - -static void ave_sess_check_tails(struct ave_dev *ave, struct ave_session *sess) -{ - ave_check_tail(ave, "hdr", 0, &sess->hdr); - ave_check_tail(ave, "mb", 0, &sess->mb); - ave_check_tail(ave, "row_a", 0, &sess->row_a); - ave_check_tail(ave, "row_b", 0, &sess->row_b); - ave_check_tail(ave, "fixed", 0, &sess->fixed); - for (int i = 0; i < 2; i++) - ave_check_tail(ave, "hevc_extra", i, &sess->hevc_extra[i]); - for (int i = 0; i < AVE_MAX_REFS; i++) { - ave_check_tail(ave, "ref_a", i, &sess->ref_a[i]); - ave_check_tail(ave, "ref_b", i, &sess->ref_b[i]); - ave_check_tail(ave, "ref_c", i, &sess->ref_c[i]); - ave_check_tail(ave, "ref_d", i, &sess->ref_d[i]); - } - for (int i = 0; i < AVE_TILE_BUFS; i++) { - ave_check_tail(ave, "tile", i, &sess->tile[i]); - ave_check_tail(ave, "tile_aux", i, &sess->tile_aux[i]); - } - for (int i = 0; i < AVE_MAX_SLOTS; i++) { - ave_check_tail(ave, "coded", i, &sess->slot[i].coded); - ave_check_tail(ave, "meta", i, &sess->slot[i].meta); - ave_check_tail(ave, "aux", i, &sess->slot[i].aux); - } - - /* - * The size of the H.264 per-frame buffer is not known; it is - * allocated generously. Report how much of it the firmware used. - */ - if (sess->p.codec == AVE_CODEC_H264 && sess->slot[0].aux.cpu) { - const struct ave_buf *b = &sess->slot[0].aux; - const u64 *w = b->cpu; - size_t n = b->size / 8; - - while (n && !w[n - 1]) - n--; - dev_dbg(ave->dev, "per-frame buffer: %#zx of %#zx bytes used\n", n * 8, b->size); - } -} - /* Work out the parameters and allocate the buffers; no firmware access. */ struct ave_session *ave_sess_create(struct ave_ctx *ctx) { @@ -539,13 +473,10 @@ void ave_sess_destroy(struct ave_ctx *ctx) if (!sess) return; - if (sess->handed && ctx->ave->fw_state == AVE_FW_BROKEN) { + if (sess->handed && ctx->ave->fw_state == AVE_FW_BROKEN) dev_warn(ctx->ave->dev, "keeping the buffers of a failed session\n"); - } else { - if (sess->handed) - ave_sess_check_tails(ctx->ave, sess); + else ave_sess_free_bufs(ctx->ave, sess); - } kfree(sess); ctx->sess = NULL; } From 07190cc827b078e322b9609014cfa85a53ac371e Mon Sep 17 00:00:00 2001 From: Ryan <81541646+RyanTheTide@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:24:54 +1100 Subject: [PATCH 1402/1579] arm64: dts: apple: t8140-j700: Drop SMC sensors the J700 does not have The MacBook Neo takes its SMC sensors from hwmon-common.dtsi and hwmon-laptop.dtsi, as the other laptops do. Five of those keys do not exist on the J700 SMC: PMVR, PHPC, TH0x, TCHP and TW0P. The board has no heatpipe and its Wi-Fi is the MT7932 rather than a module with its own sensor, and the NAND temperature is already TH0p ("Storage (NAND)"). When the J700 first took these files, macsmc-hwmon skipped keys the SMC did not have. Since the T8140 zones refer to SMC temperatures by channel index, a temperature node whose key is missing keeps an empty, hidden channel and the driver warns about it, so every boot logs macsmc-hwmon macsmc-hwmon: temperature-TH0x unavailable (-5), temp84 hidden macsmc-hwmon macsmc-hwmon: temperature-TCHP unavailable (-5), temp86 hidden macsmc-hwmon macsmc-hwmon: temperature-TW0P unavailable (-5), temp87 hidden The keys are missing on every boot (42 of 42 saved boot logs), and none of the five shows up in sysfs. Delete the five nodes in the board file, so the device tree lists only sensors the machine has and the warning is left for a sensor that is really missing. The removed temperatures come after every channel a thermal zone uses (0 to 5), so the zones keep their sensors. Only the hwmon number of the battery hotspot moves, from temp85 to temp84, and its label stays the same. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index ece0d9c6bfefb..f0a10cf0c8b3e 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -1812,6 +1812,20 @@ #include "hwmon-common.dtsi" #include "hwmon-laptop.dtsi" +/* + * The J700 SMC has no keys for these shared laptop sensors. On T8140 the + * temperature channels are indexed for the thermal zones, so a missing + * temperature key keeps an empty, hidden channel and is reported at every + * boot. The NAND temperature is described above as Storage (NAND). + */ +&smc_hwmon { + /delete-node/ power-PMVR; + /delete-node/ power-PHPC; + /delete-node/ temperature-TH0x; + /delete-node/ temperature-TCHP; + /delete-node/ temperature-TW0P; +}; + &pcie0 { /* The PIODMA guard stays closed until the retained bootstrap succeeds. */ apple,piodma = <&pcie0_piodma_diag>; From 4db3a5a22c43afdc11bcc0c64ff7e90eb8f87caa Mon Sep 17 00:00:00 2001 From: Ryan <81541646+RyanTheTide@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:36:52 +1100 Subject: [PATCH 1403/1579] wifi: mt7932: report a connect the AP did not complete at info level A Neo boot logged mt7932-fullmac 0000:01:00.0: CONNECT_FAILED: -110; recovery-required=0 at error level while NetworkManager brought up Wi-Fi. The firmware was fine. The association timed out at the AP, cfg80211 reported the failed connect to the supplicant, and the next attempt connected. Once the request has gone to the AP, a timeout, a refusal or a reset with the firmware still healthy is how the AP answered. These are the same three outcomes the connect worker already treats as retryable for the first OWE attempt. mac80211 reports association timeouts and refusals at info level, and the supplicant logs them too. Log them at info level here as well. Everything else keeps the error level: failures before the request reaches the AP, protocol and I/O errors, and any failure that leaves the link needing recovery. A firmware command timeout sets link_failed, so it still shows as an error. The connect-result host test now extracts the new helper and checks the info and error split for every outcome. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/link.c | 17 +++++++ tools/asahi/mt7932-connect-result-tests.py | 54 +++++++++++++++------ 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 538a1f15f0a3f..713d801cb1891 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -598,6 +598,20 @@ static int mt_connect_attempt(struct mt7932 *m, bool *submitted, return ret ?: READ_ONCE(m->connect_error); } +/* + * Once the request has gone to the AP, a timeout, a refusal or a reset while + * the firmware is still healthy is how the AP answered, not a driver or + * firmware fault. cfg80211 hands it to the supplicant, which retries, and + * mac80211 reports the same outcomes at info level. Keep the error level for + * failures that need recovery or point at the firmware. + */ +static bool mt_connect_failed_at_peer(struct mt7932 *m, bool auth_submitted, + int ret) +{ + return auth_submitted && !READ_ONCE(m->link_failed) && + (ret == -ETIMEDOUT || ret == -ECONNREFUSED || ret == -ECONNRESET); +} + void mt_connect_work(struct work_struct *work) { struct mt7932 *m = container_of(work, struct mt7932, connect_work); @@ -760,6 +774,9 @@ void mt_connect_work(struct work_struct *work) READ_ONCE(m->data_ready)); else if (cancelled) dev_dbg(&m->pdev->dev, "CONNECT_CANCELLED: %d\n", ret); + else if (mt_connect_failed_at_peer(m, auth_submitted, ret)) + dev_info(&m->pdev->dev, "CONNECT_FAILED: %d; recovery-required=0\n", + ret); else dev_err(&m->pdev->dev, "CONNECT_FAILED: %d; recovery-required=%u\n", ret, READ_ONCE(m->link_failed)); diff --git a/tools/asahi/mt7932-connect-result-tests.py b/tools/asahi/mt7932-connect-result-tests.py index 2f8a4bf9d4c62..87054ccfcef80 100755 --- a/tools/asahi/mt7932-connect-result-tests.py +++ b/tools/asahi/mt7932-connect-result-tests.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only -"""Compile the connect worker's report path and the complete nl80211 encoder.""" +"""Compile the connect worker's report path, its log-level helper and the +complete nl80211 encoder.""" from pathlib import Path import argparse import hashlib @@ -41,9 +42,9 @@ #define spin_unlock(l) ((void)(l)) #define mutex_lock(l) ((void)(l)) #define mutex_unlock(l) ((void)(l)) -#define dev_info(...) ((void)0) +#define dev_info(...) ((void)info_logs++) #define dev_dbg(...) ((void)0) -#define dev_err(...) ((void)0) +#define dev_err(...) ((void)err_logs++) #define for_each_valid_link(cr,l) for((l)=0;(l)<1;(l)++) struct sk_buff { unsigned values[32]; bool seen[32]; }; struct nlattr { int n; }; @@ -75,11 +76,11 @@ static int wiphy_net(int*w){return 0;} static void genlmsg_multicast_netns(int*f,int n,struct sk_buff*m,int p,int g,gfp_t t){} ENCODER -struct mt7932 {bool stopping,connect_open,assoc_request_seen,connecting,connected,disconnecting,connect_cancelled,connect_publishing,connect_lost,rf_ready,connect_pmk_valid; +struct mt7932 {bool link_failed,stopping,connect_open,assoc_request_seen,connecting,connected,disconnecting,connect_cancelled,connect_publishing,connect_lost,rf_ready,connect_pmk_valid; int connect_error,response_lock,command_mutex,data_lock,wiphy,power_work;u16 disconnect_reason;struct net_device *netdev;struct cfg80211_bss *connect_bss;u8 connect_bssid[6],connect_pmk[32],connect_sae_pwd[63],connect_sae_pwd_length,assoc_request_ies[512];unsigned assoc_request_ie_len; struct {struct {bool ht_supported;}ht_cap;}band2;bool power_tim,data_ready,station_signal_valid,station_rx_rate_valid,station_tx_rate_valid;unsigned long station_tx_bytes,station_rx_bytes,station_tx_packets,station_rx_packets,station_connected;}; static unsigned long jiffies=1; -static unsigned notifications,disconnections,retirements,releases,authorizations,carrier,work_count; +static unsigned info_logs,err_logs,notifications,disconnections,retirements,releases,authorizations,carrier,work_count; static struct cfg80211_connect_resp_params saved; static bool mt_rf_allowed(struct mt7932*m){return m->rf_ready;} static void mt_transport_snapshot(struct mt7932*m){} @@ -96,16 +97,22 @@ static void netif_carrier_on(struct net_device*n){carrier++;} static void netif_wake_queue(struct net_device*n){} static void cfg80211_port_authorized(struct net_device*n,const u8*b,const u8*p,size_t l,gfp_t f){authorizations++;} -static void report_worker(struct mt7932 *m,int ret,bool retirement_attempted){ +HELPER +static void report_worker(struct mt7932 *m,int ret,bool retirement_attempted,bool auth_submitted){ struct cfg80211_connect_resp_params response={};unsigned long flags;bool submitted=true,peer_owned=true,cancelled;u16 reason; REPORT int main(void){struct net_device n={1};struct cfg80211_bss b={}; int errors[]={0,-ETIMEDOUT,-ECONNREFUSED,-EPROTO,-ECANCELED,-ECONNRESET,-EIO}; - for(unsigned i=0;i Date: Fri, 30 Oct 2026 14:50:53 +0000 Subject: [PATCH 1404/1579] dt-bindings: hwmon: apple,smc-hwmon: describe fans the host may drive Whether a Mac's SMC firmware accepts a manual fan speed from the host, and hands the fan back when asked, is a property of that machine. Add a boolean, apple,fan-manual-control, for the fan nodes that have it. It needs the fan's target and mode keys. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/hwmon/apple,smc-hwmon.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index 4e0b4268f3ffa..e932bfa31a32c 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -95,6 +95,16 @@ patternProperties: pattern: "^[A-Za-z0-9]{4}$" description: Writeable key to enable/disable manual fan control + apple,fan-manual-control: + type: boolean + description: + The SMC firmware of this machine accepts a manual speed for this + fan from the host and returns it to firmware control when asked, + so the fan's target speed can be set from the OS. Requires + apple,fan-target and apple,fan-mode. Without it the fan is only + monitored, and driven by the thermal framework where the SMC + node is a cooling device. + "^power-[A-Za-z0-9]{4}$": $ref: "#/$defs/sensor" unevaluatedProperties: false From 65f0c0fddad9135c63a7cb79277fef912890d4e6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 14:50:53 +0000 Subject: [PATCH 1405/1579] hwmon: macsmc: let the device tree allow manual fan speeds fanX_target is only writable with the unsafe fan_control parameter, so a machine whose SMC is known to take manual speeds still needs a kernel parameter for it. Make fanX_target writable on the fans whose node has apple,fan-manual-control, and warn when the property is set on a fan without target and mode keys. fan_control stays. It is the documented mainline interface for this, and dropping it would leave existing command lines and modprobe options with an unknown-parameter warning and without the speeds they ask for. It now only adds fans the device tree does not allow. Nothing changes for machines without the property, the thermal cooling device path or the unlock and restore sequence. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/hwmon/macsmc-hwmon.rst | 12 ++++++++---- drivers/hwmon/macsmc-hwmon.c | 27 ++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/Documentation/hwmon/macsmc-hwmon.rst b/Documentation/hwmon/macsmc-hwmon.rst index 6903f76df62bf..8aef3dd2bf75e 100644 --- a/Documentation/hwmon/macsmc-hwmon.rst +++ b/Documentation/hwmon/macsmc-hwmon.rst @@ -22,10 +22,14 @@ the desktop Macs), sensors present on any given machine are described via Devicetree. The driver picks these up and registers them with hwmon when probed. -Manual fan speed is supported via the fan_control module parameter. This -is disabled by default and marked as unsafe, as it cannot be proven that -the system will fail safe if overheating due to manual fan control being -used. +Manual fan speed is available on the fans whose Devicetree node has +apple,fan-manual-control, which a machine's Devicetree sets once its SMC +firmware is known to accept a manual speed and to take the fan back. On +those fans fanX_target is writable; writing 0 hands the fan back to the +firmware. The fan_control module parameter, disabled by default and marked +as unsafe, allows manual speeds on every fan with a mode key whatever the +Devicetree says. It cannot be proven that the system will fail safe if it +overheats under a manual fan speed. sysfs interface --------------- diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index c0ca04e88d3d2..e1685c2945d0c 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -49,10 +49,15 @@ #define FLT_MANT_MASK GENMASK(22, 0) #define FLT_SIGN_MASK BIT(31) +/* + * Fans whose device tree node has apple,fan-manual-control take a target speed + * from userspace. This parameter, kept for compatibility, allows it for every + * fan with a mode key, whatever the device tree says. + */ static bool fan_control; module_param_unsafe(fan_control, bool, 0644); MODULE_PARM_DESC(fan_control, - "Override the SMC to set your own fan speeds on supported machines"); + "Allow manual fan speeds on every fan with a mode key, also where the device tree does not (default: the device tree decides)"); /* * SoC die temperature keys: temperature channels labelled "SoC die " @@ -185,6 +190,8 @@ struct macsmc_hwmon_fan { char label[MAX_LABEL_LENGTH]; u32 attrs; bool manual; + /* The device tree allows a manual speed from userspace. */ + bool manual_control; /* Index of the fan node among the fan-* nodes: its cooling device id. */ u32 index; struct macsmc_hwmon *hwmon; @@ -689,12 +696,18 @@ static int macsmc_hwmon_fan_apply(struct macsmc_hwmon *hwmon, return 0; } +/* Whether userspace may set this fan's speed. */ +static bool macsmc_hwmon_fan_user_control(const struct macsmc_hwmon_fan *fan) +{ + return fan->mode.macsmc_key && (fan->manual_control || fan_control); +} + static int macsmc_hwmon_set_fan(struct macsmc_hwmon *hwmon, int channel, long val) { struct macsmc_hwmon_fan *fan = &hwmon->fan.fans[channel]; int ret; - if (!fan_control || fan->mode.macsmc_key == 0) + if (!macsmc_hwmon_fan_user_control(fan)) return -EOPNOTSUPP; if (val < 0) return -EINVAL; @@ -899,7 +912,7 @@ static umode_t macsmc_hwmon_fan_is_visible(const struct macsmc_hwmon_fan *fan, u32 attr) { if (fan->attrs & BIT(attr)) { - if (attr == hwmon_fan_target && fan_control && fan->mode.macsmc_key) + if (attr == hwmon_fan_target && macsmc_hwmon_fan_user_control(fan)) return 0644; return 0444; @@ -1237,6 +1250,14 @@ static int macsmc_hwmon_create_fan(struct device *dev, struct apple_smc *smc, return ret; } + if (of_property_read_bool(fan_node, "apple,fan-manual-control")) { + if (fan->mode.macsmc_key && (fan->attrs & HWMON_F_TARGET)) + fan->manual_control = true; + else + dev_warn(dev, "%s: apple,fan-manual-control needs apple,fan-target and apple,fan-mode\n", + fan->label); + } + /* Initialise fan control mode to automatic */ fan->manual = false; From b3b601515f52ea96864fbcab8520086f04ff316e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 14:50:53 +0000 Subject: [PATCH 1406/1579] arm64: dts: apple: t6040-j616s: allow manual speeds on both fans The J616s SMC keeps its fans in system mode, takes a manual speed once the fan test key unlocks them (hwmon: macsmc: Unlock fan control on firmware that keeps fans in system mode), and returns both fans to system mode when the request is dropped. The batch1 fan test showed both fans following the requested targets and going back to firmware control with no warnings. Mark both fans apple,fan-manual-control so this works without the fan_control parameter. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 83b29a6c3f866..69f0070af7659 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -3868,6 +3868,19 @@ */ &smc_hwmon { #cooling-cells = <3>; + + /* + * The SMC accepts a manual speed for both fans once its fan test + * key unlocks them, and returns them to system mode when the + * request is dropped. + */ + fan-F0Ac { + apple,fan-manual-control; + }; + + fan-F1Ac { + apple,fan-manual-control; + }; }; &pcie_dart { From 8ea7d24f706848229b5641c64ab3fd51e80f2ee4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 21 Oct 2026 19:18:36 +1100 Subject: [PATCH 1407/1579] wifi: mt7932: report interrupt, ring and BSS presence observations 2.4 GHz runs at 6-17 Mbit/s with 0.5-2 s gateway round trips and no loss, while 5 GHz on the same AP runs at about 200 Mbit/s with 2 ms round trips. Power save is off on both bands. Latency without loss points at work that waits in the host or the firmware, but nothing the driver exposes shows where. Count interrupt-thread passes, descriptors consumed per receive ring, and interrupt status bits found pending outside the enabled mask. A ring whose completion is announced only by such a bit is drained only when another interrupt arrives. Count TX completion reports, firmware BSS-absence reports and the time spent absent, during which TX is held, beacons from the associated BSS that reach the host, and admitted Ethernet RX per band. Export them through ethtool next to the existing RX counters and document them. The counters are lifetime host observations. They send no firmware commands and do not change interrupt, ring, power or radio behaviour. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 14 +++++++ drivers/net/wireless/mediatek/mt7932/data.c | 12 ++++++ .../net/wireless/mediatek/mt7932/frontend.c | 1 + drivers/net/wireless/mediatek/mt7932/mt7932.c | 13 +++++- drivers/net/wireless/mediatek/mt7932/mt7932.h | 9 ++++ .../net/wireless/mediatek/mt7932/station.c | 41 +++++++++++++++++++ 6 files changed, 89 insertions(+), 1 deletion(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 7925972fe222b..fd045bcc9dab3 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -623,6 +623,20 @@ over-air MPDU counts, BA-window sizes, TX retries or firmware ACK counters. Duplicate suppression records sequences only after the protection check, so a rejected unprotected frame cannot suppress a later decrypted retry. +Transport counters describe how the host services the device: +``irq_service_passes`` counts interrupt-thread passes and +``rx_ring0_records`` to ``rx_ring8_records`` count descriptors consumed per +receive ring. ``irq_unmasked_bits`` accumulates status bits that were +pending outside the enabled interrupt mask, and ``irq_unmasked_passes`` +counts the passes that saw any. Work announced only by such a bit waits for +an unrelated interrupt, which shows up as latency rather than loss. +``tx_free_events`` counts TX completion reports. ``bss_absent_events`` and +``bss_absent_ms`` count how often and for how long the firmware reported the +BSS absent, during which TX is held. ``home_beacons`` counts beacons from +the associated BSS that reached the host, and ``rx_2ghz_records``, +``rx_5ghz_records`` and ``rx_6ghz_records`` split admitted Ethernet RX by +band. Like the RX counters, they are lifetime values. + Use the current RX bitrate from ``iw dev station dump`` while traffic is running. The driver expires that sample after one second; an idle or unsupported RX vector can leave it absent. TX bitrate is not diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index b749e18a93380..553a6fe0e1193 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -79,6 +79,15 @@ void mt_bss_presence(struct mt7932 *m, const struct mt7932_event *event) if (event->length < 40 || body[0] || !READ_ONCE(m->bss_active)) return; + /* response_lock: how often, and for how long, the firmware reports + * the BSS absent. TX is held while it is absent without quota. + */ + if (body[1] && !m->bss_absent) { + m->bss_absent_events++; + m->bss_absent_since = jiffies; + } else if (!body[1] && m->bss_absent) { + m->bss_absent_ms += jiffies_to_msecs(jiffies - m->bss_absent_since); + } spin_lock_irqsave(&m->data_lock, flags); m->bss_absent = !!body[1]; m->bss_quota = body[2]; @@ -107,6 +116,7 @@ void mt_data_complete(struct mt7932 *m, const u8 *packet, size_t length) return; } spin_lock_irqsave(&m->data_lock, flags); + m->tx_free_events++; mt_data_clean_locked(m); for (i = 0; i < count; i++) { unsigned int token = tokens[i]; @@ -269,6 +279,8 @@ void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) m->rx_ethernet++; spin_lock(&m->data_lock); m->station_rx_packets++; + m->rx_band_records[frame->band == NL80211_BAND_2GHZ ? 0 : + frame->band == NL80211_BAND_5GHZ ? 1 : 2]++; m->station_rx_bytes += frame->length; signal = mt7932_rx_signal(frame, m->antenna_mask); if (frame->signal_valid && signal > -128) { diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index cfbdcd9f494d2..51b38f7291b5f 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -711,6 +711,7 @@ void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length) if (m->connected && !m->disconnecting && !m->stopping && frame.channel == m->connect_channel && ether_addr_equal(frame.data + 16, m->connect_bssid)) { + m->home_beacons++; mt_cqm_signal(m, mt7932_rx_signal(&frame, m->antenna_mask)); if (tim && tim->datalen >= 4 && !m->power_tim) { m->power_tim = true; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index acf23bb0d31da..94388b56449e1 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -229,6 +229,7 @@ static void mt_rx_drain(struct mt7932 *m, struct mt7932_ring *q) break; dma_rmb(); length = (ctrl >> 16) & 0x3fff; + m->rx_ring_records[q - m->rx]++; if (length <= MT7932_RX_CAPACITY && (ctrl & BIT(30))) mt_receive(m, q->buffers + q->tail * RX_STRIDE, length); else @@ -274,9 +275,19 @@ static irqreturn_t mt_irq(int irq, void *data) static void mt_service(struct mt7932 *m) { - u32 causes = mt_read(m, W + 0x200) & m->irq_mask; + u32 raw = mt_read(m, W + 0x200), causes = raw & m->irq_mask; unsigned int i; + /* Status bits outside the enabled mask never raise an interrupt; if + * one of them announces ring work, that work waits for an unrelated + * cause. Record them so a test can see such a source. + */ + m->irq_passes++; + if ((raw & ~m->irq_mask) && raw != U32_MAX) { + m->irq_unmasked |= raw & ~m->irq_mask; + m->irq_unmasked_passes++; + } + /* * Acknowledge before servicing the rings. A completion that lands * while they are being drained then sets its status bit again and diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 172369bb3f159..7bba93a9cd19e 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -199,6 +199,15 @@ struct mt7932 { u64 rx_integrity, rx_malformed, rx_unsupported; u64 rx_wrong_peer, rx_unprotected, rx_duplicate, rx_no_memory; u64 rx_retry, rx_amsdu; + /* Lifetime transport observations for ethtool. The ring and cause + * counters are written only by the interrupt thread; the BSS presence + * counters are under response_lock, the data counters under data_lock. + */ + u64 irq_passes, irq_unmasked_passes, rx_ring_records[9]; + u32 irq_unmasked; + u64 tx_free_events, bss_absent_events, bss_absent_ms; + unsigned long bss_absent_since; + u64 home_beacons, rx_band_records[3]; /* TX0 descriptors published since the last doorbell; data_lock. */ unsigned int data_unkicked; bool data_ready; diff --git a/drivers/net/wireless/mediatek/mt7932/station.c b/drivers/net/wireless/mediatek/mt7932/station.c index d592b5029d637..d6d131a507ca2 100644 --- a/drivers/net/wireless/mediatek/mt7932/station.c +++ b/drivers/net/wireless/mediatek/mt7932/station.c @@ -11,6 +11,25 @@ static const char mt_rx_stat_names[][ETH_GSTRING_LEN] = { "host_rx_no_memory", "host_rx_retry_records", "host_rx_amsdu_records", + "irq_service_passes", + "irq_unmasked_passes", + "irq_unmasked_bits", + "rx_ring0_records", + "rx_ring1_records", + "rx_ring2_records", + "rx_ring3_records", + "rx_ring4_records", + "rx_ring5_records", + "rx_ring6_records", + "rx_ring7_records", + "rx_ring8_records", + "tx_free_events", + "bss_absent_events", + "bss_absent_ms", + "home_beacons", + "rx_2ghz_records", + "rx_5ghz_records", + "rx_6ghz_records", }; static int mt_get_sset_count(struct net_device *netdev, int set) @@ -33,8 +52,19 @@ static void mt_get_ethtool_stats(struct net_device *netdev, { struct mt7932 *m = *(struct mt7932 **)netdev_priv(netdev); unsigned long flags; + unsigned int i; + BUILD_BUG_ON(ARRAY_SIZE(mt_rx_stat_names) != 28); + /* The interrupt thread alone writes these; a torn pass is harmless. */ + data[9] = READ_ONCE(m->irq_passes); + data[10] = READ_ONCE(m->irq_unmasked_passes); + data[11] = READ_ONCE(m->irq_unmasked); + for (i = 0; i < ARRAY_SIZE(m->rx_ring_records); i++) + data[12 + i] = READ_ONCE(m->rx_ring_records[i]); spin_lock_irqsave(&m->response_lock, flags); + data[22] = m->bss_absent_events; + data[23] = m->bss_absent_ms; + data[24] = m->home_beacons; data[0] = m->rx_integrity; data[1] = m->rx_malformed; data[2] = m->rx_unsupported; @@ -45,6 +75,17 @@ static void mt_get_ethtool_stats(struct net_device *netdev, data[7] = m->rx_retry; data[8] = m->rx_amsdu; spin_unlock_irqrestore(&m->response_lock, flags); + if (READ_ONCE(m->data_ready)) { + spin_lock_irqsave(&m->data_lock, flags); + data[21] = m->tx_free_events; + for (i = 0; i < ARRAY_SIZE(m->rx_band_records); i++) + data[25 + i] = m->rx_band_records[i]; + spin_unlock_irqrestore(&m->data_lock, flags); + } else { + data[21] = 0; + for (i = 0; i < ARRAY_SIZE(m->rx_band_records); i++) + data[25 + i] = 0; + } } const struct ethtool_ops mt_ethtool_ops = { From d8a276e312dee81addbc1d1185b9716c81a068ce Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 22 Oct 2026 09:21:40 +1100 Subject: [PATCH 1408/1579] wifi: mt7932: invalidate pre-join station power cache Every join sends CID05 awake before EA/40. That marks mode 0 applied, so the post-association power worker skips the desired awake submission when userspace disabled station power save. A pre-join SET cannot establish the power policy of the subsequent associated firmware session. Invalidate the cache after pre-join initialization, including successful submission. The existing associated-session worker then submits the desired mode once; later identical requests still coalesce. Preserve the TIM and RF/peer/termination gates and leave failures uncached. Add hardware-free KUnit coverage for reconnects, mode-0 reapplication, TIM-gated sleep, failed submissions and terminal sessions. The transport seam exists only in the RF KUnit build. List the driver directory under the existing J700 maintainer entry. This repairs a host cache boundary. Its effect on firmware coexistence and radio latency still requires paired Bluetooth on/off qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/net/wireless/mediatek/mt7932/Makefile | 1 + drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 + .../net/wireless/mediatek/mt7932/power-test.c | 199 ++++++++++++++++++ drivers/net/wireless/mediatek/mt7932/power.c | 17 +- 5 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 drivers/net/wireless/mediatek/mt7932/power-test.c diff --git a/MAINTAINERS b/MAINTAINERS index a8c16694f668a..93fa816104d7f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2118,6 +2118,7 @@ F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ +F: drivers/net/wireless/mediatek/mt7932/ F: drivers/phy/apple/atc-j700-dp.h F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index cf7b169dab786..65854425354aa 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -2,6 +2,7 @@ obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o +mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += power-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += input-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += regulatory-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += association-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 7bba93a9cd19e..6afd60d2de8b0 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -70,6 +70,8 @@ struct mt7932 { /* Hardware-free calibration transport seam; production builds omit it. */ int (*cal_test_send)(struct mt7932 *m, const void *body, size_t length); void *cal_test_context; + int (*power_test_send)(struct mt7932 *m, const void *body, size_t length); + void *power_test_context; #endif u8 cal_requests[32][16]; u8 cal_request_seq[32]; diff --git a/drivers/net/wireless/mediatek/mt7932/power-test.c b/drivers/net/wireless/mediatek/mt7932/power-test.c new file mode 100644 index 0000000000000..4588ab006aaa3 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/power-test.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +#include "mt7932.h" + +struct mt_power_test_context { + struct kunit *test; + struct mt7932 *m; + unsigned int commands; + u8 mode; + int error; +}; + +static int mt_power_test_send(struct mt7932 *m, const void *data, size_t length) +{ + struct mt_power_test_context *ctx = m->power_test_context; + const u8 *body = data; + + lockdep_assert_held(&m->command_mutex); + KUNIT_EXPECT_EQ(ctx->test, length, (size_t)4); + if (length != 4) + return -EINVAL; + KUNIT_EXPECT_EQ(ctx->test, body[0], 0); /* owned BSS0 */ + KUNIT_EXPECT_EQ(ctx->test, body[2], 0); + KUNIT_EXPECT_EQ(ctx->test, body[3], 0); + ctx->commands++; + ctx->mode = body[1]; + return ctx->error; +} + +static void mt_power_test_device_release(struct device *dev) +{ +} + +static void mt_power_test_cleanup(void *data) +{ + struct mt7932 *m = data; + + cancel_work_sync(&m->power_work); + put_device(&m->pdev->dev); +} + +static int mt_power_test_init(struct kunit *test) +{ + struct mt_power_test_context *ctx; + struct mt7932 *m; + int ret; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + m = kunit_kzalloc(test, sizeof(*m), GFP_KERNEL); + if (!m) + return -ENOMEM; + m->pdev = kunit_kzalloc(test, sizeof(*m->pdev), GFP_KERNEL); + if (!m->pdev) + return -ENOMEM; + device_initialize(&m->pdev->dev); + m->pdev->dev.release = mt_power_test_device_release; + ret = dev_set_name(&m->pdev->dev, "mt7932-power-test"); + if (ret) { + put_device(&m->pdev->dev); + return ret; + } + mutex_init(&m->command_mutex); + spin_lock_init(&m->response_lock); + INIT_WORK(&m->power_work, mt_power_work); + ret = kunit_add_action_or_reset(test, mt_power_test_cleanup, m); + if (ret) + return ret; + m->rf_ready = true; + m->peer_valid = true; + m->power_test_send = mt_power_test_send; + m->power_test_context = ctx; + ctx->m = m; + ctx->test = test; + test->priv = ctx; + return 0; +} + +static int mt_power_test_prejoin(struct mt7932 *m) +{ + int ret; + + mutex_lock(&m->command_mutex); + ret = mt_power_init(m); + mutex_unlock(&m->command_mutex); + return ret; +} + +static void mt_power_test_run(struct mt7932 *m) +{ + schedule_work(&m->power_work); + flush_work(&m->power_work); +} + +static void mt_power_awake_after_join_test(struct kunit *test) +{ + struct mt_power_test_context *ctx = test->priv; + struct mt7932 *m = ctx->m; + unsigned int join; + + /* The old cache skipped the second SET in every mode-0 association. + * Exercise reconnect too: each session must submit both phases again. + */ + for (join = 0; join < 2; join++) { + m->connected = false; + KUNIT_ASSERT_EQ(test, mt_power_test_prejoin(m), 0); + KUNIT_EXPECT_EQ(test, ctx->commands, join * 2 + 1); + KUNIT_EXPECT_EQ(test, ctx->mode, 0); + KUNIT_EXPECT_FALSE(test, m->power_applied_valid); + m->connected = true; + mt_power_test_run(m); + KUNIT_EXPECT_EQ(test, ctx->commands, join * 2 + 2); + KUNIT_EXPECT_EQ(test, ctx->mode, 0); + KUNIT_EXPECT_TRUE(test, m->power_applied_valid); + /* Deduplication remains valid within this associated session. */ + mt_power_test_run(m); + KUNIT_EXPECT_EQ(test, ctx->commands, join * 2 + 2); + } +} + +static void mt_power_tim_gates_sleep_test(struct kunit *test) +{ + struct mt_power_test_context *ctx = test->priv; + struct mt7932 *m = ctx->m; + unsigned int mode; + + for (mode = 1; mode <= 2; mode++) { + m->power_mode = mode; + KUNIT_ASSERT_EQ(test, mt_power_test_prejoin(m), 0); + KUNIT_EXPECT_FALSE(test, m->power_tim); + m->connected = true; + mt_power_test_run(m); + KUNIT_EXPECT_EQ(test, ctx->commands, mode * 2 - 1); + KUNIT_EXPECT_FALSE(test, m->power_applied_valid); + m->power_tim = true; + mt_power_test_run(m); + KUNIT_EXPECT_EQ(test, ctx->commands, mode * 2); + KUNIT_EXPECT_EQ(test, ctx->mode, mode); + KUNIT_EXPECT_EQ(test, m->power_applied, mode); + } +} + +static void mt_power_failed_submission_retry_test(struct kunit *test) +{ + struct mt_power_test_context *ctx = test->priv; + struct mt7932 *m = ctx->m; + + ctx->error = -EIO; + KUNIT_EXPECT_EQ(test, mt_power_test_prejoin(m), -EIO); + KUNIT_EXPECT_FALSE(test, m->power_applied_valid); + ctx->error = 0; + KUNIT_ASSERT_EQ(test, mt_power_test_prejoin(m), 0); + m->connected = true; + ctx->error = -EIO; + mt_power_test_run(m); + KUNIT_EXPECT_FALSE(test, m->power_applied_valid); + ctx->error = 0; + mt_power_test_run(m); + KUNIT_EXPECT_EQ(test, ctx->commands, 4U); + KUNIT_EXPECT_TRUE(test, m->power_applied_valid); +} + +static void mt_power_terminal_session_test(struct kunit *test) +{ + struct mt_power_test_context *ctx = test->priv; + struct mt7932 *m = ctx->m; + unsigned int state; + + KUNIT_ASSERT_EQ(test, mt_power_test_prejoin(m), 0); + for (state = 0; state < 6; state++) { + m->stopping = state == 0; + m->rf_ready = state != 1; + m->connected = state != 2; + m->disconnecting = state == 3; + m->peer_valid = state != 4; + m->link_failed = state == 5; + mt_power_test_run(m); + KUNIT_EXPECT_EQ(test, ctx->commands, 1U); + KUNIT_EXPECT_FALSE(test, m->power_applied_valid); + } +} + +static struct kunit_case mt_power_test_cases[] = { + KUNIT_CASE(mt_power_awake_after_join_test), + KUNIT_CASE(mt_power_tim_gates_sleep_test), + KUNIT_CASE(mt_power_failed_submission_retry_test), + KUNIT_CASE(mt_power_terminal_session_test), + {} +}; + +static struct kunit_suite mt_power_test_suite = { + .name = "mt7932-power", + .init = mt_power_test_init, + .test_cases = mt_power_test_cases, +}; + +kunit_test_suite(mt_power_test_suite); diff --git a/drivers/net/wireless/mediatek/mt7932/power.c b/drivers/net/wireless/mediatek/mt7932/power.c index 62537ae867d90..f3c1c42863c9c 100644 --- a/drivers/net/wireless/mediatek/mt7932/power.c +++ b/drivers/net/wireless/mediatek/mt7932/power.c @@ -10,7 +10,12 @@ static int mt_power_submit(struct mt7932 *m, u8 mode) u8 body[4] = { 0, mode, 0, 0 }; /* owned BSS0, not a station/WTBL ID */ int ret; - ret = mt_request(m, 5, true, true, false, body, sizeof(body)); +#if IS_ENABLED(CONFIG_MT7932_RF_KUNIT_TEST) + if (m->power_test_send) + ret = m->power_test_send(m, body, sizeof(body)); + else +#endif + ret = mt_request(m, 5, true, true, false, body, sizeof(body)); m->power_applied_valid = !ret; if (!ret) m->power_applied = mode; @@ -22,11 +27,19 @@ static int mt_power_submit(struct mt7932 *m, u8 mode) int mt_power_init(struct mt7932 *m) { unsigned long flags; + int ret; spin_lock_irqsave(&m->response_lock, flags); m->power_tim = false; spin_unlock_irqrestore(&m->response_lock, flags); - return mt_power_submit(m, 0); + ret = mt_power_submit(m, 0); + /* This SET precedes EA/40. It cannot attest to the power policy of the + * associated BSS, whose authentication and scheduling firmware owns. + * Invalidate even on success so power_work submits the desired mode + * after association, including mode 0 when userspace disabled PS. + */ + m->power_applied_valid = false; + return ret; } /* command_mutex fences BSS reuse/retirement while the SET is submitted. */ From 3ce7b58f735d21def32b6c98aedbfcfde4e74eea Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:00:13 +1100 Subject: [PATCH 1409/1579] PCI: apple: piodma: Let the device tree alone select the J700 bootstrap The J700 radio bootstrap already runs only when the board device tree points the PCIe host's "apple,piodma" property at the engine. On top of that, the enumerate module parameter could turn it off again for diagnosis. Nothing sets it, and a board that should stay root-port-only says so in its device tree by leaving the property out. Drop the parameter so the device tree is the only switch. The bootstrap also logged every step at info level: the arena addresses, each interrupt, the request packet, the raw completion words and the validation, about eight lines of hex on every boot. Keep the single BOOTSTRAP_VALIDATED line and the error paths at their levels and move the step-by-step trace to debug, where dynamic debug can still show it. No change to what the bootstrap does. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/PCI/controller/apple-t8140.rst | 7 ++- drivers/pci/controller/Kconfig | 3 +- .../pci/controller/pcie-apple-piodma-diag.c | 50 +++++++++---------- 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 7d342d1c1b4d1..4d016f4b4ecc9 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -10,10 +10,9 @@ The root bridge forwards their BARs and DART translates endpoint DMA. This is a bring-up implementation, not general T8140 PCIe support. Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``. The board device tree selects the bootstrap by pointing the host's ``apple,piodma`` property at the engine, and -the kernel then runs it during the host probe. Passing -``pcie_apple_piodma_diag.enumerate=0`` keeps the root-only diagnostic path, -which rejects downstream configuration access. A kernel built without the -option always uses that root-only path. +the kernel then runs it during the host probe. A host without that property, +or a kernel built without the option, uses the root-only path, which rejects +downstream configuration access. Build the host and its DART in as well: the supplier retains published DMA and has no removal contract. Keeping the supplier and host built in avoids a module dependency on a provider whose lifetime cannot end at unload. diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index b758e0682ffd8..d9dc6b73aaae2 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -65,8 +65,7 @@ config PCIE_APPLE_PIODMA_DIAG Build the experimental SID17 retained bootstrap provider. It submits one fixed request, validates typed ECAM and then enumerates the two Neo radio functions when the board device tree links the PCIe host - to this engine. The enumerate=0 parameter keeps the host on its - root-port-only path instead. + to this engine. A host without that link stays root-port-only. The host and supplier must both be built in: supplier removal cannot release a published DMA arena safely. Suspend-to-idle sleep retains that arena and restores the configuration without repeating a request. diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 61b72a987d23f..2f2cde5490a27 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -57,12 +57,8 @@ /* * The board DT opts in by pointing the host's "apple,piodma" at this engine; - * a host without that property stays root-port-only. enumerate=0 keeps every - * host on its root-port-only path for diagnosis. + * a host without that property stays root-port-only. */ -static bool enumerate = true; -module_param(enumerate, bool, 0400); -MODULE_PARM_DESC(enumerate, "Prime once, validate native ECAM, and enumerate the two Neo functions (default: on)"); struct apple_piodma_request { u16 offset; @@ -290,11 +286,11 @@ static int apple_piodma_diag_arena(struct apple_piodma_diag *diag) memset(slot + PIODMA_DATA_OFFSET, 0xa5, PAGE_SIZE); *(__le32 *)(slot + PIODMA_DATA_OFFSET) = cpu_to_le32(U32_MAX); - dev_info(diag->dev, - "owned SID17 slot=%d dma=%#llx descriptor=%#llx completion=%#llx data=%#llx\n", - i, dma, dma + diag->pointer_prefix, - dma + diag->pointer_prefix + PIODMA_COMPLETION_OFFSET, - dma + PIODMA_DATA_OFFSET); + dev_dbg(diag->dev, + "owned SID17 slot=%d dma=%#llx descriptor=%#llx completion=%#llx data=%#llx\n", + i, dma, dma + diag->pointer_prefix, + dma + diag->pointer_prefix + PIODMA_COMPLETION_OFFSET, + dma + PIODMA_DATA_OFFSET); } dma_wmb(); return 0; @@ -389,8 +385,8 @@ static irqreturn_t apple_piodma_diag_irq(int irq, void *data) count = atomic_inc_return(&diag->irq_count); WRITE_ONCE(diag->last_irq_status, status); WRITE_ONCE(diag->last_irq_tag, tag); - dev_info(diag->dev, "irq=%d fabric=%#x status=%#x tag=%#x active=%#x\n", - count, fabric, status, tag, active); + dev_dbg(diag->dev, "irq=%d fabric=%#x status=%#x tag=%#x active=%#x\n", + count, fabric, status, tag, active); if (count >= PIODMA_IRQ_LIMIT || status & ~PIODMA_IRQ_COMPLETE || (status && (!tag || tag > active))) { WRITE_ONCE(diag->irq_fault, true); @@ -526,11 +522,11 @@ static int apple_piodma_diag_submit(struct apple_piodma_diag *diag, unsigned int return ret ?: -EIO; slot->submitted = true; WRITE_ONCE(diag->active_tag, tag); - dev_info(diag->dev, - "request=%u function=%u offset=%#x source=%#llx packet=%08x:%08x:%08x:%08x tag=%u\n", - index, request->function, request->offset, source, - le32_to_cpu(packet[0]), le32_to_cpu(packet[1]), - le32_to_cpu(packet[2]), le32_to_cpu(packet[3]), tag); + dev_dbg(diag->dev, + "request=%u function=%u offset=%#x source=%#llx packet=%08x:%08x:%08x:%08x tag=%u\n", + index, request->function, request->offset, source, + le32_to_cpu(packet[0]), le32_to_cpu(packet[1]), + le32_to_cpu(packet[2]), le32_to_cpu(packet[3]), tag); writel(tag, diag->engine + 0xd8); writel(upper_32_bits(descriptor) & 0x3ff, diag->engine + 0x50); writel(lower_32_bits(descriptor), diag->engine + 0x4c); @@ -543,11 +539,11 @@ static int apple_piodma_diag_submit(struct apple_piodma_diag *diag, unsigned int for (i = 0; i < ARRAY_SIZE(slot->completion); i++) slot->completion[i] = READ_ONCE(completion[i]); slot->observed = true; - dev_info(diag->dev, - "response=%u waited=%lu matched_status=%#x word=%#x completion=%08x:%08x:%08x:%08x\n", - index, waited, READ_ONCE(slot->matched_status), slot->result, - le32_to_cpu(slot->completion[0]), le32_to_cpu(slot->completion[1]), - le32_to_cpu(slot->completion[2]), le32_to_cpu(slot->completion[3])); + dev_dbg(diag->dev, + "response=%u waited=%lu matched_status=%#x word=%#x completion=%08x:%08x:%08x:%08x\n", + index, waited, READ_ONCE(slot->matched_status), slot->result, + le32_to_cpu(slot->completion[0]), le32_to_cpu(slot->completion[1]), + le32_to_cpu(slot->completion[2]), le32_to_cpu(slot->completion[3])); if (!waited) return -ETIMEDOUT; if (READ_ONCE(diag->irq_fault) || !READ_ONCE(slot->matched_status) || @@ -569,15 +565,15 @@ static int apple_piodma_diag_submit(struct apple_piodma_diag *diag, unsigned int if (ret || READ_ONCE(diag->irq_fault)) return ret ?: -EIO; } - dev_info(diag->dev, - "READ_VALIDATED request=%u word=%#x FIFO=%#x command=%#x; retained slots stable\n", - index, slot->result, fifo, command); + dev_dbg(diag->dev, + "READ_VALIDATED request=%u word=%#x FIFO=%#x command=%#x; retained slots stable\n", + index, slot->result, fifo, command); return 0; } bool apple_piodma_bootstrap_enabled(void) { - return enumerate && of_machine_is_compatible("apple,j700") && + return of_machine_is_compatible("apple,j700") && of_machine_is_compatible("apple,t8140"); } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_enabled); @@ -844,7 +840,7 @@ static int apple_piodma_diag_probe(struct platform_device *pdev) platform_set_drvdata(pdev, diag); /* Publish the CPU-owned arena and permanent device reference together. */ smp_store_release(&active_diag, diag); - dev_info(dev, "bootstrap supplier ready; no command submitted during probe\n"); + dev_dbg(dev, "bootstrap supplier ready; no command submitted during probe\n"); return 0; err_arena: From 575a429756818434e3711268c3d179f516102b05 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 01:59:49 +1100 Subject: [PATCH 1410/1579] usb: typec: tipd: Keep a probe-time mux hold for the ready worker CD321x places the connector in SAFE during port registration, before its workers may run. That request uses the Type-C class handle rather than the driver's mux handle. A transient timeout consequently leaves no held mux series: only state_valid is cleared, and initialization continues. In particular, no timeout start time or forced SAFE revalidation survives that initial failure. Apply the initial SAFE state through the worker's handle and feed a transient timeout into the existing held-mux machinery. Record its backoff and time ceiling while INITIALIZING, without enabling or scheduling the workers. ready() starts the normal fresh read, which reapplies SAFE before the connected state. Retain the existing probe deferral for an unavailable fixed host and the ordinary retry policy for non-transient providers. Exercise failed probe SAFE, delayed ready, success without an IRQ, persistent timeout, and unchanged non-transient and deferred-provider behavior. Also cover the ATC USB2-to-pin-D first handoff: a refused lock must retain the pending PIPE and powered-off SS port until a later mux request succeeds. Log the pre-lock PIPE registers under dynamic debug to distinguish a controller acknowledgment failure from lane setup failures. The first USB3_DP request already propagates its timeout through the runtime held-mux path. This change closes the earlier initialization gap; it does not establish why a particular controller failed to acknowledge. Fixes: ed289aabb019 ("usb: typec: tipd: Keep applying a Type-C state that a busy mux holds up") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 28 ++++++ drivers/phy/apple/atc.c | 5 ++ drivers/usb/typec/tipd/cd321x-mux-test.c | 85 +++++++++++++++++++ drivers/usb/typec/tipd/cd321x-pm-test.c | 22 +++++ drivers/usb/typec/tipd/cd321x-pm.h | 13 +-- drivers/usb/typec/tipd/core.c | 39 ++++++--- drivers/usb/typec/tipd/tps6598x.h | 1 + .../selftests/drivers/gpu/apple-dcp/pm.py | 49 +++++++++-- 8 files changed, 218 insertions(+), 24 deletions(-) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index 366b4b86f115e..e569cb4af4b44 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -857,6 +857,33 @@ static void atcphy_test_election_held_usb(struct kunit *test) atcphy_test_election_redrive(test, atcphy); } +/* First pin-D handoff after probe: USB2 lanes and a parked PIPE. */ +static void atcphy_test_boot_pipe_retry(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_election(test); + struct atcphy_test_election *election = test->priv; + + atcphy->mode = APPLE_ATCPHY_MODE_USB2; + atcphy->typec_mode = APPLE_ATCPHY_MODE_USB2; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + election->held = true; + KUNIT_ASSERT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB3_DP); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + KUNIT_EXPECT_TRUE(test, atcphy->pipe_pending); + KUNIT_EXPECT_TRUE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&atcphy->usb3_port_work)); + + /* The same request must finish the handoff, not use the mode cache. */ + election->held = false; + memset(election->steps.log, 0, sizeof(election->steps.log)); + KUNIT_ASSERT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, election->steps.log, "g01"); + KUNIT_EXPECT_TRUE(test, atcphy_usb3_port_ready(atcphy)); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); +} + /* * Only the fixed-hub ports of a J700 have the Type-C driver retry a timeout * until it passes; every M1 and M2 port, and fixed-hub ports on other PHYs, @@ -1210,6 +1237,7 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_host_start_without_usb3), KUNIT_CASE(atcphy_test_election_held_safe), KUNIT_CASE(atcphy_test_election_held_usb), + KUNIT_CASE(atcphy_test_boot_pipe_retry), KUNIT_CASE(atcphy_test_transient_timeouts), #if IS_REACHABLE(CONFIG_USB) KUNIT_CASE(atcphy_test_usb3_root), diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 84cef035e0865..e5ed6f7c5c8e6 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1233,6 +1233,11 @@ static int atcphy_pipehandler_lock_timeout(struct apple_atcphy *atcphy, u32 time dev_dbg(atcphy->dev, "PIPE request: req=%#x mode=%u pipe=%u host=%u dirty=%u\n", requested, atcphy->mode, atcphy->pipe_state, atcphy->host_active, atcphy->mode_dirty); + dev_dbg(atcphy->dev, "PIPE before lock: ack=%#x aon=%#x mux=%#x override=%#x\n", + readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK), + readl(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN), + readl(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL), + readl(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE)); if (requested & PIPEHANDLER_LOCK_EN) { dev_dbg(atcphy->dev, "Pipehandler lock already requested\n"); } else { diff --git a/drivers/usb/typec/tipd/cd321x-mux-test.c b/drivers/usb/typec/tipd/cd321x-mux-test.c index 5a95956d0adb4..13fe97d64520a 100644 --- a/drivers/usb/typec/tipd/cd321x-mux-test.c +++ b/drivers/usb/typec/tipd/cd321x-mux-test.c @@ -908,7 +908,92 @@ static void cd321x_mux_held_then_other_error_test(struct kunit *test) cd321x_mux_test_expect_usb(test, t); } +/* A dock already attached when probe's first SAFE request times out. */ +static struct cd321x_mux_test *cd321x_mux_test_boot(struct kunit *test, bool transient) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, transient); + + cd321x_pm_init(&t->cd.pm); + t->status = TPS_STATUS_PLUG_PRESENT; + t->data_status = TPS_DATA_STATUS_DATA_CONNECTION | TPS_DATA_STATUS_USB2_CONNECTION | + TPS_DATA_STATUS_USB3_CONNECTION; + t->phy.held = true; + KUNIT_ASSERT_EQ(test, cd321x_init_mux(&t->cd), 0); + KUNIT_EXPECT_STREQ(test, t->phy.log, "S"); + KUNIT_EXPECT_EQ(test, t->cd.pm.phase, CD321X_PM_INITIALIZING); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.update_work)); + KUNIT_EXPECT_FALSE(test, t->cd.state_valid); + + return t; +} + +static void cd321x_mux_boot_retries_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_boot(test, true); + + KUNIT_ASSERT_NE(test, t->cd.pm.mux_retry_ms, 0U); + t->cd.tps.data->ready(&t->cd.tps); + cd321x_mux_test_expect_read_now(test, t); + while (t->now_ms < TEST_ELECTIONS_MS) + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + KUNIT_EXPECT_NE(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + /* No interrupt or physical replug is needed after the lock is released. */ + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +static void cd321x_mux_boot_ceiling_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_boot(test, true); + unsigned int i; + + t->cd.tps.data->ready(&t->cd.tps); + cd321x_mux_test_expect_read_now(test, t); + for (i = 0; i < 4 * CD321X_MUX_RETRIES && cd321x_mux_test_next(t); i++) + ; + KUNIT_EXPECT_GE(test, t->now_ms, CD321X_MUX_HELD_MS); + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0U); + KUNIT_EXPECT_FALSE(test, t->cd.state_valid); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.update_work)); +} + +/* M1/M2 providers retain their ordinary three-attempt recovery budget. */ +static void cd321x_mux_boot_nontransient_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_boot(test, false); + + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0U); + t->cd.tps.data->ready(&t->cd.tps); + cd321x_mux_test_expect_read_now(test, t); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_LT(test, t->now_ms, TEST_ELECTIONS_MS); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0U); +} + +static void cd321x_mux_boot_defer_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + + cd321x_pm_init(&t->cd.pm); + t->phy.held = true; + t->phy.held_ret = -EAGAIN; + KUNIT_EXPECT_EQ(test, cd321x_init_mux(&t->cd), -EPROBE_DEFER); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0U); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); +} + static struct kunit_case cd321x_mux_cases[] = { + KUNIT_CASE(cd321x_mux_boot_retries_test), + KUNIT_CASE(cd321x_mux_boot_ceiling_test), + KUNIT_CASE(cd321x_mux_boot_nontransient_test), + KUNIT_CASE(cd321x_mux_boot_defer_test), KUNIT_CASE(cd321x_mux_held_past_budget_test), KUNIT_CASE(cd321x_mux_held_backs_off_test), KUNIT_CASE(cd321x_mux_held_event_takes_over_test), diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index aef346d7bb4d8..5598f65af9e06 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -1070,7 +1070,29 @@ static void cd321x_dp_send_test(struct kunit *test) KUNIT_EXPECT_EQ(test, cd321x_dp_send(irq, irq, true), CD321X_DP_SEND_EDGE); } +/* Probe joins the same timed series, without making the port usable early. */ +static void cd321x_probe_mux_held_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_init(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm, 1000), CD321X_MUX_RETRY_MS); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_INITIALIZING); + KUNIT_EXPECT_EQ(test, pm.attempts_left, 0U); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_ready(&pm)); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + KUNIT_EXPECT_EQ(test, pm.mux_since_ms, 1000U); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm, 1000 + CD321X_MUX_HELD_MS), 0U); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); +} + static struct kunit_case cd321x_pm_cases[] = { + KUNIT_CASE(cd321x_probe_mux_held_test), KUNIT_CASE(cd321x_reset_first_irq_before_ready_test), KUNIT_CASE(cd321x_reset_first_irq_during_prepare_test), KUNIT_CASE(cd321x_reset_setup_retry_test), diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h index 8260e559b0c87..18f586682fceb 100644 --- a/drivers/usb/typec/tipd/cd321x-pm.h +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -314,11 +314,14 @@ static inline unsigned int cd321x_pm_mux_held(struct cd321x_pm_state *pm, pm->attempts_left = 0; return 0; } - if (pm->phase == CD321X_PM_RUNNING) - pm->attempts_left = CD321X_RESUME_ATTEMPTS; - else - pm->attempts_left++; - pm->phase = CD321X_PM_REVALIDATE; + /* A probe-time SAFE timeout joins this series without enabling workers. */ + if (pm->phase != CD321X_PM_INITIALIZING) { + if (pm->phase == CD321X_PM_RUNNING) + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + else + pm->attempts_left++; + pm->phase = CD321X_PM_REVALIDATE; + } pm->mux_retries++; pm->mux_retry_ms = clamp(2 * pm->mux_retry_ms, CD321X_MUX_RETRY_MS, CD321X_MUX_RETRY_MAX_MS); diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index dd6f7b99e698a..cc6a15384ca9f 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -998,10 +998,11 @@ static void cd321x_retry_held_mux(struct cd321x *cd321x) if (!cd321x->pm.mux_retry_ms) dev_info(cd321x->tps.dev, "Type-C mux held up, retrying\n"); delay = cd321x_pm_mux_held(&cd321x->pm, now_ms); - if (delay) + /* Probe records the hold, but ready() must enable the workers first. */ + if (delay && cd321x->pm.phase != CD321X_PM_INITIALIZING) mod_delayed_work(system_freezable_wq, &cd321x->resume_work, msecs_to_jiffies(delay)); - else + else if (!delay) dev_warn(cd321x->tps.dev, "Type-C mux still held up after %u retries in %u s, waiting for the next event\n", retries, held_ms / 1000); @@ -2080,6 +2081,8 @@ static void cd321x_ready(struct tps6598x *tps) guard(mutex)(&tps->lock); /* Include failures reported while the port was still being initialized. */ if (cd321x_pm_ready(&cd321x->pm)) { + dev_dbg(tps->dev, "Type-C probe ready: held=%u\n", + !!cd321x->pm.mux_retry_ms); cd321x->state_valid = false; mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); } @@ -2142,6 +2145,27 @@ VISIBLE_IF_KUNIT void cd321x_init_work(struct cd321x *cd321x) } EXPORT_SYMBOL_IF_KUNIT(cd321x_init_work); +/* Use the same handle as the worker: timeout provenance belongs to the handle. */ +VISIBLE_IF_KUNIT int cd321x_init_mux(struct cd321x *cd321x) +{ + bool held; + int ret; + + guard(mutex)(&cd321x->tps.lock); + cd321x->state.alt = NULL; + cd321x->state.mode = TYPEC_STATE_SAFE; + cd321x->state.data = NULL; + ret = cd321x_mux_set(cd321x, &held); + cd321x->state_valid = !ret; + if (ret == -EAGAIN) + return -EPROBE_DEFER; + if (held) + cd321x_retry_held_mux(cd321x); + + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(cd321x_init_mux); + static int cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) { @@ -2178,20 +2202,14 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) goto err_put_mux; } - cd321x->state.alt = NULL; - cd321x->state.mode = TYPEC_STATE_SAFE; - cd321x->state.data = NULL; - ret = typec_set_mode(tps->port, TYPEC_STATE_SAFE); - cd321x->state_valid = !ret; /* * A fixed host can register its SuperSpeed root after the mux provider. * Wait for that consumer at probe instead of exhausting the cable retry * budget before it exists. Host startup does not depend on this port. */ - if (ret == -EAGAIN) { - ret = -EPROBE_DEFER; + ret = cd321x_init_mux(cd321x); + if (ret) goto err_put_switch; - } cd321x->tbt_notifier.notifier_call = cd321x_tbt_notify; ret = typec_thunderbolt_switch_register_notifier(cd321x->tbt_switch, @@ -2712,6 +2730,7 @@ int tipd_init(struct tps6598x *tps) int ret, cleanup_ret; bool reset_allowed = true; + dev_dbg(tps->dev, "Type-C probe start\n"); tps->reset = devm_gpiod_get_optional(tps->dev, "reset", GPIOD_OUT_LOW); if (IS_ERR(tps->reset)) return dev_err_probe(tps->dev, PTR_ERR(tps->reset), diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index f6d51724a1f5d..28900eb3ac9a7 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -490,6 +490,7 @@ static inline void tipd_debugfs_exit(void) { } #if IS_ENABLED(CONFIG_KUNIT) void cd321x_init_work(struct cd321x *cd321x); +int cd321x_init_mux(struct cd321x *cd321x); bool cd321x_tbt_switch_keep_waiting(struct device *dev); struct typec_thunderbolt_switch * cd321x_tbt_switch_get(struct device *dev, struct fwnode_handle *fwnode); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 3c2f2b8df2bd5..5eaf7fe2c95e2 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -499,6 +499,7 @@ def function(text, name): }; static struct fwnode_handle connector; static int mux_error, notifier_error, switch_error; +static bool mux_transient; static int ports,altmodes,muxes,switches,notifiers,stop_calls; struct typec_thunderbolt_switch; static bool keep_waiting=true; @@ -521,7 +522,14 @@ def function(text, name): static void *fwnode_typec_thunderbolt_switch_get(struct fwnode_handle *f) { (void)f;if(switch_error)return (void *)(intptr_t)switch_error;switches++;return &switches; } -static int typec_set_mode(void *p,unsigned long m) { (void)p;assert(m==TYPEC_STATE_SAFE);return mux_error; } +static int cd321x_mux_set(struct cd321x *c,bool *held) { + assert(c->state.mode==TYPEC_STATE_SAFE); + *held=mux_error==-ETIMEDOUT&&mux_transient;return mux_error; +} +static void cd321x_retry_held_mux(struct cd321x *c) { + assert(c->pm.phase==CD321X_PM_INITIALIZING); + assert(cd321x_pm_mux_held(&c->pm,0)); +} static int cd321x_tbt_notify(struct notifier_block *n,unsigned long e,void *d) { (void)n;(void)e;(void)d;return 0; } static int typec_thunderbolt_switch_register_notifier(void *s,struct notifier_block *n) { (void)n;if(!s)return 0;if(notifier_error)return notifier_error;notifiers++;return 0; @@ -540,11 +548,13 @@ def function(text, name): production = switch_get.replace('VISIBLE_IF_KUNIT', 'static', 1) + '\n' init_work = block(core, 'VISIBLE_IF_KUNIT void cd321x_init_work(') production += init_work.replace('VISIBLE_IF_KUNIT', 'static', 1) + '\n' + init_mux = block(core, 'VISIBLE_IF_KUNIT int cd321x_init_mux(') + production += init_mux.replace('VISIBLE_IF_KUNIT', 'static', 1) + '\n' production += function(core, 'cd321x_register_port') + '\n' + function(core, 'cd321x_unregister_port') controls = r''' static void reset(struct cd321x *c) { memset(c,0,sizeof(*c));connector.refs=0; - mux_error=notifier_error=switch_error=0;stop_calls=0;keep_waiting=true; + mux_error=notifier_error=switch_error=0;stop_calls=0;keep_waiting=true;mux_transient=false; assert(!ports&&!altmodes&&!muxes&&!switches&&!notifiers); } static void balanced(void) { @@ -608,6 +618,13 @@ def function(text, name): reset(&c);mux_error=-EIO; assert(cd321x_register_port(&c.tps,&connector)==0&&!c.state_valid); cd321x_unregister_port(&c.tps);balanced(); + /* A probe-time transient timeout keeps its series until ready enables work. */ + reset(&c);mux_error=-ETIMEDOUT;mux_transient=true; + assert(cd321x_register_port(&c.tps,&connector)==0&&!c.state_valid); + assert(c.pm.phase==CD321X_PM_INITIALIZING&&c.pm.mux_retry_ms&&c.pm.force_reconnect); + assert(!c.resume_work.pending&&!c.update_work.pending&&cd321x_pm_ready(&c.pm)); + assert(c.pm.mux_retry_ms&&c.pm.force_reconnect); + cd321x_unregister_port(&c.tps);balanced(); /* Missing router defers while suppliers may still bind, then is optional. */ reset(&c);switch_error=-EPROBE_DEFER; assert(cd321x_register_port(&c.tps,&connector)==-EPROBE_DEFER);balanced(); @@ -710,7 +727,7 @@ def kconfig(root, output): def probe_dispatch(root): - """Connect real port probe -> typec_set_mode -> mux core -> PM wrapper.""" + """Connect real port probe -> init_mux -> mux core -> PM wrapper.""" unit = ownership(root) unit = unit.replace('int main(void)', 'static int __attribute__((unused)) ownership_fixture_main(void)', 1) unit = unit.replace('#define IS_ERR_OR_NULL(p) (!(p))', '#define IS_ERR(p) ((intptr_t)(p)<0)\n #define IS_ERR_OR_NULL(p) (!(p)||IS_ERR(p))') @@ -719,13 +736,20 @@ def probe_dispatch(root): # Type-C and mutex declarations already belong to the shared PM fixture. port = port.replace('struct typec_mux_state { void *alt,*data; unsigned long mode; };', '') port = port.replace('#define guard(kind) (void)', '') - port = port.replace('static int typec_set_mode(void *p,unsigned long m) { (void)p;assert(m==TYPEC_STATE_SAFE);return mux_error; }', '') + start = port.index('static int cd321x_mux_set(') + end = port.index('static void cd321x_retry_held_mux(', start) + core = (root / 'drivers/usb/typec/tipd/core.c').read_text() + dispatch = block(core, 'static bool cd321x_mux_held(') + '\n' + block(core, 'static int cd321x_mux_set(') + timeout = block((root / 'drivers/usb/typec/mux.c').read_text(), + 'bool typec_mux_timeout_transient(') + port = port[:start] + timeout + '\n' + dispatch + '\n' + port[end:] port = port.replace('(void)f;t->port=t;ports++;return 0;', '(void)f;t->port=&probe_port;ports++;return 0;') - # Real class dispatch requires the port's mux; its allocator remains a mock. + # Both handles refer to the same route; its allocator remains a mock. declarations = 'struct typec_port { struct typec_mux *mux; };\nstatic struct typec_port probe_port;\n' - dispatch = block((root / 'drivers/usb/typec/class.c').read_text(), 'int typec_set_mode(') marker = port.index('static int tps6598x_register_port(') - port = port[:marker] + declarations + dispatch + '\n' + port[marker:] + port = port[:marker] + declarations + port[marker:] + port = port.replace('(void)f;muxes++;return &muxes;', + '(void)f;muxes++;return probe_port.mux;') import re # Only helpers used at this boundary belong in this translation unit; # the separate probe group exercises the complete retry state machine. @@ -733,13 +757,14 @@ def probe_dispatch(root): 'cd321x_pm_begin_read', 'cd321x_pm_snapshot_ready', 'cd321x_pm_complete', 'cd321x_pm_can_update', # which, once retries of a held mux exist, ends them on removal - 'cd321x_pm_mux_done'} + 'cd321x_pm_mux_done', 'cd321x_pm_mux_held'} for match in list(re.finditer(r'^static inline (?:void|bool|unsigned int) (cd321x_\w+)\(', port, re.M))[::-1]: if match.group(1) not in used: body = block(port, match.group()) port = port[:match.start()] + port[match.start() + len(body):] port = port.replace('static int mux_error, notifier_error, switch_error;', 'static int notifier_error, switch_error;') + port = port.replace('static bool mux_transient;', '') existing = set(re.findall(r'^\s*#define (\w+)', unit, re.M)) port = '\n'.join(line for line in port.splitlines() if not (line.startswith('#define ') and line.split()[1].split('(')[0] in existing)) @@ -770,7 +795,13 @@ def probe_dispatch(root): memset(&c,0,sizeof(c));prepare(&p,true,true,false);resume_error=-EIO;before=writes; assert(cd321x_register_port(&c.tps,&connector)==0 && !c.state_valid && writes==before); balanced(&p);cd321x_unregister_port(&c.tps);port_balance(); - puts("PASS: actual CD321x probe -> typec_set_mode -> typec_mux_set -> bounded PM wrapper; delayed root reprobe and rollback references"); + memset(&c,0,sizeof(c));prepare(&p,true,true,false);resume_error=0; + provider.transient_timeouts=true;fail_caller="atcphy_pipehandler_lock_timeout"; + assert(cd321x_register_port(&c.tps,&connector)==0&&!c.state_valid); + assert(c.pm.phase==CD321X_PM_INITIALIZING&&c.pm.mux_retry_ms&&c.pm.force_reconnect); + assert(!c.resume_work.pending&&!c.update_work.pending); + balanced(&p);cd321x_unregister_port(&c.tps);port_balance();fail_caller=NULL; + puts("PASS: actual CD321x probe -> init_mux -> typec_mux_set -> bounded PM wrapper; delayed root, initial PIPE timeout and rollback references"); } """ return unit + '\n' + port + '\n' + controls From 8df6578502e77c519fb6cbf2c83823ecc5367c9b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 01:59:53 +1100 Subject: [PATCH 1411/1579] usb: core: Retrain an unconnected SS port after a lane handover A fixed-hub host can enumerate a dock over USB2 before Type-C has brought up the SuperSpeed lanes. Restoring the SS root port's power already queues a warm reset, but that alone need not restart receiver detection after the partner has abandoned the earlier training attempt. Rebinding the PD port can repeat the same sequence without producing a working SS connection. Let the lane provider consume that queued reset immediately after it has restored the port, while retaining the hub lock and a runtime PM reference to the port. Send one warm reset and, only if it completes with no receiver, restart the link once through Disabled and RxDetect. Reuse the existing bounded reset-status wait and hub rescan; add no delayed recovery loop. Skip connected links, USB2 and non-root hubs, ports with children or owners, administrative disables, power holds and ports without the queued reset. Use this only after successful SS power restoration on a T8140 fixed-hub PHY. A training failure is reported without pretending that the PHY still owns a power hold which power_on() has already released. M1/M2 and ordinary Type-C ports retain their existing path. The late-SS asymmetric peer fix handles devices which had already enumerated on USB2 once the SS hub arrives. Exercise the actual helper on a software HCD: success on the warm reset, success only after receiver-detection restart, persistent absence with no new attempts, an already healthy link, a control error and port ownership. The host PM fixture also checks that retraining runs only after the lanes and PIPE are ready and port power has been restored. Fixes: 70f7c21e7d32 ("phy: apple: atc: Route the PIPE from the Type-C mux on a fixed-hub port") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 7 + drivers/usb/core/hub.c | 87 ++++++++++ drivers/usb/core/port-power-test.c | 150 +++++++++++++++++- include/linux/usb.h | 1 + .../selftests/drivers/gpu/apple-dcp/pm.py | 7 + 5 files changed, 251 insertions(+), 1 deletion(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index e5ed6f7c5c8e6..91ecba31cdc7f 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -5087,6 +5087,13 @@ static int atcphy_usb3_port_on(struct apple_atcphy *atcphy, struct atcphy_usb3_p ret = usb_hub_port_power_on(pm->root, 1); if (ret < 0) dev_warn(atcphy->dev, "Failed to power the SuperSpeed port on: %d\n", ret); + else if (!ret && atcphy->fixed_usb2 && atcphy->hw->has_t8140_dp) { + int retrain = usb_hub_port_retrain(pm->root, 1); + + if (retrain < 0) + dev_warn(atcphy->dev, "SuperSpeed handover retrain failed: %d\n", retrain); + /* Power was restored even if no receiver answered the bounded retrain. */ + } #endif return ret; } diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index c09591fe0ca68..e1e6bea87eb15 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -3479,6 +3479,93 @@ static void usb_unlock_port(struct usb_port *port_dev) __release(&port_dev->status_lock); } +/** + * usb_hub_port_retrain - restart an unconnected SuperSpeed root port + * @hdev: SuperSpeed root hub + * @port1: port number + * + * Called immediately after usb_hub_port_power_on(), under the same hub lock, + * once the caller has finished bringing up the lanes behind the port. Consume + * the queued warm reset here while keeping the port runtime-active. If that + * reset finds no receiver, restart detection once through Disabled/RxDetect. + * Each operation has a bounded wait; no further recovery work is scheduled. + * Connected links and ports with a child, an owner or a power hold are left + * alone. Only the caller's newly restored port is eligible. + * + * Return: 0 for a usable link, 1 for a port left alone, or a negative error. + * A failure does not reclaim the power hold released by power_on(). + */ +int usb_hub_port_retrain(struct usb_device *hdev, int port1) +{ + struct usb_hub *hub = usb_hub_to_struct_hub(hdev); + struct usb_interface *intf; + struct usb_port *port_dev; + u16 status, change; + int ret; + + device_lock_assert(&hdev->dev); + if (!hub || hub->disconnected || hub->quiescing) + return -ENODEV; + if (hdev->parent || !hub_is_superspeed(hdev)) + return -EOPNOTSUPP; + if (port1 < 1 || port1 > hdev->maxchild) + return -EINVAL; + port_dev = hub->ports[port1 - 1]; + if (port_dev->disabled || port_dev->held_off || port_dev->port_owner || + port_dev->child || port_dev->ignore_event || port_dev->early_stop || + !test_bit(port1, hub->warm_reset_bits)) + return 1; + + intf = to_usb_interface(hub->intfdev); + ret = usb_autopm_get_interface(intf); + if (ret < 0) + return ret; + ret = pm_runtime_resume_and_get(&port_dev->dev); + if (ret < 0) + goto put_intf; + usb_lock_port(port_dev); + ret = usb_hub_port_status(hub, port1, &status, &change); + if (ret) + goto unlock; + if (!usb_port_is_power_on(hub, status)) { + ret = 1; + goto unlock; + } + /* The caller still holds the hub lock from power_on(): this is its reset. */ + clear_bit(port1, hub->warm_reset_bits); + if ((status & (USB_PORT_STAT_CONNECTION | USB_PORT_STAT_ENABLE)) == + (USB_PORT_STAT_CONNECTION | USB_PORT_STAT_ENABLE) && + (status & USB_PORT_STAT_LINK_STATE) <= USB_SS_PORT_LS_U3) { + ret = 0; + goto unlock; + } + + dev_dbg(&port_dev->dev, "SS handover: warm reset after lane readiness\n"); + ret = set_port_feature(hdev, port1, USB_PORT_FEAT_BH_PORT_RESET); + if (!ret) + ret = hub_port_wait_reset(hub, port1, NULL, HUB_BH_RESET_TIME, true); + if (ret != -ENOTCONN) + goto rescan; + + dev_dbg(&port_dev->dev, "SS handover: restarting receiver detection\n"); + ret = hub_set_port_link_state(hub, port1, USB_SS_PORT_LS_SS_DISABLED); + if (!ret) + ret = hub_set_port_link_state(hub, port1, USB_SS_PORT_LS_RX_DETECT); + if (!ret) + ret = hub_port_wait_reset(hub, port1, NULL, HUB_BH_RESET_TIME, true); +rescan: + dev_dbg(&port_dev->dev, "SS handover: retrain result %d\n", ret); + set_bit(port1, hub->change_bits); + kick_hub_wq(hub); +unlock: + usb_unlock_port(port_dev); + pm_runtime_put(&port_dev->dev); +put_intf: + usb_autopm_put_interface(intf); + return ret; +} +EXPORT_SYMBOL_GPL(usb_hub_port_retrain); + #ifdef CONFIG_PM /* Check if a port is suspended(USB2.0 port) or in U3 state(USB3.0 port) */ diff --git a/drivers/usb/core/port-power-test.c b/drivers/usb/core/port-power-test.c index 181fa7ff0b8a8..aedab69626a90 100644 --- a/drivers/usb/core/port-power-test.c +++ b/drivers/usb/core/port-power-test.c @@ -31,6 +31,11 @@ struct port_power_test_hc { u16 change; unsigned int power_ons; bool fail_power_off; + unsigned int warm_resets; + unsigned int detect_restarts; + bool connect_on_warm; + bool connect_on_detect; + bool fail_warm_reset; }; struct port_power_test { @@ -171,8 +176,31 @@ static int port_power_test_hub_control(struct usb_hcd *hcd, u16 type_req, u16 va port_power_test_set_power(hcd, true); else if (value == USB_PORT_FEAT_RESET) hc->change |= USB_PORT_STAT_C_RESET; - else if (value == USB_PORT_FEAT_BH_PORT_RESET) + else if (value == USB_PORT_FEAT_BH_PORT_RESET) { + hc->warm_resets++; + if (hc->fail_warm_reset) + return -EIO; hc->change |= USB_PORT_STAT_C_RESET | USB_PORT_STAT_C_BH_RESET; + if (hc->connect_on_warm) { + hc->status &= ~USB_PORT_STAT_LINK_STATE; + hc->status |= USB_PORT_STAT_CONNECTION | USB_PORT_STAT_ENABLE; + } + } else if (value == USB_PORT_FEAT_LINK_STATE) { + u16 state = (index >> 3) & USB_PORT_STAT_LINK_STATE; + + hc->status &= ~USB_PORT_STAT_LINK_STATE; + hc->status |= state; + if (state == USB_SS_PORT_LS_SS_DISABLED) + hc->status &= ~(USB_PORT_STAT_CONNECTION | USB_PORT_STAT_ENABLE); + if (state == USB_SS_PORT_LS_RX_DETECT) { + hc->detect_restarts++; + if (hc->connect_on_detect) { + hc->status &= ~USB_PORT_STAT_LINK_STATE; + hc->status |= USB_PORT_STAT_CONNECTION | + USB_PORT_STAT_ENABLE; + } + } + } return 0; case ClearPortFeature: if (value == USB_PORT_FEAT_POWER && hc->fail_power_off) { @@ -928,7 +956,127 @@ static void port_power_test_connector_graph_legacy(struct kunit *test) } #endif +/* Keep the root locked from power-on through retraining, as the PHY does. */ +static struct port_power_test_hc * +port_power_test_retrain_begin(struct kunit *test, bool warm, bool detect) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_hc(t->shared_hcd); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); + scoped_guard(spinlock_irqsave, &hc->lock) { + hc->warm_resets = 0; + hc->detect_restarts = 0; + hc->connect_on_warm = warm; + hc->connect_on_detect = detect; + } + return hc; +} + +static void port_power_test_retrain_end(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_hc(t->shared_hcd); + + /* The synthetic controller has no device to enumerate after the check. */ + scoped_guard(spinlock_irqsave, &hc->lock) { + hc->status &= ~(USB_PORT_STAT_CONNECTION | USB_PORT_STAT_ENABLE | + USB_PORT_STAT_LINK_STATE); + hc->status |= USB_SS_PORT_LS_RX_DETECT; + } + usb_unlock_device(t->ss_root); +} + +static void port_power_test_retrain_warm(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_retrain_begin(test, true, false); + int usage = port_power_test_usage(t->ss_port); + + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, hc->warm_resets, 1U); + KUNIT_EXPECT_EQ(test, hc->detect_restarts, 0U); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage); + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), 1); + KUNIT_EXPECT_EQ(test, hc->warm_resets, 1U); + port_power_test_retrain_end(test); +} + +static void port_power_test_retrain_detect(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_retrain_begin(test, false, true); + + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, hc->warm_resets, 1U); + KUNIT_EXPECT_EQ(test, hc->detect_restarts, 1U); + port_power_test_retrain_end(test); +} + +static void port_power_test_retrain_bounded(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_retrain_begin(test, false, false); + + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), -ENOTCONN); + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), 1); + KUNIT_EXPECT_EQ(test, hc->warm_resets, 1U); + KUNIT_EXPECT_EQ(test, hc->detect_restarts, 1U); + port_power_test_retrain_end(test); +} + +static void port_power_test_retrain_healthy(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_retrain_begin(test, false, false); + + scoped_guard(spinlock_irqsave, &hc->lock) { + hc->status &= ~USB_PORT_STAT_LINK_STATE; + hc->status |= USB_PORT_STAT_CONNECTION | USB_PORT_STAT_ENABLE; + } + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, hc->warm_resets, 0U); + KUNIT_EXPECT_EQ(test, hc->detect_restarts, 0U); + port_power_test_retrain_end(test); +} + +static void port_power_test_retrain_error(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_retrain_begin(test, false, false); + + hc->fail_warm_reset = true; + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), -EIO); + KUNIT_EXPECT_EQ(test, hc->warm_resets, 1U); + KUNIT_EXPECT_EQ(test, hc->detect_restarts, 0U); + port_power_test_retrain_end(test); +} + +static void port_power_test_retrain_owner(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_retrain_begin(test, false, false); + + KUNIT_EXPECT_EQ(test, usb_hub_claim_port(t->ss_root, 1, (void *)t), 0); + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), 1); + KUNIT_EXPECT_EQ(test, usb_hub_release_port(t->ss_root, 1, (void *)t), 0); + KUNIT_EXPECT_EQ(test, hc->warm_resets, 0U); + KUNIT_EXPECT_EQ(test, hc->detect_restarts, 0U); + port_power_test_retrain_end(test); + usb_lock_device(t->hs_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->hs_root, 1), -EOPNOTSUPP); + usb_unlock_device(t->hs_root); +} + static struct kunit_case port_power_test_cases[] = { + KUNIT_CASE(port_power_test_retrain_warm), + KUNIT_CASE(port_power_test_retrain_detect), + KUNIT_CASE(port_power_test_retrain_bounded), + KUNIT_CASE(port_power_test_retrain_healthy), + KUNIT_CASE(port_power_test_retrain_error), + KUNIT_CASE(port_power_test_retrain_owner), #if IS_ENABLED(CONFIG_OF_DYNAMIC) KUNIT_CASE(port_power_test_connector_graph), KUNIT_CASE(port_power_test_connector_graph_legacy), diff --git a/include/linux/usb.h b/include/linux/usb.h index 8c14a6bd3e08b..25f6b0c2fb708 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -958,6 +958,7 @@ int usb_hub_release_port(struct usb_device *hdev, unsigned port1, /* port power switching for a driver that moves the lanes behind a port */ int usb_hub_port_power_off(struct usb_device *hdev, int port1); int usb_hub_port_power_on(struct usb_device *hdev, int port1); +int usb_hub_port_retrain(struct usb_device *hdev, int port1); /** * usb_make_path - returns stable device path in the usb tree diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 5eaf7fe2c95e2..faae722b9d8e1 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -214,6 +214,13 @@ def ownership(root, usb_unreachable=False): usb3_port_down=false; return 0; } +static int usb_hub_port_retrain(struct usb_device *u, int port) +{ + assert(u==&roots[3]&&port==1&&root_locked&&pm_refs==1&&!pm_phy->lock); + assert(!usb3_port_down&&pm_phy->host_active&&!pm_phy->mode_dirty&&!pm_phy->pipe_pending); + assert(pm_phy->pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3); + return 0; +} #endif ''' suffix=ns['suffix'].replace('int main(', 'int __attribute__((unused)) fixture_main(',1) From 9022f7e0bd803c2bb0497370785f6e430d52aefe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 01:59:54 +1100 Subject: [PATCH 1412/1579] usb: core: Rearm asymmetric peer recovery for a new lane handover The late-SuperSpeed peer reconnect guard survives the SS disconnect caused by its own USB2 power cycle. It also survives a Type-C mux teardown and handover, such as a tipd rebind, while the fixed USB2 hub remains attached. Devices which fall back to USB2 during that new handover then cannot get another peer reconnect after SuperSpeed returns. Count explicit SuperSpeed lane handovers when the port-power API acquires a new hold. Repeated calls on an existing hold, administrative disables and USB2 power cycles do not advance the count. Record which handover the USB2 worker serviced, so the guard permits one reconnect after a new handover while remaining latched over its own cycle. Snapshot the count before I/O: completion of an older cycle cannot consume a newer handover. Replace a pending request when a newer SS attachment becomes ready before the USB2 worker consumes it. Acquire the new device references under the existing peer/state locks and release replaced references after dropping the peer mutex. Keep the existing single worker and do not nest peer hub locks or add a retry loop. Cover the real port-power hold count and the production queue/decision helpers for a rebind, an identical pending request, its replacement, an old in-flight completion, self-cycle loop suppression and count wrap. Fixes: 5ec3b2c98e5c ("usb: core: Reconnect asymmetric hub peers after a late SuperSpeed connection") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/core/hub.c | 29 +++++++++++--- drivers/usb/core/hub.h | 20 +++++++++- drivers/usb/core/port-power-test.c | 62 ++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 7 deletions(-) diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index e1e6bea87eb15..7c5d5db928b11 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -1084,33 +1084,41 @@ enum hub_activation_type { static void hub_reconnect_ss_peer(struct usb_hub *hub) { struct usb_device *ss = hub->hdev; + struct usb_device *old_hs = NULL, *old_ss = NULL; struct usb_hub *parent, *hs_hub; struct usb_port *upstream, *peer; if (!ss->parent || !hub_is_superspeed(ss)) return; - guard(mutex)(&usb_port_peer_mutex); + mutex_lock(&usb_port_peer_mutex); parent = usb_hub_to_struct_hub(ss->parent); if (!parent || ss->portnum > ss->parent->maxchild) - return; + goto unlock; upstream = parent->ports[ss->portnum - 1]; peer = upstream->peer; /* Only firmware-described, asymmetric paths need this handover. */ if (!upstream->connector_node || !peer) - return; + goto unlock; hs_hub = usb_hub_to_struct_hub(to_usb_device(peer->dev.parent->parent)); if (!hs_hub || hs_hub->disconnected || hs_hub->hdev->level == ss->parent->level) - return; + goto unlock; /* Disconnect publishes child = NULL under device_state_lock. */ scoped_guard(spinlock_irq, &device_state_lock) { - if (!peer->child || peer->reconnect_ss || peer->ss_reconnected) - return; + if (!usb_port_may_queue_ss_reconnect(peer, ss)) + goto unlock; + /* A new SS attachment supersedes a request the USB2 worker has not read. */ + old_hs = peer->reconnect_hs; + old_ss = peer->reconnect_ss; peer->reconnect_hs = usb_get_dev(peer->child); peer->reconnect_ss = usb_get_dev(ss); } kick_hub_wq(hs_hub); +unlock: + mutex_unlock(&usb_port_peer_mutex); + usb_put_dev(old_hs); + usb_put_dev(old_ss); } static void hub_init_func2(struct work_struct *ws); @@ -2254,6 +2262,10 @@ int usb_hub_port_power_off(struct usb_device *hdev, int port1) dev_dbg(&port_dev->dev, "lane handover: power off\n"); /* The runtime PM reference is dropped once the hold ends. */ port_dev->held_off = !disabled; + if (port_dev->is_superspeed && !disabled) { + guard(mutex)(&usb_port_peer_mutex); + port_dev->ss_handover_generation++; + } /* Nothing hub_wq still has queued for the port applies to it off. */ clear_bit(port1, hub->change_bits); clear_bit(port1, hub->warm_reset_bits); @@ -6226,6 +6238,7 @@ static void hub_port_reconnect_ss(struct usb_hub *hub, int port1) { struct usb_port *port = hub->ports[port1 - 1]; struct usb_device *hs, *ss; + u32 handover = 0; bool reconnect; int ret; @@ -6237,6 +6250,8 @@ static void hub_port_reconnect_ss(struct usb_hub *hub, int port1) if (!ss) return; reconnect = usb_port_needs_ss_reconnect(port, hs, ss); + if (reconnect) + handover = port->peer->ss_handover_generation; } if (!reconnect) goto out; @@ -6248,6 +6263,8 @@ static void hub_port_reconnect_ss(struct usb_hub *hub, int port1) /* Power loss can reconnect the SS half too; it must not start a loop. */ scoped_guard(mutex, &usb_port_peer_mutex) { port->ss_reconnected = true; + /* An older in-flight cycle cannot consume a newer lane handover. */ + port->ss_reconnect_generation = handover; port->ss_reconnect_pending = true; } diff --git a/drivers/usb/core/hub.h b/drivers/usb/core/hub.h index c095b77aaf327..6cde9aaef6740 100644 --- a/drivers/usb/core/hub.h +++ b/drivers/usb/core/hub.h @@ -94,6 +94,8 @@ struct usb_hub { * @reconnect_ss: SuperSpeed hub requesting that USB2 connection restart * @ss_reconnected: the current USB2 attachment already had its late-SS handover * @ss_reconnect_pending: consume the connection event from our own power cycle + * @ss_handover_generation: explicit SS lane handovers, under usb_port_peer_mutex + * @ss_reconnect_generation: handover serviced by the last USB2 reconnect * @connector: USB Type-C connector * @req: default pm qos request for hubs without port power control * @connect_type: port's connect type @@ -140,8 +142,24 @@ struct usb_port { bool held_off; bool ss_reconnected; bool ss_reconnect_pending; + u32 ss_handover_generation; + u32 ss_reconnect_generation; }; +/* usb_port_peer_mutex protects the peer and both handover counters. */ +static inline bool usb_port_ss_reconnect_done(const struct usb_port *port) +{ + return port->ss_reconnected && port->peer && + port->ss_reconnect_generation == port->peer->ss_handover_generation; +} + +static inline bool usb_port_may_queue_ss_reconnect(const struct usb_port *port, + const struct usb_device *ss) +{ + return port->child && !usb_port_ss_reconnect_done(port) && + (port->reconnect_hs != port->child || port->reconnect_ss != ss); +} + /* Caller holds the hub lock and usb_port_peer_mutex. */ static inline void usb_port_ss_reconnect_event(struct usb_port *port, u16 status, u16 change) { @@ -160,7 +178,7 @@ static inline bool usb_port_needs_ss_reconnect(const struct usb_port *port, const struct usb_device *hs, const struct usb_device *ss) { - return hs && ss && !port->ss_reconnected && !port->is_superspeed && + return hs && ss && !usb_port_ss_reconnect_done(port) && !port->is_superspeed && port->child == hs && port->peer && port->peer->is_superspeed && port->connector_node && port->peer->connector_node == port->connector_node && diff --git a/drivers/usb/core/port-power-test.c b/drivers/usb/core/port-power-test.c index aedab69626a90..061eec179d8a0 100644 --- a/drivers/usb/core/port-power-test.c +++ b/drivers/usb/core/port-power-test.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -437,14 +438,18 @@ static void port_power_test_hold(struct kunit *test) { struct port_power_test *t = test->priv; unsigned int hs_power_ons = port_power_test_power_ons(t->hcd); + struct usb_port *ss = to_usb_port(t->ss_port); + u32 handover = ss->ss_handover_generation; int usage = port_power_test_usage(t->ss_port); usb_lock_device(t->ss_root); KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, ss->ss_handover_generation, handover + 1); KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); /* A second change of the lanes finds the port held for it */ KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, ss->ss_handover_generation, handover + 1); KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); @@ -453,6 +458,10 @@ static void port_power_test_hold(struct kunit *test) /* Nothing is held any more */ KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 1); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, ss->ss_handover_generation, handover + 2); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage); KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 2), -EINVAL); usb_unlock_device(t->ss_root); @@ -728,6 +737,10 @@ static void port_power_test_late_ss_decision(struct kunit *test) port->ss_reconnected = true; KUNIT_EXPECT_FALSE(test, usb_port_needs_ss_reconnect(port, hs, ss)); + peer->ss_handover_generation++; + KUNIT_EXPECT_TRUE(test, usb_port_needs_ss_reconnect(port, hs, ss)); + port->ss_reconnect_generation = peer->ss_handover_generation; + KUNIT_EXPECT_FALSE(test, usb_port_needs_ss_reconnect(port, hs, ss)); port->ss_reconnected = false; port->disabled = true; @@ -1070,7 +1083,56 @@ static void port_power_test_retrain_owner(struct kunit *test) usb_unlock_device(t->hs_root); } +/* A new PHY handover rearms USB2 recovery; its own power cycle never does. */ +static void port_power_test_late_ss_handover(struct kunit *test) +{ + struct usb_port *port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL); + struct usb_port *peer = kunit_kzalloc(test, sizeof(*peer), GFP_KERNEL); + struct usb_device *hs = kunit_kzalloc(test, sizeof(*hs), GFP_KERNEL); + struct usb_device *ss = kunit_kzalloc(test, sizeof(*ss), GFP_KERNEL); + struct usb_device *next = kunit_kzalloc(test, sizeof(*next), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, port); + KUNIT_ASSERT_NOT_NULL(test, peer); + KUNIT_ASSERT_NOT_NULL(test, hs); + KUNIT_ASSERT_NOT_NULL(test, ss); + KUNIT_ASSERT_NOT_NULL(test, next); + port->child = hs; + port->peer = peer; + port->ss_reconnected = true; + peer->is_superspeed = true; + peer->child = ss; + port->ss_reconnect_generation = 7; + peer->ss_handover_generation = 7; + KUNIT_EXPECT_FALSE(test, usb_port_may_queue_ss_reconnect(port, ss)); + + /* A tipd rebind changes SS even while the USB2 hub remains attached. */ + peer->ss_handover_generation++; + KUNIT_EXPECT_TRUE(test, usb_port_may_queue_ss_reconnect(port, ss)); + port->reconnect_hs = hs; + port->reconnect_ss = ss; + KUNIT_EXPECT_FALSE(test, usb_port_may_queue_ss_reconnect(port, ss)); + KUNIT_EXPECT_TRUE(test, usb_port_may_queue_ss_reconnect(port, next)); + + /* Completion of the older request must not consume the newer handover. */ + port->ss_reconnect_generation = 7; + KUNIT_EXPECT_TRUE(test, usb_port_may_queue_ss_reconnect(port, next)); + port->ss_reconnect_generation = peer->ss_handover_generation; + port->ss_reconnect_pending = true; + usb_port_ss_reconnect_event(port, USB_PORT_STAT_CONNECTION, USB_PORT_STAT_C_CONNECTION); + KUNIT_EXPECT_FALSE(test, usb_port_may_queue_ss_reconnect(port, next)); + peer->ss_handover_generation++; + KUNIT_EXPECT_TRUE(test, usb_port_may_queue_ss_reconnect(port, next)); + + peer->ss_handover_generation = U32_MAX; + port->ss_reconnect_generation = U32_MAX; + KUNIT_EXPECT_FALSE(test, usb_port_may_queue_ss_reconnect(port, next)); + peer->ss_handover_generation++; + KUNIT_EXPECT_TRUE(test, usb_port_may_queue_ss_reconnect(port, next)); +} + static struct kunit_case port_power_test_cases[] = { + KUNIT_CASE(port_power_test_late_ss_handover), KUNIT_CASE(port_power_test_retrain_warm), KUNIT_CASE(port_power_test_retrain_detect), KUNIT_CASE(port_power_test_retrain_bounded), From 17a0a425c64811ce780ce72f98b220ff23515ea1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:00:45 +1100 Subject: [PATCH 1413/1579] wifi: mt7932: document power cache and coexistence qualification Keep the post-association cache repair and observation counters. The RX protection-before-duplicate fix and its tests are already in the 7.3 base. Retain the established listen interval: the old 2.4 GHz reduction is not scoped to Bluetooth activity and could increase listen opportunities by a factor of twenty even with Bluetooth off. Document the arithmetic, evidence limits and paired hardware checks without claiming firmware wake readback or an observed PTA repair. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index fd045bcc9dab3..c2b9ff3c26808 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -672,3 +672,35 @@ Limitations are described under Station security. * SCO/headset microphone, LE Audio/ISO and simultaneous headset audio are unqualified. + +Station power cache and coexistence +---------------------------------- + +Each join submits CID05 awake before EA/40. That pre-join submission is +not the power policy of the subsequent associated firmware session. The +host invalidates its applied-mode cache after pre-join initialization so +that the existing post-association worker submits the desired policy, +including awake when station power save is disabled. Successful repeated +requests within that association still coalesce. Sleep remains gated by +TIM discovery and the normal RF, peer and termination checks. A successful +submission is not firmware readback or a measured coexistence improvement. + +The association builder retains its twenty-beacon listen interval on all +bands. The earlier proposal to reduce it to one on every 2.4 GHz association +is omitted: there is no BT activity notification connecting these drivers, +and a band-only policy would also affect BT-off associations. Twenty to one +permits twenty times as many listen opportunities, or nineteen additional +opportunities per twenty beacons. At a 100 TU beacon interval (102.4 ms), +the advertised span would fall from 2048 ms to 102.4 ms. These values do not +quantify radio wake duty cycle, milliwatts or battery life; those depend on +firmware, TIM/DTIM and actual traffic. No new module parameter, guessed PTA +command or Bluetooth link-policy override is introduced. + +Qualify with paired control/fix boots, BT off/on on 2.4 GHz, local wired +iperf3 in each direction and ten-minute gateway pings. Interleave off/on/off +and on/off/on blocks to expose drift in a busy channel. Compare each +module's BT-on penalty against its adjacent BT-off cells, retaining loss +and tail latency separately. Include a 5 GHz control, functioning BT HID, +and a twenty-second rtc1 suspend/resume followed by a fresh association. +Keep failed cells and independent rollback evidence; host tests alone do +not establish latency, throughput or Bluetooth functionality. From 89235f63fdb98cdff011683be8c18f91f4feda35 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:10:19 +1100 Subject: [PATCH 1414/1579] dt-bindings: hwmon: apple,smc: Describe die and pressure keys Describe an ordered list of SoC die temperature keys and the optional firmware thermal pressure key on the SMC hwmon node. Specify the key formats and preserve the temperature child indices used by thermal zones. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/hwmon/apple,smc-hwmon.yaml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index e932bfa31a32c..b8e962334935f 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -40,6 +40,28 @@ properties: compatible: const: apple,smc-hwmon + apple,soc-die-keys: + $ref: /schemas/types.yaml#/definitions/string-array + minItems: 1 + maxItems: 32 + items: + pattern: "^[A-Za-z0-9]{4}$" + description: + SMC FourCC keys for SoC die temperature sensors. Each key must report a + readable four-byte floating-point temperature in degrees Celsius. The + hottest reading supplies the macsmc_soc_die thermal zone. These channels + follow the temperature-* children in the listed order and are labelled + "SoC die ". Keys absent from the running firmware or with an + unsupported format are skipped. These channels do not change the indices + used by thermal-sensors references to temperature-* children. + + apple,thermal-pressure-key: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: + SMC FourCC key reporting the firmware thermal pressure level as a + readable signed 32-bit integer. This is exposed for monitoring only. + apple,discover-temperatures: type: boolean description: From a1d90d98db71a774952c8b9334bb910f428b750d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:10:19 +1100 Subject: [PATCH 1415/1579] arm64: dts: apple: Describe M3 die and pressure keys Move the J516s, J516c and J514c die key lists from the hwmon board table into their board device trees, retaining every key and its order. Add the J516s mTPL pressure key and keep the existing qualification of the unconfirmed J516s and assumed J514c sensor lists. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6030-j516s.dts | 8 ++++++++ arch/arm64/boot/dts/apple/t6031-j514c.dts | 9 +++++++++ arch/arm64/boot/dts/apple/t6031-j516c.dts | 9 +++++++++ 3 files changed, 26 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6030-j516s.dts b/arch/arm64/boot/dts/apple/t6030-j516s.dts index edc00c0c1c4df..39ee54f235c0d 100644 --- a/arch/arm64/boot/dts/apple/t6030-j516s.dts +++ b/arch/arm64/boot/dts/apple/t6030-j516s.dts @@ -97,3 +97,11 @@ &isp { apple,platform-id = <16>; }; + +&smc_hwmon { + /* M3-family die keys, not yet confirmed on J516s; absent keys are skipped. */ + apple,soc-die-keys = "Tf04", "Tf09", "Tf0A", "Tf0B", "Tf0D", + "Tf0E", "Tf14", "Tf18", "Tf19", "Tf1A", + "Tf24", "Tf25", "Tf2A", "Tf2B"; + apple,thermal-pressure-key = "mTPL"; +}; diff --git a/arch/arm64/boot/dts/apple/t6031-j514c.dts b/arch/arm64/boot/dts/apple/t6031-j514c.dts index f4f282e9d3207..42aeffa229c75 100644 --- a/arch/arm64/boot/dts/apple/t6031-j514c.dts +++ b/arch/arm64/boot/dts/apple/t6031-j514c.dts @@ -53,3 +53,12 @@ compatible = "apple,j514-macaudio", "apple,j314-macaudio", "apple,macaudio"; model = "MacBook Pro J514"; }; + +&smc_hwmon { + /* Assumed to match J516c; unavailable firmware keys are skipped. */ + apple,soc-die-keys = "Tf04", "Tf06", "Tf09", "Tf0A", "Tf0B", + "Tf14", "Tf16", "Tf19", "Tf1A", "Tf1B", + "Tf24", "Tf26", "Tf29", "Tf2A", "Tf2B", + "Tf34", "Tf36", "Tf39", "Tf3A", "Tf3B", + "Tf44", "Tf46", "Tf49", "Tf4A", "Tf4B"; +}; diff --git a/arch/arm64/boot/dts/apple/t6031-j516c.dts b/arch/arm64/boot/dts/apple/t6031-j516c.dts index 562491755117f..9b3afec0903e1 100644 --- a/arch/arm64/boot/dts/apple/t6031-j516c.dts +++ b/arch/arm64/boot/dts/apple/t6031-j516c.dts @@ -53,3 +53,12 @@ compatible = "apple,j516-macaudio", "apple,j316-macaudio", "apple,macaudio"; model = "MacBook Pro J516"; }; + +&smc_hwmon { + /* M3 Max die keys; unavailable firmware keys are skipped. */ + apple,soc-die-keys = "Tf04", "Tf06", "Tf09", "Tf0A", "Tf0B", + "Tf14", "Tf16", "Tf19", "Tf1A", "Tf1B", + "Tf24", "Tf26", "Tf29", "Tf2A", "Tf2B", + "Tf34", "Tf36", "Tf39", "Tf3A", "Tf3B", + "Tf44", "Tf46", "Tf49", "Tf4A", "Tf4B"; +}; From 565cbce31103d20e0989580519ba5603091cc9e9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:10:20 +1100 Subject: [PATCH 1416/1579] hwmon: macsmc: Read die and pressure keys from DT Use apple,soc-die-keys to append validated die temperature channels after the existing temperature children. Register the hottest-sensor zone and pressure attribute from DT data without a machine-compatible gate. Keep the current validation and failed-read behavior. All three board table entries now have matching DT data, so remove the table and the downstream soc_die_keys and board_keys parameters. Neither parameter is present in the mainline v7.3 driver or its documentation. Keep the documented fan_control override and DT fan support from the preceding fan-control series unchanged. Document the new properties and the requirement to update the DTB with the kernel when replacing the former board fallback. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/hwmon/macsmc-hwmon.rst | 18 +++ drivers/hwmon/macsmc-hwmon.c | 160 +++++---------------------- 2 files changed, 46 insertions(+), 132 deletions(-) diff --git a/Documentation/hwmon/macsmc-hwmon.rst b/Documentation/hwmon/macsmc-hwmon.rst index 8aef3dd2bf75e..2204745326cba 100644 --- a/Documentation/hwmon/macsmc-hwmon.rst +++ b/Documentation/hwmon/macsmc-hwmon.rst @@ -31,6 +31,20 @@ as unsafe, allows manual speeds on every fan with a mode key whatever the Devicetree says. It cannot be proven that the system will fail safe if it overheats under a manual fan speed. +SoC die temperatures are described by the ``apple,soc-die-keys`` string list +on the hwmon device node. The driver appends these temperature channels after +the ``temperature-*`` children, in list order, with labels ``SoC die ``. +Only readable four-byte floating-point keys are used. When thermal support is +enabled, their hottest reading supplies the ``macsmc_soc_die`` thermal zone. +A failed or implausible reading fails the zone read rather than reporting a +partial result. Keys absent from the firmware are skipped. + +The optional ``apple,thermal-pressure-key`` on the same node describes a +readable signed 32-bit firmware thermal pressure level. Neither feature needs +a module parameter. Kernels using these properties need an updated device +tree; the former downstream per-board fallback and its ``soc_die_keys`` and +``board_keys`` parameters have been removed. + sysfs interface --------------- @@ -73,3 +87,7 @@ tempX_input tempX_label Temperature sensor label +thermal_pressure_level + Firmware thermal pressure level, exposed read-only when the device tree + supplies a valid key. The integer is firmware-defined and is not a + temperature or a percentage. diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index e1685c2945d0c..af67979f51213 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -59,64 +59,6 @@ module_param_unsafe(fan_control, bool, 0644); MODULE_PARM_DESC(fan_control, "Allow manual fan speeds on every fan with a mode key, also where the device tree does not (default: the device tree decides)"); -/* - * SoC die temperature keys: temperature channels labelled "SoC die " - * whose hottest reading is the SoC die thermal zone (see - * macsmc_hwmon_die_get_temp()). They come from this parameter or, without it, - * from the machine's entry in macsmc_hwmon_boards[]. Keys the SMC does not - * have are skipped. - */ -static char *soc_die_keys; -module_param(soc_die_keys, charp, 0444); -MODULE_PARM_DESC(soc_die_keys, - "Comma-separated SMC temperature keys whose hottest reading is the SoC die thermal zone, \"none\" for no zone (default: the machine's list)"); - -/* - * Keys a machine's device tree does not list yet. The device tree comes from - * the boot loader, so these let a kernel use keys before its device tree - * catches up. board_keys=0 turns them off: the device tree alone then decides - * (soc_die_keys still applies). - */ -static bool board_keys = true; -module_param(board_keys, bool, 0444); -MODULE_PARM_DESC(board_keys, - "Use the built-in per-machine SoC die and thermal pressure keys the device tree does not list (default: on)"); - -static const struct macsmc_hwmon_board { - const char *compatible; - const char *die_keys; - const char *pressure_key; -} macsmc_hwmon_boards[] = { - /* - * MacBook Pro (16-inch, M3 Pro): key names used for the CPU - * performance-core and GPU die sensors of M3-family machines. They - * are not confirmed on this machine: the debugfs key list shows which - * temperature keys it has. - */ - { - .compatible = "apple,j516s", - .die_keys = "Tf04,Tf09,Tf0A,Tf0B,Tf0D,Tf0E,Tf14,Tf18,Tf19,Tf1A,Tf24,Tf25,Tf2A,Tf2B", - .pressure_key = "mTPL", - }, - /* - * MacBook Pro (14-inch and 16-inch, M3 Max): the same key family in - * five groups. On a J516C every one of these keys read a die - * temperature (47-62 C at idle); the keys ending in D and E repeated - * the one ending in 4 and are left out. The J514C list is assumed to - * be the same; keys the SMC does not have are skipped. - */ - { - .compatible = "apple,j516c", - .die_keys = "Tf04,Tf06,Tf09,Tf0A,Tf0B,Tf14,Tf16,Tf19,Tf1A,Tf1B,Tf24,Tf26,Tf29,Tf2A,Tf2B," - "Tf34,Tf36,Tf39,Tf3A,Tf3B,Tf44,Tf46,Tf49,Tf4A,Tf4B", - }, - { - .compatible = "apple,j514c", - .die_keys = "Tf04,Tf06,Tf09,Tf0A,Tf0B,Tf14,Tf16,Tf19,Tf1A,Tf1B,Tf24,Tf26,Tf29,Tf2A,Tf2B," - "Tf34,Tf36,Tf39,Tf3A,Tf3B,Tf44,Tf46,Tf49,Tf4A,Tf4B", - }, -}; - static bool macsmc_hwmon_is_m3(void) { return of_machine_is_compatible("apple,t6030") || @@ -126,52 +68,6 @@ static bool macsmc_hwmon_is_m3(void) of_machine_is_compatible("apple,t8122"); } -static const struct macsmc_hwmon_board *macsmc_hwmon_board(void) -{ - unsigned int i; - - if (!board_keys) - return NULL; - for (i = 0; i < ARRAY_SIZE(macsmc_hwmon_boards); i++) - if (of_machine_is_compatible(macsmc_hwmon_boards[i].compatible)) - return &macsmc_hwmon_boards[i]; - return NULL; -} - -/* The list of SoC die temperature keys, or NULL for none. */ -static const char *macsmc_hwmon_die_key_list(void) -{ - const struct macsmc_hwmon_board *board; - - if (!macsmc_hwmon_is_m3()) - return NULL; - if (soc_die_keys) - return strcmp(soc_die_keys, "none") ? soc_die_keys : NULL; - board = macsmc_hwmon_board(); - return board ? board->die_keys : NULL; -} - -/* - * The next key of a comma-separated list: returns 1 and the key, 0 at the end - * of the list, or -EINVAL for an entry that is not four characters long. - */ -static int macsmc_hwmon_next_key(const char **list, char key[5]) -{ - const char *p = *list, *end; - - while (*p == ',' || *p == ' ') - p++; - if (!*p) - return 0; - end = strchrnul(p, ','); - *list = end; - if (end - p != 4) - return -EINVAL; - memcpy(key, p, 4); - key[4] = '\0'; - return 1; -} - struct macsmc_hwmon_sensor { struct apple_smc_key_info info; smc_key macsmc_key; @@ -1068,15 +964,10 @@ static int macsmc_hwmon_create_pressure(struct macsmc_hwmon *hwmon, ret = of_property_read_string(hwmon_node, "apple,thermal-pressure-key", &key); - if (ret == -EINVAL) { - const struct macsmc_hwmon_board *board = macsmc_hwmon_board(); - - if (!board || !board->pressure_key) - return 0; - key = board->pressure_key; - } else if (ret) { + if (ret == -EINVAL) + return 0; + if (ret) return ret; - } ret = macsmc_hwmon_parse_key(hwmon->dev, hwmon->smc, &pressure, key); if (ret) @@ -1270,10 +1161,15 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, struct device_node *key_node __maybe_unused; struct macsmc_hwmon_sensor *sensor; u32 n_current = 0, n_fan = 0, n_power = 0, n_temperature = 0, n_voltage = 0; - const char *die_keys = macsmc_hwmon_die_key_list(), *list; - char key[5]; + const char *key; bool indexed; - int ret; + int n_die_keys, i, ret; + + n_die_keys = of_property_count_strings(hwmon_node, "apple,soc-die-keys"); + if (n_die_keys == -EINVAL) + n_die_keys = 0; + else if (n_die_keys < 0) + return n_die_keys; for_each_child_of_node_with_prefix(hwmon_node, key_node, "current-") { n_current++; @@ -1345,8 +1241,7 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, for_each_child_of_node_with_prefix(hwmon_node, key_node, "temperature-") { n_temperature++; } - for (list = die_keys; list && (ret = macsmc_hwmon_next_key(&list, key));) - n_temperature++; + n_temperature += n_die_keys; if (n_temperature) { hwmon->temp.sensors = devm_kcalloc(hwmon->dev, n_temperature, @@ -1372,10 +1267,13 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, macsmc_hwmon_add_temp_channel(&hwmon->temp, ret); } - for (list = die_keys; list && (ret = macsmc_hwmon_next_key(&list, key));) { - if (ret < 0) { - dev_warn(hwmon->dev, "ignoring an entry of the SoC die key list \"%s\"\n", - die_keys); + for (i = 0; i < n_die_keys; i++) { + ret = of_property_read_string_index(hwmon_node, "apple,soc-die-keys", + i, &key); + if (ret) + return ret; + if (strlen(key) != 4) { + dev_warn(hwmon->dev, "Ignoring invalid SoC die key %s\n", key); continue; } hwmon->die_listed++; @@ -1713,11 +1611,9 @@ static int macsmc_hwmon_probe(struct platform_device *pdev) return ret; } - if (macsmc_hwmon_is_m3()) { - ret = macsmc_hwmon_create_pressure(hwmon, hwmon->dev->of_node); - if (ret) - return ret; - } + ret = macsmc_hwmon_create_pressure(hwmon, hwmon->dev->of_node); + if (ret) + return ret; ret = macsmc_hwmon_discover_temperatures(hwmon); if (ret) @@ -1757,17 +1653,17 @@ static int macsmc_hwmon_probe(struct platform_device *pdev) hwmon->hwmon_dev = devm_hwmon_device_register_with_info(&pdev->dev, "macsmc_hwmon", hwmon, &hwmon->chip_info, - macsmc_hwmon_is_m3() ? macsmc_hwmon_extra_groups : NULL); + macsmc_hwmon_extra_groups); if (IS_ERR(hwmon->hwmon_dev)) return dev_err_probe(hwmon->dev, PTR_ERR(hwmon->hwmon_dev), "Probing SMC hwmon device failed\n"); - if (macsmc_hwmon_is_m3()) { + if (macsmc_hwmon_is_m3()) macsmc_hwmon_debugfs_init(hwmon); - ret = macsmc_hwmon_die_register(hwmon); - if (ret) - return ret; - } + + ret = macsmc_hwmon_die_register(hwmon); + if (ret) + return ret; /* * Released before the restore action above, which stops the cooling From 990c41f8f99018e7fe025a7c35341145087d4e16 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:07:14 +1100 Subject: [PATCH 1417/1579] arm64: dts: apple: t6040: describe the remaining non-PCI DART apertures The DART nodes otherwise default to their entire hardware address width. That omits the device-visible windows recorded by the T6040 firmware tree, including the reserved prefix of the MTP, SIO and USB windows. Translate the structural vm-base/vm-size facts from the J616s 26.6.2 ADT into the existing apple,dma-range binding for the MTP, USB, SIO, AVE, external display and remaining disabled accelerator/ACIO DARTs. These are aperture limits, not captured page tables or firmware addresses. The Apple DART driver already consumes them when setting domain geometry. Keep PCI DARTs separate: their device-visible PCI bus aperture needs the host bridge alias contract, rather than a direct copy of the DART window. Leave the active AOP, ISP and AVD windows alone: their full ADT aperture crosses aliases rejected by the inherited three-level page-table path. The ANE aperture and MPM stream are covered by the separate ANE series. No device status, stream assignment, power domain or thermal node changes. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040.dtsi | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 961af1b7b5bd4..15a3c2965a960 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -1377,6 +1377,7 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; /* ADT "remap": streams 1 2 3 are stream 0 */ apple,remap-streams = <1 0>, <2 0>, <3 0>; status = "disabled"; @@ -1607,6 +1608,7 @@ interrupts = ; power-domains = <&ps_atc0_usb>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; status = "disabled"; }; @@ -1617,6 +1619,7 @@ interrupts = ; power-domains = <&ps_atc0_usb>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; status = "disabled"; }; @@ -1691,6 +1694,7 @@ interrupts = ; power-domains = <&ps_atc1_usb>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; status = "disabled"; }; @@ -1701,6 +1705,7 @@ interrupts = ; power-domains = <&ps_atc1_usb>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; status = "disabled"; }; @@ -1775,6 +1780,7 @@ interrupts = ; power-domains = <&ps_atc2_usb>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; status = "disabled"; }; @@ -1785,6 +1791,7 @@ interrupts = ; power-domains = <&ps_atc2_usb>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; status = "disabled"; }; @@ -1816,6 +1823,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x2 0x83080000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_venc0_sys>; @@ -1826,6 +1834,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x2 0x830a0000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_venc0_sys>; @@ -1836,6 +1845,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x3 0x01020000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_scodec>; @@ -1873,6 +1883,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x3 0xa2300000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; interrupt-parent = <&aic>; interrupts = ; status = "disabled"; @@ -1882,6 +1893,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x3 0xa2340000 0x0 0x20000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; interrupt-parent = <&aic>; interrupts = ; status = "disabled"; @@ -1900,6 +1912,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x4 0x29060000 0x0 0x20000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x1c000 0x2ff 0xfffe4000>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_sio>; @@ -1940,6 +1953,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x4 0x87020000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_jpg>; @@ -1950,6 +1964,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x4 0x87080000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_jpg>; @@ -1960,6 +1975,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x5 0x21260000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_msr0>; @@ -1970,6 +1986,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x6 0x02300000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; interrupt-parent = <&aic>; interrupts = ; status = "disabled"; @@ -1979,6 +1996,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x6 0x02340000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; interrupt-parent = <&aic>; interrupts = ; status = "disabled"; @@ -2211,6 +2229,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x3 0x81a80000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; status = "disabled"; @@ -2236,6 +2255,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x3 0x89a80000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; status = "disabled"; @@ -2261,6 +2281,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x3 0x91a80000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; status = "disabled"; From 137cfcf98ebd5aa432856b8743f96a9df2762a28 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:14:21 +1100 Subject: [PATCH 1418/1579] drm/asahi: Format the client usage accounting paths Apply rustfmt to existing files that the per-client engine accounting change touches. Keep the mechanical formatting separate from the accounting and firmware timestamp changes for review. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/asahi_core.rs | 8 +- drivers/gpu/drm/asahi/file.rs | 75 ++++++++++------ drivers/gpu/drm/asahi/g15/mod.rs | 1 - drivers/gpu/drm/asahi/g17/completion.rs | 50 ++++++++--- drivers/gpu/drm/asahi/g17/mod.rs | 108 +++++++++++++++--------- drivers/gpu/drm/asahi/g17/queue/mod.rs | 106 +++++++++++++++-------- drivers/gpu/drm/asahi/queue/mod.rs | 21 ++++- rust/kernel/drm/file.rs | 4 +- 8 files changed, 250 insertions(+), 123 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index e6b4d0c9a088f..1a584932c5a77 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -12,21 +12,21 @@ mod driver; mod event; mod file; mod float; -mod fwid; mod fw; -mod gem; -mod g17; +mod fwid; #[cfg(CONFIG_DRM_ASAHI_G15)] mod g15; +mod g17; +mod gem; #[cfg(CONFIG_DRM_ASAHI_G16)] mod g16; mod gpu; mod hw; mod initdata; +mod m3; mod mem; mod memacct; -mod m3; mod microseq; mod mmu; mod object; diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 01598b691ded3..6cc411cdb7467 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,8 +16,8 @@ use crate::{ AsahiDrmDevice, // }, gem, - memacct, gpu, + memacct, mmu, queue, util::{ @@ -132,11 +132,16 @@ pub(crate) struct SyncItem { } impl SyncItem { - fn parse_one(file: &DrmFile, mut data: uapi::drm_asahi_sync, out: bool, - bundled: bool) -> Result - { + fn parse_one( + file: &DrmFile, + mut data: uapi::drm_asahi_sync, + out: bool, + bundled: bool, + ) -> Result { if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { - if !out || !bundled || data.timeline_value == 0 { return Err(EINVAL); } + if !out || !bundled || data.timeline_value == 0 { + return Err(EINVAL); + } data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; } match data.sync_type { @@ -230,12 +235,16 @@ impl SyncItem { } anchor = Some(i as usize); } - vec.push(Self::parse_one(file, sync, i >= in_count, - allow_cumulative)?, GFP_KERNEL)?; + vec.push( + Self::parse_one(file, sync, i >= in_count, allow_cumulative)?, + GFP_KERNEL, + )?; } if let Some(anchor) = anchor { let bundle = crate::g17::sync_output::SignalBundle::new( - &vec[in_count as usize..], anchor - in_count as usize)?; + &vec[in_count as usize..], + anchor - in_count as usize, + )?; vec[anchor].cumulative = Some(bundle); } else { for sync in &mut vec[in_count as usize..] { @@ -698,22 +707,28 @@ impl File { }; let size = gem::checked_object_size(data.size.try_into()?)?; - let commit = memacct::LEDGER.admit(&file.inner().mem, size as u64).inspect_err(|_| { - cls_pr_debug!( - Errors, - "gem_create: {:#x} bytes refused by the GPU memory limits\n", - size - ); - })?; + let commit = memacct::LEDGER + .admit(&file.inner().mem, size as u64) + .inspect_err(|_| { + cls_pr_debug!( + Errors, + "gem_create: {:#x} bytes refused by the GPU memory limits\n", + size + ); + })?; // Large G17 user objects are populated in full by the first VM_BIND. // Do not multiply direct reclaim/compaction latency by their folio // count. Userspace can evict its BO cache and retry after ENOMEM. - let pages_no_reclaim = gem::user_pages_no_reclaim( - size, - reg_data.gpu.as_any().is::(), - ); + let pages_no_reclaim = + gem::user_pages_no_reclaim(size, reg_data.gpu.as_any().is::()); let gem = gem::new_object( - device, size, data.flags, resv_obj, reclaim_cpu, pages_no_reclaim, commit, + device, + size, + data.flags, + resv_obj, + reclaim_cpu, + pages_no_reclaim, + commit, )?; if reg_data.gpu.prepare_new_objects() { #[cfg(CONFIG_DRM_ASAHI_G15)] @@ -1306,7 +1321,10 @@ impl File { if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH && !reg_data.gpu.as_any().is::() { - cls_pr_debug!(Errors, "queue_create: HIGH priority is not supported on this GPU\n"); + cls_pr_debug!( + Errors, + "queue_create: HIGH priority is not supported on this GPU\n" + ); return Err(EINVAL); } // SAFETY: capable() only reads the calling task's credentials (ioctl process context). @@ -1314,7 +1332,10 @@ impl File { && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as _) } && !file.is_current_master() { - cls_pr_debug!(Errors, "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n"); + cls_pr_debug!( + Errors, + "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n" + ); return Err(EACCES); } @@ -1425,9 +1446,13 @@ impl File { if guarded { copy_commands(&mut vec)?; } - let syncs = - SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count, - gpu.as_any().is::())?; + let syncs = SyncItem::parse_array( + file, + data.syncs, + data.in_sync_count, + data.out_sync_count, + gpu.as_any().is::(), + )?; mod_dev_dbg!( device, diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 371bc41b147ce..718e5b8c851d3 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -210,7 +210,6 @@ impl platform::Driver for Driver<'_> { }, }) } - } /// Firmware starts only after every fallible module/driver registration has succeeded. diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 02e7bbc5c33dc..a232b9c9eb2cc 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -13,7 +13,12 @@ use super::{ recovery::Class, status::VmStatus, }; -use crate::{driver::AsahiDrmDevice, file, hw::t8140::{dynamic, resources as cfg}, mmu}; +use crate::{ + driver::AsahiDrmDevice, + file, + hw::t8140::{dynamic, resources as cfg}, + mmu, +}; use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; use kernel::{ dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, @@ -86,7 +91,9 @@ impl Destinations { Ok(Self(destinations)) } - pub(crate) fn has_outputs(&self) -> bool { self.0.iter().any(Option::is_some) } + pub(crate) fn has_outputs(&self) -> bool { + self.0.iter().any(Option::is_some) + } fn iter(&self) -> core::slice::Iter<'_, Option> { self.0.iter() @@ -287,14 +294,17 @@ impl Completion { sources: Option>, ) -> Result> { let fence = fence::independent(contexts)?; - let ordered = !compute - && (host_predecessors || destinations.iter().any(Option::is_some)); + let ordered = !compute && (host_predecessors || destinations.iter().any(Option::is_some)); let terminal = if host_predecessors { Some(fence::independent(contexts)?) } else { None }; - let output = Output { fence, member, destinations }; + let output = Output { + fence, + member, + destinations, + }; let output = if ordered { HostOutput::Ordered(Publication::new(output, context.status().clone())?) } else { @@ -334,7 +344,9 @@ impl Completion { Fence::from_fence(&self.output.output().fence) } pub(crate) fn scheduler_fence(&self) -> Fence { - self.terminal.as_ref().map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) + self.terminal + .as_ref() + .map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) } pub(crate) fn publication(&self) -> Option> { match &self.output { @@ -382,11 +394,15 @@ impl Completion { } match self.check_source_inputs() { Ok(()) => self.output.output().member.note_retired(), - Err(error) => { self.note_input_error(error); } + Err(error) => { + self.note_input_error(error); + } } } pub(super) fn check_source_inputs(&self) -> Result { - if let Some(sources) = &self.sources { sources.check()?; } + if let Some(sources) = &self.sources { + sources.check()?; + } Ok(()) } @@ -394,7 +410,11 @@ impl Completion { self.output.output().member.work_state() } pub(crate) fn has_timestamps(&self) -> bool { - self.output.output().destinations.iter().any(Option::is_some) + self.output + .output() + .destinations + .iter() + .any(Option::is_some) } pub(crate) fn compute_timestamp_va(&self) -> Result { @@ -597,7 +617,9 @@ impl Completion { /// An exact whole-pair drain or processor stop allows rehabilitation. The /// VM pin goes first; the still-held lease excludes reuse until both settle. pub(crate) fn release_after_retirement(&self) { - if let Some(sources) = &self.sources { sources.release(); } + if let Some(sources) = &self.sources { + sources.release(); + } drop(self.vm_job.lock().take()); let lease = self.render.lock().take(); if let Some(lease) = lease.as_ref() { @@ -607,7 +629,9 @@ impl Completion { } fn release_healthy(&self) { - if let Some(sources) = &self.sources { sources.release(); } + if let Some(sources) = &self.sources { + sources.release(); + } drop(self.render.lock().take()); drop(self.vm_job.lock().take()); } @@ -617,7 +641,9 @@ impl Completion { #[track_caller] pub(crate) fn complete(&self, mut result: Result<[u64; 4]>) { if result.is_ok() { - if let Err(error) = self.check_source_inputs() { result = Err(error); } + if let Err(error) = self.check_source_inputs() { + result = Err(error); + } } if self.signalled.swap(true, Ordering::AcqRel) { return; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 6495ad2d98c13..ae5bbd61a3ce4 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -36,26 +36,26 @@ pub(crate) mod sync_output; mod trace; #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) use dependency::inherited_error; +mod capacity; mod event; mod feed; mod fence; mod freelist; pub(crate) mod fw; -mod ids; pub(crate) mod hazard; +mod ids; mod initdata; mod job; -mod capacity; mod kick; mod object; mod port; pub(crate) mod power; -mod reports; mod preparation; mod qos; mod queue; mod recovery; mod replay; +mod reports; mod runtime; pub(crate) mod status; mod teardown; @@ -75,12 +75,12 @@ use kernel::{ c_str, device::{self, Core}, drm::gem::shmem, + impl_has_delayed_work, impl_has_work, io::{ IoBase, IoSysMap, Region, // }, - impl_has_delayed_work, impl_has_work, new_condvar, new_delayed_work, new_mutex, new_work, platform, prelude::*, soc::apple::rtkit, @@ -755,10 +755,12 @@ impl Firmware { } power::IdleAction::Recheck(delay) => Some(delay), }; - Ok(match (effort, self.idle_hold(now_ns, in_flight || submitted)) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - }) + Ok( + match (effort, self.idle_hold(now_ns, in_flight || submitted)) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }, + ) } /// Drives the idle hold (see power::Hold); returns the delay in ns for its next check. A @@ -767,7 +769,10 @@ impl Firmware { const RETRY_NS: u64 = 1_000_000; let hold_ms = power::IDLE_HOLD_MS.load(Ordering::Relaxed); for _ in 0..2 { - match self.hold.step(now_ns, in_flight, self.effort.submissions(), hold_ms) { + match self + .hold + .step(now_ns, in_flight, self.effort.submissions(), hold_ms) + { power::HoldAction::None => return None, power::HoldAction::Recheck(delay) => return Some(delay), power::HoldAction::Assert => { @@ -994,7 +999,6 @@ impl Shared { self.renders.notify(); } - /// The session's port, valid while `lease` is held: shutdown and device loss join every /// lease before they detach the session. fn port<'a>(&self, lease: &'a preparation::Lease) -> Option<&'a port::Port> { @@ -1141,7 +1145,10 @@ impl Gpu { ) -> Result> { let calibration = if cfg.chip_id == crate::hw::t6040::CHIP_ID { Some(calibration::Calibration::read(dev).inspect_err(|_| { - dev_err!(dev.as_ref(), "Missing or invalid T6040 GPU calibration handoff\n"); + dev_err!( + dev.as_ref(), + "Missing or invalid T6040 GPU calibration handoff\n" + ); })?) } else { None @@ -1172,11 +1179,14 @@ impl Gpu { }; let max_frequency_khz = Self::max_frequency_khz(dev)?; let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; - let idle_delay = power::idle_delay( - node.get_opt_property(c_str!("apple,idle-off-delay-ms"))?, - )?; + let idle_delay = + power::idle_delay(node.get_opt_property(c_str!("apple,idle-off-delay-ms"))?)?; power::IDLE_HOLD_MS.store(idle_delay, Ordering::Relaxed); - dev_info!(dev.as_ref(), "GPU idle power-off delay: {} ms\n", idle_delay); + dev_info!( + dev.as_ref(), + "GPU idle power-off delay: {} ms\n", + idle_delay + ); let channels = availability::Availability::new()?; let progress = availability::Availability::new()?; let queues = availability::Availability::new()?; @@ -1282,8 +1292,12 @@ impl Gpu { let version = Self::os_firmware_version(dev); // iBoot strings can be shared by OS releases. Prefer the reported release's // candidate, but admit only an exact loaded image, regardless of that hint. - let preferred = builds.iter().filter(|build| Some(build.os_version) == version); - let others = builds.iter().filter(|build| Some(build.os_version) != version); + let preferred = builds + .iter() + .filter(|build| Some(build.os_version) == version); + let others = builds + .iter() + .filter(|build| Some(build.os_version) != version); for build in Iterator::chain(preferred, others) { if Self::identify_image(dev, build).is_ok() { dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); @@ -1306,13 +1320,10 @@ impl Gpu { let data = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; let start = text.start(); let end = start.checked_add(text.size()).ok_or(EINVAL)?; - crate::fwid::identify( - dev.as_ref().as_ref(), - start..end, - data.size(), - build.images, - ) - .inspect_err(|e| dev_err!(dev.as_ref(), "GPU firmware image not admitted: {:?}\n", e))?; + crate::fwid::identify(dev.as_ref().as_ref(), start..end, data.size(), build.images) + .inspect_err(|e| { + dev_err!(dev.as_ref(), "GPU firmware image not admitted: {:?}\n", e) + })?; Ok(()) } @@ -1372,13 +1383,21 @@ impl Gpu { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) - .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; + let primary = + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary).inspect_err(|e| { + dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e) + })?; let secondary = match cfg.secondary_root_offset { Some(_) => Some( - Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err(|e| { - dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e) - })?, + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err( + |e| { + dev_err!( + dev.as_ref(), + "Secondary coprocessor setup failed: {:?}\n", + e + ) + }, + )?, ), None => None, }; @@ -1450,13 +1469,18 @@ impl Gpu { session.uat.confirm_firmware_root_entries(root_entries)?; } let mailbox: *const Mailbox = &*session.primary.rtkit; - session.port.attach(session.init.port_views(mailbox)?, session.bridge.regs.clone())?; + session.port.attach( + session.init.port_views(mailbox)?, + session.bridge.regs.clone(), + )?; session.uat.mirror_secondary_root()?; if cfg.fabric_bridge { session.bridge.regs.enable_g17_gating()?; } let ack = shared.variant.init_ack; - session.primary.init(session.init.root_va(Role::Primary)?, ack)?; + session + .primary + .init(session.init.root_va(Role::Primary)?, ack)?; if ack == fw::variant::InitAck::Status { Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; } @@ -1469,7 +1493,12 @@ impl Gpu { secondary.notify(MSG_POWER_ON_DONE)?; } } - Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref())?; + Self::wait_for_opening( + dev, + &session.init, + &session.primary, + session.secondary.as_ref(), + )?; session .uat .activate_render_context(&session._render_global)?; @@ -1590,8 +1619,7 @@ impl gpu::Gpu for Gpu { | sync_output::FEATURE_CUMULATIVE_SIGNALS; let variant = self.shared.variant; if variant.few_primitives { - params.features |= - u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); + params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); } if variant.ray_traversal { params.features |= @@ -1694,12 +1722,16 @@ impl gpu::Gpu for Gpu { // Backend construction has rolled back all temporary leases. let waited = now_ns().saturating_sub(admission_started); let limit = job::TIMEOUT_MS as u64 * 1_000_000; - if waited >= limit { return Err(EBUSY); } + if waited >= limit { + return Err(EBUSY); + } self.shared.queue_events(); // A returned queue slot or reclaimed context wakes this. - self.shared - .queues - .block(observed, (limit - waited).div_ceil(1_000_000) as u32, true)?; + self.shared.queues.block( + observed, + (limit - waited).div_ceil(1_000_000) as u32, + true, + )?; } result => break result?, } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index d22c0f617f374..789c4451fd636 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -104,7 +104,10 @@ pub(crate) struct Queue { _slot: admission::QueueSlot, } -type ValidatedBatch = (KVec<(Validated, Order, Destinations)>, KVec); +type ValidatedBatch = ( + KVec<(Validated, Order, Destinations)>, + KVec, +); impl Queue { pub(crate) fn new( @@ -159,7 +162,11 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; // Without room the entries are released here, as they always could be. - let mut finished = if self.finished.reserve(self.frontiers[index].len(), GFP_KERNEL).is_ok() { + let mut finished = if self + .finished + .reserve(self.frontiers[index].len(), GFP_KERNEL) + .is_ok() + { Some(&mut self.finished) } else { None @@ -201,13 +208,19 @@ impl Queue { }; let index = engine.index(); if let Validated::Render { ray_query, .. } = &command { - let blocks = self.backend.render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; - if blocks > self.context.render_blocks() { return Err(EINVAL); } + let blocks = self + .backend + .render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; + if blocks > self.context.render_blocks() { + return Err(EINVAL); + } } self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { - let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; + let Validated::Compute { scratch, .. } = &command else { + return Err(EINVAL); + }; self.ensure_compute(scratch.pages())?; } // Only convert inputs when no independent host-prefix wait could keep @@ -215,21 +228,34 @@ impl Queue { if engine == Engine::Compute { super::trace::inputs(self.context.id(), order.sequence, 0, inputs.len(), None); } - let sources = if engine == Engine::Compute && !inputs.is_empty() + let sources = if engine == Engine::Compute + && !inputs.is_empty() && !destinations.has_outputs() - && self.frontiers[0].iter().all(|(sequence, fences)| - !Order::contains(order.wait_through[0], *sequence) - || fences.status() > 0) - && self.frontiers[1].iter().all(|(sequence, fences)| + && self.frontiers[0].iter().all(|(sequence, fences)| { + !Order::contains(order.wait_through[0], *sequence) || fences.status() > 0 + }) + && self.frontiers[1].iter().all(|(sequence, fences)| { !Order::contains(order.wait_through[1], *sequence) || fences.ready.raw() != fences.completed.raw() - || fences.status() > 0) - && self.failures.iter().enumerate().all(|(index, failure)| - failure.as_ref().is_none_or(|(sequence, _)| - !Order::contains(order.wait_through[index], *sequence))) - { self.backend.input_sources(inputs) } else { None }; + || fences.status() > 0 + }) + && self.failures.iter().enumerate().all(|(index, failure)| { + failure.as_ref().is_none_or(|(sequence, _)| { + !Order::contains(order.wait_through[index], *sequence) + }) + }) { + self.backend.input_sources(inputs) + } else { + None + }; if engine == Engine::Compute { - super::trace::inputs(self.context.id(), order.sequence, 1, inputs.len(), sources.as_deref()); + super::trace::inputs( + self.context.id(), + order.sequence, + 1, + inputs.len(), + sources.as_deref(), + ); } // Only earlier render ordinals covered by the explicit logical prefix @@ -237,8 +263,7 @@ impl Queue { let host_predecessors = self.frontiers[index] .iter() .take_while(|(sequence, _)| { - engine == Engine::Render - && Order::contains(order.wait_through[index], *sequence) + engine == Engine::Render && Order::contains(order.wait_through[index], *sequence) }) .filter_map(|(_, fences)| fences.publication.as_ref()); let completion = Completion::new( @@ -253,7 +278,8 @@ impl Queue { self.backend.feed(), sources.clone(), )?; - let publication = completion.publication() + let publication = completion + .publication() .map(|target| PreparedPublication::new(target, host_predecessors)) .transpose()?; if engine == Engine::Compute { @@ -290,8 +316,10 @@ impl Queue { }) }); let failure_count = failures.iter().flatten().count(); - let mut dependencies = - KVec::with_capacity(inputs.len() + count + timestamp_count + failure_count, GFP_KERNEL)?; + let mut dependencies = KVec::with_capacity( + inputs.len() + count + timestamp_count + failure_count, + GFP_KERNEL, + )?; let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count + failure_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; @@ -329,7 +357,10 @@ impl Queue { if job::fence_status(fence) > 0 { continue; } - if sources.as_ref().is_some_and(|sources| sources.contains_input(fence)) { + if sources + .as_ref() + .is_some_and(|sources| sources.contains_input(fence)) + { continue; } dependencies.push(fence.clone(), GFP_KERNEL)?; @@ -359,10 +390,11 @@ impl Queue { Engine::Render => &mut self.render, Engine::Compute => &mut self.compute, }; - let fences = scheduler - .as_mut() - .ok_or(ENODEV)? - .enqueue(packet, dependencies, early, publication)?; + let fences = + scheduler + .as_mut() + .ok_or(ENODEV)? + .enqueue(packet, dependencies, early, publication)?; self.sequence[index] = order.sequence; self.frontiers[index] .push_within_capacity((order.sequence, fences)) @@ -429,8 +461,10 @@ impl Queue { impl crate::queue::Queue for Queue { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - let pages = [super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), - super::fw::compute::ScratchRequest::new(layouts[1])?.pages()]; + let pages = [ + super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), + super::fw::compute::ScratchRequest::new(layouts[1])?.pages(), + ]; let blocks = self.backend.render_scs_blocks(pages)?; self.context.reserve_render_blocks(blocks) } @@ -444,22 +478,22 @@ impl crate::queue::Queue for Queue { bytes: &[u8], objects: Pin<&xarray::XArray>>, ) -> Result { - let mut outputs = Outputs::new( - &self.fences, - syncs, - input_count, - &self.allocation_failure, - )?; + let mut outputs = Outputs::new(&self.fences, syncs, input_count, &self.allocation_failure)?; let result = (|| { if self.context.status().get() != 0 { return Err(EIO); } let (mut pending, guards) = self.prepare_submission(bytes, objects)?; - if pending.iter().any(|(command, _, _)| matches!(command, Validated::Render { .. })) { + if pending + .iter() + .any(|(command, _, _)| matches!(command, Validated::Render { .. })) + { self.context.note_render(); } if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { - for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } + for (command, _, _) in pending.iter_mut() { + command.allow_long_job(); + } } let mut guards = guards.into_iter(); let firmware_source = pending.len() == 1 diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 2d9052c2edb51..653fd77d0c3f3 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -67,7 +67,9 @@ mod render; /// Trait implemented by all versioned queues. pub(crate) trait Queue: Send + Sync { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - if layouts != [0; 2] { return Err(EINVAL); } + if layouts != [0; 2] { + return Err(EINVAL); + } Ok(()) } @@ -266,7 +268,10 @@ impl sched::JobImpl for QueueJob::ver { Some(kernel::module::this_module::()); fn timeout_data(&self) -> QueueTimeout { - QueueTimeout { dev: self.dev.clone(), id: self.id } + QueueTimeout { + dev: self.dev.clone(), + id: self.id, + } } fn prepare(job: &mut Self) -> Option { @@ -764,13 +769,21 @@ impl Queue for Queue::ver { match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(payload_size)?; - if render.ts_frag_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + if render + .ts_frag_end_copies + .iter() + .any(|ts| ts.handle != 0 || ts.offset != 0) + { return Err(EINVAL); } } uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(payload_size)?; - if compute.ts_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + if compute + .ts_end_copies + .iter() + .any(|ts| ts.handle != 0 || ts.offset != 0) + { return Err(EINVAL); } } diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 88709a7f9b36d..7f2de64c424d6 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -163,9 +163,7 @@ impl File { // SAFETY: `printer` is the printer `drm_show_fdinfo()` passes, valid for this call; `raw` // and the region name outlive the call. - unsafe { - bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_ptr()) - }; + unsafe { bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_ptr()) }; } /// The postclose callback of a `struct drm_file`. From 05215160a86feb9a03ea6d834da97d7c281c4312 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:17:39 +1100 Subject: [PATCH 1419/1579] drm/asahi: Account per-client engine time in fdinfo Retain a per-file engine ledger in queues and completions, and report vertex, fragment and compute time through the Rust DRM fdinfo callback. Charge successful firmware timestamp intervals once at retirement, before signalling the completion fence. Shared code handles conversion and saturating atomic accumulation across classic and native backends, including the shared T6040/G16S and T8140/G17P execution path. Enable the classic firmware's existing timestamp operations without requiring application timestamp outputs. Native retirement already reads the required cells. Do not misreport independent command timer ticks as GPU execution-clock cycles or infer frequency-weighted utilization. Document the sources, limits and overhead, and register eight host tests of the production accounting code. Evidence: existing public firmware timestamp and retirement interfaces; host arithmetic and concurrency tests. Kernel build and runtime qualification remain pending an authorized build service route. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/gpu/asahi-usage-stats.rst | 80 ++++++++++++ Documentation/gpu/drivers.rst | 1 + drivers/gpu/drm/asahi/asahi_core.rs | 1 + drivers/gpu/drm/asahi/file.rs | 19 ++- drivers/gpu/drm/asahi/fw/job.rs | 4 - drivers/gpu/drm/asahi/g15/mod.rs | 3 +- drivers/gpu/drm/asahi/g15/poll.rs | 27 ++-- drivers/gpu/drm/asahi/g15/queue.rs | 11 +- drivers/gpu/drm/asahi/g16/mod.rs | 3 +- drivers/gpu/drm/asahi/g16/poll.rs | 35 ++++-- drivers/gpu/drm/asahi/g16/queue.rs | 11 +- drivers/gpu/drm/asahi/g17/completion.rs | 13 ++ drivers/gpu/drm/asahi/g17/mod.rs | 2 + drivers/gpu/drm/asahi/g17/queue/mod.rs | 4 + drivers/gpu/drm/asahi/gpu.rs | 3 + drivers/gpu/drm/asahi/queue/compute.rs | 65 +++++----- drivers/gpu/drm/asahi/queue/mod.rs | 3 + drivers/gpu/drm/asahi/queue/render.rs | 127 ++++++++++--------- drivers/gpu/drm/asahi/usage.rs | 161 ++++++++++++++++++++++++ rust/bindings/bindings_helper.h | 1 + rust/kernel/drm/file.rs | 17 +++ tools/asahi/run-host-tests.py | 3 +- tools/asahi/usage-tests.rs | 6 + 23 files changed, 492 insertions(+), 108 deletions(-) create mode 100644 Documentation/gpu/asahi-usage-stats.rst create mode 100644 drivers/gpu/drm/asahi/usage.rs create mode 100644 tools/asahi/usage-tests.rs diff --git a/Documentation/gpu/asahi-usage-stats.rst b/Documentation/gpu/asahi-usage-stats.rst new file mode 100644 index 0000000000000..f5e731e77d0c2 --- /dev/null +++ b/Documentation/gpu/asahi-usage-stats.rst @@ -0,0 +1,80 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +Asahi client engine time +======================== + +Asahi reports the following optional keys in each DRM file's fdinfo, following +:doc:`drm-usage-stats`:: + + drm-engine-vertex: 123456 ns + drm-engine-fragment: 234567 ns + drm-engine-compute: 345678 ns + +``vertex`` is the tiling/vertex stage of render commands, ``fragment`` is their +fragment stage, and ``compute`` is compute dispatch. These are separate timelines: +vertex and fragment can overlap. There is no separately submitted blit engine in +the driver's command ABI. Copies executed by compute or render are included in +the engine that executes them. + +The counters belong to the open DRM file, rather than to a PID or a queue handle. +All queues and VMs of that file contribute to the same ledger. Duplicated or +inherited file descriptors share counters and must be deduplicated using +``drm-client-id``. Closing a queue or VM does not reset its file's counters, and +completions retain the ledger until their firmware work retires. A new DRM file +starts at zero, even when another file in the same process has submitted work. + +Time source and limits +---------------------- + +Only completed stage intervals are reported. A long-running command contributes +after retirement, so polling during execution can temporarily show no change. +Counters accumulate monotonically, saturating at the largest unsigned 64-bit +value. Conversion rounds down by less than one nanosecond per retired stage. + +G13 and G14 use the firmware's internal start/end timestamp cells. The existing +timestamp microsequence operations run even when the application requests no +timestamp output. Internal buffers were already allocated for each command; +fdinfo never reads an application's timestamp buffer. + +The G15 and native G16 backends reuse the internal timestamp cells already read +by their successful retirement statistics path. The shared G17 execution backend, +including T6040/G16S and T8140/G17P, charges the timestamp pairs delivered by its +once-only completion path. These backends use the firmware's 24 MHz command timer; +the classic backend uses its hardware configuration's timer frequency. + +These are firmware stage envelopes, not hardware shader occupancy measurements. +They can include firmware scheduling, preemption and stalls between the stage's +start and end operations. Intervals are summed without attempting to merge +overlapping commands on the same engine. Missing, zero-start or reversed intervals +are ignored. Failed or cancelled commands do not contribute incomplete intervals; +successful stages that retired before a later failure remain counted. An +unexecuted G17 replay contributes only when it eventually retires successfully. +There is no submit-to-completion wall-time fallback in the current backends. + +``drm-cycles-*`` and ``drm-maxfreq-*`` are intentionally absent. The timestamp +clock is independent of the variable GPU execution clock. Treating command timer +ticks as GPU busy cycles, or multiplying time by a sampled/maximum GPU frequency, +would give incorrect frequency-weighted utilization. Per-client execution-cycle +counters would require a separate hardware measurement source. + +Accounting cost +--------------- + +Each file allocates one three-counter ledger. Queues and pending completions +retain references to it. Retirement adds one relaxed, saturating atomic update +per valid stage, with no new locks or allocations. The native backends reuse +existing timestamp reads. Classic completion callbacks read two internal cells +per stage; commands without application timestamps also execute two existing +timestamp microsequence operations per stage. Reading fdinfo loads three counters +and does not traverse queues, wait for fences or access hardware. + +Host arithmetic tests +--------------------- + +The ledger uses only ``core`` and can be tested directly from a source checkout:: + + rustc --edition 2021 --test drivers/gpu/drm/asahi/usage.rs -o /tmp/asahi-usage-test + /tmp/asahi-usage-test + +Tests cover conversion, missing/reset timestamps, saturation, overlapping stages, +client isolation, out-of-order completions and simultaneous queues sharing a file. diff --git a/Documentation/gpu/drivers.rst b/Documentation/gpu/drivers.rst index ced2721b96c75..51464c3d54891 100644 --- a/Documentation/gpu/drivers.rst +++ b/Documentation/gpu/drivers.rst @@ -9,6 +9,7 @@ GPU Driver Documentation apple-dcp-backlight apple-drm-parameters asahi-reclaim + asahi-usage-stats asahi-m4-bringup apple-dcp-m4 i915 diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 1a584932c5a77..5176ffc7010fb 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -37,6 +37,7 @@ mod slotalloc; mod stats; mod sysfs_exports; mod timestamp; +mod usage; mod util; #[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] mod vm; diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 6cc411cdb7467..ae3d2db8769ac 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -277,6 +277,8 @@ pub(crate) struct File { objects: xarray::XArray>, /// GPU memory this client created and holds. mem: Arc, + /// Completed engine time, retained by queues after handle destruction. + usage: Arc, } /// Convenience type alias for our DRM `File` type. @@ -297,9 +299,10 @@ impl drm::file::DriverFile for File { let id = gpu.ids().file.next(); let user_range = gpu.user_range(); let mem = memacct::Client::new()?; + let usage = Arc::new(crate::usage::Client::new(), GFP_KERNEL)?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - KBox::pin_init(File::new(id, user_range, mem), GFP_KERNEL) + KBox::pin_init(File::new(id, user_range, mem, usage), GFP_KERNEL) }) } @@ -313,6 +316,17 @@ impl drm::file::DriverFile for File { }) } + fn engine_stats(&self, mut emit: impl FnMut(&CStr, u64)) { + let names = [ + kernel::c_str!("vertex"), + kernel::c_str!("fragment"), + kernel::c_str!("compute"), + ]; + for (name, ns) in names.into_iter().zip(self.usage.snapshot()) { + emit(name, ns); + } + } + fn as_raw(&self) -> *mut bindings::drm_file { todo!() } @@ -326,6 +340,7 @@ impl File { id: u64, user_range: Range, mem: Arc, + usage: Arc, ) -> impl PinInit { unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { @@ -352,6 +367,7 @@ impl File { (*slot).id = id; (*slot).user_range = user_range; addr_of_mut!((*slot).mem).write(mem); + addr_of_mut!((*slot).usage).write(usage); Ok(()) }) } @@ -1358,6 +1374,7 @@ impl File { // TODO: Plumb deeper the enum uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority, data.usc_exec_base, + file.inner().usage.clone(), )?; queue.reserve_render_scs(data.render_scs_layout)?; diff --git a/drivers/gpu/drm/asahi/fw/job.rs b/drivers/gpu/drm/asahi/fw/job.rs index e4f2f9225ea05..aaa83355b437d 100644 --- a/drivers/gpu/drm/asahi/fw/job.rs +++ b/drivers/gpu/drm/asahi/fw/job.rs @@ -139,10 +139,6 @@ pub(crate) struct UserTimestamps { } impl UserTimestamps { - pub(crate) fn any(&self) -> bool { - self.start.is_some() || self.end.is_some() - } - pub(crate) fn pointers(&self) -> Result> { Ok(raw::TimestampPointers { start_addr: self diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 718e5b8c851d3..c792c1303475e 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -643,9 +643,10 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { Ok(KBox::new( - queue::Queue::new(self, vm, priority, usc_exec_base)?, + queue::Queue::new(self, vm, priority, usc_exec_base, usage)?, GFP_KERNEL, )?) } diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index cff5d87b2c84e..bf04984b7ed17 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -529,25 +529,38 @@ impl Firmware { let s = &mut self.engine.stats; // Add one stage interval read from internal timestamp cells (start, end). - let mut stage = |o: &super::object::FwObject, off: usize, sum: &mut u64| { - let rd = |x: usize| o.r32(x).unwrap_or(0) as u64; - let start = rd(off + 4) << 32 | rd(off); - let end = rd(off + 12) << 32 | rd(off + 8); + let mut stage = |o: &super::object::FwObject, + off: usize, + sum: &mut u64, + engine: crate::usage::Engine| { + let (Ok(start_lo), Ok(start_hi), Ok(end_lo), Ok(end_hi)) = + (o.r32(off), o.r32(off + 4), o.r32(off + 8), o.r32(off + 12)) + else { + return; + }; + let start = u64::from(start_hi) << 32 | u64::from(start_lo); + let end = u64::from(end_hi) << 32 | u64::from(end_lo); if start == 0 || end < start { return; } + f.job.queue.usage.add(engine, start, end, 24_000_000); *sum += end - start; lo = lo.min(start); hi = hi.max(end); }; for k in 0..f.renders as usize { let p = &scrub.passes[f.claims.pass[k] as usize]; - stage(&p.its, 0, &mut s.ta_ticks); - stage(&p.its, 0x10, &mut s.fs_ticks); + stage(&p.its, 0, &mut s.ta_ticks, crate::usage::Engine::Vertex); + stage( + &p.its, + 0x10, + &mut s.fs_ticks, + crate::usage::Engine::Fragment, + ); } for k in 0..(f.computes as usize).min(f.claims.slot_count as usize) { let c = &scrub.compute[f.claims.slot[k] as usize]; - stage(&c.its, 0, &mut s.cdm_ticks); + stage(&c.its, 0, &mut s.cdm_ticks, crate::usage::Engine::Compute); } if hi > lo { s.span_ticks += hi - lo; diff --git a/drivers/gpu/drm/asahi/g15/queue.rs b/drivers/gpu/drm/asahi/g15/queue.rs index cc87f1ccbe084..03a8f46c8a0fc 100644 --- a/drivers/gpu/drm/asahi/g15/queue.rs +++ b/drivers/gpu/drm/asahi/g15/queue.rs @@ -42,6 +42,8 @@ pub(crate) struct QueueState { /// Host priority: `DRM_ASAHI_PRIORITY_REALTIME - priority` as `file.rs` passes it (MEDIUM 2, /// LOW 3; `file.rs` refuses HIGH and REALTIME). Smaller is served first. pub(crate) priority: u32, + /// Completed stage time of the DRM file that created this queue. + pub(crate) usage: Arc, /// USC executable window base. pub(crate) usc_base: u64, /// Engine service turn of this queue (least recently served first). @@ -103,7 +105,13 @@ impl Drop for Queue { impl Queue { /// Creates a logical queue. `priority` is `DRM_ASAHI_PRIORITY_REALTIME - priority`, as /// `file.rs` passes it. - pub(crate) fn new(gpu: &Gpu, vm: mmu::Vm, priority: u32, usc_exec_base: u64) -> Result { + pub(crate) fn new( + gpu: &Gpu, + vm: mmu::Vm, + priority: u32, + usc_exec_base: u64, + usage: Arc, + ) -> Result { if priority > uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { return Err(EINVAL); } @@ -129,6 +137,7 @@ impl Queue { let state = Arc::new( QueueState { priority, + usage, usc_base: usc_exec_base, turn: AtomicU64::new(0), owned: [const { AtomicU32::new(0) }; 32], diff --git a/drivers/gpu/drm/asahi/g16/mod.rs b/drivers/gpu/drm/asahi/g16/mod.rs index c5c9829880e98..3ee9b66802b54 100644 --- a/drivers/gpu/drm/asahi/g16/mod.rs +++ b/drivers/gpu/drm/asahi/g16/mod.rs @@ -616,9 +616,10 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { Ok(KBox::new( - queue::Queue::new(self, vm, priority, usc_exec_base)?, + queue::Queue::new(self, vm, priority, usc_exec_base, usage)?, GFP_KERNEL, )?) } diff --git a/drivers/gpu/drm/asahi/g16/poll.rs b/drivers/gpu/drm/asahi/g16/poll.rs index 172296263398f..9dd6c965da6c4 100644 --- a/drivers/gpu/drm/asahi/g16/poll.rs +++ b/drivers/gpu/drm/asahi/g16/poll.rs @@ -694,13 +694,24 @@ impl Firmware { let s = &mut self.engine.stats; // Add one stage interval read from internal timestamp cells (start, end). - let mut stage = |o: &super::object::FwObject, off: usize, sum: &mut u64| { - let rd = |x: usize| o.r32(x).unwrap_or(0) as u64; - let start = rd(off + 4) << 32 | rd(off); - let end = rd(off + 12) << 32 | rd(off + 8); + let mut stage = |o: &super::object::FwObject, + off: usize, + sum: &mut u64, + engine: crate::usage::Engine| { + let (Ok(start_lo), Ok(start_hi), Ok(end_lo), Ok(end_hi)) = + (o.r32(off), o.r32(off + 4), o.r32(off + 8), o.r32(off + 12)) + else { + return; + }; + let start = u64::from(start_hi) << 32 | u64::from(start_lo); + let end = u64::from(end_hi) << 32 | u64::from(end_lo); if start == 0 || end < start { return; } + f.job + .queue + .usage + .add(engine, start, end, super::hw::COMMAND_TIMESTAMP_HZ as u32); *sum = sum.saturating_add(end - start); lo = lo.min(start); hi = hi.max(end); @@ -710,15 +721,25 @@ impl Firmware { return; }; let p = &arena.passes[f.claims.pass[k] as usize]; - stage(&p.scene, 0x600, &mut s.ta_ticks); - stage(&p.scene, 0x620, &mut s.fs_ticks); + stage( + &p.scene, + 0x600, + &mut s.ta_ticks, + crate::usage::Engine::Vertex, + ); + stage( + &p.scene, + 0x620, + &mut s.fs_ticks, + crate::usage::Engine::Fragment, + ); } for k in 0..(f.computes as usize).min(f.claims.slot_count as usize) { let Some(arena) = f.job.resources.compute.as_ref() else { return; }; let c = &arena.compute[f.claims.slot[k] as usize]; - stage(&c.its, 0, &mut s.cdm_ticks); + stage(&c.its, 0, &mut s.cdm_ticks, crate::usage::Engine::Compute); } if hi > lo { s.span_ticks = s.span_ticks.saturating_add(hi - lo); diff --git a/drivers/gpu/drm/asahi/g16/queue.rs b/drivers/gpu/drm/asahi/g16/queue.rs index 08a2093ef527b..41da5b94c7c50 100644 --- a/drivers/gpu/drm/asahi/g16/queue.rs +++ b/drivers/gpu/drm/asahi/g16/queue.rs @@ -43,6 +43,8 @@ pub(crate) struct QueueState { /// Host priority: `DRM_ASAHI_PRIORITY_REALTIME - priority` as `file.rs` passes it (MEDIUM 2, /// LOW 3; `file.rs` refuses HIGH and REALTIME). Smaller is served first. pub(crate) priority: u32, + /// Completed stage time of the DRM file that created this queue. + pub(crate) usage: Arc, /// USC executable window base. pub(crate) usc_base: u64, /// First cancelled accepted ordinal per engine; zero means none. Later scheduler @@ -102,7 +104,13 @@ pub(crate) struct Queue { impl Queue { /// Creates a logical queue. `priority` is `DRM_ASAHI_PRIORITY_REALTIME - priority`, as /// `file.rs` passes it. - pub(crate) fn new(gpu: &Gpu, vm: mmu::Vm, priority: u32, usc_exec_base: u64) -> Result { + pub(crate) fn new( + gpu: &Gpu, + vm: mmu::Vm, + priority: u32, + usc_exec_base: u64, + usage: Arc, + ) -> Result { if priority > uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { return Err(EINVAL); } @@ -127,6 +135,7 @@ impl Queue { let state = Arc::new( QueueState { priority, + usage, usc_base: usc_exec_base, cancelled_at: [const { AtomicU64::new(0) }; 2], turn: AtomicU64::new(0), diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index a232b9c9eb2cc..e7249cdc3d32c 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -255,6 +255,7 @@ pub(crate) struct Completion { terminal: Option>, compute: Option, feed: Option>, + usage: Arc, sources: Option>, #[pin] vm_job: Mutex>, @@ -292,6 +293,7 @@ impl Completion { host_predecessors: bool, feed: Option>, sources: Option>, + usage: Arc, ) -> Result> { let fence = fence::independent(contexts)?; let ordered = !compute && (host_predecessors || destinations.iter().any(Option::is_some)); @@ -322,6 +324,7 @@ impl Completion { terminal, compute, feed, + usage, sources, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), @@ -651,6 +654,16 @@ impl Completion { if result.is_ok() && self.interrupted() { result = Err(ECANCELED); } + if let Ok(ticks) = &result { + use crate::usage::Engine; + let hz = cfg::COMMAND_TIMESTAMP_HZ as u32; + if self.compute.is_some() { + self.usage.add(Engine::Compute, ticks[0], ticks[1], hz); + } else { + self.usage.add(Engine::Vertex, ticks[0], ticks[1], hz); + self.usage.add(Engine::Fragment, ticks[2], ticks[3], hz); + } + } if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { // Compute reports its dispatch envelope in the first pair, a // render its fragment pass in the second. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index ae5bbd61a3ce4..c2962c45f7afc 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1701,6 +1701,7 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { let dev = self.shared.drm()?; let owner = self.ids.queue.next(); @@ -1742,6 +1743,7 @@ impl gpu::Gpu for Gpu { backend.clone(), frontend, slot, + usage, ) { Ok(frontend) => frontend, Err(error) => { diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 789c4451fd636..e593c5879cf74 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -84,6 +84,7 @@ impl Frontend { pub(crate) struct Queue { dev: AsahiDevRef, context: Arc, + usage: Arc, backend: Arc, render: Option>, compute: Option>, @@ -116,6 +117,7 @@ impl Queue { backend: Arc, frontend: Frontend, slot: admission::QueueSlot, + usage: Arc, ) -> Result { let Frontend { fences, @@ -128,6 +130,7 @@ impl Queue { Ok(Self { dev: dev.into(), context, + usage, backend, render, compute: None, @@ -277,6 +280,7 @@ impl Queue { host_predecessors.clone().next().is_some(), self.backend.feed(), sources.clone(), + self.usage.clone(), )?; let publication = completion .publication() diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 3f2672f803ce9..343fb1c84df50 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -367,6 +367,7 @@ pub(crate) trait Gpu: Send + Sync { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result>; /// Bind a BO range as a firmware- or CPU-written timestamp buffer. fn map_timestamp_buffer( @@ -1411,6 +1412,7 @@ impl Gpu for GpuManager::ver { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { let mut kalloc = self.alloc(); let id = self.ids.queue.next(); @@ -1428,6 +1430,7 @@ impl Gpu for GpuManager::ver { id, priority, usc_exec_base, + usage, )?, GFP_KERNEL, )?) diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index c9f205708e313..ff9804c5bd10b 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -102,6 +102,10 @@ impl super::QueueInner::ver { GFP_KERNEL, )?; + let usage_timestamps = timestamps.clone(); + let usage = self.usage.clone(); + let timestamp_hz = gpu.get_cfg().base_clock_hz; + let uuid = 0; mod_dev_dbg!(self.dev, "[Submission {}] UUID = {:#x?}\n", id, uuid); @@ -150,20 +154,18 @@ impl super::QueueInner::ver { notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), })?; - if user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), - work_queue: ev_comp.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, context_store_req), - uuid, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_comp.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, context_store_req), + uuid, + unk_30_padding: 0, + })?; #[ver(G < G14X)] builder.add(microseq::WaitForIdle { @@ -174,20 +176,18 @@ impl super::QueueInner::ver { header: microseq::op::WaitForIdle2::HEADER, })?; - if user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), - work_queue: ev_comp.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, context_store_req), - uuid, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_comp.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, context_store_req), + uuid, + unk_30_padding: 0, + })?; let off = builder.offset_to(start_comp); builder.add(microseq::FinalizeCompute::ver { @@ -362,6 +362,15 @@ impl super::QueueInner::ver { comp_job.add_cb(comp, vm_bind.slot(), move |error| { if let Some(err) = error { fence.set_error(err.into()) + } else { + usage_timestamps.with(|raw, _| { + usage.add( + crate::usage::Engine::Compute, + raw.start.load(Ordering::Relaxed), + raw.end.load(Ordering::Relaxed), + timestamp_hz, + ); + }); } fence.command_complete(); diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 653fd77d0c3f3..40baaa2e6d897 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -153,6 +153,7 @@ pub(crate) struct QueueInner { notifier_list: Arc>, notifier: Arc>, usc_exec_base: u64, + usage: Arc, id: u64, #[ver(V >= V13_0B4)] counter: AtomicU64, @@ -487,6 +488,7 @@ impl Queue::ver { id: u64, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result { mod_dev_dbg!(dev, "[Queue {}] Creating queue\n", id); @@ -557,6 +559,7 @@ impl Queue::ver { notifier_list: Arc::new(notifier_list, GFP_KERNEL)?, notifier, usc_exec_base, + usage, id, #[ver(V >= V13_0B4)] counter: AtomicU64::new(0), diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 68c4856ca4200..4de0e84239a02 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -511,20 +511,18 @@ impl super::QueueInner::ver { notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), })?; - if frg_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), - work_queue: ev_frag.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_3d, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_frag.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; #[ver(G < G14X)] builder.add(microseq::WaitForIdle { @@ -535,20 +533,18 @@ impl super::QueueInner::ver { header: microseq::op::WaitForIdle2::HEADER, })?; - if frg_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), - work_queue: ev_frag.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_3d, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_frag.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; let off = builder.offset_to(start_frag); builder.add(microseq::FinalizeFragment::ver { @@ -929,9 +925,21 @@ impl super::QueueInner::ver { mod_dev_dbg!(self.dev, "[Submission {}] Add Frag\n", id); fence.add_command(); + let usage = self.usage.clone(); + let usage_timestamps = timestamps.clone(); + let timestamp_hz = gpu.get_cfg().base_clock_hz; frag_job.add_cb(frag, vm_bind.slot(), move |error| { if let Some(err) = error { fence.set_error(err.into()); + } else { + usage_timestamps.with(|raw, _| { + usage.add( + crate::usage::Engine::Fragment, + raw.frag.start.load(Ordering::Relaxed), + raw.frag.end.load(Ordering::Relaxed), + timestamp_hz, + ); + }); } fence.command_complete(); @@ -1022,20 +1030,18 @@ impl super::QueueInner::ver { unk_178: (!clustering) as u32, })?; - if vtx_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), - work_queue: ev_vtx.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_ta, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_vtx.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; #[ver(G < G14X)] builder.add(microseq::WaitForIdle { @@ -1046,20 +1052,18 @@ impl super::QueueInner::ver { header: microseq::op::WaitForIdle2::HEADER, })?; - if vtx_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), - work_queue: ev_vtx.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_ta, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_vtx.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; let off = builder.offset_to(start_vtx); builder.add(microseq::FinalizeVertex::ver { @@ -1368,9 +1372,20 @@ impl super::QueueInner::ver { mod_dev_dbg!(self.dev, "[Submission {}] Add Vertex\n", id); fence.add_command(); + let usage = self.usage.clone(); + let usage_timestamps = timestamps.clone(); vtx_job.add_cb(vtx, vm_bind.slot(), move |error| { if let Some(err) = error { fence.set_error(err.into()) + } else { + usage_timestamps.with(|raw, _| { + usage.add( + crate::usage::Engine::Vertex, + raw.vtx.start.load(Ordering::Relaxed), + raw.vtx.end.load(Ordering::Relaxed), + timestamp_hz, + ); + }); } fence.command_complete(); diff --git a/drivers/gpu/drm/asahi/usage.rs b/drivers/gpu/drm/asahi/usage.rs new file mode 100644 index 0000000000000..496c722dba651 --- /dev/null +++ b/drivers/gpu/drm/asahi/usage.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-file engine time, shared by all firmware backends. +//! +//! Queues retain their file's ledger until their last completion. Only authenticated, +//! successful completion timestamps are charged, before signalling the completion fence. +//! These are command timer ticks, not shader clock cycles. Never export them as drm-cycles. + +use core::sync::atomic::{AtomicU64, Ordering}; + +#[derive(Clone, Copy)] +pub(crate) enum Engine { + Vertex, + Fragment, + Compute, +} + +/// One ledger per DRM file, shared by all of its queues and VMs. +pub(crate) struct Client { + ns: [AtomicU64; 3], +} + +impl Client { + pub(crate) const fn new() -> Self { + Self { + ns: [const { AtomicU64::new(0) }; 3], + } + } + + /// Called exactly once per retired stage, while its internal timestamps are retained. + /// Missing, reversed or reset timestamps contribute nothing. Sub-nanosecond fractions + /// are rounded down per stage. Saturation keeps fdinfo monotonic even on overflow. + pub(crate) fn add(&self, engine: Engine, start: u64, end: u64, frequency_hz: u32) { + if start == 0 || end <= start || frequency_hz == 0 { + return; + } + let ns = ticks_to_ns(end - start, frequency_hz); + let counter = &self.ns[engine as usize]; + let mut old = counter.load(Ordering::Relaxed); + loop { + match counter.compare_exchange_weak( + old, + old.saturating_add(ns), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(value) => old = value, + } + } + } + + /// Vertex, fragment and compute, in ns. No cross-engine atomic snapshot is promised. + pub(crate) fn snapshot(&self) -> [u64; 3] { + self.ns.each_ref().map(|ns| ns.load(Ordering::Relaxed)) + } +} + +fn ticks_to_ns(ticks: u64, frequency_hz: u32) -> u64 { + const NSEC_PER_SEC: u64 = 1_000_000_000; + let hz = u64::from(frequency_hz); + // The remainder is smaller than a u32 frequency, so its product fits in u64. + (ticks / hz) + .saturating_mul(NSEC_PER_SEC) + .saturating_add(ticks % hz * NSEC_PER_SEC / hz) +} + +#[cfg(test)] +mod tests { + use super::*; + + const HZ: u32 = 24_000_000; + + #[test] + fn idle_and_invalid_intervals() { + let client = Client::new(); + assert_eq!(client.snapshot(), [0; 3]); + for (start, end, hz) in [ + (0, 100, HZ), + (100, 0, HZ), + (20, 10, HZ), + (1, 1, HZ), + (1, 2, 0), + ] { + client.add(Engine::Compute, start, end, hz); + } + assert_eq!(client.snapshot(), [0; 3]); + } + + #[test] + fn engine_and_client_isolation() { + let a = Client::new(); + let b = Client::new(); + a.add(Engine::Vertex, 10, 24_010, HZ); + a.add(Engine::Fragment, 20, 48_020, HZ); + b.add(Engine::Compute, 30, 72_030, HZ); + assert_eq!(a.snapshot(), [1_000_000, 2_000_000, 0]); + assert_eq!(b.snapshot(), [0, 0, 3_000_000]); + } + + #[test] + fn overlapping_stages_are_independent() { + let client = Client::new(); + client.add(Engine::Vertex, 1, 25, HZ); + client.add(Engine::Fragment, 13, 37, HZ); + assert_eq!(client.snapshot(), [1_000, 1_000, 0]); + } + + #[test] + fn conversion_matches_wide_reference() { + for hz in [1, HZ, 1_000_000_000, u32::MAX] { + for ticks in [0, 1, 23, 24, 25, u32::MAX as u64, u64::MAX / 2, u64::MAX] { + let expected = (u128::from(ticks) * 1_000_000_000 / u128::from(hz)) + .min(u128::from(u64::MAX)) as u64; + assert_eq!(ticks_to_ns(ticks, hz), expected); + } + } + } + + #[test] + fn fractional_ticks_round_down() { + assert_eq!(ticks_to_ns(1, HZ), 41); + assert_eq!(ticks_to_ns(3, HZ), 125); + } + + #[test] + fn out_of_order_completions_accumulate() { + let client = Client::new(); + client.add(Engine::Compute, 100, 124, HZ); + client.add(Engine::Compute, 1, 25, HZ); + assert_eq!(client.snapshot(), [0, 0, 2_000]); + } + + #[test] + fn saturation_is_monotonic() { + let client = Client::new(); + client.add(Engine::Compute, 1, u64::MAX, 1); + assert_eq!(client.snapshot()[2], u64::MAX); + client.add(Engine::Compute, 1, 25, HZ); + assert_eq!(client.snapshot()[2], u64::MAX); + } + + #[test] + fn concurrent_queues_share_one_file() { + let client = std::sync::Arc::new(Client::new()); + let threads: std::vec::Vec<_> = (0..8) + .map(|_| { + let client = client.clone(); + std::thread::spawn(move || { + for _ in 0..10_000 { + client.add(Engine::Compute, 1, 25, HZ); + } + }) + }) + .collect(); + for thread in threads { + thread.join().unwrap(); + } + assert_eq!(client.snapshot(), [0, 0, 80_000_000]); + } +} diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 23aa685923368..ff4f810297ba4 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 7f2de64c424d6..f152957397221 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -44,6 +44,11 @@ pub trait DriverFile { fn memory_stats(&self) -> Option { None } + + /// Emit completed per-engine busy time in nanoseconds for this client's fdinfo. + /// Engine names must be stable, unique and contain no whitespace or colons. + /// The default emits no engine keys. See `Documentation/gpu/drm-usage-stats.rst`. + fn engine_stats(&self, _emit: impl FnMut(&crate::str::CStr, u64)) {} } /// An open DRM File. @@ -136,6 +141,18 @@ impl File { // `driver_priv` was set by `open_callback()`. The seq_file read holds a reference to the // file, so `postclose_callback()` cannot run concurrently. let file = unsafe { File::::from_raw(raw_file) }; + file.inner().engine_stats(|name, ns| { + // SAFETY: The DRM core owns printer throughout this callback; the format and + // name are NUL-terminated, and the vararg types match %s and %llu. + unsafe { + bindings::drm_printf( + printer, + c"drm-engine-%s:\t%llu ns\n".as_ptr(), + name.as_char_ptr(), + ns as core::ffi::c_ulonglong, + ) + }; + }); let Some(stats) = file.inner().memory_stats() else { return; }; diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index bcc42fe042c0f..20c44bc4231bb 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -134,7 +134,8 @@ def mutations(flag, names, base, expect=KILL): } # Rust unit-test files built with `rustc --test`, and shell drivers. -ASAHI_RUST = ['m3-batching-tests.rs', 'm3-runtime-tests.rs', 'pmp-bootargs-tests.rs', 'pmp-power-tests.rs'] +ASAHI_RUST = ['m3-batching-tests.rs', 'm3-runtime-tests.rs', 'pmp-bootargs-tests.rs', 'pmp-power-tests.rs', + 'usage-tests.rs'] ASAHI_SHELL = ['g17-ray-query-timeout-tests.sh'] # Python that runs on the host but imports its neighbours, so it keeps the diff --git a/tools/asahi/usage-tests.rs b/tools/asahi/usage-tests.rs new file mode 100644 index 0000000000000..67bbbc58d6276 --- /dev/null +++ b/tools/asahi/usage-tests.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Run the production per-client engine ledger's tests without a kernel build. + +#[path = "../../drivers/gpu/drm/asahi/usage.rs"] +mod usage; From e2f5cd6967dfd915f6a0e90aab4721bf11fb39d7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:19:05 +1100 Subject: [PATCH 1420/1579] test: asahi: Pass the client ledger to queue construction Update the extracted G15 queue-credit constructor test for its new ledger argument and multiline signature. Include the production ledger in the fixture and record its source digest with the queue and hardware sources. The control and all six intended runtime mutation failures pass. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/g15-queue-credit-tests.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/asahi/g15-queue-credit-tests.py b/tools/asahi/g15-queue-credit-tests.py index 2c8d7f64fb3f3..ee615d28d234f 100644 --- a/tools/asahi/g15-queue-credit-tests.py +++ b/tools/asahi/g15-queue-credit-tests.py @@ -26,6 +26,7 @@ def extract(text, marker): CODE = r''' #![allow(dead_code)] +__USAGE__ use std::{sync::{Arc as StdArc,Mutex,Barrier,OnceLock,atomic::{AtomicBool,AtomicI32,AtomicU32,AtomicU64,Ordering}},ops::Deref,cell::Cell}; #[derive(Copy,Clone,Debug,PartialEq,Eq)]struct Error(i32); type Result = std::result::Result; @@ -70,7 +71,7 @@ def extract(text, marker): Gpu{shared:Arc::wrap(Shared{logical_queues:counter,crashed:AtomicBool::new(false),dev:Device{trace}})} } fn vm(gpu:&Gpu,status:i32)->mmu::Vm{mmu::Vm{state:Arc::wrap(VmState{status:Status(AtomicI32::new(status)),trace:gpu.shared.dev.trace.clone()})}} -fn new(gpu:&Gpu,status:i32,priority:u32,base:u64)->Result>{KBox::new(Queue::new(gpu,vm(gpu,status),priority,base)?,GFP_KERNEL)} +fn new(gpu:&Gpu,status:i32,priority:u32,base:u64)->Result>{KBox::new(Queue::new(gpu,vm(gpu,status),priority,base,Arc::wrap(usage::Client::new()))?,GFP_KERNEL)} fn count(gpu:&Gpu)->u32{gpu.shared.logical_queues.load(Ordering::SeqCst)} fn main(){ // One device shared across different VM objects/files: exactly58 live queues. @@ -115,12 +116,14 @@ def main(): parser.add_argument("--mutations", action="store_true") args = parser.parse_args(); args.output.mkdir(parents=True, exist_ok=True) queue = (ROOT / "drivers/gpu/drm/asahi/g15/queue.rs").read_text() + usage = (ROOT / "drivers/gpu/drm/asahi/usage.rs").read_text() hardware = (ROOT / "drivers/gpu/drm/asahi/g15/hw.rs").read_text() limit = next(line.strip() for line in hardware.splitlines() if "const MAX_QUEUES:" in line) - replacements = {"__LIMIT__": limit, "__STATE__": extract(queue, "pub(crate) struct QueueState {"), + replacements = {"__USAGE__": "mod usage {\n" + usage + "\n}", + "__LIMIT__": limit, "__STATE__": extract(queue, "pub(crate) struct QueueState {"), "__CREDIT__": extract(queue, "struct QueueCredit {"), "__CREDIT_IMPL__": extract(queue, "impl QueueCredit {"), "__CREDIT_DROP__": extract(queue, "impl Drop for QueueCredit {"), "__QUEUE__": extract(queue, "pub(crate) struct Queue {"), - "__QUEUE_DROP__": extract(queue, "impl Drop for Queue {"), "__NEW__": extract(queue, " pub(crate) fn new(gpu: &Gpu,")} + "__QUEUE_DROP__": extract(queue, "impl Drop for Queue {"), "__NEW__": extract(queue, " pub(crate) fn new(")} code = CODE for old, new in replacements.items(): code = code.replace(old, new) variants = {"control": code}; expected = {} @@ -143,6 +146,7 @@ def main(): variants[name] = variant; expected[name] = message variants["control-after"] = code receipt = {"scope": __doc__, "sources": {"queue.rs": hashlib.sha256(queue.encode()).hexdigest(), + "usage.rs": hashlib.sha256(usage.encode()).hexdigest(), "hw.rs": hashlib.sha256(hardware.encode()).hexdigest()}, "variants": {}} for name, source in variants.items(): path=args.output/(name+".rs");path.write_text(source);binary=args.output/name From 0c633c982c3526abaa7a9dce585241493e06eefc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:24:55 +1100 Subject: [PATCH 1421/1579] phy: apple: atc: Skip stopped PIPE locks on M1/M2 A USB-C dock unplug on T6000 logs "Pipehandler lock not acked." and "Failed to lock pipehandler" after xHCI unregisters. The Apple DWC3 glue intentionally exits host mode before powering off the USB3 PHY, whose power-off callback then parks PIPE. The stopped controller no longer clocks the PIPE lock acknowledgment. Enable the existing park_pipe_unlocked capability for the shared T8103 hardware description selected by M1/M2 PHYs, including T6000 through its device-tree fallback. As on T8122, stopped power-off and reset can park without this best-effort handshake. Live handoffs and host BIST still request the lock, and failed live handoffs retain their ownership. Add KUnit checks that count and refuse lock requests. Stopped ordinary ports must park without requesting one, including pending handoffs and USB3+DP shutdown which must preserve the DP mode. Live parking and host setup must still propagate the refused lock. Removing the capability makes both stopped-path regressions fail. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-test.c | 95 ++++++++++++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 1 + 2 files changed, 96 insertions(+) diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index 366b4b86f115e..318cce4627d0f 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -676,6 +676,98 @@ static void atcphy_test_stopped_reset(struct kunit *test) KUNIT_EXPECT_FALSE(test, aon & PIPEHANDLER_AON_GEN_DWC3_RESET_N); } +static int atcphy_test_count_refused_lock(struct apple_atcphy *atcphy) +{ + unsigned int *requests = kunit_get_current_test()->priv; + + (*requests)++; + return -ETIMEDOUT; +} + +static struct apple_atcphy *atcphy_test_alloc_legacy_pipe(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, false); + void *lpdptx = kunit_kzalloc(test, LPDPTX_AUX_CONTROL + 4, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, lpdptx); + mutex_init(&atcphy->lock); + atcphy_test_fake_regs(test, atcphy); + atcphy->regs.lpdptx = (__force void __iomem *)lpdptx; + atcphy->host_active = true; + kunit_activate_static_stub(test, atcphy_pipehandler_lock, + atcphy_test_count_refused_lock); + + return atcphy; +} + +/* Ordinary M1/M2 ports have no PIPE ACK clock once xHCI has stopped. */ +static void atcphy_test_legacy_stopped_power_off(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_legacy_pipe(test); + struct phy *phy = kunit_kzalloc(test, sizeof(*phy), GFP_KERNEL); + unsigned int requests = 0; + + KUNIT_ASSERT_NOT_NULL(test, phy); + phy_set_drvdata(phy, atcphy); + test->priv = &requests; + + for (unsigned int pending = 0; pending < 2; pending++) { + for (unsigned int dp = 0; dp < 2; dp++) { + atcphy->mode = dp ? APPLE_ATCPHY_MODE_USB3_DP : APPLE_ATCPHY_MODE_USB3; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; + atcphy->pipe_pending = pending; + atcphy->host_active = true; + KUNIT_ASSERT_EQ(test, atcphy_usb3_power_off(phy), 0); + KUNIT_EXPECT_EQ(test, requests, 0U); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); + KUNIT_EXPECT_FALSE(test, atcphy->host_active); + KUNIT_EXPECT_EQ(test, atcphy->mode, + dp ? APPLE_ATCPHY_MODE_USB3_DP : APPLE_ATCPHY_MODE_OFF); + } + } +} + +static void atcphy_test_legacy_stopped_reset(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_legacy_pipe(test); + unsigned int requests = 0; + u32 aon; + + test->priv = &requests; + atcphy->pipe_pending = true; + KUNIT_ASSERT_EQ(test, atcphy_dwc3_reset_assert(&atcphy->rcdev, 0), 0); + KUNIT_EXPECT_EQ(test, requests, 0U); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); + KUNIT_EXPECT_FALSE(test, atcphy->host_active); + aon = readl(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN); + KUNIT_EXPECT_TRUE(test, aon & PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + KUNIT_EXPECT_FALSE(test, aon & PIPEHANDLER_AON_GEN_DWC3_RESET_N); +} + +/* Neither a live pending handoff nor USB3 host setup may bypass the ACK. */ +static void atcphy_test_legacy_live_lock(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_legacy_pipe(test); + unsigned int requests = 0; + + test->priv = &requests; + atcphy->pipe_pending = true; + scoped_guard(mutex, &atcphy->lock) + KUNIT_EXPECT_EQ(test, atcphy_park_pipehandler(atcphy, false), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, requests, 1U); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_USB3); + KUNIT_EXPECT_TRUE(test, atcphy->pipe_pending); + KUNIT_EXPECT_TRUE(test, atcphy->mode_dirty); + + atcphy->mode_dirty = false; + scoped_guard(mutex, &atcphy->lock) + KUNIT_EXPECT_EQ(test, atcphy_configure_pipehandler(atcphy, true), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, requests, 2U); + KUNIT_EXPECT_TRUE(test, atcphy->pipe_pending); +} + /* A SuperSpeed problem does not keep the host behind a fixed hub from starting */ static void atcphy_test_host_start_without_usb3(struct kunit *test) { @@ -1207,6 +1299,9 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_stopped_power_off), KUNIT_CASE(atcphy_test_stopped_power_off_dp), KUNIT_CASE(atcphy_test_stopped_reset), + KUNIT_CASE(atcphy_test_legacy_stopped_power_off), + KUNIT_CASE(atcphy_test_legacy_stopped_reset), + KUNIT_CASE(atcphy_test_legacy_live_lock), KUNIT_CASE(atcphy_test_host_start_without_usb3), KUNIT_CASE(atcphy_test_election_held_safe), KUNIT_CASE(atcphy_test_election_held_usb), diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 84cef035e0865..1721431e725a3 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -6202,6 +6202,7 @@ static const struct atcphy_hw atcphy_hw_t8103 = { .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, .has_usb4 = true, + .park_pipe_unlocked = true, }; static const struct atcphy_hw atcphy_hw_t8122 = { From 579de97d141f1fbcaca250b7d17258da2988ce9b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 01:59:54 +1100 Subject: [PATCH 1422/1579] phy: apple: atc: Keep absent SS receivers quiet on USB2 links The USB mux state also brings up the SS PHY for USB2-only devices and cables. A bounded handover retrain which finds no SS receiver is normal there. Keep that ENOTCONN result in the helper's debug trace and reserve the PHY warning for control and reset errors. The return value still describes successful port-power restoration; no reset, retry, ownership or lane configuration behavior changes. Fixes: 8df6578502e7 ("usb: core: Retrain an unconnected SS port after a lane handover") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 91ecba31cdc7f..47e749ce29c70 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -5090,7 +5090,8 @@ static int atcphy_usb3_port_on(struct apple_atcphy *atcphy, struct atcphy_usb3_p else if (!ret && atcphy->fixed_usb2 && atcphy->hw->has_t8140_dp) { int retrain = usb_hub_port_retrain(pm->root, 1); - if (retrain < 0) + /* An absent SS receiver is normal for a USB2-only device or cable. */ + if (retrain < 0 && retrain != -ENOTCONN) dev_warn(atcphy->dev, "SuperSpeed handover retrain failed: %d\n", retrain); /* Power was restored even if no receiver answered the bounded retrain. */ } From 855164834e489dfe4fc2c5643a36ccade0305555 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 03:36:05 +1100 Subject: [PATCH 1423/1579] dt-bindings: dma: apple: name the T8140 PCIe bootstrap contract Introduce apple,t8140-pcie-bootstrap for the bounded radio PCIe bootstrap. Keep the original compatible for deployed Stage 2 DTBs and permit the old compatible as a fallback in new DTBs. Retain the no-teardown contract. Assisted-by: LLM Signed-off-by: Ryan Murray --- ...c.yaml => apple,t8140-pcie-bootstrap.yaml} | 22 +++++++++++++------ MAINTAINERS | 1 + 2 files changed, 16 insertions(+), 7 deletions(-) rename Documentation/devicetree/bindings/dma/{apple,t8140-piodma-diagnostic.yaml => apple,t8140-pcie-bootstrap.yaml} (53%) diff --git a/Documentation/devicetree/bindings/dma/apple,t8140-piodma-diagnostic.yaml b/Documentation/devicetree/bindings/dma/apple,t8140-pcie-bootstrap.yaml similarity index 53% rename from Documentation/devicetree/bindings/dma/apple,t8140-piodma-diagnostic.yaml rename to Documentation/devicetree/bindings/dma/apple,t8140-pcie-bootstrap.yaml index a6a8d7d3af77a..2db00ecb45a21 100644 --- a/Documentation/devicetree/bindings/dma/apple,t8140-piodma-diagnostic.yaml +++ b/Documentation/devicetree/bindings/dma/apple,t8140-pcie-bootstrap.yaml @@ -1,23 +1,31 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- -$id: http://devicetree.org/schemas/dma/apple,t8140-piodma-diagnostic.yaml# +$id: http://devicetree.org/schemas/dma/apple,t8140-pcie-bootstrap.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple T8140 experimental PCIe PIODMA bootstrap engine +title: Apple T8140 PCIe bootstrap engine maintainers: - Ace description: | - The J700 diagnostic driver uses this engine for a single retained request - before enabling native downstream ECAM access. It does not expose generic - DMA channels. This experimental binding describes that bounded bootstrap; - it does not establish an engine reset, removal or memory reuse contract. + This engine performs a single retained config read before native downstream + ECAM access is enabled for the T8140 radio PCIe host. It does not expose + generic DMA channels or establish an engine reset or memory reuse contract. + The host selects this engine with apple,piodma; apple,pcie-host must point + back to that host. The legacy compatible is supported for existing Stage 2 + device trees. properties: compatible: - const: apple,t8140-piodma-diagnostic + oneOf: + - const: apple,t8140-pcie-bootstrap + - const: apple,t8140-piodma-diagnostic + deprecated: true + - items: + - const: apple,t8140-pcie-bootstrap + - const: apple,t8140-piodma-diagnostic reg: items: diff --git a/MAINTAINERS b/MAINTAINERS index a8c16694f668a..72d0dab866e31 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2082,6 +2082,7 @@ W: https://aurora-silicon.github.io S: Maintained F: Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml F: Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +F: Documentation/devicetree/bindings/dma/apple,t8140-pcie-bootstrap.yaml F: Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml F: Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml F: drivers/pmdomain/apple/pmp-report.c From 5d5431357ddd250c385330e0d0f52be8b48ed78c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 03:36:05 +1100 Subject: [PATCH 1424/1579] PCI: apple: make the T8140 radio bootstrap DT selected and explicit Give the bounded PIODMA bootstrap a production name and select it only by reciprocal host/engine DT links. Accept old Stage 2 compatibles and supply the legacy fallback in the J700 DT. Keep only one success info message, after both functions pass host admission. Replace independent ownership flags with ordered state transitions shared by KUnit guard tests. Preserve the attempt limit, pre-child PM/kexec interlocks, pre-publication unwind and irreversible ownership after the first engine write. Drop unused last-IRQ trace storage. Completion and an idle FIFO do not prove DMA teardown. Public history has no qualified engine reset, so retain the arena, failure reset requirement and lifecycle restrictions. Document the history behind each guard and why link reset or bootstrap replay cannot retire it. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/PCI/controller/apple-t8140.rst | 172 ++++-- .../device_drivers/wifi/mt7932-neo.rst | 9 +- MAINTAINERS | 3 +- arch/arm64/boot/dts/apple/t8140-j700.dts | 7 +- arch/arm64/configs/asahi.config | 2 +- drivers/net/wireless/mediatek/mt7932/Kconfig | 2 +- drivers/pci/controller/Kconfig | 41 +- drivers/pci/controller/Makefile | 3 +- .../pcie-apple-t8140-bootstrap-state.h | 140 +++++ .../pcie-apple-t8140-bootstrap-test.c | 148 +++++ ...ma-diag.c => pcie-apple-t8140-bootstrap.c} | 534 +++++++++--------- ...ma-diag.h => pcie-apple-t8140-bootstrap.h} | 12 +- drivers/pci/controller/pcie-apple.c | 38 +- include/linux/soc/apple/pci-apple-piodma.h | 2 +- .../configs/apple_t8140_bootstrap.config | 3 + tools/testing/kunit/configs/mt7932.config | 2 +- 16 files changed, 756 insertions(+), 362 deletions(-) create mode 100644 drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h create mode 100644 drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c rename drivers/pci/controller/{pcie-apple-piodma-diag.c => pcie-apple-t8140-bootstrap.c} (53%) rename drivers/pci/controller/{pcie-apple-piodma-diag.h => pcie-apple-t8140-bootstrap.h} (84%) create mode 100644 tools/testing/kunit/configs/apple_t8140_bootstrap.config diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 4d016f4b4ecc9..53948d60375dc 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -1,39 +1,141 @@ .. SPDX-License-Identifier: GPL-2.0-only -T8140/J700 experimental radio PCIe bootstrap -============================================ - -The J700 path trains port 0, validates a bounded PIODMA bootstrap request, -and enables native ECAM for the MediaTek Wi-Fi and Bluetooth functions. -The root bridge forwards their BARs and DART translates endpoint DMA. - -This is a bring-up implementation, not general T8140 PCIe support. -Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``. The board device tree selects the -bootstrap by pointing the host's ``apple,piodma`` property at the engine, and -the kernel then runs it during the host probe. A host without that property, -or a kernel built without the option, uses the root-only path, which rejects -downstream configuration access. -Build the host and its DART in as well: the supplier retains published DMA -and has no removal contract. Keeping the supplier and host built in avoids -a module dependency on a provider whose lifetime cannot end at unload. - -Suspend-to-idle suspend retains the arena after checking the completed request, -idle FIFO and command state. Resume restores the existing configuration -banks without issuing a second bootstrap. Hibernation, deeper suspend and -kexec are refused while admission or retained DMA ownership is active. The -runtime kexec guard permits the kernel to support kexec on other machines. -Host removal, sysfs reset, hotplug, VFIO assignment and native AER/DPC -recovery are withheld on this retained hierarchy. Sleep is refused until -radio admission completes; the existing suspend-to-idle path remains available -after successful admission. Failed or uncertain bootstrap state -requires a full external hardware reset; there is no command retry. - -The PIODMA arena remains allocated until external reset. Controller removal, -teardown, memory reuse and arbitrary downstream devices are unqualified. -Fresh hardware qualification must check both radio functions across sleep. -Do not remove the controller or its IOMMU while this experiment is active. -These lifecycle contracts require further work before production support. +T8140 radio PCIe bootstrap +========================= + +The J700 radio link needs one validated PIODMA config read of function 0's +vendor/device word before native ECAM works. This is a bounded PCIe bootstrap, +not a generic DMA channel provider. Linux owns the SID17 DMA mapping. After +completion, the host validates byte, word and dword ECAM reads for MT7932 +Wi-Fi and Bluetooth, bridge forwarding, assigned BARs and cold bus-master +state before publishing either function to drivers. Link power states remain +disabled for this qualified pair. Other downstream devices are not admitted. + +Selection and compatibility +--------------------------- + +Build ``CONFIG_PCIE_APPLE_T8140_BOOTSTRAP=y`` with the host, DART and ASPM. +The host's ``apple,piodma`` phandle selects the engine; the engine's +``apple,pcie-host`` must point back to that same T8140 host. No machine name +or module parameter selects this path. Without the host property, downstream +config access stays closed and the host remains root-port-only. Malformed +links fail rather than silently selecting the root-only path. + +The production compatible is ``apple,t8140-pcie-bootstrap``. Existing Stage 2 +DTBs using only ``apple,t8140-piodma-diagnostic`` remain supported. New DTBs +list the production compatible followed by the legacy compatible, so older +kernels still recognize them. J700 receives its live DT from Stage 2; updating +a kernel's packaged DTB alone does not update the boot-time DT. + +Ownership and failure +--------------------- + +The state machine distinguishes CPU-owned allocation, lifecycle admission, +consumed attempt, irreversible engine ownership, completed read and admitted +radios. PM and kexec vetoes are acquired before PCI children exist. Contention +on the system transition or kexec lock defers admission without consuming the +attempt. The bounded retry and delayed host attachment retry only this +software admission. They never retry an engine command. + +A failed probe frees an unpublished coherent allocation and drops its power +reference. Host unwind releases temporary vetoes only after all children have +been removed and only before engine publication. A preflight failure consumes +the attempt, but does not pin lifecycle vetoes after host unwind. The first +engine write marks ownership irreversible, even if initialization or transfer +validation subsequently fails. The supplier, arena, IRQ and power reference +then remain retained until a full external hardware reset. Both successful +and failed retained states reject another command or a release transition. + +Completion IRQ/tag checks, a cleared command bit, FIFO capacity and stable +canaries establish admission observations. None establishes that the engine +has stopped retaining descriptor/completion addresses or can no longer issue +DMA. There is no qualified stop, drain, pointer invalidation or engine reset +sequence in the public implementation. A PCIe link reset affects the link; +it does not prove that SID17 engine ownership ended. Therefore the arena +cannot be freed after admission, and a failed bootstrap cannot safely be +retried merely by resetting the link. + +For the same reason, replaying the bootstrap in a kexec kernel or after image +restore is insufficient. A new kernel may reuse the old DMA storage and DART +mappings before it could establish any new engine state. Hibernation can +restore memory over retained ownership. Hibernation, restore, deeper suspend, +kexec, host removal, user resets, hotplug, VFIO assignment and native AER/DPC +recovery remain refused. Supporting these operations requires a proven engine +teardown/reset and endpoint recovery contract first. Renaming the provider +does not qualify those operations. + +Suspend-to-idle is allowed only after radio admission, with no IRQ fault, +idle command/FIFO checks and intact arena contents. Resume restores the +configuration banks after genpd power restoration without submitting another +command. A failed or incomplete admission still refuses suspend-to-idle. +Do not remove the supplier or its IOMMU while retained ownership exists. + +History and guard audit +----------------------- + +This implementation is derived from the GPL-2.0-only public Linux source and +its history. The original file refers to an unnamed accompanying provenance +ledger that is not distributed in this tree. That reference is not evidence +of teardown or reset semantics. No private implementation is used to extend +the hardware contract here. + +The following public commits explain the guards retained by this provider +and its host. Commit IDs refer to the history containing base +``4db3a5a22c43`` and are usable with ``git show``. + +* ``55039de66e40``: one fixed request, exact MMIO/ECAM ranges, 16 KiB pages, + SID17 translated low-36-bit DART aperture, prefixed engine pointers, + descriptor/data bounds, pristine engine state, IRQ budget/status/tag checks, + expected response and canaries. These restrict the unqualified engine to + the observed address and request contract; all remain enforced. +* ``55039de66e40`` and ``4429c97c6c7a``: typed radio identities, root forwarding, + translated endpoint DMA and cold bus-master checks protect publication. + They remain prerequisites, including admission in each radio driver's probe. +* ``f41094962726`` and ``bb2f7f1f075c``: unscan failed unpublished children and + require completed admission, so a later rescan cannot publish failed radios. +* ``dbe56bcf475c``, ``17f1649e4eee``: pre-child PM/kexec admission, nonblocking + lock order, shutdown/consumed-attempt checks and ordered state publication + prevent transition/probe deadlocks and exposure before guards exist. +* ``69a531f01634``, ``17f1649e4eee`` and ``0886eb8081df``: bounded contention + retry and delayed attachment on system_dfl_wq preserve progress without a + hardware retry. Permanent host references and non-joining cancellation avoid + waiting for a worker behind the host device lock. +* ``67eec79324c6``: device prepare veto covers snapshot ioctls whose notifier + ran before supplier probe. It complements the hibernation/restore notifier. +* ``bda690fe1fb5`` and DjDeveloperr's ``e91592e904f5``: retained suspend-to-idle + checks and configuration-only resume remain. The suspend decision stays in + device prepare, because the notifier runs before the target state is set. +* ``de84c23e4bed``, ``8edd500f3f6d``, ``90e8abce3609`` and ``c4badcb98927``: + retained hierarchy removal, reset, assignment, hotplug and AER/DPC refusal + remain. Permanent refusals return EOPNOTSUPP; admission contention defers. +* ``3a0f71240ae3`` and ``d9efe5417547``: the builtin kexec interlock and its + unload exceptions remain; an attempted transition cannot reclaim the arena. +* ``f45a057bda82``, ``dab09d9a6c9b``, ``b5144ea3ee99``, ``ecb830163924`` and + ``e23e829f3319``: the shared PCIe-C IRQ gate, interrupt ordering and callback + lifetime fixes remain untouched. A bootstrap cleanup must not undo them. + +Retired selection and presentation guards: + +* ``4a78a9f29455`` and ``1238f8c09656`` kept an immutable enumerate parameter. + ``3ce7b58f735d`` removed it: DT omission already selects root-only operation. +* ``b5773247555f`` restricted the provider to J700/T8140 machine names. The + reciprocal DT links, exact controller resources, DART/SID, root and radio + identity checks now carry that restriction without a machine-name switch. +* Diagnostic file, Kconfig and driver names are replaced by production names. + The old compatible alone remains an ABI. Raw packet and IRQ traces are debug + messages; the single success info line follows completed host admission. + +Validation +---------- + +``CONFIG_PCIE_APPLE_T8140_BOOTSTRAP_KUNIT_TEST`` tests ownership transitions, +pre-publication unwind, retained failure, sleep admission, arena bounds and +IRQ validation without hardware. The test does not qualify engine teardown. +Owner qualification must check both radio functions, Wi-Fi association and +Bluetooth power before and after three RTC-woken suspend-to-idle cycles. +Preserve error/warning logs against the baseline and keep the previous fleet +kernel as the rollback entry. Retained failure requires full hardware reset. J700's ``wifi0`` alias and PCI endpoint node allow the public bootloader to -pass the unit's own Wi-Fi address through ``local-mac-address``. The zero -placeholder in the static device tree is not a usable station address. +pass the unit's own Wi-Fi address through ``local-mac-address``. The static +zero placeholder is not a usable station address. diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 7925972fe222b..52f07a7da62ae 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -14,10 +14,11 @@ Co-authored by DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. Platform and configuration -------------------------- -Use the separately reviewed T8140 PCIe bootstrap described in -``Documentation/PCI/controller/apple-t8140.rst``. It runs during boot when -``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``. Public m1n1 must populate the J700 -``wifi0`` endpoint's own ``local-mac-address``; a zero placeholder is rejected. +Use the T8140 PCIe bootstrap described in +``Documentation/PCI/controller/apple-t8140.rst``. Build the provider with +``CONFIG_PCIE_APPLE_T8140_BOOTSTRAP=y`` and select it through reciprocal +host/engine DT links. Public m1n1 must populate the J700 ``wifi0`` endpoint's +own ``local-mac-address``; a zero placeholder is rejected. The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, ``CONFIG_CFG80211=m``, ``CONFIG_BT_MTK7932_PCIE=m``, ``CONFIG_BT_BREDR=y``, diff --git a/MAINTAINERS b/MAINTAINERS index 72d0dab866e31..230cf21505560 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2119,12 +2119,12 @@ F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ +F: drivers/pci/controller/pcie-apple-t8140-bootstrap* F: drivers/phy/apple/atc-j700-dp.h F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs F: drivers/soc/apple/mcc.c F: drivers/tty/serial/apple_dockchannel* -F: drivers/pci/controller/pcie-apple-piodma* F: include/linux/soc/apple/pci-apple-piodma.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs @@ -2132,6 +2132,7 @@ F: sound/soc/apple/aop_t8140.rs F: tools/asahi/mt7932* F: tools/testing/apple/ F: tools/testing/kunit/configs/apple_firmware.config +F: tools/testing/kunit/configs/apple_t8140_bootstrap.config F: tools/testing/kunit/configs/mt7932.config F: tools/testing/selftests/drivers/gpu/apple-dcp/ diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 3cac74815d8a8..043c6b5d085fb 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -1840,7 +1840,7 @@ &pcie0 { /* The PIODMA guard stays closed until the retained bootstrap succeeds. */ - apple,piodma = <&pcie0_piodma_diag>; + apple,piodma = <&pcie0_bootstrap>; iommu-map = <0x100 &pcie0_dart 1 2>; iommu-map-mask = <0xffff>; pinctrl-0 = <&pcie0_clkreq>; @@ -1868,8 +1868,9 @@ }; / { - pcie0_piodma_diag: piodma@390030000 { - compatible = "apple,t8140-piodma-diagnostic"; + pcie0_bootstrap: piodma@390030000 { + compatible = "apple,t8140-pcie-bootstrap", + "apple,t8140-piodma-diagnostic"; reg = <0x3 0x90030000 0x0 0x4000>, <0x3 0x90034000 0x0 0x4000>, <0x3 0x90038000 0x0 0x4000>; diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index c9a0ff32f2b47..3c8c4d225985e 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -27,7 +27,7 @@ CONFIG_NVME_APPLE=m CONFIG_MMC_SDHCI_PCI=m CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y -CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_PCIE_APPLE_T8140_BOOTSTRAP=y CONFIG_PCIEASPM=y CONFIG_NET_VENDOR_INTEL=y CONFIG_IGB=m diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index 49ff2296003c1..f16a55a528b36 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -2,7 +2,7 @@ config MT7932_FULLMAC tristate "MediaTek MT7932 Wi-Fi on the Apple MacBook Neo (J700)" depends on PCI && OF && ARM64 && IOMMU_DMA - depends on PCIE_APPLE_PIODMA_DIAG && CFG80211 + depends on PCIE_APPLE_T8140_BOOTSTRAP && CFG80211 select FW_LOADER select CRYPTO select CRYPTO_HMAC diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index d9dc6b73aaae2..7a72e76b08262 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -35,6 +35,32 @@ config PCIE_ALTERA_MSI Say Y here if you want PCIe MSI support for the Altera FPGA. This MSI driver supports Altera MSI to GIC controller IP. +config PCIE_APPLE_T8140_BOOTSTRAP + bool "Apple T8140 radio PCIe bootstrap" + depends on PCIE_APPLE=y && ARCH_APPLE && IOMMU_DMA + depends on PCIEASPM + help + Enable the radio PCIe bootstrap on Apple T8140 systems. A reciprocal + device tree link between the host and engine selects this provider. + One fixed SID17 config read primes native ECAM; typed reads and host + resource validation admit only the expected Wi-Fi and Bluetooth pair. + + Build the host and supplier in. Published DMA memory remains owned + until full hardware reset because engine teardown is not qualified. + Suspend-to-idle retains the arena and restores configuration without + issuing another request. Hibernation, deeper sleep and kexec remain + blocked after activation, as do removal and arbitrary downstream use. + +config PCIE_APPLE_T8140_BOOTSTRAP_KUNIT_TEST + tristate "Apple T8140 PCIe bootstrap tests" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Test bootstrap ownership transitions, sleep admission, DMA address + bounds and IRQ validation without accessing hardware. The tests cover + successful admission, pre-publication unwind and irreversible failure + ownership. They do not qualify engine reset or hardware teardown. + config PCIE_APPLE_MSI_DOORBELL_ADDR hex default 0xfffff000 @@ -57,21 +83,6 @@ config PCIE_APPLE If unsure, say Y if you have an Apple Silicon system. -config PCIE_APPLE_PIODMA_DIAG - bool "J700 PIODMA diagnostic (requires full reset after use)" - depends on PCIE_APPLE=y && ARCH_APPLE && IOMMU_DMA - depends on PCIEASPM - help - Build the experimental SID17 retained bootstrap provider. It submits - one fixed request, validates typed ECAM and then enumerates the two - Neo radio functions when the board device tree links the PCIe host - to this engine. A host without that link stays root-port-only. - The host and supplier must both be built in: supplier removal cannot - release a published DMA arena safely. Suspend-to-idle sleep retains that - arena and restores the configuration without repeating a request. - Hibernation and deeper suspend are rejected after activation. - This does not qualify memory reuse or arbitrary downstream devices. - config PCI_VERSATILE bool "ARM Versatile PB PCI controller" depends on ARCH_VERSATILE || COMPILE_TEST diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 3bd9cdb5b1197..978c38a61f44c 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -67,4 +67,5 @@ obj-$(CONFIG_ARM64) += pci-xgene.o endif endif -obj-$(CONFIG_PCIE_APPLE_PIODMA_DIAG) += pcie-apple-piodma-diag.o +obj-$(CONFIG_PCIE_APPLE_T8140_BOOTSTRAP) += pcie-apple-t8140-bootstrap.o +obj-$(CONFIG_PCIE_APPLE_T8140_BOOTSTRAP_KUNIT_TEST) += pcie-apple-t8140-bootstrap-test.o diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h b/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h new file mode 100644 index 0000000000000..21627bfa2ad4a --- /dev/null +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef PCIE_APPLE_T8140_BOOTSTRAP_STATE_H +#define PCIE_APPLE_T8140_BOOTSTRAP_STATE_H + +#include +#include +#include +#include +#include +#include + +#define PIODMA_PACKET_MASK GENMASK_ULL(35, 0) +#define PIODMA_POINTER_MASK GENMASK_ULL(41, 0) +#define PIODMA_POINTER_PREFIX BIT_ULL(40) +#define PIODMA_IRQ_COMPLETE (BIT(0) | BIT(6)) +#define PIODMA_IRQ_LIMIT 16 + +/* Only RELEASE before publication can end guard ownership. No reset event. */ +enum apple_piodma_state { + PIODMA_IDLE, + PIODMA_GUARDED, + PIODMA_ATTEMPTED, + PIODMA_FAILED, + PIODMA_REJECTED, + PIODMA_RETAINED, + PIODMA_READY, + PIODMA_ADMITTED, + PIODMA_FAILED_RETAINED, + PIODMA_STATE_COUNT, +}; + +enum apple_piodma_event { + PIODMA_GUARD, + PIODMA_RELEASE, + PIODMA_PRIME, + PIODMA_PUBLISH, + PIODMA_COMPLETE, + PIODMA_ADMIT, + PIODMA_FAIL, + PIODMA_EVENT_COUNT, +}; + +static inline bool apple_piodma_retained(enum apple_piodma_state state) +{ + return state >= PIODMA_RETAINED && state < PIODMA_STATE_COUNT; +} + +static inline bool apple_piodma_guarded(enum apple_piodma_state state) +{ + return state == PIODMA_GUARDED || state == PIODMA_ATTEMPTED || + state == PIODMA_FAILED || apple_piodma_retained(state); +} + +static inline bool apple_piodma_attempted(enum apple_piodma_state state) +{ + return state == PIODMA_ATTEMPTED || state == PIODMA_FAILED || + state == PIODMA_REJECTED || apple_piodma_retained(state); +} + +static inline bool apple_piodma_ready(enum apple_piodma_state state) +{ + return state == PIODMA_READY || state == PIODMA_ADMITTED; +} + +static inline bool apple_piodma_sleep_allowed(enum apple_piodma_state state, + bool suspend_to_idle) +{ + return !apple_piodma_guarded(state) || + (state == PIODMA_ADMITTED && suspend_to_idle); +} + +static inline int apple_piodma_next(enum apple_piodma_state state, + enum apple_piodma_event event) +{ + switch (event) { + case PIODMA_GUARD: + if (state == PIODMA_IDLE) + return PIODMA_GUARDED; + break; + case PIODMA_RELEASE: + if (state == PIODMA_GUARDED) + return PIODMA_IDLE; + if (state == PIODMA_FAILED) + return PIODMA_REJECTED; + break; + case PIODMA_PRIME: + if (state == PIODMA_GUARDED) + return PIODMA_ATTEMPTED; + break; + case PIODMA_PUBLISH: + if (state == PIODMA_ATTEMPTED) + return PIODMA_RETAINED; + break; + case PIODMA_COMPLETE: + if (state == PIODMA_RETAINED) + return PIODMA_READY; + break; + case PIODMA_ADMIT: + if (state == PIODMA_READY) + return PIODMA_ADMITTED; + break; + case PIODMA_FAIL: + if (state == PIODMA_ATTEMPTED) + return PIODMA_FAILED; + if (state == PIODMA_RETAINED) + return PIODMA_FAILED_RETAINED; + break; + default: + break; + } + return -EINVAL; +} + +static inline bool apple_piodma_arena_valid(u64 dma, u64 size, u64 prefix, + bool force_aperture, u64 start, u64 end) +{ + u64 last, pointer; + + return size && IS_ALIGNED(dma, SZ_16K) && + prefix == PIODMA_POINTER_PREFIX && + !check_add_overflow(dma, size - 1, &last) && + last <= PIODMA_PACKET_MASK && + !check_add_overflow(last, prefix, &pointer) && + pointer <= PIODMA_POINTER_MASK && force_aperture && + !start && end == PIODMA_PACKET_MASK; +} + +static inline bool apple_piodma_irq_invalid(unsigned int count, u32 status, + u32 tag, u32 active) +{ + return count >= PIODMA_IRQ_LIMIT || status & ~PIODMA_IRQ_COMPLETE || + (status && (!tag || tag > active)); +} + +static inline bool apple_piodma_irq_matches(u32 status, u32 tag, u32 active) +{ + return status & PIODMA_IRQ_COMPLETE && tag == active && active; +} + +#endif diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c b/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c new file mode 100644 index 0000000000000..941a1786c0483 --- /dev/null +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "pcie-apple-t8140-bootstrap-state.h" + +static void apple_piodma_lifecycle_test(struct kunit *test) +{ + static const enum apple_piodma_event events[] = { + PIODMA_GUARD, PIODMA_PRIME, PIODMA_PUBLISH, + PIODMA_COMPLETE, PIODMA_ADMIT, + }; + int state = PIODMA_IDLE; + int i; + + for (i = 0; i < ARRAY_SIZE(events); i++) { + state = apple_piodma_next(state, events[i]); + KUNIT_ASSERT_GE(test, state, 0); + KUNIT_EXPECT_TRUE(test, apple_piodma_guarded(state)); + KUNIT_EXPECT_FALSE(test, apple_piodma_sleep_allowed(state, false)); + KUNIT_EXPECT_EQ(test, apple_piodma_sleep_allowed(state, true), + i == ARRAY_SIZE(events) - 1); + } + KUNIT_EXPECT_EQ(test, state, PIODMA_ADMITTED); + KUNIT_EXPECT_TRUE(test, apple_piodma_ready(state)); + KUNIT_EXPECT_TRUE(test, apple_piodma_retained(state)); +} + +static void apple_piodma_failure_test(struct kunit *test) +{ + int state; + + /* Guard contention/unwind before PRIME does not consume the attempt. */ + state = apple_piodma_next(PIODMA_GUARDED, PIODMA_RELEASE); + KUNIT_ASSERT_EQ(test, state, PIODMA_IDLE); + KUNIT_EXPECT_FALSE(test, apple_piodma_attempted(state)); + KUNIT_EXPECT_EQ(test, apple_piodma_next(state, PIODMA_GUARD), PIODMA_GUARDED); + + /* Preflight failure consumes the attempt, but may release both vetoes. */ + state = apple_piodma_next(PIODMA_ATTEMPTED, PIODMA_FAIL); + KUNIT_ASSERT_EQ(test, state, PIODMA_FAILED); + KUNIT_EXPECT_TRUE(test, apple_piodma_guarded(state)); + KUNIT_EXPECT_FALSE(test, apple_piodma_retained(state)); + state = apple_piodma_next(state, PIODMA_RELEASE); + KUNIT_ASSERT_EQ(test, state, PIODMA_REJECTED); + KUNIT_EXPECT_FALSE(test, apple_piodma_guarded(state)); + KUNIT_EXPECT_TRUE(test, apple_piodma_attempted(state)); + KUNIT_EXPECT_TRUE(test, apple_piodma_sleep_allowed(state, false)); + KUNIT_EXPECT_LT(test, apple_piodma_next(state, PIODMA_GUARD), 0); + + /* Initialization/transfer failure must keep memory and vetoes owned. */ + state = apple_piodma_next(PIODMA_RETAINED, PIODMA_FAIL); + KUNIT_ASSERT_EQ(test, state, PIODMA_FAILED_RETAINED); + KUNIT_EXPECT_TRUE(test, apple_piodma_retained(state)); + KUNIT_EXPECT_TRUE(test, apple_piodma_guarded(state)); + KUNIT_EXPECT_FALSE(test, apple_piodma_ready(state)); + KUNIT_EXPECT_FALSE(test, apple_piodma_sleep_allowed(state, true)); +} + +static void apple_piodma_transition_guards_test(struct kunit *test) +{ + int state, event, next; + + /* Exhaust all transitions: no retained state can release or retry. */ + for (state = PIODMA_IDLE; state < PIODMA_STATE_COUNT; state++) { + for (event = PIODMA_GUARD; event < PIODMA_EVENT_COUNT; event++) { + next = apple_piodma_next(state, event); + if (apple_piodma_retained(state)) { + if (next >= 0) + KUNIT_EXPECT_TRUE(test, apple_piodma_retained(next)); + if (event == PIODMA_RELEASE || event == PIODMA_PRIME || + event == PIODMA_GUARD) + KUNIT_EXPECT_LT(test, next, 0); + } + if (event == PIODMA_ADMIT && state != PIODMA_READY) + KUNIT_EXPECT_LT(test, next, 0); + if (event == PIODMA_COMPLETE && state != PIODMA_RETAINED) + KUNIT_EXPECT_LT(test, next, 0); + } + } +} + +static void apple_piodma_arena_test(struct kunit *test) +{ + const u64 end = PIODMA_PACKET_MASK, size = 3 * SZ_16K; + const u64 top = BIT_ULL(36) - size, prefix = PIODMA_POINTER_PREFIX; + const struct { + u64 dma, size, prefix, start, end; + bool forced, valid; + } cases[] = { + { 0, size, prefix, 0, end, true, true }, + { top, size, prefix, 0, end, true, true }, + { top + SZ_16K, size, prefix, 0, end, true, false }, + { 1, size, prefix, 0, end, true, false }, + { U64_MAX - SZ_16K + 1, size, prefix, 0, end, true, false }, + { 0, 0, prefix, 0, end, true, false }, + { 0, size, 0, 0, end, true, false }, + { 0, size, prefix, 0, end, false, false }, + { 0, size, prefix, SZ_16K, end, true, false }, + { 0, size, prefix, 0, end - 1, true, false }, + }; + bool valid; + int i; + + for (i = 0; i < ARRAY_SIZE(cases); i++) { + valid = apple_piodma_arena_valid(cases[i].dma, cases[i].size, + cases[i].prefix, cases[i].forced, + cases[i].start, cases[i].end); + KUNIT_EXPECT_EQ_MSG(test, valid, cases[i].valid, "case %d", i); + } +} + +static void apple_piodma_irq_test(struct kunit *test) +{ + KUNIT_EXPECT_FALSE(test, apple_piodma_irq_invalid(1, BIT(0), 1, 1)); + KUNIT_EXPECT_FALSE(test, apple_piodma_irq_invalid(15, BIT(6), 1, 1)); + KUNIT_EXPECT_TRUE(test, apple_piodma_irq_matches(BIT(0), 1, 1)); + KUNIT_EXPECT_TRUE(test, apple_piodma_irq_matches(BIT(6), 1, 1)); + KUNIT_EXPECT_TRUE(test, apple_piodma_irq_invalid(16, BIT(0), 1, 1)); + KUNIT_EXPECT_TRUE(test, apple_piodma_irq_invalid(1, BIT(1), 1, 1)); + KUNIT_EXPECT_TRUE(test, apple_piodma_irq_invalid(1, BIT(0), 0, 1)); + KUNIT_EXPECT_TRUE(test, apple_piodma_irq_invalid(1, BIT(0), 2, 1)); + KUNIT_EXPECT_TRUE(test, apple_piodma_irq_invalid(1, BIT(0), 1, 0)); + KUNIT_EXPECT_FALSE(test, apple_piodma_irq_matches(0, 1, 1)); + KUNIT_EXPECT_FALSE(test, apple_piodma_irq_matches(BIT(0), 0, 0)); + /* A late old tag neither faults nor qualifies the active request. */ + KUNIT_EXPECT_FALSE(test, apple_piodma_irq_invalid(1, BIT(0), 1, 2)); + KUNIT_EXPECT_FALSE(test, apple_piodma_irq_matches(BIT(0), 1, 2)); +} + +static struct kunit_case apple_piodma_cases[] = { + KUNIT_CASE(apple_piodma_lifecycle_test), + KUNIT_CASE(apple_piodma_failure_test), + KUNIT_CASE(apple_piodma_transition_guards_test), + KUNIT_CASE(apple_piodma_arena_test), + KUNIT_CASE(apple_piodma_irq_test), + { } +}; + +static struct kunit_suite apple_piodma_suite = { + .name = "apple-t8140-bootstrap", + .test_cases = apple_piodma_cases, +}; + +kunit_test_suite(apple_piodma_suite); + +MODULE_DESCRIPTION("Apple T8140 PCIe bootstrap ownership and guard tests"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-t8140-bootstrap.c similarity index 53% rename from drivers/pci/controller/pcie-apple-piodma-diag.c rename to drivers/pci/controller/pcie-apple-t8140-bootstrap.c index 2f2cde5490a27..a95dd5112f201 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap.c @@ -1,16 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * J700 PIODMA diagnostic scaffolding, not a production DMA driver. + * T8140 radio PCIe bootstrap: one validated config read before native ECAM. * - * Public Linux DMA/IOMMU APIs own the SID17 mappings. The register and - * address-view contract is attributed in the accompanying provenance ledger - * to the reviewed Aurora reference and fresh ADT observations. - * - * This built-in driver deliberately retains its coherent arena and device - * power reference until a coordinator-controlled full hardware reset. There - * is no unbind interface or hardware retry. Runtime guards cover host admission - * and retained hardware ownership. Native ECAM is only - * opened by the host after a successful bootstrap and typed read checks. + * The Linux DMA/IOMMU APIs own the SID17 mappings. See + * Documentation/PCI/controller/apple-t8140.rst for the public history and + * the retained-ownership contract. Completion does not establish DMA teardown. + * The arena, IRQ and power reference remain owned until full hardware reset. */ #include #include @@ -37,11 +32,9 @@ #include #include -#include "pcie-apple-piodma-diag.h" +#include "pcie-apple-t8140-bootstrap.h" +#include "pcie-apple-t8140-bootstrap-state.h" -#define PIODMA_PACKET_MASK DMA_BIT_MASK(36) -#define PIODMA_POINTER_MASK DMA_BIT_MASK(42) -#define PIODMA_POINTER_PREFIX BIT_ULL(40) #define PIODMA_SID 17 #define PIODMA_REQUEST_COUNT 1 #define PIODMA_SLOT_SIZE (3 * PAGE_SIZE) @@ -50,10 +43,8 @@ #define PIODMA_COMPLETION_OFFSET PAGE_SIZE #define PIODMA_CONFIG_BASE 0x1cb0000000ULL #define PIODMA_CONFIG_SIZE SZ_256M -#define PIODMA_IRQ_COMPLETE (BIT(0) | BIT(6)) #define PIODMA_IRQ_MASK GENMASK(9, 0) #define PIODMA_FIFO_MASK GENMASK(5, 0) -#define PIODMA_IRQ_LIMIT 16 /* * The board DT opts in by pointing the host's "apple,piodma" at this engine; @@ -81,28 +72,22 @@ struct apple_piodma_slot { bool observed; }; -struct apple_piodma_diag { +struct apple_t8140_bootstrap { struct device *dev; struct iommu_domain *domain; void __iomem *engine; void __iomem *fabric; struct apple_piodma_slot slots[PIODMA_REQUEST_COUNT]; atomic_t irq_count; - atomic_t attempted; + atomic_t state; u32 active_tag; u64 config_source; - u32 last_irq_status; - u32 last_irq_tag; int irq; bool irq_fault; void *arena; dma_addr_t dma; u64 pointer_prefix; u8 secondary_bus; - bool retained; - bool ready; - bool guarded; - bool radio_admitted; struct notifier_block pm_notifier; struct kexec_blocker kexec_blocker; struct delayed_work admission_retry; @@ -110,7 +95,24 @@ struct apple_piodma_diag { }; /* A successful supplier probe owns its unpublished arena until full reset. */ -static struct apple_piodma_diag *active_diag; +static struct apple_t8140_bootstrap *active_bootstrap; + +/* Host device_lock serializes transitions; IRQ faults are recorded separately. */ +static enum apple_piodma_state apple_piodma_state(struct apple_t8140_bootstrap *bootstrap) +{ + return atomic_read_acquire(&bootstrap->state); +} + +static int apple_piodma_transition(struct apple_t8140_bootstrap *bootstrap, + enum apple_piodma_event event) +{ + enum apple_piodma_state state = apple_piodma_state(bootstrap); + int next = apple_piodma_next(state, event); + + if (next < 0) + return next; + return atomic_cmpxchg(&bootstrap->state, state, next) == state ? 0 : -EALREADY; +} static bool apple_piodma_trylock_sleep(unsigned int *flags) { @@ -133,7 +135,7 @@ static bool apple_piodma_trylock_sleep(unsigned int *flags) return false; } -static int apple_piodma_lock_transition(struct apple_piodma_diag *diag, +static int apple_piodma_lock_transition(struct apple_t8140_bootstrap *bootstrap, unsigned int *sleep_flags) { unsigned int attempt; @@ -142,7 +144,7 @@ static int apple_piodma_lock_transition(struct apple_piodma_diag *diag, /* Probe cannot wait on either lock while holding its device lock. */ for (attempt = 0; attempt < 50; attempt++) { if (apple_piodma_trylock_sleep(sleep_flags)) { - ret = kexec_block(&diag->kexec_blocker); + ret = kexec_block(&bootstrap->kexec_blocker); if (!ret) return 0; unlock_system_sleep(*sleep_flags); @@ -159,8 +161,8 @@ static int apple_piodma_lock_transition(struct apple_piodma_diag *diag, static int apple_piodma_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) { - struct apple_piodma_diag *diag = - container_of(nb, struct apple_piodma_diag, pm_notifier); + struct apple_t8140_bootstrap *bootstrap = + container_of(nb, struct apple_t8140_bootstrap, pm_notifier); switch (action) { case PM_SUSPEND_PREPARE: @@ -174,7 +176,7 @@ static int apple_piodma_pm_notify(struct notifier_block *nb, return NOTIFY_DONE; case PM_HIBERNATION_PREPARE: case PM_RESTORE_PREPARE: - dev_warn_ratelimited(diag->dev, + dev_warn_ratelimited(bootstrap->dev, "sleep refused: Neo guard active during radio admission or retained DMA lifetime\n"); return notifier_from_errno(-EOPNOTSUPP); default: @@ -184,13 +186,12 @@ static int apple_piodma_pm_notify(struct notifier_block *nb, static int apple_piodma_pm_prepare(struct device *dev) { - struct apple_piodma_diag *diag = dev_get_drvdata(dev); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(dev); /* Acquire admission before either PCI children or DMA pointers escape. */ - if (!diag || !smp_load_acquire(&diag->guarded)) + if (!bootstrap || !apple_piodma_guarded(apple_piodma_state(bootstrap))) return 0; - if (pm_suspend_no_platform() && READ_ONCE(diag->ready) && - smp_load_acquire(&diag->radio_admitted)) + if (apple_piodma_sleep_allowed(apple_piodma_state(bootstrap), pm_suspend_no_platform())) return 0; /* * A /dev/snapshot file may have sent its PREPARE notifier before this @@ -202,7 +203,7 @@ static int apple_piodma_pm_prepare(struct device *dev) return -EOPNOTSUPP; } -static int apple_piodma_diag_root(struct apple_piodma_diag *diag, struct pci_dev *root) +static int apple_t8140_bootstrap_root(struct apple_t8140_bootstrap *bootstrap, struct pci_dev *root) { int ret; @@ -211,23 +212,23 @@ static int apple_piodma_diag_root(struct apple_piodma_diag *diag, struct pci_dev root->vendor != PCI_VENDOR_ID_APPLE || root->device != 0x100c || root->class != PCI_CLASS_BRIDGE_PCI << 8) return -ENODEV; - ret = pci_read_config_byte(root, PCI_SECONDARY_BUS, &diag->secondary_bus); + ret = pci_read_config_byte(root, PCI_SECONDARY_BUS, &bootstrap->secondary_bus); if (ret) return pcibios_err_to_errno(ret); - if (diag->secondary_bus != 1) + if (bootstrap->secondary_bus != 1) return -EINVAL; - diag->config_source += (u64)diag->secondary_bus << 20; + bootstrap->config_source = PIODMA_CONFIG_BASE + ((u64)bootstrap->secondary_bus << 20); return 0; } -static int apple_piodma_diag_iommu(struct apple_piodma_diag *diag) +static int apple_t8140_bootstrap_iommu(struct apple_t8140_bootstrap *bootstrap) { struct of_phandle_args args; u64 range[2]; int ret; - ret = of_parse_phandle_with_args(diag->dev->of_node, "iommus", - "#iommu-cells", 0, &args); + ret = of_parse_phandle_with_args(bootstrap->dev->of_node, "iommus", + "#iommu-cells", 0, &args); if (ret) return ret; if (args.args_count != 1 || args.args[0] != PIODMA_SID || @@ -240,56 +241,53 @@ static int apple_piodma_diag_iommu(struct apple_piodma_diag *diag) ret = -EINVAL; goto out; } - diag->pointer_prefix = PIODMA_POINTER_PREFIX; + bootstrap->pointer_prefix = PIODMA_POINTER_PREFIX; ret = of_property_read_u64_array(args.np, "apple,dma-range", range, 2); if (ret) goto out; - if (diag->pointer_prefix != PIODMA_POINTER_PREFIX || range[0] || + if (bootstrap->pointer_prefix != PIODMA_POINTER_PREFIX || range[0] || range[1] != BIT_ULL(36)) { ret = -EINVAL; goto out; } - diag->domain = iommu_get_domain_for_dev(diag->dev); - if (!diag->domain || - (diag->domain->type != IOMMU_DOMAIN_DMA && - diag->domain->type != IOMMU_DOMAIN_DMA_FQ)) { + bootstrap->domain = iommu_get_domain_for_dev(bootstrap->dev); + if (!bootstrap->domain || + (bootstrap->domain->type != IOMMU_DOMAIN_DMA && + bootstrap->domain->type != IOMMU_DOMAIN_DMA_FQ)) { ret = -ENODEV; goto out; } - ret = dma_set_mask_and_coherent(diag->dev, PIODMA_PACKET_MASK); + ret = dma_set_mask_and_coherent(bootstrap->dev, PIODMA_PACKET_MASK); out: of_node_put(args.np); return ret; } -static int apple_piodma_diag_arena(struct apple_piodma_diag *diag) +static int apple_t8140_bootstrap_arena(struct apple_t8140_bootstrap *bootstrap) { - u64 last, pointer; int i; - diag->arena = dma_alloc_coherent(diag->dev, PIODMA_ARENA_SIZE, &diag->dma, GFP_KERNEL); - if (!diag->arena) + bootstrap->arena = dma_alloc_coherent(bootstrap->dev, PIODMA_ARENA_SIZE, + &bootstrap->dma, GFP_KERNEL); + if (!bootstrap->arena) return -ENOMEM; - if (!IS_ALIGNED(diag->dma, SZ_16K) || - check_add_overflow((u64)diag->dma, PIODMA_ARENA_SIZE - 1, &last) || - last > PIODMA_PACKET_MASK || - check_add_overflow(last, diag->pointer_prefix, &pointer) || - pointer > PIODMA_POINTER_MASK || - !diag->domain->geometry.force_aperture || - diag->domain->geometry.aperture_start != 0 || - diag->domain->geometry.aperture_end != PIODMA_PACKET_MASK) + if (!apple_piodma_arena_valid(bootstrap->dma, PIODMA_ARENA_SIZE, + bootstrap->pointer_prefix, + bootstrap->domain->geometry.force_aperture, + bootstrap->domain->geometry.aperture_start, + bootstrap->domain->geometry.aperture_end)) return -ERANGE; - memset(diag->arena, 0, PIODMA_ARENA_SIZE); + memset(bootstrap->arena, 0, PIODMA_ARENA_SIZE); for (i = 0; i < PIODMA_REQUEST_COUNT; i++) { - void *slot = diag->arena + i * PIODMA_SLOT_SIZE; - u64 dma = diag->dma + i * PIODMA_SLOT_SIZE; + void *slot = bootstrap->arena + i * PIODMA_SLOT_SIZE; + u64 dma = bootstrap->dma + i * PIODMA_SLOT_SIZE; memset(slot + PIODMA_DATA_OFFSET, 0xa5, PAGE_SIZE); *(__le32 *)(slot + PIODMA_DATA_OFFSET) = cpu_to_le32(U32_MAX); - dev_dbg(diag->dev, + dev_dbg(bootstrap->dev, "owned SID17 slot=%d dma=%#llx descriptor=%#llx completion=%#llx data=%#llx\n", - i, dma, dma + diag->pointer_prefix, - dma + diag->pointer_prefix + PIODMA_COMPLETION_OFFSET, + i, dma, dma + bootstrap->pointer_prefix, + dma + bootstrap->pointer_prefix + PIODMA_COMPLETION_OFFSET, dma + PIODMA_DATA_OFFSET); } dma_wmb(); @@ -297,9 +295,8 @@ static int apple_piodma_diag_arena(struct apple_piodma_diag *diag) } - -static int apple_piodma_diag_resources(struct platform_device *pdev, - struct apple_piodma_diag *diag) +static int apple_t8140_bootstrap_resources(struct platform_device *pdev, + struct apple_t8140_bootstrap *bootstrap) { static const struct { const char *name; @@ -318,7 +315,7 @@ static int apple_piodma_diag_resources(struct platform_device *pdev, if (!res || res->start != regions[i].base || resource_size(res) != SZ_16K) return -EINVAL; } - host = of_parse_phandle(diag->dev->of_node, "apple,pcie-host", 0); + host = of_parse_phandle(bootstrap->dev->of_node, "apple,pcie-host", 0); if (!host) return -EINVAL; ret = of_address_to_resource(host, 0, &ecam); @@ -329,20 +326,20 @@ static int apple_piodma_diag_resources(struct platform_device *pdev, of_node_put(host); if (ret) return ret; - diag->config_source = ecam.start; - if (diag->config_source + (1 << 12) + sizeof(u32) - 1 > ecam.end) + bootstrap->config_source = ecam.start; + if (bootstrap->config_source + (1 << 12) + sizeof(u32) - 1 > ecam.end) return -ERANGE; - diag->engine = devm_platform_ioremap_resource_byname(pdev, "engine"); - if (IS_ERR(diag->engine)) - return PTR_ERR(diag->engine); - diag->fabric = devm_platform_ioremap_resource_byname(pdev, "fabric"); - if (IS_ERR(diag->fabric)) - return PTR_ERR(diag->fabric); - diag->irq = platform_get_irq(pdev, 0); - return diag->irq < 0 ? diag->irq : 0; + bootstrap->engine = devm_platform_ioremap_resource_byname(pdev, "engine"); + if (IS_ERR(bootstrap->engine)) + return PTR_ERR(bootstrap->engine); + bootstrap->fabric = devm_platform_ioremap_resource_byname(pdev, "fabric"); + if (IS_ERR(bootstrap->fabric)) + return PTR_ERR(bootstrap->fabric); + bootstrap->irq = platform_get_irq(pdev, 0); + return bootstrap->irq < 0 ? bootstrap->irq : 0; } -static int apple_piodma_diag_idle(struct apple_piodma_diag *diag) +static int apple_t8140_bootstrap_idle(struct apple_t8140_bootstrap *bootstrap) { static const struct { u32 offset; @@ -356,45 +353,42 @@ static int apple_piodma_diag_idle(struct apple_piodma_diag *diag) u32 value; for (i = 0; i < ARRAY_SIZE(engine); i++) { - value = readl(diag->engine + engine[i].offset); + value = readl(bootstrap->engine + engine[i].offset); if (value != engine[i].expected) { - dev_err(diag->dev, "idle mismatch engine[%#x]=%#x expected=%#x\n", + dev_err(bootstrap->dev, "idle mismatch engine[%#x]=%#x expected=%#x\n", engine[i].offset, value, engine[i].expected); return -EBUSY; } } - if (readl(diag->fabric) || readl(diag->fabric + 4)) + if (readl(bootstrap->fabric) || readl(bootstrap->fabric + 4)) return -EBUSY; return 0; } -static irqreturn_t apple_piodma_diag_irq(int irq, void *data) +static irqreturn_t apple_t8140_bootstrap_irq(int irq, void *data) { - struct apple_piodma_diag *diag = data; + struct apple_t8140_bootstrap *bootstrap = data; struct apple_piodma_slot *slot; u32 fabric, status, tag, active; int count; /* Acknowledge order follows the attributed prior controller contract. */ - fabric = readl(diag->fabric); - writel(fabric, diag->fabric); - status = readl(diag->engine + 4); - writel(status, diag->engine + 4); - tag = readl(diag->engine + 0xdc) & 0xff; - active = READ_ONCE(diag->active_tag); - count = atomic_inc_return(&diag->irq_count); - WRITE_ONCE(diag->last_irq_status, status); - WRITE_ONCE(diag->last_irq_tag, tag); - dev_dbg(diag->dev, "irq=%d fabric=%#x status=%#x tag=%#x active=%#x\n", + fabric = readl(bootstrap->fabric); + writel(fabric, bootstrap->fabric); + status = readl(bootstrap->engine + 4); + writel(status, bootstrap->engine + 4); + tag = readl(bootstrap->engine + 0xdc) & 0xff; + active = READ_ONCE(bootstrap->active_tag); + count = atomic_inc_return(&bootstrap->irq_count); + dev_dbg(bootstrap->dev, "irq=%d fabric=%#x status=%#x tag=%#x active=%#x\n", count, fabric, status, tag, active); - if (count >= PIODMA_IRQ_LIMIT || status & ~PIODMA_IRQ_COMPLETE || - (status && (!tag || tag > active))) { - WRITE_ONCE(diag->irq_fault, true); + if (apple_piodma_irq_invalid(count, status, tag, active)) { + WRITE_ONCE(bootstrap->irq_fault, true); disable_irq_nosync(irq); if (active) - complete(&diag->slots[active - 1].event); - } else if (status & PIODMA_IRQ_COMPLETE && tag == active && active) { - slot = &diag->slots[active - 1]; + complete(&bootstrap->slots[active - 1].event); + } else if (apple_piodma_irq_matches(status, tag, active)) { + slot = &bootstrap->slots[active - 1]; WRITE_ONCE(slot->matched_status, READ_ONCE(slot->matched_status) | status); complete(&slot->event); } @@ -402,7 +396,7 @@ static irqreturn_t apple_piodma_diag_irq(int irq, void *data) return IRQ_HANDLED; } -static int apple_piodma_diag_initialize(struct apple_piodma_diag *diag) +static int apple_t8140_bootstrap_initialize(struct apple_t8140_bootstrap *bootstrap) { static const struct { u32 offset; @@ -419,30 +413,28 @@ static int apple_piodma_diag_initialize(struct apple_piodma_diag *diag) int i; u32 value; - /* Publish validated root/bus fields; all later failures retain ownership. */ - smp_store_release(&diag->retained, true); - writel(PIODMA_IRQ_MASK, diag->engine + 8); - writel(0, diag->fabric + 4); + writel(PIODMA_IRQ_MASK, bootstrap->engine + 8); + writel(0, bootstrap->fabric + 4); for (i = 0; i < ARRAY_SIZE(tunables); i++) { - value = readl(diag->engine + tunables[i].offset); + value = readl(bootstrap->engine + tunables[i].offset); value = (value & ~tunables[i].mask) | tunables[i].value; - writel(value, diag->engine + tunables[i].offset); + writel(value, bootstrap->engine + tunables[i].offset); } - writel(readl(diag->engine) & ~BIT(0), diag->engine); + writel(readl(bootstrap->engine) & ~BIT(0), bootstrap->engine); /* Exact prior bank values; their architectural meaning is unqualified. */ for (i = 0; i < 8; i++) { - writel(0, diag->engine + 0x24 + 4 * i); - writel(i == 1 ? 1 : 0, diag->engine + 0xf0 + 4 * i); + writel(0, bootstrap->engine + 0x24 + 4 * i); + writel(i == 1 ? 1 : 0, bootstrap->engine + 0xf0 + 4 * i); } /* Order the bank writes before their configuration readback. */ wmb(); - if (readl(diag->engine) || readl(diag->engine + 8) != PIODMA_IRQ_MASK || - readl(diag->engine + 0xf4) != 1 || READ_ONCE(diag->irq_fault)) + if (readl(bootstrap->engine) || readl(bootstrap->engine + 8) != PIODMA_IRQ_MASK || + readl(bootstrap->engine + 0xf4) != 1 || READ_ONCE(bootstrap->irq_fault)) return -EIO; return 0; } -static bool apple_piodma_diag_canary(const u8 *memory, size_t size, u8 value) +static bool apple_t8140_bootstrap_canary(const u8 *memory, size_t size, u8 value) { size_t i; @@ -452,14 +444,14 @@ static bool apple_piodma_diag_canary(const u8 *memory, size_t size, u8 value) return true; } -static int apple_piodma_diag_check_slots(struct apple_piodma_diag *diag) +static int apple_t8140_bootstrap_check_slots(struct apple_t8140_bootstrap *bootstrap) { int i, j; dma_rmb(); for (i = 0; i < PIODMA_REQUEST_COUNT; i++) { - struct apple_piodma_slot *slot = &diag->slots[i]; - void *memory = diag->arena + i * PIODMA_SLOT_SIZE; + struct apple_piodma_slot *slot = &bootstrap->slots[i]; + void *memory = bootstrap->arena + i * PIODMA_SLOT_SIZE; __le32 *packet = memory; __le32 *completion = memory + PIODMA_COMPLETION_OFFSET; __le32 *result = memory + PIODMA_DATA_OFFSET; @@ -472,42 +464,42 @@ static int apple_piodma_diag_check_slots(struct apple_piodma_diag *diag) } if ((slot->observed && le32_to_cpu(READ_ONCE(*result)) != slot->result) || (!slot->submitted && le32_to_cpu(READ_ONCE(*result)) != U32_MAX) || - !apple_piodma_diag_canary(memory + sizeof(slot->packet), + !apple_t8140_bootstrap_canary(memory + sizeof(slot->packet), PAGE_SIZE - sizeof(slot->packet), 0) || - !apple_piodma_diag_canary(memory + PIODMA_COMPLETION_OFFSET + + !apple_t8140_bootstrap_canary(memory + PIODMA_COMPLETION_OFFSET + sizeof(slot->completion), PAGE_SIZE - sizeof(slot->completion), 0) || - !apple_piodma_diag_canary(memory + PIODMA_DATA_OFFSET + sizeof(u32), + !apple_t8140_bootstrap_canary(memory + PIODMA_DATA_OFFSET + sizeof(u32), PAGE_SIZE - sizeof(u32), 0xa5)) goto changed; } return 0; changed: - dev_err(diag->dev, "slot=%d changed outside its permitted response; stop\n", i); + dev_err(bootstrap->dev, "slot=%d changed outside its permitted response; stop\n", i); return -EIO; } -static int apple_piodma_diag_submit(struct apple_piodma_diag *diag, unsigned int index) +static int apple_t8140_bootstrap_submit(struct apple_t8140_bootstrap *bootstrap, unsigned int index) { const struct apple_piodma_request *request = &requests[index]; - struct apple_piodma_slot *slot = &diag->slots[index]; - void *memory = diag->arena + index * PIODMA_SLOT_SIZE; + struct apple_piodma_slot *slot = &bootstrap->slots[index]; + void *memory = bootstrap->arena + index * PIODMA_SLOT_SIZE; __le32 *packet = memory; __le32 *result = memory + PIODMA_DATA_OFFSET; __le32 *completion = memory + PIODMA_COMPLETION_OFFSET; - u64 dma = diag->dma + index * PIODMA_SLOT_SIZE; - u64 source = diag->config_source + ((u64)request->function << 12) + request->offset; + u64 dma = bootstrap->dma + index * PIODMA_SLOT_SIZE; + u64 source = bootstrap->config_source + ((u64)request->function << 12) + request->offset; u64 dst = dma + PIODMA_DATA_OFFSET; u64 src = source & PIODMA_PACKET_MASK; - u64 descriptor = dma + diag->pointer_prefix; + u64 descriptor = dma + bootstrap->pointer_prefix; u64 comp = descriptor + PIODMA_COMPLETION_OFFSET; unsigned long waited; u32 fifo, command, tag = index + 1; int i, ret; - ret = apple_piodma_diag_check_slots(diag); - if (ret || READ_ONCE(diag->irq_fault)) + ret = apple_t8140_bootstrap_check_slots(bootstrap); + if (ret || READ_ONCE(bootstrap->irq_fault)) return ret ?: -EIO; /* Only these fixed, validated ECAM addresses use aperture1/low36. */ slot->packet[0] = cpu_to_le32(0xc007 | BIT(18) | ((src & 0xff) << 24)); @@ -516,68 +508,61 @@ static int apple_piodma_diag_submit(struct apple_piodma_diag *diag, unsigned int slot->packet[3] = cpu_to_le32(sizeof(u32)); memcpy(packet, slot->packet, sizeof(slot->packet)); dma_wmb(); - ret = readl_poll_timeout(diag->engine + 0x0c, fifo, - (fifo & PIODMA_FIFO_MASK) == 16, 10, 50000); - if (ret || READ_ONCE(diag->irq_fault)) + ret = readl_poll_timeout(bootstrap->engine + 0x0c, fifo, + (fifo & PIODMA_FIFO_MASK) == 16, 10, 50000); + if (ret || READ_ONCE(bootstrap->irq_fault)) return ret ?: -EIO; slot->submitted = true; - WRITE_ONCE(diag->active_tag, tag); - dev_dbg(diag->dev, + WRITE_ONCE(bootstrap->active_tag, tag); + dev_dbg(bootstrap->dev, "request=%u function=%u offset=%#x source=%#llx packet=%08x:%08x:%08x:%08x tag=%u\n", index, request->function, request->offset, source, le32_to_cpu(packet[0]), le32_to_cpu(packet[1]), le32_to_cpu(packet[2]), le32_to_cpu(packet[3]), tag); - writel(tag, diag->engine + 0xd8); - writel(upper_32_bits(descriptor) & 0x3ff, diag->engine + 0x50); - writel(lower_32_bits(descriptor), diag->engine + 0x4c); - writel(upper_32_bits(comp) & 0x3ff, diag->engine + 0x48); - writel(lower_32_bits(comp), diag->engine + 0x44); - writel(0x411, diag->engine + 0x54); + writel(tag, bootstrap->engine + 0xd8); + writel(upper_32_bits(descriptor) & 0x3ff, bootstrap->engine + 0x50); + writel(lower_32_bits(descriptor), bootstrap->engine + 0x4c); + writel(upper_32_bits(comp) & 0x3ff, bootstrap->engine + 0x48); + writel(lower_32_bits(comp), bootstrap->engine + 0x44); + writel(0x411, bootstrap->engine + 0x54); waited = wait_for_completion_timeout(&slot->event, msecs_to_jiffies(50)); dma_rmb(); slot->result = le32_to_cpu(READ_ONCE(*result)); for (i = 0; i < ARRAY_SIZE(slot->completion); i++) slot->completion[i] = READ_ONCE(completion[i]); slot->observed = true; - dev_dbg(diag->dev, + dev_dbg(bootstrap->dev, "response=%u waited=%lu matched_status=%#x word=%#x completion=%08x:%08x:%08x:%08x\n", index, waited, READ_ONCE(slot->matched_status), slot->result, le32_to_cpu(slot->completion[0]), le32_to_cpu(slot->completion[1]), le32_to_cpu(slot->completion[2]), le32_to_cpu(slot->completion[3])); if (!waited) return -ETIMEDOUT; - if (READ_ONCE(diag->irq_fault) || !READ_ONCE(slot->matched_status) || + if (READ_ONCE(bootstrap->irq_fault) || !READ_ONCE(slot->matched_status) || slot->result == U32_MAX || (slot->result & request->mask) != request->expected) return -EIO; /* Admission observations only: all published memory remains retained. */ - ret = readl_poll_timeout(diag->engine + 0x54, command, !(command & BIT(0)), - 10, 50000); + ret = readl_poll_timeout(bootstrap->engine + 0x54, command, !(command & BIT(0)), + 10, 50000); if (ret) return ret; - ret = readl_poll_timeout(diag->engine + 0x0c, fifo, - (fifo & PIODMA_FIFO_MASK) == 16, 10, 50000); + ret = readl_poll_timeout(bootstrap->engine + 0x0c, fifo, + (fifo & PIODMA_FIFO_MASK) == 16, 10, 50000); if (ret) return ret; for (i = 0; i < 10; i++) { usleep_range(1000, 2000); - ret = apple_piodma_diag_check_slots(diag); - if (ret || READ_ONCE(diag->irq_fault)) + ret = apple_t8140_bootstrap_check_slots(bootstrap); + if (ret || READ_ONCE(bootstrap->irq_fault)) return ret ?: -EIO; } - dev_dbg(diag->dev, + dev_dbg(bootstrap->dev, "READ_VALIDATED request=%u word=%#x FIFO=%#x command=%#x; retained slots stable\n", index, slot->result, fifo, command); return 0; } -bool apple_piodma_bootstrap_enabled(void) -{ - return of_machine_is_compatible("apple,j700") && - of_machine_is_compatible("apple,t8140"); -} -EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_enabled); - static void apple_piodma_put_supplier(void *data) { put_device(data); @@ -586,19 +571,19 @@ static void apple_piodma_put_supplier(void *data) int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) { struct device_node *node, *host_node; - struct apple_piodma_diag *diag; + struct apple_t8140_bootstrap *bootstrap; struct platform_device *pdev; struct device_link *link; bool correct_host; int ret; - if (!apple_piodma_bootstrap_enabled() || - !of_device_is_compatible(host->of_node, "apple,t8140-pcie")) + if (!of_device_is_compatible(host->of_node, "apple,t8140-pcie")) return -ENODEV; node = of_parse_phandle(host->of_node, "apple,piodma", 0); if (!node) return -EINVAL; - if (!of_device_is_compatible(node, "apple,t8140-piodma-diagnostic")) { + if (!of_device_is_compatible(node, "apple,t8140-pcie-bootstrap") && + !of_device_is_compatible(node, "apple,t8140-piodma-diagnostic")) { of_node_put(node); return -EINVAL; } @@ -611,8 +596,8 @@ int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) return -EINVAL; if (!pdev) return -EPROBE_DEFER; - diag = platform_get_drvdata(pdev); - if (!device_is_bound(&pdev->dev) || !diag) { + bootstrap = platform_get_drvdata(pdev); + if (!device_is_bound(&pdev->dev) || !bootstrap) { put_device(&pdev->dev); return -EPROBE_DEFER; } @@ -622,7 +607,7 @@ int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) return -ESHUTDOWN; } /* Host probe holds its device lock; never rebuild a consumed hierarchy. */ - if (atomic_read(&diag->attempted)) { + if (apple_piodma_attempted(apple_piodma_state(bootstrap))) { put_device(&pdev->dev); return -EALREADY; } @@ -641,25 +626,25 @@ EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_get); static void apple_piodma_retry_admission(struct work_struct *work) { - struct apple_piodma_diag *diag = - container_of(to_delayed_work(work), struct apple_piodma_diag, + struct apple_t8140_bootstrap *bootstrap = + container_of(to_delayed_work(work), struct apple_t8140_bootstrap, admission_retry); - /* Acquire the matched host's permanent reference from the scheduler. */ - struct device *host = smp_load_acquire(&diag->retry_host); + /* Pairs with the scheduler's cmpxchg of the boot-lifetime host reference. */ + struct device *host = smp_load_acquire(&bootstrap->retry_host); int ret; if (system_state > SYSTEM_RUNNING || !host || - !device_is_registered(host) || atomic_read(&diag->attempted) || - READ_ONCE(diag->guarded)) + !device_is_registered(host) || apple_piodma_attempted(apple_piodma_state(bootstrap)) || + apple_piodma_guarded(apple_piodma_state(bootstrap))) return; /* The core rechecks registration and binding under the host device lock. */ ret = device_attach(host); if (ret < 0) - dev_dbg(diag->dev, "deferred host attachment returned %d\n", ret); + dev_dbg(bootstrap->dev, "deferred host attachment returned %d\n", ret); /* Only fresh pre-child guard contention can schedule another attempt. */ } -static void apple_piodma_schedule_admission(struct apple_piodma_diag *diag, +static void apple_piodma_schedule_admission(struct apple_t8140_bootstrap *bootstrap, struct device *host) { struct device *previous; @@ -667,18 +652,18 @@ static void apple_piodma_schedule_admission(struct apple_piodma_diag *diag, if (system_state > SYSTEM_RUNNING || !device_is_registered(host)) return; /* Publish a boot-lifetime reference; this supplier has one matched host. */ - previous = cmpxchg(&diag->retry_host, NULL, get_device(host)); + previous = cmpxchg(&bootstrap->retry_host, NULL, get_device(host)); if (previous) { put_device(host); if (WARN_ON_ONCE(previous != host)) return; } - mod_delayed_work(system_dfl_wq, &diag->admission_retry, HZ); + mod_delayed_work(system_dfl_wq, &bootstrap->admission_retry, HZ); } int apple_piodma_bootstrap_guard_begin(struct device *supplier, struct device *host) { - struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); unsigned int sleep_flags; int ret; @@ -686,24 +671,25 @@ int apple_piodma_bootstrap_guard_begin(struct device *supplier, struct device *h if (system_state > SYSTEM_RUNNING) return -ESHUTDOWN; /* A worker may have waited behind a host probe that consumed its attempt. */ - if (atomic_read(&diag->attempted) || READ_ONCE(diag->guarded)) + if (apple_piodma_attempted(apple_piodma_state(bootstrap)) || + apple_piodma_guarded(apple_piodma_state(bootstrap))) return -EALREADY; /* This is the last dependency admission before PCI children exist. */ - ret = apple_piodma_lock_transition(diag, &sleep_flags); + ret = apple_piodma_lock_transition(bootstrap, &sleep_flags); if (ret) { if (ret == -EPROBE_DEFER) - apple_piodma_schedule_admission(diag, host); + apple_piodma_schedule_admission(bootstrap, host); return ret; } /* Do not wait for a worker that may be waiting on this host device lock. */ - cancel_delayed_work(&diag->admission_retry); - diag->pm_notifier.notifier_call = apple_piodma_pm_notify; - ret = register_pm_notifier(&diag->pm_notifier); + cancel_delayed_work(&bootstrap->admission_retry); + bootstrap->pm_notifier.notifier_call = apple_piodma_pm_notify; + ret = register_pm_notifier(&bootstrap->pm_notifier); if (ret) { - kexec_unblock(&diag->kexec_blocker); + kexec_unblock(&bootstrap->kexec_blocker); } else { /* Publish both vetoes before permitting system sleep to proceed. */ - smp_store_release(&diag->guarded, true); + WARN_ON_ONCE(apple_piodma_transition(bootstrap, PIODMA_GUARD)); } unlock_system_sleep(sleep_flags); return ret; @@ -712,42 +698,47 @@ EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_guard_begin); void apple_piodma_bootstrap_guard_end(struct device *supplier) { - struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); /* Post-publication ownership is irreversible until hardware reset. */ - if (!READ_ONCE(diag->guarded) || smp_load_acquire(&diag->retained)) + if (!apple_piodma_guarded(apple_piodma_state(bootstrap)) || + apple_piodma_retained(apple_piodma_state(bootstrap))) return; /* The host has removed every child before permitting sleep again. */ - smp_store_release(&diag->guarded, false); - unregister_pm_notifier(&diag->pm_notifier); - kexec_unblock(&diag->kexec_blocker); + if (WARN_ON_ONCE(apple_piodma_transition(bootstrap, PIODMA_RELEASE))) + return; + unregister_pm_notifier(&bootstrap->pm_notifier); + kexec_unblock(&bootstrap->kexec_blocker); } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_guard_end); int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) { - struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); int ret; /* Admission was completed before pci_host_common_init published children. */ - if (!smp_load_acquire(&diag->guarded)) + if (!apple_piodma_guarded(apple_piodma_state(bootstrap))) return -EINVAL; /* One hardware attempt, including permanent preflight failures. */ - if (atomic_cmpxchg(&diag->attempted, 0, 1)) + if (apple_piodma_transition(bootstrap, PIODMA_PRIME)) return -EALREADY; - ret = apple_piodma_diag_root(diag, root); - if (!ret) - ret = apple_piodma_diag_idle(diag); + ret = apple_t8140_bootstrap_root(bootstrap, root); if (!ret) - ret = apple_piodma_diag_initialize(diag); + ret = apple_t8140_bootstrap_idle(bootstrap); + if (!ret) { + /* Ownership becomes irreversible before the first engine write. */ + ret = apple_piodma_transition(bootstrap, PIODMA_PUBLISH); + if (!ret) + ret = apple_t8140_bootstrap_initialize(bootstrap); + } if (!ret) - ret = apple_piodma_diag_submit(diag, 0); + ret = apple_t8140_bootstrap_submit(bootstrap, 0); if (ret) { - dev_err(supplier, "bootstrap failure=%d; retained ownership; no retry\n", ret); + WARN_ON_ONCE(apple_piodma_transition(bootstrap, PIODMA_FAIL)); + dev_err(supplier, "bootstrap failed: %d; full hardware reset required\n", ret); } else { - WRITE_ONCE(diag->ready, true); - dev_info(supplier, "BOOTSTRAP_VALIDATED word=%#x; one retained slot\n", - diag->slots[0].result); + WARN_ON_ONCE(apple_piodma_transition(bootstrap, PIODMA_COMPLETE)); } return ret; } @@ -755,187 +746,188 @@ EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_prime); bool apple_piodma_bootstrap_retained(struct device *supplier) { - struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); /* Acquire the prime's irreversible publication and guard ownership. */ - return smp_load_acquire(&diag->retained); + return apple_piodma_retained(apple_piodma_state(bootstrap)); } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_retained); void apple_piodma_bootstrap_admitted(struct device *supplier) { - struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); - if (WARN_ON_ONCE(!diag || !diag->retained)) + if (WARN_ON_ONCE(!bootstrap || !apple_piodma_retained(apple_piodma_state(bootstrap)))) return; /* Publish the host's completed cold, resource and ASPM admission checks. */ - smp_store_release(&diag->radio_admitted, true); + WARN_ON_ONCE(apple_piodma_transition(bootstrap, PIODMA_ADMIT)); } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_admitted); int apple_piodma_radio_check(struct pci_dev *pdev) { - /* Acquire the supplier's permanent arena and PM/kexec guard ownership. */ - struct apple_piodma_diag *diag = smp_load_acquire(&active_diag); + /* Pairs with probe's smp_store_release of the boot-lifetime supplier. */ + struct apple_t8140_bootstrap *bootstrap = smp_load_acquire(&active_bootstrap); struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->bus); struct device_node *host; bool matches; - if (!apple_piodma_bootstrap_enabled() || !diag) + if (!bootstrap) return -ENODEV; /* A retained supplier alone does not prove that host admission succeeded. */ - if (!smp_load_acquire(&diag->radio_admitted) || !bridge->dev.parent || + if (apple_piodma_state(bootstrap) != PIODMA_ADMITTED || !bridge->dev.parent || pdev->vendor != PCI_VENDOR_ID_MEDIATEK || - pci_domain_nr(pdev->bus) || pdev->bus->number != diag->secondary_bus || + pci_domain_nr(pdev->bus) || pdev->bus->number != bootstrap->secondary_bus || PCI_SLOT(pdev->devfn) || (pdev->devfn != PCI_DEVFN(0, 0) && pdev->devfn != PCI_DEVFN(0, 1)) || pdev->device != (PCI_FUNC(pdev->devfn) ? 0x793b : 0x7932)) return -ENODEV; - host = of_parse_phandle(diag->dev->of_node, "apple,pcie-host", 0); + host = of_parse_phandle(bootstrap->dev->of_node, "apple,pcie-host", 0); matches = host && host == bridge->dev.parent->of_node; of_node_put(host); return matches ? 0 : -ENODEV; } EXPORT_SYMBOL_GPL(apple_piodma_radio_check); -static int apple_piodma_diag_probe(struct platform_device *pdev) +static int apple_t8140_bootstrap_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct apple_piodma_diag *diag; + struct apple_t8140_bootstrap *bootstrap; int i, ret; - if (!apple_piodma_bootstrap_enabled() || PAGE_SIZE != SZ_16K) + if (PAGE_SIZE != SZ_16K) return -ENODEV; - diag = kzalloc_obj(*diag); - if (!diag) + bootstrap = kzalloc_obj(*bootstrap); + if (!bootstrap) return -ENOMEM; - diag->dev = dev; - INIT_DELAYED_WORK(&diag->admission_retry, apple_piodma_retry_admission); - diag->kexec_blocker.reason = + bootstrap->dev = dev; + INIT_DELAYED_WORK(&bootstrap->admission_retry, apple_piodma_retry_admission); + bootstrap->kexec_blocker.reason = "Neo radio admission or retained DMA ownership"; for (i = 0; i < PIODMA_REQUEST_COUNT; i++) - init_completion(&diag->slots[i].event); - atomic_set(&diag->irq_count, 0); - atomic_set(&diag->attempted, 0); - ret = apple_piodma_diag_iommu(diag); + init_completion(&bootstrap->slots[i].event); + atomic_set(&bootstrap->irq_count, 0); + atomic_set(&bootstrap->state, PIODMA_IDLE); + ret = apple_t8140_bootstrap_iommu(bootstrap); if (ret) goto err_free; pm_runtime_enable(dev); ret = pm_runtime_resume_and_get(dev); if (ret < 0) goto err_pm; - ret = apple_piodma_diag_arena(diag); + ret = apple_t8140_bootstrap_arena(bootstrap); if (ret) goto err_arena; - ret = apple_piodma_diag_resources(pdev, diag); + ret = apple_t8140_bootstrap_resources(pdev, bootstrap); if (ret) goto err_arena; - ret = devm_request_irq(dev, diag->irq, apple_piodma_diag_irq, 0, - dev_name(dev), diag); + ret = devm_request_irq(dev, bootstrap->irq, apple_t8140_bootstrap_irq, 0, + dev_name(dev), bootstrap); if (ret) goto err_arena; /* The supplier and unpublished arena keep their device alive. */ get_device(dev); - platform_set_drvdata(pdev, diag); - /* Publish the CPU-owned arena and permanent device reference together. */ - smp_store_release(&active_diag, diag); + platform_set_drvdata(pdev, bootstrap); + /* Pairs with radio_check's smp_load_acquire of this boot-lifetime owner. */ + smp_store_release(&active_bootstrap, bootstrap); dev_dbg(dev, "bootstrap supplier ready; no command submitted during probe\n"); return 0; err_arena: - if (diag->arena) - dma_free_coherent(dev, PIODMA_ARENA_SIZE, diag->arena, diag->dma); + if (bootstrap->arena) + dma_free_coherent(dev, PIODMA_ARENA_SIZE, bootstrap->arena, bootstrap->dma); pm_runtime_put_sync(dev); err_pm: pm_runtime_disable(dev); err_free: - kfree(diag); - return dev_err_probe(dev, ret, "PIODMA diagnostic preflight failed\n"); + kfree(bootstrap); + return dev_err_probe(dev, ret, "PCIe bootstrap preflight failed\n"); } -static void apple_piodma_diag_shutdown(struct platform_device *pdev) +static void apple_t8140_bootstrap_shutdown(struct platform_device *pdev) { - struct apple_piodma_diag *diag = platform_get_drvdata(pdev); + struct apple_t8140_bootstrap *bootstrap = platform_get_drvdata(pdev); /* * Shutdown holds supplier/parent locks: never join a host attachment * here. Admission rejects shutdown, and the supplier lifetime is retained. */ - cancel_delayed_work(&diag->admission_retry); - if (READ_ONCE(diag->retained)) - dev_warn(&pdev->dev, "diagnostic arena retained; require full hardware reset\n"); + cancel_delayed_work(&bootstrap->admission_retry); + if (apple_piodma_retained(apple_piodma_state(bootstrap))) + dev_warn(&pdev->dev, "bootstrap arena retained; require full hardware reset\n"); } /* No new request is permitted once the host has admitted its endpoints. */ -static int apple_piodma_diag_sleep_check(struct apple_piodma_diag *diag) +static int apple_t8140_bootstrap_sleep_check(struct apple_t8140_bootstrap *bootstrap) { - if (!READ_ONCE(diag->ready) || READ_ONCE(diag->irq_fault)) + if (!apple_piodma_ready(apple_piodma_state(bootstrap)) || READ_ONCE(bootstrap->irq_fault)) return -EIO; - if (readl(diag->engine + 0x54) & BIT(0)) + if (readl(bootstrap->engine + 0x54) & BIT(0)) return -EBUSY; - if ((readl(diag->engine + 0x0c) & PIODMA_FIFO_MASK) != 16) + if ((readl(bootstrap->engine + 0x0c) & PIODMA_FIFO_MASK) != 16) return -EBUSY; - return apple_piodma_diag_check_slots(diag); + return apple_t8140_bootstrap_check_slots(bootstrap); } -static int apple_piodma_diag_suspend_noirq(struct device *dev) +static int apple_t8140_bootstrap_suspend_noirq(struct device *dev) { - struct apple_piodma_diag *diag = dev_get_drvdata(dev); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(dev); - if (!diag->retained) + if (!apple_piodma_retained(apple_piodma_state(bootstrap))) return 0; if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) return -EOPNOTSUPP; - return apple_piodma_diag_sleep_check(diag); + return apple_t8140_bootstrap_sleep_check(bootstrap); } -static int apple_piodma_diag_resume_noirq(struct device *dev) +static int apple_t8140_bootstrap_resume_noirq(struct device *dev) { - struct apple_piodma_diag *diag = dev_get_drvdata(dev); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(dev); int ret; - if (!diag->retained) + if (!apple_piodma_retained(apple_piodma_state(bootstrap))) return 0; - if (!READ_ONCE(diag->ready) || READ_ONCE(diag->irq_fault)) + if (!apple_piodma_ready(apple_piodma_state(bootstrap)) || READ_ONCE(bootstrap->irq_fault)) return -EIO; /* Genpd may gate this island; restore configuration, never a command. */ - ret = apple_piodma_diag_initialize(diag); + ret = apple_t8140_bootstrap_initialize(bootstrap); if (ret) return ret; - return apple_piodma_diag_sleep_check(diag); + return apple_t8140_bootstrap_sleep_check(bootstrap); } -static int apple_piodma_diag_freeze(struct device *dev) +static int apple_t8140_bootstrap_freeze(struct device *dev) { - struct apple_piodma_diag *diag = dev_get_drvdata(dev); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(dev); - return diag->retained ? -EOPNOTSUPP : 0; + return apple_piodma_retained(apple_piodma_state(bootstrap)) ? -EOPNOTSUPP : 0; } -static const struct dev_pm_ops apple_piodma_diag_pm_ops = { +static const struct dev_pm_ops apple_t8140_bootstrap_pm_ops = { .prepare = apple_piodma_pm_prepare, - .suspend_noirq = apple_piodma_diag_suspend_noirq, - .resume_noirq = apple_piodma_diag_resume_noirq, - .freeze = apple_piodma_diag_freeze, - .poweroff = apple_piodma_diag_freeze, + .suspend_noirq = apple_t8140_bootstrap_suspend_noirq, + .resume_noirq = apple_t8140_bootstrap_resume_noirq, + .freeze = apple_t8140_bootstrap_freeze, + .poweroff = apple_t8140_bootstrap_freeze, }; -static const struct of_device_id apple_piodma_diag_match[] = { +static const struct of_device_id apple_t8140_bootstrap_match[] = { + { .compatible = "apple,t8140-pcie-bootstrap" }, { .compatible = "apple,t8140-piodma-diagnostic" }, { } }; -static struct platform_driver apple_piodma_diag_driver = { - .probe = apple_piodma_diag_probe, - .shutdown = apple_piodma_diag_shutdown, +static struct platform_driver apple_t8140_bootstrap_driver = { + .probe = apple_t8140_bootstrap_probe, + .shutdown = apple_t8140_bootstrap_shutdown, .driver = { - .name = "apple-piodma-diag", - .of_match_table = apple_piodma_diag_match, - .pm = pm_sleep_ptr(&apple_piodma_diag_pm_ops), + .name = "apple-t8140-bootstrap", + .of_match_table = apple_t8140_bootstrap_match, + .pm = pm_sleep_ptr(&apple_t8140_bootstrap_pm_ops), .suppress_bind_attrs = true, }, }; -builtin_platform_driver(apple_piodma_diag_driver); -MODULE_DESCRIPTION("T8140 retained one-read PCIe bootstrap experiment"); +builtin_platform_driver(apple_t8140_bootstrap_driver); +MODULE_DESCRIPTION("Apple T8140 radio PCIe bootstrap"); MODULE_LICENSE("GPL"); diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.h b/drivers/pci/controller/pcie-apple-t8140-bootstrap.h similarity index 84% rename from drivers/pci/controller/pcie-apple-piodma-diag.h rename to drivers/pci/controller/pcie-apple-t8140-bootstrap.h index c8d29335ed0cb..b7c4eeedb92c9 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.h +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef PCIE_APPLE_PIODMA_DIAG_H -#define PCIE_APPLE_PIODMA_DIAG_H +#ifndef PCIE_APPLE_T8140_BOOTSTRAP_H +#define PCIE_APPLE_T8140_BOOTSTRAP_H #include #include @@ -9,8 +9,7 @@ struct device; struct pci_dev; -#if IS_ENABLED(CONFIG_PCIE_APPLE_PIODMA_DIAG) -bool apple_piodma_bootstrap_enabled(void); +#if IS_ENABLED(CONFIG_PCIE_APPLE_T8140_BOOTSTRAP) int apple_piodma_bootstrap_get(struct device *host, struct device **supplier); int apple_piodma_bootstrap_guard_begin(struct device *supplier, struct device *host); @@ -19,11 +18,6 @@ int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root); void apple_piodma_bootstrap_admitted(struct device *supplier); bool apple_piodma_bootstrap_retained(struct device *supplier); #else -static inline bool apple_piodma_bootstrap_enabled(void) -{ - return false; -} - static inline int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) { return -ENODEV; diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index e0b6ead2526e2..b86574e27580d 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -44,7 +44,7 @@ #include "../pci.h" #include "pci-host-common.h" -#include "pcie-apple-piodma-diag.h" +#include "pcie-apple-t8140-bootstrap.h" static ATOMIC_NOTIFIER_HEAD(apple_pcie_tunnel_notifiers); @@ -1813,10 +1813,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (pcie->hw->tunneled) WRITE_ONCE(port->started, true); if (pcie->hw->root_bus_only) - dev_info(pcie->dev, "port %pOF RUN=%#x link=%#x; downstream config %s\n", - np, apple_pcie_port_readl(port, PORT_STATUS), - apple_pcie_port_readl(port, PORT_LINKSTS), - pcie->piodma_supplier ? "waits for the bootstrap" : "blocked"); + dev_dbg(pcie->dev, "port %pOF RUN=%#x link=%#x; downstream config %s\n", + np, apple_pcie_port_readl(port, PORT_STATUS), + apple_pcie_port_readl(port, PORT_LINKSTS), + pcie->piodma_supplier ? "waits for the bootstrap" : "blocked"); return 0; @@ -2296,7 +2296,7 @@ static int apple_pcie_init(struct pci_config_window *cfg) } /* - * The T8140 experiment starts with all downstream accesses blocked. + * The T8140 host starts with all downstream accesses blocked. * Only a successful owned bootstrap and typed ECAM checks open functions0/1. * Generic config callbacks retain their normal native widths and locking. */ @@ -2651,9 +2651,9 @@ static int apple_pcie_neo_typed_ids(struct apple_pcie *pcie, struct pci_dev *roo low = readb(cfg + PCI_VENDOR_ID); high = readb(cfg + PCI_VENDOR_ID + 1); header = readb(cfg + PCI_HEADER_TYPE); - dev_info(pcie->dev, - "typed ECAM function=%d id=%#x vendor=%#x device=%#x bytes=%02x:%02x header=%#x\n", - function, id, vendor, device, low, high, header); + dev_dbg(pcie->dev, + "typed ECAM function=%d id=%#x vendor=%#x device=%#x bytes=%02x:%02x header=%#x\n", + function, id, vendor, device, low, high, header); if (id != ids[function] || vendor != (ids[function] & 0xffff) || device != ids[function] >> 16 || low != 0xc3 || high != 0x14 || (header & PCI_HEADER_TYPE_MASK) != PCI_HEADER_TYPE_NORMAL || @@ -2676,8 +2676,8 @@ static int apple_pcie_neo_check_memory(struct pci_dev *root) ret = pci_read_config_word(root, PCI_COMMAND, &command); if (!ret) ret = pci_read_config_dword(root, PCI_MEMORY_BASE, &window); - pci_info(root, "NEO_ROOT_MEMORY_READBACK status=%d command=%#x window=%#x\n", - ret, command, window); + pci_dbg(root, "NEO_ROOT_MEMORY_READBACK status=%d command=%#x window=%#x\n", + ret, command, window); if (ret || command == U16_MAX || !(command & PCI_COMMAND_MEMORY) || window == U32_MAX) return -EIO; /* Each packed register field is12 bits; PCI_MEMORY_RANGE_MASK is unsigned long. */ @@ -2706,8 +2706,8 @@ static int apple_pcie_neo_check_memory(struct pci_dev *root) if (ret) return ret; } - pci_info(root, "NEO_ROOT_MEMORY_ENABLED command=%#x window=%#x bus=%#llx-%#llx\n", - command, window, base, limit); + pci_dbg(root, "NEO_ROOT_MEMORY_ENABLED command=%#x window=%#x bus=%#llx-%#llx\n", + command, window, base, limit); return 0; } @@ -2731,8 +2731,8 @@ static int apple_pcie_neo_check_cold(struct apple_pcie *pcie, struct pci_dev *ro if (pci_read_config_word(endpoint, PCI_COMMAND, &command)) { ret = ret ?: -EIO; } else { - dev_info(pcie->dev, "enumerated %s id=%04x:%04x command=%#x\n", - pci_name(endpoint), endpoint->vendor, endpoint->device, command); + dev_dbg(pcie->dev, "enumerated %s id=%04x:%04x command=%#x\n", + pci_name(endpoint), endpoint->vendor, endpoint->device, command); if (command & PCI_COMMAND_MASTER) { pci_clear_master(endpoint); ret = ret ?: -EIO; @@ -2803,7 +2803,7 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) } pci_unlock_rescan_remove(); if (!ret) - dev_info(pcie->dev, "NEO_RADIO_READY; functions0/1 admitted with link power states disabled\n"); + dev_info(pcie->dev, "radio PCIe bootstrap complete; Wi-Fi and Bluetooth admitted\n"); out: pci_dev_put(root); return ret; @@ -2842,7 +2842,7 @@ static int apple_pcie_probe(struct platform_device *pdev) } /* Only a host that the board links to a bootstrap engine opens bus 1. */ - if (hw->root_bus_only && apple_piodma_bootstrap_enabled() && + if (hw->root_bus_only && IS_ENABLED(CONFIG_PCIE_APPLE_T8140_BOOTSTRAP) && of_property_present(dev->of_node, "apple,piodma")) { ret = apple_piodma_bootstrap_get(dev, &piodma_supplier); if (ret) @@ -2880,7 +2880,7 @@ static int apple_pcie_probe(struct platform_device *pdev) bridge->native_shpc_hotplug = false; bridge->native_aer = false; bridge->native_dpc = false; - dev_info_once(dev, "radio host: automatic AER/DPC recovery disabled for retained DMA\n"); + dev_dbg(dev, "radio host: automatic AER/DPC recovery disabled for retained DMA\n"); } if (hw->root_bus_only && !piodma_supplier) dev_info(dev, "root-port probe only; downstream config is blocked\n"); @@ -2991,7 +2991,7 @@ static int apple_pcie_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "Neo bootstrap preflight failed\n"); } if (ret) - dev_err(dev, "enumeration experiment failed=%d; host/supplier retained, no retry\n", + dev_err(dev, "radio admission failed=%d; host/supplier retained, no retry\n", ret); return 0; } diff --git a/include/linux/soc/apple/pci-apple-piodma.h b/include/linux/soc/apple/pci-apple-piodma.h index 5cb948a60943e..3b2971b8b1db4 100644 --- a/include/linux/soc/apple/pci-apple-piodma.h +++ b/include/linux/soc/apple/pci-apple-piodma.h @@ -7,7 +7,7 @@ struct pci_dev; -#if IS_ENABLED(CONFIG_PCIE_APPLE_PIODMA_DIAG) +#if IS_ENABLED(CONFIG_PCIE_APPLE_T8140_BOOTSTRAP) /* Require completed Neo admission and permanent retained-DMA protection. */ int apple_piodma_radio_check(struct pci_dev *pdev); #else diff --git a/tools/testing/kunit/configs/apple_t8140_bootstrap.config b/tools/testing/kunit/configs/apple_t8140_bootstrap.config new file mode 100644 index 0000000000000..20903b6ec30df --- /dev/null +++ b/tools/testing/kunit/configs/apple_t8140_bootstrap.config @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_PCI=y +CONFIG_PCIE_APPLE_T8140_BOOTSTRAP_KUNIT_TEST=y diff --git a/tools/testing/kunit/configs/mt7932.config b/tools/testing/kunit/configs/mt7932.config index 323269882d301..dbd1f710ae24b 100644 --- a/tools/testing/kunit/configs/mt7932.config +++ b/tools/testing/kunit/configs/mt7932.config @@ -6,7 +6,7 @@ CONFIG_PCI_MSI=y CONFIG_PCIEPORTBUS=y CONFIG_PCIEASPM=y CONFIG_PCIE_APPLE=y -CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_PCIE_APPLE_T8140_BOOTSTRAP=y CONFIG_IOMMU_SUPPORT=y CONFIG_IOMMU_DMA=y CONFIG_NET=y From 1e68ca03b1b0bba194d8358bdf4105d0b2f71580 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:06:37 +1100 Subject: [PATCH 1425/1579] rust: drm: file: pass fdinfo format strings as kernel char pointers Clippy's disallowed-methods lint rejects CStr::as_ptr() in kernel code because the kernel's char is unsigned. Use as_char_ptr() for the two new fdinfo strings, as the engine name argument already does. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/drm/file.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index f152957397221..81f0a90e4a517 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -147,7 +147,7 @@ impl File { unsafe { bindings::drm_printf( printer, - c"drm-engine-%s:\t%llu ns\n".as_ptr(), + c"drm-engine-%s:\t%llu ns\n".as_char_ptr(), name.as_char_ptr(), ns as core::ffi::c_ulonglong, ) @@ -180,7 +180,9 @@ impl File { // SAFETY: `printer` is the printer `drm_show_fdinfo()` passes, valid for this call; `raw` // and the region name outlive the call. - unsafe { bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_ptr()) }; + unsafe { + bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_char_ptr()) + }; } /// The postclose callback of a `struct drm_file`. From 628c256988eb8f82ef4e33ca28e6990aa23cf6ba Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:16:53 +1100 Subject: [PATCH 1426/1579] dt-bindings: media: apple,ave: describe T6000 and T8112 Add the M1 Pro and M2 AVE compatibles to the shared encoder binding. Both use the same three register windows, five power domains and paired DART topology. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/media/apple,t8140-ave.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml b/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml index 61e10feb5df71..cfc518d4a0237 100644 --- a/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml +++ b/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml @@ -10,14 +10,16 @@ maintainers: - Miguel Cruz <7362641+Acelogic@users.noreply.github.com> description: | - The T6041 and T8140 AVE coprocessors encode video using firmware preloaded by the - bootloader. Its two DART instances translate the same stream. The + The T6000, T6041, T8112 and T8140 AVE coprocessors encode video using firmware + preloaded by the bootloader. Its two DART instances translate the same stream. The bootloader supplies two reserved firmware regions and their IOVA mappings. properties: compatible: enum: + - apple,t6000-ave - apple,t6041-ave + - apple,t8112-ave - apple,t8140-ave reg: From 244eddc0a09840a53fd67028297f76de11f8b179 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:16:53 +1100 Subject: [PATCH 1427/1579] media: apple: ave: add probe-only T6000 and T8112 match data Select firmware segment sizes, boot identity, device type, explicit coprocessor offsets and LFS reference sizing from SoC match data. Validate both reserved segments and their loader checksums before binding. The pre-M4 command constants, MCC DSID and firmware builds still need qualification. Refuse encoding and firmware startup with an informational message on these SoCs, even if their TEXT checksum happens to match T8140. Keep the shared wire layouts and the qualified T8140 command path. Exercise exact descriptor sizes and alignment boundaries with an independent KUnit sizing suite. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/admin-guide/media/apple-ave.rst | 14 +++ MAINTAINERS | 1 + drivers/media/platform/apple/ave/.kunitconfig | 7 ++ drivers/media/platform/apple/ave/Kconfig | 12 +++ drivers/media/platform/apple/ave/Makefile | 1 + drivers/media/platform/apple/ave/ave-drv.c | 48 ++++++++- drivers/media/platform/apple/ave/ave-fw.c | 100 ++++++++++++------ drivers/media/platform/apple/ave/ave-ipc.c | 6 +- drivers/media/platform/apple/ave/ave-regs.h | 4 +- drivers/media/platform/apple/ave/ave-sess.c | 4 +- .../media/platform/apple/ave/ave-sizes-test.c | 59 +++++++++++ drivers/media/platform/apple/ave/ave-sizes.h | 21 ++++ drivers/media/platform/apple/ave/ave.h | 26 +++++ 13 files changed, 263 insertions(+), 40 deletions(-) create mode 100644 drivers/media/platform/apple/ave/.kunitconfig create mode 100644 drivers/media/platform/apple/ave/ave-sizes-test.c create mode 100644 drivers/media/platform/apple/ave/ave-sizes.h diff --git a/Documentation/admin-guide/media/apple-ave.rst b/Documentation/admin-guide/media/apple-ave.rst index 2a76d4eac23d1..fbac9d743da9b 100644 --- a/Documentation/admin-guide/media/apple-ave.rst +++ b/Documentation/admin-guide/media/apple-ave.rst @@ -45,6 +45,20 @@ build the device binds, but starting a stream fails with ``ENOTSUPP`` and a message naming the checksum, until that build's command layout has been checked. +T6000 (M1 Pro) and T8112 (M2) have probe-only support in this driver. Their +device tree nodes and both encoder DARTs remain disabled. The firmware +handoff must reserve their preloaded TEXT (0xd0000 bytes) and DATA +(0x128000 bytes), preserve the ADT remap addresses and export both CRCs. +An enabled node with a valid handoff may bind, but encoding is refused with +an informational message until per-SoC command constants, MCC DSID and the +firmware CRCs have been qualified. The T8140 command constants are never a +fallback for these devices. Enabling a node alone does not enable encoding. + +The shared Start layout uses a smaller LFS reference surface on both +pre-M4 devices: at 1920x1080 its exact descriptor size is 0x85200 for AVC +and 0x1fe00 for HEVC, before the DMA allocation is page rounded. The +``apple-ave-sizes`` KUnit suite checks these sizes and alignment boundaries. + With debugfs mounted, ``apple-ave/last_start_h264``, ``last_start_hevc``, ``last_process0_h264`` and ``last_process0_hevc`` hold the last Start command built and the first Process command of the last session, diff --git a/MAINTAINERS b/MAINTAINERS index 93fa816104d7f..859f70c1d19bb 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2808,6 +2808,7 @@ L: linux-media@vger.kernel.org S: Maintained W: https://aurora-silicon.github.io F: Documentation/admin-guide/media/apple-ave.rst +F: Documentation/devicetree/bindings/media/apple,t8140-ave.yaml F: drivers/media/platform/apple/ave/ ARM/APPLE QSPIMC DRIVER diff --git a/drivers/media/platform/apple/ave/.kunitconfig b/drivers/media/platform/apple/ave/.kunitconfig new file mode 100644 index 0000000000000..891a4969c8986 --- /dev/null +++ b/drivers/media/platform/apple/ave/.kunitconfig @@ -0,0 +1,7 @@ +CONFIG_KUNIT=y +CONFIG_EXPERT=y +CONFIG_NO_IOMEM=n +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_SUPPORT_FILTER=n +CONFIG_MEDIA_PLATFORM_DRIVERS=y +CONFIG_VIDEO_APPLE_AVE_KUNIT_TEST=y diff --git a/drivers/media/platform/apple/ave/Kconfig b/drivers/media/platform/apple/ave/Kconfig index baf6218ecdd5b..20eb7cc404192 100644 --- a/drivers/media/platform/apple/ave/Kconfig +++ b/drivers/media/platform/apple/ave/Kconfig @@ -10,9 +10,21 @@ config VIDEO_APPLE_AVE help V4L2 memory-to-memory driver for the AVE H.264/HEVC encoder in Apple T8140 (A18 Pro) SoCs, as in the MacBook Neo. + T6000 (M1 Pro) and T8112 (M2) have probe-only support; encoding + requires qualified firmware command constants. The encoder firmware is loaded by the bootloader; the driver builds every command itself. See Documentation/admin-guide/media/apple-ave.rst. To compile this driver as a module, choose M here: the module will be called apple-ave. + +config VIDEO_APPLE_AVE_KUNIT_TEST + tristate "KUnit tests for Apple AVE buffer sizing" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Test the exact LFS reference descriptor sizes for pre-M4 and T8140 + encoders, including picture alignment boundaries and the reference + 1920x1080 AVC and HEVC sizes. The tests do not need an encoder or + any firmware and can run independently of the AVE driver. diff --git a/drivers/media/platform/apple/ave/Makefile b/drivers/media/platform/apple/ave/Makefile index 53bb6b554f770..899d8990555ea 100644 --- a/drivers/media/platform/apple/ave/Makefile +++ b/drivers/media/platform/apple/ave/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_VIDEO_APPLE_AVE) += apple-ave.o apple-ave-y := ave-drv.o ave-fw.o ave-ipc.o ave-cmd.o ave-image.o ave-sess.o ave-v4l2.o +obj-$(CONFIG_VIDEO_APPLE_AVE_KUNIT_TEST) += ave-sizes-test.o diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c index b25ac9df48a67..09eb356332abc 100644 --- a/drivers/media/platform/apple/ave/ave-drv.c +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -14,14 +14,55 @@ #include #include #include +#include #include #include #include #include "ave.h" +#include "ave-regs.h" #define AVE_AUTOSUSPEND_MS 2000 +static const struct ave_coproc_regs ave_coproc_v40 = { + .vcbar = AVE_COPROC_VCBAR, + .timebase_freq = AVE_COPROC_TIMEBASE_FREQ, + .timebase = AVE_COPROC_TIMEBASE, + .ctrl = AVE_COPROC_CTRL, + .status = AVE_COPROC_STATUS, + .boot_arg = AVE_COPROC_BOOT_ARG, + .keep_mappings = AVE_COPROC_KEEP_MAPPINGS, +}; + +/* Pre-M4 wire layouts are shared; their command constants still need traces. */ +static const struct ave_variant ave_t6000 = { + .regs = &ave_coproc_v40, + .fw_text_size = 0xd0000, + .fw_data_size = 0x128000, + .boot_device_id = 11, + .device_type = 9, + .lfs_ref_scale = 1, +}; + +static const struct ave_variant ave_t8112 = { + .regs = &ave_coproc_v40, + .fw_text_size = 0xd0000, + .fw_data_size = 0x128000, + .boot_device_id = 15, + .device_type = 13, + .lfs_ref_scale = 1, +}; + +static const struct ave_variant ave_t8140 = { + .regs = &ave_coproc_v40, + .fw_text_size = 0x14c000, + .fw_data_size = 0x130000, + .boot_device_id = 29, + .device_type = 27, + .lfs_ref_scale = 2, + .encoding_text_crc = AVE_FW_TEXT_CRC_9003_78_0, +}; + /* * debugfs: apple-ave/last_start_{h264,hevc} hold the last Start built and * last_process0_{h264,hevc} the first Process of the last session, so the @@ -84,6 +125,9 @@ static int ave_probe(struct platform_device *pdev) if (!ave) return -ENOMEM; ave->dev = dev; + ave->variant = of_device_get_match_data(dev); + if (!ave->variant) + return -ENODEV; mutex_init(&ave->hw_lock); platform_set_drvdata(pdev, ave); @@ -202,7 +246,9 @@ static const struct dev_pm_ops ave_pm_ops = { }; static const struct of_device_id ave_of_match[] = { - { .compatible = "apple,t8140-ave" }, + { .compatible = "apple,t6000-ave", .data = &ave_t6000 }, + { .compatible = "apple,t8112-ave", .data = &ave_t8112 }, + { .compatible = "apple,t8140-ave", .data = &ave_t8140 }, { } }; MODULE_DEVICE_TABLE(of, ave_of_match); diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c index 9b4d3c3f67662..fb7563252ab8c 100644 --- a/drivers/media/platform/apple/ave/ave-fw.c +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -60,6 +61,12 @@ static int ave_fw_segment(struct ave_dev *ave, unsigned int index, np = of_parse_phandle(ave->dev->of_node, "memory-region", index); if (!np) return -ENOENT; + if (!of_node_name_eq(np->parent, "reserved-memory") || + !of_device_is_compatible(np, "apple,asc-mem") || + !of_property_read_bool(np, "no-map")) { + ret = -EINVAL; + goto out; + } ret = of_address_to_resource(np, 0, &res); if (ret) goto out; @@ -74,7 +81,9 @@ static int ave_fw_segment(struct ave_dev *ave, unsigned int index, seg->phys = res.start; seg->size = resource_size(&res); seg->iova = of_read_number(prop + 1, 2); - if (of_read_number(prop + 3, 2) != seg->size || + if (!seg->size || seg->phys > PHYS_ADDR_MAX - seg->size || + seg->iova > U64_MAX - seg->size || + of_read_number(prop + 3, 2) != seg->size || !IS_ALIGNED(seg->phys, SZ_16K) || !IS_ALIGNED(seg->iova, SZ_16K)) ret = -EINVAL; out: @@ -90,18 +99,17 @@ static int ave_fw_check_mapping(struct ave_dev *ave, const struct ave_fw_segment return 0; } -static u32 ave_fw_crc(const struct ave_fw_segment *seg, void **copy) +static int ave_fw_crc(const struct ave_fw_segment *seg, u32 *crc, void *copy) { void *va = memremap(seg->phys, seg->size, MEMREMAP_WB); - u32 crc; if (!va) - return 0; - crc = crc32_le(~0U, va, seg->size) ^ ~0U; + return -ENOMEM; + *crc = crc32_le(~0U, va, seg->size) ^ ~0U; if (copy) - memcpy(*copy, va, seg->size); + memcpy(copy, va, seg->size); memunmap(va); - return crc; + return 0; } /* @@ -112,7 +120,7 @@ int ave_fw_probe(struct ave_dev *ave) { struct device *dev = ave->dev; u32 text_crc, data_crc, want; - bool known = false; + bool known = !ave->variant->encoding_text_crc; int ret; if (of_count_phandle_with_args(dev->of_node, "memory-region", NULL) != 2) @@ -124,10 +132,24 @@ int ave_fw_probe(struct ave_dev *ave) if (ret) return dev_err_probe(dev, ret, "bad firmware reservation\n"); + if (ave->text.size != ave->variant->fw_text_size || + ave->data.size != ave->variant->fw_data_size) + return dev_err_probe(dev, -EINVAL, "unexpected firmware segment sizes\n"); + if ((ave->text.phys < ave->data.phys + ave->data.size && + ave->data.phys < ave->text.phys + ave->text.size) || + (ave->text.iova < ave->data.iova + ave->data.size && + ave->data.iova < ave->text.iova + ave->text.size)) + return dev_err_probe(dev, -EINVAL, "overlapping firmware segments\n"); + if (ave_fw_check_mapping(ave, &ave->text) || ave_fw_check_mapping(ave, &ave->data)) return dev_err_probe(dev, -EIO, "firmware is not mapped where it was loaded\n"); - text_crc = ave_fw_crc(&ave->text, NULL); + ret = ave_fw_crc(&ave->text, &text_crc, NULL); + if (ret) + return ret; + ret = ave_fw_chosen_crc(ave, "aurora,ave-text-crc32", &want); + if (ret || text_crc != want) + return dev_err_probe(dev, -ENODEV, "firmware text differs from loader CRC\n"); for (unsigned int i = 0; i < ARRAY_SIZE(ave_known_text_crc); i++) known |= text_crc == ave_known_text_crc[i]; if (!known) @@ -138,7 +160,9 @@ int ave_fw_probe(struct ave_dev *ave) ave->data_pristine = vmalloc(ave->data.size); if (!ave->data_pristine) return -ENOMEM; - data_crc = ave_fw_crc(&ave->data, &ave->data_pristine); + ret = ave_fw_crc(&ave->data, &data_crc, ave->data_pristine); + if (ret) + goto free_data; /* * The loader records the DATA checksum before anything runs. If it @@ -146,14 +170,21 @@ int ave_fw_probe(struct ave_dev *ave) */ ret = ave_fw_chosen_crc(ave, "aurora,ave-data-crc32", &want); if (ret || data_crc != want) { - vfree(ave->data_pristine); - ave->data_pristine = NULL; - return dev_err_probe(dev, -ENODEV, - "firmware data is not in its boot state\n"); + ret = dev_err_probe(dev, -ENODEV, "firmware data is not in its boot state\n"); + goto free_data; } + ave->data_crc = data_crc; ave->fw_state = AVE_FW_OFF; + if (!ave->variant->encoding_text_crc) + dev_info(dev, "probe only: device type %u firmware TEXT %08x DATA %08x needs qualification\n", + ave->variant->device_type, text_crc, data_crc); return 0; + +free_data: + vfree(ave->data_pristine); + ave->data_pristine = NULL; + return ret; } static void ave_fw_restore_data(struct ave_dev *ave) @@ -212,10 +243,11 @@ int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, unsigned long */ int ave_fw_check_build(struct ave_dev *ave) { - if (ave->text_crc == AVE_FW_TEXT_CRC_9003_78_0) + if (ave->variant->encoding_text_crc && + ave->text_crc == ave->variant->encoding_text_crc) return 0; - dev_err(ave->dev, "encoder firmware %08x is not validated for encoding (only %08x is)\n", - ave->text_crc, AVE_FW_TEXT_CRC_9003_78_0); + dev_info(ave->dev, "encoding refused: device type %u firmware TEXT %08x DATA %08x has no qualified command constants/MCC DSID\n", + ave->variant->device_type, ave->text_crc, ave->data_crc); return -ENOTSUPP; } @@ -231,9 +263,10 @@ void ave_fw_remove(struct ave_dev *ave) static bool ave_fw_idle(struct ave_dev *ave) { - if (readq(ave->coproc + AVE_COPROC_VCBAR) != (ave->text.iova | AVE_COPROC_VCBAR_VALID)) + if (readq(ave->coproc + ave->variant->regs->vcbar) != + (ave->text.iova | AVE_COPROC_VCBAR_VALID)) return false; - if (readl(ave->coproc + AVE_COPROC_CTRL) || readl(ave->mbox + AVE_MBOX_STATUS)) + if (readl(ave->coproc + ave->variant->regs->ctrl) || readl(ave->mbox + AVE_MBOX_STATUS)) return false; for (unsigned int i = 0; i < AVE_MBOX_SCRATCH_COUNT; i++) if (readl(ave->mbox + AVE_MBOX_SCRATCH(i))) @@ -253,15 +286,16 @@ static int ave_fw_prepare_restart(struct ave_dev *ave) { u32 status; - if (readq(ave->coproc + AVE_COPROC_VCBAR) != (ave->text.iova | AVE_COPROC_VCBAR_VALID)) { + if (readq(ave->coproc + ave->variant->regs->vcbar) != + (ave->text.iova | AVE_COPROC_VCBAR_VALID)) { dev_err(ave->dev, "coprocessor lost its code base (vcbar %#llx)\n", - readq(ave->coproc + AVE_COPROC_VCBAR)); + readq(ave->coproc + ave->variant->regs->vcbar)); return -EIO; } if (ave->fw_state != AVE_FW_HALTED) return 0; - if (readl(ave->coproc + AVE_COPROC_CTRL)) { + if (readl(ave->coproc + ave->variant->regs->ctrl)) { dev_dbg(ave->dev, "coprocessor kept power after its halt; retry once it suspends\n"); return -EAGAIN; } @@ -303,8 +337,8 @@ static int ave_fw_boot(struct ave_dev *ave) */ if (!ave_fw_idle(ave)) { dev_err(ave->dev, "coprocessor is not in its reset state (vcbar %#llx ctrl %#x)\n", - readq(ave->coproc + AVE_COPROC_VCBAR), - readl(ave->coproc + AVE_COPROC_CTRL)); + readq(ave->coproc + ave->variant->regs->vcbar), + readl(ave->coproc + ave->variant->regs->ctrl)); return -EIO; } @@ -320,7 +354,7 @@ static int ave_fw_boot(struct ave_dev *ave) log = ave->log.cpu; memset(cfg, 0, AVE_BOOT_CFG_BYTES); memset(log, 0, ave->log.size); - cfg[AVE_BOOT_CFG_DEVICE] = cpu_to_le32(AVE_BOOT_CFG_DEVICE_T8140); + cfg[AVE_BOOT_CFG_DEVICE] = cpu_to_le32(ave->variant->boot_device_id); cfg[AVE_BOOT_CFG_UNITS] = cpu_to_le32(1); cfg[AVE_BOOT_CFG_GROUPS] = cpu_to_le32(1); cfg[AVE_BOOT_CFG_SUBID_MASK] = cpu_to_le32(1); @@ -338,16 +372,16 @@ static int ave_fw_boot(struct ave_dev *ave) writel(upper_32_bits(ave->boot_cfg.dma), ave->mbox + AVE_MBOX_SCRATCH(2)); ave->data_dirty = true; - writel(1, ave->coproc + AVE_COPROC_KEEP_MAPPINGS); - writel(0, ave->coproc + AVE_COPROC_CTRL); - writel(AVE_COPROC_BOOT_ARG_VALUE, ave->coproc + AVE_COPROC_BOOT_ARG); - writel(AVE_COPROC_CTRL_RUN, ave->coproc + AVE_COPROC_CTRL); + writel(1, ave->coproc + ave->variant->regs->keep_mappings); + writel(0, ave->coproc + ave->variant->regs->ctrl); + writel(AVE_COPROC_BOOT_ARG_VALUE, ave->coproc + ave->variant->regs->boot_arg); + writel(AVE_COPROC_CTRL_RUN, ave->coproc + ave->variant->regs->ctrl); ret = readl_poll_timeout(ave->mbox + AVE_MBOX_STATUS, status, status & AVE_MBOX_STATUS_PENDING, 50, AVE_MBOX_TIMEOUT_US); if (ret) { dev_err(ave->dev, "firmware did not answer (status %#x)\n", - readl(ave->coproc + AVE_COPROC_STATUS)); + readl(ave->coproc + ave->variant->regs->status)); return ret; } for (unsigned int i = 0; i < 4; i++) @@ -377,6 +411,9 @@ int ave_fw_ensure_running(struct ave_dev *ave) int ret; lockdep_assert_held(&ave->hw_lock); + ret = ave_fw_check_build(ave); + if (ret) + return ret; switch (ave->fw_state) { case AVE_FW_RUNNING: return 0; @@ -396,7 +433,8 @@ int ave_fw_ensure_running(struct ave_dev *ave) prev = ave->fw_state; ret = ave_fw_boot(ave); if (ret) { - ave->fw_state = readl(ave->coproc + AVE_COPROC_CTRL) ? AVE_FW_HALTED : prev; + ave->fw_state = readl(ave->coproc + ave->variant->regs->ctrl) ? + AVE_FW_HALTED : prev; memset(ave->ring, 0, sizeof(ave->ring)); ave->ipc_fw_base = 0; } diff --git a/drivers/media/platform/apple/ave/ave-ipc.c b/drivers/media/platform/apple/ave/ave-ipc.c index 974f3b2e8cc90..c197671debbbe 100644 --- a/drivers/media/platform/apple/ave/ave-ipc.c +++ b/drivers/media/platform/apple/ave/ave-ipc.c @@ -157,8 +157,8 @@ int ave_ipc_negotiate(struct ave_dev *ave, const u32 hello[4]) writel(AVE_MBOX_STATUS_PENDING, ave->mbox + AVE_MBOX_STATUS); /* Tell the firmware how far its timebase is from the host's. */ - ticks = readq(ave->coproc + AVE_COPROC_TIMEBASE); - freq = readl(ave->coproc + AVE_COPROC_TIMEBASE_FREQ); + ticks = readq(ave->coproc + ave->variant->regs->timebase); + freq = readl(ave->coproc + ave->variant->regs->timebase_freq); if (freq && freq < 1000000) return -EPROTO; ticks = div64_u64(ticks, freq ? freq / 1000000 : 24); @@ -184,7 +184,7 @@ int ave_ipc_negotiate(struct ave_dev *ave, const u32 hello[4]) put_unaligned_le32(AVE_IPC_SHARED_BYTES, hs + AVE_HS_SHARED_BYTES); put_unaligned_le64(ave->heap.dma, hs + AVE_HS_HEAP); put_unaligned_le32(hello[3], hs + AVE_HS_HEAP_BYTES); - put_unaligned_le32(AVE_HS_MAGIC_VALUE, hs + AVE_HS_MAGIC); + put_unaligned_le32(ave->variant->device_type, hs + AVE_HS_DEVICE_TYPE); dma_wmb(); ave_mbox_send(ave, lower_32_bits(ave->ipc_fw_base + AVE_IPC_HANDSHAKE_OFF), diff --git a/drivers/media/platform/apple/ave/ave-regs.h b/drivers/media/platform/apple/ave/ave-regs.h index a85fde26278a6..4b625ad3b70d9 100644 --- a/drivers/media/platform/apple/ave/ave-regs.h +++ b/drivers/media/platform/apple/ave/ave-regs.h @@ -36,7 +36,6 @@ #define AVE_BOOT_MAGIC 0x08042006 #define AVE_BOOT_CFG_BYTES 0x40 #define AVE_BOOT_CFG_DEVICE 1 /* word index */ -#define AVE_BOOT_CFG_DEVICE_T8140 29 #define AVE_BOOT_CFG_UNITS 2 #define AVE_BOOT_CFG_GROUPS 3 #define AVE_BOOT_CFG_SUBID_MASK 4 @@ -64,8 +63,7 @@ #define AVE_HS_SHARED_BYTES 0x18 #define AVE_HS_HEAP 0x1c #define AVE_HS_HEAP_BYTES 0x24 -#define AVE_HS_MAGIC 0x28 -#define AVE_HS_MAGIC_VALUE 27 +#define AVE_HS_DEVICE_TYPE 0x28 /* Channel descriptors the firmware writes into the channel area */ #define AVE_CHAN_DESC_STRIDE 0x100 diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index e58d485eb61c1..c97681d7fb840 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -232,7 +232,7 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) bool hevc = p->codec == AVE_CODEC_HEVC; u32 w = p->width, h = p->height; u32 w16 = DIV_ROUND_UP(w, 16), w32 = DIV_ROUND_UP(w, 32); - u32 h32 = DIV_ROUND_UP(h, 32), w64 = DIV_ROUND_UP(w, 64), h64 = DIV_ROUND_UP(h, 64); + u32 h32 = DIV_ROUND_UP(h, 32), h64 = DIV_ROUND_UP(h, 64); const unsigned long nomap = DMA_ATTR_NO_KERNEL_MAPPING; size_t ref_a, ref_b, ref_c, ref_d, tile, mapped = 0; struct ave_buf_req *r; @@ -245,7 +245,7 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) ref_a = ALIGN(w, 32) * ALIGN(h + 16, 32); ref_b = roundup_pow_of_two(w) * roundup_pow_of_two(h) / 32; - ref_c = hevc ? 512 * w64 * h64 : ALIGN(w, 64) * (ALIGN(h, 16) / 2 + 24); + ref_c = ave_lfs_ref_size(w, h, hevc, ave->variant->lfs_ref_scale); ref_d = ALIGN(hevc ? 64 * w32 * h32 : 128 * p->mbs, SZ_16K); tile = hevc ? 2304 * w32 * h64 : ALIGN(w, 16) * ALIGN(h, 64); diff --git a/drivers/media/platform/apple/ave/ave-sizes-test.c b/drivers/media/platform/apple/ave/ave-sizes-test.c new file mode 100644 index 0000000000000..72e92f67b2d5e --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-sizes-test.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Apple AVE LFS reference descriptor sizes, not DMA allocation sizes. */ +#include +#include + +#include "ave-sizes.h" + +struct ave_size_case { + u32 width; + u32 height; + u32 avc; + u32 hevc; + u32 avc_t8140; + u32 hevc_t8140; +}; + +static const struct ave_size_case ave_size_cases[] = { + { 320, 240, 0x5a00, 0x1400, 0xb400, 0x2800 }, + { 640, 480, 0x14a00, 0x5000, 0x29400, 0xa000 }, + { 642, 482, 0x17600, 0x5800, 0x2ec00, 0xb000 }, + { 1280, 720, 0x3c000, 0xf000, 0x78000, 0x1e000 }, + { 1920, 1080, 0x85200, 0x1fe00, 0x10a400, 0x3fc00 }, + { 1920, 1088, 0x85200, 0x1fe00, 0x10a400, 0x3fc00 }, + { 1922, 1090, 0x8b800, 0x22e00, 0x117000, 0x45c00 }, + { 2560, 1600, 0x101800, 0x3e800, 0x203000, 0x7d000 }, +}; + +static void ave_size_desc(const struct ave_size_case *c, char *desc) +{ + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%ux%u", c->width, c->height); +} + +KUNIT_ARRAY_PARAM(ave_size, ave_size_cases, ave_size_desc); + +static void ave_lfs_ref_test(struct kunit *test) +{ + const struct ave_size_case *c = test->param_value; + + /* T6000 and T8112 both select scale 1; T8140 selects scale 2. */ + KUNIT_EXPECT_EQ(test, ave_lfs_ref_size(c->width, c->height, false, 1), c->avc); + KUNIT_EXPECT_EQ(test, ave_lfs_ref_size(c->width, c->height, true, 1), c->hevc); + KUNIT_EXPECT_EQ(test, ave_lfs_ref_size(c->width, c->height, false, 2), c->avc_t8140); + KUNIT_EXPECT_EQ(test, ave_lfs_ref_size(c->width, c->height, true, 2), c->hevc_t8140); +} + +static struct kunit_case ave_sizes_tests[] = { + KUNIT_CASE_PARAM(ave_lfs_ref_test, ave_size_gen_params), + { } +}; + +static struct kunit_suite ave_sizes_suite = { + .name = "apple-ave-sizes", + .test_cases = ave_sizes_tests, +}; + +kunit_test_suite(ave_sizes_suite); + +MODULE_DESCRIPTION("Apple AVE buffer sizing tests"); +MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/ave/ave-sizes.h b/drivers/media/platform/apple/ave/ave-sizes.h new file mode 100644 index 0000000000000..01e29678035b5 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-sizes.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _APPLE_AVE_SIZES_H_ +#define _APPLE_AVE_SIZES_H_ + +#include +#include +#include + +/* Start +0x7d0 LFSRef: exact descriptor bytes, before DMA page rounding. */ +static inline u32 ave_lfs_ref_size(u32 width, u32 height, bool hevc, u32 scale) +{ + u32 size; + + if (hevc) + size = 256 * DIV_ROUND_UP(width, 64) * DIV_ROUND_UP(height, 64); + else + size = ALIGN(width, 64) * (ALIGN(height, 16) / 4 + 12); + return size * scale; +} + +#endif diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h index 0f6e2b38d86ca..c0c150f7fab42 100644 --- a/drivers/media/platform/apple/ave/ave.h +++ b/drivers/media/platform/apple/ave/ave.h @@ -19,6 +19,8 @@ #include #include +#include "ave-sizes.h" + struct debugfs_blob_wrapper; struct dentry; struct dev_pm_domain_list; @@ -111,8 +113,31 @@ enum ave_image { struct ave_ctx; +/* Register offsets for the shared 0x40 coprocessor architecture. */ +struct ave_coproc_regs { + u32 vcbar; + u32 timebase_freq; + u32 timebase; + u32 ctrl; + u32 status; + u32 boot_arg; + u32 keep_mappings; +}; + +struct ave_variant { + const struct ave_coproc_regs *regs; + u32 fw_text_size; + u32 fw_data_size; + u32 boot_device_id; + u32 device_type; + u32 lfs_ref_scale; + /* Zero means no command constants or MCC DSID have been qualified. */ + u32 encoding_text_crc; +}; + struct ave_dev { struct device *dev; + const struct ave_variant *variant; void __iomem *codec; void __iomem *mbox; void __iomem *coproc; @@ -122,6 +147,7 @@ struct ave_dev { struct ave_fw_segment text; struct ave_fw_segment data; u32 text_crc; + u32 data_crc; void *data_pristine; bool data_dirty; From 38fc158c628c60ac4400f7dc543287c9ad50529f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:16:54 +1100 Subject: [PATCH 1428/1579] arm64: dts: apple: describe disabled T6000 and T8112 AVE blocks Describe the codec, mailbox and coprocessor windows, all five AVE IRQs and both SID 0 DARTs from the ADT facts. Use the existing VENC power domains and preserve the 0x8000 pre-M4 mailbox windows. Leave the encoder and both DARTs disabled pending loader handoff and owner qualification. Do not guess the DMA aperture or add an RTKit endpoint. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 39 +++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8112.dtsi | 39 +++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 2cbe84a3d5625..8357258058339 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -8,6 +8,45 @@ &SOC_NODE { + ave_dart1: iommu@20d030000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x0d030000 0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_venc_sys>; + #iommu-cells = <1>; + status = "disabled"; + }; + + ave_dart0: iommu@20d040000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x0d040000 0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_venc_sys>; + #iommu-cells = <1>; + status = "disabled"; + }; + + ave: video-codec@20d100000 { + compatible = "apple,t6000-ave"; + reg = <0x2 0x0d100000 0 0x45c000>, + <0x2 0x0d050000 0 0x8000>, + <0x2 0x0d800000 0 0x800000>; + reg-names = "codec", "mailbox", "coproc"; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + ; + power-domains = <&ps_venc_dma>, <&ps_venc_pipe4>, + <&ps_venc_pipe5>, <&ps_venc_me0>, <&ps_venc_me1>; + power-domain-names = "dma", "pipe4", "pipe5", "me0", "me1"; + iommus = <&ave_dart0 0>, <&ave_dart1 0>; + status = "disabled"; + }; + ane_dart0: iommu@285800000 { compatible = "apple,t6000-dart"; reg = <0x2 0x85800000 0x0 0x4000>; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 90f9311d4aede..d7a4246482355 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -505,6 +505,45 @@ /* Required to get >32-bit DMA via DARTs */ dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; + ave_dart0: iommu@67030000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0 0x67030000 0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_venc_sys>; + #iommu-cells = <1>; + status = "disabled"; + }; + + ave_dart1: iommu@67040000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0 0x67040000 0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_venc_sys>; + #iommu-cells = <1>; + status = "disabled"; + }; + + ave: video-codec@67100000 { + compatible = "apple,t8112-ave"; + reg = <0 0x67100000 0 0x45c000>, + <0 0x67050000 0 0x8000>, + <0 0x67800000 0 0x800000>; + reg-names = "codec", "mailbox", "coproc"; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + ; + power-domains = <&ps_venc_dma>, <&ps_venc_pipe4>, + <&ps_venc_pipe5>, <&ps_venc_me0>, <&ps_venc_me1>; + power-domain-names = "dma", "pipe4", "pipe5", "me0", "me1"; + iommus = <&ave_dart0 0>, <&ave_dart1 0>; + status = "disabled"; + }; + gpu: gpu@206400000 { compatible = "apple,agx-t8112", "apple,agx-g14g"; reg = <0x2 0x6400000 0 0x40000>, From a6eb260445383bd271b609bbe05d66e197a2f9d0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:16:33 +1100 Subject: [PATCH 1429/1579] dt-bindings: power: apple: describe firmware-managed DRAM classes Describe explicit DCS state IDs, inferred interface classes and the distinction between physical peak bandwidth and the published rounded class. Leave unaccepted lower-state rates unspecified and do not describe a PMGR achieved-state register. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../opp/operating-points-v2-apple-dram.yaml | 80 +++++++++++++++++ .../bindings/power/apple,pmp-dram.yaml | 88 +++++++++++++++++++ .../power/apple,t8140-pmp-report.yaml | 19 +++- 3 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 Documentation/devicetree/bindings/opp/operating-points-v2-apple-dram.yaml create mode 100644 Documentation/devicetree/bindings/power/apple,pmp-dram.yaml diff --git a/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dram.yaml b/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dram.yaml new file mode 100644 index 0000000000000..a201d86f1ba42 --- /dev/null +++ b/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dram.yaml @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/opp/operating-points-v2-apple-dram.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple firmware-managed DRAM performance states + +maintainers: + - Ryan Murray + +description: | + DCS firmware state IDs are not clock rates. Describe every state with an + explicit opp-level, independent of node order. Rates are optional except + for the highest state. The maximum interface rate and data width are + inferred from the interface topology and published bandwidth class. + + Lower-state rates derived from firmware bandwidth policy thresholds are + inferences, not measured clocks. Those thresholds use Q16.16 decimal GB/s; + they are not a frequency table. Leave lower-state rates absent until the + inference is accepted or independently confirmed by a public decoder. + + opp-hz is CK-equivalent Hz, rounded upwards to the nearest integer, one + half of the interface transfer rate. It does not claim a literal command + clock or LPDDR5 WCK. These OPPs describe firmware states for reporting; + they do not authorize Linux to change clocks, voltages, or firmware votes. + +allOf: + - $ref: opp-v2-base.yaml# + +properties: + compatible: + const: operating-points-v2-apple-dram + + opp-shared: true + +patternProperties: + '^opp(-?[0-9]+)*$': + type: object + additionalProperties: false + + properties: + opp-level: + minimum: 0 + maximum: 31 + + opp-hz: + maxItems: 1 + + apple,data-rate: + $ref: /schemas/types.yaml#/definitions/uint64 + minimum: 1 + description: Interface transfers per second per data pin, rounded to an integer. + + required: + - opp-level + + dependencies: + apple,data-rate: [opp-hz] + opp-hz: [apple,data-rate] + +required: + - compatible + +additionalProperties: false + +examples: + - | + opp-table-dram { + compatible = "operating-points-v2-apple-dram"; + opp-shared; + opp-0 { + opp-level = <0>; + }; + opp-1 { + opp-level = <1>; + opp-hz = /bits/ 64 <3200000000>; + apple,data-rate = /bits/ 64 <6400000000>; + }; + }; diff --git a/Documentation/devicetree/bindings/power/apple,pmp-dram.yaml b/Documentation/devicetree/bindings/power/apple,pmp-dram.yaml new file mode 100644 index 0000000000000..250c2e2019949 --- /dev/null +++ b/Documentation/devicetree/bindings/power/apple,pmp-dram.yaml @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/power/apple,pmp-dram.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple PMP DRAM reporting properties + +maintainers: + - Ryan Murray + +description: | + Static description of the firmware-managed DRAM interface. These properties + reside on the existing PMP report provider, which is available independently + of a Linux PMP firmware start. No extra MMIO resource or ownership change is + implied. The bus width and maximum data rate are inferred physical classes. + + The observed LPDDR5 family string does not distinguish LPDDR5X. Do not + report an inferred X suffix as an observed memory type. Bandwidth is in + decimal bytes per second; usable data width excludes ECC/check bits. + + The OPP table must contain exactly apple,dcs-state-count available entries, + with unique opp-level values covering zero through count minus one. Only + the highest state must have a rate. Peak bandwidth must equal that state's + apple,data-rate multiplied by apple,data-bus-width divided by eight. + The published bandwidth class may be rounded (for example 200 GB/s for + an inferred physical peak of 204.8 GB/s). + + This description does not identify an achieved-state register. PMGR desired + states and PMC floor/ceiling votes are not achieved memory states. The + T6000/T8112 PMP DRAM State/FxS reporter has bucket 3*S+M, where S is DCS + state and M is self-refresh (0), power-down (1), or active (2). That encoding + alone does not specify a transport or a readable register resource. No + current clock may be claimed from a low-power bucket or unknown state. + +select: false + +properties: + operating-points-v2: + description: Reference to an operating-points-v2-apple-dram table. + maxItems: 1 + + apple,dram-type: + $ref: /schemas/types.yaml#/definitions/string + const: lpddr5 + description: Observed DRAM family, without an inferred LPDDR5X suffix. + + apple,data-bus-width: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 8 + multipleOf: 8 + description: Usable data bits, excluding ECC/check bits. + + apple,dcs-state-count: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 32 + description: Number of firmware DCS states. + + apple,peak-bandwidth-bytes-per-second: + $ref: /schemas/types.yaml#/definitions/uint64 + minimum: 1 + description: Physical maximum bandwidth calculated from the highest OPP and data width. + + apple,bandwidth-class-bytes-per-second: + $ref: /schemas/types.yaml#/definitions/uint64 + minimum: 1 + description: Published, potentially rounded, maximum bandwidth class. + +dependencies: + operating-points-v2: [apple,dram-type] + apple,data-bus-width: [apple,dram-type] + apple,dcs-state-count: [apple,dram-type] + apple,peak-bandwidth-bytes-per-second: [apple,dram-type] + apple,bandwidth-class-bytes-per-second: [apple,dram-type] + +allOf: + - if: + required: [apple,dram-type] + then: + required: + - operating-points-v2 + - apple,data-bus-width + - apple,dcs-state-count + - apple,peak-bandwidth-bytes-per-second + - apple,bandwidth-class-bytes-per-second + +additionalProperties: true diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml index e79ea331865b5..a5f1475926430 100644 --- a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/power/apple,t8140-pmp-report.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple T8140 PMP report aperture +title: Apple PMP report aperture maintainers: - Ryan Murray @@ -18,10 +18,15 @@ description: | documented report entries may be read; the aperture is not a contiguous readable array of registers. +allOf: + - $ref: apple,pmp-dram.yaml# + properties: compatible: enum: + - apple,t6000-pmp-v2-report - apple,t6041-pmp-v2-report + - apple,t8112-pmp-v2-report - apple,t8140-pmp-v2-report reg: @@ -38,6 +43,13 @@ properties: '#size-cells': const: 0 + operating-points-v2: true + apple,dram-type: true + apple,data-bus-width: true + apple,dcs-state-count: true + apple,peak-bandwidth-bytes-per-second: true + apple,bandwidth-class-bytes-per-second: true + apple,pmp: $ref: /schemas/types.yaml#/definitions/phandle description: @@ -57,9 +69,12 @@ patternProperties: properties: compatible: oneOf: + - const: apple,t6000-pmp-v2-report-entry - const: apple,t8140-pmp-v2-report-entry - items: - - const: apple,t6041-pmp-v2-report-entry + - enum: + - apple,t6041-pmp-v2-report-entry + - apple,t8112-pmp-v2-report-entry - const: apple,t6000-pmp-v2-report-entry reg: From e2fc1e35ee3d07fc3f1ea537066e81eb5a08a328 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:16:35 +1100 Subject: [PATCH 1430/1579] pmdomain: apple: report DT DRAM classes through the PMP provider Expose read-only metadata on the existing PMP report device without introducing devfreq control or claiming a current clock. Validate explicit state coverage, rate units and peak-bandwidth arithmetic before publishing attributes. Exercise the property parser with software-node KUnit fixtures. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../ABI/testing/sysfs-driver-apple-pmp-dram | 49 ++++ drivers/pmdomain/apple/Kconfig | 12 + drivers/pmdomain/apple/Makefile | 1 + drivers/pmdomain/apple/pmp-dram-test.c | 258 ++++++++++++++++++ drivers/pmdomain/apple/pmp-dram.h | 114 ++++++++ drivers/pmdomain/apple/pmp-report.c | 48 ++++ 6 files changed, 482 insertions(+) create mode 100644 Documentation/ABI/testing/sysfs-driver-apple-pmp-dram create mode 100644 drivers/pmdomain/apple/pmp-dram-test.c create mode 100644 drivers/pmdomain/apple/pmp-dram.h diff --git a/Documentation/ABI/testing/sysfs-driver-apple-pmp-dram b/Documentation/ABI/testing/sysfs-driver-apple-pmp-dram new file mode 100644 index 0000000000000..2e7cfaba58445 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-apple-pmp-dram @@ -0,0 +1,49 @@ +What: /sys/bus/platform/devices/.../dram/type +Date: October 2026 +Contact: Ryan Murray +Description: + Read-only DRAM family from DT, currently "lpddr5". This family + includes LPDDR5X; the attribute does not assert the X suffix. + The dram group is present on an apple-pmp-report device only + when DT supplies a complete, consistent memory description. + Static attributes remain valid when firmware is stopped. + +What: /sys/bus/platform/devices/.../dram/max_data_rate +Date: October 2026 +Contact: Ryan Murray +Description: + Read-only maximum interface transfers per second per data pin + (divide by 1000000 for MT/s), from the highest DT OPP. This is + an inferred interface class, not a measured current rate or + a literal LPDDR5 command clock/WCK. The maximum state ID is + state_count minus one, independent of DT child order. + +What: /sys/bus/platform/devices/.../dram/peak_bandwidth +Date: October 2026 +Contact: Ryan Murray +Description: + Read-only inferred physical maximum in decimal bytes/second, + max_data_rate * data_bus_width / 8. This excludes ECC/check bits + and does not measure sustainable application bandwidth. + +What: /sys/bus/platform/devices/.../dram/bandwidth_class +Date: October 2026 +Contact: Ryan Murray +Description: + Read-only published maximum bandwidth class in decimal + bytes/second. It may be rounded relative to peak_bandwidth. + +What: /sys/bus/platform/devices/.../dram/data_bus_width +Date: October 2026 +Contact: Ryan Murray +Description: + Read-only inferred usable data width in bits, excluding ECC. + +What: /sys/bus/platform/devices/.../dram/state_count +Date: October 2026 +Contact: Ryan Murray +Description: + Read-only number of firmware DCS states, from DT. State IDs + are not frequencies. No current-state, current-rate, residency, + or transition-statistics attribute is provided without an + independently specified achieved-state transport. diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index e25dd30281fc4..f1293707ca8d2 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -101,3 +101,15 @@ config APPLE_PMP_REPORT_KUNIT_TEST argument validation without accessing hardware. Includes malformed tables, address arithmetic and device ordering. The PMP report driver does not need to be enabled to run this suite. + +config APPLE_PMP_DRAM_KUNIT_TEST + tristate "Apple PMP DRAM description parsing tests" if !KUNIT_ALL_TESTS + depends on KUNIT && PM + select PM_GENERIC_DOMAINS + default KUNIT_ALL_TESTS + help + Parse synthetic firmware properties including unordered state IDs, + missing and duplicate states, incomplete descriptions and overflow. + Software-node fixtures exercise the parser without a DT boot image. + These tests access no hardware and do not need the PMP driver. They + are only useful for kernel developers. diff --git a/drivers/pmdomain/apple/Makefile b/drivers/pmdomain/apple/Makefile index ff7f9192b5e39..9f035fd2827f0 100644 --- a/drivers/pmdomain/apple/Makefile +++ b/drivers/pmdomain/apple/Makefile @@ -3,4 +3,5 @@ obj-$(CONFIG_APPLE_PMGR_PWRSTATE) += pmgr-pwrstate.o obj-$(CONFIG_APPLE_PMGR_PWRSTATE) += gfx-tvm.o obj-$(CONFIG_APPLE_PMP_REPORT) += pmp-report.o obj-$(CONFIG_APPLE_PMP_REPORT_KUNIT_TEST) += pmp-report-test.o +obj-$(CONFIG_APPLE_PMP_DRAM_KUNIT_TEST) += pmp-dram-test.o obj-$(CONFIG_APPLE_PMC) += pmc.o diff --git a/drivers/pmdomain/apple/pmp-dram-test.c b/drivers/pmdomain/apple/pmp-dram-test.c new file mode 100644 index 0000000000000..5a7bd479b3ba7 --- /dev/null +++ b/drivers/pmdomain/apple/pmp-dram-test.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Exercise the actual firmware-property parser without MMIO or firmware. */ +#include +#include + +#include "pmp-dram.h" + +struct pmp_dram_fixture { + struct software_node nodes[8]; + const struct software_node *group[9]; + struct software_node_ref_args ref[1]; + struct property_entry props[8]; + struct property_entry table_props[2]; + struct property_entry opp_props[6][4]; +}; + +static void pmp_dram_unregister(void *data) +{ + struct pmp_dram_fixture *f = data; + + software_node_unregister_node_group(f->group); +} + +static struct pmp_dram_fixture *pmp_dram_fixture(struct kunit *test, u32 count, + u32 width, u64 rate, u64 bandwidth_class) +{ + static const char * const names[] = { + "opp-0", "opp-1", "opp-2", "opp-3", "opp-4", "opp-5", + }; + struct pmp_dram_fixture *f; + unsigned int i; + + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + f->ref[0].node = &f->nodes[1]; + f->props[0] = PROPERTY_ENTRY_STRING("apple,dram-type", "lpddr5"); + f->props[1] = PROPERTY_ENTRY_U32("apple,data-bus-width", width); + f->props[2] = PROPERTY_ENTRY_U32("apple,dcs-state-count", count); + f->props[3] = PROPERTY_ENTRY_U64("apple,peak-bandwidth-bytes-per-second", + rate * (width / 8)); + f->props[4] = PROPERTY_ENTRY_U64("apple,bandwidth-class-bytes-per-second", bandwidth_class); + f->props[5] = PROPERTY_ENTRY_REF_ARRAY("operating-points-v2", f->ref); + f->nodes[0].name = "pmp-dram-test"; + f->nodes[0].properties = f->props; + f->table_props[0] = PROPERTY_ENTRY_STRING("compatible", "operating-points-v2-apple-dram"); + f->nodes[1].name = "opp-table-dram"; + f->nodes[1].properties = f->table_props; + for (i = 0; i < count; i++) { + struct software_node *opp = &f->nodes[i + 2]; + + opp->name = names[i]; + opp->parent = &f->nodes[1]; + opp->properties = f->opp_props[i]; + /* Reverse order: the highest state is not the last child. */ + f->opp_props[i][0] = PROPERTY_ENTRY_U32("opp-level", count - i - 1); + if (!i) { + f->opp_props[i][1] = PROPERTY_ENTRY_U64("apple,data-rate", rate); + f->opp_props[i][2] = PROPERTY_ENTRY_U64("opp-hz", rate / 2 + rate % 2); + } + } + for (i = 0; i < count + 2; i++) + f->group[i] = &f->nodes[i]; + return f; +} + +static int pmp_dram_parse_fixture(struct kunit *test, struct pmp_dram_fixture *f, + struct apple_pmp_dram *dram) +{ + int ret; + + ret = software_node_register_node_group(f->group); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, pmp_dram_unregister, f); + KUNIT_ASSERT_EQ(test, ret, 0); + return apple_pmp_dram_parse(software_node_fwnode(&f->nodes[0]), dram); +} + +static void pmp_dram_valid(struct kunit *test) +{ + struct pmp_dram_fixture *f = pmp_dram_fixture(test, 4, 256, 6400000000ULL, 200000000000ULL); + struct apple_pmp_dram dram = {}; + + KUNIT_ASSERT_EQ(test, pmp_dram_parse_fixture(test, f, &dram), 0); + KUNIT_EXPECT_STREQ(test, dram.type, "lpddr5"); + KUNIT_EXPECT_EQ(test, dram.max_rate, 6400000000ULL); + KUNIT_EXPECT_EQ(test, dram.peak_bandwidth, 204800000000ULL); + KUNIT_EXPECT_EQ(test, dram.bandwidth_class, 200000000000ULL); + KUNIT_EXPECT_EQ(test, dram.state_count, 4U); + KUNIT_EXPECT_EQ(test, dram.width, 256U); +} + +static void pmp_dram_m2(struct kunit *test) +{ + struct pmp_dram_fixture *f = pmp_dram_fixture(test, 5, 128, 6400000000ULL, 100000000000ULL); + struct apple_pmp_dram dram = {}; + + KUNIT_ASSERT_EQ(test, pmp_dram_parse_fixture(test, f, &dram), 0); + KUNIT_EXPECT_EQ(test, dram.peak_bandwidth, 102400000000ULL); + KUNIT_EXPECT_EQ(test, dram.state_count, 5U); +} + +static void pmp_dram_m4pro(struct kunit *test) +{ + struct pmp_dram_fixture *f = pmp_dram_fixture(test, 6, 256, 8533333333ULL, 273000000000ULL); + struct apple_pmp_dram dram = {}; + + KUNIT_ASSERT_EQ(test, pmp_dram_parse_fixture(test, f, &dram), 0); + KUNIT_EXPECT_EQ(test, dram.max_rate, 8533333333ULL); + KUNIT_EXPECT_EQ(test, dram.peak_bandwidth, 273066666656ULL); + KUNIT_EXPECT_EQ(test, dram.state_count, 6U); +} + +static void pmp_dram_a18pro(struct kunit *test) +{ + struct pmp_dram_fixture *f = pmp_dram_fixture(test, 5, 64, 7500000000ULL, 60000000000ULL); + struct apple_pmp_dram dram = {}; + + KUNIT_ASSERT_EQ(test, pmp_dram_parse_fixture(test, f, &dram), 0); + KUNIT_EXPECT_EQ(test, dram.peak_bandwidth, 60000000000ULL); + KUNIT_EXPECT_EQ(test, dram.width, 64U); +} + +enum pmp_dram_fault { + DRAM_ZERO_WIDTH, DRAM_UNALIGNED_WIDTH, DRAM_ZERO_COUNT, DRAM_LARGE_COUNT, + DRAM_DUPLICATE, DRAM_MISSING, DRAM_UNKNOWN_LEVEL, DRAM_MISSING_LEVEL, + DRAM_NO_MAX, DRAM_ZERO_RATE, DRAM_BAD_CK, DRAM_CK_ONLY, DRAM_BAD_PEAK, + DRAM_OVERFLOW, DRAM_BAD_TYPE, DRAM_NO_CLASS, DRAM_ZERO_CLASS, + DRAM_NO_TABLE, DRAM_BAD_COMPATIBLE, DRAM_LOW_ABOVE_MAX, DRAM_WRONG_TYPE, +}; + +static void pmp_dram_bad_description(struct kunit *test) +{ + enum pmp_dram_fault fault = *(enum pmp_dram_fault *)test->param_value; + struct pmp_dram_fixture *f = pmp_dram_fixture(test, 4, 256, 6400000000ULL, 200000000000ULL); + struct apple_pmp_dram dram = { .max_rate = 123 }; + + switch (fault) { + case DRAM_ZERO_WIDTH: + f->props[1] = PROPERTY_ENTRY_U32("apple,data-bus-width", 0); + break; + case DRAM_UNALIGNED_WIDTH: + f->props[1] = PROPERTY_ENTRY_U32("apple,data-bus-width", 7); + break; + case DRAM_ZERO_COUNT: + f->props[2] = PROPERTY_ENTRY_U32("apple,dcs-state-count", 0); + break; + case DRAM_LARGE_COUNT: + f->props[2] = PROPERTY_ENTRY_U32("apple,dcs-state-count", 33); + break; + case DRAM_DUPLICATE: + f->opp_props[1][0] = PROPERTY_ENTRY_U32("opp-level", 3); + break; + case DRAM_MISSING: + f->group[5] = NULL; + break; + case DRAM_UNKNOWN_LEVEL: + f->opp_props[1][0] = PROPERTY_ENTRY_U32("opp-level", 4); + break; + case DRAM_MISSING_LEVEL: + memset(&f->opp_props[1][0], 0, sizeof(struct property_entry)); + break; + case DRAM_NO_MAX: + memset(&f->opp_props[0][1], 0, 2 * sizeof(struct property_entry)); + break; + case DRAM_ZERO_RATE: + f->opp_props[0][1] = PROPERTY_ENTRY_U64("apple,data-rate", 0); + break; + case DRAM_BAD_CK: + f->opp_props[0][2] = PROPERTY_ENTRY_U64("opp-hz", 6400000000ULL); + break; + case DRAM_CK_ONLY: + f->opp_props[0][1] = f->opp_props[0][2]; + memset(&f->opp_props[0][2], 0, sizeof(struct property_entry)); + break; + case DRAM_BAD_PEAK: + f->props[3] = PROPERTY_ENTRY_U64("apple,peak-bandwidth-bytes-per-second", + 200000000000ULL); + break; + case DRAM_OVERFLOW: + f->opp_props[0][1] = PROPERTY_ENTRY_U64("apple,data-rate", U64_MAX - 1); + f->opp_props[0][2] = PROPERTY_ENTRY_U64("opp-hz", (U64_MAX - 1) / 2); + break; + case DRAM_BAD_TYPE: + f->props[0] = PROPERTY_ENTRY_STRING("apple,dram-type", "lpddr5x"); + break; + case DRAM_NO_CLASS: + f->props[4] = PROPERTY_ENTRY_U64("unrelated", 1); + break; + case DRAM_ZERO_CLASS: + f->props[4] = PROPERTY_ENTRY_U64("apple,bandwidth-class-bytes-per-second", 0); + break; + case DRAM_NO_TABLE: + memset(&f->props[5], 0, sizeof(struct property_entry)); + break; + case DRAM_BAD_COMPATIBLE: + f->table_props[0] = PROPERTY_ENTRY_STRING("compatible", "operating-points-v2"); + break; + case DRAM_LOW_ABOVE_MAX: + f->opp_props[1][1] = PROPERTY_ENTRY_U64("apple,data-rate", 7500000000ULL); + f->opp_props[1][2] = PROPERTY_ENTRY_U64("opp-hz", 3750000000ULL); + break; + case DRAM_WRONG_TYPE: + f->props[2] = PROPERTY_ENTRY_STRING("apple,dcs-state-count", "four"); + break; + } + KUNIT_EXPECT_LT(test, pmp_dram_parse_fixture(test, f, &dram), 0); + /* Never publish partially parsed values or a fallback maximum. */ + KUNIT_EXPECT_EQ(test, dram.max_rate, 123ULL); +} + +static const enum pmp_dram_fault pmp_dram_faults[] = { + DRAM_ZERO_WIDTH, DRAM_UNALIGNED_WIDTH, DRAM_ZERO_COUNT, DRAM_LARGE_COUNT, + DRAM_DUPLICATE, DRAM_MISSING, DRAM_UNKNOWN_LEVEL, DRAM_MISSING_LEVEL, + DRAM_NO_MAX, DRAM_ZERO_RATE, DRAM_BAD_CK, DRAM_CK_ONLY, DRAM_BAD_PEAK, + DRAM_OVERFLOW, DRAM_BAD_TYPE, DRAM_NO_CLASS, DRAM_ZERO_CLASS, + DRAM_NO_TABLE, DRAM_BAD_COMPATIBLE, DRAM_LOW_ABOVE_MAX, DRAM_WRONG_TYPE, +}; + +KUNIT_ARRAY_PARAM(pmp_dram_fault, pmp_dram_faults, NULL); + +static void pmp_dram_old_dt(struct kunit *test) +{ + struct pmp_dram_fixture *f = pmp_dram_fixture(test, 4, 256, 6400000000ULL, 200000000000ULL); + struct apple_pmp_dram dram = {}; + + memset(f->props, 0, sizeof(f->props)); + KUNIT_EXPECT_EQ(test, pmp_dram_parse_fixture(test, f, &dram), -ENODATA); +} + +static void pmp_dram_partial_dt(struct kunit *test) +{ + struct pmp_dram_fixture *f = pmp_dram_fixture(test, 4, 256, 6400000000ULL, 200000000000ULL); + struct apple_pmp_dram dram = {}; + + f->props[0] = PROPERTY_ENTRY_U32("apple,data-bus-width", 128); + memset(&f->props[1], 0, sizeof(struct property_entry)); + KUNIT_EXPECT_EQ(test, pmp_dram_parse_fixture(test, f, &dram), -EINVAL); +} + +static struct kunit_case pmp_dram_cases[] = { + KUNIT_CASE(pmp_dram_valid), + KUNIT_CASE(pmp_dram_m2), + KUNIT_CASE(pmp_dram_m4pro), + KUNIT_CASE(pmp_dram_a18pro), + KUNIT_CASE_PARAM(pmp_dram_bad_description, pmp_dram_fault_gen_params), + KUNIT_CASE(pmp_dram_old_dt), + KUNIT_CASE(pmp_dram_partial_dt), + {} +}; + +static struct kunit_suite pmp_dram_suite = { + .name = "apple-pmp-dram", + .test_cases = pmp_dram_cases, +}; + +kunit_test_suite(pmp_dram_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/pmdomain/apple/pmp-dram.h b/drivers/pmdomain/apple/pmp-dram.h new file mode 100644 index 0000000000000..08a7fcf9c17c8 --- /dev/null +++ b/drivers/pmdomain/apple/pmp-dram.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _APPLE_PMP_DRAM_H +#define _APPLE_PMP_DRAM_H + +#include +#include +#include +#include +#include + +#define APPLE_PMP_DRAM_MAX_STATES 32 + +struct apple_pmp_dram { + const char *type; + u32 width; + u32 state_count; + u64 max_rate; + u64 peak_bandwidth; + u64 bandwidth_class; +}; + +/* + * Read metadata only: never register these OPPs with a clock/voltage provider. + * Unknown lower-state rates are deliberately absent. Firmware state IDs, not + * child order or frequency sorting, identify the entries. + */ +static inline int apple_pmp_dram_parse(const struct fwnode_handle *node, + struct apple_pmp_dram *dram) +{ + struct apple_pmp_dram parsed = {}; + struct fwnode_reference_args ref; + struct fwnode_handle *opp; + u64 rates[APPLE_PMP_DRAM_MAX_STATES] = {}; + u64 seen = 0, peak; + u32 level; + unsigned int i; + int ret; + + /* Old DTs with no memory description retain the existing interface. */ + if (!fwnode_property_present(node, "apple,dram-type") && + !fwnode_property_present(node, "apple,data-bus-width") && + !fwnode_property_present(node, "apple,dcs-state-count") && + !fwnode_property_present(node, "apple,peak-bandwidth-bytes-per-second") && + !fwnode_property_present(node, "apple,bandwidth-class-bytes-per-second") && + !fwnode_property_present(node, "operating-points-v2")) + return -ENODATA; + + if (fwnode_property_read_string(node, "apple,dram-type", &parsed.type) || + strcmp(parsed.type, "lpddr5") || + fwnode_property_read_u32(node, "apple,data-bus-width", &parsed.width) || + !parsed.width || parsed.width % 8 || + fwnode_property_read_u32(node, "apple,dcs-state-count", &parsed.state_count) || + !parsed.state_count || parsed.state_count > APPLE_PMP_DRAM_MAX_STATES || + fwnode_property_read_u64(node, "apple,peak-bandwidth-bytes-per-second", + &parsed.peak_bandwidth) || + fwnode_property_read_u64(node, "apple,bandwidth-class-bytes-per-second", + &parsed.bandwidth_class) || !parsed.bandwidth_class) + return -EINVAL; + + ret = fwnode_property_get_reference_args(node, "operating-points-v2", NULL, 0, 0, &ref); + if (ret) + return ret; + if (fwnode_property_match_string(ref.fwnode, "compatible", + "operating-points-v2-apple-dram") != 0) { + ret = -EINVAL; + goto out; + } + fwnode_for_each_available_child_node(ref.fwnode, opp) { + u64 rate, ck; + + if (fwnode_property_read_u32(opp, "opp-level", &level) || + level >= parsed.state_count || (seen & BIT_ULL(level))) + goto bad_opp; + seen |= BIT_ULL(level); + if (!fwnode_property_present(opp, "apple,data-rate")) { + if (fwnode_property_present(opp, "opp-hz")) + goto bad_opp; + continue; + } + if (fwnode_property_read_u64(opp, "apple,data-rate", &rate) || !rate || + fwnode_property_read_u64(opp, "opp-hz", &ck) || !ck || + ck != rate / 2 + rate % 2) + goto bad_opp; + rates[level] = rate; + } + if (seen != GENMASK_ULL(parsed.state_count - 1, 0)) { + ret = -EINVAL; + goto out; + } + parsed.max_rate = rates[parsed.state_count - 1]; + if (!parsed.max_rate || + check_mul_overflow(parsed.max_rate, (u64)parsed.width / 8, &peak) || + peak != parsed.peak_bandwidth) { + ret = -EINVAL; + goto out; + } + for (i = 0; i + 1 < parsed.state_count; i++) { + if (rates[i] >= parsed.max_rate) { + ret = -EINVAL; + goto out; + } + } + /* Publish only a complete, internally consistent description. */ + *dram = parsed; +out: + fwnode_handle_put(ref.fwnode); + return ret; +bad_opp: + fwnode_handle_put(opp); + ret = -EINVAL; + goto out; +} + +#endif diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index aaa38c06daaaf..1641181d57ed7 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -32,6 +32,7 @@ #include #include "pmp-report-validation.h" +#include "pmp-dram.h" #define PMP_REPORT_READY 0x1 @@ -81,6 +82,7 @@ struct apple_pmp_start; struct apple_pmp_report { struct device *dev; + struct apple_pmp_dram dram; const struct apple_pmp_report_offsets *offsets; void __iomem *base; /* Protects the request word, @shadow, @known and @defer. */ @@ -105,6 +107,43 @@ static bool apple_pmp_report_running(const struct apple_pmp_report *rep) return rep->offsets->running_any ? status : status & PMP_REPORT_READY; } +static ssize_t type_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct apple_pmp_report *rep = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%s\n", rep->dram.type); +} +static DEVICE_ATTR_RO(type); + +#define APPLE_PMP_DRAM_ATTR(_name, _field, _format) \ +static ssize_t _name##_show(struct device *dev, struct device_attribute *attr, char *buf) \ +{ \ + struct apple_pmp_report *rep = dev_get_drvdata(dev); \ + return sysfs_emit(buf, _format "\n", rep->dram._field); \ +} \ +static DEVICE_ATTR_RO(_name) + +APPLE_PMP_DRAM_ATTR(data_bus_width, width, "%u"); +APPLE_PMP_DRAM_ATTR(state_count, state_count, "%u"); +APPLE_PMP_DRAM_ATTR(max_data_rate, max_rate, "%llu"); +APPLE_PMP_DRAM_ATTR(peak_bandwidth, peak_bandwidth, "%llu"); +APPLE_PMP_DRAM_ATTR(bandwidth_class, bandwidth_class, "%llu"); + +static struct attribute *apple_pmp_dram_attrs[] = { + &dev_attr_type.attr, + &dev_attr_data_bus_width.attr, + &dev_attr_state_count.attr, + &dev_attr_max_data_rate.attr, + &dev_attr_peak_bandwidth.attr, + &dev_attr_bandwidth_class.attr, + NULL, +}; + +static const struct attribute_group apple_pmp_dram_group = { + .name = "dram", + .attrs = apple_pmp_dram_attrs, +}; + static const struct apple_pmp_report_offsets apple_pmp_offsets_t6030; /* Bound report regions, protected by RCU until managed teardown drains readers. */ @@ -1147,6 +1186,15 @@ static int apple_pmp_report_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&rep->ready_work, apple_pmp_report_ready_work); platform_set_drvdata(pdev, rep); + ret = apple_pmp_dram_parse(dev_fwnode(dev), &rep->dram); + if (ret && ret != -ENODATA) + return dev_err_probe(dev, ret, "invalid DRAM description\n"); + if (!ret) { + ret = devm_device_add_group(dev, &apple_pmp_dram_group); + if (ret) + return ret; + } + INIT_LIST_HEAD(&rep->list); ret = devm_add_action_or_reset(dev, apple_pmp_report_release, rep); if (ret) From ec8e993e852e8f056897e4577e9ec1c77171e35e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:16:36 +1100 Subject: [PATCH 1431/1579] arm64: dts: apple: describe DRAM maximum rates and bandwidth classes Add the observed LPDDR5 family and DCS state counts for T6000, T8112, T6040/T6041 and T8140. Encode the inferred widths and maximum interface rates separately from the published bandwidth classes. Lower-state rates and unaccepted achieved-state registers remain absent. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6000.dtsi | 39 +++++++++++++++++++++++ arch/arm64/boot/dts/apple/t6040.dtsi | 47 ++++++++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8112.dtsi | 43 +++++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8140.dtsi | 43 +++++++++++++++++++++++++ 4 files changed, 172 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index db7ea6eac5a6f..fc1c92bc0ea8f 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -39,3 +39,42 @@ &gpu { compatible = "apple,agx-t6000", "apple,agx-g13x"; }; + +/* + * DRAM interface class: width and maximum rate are inferred from topology + * and the published bandwidth class. Lower-state rates remain unqualified. + * State IDs below describe firmware DCS states, not PMGR codes. + */ +/ { + dram_opp: opp-table-dram { + compatible = "operating-points-v2-apple-dram"; + opp-shared; + + opp-0 { + opp-level = <0>; + }; + + opp-1 { + opp-level = <1>; + }; + + opp-2 { + opp-level = <2>; + }; + + opp-3 { + opp-level = <3>; + opp-hz = /bits/ 64 <3200000000>; + apple,data-rate = /bits/ 64 <6400000000>; + }; + }; +}; + +&pmp_report { + operating-points-v2 = <&dram_opp>; + apple,dram-type = "lpddr5"; + apple,data-bus-width = <256>; + apple,dcs-state-count = <4>; + apple,peak-bandwidth-bytes-per-second = /bits/ 64 <204800000000>; + apple,bandwidth-class-bytes-per-second = /bits/ 64 <200000000000>; +}; diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 15a3c2965a960..8a5abbb0fcbfe 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -2290,3 +2290,50 @@ }; #include "t6040-pmgr.dtsi" + +/* + * DRAM interface class: width and maximum rate are inferred from topology + * and the published bandwidth class. Lower-state rates remain unqualified. + * State IDs below describe firmware DCS states, not PMGR codes. + */ +/ { + dram_opp: opp-table-dram { + compatible = "operating-points-v2-apple-dram"; + opp-shared; + + opp-0 { + opp-level = <0>; + }; + + opp-1 { + opp-level = <1>; + }; + + opp-2 { + opp-level = <2>; + }; + + opp-3 { + opp-level = <3>; + }; + + opp-4 { + opp-level = <4>; + }; + + opp-5 { + opp-level = <5>; + opp-hz = /bits/ 64 <4266666667>; + apple,data-rate = /bits/ 64 <8533333333>; + }; + }; +}; + +&pmp_report { + operating-points-v2 = <&dram_opp>; + apple,dram-type = "lpddr5"; + apple,data-bus-width = <256>; + apple,dcs-state-count = <6>; + apple,peak-bandwidth-bytes-per-second = /bits/ 64 <273066666656>; + apple,bandwidth-class-bytes-per-second = /bits/ 64 <273000000000>; +}; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 90f9311d4aede..c97093436f865 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -2522,3 +2522,46 @@ #include "t8112-pmgr.dtsi" #include "hwmon-common.dtsi" + +/* + * DRAM interface class: width and maximum rate are inferred from topology + * and the published bandwidth class. Lower-state rates remain unqualified. + * State IDs below describe firmware DCS states, not PMGR codes. + */ +/ { + dram_opp: opp-table-dram { + compatible = "operating-points-v2-apple-dram"; + opp-shared; + + opp-0 { + opp-level = <0>; + }; + + opp-1 { + opp-level = <1>; + }; + + opp-2 { + opp-level = <2>; + }; + + opp-3 { + opp-level = <3>; + }; + + opp-4 { + opp-level = <4>; + opp-hz = /bits/ 64 <3200000000>; + apple,data-rate = /bits/ 64 <6400000000>; + }; + }; +}; + +&pmp_report { + operating-points-v2 = <&dram_opp>; + apple,dram-type = "lpddr5"; + apple,data-bus-width = <128>; + apple,dcs-state-count = <5>; + apple,peak-bandwidth-bytes-per-second = /bits/ 64 <102400000000>; + apple,bandwidth-class-bytes-per-second = /bits/ 64 <100000000000>; +}; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 9fdd91a302412..450470726c3e6 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1778,3 +1778,46 @@ }; #include "t8140-pmgr.dtsi" + +/* + * DRAM interface class: width and maximum rate are inferred from topology + * and the published bandwidth class. Lower-state rates remain unqualified. + * State IDs below describe firmware DCS states, not PMGR codes. + */ +/ { + dram_opp: opp-table-dram { + compatible = "operating-points-v2-apple-dram"; + opp-shared; + + opp-0 { + opp-level = <0>; + }; + + opp-1 { + opp-level = <1>; + }; + + opp-2 { + opp-level = <2>; + }; + + opp-3 { + opp-level = <3>; + }; + + opp-4 { + opp-level = <4>; + opp-hz = /bits/ 64 <3750000000>; + apple,data-rate = /bits/ 64 <7500000000>; + }; + }; +}; + +&pmp_report { + operating-points-v2 = <&dram_opp>; + apple,dram-type = "lpddr5"; + apple,data-bus-width = <64>; + apple,dcs-state-count = <5>; + apple,peak-bandwidth-bytes-per-second = /bits/ 64 <60000000000>; + apple,bandwidth-class-bytes-per-second = /bits/ 64 <60000000000>; +}; From 0714fc8edfee56f89bddda2aa2a8eef6d5a4fb6d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:23:44 +1100 Subject: [PATCH 1432/1579] MAINTAINERS: cover Apple PMP DRAM reporting documentation Keep the new DRAM ABI and OPP binding under the existing Apple machine support entry alongside the PMP report driver. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 75dbf0ccaf520..b1004b10201ff 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2708,6 +2708,7 @@ B: https://github.com/AsahiLinux/linux/issues C: irc://irc.oftc.net/asahi-dev T: git https://github.com/AsahiLinux/linux.git F: Documentation/ABI/testing/sysfs-driver-apple-dart +F: Documentation/ABI/testing/sysfs-driver-apple-pmp-dram F: Documentation/devicetree/bindings/arm/apple.yaml F: Documentation/devicetree/bindings/arm/apple/* F: Documentation/devicetree/bindings/clock/apple,nco.yaml @@ -2730,6 +2731,7 @@ F: Documentation/devicetree/bindings/npu/apple,ane.yaml F: Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml F: Documentation/devicetree/bindings/nvmem/apple,efuses.yaml F: Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml +F: Documentation/devicetree/bindings/opp/operating-points-v2-apple-dram.yaml F: Documentation/devicetree/bindings/pci/apple,pcie.yaml F: Documentation/devicetree/bindings/phy/apple,atcphy.yaml F: Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml From 6ca0e463707f6084acaba86d930197f550ae1698 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:32:40 +1100 Subject: [PATCH 1433/1579] dmaengine: apple-admac: release the power domain without channel users Hold a runtime PM reference from channel resource allocation through stop, synchronization and SRAM release. This covers the atomic DMA callbacks and configuration accesses as well as active transfers. Drain the dedicated interrupt before the last reference allows genpd to remove power. Reprogram controller and channel state through the existing allocation and configuration paths before the next transfer. Invalidate the one-time CTL initialization when a DT-described power domain may have lost state. Keep MMIO out of the PM callbacks: firmware-managed controllers are inaccessible until the audio client completes its power handshake. Do not reset shared firmware state or touch unallocated channels. Keep system sleep behavior unchanged rather than force-suspending allocated cyclic channels. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/dma/apple-admac.c | 64 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index a24bd66c86d81..90d51641a6a35 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -680,6 +681,11 @@ static int admac_alloc_chan_resources(struct dma_chan *chan) struct admac_data *ad = adchan->host; int ret; + /* Keep MMIO accessible for config, status and the atomic DMA callbacks. */ + ret = pm_runtime_resume_and_get(ad->dev); + if (ret < 0) + return ret; + dma_cookie_init(&adchan->chan); /* @@ -699,8 +705,10 @@ static int admac_alloc_chan_resources(struct dma_chan *chan) ret = admac_alloc_sram_carveout(ad, admac_chan_direction(adchan->no), &adchan->carveout); - if (ret < 0) + if (ret < 0) { + pm_runtime_put_autosuspend(ad->dev); return ret; + } writel_relaxed(adchan->carveout, ad->base + REG_CHAN_SRAM_CARVEOUT(ad, adchan->no)); @@ -715,6 +723,7 @@ static void admac_free_chan_resources(struct dma_chan *chan) admac_synchronize(chan); admac_free_sram_carveout(adchan->host, admac_chan_direction(adchan->no), adchan->carveout); + pm_runtime_put_autosuspend(adchan->host->dev); } static struct dma_chan *admac_dma_of_xlate(struct of_phandle_args *dma_spec, @@ -976,6 +985,37 @@ static int admac_device_config(struct dma_chan *chan, return 0; } +static int admac_runtime_suspend(struct device *dev) +{ + struct admac_data *ad = dev_get_drvdata(dev); + + /* No channel resources remain; drain handlers before genpd gates MMIO. */ + disable_irq(ad->irq); + if (dev->pm_domain) + ad->ctl_programmed = false; + + return 0; +} + +static int admac_runtime_resume(struct device *dev) +{ + struct admac_data *ad = dev_get_drvdata(dev); + + /* + * Defer register restoration to alloc/config, before any transfer. Some + * controllers are powered by firmware only after the audio handshake. + * Restore only the requested channel, never reset the shared fabric. + */ + enable_irq(ad->irq); + + return 0; +} + +/* Allocated channels must stay powered, including paused cyclic transfers. */ +static const struct dev_pm_ops admac_pm_ops = { + RUNTIME_PM_OPS(admac_runtime_suspend, admac_runtime_resume, NULL) +}; + static int admac_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; @@ -1093,24 +1133,37 @@ static int admac_probe(struct platform_device *pdev) goto free_reset; } + pm_runtime_get_noresume(ad->dev); + err = pm_runtime_set_active(ad->dev); + if (err) + goto put_pm; + pm_runtime_set_autosuspend_delay(ad->dev, 100); + pm_runtime_use_autosuspend(ad->dev); + pm_runtime_enable(ad->dev); + err = dma_async_device_register(&ad->dma); if (err) { dev_err_probe(&pdev->dev, err, "failed to register DMA device\n"); - goto free_irq; + goto disable_pm; } err = of_dma_controller_register(pdev->dev.of_node, admac_dma_of_xlate, ad); if (err) { dma_async_device_unregister(&ad->dma); dev_err_probe(&pdev->dev, err, "failed to register with OF\n"); - goto free_irq; + goto disable_pm; } dev_info(&pdev->dev, "Audio DMA Controller\n"); + pm_runtime_put_autosuspend(ad->dev); return 0; -free_irq: +disable_pm: + pm_runtime_disable(ad->dev); + pm_runtime_dont_use_autosuspend(ad->dev); +put_pm: + pm_runtime_put_noidle(ad->dev); free_irq(ad->irq, ad); free_reset: reset_control_rearm(ad->rstc); @@ -1123,6 +1176,8 @@ static void admac_remove(struct platform_device *pdev) of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&ad->dma); + pm_runtime_disable(ad->dev); + pm_runtime_dont_use_autosuspend(ad->dev); free_irq(ad->irq, ad); reset_control_rearm(ad->rstc); } @@ -1163,6 +1218,7 @@ static struct platform_driver apple_admac_driver = { .driver = { .name = "apple-admac", .of_match_table = admac_of_match, + .pm = pm_ptr(&admac_pm_ops), }, .probe = admac_probe, .remove = admac_remove, From ba7b77683de2472099c750dbd2f4a7876ab9497d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:33:30 +1100 Subject: [PATCH 1434/1579] PCI: apple: select PCIe-C initialization by match data and DT The existing PCIe-C implementation matches T8103 and T6000, but T8112 falls through to T8103 despite having Intr2AXI instead of oe-fabric. ACIO can then admit a port whose T8110 DART has no tunables, or leave PCIe-C disabled because no bootloader preinit handoff exists. Reuse the existing tunnel cold-init, reset, IRQ and hotplug sequences. Give T8112 and T6000 shared Intr2AXI match data, require the documented apple,pciec-kernel-init property, and make ACIO use the same admission helper as the host driver. Remove the cold-init module parameter and machine allowlist. Retain the firmware-handoff and T8103 oe-fabric paths. The existing implementation and original authorship remain in history, including Chris Kearney's T8103 support (43a6da03266fc5) and Wes Grimes' T600x/T602x cold initialization (eb20efd9dd4a3c). Host tests cover the M2 fallback regression, missing DART tunables, DT opt-in, firmware handoff, and original M1 behavior using production C. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/pci/apple,pcie.yaml | 6 +- drivers/pci/controller/pcie-apple.c | 130 ++++++------- drivers/thunderbolt/apple.c | 6 +- tools/asahi/pciec-policy-tests.py | 176 ++++++++++++++++++ tools/asahi/run-host-tests.py | 1 + 5 files changed, 248 insertions(+), 71 deletions(-) create mode 100644 tools/asahi/pciec-policy-tests.py diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 02120d1934339..921624dfb9d7a 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -123,7 +123,11 @@ properties: apple,pciec-kernel-init: type: boolean - description: The kernel initializes the tunneled controller from reset. + description: | + The kernel initializes the tunneled controller when firmware has not + already initialized it. The compatible selects the initialization + sequence. T600x and T8112 use the Intr2AXI bridge and require tunables + for their associated DART before the tunnel children can be populated. apple,tunable-rc: $ref: '#/$defs/tunable' diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index e0b6ead2526e2..b674e84c07acf 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -68,57 +68,6 @@ static int link_up_timeout = 500; module_param(link_up_timeout, int, 0644); MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); -static bool tunnel_kernel_init = true; -module_param(tunnel_kernel_init, bool, 0444); -MODULE_PARM_DESC(tunnel_kernel_init, - "Cold-initialize T600x/T602x PCIe-C ports without an m1n1 handoff (default: on)"); - -static const char *const apple_pcie_tunnel_cold_init_machines[] = { - "apple,t6000", "apple,t6001", "apple,t6020", "apple,t6021", NULL -}; - -/** - * apple_pcie_tunnel_needs_cold_init() - whether the kernel brings up a port - * @np: PCIe-C host node - * - * No m1n1 performs the PCIe-C preinit handoff on T600x/T602x, so the kernel - * cold-initializes these ports instead; pcie_apple.tunnel_kernel_init=0 turns - * that off. It is a parameter rather than a DT property because m1n1 hands - * every installed kernel the same DT. The tunnel's DART cannot probe without - * its tunables, which m1n1 does not provide, so ports whose DART lacks them - * are refused. - * - * Unlike "apple,pciec-kernel-init" (t8103), this does not select the t8103 - * power and resume sequences. - */ -bool apple_pcie_tunnel_needs_cold_init(struct device_node *np) -{ - struct of_phandle_args iommu_spec = {}; - struct device_node *dart; - bool tunables; - u32 status; - - if (!tunnel_kernel_init || - !of_device_is_compatible(np, "apple,t6000-pciec") || - !of_machine_compatible_match(apple_pcie_tunnel_cold_init_machines)) - return false; - - if (!of_property_read_u32(np, "apple,pciec-preinit-status", &status) && - status == 1) - return false; - - if (of_map_iommu_id(np, 0, &iommu_spec) || !iommu_spec.np) - return false; - dart = iommu_spec.np; - tunables = of_property_present(dart, "apple,tunable"); - if (!tunables) - pr_warn_once("%pOF: no PCIe-C DART tunables, not cold-initializing\n", - dart); - of_node_put(dart); - return tunables; -} -EXPORT_SYMBOL_GPL(apple_pcie_tunnel_needs_cold_init); - /* T8103 (original M1) and related SoCs */ #define CORE_RC_PHYIF_CTL 0x00024 #define CORE_RC_PHYIF_CTL_RUN BIT(0) @@ -250,6 +199,7 @@ struct hw_info { bool retain_clocks; bool root_bus_only; bool tunneled; + bool tunnel_intr2axi; }; static const struct hw_info t8103_hw = { @@ -301,6 +251,16 @@ static const struct hw_info t8103_pciec_hw = { .tunneled = true, }; +/* T600x and T8112 use Intr2AXI instead of the T8103 oe-fabric. */ +static const struct hw_info t8112_pciec_hw = { + .port_msiaddr = PORT_MSIADDR, + .port_perst = PORT_PERST, + .port_rid2sid = PORT_RID2SID, + .max_rid2sid = 64, + .tunneled = true, + .tunnel_intr2axi = true, +}; + /* * t8122 PCIe-C keeps the t8103 doorbell and RID2SID registers, but MSI * delivery uses the t602x map. The map is indexed by the MSI data, and @@ -315,6 +275,46 @@ static const struct hw_info t8122_pciec_hw = { .tunneled = true, }; +static const struct of_device_id apple_pcie_of_match[]; + +/** + * apple_pcie_tunnel_needs_cold_init() - whether the kernel brings up a port + * @np: PCIe-C host node + * + * DT selects kernel initialization; match data selects the hardware sequence. + * Never reset a qualified firmware handoff. Intr2AXI controllers also need + * DART tunables before ACIO may populate their cable-powered children. + */ +bool apple_pcie_tunnel_needs_cold_init(struct device_node *np) +{ + const struct of_device_id *match = of_match_node(apple_pcie_of_match, np); + struct of_phandle_args iommu_spec = {}; + const struct hw_info *hw; + bool tunables; + u32 status; + + if (!match || !of_property_read_bool(np, "apple,pciec-kernel-init")) + return false; + hw = match->data; + if (!hw->tunneled) + return false; + if (!of_property_read_u32(np, "apple,pciec-preinit-status", &status) && + status == 1) + return false; + if (!hw->tunnel_intr2axi) + return true; + + if (of_map_iommu_id(np, 0, &iommu_spec) || !iommu_spec.np) + return false; + tunables = of_property_present(iommu_spec.np, "apple,tunable"); + if (!tunables) + pr_warn_once("%pOF: no PCIe-C DART tunables, not cold-initializing\n", + iommu_spec.np); + of_node_put(iommu_spec.np); + return tunables; +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_needs_cold_init); + struct apple_pcie { struct mutex lock; struct device *dev; @@ -991,9 +991,9 @@ static int apple_pcie_tunnel_reinitialize(struct apple_pcie_port *port) } /* - * Bring up a port that no firmware handoff left running (t8103, or - * T600x/T602x with pcie_apple.tunnel_kernel_init): tunables and the reset - * table, then PERST and APPCLK, and only then the root-port config-space + * Bring up a port selected by apple,pciec-kernel-init that no firmware + * handoff left running: tunables and the reset table, then PERST and APPCLK, + * and only then the root-port config-space * tunables. Stop before releasing the tunnel reset or starting link * training. */ @@ -1668,8 +1668,8 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, * m1n1 owns PCIe-C cold initialization on T6020. Replaying the * port reset against that live handoff raises an asynchronous * SError. t8103 boots with ATC_PCIE off and no handoff, so - * apple,pciec-kernel-init selects the in-kernel sequence, as - * pcie_apple.tunnel_kernel_init does on T600x/T602x. + * apple,pciec-kernel-init selects the in-kernel sequence on each + * supported generation. */ ret = of_property_read_u32(pcie->dev->of_node, "apple,pciec-preinit-status", @@ -1688,7 +1688,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, } } else if (!preinit_ok) { dev_err(pcie->dev, - "PCIe-C requires an m1n1 preinit handoff or pcie_apple.tunnel_kernel_init=1\n"); + "PCIe-C requires a firmware handoff or apple,pciec-kernel-init\n"); ret = -ENODEV; goto err_teardown; } else { @@ -2889,9 +2889,9 @@ static int apple_pcie_probe(struct platform_device *pdev) return PTR_ERR(pcie->base); if (pcie->hw->tunneled) { pcie->kernel_init = - of_device_is_compatible(dev->of_node, "apple,t8103-pciec") && + !hw->tunnel_intr2axi && of_property_read_bool(dev->of_node, "apple,pciec-kernel-init"); - pcie->tunnel_cold_init = + pcie->tunnel_cold_init = hw->tunnel_intr2axi && apple_pcie_tunnel_needs_cold_init(dev->of_node); ret = apple_pcie_tunnel_init_resources(pdev, pcie); if (ret) @@ -3288,12 +3288,11 @@ static struct apple_tunable *apple_pcie_tunable_once(struct device_node *np, * probes immediately and its invalidate command never completes while * APPCLK is still gated. */ -static const struct of_device_id apple_pcie_of_match[]; - int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) { struct device_node *np, *port_np = NULL; const struct of_device_id *match; + const struct hw_info *hw; struct apple_pcie *pcie; struct apple_pcie_port *port; struct apple_pcie_map config = {}, debug = {}, fabric = {}, portmap = {}; @@ -3304,9 +3303,7 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) np = NULL; for_each_available_child_of_node(tunnel, np) { - kernel_init = of_device_is_compatible(np, "apple,t8103-pciec") && - of_property_read_bool(np, "apple,pciec-kernel-init"); - if (kernel_init || apple_pcie_tunnel_needs_cold_init(np)) + if (apple_pcie_tunnel_needs_cold_init(np)) break; } if (!np) @@ -3324,6 +3321,8 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) ret = -ENODEV; goto out_np; } + hw = match->data; + kernel_init = !hw->tunnel_intr2axi; ret = apple_pcie_map_named(np, "port0", &portmap); if (ret) @@ -3720,8 +3719,9 @@ static const struct dev_pm_ops apple_pcie_pm_ops = { static const struct of_device_id apple_pcie_of_match[] = { { .compatible = "apple,t8122-pciec", .data = &t8122_pciec_hw }, + { .compatible = "apple,t8112-pciec", .data = &t8112_pciec_hw }, { .compatible = "apple,t8103-pciec", .data = &t8103_pciec_hw }, - { .compatible = "apple,t6000-pciec", .data = &t8103_pciec_hw }, + { .compatible = "apple,t6000-pciec", .data = &t8112_pciec_hw }, /* T8132 currently inherits controller and PHY setup from m1n1. */ { .compatible = "apple,t8132-pcie", .data = &t602x_hw }, { .compatible = "apple,t6020-pcie", .data = &t602x_hw }, diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index c6625c5cbb5d3..52353e5d9059b 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -855,11 +855,7 @@ apple_cio_pcie_tunnel_is_preinitialized(struct device_node *parent) if (!of_property_read_u32(child, "apple,pciec-preinit-status", &status) && status == 1) return true; - /* t8103 brings the port up itself. */ - if (of_device_is_compatible(child, "apple,t8103-pciec") && - of_property_read_bool(child, "apple,pciec-kernel-init")) - return true; - /* T600x/T602x without a handoff, when opted in. */ + /* Use the PCI driver's generation and DT admission checks. */ return apple_pcie_tunnel_needs_cold_init(child); } diff --git a/tools/asahi/pciec-policy-tests.py b/tools/asahi/pciec-policy-tests.py new file mode 100644 index 0000000000000..825caefac0912 --- /dev/null +++ b/tools/asahi/pciec-policy-tests.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise PCIe-C cold-init admission and ACIO admission using production C.""" +import argparse +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] + + +def function(source, name): + start = re.search(r'^(?:static\s+)?bool\s+' + name + r'\(', source, re.M).start() + end = source.index('{', start) + 1 + depth = 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +FIXTURE = r''' +#include +#include +#include +#include +#include +typedef uint32_t u32; +#define BIT(n) (1U << (n)) +#define GENMASK(h, l) ((~0U << (l)) & (~0U >> (31 - (h)))) +struct device_node { + const char *compat[3]; + bool kernel_init, tunables; + int status, refs; + struct device_node *dart, *child; +}; +struct of_device_id { const char *compatible; const void *data; }; +struct of_phandle_args { struct device_node *np; }; +static int maps; +static bool of_device_is_compatible(struct device_node *np, const char *compat) +{ + for (int i = 0; np && i < 3 && np->compat[i]; i++) + if (!strcmp(np->compat[i], compat)) + return true; + return false; +} +static const struct of_device_id *of_match_node(const struct of_device_id *ids, + struct device_node *np) +{ + for (int i = 0; np && i < 3 && np->compat[i]; i++) + for (const struct of_device_id *id = ids; id->compatible; id++) + if (!strcmp(np->compat[i], id->compatible)) + return id; + return NULL; +} +static bool of_property_read_bool(struct device_node *np, const char *name) +{ + assert(!strcmp(name, "apple,pciec-kernel-init")); + return np->kernel_init; +} +static bool of_property_present(struct device_node *np, const char *name) +{ + assert(!strcmp(name, "apple,tunable")); + return np->tunables; +} +static int of_property_read_u32(struct device_node *np, const char *name, u32 *v) +{ + assert(!strcmp(name, "apple,pciec-preinit-status")); + if (np->status < 0) + return -1; + *v = np->status; + return 0; +} +static int of_map_iommu_id(struct device_node *np, int rid, struct of_phandle_args *a) +{ + assert(rid == 0); + maps++; + if (!np->dart) + return -1; + a->np = np->dart; + a->np->refs++; + return 0; +} +static void of_node_put(struct device_node *np) { assert(np->refs > 0); np->refs--; } +#define pr_warn_once(...) ((void)0) +#define for_each_available_child_of_node_scoped(parent, child) \ + for (struct device_node *child = (parent)->child; child; child = NULL) +/* PRODUCTION */ +static void expect(struct device_node *host, bool cold, bool admitted) +{ + struct device_node parent = { .child = host }; + maps = 0; + assert(apple_pcie_tunnel_needs_cold_init(host) == cold); + assert(apple_cio_pcie_tunnel_is_preinitialized(&parent) == admitted); + assert(!host->dart || host->dart->refs == 0); + if (!host->kernel_init || host->status == 1) + assert(maps == 0); +} +int main(void) +{ + struct device_node dart = { .tunables = true }; + const char *intr2axi[] = { "apple,t8112-pciec", "apple,t6000-pciec" }; + for (unsigned int i = 0; i < 2; i++) { + /* M2 must win over its T8103 fallback, without a board allowlist. */ + struct device_node host = { .compat = { intr2axi[i], "apple,t8103-pciec" }, + .status = -1, .dart = &dart }; + const struct of_device_id *id = of_match_node(apple_pcie_of_match, &host); + assert(id && ((const struct hw_info *)id->data)->tunnel_intr2axi); + expect(&host, false, false); /* No implicit cold reset. */ + host.kernel_init = true; + expect(&host, true, true); + dart.tunables = false; + expect(&host, false, false); /* Do not let the fallback bypass DART admission. */ + host.status = 1; + expect(&host, false, true); /* Preserve a firmware handoff. */ + host.status = 0; + expect(&host, false, false); + dart.tunables = true; + expect(&host, true, true); /* Failed/missing handoff can be initialized. */ + host.dart = NULL; + expect(&host, false, false); + } + struct device_node m1 = { .compat = { "apple,t8103-pciec" }, .status = -1 }; + expect(&m1, false, false); + m1.kernel_init = true; + expect(&m1, true, true); /* Original M1 does not need T8110 DART tunables. */ + m1.status = 1; + expect(&m1, false, true); + struct device_node regular = { .compat = { "apple,pcie" }, .kernel_init = true, + .status = -1 }; + expect(®ular, false, false); + regular.compat[0] = "apple,unknown-pciec"; + expect(®ular, false, false); + puts("PASS: M1, M1 Pro, M2, firmware handoff, missing DART and DT opt-in"); + return 0; +} +''' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--out', type=Path) + args = parser.parse_args() + pci = (ROOT / 'drivers/pci/controller/pcie-apple.c').read_text() + acio = (ROOT / 'drivers/thunderbolt/apple.c').read_text() + defines = '\n'.join(re.findall(r'^#define .*$', pci, re.M)) + info = pci[pci.index('struct hw_info {'):pci.index('static const struct of_device_id apple_pcie_of_match[];')] + matches = re.search(r'static const struct of_device_id apple_pcie_of_match\[\] = \{.*?\n\};', pci, re.S)[0] + production = '\n'.join((defines, info, matches, + function(pci, 'apple_pcie_tunnel_needs_cold_init'), + function(acio, 'apple_cio_pcie_tunnel_is_preinitialized'))) + code = FIXTURE.replace('/* PRODUCTION */', production) + with tempfile.TemporaryDirectory(prefix='pciec-policy-') as tmp: + out = args.out or Path(tmp) + out.mkdir(parents=True, exist_ok=True) + cases = {'production': code, + 'm2-fallback': code.replace('{ .compatible = "apple,t8112-pciec",\t.data = &t8112_pciec_hw },', ''), + 'dart-bypass': code.replace('tunables = of_property_present(iommu_spec.np, "apple,tunable");', 'tunables = true;')} + for name, body in cases.items(): + source = out / (name + '.c') + source.write_text(body) + binary = out / name + subprocess.run(['cc', '-std=gnu11', '-Wall', '-Werror', '-Wno-unused-function', str(source), '-o', str(binary)], check=True) + run = subprocess.run([str(binary)], capture_output=True, text=True) + (out / (name + '.log')).write_text(run.stdout + run.stderr) + if name == 'production': + assert run.returncode == 0, run.stderr + print(run.stdout, end='') + else: + assert run.returncode < 0 and 'Assertion' in run.stderr, (name, run.returncode, run.stderr) + print('PASS negative control:', name) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index bcc42fe042c0f..028816f12be4b 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -114,6 +114,7 @@ def mutations(flag, names, base, expect=KILL): + mutations('--mutation', ('sep-syscore', 'wake-irq', 'rx-suspend', 'stale-policy', 'usb-rpm', 'vlp-client', 'six-band'), out('--output')) + [(out('--output') + ['--old-usb'], KILL), (out('--output') + ['--old-regulatory'], KILL)], + 'pciec-policy-tests.py': [(out(), PASS)], 'pmp-j613-hwmon-tests.py': [([], PASS)], 'qspi-nvram-tests.py': [(out('--output'), PASS)], 'rtkit-oslog-tests.py': [(out(), PASS)] From 18533fba66c746252418ddbe2c6860e00bf054b9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:33:33 +1100 Subject: [PATCH 1435/1579] arm64: dts: apple: opt T600x and T8112 PCIe-C into kernel init Select the existing kernel cold-init sequence for T600x and T8112 PCIe-C root ports. Both SoCs already have the tunnel containers and ACIO phandles. Keep those nodes cable-controlled: the Thunderbolt core populates their DART and host only when the PCIe adapters are enabled. Keep the published T600x DART tunables includes for ports 0 through 2. Other ports must receive their own DART tunables from the bootloader; the host refuses cold init without them. No other SoC's calibration is substituted. T8112 needs the companion m1n1 runtime DART tunables handoff. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t600x-pciec.dtsi | 4 ++++ arch/arm64/boot/dts/apple/t8112.dtsi | 2 ++ 2 files changed, 6 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-pciec.dtsi b/arch/arm64/boot/dts/apple/t600x-pciec.dtsi index 808048a7fcece..c58c66eda9bc9 100644 --- a/arch/arm64/boot/dts/apple/t600x-pciec.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pciec.dtsi @@ -22,6 +22,7 @@ apciec0: pcie@730000000 { compatible = "apple,t6000-pciec"; + apple,pciec-kernel-init; device_type = "pci"; reg = <0x7 0x30000000 0x0 0x10000000>, @@ -92,6 +93,7 @@ apciec1: pcie@b30000000 { compatible = "apple,t6000-pciec"; + apple,pciec-kernel-init; device_type = "pci"; reg = <0xb 0x30000000 0x0 0x10000000>, @@ -162,6 +164,7 @@ apciec2: pcie@f30000000 { compatible = "apple,t6000-pciec"; + apple,pciec-kernel-init; device_type = "pci"; reg = <0xf 0x30000000 0x0 0x10000000>, @@ -231,6 +234,7 @@ apciec3: pcie@1330000000 { compatible = "apple,t6000-pciec"; + apple,pciec-kernel-init; device_type = "pci"; reg = <0x13 0x30000000 0x0 0x10000000>, diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 90f9311d4aede..d9e6e9ae66b0d 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -2023,6 +2023,7 @@ apciec0: pcie@3b0000000 { compatible = "apple,t8112-pciec", "apple,t8103-pciec"; + apple,pciec-kernel-init; device_type = "pci"; reg = <0x3 0xb0000000 0x0 0x10000000>, @@ -2243,6 +2244,7 @@ apciec1: pcie@530000000 { compatible = "apple,t8112-pciec", "apple,t8103-pciec"; + apple,pciec-kernel-init; device_type = "pci"; reg = <0x5 0x30000000 0x0 0x10000000>, From 7d154821ed5afc0b027df74792166cad684bed17 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:01:15 +1100 Subject: [PATCH 1436/1579] media: apple-ave: refuse system sleep when firmware cannot halt A failed firmware shutdown leaves the encoder potentially issuing DMA. Returning success lets genpd power down its domains and DARTs anyway. Propagate the shutdown error so device PM aborts sleep before that boundary. Restart the m2m scheduler on this failure because PM does not call resume for the device that failed suspend. Closed sessions already reopen on their next frame. Exercise the production callbacks with halt failures, repeated cycles and rollback; the pre-fix callback fails the runtime negative control. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/ave-drv.c | 15 ++- tools/asahi/ave-pm-tests.py | 116 +++++++++++++++++++++ tools/asahi/run-host-tests.py | 2 + 3 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 tools/asahi/ave-pm-tests.py diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c index b25ac9df48a67..eec887796cf99 100644 --- a/drivers/media/platform/apple/ave/ave-drv.c +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -170,6 +170,7 @@ static int ave_runtime_resume(struct device *dev) static int ave_suspend(struct device *dev) { struct ave_dev *ave = dev_get_drvdata(dev); + int ret; /* Let a running job finish; the m2m core holds back new ones. */ v4l2_m2m_suspend(ave->m2m_dev); @@ -181,10 +182,18 @@ static int ave_suspend(struct device *dev) */ mutex_lock(&ave->hw_lock); ave_v4l2_suspend(ave); - if (ave_fw_shutdown(ave)) - dev_warn(dev, "firmware not answering, it cannot be stopped for sleep\n"); + ret = ave_fw_shutdown(ave); mutex_unlock(&ave->hw_lock); - return 0; + if (ret) { + /* + * genpd may gate the encoder and its DARTs after this callback. + * A stuck firmware may still DMA, so abort before that happens. + * PM does not resume a device whose suspend callback failed. + */ + dev_err(dev, "firmware cannot be stopped for sleep: %d\n", ret); + v4l2_m2m_resume(ave->m2m_dev); + } + return ret; } static int ave_resume(struct device *dev) diff --git a/tools/asahi/ave-pm-tests.py b/tools/asahi/ave-pm-tests.py new file mode 100644 index 0000000000000..0a86cae413d28 --- /dev/null +++ b/tools/asahi/ave-pm-tests.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise the AVE sleep callbacks with a firmware halt failure. + +Compile the production callbacks against a simulated m2m scheduler and firmware. +No encoder registers, firmware or media device are needed. The old callback is a +negative control: it must fail the stuck-firmware assertion after compiling. +""" +import argparse +import os +from pathlib import Path +import resource +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = 'drivers/media/platform/apple/ave/ave-drv.c' + + +def function(source, name): + start = source.index('static int ' + name + '(') + end = source.index('\n}', start) + 2 + return source[start:end] + + +PREFIX = r''' +#include +#include +#include +#include +struct scheduler { bool stopped; unsigned pauses, restarts; }; +struct ave_dev { struct scheduler *m2m_dev; int hw_lock; }; +struct device { struct ave_dev *data; }; +static int shutdown_error; +static unsigned closes, halts; +static void *dev_get_drvdata(struct device *d) { return d->data; } +static void mutex_lock(int *l) { assert(!*l); *l = 1; } +static void mutex_unlock(int *l) { assert(*l == 1); *l = 0; } +static void v4l2_m2m_suspend(struct scheduler *s) { + assert(!s->stopped); s->stopped = true; s->pauses++; +} +static void v4l2_m2m_resume(struct scheduler *s) { + assert(s->stopped); s->stopped = false; s->restarts++; +} +static void ave_v4l2_suspend(struct ave_dev *a) { + assert(a->hw_lock && a->m2m_dev->stopped); closes++; +} +static int ave_fw_shutdown(struct ave_dev *a) { + assert(a->hw_lock && a->m2m_dev->stopped && closes > halts); + halts++; return shutdown_error; +} +#define dev_warn(...) ((void)0) +#define dev_err(...) ((void)0) +''' +CASES = r''' +int main(void) { + struct scheduler s = {0}; + struct ave_dev a = {.m2m_dev = &s}; + struct device d = {.data = &a}; + for (unsigned n = 0; n < 3; n++) { + shutdown_error = 0; + assert(ave_suspend(&d) == 0); + assert(s.stopped && !a.hw_lock); + assert(ave_resume(&d) == 0); + assert(!s.stopped && !a.hw_lock); + } + /* PM does not call resume for a device whose suspend returned an error. */ + shutdown_error = -EBUSY; + assert(ave_suspend(&d) == -EBUSY); + assert(!s.stopped && !a.hw_lock); + /* An aborted sleep must not leave the scheduler stopped for the next one. */ + shutdown_error = -EIO; + assert(ave_suspend(&d) == -EIO); + assert(!s.stopped && !a.hw_lock); + shutdown_error = 0; + assert(ave_suspend(&d) == 0); + assert(s.stopped); + assert(ave_resume(&d) == 0); + assert(!s.stopped && !a.hw_lock); + assert(s.pauses == 6 && s.restarts == 6 && closes == 6 && halts == 6); + puts("AVE PM: normal/repeated sleep, halt refusal, error propagation, rollback and retry passed"); +} +''' + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--out') + p.add_argument('--old-ref', help='expect this revision to fail the runtime control') + args = p.parse_args() + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + source = ((ROOT / SOURCE).read_text() if not args.old_ref else subprocess.check_output( + ['git', 'show', args.old_ref + ':' + SOURCE], cwd=ROOT, text=True)) + with tempfile.TemporaryDirectory(prefix='ave-pm-') as tmp: + out = Path(args.out or tmp) + out.mkdir(parents=True, exist_ok=True) + c = out / 'ave-pm.c' + c.write_text(PREFIX + function(source, 'ave_suspend') + '\n' + + function(source, 'ave_resume') + CASES) + exe = out / 'ave-pm' + subprocess.run([os.environ.get('CC', 'cc'), '-std=c11', '-Wall', '-Wextra', + '-Werror', str(c), '-o', str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + (out / 'run.log').write_text(r.stdout + r.stderr) + if args.old_ref: + if r.returncode != -6 or 'Assertion' not in r.stderr: + raise SystemExit('old callback did not fail its runtime assertion') + print('AVE PM: old callback compiled and failed the halt-refusal assertion') + elif r.returncode: + raise SystemExit(r.stdout + r.stderr) + else: + print(r.stdout, end='') + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index bcc42fe042c0f..e7e60bad2e6cd 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -50,6 +50,8 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'ave-pm-tests.py': [(out(), PASS), + (out() + ['--old-ref', '4db3a5a22c43'], PASS)], 'apple-dart-fw-tests.py': [(out(), PASS)], 'apple-dart-range-tests.py': [([], PASS)], 'atc-usb3-tests.py': [(out(), PASS), (out() + ['--mutations'], PASS)], From 68e9ecbe10d70a2578604ae098fa86e29a5481b8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:07:13 +1100 Subject: [PATCH 1437/1579] HID: dockchannel: notify HID drivers across system sleep DockChannel marks the transport suspended but never informs its HID drivers of sleep or resume. The magicmouse actuator cache consequently cannot learn that system sleep returned control to the device firmware. Its existing cache reset only runs on reset-resume. Propagate suspend and resume notifications with feature requests still available. Unwind successful HID suspends if a later one fails, and resume all interfaces even if one reports an error. Add a normal magicmouse resume callback that forgets the cached haptic mode without issuing any commands. Host controls execute the production callbacks through repeated sleeps, suspend failure, resume failure and the haptic state reset. The old transport compiles but fails the haptic-cache assertion. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 35 ++++- drivers/hid/hid-magicmouse.c | 10 ++ tools/asahi/mtp-pm-tests.py | 138 ++++++++++++++++++ tools/asahi/run-host-tests.py | 2 + 4 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 tools/asahi/mtp-pm-tests.py diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 11450d904fc8a..6d97f51971b29 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -1755,17 +1755,50 @@ static void dockchannel_hid_remove(struct platform_device *pdev) static int dockchannel_hid_suspend(struct device *dev) { struct dockchannel_hid *dchid = dev_get_drvdata(dev); + int i, ret; + /* HID drivers may still need the transport while preparing for sleep. */ + for (i = 0; i < MAX_INTERFACES; i++) { + struct dchid_iface *iface = dchid->ifaces[i]; + + if (!iface || !iface->hid) + continue; + ret = hid_driver_suspend(iface->hid, PMSG_SUSPEND); + if (ret) + goto unwind; + } WRITE_ONCE(dchid->suspended, true); return 0; + +unwind: + /* PM does not call our resume when this callback fails. */ + while (--i >= 0) { + struct dchid_iface *iface = dchid->ifaces[i]; + + if (iface && iface->hid) + hid_driver_resume(iface->hid); + } + return ret; } static int dockchannel_hid_resume(struct device *dev) { struct dockchannel_hid *dchid = dev_get_drvdata(dev); + int i, ret = 0; + /* Allow feature requests again before notifying any HID driver. */ WRITE_ONCE(dchid->suspended, false); - return 0; + for (i = 0; i < MAX_INTERFACES; i++) { + struct dchid_iface *iface = dchid->ifaces[i]; + int err; + + if (!iface || !iface->hid) + continue; + err = hid_driver_resume(iface->hid); + if (!ret) + ret = err; + } + return ret; } static DEFINE_SIMPLE_DEV_PM_OPS(dockchannel_hid_pm_ops, dockchannel_hid_suspend, diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 2d23290497b51..c2d96ac9dd4bd 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -2481,6 +2481,15 @@ static void magicmouse_remove(struct hid_device *hdev) } #ifdef CONFIG_PM +static int magicmouse_resume(struct hid_device *hdev) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + if (msc) + magicmouse_haptic_forget_mode(msc); + return 0; +} + static int magicmouse_reset_resume(struct hid_device *hdev) { struct magicmouse_sc *msc = hid_get_drvdata(hdev); @@ -2567,6 +2576,7 @@ static struct hid_driver magicmouse_driver = { .input_mapping = magicmouse_input_mapping, .input_configured = magicmouse_input_configured, #ifdef CONFIG_PM + .resume = magicmouse_resume, .reset_resume = magicmouse_reset_resume, #endif }; diff --git a/tools/asahi/mtp-pm-tests.py b/tools/asahi/mtp-pm-tests.py new file mode 100644 index 0000000000000..7d5b84d662ef0 --- /dev/null +++ b/tools/asahi/mtp-pm-tests.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Run production MTP PM and magicmouse resume callbacks without a device. + +Checks notification order, suspend rollback, resume-error handling and the +actuator mode cache. The old transport must compile and fail at runtime. +""" +import argparse +import os +from pathlib import Path +import resource +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +TRANSPORT = 'drivers/hid/dockchannel-hid/dockchannel-hid.c' + + +def function(source, name): + for kind in ('int', 'void'): + prefix = 'static ' + kind + ' ' + name + '(' + if prefix in source: + start = source.index(prefix) + return source[start:source.index('\n}', start)+2] + raise ValueError(name) + + +PREFIX = r''' +#include +#include +#include +#include +#define MAX_INTERFACES 16 +#define PMSG_SUSPEND 1 +#define HID_HAPTIC_MODE_DEVICE 0 +#define HID_HAPTIC_MODE_HOST 1 +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x) = (v)) +struct haptics { int mode; }; +struct magicmouse_sc { struct haptics *haptics; bool haptic_button_down, haptic_deep_down; }; +struct hid_device { struct magicmouse_sc *data; int id; }; +struct dchid_iface { struct hid_device *hid; }; +struct dockchannel_hid { struct dchid_iface *ifaces[MAX_INTERFACES]; bool suspended; }; +struct device { struct dockchannel_hid *data; }; +static struct dockchannel_hid *transport; +static unsigned suspends[16], resumes[16], key_releases; +static int suspend_error_at = -1, resume_error_at = -1; +static void *dev_get_drvdata(struct device *d) { return d->data; } +static void *hid_get_drvdata(struct hid_device *d) { return d->data; } +static void magicmouse_force_key(struct magicmouse_sc *m, bool down) { + assert(!down); key_releases++; +} +static int hid_driver_suspend(struct hid_device *h, int state) { + assert(state == PMSG_SUSPEND && !transport->suspended); + suspends[h->id]++; + return h->id == suspend_error_at ? -EBUSY : 0; +} +''' +MIDDLE = r''' +static int hid_driver_resume(struct hid_device *h) { + assert(!transport->suspended); + resumes[h->id]++; + magicmouse_resume(h); + return h->id == resume_error_at ? -EIO : 0; +} +''' +CASES = r''' +int main(void) { + struct haptics hap = {.mode = HID_HAPTIC_MODE_HOST}; + struct magicmouse_sc m = {.haptics = &hap, .haptic_button_down = true, .haptic_deep_down = true}; + struct hid_device h0 = {.data = &m, .id = 0}, h4 = {.id = 4}, h8 = {.id = 8}; + struct dchid_iface i0 = {.hid = &h0}, i4 = {.hid = &h4}, i8 = {.hid = &h8}, empty = {0}; + struct dockchannel_hid t = {.ifaces = {[0] = &i0, [3] = &empty, [4] = &i4, [8] = &i8}}; + struct device d = {.data = &t}; transport = &t; + assert(!dockchannel_hid_suspend(&d) && t.suspended); + assert(!dockchannel_hid_resume(&d) && !t.suspended); + assert(hap.mode == HID_HAPTIC_MODE_DEVICE); + assert(!m.haptic_button_down && !m.haptic_deep_down && key_releases == 1); + assert(suspends[0] == 1 && resumes[0] == 1 && resumes[8] == 1); + /* A later HID failure rolls back only the drivers that suspended. */ + suspend_error_at = 4; + assert(dockchannel_hid_suspend(&d) == -EBUSY && !t.suspended); + assert(resumes[0] == 2 && resumes[4] == 1 && suspends[8] == 1); + suspend_error_at = 0; + assert(dockchannel_hid_suspend(&d) == -EBUSY && !t.suspended); + assert(resumes[0] == 2 && suspends[4] == 2); + suspend_error_at = -1; + assert(!dockchannel_hid_suspend(&d) && t.suspended); + /* One failed resume cannot keep the other interfaces suspended. */ + resume_error_at = 4; + assert(dockchannel_hid_resume(&d) == -EIO && !t.suspended); + assert(resumes[0] == 3 && resumes[4] == 2 && resumes[8] == 2); + resume_error_at = -1; + for (unsigned n = 0; n < 3; n++) { + hap.mode = HID_HAPTIC_MODE_HOST; + assert(!dockchannel_hid_suspend(&d)); + assert(!dockchannel_hid_resume(&d)); + assert(hap.mode == HID_HAPTIC_MODE_DEVICE); + } + puts("MTP PM: HID notifications, rollback, all-interface resume and haptic cache passed"); +} +''' + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--out') + p.add_argument('--old-transport') + args = p.parse_args() + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + source = ((ROOT / TRANSPORT).read_text() if not args.old_transport else subprocess.check_output( + ['git', 'show', args.old_transport + ':' + TRANSPORT], cwd=ROOT, text=True)) + mouse = (ROOT / 'drivers/hid/hid-magicmouse.c').read_text() + with tempfile.TemporaryDirectory(prefix='mtp-pm-') as tmp: + out = Path(args.out or tmp) + out.mkdir(parents=True, exist_ok=True) + c, exe = out/'mtp-pm.c', out/'mtp-pm' + c.write_text(PREFIX + function(mouse, 'magicmouse_haptic_forget_mode') + '\n' + + function(mouse, 'magicmouse_resume') + MIDDLE + + function(source, 'dockchannel_hid_suspend') + '\n' + + function(source, 'dockchannel_hid_resume') + CASES) + subprocess.run([os.environ.get('CC', 'cc'), '-std=c11', '-Wall', '-Wextra', + '-Wno-unused-parameter', '-Wno-unused-function', + str(c), '-o', str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + (out/'run.log').write_text(r.stdout+r.stderr) + if args.old_transport: + if r.returncode != -6 or 'Assertion' not in r.stderr: + raise SystemExit('old transport did not fail its runtime assertion') + print('MTP PM: old transport compiled and failed the haptic-cache assertion') + elif r.returncode: + raise SystemExit(r.stdout+r.stderr) + else: + print(r.stdout, end='') + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index e7e60bad2e6cd..245a6fa1aa6d4 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -50,6 +50,8 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'mtp-pm-tests.py': [(out(), PASS), + (out() + ['--old-transport', '4db3a5a22c43'], PASS)], 'ave-pm-tests.py': [(out(), PASS), (out() + ['--old-ref', '4db3a5a22c43'], PASS)], 'apple-dart-fw-tests.py': [(out(), PASS)], From 8d54cd288f050aff64272a0714278b27be735e39 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:34:16 +1100 Subject: [PATCH 1438/1579] i2c: pasemi: runtime suspend Apple platform controllers between transfers Acquire a runtime PM reference around both I2C and SMBus transfers, including error recovery, and unwind it on every error. Leave the PCI controller's PM policy unchanged. Restore the divider, revision- dependent enable bit and FIFO state before allowing another transfer after domain power loss. Set up the interrupt and runtime PM before publishing the adapter, since client probes can transfer immediately. Drain the interrupt before disabling the reference clock. Order managed cleanup so client removal finishes while PM and the IRQ still work, then stop PM before freeing the IRQ and clock. Keep polling available when DT supplies no interrupt, but propagate resource and IRQ registration failures. Add KUnit coverage of failed resumes on both transfer entry points and early I2C/SMBus failures, checking PM reference balance and preventing MMIO after a failed resume. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/i2c/busses/Kconfig | 11 ++ drivers/i2c/busses/i2c-pasemi-core-test.c | 132 ++++++++++++++++++++++ drivers/i2c/busses/i2c-pasemi-core.c | 79 +++++++++---- drivers/i2c/busses/i2c-pasemi-core.h | 3 + drivers/i2c/busses/i2c-pasemi-platform.c | 103 ++++++++++++++--- 5 files changed, 295 insertions(+), 33 deletions(-) create mode 100644 drivers/i2c/busses/i2c-pasemi-core-test.c diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index d7b89508311ff..5001cc2640718 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -1007,6 +1007,17 @@ config I2C_APPLE This driver can also be built as a module. If so, the module will be called i2c-apple. +config I2C_PASEMI_KUNIT_TEST + bool "KUnit tests for PA Semi/Apple I2C runtime PM" if !KUNIT_ALL_TESTS + depends on KUNIT && PM && I2C_APPLE + default KUNIT_ALL_TESTS + help + Test runtime PM reference handling on resume and transfer failures. + Exercise both I2C and SMBus transfer entry points with a fake PM + domain and register storage. Failed resumes must not touch MMIO, + and transfer errors must release their power references. + No controller hardware is needed. These tests are for developers. + config I2C_PCA_PLATFORM tristate "PCA9564/PCA9665 as platform device" select I2C_ALGOPCA diff --git a/drivers/i2c/busses/i2c-pasemi-core-test.c b/drivers/i2c/busses/i2c-pasemi-core-test.c new file mode 100644 index 0000000000000..940d2ce520e8c --- /dev/null +++ b/drivers/i2c/busses/i2c-pasemi-core-test.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by i2c-pasemi-core.c to exercise the real transfer entry points. */ + +#include +#include +#include + +struct pasemi_pm_test { + struct pasemi_smbus smbus; + int resume_error; + int resumes; +}; + +static int pasemi_test_resume(struct device *dev) +{ + struct pasemi_pm_test *ctx = dev_get_drvdata(dev); + + ctx->resumes++; + return ctx->resume_error; +} + +static struct dev_pm_domain pasemi_test_domain = { + .ops.runtime_resume = pasemi_test_resume, +}; + +static void pasemi_test_disable_pm(void *arg) +{ + struct device *dev = arg; + + pm_runtime_disable(dev); + pm_runtime_dont_use_autosuspend(dev); + dev->pm_domain = NULL; +} + +static int pasemi_pm_test_init(struct kunit *test) +{ + struct pasemi_pm_test *ctx; + struct device *dev; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + dev = kunit_device_register(test, "pasemi-pm"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + dev_set_drvdata(dev, ctx); + dev->pm_domain = &pasemi_test_domain; + ctx->smbus.dev = dev; + ctx->smbus.runtime_pm = true; + ctx->smbus.adapter.algo_data = &ctx->smbus; + /* A baseline reference of one prevents autosuspend racing assertions. */ + pm_runtime_set_autosuspend_delay(dev, -1); + pm_runtime_use_autosuspend(dev); + pm_runtime_enable(dev); + test->priv = ctx; + return kunit_add_action_or_reset(test, pasemi_test_disable_pm, dev); +} + +static void pasemi_i2c_resume_failure_test(struct kunit *test) +{ + struct pasemi_pm_test *ctx = test->priv; + struct i2c_msg msg = { .addr = 0x10 }; + + ctx->resume_error = -EIO; + /* ioaddr is NULL: neither path may access MMIO after a failed resume. */ + KUNIT_EXPECT_EQ(test, pasemi_i2c_xfer(&ctx->smbus.adapter, &msg, 1), -EIO); + KUNIT_EXPECT_EQ(test, ctx->resumes, 1); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->smbus.dev->power.usage_count), 1); +} + +static void pasemi_smb_resume_failure_test(struct kunit *test) +{ + struct pasemi_pm_test *ctx = test->priv; + union i2c_smbus_data data; + int ret; + + ctx->resume_error = -EIO; + ret = pasemi_smb_xfer(&ctx->smbus.adapter, 0x10, 0, I2C_SMBUS_READ, + 0, I2C_SMBUS_BYTE_DATA, &data); + KUNIT_EXPECT_EQ(test, ret, -EIO); + KUNIT_EXPECT_EQ(test, ctx->resumes, 1); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->smbus.dev->power.usage_count), 1); +} + +static void pasemi_test_mmio(struct kunit *test, u32 status) +{ + struct pasemi_pm_test *ctx = test->priv; + void *regs = kunit_kzalloc(test, 0x30, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, regs); + ctx->smbus.ioaddr = (void __iomem __force *)regs; + iowrite32(status, ctx->smbus.ioaddr + REG_SMSTA); +} + +static void pasemi_i2c_busy_failure_test(struct kunit *test) +{ + struct pasemi_pm_test *ctx = test->priv; + struct i2c_msg msg = { .addr = 0x10 }; + + pasemi_test_mmio(test, SMSTA_XIP); + KUNIT_EXPECT_EQ(test, pasemi_i2c_xfer(&ctx->smbus.adapter, &msg, 1), -EIO); + KUNIT_EXPECT_EQ(test, ctx->resumes, 1); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->smbus.dev->power.usage_count), 1); +} + +static void pasemi_smb_invalid_failure_test(struct kunit *test) +{ + struct pasemi_pm_test *ctx = test->priv; + union i2c_smbus_data data; + int ret; + + pasemi_test_mmio(test, SMSTA_MTE); + ret = pasemi_smb_xfer(&ctx->smbus.adapter, 0x10, 0, I2C_SMBUS_READ, + 0, -1, &data); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, ctx->resumes, 1); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->smbus.dev->power.usage_count), 1); +} + +static struct kunit_case pasemi_pm_test_cases[] = { + KUNIT_CASE(pasemi_i2c_resume_failure_test), + KUNIT_CASE(pasemi_smb_resume_failure_test), + KUNIT_CASE(pasemi_i2c_busy_failure_test), + KUNIT_CASE(pasemi_smb_invalid_failure_test), + {} +}; + +static struct kunit_suite pasemi_pm_test_suite = { + .name = "i2c-pasemi-runtime-pm", + .init = pasemi_pm_test_init, + .test_cases = pasemi_pm_test_cases, +}; + +kunit_test_suite(pasemi_pm_test_suite); diff --git a/drivers/i2c/busses/i2c-pasemi-core.c b/drivers/i2c/busses/i2c-pasemi-core.c index f4eca44ed1839..90d73a338bd27 100644 --- a/drivers/i2c/busses/i2c-pasemi-core.c +++ b/drivers/i2c/busses/i2c-pasemi-core.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -244,19 +245,29 @@ static int pasemi_i2c_xfer(struct i2c_adapter *adapter, struct pasemi_smbus *smbus = adapter->algo_data; int ret, i; + if (smbus->runtime_pm) { + ret = pm_runtime_resume_and_get(smbus->dev); + if (ret < 0) + return ret; + } + ret = pasemi_smb_clear(smbus); if (ret) - return ret; + goto out; for (i = 0; i < num && !ret; i++) ret = pasemi_i2c_xfer_msg(adapter, &msgs[i], (i == (num - 1))); +out: + if (smbus->runtime_pm) + pm_runtime_put_autosuspend(smbus->dev); return ret ? ret : num; } -static int pasemi_smb_xfer(struct i2c_adapter *adapter, - u16 addr, unsigned short flags, char read_write, u8 command, - int size, union i2c_smbus_data *data) +static int pasemi_smb_xfer_unlocked(struct i2c_adapter *adapter, u16 addr, + unsigned short flags, char read_write, + u8 command, int size, + union i2c_smbus_data *data) { struct pasemi_smbus *smbus = adapter->algo_data; unsigned int rd; @@ -414,16 +425,47 @@ static u32 pasemi_smb_func(struct i2c_adapter *adapter) I2C_FUNC_SMBUS_BLOCK_PROC_CALL | I2C_FUNC_I2C; } +static int pasemi_smb_xfer(struct i2c_adapter *adapter, + u16 addr, unsigned short flags, char read_write, + u8 command, int size, union i2c_smbus_data *data) +{ + struct pasemi_smbus *smbus = adapter->algo_data; + int ret; + + if (smbus->runtime_pm) { + ret = pm_runtime_resume_and_get(smbus->dev); + if (ret < 0) + return ret; + } + + ret = pasemi_smb_xfer_unlocked(adapter, addr, flags, read_write, command, + size, data); + if (smbus->runtime_pm) + pm_runtime_put_autosuspend(smbus->dev); + + return ret; +} + static const struct i2c_algorithm smbus_algorithm = { .xfer = pasemi_i2c_xfer, .smbus_xfer = pasemi_smb_xfer, .functionality = pasemi_smb_func, }; -int pasemi_i2c_common_probe(struct pasemi_smbus *smbus) +void pasemi_i2c_init(struct pasemi_smbus *smbus) { - int error; + init_completion(&smbus->irq_completion); + + if (smbus->hw_rev != PASEMI_HW_REV_PCI) + smbus->hw_rev = reg_read(smbus, REG_REV); + + reg_write(smbus, REG_IMASK, 0); + pasemi_reset(smbus); +} +EXPORT_SYMBOL_GPL(pasemi_i2c_init); +int pasemi_i2c_register_adapter(struct pasemi_smbus *smbus) +{ smbus->adapter.owner = THIS_MODULE; snprintf(smbus->adapter.name, sizeof(smbus->adapter.name), "PA Semi SMBus adapter (%s)", dev_name(smbus->dev)); @@ -432,21 +474,14 @@ int pasemi_i2c_common_probe(struct pasemi_smbus *smbus) /* set up the sysfs linkage to our parent device */ smbus->adapter.dev.parent = smbus->dev; - smbus->use_irq = 0; - init_completion(&smbus->irq_completion); - - if (smbus->hw_rev != PASEMI_HW_REV_PCI) - smbus->hw_rev = reg_read(smbus, REG_REV); - - reg_write(smbus, REG_IMASK, 0); - - pasemi_reset(smbus); - - error = devm_i2c_add_adapter(smbus->dev, &smbus->adapter); - if (error) - return error; + return devm_i2c_add_adapter(smbus->dev, &smbus->adapter); +} +EXPORT_SYMBOL_GPL(pasemi_i2c_register_adapter); - return 0; +int pasemi_i2c_common_probe(struct pasemi_smbus *smbus) +{ + pasemi_i2c_init(smbus); + return pasemi_i2c_register_adapter(smbus); } EXPORT_SYMBOL_GPL(pasemi_i2c_common_probe); @@ -460,6 +495,10 @@ irqreturn_t pasemi_irq_handler(int irq, void *dev_id) } EXPORT_SYMBOL_GPL(pasemi_irq_handler); +#if IS_ENABLED(CONFIG_I2C_PASEMI_KUNIT_TEST) +#include "i2c-pasemi-core-test.c" +#endif + MODULE_LICENSE("GPL"); MODULE_AUTHOR("Olof Johansson "); MODULE_DESCRIPTION("PA Semi PWRficient SMBus driver"); diff --git a/drivers/i2c/busses/i2c-pasemi-core.h b/drivers/i2c/busses/i2c-pasemi-core.h index 88821f4e8a9f8..f32300648319d 100644 --- a/drivers/i2c/busses/i2c-pasemi-core.h +++ b/drivers/i2c/busses/i2c-pasemi-core.h @@ -18,9 +18,12 @@ struct pasemi_smbus { unsigned int clk_div; int hw_rev; int use_irq; + bool runtime_pm; struct completion irq_completion; }; int pasemi_i2c_common_probe(struct pasemi_smbus *smbus); +void pasemi_i2c_init(struct pasemi_smbus *smbus); +int pasemi_i2c_register_adapter(struct pasemi_smbus *smbus); irqreturn_t pasemi_irq_handler(int irq, void *dev_id); diff --git a/drivers/i2c/busses/i2c-pasemi-platform.c b/drivers/i2c/busses/i2c-pasemi-platform.c index a486a37d38634..efeaeb43c16d6 100644 --- a/drivers/i2c/busses/i2c-pasemi-platform.c +++ b/drivers/i2c/busses/i2c-pasemi-platform.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "i2c-pasemi-core.h" @@ -18,8 +19,53 @@ struct pasemi_platform_i2c_data { struct pasemi_smbus smbus; struct clk *clk_ref; + int irq; }; +static int pasemi_platform_i2c_runtime_suspend(struct device *dev) +{ + struct pasemi_platform_i2c_data *data = dev_get_drvdata(dev); + + if (data->smbus.use_irq) + disable_irq(data->irq); + clk_disable_unprepare(data->clk_ref); + + return 0; +} + +static int pasemi_platform_i2c_runtime_resume(struct device *dev) +{ + struct pasemi_platform_i2c_data *data = dev_get_drvdata(dev); + int error; + + error = clk_prepare_enable(data->clk_ref); + if (error) + return error; + + /* Restore the divider, enable bit and FIFOs after domain power loss. */ + pasemi_i2c_init(&data->smbus); + if (data->smbus.use_irq) + enable_irq(data->irq); + + return 0; +} + +static DEFINE_RUNTIME_DEV_PM_OPS(pasemi_platform_i2c_pm_ops, + pasemi_platform_i2c_runtime_suspend, + pasemi_platform_i2c_runtime_resume, NULL); + +static void pasemi_platform_i2c_disable_pm(void *arg) +{ + struct device *dev = arg; + + /* Registered before the adapter, after the IRQ: clients are gone now. */ + pm_runtime_disable(dev); + pm_runtime_dont_use_autosuspend(dev); + if (!pm_runtime_status_suspended(dev)) + pasemi_platform_i2c_runtime_suspend(dev); + pm_runtime_set_suspended(dev); +} + static int pasemi_platform_i2c_calc_clk_div(struct pasemi_platform_i2c_data *data, u32 frequency) @@ -66,7 +112,7 @@ static int pasemi_platform_i2c_probe(struct platform_device *pdev) if (of_property_read_u32(dev->of_node, "clock-frequency", &frequency)) frequency = I2C_MAX_STANDARD_MODE_FREQ; - data->clk_ref = devm_clk_get_enabled(dev, NULL); + data->clk_ref = devm_clk_get(dev, NULL); if (IS_ERR(data->clk_ref)) return PTR_ERR(data->clk_ref); @@ -74,23 +120,54 @@ static int pasemi_platform_i2c_probe(struct platform_device *pdev) if (error) return error; - smbus->adapter.dev.of_node = pdev->dev.of_node; - error = pasemi_i2c_common_probe(smbus); + platform_set_drvdata(pdev, data); + error = clk_prepare_enable(data->clk_ref); if (error) return error; - irq_num = platform_get_irq(pdev, 0); - error = devm_request_irq(smbus->dev, irq_num, pasemi_irq_handler, 0, "pasemi_apple_i2c", (void *)smbus); - - if (!error) + pasemi_i2c_init(smbus); + irq_num = platform_get_irq_optional(pdev, 0); + if (irq_num < 0 && irq_num != -ENXIO) { + error = irq_num; + goto disable_clk; + } + if (irq_num > 0) { + error = devm_request_irq(dev, irq_num, pasemi_irq_handler, 0, + "pasemi_apple_i2c", smbus); + if (error) + goto disable_clk; smbus->use_irq = 1; - platform_set_drvdata(pdev, data); - - return 0; + data->irq = irq_num; + } + + pm_runtime_get_noresume(dev); + error = pm_runtime_set_active(dev); + if (error) { + pm_runtime_put_noidle(dev); + goto disable_clk; + } + pm_runtime_set_autosuspend_delay(dev, 100); + pm_runtime_use_autosuspend(dev); + pm_runtime_enable(dev); + error = devm_add_action_or_reset(dev, pasemi_platform_i2c_disable_pm, dev); + if (error) { + pm_runtime_put_noidle(dev); + return error; + } + + smbus->runtime_pm = true; + smbus->adapter.dev.of_node = dev->of_node; + error = pasemi_i2c_register_adapter(smbus); + pm_runtime_put_autosuspend(dev); + return error; + +disable_clk: + if (smbus->use_irq) + devm_free_irq(dev, data->irq, smbus); + clk_disable_unprepare(data->clk_ref); + return error; } -static void pasemi_platform_i2c_remove(struct platform_device *pdev) { } - static const struct of_device_id pasemi_platform_i2c_of_match[] = { { .compatible = "apple,t8103-i2c" }, { .compatible = "apple,i2c" }, @@ -102,9 +179,9 @@ static struct platform_driver pasemi_platform_i2c_driver = { .driver = { .name = "i2c-apple", .of_match_table = pasemi_platform_i2c_of_match, + .pm = pm_ptr(&pasemi_platform_i2c_pm_ops), }, .probe = pasemi_platform_i2c_probe, - .remove = pasemi_platform_i2c_remove, }; module_platform_driver(pasemi_platform_i2c_driver); From 33a3574ab9b0029887420b3703b79c03d5da30ab Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:41:27 +1100 Subject: [PATCH 1439/1579] dt-bindings: power: apple: keep vendor property dependencies intact Use YAML block lists so commas inside vendor property names remain part of each dependency. The binding example and DT gates exposed these incorrect flow lists. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../opp/operating-points-v2-apple-dram.yaml | 6 ++++-- .../bindings/power/apple,pmp-dram.yaml | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dram.yaml b/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dram.yaml index a201d86f1ba42..76ae3c6f875ae 100644 --- a/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dram.yaml +++ b/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dram.yaml @@ -56,8 +56,10 @@ patternProperties: - opp-level dependencies: - apple,data-rate: [opp-hz] - opp-hz: [apple,data-rate] + apple,data-rate: + - opp-hz + opp-hz: + - apple,data-rate required: - compatible diff --git a/Documentation/devicetree/bindings/power/apple,pmp-dram.yaml b/Documentation/devicetree/bindings/power/apple,pmp-dram.yaml index 250c2e2019949..8c0cc99eb2177 100644 --- a/Documentation/devicetree/bindings/power/apple,pmp-dram.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmp-dram.yaml @@ -68,15 +68,21 @@ properties: description: Published, potentially rounded, maximum bandwidth class. dependencies: - operating-points-v2: [apple,dram-type] - apple,data-bus-width: [apple,dram-type] - apple,dcs-state-count: [apple,dram-type] - apple,peak-bandwidth-bytes-per-second: [apple,dram-type] - apple,bandwidth-class-bytes-per-second: [apple,dram-type] + operating-points-v2: + - apple,dram-type + apple,data-bus-width: + - apple,dram-type + apple,dcs-state-count: + - apple,dram-type + apple,peak-bandwidth-bytes-per-second: + - apple,dram-type + apple,bandwidth-class-bytes-per-second: + - apple,dram-type allOf: - if: - required: [apple,dram-type] + required: + - apple,dram-type then: required: - operating-points-v2 From c73a5b1a27f6060719614a3931c9a8d182668c30 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:51:36 +1100 Subject: [PATCH 1440/1579] pmdomain: apple: fix DRAM KUnit reference and module metadata Initialize references with SOFTWARE_NODE_REFERENCE and provide a module description. This fixes W=1 compilation and the missing-description warning from the full modules build. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmp-dram-test.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/pmdomain/apple/pmp-dram-test.c b/drivers/pmdomain/apple/pmp-dram-test.c index 5a7bd479b3ba7..c662b99ff3df1 100644 --- a/drivers/pmdomain/apple/pmp-dram-test.c +++ b/drivers/pmdomain/apple/pmp-dram-test.c @@ -32,7 +32,7 @@ static struct pmp_dram_fixture *pmp_dram_fixture(struct kunit *test, u32 count, f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, f); - f->ref[0].node = &f->nodes[1]; + f->ref[0] = SOFTWARE_NODE_REFERENCE(&f->nodes[1]); f->props[0] = PROPERTY_ENTRY_STRING("apple,dram-type", "lpddr5"); f->props[1] = PROPERTY_ENTRY_U32("apple,data-bus-width", width); f->props[2] = PROPERTY_ENTRY_U32("apple,dcs-state-count", count); @@ -255,4 +255,5 @@ static struct kunit_suite pmp_dram_suite = { kunit_test_suite(pmp_dram_suite); +MODULE_DESCRIPTION("KUnit tests for Apple PMP DRAM property parsing"); MODULE_LICENSE("Dual MIT/GPL"); From 2bdb22771cb2c0cb031e3d4ad8e70c5718cff99d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:51:43 +1100 Subject: [PATCH 1441/1579] drm/asahi: Format the G17 statistics integration paths Apply rustfmt to the existing files touched by the qualified statistics snapshot, keeping formatting separate from the decoder change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/driver.rs | 11 ++--- drivers/gpu/drm/asahi/g17/channel.rs | 7 ++- drivers/gpu/drm/asahi/g17/port.rs | 49 +++++++++++++++------ drivers/gpu/drm/asahi/g17/runtime/events.rs | 29 +++++++----- drivers/gpu/drm/asahi/sysfs_exports.rs | 6 ++- 5 files changed, 70 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index ab68573881659..2733ed372875a 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -220,10 +220,7 @@ impl platform::Driver for AsahiPlatformDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; - fn unbind<'bound>( - _pdev: &'bound platform::Device>, - this: Pin<&Self::Data<'bound>>, - ) { + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { if let Some(gpu) = this.g17.as_ref() { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); @@ -390,7 +387,11 @@ impl AsahiPlatformDriver { if let Err(error) = crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) { - dev_warn!(pdev.as_ref(), "idle_off_delay_ms not registered: {:?}\n", error); + dev_warn!( + pdev.as_ref(), + "idle_off_delay_ms not registered: {:?}\n", + error + ); } Ok(AsahiPlatformDriverData { _device: reg.device().into(), diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 568647ebb444a..7b99daf4c1fce 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -85,7 +85,12 @@ impl Rings { } } - fn consume_report(init: &InitData, role: Role, ring: ReportRing, reports: &mut super::reports::Reports) -> Result { + fn consume_report( + init: &InitData, + role: Role, + ring: ReportRing, + reports: &mut super::reports::Reports, + ) -> Result { let (state, entries, capacity, allowed_types) = ring.geometry(); let consumer_word = init.status_word(role, state)?; let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 6d0e87c14b947..ff95b07fa759b 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -225,7 +225,11 @@ impl Port { /// Exposes `record` in its reserved slot. Inner queue records must be visible first. Returns /// the new producer, the retained consumer witness for this slot. - pub(super) fn publish_compute(&self, mut claim: OuterClaim<'_>, record: &WorkSlot) -> Result { + pub(super) fn publish_compute( + &self, + mut claim: OuterClaim<'_>, + record: &WorkSlot, + ) -> Result { if record.data_master != DataMaster::Compute as u32 || !core::ptr::eq(claim.port, self) { return Err(EINVAL); } @@ -242,7 +246,8 @@ impl Port { unsafe { core::ptr::copy_nonoverlapping( core::ptr::from_ref(record).cast::(), - ring.slots.add(producer as usize * core::mem::size_of::()), + ring.slots + .add(producer as usize * core::mem::size_of::()), core::mem::size_of::(), ); } @@ -280,13 +285,16 @@ impl Port { let (fragment, tiling) = self.render_rings(priority)?; let cursors = [fragment.cursors()?, tiling.cursors()?]; let room = |[first, second, producer]: [u32; 3]| { - let room = |consumer: u32| (consumer + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS; + let room = + |consumer: u32| (consumer + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS; room(first).min(room(second)) }; - Ok((room(cursors[0]) > 0 && room(cursors[1]) > 0).then_some(RenderRoom { - priority, - producers: [cursors[0][2], cursors[1][2]], - })) + Ok( + (room(cursors[0]) > 0 && room(cursors[1]) > 0).then_some(RenderRoom { + priority, + producers: [cursors[0][2], cursors[1][2]], + }), + ) } /// Copies a render pair's fragment, then tiling record into the slots after the producers @@ -312,7 +320,10 @@ impl Port { return Err(EINVAL); } // SAFETY: The ring view spans WORK_RING_SLOTS records; `producer` is in range. - let slot = unsafe { ring.slots.add(producer as usize * core::mem::size_of::()) }; + let slot = unsafe { + ring.slots + .add(producer as usize * core::mem::size_of::()) + }; Ok((slot, (producer + 1) % WORK_RING_SLOTS)) }; let (fragment_slot, fragment_next) = slot(&fragment_ring, room.producers[0])?; @@ -385,7 +396,10 @@ impl Port { policy: Policy, ) -> Result { let view = self.qos_view()?; - let publication = self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock)?; + let publication = self + .qos + .lock() + .publish(&view, owner, scheduler, policy, qos::clock)?; self.rx_update(); Ok(publication) } @@ -399,7 +413,9 @@ impl Port { ) -> Result<[qos::Publication; 2]> { let view = self.qos_view()?; let publications = - self.qos.lock().publish_owners(&view, owners, scheduler, policy, qos::clock)?; + self.qos + .lock() + .publish_owners(&view, owners, scheduler, policy, qos::clock)?; self.rx_update(); Ok(publications) } @@ -485,13 +501,18 @@ impl Port { } pub(super) fn note_gpu_activity(&self) { - self.last_submission_ns.store(super::now_ns(), Ordering::Relaxed); + self.last_submission_ns + .store(super::now_ns(), Ordering::Relaxed); self.demand.set_active(true); } pub(super) fn write_power(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "effort_full={} direct_submissions={} last_submission_ns={}", - self.effort_full.load(Ordering::SeqCst), self.submissions(), - self.last_submission_ns.load(Ordering::Relaxed)) + writeln!( + f, + "effort_full={} direct_submissions={} last_submission_ns={}", + self.effort_full.load(Ordering::SeqCst), + self.submissions(), + self.last_submission_ns.load(Ordering::Relaxed) + ) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index a59f496ab4f3b..55d517a60ade8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -47,7 +47,8 @@ impl event::Transport for Service<'_> { role: Role, mut consume: impl FnMut(&mut Self, Event) -> Result, ) -> Result { - let mut events = channel::Events::new(role, self.firmware.primary.state.shared.variant.graph); + let mut events = + channel::Events::new(role, self.firmware.primary.state.shared.variant.graph); while let Some(event) = events.next(&self.firmware.primary.state.shared.dev, &self.firmware.init)? { @@ -232,7 +233,9 @@ impl recovery::Queues for Service<'_> { }) } fn report_blamed_render(&self, qid: u8) { - self.firmware.queues.report_render_recovery(qid, &self.firmware.port); + self.firmware + .queues + .report_render_recovery(qid, &self.firmware.port); } fn settle_blamed_render(&mut self, qid: u8, error: Error) { let result = self @@ -264,11 +267,9 @@ impl Firmware { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - queue.release_witnessed( - state, - |owner| port.qos_complete(&[owner]), - &mut |event| deferred.push(event), - )?; + queue.release_witnessed(state, |owner| port.qos_complete(&[owner]), &mut |event| { + deferred.push(event) + })?; } Ok(()) } @@ -313,7 +314,10 @@ impl Firmware { }); // Publishers without the device mutex stamp their tickets with this // mirror; they rejoin only after preparation admission reopens. - service.firmware.port.set_generation(service.firmware.recovery.generation()); + service + .firmware + .port + .set_generation(service.firmware.recovery.generation()); let result = result?; if let Some(error) = result.target_error { dev_warn!( @@ -394,7 +398,8 @@ impl crate::g17::Shared { deferred.finish(); crate::g17::trace::event(1); if recover { - self.preparations.wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; + self.preparations + .wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; } let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -451,8 +456,10 @@ impl crate::g17::Shared { || self.progress.has_waiters() || self.queues.has_waiters() || self.renders.has_waiters() - || now.saturating_sub(self.housekeeping_ns.load(core::sync::atomic::Ordering::Relaxed)) - >= HOUSEKEEPING_NS; + || now.saturating_sub( + self.housekeeping_ns + .load(core::sync::atomic::Ordering::Relaxed), + ) >= HOUSEKEEPING_NS; let mut lingers = None; let mut pools = None; let mut shrinking = false; diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index cee5d52a4e261..e7f842d8ac6fd 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -29,7 +29,11 @@ pub(crate) fn register_idle( // SAFETY: `word` is a static atomic that outlives the attribute; the C side reads and writes // it with READ_ONCE/WRITE_ONCE. let ret = unsafe { asahi_sysfs_register_idle(dev.cast(), word) }; - if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { Ok(()) } + if ret < 0 { + Err(kernel::error::Error::from_errno(ret)) + } else { + Ok(()) + } } /// Register only after initialization succeeds, keeping the snapshot Arc alive. From fdcf196afe4a1a2fc6fe56ba03c72d2e136f525b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 05:28:57 +1100 Subject: [PATCH 1442/1579] drm/asahi: Export qualified G16S and G17P statistics Decode bounded tiler, renderer and device percentages and the logical pstate from complete copied records in one shared G17 path. Merge firmware roles in drain order, bound the utilisation averaging window, and invalidate epochs on restart, suspend/resume, quarantine and observed gating. Keep stable logical states valid without inventing a periodic-report deadline. Read a coherent snapshot under the device mutex without waking the GPU. Leave ambiguous A/B requested frequency, achieved frequency, busy time, measured power and temperature unsupported; preserve the legacy G13/G14 formatter. Add shared KUnit/host SPEC vectors, transport and formatter tests, legacy byte comparisons, negative controls and ABI documentation. Resolve the existing driver Clippy diagnostics with typed conversions and explicit contracts. Enforce bound-device access during SG preparation with a joined binding lease instead of unchecked lifetime casts; exercise removal and rebind with host concurrency controls. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../ABI/testing/sysfs-driver-asahi-agx-stats | 57 +- drivers/gpu/drm/asahi/alloc.rs | 18 +- drivers/gpu/drm/asahi/asahi_core.rs | 3 + drivers/gpu/drm/asahi/binding.rs | 130 +++++ drivers/gpu/drm/asahi/driver.rs | 37 +- drivers/gpu/drm/asahi/file.rs | 17 +- drivers/gpu/drm/asahi/float.rs | 4 +- drivers/gpu/drm/asahi/fw/event.rs | 2 +- drivers/gpu/drm/asahi/fw/types.rs | 6 + drivers/gpu/drm/asahi/fwid.rs | 2 +- drivers/gpu/drm/asahi/g15/mod.rs | 6 + drivers/gpu/drm/asahi/g16/adt.rs | 2 +- drivers/gpu/drm/asahi/g16/boot.rs | 32 +- drivers/gpu/drm/asahi/g16/encode.rs | 50 +- drivers/gpu/drm/asahi/g16/encode/compute.rs | 2 +- drivers/gpu/drm/asahi/g16/encode/render.rs | 92 ++-- drivers/gpu/drm/asahi/g16/engine.rs | 2 +- drivers/gpu/drm/asahi/g16/event.rs | 2 +- drivers/gpu/drm/asahi/g16/fw/channels.rs | 4 +- drivers/gpu/drm/asahi/g16/fw/initdata.rs | 20 +- drivers/gpu/drm/asahi/g16/initdata.rs | 9 +- drivers/gpu/drm/asahi/g16/job.rs | 4 + drivers/gpu/drm/asahi/g16/mod.rs | 1 + drivers/gpu/drm/asahi/g16/object.rs | 8 +- drivers/gpu/drm/asahi/g16/params.rs | 4 +- drivers/gpu/drm/asahi/g16/poll.rs | 14 +- drivers/gpu/drm/asahi/g16/pools.rs | 4 +- drivers/gpu/drm/asahi/g16/pools/lifetime.rs | 2 +- drivers/gpu/drm/asahi/g16/recovery.rs | 5 +- drivers/gpu/drm/asahi/g16/recovery/logic.rs | 2 +- drivers/gpu/drm/asahi/g16/regs.rs | 2 +- drivers/gpu/drm/asahi/g16/uat.rs | 10 +- drivers/gpu/drm/asahi/g17/channel.rs | 85 ++- drivers/gpu/drm/asahi/g17/fence.rs | 228 ++++++-- drivers/gpu/drm/asahi/g17/initdata.rs | 150 +++-- drivers/gpu/drm/asahi/g17/mod.rs | 27 +- drivers/gpu/drm/asahi/g17/port.rs | 16 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 514 ++++++++++++++---- drivers/gpu/drm/asahi/g17/queue/render.rs | 185 +++++-- .../drm/asahi/g17/queue/render/publication.rs | 8 +- drivers/gpu/drm/asahi/g17/reports.rs | 54 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 4 + drivers/gpu/drm/asahi/g17/stats.rs | 40 ++ drivers/gpu/drm/asahi/g17/telemetry.rs | 246 +++++++++ drivers/gpu/drm/asahi/gem.rs | 12 +- drivers/gpu/drm/asahi/gpu.rs | 16 +- drivers/gpu/drm/asahi/initdata.rs | 44 +- drivers/gpu/drm/asahi/mem.rs | 6 +- drivers/gpu/drm/asahi/microseq.rs | 2 +- drivers/gpu/drm/asahi/mmu.rs | 174 ++++-- drivers/gpu/drm/asahi/object.rs | 40 +- drivers/gpu/drm/asahi/pgtable.rs | 37 +- drivers/gpu/drm/asahi/queue/compute.rs | 19 +- drivers/gpu/drm/asahi/queue/mod.rs | 23 +- drivers/gpu/drm/asahi/queue/render.rs | 159 +++--- drivers/gpu/drm/asahi/stats.rs | 8 +- drivers/gpu/drm/asahi/sysfs.c | 92 +++- drivers/gpu/drm/asahi/sysfs.h | 22 +- drivers/gpu/drm/asahi/sysfs_exports.rs | 73 ++- drivers/gpu/drm/asahi/util.rs | 11 +- drivers/gpu/drm/asahi/workqueue.rs | 2 +- tools/asahi/binding-lifetime-tests.py | 125 +++++ tools/asahi/g15-probe-init-tests.py | 8 + tools/asahi/g17-capacity-fault-tests.py | 10 +- .../asahi/g17-telemetry-integration-tests.py | 371 +++++++++++++ tools/asahi/g17-telemetry-tests.rs | 6 + tools/asahi/g17-unpublished-tests.py | 5 +- tools/asahi/run-host-tests.py | 8 + 68 files changed, 2715 insertions(+), 668 deletions(-) create mode 100644 drivers/gpu/drm/asahi/binding.rs create mode 100644 drivers/gpu/drm/asahi/g17/stats.rs create mode 100644 drivers/gpu/drm/asahi/g17/telemetry.rs create mode 100644 tools/asahi/binding-lifetime-tests.py create mode 100644 tools/asahi/g17-telemetry-integration-tests.py create mode 100644 tools/asahi/g17-telemetry-tests.rs diff --git a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats index 11a964e6f17d8..10cba6c35673f 100644 --- a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats +++ b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats @@ -64,5 +64,58 @@ Description: `#[repr(C)]` on the Rust side and its offsets are asserted at build time in `sysfs.c`; readers must parse `key value` lines and must not assume binary layout of this file. - G17 firmware has no supported decoder for this snapshot format; - its file reports unsupported rather than fabricated zero counters. + + G16S (T6040/V25G83) and G17P (T8140/V26A428) use a separate, + qualified snapshot. The G13/G14 output above is unchanged. + The following keys replace the legacy keys on these firmwares: + + utilisation_status, pstate_status + Each is valid, stale, or unavailable. Numeric metrics + are printed only while their capability is valid; + otherwise their value is the string unsupported. + + util_tiler_percent, util_renderer_percent, util_device_percent + Unsigned integer percent (0..100), from tag 0x13 + words 3, 4 and 6 respectively (zero-based). Device + utilisation is independent of the overlapping engines; + it is never their sum. Each complete valid packet from + either role contributes one sample to a shared integer + mean, rounded down. Reads do not consume samples. + A packet with any value above 100 invalidates the whole + utilisation capability. Zero is a valid sample. + + pstate Logical selector 0..10 from tag 0x10 word 6. Words 1 + and 2 form a little-endian u64 firmware timestamp. + The first timestamp establishes a baseline. Invalid + selectors and backwards timestamps remove pstate + validity; timestamps are never extended across epochs. + P0 is a logical state, not a core-power measurement. + + requested_frequency_hz, achieved_frequency_hz, busy_ns, + power_mw, temperature_raw + Always unsupported on these firmwares. The statistics + packet does not identify performance table A or B; + a selector is neither a host request nor a measured + frequency. No qualified busy-time duration, power unit + or temperature field is available. + + The host uses a 16 ms age/averaging-window limit for V25G83 and + 32 ms for V26A428 (two nominal periods). These are conservative + host policies, not firmware guarantees about packet duration. + The oldest contributing utilisation sample bounds its window. + A logical pstate has no qualified periodic-report cadence and + remains valid until a replacement or an epoch boundary; its + host receive time is retained in diagnostics, not used to invent + a periodic-report deadline. + Suspend/resume, restart, statistics-ring quarantine, observed + core gating and teardown end validity. After an epoch boundary, + each ring's already queued records are discarded before new + baselines are accepted. A quarantined statistics ring disables + both roles' exports until reprobe; a trace-ring error does not. + + Reads take the device mutex and do not wake the GPU. The + firmware_reports debugfs view retains per-role raw packets, + counts and host monotonic receive times, plus capability status, + policy, timestamp and diagnostic tick deltas. Tick deltas are + never accumulated as GPU busy time. Firmware role streams are + merged in drain order without deduplication. diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs index d815fdf1040ac..10e367b1a15fa 100644 --- a/drivers/gpu/drm/asahi/alloc.rs +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -162,14 +162,14 @@ impl Drop for GenericAlloc { // the invariant, and greater than debug_len as checked above. unsafe { let p = p.as_ptr().add(self.debug_offset - debug_len); - (p as *mut u32).write(STATE_DEAD); + p.cast::().write(STATE_DEAD); } } } if debug_enabled(DebugFlags::FillAllocations) { if let Some(p) = self.ptr() { // SAFETY: Writing to our inner base pointer with our known inner size is safe. - unsafe { (p.as_ptr() as *mut u8).write_bytes(0xde, self.size()) }; + unsafe { p.as_ptr().cast::().write_bytes(0xde, self.size()) }; } } if self.padding != 0 { @@ -178,7 +178,7 @@ impl Drop for GenericAlloc { // the inner base pointer, after `size()` bytes. let guard = unsafe { core::slice::from_raw_parts( - (p.as_ptr() as *mut u8 as *const u8).add(self.size()), + p.as_ptr().cast::().cast_const().add(self.size()), self.padding, ) }; @@ -313,8 +313,8 @@ pub(crate) trait Allocator { unsafe { let p = p.as_ptr(); p.write_bytes(0x42, debug_offset - 2 * debug_len); - let cur = p.add(debug_offset - debug_len) as *mut AllocDebugData; - let prev = p.add(debug_offset - 2 * debug_len) as *mut AllocDebugData; + let cur = p.add(debug_offset - debug_len).cast::(); + let prev = p.add(debug_offset - 2 * debug_len).cast::(); prev.copy_from(cur, 1); cur.copy_from(&debug, 1); }; @@ -344,7 +344,7 @@ pub(crate) trait Allocator { if debug_enabled(DebugFlags::FillAllocations) { if let Some(p) = ret.ptr() { // SAFETY: Writing to our inner base pointer with our known inner size is safe. - unsafe { (p.as_ptr() as *mut u8).write_bytes(0xaa, ret.size()) }; + unsafe { p.as_ptr().cast::().write_bytes(0xaa, ret.size()) }; } } @@ -354,7 +354,7 @@ pub(crate) trait Allocator { // the inner base pointer, after `size()` bytes. let guard = unsafe { core::slice::from_raw_parts_mut( - (p.as_ptr() as *mut u8).add(ret.size()), + p.as_ptr().cast::().add(ret.size()), padding, ) }; @@ -536,7 +536,7 @@ impl Allocator for SimpleAllocator { ); let mut obj = crate::gem::new_kernel_object(&self.dev, size_aligned)?; - let p = obj.vmap()?.addr as *mut u8; + let p = obj.vmap()?.addr.cast::(); if debug_enabled(DebugFlags::FillAllocations) { let obj_size = obj.size(); if let Ok(vmap) = obj.vmap() { @@ -990,7 +990,7 @@ impl HeapAllocator { let (obj_start, obj_size, p) = self.mm.with_inner(|inner| -> Result<_> { let idx = idx.unwrap_or(inner.backing_objects.len() - 1); let obj = &mut inner.backing_objects[idx]; - let p = obj.0.vmap()?.addr as *mut u8; + let p = obj.0.vmap()?.addr.cast::(); Ok((obj.2, obj.0.size(), p)) })?; assert!(obj_start <= start); diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 5176ffc7010fb..57c3f16e7df2a 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -4,6 +4,7 @@ //! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. mod alloc; +mod binding; mod buffer; mod channel; mod cleanup; @@ -85,6 +86,8 @@ struct DriverModule { impl kernel::InPlaceModule for DriverModule { fn init(module: &'static kernel::ThisModule) -> impl PinInit { + // SAFETY: Module init runs once, before either adapter is registered below. + unsafe { binding::init() }; try_pin_init!(Self { // Initialize the queue first, before registration can invoke probe. _cleanup: cleanup::QueueOwner::new()?, diff --git a/drivers/gpu/drm/asahi/binding.rs b/drivers/gpu/drm/asahi/binding.rs new file mode 100644 index 0000000000000..5e0fac1f29ab2 --- /dev/null +++ b/drivers/gpu/drm/asahi/binding.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scoped DMA-device access during probe and runtime mapping preparation. +//! +//! The platform owner closes admission and joins leases before unbind returns. +//! VMs retain the gate, so an old VM cannot acquire access after removal/rebind. + +use crate::driver::AsahiDrmDevice; +use kernel::{ + device::{Bound, Device}, + new_condvar, new_mutex, + prelude::*, + sync::{aref::ARef, Arc, CondVar, Mutex}, +}; + +kernel::sync::global_lock! { + // SAFETY: Module initialization initializes this lock before registering adapters. + unsafe(uninit) static DEVICES: Mutex>> = KVec::new(); +} + +/// Initialize the registry before a platform adapter can probe. +/// +/// # Safety +/// +/// Called exactly once per module instance, before any other binding operation. +pub(crate) unsafe fn init() { + // SAFETY: The module initializer establishes the single-call ordering above. + unsafe { DEVICES.init() }; +} + +struct State { + live: bool, + users: usize, +} + +#[pin_data] +pub(crate) struct Gate { + key: usize, + dev: ARef, + #[pin] + state: Mutex, + #[pin] + changed: CondVar, +} + +/// The platform binding's owner; Drop also covers errors during probe. +pub(crate) struct Owner(Arc); + +impl Owner { + /// Register while probe supplies the parent bound-device proof. + /// + /// # Safety + /// + /// The platform adapter must call close before unbind returns, independently + /// of this owner's Drop. On probe error the owner must be closed/dropped + /// before the bound probe callback returns. + pub(crate) unsafe fn new(dev: &Device, drm: &AsahiDrmDevice) -> Result { + let gate = Arc::pin_init( + pin_init!(Gate { + key: core::ptr::from_ref(drm) as usize, + dev: dev.into(), + state <- new_mutex!(State { live: true, users: 0 }, "asahi::binding::state"), + changed <- new_condvar!("asahi::binding::changed"), + }), + GFP_KERNEL, + )?; + DEVICES.lock().push(gate.clone(), GFP_KERNEL)?; + Ok(Self(gate)) + } + + /// Called from platform unbind before devres releases the parent's state. + pub(crate) fn close(&self) { + DEVICES.lock().retain(|gate| !Arc::ptr_eq(gate, &self.0)); + let mut state = self.0.state.lock(); + state.live = false; + while state.users != 0 { + self.0.changed.wait(&mut state); + } + } +} + +impl Drop for Owner { + fn drop(&mut self) { + self.close(); + } +} + +/// Look up the binding for this DRM instance, not just its reusable parent address. +pub(crate) fn lookup(drm: &AsahiDrmDevice) -> Result> { + let key = core::ptr::from_ref(drm) as usize; + DEVICES + .lock() + .iter() + .find(|gate| gate.key == key) + .cloned() + .ok_or(ENODEV) +} + +impl Gate { + /// Covers only SG-table acquisition/segment preparation, not firmware waits. + pub(crate) fn enter(&self) -> Result> { + let mut state = self.state.lock(); + if !state.live { + return Err(ENODEV); + } + state.users = state.users.checked_add(1).ok_or(EOVERFLOW)?; + Ok(Lease(self)) + } +} + +pub(crate) struct Lease<'a>(&'a Gate); + +impl Lease<'_> { + pub(crate) fn device(&self) -> &Device { + // SAFETY: Owner starts during bound probe, rejects new leases on close, + // and joins existing leases before unbind returns. This borrow cannot + // outlive its lease; the ARef also retains the device allocation. + unsafe { self.0.dev.as_bound() } + } +} + +impl Drop for Lease<'_> { + fn drop(&mut self) { + let mut state = self.0.state.lock(); + state.users -= 1; + if state.users == 0 { + self.0.changed.notify_all(); + } + } +} diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 2733ed372875a..caf0ef32ca427 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -45,6 +45,7 @@ pub(crate) struct AsahiPlatformDriver; // #[pin_data(PinnedDrop)] pub(crate) struct AsahiPlatformDriverData<'bound> { + binding: crate::binding::Owner, _device: ARef, _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. @@ -225,6 +226,7 @@ impl platform::Driver for AsahiPlatformDriver { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); } + this.close_binding(); } /// Device probe function. @@ -243,6 +245,7 @@ impl platform::Driver for AsahiPlatformDriver { ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), }; + // SAFETY: The selected SoC's UAT output width bounds all device DMA. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; @@ -259,6 +262,9 @@ impl platform::Driver for AsahiPlatformDriver { .required_by(pdev.as_ref())?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { @@ -307,6 +313,7 @@ impl platform::Driver for AsahiPlatformDriver { )?; let driver = AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, @@ -331,6 +338,9 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::g15::Gpu::new(pdev, &drm, cfg)?; let data = try_pin_init!(AsahiDrmRegistrationData { gpu: gpu as Arc, @@ -340,6 +350,7 @@ impl AsahiPlatformDriver { // G15 retains its firmware-visible state independently until reboot, even on error. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, @@ -360,6 +371,9 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res)?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { @@ -375,9 +389,14 @@ impl AsahiPlatformDriver { return Err(error); } }; - // G17's firmware channel format has no supported stats decoder yet. + let stats = gpu::Gpu::stats_snapshot(&*gpu); let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { + let snapshot = stats.as_ref().map_or(core::ptr::null(), Arc::as_ptr); + let raw_stats = match crate::sysfs_exports::register_g17( + raw_dev, + snapshot, + module_parameters::stats_export.value() != 0, + ) { Ok(handle) => handle, Err(error) => { gpu.shutdown(); @@ -394,6 +413,7 @@ impl AsahiPlatformDriver { ); } Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: Some(gpu), @@ -401,7 +421,7 @@ impl AsahiPlatformDriver { g16: None, raw_dev, raw_stats, - stats: None, + stats, }) } } @@ -417,6 +437,9 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G16Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::g16::Gpu::new(pdev, &drm, cfg, res)?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { @@ -428,6 +451,7 @@ impl AsahiPlatformDriver { let raw_dev = pdev.as_ref().as_raw(); let raw_stats = crate::sysfs_exports::register(raw_dev, core::ptr::null(), false)?; Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, @@ -439,8 +463,15 @@ impl AsahiPlatformDriver { } } +impl AsahiPlatformDriverData<'_> { + pub(crate) fn close_binding(&self) { + self.binding.close(); + } +} + impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { + self.binding.close(); // `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index ae3d2db8769ac..244dca39e41ac 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -215,15 +215,15 @@ impl SyncItem { let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; let mut anchor = None; - // SAFETY: We only read this once, so there are no TOCTOU issues. - let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); + // Read each record once so validation and use see the same retained bytes. + let mut reader = UserSlice::new(UserPtr::from_addr(ptr as usize), size).reader(); for i in 0..count { let mut sync: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE reader.read_raw(unsafe { - core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) + core::slice::from_raw_parts_mut(sync.as_mut_ptr().cast::>(), STRIDE) })?; // SAFETY: All bit patterns in the struct are valid @@ -342,6 +342,9 @@ impl File { mem: Arc, usage: Arc, ) -> impl PinInit { + // SAFETY: The initializer supplies aligned, pinned storage for Self. + // XArray::new is infallible; each pinned array is initialized in place, + // then every remaining field is written before returning success. unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); @@ -462,13 +465,13 @@ impl File { let size = core::mem::size_of::().min(data.size.try_into()?); - // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. + // Write the retained result to this user slice once. let mut params_writer = UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { - core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) + core::slice::from_raw_parts(core::ptr::from_ref(¶ms).cast::(), size) })?; Ok(0) @@ -1343,9 +1346,9 @@ impl File { ); return Err(EINVAL); } - // SAFETY: capable() only reads the calling task's credentials (ioctl process context). if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as _) } + // SAFETY: capable() reads the current ioctl task's credentials. + && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as i32) } && !file.is_current_master() { cls_pr_debug!( diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index 224e92c6d97e2..52261d10757d1 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -140,12 +140,12 @@ macro_rules! from_ints { ($u:ty, $i:ty) => { impl From<$i> for F32 { fn from(v: $i) -> F32 { - F32U::from_i64(v as i64).pack() + F32U::from_i64(i64::from(v)).pack() } } impl From<$u> for F32 { fn from(v: $u) -> F32 { - F32U::from_u64(v as u64).pack() + F32U::from_u64(u64::from(v)).pack() } } }; diff --git a/drivers/gpu/drm/asahi/fw/event.rs b/drivers/gpu/drm/asahi/fw/event.rs index 52bc456f58707..e26bc6ceb3448 100644 --- a/drivers/gpu/drm/asahi/fw/event.rs +++ b/drivers/gpu/drm/asahi/fw/event.rs @@ -73,7 +73,7 @@ pub(crate) mod raw { // We could use fetch_add, but the non-LSE atomic // sequence Rust produces confuses the hypervisor. let v = self.0.load(Ordering::Relaxed); - self.0.store(v + (amount as u64), Ordering::Relaxed); + self.0.store(v + u64::from(amount), Ordering::Relaxed); } } diff --git a/drivers/gpu/drm/asahi/fw/types.rs b/drivers/gpu/drm/asahi/fw/types.rs index f55d6cec6b8ca..11def551029f5 100644 --- a/drivers/gpu/drm/asahi/fw/types.rs +++ b/drivers/gpu/drm/asahi/fw/types.rs @@ -117,6 +117,8 @@ macro_rules! default_zeroed { (<$($lt:lifetime),*>, $type:ty) => { impl<$($lt),*> Default for $type { fn default() -> $type { + // SAFETY: The macro's contract requires every field of this + // type to admit the all-zero bit pattern. unsafe { core::mem::zeroed() } } } @@ -126,6 +128,8 @@ macro_rules! default_zeroed { ($type:ty) => { impl Default for $type { fn default() -> $type { + // SAFETY: The macro's contract requires every field of this + // type to admit the all-zero bit pattern. unsafe { core::mem::zeroed() } } } @@ -144,6 +148,7 @@ unsafe impl Zeroable for Pad {} impl Default for Pad { fn default() -> Self { + // SAFETY: The wrapper contains only an array of bytes. unsafe { core::mem::zeroed() } } } @@ -170,6 +175,7 @@ unsafe impl Zeroable for Array {} impl Default for Array { fn default() -> Self { + // SAFETY: Every element admits the all-zero bit pattern through T: Zeroable. unsafe { core::mem::zeroed() } } } diff --git a/drivers/gpu/drm/asahi/fwid.rs b/drivers/gpu/drm/asahi/fwid.rs index be81eac5dfb74..34c65a0d8a097 100644 --- a/drivers/gpu/drm/asahi/fwid.rs +++ b/drivers/gpu/drm/asahi/fwid.rs @@ -50,7 +50,7 @@ impl fmt::Display for Hex<'_> { pub(crate) fn copy_physical(phys: u64, size: usize) -> Result> { // SAFETY: memremap() of a boot-loader range creates a temporary mapping; it has no other // side effects. - let map = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WC as _) }; + let map = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WC as usize) }; if map.is_null() { return Err(ENOMEM); } diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index c792c1303475e..e39180f02248a 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -176,6 +176,12 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { + if let Some(driver) = &this._driver { + driver.close_binding(); + } + } + /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true diff --git a/drivers/gpu/drm/asahi/g16/adt.rs b/drivers/gpu/drm/asahi/g16/adt.rs index a34ecc901f615..804703b19cecc 100644 --- a/drivers/gpu/drm/asahi/g16/adt.rs +++ b/drivers/gpu/drm/asahi/g16/adt.rs @@ -94,7 +94,7 @@ fn performance_source(regions: &[Range]) -> Result<(KVVec, u64)> { let (start, size) = source.ok_or(ENODEV)?; // SAFETY: The sole accepted DT source above is the bounded reserved original ADT export. // WB matches its System RAM mapping; no writes are made through this temporary mapping. - let mapping = unsafe { bindings::memremap(start, size, bindings::MEMREMAP_WB as _) }; + let mapping = unsafe { bindings::memremap(start, size, bindings::MEMREMAP_WB as usize) }; if mapping.is_null() { return Err(ENOMEM); } diff --git a/drivers/gpu/drm/asahi/g16/boot.rs b/drivers/gpu/drm/asahi/g16/boot.rs index 450d1c25268ab..e43778bb02a1f 100644 --- a/drivers/gpu/drm/asahi/g16/boot.rs +++ b/drivers/gpu/drm/asahi/g16/boot.rs @@ -147,6 +147,10 @@ unsafe fn shared<'a>(cookie: *mut c_void) -> ArcBorrow<'a, Shared> { } /// RTKit crash callback: latches fail-stop. +/// +/// # Safety +/// +/// `cookie` must be the live Shared cookie registered with these RTKit operations. unsafe extern "C" fn rtkit_crashed(cookie: *mut c_void, _log: *const c_void, _size: usize) { // SAFETY: RTKit passes back the registered cookie. let shared = unsafe { shared(cookie) }; @@ -158,6 +162,10 @@ unsafe extern "C" fn rtkit_crashed(cookie: *mut c_void, _log: *const c_void, _si } /// RTKit message callback. +/// +/// # Safety +/// +/// `cookie` must be the live Shared cookie registered with these RTKit operations. unsafe extern "C" fn rtkit_recv(cookie: *mut c_void, endpoint: u8, message: u64) { // SAFETY: RTKit passes back the registered cookie. let shared = unsafe { shared(cookie) }; @@ -179,6 +187,10 @@ unsafe extern "C" fn rtkit_recv(cookie: *mut c_void, endpoint: u8, message: u64) /// EP 0x20 ring-service request, in the mailbox interrupt: wake only. The drain and the /// backend's completion check run in the poll work; the shared state outlives the mailbox. +/// +/// # Safety +/// +/// `cookie` must be the live Shared cookie registered with these RTKit operations. unsafe extern "C" fn rtkit_recv_early(cookie: *mut c_void, endpoint: u8, message: u64) -> bool { // SAFETY: RTKit passes back the registered cookie. let shared = unsafe { shared(cookie) }; @@ -194,6 +206,11 @@ unsafe extern "C" fn rtkit_recv_early(cookie: *mut c_void, endpoint: u8, message } /// Only the inherited crash buffer inside this session's firmware DATA, once. +/// +/// # Safety +/// +/// `cookie` must be the registered live Shared cookie. `bfr` must point to the +/// writable RTKit buffer descriptor, exclusively borrowed for this callback. unsafe extern "C" fn rtkit_shmem_setup( cookie: *mut c_void, endpoint: u8, @@ -203,7 +220,7 @@ unsafe extern "C" fn rtkit_shmem_setup( let shared = unsafe { shared(cookie) }; // SAFETY: RTKit passes a valid buffer descriptor it owns for the duration of the call. let bfr = unsafe { &mut *bfr }; - let (iova, size) = (bfr.iova as u64, bfr.size as u64); + let (iova, size) = (bfr.iova, bfr.size as u64); let data = &shared.data; let data_size = data.end - data.start; if endpoint != 1 @@ -245,6 +262,10 @@ unsafe extern "C" fn rtkit_shmem_setup( } /// Retained until reboot: no qualified stop path. +/// +/// # Safety +/// +/// This no-op callback accesses neither pointer and releases no storage. unsafe extern "C" fn rtkit_shmem_destroy( _cookie: *mut c_void, _bfr: *mut bindings::apple_rtkit_shmem, @@ -436,8 +457,13 @@ pub(crate) fn admit( } let adt = adt::read(dev).inspect_err(|e| dev_err!(dev, "ADT GPU regions not admitted: {:?}\n", e))?; - crate::fwid::identify(dev, adt.text.clone(), adt.data.end - adt.data.start, cfg.firmware) - .inspect_err(|e| dev_err!(dev, "loaded GFX TEXT identity mismatch: {:?}\n", e))?; + crate::fwid::identify( + dev, + adt.text.clone(), + adt.data.end - adt.data.start, + cfg.firmware, + ) + .inspect_err(|e| dev_err!(dev, "loaded GFX TEXT identity mismatch: {:?}\n", e))?; dev_info!(dev, "M4: firmware identity accepted\n"); // Firmware objects are cleaned to the point of coherency by their DMA mapping: that needs // a device that is not DMA-coherent, as the synthesized node is. diff --git a/drivers/gpu/drm/asahi/g16/encode.rs b/drivers/gpu/drm/asahi/g16/encode.rs index 370f1848e1cb7..a9bf84943b805 100644 --- a/drivers/gpu/drm/asahi/g16/encode.rs +++ b/drivers/gpu/drm/asahi/g16/encode.rs @@ -118,23 +118,27 @@ impl Geometry { /// The geometry of render `u` with layer metadata at `heap`. The render was validated /// (width, height, layers >= 1, samples 1, 2 or 4, utiles 16 or 32). pub(crate) fn new(u: &uapi::drm_asahi_cmd_render, heap: u64) -> Self { - let (w, h, l) = (u.width_px as u32, u.height_px as u32, u.layers as u32); + let (w, h, l) = ( + u32::from(u.width_px), + u32::from(u.height_px), + u32::from(u.layers), + ); let empty = u.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES != 0; - let log2s = (u.samples as u32).max(1).ilog2(); + let log2s = u32::from(u.samples).max(1).ilog2(); let mut g = Geometry::default(); - g.tx = div_round_up(w as u64, 32) as u32; - g.ty = div_round_up(h as u64, 32) as u32; - g.bx = align(div_round_up(g.tx as u64, 4), 4) as u32; - g.by = align(div_round_up(g.ty as u64, 4), 4) as u32; + g.tx = div_round_up(u64::from(w), 32) as u32; + g.ty = div_round_up(u64::from(h), 32) as u32; + g.bx = align(div_round_up(u64::from(g.tx), 4), 4) as u32; + g.by = align(div_round_up(u64::from(g.ty), 4), 4) as u32; g.sx = g.bx * if u.utile_width_px == 16 { 2 } else { 1 }; g.sy = g.by * if u.utile_height_px == 16 { 2 } else { 1 }; - g.e = (4 * g.sx as u64) * (4 * g.sy as u64); + g.e = (4 * u64::from(g.sx)) * (4 * u64::from(g.sy)); g.pixels = w.wrapping_sub(1) | h.wrapping_sub(1) << 16; g.screen = g.tx.wrapping_sub(1) | g.ty.wrapping_sub(1) << 12; - g.x_blocks = (g.bx as u64) << 10 | (g.bx as u64) << 18 | 3 * g.bx as u64; - g.y_blocks = (g.by as u64) << 18 | (g.by as u64) << 10 | 3 * g.by as u64; + g.x_blocks = u64::from(g.bx) << 10 | u64::from(g.bx) << 18 | (3 * u64::from(g.bx)); + g.y_blocks = u64::from(g.by) << 18 | u64::from(g.by) << 10 | (3 * u64::from(g.by)); g.blocks = g.bx * g.by; g.tiles = g.tx * g.ty; g.region_stride = (5 * g.e / 64) as u32; @@ -151,8 +155,8 @@ impl Geometry { 0x4000 }; g.utile_config = utile_w | utile_h | log2s; - g.tilemap_bytes = align(5 * g.e * l as u64, 0x1000); - g.tpc_bytes = g.e * 8 * l as u64; + g.tilemap_bytes = align(5 * g.e * u64::from(l), 0x1000); + g.tpc_bytes = g.e * 8 * u64::from(l); g.layer_metadata = heap + if l > 1 { 0x200 } else { 0 }; g.layer_mode = if l == 1 { 0x8000 @@ -160,21 +164,21 @@ impl Geometry { 0xc000 | (if empty { 0x2000 } else { 0x1000 }) | l.wrapping_sub(1) }; g.tilebuffer_control = div_round_up( - u.sample_size_B as u64 - * u.utile_width_px as u64 - * u.utile_height_px as u64 - * u.samples as u64, + u64::from(u.sample_size_B) + * u64::from(u.utile_width_px) + * u64::from(u.utile_height_px) + * u64::from(u.samples), 2048, ) as u32 | log2s << 17; - g.tile_mode = 0x280 | (l > 1) as u32 | if empty { 1 << 16 } else { 0 }; + g.tile_mode = 0x280 | u32::from(l > 1) | if empty { 1 << 16 } else { 0 }; g.isp_control = 0xc000 | (u.flags & (1 << 18)); g.native_tiles = 0x3717fu64 - | ((l > 1) as u64) << 32 - | (g.tx.wrapping_sub(1) as u64) << 44 - | (g.ty.wrapping_sub(1) as u64) << 53 + | u64::from(l > 1) << 32 + | u64::from(g.tx.wrapping_sub(1)) << 44 + | u64::from(g.ty.wrapping_sub(1)) << 53 | 137438953472u64 - | ((g.utile_config & 0xf000) as u64) << 28; + | u64::from(g.utile_config & 0xf000) << 28; g } } @@ -223,7 +227,7 @@ pub(crate) fn cluster_eligible(_u: &uapi::drm_asahi_cmd_render, _geo: &Geometry) /// Program address. pub(crate) fn prog_addr(usc_base: u64, p: &uapi::drm_asahi_bg_eot) -> u64 { - usc_base.wrapping_add((p.usc & !63) as u64) + usc_base.wrapping_add(u64::from(p.usc & !63)) } /// Program resource word, 64-bit with RSRC_SPEC_HI. @@ -233,9 +237,9 @@ pub(crate) fn prog_rsrc( hi: u32, ) -> u64 { if u.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI != 0 { - (hi as u64) << 32 | p.rsrc_spec as u64 + u64::from(hi) << 32 | u64::from(p.rsrc_spec) } else { - p.rsrc_spec as u64 + u64::from(p.rsrc_spec) } } diff --git a/drivers/gpu/drm/asahi/g16/encode/compute.rs b/drivers/gpu/drm/asahi/g16/encode/compute.rs index 4089533f67ef7..08a4f9ee9cefd 100644 --- a/drivers/gpu/drm/asahi/g16/encode/compute.rs +++ b/drivers/gpu/drm/asahi/g16/encode/compute.rs @@ -221,7 +221,7 @@ pub(crate) fn encode_compute( let start = a.pointer & !127; if a.size == 0 || end <= start - || (end - start) / 128 > u32::MAX as u64 + || (end - start) / 128 > u64::from(u32::MAX) || !gpu(start, end - start, 128) { return Err(ComputeEncodeError::Attachment); diff --git a/drivers/gpu/drm/asahi/g16/encode/render.rs b/drivers/gpu/drm/asahi/g16/encode/render.rs index ec143a818c46e..b054ec08198b7 100644 --- a/drivers/gpu/drm/asahi/g16/encode/render.rs +++ b/drivers/gpu/drm/asahi/g16/encode/render.rs @@ -112,7 +112,7 @@ fn timestamps( put64(q, base + 0x24, if user { work + user_pair } else { 0 }); put64(q, base + 0x2c, work + auxiliary); put64(q, base + 0x34, work + context); - put64(q, base + 0x44, uuid as u64); + put64(q, base + 0x44, u64::from(uuid)); } put32(q, off + 0x4c, 1); } @@ -160,7 +160,7 @@ pub(crate) fn encode_ta( compact(p.heap_gva) | 1 << 63, 0x003a0012006b0003, 1, - geo.utile_config as u64, + u64::from(geo.utile_config), u.ppp_multisamplectl, compact(p.pre_gva[0]), compact(p.pre_gva[0]), @@ -179,18 +179,18 @@ pub(crate) fn encode_ta( 0, 0, 0x8860, - geo.region_stride as u64, - geo.region_stride as u64, + u64::from(geo.region_stride), + u64::from(geo.region_stride), 0x88, - (2 | (u.ppp_ctrl & 0x201)) as u64, - geo.pixels as u64, - geo.screen as u64, + u64::from(2 | (u.ppp_ctrl & 0x201)), + u64::from(geo.pixels), + u64::from(geo.screen), geo.x_blocks, geo.y_blocks, - geo.blocks as u64, - geo.tpc_stride as u64, + u64::from(geo.blocks), + u64::from(geo.tpc_stride), 0x100, - geo.layer_mode as u64, + u64::from(geo.layer_mode), 0, u64::MAX, u64::MAX, @@ -207,15 +207,15 @@ pub(crate) fn encode_ta( 0, 8, 0xff0000, - r.parameter_id as u64, + u64::from(r.parameter_id), compact(r.scene_list), compact(r.scene_list), scratch_word, 0x6000400020, 0x200000001, - r.uuid as u64, - r.uuid as u64, - r.uuid as u64, + u64::from(r.uuid), + u64::from(r.uuid), + u64::from(r.uuid), // USC base is a shared queue property carried by the render refs. r.usc_base, ]; @@ -307,33 +307,33 @@ pub(crate) fn encode_fs( q.fill(0); let val: [u64; 82] = [ 1, - geo.utile_config as u64, + u64::from(geo.utile_config), prog_rsrc(u, &u.eot, u.bg_eot_rsrc_spec_hi), prog_addr(usc, &u.eot), prog_rsrc(u, &u.bg, u.bg_rsrc_spec_hi), prog_addr(usc, &u.bg), - u.isp_merge_upper_x as u64, - u.isp_merge_upper_y as u64, + u64::from(u.isp_merge_upper_x), + u64::from(u.isp_merge_upper_y), 0, 0, 0, 0, u.ppp_multisamplectl, - geo.isp_blocks as u64, - geo.isp_blocks as u64, - geo.screen as u64, + u64::from(geo.isp_blocks), + u64::from(geo.isp_blocks), + u64::from(geo.screen), 0, 0xff0200, p.heap_gva & !63, u.isp_scissor_base & !3, u.isp_dbias_base & !3, - geo.isp_control as u64, - (u.height_px as u64) << 32 | u.width_px as u64, + u64::from(geo.isp_control), + u64::from(u.height_px) << 32 | u64::from(u.width_px), 0x100000, - geo.tilebuffer_control as u64, - u.isp_zls_pixels as u64, - u.isp_bgobjdepth as u64, - u.isp_bgobjvals as u64, + u64::from(geo.tilebuffer_control), + u64::from(u.isp_zls_pixels), + u64::from(u.isp_bgobjdepth), + u64::from(u.isp_bgobjvals), u.isp_oclqry_base & !15, u.zls_ctrl, 0x04040404, @@ -346,10 +346,10 @@ pub(crate) fn encode_fs( 0, 0, 0, - u.depth.stride as u64, - u.depth.stride as u64, - u.stencil.stride as u64, - u.stencil.stride as u64, + u64::from(u.depth.stride), + u64::from(u.depth.stride), + u64::from(u.stencil.stride), + u64::from(u.stencil.stride), 0, 0, 0, @@ -360,8 +360,8 @@ pub(crate) fn encode_fs( 0, p.tilemap_gva, geo.layer_metadata, - (geo.region_stride as u64) << 26, - geo.tile_mode as u64, + u64::from(geo.region_stride) << 26, + u64::from(geo.tile_mode), 0, 0, 0, @@ -382,10 +382,10 @@ pub(crate) fn encode_fs( 8, 0x6000400020, 0x200000001, - r.uuid as u64, - r.uuid as u64, - r.uuid as u64, - r.parameter_id as u64, + u64::from(r.uuid), + u64::from(r.uuid), + u64::from(r.uuid), + u64::from(r.parameter_id), compact(r.scene_list), usc, ]; @@ -423,18 +423,22 @@ pub(crate) fn encode_fs( put64(w, 0x938, u.zls_ctrl); put64(w, 0x940, 0x04040404); put64(w, 0x948, u.depth.base); - put64(w, 0x950, u.depth.stride as u64); + put64(w, 0x950, u64::from(u.depth.stride)); put64(w, 0x960, u.depth.base); put64(w, 0x968, u.depth.base); put64(w, 0x978, u.stencil.base); - put64(w, 0x980, u.stencil.stride as u64); + put64(w, 0x980, u64::from(u.stencil.stride)); put64(w, 0x990, u.stencil.base); put64(w, 0x998, u.stencil.base); - put64(w, 0x9b8, geo.tilebuffer_control as u64); - put64(w, 0x9c0, geo.isp_control as u64); - put64(w, 0x9c8, (u.height_px as u64) << 32 | u.width_px as u64); + put64(w, 0x9b8, u64::from(geo.tilebuffer_control)); + put64(w, 0x9c0, u64::from(geo.isp_control)); + put64( + w, + 0x9c8, + u64::from(u.height_px) << 32 | u64::from(u.width_px), + ); put64(w, 0x9d0, 0x100000); - put64(w, 0x9d8, geo.tile_mode as u64); + put64(w, 0x9d8, u64::from(geo.tile_mode)); put64(w, 0x9f4, prog_rsrc(u, &u.eot, u.bg_eot_rsrc_spec_hi)); put64(w, 0x9fc, prog_addr(usc, &u.eot)); put64( @@ -446,9 +450,9 @@ pub(crate) fn encode_fs( put64( w, 0xa28, - (u.isp_bgobjvals as u64) << 32 | u.isp_bgobjdepth as u64, + u64::from(u.isp_bgobjvals) << 32 | u64::from(u.isp_bgobjdepth), ); - put64(w, 0xa50, u.isp_zls_pixels as u64); + put64(w, 0xa50, u64::from(u.isp_zls_pixels)); metadata(w, 0xbc0, r, 1, target); internal_pair(w, p, 1, ts_va); put64(w, 0xc46, r.uma_mgr[0]); diff --git a/drivers/gpu/drm/asahi/g16/engine.rs b/drivers/gpu/drm/asahi/g16/engine.rs index 6e5700b78088e..a1202afb25f0d 100644 --- a/drivers/gpu/drm/asahi/g16/engine.rs +++ b/drivers/gpu/drm/asahi/g16/engine.rs @@ -1096,7 +1096,7 @@ impl Firmware { kind: channel as u32, head: head as u16, event, - new_queue: fresh as u8, + new_queue: u8::from(fresh), }; self.channels.post(channel, &record)?; self.session.ring(channel)?; diff --git a/drivers/gpu/drm/asahi/g16/event.rs b/drivers/gpu/drm/asahi/g16/event.rs index 4fabea819b854..26ef79a2c3a9a 100644 --- a/drivers/gpu/drm/asahi/g16/event.rs +++ b/drivers/gpu/drm/asahi/g16/event.rs @@ -30,7 +30,7 @@ pub(crate) struct ErrorEvent { impl ErrorEvent { pub(crate) fn generation(&self) -> u64 { - self.raw[1] as u64 | ((self.raw[2] as u64) << 32) + u64::from(self.raw[1]) | (u64::from(self.raw[2]) << 32) } } diff --git a/drivers/gpu/drm/asahi/g16/fw/channels.rs b/drivers/gpu/drm/asahi/g16/fw/channels.rs index 87b207f5b5ea7..9b5ae1e7c4b08 100644 --- a/drivers/gpu/drm/asahi/g16/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g16/fw/channels.rs @@ -34,7 +34,7 @@ const _: () = assert!(core::mem::offset_of!(SubmitRecord, new_queue) == 0x17); impl SubmitRecord { /// The little-endian record bytes. - pub(crate) fn to_bytes(&self) -> [u8; 0x18] { + pub(crate) fn to_bytes(self) -> [u8; 0x18] { let mut b = [0u8; 0x18]; b[0..8].copy_from_slice(&self.unk_0.to_le_bytes()); b[8..16].copy_from_slice(&self.qinfo.to_le_bytes()); @@ -80,7 +80,7 @@ impl DevCtl { } /// The little-endian record bytes. - pub(crate) fn to_bytes(&self) -> [u8; 0x40] { + pub(crate) fn to_bytes(self) -> [u8; 0x40] { let mut b = [0u8; 0x40]; for (i, w) in self.words.iter().enumerate() { b[4 * i..4 * i + 4].copy_from_slice(&w.to_le_bytes()); diff --git a/drivers/gpu/drm/asahi/g16/fw/initdata.rs b/drivers/gpu/drm/asahi/g16/fw/initdata.rs index 40fb02cd3bda5..9acbb4ac763a5 100644 --- a/drivers/gpu/drm/asahi/g16/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g16/fw/initdata.rs @@ -272,7 +272,11 @@ pub(crate) fn loader_span(slot: usize, bank: usize) -> Result<(u64, u32, u32), ( return Err(()); } let offset = e.pa & (PAGE - 1); - let bytes = if slot == 3 { 0x18000 } else { e.length as u64 }; + let bytes = if slot == 3 { + 0x18000 + } else { + u64::from(e.length) + }; let size = (offset + bytes + PAGE - 1) & !(PAGE - 1); Ok(( (e.pa & PA_MASK) + if bank != 0 { 0x2000000 } else { 0 }, @@ -305,9 +309,8 @@ pub(crate) fn loader_encode(slot: usize, va: u64, record: &mut [u8]) -> Result<( return Ok(()); } let offset = e.pa & (PAGE - 1); - let span = (offset + e.length as u64 + PAGE - 1) & !(PAGE - 1); - if va < HOST_START - || va >= HOST_END + let span = (offset + u64::from(e.length) + PAGE - 1) & !(PAGE - 1); + if !(HOST_START..HOST_END).contains(&va) || va & (PAGE - 1) != offset || span > HOST_END - (va - offset) { @@ -318,7 +321,7 @@ pub(crate) fn loader_encode(slot: usize, va: u64, record: &mut [u8]) -> Result<( put32(out, 0x10, e.length)?; put32(out, 0x14, e.requested)?; put64(out, 0x18, e.tag)?; - put64(out, 0x20, e.mode as u64) + put64(out, 0x20, u64::from(e.mode)) } pub(crate) const PERF_STATES: usize = 10; @@ -371,10 +374,11 @@ pub(crate) struct Input { } pub(crate) fn object_va(va: u64, size: usize) -> bool { - va >= HOST_START && va < HOST_END && va & 7 == 0 && size != 0 && size as u64 <= HOST_END - va + (HOST_START..HOST_END).contains(&va) && va & 7 == 0 && size != 0 && size as u64 <= HOST_END - va } fn bootstrap_alias(va: u64) -> bool { - va >= BOOTSTRAP_START && va <= BOOTSTRAP_START + BOOTSTRAP_SIZE - 0x8000 && va & (PAGE - 1) == 0 + (BOOTSTRAP_START..=BOOTSTRAP_START + BOOTSTRAP_SIZE - 0x8000).contains(&va) + && va & (PAGE - 1) == 0 } pub(crate) fn encode_root(out: &mut [u8], va: &[u64; OBJECT_COUNT]) -> Result<(), ()> { @@ -410,7 +414,7 @@ pub(crate) fn encode_root(out: &mut [u8], va: &[u64; OBJECT_COUNT]) -> Result<() out[at..at + 8].copy_from_slice(&[8, 14, 14, *shift, c[0], c[1], 0, 0x40]); put64(out, at + 8, 1)?; put64(out, at + 0x10, PA_MASK)?; - put64(out, at + 0x18, (*count as u64 - 1) << shift)?; + put64(out, at + 0x18, (u64::from(*count) - 1) << shift)?; } Ok(()) } diff --git a/drivers/gpu/drm/asahi/g16/initdata.rs b/drivers/gpu/drm/asahi/g16/initdata.rs index 24e7f1f83b78c..ecdde0579133a 100644 --- a/drivers/gpu/drm/asahi/g16/initdata.rs +++ b/drivers/gpu/drm/asahi/g16/initdata.rs @@ -158,11 +158,12 @@ pub(crate) fn build( continue; } let offset = entry.pa & (wire::PAGE - 1); - let at = objects.reserve_io((offset + entry.length as u64).next_multiple_of(wire::PAGE))?; + let at = + objects.reserve_io((offset + u64::from(entry.length)).next_multiple_of(wire::PAGE))?; input.loader_va[slot] = at + offset; for bank in 0..if slot == 3 { 2 } else { 1 } { - let (pa, size, delta) = wire::loader_span(slot, bank).map_err(|_| EINVAL)?; - objects.map_mmio_class(at + delta as u64, pa, size as u64, entry.attrs)?; + let (pa, size, delta) = wire::loader_span(slot, bank).map_err(|()| EINVAL)?; + objects.map_mmio_class(at + u64::from(delta), pa, u64::from(size), entry.attrs)?; } } input.scratch = objects.reserve_io(0x8000)?; @@ -183,7 +184,7 @@ pub(crate) fn build( } let array: &mut [&mut [u8]; wire::OBJECT_COUNT] = (&mut slices[..]).try_into().map_err(|_| EINVAL)?; - wire::encode(array, &input, 9).map_err(|_| EINVAL)?; + wire::encode(array, &input, 9).map_err(|()| EINVAL)?; } let mut loader = [0u8; wire::LOADER_BYTES]; loader.copy_from_slice(&images[id::MAIN][0x640..0x640 + wire::LOADER_BYTES]); diff --git a/drivers/gpu/drm/asahi/g16/job.rs b/drivers/gpu/drm/asahi/g16/job.rs index fb22514e1c61b..9787907e68c0d 100644 --- a/drivers/gpu/drm/asahi/g16/job.rs +++ b/drivers/gpu/drm/asahi/g16/job.rs @@ -121,6 +121,10 @@ pub(crate) struct Job { } impl Job { + #[expect( + clippy::too_many_arguments, + reason = "The job owns independently prepared command, mapping and fence resources." + )] pub(crate) fn new( id: u64, queue: Arc, diff --git a/drivers/gpu/drm/asahi/g16/mod.rs b/drivers/gpu/drm/asahi/g16/mod.rs index 3ee9b66802b54..ef1e6f360db31 100644 --- a/drivers/gpu/drm/asahi/g16/mod.rs +++ b/drivers/gpu/drm/asahi/g16/mod.rs @@ -171,6 +171,7 @@ impl platform::Driver for Driver { if let Some(gpu) = &driver.g16 { gpu.retain(); } + driver.close_binding(); } } } diff --git a/drivers/gpu/drm/asahi/g16/object.rs b/drivers/gpu/drm/asahi/g16/object.rs index 11072b883e288..0f26cb3276738 100644 --- a/drivers/gpu/drm/asahi/g16/object.rs +++ b/drivers/gpu/drm/asahi/g16/object.rs @@ -175,7 +175,7 @@ impl FwObject { let p = self.ptr(offset, 4)?; if p as usize % 4 == 0 { // SAFETY: `p` is valid and aligned for a u32. - unsafe { (p as *mut u32).write_volatile(value.to_le()) }; + unsafe { p.cast::().write_volatile(value.to_le()) }; Ok(()) } else { self.write(offset, &value.to_le_bytes()) @@ -187,7 +187,7 @@ impl FwObject { let p = self.ptr(offset, 8)?; if p as usize % 8 == 0 { // SAFETY: `p` is valid and aligned for a u64. - unsafe { (p as *mut u64).write_volatile(value.to_le()) }; + unsafe { p.cast::().write_volatile(value.to_le()) }; Ok(()) } else { self.write(offset, &value.to_le_bytes()) @@ -624,8 +624,8 @@ impl Objects { let bound = unsafe { self.dev.as_ref().as_ref().as_bound() }; let sgt = gem::held_sg_table(&backing.gem, bound)?; for range in sgt.iter() { - let pa = range.dma_address() as u64; - let len = range.dma_len() as u64; + let pa = range.dma_address(); + let len = range.dma_len(); // EPERM for a page the UAT cannot name. if pa == 0 || (pa | len) & (hw::PAGE - 1) != 0 diff --git a/drivers/gpu/drm/asahi/g16/params.rs b/drivers/gpu/drm/asahi/g16/params.rs index 90540338f39a2..d2ef01e26b3a3 100644 --- a/drivers/gpu/drm/asahi/g16/params.rs +++ b/drivers/gpu/drm/asahi/g16/params.rs @@ -60,8 +60,8 @@ pub(crate) fn fill(gpu: &Gpu, params: &mut uapi::drm_asahi_params_global) -> Res if gpu.cfg.chip_id != 0x8132 || gpu.cfg.gpu_revision != 0x10 { return Err(ENODEV); } - params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64; + params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS); params.gpu_generation = GPU_GENERATION; params.gpu_variant = gpu.cfg.gpu_variant as u32; params.gpu_revision = gpu.cfg.gpu_revision; diff --git a/drivers/gpu/drm/asahi/g16/poll.rs b/drivers/gpu/drm/asahi/g16/poll.rs index 9dd6c965da6c4..018c335a930b0 100644 --- a/drivers/gpu/drm/asahi/g16/poll.rs +++ b/drivers/gpu/drm/asahi/g16/poll.rs @@ -172,7 +172,7 @@ impl Shared { // Dropping the previous handle cancels a still pending expiry before the timer // is started again. drop(handle.take()); - *handle = Some(self.clone().start(Delta::from_micros(us as i64))); + *handle = Some(self.clone().start(Delta::from_micros(i64::from(us)))); } self.mod_poll(1); } @@ -572,8 +572,8 @@ impl Firmware { let s = &mut self.engine.stats; if result.is_ok() { s.jobs += 1; - s.job_cmds += p.count() as u64; - s.job_max = s.job_max.max(p.count() as u64); + s.job_cmds += u64::from(p.count()); + s.job_max = s.job_max.max(u64::from(p.count())); } complete(done, &job, Ok(())); // The rest of a compute horizon, after the first job (fence order). @@ -584,8 +584,8 @@ impl Firmware { Err(error) => complete(done, j2, Err(error)), Ok(()) => { s.jobs += 1; - s.job_cmds += p2.count() as u64; - s.job_max = s.job_max.max(p2.count() as u64); + s.job_cmds += u64::from(p2.count()); + s.job_max = s.job_max.max(u64::from(p2.count())); } } complete(done, j2, Ok(())); @@ -761,8 +761,8 @@ impl Firmware { s.first_ns[kind] += first - f.published_ns; s.last_ns[kind] += last - f.published_ns; } - s.count[0] += f.computes as u64; - s.count[1] += f.renders as u64; + s.count[0] += u64::from(f.computes); + s.count[1] += u64::from(f.renders); if kind == 2 { s.count[2] += f.count as u64; } diff --git a/drivers/gpu/drm/asahi/g16/pools.rs b/drivers/gpu/drm/asahi/g16/pools.rs index 1b0406aae6d51..d4a85cc6f3d36 100644 --- a/drivers/gpu/drm/asahi/g16/pools.rs +++ b/drivers/gpu/drm/asahi/g16/pools.rs @@ -99,7 +99,7 @@ fn read64(o: &FwObject, at: usize) -> Result { let high = o.r32(at + 4)?; let low = o.r32(at)?; if high == o.r32(at + 4)? { - return Ok((high as u64) << 32 | low as u64); + return Ok(u64::from(high) << 32 | u64::from(low)); } } Err(EAGAIN) @@ -347,7 +347,7 @@ impl PbEpoch { buffer::page_record(heap, i as u32).to_le_bytes() })?; write_records(&self.blocks, buffer::MAX_BLOCKS as usize, |i| { - (buffer::block_record(heap, i as u32) as u64).to_le_bytes() + u64::from(buffer::block_record(heap, i as u32)).to_le_bytes() })?; self.ring.w32(0, buffer::MAX_BLOCKS)?; self.ring.w32(4, buffer::MAX_BLOCKS)?; diff --git a/drivers/gpu/drm/asahi/g16/pools/lifetime.rs b/drivers/gpu/drm/asahi/g16/pools/lifetime.rs index 58d482f1ea94b..baf623b2722d7 100644 --- a/drivers/gpu/drm/asahi/g16/pools/lifetime.rs +++ b/drivers/gpu/drm/asahi/g16/pools/lifetime.rs @@ -66,7 +66,7 @@ impl Lifetime { State::Retired => counts.retired += 1, State::Quarantined => counts.quarantined += 1, } - counts.reusable += self.available(k) as u32; + counts.reusable += u32::from(self.available(k)); } counts } diff --git a/drivers/gpu/drm/asahi/g16/recovery.rs b/drivers/gpu/drm/asahi/g16/recovery.rs index cc3f3525c101d..e15590048dacb 100644 --- a/drivers/gpu/drm/asahi/g16/recovery.rs +++ b/drivers/gpu/drm/asahi/g16/recovery.rs @@ -126,8 +126,7 @@ impl Firmware { } pub(crate) fn recovery_pending(&self) -> Result { - Ok(self.recovery.running || self.new_recovery() - || self.firmware_r32(logic::PHASE)? != 0) + Ok(self.recovery.running || self.new_recovery() || self.firmware_r32(logic::PHASE)? != 0) } pub(crate) fn recovery_idle(&self) -> Result { @@ -289,7 +288,7 @@ impl Firmware { // The firmware is stopped while its published generation is consumed. let lo = self.firmware_r32(logic::GENERATION)?; let hi = self.firmware_r32(logic::GENERATION + 4)?; - Ok(lo as u64 | ((hi as u64) << 32)) + Ok(u64::from(lo) | (u64::from(hi) << 32)) } /// Resolve the snapshot while the firmware is stopped. A missing retained diff --git a/drivers/gpu/drm/asahi/g16/recovery/logic.rs b/drivers/gpu/drm/asahi/g16/recovery/logic.rs index 2d29a39337df8..4d8bcb716eda5 100644 --- a/drivers/gpu/drm/asahi/g16/recovery/logic.rs +++ b/drivers/gpu/drm/asahi/g16/recovery/logic.rs @@ -142,7 +142,7 @@ fn host_deadline(now: u64, milliseconds: u32) -> u64 { if milliseconds == 0 { u64::MAX } else { - now.saturating_add(milliseconds.max(100) as u64 * 1_000_000) + now.saturating_add(u64::from(milliseconds.max(100)) * 1_000_000) } } diff --git a/drivers/gpu/drm/asahi/g16/regs.rs b/drivers/gpu/drm/asahi/g16/regs.rs index 86c98be692c73..e264e319c2a64 100644 --- a/drivers/gpu/drm/asahi/g16/regs.rs +++ b/drivers/gpu/drm/asahi/g16/regs.rs @@ -104,7 +104,7 @@ impl Mapping { } // SAFETY: memremap() of a range outside System RAM creates a new mapping; the range is // an ADT-described GPU region that the caller admitted. - let addr = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WC as _) }; + let addr = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WC as usize) }; if addr.is_null() { return Err(ENOMEM); } diff --git a/drivers/gpu/drm/asahi/g16/uat.rs b/drivers/gpu/drm/asahi/g16/uat.rs index 54104712c294c..72ca9cb94b846 100644 --- a/drivers/gpu/drm/asahi/g16/uat.rs +++ b/drivers/gpu/drm/asahi/g16/uat.rs @@ -294,7 +294,7 @@ impl Window { let address = va | ((index as u64) << shift); let off = address.checked_sub(hw::UPPER).ok_or(EINVAL)?; let expected = if off < 0x60000 { - self.text_pa + off + (self.text_pa + off) | if (0x50000..0x58000).contains(&off) { 0x483 } else { @@ -302,7 +302,7 @@ impl Window { } } else if off < 0x19c000 { let data_offset = off - 0x60000; - self.data_pa + data_offset + (self.data_pa + data_offset) | if data_offset < 0xc000 { 0x00200000000004c3 } else { @@ -407,9 +407,9 @@ impl Window { continue; } let expected = if index == 0 { - self.root.phys() + hw::PAGE | 0x403 + (self.root.phys() + hw::PAGE) | 0x403 } else { - self.root.phys() + 0x10000 | 0x00200000000004cb + (self.root.phys() + 0x10000) | 0x00200000000004cb }; if self.firmware_root[index] != 0 || observed != expected { return Err(EIO); @@ -460,7 +460,7 @@ impl Window { let handoff = uat.lock_m4_handoff()?; let result = self .verify_root() - .and_then(|_| self.sync_inner(shadow_l2, Some(range.clone()))); + .and_then(|()| self.sync_inner(shadow_l2, Some(range.clone()))); drop(handoff); result?; // This includes global final-level entries. Never replace this with ASID-only TLBI: diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 7b99daf4c1fce..7b4c9de0e0fd9 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -46,18 +46,59 @@ impl ReportRing { pub(crate) struct Rings { quarantined: [[bool; 2]; 2], reports: super::reports::Reports, + telemetry: super::telemetry::Telemetry, + max_age_ns: u64, + suspended: bool, + discard_statistics: [bool; 2], + statistics_epoch: u64, } impl Rings { - pub(crate) fn new() -> Result { + pub(crate) fn new(graph: super::fw::variant::InitGraph) -> Result { Ok(Self { quarantined: [[false; 2]; 2], reports: super::reports::Reports::new()?, + telemetry: super::telemetry::Telemetry::default(), + max_age_ns: super::stats::max_age_ns(graph), + suspended: false, + discard_statistics: [false; 2], + statistics_epoch: 0, }) } + pub(super) fn invalidate_statistics(&mut self) { + self.telemetry.invalidate(); + // The packets have no epoch marker. Discard each role's already + // queued records before accepting a new baseline after the boundary. + self.discard_statistics = [true; 2]; + } + + pub(super) fn suspend_statistics(&mut self, suspended: bool) { + self.suspended = suspended; + self.invalidate_statistics(); + } + + pub(super) fn statistics_suspended(&self) -> bool { + self.suspended + } + + pub(super) fn observe_statistics_power(&mut self, powered: bool, epoch: u64) { + if !powered || epoch != self.statistics_epoch { + self.invalidate_statistics(); + } + self.statistics_epoch = epoch; + } + + pub(super) fn read_statistics(&self, now_ns: u64) -> super::telemetry::Readout { + self.telemetry.read(now_ns, self.max_age_ns) + } + pub(super) fn write_reports(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - self.reports.write(f) + self.reports.write(f)?; + let view = self.read_statistics(super::now_ns()); + writeln!(f, "qualified max_age_ns={} util_status={} util_host_ns={} util_samples={} pstate_status={} pstate_host_ns={} pstate_timestamp={} delta_ticks={:?}", + self.max_age_ns, view.util_status, view.util_host_ns, view.util_samples, + view.pstate_status, view.pstate_host_ns, view.pstate_timestamp, self.telemetry.delta_ticks) } pub(crate) fn drain_trace(&mut self, dev: &Device, init: &InitData, role: Role) { @@ -69,12 +110,29 @@ impl Rings { } fn drain_report(&mut self, dev: &Device, init: &InitData, role: Role, ring: ReportRing) { - let quarantined = &mut self.quarantined[role as usize][ring as usize]; - if *quarantined { + if self.quarantined[role as usize][ring as usize] { return; } - if let Err(error) = Self::consume_report(init, role, ring, &mut self.reports) { - *quarantined = true; + let statistics = matches!(ring, ReportRing::Statistics); + let accept = !self.suspended + && !self.discard_statistics[role as usize] + && !self + .quarantined + .iter() + .any(|rings| rings[ReportRing::Statistics as usize]); + if let Err(error) = Self::consume_report( + init, + role, + ring, + &mut self.reports, + &mut self.telemetry, + self.max_age_ns, + accept, + ) { + self.quarantined[role as usize][ring as usize] = true; + if statistics { + self.invalidate_statistics(); + } dev_err!( dev.as_ref(), "G17 {:?} {:?} ring stopped: {:?}\n", @@ -82,6 +140,8 @@ impl Rings { ring, error ); + } else if statistics && !self.suspended { + self.discard_statistics[role as usize] = false; } } @@ -90,6 +150,9 @@ impl Rings { role: Role, ring: ReportRing, reports: &mut super::reports::Reports, + telemetry: &mut super::telemetry::Telemetry, + max_age_ns: u64, + accept_statistics: bool, ) -> Result { let (state, entries, capacity, allowed_types) = ring.geometry(); let consumer_word = init.status_word(role, state)?; @@ -104,8 +167,8 @@ impl Rings { } while consumer != producer { let offset = entries + consumer as usize * EVENT_RECORD_SIZE; - // Copy all words before returning the slot. Interpretation is a - // separate firmware qualification step, never a scheduling input. + // Decode only the complete copy, before returning the slot. + // Statistics never affect scheduling or power control. let words = init.status_read_words::(role, offset)?; super::trace::report(role as u8, ring as u8, &words); let tag = words[0]; @@ -113,7 +176,11 @@ impl Rings { return Err(EIO); } if matches!(ring, ReportRing::Statistics) { - reports.record(role, words, super::now_ns()); + let host_ns = super::now_ns(); + reports.record(role, words, host_ns); + if accept_statistics { + telemetry.record(&words, host_ns, max_age_ns); + } } consumer = (consumer + 1) & (capacity - 1); fence(Ordering::SeqCst); diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 8245e8bca71e6..81d1d6fe43960 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -68,7 +68,9 @@ static EXPORT_OPS: AtomicUsize = AtomicUsize::new(0); #[vtable] impl FenceOps for ExportFence { - fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("asahi") } + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("asahi") + } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("g17-aggregate") } @@ -76,7 +78,13 @@ impl FenceOps for ExportFence { #[inline(never)] fn export_fallback(contexts: &FenceContexts) -> Result> { - let fence = contexts.new_fence(0, ExportFence { source: None, scoped: false })?; + let fence = contexts.new_fence( + 0, + ExportFence { + source: None, + scoped: false, + }, + )?; // SAFETY: Unique, initialized and still private. Only this constructor // publishes ExportFence objects; its payload is immutable after sharing. let ops = unsafe { @@ -86,8 +94,8 @@ fn export_fallback(contexts: &FenceContexts) -> Result> let previous = EXPORT_OPS.load(Ordering::Acquire); if previous == 0 { match EXPORT_OPS.compare_exchange(0, ops, Ordering::AcqRel, Ordering::Acquire) { - Ok(_) => {}, - Err(previous) if previous == ops => {}, + Ok(_) => {} + Err(previous) if previous == ops => {} Err(_) => return Err(EIO), } } else if previous != ops { @@ -96,21 +104,32 @@ fn export_fallback(contexts: &FenceContexts) -> Result> Ok(fence) } -/// Caller retains the leaf fence throughout lookup. Exact registered operations -/// identify the repr(C) FenceObject payload, not a driver or timeline name. -pub(super) unsafe fn exported_source_raw(raw: *mut bindings::dma_fence) - -> Option> -{ +/// Find the submission behind a leaf fence with the exact registered operations. +/// Those operations identify the repr(C) FenceObject payload, not its name. +/// +/// # Safety +/// +/// `raw` must be non-null and point to an initialized dma_fence. The caller +/// must retain a fence reference throughout this lookup. +pub(super) unsafe fn exported_source_raw(raw: *mut bindings::dma_fence) -> Option> { let expected = EXPORT_OPS.load(Ordering::Acquire); - if expected == 0 { return None; } + if expected == 0 { + return None; + } // SAFETY: The caller owns a live fence reference. ExportFence has a unique // timeline callback and nontrivial payload drop. No cast precedes the exact // registered vtable check. FenceObject begins with its dma_fence. let object = unsafe { - if (*raw).ops as usize != expected { return None; } + if (*raw).ops as usize != expected { + return None; + } &*(raw as *const FenceObject) }; - object.source.as_ref().filter(|source| source.firmware_source).cloned() + object + .source + .as_ref() + .filter(|source| source.firmware_source) + .cloned() } struct State { @@ -144,33 +163,50 @@ impl Submission { Self::with_work(Some(work), firmware_source) } fn with_work(work: Option>, firmware_source: bool) -> Result> { - Arc::pin_init(pin_init!(Self { - state <- new_mutex!(State { work, ready: None }, "G17 submission work state"), - remaining: AtomicU8::new(1), - error: AtomicI32::new(0), - owners: AtomicU8::new(1), - firmware_source, - retired: AtomicBool::new(false), - }), GFP_KERNEL) + Arc::pin_init( + pin_init!(Self { + state <- new_mutex!(State { work, ready: None }, "G17 submission work state"), + remaining: AtomicU8::new(1), + error: AtomicI32::new(0), + owners: AtomicU8::new(1), + firmware_source, + retired: AtomicBool::new(false), + }), + GFP_KERNEL, + ) } pub(super) fn source_status(&self) -> Result { - let complete = self.remaining.load(Ordering::Acquire) == 0 - || self.retired.load(Ordering::Acquire); + let complete = + self.remaining.load(Ordering::Acquire) == 0 || self.retired.load(Ordering::Acquire); let error = self.error.load(Ordering::Acquire); - if error != 0 { return Err(super::dependency::inherited_error(Error::from_errno(error))); } + if error != 0 { + return Err(super::dependency::inherited_error(Error::from_errno(error))); + } Ok(complete) } pub(super) fn same_vm(&self, status: &Arc) -> bool { - self.state.lock().work.as_ref().is_some_and(|work| Arc::ptr_eq(work.status(), status)) + self.state + .lock() + .work + .as_ref() + .is_some_and(|work| Arc::ptr_eq(work.status(), status)) + } + pub(super) fn ready(&self) -> Option { + self.state.lock().ready.clone() } - pub(super) fn ready(&self) -> Option { self.state.lock().ready.clone() } fn set_ready(&self, ready: Fence) { - if !self.firmware_source { return; } + if !self.firmware_source { + return; + } let mut state = self.state.lock(); - if state.work.is_some() && state.ready.is_none() { state.ready = Some(ready); } + if state.work.is_some() && state.ready.is_none() { + state.ready = Some(ready); + } } fn note_error(&self, error: Error) -> bool { - self.error.compare_exchange(0, error.to_errno(), Ordering::AcqRel, Ordering::Acquire).is_ok() + self.error + .compare_exchange(0, error.to_errno(), Ordering::AcqRel, Ordering::Acquire) + .is_ok() } fn release_state(&self) { let old = { @@ -182,15 +218,26 @@ impl Submission { fn owner_done(&self) { // Every hardware owner retains its Completion/Member through exact // retirement or stop. Exported fence FDs must not retain failed pins. - if self.owners.fetch_sub(1, Ordering::AcqRel) == 1 { self.release_state(); } + if self.owners.fetch_sub(1, Ordering::AcqRel) == 1 { + self.release_state(); + } } fn member_done(&self, fence: &UserFence, result: Result) { - if let Err(error) = result { self.note_error(error); } + if let Err(error) = result { + self.note_error(error); + } if self.remaining.fetch_sub(1, Ordering::AcqRel) == 1 { let error = self.error.load(Ordering::Acquire); - if error != 0 { fence.set_error(Error::from_errno(error)); } - else { self.release_state(); } - if fence.scoped { signal_exported(fence); } else { fence.signal(); } + if error != 0 { + fence.set_error(Error::from_errno(error)); + } else { + self.release_state(); + } + if fence.scoped { + signal_exported(fence); + } else { + fence.signal(); + } } } } @@ -200,16 +247,28 @@ pub(crate) struct Aggregate(UserFence); impl Aggregate { fn source(&self) -> &Arc { // Only Outputs::publish constructs Aggregate, after setting the payload. - self.0.source.as_ref().expect("published aggregate has submission state") + self.0 + .source + .as_ref() + .expect("published aggregate has submission state") + } + fn fence(&self) -> Fence { + Fence::from_fence(&self.0) } - fn fence(&self) -> Fence { Fence::from_fence(&self.0) } pub(crate) fn member(&self) -> Member { self.source().remaining.fetch_add(1, Ordering::Relaxed); self.source().owners.fetch_add(1, Ordering::Relaxed); - Member { aggregate: self.clone(), complete: AtomicBool::new(false) } + Member { + aggregate: self.clone(), + complete: AtomicBool::new(false), + } } fn finish_enqueue(&self, result: Result) { - if let Err(error) = result { self.source().error.store(error.to_errno(), Ordering::Release); } + if let Err(error) = result { + self.source() + .error + .store(error.to_errno(), Ordering::Release); + } self.source().member_done(&self.0, result); self.source().owner_done(); } @@ -220,22 +279,38 @@ pub(crate) struct Member { complete: AtomicBool, } impl Member { - pub(super) fn needs_source_ready(&self) -> bool { self.aggregate.source().firmware_source } + pub(super) fn needs_source_ready(&self) -> bool { + self.aggregate.source().firmware_source + } pub(crate) fn work_state(&self) -> Result> { - self.aggregate.source().state.lock().work.clone().ok_or(ECANCELED) + self.aggregate + .source() + .state + .lock() + .work + .clone() + .ok_or(ECANCELED) } pub(super) fn is_source(&self, source: &Arc) -> bool { self.aggregate.source().firmware_source && Arc::ptr_eq(self.aggregate.source(), source) } - pub(super) fn set_source_ready(&self, ready: Fence) { self.aggregate.source().set_ready(ready); } - pub(super) fn note_error(&self, error: Error) -> bool { self.aggregate.source().note_error(error) } + pub(super) fn set_source_ready(&self, ready: Fence) { + self.aggregate.source().set_ready(ready); + } + pub(super) fn note_error(&self, error: Error) -> bool { + self.aggregate.source().note_error(error) + } pub(super) fn note_retired(&self) { let source = self.aggregate.source(); - if source.firmware_source { source.retired.store(true, Ordering::Release); } + if source.firmware_source { + source.retired.store(true, Ordering::Release); + } } pub(crate) fn complete(&self, result: Result) { if !self.complete.swap(true, Ordering::AcqRel) { - self.aggregate.source().member_done(&self.aggregate.0, result); + self.aggregate + .source() + .member_done(&self.aggregate.0, result); } } } @@ -278,7 +353,10 @@ mod tests { let member = aggregate.member(); aggregate.finish_enqueue(Ok(())); member.complete(Err(EIO)); - assert_eq!(super::super::job::fence_status(&aggregate.fence()), EIO.to_errno()); + assert_eq!( + super::super::job::fence_status(&aggregate.fence()), + EIO.to_errno() + ); assert!(source.state.lock().ready.is_some()); drop(member); // Keep both the exported fence and source state alive, as userspace can. @@ -291,7 +369,10 @@ mod tests { let aggregate = aggregate(&contexts()); drop(aggregate.member()); aggregate.finish_enqueue(Err(ENOMEM)); - assert_eq!(super::super::job::fence_status(&aggregate.fence()), ENOMEM.to_errno()); + assert_eq!( + super::super::job::fence_status(&aggregate.fence()), + ENOMEM.to_errno() + ); } #[test] @@ -318,7 +399,9 @@ mod tests { } assert!(super::super::job::fence_status(&aggregate.fence()) > 0); assert_eq!(source.remaining.load(Ordering::Acquire), 0); - for member in &mut members { drop(member.take()); } + for member in &mut members { + drop(member.take()); + } assert_eq!(source.owners.load(Ordering::Acquire), 0); } } @@ -333,7 +416,9 @@ fn install(outputs: impl Iterator, fence: &Fence) { for mut sync in outputs { if let Some(chain) = sync.chain_fence.take() { sync.syncobj.add_point(chain, fence, sync.timeline_value); - } else { sync.syncobj.replace_fence(Some(fence)); } + } else { + sync.syncobj.replace_fence(Some(fence)); + } } } @@ -346,10 +431,15 @@ pub(crate) struct Outputs { cumulative: Option, } impl Outputs { - pub(crate) fn new(contexts: &FenceContexts, mut syncs: KVec, - input_count: usize, allocation_failure: &Fence) -> Result - { - if input_count > syncs.len() { return Err(EINVAL); } + pub(crate) fn new( + contexts: &FenceContexts, + mut syncs: KVec, + input_count: usize, + allocation_failure: &Fence, + ) -> Result { + if input_count > syncs.len() { + return Err(EINVAL); + } let mut cumulative = syncs.iter_mut().find_map(|sync| sync.cumulative.take()); let fallback = match export_fallback(contexts) { Ok(fence) => fence, @@ -362,11 +452,23 @@ impl Outputs { return Err(error); } }; - Ok(Self { fallback: Some(fallback), syncs, input_count, - submission: None, result: Err(ECANCELED), cumulative }) - } - pub(crate) fn inputs(&self) -> &[SyncItem] { &self.syncs[..self.input_count] } - pub(crate) fn publish(&mut self, context: Arc, firmware_source: bool) -> Result { + Ok(Self { + fallback: Some(fallback), + syncs, + input_count, + submission: None, + result: Err(ECANCELED), + cumulative, + }) + } + pub(crate) fn inputs(&self) -> &[SyncItem] { + &self.syncs[..self.input_count] + } + pub(crate) fn publish( + &mut self, + context: Arc, + firmware_source: bool, + ) -> Result { let source = Submission::new(context, firmware_source)?; let mut fence = self.fallback.take().ok_or(EINVAL)?; fence.source = Some(source); @@ -381,7 +483,10 @@ impl Outputs { } Ok(submission) } - pub(crate) fn finish(mut self, result: Result) -> Result { self.result = result; result } + pub(crate) fn finish(mut self, result: Result) -> Result { + self.result = result; + result + } } impl Drop for Outputs { fn drop(&mut self) { @@ -393,11 +498,16 @@ impl Drop for Outputs { return; } if let Some(fallback) = self.fallback.take() { - if let Err(error) = self.result { fallback.set_error(error); } + if let Err(error) = self.result { + fallback.set_error(error); + } if let Some(bundle) = &mut self.cumulative { bundle.fail(&Fence::from_fence(&fallback)); } else { - install(self.syncs.drain(self.input_count..), &Fence::from_fence(&fallback)); + install( + self.syncs.drain(self.input_count..), + &Fence::from_fence(&fallback), + ); } fallback.signal(); } diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index c8763f40871d1..578bb7533f3c9 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -41,7 +41,11 @@ pub(crate) struct ComputeCompletion<'a> { /// Records in the compute completion ring that the host scans. pub(super) const COMPUTE_COMPLETION_SLOTS: usize = { let slots = abi::COMPLETION_RING_SIZE / 0x40; - if slots < 0x800 { slots } else { 0x800 } + if slots < 0x800 { + slots + } else { + 0x800 + } }; /// Timestamp bits of a completion record's start and end words. @@ -203,7 +207,13 @@ struct BuildEnv<'a> { impl BuildEnv<'_> { /// An object at a fixed offset of the kernel window. - fn fixed(&self, offset: u64, size: usize, prot: mmu::Prot, cpu: CpuMap) -> Result { + fn fixed( + &self, + offset: u64, + size: usize, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { KernelObject::new( self.dev, self.vm, @@ -215,7 +225,13 @@ impl BuildEnv<'_> { } /// An object placed in the dynamic part of the kernel window. - fn dynamic(&self, size: usize, align: u64, prot: mmu::Prot, cpu: CpuMap) -> Result { + fn dynamic( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { KernelObject::new( self.dev, self.vm, @@ -320,7 +336,12 @@ impl InitData { /// Initialize a fresh physical compute QID's row before exposing its queue. An installed /// or rebound queue keeps the row's firmware-owned counters and must never call this again. /// Called for a fresh row or after its free-list release was consumed. - pub(super) fn initialize_compute_pool_blocks(&self, slot: u16, page_list_va: u64, blocks: usize) -> Result { + pub(super) fn initialize_compute_pool_blocks( + &self, + slot: u16, + page_list_va: u64, + blocks: usize, + ) -> Result { if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { return Err(EINVAL); } @@ -682,23 +703,21 @@ impl InitData { /// Raw views of the memory publishers share without the device mutex: every work ring, /// the completion stamps, the QoS table and the recovery state word. They stay valid for /// this session. - pub(super) fn port_views( - &self, - mailbox: *const super::Mailbox, - ) -> Result { - use super::fw::{channels, queue::{DataMaster, Policy}}; + pub(super) fn port_views(&self, mailbox: *const super::Mailbox) -> Result { + use super::fw::{ + channels, + queue::{DataMaster, Policy}, + }; let ring_of = |priority: u8, engine: DataMaster| -> Result { let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; - let word = |offset: usize| -> Result<*const AtomicU32> { - Ok(state.word(base + offset)?) - }; + let word = + |offset: usize| -> Result<*const AtomicU32> { Ok(state.word(base + offset)?) }; Ok(super::port::RingView { slots: self.object(self.bundle)?.pointer( abi::work_ring_offset(channel), - channels::WORK_RING_SLOTS as usize - * core::mem::size_of::(), + channels::WORK_RING_SLOTS as usize * core::mem::size_of::(), )?, consumers: [ word(abi::ChannelState::CONSUMER)?, @@ -707,13 +726,14 @@ impl InitData { producer: word(abi::ChannelState::PRODUCER)?, }) }; - let rings = |engine: DataMaster| -> Result<[super::port::RingView; Policy::PRIORITIES as usize]> { - let mut views = [ring_of(0, engine)?; Policy::PRIORITIES as usize]; - for (priority, view) in views.iter_mut().enumerate().skip(1) { - *view = ring_of(priority as u8, engine)?; - } - Ok(views) - }; + let rings = + |engine: DataMaster| -> Result<[super::port::RingView; Policy::PRIORITIES as usize]> { + let mut views = [ring_of(0, engine)?; Policy::PRIORITIES as usize]; + for (priority, view) in views.iter_mut().enumerate().skip(1) { + *view = ring_of(priority as u8, engine)?; + } + Ok(views) + }; Ok(super::port::Views { compute: rings(DataMaster::Compute)?, tiling: rings(DataMaster::Tiling)?, @@ -775,13 +795,13 @@ impl InitData { } fence(Ordering::Acquire); let mut newest = None; - for slot in 0..COMPUTE_COMPLETION_SLOTS { + for (slot, seen_control) in seen.iter_mut().enumerate() { let completion = ComputeCompletion { object, offset: slot * 0x40, }; let control = completion.read(0)?; - seen[slot] = control; + *seen_control = control; if control != 0 { let end = completion.read(5)? & COMPLETION_TIME_MASK; if newest.is_none_or(|(_, latest)| end > latest) { @@ -814,6 +834,10 @@ impl InitData { } /// Constructs the graph after the firmware has published the high-root entries. + #[expect( + clippy::too_many_arguments, + reason = "The firmware graph builder keeps its independent input objects explicit." + )] pub(super) fn build( &mut self, dev: &AsahiDrmDevice, @@ -891,7 +915,8 @@ impl InitData { table.init(); Ok(()) })?; - let page_pool = self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + let page_pool = + self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(page_pool)? .initialize::(0, |table| { table.init(low::RENDER_PAGE_LIST, low::COMPUTE_POOL_PAGE_LIST); @@ -975,7 +1000,11 @@ impl InitData { fn build_render_pool(&mut self, env: &BuildEnv<'_>) -> Result { use mmu::PROT_GPU_SHARED_RW as GS; use CpuMap::WriteBack as WB; - let pages = self.add(KernelObject::backing(env.dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; + let pages = self.add(KernelObject::backing( + env.dev, + low::RENDER_PAGE_LIST_SIZE, + WB, + )?)?; self.object_mut(pages)? .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |list| { abi::fill_page_list( @@ -983,10 +1012,18 @@ impl InitData { (0..low::RENDER_BLOCKS_INITIAL as usize).map(low::render_block), ) })?; - let runs = self.add(KernelObject::backing(env.dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; + let runs = self.add(KernelObject::backing( + env.dev, + low::RENDER_RUN_LIST_SIZE, + WB, + )?)?; let first_block = self.objects.len(); for _ in 0..low::RENDER_BLOCKS_ALLOCATED { - self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + self.add(KernelObject::backing( + env.dev, + abi::FREE_LIST_BLOCK_SIZE, + WB, + )?)?; } self.render_pages = pages; self.render_runs = runs; @@ -994,7 +1031,12 @@ impl InitData { self.alias(pages, env.render, low::RENDER_PAGE_LIST, GS)?; self.alias(runs, env.render, low::RENDER_RUN_LIST, GS)?; for block in 0..low::RENDER_BLOCKS_MAPPED { - self.alias(first_block + block, env.render, low::render_block(block), GS)?; + self.alias( + first_block + block, + env.render, + low::render_block(block), + GS, + )?; } Ok(first_block) } @@ -1031,12 +1073,8 @@ impl InitData { .initialize::<[abi::FreeListRunSlot; mmu::UAT_PGSZ / 0x40]>(0, |list| { abi::fill_run_list(list, low::COMPUTE_FREE_LIST_BLOCKS) })?; - let ready = self.add(env.fixed( - high::COMPUTE_READY_FREE_LIST, - mmu::UAT_PGSZ, - GFS, - WC, - )?)?; + let ready = + self.add(env.fixed(high::COMPUTE_READY_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; self.object_mut(ready)?.write( 0, abi::FreeListControl::new(&abi::FreeListArgs { @@ -1427,15 +1465,15 @@ impl InitData { /// Whether a firmware that acknowledges in status A accepted the graph (`+0x14` = 1). pub(super) fn init_accepted(&self) -> Result { - Ok(self.status_word(Role::Primary, 0x14)?.load(Ordering::Acquire) == 1) + Ok(self + .status_word(Role::Primary, 0x14)? + .load(Ordering::Acquire) + == 1) } pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { - self.object(self.cluster)?.word( - self.status_offset(role) - .checked_add(offset) - .ok_or(EINVAL)?, - ) + self.object(self.cluster)? + .word(self.status_offset(role).checked_add(offset).ok_or(EINVAL)?) } pub(super) fn status_read_words( @@ -1443,11 +1481,8 @@ impl InitData { role: Role, offset: usize, ) -> Result<[u32; N]> { - self.object(self.cluster)?.read_words( - self.status_offset(role) - .checked_add(offset) - .ok_or(EINVAL)?, - ) + self.object(self.cluster)? + .read_words(self.status_offset(role).checked_add(offset).ok_or(EINVAL)?) } pub(super) fn control_counters(&self) -> Result<[u32; 3]> { @@ -1540,13 +1575,15 @@ impl InitData { Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), }; - let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) + let end = offset + .checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) .ok_or(EOVERFLOW)?; if end > extent { return Err(EINVAL); } let pointer_at = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + core::mem::offset_of!(abi::MainConfig, view_va) + view * size_of::(); + + core::mem::offset_of!(abi::MainConfig, view_va) + + view * size_of::(); fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; @@ -1559,7 +1596,8 @@ impl InitData { impl super::recovery::Memory for InitData { fn fault_sources(&self) -> Result { - let mask = self.object(self.cluster)? + let mask = self + .object(self.cluster)? .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { @@ -1567,13 +1605,19 @@ impl super::recovery::Memory for InitData { // address is unknown. Zero keeps ownership unknown and cannot override // firmware blame, but preserves the progress evidence for diagnostics. let key = super::recovery::slot_key(slot, self.recovery_slot_keys).unwrap_or(0); - let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, - abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) - .ok().map(|words| words[0])?; + let monitor = self + .recovery_view_words::<6>( + abi::recovery::PROGRESS_VIEW, + abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE, + ) + .ok() + .map(|words| words[0])?; Some((key, monitor)) }); - sources.reason = self.recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) - .ok().map(|words| words[0]); + sources.reason = self + .recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) + .ok() + .map(|words| words[0]); Ok(sources) } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index c2962c45f7afc..2a0a9c0a63222 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -57,8 +57,10 @@ mod recovery; mod replay; mod reports; mod runtime; +pub(crate) mod stats; pub(crate) mod status; mod teardown; +pub(crate) mod telemetry; mod timeout; mod validation; @@ -734,6 +736,16 @@ impl Firmware { /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, /// unretired work; quarantined work retaining mappings is excluded. Submissions published /// without the device mutex count as activity, and a clear that races one is abandoned. + fn observe_statistics_power(&mut self) { + if self.rings.statistics_suspended() { + return; + } + self.rings.observe_statistics_power( + self.bridge.regs.g17_core_power() & 0xf != 0, + self.port.statistics_epoch(), + ); + } + fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { let submissions = self.port.submissions(); let submitted = submissions != self.seen_submissions; @@ -1130,6 +1142,7 @@ pub(crate) struct Gpu { geometry: mmu::UatGeometry, ids: gpu::SequenceIDs, shared: Arc, + stats: Arc, _capacity: Pin>>, _reports: Pin>>, _power: Pin>>, @@ -1256,6 +1269,13 @@ impl Gpu { directory.read_only_file(c"power", reports::PowerView(shared.clone())), GFP_KERNEL, )?; + let stats = Arc::new( + crate::stats::StatsSnapshot { + g17_reader: Some(stats::Reader(shared.clone())), + ..Default::default() + }, + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -1264,6 +1284,7 @@ impl Gpu { geometry, ids: Default::default(), shared, + stats, _capacity: capacity, _reports: reports, _power: power, @@ -1420,7 +1441,7 @@ impl Gpu { shared.feed.clone(), )?, seen_submissions: 0, - rings: channel::Rings::new()?, + rings: channel::Rings::new(shared.variant.graph)?, recovery: recovery::State::new(), effort: power::Effort::new(), hold: power::Hold::new(), @@ -1591,7 +1612,7 @@ impl gpu::Gpu for Gpu { } fn stats_snapshot(&self) -> Option> { - None + Some(self.stats.clone()) } fn manager(&self) -> Option<&dyn gpu::GpuManager> { @@ -1722,7 +1743,7 @@ impl gpu::Gpu for Gpu { // until the scheduler release is consumed and reclaimed. // Backend construction has rolled back all temporary leases. let waited = now_ns().saturating_sub(admission_started); - let limit = job::TIMEOUT_MS as u64 * 1_000_000; + let limit = u64::from(job::TIMEOUT_MS) * 1_000_000; if waited >= limit { return Err(EBUSY); } diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index ff95b07fa759b..6f9d1c7d956c0 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -116,6 +116,7 @@ pub(super) struct Port { /// Full firmware effort is applied; cleared before an idle clear rechecks `submissions`. effort_full: AtomicBool, demand: super::power::Demand, + statistics_epoch: AtomicU64, last_submission_ns: AtomicU64, submissions: AtomicU64, } @@ -172,6 +173,7 @@ impl Port { publications: AtomicU64::new(0), effort_full: AtomicBool::new(false), demand, + statistics_epoch: AtomicU64::new(0), last_submission_ns: AtomicU64::new(0), submissions: AtomicU64::new(0), }), @@ -500,7 +502,21 @@ impl Port { self.demand.set_active(full); } + pub(super) fn statistics_epoch(&self) -> u64 { + self.statistics_epoch.load(Ordering::Acquire) + } + pub(super) fn note_gpu_activity(&self) { + // Observe gating before activity can wake the cores. A lockless + // publisher cannot clear the decoder under the device mutex; this + // epoch makes both the next drain and the next reader discard it. + if self + .regs + .as_ref() + .is_some_and(|regs| regs.g17_core_power() & 0xf == 0) + { + self.statistics_epoch.fetch_add(1, Ordering::Release); + } self.last_submission_ns .store(super::now_ns(), Ordering::Relaxed); self.demand.set_active(true); diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 7ff828b1e3602..368e95f144c24 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -35,7 +35,8 @@ const NATIVE_WAIT_STRIDE: usize = 0x60; // A submission can name every physical compute channel. Keep the complete // wait list inside its own cached slot, including the last retained slot. const CACHED_RECORD_STRIDE: usize = (NATIVE_WAIT_OFFSET - + crate::g17::kick::COMPUTE_CHANNELS * NATIVE_WAIT_STRIDE).next_power_of_two(); + + crate::g17::kick::COMPUTE_CHANNELS * NATIVE_WAIT_STRIDE) + .next_power_of_two(); const CONFIG: usize = 0; const ANNOUNCE: usize = 0x800; const PREDECESSOR: usize = 0x400; @@ -280,7 +281,10 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self.graph.word(CURSORS + self.pointers.producer)?.load(Ordering::Relaxed), + producer: self + .graph + .word(CURSORS + self.pointers.producer)? + .load(Ordering::Relaxed), consumer: self.graph.word(CURSORS)?.load(Ordering::Relaxed), count: self.graph.word(CURSORS + 0x60)?.load(Ordering::Relaxed), }) @@ -332,7 +336,9 @@ impl Graph { return Err(EIO); } for index in 0..ITEM_RING_ENTRIES as usize { - self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); + self.graph + .dword(ITEMS + index * 8)? + .store(0, Ordering::Relaxed); } fence(Ordering::SeqCst); self.graph @@ -685,7 +691,10 @@ impl Queue { /// never runs, or the failed command itself, whose ring entry it does not consume (a ray /// traversal's, on Neo). Only a kill of the queue's context gives them back. pub(crate) fn fault_pending(&self) -> bool { - self.fault_stamp.is_some() && self.quarantined && self.awaiting_witness && self.exit.is_none() + self.fault_stamp.is_some() + && self.quarantined + && self.awaiting_witness + && self.exit.is_none() } /// The stamp that proves retirement: the stored one, or a newer reported fault's. @@ -724,7 +733,9 @@ impl Queue { } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { - !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + !self + .context() + .is_some_and(|bound| Arc::ptr_eq(bound, context)) || (self.active.len == 0 && self.replays.len == 0) } pub(crate) fn in_flight(&self) -> bool { @@ -801,7 +812,10 @@ impl Queue { qid: self.qid(), // Accounting completes only for bound owners; a detached queue has no // pending work and never reaches this. - qos: self.binding.as_ref().map_or(0, |binding| binding.context.qos_id()), + qos: self + .binding + .as_ref() + .map_or(0, |binding| binding.context.qos_id()), data_master: DataMaster::Compute as u8, } } @@ -818,7 +832,8 @@ impl Queue { } pub(crate) fn pool_release_stalled(&self, now: u64) -> bool { - self.pool_release_pending() && crate::g17::timeout::background_stalled(self.release_sent_ns, now) + self.pool_release_pending() + && crate::g17::timeout::background_stalled(self.release_sent_ns, now) } /// A closed owner can still retain its last commands or their completion @@ -827,7 +842,12 @@ impl Queue { pub(crate) fn release_settling(&self) -> bool { // release_owner keeps the old owner while its last commands retire. // Waiting only once owner is None misses precisely that interval. - release_pending(self.released, self.quarantined, self.active.len, self.retire_pending) + release_pending( + self.released, + self.quarantined, + self.active.len, + self.retire_pending, + ) } fn selected(&self, masks: Option<[u64; 2]>) -> bool { @@ -1029,8 +1049,13 @@ impl Queue { let front = self.active.front().ok_or(EIO)?; if let Some(ticket) = front.ticket { crate::g17::recovery::note_retired(); - crate::g17::trace::complete(crate::g17::trace::COMPUTE, ticket.qid, - ticket.ordinal, &front.packet, [timestamps[0], timestamps[1]]); + crate::g17::trace::complete( + crate::g17::trace::COMPUTE, + ticket.qid, + ticket.ordinal, + &front.packet, + [timestamps[0], timestamps[1]], + ); } self.retire_pending = true; defer(Deferred::Retired( @@ -1251,7 +1276,9 @@ impl Queue { /// Validate the complete unpublished item budget before changing descriptors, /// work counts or queue producers. Leave one item empty across ring wrap. fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { - if cursors.count != ITEM_RING_ENTRIES { return Err(EIO); } + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { return Err(ERANGE); } @@ -1260,14 +1287,18 @@ fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { let _ = crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeItems); return Err(EAGAIN); } - Ok(()) + Ok(()) } /// Retained commands are capacity, not a failed VM. Nothing has been written /// when this check runs, so the outer publisher can wait for retirement under /// its original admission deadline, just as it does for a full firmware ring. fn retained_capacity( - quarantined: bool, released: bool, active: usize, replays: usize, replay_front: bool, + quarantined: bool, + released: bool, + active: usize, + replays: usize, + replay_front: bool, ) -> Result { if quarantined || released { return Err(EBUSY); @@ -1293,8 +1324,13 @@ mod admission_tests { let waits = crate::g17::kick::COMPUTE_CHANNELS; for slot in 0..RECORD_SLOTS { for index in 0..waits { - let start = record_start(slot, NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, - NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).unwrap(); + let start = record_start( + slot, + NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, + CACHED_RECORD_STRIDE, + ) + .unwrap(); assert!(start >= slot * CACHED_RECORD_STRIDE); assert!(start + NATIVE_WAIT_STRIDE <= (slot + 1) * CACHED_RECORD_STRIDE); } @@ -1305,13 +1341,27 @@ mod admission_tests { fn record_window_cannot_overwrite_the_next_slot() { // The nineteenth wait used to cross the old slot boundary silently. let offset = NATIVE_WAIT_OFFSET + 18 * NATIVE_WAIT_STRIDE; - assert_eq!(record_start(0, offset, NATIVE_WAIT_STRIDE, RECORD_STRIDE), Err(EINVAL)); + assert_eq!( + record_start(0, offset, NATIVE_WAIT_STRIDE, RECORD_STRIDE), + Err(EINVAL) + ); assert!(record_start(0, offset, NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).is_ok()); - assert_eq!(record_start(RECORD_SLOTS, 0, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); - assert_eq!(record_start(0, usize::MAX, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); - assert_eq!(record_start(0, CACHED_RECORD_STRIDE, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); - assert_eq!(record_start(0, CACHED_RECORD_STRIDE - 1, 1, CACHED_RECORD_STRIDE), - Ok(CACHED_RECORD_STRIDE - 1)); + assert_eq!( + record_start(RECORD_SLOTS, 0, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, usize::MAX, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, CACHED_RECORD_STRIDE, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, CACHED_RECORD_STRIDE - 1, 1, CACHED_RECORD_STRIDE), + Ok(CACHED_RECORD_STRIDE - 1) + ); } #[test] @@ -1320,11 +1370,17 @@ mod admission_tests { for sequence in 0..IN_FLIGHT { active.push(sequence).unwrap(); } - assert_eq!(retained_capacity(false, false, active.len, 0, false), Err(EAGAIN)); + assert_eq!( + retained_capacity(false, false, active.len, 0, false), + Err(EAGAIN) + ); assert_eq!(active.front(), Some(&0)); assert_eq!(active.len, IN_FLIGHT); assert_eq!(active.pop(), Some(0)); - assert_eq!(retained_capacity(false, false, active.len, 0, false), Ok(())); + assert_eq!( + retained_capacity(false, false, active.len, 0, false), + Ok(()) + ); assert_eq!(active.front(), Some(&1)); } @@ -1332,28 +1388,114 @@ mod admission_tests { fn replay_front_and_terminal_ownership_remain_distinct() { assert_eq!(retained_capacity(false, false, 0, 1, false), Err(EAGAIN)); assert_eq!(retained_capacity(false, false, 0, 1, true), Ok(())); - assert_eq!(retained_capacity(false, false, IN_FLIGHT, 1, true), Err(EAGAIN)); + assert_eq!( + retained_capacity(false, false, IN_FLIGHT, 1, true), + Err(EAGAIN) + ); for (quarantined, released) in [(true, false), (false, true), (true, true)] { - assert_eq!(retained_capacity(quarantined, released, 0, 1, true), Err(EBUSY)); + assert_eq!( + retained_capacity(quarantined, released, 0, 1, true), + Err(EBUSY) + ); } } #[test] fn item_refusal_is_retryable_and_preserves_the_empty_slot() { let n = ITEM_RING_ENTRIES; - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, false), Ok(())); - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, true), Err(EAGAIN)); - assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, false), Ok(())); - assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, true), Err(EAGAIN)); - assert_eq!(item_capacity(Cursors { producer: 10, consumer: 10, count: n }, true), Ok(())); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 4, + count: n + }, + false + ), + Ok(()) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 4, + count: n + }, + true + ), + Err(EAGAIN) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n - 2, + consumer: 2, + count: n + }, + false + ), + Ok(()) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n - 2, + consumer: 2, + count: n + }, + true + ), + Err(EAGAIN) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 10, + consumer: 10, + count: n + }, + true + ), + Ok(()) + ); } #[test] fn bad_item_cursors_are_not_backpressure() { let n = ITEM_RING_ENTRIES; - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 0, count: 0 }, false), Err(EIO)); - assert_eq!(item_capacity(Cursors { producer: n, consumer: 0, count: n }, false), Err(ERANGE)); - assert_eq!(item_capacity(Cursors { producer: 0, consumer: n, count: n }, true), Err(ERANGE)); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 0, + count: 0 + }, + false + ), + Err(EIO) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n, + consumer: 0, + count: n + }, + false + ), + Err(ERANGE) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: n, + count: n + }, + true + ), + Err(ERANGE) + ); } #[test] @@ -1389,7 +1531,9 @@ impl Queue { Ok(true) } pub(crate) fn reusable(&self) -> bool { - if self.growing { return false; } + if self.growing { + return false; + } self.installed // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) @@ -1454,8 +1598,12 @@ impl Queue { } context.mark_published(); context.note_compute_binding(true); - crate::g17::trace::bind(crate::g17::trace::COMPUTE, self.qid(), context.id(), - if same_vm { 2 } else { 1 }); + crate::g17::trace::bind( + crate::g17::trace::COMPUTE, + self.qid(), + context.id(), + if same_vm { 2 } else { 1 }, + ); self.owner = Some(owner); self.unpublished_binding = true; self.released = false; @@ -1523,7 +1671,8 @@ impl Queue { } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { - self.pool_phase == PoolPhase::Populated && self.pool.populated() + self.pool_phase == PoolPhase::Populated + && self.pool.populated() && (self.reusable() || self.recycled_pending()) && !self.source_hint.has_readers(&self.source_readers) } @@ -1619,9 +1768,8 @@ impl Queue { if !self.vacate_candidate() { return Err(EBUSY); } - let record = ControlRecord::FreeListRelease( - self.pool.release_record(self.free_list_generation), - ); + let record = + ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); self.release_sent_ns = Some(crate::g17::now_ns()); @@ -1636,7 +1784,9 @@ impl Queue { consumer: u32, producer: u32, ) -> Result> { - if self.growing { return Ok(None); } + if self.growing { + return Ok(None); + } let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { return Ok(None); }; @@ -1645,11 +1795,18 @@ impl Queue { } // The retained owner binding's client alias pins the backing: detach it // first (the owner has exited; `reusable()` proved no previous binding). - let alias = self.binding.as_mut().and_then(|binding| binding.pool_alias.take()); + let alias = self + .binding + .as_mut() + .and_then(|binding| binding.pool_alias.take()); let pages = self.pool.take_pages(); - let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); + let descriptors = self + .binding + .as_mut() + .and_then(|binding| binding.descriptors.take()); let ring = self.graph.vacate_ring(); - self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; + self.pool_phase = PoolPhase::Vacant; + self.release_sent_ns = None; Ok(Some(Vacated { _alias: alias, _pages: pages, @@ -1664,7 +1821,9 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if !self.can_rearm() { return Err(EAGAIN); } + if !self.can_rearm() { + return Err(EAGAIN); + } self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; @@ -1678,27 +1837,50 @@ impl Queue { /// Only a capacity increase waits for this queue's previously published work. /// Other queues and ordinary submissions keep their existing concurrency. - pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { - if self.growing { return Err(EAGAIN); } - if self.quarantined || self.released || self.exit.is_some() - || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() + pub(crate) fn begin_growth( + &mut self, + init: &crate::g17::initdata::InitData, + blocks: usize, + ) -> Result { + if self.growing { + return Err(EAGAIN); + } + if self.quarantined + || self.released + || self.exit.is_some() + || self.active.len != 0 + || self.replays.len != 0 + || self.previous.is_some() || (self.installed && !self.retirement_ready) - || self.pool_phase != PoolPhase::Populated { return Err(EAGAIN); } + || self.pool_phase != PoolPhase::Populated + { + return Err(EAGAIN); + } if self.installed { - let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); + let record = + ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); - self.release_sent_ns = Some(crate::g17::now_ns()); - } else { self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; } + self.release_sent_ns = Some(crate::g17::now_ns()); + } else { + self.pool_phase = PoolPhase::Vacant; + self.release_sent_ns = None; + } self.growing = true; self.growth_blocks = blocks; Ok(self.installed) } - pub(crate) fn finish_growth(&mut self, init: &crate::g17::initdata::InitData, + pub(crate) fn finish_growth( + &mut self, + init: &crate::g17::initdata::InitData, pages: &mut Option, - alias: &mut Option, blocks: usize) -> Result { - if !self.growing || blocks < self.growth_blocks { return Err(EAGAIN); } + alias: &mut Option, + blocks: usize, + ) -> Result { + if !self.growing || blocks < self.growth_blocks { + return Err(EAGAIN); + } if let PoolPhase::ReleaseSent(cursor) = self.pool_phase { let consumer = init.control_consumer()?.load(Ordering::Acquire); let producer = init.control_producer()?.load(Ordering::Acquire); @@ -1706,7 +1888,9 @@ impl Queue { return Err(EAGAIN); } } - if alias.is_none() { return Err(EINVAL); } + if alias.is_none() { + return Err(EINVAL); + } let fresh = pages.as_ref().ok_or(EINVAL)?; let binding = self.binding.as_mut().ok_or(EFAULT)?; // No ownership moves before every fallible descriptor/header operation. @@ -1718,11 +1902,19 @@ impl Queue { self.release_sent_ns = None; self.growing = false; self.growth_blocks = 0; - Ok(Vacated { _alias: old_alias, _pages: pages.take(), _descriptors: None, _ring: None }) + Ok(Vacated { + _alias: old_alias, + _pages: pages.take(), + _descriptors: None, + _ring: None, + }) } pub(crate) fn growth_token(&self) -> Option { - self.growing.then_some(Growth { qid: self.qid(), phase: self.pool_phase }) + self.growing.then_some(Growth { + qid: self.qid(), + phase: self.pool_phase, + }) } /// Transfer an abandoned builder's release back to the event worker. Do not @@ -1885,7 +2077,9 @@ fn record_start(slot: usize, offset: usize, window: usize, stride: usize) -> Res if slot >= RECORD_SLOTS || offset > stride || window > stride - offset { return Err(EINVAL); } - slot.checked_mul(stride).and_then(|start| start.checked_add(offset)).ok_or(EINVAL) + slot.checked_mul(stride) + .and_then(|start| start.checked_add(offset)) + .ok_or(EINVAL) } impl Graph { @@ -1957,8 +2151,7 @@ pub(crate) trait Host { ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; - fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) - -> Result; + fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; fn notify(&mut self, message: u64) -> Result; fn note_submission(&mut self) -> Result; } @@ -2023,8 +2216,12 @@ impl Queue { let first = !self.installed && self.ordinal == 0; // The first prototype only waits on an installed consumer channel. // A retry without a published source falls back to completion on CPU. - if first && !native.is_empty() { return Err(EAGAIN); } - if native.len() > crate::g17::kick::COMPUTE_CHANNELS { return Err(EINVAL); } + if first && !native.is_empty() { + return Err(EAGAIN); + } + if native.len() > crate::g17::kick::COMPUTE_CHANNELS { + return Err(EINVAL); + } let entry_signal = first || dependencies .iter() @@ -2081,7 +2278,9 @@ impl Queue { let policy = packet.context.policy(); let ring = 1u8 << policy.priority(); let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; - let cached = self.config_cache.and_then(|(config, at)| (at == epoch).then_some(config)); + let cached = self + .config_cache + .and_then(|(config, at)| (at == epoch).then_some(config)); let send_config = first || super::render::publication::required(cached, config); let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; @@ -2155,10 +2354,16 @@ impl Queue { let cursors = self.graph.cursors()?; let mut wait_vas = KVec::with_capacity(native.len(), GFP_KERNEL)?; for (index, wait) in native.iter().enumerate() { - let record = wait.bytes(crate::g17::fw::stamp_value(kick_timestamp.get() as u32), - packet.order.sequence); - let address = self.graph.record(record_slot, NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, - NATIVE_WAIT_STRIDE, record)?; + let record = wait.bytes( + crate::g17::fw::stamp_value(kick_timestamp.get() as u32), + packet.order.sequence, + ); + let address = self.graph.record( + record_slot, + NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, + record, + )?; wait_vas.push(address, GFP_KERNEL)?; } let prepared_kick = self.kick.prepare_entry(&kick_args)?; @@ -2207,8 +2412,11 @@ impl Queue { packet.mark_published(); visible = true; }) - })().or_else(|error| { - if !visible { self.pool.cancel_unpublished(target)?; } + })() + .or_else(|error| { + if !visible { + self.pool.cancel_unpublished(target)?; + } Err(error) })?; visible = true; @@ -2336,8 +2544,13 @@ impl Queue { .replays .front() .is_some_and(|front| Arc::ptr_eq(front, packet)); - retained_capacity(self.quarantined, self.released, self.active.len, - self.replays.len, replay_front)?; + retained_capacity( + self.quarantined, + self.released, + self.active.len, + self.replays.len, + replay_front, + )?; if !packet.context.is_current() || !self .binding @@ -2425,17 +2638,30 @@ impl Queue { u8::from(self.quarantined), u8::from(self.awaiting_witness), u8::from(self.previous.is_some()), u8::from(self.exit.is_some()), u8::from(self.growing), self.pool_phase, self.active.len, IN_FLIGHT)?; match self.graph.cursors() { - Ok(cursors) if cursors.count != 0 => writeln!(f, "items_used={} items_total={}", - cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, cursors.count - 1), + Ok(cursors) if cursors.count != 0 => writeln!( + f, + "items_used={} items_total={}", + cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, + cursors.count - 1 + ), _ => writeln!(f, "items_unavailable=1"), }?; - writeln!(f, "compute_settling qid={} release={} pool_release={}", self.qid(), - u8::from(self.release_settling()), u8::from(self.pool_release_pending()))?; + writeln!( + f, + "compute_settling qid={} release={} pool_release={}", + self.qid(), + u8::from(self.release_settling()), + u8::from(self.pool_release_pending()) + )?; if let Some(binding) = self.binding.as_ref() { - binding.context.write_capacity(f, "compute", usize::from(self.qid()))?; + binding + .context + .write_capacity(f, "compute", usize::from(self.qid()))?; } if let Some(previous) = self.previous.as_ref() { - previous.context.write_capacity(f, "compute_previous", usize::from(self.qid()))?; + previous + .context + .write_capacity(f, "compute_previous", usize::from(self.qid()))?; } Ok(()) } @@ -2544,10 +2770,18 @@ impl Queue { /// An open client's idle channel that [`Self::share_idle`] can hand over. pub(crate) fn shareable(&self) -> bool { - self.claims == 0 && self.installed && self.owner.is_some() && self.retirement_ready - && !self.growing && !self.quarantined && !self.awaiting_witness - && self.exit.is_none() && self.active.len == 0 && self.replays.len == 0 - && self.previous.is_none() && self.pool_phase == PoolPhase::Populated + self.claims == 0 + && self.installed + && self.owner.is_some() + && self.retirement_ready + && !self.growing + && !self.quarantined + && !self.awaiting_witness + && self.exit.is_none() + && self.active.len == 0 + && self.replays.len == 0 + && self.previous.is_none() + && self.pool_phase == PoolPhase::Populated && !self.source_hint.has_readers(&self.source_readers) } @@ -2560,7 +2794,8 @@ impl Queue { impl Queue { pub(crate) fn can_rearm(&self) -> bool { - self.pool_phase == PoolPhase::Vacant && (self.reusable() || self.recycled_pending()) + self.pool_phase == PoolPhase::Vacant + && (self.reusable() || self.recycled_pending()) && !self.source_hint.has_readers(&self.source_readers) } } @@ -2571,19 +2806,35 @@ impl Queue { /// caller validates the current preparation epoch and cancels only a true /// result's never-published QID; installed rings and identities stay intact. pub(crate) fn rollback_unpublished( - &mut self, owner: u64, context: &Arc, + &mut self, + owner: u64, + context: &Arc, ) -> Result<(bool, Option)> { - if self.owner != Some(owner) || !self.unpublished_binding - || self.binding.as_ref().is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) - || self.growing || self.quarantined || self.awaiting_witness - || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 - || self.retire_pending || self.failure_status_pending + if self.owner != Some(owner) + || !self.unpublished_binding + || self + .binding + .as_ref() + .is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) + || self.growing + || self.quarantined + || self.awaiting_witness + || self.exit.is_some() + || self.active.len != 0 + || self.replays.len != 0 + || self.retire_pending + || self.failure_status_pending || self.pool_phase != PoolPhase::Populated - || self.previous.as_ref().is_some_and(|previous| previous.publication.is_some()) + || self + .previous + .as_ref() + .is_some_and(|previous| previous.publication.is_some()) { return Ok((false, None)); } - if !self.installed { return Ok((true, None)); } + if !self.installed { + return Ok((true, None)); + } let mut retired = None; if let Some(previous) = self.previous.as_ref() { self.graph.set_owner(&previous.context)?; @@ -2604,37 +2855,61 @@ impl Queue { } } - impl Queue { /// Same immutable singleton identity and current physical epoch. - pub(crate) fn has_input_source(&self, source: &Arc, - status: &Arc, epoch: u64) -> bool - { - !self.quarantined && self.replays.len == 0 && self.active.iter().any(|active| - active.ticket.is_some_and(|ticket| ticket.epoch == epoch) - && Arc::ptr_eq(active.packet.completion.status(), status) - && active.packet.completion.is_input_source(source)) - } - - pub(crate) fn source_dependency(&mut self, source: &crate::g17::sync_input::Source, - status: &Arc, epoch: u64) -> Result> - { + pub(crate) fn has_input_source( + &self, + source: &Arc, + status: &Arc, + epoch: u64, + ) -> bool { + !self.quarantined + && self.replays.len == 0 + && self.active.iter().any(|active| { + active.ticket.is_some_and(|ticket| ticket.epoch == epoch) + && Arc::ptr_eq(active.packet.completion.status(), status) + && active.packet.completion.is_input_source(source) + }) + } + + pub(crate) fn source_dependency( + &mut self, + source: &crate::g17::sync_input::Source, + status: &Arc, + epoch: u64, + ) -> Result> { for active in self.active.iter() { - if !active.packet.completion.is_input_source(&source.submission) { continue; } - if !Arc::ptr_eq(active.packet.completion.status(), status) { return Err(EINVAL); } - if self.quarantined || self.replays.len != 0 { return Err(EAGAIN); } + if !active.packet.completion.is_input_source(&source.submission) { + continue; + } + if !Arc::ptr_eq(active.packet.completion.status(), status) { + return Err(EINVAL); + } + if self.quarantined || self.replays.len != 0 { + return Err(EAGAIN); + } let ticket = active.ticket.ok_or(EAGAIN)?; - if ticket.epoch != epoch { return Err(EAGAIN); } + if ticket.epoch != epoch { + return Err(EAGAIN); + } self.valid_ticket(active, &ticket)?; - if active.packet.cancel_requested() { return Err(ECANCELED); } + if active.packet.cancel_requested() { + return Err(ECANCELED); + } source.submission.source_status()?; - let dependency = crate::g17::fw::kick::KickDependency::new(self.qid(), - crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?).ok_or(EIO)?; + let dependency = crate::g17::fw::kick::KickDependency::new( + self.qid(), + crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?, + ) + .ok_or(EIO)?; let hold = self.source_hint.hold(&self.source_readers)?; - return Ok(Some((active.published, dependency, + return Ok(Some(( + active.published, + dependency, self.graph.support.gpu_va() + STAMP as u64, - u32::from(self.qid()), hold))); + u32::from(self.qid()), + hold, + ))); } Ok(None) } @@ -2650,3 +2925,12 @@ impl Queue { } } } + +/// Publication, dependency, stamp address, queue and the retained source hold. +pub(crate) type SourceDependency = ( + u64, + crate::g17::fw::kick::KickDependency, + u64, + u32, + crate::g17::sync_input::Hold, +); diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 540f7cc76e8e7..2436cfd33ed35 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -248,8 +248,7 @@ impl Pair { let layout = target::Layout::new(pass, clusters, variant.programs)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; - let (fragment, fragment_alias) = - kick::Queue::render(alloc, ids[1], context.vm(), rings)?; + let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm(), rings)?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -336,7 +335,11 @@ impl Pair { self.replay.front().cloned() } pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { - if !self.replay.front().is_some_and(|front| Arc::ptr_eq(front, packet)) { + if !self + .replay + .front() + .is_some_and(|front| Arc::ptr_eq(front, packet)) + { return Err(EIO); } self.replay.pop(); @@ -363,14 +366,20 @@ impl Pair { addresses: RenderAddresses, ) -> Result { if !self.room() - || self.replay.front().is_some_and(|front| !Arc::ptr_eq(front, packet)) + || self + .replay + .front() + .is_some_and(|front| !Arc::ptr_eq(front, packet)) || !Arc::ptr_eq(&self.memory.context, &packet.context) || !packet.context.is_current() { return Err(EBUSY); } let Validated::Render { - pass, attachments, ray_query, .. + pass, + attachments, + ray_query, + .. } = &packet.command else { return Err(EINVAL); @@ -465,11 +474,9 @@ impl Pair { true => attachments[1].as_slice(), false => &[][..], }; - let result = self.memory.write_descriptors( - &args, - attachments, - packet.context.id().try_into()?, - ); + let result = + self.memory + .write_descriptors(&args, attachments, packet.context.id().try_into()?); let mcache = match result { Ok(mcache) => mcache, Err(error) => { @@ -669,7 +676,9 @@ impl Pair { .filter(|_| self.recovery_generation == epoch); let present = [0, 1].map(|stage| { let mut config = configs[stage]; - let reload = kicks[stage].prev().completion_value(payload.wrapping_sub(1)); + let reload = kicks[stage] + .prev() + .completion_value(payload.wrapping_sub(1)); if config.completion_seed_valid != 0 && self.installed[stage] && held.is_some_and(|held| held[stage] == reload) @@ -687,7 +696,10 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - let Validated::Render { pass, ray_query, .. } = &packet.command else { + let Validated::Render { + pass, ray_query, .. + } = &packet.command + else { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; @@ -790,10 +802,7 @@ impl Pair { windows[0].target().try_into()?, self.outer_started & ring == 0, )?; - let event_slot = memory - .graph - .support - .word(abi::storage::EVENT_SLOTS + 4)?; + let event_slot = memory.graph.support.word(abi::storage::EVENT_SLOTS + 4)?; let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); let inner = memory.graph.support.word(0x28000)?; let next_inner = (next_ordinal as u32).wrapping_mul(2); @@ -948,7 +957,11 @@ impl Pair { let policy = packet.context.policy(); // A registration of another class can only follow a rebind, which // requires the pair to have drained. - if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { + if self + .registered + .is_some_and(|class| class != policy.priority()) + && self.in_flight() + { return Err(EIO); } Ok((epoch, prepared.mcache, policy)) @@ -986,8 +999,12 @@ impl Pair { host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; host.notify_render(priority)?; - crate::g17::trace::publish(crate::g17::trace::RENDER, self.qids()[1], self.ordinal as u32, - packet); + crate::g17::trace::publish( + crate::g17::trace::RENDER, + self.qids()[1], + self.ordinal as u32, + packet, + ); host.note_submission() })(); if let Err(error) = publish { @@ -1053,9 +1070,14 @@ impl Pair { active.ticket.ordinal, pass.width, pass.height, pass.layers, target::Layout::new(pass, self.clusters, self.variant.programs)); } - pr_warn!("G17: recovery render ordinal={} published={} ctx={} seq={} snapshot={:?}\n", - active.ticket.ordinal, active.outer_published, active.packet.context.id(), - active.packet.order.sequence, observation::snapshot(&self.memory, active.ticket.ordinal)); + pr_warn!( + "G17: recovery render ordinal={} published={} ctx={} seq={} snapshot={:?}\n", + active.ticket.ordinal, + active.outer_published, + active.packet.context.id(), + active.packet.order.sequence, + observation::snapshot(&self.memory, active.ticket.ordinal) + ); } } @@ -1063,7 +1085,12 @@ impl Pair { /// `stamps` still lag it, and reports whether that has held for /// [`retirement::STAMP_FALLBACK_NS`] (`now`, host ns). The observation scan then retires it /// on its pass record instead of waiting for stamps no later event will move. - pub(crate) fn note_stamp_wait(&mut self, stamps: [u32; 2], state: u32, now: u64) -> Result { + pub(crate) fn note_stamp_wait( + &mut self, + stamps: [u32; 2], + state: u32, + now: u64, + ) -> Result { if self.quarantined || self.prepared.is_some() || state != 0 { return Ok(false); } @@ -1074,7 +1101,10 @@ impl Pair { return Ok(false); } let published = front.ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; - if stamps.into_iter().all(|stamp| retirement::stamp_covers(stamp, published)) { + if stamps + .into_iter() + .all(|stamp| retirement::stamp_covers(stamp, published)) + { front.stamp_wait_since = None; return Ok(false); } @@ -1113,14 +1143,25 @@ impl Pair { /// Whether the front group's stamp wait is latched: the observation scan rechecks the pair /// without a completion notification, which may never come for such a group. pub(crate) fn stamp_waiting(&self) -> bool { - self.active.front().is_some_and(|front| front.stamp_wait_since.is_some()) + self.active + .front() + .is_some_and(|front| front.stamp_wait_since.is_some()) } /// The watchdog of `packet` expired while it is this pair's front: if the firmware executed /// it and consumed its ring entries while the stamps lag, it retires on its pass record now /// instead of failing. Failing would give up on the stamps as well. - pub(crate) fn expire_stamp_wait(&mut self, packet: &Arc, stamps: [u32; 2], state: u32) -> Result { - if !self.active.front().is_some_and(|front| Arc::ptr_eq(&front.packet, packet)) { + pub(crate) fn expire_stamp_wait( + &mut self, + packet: &Arc, + stamps: [u32; 2], + state: u32, + ) -> Result { + if !self + .active + .front() + .is_some_and(|front| Arc::ptr_eq(&front.packet, packet)) + { return Ok(false); } if !self.note_stamp_wait(stamps, state, crate::g17::now_ns())? && !self.stamp_waiting() { @@ -1183,7 +1224,9 @@ impl Pair { return Err(EINVAL); }; let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; - self.memory.manager.note_required(required, crate::g17::now_ns()); + self.memory + .manager + .note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); } @@ -1246,7 +1289,10 @@ impl Pair { /// A slot stamp must name retained work before its execution markers are meaningful. pub(crate) fn pass_started(&self, stamp: u64) -> Option { const MASK: u64 = (1 << 40) - 1; - let ordinal = self.active.iter().map(|active| active.ticket.ordinal) + let ordinal = self + .active + .iter() + .map(|active| active.ticket.ordinal) .find(|ordinal| ordinal.wrapping_add(1) & MASK == stamp & MASK)?; Some(observation::snapshot(&self.memory, ordinal).ok()?.pass[0] != 0) } @@ -1278,7 +1324,11 @@ impl Pair { self.quarantined = true; // Successors published after recovery can still fail behind its spared // head. Classify the whole failing group before any fence can settle. - if self.active.iter().any(|active| active.packet.completion.spared()) { + if self + .active + .iter() + .any(|active| active.packet.completion.spared()) + { for active in self.active.iter() { active.packet.completion.classify(recovery::Class::Spared); } @@ -1394,8 +1444,13 @@ impl Pair { front.flist_completed = true; } crate::g17::recovery::note_retired(); - crate::g17::trace::complete(crate::g17::trace::RENDER, qids[1], ordinal as u32, - &front.packet, [timestamps[0], timestamps[3]]); + crate::g17::trace::complete( + crate::g17::trace::RENDER, + qids[1], + ordinal as u32, + &front.packet, + [timestamps[0], timestamps[3]], + ); defer(Deferred::Retired( front.packet.completion.clone(), Ok(timestamps), @@ -1455,7 +1510,13 @@ impl Pair { // A torn sample is no proof; a later poll or the timeout looks again. let witness = |executed| { matches!( - retirement::abandoned(&active.ticket, snapshot, stamps, state, executed), + retirement::abandoned( + &active.ticket, + snapshot, + stamps, + state, + executed + ), Ok(true) ) }; @@ -1464,7 +1525,11 @@ impl Pair { // another queue waited on never: the wait ended when firmware retired it. let lost = (active.packet.completion.interrupted() && !snapshot.untouched()) || active.packet.completion.has_firmware_consumer(); - Ok((unexecuted && !lost, !unexecuted && witness(true), unexecuted && lost)) + Ok(( + unexecuted && !lost, + !unexecuted && witness(true), + unexecuted && lost, + )) })() .unwrap_or_else(|error: Error| { failure = Some(error); @@ -1728,7 +1793,8 @@ impl Pair { /// The firmware acknowledged a kill of this pair's context (scheduler page): it no longer /// walks that context's work list, so the commands it never unlinked can be released. fn context_killed(&self) -> bool { - crate::g17::queue::compute::exit::State::read(&self.memory.context).is_ok_and(|state| state.killed != 0) + crate::g17::queue::compute::exit::State::read(&self.memory.context) + .is_ok_and(|state| state.killed != 0) } /// The pair's context may be killed although its commands were not retired: they all ran @@ -1867,15 +1933,13 @@ impl Pair { // own even with successors queued, so the pair is not quarantined into a sticky EIO. It // fails with ECANCELED, which spared work never reports as device loss; consecutive // abandoned spared successors are retired the same way. - let front_spared = self.active.front().is_some_and(|a| { - Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared() - }); + let front_spared = self + .active + .front() + .is_some_and(|a| Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared()); if !self.quarantined && (self.active.len() == 1 || front_spared) { let mut retired = false; - loop { - let Some(active) = self.active.front() else { - break; - }; + while let Some(active) = self.active.front() { let spared = active.packet.completion.spared(); if retired && !spared { break; @@ -2146,12 +2210,25 @@ impl Pair { } /// The render prefix of `packet` on this pair, as the registry collects it /// for an owner whose only pair this is. - pub(crate) fn own_dependencies(&self, owner: u64, packet: &Arc) -> Result { + pub(crate) fn own_dependencies( + &self, + owner: u64, + packet: &Arc, + ) -> Result { let mut dependencies = RenderDependencies::EMPTY; let mut frontier = crate::g17::dependency::Frontier::new(); - self.collect_dependencies(owner, packet.completion.status(), packet.order.wait_through[0], &mut frontier, false)?; + self.collect_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + false, + )?; for dependency in frontier.as_slice() { - *dependencies.render.get_mut(dependencies.render_count).ok_or(EOVERFLOW)? = *dependency; + *dependencies + .render + .get_mut(dependencies.render_count) + .ok_or(EOVERFLOW)? = *dependency; dependencies.render_count += 1; } if dependencies.render_count != 0 @@ -2165,7 +2242,10 @@ impl Pair { &mut tiling, )?; for dependency in tiling.as_slice() { - *dependencies.tiling.get_mut(dependencies.tiling_count).ok_or(EOVERFLOW)? = *dependency; + *dependencies + .tiling + .get_mut(dependencies.tiling_count) + .ok_or(EOVERFLOW)? = *dependency; dependencies.tiling_count += 1; } } @@ -2178,8 +2258,12 @@ impl Pair { /// not left the pair yet. pub(crate) fn switch_in_flight(&self) -> Option { let (from, first) = self.switch?; - (self.replay.len() != 0 || self.active.iter().any(|active| active.ticket.ordinal == first)) - .then_some(from) + (self.replay.len() != 0 + || self + .active + .iter() + .any(|active| active.ticket.ordinal == first)) + .then_some(from) } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { @@ -2344,7 +2428,9 @@ impl Pair { self.memory.context = binding.context.clone(); self.owner = Some(binding.owner); self.initial_packet = binding.initial_packet; - if foreign { self.graph_started = false; } + if foreign { + self.graph_started = false; + } self.context_update = true; if foreign { self.cursors = [0; 2]; @@ -2395,4 +2481,3 @@ mod completion_noted_tests { assert!(completion_noted([3, 68], both)); } } - diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index c1985c72f6ec1..d65c40c7cfade 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -41,6 +41,10 @@ pub(super) struct Lane<'a> { pub(super) window: Window, } impl<'a> Lane<'a> { + #[expect( + clippy::too_many_arguments, + reason = "The publication lane describes independent fields of the firmware wire record." + )] pub(super) fn new( graph: &'a memory::Graph, stage: usize, @@ -94,7 +98,9 @@ impl<'a> Lane<'a> { push(announce_va); Ok(Self { items, - producer: graph.queues.word(memory::POINTERS[stage] + graph.pointers.producer)?, + producer: graph + .queues + .word(memory::POINTERS[stage] + graph.pointers.producer)?, values, config, config_destination, diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs index b67512f7eec6b..246aa5f02edbf 100644 --- a/drivers/gpu/drm/asahi/g17/reports.rs +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -4,7 +4,10 @@ //! treating another firmware's pstate report as an achieved GPU frequency. //! Writers and readers hold the device mutex; reads never power the GPU up. -use super::{fw::channels::{EVENT_WORDS, STATISTICS_TYPES}, Role}; +use super::{ + fw::channels::{EVENT_WORDS, STATISTICS_TYPES}, + Role, +}; use kernel::prelude::*; #[derive(Clone, Copy, Default)] @@ -40,10 +43,25 @@ impl Reports { } pub(super) fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "format=1 units=raw host_time=monotonic_ns achieved_frequency=unqualified")?; - for (index, sample) in self.samples.iter().enumerate().filter(|(_, s)| s.count != 0) { - writeln!(f, "role={} tag={:#x} count={} host_ns={} words={:08x?}", - index / 32, index % 32, sample.count, sample.host_ns, sample.words)?; + writeln!( + f, + "format=1 units=raw host_time=monotonic_ns achieved_frequency=unqualified" + )?; + for (index, sample) in self + .samples + .iter() + .enumerate() + .filter(|(_, s)| s.count != 0) + { + writeln!( + f, + "role={} tag={:#x} count={} host_ns={} words={:08x?}", + index / 32, + index % 32, + sample.count, + sample.host_ns, + sample.words + )?; } Ok(()) } @@ -54,15 +72,29 @@ pub(super) struct View(pub(super) kernel::sync::Arc); impl kernel::debugfs::Writer for View { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - let state = self.0.state.lock(); - let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; - writeln!(f, "online=1 graph={:?} sampled_ns={}", self.0.variant.graph, super::now_ns())?; + let mut state = self.0.state.lock(); + let Some(firmware) = state.as_deref_mut() else { + return writeln!(f, "online=0"); + }; + firmware.observe_statistics_power(); + writeln!( + f, + "online=1 graph={:?} sampled_ns={}", + self.0.variant.graph, + super::now_ns() + )?; write_power(firmware, f)?; firmware.rings.write_reports(f) } } -fn write_power(firmware: &super::Firmware, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { +fn write_power( + firmware: &super::Firmware, + f: &mut kernel::fmt::Formatter<'_>, +) -> kernel::fmt::Result { + if firmware.rings.statistics_suspended() { + return writeln!(f, "power_management=firmware suspended=1"); + } let core_power = firmware.bridge.regs.g17_core_power(); writeln!(f, "power_management=firmware sampled_ns={} core_power={:#x} cores_powered={} host_hold={} idle_hold_ms={}", super::now_ns(), core_power, core_power & 0xf != 0, firmware.hold.active(), @@ -76,7 +108,9 @@ pub(super) struct PowerView(pub(super) kernel::sync::Arc); impl kernel::debugfs::Writer for PowerView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let state = self.0.state.lock(); - let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; + let Some(firmware) = state.as_deref() else { + return writeln!(f, "online=0"); + }; write_power(firmware, f) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 55d517a60ade8..3e779bba4fbbf 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -38,9 +38,11 @@ impl event::Transport for Service<'_> { } fn statistics(&mut self, role: Role) { let firmware = &mut *self.firmware; + firmware.observe_statistics_power(); firmware .rings .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); + firmware.observe_statistics_power(); } fn drain( &mut self, @@ -298,6 +300,7 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { + self.rings.invalidate_statistics(); if self.recovery.restart_limit(super::now_ns()) { pr_warn!("G17: restart storm limit reached\n"); return Err(ETIMEDOUT); @@ -318,6 +321,7 @@ impl Firmware { .firmware .port .set_generation(service.firmware.recovery.generation()); + service.firmware.rings.invalidate_statistics(); let result = result?; if let Some(error) = result.target_error { dev_warn!( diff --git a/drivers/gpu/drm/asahi/g17/stats.rs b/drivers/gpu/drm/asahi/g17/stats.rs new file mode 100644 index 0000000000000..4541ab8d40220 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/stats.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! A sysfs reader owns the session, never a pointer into detachable Firmware. + +use super::{fw::variant::InitGraph, telemetry::Readout, Shared}; +use core::sync::atomic::Ordering; +use kernel::sync::Arc; + +pub(crate) struct Reader(pub(super) Arc); + +impl Reader { + pub(crate) fn read(&self) -> Readout { + let mut state = self.0.state.lock(); + let Some(firmware) = state.as_deref_mut() else { + return Readout::default(); + }; + if self.0.crashed.load(Ordering::Acquire) || firmware.recovery.pending() { + firmware.rings.invalidate_statistics(); + return firmware.rings.read_statistics(super::now_ns()); + } + firmware.observe_statistics_power(); + firmware.rings.read_statistics(super::now_ns()) + } + + pub(crate) fn suspend(&self, suspended: bool) { + if let Some(firmware) = self.0.state.lock().as_deref_mut() { + firmware.rings.suspend_statistics(suspended); + } + } +} + +/// Two nominal periods is a conservative host age/window policy. The firmware +/// does not guarantee a packet duration or a pstate report cadence. Logical +/// pstate validity is controlled by records and epoch boundaries, not this age. +pub(super) const fn max_age_ns(graph: InitGraph) -> u64 { + match graph { + InitGraph::V25G83 => 16_000_000, + InitGraph::G17P => 32_000_000, + } +} diff --git a/drivers/gpu/drm/asahi/g17/telemetry.rs b/drivers/gpu/drm/asahi/g17/telemetry.rs new file mode 100644 index 0000000000000..a048665dcab5c --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/telemetry.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Qualified V25G83/V26A428 statistics, decoded from complete copied records. +//! No record measures busy time, power, temperature or achieved frequency. +//! A/B table selection is absent, so neither nominal candidate is exported as +//! a requested or achieved frequency. All state is protected by the device mutex. + +pub(super) const UNAVAILABLE: u32 = 0; +pub(super) const VALID: u32 = 1; +pub(super) const STALE: u32 = 2; + +/// C read callback result; independent capabilities never imply one another. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(C)] +pub(crate) struct Readout { + pub(crate) util_status: u32, + pub(crate) pstate_status: u32, + pub(crate) tiler: u32, + pub(crate) renderer: u32, + pub(crate) device: u32, + pub(crate) pstate: u32, + pub(crate) util_host_ns: u64, + pub(crate) pstate_host_ns: u64, + pub(crate) util_samples: u64, + pub(crate) pstate_timestamp: u64, +} + +const _: () = { + assert!(core::mem::offset_of!(Readout, util_host_ns) == 24); + assert!(core::mem::size_of::() == 56); +}; + +#[derive(Default)] +pub(super) struct Telemetry { + view: Readout, + sums: [u64; 3], + window_ns: u64, + timestamp: Option, + /// Diagnostic interval only, never used as busy time. + pub(super) delta_ticks: Option, +} + +impl Telemetry { + /// End an epoch. Preserve the reason that formerly valid data is stale, + /// but never carry an average or a timestamp baseline into a new epoch. + pub(super) fn invalidate(&mut self) { + if self.view.util_status != UNAVAILABLE { + self.view.util_status = STALE; + } + if self.view.pstate_status != UNAVAILABLE { + self.view.pstate_status = STALE; + } + self.view.util_samples = 0; + self.timestamp = None; + self.delta_ticks = None; + } + + /// `max_age_ns` is a host policy (two nominal periods), not a packet duration. + /// Every role contributes one equally weighted sample to the same bounded + /// collection window. Reads do not consume samples or change the mean. + pub(super) fn record(&mut self, words: &[u32; 18], host_ns: u64, max_age_ns: u64) { + match words[0] { + 0x13 => { + let values = [words[3], words[4], words[6]]; + if values.iter().any(|&value| value > 100) { + self.view.util_status = UNAVAILABLE; + self.view.util_samples = 0; + return; + } + if self.view.util_samples == 0 + || host_ns.saturating_sub(self.window_ns) >= max_age_ns + || host_ns < self.window_ns + || self.view.util_samples >= u64::MAX / 100 + { + self.sums = [0; 3]; + self.view.util_samples = 0; + self.window_ns = host_ns; + } + for (sum, value) in self.sums.iter_mut().zip(values) { + *sum += u64::from(value); + } + self.view.util_samples += 1; + self.view.util_host_ns = host_ns; + self.view.util_status = VALID; + self.view.tiler = (self.sums[0] / self.view.util_samples) as u32; + self.view.renderer = (self.sums[1] / self.view.util_samples) as u32; + self.view.device = (self.sums[2] / self.view.util_samples) as u32; + } + 0x10 => { + let timestamp = u64::from(words[1]) | (u64::from(words[2]) << 32); + if self.timestamp.is_some_and(|last| timestamp < last) { + self.invalidate(); + self.view.pstate_status = UNAVAILABLE; + return; + } + self.delta_ticks = self.timestamp.map(|last| timestamp - last); + self.timestamp = Some(timestamp); + self.view.pstate_host_ns = host_ns; + self.view.pstate_timestamp = timestamp; + self.view.pstate_status = if words[6] <= 10 { VALID } else { UNAVAILABLE }; + self.view.pstate = words[6]; + } + _ => {} + } + } + + pub(super) fn read(&self, now_ns: u64, max_age_ns: u64) -> Readout { + let mut view = self.view; + let expired = |received| now_ns < received || now_ns - received >= max_age_ns; + // The oldest contributing sample bounds the whole averaging window. + if view.util_status == VALID && expired(self.window_ns) { + view.util_status = STALE; + } + // A pstate report describes a state transition, with no qualified + // cadence. Its validity ends on replacement or epoch invalidation, + // rather than an invented periodic-report deadline. + view + } +} + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_telemetry))] +#[cfg(any(test, CONFIG_KUNIT))] +mod tests { + use super::*; + + const AGE: u64 = 16_000_000; + const U1: [u32; 18] = [ + 0x13, 0, 0, 25, 75, 0xdeadbeef, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + const U2: [u32; 18] = [ + 0x13, 0, 0, 35, 85, 0x12345678, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + const U3: [u32; 18] = [0x13, 0, 0, 101, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + const P1: [u32; 18] = [0x10, 0x10, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + const P2A: [u32; 18] = [0x10, 0x1000, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + const P2B: [u32; 18] = [0x10, 0x2500, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + const P3: [u32; 18] = [ + 0x10, 0x3000, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + + #[test] + fn spec_u1_u2_u3_both_firmware_policies() { + for age in [AGE, 2 * AGE] { + let mut stats = Telemetry::default(); + assert_eq!(stats.read(0, age), Readout::default()); + stats.record(&U1, 100, age); + let view = stats.read(100, age); + assert_eq!((view.tiler, view.renderer, view.device), (25, 75, 50)); + assert_eq!(view.util_status, VALID); + stats.record(&U2, 101, age); // May be the other role; no deduplication. + let view = stats.read(101, age); + assert_eq!((view.tiler, view.renderer, view.device), (30, 80, 57)); + assert_eq!(view.util_samples, 2); + stats.record(&U3, 102, age); + assert_eq!(stats.read(102, age).util_status, UNAVAILABLE); + stats.record(&U2, 103, age); + assert_eq!(stats.read(103, age).device, 65); + } + } + + #[test] + fn spec_p1_p2_p3_and_regression() { + let mut stats = Telemetry::default(); + stats.record(&P1, 100, AGE); + let view = stats.read(100, AGE); + assert_eq!(view.pstate, 6); + assert_eq!(view.pstate_status, VALID); + assert_eq!(view.pstate_timestamp, 0x1_0000_0010); + assert_eq!(stats.delta_ticks, None); // First sample is only a baseline. + stats.invalidate(); + stats.record(&P2A, 101, AGE); + stats.record(&P2B, 102, AGE); + assert_eq!(stats.delta_ticks, Some(5376)); + assert_eq!( + stats.delta_ticks.unwrap() * 1_000_000_000 / 24_000_000, + 224_000 + ); + stats.record(&P3, 103, AGE); + assert_eq!(stats.read(103, AGE).pstate_status, UNAVAILABLE); + stats.record(&U1, 104, AGE); + stats.record(&P2A, 105, AGE); // Backwards, including across role drain order. + assert_eq!(stats.read(105, AGE).pstate_status, UNAVAILABLE); + assert_eq!(stats.read(105, AGE).util_status, STALE); + assert_eq!(stats.delta_ticks, None); + stats.record(&P2A, 106, AGE); // New epoch establishes a new baseline. + assert_eq!(stats.read(106, AGE).pstate_status, VALID); + assert_eq!(stats.delta_ticks, None); + stats.record(&P2A, 107, AGE); // Equal timestamps are non-regressing. + assert_eq!(stats.delta_ticks, Some(0)); + } + + #[test] + fn age_epoch_and_zero_are_independent() { + let mut stats = Telemetry::default(); + stats.record(&U1, 100, AGE); + stats.record(&P1, 200, AGE); + assert_eq!(stats.read(99, AGE).util_status, STALE); + assert_eq!(stats.read(100 + AGE - 1, AGE).util_status, VALID); + let view = stats.read(100 + AGE, AGE); + assert_eq!(view.util_status, STALE); + assert_eq!(view.pstate_status, VALID); + assert_eq!(stats.read(200 + AGE, AGE).pstate_status, VALID); + assert_eq!(stats.read(u64::MAX, AGE).pstate_status, VALID); + stats.record(&U2, 100 + AGE, AGE); + assert_eq!(stats.read(100 + AGE, AGE).device, 65); // Old window was discarded. + stats.invalidate(); // Suspend, restart, quarantine, gate or teardown. + let view = stats.read(100 + AGE, AGE); + assert_eq!(view.util_status, STALE); + assert_eq!(view.pstate_status, STALE); + let mut zero = [0; 18]; + zero[0] = 0x13; + stats.record(&zero, 101 + AGE, AGE); + let view = stats.read(101 + AGE, AGE); + assert_eq!(view.util_status, VALID); + assert_eq!((view.tiler, view.renderer, view.device), (0, 0, 0)); + zero[0] = 0x10; + stats.record(&zero, 102 + AGE, AGE); + assert_eq!(stats.read(102 + AGE, AGE).pstate, 0); + assert_eq!(stats.read(102 + AGE, AGE).pstate_status, VALID); + } + + #[test] + fn invalid_components_and_selectors_never_clamp() { + for word in [3, 4, 6] { + let mut stats = Telemetry::default(); + let mut words = U1; + words[word] = u32::MAX; + stats.record(&words, 1, AGE); + assert_eq!(stats.read(1, AGE).util_status, UNAVAILABLE); + } + for selector in [11, 15, 16, u32::MAX] { + let mut stats = Telemetry::default(); + let mut words = P1; + words[6] = selector; + stats.record(&words, 1, AGE); + assert_eq!(stats.read(1, AGE).pstate_status, UNAVAILABLE); + } + let mut stats = Telemetry::default(); + for tag in [0x11, 0x12, 0x18, 0x1d, u32::MAX] { + let mut words = U1; + words[0] = tag; + stats.record(&words, 1, AGE); + } + assert_eq!(stats.read(1, AGE), Readout::default()); + } +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 00b44cbe62887..f100820a0c75e 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -41,9 +41,7 @@ use crate::{ AsahiDrmDevice, AsahiDrmDriver, // }, - file, - memacct, - mmu, + file, memacct, mmu, }; const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; @@ -133,9 +131,6 @@ pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { Ok(()) } -unsafe impl Send for AsahiObject {} -unsafe impl Sync for AsahiObject {} - /// A shared reference to a GEM object for this driver. pub(crate) struct ObjectRef { /// The underlying GEM object reference @@ -503,6 +498,7 @@ pub(crate) fn prepare_new_object_g15(gem: &Object) -> Result { Ok(()) } - #[cfg(CONFIG_DRM_ASAHI_G15)] -extern "C" { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } +extern "C" { + fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; +} diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 343fb1c84df50..e9f1008fc5a72 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -311,7 +311,7 @@ pub(crate) trait Gpu: Send + Sync { flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES); } if self.soft_faults() && self.supports_soft_faults() { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS); } flags } @@ -1100,7 +1100,7 @@ impl GpuManager::ver { virt_addr: U64(iova + off as u64), total_size: (map.size * map.count * dies) as u32, element_size: map.size as u32, - readwrite: U64(map.writable as u64), + readwrite: U64(u64::from(map.writable)), }; }); @@ -1150,20 +1150,14 @@ impl GpuManager::ver { /// Fetch the GPU MMU fault information from the hardware registers. fn get_fault_info(&self) -> Option { // TODO: handle differently - let Some(guard) = self.dev.registration_guard() else { - return None; - }; + let guard = self.dev.registration_guard()?; guard.registration_data_with(|reg_data| { let res = ®_data.resources; let info = res.get_fault_info(self.cfg); - if info.is_some() { - dev_err!( - self.dev.as_ref(), - " Fault info: {:#x?}\n", - info.as_ref().unwrap() - ); + if let Some(fault) = &info { + dev_err!(self.dev.as_ref(), " Fault info: {:#x?}\n", fault); } info }) diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index bc79ac73739a5..75cbb1c03c6b1 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -137,11 +137,11 @@ impl<'a> InitDataBuilder::ver<'a> { continue; } - let f_khz = (ps.freq_hz / 1000) as u64; - let v_max = ps.max_volt_mv() as u64; + let f_khz = u64::from(ps.freq_hz / 1000); + let v_max = u64::from(ps.max_volt_mv()); t1.push( - (1000000000 * (curve_cfg.t1_coef as u64) / (f_khz * v_max)) + (1000000000 * u64::from(curve_cfg.t1_coef) / (f_khz * v_max)) .try_into() .unwrap(), GFP_KERNEL, @@ -149,7 +149,7 @@ impl<'a> InitDataBuilder::ver<'a> { for (j, scale) in curve_cfg.t3_scales.iter().enumerate() { t3[j].push( - (t3_coef as u64 * 1000000100 * *scale as u64 / (f_khz * v_max * 6)) + (u64::from(t3_coef) * 1000000100 * u64::from(*scale) / (f_khz * v_max * 6)) .try_into() .unwrap(), GFP_KERNEL, @@ -423,7 +423,7 @@ impl<'a> InitDataBuilder::ver<'a> { #[ver(G >= G14X)] fast_die1_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[1]), #[ver(V < V13_0B4)] - fast_die0_sensor_present: U64(cfg.fast_die0_sensor_present as u64), + fast_die0_sensor_present: U64(u64::from(cfg.fast_die0_sensor_present)), unk_163c: 1, unk_3644: 0, hws1 <- Self::hw_shared1(cfg), @@ -487,10 +487,12 @@ impl<'a> InitDataBuilder::ver<'a> { } if !dyncfg.hw_data_b.is_empty() { + // SAFETY: This closure owns the fully sized, initially zeroed + // wire allocation; the byte view remains within that allocation. unsafe { let mut matches: bool = true; let sla = core::slice::from_raw_parts( - raw as *const raw::HwDataA::ver as *const u8, + core::ptr::from_ref(raw).cast::(), core::mem::size_of::(), ); if sla.len() != dyncfg.hw_data_a.len() { @@ -502,8 +504,10 @@ impl<'a> InitDataBuilder::ver<'a> { dyncfg.hw_data_a.len(), ); } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_a.len()) { - if sla[i] != dyncfg.hw_data_a[i] { + for (i, (actual, expected)) in + sla.iter().zip(&dyncfg.hw_data_a).enumerate() + { + if actual != expected { matches = false; dev_err!(self.dev.as_ref(), "!!! Hwdata A first mismatch: {i}"); break; @@ -670,10 +674,12 @@ impl<'a> InitDataBuilder::ver<'a> { } if !dyncfg.hw_data_b.is_empty() { + // SAFETY: This closure owns the fully sized, initially zeroed + // wire allocation; the byte view remains within that allocation. unsafe { let mut matches: bool = true; let sla = core::slice::from_raw_parts( - raw as *const raw::HwDataB::ver as *const u8, + core::ptr::from_ref(raw).cast::(), core::mem::size_of::(), ); if sla.len() != dyncfg.hw_data_b.len() { @@ -685,8 +691,10 @@ impl<'a> InitDataBuilder::ver<'a> { dyncfg.hw_data_b.len(), ); } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_b.len()) { - if sla[i] != dyncfg.hw_data_b[i] { + for (i, (actual, expected)) in + sla.iter().zip(&dyncfg.hw_data_b).enumerate() + { + if actual != expected { matches = false; dev_err!(self.dev.as_ref(), "!!! Hwdata B first mismatch: {i}"); break; @@ -824,10 +832,12 @@ impl<'a> InitDataBuilder::ver<'a> { } if !dyncfg.hw_globals.is_empty() { + // SAFETY: This closure owns the fully sized, initially zeroed + // wire allocation; the byte view remains within that allocation. unsafe { let mut matches: bool = true; let sla = core::slice::from_raw_parts( - raw as *const raw::Globals::ver as *const u8, + core::ptr::from_ref(raw).cast::(), core::mem::size_of::(), ); if sla.len() != dyncfg.hw_globals.len() { @@ -839,8 +849,10 @@ impl<'a> InitDataBuilder::ver<'a> { dyncfg.hw_globals.len(), ); } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_globals.len()) { - if sla[i] != dyncfg.hw_globals[i] { + for (i, (actual, expected)) in + sla.iter().zip(&dyncfg.hw_globals).enumerate() + { + if actual != expected { matches = false; dev_err!(self.dev.as_ref(), "!!! Globals first mismatch: {i}"); break; @@ -971,12 +983,12 @@ impl<'a> InitDataBuilder::ver<'a> { num_entries: usize, ) -> raw::UatLevelInfo { raw::UatLevelInfo { - index_shift: index_shift as _, + index_shift: index_shift as u8, unk_1: 14, unk_2: 14, unk_3: 8, unk_4: 0x4000, - num_entries: num_entries as _, + num_entries: num_entries as u16, unk_8: U64(1), unk_10: U64(((1u64 << cfg.uat_oas) - 1) & !(mmu::UAT_PGMSK as u64)), index_mask: U64(((num_entries - 1) << index_shift) as u64), diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 5f0ab2b4aea4f..79071cbc434a0 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -71,7 +71,7 @@ pub(crate) fn tlbi_asid(asid: Asid) { asm!( ".arch armv8.4-a", "tlbi aside1os, {x}", - x = in(reg) ((asid as u64) << 48) + x = in(reg) (u64::from(asid) << 48) ); } } @@ -85,7 +85,7 @@ pub(crate) fn tlbi_page(asid: Asid, va: usize) { return; } - let val: u64 = ((asid as u64) << 48) | ((va as u64 >> 12) & 0xffffffffffc); + let val: u64 = (u64::from(asid) << 48) | ((va as u64 >> 12) & 0xffffffffffc); // SAFETY: tlbi is always safe by definition unsafe { asm!( @@ -147,7 +147,7 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { let bits = 5 * exp; let mut base = (num + (1 << bits) - 1) >> bits; - val |= (exp as u64) << 44; + val |= u64::from(exp) << 44; while base > 32 { // SAFETY: tlbi is always safe by definition diff --git a/drivers/gpu/drm/asahi/microseq.rs b/drivers/gpu/drm/asahi/microseq.rs index cbdb5de62e921..42cd40b8220f6 100644 --- a/drivers/gpu/drm/asahi/microseq.rs +++ b/drivers/gpu/drm/asahi/microseq.rs @@ -45,7 +45,7 @@ impl Builder { pub(crate) fn add(&mut self, op: T) -> Result { let off = self.ops.len(); let p: *const T = &op; - let p: *const u8 = p as *const u8; + let p: *const u8 = p.cast::(); // SAFETY: Microseq operations always have no padding bytes, so it is safe to // access them as a byte slice. let s: &[u8] = unsafe { core::slice::from_raw_parts(p, core::mem::size_of::()) }; diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 3522010949e35..12b0a7457fff3 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -108,8 +108,6 @@ pub(crate) use pgtable::{ UAT_TOP_LEVEL_SHIFT, // }; -use pin_init; - const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; @@ -992,6 +990,7 @@ impl VmInner { /// Shared reference to a virtual memory address space ([`Vm`]). #[derive(Clone)] pub(crate) struct Vm { + device_binding: Arc, dev: ARef, id: u64, mapping_epoch: Option>, @@ -1220,14 +1219,17 @@ impl TimestampVmap { /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused /// when the destination is prepared, and ignored here. pub(crate) fn write(&self, offset: usize, value: u64) { - if offset.checked_add(size_of::()).is_none_or(|end| end > self.size()) { + if offset + .checked_add(size_of::()) + .is_none_or(|end| end > self.size()) + { return; } // Completion runs in task context. Bindings of the same BO share this lock, // so aligned u64 and overlapping unaligned byte stores never race in Rust. let _guard = self.0.owner().timestamp_lock.lock(); - // SAFETY: The complete extent was checked above, inside this pinned vmap. let base = self.0.addr.cast::(); + // SAFETY: The complete extent was checked above, inside this pinned vmap. let pointer = unsafe { base.add(offset) }; if pointer as usize % core::mem::align_of::() == 0 { // SAFETY: All eight bytes were checked above; the vmap pins the backing. @@ -1450,7 +1452,7 @@ impl KernelMapping { } else { // Otherwise, check if this slot is the active one, otherwise return // Also check that we actually own this slot - let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; + let ttb = owner.ttb() | TTBR_VALID | u64::from(slot) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); uat_inner.handoff_lock(); @@ -1693,7 +1695,7 @@ impl UatShared { /// Writes back context-table slot `idx` after it changed, where this SoC needs it. fn publish_slot(&self, idx: usize) { if self.quirks.clean_tables { - let slot = &self.ttbs()[idx] as *const SlotTTBS as *const u8; + let slot = core::ptr::from_ref(&self.ttbs()[idx]).cast::(); pgtable::clean_poc(slot, size_of::()); pgtable::poc_barrier(); } @@ -1702,13 +1704,20 @@ impl UatShared { /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant - unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() + unsafe { self.handoff_rgn.map.ptr().cast::().as_ref() }.unwrap() } /// Returns the TTBAT area fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { // SAFETY: pointer is non-null per the type invariant - unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() + unsafe { + self.ttbs_rgn + .map + .ptr() + .cast::<[SlotTTBS; UAT_NUM_CTX]>() + .as_ref() + } + .unwrap() } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. @@ -1896,7 +1905,9 @@ impl Handoff { while self.lock_fw.load(Ordering::Relaxed) != 0 { if self.turn.load(Ordering::Relaxed) != 0 { self.lock_ap.store(0, Ordering::Relaxed); - while self.turn.load(Ordering::Relaxed) != 0 {} + while self.turn.load(Ordering::Relaxed) != 0 { + core::hint::spin_loop() + } self.lock_ap.store(1, Ordering::Relaxed); fence(Ordering::SeqCst); } @@ -2003,10 +2014,7 @@ impl HandoffFlush { } } -fn prefetch_sgt_segments( - dev: &device::Device, - gem: &gem::Object, -) -> Result { +fn prefetch_sgt_segments(dev: &device::Device, gem: &gem::Object) -> Result { let sgt = crate::gem::held_sg_table(gem, dev)?; let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; @@ -2014,7 +2022,10 @@ fn prefetch_sgt_segments( segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(SgSegments { segments, _lease: sgt.into_lease() }) + Ok(SgSegments { + segments, + _lease: sgt.into_lease(), + }) } /// Reuse cached segments without allocating another copy for each bind. The bind lease was @@ -2124,7 +2135,9 @@ impl Vm { )?; let gpuvm = ARef::from(&*unique_gpuvm); + let device_binding = crate::binding::lookup(&dev)?; Ok(Vm { + device_binding, dev: dev.as_ref().as_ref().into(), id, gpuvm, @@ -2334,7 +2347,13 @@ impl Vm { guard_size: usize, ) -> Result { self.map_in_range_with_mode( - gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Best, + gem, + object_range, + alignment, + range, + prot, + guard_size, + mm::InsertMode::Best, ) } @@ -2349,7 +2368,13 @@ impl Vm { guard_size: usize, ) -> Result { self.map_in_range_with_mode( - gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Low, + gem, + object_range, + alignment, + range, + prot, + guard_size, + mm::InsertMode::Low, ) } @@ -2377,15 +2402,24 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let vm_bo = self - .gpuvm - .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; + // End the bound-device lease before entering firmware/GPUVM waits. + let (_bind_lease, vm_bo) = { + let binding = self.device_binding.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = self.gpuvm.obtain( + gem, + pin_init!(VmBoData { + segments <- new_mutex!(None) + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; let size = object_range.range(); + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { self.gpuvm.exec_lock(/*Some(gem)*/ None, false)? }; @@ -2456,13 +2490,20 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let vm_bo = self - .gpuvm - .obtain(&gem, pin_init!(VmBoData { segments <-new_mutex!(None) }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; + // End the bound-device lease before entering firmware/GPUVM waits. + let (_bind_lease, vm_bo) = { + let binding = self.device_binding.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = self.gpuvm.obtain( + gem, + pin_init!(VmBoData { + segments <- new_mutex!(None) + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { @@ -2536,7 +2577,7 @@ impl Vm { gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; + let dev: &device::Device = device.as_ref().as_ref(); let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm @@ -2554,6 +2595,8 @@ impl Vm { // Mapping needs a complete context let mut ctx = StepContext::new(Some(device.as_ref().as_ref()), prot, &mut resources)?; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { self.gpuvm.exec_lock(/*Some(&gem)*/ None, false)? }; @@ -2626,6 +2669,8 @@ impl Vm { size: usize, prot: Prot, ) -> Result { + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; if (iova as usize | phys | size) & UAT_PGMSK != 0 { @@ -2730,6 +2775,8 @@ impl Vm { let mut resources = VmOpResources::for_unmap()?; let mut ctx = StepContext::new(dev, Prot::default(), &mut resources)?; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; mod_dev_dbg!(self.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); @@ -2906,7 +2953,11 @@ impl Uat { #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Map a region given by its physical range, write-combined. - fn map_physical(range: &Range, min_size: usize, reserved_ram: bool) -> Result { + fn map_physical( + range: &Range, + min_size: usize, + reserved_ram: bool, + ) -> Result { let size: usize = (range.end.checked_sub(range.start).ok_or(EINVAL)?).try_into()?; if size < min_size || range.start & (UAT_PGMSK as PhysicalAddr) != 0 { return Err(EINVAL); @@ -2915,13 +2966,11 @@ impl Uat { let mut raw: kernel::bindings::resource = unsafe { core::mem::zeroed() }; raw.start = range.start; raw.end = range.end - 1; - raw.flags = kernel::bindings::IORESOURCE_MEM as _; + raw.flags = kernel::bindings::IORESOURCE_MEM as usize; // SAFETY: `Resource` is a transparent wrapper around `Opaque`, and // `raw` describes a valid memory range. let res: io::resource::Resource = unsafe { core::mem::transmute(kernel::types::Opaque::new(raw)) }; - // SAFETY: The UAT regions are only accessed through this mapping by the CPU; the GPU - // reaches them through its own translation. let flags = io::mem::MemFlag::WC; #[cfg(CONFIG_DRM_ASAHI_G15)] let flags = if reserved_ram { @@ -2938,6 +2987,8 @@ impl Uat { }; #[cfg(not(CONFIG_DRM_ASAHI_G15))] let _ = reserved_ram; + // SAFETY: The UAT regions are only accessed through this mapping by the CPU; + // the GPU reaches them through its own translation. let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }?; Ok(UatRegion { base: range.start, @@ -2994,7 +3045,7 @@ impl Uat { // SAFETY: The Handoff struct layout matches the firmware's view of memory at this // address, and the region is at least large enough per the check above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; let inner = Arc::pin_init( try_pin_init!(UatInner { @@ -3264,9 +3315,9 @@ impl Uat { { return Err(EIO); } - for index in UAT_USER_CTX_START..UAT_NUM_CTX { - if ttbs[index].ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] - || ttbs[index].ttb1.load(Ordering::Acquire) != 0 + for (index, table) in ttbs.iter().enumerate().skip(UAT_USER_CTX_START) { + if table.ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] + || table.ttb1.load(Ordering::Acquire) != 0 { return Err(EIO); } @@ -3367,14 +3418,19 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { - for (value, pte) in snapshot.iter_mut().zip(ptes) { - *value = pte.load(Ordering::Relaxed); - } - for pte in &ptes[owned..] { - pte.store(0, Ordering::Relaxed); - } - }) + pgtable::with_root_entries( + self.inner.reserved.as_deref(), + self.kernel_vm.ttb(), + UAT_PGSZ / size_of::(), + |ptes| { + for (value, pte) in snapshot.iter_mut().zip(ptes) { + *value = pte.load(Ordering::Relaxed); + } + for pte in &ptes[owned..] { + pte.store(0, Ordering::Relaxed); + } + }, + ) }?; mem::sync(); @@ -3387,11 +3443,16 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), snapshot.len(), |ptes| { - ptes.iter() - .zip(snapshot) - .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) - }) + pgtable::with_root_entries( + self.inner.reserved.as_deref(), + self.kernel_vm.ttb(), + snapshot.len(), + |ptes| { + ptes.iter() + .zip(snapshot) + .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) + }, + ) }?; if !matches { return Err(EIO); @@ -3479,7 +3540,7 @@ impl Uat { // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, // and the region is at least large enough per the size specified above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); let shared_quirks = quirks; @@ -3608,7 +3669,10 @@ impl Uat { inner.handoff().init(firmware, quirks.handoff_slot)?; if quirks.clean_tables { - pgtable::clean_poc(inner.handoff() as *const Handoff as *const u8, HANDOFF_SIZE); + pgtable::clean_poc( + core::ptr::from_ref(inner.handoff()).cast::(), + HANDOFF_SIZE, + ); pgtable::poc_barrier(); } @@ -3636,7 +3700,7 @@ impl Uat { .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } if quirks.clean_tables { - pgtable::clean_poc(ttbs.as_ptr() as *const u8, SLOTS_SIZE); + pgtable::clean_poc(ttbs.as_ptr().cast::(), SLOTS_SIZE); pgtable::poc_barrier(); } diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index ecf040670cfaa..0c9e6f62b5bbe 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -152,9 +152,9 @@ macro_rules! inner_ptr { // we don't actually read from `outer` (which would be UB) nor create an intermediate // reference. let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p as *const u8; + let inner = p.cast::(); // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer as *const u8) }; + let off = unsafe { inner.offset_from(outer.cast::()) }; // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer // object. unsafe { $gpuva.offset(off.try_into().unwrap(), p) } @@ -234,9 +234,9 @@ macro_rules! inner_weak_ptr { // we don't actually read from `outer` (which would be UB) nor create an intermediate // reference. let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p as *const u8; + let inner = p.cast::(); // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer as *const u8) }; + let off = unsafe { inner.offset_from(outer.cast::()) }; // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer // object. unsafe { $gpuva.offset(off.try_into().unwrap(), p) } @@ -298,11 +298,15 @@ impl> GpuObject { core::any::type_name::(), alloc.gpu_ptr() ); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'static>; + let p = alloc + .ptr() + .ok_or(EINVAL)? + .as_ptr() + .cast::>(); let mut raw = callback(&inner); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant, and the type is // identical to the type of `raw` other than the lifetime. - unsafe { p.copy_from(&mut raw as *mut _ as *mut u8 as *mut _, 1) }; + unsafe { p.copy_from(core::ptr::from_mut(&mut raw).cast::().cast(), 1) }; mem::forget(raw); Ok(Self { raw: p, @@ -337,10 +341,14 @@ impl> GpuObject { core::any::type_name::(), alloc.gpu_ptr() ); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; + let p = alloc + .ptr() + .ok_or(EINVAL)? + .as_ptr() + .cast::>>(); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. let raw = callback(&inner, unsafe { &mut *p })?; - if p as *mut T::Raw<'_> != raw as *mut _ { + if !core::ptr::eq(p.cast::>(), raw) { dev_err!( alloc.device().as_ref(), "Allocation callback returned a mismatched reference ({})\n", @@ -349,7 +357,7 @@ impl> GpuObject { return Err(EINVAL); } Ok(Self { - raw: p as *mut u8 as *mut T::Raw<'static>, + raw: p.cast::().cast::>(), gpu_ptr, alloc, inner, @@ -400,14 +408,14 @@ impl> GpuObject { alloc.gpu_ptr() ); let inner = inner_init(gpu_ptr); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'_>; + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr().cast::>(); let ret = Self { - raw: p as *mut u8 as *mut T::Raw<'static>, + raw: p.cast::().cast::>(), gpu_ptr, alloc, inner: KBox::init(inner, GFP_KERNEL)?, }; - let q = &*ret.inner as *const T; + let q = core::ptr::from_ref::(&*ret.inner); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. unsafe { pin_init::raw_try_init(p, raw_init(&*q, gpu_ptr)) }?; Ok(ret) @@ -437,7 +445,7 @@ impl> GpuObject { ) -> RetVal { // SAFETY: `self.raw` is valid per the type invariant, and the second half is just // converting lifetimes. - unsafe { callback(&mut *self.raw, &mut *(&mut *self.inner as *mut _)) } + unsafe { callback(&mut *self.raw, &mut *core::ptr::from_mut(&mut *self.inner)) } } /// Access the inner `Raw` data given a user-supplied callback. @@ -449,7 +457,7 @@ impl> GpuObject { ) -> RetVal { // SAFETY: `self.raw` is valid per the type invariant, and the second half is just // converting lifetimes. - unsafe { callback(&*self.raw, &*(&*self.inner as *const _)) } + unsafe { callback(&*self.raw, &*core::ptr::from_ref(&*self.inner)) } } } @@ -475,8 +483,8 @@ where f.debug_struct(core::any::type_name::()) // SAFETY: `self.raw` is valid per the type invariant. .field("raw", &format_args!("{:#X?}", unsafe { &*self.raw })) - .field("inner", &format_args!("{:#X?}", &self.inner)) - .field("alloc", &format_args!("{:?}", &self.alloc)) + .field("inner", &format_args!("{:#X?}", self.inner)) + .field("alloc", &format_args!("{:?}", self.alloc)) .finish() } } diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 2fabe6bee88d6..0f915607cb1f4 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -153,9 +153,8 @@ impl ReservedTables { pub(crate) unsafe fn add(&mut self, res: kernel::io::resource::Resource) -> Result { let start = res.start(); // SAFETY: The caller guarantees the region; write-back matches how the firmware maps it. - let map = unsafe { - kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? - }; + let map = + unsafe { kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? }; self.starts.push(start, GFP_KERNEL)?; self.regions.push(map, GFP_KERNEL)?; Ok(()) @@ -671,7 +670,7 @@ impl UatPageTable { let driver_tables = &mut self.driver_tables; let oas_mask = self.oas_mask; let entry = |p: *mut u8| -> Result> { - let uptep = p as *const _ as *const Pte; + let uptep = p.cast_const().cast::(); // SAFETY: with_table() ensures the pointer is valid, and our index is // aligned so it is safe to deref as an AtomicU64. let upte = unsafe { &*uptep }; @@ -679,7 +678,10 @@ impl UatPageTable { // Allocate if requested if upte_val == 0 && alloc { let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + mod_pr_debug!( + "UATPageTable::with_pages: alloc PT at {:#x}\n", + pt_page.phys() + ); if clean { // A non-snooping walker must never follow a parent to a // table whose zeroes are still in the CPU caches. @@ -755,7 +757,7 @@ impl UatPageTable { iova ); let leaves = |p: *mut u8| -> Result { - let ptep = p as *const _ as *const Pte; + let ptep = p.cast_const().cast::(); // SAFETY: We know this is a valid pointer to PTEs and the range is valid and // checked by with_table(). let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; @@ -769,7 +771,13 @@ impl UatPageTable { // SAFETY: Page table addresses are either allocated by us, or // firmware-managed tables in RAM or in a reserved region. unsafe { - with_table(reserved.as_deref(), phys, idx * PTE_SIZE, count * PTE_SIZE, leaves) + with_table( + reserved.as_deref(), + phys, + idx * PTE_SIZE, + count * PTE_SIZE, + leaves, + ) }?; let block = 1 << (UAT_PGBIT + UAT_LVBIT); @@ -806,7 +814,10 @@ impl UatPageTable { } if let Some((parent_phys, parent_index)) = parent { let oas_mask = self.oas_mask; - let in_reserved = self.reserved.as_ref().is_some_and(|r| r.contains(parent_phys)); + let in_reserved = self + .reserved + .as_ref() + .is_some_and(|r| r.contains(parent_phys)); // SAFETY: Only looks the page up; nothing is accessed through it. if !in_reserved && unsafe { Page::borrow_phys(&parent_phys) }.is_none() { return Err(EIO); @@ -832,7 +843,15 @@ impl UatPageTable { // its child descriptor is detached. A changed descriptor indicates // an ownership violation; fail closed instead of freeing a table // that may still be reachable. - unsafe { with_table(reserved, parent_phys, parent_index * PTE_SIZE, PTE_SIZE, detach) }?; + unsafe { + with_table( + reserved, + parent_phys, + parent_index * PTE_SIZE, + PTE_SIZE, + detach, + ) + }?; } if let Some(tables) = self.driver_tables.as_mut() { let index = tables.iter().position(|&table| table == phys).ok_or(EIO)?; diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index ff9804c5bd10b..a89fcaf0b8dcb 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -35,6 +35,10 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::Compute; #[versions(AGX)] impl super::QueueInner::ver { /// Submit work to a compute queue. + #[expect( + clippy::too_many_arguments, + reason = "Submission inputs are retained separately by the existing queue interface." + )] pub(super) fn submit_compute( &self, gpu: &gpu::GpuManager::ver, @@ -54,9 +58,10 @@ impl super::QueueInner::ver { return Err(EINVAL); } - let mut user_timestamps: fw::job::UserTimestamps = Default::default(); - user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts.start)?; - user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts.end)?; + let user_timestamps = fw::job::UserTimestamps { + start: common::get_timestamp_object(objects, cmdbuf.ts.start)?, + end: common::get_timestamp_object(objects, cmdbuf.ts.end)?, + }; // This sequence number increases per new client/VM? assigned to some slot, // but it's unclear *which* slot... @@ -214,7 +219,7 @@ impl super::QueueInner::ver { #[ver(G == G14 && V < V13_0B4)] unk_5c_g14: U64(0), restart_branch_offset: off, - has_attachments: (attachments.count > 0) as u32, + has_attachments: u32::from(attachments.count > 0), #[ver(V >= V13_0B4)] unk_64: Default::default(), #[ver(V >= V13_0B4)] @@ -316,8 +321,8 @@ impl super::QueueInner::ver { unk_18: 0x0, // fixed unk_mask: 0xffffffff, sampler_array: U64(cmdbuf.sampler_heap), - sampler_count: cmdbuf.sampler_count as u32, - sampler_max: (cmdbuf.sampler_count as u32) + 1, + sampler_count: cmdbuf.sampler_count, + sampler_max: cmdbuf.sampler_count + 1, }), meta <- try_init!(fw::job::raw::JobMeta { unk_0: 0, @@ -328,7 +333,7 @@ impl super::QueueInner::ver { stamp_value: ev_comp.value.next(), stamp_slot: ev_comp.slot, evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, + flush_stamps: u32::from(flush_stamps), uuid, event_seq: ev_comp.event_seq as u32, }), diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 40baaa2e6d897..93c4eb0351187 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -364,7 +364,7 @@ impl sched::JobImpl for QueueJob::ver { return Err(ENODEV); }; - return guard.registration_data_with(|reg_data| { + guard.registration_data_with(|reg_data| { let gpu = match reg_data .gpu .clone() @@ -439,7 +439,7 @@ impl sched::JobImpl for QueueJob::ver { core::mem::drop(frag_job); Ok(Some(Fence::from_fence(&job.fence))) - }); + }) } fn timed_out(job: &mut QueueTimeout, _finished: bool) -> sched::Status { @@ -637,8 +637,11 @@ const COMPUTE_SUPPORTED_SIZE: usize = // SAFETY: All bit patterns are valid by construction. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_header {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_render {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_compute {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_attachment {} fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result { @@ -649,8 +652,10 @@ fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result, size: usize) -> Result> 7; - attachments.list[i as usize] = microseq::Attachment { + attachments.list[i] = microseq::Attachment { address: U64(att.pointer), size: cache_lines.try_into()?, unk_c: 0x17, @@ -769,7 +774,7 @@ impl Queue for Queue::ver { while !cmdbuf.is_empty() { let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; let payload_size = header.size as usize; - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(payload_size)?; if render @@ -794,7 +799,7 @@ impl Queue for Queue::ver { } nr_commands += 1; - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { last_render = nr_commands; nr_render += 1; @@ -886,7 +891,7 @@ impl Queue for Queue::ver { EINVAL })? { let mut alloc = gpu.alloc(); - let queue_job = match header.cmd_type as u32 { + let queue_job = match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => job.get_vtx()?, uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => job.get_comp()?, _ => return Err(EINVAL), @@ -916,7 +921,7 @@ impl Queue for Queue::ver { } } - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { let render: uapi::drm_asahi_cmd_render = cmdbuf .read_supported(header_size, RENDER_SUPPORTED_SIZE) diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 4de0e84239a02..e02411e2417c0 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -47,9 +47,9 @@ impl super::QueueInner::ver { cmdbuf: &uapi::drm_asahi_cmd_render, num_clusters: u32, ) -> Result { - let width: u32 = cmdbuf.width_px as u32; - let height: u32 = cmdbuf.height_px as u32; - let layers: u32 = cmdbuf.layers as u32; + let width: u32 = u32::from(cmdbuf.width_px); + let height: u32 = u32::from(cmdbuf.height_px); + let layers: u32 = u32::from(cmdbuf.layers); if layers == 0 || layers > 2048 { cls_pr_debug!(Errors, "Layer count invalid ({})\n", layers); @@ -68,8 +68,8 @@ impl super::QueueInner::ver { let tile_width = 32u32; let tile_height = 32u32; - let utile_width = cmdbuf.utile_width_px as u32; - let utile_height = cmdbuf.utile_height_px as u32; + let utile_width = u32::from(cmdbuf.utile_width_px); + let utile_height = u32::from(cmdbuf.utile_height_px); match (utile_width, utile_height) { (32, 32) | (32, 16) | (16, 16) => (), @@ -153,7 +153,7 @@ impl super::QueueInner::ver { tpc_size, meta1_layer_stride, #[ver(G < G14X)] - meta1_blocks: meta1_layer_stride * (cmdbuf.layers as u32), + meta1_blocks: meta1_layer_stride * u32::from(cmdbuf.layers), #[ver(G >= G14X)] meta1_blocks: meta1_layer_stride, layermeta_size: if layers > 1 { 0x100 } else { 0 }, @@ -182,6 +182,10 @@ impl super::QueueInner::ver { } /// Submit work to a render queue. + #[expect( + clippy::too_many_arguments, + reason = "Submission inputs are retained separately by the existing queue interface." + )] pub(super) fn submit_render( &self, gpu: &gpu::GpuManager::ver, @@ -199,7 +203,7 @@ impl super::QueueInner::ver { & !(uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING - | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT) as u32 + | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT) != 0 { cls_pr_debug!(Errors, "Invalid flags ({:#x})\n", cmdbuf.flags); @@ -235,8 +239,7 @@ impl super::QueueInner::ver { let mut clustering = nclusters > 1; if debug_enabled(debug::DebugFlags::DisableClustering) - || cmdbuf.flags - & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING as u32 + || cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING != 0 { clustering = false; @@ -260,7 +263,7 @@ impl super::QueueInner::ver { // but it's unclear *which* slot... let slot_client_seq: u8 = (self.id & 0xff) as u8; - let tile_info = Self::get_tiling_params(&cmdbuf, if clustering { nclusters } else { 1 })?; + let tile_info = Self::get_tiling_params(cmdbuf, if clustering { nclusters } else { 1 })?; let buffer = &self.buffer; let notifier = self.notifier.clone(); @@ -375,9 +378,7 @@ impl super::QueueInner::ver { if cmdbuf.layers > 1 { tile_config |= 1; } - if cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES as u32 - != 0 - { + if cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES != 0 { tile_config |= 0x10000; } @@ -398,19 +399,19 @@ impl super::QueueInner::ver { // Calculate the number of 2KiB blocks to allocate per utile. This is // just a bit of dimensional analysis. let pixels_per_utile: u32 = - (cmdbuf.utile_width_px as u32) * (cmdbuf.utile_height_px as u32); + u32::from(cmdbuf.utile_width_px) * u32::from(cmdbuf.utile_height_px); let samples_per_utile: u32 = pixels_per_utile << samples_log2; - let utile_size_bytes: u32 = (cmdbuf.sample_size_B as u32) * samples_per_utile; + let utile_size_bytes: u32 = u32::from(cmdbuf.sample_size_B) * samples_per_utile; let block_size_bytes: u32 = 2048; let blocks_per_utile: u32 = utile_size_bytes.div_ceil(block_size_bytes); #[ver(G >= G14X)] let frg_tilecfg = 0x0000000_00036011 - | (((tile_info.tiles_x - 1) as u64) << 44) - | (((tile_info.tiles_y - 1) as u64) << 53) + | (u64::from(tile_info.tiles_x - 1) << 44) + | (u64::from(tile_info.tiles_y - 1) << 53) | (if unk1 { 0 } else { 0x20_00000000 }) | (if cmdbuf.layers > 1 { 0x1_00000000 } else { 0 }) - | ((utile_config as u64 & 0xf000) << 28); + | ((u64::from(utile_config) & 0xf000) << 28); // TODO: check #[ver(V >= V13_0B4)] @@ -428,9 +429,9 @@ impl super::QueueInner::ver { let frg_unk_140 = 0x8c60; let frg_unk_158 = 0x1c; #[ver(G >= G14)] - let load_bgobjvals = cmdbuf.isp_bgobjvals as u64; + let load_bgobjvals = u64::from(cmdbuf.isp_bgobjvals); #[ver(G < G14)] - let load_bgobjvals = cmdbuf.isp_bgobjvals as u64 | 0x400; + let load_bgobjvals = u64::from(cmdbuf.isp_bgobjvals) | 0x400; let reload_zlsctrl = cmdbuf.zls_ctrl; let iogpu_unk54 = 0x3a0012006b0003; let iogpu_unk56 = 1; @@ -441,12 +442,12 @@ impl super::QueueInner::ver { #[ver(G >= G14X)] let vtx_unk_f0 = 0x1c; #[ver(G < G14)] - let vtx_unk_f0 = 0x1c + (align(tile_info.meta1_blocks, 4) as u64); + let vtx_unk_f0 = 0x1c + u64::from(align(tile_info.meta1_blocks, 4)); let vtx_unk_118 = 0x1c; // DRM_ASAHI_RENDER_DBIAS_IS_INT chosen to match hardware bit. - let isp_ctl = 0xc000u32 - | (cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT as u32); + let isp_ctl = + 0xc000u32 | (cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT); // Always allow preemption at the UAPI level let no_preemption = false; @@ -575,7 +576,7 @@ impl super::QueueInner::ver { #[ver(G == G14 && V < V13_0B4)] unk_8c_g14: U64(0), restart_branch_offset: off, - has_attachments: (fragment_attachments.count > 0) as u32, + has_attachments: u32::from(fragment_attachments.count > 0), #[ver(V >= V13_0B4)] unk_9c: Default::default(), })?; @@ -597,10 +598,10 @@ impl super::QueueInner::ver { |inner, _ptr| { let vm_slot = vm_bind.slot(); let aux_fb_info = fw::fragment::raw::AuxFBInfo::ver { - isp_ctl: isp_ctl, + isp_ctl, unk2: 0, - width: cmdbuf.width_px as u32, - height: cmdbuf.height_px as u32, + width: u32::from(cmdbuf.width_px), + height: u32::from(cmdbuf.height_px), #[ver(V >= V13_0B4)] unk3: U64(0x100000), }; @@ -619,7 +620,7 @@ impl super::QueueInner::ver { unk_buffer_buf: inner.scene.kernel_buffer_pointer(), tvb_tilemap: inner.scene.tvb_tilemap_pointer(), ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), - samples: cmdbuf.samples as u32, + samples: u32::from(cmdbuf.samples), tiles_per_mtile_y: tile_info.tiles_per_mtile_y as u16, tiles_per_mtile_x: tile_info.tiles_per_mtile_x as u16, unk_50: U64(0), @@ -627,21 +628,21 @@ impl super::QueueInner::ver { isp_merge_upper_x: F32::from_bits(cmdbuf.isp_merge_upper_x), isp_merge_upper_y: F32::from_bits(cmdbuf.isp_merge_upper_y), unk_68: U64(0), - tile_count: U64(tile_info.tiles as u64), + tile_count: U64(u64::from(tile_info.tiles)), #[ver(G < G14X)] job_params1 <- try_init!(fw::fragment::raw::JobParameters1::ver { utile_config, unk_4: 0, bg: fw::fragment::raw::BackgroundProgram { - rsrc_spec: U64(cmdbuf.bg.rsrc_spec as u64), - address: U64(cmdbuf.bg.usc as u64), + rsrc_spec: U64(u64::from(cmdbuf.bg.rsrc_spec)), + address: U64(u64::from(cmdbuf.bg.usc)), }, ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), isp_scissor_base: U64(cmdbuf.isp_scissor_base), isp_dbias_base: U64(cmdbuf.isp_dbias_base), isp_oclqry_base: U64(cmdbuf.isp_oclqry_base), aux_fb_info, - isp_zls_pixels: U64(cmdbuf.isp_zls_pixels as u64), + isp_zls_pixels: U64(u64::from(cmdbuf.isp_zls_pixels)), zls_ctrl: U64(cmdbuf.zls_ctrl), #[ver(G >= G14)] unk_58_g14_0: U64(g14_unk), @@ -653,21 +654,21 @@ impl super::QueueInner::ver { s_store: U64(cmdbuf.stencil.base), #[ver(G >= G14)] unk_68_g14_0: Default::default(), - z_load_stride: U64(cmdbuf.depth.stride as u64), - z_store_stride: U64(cmdbuf.depth.stride as u64), - s_load_stride: U64(cmdbuf.stencil.stride as u64), - s_store_stride: U64(cmdbuf.stencil.stride as u64), + z_load_stride: U64(u64::from(cmdbuf.depth.stride)), + z_store_stride: U64(u64::from(cmdbuf.depth.stride)), + s_load_stride: U64(u64::from(cmdbuf.stencil.stride)), + s_store_stride: U64(u64::from(cmdbuf.stencil.stride)), z_load_comp: U64(cmdbuf.depth.comp_base), - z_load_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_load_comp_stride: U64(u64::from(cmdbuf.depth.comp_stride)), z_store_comp: U64(cmdbuf.depth.comp_base), - z_store_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_store_comp_stride: U64(u64::from(cmdbuf.depth.comp_stride)), s_load_comp: U64(cmdbuf.stencil.comp_base), - s_load_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_load_comp_stride: U64(u64::from(cmdbuf.stencil.comp_stride)), s_store_comp: U64(cmdbuf.stencil.comp_base), - s_store_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_store_comp_stride: U64(u64::from(cmdbuf.stencil.comp_stride)), tvb_tilemap: inner.scene.tvb_tilemap_pointer(), tvb_layermeta: inner.scene.tvb_layermeta_pointer(), - mtile_stride_dwords: U64((4 * tile_info.params.rgn_size as u64) << 24), + mtile_stride_dwords: U64((4 * u64::from(tile_info.params.rgn_size)) << 24), tvb_heapmeta: inner.scene.tvb_heapmeta_pointer(), tile_config: U64(tile_config), aux_fb: inner.aux_fb.gpu_pointer(), @@ -737,7 +738,7 @@ impl super::QueueInner::ver { r.add(0x15021, isp_ctl.into()); // aux_fb_info.unk_1 r.add( 0x15211, - ((cmdbuf.height_px as u64) << 32) | cmdbuf.width_px as u64, + (u64::from(cmdbuf.height_px) << 32) | u64::from(cmdbuf.width_px), ); // aux_fb_info.{width, heigh r.add(0x15049, 0x100000); // s2.aux_fb_info.unk3 r.add(0x10051, blocks_per_utile.into()); // s1.unk_2c @@ -756,21 +757,21 @@ impl super::QueueInner::ver { r.add(0x15221, 0); r.add(0x15239, 0); r.add(0x15229, 0); - r.add(0x15401, cmdbuf.depth.stride as u64); // load - r.add(0x15421, cmdbuf.depth.stride as u64); // store - r.add(0x15409, cmdbuf.stencil.stride as u64); // load - r.add(0x15429, cmdbuf.stencil.stride as u64); + r.add(0x15401, u64::from(cmdbuf.depth.stride)); // load + r.add(0x15421, u64::from(cmdbuf.depth.stride)); // store + r.add(0x15409, u64::from(cmdbuf.stencil.stride)); // load + r.add(0x15429, u64::from(cmdbuf.stencil.stride)); r.add(0x153c1, cmdbuf.depth.comp_base); // load - r.add(0x15411, cmdbuf.depth.comp_stride as u64); // load + r.add(0x15411, u64::from(cmdbuf.depth.comp_stride)); // load r.add(0x153c9, cmdbuf.depth.comp_base); // store - r.add(0x15431, cmdbuf.depth.comp_stride as u64); // store + r.add(0x15431, u64::from(cmdbuf.depth.comp_stride)); // store r.add(0x153d1, cmdbuf.stencil.comp_base); // load - r.add(0x15419, cmdbuf.stencil.comp_stride as u64); // load + r.add(0x15419, u64::from(cmdbuf.stencil.comp_stride)); // load r.add(0x153d9, cmdbuf.stencil.comp_base); // store - r.add(0x15439, cmdbuf.stencil.comp_stride as u64); // store + r.add(0x15439, u64::from(cmdbuf.stencil.comp_stride)); // store r.add(0x16429, inner.scene.tvb_tilemap_pointer().into()); r.add(0x16060, inner.scene.tvb_layermeta_pointer().into()); - r.add(0x16431, (4 * tile_info.params.rgn_size as u64) << 24); // ISP_RGN? + r.add(0x16431, (4 * u64::from(tile_info.params.rgn_size)) << 24); // ISP_RGN? r.add(0x10039, tile_config); // tile_config ISP_CTL? r.add(0x16451, 0x0); // ISP_RENDER_ORIGIN r.add(0x11821, cmdbuf.fragment_helper.binary.into()); @@ -807,28 +808,28 @@ impl super::QueueInner::ver { unk_118: U64(0x0), unk_120: Default::default(), unk_partial_bg: fw::fragment::raw::BackgroundProgram { - rsrc_spec: U64(cmdbuf.partial_bg.rsrc_spec as u64), - address: U64(cmdbuf.partial_bg.usc as u64), + rsrc_spec: U64(u64::from(cmdbuf.partial_bg.rsrc_spec)), + address: U64(u64::from(cmdbuf.partial_bg.usc)), }, unk_258: U64(0), unk_260: U64(0), unk_268: U64(0), unk_270: U64(0), partial_bg: fw::fragment::raw::BackgroundProgram { - rsrc_spec: U64(cmdbuf.partial_bg.rsrc_spec as u64), - address: U64(cmdbuf.partial_bg.usc as u64), + rsrc_spec: U64(u64::from(cmdbuf.partial_bg.rsrc_spec)), + address: U64(u64::from(cmdbuf.partial_bg.usc)), }, zls_ctrl: U64(reload_zlsctrl), unk_290: U64(g14_unk), z_load: U64(cmdbuf.depth.base), - z_partial_stride: U64(cmdbuf.depth.stride as u64), - z_partial_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_partial_stride: U64(u64::from(cmdbuf.depth.stride)), + z_partial_comp_stride: U64(u64::from(cmdbuf.depth.comp_stride)), z_store: U64(cmdbuf.depth.base), z_partial: U64(cmdbuf.depth.base), z_partial_comp: U64(cmdbuf.depth.comp_base), s_load: U64(cmdbuf.stencil.base), - s_partial_stride: U64(cmdbuf.stencil.stride as u64), - s_partial_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_partial_stride: U64(u64::from(cmdbuf.stencil.stride)), + s_partial_comp_stride: U64(u64::from(cmdbuf.stencil.comp_stride)), s_store: U64(cmdbuf.stencil.base), s_partial: U64(cmdbuf.stencil.base), s_partial_comp: U64(cmdbuf.stencil.comp_base), @@ -848,13 +849,13 @@ impl super::QueueInner::ver { ), isp_bgobjdepth: cmdbuf.isp_bgobjdepth, isp_bgobjvals: cmdbuf.isp_bgobjvals, - sample_size: cmdbuf.sample_size_B as u32, + sample_size: u32::from(cmdbuf.sample_size_B), unk_37c: 0x0, unk_380: U64(0x0), unk_388: U64(0x0), #[ver(V >= V13_0B4)] unk_390_0: U64(0x0), - isp_zls_pixels: U64(cmdbuf.isp_zls_pixels as u64), + isp_zls_pixels: U64(u64::from(cmdbuf.isp_zls_pixels)), }), unk_758_flag: 0, unk_75c_flag: 0, @@ -871,12 +872,11 @@ impl super::QueueInner::ver { unk_18: 0x0, // fixed unk_mask: 0xffffffffu32, sampler_array: U64(cmdbuf.sampler_heap), - sampler_count: cmdbuf.sampler_count as u32, - sampler_max: (cmdbuf.sampler_count as u32) + 1, + sampler_count: u32::from(cmdbuf.sampler_count), + sampler_max: u32::from(cmdbuf.sampler_count) + 1, }), - process_empty_tiles: (cmdbuf.flags - & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES as u32 - != 0) as u32, + process_empty_tiles: u32::from(cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES != 0), // TODO: needs to be investigated no_clear_pipeline_textures: 1, // TODO: needs to be investigated @@ -887,13 +887,13 @@ impl super::QueueInner::ver { meta <- try_init!(fw::job::raw::JobMeta { unk_0: 0, unk_2: 0, - no_preemption: no_preemption as u8, + no_preemption: u8::from(no_preemption), stamp: ev_frag.stamp_pointer, fw_stamp: ev_frag.fw_stamp_pointer, stamp_value: ev_frag.value.next(), stamp_slot: ev_frag.slot, evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, + flush_stamps: u32::from(flush_stamps), uuid: uuid_3d, event_seq: ev_frag.event_seq as u32, }), @@ -1027,7 +1027,7 @@ impl super::QueueInner::ver { #[ver(V < V13_0B4)] unk_178: 0x0, // padding? #[ver(V >= V13_0B4)] - unk_178: (!clustering) as u32, + unk_178: u32::from(!clustering), })?; builder.add(microseq::Timestamp::ver { @@ -1089,7 +1089,7 @@ impl super::QueueInner::ver { #[ver(G >= G14 && V < V13_0B4)] unk_68_g14: U64(0), restart_branch_offset: off, - has_attachments: (vertex_attachments.count > 0) as u32, + has_attachments: u32::from(vertex_attachments.count > 0), #[ver(V >= V13_0B4)] unk_74: Default::default(), // Ventura })?; @@ -1140,7 +1140,7 @@ impl super::QueueInner::ver { tvb_cluster_meta1: inner .scene .meta_1_pointer() - .map(|x| x.or((tile_info.meta1_layer_stride as u64) << 50)), + .map(|x| x.or(u64::from(tile_info.meta1_layer_stride) << 50)), utile_config, unk_4c: 0, ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), // fixed @@ -1268,8 +1268,8 @@ impl super::QueueInner::ver { r.add(0x10121, tile_info.params.ppp_ctrl.into()); // PPP_CTRL r.add( 0x10129, - tile_info.params.x_max as u64 - | ((tile_info.params.y_max as u64) << 16), + u64::from(tile_info.params.x_max) + | (u64::from(tile_info.params.y_max) << 16), ); // PPP_SCREEN r.add(0x101b9, tile_info.params.te_screen.into()); // TE_SCREEN r.add(0x1c069, tile_info.params.te_mtile1.into()); // TE_MTILE1 @@ -1323,26 +1323,25 @@ impl super::QueueInner::ver { unk_18: 0x0, // fixed unk_mask: 0xffffffffu32, sampler_array: U64(cmdbuf.sampler_heap), - sampler_count: cmdbuf.sampler_count as u32, - sampler_max: (cmdbuf.sampler_count as u32) + 1, + sampler_count: u32::from(cmdbuf.sampler_count), + sampler_max: u32::from(cmdbuf.sampler_count) + 1, }), unk_55c: 0, unk_560: 0, sync_grow: 0, unk_568: 0, - uses_scratch: (cmdbuf.flags - & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH as u32 - != 0) as u32, + uses_scratch: u32::from(cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH != 0), meta <- try_init!(fw::job::raw::JobMeta { unk_0: 0, unk_2: 0, - no_preemption: no_preemption as u8, + no_preemption: u8::from(no_preemption), stamp: ev_vtx.stamp_pointer, fw_stamp: ev_vtx.fw_stamp_pointer, stamp_value: ev_vtx.value.next(), stamp_slot: ev_vtx.slot, evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, + flush_stamps: u32::from(flush_stamps), uuid: uuid_ta, event_seq: ev_vtx.event_seq as u32, }), diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs index 772d969d9baff..d07de2c6e7384 100644 --- a/drivers/gpu/drm/asahi/stats.rs +++ b/drivers/gpu/drm/asahi/stats.rs @@ -11,7 +11,8 @@ //! the versioned context where `StatsMsg` is nameable; `note_job()` is called //! from the `recv_message` rtkit callback (so the firmware mailbox IRQ //! context); the sysfs `show` callback runs in arbitrary process context. All -//! fields use `AtomicU*` so neither side takes a lock. +//! legacy fields use `AtomicU*`. G17 instead uses the optional reader under +//! its device mutex and does not populate these legacy atomic fields. //! //! Field semantics (firmware units, late-validated): //! - `util1..util4`: raw `Utilization` u32s from the firmware. Meaning @@ -58,6 +59,9 @@ pub(crate) struct StatsSnapshot { /// `JobFence::command_complete` when the last command of a submission /// completes. pub(crate) jobs: AtomicU64, + /// Optional qualified reader. The legacy C layout above remains a prefix; + /// C accesses this extension only through the registered Rust callbacks. + pub(crate) g17_reader: Option, } // Keep the Rust half of the C sysfs layout contract checked as well. @@ -65,7 +69,7 @@ const _: () = { assert!(core::mem::offset_of!(StatsSnapshot, pstate) == 16); assert!(core::mem::offset_of!(StatsSnapshot, busy_ns) == 40); assert!(core::mem::offset_of!(StatsSnapshot, jobs) == 48); - assert!(core::mem::size_of::() == 56); + assert!(core::mem::offset_of!(StatsSnapshot, g17_reader) == 56); }; impl StatsSnapshot { diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 7d7d785a83133..a704e3f1f7afd 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -16,7 +18,7 @@ #include "sysfs.h" -/* Mirror of repr(C) Rust atomics; READ_ONCE accesses the current values. */ +/* Legacy prefix of repr(C) Rust atomics; READ_ONCE accesses current values. */ struct asahi_stats_snapshot { u32 util1; u32 util2; @@ -48,8 +50,69 @@ struct asahi_stats_attribute { const struct asahi_stats_snapshot *snapshot; const struct asahi_gpu_memory *memory; bool export_enabled; + asahi_stats_read_fn read; + asahi_stats_suspend_fn suspend; + struct notifier_block pm_notifier; }; +/* 0: unavailable, 1: valid, 2: stale; matches g17::telemetry. */ +static const char *asahi_stats_quality(u32 status) +{ + return status == 1 ? "valid" : status == 2 ? "stale" : "unavailable"; +} + +static ssize_t agx_qualified_stats_show(struct asahi_stats_attribute *stats, char *buf) +{ + struct asahi_qualified_stats snap; + ssize_t n = 0; + + stats->read(stats->snapshot, &snap); + n += sysfs_emit_at(buf, n, "utilisation_status %s\n", + asahi_stats_quality(snap.util_status)); + if (snap.util_status == 1) { + n += sysfs_emit_at(buf, n, "util_tiler_percent %u\n", snap.tiler); + n += sysfs_emit_at(buf, n, "util_renderer_percent %u\n", snap.renderer); + n += sysfs_emit_at(buf, n, "util_device_percent %u\n", snap.device); + } else { + n += sysfs_emit_at(buf, n, "util_tiler_percent unsupported\n" + "util_renderer_percent unsupported\n" + "util_device_percent unsupported\n"); + } + n += sysfs_emit_at(buf, n, "pstate_status %s\n", + asahi_stats_quality(snap.pstate_status)); + if (snap.pstate_status == 1) + n += sysfs_emit_at(buf, n, "pstate %u\n", snap.pstate); + else + n += sysfs_emit_at(buf, n, "pstate unsupported\n"); + /* No A/B selector, packet duration, measured power or temperature. */ + n += sysfs_emit_at(buf, n, "requested_frequency_hz unsupported\n" + "achieved_frequency_hz unsupported\n" + "busy_ns unsupported\n" + "power_mw unsupported\n" + "temperature_raw unsupported\n"); + return n; +} + +static int asahi_stats_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +{ + struct asahi_stats_attribute *stats = + container_of(nb, struct asahi_stats_attribute, pm_notifier); + + switch (action) { + case PM_SUSPEND_PREPARE: + case PM_HIBERNATION_PREPARE: + case PM_RESTORE_PREPARE: + stats->suspend(stats->snapshot, true); + break; + case PM_POST_SUSPEND: + case PM_POST_HIBERNATION: + case PM_POST_RESTORE: + stats->suspend(stats->snapshot, false); + break; + } + return NOTIFY_OK; +} + static ssize_t agx_stats_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -60,6 +123,8 @@ static ssize_t agx_stats_show(struct device *dev, if (!stats->export_enabled) return sysfs_emit(buf, "unsupported\n"); + if (stats->read) + return agx_qualified_stats_show(stats, buf); n += scnprintf(buf + n, PAGE_SIZE - n, "busy_ns %llu\n", (unsigned long long)READ_ONCE(snap->busy_ns)); n += scnprintf(buf + n, PAGE_SIZE - n, "jobs %llu\n", @@ -118,14 +183,18 @@ static ssize_t gpu_memory_show(struct device *dev, * registered. Returns 0 on success, or a negative errno. */ int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, const void *memory, void **handle) + int export_enabled, const void *memory, void **handle, + asahi_stats_read_fn read, asahi_stats_suspend_fn suspend) { struct asahi_stats_attribute *stats; int ret; - if (!dev || !handle || !memory || (export_enabled && !snapshot)) + if (!dev || !handle || !memory || ((export_enabled || read) && !snapshot) || + (!!read != !!suspend)) return -ENODEV; + BUILD_BUG_ON(offsetof(struct asahi_qualified_stats, util_host_ns) != 24); + BUILD_BUG_ON(sizeof(struct asahi_qualified_stats) != 56); BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, refused) != 24); BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, client_limit) != 40); BUILD_BUG_ON(sizeof(struct asahi_gpu_memory) != 48); @@ -145,6 +214,16 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, stats->attr.show = agx_stats_show; stats->snapshot = snapshot; stats->export_enabled = export_enabled; + stats->read = read; + stats->suspend = suspend; + if (suspend) { + stats->pm_notifier.notifier_call = asahi_stats_pm_notify; + ret = register_pm_notifier(&stats->pm_notifier); + if (ret) { + kfree(stats); + return ret; + } + } sysfs_attr_init(&stats->memory_attr.attr); stats->memory_attr.attr.name = "gpu_memory"; @@ -154,6 +233,8 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, ret = device_create_file(dev, &stats->attr); if (ret) { + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); kfree(stats); return ret; } @@ -161,6 +242,8 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, ret = device_create_file(dev, &stats->memory_attr); if (ret) { device_remove_file(dev, &stats->attr); + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); kfree(stats); return ret; } @@ -217,6 +300,9 @@ void asahi_sysfs_unregister(struct device *dev, void *handle) if (!dev || !stats) return; + /* Both notifier removal and file removal join readers of the Rust Arc. */ + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); /* Removal drains active show callbacks before either allocation dies. */ device_remove_file(dev, &stats->memory_attr); device_remove_file(dev, &stats->attr); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index f1873f9be6caa..d43f482903e55 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -3,10 +3,30 @@ #ifndef _ASAHI_SYSFS_H #define _ASAHI_SYSFS_H +#include + struct device; +/* Mirrors g17::telemetry::Readout, filled under the Rust device mutex. */ +struct asahi_qualified_stats { + u32 util_status; + u32 pstate_status; + u32 tiler; + u32 renderer; + u32 device; + u32 pstate; + u64 util_host_ns; + u64 pstate_host_ns; + u64 util_samples; + u64 pstate_timestamp; +}; + +typedef void (*asahi_stats_read_fn)(const void *snapshot, struct asahi_qualified_stats *out); +typedef void (*asahi_stats_suspend_fn)(const void *snapshot, bool suspended); + int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, const void *memory, void **handle); + int export_enabled, const void *memory, void **handle, + asahi_stats_read_fn read, asahi_stats_suspend_fn suspend); int asahi_sysfs_register_idle(struct device *dev, u32 *word); void asahi_sysfs_unregister(struct device *dev, void *handle); diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index e7f842d8ac6fd..7c34f0627b020 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -3,16 +3,21 @@ //! Per-device sysfs registration. The caller retains the snapshot Arc until //! unregister has drained active readers and freed the C attribute wrapper. -use crate::{memacct::Ledger, stats::StatsSnapshot}; +use crate::{g17::telemetry::Readout, memacct::Ledger, stats::StatsSnapshot}; use core::ffi::{c_int, c_void}; +type ReadStats = unsafe extern "C" fn(*const c_void, *mut Readout); +type SuspendStats = unsafe extern "C" fn(*const c_void, bool); + extern "C" { fn asahi_sysfs_register( dev: *mut c_void, - snapshot: *const StatsSnapshot, + snapshot: *const c_void, export_enabled: c_int, memory: *const Ledger, handle: *mut *mut c_void, + read: Option, + suspend: Option, ) -> c_int; fn asahi_sysfs_unregister(dev: *mut c_void, handle: *mut c_void); } @@ -43,6 +48,66 @@ pub(crate) fn register( dev: *mut kernel::bindings::device, snapshot: *const StatsSnapshot, export_enabled: bool, +) -> kernel::error::Result<*mut c_void> { + register_with_callbacks(dev, snapshot, export_enabled, None, None) +} + +/// The caller retains the G17 snapshot Arc until unregister joins both sysfs +/// readers and the PM notifier. The callbacks never power the GPU up. +pub(crate) fn register_g17( + dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, + export_enabled: bool, +) -> kernel::error::Result<*mut c_void> { + register_with_callbacks( + dev, + snapshot, + export_enabled, + Some(read_g17), + Some(suspend_g17), + ) +} + +/// Copy the locked qualified snapshot to the C formatter's local output. +/// +/// # Safety +/// +/// `snapshot` must name a live StatsSnapshot retained by the registration. +/// `out` must be aligned and writable for one Readout for this call. +unsafe extern "C" fn read_g17(snapshot: *const c_void, out: *mut Readout) { + // SAFETY: C calls only while registration retains the snapshot Arc, and + // supplies a writable, aligned output for this call. + let snapshot = unsafe { &*snapshot.cast::() }; + let view = snapshot + .g17_reader + .as_ref() + .map_or_else(Readout::default, |reader| reader.read()); + // SAFETY: As above; the C output is initialized in one locked read. + unsafe { out.write(view) }; +} + +/// End the telemetry epoch at either side of system sleep. +/// +/// # Safety +/// +/// `snapshot` must name a live StatsSnapshot. The caller must join this +/// callback before releasing the registration's owning Arc. +unsafe extern "C" fn suspend_g17(snapshot: *const c_void, suspended: bool) { + // SAFETY: The notifier is unregistered and joined before the Arc is dropped. + if let Some(reader) = unsafe { &*snapshot.cast::() } + .g17_reader + .as_ref() + { + reader.suspend(suspended); + } +} + +fn register_with_callbacks( + dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, + export_enabled: bool, + read: Option, + suspend: Option, ) -> kernel::error::Result<*mut c_void> { let mut handle = core::ptr::null_mut(); // SAFETY: The bound driver keeps the device and optional snapshot alive until @@ -50,10 +115,12 @@ pub(crate) fn register( let ret = unsafe { asahi_sysfs_register( dev.cast(), - snapshot, + snapshot.cast(), c_int::from(export_enabled), &crate::memacct::LEDGER, &mut handle, + read, + suspend, ) }; if ret < 0 { diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs index b51b027eb573c..6e5540caeac3f 100644 --- a/drivers/gpu/drm/asahi/util.rs +++ b/drivers/gpu/drm/asahi/util.rs @@ -96,6 +96,13 @@ pub(crate) fn gcd(in_n: u64, in_m: u64) -> u64 { m } +/// A value that can be populated from an untrusted byte record. +/// +/// # Safety +/// +/// Every bit pattern must be valid for Self. Its byte representation must +/// contain no uninitialized bytes (including padding) when produced by Default, +/// since Reader temporarily views that storage as a byte slice. pub(crate) unsafe trait AnyBitPattern: Default + Sized + Copy + 'static {} pub(crate) struct Reader<'a> { @@ -117,7 +124,9 @@ impl<'a> Reader<'a> { // SAFETY: The output pointer is valid, and the size does not exceed // the type size, and all bit patterns are valid. - let dst = unsafe { core::slice::from_raw_parts_mut(&mut obj as *mut _ as *mut u8, size) }; + let dst = unsafe { + core::slice::from_raw_parts_mut(core::ptr::from_mut(&mut obj).cast::(), size) + }; dst.copy_from_slice(&src[..size]); self.offset = end; diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index d6695a1b3ee96..85281346296d5 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -803,7 +803,7 @@ impl WorkQueue::ver { )?; let inner = WorkQueueInner::ver { - dev: dev.into(), + dev, event_manager, info: Some(KBox::new(info, GFP_KERNEL)?), gpu_context, diff --git a/tools/asahi/binding-lifetime-tests.py b/tools/asahi/binding-lifetime-tests.py new file mode 100644 index 0000000000000..dedcbcd4326e5 --- /dev/null +++ b/tools/asahi/binding-lifetime-tests.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the production bound-device gate with host locking/refcount adapters.""" +import argparse +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +HEADER = r''' +#![allow(dead_code)] +extern crate self as kernel; +mod prelude { + pub type Result = std::result::Result; + pub const ENODEV:i32=-19; + pub const EOVERFLOW:i32=-75; + pub const GFP_KERNEL:()=(); + pub struct KVec(Vec); + impl KVec { pub const fn new()->Self {Self(Vec::new())} pub fn push(&mut self,v:T,_:())->Result {self.0.push(v);Ok(())} } + impl std::ops::Deref for KVec{type Target=Vec;fn deref(&self)->&Vec{&self.0}} + impl std::ops::DerefMut for KVec{fn deref_mut(&mut self)->&mut Vec{&mut self.0}} +} +#[macro_export] +macro_rules! new_mutex {($v:expr $(,$name:expr)?)=>{$crate::sync::Mutex::new($v)};} +#[macro_export] +macro_rules! global_lock { (unsafe(uninit) static $name:ident: Mutex<$ty:ty> = $value:expr;) => { static $name:$crate::sync::Mutex<$ty>=$crate::sync::Mutex::new($value); }; } +#[macro_export] +macro_rules! new_condvar {($name:expr)=>{$crate::sync::CondVar(std::sync::Condvar::new())};} +macro_rules! pin_init {($($t:tt)*)=>{$($t)*};} +mod sync { + pub use crate::global_lock; + pub struct Arc(std::sync::Arc); + impl Clone for Arc{fn clone(&self)->Self{Self(self.0.clone())}} + impl std::ops::Deref for Arc{type Target=T;fn deref(&self)->&T{&self.0}} + impl Arc{pub fn pin_init(v:T,_:())->crate::prelude::Result{Ok(Self(std::sync::Arc::new(v)))} pub fn ptr_eq(a:&Self,b:&Self)->bool{std::sync::Arc::ptr_eq(&a.0,&b.0)}} + pub struct Mutex(std::sync::Mutex); + impl Mutex{pub const fn new(v:T)->Self{Self(std::sync::Mutex::new(v))} pub unsafe fn init(&self){} pub fn lock(&self)->Guard<'_,T>{Guard(Some(self.0.lock().unwrap()))}} + pub struct Guard<'a,T>(Option>); + impl std::ops::Deref for Guard<'_,T>{type Target=T;fn deref(&self)->&T{self.0.as_ref().unwrap()}} + impl std::ops::DerefMut for Guard<'_,T>{fn deref_mut(&mut self)->&mut T{self.0.as_mut().unwrap()}} + pub struct CondVar(pub std::sync::Condvar); + impl CondVar {pub fn wait(&self,g:&mut Guard<'_,T>){let old=g.0.take().unwrap();g.0=Some(self.0.wait(old).unwrap());} pub fn notify_all(&self){self.0.notify_all();}} + pub mod aref { + pub struct ARef(&'static T); + impl std::ops::Deref for ARef{type Target=T;fn deref(&self)->&T{self.0}} + impl From<&crate::device::Device> for ARef{fn from(d:&crate::device::Device)->Self { + // The host fixtures leak their devices for the test process. + Self(unsafe{&*(d as *const _ as *const crate::device::Device)}) + }} + } +} +mod device { + pub struct Normal; + pub struct Bound; + #[repr(C)] + pub struct Device{pub bound:std::sync::atomic::AtomicBool,pub marker:std::marker::PhantomData} + impl Device{pub fn new()->Self{Self{bound:std::sync::atomic::AtomicBool::new(true),marker:std::marker::PhantomData}}} + impl Device {pub unsafe fn as_bound(&self)->&Device{assert!(self.bound.load(std::sync::atomic::Ordering::SeqCst));unsafe{&*(self as *const _ as *const Device)}}} +} +mod driver {pub struct AsahiDrmDevice(pub u64);} +mod binding; +''' +TESTS = r''' +use {binding::*,device::*,driver::AsahiDrmDevice}; +use std::{sync::{mpsc,atomic::Ordering},time::Duration}; +fn device()->&'static Device{Box::leak(Box::new(Device::new()))} +#[test] +fn close_joins_existing_access_and_rejects_future_access() { + let dev=device();let drm=Box::new(AsahiDrmDevice(1)); + let owner=unsafe{Owner::new(dev,&drm)}.unwrap(); + let gate=lookup(&drm).unwrap();let access=gate.enter().unwrap();let _=access.device(); + let (started_rx,started)= {let (a,b)=mpsc::channel();(b,a)}; + let (done,finished)=mpsc::channel(); + let thread=std::thread::spawn(move||{started.send(()).unwrap();owner.close();done.send(()).unwrap();}); + started_rx.recv().unwrap(); + assert!(finished.recv_timeout(Duration::from_millis(50)).is_err(),"unbind returned with a live device borrow"); + drop(access);finished.recv_timeout(Duration::from_secs(2)).unwrap();thread.join().unwrap(); + dev.bound.store(false,Ordering::SeqCst); + assert!(gate.enter().is_err());assert!(lookup(&drm).is_err()); +} +#[test] +fn rebind_cannot_reopen_an_old_vm_gate() { + let dev=device();let a=Box::new(AsahiDrmDevice(2));let b=Box::new(AsahiDrmDevice(3)); + let first=unsafe{Owner::new(dev,&a)}.unwrap();let old=lookup(&a).unwrap(); + first.close();let second=unsafe{Owner::new(dev,&b)}.unwrap(); + assert!(old.enter().is_err());assert!(lookup(&a).is_err()); + let new=lookup(&b).unwrap();let access=new.enter().unwrap();let _=access.device();drop(access);second.close(); +} +#[test] +fn failed_probe_owner_drop_closes_retained_vm_handles() { + let dev=device();let drm=Box::new(AsahiDrmDevice(4)); + let owner=unsafe{Owner::new(dev,&drm)}.unwrap();let gate=lookup(&drm).unwrap();drop(owner); + dev.bound.store(false,Ordering::SeqCst); + assert!(gate.enter().is_err());assert!(lookup(&drm).is_err()); +} +''' + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('--out', type=Path) + ap.add_argument('--mutation', choices=['no-close', 'no-join']) + args = ap.parse_args() + work = (args.out or Path(tempfile.mkdtemp(prefix='asahi-binding-'))).resolve() + work.mkdir(parents=True, exist_ok=True) + source = (ROOT / 'drivers/gpu/drm/asahi/binding.rs').read_text() + source = '\n'.join(line for line in source.splitlines() + if line.strip() not in ('#[pin_data]', '#[pin]')) + source = source.replace(' <- ', ': ') + source = source.replace('// SAFETY: Module initialization initializes this lock before registering adapters.', '') + if args.mutation == 'no-close': + source = source.replace('state.live = false;', 'state.live = true;') + if args.mutation == 'no-join': + source = source.replace('while state.users != 0 {', 'while false {') + (work / 'binding.rs').write_text(source) + (work / 'test.rs').write_text(HEADER + TESTS) + subprocess.run(['rustc', '--edition=2021', '--test', str(work / 'test.rs'), '-o', str(work / 'test')], check=True) + subprocess.run([str(work / 'test')], check=True) + + +if __name__ == '__main__': + try: + main() + except subprocess.CalledProcessError as error: + raise SystemExit(error.returncode if error.returncode > 0 else 128 - error.returncode) diff --git a/tools/asahi/g15-probe-init-tests.py b/tools/asahi/g15-probe-init-tests.py index 8ab22935d9c53..9d6ec7c0096eb 100644 --- a/tools/asahi/g15-probe-init-tests.py +++ b/tools/asahi/g15-probe-init-tests.py @@ -77,6 +77,7 @@ def item(text, marker): static FAIL_REGISTRATION: Cell = const { Cell::new(0) }; static COMPLETED_REGISTRATION: Cell = const { Cell::new(0) }; static ACTIVATED: Cell = const { Cell::new(false) }; + static BINDING_READY: Cell = const { Cell::new(false) }; static ATTACH_ERROR: Cell = const { Cell::new(false) }; static PROBE_ERROR: Cell = const { Cell::new(0) }; } @@ -140,6 +141,11 @@ def item(text, marker): pub trait InPlaceModule: Sized { fn init(module: &'static ThisModule) -> impl PinInit; } +// Registry synchronization is covered by binding-lifetime-tests; this adapter +// verifies that the actual module initializer orders it before registration. +mod binding { + pub unsafe fn init() { crate::BINDING_READY.set(true); } +} mod cleanup { pub struct QueueOwner; impl QueueOwner { pub fn new() -> crate::Result { Ok(Self) } } @@ -150,6 +156,7 @@ def item(text, marker): pub struct Registration { stage: u8, marker: PhantomData } impl Registration { pub fn new(name: &'static std::ffi::CStr, _module: &'static ThisModule) -> impl PinInit { + assert!(BINDING_READY.get(), "adapter registered before binding registry init"); try_pin_init!(Self { stage: { let stage = match name.to_bytes() { @@ -198,6 +205,7 @@ def item(text, marker): } fn reset(fail_alloc: bool, touch: bool, error: i32) { MODULE_READY.store(true, Ordering::Release); + BINDING_READY.set(false); FAIL_ALLOC.set(fail_alloc); TOUCH.set(touch); PROBE_ERROR.set(error); RETAINED.set(false); ALLOCS.set(0); PROBES.set(0); } diff --git a/tools/asahi/g17-capacity-fault-tests.py b/tools/asahi/g17-capacity-fault-tests.py index 7c33e92a74812..c03764ca28d60 100644 --- a/tools/asahi/g17-capacity-fault-tests.py +++ b/tools/asahi/g17-capacity-fault-tests.py @@ -7,6 +7,7 @@ """ from pathlib import Path import os +import re import subprocess import tempfile @@ -278,13 +279,18 @@ def main(): fault_fix = 'Previous::retire_witnessed(&mut self.previous);' assert source.count(idle_fix) == 1 assert source.count(fault_fix) == 2 + def mutate(pattern, replacement): + changed, count = re.subn(pattern, replacement, source) + assert count == 1, (pattern, count) + return changed + controls = [('fixed', source, True), ('missing-idle-close', source.replace(idle_fix, ''), False), ('missing-fault-release', source.replace(fault_fix, '', 1), False), ('missing-kill-release', source.replace('Previous::retire_witnessed(&mut self.previous);\n self.exit', 'self.exit'), False), ('missing-reclaim-wake', source.replace('self.queue_events();', ''), False), - ('share-before-retirement', source.replace('&& self.owner.is_some() && self.retirement_ready', '&& self.owner.is_some()'), False), - ('share-during-kill', source.replace('&& self.exit.is_none() && self.active.len', '&& self.active.len'), False)] + ('share-before-retirement', mutate(r'(&&\s*self\.owner\.is_some\(\))\s*&&\s*self\.retirement_ready', r'\1'), False), + ('share-during-kill', mutate(r'&&\s*self\.exit\.is_none\(\)\s*(?=&&\s*self\.active\.len)', ''), False)] with tempfile.TemporaryDirectory(prefix='g17-capacity-') as tmp: for name, rust, passes in controls: path = Path(tmp) / (name + '.rs') diff --git a/tools/asahi/g17-telemetry-integration-tests.py b/tools/asahi/g17-telemetry-integration-tests.py new file mode 100644 index 0000000000000..5c36b6beed579 --- /dev/null +++ b/tools/asahi/g17-telemetry-integration-tests.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production ring transport and C formatter without hardware.""" +import argparse +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +DRIVER = ROOT / 'drivers/gpu/drm/asahi' + + +def item(source, signature): + start = source.index(signature) + end = source.index('{', start) + 1 + depth = 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +RUST_HEADER = r''' +#![allow(dead_code)] +extern crate self as kernel; +mod prelude { + pub type Result = std::result::Result; + pub const EIO: i32 = -5; + pub const EAGAIN: i32 = -11; +} +mod device { + pub struct Device; + impl AsRef for Device { fn as_ref(&self) -> &Device { self } } +} +mod fmt { pub use std::fmt::*; } +macro_rules! dev_err { ($dev:expr, $($t:tt)*) => { let _ = ($dev, format_args!($($t)*)); }; } +mod g17 { + use std::sync::atomic::{AtomicU64, Ordering}; + static NOW: AtomicU64 = AtomicU64::new(1); + fn now_ns() -> u64 { NOW.load(Ordering::Relaxed) } + #[derive(Clone, Copy, Debug)] + enum Role { Primary = 0, Secondary = 1 } + mod trace { pub fn report(_:u8, _:u8, _:&[u32;18]) {} pub fn message(_:u32,_:u64) {} pub const CONTROL:u32=0; } + mod fw { + pub mod variant { + #[derive(Clone, Copy)] + pub enum InitGraph { V25G83, G17P } + } + pub mod channels { + pub const EVENT_WORDS:usize=18; + pub const EVENT_RECORD_SIZE:usize=72; + pub const RX_PRODUCER_OFFSET:usize=0x20; + pub const TRACE_SLOTS:u32=4; + pub const TRACE_TYPES:u32=1; + pub const STATISTICS_SLOTS:u32=8; + pub const STATISTICS_TYPES:u32=0x5fff0000; + pub const DEVICE_CONTROL_SLOTS:u32=8; + pub struct ControlRecord; + #[derive(Clone, Copy)] + pub enum Opcode { Dummy } + impl Opcode { pub fn asserts_gpu_power(self)->bool { false } } + impl ControlRecord { + pub fn bytes(&self)->&[u8] { &[] } + pub fn opcode(&self)->Opcode { Opcode::Dummy } + pub fn asserts_gpu_power(&self)->bool { false } + } + } + pub mod initdata { pub mod status { + pub const TRACE_STATE:u32=0; + pub const TRACE_RING:u32=0x100; + pub const STATS_STATE:u32=0x1000; + pub const STATS_RING:u32=0x2000; + } } + } + mod initdata { + use super::{Role,fw::{channels::*,initdata::status::*}}; + use crate::prelude::*; + use std::{cell::RefCell,sync::atomic::AtomicU32}; + pub struct InitData { + pub cursors:[[AtomicU32;2];2], + pub slots:RefCell<[[[u32;18];8];2]>, + } + impl InitData { + pub fn new()->Self { Self { cursors:std::array::from_fn(|_|std::array::from_fn(|_|AtomicU32::new(0))), slots:RefCell::new([[[0;18];8];2]) } } + pub fn push(&self,role:Role,words:[u32;18]) { + use std::sync::atomic::Ordering::Relaxed; + let producer=self.cursors[role as usize][1].load(Relaxed); + self.slots.borrow_mut()[role as usize][producer as usize]=words; + self.cursors[role as usize][1].store((producer+1)%STATISTICS_SLOTS,Relaxed); + } + pub fn status_word(&self,role:Role,offset:usize)->Result<&AtomicU32> { + Ok(&self.cursors[role as usize][usize::from(offset==STATS_STATE as usize+RX_PRODUCER_OFFSET)]) + } + pub fn status_read_words(&self,role:Role,offset:usize)->Result<[u32;N]> { + let mut words=[0;N]; + words.copy_from_slice(&self.slots.borrow()[role as usize][(offset-STATS_RING as usize)/EVENT_RECORD_SIZE][..N]); + Ok(words) + } + pub fn control_consumer(&self)->Result<&AtomicU32> { Ok(&self.cursors[0][0]) } + pub fn control_producer(&self)->Result<&AtomicU32> { Ok(&self.cursors[0][1]) } + pub fn publish_control(&self,_:u32,_:&[u8],_:bool,_:u32)->Result { Ok(()) } + } + } + mod reports { + use super::Role; + pub struct Reports; + impl Reports { + pub fn new()->crate::prelude::Result { Ok(Self) } + pub fn record(&mut self,_:Role,_:[u32;18],_:u64) {} + pub fn write(&self,_:&mut crate::fmt::Formatter<'_>)->crate::fmt::Result { Ok(()) } + } + } +''' +RUST_TESTS = r''' + #[test] + fn roles_merge_and_any_statistics_quarantine_disables_both() { + use {channel::Rings,fw::variant::InitGraph,Role::*}; + let dev=crate::device::Device; + let init=initdata::InitData::new(); + let mut rings=Rings::new(InitGraph::G17P).unwrap(); + let sample=|ta,render,device|[0x13,0,0,ta,render,0xdeadbeef,device,0,0,0,0,0,0,0,0,0,0,0]; + init.push(Primary,sample(25,75,50)); + init.push(Secondary,sample(35,85,65)); + rings.drain_statistics(&dev,&init,Primary); + rings.drain_statistics(&dev,&init,Secondary); + let view=rings.read_statistics(1); + assert_eq!((view.tiler,view.renderer,view.device,view.util_samples),(30,80,57,2)); + assert_eq!(view.util_status,telemetry::VALID); + init.cursors[1][1].store(fw::channels::STATISTICS_SLOTS,Ordering::Relaxed); + rings.drain_statistics(&dev,&init,Secondary); + assert_eq!(rings.read_statistics(1).util_status,telemetry::STALE); + init.cursors[1][1].store(1,Ordering::Relaxed); + for _ in 0..3 { + init.push(Primary,sample(100,100,100)); + rings.drain_statistics(&dev,&init,Primary); + } + assert_eq!(rings.read_statistics(1).util_status,telemetry::STALE); + } + #[test] + fn suspend_gating_and_restart_discard_queued_epoch_records() { + use {channel::Rings,fw::variant::InitGraph,Role::*}; + let dev=crate::device::Device; + let init=initdata::InitData::new(); + let mut rings=Rings::new(InitGraph::V25G83).unwrap(); + let words=[0x13,0,0,25,75,0,50,0,0,0,0,0,0,0,0,0,0,0]; + init.push(Primary,words); + rings.drain_statistics(&dev,&init,Primary); + assert_eq!(rings.read_statistics(1).util_status,telemetry::VALID); + for boundary in 0..4 { + init.push(Primary,words); // Unread at the epoch boundary. + match boundary { + 0=> { rings.suspend_statistics(true); rings.drain_statistics(&dev,&init,Primary); rings.suspend_statistics(false); init.push(Primary,words); } + 1=>rings.observe_statistics_power(false,0), + 2=>rings.observe_statistics_power(true,1), // Wake observed by a fast publisher. + _=>rings.invalidate_statistics(), // Restart handshake. + } + assert_eq!(rings.read_statistics(1).util_status,telemetry::STALE); + rings.drain_statistics(&dev,&init,Primary); + assert_eq!(rings.read_statistics(1).util_status,telemetry::STALE); + init.push(Primary,words); + rings.drain_statistics(&dev,&init,Primary); + assert_eq!(rings.read_statistics(1).util_status,telemetry::VALID); + } + } + #[test] + fn pstate_uses_drain_order_without_role_deduplication() { + use {channel::Rings,fw::variant::InitGraph,Role::*}; + let dev=crate::device::Device; + let init=initdata::InitData::new(); + let mut rings=Rings::new(InitGraph::G17P).unwrap(); + init.push(Primary,[0x10,0x1000,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0]); + init.push(Secondary,[0x10,0x2500,0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0]); + rings.drain_statistics(&dev,&init,Primary); + rings.drain_statistics(&dev,&init,Secondary); + assert_eq!(rings.read_statistics(1).pstate,6); + init.push(Primary,[0x10,0x1000,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0]); + rings.drain_statistics(&dev,&init,Primary); + assert_eq!(rings.read_statistics(1).pstate_status,telemetry::UNAVAILABLE); + } +} +''' + +C_HEADER = r''' +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint64_t u64; +#define PAGE_SIZE 4096 +#define READ_ONCE(x) (x) +#define container_of(ptr,type,member) ((type*)((char*)(ptr)-offsetof(type,member))) +struct device {}; +struct device_attribute { + struct { const char *name; unsigned mode; } attr; + ssize_t (*show)(struct device*,struct device_attribute*,char*); +}; +struct notifier_block { int (*notifier_call)(struct notifier_block*,unsigned long,void*); }; +#define PM_SUSPEND_PREPARE 1 +#define PM_HIBERNATION_PREPARE 2 +#define PM_RESTORE_PREPARE 3 +#define PM_POST_SUSPEND 4 +#define PM_POST_HIBERNATION 5 +#define PM_POST_RESTORE 6 +#define NOTIFY_OK 1 +#define ENODEV 19 +#define ENOMEM 12 +#define BUILD_BUG_ON(x) _Static_assert(!(x),#x) +#define kzalloc_obj(x) calloc(1,sizeof(x)) +#define sysfs_attr_init(x) ((void)0) +#define kfree(x) free(x) +static int registered, removed, created, fail_file, fail_notifier, suspend_calls; +static int register_pm_notifier(struct notifier_block *nb) { (void)nb; if(fail_notifier)return -12; registered++; return 0; } +static int unregister_pm_notifier(struct notifier_block *nb) { (void)nb; removed++; return 0; } +static int device_create_file(struct device *dev,struct device_attribute *attr) { (void)dev;(void)attr; return ++created==fail_file?-12:0; } +static void device_remove_file(struct device *dev,struct device_attribute *attr) { (void)dev;(void)attr; } +static int emit(char *buf,size_t size,const char *fmt,...) { + va_list ap;va_start(ap,fmt);int n=vsnprintf(buf,size,fmt,ap);va_end(ap);return n<(int)size?n:(int)size-1; +} +#define scnprintf emit +#define sysfs_emit(buf,...) emit(buf,PAGE_SIZE,__VA_ARGS__) +#define sysfs_emit_at(buf,at,...) emit(buf+at,PAGE_SIZE-(at),__VA_ARGS__) +static ssize_t gpu_memory_show(struct device *dev,struct device_attribute *attr,char *buf) { (void)dev;(void)attr;(void)buf;return 0; } +static u32 *asahi_idle_word; +static struct device_attribute dev_attr_idle_off_delay_ms; +''' +C_TESTS = r''' +static struct asahi_qualified_stats current; +static void read_stats(const void *ptr,struct asahi_qualified_stats *out) { (void)ptr;*out=current; } +static void suspend_stats(const void *ptr,bool suspended) { (void)ptr;suspend_calls+=suspended?1:10; } +int main(void) { + struct asahi_stats_snapshot snap; + struct asahi_stats_attribute stats={.snapshot=&snap,.export_enabled=true}; + char old[PAGE_SIZE],now[PAGE_SIZE]; + for(unsigned seed=0;seed<512;seed++) { + snap=(struct asahi_stats_snapshot){ + .util1=seed, .util2=seed+11, .util3=~seed, .util4=seed*0x01010101u, + .pstate=seed%16, .avg_power_mw=seed*1234567u, + .temperature_raw=seed^0xff00aa11u, .temperature_scale=seed+37, + .temperature_tmin=seed+42, .temperature_tmax=seed+73, + .busy_ns=((u64)seed<<32)|(UINT32_MAX-seed), + .jobs=UINT64_MAX-(u64)seed*101, + }; + ssize_t before=old_agx_stats_show(NULL,&stats.attr,old); + ssize_t after=agx_stats_show(NULL,&stats.attr,now); + assert(before==after && !memcmp(old,now,before+1)); + } + stats.export_enabled=false; + assert(agx_stats_show(NULL,&stats.attr,now)==12); + assert(!strcmp(now,"unsupported\n")); + stats.export_enabled=true;stats.read=read_stats; + for(unsigned status=0;status<3;status++) { + current=(struct asahi_qualified_stats){.util_status=status,.pstate_status=status,.tiler=30,.renderer=80,.device=57,.pstate=6}; + agx_stats_show(NULL,&stats.attr,now); + assert(strstr(now,"busy_ns unsupported\n")); + assert(strstr(now,"power_mw unsupported\n")); + assert(strstr(now,"temperature_raw unsupported\n")); + assert(strstr(now,"requested_frequency_hz unsupported\n")); + assert(strstr(now,"achieved_frequency_hz unsupported\n")); + if(status==1) { + assert(strstr(now,"util_tiler_percent 30\nutil_renderer_percent 80\nutil_device_percent 57\n")); + assert(strstr(now,"pstate 6\n")); + } else { + assert(strstr(now,"util_device_percent unsupported\n")); + assert(strstr(now,"pstate unsupported\n")); + assert(strstr(now,status==2?"pstate_status stale\n":"pstate_status unavailable\n")); + } + } + struct device dev; + struct asahi_gpu_memory memory={0}; + void *handle=NULL; + assert(!asahi_sysfs_register(&dev,&snap,1,&memory,&handle,read_stats,suspend_stats)); + struct asahi_stats_attribute *registered_stats=handle; + assert(registered==1 && removed==0); + asahi_stats_pm_notify(®istered_stats->pm_notifier,PM_SUSPEND_PREPARE,NULL); + asahi_stats_pm_notify(®istered_stats->pm_notifier,PM_POST_SUSPEND,NULL); + assert(suspend_calls==11); + asahi_sysfs_unregister(&dev,handle); + assert(removed==1); + for(int failure=1;failure<=2;failure++) { + created=0;fail_file=failure;handle=NULL; + assert(asahi_sysfs_register(&dev,&snap,1,&memory,&handle,read_stats,suspend_stats)==-12); + assert(registered==removed); + assert(handle==NULL); + } + fail_file=0;fail_notifier=1;created=0; + assert(asahi_sysfs_register(&dev,&snap,1,&memory,&handle,read_stats,suspend_stats)==-12); + assert(created==0 && registered==removed); + puts("PASS: 512 byte-identical legacy outputs, G17 capabilities, PM lifecycle and registration rollback"); +} +''' + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('--out', type=Path) + ap.add_argument('--baseline', default='ebcf9cb777a434867e9d880892ee2b46887728bb', + help='revision containing the legacy formatter') + ap.add_argument('--mutation', choices=['fields', 'range', 'timestamp', 'age', + 'quarantine', 'epoch', 'legacy-format']) + args = ap.parse_args() + work = args.out or Path(tempfile.mkdtemp(prefix='asahi-g17-telemetry-')) + work.mkdir(parents=True, exist_ok=True) + work = work.resolve() + channel = (DRIVER / 'g17/channel.rs').read_text().split('/// One bounded event-ring pass.')[0] + # Compile the production transport and decoder; only MMIO/shared memory, + # kernel logging and allocation adapters are replaced by host fixtures. + channel = '\n'.join(line for line in channel.splitlines() if not line.startswith('//!')) + (work / 'channel.rs').write_text(channel) + telemetry = (DRIVER / 'g17/telemetry.rs').read_text() + changes = { + 'fields': ('[words[3], words[4], words[6]]', '[words[3], words[4], words[5]]'), + 'range': ('value > 100', 'value > 101'), + 'timestamp': ('u64::from(words[1]) | (u64::from(words[2]) << 32)', + 'u64::from(words[2]) | (u64::from(words[1]) << 32)'), + 'age': ('now_ns - received >= max_age_ns', 'now_ns - received > max_age_ns'), + 'quarantine': ('.any(|rings| rings[ReportRing::Statistics as usize])', '.any(|_| false)'), + 'epoch': ('self.discard_statistics = [true; 2]', 'self.discard_statistics = [false; 2]'), + } + if args.mutation in changes: + before, after = changes[args.mutation] + if args.mutation in ('quarantine', 'epoch'): + assert before in channel + channel = channel.replace(before, after, 1) + (work / 'channel.rs').write_text(channel) + else: + assert before in telemetry + telemetry = telemetry.replace(before, after, 1) + (work / 'telemetry.rs').write_text(telemetry) + policy = item((DRIVER / 'g17/stats.rs').read_text(), 'pub(super) const fn max_age_ns') + (work / 'rings.rs').write_text(RUST_HEADER + 'mod stats { use super::fw::variant::InitGraph;\n' + + policy + '\n}\nmod telemetry;\nmod channel;\n' + RUST_TESTS) + # Rust resolves child modules relative to the inline parent's directory. + (work / 'g17').mkdir(exist_ok=True) + for name in ['channel.rs', 'telemetry.rs']: + (work / 'g17' / name).write_text((work / name).read_text()) + subprocess.run(['rustc', '--edition=2021', '--test', str(work / 'rings.rs'), '-o', str(work / 'rings')], check=True) + subprocess.run([str(work / 'rings')], check=True) + source = (DRIVER / 'sysfs.c').read_text() + if args.mutation == 'legacy-format': + source = source.replace('READ_ONCE(snap->util1)', 'READ_ONCE(snap->util2)', 1) + header = (DRIVER / 'sysfs.h').read_text() + baseline = subprocess.check_output(['git', '-C', str(ROOT), 'show', args.baseline + ':drivers/gpu/drm/asahi/sysfs.c'], text=True) + pieces = [C_HEADER] + for signature in ['struct asahi_qualified_stats']: + pieces.append(item(header, signature) + ';') + pieces += [line for line in header.splitlines() if line.startswith('typedef')] + for signature in ['struct asahi_stats_snapshot', 'struct asahi_gpu_memory', 'struct asahi_stats_attribute']: + pieces.append(item(source, signature) + ';') + pieces.append(item(baseline, 'static ssize_t agx_stats_show').replace('agx_stats_show(', 'old_agx_stats_show(', 1)) + for signature in ['static const char *asahi_stats_quality', 'static ssize_t agx_qualified_stats_show', + 'static int asahi_stats_pm_notify', 'static ssize_t agx_stats_show', + 'int asahi_sysfs_register(', 'void asahi_sysfs_unregister(']: + pieces.append(item(source, signature)) + pieces.append(C_TESTS) + (work / 'formatter.c').write_text('\n'.join(pieces)) + subprocess.run(['cc', '-std=gnu11', '-Wall', '-Wextra', '-Werror', '-Wno-unused-parameter', + str(work / 'formatter.c'), '-o', str(work / 'formatter')], check=True) + subprocess.run([str(work / 'formatter')], check=True) + + +if __name__ == '__main__': + try: + main() + except subprocess.CalledProcessError as error: + print(f'child exit: {error.returncode}') + raise SystemExit(error.returncode if error.returncode > 0 else 128 - error.returncode) diff --git a/tools/asahi/g17-telemetry-tests.rs b/tools/asahi/g17-telemetry-tests.rs new file mode 100644 index 0000000000000..bfedf6ea03155 --- /dev/null +++ b/tools/asahi/g17-telemetry-tests.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Run the production decoder's synthetic vectors and epoch tests on the host. +#![allow(dead_code)] +#[path = "../../drivers/gpu/drm/asahi/g17/telemetry.rs"] +mod telemetry; diff --git a/tools/asahi/g17-unpublished-tests.py b/tools/asahi/g17-unpublished-tests.py index a162b2651a037..f06eedf7f64f8 100644 --- a/tools/asahi/g17-unpublished-tests.py +++ b/tools/asahi/g17-unpublished-tests.py @@ -4,6 +4,7 @@ from pathlib import Path import importlib.util import os +import re import subprocess import tempfile ROOT=Path(__file__).resolve().parents[2]/'drivers/gpu/drm/asahi/g17' @@ -93,9 +94,11 @@ def main(): assert compute.count('visible = true;\n self.unpublished_binding = false;')==2 backend=(ROOT/'runtime/backend.rs').read_text() assert 'if !lease.is_current() { drop(state); drop(lease); continue; }' in backend + no_kill, count = re.subn(r'\|\|\s*self\.exit\.is_some\(\)\s*(?=\|\|\s*self\.active\.len\s*!=\s*0)', '', source) + assert count == 1, count cases=[('fixed',source),('omit-first-failure-rollback',source.replace('self.rollback_unpublished_compute().is_err()','false')), ('rollback-published-binding',source.replace('|| !self.unpublished_binding','')), - ('rollback-issued-kill',source.replace('|| self.exit.is_some() || self.active.len != 0','|| self.active.len != 0'))] + ('rollback-issued-kill',no_kill)] with tempfile.TemporaryDirectory(prefix='g17-unpublished-') as tmp: for name,code in cases: path=Path(tmp)/(name+'.rs');path.write_text(code);binary=path.with_suffix('') diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index cacece95bc905..100da4247b510 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -54,6 +54,8 @@ def mutations(flag, names, base, expect=KILL): (out() + ['--old-transport', '4db3a5a22c43'], PASS)], 'ave-pm-tests.py': [(out(), PASS), (out() + ['--old-ref', '4db3a5a22c43'], PASS)], + 'binding-lifetime-tests.py': [(out(), PASS)] + + mutations('--mutation', ('no-close', 'no-join'), out()), 'apple-dart-fw-tests.py': [(out(), PASS)], 'apple-dart-range-tests.py': [([], PASS)], 'atc-usb3-tests.py': [(out(), PASS), (out() + ['--mutations'], PASS)], @@ -93,6 +95,9 @@ def mutations(flag, names, base, expect=KILL): 'g15-retention-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 'g15-stats-drain-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 'g15-zero-words-tests.py': [([], PASS), (['--mutations'], PASS)], + 'g17-telemetry-integration-tests.py': [(out(), PASS)] + + mutations('--mutation', ('fields', 'range', 'timestamp', 'age', + 'quarantine', 'epoch', 'legacy-format'), out()), 'g17-admission-deadline-tests.py': [([], PASS)], 'g17-admission-order-tests.py': [([], PASS)], 'g17-capacity-fault-tests.py': [([], PASS)], @@ -143,6 +148,9 @@ def mutations(flag, names, base, expect=KILL): 'usage-tests.rs'] ASAHI_SHELL = ['g17-ray-query-timeout-tests.sh'] +# Production G17 decoder vectors also build directly with rustc --test. +ASAHI_RUST.append('g17-telemetry-tests.rs') + # Python that runs on the host but imports its neighbours, so it keeps the # script directory on sys.path (-E -s instead of -I). SUITES = [ From 6156b0c17bc35fe4c26ffd42f59df8814e0c0c06 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 03:30:06 +1100 Subject: [PATCH 1443/1579] wifi: mt7932: add WPA2 PMF and J700 LPI/TPE policy Retain cfg80211's optional/required MFP policy, validate WPA2 RSN admission and encode selected PSK-SHA256 as EA/40 AKM 0x0400. Advertise BIP-CMAC-128 with firmware-owned handshake, IGTK installation and BIP verification. Bind t-2862 country class data to the clean t-3460 CID5d class/TPE ABI. Project finite AP limits with t-3573's immutable J700 gain matrix and exact primary/bandwidth map, preserving signed half-dB and sentinel semantics. Keep SP/AFC and unknown layouts unsupported. Preserve aurora-wip's v2 VLP admission and firmware command sequence, including finite/absent TPE and AP-wide 160 MHz announcements. Retain v2 pages when v4 is installed alongside it, using v4 as the LP/extra-channel sidecar. New LPI and v4-only VLP use class publication and projected TPE. Restore the idle country policy and unconstrained TPE after retiring a new class connection. Partial publication closes RF admission until reset. Cover MFP/AKM encoders, class selectors, package ownership, legacy VLP regression and the gain SPEC's boundary/projection vectors with KUnit. Firmware acceptance and RF compliance require owner qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../wifi/mt7932-j700-antenna.rst | 56 +++ .../device_drivers/wifi/mt7932-neo.rst | 275 +++++++++++- MAINTAINERS | 1 + drivers/net/wireless/mediatek/mt7932/Makefile | 2 + .../wireless/mediatek/mt7932/association.h | 117 ++++++ .../net/wireless/mediatek/mt7932/frontend.c | 45 +- drivers/net/wireless/mediatek/mt7932/link.c | 71 +++- drivers/net/wireless/mediatek/mt7932/mt7932.h | 6 +- .../wireless/mediatek/mt7932/pmf-lpi-test.c | 396 ++++++++++++++++++ .../mediatek/mt7932/regulatory-test.c | 246 ++++++++++- .../net/wireless/mediatek/mt7932/regulatory.h | 157 ++++++- drivers/net/wireless/mediatek/mt7932/six.h | 298 +++++++++++++ .../net/wireless/mediatek/mt7932/startup.c | 267 +++++++++++- .../asahi/mt7932-association-events-tests.py | 2 +- tools/asahi/mt7932-policy-classes-tests.py | 180 ++++++++ tools/asahi/mt7932-policy-classes.py | 234 +++++++++++ 16 files changed, 2272 insertions(+), 81 deletions(-) create mode 100644 Documentation/networking/device_drivers/wifi/mt7932-j700-antenna.rst create mode 100644 drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c create mode 100644 drivers/net/wireless/mediatek/mt7932/six.h create mode 100644 tools/asahi/mt7932-policy-classes-tests.py create mode 100644 tools/asahi/mt7932-policy-classes.py diff --git a/Documentation/networking/device_drivers/wifi/mt7932-j700-antenna.rst b/Documentation/networking/device_drivers/wifi/mt7932-j700-antenna.rst new file mode 100644 index 0000000000000..3884d3fc935e3 --- /dev/null +++ b/Documentation/networking/device_drivers/wifi/mt7932-j700-antenna.rst @@ -0,0 +1,56 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +J700 6 GHz antenna-gain projection +================================= + +The clean t-3573 antenna-gain SPEC (2026-10-31), sections 1, 3 and 6, +establishes these immutable board facts. They apply only to the J700/MT7932 +board admitted by the driver's existing machine, PCI function and supplier +checks. They are not generic PCI 14c3:7932 properties. No external resource, +undocumented device-tree property or module parameter is required. + +Values are signed half-dB gain offsets in SISO, CDD, MIMO order:: + + G1 16 16 10 + G2 14 14 8 + G3 15 16 10 + G4 15 15 9 + G5 16 16 10 + G6 15 16 10 + +The primary-channel/bandwidth lookup is:: + + primary 20 40 80 160 MHz + 2 1 - - - + 1..29 step 4 1 1 1 1 + 33..61 step 4 2 2 2 2 + 65..93 step 4 3 3 3 3 + 97..109 step 4 4 4 4 4 + 113 4 4 5 4 + 117 5 4 5 4 + 121..125 step 4 5 5 5 4 + 129..181 step 4 5 5 5 5 + 185 5 6 5 5 + 189 6 6 5 5 + 193..221 step 4 6 6 6 6 + 225..229 step 4 6 6 - - + 233 6 - - - + +A finite TPE cell subtracts the mapped gain independently for each mode. +The calculation uses a wider signed type and refuses unrepresentable +results. 0x7f is absence of a constraint and survives unchanged; no missing +gain is replaced by zero. Unknown primary/bandwidth mappings refuse finite +cells. The encoder covers four bandwidths; station operation remains at most +80 MHz. Country/class, SAR and hardware ceilings remain independent bounds. + +For example, channel 117 with EIRP limits (40,40,40,40) half-dBm produces +SISO and CDD (24,25,24,25), MIMO (30,31,30,31). The same vectors apply to +LPI and VLP. PSD -1 dBm/MHz converts to EIRP (24,30,36,42) half-dBm; with G1 +this produces SISO/CDD (8,14,20,26), MIMO (14,20,26,32). + +KUnit exercises these vectors, every boundary row, signed arithmetic, +sentinels, unsupported edge bandwidths, and legacy v2 VLP compatibility. +No 2.4/5 GHz mode matrix, OTA resource parser, calibration derivation or +firmware readback is established. Firmware acceptance and conducted/radiated +compliance need separate owner qualification. See :doc:`mt7932-neo` for +class selection, legacy compatibility, lifetime and installation. diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 27278ed431987..ceb85331324ad 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -75,12 +75,20 @@ Country packages Every regulatory change selects a country package again, without a reload. For country CC the driver uses the first of these that exists and is valid: -1. ``policy-v2/CC.bin``: J7RP version 2 (version 1 plus 6 GHz VLP pages); -2. ``policy/CC.bin``: J7RP version 1 (2.4 and 5 GHz, at most 16 KiB); -3. ``policy/world-XZ.bin``: the world package. +1. ``policy-v4/CC.bin``: J7RP version 4 (bound LP and optional VLP class data); +2. ``policy-v2/CC.bin``: J7RP version 2 (version 1 plus 6 GHz VLP pages); +3. ``policy/CC.bin``: J7RP version 1 (2.4 and 5 GHz, at most 16 KiB); +4. ``policy/world-XZ.bin``: the world package. + +When valid v4 and v2 packages coexist, the driver keeps v2's original base +and VLP pages and retains v4 as a class sidecar. LP and additional VLP +permissions extend the exposed channel set, but never change the legacy +VLP page bytes or its original numeric clamp. Both files remain held until +publication; the sidecar hash is exposed with the primary package hash. Country 00 uses only the world package. ``policy/`` holds version 1 only and -``policy-v2/`` version 2 only; a package in the wrong directory is refused. +``policy-v2/`` version 2 only and ``policy-v4/`` version 4 only; a package in +the wrong directory is refused. Version 3 archives remain unbound and refused. Kernels without version 2 support never read ``policy-v2/``, so version 2 packages can be installed while those kernels remain bootable. @@ -101,7 +109,8 @@ fallback reason and the result of each lookup. Station security ---------------- -The driver advertises PSK and SAE with CCMP only. Firmware performs SAE +The driver advertises PSK, PSK-SHA256, SAE and OWE with CCMP, plus +BIP-CMAC-128 for management protection. Firmware performs SAE authentication and the four-way handshake and owns PTK, GTK and IGTK; there is no host key-install interface or external-auth exchange. @@ -115,14 +124,20 @@ required or optional management frame protection (MFP). Firmware uses required MFP for the SAE arm. On a PSK+SAE transition network, the first AKM supplied by userspace selects the arm: PSK keeps the existing binary-PMK request and disables MFP, while SAE uses the password and required MFP. PSK with required -MFP and PSK-SHA256 are unsupported. - -BIP-CMAC-128 is not advertised because WPA2 with required MFP has not been -implemented or qualified. Advertising BIP would also make supplicants request -required MFP on WPA2 networks. iwd requires BIP-CMAC-128 before selecting SAE, -so it remains WPA2-only with this driver, including on transition networks; -it cannot join a pure-SAE network. SAE offload support here targets -NetworkManager with wpa_supplicant, pending the hardware qualification below. +MFP is rejected on legacy AKM 2. WPA2 PMF uses the explicitly selected +PSK-SHA256 AKM (6), with a binary 32-byte PMK and optional or required MFP. +The only EA/40 security delta is little-endian 0x0400 at body +0x14. The +host retains the MFP policy; there is no standalone optional/required field. + +Before any association command, the host validates the candidate's RSN IE, +selected AKM, CCMP and MFPC/MFPR. A required request cannot downgrade to +AKM 2 or a non-MFPC AP. Only BIP-CMAC-128 is accepted as an explicit group +management suite; the standard omitted-suite default is also CMAC-128. +Firmware owns IGTK extraction, install/removal and BIP verification. No +add_key or set_default_mgmt_key operation is added. iwd's BIP gate is now +satisfied, but WPA2-PMF and iwd behavior still require hardware qualification. +The admitted PMF state is host policy, not a firmware readback. Selectors +0x40/0x42 retain their normal completion contract; neither reports PMF state. The standard SAE PWE settings (unspecified, hunting-and-pecking, hash-to-element and both) are accepted; firmware chooses the exchange without @@ -183,9 +198,10 @@ Only SAE and OWE with required MFP are admitted on 6 GHz. Hunting-and-pecking only is refused there; firmware owns H2E. These checks extend the SAE and OWE layers without another crypto or authentication implementation. -Wildcard 6 GHz scans listen passively. Hidden SSIDs use directed scanning -when NO_IR permits it. RNR-only requests remain passive until target-list -serialization is implemented. Full-band scans are split into bounded +Legacy VLP channels retain directed hidden-SSID probing and the original +connect discovery sequence. Channels added only by class packages listen +passively until the target supplies its class/TPE. RNR-only requests remain +passive until target-list serialization is implemented. Full-band scans are split into bounded 32-channel commands. Firmware event channel layout is checked before OCA association calibration, queue binding and the two association power pages. @@ -555,11 +571,234 @@ NO_IR, VLP-client restrictions and runtime EIRP/PSD reductions remain authoritative. The driver preserves 0xc4/0x7f sentinels when reducing numeric native half-dBm cells. LPI, SP and AFC are outside this contract. -The active association path submits the two native selector pages after -queue binding and does not build a 60-byte TPE. Generic runtime D7 +The legacy v2 path retains its two native selector pages after queue +binding. Explicit VLP APs on these channels retain their original admission +and command sequence, including ordinary finite or absent TPE and AP-wide +160 MHz announcements (the station still negotiates at most 80 MHz). No +pre-association class/TPE SET is inserted on this compatibility path. Generic runtime D7 requests for band 2 (6 GHz) remain rejected by the raw planner. RF behavior, rate accounting and calibration still require hardware qualification. +6 GHz LPI table archive (J7RP v3, unbound) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The class archive producer audits the authenticated text tables and preserves +LPI alongside VLP. Version 3 does **not** enable LPI operation. Its binding +revision zero remains unbound; version 4 below explicitly binds the same data +to the established class/TPE ABI. Do not infer a command selector from +the VLP tag ``0x2a``, use larger LP values with that tag, or transplant a +related softmac driver's CLC command into this fullmac driver. + +Run an audit without producing packages:: + + python3 tools/asahi/mt7932-policy-classes.py --tables /path/to/IZUBA + +Opt in to **host data archives**, separate from operational v1/v2 packages:: + + python3 tools/asahi/mt7932-policy-classes.py --tables /path/to/IZUBA \ + --legacy-dir /path/to/policy --output-dir /path/to/private-archive-root + python3 tools/asahi/mt7932-policy-classes-tests.py + +Archives use ``policy-v3/CC.bin`` exclusively. Never place them in ``policy/`` +or ``policy-v2/``. The driver does not search policy-v3 and rejects version 3 +if it is mistakenly placed in either operational directory. The installer +archive API is explicit and separate from deployment; deployment remains +v1/v2. No Apple input or generated archive may be committed or redistributed. + +Authenticated table coverage +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For the same 25G83 hashes as v2, the class audit distinguishes section labels, +alphabetic countries, empty sections and usable primaries: + +.. list-table:: + :header-rows: 1 + + * - Class + - Sections + - Countries + - Nonempty countries + - Eligible + * - LP (LPI) + - 135 + - 132 + - 129 + - 129 + * - VLP + - 127 + - 124 + - 120 + - 120 + * - SP + - 112 + - 109 + - 8 + - 8 + +A2/A4/A5 are non-country labels in every class. CN/KH/MH have sections but no +6 GHz matrix row; AR has no usable VLP primary. Every eligible LPI country +has a legacy v1 package. LPI adds AE/AR/CO/EG/GT/JO/PE/SV/TN to the VLP set. +SP is audit-only: no SP archive section, command, AFC permission or runtime +support is produced. The report lists countries and primary counts for all +three classes, including the reasons each unusable section was excluded. + +LP uses matrix mode 1; VLP uses mode 2; SP uses mode 3. These are table +operating-class values, **not firmware command selectors**. Require both the +class-specific matrix mode and at least one projected numeric power cell on +a primary; all-sentinel rows never grant permission. Retain the v2 parsing, +row projection, sentinel and one-SS pad rules separately for each class. +Thus the table-derived LP primary counts are AU/GB/DE/JP=24, US/CA=59; +the corresponding VLP counts are 24 and 50. These counts describe table data, +not a live cfg80211 or firmware grant. + +Unbound v3 byte contract +^^^^^^^^^^^^^^^^^^^^^^^ + +All integers are little endian. The outer J7RP header and legacy mode/table +records are identical to v1 except version=3 and the total file length. +After the thirteen legacy tables append this 16-byte class directory:: + + Offset Size Meaning + 0 4 ASCII 6GCL + 4 1 Archive schema revision 1 + 5 1 Section count: 1 or 2 + 6 1 Firmware binding revision: 0 (unbound data) + 7 1 Flags: zero + 8 4 Directory size, including header and every section + 12 4 Reserved: zero + +Each section has this 24-byte header followed by a 32-byte one-SS pad and +110 table records of 122 bytes each (total section size 13,476 bytes):: + + Offset Size Meaning + 0 1 J7RP class ID: 1=VLP, 2=LPI (not a firmware enum) + 1 1 Indoor-only requirement: 0 for VLP, 1 for LPI + 2 1 Cell units contract: 1, native half-dBm and v2 sentinels + 3 1 Table record count: 110 + 4 4 Section size including header: 13476 + 8 8 Nonzero primary bitmap; bit i denotes channel 1+4*i + 16 2 Alpha2 identical to the outer header + 18 6 Reserved: zero + 24 32 One-SS pad, independently derived for this class + 56 13420 110 records: channel byte followed by 121 projected cells + +Records follow the exact v2 110-channel matrix order, including channel 2 +and excluding 231. Sections are ordered by increasing class ID, with no +duplicates. A usable LPI section is mandatory; a usable VLP section is +optional, so countries without VLP data can still be archived. Reject unknown +classes, binding revisions, flags, units, reserved fields, mismatched countries, +wrong channel order/count/size, empty or out-of-range primary masks, a masked +primary with all-sentinel cells, truncation and trailing bytes. The maximum +archive size is 65,536 bytes. SP must never be serialized. These bodies are +**table records, not CID 0x5d or CID 0x95 packets**. + +Build all selected archives before publication. Refuse a nonempty output root. +Use the v2 inventory rules with ``policy-v3/`` replacing ``policy-v2/``; roots +and directories have mode 0700, files 0600. Preserve every legacy byte except +the version and total size. Audit output contains only counts, country names, +skip reasons, hashes and lengths, never source rows or rate values. + +6 GHz runtime binding (J7RP v4) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The clean 2026-10-31 firmware ABI specification establishes CID 0x5d byte ++0x01[6:4] as the power class: 0 SP, 1 LPI, 2 VLP. Normal infrastructure +per-rate pages use submode 0 and byte 1 of 0x1a for LP data or 0x2a for VLP +data. SP stays disabled because AFC is not implemented. No module parameter +or forced power mode is exposed. + +Version 4 is byte-for-byte version 3 table data, except the J7RP version +halfword is 4 and the 6GCL directory binding revision is 1. It uses only +``policy-v4/CC.bin``. Section order, independent class masks, indoor metadata, +units, bitmap and 110 projected records retain the v3 contract. It requires +LP and permits optional VLP; SP, duplicate/reordered classes, reserved fields, +invalid channel order and sentinel-only permitted primaries are rejected. +These files are data, not captured command envelopes. Older kernels never +search this directory, and v1/v2 remain available for rollback. + +Produce bound packages from the same authenticated inputs:: + + python3 tools/asahi/mt7932-policy-classes.py --tables /path/to/IZUBA \ + --legacy-dir /path/to/policy --output-dir /path/to/new-root --runtime + +For new LPI and v4-only VLP associations the driver validates the complete IE +stream, HE Operation size, primary/center/width and default-client TPE. +Regulatory Info 0 selects LPI, 2 selects VLP; SP and other unqualified values +fail closed. Missing/malformed/duplicate HE Operation never defaults to LPI. +The selected class mask and cfg80211 permissions must allow every occupied +20 MHz channel. The exact installed class-package hash is checked again. +Normal class pages precede the separate 60-byte TPE and EA/40. Partial SET +failure closes RF admission and requires checked reset; no rollback ACK is +invented. The single station has no concurrent BSS class arbitration. + +Existing v2 VLP channels instead retain the original startup CID5d/CID95 +pages and the two 1020-byte CID5d pages after queue binding. Their TPE never +selected admission in aurora-wip, so finite, absent or unsupported TPE does +not introduce a new refusal or pre-association SET on this path. HE Regulatory +Info must explicitly select VLP; it can never authorize SP or select LPI by +fallback. Existing SAE/OWE admission is unchanged. + +The new TPE encoder writes the fixed header, enable, band 3, primary and one +to four SISO/CDD/MIMO bandwidth limits. EIRP interpretations retain half-dBm +bytes. PSD uses the minimum of all advertised 20 MHz values, then adds +26/32/38/44 half-dB ticks for 20/40/80/160 MHz; this conservatively bounds +narrower allocations. Multiple applicable envelopes are intersected; PSD +-128 forbids transmission and is rejected. The 0x7f sentinel survives +conversion and gain subtraction. Signed arithmetic underflow or overflow +is rejected. Extended PSD and subordinate-AP envelopes cannot supply a +station limit. Missing applicable TPE is refused on the new class path. + +The t-3573 J700 antenna-gain SPEC supplies immutable board properties. A fixed +six-by-three signed half-dB table is bound to the existing J700-only probe; +no device-tree property or module parameter is added. SISO/CDD/MIMO tuples +are (16,16,10), (14,14,8), (15,16,10), (15,15,9), (16,16,10), (15,16,10). +See :doc:`mt7932-j700-antenna` for the immutable matrix and lookup contract. +The exact primary-channel/bandwidth map includes the boundary exceptions at +113, 117, 121, 125, 185 and 189. Each finite TPE cell subtracts its own mapped +gain. Both LPI and v4-only VLP use this matrix; neither changes the other +country/class, SAR or hardware ceilings. Missing channel/bandwidth mappings +fail closed for finite cells. No 2.4/5 GHz gains are inferred. + +Country changes retire the connection. Each new class connection reparses +AP information. Changed or invalid class/TPE in its connected AP's beacon +closes data admission and schedules disconnect. After peer/DMA retirement, +RF admission stays fenced while the installed VLP policy is restored and a +known all-0x7f TPE body removes the previous AP ceiling. A subsequent legacy +VLP join uses its original association sequence. Firmware acceptance and RF +compliance, including clearing across class transitions, remain unqualified. +There is no authoritative active-class or PMF readback. Debug metadata +excludes BSSIDs and key material. + +Requirements before runtime enablement +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The static ABI now establishes the class selector and 60-byte TPE transport. +The remaining hardware qualification gates are described below, including changes on association, reassociation, AP changes, country +changes, disconnect and reset. Do not send +unbound archive bytes as commands or claim readback of a firmware-active +class from table presence. Public mac80211 handles HE Operation Regulatory +Info separately from TPE; TPE values alone do not identify the AP class. + +Association must use the target BSS's validated HE 6 GHz Operation Information +and applicable client-category TPE, intersect the country/class primary mask +with cfg80211 permissions, channel width, EIRP and PSD, and enforce indoor-only +rules. LPI cannot be selected for a VLP AP. SP requires separate AFC semantics +and stays unsupported. An LPI AP without country LPI permission must never +cause an LPI power increase; VLP fallback must retain all VLP restrictions and +only occur when allowed for that connection. Unknown or malformed HE/TPE +information cannot authorize higher power. Preserve sentinel meanings and +round power caps downward. Revalidate changing BSS/regulatory state before +submission and abort rather than transmit with stale, more permissive limits. + +The Neo test plan requires a 6 GHz AP explicitly advertising LPI in HE 6 GHz +Operation Regulatory Info, with inspectable client TPEs and a legal indoor +configuration. A phone hotspot may advertise only VLP and cannot close this +gate. Capture the advertised class and TPE independently, verify host selection +and firmware acceptance/readback, then test SAE-H2E and OWE, reconnect, VLP +fallback, forbidden-class/country/indoor cases, country changes, reset, power +caps and v1/v2 rollback. Controller approval with explicit Neo identity and +all session/resource/ownership gates is required for every hardware run. + Local firmware inputs --------------------- diff --git a/MAINTAINERS b/MAINTAINERS index 51b76c00ef905..2526d667b8de7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2111,6 +2111,7 @@ F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml F: Documentation/gpu/apple-dcp-backlight.rst F: Documentation/hid/apple-j700-multitouch.rst +F: Documentation/networking/device_drivers/wifi/mt7932-j700-antenna.rst F: Documentation/networking/device_drivers/wifi/mt7932-neo.rst F: arch/arm64/boot/dts/apple/t8140* F: arch/arm64/configs/j700*.config diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index 65854425354aa..6c47a8d282753 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -7,3 +7,5 @@ mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += input-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += regulatory-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += association-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += six-test.o + +mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += pmf-lpi-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/association.h b/drivers/net/wireless/mediatek/mt7932/association.h index 2028a46558de6..011a029f20357 100644 --- a/drivers/net/wireless/mediatek/mt7932/association.h +++ b/drivers/net/wireless/mediatek/mt7932/association.h @@ -56,6 +56,12 @@ mt7932_connect_auth(const struct cfg80211_connect_params *p) (p->mfp == NL80211_MFP_NO || p->mfp == NL80211_MFP_OPTIONAL)) return WLAN_AKM_SUITE_PSK; break; + case WLAN_AKM_SUITE_PSK_SHA256: + if ((versions & NL80211_WPA_VERSION_2) && p->crypto.psk && + (p->mfp == NL80211_MFP_OPTIONAL || + p->mfp == NL80211_MFP_REQUIRED)) + return WLAN_AKM_SUITE_PSK_SHA256; + break; case WLAN_AKM_SUITE_OWE: if (p->auth_type == NL80211_AUTHTYPE_OPEN_SYSTEM && p->mfp == NL80211_MFP_REQUIRED && !p->crypto.psk && @@ -82,6 +88,117 @@ mt7932_connect_auth(const struct cfg80211_connect_params *p) return -EOPNOTSUPP; } +/* A selected AKM is not an MFP policy field. Firmware owns all handshake + * keys, including IGTK, and BIP verification. No host management-key command + * or optional/required bit has been established for EA/40. + */ +static inline int mt7932_connect_psk_akm(u8 *body, size_t length, u32 akm) +{ + if (length < 856 || body[1] != 4 || + (akm != WLAN_AKM_SUITE_PSK && akm != WLAN_AKM_SUITE_PSK_SHA256)) + return -EINVAL; + put_unaligned_le16(akm == WLAN_AKM_SUITE_PSK_SHA256 ? 0x0400 : 0x0008, + body + 0x14); + return 0; +} + +/* Validate the entire RSN payload before using capabilities for admission. + * The group management suite defaults to BIP-CMAC-128 when omitted. Reject + * unsupported explicit suites rather than letting firmware choose one. + */ +static inline int mt7932_rsn_admit(const u8 *rsn, size_t length, u32 akm, + enum nl80211_mfp mfp, bool *pmf) +{ + unsigned int count, i; + u16 caps = 0; + bool selected = false, ccmp = false; + + *pmf = false; + if (mfp > NL80211_MFP_OPTIONAL || length < 8 || + get_unaligned_le16(rsn) != 1 || + get_unaligned_be32(rsn + 2) != WLAN_CIPHER_SUITE_CCMP) + return -EINVAL; + rsn += 6; + length -= 6; + count = get_unaligned_le16(rsn); + rsn += 2; + length -= 2; + if (!count || count > length / 4) + return -EINVAL; + for (i = 0; i < count; i++) + ccmp |= get_unaligned_be32(rsn + 4 * i) == + WLAN_CIPHER_SUITE_CCMP; + rsn += 4 * count; + length -= 4 * count; + if (!ccmp || length < 2) + return -EINVAL; + count = get_unaligned_le16(rsn); + rsn += 2; + length -= 2; + if (!count || count > length / 4) + return -EINVAL; + for (i = 0; i < count; i++) + selected |= get_unaligned_be32(rsn + 4 * i) == akm; + rsn += 4 * count; + length -= 4 * count; + if (!selected) + return -EOPNOTSUPP; + if (length) { + if (length < 2) + return -EINVAL; + caps = get_unaligned_le16(rsn); + rsn += 2; + length -= 2; + } + if (length) { + if (length < 2) + return -EINVAL; + count = get_unaligned_le16(rsn); + rsn += 2; + length -= 2; + if (count > length / 16) + return -EINVAL; + rsn += 16 * count; + length -= 16 * count; + } + if (length && (length != 4 || + get_unaligned_be32(rsn) != WLAN_CIPHER_SUITE_AES_CMAC)) + return -EOPNOTSUPP; + if ((caps & BIT(6)) && !(caps & BIT(7))) + return -EINVAL; + /* Legacy PSK has no established PMF firmware selection. Optional may + * join it without PMF; required must never downgrade to that arm. + */ + if (akm == WLAN_AKM_SUITE_PSK) + return mfp == NL80211_MFP_REQUIRED || (caps & BIT(6)) ? + -EOPNOTSUPP : + 0; + if (mfp == NL80211_MFP_NO || !(caps & BIT(7))) + return -EOPNOTSUPP; + *pmf = true; + return 0; +} + +static inline int mt7932_connect_admit(const u8 *ies, size_t length, u32 akm, + enum nl80211_mfp mfp, bool *pmf) +{ + const struct element *elem, *rsn = NULL; + size_t parsed = 0; + + *pmf = false; + for_each_element(elem, ies, length) { + parsed += elem->datalen + 2; + if (elem->id != WLAN_EID_RSN) + continue; + if (rsn) + return -EINVAL; + rsn = elem; + } + if (parsed != length || !rsn) + return -EINVAL; + return mt7932_rsn_admit(rsn->data, rsn->datalen, akm, mfp, pmf); +} + /* The SAE offload needs both the password and a WPA passphrase-derived PMK. * This is PBKDF2-HMAC-SHA1(password, SSID, 4096, 32), not host SAE crypto. * Run only in sleepable context and erase every keyed intermediate. diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 51b38f7291b5f..d309646c00b8d 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -168,12 +168,14 @@ static int mt_scan_submit_next(struct mt7932 *m) bool passive = false, directed = false; int ret; - /* Hidden networks are found by a directed probe; supplicants send the - * hidden SSIDs together with a wildcard entry. - */ + /* Hidden SSIDs retain directed probing on the legacy VLP channels. */ for (i = 0; i < request->n_ssids; i++) directed |= !!request->ssids[i].ssid_len; + /* Retain directed discovery on the installed legacy VLP channels. + * New class-only channels need an AP class/TPE before transmission. + */ + /* 2GHz active/passive, then 5GHz active/passive. Never put a NO_IR * channel in a firmware request which can emit probe requests. */ @@ -185,9 +187,13 @@ static int mt_scan_submit_next(struct mt7932 *m) for (i = m->scan_cursor; i < request->n_channels; i++) { struct ieee80211_channel *channel = request->channels[i]; bool radar = channel->flags & IEEE80211_CHAN_RADAR; - bool listen = !request->n_ssids || radar || - (channel->band == NL80211_BAND_6GHZ && !directed) || - (channel->flags & IEEE80211_CHAN_NO_IR); + bool listen = + !request->n_ssids || radar || + (channel->band == NL80211_BAND_6GHZ && + (!directed || + !(m->six_legacy_channels & + BIT_ULL((channel->hw_value - 1) / 4)))) || + (channel->flags & IEEE80211_CHAN_NO_IR); /* Radar channels are never joined; while associated, * a passive dwell there only delays the home channel. @@ -263,6 +269,7 @@ int mt_connect_discover(struct mt7932 *m) bool passive = attempt >= 6; band = 1 + (attempt / 2) % 3; + passive |= band == 3 && !m->six_legacy_channels; sband = band == 1 ? &m->band2 : band == 2 ? &m->band5 : &m->band6; if (band == 3 && m->connect_akm != WLAN_AKM_SUITE_SAE && m->connect_akm != WLAN_AKM_SUITE_OWE) @@ -278,6 +285,9 @@ int mt_connect_discover(struct mt7932 *m) IEEE80211_CHAN_RADAR)) continue; passive |= !!(channel->flags & IEEE80211_CHAN_NO_IR); + passive |= band == 3 && + !(m->six_legacy_channels & + BIT_ULL((channel->hw_value - 1) / 4)); if (count < ARRAY_SIZE(channels)) channels[count++] = channel->hw_value; } @@ -712,6 +722,26 @@ void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length) frame.channel == m->connect_channel && ether_addr_equal(frame.data + 16, m->connect_bssid)) { m->home_beacons++; + if (frame.band == NL80211_BAND_6GHZ && + m->six_ap_valid) { + struct mt7932_6g_ap ap; + + if (mt7932_6g_ap_parse(&ap, frame.data + 36, + frame.length - 36, + frame.channel, + channel->flags) || + memcmp(&ap, &m->six_ap, sizeof(ap))) { + m->connected = false; + m->disconnecting = true; + m->six_ap_valid = false; + m->disconnect_local = true; + m->disconnect_reason = + WLAN_REASON_DEAUTH_LEAVING; + netif_carrier_off(m->netdev); + netif_stop_queue(m->netdev); + schedule_work(&m->disconnect_work); + } + } mt_cqm_signal(m, mt7932_rx_signal(&frame, m->antenna_mask)); if (tim && tim->datalen >= 4 && !m->power_tim) { m->power_tim = true; @@ -725,7 +755,8 @@ void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length) mt7932_rx_signal(&frame, m->antenna_mask) * 100, GFP_ATOMIC); if (bss) { m->beacons++; - dev_dbg_ratelimited(&m->pdev->dev, "BSS_RECEIVED: channel=%u BSSID=%pM\n", frame.channel, bss->bssid); + dev_dbg_ratelimited(&m->pdev->dev, "BSS_RECEIVED: channel=%u\n", + frame.channel); cfg80211_put_bss(m->wiphy, bss); } } diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 713d801cb1891..c2b45bea58879 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -48,6 +48,7 @@ void mt_rf_fail_locked(struct mt7932 *m, int error) if (error >= 0) error = -EIO; WRITE_ONCE(m->rf_ready, false); + m->six_ap_valid = false; WRITE_ONCE(m->link_failed, true); if (!m->cal_state.error) m->cal_state.error = error; @@ -296,6 +297,16 @@ static void mt_peer_release(struct mt7932 *m) m->peer_valid = false; m->peer_wtbl = 0; m->event_cipher = 0; + m->six_ap_valid = false; + m->connect_pmf = false; + /* Restore the installed VLP policy after a class/TPE connection. This + * runs only after peer/DMA retirement; scans and joins stay fenced. + */ + if (m->six_class_dirty && !m->reg_pending && !m->stopping) { + m->reg_generation++; + WRITE_ONCE(m->reg_pending, true); + schedule_work(&m->startup_work); + } } /* No lifecycle/command lock held on entry. RX/IRQ and TX-free processing stay @@ -430,6 +441,20 @@ static int mt_connect_prepare(struct mt7932 *m, bool *peer_owned) m->connect_mfp, m->connect_sae_pwe) || (m->connect_bss->channel->band == NL80211_BAND_6GHZ && !m->six_power_valid)) return -EOPNOTSUPP; + if (m->connect_akm == WLAN_AKM_SUITE_PSK || + m->connect_akm == WLAN_AKM_SUITE_PSK_SHA256) { + const struct cfg80211_bss_ies *ies; + + rcu_read_lock(); + ies = rcu_dereference(m->connect_bss->ies); + ret = ies ? mt7932_connect_admit(ies->data, ies->len, + m->connect_akm, m->connect_mfp, + &m->connect_pmf) : + -EINVAL; + rcu_read_unlock(); + if (ret) + return ret; + } if (m->connect_bss->channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_NO_20MHZ | IEEE80211_CHAN_RADAR)) { return -EINVAL; @@ -527,6 +552,40 @@ static int mt_connect_attempt(struct mt7932 *m, bool *submitted, m->connect_sae_pwd_length, m->connect_owe); if (ret < 0) goto free; + if (m->connect_akm == WLAN_AKM_SUITE_PSK || + m->connect_akm == WLAN_AKM_SUITE_PSK_SHA256) { + ret = mt7932_connect_psk_akm(body, 856, m->connect_akm); + if (ret) + goto free; + } + if (m->connect_band == NL80211_BAND_6GHZ) { + const struct cfg80211_bss_ies *ies; + struct mt7932_6g_ap ap; + + rcu_read_lock(); + ies = rcu_dereference(m->connect_bss->ies); + ret = ies ? mt7932_6g_connect_ap(&ap, ies->data, ies->len, + m->connect_channel, + m->connect_bss->channel->flags, + m->six_legacy_channels) : + -EINVAL; + rcu_read_unlock(); + if (ret < 0) + goto free; + if (ret == 1) { + m->six_legacy_assoc = true; + ret = 0; + goto identity; + } + ret = mt_6g_connect_policy(m, &ap); + if (ret) + goto free; + spin_lock_irq(&m->response_lock); + m->six_ap = ap; + m->six_ap_valid = true; + spin_unlock_irq(&m->response_lock); + } +identity: /* Reprogram the same on-air station identity for either attempt. */ memcpy(mac + 2, m->mac, ETH_ALEN); *submitted = true; @@ -547,8 +606,10 @@ static int mt_connect_attempt(struct mt7932 *m, bool *submitted, m->connect_pmk_valid = false; if (ret) goto out; - dev_dbg(&m->pdev->dev, "CONNECT_SUBMITTED: BSSID=%pM channel=%u\n", - m->connect_bssid, m->connect_channel); + dev_dbg(&m->pdev->dev, + "CONNECT_SUBMITTED: channel=%u AKM=%08x MFP=%u admitted-PMF=%u\n", + m->connect_channel, m->connect_akm, m->connect_mfp, + m->connect_pmf); *auth_submitted = true; deadline = jiffies + msecs_to_jiffies(10000); /* Genuine D7 requests must remain serviceable during asynchronous waits. */ @@ -575,7 +636,8 @@ static int mt_connect_attempt(struct mt7932 *m, bool *submitted, } mt7932_peer_queue(queue, m->peer_ids.station[0], true); ret = mt_request(m, 0x6b, true, true, false, queue, sizeof(queue)); - if (!ret && m->connect_band == NL80211_BAND_6GHZ) { + if (!ret && m->connect_band == NL80211_BAND_6GHZ && + m->six_legacy_assoc) { unsigned int i; for (i = 0; !ret && i < 2; i++) @@ -854,6 +916,9 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, m->connect_owe = auth == WLAN_AKM_SUITE_OWE; m->connect_akm = auth; m->connect_mfp = params->mfp; + m->connect_pmf = false; + m->six_ap_valid = false; + m->six_legacy_assoc = false; m->connect_sae_pwe = params->crypto.sae_pwe; memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); memzero_explicit(m->connect_sae_pwd, sizeof(m->connect_sae_pwd)); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 6afd60d2de8b0..28c339e76ed29 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -112,12 +112,15 @@ struct mt7932 { struct ieee80211_channel channels6[MT7932_CHANNELS_6G]; struct ieee80211_sband_iftype_data he6; u8 six_assoc_power[2][1020]; - bool six_power_valid; + bool six_power_valid, six_legacy_assoc, six_ap_valid, six_class_dirty; + u64 six_legacy_channels; + struct mt7932_6g_ap six_ap; u8 scan_band, scan_batch; unsigned int scan_cursor; enum nl80211_band connect_band, scan_home_band; u32 connect_akm; enum nl80211_mfp connect_mfp; + bool connect_pmf; /* admitted request, never firmware readback */ enum nl80211_sae_pwe_mechanism connect_sae_pwe; struct ieee80211_rate rates[12]; struct work_struct startup_work; @@ -222,6 +225,7 @@ struct mt7932 { bool mt_rf_allowed(struct mt7932 *m); void mt_retry_missing_policy(struct mt7932 *m); +int mt_6g_connect_policy(struct mt7932 *m, const struct mt7932_6g_ap *ap); void mt_reg_apply_deferred_locked(struct mt7932 *m); void mt_policy_debugfs_init(struct mt7932 *m); int mt_net_open(struct net_device *netdev); diff --git a/drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c b/drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c new file mode 100644 index 0000000000000..6d8320f308348 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c @@ -0,0 +1,396 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +#include "association.h" +#include "regulatory.h" + +/* Standard RSN IE: CCMP, AKM 6, MFPC, empty PMKID list, BIP-CMAC-128. */ +static const u8 pmf_rsn[] = { + 48, 26, 1, 0, 0, 15, 172, 4, 1, 0, 0, 15, 172, 4, + 1, 0, 0, 15, 172, 6, 128, 0, 0, 0, 0, 15, 172, 6, +}; + +static void mt_pmf_admission_test(struct kunit *test) +{ + u8 ies[2 * sizeof(pmf_rsn)], pmk[32] = {}; + struct cfg80211_connect_params request = { + .crypto = { + .wpa_versions = NL80211_WPA_VERSION_2, + .n_ciphers_pairwise = 1, + .ciphers_pairwise = {WLAN_CIPHER_SUITE_CCMP}, + .cipher_group = WLAN_CIPHER_SUITE_CCMP, + .n_akm_suites = 1, + .akm_suites = {WLAN_AKM_SUITE_PSK_SHA256}, + .psk = pmk, + }, + }; + bool pmf; + unsigned int i; + + for (i = NL80211_MFP_REQUIRED; i <= NL80211_MFP_OPTIONAL; i++) { + request.mfp = i; + KUNIT_EXPECT_EQ(test, mt7932_connect_auth(&request), + WLAN_AKM_SUITE_PSK_SHA256); + KUNIT_EXPECT_EQ(test, + mt7932_connect_admit(pmf_rsn, sizeof(pmf_rsn), + WLAN_AKM_SUITE_PSK_SHA256, + i, &pmf), + 0); + KUNIT_EXPECT_TRUE(test, pmf); + } + request.mfp = NL80211_MFP_NO; + KUNIT_EXPECT_LT(test, mt7932_connect_auth(&request), 0); + request.mfp = NL80211_MFP_REQUIRED; + request.crypto.wpa_versions = NL80211_WPA_VERSION_3; + KUNIT_EXPECT_LT(test, mt7932_connect_auth(&request), 0); + request.crypto.wpa_versions = NL80211_WPA_VERSION_2; + request.crypto.n_akm_suites = 2; + request.crypto.akm_suites[0] = WLAN_AKM_SUITE_PSK; + request.crypto.akm_suites[1] = WLAN_AKM_SUITE_PSK_SHA256; + KUNIT_EXPECT_LT(test, mt7932_connect_auth(&request), 0); + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(pmf_rsn, sizeof(pmf_rsn), + WLAN_AKM_SUITE_PSK_SHA256, + NL80211_MFP_NO, &pmf), + 0); + for (i = 0; i < sizeof(pmf_rsn); i++) + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(pmf_rsn, i, + WLAN_AKM_SUITE_PSK_SHA256, + NL80211_MFP_REQUIRED, &pmf), + 0); + memcpy(ies, pmf_rsn, sizeof(pmf_rsn)); + ies[20] = 0; /* no MFPC */ + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(ies, sizeof(pmf_rsn), + WLAN_AKM_SUITE_PSK_SHA256, + NL80211_MFP_REQUIRED, &pmf), + 0); + ies[20] = 64; /* MFPR without MFPC */ + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(ies, sizeof(pmf_rsn), + WLAN_AKM_SUITE_PSK_SHA256, + NL80211_MFP_OPTIONAL, &pmf), + 0); + ies[20] = 128; + ies[27] = 11; /* unqualified BIP-GMAC-128 */ + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(ies, sizeof(pmf_rsn), + WLAN_AKM_SUITE_PSK_SHA256, + NL80211_MFP_REQUIRED, &pmf), + 0); + memcpy(ies, pmf_rsn, sizeof(pmf_rsn)); + memcpy(ies + sizeof(pmf_rsn), pmf_rsn, sizeof(pmf_rsn)); + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(ies, sizeof(ies), + WLAN_AKM_SUITE_PSK_SHA256, + NL80211_MFP_REQUIRED, &pmf), + 0); + ies[1] = + 20; /* no PMKID list or explicit management cipher: CMAC default */ + KUNIT_EXPECT_EQ(test, + mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK_SHA256, + NL80211_MFP_REQUIRED, &pmf), + 0); + ies[19] = 2; /* legacy PSK cannot satisfy required PMF */ + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK, + NL80211_MFP_REQUIRED, &pmf), + 0); + KUNIT_EXPECT_EQ(test, + mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK, + NL80211_MFP_OPTIONAL, &pmf), + 0); + KUNIT_EXPECT_FALSE(test, pmf); + ies[20] |= 64; + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK, + NL80211_MFP_OPTIONAL, &pmf), + 0); +} + +static void mt_pmf_body_test(struct kunit *test) +{ + static const struct mt7932_peer_ids ids = { { 1, 2 }, { 3, 4, 5, 6 } }; + static const u8 peer[] = { 2, 0, 0, 0, 0, 1 }; + u8 *body, *legacy, pmk[32] = {}; + + body = kunit_kzalloc(test, 856, GFP_KERNEL); + legacy = kunit_kzalloc(test, 856, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, body); + KUNIT_ASSERT_NOT_NULL(test, legacy); + KUNIT_ASSERT_EQ(test, + mt7932_connect_body(body, 856, "test", 4, pmk, peer, + NL80211_BAND_5GHZ, 36, 20, 2, &ids, + false, NULL, 0, false), + 856); + memcpy(legacy, body, 856); + KUNIT_ASSERT_EQ(test, + mt7932_connect_psk_akm(body, 856, + WLAN_AKM_SUITE_PSK_SHA256), + 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 0x14), 0x0400); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(body + 0x40), 32U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(body + 0x44), 6U); + KUNIT_EXPECT_MEMEQ(test, body + 0x4c, pmk, 32); + KUNIT_ASSERT_EQ(test, + mt7932_connect_psk_akm(body, 856, WLAN_AKM_SUITE_PSK), 0); + KUNIT_EXPECT_MEMEQ(test, body, legacy, + 856); /* only selected AKM changed */ + KUNIT_EXPECT_LT(test, + mt7932_connect_psk_akm(body, 855, + WLAN_AKM_SUITE_PSK_SHA256), + 0); +} + +/* HE operation + uniform default-client PSD TPE at -1 dBm/MHz. */ +static const u8 lpi_ies[] = { + 255, 12, 36, 0, 0, 2, 0, 0, 0, 5, 2, 7, 0, 6, 195, 2, 24, 254, +}; + +static void mt_6g_ap_test(struct kunit *test) +{ + struct mt7932_6g_ap ap; + u8 ies[sizeof(lpi_ies) + 14]; + unsigned int i; + + KUNIT_ASSERT_EQ(test, + mt7932_6g_ap_parse(&ap, lpi_ies, sizeof(lpi_ies), 5, 0), + 0); + KUNIT_EXPECT_EQ(test, ap.mode, MT7932_6G_LPI); + KUNIT_EXPECT_EQ(test, ap.count, 4); + KUNIT_EXPECT_EQ(test, ap.limit[0], 24); + KUNIT_EXPECT_EQ(test, ap.limit[3], 42); + for (i = 0; i < sizeof(lpi_ies); i++) + KUNIT_EXPECT_LT(test, mt7932_6g_ap_parse(&ap, lpi_ies, i, 5, 0), + 0); + memcpy(ies, lpi_ies, sizeof(lpi_ies)); + ies[10] |= 8; /* SP */ + KUNIT_EXPECT_LT(test, + mt7932_6g_ap_parse(&ap, ies, sizeof(lpi_ies), 5, 0), 0); + ies[10] = 2 | 16; /* VLP */ + KUNIT_EXPECT_EQ(test, + mt7932_6g_ap_parse(&ap, ies, sizeof(lpi_ies), 5, 0), 0); + KUNIT_EXPECT_EQ(test, ap.mode, MT7932_6G_VLP); + KUNIT_EXPECT_LT(test, + mt7932_6g_ap_parse(&ap, ies, sizeof(lpi_ies), 5, + IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT), + 0); + ies[10] = 2 | 120; /* reserved cannot authorize LPI */ + KUNIT_EXPECT_LT(test, + mt7932_6g_ap_parse(&ap, ies, sizeof(lpi_ies), 5, 0), 0); + memcpy(ies, lpi_ies, sizeof(lpi_ies)); + KUNIT_EXPECT_LT(test, + mt7932_6g_ap_parse(&ap, ies, sizeof(lpi_ies), 5, + IEEE80211_CHAN_NO_IR), + 0); + KUNIT_EXPECT_LT(test, + mt7932_6g_ap_parse(&ap, ies, sizeof(lpi_ies), 9, 0), 0); + memcpy(ies + sizeof(lpi_ies), lpi_ies, 14); + KUNIT_EXPECT_LT(test, mt7932_6g_ap_parse(&ap, ies, sizeof(ies), 5, 0), + 0); + memcpy(ies, lpi_ies, sizeof(lpi_ies)); + ies[17] = + 0x80; /* PSD forbids transmission, not a very low finite cap. */ + KUNIT_EXPECT_EQ(test, + mt7932_6g_ap_parse(&ap, ies, sizeof(lpi_ies), 5, 0), + -EACCES); + /* TPE alone cannot select LPI. */ + KUNIT_EXPECT_LT(test, mt7932_6g_ap_parse(&ap, lpi_ies + 14, 4, 5, 0), + 0); +} + +static void mt_6g_tpe_body_test(struct kunit *test) +{ + static const s8 gain[3][4] = { { 2, 4, 6, 8 }, + { 4, 6, 8, 10 }, + { 6, 8, 10, 12 } }; + struct mt7932_6g_ap ap = { + .mode = MT7932_6G_LPI, + .primary = 5, + .count = 3, + .limit = { 30, -10, 127, 0 }, + }; + u8 body[61]; + + memset(body, 0xa5, sizeof(body)); + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), 60); + KUNIT_EXPECT_EQ(test, body[60], 0xa5); + KUNIT_EXPECT_EQ(test, body[0], 0); + KUNIT_EXPECT_EQ(test, body[1], 2); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 2), 60); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(body + 4), 0x03010301U); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(body + 8, 0, 36), NULL); + KUNIT_EXPECT_EQ(test, body[44], 1); + KUNIT_EXPECT_EQ(test, body[45], 3); + KUNIT_EXPECT_EQ(test, body[46], 5); + KUNIT_EXPECT_EQ(test, body[47], 3); + KUNIT_EXPECT_EQ(test, body[48], 28); + KUNIT_EXPECT_EQ(test, (s8)body[49], -14); + KUNIT_EXPECT_EQ(test, body[52], 26); + KUNIT_EXPECT_EQ(test, body[56], 24); + KUNIT_EXPECT_EQ(test, body[50], 127); + KUNIT_EXPECT_EQ(test, body[51], 127); + KUNIT_EXPECT_EQ(test, body[58], 127); + KUNIT_EXPECT_EQ(test, body[59], 127); + KUNIT_EXPECT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, NULL), + -EOPNOTSUPP); + ap.limit[0] = -128; + KUNIT_EXPECT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), -ERANGE); + memset(ap.limit, 127, 4); + KUNIT_EXPECT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, NULL), 60); + KUNIT_EXPECT_EQ(test, mt7932_6g_tpe_body(body, 59, &ap, gain), -EINVAL); + ap.count = 5; + KUNIT_EXPECT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), -EINVAL); +} + +/* t-3573 section 3: bandwidth-specific boundary rows and all six groups. */ +static void mt_6g_j700_gain_test(struct kunit *test) +{ + static const struct { + u8 channel; + u8 group[4]; + } vectors[] = { + { 1, { 1, 1, 1, 1 } }, { 29, { 1, 1, 1, 1 } }, + { 33, { 2, 2, 2, 2 } }, { 61, { 2, 2, 2, 2 } }, + { 65, { 3, 3, 3, 3 } }, { 93, { 3, 3, 3, 3 } }, + { 97, { 4, 4, 4, 4 } }, { 109, { 4, 4, 4, 4 } }, + { 113, { 4, 4, 5, 4 } }, { 117, { 5, 4, 5, 4 } }, + { 121, { 5, 5, 5, 4 } }, { 125, { 5, 5, 5, 4 } }, + { 129, { 5, 5, 5, 5 } }, { 181, { 5, 5, 5, 5 } }, + { 185, { 5, 6, 5, 5 } }, { 189, { 6, 6, 5, 5 } }, + { 193, { 6, 6, 6, 6 } }, { 221, { 6, 6, 6, 6 } }, + { 225, { 6, 6, 0, 0 } }, { 229, { 6, 6, 0, 0 } }, + { 233, { 6, 0, 0, 0 } }, { 2, { 1, 0, 0, 0 } }, + }; + static const s8 expected[][3] = { + { 16, 16, 10 }, { 14, 14, 8 }, { 15, 16, 10 }, + { 15, 15, 9 }, { 16, 16, 10 }, { 15, 16, 10 }, + }; + s8 gain[3]; + unsigned int i, bw; + + for (i = 0; i < ARRAY_SIZE(vectors); i++) { + for (bw = 0; bw < 4; bw++) { + int ret = mt7932_j700_6g_gain(vectors[i].channel, bw, + gain); + u8 group = vectors[i].group[bw]; + + if (!group) { + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + continue; + } + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_MEMEQ(test, gain, expected[group - 1], 3); + } + } + KUNIT_EXPECT_EQ(test, mt7932_j700_6g_gain(0, 0, gain), -EINVAL); + KUNIT_EXPECT_EQ(test, mt7932_j700_6g_gain(3, 0, gain), -EINVAL); + KUNIT_EXPECT_EQ(test, mt7932_j700_6g_gain(237, 0, gain), -EINVAL); + KUNIT_EXPECT_EQ(test, mt7932_j700_6g_gain(5, 4, gain), -EINVAL); +} + +static void mt_6g_finite_projection_test(struct kunit *test) +{ + struct mt7932_6g_ap ap = { + .mode = MT7932_6G_LPI, + .primary = 117, + .count = 4, + .limit = { 40, 40, 40, 40 }, + }; + /* Channel 117 uses G5/G4/G5/G4, not a frequency-only group. */ + static const u8 expected[] = { 24, 25, 24, 25, 24, 25, + 24, 25, 30, 31, 30, 31 }; + s8 gain[3][4]; + u8 body[60], lpi[60]; + + KUNIT_ASSERT_EQ(test, mt7932_j700_6g_gains(&ap, gain), 0); + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), 60); + KUNIT_EXPECT_MEMEQ(test, body + 48, expected, sizeof(expected)); + memcpy(lpi, body, 60); + ap.mode = MT7932_6G_VLP; + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), 60); + KUNIT_EXPECT_MEMEQ(test, lpi, body, 60); + /* PSD -1 dBm/MHz => 24/30/36/42 half-dBm EIRP; G1 gains. */ + KUNIT_ASSERT_EQ(test, + mt7932_6g_ap_parse(&ap, lpi_ies, sizeof(lpi_ies), 5, 0), + 0); + KUNIT_ASSERT_EQ(test, mt7932_j700_6g_gains(&ap, gain), 0); + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), 60); + KUNIT_EXPECT_EQ(test, body[48], 8); + KUNIT_EXPECT_EQ(test, body[51], 26); + KUNIT_EXPECT_EQ(test, body[56], 14); + KUNIT_EXPECT_EQ(test, body[59], 32); + ap.primary = 233; + KUNIT_EXPECT_EQ(test, mt7932_j700_6g_gains(&ap, gain), -EOPNOTSUPP); + memset(ap.limit, 127, 4); + ap.limit[0] = 40; + KUNIT_ASSERT_EQ(test, mt7932_j700_6g_gains(&ap, gain), 0); + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), 60); + KUNIT_EXPECT_EQ(test, body[49], 127); + /* Signed subtraction, overflow in both directions, and sentinel. */ + ap.count = 1; + ap.limit[0] = 0; + gain[0][0] = -7; + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), 60); + KUNIT_EXPECT_EQ(test, body[48], 7); + ap.limit[0] = 126; + KUNIT_EXPECT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), -ERANGE); + ap.limit[0] = -128; + gain[0][0] = 1; + KUNIT_EXPECT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), -ERANGE); +} + +static void mt_6g_legacy_vlp_finite_test(struct kunit *test) +{ + struct mt7932_6g_ap ap; + u8 ies[sizeof(lpi_ies)]; + + memcpy(ies, lpi_ies, sizeof(ies)); + ies[10] |= 16; /* VLP, ordinary finite default-client PSD TPE */ + /* Legacy admission selects no new class/TPE SETs, regardless of TPE. + * The caller retains the two original post-CID6b cached pages. + */ + KUNIT_EXPECT_EQ(test, + mt7932_6g_connect_ap(&ap, ies, sizeof(ies), 5, 0, + BIT_ULL(1)), + 1); + KUNIT_EXPECT_EQ(test, + mt7932_6g_connect_ap(&ap, ies, 14, 5, 0, BIT_ULL(1)), 1); + ies[10] = 3 | 16; /* AP advertises 160 MHz; STA still negotiates <=80 */ + KUNIT_EXPECT_EQ(test, + mt7932_6g_connect_ap(&ap, ies, sizeof(ies), 5, 0, + BIT_ULL(1)), + 1); + ies[10] = 2 | 16; + KUNIT_EXPECT_EQ(test, + mt7932_6g_connect_ap(&ap, ies, sizeof(ies), 5, 0, 0), 0); + KUNIT_EXPECT_EQ(test, ap.mode, MT7932_6G_VLP); + KUNIT_EXPECT_EQ(test, ap.limit[0], 24); + ies[10] = 2; /* LPI must never take the legacy VLP bypass. */ + KUNIT_EXPECT_EQ(test, + mt7932_6g_connect_ap(&ap, ies, sizeof(ies), 5, 0, + BIT_ULL(1)), + 0); + KUNIT_EXPECT_EQ(test, ap.mode, MT7932_6G_LPI); +} + +static struct kunit_case mt_pmf_lpi_cases[] = { + KUNIT_CASE(mt_pmf_admission_test), + KUNIT_CASE(mt_pmf_body_test), + KUNIT_CASE(mt_6g_ap_test), + KUNIT_CASE(mt_6g_tpe_body_test), + KUNIT_CASE(mt_6g_j700_gain_test), + KUNIT_CASE(mt_6g_finite_projection_test), + KUNIT_CASE(mt_6g_legacy_vlp_finite_test), + {} +}; + +static struct kunit_suite mt_pmf_lpi_suite = { + .name = "mt7932-pmf-lpi", + .test_cases = mt_pmf_lpi_cases, +}; + +kunit_test_suite(mt_pmf_lpi_suite); + +MODULE_LICENSE("GPL"); diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c index 544497108d4e9..be1dc20a8d477 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c +++ b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c @@ -551,6 +551,127 @@ static u8 *pkg_build_v2(struct kunit *test, const char *alpha2, u64 primaries, s return data; } +static u8 *pkg_build_v4(struct kunit *test, size_t *size) +{ + size_t base_size; + u8 *base = pkg_build(test, "AU", &base_size), *data, *directory; + unsigned int i, j; + + *size = base_size + 16 + 2 * 13476; + data = kunit_kzalloc(test, *size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, data); + memcpy(data, base, base_size); + put_unaligned_le16(4, data + 4); + put_unaligned_le32(*size, data + 12); + directory = data + base_size; + memcpy(directory, "6GCL", 4); + directory[4] = 1; + directory[5] = 2; + directory[6] = 1; + put_unaligned_le32(16 + 2 * 13476, directory + 8); + for (i = 0; i < 2; i++) { + u8 *section = directory + 16 + i * 13476; + unsigned int channel = 1; + + section[0] = i + 1; + section[1] = i; + section[2] = 1; + section[3] = 110; + put_unaligned_le32(13476, section + 4); + put_unaligned_le64(BIT_ULL(i ? 1 : 9), section + 8); + memcpy(section + 16, "AU", 2); + for (j = 0; j < 110; j++) { + u8 *record = section + 56 + j * 122; + + record[0] = j ? channel : 2; + memset(record + 1, i ? 40 : 20, 121); + if (j) { + channel += 2; + while (channel == 31 || channel == 63 || + channel == 95 || channel == 127 || + channel == 159 || channel == 191 || + channel == 223 || channel == 231) + channel += 2; + } + } + } + return data; +} + +static void mt7932_policy_v4_classes_test(struct kunit *test) +{ + struct mt7932_policy *policy = + kunit_kzalloc(test, sizeof(*policy), GFP_KERNEL); + size_t size, base_size; + u8 *data = pkg_build_v4(test, &size), *body, *ext; + unsigned int first, total = 0; + int length; + + KUNIT_ASSERT_NOT_NULL(test, policy); + body = kunit_kzalloc(test, 1021, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, body); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(policy, data, size, "AU"), 0); + KUNIT_EXPECT_EQ(test, policy->six_class[MT7932_6G_LPI].channels, + BIT_ULL(1)); + KUNIT_EXPECT_EQ(test, policy->six_class[MT7932_6G_VLP].channels, + BIT_ULL(9)); + KUNIT_EXPECT_TRUE(test, + mt7932_policy_permits(policy, NL80211_BAND_6GHZ, 5)); + KUNIT_EXPECT_TRUE(test, + mt7932_policy_permits(policy, NL80211_BAND_6GHZ, 37)); + KUNIT_EXPECT_FALSE(test, + mt7932_policy_permits(policy, NL80211_BAND_6GHZ, 1)); + KUNIT_EXPECT_LT(test, + mt7932_6g_class_page(body, 1020, policy, MT7932_6G_SP, + "AU", 0), + 0); + for (first = 0; first < 110; first += 8) { + length = mt7932_6g_class_page(body, 1020, policy, MT7932_6G_LPI, + "AU", first); + KUNIT_ASSERT_GT(test, length, 0); + KUNIT_EXPECT_EQ(test, body[0], 0); + KUNIT_EXPECT_EQ(test, body[1], 0x1a); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 2), length); + KUNIT_EXPECT_EQ(test, body[5], 3); + KUNIT_EXPECT_EQ(test, body[6], first == 104); + KUNIT_EXPECT_EQ(test, body[45], 40); + total += body[4]; + } + KUNIT_EXPECT_EQ(test, total, 110U); + KUNIT_EXPECT_EQ(test, length, 776); + KUNIT_EXPECT_EQ(test, + mt7932_6g_class_page(body, 1020, policy, MT7932_6G_VLP, + "AU", 0), + 1020); + KUNIT_EXPECT_EQ(test, body[1], 0x2a); + KUNIT_EXPECT_EQ(test, body[45], 20); + KUNIT_EXPECT_LT(test, + mt7932_6g_class_page(body, 1019, policy, MT7932_6G_LPI, + "AU", 0), + 0); + KUNIT_EXPECT_LT(test, + mt7932_6g_class_page(body, 1020, policy, MT7932_6G_LPI, + "AU", 1), + 0); + base_size = size - 16 - 2 * 13476; + ext = data + base_size; + /* Unbound v3 cannot acquire runtime authority through a path rename. */ + put_unaligned_le16(3, data + 4); + ext[6] = 0; + KUNIT_EXPECT_LT(test, mt7932_policy_parse(policy, data, size, "AU"), 0); + put_unaligned_le16(4, data + 4); + KUNIT_EXPECT_LT(test, mt7932_policy_parse(policy, data, size, "AU"), 0); + ext[6] = 1; + KUNIT_EXPECT_LT(test, mt7932_policy_parse(policy, data, size - 1, "AU"), + 0); + KUNIT_EXPECT_LT(test, mt7932_policy_parse(policy, data, size, "NZ"), 0); + ext[16 + 13476] = 3; /* never serialize SP */ + KUNIT_EXPECT_LT(test, mt7932_policy_parse(policy, data, size, "AU"), 0); + ext[16 + 13476] = 2; + ext[16 + 56] = 1; /* first record must be channel 2 */ + KUNIT_EXPECT_LT(test, mt7932_policy_parse(policy, data, size, "AU"), 0); +} + #define FAKE_FILES 8 #define FAKE_LOG 16 @@ -590,8 +711,9 @@ static void fake_put(void *ctx, struct mt7932_policy_file *file) { struct fake_fs *fs = ctx; - fs->held--; + fs->held -= file->class_handle ? 2 : 1; file->handle = NULL; + file->class_handle = NULL; } static void fake_add(struct fake_fs *fs, const char *path, const u8 *data, size_t size) @@ -665,10 +787,11 @@ static void mt7932_policy_order_test(struct kunit *test) char path[MT7932_POLICY_PATH_MAX]; struct select_env *env = select_env(test); - KUNIT_ASSERT_EQ(test, mt7932_policy_candidates("AU", sources), 3); - KUNIT_EXPECT_EQ(test, sources[0], MT7932_POLICY_V2); - KUNIT_EXPECT_EQ(test, sources[1], MT7932_POLICY_V1); - KUNIT_EXPECT_EQ(test, sources[2], MT7932_POLICY_WORLD); + KUNIT_ASSERT_EQ(test, mt7932_policy_candidates("AU", sources), 4); + KUNIT_EXPECT_EQ(test, sources[0], MT7932_POLICY_V4); + KUNIT_EXPECT_EQ(test, sources[1], MT7932_POLICY_V2); + KUNIT_EXPECT_EQ(test, sources[2], MT7932_POLICY_V1); + KUNIT_EXPECT_EQ(test, sources[3], MT7932_POLICY_WORLD); KUNIT_ASSERT_EQ(test, mt7932_policy_candidates("00", sources), 1); KUNIT_EXPECT_EQ(test, sources[0], MT7932_POLICY_WORLD); mt7932_policy_path(path, sizeof(path), MT7932_POLICY_V2, "AU"); @@ -680,13 +803,18 @@ static void mt7932_policy_order_test(struct kunit *test) /* Nothing installed: every candidate is asked for, in order. */ KUNIT_EXPECT_EQ(test, select_run(env, "NZ", false), -ENOENT); - KUNIT_ASSERT_EQ(test, env->fs.requests, 3); - KUNIT_EXPECT_STREQ(test, env->fs.log[0], "mediatek/mt7932/policy-v2/NZ.bin"); - KUNIT_EXPECT_STREQ(test, env->fs.log[1], "mediatek/mt7932/policy/NZ.bin"); - KUNIT_EXPECT_STREQ(test, env->fs.log[2], "mediatek/mt7932/policy/world-XZ.bin"); + KUNIT_ASSERT_EQ(test, env->fs.requests, 4); + KUNIT_EXPECT_STREQ(test, env->fs.log[0], + "mediatek/mt7932/policy-v4/NZ.bin"); + KUNIT_EXPECT_STREQ(test, env->fs.log[1], + "mediatek/mt7932/policy-v2/NZ.bin"); + KUNIT_EXPECT_STREQ(test, env->fs.log[2], + "mediatek/mt7932/policy/NZ.bin"); + KUNIT_EXPECT_STREQ(test, env->fs.log[3], + "mediatek/mt7932/policy/world-XZ.bin"); KUNIT_EXPECT_PTR_EQ(test, env->file.handle, NULL); KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_NONE); - KUNIT_EXPECT_EQ(test, env->choice.n_tried, 3); + KUNIT_EXPECT_EQ(test, env->choice.n_tried, 4); /* cfg80211's numeric codes have no package of their own. */ KUNIT_EXPECT_EQ(test, mt7932_policy_candidates("98", sources), 1); @@ -717,7 +845,7 @@ static void mt7932_policy_v2_preferred_test(struct kunit *test) fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size); KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); - KUNIT_EXPECT_EQ(test, env->fs.requests, 1); + KUNIT_EXPECT_EQ(test, env->fs.requests, 2); KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_V2); KUNIT_EXPECT_EQ(test, env->choice.version, 2); KUNIT_EXPECT_EQ(test, env->choice.size, v2_size); @@ -738,8 +866,8 @@ static void mt7932_policy_v2_preferred_test(struct kunit *test) /* Malformed v2: v1 is used and 6 GHz leaves the firmware domain. */ put_unaligned_le32(v2_size - 1, v2 + 12); KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); - KUNIT_EXPECT_EQ(test, env->fs.requests, 2); - KUNIT_EXPECT_EQ(test, env->choice.tried[0].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->fs.requests, 3); + KUNIT_EXPECT_EQ(test, env->choice.tried[1].error, -EINVAL); KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_V1); KUNIT_EXPECT_EQ(test, env->choice.version, 1); KUNIT_EXPECT_EQ(test, env->choice.fallback, 0); @@ -752,6 +880,64 @@ static void mt7932_policy_v2_preferred_test(struct kunit *test) KUNIT_EXPECT_EQ(test, env->fs.held, 0); } +static void mt7932_policy_v4_preferred_test(struct kunit *test) +{ + struct select_env *env = select_env(test); + struct mt7932_policy *baseline = + kunit_kzalloc(test, sizeof(*baseline), GFP_KERNEL); + size_t size, v2_size; + u8 *data = pkg_build_v4(test, &size); + u8 *v2 = pkg_build_v2(test, "AU", BIT_ULL(9), &v2_size); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, baseline); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(baseline, v2, v2_size, "AU"), + 0); + fake_add(&env->fs, "mediatek/mt7932/policy-v4/AU.bin", data, size); + fake_add(&env->fs, "mediatek/mt7932/policy-v2/AU.bin", v2, v2_size); + KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_V2); + KUNIT_EXPECT_EQ(test, env->choice.version, 2); + KUNIT_EXPECT_TRUE(test, env->choice.class_valid); + KUNIT_EXPECT_EQ(test, env->choice.six_primaries, 2); + KUNIT_EXPECT_EQ(test, env->policy.legacy_channels, BIT_ULL(9)); + KUNIT_EXPECT_EQ(test, env->fs.requests, 2); + KUNIT_EXPECT_EQ(test, env->fs.held, 2); + /* Every original startup CID/body, and both post-queue association + * pages, are retained byte for byte even with the LPI sidecar. + */ + KUNIT_ASSERT_EQ(test, env->policy.six_pages, baseline->six_pages); + for (i = 0; i < baseline->six_pages; i++) { + KUNIT_EXPECT_EQ(test, env->policy.six_cid[i], + baseline->six_cid[i]); + KUNIT_EXPECT_EQ(test, env->policy.six_length[i], + baseline->six_length[i]); + KUNIT_EXPECT_MEMEQ(test, env->policy.six_page[i], + baseline->six_page[i], + baseline->six_length[i]); + } + for (i = 0; i < 2; i++) + KUNIT_EXPECT_MEMEQ(test, env->policy.assoc_power[i], + baseline->assoc_power[i], 1020); + /* Extra LP channels must not change the legacy VLP numeric clamp. */ + env->reg.power[mt7932_6g_reg_index(5)] = -1; + env->reg.power[mt7932_6g_reg_index(37)] = 14; + KUNIT_EXPECT_EQ(test, + mt7932_6g_limits_mask(&env->reg, + env->policy.legacy_channels) + .eirp, + 28); + KUNIT_EXPECT_EQ(test, mt7932_6g_limits(&env->reg).eirp, -2); + data[4] = 3; + KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_V2); + KUNIT_EXPECT_FALSE(test, env->choice.class_valid); + KUNIT_EXPECT_EQ(test, env->choice.tried[0].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->choice.six_primaries, 1); + fake_put(&env->fs, &env->file); + KUNIT_EXPECT_EQ(test, env->fs.held, 0); +} + /* Each directory accepts one version, and the size limits hold. */ static void mt7932_policy_version_test(struct kunit *test) { @@ -771,8 +957,8 @@ static void mt7932_policy_version_test(struct kunit *test) fake_add(&env->fs, "mediatek/mt7932/policy/AU.bin", v2, v2_size); fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size); KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); - KUNIT_EXPECT_EQ(test, env->choice.tried[0].error, -ENOENT); - KUNIT_EXPECT_EQ(test, env->choice.tried[1].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->choice.tried[1].error, -ENOENT); + KUNIT_EXPECT_EQ(test, env->choice.tried[2].error, -EINVAL); KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_WORLD); KUNIT_EXPECT_EQ(test, env->choice.fallback, -EINVAL); KUNIT_EXPECT_EQ(test, env->choice.six_primaries, 0); @@ -782,7 +968,7 @@ static void mt7932_policy_version_test(struct kunit *test) fake_add(&env->fs, "mediatek/mt7932/policy-v2/AU.bin", v1, v1_size); fake_add(&env->fs, "mediatek/mt7932/policy/AU.bin", v1, v1_size); KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); - KUNIT_EXPECT_EQ(test, env->choice.tried[0].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->choice.tried[1].error, -EINVAL); KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_V1); /* Over 16 KiB as v1, over 64 KiB as v2. */ @@ -795,8 +981,8 @@ static void mt7932_policy_version_test(struct kunit *test) fake_add(&env->fs, "mediatek/mt7932/policy/AU.bin", big, big_size); fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size); KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); - KUNIT_EXPECT_EQ(test, env->choice.tried[0].error, -EINVAL); KUNIT_EXPECT_EQ(test, env->choice.tried[1].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->choice.tried[2].error, -EINVAL); KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_WORLD); /* Another country's package under this country's name. */ @@ -932,6 +1118,29 @@ static void mt7932_policy_switch_test(struct kunit *test) KUNIT_EXPECT_EQ(test, env->fs.held, 0); } +/* Unbound class archives must never be consumed as firmware command pages. */ +static void mt7932_policy_unbound_test(struct kunit *test) +{ + struct select_env *env = select_env(test); + struct mt7932_policy policy; + size_t size, xz_size; + u8 *data, *xz; + + data = pkg_build_v2(test, "AU", BIT_ULL(1), &size); + xz = pkg_build(test, "XZ", &xz_size); + put_unaligned_le16(3, data + 4); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "AU"), + -EINVAL); + fake_add(&env->fs, "mediatek/mt7932/policy-v2/AU.bin", data, size); + fake_add(&env->fs, "mediatek/mt7932/policy/AU.bin", data, size); + fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size); + KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); + KUNIT_EXPECT_EQ(test, env->choice.tried[1].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->choice.tried[2].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_WORLD); + KUNIT_EXPECT_EQ(test, env->choice.six_primaries, 0); +} + static struct kunit_case mt7932_policy_test_cases[] = { KUNIT_CASE(mt7932_reg_relaxation_test), KUNIT_CASE(mt7932_policy_valid_test), @@ -947,8 +1156,11 @@ static struct kunit_case mt7932_policy_test_cases[] = { KUNIT_CASE(mt7932_6g_power_limit_test), KUNIT_CASE(mt7932_policy_corruption_test), KUNIT_CASE(mt7932_policy_order_test), + KUNIT_CASE(mt7932_policy_v4_classes_test), + KUNIT_CASE(mt7932_policy_v4_preferred_test), KUNIT_CASE(mt7932_policy_v2_preferred_test), KUNIT_CASE(mt7932_policy_version_test), + KUNIT_CASE(mt7932_policy_unbound_test), KUNIT_CASE(mt7932_policy_fallback_test), KUNIT_CASE(mt7932_policy_switch_test), {} diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory.h b/drivers/net/wireless/mediatek/mt7932/regulatory.h index 4bf7a78a4e113..24dcbe6336599 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory.h +++ b/drivers/net/wireless/mediatek/mt7932/regulatory.h @@ -9,17 +9,26 @@ #include #include "channels.h" +#include "six.h" #define MT7932_POLICY_TABLES 13 #define MT7932_POLICY_MODES 3 #define MT7932_POLICY_6G_PAGES 32 +#define MT7932_6G_RECORDS 110 + +struct mt7932_6g_table { + const u8 *records, *bitmap; + u64 channels; +}; + struct mt7932_policy { const u8 *mode[MT7932_POLICY_MODES]; const u8 *table[MT7932_POLICY_TABLES]; u16 length[MT7932_POLICY_TABLES]; u8 modes, six_pages; - u64 six_channels; + struct mt7932_6g_table six_class[3]; + u64 six_channels, legacy_channels; const u8 *six_page[MT7932_POLICY_6G_PAGES]; u16 six_length[MT7932_POLICY_6G_PAGES]; u8 six_cid[MT7932_POLICY_6G_PAGES]; @@ -56,6 +65,7 @@ static inline int mt7932_policy_6g_parse(struct mt7932_policy *out, memcmp(data + 16, alpha2, 2) || data[18] != 1 || memchr_inv(data + 19, 0, 5)) return -EINVAL; out->six_channels = get_unaligned_le64(data + 8); + out->legacy_channels = out->six_channels; if (!out->six_channels || (out->six_channels & ~GENMASK_ULL(58, 0))) return -EINVAL; out->six_pages = data[6]; @@ -106,6 +116,113 @@ static inline int mt7932_policy_6g_parse(struct mt7932_policy *out, return at == size && out->assoc_power[0] && out->assoc_power[1] ? 0 : -EINVAL; } +/* Version 4 binds the unchanged v3 class data to the SPEC's CID5d ABI. + * Version 3, binding revision zero, remains an unbound archive and is refused. + */ +static inline int mt7932_policy_classes_parse(struct mt7932_policy *out, + const u8 *data, size_t size, + const u8 alpha2[2]) +{ + size_t at = 16; + unsigned int i, previous = 0; + + if (size < at || memcmp(data, "6GCL", 4) || data[4] != 1 || !data[5] || + data[5] > 2 || data[6] != 1 || data[7] || + get_unaligned_le32(data + 8) != size || + get_unaligned_le32(data + 12)) + return -EINVAL; + for (i = 0; i < data[5]; i++) { + const u8 *section = data + at, *record; + struct mt7932_6g_table *table; + unsigned int id, mode, j, channel = 1; + u64 mask; + + if (size - at < 56) + return -EINVAL; + id = section[0]; + if ((id != 1 && id != 2) || id <= previous || + section[1] != id - 1 || section[2] != 1 || + section[3] != MT7932_6G_RECORDS || + get_unaligned_le32(section + 4) != + 56 + MT7932_6G_RECORDS * 122 || + size - at < 56 + MT7932_6G_RECORDS * 122 || + memcmp(section + 16, alpha2, 2) || + memchr_inv(section + 18, 0, 6)) + return -EINVAL; + mask = get_unaligned_le64(section + 8); + if (!mask || (mask & ~GENMASK_ULL(58, 0))) + return -EINVAL; + record = section + 56; + for (j = 0; j < MT7932_6G_RECORDS; j++, record += 122) { + unsigned int k; + bool power = false; + + /* Authenticated matrix order: channel 2, then primary and + * bonded centers up to 160 MHz (excluding channel 231). + */ + if (record[0] != (j ? channel : 2)) + return -EINVAL; + if (j) { + channel += 2; + while (channel == 31 || channel == 63 || + channel == 95 || channel == 127 || + channel == 159 || channel == 191 || + channel == 223 || channel == 231) + channel += 2; + } + if (record[0] % 4 != 1 || + !(mask & BIT_ULL((record[0] - 1) / 4))) + continue; + for (k = 1; k < 122; k++) + power |= record[k] != 0xc4 && record[k] != 0x7f; + if (!power) + return -EINVAL; + } + mode = id == 1 ? MT7932_6G_VLP : MT7932_6G_LPI; + table = &out->six_class[mode]; + table->channels = mask; + table->bitmap = section + 24; + table->records = section + 56; + out->six_channels |= mask; + previous = id; + at += 56 + MT7932_6G_RECORDS * 122; + } + return at == size && out->six_class[MT7932_6G_LPI].records ? 0 : + -EINVAL; +} + +/* Normal infrastructure publication: submode zero, no invented class bits. + * Class IDs and data must agree; callers cannot relabel VLP data as LPI. + */ +static inline int mt7932_6g_class_page(u8 *body, size_t capacity, + const struct mt7932_policy *policy, + unsigned int mode, const u8 alpha2[2], + unsigned int first) +{ + const struct mt7932_6g_table *table; + unsigned int count, length; + + if (mode != MT7932_6G_LPI && mode != MT7932_6G_VLP) + return -EOPNOTSUPP; + table = &policy->six_class[mode]; + if (!table->records || first >= MT7932_6G_RECORDS || first % 8) + return -EINVAL; + count = min(8U, MT7932_6G_RECORDS - first); + length = 44 + count * 122; + if (capacity < length) + return -ENOSPC; + memset(body, 0, 44); + body[1] = 0x0a | (mode << 4); + put_unaligned_le16(length, body + 2); + body[4] = count; + body[5] = 3; + body[6] = first + count == MT7932_6G_RECORDS; + memcpy(body + 8, alpha2, 2); + memcpy(body + 12, table->bitmap, 32); + memcpy(body + 44, table->records + first * 122, count * 122); + return length; +} + static inline int mt7932_policy_parse(struct mt7932_policy *out, const u8 *data, size_t size, const u8 alpha2[2]) @@ -115,11 +232,13 @@ static inline int mt7932_policy_parse(struct mt7932_policy *out, memset(out, 0, sizeof(*out)); version = size >= 6 ? get_unaligned_le16(data + 4) : 0; - if (size < at || size > (version == 2 ? 65536 : 16384) || memcmp(data, "J7RP", 4) || - (version != 1 && version != 2) || memcmp(data + 6, alpha2, 2) || - data[8] < 2 || data[8] > MT7932_POLICY_MODES || - data[9] != MT7932_POLICY_TABLES || data[10] || data[11] || - get_unaligned_le32(data + 12) != size) + if (size < at || + size > (version == 2 || version == 4 ? 65536 : 16384) || + memcmp(data, "J7RP", 4) || + (version != 1 && version != 2 && version != 4) || + memcmp(data + 6, alpha2, 2) || data[8] < 2 || + data[8] > MT7932_POLICY_MODES || data[9] != MT7932_POLICY_TABLES || + data[10] || data[11] || get_unaligned_le32(data + 12) != size) return -EINVAL; out->modes = data[8]; for (i = 0; i < out->modes; i++, at += 328) { @@ -175,6 +294,9 @@ static inline int mt7932_policy_parse(struct mt7932_policy *out, out->length[i] = n; at += n; } + if (version == 4) + return mt7932_policy_classes_parse(out, data + at, size - at, + alpha2); return version == 2 ? mt7932_policy_6g_parse(out, data + at, size - at, alpha2) : at == size ? 0 : -EINVAL; } @@ -294,7 +416,7 @@ static inline unsigned int mt7932_6g_reg_index(unsigned int channel) } static inline struct mt7932_6g_limits -mt7932_6g_limits(const struct mt7932_reg_snapshot *reg) +mt7932_6g_limits_mask(const struct mt7932_reg_snapshot *reg, u64 channels) { struct mt7932_6g_limits limits = { .eirp = 28, .psd = 256 }; unsigned int i, begin = reg->domain[8] + reg->domain[9]; @@ -303,6 +425,8 @@ mt7932_6g_limits(const struct mt7932_reg_snapshot *reg) const u8 *entry = reg->domain + 12 + i * 8; unsigned int index = mt7932_6g_reg_index(get_unaligned_le16(entry)); + if (!(channels & BIT_ULL((get_unaligned_le16(entry) - 1) / 4))) + continue; limits.eirp = min_t(int, limits.eirp, 2 * reg->power[index]); if (get_unaligned_le32(entry + 4) & IEEE80211_CHAN_PSD) limits.psd = min_t(int, limits.psd, 2 * reg->psd[index]); @@ -310,6 +434,12 @@ mt7932_6g_limits(const struct mt7932_reg_snapshot *reg) return limits; } +static inline struct mt7932_6g_limits +mt7932_6g_limits(const struct mt7932_reg_snapshot *reg) +{ + return mt7932_6g_limits_mask(reg, ~0ULL); +} + /* Remove the channels the country package forbids from a CID0f domain. */ static inline void mt7932_policy_filter(struct mt7932_reg_snapshot *reg, const struct mt7932_policy *policy) @@ -344,18 +474,21 @@ static inline void mt7932_policy_filter(struct mt7932_reg_snapshot *reg, */ enum mt7932_policy_source { MT7932_POLICY_NONE, - MT7932_POLICY_V2, /* policy-v2/.bin */ - MT7932_POLICY_V1, /* policy/.bin */ - MT7932_POLICY_WORLD, /* policy/world-XZ.bin */ + MT7932_POLICY_V4, /* policy-v4/.bin */ + MT7932_POLICY_V2, /* policy-v2/.bin */ + MT7932_POLICY_V1, /* policy/.bin */ + MT7932_POLICY_WORLD, /* policy/world-XZ.bin */ }; -#define MT7932_POLICY_CANDIDATES 3 +#define MT7932_POLICY_CANDIDATES 4 #define MT7932_POLICY_PATH_MAX 48 struct mt7932_policy_file { const void *handle; const u8 *data; size_t size; + /* Optional v4 LPI sidecar; the primary remains the original v2. */ + const void *class_handle; }; /* get() returns 0 with @file filled in, or an error; put() releases it. */ @@ -372,6 +505,8 @@ struct mt7932_policy_choice { u16 version; u32 size; u8 sha256[SHA256_DIGEST_SIZE]; + u8 class_sha256[SHA256_DIGEST_SIZE]; + bool class_valid; /* 6 GHz primaries left in the firmware domain. */ u8 six_primaries; /* Why the country's own packages were not used: -ENOENT, -EINVAL, or diff --git a/drivers/net/wireless/mediatek/mt7932/six.h b/drivers/net/wireless/mediatek/mt7932/six.h new file mode 100644 index 0000000000000..1d2a4f68f354e --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/six.h @@ -0,0 +1,298 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_SIX_H +#define MT7932_SIX_H + +#include +#include + +#include "channels.h" + +/* Firmware class numbering, distinct from the J7RP archive class IDs. */ +enum mt7932_6g_class { + MT7932_6G_SP, + MT7932_6G_LPI, + MT7932_6G_VLP, +}; + +struct mt7932_6g_ap { + u8 mode, primary, center, bandwidth; + u8 count; + s8 limit[4]; /* half dBm EIRP, 0x7f means no applicable limit */ +}; + +static inline int mt7932_6g_class(u8 regulatory, u32 flags) +{ + if (flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_IR | + IEEE80211_CHAN_NO_HE | IEEE80211_CHAN_NO_20MHZ | + IEEE80211_CHAN_RADAR)) + return -EACCES; + switch (regulatory) { + case 0: + return MT7932_6G_LPI; + case 1: + /* SP requires AFC; there is no SP-to-LPI fallback. */ + return -EOPNOTSUPP; + case 2: + return flags & IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT ? + -EACCES : + MT7932_6G_VLP; + default: + /* The static mapping's catch-all is not class authorization. */ + return -EOPNOTSUPP; + } +} + +/* A conservative reduction across every advertised 20 MHz PSD value also + * bounds each narrower allocation, without assuming per-subchannel ordering. + * floor(2 * 10log10(BW_MHz)) always rounds toward the restrictive limit. + */ +static inline int mt7932_6g_tpe(struct mt7932_6g_ap *ap, const u8 *data, + u8 length) +{ + static const u8 psd_gain[] = { 26, 32, 38, 44 }; + unsigned int count, interpretation, category, n, i; + int psd = 127; + + if (!ieee80211_valid_tpe_element(data, length)) + return -EINVAL; + count = FIELD_GET(IEEE80211_TX_PWR_ENV_INFO_COUNT, data[0]); + interpretation = + FIELD_GET(IEEE80211_TX_PWR_ENV_INFO_INTERPRET, data[0]); + category = FIELD_GET(IEEE80211_TX_PWR_ENV_INFO_CATEGORY, data[0]); + /* A station is a default client, never a subordinate AP. */ + if (category != IEEE80211_TPE_CAT_6GHZ_DEFAULT) + return 0; + if (interpretation == IEEE80211_TPE_LOCAL_EIRP || + interpretation == IEEE80211_TPE_REG_CLIENT_EIRP) { + n = min(count + 1, 4U); + for (i = 0; i < n; i++) + ap->limit[i] = min_t(s8, ap->limit[i], data[i + 1]); + ap->count = max_t(u8, ap->count, n); + return 0; + } + if (interpretation != IEEE80211_TPE_LOCAL_EIRP_PSD && + interpretation != IEEE80211_TPE_REG_CLIENT_EIRP_PSD) + return -EOPNOTSUPP; + n = count ? BIT(count - 1) : 1; + /* Extended/320 MHz PSD layouts are outside the established binding. */ + if (count > 4 || length != n + 1) + return -EOPNOTSUPP; + for (i = 0; i < n; i++) { + /* A PSD byte of -128 forbids that 20 MHz subchannel. There + * is no supported puncturing path that could omit it here. + */ + if (data[i + 1] == 0x80) + return -EACCES; + psd = min(psd, (int)(s8)data[i + 1]); + } + for (i = 0; i < 4; i++) { + int limit = psd == 127 ? 127 : min(psd + psd_gain[i], 126); + + ap->limit[i] = min_t(int, ap->limit[i], limit); + } + ap->count = 4; + return 0; +} + +/* Full TLV validation prevents a truncated or duplicate HE element from + * becoming authority for LPI. TPE never selects the class. + */ +static inline int mt7932_6g_ap_parse(struct mt7932_6g_ap *ap, const u8 *ies, + size_t length, u8 primary, u32 flags) +{ + const struct element *elem; + bool operation = false; + size_t parsed = 0; + int mode, ret; + + memset(ap, 0, sizeof(*ap)); + memset(ap->limit, 127, sizeof(ap->limit)); + for_each_element(elem, ies, length) { + parsed += elem->datalen + 2; + if (elem->id == WLAN_EID_TX_POWER_ENVELOPE) { + ret = mt7932_6g_tpe(ap, elem->data, elem->datalen); + if (ret) + return ret; + } + if (elem->id == WLAN_EID_EXTENSION && elem->datalen && + elem->data[0] == WLAN_EID_EXT_HE_OPERATION) { + const struct ieee80211_he_operation *he; + const struct ieee80211_he_6ghz_oper *six; + u8 width, center, regulatory; + + if (operation || elem->datalen < 1 + sizeof(*he) || + elem->datalen != + ieee80211_he_oper_size(elem->data + 1)) + return -EINVAL; + he = (const void *)(elem->data + 1); + six = ieee80211_he_6ghz_oper(he); + if (!six || six->primary != primary || + !mt7932_channel_band(NL80211_BAND_6GHZ, primary)) + return -EINVAL; + regulatory = + FIELD_GET(IEEE80211_HE_6GHZ_OPER_CTRL_REG_INFO, + six->control); + mode = mt7932_6g_class(regulatory, flags); + if (mode < 0) + return mode; + width = FIELD_GET(IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH, + six->control); + if (width > 2 || six->ccfs1) + return -EOPNOTSUPP; + center = width ? mt7932_channel_6g_center(primary, + width == 2) : + primary; + if (!center || six->ccfs0 != center) + return -EINVAL; + ap->mode = mode; + ap->primary = primary; + ap->center = center; + ap->bandwidth = width; + operation = true; + } + } + if (parsed != length || !operation || !ap->count) + return -EINVAL; + /* Every bandwidth we may use needs an advertised bound. */ + return ap->count > ap->bandwidth ? 0 : -EOPNOTSUPP; +} + +/* Return 1 for the existing v2 VLP association sequence. Its admission and + * commands never depended on TPE, including finite TPE or AP-wide 160 MHz. + * LPI and newly available VLP channels use the strict class/TPE path. + */ +static inline int mt7932_6g_connect_ap(struct mt7932_6g_ap *ap, const u8 *ies, + size_t length, u8 primary, u32 flags, + u64 legacy_channels) +{ + const struct element *elem; + const struct ieee80211_he_operation *he; + const struct ieee80211_he_6ghz_oper *six; + u8 regulatory; + + elem = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_OPERATION, ies, length); + if (!elem || elem->datalen < 1 + sizeof(*he) || + elem->datalen != ieee80211_he_oper_size(elem->data + 1)) + return -EINVAL; + he = (const void *)(elem->data + 1); + six = ieee80211_he_6ghz_oper(he); + if (!six || six->primary != primary || + !mt7932_channel_band(NL80211_BAND_6GHZ, primary)) + return -EINVAL; + regulatory = FIELD_GET(IEEE80211_HE_6GHZ_OPER_CTRL_REG_INFO, six->control); + if (regulatory == 2 && + (legacy_channels & BIT_ULL((primary - 1) / 4))) + return 1; + return mt7932_6g_ap_parse(ap, ies, length, primary, flags); +} + +/* Immutable J700 board properties, in signed half-dB ticks, from the + * t-3573 antenna-gain SPEC sections 1 and 3. The probe is J700-only. + * The bandwidth map deliberately differs from a frequency-only lookup. + */ +static inline int mt7932_j700_6g_gain(u8 primary, unsigned int bw, s8 gain[3]) +{ + static const s8 groups[6][3] = { + { 16, 16, 10 }, { 14, 14, 8 }, { 15, 16, 10 }, + { 15, 15, 9 }, { 16, 16, 10 }, { 15, 16, 10 }, + }; + unsigned int group; + + if (bw > 3 || (primary != 2 && + (primary < 1 || primary > 233 || (primary - 1) % 4))) + return -EINVAL; + if ((primary == 2 && bw) || (primary == 233 && bw) || + (primary >= 225 && bw >= 2)) + return -EOPNOTSUPP; + if (primary == 2 || primary <= 29) + group = 0; + else if (primary <= 61) + group = 1; + else if (primary <= 93) + group = 2; + else if (primary <= 109) + group = 3; + else if (primary == 113) + group = bw == 2 ? 4 : 3; + else if (primary == 117) + group = bw == 1 || bw == 3 ? 3 : 4; + else if (primary <= 125) + group = bw == 3 ? 3 : 4; + else if (primary <= 181) + group = 4; + else if (primary == 185) + group = bw == 1 ? 5 : 4; + else if (primary == 189) + group = bw < 2 ? 5 : 4; + else + group = 5; + memcpy(gain, groups[group], 3); + return 0; +} + +static inline int mt7932_j700_6g_gains(const struct mt7932_6g_ap *ap, + s8 gain[3][4]) +{ + unsigned int bw, mode; + + memset(gain, 0, 12); + for (bw = 0; bw < ap->count && bw < 4; bw++) { + s8 values[3]; + int ret; + + if (ap->limit[bw] == 127) + continue; + ret = mt7932_j700_6g_gain(ap->primary, bw, values); + if (ret) + return ret; + for (mode = 0; mode < 3; mode++) + gain[mode][bw] = values[mode]; + } + return 0; +} + +/* Gains use the same half-dB units as the AP limit. A NULL gain source is + * usable only for unconstrained limits: zero gain must never be guessed. + */ +static inline int mt7932_6g_tpe_body(u8 *body, size_t capacity, + const struct mt7932_6g_ap *ap, + const s8 gain[3][4]) +{ + unsigned int group, i; + + if (capacity < 60 || !ap->count || ap->count > 4 || + (ap->mode != MT7932_6G_LPI && ap->mode != MT7932_6G_VLP) || + !mt7932_channel_band(NL80211_BAND_6GHZ, ap->primary)) + return -EINVAL; + for (i = 0; i < ap->count; i++) + if (ap->limit[i] != 127 && !gain) + return -EOPNOTSUPP; + memset(body, 0, 60); + body[1] = 2; + put_unaligned_le16(60, body + 2); + body[4] = 1; + body[5] = 3; + body[6] = 1; + body[7] = 3; + body[44] = 1; + body[45] = 3; + body[46] = ap->primary; + body[47] = ap->count; + memset(body + 48, 127, 12); + for (group = 0; group < 3; group++) { + for (i = 0; i < ap->count; i++) { + int limit; + + if (ap->limit[i] == 127) + continue; + limit = ap->limit[i] - gain[group][i]; + /* Saturating upward at -128 would relax an unrepresentable cap. */ + if (limit < -128 || limit > 126) + return -ERANGE; + body[48 + group * 4 + i] = limit; + } + } + return 60; +} + +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 7962a5fd95a54..62aedd18c7024 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -149,7 +149,8 @@ static noinline_for_stack int mt_publish_policy(struct mt7932 *m, { u8 dbdc[36] = {0,0,0,0,2,0,36}; unsigned int i; - struct mt7932_6g_limits limits = mt7932_6g_limits(reg); + struct mt7932_6g_limits limits = + mt7932_6g_limits_mask(reg, policy->legacy_channels); u8 body[1020]; unsigned int j; int ret; @@ -180,6 +181,7 @@ static noinline_for_stack int mt_publish_policy(struct mt7932 *m, return ret; } m->six_power_valid = false; + m->six_legacy_channels = policy->legacy_channels; for (i = 0; i < policy->six_pages; i++) { memcpy(body, policy->six_page[i], policy->six_length[i]); if (policy->six_cid[i] == 0x5d) @@ -192,7 +194,22 @@ static noinline_for_stack int mt_publish_policy(struct mt7932 *m, if (ret) return ret; } - if (policy->six_pages) + if (m->six_class_dirty) { + struct mt7932_6g_ap clear = m->six_ap; + + /* The established all-0x7f command removes the prior AP ceiling. + * Publish VLP's country tables first, while RF admission is fenced. + */ + memset(clear.limit, 127, sizeof(clear.limit)); + ret = mt7932_6g_tpe_body(body, sizeof(body), &clear, NULL); + if (ret < 0) + return ret; + ret = mt_request(m, 0x5d, true, true, false, body, 60); + if (ret) + return ret; + m->six_class_dirty = false; + } + if (policy->six_channels) m->six_power_valid = true; /* Native SETs have no response/atomic policy-commit event. Command * submission is not misrepresented as firmware readback or an ACK. @@ -371,7 +388,132 @@ static int mt_policy_get(void *ctx, const char *path, struct mt7932_policy_file static void mt_policy_put(void *ctx, struct mt7932_policy_file *file) { release_firmware(file->handle); + release_firmware(file->class_handle); file->handle = NULL; + file->class_handle = NULL; +} + +/* command_mutex held. Reopen and authenticate the exact installed policy, + * then publish the AP's class before EA/40. No active-class readback exists. + */ +int mt_6g_connect_policy(struct mt7932 *m, const struct mt7932_6g_ap *ap) +{ + struct mt7932_policy_file file = {}; + struct mt7932_policy *policy; + struct mt7932_6g_limits limits; + struct mt7932_reg_snapshot *reg; + u8 hash[SHA256_DIGEST_SIZE], *body; + char path[MT7932_POLICY_PATH_MAX]; + u8 tpe[60]; + s8 gain[3][4]; + bool sidecar = m->policy_choice.class_valid; + u64 channels; + unsigned long flags; + unsigned int first, channel, begin, end; + int ret, length; + bool submitted = false; + + lockdep_assert_held(&m->command_mutex); + ret = mt7932_j700_6g_gains(ap, gain); + if (ret) + return ret; + ret = mt7932_6g_tpe_body(tpe, sizeof(tpe), ap, gain); + if (ret < 0) + return ret; + policy = kzalloc_obj(*policy, GFP_KERNEL); + reg = kzalloc_obj(*reg, GFP_KERNEL); + body = kmalloc(1020, GFP_KERNEL); + if (!policy || !reg || !body) { + ret = -ENOMEM; + goto free; + } + mt7932_policy_path(path, sizeof(path), + sidecar ? MT7932_POLICY_V4 : m->policy_choice.source, + m->policy_choice.alpha2); + ret = mt_policy_get(m, path, &file); + if (ret) + goto free; + sha256(file.data, file.size, hash); + if (memcmp(hash, + sidecar ? m->policy_choice.class_sha256 : + m->policy_choice.sha256, + sizeof(hash))) { + ret = -ESTALE; + goto free; + } + ret = mt7932_policy_parse(policy, file.data, file.size, + m->policy_choice.domain); + if (ret) + goto free; + channels = policy->six_class[ap->mode].channels; + begin = ap->center - 2 * (BIT(ap->bandwidth) - 1); + end = ap->center + 2 * (BIT(ap->bandwidth) - 1); + for (channel = begin; channel <= end; channel += 4) { + int frequency = ieee80211_channel_to_frequency(channel, NL80211_BAND_6GHZ); + struct ieee80211_channel *c = ieee80211_get_channel(m->wiphy, frequency); + + if (!c || !(channels & BIT_ULL((channel - 1) / 4)) || + mt7932_6g_class(ap->mode == MT7932_6G_LPI ? 0 : 2, + c->flags) < 0) { + ret = -EACCES; + goto free; + } + } + spin_lock_irqsave(&m->response_lock, flags); + *reg = m->reg_desired; + spin_unlock_irqrestore(&m->response_lock, flags); + limits = mt7932_6g_limits(reg); + if (ap->mode == MT7932_6G_LPI) { + /* The LP table, cfg80211 and AP envelope all remain upper bounds. + * Reusing VLP's fixed 14 dBm ceiling would obscure the LP data. + */ + limits.eirp = 254; + for (channel = 0; channel < MT7932_CHANNELS_6G; channel++) { + unsigned int index = + mt7932_6g_reg_index(1 + 4 * channel); + + if (channels & BIT_ULL(channel)) + limits.eirp = min_t(int, limits.eirp, + 2 * reg->power[index]); + } + } + m->six_legacy_assoc = false; + for (first = 0; first < MT7932_6G_RECORDS; first += 8) { + length = mt7932_6g_class_page(body, 1020, policy, ap->mode, + m->policy_choice.domain, first); + if (length < 0) { + ret = length; + goto free; + } + mt7932_6g_limit_page(body, &limits); + submitted = true; + m->six_class_dirty = true; + ret = mt_request(m, 0x5d, true, true, false, body, length); + if (ret) + goto free; + } + dev_dbg(&m->pdev->dev, "CID5D_CLASS: byte1=%02x band=3 channel=%u\n", + 0x0a | (ap->mode << 4), ap->primary); + ret = mt_request(m, 0x5d, true, true, false, tpe, sizeof(tpe)); + if (!ret) + dev_dbg(&m->pdev->dev, + "CID5D_TPE: length=60 band=3 channel=%u count=%u\n", + ap->primary, ap->count); +free: + /* No atomic SET/rollback exists. A partial publication closes RF + * admission; a checked firmware reset is required before reuse. + */ + if (ret && submitted) { + spin_lock_irqsave(&m->response_lock, flags); + mt_rf_fail_locked(m, ret); + spin_unlock_irqrestore(&m->response_lock, flags); + } + if (file.handle) + mt_policy_put(m, &file); + kfree(body); + kfree(reg); + kfree(policy); + return ret; } static void mt_policy_report(struct mt7932 *m, const struct mt7932_policy_choice *choice, @@ -402,19 +544,26 @@ static void mt_startup_work(struct work_struct *work) }; struct mt7932_policy_choice choice; struct mt7932_policy_file file; - struct mt7932_reg_snapshot reg; + struct mt7932_reg_snapshot *reg; struct mt7932_policy *policy; unsigned long flags; u32 generation; bool submitted; int ret; + reg = kzalloc_obj(*reg, GFP_KERNEL); + if (!reg) { + dev_err(&m->pdev->dev, + "REGULATORY_BLOCKED: snapshot allocation failed\n"); + return; + } for (;;) { spin_lock_irqsave(&m->response_lock, flags); generation = m->reg_generation; - reg = m->reg_desired; + *reg = m->reg_desired; if (m->stopping || generation == m->reg_attempted) { spin_unlock_irqrestore(&m->response_lock, flags); + kfree(reg); return; } m->reg_attempted = generation; @@ -435,12 +584,12 @@ static void mt_startup_work(struct work_struct *work) flush_work(&m->scan_finish_work); flush_work(&m->cal_work); } - ret = reg.error; + ret = reg->error; if (READ_ONCE(m->retired_scan_seq)) ret = -EBUSY; memset(&file, 0, sizeof(file)); memset(&choice, 0, sizeof(choice)); - memcpy(choice.alpha2, reg.alpha2, 2); + memcpy(choice.alpha2, reg->alpha2, 2); policy = NULL; submitted = false; if (READ_ONCE(m->policy_failed) || READ_ONCE(m->link_failed) || @@ -448,8 +597,10 @@ static void mt_startup_work(struct work_struct *work) ret = -EIO; if (!ret) { policy = kzalloc_obj(*policy, GFP_KERNEL); - ret = policy ? mt7932_policy_select(&loader, ®, policy, &file, &choice) : - -ENOMEM; + ret = policy ? + mt7932_policy_select(&loader, reg, policy, + &file, &choice) : + -ENOMEM; mt_policy_report(m, &choice, ret); if (!ret) mt_policy_disable(m, policy, generation); @@ -459,7 +610,7 @@ static void mt_startup_work(struct work_struct *work) goto next; if (!ret) { submitted = true; - ret = mt_publish_policy(m, ®, policy); + ret = mt_publish_policy(m, reg, policy); if (!ret && !m->startup_started) ret = mt_startup_once(m); /* A partial SET sequence is not a rollback. Only missing or @@ -483,13 +634,15 @@ static void mt_startup_work(struct work_struct *work) m->policy_choice = choice; } if (ret) - dev_warn_ratelimited(&m->pdev->dev, "REGULATORY_BLOCKED: %c%c generation=%u error=%d recovery-required=%u\n", - reg.alpha2[0], reg.alpha2[1], generation, ret, - m->policy_failed || m->link_failed); + dev_warn_ratelimited(&m->pdev->dev, + "REGULATORY_BLOCKED: %c%c generation=%u error=%d recovery-required=%u\n", + reg->alpha2[0], reg->alpha2[1], generation, ret, + m->policy_failed || m->link_failed); else - dev_dbg(&m->pdev->dev, "REGULATORY_READY: %c%c firmware=%c%c generation=%u current=%u\n", - reg.alpha2[0], reg.alpha2[1], reg.domain[0], reg.domain[1], - generation, !m->reg_pending); + dev_dbg(&m->pdev->dev, + "REGULATORY_READY: %c%c firmware=%c%c generation=%u current=%u\n", + reg->alpha2[0], reg->alpha2[1], reg->domain[0], + reg->domain[1], generation, !m->reg_pending); spin_unlock_irqrestore(&m->response_lock, flags); next: mutex_unlock(&m->command_mutex); @@ -542,8 +695,6 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request rule = freq_reg_info(m->wiphy, MHZ_TO_KHZ(channel->center_freq)); if (!IS_ERR(rule) && !(rule->flags & NL80211_RRF_NO_IR)) flags &= ~IEEE80211_CHAN_NO_IR; - if (flags & IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT) - flags |= IEEE80211_CHAN_DISABLED; } reg.power[i] = channel->max_power; @@ -611,6 +762,7 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request return; } m->reg_desired = reg; + m->six_ap_valid = false; m->reg_deferral = false; m->reg_generation++; /* The next package pass records its own set. */ @@ -655,9 +807,13 @@ static const struct cfg80211_ops mt_cfg_ops = { int mt_register_regulatory_gate(struct mt7932 *m) { static const int rates[] = {10,20,55,110,60,90,120,180,240,360,480,540}; - static const u32 ciphers[] = { WLAN_CIPHER_SUITE_CCMP }; + static const u32 ciphers[] = { WLAN_CIPHER_SUITE_CCMP, + WLAN_CIPHER_SUITE_AES_CMAC }; static const u32 akms[] = { - WLAN_AKM_SUITE_PSK, WLAN_AKM_SUITE_SAE, WLAN_AKM_SUITE_OWE, + WLAN_AKM_SUITE_PSK, + WLAN_AKM_SUITE_PSK_SHA256, + WLAN_AKM_SUITE_SAE, + WLAN_AKM_SUITE_OWE, }; unsigned int i; int ret; @@ -811,9 +967,8 @@ static const char *mt7932_policy_fallback_reason(int fallback) } static const char *const mt7932_policy_names[] = { - [MT7932_POLICY_NONE] = "none", - [MT7932_POLICY_V2] = "policy-v2", - [MT7932_POLICY_V1] = "policy", + [MT7932_POLICY_NONE] = "none", [MT7932_POLICY_V4] = "policy-v4", + [MT7932_POLICY_V2] = "policy-v2", [MT7932_POLICY_V1] = "policy", [MT7932_POLICY_WORLD] = "world", }; @@ -824,6 +979,7 @@ unsigned int mt7932_policy_candidates(const u8 alpha2[2], /* 00 and the other numeric codes have no country package. */ if (mt7932_policy_country(alpha2)) { + sources[n++] = MT7932_POLICY_V4; sources[n++] = MT7932_POLICY_V2; sources[n++] = MT7932_POLICY_V1; } @@ -835,6 +991,10 @@ void mt7932_policy_path(char *buf, size_t len, enum mt7932_policy_source source, const u8 alpha2[2]) { switch (source) { + case MT7932_POLICY_V4: + scnprintf(buf, len, "mediatek/mt7932/policy-v4/%c%c.bin", + alpha2[0], alpha2[1]); + break; case MT7932_POLICY_V2: scnprintf(buf, len, "mediatek/mt7932/policy-v2/%c%c.bin", alpha2[0], alpha2[1]); break; @@ -860,8 +1020,57 @@ static int mt7932_policy_check_version(enum mt7932_policy_source source, const u8 *data, size_t size) { unsigned int version = size >= 6 ? get_unaligned_le16(data + 4) : 0; + unsigned int expected = 1; - return version == (source == MT7932_POLICY_V2 ? 2 : 1) ? 0 : -EINVAL; + if (source == MT7932_POLICY_V4) + expected = 4; + else if (source == MT7932_POLICY_V2) + expected = 2; + return version == expected ? 0 : -EINVAL; +} + +/* Prefer the original v2 VLP publication whenever it is installed. Retain + * v4's LP view as a sidecar, rather than replacing the working VLP pages or + * changing the two post-queue association SETs. Both files stay pinned until + * publication and both hashes are checked again before class selection. + */ +static int mt7932_policy_keep_vlp(const struct mt7932_policy_loader *loader, + struct mt7932_policy *policy, + struct mt7932_policy_file *file, + struct mt7932_policy_choice *choice) +{ + struct mt7932_policy_file legacy = {}; + struct mt7932_policy *old; + struct mt7932_6g_table lpi = policy->six_class[MT7932_6G_LPI]; + struct mt7932_6g_table vlp = policy->six_class[MT7932_6G_VLP]; + char path[MT7932_POLICY_PATH_MAX]; + + old = kzalloc_obj(*old, GFP_KERNEL); + if (!old) + return -ENOMEM; + mt7932_policy_path(path, sizeof(path), MT7932_POLICY_V2, + choice->alpha2); + if (loader->get(loader->ctx, path, &legacy)) + goto free; + if (mt7932_policy_check_version(MT7932_POLICY_V2, legacy.data, + legacy.size) || + mt7932_policy_parse(old, legacy.data, legacy.size, + choice->alpha2)) { + loader->put(loader->ctx, &legacy); + goto free; + } + sha256(file->data, file->size, choice->class_sha256); + choice->class_valid = true; + legacy.class_handle = file->handle; + *file = legacy; + *policy = *old; + policy->six_class[MT7932_6G_LPI] = lpi; + policy->six_class[MT7932_6G_VLP] = vlp; + policy->six_channels |= lpi.channels | vlp.channels; + choice->source = MT7932_POLICY_V2; +free: + kfree(old); + return 0; } int mt7932_policy_select(const struct mt7932_policy_loader *loader, @@ -908,6 +1117,15 @@ int mt7932_policy_select(const struct mt7932_policy_loader *loader, if (ret) return ret; + if (choice->source == MT7932_POLICY_V4) { + ret = mt7932_policy_keep_vlp(loader, policy, file, choice); + if (ret) { + loader->put(loader->ctx, file); + memset(file, 0, sizeof(*file)); + return ret; + } + } + /* * A country without a usable package of its own keeps its cfg80211 * identity and channel list; the firmware gets the world profile and @@ -965,6 +1183,9 @@ static int mt_policy_show(struct seq_file *s, void *unused) seq_printf(s, "size: %u\n", c->size); seq_printf(s, "sha256: %8phN\n", c->sha256); seq_printf(s, "6ghz_primaries: %u\n", c->six_primaries); + if (c->class_valid) + seq_printf(s, "class_source: policy-v4\nclass_sha256: %*phN\n", + SHA256_DIGEST_SIZE, c->class_sha256); } seq_printf(s, "fallback: %s\n", mt7932_policy_fallback_reason(c->fallback)); for (i = 0; i < c->n_tried; i++) { diff --git a/tools/asahi/mt7932-association-events-tests.py b/tools/asahi/mt7932-association-events-tests.py index 436ab4d00b75f..7ac7ef7709e08 100644 --- a/tools/asahi/mt7932-association-events-tests.py +++ b/tools/asahi/mt7932-association-events-tests.py @@ -85,7 +85,7 @@ def function(source, name): bool connecting,connected,disconnecting,disconnect_seen,disconnect_event_received; bool disconnect_local,connect_open,assoc_request_seen,peer_valid,link_failed,stopping,rf_ready; u8 connect_channel,connect_center,peer_wtbl,smart_version,waiting; - enum nl80211_band connect_band; u8 six_assoc_power[2][1020]; + enum nl80211_band connect_band; u8 six_assoc_power[2][1020]; bool six_legacy_assoc; int waiting_eid; struct cfg80211_chan_def connect_chandef; struct test_band band2,band5; diff --git a/tools/asahi/mt7932-policy-classes-tests.py b/tools/asahi/mt7932-policy-classes-tests.py new file mode 100644 index 0000000000000..f4fc6f021c1d6 --- /dev/null +++ b/tools/asahi/mt7932-policy-classes-tests.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Synthetic interoperability and rejection tests for unbound class archives.""" +import contextlib +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import struct +import tempfile +import unittest +from unittest import mock + + +def load(name, file): + spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(file)) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +gen = load('classes', 'mt7932-policy-classes.py') +fixture = load('fixtures', 'mt7932-policy-6ghz-tests.py') + + +class ClassTests(unittest.TestCase): + def package(self, names=('VLP', 'LP'), mode='1/2/3'): + powers = {'ZZ,' + name: fixture.groups() for name in names} + return gen.archive(fixture.legacy('ZZ'), 'ZZ', fixture.CHANNELS, + [mode] * 110, powers) + + def test_independent_class_masks_and_layout(self): + powers = {'ZZ,LP': fixture.groups(), 'ZZ,VLP': fixture.groups()} + legal = ['1'] * 110 + legal[fixture.CHANNELS.index(5)] = '2' + blob = gen.archive(fixture.legacy('ZZ'), 'ZZ', fixture.CHANNELS, legal, powers) + masks = gen.parse_archive(blob, 'ZZ') + self.assertEqual(masks[1], 2) + self.assertEqual(masks[2], ((1 << 59) - 1) ^ 2) + prefix = len(fixture.legacy('ZZ')) + self.assertEqual(len(blob), prefix + 16 + 2 * (24 + 32 + 110 * 122)) + self.assertEqual(blob[prefix:prefix + 8], b'6GCL\x01\x02\0\0') + self.assertEqual(blob[prefix + 16:prefix + 20], bytes((1, 0, 1, 110))) + self.assertEqual(blob[prefix + 16 + 13476:prefix + 20 + 13476], bytes((2, 1, 1, 110))) + restored = bytearray(blob[:prefix]) + struct.pack_into(' 65536 or data[:4] != b'J7RP' or ( + struct.unpack_from(' len(data): + raise ValueError('truncated legacy table') + at += struct.unpack_from(' len(data) - at + or not 0 < mask < (1 << 59) or alpha2 != country.encode('ascii') or any(reserved)): + raise ValueError('class header') + start = at + 56 + channels = [data[start + index * 122] for index in range(records)] + expected = [2] + [ch for ch in range(1, 234, 2) + if six.power.channel_width_class(3, ch) <= 3 and ch != 231] + if channels != expected: + raise ValueError('class channel order') + for bit in range(59): + if mask & (1 << bit): + index = channels.index(1 + 4 * bit) + values = data[start + index * 122 + 1:start + (index + 1) * 122] + if all(value in (0xc4, 0x7f) for value in values): + raise ValueError('primary has no power data') + result[cid] = mask + previous = cid + at += length + if at != len(data) or 2 not in result: + raise ValueError('trailing data or missing LPI') + return result + + +def bind(archive_data, country): + """v4: unchanged class data, binding revision 1 (SPEC 2026-10-31).""" + parse_archive(archive_data, country) + data = bytearray(archive_data) + at = 16 + data[8] * 328 + for _ in range(13): + at += struct.unpack_from(' Date: Sat, 31 Oct 2026 06:11:28 +1100 Subject: [PATCH 1444/1579] selftests: apple: cover DRAM reporting in the PMP graph harness The report probe now parses DRAM metadata and registers a sysfs group, so the graph harness needs typed boundaries for those operations. Extract the production DRAM structure, readers and group, and exercise the unchanged probe with absent metadata, valid metadata, parser failures and group registration failure. Check all six cached read-only attributes and retain the original startup, child-seeding and supplier checks. Add seven DRAM mutation controls alongside the existing fourteen controls. Keep --ref usable against older report sources while requiring the DRAM hook in the current source. The firmware-property parser remains covered by its KUnit suite; the host parser boundary checks the device and output arguments and injects success and error results for probe integration. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/t6030-pmp-graph-tests.c | 138 ++++++++++++++++++++++++++- tools/asahi/t6030-pmp-graph-tests.py | 35 ++++++- 2 files changed, 169 insertions(+), 4 deletions(-) diff --git a/tools/asahi/t6030-pmp-graph-tests.c b/tools/asahi/t6030-pmp-graph-tests.c index 9f018ee94d628..e9ed89ca13e92 100644 --- a/tools/asahi/t6030-pmp-graph-tests.c +++ b/tools/asahi/t6030-pmp-graph-tests.c @@ -7,10 +7,11 @@ #include #include #include +#include #include #include #include -typedef uint64_t u64; +typedef unsigned long long u64; typedef uint32_t u32; typedef uint16_t u16; typedef uint8_t u8; @@ -55,6 +56,34 @@ struct device { struct device *parent; void *data; }; +#include "dram-type.h" +#if HOST_HAS_DRAM +struct fwnode_handle { int unused; }; +struct attribute { + const char *name; + unsigned int mode; +}; +struct device_attribute { + struct attribute attr; + ssize_t (*show)(struct device *, struct device_attribute *, char *); +}; +struct attribute_group { + const char *name; + struct attribute **attrs; +}; +#define DEVICE_ATTR_RO(n) struct device_attribute dev_attr_##n = { { #n, 0444 }, n##_show } +static int sysfs_emit(char *buf, const char *format, ...) + __attribute__((format(printf, 2, 3))); +static int sysfs_emit(char *buf, const char *format, ...) +{ + va_list args; + va_start(args, format); + int ret = vsnprintf(buf, 4096, format, args); + va_end(args); + return ret; +} +static int devm_device_add_group(struct device *, const struct attribute_group *); +#endif struct platform_device { struct device dev; }; @@ -106,6 +135,7 @@ struct delayed_work { }; struct apple_pmp_report { struct device *dev; + struct apple_pmp_dram dram; const struct apple_pmp_report_offsets *offsets; void *base; int lock; @@ -250,6 +280,33 @@ static void rcu_read_unlock(void) static struct list_head apple_pmp_starts, apple_pmp_reports; static struct device_node report_node, pmp_node, entries[4]; static struct platform_device report_dev, entry_devs[4]; +#if HOST_HAS_DRAM +/* Typed parser boundary: the real property parser is tested by pmp-dram KUnit. */ +static struct fwnode_handle report_fwnode; +static int dram_parse_error, dram_group_error; +static unsigned int dram_parse_calls, dram_group_calls; +static const struct attribute_group *registered_group; +static const struct apple_pmp_dram parsed_dram = { + .type = "lpddr5", .width = 256, .state_count = 4, + .max_rate = 6400000000ULL, .peak_bandwidth = 204800000000ULL, + .bandwidth_class = 200000000000ULL, +}; +static struct fwnode_handle *dev_fwnode(struct device *dev) +{ + assert(dev == &report_dev.dev); + return &report_fwnode; +} +static int apple_pmp_dram_parse(const struct fwnode_handle *node, + struct apple_pmp_dram *dram) +{ + struct apple_pmp_report *rep = report_dev.dev.data; + assert(node == &report_fwnode && rep && dram == &rep->dram); + assert(++dram_parse_calls == 1); + if (!dram_parse_error) + *dram = parsed_dram; + return dram_parse_error; +} +#endif static u64 regs[0x14000 / 8]; static u8 ranges[64], devices[17 * 0x7c]; static struct work_struct *pending; @@ -603,6 +660,38 @@ static const struct apple_pmp_start_ops apple_pmp_start_hw_ops = { hw_check, static const struct apple_pmp_start_timing apple_pmp_start_timing = { 2, 0, 2, 2 }; #include "production.h" +#if HOST_HAS_DRAM +static int devm_device_add_group(struct device *dev, const struct attribute_group *group) +{ + assert(dev == &report_dev.dev && group == &apple_pmp_dram_group); + assert(dram_parse_calls == 1 && !dram_parse_error); + assert(++dram_group_calls == 1 && !registered_group); + if (!dram_group_error) + registered_group = group; + return dram_group_error; +} +static void check_dram_attributes(void) +{ + static const char * const names[] = { + "type", "data_bus_width", "state_count", "max_data_rate", + "peak_bandwidth", "bandwidth_class", + }; + static const char * const values[] = { + "lpddr5\n", "256\n", "4\n", "6400000000\n", + "204800000000\n", "200000000000\n", + }; + assert(registered_group && !strcmp(registered_group->name, "dram")); + for (unsigned int i = 0; i < ARRAY_SIZE(names); i++) { + struct attribute *attr = registered_group->attrs[i]; + assert(attr && !strcmp(attr->name, names[i]) && attr->mode == 0444); + struct device_attribute *da = container_of(attr, struct device_attribute, attr); + char buf[4096]; + assert(da->show(&report_dev.dev, da, buf) == (ssize_t)strlen(values[i])); + assert(!strcmp(buf, values[i])); + } + assert(!registered_group->attrs[ARRAY_SIZE(names)]); +} +#endif static const struct apple_pmp_report_offsets * of_device_get_match_data(struct device *d) { @@ -699,6 +788,12 @@ static struct apple_pmp_report *setup(void) hw_binds = hw_ready = true; hw_ack = BIT_ULL(7); enables = commands = 0; +#if HOST_HAS_DRAM + dram_parse_error = -ENODATA; + dram_group_error = 0; + dram_parse_calls = dram_group_calls = 0; + registered_group = NULL; +#endif return NULL; } static int g15_vote(void) @@ -719,6 +814,9 @@ int main(void) setup(); assert(apple_pmp_report_probe(&report_dev) == 0); struct apple_pmp_report *r = report_dev.dev.data; +#if HOST_HAS_DRAM + assert(dram_parse_calls == 1 && !dram_group_calls && !registered_group); +#endif /* The old passive descriptor fails the real DCP entry waiter here. */ if (!r->start) assert(apple_pmp_report_wait_ready(&entries[0], 30000) == 0); @@ -846,6 +944,44 @@ int main(void) assert(g15_vote() == -EINVAL && commands == 0 && !enables); cases++; cleanup(); + +#if HOST_HAS_DRAM + /* Metadata registration leaves the original managed-start graph intact. */ + setup(); + dram_parse_error = 0; + assert(apple_pmp_report_probe(&report_dev) == 0); + assert(dram_parse_calls == 1 && dram_group_calls == 1); + /* Reads use cached values, even if a subsequent parse would fail. */ + dram_parse_error = -EIO; + check_dram_attributes(); + assert(dram_parse_calls == 1 && dram_group_calls == 1); + assert(!enables && !pmp_node.available); + assert(g15_vote() == 0 && enables == 1 && commands == 1); + cases++; + cleanup(); + + /* Errors must propagate before child seeding or startup ownership. */ + const int errors[] = { -EINVAL, -EPROBE_DEFER, -EIO }; + for (unsigned int i = 0; i < ARRAY_SIZE(errors); i++) { + setup(); + dram_parse_error = errors[i]; + assert(apple_pmp_report_probe(&report_dev) == errors[i]); + assert(dram_parse_calls == 1 && !dram_group_calls && !registered_group); + assert(!action_count && !enables && !commands && !pending); + assert(!regs[0x1180 / 8] && !pmp_node.available); + cases++; + cleanup(); + } + setup(); + dram_parse_error = 0; + dram_group_error = -ENOMEM; + assert(apple_pmp_report_probe(&report_dev) == -ENOMEM); + assert(dram_parse_calls == 1 && dram_group_calls == 1 && !registered_group); + assert(!action_count && !enables && !commands && !pending); + assert(!regs[0x1180 / 8] && !pmp_node.available); + cases++; + cleanup(); +#endif printf("PASS: %u actual overlay/probe/entry/DCP7/managed-start/supplier graph cases\n", cases); return 0; diff --git a/tools/asahi/t6030-pmp-graph-tests.py b/tools/asahi/t6030-pmp-graph-tests.py index 6ba26a09c0176..bbe4dd99ea6c0 100644 --- a/tools/asahi/t6030-pmp-graph-tests.py +++ b/tools/asahi/t6030-pmp-graph-tests.py @@ -3,7 +3,9 @@ """Execute the actual report probe -> entry seed -> DCP -> PMP -> G15 graph. The actual overlay supplies the report entries. OF storage, allocation, MMIO, -workqueue execution and the PMP hardware operations are typed host boundaries. +workqueue execution, DRAM parsing, sysfs registration and PMP hardware operations +are typed host boundaries. The DRAM parser itself has firmware-property KUnit +coverage; this suite checks its probe integration and the actual sysfs readers. The production probe, descriptor validation, startup, waiter and supplier/command bodies are extracted verbatim. This is not a firmware or hardware test. """ @@ -51,7 +53,20 @@ def main(): offsets = re.search(r'static const struct apple_pmp_report_offsets apple_pmp_offsets_t6030 = \{.*?\n\};', source, re.S)[0] functions = '\n'.join(extract.function(source, n) for n in names) validation = re.sub(r'^#include .*\n', '', (ROOT / 'drivers/pmdomain/apple/pmp-report-validation.h').read_text(), flags=re.M) - code = offsets + '\n' + validation + '\n' + functions + dram_type = re.search(r'struct apple_pmp_dram \{.*?\n\};', + (ROOT / 'drivers/pmdomain/apple/pmp-dram.h').read_text(), re.S)[0] + has_dram = 'ret = apple_pmp_dram_parse(' in source + assert has_dram or args.ref, 'DRAM probe hook missing from production source' + (args.output / 'dram-type.h').write_text(f'#define HOST_HAS_DRAM {int(has_dram)}\n' + dram_type + '\n') + # Keep the real readers, modes and group wired into the extracted probe. + dram_attrs = source[source.index('static ssize_t type_show('): + source.index('static const struct apple_pmp_report_offsets apple_pmp_offsets_t6030;')] if has_dram else '' + # Kernel sysfs callbacks intentionally leave the attribute argument unused. + # Match the kernel warning policy only around those verbatim callbacks. + dram_attrs = ('#pragma GCC diagnostic push\n' + '#pragma GCC diagnostic ignored "-Wunused-parameter"\n' + + dram_attrs + '#pragma GCC diagnostic pop\n') + code = offsets + '\n' + validation + '\n' + dram_attrs + '\n' + functions # The actual overlay must still describe a halted supplier and disabled DMA/mailbox. assert overlay.count('status = \"disabled\";') >= 3 (args.output / 'production.h').write_text(code) @@ -70,7 +85,21 @@ def main(): print(result.stdout + result.stderr) mutations = [] if args.mutations: - controls = [ + dram_controls = [ + ('no-dram-parse', 'ret = apple_pmp_dram_parse(dev_fwnode(dev), &rep->dram);', + 'ret = -ENODATA; (void)apple_pmp_dram_parse; (void)dev_fwnode;'), + ('reject-absent-dram', 'ret && ret != -ENODATA', 'ret'), + ('ignore-invalid-dram', 'ret && ret != -ENODATA', 'false'), + ('no-dram-group', 'ret = devm_device_add_group(dev, &apple_pmp_dram_group);', + 'ret = 0; (void)devm_device_add_group; (void)apple_pmp_dram_group;'), + ('ignore-dram-group-error', 'ret = devm_device_add_group(dev, &apple_pmp_dram_group);', + '(void)devm_device_add_group(dev, &apple_pmp_dram_group); ret = 0;'), + ('dram-group-without-metadata', 'if (!ret) {\n\t\tret = devm_device_add_group', + 'if (ret == -ENODATA) {\n\t\tret = devm_device_add_group'), + ('dram-wrong-rate-reader', 'APPLE_PMP_DRAM_ATTR(max_data_rate, max_rate,', + 'APPLE_PMP_DRAM_ATTR(max_data_rate, peak_bandwidth,'), + ] + controls = (dram_controls if has_dram else []) + [ ('no-managed-owner', '.starts_pmp = IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK)', '.starts_pmp = false'), ('no-managed-probe', 'ret = apple_pmp_start_init(rep);', 'ret = 0; (void)apple_pmp_start_init;'), ('no-child-seeding', 'apple_pmp_report_entry_set_state(&ent->genpd, true);', '(void)0;'), From a31dbc0510b69aeff06b5f86204fed6d082b5474 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 06:12:38 +1100 Subject: [PATCH 1445/1579] tools: asahi: run the mt7932 policy class tests with the host suite run-host-tests.py refuses unregistered test files, so register the new 6 GHz class policy tests next to the existing mt7932 policy tests. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/run-host-tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index cacece95bc905..66c8779f6cc64 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -112,6 +112,7 @@ def mutations(flag, names, base, expect=KILL): 'mt7932-connect-result-tests.py': [(out('--output'), PASS)], 'mt7932-owe-retry-tests.py': [(out(), PASS)], 'mt7932-policy-6ghz-tests.py': [([], PASS)], + 'mt7932-policy-classes-tests.py': [([], PASS)], 'neo-auxiliary-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutation'], KILL), (out('--output') + ['--old'], KILL)], 'neo-stability-tests.py': [(out('--output'), PASS)] From 4f30dbc420ade9a296363fbb248f5d379dd6b88c Mon Sep 17 00:00:00 2001 From: Ryan <81541646+RyanTheTide@users.noreply.github.com> Date: Sat, 31 Oct 2026 06:31:28 +1100 Subject: [PATCH 1446/1579] tools: asahi: refresh Neo stability controls for 6 GHz classes Extract the production 6 GHz class enum and selector alongside regulatory helpers, and supply the notifier's six_ap_valid field in the host mock. Keep VLP-client restrictions in class admission rather than expecting the country domain to discard channels that can still admit LPI. Retarget the VLP-client mutation to the production selector and assert both VLP denial and LPI admission. Preserve the existing band, stale-policy, USB, wake and syscore controls, including old-source negative controls. Check that a regulatory update invalidates the cached AP decision. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/neo-stability-tests.py | 38 +++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/tools/asahi/neo-stability-tests.py b/tools/asahi/neo-stability-tests.py index ff02d627e5711..16256c1843fd8 100644 --- a/tools/asahi/neo-stability-tests.py +++ b/tools/asahi/neo-stability-tests.py @@ -116,6 +116,7 @@ def function(source, name): #define for_each_set_bit(i,b,n) for((i)=0;(i)<(n);(i)++) if((b)[(i)/64]&(1ULL<<((i)%64))) #define bitmap_zero(b,n) memset((b),0,((n)+63)/64*8) #define IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT 128 +#define IEEE80211_CHAN_NO_HE 256 #define NL80211_RRF_NO_IR (1<<7) struct ieee80211_channel {unsigned flags,hw_value;int max_power;enum nl80211_band band;int center_freq;s8 psd;}; static u64 get_unaligned_le64(const u8*p){return get_unaligned_le32(p)|(u64)get_unaligned_le32(p+4)<<32;} @@ -131,7 +132,7 @@ def function(source, name): static void rtnl_unlock(void){assert(rtnl_depth--==1);} static void wiphy_lock(struct wiphy*w){assert(rtnl_depth&& !wiphy_depth++);} static void wiphy_unlock(struct wiphy*w){assert(wiphy_depth--==1);} -struct mt7932 {struct ieee80211_channel channels[MT7932_CHANNELS_2G],channels5[MT7932_CHANNELS_5G],channels6[MT7932_CHANNELS_6G];struct wiphy*wiphy;struct net_device*netdev;struct {struct device dev;}*pdev;struct mt7932_reg_snapshot reg_desired,reg_deferred;unsigned reg_generation,response_lock,startup_work;bool reg_pending,stopping,interface_registered,connecting,connected,disconnecting,reg_deferral,data_ready;int data_lock;int connect_error,assoc_start,assoc_done,discovery_done;uint64_t policy_disabled[(MT7932_CHANNELS+63)/64];}; +struct mt7932 {struct ieee80211_channel channels[MT7932_CHANNELS_2G],channels5[MT7932_CHANNELS_5G],channels6[MT7932_CHANNELS_6G];struct wiphy*wiphy;struct net_device*netdev;struct {struct device dev;}*pdev;struct mt7932_reg_snapshot reg_desired,reg_deferred;unsigned reg_generation,response_lock,startup_work;bool reg_pending,stopping,interface_registered,connecting,connected,disconnecting,reg_deferral,data_ready,six_ap_valid;int data_lock;int connect_error,assoc_start,assoc_done,discovery_done;uint64_t policy_disabled[(MT7932_CHANNELS+63)/64];}; #define spin_lock(l) ((void)(l)) #define spin_unlock(l) ((void)(l)) static void mt_data_kick_locked(struct mt7932*m){assert(0 && "unexpected TX boundary");} @@ -191,7 +192,14 @@ def function(source, name): memset(tables[t]+44+j*122,0xc4,122);tables[t][44+j*122]=c->hw_value; if(i!=10)tables[t][45+j*122]=1; } - /* 6 GHz: VLP permission bitmap, NO_IR until cfg80211's rule lifts it, PSD limits. */ + /* The VLP-client flag gates the selected AP class, not the whole + * country domain: the same channel can still admit LPI. + */ + assert(mt7932_6g_class(2,0)==MT7932_6G_VLP); + assert(mt7932_6g_class(2,IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT)==-EACCES); + assert(mt7932_6g_class(0,IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT)==MT7932_6G_LPI); + assert(mt7932_6g_class(1,0)==-EOPNOTSUPP); + /* Country bitmap, NO_IR until cfg80211's rule lifts it, PSD limits. */ p.six_channels=GENMASK_ULL(MT7932_CHANNELS_6G-1,0)&~BIT_ULL(3); for(unsigned k=0;khw_value&&!f); - else{unsigned k=i-N25+(i>=VLP);assert(get_unaligned_le16(e)==mt7932_channel_6g(k)&&f==IEEE80211_CHAN_PSD);} + else{unsigned k=i-N25;u32 expected=IEEE80211_CHAN_PSD|(i==VLP?IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT:0);assert(get_unaligned_le16(e)==mt7932_channel_6g(k)&&f==expected);} } for(unsigned k=0;k=N25-1?c!=13:true);} + assert(reg.length==12+(MT7932_CHANNELS-2)*8&®.domain[8]==12&®.domain[9]==MT7932_CHANNELS_5G&®.domain[10]==MT7932_CHANNELS_6G-1); + for(unsigned i=0;i=N25-1?c!=13:true);} /* A rule that keeps NO_IR, or no rule at all, leaves the channel passive. */ m.channels6[0].flags|=IEEE80211_CHAN_NO_IR;m.channels6[1].flags|=IEEE80211_CHAN_NO_IR; rule6.flags=NL80211_RRF_NO_IR;mt_policy_disable(&m,&p,generation);assert(m.channels6[0].flags&IEEE80211_CHAN_NO_IR); @@ -255,6 +264,7 @@ def main(): 'drivers/soc/apple/dockchannel.c': ['dockchannel_init_wakeup'], 'drivers/soc/apple/pm_shim.c': ['sep_pm_release_domains', 'sep_pm_keep_domains'], 'drivers/usb/dwc3/dwc3-apple.c': ['dwc3_apple_core_stop', 'dwc3_apple_init', 'dwc3_apple_exit'], + 'drivers/net/wireless/mediatek/mt7932/six.h': ['mt7932_6g_class'], 'drivers/net/wireless/mediatek/mt7932/startup.c': ['mt_channel', 'mt_policy_disable', 'mt_regulatory_notify'], } bodies = {} @@ -267,7 +277,13 @@ def main(): bodies[name] = function(text, name) headers = [ROOT/'drivers/net/wireless/mediatek/mt7932/channels.h', ROOT/'drivers/net/wireless/mediatek/mt7932/regulatory.h'] - header = '\n'.join(re.sub(r'^#include .*\n', '', path.read_text(), flags=re.M) + # regulatory.h's class parser/pages depend on the production enum. + # Extract only that declaration, avoiding the unrelated HE/TPE helpers. + six = (ROOT/'drivers/net/wireless/mediatek/mt7932/six.h').read_text() + power_class = re.search(r'^enum mt7932_6g_class \{[^}]*\};', six, re.M) + if not power_class: + raise ValueError('MT7932 6 GHz class enum') + header = power_class[0] + '\n' + '\n'.join(re.sub(r'^#include .*\n', '', path.read_text(), flags=re.M) for path in headers) # The receive IRQ request statement from dockchannel_init(), run against a recording stub. dock = (ROOT/'drivers/soc/apple/dockchannel.c').read_text() @@ -287,7 +303,7 @@ def main(): 'rx-suspend': ('IRQF_NO_AUTOEN | IRQF_NO_SUSPEND', 'IRQF_NO_AUTOEN'), 'stale-policy': ('generation != READ_ONCE(m->reg_generation)', 'false'), 'usb-rpm': ('pm_runtime_put_autosuspend(appledwc->dev);', '(void)0;'), - 'vlp-client': ('if (flags & IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT)', 'if (0)'), + 'vlp-client': ('return flags & IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT ?', 'return false ?'), 'six-band': ('(policy->six_channels & BIT_ULL((channel - 1) / 4))', 'true'), } if args.mutation: From ca56861568212bad5ee53d984a15f5bfc3316967 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 14:09:08 +1100 Subject: [PATCH 1447/1579] media: apple: ave: select command layouts by firmware TEXT CRC Describe Start/Process sizes, field offsets, descriptors, constant words, subtypes and working-buffer sizes per firmware. Keep the J700 25G83 wire format and select its layout only for loader-verified TEXT CRC 69c54454. Add the independently derived 26A428 command layout and T6041 probe match data with the measured DATA size. Its TEXT CRC and boot sequence remain unqualified, so the new layout has no CRC lookup entry and encoding stays disabled. Preserve the loader reservations, CRC checks and disabled DT nodes. Add host checks of the compiled builders, allocation, reply routing and qualification gates, including byte comparisons against the J700 baseline. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/admin-guide/media/apple-ave.rst | 50 +- drivers/media/platform/apple/ave/Kconfig | 3 +- drivers/media/platform/apple/ave/Makefile | 2 +- drivers/media/platform/apple/ave/ave-cmd.c | 490 ++++++++--------- drivers/media/platform/apple/ave/ave-drv.c | 24 +- drivers/media/platform/apple/ave/ave-fw.c | 21 +- drivers/media/platform/apple/ave/ave-image.c | 251 ++++++++- drivers/media/platform/apple/ave/ave-layout.c | 494 ++++++++++++++++++ drivers/media/platform/apple/ave/ave-layout.h | 135 +++++ drivers/media/platform/apple/ave/ave-regs.h | 2 - drivers/media/platform/apple/ave/ave-sess.c | 52 +- drivers/media/platform/apple/ave/ave.h | 24 +- drivers/media/platform/apple/ave/tests/README | 29 + .../media/platform/apple/ave/tests/boundary.h | 87 +++ drivers/media/platform/apple/ave/tests/run.py | 98 ++++ drivers/media/platform/apple/ave/tests/shim.h | 105 ++++ drivers/media/platform/apple/ave/tests/test.c | 363 +++++++++++++ 17 files changed, 1895 insertions(+), 335 deletions(-) create mode 100644 drivers/media/platform/apple/ave/ave-layout.c create mode 100644 drivers/media/platform/apple/ave/ave-layout.h create mode 100644 drivers/media/platform/apple/ave/tests/README create mode 100644 drivers/media/platform/apple/ave/tests/boundary.h create mode 100644 drivers/media/platform/apple/ave/tests/run.py create mode 100644 drivers/media/platform/apple/ave/tests/shim.h create mode 100644 drivers/media/platform/apple/ave/tests/test.c diff --git a/Documentation/admin-guide/media/apple-ave.rst b/Documentation/admin-guide/media/apple-ave.rst index fbac9d743da9b..d43e759481cd6 100644 --- a/Documentation/admin-guide/media/apple-ave.rst +++ b/Documentation/admin-guide/media/apple-ave.rst @@ -40,10 +40,10 @@ it restores before every firmware boot. No firmware files are needed. The driver builds every command itself, including the two large session commands (Start and Process), from the session settings and a set of constant fields known for one firmware build: -AppleAVE2FW 9003.78.0, TEXT checksum ``69c54454``. With any other firmware -build the device binds, but starting a stream fails with ``ENOTSUPP`` and a -message naming the checksum, until that build's command layout has been -checked. +AppleAVE2FW 9003.78.0, TEXT checksum ``69c54454``. T8140 rejects an unrecognized firmware at probe. Probe-only devices may +bind with another loader-verified build, but starting a stream fails with +``ENOTSUPP`` and a message naming the checksum until its command layout and +hardware boot sequence have been qualified. T6000 (M1 Pro) and T8112 (M2) have probe-only support in this driver. Their device tree nodes and both encoder DARTs remain disabled. The firmware @@ -59,6 +59,48 @@ pre-M4 devices: at 1920x1080 its exact descriptor size is 0x85200 for AVC and 0x1fe00 for HEVC, before the DMA allocation is page rounded. The ``apple-ave-sizes`` KUnit suite checks these sizes and alignment boundaries. +T6041 (M4 Pro, including J616s reporting chip ID T6040) also has probe-only +support. Its TEXT and DATA reservations must be 0x14c000 and 0x12c000 bytes. +The existing AVE node and both DARTs remain disabled; the loader must publish +and preserve the live reservations and both pre-start CRCs before enabling +those nodes. A probe with a valid handoff reports the verified CRCs without +starting the coprocessor. An unknown layout refuses stream start before +allocating session buffers or acquiring runtime power. + +Command sizes, offsets, descriptor groups, constants, Process subtype and +buffer sizes are selected by the verified TEXT CRC, independently of the +SoC match data. Only ``69c54454`` currently selects an encoding layout. +There is no fallback layout and zero is not a wildcard CRC. Hardware boot +qualification is a separate requirement; presenting the J700 CRC on T6041 +does not authorize the J700 boot sequence. + +The driver describes the command layout observed on J616s with macOS 27.0 +26A428, including counted Start groups, packed AVC fields and sized Process +descriptors. It is deliberately absent from the CRC lookup: its TEXT CRC +and its relationship to the firmware supplied to Linux are still unknown. +The boot-config words, coprocessor start registers, DMA aperture and output +metadata/slice-header extraction also need qualification before encoding +can be enabled. The 26A428 capture and the J700 capture used different OS +versions; a SoC compatible cannot identify a command ABI. + +The public API continues to offer 8-bit NV12 and VBR/CBR. CQ and Main10 +layout facts are exercised by host tests without exposing new V4L2 modes. +The 26A428 layout has no proven fields for runtime bit-rate updates, forced +IDR or HEVC low delay. Such updates are refused rather than written at J700 +offsets. Frame rates other than 30 fps and unobserved no-reordering modes +are also refused for that layout. + +Host validation needs a C compiler with AddressSanitizer and UBSan and Python:: + + python3 drivers/media/platform/apple/ave/tests/run.py + +The tests compile the actual builder, allocation and reply functions with a +fake IPC/DMA boundary. They require no encoder, firmware or captured command +blocks. To compare all bytes of 288 synthetic J700 sessions against an older +checkout, add ``--compare-baseline /path/to/old/drivers/media/platform/apple/ave``. +Private capture comparisons must keep captured images outside the source tree +and mask only addresses, session tokens, clocks and host bookkeeping fields. + With debugfs mounted, ``apple-ave/last_start_h264``, ``last_start_hevc``, ``last_process0_h264`` and ``last_process0_hevc`` hold the last Start command built and the first Process command of the last session, diff --git a/drivers/media/platform/apple/ave/Kconfig b/drivers/media/platform/apple/ave/Kconfig index 20eb7cc404192..9284ff96874cb 100644 --- a/drivers/media/platform/apple/ave/Kconfig +++ b/drivers/media/platform/apple/ave/Kconfig @@ -10,7 +10,8 @@ config VIDEO_APPLE_AVE help V4L2 memory-to-memory driver for the AVE H.264/HEVC encoder in Apple T8140 (A18 Pro) SoCs, as in the MacBook Neo. - T6000 (M1 Pro) and T8112 (M2) have probe-only support; encoding + T6000 (M1 Pro), T6041 (M4 Pro) and T8112 (M2) have probe-only + support; encoding requires qualified firmware command constants. The encoder firmware is loaded by the bootloader; the driver builds diff --git a/drivers/media/platform/apple/ave/Makefile b/drivers/media/platform/apple/ave/Makefile index 899d8990555ea..0487aab585666 100644 --- a/drivers/media/platform/apple/ave/Makefile +++ b/drivers/media/platform/apple/ave/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_VIDEO_APPLE_AVE) += apple-ave.o -apple-ave-y := ave-drv.o ave-fw.o ave-ipc.o ave-cmd.o ave-image.o ave-sess.o ave-v4l2.o +apple-ave-y := ave-drv.o ave-fw.o ave-ipc.o ave-cmd.o ave-layout.o ave-image.o ave-sess.o ave-v4l2.o obj-$(CONFIG_VIDEO_APPLE_AVE_KUNIT_TEST) += ave-sizes-test.o diff --git a/drivers/media/platform/apple/ave/ave-cmd.c b/drivers/media/platform/apple/ave/ave-cmd.c index 7a03ed53b457b..3360f8e4edac3 100644 --- a/drivers/media/platform/apple/ave/ave-cmd.c +++ b/drivers/media/platform/apple/ave/ave-cmd.c @@ -94,6 +94,8 @@ static bool ave_cmd_others_open(struct ave_ctx *ctx) */ static u32 ave_session_arg2(struct ave_ctx *ctx) { + if (ctx->ave->layout->counted && ctx->sess->p.codec == AVE_CODEC_H264) + return ctx->sess->p.bframes ? AVE_SESSION_ARG2_H264 : 0x140; if (ave_cmd_others_open(ctx)) return 0; return ctx->sess->p.codec == AVE_CODEC_HEVC ? AVE_SESSION_ARG2_HEVC : @@ -134,7 +136,7 @@ int ave_cmd_config(struct ave_dev *ave) ave_put32(cmd, AVE_CONFIG_ALL_UNITS, U32_MAX); cmd[AVE_CONFIG_OPT_A] = 1; cmd[AVE_CONFIG_OPT_B] = 1; - ave_put32(cmd, AVE_CONFIG_MCC_DSID, AVE_CONFIG_MCC_DSID_T8140); + ave_put32(cmd, AVE_CONFIG_MCC_DSID, ave->layout->mcc_dsid); ret = ave_ipc_submit(ave, buf, AVE_CONFIG_BYTES, AVE_CMD_GLOBAL_TIMEOUT_US, true); return ret ?: ave_ipc_settle(ave, AVE_CMD_WAIT_MS); } @@ -215,150 +217,71 @@ int ave_cmd_close(struct ave_ctx *ctx) /* Start (id 6) */ -#define AVE_START_SURFACE 0x48 /* and an unused second one at 0x58 */ -#define AVE_START_SURFACE_BYTES 0x12c000 -#define AVE_START_SURFACE2 0x58 -#define AVE_START_REF_A(i) (0x390 + 0x20 * (i)) -#define AVE_START_REF_B(i) (0x3a0 + 0x20 * (i)) -#define AVE_START_REF_C(i) (0x7d0 + 0x20 * (i)) -#define AVE_START_REF_C_ALIAS(i) (0x7e0 + 0x20 * (i)) -#define AVE_START_MB 0xc10 -#define AVE_START_CODED(s) (0xd50 + 0x10 * (s)) -#define AVE_START_META(s) (0xf50 + 0x10 * (s)) #define AVE_START_ARRAY_LEN 30 -#define AVE_START_REF_D(i) (0x1130 + 0x10 * (i)) -#define AVE_START_ROW_A 0x1350 -#define AVE_START_ROW_B 0x1390 -#define AVE_START_FIXED 0x13d0 -#define AVE_START_FIXED_BYTES 0x4000 -#define AVE_START_HEVC_EXTRA(i) (0x2520 + 0x10 * (i)) - -static const u16 ave_start_tile_off[AVE_TILE_BUFS] = { - 0x1410, 0x1430, 0x1490, 0x14b0, /* both codecs */ - 0x1510, 0x1530, 0x1590, 0x15b0, /* H.264 only */ -}; - -/* Session configuration words */ -#define AVE_START_CODEC 0xc0 -#define AVE_START_8BIT 0x208 -#define AVE_START_CONST_214 0x214 -#define AVE_START_CONST_218 0x218 -#define AVE_START_FPS 0x220 -#define AVE_START_RC_MODE 0x234 -#define AVE_START_BITRATE 0x238 -#define AVE_START_INIT_QP(i) (0x240 + 4 * (i)) #define AVE_START_INIT_QP_VALUE 26 -#define AVE_START_LIMITS 0x250 -#define AVE_START_LIMIT_BYTES 0x258 -#define AVE_START_LIMIT_SECS 0x260 -#define AVE_START_F64_288 0x288 -#define AVE_START_F64_290 0x290 -#define AVE_START_MIN_QP 0x298 -#define AVE_START_MAX_QP 0x29c -#define AVE_START_GOP_FLAGS 0x2a0 -#define AVE_START_P_FRAMES 0x2ac -#define AVE_START_MAX_B 0x2b0 -#define AVE_START_REF_DEPTH 0x2b4 -#define AVE_START_IDR_INTERVAL 0x2b8 -#define AVE_START_F64_2C0 0x2c0 -#define AVE_START_F64_2C8 0x2c8 -#define AVE_START_RC_FLAGS 0x300 -#define AVE_START_MIN_QP2 0x308 -#define AVE_START_MAX_QP2 0x30c -#define AVE_START_RC_310 0x310 -#define AVE_START_SIZE_CLASS 0x320 -#define AVE_START_CODED_W 0x368 -#define AVE_START_CODED_H 0x36c -#define AVE_START_CODEC_FLAGS 0x370 -#define AVE_START_H264_380 0x380 -#define AVE_START_H264_384 0x384 -#define AVE_START_CODEC_2588 0x2588 /* 2 for H.264 High and HEVC */ -#define AVE_START_BIT_DEPTH 0x2598 -#define AVE_START_CODED_H2 0x25dc - -/* H.264 parameter-set words */ -#define AVE_START_H264_PROFILE 0x291c -#define AVE_START_H264_LEVEL 0x2938 -#define AVE_START_H264_MAX_REFS 0x2d44 -#define AVE_START_H264_WIDTH_MBS 0x2d4c -#define AVE_START_H264_HEIGHT_MBS 0x2d50 -#define AVE_START_H264_CROP 0x2db4 -#define AVE_START_H264_CROP_LEFT 0x2db8 -#define AVE_START_H264_CROP_RIGHT 0x2dbc -#define AVE_START_H264_CROP_TOP 0x2dc0 -#define AVE_START_H264_CROP_BOTTOM 0x2dc4 -#define AVE_START_H264_CABAC 0x2fd8 -#define AVE_START_H264_8X8 0x3023 /* byte */ - -/* HEVC parameter-set words; most appear twice */ -#define AVE_START_HEVC_PAD 0x26e0 -#define AVE_START_HEVC_RC_270C 0x270c /* 2 in bit-rate modes, 5 in CQ */ -#define AVE_START_HEVC_RC_9604 0x9604 /* 0 in bit-rate modes, 2 in CQ */ -#define AVE_START_HEVC_RC_9608 0x9608 /* 0x10000 in bit-rate modes, 0 in CQ */ -#define AVE_START_HEVC_PROFILE_A 0x2938 -#define AVE_START_HEVC_COMPAT_A 0x293d /* two bytes */ -#define AVE_START_HEVC_8BIT_A 0x2962 /* byte */ -#define AVE_START_HEVC_LEVEL_A 0x296c -#define AVE_START_HEVC_PROFILE_B 0x5904 -#define AVE_START_HEVC_COMPAT_B 0x5909 -#define AVE_START_HEVC_8BIT_B 0x592e -#define AVE_START_HEVC_LEVEL_B 0x5938 -#define AVE_START_HEVC_WIDTH 0x5ab4 -#define AVE_START_HEVC_HEIGHT 0x5ab8 -#define AVE_START_HEVC_WIDTH_CTB 0x5abc -#define AVE_START_HEVC_HEIGHT_CTB 0x5ac0 -#define AVE_START_HEVC_CONF 0x5ac4 -#define AVE_START_HEVC_CONF_LEFT 0x5ac8 -#define AVE_START_HEVC_CONF_RIGHT 0x5acc -#define AVE_START_HEVC_CONF_TOP 0x5ad0 -#define AVE_START_HEVC_CONF_BOTTOM 0x5ad4 -#define AVE_START_HEVC_DEPTH_LUMA 0x5ad8 -#define AVE_START_HEVC_DEPTH_CHROMA 0x5adc -/* HEVC reference structure with B-frames (reordering on) */ -#define AVE_START_HEVC_REFS 0x259c -#define AVE_START_HEVC_REORDER_COPY 0x2e54 -#define AVE_START_HEVC_DPB_MINUS1 0x5b00 -#define AVE_START_HEVC_REORDER 0x5b04 -#define AVE_START_HEVC_REORDER_ON 0x95fc - -static void ave_start_surfaces(struct ave_session *sess, u8 *cmd) + +static void ave_start_surfaces(struct ave_session *sess, u8 *cmd, + const struct ave_fw_layout *fw) { const struct ave_params *p = &sess->p; bool hevc = p->codec == AVE_CODEC_HEVC; unsigned int tiles = hevc ? 4 : AVE_TILE_BUFS; + const struct ave_start_layout *l = &fw->start[p->codec - 1]; /* Clear every descriptor the image may carry, then fill ours. */ for (unsigned int i = 0; i < AVE_MAX_REFS; i++) { - ave_put_surface(cmd, AVE_START_REF_A(i), 0, 0); - ave_put_surface(cmd, AVE_START_REF_B(i), 0, 0); - ave_put_surface(cmd, AVE_START_REF_C(i), 0, 0); - ave_put_surface(cmd, AVE_START_REF_C_ALIAS(i), 0, 0); - ave_put_surface(cmd, AVE_START_REF_D(i), 0, 0); + ave_put_surface(cmd, l->ref_a + l->ref_stride * i, 0, 0); + ave_put_surface(cmd, l->ref_b + l->ref_stride * i, 0, 0); + ave_put_surface(cmd, l->ref_c + l->ref_c_stride * i, 0, 0); + ave_put_surface(cmd, l->ref_c_alias + l->ref_c_stride * i, 0, 0); + ave_put_surface(cmd, l->ref_d + 0x10 * i, 0, 0); } for (unsigned int s = 0; s < AVE_START_ARRAY_LEN; s++) { - ave_put_surface(cmd, AVE_START_CODED(s), 0, 0); - ave_put_surface(cmd, AVE_START_META(s), 0, 0); + ave_put_surface(cmd, l->coded + 0x10 * s, 0, 0); + ave_put_surface(cmd, l->meta + 0x10 * s, 0, 0); } - ave_put_buf(cmd, AVE_START_SURFACE, &sess->hdr, AVE_START_SURFACE_BYTES); - ave_put_surface(cmd, AVE_START_SURFACE2, 0, 0); + ave_put_buf(cmd, 0x48, &sess->hdr, fw->hdr_bytes); + ave_put_surface(cmd, 0x58, 0, 0); for (unsigned int i = 0; i < p->refs; i++) { - ave_put_buf(cmd, AVE_START_REF_A(i), &sess->ref_a[i], sess->ref_a[i].size); - ave_put_buf(cmd, AVE_START_REF_B(i), &sess->ref_b[i], sess->ref_b[i].size); - ave_put_buf(cmd, AVE_START_REF_C(i), &sess->ref_c[i], sess->ref_c[i].size); - ave_put_surface(cmd, AVE_START_REF_C_ALIAS(i), sess->ref_c[i].dma, 0); - ave_put_buf(cmd, AVE_START_REF_D(i), &sess->ref_d[i], sess->ref_d[i].size); + ave_put_buf(cmd, l->ref_a + l->ref_stride * i, + &sess->ref_a[i], sess->ref_a[i].size); + ave_put_buf(cmd, l->ref_b + l->ref_stride * i, + &sess->ref_b[i], sess->ref_b[i].size); + ave_put_buf(cmd, l->ref_c + l->ref_c_stride * i, + &sess->ref_c[i], sess->ref_c[i].size); + ave_put_surface(cmd, l->ref_c_alias + l->ref_c_stride * i, sess->ref_c[i].dma, 0); + ave_put_buf(cmd, l->ref_d + 0x10 * i, &sess->ref_d[i], sess->ref_d[i].size); + } + if (fw->counted) { + ave_put32(cmd, l->refs_count, p->refs); + ave_put32(cmd, l->work_count, p->refs); + ave_put32(cmd, l->lfs_count, p->refs); + ave_put32(cmd, l->slots_count, p->slots); + ave_put32(cmd, l->meta_count, p->slots); + ave_put32(cmd, l->mb_count, p->rc_mode == 3 ? 3 : 1); + if (p->rc_mode == 3) + for (unsigned int i = 0; i < 2; i++) + ave_put_buf(cmd, l->mb + 0x10 * (i + 1), + &sess->mb_extra[i], sess->mb_extra[i].size); + ave_put32(cmd, l->tiles_count, tiles); + for (unsigned int i = 0; i < p->refs; i++) { + ave_put_buf(cmd, l->ref_e + l->ref_stride * i, + &sess->ref_e[i], sess->ref_e[i].size); + ave_put_buf(cmd, l->ref_f + l->ref_stride * i, + &sess->ref_f[i], sess->ref_f[i].size); + } } - ave_put_buf(cmd, AVE_START_MB, &sess->mb, sess->mb.size); + ave_put_buf(cmd, l->mb, &sess->mb, sess->mb.size); for (unsigned int s = 0; s < p->slots; s++) { - ave_put_buf(cmd, AVE_START_CODED(s), &sess->slot[s].coded, p->coded_cap); - ave_put_buf(cmd, AVE_START_META(s), &sess->slot[s].meta, sess->slot[s].meta.size); + ave_put_buf(cmd, l->coded + 0x10 * s, &sess->slot[s].coded, p->coded_cap); + ave_put_buf(cmd, l->meta + 0x10 * s, &sess->slot[s].meta, sess->slot[s].meta.size); } - ave_put_buf(cmd, AVE_START_ROW_A, &sess->row_a, sess->row_a.size); - ave_put_buf(cmd, AVE_START_ROW_B, &sess->row_b, sess->row_b.size); - ave_put_buf(cmd, AVE_START_FIXED, &sess->fixed, AVE_START_FIXED_BYTES); + ave_put_buf(cmd, l->row_a, &sess->row_a, sess->row_a.size); + ave_put_buf(cmd, l->row_b, &sess->row_b, sess->row_b.size); + ave_put_buf(cmd, l->fixed, &sess->fixed, SZ_16K); for (unsigned int i = 0; i < AVE_TILE_BUFS; i++) { - u32 off = ave_start_tile_off[i]; + u32 off = l->tiles[i]; if (i < tiles) { ave_put_buf(cmd, off, &sess->tile[i], sess->tile[i].size); @@ -370,112 +293,154 @@ static void ave_start_surfaces(struct ave_session *sess, u8 *cmd) } if (hevc) for (unsigned int i = 0; i < 2; i++) - ave_put_buf(cmd, AVE_START_HEVC_EXTRA(i), &sess->hevc_extra[i], + ave_put_buf(cmd, l->hevc_extra + 0x10 * i, &sess->hevc_extra[i], sess->hevc_extra[i].size); } -static void ave_start_config(const struct ave_params *p, u8 *cmd) +static void ave_start_put32(u8 *cmd, u32 off, u32 val) +{ + if (off) + ave_put32(cmd, off, val); +} + +static void ave_start_config(const struct ave_params *p, u8 *cmd, + const struct ave_fw_layout *fw) { bool hevc = p->codec == AVE_CODEC_HEVC; + bool cq = p->rc_mode == 3; + u32 depth = p->bit_depth ?: 8; u32 crop_r = (p->cw - p->width) / 2, crop_b = (p->ch - p->height) / 2; bool crop = crop_r || crop_b; - - ave_put32(cmd, AVE_START_CODEC, p->codec); - ave_put32(cmd, AVE_START_8BIT, hevc ? 1 : 0); - ave_put32(cmd, AVE_START_CONST_214, 0x39d); - ave_put32(cmd, AVE_START_CONST_218, 0x18); - ave_put32(cmd, AVE_START_FPS, p->fps_int); + const struct ave_start_layout *l = &fw->start[p->codec - 1]; + + ave_start_put32(cmd, l->codec, p->codec); + ave_start_put32(cmd, l->flag_8bit, + fw->counted ? 0x10000 | (hevc ? (depth == 8 ? 0x41 : 0x40) : 0) | + (cq ? 0x10 : 0) : hevc ? 1 : 0); + ave_start_put32(cmd, l->geometry_mode, (hevc ? 0x40 : 0) | (cq ? 0 : 2)); + ave_start_put32(cmd, l->geometry_depth, depth); + ave_start_put32(cmd, l->const_214, 0x39d); + ave_start_put32(cmd, l->const_218, 0x18); + ave_start_put32(cmd, l->fps, p->fps_int); /* Rate control */ - ave_put32(cmd, AVE_START_RC_MODE, p->rc_mode); - ave_put32(cmd, AVE_START_BITRATE, p->bitrate); + ave_start_put32(cmd, l->rc_mode, p->rc_mode); + ave_start_put32(cmd, l->bitrate, p->bitrate); for (unsigned int i = 0; i < 3; i++) - ave_put32(cmd, AVE_START_INIT_QP(i), AVE_START_INIT_QP_VALUE); - ave_put32(cmd, AVE_START_LIMITS, p->limit_bytes ? 1 : 0); - if (p->limit_bytes) { - ave_put64(cmd, AVE_START_LIMIT_BYTES, ave_f64(p->limit_bytes, 1)); - ave_put64(cmd, AVE_START_LIMIT_SECS, ave_f64(1, 1)); + ave_put32(cmd, (l->init_qp + 4 * i), cq ? p->initial_qp : AVE_START_INIT_QP_VALUE); + ave_start_put32(cmd, l->limits, p->limit_bytes ? 1 : 0); + if (l->limit_bytes && p->limit_bytes) { + ave_put64(cmd, l->limit_bytes, + ave_f64((u64)p->limit_bytes * (fw->counted ? 8 : 1), 1)); + ave_put64(cmd, l->limit_secs, ave_f64(1, 1)); } - ave_put64(cmd, AVE_START_F64_288, ave_f64(1, 1)); - ave_put64(cmd, AVE_START_F64_290, ave_f64(1, 2)); - ave_put64(cmd, AVE_START_F64_2C0, AVE_F64_NEG | ave_f64(1, 1)); - ave_put64(cmd, AVE_START_F64_2C8, AVE_F64_NEG | ave_f64(1, 1)); - ave_put32(cmd, AVE_START_MIN_QP, p->min_qp); - ave_put32(cmd, AVE_START_MAX_QP, p->max_qp); - ave_put32(cmd, AVE_START_MIN_QP2, p->min_qp); - ave_put32(cmd, AVE_START_MAX_QP2, p->max_qp); - ave_put32(cmd, AVE_START_RC_FLAGS, hevc ? 0x736 : 0x8202); - ave_put32(cmd, AVE_START_RC_310, 30); + ave_put64(cmd, l->f64_288, ave_f64(1, 1)); + ave_put64(cmd, l->f64_290, ave_f64(1, 2)); + ave_put64(cmd, l->f64_2c0, AVE_F64_NEG | ave_f64(1, 1)); + ave_put64(cmd, l->f64_2c8, AVE_F64_NEG | ave_f64(1, 1)); + ave_start_put32(cmd, l->min_qp, p->min_qp); + ave_start_put32(cmd, l->max_qp, p->max_qp); + ave_start_put32(cmd, l->min_qp2, p->min_qp); + ave_start_put32(cmd, l->max_qp2, p->max_qp); + ave_start_put32(cmd, l->rc_flags, cq ? 0 : hevc ? 0x736 : 0x8202); + ave_start_put32(cmd, l->rc_310, cq ? 0 : 30); if (!hevc) - ave_put32(cmd, AVE_START_H264_380, 1); + ave_start_put32(cmd, l->h264_380, 1); /* GOP structure: with B-frames as captured, or P-only, or intra-only. */ - ave_put32(cmd, AVE_START_GOP_FLAGS, p->bframes ? 0x600 : 0x200); - ave_put32(cmd, AVE_START_P_FRAMES, p->intra_only ? 0 : 1); - ave_put32(cmd, AVE_START_MAX_B, p->bframes ? 3 : 0); - ave_put32(cmd, AVE_START_REF_DEPTH, p->bframes ? 3 : 1); - ave_put32(cmd, AVE_START_IDR_INTERVAL, p->gop); + ave_start_put32(cmd, l->gop_flags, (p->bframes ? 0x600 : 0x200) | + (fw->counted && cq ? 0x10000 | (hevc ? 0x100 : 0) : 0)); + ave_start_put32(cmd, l->p_frames, p->intra_only ? 0 : 1); + ave_start_put32(cmd, l->max_b, p->bframes ? 3 : 0); + ave_start_put32(cmd, l->ref_depth, p->bframes ? 3 : 1); + ave_start_put32(cmd, l->idr_interval, p->gop); /* Picture size */ - ave_put32(cmd, AVE_START_SIZE_CLASS, p->width * p->height > 1280 * 720 ? 3 : 1); - ave_put32(cmd, AVE_START_CODED_W, p->cw); - ave_put32(cmd, AVE_START_CODED_H, p->ch); - ave_put32(cmd, AVE_START_CODED_H2, p->ch); - ave_put32(cmd, AVE_START_CODEC_FLAGS, hevc ? 0x2000002 : 2); - ave_put32(cmd, AVE_START_BIT_DEPTH, 8); + ave_start_put32(cmd, l->size_class, p->width * p->height > 1280 * 720 ? 3 : 1); + ave_start_put32(cmd, l->coded_w, p->cw); + ave_start_put32(cmd, l->coded_h, p->ch); + ave_start_put32(cmd, l->coded_h2, p->ch); + ave_start_put32(cmd, l->codec_flags, hevc ? 0x2000002 : 2); + ave_start_put32(cmd, l->bit_depth, depth); if (!hevc) { - ave_put32(cmd, AVE_START_H264_384, p->width * p->height <= 320 * 240 ? 0x40 : 0x10); - ave_put32(cmd, AVE_START_CODEC_2588, p->high ? 2 : 0); - ave_put32(cmd, AVE_START_H264_PROFILE, p->profile); - ave_put32(cmd, AVE_START_H264_LEVEL, p->level); - ave_put32(cmd, AVE_START_H264_MAX_REFS, p->refs - 1); - ave_put32(cmd, AVE_START_H264_WIDTH_MBS, p->cw / 16 - 1); - ave_put32(cmd, AVE_START_H264_HEIGHT_MBS, p->ch / 16 - 1); - ave_put32(cmd, AVE_START_H264_CROP, crop); - ave_put32(cmd, AVE_START_H264_CROP_LEFT, 0); - ave_put32(cmd, AVE_START_H264_CROP_RIGHT, crop_r); - ave_put32(cmd, AVE_START_H264_CROP_TOP, 0); - ave_put32(cmd, AVE_START_H264_CROP_BOTTOM, crop_b); - ave_put32(cmd, AVE_START_H264_CABAC, p->profile != 2); - cmd[AVE_START_H264_8X8] = p->high; + ave_start_put32(cmd, l->h264_384, p->width * p->height <= 320 * 240 ? 0x40 : 0x10); + ave_start_put32(cmd, l->codec_2588, p->high ? 2 : 0); + ave_start_put32(cmd, l->h264_profile, p->profile); + ave_start_put32(cmd, l->h264_level, p->level); + ave_start_put32(cmd, l->h264_max_refs, + (p->refs - 1) | (fw->counted ? (p->cw / 16) << 16 : 0)); + if (fw->counted) { + static const u8 level_idc[] = { + 10, 9, 11, 12, 13, 20, 21, 22, 30, + 31, 32, 40, 41, 42, 50, 51, 52, + }; + + ave_put32(cmd, l->profile_idc, p->high ? 100 : p->profile == 2 ? 66 : 77); + ave_put32(cmd, l->level_idc, level_idc[p->level - 1]); + put_unaligned_le16(p->refs - 1, cmd + l->max_refs_copy); + ave_put32(cmd, l->mb_size, ((p->ch / 16 - 1) << 16) | (p->cw / 16 - 1)); + cmd[l->mb_height] = p->ch / 16; + cmd[l->mb_height + 2] = 1; + cmd[l->mb_height + 3] = 1; + cmd[l->crop_copy] = crop; + put_unaligned_le16(crop_r, cmd + l->crop_right_copy); + put_unaligned_le16(crop_b, cmd + l->crop_bottom_copy); + cmd[l->codec_byte] = !cq; + /* CABAC flag in the second, packed picture structure. */ + cmd[l->cabac_copy] = p->profile != 2; + cmd[l->transform_copy] = p->high; + } + ave_start_put32(cmd, l->h264_width_mbs, p->cw / 16 - 1); + ave_start_put32(cmd, l->h264_height_mbs, p->ch / 16 - 1); + ave_start_put32(cmd, l->h264_crop, crop); + ave_start_put32(cmd, l->h264_crop_left, 0); + ave_start_put32(cmd, l->h264_crop_right, crop_r); + ave_start_put32(cmd, l->h264_crop_top, 0); + ave_start_put32(cmd, l->h264_crop_bottom, crop_b); + ave_start_put32(cmd, l->h264_cabac, p->profile != 2); + cmd[l->h264_8x8] = p->high; return; } - ave_put32(cmd, AVE_START_HEVC_PAD, 1 | + ave_start_put32(cmd, l->hevc_pad, 1 | (p->cw > p->width ? BIT(16) | BIT(18) : 0) | (p->ch > p->height ? BIT(17) | BIT(19) : 0)); - ave_put32(cmd, AVE_START_HEVC_PROFILE_A, p->profile); - ave_put32(cmd, AVE_START_HEVC_PROFILE_B, p->profile); - cmd[AVE_START_HEVC_COMPAT_A] = 1; - cmd[AVE_START_HEVC_COMPAT_A + 1] = 1; - cmd[AVE_START_HEVC_COMPAT_B] = 1; - cmd[AVE_START_HEVC_COMPAT_B + 1] = 1; - cmd[AVE_START_HEVC_8BIT_A] = 1; - cmd[AVE_START_HEVC_8BIT_B] = 1; + ave_start_put32(cmd, l->hevc_profile_a, p->profile); + ave_start_put32(cmd, l->hevc_profile_b, p->profile); + cmd[l->hevc_compat_a] = depth == 8; + cmd[l->hevc_compat_a + 1] = 1; + cmd[l->hevc_compat_b] = depth == 8; + cmd[l->hevc_compat_b + 1] = 1; + cmd[l->hevc_8bit_a] = depth == 8; + cmd[l->hevc_8bit_b] = depth == 8; /* Rate-control words that only change in constant-quality mode */ - ave_put32(cmd, AVE_START_HEVC_RC_270C, 2); - ave_put32(cmd, AVE_START_HEVC_RC_9604, 0); - ave_put32(cmd, AVE_START_HEVC_RC_9608, 0x10000); - ave_put32(cmd, AVE_START_HEVC_LEVEL_A, p->level); - ave_put32(cmd, AVE_START_HEVC_LEVEL_B, p->level); - ave_put32(cmd, AVE_START_HEVC_WIDTH, p->cw); - ave_put32(cmd, AVE_START_HEVC_HEIGHT, p->ch); - ave_put32(cmd, AVE_START_HEVC_WIDTH_CTB, DIV_ROUND_UP(p->cw, 32)); - ave_put32(cmd, AVE_START_HEVC_HEIGHT_CTB, DIV_ROUND_UP(p->ch, 32)); - ave_put32(cmd, AVE_START_HEVC_CONF, crop); - ave_put32(cmd, AVE_START_HEVC_CONF_LEFT, 0); - ave_put32(cmd, AVE_START_HEVC_CONF_RIGHT, crop_r); - ave_put32(cmd, AVE_START_HEVC_CONF_TOP, 0); - ave_put32(cmd, AVE_START_HEVC_CONF_BOTTOM, crop_b); - ave_put32(cmd, AVE_START_HEVC_DEPTH_LUMA, 0); - ave_put32(cmd, AVE_START_HEVC_DEPTH_CHROMA, 0); + ave_start_put32(cmd, l->hevc_cq, cq); + ave_start_put32(cmd, l->hevc_cq_copy, cq); + ave_start_put32(cmd, l->hevc_cq_codec, cq); + ave_start_put32(cmd, l->hevc_cq_picture, cq ? 0x100 : 0); + ave_start_put32(cmd, l->hevc_rc_270c, 2); + ave_start_put32(cmd, l->hevc_rc_9604, cq ? 2 : 0); + ave_start_put32(cmd, l->hevc_rc_9608, cq ? 0 : 0x10000); + ave_start_put32(cmd, l->hevc_level_a, p->level); + ave_start_put32(cmd, l->hevc_level_b, p->level); + ave_start_put32(cmd, l->hevc_width, p->cw); + ave_start_put32(cmd, l->hevc_height, p->ch); + ave_start_put32(cmd, l->hevc_width_ctb, DIV_ROUND_UP(p->cw, 32)); + ave_start_put32(cmd, l->hevc_height_ctb, DIV_ROUND_UP(p->ch, 32)); + ave_start_put32(cmd, l->hevc_conf, crop); + ave_start_put32(cmd, l->hevc_conf_left, 0); + ave_start_put32(cmd, l->hevc_conf_right, crop_r); + ave_start_put32(cmd, l->hevc_conf_top, 0); + ave_start_put32(cmd, l->hevc_conf_bottom, crop_b); + ave_start_put32(cmd, l->hevc_depth_luma, depth - 8); + ave_start_put32(cmd, l->hevc_depth_chroma, depth - 8); /* Reference structure: B-frame pyramid or low-delay P (ave-image.c) */ - ave_put32(cmd, AVE_START_HEVC_REFS, p->bframes ? 4 : 1); - ave_put32(cmd, AVE_START_HEVC_DPB_MINUS1, 4); - ave_put32(cmd, AVE_START_HEVC_REORDER, p->bframes ? 2 : 0); - ave_put32(cmd, AVE_START_HEVC_REORDER_COPY, p->bframes ? 2 : 0); - ave_put32(cmd, AVE_START_HEVC_REORDER_ON, p->bframes); + ave_start_put32(cmd, l->hevc_refs, p->bframes ? 4 : 1); + ave_start_put32(cmd, l->hevc_dpb_minus1, 4); + ave_start_put32(cmd, l->hevc_reorder, p->bframes ? 2 : 0); + ave_start_put32(cmd, l->hevc_reorder_copy, p->bframes ? 2 : 0); + ave_start_put32(cmd, l->hevc_reorder_on, p->bframes); } int ave_cmd_start(struct ave_ctx *ctx) @@ -483,19 +448,22 @@ int ave_cmd_start(struct ave_ctx *ctx) struct ave_dev *ave = ctx->ave; struct ave_session *sess = ctx->sess; const struct ave_params *p = &sess->p; - size_t len = p->codec == AVE_CODEC_HEVC ? AVE_START_HEVC_BYTES : AVE_START_H264_BYTES; + const struct ave_fw_layout *fw = ave->layout; + size_t len = fw->bytes[p->codec - 1][AVE_IMAGE_START]; u8 *cmd; int ret; + if (!p->refs || p->refs > AVE_MAX_REFS || !p->slots || p->slots > fw->max_slots) + return -EINVAL; cmd = ave_ipc_cmd(ave, AVE_BUF_CMD, len); if (!cmd) return -EBUSY; - ave_image_start(cmd, len, p->codec, p->reorder_off, + ave_image_start(cmd, len, fw, p->codec, p->reorder_off, p->codec == AVE_CODEC_HEVC && !p->bframes); ave_cmd_header(ctx, cmd, AVE_OP_START, AVE_OP_START, AVE_SESSION_ARG0, ave_session_setup_arg2(ctx)); - ave_start_surfaces(sess, cmd); - ave_start_config(&sess->p, cmd); + ave_start_surfaces(sess, cmd, fw); + ave_start_config(&sess->p, cmd, fw); ave_debugfs_save(ave, p->codec, AVE_IMAGE_START, cmd, len); ret = ave_ipc_submit(ave, AVE_BUF_CMD, len, AVE_CMD_SESSION_TIMEOUT_US, true); @@ -507,63 +475,6 @@ int ave_cmd_start(struct ave_ctx *ctx) /* Process (id 8) */ -struct ave_proc_layout { - u32 first_frame; - u32 y, uv; /* {u64 address, u32 size, u32 stride} */ - u32 in_w, in_h; - u32 coded_cap; - u32 fps_int; - u32 bitrate; - u32 limits, limit_bytes, limit_secs; - u32 const5; - u32 fps; /* f64 */ - u32 timescale; /* u64 */ - u32 slot; - u32 coded, meta; /* u64 addresses */ - u32 ordinal; - u32 pts; /* u64 */ - u32 force_key; /* 1 on a forced key frame */ - u32 force_type; /* 2 on a forced key frame */ - u32 frame_buf; /* H.264 only */ - u32 shdr_table; /* HEVC only */ - u32 crop; /* HEVC only */ - u32 low_delay; /* HEVC only: 3 with reordering off */ -}; - -static const struct ave_proc_layout ave_proc_h264 = { - .first_frame = 0x12c0, - .y = 0x5830, .uv = 0x5850, - .in_w = 0x58e8, .in_h = 0x58ec, - .coded_cap = 0x61c8, - .fps_int = 0x1610, - .bitrate = 0x1620, - .limits = 0x1628, .limit_bytes = 0x1630, .limit_secs = 0x1638, - .const5 = 0x6238, - .fps = 0x6268, .timescale = 0x6278, - .slot = 0x61b4, .coded = 0x61b8, .meta = 0x61c0, - .ordinal = 0x6228, .pts = 0x6270, - .force_key = 0x1658, .force_type = 0x29f8, - .frame_buf = 0x58c8, -}; - -static const struct ave_proc_layout ave_proc_hevc = { - .first_frame = 0x5d90, - .y = 0xa300, .uv = 0xa320, - .in_w = 0xa3b8, .in_h = 0xa3bc, - .coded_cap = 0xac98, - .fps_int = 0x60e0, - .bitrate = 0x60f0, - .limits = 0x60f8, .limit_bytes = 0x6100, .limit_secs = 0x6108, - .const5 = 0xad08, - .fps = 0xad38, .timescale = 0xad48, - .slot = 0xac84, .coded = 0xac88, .meta = 0xac90, - .ordinal = 0xacf8, .pts = 0xad40, - .force_key = 0x6128, .force_type = 0x74c8, - .shdr_table = 0xd88, - .crop = 0x74dc, - .low_delay = 0xcf8, -}; - #define AVE_PROC_TIMESCALE 14400000ULL #define AVE_PROC_SHDR_ENTRIES 256 #define AVE_PROC_SHDR_STRIDE 0x400 @@ -576,24 +487,29 @@ int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff struct ave_dev *ave = ctx->ave; struct ave_session *sess = ctx->sess; const struct ave_params *p = &sess->p; - const struct ave_proc_layout *l = p->codec == AVE_CODEC_HEVC ? &ave_proc_hevc : - &ave_proc_h264; - size_t len = p->codec == AVE_CODEC_HEVC ? AVE_PROCESS_HEVC_BYTES : AVE_PROCESS_H264_BYTES; - struct ave_slot *s = &sess->slot[slot]; + const struct ave_fw_layout *fw = ave->layout; + const struct ave_proc_layout *l = &fw->process[p->codec - 1]; + size_t len = fw->bytes[p->codec - 1][AVE_IMAGE_PROCESS]; + struct ave_slot *s; unsigned int buf = AVE_BUF_PROC(sess->index * AVE_MAX_SLOTS + slot); dma_addr_t y = input; u64 pts; u8 *cmd; int ret; + if (slot >= p->slots || p->slots > fw->max_slots) + return -EINVAL; + s = &sess->slot[slot]; + if ((!l->force_key && force_key) || (!l->bitrate && bitrate != p->bitrate)) + return -EOPNOTSUPP; ret = ave_ipc_wait_ack(ave, buf); if (ret) return ret; cmd = ave_ipc_cmd(ave, buf, len); if (!cmd) return -EBUSY; - ave_image_process(cmd, len, p->codec); - ave_cmd_header(ctx, cmd, AVE_OP_PROCESS, AVE_PROC_SUBTYPE + slot, 0, + ave_image_process(cmd, len, fw, p->codec); + ave_cmd_header(ctx, cmd, AVE_OP_PROCESS, fw->process_subtype + slot, 0, ave_session_arg2(ctx)); /* Input picture: NV12, chroma right after p->height luma lines. */ @@ -610,9 +526,11 @@ int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff ave_put32(cmd, l->coded_cap, p->coded_cap); ave_put32(cmd, l->fps_int, p->fps_int); /* The bit rate may change while streaming; it takes effect here. */ - ave_put32(cmd, l->bitrate, bitrate); - ave_put32(cmd, l->limits, p->limit_bytes ? 1 : 0); - if (p->limit_bytes) { + if (l->bitrate) + ave_put32(cmd, l->bitrate, bitrate); + if (l->limits) + ave_put32(cmd, l->limits, p->limit_bytes ? 1 : 0); + if (l->limit_bytes && p->limit_bytes) { ave_put64(cmd, l->limit_bytes, ave_f64(p->limit_bytes, 1)); ave_put64(cmd, l->limit_secs, ave_f64(1, 1)); } @@ -627,8 +545,14 @@ int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff ave_put32(cmd, l->slot, slot); ave_put64(cmd, l->coded, s->coded.dma); ave_put64(cmd, l->meta, s->meta.dma); + if (l->meta_size) + ave_put32(cmd, l->meta_size, s->meta.size); + if (l->extra) + ave_put_buf(cmd, l->extra, &s->extra, SZ_16K); + if (l->frame_size) + ave_put32(cmd, l->frame_size, p->rc_mode == 3 ? 0 : s->aux.size); if (l->frame_buf) - ave_put64(cmd, l->frame_buf, s->aux.dma); + ave_put64(cmd, l->frame_buf, fw->counted && p->rc_mode == 3 ? 0 : s->aux.dma); if (l->shdr_table) for (unsigned int i = 0; i < AVE_PROC_SHDR_ENTRIES; i++) ave_put64(cmd, l->shdr_table + 8 * i, @@ -636,14 +560,16 @@ int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buff ave_put32(cmd, l->ordinal, sess->frames); pts = mul_u64_u64_div_u64((u64)sess->frames * p->fps_den, AVE_PROC_TIMESCALE, p->fps_num); ave_put64(cmd, l->pts, pts); - ave_put32(cmd, l->force_key, force_key); - ave_put32(cmd, l->force_type, force_key ? AVE_PROC_FORCED_TYPE : 0); + if (l->force_key) + ave_put32(cmd, l->force_key, force_key); + if (l->force_type) + ave_put32(cmd, l->force_type, force_key ? AVE_PROC_FORCED_TYPE : 0); if (l->low_delay) ave_put32(cmd, l->low_delay, p->bframes ? 0 : AVE_PROC_LOW_DELAY); if (!sess->frames) ave_debugfs_save(ave, p->codec, AVE_IMAGE_PROCESS, cmd, len); - ret = ave_ipc_submit(ave, buf, len, AVE_CMD_SESSION_TIMEOUT_US, false); + ret = ave_ipc_submit(ave, buf, len, fw->process_timeout_us, false); if (ret) return ret; s->src = src; diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c index da85e5dd25757..43c9ddb089e83 100644 --- a/drivers/media/platform/apple/ave/ave-drv.c +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -53,6 +53,13 @@ static const struct ave_variant ave_t8112 = { .lfs_ref_scale = 1, }; +/* CRC, boot-config words and coprocessor start registers remain unqualified. */ +static const struct ave_variant ave_t6041 = { + .fw_text_size = 0x14c000, + .fw_data_size = 0x12c000, + .lfs_ref_scale = 2, +}; + static const struct ave_variant ave_t8140 = { .regs = &ave_coproc_v40, .fw_text_size = 0x14c000, @@ -60,7 +67,7 @@ static const struct ave_variant ave_t8140 = { .boot_device_id = 29, .device_type = 27, .lfs_ref_scale = 2, - .encoding_text_crc = AVE_FW_TEXT_CRC_9003_78_0, + .boot_qualified = true, }; /* @@ -68,17 +75,12 @@ static const struct ave_variant ave_t8140 = { * last_process0_{h264,hevc} the first Process of the last session, so the * generated commands can be compared with reference captures. */ -static const size_t ave_image_bytes[2][AVE_IMAGE_COUNT] = { - { AVE_START_H264_BYTES, AVE_PROCESS_H264_BYTES }, - { AVE_START_HEVC_BYTES, AVE_PROCESS_HEVC_BYTES }, -}; - void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, enum ave_image kind, const void *cmd, size_t len) { void *dst = ave->dump[codec - 1][kind]; - if (dst && len == ave_image_bytes[codec - 1][kind]) + if (ave->layout && dst && len == ave->layout->bytes[codec - 1][kind]) memcpy(dst, cmd, len); } @@ -89,21 +91,22 @@ static void ave_debugfs_init(struct ave_dev *ave) { "last_start_hevc", "last_process0_hevc" }, }; - if (!IS_ENABLED(CONFIG_DEBUG_FS)) + if (!IS_ENABLED(CONFIG_DEBUG_FS) || !ave->layout) return; ave->debugfs = debugfs_create_dir("apple-ave", NULL); for (unsigned int c = 0; c < 2; c++) for (unsigned int k = 0; k < AVE_IMAGE_COUNT; k++) { struct debugfs_blob_wrapper *blob; + size_t size = ave->layout->bytes[c][k]; blob = devm_kzalloc(ave->dev, sizeof(*blob), GFP_KERNEL); - ave->dump[c][k] = devm_kzalloc(ave->dev, ave_image_bytes[c][k], GFP_KERNEL); + ave->dump[c][k] = devm_kzalloc(ave->dev, size, GFP_KERNEL); if (!blob || !ave->dump[c][k]) { ave->dump[c][k] = NULL; continue; } blob->data = ave->dump[c][k]; - blob->size = ave_image_bytes[c][k]; + blob->size = ave->layout->bytes[c][k]; debugfs_create_blob(names[c][k], 0400, ave->debugfs, blob); } } @@ -256,6 +259,7 @@ static const struct dev_pm_ops ave_pm_ops = { static const struct of_device_id ave_of_match[] = { { .compatible = "apple,t6000-ave", .data = &ave_t6000 }, + { .compatible = "apple,t6041-ave", .data = &ave_t6041 }, { .compatible = "apple,t8112-ave", .data = &ave_t8112 }, { .compatible = "apple,t8140-ave", .data = &ave_t8140 }, { } diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c index fb7563252ab8c..2e1be782df86a 100644 --- a/drivers/media/platform/apple/ave/ave-fw.c +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -27,8 +27,8 @@ #include "ave-regs.h" /* - * Firmware builds whose command layouts this driver implements, by the CRC32 - * of their TEXT segment. The command formats change between builds. + * Legacy TEXT CRCs accepted at T8140 probe. Encoding separately requires + * a layout from ave_layout_for_crc(); command formats change between builds. */ static const u32 ave_known_text_crc[] = { 0x2ce7359e, /* AppleAVE2FW 9003.78.0 */ @@ -120,7 +120,7 @@ int ave_fw_probe(struct ave_dev *ave) { struct device *dev = ave->dev; u32 text_crc, data_crc, want; - bool known = !ave->variant->encoding_text_crc; + bool known = !ave->variant->boot_qualified; int ret; if (of_count_phandle_with_args(dev->of_node, "memory-region", NULL) != 2) @@ -156,6 +156,7 @@ int ave_fw_probe(struct ave_dev *ave) return dev_err_probe(dev, -ENODEV, "unsupported firmware (text CRC %08x)\n", text_crc); ave->text_crc = text_crc; + ave->layout = ave_layout_for_crc(text_crc); ave->data_pristine = vmalloc(ave->data.size); if (!ave->data_pristine) @@ -176,9 +177,9 @@ int ave_fw_probe(struct ave_dev *ave) ave->data_crc = data_crc; ave->fw_state = AVE_FW_OFF; - if (!ave->variant->encoding_text_crc) - dev_info(dev, "probe only: device type %u firmware TEXT %08x DATA %08x needs qualification\n", - ave->variant->device_type, text_crc, data_crc); + if (!ave->variant->boot_qualified || !ave->layout) + dev_info(dev, "probe only: firmware TEXT %08x DATA %08x needs qualification\n", + text_crc, data_crc); return 0; free_data: @@ -243,11 +244,11 @@ int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, unsigned long */ int ave_fw_check_build(struct ave_dev *ave) { - if (ave->variant->encoding_text_crc && - ave->text_crc == ave->variant->encoding_text_crc) + if (ave->variant->boot_qualified && ave->variant->regs && ave->layout && + ave->layout == ave_layout_for_crc(ave->text_crc)) return 0; - dev_info(ave->dev, "encoding refused: device type %u firmware TEXT %08x DATA %08x has no qualified command constants/MCC DSID\n", - ave->variant->device_type, ave->text_crc, ave->data_crc); + dev_info(ave->dev, "encoding refused: firmware TEXT %08x DATA %08x has no qualified command layout/boot sequence\n", + ave->text_crc, ave->data_crc); return -ENOTSUPP; } diff --git a/drivers/media/platform/apple/ave/ave-image.c b/drivers/media/platform/apple/ave/ave-image.c index 273dd2a69b013..853d0a5bccc84 100644 --- a/drivers/media/platform/apple/ave/ave-image.c +++ b/drivers/media/platform/apple/ave/ave-image.c @@ -236,6 +236,232 @@ static const struct ave_word ave_hevc_process[] = { W(0xad68, 0xffffffff), }; +/* + * 26A428: independently derived invariant words from the field ledger. + * Computed fields overwrite any invariant default; no command images or + * captured addresses are stored here. This layout has no qualified CRC yet. + */ +static const struct ave_word ave_26a428_h264_start[] = { + /* Invariant bytes of a word whose crop flag is computed. */ + W(0x3ac4, 0x10001), + /* Byte zero stays set while byte three selects the 8x8 transform. */ + W(0x3820, 1), + W(0x50, 0xec000), + W(0x90, 0x2), + W(0xb10, 0x10000), + W(0xb20, 0x10000), + W(0xb30, 0x10000), + W(0xdb8, 0x2), + W(0xfe0, 0x2), + W(0x19b8, 0x1), + W(0x1a00, 0x1), + W(0x1a48, 0x1), + W(0x1a58, 0x4000), + W(0x1bb0, 0x4), + W(0x1bb4, 0x8), + W(0x31f0, 0x1000), + W(0x320c, 0x1e), + W(0x321c, 0x36), + W(0x3234, 0x8), + RUN(0x3248, 4, 0x1), + W(0x3258, 0x1), + W(0x325c, 0xffffffff), + W(0x3298, 0x1), + W(0x35f4, 0x39d), + W(0x35f8, 0x18), + W(0x3600, 0x1e), + W(0x366c, 0x3ff00000), + W(0x3674, 0x3fe00000), + W(0x36f4, 0xbff00000), + W(0x36fc, 0xbff00000), + W(0x3700, 0x1), + RUN(0x3714, 3, 0x1), + W(0x3728, 0x1), + RUN(0x372c, 2, 0x1), + W(0x373c, 0x7), + RUN(0x3780, 2, 0xffffffff), + W(0x37c0, 0x1), + W(0x37c4, 0x80800), + W(0x37cc, 0x60005), + W(0x37d8, 0x1000000), + W(0x37dc, 0x5), + W(0x37e0, 0x20100), + W(0x37e4, 0x20002), + W(0x3810, 0x1010100), + W(0x3814, 0x1a1a0000), + W(0x3818, 0xf3f3f300), + W(0x381c, 0xf3f3f3f3), + W(0x3824, 0xf3f30000), + W(0x3828, 0xf3f3f3f3), + W(0x382c, 0x1f3), + W(0x3834, 0x100), + W(0x3838, 0x1), + W(0x383c, 0x8), + W(0x3848, 0x100), + W(0x3860, 0x1), + W(0x3864, 0x4), + W(0x3874, 0x15), + W(0x3884, 0x1), + W(0x393c, 0x8), + W(0x3aa4, 0x1), + W(0x3ab0, 0x20001), + W(0x3ad0, 0x1000000), + W(0x3ad4, 0x5), + W(0x3ad8, 0x20100), + W(0x3adc, 0x20002), + W(0x3b00, 0xf3000000), + W(0x3b04, 0xf3f3f3f3), + W(0x3b08, 0x1f3f3), + W(0x3b10, 0xf3f3f3f3), + W(0x3b14, 0x1f3f3f3), + W(0x3b1c, 0x1), +}; + +static const struct ave_word ave_26a428_hevc_start[] = { + /* Constraint bytes: the max-8-bit byte is computed. */ + W(0x3a60, 0x01000101), + W(0x6530, 0x01000000), + W(0x50, 0xec000), + W(0x68, 0x2), + W(0x90, 0x2), + W(0x94, 0x5), + W(0xb10, 0x10000), + W(0xb20, 0x10000), + W(0xb30, 0x10000), + W(0xb40, 0x10000), + W(0xb50, 0x10000), + W(0xdb8, 0x2), + W(0xdbc, 0x5), + W(0xfe0, 0x2), + W(0xfe4, 0x5), + W(0x19b8, 0x1), + W(0x19c8, 0x4000), + W(0x1a00, 0x1), + W(0x1a48, 0x1), + W(0x1a58, 0x4000), + RUN(0x1bb0, 2, 0x4), + W(0x31f0, 0x1000), + W(0x320c, 0x1e), + W(0x321c, 0x36), + W(0x3248, 0x2), + RUN(0x324c, 4, 0x1), + RUN(0x325c, 2, 0xffffffff), + W(0x3298, 0x1), + W(0x35f0, 0xa), + W(0x35f4, 0x39d), + W(0x35f8, 0x18), + W(0x3600, 0x1e), + W(0x366c, 0x3ff00000), + W(0x3674, 0x3fe00000), + W(0x36dc, 0x1), + RUN(0x36e0, 2, 0x3), + W(0x36e8, 0x1e), + W(0x36f4, 0xbff00000), + W(0x36fc, 0xbff00000), + W(0x3700, 0x1), + RUN(0x3714, 3, 0x1), + W(0x3728, 0x1), + RUN(0x372c, 2, 0x1), + W(0x373c, 0x7), + W(0x3764, 0x4), + RUN(0x3780, 2, 0xffffffff), + W(0x37b8, 0x100), + W(0x37bc, 0x1), + W(0x37c4, 0x4), + W(0x37cc, 0x100), + W(0x37e4, 0x1), + W(0x37e8, 0x4), + RUN(0x37f8, 2, 0x15), + W(0x3800, 0x2), + W(0x3804, 0x2), + W(0x3808, 0x1), + W(0x38c0, 0x8), + W(0x3a20, 0x10100), + W(0x3a2c, 0x1), + W(0x3a34, 0x1), + W(0x3a5c, 0x1010001), + W(0x3a64, 0x1), + RUN(0x3b8c, 7, 0x1010101), + W(0x3ba8, 0x101), + W(0x3bbc, 0x1010101), + W(0x3bc0, 0x101), + RUN(0x4940, 7, 0x4), + W(0x495c, 0x2), + W(0x4da0, 0x3e9), + W(0x4da4, 0x5dc0), + W(0x64fc, 0x1), + W(0x6504, 0x1), + W(0x652c, 0x1010001), + W(0x6534, 0x1), + W(0x66b0, 0x1), + W(0x66e4, 0x7), + RUN(0x66ec, 7, 0x4), + W(0x6708, 0x2), + W(0x6744, 0x2), + W(0x674c, 0x3), + W(0x6750, 0x1), + W(0x6758, 0x1), + W(0x8004, 0x100), + RUN(0x8008, 2, 0x7), + W(0x8010, 0x1), + W(0x8018, 0x1), + W(0x80a0, 0x1), + W(0x80b4, 0x100), + W(0x80b8, 0x5), + W(0x80c0, 0x1), + RUN(0x80c4, 3, 0x2), + W(0x80f4, 0x3e9), + W(0x80f8, 0x5dc0), + W(0x8104, 0x101), + W(0x8108, 0x1), + W(0x810c, 0x3), + W(0x8110, 0x5), + RUN(0x8114, 3, 0x17), + RUN(0x8128, 2, 0xbebb), + W(0x815c, 0x100), + W(0x9de8, 0x10000), + W(0x9df4, 0x2), + RUN(0x9dfc, 7, 0xfffffff3), + RUN(0x9e1c, 7, 0xfffffff3), + W(0x9e3c, 0x100), + W(0xa64c, 0x10000), + W(0xbf10, 0x2), + W(0xbf50, 0x1), + RUN(0x13eac, 2, 0x1), + W(0x13eb8, 0x1), +}; + +static const struct ave_word ave_26a428_h264_process[] = { + W(0xdc0, 0x1), + W(0x1328, 0x1e), + W(0x25e4, 0x1), + W(0x25ec, 0x1), + W(0x29dc, 0xffffffff), + W(0x5660, 0x5), + W(0x575c, 0x403e0000), + W(0x5768, 0xdbba00), + W(0x58b8, 0x10000), + W(0x58c8, 0x4000), +}; + +static const struct ave_word ave_26a428_hevc_process[] = { + W(0x9a4, 0x1), + W(0x9ac, 0x2), + W(0xaa8, 0x10100), + W(0xb30, 0x10000), + RUN(0xb38, 2, 0x5), + W(0x1568, 0x1), + W(0x1ad0, 0x1e), + W(0x3184, 0xffffffff), + W(0x3194, 0x1), + W(0x5e08, 0x5), + W(0x5f04, 0x403e0000), + W(0x5f10, 0xdbba00), + W(0x5f48, 0xffffffff), + W(0x6060, 0x10000), + W(0x6070, 0x4000), +}; + static void ave_put_words(u8 *cmd, size_t len, const struct ave_word *w, size_t n) { for (size_t i = 0; i < n; i++) @@ -260,9 +486,19 @@ static void ave_start_codec_bytes(u8 *cmd, enum ave_codec codec, bool reorder_of cmd[0x2585] = codec == AVE_CODEC_H264 && reorder_off ? 0 : 1; } -void ave_image_start(u8 *cmd, size_t len, enum ave_codec codec, bool reorder_off, +void ave_image_start(u8 *cmd, size_t len, const struct ave_fw_layout *fw, + enum ave_codec codec, bool reorder_off, bool hevc_low_delay) { + if (fw->counted) { + if (codec == AVE_CODEC_HEVC) + ave_put_words(cmd, len, ave_26a428_hevc_start, + ARRAY_SIZE(ave_26a428_hevc_start)); + else + ave_put_words(cmd, len, ave_26a428_h264_start, + ARRAY_SIZE(ave_26a428_h264_start)); + return; + } ave_put_words(cmd, len, ave_start_common, ARRAY_SIZE(ave_start_common)); if (codec == AVE_CODEC_HEVC) { ave_put_words(cmd, len, ave_hevc_start, ARRAY_SIZE(ave_hevc_start)); @@ -275,8 +511,19 @@ void ave_image_start(u8 *cmd, size_t len, enum ave_codec codec, bool reorder_off ave_start_codec_bytes(cmd, codec, reorder_off); } -void ave_image_process(u8 *cmd, size_t len, enum ave_codec codec) +void ave_image_process(u8 *cmd, size_t len, const struct ave_fw_layout *fw, + enum ave_codec codec) { + if (fw->counted) { + if (codec == AVE_CODEC_HEVC) + ave_put_words(cmd, len, ave_26a428_hevc_process, + ARRAY_SIZE(ave_26a428_hevc_process)); + else + ave_put_words(cmd, len, ave_26a428_h264_process, + ARRAY_SIZE(ave_26a428_h264_process)); + return; + } + if (codec == AVE_CODEC_HEVC) ave_put_words(cmd, len, ave_hevc_process, ARRAY_SIZE(ave_hevc_process)); else diff --git a/drivers/media/platform/apple/ave/ave-layout.c b/drivers/media/platform/apple/ave/ave-layout.c new file mode 100644 index 0000000000000..35761b36190ba --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-layout.c @@ -0,0 +1,494 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Apple AVE command layouts, identified by firmware TEXT CRC. */ +#include "ave.h" + +const struct ave_fw_layout ave_layout_9003_78_0 = { + .name = "9003.78.0 / 25G83", + .bytes = { { 0x3180, 0x63d8 }, { 0x13f28, 0xb1c0 } }, + .hdr_bytes = 0x12c000, + .meta_bytes = 0xc000, + .process_timeout_us = 94400000, + .process_subtype = 0x15, + .max_slots = 5, + .mcc_dsid = 0x12, + .start = { + [0] = { + .ref_a = 0x390, + .ref_b = 0x3a0, + .ref_c = 0x7d0, + .ref_c_alias = 0x7e0, + .ref_d = 0x1130, + .ref_stride = 0x20, + .ref_c_stride = 0x20, + .mb = 0xc10, + .coded = 0xd50, + .meta = 0xf50, + .row_a = 0x1350, + .row_b = 0x1390, + .fixed = 0x13d0, + .hevc_extra = 0x2520, + .tiles = { 0x1410, 0x1430, 0x1490, 0x14b0, 0x1510, + 0x1530, 0x1590, 0x15b0 }, + .codec = 0xc0, + .flag_8bit = 0x208, + .const_214 = 0x214, + .const_218 = 0x218, + .fps = 0x220, + .rc_mode = 0x234, + .bitrate = 0x238, + .init_qp = 0x240, + .limits = 0x250, + .limit_bytes = 0x258, + .limit_secs = 0x260, + .f64_288 = 0x288, + .f64_290 = 0x290, + .min_qp = 0x298, + .max_qp = 0x29c, + .gop_flags = 0x2a0, + .p_frames = 0x2ac, + .max_b = 0x2b0, + .ref_depth = 0x2b4, + .idr_interval = 0x2b8, + .f64_2c0 = 0x2c0, + .f64_2c8 = 0x2c8, + .rc_flags = 0x300, + .min_qp2 = 0x308, + .max_qp2 = 0x30c, + .rc_310 = 0x310, + .size_class = 0x320, + .coded_w = 0x368, + .coded_h = 0x36c, + .codec_flags = 0x370, + .h264_380 = 0x380, + .h264_384 = 0x384, + .codec_2588 = 0x2588, + .bit_depth = 0x2598, + .coded_h2 = 0x25dc, + .h264_profile = 0x291c, + .h264_level = 0x2938, + .h264_max_refs = 0x2d44, + .h264_width_mbs = 0x2d4c, + .h264_height_mbs = 0x2d50, + .h264_crop = 0x2db4, + .h264_crop_left = 0x2db8, + .h264_crop_right = 0x2dbc, + .h264_crop_top = 0x2dc0, + .h264_crop_bottom = 0x2dc4, + .h264_cabac = 0x2fd8, + .h264_8x8 = 0x3023, + .hevc_pad = 0x26e0, + .hevc_rc_270c = 0x270c, + .hevc_rc_9604 = 0x9604, + .hevc_rc_9608 = 0x9608, + .hevc_profile_a = 0x2938, + .hevc_compat_a = 0x293d, + .hevc_8bit_a = 0x2962, + .hevc_level_a = 0x296c, + .hevc_profile_b = 0x5904, + .hevc_compat_b = 0x5909, + .hevc_8bit_b = 0x592e, + .hevc_level_b = 0x5938, + .hevc_width = 0x5ab4, + .hevc_height = 0x5ab8, + .hevc_width_ctb = 0x5abc, + .hevc_height_ctb = 0x5ac0, + .hevc_conf = 0x5ac4, + .hevc_conf_left = 0x5ac8, + .hevc_conf_right = 0x5acc, + .hevc_conf_top = 0x5ad0, + .hevc_conf_bottom = 0x5ad4, + .hevc_depth_luma = 0x5ad8, + .hevc_depth_chroma = 0x5adc, + .hevc_refs = 0x259c, + .hevc_reorder_copy = 0x2e54, + .hevc_dpb_minus1 = 0x5b00, + .hevc_reorder = 0x5b04, + .hevc_reorder_on = 0x95fc, + }, + [1] = { + .ref_a = 0x390, + .ref_b = 0x3a0, + .ref_c = 0x7d0, + .ref_c_alias = 0x7e0, + .ref_d = 0x1130, + .ref_stride = 0x20, + .ref_c_stride = 0x20, + .mb = 0xc10, + .coded = 0xd50, + .meta = 0xf50, + .row_a = 0x1350, + .row_b = 0x1390, + .fixed = 0x13d0, + .hevc_extra = 0x2520, + .tiles = { 0x1410, 0x1430, 0x1490, 0x14b0, 0x1510, + 0x1530, 0x1590, 0x15b0 }, + .codec = 0xc0, + .flag_8bit = 0x208, + .const_214 = 0x214, + .const_218 = 0x218, + .fps = 0x220, + .rc_mode = 0x234, + .bitrate = 0x238, + .init_qp = 0x240, + .limits = 0x250, + .limit_bytes = 0x258, + .limit_secs = 0x260, + .f64_288 = 0x288, + .f64_290 = 0x290, + .min_qp = 0x298, + .max_qp = 0x29c, + .gop_flags = 0x2a0, + .p_frames = 0x2ac, + .max_b = 0x2b0, + .ref_depth = 0x2b4, + .idr_interval = 0x2b8, + .f64_2c0 = 0x2c0, + .f64_2c8 = 0x2c8, + .rc_flags = 0x300, + .min_qp2 = 0x308, + .max_qp2 = 0x30c, + .rc_310 = 0x310, + .size_class = 0x320, + .coded_w = 0x368, + .coded_h = 0x36c, + .codec_flags = 0x370, + .h264_380 = 0x380, + .h264_384 = 0x384, + .codec_2588 = 0x2588, + .bit_depth = 0x2598, + .coded_h2 = 0x25dc, + .h264_profile = 0x291c, + .h264_level = 0x2938, + .h264_max_refs = 0x2d44, + .h264_width_mbs = 0x2d4c, + .h264_height_mbs = 0x2d50, + .h264_crop = 0x2db4, + .h264_crop_left = 0x2db8, + .h264_crop_right = 0x2dbc, + .h264_crop_top = 0x2dc0, + .h264_crop_bottom = 0x2dc4, + .h264_cabac = 0x2fd8, + .h264_8x8 = 0x3023, + .hevc_pad = 0x26e0, + .hevc_rc_270c = 0x270c, + .hevc_rc_9604 = 0x9604, + .hevc_rc_9608 = 0x9608, + .hevc_profile_a = 0x2938, + .hevc_compat_a = 0x293d, + .hevc_8bit_a = 0x2962, + .hevc_level_a = 0x296c, + .hevc_profile_b = 0x5904, + .hevc_compat_b = 0x5909, + .hevc_8bit_b = 0x592e, + .hevc_level_b = 0x5938, + .hevc_width = 0x5ab4, + .hevc_height = 0x5ab8, + .hevc_width_ctb = 0x5abc, + .hevc_height_ctb = 0x5ac0, + .hevc_conf = 0x5ac4, + .hevc_conf_left = 0x5ac8, + .hevc_conf_right = 0x5acc, + .hevc_conf_top = 0x5ad0, + .hevc_conf_bottom = 0x5ad4, + .hevc_depth_luma = 0x5ad8, + .hevc_depth_chroma = 0x5adc, + .hevc_refs = 0x259c, + .hevc_reorder_copy = 0x2e54, + .hevc_dpb_minus1 = 0x5b00, + .hevc_reorder = 0x5b04, + .hevc_reorder_on = 0x95fc, + }, + }, + .process = { + [0] = { + .first_frame = 0x12c0, + .y = 0x5830, + .uv = 0x5850, + .in_w = 0x58e8, + .in_h = 0x58ec, + .coded_cap = 0x61c8, + .fps_int = 0x1610, + .bitrate = 0x1620, + .limits = 0x1628, + .limit_bytes = 0x1630, + .limit_secs = 0x1638, + .const5 = 0x6238, + .fps = 0x6268, + .timescale = 0x6278, + .slot = 0x61b4, + .coded = 0x61b8, + .meta = 0x61c0, + .ordinal = 0x6228, + .pts = 0x6270, + .force_key = 0x1658, + .force_type = 0x29f8, + .frame_buf = 0x58c8, + }, + [1] = { + .first_frame = 0x5d90, + .y = 0xa300, + .uv = 0xa320, + .in_w = 0xa3b8, + .in_h = 0xa3bc, + .coded_cap = 0xac98, + .fps_int = 0x60e0, + .bitrate = 0x60f0, + .limits = 0x60f8, + .limit_bytes = 0x6100, + .limit_secs = 0x6108, + .const5 = 0xad08, + .fps = 0xad38, + .timescale = 0xad48, + .slot = 0xac84, + .coded = 0xac88, + .meta = 0xac90, + .ordinal = 0xacf8, + .pts = 0xad40, + .force_key = 0x6128, + .force_type = 0x74c8, + .shdr_table = 0xd88, + .crop = 0x74dc, + .low_delay = 0xcf8, + }, + }, +}; + +const struct ave_fw_layout ave_layout_26a428 = { + .name = "26A428 (CRC unqualified)", + .bytes = { { 0x3b40, 0x6df8 }, { 0x13ee0, 0x78b8 } }, + .hdr_bytes = 0xec000, + .meta_bytes = 0x10000, + .process_timeout_us = 46400000, + .process_subtype = 0x13, + .max_slots = 7, + .mcc_dsid = 0x12, + .counted = true, + .start = { + [0] = { + .ref_a = 0x98, + .ref_b = 0xa8, + .ref_c = 0xfe8, + .ref_c_alias = 0xff8, + .ref_d = 0xdc0, + .ref_e = 0xb8, + .ref_f = 0xc8, + .ref_stride = 0x40, + .ref_c_stride = 0x40, + .mb = 0x1870, + .coded = 0x920, + .meta = 0xb08, + .row_a = 0x19c0, + .row_b = 0x1a08, + .fixed = 0x1a50, + .hevc_extra = 0x70, + .tiles = { 0x1bb8, 0x1bd8, 0x1db8, 0x1dd8, 0x1bf8, + 0x1c18, 0x1df8, 0x1e18 }, + .refs_count = 0x94, + .work_count = 0xdbc, + .lfs_count = 0xfe4, + .slots_count = 0x918, + .meta_count = 0xb00, + .mb_count = 0x1868, + .row_a_count = 0x19b8, + .row_b_count = 0x1a00, + .fixed_count = 0x1a48, + .tiles_count = 0x1bb4, + .codec = 0x3248, + .flag_8bit = 0x35e8, + .const_214 = 0x35f4, + .const_218 = 0x35f8, + .fps = 0x3600, + .rc_mode = 0x3614, + .bitrate = 0x3618, + .init_qp = 0x3620, + .limits = 0x3630, + .limit_bytes = 0x3638, + .limit_secs = 0x3640, + .f64_288 = 0x3668, + .f64_290 = 0x3670, + .min_qp = 0x3678, + .max_qp = 0x367c, + .gop_flags = 0x36d0, + .p_frames = 0x36dc, + .max_b = 0x36e0, + .ref_depth = 0x36e4, + .idr_interval = 0x36e8, + .f64_2c0 = 0x36f0, + .f64_2c8 = 0x36f8, + .rc_flags = 0x3734, + .min_qp2 = 0x3740, + .max_qp2 = 0x3744, + .rc_310 = 0x3748, + .size_class = 0x3760, + .coded_w = 0x322c, + .coded_h = 0x3230, + .coded_h2 = 0x32a0, + .bit_depth = 0x383c, + .h264_profile = 0x37b0, + .h264_level = 0x37b8, + .h264_max_refs = 0x37d0, + .h264_crop = 0x3800, + .h264_crop_right = 0x3804, + .h264_crop_bottom = 0x3808, + .h264_cabac = 0x380c, + .h264_8x8 = 0x3823, + .geometry_mode = 0x31f8, + .geometry_depth = 0x3234, + .profile_idc = 0x3a98, + .level_idc = 0x3aa0, + .max_refs_copy = 0x3abe, + .mb_size = 0x3ac0, + .mb_height = 0x37d4, + .crop_copy = 0x3ac7, + .crop_right_copy = 0x3aca, + .crop_bottom_copy = 0x3ace, + .codec_byte = 0x3830, + .cabac_copy = 0x3af9, + .transform_copy = 0x3b0d, + }, + [1] = { + .ref_a = 0x98, + .ref_b = 0xa8, + .ref_c = 0xfe8, + .ref_c_alias = 0xff8, + .ref_d = 0xdc0, + .ref_e = 0xb8, + .ref_f = 0xc8, + .ref_stride = 0x40, + .ref_c_stride = 0x40, + .mb = 0x1870, + .coded = 0x920, + .meta = 0xb08, + .row_a = 0x19c0, + .row_b = 0x1a08, + .fixed = 0x1a50, + .hevc_extra = 0x70, + .tiles = { 0x1bb8, 0x1bd8, 0x1db8, 0x1dd8, 0x1bf8, + 0x1c18, 0x1df8, 0x1e18 }, + .refs_count = 0x94, + .work_count = 0xdbc, + .lfs_count = 0xfe4, + .slots_count = 0x918, + .meta_count = 0xb00, + .mb_count = 0x1868, + .row_a_count = 0x19b8, + .row_b_count = 0x1a00, + .fixed_count = 0x1a48, + .tiles_count = 0x1bb4, + .codec = 0x3248, + .flag_8bit = 0x35e8, + .const_214 = 0x35f4, + .const_218 = 0x35f8, + .fps = 0x3600, + .rc_mode = 0x3614, + .bitrate = 0x3618, + .init_qp = 0x3620, + .limits = 0x3630, + .limit_bytes = 0x3638, + .limit_secs = 0x3640, + .f64_288 = 0x3668, + .f64_290 = 0x3670, + .min_qp = 0x3678, + .max_qp = 0x367c, + .gop_flags = 0x36d0, + .p_frames = 0x36dc, + .max_b = 0x36e0, + .ref_depth = 0x36e4, + .idr_interval = 0x36e8, + .f64_2c0 = 0x36f0, + .f64_2c8 = 0x36f8, + .rc_flags = 0x3734, + .min_qp2 = 0x3740, + .max_qp2 = 0x3744, + .rc_310 = 0x3748, + .size_class = 0x3760, + .coded_w = 0x322c, + .coded_h = 0x3230, + .coded_h2 = 0x32a0, + .bit_depth = 0x37c0, + .hevc_pad = 0x37ec, + .hevc_refs = 0x37c4, + .hevc_rc_9604 = 0x9dec, + .hevc_rc_9608 = 0x9df0, + .hevc_profile_a = 0x3a38, + .hevc_compat_a = 0x3a3d, + .hevc_8bit_a = 0x3a62, + .hevc_level_a = 0x3a6c, + .hevc_profile_b = 0x6508, + .hevc_compat_b = 0x650d, + .hevc_8bit_b = 0x6532, + .hevc_level_b = 0x653c, + .hevc_width = 0x66b8, + .hevc_height = 0x66bc, + .hevc_width_ctb = 0x66c0, + .hevc_height_ctb = 0x66c4, + .hevc_conf = 0x66c8, + .hevc_conf_left = 0x66cc, + .hevc_conf_right = 0x66d0, + .hevc_conf_top = 0x66d4, + .hevc_conf_bottom = 0x66d8, + .hevc_depth_luma = 0x66dc, + .hevc_depth_chroma = 0x66e0, + .hevc_reorder_copy = 0x495c, + .hevc_dpb_minus1 = 0x6704, + .hevc_reorder = 0x6708, + .geometry_mode = 0x31f8, + .geometry_depth = 0x3234, + .hevc_cq = 0x3708, + .hevc_cq_copy = 0xbefc, + .hevc_cq_codec = 0x37c8, + .hevc_cq_picture = 0x9e38, + }, + }, + .process = { + [0] = { + .first_frame = 0xdc0, + .fps_int = 0x1328, + .in_w = 0x5648, + .in_h = 0x564c, + .ordinal = 0x5650, + .const5 = 0x5660, + .fps = 0x5758, + .pts = 0x5760, + .timescale = 0x5768, + .slot = 0x57b8, + .y = 0x57c0, + .uv = 0x57e0, + .coded = 0x58a0, + .coded_cap = 0x58a8, + .meta = 0x58b0, + .meta_size = 0x58b8, + .extra = 0x58c0, + .frame_buf = 0x58d8, + .frame_size = 0x58e0, + }, + [1] = { + .first_frame = 0x1568, + .fps_int = 0x1ad0, + .in_w = 0x5df0, + .in_h = 0x5df4, + .ordinal = 0x5df8, + .const5 = 0x5e08, + .fps = 0x5f00, + .pts = 0x5f08, + .timescale = 0x5f10, + .slot = 0x5f60, + .y = 0x5f68, + .uv = 0x5f88, + .coded = 0x6048, + .coded_cap = 0x6050, + .meta = 0x6058, + .meta_size = 0x6060, + .extra = 0x6068, + .crop = 0x3190, + }, + }, +}; + +const struct ave_fw_layout *ave_layout_for_crc(u32 crc) +{ + if (crc == AVE_FW_TEXT_CRC_9003_78_0) + return &ave_layout_9003_78_0; + /* 26A428 has no verified loader TEXT CRC: no placeholder or fallback. */ + return NULL; +} diff --git a/drivers/media/platform/apple/ave/ave-layout.h b/drivers/media/platform/apple/ave/ave-layout.h new file mode 100644 index 0000000000000..d43225a9af5dd --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-layout.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _APPLE_AVE_LAYOUT_H_ +#define _APPLE_AVE_LAYOUT_H_ + +/* Zero offsets denote fields absent from a firmware's wire format. */ +struct ave_start_layout { + u16 ref_a, ref_b, ref_c, ref_c_alias, ref_d; + u16 ref_e, ref_f, ref_stride, ref_c_stride; + u16 mb, coded, meta, row_a, row_b, fixed, hevc_extra; + u16 tiles[8]; + u16 refs_count, work_count, lfs_count, slots_count, meta_count; + u16 mb_count, row_a_count, row_b_count, fixed_count, tiles_count; + u16 codec; + u16 flag_8bit; + u16 const_214; + u16 const_218; + u16 fps; + u16 rc_mode; + u16 bitrate; + u16 init_qp; + u16 limits; + u16 limit_bytes; + u16 limit_secs; + u16 f64_288; + u16 f64_290; + u16 min_qp; + u16 max_qp; + u16 gop_flags; + u16 p_frames; + u16 max_b; + u16 ref_depth; + u16 idr_interval; + u16 f64_2c0; + u16 f64_2c8; + u16 rc_flags; + u16 min_qp2; + u16 max_qp2; + u16 rc_310; + u16 size_class; + u16 coded_w; + u16 coded_h; + u16 codec_flags; + u16 h264_380; + u16 h264_384; + u16 codec_2588; + u16 bit_depth; + u16 coded_h2; + u16 h264_profile; + u16 h264_level; + u16 h264_max_refs; + u16 h264_width_mbs; + u16 h264_height_mbs; + u16 h264_crop; + u16 h264_crop_left; + u16 h264_crop_right; + u16 h264_crop_top; + u16 h264_crop_bottom; + u16 h264_cabac; + u16 h264_8x8; + u16 hevc_pad; + u16 hevc_rc_270c; + u16 hevc_rc_9604; + u16 hevc_rc_9608; + u16 hevc_profile_a; + u16 hevc_compat_a; + u16 hevc_8bit_a; + u16 hevc_level_a; + u16 hevc_profile_b; + u16 hevc_compat_b; + u16 hevc_8bit_b; + u16 hevc_level_b; + u16 hevc_width; + u16 hevc_height; + u16 hevc_width_ctb; + u16 hevc_height_ctb; + u16 hevc_conf; + u16 hevc_conf_left; + u16 hevc_conf_right; + u16 hevc_conf_top; + u16 hevc_conf_bottom; + u16 hevc_depth_luma; + u16 hevc_depth_chroma; + u16 hevc_refs; + u16 hevc_reorder_copy; + u16 hevc_dpb_minus1; + u16 hevc_reorder; + u16 hevc_reorder_on; + u16 geometry_mode, geometry_depth; + u16 profile_idc, level_idc, max_refs_copy, mb_size, mb_height; + u16 crop_copy, crop_right_copy, crop_bottom_copy; + u16 codec_byte, cabac_copy, transform_copy, hevc_cq, hevc_cq_copy; + u16 hevc_cq_codec, hevc_cq_picture; +}; + +struct ave_proc_layout { + u16 first_frame; + u16 y, uv; /* {u64 address, u32 size, u32 stride} */ + u16 in_w, in_h; + u16 coded_cap; + u16 fps_int; + u16 bitrate; + u16 limits, limit_bytes, limit_secs; + u16 const5; + u16 fps; /* f64 */ + u16 timescale; /* u64 */ + u16 slot; + u16 coded, meta; /* u64 addresses */ + u16 ordinal; + u16 pts; /* u64 */ + u16 force_key; /* 1 on a forced key frame */ + u16 force_type; /* 2 on a forced key frame */ + u16 frame_buf; /* H.264 only */ + u16 shdr_table; /* HEVC only */ + u16 crop; /* HEVC only */ + u16 meta_size, extra, frame_size; + u16 low_delay; /* HEVC only: 3 with reordering off */ +}; + +struct ave_fw_layout { + const char *name; + size_t bytes[2][AVE_IMAGE_COUNT]; + struct ave_start_layout start[2]; + struct ave_proc_layout process[2]; + u32 hdr_bytes, meta_bytes; + u32 process_timeout_us; + u8 process_subtype, max_slots, mcc_dsid; + bool counted; +}; + +extern const struct ave_fw_layout ave_layout_9003_78_0; +/* Not selectable until its loader TEXT CRC is measured and qualified. */ +extern const struct ave_fw_layout ave_layout_26a428; +const struct ave_fw_layout *ave_layout_for_crc(u32 crc); + +#endif diff --git a/drivers/media/platform/apple/ave/ave-regs.h b/drivers/media/platform/apple/ave/ave-regs.h index 4b625ad3b70d9..6e657dcd1fb0c 100644 --- a/drivers/media/platform/apple/ave/ave-regs.h +++ b/drivers/media/platform/apple/ave/ave-regs.h @@ -131,8 +131,6 @@ enum ave_opcode { #define AVE_TOKEN_BASE 300 /* the first Open carries 400 */ #define AVE_CLIENT_STEP 10 -/* Process slots: the command subtype is this plus the slot index */ -#define AVE_PROC_SUBTYPE 0x15 /* Completes sent to drain a session */ #define AVE_DRAIN_COMPLETES 4 diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index c97681d7fb840..267c5abc73eed 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -47,7 +47,6 @@ #define AVE_SHDR_BYTES (256 * 0x400) #define AVE_SHDR_ENTRY_BYTES 0x400 #define AVE_CODED_MIN 0x3c000 -#define AVE_HDR_BYTES 0x12c000 /* H.264 per-frame buffer (Process +0x58c8): unused in bit-rate modes */ #define AVE_H264_FRAME_BUF SZ_16K @@ -181,6 +180,10 @@ static int ave_sess_params(struct ave_ctx *ctx, struct ave_params *p) ALIGN(p->width, 16)) * ALIGN(p->height, 16) / 2, SZ_16K); p->coded_cap = max_t(u32, p->coded_cap, AVE_CODED_MIN); + if (ctx->ave->layout->counted && + (p->fps_num != 30ULL * p->fps_den || + (!p->bframes && (p->codec == AVE_CODEC_HEVC || (!baseline && !p->intra_only))))) + return -EOPNOTSUPP; return 0; } @@ -201,8 +204,8 @@ struct ave_buf_req { unsigned long attrs; }; -#define AVE_MAX_BUF_REQS (1 + 4 * AVE_MAX_REFS + 4 + 2 * AVE_TILE_BUFS + 2 + \ - 3 * AVE_MAX_SLOTS) +#define AVE_MAX_BUF_REQS (1 + 6 * AVE_MAX_REFS + 6 + 2 * AVE_TILE_BUFS + 2 + \ + 4 * AVE_MAX_SLOTS) static int ave_buf_req_cmp(const void *a, const void *b) { @@ -229,6 +232,7 @@ static void ave_req(struct ave_buf_req *r, unsigned int *n, struct ave_buf *b, s static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) { const struct ave_params *p = &sess->p; + const struct ave_fw_layout *fw = ave->layout; bool hevc = p->codec == AVE_CODEC_HEVC; u32 w = p->width, h = p->height; u32 w16 = DIV_ROUND_UP(w, 16), w32 = DIV_ROUND_UP(w, 32); @@ -244,19 +248,30 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) return -ENOMEM; ref_a = ALIGN(w, 32) * ALIGN(h + 16, 32); + if (p->bit_depth == 10) + ref_a = ref_a * 5 / 4; ref_b = roundup_pow_of_two(w) * roundup_pow_of_two(h) / 32; ref_c = ave_lfs_ref_size(w, h, hevc, ave->variant->lfs_ref_scale); ref_d = ALIGN(hevc ? 64 * w32 * h32 : 128 * p->mbs, SZ_16K); tile = hevc ? 2304 * w32 * h64 : ALIGN(w, 16) * ALIGN(h, 64); + if (p->bit_depth == 10) + tile *= 2; - ave_req(r, &n, &sess->hdr, AVE_HDR_BYTES, 0, nomap); + ave_req(r, &n, &sess->hdr, fw->hdr_bytes, 0, nomap); for (unsigned int i = 0; i < p->refs; i++) { ave_req(r, &n, &sess->ref_a[i], ref_a, ref_a * 3 / 2, nomap); ave_req(r, &n, &sess->ref_b[i], ref_b, ref_b * 5 / 4, nomap); ave_req(r, &n, &sess->ref_c[i], ref_c, 0, nomap); ave_req(r, &n, &sess->ref_d[i], ref_d, 0, nomap); + if (fw->counted) { + ave_req(r, &n, &sess->ref_e[i], ref_a / 2, 0, nomap); + ave_req(r, &n, &sess->ref_f[i], ref_b / 4, 0, nomap); + } } ave_req(r, &n, &sess->mb, ALIGN(82 * p->mbs, SZ_16K), 0, nomap); + if (fw->counted && p->rc_mode == 3) + for (unsigned int i = 0; i < 2; i++) + ave_req(r, &n, &sess->mb_extra[i], ALIGN(82 * p->mbs, SZ_16K), 0, nomap); ave_req(r, &n, &sess->row_a, hevc ? SZ_16K : ALIGN(256 * w16, SZ_16K), 0, nomap); ave_req(r, &n, &sess->row_b, ALIGN(hevc ? 768 * w32 : 1024 * w16, SZ_16K), 0, nomap); ave_req(r, &n, &sess->fixed, SZ_16K, 0, nomap); @@ -269,14 +284,18 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) ave_req(r, &n, &sess->hevc_extra[i], ALIGN(w * h32, SZ_16K), 0, nomap); for (unsigned int s = 0; s < p->slots; s++) { ave_req(r, &n, &sess->slot[s].coded, p->coded_cap, 0, 0); - ave_req(r, &n, &sess->slot[s].meta, AVE_META_BYTES, 0, 0); - /* - * HEVC: slice headers. H.264: the per-frame buffer of Process - * +0x58c8, which the firmware never wrote in any bit-rate - * mode on hardware; one page. - */ - ave_req(r, &n, &sess->slot[s].aux, hevc ? AVE_SHDR_BYTES : AVE_H264_FRAME_BUF, 0, - hevc ? 0 : nomap); + ave_req(r, &n, &sess->slot[s].meta, fw->meta_bytes, 0, 0); + /* J700 has HEVC slice headers; 26A428 has a new small slot buffer. */ + if (fw->counted) { + ave_req(r, &n, &sess->slot[s].extra, SZ_16K, 0, 0); + if (!hevc && p->rc_mode != 3) + ave_req(r, &n, &sess->slot[s].aux, + ALIGN(432 * p->mbs, SZ_16K), 0, nomap); + } else { + ave_req(r, &n, &sess->slot[s].aux, + hevc ? AVE_SHDR_BYTES : AVE_H264_FRAME_BUF, 0, + hevc ? 0 : nomap); + } } sort(r, n, sizeof(*r), ave_buf_req_cmp, NULL); @@ -302,8 +321,12 @@ static void ave_sess_free_bufs(struct ave_dev *ave, struct ave_session *sess) ave_free(ave, &sess->ref_b[i]); ave_free(ave, &sess->ref_c[i]); ave_free(ave, &sess->ref_d[i]); + ave_free(ave, &sess->ref_e[i]); + ave_free(ave, &sess->ref_f[i]); } ave_free(ave, &sess->mb); + for (unsigned int i = 0; i < 2; i++) + ave_free(ave, &sess->mb_extra[i]); ave_free(ave, &sess->row_a); ave_free(ave, &sess->row_b); ave_free(ave, &sess->fixed); @@ -317,6 +340,7 @@ static void ave_sess_free_bufs(struct ave_dev *ave, struct ave_session *sess) ave_free(ave, &sess->slot[s].coded); ave_free(ave, &sess->slot[s].meta); ave_free(ave, &sess->slot[s].aux); + ave_free(ave, &sess->slot[s].extra); ave_free(ave, &sess->slot[s].bounce); } } @@ -498,7 +522,7 @@ void ave_sess_prepare_slot(struct ave_session *sess, unsigned int slot) struct ave_slot *s = &sess->slot[slot]; memset(s->meta.cpu, 0, s->meta.size); - if (sess->p.codec == AVE_CODEC_HEVC) + if (sess->p.codec == AVE_CODEC_HEVC && s->aux.cpu) memset(s->aux.cpu, 0, AVE_SHDR_ENTRY_BYTES); } @@ -510,7 +534,7 @@ void ave_sess_reply(struct ave_dev *ave, const u8 *msg, u32 len) { u64 client = get_unaligned_le64(msg + AVE_REPLY_CLIENT); u32 subtype = get_unaligned_le32(msg + AVE_REPLY_SUBTYPE); - u32 slot = subtype - AVE_PROC_SUBTYPE; + u32 slot = subtype - ave->layout->process_subtype; struct ave_session *sess = NULL; struct ave_ctx *ctx = NULL; struct ave_slot *s; diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h index c0c150f7fab42..c880b9a62d235 100644 --- a/drivers/media/platform/apple/ave/ave.h +++ b/drivers/media/platform/apple/ave/ave.h @@ -27,7 +27,7 @@ struct dev_pm_domain_list; struct iommu_domain; /* Frames the firmware can hold at once per session (Process "slots"). */ -#define AVE_MAX_SLOTS 5 +#define AVE_MAX_SLOTS 7 /* Sessions one firmware boot serves at once (macOS was seen running two). */ #define AVE_MAX_SESSIONS 2 @@ -103,10 +103,7 @@ enum ave_image { AVE_IMAGE_COUNT, }; -#define AVE_START_H264_BYTES 0x3180 -#define AVE_START_HEVC_BYTES 0x13f28 -#define AVE_PROCESS_H264_BYTES 0x63d8 -#define AVE_PROCESS_HEVC_BYTES 0xb1c0 +#include "ave-layout.h" /* The firmware build whose command layouts the driver generates */ #define AVE_FW_TEXT_CRC_9003_78_0 0x69c54454 @@ -131,13 +128,14 @@ struct ave_variant { u32 boot_device_id; u32 device_type; u32 lfs_ref_scale; - /* Zero means no command constants or MCC DSID have been qualified. */ - u32 encoding_text_crc; + /* Register sequence and boot/hello configuration have been qualified. */ + bool boot_qualified; }; struct ave_dev { struct device *dev; const struct ave_variant *variant; + const struct ave_fw_layout *layout; void __iomem *codec; void __iomem *mbox; void __iomem *coproc; @@ -206,12 +204,15 @@ struct ave_params { u32 level; /* firmware level enum, 1-based */ bool high; u32 coded_cap; + u32 bit_depth; /* zero also means the existing 8-bit API */ + u32 initial_qp; /* CQ only; bit-rate modes use 26 */ }; struct ave_slot { struct ave_buf coded; struct ave_buf meta; struct ave_buf aux; /* H.264 per-frame buffer, HEVC slice headers */ + struct ave_buf extra; /* counted layout: per-slot 16 KiB buffer */ struct ave_buf bounce; /* input copied to the encoder's stride */ struct vb2_v4l2_buffer *src; u32 status; @@ -229,7 +230,10 @@ struct ave_session { struct ave_buf ref_b[AVE_MAX_REFS]; struct ave_buf ref_c[AVE_MAX_REFS]; struct ave_buf ref_d[AVE_MAX_REFS]; + struct ave_buf ref_e[AVE_MAX_REFS]; + struct ave_buf ref_f[AVE_MAX_REFS]; struct ave_buf mb; + struct ave_buf mb_extra[2]; struct ave_buf row_a, row_b, fixed; struct ave_buf tile[AVE_TILE_BUFS]; struct ave_buf tile_aux[AVE_TILE_BUFS]; @@ -345,9 +349,11 @@ bool ave_sess_stalled(struct ave_session *sess); bool ave_sess_starved(struct ave_session *sess); /* ave-image.c */ -void ave_image_start(u8 *cmd, size_t len, enum ave_codec codec, bool reorder_off, +void ave_image_start(u8 *cmd, size_t len, const struct ave_fw_layout *fw, + enum ave_codec codec, bool reorder_off, bool hevc_low_delay); -void ave_image_process(u8 *cmd, size_t len, enum ave_codec codec); +void ave_image_process(u8 *cmd, size_t len, const struct ave_fw_layout *fw, + enum ave_codec codec); /* ave-drv.c */ void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, enum ave_image kind, diff --git a/drivers/media/platform/apple/ave/tests/README b/drivers/media/platform/apple/ave/tests/README new file mode 100644 index 0000000000000..d315f9d4d4bc8 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/README @@ -0,0 +1,29 @@ +AVE host tests +============== + +Run python3 run.py with a C compiler providing ASan and UBSan (CC may name +its executable). No encoder, firmware or captured command image is needed. + +run.py reads the real AVE structs and compiles the unmodified bodies of +ave-image.c, ave-layout.c and ave-cmd.c, plus the real buffer-allocation, +free, reply and firmware-build-gate functions. Only includes and unused +kernel-facing declarations are omitted. shim.h supplies host primitives; +boundary.h replaces IPC, DMA and logging. This does not validate the kernel +API integration, DART mappings, boot sequence or output extraction. + +--compare-baseline PATH builds an older AVE source directory and compares +all bytes of 288 synthetic sessions (1,440 Start/Process commands, including +headers, addresses and zero padding). It covers both codecs, geometry, +profiles, B-frame settings, rates, CBR/VBR and forced-key-frame fields. + +--build PATH retains the executable for offline capture QA. Its --case +interface emits synthetic commands; capture inputs and generated artifacts +must stay outside the public source tree. Compare only with the matching +firmware layout, mask enumerated pointer fields and host-only headers, and +report inherited differences explicitly. The 26A428 description is not +reachable through the runtime CRC lookup. + +The host shim intentionally uses stdint types, Linux's private ENOTSUPP +value and side-effect-free calls to kernel-like arithmetic macros. Those +produce checkpatch PREFER_KERNEL_TYPES, ENOTSUPP and MACRO_ARG_REUSE notices. +MAINTAINERS already covers the entire AVE directory (FILE_PATH_CHANGES). diff --git a/drivers/media/platform/apple/ave/tests/boundary.h b/drivers/media/platform/apple/ave/tests/boundary.h new file mode 100644 index 0000000000000..22aa6bba0ff66 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/boundary.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Only the hardware/IPC boundary is mocked. The test uses real session types. */ +struct ave_dev { + struct ave_ctx *sessions[AVE_MAX_SESSIONS]; + u32 next_client, text_crc, data_crc; + const struct ave_fw_layout *layout; + const struct ave_variant *variant; + enum ave_fw_state fw_state; + void *dev; +}; + +struct ave_ctx { + struct ave_dev *ave; + struct ave_session *sess; + int ready; +}; + +struct vb2_v4l2_buffer { + int unused; +}; + +static u8 wire[0x1c000 + 32]; +static size_t wire_len; +static unsigned int submissions; +static u64 next_dma = 0x10002000000ULL; +static unsigned int alloc_count, free_count; +static void ave_free(struct ave_dev *ave, struct ave_buf *buf) +{ + if (buf->cpu) + free_count++; + memset(buf, 0, sizeof(*buf)); +} + +static int ave_alloc_padded(struct ave_dev *ave, struct ave_buf *buf, + size_t size, size_t alloc, unsigned long attrs) +{ + assert(size); + buf->size = size; + buf->alloc = ALIGN(max(size, alloc), SZ_16K); + buf->dma = next_dma; + next_dma += ALIGN(buf->alloc, 0x100000); + buf->attrs = attrs; + buf->cpu = (void *)1; + alloc_count++; + return 0; +} + +static void *ave_ipc_cmd(struct ave_dev *ave, unsigned int buf, size_t len) +{ + assert(len <= sizeof(wire) - 32); + memset(wire, 0, len); + memset(wire + len, 0xa5, 32); + wire_len = len; + return wire; +} + +static int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, + u64 timeout, bool reply) +{ + assert(len == wire_len); + for (unsigned int i = 0; i < 32; i++) + assert(wire[len + i] == 0xa5); + put_unaligned_le64(timeout, wire + AVE_CMD_TIMEOUT); + put_unaligned_le64(1234567, wire + AVE_CMD_CREATED); + submissions++; + return 0; +} + +static int ave_ipc_scalar_buf(struct ave_dev *ave) +{ + return 1; +} + +static int ave_ipc_settle(struct ave_dev *ave, unsigned int ms) +{ + return 0; +} + +static int ave_ipc_wait_ack(struct ave_dev *ave, unsigned int buf) +{ + return 0; +} + +static void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, + enum ave_image kind, const void *cmd, size_t len) +{ +} diff --git a/drivers/media/platform/apple/ave/tests/run.py b/drivers/media/platform/apple/ave/tests/run.py new file mode 100644 index 0000000000000..6dc7482914884 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/run.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Host checks of the actual AVE builders with a fake IPC/DMA boundary. + +Only includes and unused kernel-facing declarations are omitted. Function +bodies are compiled unchanged. --baseline accepts an older AVE source dir +for byte comparisons; --build retains the executable for private capture QA. +No firmware, command images, or reference captures are part of these tests. +""" +import argparse +import os +from pathlib import Path +import re +import subprocess +import tempfile + + +def declaration(text, name): + match = re.search(r'^(?:struct|enum) ' + name + r' \{', text, re.M) + end = text.index('\n};', match.start()) + 3 + return text[match.start():end] + '\n' + + +def function(text, name): + match = re.search(r'^(?:static )?(?:const )?[\w *]+\b' + name + r'\([^;]+?\n\{', text, re.M) + end = text.index('\n}', match.end()) + 2 + return text[match.start():end] + '\n' + + +def source(text): + return re.sub(r'^#include .*\n', '', text, flags=re.M) + + +def build(src, out, baseline): + header = (src / 'ave.h').read_text() + code = (Path(__file__).parent / 'shim.h').read_text() + for name in ('AVE_MAX_SLOTS', 'AVE_MAX_REFS', 'AVE_TILE_BUFS', + 'AVE_MAX_SESSIONS', 'AVE_FW_TEXT_CRC_9003_78_0'): + code += re.search(r'^#define ' + name + r'\s+[^\n]+', header, re.M)[0] + '\n' + for name in ('ave_codec', 'ave_buf', 'ave_fw_state', 'ave_image', + 'ave_coproc_regs', 'ave_variant', 'ave_params', 'ave_slot', 'ave_session'): + code += declaration(header, name) + if baseline: + code = '#define TEST_BASELINE 1\n' + code + code += '\n'.join(re.findall(r'^#define AVE_(?:START|PROCESS)_\w+.*$', header, re.M)) + '\n' + else: + code += source((src / 'ave-layout.h').read_text()) + code += source((src / 'ave-layout.c').read_text()) + code += source((src / 'ave-regs.h').read_text()) + code += source((src / 'ave-sizes.h').read_text()) + code += (Path(__file__).parent / 'boundary.h').read_text() + code += source((src / 'ave-image.c').read_text()) + code += source((src / 'ave-cmd.c').read_text()) + session = (src / 'ave-sess.c').read_text() + code += declaration(session, 'ave_buf_req') + lines = iter(session.splitlines()) + for line in lines: + if line.startswith('#define '): + code += line + '\n' + while line.endswith('\\'): + line = next(lines) + code += line + '\n' + for name in ('ave_buf_req_cmp', 'ave_req', 'ave_sess_alloc', 'ave_sess_free_bufs', 'ave_sess_reply'): + code += function(session, name) + if not baseline: + code += function((src / 'ave-fw.c').read_text(), 'ave_fw_check_build') + code += (Path(__file__).parent / 'test.c').read_text() + out.mkdir(parents=True, exist_ok=True) + (out / 'host.c').write_text(code) + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-O1', '-g', + '-Wall', '-Wextra', '-Werror', '-Wno-unused-parameter', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', + str(out / 'host.c'), '-o', str(out / 'ave-test')], check=True) + return out / 'ave-test' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--baseline', type=Path) + parser.add_argument('--build', type=Path) + parser.add_argument('--compare-baseline', type=Path) + args = parser.parse_args() + src = args.baseline or Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory(prefix='ave-host-') as tmp: + binary = build(src, args.build or Path(tmp), bool(args.baseline)) + subprocess.run([str(binary)], check=True) + if args.compare_baseline: + old = build(args.compare_baseline, Path(tmp) / 'baseline', True) + before = subprocess.check_output([str(old), '--matrix']) + after = subprocess.check_output([str(binary), '--matrix']) + if before != after: + mismatch = next(i for i, pair in enumerate(zip(before, after)) if pair[0] != pair[1]) + raise AssertionError(f'J700 matrix differs at stream byte {mismatch}') + print(f'ok: 288 J700 sessions / 1440 commands byte-identical ({len(after)} bytes)') + + +if __name__ == '__main__': + main() diff --git a/drivers/media/platform/apple/ave/tests/shim.h b/drivers/media/platform/apple/ave/tests/shim.h new file mode 100644 index 0000000000000..bc3dea7f80fc1 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/shim.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Host types/primitives, never an alternative command implementation. */ +#include +#include +#include +#include +#include +#include +#include + +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +typedef int32_t s32; +typedef int64_t s64; +typedef uint64_t dma_addr_t; +#define AVE_BUF_CMD 0 +#define AVE_BUF_PROC(s) (17 + (s)) +#define SZ_16K 0x4000U +#define U32_MAX UINT32_MAX +#define GENMASK_ULL(h, l) ((~0ULL >> (63 - (h))) & (~0ULL << (l))) +#define BIT(n) (1U << (n)) +#define BIT_ULL(n) (1ULL << (n)) +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define DIV_ROUND_UP(x, a) (((x) + (a) - 1) / (a)) +#define ALIGN(x, a) (((x) + (a) - 1) & ~((a) - 1)) +#define max(x, y) ((x) > (y) ? (x) : (y)) +#define GFP_KERNEL 0 +#define DMA_ATTR_NO_KERNEL_MAPPING 1 +static inline void *kcalloc(size_t n, size_t size, int flags) +{ + return calloc(n, size); +} + +#define kfree(p) free(p) +static inline void sort(void *p, size_t n, size_t size, + int (*cmp)(const void *, const void *), void *unused) +{ + qsort(p, n, size, cmp); +} + +#define dev_err(...) test_log(__VA_ARGS__) +#define dev_info(...) test_log(__VA_ARGS__) +#define dev_dbg(...) test_log(__VA_ARGS__) +#define dev_warn(...) test_log(__VA_ARGS__) +#define WARN_ON_ONCE(c) ((c) ? (abort(), 1) : 0) +#define ENOTSUPP 524 +static inline void test_log(void *dev, const char *fmt, ...) +{ +} + +#define atomic_inc(p) (++*(p)) +static unsigned long jiffies = 42; +#ifndef TEST_BASELINE +static inline void put_unaligned_le16(u16 v, void *p) +{ + u8 *b = p; + + for (unsigned int i = 0; i < 2; i++) + b[i] = v >> (8 * i); +} +#endif +static inline void put_unaligned_le32(u32 v, void *p) +{ + u8 *b = p; + + for (unsigned int i = 0; i < 4; i++) + b[i] = v >> (8 * i); +} + +static inline void put_unaligned_le64(u64 v, void *p) +{ + u8 *b = p; + + for (unsigned int i = 0; i < 8; i++) + b[i] = v >> (8 * i); +} + +static inline u32 get_unaligned_le32(const void *p) +{ + const u8 *b = p; + + return (u32)b[0] | (u32)b[1] << 8 | (u32)b[2] << 16 | (u32)b[3] << 24; +} + +static inline u64 get_unaligned_le64(const void *p) +{ + return get_unaligned_le32(p) | + (u64)get_unaligned_le32((const u8 *)p + 4) << 32; +} + +static inline u64 mul_u64_u64_div_u64(u64 a, u64 b, u64 c) +{ + return (__uint128_t)a * b / c; +} + +static inline u32 roundup_pow_of_two(u32 v) +{ + u32 n = 1; + + while (n < v) + n *= 2; + return n; +} diff --git a/drivers/media/platform/apple/ave/tests/test.c b/drivers/media/platform/apple/ave/tests/test.c new file mode 100644 index 0000000000000..486b7dc60d225 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/test.c @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: GPL-2.0-only +static const struct ave_coproc_regs test_regs; +static const struct ave_variant test_variant = { + .lfs_ref_scale = 2, +#ifndef TEST_BASELINE + .boot_qualified = true, + .regs = &test_regs, +#endif +}; + +static void params(struct ave_params *p, unsigned int codec, u32 w, u32 h) +{ + memset(p, 0, sizeof(*p)); + p->codec = codec; + p->width = w; + p->height = h; + p->cw = ALIGN(w, 16); + p->ch = ALIGN(h, 16); + p->stride = ALIGN(w, 64); + p->mbs = p->cw / 16 * (p->ch / 16); + p->fps_num = 30; + p->fps_int = 30; + p->fps_den = 1; + p->bframes = true; + p->refs = 5; + p->slots = 5; + p->gop = 30; + p->rc_mode = 4; + p->bitrate = 4000000; + p->limit_bytes = 1000000; + p->max_qp = 51; + p->profile = codec == 1 ? 4 : 1; + p->level = codec == 1 ? 12 : 6; + p->coded_cap = + max(ALIGN(3 * (codec == 2 ? ALIGN(w, 32) : p->cw) * p->ch / 2, + SZ_16K), + 0x3c000U); +} + +static void start(struct ave_ctx *ctx) +{ + assert(ave_sess_alloc(ctx->ave, ctx->sess) == 0); + assert(ave_cmd_open(ctx) == 0); + assert(ave_cmd_start(ctx) == 0); +} + +static void frame(struct ave_ctx *ctx, unsigned int slot, bool force) +{ + struct vb2_v4l2_buffer src; + + assert(ave_cmd_process(ctx, slot, &src, 0x10012340000ULL, + ctx->sess->p.bitrate, force) == 0); +} + +#ifndef TEST_BASELINE +static void facts(void) +{ + struct ave_dev dev = { .variant = &test_variant, + .layout = &ave_layout_26a428 }; + struct ave_session session = {}; + struct ave_ctx ctx = { .ave = &dev, .sess = &session }; + u8 reply[0x50] = {}; + + assert(!ave_layout_for_crc(0)); + assert(!ave_layout_for_crc(0xffffffff)); + assert(!ave_layout_for_crc(0x2ce7359e)); + assert(ave_layout_for_crc(0x69c54454) == &ave_layout_9003_78_0); + assert(ave_fw_check_build(&dev) == -ENOTSUPP); + dev.text_crc = 0x69c54454; + assert(ave_fw_check_build(&dev) == -ENOTSUPP); + dev.layout = ave_layout_for_crc(dev.text_crc); + assert(ave_fw_check_build(&dev) == 0); + struct ave_variant probe_only = test_variant; + + probe_only.boot_qualified = false; + probe_only.regs = NULL; + dev.variant = &probe_only; + assert(ave_fw_check_build(&dev) == -ENOTSUPP); + dev.variant = &test_variant; + dev.layout = &ave_layout_26a428; + + for (unsigned int codec = 1; codec <= 2; codec++) { + params(&session.p, codec, 1920, 1080); + start(&ctx); + assert(wire_len == (codec == 1 ? 0x3b40U : 0x13ee0U)); + assert(get_unaligned_le32(wire + 0x50) == 0xec000); + assert(get_unaligned_le32(wire + 0x94) == 5); + assert(get_unaligned_le32(wire + 0x918) == 5); + assert(get_unaligned_le32(wire + 0xb00) == 5); + assert(get_unaligned_le32(wire + 0xb10) == 0x10000); + assert(get_unaligned_le32(wire + 0xdbc) == 5); + assert(get_unaligned_le32(wire + 0xfe4) == 5); + assert(get_unaligned_le32(wire + 0xc0) == 1920 * 1120 / 2); + assert(get_unaligned_le32(wire + 0xd0) == 2048 * 2048 / 128); + assert(get_unaligned_le32(wire + 0x322c) == 1920); + assert(get_unaligned_le32(wire + 0x3230) == 1088); + assert(get_unaligned_le32(wire + 0x3618) == 4000000); + assert(get_unaligned_le64(wire + 0x3638) == + 0x415e848000000000ULL); /* 8e6 */ + if (codec == 1) { + assert(get_unaligned_le32(wire + 0x37d0) == + (120U << 16 | 4)); + assert(wire[0x37d4] == 68 && wire[0x37d6] == 1); + assert(get_unaligned_le32(wire + 0x3a98) == 77); + assert(get_unaligned_le32(wire + 0x3aa0) == 40); + assert(get_unaligned_le32(wire + 0x3ac0) == + (67U << 16 | 119)); + assert(wire[0x3ac7] == 1); + assert(get_unaligned_le32(wire + 0x3acc) >> 16 == 4); + } else { + assert(get_unaligned_le32(wire + 0x68) == 2); + assert(get_unaligned_le32(wire + 0x66b8) == 1920); + assert(get_unaligned_le32(wire + 0x66bc) == 1088); + assert(get_unaligned_le32(wire + 0x6708) == 2); + } + session.frames = 0; + frame(&ctx, 0, false); + assert(wire_len == (codec == 1 ? 0x6df8U : 0x78b8U)); + assert(get_unaligned_le32(wire + 0x20) == 0x13); + assert(get_unaligned_le64(wire + 0x30) == 46400000); + assert(get_unaligned_le32(wire + + (codec == 1 ? 0xdc0 : 0x1568)) == 1); + assert(get_unaligned_le32(wire + + (codec == 1 ? 0x58b8 : 0x6060)) == + 0x10000); + assert(get_unaligned_le32(wire + + (codec == 1 ? 0x58c8 : 0x6070)) == + 0x4000); + if (codec == 1) + assert(get_unaligned_le32(wire + 0x58e0) == 0x360000); + frame(&ctx, 4, false); + assert(get_unaligned_le32(wire + 0x20) == 0x17); + assert(get_unaligned_le32(wire + + (codec == 1 ? 0xdc0 : 0x1568)) == 0); + assert(get_unaligned_le64(wire + + (codec == 1 ? 0x5760 : 0x5f08)) == + 480000); + unsigned int before = submissions; + + assert(ave_cmd_process(&ctx, session.p.slots, NULL, 0, + session.p.bitrate, false) == -EINVAL); + + assert(ave_cmd_process(&ctx, 0, NULL, 0, session.p.bitrate, + true) == -EOPNOTSUPP); + assert(ave_cmd_process(&ctx, 0, NULL, 0, 12000000, false) == + -EOPNOTSUPP); + assert(submissions == before); + dev.sessions[0] = &ctx; + put_unaligned_le64(session.client, reply + 0x10); + put_unaligned_le32(0x17, reply + 0x20); + ave_sess_reply(&dev, reply, sizeof(reply)); + assert(!session.slot[4].busy && session.slot[4].done); + assert(ctx.ready == (int)codec); + dev.sessions[0] = NULL; + ave_sess_free_bufs(&dev, &session); + memset(&session, 0, sizeof(session)); + } + for (unsigned int codec = 1; codec <= 2; codec++) { + params(&session.p, codec, 1920, 1080); + session.p.rc_mode = 3; + session.p.slots = 7; + session.p.initial_qp = 28; + session.p.max_qp = 48; + session.p.bitrate = codec == 1 ? 14100000 : 7050000; + session.p.limit_bytes = 0; + session.p.coded_cap = 0x458000; + start(&ctx); + assert(get_unaligned_le32(wire + 0x918) == 7); + assert(get_unaligned_le32(wire + 0xb00) == 7); + assert(get_unaligned_le32(wire + 0x1868) == 3); + assert(get_unaligned_le32(wire + 0x1888) == 0xa4000); + assert(get_unaligned_le32(wire + 0x1898) == 0xa4000); + assert(get_unaligned_le32(wire + 0x988) == 0x458000); + assert(get_unaligned_le32(wire + 0x3620) == 28); + assert(get_unaligned_le32(wire + 0x3734) == 0); + assert(get_unaligned_le32(wire + 0x36d0) == + (codec == 1 ? 0x10600U : 0x10700U)); + if (codec == 1) { + assert(wire[0x3830] == 0); + } else { + assert(get_unaligned_le32(wire + 0x3708) == 1); + assert(get_unaligned_le32(wire + 0x37c8) == 1); + assert(get_unaligned_le32(wire + 0x9e38) == 0x100); + } + frame(&ctx, 6, false); + assert(get_unaligned_le32(wire + 0x20) == 0x19); + if (codec == 1) { + assert(get_unaligned_le64(wire + 0x58d8) == 0); + assert(get_unaligned_le32(wire + 0x58e0) == 0); + } + ave_sess_free_bufs(&dev, &session); + memset(&session, 0, sizeof(session)); + } + params(&session.p, 2, 1920, 1080); + session.p.bit_depth = 10; + session.p.profile = 2; + session.p.min_qp = -12; + session.p.stride *= 2; + session.p.coded_cap = session.p.coded_cap * 5 / 4; + start(&ctx); + assert(get_unaligned_le32(wire + 0xa0) == 0x290400); + assert(get_unaligned_le32(wire + 0xc0) == 0x148200); + assert(get_unaligned_le32(wire + 0x1bc0) == 0x47b800); + assert(get_unaligned_le32(wire + 0x35e8) == 0x10040); + assert(get_unaligned_le32(wire + 0x3a60) == 0x01000101); + assert(get_unaligned_le32(wire + 0x6530) == 0x01000000); + assert(get_unaligned_le32(wire + 0x66dc) == 2); + ave_sess_free_bufs(&dev, &session); + memset(&session, 0, sizeof(session)); + + params(&session.p, 1, 1366, 768); + session.p.profile = 6; + session.p.high = true; + start(&ctx); + assert(get_unaligned_le32(wire + 0x3a98) == 100); + assert(wire[0x3b0d] == 1 && wire[0x3823] == 1 && wire[0x3820] == 1); + assert(get_unaligned_le32(wire + 0x3ac8) >> 16 == 5); + ave_sess_free_bufs(&dev, &session); + params(&session.p, 1, 1920, 1080); + session.p.bframes = false; + session.p.profile = 2; + session.p.refs = 2; + session.p.slots = 3; + start(&ctx); + frame(&ctx, 2, false); + assert(get_unaligned_le32(wire + 0x20) == 0x15); + assert(get_unaligned_le32(wire + 0x2c) == 0x140); + ave_sess_free_bufs(&dev, &session); + + assert(alloc_count == free_count); + puts("ok: CRC/boot gates, T6040/CQ/Main10, bounds, replies, refusal, allocations"); +} +#endif + +int main(int argc, char **argv) +{ + struct ave_dev dev = { + .variant = &test_variant, +#ifndef TEST_BASELINE + .layout = &ave_layout_9003_78_0, +#endif + }; + struct ave_session session = {}; + struct ave_ctx ctx = { .ave = &dev, .sess = &session }; + + if (argc > 1 && !strcmp(argv[1], "--matrix")) { + /* Byte-for-byte regression input, including addresses and headers. */ + static const u32 geometry[][2] = { + { 320, 240 }, { 1280, 720 }, { 1366, 768 }, + { 1920, 1080 }, { 2408, 1506 }, { 3840, 2160 }, + }; + static const u32 rates[] = { 1000000, 4000000, 12000000 }; + + for (unsigned int i = 0; i < 288; i++) { + u32 codec = i < 216 ? 1 : 2; + u32 n = codec == 1 ? i : i - 216; + u32 mode = n % 2 ? 4 : 2; + + n /= 2; + u32 rate = rates[n % 3]; + + n /= 3; + u32 b = n % 2; + + n /= 2; + u32 profile = codec == 1 ? 2 + 2 * (n % 3) : 1; + + if (codec == 1) + n /= 3; + params(&session.p, codec, geometry[n][0], + geometry[n][1]); + session.p.profile = profile; + session.p.high = codec == 1 && profile == 6; + session.p.bframes = b; + session.p.refs = b ? 5 : 2; + session.p.slots = b ? 5 : 3; + session.p.reorder_off = !b; + session.p.bitrate = rate; + session.p.rc_mode = mode; + session.p.limit_bytes = mode == 4 ? rate / 4 : 0; + start(&ctx); + assert(fwrite(wire, 1, wire_len, stdout) == wire_len); + for (unsigned int slot = 0; slot < session.p.slots; + slot++) { + frame(&ctx, slot, slot == 1); + assert(fwrite(wire, 1, wire_len, stdout) == + wire_len); + } + ave_sess_free_bufs(&dev, &session); + memset(&session, 0, sizeof(session)); + } + return 0; + } + + if (argc > 1 && !strcmp(argv[1], "--case")) { + /* An optional synthetic case for offline, private capture comparison. */ + assert(argc == 20); +#ifndef TEST_BASELINE + if (atoi(argv[2])) + dev.layout = &ave_layout_26a428; +#endif + params(&session.p, atoi(argv[3]), atoi(argv[4]), atoi(argv[5])); + session.p.profile = atoi(argv[6]); + session.p.high = session.p.profile == 6; + session.p.level = atoi(argv[7]); + session.p.bitrate = atoi(argv[8]); + session.p.rc_mode = atoi(argv[9]); + session.p.bframes = atoi(argv[10]); + session.p.refs = atoi(argv[11]); + session.p.slots = atoi(argv[12]); + session.p.gop = atoi(argv[13]); + session.p.fps_int = atoi(argv[14]); + session.p.fps_num = session.p.fps_int; + session.p.stride = atoi(argv[15]); + session.p.reorder_off = atoi(argv[16]); + session.p.intra_only = session.p.gop == 1 && + session.p.codec == 1; + session.p.limit_bytes = + session.p.rc_mode == 4 ? session.p.bitrate / 4 : 0; +#ifndef TEST_BASELINE + session.p.initial_qp = atoi(argv[17]); + session.p.bit_depth = atoi(argv[18]); + if (session.p.bit_depth == 10) + session.p.min_qp = -12; + if (session.p.bit_depth == 10) + session.p.coded_cap = session.p.coded_cap * 5 / 4; + if (session.p.rc_mode == 3) { + session.p.max_qp = 48; + session.p.coded_cap = session.p.initial_qp == 40 ? 0x3a4000 : + session.p.initial_qp == 28 ? 0x458000 : 0x5fc000; + } +#endif + start(&ctx); + if (atoi(argv[19])) { + session.frames = 0; + frame(&ctx, 0, false); + } + assert(fwrite(wire, 1, wire_len, stdout) == wire_len); + return 0; + } +#ifndef TEST_BASELINE + facts(); +#else + (void)test_regs; +#endif + /* Exercise alignment, profile/GOP/rate combinations under sanitizers. */ + static const u32 sizes[][2] = { + { 320, 240 }, { 640, 480 }, { 1280, 720 }, { 1366, 768 }, + { 1920, 1080 }, { 2408, 1506 }, { 3840, 2160 }, + }; + for (unsigned int c = 1; c <= 2; c++) + for (unsigned int i = 0; i < ARRAY_SIZE(sizes); i++) { + params(&session.p, c, sizes[i][0], sizes[i][1]); + start(&ctx); + for (unsigned int s = 0; s < session.p.slots; s++) + frame(&ctx, s, true); + ave_sess_free_bufs(&dev, &session); + memset(&session, 0, sizeof(session)); + } + assert(alloc_count == free_count); + puts("ok: J700 builder and allocation alignment matrix"); + return 0; +} From abe1561590813e9fede3a843a46cfce418c778da Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 14:42:10 +0000 Subject: [PATCH 1448/1579] dt-bindings: npu: apple,ane: let the T6041 ANE name its MPM stream The T6041 ADT gives the ANE two IOMMU parents on its DART: stream 0 for the engine and stream 11 for its MPM. Allow the three DARTs' stream 11 after their stream 0 on the T6041 and keep three entries everywhere else. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/npu/apple,ane.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/npu/apple,ane.yaml b/Documentation/devicetree/bindings/npu/apple,ane.yaml index feae5b133ac2f..093afc4fd6a53 100644 --- a/Documentation/devicetree/bindings/npu/apple,ane.yaml +++ b/Documentation/devicetree/bindings/npu/apple,ane.yaml @@ -58,7 +58,11 @@ properties: - const: ane iommus: - maxItems: 3 + description: + Stream 0 of each of the three DARTs, followed on T6041 by stream 11 of + each, which the ANE's MPM uses. + minItems: 3 + maxItems: 6 power-domains: description: @@ -106,6 +110,12 @@ allOf: properties: interrupts: maxItems: 1 + iommus: + minItems: 6 + else: + properties: + iommus: + maxItems: 3 - if: properties: compatible: From 8dfa5f026efff93d86ae870b145c75937d8966a9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 14:42:10 +0000 Subject: [PATCH 1449/1579] arm64: dts: apple: t6040: complete the ANE DART description from the ADT The ADT's dart-ane0 translates from vm-base 0x100_0000_0000 for 0x300_0000_0000 bytes, and iBoot preloads the ANE firmware at DVA 0x100_0000_0000 and 0x100_000c_0000, so the DARTs need that window as on the T8140. The ANE also has a second IOMMU parent, mapper-ane0-mpm on stream 11. Give the three ANE DARTs apple,dma-range and the ANE stream 11 on each. All of these nodes stay disabled until there is an H16 firmware backend, so nothing changes at runtime. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040.dtsi | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 8a5abbb0fcbfe..e0326a9bea226 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -1926,6 +1926,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; + apple,dma-range = <0x100 0x0 0x300 0x0>; status = "disabled"; }; @@ -1936,6 +1937,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; + apple,dma-range = <0x100 0x0 0x300 0x0>; status = "disabled"; }; @@ -1946,6 +1948,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; + apple,dma-range = <0x100 0x0 0x300 0x0>; status = "disabled"; }; @@ -2159,7 +2162,8 @@ reg-names = "engine"; interrupt-parent = <&aic>; interrupts = ; - iommus = <&ane0_dart0 0>, <&ane0_dart1 0>, <&ane0_dart2 0>; + iommus = <&ane0_dart0 0>, <&ane0_dart1 0>, <&ane0_dart2 0>, + <&ane0_dart0 11>, <&ane0_dart1 11>, <&ane0_dart2 11>; power-domains = <&ps_ane_cpu>; status = "disabled"; }; From 1da4993678d74ef935b967ecb13e85e4683cf32e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:44:55 +1100 Subject: [PATCH 1450/1579] dt-bindings: npu: apple,ane: describe H16 resources and firmware handoff Describe the seven H16 ranges using neutral auxiliary names, the five real power suppliers, and bootloader-provided TEXT/DATA reservations. Keep the original H14 firmware-alias reservation distinct from physical preload reservations. Allow the known H17 resource geometry and document that absence of the exclave marker does not establish Linux ownership. Build on draft #638's SID 11 binding without replacing its shared-domain policy with an untested remap. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/npu/apple,ane.yaml | 161 +++++++++++++++--- 1 file changed, 138 insertions(+), 23 deletions(-) diff --git a/Documentation/devicetree/bindings/npu/apple,ane.yaml b/Documentation/devicetree/bindings/npu/apple,ane.yaml index 093afc4fd6a53..fea47a60b6854 100644 --- a/Documentation/devicetree/bindings/npu/apple,ane.yaml +++ b/Documentation/devicetree/bindings/npu/apple,ane.yaml @@ -15,9 +15,11 @@ description: drives the engine through its register window. On T602x and T8112 a coprocessor runs the ANE firmware and the host talks to it through an ASC mailbox. T8140 has two interrupts and one system power domain; its - coprocessor uses a different firmware interface. T6041 likewise needs a - generation-specific firmware backend; it has one interrupt and its CPU - power state supplies the engine through the ANE system domain. + coprocessor is exclave-assigned and cannot be used directly without a + Linux ownership provider. T6041 has one interrupt and preloaded firmware; + its CPU, MPM and task domains depend on the ANE system domain, and its + base domain depends on the task domain. Interrupt roles on T6041 and + T8140 are not yet established. properties: compatible: @@ -34,20 +36,18 @@ properties: reg: minItems: 1 items: - - description: Engine registers. On T602x and T8112 this is the whole - ANE block, including the coprocessor. + - description: Engine registers, including the coprocessor on firmware + managed generations - description: Power manager (PMGR) registers of the ANE - - description: ANE power state registers in the PMGR - - description: eFuse words that give the SoC revision to the ANE - firmware + - description: Power state registers or first auxiliary window + - description: Revision registers or second auxiliary window + - description: Third auxiliary window + - description: Fourth auxiliary window + - description: Fifth auxiliary window reg-names: minItems: 1 - items: - - const: engine - - const: pmgr - - const: set - - const: fuse + maxItems: 7 interrupts: minItems: 1 @@ -80,10 +80,47 @@ properties: description: Mailbox of the ANE coprocessor memory-region: - maxItems: 1 + minItems: 1 + maxItems: 2 description: - IOVA-only reservation of the firmware entry alias in the ANE DART - address space. + On T602x, the IOVA-only reservation of the firmware entry alias. + On T6041 and T8140, bootloader-provided physical TEXT and DATA preload + reservations in that order. Each reservation must have no-map, must + not be reusable, and must remain reserved until reboot. Physical + addresses vary per boot and must not be hard-coded in a board DTS. + + memory-region-names: + items: + - const: firmware-text + - const: firmware-data + + apple,firmware-iovas: + $ref: /schemas/types.yaml#/definitions/uint64-array + items: + - const: 0x10000000000 + - const: 0x100000c0000 + description: + Device addresses of the preloaded TEXT and DATA segments. Their sizes + are 0xc0000 and 0x2ac000 bytes respectively. The bootloader supplies + this property together with the physical reservations. + + apple,ane-type: + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0x110, 0x200] + description: Firmware generation identifier supplied by the bootloader + + apple,firmware-preloaded: + type: boolean + description: + Firmware has been personalized and preloaded by iBoot. The driver + must preserve its runtime patches and must not stage a disk image. + + apple,exclave-assigned: + type: boolean + description: + The ANE belongs to an exclave. Direct access requires a separate + Linux ownership provider to acquire a quiesced device and isolate + the exclave DART contexts. Absence does not prove a handoff on T8140. required: - compatible @@ -102,6 +139,56 @@ allOf: properties: interrupts: minItems: 2 + power-domains: + maxItems: 1 + apple,ane-type: + const: 0x200 + - if: + properties: + compatible: + const: apple,t8140-ane + then: + properties: + reg: + maxItems: 6 + reg-names: + minItems: 1 + items: + - const: engine + - const: pmgr + - const: aux-0 + - const: aux-1 + - const: aux-2 + - const: aux-3 + - if: + required: + - apple,firmware-iovas + then: + properties: + memory-region: + minItems: 2 + required: + - memory-region + - memory-region-names + - apple,firmware-preloaded + - if: + properties: + compatible: + enum: + - apple,t6041-ane + - apple,t8140-ane + required: + - memory-region + then: + required: + - memory-region-names + - apple,firmware-iovas + - if: + required: + - memory-region-names + then: + required: + - apple,firmware-iovas - if: properties: compatible: @@ -110,8 +197,27 @@ allOf: properties: interrupts: maxItems: 1 + reg: + minItems: 7 + reg-names: + items: + - const: engine + - const: pmgr + - const: aux-0 + - const: aux-1 + - const: aux-2 + - const: aux-3 + - const: aux-4 iommus: minItems: 6 + power-domains: + minItems: 5 + maxItems: 5 + apple,ane-type: + const: 0x110 + required: + - apple,ane-type + - apple,firmware-preloaded else: properties: iommus: @@ -125,24 +231,33 @@ allOf: - apple,t6041-ane then: properties: - reg: - maxItems: 1 - reg-names: - maxItems: 1 interrupt-names: false iommus: minItems: 3 - power-domains: - maxItems: 1 resets: false mboxes: false - memory-region: false else: properties: interrupts: maxItems: 1 power-domains: minItems: 5 + reg: + maxItems: 4 + reg-names: + minItems: 1 + items: + - const: engine + - const: pmgr + - const: set + - const: fuse + memory-region: + maxItems: 1 + memory-region-names: false + apple,firmware-iovas: false + apple,ane-type: false + apple,firmware-preloaded: false + apple,exclave-assigned: false required: - interrupt-names From 2458c5b9da986b3bc335b5c6003953e7b567e0c1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:44:55 +1100 Subject: [PATCH 1451/1579] arm64: dts: apple: t6040: describe the confirmed H16 ANE resources Add the six auxiliary ranges, firmware generation and preload marker. Attach the system, CPU, MPM, task and base suppliers; their existing PMGR parents already encode the required ordering. Retain draft #638's DART aperture and stream 11 mappings without duplicating those properties. Keep all ANE nodes disabled. Dynamic firmware reservations must be provided by the bootloader, and no physical preload address is fixed in the board description. Virtual no-state gates cannot be represented by the current register-backed PMGR binding. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040.dtsi | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index e0326a9bea226..6c54e05caa743 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -2158,13 +2158,24 @@ /* Firmware interface support is required before enabling the ANE. */ ane: ane@484000000 { compatible = "apple,t6041-ane"; - reg = <0x4 0x84000000 0x0 0x2000000>; - reg-names = "engine"; + reg = <0x4 0x84000000 0x0 0x2000000>, + <0x5 0x02280000 0x0 0x18000>, + <0x5 0x022a0000 0x0 0x4000>, + <0x5 0x003c0000 0x0 0x30000>, + <0x2 0x12000000 0x0 0xff4000>, + <0x5 0x22000000 0x0 0x2000000>, + <0x5 0x08374000 0x0 0x4000>; + reg-names = "engine", "pmgr", "aux-0", "aux-1", "aux-2", + "aux-3", "aux-4"; interrupt-parent = <&aic>; interrupts = ; iommus = <&ane0_dart0 0>, <&ane0_dart1 0>, <&ane0_dart2 0>, <&ane0_dart0 11>, <&ane0_dart1 11>, <&ane0_dart2 11>; - power-domains = <&ps_ane_cpu>; + power-domains = <&ps_ane_sys>, <&ps_ane_cpu>, <&ps_ane_mpm>, + <&ps_ane_td>, <&ps_ane_base>; + apple,ane-type = <0x110>; + apple,firmware-preloaded; + /* TEXT/DATA reservations and IOVAs must come from iBoot. */ status = "disabled"; }; From 6c7ffe2fc3690af8aac32f3dfb499bea81b80173 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:44:56 +1100 Subject: [PATCH 1452/1579] accel/ane: fence H16 firmware and H17 ownership before registration Recognize H16 and H17 in the shared firmware module without entering the H14 staging path. Refuse H17 with EOPNOTSUPP until a Linux exclave ownership backend exists, even if DT omits the ownership marker. Check before driver registration because platform core DMA configuration and single-domain power-on precede the driver's probe callback. Record H16 geometry and validate dynamic no-map preload reservations. Keep H16 fenced: the specification does not define the 27.x CONFIG_GET capability reply sufficiently to validate address width, configuration size or target limits. H14's nonzero-word predicate cannot prove those contracts. Do not expose program loading or infer task descriptor fields. Add KUnit coverage for changing physical placement, adjacency, overlap, alignment, overflow, segment size and DVA order. Document the remaining contracts required for execution support. This is preparation and refusal handling, not completed H16 inference support. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 63 ++++++++ Documentation/accel/index.rst | 1 + MAINTAINERS | 1 + drivers/accel/Makefile | 1 + drivers/accel/ane/Kconfig | 13 ++ drivers/accel/ane/Makefile | 2 + drivers/accel/ane/ane_fw_platform.c | 166 ++++++++++++++++++++ drivers/accel/ane/ane_fw_platform.h | 68 ++++++++ drivers/accel/ane/ane_fw_platform_test.c | 53 +++++++ drivers/accel/ane/ane_t6021.h | 2 + drivers/accel/ane/ane_t6021_rtclient_main.c | 51 +++++- 11 files changed, 420 insertions(+), 1 deletion(-) create mode 100644 Documentation/accel/apple-ane.rst create mode 100644 drivers/accel/ane/ane_fw_platform.c create mode 100644 drivers/accel/ane/ane_fw_platform.h create mode 100644 drivers/accel/ane/ane_fw_platform_test.c diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst new file mode 100644 index 0000000000000..fafa4702ee69d --- /dev/null +++ b/Documentation/accel/apple-ane.rst @@ -0,0 +1,63 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Apple ANE firmware preflight +=========================== + +The shared ANE firmware module recognizes the H16 (``apple,t6041-ane``, +physical T6040/M4 Pro) and H17 (``apple,t8140-ane``) generations. Recognition +is not an inference support claim: both remain unavailable for execution. +They never enter the H13 task-manager or H14 disk-firmware boot paths. + +H17 ownership +------------- + +Linux has no ANE exclave ownership handoff. The module refuses registration +with ``-EOPNOTSUPP`` and one informational message when an H17 node is +present, including a disabled node. This check occurs before platform core +DMA configuration and automatic power-domain attachment. A probe-time +check alone would be too late. Removing ``apple,exclave-assigned`` does not +establish ownership and does not bypass refusal. + +H16 preload handoff +------------------ + +The bootloader must reserve the personalized TEXT and DATA preload segments +as separate, non-reusable ``no-map`` reserved-memory nodes, attach them in +that order through ``memory-region`` and name them ``firmware-text`` and +``firmware-data``. It must also supply ``apple,firmware-iovas`` as two 64-bit +values: ``0x10000000000`` and ``0x100000c0000``. Their sizes must be +``0xc0000`` and ``0x2ac000``. Physical placement is boot-dependent; board DT +sources must not contain captured physical addresses. The reservations must +remain intact until reboot, including after driver failure. + +H16 preflight checks resource geometry, firmware identity and these ranges +without MMIO or power access. No disk firmware is requested. Missing +reservations fail with ``-ENODATA``. Malformed reservations fail with +``-EINVAL``. Even a valid handoff currently fails with ``-EOPNOTSUPP``: +the 27.x CONFIG_GET capability reply and runtime-patch allow-list are not +defined sufficiently to implement the required validation gate. No accel +node or program-loading interface is published. + +The H16 DT nodes stay disabled. The five real PMGR providers already encode +the system/CPU/MPM/task/base dependency graph. The PMGR binding does not +represent the virtual no-state gates, so no fictitious register-backed +domains are created for them. SID 0 and SID 11 on all three DARTs use the +shared DMA domain; no untested stream remapping is added. + +Remaining implementation gates +------------------------------ + +A later change needs a sanitized, versioned 27.x CONFIG_GET reply contract +covering address width, configuration size, resource limits and supported +targets. It also needs the live channel-table schema (including slot-size +semantics), a decoded allow-list of iBoot runtime patches before programming +an unlatched RVBAR, and a complete HWX target/section/relocation validation +contract. The current section-only ABI does not carry the original HWX +container identity. A caller assertion of subtype is insufficient. + +Only after these contracts are implemented and validated may H16 expose +PROG_LOAD, PROC_CREATE and EXEC. Completion must be the matching finished +event, and any uncertain DMA state must remain quarantined until reboot +unless a reset has been proved. H17 additionally requires an ownership +backend before any direct-engine access. There are no module parameters to +bypass these gates. diff --git a/Documentation/accel/index.rst b/Documentation/accel/index.rst index cbc7d4c3876a1..ba0f4f8dc6cef 100644 --- a/Documentation/accel/index.rst +++ b/Documentation/accel/index.rst @@ -8,6 +8,7 @@ Compute Accelerators :maxdepth: 1 introduction + apple-ane amdxdna/index qaic/index rocket/index diff --git a/MAINTAINERS b/MAINTAINERS index a903e83bd0ecb..dcbb7b76c44d1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8184,6 +8184,7 @@ L: asahi@lists.linux.dev L: dri-devel@lists.freedesktop.org S: Maintained F: Documentation/ABI/testing/sysfs-class-accel-ane +F: Documentation/accel/apple-ane.rst F: Documentation/devicetree/bindings/npu/apple,ane.yaml F: drivers/accel/ane/ F: include/uapi/drm/ane_accel.h diff --git a/drivers/accel/Makefile b/drivers/accel/Makefile index 235ebf723025c..f14fa5f5942f5 100644 --- a/drivers/accel/Makefile +++ b/drivers/accel/Makefile @@ -2,6 +2,7 @@ obj-$(CONFIG_DRM_ACCEL_AMDXDNA) += amdxdna/ obj-$(CONFIG_DRM_ACCEL_ANE) += ane/ +obj-$(CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST) += ane/ obj-$(CONFIG_DRM_ACCEL_ARM_ETHOSU) += ethosu/ obj-$(CONFIG_DRM_ACCEL_HABANALABS) += habanalabs/ obj-$(CONFIG_DRM_ACCEL_IVPU) += ivpu/ diff --git a/drivers/accel/ane/Kconfig b/drivers/accel/ane/Kconfig index a14e621707a54..0adf7a05bee7f 100644 --- a/drivers/accel/ane/Kconfig +++ b/drivers/accel/ane/Kconfig @@ -12,6 +12,8 @@ config DRM_ACCEL_ANE Choose this option if you have an Apple SoC with a Neural Engine: M1 (T8103), M1 Pro, Max and Ultra (T6000, T6001, T6002), M2 (T8112), or M2 Pro, Max and Ultra (T6020, T6021, T6022). + H16 (M4 Pro) and H17 are recognized for firmware preflight and + ownership refusal only; execution support is not available. The interface exposed to userspace is described in include/uapi/drm/ane_accel.h. @@ -33,3 +35,14 @@ config DRM_ACCEL_ANE_KUNIT_TEST past them and with a size product that overflows. If unsure, say N. + +config DRM_ACCEL_ANE_FW_KUNIT_TEST + tristate "KUnit tests for ANE firmware preflight" if !KUNIT_ALL_TESTS + depends on DRM_ACCEL && KUNIT + default KUNIT_ALL_TESTS + help + Check the geometry and ordering of the bootloader-provided firmware + preload ranges without accessing hardware. These tests can run on + architectures other than Apple arm64. + + If unsure, say N. diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile index 8103e33a112a4..dadc72ff98dc1 100644 --- a/drivers/accel/ane/Makefile +++ b/drivers/accel/ane/Makefile @@ -2,6 +2,7 @@ obj-$(CONFIG_DRM_ACCEL_ANE) += ane.o ane_t6021.o obj-$(CONFIG_DRM_ACCEL_ANE_KUNIT_TEST) += ane_drv_test.o +obj-$(CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST) += ane_fw_platform_test.o ane-y := \ ane_drv.o \ @@ -10,6 +11,7 @@ ane-y := \ ane_boost.o ane_t6021-y := \ + ane_fw_platform.o \ ane_t6021_rtclient_main.o \ ane_t6021_fwload.o \ ane_t6021_boot.o diff --git a/drivers/accel/ane/ane_fw_platform.c b/drivers/accel/ane/ane_fw_platform.c new file mode 100644 index 0000000000000..b6741b83de2d5 --- /dev/null +++ b/drivers/accel/ane/ane_fw_platform.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * H16/H17 facts from the 2026-10-31 H16/H17 specification, sections 3-7, + * 12 and 15. This preflight performs no MMIO, DMA mapping or power access. + */ +#include +#include +#include +#include + +#include "ane_fw_platform.h" + +static const char * const ane_h16_reg_names[] = { + "engine", "pmgr", "aux-0", "aux-1", "aux-2", "aux-3", "aux-4", +}; + +static const u32 ane_h16_reg_sizes[] = { + 0x2000000, 0x18000, 0x4000, 0x30000, 0xff4000, 0x2000000, 0x4000, +}; + +/* Offsets in reg[1]; genpd, never the ANE driver, owns transitions. */ +static const u32 ane_h16_power_words[] = { + 0x3c0, 0xc008, 0xc000, 0xc010, 0xc018, +}; + +const struct ane_fw_platform ane_h16_platform = { + /* The DT compatible is t6041; the physical M4 Pro is T6040. */ + .soc = 0x6040, + .chip_family = 16, + .ane_type = 0x110, + .hwx_subtype = 7, + .task_generation = 17, + .interrupt_count = 1, + .owner = ANE_FW_OWNER_AP, + .reg_names = ane_h16_reg_names, + .reg_sizes = ane_h16_reg_sizes, + .reg_count = ARRAY_SIZE(ane_h16_reg_names), + .wrapper = 0x1600000, + .mailbox = 0x1608000, + .scratch = 0x1880020, + .doorbell = 0x1884000, + .irq_status = 0x188c000, + .irq_ack = 0x1890000, + .power_words = ane_h16_power_words, + .power_count = ARRAY_SIZE(ane_h16_power_words), + .preload_dva = { 0x10000000000, 0x100000c0000 }, + .preload_size = { 0xc0000, 0x2ac000 }, + .stream_mask = BIT(0) | BIT(11), + .firmware_preloaded = true, + .safe_reset = false, +}; + +const struct ane_fw_platform ane_h17_platform = { + .soc = 0x8140, + .chip_family = 17, + .ane_type = 0x200, + .hwx_subtype = 9, + .task_generation = 19, + .interrupt_count = 2, + .owner = ANE_FW_OWNER_EXCLAVE, + .preload_dva = { 0x10000000000, 0x100000c0000 }, + .preload_size = { 0xc0000, 0x2ac000 }, + .stream_mask = BIT(0) | BIT(11), + .firmware_preloaded = true, + .safe_reset = false, + /* No H16 register-map inference is executable on H17. */ +}; + +static int ane_fw_preload_check(struct device *dev, + const struct ane_fw_platform *soc) +{ + struct device_node *np = dev->of_node; + struct device_node *reserved __free(device_node) = + of_find_node_by_path("/reserved-memory"); + u64 phys[2], size[2], dva[2]; + unsigned int i; + + if (!reserved || + of_property_count_u32_elems(np, "memory-region") != 2 || + of_property_count_strings(np, "memory-region-names") != 2 || + of_property_count_u64_elems(np, "apple,firmware-iovas") != 2 || + of_property_read_u64_array(np, "apple,firmware-iovas", dva, 2)) + return -ENODATA; + + for (i = 0; i < 2; i++) { + struct device_node *mem; + struct reserved_mem *rmem; + const char *name; + bool valid; + + if (of_property_read_string_index(np, "memory-region-names", i, + &name) || + strcmp(name, i ? "firmware-data" : "firmware-text")) + return -EINVAL; + mem = of_parse_phandle(np, "memory-region", i); + if (!mem) + return -EINVAL; + rmem = of_reserved_mem_lookup(mem); + valid = rmem && mem->parent == reserved && + of_device_is_available(mem) && + of_property_read_bool(mem, "no-map") && + !of_property_read_bool(mem, "reusable"); + if (valid) { + phys[i] = rmem->base; + size[i] = rmem->size; + } + of_node_put(mem); + if (!valid) + return -EINVAL; + } + + return ane_fw_preload_ranges_valid(soc, phys, size, dva) ? 0 : -EINVAL; +} + +int ane_fw_platform_preflight(struct platform_device *pdev, + const struct ane_fw_platform *soc) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct resource *res; + u32 ane_type; + unsigned int i; + int ret; + + /* No backend has acquired ownership, even if DT omits the marker. + * Check before resources, firmware, power, IOMMU or engine access. + */ + if (soc->owner == ANE_FW_OWNER_EXCLAVE) { + dev_info(dev, "H17 ANE: Linux exclave ownership handoff unavailable; refusing probe\n"); + return -EOPNOTSUPP; + } + if (of_property_read_bool(np, "apple,exclave-assigned")) { + dev_info(dev, "ANE: exclave-assigned device has no Linux ownership provider\n"); + return -EOPNOTSUPP; + } + + if (!of_property_read_bool(np, "apple,firmware-preloaded") || + of_property_read_u32(np, "apple,ane-type", &ane_type) || + ane_type != soc->ane_type) + return dev_err_probe(dev, -EINVAL, "H16 firmware identity missing or invalid\n"); + + if (of_property_count_strings(np, "reg-names") != soc->reg_count || + platform_get_resource(pdev, IORESOURCE_MEM, soc->reg_count) || + platform_irq_count(pdev) != soc->interrupt_count) + return dev_err_probe(dev, -EINVAL, "H16 resource count mismatch\n"); + + for (i = 0; i < soc->reg_count; i++) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + soc->reg_names[i]); + if (!res || resource_size(res) != soc->reg_sizes[i]) + return dev_err_probe(dev, -EINVAL, "H16 resource geometry mismatch\n"); + } + + ret = ane_fw_preload_check(dev, soc); + if (ret) + return dev_err_probe(dev, ret, + "H16 requires bootloader-reserved no-map TEXT/DATA preload ranges\n"); + + /* The 27.x CONFIG_GET capability reply and preload runtime-patch + * allow-list have not been specified. The H14 check (+8 != 0) does + * not establish the address width, target or resource limits. Do not + * start firmware or expose PROG_LOAD using that weaker predicate. + */ + dev_info(dev, "H16 ANE: 27.x CONFIG_GET contract unavailable; refusing before power access\n"); + return -EOPNOTSUPP; +} diff --git a/drivers/accel/ane/ane_fw_platform.h b/drivers/accel/ane/ane_fw_platform.h new file mode 100644 index 0000000000000..93d8768217ebd --- /dev/null +++ b/drivers/accel/ane/ane_fw_platform.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_FW_PLATFORM_H__ +#define __ANE_FW_PLATFORM_H__ + +#include +#include +#include +#include +#include +#include + +struct platform_device; + +enum ane_fw_owner { + ANE_FW_OWNER_AP, + ANE_FW_OWNER_EXCLAVE, +}; + +/* Later firmware generations must never enter the H14 staging path. */ +struct ane_fw_platform { + u32 soc; + u16 chip_family; + u16 ane_type; + u8 hwx_subtype; + u8 task_generation; + u8 interrupt_count; + enum ane_fw_owner owner; + const char * const *reg_names; + const u32 *reg_sizes; + u32 reg_count; + u32 wrapper; + u32 mailbox; + u32 scratch; + u32 doorbell; + u32 irq_status; + u32 irq_ack; + const u32 *power_words; + u32 power_count; + u64 preload_dva[2]; + u64 preload_size[2]; + u16 stream_mask; + bool firmware_preloaded; + bool safe_reset; +}; + +/* Pure range check: the boot loader supplies the physical addresses. */ +static inline bool +ane_fw_preload_ranges_valid(const struct ane_fw_platform *soc, const u64 phys[2], + const u64 size[2], const u64 dva[2]) +{ + u64 end[2]; + unsigned int i; + + for (i = 0; i < 2; i++) { + if (!phys[i] || !IS_ALIGNED(phys[i], SZ_16K) || + size[i] != soc->preload_size[i] || + dva[i] != soc->preload_dva[i] || + check_add_overflow(phys[i], size[i], &end[i])) + return false; + } + return end[0] <= phys[1] || end[1] <= phys[0]; +} + +extern const struct ane_fw_platform ane_h16_platform, ane_h17_platform; +int ane_fw_platform_preflight(struct platform_device *pdev, + const struct ane_fw_platform *soc); + +#endif /* __ANE_FW_PLATFORM_H__ */ diff --git a/drivers/accel/ane/ane_fw_platform_test.c b/drivers/accel/ane/ane_fw_platform_test.c new file mode 100644 index 0000000000000..0f7171ec95a7e --- /dev/null +++ b/drivers/accel/ane/ane_fw_platform_test.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "ane_fw_platform.h" + +static void ane_preload_ranges_test(struct kunit *test) +{ + const struct ane_fw_platform soc = { + .preload_dva = { 0x10000000000, 0x100000c0000 }, + .preload_size = { 0xc0000, 0x2ac000 }, + }; + u64 phys[] = { 0x100000000, 0x200000000 }; + u64 size[] = { 0xc0000, 0x2ac000 }; + u64 dva[] = { 0x10000000000, 0x100000c0000 }; + + KUNIT_EXPECT_TRUE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + /* Physical placement may change independently on every boot. */ + phys[0] = 0x300000000; + KUNIT_EXPECT_TRUE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + phys[0] = phys[1] + SZ_16K; + KUNIT_EXPECT_FALSE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + phys[0] = phys[1] - size[0]; + KUNIT_EXPECT_TRUE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + phys[0]++; + KUNIT_EXPECT_FALSE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + phys[0] = 0; + KUNIT_EXPECT_FALSE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + phys[0] = U64_MAX - SZ_16K + 1; + KUNIT_EXPECT_FALSE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + phys[0] = 0x100000000; + size[1] -= SZ_16K; + KUNIT_EXPECT_FALSE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + size[1] = 0x2ac000; + dva[0] = 0x100000c0000; + dva[1] = 0x10000000000; + KUNIT_EXPECT_FALSE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); +} + +static struct kunit_case ane_fw_platform_cases[] = { + KUNIT_CASE(ane_preload_ranges_test), + {} +}; + +static struct kunit_suite ane_fw_platform_suite = { + .name = "ane-fw-platform", + .test_cases = ane_fw_platform_cases, +}; + +kunit_test_suite(ane_fw_platform_suite); + +MODULE_DESCRIPTION("KUnit tests for ANE firmware preload range validation"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_t6021.h b/drivers/accel/ane/ane_t6021.h index cf04c6699710e..ce134209c5e06 100644 --- a/drivers/accel/ane/ane_t6021.h +++ b/drivers/accel/ane/ane_t6021.h @@ -742,6 +742,7 @@ bool ane_t6021_fwload_options_ok(void); bool ane_t6021_fwload_placement_ok(struct device *dev); bool ane_t6021_fwload_requested(void); +struct ane_fw_platform; struct ane_fw_image; struct ane_asc_tunables; @@ -762,6 +763,7 @@ struct ane_asc_tunables; * trace_td; 0 = trace_td unsupported. */ struct ane_t602x_soc { + const struct ane_fw_platform *platform; u32 soc; u32 soc_revision; bool revision_fuse; diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 40e643f58a8ca..04f38fc9bd4e8 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -65,6 +65,7 @@ #include #include #include +#include #include #include #include @@ -90,6 +91,7 @@ #include #include +#include "ane_fw_platform.h" #include "ane_stats.h" #include "ane_t6021.h" #include "ane_t6021_boot.h" @@ -2483,9 +2485,16 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, static int ane_rtclient_probe(struct platform_device *pdev) { + const struct ane_t602x_soc *soc = of_device_get_match_data(&pdev->dev); bool hardware_touched = false; int ret; + if (!soc) + return -ENODEV; + /* Later generations cannot enter H14 boot or parameter paths. */ + if (soc->platform) + return ane_fw_platform_preflight(pdev, soc->platform); + mutex_lock(&ane_rtclient_probe_guard_lock); if (ane_rtclient_probe_failed_dirty) { mutex_unlock(&ane_rtclient_probe_guard_lock); @@ -2526,11 +2535,21 @@ static void ane_rtclient_remove(struct platform_device *pdev) "remove HELD: no teardown — reboot reclaims\n"); } +static const struct ane_t602x_soc ane_h16_soc = { + .platform = &ane_h16_platform, +}; + +static const struct ane_t602x_soc ane_h17_soc = { + .platform = &ane_h17_platform, +}; + static const struct of_device_id ane_rtclient_of_match[] = { { .compatible = "apple,t6020-ane", .data = &ane_t6020_soc }, { .compatible = "apple,t6021-ane", .data = &ane_t6021_soc }, { .compatible = "apple,t6022-ane", .data = &ane_t6022_soc }, { .compatible = "apple,t8112-ane", .data = &ane_t8112_soc }, + { .compatible = "apple,t6041-ane", .data = &ane_h16_soc }, + { .compatible = "apple,t8140-ane", .data = &ane_h17_soc }, { } }; MODULE_DEVICE_TABLE(of, ane_rtclient_of_match); @@ -2588,8 +2607,38 @@ static struct platform_driver ane_rtclient_driver = { static int __init ane_rtclient_init(void) { + const struct of_device_id *match; + struct device_node *np; int ret; + /* platform_probe() powers a single genpd, and DMA configuration + * happens even earlier. Refuse later generations before registering + * the driver, including disabled nodes, so these core paths cannot + * precede the ownership and firmware preflight. These SoCs have one + * ANE generation; refusing registration cannot hide an H14 engine. + */ + for_each_matching_node_and_match(np, ane_rtclient_of_match, &match) { + const struct ane_t602x_soc *soc = match->data; + struct platform_device *pdev; + + if (!soc->platform) + continue; + if (soc->platform->owner == ANE_FW_OWNER_EXCLAVE) { + of_node_put(np); + pr_info("ane: H17 ANE: Linux exclave ownership handoff unavailable; refusing registration\n"); + return -EOPNOTSUPP; + } + pdev = of_find_device_by_node(np); + of_node_put(np); + if (!pdev) { + pr_info("ane: H16 ANE: firmware transport unvalidated; keeping disabled\n"); + return -EOPNOTSUPP; + } + ret = ane_fw_platform_preflight(pdev, soc->platform); + put_device(&pdev->dev); + return ret; + } + WRITE_ONCE(ane_rtclient_probe_failed_dirty, false); ret = platform_driver_register(&ane_rtclient_driver); @@ -2607,4 +2656,4 @@ static void __exit ane_rtclient_exit(void) module_exit(ane_rtclient_exit); MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("Apple Neural Engine (T6021/M2) installed module"); +MODULE_DESCRIPTION("Apple Neural Engine firmware driver with H16/H17 preflight"); From 501bd20b3f7df7abc04fd76f7d875e87486040ed Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 03:17:10 +1100 Subject: [PATCH 1453/1579] accel/ane: drop unused legacy exchange ring pointers The legacy exchange assigns BUF_T2H and IO_T2H pointers without reading them. Remove the unused locals and assignments so W=1 is clean for the shared firmware transport. Ring service and completion behavior do not change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane_t6021_rtclient_main.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 04f38fc9bd4e8..7f1e9afbb7579 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -483,7 +483,6 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, { struct ane_t6021 *a = ane->fw; u64 *io, *malloc_ring, header; - u64 *t2h_buf, *t2h_ioq; void __iomem *ipi = ane->engine + ANE_IPI_OFF; unsigned int cursor = ane->legacy_malloc_cursor; unsigned long deadline; @@ -501,8 +500,6 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, io = a->boot_ipc + ane_t6021_chman_layout[channel].off + (size_t)ane->legacy_cmd_cursor[channel] * 64; malloc_ring = a->boot_ipc + ane_t6021_chman_layout[5].off; - t2h_buf = a->boot_ipc + ane_t6021_chman_layout[4].off; - t2h_ioq = a->boot_ipc + ane_t6021_chman_layout[6].off; if (!(READ_ONCE(io[0]) & 1)) { result = -EBUSY; From dd470c053e4be9b7ca7d5c92987c12fa29f2c8f4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 03:48:08 +1100 Subject: [PATCH 1454/1579] accel/ane: count H16 interrupts without creating IRQ mappings platform_irq_count() calls platform_get_irq_optional() and can create IRQ mappings. Preflight must only inspect the DT description before refusing the unsupported firmware contract. Use of_irq_count() to parse and count the interrupt specifiers without allocating an IRQ mapping. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane_fw_platform.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/accel/ane/ane_fw_platform.c b/drivers/accel/ane/ane_fw_platform.c index b6741b83de2d5..74f198504f105 100644 --- a/drivers/accel/ane/ane_fw_platform.c +++ b/drivers/accel/ane/ane_fw_platform.c @@ -5,6 +5,7 @@ */ #include #include +#include #include #include @@ -141,7 +142,7 @@ int ane_fw_platform_preflight(struct platform_device *pdev, if (of_property_count_strings(np, "reg-names") != soc->reg_count || platform_get_resource(pdev, IORESOURCE_MEM, soc->reg_count) || - platform_irq_count(pdev) != soc->interrupt_count) + of_irq_count(np) != soc->interrupt_count) return dev_err_probe(dev, -EINVAL, "H16 resource count mismatch\n"); for (i = 0; i < soc->reg_count; i++) { From d89527ddb6a3ba118cf2f4510f197172471dd6d4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 06:25:41 +1100 Subject: [PATCH 1455/1579] accel/ane: validate the SPEC-r2 channel, reply and task contracts Parse the eight 27.x channel descriptors by name, checking owned-memory bounds, counts, types and overlaps. Use copied dynamic geometry in the shared exchange path and detect descriptor mutation. Distinguish a zero successful acknowledgment from a nonzero firmware refusal; quarantine failed exchanges without retiring corrupted slots or resetting hardware. Add version-12 RTKit ordering and exact CONFIG_GET/completion predicates. Keep one H16 execution deadline across acknowledgment and finish, retain all firmware-visible memory on failure and reject new BO work while quarantined. Preserve the legacy reserved-word and completion behavior. Add H16 nine-word task-stream/register and local relocation checks, with KUnit coverage. Add the device-free add-128 input/oracle selftest from the SPEC-r2 hashes. These components do not enable H16: the pristine preload comparison reference and custom HWX descriptor/aperture byte layouts are still required before a complete original-container ABI or boot path can be admitted. Keep H17 ownership refusal unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 61 ++-- MAINTAINERS | 1 + drivers/accel/ane/Kconfig | 6 +- drivers/accel/ane/Makefile | 3 +- drivers/accel/ane/ane_fw_platform.c | 10 +- drivers/accel/ane/ane_fw_protocol.h | 244 ++++++++++++++++ drivers/accel/ane/ane_fw_protocol_test.c | 260 ++++++++++++++++++ drivers/accel/ane/ane_h16_tasks.h | 143 ++++++++++ drivers/accel/ane/ane_h16_tasks_test.c | 103 +++++++ drivers/accel/ane/ane_t6021_rtclient_main.c | 221 +++++++++++---- tools/testing/selftests/Makefile | 1 + tools/testing/selftests/drivers/ane/Makefile | 3 + .../selftests/drivers/ane/h16-fixture.py | 65 +++++ 13 files changed, 1049 insertions(+), 72 deletions(-) create mode 100644 drivers/accel/ane/ane_fw_protocol.h create mode 100644 drivers/accel/ane/ane_fw_protocol_test.c create mode 100644 drivers/accel/ane/ane_h16_tasks.h create mode 100644 drivers/accel/ane/ane_h16_tasks_test.c create mode 100644 tools/testing/selftests/drivers/ane/Makefile create mode 100755 tools/testing/selftests/drivers/ane/h16-fixture.py diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index fafa4702ee69d..6e636062bd2d0 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -34,8 +34,9 @@ H16 preflight checks resource geometry, firmware identity and these ranges without MMIO or power access. No disk firmware is requested. Missing reservations fail with ``-ENODATA``. Malformed reservations fail with ``-EINVAL``. Even a valid handoff currently fails with ``-EOPNOTSUPP``: -the 27.x CONFIG_GET capability reply and runtime-patch allow-list are not -defined sufficiently to implement the required validation gate. No accel +the pinned immutable-image comparison reference and the byte layouts of the +custom HWX descriptors/aperture records are not supplied. The section-only +load ABI is explicitly refused for H16. No accel node or program-loading interface is published. The H16 DT nodes stay disabled. The five real PMGR providers already encode @@ -44,20 +45,48 @@ represent the virtual no-state gates, so no fictitious register-backed domains are created for them. SID 0 and SID 11 on all three DARTs use the shared DMA domain; no untested stream remapping is added. +Implemented SPEC-r2 components +------------------------------ + +The shared transport has an eight-descriptor dynamic channel parser with +unique names, bounded counts, owned-surface bounds and overlap checks. The +runtime uses copied descriptor geometry, verifies the descriptor snapshot, +and checks the complete IO reply envelope. Zero acknowledgment is success; +a nonzero acknowledgment is a firmware refusal, never a retry or success. +Malformed replies, guard changes and timeouts quarantine the instance. A +quarantined queue cannot allocate another BO, and no reset is attempted. + +The version-12 RTKit management state machine validates HELLO, endpoint maps, +system endpoint start, IOP/AP power acknowledgments and application endpoint +start ordering. It does not invent responses for unknown message types or +unimplemented system endpoint buffer requests. Its caller must provide one +absolute session deadline and a bounded 64-bit mailbox transport. + +H16 validation helpers walk nine fixed task-header words, the documented +conditional extra word, dense/scatter records and the nine allowed register +blocks. A separate helper checks the eight-byte Mach-O relocation entries, +unique aligned offsets, the actual kernel-section ordinal and addend bounds. +These helpers do not substitute for complete container/tensor/program and +16-byte task-aperture metadata validation. + +The device-free selftest in ``tools/testing/selftests/drivers/ane`` generates +the SPEC-r2 fp16 add-128 inputs and CPU oracle and verifies all three pinned +buffer hashes. It can also check a supplied HWX against the pinned size/hash. +It never opens an accel device or claims a Linux inference result. + Remaining implementation gates ------------------------------ -A later change needs a sanitized, versioned 27.x CONFIG_GET reply contract -covering address width, configuration size, resource limits and supported -targets. It also needs the live channel-table schema (including slot-size -semantics), a decoded allow-list of iBoot runtime patches before programming -an unlatched RVBAR, and a complete HWX target/section/relocation validation -contract. The current section-only ABI does not carry the original HWX -container identity. A caller assertion of subtype is insufficient. - -Only after these contracts are implemented and validated may H16 expose -PROG_LOAD, PROC_CREATE and EXEC. Completion must be the matching finished -event, and any uncertain DMA state must remain quarantined until reboot -unless a reset has been proved. H17 additionally requires an ownership -backend before any direct-engine access. There are no module parameters to -bypass these gates. +SPEC-r2 defines the 27.x channel and CONFIG_GET contract, including the exact +24 MHz reply requirement. It does not give the byte offsets and widths of +the custom kind-3 tensor/kind-4 program descriptors or 16-byte task-aperture +records, nor an available pristine-image comparison reference/manifest. +The 43 patchbay record widths/order and original immutable bytes must be +proved against the SHA-pinned payload, not inferred from mutable live bytes. +A SHA-256 digest alone cannot reconstruct that reference. + +Until those exact layouts and the validated comparison reference are +available, H16 remains refused before power access. +No physical RTKit boot or complete HWX load path is enabled. H17 additionally +requires a Linux ownership backend before any direct-engine access. There +are no module parameters to bypass these gates. diff --git a/MAINTAINERS b/MAINTAINERS index dcbb7b76c44d1..7f91cfd21f6e6 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8188,6 +8188,7 @@ F: Documentation/accel/apple-ane.rst F: Documentation/devicetree/bindings/npu/apple,ane.yaml F: drivers/accel/ane/ F: include/uapi/drm/ane_accel.h +F: tools/testing/selftests/drivers/ane/ DRM ACCEL DRIVERS FOR INTEL VPU M: Karol Wachowski diff --git a/drivers/accel/ane/Kconfig b/drivers/accel/ane/Kconfig index 0adf7a05bee7f..c12d49c38a59a 100644 --- a/drivers/accel/ane/Kconfig +++ b/drivers/accel/ane/Kconfig @@ -41,8 +41,8 @@ config DRM_ACCEL_ANE_FW_KUNIT_TEST depends on DRM_ACCEL && KUNIT default KUNIT_ALL_TESTS help - Check the geometry and ordering of the bootloader-provided firmware - preload ranges without accessing hardware. These tests can run on - architectures other than Apple arm64. + Check preload ranges, 27.x channel and reply validation, RTKit + management ordering, and H16 task/relocation bounds without hardware. + These tests can run on architectures other than Apple arm64. If unsure, say N. diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile index dadc72ff98dc1..db8286c2ee3cf 100644 --- a/drivers/accel/ane/Makefile +++ b/drivers/accel/ane/Makefile @@ -2,7 +2,8 @@ obj-$(CONFIG_DRM_ACCEL_ANE) += ane.o ane_t6021.o obj-$(CONFIG_DRM_ACCEL_ANE_KUNIT_TEST) += ane_drv_test.o -obj-$(CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST) += ane_fw_platform_test.o +obj-$(CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST) += ane_fw_platform_test.o \ + ane_fw_protocol_test.o ane_h16_tasks_test.o ane-y := \ ane_drv.o \ diff --git a/drivers/accel/ane/ane_fw_platform.c b/drivers/accel/ane/ane_fw_platform.c index 74f198504f105..99c2ba09418ac 100644 --- a/drivers/accel/ane/ane_fw_platform.c +++ b/drivers/accel/ane/ane_fw_platform.c @@ -157,11 +157,11 @@ int ane_fw_platform_preflight(struct platform_device *pdev, return dev_err_probe(dev, ret, "H16 requires bootloader-reserved no-map TEXT/DATA preload ranges\n"); - /* The 27.x CONFIG_GET capability reply and preload runtime-patch - * allow-list have not been specified. The H14 check (+8 != 0) does - * not establish the address width, target or resource limits. Do not - * start firmware or expose PROG_LOAD using that weaker predicate. + /* SPEC-r2 defines CONFIG_GET and dynamic channels. Opening this gate + * additionally needs a pinned immutable-image comparison reference and + * the exact custom HWX descriptor/aperture layouts. Neither may be + * replaced by a broad preload-diff rule or a section-only LOAD ioctl. */ - dev_info(dev, "H16 ANE: 27.x CONFIG_GET contract unavailable; refusing before power access\n"); + dev_info(dev, "H16 ANE: pinned preload reference and HWX layout unavailable; refusing before power access\n"); return -EOPNOTSUPP; } diff --git a/drivers/accel/ane/ane_fw_protocol.h b/drivers/accel/ane/ane_fw_protocol.h new file mode 100644 index 0000000000000..aa238dfb2eb2a --- /dev/null +++ b/drivers/accel/ane/ane_fw_protocol.h @@ -0,0 +1,244 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_FW_PROTOCOL_H__ +#define __ANE_FW_PROTOCOL_H__ + +#include +#include +#include +#include +#include +#include + +#define ANE_FW_CHANNELS 8 +#define ANE_FW_DESCRIPTOR_SIZE 0x100 +#define ANE_FW_TABLE_SIZE (ANE_FW_CHANNELS * ANE_FW_DESCRIPTOR_SIZE) +#define ANE_FW_SLOT_SIZE 64 +#define ANE_FW_TICK_RATE 24000000U + +struct ane_fw_channel { + u64 offset; + u32 slots; + u32 type; + u32 bit; +}; + +static inline bool ane_fw_contains(u64 base, u64 bytes, u64 addr, u64 size) +{ + return size <= bytes && addr >= base && addr - base <= bytes - size; +} + +/* Parse a host-owned descriptor snapshot; output indices are logical names. */ +static inline int +ane_fw_channels_parse(const void *table, u64 ipc_dva, size_t ipc_size, + u64 table_dva, struct ane_fw_channel channels[ANE_FW_CHANNELS]) +{ + static const char * const names[ANE_FW_CHANNELS] = { + "TERMINAL", "IO", "DEBUG", "BUF_H2T", "BUF_T2H", + "SHAREDMALLOC", "IO_T2H", "DATA_CHAIN_H2T", + }; + struct ane_fw_channel parsed[ANE_FW_CHANNELS] = {}; + u64 table_off; + u32 seen = 0; + unsigned int i, j; + + if (!table || !ane_fw_contains(ipc_dva, ipc_size, table_dva, ANE_FW_TABLE_SIZE)) + return -EPROTO; + table_off = table_dva - ipc_dva; + for (i = 0; i < ANE_FW_CHANNELS; i++) { + const u8 *desc = (const u8 *)table + i * ANE_FW_DESCRIPTOR_SIZE; + struct ane_fw_channel *c; + u64 slots, ring, end; + + if (!memchr(desc, 0, 64)) + return -EPROTO; + for (j = 0; j < ANE_FW_CHANNELS; j++) + if (!strcmp((const char *)desc, names[j])) + break; + if (j == ANE_FW_CHANNELS || (seen & BIT(j))) + return -EPROTO; + seen |= BIT(j); + c = &parsed[j]; + c->type = get_unaligned_le32(desc + 0x40); + c->bit = get_unaligned_le32(desc + 0x44); + slots = get_unaligned_le64(desc + 0x48); + ring = get_unaligned_le64(desc + 0x50); + if (c->type > 2 || c->bit >= 32 || !slots || slots > 0x400 || + (ring & (ANE_FW_SLOT_SIZE - 1)) || + !ane_fw_contains(ipc_dva, ipc_size, ring, + slots * ANE_FW_SLOT_SIZE)) + return -EPROTO; + c->offset = ring - ipc_dva; + c->slots = slots; + end = c->offset + slots * ANE_FW_SLOT_SIZE; + if (c->offset < table_off + ANE_FW_TABLE_SIZE && end > table_off) + return -EPROTO; + } + if (seen != GENMASK(ANE_FW_CHANNELS - 1, 0) || + parsed[1].type != 0 || parsed[5].type != 1) + return -EPROTO; + for (i = 0; i < ANE_FW_CHANNELS; i++) { + u64 end = parsed[i].offset + parsed[i].slots * ANE_FW_SLOT_SIZE; + + for (j = i + 1; j < ANE_FW_CHANNELS; j++) + if (parsed[i].offset < parsed[j].offset + + parsed[j].slots * ANE_FW_SLOT_SIZE && + parsed[j].offset < end) + return -EPROTO; + } + memcpy(channels, parsed, sizeof(parsed)); + return 0; +} + +/* Call only after parsing the table and checking its immutable snapshot. */ +static inline void +ane_fw_channels_host_init(void *ipc, const struct ane_fw_channel *channels) +{ + unsigned int i, slot; + + for (i = 0; i < ANE_FW_CHANNELS; i++) { + const struct ane_fw_channel *c = &channels[i]; + + if (c->type) + continue; + for (slot = 0; slot < c->slots; slot++) { + u8 *p = (u8 *)ipc + c->offset + slot * ANE_FW_SLOT_SIZE; + + memset(p, 0, 24); + put_unaligned_le64(1, p); + } + } +} + +/* A firmware refusal is distinct from corruption and never means success. */ +static inline int +ane_fw_reply_status(u64 header, u64 slot_length, u64 slot_result, + u64 command_dva, size_t length, u16 opcode, const void *command, + bool zero_reserved) +{ + const u8 *p = command; + + if (!(header & 1)) + return -EAGAIN; + if (!command || length < 8 || (command_dva & 1) || header != (command_dva | 1) || + slot_length != length || slot_result || + (zero_reserved && get_unaligned_le32(p)) || + get_unaligned_le16(p + 4) != opcode) + return -EPROTO; + return get_unaligned_le16(p + 6) ? -EREMOTEIO : 0; +} + +static inline int ane_fw_config_status(const void *command, size_t length) +{ + if (!command || length != 16 || get_unaligned_le32((const u8 *)command + 8) != + ANE_FW_TICK_RATE) + return -EPROTO; + return 0; +} + +/* Only a matching state-1 event completes an execution. */ +static inline int +ane_fw_finished_event(const void *event, size_t length, u64 cookie, + u32 program, u32 process) +{ + const u8 *p = event; + u32 state; + + if (!p || length != 0x28 || get_unaligned_le32(p + 4) != 0x300 || + get_unaligned_le64(p + 8) != cookie || + get_unaligned_le32(p + 0x10) != program || + get_unaligned_le32(p + 0x14) != process || + get_unaligned_le32(p + 0x18)) + return -EPROTO; + state = get_unaligned_le32(p + 0x1c); + return state > 1 ? -EPROTO : state; +} + +enum ane_fw_rtk_phase { + ANE_FW_RTK_HELLO, + ANE_FW_RTK_EPMAP, + ANE_FW_RTK_IOP_ACK, + ANE_FW_RTK_AP_ACK, + ANE_FW_RTK_READY, +}; + +struct ane_fw_rtk { + u32 endpoints[8]; + u8 fragments; + enum ane_fw_rtk_phase phase; +}; + +#define ANE_FW_RTK_TYPE(t) ((u64)(t) << 52) +#define ANE_FW_RTK_LAST BIT_ULL(51) + +/* The caller owns one absolute deadline for the complete management session. */ +static inline int +ane_fw_rtk_receive(struct ane_fw_rtk *rtk, u64 word0, u64 word1, + void *ctx, int (*send)(void *ctx, u64 word0, u64 word1)) +{ + static const u8 system_eps[] = { 1, 2, 3, 4, 8, 10 }; + u32 type = (word0 >> 52) & 0xff; + unsigned int base, ep, i; + int ret; + + if (word1 & 0xff) + return -EOPNOTSUPP; + switch (type) { + case 1: + if (rtk->phase != ANE_FW_RTK_HELLO || (word0 & 0xffff) > 12 || + ((word0 >> 16) & 0xffff) < 12) + return -EPROTO; + ret = send(ctx, ANE_FW_RTK_TYPE(2) | (12 << 16) | 12, 0); + if (!ret) + rtk->phase = ANE_FW_RTK_EPMAP; + return ret; + case 8: + if (rtk->phase != ANE_FW_RTK_EPMAP) + return -EPROTO; + base = (word0 >> 32) & 7; + if (rtk->fragments & BIT(base)) + return -EPROTO; + rtk->fragments |= BIT(base); + rtk->endpoints[base] = word0; + ret = send(ctx, ANE_FW_RTK_TYPE(8) | ((u64)base << 32) | + (word0 & ANE_FW_RTK_LAST ? ANE_FW_RTK_LAST : 1), 0); + if (ret || !(word0 & ANE_FW_RTK_LAST)) + return ret; + for (i = 0; i < ARRAY_SIZE(system_eps); i++) { + ep = system_eps[i]; + if (!(rtk->endpoints[0] & BIT(ep))) + continue; + ret = send(ctx, ANE_FW_RTK_TYPE(5) | ((u64)ep << 32) | 2, 0); + if (ret) + return ret; + } + ret = send(ctx, ANE_FW_RTK_TYPE(6) | 0x20, 0); + if (!ret) + rtk->phase = ANE_FW_RTK_IOP_ACK; + return ret; + case 7: + if (rtk->phase != ANE_FW_RTK_IOP_ACK || (word0 & 0xffff) != 0x20) + return -EPROTO; + ret = send(ctx, ANE_FW_RTK_TYPE(0xb) | 0x20, 0); + if (!ret) + rtk->phase = ANE_FW_RTK_AP_ACK; + return ret; + case 0xb: + if (rtk->phase != ANE_FW_RTK_AP_ACK || (word0 & 0xffff) != 0x20) + return -EPROTO; + for (ep = 0x20; ep < 0x100; ep++) { + if (!(rtk->endpoints[ep / 32] & BIT(ep % 32))) + continue; + ret = send(ctx, ANE_FW_RTK_TYPE(5) | ((u64)ep << 32) | 2, 0); + if (ret) + return ret; + } + rtk->phase = ANE_FW_RTK_READY; + return 0; + case 0xc: + return 0; + default: + return -EOPNOTSUPP; + } +} + +#endif /* __ANE_FW_PROTOCOL_H__ */ diff --git a/drivers/accel/ane/ane_fw_protocol_test.c b/drivers/accel/ane/ane_fw_protocol_test.c new file mode 100644 index 0000000000000..e77db9b2c3a51 --- /dev/null +++ b/drivers/accel/ane/ane_fw_protocol_test.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include + +#include "ane_fw_protocol.h" + +#define TEST_IPC_SIZE 0x10000 +#define TEST_IPC_DVA 0x20000000000ULL +#define TEST_TABLE_OFF 0x1000 + +static void *ane_test_table(struct kunit *test) +{ + static const char * const names[] = { + "TERMINAL", "IO", "DEBUG", "BUF_H2T", "BUF_T2H", + "SHAREDMALLOC", "IO_T2H", "DATA_CHAIN_H2T", + }; + void *ipc = kunit_kzalloc(test, TEST_IPC_SIZE, GFP_KERNEL); + unsigned int i; + + if (!ipc) + return NULL; + for (i = 0; i < ANE_FW_CHANNELS; i++) { + u8 *d = ipc + TEST_TABLE_OFF + i * ANE_FW_DESCRIPTOR_SIZE; + + strscpy((char *)d, names[i], 64); + put_unaligned_le32(i == 0 ? 2 : (i == 4 || i == 5 || i == 6), d + 0x40); + put_unaligned_le32(31 - i, d + 0x44); + put_unaligned_le64(i + 1, d + 0x48); + put_unaligned_le64(TEST_IPC_DVA + 0x2000 + i * 0x400, d + 0x50); + } + return ipc; +} + +static int ane_test_channels_parse(void *ipc, struct ane_fw_channel *channels) +{ + return ane_fw_channels_parse(ipc + TEST_TABLE_OFF, TEST_IPC_DVA, TEST_IPC_SIZE, + TEST_IPC_DVA + TEST_TABLE_OFF, channels); +} + +static void ane_dynamic_channels_test(struct kunit *test) +{ + struct ane_fw_channel channels[ANE_FW_CHANNELS]; + u8 *ipc = ane_test_table(test); + u8 *io; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ipc); + KUNIT_ASSERT_EQ(test, ane_test_channels_parse(ipc, channels), 0); + KUNIT_EXPECT_EQ(test, channels[1].bit, 30U); + KUNIT_EXPECT_EQ(test, channels[1].slots, 2U); + KUNIT_EXPECT_EQ(test, channels[1].offset, 0x2400ULL); + { + u8 saved[ANE_FW_DESCRIPTOR_SIZE]; + u8 *first = ipc + TEST_TABLE_OFF + ANE_FW_DESCRIPTOR_SIZE; + u8 *last = ipc + TEST_TABLE_OFF + 7 * ANE_FW_DESCRIPTOR_SIZE; + + memcpy(saved, first, sizeof(saved)); + memcpy(first, last, sizeof(saved)); + memcpy(last, saved, sizeof(saved)); + KUNIT_ASSERT_EQ(test, ane_test_channels_parse(ipc, channels), 0); + KUNIT_EXPECT_EQ(test, channels[1].offset, 0x2400ULL); + } + memset(ipc + 0x2000, 0xa5, 0x2000); + ane_fw_channels_host_init(ipc, channels); + io = ipc + channels[1].offset; + KUNIT_EXPECT_EQ(test, get_unaligned_le64(io), 1ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(io + 8), 0ULL); + KUNIT_EXPECT_EQ(test, io[24], (u8)0xa5); + for (i = 0; i < 24; i++) + KUNIT_EXPECT_EQ(test, ipc[0x2000 + i], (u8)0xa5); +} + +static void ane_channel_snapshot_test(struct kunit *test) +{ + struct ane_fw_channel channels[ANE_FW_CHANNELS]; + u8 *ipc = ane_test_table(test); + u8 *snapshot = kunit_kmalloc(test, ANE_FW_TABLE_SIZE, GFP_KERNEL); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, ipc); + KUNIT_ASSERT_NOT_NULL(test, snapshot); + memcpy(snapshot, ipc + TEST_TABLE_OFF, ANE_FW_TABLE_SIZE); + put_unaligned_le64(0, ipc + TEST_TABLE_OFF + ANE_FW_DESCRIPTOR_SIZE + 0x48); + ret = ane_fw_channels_parse(snapshot, TEST_IPC_DVA, TEST_IPC_SIZE, + TEST_IPC_DVA + TEST_TABLE_OFF, channels); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, channels[1].slots, 2U); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); +} + +static void ane_channel_rejections_test(struct kunit *test) +{ + struct ane_fw_channel channels[ANE_FW_CHANNELS]; + u8 *ipc = ane_test_table(test); + u8 *d; + + KUNIT_ASSERT_NOT_NULL(test, ipc); + d = ipc + TEST_TABLE_OFF + ANE_FW_DESCRIPTOR_SIZE; + put_unaligned_le64(0, d + 0x48); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + put_unaligned_le64(0x401, d + 0x48); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + put_unaligned_le64(2, d + 0x48); + put_unaligned_le64(TEST_IPC_DVA + TEST_TABLE_OFF, d + 0x50); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + put_unaligned_le64(TEST_IPC_DVA + 0x2000, d + 0x50); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + put_unaligned_le64(TEST_IPC_DVA + TEST_IPC_SIZE - 64, d + 0x50); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + put_unaligned_le64(TEST_IPC_DVA + 0x2400, d + 0x50); + put_unaligned_le32(32, d + 0x44); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + put_unaligned_le32(30, d + 0x44); + put_unaligned_le32(1, d + 0x40); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + put_unaligned_le32(0, d + 0x40); + strscpy((char *)d, "TERMINAL", 64); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + memset(d, 0x41, 64); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + KUNIT_EXPECT_FALSE(test, ane_fw_contains(16, 64, U64_MAX, 32)); + KUNIT_EXPECT_FALSE(test, ane_fw_contains(16, 64, 16, 65)); +} + +static void ane_reply_status_test(struct kunit *test) +{ + u8 command[16] = {}; + const u64 dva = 0x30000000000; + + put_unaligned_le16(3, command + 4); + put_unaligned_le32(ANE_FW_TICK_RATE, command + 8); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva, 16, 16, dva, 16, 3, + command, true), -EAGAIN); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 16, 0, dva, 16, 3, + command, true), 0); + KUNIT_EXPECT_EQ(test, ane_fw_config_status(command, sizeof(command)), 0); + put_unaligned_le16(1, command + 6); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 16, 0, dva, 16, 3, + command, true), -EREMOTEIO); + put_unaligned_le16(0xffff, command + 6); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 16, 0, dva, 16, 3, + command, true), -EREMOTEIO); + put_unaligned_le16(0, command + 6); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 15, 0, dva, 16, 3, + command, true), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 16, 1, dva, 16, 3, + command, true), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 16, 0, dva, 16, 4, + command, true), -EPROTO); + put_unaligned_le32(1, command); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 16, 0, dva, 16, 3, + command, true), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 16, 0, dva, 16, 3, + command, false), 0); + put_unaligned_le32(1, command + 8); + KUNIT_EXPECT_EQ(test, ane_fw_config_status(command, sizeof(command)), -EPROTO); +} + +static void ane_finished_event_test(struct kunit *test) +{ + u8 event[0x28] = {}; + + put_unaligned_le32(0x300, event + 4); + put_unaligned_le64(42, event + 8); + put_unaligned_le32(3, event + 0x10); + put_unaligned_le32(4, event + 0x14); + KUNIT_EXPECT_EQ(test, ane_fw_finished_event(event, sizeof(event), 42, 3, 4), 0); + put_unaligned_le32(1, event + 0x1c); + KUNIT_EXPECT_EQ(test, ane_fw_finished_event(event, sizeof(event), 42, 3, 4), 1); + KUNIT_EXPECT_EQ(test, ane_fw_finished_event(event, sizeof(event), 41, 3, 4), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_fw_finished_event(event, sizeof(event), 42, 3, 5), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_fw_finished_event(event, 0x20, 42, 3, 4), -EPROTO); + put_unaligned_le32(2, event + 0x1c); + KUNIT_EXPECT_EQ(test, ane_fw_finished_event(event, sizeof(event), 42, 3, 4), -EPROTO); +} + +struct ane_test_tx { + u64 words[16]; + unsigned int count; +}; + +static int ane_test_send(void *ctx, u64 word0, u64 word1) +{ + struct ane_test_tx *tx = ctx; + + if (word1 || tx->count == ARRAY_SIZE(tx->words)) + return -EOVERFLOW; + tx->words[tx->count++] = word0; + return 0; +} + +static void ane_rtk_order_test(struct kunit *test) +{ + struct ane_fw_rtk rtk = {}; + struct ane_test_tx tx = {}; + + KUNIT_ASSERT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(1) | + (13 << 16) | 11, 0, &tx, ane_test_send), 0); + KUNIT_EXPECT_EQ(test, tx.words[0], ANE_FW_RTK_TYPE(2) | (12 << 16) | 12); + KUNIT_ASSERT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(8) | + BIT(2) | BIT(8), 0, &tx, ane_test_send), 0); + KUNIT_EXPECT_EQ(test, tx.words[1], ANE_FW_RTK_TYPE(8) | 1); + KUNIT_ASSERT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(8) | + BIT_ULL(32) | ANE_FW_RTK_LAST | BIT(0) | BIT(5), 0, + &tx, ane_test_send), 0); + KUNIT_EXPECT_EQ(test, rtk.phase, ANE_FW_RTK_IOP_ACK); + KUNIT_EXPECT_EQ(test, tx.count, 6U); + KUNIT_EXPECT_EQ(test, tx.words[2], ANE_FW_RTK_TYPE(8) | BIT_ULL(32) | + ANE_FW_RTK_LAST); + KUNIT_EXPECT_EQ(test, tx.words[3], ANE_FW_RTK_TYPE(5) | (2ULL << 32) | 2); + KUNIT_EXPECT_EQ(test, tx.words[4], ANE_FW_RTK_TYPE(5) | (8ULL << 32) | 2); + KUNIT_ASSERT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(7) | 0x20, + 0, &tx, ane_test_send), 0); + KUNIT_EXPECT_EQ(test, tx.words[6], ANE_FW_RTK_TYPE(0xb) | 0x20); + KUNIT_ASSERT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(0xb) | 0x20, + 0, &tx, ane_test_send), 0); + KUNIT_EXPECT_EQ(test, rtk.phase, ANE_FW_RTK_READY); + KUNIT_EXPECT_EQ(test, tx.words[7], ANE_FW_RTK_TYPE(5) | (32ULL << 32) | 2); + KUNIT_EXPECT_EQ(test, tx.words[8], ANE_FW_RTK_TYPE(5) | (37ULL << 32) | 2); +} + +static void ane_rtk_rejections_test(struct kunit *test) +{ + struct ane_fw_rtk rtk = {}; + struct ane_test_tx tx = {}; + u64 hello = ANE_FW_RTK_TYPE(1) | (12 << 16) | 12; + int ret; + + KUNIT_EXPECT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(1) | + (2 << 16) | 1, 0, &tx, ane_test_send), -EPROTO); + KUNIT_EXPECT_EQ(test, tx.count, 0U); + ret = ane_fw_rtk_receive(&rtk, hello, 2, &tx, ane_test_send); + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, tx.count, 0U); + KUNIT_EXPECT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(7) | 0x20, + 0, &tx, ane_test_send), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(0xff), + 0, &tx, ane_test_send), -EOPNOTSUPP); +} + +static struct kunit_case ane_protocol_cases[] = { + KUNIT_CASE(ane_dynamic_channels_test), + KUNIT_CASE(ane_channel_snapshot_test), + KUNIT_CASE(ane_channel_rejections_test), + KUNIT_CASE(ane_reply_status_test), + KUNIT_CASE(ane_finished_event_test), + KUNIT_CASE(ane_rtk_order_test), + KUNIT_CASE(ane_rtk_rejections_test), + {} +}; + +static struct kunit_suite ane_protocol_suite = { + .name = "ane-fw-protocol", + .test_cases = ane_protocol_cases, +}; + +kunit_test_suite(ane_protocol_suite); +MODULE_DESCRIPTION("KUnit tests for ANE 27.x channel and RTKit contracts"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_tasks.h b/drivers/accel/ane/ane_h16_tasks.h new file mode 100644 index 0000000000000..8415e692c5239 --- /dev/null +++ b/drivers/accel/ane/ane_h16_tasks.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_TASKS_H__ +#define __ANE_H16_TASKS_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define ANE_H16_TASK_MAX 0x60000 + +/* Byte addresses in the source-backed H16 register image, exclusive ends. */ +static inline bool ane_h16_register_valid(u32 word) +{ + static const struct { + u32 start; + u32 end; + } blocks[] = { + { 0x0000, 0x005c }, { 0x4100, 0x41a4 }, { 0x44d0, 0x44d4 }, + { 0x4500, 0x453c }, { 0x4900, 0x4930 }, { 0x4d00, 0x4e44 }, + { 0x5100, 0x5154 }, { 0x5500, 0x5620 }, { 0x5900, 0x5930 }, + }; + u32 addr = word * 4; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(blocks); i++) + if (addr >= blocks[i].start && addr < blocks[i].end) + return true; + return false; +} + +/* This structural gate does not replace container/tensor/aperture validation. */ +static inline int +ane_h16_task_stream_validate(const void *text, size_t size, u32 expected_tasks) +{ + const u8 *p = text; + size_t pos = 16; + u32 tasks = 0; + + if (!p || !expected_tasks || size < 16 || size > ANE_H16_TASK_MAX || + (size & 3) || memchr_inv(p, 0, 16)) + return -ENOEXEC; + while (pos < size) { + u32 words, header_words = 9; + size_t end, padded, rec; + + if (size - pos < 16) + return -ENOEXEC; + words = (get_unaligned_le32(p + pos) >> 16) & 0x7ff; + if (!words) { + if (memchr_inv(p + pos, 0, 16)) + return -ENOEXEC; + pos += 16; + continue; + } + if (words < 9 || words > (size - pos) / 4) + return -ENOEXEC; + end = pos + words * 4; + if ((get_unaligned_le32(p + pos + 8 * 4) & 3) == 3) + header_words++; + if (words < header_words) + return -ENOEXEC; + rec = pos + header_words * 4; + while (rec < end) { + u32 h = get_unaligned_le32(p + rec); + u32 base = h & 0x7fff; + u32 mask = (h >> 15) & 0xffff; + u32 count, i; + + rec += 4; + if (h & BIT(31)) + count = 1 + hweight32(mask); + else + count = ((h >> 15) & 0x3f) + 1; + if (count > (end - rec) / 4 || !ane_h16_register_valid(base)) + return -ENOEXEC; + if (h & BIT(31)) { + for (i = 0; i < 16; i++) + if ((mask & BIT(i)) && + !ane_h16_register_valid(base + i + 1)) + return -ENOEXEC; + } else { + for (i = 1; i < count; i++) + if (!ane_h16_register_valid(base + i)) + return -ENOEXEC; + } + rec += count * 4; + } + padded = ALIGN(end, 16); + if (padded > size || memchr_inv(p + end, 0, padded - end)) + return -ENOEXEC; + pos = padded; + if (++tasks > expected_tasks) + return -ENOEXEC; + } + return tasks == expected_tasks ? 0 : -ENOEXEC; +} + +/* Section ordinal is supplied only by the validated complete section table. + * This checks Mach-O relocation entries, not the distinct aperture records. + */ +static inline int +ane_h16_relocations_validate(const void *text, size_t text_size, + const void *relocs, size_t count, u32 ordinal, + u64 kernel_vm, u64 kernel_size, + unsigned long *seen, size_t seen_bits) +{ + const u8 *r = relocs; + u64 kernel_end; + size_t i; + + if (!count) + return 0; + if (!text || !relocs || !seen || !ordinal || ordinal > 0xffffff || + !kernel_size || text_size > ANE_H16_TASK_MAX || (text_size & 3) || + count > text_size / 4 || seen_bits < text_size / 4 || + check_add_overflow(kernel_vm, kernel_size, &kernel_end)) + return -ENOEXEC; + bitmap_zero(seen, text_size / 4); + for (i = 0; i < count; i++) { + u32 patch = get_unaligned_le32(r + i * 8); + u32 info = get_unaligned_le32(r + i * 8 + 4); + u32 addend; + u64 target; + + if ((patch & 3) || text_size < 4 || patch > text_size - 4 || + (info >> 24) != 7 || (info & 0xffffff) != ordinal || + test_and_set_bit(patch / 4, seen)) + return -ENOEXEC; + addend = get_unaligned_le32((const u8 *)text + patch); + if (addend >= kernel_size || + check_add_overflow(kernel_vm, (u64)addend, &target) || + target >= kernel_end) + return -ENOEXEC; + } + return 0; +} + +#endif /* __ANE_H16_TASKS_H__ */ diff --git a/drivers/accel/ane/ane_h16_tasks_test.c b/drivers/accel/ane/ane_h16_tasks_test.c new file mode 100644 index 0000000000000..321e7bb0ffb3c --- /dev/null +++ b/drivers/accel/ane/ane_h16_tasks_test.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "ane_h16_tasks.h" + +static void ane_h16_dense_task_test(struct kunit *test) +{ + u8 text[64] = {}; + + /* Nine header words, one dense record and its one-word payload. */ + put_unaligned_le32(11 << 16, text + 16); + put_unaligned_le32(0x4100 / 4, text + 16 + 9 * 4); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), 0); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 2), -ENOEXEC); + put_unaligned_le32(0x41a4 / 4, text + 16 + 9 * 4); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + put_unaligned_le32(0x419c / 4 | (1 << 15), text + 16 + 9 * 4); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + put_unaligned_le32(0x4100 / 4, text + 16 + 9 * 4); + text[63] = 1; + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); +} + +static void ane_h16_scatter_task_test(struct kunit *test) +{ + u8 text[64] = {}; + + put_unaligned_le32(12 << 16, text + 16); + put_unaligned_le32(BIT(31) | BIT(15) | 21, text + 16 + 9 * 4); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), 0); + /* The second scatter write would leave the common register block. */ + put_unaligned_le32(BIT(31) | BIT(15) | 22, text + 16 + 9 * 4); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); +} + +static void ane_h16_extra_header_test(struct kunit *test) +{ + u8 text[64] = {}; + + put_unaligned_le32(10 << 16, text + 16); + put_unaligned_le32(3, text + 16 + 8 * 4); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), 0); + put_unaligned_le32(9 << 16, text + 16); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + KUNIT_EXPECT_FALSE(test, ane_h16_register_valid(0x44d4 / 4)); + KUNIT_EXPECT_TRUE(test, ane_h16_register_valid(0x44d0 / 4)); + KUNIT_EXPECT_FALSE(test, ane_h16_register_valid(0x5930 / 4)); +} + +static void ane_h16_relocation_test(struct kunit *test) +{ + u8 text[64] = {}; + u8 relocs[16] = {}; + DECLARE_BITMAP(seen, 16); + int ret; + + put_unaligned_le32(4, relocs); + put_unaligned_le32(0x07000003, relocs + 4); + put_unaligned_le32(0x20, text + 4); + ret = ane_h16_relocations_validate(text, sizeof(text), + relocs, 1, 3, 0x30020000, 0x100, seen, 16); + KUNIT_EXPECT_EQ(test, ret, 0); + memcpy(relocs + 8, relocs, 8); + ret = ane_h16_relocations_validate(text, sizeof(text), + relocs, 2, 3, 0x30020000, 0x100, seen, 16); + KUNIT_EXPECT_EQ(test, ret, -ENOEXEC); + ret = ane_h16_relocations_validate(text, sizeof(text), + relocs, 1, 4, 0x30020000, 0x100, seen, 16); + KUNIT_EXPECT_EQ(test, ret, -ENOEXEC); + put_unaligned_le32(0x0f000003, relocs + 4); + ret = ane_h16_relocations_validate(text, sizeof(text), + relocs, 1, 3, 0x30020000, 0x100, seen, 16); + KUNIT_EXPECT_EQ(test, ret, -ENOEXEC); + put_unaligned_le32(0x07000003, relocs + 4); + put_unaligned_le32(0x100, text + 4); + ret = ane_h16_relocations_validate(text, sizeof(text), + relocs, 1, 3, 0x30020000, 0x100, seen, 16); + KUNIT_EXPECT_EQ(test, ret, -ENOEXEC); + ret = ane_h16_relocations_validate(text, sizeof(text), + relocs, 1, 3, U64_MAX - 8, 0x100, seen, 16); + KUNIT_EXPECT_EQ(test, ret, -ENOEXEC); + ret = ane_h16_relocations_validate(text, sizeof(text), + relocs, 1, 3, 0x30020000, 0, seen, 16); + KUNIT_EXPECT_EQ(test, ret, -ENOEXEC); +} + +static struct kunit_case ane_h16_task_cases[] = { + KUNIT_CASE(ane_h16_dense_task_test), + KUNIT_CASE(ane_h16_scatter_task_test), + KUNIT_CASE(ane_h16_extra_header_test), + KUNIT_CASE(ane_h16_relocation_test), + {} +}; + +static struct kunit_suite ane_h16_task_suite = { + .name = "ane-h16-tasks", + .test_cases = ane_h16_task_cases, +}; + +kunit_test_suite(ane_h16_task_suite); +MODULE_DESCRIPTION("KUnit tests for H16 task and relocation validation"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 7f1e9afbb7579..a50eeef08a90f 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -92,6 +92,7 @@ #include #include "ane_fw_platform.h" +#include "ane_fw_protocol.h" #include "ane_stats.h" #include "ane_t6021.h" #include "ane_t6021_boot.h" @@ -215,6 +216,12 @@ struct ane_rtclient { /* ChMan descriptor table validated against the 'IPC ' surface. */ bool chman_ok; + struct ane_fw_channel channels[ANE_FW_CHANNELS]; + u8 channel_snapshot[ANE_FW_TABLE_SIZE]; + u64 channel_table_offset; + u64 call_cookie; + u32 call_program; + u32 call_process; struct ane_legacy_buffer *legacy_buffers; u32 legacy_allocated; size_t legacy_bytes; @@ -402,7 +409,7 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, struct ane_legacy_buffer *command, size_t length, u16 opcode, unsigned int channel, - unsigned int timeout_ms); + unsigned long deadline); /* ---- boot helpers (shared with boot.c / fwload.c) ---- */ @@ -444,6 +451,31 @@ static void ane_rtclient_validate_chman(struct ane_rtclient *ane) "chman: no host IPC surface — table not validated\n"); return; } + if (ane->soc->platform) { + int ret; + + if (!ane_fw_contains(a->boot_ipc_iova, a->boot_ipc_size, + a->boot_scratch_result, ANE_FW_TABLE_SIZE)) { + dev_err(ane->dev, "27.x channel table outside IPC\n"); + return; + } + ane->channel_table_offset = a->boot_scratch_result - a->boot_ipc_iova; + dma_rmb(); + memcpy(ane->channel_snapshot, + a->boot_ipc + ane->channel_table_offset, ANE_FW_TABLE_SIZE); + ret = ane_fw_channels_parse(ane->channel_snapshot, a->boot_ipc_iova, + a->boot_ipc_size, a->boot_scratch_result, + ane->channels); + dma_rmb(); + if (ret || memcmp(ane->channel_snapshot, + a->boot_ipc + ane->channel_table_offset, + ANE_FW_TABLE_SIZE)) { + dev_err(ane->dev, "27.x channel table rejected or changed\n"); + return; + } + ane->chman_ok = true; + return; + } if (a->boot_ipc_size < ANE_T6021_CHMAN_TOTAL) { dev_warn(ane->dev, "chman: IPC surface %#llx bytes < fw layout %#x — table not validated\n", @@ -467,10 +499,39 @@ static void ane_rtclient_validate_chman(struct ane_rtclient *ane) } ane->chman_ok = !bad; + if (!bad) + for (i = 0; i < ANE_FW_CHANNELS; i++) { + const struct ane_t6021_chman_static *c = &ane_t6021_chman_layout[i]; + + ane->channels[i] = (struct ane_fw_channel) { + .offset = c->off, .slots = c->size, + .type = c->type, .bit = c->bit, + }; + } dev_info(ane->dev, "chman: table %s (mismatch mask %#x)\n", bad ? "NOT VALIDATED" : "VALIDATED", bad); } +static bool ane_rtclient_channels_unchanged(struct ane_rtclient *ane) +{ + struct ane_t6021 *a = ane->fw; + + if (!a || !ane->chman_ok) + return false; + dma_rmb(); + if (ane->soc->platform) + return !memcmp(ane->channel_snapshot, + a->boot_ipc + ane->channel_table_offset, + ANE_FW_TABLE_SIZE); + return !ane_t6021_chman_check(a->boot_ipc, a->boot_ipc_iova); +} + +static void __iomem *ane_rtclient_doorbell(struct ane_rtclient *ane) +{ + return ane->engine + (ane->soc->platform ? + ane->soc->platform->doorbell : ANE_IPI_OFF); +} + /* Legacy ChMan exchange (post-DONE 13.5 transport). Resends to the same * 64-byte IO slot would wedge the ring; cursor advances after each * completed command (the fw treats bit0 as host-owned). @@ -479,43 +540,52 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, struct ane_legacy_buffer *command, size_t length, u16 opcode, unsigned int channel, - unsigned int timeout_ms) + unsigned long deadline) { struct ane_t6021 *a = ane->fw; u64 *io, *malloc_ring, header; - void __iomem *ipi = ane->engine + ANE_IPI_OFF; + void __iomem *ipi = ane_rtclient_doorbell(ane); unsigned int cursor = ane->legacy_malloc_cursor; - unsigned long deadline; - u32 *reply; int result = -ETIMEDOUT; - if (!ane->held || !ane->chman_ok || - ane_t6021_chman_check(a->boot_ipc, a->boot_ipc_iova)) + if (!ane->held || !ane_rtclient_channels_unchanged(ane)) return -EPROTO; if (length < 8 || length > command->size || channel >= ANE_T6021_CHMAN_COUNT) { result = -EINVAL; goto out; } - io = a->boot_ipc + ane_t6021_chman_layout[channel].off + + io = a->boot_ipc + ane->channels[channel].offset + (size_t)ane->legacy_cmd_cursor[channel] * 64; - malloc_ring = a->boot_ipc + ane_t6021_chman_layout[5].off; + malloc_ring = a->boot_ipc + ane->channels[5].offset; if (!(READ_ONCE(io[0]) & 1)) { result = -EBUSY; goto out; } - reply = command->cpu; - ((u16 *)reply)[2] = opcode; + if (time_after_eq(jiffies, deadline)) + goto out; + put_unaligned_le16(opcode, (u8 *)command->cpu + 4); + if (ane->soc->platform) + memset((u8 *)command->cpu + length, 0xa5, command->size - length); WRITE_ONCE(io[1], length); WRITE_ONCE(io[2], length); dma_wmb(); WRITE_ONCE(io[0], command->dma); dma_wmb(); - writel(BIT(ane_t6021_chman_layout[channel].bit), ipi); - deadline = jiffies + msecs_to_jiffies(timeout_ms); + writel(BIT(ane->channels[channel].bit), ipi); while (time_before(jiffies, deadline)) { - u32 pending = readl(ipi + 0x8000); + u32 pending; + + if (atomic_read(&ane_t6021_quarantined)) { + result = -EIO; + goto out; + } + if (!ane_rtclient_channels_unchanged(ane)) { + result = -EPROTO; + goto out; + } + pending = readl(ipi + 0x8000); if (pending) { writel(pending, ipi + 0xc000); @@ -545,7 +615,7 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, * is a malformed ring and we stop. */ if (alloc_hdr || tag > U32_MAX) { - result = -EOPNOTSUPP; + result = -EPROTO; goto out; } if (ane_rtclient_legacy_alloc(ane, size)) { @@ -558,22 +628,36 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, dma_wmb(); WRITE_ONCE(slot[0], buf->dma | 1); dma_wmb(); - writel(BIT(5), ipi); + writel(BIT(ane->channels[5].bit), ipi); cursor = (cursor + 1) % - ane_t6021_chman_layout[5].size; + ane->channels[5].slots; } } + /* Allocation service cannot extend the command deadline. */ + if (time_after_eq(jiffies, deadline)) + goto out; header = READ_ONCE(io[0]); if (header & 1) { dma_rmb(); - result = (header == (command->dma | 1) && - READ_ONCE(io[1]) == length && - READ_ONCE(io[2]) == 0 && - ((u16 *)reply)[2] == opcode && - ((u16 *)reply)[3] == 0) ? 0 : -EPROTO; - ane->legacy_cmd_cursor[channel] = - (ane->legacy_cmd_cursor[channel] + 1) % - ane_t6021_chman_layout[channel].size; + result = ane_fw_reply_status(header, READ_ONCE(io[1]), + READ_ONCE(io[2]), command->dma, + length, opcode, command->cpu, + !!ane->soc->platform); + /* Corruption does not retire the unknown slot. A refusal + * retires ownership but remains a nonzero result. + */ + if (!ane_rtclient_channels_unchanged(ane) || + (ane->soc->platform && + memchr_inv((u8 *)command->cpu + length, 0xa5, + command->size - length))) + result = -EPROTO; + if (result == -EREMOTEIO) + dev_info(ane->dev, "firmware refused op=%#x ack=%#x; no retry\n", + opcode, get_unaligned_le16((u8 *)command->cpu + 6)); + if (!result || result == -EREMOTEIO) + ane->legacy_cmd_cursor[channel] = + (ane->legacy_cmd_cursor[channel] + 1) % + ane->channels[channel].slots; goto out; } /* Fast poll: check the rings every 50 us instead of @@ -770,11 +854,11 @@ static const void *ane_rtclient_fw_cpu(struct ane_rtclient *ane, u64 iova, for (i = 0; i < ane->legacy_allocated; i++) { const struct ane_legacy_buffer *b = &ane->legacy_buffers[i]; - if (iova >= b->dma && iova - b->dma + len <= b->size) + if (ane_fw_contains(b->dma, b->size, iova, len)) return b->cpu + (iova - b->dma); } - if (iova >= ane->fw->boot_ipc_iova && - iova - ane->fw->boot_ipc_iova + len <= ane->fw->boot_ipc_size) + if (ane_fw_contains(ane->fw->boot_ipc_iova, ane->fw->boot_ipc_size, + iova, len)) return ane->fw->boot_ipc + (iova - ane->fw->boot_ipc_iova); return NULL; } @@ -789,23 +873,38 @@ static const void *ane_rtclient_fw_cpu(struct ane_rtclient *ane, u64 iova, static bool ane_rtclient_drain_t2h(struct ane_rtclient *ane, unsigned int channel) { - const struct ane_t6021_chman_static *c = &ane_t6021_chman_layout[channel]; + const struct ane_fw_channel *c = &ane->channels[channel]; unsigned int n = 0, slot_i = ane->legacy_cmd_cursor[channel]; - void __iomem *ipi = ane->engine + ANE_IPI_OFF; + void __iomem *ipi = ane_rtclient_doorbell(ane); bool finished = false; - if (!ane->fw || !ane->fw->boot_ipc) + if (!ane_rtclient_channels_unchanged(ane)) { + atomic_set(&ane_t6021_quarantined, 1); + return false; + } + if (c->type != 1) return false; - while (n < c->size) { - u64 *slot = ane->fw->boot_ipc + c->off + (size_t)slot_i * 64; + while (n < c->slots) { + u64 *slot = ane->fw->boot_ipc + c->offset + (size_t)slot_i * 64; u64 hdr = READ_ONCE(slot[0]), len = READ_ONCE(slot[1]); if (hdr & 1) break; dma_rmb(); - dev_dbg(ane->dev, "T2H ch=%s slot=%u hdr=%016llx len=%#llx\n", - c->name, slot_i, hdr, len); - if (channel == 6 && len >= ANE_T2H_CALL_STATE_OFF + 4) { + dev_dbg(ane->dev, "T2H ch=%u slot=%u len=%#llx\n", + channel, slot_i, len); + if (channel == 6 && ane->soc->platform) { + const void *ev = ane_rtclient_fw_cpu(ane, hdr, len); + int status = ane_fw_finished_event(ev, len, ane->call_cookie, + ane->call_program, ane->call_process); + + if (status < 0) { + atomic_set(&ane_t6021_quarantined, 1); + return false; + } + if (status == 1) + finished = true; + } else if (channel == 6 && len >= ANE_T2H_CALL_STATE_OFF + 4) { const u8 *ev = ane_rtclient_fw_cpu(ane, hdr, ANE_T2H_CALL_STATE_OFF + 4); @@ -825,7 +924,7 @@ static bool ane_rtclient_drain_t2h(struct ane_rtclient *ane, WRITE_ONCE(slot[0], hdr | 1); dma_wmb(); writel(BIT(c->bit), ipi); - slot_i = (slot_i + 1) % c->size; + slot_i = (slot_i + 1) % c->slots; ane->legacy_cmd_cursor[channel] = slot_i; } return finished; @@ -884,17 +983,20 @@ static int ane_rtclient_call_wait_traced(struct ane_rtclient *ane, * Return zero on the event, or -ETIMEDOUT. */ static int ane_rtclient_call_wait(struct ane_rtclient *ane, - unsigned int timeout_ms) + unsigned long deadline) { - unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); unsigned int poll_us = READ_ONCE(call_poll_us); ktime_t tight_end = ktime_add_ns(ktime_get(), (u64)poll_us * NSEC_PER_USEC); if (ane_t6021_tracing) return ane_rtclient_call_wait_traced(ane, deadline); + if (ane->soc->platform && time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; do { + if (atomic_read(&ane_t6021_quarantined)) + return -EIO; if (ane_rtclient_drain_t2h(ane, 6)) return 0; if (poll_us && ktime_before(ktime_get(), tight_end)) @@ -902,6 +1004,10 @@ static int ane_rtclient_call_wait(struct ane_rtclient *ane, else usleep_range(50, 100); } while (time_before(jiffies, deadline)); + if (atomic_read(&ane_t6021_quarantined)) + return -EIO; + if (ane->soc->platform) + return -ETIMEDOUT; return ane_rtclient_drain_t2h(ane, 6) ? 0 : -ETIMEDOUT; } @@ -911,6 +1017,7 @@ static int ane_rtclient_command(struct ane_rtclient *ane, unsigned int channel, unsigned int timeout_ms) { + unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); int ret; u64 stats_ticket = 0; u64 stats_submit_ns = 0; @@ -946,7 +1053,8 @@ static int ane_rtclient_command(struct ane_rtclient *ane, stats_submit_ns, 1); } ret = ane_rtclient_legacy_exchange(ane, command, length, opcode, - channel, timeout_ms); + channel, ane->soc->platform ? deadline : + jiffies + msecs_to_jiffies(timeout_ms)); if (ret) { ane_t6021_tracing = false; if (stats_call) @@ -961,9 +1069,12 @@ static int ane_rtclient_command(struct ane_rtclient *ane, if (opcode == CSNE_CMD_PROCEDURE_CALL) { if (ane_t6021_tracing) ane_t6021_trace_add(ANE_TR_ACK, 0); - ret = ane_rtclient_call_wait(ane, timeout_ms); + ret = ane_rtclient_call_wait(ane, ane->soc->platform ? deadline : + jiffies + msecs_to_jiffies(timeout_ms)); ane_t6021_tracing = false; - if (!ret && call_settle_us) + if (!ret) + dma_rmb(); + if (!ret && !ane->soc->platform && call_settle_us) usleep_range(call_settle_us, call_settle_us + 100); if (ret) { if (stats_call) @@ -988,6 +1099,8 @@ static int ane_rtclient_command(struct ane_rtclient *ane, */ ane_rtclient_drain_t2h(ane, 4); ane_rtclient_drain_t2h(ane, 6); + if (atomic_read(&ane_t6021_quarantined)) + return -EIO; /* BOs are dma_alloc_coherent memory mapped write-combined for the * CPU, so no cache maintenance is needed on either side. */ @@ -1057,6 +1170,9 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, int ret, i, j; BUILD_BUG_ON(DRM_ANE_MAX_SECTIONS != ANE_T6021_LOAD_SEC_COUNT); + /* H16 requires the original HWX, not caller-asserted section identity. */ + if (ane->soc->platform) + return -EOPNOTSUPP; if (user->pad || user->section_count < 1 || user->section_count > ANE_T6021_LOAD_SEC_COUNT || user->generic_count > DRM_ANE_MAX_BINDS || !fd) @@ -1372,7 +1488,17 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, *(u32 *)(cmd + 0x0c) = cpu_to_le32(user->proc_id); *(u64 *)(cmd + 0x10) = cpu_to_le64(0); *(u32 *)(cmd + 0x18) = cpu_to_le32(user->priority); - *(u64 *)(cmd + 0x20) = cpu_to_le64(ANE_CALL_COOKIE); + if (ane->soc->platform) { + if (ane->call_cookie == U64_MAX) { + ret = -EOVERFLOW; + goto unlock; + } + ane->call_cookie++; + ane->call_program = user->prog_id; + ane->call_process = user->proc_id; + } + *(u64 *)(cmd + 0x20) = cpu_to_le64(ane->soc->platform ? + ane->call_cookie : ANE_CALL_COOKIE); *(u32 *)(cmd + 0x28) = cpu_to_le32(user->count); for (i = 0; i < user->count; i++) { struct ane_t6021_bo *bo = NULL, *b; @@ -1450,6 +1576,8 @@ static int ane_t6021_bo_init_ioctl(struct drm_device *drm, void *data, struct ane_t6021_bo *bo; struct ane_rtclient *ane; + if (atomic_read(&ane_t6021_quarantined)) + return -EIO; if (args->pad || args->size == 0 || args->size > ANE_T6021_BO_MAX || !fd) return -EINVAL; @@ -2334,14 +2462,13 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, * reboot. */ struct ane_legacy_buffer *command = ane->cmd_buf; + unsigned long query_deadline = jiffies + msecs_to_jiffies(3000); int qret; if (command) { memset(command->cpu, 0, SZ_16K); - qret = ane_rtclient_legacy_exchange(ane, - command, - 16, 0x03, - 1, 3000); + qret = ane_rtclient_legacy_exchange(ane, command, 16, 0x03, 1, + query_deadline); dev_dbg(dev, "LEGACY CONFIG_GET words %08x %08x result=%d (DMA remains held)\n", READ_ONCE(((u32 *)command->cpu)[1]), diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 90ea8b9c4dd51..1f2c4b5b33fa8 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -20,6 +20,7 @@ TARGETS += dax TARGETS += devices/error_logs TARGETS += devices/probe TARGETS += dmabuf-heaps +TARGETS += drivers/ane TARGETS += drivers/apple TARGETS += drivers/dma-buf TARGETS += drivers/gpu/apple-afkv2 diff --git a/tools/testing/selftests/drivers/ane/Makefile b/tools/testing/selftests/drivers/ane/Makefile new file mode 100644 index 0000000000000..5034bf537707f --- /dev/null +++ b/tools/testing/selftests/drivers/ane/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 +TEST_PROGS := h16-fixture.py +include ../../lib.mk diff --git a/tools/testing/selftests/drivers/ane/h16-fixture.py b/tools/testing/selftests/drivers/ane/h16-fixture.py new file mode 100755 index 0000000000000..1e74809b07732 --- /dev/null +++ b/tools/testing/selftests/drivers/ane/h16-fixture.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Prepare and verify SPEC-r2 H16G add-128 owner inputs without hardware.""" + +import argparse +import hashlib +import pathlib +import struct + +HWX_SIZE = 49152 +HWX_SHA256 = "231f54c28ea6b14a2867fac78eac1a2cb22aa3b0eeaa28113ed03c520b83a1d3" +EXPECTED_HASHES = { + "a.bin": "3a944e8c891c9e4b00d1cd12bbec5f17cde432fafc1fce0df5809b3deb513e03", + "b.bin": "779fff95d69316770cfff014c8bab4249c2f797c673a4518af80ab87b8d86a05", + "expected.bin": "d113d9ae6734b7a7e233458229f64f382da99a3f93868eef3abb06a29f7f4ce8", +} +FIRST_VALUES = ( + -2.25, -1.875, -1.5, -1.125, -0.75, -0.375, 0, 0.375, + 0.75, 1.125, 1.5, -0.875, -0.5, -0.125, 0.25, 0.625, +) + + +def buffers(): + a = [(i % 17 - 8) * 0.125 for i in range(16384)] + b = [(i % 11 - 5) * 0.25 for i in range(16384)] + expected = [x + y for x, y in zip(a, b)] + if tuple(expected[:16]) != FIRST_VALUES: + raise ValueError("unexpected CPU oracle") + result = { + "a.bin": b"".join(struct.pack(" Date: Sat, 31 Oct 2026 15:07:03 +1100 Subject: [PATCH 1456/1579] iio: aop: Decode the full lid angle report and expose its scale The LAS service already uses the same EPIC fake-HID receive path as ALS, but the decoder treats byte 1 of every report as an angle. Report 1 has a 9-bit little-endian angle; for example, 256 degrees becomes zero with the old decoder. Other report IDs can also overwrite the cached angle. Accept only complete report-1 samples in the 0..360 range and ignore the padding bits. Keep the raw count in degrees and expose pi/180 through in_angl_scale so IIO consumers can convert it to radians. A fixed scale does not need to wait for the first reading. Processed ALS channels keep their existing interface through the default no-scale implementation. Add host tests of the production decoder and IIO callback, document the existing J616s DT/config wiring and a hardware test plan, and register the new documentation and test under the existing AOP maintainer entry. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/iio/aop-sensors.rst | 65 ++++++++ Documentation/iio/index.rst | 1 + MAINTAINERS | 2 + drivers/iio/common/aop_sensors/aop_las.rs | 28 +++- rust/bindings/bindings_helper.h | 1 + rust/kernel/iio/common/aop_sensors.rs | 18 +++ .../testing/selftests/drivers/apple/Makefile | 3 +- .../selftests/drivers/apple/aop-las.py | 143 ++++++++++++++++++ 8 files changed, 256 insertions(+), 5 deletions(-) create mode 100644 Documentation/iio/aop-sensors.rst create mode 100755 tools/testing/selftests/drivers/apple/aop-las.py diff --git a/Documentation/iio/aop-sensors.rst b/Documentation/iio/aop-sensors.rst new file mode 100644 index 0000000000000..2292ad208350a --- /dev/null +++ b/Documentation/iio/aop-sensors.rst @@ -0,0 +1,65 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +================= +Apple AOP sensors +================= + +The AOP firmware announces sensor services over EPIC. The AOP core matches +each service to a child node in the device tree and forwards its fake-HID +reports to the corresponding IIO driver. Ambient light (``als``) and lid +angle (``las``) use this same transport; no HID bus device is created. + +Lid angle +========= + +Enable ``CONFIG_APPLE_AOP`` and ``CONFIG_IIO_AOP_SENSOR_LAS``. The AOP must +have a ``las`` child with a SoC-specific compatible followed by +``apple,aop-las``. The T6040 device tree already supplies this child for +J616s, and ``asahi.config`` enables the driver. The bootloader must hand +over the AOP firmware memory regions before the AOP can start. + +The device is named ``aop-sensors-las``. Its ``in_angl_raw`` attribute is +the latest valid report-1 angle, in whole degree counts from 0 to 360. +Multiply it by ``in_angl_scale`` (0.017453293) to obtain radians as +specified by the IIO angle ABI. The scale is fixed and can be read before +the first sample; a raw read before the first valid sample returns +``ENODATA``. Other report IDs, truncated reports and out-of-range angles +leave the last valid reading unchanged. + +Report 1 carries a 9-bit little-endian value following the report ID. +The remaining bits in the last byte are padding. Reads return the cached +unsolicited report, rather than issuing a synchronous HID feature request. + +Verification +============ + +Host regression tests require Python 3 and a Rust compiler:: + + tools/testing/selftests/drivers/apple/aop-las.py + +The tests compile the production decoder and shared IIO read callback +against host mocks. They cover all 16-bit payloads (including padding), +report IDs, short reports, invalid readings, the scale before a sample, +last-valid-sample behaviour and processed channels without a scale. +They do not test EPIC transport or firmware behaviour. + +On J616s, or another supported laptop: + +1. Build with ``asahi.config`` and the board's DTB. Check that the AOP, + LAS and ALS options survived configuration, and that the bootloader + supplies the AOP memory regions. +2. Boot and locate the IIO device whose ``name`` is ``aop-sensors-las``. + Check ``in_angl_scale`` is 0.017453293. Read ``in_angl_raw`` repeatedly + while moving the lid through several safe positions. Expect whole + degree counts that follow the hinge position. Do not force the hinge + beyond its mechanical limit to test values above 255; the host tests + cover that encoding. +3. Confirm the value updates after boot and after resume, and that ALS + still reports illuminance. Check the kernel log for AOP timeouts, + DART faults or warnings. Repeat on an older supported AOP platform + to check the shared receive path. +4. If raw reads keep returning ``ENODATA``, check that firmware announces + ``las`` and sends EPIC report subtype 0xc4 on that service's channel. + A readable feature report alone does not prove unsolicited reports + arrive. Establish that transport behaviour before adding any polling + or firmware setup request. diff --git a/Documentation/iio/index.rst b/Documentation/iio/index.rst index b02b879b053a1..f7ccdaac25d7a 100644 --- a/Documentation/iio/index.rst +++ b/Documentation/iio/index.rst @@ -38,6 +38,7 @@ Industrial I/O Kernel Drivers adxl313 adxl380 adxl345 + aop-sensors bno055 ep93xx_adc opt4060 diff --git a/MAINTAINERS b/MAINTAINERS index a903e83bd0ecb..2250beb8efb91 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2111,6 +2111,7 @@ F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml F: Documentation/gpu/apple-dcp-backlight.rst F: Documentation/hid/apple-j700-multitouch.rst +F: Documentation/iio/aop-sensors.rst F: Documentation/networking/device_drivers/wifi/mt7932-j700-antenna.rst F: Documentation/networking/device_drivers/wifi/mt7932-neo.rst F: arch/arm64/boot/dts/apple/t8140* @@ -2136,6 +2137,7 @@ F: tools/testing/apple/ F: tools/testing/kunit/configs/apple_firmware.config F: tools/testing/kunit/configs/apple_t8140_bootstrap.config F: tools/testing/kunit/configs/mt7932.config +F: tools/testing/selftests/drivers/apple/aop-las.py F: tools/testing/selftests/drivers/gpu/apple-dcp/ APPLETALK NETWORK LAYER diff --git a/drivers/iio/common/aop_sensors/aop_las.rs b/drivers/iio/common/aop_sensors/aop_las.rs index 7e0f5ccc01b19..598c1b607a5d7 100644 --- a/drivers/iio/common/aop_sensors/aop_las.rs +++ b/drivers/iio/common/aop_sensors/aop_las.rs @@ -17,7 +17,21 @@ struct MsgProc; impl MessageProcessor for MsgProc { fn process(&self, message: &[u8]) -> Option { - message.get(1).map(|angle| u64::from(*angle) * MICRO) + // Report 1 contains a little-endian, 9-bit angle in degrees. + // Other report IDs carry sensor properties, not angle samples. + if *message.first()? != 1 { + return None; + } + let angle = u16::from_le_bytes([*message.get(1)?, *message.get(2)?]) & 0x1ff; + if angle > 360 { + return None; + } + Some(u64::from(angle) * MICRO) + } + + fn scale(&self) -> Option<(i32, i32)> { + // IIO_ANGL uses radians: one raw count is pi / 180 radians. + Some((0, 17_453_293)) } } @@ -59,9 +73,15 @@ impl platform::Driver for IIOAopLasDriver { let ty = bindings::BINDINGS_IIO_ANGL; let data = AopSensorData::new(dev.into(), ty, MsgProc)?; let listener = FakehidRegistration::new(adata, service, data.clone())?; - let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_RAW; - let iio = - IIORegistration::::new(data, c"aop-sensors-las", ty, info_mask, kernel::module::this_module::())?; + let info_mask = (1 << bindings::BINDINGS_IIO_CHAN_INFO_RAW) + | (1 << bindings::BINDINGS_IIO_CHAN_INFO_SCALE); + let iio = IIORegistration::::new( + data, + c"aop-sensors-las", + ty, + info_mask, + kernel::module::this_module::(), + )?; Ok(IIOAopLasDriver { listener, _iio: iio, diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index ff4f810297ba4..9e384a40cd494 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -163,6 +163,7 @@ const snd_ctl_elem_iface_t BINDINGS_SNDRV_CTL_ELEM_IFACE_MIXER = SNDRV_CTL_ELEM_ const u32 BINDINGS_IIO_CHAN_INFO_RAW = IIO_CHAN_INFO_RAW; const u32 BINDINGS_IIO_CHAN_INFO_PROCESSED = IIO_CHAN_INFO_PROCESSED; +const u32 BINDINGS_IIO_CHAN_INFO_SCALE = IIO_CHAN_INFO_SCALE; const u32 BINDINGS_IIO_ANGL = IIO_ANGL; const u32 BINDINGS_IIO_LIGHT = IIO_LIGHT; diff --git a/rust/kernel/iio/common/aop_sensors.rs b/rust/kernel/iio/common/aop_sensors.rs index 7c80cef56b92f..94da1c11b1fa6 100644 --- a/rust/kernel/iio/common/aop_sensors.rs +++ b/rust/kernel/iio/common/aop_sensors.rs @@ -41,6 +41,12 @@ pub trait MessageProcessor: Send + Sync { /// Returns the reading carried by `message` in micro-units of the channel (millionths of a /// lux, of a degree, ...), or `None` when the message carries no reading. fn process(&self, message: &[u8]) -> Option; + + /// Returns the fixed scale of a raw channel as an integer and a fractional part in + /// billionths, or `None` for a channel without a scale attribute. + fn scale(&self) -> Option<(i32, i32)> { + None + } } /// The state shared between a sensor's report listener and its IIO device: the latest reading. @@ -99,6 +105,18 @@ unsafe extern "C" fn aop_read_raw( if data.ty != ty { return EINVAL.to_errno(); } + // A fixed scale is available even before the first sensor report. + if mask == bindings::BINDINGS_IIO_CHAN_INFO_SCALE as isize { + let Some((int, frac)) = data.msg_proc.scale() else { + return EINVAL.to_errno(); + }; + // SAFETY: `val` and `val2` point to the caller's result variables. + unsafe { + *val = int; + *val2 = frac; + } + return bindings::IIO_VAL_INT_PLUS_NANO as i32; + } // A sensor that has not reported yet, for instance because it lacks its // calibration, has no reading rather than a reading of zero. if !data.valid.load(Acquire) { diff --git a/tools/testing/selftests/drivers/apple/Makefile b/tools/testing/selftests/drivers/apple/Makefile index 932df4d386328..0408fc701dfa0 100644 --- a/tools/testing/selftests/drivers/apple/Makefile +++ b/tools/testing/selftests/drivers/apple/Makefile @@ -5,9 +5,10 @@ # dwc3-initial-role.py checks initial host-role admission before core reset. # sn201202x-transport.py checks the production SPMI selection and payload path. # bcm4377 tests exercise DMA ownership, context addresses and firmware handshakes. +# aop-las.py checks the production angle decoder and IIO callback on the host. TEST_PROGS := macsmc-j713.py atc-t8132-usb3.py pmgr-inherited.py dwc3-initial-role.py \ sn201202x-transport.py bcm4377-dma-window.py bcm4377-radio-sids.py \ - bcm4377-state-wait.py bcm4377-rti-address.py + bcm4377-state-wait.py bcm4377-rti-address.py aop-las.py TEST_FILES := j813-peripherals.sh include ../../lib.mk diff --git a/tools/testing/selftests/drivers/apple/aop-las.py b/tools/testing/selftests/drivers/apple/aop-las.py new file mode 100755 index 0000000000000..ce4da0dcec63d --- /dev/null +++ b/tools/testing/selftests/drivers/apple/aop-las.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run the production LAS decoder and IIO read callback with host Rust mocks.""" +import os +from pathlib import Path +import shlex +import subprocess +import tempfile + + +def block(source, marker): + start = source.index(marker) + brace = source.index("{", start) + depth = 1 + end = brace + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +def main(): + root = Path(__file__).resolve().parents[5] + common = (root / "rust/kernel/iio/common/aop_sensors.rs").read_text() + las = (root / "drivers/iio/common/aop_sensors/aop_las.rs").read_text() + parts = [r''' +#![allow(non_camel_case_types)] +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering::{Acquire, Relaxed, Release}}; +const MICRO: u64 = 1_000_000; +const MAX_MICRO: u64 = i32::MAX as u64 * MICRO + MICRO - 1; +#[derive(Debug)] +struct Error(i32); +impl Error { fn to_errno(&self) -> i32 { self.0 } } +const EINVAL: Error = Error(-22); +const ENODATA: Error = Error(-61); +type Result = std::result::Result; +mod bindings { + pub const BINDINGS_IIO_CHAN_INFO_RAW: u32 = 0; + pub const BINDINGS_IIO_CHAN_INFO_PROCESSED: u32 = 1; + pub const BINDINGS_IIO_CHAN_INFO_SCALE: u32 = 2; + pub const IIO_VAL_INT: u32 = 1; + pub const IIO_VAL_INT_PLUS_MICRO: u32 = 2; + pub const IIO_VAL_INT_PLUS_NANO: u32 = 3; + pub struct iio_dev { pub priv_: *const std::ffi::c_void } + pub struct iio_chan_spec { pub type_: u32 } +} +struct Arc(std::marker::PhantomData); +impl Arc { + unsafe fn borrow<'a>(p: *const std::ffi::c_void) -> &'a T { + unsafe { &*p.cast::() } + } +} +trait FakehidListener { + fn process_fakehid_report(&self, data: &[u8]) -> Result<()>; +} +struct AopSensorData { + ty: u32, value: AtomicU64, valid: AtomicBool, msg_proc: T, +} +struct MsgProc; +''', block(common, "pub trait MessageProcessor"), + block(las, "impl MessageProcessor for MsgProc"), + block(common, "impl FakehidListener"), + block(common, 'unsafe extern "C" fn aop_read_raw'), r''' +fn sensor(msg_proc: T) -> AopSensorData { + AopSensorData { + ty: 17, value: AtomicU64::new(0), valid: AtomicBool::new(false), msg_proc, + } +} +fn read(data: &AopSensorData, mask: u32) -> (i32, i32, i32) { + let mut dev = bindings::iio_dev { priv_: (data as *const AopSensorData).cast() }; + let chan = bindings::iio_chan_spec { type_: data.ty }; + let (mut val, mut val2) = (-1, -1); + let ret = unsafe { aop_read_raw::(&mut dev, &chan, &mut val, &mut val2, mask as isize) }; + (ret, val, val2) +} +#[test] +fn full_angle_and_padding() { + // Synthetic reports: cover bit 8, both limits, invalid values and every padding pattern. + for wire in 0..=u16::MAX { + let bytes = wire.to_le_bytes(); + let angle = wire & 0x1ff; + let expected = if angle <= 360 { Some(u64::from(angle) * MICRO) } else { None }; + assert_eq!(MsgProc.process(&[1, bytes[0], bytes[1]]), expected); + } + assert_eq!(MsgProc.process(&[1, 99, 0]), Some(99 * MICRO)); + assert_eq!(MsgProc.process(&[1, 0, 1]), Some(256 * MICRO)); + assert_eq!(MsgProc.process(&[1, 104, 1]), Some(360 * MICRO)); +} +#[test] +fn report_identity_and_truncation() { + for id in 0..=u8::MAX { + if id != 1 { assert_eq!(MsgProc.process(&[id, 99, 0]), None); } + } + for truncated in [&[][..], &[1][..], &[1, 99][..]] { + assert_eq!(MsgProc.process(truncated), None); + } +} +#[test] +fn raw_scale_and_last_valid_sample() { + use bindings::*; + let data = sensor(MsgProc); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_RAW).0, -61); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_SCALE), + (IIO_VAL_INT_PLUS_NANO as i32, 0, 17_453_293)); + data.process_fakehid_report(&[2, 39]).unwrap(); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_RAW).0, -61); + data.process_fakehid_report(&[1, 99, 0]).unwrap(); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_RAW), (1, 99, 0)); + for report in [&[3, 0, 0, 0, 0, 1][..], &[1, 105, 1][..], &[1, 42][..]] { + data.process_fakehid_report(report).unwrap(); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_RAW), (1, 99, 0)); + } + let (_, int, nano) = read(&data, BINDINGS_IIO_CHAN_INFO_SCALE); + let radians = 99.0 * (int as f64 + nano as f64 / 1e9); + assert!((radians - 99.0_f64.to_radians()).abs() < 1e-7); +} +struct Processed; +impl MessageProcessor for Processed { + fn process(&self, _: &[u8]) -> Option { Some(12_345_678) } +} +#[test] +fn processed_channel_unchanged() { + use bindings::*; + let data = sensor(Processed); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_PROCESSED).0, -61); + data.process_fakehid_report(&[]).unwrap(); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_PROCESSED), (2, 12, 345_678)); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_SCALE).0, -22); + assert_eq!(read(&data, 999).0, -22); +} +'''] + with tempfile.TemporaryDirectory(prefix="aop-las-") as tmp: + source = Path(tmp) / "test.rs" + binary = Path(tmp) / "test" + source.write_text("\n".join(parts)) + subprocess.run(shlex.split(os.environ.get("RUSTC", "rustc")) + [ + "--edition=2021", "--test", "-Dwarnings", str(source), "-o", str(binary), + ], check=True) + subprocess.run([str(binary)], check=True) + + +if __name__ == "__main__": + main() From ca904007b1c4c6e566aae64da3fcc996a5210c8e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:18:35 +0000 Subject: [PATCH 1457/1579] iio: aop: expose CCT from the documented Redbird report Redbird already provides firmware-computed illuminance. Add its reported 122-byte payload's CCT float at 0x45 as a processed IIO_CCT channel in Kelvin. Retain the existing lux offsets and 200 ms reporting interval. Reject other payload lengths for CCT rather than applying host event structure offsets to the fake-HID transport. Extend the common registration with a default single-channel API and per-channel validity and saturation, so CT720, CT817 and the lid-angle client retain their interfaces. Use synthetic reports to exercise the production decoder, cache and read callback, including invalid channels, short reports, independent validity and signed IIO saturation. The wire layout is a public protocol fact, not copied implementation: Link: https://github.com/triutama133/ubuntu_asahi_patch/blob/main/kernel-patches/README.md J616s report-layout and colour-reference validation remain necessary; this is not a claim of measured colour-temperature parity. Also document the existing T6040 memory policy and static DRAM interface, and provide a bounded libc memcpy workload for Linux-side comparison. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../ABI/testing/sysfs-driver-aop-als | 13 ++ Documentation/iio/apple-aop-als.rst | 40 +++++ Documentation/iio/index.rst | 1 + drivers/iio/common/aop_sensors/aop_als.rs | 86 ++++++++-- rust/bindings/bindings_helper.h | 1 + rust/kernel/iio/common/aop_sensors.rs | 99 ++++++++---- tools/asahi/aop-als-channel-tests.py | 151 ++++++++++++++++++ tools/asahi/run-host-tests.py | 1 + .../selftests/drivers/apple/j616s-memory.rst | 51 ++++++ .../selftests/drivers/apple/memcpy-bw.c | 130 +++++++++++++++ 10 files changed, 535 insertions(+), 38 deletions(-) create mode 100644 Documentation/ABI/testing/sysfs-driver-aop-als create mode 100644 Documentation/iio/apple-aop-als.rst create mode 100644 tools/asahi/aop-als-channel-tests.py create mode 100644 tools/testing/selftests/drivers/apple/j616s-memory.rst create mode 100644 tools/testing/selftests/drivers/apple/memcpy-bw.c diff --git a/Documentation/ABI/testing/sysfs-driver-aop-als b/Documentation/ABI/testing/sysfs-driver-aop-als new file mode 100644 index 0000000000000..c5b6acd796242 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-aop-als @@ -0,0 +1,13 @@ +What: /sys/bus/iio/devices/iio:deviceX/in_cct_input +Date: October 2026 +Contact: linux-iio@vger.kernel.org +Description: + Firmware-computed correlated colour temperature in Kelvin on + aop-sensors-als devices with a Redbird (VD6286) sensor. + Fractional Kelvin are reported with micro resolution. There + is no raw photodiode-to-CCT conversion in the host driver. + Only the documented 122-byte fake-HID report is decoded. + Before a valid positive finite CCT arrives, reading returns + ENODATA. A report in another layout or without a valid CCT + leaves the last valid reading in place, independently of lux. + CT720 and CT817 devices do not expose this attribute. diff --git a/Documentation/iio/apple-aop-als.rst b/Documentation/iio/apple-aop-als.rst new file mode 100644 index 0000000000000..a19625a84be75 --- /dev/null +++ b/Documentation/iio/apple-aop-als.rst @@ -0,0 +1,40 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +Apple AOP ambient light reporting +================================ + +The ALS driver consumes the payload of EPIC fake-HID reports, after the AOP +transport has removed the EPIC headers. These offsets do not describe macOS +host-side event objects. + +Redbird (VD6286) illuminance is the little-endian firmware float at payload +byte 0x28. Linux converts it to microlux using integer arithmetic and exports +it through ``in_illuminance_input``. The driver requests a 200000 microsecond +report interval, or 5 Hz, and loads the device's factory calibration from +``apple/aop-als-cal.bin`` when installed. Calibration is device-specific and +must not be distributed with test fixtures. + +The `public Redbird report description +`_ +identifies a 122-byte report carrying a firmware-computed CCT float at byte +0x45. The driver exports that layout as ``in_cct_input``, in Kelvin, without +recalculating colour temperature from raw channels. Other lengths do not +supply a CCT reading. This layout must be verified on each firmware family +before its CCT readings are used for display policy. The existing CT720 and +CT817 illuminance layouts have no established CCT field in this driver. + +Read both channels by finding the IIO device by name rather than assuming a +device number:: + + for d in /sys/bus/iio/devices/iio:device*; do + [ "$(cat "$d/name" 2>/dev/null)" = aop-sensors-als ] || continue + cat "$d/in_illuminance_input" + [ ! -e "$d/in_cct_input" ] || cat "$d/in_cct_input" + done + +These attributes contain the last valid reading, not a sample counter. Polling +them five times per second cannot prove a 5 Hz arrival cadence. Measure that +at the report callback or transport with tracing, and verify the payload +length and field correspondence there. Keep raw captures private. A CCT value +remaining unchanged while lux changes can also mean an unsupported payload +layout or a report with invalid CCT; it does not prove steady illumination. diff --git a/Documentation/iio/index.rst b/Documentation/iio/index.rst index b02b879b053a1..42b097088f87b 100644 --- a/Documentation/iio/index.rst +++ b/Documentation/iio/index.rst @@ -38,6 +38,7 @@ Industrial I/O Kernel Drivers adxl313 adxl380 adxl345 + apple-aop-als bno055 ep93xx_adc opt4060 diff --git a/drivers/iio/common/aop_sensors/aop_als.rs b/drivers/iio/common/aop_sensors/aop_als.rs index bec8459aba2c4..5e6c5001ad2b8 100644 --- a/drivers/iio/common/aop_sensors/aop_als.rs +++ b/drivers/iio/common/aop_sensors/aop_als.rs @@ -12,6 +12,7 @@ use kernel::{ module_platform_driver, of, platform, prelude::*, soc::apple::aop::{EPICService, FakehidRegistration, AOP}, + sync::Arc, }; const EPIC_SUBTYPE_GET_AOP_PROPERTY: u16 = 0xa; @@ -39,7 +40,12 @@ const ALS_VERBOSITY: u32 = 6; /// The reporting interval in microseconds. const ALS_INTERVAL_US: u32 = 200000; const LUX_OFFSET_CT720: usize = 0x1d; +// These offsets address the fake-HID EPIC payload, not a host HID event. const LUX_OFFSET_VD6286: usize = 0x28; +// The documented 122-byte Redbird report carries firmware-computed Kelvin. +// Other report layouts must be decoded separately, even for the same sensor. +const VD6286_COLOR_REPORT_LEN: usize = 122; +const CCT_OFFSET_VD6286: usize = 0x45; /* * The CT817 sends a 32-byte report: two raw channel floats at +0x08 and * +0x0c and the lux float at +0x1c, one byte below the CT720's, so the two @@ -168,17 +174,62 @@ fn f32_to_micro(f: u32) -> Option { struct MsgProc(usize); +fn report_float(message: &[u8], offset: usize) -> Option { + let raw = message.get(offset..offset + 4)?; + f32_to_micro(u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]])) +} + impl MessageProcessor for MsgProc { fn process(&self, message: &[u8]) -> Option { - let raw = message.get(self.0..self.0 + 4)?; - f32_to_micro(u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]])) + report_float(message, self.0) + } + + fn process_channel(&self, message: &[u8], channel: usize) -> Option { + match channel { + 0 => self.process(message), + 1 if self.0 == LUX_OFFSET_VD6286 && message.len() == VD6286_COLOR_REPORT_LEN => { + // Zero and negative CCT do not describe a colour temperature. + report_float(message, CCT_OFFSET_VD6286).filter(|value| *value != 0) + } + _ => None, + } } } +enum AlsIio { + Light { + _registration: IIORegistration, + }, + Color { + _registration: IIORegistration, + }, +} + struct IIOAopAlsDriver { /// Declared first: the subscription ends before the IIO device goes. listener: FakehidRegistration, - _iio: IIORegistration, + _iio: AlsIio, +} + +fn register_channels( + pdev: &platform::Device, + aop: &Arc, + service: EPICService, + offset: usize, + types: [u32; N], +) -> Result<(FakehidRegistration, IIORegistration)> { + let data = AopSensorData::new_channels(pdev.as_ref().into(), types, MsgProc(offset))?; + let listener = FakehidRegistration::new(aop.clone(), service, data.clone())?; + enable_als(aop.as_ref(), pdev, &service, offset)?; + let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_PROCESSED; + let iio = IIORegistration::new_channels( + data, + c"aop-sensors-als", + types, + info_mask, + kernel::module::this_module::(), + )?; + Ok((listener, iio)) } kernel::of_device_table!( @@ -211,12 +262,29 @@ impl platform::Driver for IIOAopAlsDriver { let service = unsafe { *service }; let ty = bindings::BINDINGS_IIO_LIGHT; let offset = get_lux_offset(adata.as_ref(), pdev, &service)?; - let data = AopSensorData::new(dev.into(), ty, MsgProc(offset))?; - let listener = FakehidRegistration::new(adata.clone(), service, data.clone())?; - enable_als(adata.as_ref(), pdev, &service, offset)?; - let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_PROCESSED; - let iio = - IIORegistration::::new(data, c"aop-sensors-als", ty, info_mask, kernel::module::this_module::())?; + let (listener, iio) = if offset == LUX_OFFSET_VD6286 { + let (listener, registration) = register_channels( + pdev, + &adata, + service, + offset, + [ty, bindings::BINDINGS_IIO_CCT], + )?; + ( + listener, + AlsIio::Color { + _registration: registration, + }, + ) + } else { + let (listener, registration) = register_channels(pdev, &adata, service, offset, [ty])?; + ( + listener, + AlsIio::Light { + _registration: registration, + }, + ) + }; Ok(IIOAopAlsDriver { listener, _iio: iio, diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index ff4f810297ba4..0a35cbf80a180 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -165,6 +165,7 @@ const u32 BINDINGS_IIO_CHAN_INFO_RAW = IIO_CHAN_INFO_RAW; const u32 BINDINGS_IIO_CHAN_INFO_PROCESSED = IIO_CHAN_INFO_PROCESSED; const u32 BINDINGS_IIO_ANGL = IIO_ANGL; const u32 BINDINGS_IIO_LIGHT = IIO_LIGHT; +const u32 BINDINGS_IIO_CCT = IIO_CCT; const xa_mark_t RUST_CONST_HELPER_XA_PRESENT = XA_PRESENT; diff --git a/rust/kernel/iio/common/aop_sensors.rs b/rust/kernel/iio/common/aop_sensors.rs index 7c80cef56b92f..5a7c20a30757f 100644 --- a/rust/kernel/iio/common/aop_sensors.rs +++ b/rust/kernel/iio/common/aop_sensors.rs @@ -41,28 +41,48 @@ pub trait MessageProcessor: Send + Sync { /// Returns the reading carried by `message` in micro-units of the channel (millionths of a /// lux, of a degree, ...), or `None` when the message carries no reading. fn process(&self, message: &[u8]) -> Option; + + /// Decodes a channel of a multi-channel sensor. Single-channel processors need only + /// implement [`Self::process`]. A missing channel leaves its previous reading in place. + fn process_channel(&self, message: &[u8], channel: usize) -> Option { + if channel == 0 { + self.process(message) + } else { + None + } + } } /// The state shared between a sensor's report listener and its IIO device: the latest reading. -pub struct AopSensorData { +pub struct AopSensorData { dev: ARef, - ty: u32, + ty: [u32; N], /// The latest reading in micro-units, at most [`MAX_MICRO`]. - value: Atomic, + value: [Atomic; N], /// Whether a reading has arrived at all; until then reads return `ENODATA`. - valid: Atomic, + valid: [Atomic; N], msg_proc: T, } impl AopSensorData { /// Creates the state for one channel of type `ty` whose reports `msg_proc` decodes. pub fn new(dev: ARef, ty: u32, msg_proc: T) -> Result>> { + Self::new_channels(dev, [ty], msg_proc) + } +} + +impl AopSensorData { + /// Creates state for the channels in `ty`, in the order decoded by the processor. + pub fn new_channels(dev: ARef, ty: [u32; N], msg_proc: T) -> Result> { + if N == 0 { + return Err(EINVAL); + } Ok(Arc::new( AopSensorData { dev, ty, - value: Atomic::new(0), - valid: Atomic::new(false), + value: [const { Atomic::new(0) }; N], + valid: [const { Atomic::new(false) }; N], msg_proc, }, GFP_KERNEL, @@ -70,13 +90,15 @@ impl AopSensorData { } } -impl FakehidListener for AopSensorData { +impl FakehidListener for AopSensorData { fn process_fakehid_report(&self, data: &[u8]) -> Result<()> { // A report without a reading leaves the last reading in place. - if let Some(value) = self.msg_proc.process(data) { - self.value.store(value.min(MAX_MICRO), Relaxed); - // Publishes the reading above to readers that see `valid`. - self.valid.store(true, Release); + for channel in 0..N { + if let Some(value) = self.msg_proc.process_channel(data, channel) { + self.value[channel].store(value.min(MAX_MICRO), Relaxed); + // Publishes the reading above to readers that see `valid`. + self.valid[channel].store(true, Release); + } } Ok(()) } @@ -84,7 +106,7 @@ impl FakehidListener for AopSensorData { /// The `read_raw` callback: processed values are reported with their micro part, raw values as /// integers. -unsafe extern "C" fn aop_read_raw( +unsafe extern "C" fn aop_read_raw( dev: *mut bindings::iio_dev, chan: *const bindings::iio_chan_spec, val: *mut i32, @@ -93,18 +115,20 @@ unsafe extern "C" fn aop_read_raw( ) -> i32 { // SAFETY: The IIO core calls this with the device and channel that `IIORegistration::new()` // registered; `priv_` is the `Arc` it stored there. - let data = unsafe { Arc::>::borrow((*dev).priv_.cast()) }; + let data = unsafe { Arc::>::borrow((*dev).priv_.cast()) }; // SAFETY: `chan` is one of the registered channel specs. let ty = unsafe { (*chan).type_ }; - if data.ty != ty { + // SAFETY: the registration assigns each channel its index as its address. + let channel = unsafe { (*chan).address as usize }; + if data.ty.get(channel) != Some(&ty) { return EINVAL.to_errno(); } // A sensor that has not reported yet, for instance because it lacks its // calibration, has no reading rather than a reading of zero. - if !data.valid.load(Acquire) { + if !data.valid[channel].load(Acquire) { return ENODATA.to_errno(); } - let micro = data.value.load(Relaxed); + let micro = data.value[channel].load(Relaxed); // Both parts fit an `i32` because stored readings never exceed `MAX_MICRO`. let (int, frac) = ((micro / MICRO) as i32, (micro % MICRO) as i32); // SAFETY: `val` and `val2` point to the caller's result variables. @@ -121,18 +145,18 @@ unsafe extern "C" fn aop_read_raw( } } -struct IIOSpec { - spec: [bindings::iio_chan_spec; 1], +struct IIOSpec { + spec: [bindings::iio_chan_spec; N], vtable: bindings::iio_info, _p: PhantomPinned, } /// TODO: add documentation -pub struct IIORegistration { +pub struct IIORegistration { dev: *mut bindings::iio_dev, - spec: Pin>, + spec: Pin>>, registered: bool, - _p: PhantomData>, + _p: PhantomData>, } impl IIORegistration { @@ -144,10 +168,27 @@ impl IIORegistration { info_mask: usize, module: &ThisModule, ) -> Result { + Self::new_channels(data, name, [ty], info_mask, module) + } +} + +impl IIORegistration { + /// Registers the sensor's channels. The types must match those of `data`. + pub fn new_channels( + data: Arc>, + name: &'static CStr, + ty: [u32; N], + info_mask: usize, + module: &ThisModule, + ) -> Result { + if ty != data.ty { + return Err(EINVAL); + } let spec = KBox::pin( IIOSpec { - spec: [bindings::iio_chan_spec { - type_: ty, + spec: core::array::from_fn(|channel| bindings::iio_chan_spec { + type_: ty[channel], + address: channel as _, __bindgen_anon_1: bindings::iio_chan_spec__bindgen_ty_1 { scan_type: bindings::iio_scan_type { __bindgen_anon_1: bindings::iio_scan_type__bindgen_ty_1 { @@ -160,9 +201,9 @@ impl IIORegistration { }, info_mask_separate: info_mask, ..Default::default() - }], + }), vtable: bindings::iio_info { - read_raw: Some(aop_read_raw::), + read_raw: Some(aop_read_raw::), ..Default::default() }, _p: PhantomPinned, @@ -197,19 +238,19 @@ impl IIORegistration { } } -impl Drop for IIORegistration { +impl Drop for IIORegistration { fn drop(&mut self) { if self.dev != ptr::null_mut() { unsafe { if self.registered { bindings::iio_device_unregister(self.dev); } - Arc::>::from_foreign((*self.dev).priv_.cast()); + Arc::>::from_foreign((*self.dev).priv_.cast()); bindings::iio_device_free(self.dev); } } } } -unsafe impl Send for IIORegistration {} -unsafe impl Sync for IIORegistration {} +unsafe impl Send for IIORegistration {} +unsafe impl Sync for IIORegistration {} diff --git a/tools/asahi/aop-als-channel-tests.py b/tools/asahi/aop-als-channel-tests.py new file mode 100644 index 0000000000000..d88d8405ab5f1 --- /dev/null +++ b/tools/asahi/aop-als-channel-tests.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production report decoder, per-channel cache and IIO callback.""" +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] + + +def source(): + common = (ROOT / 'rust/kernel/iio/common/aop_sensors.rs').read_text() + als = (ROOT / 'drivers/iio/common/aop_sensors/aop_als.rs').read_text() + common = common[common.index('pub const MICRO:'):common.index('struct IIOSpec')] + constants = '\n'.join(line for line in als.splitlines() + if line.startswith('const LUX_OFFSET_') or + line.startswith('const CCT_OFFSET_') or + line.startswith('const VD6286_COLOR_REPORT_LEN')) + conversion = als[als.index('fn f32_to_micro('):als.index('enum AlsIio')] + return STUBS + common + constants + conversion + TESTS + + +STUBS = r''' +#![allow(dead_code, non_camel_case_types)] +use std::{ops::Deref, sync::atomic::Ordering::{Acquire, Relaxed, Release}}; +#[derive(Debug)] pub struct Error(i32); +impl Error { fn to_errno(&self)->i32 { self.0 } } +const EINVAL:Error=Error(-22); const ENODATA:Error=Error(-61); +type Result = std::result::Result; +const GFP_KERNEL:u32=0; +mod device { pub struct Device; } +type ARef = std::sync::Arc; +pub struct Arc(std::sync::Arc); +impl Arc { + fn new(value:T,_flags:u32)->Result { Ok(Self(std::sync::Arc::new(value))) } + unsafe fn borrow<'a>(ptr:*const T)->&'a T { unsafe { &*ptr } } +} +impl Deref for Arc { type Target=T; fn deref(&self)->&T { &self.0 } } +struct Atomic(std::sync::Mutex); +impl Atomic { + const fn new(value:T)->Self { Self(std::sync::Mutex::new(value)) } + fn store(&self,value:T,_order:std::sync::atomic::Ordering) { *self.0.lock().unwrap()=value; } + fn load(&self,_order:std::sync::atomic::Ordering)->T { *self.0.lock().unwrap() } +} +trait FakehidListener { fn process_fakehid_report(&self, data:&[u8])->Result<()>; } +mod bindings { + pub const BINDINGS_IIO_CHAN_INFO_PROCESSED:u32=2; + pub const BINDINGS_IIO_CHAN_INFO_RAW:u32=1; + pub const IIO_VAL_INT_PLUS_MICRO:u32=2; + pub const IIO_VAL_INT:u32=1; + pub struct iio_dev { pub priv_:*mut std::ffi::c_void } + pub struct iio_chan_spec { pub type_:u32, pub address:u64 } +} +''' + +TESTS = r''' +fn put(report:&mut [u8],offset:usize,value:f32) { + report[offset..offset+4].copy_from_slice(&value.to_le_bytes()); +} +fn read(data:&AopSensorData,channel:u64,ty:u32)->(i32,i32,i32) { + let mut dev=bindings::iio_dev{priv_:(data as *const AopSensorData).cast_mut().cast()}; + let chan=bindings::iio_chan_spec{type_:ty,address:channel}; + let(mut a,mut b)=(-1,-1); + let ret=unsafe { aop_read_raw::(&mut dev,&chan,&mut a,&mut b,2) }; + (ret,a,b) +} +#[test] fn redbird_firmware_channels_and_abi() { + let mut report=[0u8;122]; + // Distinct sentinels at the host event's offsets catch coordinate-system mistakes. + put(&mut report,0x14,900.0); put(&mut report,0x34,1200.0); + put(&mut report,0x28,321.25); put(&mut report,0x45,5000.5); + let data=AopSensorData::new_channels(std::sync::Arc::new(device::Device),[5,6],MsgProc(LUX_OFFSET_VD6286)).unwrap(); + assert_eq!(read(&data,0,5).0,-61); assert_eq!(read(&data,1,6).0,-61); + data.process_fakehid_report(&report).unwrap(); + assert_eq!(read(&data,0,5),(2,321,250000)); + assert_eq!(read(&data,1,6),(2,5000,500000)); + assert_eq!(read(&data,2,6).0,-22); assert_eq!(read(&data,1,5).0,-22); + // Invalid lux does not prevent a valid CCT from advancing. + put(&mut report,0x28,f32::NAN); put(&mut report,0x45,4100.75); + data.process_fakehid_report(&report).unwrap(); + assert_eq!(read(&data,0,5),(2,321,250000)); assert_eq!(read(&data,1,6),(2,4100,750000)); + // Invalid CCT does not prevent lux from advancing; preserve last valid CCT. + for bad in [0.0,-1.0,f32::NAN,f32::INFINITY,f32::NEG_INFINITY] { + put(&mut report,0x45,bad); put(&mut report,0x28,0.125); + data.process_fakehid_report(&report).unwrap(); + assert_eq!(read(&data,0,5),(2,0,125000)); assert_eq!(read(&data,1,6),(2,4100,750000)); + } + // Storage saturation protects both signed IIO output parts. + put(&mut report,0x45,f32::MAX); data.process_fakehid_report(&report).unwrap(); + assert_eq!(read(&data,1,6),(2,i32::MAX,999999)); +} +#[test] fn unknown_layout_does_not_fabricate_cct() { + let decoder=MsgProc(LUX_OFFSET_VD6286); + let mut report=vec![0u8;140]; put(&mut report,0x28,2.0); put(&mut report,0x45,6000.0); + for length in 0..140 { + let sample=&report[..length]; + assert_eq!(decoder.process_channel(sample,1),if length==122 {Some(6000000000)} else {None}); + assert_eq!(decoder.process_channel(sample,0),if length>=44 {Some(2000000)} else {None}); + } + assert_eq!(decoder.process_channel(&report,2),None); + let data=AopSensorData::new_channels(std::sync::Arc::new(device::Device),[5,6],decoder).unwrap(); + data.process_fakehid_report(&report[..121]).unwrap(); + assert_eq!(read(&data,0,5),(2,2,0)); assert_eq!(read(&data,1,6).0,-61); +} +#[test] fn single_channel_clients_and_other_sensors() { + struct Scalar; + impl MessageProcessor for Scalar { fn process(&self,_m:&[u8])->Option { Some(99000000) } } + let scalar=AopSensorData::new(std::sync::Arc::new(device::Device),7,Scalar).unwrap(); + scalar.process_fakehid_report(&[]).unwrap(); assert_eq!(read(&scalar,0,7),(2,99,0)); + assert_eq!(Scalar.process_channel(&[],1),None); + for offset in [LUX_OFFSET_CT720,LUX_OFFSET_CT817] { + let mut report=[0u8;122]; put(&mut report,offset,0.5); put(&mut report,0x45,6000.0); + let decoder=MsgProc(offset); + assert_eq!(decoder.process(&report),Some(500000)); + assert_eq!(decoder.process_channel(&report,1),None); + } + assert_eq!(f32_to_micro((-0.25f32).to_bits()),Some(0)); + assert_eq!(f32_to_micro(1u32),Some(0)); +} +''' + + +def main(): + code = source() + mutations = { + 'host-cct-offset': ('CCT_OFFSET_VD6286: usize = 0x45', 'CCT_OFFSET_VD6286: usize = 0x34'), + 'unknown-layout': ('message.len() == VD6286_COLOR_REPORT_LEN', 'message.len() >= VD6286_COLOR_REPORT_LEN - 1'), + 'cct-zero-valid': ('.filter(|value| *value != 0)', ''), + 'channel-alias': ('report_float(message, CCT_OFFSET_VD6286)', 'report_float(message, LUX_OFFSET_VD6286)'), + 'missing-saturation': ('value.min(MAX_MICRO)', 'value'), + } + with tempfile.TemporaryDirectory(prefix='aop-als-channels-') as directory: + directory = Path(directory) + for name, text in [('production', code)] + [ + (name, code.replace(old, new)) for name, (old, new) in mutations.items()]: + assert text != code or name == 'production', name + path = directory / 'channels.rs' + path.write_text(text) + binary = directory / 'channels' + subprocess.run(['rustc', '--edition=2021', '-Dwarnings', '--test', str(path), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)], text=True, capture_output=True) + if name == 'production': + print(result.stdout) + assert result.returncode == 0, result.stderr + else: + assert result.returncode == 101 and 'panicked at' in result.stdout + result.stderr, (name, result) + print('killed:', name) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 24f7a9e9134fe..fc8b52ba6ada0 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -50,6 +50,7 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'aop-als-channel-tests.py': [([], PASS)], 'mtp-pm-tests.py': [(out(), PASS), (out() + ['--old-transport', '4db3a5a22c43'], PASS)], 'ave-pm-tests.py': [(out(), PASS), diff --git a/tools/testing/selftests/drivers/apple/j616s-memory.rst b/tools/testing/selftests/drivers/apple/j616s-memory.rst new file mode 100644 index 0000000000000..2b493a333191c --- /dev/null +++ b/tools/testing/selftests/drivers/apple/j616s-memory.rst @@ -0,0 +1,51 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +J616s memory policy checks +========================= + +The T6040 uses the ``apple,t6041-pmc`` and ``apple,t6041-clpc`` compatibles. +The J616s DT enables both the PMC and the PMP. ``apple,pmp`` gates PMC probe +on PMP readiness; static voters 0, 1 and 3 enable PMP, CLPC and AP requests on +the DCS and fabric rails. Do not enable the PMP voter before its owner is +running: its boot-time floor can be the maximum state. + +For CPU demand, CLPC samples every 4 ms, qualifies three consecutive samples +with at least 50 percent busy time on a CPU in a cluster, and uses the cluster +frequency quartile to request DCS states 1, 2, 5, 5 and fabric states 0, 1, 2, +2. Sustained work in the upper half of a cluster's range requests DCS state 5 +(the sixth state). On idle, the host floor is released after a 32 ms hold; +firmware and display clients may continue requesting memory states. The +host's zero floor does not imply that DRAM actually runs in state zero. + +``/sys/kernel/debug/apple-pmc/*/status`` reports requested and honoured agent +floors and rail enable masks. The CLPC device's debugfs ``status`` reports its +CPU requests, current vote and error count. These are policy inputs, not DCS +achieved-state residency. In particular, neither an honoured floor nor the +``pmc-dcs`` thermal cooling state is a current DRAM clock. + +The PMP report provider's ``dram/`` sysfs group contains static metadata:: + + find /sys/devices/platform -type d -name dram -print + +On a J616s DT carrying the DRAM description, its attributes report type +``lpddr5``, 6 states, width 256, maximum data rate 8533333333 transfers/s, +physical peak 273066666656 bytes/s, and published class 273000000000 bytes/s. +The group is conditional on valid boot DT metadata; a missing group with an +older boot DT does not imply a disabled DCS governor. It contains no current +rate or residency counters. + +For a reproducible application bandwidth workload, build on Linux:: + + cc -O2 -Wall -Wextra -Werror -pthread \ + tools/testing/selftests/drivers/apple/memcpy-bw.c -o /tmp/memcpy-bw + /tmp/memcpy-bw 8 256 10 + +Each of eight threads copies between its own two 256 MiB buffers for ten +seconds, after first-touch initialization. Output counts one source read and +one destination write per completed libc copy, in decimal GB/s, over the +wall time until the last thread finishes. It is an application estimate, not +an AMCC bus measurement: write allocation, cache effects and libc algorithm +can change the physical traffic. Record libc, CPU governor, display topology, +thermal state and the exact kernel/DT alongside the result. Compare idle, +bounded CPU load, memcpy, and recovery using the same observation method. +Do not infer lower-state rates from bandwidth response thresholds. diff --git a/tools/testing/selftests/drivers/apple/memcpy-bw.c b/tools/testing/selftests/drivers/apple/memcpy-bw.c new file mode 100644 index 0000000000000..9c8961e355356 --- /dev/null +++ b/tools/testing/selftests/drivers/apple/memcpy-bw.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Bounded application memcpy throughput; this is not an AMCC bus counter. */ +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include +#include +#include + +#define MAX_THREADS 64 + +static pthread_barrier_t ready; +static uint64_t start_ns, duration_ns; +static size_t bytes; +/* A volatile function pointer keeps each libc copy observable to the compiler. */ +static void *(*volatile copy_memory)(void *, const void *, size_t) = memcpy; + +struct worker { + void *src, *dst; + uint64_t copies, end_ns; +}; + +static uint64_t now_ns(void) +{ + struct timespec ts; + + if (clock_gettime(CLOCK_MONOTONIC, &ts)) { + perror("clock_gettime"); + exit(1); + } + return (uint64_t)ts.tv_sec * 1000000000 + ts.tv_nsec; +} + +static void *copy_worker(void *arg) +{ + struct worker *worker = arg; + + memset(worker->src, 0x5a, bytes); + memset(worker->dst, 0, bytes); + pthread_barrier_wait(&ready); + pthread_barrier_wait(&ready); + do { + copy_memory(worker->dst, worker->src, bytes); + worker->copies++; + worker->end_ns = now_ns(); + } while (worker->end_ns - start_ns < duration_ns); + return NULL; +} + +static unsigned long number(const char *arg, unsigned long max) +{ + char *end; + unsigned long value; + + errno = 0; + value = strtoul(arg, &end, 10); + if (errno || !*arg || *end || !value || value > max) { + fprintf(stderr, "Invalid argument: %s (expected 1..%lu)\n", arg, max); + exit(2); + } + return value; +} + +int main(int argc, char **argv) +{ + struct worker workers[MAX_THREADS] = {}; + pthread_t threads[MAX_THREADS]; + unsigned long count = 8, mib = 256, seconds = 10; + uint64_t copies = 0, end_ns = 0; + unsigned long i; + int ret; + + if (argc > 4) { + fprintf(stderr, "Usage: %s [threads [MiB-per-buffer [seconds]]]\n", argv[0]); + return 2; + } + if (argc > 1) + count = number(argv[1], MAX_THREADS); + if (argc > 2) + mib = number(argv[2], 1024); + if (argc > 3) + seconds = number(argv[3], 60); + bytes = mib * 1024 * 1024; + duration_ns = seconds * 1000000000ULL; + for (i = 0; i < count; i++) { + ret = posix_memalign(&workers[i].src, 16384, bytes); + if (!ret) + ret = posix_memalign(&workers[i].dst, 16384, bytes); + if (ret) { + fprintf(stderr, "Allocation: %s\n", strerror(ret)); + return 1; + } + } + ret = pthread_barrier_init(&ready, NULL, count + 1); + if (ret) { + fprintf(stderr, "Barrier: %s\n", strerror(ret)); + return 1; + } + for (i = 0; i < count; i++) { + ret = pthread_create(&threads[i], NULL, copy_worker, &workers[i]); + if (ret) { + fprintf(stderr, "Thread: %s\n", strerror(ret)); + /* Terminate any workers already waiting at the barrier. */ + exit(1); + } + } + pthread_barrier_wait(&ready); + start_ns = now_ns(); + pthread_barrier_wait(&ready); + for (i = 0; i < count; i++) { + pthread_join(threads[i], NULL); + copies += workers[i].copies; + if (workers[i].end_ns > end_ns) + end_ns = workers[i].end_ns; + if (memcmp(workers[i].src, workers[i].dst, bytes)) { + fprintf(stderr, "Copy verification failed\n"); + return 1; + } + free(workers[i].src); + free(workers[i].dst); + } + pthread_barrier_destroy(&ready); + printf("threads=%lu bytes_per_buffer=%zu elapsed_s=%.6f copies=%llu " + "application_rd_wr_GBps=%.3f\n", count, bytes, + (end_ns - start_ns) / 1e9, (unsigned long long)copies, + 2.0 * bytes * copies / (end_ns - start_ns)); + return 0; +} From 5a93c9fcd577310c2e52a75c4b270c21e428461d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:30:06 +0000 Subject: [PATCH 1458/1579] selftests: apple: collect bounded J616s parity snapshots Record policy, cached IIO values, requested CPU clocks and static memory metadata around bounded CPU and memcpy workloads. Preserve read failures and terminate only the helper's own child processes on interruption or partial startup failure. Check the board model before starting work. Document the resource footprint and the limits of polling cached values. This collector does not measure achieved DCS residency or ALS cadence. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/apple/j616s-memory.rst | 13 +++ .../selftests/drivers/apple/j616s-parity.py | 99 +++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 tools/testing/selftests/drivers/apple/j616s-parity.py diff --git a/tools/testing/selftests/drivers/apple/j616s-memory.rst b/tools/testing/selftests/drivers/apple/j616s-memory.rst index 2b493a333191c..da1dfa386e092 100644 --- a/tools/testing/selftests/drivers/apple/j616s-memory.rst +++ b/tools/testing/selftests/drivers/apple/j616s-memory.rst @@ -49,3 +49,16 @@ can change the physical traffic. Record libc, CPU governor, display topology, thermal state and the exact kernel/DT alongside the result. Compare idle, bounded CPU load, memcpy, and recovery using the same observation method. Do not infer lower-state rates from bandwidth response thresholds. + +The capture helper checks the board model, records available policy and IIO +attributes, then runs two seconds idle, five seconds of fourteen CPU workers, +two seconds recovery, the ten-second memcpy workload, and two seconds recovery:: + + python3 tools/testing/selftests/drivers/apple/j616s-parity.py \ + --memcpy /tmp/memcpy-bw --output /tmp/j616s-parity-NEW + +The output directory must be new. Each read failure is recorded in JSON, and +the helper terminates only its own workload processes on failure or interrupt. +The memcpy phase needs 4 GiB plus overhead. Polling at 20 Hz can itself cause +CPU wakeups; use low-overhead counter snapshots for final idle-residency and +power comparisons. Cached IIO values do not establish report arrival cadence. diff --git a/tools/testing/selftests/drivers/apple/j616s-parity.py b/tools/testing/selftests/drivers/apple/j616s-parity.py new file mode 100644 index 0000000000000..d8835771f36c8 --- /dev/null +++ b/tools/testing/selftests/drivers/apple/j616s-parity.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Collect J616s policy snapshots around bounded CPU and memcpy workloads.""" +import argparse +import json +import pathlib +import subprocess +import time + +parser = argparse.ArgumentParser() +parser.add_argument('--memcpy', required=True, help='Path to public memcpy-bw binary') +parser.add_argument('--output', required=True, help='New private evidence directory') +args = parser.parse_args() +try: + compatible = pathlib.Path('/proc/device-tree/compatible').read_bytes() +except OSError as error: + raise SystemExit(f'Cannot read device identity: {error.strerror}') +if b'apple,j616s' not in compatible.split(b'\0'): + raise SystemExit('Device identity is not apple,j616s') +output = pathlib.Path(args.output) +output.mkdir(parents=True, exist_ok=False) + +# Read only documented policy, cached IIO readings, and requested clock attributes. +paths = sorted(set( + list(pathlib.Path('/sys/kernel/debug/apple-pmc').glob('*/status')) + + list(pathlib.Path('/sys/kernel/debug').glob('*performance-controller*/status')) + + list(pathlib.Path('/sys/devices/system/cpu/cpufreq').glob('policy*/scaling_cur_freq')) +)) +als = [] +for device in pathlib.Path('/sys/bus/iio/devices').glob('iio:device*'): + try: + if (device / 'name').read_text().strip() == 'aop-sensors-als': + als += [device / 'in_illuminance_input', device / 'in_cct_input'] + except OSError: + pass +paths += als + +def read(path): + try: + return path.read_text().strip() + except OSError as error: + return {'error': error.strerror, 'errno': error.errno} + +identity = {'kernel': subprocess.check_output(['uname', '-r'], text=True).strip(), + 'compatible': compatible.decode(errors='replace').split('\0'), + 'selected_paths': [str(path) for path in paths], + 'dram': {}} +for group in pathlib.Path('/sys/devices/platform').rglob('dram'): + for name in ['type', 'state_count', 'data_bus_width', 'max_data_rate', 'peak_bandwidth', 'bandwidth_class']: + identity['dram'][str(group / name)] = read(group / name) +(output / 'identity.json').write_text(json.dumps(identity, indent=2) + '\n') +children = [] +with (output / 'snapshots.jsonl').open('x') as log: + def sample(phase): + record = {'monotonic_ns': time.monotonic_ns(), 'phase': phase, + 'values': {str(path): read(path) for path in paths}} + log.write(json.dumps(record) + '\n') + log.flush() + + def observe(phase, seconds): + end = time.monotonic() + seconds + while time.monotonic() < end: + sample(phase) + time.sleep(0.05) + + try: + observe('idle', 2) + with open('/dev/null', 'wb') as null: + for _ in range(14): + children.append(subprocess.Popen(['yes'], stdout=null)) + observe('all-core', 5) + for child in children: + child.terminate() + for child in children: + child.wait() + children = [] + observe('cpu-recovery', 2) + with (output / 'memcpy.txt').open('x') as metrics: + child = subprocess.Popen([args.memcpy, '8', '256', '10'], stdout=metrics, stderr=subprocess.STDOUT) + children = [child] + deadline = time.monotonic() + 60 + while child.poll() is None and time.monotonic() < deadline: + sample('memcpy-including-initialization') + time.sleep(0.05) + if child.poll() is None: + raise RuntimeError('memcpy did not finish within 60 seconds') + (output / 'memcpy-exit.txt').write_text(str(child.returncode) + '\n') + children = [] + observe('memory-recovery', 2) + finally: + for child in children: + if child.poll() is None: + child.terminate() + for child in children: + try: + child.wait(timeout=2) + except subprocess.TimeoutExpired: + child.kill() + child.wait() From 64357f8873ad1221d3bc98e89a0455235cf59b94 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 15:31:33 +1100 Subject: [PATCH 1459/1579] media: apple: isp: report and propagate sensor tuning failures NV12 setup ignored every setfile error except EINTR, including a bad header, a short file and a rejected firmware upload. This made an unusable tuning file look like a stream running with model tuning. Keep absent tuning optional, but fail setup for other read, validation and upload errors. Log failed uploads and add a debug message for a successful upload. Exercise failure propagation and missing-file fallback at the capture setup boundary with KUnit static stubs. Distinguish model-wide sensor setfiles from per-module iCCl factory data in Kconfig and document the existing NR commands, P010 omissions and the unimplemented factory calibration/ANE interfaces. Do not change the qualified preview sequence or guess undocumented firmware parameters. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../media/drivers/apple-isp-j616s.rst | 39 ++++++++++++++ drivers/media/platform/apple/isp/Kconfig | 6 ++- .../media/platform/apple/isp/isp-cam-test.c | 51 +++++++++++++++++++ drivers/media/platform/apple/isp/isp-cam.c | 32 +++++++++--- 4 files changed, 118 insertions(+), 10 deletions(-) diff --git a/Documentation/driver-api/media/drivers/apple-isp-j616s.rst b/Documentation/driver-api/media/drivers/apple-isp-j616s.rst index 9a2bf93d7b059..4a5fec6fd37f4 100644 --- a/Documentation/driver-api/media/drivers/apple-isp-j616s.rst +++ b/Documentation/driver-api/media/drivers/apple-isp-j616s.rst @@ -41,6 +41,45 @@ firmware-derived output modes still need capture validation on 26.6.2. Only our source and synthetic protocol tests are included; no Apple binary, setfile contents, calibration table or firmware image is added. +Sensor tuning and noise reduction +--------------------------------- + +NV12 setup loads model-wide sensor tuning using ``CH_SET_FILE_LOAD`` +(0x0111). T8140/IMX558 selects ``apple/isp_1925_03XX.dat``; T6040/IMX958 +selects ``apple/isp_2224_01XX.dat``. The T6040 lens-shading fixup is applied +only to the copied firmware buffer. A missing file allows streaming with a +warning. An invalid file, interrupted read or failed firmware upload fails +stream setup instead of silently continuing. A debug message identifies a +successful upload without dumping its contents. + +The NV12 command sequence already sends ``CH_CNR_START`` (0x0a2f), +``CH_MBNR_ENABLE`` (0x0a3a, use case 0, mode 1, chroma enabled), +``APPLE_CH_TEMPORAL_FILTER_START`` (0xc100), +``APPLE_CH_MOTION_HISTORY_START`` (0xc102) and +``APPLE_CH_TEMPORAL_FILTER_ENABLE`` (0xc113). The temporal-start record has +one fixed argument of 1 and another supplied by ``apple,temporal-filter`` +(default 0; J700 supplies 1). The latter's meaning is not established here; +it must not be treated as a boolean enable or an output-pool count. +Command submission alone does not establish equivalence to a host-side +motion-compensated filter or measure the firmware's resulting noise reduction. + +The P010 preview still uses its qualified minimal sequence, which skips +these explicit NR commands and setfile upload. An earlier NV12 stream may +have left tuning in the resident firmware. Compare cold-boot P010 with +NV12-to-P010 transitions when assessing image quality. + +Sensor setfiles are distinct from individual camera-module factory +calibration (iCCl). The driver has no iCCl parser, serial binding or upload +command. The installer preserves raw factory records in the local fallback +archive; this does not mean the ISP consumes them. No factory record or +vendor payload belongs in the kernel source or a distributed firmware package. + +The separate ANE video denoiser is not implemented. The board parameter facts +are auto mode 2, total-gain entry/exit thresholds 2304/1792 on J700 and +3584/2816 on J616s, an FPS threshold of 30 and a 30-frame ramp-down. The FPS +comparison direction, gain units and ANE/ISP interchange need a confirmed +interface before implementation. + Public source audit ------------------- diff --git a/drivers/media/platform/apple/isp/Kconfig b/drivers/media/platform/apple/isp/Kconfig index 7a797a1a7e1a3..9b28a0de50797 100644 --- a/drivers/media/platform/apple/isp/Kconfig +++ b/drivers/media/platform/apple/isp/Kconfig @@ -14,9 +14,11 @@ config VIDEO_APPLE_ISP Driver for the image signal processor that runs the built-in camera of Apple silicon machines. The ISP firmware is loaded by the bootloader. When a stream starts, the driver loads the - calibration file for the detected camera module, + sensor tuning file for the detected camera model, apple/isp__XX.dat, from the firmware search - path; if it is missing, the stream runs without calibration. + path; if it is missing, the stream runs without sensor tuning. + Invalid files and firmware upload errors fail the stream setup. + These setfiles do not contain per-module factory calibration. config VIDEO_APPLE_ISP_KUNIT_TEST bool "KUnit tests for the Apple ISP driver" if !KUNIT_ALL_TESTS diff --git a/drivers/media/platform/apple/isp/isp-cam-test.c b/drivers/media/platform/apple/isp/isp-cam-test.c index 27c9e23f348aa..8991abed7d88b 100644 --- a/drivers/media/platform/apple/isp/isp-cam-test.c +++ b/drivers/media/platform/apple/isp/isp-cam-test.c @@ -69,6 +69,8 @@ static const struct isp_test_command isp_test_preview_commands[] = { struct isp_test_command_trace { unsigned int count; unsigned int fail_at; + int setfile_error; + unsigned int setfile_calls; }; static int isp_test_send(struct apple_isp *isp, void *args, u32 insize, @@ -158,6 +160,53 @@ static void isp_test_preview_command_failures(struct kunit *test) } } +static int isp_test_load_setfile(struct apple_isp *isp, u32 ch) +{ + struct kunit *test = kunit_get_current_test(); + struct isp_test_command_trace *trace = test->priv; + + trace->setfile_calls++; + return trace->setfile_error; +} + +static void isp_test_setfile_failure_stops_setup(struct kunit *test) +{ + static const int errors[] = { -EINVAL, -EIO, -ETIME, -EINTR, -ENOMEM }; + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_test_command_trace trace; + + isp_get_current_format(isp)->pixelformat = V4L2_PIX_FMT_NV12; + kunit_activate_static_stub(test, isp_ch_load_setfile, isp_test_load_setfile); + test->priv = &trace; + for (unsigned int i = 0; i < ARRAY_SIZE(errors); i++) { + trace = (struct isp_test_command_trace){ .setfile_error = errors[i] }; + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), errors[i]); + /* Only the flicker command preceded the rejected tuning. */ + KUNIT_EXPECT_EQ(test, trace.count, 1); + KUNIT_EXPECT_EQ(test, trace.setfile_calls, 1); + } +} + +static void isp_test_optional_setfile_continues_setup(struct kunit *test) +{ + static const int results[] = { 0, -ENOENT }; + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_test_command_trace trace; + + isp_get_current_format(isp)->pixelformat = V4L2_PIX_FMT_NV12; + kunit_activate_static_stub(test, isp_ch_load_setfile, isp_test_load_setfile); + test->priv = &trace; + for (unsigned int i = 0; i < ARRAY_SIZE(results); i++) { + trace = (struct isp_test_command_trace){ + .setfile_error = results[i], .fail_at = 2, + }; + /* The SBS command after tuning, rather than tuning, fails. */ + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), -ETIME); + KUNIT_EXPECT_EQ(test, trace.count, 2); + KUNIT_EXPECT_EQ(test, trace.setfile_calls, 1); + } +} + static int isp_test_config_reply(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) { @@ -319,6 +368,8 @@ static void isp_test_command_ack_failure_latches(struct kunit *test) } static struct kunit_case isp_cam_test_cases[] = { + KUNIT_CASE(isp_test_setfile_failure_stops_setup), + KUNIT_CASE(isp_test_optional_setfile_continues_setup), KUNIT_CASE(isp_test_command_ack_failure_latches), KUNIT_CASE(isp_test_sensor_geometry), KUNIT_CASE(isp_test_sensor_config_list), diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index a5cbb2b4d8eb0..8483f511f6299 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -3,6 +3,7 @@ #include #include +#include #include "isp-cam.h" #include "isp-cmd.h" @@ -512,14 +513,20 @@ static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) u32 magic; int err; + KUNIT_STATIC_STUB_REDIRECT(isp_ch_load_setfile, isp, ch); + if (WARN_ON_ONCE(setfile->size > isp->data_surf->size)) return -ENOSPC; err = firmware_request_nowarn(&fw, setfile->path, isp->dev); if (err) { - dev_warn_once(isp->dev, - "setfile '%s' not loaded (%d), streaming without calibration\n", - setfile->path, err); + if (err == -ENOENT) + dev_warn_once(isp->dev, + "setfile '%s' missing, streaming without sensor tuning\n", + setfile->path); + else + dev_err(isp->dev, "failed to read setfile '%s': %d\n", + setfile->path, err); return err; } @@ -545,8 +552,16 @@ static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) if (isp->hw->gen == ISP_GEN_T6040) isp_setfile_fixup_h17(isp->data_surf->virt, len); - return isp_cmd_ch_set_file_load(isp, ch, isp->data_surf->iova, - setfile->size); + err = isp_cmd_ch_set_file_load(isp, ch, isp->data_surf->iova, + setfile->size); + if (err) + dev_err(isp->dev, "ch %u: setfile '%s' load failed: %d\n", + ch, setfile->path, err); + else + dev_dbg(isp->dev, "ch %u: loaded sensor tuning '%s' (0x%zx bytes)\n", + ch, setfile->path, setfile->size); + + return err; } static int isp_ch_configure_frame_rate(struct apple_isp *isp, u32 ch) @@ -613,11 +628,12 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) goto select_config; /* - * The setfile isn't requisite but then we don't get calibration. If - * loading it was interrupted by a signal, give up on the stream. + * Missing sensor tuning is optional. A present but invalid setfile or + * a failed firmware upload must not silently select firmware defaults. + * This model-wide tuning is separate from per-module factory data. */ err = isp_ch_load_setfile(isp, ch); - if (err == -EINTR) + if (err && err != -ENOENT) return err; if (isp->hw->lpdp) { From 0ab62d2f77797be2f5b375d2fb2545fcb49f4084 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 15:33:36 +1100 Subject: [PATCH 1460/1579] media: apple: isp: account for NV12 flicker replies in tuning tests The preview fixture describes P010, whose flicker request has no output. The new tuning failure cases use NV12, whose existing flicker request returns a 12-byte record. Keep both expectations in the shared transport stub so the tests validate the appropriate command ABI. Clarify that a completed tuning upload is a transport result rather than proof that the firmware applied every coefficient; its NV12 reply fields remain undecoded. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/driver-api/media/drivers/apple-isp-j616s.rst | 2 ++ drivers/media/platform/apple/isp/isp-cam-test.c | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Documentation/driver-api/media/drivers/apple-isp-j616s.rst b/Documentation/driver-api/media/drivers/apple-isp-j616s.rst index 4a5fec6fd37f4..329b811dce70e 100644 --- a/Documentation/driver-api/media/drivers/apple-isp-j616s.rst +++ b/Documentation/driver-api/media/drivers/apple-isp-j616s.rst @@ -51,6 +51,8 @@ only to the copied firmware buffer. A missing file allows streaming with a warning. An invalid file, interrupted read or failed firmware upload fails stream setup instead of silently continuing. A debug message identifies a successful upload without dumping its contents. +This confirms transport completion only: the NV12 path does not decode the +setfile reply fields, so it does not prove that every coefficient was applied. The NV12 command sequence already sends ``CH_CNR_START`` (0x0a2f), ``CH_MBNR_ENABLE`` (0x0a3a, use case 0, mode 1, chroma enabled), diff --git a/drivers/media/platform/apple/isp/isp-cam-test.c b/drivers/media/platform/apple/isp/isp-cam-test.c index 8991abed7d88b..162e2ff1f0258 100644 --- a/drivers/media/platform/apple/isp/isp-cam-test.c +++ b/drivers/media/platform/apple/isp/isp-cam-test.c @@ -79,6 +79,7 @@ static int isp_test_send(struct apple_isp *isp, void *args, u32 insize, struct kunit *test = kunit_get_current_test(); struct isp_test_command_trace *trace = test->priv; const struct isp_test_command *expected; + u32 expected_outsize; if (trace->count >= ARRAY_SIZE(isp_test_preview_commands)) { KUNIT_FAIL(test, "unexpected extra command"); @@ -86,7 +87,9 @@ static int isp_test_send(struct apple_isp *isp, void *args, u32 insize, } expected = &isp_test_preview_commands[trace->count++]; KUNIT_EXPECT_EQ(test, insize, expected->size); - KUNIT_EXPECT_EQ(test, outsize, expected->words[1] == 0x117 ? insize : 0); + expected_outsize = expected->words[1] == 0x117 || + (expected->words[1] == 0x24 && !isp_is_p010(isp)) ? insize : 0; + KUNIT_EXPECT_EQ(test, outsize, expected_outsize); if (insize == expected->size) KUNIT_EXPECT_MEMEQ(test, args, expected->words, insize); if (trace->count == trace->fail_at) From 1ab7d910cb984b0baceb61b814576fb40a7b7840 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 14:58:48 +1100 Subject: [PATCH 1461/1579] accel/ane: apply the H16 macOS 27.0 runtime wire contract Use the observed H16 command IDs, sizes and channel indices rather than the H14 procedure ABI. Prepare the zero-word/u16-ID/u16-ack envelope and reject invalid runtime packets before publication. Retain descriptor wire indices independently of logical channel names. Validate 0x60-byte 0x0300 reports as diagnostics without interpreting the undecoded body as legacy completion. Record the single J616s IRQ as the channel reply/event doorbell. Keep cold boot, preload and original-HWX gates closed and allocation ceilings unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 80 +++++++++++--- drivers/accel/ane/ane_fw_platform.c | 12 ++- drivers/accel/ane/ane_fw_platform.h | 6 ++ drivers/accel/ane/ane_fw_protocol.h | 15 ++- drivers/accel/ane/ane_fw_protocol_test.c | 113 ++++++++++++++++++++ drivers/accel/ane/ane_h16_protocol.h | 79 ++++++++++++++ drivers/accel/ane/ane_t6021_rtclient_main.c | 33 ++++-- 7 files changed, 306 insertions(+), 32 deletions(-) create mode 100644 drivers/accel/ane/ane_h16_protocol.h diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index 6e636062bd2d0..b16cb55f783df 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -34,10 +34,9 @@ H16 preflight checks resource geometry, firmware identity and these ranges without MMIO or power access. No disk firmware is requested. Missing reservations fail with ``-ENODATA``. Malformed reservations fail with ``-EINVAL``. Even a valid handoff currently fails with ``-EOPNOTSUPP``: -the pinned immutable-image comparison reference and the byte layouts of the -custom HWX descriptors/aperture records are not supplied. The section-only -load ABI is explicitly refused for H16. No accel -node or program-loading interface is published. +the strict preload validator, original-container HWX ABI and cold boot +backend are incomplete. The section-only load ABI is explicitly refused +for H16. No accel node or program-loading interface is published. The H16 DT nodes stay disabled. The five real PMGR providers already encode the system/CPU/MPM/task/base dependency graph. The PMGR binding does not @@ -77,16 +76,63 @@ It never opens an accel device or claims a Linux inference result. Remaining implementation gates ------------------------------ -SPEC-r2 defines the 27.x channel and CONFIG_GET contract, including the exact -24 MHz reply requirement. It does not give the byte offsets and widths of -the custom kind-3 tensor/kind-4 program descriptors or 16-byte task-aperture -records, nor an available pristine-image comparison reference/manifest. -The 43 patchbay record widths/order and original immutable bytes must be -proved against the SHA-pinned payload, not inferred from mutable live bytes. -A SHA-256 digest alone cannot reconstruct that reference. - -Until those exact layouts and the validated comparison reference are -available, H16 remains refused before power access. -No physical RTKit boot or complete HWX load path is enabled. H17 additionally -requires a Linux ownership backend before any direct-engine access. There -are no module parameters to bypass these gates. +SPEC-r3 supplies the custom descriptor/aperture layouts and a normalized +immutable preload digest with 42 ordered patchbay records. The complete +validators, original-container ABI and physical boot backend still need +implementation. Off-list preload mutations remain refused. + +The 2026-10-31 H16 runtime observation did not capture cold boot, scratch +negotiation, RTKit management, CONFIG_GET or RESOURCE_INFO_GET. The version-12 +and 24 MHz predicates remain conservative proposed startup gates; runtime +traffic does not prove them. Resource and allocation ceilings stay unchanged. +H17 additionally requires a Linux ownership backend before direct access. +There are no module parameters to bypass these gates. + +H16 macOS 27.0 runtime contract +------------------------------ + +J616s has one ANE interrupt, 1054, used as the shared-memory channel +doorbell/completion line for all firmware-to-host replies and events. +It has no ANE exclave assignment. This identifies the interrupt role, not +its cold-start masking/acknowledgment sequence. H17's two IRQ roles remain +unknown. ANE runtime uses its own channel transport; it does not establish +an A7IOP/RTBuddy boot sequence. + +Host command bytes 0..3 are zero, the little-endian u16 ID is at +4 and +the u16 acknowledgment at +6. All observed successful replies had zero +acknowledgment. Nonzero meanings remain unknown and are refused. + +================== ====== ======= ================== +Command ID Channel Bytes +================== ====== ======= ================== +LOAD_PROGRAM 0x0200 1 0x1c0 +UNLOAD_PROGRAM 0x0201 1 0xc +CREATE_PROCESS 0x0202 1 0x258 +TERMINATE_PROCESS 0x0203 1 0x10 +EXECUTE 0x020c 7 0xca8/0xcc0/0xcd8 +POWER_GATE 0x002d 7 0xc +EVENT_REPORT 0x0300 6 0x60 (firmware to host) +================== ====== ======= ================== + +These sizes bound the observed version, not future firmware. POWER_GATE has +a u32 Boolean at +8. The execute-size increment of 0x18 does not establish +buffer-record semantics. The channel numbers are observed table indices; +their mapping to channel names remains unproved. The parser retains each +descriptor's original index rather than assuming name order is wire order. + +The transport prepares the H16 envelope and rejects unrecognized runtime +IDs, wrong sizes, channels and power-gate values before publication. +H14's 0x0204 procedure packet and short CREATE_PROCESS body remain H14-only; +H16's body serializers and completion correlation are not implemented. +The H16 runtime command table does not authorize startup commands. + +The event report starts with u32 0x0000fe00 through 0x0000fe03, then u16 +0x0300 and zero u16 at +6. Its body is undecoded. Valid reports are consumed +as diagnostics and never complete an inference or release its buffers. +The legacy 0x28-byte state-1 completion predicate must not be used for H16. +Unknown short channel-6 messages fail closed. A timeout retains mappings +and quarantines the instance; it does not trigger a retry or reset. + +KUnit covers all seven runtime IDs and their sizes/directions/channels, +the u16 envelope and refusal semantics, and report validation without +inventing completion fields. All test packets are synthetic. diff --git a/drivers/accel/ane/ane_fw_platform.c b/drivers/accel/ane/ane_fw_platform.c index 99c2ba09418ac..fe76586f779cd 100644 --- a/drivers/accel/ane/ane_fw_platform.c +++ b/drivers/accel/ane/ane_fw_platform.c @@ -32,6 +32,8 @@ const struct ane_fw_platform ane_h16_platform = { .hwx_subtype = 7, .task_generation = 17, .interrupt_count = 1, + /* J616s IRQ 1054 drains replies and events from all F2H channels. */ + .irq_role = ANE_FW_IRQ_CHANNELS, .owner = ANE_FW_OWNER_AP, .reg_names = ane_h16_reg_names, .reg_sizes = ane_h16_reg_sizes, @@ -157,11 +159,11 @@ int ane_fw_platform_preflight(struct platform_device *pdev, return dev_err_probe(dev, ret, "H16 requires bootloader-reserved no-map TEXT/DATA preload ranges\n"); - /* SPEC-r2 defines CONFIG_GET and dynamic channels. Opening this gate - * additionally needs a pinned immutable-image comparison reference and - * the exact custom HWX descriptor/aperture layouts. Neither may be - * replaced by a broad preload-diff rule or a section-only LOAD ioctl. + /* SPEC-r3 supplies the preload manifest and HWX layouts. Their full + * validators, original-container ABI and cold boot backend are not yet + * implemented. The runtime capture did not observe firmware startup. + * Neither a broad preload-diff rule nor a section-only LOAD opens this. */ - dev_info(dev, "H16 ANE: pinned preload reference and HWX layout unavailable; refusing before power access\n"); + dev_info(dev, "H16 ANE: preload validation, HWX ABI and boot backend incomplete; refusing before power access\n"); return -EOPNOTSUPP; } diff --git a/drivers/accel/ane/ane_fw_platform.h b/drivers/accel/ane/ane_fw_platform.h index 93d8768217ebd..a7d4ed4f344f0 100644 --- a/drivers/accel/ane/ane_fw_platform.h +++ b/drivers/accel/ane/ane_fw_platform.h @@ -16,6 +16,11 @@ enum ane_fw_owner { ANE_FW_OWNER_EXCLAVE, }; +enum ane_fw_irq_role { + ANE_FW_IRQ_UNKNOWN, + ANE_FW_IRQ_CHANNELS, +}; + /* Later firmware generations must never enter the H14 staging path. */ struct ane_fw_platform { u32 soc; @@ -24,6 +29,7 @@ struct ane_fw_platform { u8 hwx_subtype; u8 task_generation; u8 interrupt_count; + enum ane_fw_irq_role irq_role; enum ane_fw_owner owner; const char * const *reg_names; const u32 *reg_sizes; diff --git a/drivers/accel/ane/ane_fw_protocol.h b/drivers/accel/ane/ane_fw_protocol.h index aa238dfb2eb2a..ec2aca2b35845 100644 --- a/drivers/accel/ane/ane_fw_protocol.h +++ b/drivers/accel/ane/ane_fw_protocol.h @@ -20,6 +20,7 @@ struct ane_fw_channel { u32 slots; u32 type; u32 bit; + u32 index; }; static inline bool ane_fw_contains(u64 base, u64 bytes, u64 addr, u64 size) @@ -58,6 +59,7 @@ ane_fw_channels_parse(const void *table, u64 ipc_dva, size_t ipc_size, return -EPROTO; seen |= BIT(j); c = &parsed[j]; + c->index = i; c->type = get_unaligned_le32(desc + 0x40); c->bit = get_unaligned_le32(desc + 0x44); slots = get_unaligned_le64(desc + 0x48); @@ -109,6 +111,17 @@ ane_fw_channels_host_init(void *ipc, const struct ane_fw_channel *channels) } } +/* H14 preserves its reserved word; H16 requires bytes 0..3 to be zero. */ +static inline void ane_fw_command_header(void *command, u16 opcode, bool zero_reserved) +{ + u8 *p = command; + + if (zero_reserved) + put_unaligned_le32(0, p); + put_unaligned_le16(opcode, p + 4); + put_unaligned_le16(0, p + 6); +} + /* A firmware refusal is distinct from corruption and never means success. */ static inline int ane_fw_reply_status(u64 header, u64 slot_length, u64 slot_result, @@ -135,7 +148,7 @@ static inline int ane_fw_config_status(const void *command, size_t length) return 0; } -/* Only a matching state-1 event completes an execution. */ +/* Legacy H14/comparator layout only; H16 27.0 reports are 0x60 bytes. */ static inline int ane_fw_finished_event(const void *event, size_t length, u64 cookie, u32 program, u32 process) diff --git a/drivers/accel/ane/ane_fw_protocol_test.c b/drivers/accel/ane/ane_fw_protocol_test.c index e77db9b2c3a51..86fe430afde20 100644 --- a/drivers/accel/ane/ane_fw_protocol_test.c +++ b/drivers/accel/ane/ane_fw_protocol_test.c @@ -4,6 +4,114 @@ #include #include "ane_fw_protocol.h" +#include "ane_h16_protocol.h" + +static void ane_h16_id_table_test(struct kunit *test) +{ + static const struct ane_h16_command_info expected[] = { + { 0x200, 0x1c0, 0x1c0, 1, 1, false }, + { 0x201, 0xc, 0xc, 1, 1, false }, + { 0x202, 0x258, 0x258, 1, 1, false }, + { 0x203, 0x10, 0x10, 1, 1, false }, + { 0x20c, 0xca8, 0xcd8, 0x18, 7, false }, + { 0x2d, 0xc, 0xc, 1, 7, false }, + { 0x300, 0x60, 0x60, 1, 6, true }, + }; + u8 *command = kunit_kzalloc(test, 0xcd8, GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, command); + for (i = 0; i < ARRAY_SIZE(expected); i++) { + const struct ane_h16_command_info *e = &expected[i]; + const struct ane_h16_command_info *info = ane_h16_command_info(e->id); + size_t length; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, info); + KUNIT_EXPECT_EQ(test, info->min_size, e->min_size); + KUNIT_EXPECT_EQ(test, info->max_size, e->max_size); + KUNIT_EXPECT_EQ(test, info->size_step, e->size_step); + KUNIT_EXPECT_EQ(test, info->channel, e->channel); + KUNIT_EXPECT_EQ(test, info->from_firmware, e->from_firmware); + if (e->from_firmware) { + ret = ane_h16_command_prepare(command, e->min_size, e->id, e->channel); + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + continue; + } + for (length = e->min_size; length <= e->max_size; length += e->size_step) { + ret = ane_h16_command_prepare(command, length, e->id, e->channel); + KUNIT_EXPECT_EQ(test, ret, 0); + } + ret = ane_h16_command_prepare(command, e->min_size - 1, e->id, e->channel); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = ane_h16_command_prepare(command, e->max_size + 1, e->id, e->channel); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = ane_h16_command_prepare(command, e->min_size, e->id, e->channel ^ 1); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + } + KUNIT_EXPECT_PTR_EQ(test, ane_h16_command_info(0x204), NULL); + KUNIT_EXPECT_PTR_EQ(test, ane_h16_command_info(0x301), NULL); + KUNIT_EXPECT_EQ(test, ane_h16_command_prepare(command, 0xca8, 0x204, 7), + -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, ane_h16_command_prepare(command, 0xca9, 0x20c, 7), -EINVAL); + /* The H14 CREATE_PROCESS size must never be sent to H16. */ + KUNIT_EXPECT_EQ(test, ane_h16_command_prepare(command, 0x10, 0x202, 1), -EINVAL); +} + +static void ane_h16_envelope_test(struct kunit *test) +{ + u8 command[0xc]; + const u64 dva = 0x30000000000; + + memset(command, 0xa5, sizeof(command)); + put_unaligned_le32(1, command + 8); + KUNIT_ASSERT_EQ(test, ane_h16_command_prepare(command, sizeof(command), 0x2d, 7), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(command), 0U); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(command + 4), (u16)0x2d); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(command + 6), (u16)0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(command + 8), 1U); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 0xc, 0, dva, 0xc, + 0x2d, command, true), 0); + /* Test the upper byte too: acknowledgment is u16, not u8. */ + put_unaligned_le16(0x100, command + 6); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 0xc, 0, dva, 0xc, + 0x2d, command, true), -EREMOTEIO); + put_unaligned_le32(0, command + 8); + KUNIT_ASSERT_EQ(test, ane_h16_command_prepare(command, sizeof(command), 0x2d, 7), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(command + 6), (u16)0); + put_unaligned_le32(2, command + 8); + KUNIT_EXPECT_EQ(test, ane_h16_command_prepare(command, sizeof(command), 0x2d, 7), + -EINVAL); + KUNIT_EXPECT_EQ(test, ane_h16_command_prepare(NULL, sizeof(command), 0x2d, 7), + -EINVAL); +} + +static void ane_h16_report_test(struct kunit *test) +{ + u8 event[0x60] = {}; + unsigned int n; + + put_unaligned_le16(0x300, event + 4); + for (n = 0; n < 4; n++) { + put_unaligned_le32(0xfe00 | n, event); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, sizeof(event), 6), 0); + /* A report can never satisfy the legacy completion predicate. */ + KUNIT_EXPECT_EQ(test, ane_fw_finished_event(event, sizeof(event), 0, 0, 0), + -EPROTO); + } + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, sizeof(event), 7), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, 0x28, 6), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, 0xc, 6), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(NULL, 0x60, 6), -EPROTO); + put_unaligned_le32(0xfe04, event); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, sizeof(event), 6), -EPROTO); + put_unaligned_le32(0xfe00, event); + put_unaligned_le16(0x301, event + 4); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, sizeof(event), 6), -EPROTO); + put_unaligned_le16(0x300, event + 4); + put_unaligned_le16(0x100, event + 6); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, sizeof(event), 6), -EPROTO); +} #define TEST_IPC_SIZE 0x10000 #define TEST_IPC_DVA 0x20000000000ULL @@ -50,6 +158,7 @@ static void ane_dynamic_channels_test(struct kunit *test) KUNIT_EXPECT_EQ(test, channels[1].bit, 30U); KUNIT_EXPECT_EQ(test, channels[1].slots, 2U); KUNIT_EXPECT_EQ(test, channels[1].offset, 0x2400ULL); + KUNIT_EXPECT_EQ(test, channels[1].index, 1U); { u8 saved[ANE_FW_DESCRIPTOR_SIZE]; u8 *first = ipc + TEST_TABLE_OFF + ANE_FW_DESCRIPTOR_SIZE; @@ -60,6 +169,7 @@ static void ane_dynamic_channels_test(struct kunit *test) memcpy(last, saved, sizeof(saved)); KUNIT_ASSERT_EQ(test, ane_test_channels_parse(ipc, channels), 0); KUNIT_EXPECT_EQ(test, channels[1].offset, 0x2400ULL); + KUNIT_EXPECT_EQ(test, channels[1].index, 7U); } memset(ipc + 0x2000, 0xa5, 0x2000); ane_fw_channels_host_init(ipc, channels); @@ -240,6 +350,9 @@ static void ane_rtk_rejections_test(struct kunit *test) } static struct kunit_case ane_protocol_cases[] = { + KUNIT_CASE(ane_h16_id_table_test), + KUNIT_CASE(ane_h16_envelope_test), + KUNIT_CASE(ane_h16_report_test), KUNIT_CASE(ane_dynamic_channels_test), KUNIT_CASE(ane_channel_snapshot_test), KUNIT_CASE(ane_channel_rejections_test), diff --git a/drivers/accel/ane/ane_h16_protocol.h b/drivers/accel/ane/ane_h16_protocol.h new file mode 100644 index 0000000000000..f368212d230ba --- /dev/null +++ b/drivers/accel/ane/ane_h16_protocol.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_PROTOCOL_H__ +#define __ANE_H16_PROTOCOL_H__ + +#include "ane_fw_protocol.h" + +/* Derived field facts from H16/macOS 27.0, 2026-10-31. No payload bytes. */ +enum ane_h16_command { + ANE_H16_LOAD_PROGRAM = 0x0200, + ANE_H16_UNLOAD_PROGRAM = 0x0201, + ANE_H16_CREATE_PROCESS = 0x0202, + ANE_H16_TERMINATE_PROCESS = 0x0203, + ANE_H16_EXECUTE = 0x020c, + ANE_H16_POWER_GATE = 0x002d, + ANE_H16_EVENT_REPORT = 0x0300, +}; + +struct ane_h16_command_info { + u16 id; + u16 min_size; + u16 max_size; + u8 size_step; + u8 channel; + bool from_firmware; +}; + +static inline const struct ane_h16_command_info *ane_h16_command_info(u16 id) +{ + static const struct ane_h16_command_info commands[] = { + { ANE_H16_LOAD_PROGRAM, 0x1c0, 0x1c0, 1, 1, false }, + { ANE_H16_UNLOAD_PROGRAM, 0xc, 0xc, 1, 1, false }, + { ANE_H16_CREATE_PROCESS, 0x258, 0x258, 1, 1, false }, + { ANE_H16_TERMINATE_PROCESS, 0x10, 0x10, 1, 1, false }, + { ANE_H16_EXECUTE, 0xca8, 0xcd8, 0x18, 7, false }, + { ANE_H16_POWER_GATE, 0xc, 0xc, 1, 7, false }, + { ANE_H16_EVENT_REPORT, 0x60, 0x60, 1, 6, true }, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(commands); i++) + if (commands[i].id == id) + return &commands[i]; + return NULL; +} + +/* Validate the runtime wire shape, never infer a body from the H14 ABI. */ +static inline int +ane_h16_command_prepare(void *command, size_t length, u16 id, unsigned int channel) +{ + const struct ane_h16_command_info *info = ane_h16_command_info(id); + + if (!info || info->from_firmware) + return -EOPNOTSUPP; + if (!command || channel != info->channel || length < info->min_size || + length > info->max_size || (length - info->min_size) % info->size_step) + return -EINVAL; + if (id == ANE_H16_POWER_GATE && + get_unaligned_le32((u8 *)command + 8) > 1) + return -EINVAL; + ane_fw_command_header(command, id, true); + return 0; +} + +/* Reports are diagnostics only: cookie, state and completion are undecoded. */ +static inline int +ane_h16_event_report_status(const void *event, size_t length, unsigned int channel) +{ + const u8 *p = event; + const struct ane_h16_command_info *info = + ane_h16_command_info(ANE_H16_EVENT_REPORT); + + if (!p || channel != info->channel || length != info->min_size || + (get_unaligned_le32(p) & ~3U) != 0xfe00 || + get_unaligned_le16(p + 4) != info->id || get_unaligned_le16(p + 6)) + return -EPROTO; + return 0; +} + +#endif /* __ANE_H16_PROTOCOL_H__ */ diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index a50eeef08a90f..b4e44b65e95ae 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -92,6 +92,7 @@ #include #include "ane_fw_platform.h" +#include "ane_h16_protocol.h" #include "ane_fw_protocol.h" #include "ane_stats.h" #include "ane_t6021.h" @@ -565,7 +566,14 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, } if (time_after_eq(jiffies, deadline)) goto out; - put_unaligned_le16(opcode, (u8 *)command->cpu + 4); + if (ane->soc->platform) { + result = ane_h16_command_prepare(command->cpu, length, opcode, + ane->channels[channel].index); + if (result) + goto out; + } else { + ane_fw_command_header(command->cpu, opcode, false); + } if (ane->soc->platform) memset((u8 *)command->cpu + length, 0xa5, command->size - length); WRITE_ONCE(io[1], length); @@ -893,17 +901,17 @@ static bool ane_rtclient_drain_t2h(struct ane_rtclient *ane, dma_rmb(); dev_dbg(ane->dev, "T2H ch=%u slot=%u len=%#llx\n", channel, slot_i, len); - if (channel == 6 && ane->soc->platform) { + if (ane->soc->platform) { const void *ev = ane_rtclient_fw_cpu(ane, hdr, len); - int status = ane_fw_finished_event(ev, len, ane->call_cookie, - ane->call_program, ane->call_process); + int status = ane_h16_event_report_status(ev, len, c->index); if (status < 0) { atomic_set(&ane_t6021_quarantined, 1); return false; } - if (status == 1) - finished = true; + /* The captured H16 report has no decoded completion field. + * Consume diagnostics without retiring an execution or its BOs. + */ } else if (channel == 6 && len >= ANE_T2H_CALL_STATE_OFF + 4) { const u8 *ev = ane_rtclient_fw_cpu(ane, hdr, ANE_T2H_CALL_STATE_OFF + 4); @@ -1021,6 +1029,8 @@ static int ane_rtclient_command(struct ane_rtclient *ane, int ret; u64 stats_ticket = 0; u64 stats_submit_ns = 0; + bool execute = ane->soc->platform ? opcode == ANE_H16_EXECUTE : + opcode == CSNE_CMD_PROCEDURE_CALL; ane_t6021_tracing = opcode == CSNE_CMD_PROCEDURE_CALL && trace_td && ane->soc->trace_td_off; @@ -1043,8 +1053,7 @@ static int ane_rtclient_command(struct ane_rtclient *ane, * ring, as on ane.ko: stats=0 or a failed probe allocation leaves * stats_slots NULL, and the hooks must not run on a NULL ring. */ - bool stats_call = ane->stats_slots && - opcode == CSNE_CMD_PROCEDURE_CALL; + bool stats_call = ane->stats_slots && execute; if (stats_call) { stats_submit_ns = ktime_get_ns(); @@ -1066,7 +1075,7 @@ static int ane_rtclient_command(struct ane_rtclient *ane, atomic_set(&ane_t6021_quarantined, 1); return ret; } - if (opcode == CSNE_CMD_PROCEDURE_CALL) { + if (execute) { if (ane_t6021_tracing) ane_t6021_trace_add(ANE_TR_ACK, 0); ret = ane_rtclient_call_wait(ane, ane->soc->platform ? deadline : @@ -1395,6 +1404,9 @@ static int ane_rtclient_create_process(struct ane_rtclient *ane, struct ane_t6021_prog *prog = ane_t6021_prog_by_id(prog_id); int ret; + /* H16 uses 0x258 bytes; its body is not the H14 0x10-byte body. */ + if (ane->soc->platform) + return -EOPNOTSUPP; if (!prog) return -ENOENT; if (prog->proc_id != U32_MAX) { @@ -1438,6 +1450,9 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, size_t cmd_size; int ret, i, pinned = 0; + /* H16 EXECUTE is 0x020c on channel 7 with an undecoded body. */ + if (ane->soc->platform) + return -EOPNOTSUPP; if (user->pad || user->count < 1 || user->count > DRM_ANE_MAX_BINDS || user->priority < 2 || user->priority > 7 || !fd) return -EINVAL; From f272eb10be76910f6645aab0fa051421b59d2d20 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 15:49:49 +1100 Subject: [PATCH 1462/1579] dt-bindings: hwmon: apple,smc-hwmon: Deprecate host fan cooling Keep the former cooling-device specifier valid for older device trees, but describe that fan policy stays with the SMC firmware. Manual target writes remain available through hwmon. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/hwmon/apple,smc-hwmon.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index b8e962334935f..fcfe475d4dcf2 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -80,12 +80,10 @@ properties: "#cooling-cells": const: 3 + deprecated: true description: - Makes the fans cooling devices for thermal zones. The first cell is the - zero-based index of a fan, counting the fan-* child nodes in order; the - other two are the lower and upper cooling state. State 0 leaves the fan - to the SMC; states 1 to 8 run it at speeds spread evenly from its - minimum to its maximum. A fan needs all four of its speed and mode keys. + Former fan cooling-device specifier. Fan policy is handled by the SMC + firmware; the driver no longer registers fan cooling devices. patternProperties: "^current-[A-Za-z0-9]{4}$": From cc3766d3c0173d83bac1556dd67d5ad3c6c19197 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 15:49:49 +1100 Subject: [PATCH 1463/1579] hwmon: macsmc: Leave fan policy to firmware until a user requests control The host thermal ladder assumed the SMC needed a manual request to run the fans. Sustained macOS measurements instead show firmware ramping the fans while the host makes no SMC key writes. Remove the autonomous fan cooling devices and resume work; retain explicit fanX_target control and release on suspend, shutdown and removal. Read the mode before requesting manual control. In system mode, unlock first and wait using reads instead of provoking a refused mode write. Firmware without an unlock key fails without issuing a known unsupported write. Exercise manual targets, zero refused writes, untouched lifecycle paths, suspend release, resume without reapplication, and unlock failures with the existing KUnit SMC model. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/hwmon/macsmc-hwmon.rst | 5 + drivers/hwmon/macsmc-hwmon-test.c | 120 ++++------- drivers/hwmon/macsmc-hwmon.c | 293 ++++++--------------------- 3 files changed, 99 insertions(+), 319 deletions(-) diff --git a/Documentation/hwmon/macsmc-hwmon.rst b/Documentation/hwmon/macsmc-hwmon.rst index 2204745326cba..b0c78c3412088 100644 --- a/Documentation/hwmon/macsmc-hwmon.rst +++ b/Documentation/hwmon/macsmc-hwmon.rst @@ -31,6 +31,11 @@ as unsafe, allows manual speeds on every fan with a mode key whatever the Devicetree says. It cannot be proven that the system will fail safe if it overheats under a manual fan speed. +The driver leaves fan policy to the SMC until userspace writes a nonzero +``fanX_target``. Thermal zones do not take fans into manual mode. Suspend, +shutdown and removal release only fans taken over by such a request; resume +does not reapply it. A new target write is required after resume. + SoC die temperatures are described by the ``apple,soc-die-keys`` string list on the hwmon device node. The driver appends these temperature channels after the ``temperature-*`` children, in list order, with labels ``SoC die ``. diff --git a/drivers/hwmon/macsmc-hwmon-test.c b/drivers/hwmon/macsmc-hwmon-test.c index 312e752565e04..610a019ee59b8 100644 --- a/drivers/hwmon/macsmc-hwmon-test.c +++ b/drivers/hwmon/macsmc-hwmon-test.c @@ -2,7 +2,7 @@ /* * KUnit tests for the Apple SMC hwmon driver: channel numbering of the * temperature sensors, which thermal zones depend on, and manual fan control - * by userspace and by the thermal framework. + * by userspace. * * Included by macsmc-hwmon.c. */ @@ -106,6 +106,7 @@ struct macsmc_fan_test { u8 mode[2]; u32 target[2]; u8 ftst; + int writes; int ftst_writes; int refused; smc_key fail_key; @@ -149,6 +150,7 @@ static int fan_test_write(struct apple_smc *smc, smc_key key, const void *buf, s u32 value = 0; int i = fan_test_index(key); + t->writes++; memcpy(&value, buf, min(size, sizeof(value))); if (key == t->fail_key) return -EIO; @@ -218,9 +220,6 @@ static int macsmc_fan_test_init(struct kunit *test) t->hwmon.fan.count = 2; t->hwmon.fan.unlock_key = SMC_KEY(Ftst); mutex_init(&t->hwmon.fan.lock); - INIT_WORK(&t->hwmon.fan.cool_work, macsmc_hwmon_fans_cool_work); - for (i = 0; i < 2; i++) - t->fans[i].hwmon = &t->hwmon; test->priv = t; fan_control = true; kunit_activate_static_stub(test, apple_smc_read, fan_test_read); @@ -261,6 +260,7 @@ static void macsmc_fan_test_unlock_and_release(struct kunit *test) KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 0), 0); fan_test_released(test, t); KUNIT_EXPECT_EQ(test, t->ftst_writes, 2); + KUNIT_EXPECT_EQ(test, t->refused, 0); } /* Targets are clamped to [min, max]; negative ones are refused. */ @@ -303,14 +303,14 @@ static void macsmc_fan_test_two_fans(struct kunit *test) fan_test_released(test, t); } -/* Locked firmware without the unlock key: refused, nothing else written. */ +/* System mode without an unlock key: fail without attempting a mode write. */ static void macsmc_fan_test_no_unlock_key(struct kunit *test) { struct macsmc_fan_test *t = test->priv; t->hwmon.fan.unlock_key = 0; - KUNIT_EXPECT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), -EIO); - KUNIT_EXPECT_EQ(test, t->ftst_writes, 0); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, t->writes, 0); KUNIT_EXPECT_EQ(test, t->mode[0], MACSMC_FAN_MODE_SYSTEM); KUNIT_EXPECT_EQ(test, t->hwmon.fan.manual, 0); } @@ -330,97 +330,51 @@ static void macsmc_fan_test_legacy(struct kunit *test) KUNIT_EXPECT_EQ(test, t->refused, 0); } -/* Cooling states spread over [min, max]; 0 asks for nothing. */ -static void macsmc_fan_test_state_rpm(struct kunit *test) -{ - KUNIT_EXPECT_EQ(test, macsmc_hwmon_fan_state_rpm(1350, 5777, 0), 0); - KUNIT_EXPECT_EQ(test, macsmc_hwmon_fan_state_rpm(1350, 5777, 1), 1350); - KUNIT_EXPECT_EQ(test, macsmc_hwmon_fan_state_rpm(1350, 5777, 4), 3247); - KUNIT_EXPECT_EQ(test, macsmc_hwmon_fan_state_rpm(1350, 5777, - MACSMC_FAN_COOLING_STATES), 5777); - /* Never past the maximum, whatever the state. */ - KUNIT_EXPECT_EQ(test, macsmc_hwmon_fan_state_rpm(1350, 5777, - MACSMC_FAN_COOLING_STATES + 5), 5777); -} - -/* Cooling needs no fan_control: unlock for a state, release at state 0. */ -static void macsmc_fan_test_cooling(struct kunit *test) +/* Suspend, resume and shutdown do not write untouched fan keys. */ +static void macsmc_fan_test_untouched(struct kunit *test) { struct macsmc_fan_test *t = test->priv; + struct platform_device pdev = {}; - fan_control = false; - t->fans[0].cool_state = 1; - macsmc_hwmon_fans_apply(&t->hwmon); - KUNIT_EXPECT_EQ(test, t->ftst, 1); - KUNIT_EXPECT_EQ(test, t->mode[0], MACSMC_FAN_MODE_MANUAL); - KUNIT_EXPECT_EQ(test, t->target[0], FAN_TEST_F1350); - /* The other fan asks for nothing and stays with the firmware. */ - KUNIT_EXPECT_EQ(test, t->mode[1], MACSMC_FAN_MODE_AUTO); - KUNIT_EXPECT_FALSE(test, t->fans[1].manual); - - t->fans[0].cool_state = MACSMC_FAN_COOLING_STATES; - macsmc_hwmon_fans_apply(&t->hwmon); - KUNIT_EXPECT_EQ(test, t->target[0], FAN_TEST_F5777); - KUNIT_EXPECT_EQ(test, t->ftst_writes, 1); - - t->fans[0].cool_state = 0; - macsmc_hwmon_fans_apply(&t->hwmon); + platform_set_drvdata(&pdev, &t->hwmon); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_suspend(&pdev.dev), 0); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_resume(&pdev.dev), 0); + macsmc_hwmon_shutdown(&pdev); + KUNIT_EXPECT_EQ(test, t->writes, 0); fan_test_released(test, t); } -/* A fan runs at the higher of the userspace target and the cooling state. */ -static void macsmc_fan_test_cooling_and_user(struct kunit *test) +/* Suspend releases a manual target; resume never reapplies it. */ +static void macsmc_fan_test_suspend_manual(struct kunit *test) { struct macsmc_fan_test *t = test->priv; + struct device dev = {}; + int writes; + dev_set_drvdata(&dev, &t->hwmon); KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), 0); - t->fans[0].cool_state = 1; - macsmc_hwmon_fans_apply(&t->hwmon); - KUNIT_EXPECT_EQ(test, t->target[0], FAN_TEST_F2500); - - t->fans[0].cool_state = MACSMC_FAN_COOLING_STATES; - macsmc_hwmon_fans_apply(&t->hwmon); - KUNIT_EXPECT_EQ(test, t->target[0], FAN_TEST_F5777); - /* Userspace cannot take the fan below what cooling asks for. */ - KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 1350), 0); - KUNIT_EXPECT_EQ(test, t->target[0], FAN_TEST_F5777); - KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 0), 0); - KUNIT_EXPECT_EQ(test, t->mode[0], MACSMC_FAN_MODE_MANUAL); - - t->fans[0].cool_state = 0; - macsmc_hwmon_fans_apply(&t->hwmon); - fan_test_released(test, t); -} - -/* Once stopped (suspend, shutdown, removal), no state takes a fan back. */ -static void macsmc_fan_test_cooling_stopped(struct kunit *test) -{ - struct macsmc_fan_test *t = test->priv; - - t->fans[0].cool_state = 4; - t->fans[1].cool_state = 4; - macsmc_hwmon_fans_apply(&t->hwmon); - KUNIT_EXPECT_EQ(test, t->hwmon.fan.manual, 2); - - macsmc_hwmon_fans_stop(&t->hwmon); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_suspend(&dev), 0); fan_test_released(test, t); - - macsmc_hwmon_fans_apply(&t->hwmon); + writes = t->writes; + KUNIT_EXPECT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), -EBUSY); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_resume(&dev), 0); + KUNIT_EXPECT_EQ(test, t->writes, writes); fan_test_released(test, t); - KUNIT_EXPECT_EQ(test, t->ftst_writes, 2); + /* A fresh userspace request is still accepted after resume. */ + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), 0); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 0), 0); + KUNIT_EXPECT_EQ(test, t->refused, 0); } /* A failed unlock leaves the fan with the firmware. */ -static void macsmc_fan_test_cooling_unlock_fails(struct kunit *test) +static void macsmc_fan_test_unlock_fails(struct kunit *test) { struct macsmc_fan_test *t = test->priv; t->fail_key = SMC_KEY(Ftst); - t->fans[0].cool_state = MACSMC_FAN_COOLING_STATES; - macsmc_hwmon_fans_apply(&t->hwmon); - KUNIT_EXPECT_EQ(test, t->mode[0], MACSMC_FAN_MODE_SYSTEM); - KUNIT_EXPECT_EQ(test, t->hwmon.fan.manual, 0); - KUNIT_EXPECT_FALSE(test, t->hwmon.fan.unlocked); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), -EIO); + fan_test_released(test, t); + KUNIT_EXPECT_EQ(test, t->refused, 0); } static struct kunit_case macsmc_fan_test_cases[] = { @@ -430,11 +384,9 @@ static struct kunit_case macsmc_fan_test_cases[] = { KUNIT_CASE(macsmc_fan_test_two_fans), KUNIT_CASE(macsmc_fan_test_no_unlock_key), KUNIT_CASE(macsmc_fan_test_legacy), - KUNIT_CASE(macsmc_fan_test_state_rpm), - KUNIT_CASE(macsmc_fan_test_cooling), - KUNIT_CASE(macsmc_fan_test_cooling_and_user), - KUNIT_CASE(macsmc_fan_test_cooling_stopped), - KUNIT_CASE(macsmc_fan_test_cooling_unlock_fails), + KUNIT_CASE(macsmc_fan_test_untouched), + KUNIT_CASE(macsmc_fan_test_suspend_manual), + KUNIT_CASE(macsmc_fan_test_unlock_fails), {} }; diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index af67979f51213..a8e47ee4b98ff 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -36,7 +36,6 @@ #include #include #include -#include #define MAX_LABEL_LENGTH 32 @@ -88,13 +87,6 @@ struct macsmc_hwmon_fan { bool manual; /* The device tree allows a manual speed from userspace. */ bool manual_control; - /* Index of the fan node among the fan-* nodes: its cooling device id. */ - u32 index; - struct macsmc_hwmon *hwmon; - /* Speed asked for through the fan_control target, 0 for none. */ - long user_rpm; - /* Cooling state asked for by the thermal framework, 0 for none. */ - unsigned long cool_state; }; struct macsmc_hwmon_sensors { @@ -109,13 +101,6 @@ struct macsmc_hwmon_sensors { /* The firmware's own fan policy; it refuses a manual mode request. */ #define MACSMC_FAN_MODE_SYSTEM 3 -/* - * Cooling states of a fan with #cooling-cells: 0 leaves the fan to the - * firmware, 1 to MACSMC_FAN_COOLING_STATES run it at speeds spread evenly - * from its minimum to its maximum. - */ -#define MACSMC_FAN_COOLING_STATES 8 - /* How long the firmware gets to leave or re-enter its system mode */ #define MACSMC_FAN_MODE_TIMEOUT_MS 10000 #define MACSMC_FAN_MODE_POLL_MS 100 @@ -134,8 +119,6 @@ struct macsmc_hwmon_fans { bool unlocked; /* Fans in manual mode */ u32 manual; - /* Applies the cooling states, which can take seconds to unlock. */ - struct work_struct cool_work; /* Suspended, shut down or going away: every fan stays with the SMC. */ bool stopped; }; @@ -497,99 +480,52 @@ static int macsmc_hwmon_fan_manual(struct macsmc_hwmon *hwmon, if (fan->manual) return 0; - ret = macsmc_hwmon_write_key(hwmon->smc, &fan->mode, MACSMC_FAN_MODE_MANUAL); - if (ret < 0 && hwmon->fan.unlock_key && !hwmon->fan.unlocked && - !macsmc_hwmon_fan_read_mode(hwmon, fan, &mode) && - mode == MACSMC_FAN_MODE_SYSTEM) { - ret = apple_smc_write_u8(hwmon->smc, hwmon->fan.unlock_key, 1); - if (ret < 0) { - /* The key may have been set anyway; clear it. */ - apple_smc_write_u8(hwmon->smc, hwmon->fan.unlock_key, 0); - return ret; + /* Do not send a mode write the firmware is known to refuse. */ + ret = macsmc_hwmon_fan_read_mode(hwmon, fan, &mode); + if (ret) + return ret; + + if (mode == MACSMC_FAN_MODE_SYSTEM) { + if (!hwmon->fan.unlock_key) + return -EOPNOTSUPP; + + if (!hwmon->fan.unlocked) { + ret = apple_smc_write_u8(hwmon->smc, hwmon->fan.unlock_key, 1); + if (ret < 0) { + /* The key may have been set anyway; clear it. */ + apple_smc_write_u8(hwmon->smc, hwmon->fan.unlock_key, 0); + return ret; + } + hwmon->fan.unlocked = true; } - hwmon->fan.unlocked = true; - /* Ask again once the fan has left system mode. */ timeout = jiffies + msecs_to_jiffies(MACSMC_FAN_MODE_TIMEOUT_MS); for (;;) { ret = macsmc_hwmon_fan_read_mode(hwmon, fan, &mode); - if (!ret && mode == MACSMC_FAN_MODE_SYSTEM) - ret = -EBUSY; - else if (!ret) - ret = macsmc_hwmon_write_key(hwmon->smc, &fan->mode, - MACSMC_FAN_MODE_MANUAL); - if (ret >= 0 || time_after(jiffies, timeout)) + if (ret || mode != MACSMC_FAN_MODE_SYSTEM) + break; + if (time_after(jiffies, timeout)) { + ret = -ETIMEDOUT; break; + } msleep(MACSMC_FAN_MODE_POLL_MS); } - if (ret < 0) { - dev_err(hwmon->dev, "%s refused manual mode after unlock (%d)\n", - fan->label, ret); - if (!hwmon->fan.manual) - macsmc_hwmon_fans_relock(hwmon); - return ret; - } + if (ret) + goto relock; } + + ret = macsmc_hwmon_write_key(hwmon->smc, &fan->mode, MACSMC_FAN_MODE_MANUAL); if (ret < 0) - return ret; + goto relock; fan->manual = true; hwmon->fan.manual++; return 0; -} - -/* The speed of a cooling state of a fan running from @min to @max. */ -static long macsmc_hwmon_fan_state_rpm(long min, long max, unsigned long state) -{ - if (!state) - return 0; - state = min(state, MACSMC_FAN_COOLING_STATES); - return min + DIV_ROUND_CLOSEST((max - min) * (long)(state - 1), - MACSMC_FAN_COOLING_STATES - 1); -} - -/* - * Run a fan at the higher of the userspace target and the speed of its - * cooling state, or hand it to the firmware when neither asks for anything. - * Any failure leaves the fan with the firmware. Called with the fan lock held. - */ -static int macsmc_hwmon_fan_apply(struct macsmc_hwmon *hwmon, - struct macsmc_hwmon_fan *fan) -{ - unsigned long state = READ_ONCE(fan->cool_state); - long min, max, rpm; - int ret; - if (hwmon->fan.stopped || (!fan->user_rpm && !state)) - return macsmc_hwmon_fan_auto(hwmon, fan); - - /* - * The SMC does no sanity checks on requested fan speeds, so we need to. - */ - ret = macsmc_hwmon_read_key(hwmon->smc, &fan->min, 1, &min); - if (!ret) - ret = macsmc_hwmon_read_key(hwmon->smc, &fan->max, 1, &max); - if (!ret && (min <= 0 || max < min)) - ret = -EIO; - if (ret) { - macsmc_hwmon_fan_auto(hwmon, fan); - return ret; - } - - rpm = max(fan->user_rpm, macsmc_hwmon_fan_state_rpm(min, max, state)); - rpm = clamp(rpm, min, max); - - ret = macsmc_hwmon_fan_manual(hwmon, fan); - if (ret) - return ret; - - ret = macsmc_hwmon_write_key(hwmon->smc, &fan->set, rpm); - if (ret < 0) { - macsmc_hwmon_fan_auto(hwmon, fan); - return ret; - } - - return 0; +relock: + if (hwmon->fan.unlocked && !hwmon->fan.manual) + macsmc_hwmon_fans_relock(hwmon); + return ret; } /* Whether userspace may set this fan's speed. */ @@ -601,6 +537,7 @@ static bool macsmc_hwmon_fan_user_control(const struct macsmc_hwmon_fan *fan) static int macsmc_hwmon_set_fan(struct macsmc_hwmon *hwmon, int channel, long val) { struct macsmc_hwmon_fan *fan = &hwmon->fan.fans[channel]; + long min, max; int ret; if (!macsmc_hwmon_fan_user_control(fan)) @@ -610,11 +547,25 @@ static int macsmc_hwmon_set_fan(struct macsmc_hwmon *hwmon, int channel, long va guard(mutex)(&hwmon->fan.lock); - /* 0 drops the request; the fan goes back to the firmware unless cooling. */ - fan->user_rpm = val; - ret = macsmc_hwmon_fan_apply(hwmon, fan); - if (ret) - fan->user_rpm = 0; + if (!val) + return macsmc_hwmon_fan_auto(hwmon, fan); + if (hwmon->fan.stopped) + return -EBUSY; + + /* The SMC does no sanity checks on requested fan speeds. */ + ret = macsmc_hwmon_read_key(hwmon->smc, &fan->min, 1, &min); + if (!ret) + ret = macsmc_hwmon_read_key(hwmon->smc, &fan->max, 1, &max); + if (!ret && (min <= 0 || max < min)) + ret = -EIO; + if (!ret) + ret = macsmc_hwmon_fan_manual(hwmon, fan); + if (!ret) { + ret = macsmc_hwmon_write_key(hwmon->smc, &fan->set, clamp(val, min, max)); + if (ret >= 0) + return 0; + } + macsmc_hwmon_fan_auto(hwmon, fan); return ret; } @@ -635,129 +586,15 @@ static void macsmc_hwmon_fans_restore(struct macsmc_hwmon *hwmon) macsmc_hwmon_fan_auto(hwmon, &hwmon->fan.fans[i]); } -/* Bring every fan in line with its requests, unless stopped. */ -static void macsmc_hwmon_fans_apply(struct macsmc_hwmon *hwmon) -{ - u32 i; - int ret; - - guard(mutex)(&hwmon->fan.lock); - for (i = 0; i < hwmon->fan.count; i++) { - struct macsmc_hwmon_fan *fan = &hwmon->fan.fans[i]; - - ret = macsmc_hwmon_fan_apply(hwmon, fan); - if (ret) - dev_warn_ratelimited(hwmon->dev, - "%s left to the SMC: cannot run cooling state %lu (%d)\n", - fan->label, READ_ONCE(fan->cool_state), ret); - } -} - -static void macsmc_hwmon_fans_cool_work(struct work_struct *work) +static void macsmc_hwmon_fans_restore_action(void *data) { - macsmc_hwmon_fans_apply(container_of(work, struct macsmc_hwmon, fan.cool_work)); -} + struct macsmc_hwmon *hwmon = data; -/* - * Stop applying cooling states and hand every fan back to the firmware, for - * suspend, shutdown and removal. - */ -static void macsmc_hwmon_fans_stop(struct macsmc_hwmon *hwmon) -{ scoped_guard(mutex, &hwmon->fan.lock) hwmon->fan.stopped = true; - cancel_work_sync(&hwmon->fan.cool_work); macsmc_hwmon_fans_restore(hwmon); } -static void macsmc_hwmon_fans_restore_action(void *data) -{ - macsmc_hwmon_fans_stop(data); -} - -#if IS_ENABLED(CONFIG_THERMAL_OF) -static int macsmc_hwmon_fan_get_max_state(struct thermal_cooling_device *cdev, - unsigned long *state) -{ - *state = MACSMC_FAN_COOLING_STATES; - return 0; -} - -static int macsmc_hwmon_fan_get_cur_state(struct thermal_cooling_device *cdev, - unsigned long *state) -{ - struct macsmc_hwmon_fan *fan = cdev->devdata; - - *state = READ_ONCE(fan->cool_state); - return 0; -} - -/* - * Unlocking the fans takes the firmware seconds, so the state is applied - * from a work item rather than from the thermal zone update. - */ -static int macsmc_hwmon_fan_set_cur_state(struct thermal_cooling_device *cdev, - unsigned long state) -{ - struct macsmc_hwmon_fan *fan = cdev->devdata; - - if (state > MACSMC_FAN_COOLING_STATES) - return -EINVAL; - - WRITE_ONCE(fan->cool_state, state); - queue_work(system_freezable_wq, &fan->hwmon->fan.cool_work); - return 0; -} - -static const struct thermal_cooling_device_ops macsmc_hwmon_fan_cooling_ops = { - .get_max_state = macsmc_hwmon_fan_get_max_state, - .get_cur_state = macsmc_hwmon_fan_get_cur_state, - .set_cur_state = macsmc_hwmon_fan_set_cur_state, -}; - -/* - * With #cooling-cells = <3> on the hwmon node, every fan is a cooling device - * for the thermal zones that map it as <&hwmon fan-index lower upper>, the - * index counting the fan-* nodes in order. Without it the fans stay with the - * firmware as before. - */ -static void macsmc_hwmon_fans_register_cooling(struct macsmc_hwmon *hwmon) -{ - const u32 needed = HWMON_F_MIN | HWMON_F_MAX | HWMON_F_TARGET; - u32 cells, i; - - if (!hwmon->fan.count || - of_property_read_u32(hwmon->dev->of_node, "#cooling-cells", &cells)) - return; - if (cells != 3) { - dev_warn(hwmon->dev, "#cooling-cells must be 3; fans left to the SMC\n"); - return; - } - - for (i = 0; i < hwmon->fan.count; i++) { - struct macsmc_hwmon_fan *fan = &hwmon->fan.fans[i]; - struct thermal_cooling_device *cdev; - - if ((fan->attrs & needed) != needed || !fan->mode.macsmc_key) { - dev_warn(hwmon->dev, "%s has no speed or mode keys; not a cooling device\n", - fan->label); - continue; - } - - cdev = devm_thermal_of_cooling_device_register(hwmon->dev, fan->index, - "macsmc-fan", fan, - &macsmc_hwmon_fan_cooling_ops); - if (IS_ERR(cdev)) - dev_warn(hwmon->dev, "%s: cannot register a cooling device (%pe)\n", - fan->label, cdev); - } -} -#else -static void macsmc_hwmon_fans_register_cooling(struct macsmc_hwmon *hwmon) -{ -} -#endif - static int macsmc_hwmon_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long *val) { @@ -1208,8 +1045,6 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, for_each_child_of_node_with_prefix(hwmon_node, key_node, "fan-") { struct macsmc_hwmon_fan *fan = &hwmon->fan.fans[hwmon->fan.count]; - fan->hwmon = hwmon; - fan->index = n_fan++; if (!macsmc_hwmon_create_fan(hwmon->dev, hwmon->smc, key_node, fan)) hwmon->fan.count++; } @@ -1603,7 +1438,6 @@ static int macsmc_hwmon_probe(struct platform_device *pdev) ret = devm_mutex_init(&pdev->dev, &hwmon->fan.lock); if (ret) return ret; - INIT_WORK(&hwmon->fan.cool_work, macsmc_hwmon_fans_cool_work); ret = macsmc_hwmon_populate_sensors(hwmon, hwmon->dev->of_node); if (ret) { @@ -1665,12 +1499,6 @@ static int macsmc_hwmon_probe(struct platform_device *pdev) if (ret) return ret; - /* - * Released before the restore action above, which stops the cooling - * work and hands the fans back. - */ - macsmc_hwmon_fans_register_cooling(hwmon); - dev_dbg(hwmon->dev, "Registered SMC hwmon device. Sensors:\n"); dev_dbg(hwmon->dev, "Current: %d, Fans: %d, Power: %d, Temperature: %d, Voltage: %d", @@ -1692,27 +1520,22 @@ static void macsmc_hwmon_shutdown(struct platform_device *pdev) struct macsmc_hwmon *hwmon = platform_get_drvdata(pdev); if (hwmon) - macsmc_hwmon_fans_stop(hwmon); + macsmc_hwmon_fans_restore_action(hwmon); } static int macsmc_hwmon_suspend(struct device *dev) { - macsmc_hwmon_fans_stop(dev_get_drvdata(dev)); + macsmc_hwmon_fans_restore_action(dev_get_drvdata(dev)); return 0; } -/* Cooling states asked for before suspend apply again; the zones update them. */ +/* Manual requests end at suspend; resume never writes fan keys. */ static int macsmc_hwmon_resume(struct device *dev) { struct macsmc_hwmon *hwmon = dev_get_drvdata(dev); - scoped_guard(mutex, &hwmon->fan.lock) { - hwmon->fan.stopped = false; - /* The userspace targets were dropped when the fans went back. */ - for (u32 i = 0; i < hwmon->fan.count; i++) - hwmon->fan.fans[i].user_rpm = 0; - } - queue_work(system_freezable_wq, &hwmon->fan.cool_work); + guard(mutex)(&hwmon->fan.lock); + hwmon->fan.stopped = false; return 0; } From b8e84d08eef58535786a4ae9ca695d577d1439b4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 15:49:50 +1100 Subject: [PATCH 1464/1579] power: supply: macsmc: Preserve firmware charge settings at probe Probing a battery should not reset its persistent charge policy. Stop resetting the charging inhibitors and undocumented CH0K/CH0B keys merely because the driver bound. The earlier CHLS repair cleared a force-discharge bit left without a limit by old kernels. Warn about that state without changing it; an explicit charge_behaviour=auto request still clears it while preserving the limit byte. Keep the existing serialization and explicit charge controls. Update the KUnit coverage to verify read-only observation and explicit recovery. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/power/supply/macsmc-power-test.c | 39 ++++++++---------------- drivers/power/supply/macsmc-power.c | 37 +++------------------- 2 files changed, 17 insertions(+), 59 deletions(-) diff --git a/drivers/power/supply/macsmc-power-test.c b/drivers/power/supply/macsmc-power-test.c index 4a86c9bd9f155..b6f9b50173d01 100644 --- a/drivers/power/supply/macsmc-power-test.c +++ b/drivers/power/supply/macsmc-power-test.c @@ -637,41 +637,28 @@ static void macsmc_charge_serialized_test(struct kunit *test) KUNIT_EXPECT_EQ(test, t->chls, 0x013c); } -/* Probe clears force-discharge left without a limit, and only that. */ +/* A stale force-discharge bit is cleared only by an explicit auto request. */ static void macsmc_charge_clear_stray_discharge_test(struct kunit *test) { - static const struct { - u16 before, after; - unsigned int writes; - } cases[] = { - { 0x0100, 0x0000, 1 }, - { 0x0164, 0x0064, 1 }, - { 0x0105, 0x0005, 1 }, - { 0x0150, 0x0150, 0 }, /* auto-discharge with a limit stays */ - { 0x0050, 0x0050, 0 }, - { 0x0000, 0x0000, 0 }, - }; + static const u16 values[] = { 0x0100, 0x0164, 0x0105, 0x0150, 0x0050, 0x0000 }; struct macsmc_charge_test *t = test->priv; unsigned int i; t->power.has_chls = true; - for (i = 0; i < ARRAY_SIZE(cases); i++) { - t->chls = cases[i].before; + for (i = 0; i < ARRAY_SIZE(values); i++) { + t->chls = values[i]; t->nops = 0; - KUNIT_EXPECT_EQ_MSG(test, macsmc_battery_clear_stray_discharge(&t->power), 0, + macsmc_battery_get_charge_behaviour(&t->power); + KUNIT_EXPECT_EQ_MSG(test, t->chls, values[i], "read case %u", i); + KUNIT_EXPECT_EQ(test, charge_test_writes(t), 0U); + KUNIT_EXPECT_EQ_MSG(test, + charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO), + 0, "case %u", i); + KUNIT_EXPECT_EQ_MSG(test, t->chls, values[i] & ~CHLS_FORCE_DISCHARGE, "case %u", i); - KUNIT_EXPECT_EQ_MSG(test, t->chls, cases[i].after, "case %u", i); - KUNIT_EXPECT_EQ_MSG(test, charge_test_writes(t), cases[i].writes, "case %u", i); + KUNIT_EXPECT_EQ_MSG(test, charge_test_writes(t), 1U, "case %u", i); } - - t->chls = 0x0100; - t->fail_key = SMC_KEY(CHLS); - t->fail_err = -ETIMEDOUT; - KUNIT_EXPECT_EQ(test, macsmc_battery_clear_stray_discharge(&t->power), -ETIMEDOUT); - t->fail_write = true; - t->fail_err = -EIO; - KUNIT_EXPECT_EQ(test, macsmc_battery_clear_stray_discharge(&t->power), -EIO); - KUNIT_EXPECT_EQ(test, t->chls, 0x0100); } static struct kunit_case macsmc_charge_test_cases[] = { diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index 36d59e5ea315a..1bc23703654a4 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -786,25 +786,6 @@ static int macsmc_battery_set_property(struct power_supply *psy, return ret < 0 ? ret : 0; } -/* - * An earlier kernel could leave force-discharge set in CHLS without a charge limit, which this - * driver never asks of the SMC. Clear the bit once at probe; the limit byte is kept. - */ -static int macsmc_battery_clear_stray_discharge(struct macsmc_power *power) -{ - u16 chls; - int ret; - - guard(mutex)(&power->charge_lock); - ret = apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &chls); - if (ret) - return ret; - if (!(chls & CHLS_FORCE_DISCHARGE) || macsmc_chls_has_limit(chls)) - return 0; - ret = apple_smc_write_u16(power->smc, SMC_KEY(CHLS), chls & ~CHLS_FORCE_DISCHARGE); - return ret < 0 ? ret : 0; -} - static int macsmc_battery_property_is_writeable(struct power_supply *psy, enum power_supply_property psp) { @@ -1108,26 +1089,16 @@ static int macsmc_power_probe(struct platform_device *pdev) of_machine_is_compatible("apple,j700"); power->status_from_current = of_machine_is_compatible("apple,j700"); - /* Reset "Optimised Battery Charging" flags to default state */ - if (power->has_chte) - apple_smc_write_u32(smc, SMC_KEY(CHTE), 0); - else if (power->has_ch0c) - apple_smc_write_u8(smc, SMC_KEY(CH0C), 0); - - if (power->has_ch0i) - apple_smc_write_u8(smc, SMC_KEY(CH0I), 0); - - apple_smc_write_u8(smc, SMC_KEY(CH0K), 0); - apple_smc_write_u8(smc, SMC_KEY(CH0B), 0); - + /* Preserve firmware charge policy until userspace changes it. */ /* Detect charge limit method (CHWA vs CHLS) */ if (apple_smc_read_flag(power->smc, SMC_KEY(CHWA), &flag) == 0) power->has_chwa = true; else if (apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16) >= 0) power->has_chls = true; - if (power->has_chls && macsmc_battery_clear_stray_discharge(power)) - dev_warn(dev, "Failed to clear force-discharge without a charge limit\n"); + if (power->has_chls && (vu16 & CHLS_FORCE_DISCHARGE) && + !macsmc_chls_has_limit(vu16)) + dev_warn(dev, "CHLS force-discharge without a limit; select charge_behaviour=auto to clear\n"); /* * CHLS carries auto-discharge, so a CHLS-only Mac can select auto From 03ce684ab51ace54241c96463bc0674a6339c5e1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 15:49:50 +1100 Subject: [PATCH 1465/1579] arm64: dts: apple: Use firmware fan policy and expose thermal pressure Both firmware key inventories include the readable signed 32-bit mTPL key. Select it through the existing thermal-pressure property so the shared driver exposes its read-only level. On J616s macOS polls this key once per second; it is distinct from the temperature keys. Remove the J616s host active fan trips and maps. The SMC ramps the fans under sustained load without host key writes, so the earlier assumption behind the Linux fan ladder does not hold. Preserve every temperature channel, passive CPU/GPU cooling map, critical trip, and explicit manual fan control property. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 66 ++--------------------- arch/arm64/boot/dts/apple/t8140-j700.dts | 2 + 2 files changed, 6 insertions(+), 62 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 69f0070af7659..79ed4de3c98be 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -322,41 +322,9 @@ * GPU is powered off its sensors report placeholder values from -4 to * 4 C. * - * The fans: left to themselves (SMC mode 3, with no OS asking for - * cooling) they stay off, or switch on late and ramp slowly, while the - * dies heat up under load. So the CPU die zone and the GPU zones also - * run both fans through a ladder of active trips; a fan takes the - * highest state any zone asks for and goes back to the SMC once no - * zone asks for anything. States 1-8 of a fan run it from its minimum - * (1350 rpm) to its maximum (5777 rpm) in even steps. + * Fan speeds are governed by the SMC firmware. Manual speeds are + * available only through an explicit userspace fan target request. */ -#define J616S_FAN_TRIP(label, node, temp, hyst) \ - label: node { \ - temperature = ; \ - hysteresis = ; \ - type = "active"; \ - }; - -#define J616S_FAN_TRIPS(zone) \ - J616S_FAN_TRIP(zone##_fan1, trip-fan1, 75000, 10000) \ - J616S_FAN_TRIP(zone##_fan2, trip-fan2, 80000, 5000) \ - J616S_FAN_TRIP(zone##_fan3, trip-fan3, 85000, 5000) \ - J616S_FAN_TRIP(zone##_fan4, trip-fan4, 90000, 5000) \ - J616S_FAN_TRIP(zone##_fan5, trip-fan5, 95000, 5000) - -#define J616S_FAN_MAP(trip_label, node, state) \ - node { \ - trip = <&trip_label>; \ - cooling-device = <&smc_hwmon 0 state state>, \ - <&smc_hwmon 1 state state>; \ - }; - -#define J616S_FAN_MAPS(zone) \ - J616S_FAN_MAP(zone##_fan1, map-fan1, 1) \ - J616S_FAN_MAP(zone##_fan2, map-fan2, 2) \ - J616S_FAN_MAP(zone##_fan3, map-fan3, 3) \ - J616S_FAN_MAP(zone##_fan4, map-fan4, 5) \ - J616S_FAN_MAP(zone##_fan5, map-fan5, 8) thermal-zones { cpu-die-thermal { @@ -365,7 +333,6 @@ thermal-sensors = <&smc_hwmon 0>; /* TCMz */ trips { - J616S_FAN_TRIPS(cpu_die) cpu_die_passive: cpu-die-passive { temperature = <113000>; @@ -381,7 +348,6 @@ }; cooling-maps { - J616S_FAN_MAPS(cpu_die) map-e { trip = <&cpu_die_passive>; @@ -928,7 +894,6 @@ thermal-sensors = <&smc_hwmon 17>; /* Tg05 */ trips { - J616S_FAN_TRIPS(gpu0) gpu0_passive: trip-passive { temperature = <101000>; @@ -944,7 +909,6 @@ }; cooling-maps { - J616S_FAN_MAPS(gpu0) map-e { trip = <&gpu0_passive>; @@ -983,7 +947,6 @@ thermal-sensors = <&smc_hwmon 18>; /* Tg0S */ trips { - J616S_FAN_TRIPS(gpu1) gpu1_passive: trip-passive { temperature = <101000>; @@ -999,7 +962,6 @@ }; cooling-maps { - J616S_FAN_MAPS(gpu1) map-e { trip = <&gpu1_passive>; @@ -1038,7 +1000,6 @@ thermal-sensors = <&smc_hwmon 19>; /* Tg0Y */ trips { - J616S_FAN_TRIPS(gpu2) gpu2_passive: trip-passive { temperature = <101000>; @@ -1054,7 +1015,6 @@ }; cooling-maps { - J616S_FAN_MAPS(gpu2) map-e { trip = <&gpu2_passive>; @@ -1093,7 +1053,6 @@ thermal-sensors = <&smc_hwmon 20>; /* Tg0k */ trips { - J616S_FAN_TRIPS(gpu3) gpu3_passive: trip-passive { temperature = <101000>; @@ -1109,7 +1068,6 @@ }; cooling-maps { - J616S_FAN_MAPS(gpu3) map-e { trip = <&gpu3_passive>; @@ -1148,7 +1106,6 @@ thermal-sensors = <&smc_hwmon 21>; /* Tg0z */ trips { - J616S_FAN_TRIPS(gpu4) gpu4_passive: trip-passive { temperature = <101000>; @@ -1164,7 +1121,6 @@ }; cooling-maps { - J616S_FAN_MAPS(gpu4) map-e { trip = <&gpu4_passive>; @@ -1203,7 +1159,6 @@ thermal-sensors = <&smc_hwmon 22>; /* Tg1V */ trips { - J616S_FAN_TRIPS(gpu5) gpu5_passive: trip-passive { temperature = <101000>; @@ -1219,7 +1174,6 @@ }; cooling-maps { - J616S_FAN_MAPS(gpu5) map-e { trip = <&gpu5_passive>; @@ -1258,7 +1212,6 @@ thermal-sensors = <&smc_hwmon 23>; /* Tg1l */ trips { - J616S_FAN_TRIPS(gpu6) gpu6_passive: trip-passive { temperature = <101000>; @@ -1274,7 +1227,6 @@ }; cooling-maps { - J616S_FAN_MAPS(gpu6) map-e { trip = <&gpu6_passive>; @@ -3851,23 +3803,13 @@ }; }; -/* - * Fan speeds, limits, targets and modes, and the common SMC sensors; every - * key exists on this machine's SMC. Both fans list their target (F0Tg, - * F1Tg) and mode (F0Md, F1Md) keys, so with the driver's fan_control - * parameter fanN_target becomes writable; by default the attributes are - * read-only, and the thermal zones above drive the fans. - */ +/* Fan monitoring and explicit userspace manual control. */ #include "hwmon-common.dtsi" #include "hwmon-fan-dual.dtsi" #include "hwmon-laptop.dtsi" -/* - * Both fans are cooling devices of the CPU die and GPU zones, as - * <&smc_hwmon fan-index lower upper>. - */ &smc_hwmon { - #cooling-cells = <3>; + apple,thermal-pressure-key = "mTPL"; /* * The SMC accepts a manual speed for both fans once its fan test diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 043c6b5d085fb..a3044890100e2 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -625,6 +625,8 @@ }; &smc_hwmon { + apple,thermal-pressure-key = "mTPL"; + /* * Further channels, reported to userspace. They follow the six CPU * die temperatures in t8140.dtsi, which the thermal zones above From 57ceb18a8c45508fda3e2ce6e55f7e41b99b7e26 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 15:53:28 +1100 Subject: [PATCH 1466/1579] drm/apple: decode the return status of gamma table RPCs The gamma RPC request ends with an input pointer null flag, but the response ends with a 32-bit return status. Treating that status as a one-byte null flag misses failures with a zero low byte and diagnoses an empty failed external read as a malformed calibration table. Check the full status before validating the table, and copy only table data into the request base. Report the external empty status-2 reply at debug level. Keep warnings for internal panels, malformed successful tables and other failures. Add debug results for A420 and A421 so a lit panel trial can distinguish a failed RPC from unusable table contents. Exercise status admission, table validation, independent RGB composition and base restoration through the production callbacks in host selftests. This does not invent an identity base: neither its wire encoding nor external A421 support has been established. A failed read still leaves GAMMA_LUT unavailable and needs further protocol qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.h | 6 +- drivers/gpu/drm/apple/iomfb_template.c | 21 ++- .../selftests/drivers/gpu/apple-dcp/README | 6 + .../drivers/gpu/apple-dcp/desktop.py | 154 ++++++++++++++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 1 + 5 files changed, 181 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 7c60a81207ad7..7fb06f8d5eea2 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -476,7 +476,8 @@ struct dcp_bool_arg { /* * A420 get_gamma_table / A421 set_gamma_table: three channels (red, green, - * blue) of 257 u32 entries followed by a null flag, 3088 bytes in and out. + * blue) of 257 u32 entries. The request appends an input pointer null flag; + * the reply appends a return status instead. Both are 3088 bytes. */ #define IOMFB_GAMMA_ENTRIES 257 @@ -488,8 +489,7 @@ struct iomfb_gamma_table_req { struct iomfb_gamma_table_resp { __le32 table[3][IOMFB_GAMMA_ENTRIES]; - u8 table_null; - u8 padding[3]; + __le32 ret; } __packed; #endif diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index b649590127626..4e218bf664b4f 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2875,6 +2875,11 @@ static u32 dcp_gamma_lookup(const __le32 *fw, u16 v) static void dcp_gamma_set_done(struct apple_dcp *dcp, void *out, void *cookie) { + struct iomfb_gamma_table_resp *resp = out; + + if (resp) + dev_dbg(dcp->dev, "A421 gamma table status %u\n", + le32_to_cpu(resp->ret)); do_swap(dcp, NULL, NULL); } @@ -2905,23 +2910,31 @@ static void dcp_gamma_read(struct apple_dcp *dcp, void *out, void *cookie) u32 ch, i; /* A table to compose with rises from its first to its last entry. */ - for (ch = 0; resp && ch < 3; ch++) { + for (ch = 0; resp && !le32_to_cpu(resp->ret) && ch < 3; ch++) { for (i = 1; i < IOMFB_GAMMA_ENTRIES; i++) if (le32_to_cpu(resp->table[ch][i]) < le32_to_cpu(resp->table[ch][i - 1])) break; - if (resp->table_null || i < IOMFB_GAMMA_ENTRIES || + if (i < IOMFB_GAMMA_ENTRIES || !le32_to_cpu(resp->table[ch][IOMFB_GAMMA_ENTRIES - 1])) break; } if (!resp || ch < 3) { - dev_warn(dcp->dev, "unusable firmware gamma table, GAMMA_LUT ignored\n"); + /* External firmware can return no table with status 2. */ + if (resp && !dcp_has_panel(dcp) && le32_to_cpu(resp->ret) == 2 && + !memchr_inv(resp->table, 0, sizeof(resp->table))) + dev_dbg(dcp->dev, + "A420 returned no gamma table (status 2), GAMMA_LUT ignored\n"); + else + dev_warn(dcp->dev, "unusable firmware gamma table, GAMMA_LUT ignored\n"); d->gamma_unusable = true; do_swap(dcp, NULL, NULL); return; } - memcpy(d->gamma_fw, resp, sizeof(*d->gamma_fw)); + memset(d->gamma_fw, 0, sizeof(*d->gamma_fw)); + memcpy(d->gamma_fw->table, resp->table, sizeof(resp->table)); d->gamma_fw_read = true; + dev_dbg(dcp->dev, "A420 firmware gamma table read\n"); dcp_gamma_set(dcp); } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index 6777d85cf3431..7d65d5f848e34 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -51,6 +51,12 @@ concurrency and system suspend require separate kernel/hardware qualification. Kernel KUnit tests and apple-afkv2 remain separate tests in their existing standard layouts. No archived diagnostic script is a dependency of this suite. +The display group's desktop-gamma fixture executes the production A420/A421 +callbacks and composition with synthetic tables. It checks full return-status +handling, the external empty status-2 reply, internal-panel diagnostics, +malformed tables, independent RGB interpolation and restoration of the base. +It does not establish firmware table encoding or external A421 support. + The runner preserves host artifacts on failure as well as success. Existing physical-disconnect fixture warnings for unused mock boundaries and signed host/kernel ABI comparisons are handled only by those fixtures' original diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py b/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py index 8d6a44988faa3..b7f4f7dd633dc 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py @@ -942,3 +942,157 @@ def identity_trace_budget(root): return 0; } ''' + + +def gamma(root): + base = root / 'drivers/gpu/drm/apple' + header = (base / 'iomfb.h').read_text() + template = (base / 'iomfb_template.c').read_text() + prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32, __le32; +typedef int64_t s64; +#define __packed __attribute__((packed)) +#define le32_to_cpu(v) (v) +#define cpu_to_le32(v) (v) +#define div_s64(a, b) ((a) / (b)) +''' + prefix += next(line for line in header.splitlines() + if line.startswith('#define IOMFB_GAMMA_ENTRIES')) + '\n' + for name in ('req', 'resp'): + prefix += block(header, 'struct iomfb_gamma_table_' + name + ' {') + ';\n' + prefix += r''' +struct drm_color_lut { u16 red, green, blue, reserved; }; +struct dcp_desktop_state { + struct iomfb_gamma_table_req *gamma_fw, *gamma_out; + struct drm_color_lut gamma_lut[IOMFB_GAMMA_ENTRIES]; + bool gamma_user, gamma_fw_read, gamma_unusable; +}; +struct apple_dcp { struct dcp_desktop_state *desktop; bool panel; void *dev; }; +static unsigned warnings, debug, swaps, writes; +static bool dcp_has_panel(struct apple_dcp *dcp) { return dcp->panel; } +static void *memchr_inv(const void *p, int c, size_t n) +{ + const unsigned char *bytes = p; + for (size_t i = 0; i < n; i++) + if (bytes[i] != c) return (void *)(bytes + i); + return NULL; +} +#define dev_warn(dev, ...) ((void)(dev), warnings++) +#define dev_dbg(dev, ...) ((void)(dev), debug++) +static void do_swap(struct apple_dcp *dcp, void *out, void *cookie) +{ assert(dcp && !out && !cookie); swaps++; } +static const int dcp_set_gamma_table_method; +static struct iomfb_gamma_table_req sent; +static void dcp_push(struct apple_dcp *dcp, bool oob, const int *method, + size_t in_size, size_t out_size, void *in, + void (*cb)(struct apple_dcp *, void *, void *), void *cookie) +{ + assert(!oob && method == &dcp_set_gamma_table_method); + assert(in_size == sizeof(sent)); + assert(out_size == sizeof(struct iomfb_gamma_table_resp)); + struct iomfb_gamma_table_resp reply = {0}; + memcpy(&sent, in, sizeof(sent)); + writes++; + cb(dcp, &reply, cookie); +} +''' + production = '\n'.join(block(template, 'static ' + signature) for signature in ( + 'u32 dcp_gamma_lookup(', 'void dcp_gamma_set_done(', + 'void dcp_gamma_set(', 'void dcp_gamma_read(')) + suffix = r''' +static struct iomfb_gamma_table_req fw, output; +static struct dcp_desktop_state state; +static struct apple_dcp dcp; +static struct iomfb_gamma_table_resp reply; +static void reset(bool panel) +{ + memset(&fw, 0xa5, sizeof(fw)); + memset(&output, 0, sizeof(output)); + memset(&state, 0, sizeof(state)); + memset(&reply, 0, sizeof(reply)); + state.gamma_fw = &fw; + state.gamma_out = &output; + dcp = (struct apple_dcp){ .desktop = &state, .panel = panel }; + warnings = debug = swaps = writes = 0; +} +static void increasing(void) +{ + /* Arbitrary synthetic scales: no assertion about the firmware encoding. */ + for (unsigned ch = 0; ch < 3; ch++) + for (unsigned i = 0; i < IOMFB_GAMMA_ENTRIES; i++) + reply.table[ch][i] = (ch + 1) * i * i; +} +static void rejected(bool warn) +{ + dcp_gamma_read(&dcp, &reply, NULL); + assert(state.gamma_unusable && !state.gamma_fw_read); + assert(warnings == warn && swaps == 1 && writes == 0); + assert(debug == !warn); + /* Failure must not turn the reply into a calibration base. */ + const unsigned char *bytes = (const void *)&fw; + for (size_t i = 0; i < sizeof(fw); i++) assert(bytes[i] == 0xa5); +} +int main(void) +{ + _Static_assert(sizeof(reply) == 3088, "gamma response size"); + _Static_assert(offsetof(struct iomfb_gamma_table_resp, ret) == 3084, + "gamma return status offset"); + for (unsigned panel = 0; panel < 2; panel++) { + reset(panel); reply.ret = 2; rejected(panel); + reset(panel); reply.ret = 1; rejected(true); + reset(panel); reply.ret = 0x100; increasing(); rejected(true); + reset(panel); reply.ret = 2; increasing(); rejected(true); + reset(panel); rejected(true); /* Successful but empty table. */ + for (unsigned ch = 0; ch < 3; ch++) { + reset(panel); increasing(); + reply.table[ch][10] = reply.table[ch][9] - 1; + rejected(true); + reset(panel); increasing(); + memset(reply.table[ch], 0, sizeof(reply.table[ch])); + rejected(true); + } + reset(panel); + dcp_gamma_read(&dcp, NULL, NULL); + assert(state.gamma_unusable && !state.gamma_fw_read); + assert(warnings == 1 && swaps == 1 && writes == 0); + + reset(panel); increasing(); + dcp_gamma_read(&dcp, &reply, NULL); + assert(state.gamma_fw_read && !state.gamma_unusable); + assert(!warnings && writes == 1 && swaps == 1); + assert(!sent.table_null); + assert(!memcmp(sent.table, reply.table, sizeof(sent.table))); + assert(!sent.padding[0] && !sent.padding[1] && !sent.padding[2]); + + /* Nonlinear, independent RGB channels; endpoints and interpolation. */ + state.gamma_user = true; + for (unsigned i = 0; i < IOMFB_GAMMA_ENTRIES; i++) { + state.gamma_lut[i].red = 0; + state.gamma_lut[i].green = 0x7fff; + state.gamma_lut[i].blue = 0xffff; + } + dcp_gamma_set(&dcp); + assert(writes == 2 && swaps == 2 && !warnings); + for (unsigned i = 0; i < IOMFB_GAMMA_ENTRIES; i++) { + assert(sent.table[0][i] == 0); + assert(sent.table[1][i] == 32767); + assert(sent.table[2][i] == 196608); + } + state.gamma_user = false; + dcp_gamma_set(&dcp); + assert(writes == 3 && swaps == 3 && !warnings); + assert(!memcmp(sent.table, reply.table, sizeof(sent.table))); + } + puts("gamma RPC status, admission, composition and restoration passed"); + return 0; +} +''' + return prefix + production + suffix diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index f836743f2fc8d..b156582f145f6 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -54,6 +54,7 @@ def main(): 'orientation-faults', 'backend-change', 'safe-first', 'failed-flip-return', 'live-park')) tests = [ + ('display', 'desktop-gamma', desktop.gamma, ((),), ()), ('lifecycle', 'desktop-modeset-unmap', desktop.modeset_unmap, ((),), ()), ('lifecycle', 'desktop-unknown-modeset-recovery', desktop.modeset_unknown, ((),), ()), ('display', 'panel-backlight-ramp', display.backlight, ((),), ()), From 3db457a10d5d2a5911732591570d707e06e264c9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 15:58:22 +1100 Subject: [PATCH 1467/1579] hwmon: macsmc: Keep explicit control of write-only legacy mode keys The mode preflight should not introduce a read requirement for a legacy write-only mode key. Use the metadata to keep its existing direct write path, still solely under an explicit user target. Readable system-mode keys keep the unlock-before-write sequence. Cover this metadata case with the SMC model, asserting no mode read and no unlock operation for the legacy write-only path. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hwmon/macsmc-hwmon-test.c | 19 +++++++++++++++++++ drivers/hwmon/macsmc-hwmon.c | 5 +++++ 2 files changed, 24 insertions(+) diff --git a/drivers/hwmon/macsmc-hwmon-test.c b/drivers/hwmon/macsmc-hwmon-test.c index 610a019ee59b8..1f8fa33dc2e0e 100644 --- a/drivers/hwmon/macsmc-hwmon-test.c +++ b/drivers/hwmon/macsmc-hwmon-test.c @@ -109,6 +109,7 @@ struct macsmc_fan_test { int writes; int ftst_writes; int refused; + int mode_reads; smc_key fail_key; }; @@ -125,6 +126,7 @@ static int fan_test_read(struct apple_smc *smc, smc_key key, void *buf, size_t s switch (key & 0xff00ffff) { case __SMC_KEY('F', 0, 'M', 'd'): + t->mode_reads++; value = t->mode[i]; break; case __SMC_KEY('F', 0, 'M', 'n'): @@ -330,6 +332,22 @@ static void macsmc_fan_test_legacy(struct kunit *test) KUNIT_EXPECT_EQ(test, t->refused, 0); } +/* A legacy write-only mode key keeps working without a new read requirement. */ +static void macsmc_fan_test_legacy_write_only(struct kunit *test) +{ + struct macsmc_fan_test *t = test->priv; + + t->legacy = true; + t->mode[0] = MACSMC_FAN_MODE_AUTO; + t->fans[0].mode.info.flags &= ~APPLE_SMC_READABLE; + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), 0); + KUNIT_EXPECT_EQ(test, t->mode[0], MACSMC_FAN_MODE_MANUAL); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 0), 0); + KUNIT_EXPECT_EQ(test, t->mode[0], MACSMC_FAN_MODE_AUTO); + KUNIT_EXPECT_EQ(test, t->mode_reads, 0); + KUNIT_EXPECT_EQ(test, t->ftst_writes, 0); +} + /* Suspend, resume and shutdown do not write untouched fan keys. */ static void macsmc_fan_test_untouched(struct kunit *test) { @@ -384,6 +402,7 @@ static struct kunit_case macsmc_fan_test_cases[] = { KUNIT_CASE(macsmc_fan_test_two_fans), KUNIT_CASE(macsmc_fan_test_no_unlock_key), KUNIT_CASE(macsmc_fan_test_legacy), + KUNIT_CASE(macsmc_fan_test_legacy_write_only), KUNIT_CASE(macsmc_fan_test_untouched), KUNIT_CASE(macsmc_fan_test_suspend_manual), KUNIT_CASE(macsmc_fan_test_unlock_fails), diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index a8e47ee4b98ff..097fa6c40c610 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -480,6 +480,10 @@ static int macsmc_hwmon_fan_manual(struct macsmc_hwmon *hwmon, if (fan->manual) return 0; + /* Older firmware may expose a write-only mode key. */ + if (!(fan->mode.info.flags & APPLE_SMC_READABLE)) + goto set_manual; + /* Do not send a mode write the firmware is known to refuse. */ ret = macsmc_hwmon_fan_read_mode(hwmon, fan, &mode); if (ret) @@ -514,6 +518,7 @@ static int macsmc_hwmon_fan_manual(struct macsmc_hwmon *hwmon, goto relock; } +set_manual: ret = macsmc_hwmon_write_key(hwmon->smc, &fan->mode, MACSMC_FAN_MODE_MANUAL); if (ret < 0) goto relock; From 8957f1851e8177aae939220da462dfcfe79de015 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 16:08:27 +1100 Subject: [PATCH 1468/1579] dt-bindings: display: apple: Add T8140 display subsystem compatible The T8140 display subsystem uses apple,t8140-display-subsystem followed by the common apple,display-subsystem fallback. Describe that pair so t8140-j700 passes schema validation without changing its device tree. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/display/apple,display-subsystem.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml b/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml index a7339054136a8..383286892b97e 100644 --- a/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml +++ b/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml @@ -21,6 +21,9 @@ properties: - apple,t8132-display-subsystem - const: apple,t8132-display-diagnostics deprecated: true + - items: + - const: apple,t8140-display-subsystem + - const: apple,display-subsystem - items: - const: apple,t8132-display-subsystem - const: apple,t8132-display-diagnostics From 1bf01cffd7f64c88aecf37299aaca2bfbdd5bf34 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 16:08:31 +1100 Subject: [PATCH 1469/1579] dt-bindings: display: simple-framebuffer: Allow post-init-providers The inherited Apple framebuffer lists post-init-providers to break an initialization dependency cycle. Allow the property in the strict simple-framebuffer binding. The common dtschema post-init-providers.yaml schema already defines the property and validates its phandle list. Use a boolean schema here so the binding does not duplicate its type or redefine its meaning. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/display/simple-framebuffer.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/display/simple-framebuffer.yaml b/Documentation/devicetree/bindings/display/simple-framebuffer.yaml index 45ffdebc9d86b..66e7347805bdd 100644 --- a/Documentation/devicetree/bindings/display/simple-framebuffer.yaml +++ b/Documentation/devicetree/bindings/display/simple-framebuffer.yaml @@ -118,6 +118,8 @@ properties: - x8r8g8b8 - x8b8g8r8 + post-init-providers: true + display: $ref: /schemas/types.yaml#/definitions/phandle description: Primary display hardware node From def29c759a9e8d1bd5d3fc50caf913ad8edb6350 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 05:59:37 +0000 Subject: [PATCH 1470/1579] spi: apple-qspi: admit verified NVRAM writes on the J700 asahi-bless cannot select a one-time boot volume on the MacBook Neo (J700). The T8140 path admitted programs and erases inside the nvram partition with no runtime opt-in and no readback, while the J613 and J713 go through the read-only, opt-in, verified policy. Writes still never reached the flash, because the partition was read-only in MTD. Put the J700 on the same policy. The controller starts read-only and root opts in at runtime through the read_only parameter. A write needs the J700 board and the controller at 0x385110000. Programs are single-wire, three-byte, within one 256-byte page; erases are aligned 4 KiB sectors; both must lie inside the nvram partition. Every mutation checks the JEDEC ID and status first, is verified by readback, and a fault latches the driver read-only. The J613/J713 partition is pinned at 0x700000. The J700's comes from its device tree (0x24000, size 0x1d0000), so the range check now takes the partition bounds, and a DT window must fit in three-byte addressing. The J700 flash is a W25Q16JV (JEDEC ef6515). iBoot leaves BP0 set, which protects the top 64 KiB, so the J700 tolerates the block-protect bits as found: the driver never changes them, the flash ignores writes in that range, and readback then latches read-only. CMP, suspend, WPS, BUSY and WEL are still refused. T8140 data reads keep their multi-line pass-through. KUnit and the host harness now cover the shared policy at the J700 bounds, the J700 board gate and the block-protect tolerance. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/Kconfig | 16 +-- drivers/spi/spi-apple-qspi-policy-test.c | 128 +++++++++++++---------- drivers/spi/spi-apple-qspi-policy.h | 95 ++++++----------- drivers/spi/spi-apple-qspi.c | 90 ++++++++-------- tools/asahi/qspi-nvram-tests.py | 55 +++++----- 5 files changed, 186 insertions(+), 198 deletions(-) diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index ad875e80d8ba2..8b53e2cf36b8d 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -148,20 +148,20 @@ config SPI_APPLE_QSPI Unlike the ordinary Apple SPIMC block, QSPIMC accepts packed SPI memory descriptors and is used for the boot NOR on T8140 systems. - T8122/T8132 controllers default to read-only access. Only the pinned - J613 NVRAM layout admits bounded writes after a runtime opt-in. - On T8140 only the nvram partition accepts programs and erases; the - rest of the boot NOR is read-only. + The boot NOR defaults to read-only access. On the J613, J713 and + J700 a runtime opt-in admits bounded, verified page programs and + 4 KiB erases inside the nvram partition only; the rest of the boot + NOR stays read-only. config SPI_APPLE_QSPI_KUNIT_TEST - tristate "KUnit tests for the Apple QSPIMC boot NOR write window" if !KUNIT_ALL_TESTS + tristate "KUnit tests for the Apple QSPIMC NVRAM write policy" if !KUNIT_ALL_TESTS depends on KUNIT depends on SPI_APPLE_QSPI || COMPILE_TEST default KUNIT_ALL_TESTS help - Tests which T8140 boot NOR requests the QSPIMC driver admits: write - enable control, and programs and erases inside the nvram partition - only. No hardware is accessed. + Tests which boot NOR requests the QSPIMC driver admits: write + enable control, and page programs and 4 KiB erases inside the + nvram partition only. No hardware is accessed. config SPI_AR934X tristate "Qualcomm Atheros AR934X/QCA95XX SPI controller driver" diff --git a/drivers/spi/spi-apple-qspi-policy-test.c b/drivers/spi/spi-apple-qspi-policy-test.c index 9ec58d587c60f..b1ecbdbe4e26e 100644 --- a/drivers/spi/spi-apple-qspi-policy-test.c +++ b/drivers/spi/spi-apple-qspi-policy-test.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* - * KUnit tests for the T8140 boot NOR write window of the Apple QSPIMC driver. + * KUnit tests for the Apple QSPIMC NVRAM mutation policy, at the J700 and + * J613/J713 nvram partitions. */ #include @@ -14,59 +15,57 @@ static int nv(unsigned int opcode, unsigned int addr_bytes, unsigned long long addr, unsigned int len, int data_out) { - return apple_qspi_window_allowed(opcode, addr_bytes, addr, len, - data_out, NV_START, NV_END); + return apple_qspi_mutation_allowed(opcode, addr_bytes, addr, len, + data_out, NV_START, NV_END); } -static void qspi_window_write_enable(struct kunit *test) +static void qspi_policy_write_enable(struct kunit *test) { KUNIT_EXPECT_TRUE(test, nv(0x06, 0, 0, 0, 0)); KUNIT_EXPECT_TRUE(test, nv(0x04, 0, 0, 0, 0)); KUNIT_EXPECT_FALSE(test, nv(0x06, 3, NV_START, 0, 0)); KUNIT_EXPECT_FALSE(test, nv(0x06, 0, 0, 1, 1)); - /* Write-enable control needs no window. */ - KUNIT_EXPECT_TRUE(test, apple_qspi_window_allowed(0x06, 0, 0, 0, 0, 0, 0)); + /* Write-enable control is virtual and needs no window. */ + KUNIT_EXPECT_TRUE(test, apple_qspi_mutation_allowed(0x06, 0, 0, 0, 0, 0, 0)); } -static void qspi_window_program(struct kunit *test) +static void qspi_policy_program(struct kunit *test) { KUNIT_EXPECT_TRUE(test, nv(0x02, 3, NV_START, 256, 1)); KUNIT_EXPECT_TRUE(test, nv(0x02, 3, NV_START + 0x1234, 1, 1)); KUNIT_EXPECT_TRUE(test, nv(0x02, 3, NV_END - 256, 256, 1)); - KUNIT_EXPECT_TRUE(test, nv(0x12, 4, NV_START, 256, 1)); - /* Any byte outside, or a wrap within a page outside, is refused. */ + /* Any byte outside the partition is refused. */ KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START - 256, 256, 1)); KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START - 1, 2, 1)); KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_END, 1, 1)); KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_END - 1, 2, 1)); KUNIT_EXPECT_FALSE(test, nv(0x02, 3, 0, 256, 1)); - /* A program needs data out, and at most one 4 KiB block's worth. */ + /* A program stays within one page and needs data out. */ + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START + 0xff, 2, 1)); + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START, 257, 1)); KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START, 0, 1)); KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START, 256, 0)); - KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START, 4097, 1)); + /* Four-byte programs are not admitted. */ + KUNIT_EXPECT_FALSE(test, nv(0x12, 4, NV_START, 256, 1)); } -static void qspi_window_erase(struct kunit *test) +static void qspi_policy_erase(struct kunit *test) { KUNIT_EXPECT_TRUE(test, nv(0x20, 3, NV_START, 0, 0)); - KUNIT_EXPECT_TRUE(test, nv(0x20, 3, NV_START + 0x123, 0, 0)); - KUNIT_EXPECT_TRUE(test, nv(0x20, 3, NV_END - 1, 0, 0)); - KUNIT_EXPECT_TRUE(test, nv(0x21, 4, NV_START, 0, 0)); - KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_START - 1, 0, 0)); + KUNIT_EXPECT_TRUE(test, nv(0x20, 3, NV_END - 0x1000, 0, 0)); + /* Only aligned 4 KiB sector erases inside the partition. */ + KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_START + 0x123, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_START - 0x1000, 0, 0)); KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_END, 0, 0)); - /* 32 and 64 KiB erases cover the whole aligned block. */ - KUNIT_EXPECT_TRUE(test, nv(0x52, 3, 0x28000, 0, 0)); - KUNIT_EXPECT_FALSE(test, nv(0x52, 3, NV_START, 0, 0)); - KUNIT_EXPECT_TRUE(test, nv(0xd8, 3, 0x30000, 0, 0)); - KUNIT_EXPECT_TRUE(test, nv(0xdc, 4, 0x1e0000, 0, 0)); - KUNIT_EXPECT_FALSE(test, nv(0xd8, 3, NV_START, 0, 0)); - KUNIT_EXPECT_FALSE(test, nv(0xd8, 3, 0x1f0000, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x21, 4, NV_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x52, 3, 0x28000, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0xd8, 3, 0x30000, 0, 0)); /* An erase carries no data. */ KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_START, 1, 0)); KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_START, 0, 1)); } -static void qspi_window_refused_commands(struct kunit *test) +static void qspi_policy_refused_commands(struct kunit *test) { static const u8 refused[] = { 0x01, 0x31, 0x11, /* status register writes */ @@ -86,48 +85,67 @@ static void qspi_window_refused_commands(struct kunit *test) } } -static void qspi_window_address_width(struct kunit *test) +static void qspi_policy_address_width(struct kunit *test) { - /* Three address bytes cannot carry an address above 16 MiB. */ - KUNIT_EXPECT_FALSE(test, nv(0x02, 3, (1ULL << 24) + NV_START, 1, 1)); - KUNIT_EXPECT_FALSE(test, nv(0x20, 3, (1ULL << 24) + NV_START, 0, 0)); - KUNIT_EXPECT_FALSE(test, nv(0x12, 4, (1ULL << 32) + NV_START, 1, 1)); KUNIT_EXPECT_FALSE(test, nv(0x02, 2, NV_START, 1, 1)); + KUNIT_EXPECT_FALSE(test, nv(0x02, 4, NV_START, 1, 1)); KUNIT_EXPECT_FALSE(test, nv(0x20, 0, NV_START, 0, 0)); - KUNIT_EXPECT_FALSE(test, nv(0x20, 5, NV_START, 0, 0)); - /* A window that reaches the top of 24-bit addressing. */ - KUNIT_EXPECT_TRUE(test, apple_qspi_window_allowed(0x20, 3, 0xfff000, 0, 0, - 0xff0000, 1ULL << 24)); - /* A program that crosses 16 MiB cannot use three address bytes. */ - KUNIT_EXPECT_FALSE(test, apple_qspi_window_allowed(0x02, 3, 0xffffff, 2, 1, - 0xff0000, 1ULL << 25)); - KUNIT_EXPECT_TRUE(test, apple_qspi_window_allowed(0x12, 4, 0xffffff, 2, 1, - 0xff0000, 1ULL << 25)); + KUNIT_EXPECT_FALSE(test, nv(0x20, 4, NV_START, 0, 0)); + /* Addresses are not truncated to the wire width before the check. */ + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, (1ULL << 24) + NV_START, 1, 1)); + KUNIT_EXPECT_FALSE(test, nv(0x20, 3, (1ULL << 24) + NV_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, ~0ULL, 1, 1)); +} + +static void qspi_policy_empty(struct kunit *test) +{ + KUNIT_EXPECT_FALSE(test, apple_qspi_mutation_allowed(0x02, 3, 0, 1, 1, 0, 0)); + KUNIT_EXPECT_FALSE(test, apple_qspi_mutation_allowed(0x20, 3, 0, 0, 0, 0, 0)); + KUNIT_EXPECT_FALSE(test, apple_qspi_mutation_allowed(0x20, 3, NV_START, 0, 0, + NV_START, NV_START)); +} + +static void qspi_policy_j613(struct kunit *test) +{ + unsigned long long s = APPLE_QSPI_NVRAM_START, e = APPLE_QSPI_NVRAM_END; + + KUNIT_EXPECT_TRUE(test, apple_qspi_mutation_allowed(0x02, 3, s, 256, 1, s, e)); + KUNIT_EXPECT_TRUE(test, apple_qspi_mutation_allowed(0x20, 3, e - 0x1000, 0, 0, s, e)); + KUNIT_EXPECT_FALSE(test, apple_qspi_mutation_allowed(0x02, 3, s - 1, 1, 1, s, e)); + KUNIT_EXPECT_FALSE(test, apple_qspi_mutation_allowed(0x20, 3, e, 0, 0, s, e)); } -static void qspi_window_empty(struct kunit *test) +static void qspi_policy_status(struct kunit *test) { - KUNIT_EXPECT_FALSE(test, apple_qspi_window_allowed(0x02, 3, 0, 1, 1, 0, 0)); - KUNIT_EXPECT_FALSE(test, apple_qspi_window_allowed(0x20, 3, 0, 0, 0, 0, 0)); - KUNIT_EXPECT_FALSE(test, apple_qspi_window_allowed(0x20, 3, NV_START, 0, 0, - NV_START, NV_START)); + /* BUSY and WEL are never admitted. */ + KUNIT_EXPECT_FALSE(test, apple_qspi_status_allows_write(0x01, 0, 0, 0x1c)); + KUNIT_EXPECT_FALSE(test, apple_qspi_status_allows_write(0x02, 0, 0, 0x1c)); + /* Block protect only where the board allows it. */ + KUNIT_EXPECT_FALSE(test, apple_qspi_status_allows_write(0x04, 0, 0, 0)); + KUNIT_EXPECT_TRUE(test, apple_qspi_status_allows_write(0x04, 0x02, 0, 0x1c)); + /* CMP, SUS and WPS are refused everywhere. */ + KUNIT_EXPECT_FALSE(test, apple_qspi_status_allows_write(0, 0x40, 0, 0x1c)); + KUNIT_EXPECT_FALSE(test, apple_qspi_status_allows_write(0, 0x80, 0, 0x1c)); + KUNIT_EXPECT_FALSE(test, apple_qspi_status_allows_write(0, 0, 0x04, 0x1c)); } -static struct kunit_case apple_qspi_window_cases[] = { - KUNIT_CASE(qspi_window_write_enable), - KUNIT_CASE(qspi_window_program), - KUNIT_CASE(qspi_window_erase), - KUNIT_CASE(qspi_window_refused_commands), - KUNIT_CASE(qspi_window_address_width), - KUNIT_CASE(qspi_window_empty), +static struct kunit_case apple_qspi_policy_cases[] = { + KUNIT_CASE(qspi_policy_write_enable), + KUNIT_CASE(qspi_policy_program), + KUNIT_CASE(qspi_policy_erase), + KUNIT_CASE(qspi_policy_refused_commands), + KUNIT_CASE(qspi_policy_address_width), + KUNIT_CASE(qspi_policy_empty), + KUNIT_CASE(qspi_policy_j613), + KUNIT_CASE(qspi_policy_status), {} }; -static struct kunit_suite apple_qspi_window_suite = { - .name = "apple-qspi-nvram-window", - .test_cases = apple_qspi_window_cases, +static struct kunit_suite apple_qspi_policy_suite = { + .name = "apple-qspi-nvram-policy", + .test_cases = apple_qspi_policy_cases, }; -kunit_test_suite(apple_qspi_window_suite); +kunit_test_suite(apple_qspi_policy_suite); -MODULE_DESCRIPTION("KUnit tests for the Apple QSPIMC boot NOR write window"); +MODULE_DESCRIPTION("KUnit tests for the Apple QSPIMC NVRAM mutation policy"); MODULE_LICENSE("GPL"); diff --git a/drivers/spi/spi-apple-qspi-policy.h b/drivers/spi/spi-apple-qspi-policy.h index 5d788d69f4f8b..12d8ac41dd26c 100644 --- a/drivers/spi/spi-apple-qspi-policy.h +++ b/drivers/spi/spi-apple-qspi-policy.h @@ -2,21 +2,36 @@ #ifndef APPLE_QSPI_POLICY_H #define APPLE_QSPI_POLICY_H +/* The pinned J613/J713 nvram partition. */ #define APPLE_QSPI_NVRAM_START 0x700000ULL #define APPLE_QSPI_NVRAM_END 0x800000ULL #define APPLE_QSPI_PAGE_SIZE 256U #define APPLE_QSPI_ERASE_SIZE 4096U -/* Subtraction avoids overflow; addr is not truncated to wire width first. */ -static inline int apple_qspi_range_allowed(unsigned long long addr, unsigned int len) +/* + * [start, end) is the board's nvram partition. Subtraction avoids overflow; + * addr is not truncated to wire width first. + */ +static inline int apple_qspi_range_allowed(unsigned long long addr, + unsigned int len, unsigned long long start, + unsigned long long end) { - return len && addr >= APPLE_QSPI_NVRAM_START && - addr < APPLE_QSPI_NVRAM_END && len <= APPLE_QSPI_NVRAM_END - addr; + return len && start < end && addr >= start && addr < end && + len <= end - addr; } +/* + * May a write-path request reach the boot NOR? Write enable control is + * virtual and admitted. A single-wire, three-byte page program that stays + * within one 256-byte page, or an aligned 4 KiB sector erase, is admitted + * only when every byte it can change lies inside [start, end). Everything + * else (status and configuration writes, larger erases, chip erase, + * address-mode, reset and OTP commands) is refused. + */ static inline int apple_qspi_mutation_allowed(unsigned int opcode, unsigned int addr_bytes, unsigned long long addr, - unsigned int len, int data_out) + unsigned int len, int data_out, + unsigned long long start, unsigned long long end) { switch (opcode) { case 0x06: /* Virtual WREN, never sent independently. */ @@ -24,76 +39,30 @@ static inline int apple_qspi_mutation_allowed(unsigned int opcode, return !addr_bytes && !len && !data_out; case 0x02: /* PP: prevent page wrap, even if SPI-NOR gets geometry wrong. */ return addr_bytes == 3 && data_out && len <= APPLE_QSPI_PAGE_SIZE && - apple_qspi_range_allowed(addr, len) && + apple_qspi_range_allowed(addr, len, start, end) && len <= APPLE_QSPI_PAGE_SIZE - (addr & (APPLE_QSPI_PAGE_SIZE - 1)); case 0x20: /* SE: hard-code the entire affected 4 KiB range. */ return addr_bytes == 3 && !len && !data_out && !(addr & (APPLE_QSPI_ERASE_SIZE - 1)) && - apple_qspi_range_allowed(addr, APPLE_QSPI_ERASE_SIZE); + apple_qspi_range_allowed(addr, APPLE_QSPI_ERASE_SIZE, + start, end); default: return 0; } } /* - * T8140 boot NOR: may a request that does not read data reach the flash? - * Write-enable control is admitted. A page program, or a 4, 32 or 64 KiB - * erase, is admitted only when every byte it can change lies inside - * [start, end), the nvram partition. A program counts the whole 4 KiB - * blocks it touches, which covers a write that wraps within its page. Status - * and configuration register writes, chip erase, address-mode, reset and OTP - * commands are refused. The address must fit in addr_bytes: the controller - * sends only those bytes, so a wider value would alias a lower address. + * Protection and activity bits must permit a write; JEDEC ID is checked + * separately. BUSY/WEL, CMP/SUS and WPS are always refused. bp_ok names the + * SR1 block-protect bits a board's firmware leaves set at boot: they stay + * unchanged, the flash ignores programs and erases in the range they protect, + * and the driver's readback then latches it read-only. SRP, TB, SEC, QE and + * LB stay unchanged. */ -static inline int apple_qspi_window_allowed(unsigned int opcode, - unsigned int addr_bytes, unsigned long long addr, - unsigned int len, int data_out, - unsigned long long start, unsigned long long end) -{ - unsigned long long block, first, last; - - switch (opcode) { - case 0x06: /* WREN */ - case 0x04: /* WRDI */ - return !addr_bytes && !len && !data_out; - case 0x02: /* PP */ - case 0x12: /* PP_4B */ - if (!data_out || !len || len > APPLE_QSPI_ERASE_SIZE) - return 0; - block = APPLE_QSPI_ERASE_SIZE; - break; - case 0x20: /* BE_4K */ - case 0x21: /* BE_4K_4B */ - block = 0x1000; - break; - case 0x52: /* BE_32K */ - case 0x5c: /* BE_32K_4B */ - block = 0x8000; - break; - case 0xd8: /* SE */ - case 0xdc: /* SE_4B */ - block = 0x10000; - break; - default: - return 0; - } - if (opcode != 0x02 && opcode != 0x12 && (len || data_out)) - return 0; - if ((addr_bytes != 3 && addr_bytes != 4) || addr >> (8 * addr_bytes)) - return 0; - - /* [first, last) are the blocks the flash can change. */ - first = addr & ~(block - 1); - last = ((addr + (len ? len : 1) - 1) | (block - 1)) + 1; - return last <= 1ULL << (8 * addr_bytes) && - start <= first && last <= end; -} - -/* Protection and activity bits must permit a write; JEDEC ID is checked separately. */ static inline int apple_qspi_status_allows_write(unsigned int sr1, - unsigned int sr2, unsigned int sr3) + unsigned int sr2, unsigned int sr3, unsigned int bp_ok) { - /* BUSY/WEL, BP[2:0]; CMP/SUS; WPS. SRP, TB, SEC, QE, LB stay unchanged. */ - return !(sr1 & 0x1f) && !(sr2 & 0xc0) && !(sr3 & 0x04); + return !(sr1 & 0x03) && !(sr1 & 0x1c & ~bp_ok) && + !(sr2 & 0xc0) && !(sr3 & 0x04); } #endif diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index c594a241dcef8..e94fe6eb3fd5f 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -7,10 +7,10 @@ * command FIFO. This driver intentionally implements the PIO spi-mem path * only; that is sufficient for SPI NOR discovery and access. * - * On T8140 reads reach the whole boot NOR, but programs and erases are - * admitted only inside its unique, aligned nvram partition. Without that - * partition the NOR is read-only. - * + * The boot NOR starts read-only. On the qualified boards (J613, J713 and + * the J700) root may opt in at runtime to bounded, verified programs and + * 4 KiB erases inside the nvram partition; everything else stays + * read-only. On T8140 data reads reach the whole NOR. */ #include @@ -88,6 +88,10 @@ #define APPLE_QSPI_MAX_HZ 40000000 #define APPLE_QSPI_TIMEOUT_MS 1000 +/* Winbond W25Q64 (J613/J713) and W25Q16 (J700) boot NOR JEDEC IDs. */ +static const u8 apple_qspi_jedec_w25q64[3] = { 0xef, 0x65, 0x17 }; +static const u8 apple_qspi_jedec_w25q16[3] = { 0xef, 0x65, 0x15 }; + static DEFINE_MUTEX(apple_qspi_policy_lock); static bool read_only = true; static bool runtime_ready; @@ -125,7 +129,7 @@ static const struct kernel_param_ops apple_qspi_read_only_ops = { .get = apple_qspi_get_read_only, }; module_param_cb(read_only, &apple_qspi_read_only_ops, &read_only, 0644); -MODULE_PARM_DESC(read_only, "Default true; root runtime opt-in for bounded J613/J713 NVRAM writes"); +MODULE_PARM_DESC(read_only, "Default true; root runtime opt-in for bounded J613/J713/J700 NVRAM writes"); static bool apple_qspi_opcode_is_read(u8 opcode) { @@ -149,12 +153,16 @@ static bool apple_qspi_opcode_is_read(u8 opcode) } struct apple_qspi { - /* T8122/T8132 use the serialized, verified NVRAM policy. */ + /* Every write-path request goes through the verified NVRAM policy. */ bool nvram_policy; - /* T8140 admits programs and erases only inside nvram. */ + /* T8140: data reads of any shape pass straight to the controller. */ bool nvram_window; + /* The nvram partition [window_start, window_end). */ u64 window_start; u64 window_end; + /* Expected JEDEC ID and the SR1 block-protect bits left by firmware. */ + u8 jedec[3]; + u8 bp_ok; bool nvram_write_candidate; bool write_fault; void __iomem *base; @@ -652,7 +660,7 @@ static int apple_qspi_nvram_modify(struct spi_mem *mem, const struct spi_mem_op ret = apple_qspi_read_register(mem, 0x9f, id, sizeof(id)); if (ret) goto out; - if (id[0] != 0xef || id[1] != 0x65 || id[2] != 0x17) { + if (memcmp(id, qspi->jedec, sizeof(id))) { ret = -ENODEV; goto out; } @@ -660,7 +668,8 @@ static int apple_qspi_nvram_modify(struct spi_mem *mem, const struct spi_mem_op if (ret) goto out; /* No unlocking. Conservatively refuse BP/CMP/WPS, BUSY/WEL or suspend. */ - if (!apple_qspi_status_allows_write(before[0], before[1], before[2])) { + if (!apple_qspi_status_allows_write(before[0], before[1], before[2], + qspi->bp_ok)) { ret = -EACCES; goto out; } @@ -780,7 +789,8 @@ static int apple_qspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) } if (!apple_qspi_mutation_allowed(op->cmd.opcode, op->addr.nbytes, op->addr.val, op->data.nbytes, - op->data.dir == SPI_MEM_DATA_OUT)) { + op->data.dir == SPI_MEM_DATA_OUT, + qspi->window_start, qspi->window_end)) { denial = "mutation-policy"; goto out; } @@ -802,36 +812,14 @@ static int apple_qspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) return ret; } -/* T8140: data reads pass; anything else must stay inside the nvram window. */ -static int apple_qspi_window_exec_op(struct spi_mem *mem, - const struct spi_mem_op *op) -{ - struct apple_qspi *qspi = spi_controller_get_devdata(mem->spi->controller); - - if (op->data.dir == SPI_MEM_DATA_IN && op->data.nbytes) - return apple_qspi_transfer_op(mem, op); - if (!op->dummy.nbytes && - apple_qspi_window_allowed(op->cmd.opcode, op->addr.nbytes, - op->addr.val, op->data.nbytes, - op->data.dir == SPI_MEM_DATA_OUT, - qspi->window_start, qspi->window_end)) - return apple_qspi_transfer_op(mem, op); - - dev_warn_ratelimited(&mem->spi->dev, - "boot NOR request denied: opcode=%02x addr=%#llx addr_bytes=%u len=%u dir=%u nvram=[%#llx, %#llx)\n", - op->cmd.opcode, (unsigned long long)op->addr.val, - op->addr.nbytes, op->data.nbytes, op->data.dir, - (unsigned long long)qspi->window_start, - (unsigned long long)qspi->window_end); - return -EACCES; -} - static int apple_qspi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) { struct apple_qspi *qspi = spi_controller_get_devdata(mem->spi->controller); - if (qspi->nvram_window) - return apple_qspi_window_exec_op(mem, op); + /* T8140 keeps its multi-line data reads; spi-mem serializes them. */ + if (qspi->nvram_window && op->data.dir == SPI_MEM_DATA_IN && + op->data.nbytes) + return apple_qspi_transfer_op(mem, op); if (!qspi->nvram_policy) return apple_qspi_transfer_op(mem, op); return apple_qspi_exec_op(mem, op); @@ -853,6 +841,9 @@ static bool apple_qspi_nvram_write_board(struct device_node *node, if (of_device_is_compatible(node, "apple,t8132-qspi")) return of_machine_is_compatible("apple,j713") && base == 0x3ad118000ULL; + if (of_device_is_compatible(node, "apple,t8140-qspi")) + return of_machine_is_compatible("apple,j700") && + base == 0x385110000ULL; return false; } @@ -894,7 +885,8 @@ static bool apple_qspi_nvram_layout(struct device_node *node) /* * The one "nvram" partition of the flash on CS0, 4 KiB aligned and within - * 32-bit addressing. Without it, no program or erase is admitted. + * three-byte addressing, since the policy only admits three-byte programs + * and erases. Without it, no program or erase is admitted. */ static void apple_qspi_nvram_window(struct device *dev, struct apple_qspi *qspi) { @@ -933,7 +925,7 @@ static void apple_qspi_nvram_window(struct device *dev, struct apple_qspi *qspi) } } of_node_put(parts); - if (found != 1 || start >= end || end > 1ULL << 32 || + if (found != 1 || start >= end || end > 1ULL << 24 || !IS_ALIGNED(start, APPLE_QSPI_ERASE_SIZE) || !IS_ALIGNED(end, APPLE_QSPI_ERASE_SIZE)) start = end = 0; @@ -941,8 +933,7 @@ static void apple_qspi_nvram_window(struct device *dev, struct apple_qspi *qspi) qspi->window_start = start; qspi->window_end = end; if (start < end) - dev_dbg(dev, "boot NOR read-only outside nvram [%#llx, %#llx)\n", - start, end); + dev_dbg(dev, "nvram partition [%#llx, %#llx)\n", start, end); else dev_dbg(dev, "boot NOR read-only: no usable nvram partition\n"); } @@ -961,11 +952,19 @@ static int apple_qspi_probe(struct platform_device *pdev) return -ENOMEM; qspi = spi_controller_get_devdata(ctlr); + qspi->nvram_policy = true; qspi->nvram_window = of_device_is_compatible(dev->of_node, "apple,t8140-qspi"); - if (qspi->nvram_window) + if (qspi->nvram_window) { + /* J700: W25Q16JV; iBoot leaves BP0 set over the top 64 KiB. */ apple_qspi_nvram_window(dev, qspi); - qspi->nvram_policy = of_device_is_compatible(dev->of_node, "apple,t8122-qspi") || - of_device_is_compatible(dev->of_node, "apple,t8132-qspi"); + memcpy(qspi->jedec, apple_qspi_jedec_w25q16, sizeof(qspi->jedec)); + qspi->bp_ok = 0x1c; + } else { + qspi->window_start = APPLE_QSPI_NVRAM_START; + qspi->window_end = APPLE_QSPI_NVRAM_END; + memcpy(qspi->jedec, apple_qspi_jedec_w25q64, sizeof(qspi->jedec)); + qspi->bp_ok = 0; + } if (of_device_is_compatible(dev->of_node, "apple,t8122-qspi") && !of_machine_is_compatible("apple,j613")) return -ENODEV; @@ -979,7 +978,8 @@ static int apple_qspi_probe(struct platform_device *pdev) qspi->nvram_write_candidate = apple_qspi_nvram_write_board(dev->of_node, res->start) && - apple_qspi_nvram_layout(dev->of_node); + (qspi->nvram_window ? qspi->window_start < qspi->window_end : + apple_qspi_nvram_layout(dev->of_node)); qspi->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(qspi->base)) return PTR_ERR(qspi->base); @@ -1016,7 +1016,7 @@ static int apple_qspi_probe(struct platform_device *pdev) ctlr->num_chipselect = 1; ctlr->max_native_cs = 1; ctlr->mode_bits = SPI_CPHA | SPI_CPOL; - if (!qspi->nvram_policy) + if (qspi->nvram_window) ctlr->mode_bits |= SPI_RX_DUAL | SPI_RX_QUAD | SPI_TX_DUAL | SPI_TX_QUAD; ctlr->bits_per_word_mask = SPI_BPW_MASK(8); diff --git a/tools/asahi/qspi-nvram-tests.py b/tools/asahi/qspi-nvram-tests.py index 5b48039e8dbab..427d0cd8479c0 100644 --- a/tools/asahi/qspi-nvram-tests.py +++ b/tools/asahi/qspi-nvram-tests.py @@ -68,6 +68,7 @@ def function(source, signature): struct apple_qspi { bool nvram_policy, nvram_write_candidate, write_fault, nvram_window; u64 window_start, window_end; + u8 jedec[3]; u8 bp_ok; }; struct spi_controller { struct apple_qspi *q; }; struct spi_device { struct spi_controller *controller; unsigned int cs; int dev; }; @@ -103,7 +104,7 @@ def function(source, signature): memcpy(o->data.buf.in,flash+o->addr.val,o->data.nbytes); if(corrupt_readback&&programmed)((u8*)o->data.buf.in)[0]^=1; break; - case 0x06:assert(!sr[0]);sr[0]|=2;break; + case 0x06:assert(!(sr[0]&3));sr[0]|=2;break; case 0x04:sr[0]&=~2;break; case 0x02: assert(sr[0]&2);assert(o->addr.val>=0x700000&&o->addr.val+o->data.nbytes<=0x800000); @@ -123,7 +124,7 @@ def function(source, signature): static struct spi_mem mem={&spi}; static struct kernel_param parameter={&read_only}; static void reset(void){ - memset(flash,0xff,sizeof(flash));memset(sr,0,sizeof(sr));q=(struct apple_qspi){.nvram_policy=true,.nvram_write_candidate=true}; + memset(flash,0xff,sizeof(flash));memset(sr,0,sizeof(sr));q=(struct apple_qspi){.nvram_policy=true,.nvram_write_candidate=true,.window_start=0x700000,.window_end=0x800000,.jedec={0xef,0x65,0x17}}; read_only=true;runtime_ready=false;write_fault_latched=false;wire_count=0;fail_opcode=0;spi.cs=0; corrupt_readback=false;stuck_busy=false;programmed=false;deny_alloc=false;jiffies=0; jedec[0]=0xef;jedec[1]=0x65;jedec[2]=0x17; @@ -136,7 +137,7 @@ def function(source, signature): static bool sent(unsigned int o){for(unsigned int i=0;i= APPLE_QSPI_NVRAM_START', 'addr >= 1'), + ('j700-board', 'of_machine_is_compatible("apple,j700")', 'true'), + ('range', 'addr >= start &&', 'addr >= 1 &&'), + ('bp-ok', '!(sr1 & 0x1c & ~bp_ok)', '1'), ('read-only', 'read_only || !qspi->nvram_write_candidate || qspi->write_fault', '!qspi->nvram_write_candidate || qspi->write_fault'), ('fault-latch', 'write_fault_latched = true;', 'write_fault_latched = false;')]: From 7907a2410557e3def4a9b729aa40d201ea8a5cb8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 05:59:37 +0000 Subject: [PATCH 1471/1579] arm64: configs: asahi: Use 4 KiB SPI NOR erase sectors The J700 nvram partition starts at 0x24000 on its boot NOR. SPI-NOR picks the largest uniform erase size unless MTD_SPI_NOR_USE_4K_SECTORS is set, so the flash reports 64 KiB erase blocks and MTD forces the partition read-only because it does not start on an erase boundary. asahi-bless then cannot set a boot volume. There is no device tree or per-flash way to choose the erase size; this option is the only switch, and it is the upstream default. The Fedora base config turns it off. Set it again. The Apple NVRAM partitions are managed in 4 KiB units, and the QSPI NVRAM policy on the J613, J713 and J700 only admits 4 KiB erases anyway. Other SPI NOR users only see smaller erase blocks. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 3c8c4d225985e..7e106922b1c9a 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -51,6 +51,8 @@ CONFIG_APPLE_WATCHDOG=y # m1n1 hands the ADT and its own log over as phram regions (/dev/mtd*). CONFIG_MTD=y CONFIG_MTD_PHRAM=m +# Apple NVRAM partitions are 4 KiB aligned (J700: 0x24000). +CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y CONFIG_VIDEO_APPLE_AVD=m CONFIG_VIDEO_APPLE_AVE=m CONFIG_VIDEO_APPLE_ISP=m From 427136e56f4461a38186a6c6bdc44a3d336030c1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 01:53:55 +0000 Subject: [PATCH 1472/1579] spi: apple-qspi: describe the qualified NVRAM boards as data Each board that may write its NVRAM was qualified by its own compatible check in apple_qspi_nvram_write_board(), and probe then picked the partition bounds, JEDEC ID and block-protect allowance from whether the controller was T8140. The J613/J713 partition was a pair of constants, checked by a second device tree walk, while the J700 one came from the device tree unchecked. Put the qualified boards in one table: controller compatible, machine, controller aperture, the nvram partition the device tree must describe, the flash JEDEC ID and the block-protect bits its firmware leaves set. Every controller now reads its nvram partition from the device tree with the same walk, and a board is a write candidate only when that partition is exactly the one in its table entry. The T8140 multi-line read pass-through comes from the compatible match data. The J613 and J713 keep the 0x700000 partition and the W25Q64 ID, and the J700 keeps 0x24000, the W25Q16 ID and its BP tolerance, now with its partition pinned as well. The KUnit 0x700000 case covers the partition edges against its neighbours, and the host harness reads the table and gains controller, aperture and partition controls. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi-policy-test.c | 42 +++++-- drivers/spi/spi-apple-qspi-policy.h | 3 - drivers/spi/spi-apple-qspi.c | 144 ++++++++++++----------- tools/asahi/qspi-nvram-tests.py | 23 +++- 4 files changed, 125 insertions(+), 87 deletions(-) diff --git a/drivers/spi/spi-apple-qspi-policy-test.c b/drivers/spi/spi-apple-qspi-policy-test.c index b1ecbdbe4e26e..34563b7b37ac4 100644 --- a/drivers/spi/spi-apple-qspi-policy-test.c +++ b/drivers/spi/spi-apple-qspi-policy-test.c @@ -105,14 +105,42 @@ static void qspi_policy_empty(struct kunit *test) NV_START, NV_START)); } -static void qspi_policy_j613(struct kunit *test) +/* The J613 and J713 nvram partition: 0x700000, 1 MiB. */ +#define NV7_START 0x700000ULL +#define NV7_END 0x800000ULL + +static int nv7(unsigned int opcode, unsigned int addr_bytes, + unsigned long long addr, unsigned int len, int data_out) { - unsigned long long s = APPLE_QSPI_NVRAM_START, e = APPLE_QSPI_NVRAM_END; + return apple_qspi_mutation_allowed(opcode, addr_bytes, addr, len, + data_out, NV7_START, NV7_END); +} - KUNIT_EXPECT_TRUE(test, apple_qspi_mutation_allowed(0x02, 3, s, 256, 1, s, e)); - KUNIT_EXPECT_TRUE(test, apple_qspi_mutation_allowed(0x20, 3, e - 0x1000, 0, 0, s, e)); - KUNIT_EXPECT_FALSE(test, apple_qspi_mutation_allowed(0x02, 3, s - 1, 1, 1, s, e)); - KUNIT_EXPECT_FALSE(test, apple_qspi_mutation_allowed(0x20, 3, e, 0, 0, s, e)); +static void qspi_policy_nvram_700000(struct kunit *test) +{ + /* First and last page and sector of the partition. */ + KUNIT_EXPECT_TRUE(test, nv7(0x02, 3, NV7_START, 256, 1)); + KUNIT_EXPECT_TRUE(test, nv7(0x02, 3, NV7_END - 256, 256, 1)); + KUNIT_EXPECT_TRUE(test, nv7(0x20, 3, NV7_START, 0, 0)); + KUNIT_EXPECT_TRUE(test, nv7(0x20, 3, NV7_END - 0x1000, 0, 0)); + /* syscfg ends at 0x700000 and the NOR is 8 MiB: both stay out. */ + KUNIT_EXPECT_FALSE(test, nv7(0x02, 3, NV7_START - 1, 1, 1)); + KUNIT_EXPECT_FALSE(test, nv7(0x02, 3, NV7_START - 256, 256, 1)); + KUNIT_EXPECT_FALSE(test, nv7(0x20, 3, NV7_START - 0x1000, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv7(0x02, 3, NV7_END, 1, 1)); + KUNIT_EXPECT_FALSE(test, nv7(0x02, 3, NV7_END - 1, 2, 1)); + KUNIT_EXPECT_FALSE(test, nv7(0x20, 3, NV7_END, 0, 0)); + /* iBoot at 0 and the J700 window are outside this partition. */ + KUNIT_EXPECT_FALSE(test, nv7(0x02, 3, 0, 256, 1)); + KUNIT_EXPECT_FALSE(test, nv7(0x20, 3, NV_START, 0, 0)); + /* Larger erases and four-byte forms are still refused here. */ + KUNIT_EXPECT_FALSE(test, nv7(0x52, 3, NV7_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv7(0xd8, 3, NV7_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv7(0x21, 4, NV7_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv7(0x12, 4, NV7_START, 256, 1)); + /* An address above 16 MiB is not aliased onto the partition. */ + KUNIT_EXPECT_FALSE(test, nv7(0x02, 3, (1ULL << 24) + NV7_START, 1, 1)); + KUNIT_EXPECT_FALSE(test, nv7(0x20, 3, (1ULL << 24) + NV7_START, 0, 0)); } static void qspi_policy_status(struct kunit *test) @@ -136,7 +164,7 @@ static struct kunit_case apple_qspi_policy_cases[] = { KUNIT_CASE(qspi_policy_refused_commands), KUNIT_CASE(qspi_policy_address_width), KUNIT_CASE(qspi_policy_empty), - KUNIT_CASE(qspi_policy_j613), + KUNIT_CASE(qspi_policy_nvram_700000), KUNIT_CASE(qspi_policy_status), {} }; diff --git a/drivers/spi/spi-apple-qspi-policy.h b/drivers/spi/spi-apple-qspi-policy.h index 12d8ac41dd26c..43c14008a6a32 100644 --- a/drivers/spi/spi-apple-qspi-policy.h +++ b/drivers/spi/spi-apple-qspi-policy.h @@ -2,9 +2,6 @@ #ifndef APPLE_QSPI_POLICY_H #define APPLE_QSPI_POLICY_H -/* The pinned J613/J713 nvram partition. */ -#define APPLE_QSPI_NVRAM_START 0x700000ULL -#define APPLE_QSPI_NVRAM_END 0x800000ULL #define APPLE_QSPI_PAGE_SIZE 256U #define APPLE_QSPI_ERASE_SIZE 4096U diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index e94fe6eb3fd5f..bce570395547d 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -88,9 +88,40 @@ #define APPLE_QSPI_MAX_HZ 40000000 #define APPLE_QSPI_TIMEOUT_MS 1000 -/* Winbond W25Q64 (J613/J713) and W25Q16 (J700) boot NOR JEDEC IDs. */ -static const u8 apple_qspi_jedec_w25q64[3] = { 0xef, 0x65, 0x17 }; -static const u8 apple_qspi_jedec_w25q16[3] = { 0xef, 0x65, 0x15 }; +/* Per-SoC controller differences, from the compatible. */ +struct apple_qspi_soc { + /* T8140: data reads of any shape pass straight to the controller. */ + bool multi_io_reads; +}; + +/* + * A board whose NVRAM may be written: the controller compatible, the + * machine, the controller aperture, the nvram partition the device tree + * must describe, the boot NOR JEDEC ID and the SR1 block-protect bits its + * firmware leaves set at boot. + */ +struct apple_qspi_board { + const char *controller; + const char *machine; + resource_size_t base; + u32 nvram_start; + u32 nvram_size; + u8 jedec[3]; + u8 bp_ok; +}; + +#define APPLE_QSPI_JEDEC_W25Q64 { 0xef, 0x65, 0x17 } +#define APPLE_QSPI_JEDEC_W25Q16 { 0xef, 0x65, 0x15 } + +static const struct apple_qspi_board apple_qspi_boards[] = { + { "apple,t8122-qspi", "apple,j613", 0x2a1118000ULL, + 0x700000, 0x100000, APPLE_QSPI_JEDEC_W25Q64, 0 }, + { "apple,t8132-qspi", "apple,j713", 0x3ad118000ULL, + 0x700000, 0x100000, APPLE_QSPI_JEDEC_W25Q64, 0 }, + /* iBoot leaves BP0 set on the J700, over the top 64 KiB. */ + { "apple,t8140-qspi", "apple,j700", 0x385110000ULL, + 0x24000, 0x1d0000, APPLE_QSPI_JEDEC_W25Q16, 0x1c }, +}; static DEFINE_MUTEX(apple_qspi_policy_lock); static bool read_only = true; @@ -129,7 +160,7 @@ static const struct kernel_param_ops apple_qspi_read_only_ops = { .get = apple_qspi_get_read_only, }; module_param_cb(read_only, &apple_qspi_read_only_ops, &read_only, 0644); -MODULE_PARM_DESC(read_only, "Default true; root runtime opt-in for bounded J613/J713/J700 NVRAM writes"); +MODULE_PARM_DESC(read_only, "Default true; root runtime opt-in for bounded NVRAM writes on qualified boards"); static bool apple_qspi_opcode_is_read(u8 opcode) { @@ -604,7 +635,7 @@ static int apple_qspi_transfer_op(struct spi_mem *mem, return ret; } -/* Private transfers used by the serialized J613 write policy. */ +/* Private transfers used by the serialized NVRAM write policy. */ static int apple_qspi_read_register(struct spi_mem *mem, u8 opcode, u8 *buf, u32 len) { struct spi_mem_op op = SPI_MEM_OP(SPI_MEM_OP_CMD(opcode, 1), @@ -832,55 +863,20 @@ static const struct spi_controller_mem_ops apple_qspi_mem_ops = { }; /* Keep write qualification tied to the board and its controller aperture. */ -static bool apple_qspi_nvram_write_board(struct device_node *node, - resource_size_t base) +static const struct apple_qspi_board * +apple_qspi_nvram_write_board(struct device_node *node, resource_size_t base) { - if (of_device_is_compatible(node, "apple,t8122-qspi")) - return of_machine_is_compatible("apple,j613") && - base == 0x2a1118000ULL; - if (of_device_is_compatible(node, "apple,t8132-qspi")) - return of_machine_is_compatible("apple,j713") && - base == 0x3ad118000ULL; - if (of_device_is_compatible(node, "apple,t8140-qspi")) - return of_machine_is_compatible("apple,j700") && - base == 0x385110000ULL; - return false; -} - -/* Require the pinned partition and single physical chip select. */ -static bool apple_qspi_nvram_layout(struct device_node *node) -{ - struct device_node *flash, *parts, *part; - u32 reg[2], cs; - const char *label; - bool found = false; - - /* OF name matching excludes the unit address (@0); reg pins CS0. */ - flash = of_get_child_by_name(node, "flash"); - if (!flash) - return false; - if (!of_device_is_compatible(flash, "jedec,spi-nor") || - of_property_read_u32(flash, "reg", &cs) || cs) { - of_node_put(flash); - return false; - } - parts = of_get_child_by_name(flash, "partitions"); - of_node_put(flash); - if (!parts) - return false; - if (of_device_is_compatible(parts, "fixed-partitions")) { - for_each_available_child_of_node(parts, part) { - if (!of_property_read_string(part, "label", &label) && - !strcmp(label, "nvram") && - of_property_count_u32_elems(part, "reg") == 2 && - !of_property_read_u32_array(part, "reg", reg, 2) && - reg[0] == APPLE_QSPI_NVRAM_START && - reg[1] == APPLE_QSPI_NVRAM_END - APPLE_QSPI_NVRAM_START) - found = true; - } + const struct apple_qspi_board *board; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(apple_qspi_boards); i++) { + board = &apple_qspi_boards[i]; + if (of_device_is_compatible(node, board->controller) && + of_machine_is_compatible(board->machine) && + base == board->base) + return board; } - of_node_put(parts); - return found; + return NULL; } /* @@ -941,30 +937,26 @@ static void apple_qspi_nvram_window(struct device *dev, struct apple_qspi *qspi) static int apple_qspi_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + const struct apple_qspi_board *board; + const struct apple_qspi_soc *soc; struct spi_controller *ctlr; struct apple_qspi *qspi; struct resource *res; u32 depths, version; int irq, ret; + soc = of_device_get_match_data(dev); + if (!soc) + return -ENODEV; + ctlr = devm_spi_alloc_host(dev, sizeof(*qspi)); if (!ctlr) return -ENOMEM; qspi = spi_controller_get_devdata(ctlr); qspi->nvram_policy = true; - qspi->nvram_window = of_device_is_compatible(dev->of_node, "apple,t8140-qspi"); - if (qspi->nvram_window) { - /* J700: W25Q16JV; iBoot leaves BP0 set over the top 64 KiB. */ - apple_qspi_nvram_window(dev, qspi); - memcpy(qspi->jedec, apple_qspi_jedec_w25q16, sizeof(qspi->jedec)); - qspi->bp_ok = 0x1c; - } else { - qspi->window_start = APPLE_QSPI_NVRAM_START; - qspi->window_end = APPLE_QSPI_NVRAM_END; - memcpy(qspi->jedec, apple_qspi_jedec_w25q64, sizeof(qspi->jedec)); - qspi->bp_ok = 0; - } + qspi->nvram_window = soc->multi_io_reads; + apple_qspi_nvram_window(dev, qspi); if (of_device_is_compatible(dev->of_node, "apple,t8122-qspi") && !of_machine_is_compatible("apple,j613")) return -ENODEV; @@ -976,10 +968,14 @@ static int apple_qspi_probe(struct platform_device *pdev) return dev_err_probe(dev, -EINVAL, "QSPIMC aperture is too small\n"); - qspi->nvram_write_candidate = apple_qspi_nvram_write_board(dev->of_node, - res->start) && - (qspi->nvram_window ? qspi->window_start < qspi->window_end : - apple_qspi_nvram_layout(dev->of_node)); + /* The device tree must describe exactly the board's nvram partition. */ + board = apple_qspi_nvram_write_board(dev->of_node, res->start); + if (board && qspi->window_start == board->nvram_start && + qspi->window_end == (u64)board->nvram_start + board->nvram_size) { + memcpy(qspi->jedec, board->jedec, sizeof(qspi->jedec)); + qspi->bp_ok = board->bp_ok; + qspi->nvram_write_candidate = true; + } qspi->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(qspi->base)) return PTR_ERR(qspi->base); @@ -1125,10 +1121,16 @@ static int apple_qspi_resume(struct device *dev) static DEFINE_SIMPLE_DEV_PM_OPS(apple_qspi_pm_ops, apple_qspi_suspend, apple_qspi_resume); +static const struct apple_qspi_soc apple_qspi_single_io = { }; + +static const struct apple_qspi_soc apple_qspi_t8140 = { + .multi_io_reads = true, +}; + static const struct of_device_id apple_qspi_of_match[] = { - { .compatible = "apple,t8122-qspi" }, - { .compatible = "apple,t8132-qspi" }, - { .compatible = "apple,t8140-qspi" }, + { .compatible = "apple,t8122-qspi", .data = &apple_qspi_single_io }, + { .compatible = "apple,t8132-qspi", .data = &apple_qspi_single_io }, + { .compatible = "apple,t8140-qspi", .data = &apple_qspi_t8140 }, {} }; MODULE_DEVICE_TABLE(of, apple_qspi_of_match); diff --git a/tools/asahi/qspi-nvram-tests.py b/tools/asahi/qspi-nvram-tests.py index 427d0cd8479c0..930e41a1354bb 100644 --- a/tools/asahi/qspi-nvram-tests.py +++ b/tools/asahi/qspi-nvram-tests.py @@ -5,6 +5,7 @@ import argparse import contextlib import json +import re import subprocess import tempfile @@ -145,9 +146,15 @@ def function(source, signature): for(unsigned int c=0;cnvram_start==0x700000&&qb->nvram_size==0x100000&&!memcmp(qb->jedec,jedec,3)&&!qb->bp_ok); + machine="apple,j700";qb=apple_qspi_nvram_write_board(&t8140,0x385110000ULL); + assert(qb&&qb->nvram_start==0x24000&&qb->nvram_size==0x1d0000&&qb->jedec[2]==0x15&&qb->bp_ok==0x1c); puts("PASS J613/J713/J700 board, controller and full-width aperture qualification (60 combinations)"); u8 data[256];memset(data,0x55,sizeof(data));reset(); struct spi_mem_op wren=SPI_MEM_OP(SPI_MEM_OP_CMD(0x06,1),SPI_MEM_OP_NO_ADDR,SPI_MEM_OP_NO_DUMMY,SPI_MEM_OP_NO_DATA); @@ -245,15 +252,18 @@ def main(): args.output.mkdir(parents=True, exist_ok=True) source = (ROOT / 'drivers/spi/spi-apple-qspi.c').read_text() policy = (ROOT / 'drivers/spi/spi-apple-qspi-policy.h').read_text() + boards = (''.join(m.group(0) + '\n' for m in re.finditer(r'#define APPLE_QSPI_JEDEC_\w+.*', source)) + + function(source, 'struct apple_qspi_board {').rstrip() + ';\n' + + function(source, 'static const struct apple_qspi_board apple_qspi_boards[]').rstrip() + ';\n') signatures = ['static int apple_qspi_set_read_only(', 'static int apple_qspi_get_read_only(', - 'static bool apple_qspi_nvram_write_board(', + 'static const struct apple_qspi_board *\napple_qspi_nvram_write_board(', 'static bool apple_qspi_opcode_is_read(', 'static int apple_qspi_width_mode(', 'static int apple_qspi_shape_op(', 'static bool apple_qspi_supports_op(', 'static int apple_qspi_read_register(', 'static int apple_qspi_command(', 'static int apple_qspi_read_array(', 'static int apple_qspi_status(', 'static int apple_qspi_nvram_modify(', 'static int apple_qspi_exec_op(', 'static int apple_qspi_mem_exec_op('] - code = PREFIX + policy + '\n'.join(function(source, s) for s in signatures) + CONTROLS + code = PREFIX + policy + boards + '\n'.join(function(source, s) for s in signatures) + CONTROLS output = contextlib.nullcontext(str(args.output)) if args.output else tempfile.TemporaryDirectory(prefix='qspi-nvram-') with output as directory: current = run(code, Path(directory) / 'current.c') @@ -261,9 +271,10 @@ def main(): if current.returncode: raise SystemExit(current.stderr) for name, old, new in [ - ('j713-board', 'of_machine_is_compatible("apple,j713")', 'true'), - ('j713-aperture', 'base == 0x3ad118000ULL', 'base != 0'), - ('j700-board', 'of_machine_is_compatible("apple,j700")', 'true'), + ('controller', 'of_device_is_compatible(node, board->controller) &&', ''), + ('board', 'of_machine_is_compatible(board->machine) &&', ''), + ('aperture', 'base == board->base', 'base != 0'), + ('j713-partition', '"apple,j713", 0x3ad118000ULL,\n\t 0x700000', '"apple,j713", 0x3ad118000ULL,\n\t 0x680000'), ('range', 'addr >= start &&', 'addr >= 1 &&'), ('bp-ok', '!(sr1 & 0x1c & ~bp_ok)', '1'), ('read-only', 'read_only || !qspi->nvram_write_candidate || qspi->write_fault', From 5c9287e18940bcc86a7696aa3f756d6c2d53b0e1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 01:54:06 +0000 Subject: [PATCH 1473/1579] spi: apple-qspi: add T6041 and admit NVRAM writes on the J616s The M4 Pro (T6041) boot NOR sits behind the same QSPIMC controller as the M3, M4 and A18 Pro parts, at 0x429118000. The J616s ADT places its nvram partition (anvram, "nvram,nor") at 0x700000 with a size of 1 MiB, the J613 and J713 layout, next to iboot at 0 and syscfg at 0x680000. Without a T6041 match the driver does not bind, so asahi-bless cannot select a boot volume from Linux. Match apple,t6041-qspi with single-line transfers, as on T8122 and T8132. Qualify the J616s for the verified NVRAM policy at its controller aperture and that partition, with the W25Q64 ID the other 0x700000 boards use and no block-protect bits tolerated. The flash ID is checked before every program or erase, so a different part is refused before any write enable reaches it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/Kconfig | 7 ++++--- drivers/spi/spi-apple-qspi-policy-test.c | 4 ++-- drivers/spi/spi-apple-qspi.c | 11 +++++++---- tools/asahi/qspi-nvram-tests.py | 17 +++++++++-------- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index 8b53e2cf36b8d..dbaf0d6949249 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -146,10 +146,11 @@ config SPI_APPLE_QSPI help This enables support for Apple's command-FIFO QSPIMC controller. Unlike the ordinary Apple SPIMC block, QSPIMC accepts packed SPI - memory descriptors and is used for the boot NOR on T8140 systems. + memory descriptors and is used for the boot NOR on M3 and later + systems. - The boot NOR defaults to read-only access. On the J613, J713 and - J700 a runtime opt-in admits bounded, verified page programs and + The boot NOR defaults to read-only access. On the J613, J713, J616s + and J700 a runtime opt-in admits bounded, verified page programs and 4 KiB erases inside the nvram partition only; the rest of the boot NOR stays read-only. diff --git a/drivers/spi/spi-apple-qspi-policy-test.c b/drivers/spi/spi-apple-qspi-policy-test.c index 34563b7b37ac4..8271f7a19f06d 100644 --- a/drivers/spi/spi-apple-qspi-policy-test.c +++ b/drivers/spi/spi-apple-qspi-policy-test.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* * KUnit tests for the Apple QSPIMC NVRAM mutation policy, at the J700 and - * J613/J713 nvram partitions. + * J613/J713/J616s nvram partitions. */ #include @@ -105,7 +105,7 @@ static void qspi_policy_empty(struct kunit *test) NV_START, NV_START)); } -/* The J613 and J713 nvram partition: 0x700000, 1 MiB. */ +/* The J613, J713 and J616s nvram partition: 0x700000, 1 MiB. */ #define NV7_START 0x700000ULL #define NV7_END 0x800000ULL diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index bce570395547d..1c4d6794803ca 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -7,10 +7,10 @@ * command FIFO. This driver intentionally implements the PIO spi-mem path * only; that is sufficient for SPI NOR discovery and access. * - * The boot NOR starts read-only. On the qualified boards (J613, J713 and - * the J700) root may opt in at runtime to bounded, verified programs and - * 4 KiB erases inside the nvram partition; everything else stays - * read-only. On T8140 data reads reach the whole NOR. + * The boot NOR starts read-only. On the qualified boards (J613, J713, + * J616s and the J700) root may opt in at runtime to bounded, verified + * programs and 4 KiB erases inside the nvram partition; everything else + * stays read-only. On T8140 data reads reach the whole NOR. */ #include @@ -118,6 +118,8 @@ static const struct apple_qspi_board apple_qspi_boards[] = { 0x700000, 0x100000, APPLE_QSPI_JEDEC_W25Q64, 0 }, { "apple,t8132-qspi", "apple,j713", 0x3ad118000ULL, 0x700000, 0x100000, APPLE_QSPI_JEDEC_W25Q64, 0 }, + { "apple,t6041-qspi", "apple,j616s", 0x429118000ULL, + 0x700000, 0x100000, APPLE_QSPI_JEDEC_W25Q64, 0 }, /* iBoot leaves BP0 set on the J700, over the top 64 KiB. */ { "apple,t8140-qspi", "apple,j700", 0x385110000ULL, 0x24000, 0x1d0000, APPLE_QSPI_JEDEC_W25Q16, 0x1c }, @@ -1130,6 +1132,7 @@ static const struct apple_qspi_soc apple_qspi_t8140 = { static const struct of_device_id apple_qspi_of_match[] = { { .compatible = "apple,t8122-qspi", .data = &apple_qspi_single_io }, { .compatible = "apple,t8132-qspi", .data = &apple_qspi_single_io }, + { .compatible = "apple,t6041-qspi", .data = &apple_qspi_single_io }, { .compatible = "apple,t8140-qspi", .data = &apple_qspi_t8140 }, {} }; diff --git a/tools/asahi/qspi-nvram-tests.py b/tools/asahi/qspi-nvram-tests.py index 930e41a1354bb..3939be8208c5b 100644 --- a/tools/asahi/qspi-nvram-tests.py +++ b/tools/asahi/qspi-nvram-tests.py @@ -138,24 +138,24 @@ def function(source, signature): static bool sent(unsigned int o){for(unsigned int i=0;invram_start==0x700000&&qb->nvram_size==0x100000&&!memcmp(qb->jedec,jedec,3)&&!qb->bp_ok); machine="apple,j700";qb=apple_qspi_nvram_write_board(&t8140,0x385110000ULL); assert(qb&&qb->nvram_start==0x24000&&qb->nvram_size==0x1d0000&&qb->jedec[2]==0x15&&qb->bp_ok==0x1c); - puts("PASS J613/J713/J700 board, controller and full-width aperture qualification (60 combinations)"); + puts("PASS J613/J713/J700/J616s board, controller and full-width aperture qualification (140 combinations)"); u8 data[256];memset(data,0x55,sizeof(data));reset(); struct spi_mem_op wren=SPI_MEM_OP(SPI_MEM_OP_CMD(0x06,1),SPI_MEM_OP_NO_ADDR,SPI_MEM_OP_NO_DUMMY,SPI_MEM_OP_NO_DATA); /* T8140 (J700) shares the verified policy at its DT nvram window; its data @@ -274,7 +274,8 @@ def main(): ('controller', 'of_device_is_compatible(node, board->controller) &&', ''), ('board', 'of_machine_is_compatible(board->machine) &&', ''), ('aperture', 'base == board->base', 'base != 0'), - ('j713-partition', '"apple,j713", 0x3ad118000ULL,\n\t 0x700000', '"apple,j713", 0x3ad118000ULL,\n\t 0x680000'), + ('j616s-aperture', '"apple,j616s", 0x429118000ULL', '"apple,j616s", 0x429118800ULL'), + ('j616s-partition', '"apple,j616s", 0x429118000ULL,\n\t 0x700000', '"apple,j616s", 0x429118000ULL,\n\t 0x680000'), ('range', 'addr >= start &&', 'addr >= 1 &&'), ('bp-ok', '!(sr1 & 0x1c & ~bp_ok)', '1'), ('read-only', 'read_only || !qspi->nvram_write_candidate || qspi->write_fault', From 3d9601c360bfb63b89bee2226b2d5d3fd9f86d26 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 01:54:14 +0000 Subject: [PATCH 1474/1579] arm64: dts: apple: t6040-j616s: enable the boot NOR and its nvram Describe the J616s boot NOR on the QSPIMC controller with the nvram partition from its ADT (/arm-io/qspi/spinor/anvram, 0x700000, 1 MiB), so asahi-nvram and asahi-bless can find it. Use single-line transfers at 10 MHz, as on the J713. Add the nvram and spi0 aliases the J700 also carries. The SoC node stays disabled for boards that do not describe their flash. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 29 +++++++++++++++++++++++ arch/arm64/boot/dts/apple/t6040.dtsi | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 69f0070af7659..d9ee12939c232 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -21,6 +21,8 @@ aliases { keyboard = &keyboard; + nvram = &nvram; + spi0 = &qspi; wifi0 = "/pcie@1cb0000000/pci@0,0/wifi@0,0"; bluetooth0 = "/pcie@1cb0000000/pci@0,0/bluetooth@0,1"; /* used by the loader to hand over the display */ @@ -1430,6 +1432,33 @@ status = "okay"; }; +/* + * Boot NOR. The partition is the J616s ADT /arm-io/qspi/spinor/anvram, + * flash-relative; asahi-bless and asahi-nvram find it by its label. + */ +&qspi { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <10000000>; + spi-rx-bus-width = <1>; + spi-tx-bus-width = <1>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + nvram: partition@700000 { + label = "nvram"; + reg = <0x700000 0x100000>; + }; + }; + }; +}; + /* * Touch ID sensor (Mesa 0x3356). Its power is NUB GPIO 53 and its * data-ready line is AP GPIO 116, as the J616s ADT describes them. diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 6c54e05caa743..f2c603b3f01e6 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -2124,7 +2124,7 @@ power-domains = <&ps_qspi>; #address-cells = <1>; #size-cells = <0>; - /* The firmware NOR needs a qualified partition/write policy. */ + /* Boards enable it with their flash and its nvram partition. */ status = "disabled"; }; From eac79241035dcc1a1268fb7e96e833e1adea8fa4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:09:04 +0000 Subject: [PATCH 1475/1579] spi: apple-qspi: use the J616s Macronix boot NOR ID and registers macOS reports the J616s boot NOR JEDEC ID as c22b27, a Macronix part, not the Winbond W25Q64 the J613 and J713 carry. Pin that ID in the J616s table entry. The write policy and the flash_status attribute read status registers with Winbond opcodes: 05h, 35h and 15h. On Macronix parts 35h is EQIO, which switches the flash to QPI mode, and the status register keeps four block-protect bits, BP[3:0] at bits 5:2, where Winbond has three and TB. For a Macronix part, read only the status (05h) and configuration (15h) registers, refuse 35h from SPI-NOR as well, and check WIP, WEL and all four BP bits before a write. flash_status reads the JEDEC ID first and sends 35h only to Winbond parts. The J616s still admits only 3-byte page programs and 4 KiB sector erases (20h) inside its nvram partition, with readback verification. SPI-NOR has no table entry for c22b27; it binds through the generic Macronix entry and takes geometry and erase types from SFDP. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi-policy-test.c | 19 +++++ drivers/spi/spi-apple-qspi-policy.h | 14 ++++ drivers/spi/spi-apple-qspi.c | 89 ++++++++++++++++++------ tools/asahi/qspi-nvram-tests.py | 20 +++++- 4 files changed, 119 insertions(+), 23 deletions(-) diff --git a/drivers/spi/spi-apple-qspi-policy-test.c b/drivers/spi/spi-apple-qspi-policy-test.c index 8271f7a19f06d..f6dc555057443 100644 --- a/drivers/spi/spi-apple-qspi-policy-test.c +++ b/drivers/spi/spi-apple-qspi-policy-test.c @@ -157,6 +157,24 @@ static void qspi_policy_status(struct kunit *test) KUNIT_EXPECT_FALSE(test, apple_qspi_status_allows_write(0, 0, 0x04, 0x1c)); } +static void qspi_policy_mx_status(struct kunit *test) +{ + /* The J616s Macronix part: no block protection is tolerated. */ + KUNIT_EXPECT_TRUE(test, apple_qspi_mx_status_allows_write(0x00, 0)); + /* QE and SRWD stay as found and do not block a write. */ + KUNIT_EXPECT_TRUE(test, apple_qspi_mx_status_allows_write(0x40, 0)); + KUNIT_EXPECT_TRUE(test, apple_qspi_mx_status_allows_write(0x80, 0)); + /* WIP and WEL are never admitted. */ + KUNIT_EXPECT_FALSE(test, apple_qspi_mx_status_allows_write(0x01, 0x3c)); + KUNIT_EXPECT_FALSE(test, apple_qspi_mx_status_allows_write(0x02, 0x3c)); + /* Each of BP0 to BP3, including BP3 at bit 5, which Winbond calls TB. */ + KUNIT_EXPECT_FALSE(test, apple_qspi_mx_status_allows_write(0x04, 0)); + KUNIT_EXPECT_FALSE(test, apple_qspi_mx_status_allows_write(0x08, 0)); + KUNIT_EXPECT_FALSE(test, apple_qspi_mx_status_allows_write(0x10, 0)); + KUNIT_EXPECT_FALSE(test, apple_qspi_mx_status_allows_write(0x20, 0)); + KUNIT_EXPECT_TRUE(test, apple_qspi_mx_status_allows_write(0x3c, 0x3c)); +} + static struct kunit_case apple_qspi_policy_cases[] = { KUNIT_CASE(qspi_policy_write_enable), KUNIT_CASE(qspi_policy_program), @@ -166,6 +184,7 @@ static struct kunit_case apple_qspi_policy_cases[] = { KUNIT_CASE(qspi_policy_empty), KUNIT_CASE(qspi_policy_nvram_700000), KUNIT_CASE(qspi_policy_status), + KUNIT_CASE(qspi_policy_mx_status), {} }; diff --git a/drivers/spi/spi-apple-qspi-policy.h b/drivers/spi/spi-apple-qspi-policy.h index 43c14008a6a32..5acd9a62d5291 100644 --- a/drivers/spi/spi-apple-qspi-policy.h +++ b/drivers/spi/spi-apple-qspi-policy.h @@ -62,4 +62,18 @@ static inline int apple_qspi_status_allows_write(unsigned int sr1, return !(sr1 & 0x03) && !(sr1 & 0x1c & ~bp_ok) && !(sr2 & 0xc0) && !(sr3 & 0x04); } + +/* JEDEC manufacturer ID of Macronix parts. */ +#define APPLE_QSPI_MFR_MACRONIX 0xc2 + +/* + * The Macronix status register holds WIP, WEL and four block-protect bits, + * BP[3:0] at bits 5:2. BUSY and WEL are always refused, and block-protect + * bits only where bp_ok allows them. QE and SRWD stay unchanged. + */ +static inline int apple_qspi_mx_status_allows_write(unsigned int sr, + unsigned int bp_ok) +{ + return !(sr & 0x03) && !(sr & 0x3c & ~bp_ok); +} #endif diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index 1c4d6794803ca..f4b4b6a0d5202 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -112,14 +112,16 @@ struct apple_qspi_board { #define APPLE_QSPI_JEDEC_W25Q64 { 0xef, 0x65, 0x17 } #define APPLE_QSPI_JEDEC_W25Q16 { 0xef, 0x65, 0x15 } +#define APPLE_QSPI_JEDEC_MX_2B27 { 0xc2, 0x2b, 0x27 } static const struct apple_qspi_board apple_qspi_boards[] = { { "apple,t8122-qspi", "apple,j613", 0x2a1118000ULL, 0x700000, 0x100000, APPLE_QSPI_JEDEC_W25Q64, 0 }, { "apple,t8132-qspi", "apple,j713", 0x3ad118000ULL, 0x700000, 0x100000, APPLE_QSPI_JEDEC_W25Q64, 0 }, + /* Macronix; the ID macOS reports for the J616s boot NOR. */ { "apple,t6041-qspi", "apple,j616s", 0x429118000ULL, - 0x700000, 0x100000, APPLE_QSPI_JEDEC_W25Q64, 0 }, + 0x700000, 0x100000, APPLE_QSPI_JEDEC_MX_2B27, 0 }, /* iBoot leaves BP0 set on the J700, over the top 64 KiB. */ { "apple,t8140-qspi", "apple,j700", 0x385110000ULL, 0x24000, 0x1d0000, APPLE_QSPI_JEDEC_W25Q16, 0x1c }, @@ -664,12 +666,24 @@ static int apple_qspi_read_array(struct spi_mem *mem, u64 addr, u8 *buf, u32 len return apple_qspi_transfer_op(mem, &op); } -static int apple_qspi_status(struct spi_mem *mem, u8 sr[3]) +/* + * Winbond parts have three status registers (05h, 35h, 15h). On Macronix + * parts 35h is EQIO, which switches the flash to QPI mode, so only the + * status (05h) and configuration (15h) registers are read there. + */ +static int apple_qspi_status(struct spi_mem *mem, u8 mfr, u8 sr[3]) { - static const u8 opcode[] = { 0x05, 0x35, 0x15 }; - int i, ret; - - for (i = 0; i < 3; i++) { + static const u8 winbond[] = { 0x05, 0x35, 0x15 }; + static const u8 macronix[] = { 0x05, 0x15 }; + const u8 *opcode = winbond; + int i, n = ARRAY_SIZE(winbond), ret; + + if (mfr == APPLE_QSPI_MFR_MACRONIX) { + opcode = macronix; + n = ARRAY_SIZE(macronix); + } + sr[2] = 0; + for (i = 0; i < n; i++) { ret = apple_qspi_read_register(mem, opcode[i], &sr[i], 1); if (ret) return ret; @@ -697,11 +711,13 @@ static int apple_qspi_nvram_modify(struct spi_mem *mem, const struct spi_mem_op ret = -ENODEV; goto out; } - ret = apple_qspi_status(mem, before); + ret = apple_qspi_status(mem, id[0], before); if (ret) goto out; /* No unlocking. Conservatively refuse BP/CMP/WPS, BUSY/WEL or suspend. */ - if (!apple_qspi_status_allows_write(before[0], before[1], before[2], + if (id[0] == APPLE_QSPI_MFR_MACRONIX ? + !apple_qspi_mx_status_allows_write(before[0], qspi->bp_ok) : + !apple_qspi_status_allows_write(before[0], before[1], before[2], qspi->bp_ok)) { ret = -EACCES; goto out; @@ -755,7 +771,7 @@ static int apple_qspi_nvram_modify(struct spi_mem *mem, const struct spi_mem_op ret = apple_qspi_command(mem, 0x04); if (ret) goto fault; - ret = apple_qspi_status(mem, after); + ret = apple_qspi_status(mem, id[0], after); if (ret) goto fault; if (memcmp(before, after, sizeof(before))) { @@ -804,6 +820,12 @@ static int apple_qspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) if (!apple_qspi_supports_op(mem, op) || apple_qspi_shape_op(op, &shape)) return -EOPNOTSUPP; mutex_lock(&apple_qspi_policy_lock); + /* 35h is EQIO, not a register read, on Macronix parts. */ + if (op->cmd.opcode == 0x35 && + qspi->jedec[0] == APPLE_QSPI_MFR_MACRONIX) { + ret = -EOPNOTSUPP; + goto out; + } if (op->data.dir == SPI_MEM_DATA_IN && op->data.nbytes && apple_qspi_opcode_is_read(op->cmd.opcode)) { ret = apple_qspi_transfer_op(mem, op); @@ -1043,15 +1065,24 @@ static int apple_qspi_match_cs0(struct device *dev, const void *unused) !spi_get_chipselect(to_spi_device(dev), 0); } +static int apple_qspi_flash_read_reg(struct spi_mem *mem, u8 opcode, + u8 *buf, unsigned int len) +{ + struct spi_mem_op op = SPI_MEM_OP(SPI_MEM_OP_CMD(opcode, 1), + SPI_MEM_OP_NO_ADDR, SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_IN(len, buf, 1)); + + return spi_mem_exec_op(mem, &op); +} + static ssize_t flash_status_show(struct device *dev, struct device_attribute *attr, char *buf) { - static const u8 opcodes[] = { 0x9f, 0x05, 0x35, 0x15 }; struct spi_controller *ctlr = dev_get_drvdata(dev); struct spi_mem mem = { }; struct device *child; u8 *data; - int i, ret; + int ret; child = device_find_child(&ctlr->dev, NULL, apple_qspi_match_cs0); if (!child) @@ -1065,18 +1096,34 @@ static ssize_t flash_status_show(struct device *dev, /* spi-mem supplies queue, bus and I/O serialization for each read. * These are sequential samples, not an atomic protection snapshot. + * The JEDEC ID decides which registers exist: 35h is SR2 on Winbond + * parts but EQIO on Macronix ones, so it is sent to Winbond only. */ - for (i = 0; i < ARRAY_SIZE(opcodes); i++) { - struct spi_mem_op op = SPI_MEM_OP(SPI_MEM_OP_CMD(opcodes[i], 1), - SPI_MEM_OP_NO_ADDR, SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_DATA_IN(i ? 1 : 3, data + (i ? i + 2 : 0), 1)); - - ret = spi_mem_exec_op(&mem, &op); - if (ret) - goto out_free; + ret = apple_qspi_flash_read_reg(&mem, 0x9f, data, 3) ?: + apple_qspi_flash_read_reg(&mem, 0x05, data + 3, 1); + if (ret) + goto out_free; + switch (data[0]) { + case 0xef: + ret = apple_qspi_flash_read_reg(&mem, 0x35, data + 4, 1) ?: + apple_qspi_flash_read_reg(&mem, 0x15, data + 5, 1); + if (!ret) + ret = sysfs_emit(buf, "jedec=%02x%02x%02x sr1=%02x sr2=%02x sr3=%02x\n", + data[0], data[1], data[2], data[3], + data[4], data[5]); + break; + case APPLE_QSPI_MFR_MACRONIX: + ret = apple_qspi_flash_read_reg(&mem, 0x15, data + 4, 1); + if (!ret) + ret = sysfs_emit(buf, "jedec=%02x%02x%02x sr=%02x cr=%02x\n", + data[0], data[1], data[2], data[3], + data[4]); + break; + default: + ret = sysfs_emit(buf, "jedec=%02x%02x%02x sr1=%02x\n", + data[0], data[1], data[2], data[3]); + break; } - ret = sysfs_emit(buf, "jedec=%02x%02x%02x sr1=%02x sr2=%02x sr3=%02x\n", - data[0], data[1], data[2], data[3], data[4], data[5]); out_free: kfree(data); out_put: diff --git a/tools/asahi/qspi-nvram-tests.py b/tools/asahi/qspi-nvram-tests.py index 3939be8208c5b..b458f3b06f3a8 100644 --- a/tools/asahi/qspi-nvram-tests.py +++ b/tools/asahi/qspi-nvram-tests.py @@ -98,7 +98,7 @@ def function(source, signature): switch(o->cmd.opcode){ case 0x9f:memcpy(o->data.buf.in,jedec,o->data.nbytes);break; case 0x05:*(u8*)o->data.buf.in=sr[0]|(stuck_busy&&programmed?1:0);break; - case 0x35:*(u8*)o->data.buf.in=sr[1];break; + case 0x35:assert(jedec[0]!=0xc2);*(u8*)o->data.buf.in=sr[1];break; case 0x15:*(u8*)o->data.buf.in=sr[2];break; case 0x03: assert(o->addr.val+o->data.nbytes<=sizeof(flash)); @@ -152,7 +152,7 @@ def function(source, signature): const struct apple_qspi_board *qb; struct device_node t6041={"apple,t6041-qspi"},t8140={"apple,t8140-qspi"}; machine="apple,j616s";qb=apple_qspi_nvram_write_board(&t6041,0x429118000ULL); - assert(qb&&qb->nvram_start==0x700000&&qb->nvram_size==0x100000&&!memcmp(qb->jedec,jedec,3)&&!qb->bp_ok); + assert(qb&&qb->nvram_start==0x700000&&qb->nvram_size==0x100000&&qb->jedec[0]==0xc2&&qb->jedec[1]==0x2b&&qb->jedec[2]==0x27&&!qb->bp_ok); machine="apple,j700";qb=apple_qspi_nvram_write_board(&t8140,0x385110000ULL); assert(qb&&qb->nvram_start==0x24000&&qb->nvram_size==0x1d0000&&qb->jedec[2]==0x15&&qb->bp_ok==0x1c); puts("PASS J613/J713/J700/J616s board, controller and full-width aperture qualification (140 combinations)"); @@ -223,6 +223,19 @@ def function(source, signature): assert(sent(0x04)&&apple_qspi_set_read_only("N",¶meter)==-EPERM);denied(o); } puts("PASS read-only startup, exact opcode/range/page/erase admission, WREN ordering, readback and fault latch"); + /* J616s Macronix part: 35h (EQIO there) never reaches the wire. */ + reset();jedec[0]=0xc2;jedec[1]=0x2b;jedec[2]=0x27;memcpy(q.jedec,jedec,3);arm();memset(data,0x55,sizeof(data)); + o=program(0x700000,256,data);wire_count=0;assert(!apple_qspi_exec_op(&mem,&o)&&sent(0x02)&&!sent(0x35)&&sent(0x15)); + assert(!memcmp(flash+0x700000,data,256)&&!sr[0]); + erase.addr.val=0x701000; + wire_count=0;assert(!apple_qspi_exec_op(&mem,&erase)&&sent(0x20)&&!sent(0x35)); + struct spi_mem_op rdcr2=SPI_MEM_OP(SPI_MEM_OP_CMD(0x35,1),SPI_MEM_OP_NO_ADDR,SPI_MEM_OP_NO_DUMMY,SPI_MEM_OP_DATA_IN(1,id,1)); + wire_count=0;assert(apple_qspi_exec_op(&mem,&rdcr2)<0&&!wire_count); + for(unsigned int bp=0x04;bp<=0x20;bp<<=1){reset();jedec[0]=0xc2;jedec[1]=0x2b;jedec[2]=0x27;memcpy(q.jedec,jedec,3);arm(); + sr[0]=bp;wire_count=0;o=program(0x700000,1,data);assert(apple_qspi_exec_op(&mem,&o)==-EACCES&&!sent(0x06)&&!sent(0x35));} + reset();memcpy(q.jedec,(u8[]){0xc2,0x2b,0x27},3);arm();wire_count=0;o=program(0x700000,1,data); + assert(apple_qspi_exec_op(&mem,&o)==-ENODEV&&!sent(0x06)); + puts("PASS Macronix J616s part: no 35h on the wire, SR/CR checks, BP0-BP3 refusal and JEDEC mismatch"); return 0; } ''' @@ -278,6 +291,9 @@ def main(): ('j616s-partition', '"apple,j616s", 0x429118000ULL,\n\t 0x700000', '"apple,j616s", 0x429118000ULL,\n\t 0x680000'), ('range', 'addr >= start &&', 'addr >= 1 &&'), ('bp-ok', '!(sr1 & 0x1c & ~bp_ok)', '1'), + ('mx-bp3', '!(sr & 0x3c & ~bp_ok)', '!(sr & 0x1c & ~bp_ok)'), + ('mx-eqio', 'opcode = macronix;', 'opcode = winbond;'), + ('mx-eqio-read', 'qspi->jedec[0] == APPLE_QSPI_MFR_MACRONIX) {', '0) {'), ('read-only', 'read_only || !qspi->nvram_write_candidate || qspi->write_fault', '!qspi->nvram_write_candidate || qspi->write_fault'), ('fault-latch', 'write_fault_latched = true;', 'write_fault_latched = false;')]: From 777dc885980c40da93666c42acb66617a2f27071 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 04:09:29 +0000 Subject: [PATCH 1476/1579] spi: apple-qspi: runtime suspend the T8140 controller The T8140 boot NOR controller keeps its genpd domain active between accesses. Enable runtime PM only for this SoC, independently of the board qualification and NVRAM write policy. Let spi-mem acquire the runtime reference around exec_op. This covers all private JEDEC/status reads, WREN, completion polling, WRDI and readback in a verified mutation. Mark the controller busy after the whole operation, since spi-mem releases its reference with a plain pm_runtime_put(). Use a 100 ms autosuspend delay to group successive 4 KiB read chunks and userspace NVRAM access bursts without retaining the domain for long idle periods. Hold an initial reference through registration, NOR discovery and the last probe MMIO, and balance it on failure. Drain and disable the IRQ before genpd can gate the block. On resume, reuse the reset helper to restore timing, FIFO thresholds and interrupt state before enabling the IRQ; mode, divider and transfer descriptors are set per operation. A failed reset leaves the IRQ disabled and prevents the memory access. Use the force helpers for system sleep so an already suspended block is not accessed. Keep the SPI queue rollback and reopen behavior, and leave other SoCs on their existing system-sleep path. Extend the host harness to check that read, refusal, verified program and error paths update the idle stamp under the outer SPI PM reference. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi.c | 93 +++++++++++++++++++++++++++------ tools/asahi/qspi-nvram-tests.py | 36 ++++++++++++- 2 files changed, 112 insertions(+), 17 deletions(-) diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index f4b4b6a0d5202..601929b85f1fd 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -87,11 +88,13 @@ #define APPLE_QSPI_MAX_DATA 4096 #define APPLE_QSPI_MAX_HZ 40000000 #define APPLE_QSPI_TIMEOUT_MS 1000 +#define APPLE_QSPI_AUTOSUSPEND_MS 100 /* Per-SoC controller differences, from the compatible. */ struct apple_qspi_soc { /* T8140: data reads of any shape pass straight to the controller. */ bool multi_io_reads; + bool runtime_pm; }; /* @@ -869,15 +872,21 @@ static int apple_qspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) static int apple_qspi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) { - struct apple_qspi *qspi = spi_controller_get_devdata(mem->spi->controller); + struct spi_controller *ctlr = mem->spi->controller; + struct apple_qspi *qspi = spi_controller_get_devdata(ctlr); + int ret; /* T8140 keeps its multi-line data reads; spi-mem serializes them. */ - if (qspi->nvram_window && op->data.dir == SPI_MEM_DATA_IN && - op->data.nbytes) - return apple_qspi_transfer_op(mem, op); - if (!qspi->nvram_policy) - return apple_qspi_transfer_op(mem, op); - return apple_qspi_exec_op(mem, op); + if (!qspi->nvram_policy || (qspi->nvram_window && + op->data.dir == SPI_MEM_DATA_IN && + op->data.nbytes)) + ret = apple_qspi_transfer_op(mem, op); + else + ret = apple_qspi_exec_op(mem, op); + /* spi-mem drops its runtime reference with pm_runtime_put(). */ + if (ctlr->auto_runtime_pm) + pm_runtime_mark_last_busy(ctlr->dev.parent); + return ret; } static const struct spi_controller_mem_ops apple_qspi_mem_ops = { @@ -1043,10 +1052,26 @@ static int apple_qspi_probe(struct platform_device *pdev) ctlr->mem_ops = &apple_qspi_mem_ops; platform_set_drvdata(pdev, ctlr); + /* Keep probe and SPI NOR discovery powered until registration finishes. */ + if (soc->runtime_pm && IS_ENABLED(CONFIG_PM)) { + pm_runtime_get_noresume(dev); + ret = devm_pm_runtime_set_active_enabled(dev); + if (ret) { + pm_runtime_put_noidle(dev); + return ret; + } + /* Coalesce NOR read chunks and userspace NVRAM access bursts. */ + pm_runtime_set_autosuspend_delay(dev, APPLE_QSPI_AUTOSUSPEND_MS); + pm_runtime_use_autosuspend(dev); + ctlr->auto_runtime_pm = true; + } ret = devm_spi_register_controller(dev, ctlr); - if (ret) + if (ret) { + if (ctlr->auto_runtime_pm) + pm_runtime_put_noidle(dev); return dev_err_probe(dev, ret, "failed to register SPI controller\n"); + } mutex_lock(&apple_qspi_policy_lock); runtime_ready = true; @@ -1055,6 +1080,8 @@ static int apple_qspi_probe(struct platform_device *pdev) dev_dbg(dev, "QSPIMC version %#x, %lu Hz clock, txq=%u rxq=%u\n", version, clk_get_rate(qspi->clk), qspi->tx_depth, qspi->rx_depth); + if (ctlr->auto_runtime_pm) + pm_runtime_put_autosuspend(dev); return 0; } @@ -1138,11 +1165,41 @@ static struct attribute *apple_qspi_attrs[] = { }; ATTRIBUTE_GROUPS(apple_qspi); +/* No IRQ handler may read the block while genpd has it powered off. */ +static int apple_qspi_runtime_suspend(struct device *dev) +{ + struct spi_controller *ctlr = dev_get_drvdata(dev); + struct apple_qspi *qspi = spi_controller_get_devdata(ctlr); + + disable_irq(qspi->irq); + return 0; +} + +static int apple_qspi_runtime_resume(struct device *dev) +{ + struct spi_controller *ctlr = dev_get_drvdata(dev); + struct apple_qspi *qspi = spi_controller_get_devdata(ctlr); + int ret; + + ret = apple_qspi_hw_reset(qspi); + if (ret) + return ret; + enable_irq(qspi->irq); + return 0; +} + static int apple_qspi_suspend(struct device *dev) { struct spi_controller *ctlr = dev_get_drvdata(dev); + int ret; - return spi_controller_suspend(ctlr); + ret = spi_controller_suspend(ctlr); + if (ret || !ctlr->auto_runtime_pm) + return ret; + ret = pm_runtime_force_suspend(dev); + if (ret) + spi_controller_resume(ctlr); + return ret; } /* @@ -1157,23 +1214,29 @@ static int apple_qspi_resume(struct device *dev) /* * Reopen the controller even if the reset failed. Left suspended, it - * would fail every transfer until reboot; a transfer that times out - * resets it again instead. + * would fail every queued transfer until reboot. Runtime PM still + * refuses MMIO if the controller could not be brought back up. */ - ret = apple_qspi_hw_reset(qspi); + if (ctlr->auto_runtime_pm) + ret = pm_runtime_force_resume(dev); + else + ret = apple_qspi_hw_reset(qspi); if (ret) dev_err(dev, "controller reset failed on resume: %d\n", ret); return spi_controller_resume(ctlr); } -static DEFINE_SIMPLE_DEV_PM_OPS(apple_qspi_pm_ops, apple_qspi_suspend, - apple_qspi_resume); +static const struct dev_pm_ops apple_qspi_pm_ops = { + SYSTEM_SLEEP_PM_OPS(apple_qspi_suspend, apple_qspi_resume) + RUNTIME_PM_OPS(apple_qspi_runtime_suspend, apple_qspi_runtime_resume, NULL) +}; static const struct apple_qspi_soc apple_qspi_single_io = { }; static const struct apple_qspi_soc apple_qspi_t8140 = { .multi_io_reads = true, + .runtime_pm = true, }; static const struct of_device_id apple_qspi_of_match[] = { @@ -1191,7 +1254,7 @@ static struct platform_driver apple_qspi_driver = { .name = "apple-qspi", .dev_groups = apple_qspi_groups, .of_match_table = apple_qspi_of_match, - .pm = pm_sleep_ptr(&apple_qspi_pm_ops), + .pm = pm_ptr(&apple_qspi_pm_ops), }, }; module_platform_driver(apple_qspi_driver); diff --git a/tools/asahi/qspi-nvram-tests.py b/tools/asahi/qspi-nvram-tests.py index b458f3b06f3a8..a13ce3ac46e17 100644 --- a/tools/asahi/qspi-nvram-tests.py +++ b/tools/asahi/qspi-nvram-tests.py @@ -70,8 +70,11 @@ def function(source, signature): bool nvram_policy, nvram_write_candidate, write_fault, nvram_window; u64 window_start, window_end; u8 jedec[3]; u8 bp_ok; + int irq; }; -struct spi_controller { struct apple_qspi *q; }; +struct spi_controller; +struct device { struct device *parent; struct spi_controller *controller; }; +struct spi_controller { struct apple_qspi *q; bool auto_runtime_pm; struct device dev; }; struct spi_device { struct spi_controller *controller; unsigned int cs; int dev; }; struct spi_mem { struct spi_device *spi; }; struct spi_mem_op { @@ -92,7 +95,12 @@ def function(source, signature): static u8 flash[0x800000], sr[3], jedec[3]={0xef,0x65,0x17}; static unsigned int wire[4096],wire_count,fail_opcode; static bool corrupt_readback,stuck_busy,programmed; +static unsigned int pm_refs,busy_calls,busy_wire; +static void pm_runtime_mark_last_busy(struct device*d){ + assert(d&&pm_refs);busy_calls++;busy_wire=wire_count; +} static int apple_qspi_transfer_op(struct spi_mem*m,const struct spi_mem_op*o){ + if(m->spi->controller->auto_runtime_pm)assert(pm_refs); assert(wire_countcmd.opcode; if(o->cmd.opcode==fail_opcode)return -EIO; switch(o->cmd.opcode){ @@ -121,6 +129,7 @@ def function(source, signature): CONTROLS = r''' static struct apple_qspi q; static struct spi_controller ctl={&q}; +static struct device parent; static struct spi_device spi={&ctl}; static struct spi_mem mem={&spi}; static struct kernel_param parameter={&read_only}; @@ -129,6 +138,8 @@ def function(source, signature): read_only=true;runtime_ready=false;write_fault_latched=false;wire_count=0;fail_opcode=0;spi.cs=0; corrupt_readback=false;stuck_busy=false;programmed=false;deny_alloc=false;jiffies=0; jedec[0]=0xef;jedec[1]=0x65;jedec[2]=0x17; + ctl.auto_runtime_pm=false;ctl.dev.parent=&parent; + pm_refs=busy_calls=busy_wire=0; } static struct spi_mem_op program(u64 addr,unsigned int n,const u8*b){ struct spi_mem_op o=SPI_MEM_OP(SPI_MEM_OP_CMD(0x02,1),SPI_MEM_OP_ADDR(3,addr,1),SPI_MEM_OP_NO_DUMMY,SPI_MEM_OP_NO_DATA); @@ -236,6 +247,26 @@ def function(source, signature): reset();memcpy(q.jedec,(u8[]){0xc2,0x2b,0x27},3);arm();wire_count=0;o=program(0x700000,1,data); assert(apple_qspi_exec_op(&mem,&o)==-ENODEV&&!sent(0x06)); puts("PASS Macronix J616s part: no 35h on the wire, SR/CR checks, BP0-BP3 refusal and JEDEC mismatch"); + /* Model spi-mem's outer reference. Every private JEDEC/status/WREN, + * completion poll, WRDI and readback must finish before the idle stamp. + * Hardware tests must still prove genpd and the actual suspend timer. */ + reset();q.nvram_window=true;ctl.auto_runtime_pm=true;pm_refs=1; + assert(!apple_qspi_mem_exec_op(&mem,&t8140_read)); + assert(busy_calls==1&&busy_wire==wire_count&&wire_count==1); + wire_count=0;assert(apple_qspi_mem_exec_op(&mem,&wren)==-EACCES); + assert(busy_calls==2&&!wire_count); + arm();o=program(0x700000,256,data);stuck_busy=false; + assert(!apple_qspi_mem_exec_op(&mem,&o)); + assert(busy_calls==3&&busy_wire==wire_count&&wire[wire_count-1]==0x03); + wire_count=0;fail_opcode=0x03; + assert(apple_qspi_mem_exec_op(&mem,&t8140_read)==-EIO); + assert(busy_calls==4&&busy_wire==wire_count); + fail_opcode=0x02;o.addr.val=0x700100; + assert(apple_qspi_mem_exec_op(&mem,&o)==-EIO&&q.write_fault&&read_only); + assert(busy_calls==5&&busy_wire==wire_count&&wire[wire_count-1]==0x04); + pm_refs=0; + reset();assert(!apple_qspi_mem_exec_op(&mem,&t8140_read));assert(!busy_calls); + puts("PASS runtime-PM idle stamp after read, denied write, full verified program and transfer error; other SoCs unchanged"); return 0; } ''' @@ -296,7 +327,8 @@ def main(): ('mx-eqio-read', 'qspi->jedec[0] == APPLE_QSPI_MFR_MACRONIX) {', '0) {'), ('read-only', 'read_only || !qspi->nvram_write_candidate || qspi->write_fault', '!qspi->nvram_write_candidate || qspi->write_fault'), - ('fault-latch', 'write_fault_latched = true;', 'write_fault_latched = false;')]: + ('fault-latch', 'write_fault_latched = true;', 'write_fault_latched = false;'), + ('pm-last-busy', 'pm_runtime_mark_last_busy(ctlr->dev.parent);', '(void)ctlr;')]: assert old in code mutant = run(code.replace(old, new), Path(directory) / (name + '.c')) if not mutant.returncode: From 52cbd7f4041605e6afe8b969410baaf767f626a8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 16:35:57 +1100 Subject: [PATCH 1477/1579] arm64: dts: apple: Isolate the final J616s firmware fan policy Keep ladder definitions and call sites stable while removing their fan trip/map instances in a final policy section. This makes independent CPU ladder updates apply cleanly in either aggregate PR landing order without restoring automatic fan writes. Drop the optional fan map on the passive trip as well, preserving the trip and its CPU cooling maps. The effective device tree retains firmware fan ownership, all existing sensor channels, and passive/critical protection. No GPU temperature producer, conversion or validity policy changes are included. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 162 +++++++++++++++++++++- 1 file changed, 160 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 79ed4de3c98be..df0ee7681ff47 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -322,9 +322,41 @@ * GPU is powered off its sensors report placeholder values from -4 to * 4 C. * - * Fan speeds are governed by the SMC firmware. Manual speeds are - * available only through an explicit userspace fan target request. + * The fans: left to themselves (SMC mode 3, with no OS asking for + * cooling) they stay off, or switch on late and ramp slowly, while the + * dies heat up under load. So the CPU die zone and the GPU zones also + * run both fans through a ladder of active trips; a fan takes the + * highest state any zone asks for and goes back to the SMC once no + * zone asks for anything. States 1-8 of a fan run it from its minimum + * (1350 rpm) to its maximum (5777 rpm) in even steps. */ +#define J616S_FAN_TRIP(label, node, temp, hyst) \ + label: node { \ + temperature = ; \ + hysteresis = ; \ + type = "active"; \ + }; + +#define J616S_FAN_TRIPS(zone) \ + J616S_FAN_TRIP(zone##_fan1, trip-fan1, 75000, 10000) \ + J616S_FAN_TRIP(zone##_fan2, trip-fan2, 80000, 5000) \ + J616S_FAN_TRIP(zone##_fan3, trip-fan3, 85000, 5000) \ + J616S_FAN_TRIP(zone##_fan4, trip-fan4, 90000, 5000) \ + J616S_FAN_TRIP(zone##_fan5, trip-fan5, 95000, 5000) + +#define J616S_FAN_MAP(trip_label, node, state) \ + node { \ + trip = <&trip_label>; \ + cooling-device = <&smc_hwmon 0 state state>, \ + <&smc_hwmon 1 state state>; \ + }; + +#define J616S_FAN_MAPS(zone) \ + J616S_FAN_MAP(zone##_fan1, map-fan1, 1) \ + J616S_FAN_MAP(zone##_fan2, map-fan2, 2) \ + J616S_FAN_MAP(zone##_fan3, map-fan3, 3) \ + J616S_FAN_MAP(zone##_fan4, map-fan4, 5) \ + J616S_FAN_MAP(zone##_fan5, map-fan5, 8) thermal-zones { cpu-die-thermal { @@ -333,6 +365,7 @@ thermal-sensors = <&smc_hwmon 0>; /* TCMz */ trips { + J616S_FAN_TRIPS(cpu_die) cpu_die_passive: cpu-die-passive { temperature = <113000>; @@ -348,6 +381,7 @@ }; cooling-maps { + J616S_FAN_MAPS(cpu_die) map-e { trip = <&cpu_die_passive>; @@ -894,6 +928,7 @@ thermal-sensors = <&smc_hwmon 17>; /* Tg05 */ trips { + J616S_FAN_TRIPS(gpu0) gpu0_passive: trip-passive { temperature = <101000>; @@ -909,6 +944,7 @@ }; cooling-maps { + J616S_FAN_MAPS(gpu0) map-e { trip = <&gpu0_passive>; @@ -947,6 +983,7 @@ thermal-sensors = <&smc_hwmon 18>; /* Tg0S */ trips { + J616S_FAN_TRIPS(gpu1) gpu1_passive: trip-passive { temperature = <101000>; @@ -962,6 +999,7 @@ }; cooling-maps { + J616S_FAN_MAPS(gpu1) map-e { trip = <&gpu1_passive>; @@ -1000,6 +1038,7 @@ thermal-sensors = <&smc_hwmon 19>; /* Tg0Y */ trips { + J616S_FAN_TRIPS(gpu2) gpu2_passive: trip-passive { temperature = <101000>; @@ -1015,6 +1054,7 @@ }; cooling-maps { + J616S_FAN_MAPS(gpu2) map-e { trip = <&gpu2_passive>; @@ -1053,6 +1093,7 @@ thermal-sensors = <&smc_hwmon 20>; /* Tg0k */ trips { + J616S_FAN_TRIPS(gpu3) gpu3_passive: trip-passive { temperature = <101000>; @@ -1068,6 +1109,7 @@ }; cooling-maps { + J616S_FAN_MAPS(gpu3) map-e { trip = <&gpu3_passive>; @@ -1106,6 +1148,7 @@ thermal-sensors = <&smc_hwmon 21>; /* Tg0z */ trips { + J616S_FAN_TRIPS(gpu4) gpu4_passive: trip-passive { temperature = <101000>; @@ -1121,6 +1164,7 @@ }; cooling-maps { + J616S_FAN_MAPS(gpu4) map-e { trip = <&gpu4_passive>; @@ -1159,6 +1203,7 @@ thermal-sensors = <&smc_hwmon 22>; /* Tg1V */ trips { + J616S_FAN_TRIPS(gpu5) gpu5_passive: trip-passive { temperature = <101000>; @@ -1174,6 +1219,7 @@ }; cooling-maps { + J616S_FAN_MAPS(gpu5) map-e { trip = <&gpu5_passive>; @@ -1212,6 +1258,7 @@ thermal-sensors = <&smc_hwmon 23>; /* Tg1l */ trips { + J616S_FAN_TRIPS(gpu6) gpu6_passive: trip-passive { temperature = <101000>; @@ -1227,6 +1274,7 @@ }; cooling-maps { + J616S_FAN_MAPS(gpu6) map-e { trip = <&gpu6_passive>; @@ -4465,3 +4513,113 @@ status = "disabled"; }; }; + +/* + * The SMC owns automatic fan policy. Drop the host fan trip/map instances + * after the board's thermal descriptions, retaining all passive and critical + * protection. Keep this final policy separate from ladder descriptions so + * independent ladder updates cannot restore autonomous SMC writes. + */ + +#define J616S_DELETE_FAN_TRIPS \ + /delete-node/ trip-fan1; \ + /delete-node/ trip-fan2; \ + /delete-node/ trip-fan3; \ + /delete-node/ trip-fan4; \ + /delete-node/ trip-fan5; + +/* Some CPU ladders also map full fan speed to the passive trip. */ +#define J616S_DELETE_FAN_MAPS \ + /delete-node/ map-fan1; \ + /delete-node/ map-fan2; \ + /delete-node/ map-fan3; \ + /delete-node/ map-fan4; \ + /delete-node/ map-fan5; \ + /delete-node/ map-fan6; + +/ { + thermal-zones { + cpu-die-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + + gpu0-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + + gpu1-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + + gpu2-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + + gpu3-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + + gpu4-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + + gpu5-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + + gpu6-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + }; +}; + +#undef J616S_DELETE_FAN_TRIPS +#undef J616S_DELETE_FAN_MAPS From 802ce68d3d0d3349adf0cba4696a59c879475e39 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 16:36:26 +1100 Subject: [PATCH 1478/1579] accel/ane: draft H16 execute serialization and event correlation Encode the sanitized H16 execute header, fixed ordinary buffer slots and custom BAR tail from validated kernel resource metadata. Check mapping geometry and conservative BAR bounds before writing any output. Decode channel-6 UUID and identifier correlation with explicit progress, success and failure states. Accept rolling notification words without treating channel-7 acknowledgments as completion. Keep live submission, pending tracking and mapping retirement gated. Add synthetic KUnit coverage for wire layout, all buffer counts, tail lengths, overflow and malformed or mismatched events. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 68 ++++- drivers/accel/ane/Makefile | 2 +- drivers/accel/ane/ane_fw_protocol_test.c | 2 +- drivers/accel/ane/ane_h16_execute.h | 208 +++++++++++++ drivers/accel/ane/ane_h16_execute_test.c | 317 ++++++++++++++++++++ drivers/accel/ane/ane_h16_protocol.h | 3 +- drivers/accel/ane/ane_t6021_rtclient_main.c | 4 +- 7 files changed, 585 insertions(+), 19 deletions(-) create mode 100644 drivers/accel/ane/ane_h16_execute.h create mode 100644 drivers/accel/ane/ane_h16_execute_test.c diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index b16cb55f783df..ef546815b85eb 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -115,24 +115,66 @@ EVENT_REPORT 0x0300 6 0x60 (firmware to host) ================== ====== ======= ================== These sizes bound the observed version, not future firmware. POWER_GATE has -a u32 Boolean at +8. The execute-size increment of 0x18 does not establish -buffer-record semantics. The channel numbers are observed table indices; +a u32 Boolean at +8. The execute-size increment of 0x18 counts custom BAR +records. The channel numbers are observed table indices; their mapping to channel names remains unproved. The parser retains each descriptor's original index rather than assuming name order is wire order. The transport prepares the H16 envelope and rejects unrecognized runtime IDs, wrong sizes, channels and power-gate values before publication. H14's 0x0204 procedure packet and short CREATE_PROCESS body remain H14-only; -H16's body serializers and completion correlation are not implemented. +H16's standalone serializer and event classifier are described below; +they are not connected to inference submission. The H16 runtime command table does not authorize startup commands. -The event report starts with u32 0x0000fe00 through 0x0000fe03, then u16 -0x0300 and zero u16 at +6. Its body is undecoded. Valid reports are consumed -as diagnostics and never complete an inference or release its buffers. -The legacy 0x28-byte state-1 completion predicate must not be used for H16. -Unknown short channel-6 messages fail closed. A timeout retains mappings -and quarantines the instance; it does not trigger a retry or reset. - -KUnit covers all seven runtime IDs and their sizes/directions/channels, -the u16 envelope and refusal semantics, and report validation without -inventing completion fields. All test packets are synthetic. +H16 execute and event fields +--------------------------- + +The macOS 27.0 field contract has a fixed 0xca8-byte execute body followed +by zero, one or two observed 0x18-byte custom BAR records. The 64-slot +ordinary buffer array starts at +0x60 with a 0x30-byte stride; the valid +count is at +0x28. A descriptor carries flags at +0, resource index at +4, +numeric class at +8, the fixed value 3 at +0x10, a u64 DVA at +0x18 and a +u64 byte span at +0x20. Other descriptor bytes are zero. The UUID at +execute +0x20 is independent of program/process/procedure IDs at +8/+0xc/+0x10. +The observed priority is 5 at +0x18; request flags and reserved bytes are zero. + +The custom BAR count is at +0xca0 and records start at +0xca8. Each carries +u32 configuration/value-type/BAR selectors, u8 bit offset/width, u16 flags +and a u64 value. ``ane_h16_execute_encode()`` accepts kernel-owned resource +metadata, checks nonempty counts up to 64 and overflow-free mapping spans, +and zeroes unused slots and reserved space. The caller must first match +indices, numeric classes, permissions, selectors and values to immutable +validated process metadata and keep mappings pinned. The helper does not +establish that authority. Its two-BAR ceiling, 64-bit field bound and +same-configuration/BAR overlap refusal are conservative draft restrictions, +not established firmware-wide limits. Unknown class and selector semantics +are not guessed. There is no userspace interface for these structures. + +Channel 7 acknowledges submission only. Channel 6 carries 0x60-byte 0x0300 +events, with UUID at +8, program/process/procedure at +0x10/+0x14/+0x18 and +state at +0x1c. State 0 is start, 3 hardware-start, 1 successful terminal +completion and 2 terminal failure. Failure was established from field use; +it was not present in the runtime sample. The first word selects report +slots or rolls independently; 0xfe00..0xfe03 is not a universal prefix or a +completion key. CPU-load and timestamp fields are diagnostic. + +``ane_h16_event_decode()`` validates the channel, exact size, command and +zero u16 acknowledgment before returning fields. The result helper requires +all four identity fields to match an outstanding request. Destination 1 is +routed away from the general scheduler; unknown states/destinations refuse. +A caller must register a unique pending UUID before publication, consume it +atomically once on a terminal report, synchronize DMA, and apply quarantine +rules before considering any mapping release. A classifier result alone +cannot prove safe release or reject duplicate events without pending state. + +The live H16 path still consumes reports as diagnostics. It has no pending +UUID table, serializer call or inference interface. The legacy 0x28-byte +completion predicate remains H14-only. Short channel-6 CPU-load reports and +channel-5 allocation operations are outside these helpers and remain gated. +Timeouts still retain mappings and quarantine without retry or reset. + +KUnit covers byte-exact synthetic execute fields, all 1..64 buffer counts, +zero/one/two custom BAR lengths, wide DVAs/spans, reserved-zero output, +boundary and overflow refusal, all event states, interleaved identities, +ISP routing and malformed reports. No test packet is copied from a capture. diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile index db8286c2ee3cf..c4f0954656118 100644 --- a/drivers/accel/ane/Makefile +++ b/drivers/accel/ane/Makefile @@ -3,7 +3,7 @@ obj-$(CONFIG_DRM_ACCEL_ANE) += ane.o ane_t6021.o obj-$(CONFIG_DRM_ACCEL_ANE_KUNIT_TEST) += ane_drv_test.o obj-$(CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST) += ane_fw_platform_test.o \ - ane_fw_protocol_test.o ane_h16_tasks_test.o + ane_fw_protocol_test.o ane_h16_tasks_test.o ane_h16_execute_test.o ane-y := \ ane_drv.o \ diff --git a/drivers/accel/ane/ane_fw_protocol_test.c b/drivers/accel/ane/ane_fw_protocol_test.c index 86fe430afde20..fb98428f3f238 100644 --- a/drivers/accel/ane/ane_fw_protocol_test.c +++ b/drivers/accel/ane/ane_fw_protocol_test.c @@ -104,7 +104,7 @@ static void ane_h16_report_test(struct kunit *test) KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, 0xc, 6), -EPROTO); KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(NULL, 0x60, 6), -EPROTO); put_unaligned_le32(0xfe04, event); - KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, sizeof(event), 6), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, sizeof(event), 6), 0); put_unaligned_le32(0xfe00, event); put_unaligned_le16(0x301, event + 4); KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, sizeof(event), 6), -EPROTO); diff --git a/drivers/accel/ane/ane_h16_execute.h b/drivers/accel/ane/ane_h16_execute.h new file mode 100644 index 0000000000000..df3cb515e9e2c --- /dev/null +++ b/drivers/accel/ane/ane_h16_execute.h @@ -0,0 +1,208 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_EXECUTE_H__ +#define __ANE_H16_EXECUTE_H__ + +#include + +#include "ane_h16_protocol.h" + +#define ANE_H16_EXECUTE_BASE_SIZE 0xca8 +#define ANE_H16_BUFFER_OFFSET 0x60 +#define ANE_H16_BUFFER_STRIDE 0x30 +#define ANE_H16_MAX_BUFFERS 64 +#define ANE_H16_CUSTOM_BAR_STRIDE 0x18 +/* Conservative observed-version ceiling, not a firmware maximum. */ +#define ANE_H16_MAX_CUSTOM_BARS 2 + +struct ane_h16_request_key { + u64 uuid; + u32 program; + u32 process; + u32 procedure; +}; + +/* Kernel-owned, validated process resources and pinned mapping geometry. */ +struct ane_h16_execute_buffer { + u32 index; + u32 class; + u64 mapping_dva; + u64 mapping_size; + u64 offset; + u64 size; +}; + +/* Selectors and value interpretation must come from validated program metadata. */ +struct ane_h16_custom_bar { + u32 configuration; + u32 value_type; + u32 bar; + u8 bit_offset; + u8 bit_width; + u16 flags; + u64 value; +}; + +struct ane_h16_execute { + struct ane_h16_request_key key; + const struct ane_h16_execute_buffer *buffers; + unsigned int buffer_count; + const struct ane_h16_custom_bar *bars; + unsigned int bar_count; +}; + +/* + * Encode only after matching resources, permissions and BAR selectors to the + * immutable loaded program. This helper checks geometry, not mapping ownership. + * Inputs must remain immutable and must not alias the destination. On error the + * destination is untouched. No mapping is published or released by this helper. + */ +static inline int +ane_h16_execute_encode(void *command, size_t length, const struct ane_h16_execute *call) +{ + u8 *p = command; + unsigned int i, j; + size_t expected; + + if (!p || !call || !call->buffers || !call->buffer_count || + call->buffer_count > ANE_H16_MAX_BUFFERS || + call->bar_count > ANE_H16_MAX_CUSTOM_BARS || + (call->bar_count && !call->bars)) + return -EINVAL; + expected = ANE_H16_EXECUTE_BASE_SIZE + call->bar_count * ANE_H16_CUSTOM_BAR_STRIDE; + if (length != expected) + return -EINVAL; + for (i = 0; i < call->buffer_count; i++) { + const struct ane_h16_execute_buffer *b = &call->buffers[i]; + + if (!b->size || b->mapping_size > U64_MAX - b->mapping_dva || + b->offset > b->mapping_size || b->size > b->mapping_size - b->offset) + return -EINVAL; + } + for (i = 0; i < call->bar_count; i++) { + const struct ane_h16_custom_bar *b = &call->bars[i]; + unsigned int end = b->bit_offset + b->bit_width; + + /* Conservative 64-bit field bound until wider fields are established. */ + if (!b->bit_width || end > 64) + return -EINVAL; + for (j = 0; j < i; j++) { + const struct ane_h16_custom_bar *a = &call->bars[j]; + + if (a->configuration == b->configuration && a->bar == b->bar && + a->bit_offset < end && + b->bit_offset < a->bit_offset + a->bit_width) + return -EINVAL; + } + } + + memset(p, 0, length); + put_unaligned_le16(ANE_H16_EXECUTE, p + 4); + put_unaligned_le32(call->key.program, p + 8); + put_unaligned_le32(call->key.process, p + 0xc); + put_unaligned_le32(call->key.procedure, p + 0x10); + put_unaligned_le32(5, p + 0x18); + put_unaligned_le64(call->key.uuid, p + 0x20); + put_unaligned_le32(call->buffer_count, p + 0x28); + for (i = 0; i < call->buffer_count; i++) { + const struct ane_h16_execute_buffer *b = &call->buffers[i]; + u8 *d = p + ANE_H16_BUFFER_OFFSET + i * ANE_H16_BUFFER_STRIDE; + + put_unaligned_le32(1, d); + put_unaligned_le32(b->index, d + 4); + put_unaligned_le32(b->class, d + 8); + put_unaligned_le32(3, d + 0x10); + put_unaligned_le64(b->mapping_dva + b->offset, d + 0x18); + put_unaligned_le64(b->size, d + 0x20); + } + put_unaligned_le32(call->bar_count, p + 0xca0); + for (i = 0; i < call->bar_count; i++) { + const struct ane_h16_custom_bar *b = &call->bars[i]; + u8 *d = p + ANE_H16_EXECUTE_BASE_SIZE + i * ANE_H16_CUSTOM_BAR_STRIDE; + + put_unaligned_le32(b->configuration, d); + put_unaligned_le32(b->value_type, d + 4); + put_unaligned_le32(b->bar, d + 8); + d[0xc] = b->bit_offset; + d[0xd] = b->bit_width; + put_unaligned_le16(b->flags, d + 0xe); + put_unaligned_le64(b->value, d + 0x10); + } + return 0; +} + +enum ane_h16_event_state { + ANE_H16_EVENT_START, + ANE_H16_EVENT_SUCCESS, + ANE_H16_EVENT_FAILURE, + ANE_H16_EVENT_HW_START, +}; + +struct ane_h16_event { + struct ane_h16_request_key key; + u32 state; + u32 cpu_load; + u64 timestamp; + u8 destination; +}; + +/* Parse a bounded host-owned snapshot. Diagnostic words are not predicates. */ +static inline int +ane_h16_event_decode(const void *event, size_t length, unsigned int channel, + struct ane_h16_event *decoded) +{ + const u8 *p = event; + struct ane_h16_event e = {}; + + if (!decoded || ane_h16_event_report_status(event, length, channel)) + return -EPROTO; + e.key.uuid = get_unaligned_le64(p + 8); + e.key.program = get_unaligned_le32(p + 0x10); + e.key.process = get_unaligned_le32(p + 0x14); + e.key.procedure = get_unaligned_le32(p + 0x18); + e.state = get_unaligned_le32(p + 0x1c); + e.cpu_load = get_unaligned_le32(p + 0x20); + e.timestamp = get_unaligned_le64(p + 0x28); + e.destination = p[0x30]; + if (e.state > ANE_H16_EVENT_HW_START || e.destination > 1) + return -EPROTO; + *decoded = e; + return 0; +} + +enum ane_h16_event_result { + ANE_H16_EVENT_PROGRESS, + ANE_H16_EVENT_COMPLETE, + ANE_H16_EVENT_FAILED, +}; + +/* + * For the general scheduler only. Look up an outstanding UUID before calling; + * remove it atomically on terminal state to reject duplicates and late reports. + * COMPLETE/FAILED classify an event, not permission to unmap: DMA sync and + * failure quarantine remain the caller's responsibility. Never call on ch7 ACK. + */ +static inline int +ane_h16_event_result(const void *event, size_t length, unsigned int channel, + const struct ane_h16_request_key *pending) +{ + struct ane_h16_event e; + int ret; + + if (!pending) + return -EINVAL; + ret = ane_h16_event_decode(event, length, channel, &e); + if (ret) + return ret; + if (e.destination) + return -ENOMSG; + if (e.key.uuid != pending->uuid || e.key.program != pending->program || + e.key.process != pending->process || e.key.procedure != pending->procedure) + return -EPROTO; + if (e.state == ANE_H16_EVENT_SUCCESS) + return ANE_H16_EVENT_COMPLETE; + if (e.state == ANE_H16_EVENT_FAILURE) + return ANE_H16_EVENT_FAILED; + return ANE_H16_EVENT_PROGRESS; +} + +#endif /* __ANE_H16_EXECUTE_H__ */ diff --git a/drivers/accel/ane/ane_h16_execute_test.c b/drivers/accel/ane/ane_h16_execute_test.c new file mode 100644 index 0000000000000..933011c18a77a --- /dev/null +++ b/drivers/accel/ane/ane_h16_execute_test.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include + +#include "ane_h16_execute.h" + +#define TEST_COMMAND_SIZE 0xcd8 + +static const struct ane_h16_request_key test_key = { + .uuid = 0x8877665544332211ULL, + .program = 0x10203040, + .process = 0x50607080, + .procedure = 0x90a0b0c0, +}; + +static struct ane_h16_execute_buffer test_buffer(void) +{ + return (struct ane_h16_execute_buffer) { + .index = 7, + .class = 1, + .mapping_dva = 0x12300000000ULL, + .mapping_size = 0x300000000ULL, + .offset = 0x10000, + .size = 0x200000000ULL, + }; +} + +static void ane_h16_execute_layout_test(struct kunit *test) +{ + static const u8 header[] = { + 0, 0, 0, 0, 0x0c, 2, 0, 0, + 0x40, 0x30, 0x20, 0x10, 0x80, 0x70, 0x60, 0x50, + 0xc0, 0xb0, 0xa0, 0x90, 0, 0, 0, 0, + 5, 0, 0, 0, 0, 0, 0, 0, + 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, + 1, 0, 0, 0, + }; + static const u8 descriptor[] = { + 1, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0x23, 1, 0, 0, + 0, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + }; + struct ane_h16_execute_buffer buffer = test_buffer(); + struct ane_h16_execute call = { test_key, &buffer, 1, NULL, 0 }; + u8 *storage = kunit_kmalloc(test, TEST_COMMAND_SIZE + 2, GFP_KERNEL); + u8 *p; + + KUNIT_ASSERT_NOT_NULL(test, storage); + memset(storage, 0xa5, TEST_COMMAND_SIZE + 2); + p = storage + 1; /* Exercise unaligned output and both destination guards. */ + KUNIT_ASSERT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), 0); + KUNIT_EXPECT_MEMEQ(test, p, header, sizeof(header)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(p + sizeof(header), 0, 0x60 - sizeof(header)), NULL); + KUNIT_EXPECT_MEMEQ(test, p + 0x60, descriptor, sizeof(descriptor)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(p + 0x90, 0, 0xca8 - 0x90), NULL); + KUNIT_EXPECT_EQ(test, storage[0], (u8)0xa5); + KUNIT_EXPECT_EQ(test, p[0xca8], (u8)0xa5); +} + +static void ane_h16_execute_counts_test(struct kunit *test) +{ + struct ane_h16_execute_buffer *buffers; + struct ane_h16_execute call = { .key = test_key }; + u8 *p = kunit_kmalloc(test, TEST_COMMAND_SIZE, GFP_KERNEL); + unsigned int n, i; + + buffers = kunit_kcalloc(test, 64, sizeof(*buffers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, p); + KUNIT_ASSERT_NOT_NULL(test, buffers); + for (i = 0; i < 64; i++) { + buffers[i] = test_buffer(); + buffers[i].index = i; + buffers[i].class = 8; /* Numeric classes have no guessed semantic name. */ + } + call.buffers = buffers; + for (n = 1; n <= 64; n++) { + size_t end = 0x60 + n * 0x30; + + call.buffer_count = n; + memset(p, 0xa5, TEST_COMMAND_SIZE); + KUNIT_ASSERT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(p + 0x28), n); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(p + end - 0x30 + 4), n - 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(p + end - 0x30 + 8), 8U); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(p + end, 0, 0xca8 - end), NULL); + } +} + +static void ane_h16_custom_bar_test(struct kunit *test) +{ + static const u8 first_bar[] = { + 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, + 0, 32, 0x34, 0x12, 0x18, 0x17, 0x16, 0x15, 0x14, 0x13, 0x12, 0x11, + }; + struct ane_h16_execute_buffer buffer = test_buffer(); + struct ane_h16_custom_bar bars[2] = { + { 0x01020304, 0x05060708, 0x090a0b0c, 0, 32, 0x1234, + 0x1112131415161718ULL }, + { 0x01020304, 0x05060708, 0x090a0b0c, 32, 32, 0, 0x2122232425262728ULL }, + }; + struct ane_h16_execute call = { test_key, &buffer, 1, bars, 0 }; + u8 *p = kunit_kmalloc(test, TEST_COMMAND_SIZE + 1, GFP_KERNEL); + unsigned int n; + + KUNIT_ASSERT_NOT_NULL(test, p); + for (n = 0; n <= 2; n++) { + size_t length = 0xca8 + n * 0x18; + + call.bar_count = n; + memset(p, 0xa5, TEST_COMMAND_SIZE + 1); + KUNIT_ASSERT_EQ(test, ane_h16_execute_encode(p, length, &call), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(p + 0xca0), n); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(p + 0xca4), 0U); + KUNIT_EXPECT_EQ(test, p[length], (u8)0xa5); + if (n) + KUNIT_EXPECT_MEMEQ(test, p + 0xca8, first_bar, sizeof(first_bar)); + if (n == 2) { + KUNIT_EXPECT_EQ(test, p[0xccc], (u8)32); + KUNIT_EXPECT_EQ(test, p[0xccd], (u8)32); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p + 0xcd0), + 0x2122232425262728ULL); + } + } +} + +static void ane_h16_execute_rejections_test(struct kunit *test) +{ + struct ane_h16_execute_buffer buffer = test_buffer(); + struct ane_h16_execute call = { test_key, &buffer, 1, NULL, 0 }; + u8 *p = kunit_kmalloc(test, TEST_COMMAND_SIZE, GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, p); + memset(p, 0xa5, TEST_COMMAND_SIZE); + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(NULL, 0xca8, &call), -EINVAL); + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, NULL), -EINVAL); + for (i = 0; i < 4; i++) { + static const size_t lengths[] = { 0, 0xca7, 0xca9, 0xcc0 }; + + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, lengths[i], &call), -EINVAL); + } + call.buffer_count = 0; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + call.buffer_count = 65; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + call.buffer_count = 1; + call.buffers = NULL; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + call.buffers = &buffer; + call.bar_count = 3; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcf0, &call), -EINVAL); + call.bar_count = 1; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcc0, &call), -EINVAL); + call.bar_count = 0; + buffer.size = 0; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + buffer = test_buffer(); + buffer.offset = buffer.mapping_size + 1; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + buffer.offset = buffer.mapping_size; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + buffer = test_buffer(); + buffer.size = buffer.mapping_size - buffer.offset + 1; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + buffer = test_buffer(); + buffer.mapping_dva = U64_MAX - buffer.mapping_size + 1; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(p, 0xa5, TEST_COMMAND_SIZE), NULL); + /* The exact last byte of a mapping is valid. */ + buffer = test_buffer(); + buffer.offset = buffer.mapping_size - 1; + buffer.size = 1; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), 0); +} + +static void ane_h16_bar_rejections_test(struct kunit *test) +{ + struct ane_h16_execute_buffer buffer = test_buffer(); + struct ane_h16_custom_bar bars[2] = { { .bit_width = 32 }, { .bit_width = 32 } }; + struct ane_h16_execute call = { test_key, &buffer, 1, bars, 2 }; + u8 *p = kunit_kmalloc(test, TEST_COMMAND_SIZE, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, p); + memset(p, 0xa5, TEST_COMMAND_SIZE); + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), -EINVAL); + bars[1].bit_offset = 31; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), -EINVAL); + bars[1].bit_offset = 33; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), -EINVAL); + bars[1].bit_offset = 255; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), -EINVAL); + bars[1].bit_offset = 32; + bars[1].bit_width = 0; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(p, 0xa5, TEST_COMMAND_SIZE), NULL); + bars[1].bit_width = 32; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), 0); + bars[1].bit_offset = 0; + bars[1].bar = 1; /* Distinct BARs do not overlap. */ + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), 0); + bars[1].bar = 0; + bars[1].configuration = 1; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), 0); +} + +static void ane_h16_test_event(u8 *p, u32 state) +{ + memset(p, 0, 0x60); + put_unaligned_le32(0x12345678, p); /* A rolling word, not fe0N. */ + put_unaligned_le16(0x300, p + 4); + put_unaligned_le64(test_key.uuid, p + 8); + put_unaligned_le32(test_key.program, p + 0x10); + put_unaligned_le32(test_key.process, p + 0x14); + put_unaligned_le32(test_key.procedure, p + 0x18); + put_unaligned_le32(state, p + 0x1c); + put_unaligned_le32(10, p + 0x20); + put_unaligned_le64(0x123456789abcdef0ULL, p + 0x28); + memset(p + 0x31, 0xff, 0x2f); /* Diagnostics do not decide completion. */ +} + +static void ane_h16_event_correlation_test(struct kunit *test) +{ + static const int results[] = { 0, 1, 2, 0 }; + struct ane_h16_event e; + u8 storage[0x61], *p = storage + 1; + unsigned int state, n; + + for (state = 0; state < 4; state++) { + ane_h16_test_event(p, state); + KUNIT_ASSERT_EQ(test, ane_h16_event_decode(p, 0x60, 6, &e), 0); + KUNIT_EXPECT_EQ(test, e.key.uuid, test_key.uuid); + KUNIT_EXPECT_EQ(test, e.state, state); + KUNIT_EXPECT_EQ(test, e.cpu_load, 10U); + KUNIT_EXPECT_EQ(test, e.timestamp, 0x123456789abcdef0ULL); + KUNIT_EXPECT_EQ(test, ane_h16_event_result(p, 0x60, 6, &test_key), results[state]); + for (n = 0; n < 4; n++) { + put_unaligned_le32(0xfe00 + n, p); + KUNIT_EXPECT_EQ(test, ane_h16_event_result(p, 0x60, 6, &test_key), + results[state]); + } + } + ane_h16_test_event(p, 1); + for (n = 0; n < 4; n++) { + struct ane_h16_request_key other = test_key; + + switch (n) { + case 0: + other.uuid ^= BIT_ULL(63); + break; + case 1: + other.program++; + break; + case 2: + other.process++; + break; + case 3: + other.procedure++; + break; + } + KUNIT_EXPECT_EQ(test, ane_h16_event_result(p, 0x60, 6, &other), -EPROTO); + } + p[0x30] = 1; /* ISP traffic cannot retire a general scheduler request. */ + KUNIT_EXPECT_EQ(test, ane_h16_event_result(p, 0x60, 6, &test_key), -ENOMSG); +} + +static void ane_h16_event_rejections_test(struct kunit *test) +{ + struct ane_h16_event e; + u8 p[0x61]; + size_t n; + + ane_h16_test_event(p, 1); + memset(&e, 0xa5, sizeof(e)); + for (n = 0; n < 0x60; n++) + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(p, n, 6, &e), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(p, sizeof(p), 6, &e), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(p, 0x60, 7, &e), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(NULL, 0x60, 6, &e), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(p, 0x60, 6, NULL), -EPROTO); + put_unaligned_le16(0x20c, p + 4); /* An ACK is never a terminal event. */ + KUNIT_EXPECT_EQ(test, ane_h16_event_result(p, 0x60, 7, &test_key), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_result(p, 0x60, 6, &test_key), -EPROTO); + put_unaligned_le16(0x300, p + 4); + put_unaligned_le16(0x100, p + 6); + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(p, 0x60, 6, &e), -EPROTO); + put_unaligned_le16(0, p + 6); + put_unaligned_le32(4, p + 0x1c); + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(p, 0x60, 6, &e), -EPROTO); + put_unaligned_le32(1, p + 0x1c); + p[0x30] = 2; + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(p, 0x60, 6, &e), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_result(p, 0x60, 6, NULL), -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(&e, 0xa5, sizeof(e)), NULL); +} + +static struct kunit_case ane_h16_execute_cases[] = { + KUNIT_CASE(ane_h16_execute_layout_test), + KUNIT_CASE(ane_h16_execute_counts_test), + KUNIT_CASE(ane_h16_custom_bar_test), + KUNIT_CASE(ane_h16_execute_rejections_test), + KUNIT_CASE(ane_h16_bar_rejections_test), + KUNIT_CASE(ane_h16_event_correlation_test), + KUNIT_CASE(ane_h16_event_rejections_test), + {} +}; + +static struct kunit_suite ane_h16_execute_suite = { + .name = "ane-h16-execute", + .test_cases = ane_h16_execute_cases, +}; + +kunit_test_suite(ane_h16_execute_suite); +MODULE_DESCRIPTION("KUnit tests for H16 execute serialization and event correlation"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_protocol.h b/drivers/accel/ane/ane_h16_protocol.h index f368212d230ba..a3ef6a9c25ae8 100644 --- a/drivers/accel/ane/ane_h16_protocol.h +++ b/drivers/accel/ane/ane_h16_protocol.h @@ -61,7 +61,7 @@ ane_h16_command_prepare(void *command, size_t length, u16 id, unsigned int chann return 0; } -/* Reports are diagnostics only: cookie, state and completion are undecoded. */ +/* The first word selects report slots; rolling words are also valid. */ static inline int ane_h16_event_report_status(const void *event, size_t length, unsigned int channel) { @@ -70,7 +70,6 @@ ane_h16_event_report_status(const void *event, size_t length, unsigned int chann ane_h16_command_info(ANE_H16_EVENT_REPORT); if (!p || channel != info->channel || length != info->min_size || - (get_unaligned_le32(p) & ~3U) != 0xfe00 || get_unaligned_le16(p + 4) != info->id || get_unaligned_le16(p + 6)) return -EPROTO; return 0; diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index b4e44b65e95ae..5f1e5e66bd22d 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -909,7 +909,7 @@ static bool ane_rtclient_drain_t2h(struct ane_rtclient *ane, atomic_set(&ane_t6021_quarantined, 1); return false; } - /* The captured H16 report has no decoded completion field. + /* H16 submission and pending-UUID tracking are not wired up. * Consume diagnostics without retiring an execution or its BOs. */ } else if (channel == 6 && len >= ANE_T2H_CALL_STATE_OFF + 4) { @@ -1450,7 +1450,7 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, size_t cmd_size; int ret, i, pinned = 0; - /* H16 EXECUTE is 0x020c on channel 7 with an undecoded body. */ + /* H16 EXECUTE needs validated resources and pending-UUID tracking. */ if (ane->soc->platform) return -EOPNOTSUPP; if (user->pad || user->count < 1 || user->count > DRM_ANE_MAX_BINDS || From 1139288fe0edd6c52af11fa0bc40fe499470d89b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 16:40:53 +1100 Subject: [PATCH 1479/1579] rust: diagnose bindgen empty-record alignment before building An unprepared source tree falls back to bindgen on PATH. Stock bindgen loses the alignment of a defined empty GNU C struct, so the default Asahi configuration fails the sched_statistics layout assertion. The existing pinned generator repairs this, but the availability check does not explain that it must be prepared before configuring the kernel. Probe the selected generator's behavior rather than its version and point to the pinned build recipe. Keep this a warning so configuration cannot silently remove Rust drivers. Every generated kernel layout assertion remains enabled. Cover patched and unpatched tools with the same version and a failed probe that emits partial output. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/rust/bindgen-layout-checks.rst | 8 ++++ scripts/rust_is_available.sh | 26 +++++++++++++ scripts/rust_is_available_bindgen_alignment.h | 3 ++ scripts/rust_is_available_test.py | 39 ++++++++++++++++++- 4 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 scripts/rust_is_available_bindgen_alignment.h diff --git a/Documentation/rust/bindgen-layout-checks.rst b/Documentation/rust/bindgen-layout-checks.rst index 4e0a996bd701d..eace444abbcd8 100644 --- a/Documentation/rust/bindgen-layout-checks.rst +++ b/Documentation/rust/bindgen-layout-checks.rst @@ -58,6 +58,14 @@ builds. Packagers prepare this entry point in their build roots. Make does not fetch tools or install anything system-wide. If the entry point is absent, Kbuild uses ``bindgen`` from PATH and the same mandatory assertions apply. +``make LLVM=1 rustavailable`` probes the selected generator's empty-record +alignment support, including locally patched versions. An affected generator +prints the preparation command above. This is a warning so it cannot silently +disable Rust drivers during configuration; the layout assertions still reject +incorrect bindings when compiling the kernel. Prepare the tool before running +``defconfig`` and merging ``asahi.config``, then keep the resolved configuration +unchanged for the kernel build. + The generated bindings are consumed by the Rust kernel crate and by several drivers. After changing the bindings generator, rebuild the kernel Image and all modules together. diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index c30983562a2f1..51e51790cb4f6 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -222,6 +222,32 @@ if [ "$bindgen_libclang_cversion" -ge 2200000 ] && fi fi +# Check the behavior, not the version: distributions can backport the repair. +# Warn rather than making CONFIG_RUST unavailable, since that would silently +# remove Rust drivers from a configuration. The kernel's layout assertions +# remain the final check for every generated type. +bindgen_alignment_output=$( \ + LC_ALL=C "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_alignment.h \ + --rust-target 1.85 2>/dev/null +) || bindgen_alignment_code=$? +if [ -n "$bindgen_alignment_code" ] || + ! echo "$bindgen_alignment_output" | grep -Eq '#\[repr\(.*align\(64\)'; then + echo >&2 "***" + if [ -n "$bindgen_alignment_code" ]; then + echo >&2 "*** Could not check empty struct alignment with '$BINDGEN'." + echo >&2 "*** Exit code: $bindgen_alignment_code" + else + echo >&2 "*** Rust bindings generator '$BINDGEN'" + echo >&2 "*** does not preserve empty struct alignment." + fi + echo >&2 "*** Default Asahi builds can fail the sched_statistics layout assertion." + echo >&2 "*** Prepare the corrected generator before configuring the kernel:" + echo >&2 "*** python3 tools/aurora-bindgen/build.py" + echo >&2 "*** See Documentation/rust/bindgen-layout-checks.rst for details." + echo >&2 "***" + warning=1 +fi + # If the C compiler is Clang, then we can also check whether its version # matches the `libclang` version used by the Rust bindings generator. # diff --git a/scripts/rust_is_available_bindgen_alignment.h b/scripts/rust_is_available_bindgen_alignment.h new file mode 100644 index 0000000000000..d27e21fa9003d --- /dev/null +++ b/scripts/rust_is_available_bindgen_alignment.h @@ -0,0 +1,3 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* A defined GNU C empty record still has alignment, despite having no storage. */ +struct rust_bindgen_aligned_empty {} __attribute__((aligned(64))); diff --git a/scripts/rust_is_available_test.py b/scripts/rust_is_available_test.py index 22bdff980c35e..36da4e576396f 100755 --- a/scripts/rust_is_available_test.py +++ b/scripts/rust_is_available_test.py @@ -20,6 +20,7 @@ class TestRustIsAvailable(unittest.TestCase): class Expected(enum.Enum): SUCCESS = enum.auto() SUCCESS_WITH_WARNINGS = enum.auto() + SUCCESS_WITH_OPTIONAL_WARNINGS = enum.auto() SUCCESS_WITH_EXTRA_OUTPUT = enum.auto() FAILURE = enum.auto() @@ -54,7 +55,8 @@ def generate_rustc(cls, stdout): """) @classmethod - def generate_bindgen(cls, version_stdout, libclang_stderr, libclang_22_patched=False): + def generate_bindgen(cls, version_stdout, libclang_stderr, libclang_22_patched=False, + empty_alignment=True): if libclang_stderr is None: libclang_case = f"raise SystemExit({cls.bindgen_default_bindgen_libclang_failure_exit_code})" else: @@ -71,6 +73,8 @@ def generate_bindgen(cls, version_stdout, libclang_stderr, libclang_22_patched=F {libclang_case} elif "rust_is_available_bindgen_libclang_22.h" in " ".join(sys.argv): {libclang_22_case} +elif "rust_is_available_bindgen_alignment.h" in " ".join(sys.argv): + print({repr('#[repr(C, align(64))]' if empty_alignment else '#[repr(C)]')}) else: print({repr(version_stdout)}) """) @@ -142,6 +146,10 @@ def run_script(self, expected, override_env): # be checked further by the test). self.assertEqual(result.returncode, 0) self.assertNotEqual(result.stderr, b"") + elif expected == self.Expected.SUCCESS_WITH_OPTIONAL_WARNINGS: + self.assertEqual(result.returncode, 0) + if result.stderr: + self.assertIn(b"Please see Documentation/rust/quick-start.rst for details", result.stderr) elif expected == self.Expected.SUCCESS_WITH_WARNINGS: # When expecting a success with warnings, the script should return 0 # and it should output at least the instructions to `stderr`. @@ -338,6 +346,33 @@ def test_success_bindgen_version(self): bindgen = self.generate_bindgen_version(bindgen_stdout) result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen }) + def test_bindgen_empty_alignment_unpatched(self): + # The same version can be patched by a distribution, so test behavior. + bindgen = self.generate_bindgen("bindgen 0.73.2", + self.bindgen_default_bindgen_libclang_stderr, + empty_alignment=False) + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) + self.assertIn("does not preserve empty struct alignment", result.stderr) + self.assertIn("python3 tools/aurora-bindgen/build.py", result.stderr) + + def test_bindgen_empty_alignment_patched(self): + bindgen = self.generate_bindgen("bindgen 0.73.2", + self.bindgen_default_bindgen_libclang_stderr) + self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen }) + + def test_bindgen_empty_alignment_probe_failure(self): + bindgen = self.generate_executable(f'''#!/usr/bin/env python3 +import os +import sys +if "rust_is_available_bindgen_alignment.h" in " ".join(sys.argv): + print("#[repr(C, align(64))]") + print("probe failed", file=sys.stderr) + raise SystemExit(1) +os.execv({str(self.default_bindgen)!r}, [{str(self.default_bindgen)!r}] + sys.argv[1:]) +''') + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) + self.assertIn("Could not check empty struct alignment", result.stderr) + def test_success_bindgen_libclang(self): for stderr in ( f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} (https://github.com/llvm/llvm-project.git 4a2c05b05ed07f1f620e94f6524a8b4b2760a0b1) [-W#pragma-messages], err: false", @@ -370,7 +405,7 @@ def test_success_clang_version(self): def test_success_real_programs(self): for cc in ["gcc", "clang"]: with self.subTest(cc=cc): - result = self.run_script(self.Expected.SUCCESS, { + result = self.run_script(self.Expected.SUCCESS_WITH_OPTIONAL_WARNINGS, { "PATH": os.environ["PATH"], "RUSTC": "rustc", "BINDGEN": "bindgen", From e8beff693c2504f22eef4eb9000a0ac37e7aeb70 Mon Sep 17 00:00:00 2001 From: Ryan <81541646+RyanTheTide@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:53:26 +1100 Subject: [PATCH 1480/1579] ASoC: tas2764: Make the SDZ supply optional The Apple M1 and M2 laptops drive the SDZ pins of their amps from one GPIO, described as shutdown-gpios on every amp and shared between them by the GPIO core since commit 7ae241b6740c ("ASoC: tas2764: Initialise SDZ GPIO low"). Only the J616s models the line as a fixed regulator (SDZ-supply), which its SDZ fault group needs. The driver still asks for the SDZ supply with devm_regulator_get(), so every amp on the GPIO boards gets a dummy regulator and warns at boot: tas2764 1-0038: supply SDZ not found, using dummy regulator Ask for it with devm_regulator_get_optional() and skip the regulator calls when there is none. The dummy regulator never switched anything, so the SDZ line behaves exactly as before on every board: the GPIO boards still raise and drop SDZ through shutdown-gpios, and the J616s still gets its real supply. A board that sets apple,sdz-group-fail-closed without an SDZ supply is still refused at probe. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/codecs/tas2764.c | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/sound/soc/codecs/tas2764.c b/sound/soc/codecs/tas2764.c index 399aa42c6414a..4bb710efe35f1 100644 --- a/sound/soc/codecs/tas2764.c +++ b/sound/soc/codecs/tas2764.c @@ -55,7 +55,7 @@ struct tas2764_priv { struct snd_soc_component *component; struct gpio_desc *reset_gpio; struct gpio_desc *sdz_gpio; - struct regulator *sdz_reg; + struct regulator *sdz_reg; /* NULL when the SDZ line is a GPIO or not described */ struct regmap *regmap; struct device *dev; int irq; @@ -106,7 +106,7 @@ static int tas2764_sdz_enable(struct tas2764_priv *tas2764) int ret; if (!tas2764->sdz_group) - return regulator_enable(tas2764->sdz_reg); + return tas2764->sdz_reg ? regulator_enable(tas2764->sdz_reg) : 0; mutex_lock(&tas2764_sdz_groups_lock); ret = READ_ONCE(tas2764->sdz_group->failed) ? -EIO : regulator_enable(tas2764->sdz_reg); @@ -141,8 +141,10 @@ static int tas2764_group_join(struct device *dev, struct tas2764_priv *tas2764) if (!tas2764->irq) return dev_err_probe(dev, -EINVAL, "SDZ group without a fault interrupt\n"); supply = of_parse_phandle(dev->of_node, "SDZ-supply", 0); - if (!supply) + if (!supply || !tas2764->sdz_reg) { + of_node_put(supply); return dev_err_probe(dev, -EINVAL, "SDZ group without an SDZ supply\n"); + } mutex_lock(&tas2764_sdz_groups_lock); list_for_each_entry(group, &tas2764_sdz_groups, node) { @@ -295,7 +297,8 @@ static int tas2764_codec_suspend(struct snd_soc_component *component) if (tas2764->sdz_gpio) gpiod_set_value_cansleep(tas2764->sdz_gpio, 0); - regulator_disable(tas2764->sdz_reg); + if (tas2764->sdz_reg) + regulator_disable(tas2764->sdz_reg); } regcache_cache_only(tas2764->regmap, true); @@ -312,7 +315,7 @@ static int tas2764_codec_resume(struct snd_soc_component *component) int ret; if (!tas2764->sdz_group) { - ret = regulator_enable(tas2764->sdz_reg); + ret = tas2764->sdz_reg ? regulator_enable(tas2764->sdz_reg) : 0; if (ret) { dev_err(tas2764->dev, "Failed to enable regulator\n"); return ret; @@ -1011,7 +1014,8 @@ static void tas2764_codec_remove(struct snd_soc_component *component) if (tas2764_group_failed(tas2764)) return; - regulator_disable(tas2764->sdz_reg); + if (tas2764->sdz_reg) + regulator_disable(tas2764->sdz_reg); } static DECLARE_TLV_DB_SCALE(tas2764_digital_tlv, 1100, 50, 0); @@ -1116,10 +1120,19 @@ static int tas2764_parse_dt(struct device *dev, struct tas2764_priv *tas2764) { int ret = 0; - tas2764->sdz_reg = devm_regulator_get(dev, "SDZ"); - if (IS_ERR(tas2764->sdz_reg)) - return dev_err_probe(dev, PTR_ERR(tas2764->sdz_reg), - "Failed to get SDZ supply\n"); + /* + * The SDZ supply is optional: most boards drive SDZ from a GPIO + * (shutdown-gpios, shared between amps through the GPIO core), and only + * boards that model the line as a regulator have one. A dummy regulator + * in its place did nothing but warn on every boot. + */ + tas2764->sdz_reg = devm_regulator_get_optional(dev, "SDZ"); + if (IS_ERR(tas2764->sdz_reg)) { + if (PTR_ERR(tas2764->sdz_reg) != -ENODEV) + return dev_err_probe(dev, PTR_ERR(tas2764->sdz_reg), + "Failed to get SDZ supply\n"); + tas2764->sdz_reg = NULL; + } tas2764->reset_gpio = devm_gpiod_get_optional(tas2764->dev, "reset", GPIOD_OUT_HIGH); From 653fe41ecf029954c7906db9efb11ddca7a4f72d Mon Sep 17 00:00:00 2001 From: Ryan <81541646+RyanTheTide@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:53:31 +1100 Subject: [PATCH 1481/1579] leds: pwm: Do not warn when keeping an unprogrammed PWM off With default-state = "keep", led_pwm_add() reads the PWM state back and warns when it has no period. A PWM that the bootloader left disabled and never programmed reads back exactly that, so every boot of the Apple M1 Pro and M2 laptops logs leds_pwm led-controller: failed to read period for kbd_backlight, default to off although nothing failed. The firmware hands over with the keyboard backlight off and the FPWM block in its reset state (no on or off cycles), and pwm-apple reports that faithfully as disabled with a period of 0. The DT already gives the period in the pwms specifier. Keeping the state of a disabled PWM means keeping the LED off, which is what the driver then does. Warn only when the PWM reads back enabled without a period, the case where the state really could not be read. Nothing else changes: the LED still starts off and is programmed with the period from the DT. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/leds/leds-pwm.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/leds/leds-pwm.c b/drivers/leds/leds-pwm.c index efe5a97a1f4d6..9633626053a51 100644 --- a/drivers/leds/leds-pwm.c +++ b/drivers/leds/leds-pwm.c @@ -109,9 +109,16 @@ static int led_pwm_add(struct device *dev, struct led_pwm_priv *priv, if (led_data->pwmstate.period) break; led->default_state = LEDS_DEFSTATE_OFF; - dev_warn(dev, - "failed to read period for %s, default to off", - led->name); + /* + * A PWM the bootloader left disabled and never programmed has + * no period to read back, and keeping its state means keeping + * the LED off. Only an enabled PWM without a period is a + * failure to read it. + */ + if (led_data->pwmstate.enabled) + dev_warn(dev, + "failed to read period for %s, default to off", + led->name); fallthrough; default: pwm_init_state(led_data->pwm, &led_data->pwmstate); From 4cf1b7cb0c983e3b81934ffb4ff0dfd6978d98b1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:01:07 +1100 Subject: [PATCH 1482/1579] wifi: brcmfmac: report short GI for HT and VHT ratespecs Preserve the firmware short-GI bit when converting HT and VHT ratespecs to cfg80211 rate information. Without it, nl80211 reports the long-GI bitrate even when the ratespec requests short GI; VHT MCS9 NSS2 at 40 MHz is reported as 360 instead of 400 Mbit/s. Only interpret this bit for HT and VHT. The same bit has different meanings for other encodings, whose guard intervals retain their existing handling. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 989546c595291..0aac63136e979 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -3210,6 +3210,10 @@ static void brcmf_convert_ratespec_to_rateinfo(u32 ratespec, /* Fill in nothing */ break; } + if ((BRCMF_RSPEC_ISHT(ratespec) || BRCMF_RSPEC_ISVHT(ratespec)) && + BRCMF_RSPEC_ISSGI(ratespec)) + rateinfo->flags |= RATE_INFO_FLAGS_SHORT_GI; + if (BRCMF_RSPEC_ISHT(ratespec)) { rateinfo->flags |= RATE_INFO_FLAGS_MCS; rateinfo->mcs = ratespec & BRCMF_RSPEC_HT_MCS_MASK; From 1e488e327b024df2ac37ac1c82b900eee018f29a Mon Sep 17 00:00:00 2001 From: Ryan <81541646+RyanTheTide@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:06:24 +1100 Subject: [PATCH 1483/1579] arm64: dts: apple: Describe the boot NOR flash supply The spi-nor core takes the flash supply with devm_regulator_get_enable() since commit d8bde4fd6636 ("mtd: spi-nor: support vcc-supply regulator"). No Apple DT describes it, so every boot logs spi-nor spi0.0: supply vcc not found, using dummy regulator The boot NOR holds the system firmware and the NVRAM and is powered whenever the SoC is, from a PMU rail that no OS switches. In the M1 Pro (J316s), M2 (J413) and J700 ADTs the spinor node lists no supply or power function. Describe that rail as an always-on fixed regulator and point the flash at it, in spi1-nvram.dtsi for the boards that share it and in the J613 25G83 QSPI, J713 and J700 flash nodes. No voltage is given since none is known from the platform. Nothing changes electrically: the regulator has no control line and is always on, as the rail already is. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/spi1-nvram.dtsi | 11 +++++++++++ arch/arm64/boot/dts/apple/t8122-j613-25g83-qspi.dtsi | 1 + arch/arm64/boot/dts/apple/t8132-j713.dts | 11 +++++++++++ arch/arm64/boot/dts/apple/t8140-j700.dts | 11 +++++++++++ 4 files changed, 34 insertions(+) diff --git a/arch/arm64/boot/dts/apple/spi1-nvram.dtsi b/arch/arm64/boot/dts/apple/spi1-nvram.dtsi index d2720b3077741..268d9ffb94044 100644 --- a/arch/arm64/boot/dts/apple/spi1-nvram.dtsi +++ b/arch/arm64/boot/dts/apple/spi1-nvram.dtsi @@ -11,6 +11,16 @@ aliases { nvram = &nvram; }; + + /* + * The boot NOR runs from a PMU rail that is on whenever the SoC is + * and that no OS switches. The ADT gives the flash no supply. + */ + nor_flash_vcc: regulator-nor-flash-vcc { + compatible = "regulator-fixed"; + regulator-name = "nor-flash-vcc"; + regulator-always-on; + }; }; &spi1 { @@ -20,6 +30,7 @@ compatible = "jedec,spi-nor"; reg = <0x0>; spi-max-frequency = <25000000>; + vcc-supply = <&nor_flash_vcc>; partitions { compatible = "fixed-partitions"; diff --git a/arch/arm64/boot/dts/apple/t8122-j613-25g83-qspi.dtsi b/arch/arm64/boot/dts/apple/t8122-j613-25g83-qspi.dtsi index 2a46a7da98168..e37fde644ad03 100644 --- a/arch/arm64/boot/dts/apple/t8122-j613-25g83-qspi.dtsi +++ b/arch/arm64/boot/dts/apple/t8122-j613-25g83-qspi.dtsi @@ -31,6 +31,7 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <10000000>; + vcc-supply = <&nor_flash_vcc>; partitions { compatible = "fixed-partitions"; diff --git a/arch/arm64/boot/dts/apple/t8132-j713.dts b/arch/arm64/boot/dts/apple/t8132-j713.dts index da55888e6d5e4..efaadea58306b 100644 --- a/arch/arm64/boot/dts/apple/t8132-j713.dts +++ b/arch/arm64/boot/dts/apple/t8132-j713.dts @@ -54,6 +54,16 @@ default-state = "keep"; }; }; + + /* + * The boot NOR runs from a PMU rail that is on whenever the SoC is + * and that no OS switches. The ADT gives the flash no supply. + */ + nor_flash_vcc: regulator-nor-flash-vcc { + compatible = "regulator-fixed"; + regulator-name = "nor-flash-vcc"; + regulator-always-on; + }; }; /* Enable DVFS only with the J713 board thermal owner. */ @@ -72,6 +82,7 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <10000000>; + vcc-supply = <&nor_flash_vcc>; spi-rx-bus-width = <1>; spi-tx-bus-width = <1>; diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index a3044890100e2..ca10179b0a73c 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -69,6 +69,16 @@ reg = <0x100 0 0x2 0>; /* To be filled by loader */ }; + /* + * The boot NOR runs from a PMU rail that is on whenever the SoC is + * and that no OS switches. The ADT gives the flash no supply. + */ + nor_flash_vcc: regulator-nor-flash-vcc { + compatible = "regulator-fixed"; + regulator-name = "nor-flash-vcc"; + regulator-always-on; + }; + /* * The J700 is fanless: the CPUs are the only cooling devices. Every * die sensor gets a zone that throttles both clusters, since they share @@ -460,6 +470,7 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <25000000>; + vcc-supply = <&nor_flash_vcc>; partitions { compatible = "fixed-partitions"; From aaa776fb39de4ea995aee97c93cc19040e738350 Mon Sep 17 00:00:00 2001 From: Ryan <81541646+RyanTheTide@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:06:29 +1100 Subject: [PATCH 1484/1579] arm64: dts: apple: t8140-j700: Describe the headphone codec supplies The cs42l42 driver, which also runs the CS42L83, takes the VA, VP, VCP, VD_FILT and VL supplies with devm_regulator_bulk_get(). The J700 DT describes none of them, so every boot logs five lines like cs42l83 3-0048: supply VA not found, using dummy regulator The J700 ADT gives the codec (/arm-io/i2c0/audio-codec-output) only a reset GPIO and an interrupt, with no supply or power function: its rails come from the PMU, are on whenever the SoC is, and no OS switches them. Describe them as one always-on fixed regulator and point all five supplies at it. No voltage is given since none is known from the platform. Nothing changes electrically: the regulator has no control line and is always on, as the rails already are. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index ca10179b0a73c..84aa5febe23fc 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -394,6 +394,17 @@ }; }; }; + + /* + * The headphone codec runs from PMU rails that are on whenever the + * SoC is and that no OS switches. The ADT gives the codec a reset + * line and no supply or power function. + */ + jack_codec_supply: regulator-jack-codec { + compatible = "regulator-fixed"; + regulator-name = "jack-codec"; + regulator-always-on; + }; }; #include "isp-imx558.dtsi" @@ -513,6 +524,11 @@ jack_codec: codec@48 { compatible = "cirrus,cs42l83"; reg = <0x48>; + VA-supply = <&jack_codec_supply>; + VCP-supply = <&jack_codec_supply>; + VD_FILT-supply = <&jack_codec_supply>; + VL-supply = <&jack_codec_supply>; + VP-supply = <&jack_codec_supply>; reset-gpios = <&pinctrl_ap 103 GPIO_ACTIVE_HIGH>; interrupt-parent = <&pinctrl_ap>; interrupts = <104 IRQ_TYPE_LEVEL_LOW>; From 2490d944a25ca23d65f4358fa4d0e0f4e2c796fa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:10:51 +1100 Subject: [PATCH 1485/1579] dt-bindings: iio: imu: Add Apple AOP accelerometer and gyroscope The Always-On Processor of Apple laptops reads the machine's inertial sensor and reports acceleration and angular rate as two services. Describe their child nodes, and allow them under the T6041 AOP. A service starts reporting only once it has been sent the machine's three calibration packets. They are machine data from the Apple device tree, so the bootloader fills them in at every boot and the device tree sources carry none. Based on Eryk Wieliczko's J713 work (dirtyroom-linux 63e6467a). Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/iio/imu/apple,aop-motion.yaml | 69 +++++++++++++++++++ .../bindings/soc/apple/apple,t6041-aop.yaml | 20 ++++++ MAINTAINERS | 1 + 3 files changed, 90 insertions(+) create mode 100644 Documentation/devicetree/bindings/iio/imu/apple,aop-motion.yaml diff --git a/Documentation/devicetree/bindings/iio/imu/apple,aop-motion.yaml b/Documentation/devicetree/bindings/iio/imu/apple,aop-motion.yaml new file mode 100644 index 0000000000000..e358c89ab18a3 --- /dev/null +++ b/Documentation/devicetree/bindings/iio/imu/apple,aop-motion.yaml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iio/imu/apple,aop-motion.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple AOP accelerometer and gyroscope + +maintainers: + - Ryan Murray + +description: | + The Always-On Processor of Apple laptops reads the machine's inertial + sensor and reports acceleration and angular rate as two of its services. + Each node identifies one of those services. It has no application + processor MMIO or interrupt resource of its own. + + A service starts reporting only once it has been sent the machine's three + calibration packets. The bootloader copies them from the Apple device tree + into the calibration properties below at every boot, so they are absent + from the device tree sources. Without them the sensor stays off. + +properties: + compatible: + oneOf: + - items: + - enum: + - apple,t6041-aop-accel + - apple,t8132-aop-accel + - const: apple,aop-accel + - items: + - enum: + - apple,t6041-aop-gyro + - apple,t8132-aop-gyro + - const: apple,aop-gyro + + label: + description: + Where the sensor sits, "accel-base" for an accelerometer in the base + of a laptop. + + apple,range-intrinsic-cal: + $ref: /schemas/types.yaml#/definitions/uint8-array + description: + The sensor's range calibration container, filled in by the bootloader. + + apple,nominal-extrinsic-cal: + $ref: /schemas/types.yaml#/definitions/uint8-array + description: + The calibration container for the sensor's nominal mounting, filled in + by the bootloader. + + apple,range-extrinsic-cal: + $ref: /schemas/types.yaml#/definitions/uint8-array + description: + The sensor's range mounting calibration container, filled in by the + bootloader. + +required: + - compatible + +additionalProperties: false + +examples: + - | + accel { + compatible = "apple,t6041-aop-accel", "apple,aop-accel"; + label = "accel-base"; + }; diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t6041-aop.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t6041-aop.yaml index e8a2df805059a..6f31a022c6f35 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,t6041-aop.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,t6041-aop.yaml @@ -86,6 +86,18 @@ properties: required: - compatible + accel: + type: object + $ref: /schemas/iio/imu/apple,aop-motion.yaml# + description: The accelerometer service. + unevaluatedProperties: false + + gyro: + type: object + $ref: /schemas/iio/imu/apple,aop-motion.yaml# + description: The gyroscope service. + unevaluatedProperties: false + required: - compatible - reg @@ -120,5 +132,13 @@ examples: las { compatible = "apple,t6041-aop-las", "apple,aop-las"; }; + + accel { + compatible = "apple,t6041-aop-accel", "apple,aop-accel"; + }; + + gyro { + compatible = "apple,t6041-aop-gyro", "apple,aop-gyro"; + }; }; }; diff --git a/MAINTAINERS b/MAINTAINERS index c053af5b9269f..65718ddea5658 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2093,6 +2093,7 @@ F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: Documentation/PCI/controller/apple-t8140.rst +F: Documentation/devicetree/bindings/iio/imu/apple,aop-motion.yaml F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml F: Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml From 3cadfe50e124ee450ca7c7905d17d644b6f7c1b8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:10:51 +1100 Subject: [PATCH 1486/1579] soc: apple: aop: Register the accelerometer and gyroscope services Create service devices for the "accel" and "gyro" services the AOP firmware announces, so that their driver can bind to the matching device tree children. The T8132 firmware sends its motion samples with report subtypes 0xc1 (gyroscope) and 0xc2 (accelerometer) besides the fake-HID 0xc4, all with the same payload, so hand those to the fake-HID listeners too. Based on Eryk Wieliczko's J713 work (dirtyroom-linux 63e6467a), which found both the services and the report subtypes. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/aop.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index c59719d048158..3d4356231e6d6 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -121,6 +121,10 @@ const EPIC_CATEGORY_NOTIFY: u8 = 0x10; const EPIC_CATEGORY_REPLY: u8 = 0x20; const EPIC_SUBTYPE_STD_SERVICE: u16 = 0xc0; const EPIC_SUBTYPE_FAKEHID_REPORT: u16 = 0xc4; +/// The T8132 firmware sends its gyroscope and accelerometer samples with +/// these report subtypes; they carry the same payload as a fake-HID report. +const EPIC_SUBTYPE_GYRO_REPORT: u16 = 0xc1; +const EPIC_SUBTYPE_ACCEL_REPORT: u16 = 0xc2; const EPIC_SUBTYPE_RETCODE: u16 = 0x84; const EPIC_SUBTYPE_RETCODE_PAYLOAD: u16 = 0xa0; const EPIC_SUBTYPE_STRING: u16 = 0x8a; @@ -756,7 +760,10 @@ impl AFKEndpoint { let name = &name[..name.iter().position(|x| *x == 0).unwrap_or(name.len())]; let chan = le_u32(data, EPIC_ANNOUNCE_CHANNEL_OFFSET); return Into::>::into(client).register_service(self, chan, name); - } else if subtype == EPIC_SUBTYPE_FAKEHID_REPORT { + } else if subtype == EPIC_SUBTYPE_FAKEHID_REPORT + || subtype == EPIC_SUBTYPE_GYRO_REPORT + || subtype == EPIC_SUBTYPE_ACCEL_REPORT + { return client.process_fakehid_report(self, qhdr.channel, data); } else if client.process_report(self, qhdr.channel, subtype, data)? { return Ok(()); @@ -1471,6 +1478,8 @@ impl AopData { b"aop-audio" => c_str!("audio"), b"las" => c_str!("las"), b"als" => c_str!("als"), + b"accel" => c_str!("accel"), + b"gyro" => c_str!("gyro"), _ => { return Ok(()); } From d2da58eb27a775640498c7a90315acc5d1aee5dd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:10:51 +1100 Subject: [PATCH 1487/1579] iio: aop: Add the accelerometer and gyroscope driver Add one driver for the AOP accelerometer and gyroscope, matched by the generic apple,aop-accel and apple,aop-gyro compatibles, so every machine with the services shares it. At probe the driver sends the three calibration packets the bootloader copied from the Apple device tree (properties 75, 76 and 77, each the location 0 entry of its container), then a 20000 us report interval. A sample report is 22 bytes with X, Y and Z as signed little-endian 32-bit values at byte 6, in 1/65536 g or 1/65536 degree per second. Without the calibration the driver leaves the sensor off and does not bind. Each sensor is an IIO device with raw X, Y and Z channels, a fixed scale in m/s^2 or rad/s and its sampling frequency. Samples are stored under a lock so a read never mixes two reports. A kfifo buffer takes X, Y and Z together with a timestamp taken when the report arrives, which is what iio-sensor-proxy and other buffer users read. Around system sleep the driver sets the interval to zero, so the AOP does not wake the application processor 50 times a second, and resends the calibration and interval on wake. Host tests run the production decoder and calibration selector against synthetic reports and containers. Based on Eryk Wieliczko's J713 driver (dirtyroom-linux 63e6467a), which first got both sensors reporting at 50 Hz on the M4 MacBook Air. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/iio/aop-sensors.rst | 32 +- drivers/iio/common/aop_sensors/Kconfig | 15 + drivers/iio/common/aop_sensors/Makefile | 1 + drivers/iio/common/aop_sensors/aop_motion.rs | 627 ++++++++++++++++++ rust/bindings/bindings_helper.h | 10 + .../testing/selftests/drivers/apple/Makefile | 4 +- .../selftests/drivers/apple/aop-motion.py | 114 ++++ 7 files changed, 798 insertions(+), 5 deletions(-) create mode 100644 drivers/iio/common/aop_sensors/aop_motion.rs create mode 100755 tools/testing/selftests/drivers/apple/aop-motion.py diff --git a/Documentation/iio/aop-sensors.rst b/Documentation/iio/aop-sensors.rst index 2292ad208350a..0129bde3f7fa1 100644 --- a/Documentation/iio/aop-sensors.rst +++ b/Documentation/iio/aop-sensors.rst @@ -6,8 +6,9 @@ Apple AOP sensors The AOP firmware announces sensor services over EPIC. The AOP core matches each service to a child node in the device tree and forwards its fake-HID -reports to the corresponding IIO driver. Ambient light (``als``) and lid -angle (``las``) use this same transport; no HID bus device is created. +reports to the corresponding IIO driver. Ambient light (``als``), lid +angle (``las``), acceleration (``accel``) and angular rate (``gyro``) use +this same transport; no HID bus device is created. Lid angle ========= @@ -30,12 +31,35 @@ Report 1 carries a 9-bit little-endian value following the report ID. The remaining bits in the last byte are padding. Reads return the cached unsolicited report, rather than issuing a synchronous HID feature request. -Verification -============ +Accelerometer and gyroscope +=========================== + +Enable ``CONFIG_IIO_AOP_SENSOR_MOTION``. The AOP must have ``accel`` and +``gyro`` children with a SoC-specific compatible followed by +``apple,aop-accel`` or ``apple,aop-gyro``. Each service reports only after +it has been sent the machine's three calibration packets, which the +bootloader copies from the Apple device tree into the child's +``apple,range-intrinsic-cal``, ``apple,nominal-extrinsic-cal`` and +``apple,range-extrinsic-cal`` properties. Without them the driver leaves +the sensor off and does not bind. The packets are sent unchanged, then the +50 Hz report interval. + +The devices are named ``aop-sensors-accel`` and ``aop-sensors-gyro``. Each +has ``in_accel_{x,y,z}_raw`` or ``in_anglvel_{x,y,z}_raw``, a shared scale +(acceleration in m/s^2 including gravity, angular rate in rad/s) and +``sampling_frequency``. The axes are the sensor's, in the order it reports +them; the mapping to the machine's axes is not established yet. A raw read +before the first sample returns ``ENODATA``. + +Both devices have a buffer that takes X, Y and Z together, with an optional +timestamp taken when the report arrives. The reports stop when the system +goes to sleep and start again, with the calibration resent, when it wakes. + Host regression tests require Python 3 and a Rust compiler:: tools/testing/selftests/drivers/apple/aop-las.py + tools/testing/selftests/drivers/apple/aop-motion.py The tests compile the production decoder and shared IIO read callback against host mocks. They cover all 16-bit payloads (including padding), diff --git a/drivers/iio/common/aop_sensors/Kconfig b/drivers/iio/common/aop_sensors/Kconfig index b2c4397787dbf..df1277d4082af 100644 --- a/drivers/iio/common/aop_sensors/Kconfig +++ b/drivers/iio/common/aop_sensors/Kconfig @@ -20,3 +20,18 @@ config IIO_AOP_SENSOR_ALS help Module to handle the ambient light sensor attached to the AOP coprocessor on Apple laptops. + +config IIO_AOP_SENSOR_MOTION + tristate "AOP accelerometer and gyroscope" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + depends on SYSFS + select APPLE_AOP + select IIO_BUFFER + select IIO_KFIFO_BUF + help + Module to handle the accelerometer and gyroscope that the AOP + coprocessor of Apple laptops reads. Each sensor is an IIO device + with X, Y and Z channels, readable directly or through a buffer. + The bootloader has to pass the machine's sensor calibration in + the device tree. diff --git a/drivers/iio/common/aop_sensors/Makefile b/drivers/iio/common/aop_sensors/Makefile index 8da5a19efe0f0..4b769a78d44c5 100644 --- a/drivers/iio/common/aop_sensors/Makefile +++ b/drivers/iio/common/aop_sensors/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_IIO_AOP_SENSOR_LAS) += aop_las.o obj-$(CONFIG_IIO_AOP_SENSOR_ALS) += aop_als.o +obj-$(CONFIG_IIO_AOP_SENSOR_MOTION) += aop_motion.o diff --git a/drivers/iio/common/aop_sensors/aop_motion.rs b/drivers/iio/common/aop_sensors/aop_motion.rs new file mode 100644 index 0000000000000..a3d1cd0eff6c7 --- /dev/null +++ b/drivers/iio/common/aop_sensors/aop_motion.rs @@ -0,0 +1,627 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple AOP accelerometer and gyroscope driver +//! +//! The Always-On Processor reads the machine's inertial sensor and reports +//! acceleration and angular rate as two services, one fake-HID report per +//! sample. A service starts reporting once it has been sent the machine's +//! three calibration packets and a report interval. + +use core::{ + ffi::{c_int, c_void}, + marker::PhantomPinned, + ptr, +}; + +use kernel::{ + bindings, c_str, + device::{self, Core}, + module_platform_driver, new_spinlock, of, platform, + prelude::*, + soc::apple::aop::{EPICService, FakehidListener, FakehidRegistration, AOP}, + sync::{ + aref::ARef, + atomic::{Atomic, Relaxed}, + Arc, SpinLock, + }, + types::{ForeignOwnable, Opaque}, +}; + +/// The EPIC notification that writes a property of a sensor service. +const SUBTYPE_SET_PROPERTY: u16 = 4; +/// The report interval property, in microseconds. Zero stops the reports. +const PROP_INTERVAL: u32 = 0; +/// 50 Hz. +const INTERVAL_US: u32 = 20_000; +/// The calibration properties of a service and the device tree properties the +/// bootloader fills them from. They are written in this order, before the +/// report interval. +const CALIBRATION: [(u32, &CStr); 3] = [ + (75, c_str!("apple,range-intrinsic-cal")), + (76, c_str!("apple,nominal-extrinsic-cal")), + (77, c_str!("apple,range-extrinsic-cal")), +]; +/// A sample report is 22 bytes; X, Y and Z are signed little-endian 32-bit +/// values in 1/65536 of the sensor's unit (g or degrees per second), starting +/// at byte 6. The other bytes carry no sample. +const REPORT_LEN: usize = 22; +const XYZ_OFFSET: usize = 6; + +/// What one of the two services measures. +struct Kind { + name: &'static CStr, + chan_type: u32, + /// The size of one raw count in m/s^2 or rad/s, in billionths. + scale_nano: i32, +} + +/// 9.80665 / 65536 m/s^2 per count. +static ACCEL: Kind = Kind { + name: c"aop-sensors-accel", + chan_type: bindings::BINDINGS_IIO_ACCEL, + scale_nano: 149_638, +}; + +/// pi / (180 * 65536) rad/s per count. +static GYRO: Kind = Kind { + name: c"aop-sensors-gyro", + chan_type: bindings::BINDINGS_IIO_ANGL_VEL, + scale_nano: 266, +}; + +/// Decodes a sample report into its X, Y and Z counts. +fn decode(report: &[u8]) -> Option<[i32; 3]> { + if report.len() != REPORT_LEN { + return None; + } + let axis = |i: usize| { + let at = XYZ_OFFSET + 4 * i; + i32::from_le_bytes([report[at], report[at + 1], report[at + 2], report[at + 3]]) + }; + Some([axis(0), axis(1), axis(2)]) +} + +fn le_u32(bytes: &[u8], at: usize) -> Option { + Some(u32::from_le_bytes(bytes.get(at..at + 4)?.try_into().ok()?)) +} + +/// Returns the packet for location 0 of a calibration property. +/// +/// The property is a container: byte 3 is 1 and byte 2 counts the entries +/// that follow the 4-byte header. Each entry is a little-endian 32-bit +/// location and length, then that many bytes of packet. +fn location0_packet(prop: &[u8]) -> Option<&[u8]> { + if prop.len() < 4 || prop[3] != 1 { + return None; + } + let mut at = 4; + for _ in 0..prop[2] { + let location = le_u32(prop, at)?; + let len = usize::try_from(le_u32(prop, at + 4)?).ok()?; + let start = at + 8; + let packet = prop.get(start..start.checked_add(len)?)?; + if location == 0 { + return Some(packet); + } + at = start + len; + } + None +} + +/// Starts and stops the reports of one service. +struct Control { + dev: ARef, + aop: Arc, + service: EPICService, + packets: [KVec; 3], +} + +impl Control { + fn new(dev: &device::Device, aop: Arc, service: EPICService) -> Result> { + let node = dev.fwnode().ok_or(ENODEV)?; + let mut packets = [KVec::new(), KVec::new(), KVec::new()]; + for ((_, name), packet) in CALIBRATION.iter().zip(packets.iter_mut()) { + if !node.property_present(name) { + dev_info!(dev, "no {:?} from the bootloader, sensor left off\n", name); + return Err(ENODEV); + } + let len = node.property_count_elem::(name)?; + let prop = node + .property_read_array_vec::(name, len)? + .required_by(dev)?; + let Some(data) = location0_packet(&prop) else { + dev_err!(dev, "{:?} holds no calibration for location 0\n", name); + return Err(EINVAL); + }; + packet.extend_from_slice(data, GFP_KERNEL)?; + } + Ok(Arc::new( + Self { + dev: dev.into(), + aop, + service, + packets, + }, + GFP_KERNEL, + )?) + } + + fn set(&self, prop: u32, value: &[u8]) -> Result { + let mut msg = KVec::with_capacity(8 + value.len(), GFP_KERNEL)?; + msg.extend_from_slice(&0u32.to_le_bytes(), GFP_KERNEL)?; + msg.extend_from_slice(&prop.to_le_bytes(), GFP_KERNEL)?; + msg.extend_from_slice(value, GFP_KERNEL)?; + let status = self + .aop + .epic_call(&self.service, SUBTYPE_SET_PROPERTY, &msg)?; + if status != 0 { + dev_err!( + self.dev, + "property {} refused: status {:#x}\n", + prop, + status + ); + return Err(EIO); + } + Ok(()) + } + + /// Sends the calibration, then the interval that starts the reports. + fn start(&self) -> Result { + for ((prop, _), packet) in CALIBRATION.iter().zip(self.packets.iter()) { + self.set(*prop, packet)?; + } + self.set(PROP_INTERVAL, &INTERVAL_US.to_le_bytes()) + } + + fn stop(&self) -> Result { + self.set(PROP_INTERVAL, &0u32.to_le_bytes()) + } +} + +/// The latest sample, read by the IIO callbacks. +#[derive(Clone, Copy, Default)] +struct Sample { + xyz: [i32; 3], + valid: bool, +} + +/// One buffer scan: X, Y and Z, then the timestamp at the next 8-byte +/// boundary, where the IIO core expects it. +#[repr(C, align(8))] +struct Scan { + xyz: [i32; 3], + pad: u32, + timestamp: i64, +} + +/// The state shared by the report listener and the IIO device. +#[pin_data] +struct State { + kind: &'static Kind, + /// The IIO device, which outlives the report listener. + iio: IioPtr, + #[pin] + sample: SpinLock, +} + +struct IioPtr(*mut bindings::iio_dev); + +// SAFETY: The IIO device may be used from any thread; the functions this +// driver calls on it from the listener are safe from any context. +unsafe impl Send for IioPtr {} +// SAFETY: See above. +unsafe impl Sync for IioPtr {} + +impl FakehidListener for State { + fn process_fakehid_report(&self, data: &[u8]) -> Result<()> { + let xyz = decode(data).ok_or(EMSGSIZE)?; + *self.sample.lock() = Sample { xyz, valid: true }; + // SAFETY: The listener is removed before the IIO device is freed. + let timestamp = unsafe { bindings::iio_get_time_ns(self.iio.0) }; + let scan = Scan { + xyz, + pad: 0, + timestamp, + }; + // SAFETY: As above; `scan` holds the full scan of all three axes, + // which is the only scan mask the device offers, followed by the + // timestamp. The core copies what the enabled buffers take. + unsafe { bindings::iio_push_to_buffers(self.iio.0, ptr::from_ref(&scan).cast()) }; + Ok(()) + } +} + +/// The only scan the device offers: X, Y and Z. +static SCAN_MASKS: [usize; 2] = [0b111, 0]; + +/// The `read_raw` callback of both devices. +/// +/// # Safety +/// +/// The IIO core calls it with a device registered by [`MotionIio`], one of its +/// channels and valid result pointers. +unsafe extern "C" fn motion_read_raw( + indio_dev: *mut bindings::iio_dev, + chan: *const bindings::iio_chan_spec, + val: *mut i32, + val2: *mut i32, + mask: isize, +) -> i32 { + // SAFETY: The IIO core calls this with the device `MotionIio::new()` + // registered, whose private data is the `Arc` stored there. + let state = unsafe { Arc::::borrow((*indio_dev).priv_.cast()) }; + // SAFETY: `chan` is one of the registered channels. + let axis = unsafe { (*chan).address }; + let (int, frac, ret) = if mask == bindings::BINDINGS_IIO_CHAN_INFO_RAW as isize { + let sample = *state.sample.lock(); + if !sample.valid { + return ENODATA.to_errno(); + } + let Some(&count) = sample.xyz.get(axis) else { + return EINVAL.to_errno(); + }; + (count, 0, bindings::IIO_VAL_INT) + } else if mask == bindings::BINDINGS_IIO_CHAN_INFO_SCALE as isize { + (0, state.kind.scale_nano, bindings::IIO_VAL_INT_PLUS_NANO) + } else if mask == bindings::BINDINGS_IIO_CHAN_INFO_SAMP_FREQ as isize { + ((1_000_000 / INTERVAL_US) as i32, 0, bindings::IIO_VAL_INT) + } else { + return EINVAL.to_errno(); + }; + // SAFETY: `val` and `val2` point to the caller's result variables. + unsafe { + *val = int; + *val2 = frac; + } + ret as i32 +} + +fn axis_channel(chan_type: u32, modifier: u32, index: i32) -> bindings::iio_chan_spec { + let mut chan = bindings::iio_chan_spec { + type_: chan_type, + channel2: modifier as i32, + address: index as usize, + scan_index: index, + __bindgen_anon_1: bindings::iio_chan_spec__bindgen_ty_1 { + scan_type: bindings::iio_scan_type { + __bindgen_anon_1: bindings::iio_scan_type__bindgen_ty_1 { sign: b's' }, + realbits: 32, + storagebits: 32, + ..Default::default() + }, + }, + info_mask_separate: 1usize << bindings::BINDINGS_IIO_CHAN_INFO_RAW, + info_mask_shared_by_type: (1usize << bindings::BINDINGS_IIO_CHAN_INFO_SCALE) + | (1usize << bindings::BINDINGS_IIO_CHAN_INFO_SAMP_FREQ), + ..Default::default() + }; + chan.set_modified(1); + chan +} + +fn timestamp_channel(index: i32) -> bindings::iio_chan_spec { + bindings::iio_chan_spec { + type_: bindings::BINDINGS_IIO_TIMESTAMP, + channel: -1, + scan_index: index, + __bindgen_anon_1: bindings::iio_chan_spec__bindgen_ty_1 { + scan_type: bindings::iio_scan_type { + __bindgen_anon_1: bindings::iio_scan_type__bindgen_ty_1 { sign: b's' }, + realbits: 64, + storagebits: 64, + ..Default::default() + }, + }, + ..Default::default() + } +} + +struct Spec { + channels: [bindings::iio_chan_spec; 4], + info: bindings::iio_info, + _p: PhantomPinned, +} + +/// The IIO device of a service, with its buffer. +struct MotionIio { + dev: *mut bindings::iio_dev, + _spec: Pin>, + registered: bool, +} + +impl MotionIio { + /// Allocates the device and its buffer; [`MotionIio::register`] makes it + /// visible. + fn new(pdev: &platform::Device>, kind: &'static Kind) -> Result<(Self, Arc)> { + let dev = pdev.as_ref(); + let spec = KBox::pin( + Spec { + channels: [ + axis_channel(kind.chan_type, bindings::BINDINGS_IIO_MOD_X, 0), + axis_channel(kind.chan_type, bindings::BINDINGS_IIO_MOD_Y, 1), + axis_channel(kind.chan_type, bindings::BINDINGS_IIO_MOD_Z, 2), + timestamp_channel(3), + ], + info: bindings::iio_info { + read_raw: Some(motion_read_raw), + ..Default::default() + }, + _p: PhantomPinned, + }, + GFP_KERNEL, + )?; + // SAFETY: `dev` is a valid device. + let indio_dev = unsafe { bindings::iio_device_alloc(dev.as_raw(), 0) }; + if indio_dev.is_null() { + return Err(ENOMEM); + } + let state = match Arc::pin_init( + pin_init!(State { + kind, + iio: IioPtr(indio_dev), + sample <- new_spinlock!(Sample::default()), + }), + GFP_KERNEL, + ) { + Ok(state) => state, + Err(e) => { + // SAFETY: Allocated above and not registered. + unsafe { bindings::iio_device_free(indio_dev) }; + return Err(e); + } + }; + // SAFETY: `indio_dev` was allocated above and is not registered yet; + // `spec` is pinned and lives as long as the device. + unsafe { + (*indio_dev).priv_ = state.clone().into_foreign().cast(); + (*indio_dev).name = kind.name.as_char_ptr(); + (*indio_dev).channels = spec.channels.as_ptr(); + (*indio_dev).num_channels = spec.channels.len() as i32; + (*indio_dev).info = &spec.info; + (*indio_dev).modes = bindings::INDIO_DIRECT_MODE as i32; + (*indio_dev).available_scan_masks = SCAN_MASKS.as_ptr(); + } + let this = Self { + dev: indio_dev, + _spec: spec, + registered: false, + }; + // SAFETY: Both devices are valid. The buffer is released with the + // platform device's resources; the IIO device holds its own + // reference to it until it is freed. + let ret = unsafe { + bindings::devm_iio_kfifo_buffer_setup_ext( + dev.as_raw(), + indio_dev, + ptr::null(), + ptr::null_mut(), + ) + }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + Ok((this, state)) + } + + fn register(&mut self) -> Result { + let module = kernel::module::this_module::(); + // SAFETY: `self.dev` is a fully set up, unregistered IIO device. + let ret = unsafe { bindings::__iio_device_register(self.dev, module.as_ptr()) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + self.registered = true; + Ok(()) + } +} + +impl Drop for MotionIio { + fn drop(&mut self) { + // SAFETY: `self.dev` was allocated in `new()` with an `Arc` + // as its private data, and nothing uses it after this. + unsafe { + if self.registered { + bindings::iio_device_unregister(self.dev); + } + drop(Arc::::from_foreign((*self.dev).priv_.cast())); + bindings::iio_device_free(self.dev); + } + } +} + +// SAFETY: The IIO core serializes registration and removal internally. +unsafe impl Send for MotionIio {} +// SAFETY: See above; `MotionIio` has no methods taking `&self`. +unsafe impl Sync for MotionIio {} + +/// Stops the reports around system sleep, so that the AOP does not wake the +/// application processor 50 times a second, and starts them again after. +#[repr(C)] +struct PmHook { + /// First, so that the callback finds the hook from it. + nb: Opaque, + control: Arc, + registered: Atomic, +} + +/// The PM notifier callback. +/// +/// # Safety +/// +/// `nb` is the notifier block of a registered [`PmHook`]. +unsafe extern "C" fn pm_event( + nb: *mut bindings::notifier_block, + action: usize, + _data: *mut c_void, +) -> c_int { + // SAFETY: `nb` is the first field of a registered `PmHook`. + let hook = unsafe { &*nb.cast::() }; + let ret = match action as u32 { + bindings::PM_SUSPEND_PREPARE | bindings::PM_HIBERNATION_PREPARE => hook.control.stop(), + bindings::PM_POST_SUSPEND | bindings::PM_POST_HIBERNATION => hook.control.start(), + _ => return bindings::NOTIFY_DONE as c_int, + }; + if let Err(e) = ret { + dev_warn!( + hook.control.dev, + "reports not switched for sleep: {:?}\n", + e + ); + } + bindings::NOTIFY_OK as c_int +} + +/// # Safety +/// +/// `nb` stays valid and pinned until [`pm_unregister`] is called on it. +#[cfg(CONFIG_PM_SLEEP)] +unsafe fn pm_register(nb: *mut bindings::notifier_block) -> c_int { + // SAFETY: By the caller's contract. + unsafe { bindings::register_pm_notifier(nb) } +} + +/// # Safety +/// +/// `nb` was registered by [`pm_register`]. +#[cfg(CONFIG_PM_SLEEP)] +unsafe fn pm_unregister(nb: *mut bindings::notifier_block) { + // SAFETY: By the caller's contract. + unsafe { bindings::unregister_pm_notifier(nb) }; +} + +/// Without system sleep there is nothing to switch. +/// +/// # Safety +/// +/// None; the signature matches the one with system sleep. +#[cfg(not(CONFIG_PM_SLEEP))] +unsafe fn pm_register(_nb: *mut bindings::notifier_block) -> c_int { + 0 +} + +/// # Safety +/// +/// None; the signature matches the one with system sleep. +#[cfg(not(CONFIG_PM_SLEEP))] +unsafe fn pm_unregister(_nb: *mut bindings::notifier_block) {} + +// SAFETY: The notifier block is only touched by the PM core, which +// serializes registration, removal and the callbacks itself. +unsafe impl Send for PmHook {} +// SAFETY: See above; the other fields are `Sync`. +unsafe impl Sync for PmHook {} + +impl PmHook { + fn new(control: Arc) -> Result>> { + let hook = KBox::pin( + Self { + nb: Opaque::new(bindings::notifier_block { + notifier_call: Some(pm_event), + next: ptr::null_mut(), + priority: 0, + }), + control, + registered: Atomic::new(false), + }, + GFP_KERNEL, + )?; + // SAFETY: The block is pinned inside the hook, which unregisters it + // before it goes. + let ret = unsafe { pm_register(hook.nb.get()) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + hook.registered.store(true, Relaxed); + Ok(hook) + } + + /// Returns once no callback is running any more; repeated calls do + /// nothing. + fn unregister(&self) { + if self.registered.xchg(false, Relaxed) { + // SAFETY: `nb` was registered by `new()`. + unsafe { pm_unregister(self.nb.get()) }; + } + } +} + +impl Drop for PmHook { + fn drop(&mut self) { + self.unregister(); + } +} + +struct IIOAopMotionDriver { + /// Fields are declared in teardown order: sleep handling first, then the + /// subscription, which ends before the IIO device goes. + pm: Pin>, + listener: FakehidRegistration, + control: Arc, + _iio: MotionIio, +} + +kernel::of_device_table!( + OF_TABLE, + &'static Kind, + [ + (of::DeviceId::new(c_str!("apple,aop-accel")), &ACCEL), + (of::DeviceId::new(c_str!("apple,aop-gyro")), &GYRO), + ] +); + +impl platform::Driver for IIOAopMotionDriver { + type IdInfo = &'static Kind; + type Data<'bound> = IIOAopMotionDriver; + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe<'bound>( + pdev: &'bound platform::Device>, + info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound { + let kind = *info.ok_or(ENODEV)?; + let dev = pdev.as_ref(); + // SAFETY: This driver binds only to the service devices the AOP core + // registers, and `pdev` is being probed. + let aop = unsafe { ::from_child(dev) }?; + // SAFETY: `dev` is a valid device; the AOP core attached its service + // record as the platform data of every service device it registers. + let service = unsafe { (*dev.as_raw()).platform_data.cast::() }; + if service.is_null() { + return Err(ENODEV); + } + // SAFETY: The record lives as long as the device it is attached to. + let service = unsafe { *service }; + + let control = Control::new(dev, aop.clone(), service)?; + let (mut iio, state) = MotionIio::new(pdev, kind)?; + let listener = FakehidRegistration::new(aop, service, state)?; + let pm = PmHook::new(control.clone())?; + control.start()?; + if let Err(e) = iio.register() { + let _ = control.stop(); + return Err(e); + } + Ok(IIOAopMotionDriver { + pm, + listener, + control, + _iio: iio, + }) + } + + fn unbind(_dev: &platform::Device>, this: Pin<&Self>) { + this.pm.unregister(); + let _ = this.control.stop(); + this.listener.unregister(); + } +} + +module_platform_driver! { + type: IIOAopMotionDriver, + name: "iio_aop_motion", + description: "AOP accelerometer and gyroscope driver", + license: "Dual MIT/GPL", + alias: ["platform:iio_aop_motion"], +} diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 9e384a40cd494..c8fab904402b5 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -69,7 +69,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -107,6 +109,7 @@ #include #include #include +#include #include #include #include @@ -166,6 +169,13 @@ const u32 BINDINGS_IIO_CHAN_INFO_PROCESSED = IIO_CHAN_INFO_PROCESSED; const u32 BINDINGS_IIO_CHAN_INFO_SCALE = IIO_CHAN_INFO_SCALE; const u32 BINDINGS_IIO_ANGL = IIO_ANGL; const u32 BINDINGS_IIO_LIGHT = IIO_LIGHT; +const u32 BINDINGS_IIO_ACCEL = IIO_ACCEL; +const u32 BINDINGS_IIO_ANGL_VEL = IIO_ANGL_VEL; +const u32 BINDINGS_IIO_TIMESTAMP = IIO_TIMESTAMP; +const u32 BINDINGS_IIO_MOD_X = IIO_MOD_X; +const u32 BINDINGS_IIO_MOD_Y = IIO_MOD_Y; +const u32 BINDINGS_IIO_MOD_Z = IIO_MOD_Z; +const u32 BINDINGS_IIO_CHAN_INFO_SAMP_FREQ = IIO_CHAN_INFO_SAMP_FREQ; const xa_mark_t RUST_CONST_HELPER_XA_PRESENT = XA_PRESENT; diff --git a/tools/testing/selftests/drivers/apple/Makefile b/tools/testing/selftests/drivers/apple/Makefile index 0408fc701dfa0..a325eed4b3ba3 100644 --- a/tools/testing/selftests/drivers/apple/Makefile +++ b/tools/testing/selftests/drivers/apple/Makefile @@ -6,9 +6,11 @@ # sn201202x-transport.py checks the production SPMI selection and payload path. # bcm4377 tests exercise DMA ownership, context addresses and firmware handshakes. # aop-las.py checks the production angle decoder and IIO callback on the host. +# aop-motion.py checks the motion report decoder and calibration selector. TEST_PROGS := macsmc-j713.py atc-t8132-usb3.py pmgr-inherited.py dwc3-initial-role.py \ sn201202x-transport.py bcm4377-dma-window.py bcm4377-radio-sids.py \ - bcm4377-state-wait.py bcm4377-rti-address.py aop-las.py + bcm4377-state-wait.py bcm4377-rti-address.py aop-las.py \ + aop-motion.py TEST_FILES := j813-peripherals.sh include ../../lib.mk diff --git a/tools/testing/selftests/drivers/apple/aop-motion.py b/tools/testing/selftests/drivers/apple/aop-motion.py new file mode 100755 index 0000000000000..592474bf709d0 --- /dev/null +++ b/tools/testing/selftests/drivers/apple/aop-motion.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run the production motion report decoder and calibration selector on the host.""" +import os +from pathlib import Path +import shlex +import subprocess +import tempfile + + +def block(source, marker): + start = source.index(marker) + brace = source.index("{", start) + depth = 1 + end = brace + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +def const(source, name): + start = source.index(f"const {name}") + return source[start:source.index(";", start) + 1] + + +def main(): + root = Path(__file__).resolve().parents[5] + motion = (root / "drivers/iio/common/aop_sensors/aop_motion.rs").read_text() + parts = [const(motion, "REPORT_LEN"), const(motion, "XYZ_OFFSET"), + block(motion, "fn decode("), block(motion, "fn le_u32("), + block(motion, "fn location0_packet("), r''' +fn report(prefix: u8, xyz: [i32; 3], suffix: u8) -> Vec { + let mut r = vec![prefix; 6]; + for v in xyz { + r.extend_from_slice(&v.to_le_bytes()); + } + r.extend_from_slice(&[suffix; 4]); + r +} +#[test] +fn decodes_signed_axes() { + // Synthetic reports; the bytes around the samples carry no sample. + for (prefix, suffix) in [(0, 0), (0xff, 0xff), (0x5a, 0xa5)] { + for xyz in [[65536, -65536, 0], [i32::MIN, i32::MAX, -1]] { + assert_eq!(decode(&report(prefix, xyz, suffix)), Some(xyz)); + } + } +} +#[test] +fn rejects_other_lengths() { + let r = report(0, [1, 2, 3], 0); + for len in 0..40 { + if len == REPORT_LEN { + continue; + } + let mut v = r.clone(); + v.resize(len, 0); + assert_eq!(decode(&v), None); + } +} +fn container(entries: &[(u32, &[u8])]) -> Vec { + let mut c = vec![2, 0, entries.len() as u8, 1]; + for (location, packet) in entries { + c.extend_from_slice(&location.to_le_bytes()); + c.extend_from_slice(&(packet.len() as u32).to_le_bytes()); + c.extend_from_slice(packet); + } + c +} +#[test] +fn selects_location_zero() { + let a = [1u8; 44]; + let b = [2u8; 40]; + let one = container(&[(0, &a)]); + assert_eq!(location0_packet(&one), Some(&a[..])); + // Trailing bytes after the entries are not part of the packet. + let mut padded = one.clone(); + padded.extend_from_slice(&[0; 4]); + assert_eq!(location0_packet(&padded), Some(&a[..])); + let two = container(&[(3, &b), (0, &a)]); + assert_eq!(location0_packet(&two), Some(&a[..])); + assert_eq!(location0_packet(&container(&[(1, &a)])), None); +} +#[test] +fn rejects_malformed_containers() { + let a = [1u8; 44]; + let good = container(&[(0, &a)]); + for len in 0..good.len() { + assert_eq!(location0_packet(&good[..len]), None, "truncated to {len}"); + } + let mut not_container = good.clone(); + not_container[3] = 0; + assert_eq!(location0_packet(¬_container), None); + let mut huge = good.clone(); + huge[8..12].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(location0_packet(&huge), None); + let mut many = good.clone(); + many[2] = 255; + assert_eq!(location0_packet(&many), Some(&a[..])); +} +'''] + with tempfile.TemporaryDirectory(prefix="aop-motion-") as tmp: + source = Path(tmp) / "test.rs" + binary = Path(tmp) / "test" + source.write_text("\n".join(parts)) + subprocess.run(shlex.split(os.environ.get("RUSTC", "rustc")) + [ + "--edition=2021", "--test", "-Dwarnings", str(source), "-o", str(binary), + ], check=True) + subprocess.run([str(binary)], check=True) + + +if __name__ == "__main__": + main() From f185987448bd0e421dbfdf1b4de0439f67216543 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:10:51 +1100 Subject: [PATCH 1488/1579] arm64: dts: apple: t6040: Add the AOP accelerometer and gyroscope Add the accel and gyro children of the T6040 AOP. The bootloader adds their calibration from the Apple device tree. On the J616s the sensor is on the logic board, so label the accelerometer accel-base, which keeps desktops from rotating the screen when the laptop tilts. Enable the driver in asahi.config. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 5 +++++ arch/arm64/boot/dts/apple/t6040.dtsi | 9 +++++++++ arch/arm64/configs/asahi.config | 1 + 3 files changed, 15 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 69f0070af7659..35ef8168eec4f 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -4013,6 +4013,11 @@ status = "okay"; }; +/* The sensor is on the logic board, in the base. */ +&aop_accel { + label = "accel-base"; +}; + &admac0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 6c54e05caa743..59bc828d032fb 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -1295,6 +1295,15 @@ aop_las: las { compatible = "apple,t6041-aop-las", "apple,aop-las"; }; + + /* The calibration properties come from the bootloader. */ + aop_accel: accel { + compatible = "apple,t6041-aop-accel", "apple,aop-accel"; + }; + + aop_gyro: gyro { + compatible = "apple,t6041-aop-gyro", "apple,aop-gyro"; + }; }; /* ADT lists the interrupts as 1, 0, 3, 2 of the block: send-empty is the lowest. */ diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 3c8c4d225985e..d43449d9d1a6d 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -118,6 +118,7 @@ CONFIG_APPLE_PMP_THERMAL=m CONFIG_APPLE_T8132_PLATFORM_HANDOFF=y CONFIG_IIO_AOP_SENSOR_LAS=m CONFIG_IIO_AOP_SENSOR_ALS=m +CONFIG_IIO_AOP_SENSOR_MOTION=m CONFIG_RUST_FW_LOADER_ABSTRACTIONS=y CONFIG_PWM_APPLE=m CONFIG_APPLE_AIC=y From 2d24fbce39a391c5e450deda3793b71d7aa25ea2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:08:40 +1100 Subject: [PATCH 1489/1579] sched/fair: Rotate misfit tasks with tasks on busy bigger CPUs Load balancing moves a misfit task to a bigger CPU only when the bigger CPU's group has spare capacity (update_sd_pick_busiest() rejects a misfit source when the local group is not group_has_spare). When every bigger CPU already runs one task, nothing moves: the task that started on a small CPU keeps it for as long as the others run. Equal tasks then make unequal progress, and work split evenly across all CPUs ends with a tail in which the tasks that started on small CPUs finish alone on big ones. On the M4 Pro (T6040, 10 P cores at capacity 1024 and 4 E cores at 383) a 14-thread CoreMark runs for about 15 s with all CPUs busy, then the 4 threads that ran on the E cores finish alone on P cores about 5 s later. It scores 425k iter/s against 546k on macOS on the same machine. Rotating the benchmark's threads across all CPUs every 20 ms from user space gives 545k, so the loss is all placement. Once a task has been misfit on a CPU for four base slices, have the tick queue a work item that swaps it with the task on a busy bigger CPU, using migrate_swap(). Bigger CPUs are taken round robin, so each task on them takes its turn on a smaller CPU and the big CPUs are time shared the way one runqueue would share them. The swap only happens when every allowed bigger CPU runs exactly one fair task, since an idle one pulls the misfit task through the existing misfit balancing, and it never moves down a task with a higher weight or a higher uclamp_min than the misfit task. Kernel threads and pinned tasks are left alone. migrate_swap() was built only for NUMA balancing; build it on every SMP kernel, and keep the NUMA tracepoint for the NUMA caller. The MISFIT_ROTATE scheduler feature is on by default, like the other fair class heuristics. Assisted-by: LLM Signed-off-by: Ryan Murray --- kernel/sched/core.c | 10 ++- kernel/sched/fair.c | 133 ++++++++++++++++++++++++++++++++++++++++ kernel/sched/features.h | 7 +++ kernel/sched/sched.h | 23 ++++++- 4 files changed, 169 insertions(+), 4 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 30aec75df0efe..96e6eaf36b856 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3476,7 +3476,11 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) } #endif /* CONFIG_SMP */ -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_SMP +/* + * Swapping two running tasks is used by NUMA balancing and by misfit + * rotation on asymmetric CPU capacity systems. + */ static void __migrate_swap_task(struct task_struct *p, int cpu) { if (task_on_rq_queued(p)) { @@ -3574,13 +3578,15 @@ int migrate_swap(struct task_struct *cur, struct task_struct *p, if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr)) goto out; +#ifdef CONFIG_NUMA_BALANCING trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); +#endif ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); out: return ret; } -#endif /* CONFIG_NUMA_BALANCING */ +#endif /* CONFIG_SMP */ /*** * kick_process - kick a running thread to enter/exit the kernel diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 90ea4c950b952..b053e300ba43c 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6353,6 +6353,7 @@ static inline void update_misfit_status(struct task_struct *p, struct rq *rq) task_fits_cpu(p, cpu)) { rq->misfit_task_load = 0; + rq->misfit_task = NULL; return; } @@ -6361,6 +6362,11 @@ static inline void update_misfit_status(struct task_struct *p, struct rq *rq) * task_h_load() returns 0. */ rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1); + + if (rq->misfit_task != p) { + rq->misfit_task = p; + rq->misfit_since = jiffies; + } } void __setparam_fair(struct task_struct *p, const struct sched_attr *attr) @@ -15025,6 +15031,130 @@ static __latent_entropy void sched_balance_softirq(void) sched_balance_domains(this_rq, idle); } +/* + * Misfit rotation. + * + * Load balancing moves a misfit task to a bigger CPU only when a bigger CPU + * has spare capacity. When every bigger CPU runs one task of its own, the + * misfit task stays on its small CPU for as long as those tasks run, so + * equal tasks make unequal progress. Work split evenly across all CPUs then + * ends with a tail in which the tasks that started on small CPUs finish + * alone on big ones. + * + * Once a task has been misfit on a CPU for a few base slices, swap it with + * the task on a busy bigger CPU. Bigger CPUs are taken round robin, so each + * task on them takes its turn on a smaller CPU: the big CPUs are time + * shared, as one runqueue would share them. It only applies when every + * allowed bigger CPU runs exactly one fair task (an idle one pulls the + * misfit task through load balancing instead), and it never moves down a + * task with a higher weight or a higher uclamp_min than the misfit task. + */ +static int misfit_rotate_last_cpu; + +static unsigned long misfit_rotate_period(void) +{ + return max(1UL, nsecs_to_jiffies(4ULL * sysctl_sched_base_slice)); +} + +static bool misfit_rotate_candidate(struct task_struct *p) +{ + return p && p->sched_class == &fair_sched_class && + !(p->flags & (PF_EXITING | PF_KTHREAD)) && + p->nr_cpus_allowed > 1; +} + +static void misfit_rotate_work(struct work_struct *work) +{ + struct rq *src_rq = container_of(work, struct rq, misfit_rotate_work); + int src_cpu = cpu_of(src_rq), dst_cpu = -1, cpu, start; + unsigned long src_cap = arch_scale_cpu_capacity(src_cpu); + struct task_struct *p, *t = NULL; + bool idle = false; + + p = xchg(&src_rq->misfit_rotate_p, NULL); + if (!p) + return; + + /* The task may have moved, slept or shrunk since the tick. */ + if (task_cpu(p) != src_cpu || !task_on_rq_queued(p) || + !misfit_rotate_candidate(p) || task_fits_cpu(p, src_cpu)) + goto out; + + rcu_read_lock(); + start = READ_ONCE(misfit_rotate_last_cpu); + for_each_cpu_wrap(cpu, p->cpus_ptr, start + 1) { + struct rq *rq = cpu_rq(cpu); + struct task_struct *c; + + if (arch_scale_cpu_capacity(cpu) <= src_cap || !cpu_active(cpu)) + continue; + + if (!READ_ONCE(rq->nr_running)) { + idle = true; + break; + } + + if (t || READ_ONCE(rq->nr_running) != 1) + continue; + + c = rcu_dereference_all(rq->curr); + if (!misfit_rotate_candidate(c) || + !cpumask_test_cpu(src_cpu, c->cpus_ptr)) + continue; + + if (c->se.load.weight > p->se.load.weight || + uclamp_eff_value(c, UCLAMP_MIN) > uclamp_eff_value(p, UCLAMP_MIN)) + continue; + + t = c; + dst_cpu = cpu; + } + + if (idle) + t = NULL; + if (t) { + get_task_struct(t); + WRITE_ONCE(misfit_rotate_last_cpu, dst_cpu); + } + rcu_read_unlock(); + + if (t) { + migrate_swap(p, t, dst_cpu, src_cpu); + put_task_struct(t); + } +out: + put_task_struct(p); +} + +/* Called from the tick on this CPU, so current is rq->curr. */ +static void misfit_rotate_trigger(struct rq *rq) +{ + struct task_struct *p = current; + + if (!sched_feat(MISFIT_ROTATE) || !sched_asym_cpucap_active()) + return; + + if (READ_ONCE(rq->misfit_task) != p || !READ_ONCE(rq->misfit_task_load) || + READ_ONCE(rq->nr_running) != 1 || !misfit_rotate_candidate(p)) + return; + + if (time_before(jiffies, READ_ONCE(rq->misfit_since) + misfit_rotate_period())) + return; + + /* Whatever the outcome, try again no sooner than one period from now. */ + WRITE_ONCE(rq->misfit_since, jiffies); + + if (READ_ONCE(rq->misfit_rotate_p)) + return; + + get_task_struct(p); + if (cmpxchg(&rq->misfit_rotate_p, NULL, p)) { + put_task_struct(p); + return; + } + queue_work_on(cpu_of(rq), system_highpri_wq, &rq->misfit_rotate_work); +} + /* * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. */ @@ -15037,6 +15167,8 @@ void sched_balance_trigger(struct rq *rq) if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq)))) return; + misfit_rotate_trigger(rq); + if (time_after_eq(jiffies, rq->next_balance)) raise_softirq(SCHED_SOFTIRQ); @@ -15942,6 +16074,7 @@ __init void init_sched_fair_class(void) zalloc_cpumask_var_node(&per_cpu(select_rq_mask, i), GFP_KERNEL, cpu_to_node(i)); zalloc_cpumask_var_node(&per_cpu(should_we_balance_tmpmask, i), GFP_KERNEL, cpu_to_node(i)); + INIT_WORK(&cpu_rq(i)->misfit_rotate_work, misfit_rotate_work); #ifdef CONFIG_CFS_BANDWIDTH INIT_CSD(&cpu_rq(i)->cfsb_csd, __cfsb_csd_unthrottle, cpu_rq(i)); diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 8f0dee8fc4755..07132f33d1e4a 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -135,6 +135,13 @@ SCHED_FEAT(WA_BIAS, true) */ SCHED_FEAT(UTIL_EST, true) +/* + * On asymmetric CPU capacity systems, swap a task that has been stuck as a + * misfit on a smaller CPU with a task on a busy bigger CPU, so equal tasks + * get an equal share of the big CPUs. + */ +SCHED_FEAT(MISFIT_ROTATE, true) + SCHED_FEAT(LATENCY_WARN, false) /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 4c25fbe84fb54..24a25e3dc8c27 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1285,6 +1285,16 @@ struct rq { unsigned char idle_balance; unsigned long misfit_task_load; + /* + * Misfit rotation: the misfit task (compared, never dereferenced), + * the jiffies when it became misfit here, and the task handed (with + * a reference) to the work that swaps it with the task on a busy + * higher-capacity CPU. + */ + struct task_struct *misfit_task; + unsigned long misfit_since; + struct task_struct *misfit_rotate_p; + struct work_struct misfit_rotate_work; /* For active balancing */ int active_balance; @@ -2117,6 +2127,17 @@ static inline int sched_numa_find_closest(const struct cpumask *cpus, int cpu) #endif /* !CONFIG_NUMA */ +#ifdef CONFIG_SMP +extern int migrate_swap(struct task_struct *p, struct task_struct *t, + int cpu, int scpu); +#else +static inline int migrate_swap(struct task_struct *p, struct task_struct *t, + int cpu, int scpu) +{ + return -EINVAL; +} +#endif + #ifdef CONFIG_NUMA_BALANCING /* The regions in numa_faults array from task_struct */ @@ -2129,8 +2150,6 @@ enum numa_faults_stats { extern void sched_setnuma(struct task_struct *p, int node); extern int migrate_task_to(struct task_struct *p, int cpu); -extern int migrate_swap(struct task_struct *p, struct task_struct *t, - int cpu, int scpu); extern void init_numa_balancing(u64 clone_flags, struct task_struct *p); #else /* !CONFIG_NUMA_BALANCING: */ From aa00d6703a8a27df8311b9e3202c3c3c997e954d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 05:17:49 +0000 Subject: [PATCH 1490/1579] HID: dockchannel: freeze interface creation across system sleep The transport now notifies HID drivers across suspend and resume, with suspend rollback and all-interface resume error handling. Keep those callbacks and make the interface creation workqueue freezable so no HID device is added, or dropped after failed registration, while they walk the interfaces. This carries the remaining transport change from the click-strength resume series; the newer sleep notifications already provide its hid_driver_resume() calls. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 6d97f51971b29..da0f2ab1403d9 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -1698,7 +1698,11 @@ static int dockchannel_hid_probe(struct platform_device *pdev) if (!dchid->ring) return -ENOMEM; } - dchid->new_iface_wq = alloc_ordered_workqueue("dchid-new", 0); + /* + * Freeze interface creation across system sleep so the HID devices + * stay stable while the suspend and resume callbacks walk them. + */ + dchid->new_iface_wq = alloc_ordered_workqueue("dchid-new", WQ_FREEZABLE); if (!dchid->new_iface_wq) return -ENOMEM; From bd1fabcf757027dc53e154fc881d56d245c65bdf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 05:17:49 +0000 Subject: [PATCH 1491/1579] HID: apple-mtp-haptic: check the click strength again on resume The strength from apple,click-strength is written into the actuator firmware at probe only. Check it again on resume and reset_resume, and write it back if the firmware no longer has it, the same way as at probe. On the MTP machines the coprocessor keeps running across suspend-to-idle and nothing is known to clear the waveforms, so this is expected to find the strength in place and write nothing. That has not been confirmed on hardware, and the check costs two feature report reads. A debug message now says when a strength is written, so a resume that had to restore it shows up with dynamic debug. The MTP firmware is not restarted while Linux runs. rtkit-helper has no crash recovery, it and dockchannel-hid both refuse sysfs unbind, and the coprocessor announces its interfaces once per boot. The actuator can only come back as a new HID device, and probe already sets the strength there. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/hid-apple-mtp-haptic.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/hid/hid-apple-mtp-haptic.c b/drivers/hid/hid-apple-mtp-haptic.c index dd35d72ffe023..b8e3ee2f605f2 100644 --- a/drivers/hid/hid-apple-mtp-haptic.c +++ b/drivers/hid/hid-apple-mtp-haptic.c @@ -68,6 +68,9 @@ EXPORT_SYMBOL_GPL(apple_taptic_switch_modes); * fainter than the same pad gives under macOS. A machine whose device tree * gives the actuator a click strength has it written over the firmware's, * leaving the shape of each waveform as the firmware has it. + * + * The strength is firmware state, so it is checked again on every resume + * the transport passes on, and written if the firmware no longer has it. */ #define APPLE_TAPTIC_PRESS_WAVEFORM 0x22 #define APPLE_TAPTIC_RELEASE_WAVEFORM 0x23 @@ -93,6 +96,8 @@ static int apple_taptic_set_strength(struct hid_device *haptic_hdev, u8 report, if (buf[APPLE_TAPTIC_WAVEFORM_STRENGTH] == strength) return 0; + hid_dbg(haptic_hdev, "click waveform 0x%02x strength 0x%02x -> 0x%02x\n", + report, buf[APPLE_TAPTIC_WAVEFORM_STRENGTH], strength); buf[APPLE_TAPTIC_WAVEFORM_STRENGTH] = strength; len = hid_hw_raw_request(haptic_hdev, report, buf, len, HID_FEATURE_REPORT, HID_REQ_SET_REPORT); @@ -176,6 +181,12 @@ static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid return 0; } +static int apple_actuator_resume(struct hid_device *haptic_hdev) +{ + apple_taptic_set_click_strength(haptic_hdev); + return 0; +} + static bool actuator_match(struct hid_device *hdev, bool ignore_special_drivers) { return apple_taptic_is_actuator(hdev); @@ -201,7 +212,9 @@ static struct hid_driver apple_actuator_driver = { .id_table = apple_haptic_devices, .probe = apple_actuator_probe, .match = actuator_match, - .remove = actuator_remove + .remove = actuator_remove, + .resume = pm_ptr(apple_actuator_resume), + .reset_resume = pm_ptr(apple_actuator_resume), }; module_hid_driver(apple_actuator_driver); From fb7e51e66d320e6c0e1403e91fdc08dbfc8b7b4a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 23:41:33 +1100 Subject: [PATCH 1492/1579] accel/ane: t6021: open the T8112 power gate around ane_sys_cpu The base M2 (T8112) has a power gate word at +0x8b8 of the ANE "set" window. It reads 0x30000000 (bits 29:28 set, closed) at boot, and macOS clears it and waits for bits 29:28 to read 0 after ANE_SYS is on and before it writes the ANE_SYS_CPU power state. On power off it writes 0x30000000 back after the states are off. The driver only read the word after power up and refused when it was closed, which is what a first run on a T8112 reported: "PWGATE closed (word 0x30000000, ane_cpu 0x1f0003ff)", with no DART or ANE fault. Do what macOS does, in its order. power-domains[0] of the T8112 node is ane_sys_cpu, whose parent is ane_sys, so a genpd notifier on that domain runs after ane_sys is on and before ane_sys_cpu is written (PRE_ON), and after ane_sys_cpu is off and before ane_sys goes off (OFF). The notifier opens the gate on PRE_ON and closes it on OFF. It writes only the two known values: any other word fails the power up without a write, and a gate that does not open within 100 ms is closed again and fails it. The notifier is registered before the device link that resumes the domain. The SoCs without pwgate_off in their match data (T6020, T6021, T6022) are unchanged. The check after power up stays, for a boot that left ane_sys_cpu on with the gate closed. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane_t6021.h | 5 +- drivers/accel/ane/ane_t6021_rtclient_main.c | 121 +++++++++++++++++++- 2 files changed, 121 insertions(+), 5 deletions(-) diff --git a/drivers/accel/ane/ane_t6021.h b/drivers/accel/ane/ane_t6021.h index ce134209c5e06..bf6d0f3e44202 100644 --- a/drivers/accel/ane/ane_t6021.h +++ b/drivers/accel/ane/ane_t6021.h @@ -754,8 +754,9 @@ struct ane_asc_tunables; * for this SoC (T6021 only), so fw_alias_reserved=1 may map it; the * other SoCs always run the firmware from driver-owned memory. * ps_cpu_off: the ANE CPU ps word in the DT "pmgr" window (probe guard). - * pwgate_off: the PWGATE word in the DT "set" window, checked - * open before any engine read; 0 = not checked. + * pwgate_off: the power gate word in the DT "set" window. The driver + * opens it before ane_sys_cpu powers on, closes it after that state + * powers off, and checks it open before any engine read; 0 = no gate. * pmu_pa: the page of the seven ANE ps words; the firmware's power * service writes them through its DART at IOVA == PA. ps_off: the * first of the seven in that page. diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 5f1e5e66bd22d..77695f89c9808 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -65,6 +65,7 @@ #include #include #include +#include #include #include #include @@ -2063,6 +2064,13 @@ struct ane_rtclient_pd { struct device **pd_dev; struct device_link **pd_link; int count; + + /* T8112 power gate, kept in step with power-domains[0]. */ + struct notifier_block pwgate_nb; + void __iomem *pwgate_base; + void __iomem *pwgate; + bool pwgate_nb_added; + bool pwgate_opened; }; static LIST_HEAD(ane_rtclient_pd_list); @@ -2074,11 +2082,103 @@ static DEFINE_MUTEX(ane_rtclient_probe_guard_lock); static void ane_rtclient_pd_free(struct ane_rtclient_pd *pd) { list_del(&pd->list); + if (pd->pwgate_base) + iounmap(pd->pwgate_base); kfree(pd->pd_dev); kfree(pd->pd_link); kfree(pd); } +/* + * T8112 power gate. A word in the "set" window gates the ANE: bits 29:28 + * set (the word reads 0x30000000) is closed, both clear is open. It must be + * open before the ANE CPU power state (ane_sys_cpu) turns on and is closed + * again after that state turns off, with ane_sys, the parent, on both + * times. power-domains[0] is ane_sys_cpu, and genpd calls this notifier + * after ane_sys is on and before ane_sys_cpu is written (PRE_ON), and + * after ane_sys_cpu is off and before ane_sys goes off (OFF), which is that + * order. Only the two known values are written. Any other word fails the + * power-up without a write. + */ +#define ANE_PWGATE_CLOSED GENMASK(29, 28) +#define ANE_PWGATE_POLL_US 10 +#define ANE_PWGATE_TIMEOUT_US 100000 + +static int ane_rtclient_pwgate_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct ane_rtclient_pd *pd = + container_of(nb, struct ane_rtclient_pd, pwgate_nb); + u32 gate; + int ret; + + switch (action) { + case GENPD_NOTIFY_PRE_ON: + gate = readl(pd->pwgate); + if (!(gate & ANE_PWGATE_CLOSED)) + return NOTIFY_OK; + if (gate != ANE_PWGATE_CLOSED) { + dev_err(pd->dev, "power gate word %#x is not a known state\n", + gate); + return notifier_from_errno(-EIO); + } + writel(0, pd->pwgate); + ret = readl_poll_timeout_atomic(pd->pwgate, gate, + !(gate & ANE_PWGATE_CLOSED), + ANE_PWGATE_POLL_US, + ANE_PWGATE_TIMEOUT_US); + if (ret) { + writel(ANE_PWGATE_CLOSED, pd->pwgate); + dev_err(pd->dev, "power gate did not open (word %#x)\n", + gate); + return notifier_from_errno(ret); + } + pd->pwgate_opened = true; + return NOTIFY_OK; + case GENPD_NOTIFY_OFF: + if (!pd->pwgate_opened) + return NOTIFY_OK; + pd->pwgate_opened = false; + writel(ANE_PWGATE_CLOSED, pd->pwgate); + ret = readl_poll_timeout_atomic(pd->pwgate, gate, + (gate & ANE_PWGATE_CLOSED) == + ANE_PWGATE_CLOSED, + ANE_PWGATE_POLL_US, + ANE_PWGATE_TIMEOUT_US); + if (ret) + dev_warn(pd->dev, "power gate did not close (word %#x)\n", + gate); + return NOTIFY_OK; + default: + return NOTIFY_DONE; + } +} + +/* + * Register the power gate notifier on power-domains[0] before its device + * link is created: the link resumes the domain at once. + */ +static int ane_rtclient_pwgate_add(struct ane_rtclient *ane, + struct ane_rtclient_pd *pd) +{ + int ret; + + if (!ane->soc->pwgate_off || pd->pwgate_nb_added) + return 0; + if (!pd->pwgate_base) { + pd->pwgate_base = of_iomap(ane->dev->of_node, 2); + if (!pd->pwgate_base) + return -ENOMEM; + pd->pwgate = pd->pwgate_base + ane->soc->pwgate_off; + } + pd->pwgate_nb.notifier_call = ane_rtclient_pwgate_notify; + ret = dev_pm_genpd_add_notifier(pd->pd_dev[0], &pd->pwgate_nb); + if (ret) + return ret; + pd->pwgate_nb_added = true; + return 0; +} + static int ane_rtclient_attach_genpd(struct ane_rtclient *ane) { struct device *dev = ane->dev; @@ -2127,6 +2227,11 @@ static int ane_rtclient_attach_genpd(struct ane_rtclient *ane) goto out; } } + if (i == 0) { + err = ane_rtclient_pwgate_add(ane, pd); + if (err) + goto out; + } if (!pd->pd_link[i]) { if (!ane_rtclient_pinned) { if (!try_module_get(THIS_MODULE)) { @@ -2171,6 +2276,15 @@ static void ane_rtclient_detach_genpd(struct device *dev) for (i = count - 1; i >= 0; i--) { if (pd->pd_link[i]) device_link_del(pd->pd_link[i]); + /* + * genpd keeps a notifier past detach, so drop it first. + * The domain may then go off with the gate open, the + * state between the two power-up steps. + */ + if (i == 0 && pd->pwgate_nb_added) { + dev_pm_genpd_remove_notifier(pd->pd_dev[0]); + pd->pwgate_nb_added = false; + } if (pd->pd_dev[i]) dev_pm_domain_detach(pd->pd_dev[i], true); } @@ -2282,9 +2396,10 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, pm_runtime_disable(dev); return -EPROBE_DEFER; } - /* T8112: macOS (type 0x70) opens PWGATE (bits 29:28 = 0) before - * the ps words. This driver only reads it: an engine read behind a - * closed gate is untested, so it refuses first. + /* T8112: the power gate notifier opened the gate before ane_sys_cpu + * turned on. A gate that still reads closed means the state was + * already on without it (no power-up ran), so refuse before any + * engine read. */ if (ane->soc->pwgate_off) { void __iomem *set = devm_of_iomap(dev, dev->of_node, 2, NULL); From fefab16fab0899cdd2baaefcd524ab9069aa4dbe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 23:41:33 +1100 Subject: [PATCH 1493/1579] accel/ane: t6021: say once at info level that the firmware is missing The M2 family ANE runs an Apple firmware image that is fetched on the machine itself and never shipped with the kernel. A system that has not fetched it yet is in a normal state, but the probe logged a firmware loader warning and an error on every boot. With the T8112 node enabled that would put two error lines in every base M2 boot log. Request the image without the loader warning and, when the file does not exist, log one info line that names the file and the tool that installs it, and fail the probe with -ENODEV. This happens before any hardware access, as before. Other load errors still log an error. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane_t6021_fwload.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index 4dc4ce27b1674..f39e4b7e8b05c 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -533,11 +533,20 @@ int ane_t6021_fwload_check(struct device *dev) if (!fw_load) return 0; - ret = request_firmware(&fw, img->name, dev); + /* + * The image is installed on the machine itself from Apple's files, so + * a missing file is the normal state of a system that has not fetched + * it yet: say so once and leave the ANE off. Other errors are errors. + */ + ret = firmware_request_nowarn(&fw, img->name, dev); + if (ret == -ENOENT) { + dev_info(dev, + "firmware %s is not installed, so the ANE stays off; install it with omarchy-ane-firmware-fetch\n", + img->name); + return -ENODEV; + } if (ret) { - dev_err(dev, - "cannot load firmware %s: %d; install it with omarchy-ane-firmware-fetch (or equivalent for your distribution)\n", - img->name, ret); + dev_err(dev, "cannot load firmware %s: %d\n", img->name, ret); return ret; } From 0f6547f642377d7b6828fa94382ce71de2272f92 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 30 Oct 2026 23:41:33 +1100 Subject: [PATCH 1494/1579] arm64: dts: apple: t8112: enable the Neural Engine Enable the ANE, its mailbox, its three DARTs and the seven ANE power states on every T8112 board. The driver now opens the ANE power gate in the order macOS uses, which was the step that stopped the first run on a T8112. Without the firmware image the probe stops before any hardware access and logs one info line, so a machine that has not fetched it is not affected. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8112-pmgr.dtsi | 12 +----------- arch/arm64/boot/dts/apple/t8112.dtsi | 5 ----- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index 26ab30e219f12..9ff4ed911d482 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -1132,10 +1132,7 @@ power-domains = <&ps_venc_pipe5>, <&ps_venc_pipe4>; }; - /* - * These states serve only the ANE. They stay disabled with the ane - * node in t8112.dtsi; enable them together with it. - */ + /* These states serve only the ANE. */ ps_ane_sys_cpu: power-controller@c008 { compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc008 4>; @@ -1143,7 +1140,6 @@ #reset-cells = <0>; label = "ane_sys_cpu"; power-domains = <&ps_ane_sys>; - status = "disabled"; }; ps_ane_td: power-controller@c010 { @@ -1153,7 +1149,6 @@ #reset-cells = <0>; label = "ane_td"; power-domains = <&ps_ane_sys_cpu>; - status = "disabled"; }; ps_ane_base: power-controller@c018 { @@ -1163,7 +1158,6 @@ #reset-cells = <0>; label = "ane_base"; power-domains = <&ps_ane_td>; - status = "disabled"; }; ps_ane_set1: power-controller@c020 { @@ -1173,7 +1167,6 @@ #reset-cells = <0>; label = "ane_set1"; power-domains = <&ps_ane_base>; - status = "disabled"; }; ps_ane_set2: power-controller@c028 { @@ -1183,7 +1176,6 @@ #reset-cells = <0>; label = "ane_set2"; power-domains = <&ps_ane_set1>; - status = "disabled"; }; ps_ane_set3: power-controller@c030 { @@ -1193,7 +1185,6 @@ #reset-cells = <0>; label = "ane_set3"; power-domains = <&ps_ane_set2>; - status = "disabled"; }; ps_ane_set4: power-controller@c038 { @@ -1203,7 +1194,6 @@ #reset-cells = <0>; label = "ane_set4"; power-domains = <&ps_ane_set3>; - status = "disabled"; }; ps_disp0_cpu0: power-controller@10000 { diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index e19ef0b98292a..23369ae0246a2 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -1788,7 +1788,6 @@ interrupts = ; interrupt-names = "recv-not-empty"; #mbox-cells = <0>; - status = "disabled"; }; ane_dart0: iommu@26b800000 { @@ -1798,7 +1797,6 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - status = "disabled"; }; ane_dart1: iommu@26b810000 { @@ -1808,7 +1806,6 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - status = "disabled"; }; ane_dart2: iommu@26b820000 { @@ -1818,7 +1815,6 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - status = "disabled"; }; ane: ane@26a000000 { @@ -1836,7 +1832,6 @@ <&ps_ane_base>, <&ps_ane_set1>, <&ps_ane_set2>, <&ps_ane_set3>, <&ps_ane_set4>; mboxes = <&ane_mbox>; - status = "disabled"; }; dispext0_dart: iommu@271304000 { From 3c5441114af9f9e6f93fadf94710269b27676b54 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:17:49 +1100 Subject: [PATCH 1495/1579] media: apple: isp: report the frame gain and exposure Userspace has no way to tell how hard the sensor is working. Gain is the signal a camera pipeline needs to decide when the picture is noisy enough to denoise, and the firmware already writes it into every frame's metadata buffer, which the driver maps but never reads. The metadata buffer opens with a table of section offsets. On H17 firmware the fourth section starts with the frame's exposure: the integration time in microseconds, then the total sensor gain in 1/256 steps with 256 as unity. This was established on T6040 and T8140 by watching auto exposure converge on a fixed scene, where the gain climbed from 260 to its final value once the integration time hit the frame period, and by stepping the frame rate, which capped the integration time at the frame period. Read the section when a metadata buffer comes back, through the table and bounded by the buffer size, and report the last values as read-only, volatile V4L2_CID_GAIN and V4L2_CID_EXPOSURE_ABSOLUTE controls. Only hardware whose match data says its metadata carries the report gets the controls. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 2 + drivers/media/platform/apple/isp/isp-drv.h | 9 ++ .../media/platform/apple/isp/isp-v4l2-test.c | 46 ++++++++++ drivers/media/platform/apple/isp/isp-v4l2.c | 92 ++++++++++++++++++- 4 files changed, 148 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 084cca4dc3707..15612805bf7fb 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -827,6 +827,7 @@ static const struct apple_isp_hw apple_isp_hw_t8140 = { .boot_mode = 1, .capture_meta_size = ISP_CAPTURE_META_SIZE_T8140, .resident_fw = true, + .meta_exposure = true, }; /* The same for T6040, from the J616s ADT's DART address filter. */ @@ -885,6 +886,7 @@ static const struct apple_isp_hw apple_isp_hw_t6040 = { .boot_mode = 1, .shared_size = 0xde600000, .resident_fw = true, + .meta_exposure = true, }; static const struct of_device_id apple_isp_of_match[] = { diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index dbe6b6ed5a2c4..0515570993ac6 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -201,6 +201,9 @@ struct apple_isp_hw { /* size of a capture metadata buffer, 0 for no such pool */ u32 capture_meta_size; + /* metadata buffers carry the frame's exposure report (H17 firmware) */ + bool meta_exposure; + /* * The firmware can be started only once per system boot, so it is * started at probe and kept running while the driver is bound. @@ -302,6 +305,12 @@ struct apple_isp { struct v4l2_device v4l2_dev; struct v4l2_ctrl_handler ctrl_handler; struct v4l2_ctrl *exposure_priority; + /* read-only exposure report of the last frame, NULL without one */ + struct v4l2_ctrl *gain; + struct v4l2_ctrl *exposure; + /* last reported total gain (1/256 steps) and exposure (us) */ + u32 frame_gain; + u32 frame_exposure_us; struct vb2_queue vbq; struct mutex video_lock; unsigned int sequence; diff --git a/drivers/media/platform/apple/isp/isp-v4l2-test.c b/drivers/media/platform/apple/isp/isp-v4l2-test.c index 398e8e92356ab..88f428b3b0c27 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2-test.c +++ b/drivers/media/platform/apple/isp/isp-v4l2-test.c @@ -348,7 +348,53 @@ static void isp_test_imx958_last_lease_requeue(struct kunit *test) } } +/* The exposure report is found through the section table, never past it. */ +static void isp_test_meta_exposure(struct kunit *test) +{ + static const struct apple_isp_hw hw = { .meta_exposure = true }; + static const struct apple_isp_hw hw_none = { }; + struct apple_isp *isp = isp_test_alloc(test); + struct isp_surf meta = { .size = 0x2000 }; + u8 *virt = kunit_kzalloc(test, meta.size, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, virt); + meta.virt = virt; + isp->hw = &hw; + isp->frame_gain = 256; + + /* fourth table entry: the section at 0x1870 */ + put_unaligned_le32(0x1870, virt + 0x1c); + put_unaligned_le32(32475, virt + 0x1870); + put_unaligned_le16(0x0800, virt + 0x1874); + put_unaligned_le16(3253, virt + 0x1876); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_exposure_us, 32475); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + + /* a section that would end past the buffer is ignored */ + put_unaligned_le32(meta.size - 4, virt + 0x1c); + put_unaligned_le32(1000, virt + meta.size - 4); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + + /* so is an unwritten section */ + put_unaligned_le32(0x1000, virt + 0x1c); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_exposure_us, 32475); + + /* and firmware without the report */ + put_unaligned_le32(0x1870, virt + 0x1c); + put_unaligned_le16(600, virt + 0x1876); + isp->hw = &hw_none; + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + isp->hw = &hw; + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 600); +} + static struct kunit_case isp_v4l2_test_cases[] = { + KUNIT_CASE(isp_test_meta_exposure), KUNIT_CASE(isp_test_imx958_last_lease_requeue), KUNIT_CASE(isp_test_p010_report_defers_completion), KUNIT_CASE(isp_test_p010_mode_selection), diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 431c85c6695c9..2e858d392292d 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -68,6 +69,45 @@ static bool isp_p010_buffer_matches(const struct isp_format *fmt, !desc->iovas[2] && !desc->iovas[3] && desc->tag == buf->tag; } +/* + * A metadata buffer starts with a table of section offsets. On H17 firmware + * the fourth section opens with the frame's exposure: the integration time in + * microseconds (le32 at 0) and the total sensor gain in 1/256 steps, 256 for + * unity (le16 at 6). Observed on T6040 and T8140 with AE converging under a + * fixed scene and with the frame rate capping the integration time. + */ +#define ISP_META_SECTIONS 0x10 +#define ISP_META_SECTION_EXPOSURE 3 +#define ISP_META_EXPOSURE_TIME 0x0 +#define ISP_META_EXPOSURE_GAIN 0x6 +#define ISP_META_EXPOSURE_SIZE 0x8 + +static void isp_meta_read_exposure(struct apple_isp *isp, + const struct isp_surf *meta) +{ + const u8 *virt = meta->virt; + u32 offset, time; + u16 gain; + + if (!isp->hw->meta_exposure || !virt) + return; + + offset = get_unaligned_le32(virt + ISP_META_SECTIONS + + 4 * ISP_META_SECTION_EXPOSURE); + if (offset < ISP_META_SECTIONS || offset > meta->size || + meta->size - offset < ISP_META_EXPOSURE_SIZE) + return; + + time = get_unaligned_le32(virt + offset + ISP_META_EXPOSURE_TIME); + gain = get_unaligned_le16(virt + offset + ISP_META_EXPOSURE_GAIN); + /* An unwritten section or one this layout does not describe */ + if (!time || !gain) + return; + + WRITE_ONCE(isp->frame_exposure_us, time); + WRITE_ONCE(isp->frame_gain, gain); +} + void ipc_bt_complete(struct apple_isp *isp) { struct isp_buffer *buf, *tmp; @@ -128,6 +168,7 @@ int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) struct isp_surf *meta = isp->meta_surfs[j]; if ((u32)bufd->iovas[0] == (u32)meta->iova) { WARN_ON(!meta->submitted); + isp_meta_read_exposure(isp, meta); if (isp_is_p010(isp)) __set_bit(j, &isp->meta_returned); else @@ -1062,6 +1103,53 @@ static int isp_vb2_queue_init(struct apple_isp *isp) return vb2_queue_init(vbq); } +static int isp_g_volatile_ctrl(struct v4l2_ctrl *ctrl) +{ + struct apple_isp *isp = + container_of(ctrl->handler, struct apple_isp, ctrl_handler); + + switch (ctrl->id) { + case V4L2_CID_GAIN: + ctrl->val = READ_ONCE(isp->frame_gain); + return 0; + case V4L2_CID_EXPOSURE_ABSOLUTE: + /* in 100 us units */ + ctrl->val = DIV_ROUND_CLOSEST(READ_ONCE(isp->frame_exposure_us), + 100); + return 0; + } + + return -EINVAL; +} + +static const struct v4l2_ctrl_ops isp_exposure_ctrl_ops = { + .g_volatile_ctrl = isp_g_volatile_ctrl, +}; + +/* + * Auto exposure picks the gain and exposure of each frame. Report the last + * frame's values read-only, so that userspace can follow the scene light, for + * example to decide when to denoise. The gain is the firmware's total sensor + * gain in 1/256 steps. + */ +static void isp_init_exposure_ctrls(struct apple_isp *isp) +{ + isp->frame_gain = 256; + isp->gain = v4l2_ctrl_new_std(&isp->ctrl_handler, + &isp_exposure_ctrl_ops, V4L2_CID_GAIN, + 0, U16_MAX, 1, 256); + isp->exposure = v4l2_ctrl_new_std(&isp->ctrl_handler, + &isp_exposure_ctrl_ops, + V4L2_CID_EXPOSURE_ABSOLUTE, + 0, 10000, 1, 0); + if (isp->gain) + isp->gain->flags |= V4L2_CTRL_FLAG_READ_ONLY | + V4L2_CTRL_FLAG_VOLATILE; + if (isp->exposure) + isp->exposure->flags |= V4L2_CTRL_FLAG_READ_ONLY | + V4L2_CTRL_FLAG_VOLATILE; +} + int apple_isp_setup_video(struct apple_isp *isp) { struct video_device *vdev = &isp->vdev; @@ -1116,10 +1204,12 @@ int apple_isp_setup_video(struct apple_isp *isp) * Auto exposure is always on. By default it holds the frame rate, as * V4L2 specifies; with exposure priority it may slow down in low light. */ - v4l2_ctrl_handler_init(&isp->ctrl_handler, 1); + v4l2_ctrl_handler_init(&isp->ctrl_handler, 3); isp->exposure_priority = v4l2_ctrl_new_std(&isp->ctrl_handler, NULL, V4L2_CID_EXPOSURE_AUTO_PRIORITY, 0, 1, 1, 0); + if (isp->hw->meta_exposure) + isp_init_exposure_ctrls(isp); if (isp->ctrl_handler.error) { err = isp->ctrl_handler.error; dev_err(isp->dev, "failed to create controls: %d\n", err); From aaab795be06fc5bb03d974aca636dc5012e72fe7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:22:51 +1100 Subject: [PATCH 1496/1579] media: apple-isp: derive call sizes from processed crop extents Add common call resolutions when an established processed crop contains an exact-aspect, chroma-aligned crop at least as large as the output. Keep the sensor configuration, pixel-format eligibility and original preset order, and avoid duplicate sizes. Also offer a sensor-sized square and smaller 4:3 outputs when larger call sizes do not fit. Do not use sensor input heights as output capabilities: they can include stacked exposures. Leave sizes requiring enlargement or unestablished crop extents unavailable. Add KUnit tests for geometry and size tables, and document enumeration and capture qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../media/drivers/apple-isp-j616s.rst | 51 ++++++++ .../media/platform/apple/isp/isp-cam-test.c | 112 ++++++++++++++++++ drivers/media/platform/apple/isp/isp-cam.c | 104 ++++++++++++++++ 3 files changed, 267 insertions(+) diff --git a/Documentation/driver-api/media/drivers/apple-isp-j616s.rst b/Documentation/driver-api/media/drivers/apple-isp-j616s.rst index 329b811dce70e..26529b211b79c 100644 --- a/Documentation/driver-api/media/drivers/apple-isp-j616s.rst +++ b/Documentation/driver-api/media/drivers/apple-isp-j616s.rst @@ -361,3 +361,54 @@ The T6040 device tree's current fixed TEXT/DATA reservations should eventually come from a bootloader handoff describing the loaded image and mapping lifetimes. They are firmware-build-specific. This change does not implement that handoff or replace those reservations with 25G76 addresses. + +Capture size enumeration +------------------------ + +The shared driver preserves the order of its device-tree presets and any +validated firmware output modes. It then adds common video-call sizes that +fit within those presets' processed crop extents. The added preset retains +its source firmware configuration and pixel-format eligibility. Crops are +centred on even coordinates, have the exact output aspect ratio and never +require upscaling. Sensor input dimensions alone are not a processed output +capability: input rows can contain stacked exposures. + +The common sizes include 640x480, 1280x720, 1920x1080, 1080x1920, +1760x1328, 1328x1760, 1552x1552, 1280x960 and 960x1280. A sensor that +cannot provide the larger square also gets the largest even square that +fits its established crops. Existing sizes are not duplicated. This is a +shared output policy, not a list indexed by board or sensor model. + +Neither the crop nor output-configuration command reports scaler limits +or rotation support. The driver does not infer a larger processed image +from raw input dimensions, rotate a portrait image into a landscape one, +or advertise a size that requires enlargement. Consequently, an exact +macOS size may remain unavailable even though a smaller size with the +same aspect ratio can be offered. + +Read-only inspection on the running kernel:: + + v4l2-ctl -d /dev/video0 --list-formats-ext + v4l2-ctl -d /dev/video0 --log-status + dmesg | grep -A20 'sensor config' + +The status ioctl prints already cached firmware configuration replies; +it does not start capture or send additional firmware queries. The cache +is currently populated only on the existing supported P010 discovery path. +A missing cache is not evidence that the sensor has no other modes. + +After booting a test kernel, enumerate again and qualify each added size +that is actually offered. For example, for 640x480 NV12:: + + v4l2-ctl -d /dev/video0 \ + --set-fmt-video=width=640,height=480,pixelformat=NV12 \ + --get-fmt-video --stream-mmap=4 --stream-count=30 --stream-to=/dev/null + gst-launch-1.0 -v v4l2src device=/dev/video0 num-buffers=30 ! \ + video/x-raw,format=NV12,width=640,height=480,framerate=30/1 ! \ + fakesink sync=false + +Repeat with each added width and height and, when offered, P010. Check +that the negotiated dimensions equal the requested dimensions: S_FMT may +otherwise choose a nearby preset. Confirm 30 completed frames, correct +aspect ratio and chroma, and no new ISP or DART errors. Enumeration and unit tests do not establish successful capture at a +new size; capture qualification must be performed separately. diff --git a/drivers/media/platform/apple/isp/isp-cam-test.c b/drivers/media/platform/apple/isp/isp-cam-test.c index 162e2ff1f0258..0b2f63685c7b4 100644 --- a/drivers/media/platform/apple/isp/isp-cam-test.c +++ b/drivers/media/platform/apple/isp/isp-cam-test.c @@ -370,7 +370,119 @@ static void isp_test_command_ack_failure_latches(struct kunit *test) } } +static void isp_test_call_crop(struct kunit *test) +{ + struct isp_preset source = { + .index = 7, .input_dim = { 1936, 1096 }, + .output_dim = { 1920, 1080 }, .crop_offset = { 8, 8 }, + .crop_size = { 1920, 1080 }, .p010 = true, + }; + struct isp_preset result; + + KUNIT_ASSERT_TRUE(test, + isp_size_from_crop(&source, (struct coord){ 640, 480 }, &result)); + KUNIT_EXPECT_EQ(test, result.index, 7); + KUNIT_EXPECT_TRUE(test, result.p010); + KUNIT_EXPECT_EQ(test, result.crop_offset.x, 248); + KUNIT_EXPECT_EQ(test, result.crop_offset.y, 8); + KUNIT_EXPECT_EQ(test, result.crop_size.x, 1440); + KUNIT_EXPECT_EQ(test, result.crop_size.y, 1080); + KUNIT_EXPECT_EQ(test, result.output_dim.x, 640); + KUNIT_EXPECT_EQ(test, result.output_dim.y, 480); +} + +static void isp_test_call_crop_bounds(struct kunit *test) +{ + struct isp_preset source = { + .input_dim = { 1920, 2160 }, .output_dim = { 1920, 1080 }, + .crop_size = { 1920, 1080 }, + }; + struct isp_preset result; + static const struct coord rejected[] = { + { 1080, 1920 }, { 1760, 1328 }, { 1552, 1552 }, + { 1922, 1080 }, { 0, 480 }, { 641, 480 }, + }; + + /* Stacked input rows do not make a portrait or a larger square. */ + for (unsigned int i = 0; i < ARRAY_SIZE(rejected); i++) + KUNIT_EXPECT_FALSE(test, isp_size_from_crop(&source, rejected[i], &result)); + source.crop_offset.x = 2; + KUNIT_EXPECT_FALSE(test, + isp_size_from_crop(&source, (struct coord){ 640, 480 }, &result)); + source.crop_offset.x = U32_MAX; + KUNIT_EXPECT_FALSE(test, + isp_size_from_crop(&source, (struct coord){ 640, 480 }, &result)); +} + +static void isp_test_call_size_table(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_preset source = { + .index = 7, .input_dim = { 1920, 1920 }, + .output_dim = { 1920, 1920 }, .crop_size = { 1920, 1920 }, + }; + + isp->presets = &source; + isp->num_presets = 1; + KUNIT_ASSERT_EQ(test, isp_add_call_sizes(isp), 0); + KUNIT_EXPECT_EQ(test, isp->num_presets, ARRAY_SIZE(isp_call_sizes) + 1); + KUNIT_EXPECT_EQ(test, isp->presets[0].output_dim.x, 1920); + KUNIT_EXPECT_EQ(test, isp->presets[0].output_dim.y, 1920); + for (unsigned int i = 0; i < ARRAY_SIZE(isp_call_sizes); i++) { + struct isp_preset *preset = &isp->presets[i + 1]; + + KUNIT_EXPECT_EQ(test, preset->output_dim.x, isp_call_sizes[i].x); + KUNIT_EXPECT_EQ(test, preset->output_dim.y, isp_call_sizes[i].y); + KUNIT_EXPECT_EQ(test, preset->index, 7); + KUNIT_EXPECT_FALSE(test, preset->p010); + KUNIT_EXPECT_EQ(test, preset->crop_size.x * preset->output_dim.y, + preset->crop_size.y * preset->output_dim.x); + KUNIT_EXPECT_EQ(test, (preset->crop_offset.x | preset->crop_offset.y | + preset->crop_size.x | preset->crop_size.y) & 1, 0); + KUNIT_EXPECT_LE(test, preset->crop_offset.x + preset->crop_size.x, 1920); + KUNIT_EXPECT_LE(test, preset->crop_offset.y + preset->crop_size.y, 1920); + } + /* Rebuilding preserves order and never duplicates an existing size. */ + KUNIT_ASSERT_EQ(test, isp_add_call_sizes(isp), 0); + KUNIT_EXPECT_EQ(test, isp->num_presets, ARRAY_SIZE(isp_call_sizes) + 1); +} + +static void isp_test_call_size_capabilities(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_preset sources[] = { + { .index = 5, .input_dim = { 1920, 2160 }, + .output_dim = { 1920, 1080 }, .crop_size = { 1920, 1080 }, .p010 = true }, + { .index = 5, .input_dim = { 1920, 2160 }, + .output_dim = { 1280, 720 }, .crop_size = { 1280, 720 }, .p010 = true }, + { .index = 9, .input_dim = { 1536, 3024 }, + .output_dim = { 1536, 3024 }, .crop_size = { 1536, 3024 }, .p010 = true }, + }; + + isp->presets = sources; + isp->num_presets = ARRAY_SIZE(sources); + KUNIT_ASSERT_EQ(test, isp_add_call_sizes(isp), 0); + KUNIT_EXPECT_EQ(test, isp->num_presets, 9); + for (unsigned int i = 0; i < ARRAY_SIZE(sources); i++) + KUNIT_EXPECT_MEMEQ(test, &isp->presets[i], &sources[i], sizeof(sources[i])); + for (unsigned int i = ARRAY_SIZE(sources); i < isp->num_presets; i++) { + struct isp_preset *preset = &isp->presets[i]; + + KUNIT_EXPECT_TRUE(test, preset->p010); + KUNIT_EXPECT_FALSE(test, preset->output_dim.x == 1760 && + preset->output_dim.y == 1328); + KUNIT_EXPECT_FALSE(test, preset->output_dim.x == 1552 && + preset->output_dim.y == 1552); + } + KUNIT_EXPECT_EQ(test, isp->presets[8].output_dim.x, 1536); + KUNIT_EXPECT_EQ(test, isp->presets[8].output_dim.y, 1536); +} + static struct kunit_case isp_cam_test_cases[] = { + KUNIT_CASE(isp_test_call_crop), + KUNIT_CASE(isp_test_call_crop_bounds), + KUNIT_CASE(isp_test_call_size_table), + KUNIT_CASE(isp_test_call_size_capabilities), KUNIT_CASE(isp_test_setfile_failure_stops_setup), KUNIT_CASE(isp_test_optional_setfile_continues_setup), KUNIT_CASE(isp_test_command_ack_failure_latches), diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 8483f511f6299..8ea316f1a917e 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -2,6 +2,7 @@ /* Copyright 2023 Eileen Yoon */ #include +#include #include #include @@ -332,6 +333,107 @@ static int isp_ch_cache_sensor_configs(struct apple_isp *isp, u32 ch, u32 count) return err; } +/* Common call sizes, offered only when a known processed crop contains them. */ +static const struct coord isp_call_sizes[] = { + { 640, 480 }, { 1280, 720 }, { 1920, 1080 }, { 1080, 1920 }, + { 1760, 1328 }, { 1328, 1760 }, { 1552, 1552 }, + { 1280, 960 }, { 960, 1280 }, +}; + +/* + * CH_CROP[_SCL1]_SET takes x/y/width/height independently of the output + * dimensions in CH_OUTPUT_CONFIG[_SCL1]_SET. Stay within an established + * processed crop, rather than the sensor input (which may contain stacked + * rows). Use the largest exact-aspect crop, centred on the chroma grid, + * with no upscaling. Neither command reports a scaler limit or rotation. + */ +static bool isp_size_from_crop(const struct isp_preset *source, + struct coord size, struct isp_preset *preset) +{ + u32 divisor, x, y, scale; + + if (!size.x || !size.y || (size.x | size.y) & 1 || + !source->crop_size.x || !source->crop_size.y || + (source->crop_offset.x | source->crop_offset.y) & 1 || + source->crop_size.x > 16384 || source->crop_size.y > 16384 || + source->crop_offset.x > source->input_dim.x || + source->crop_size.x > source->input_dim.x - source->crop_offset.x || + source->crop_offset.y > source->input_dim.y || + source->crop_size.y > source->input_dim.y - source->crop_offset.y) + return false; + + divisor = gcd(size.x, size.y); + x = size.x / divisor; + y = size.y / divisor; + scale = round_down(min(source->crop_size.x / x, + source->crop_size.y / y), 2); + if (scale < divisor) + return false; + + *preset = *source; + preset->output_dim = size; + preset->crop_size = (struct coord){ x * scale, y * scale }; + preset->crop_offset.x += round_down((source->crop_size.x - x * scale) / 2, 2); + preset->crop_offset.y += round_down((source->crop_size.y - y * scale) / 2, 2); + return true; +} + +static unsigned int isp_append_size(struct isp_preset *presets, unsigned int num, + unsigned int sources, struct coord size) +{ + struct isp_preset candidate, best; + u32 area, best_area = 0; + + for (unsigned int i = 0; i < num; i++) + if (presets[i].output_dim.x == size.x && + presets[i].output_dim.y == size.y) + return num; + + /* Never use a generated size as evidence for another generated size. */ + for (unsigned int i = 0; i < sources; i++) { + if (!isp_size_from_crop(&presets[i], size, &candidate)) + continue; + area = candidate.crop_size.x * candidate.crop_size.y; + if (area > best_area) { + best = candidate; + best_area = area; + } + } + if (best_area) + presets[num++] = best; + return num; +} + +static int isp_add_call_sizes(struct apple_isp *isp) +{ + unsigned int sources = isp->num_presets, num = sources; + struct isp_preset *presets; + struct isp_preset candidate; + u32 square = 0; + + presets = devm_kcalloc(isp->dev, sources + ARRAY_SIZE(isp_call_sizes) + 1, + sizeof(*presets), GFP_KERNEL); + if (!presets) + return -ENOMEM; + memcpy(presets, isp->presets, sources * sizeof(*presets)); + + for (unsigned int i = 0; i < ARRAY_SIZE(isp_call_sizes); i++) + num = isp_append_size(presets, num, sources, isp_call_sizes[i]); + + /* A smaller sensor can still offer a square without enlarging pixels. */ + for (unsigned int i = 0; i < sources; i++) { + u32 side = round_down(min(presets[i].crop_size.x, + presets[i].crop_size.y), 2); + + if (isp_size_from_crop(&presets[i], (struct coord){ side, side }, &candidate)) + square = max(square, side); + } + num = isp_append_size(presets, num, sources, (struct coord){ square, square }); + isp->presets = presets; + isp->num_presets = num; + return 0; +} + static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) { struct isp_format *fmt = isp_get_format(isp, ch); @@ -381,6 +483,8 @@ static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) err = isp_check_presets(isp, args->num_presets); if (!err) err = isp_ch_cache_sensor_configs(isp, ch, args->num_presets); + if (!err) + err = isp_add_call_sizes(isp); exit: kfree(args); From adf7d97d41a7ae9446b8bffd4cb06d38d9820f04 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:22:53 +1100 Subject: [PATCH 1497/1579] drm/apple: preserve optional firmware panel colour characteristics The panel parser currently skips Chromaticity and DefaultWhitePoint, and the identity EDID builder always emits zero RGB primaries and unknown gamma. Retain complete optional firmware colour data and use the same validated encoder as the external metadata EDID path. Keep identity usable when optional colour data is absent, malformed or duplicated. Preserve established colour data across a same-panel ProductAttributes-only update, without carrying it across different panel serials. Validated raw EDIDs retain precedence. Add synthetic P3 and sRGB regression inputs through the production parser and builder, including packing, checksum, malformed descriptions and partial identity updates. No panel calibration or model table is added. Document the missing native data handoff and read-only verification requirements. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../gpu/apple-dcp-panel-colorimetry.rst | 158 ++++++++++++++++++ Documentation/gpu/apple-dcp-t6041-panel.rst | 7 +- Documentation/gpu/drivers.rst | 1 + drivers/gpu/drm/apple/dcp-display-metadata.c | 75 +++++---- drivers/gpu/drm/apple/dcp.c | 6 + drivers/gpu/drm/apple/parser.c | 24 ++- .../drivers/gpu/apple-dcp/colorimetry.py | 107 ++++++++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 2 + 8 files changed, 347 insertions(+), 33 deletions(-) create mode 100644 Documentation/gpu/apple-dcp-panel-colorimetry.rst create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/colorimetry.py diff --git a/Documentation/gpu/apple-dcp-panel-colorimetry.rst b/Documentation/gpu/apple-dcp-panel-colorimetry.rst new file mode 100644 index 0000000000000..e4b177f7ba2d3 --- /dev/null +++ b/Documentation/gpu/apple-dcp-panel-colorimetry.rst @@ -0,0 +1,158 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +Apple internal-panel colour metadata +==================================== + +The panel identity EDID and its colour description are separate inputs. A +checksum-valid EDID with zero RGB primaries does not describe an sRGB panel; +it describes unknown primaries. A Display P3 default profile must not be +inferred merely from the Apple vendor ID or the DCP/SoC generation. + +Sources and publication path +---------------------------- + +The existing paths in appledrm are: + +* External displays: ``epic/dpavservep.c`` obtains an EDID with the existing + DCPAV service command. ``dcp_get_modes()`` in ``iomfb.c`` publishes it. +* Desktop internal panels: the ``DisplayAttributes`` dictionary reaches + ``dcpep_process_chunks()`` in ``iomfb_template.c`` or the existing dictionary + setters in ``iomfb_v26_6_desktop.c``. ``parse_panel_metadata()`` in + ``parser.c`` decodes the description; ``dcp_panel_metadata_replace()`` in + ``dcp.c`` retains it. ``dcp_panel_edid_snapshot()`` prefers a validated raw + panel EDID and otherwise calls ``dcp_panel_edid()`` in + ``dcp-display-metadata.c``. ``dcp_get_modes()`` publishes that snapshot with + ``drm_edid_connector_update()``. The actual mode catalog remains DCP's + ``TimingElements``, rather than the EDID's identity timing summary. +* H17P internal panels currently have no supported internal EDID command + transport and do not enter the desktop identity fallback. An empty EDID on + that path is a distinct missing publication path, not evidence for P3. + +Before the optional-colour fix, the desktop parser retained only +``ProductAttributes`` and the panel builder unconditionally emitted zero RGB +primaries, nominal D65 white and unknown gamma (0xff). The shared colour +encoder now validates and encodes a complete firmware description. Incomplete, +duplicate or malformed optional fields leave panel identity usable. A +standalone ``ProductAttributes`` update for the same panel retains its previous +colour description. A different panel serial cannot inherit that description. +Validated raw EDIDs retain precedence and are not rewritten by this fix. +There is no model/gamut table or new platform switch in this implementation. + +The firmware fields to verify are precisely: + +* ``DisplayAttributes.Chromaticity.Red.X`` and ``.Y``; +* ``DisplayAttributes.Chromaticity.Green.X`` and ``.Y``; +* ``DisplayAttributes.Chromaticity.Blue.X`` and ``.Y``; +* ``DisplayAttributes.DefaultWhitePoint.X``, ``.Y`` and ``.Gamma``. + +The existing parser interprets these as unsigned 16.16 values. EDID base-block +chromaticities round to units of 1/1024; gamma rounds to a hundredth. An absent +transfer characteristic must not be replaced by an invented panel gamma. +The optional values are validated before any colour bytes are changed, +including nonzero/bounded coordinates, nondegenerate primary triangle and +white-point containment. These are metadata validation rules, not panel +calibration or a measurement of its optical output. + +Data still required +------------------ + +The current ``dcp-panel-adt.c`` reader consumes panel identity and dimensions, +not colour calibration. Neither its implementation nor the panel binding +establishes an ADT native-primary property. Do not interpret arbitrary ADT +calibration blobs, display mode colourimetry IDs, or an ICC profile's +D50-adapted matrix as native panel xy coordinates. + +First inspect the cached firmware ``DisplayAttributes`` dictionary for the +fields above. If they are absent, obtain verified native panel xy coordinates, +white and transfer characteristic through a documented ADT/firmware producer. +The fallback then belongs on the existing ``panel`` child of the DCP node, +with a binding in ``Documentation/devicetree/bindings/display/apple,dcp.yaml`` +and a shared reader alongside ``dcp_panel_read_dimensions()`` in ``dcp.c``. +A concrete proposed representation is ``apple,native-chromaticities`` (eight +u32 16.16 cells, Rx/Ry/Gx/Gy/Bx/By/Wx/Wy) plus ``apple,native-gamma`` (one +u32 16.16 cell). These names are a proposal, not supported properties in this +patch. The bootloader can populate them from verified native data; static +board DT values require an authoritative panel specification. Firmware data +should take precedence over a DT fallback. The internal metadata store and +publication path must also be shared with H17P rather than widening the +external-display service/ownership exceptions. + +There is currently no verified runtime colour payload in the investigation +that warrants populating these properties. The optional-colour fix therefore +does not assert that a particular panel now advertises P3, and it does not +manufacture a colour description when firmware omits it. + +Expected gamut and userspace +--------------------------- + +Apple specifies the 2024 16-inch MacBook Pro as wide colour P3 [1]_, but the +MacBook Neo as sRGB [2]_. Neo must not receive a P3 default. Standard Display P3 +uses native xy R (0.680, 0.320), G (0.265, 0.690), B (0.150, 0.060), white +(0.3127, 0.3290) [3]_. These standard values are useful test fixtures, not a +substitute for a particular panel's native calibration. ICC rXYZ/gXYZ/bXYZ +values expressed in a D50 connection space require the profile's chromatic +adaptation to be undone before comparison with a D65 EDID. + +Mutter 51's ``meta-color-device.c`` rejects zero primaries when generating an +EDID ICC profile, and also needs a usable gamma. Its SDR native colour mode +uses EDID primaries and transfer information; its default colour mode still +uses sRGB [4]_. Thus EDID/profile detection alone does not prove wide-gamut +output is selected. The DRM ``Colorspace`` property describes output signal +encoding, not an inventory of ICC profiles; a P3 label need not appear there. + +Fedora, Arch Linux ARM and NixOS need the same usable kernel metadata and +compatible Mutter/colord colour-management stack. With a valid native EDID, +a separate distro-supplied ICC is not inherently required just to identify +the gamut. A measured/complex transfer curve, missing firmware data, or an +older compositor can still require a userspace profile or compositor work. +No three-distro runtime parity or post-patch profile selection is established +by the host selftests. Preserve an existing calibrated profile until native +EDID and the effective compositor colour mode have been checked. + +Read-only verification +---------------------- + +Run these through the hardware controller with explicit device identity and +policy approval. Reading the EDID attribute size with stat is insufficient; +read its contents:: + + for edid in /sys/class/drm/card*-*/edid; do + bytes=$(cat "$edid" | wc -c) + printf '%s: %s bytes\n' "$edid" "$bytes" + if test "$bytes" -gt 0; then + edid-decode "$edid" + fi + done + modetest -M apple -c + colormgr get-devices-by-kind display + colormgr get-profiles + +Sysfs may report zero file size even for a populated EDID. Missing tools +should be reported; do not install packages or change a display mode to obtain +this evidence. Firmware dictionary +inspection reads only the cached ``DisplayAttributes`` debugfs file, after +controller approval. Do not enable new trace streams or copy raw panel data +into public fixtures. + +Check nonzero decoded native primaries/white, gamma, EDID checksum, the internal +connector and its unchanged mode list, then the display device's default +profile in GNOME Colour or colormgr. A successful result records whether the +profile is generated from EDID or explicitly assigned, and the effective +Mutter colour mode. Compare profiles after accounting for ICC chromatic +adaptation. Publish only sanitized colour summaries, not raw EDIDs, serials, +firmware dictionaries or calibration records. + +Host regression command (no hardware access):: + + python3 tools/testing/selftests/drivers/gpu/apple-dcp/run.py \ + --output /tmp/apple-dcp-results + +The ``panel-colorimetry`` test uses synthetic P3 and sRGB primaries with a +synthetic gamma of 2.2 through the production parser and encoder. It checks packed coordinates, gamma, checksums, +missing/malformed/duplicate fields, degenerate geometry and same-panel identity +updates. Existing external EDID and panel identity tests remain relevant. + +.. [1] https://support.apple.com/en-gb/121554 +.. [2] https://www.apple.com/macbook-neo/specs/ +.. [3] https://www.w3.org/TR/css-color-4/#predefined-display-p3 +.. [4] https://gitlab.gnome.org/GNOME/mutter/-/blob/51.0/src/backends/meta-color-device.c diff --git a/Documentation/gpu/apple-dcp-t6041-panel.rst b/Documentation/gpu/apple-dcp-t6041-panel.rst index bdf15e644d68f..fa45c33f678ad 100644 --- a/Documentation/gpu/apple-dcp-t6041-panel.rst +++ b/Documentation/gpu/apple-dcp-t6041-panel.rst @@ -140,8 +140,11 @@ programmed mode; the original DCP sync positions and mode IDs are unchanged. This also lets a 118-line porch and 82-line notch-expanded pulse publish an identity instead of failing the base EDID size check. The EDID records model year 2024 (not a manufacture date) and a nominal D65 -reference white point required by EDID parsing. No calibrated primaries, gamma, -new mode or machine serial is claimed. The ADT reader, +reference white point required by EDID parsing. When a complete valid optional +firmware colour description is available, the shared encoder supplies its +primaries, white and gamma instead; see :doc:`apple-dcp-panel-colorimetry`. +Missing colour data does not imply calibrated primaries or gamma. No new mode +or machine serial is claimed. The ADT reader, EDID fallback and real EDID setters are restricted to desktop 26.6; T8140 continues to use its existing path. diff --git a/Documentation/gpu/drivers.rst b/Documentation/gpu/drivers.rst index 51464c3d54891..38f99ddda7c83 100644 --- a/Documentation/gpu/drivers.rst +++ b/Documentation/gpu/drivers.rst @@ -7,6 +7,7 @@ GPU Driver Documentation amdgpu/index apple-dcp-backlight + apple-dcp-panel-colorimetry apple-drm-parameters asahi-reclaim asahi-usage-stats diff --git a/drivers/gpu/drm/apple/dcp-display-metadata.c b/drivers/gpu/drm/apple/dcp-display-metadata.c index 3a40d288b024c..94518eb9d0824 100644 --- a/drivers/gpu/drm/apple/dcp-display-metadata.c +++ b/drivers/gpu/drm/apple/dcp-display-metadata.c @@ -89,26 +89,13 @@ static s64 dcp_metadata_side(const u32 *a, const u32 *b, const u32 *point) ((s64)b[1] - a[1]) * ((s64)point[0] - a[0]); } -/* Build only a base block: the authoritative firmware mode list stays separate. */ -static int dcp_metadata_block(struct edid *edid, const struct dcp_display_metadata *metadata, - const struct drm_display_mode *mode) +/* Validate the complete colour description before changing any EDID bytes. */ +static int dcp_metadata_color(struct edid *edid, const struct dcp_display_metadata *metadata) { - u32 coordinate[8], manufacturer = 0, gamma; - u8 checksum = 0; - int ret; + u32 coordinate[8], gamma; - if (!metadata->width_cm || metadata->width_cm > 255 || - !metadata->height_cm || metadata->height_cm > 255 || - metadata->product > U16_MAX || metadata->week > 53 || - metadata->year < 1990 || metadata->year > 2245 || - metadata->gamma < 65536 || metadata->gamma > (354U << 16) / 100 || - strnlen(metadata->manufacturer, 4) != 3) + if (metadata->gamma < 65536 || metadata->gamma > (354U << 16) / 100) return -EINVAL; - for (unsigned int i = 0; i < 3; i++) { - if (metadata->manufacturer[i] < 'A' || metadata->manufacturer[i] > 'Z') - return -EINVAL; - manufacturer = manufacturer << 5 | (metadata->manufacturer[i] - '@'); - } for (unsigned int i = 0; i < 8; i++) { u32 value = i < 6 ? metadata->chromaticity[i / 2][i % 2] : metadata->white[i - 6]; @@ -138,6 +125,42 @@ static int dcp_metadata_block(struct edid *edid, const struct dcp_display_metada if (gamma > 254) return -EINVAL; + edid->gamma = gamma; + edid->red_green_lo = (coordinate[0] & 3) << 6 | (coordinate[1] & 3) << 4 | + (coordinate[2] & 3) << 2 | (coordinate[3] & 3); + edid->blue_white_lo = (coordinate[4] & 3) << 6 | (coordinate[5] & 3) << 4 | + (coordinate[6] & 3) << 2 | (coordinate[7] & 3); + edid->red_x = coordinate[0] >> 2; + edid->red_y = coordinate[1] >> 2; + edid->green_x = coordinate[2] >> 2; + edid->green_y = coordinate[3] >> 2; + edid->blue_x = coordinate[4] >> 2; + edid->blue_y = coordinate[5] >> 2; + edid->white_x = coordinate[6] >> 2; + edid->white_y = coordinate[7] >> 2; + return 0; +} + +/* Build only a base block: the authoritative firmware mode list stays separate. */ +static int dcp_metadata_block(struct edid *edid, const struct dcp_display_metadata *metadata, + const struct drm_display_mode *mode) +{ + u32 manufacturer = 0; + u8 checksum = 0; + int ret; + + if (!metadata->width_cm || metadata->width_cm > 255 || + !metadata->height_cm || metadata->height_cm > 255 || + metadata->product > U16_MAX || metadata->week > 53 || + metadata->year < 1990 || metadata->year > 2245 || + strnlen(metadata->manufacturer, 4) != 3) + return -EINVAL; + for (unsigned int i = 0; i < 3; i++) { + if (metadata->manufacturer[i] < 'A' || metadata->manufacturer[i] > 'Z') + return -EINVAL; + manufacturer = manufacturer << 5 | (metadata->manufacturer[i] - '@'); + } + memset(edid, 0, sizeof(*edid)); memcpy(edid->header, "\x00\xff\xff\xff\xff\xff\xff\x00", 8); edid->product_id.manufacturer_name = cpu_to_be16(manufacturer); @@ -151,20 +174,10 @@ static int dcp_metadata_block(struct edid *edid, const struct dcp_display_metada edid->input = DRM_EDID_INPUT_DIGITAL; edid->width_cm = metadata->width_cm; edid->height_cm = metadata->height_cm; - edid->gamma = gamma; + ret = dcp_metadata_color(edid, metadata); + if (ret) + return ret; edid->features = DRM_EDID_FEATURE_PREFERRED_TIMING | DRM_EDID_FEATURE_RGB; - edid->red_green_lo = (coordinate[0] & 3) << 6 | (coordinate[1] & 3) << 4 | - (coordinate[2] & 3) << 2 | (coordinate[3] & 3); - edid->blue_white_lo = (coordinate[4] & 3) << 6 | (coordinate[5] & 3) << 4 | - (coordinate[6] & 3) << 2 | (coordinate[7] & 3); - edid->red_x = coordinate[0] >> 2; - edid->red_y = coordinate[1] >> 2; - edid->green_x = coordinate[2] >> 2; - edid->green_y = coordinate[3] >> 2; - edid->blue_x = coordinate[4] >> 2; - edid->blue_y = coordinate[5] >> 2; - edid->white_x = coordinate[6] >> 2; - edid->white_y = coordinate[7] >> 2; memset(edid->standard_timings, 1, sizeof(edid->standard_timings)); ret = dcp_metadata_timing(&edid->detailed_timings[0], mode, metadata); if (!ret) @@ -400,6 +413,8 @@ const struct drm_edid *dcp_panel_edid(const struct dcp_display_metadata *metadat block.white_x = 320 >> 2; block.white_y = 337 >> 2; block.blue_white_lo = (320 & 3) << 2 | (337 & 3); + /* Missing or malformed optional colour data leaves the identity usable. */ + dcp_metadata_color(&block, metadata); block.features = DRM_EDID_FEATURE_PREFERRED_TIMING; memset(block.standard_timings, 1, sizeof(block.standard_timings)); snprintf(name, sizeof(name), "%.3s %04x", metadata->manufacturer, metadata->product); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 2c3404593e69a..698df18df9400 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -909,6 +909,12 @@ void dcp_panel_metadata_replace(struct apple_dcp *dcp, merged.serial = old->serial; if (same_serial && !merged.serial_text[0]) memcpy(merged.serial_text, old->serial_text, sizeof(merged.serial_text)); + /* Standalone ProductAttributes setters omit the colour description. */ + if (same_serial && !merged.gamma) { + memcpy(merged.chromaticity, old->chromaticity, sizeof(merged.chromaticity)); + memcpy(merged.white, old->white, sizeof(merged.white)); + merged.gamma = old->gamma; + } if (!merged.year) merged.year = old->year; if (!merged.week) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index ca0883907a7d6..0db027145b8f8 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -1709,7 +1709,8 @@ int parse_panel_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metada { struct dcp_display_metadata parsed = {}; struct iterator it; - bool product = false; + bool product = false, chromaticity = false, white = false; + unsigned int color_seen = 0; int ret = iterator_begin(handle, &it, true); if (ret) @@ -1725,6 +1726,21 @@ int parse_panel_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metada return -EINVAL; product = true; ret = parse_metadata_product(handle, &parsed, true); + } else if (!strcmp(key, "Chromaticity")) { + struct dcp_parse_ctx color = *handle; + + /* Optional colour data must not invalidate panel identity. */ + chromaticity = !parse_metadata_chromaticity(&color, &parsed) && + !(color_seen & 1); + color_seen |= 1; + ret = skip(handle); + } else if (!strcmp(key, "DefaultWhitePoint")) { + struct dcp_parse_ctx color = *handle; + + white = !parse_metadata_point(&color, parsed.white, &parsed.gamma) && + !(color_seen & 2); + color_seen |= 2; + ret = skip(handle); } else { ret = skip(handle); } @@ -1733,6 +1749,12 @@ int parse_panel_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metada } if (!product) return -ENODATA; + /* Never combine incomplete or duplicate descriptions with a nominal white. */ + if (!chromaticity || !white) { + memset(parsed.chromaticity, 0, sizeof(parsed.chromaticity)); + memset(parsed.white, 0, sizeof(parsed.white)); + parsed.gamma = 0; + } *out = parsed; return 0; } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/colorimetry.py b/tools/testing/selftests/drivers/gpu/apple-dcp/colorimetry.py new file mode 100644 index 0000000000000..0d13f17040ea4 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/colorimetry.py @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Synthetic firmware colour data through the production parser and EDID builder.""" +import copy +import struct +import desktop +import rate + + +def panel(root, output): + fixture = rate.parser_fixture(root) + unit = desktop.panel_identity(root).split('int main(int argc, char **argv)')[0] + # Standard colour-space coordinates, never a captured panel calibration. + p3 = {'Chromaticity': {'Red': {'X': 44564, 'Y': 20972}, + 'Green': {'X': 17367, 'Y': 45220}, + 'Blue': {'X': 9830, 'Y': 3932}}, + 'DefaultWhitePoint': {'X': 20493, 'Y': 21561, 'Gamma': 144179}, + 'ProductAttributes': {'ManufacturerID': 'TST', 'ProductID': 42, + 'ProductName': 'Synthetic', 'SerialNumber': 123}} + srgb = copy.deepcopy(p3) + srgb['Chromaticity']['Red'] = {'X': 41943, 'Y': 21627} + srgb['Chromaticity']['Green'] = {'X': 19661, 'Y': 39322} + absent = {'ProductAttributes': p3['ProductAttributes']} + white_only = copy.deepcopy(p3); del white_only['Chromaticity'] + chroma_only = copy.deepcopy(p3); del chroma_only['DefaultWhitePoint'] + zero = copy.deepcopy(p3); zero['Chromaticity']['Red']['X'] = 0 + gamma = copy.deepcopy(p3); del gamma['DefaultWhitePoint']['Gamma'] + bad_gamma = copy.deepcopy(p3); bad_gamma['DefaultWhitePoint']['Gamma'] = 65535 + wrong_type = copy.deepcopy(p3); wrong_type['Chromaticity'] = 'P3' + # The wire encoder supports an ordered multimap to exercise duplicate keys. + pairs = fixture['ObjectPairs'] + duplicate = pairs(list(p3.items()) + [('Chromaticity', p3['Chromaticity'])]) + duplicate_white = pairs(list(p3.items()) + [('DefaultWhitePoint', p3['DefaultWhitePoint'])]) + duplicate_bad = pairs(list(zero.items()) + [('Chromaticity', p3['Chromaticity'])]) + inputs = [p3, srgb, absent, white_only, chroma_only, zero, gamma, bad_gamma, + wrong_type, duplicate, duplicate_white, duplicate_bad] + for i, value in enumerate(inputs): + raw = bytearray(struct.pack('raw; + if(color) { + u32 expected[8]={metadata->chromaticity[0][0],metadata->chromaticity[0][1], + metadata->chromaticity[1][0],metadata->chromaticity[1][1], + metadata->chromaticity[2][0],metadata->chromaticity[2][1], + metadata->white[0],metadata->white[1]}; + for(unsigned i=0;i<8;i++) { + u8 low=raw[i<4?25:26]; + u32 coordinate=(u32)raw[27+i]<<2 | ((low>>(6-2*(i%4)))&3); + assert(coordinate==(expected[i]+32)/64); + } + assert(raw[23]==120); + } else { + assert(raw[23]==255); + for(unsigned i=27;i<33;i++)assert(!raw[i]); + assert(!raw[25] && !(raw[26]&0xf0)); + } + if(path) {FILE *file=fopen(path,"wb");assert(file);assert(fwrite(raw,128,1,file)==1);fclose(file);} + drm_edid_free(edid); +} +int main(int argc, char **argv) +{ + assert(argc==3); + struct dcp_display_metadata p3=decode(color_0,sizeof(color_0)); + struct dcp_display_metadata srgb=decode(color_1,sizeof(color_1)); + assert(p3.gamma==144179 && p3.chromaticity[0][0]==44564 && p3.white[0]==20493); + assert(srgb.chromaticity[0][0]==41943); + check(&p3,true,argv[1]);check(&srgb,true,argv[2]); +''' + for i in range(2, len(inputs)): + unit += f'{{struct dcp_display_metadata m=decode(color_{i},sizeof(color_{i}));assert(!m.gamma && !m.white[0] && !m.chromaticity[0][0]);check(&m,false,NULL);}}\n' + unit += r''' + struct dcp_display_metadata invalid=p3; + memcpy(invalid.chromaticity[1],invalid.chromaticity[0],sizeof(invalid.chromaticity[0])); + check(&invalid,false,NULL); + invalid=p3;invalid.white[0]=1;invalid.white[1]=1;check(&invalid,false,NULL); + invalid=p3;invalid.chromaticity[0][0]=65535;check(&invalid,false,NULL); + /* A ProductAttributes-only setter must preserve the established colour. */ + struct apple_dcp d={.fw_compat=5,.has_panel=true,.connector=&d}; + __typeof__(*d.desktop) state={0};d.desktop=&state;notifications=0; + dcp_panel_metadata_replace(&d,&p3);assert(notifications==1); + struct dcp_display_metadata partial=decode(color_2,sizeof(color_2)); + dcp_panel_metadata_replace(&d,&partial); + assert(notifications==1 && !memcmp(&state.panel_metadata,&p3,sizeof(p3))); + /* A replacement panel must not inherit another serial's colour data. */ + partial.serial++; + dcp_panel_metadata_replace(&d,&partial); + assert(notifications==2 && !state.panel_metadata.gamma); + return 0; +} +''' + return unit, ((str(output/'panel-display-p3.edid'), str(output/'panel-srgb.edid')),) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index b156582f145f6..f1623a4ea8f64 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -32,6 +32,7 @@ import boot import m4 import display +import colorimetry def parking_unit(root): @@ -54,6 +55,7 @@ def main(): 'orientation-faults', 'backend-change', 'safe-first', 'failed-flip-return', 'live-park')) tests = [ + ('display', 'panel-colorimetry', lambda r: colorimetry.panel(r, output), ((),), ('-Wno-unused-function', '-Wno-unused-const-variable')), ('display', 'desktop-gamma', desktop.gamma, ((),), ()), ('lifecycle', 'desktop-modeset-unmap', desktop.modeset_unmap, ((),), ()), ('lifecycle', 'desktop-unknown-modeset-recovery', desktop.modeset_unknown, ((),), ()), From a97f7b8be0ebf1c712a72b695abaaaa7f0dc1181 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 16:36:28 +1100 Subject: [PATCH 1498/1579] drm/asahi: present the GPU driver as Aurora The GPU driver now identifies itself as Aurora everywhere userspace can see it. Our G13 to G17 code has moved well past where it started, and M3 and newer never had anything to do with the old name. What changes: - DRM name "aurora", desc "Aurora GPU for Apple AGX", version 1.0.0. The major version names the UAPI generation, the minor goes up with each new ioctl, flag or feature bit, and the patch level marks fixes userspace may need to detect. The old 0.0.0 said nothing. - The module is aurora.ko and keeps "asahi" as a module alias, so "modprobe asahi" and modinfo still work. The main platform driver is "aurora" and the G15 and G16 ones are "aurora-g15" and "aurora-g16". - debugfs moves to aurora-g15, aurora-g16 and aurora-g17. - dma-fence driver and timeline names, scheduler, workqueue, shrinker and cooling device names use aurora. Log lines get their prefix from the module name, so the two memacct messages drop their own copy. - Kconfig prompts and help say Aurora. What stays, on purpose: - CONFIG_DRM_ASAHI and the other Kconfig symbols. Every distro config and build gate names them, and a renamed symbol would silently drop the GPU driver from those builds. - include/uapi/drm/asahi_drm.h and its ioctl, struct and enum names. That header is ABI and API for Mesa and other userspace, and the ioctl numbers and layouts are unchanged. - Firmware paths and the asahi,* device tree properties the loaders pass in. Those are contracts with the boot chain and the firmware layout, not branding. - The source directory and internal Rust and C symbol names, which can follow later without any userspace effect. - Copyright, SPDX and author credit, unchanged. Userspace that matches the DRM name needs to know "aurora": aurora-silicon Mesa accepts both names from the companion Mesa change, and that has to ship before this kernel. The in-tree selftests and the G16 hardware tests accept both names. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/gpu/asahi-g17-capacity.rst | 2 +- drivers/gpu/drm/asahi/Kconfig | 11 +++++++---- drivers/gpu/drm/asahi/Makefile | 8 ++++---- drivers/gpu/drm/asahi/asahi_core.rs | 5 +++-- drivers/gpu/drm/asahi/cleanup.rs | 2 +- drivers/gpu/drm/asahi/driver.rs | 12 +++++++++--- drivers/gpu/drm/asahi/g15/engine.rs | 8 ++++---- drivers/gpu/drm/asahi/g15/fence.rs | 6 +++--- drivers/gpu/drm/asahi/g15/glue.c | 6 +++--- drivers/gpu/drm/asahi/g15/job.rs | 2 +- drivers/gpu/drm/asahi/g15/mod.rs | 2 +- drivers/gpu/drm/asahi/g15/poll.rs | 12 ++++++------ drivers/gpu/drm/asahi/g15/power.rs | 4 ++-- drivers/gpu/drm/asahi/g15/power/pstate.rs | 4 ++-- drivers/gpu/drm/asahi/g15/recovery.rs | 2 +- drivers/gpu/drm/asahi/g15/slots.rs | 4 ++-- drivers/gpu/drm/asahi/g16/engine.rs | 6 +++--- drivers/gpu/drm/asahi/g16/fence.rs | 6 +++--- drivers/gpu/drm/asahi/g16/glue.c | 2 +- drivers/gpu/drm/asahi/g16/job.rs | 2 +- drivers/gpu/drm/asahi/g16/mod.rs | 4 ++-- drivers/gpu/drm/asahi/g16/poll.rs | 8 ++++---- drivers/gpu/drm/asahi/g17/availability.rs | 2 +- drivers/gpu/drm/asahi/g17/fence.rs | 4 ++-- drivers/gpu/drm/asahi/g17/job.rs | 4 ++-- drivers/gpu/drm/asahi/g17/mod.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 2 +- drivers/gpu/drm/asahi/g17/sync_scope.c | 2 +- drivers/gpu/drm/asahi/memacct.rs | 4 ++-- drivers/gpu/drm/asahi/mmu/residency.rs | 2 +- drivers/gpu/drm/asahi/queue/mod.rs | 6 +++--- drivers/gpu/drm/asahi/sysfs.c | 2 +- tools/asahi/g15-probe-init-tests.py | 8 ++++---- tools/asahi/g17-capacity-watch.sh | 2 +- .../selftests/drivers/gpu/asahi-m4/README | 2 +- .../gpu/asahi-m4/native-ioctl-regression.c | 7 +++++-- .../drivers/gpu/asahi-m4/run-hardware-tests.py | 4 ++-- .../asahi-m4/run-native-ioctl-regression.py | 18 ++++++++++++------ 38 files changed, 104 insertions(+), 85 deletions(-) diff --git a/Documentation/gpu/asahi-g17-capacity.rst b/Documentation/gpu/asahi-g17-capacity.rst index 68d261682f842..43c61b6364688 100644 --- a/Documentation/gpu/asahi-g17-capacity.rst +++ b/Documentation/gpu/asahi-g17-capacity.rst @@ -1,7 +1,7 @@ G17 capacity diagnostics ======================= -The read-only ``/sys/kernel/debug/asahi-g17/capacity`` file samples the live +The read-only ``/sys/kernel/debug/aurora-g17/capacity`` file samples the live firmware session. It does not wake firmware or reclaim resources. An absent session reports ``online=0``. The format is diagnostic, not a stable ABI. Counts exclude reserved identities. Sub-pool locks and atomic snapshots mean diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 3903ee68b162b..647f8f37507d4 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -9,7 +9,7 @@ config RUST_DRM_GPUVM select DRM_GPUVM config DRM_ASAHI - tristate "Asahi (DRM support for Apple AGX GPUs)" + tristate "Aurora (DRM support for Apple AGX GPUs)" depends on RUST depends on DRM=y depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) @@ -23,7 +23,10 @@ config DRM_ASAHI select RUST_APPLE_RTKIT select CRYPTO_LIB_SHA256 help - DRM driver for Apple AGX GPUs (G13x, found in the M1 SoC family) + DRM driver for Apple AGX GPUs, from the M1 family onwards. + + To compile this driver as a module, choose M here: the module will + be called aurora. It keeps "asahi" as a module alias. config DRM_ASAHI_G16 bool "Apple M4 (G16G) support (experimental)" @@ -31,8 +34,8 @@ config DRM_ASAHI_G16 depends on APPLE_PMP=y || APPLE_PMP=DRM_ASAHI depends on THERMAL_OF help - Rust Asahi backend for the qualified J713/T8132 GPU firmware. - Load asahi explicitly after the native display and GFX TVM supplier + Rust backend for the qualified J713/T8132 GPU firmware. + Load aurora explicitly after the native display and GFX TVM supplier are ready. Firmware-visible storage remains retained until a verified retirement or reboot. Keep the working kernel available during bring-up. A rendering VM pins about 0.7 GiB; memory-cgroup limits are advisable. diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index fd8421f27590d..db13d06336b3c 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,9 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o +obj-$(CONFIG_DRM_ASAHI) += aurora.o +aurora-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o CFLAGS_g17/trace.o := -I$(src)/g17 -asahi-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o +aurora-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o -asahi-$(CONFIG_DRM_ASAHI_G16) += g16/glue.o +aurora-$(CONFIG_DRM_ASAHI_G16) += g16/glue.o diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 57c3f16e7df2a..42843e3459217 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -92,7 +92,7 @@ impl kernel::InPlaceModule for DriverModule { // Initialize the queue first, before registration can invoke probe. _cleanup: cleanup::QueueOwner::new()?, #[cfg(CONFIG_DRM_ASAHI_G15)] - _g15 <- kernel::driver::Registration::new(c"asahi-g15", module), + _g15 <- kernel::driver::Registration::new(c"aurora-g15", module), _driver <- kernel::driver::Registration::new( ::NAME, module, @@ -107,9 +107,10 @@ impl kernel::InPlaceModule for DriverModule { module! { type: DriverModule, - name: "asahi", + name: "aurora", description: "AGX GPU driver for Apple silicon SoCs", license: "Dual MIT/GPL", + alias: ["asahi"], params: { debug_flags: u64 { default: 0, diff --git a/drivers/gpu/drm/asahi/cleanup.rs b/drivers/gpu/drm/asahi/cleanup.rs index e3dd5ad95dc0c..52436b40304c0 100644 --- a/drivers/gpu/drm/asahi/cleanup.rs +++ b/drivers/gpu/drm/asahi/cleanup.rs @@ -33,7 +33,7 @@ impl QueueOwner { // SAFETY: The name is NUL terminated and contains no format specifiers. let raw = unsafe { bindings::alloc_workqueue_noprof( - c_str!("asahi-ts-cleanup").as_char_ptr(), + c_str!("aurora-ts-cleanup").as_char_ptr(), bindings::wq_flags_WQ_UNBOUND | bindings::wq_flags_WQ_MEM_RECLAIM, 0, ) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index caf0ef32ca427..a89728cc2f2b3 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -79,12 +79,18 @@ pub(crate) type AsahiDevRef = ARef; pub(crate) type AsahiRegData<'drm> = AsahiDrmRegistrationData<'drm>; /// DRM Driver metadata +/// +/// The major version names the UAPI generation in include/uapi/drm/asahi_drm.h +/// and only changes if that ABI ever breaks. The minor version goes up with +/// each new ioctl, flag or feature bit, so userspace can tell what a kernel +/// accepts without probing. The patch level marks fixes userspace may need to +/// detect. const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { - major: 0, + major: 1, minor: 0, patchlevel: 0, - name: c"asahi", - desc: c"Apple AGX Graphics", + name: c"aurora", + desc: c"Aurora GPU for Apple AGX", }; /// DRM Driver implementation for `AsahiDriver`. diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index e5c18a15e0c3b..51b3cfd10d594 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -221,7 +221,7 @@ struct Prefix { fs_ts: bool, } -/// Engine statistics, printed by `asahi-g15/stats`. +/// Engine statistics, printed by `aurora-g15/stats`. #[derive(Default)] pub(crate) struct Stats { pub(crate) pub_ns: u64, @@ -271,7 +271,7 @@ pub(crate) struct Stats { } /// Host-side phase timers of publication and polling, on with `debug_flags` bit 49 -/// (`Debug1`); printed as the `prof:` line of `asahi-g15/stats`. A measurement aid: nothing is +/// (`Debug1`); printed as the `prof:` line of `aurora-g15/stats`. A measurement aid: nothing is /// timed while the flag is off. #[derive(Default)] pub(crate) struct Prof { @@ -391,9 +391,9 @@ pub(crate) struct Engine { pub(crate) stats: Stats, /// Phase timers (debug_flags bit 49). pub(crate) prof: Prof, - /// The `asahi-g15/stats` file. + /// The `aurora-g15/stats` file. pub(crate) debugfs: Option>>>>, - /// The `asahi-g15/pstate` file. + /// The `aurora-g15/pstate` file. pub(crate) debugfs_pstate: Option>>>>, scratch: Option>, } diff --git a/drivers/gpu/drm/asahi/g15/fence.rs b/drivers/gpu/drm/asahi/g15/fence.rs index a959ff00d4bb1..bb85764228070 100644 --- a/drivers/gpu/drm/asahi/g15/fence.rs +++ b/drivers/gpu/drm/asahi/g15/fence.rs @@ -32,7 +32,7 @@ pub(crate) struct JobFenceOps; #[vtable] impl FenceOps for JobFenceOps { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("asahi") + c_str!("aurora") } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { @@ -63,11 +63,11 @@ pub(crate) struct Timeline { impl Timeline { pub(crate) fn new() -> Result> { - let errors = FenceContexts::new(1, c_str!("asahi_g15_refused"), ERROR_KEY)?; + let errors = FenceContexts::new(1, c_str!("aurora_g15_refused"), ERROR_KEY)?; let allocation_failure = error_fence(&errors, ENOMEM)?; Arc::pin_init( try_pin_init!(Self { - jobs: FenceContexts::new(1, c_str!("asahi_g15_queue"), TIMELINE_KEY)?, + jobs: FenceContexts::new(1, c_str!("aurora_g15_queue"), TIMELINE_KEY)?, errors, pending <- new_mutex!(KVec::new(), "g15::Timeline::pending"), allocation_failure, diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index 662c9c8258e21..b949a4b9b61ed 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -267,12 +267,12 @@ void asahi_g15_reprobe(void) const struct device_driver *drv; int ret; - drv = driver_find("asahi-g15", &platform_bus_type); + drv = driver_find("aurora-g15", &platform_bus_type); if (!drv) { - pr_err("asahi-g15: registered adapter not found for activation\n"); + pr_err("aurora-g15: registered adapter not found for activation\n"); return; } ret = driver_attach(drv); if (ret) - pr_err("asahi-g15: activation attach failed: %d\n", ret); + pr_err("aurora-g15: activation attach failed: %d\n", ret); } diff --git a/drivers/gpu/drm/asahi/g15/job.rs b/drivers/gpu/drm/asahi/g15/job.rs index 159eea1dec3bc..5f6669f06d1b0 100644 --- a/drivers/gpu/drm/asahi/g15/job.rs +++ b/drivers/gpu/drm/asahi/g15/job.rs @@ -243,7 +243,7 @@ impl Scheduler { hw::sizes::HORIZON_JOBS, 0, hw::tuning::SCHED_TIMEOUT_MS, - c_str!("asahi_g15"), + c_str!("aurora_g15"), )?; // With one run queue, only the kernel priority is valid. let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index e39180f02248a..713a1360b5687 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -441,7 +441,7 @@ impl Gpu { *shared.state.lock() = Some(firmware); // The engine's statistics file (a debugfs failure does not fail the probe). if let Err(error) = shared.start_engine() { - dev_warn!(dev.as_ref(), "G15: no asahi-g15 debugfs ({:?})\n", error); + dev_warn!(dev.as_ref(), "G15: no aurora-g15 debugfs ({:?})\n", error); } let gpu = Arc::new( Gpu { diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index bf04984b7ed17..566995eb342c9 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -54,7 +54,7 @@ impl HrTimerCallback for Shared { } } -/// The `asahi-g15/stats` debugfs file contents. +/// The `aurora-g15/stats` debugfs file contents. fn stats_show(shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { let state = shared.state.lock(); match state.as_deref() { @@ -63,7 +63,7 @@ fn stats_show(shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { } } -/// The `asahi-g15/pstate` file contents. +/// The `aurora-g15/pstate` file contents. fn pstate_show(shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { let state = shared.state.lock(); match state.as_deref() { @@ -160,10 +160,10 @@ impl Shared { self.mod_poll(1); } - /// Starts what the engine needs besides the firmware state: the `asahi-g15/stats` and - /// `asahi-g15/pstate` files. + /// Starts what the engine needs besides the firmware state: the `aurora-g15/stats` and + /// `aurora-g15/pstate` files. pub(crate) fn start_engine(self: &Arc) -> Result { - let dir = debugfs::Dir::new(c_str!("asahi-g15")); + let dir = debugfs::Dir::new(c_str!("aurora-g15")); let file = KBox::pin_init( dir.read_callback_file(c_str!("stats"), self.clone(), &stats_show), GFP_KERNEL, @@ -654,7 +654,7 @@ impl Firmware { } } - /// The `asahi-g15/stats` contents. Scripts parse the keys: keep them stable. + /// The `aurora-g15/stats` contents. Scripts parse the keys: keep them stable. pub(crate) fn write_stats(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let s = &self.engine.stats; let slots = &self.slots.stats; diff --git a/drivers/gpu/drm/asahi/g15/power.rs b/drivers/gpu/drm/asahi/g15/power.rs index 58b6636fde383..69680f4636499 100644 --- a/drivers/gpu/drm/asahi/g15/power.rs +++ b/drivers/gpu/drm/asahi/g15/power.rs @@ -13,7 +13,7 @@ //! - +0x2c idle decision enabled (template 1, unchanged). //! //! DVFS is firmware-run on the ADT tables. The state it runs at is read back by -//! `asahi-g15/pstate` ([`pstate`], [`super::Firmware::write_pstate`]): the newest power-state +//! `aurora-g15/pstate` ([`pstate`], [`super::Firmware::write_pstate`]): the newest power-state //! record of the stats ring, and HardwareDataA's state words. mod idle; @@ -50,7 +50,7 @@ impl Firmware { }) } - /// The `asahi-g15/pstate` file ([`pstate::write`]): the stats ring's records so far, those + /// The `aurora-g15/pstate` file ([`pstate::write`]): the stats ring's records so far, those /// it holds unconsumed (read, not consumed), HardwareDataA's words and table A of the /// device tree's operating points. Read-only: nothing firmware-visible is written. pub(crate) fn write_pstate(&self, f: &mut impl fmt::Write) -> fmt::Result { diff --git a/drivers/gpu/drm/asahi/g15/power/pstate.rs b/drivers/gpu/drm/asahi/g15/power/pstate.rs index 411fe22413893..0ea68f02ac180 100644 --- a/drivers/gpu/drm/asahi/g15/power/pstate.rs +++ b/drivers/gpu/drm/asahi/g15/power/pstate.rs @@ -12,7 +12,7 @@ //! An index counts the firmware's DVFS states, 0 (off) to 8; its frequency is the state's //! table-A operating point ([`super::super::board`]), taken from the device tree. //! -//! [`write`] prints the `asahi-g15/pstate` file. Its first line is what a sampler reads: +//! [`write`] prints the `aurora-g15/pstate` file. Its first line is what a sampler reads: //! `pstate freq_khz source `, from the newest power-state record //! seen, or HardwareDataA's current state when the stats ring has carried none. //! @@ -181,7 +181,7 @@ impl fmt::Display for Khz { } } -/// The `asahi-g15/pstate` file: +/// The `aurora-g15/pstate` file: /// /// ```text /// pstate freq_khz source diff --git a/drivers/gpu/drm/asahi/g15/recovery.rs b/drivers/gpu/drm/asahi/g15/recovery.rs index 0827385e92c5f..b81734b1d5f2d 100644 --- a/drivers/gpu/drm/asahi/g15/recovery.rs +++ b/drivers/gpu/drm/asahi/g15/recovery.rs @@ -197,7 +197,7 @@ impl Firmware { self.recovery.armed.is_none() && !self.new_errors() } - /// `(recoveries, hang_kills)` for `asahi-g15/stats`. + /// `(recoveries, hang_kills)` for `aurora-g15/stats`. pub(crate) fn recovery_counts(&self) -> (u64, u64) { (self.recovery.recoveries, self.recovery.hang_kills) } diff --git a/drivers/gpu/drm/asahi/g15/slots.rs b/drivers/gpu/drm/asahi/g15/slots.rs index aae6d8804ee7f..8767eea321a0d 100644 --- a/drivers/gpu/drm/asahi/g15/slots.rs +++ b/drivers/gpu/drm/asahi/g15/slots.rs @@ -376,7 +376,7 @@ impl ClearQueue { // SAFETY: The name is a valid NUL-terminated string the C side copies. let ptr = unsafe { bindings::alloc_ordered_workqueue( - c_str!("asahi-g15-clear").as_char_ptr(), + c_str!("aurora-g15-clear").as_char_ptr(), bindings::wq_flags_WQ_MEM_RECLAIM, ) }; @@ -535,7 +535,7 @@ impl Claims { } } -/// Slot accounting statistics (`asahi-g15/stats`). +/// Slot accounting statistics (`aurora-g15/stats`). #[derive(Copy, Clone, Default)] pub(crate) struct SlotStats { /// Clustering buffers zeroed at claim, pass slots reset at claim, compute slots scrubbed diff --git a/drivers/gpu/drm/asahi/g16/engine.rs b/drivers/gpu/drm/asahi/g16/engine.rs index a1202afb25f0d..f53293578316f 100644 --- a/drivers/gpu/drm/asahi/g16/engine.rs +++ b/drivers/gpu/drm/asahi/g16/engine.rs @@ -241,7 +241,7 @@ enum PublicationError { Uncertain(Error), } -/// Engine statistics, printed by `asahi-g16/stats`. +/// Engine statistics, printed by `aurora-g16/stats`. #[derive(Default)] pub(crate) struct Stats { pub(crate) pub_ns: u64, @@ -294,7 +294,7 @@ pub(crate) struct Stats { } /// Host-side phase timers of publication and polling, on with `debug_flags` bit 49 -/// (`Debug1`); printed as the `prof:` line of `asahi-g16/stats`. A measurement aid: nothing is +/// (`Debug1`); printed as the `prof:` line of `aurora-g16/stats`. A measurement aid: nothing is /// timed while the flag is off. #[derive(Default)] pub(crate) struct Prof { @@ -411,7 +411,7 @@ pub(crate) struct Engine { pub(crate) stats: Stats, /// Phase timers (debug_flags bit 49). pub(crate) prof: Prof, - /// The `asahi-g16/stats` file. + /// The `aurora-g16/stats` file. pub(crate) debugfs: Option>>>>, scratch: Option>, } diff --git a/drivers/gpu/drm/asahi/g16/fence.rs b/drivers/gpu/drm/asahi/g16/fence.rs index afd89cc588a6a..01ba768b313b7 100644 --- a/drivers/gpu/drm/asahi/g16/fence.rs +++ b/drivers/gpu/drm/asahi/g16/fence.rs @@ -76,7 +76,7 @@ pub(crate) struct JobFenceOps; #[vtable] impl FenceOps for JobFenceOps { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("asahi") + c_str!("aurora") } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { @@ -107,11 +107,11 @@ pub(crate) struct Timeline { impl Timeline { pub(crate) fn new() -> Result> { - let errors = FenceContexts::new(1, c_str!("asahi_g16_refused"), ERROR_KEY)?; + let errors = FenceContexts::new(1, c_str!("aurora_g16_refused"), ERROR_KEY)?; let allocation_failure = error_fence(&errors, ENOMEM)?; Arc::pin_init( try_pin_init!(Self { - jobs: FenceContexts::new(1, c_str!("asahi_g16_queue"), TIMELINE_KEY)?, + jobs: FenceContexts::new(1, c_str!("aurora_g16_queue"), TIMELINE_KEY)?, errors, pending <- new_mutex!(KVec::new(), "g16::Timeline::pending"), allocation_failure, diff --git a/drivers/gpu/drm/asahi/g16/glue.c b/drivers/gpu/drm/asahi/g16/glue.c index 0160187a0dd2c..db0576a1e0fe0 100644 --- a/drivers/gpu/drm/asahi/g16/glue.c +++ b/drivers/gpu/drm/asahi/g16/glue.c @@ -101,7 +101,7 @@ struct asahi_g16_cooling *asahi_g16_cooling_register(struct device *dev) goto put_node; } cooling->cdev = thermal_of_cooling_device_register(dev->of_node, 0, - "asahi-g16g", cooling, + "aurora-g16g", cooling, &asahi_g16_cooling_ops); if (IS_ERR(cooling->cdev)) { ret = PTR_ERR(cooling->cdev); diff --git a/drivers/gpu/drm/asahi/g16/job.rs b/drivers/gpu/drm/asahi/g16/job.rs index 9787907e68c0d..31f06c80ab8bf 100644 --- a/drivers/gpu/drm/asahi/g16/job.rs +++ b/drivers/gpu/drm/asahi/g16/job.rs @@ -471,7 +471,7 @@ impl Scheduler { hw::sizes::SCHEDULER_CREDITS, 0, hw::tuning::SCHED_TIMEOUT_MS, - c_str!("asahi_g16"), + c_str!("aurora_g16"), )?; // With one run queue, only the kernel priority is valid. let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; diff --git a/drivers/gpu/drm/asahi/g16/mod.rs b/drivers/gpu/drm/asahi/g16/mod.rs index ef1e6f360db31..2c10fedd1e0be 100644 --- a/drivers/gpu/drm/asahi/g16/mod.rs +++ b/drivers/gpu/drm/asahi/g16/mod.rs @@ -91,7 +91,7 @@ use crate::driver::{AsahiDrmDevice, AsahiPlatformDriver}; use crate::{alloc, gem, gpu, mmu, regs as shared_regs}; /// Name of the G16 platform driver. -pub(crate) const DRIVER_NAME: &CStr = c_str!("asahi-g16"); +pub(crate) const DRIVER_NAME: &CStr = c_str!("aurora-g16"); /// GPU generation reported to userspace. pub(crate) const GPU_GENERATION: u32 = 16; @@ -398,7 +398,7 @@ impl Gpu { *shared.state.lock() = Some(firmware); // The engine's statistics file (a debugfs failure does not fail the probe). if let Err(error) = shared.start_engine() { - dev_warn!(dev.as_ref(), "G16: no asahi-g16 debugfs ({:?})\n", error); + dev_warn!(dev.as_ref(), "G16: no aurora-g16 debugfs ({:?})\n", error); } let gpu = Arc::new( Gpu { diff --git a/drivers/gpu/drm/asahi/g16/poll.rs b/drivers/gpu/drm/asahi/g16/poll.rs index 018c335a930b0..b94fb3a4cd36f 100644 --- a/drivers/gpu/drm/asahi/g16/poll.rs +++ b/drivers/gpu/drm/asahi/g16/poll.rs @@ -56,7 +56,7 @@ impl HrTimerCallback for Shared { } } -/// The `asahi-g16/stats` debugfs file contents. +/// The `aurora-g16/stats` debugfs file contents. fn stats_show(shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { let state = shared.state.lock(); match state.as_deref() { @@ -177,9 +177,9 @@ impl Shared { self.mod_poll(1); } - /// Starts what the engine needs besides the firmware state: the `asahi-g16/stats` file. + /// Starts what the engine needs besides the firmware state: the `aurora-g16/stats` file. pub(crate) fn start_engine(self: &Arc) -> Result { - let dir = debugfs::Dir::new(c_str!("asahi-g16")); + let dir = debugfs::Dir::new(c_str!("aurora-g16")); let file = KBox::pin_init( dir.read_callback_file(c_str!("stats"), self.clone(), &stats_show), GFP_KERNEL, @@ -834,7 +834,7 @@ impl Firmware { } } - /// The `asahi-g16/stats` contents. Scripts parse the keys: keep them stable. + /// The `aurora-g16/stats` contents. Scripts parse the keys: keep them stable. pub(crate) fn write_stats(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let s = &self.engine.stats; let slots = &self.slots.stats; diff --git a/drivers/gpu/drm/asahi/g17/availability.rs b/drivers/gpu/drm/asahi/g17/availability.rs index 8014266f0b0ab..c76a20db7007a 100644 --- a/drivers/gpu/drm/asahi/g17/availability.rs +++ b/drivers/gpu/drm/asahi/g17/availability.rs @@ -40,7 +40,7 @@ impl Availability { State { generation: 0, waiters: None }, "g17::Availability" ), - fences: FenceContexts::new(1, c_str!("asahi_resource"), kernel::static_lock_class!())?, + fences: FenceContexts::new(1, c_str!("aurora_resource"), kernel::static_lock_class!())?, }), GFP_KERNEL, ) diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 81d1d6fe43960..71107371640bd 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -32,7 +32,7 @@ fn signal_exported(fence: &dyn RawDmaFence) { #[vtable] impl FenceOps for CompletionFence { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("asahi") + c_str!("aurora") } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { @@ -69,7 +69,7 @@ static EXPORT_OPS: AtomicUsize = AtomicUsize::new(0); #[vtable] impl FenceOps for ExportFence { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("asahi") + c_str!("aurora") } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("g17-aggregate") diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index ad9adcecf0f81..76cc8809358f6 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -418,8 +418,8 @@ impl Fences { impl Scheduler { pub(crate) fn new(dev: &AsahiDrmDevice, backend: Arc, engine: Engine) -> Result { let (credits, name) = match engine { - Engine::Render => (RENDER_CREDITS, c_str!("asahi_g17_render")), - Engine::Compute => (COMPUTE_CREDITS, c_str!("asahi_g17_compute")), + Engine::Render => (RENDER_CREDITS, c_str!("aurora_g17_render")), + Engine::Compute => (COMPUTE_CREDITS, c_str!("aurora_g17_compute")), }; let scheduler = sched::Scheduler::new(dev.as_ref().as_ref(), 1, credits, 0, TIMEOUT_MS, name)?; let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 2a0a9c0a63222..cc3b1297f0467 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1256,7 +1256,7 @@ impl Gpu { GFP_KERNEL, )?; - let directory = kernel::debugfs::Dir::new(c"asahi-g17"); + let directory = kernel::debugfs::Dir::new(c"aurora-g17"); let capacity = KBox::pin_init( directory.read_only_file(c"capacity", capacity::View(shared.clone())), GFP_KERNEL, diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index e593c5879cf74..ab6b6a55f93e8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -69,7 +69,7 @@ impl Frontend { return Err(EINVAL); } let window = UscWindow::new(usc_base, range.clone())?; - let fences = FenceContexts::new(1, c_str!("asahi_g17_queue"), FENCE_KEY)?; + let fences = FenceContexts::new(1, c_str!("aurora_g17_queue"), FENCE_KEY)?; let allocation_failure = super::fence::allocation_failure(&fences)?; Ok(Self { fences, diff --git a/drivers/gpu/drm/asahi/g17/sync_scope.c b/drivers/gpu/drm/asahi/g17/sync_scope.c index a3153f3122b94..73dc096d119c9 100644 --- a/drivers/gpu/drm/asahi/g17/sync_scope.c +++ b/drivers/gpu/drm/asahi/g17/sync_scope.c @@ -57,7 +57,7 @@ static struct asahi_g17_scope *to_scope(struct dma_fence *fence) static const char *scope_driver_name(struct dma_fence *fence) { - return "asahi"; + return "aurora"; } static const char *scope_timeline_name(struct dma_fence *fence) diff --git a/drivers/gpu/drm/asahi/memacct.rs b/drivers/gpu/drm/asahi/memacct.rs index b7c996d0c6f6e..12fc90b23d8eb 100644 --- a/drivers/gpu/drm/asahi/memacct.rs +++ b/drivers/gpu/drm/asahi/memacct.rs @@ -131,7 +131,7 @@ fn try_add_within(counter: &AtomicU64, bytes: u64, limit: u64, extra: u64) -> bo fn sub(counter: &AtomicU64, bytes: u64) { let prev = counter.fetch_sub(bytes, Ordering::AcqRel); if prev < bytes { - pr_warn!("asahi: GPU memory accounting underflow ({} < {})\n", prev, bytes); + pr_warn!("GPU memory accounting underflow ({} < {})\n", prev, bytes); counter.store(0, Ordering::Relaxed); } } @@ -162,7 +162,7 @@ impl Ledger { ); self.set_limits(limits); pr_info!( - "asahi: GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", + "GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", limits.device >> 20, limits.client >> 20 ); diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 49c2241bc399f..1df852184b31d 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -266,7 +266,7 @@ impl VmShrinker { )?; // SAFETY: The format is static and has no format arguments. let raw = - unsafe { kernel::bindings::shrinker_alloc(0, c_str!("asahi-idle-vm").as_char_ptr()) }; + unsafe { kernel::bindings::shrinker_alloc(0, c_str!("aurora-idle-vm").as_char_ptr()) }; if raw.is_null() { return Err(ENOMEM); } diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 93c4eb0351187..aa5dc1fba4723 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -194,7 +194,7 @@ impl JobFence::ver { #[vtable] impl dma_fence::FenceOps for JobFence::ver { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("asahi") + c_str!("aurora") } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("queue") @@ -468,7 +468,7 @@ impl Drop for QueueJob::ver { } } -static QUEUE_NAME: &CStr = c_str!("asahi_fence"); +static QUEUE_NAME: &CStr = c_str!("aurora_fence"); static QUEUE_CLASS_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); #[versions(AGX)] @@ -526,7 +526,7 @@ impl Queue::ver { WQ_SIZE, 0, 100000, - c_str!("asahi_sched"), + c_str!("aurora_sched"), )?; let entity = sched::Entity::new(&sched, sched::Priority::Kernel)?; diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index a704e3f1f7afd..7132e6de41c09 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -316,4 +316,4 @@ EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); MODULE_AUTHOR("AGX driver maintainer"); MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("Asahi AGX firmware stats sysfs shim"); +MODULE_DESCRIPTION("AGX firmware stats sysfs shim"); diff --git a/tools/asahi/g15-probe-init-tests.py b/tools/asahi/g15-probe-init-tests.py index 9d6ec7c0096eb..65cc9c0b69c9f 100644 --- a/tools/asahi/g15-probe-init-tests.py +++ b/tools/asahi/g15-probe-init-tests.py @@ -160,8 +160,8 @@ def item(text, marker): try_pin_init!(Self { stage: { let stage = match name.to_bytes() { - b"asahi-g15" => 1, - b"asahi-g16" => 3, + b"aurora-g15" => 1, + b"aurora-g16" => 3, _ => 2, }; // Model driver_attach and a late driver_add_groups failure. A deferred @@ -338,7 +338,7 @@ def main(): g16_registration = aliases + "\n" + item(core, "fn register_g16(") module_struct = item(core, "#[pin_data]\nstruct DriverModule") module_init = item(core, " fn init(module:") - module_code = module_struct + "\nimpl InPlaceModule for DriverModule {\n" + module_init + "\n}\nimpl ModuleMetadata for DriverModule { const NAME: &'static std::ffi::CStr = c\"asahi\"; }\n" + module_code = module_struct + "\nimpl InPlaceModule for DriverModule {\n" + module_init + "\n}\nimpl ModuleMetadata for DriverModule { const NAME: &'static std::ffi::CStr = c\"aurora\"; }\n" kbox = item(sources["rust/kernel/alloc/kbox.rs"], " fn try_pin_init(") assert "Box::<_, A>::new_uninit(flags)?.write_pin_init(init)" in kbox reprobe = item(sources["drivers/gpu/drm/asahi/g15/glue.c"], "void asahi_g15_reprobe(void)\n{") @@ -408,7 +408,7 @@ def main(): if name == "activation-before-g16-registration" and not enable_g16: continue # Macro definitions precede the fake registration, just as kernel prelude exports them. - source = macro + "\n" + PREFIX.replace("#![allow(dead_code, unused_variables)]\n", "").replace(" // SET_DRVDATA", set_data) + "\n" + ready + "\n" + driver + "\nimpl platform::Driver for Driver<'_> {\n type IdInfo = &'static Config;\n type Data<'bound> = Driver<'bound>;\n" + method + "\n}\nmod g15 { use super::*; pub type Driver<'a> = super::Driver<'a>; " + activate + "\n}\ntype G15Registration = driver::Registration>>;\n" + "mod g16 { pub struct Driver; pub const DRIVER_NAME: &std::ffi::CStr = c\"asahi-g16\"; }\n" + g16_registration + "\n" + module_variant + "\n" + SUFFIX.replace(" // CALLBACK", callback) + source = macro + "\n" + PREFIX.replace("#![allow(dead_code, unused_variables)]\n", "").replace(" // SET_DRVDATA", set_data) + "\n" + ready + "\n" + driver + "\nimpl platform::Driver for Driver<'_> {\n type IdInfo = &'static Config;\n type Data<'bound> = Driver<'bound>;\n" + method + "\n}\nmod g15 { use super::*; pub type Driver<'a> = super::Driver<'a>; " + activate + "\n}\ntype G15Registration = driver::Registration>>;\n" + "mod g16 { pub struct Driver; pub const DRIVER_NAME: &std::ffi::CStr = c\"aurora-g16\"; }\n" + g16_registration + "\n" + module_variant + "\n" + SUFFIX.replace(" // CALLBACK", callback) source = "#![allow(dead_code, unused_variables, private_bounds, private_interfaces)]\n" + source label = ("g16-" if enable_g16 else "") + name src = out / (label + ".rs"); src.write_text(source) diff --git a/tools/asahi/g17-capacity-watch.sh b/tools/asahi/g17-capacity-watch.sh index 5438413b2ec21..f3d203e506833 100644 --- a/tools/asahi/g17-capacity-watch.sh +++ b/tools/asahi/g17-capacity-watch.sh @@ -2,7 +2,7 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT # Run inside an already authorized test session. Does not mount debugfs. set -eu -capacity_file=${1:-/sys/kernel/debug/asahi-g17/capacity} +capacity_file=${1:-/sys/kernel/debug/aurora-g17/capacity} interval=${2:-5} case "$interval" in ''|*[!0-9]*|0) echo 'interval must be positive integer seconds' >&2; exit 2;; esac test -r "$capacity_file" diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/README b/tools/testing/selftests/drivers/gpu/asahi-m4/README index c88abd122428e..040a90a056ef6 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/README +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/README @@ -64,7 +64,7 @@ paired native Mesa installation: make hardware MESA=/path/to/installed/mesa RENDER_NODE=/dev/dri/renderD128 ROUNDS=32 -This builds native-gl46.c, verifies asahi-g16 owns the render node, selects the +This builds native-gl46.c, verifies aurora-g16 (asahi-g16 on older kernels) owns the render node, selects the requested Mesa libraries, clears software/translation/API-version overrides, and runs native OpenGL 4.6 triangle readback, compute and geometry-feedback checks. Results include kernel/boot identity and Mesa/binary hashes. Ordinary diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/native-ioctl-regression.c b/tools/testing/selftests/drivers/gpu/asahi-m4/native-ioctl-regression.c index 5f19e9f6eef97..aa324b7976525 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/native-ioctl-regression.c +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/native-ioctl-regression.c @@ -170,7 +170,10 @@ struct compute_stats { static int compute_stats(struct compute_stats *s) { - FILE *file = fopen("/sys/kernel/debug/asahi-g16/stats", "re"); + /* Kernels before the Aurora rename used "asahi-g16". */ + FILE *file = fopen("/sys/kernel/debug/aurora-g16/stats", "re"); + if (!file) + file = fopen("/sys/kernel/debug/asahi-g16/stats", "re"); char *line = NULL; size_t size = 0; bool compute = false, mixed = false, publications = false; @@ -410,7 +413,7 @@ static int identity(const char *release, const char *boot, const char *source, c if (n < 0) return -1; target[n] = 0; const char *owner = strrchr(target, '/'); - if (!owner || strcmp(owner + 1, "asahi-g16") || + if (!owner || (strcmp(owner + 1, "aurora-g16") && strcmp(owner + 1, "asahi-g16")) || access("/sys/module/gpu_m4_drm", F_OK) == 0 || access("/sys/module/apple_gpu_m4_power", F_OK) == 0) return -1; return 0; diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/run-hardware-tests.py b/tools/testing/selftests/drivers/gpu/asahi-m4/run-hardware-tests.py index b2f01f06cefc8..d9b09d5820a04 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/run-hardware-tests.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/run-hardware-tests.py @@ -31,8 +31,8 @@ def main(): node = args.render_node.resolve(strict=True) device = (Path('/sys/class/drm') / node.name / 'device').resolve(strict=True) owners = [p / 'driver' for p in [device, *device.parents] if (p / 'driver').is_symlink()] - if not any(p.resolve().name == 'asahi-g16' for p in owners): - parser.error('Selected render node is not owned by the Rust asahi-g16 platform driver') + if not any(p.resolve().name in ('aurora-g16', 'asahi-g16') for p in owners): + parser.error('Selected render node is not owned by the Rust G16 platform driver (aurora-g16)') output = (args.output or Path(tempfile.mkdtemp(prefix='asahi-m4-hardware-'))).resolve() output.mkdir(parents=True, exist_ok=True) environment = os.environ.copy() diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/run-native-ioctl-regression.py b/tools/testing/selftests/drivers/gpu/asahi-m4/run-native-ioctl-regression.py index 06b71eb83a90c..0932dc9f13fe5 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/run-native-ioctl-regression.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/run-native-ioctl-regression.py @@ -20,7 +20,11 @@ UNPATCHED = {'78d2519c82fca4c0403315e040d595628bc5a4d6', 'df583754ae2b3607af4155ba0442bc93077bdce8'} FAULT = re.compile(r'Kernel panic|BUG:|WARNING: CPU|SError|Oops:|Unable to handle kernel|' - r'\b(?:asahi|G16|GPU|PPL)\b.*(?:fault|fail(?:ed|ure)?|timeout|timed out|quarantin)', re.I) + r'\b(?:aurora|asahi|G16|GPU|PPL)\b.*(?:fault|fail(?:ed|ure)?|timeout|timed out|quarantin)', re.I) +# The G16 platform driver and its debugfs directory; kernels before the Aurora +# rename used the asahi-g16 name. +G16_DRIVERS = ('aurora-g16', 'asahi-g16') +G16_STATS = [Path('/sys/kernel/debug') / name / 'stats' for name in G16_DRIVERS] EXPECTED_REJECTION = re.compile(r'IOCTL: submit failed! \(submission ID: \d+ err: E2BIG\)\s*$') @@ -41,7 +45,8 @@ def self_test(): 'Kernel panic - not syncing: ' + expected, expected + ' GPU fault bank 0', expected.replace('err: E2BIG', 'err: EIO'), - 'asahi-g16: GPU fault bank 0; ' + expected] + 'asahi-g16: GPU fault bank 0; ' + expected, + 'aurora-g16: GPU fault bank 0; ' + expected] for line in harmless: assert not kernel_fault(line), 'False fault match: ' + line for line in failures: @@ -96,8 +101,9 @@ def preflight(args): if not stat.S_ISCHR(node.stat().st_mode): raise RuntimeError('Expected a DRM character device') device = Path('/sys/class/drm') / node.name / 'device' - if (device / 'driver').resolve(strict=True).name != 'asahi-g16': - raise RuntimeError('Rust asahi-g16 does not own the requested render node') + owner = (device / 'driver').resolve(strict=True).name + if owner not in G16_DRIVERS: + raise RuntimeError('The Rust G16 platform driver does not own the requested render node') if any((Path('/sys/module') / name).exists() for name in ['gpu_m4_drm', 'apple_gpu_m4_power']): raise RuntimeError('Legacy C GPU provider is loaded') active = root_json('/run/aurorasilicon/m4-asahi-rust.json') @@ -118,7 +124,7 @@ def preflight(args): raise RuntimeError('Qualification marker already exists; rollback supervisor may disarm') return {'boot_id': boot, 'source_commit': args.expected_source, 'kernel_release': os.uname().release, 'kernel_image_sha256': sha(image), 'asahi_module_sha256': sha(module), - 'source_state_sha256': sha(source_path), 'render_node': str(node), 'owner': 'asahi-g16', + 'source_state_sha256': sha(source_path), 'render_node': str(node), 'owner': owner, 'rollback_guard': guard, 'activation': active} @@ -149,7 +155,7 @@ def compile_helper(args, output): def snapshot(): - stats = Path('/sys/kernel/debug/asahi-g16/stats').read_text() + stats = next(p for p in G16_STATS if p.exists()).read_text() published = re.search(r'^publications: compute (\d+) render (\d+)\s*$', stats, re.M) if not published: raise RuntimeError('Missing actual firmware-producer publication counters; refuse live regression') From 6dd6ab1cf12f98ac09f4d83766550eec45da1dc6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:28:56 +1100 Subject: [PATCH 1499/1579] drm/apple: keep native chromaticities independent of optional gamma Report valid firmware primaries and white even when gamma is absent, leaving EDID gamma explicitly unknown. Preserve the external metadata path's stricter transfer-characteristic requirement. Do not infer absence of native coordinates from a zero optional gamma, or inherit a previous gamma when a new complete native description omits it. Exercise both cases in the synthetic panel colour regression. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/gpu/apple-dcp-panel-colorimetry.rst | 8 +++++--- drivers/gpu/drm/apple/dcp-display-metadata.c | 9 ++++++--- drivers/gpu/drm/apple/dcp.c | 2 +- drivers/gpu/drm/apple/parser.c | 12 +++++++----- .../selftests/drivers/gpu/apple-dcp/colorimetry.py | 13 +++++++++++-- 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/Documentation/gpu/apple-dcp-panel-colorimetry.rst b/Documentation/gpu/apple-dcp-panel-colorimetry.rst index e4b177f7ba2d3..ff31c645115c1 100644 --- a/Documentation/gpu/apple-dcp-panel-colorimetry.rst +++ b/Documentation/gpu/apple-dcp-panel-colorimetry.rst @@ -31,7 +31,8 @@ The existing paths in appledrm are: Before the optional-colour fix, the desktop parser retained only ``ProductAttributes`` and the panel builder unconditionally emitted zero RGB primaries, nominal D65 white and unknown gamma (0xff). The shared colour -encoder now validates and encodes a complete firmware description. Incomplete, +encoder now validates and encodes native primaries and white independently +of an optional gamma. Incomplete, duplicate or malformed optional fields leave panel identity usable. A standalone ``ProductAttributes`` update for the same panel retains its previous colour description. A different panel serial cannot inherit that description. @@ -43,11 +44,12 @@ The firmware fields to verify are precisely: * ``DisplayAttributes.Chromaticity.Red.X`` and ``.Y``; * ``DisplayAttributes.Chromaticity.Green.X`` and ``.Y``; * ``DisplayAttributes.Chromaticity.Blue.X`` and ``.Y``; -* ``DisplayAttributes.DefaultWhitePoint.X``, ``.Y`` and ``.Gamma``. +* ``DisplayAttributes.DefaultWhitePoint.X``, ``.Y`` and optional ``.Gamma``. The existing parser interprets these as unsigned 16.16 values. EDID base-block chromaticities round to units of 1/1024; gamma rounds to a hundredth. An absent -transfer characteristic must not be replaced by an invented panel gamma. +transfer characteristic remains unknown (0xff) even when valid primaries and +white can be published; it must not be replaced by an invented panel gamma. The optional values are validated before any colour bytes are changed, including nonzero/bounded coordinates, nondegenerate primary triangle and white-point containment. These are metadata validation rules, not panel diff --git a/drivers/gpu/drm/apple/dcp-display-metadata.c b/drivers/gpu/drm/apple/dcp-display-metadata.c index 94518eb9d0824..3be02ae5df3fa 100644 --- a/drivers/gpu/drm/apple/dcp-display-metadata.c +++ b/drivers/gpu/drm/apple/dcp-display-metadata.c @@ -94,7 +94,8 @@ static int dcp_metadata_color(struct edid *edid, const struct dcp_display_metada { u32 coordinate[8], gamma; - if (metadata->gamma < 65536 || metadata->gamma > (354U << 16) / 100) + if (metadata->gamma && + (metadata->gamma < 65536 || metadata->gamma > (354U << 16) / 100)) return -EINVAL; for (unsigned int i = 0; i < 8; i++) { u32 value = i < 6 ? metadata->chromaticity[i / 2][i % 2] : metadata->white[i - 6]; @@ -121,8 +122,9 @@ static int dcp_metadata_color(struct edid *edid, const struct dcp_display_metada return -EINVAL; } } - gamma = DIV_ROUND_CLOSEST(metadata->gamma * 100, 65536) - 100; - if (gamma > 254) + gamma = metadata->gamma ? + DIV_ROUND_CLOSEST(metadata->gamma * 100, 65536) - 100 : 255; + if (metadata->gamma && gamma > 254) return -EINVAL; edid->gamma = gamma; @@ -153,6 +155,7 @@ static int dcp_metadata_block(struct edid *edid, const struct dcp_display_metada !metadata->height_cm || metadata->height_cm > 255 || metadata->product > U16_MAX || metadata->week > 53 || metadata->year < 1990 || metadata->year > 2245 || + metadata->gamma < 65536 || metadata->gamma > (354U << 16) / 100 || strnlen(metadata->manufacturer, 4) != 3) return -EINVAL; for (unsigned int i = 0; i < 3; i++) { diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 698df18df9400..01485347116bf 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -910,7 +910,7 @@ void dcp_panel_metadata_replace(struct apple_dcp *dcp, if (same_serial && !merged.serial_text[0]) memcpy(merged.serial_text, old->serial_text, sizeof(merged.serial_text)); /* Standalone ProductAttributes setters omit the colour description. */ - if (same_serial && !merged.gamma) { + if (same_serial && !merged.chromaticity[0][0] && !merged.white[0]) { memcpy(merged.chromaticity, old->chromaticity, sizeof(merged.chromaticity)); memcpy(merged.white, old->white, sizeof(merged.white)); merged.gamma = old->gamma; diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 0db027145b8f8..fff8b3465c60c 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -1380,7 +1380,8 @@ static int parse_metadata_number(struct dcp_parse_ctx *handle, u32 *out, u32 min return ret; } -static int parse_metadata_point(struct dcp_parse_ctx *handle, u32 point[2], u32 *gamma) +static int parse_metadata_point(struct dcp_parse_ctx *handle, u32 point[2], u32 *gamma, + bool optional_gamma) { struct iterator it; unsigned int seen = 0; @@ -1419,7 +1420,8 @@ static int parse_metadata_point(struct dcp_parse_ctx *handle, u32 point[2], u32 * A white point is a chromaticity. A primary is passed on as the * sink reports it: real EDIDs carry primaries with x + y above 1. */ - if (seen != (gamma ? 7 : 3) || (gamma && point[0] + point[1] > 65536)) + if ((seen & 3) != 3 || (gamma && !optional_gamma && !(seen & 4)) || + (gamma && point[0] + point[1] > 65536)) return -EINVAL; return 0; } @@ -1448,7 +1450,7 @@ static int parse_metadata_chromaticity(struct dcp_parse_ctx *handle, if (seen & 1U << color) return -EINVAL; seen |= 1U << color; - ret = parse_metadata_point(handle, out->chromaticity[color], NULL); + ret = parse_metadata_point(handle, out->chromaticity[color], NULL, false); } else { ret = skip(handle); } @@ -1667,7 +1669,7 @@ int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_meta if (seen & 8) return -EINVAL; seen |= 8; - ret = parse_metadata_point(handle, parsed.white, &parsed.gamma); + ret = parse_metadata_point(handle, parsed.white, &parsed.gamma, false); } else if (!strcmp(key, "ProductAttributes")) { if (seen & 16) return -EINVAL; @@ -1737,7 +1739,7 @@ int parse_panel_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metada } else if (!strcmp(key, "DefaultWhitePoint")) { struct dcp_parse_ctx color = *handle; - white = !parse_metadata_point(&color, parsed.white, &parsed.gamma) && + white = !parse_metadata_point(&color, parsed.white, &parsed.gamma, true) && !(color_seen & 2); color_seen |= 2; ret = skip(handle); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/colorimetry.py b/tools/testing/selftests/drivers/gpu/apple-dcp/colorimetry.py index 0d13f17040ea4..f4cd307926988 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/colorimetry.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/colorimetry.py @@ -64,7 +64,7 @@ def panel(root, output): u32 coordinate=(u32)raw[27+i]<<2 | ((low>>(6-2*(i%4)))&3); assert(coordinate==(expected[i]+32)/64); } - assert(raw[23]==120); + assert(raw[23]==(metadata->gamma?120:255)); } else { assert(raw[23]==255); for(unsigned i=27;i<33;i++)assert(!raw[i]); @@ -83,6 +83,9 @@ def panel(root, output): check(&p3,true,argv[1]);check(&srgb,true,argv[2]); ''' for i in range(2, len(inputs)): + if i == 6: + unit += f'{{struct dcp_display_metadata m=decode(color_{i},sizeof(color_{i}));assert(!m.gamma && m.white[0] && m.chromaticity[0][0]);check(&m,true,NULL);}}\n' + continue unit += f'{{struct dcp_display_metadata m=decode(color_{i},sizeof(color_{i}));assert(!m.gamma && !m.white[0] && !m.chromaticity[0][0]);check(&m,false,NULL);}}\n' unit += r''' struct dcp_display_metadata invalid=p3; @@ -97,10 +100,16 @@ def panel(root, output): struct dcp_display_metadata partial=decode(color_2,sizeof(color_2)); dcp_panel_metadata_replace(&d,&partial); assert(notifications==1 && !memcmp(&state.panel_metadata,&p3,sizeof(p3))); + /* Newly reported primaries with unknown gamma must not inherit an old gamma. */ + struct dcp_display_metadata unknown_gamma=decode(color_6,sizeof(color_6)); + dcp_panel_metadata_replace(&d,&unknown_gamma); + assert(notifications==2 && !state.panel_metadata.gamma && state.panel_metadata.white[0]); + dcp_panel_metadata_replace(&d,&partial); + assert(notifications==2 && !state.panel_metadata.gamma && state.panel_metadata.white[0]); /* A replacement panel must not inherit another serial's colour data. */ partial.serial++; dcp_panel_metadata_replace(&d,&partial); - assert(notifications==2 && !state.panel_metadata.gamma); + assert(notifications==3 && !state.panel_metadata.gamma); return 0; } ''' From f6f6e2979142229af745816ea275997c85918ba0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:29:22 +1100 Subject: [PATCH 1500/1579] media: apple: ave: release initial power-domain supplier references AVE attaches its five power domains with PD_FLAG_DEV_LINK_ON, which adds active runtime-PM supplier references. The encoder itself starts suspended and boots firmware only for a session. Enabling runtime PM without reconciling this initial state leaves those references held until the first session has resumed and suspended the encoder. Set the suspended state explicitly while runtime PM is still disabled. The PM core releases the supplier link references even when the consumer was already suspended. Keep the existing runtime callbacks, autosuspend delay and session power lifecycle. Compile the production probe in a host regression harness covering a never-opened device, immediate registration-time use, repeated cycles and probe failures. Removing the fix must fail the idle-supplier check. Fixes: 3b0dd96e31de ("media: apple: ave: turn the J700 encoder transport into a V4L2 encoder") Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/media/platform/apple/ave/ave-drv.c | 10 ++ tools/asahi/ave-probe-pm-tests.py | 190 +++++++++++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 tools/asahi/ave-probe-pm-tests.py diff --git a/MAINTAINERS b/MAINTAINERS index c053af5b9269f..89d27cd0b0ddf 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2817,6 +2817,7 @@ W: https://aurora-silicon.github.io F: Documentation/admin-guide/media/apple-ave.rst F: Documentation/devicetree/bindings/media/apple,t8140-ave.yaml F: drivers/media/platform/apple/ave/ +F: tools/asahi/ave*-tests.py ARM/APPLE QSPIMC DRIVER M: Ryan Murray diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c index 43c9ddb089e83..a318ab6b25bcc 100644 --- a/drivers/media/platform/apple/ave/ave-drv.c +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -156,6 +156,16 @@ static int ave_probe(struct platform_device *pdev) if (ret < 0) return dev_err_probe(dev, ret, "cannot attach power domains\n"); + /* + * Domain attachment took active supplier references, but the driver + * starts firmware only on first use and begins suspended. Reconcile that + * state before enabling runtime PM, otherwise those references survive + * until the first session has resumed and suspended the encoder. + */ + ret = pm_runtime_set_suspended(dev); + if (ret) + return ret; + pm_runtime_set_autosuspend_delay(dev, AVE_AUTOSUSPEND_MS); pm_runtime_use_autosuspend(dev); ret = devm_pm_runtime_enable(dev); diff --git a/tools/asahi/ave-probe-pm-tests.py b/tools/asahi/ave-probe-pm-tests.py new file mode 100644 index 0000000000000..dcb9e02de2115 --- /dev/null +++ b/tools/asahi/ave-probe-pm-tests.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Compile the AVE probe unchanged against a supplier-reference model. + +Exercise never-opened idle, immediate registration-time use and probe errors. +The model follows runtime.c: setting SUSPENDED drops supplier references even +when the consumer status was already SUSPENDED. This tests probe ordering and +reference ownership, not hardware power gating or firmware restart. +""" +import argparse +import os +from pathlib import Path +import resource +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = 'drivers/media/platform/apple/ave/ave-drv.c' + +PREFIX = r''' +#include +#include +#include +#include +#include +#include +struct ave_dev; +struct device { bool enabled, suspended; unsigned supplier_refs; struct ave_dev *data; }; +struct platform_device { struct device dev; }; +struct iommu_domain { unsigned type; }; +struct ave_dev { struct device *dev; const void *variant; int hw_lock; + void *codec, *mbox, *coproc, *pd_list; struct iommu_domain *domain; }; +struct dev_pm_domain_attach_data { + const char *const *pd_names; unsigned num_pd_names, pd_flags; +}; +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#define GFP_KERNEL 0 +#define __IOMMU_DOMAIN_PAGING 1 +#define PD_FLAG_DEV_LINK_ON 1 +#define AVE_AUTOSUSPEND_MS 2000 +#define DMA_BIT_MASK(x) ((UINT64_C(1) << (x)) - 1) +#define IS_ERR(x) ((uintptr_t)(x) >= (uintptr_t)-4095) +#define PTR_ERR(x) ((int)(intptr_t)(x)) +static const char *const ave_pd_names[] = {"dma", "pipe4", "pipe5", "me0", "me1"}; +static struct ave_dev context; +static struct iommu_domain domain = {.type = __IOMMU_DOMAIN_PAGING}; +static int attach_error, state_error, enable_error, fw_error, register_error; +static unsigned registrations, fw_probes, fw_removes, state_updates; +static void *devm_kzalloc(struct device *d, size_t n, int f) +{ (void)d; (void)f; assert(n == sizeof(context)); return &context; } +static const void *of_device_get_match_data(struct device *d) { return d; } +static void mutex_init(int *m) { *m = 0; } +static void platform_set_drvdata(struct platform_device *p, void *v) { p->dev.data = v; } +static void *devm_platform_ioremap_resource_byname(struct platform_device *p, const char *n) +{ (void)p; return (void *)n; } +static struct iommu_domain *iommu_get_domain_for_dev(struct device *d) +{ (void)d; return &domain; } +static int dma_set_mask_and_coherent(struct device *d, uint64_t mask) +{ (void)d; assert(mask == DMA_BIT_MASK(42)); return 0; } +#define dev_err_probe(d, e, ...) (e) +static int devm_pm_domain_attach_list(struct device *d, + const struct dev_pm_domain_attach_data *data, void **list) +{ + assert(!d->enabled && d->suspended); + assert(data->num_pd_names == 5 && data->pd_flags == PD_FLAG_DEV_LINK_ON); + assert(!*list); + if (attach_error) return attach_error; + /* Five virtual devices plus the two preactivated DART supplier links. */ + d->supplier_refs = 7; + *list = d; + return 5; +} +static int pm_runtime_set_suspended(struct device *d) +{ + assert(!d->enabled); + state_updates++; + if (state_error) return state_error; + d->suspended = true; + d->supplier_refs = 0; + return 0; +} +static void pm_runtime_set_autosuspend_delay(struct device *d, unsigned delay) +{ (void)d; assert(delay == 2000); } +static void pm_runtime_use_autosuspend(struct device *d) { (void)d; } +static int devm_pm_runtime_enable(struct device *d) +{ + assert(!d->enabled); + if (enable_error) return enable_error; + d->enabled = true; + return 0; +} +static int ave_fw_probe(struct ave_dev *a) { (void)a; fw_probes++; return fw_error; } +static void ave_fw_remove(struct ave_dev *a) { (void)a; fw_removes++; } +static int ave_v4l2_register(struct ave_dev *a) +{ + struct device *d = a->dev; + registrations++; + assert(d->enabled && d->suspended && d->supplier_refs == 0); + if (register_error) return register_error; + /* A client can open immediately once its video device is published. */ + for (unsigned n = 0; n < 10; n++) { + d->suspended = false; + d->supplier_refs = 7; + assert(d->enabled && !d->suspended); + d->supplier_refs = 0; + d->suspended = true; + } + return 0; +} +static void ave_debugfs_init(struct ave_dev *a) { (void)a; } +''' + +CASES = r''' +static void reset(struct platform_device *p) +{ + *p = (struct platform_device){.dev.suspended = true}; + context = (struct ave_dev){0}; + attach_error = state_error = enable_error = fw_error = register_error = 0; + registrations = fw_probes = fw_removes = state_updates = 0; +} +int main(void) +{ + struct platform_device p; + reset(&p); + assert(ave_probe(&p) == 0); + assert(state_updates == 1 && registrations == 1 && fw_probes == 1); + assert(p.dev.suspended && !p.dev.supplier_refs); + reset(&p); + attach_error = -EIO; + assert(ave_probe(&p) == -EIO); + assert(!state_updates && !p.dev.enabled && !registrations); + reset(&p); + state_error = -EBUSY; + assert(ave_probe(&p) == -EBUSY); + assert(!p.dev.enabled && !fw_probes && !registrations); + reset(&p); + enable_error = -ENOMEM; + assert(ave_probe(&p) == -ENOMEM); + assert(!p.dev.supplier_refs && !fw_probes && !registrations); + reset(&p); + fw_error = -EINVAL; + assert(ave_probe(&p) == -EINVAL); + assert(!p.dev.supplier_refs && !registrations); + reset(&p); + register_error = -EIO; + assert(ave_probe(&p) == -EIO); + assert(!p.dev.supplier_refs && fw_removes == 1); + puts("AVE probe PM: idle suppliers, immediate use, 10 cycles and 5 error paths passed"); +} +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source', type=Path) + parser.add_argument('--out', type=Path) + parser.add_argument('--negative-control', action='store_true') + args = parser.parse_args() + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + source = (args.source or ROOT / SOURCE).read_text() + start = source.index('static int ave_probe(') + probe = source[start:source.index('\n}', start) + 2] + if args.negative_control: + old = '\tret = pm_runtime_set_suspended(dev);\n\tif (ret)\n\t\treturn ret;\n' + if probe.count(old) != 1: + raise SystemExit('cannot locate the production fix for the negative control') + probe = probe.replace(old, '') + with tempfile.TemporaryDirectory(prefix='ave-probe-pm-') as tmp: + out = args.out or Path(tmp) + out.mkdir(parents=True, exist_ok=True) + c = out / 'ave-probe-pm.c' + c.write_text(PREFIX + probe + CASES) + exe = out / 'ave-probe-pm' + subprocess.run([os.environ.get('CC', 'cc'), '-std=c11', '-Wall', '-Wextra', + '-Werror', '-Wno-unused-function', '-fsanitize=address,undefined', + '-fno-omit-frame-pointer', str(c), '-o', str(exe)], check=True) + run = subprocess.run([str(exe)], capture_output=True, text=True) + (out / 'run.log').write_text(run.stdout + run.stderr) + if args.negative_control: + if run.returncode != -6 or 'supplier_refs == 0' not in run.stderr: + raise SystemExit('pre-fix probe did not fail the idle-supplier assertion') + print('AVE probe PM: pre-fix probe compiled and failed the idle-supplier assertion') + elif run.returncode: + raise SystemExit(run.stdout + run.stderr) + else: + print(run.stdout, end='') + + +if __name__ == '__main__': + main() From 49862608f3e1195c650fbfa25fc2c757f48dba14 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:31:53 +1100 Subject: [PATCH 1501/1579] accel/ane: prepare H16 UUID submission and validate original HWX Track pending execution UUIDs before channel-7 publication and retire only matching terminal channel-6 events. Keep one deadline across acknowledgment and completion, order coherent DMA accesses, and quarantine failed or uncertain requests without exposing private output mappings. Validate complete HWX containers, tensor and program descriptors, aperture records, symbols and payload-only kernel relocations. Match execute buffers to kernel-owned process resources and revoke per-file authority on close. Add hardware-free KUnit coverage for validation and pending request paths. Keep H16 registration and section-only loading refused until preload, startup, the original-container ABI and process-resource decoding exist. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 96 ++- drivers/accel/ane/Kconfig | 11 +- drivers/accel/ane/Makefile | 3 +- drivers/accel/ane/ane_fw_platform.c | 7 +- drivers/accel/ane/ane_h16_hwx.c | 619 ++++++++++++++++++++ drivers/accel/ane/ane_h16_hwx.h | 55 ++ drivers/accel/ane/ane_h16_hwx_test.c | 427 ++++++++++++++ drivers/accel/ane/ane_h16_pending.h | 143 +++++ drivers/accel/ane/ane_h16_pending_test.c | 325 ++++++++++ drivers/accel/ane/ane_h16_resources.h | 56 ++ drivers/accel/ane/ane_t6021_rtclient_main.c | 274 ++++++++- 11 files changed, 1963 insertions(+), 53 deletions(-) create mode 100644 drivers/accel/ane/ane_h16_hwx.c create mode 100644 drivers/accel/ane/ane_h16_hwx.h create mode 100644 drivers/accel/ane/ane_h16_hwx_test.c create mode 100644 drivers/accel/ane/ane_h16_pending.h create mode 100644 drivers/accel/ane/ane_h16_pending_test.c create mode 100644 drivers/accel/ane/ane_h16_resources.h diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index ef546815b85eb..c52b141dd21dc 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -34,9 +34,9 @@ H16 preflight checks resource geometry, firmware identity and these ranges without MMIO or power access. No disk firmware is requested. Missing reservations fail with ``-ENODATA``. Malformed reservations fail with ``-EINVAL``. Even a valid handoff currently fails with ``-EOPNOTSUPP``: -the strict preload validator, original-container HWX ABI and cold boot -backend are incomplete. The section-only load ABI is explicitly refused -for H16. No accel node or program-loading interface is published. +the strict preload validator, original-container HWX ABI, firmware process +resource decoder and cold boot backend are incomplete. The section-only +load ABI is explicitly refused for H16. No accel node or program-loading interface is published. The H16 DT nodes stay disabled. The five real PMGR providers already encode the system/CPU/MPM/task/base dependency graph. The PMGR binding does not @@ -65,8 +65,8 @@ H16 validation helpers walk nine fixed task-header words, the documented conditional extra word, dense/scatter records and the nine allowed register blocks. A separate helper checks the eight-byte Mach-O relocation entries, unique aligned offsets, the actual kernel-section ordinal and addend bounds. -These helpers do not substitute for complete container/tensor/program and -16-byte task-aperture metadata validation. +Original-container, tensor, program and aperture validation is provided +separately as described below. The device-free selftest in ``tools/testing/selftests/drivers/ane`` generates the SPEC-r2 fp16 add-128 inputs and CPU oracle and verifies all three pinned @@ -78,8 +78,9 @@ Remaining implementation gates SPEC-r3 supplies the custom descriptor/aperture layouts and a normalized immutable preload digest with 42 ordered patchbay records. The complete -validators, original-container ABI and physical boot backend still need -implementation. Off-list preload mutations remain refused. +preload validator, original-container ABI, process-resource decoder and +physical boot backend still need implementation. Off-list preload mutations +remain refused. The 2026-10-31 H16 runtime observation did not capture cold boot, scratch negotiation, RTKit management, CONFIG_GET or RESOURCE_INFO_GET. The version-12 @@ -123,8 +124,8 @@ descriptor's original index rather than assuming name order is wire order. The transport prepares the H16 envelope and rejects unrecognized runtime IDs, wrong sizes, channels and power-gate values before publication. H14's 0x0204 procedure packet and short CREATE_PROCESS body remain H14-only; -H16's standalone serializer and event classifier are described below; -they are not connected to inference submission. +H16's serializer and UUID completion table are connected to the shared +submission and event paths, subject to validated process resources. The H16 runtime command table does not authorize startup commands. H16 execute and event fields @@ -168,13 +169,80 @@ atomically once on a terminal report, synchronize DMA, and apply quarantine rules before considering any mapping release. A classifier result alone cannot prove safe release or reject duplicate events without pending state. -The live H16 path still consumes reports as diagnostics. It has no pending -UUID table, serializer call or inference interface. The legacy 0x28-byte -completion predicate remains H14-only. Short channel-6 CPU-load reports and -channel-5 allocation operations are outside these helpers and remain gated. -Timeouts still retain mappings and quarantine without retry or reset. +The shared H16 submission path allocates a non-reused 64-bit UUID, installs +it in a bounded pending table before publishing channel 7, and waits for +both the transport reply and its own terminal event. Events may arrive +before the acknowledgment. States 0 and 3 leave the UUID pending; states +1 and 2 retire it once after matching every identifier. Unknown, duplicate +and late UUIDs cannot retire another request. An identifier mismatch for a pending UUID or a malformed report +quarantines the queue. ISP-destination reports and short +CPU-load notifications cannot complete a general-scheduler request. + +One absolute deadline covers acknowledgment and terminal completion; a +report at or beyond that deadline cannot succeed. Failure state 2, +transport refusal, timeout or corruption prevents output copy-back and +quarantines the queue without retry or reset. Firmware-visible process +mappings remain resident, including after file close. Closing a file +revokes its process capabilities before freeing its staging BO handles. + +Only private coherent DMA mappings may appear in H16 execute descriptors. +Userspace BOs serve as staging buffers: inputs are snapshotted before +publication and outputs are copied back after validated terminal success. +Write barriers order input and command stores before publication; DMA read +barriers order output reads after completion. Streaming DMA sync APIs must +not be applied to these coherent allocations. There is no userspace alias +to code, scratch or live firmware buffers. + +The resource map must come from the original HWX and a decoded firmware +process reply. A userspace index, numeric class or program/process ID cannot +create that map or confer ownership. Only established input/output classes +are admitted; custom BAR requests require a validated selector map and +remain unavailable. The legacy 0x28-byte completion predicate remains +H14-only. Channel-5 allocation/control/free operations require a validated +H16 reply provider and remain unavailable; H14 allocation replies are never +reused for H16. + +No original-container load ABI or H16 process-resource decoder is supplied +by this change. Consequently no H16 process can yet be installed, inference +remains refused and no accel node is registered. The submission integration +is preparatory and does not open the boot, preload, ownership, firmware +version or resource gates. KUnit covers byte-exact synthetic execute fields, all 1..64 buffer counts, zero/one/two custom BAR lengths, wide DVAs/spans, reserved-zero output, boundary and overflow refusal, all event states, interleaved identities, ISP routing and malformed reports. No test packet is copied from a capture. + +Original HWX validation +----------------------- + +The validator consumes one kernel-owned immutable snapshot of the complete +HWX. It requires subtype 7 and bounded, exactly consumed load commands; +rejects unknown command kinds, duplicate descriptors, overlapping segments, +sections and file regions; and validates symbols against the complete +section-ordinal table. Tensor names, directions, normalized fp16 dimensions, +overflow-checked strides and full batched spans must agree with their +surfaces and buffer references. There must be at least one input and one +output, with the single output last. + +The linear and scratch program descriptors have exact fixed fields and +zero reserved bytes. All five 16-byte aperture records must equal whole +surface segment bases in order, followed by the optional kernel base; +reserved halves and unused records are zero. Interior, duplicate, reordered +and scratch/text aperture addresses are refused. Task counts and word counts +must agree with the independent stream walk. Relocations use the actual +kernel section ordinal, unique bounded offsets and valid addends, and may +patch only packed-record payload words. Headers, register selectors and +padding remain immutable. + +SHA-256 covers the entire original snapshot, including load commands and +padding. The snapshot digest is rechecked before execution. Validation +provides metadata, not allocation authority or permission to load a program; +the original-container ABI and firmware resource map remain required. + +The hardware-free KUnit suites cover pending UUID lifecycle, acknowledgment +versus completion, interleaving, progress, failures, malformed and mismatched +reports, routing, duplicates, timeouts, capacity and UUID exhaustion. HWX +tests generate synthetic linear/scratch forms with optional kernel data, +then exercise command, geometry, tensor, aperture, symbol, relocation and +digest refusal paths. No hardware inference result is implied. diff --git a/drivers/accel/ane/Kconfig b/drivers/accel/ane/Kconfig index c12d49c38a59a..fb3e337b61b42 100644 --- a/drivers/accel/ane/Kconfig +++ b/drivers/accel/ane/Kconfig @@ -8,6 +8,7 @@ config DRM_ACCEL_ANE depends on APPLE_DART || COMPILE_TEST depends on MMU select CRYPTO_LIB_SHA256 + select DRM_ACCEL_ANE_H16_VALIDATION help Choose this option if you have an Apple SoC with a Neural Engine: M1 (T8103), M1 Pro, Max and Ultra (T6000, T6001, T6002), M2 @@ -36,13 +37,19 @@ config DRM_ACCEL_ANE_KUNIT_TEST If unsure, say N. +config DRM_ACCEL_ANE_H16_VALIDATION + tristate + select CRYPTO_LIB_SHA256 + config DRM_ACCEL_ANE_FW_KUNIT_TEST - tristate "KUnit tests for ANE firmware preflight" if !KUNIT_ALL_TESTS + tristate "KUnit tests for ANE firmware validation" if !KUNIT_ALL_TESTS depends on DRM_ACCEL && KUNIT + select DRM_ACCEL_ANE_H16_VALIDATION default KUNIT_ALL_TESTS help Check preload ranges, 27.x channel and reply validation, RTKit - management ordering, and H16 task/relocation bounds without hardware. + management ordering, H16 HWX/resource bounds and UUID retirement + without hardware. These tests can run on architectures other than Apple arm64. If unsure, say N. diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile index c4f0954656118..cae069760fcdb 100644 --- a/drivers/accel/ane/Makefile +++ b/drivers/accel/ane/Makefile @@ -1,9 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_DRM_ACCEL_ANE) += ane.o ane_t6021.o +obj-$(CONFIG_DRM_ACCEL_ANE_H16_VALIDATION) += ane_h16_hwx.o obj-$(CONFIG_DRM_ACCEL_ANE_KUNIT_TEST) += ane_drv_test.o obj-$(CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST) += ane_fw_platform_test.o \ - ane_fw_protocol_test.o ane_h16_tasks_test.o ane_h16_execute_test.o + ane_fw_protocol_test.o ane_h16_tasks_test.o ane_h16_execute_test.o ane_h16_pending_test.o ane_h16_hwx_test.o ane-y := \ ane_drv.o \ diff --git a/drivers/accel/ane/ane_fw_platform.c b/drivers/accel/ane/ane_fw_platform.c index fe76586f779cd..1a8f69a329abe 100644 --- a/drivers/accel/ane/ane_fw_platform.c +++ b/drivers/accel/ane/ane_fw_platform.c @@ -159,10 +159,9 @@ int ane_fw_platform_preflight(struct platform_device *pdev, return dev_err_probe(dev, ret, "H16 requires bootloader-reserved no-map TEXT/DATA preload ranges\n"); - /* SPEC-r3 supplies the preload manifest and HWX layouts. Their full - * validators, original-container ABI and cold boot backend are not yet - * implemented. The runtime capture did not observe firmware startup. - * Neither a broad preload-diff rule nor a section-only LOAD opens this. + /* Original HWX validation does not establish firmware ownership, + * preload integrity, startup state or a decoded process resource map. + * Keep the section-only ABI and incomplete boot backend refused. */ dev_info(dev, "H16 ANE: preload validation, HWX ABI and boot backend incomplete; refusing before power access\n"); return -EOPNOTSUPP; diff --git a/drivers/accel/ane/ane_h16_hwx.c b/drivers/accel/ane/ane_h16_hwx.c new file mode 100644 index 0000000000000..a29e5f81f0aa2 --- /dev/null +++ b/drivers/accel/ane/ane_h16_hwx.c @@ -0,0 +1,619 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include + +#include "ane_h16_hwx.h" +#include "ane_h16_tasks.h" + +#define HWX_COMMANDS_MAX 64 +#define HWX_SYMBOLS_MAX 4096 +#define HWX_SEGMENTS_MAX 8 + +struct hwx_range { + u64 start, size; +}; + +struct hwx_parser { + const u8 *bytes; + size_t size, command_end; + struct ane_h16_hwx info; + struct hwx_range vm[HWX_SEGMENTS_MAX], file[HWX_SEGMENTS_MAX + 4]; + struct ane_h16_hwx_section sections[ANE_H16_SECTIONS]; + u32 tensors[ANE_H16_SURFACES], refs[ANE_H16_SURFACES]; + u32 tensor_count, ref_count, segment_count, file_count, section_count; + u32 symoff, nsyms, stroff, strsize; + bool guard, symtab, metadata; +}; + +static bool hwx_fits(u64 start, u64 length, u64 size) +{ + return start <= size && length <= size - start; +} + +static bool hwx_name(const u8 *p, const char *name) +{ + size_t length = strlen(name); + + return length < 16 && !memcmp(p, name, length) && + !memchr_inv(p + length, 0, 16 - length); +} + +static bool hwx_zero(const u8 *p, size_t start, size_t end) +{ + return !memchr_inv(p + start, 0, end - start); +} + +static int hwx_range_add(struct hwx_range *ranges, u32 *count, u32 max, + u64 start, u64 size) +{ + u64 end; + unsigned int i; + + if (!size || check_add_overflow(start, size, &end) || *count == max) + return -ENOEXEC; + for (i = 0; i < *count; i++) + if (start < ranges[i].start + ranges[i].size && + ranges[i].start < end) + return -ENOEXEC; + ranges[(*count)++] = (struct hwx_range) { start, size }; + return 0; +} + +static int hwx_file_add(struct hwx_parser *p, u64 start, u64 size) +{ + if (start < p->command_end || !hwx_fits(start, size, p->size)) + return -ENOEXEC; + return hwx_range_add(p->file, &p->file_count, ARRAY_SIZE(p->file), start, size); +} + +static int hwx_segment(struct hwx_parser *p, const u8 *c, u32 length) +{ + struct ane_h16_hwx_section *dest; + u64 vm, size, file, filesz; + u32 nsects, prot, i; + bool text, kernel, scratch, surface, guard; + + if (length < 72) + return -ENOEXEC; + vm = get_unaligned_le64(c + 24); + size = get_unaligned_le64(c + 32); + file = get_unaligned_le64(c + 40); + filesz = get_unaligned_le64(c + 48); + prot = get_unaligned_le32(c + 60); + nsects = get_unaligned_le32(c + 64); + if (nsects > ANE_H16_SECTIONS || length != 72 + nsects * 80 || + prot != get_unaligned_le32(c + 56) || + hwx_range_add(p->vm, &p->segment_count, ARRAY_SIZE(p->vm), vm, size)) + return -ENOEXEC; + text = hwx_name(c + 8, "__TEXT"); + kernel = hwx_name(c + 8, "__KERN_0"); + scratch = hwx_name(c + 8, "__DATA"); + surface = hwx_name(c + 8, "__FVMLIB"); + guard = hwx_name(c + 8, "__PAGEZERO"); + if (!text && !kernel && !scratch && !surface && !guard) + return -ENOEXEC; + if (guard) { + if (p->guard || vm || size != SZ_16K || file || filesz || prot || nsects) + return -ENOEXEC; + if (get_unaligned_le32(c + 68) != 4) + return -ENOEXEC; + p->guard = true; + return 0; + } + if (surface || scratch) { + if (file || filesz || !IS_ALIGNED(vm, SZ_16K) || + !IS_ALIGNED(size, SZ_16K) || nsects != 1 || size > SZ_256M) + return -ENOEXEC; + } else if (!filesz || filesz > size || size > SZ_256M || + !IS_ALIGNED(vm, SZ_16K) || !IS_ALIGNED(file, SZ_16K) || + hwx_file_add(p, file, filesz)) { + return -ENOEXEC; + } + if (get_unaligned_le32(c + 68) != (surface ? 6 : text ? + (p->info.scratch.size ? 4 : 0) : 4)) + return -ENOEXEC; + if ((text && (prot != 5 || nsects != 2 || p->info.text.size)) || + (kernel && (prot != 1 || nsects != 1 || p->info.kernel.size)) || + (scratch && (prot != 3 || p->info.scratch.size)) || + (surface && (prot < 1 || prot > 2 || + p->info.surface_count == ANE_H16_SURFACES))) + return -ENOEXEC; + + for (i = 0; i < nsects; i++) { + const u8 *s = c + 72 + i * 80; + struct ane_h16_hwx_section section = { + .vm = get_unaligned_le64(s + 32), + .size = get_unaligned_le64(s + 40), + .offset = get_unaligned_le32(s + 48), + .reloff = get_unaligned_le32(s + 56), + .nreloc = get_unaligned_le32(s + 60), + .ordinal = p->section_count + 1, + }; + u32 alignment = get_unaligned_le32(s + 52), j; + bool task = text && hwx_name(s, "__text"); + + if (p->section_count == ANE_H16_SECTIONS || !section.size || + memcmp(s + 16, c + 8, 16) || alignment > 31 || + !IS_ALIGNED(section.vm, 1ULL << alignment) || + section.vm < vm || !hwx_fits(section.vm - vm, section.size, size) || + !hwx_zero(s, 68, 80) || section.reloff > p->size) + return -ENOEXEC; + for (j = 0; j < p->section_count; j++) { + const struct ane_h16_hwx_section *a = &p->sections[j]; + + if (section.vm < a->vm + a->size && a->vm < section.vm + section.size) + return -ENOEXEC; + } + if (surface || scratch) { + if (section.offset || section.reloff || section.nreloc || section.vm != vm) + return -ENOEXEC; + } else if (section.offset < file || + !IS_ALIGNED(section.offset, 1ULL << alignment) || + !hwx_fits(section.offset - file, section.size, filesz) || + section.offset - file != section.vm - vm) { + return -ENOEXEC; + } + if (section.nreloc) { + if (!task || section.nreloc > ANE_H16_TASK_MAX / 4 || + hwx_file_add(p, section.reloff, (u64)section.nreloc * 8)) + return -ENOEXEC; + } + if (text) { + if (task) + dest = &p->info.text; + else if (hwx_name(s, "__const")) + dest = &p->info.constants; + else + return -ENOEXEC; + } else if (kernel && hwx_name(s, "__kern_0")) { + dest = &p->info.kernel; + if (!IS_ALIGNED(section.vm, 64) || section.size > SZ_256M) + return -ENOEXEC; + } else if (scratch && hwx_name(s, "__bss")) { + dest = &p->info.scratch; + } else if (surface && hwx_name(s, prot == 1 ? "__const" : "__data")) { + struct ane_h16_hwx_surface *b = &p->info.surfaces[p->info.surface_count++]; + + b->allocation = size; + b->direction = prot; + dest = &b->section; + } else { + return -ENOEXEC; + } + if (get_unaligned_le32(s + 64) != (task ? + (section.nreloc ? 0x128 : 0x28) : surface ? + (prot == 1 ? 0x21 : 0x23) : scratch ? 0x25 : 0x26)) + return -ENOEXEC; + if (dest->size) + return -ENOEXEC; + *dest = section; + p->sections[p->section_count++] = section; + } + return 0; +} + +static bool hwx_format(u32 code) +{ + static const u8 known[] = { + 0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 15, 16, 17, + 18, 20, 21, 23, 27, 30, 31, 36, 41, 43, 72, 145, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(known); i++) + if (known[i] == code) + return true; + return false; +} + +static int hwx_program(struct hwx_parser *p) +{ + const u8 *c = p->bytes + p->info.program_offset; + struct ane_h16_hwx *info = &p->info; + u32 length = get_unaligned_le32(c + 4), records, i; + bool mixed = length == 0x8c0; + /* Nonzero cells only; every other byte is checked below. */ + static const u16 common[] = { + 0, 4, 8, 0xc, 0x10, 0x14, 0x20, 0x24, 0x40, 0x44, + 0x810, 0x814, 0x820, 0x824, 0x830, 0x838, 0x83c, + 0x844, 0x848, 0x858, 0x860, + }; + u8 *expected; + int ret = -ENOEXEC; + + if (length != 0x8a0 && !mixed) + return -ENOEXEC; + if (get_unaligned_le32(c + 0xc) != (mixed ? 0x22a : 0x222) || + get_unaligned_le64(c + 0x10) != info->text.vm || + get_unaligned_le64(c + 0x810) != info->text.vm || + get_unaligned_le64(c + 0x20) != info->constants.vm || + get_unaligned_le32(c + 0x820) != 4 || + (u64)get_unaligned_le32(c + 0x824) * 4 != info->text.size || + get_unaligned_le32(c + 0x838) != U32_MAX || + get_unaligned_le32(c + 0x83c) != U32_MAX || + get_unaligned_le32(c + 0x844) != 0xffff || + get_unaligned_le32(c + 0x848) != (mixed ? 0x8b8 : 0x898)) + return -ENOEXEC; + records = get_unaligned_le32(c + 0x860); + if (records != 0x10 && records != 0x12 && records != 0x1f && records != 0x20) + return -ENOEXEC; + info->tasks = get_unaligned_le32(c + 0x830); + if (mixed) { + u32 usable = get_unaligned_le32(c + 0x858); + + if (!info->scratch.size || !usable || usable > info->scratch.size || + get_unaligned_le64(c + 0x40) != info->scratch.vm || + get_unaligned_le32(c + 0x880) != usable) + return -ENOEXEC; + } else if (info->scratch.size || get_unaligned_le64(c + 0x40) || + get_unaligned_le32(c + 0x858)) { + return -ENOEXEC; + } + for (i = 0; i < 5; i++) { + u64 base = i < info->surface_count ? info->surfaces[i].section.vm : + i == info->surface_count ? info->kernel.vm : 0; + + if (get_unaligned_le64(c + 0x70 + i * 16) != base || + get_unaligned_le64(c + 0x78 + i * 16)) + return -ENOEXEC; + } + if (!hwx_format(get_unaligned_le32(c + (mixed ? 0x8b0 : 0x890)))) + return -ENOEXEC; + + /* Construct the exact fixed trailer and reserved-space expectation. */ + expected = kzalloc(length, GFP_KERNEL); + if (!expected) + return -ENOMEM; + for (i = 0; i < ARRAY_SIZE(common); i++) + memcpy(expected + common[i], c + common[i], 4); + for (i = 0; i < 5; i++) + memcpy(expected + 0x70 + i * 16, c + 0x70 + i * 16, 8); + put_unaligned_le32(4, expected + 0x868); + put_unaligned_le32(mixed ? 4 : 2, expected + 0x870); + put_unaligned_le32(mixed ? 2 : 7, expected + 0x878); + put_unaligned_le32(8, expected + 0x87c); + put_unaligned_le32(mixed ? get_unaligned_le32(c + 0x858) : 0x461c4000, + expected + 0x880); + put_unaligned_le32(mixed ? 3 : 9, expected + 0x888); + put_unaligned_le32(8, expected + 0x88c); + if (mixed) { + put_unaligned_le32(0x08020004, expected + 0x890); + put_unaligned_le32(8, expected + 0x894); + put_unaligned_le32(7, expected + 0x898); + put_unaligned_le32(8, expected + 0x89c); + put_unaligned_le32(0x461c4000, expected + 0x8a0); + put_unaligned_le32(9, expected + 0x8a8); + put_unaligned_le32(8, expected + 0x8ac); + } + memcpy(expected + (mixed ? 0x8b0 : 0x890), c + (mixed ? 0x8b0 : 0x890), 4); + memcpy(expected + (mixed ? 0x8b8 : 0x898), "main", 5); + if (!memcmp(c, expected, length)) + ret = ane_h16_task_stream_validate(p->bytes + info->text.offset, + info->text.size, info->tasks); + kfree(expected); + return ret; +} + +static int hwx_tensor(struct hwx_parser *p, u32 index) +{ + struct ane_h16_hwx_surface *b = &p->info.surfaces[index]; + const u8 *c = p->bytes + p->tensors[index]; + const u8 *ref = p->bytes + p->refs[index]; + u32 length = get_unaligned_le32(c + 4), shortoff; + u64 n, channels, height, width, row, plane, batch, minimum; + size_t symbol_length, short_length; + unsigned int i; + + if (length < 0xd48 || length > SZ_16K || get_unaligned_le32(c + 0xc) != 0x34a || + get_unaligned_le32(c + 0x10) != 3 || + get_unaligned_le32(c + 0x14) != b->direction || + get_unaligned_le32(c + 0x18) != 0xd38 || !hwx_zero(c, 0x1c, 0x20) || + get_unaligned_le32(c + 0x20) != 0xd3d || + get_unaligned_le32(c + 0x24) != 5 || + get_unaligned_le32(c + 0x38) != 1 || !hwx_zero(c, 0x3c, 0x50) || + get_unaligned_le64(c + 0x68) != 2 || + get_unaligned_le32(c + 0x78) != 1 || !hwx_zero(c, 0x88, 0xd38) || + memcmp(c + 0xd38, "main", 5)) + return -ENOEXEC; + n = get_unaligned_le32(c + 0x28); + channels = get_unaligned_le32(c + 0x2c); + height = get_unaligned_le32(c + 0x30); + width = get_unaligned_le32(c + 0x34); + batch = get_unaligned_le64(c + 0x50); + plane = get_unaligned_le64(c + 0x58); + row = get_unaligned_le64(c + 0x60); + if (!n || !channels || !height || !width || + check_mul_overflow(width, 2ULL, &minimum) || row < minimum || + check_mul_overflow(height, row, &minimum) || plane < minimum || + check_mul_overflow(channels, plane, &minimum) || batch < minimum || + get_unaligned_le64(c + 0x70) != batch || + get_unaligned_le64(c + 0x80) != batch || + check_mul_overflow(n, batch, &b->span) || b->span > b->section.size) + return -ENOEXEC; + symbol_length = strnlen(c + 0xd3d, min_t(u32, 256, length - 0xd3d)); + if (!symbol_length || symbol_length == min_t(u32, 256, length - 0xd3d)) + return -ENOEXEC; + shortoff = get_unaligned_le32(c + 0x7c); + if (shortoff != 0xd3d + symbol_length + 1 || shortoff >= length) + return -ENOEXEC; + short_length = strnlen(c + shortoff, length - shortoff); + if (!short_length || short_length == length - shortoff || + short_length >= sizeof(b->name) || + !hwx_zero(c, shortoff + short_length + 1, length)) + return -ENOEXEC; + memcpy(b->name, c + shortoff, short_length); + b->symbol_offset = p->tensors[index] + 0xd3d; + if (get_unaligned_le64(ref + 8) != 0x18 || + get_unaligned_le64(ref + 0x10) != b->section.vm || + memcmp(ref + 0x18, b->name, sizeof(b->name))) + return -ENOEXEC; + for (i = 0; i < index; i++) + if (!strcmp(b->name, p->info.surfaces[i].name) || + !strcmp(p->bytes + b->symbol_offset, + p->bytes + p->info.surfaces[i].symbol_offset)) + return -ENOEXEC; + return 0; +} + +static int hwx_symbols(struct hwx_parser *p) +{ + u32 found = 0, i, j; + + if (!p->symtab || !p->nsyms || p->nsyms > HWX_SYMBOLS_MAX || + !p->strsize || p->strsize > SZ_256K || + hwx_file_add(p, p->symoff, (u64)p->nsyms * 16) || + hwx_file_add(p, p->stroff, p->strsize) || p->bytes[p->stroff]) + return -ENOEXEC; + for (i = 0; i < p->nsyms; i++) { + const u8 *s = p->bytes + p->symoff + i * 16; + u32 strx = get_unaligned_le32(s); + u64 value = get_unaligned_le64(s + 8); + const struct ane_h16_hwx_section *section; + const char *name; + + if (strx >= p->strsize || !s[5] || s[5] > p->section_count || + (s[4] & ~1) != 0xe) + return -ENOEXEC; + name = p->bytes + p->stroff + strx; + if (!*name || strnlen(name, min_t(u32, 256, p->strsize - strx)) == + min_t(u32, 256, p->strsize - strx)) + return -ENOEXEC; + for (j = 0; j < i; j++) { + u32 previous = get_unaligned_le32(p->bytes + p->symoff + j * 16); + + if (!strcmp(name, p->bytes + p->stroff + previous)) + return -ENOEXEC; + } + section = &p->sections[s[5] - 1]; + if (value < section->vm || value - section->vm >= section->size) + return -ENOEXEC; + for (j = 0; j < p->info.surface_count; j++) { + const struct ane_h16_hwx_surface *b = &p->info.surfaces[j]; + + if (strcmp(name, b->name)) + continue; + if ((found & BIT(j)) || s[5] != b->section.ordinal || + value != b->section.vm) + return -ENOEXEC; + found |= BIT(j); + } + } + return found == BIT(p->info.surface_count) - 1 ? 0 : -ENOEXEC; +} + +/* The stream has already passed the complete structural walk. Clear every + * payload site from the relocation bitmap: any remaining bit names a prefix, + * header, register selector or padding byte, which must never be relocated. + */ +static bool hwx_relocations_in_payload(const struct hwx_parser *p, unsigned long *sites) +{ + const u8 *text = p->bytes + p->info.text.offset; + size_t pos = 16; + + while (pos < p->info.text.size) { + u32 words = (get_unaligned_le32(text + pos) >> 16) & 0x7ff; + size_t end, record; + + if (!words) { + pos += 16; + continue; + } + end = pos + words * 4; + record = pos + 9 * 4; + if ((get_unaligned_le32(text + pos + 8 * 4) & 3) == 3) + record += 4; + while (record < end) { + u32 header = get_unaligned_le32(text + record); + u32 count = header & BIT(31) ? + 1 + hweight32((header >> 15) & 0xffff) : + 1 + ((header >> 15) & 0x3f); + + record += 4; + bitmap_clear(sites, record / 4, count); + record += count * 4; + } + pos = ALIGN(end, 16); + } + return bitmap_empty(sites, p->info.text.size / 4); +} + +int ane_h16_hwx_validate(const void *bytes, size_t size, struct ane_h16_hwx *info) +{ + struct hwx_parser *p; + const void *relocations; + const u8 *c = bytes; + size_t pos = 32; + u32 count, length, i; + unsigned long *seen = NULL; + int ret = -ENOEXEC; + + if (!c || !info || size < 32 || size > ANE_H16_HWX_MAX || + get_unaligned_le32(c) != 0xbeefface || get_unaligned_le32(c + 4) != 0x80 || + get_unaligned_le32(c + 8) != 7 || get_unaligned_le32(c + 12) != 2 || + get_unaligned_le32(c + 24) != 0x200000 || get_unaligned_le32(c + 28)) + return -ENOEXEC; + count = get_unaligned_le32(c + 16); + length = get_unaligned_le32(c + 20); + if (!count || count > HWX_COMMANDS_MAX || !hwx_fits(32, length, size)) + return -ENOEXEC; + p = kzalloc_obj(*p); + if (!p) + return -ENOMEM; + p->bytes = bytes; + p->size = size; + p->command_end = 32 + (size_t)length; + for (i = 0; i < count; i++) { + u32 cmd, len; + + if (!hwx_fits(pos, 8, p->command_end)) + goto invalid; + c = p->bytes + pos; + cmd = get_unaligned_le32(c); + len = get_unaligned_le32(c + 4); + if (len < 8 || (len & 7) || !hwx_fits(pos, len, p->command_end)) + goto invalid; + switch (cmd) { + case 0x19: + ret = hwx_segment(p, c, len); + if (ret) + goto out; + break; + case 2: + if (p->symtab || len != 24) + goto invalid; + p->symtab = true; + p->symoff = get_unaligned_le32(c + 8); + p->nsyms = get_unaligned_le32(c + 12); + p->stroff = get_unaligned_le32(c + 16); + p->strsize = get_unaligned_le32(c + 20); + break; + case 4: + if (len < 16) + goto invalid; + if (get_unaligned_le32(c + 8) == 3) { + if (p->tensor_count == ANE_H16_SURFACES || len < 0xd48) + goto invalid; + p->tensors[p->tensor_count++] = pos; + } else if (get_unaligned_le32(c + 8) == 4) { + if (p->info.program_offset || (len != 0x8a0 && len != 0x8c0)) + goto invalid; + p->info.program_offset = pos; + } else { + goto invalid; + } + break; + case 0x40: + if (len != 32 || p->ref_count == ANE_H16_SURFACES) + goto invalid; + p->refs[p->ref_count++] = pos; + break; + case 8: { + size_t annotation; + + /* Compiler annotation has no authority over mappings. */ + if (p->metadata || len > SZ_4K || len == 8) + goto invalid; + annotation = strnlen(c + 8, len - 8); + if (annotation == len - 8 || !hwx_zero(c, 8 + annotation, len)) + goto invalid; + p->metadata = true; + break; + } + default: + goto invalid; + } + pos += len; + } + if (pos != p->command_end || !p->guard || !p->info.program_offset || + !p->info.text.size || !p->info.constants.size || + p->info.surface_count < 2 || p->ref_count != p->info.surface_count || + p->tensor_count != p->info.surface_count) + goto invalid; + for (i = 0; i < p->info.surface_count; i++) { + /* Inputs in segment order, exactly one output last. */ + if (p->info.surfaces[i].direction != (i == p->info.surface_count - 1 ? 2 : 1)) + goto invalid; + ret = hwx_tensor(p, i); + if (ret) + goto out; + } + ret = hwx_program(p); + if (ret) + goto out; + ret = hwx_symbols(p); + if (ret) + goto out; + if (p->info.text.nreloc) { + seen = bitmap_zalloc(p->info.text.size / 4, GFP_KERNEL); + if (!seen) { + ret = -ENOMEM; + goto out; + } + } + relocations = p->info.text.nreloc ? p->bytes + p->info.text.reloff : NULL; + ret = ane_h16_relocations_validate(p->bytes + p->info.text.offset, + p->info.text.size, relocations, + p->info.text.nreloc, p->info.kernel.ordinal, + p->info.kernel.vm, p->info.kernel.size, + seen, p->info.text.size / 4); + if (!ret && p->info.text.nreloc && !hwx_relocations_in_payload(p, seen)) + ret = -ENOEXEC; + if (!ret) + *info = p->info; + goto out; +invalid: + ret = -ENOEXEC; +out: + bitmap_free(seen); + kfree(p); + return ret; +} +EXPORT_SYMBOL_GPL(ane_h16_hwx_validate); + +int ane_h16_hwx_snapshot_create(const void *bytes, size_t size, + struct ane_h16_hwx_snapshot *snapshot) +{ + struct ane_h16_hwx_snapshot copy = {}; + int ret; + + if (!bytes || !snapshot || size < 32 || size > ANE_H16_HWX_MAX) + return -EINVAL; + copy.bytes = kvmalloc(size, GFP_KERNEL); + if (!copy.bytes) + return -ENOMEM; + memcpy(copy.bytes, bytes, size); + copy.size = size; + ret = ane_h16_hwx_validate(copy.bytes, size, ©.info); + if (ret) { + kvfree(copy.bytes); + return ret; + } + sha256(copy.bytes, size, copy.digest); + *snapshot = copy; + return 0; +} +EXPORT_SYMBOL_GPL(ane_h16_hwx_snapshot_create); + +int ane_h16_hwx_snapshot_check(const struct ane_h16_hwx_snapshot *snapshot) +{ + u8 digest[SHA256_DIGEST_SIZE]; + + if (!snapshot || !snapshot->bytes || snapshot->size < 32 || + snapshot->size > ANE_H16_HWX_MAX) + return -ENOEXEC; + sha256(snapshot->bytes, snapshot->size, digest); + return memcmp(digest, snapshot->digest, sizeof(digest)) ? -ENOEXEC : 0; +} +EXPORT_SYMBOL_GPL(ane_h16_hwx_snapshot_check); + +void ane_h16_hwx_snapshot_destroy(struct ane_h16_hwx_snapshot *snapshot) +{ + kvfree(snapshot->bytes); + memset(snapshot, 0, sizeof(*snapshot)); +} +EXPORT_SYMBOL_GPL(ane_h16_hwx_snapshot_destroy); + +MODULE_DESCRIPTION("H16 original HWX validation"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_hwx.h b/drivers/accel/ane/ane_h16_hwx.h new file mode 100644 index 0000000000000..ba913c0f25d12 --- /dev/null +++ b/drivers/accel/ane/ane_h16_hwx.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_HWX_H__ +#define __ANE_H16_HWX_H__ + +#include +#include +#include + +#define ANE_H16_SURFACES 4 +#define ANE_H16_SECTIONS 8 +#define ANE_H16_HWX_MAX (SZ_256M + SZ_1M) + +struct ane_h16_hwx_section { + u64 vm; + u64 size; + u32 offset; + u32 reloff; + u32 nreloc; + u32 ordinal; +}; + +struct ane_h16_hwx_surface { + struct ane_h16_hwx_section section; + u64 allocation; + u64 span; + u32 direction; + u32 symbol_offset; + char name[8]; +}; + +struct ane_h16_hwx { + struct ane_h16_hwx_section text, constants, kernel, scratch; + struct ane_h16_hwx_surface surfaces[ANE_H16_SURFACES]; + u32 surface_count; + u32 tasks; + u32 program_offset; +}; + +/* Input must be one immutable kernel-owned snapshot of the original file. */ +int ane_h16_hwx_validate(const void *bytes, size_t size, struct ane_h16_hwx *info); + +struct ane_h16_hwx_snapshot { + void *bytes; + size_t size; + u8 digest[SHA256_DIGEST_SIZE]; + struct ane_h16_hwx info; +}; + +/* Snapshot before validation; callers never publish or mutate snapshot bytes. */ +int ane_h16_hwx_snapshot_create(const void *bytes, size_t size, + struct ane_h16_hwx_snapshot *snapshot); +int ane_h16_hwx_snapshot_check(const struct ane_h16_hwx_snapshot *snapshot); +void ane_h16_hwx_snapshot_destroy(struct ane_h16_hwx_snapshot *snapshot); + +#endif /* __ANE_H16_HWX_H__ */ diff --git a/drivers/accel/ane/ane_h16_hwx_test.c b/drivers/accel/ane/ane_h16_hwx_test.c new file mode 100644 index 0000000000000..b7f434748b829 --- /dev/null +++ b/drivers/accel/ane/ane_h16_hwx_test.c @@ -0,0 +1,427 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "ane_h16_hwx.h" +#include "ane_h16_tasks.h" + +struct hwx_fixture { + u8 *bytes; + size_t size, cursor; + u32 commands, program, tensor[2], segment[2], text_command, symtab, reloc; + u32 symbols, strings; +}; + +static u8 *fixture_command(struct hwx_fixture *f, u32 command, u32 length) +{ + u8 *c = f->bytes + f->cursor; + + put_unaligned_le32(command, c); + put_unaligned_le32(length, c + 4); + f->cursor += length; + f->commands++; + return c; +} + +static u8 *fixture_segment(struct hwx_fixture *f, const char *name, u64 vm, + u64 size, u32 offset, u32 prot, u32 flags, u32 sections) +{ + u8 *c = fixture_command(f, 0x19, 72 + sections * 80); + + strscpy(c + 8, name, 16); + put_unaligned_le64(vm, c + 24); + put_unaligned_le64(size, c + 32); + put_unaligned_le64(offset, c + 40); + put_unaligned_le64(offset ? size : 0, c + 48); + put_unaligned_le32(prot, c + 56); + put_unaligned_le32(prot, c + 60); + put_unaligned_le32(sections, c + 64); + put_unaligned_le32(flags, c + 68); + return c + 72; +} + +static void fixture_section(u8 *s, const char *segment, const char *name, + u64 vm, u64 size, u32 offset, u32 alignment, u32 flags) +{ + strscpy(s, name, 16); + strscpy(s + 16, segment, 16); + put_unaligned_le64(vm, s + 32); + put_unaligned_le64(size, s + 40); + put_unaligned_le32(offset, s + 48); + put_unaligned_le32(alignment, s + 52); + put_unaligned_le32(flags, s + 64); +} + +static struct hwx_fixture *fixture_create(struct kunit *test, bool mixed, bool kernel) +{ + struct hwx_fixture *f; + u64 surface = 0x30000000 + (mixed ? SZ_16K : 0); + u64 text = surface + 2 * SZ_16K, kern = text + SZ_32K; + u32 i, strings_length = 1; + u8 *c, *s, *text_section; + static const char * const names[] = { "in", "out" }; + static const char * const symbols[] = { "input", "output" }; + + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + if (!f) + return NULL; + f->size = kernel ? SZ_64K : SZ_32K + SZ_16K; + f->bytes = kunit_kzalloc(test, f->size, GFP_KERNEL); + if (!f->bytes) + return NULL; + f->cursor = 32; + fixture_segment(f, "__PAGEZERO", 0, SZ_16K, 0, 0, 4, 0); + if (mixed) { + s = fixture_segment(f, "__DATA", 0x30000000, SZ_16K, 0, 3, 4, 1); + fixture_section(s, "__DATA", "__bss", 0x30000000, SZ_16K, 0, 6, 0x25); + } + for (i = 0; i < 2; i++) { + f->segment[i] = f->cursor; + s = fixture_segment(f, "__FVMLIB", surface + i * SZ_16K, SZ_16K, + 0, i + 1, 6, 1); + fixture_section(s, "__FVMLIB", i ? "__data" : "__const", + surface + i * SZ_16K, SZ_16K, 0, 14, i ? 0x23 : 0x21); + } + f->text_command = f->cursor; + text_section = fixture_segment(f, "__TEXT", text, SZ_32K, SZ_16K, 5, + mixed ? 4 : 0, 2); + fixture_section(text_section, "__TEXT", "__text", text, 64, SZ_16K, + 14, kernel ? 0x128 : 0x28); + fixture_section(text_section + 80, "__TEXT", "__const", text + 64, + SZ_16K, SZ_16K + 64, 6, 0x26); + if (kernel) { + s = fixture_segment(f, "__KERN_0", kern, SZ_16K, SZ_32K + SZ_16K, 1, 4, 1); + fixture_section(s, "__KERN_0", "__kern_0", kern, 64, + SZ_32K + SZ_16K, 6, 0x26); + } + for (i = 0; i < 2; i++) { + c = fixture_command(f, 0x40, 32); + put_unaligned_le64(0x18, c + 8); + put_unaligned_le64(surface + i * SZ_16K, c + 0x10); + strscpy(c + 0x18, names[i], 8); + } + f->program = f->cursor; + c = fixture_command(f, 4, mixed ? 0x8c0 : 0x8a0); + put_unaligned_le32(4, c + 8); + put_unaligned_le32(mixed ? 0x22a : 0x222, c + 0xc); + put_unaligned_le64(text, c + 0x10); + put_unaligned_le64(text + 64, c + 0x20); + if (mixed) + put_unaligned_le64(0x30000000, c + 0x40); + for (i = 0; i < 2; i++) + put_unaligned_le64(surface + i * SZ_16K, c + 0x70 + i * 16); + if (kernel) + put_unaligned_le64(kern, c + 0x90); + put_unaligned_le64(text, c + 0x810); + put_unaligned_le32(4, c + 0x820); + put_unaligned_le32(16, c + 0x824); + put_unaligned_le32(1, c + 0x830); + put_unaligned_le32(U32_MAX, c + 0x838); + put_unaligned_le32(U32_MAX, c + 0x83c); + put_unaligned_le32(0xffff, c + 0x844); + put_unaligned_le32(mixed ? 0x8b8 : 0x898, c + 0x848); + put_unaligned_le32(0x10, c + 0x860); + put_unaligned_le32(4, c + 0x868); + put_unaligned_le32(mixed ? 4 : 2, c + 0x870); + put_unaligned_le32(mixed ? 2 : 7, c + 0x878); + put_unaligned_le32(8, c + 0x87c); + put_unaligned_le32(mixed ? 256 : 0x461c4000, c + 0x880); + put_unaligned_le32(mixed ? 3 : 9, c + 0x888); + put_unaligned_le32(8, c + 0x88c); + if (mixed) { + put_unaligned_le32(256, c + 0x858); + put_unaligned_le32(0x08020004, c + 0x890); + put_unaligned_le32(8, c + 0x894); + put_unaligned_le32(7, c + 0x898); + put_unaligned_le32(8, c + 0x89c); + put_unaligned_le32(0x461c4000, c + 0x8a0); + put_unaligned_le32(9, c + 0x8a8); + put_unaligned_le32(8, c + 0x8ac); + } + memcpy(c + (mixed ? 0x8b8 : 0x898), "main", 5); + for (i = 0; i < 2; i++) { + u32 shortoff = 0xd3d + strlen(symbols[i]) + 1; + + f->tensor[i] = f->cursor; + c = fixture_command(f, 4, 0xd48); + put_unaligned_le32(3, c + 8); + put_unaligned_le32(0x34a, c + 0xc); + put_unaligned_le32(3, c + 0x10); + put_unaligned_le32(i + 1, c + 0x14); + put_unaligned_le32(0xd38, c + 0x18); + put_unaligned_le32(0xd3d, c + 0x20); + put_unaligned_le32(5, c + 0x24); + put_unaligned_le32(1, c + 0x28); + put_unaligned_le32(1, c + 0x2c); + put_unaligned_le32(1, c + 0x30); + put_unaligned_le32(128, c + 0x34); + put_unaligned_le32(1, c + 0x38); + put_unaligned_le64(256, c + 0x50); + put_unaligned_le64(256, c + 0x58); + put_unaligned_le64(256, c + 0x60); + put_unaligned_le64(2, c + 0x68); + put_unaligned_le64(256, c + 0x70); + put_unaligned_le32(1, c + 0x78); + put_unaligned_le32(shortoff, c + 0x7c); + put_unaligned_le64(256, c + 0x80); + memcpy(c + 0xd38, "main", 5); + memcpy(c + 0xd3d, symbols[i], strlen(symbols[i]) + 1); + memcpy(c + shortoff, names[i], strlen(names[i]) + 1); + } + f->symtab = f->cursor; + c = fixture_command(f, 2, 24); + f->symbols = f->cursor; + f->strings = f->symbols + 2 * 16; + for (i = 0; i < 2; i++) { + u8 *symbol = f->bytes + f->symbols + i * 16; + + put_unaligned_le32(strings_length, symbol); + symbol[4] = 0xf; + symbol[5] = (mixed ? 2 : 1) + i; + put_unaligned_le64(surface + i * SZ_16K, symbol + 8); + memcpy(f->bytes + f->strings + strings_length, names[i], strlen(names[i]) + 1); + strings_length += strlen(names[i]) + 1; + } + put_unaligned_le32(f->symbols, c + 8); + put_unaligned_le32(2, c + 12); + put_unaligned_le32(f->strings, c + 16); + put_unaligned_le32(strings_length, c + 20); + f->reloc = ALIGN(f->strings + strings_length, 8); + if (kernel) { + put_unaligned_le32(f->reloc, text_section + 56); + put_unaligned_le32(1, text_section + 60); + put_unaligned_le32(56, f->bytes + f->reloc); + put_unaligned_le32(0x07000000 | (mixed ? 6 : 5), f->bytes + f->reloc + 4); + put_unaligned_le32(32, f->bytes + SZ_16K + 56); + } + put_unaligned_le32(11 << 16, f->bytes + SZ_16K + 16); + put_unaligned_le32(0x4100 / 4, f->bytes + SZ_16K + 52); + put_unaligned_le32(0xbeefface, f->bytes); + put_unaligned_le32(0x80, f->bytes + 4); + put_unaligned_le32(7, f->bytes + 8); + put_unaligned_le32(2, f->bytes + 12); + put_unaligned_le32(f->commands, f->bytes + 16); + put_unaligned_le32(f->cursor - 32, f->bytes + 20); + put_unaligned_le32(0x200000, f->bytes + 24); + return f; +} + +static void hwx_forms_test(struct kunit *test) +{ + unsigned int variant; + + for (variant = 0; variant < 4; variant++) { + struct hwx_fixture *f = fixture_create(test, variant & 1, variant & 2); + struct ane_h16_hwx info = {}; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), 0); + KUNIT_EXPECT_EQ(test, info.surface_count, 2U); + KUNIT_EXPECT_EQ(test, info.surfaces[0].span, 256ULL); + KUNIT_EXPECT_EQ(test, info.tasks, 1U); + KUNIT_EXPECT_EQ(test, info.text.ordinal, (variant & 1) ? 4U : 3U); + KUNIT_EXPECT_EQ(test, info.text.nreloc, (variant & 2) ? 1U : 0U); + } +} + +static void hwx_header_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx info; + u32 offset; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (offset = 0; offset < 32; offset += 4) { + u32 saved = get_unaligned_le32(f->bytes + offset); + + put_unaligned_le32(U32_MAX, f->bytes + offset); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(saved, f->bytes + offset); + } + for (offset = 0; offset < 32; offset++) + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, offset, &info), -ENOEXEC); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size - 1, &info), -ENOEXEC); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, ANE_H16_HWX_MAX + 1, &info), -ENOEXEC); + /* A truncated command must not inherit a previous command's success. */ + put_unaligned_le32(f->cursor - 33, f->bytes + 20); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); +} + +static void hwx_command_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx info; + u32 pos; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (pos = 32; pos < f->cursor;) { + u32 command = get_unaligned_le32(f->bytes + pos); + u32 length = get_unaligned_le32(f->bytes + pos + 4); + + put_unaligned_le32(0xffffffff, f->bytes + pos); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(command, f->bytes + pos); + put_unaligned_le32(0, f->bytes + pos + 4); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(length, f->bytes + pos + 4); + pos += length; + } +} + +static void hwx_tensor_test(struct kunit *test) +{ + static const u32 fields[] = { + 0xc, 0x10, 0x14, 0x18, 0x1c, 0x20, 0x24, 0x28, 0x2c, + 0x30, 0x34, 0x38, 0x3c, 0x50, 0x58, 0x60, 0x68, 0x70, + 0x78, 0x7c, 0x80, 0x88, 0xd38, 0xd44, + }; + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx info; + u8 *tensor; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + tensor = f->bytes + f->tensor[0]; + for (i = 0; i < ARRAY_SIZE(fields); i++) { + u32 saved = get_unaligned_le32(tensor + fields[i]); + + put_unaligned_le32(U32_MAX, tensor + fields[i]); + KUNIT_EXPECT_EQ_MSG(test, ane_h16_hwx_validate(f->bytes, f->size, &info), + -ENOEXEC, "tensor field %#x", fields[i]); + put_unaligned_le32(saved, tensor + fields[i]); + } + /* Four batches fit in the allocation; span is not just one batch. */ + put_unaligned_le32(4, tensor + 0x28); + KUNIT_ASSERT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), 0); + KUNIT_EXPECT_EQ(test, info.surfaces[0].span, 1024ULL); + put_unaligned_le64(U64_MAX, tensor + 0x50); + put_unaligned_le64(U64_MAX, tensor + 0x70); + put_unaligned_le64(U64_MAX, tensor + 0x80); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); +} + +static void hwx_program_test(struct kunit *test) +{ + static const u32 fields[] = { + 0xc, 0x10, 0x18, 0x20, 0x28, 0x40, 0x48, 0x70, 0x78, 0x80, 0x88, + 0x90, 0xa0, 0xb0, 0xb8, 0xc0, 0x80c, 0x810, 0x818, 0x820, 0x824, + 0x828, 0x830, 0x834, 0x838, 0x83c, 0x840, 0x844, 0x848, 0x84c, + 0x858, 0x85c, 0x860, 0x864, 0x868, 0x870, 0x878, 0x880, 0x890, 0x89c, + }; + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx info; + u8 *program; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + program = f->bytes + f->program; + for (i = 0; i < ARRAY_SIZE(fields); i++) { + u32 saved = get_unaligned_le32(program + fields[i]); + + put_unaligned_le32(saved ^ U32_MAX, program + fields[i]); + KUNIT_EXPECT_EQ_MSG(test, ane_h16_hwx_validate(f->bytes, f->size, &info), + -ENOEXEC, "program field %#x", fields[i]); + put_unaligned_le32(saved, program + fields[i]); + } +} + +static void hwx_geometry_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx info; + u8 *segment; + unsigned int variant; + + KUNIT_ASSERT_NOT_NULL(test, f); + segment = f->bytes + f->segment[0]; + for (variant = 0; variant < 8; variant++) { + u32 fields[] = { 24, 32, 40, 48, 56, 60, 64, 72 + 52 }; + u32 saved = get_unaligned_le32(segment + fields[variant]); + + put_unaligned_le32(U32_MAX, segment + fields[variant]); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(saved, segment + fields[variant]); + } + put_unaligned_le64(U64_MAX - SZ_16K + 1, segment + 24); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); +} + +static void hwx_symbols_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx info; + u32 field; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (field = 8; field < 24; field += 4) { + u32 saved = get_unaligned_le32(f->bytes + f->symtab + field); + + put_unaligned_le32(U32_MAX, f->bytes + f->symtab + field); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(saved, f->bytes + f->symtab + field); + } + f->bytes[f->symbols + 5] = 0; + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + f->bytes[f->symbols + 5] = 1; + f->bytes[f->strings + 1] = 'X'; + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); +} + +static void hwx_relocations_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, true, true); + struct ane_h16_hwx info; + + KUNIT_ASSERT_NOT_NULL(test, f); + /* Scratch changes the section ordinal; a fixed ordinal is invalid. */ + put_unaligned_le32(0x07000005, f->bytes + f->reloc + 4); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(0x07000006, f->bytes + f->reloc + 4); + put_unaligned_le32(64, f->bytes + SZ_16K + 56); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(32, f->bytes + SZ_16K + 56); + put_unaligned_le32(57, f->bytes + f->reloc); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + /* A small addend in a header or the prefix must not be patched. */ + put_unaligned_le32(20, f->bytes + f->reloc); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(4, f->bytes + f->reloc); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); +} + +static void hwx_snapshot_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx_snapshot snapshot = {}; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, ane_h16_hwx_snapshot_create(f->bytes, f->size, &snapshot), 0); + f->bytes[f->tensor[0] + 0x24] = 0; + KUNIT_EXPECT_EQ(test, ane_h16_hwx_snapshot_check(&snapshot), 0); + /* Even padding and original command bytes are covered by the digest. */ + ((u8 *)snapshot.bytes)[f->program + 0x100] = 1; + KUNIT_EXPECT_EQ(test, ane_h16_hwx_snapshot_check(&snapshot), -ENOEXEC); + ane_h16_hwx_snapshot_destroy(&snapshot); + KUNIT_EXPECT_PTR_EQ(test, snapshot.bytes, NULL); +} + +static struct kunit_case hwx_cases[] = { + KUNIT_CASE(hwx_forms_test), + KUNIT_CASE(hwx_header_test), + KUNIT_CASE(hwx_command_test), + KUNIT_CASE(hwx_tensor_test), + KUNIT_CASE(hwx_program_test), + KUNIT_CASE(hwx_geometry_test), + KUNIT_CASE(hwx_symbols_test), + KUNIT_CASE(hwx_relocations_test), + KUNIT_CASE(hwx_snapshot_test), + {} +}; + +static struct kunit_suite hwx_suite = { + .name = "ane-h16-hwx", + .test_cases = hwx_cases, +}; + +kunit_test_suite(hwx_suite); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_pending.h b/drivers/accel/ane/ane_h16_pending.h new file mode 100644 index 0000000000000..b74b20a0b0ea2 --- /dev/null +++ b/drivers/accel/ane/ane_h16_pending.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_PENDING_H__ +#define __ANE_H16_PENDING_H__ + +#include + +#include "ane_h16_execute.h" + +#define ANE_H16_PENDING_MAX 64 + +struct ane_h16_pending { + struct ane_h16_request_key key; + unsigned long deadline; + int status; + bool active; + bool terminal; +}; + +/* All operations, including event dispatch, require the firmware mutex. */ +struct ane_h16_pending_table { + struct ane_h16_pending *slots[ANE_H16_PENDING_MAX]; + u64 next_uuid; + int error; + bool quarantined; +}; + +static inline int +ane_h16_pending_insert(struct ane_h16_pending_table *table, + struct ane_h16_pending *request, unsigned long now) +{ + unsigned int i; + + if (table->quarantined) + return -EIO; + if (!request || request->active || time_after_eq(now, request->deadline)) + return -EINVAL; + for (i = 0; i < ANE_H16_PENDING_MAX; i++) + if (table->slots[i] == request) + return -EEXIST; + if (table->next_uuid == U64_MAX) + return -EOVERFLOW; + for (i = 0; i < ANE_H16_PENDING_MAX; i++) { + if (table->slots[i]) + continue; + /* Never reuse a UUID, including after pre-publication failure. */ + request->key.uuid = ++table->next_uuid; + request->status = -EINPROGRESS; + request->terminal = false; + request->active = true; + table->slots[i] = request; + return 0; + } + return -ENOSPC; +} + +/* Removal is not permission to release DMA backing. */ +static inline void +ane_h16_pending_remove(struct ane_h16_pending_table *table, + struct ane_h16_pending *request) +{ + unsigned int i; + + for (i = 0; i < ANE_H16_PENDING_MAX; i++) + if (table->slots[i] == request) + table->slots[i] = NULL; + request->active = false; +} + +/* A published command with an uncertain outcome pins all its mappings. */ +static inline void +ane_h16_pending_abort(struct ane_h16_pending_table *table, + struct ane_h16_pending *request, int status) +{ + table->quarantined = true; + table->error = status; + ane_h16_pending_remove(table, request); + request->status = status; +} + +static inline int +ane_h16_pending_event(struct ane_h16_pending_table *table, const void *bytes, + size_t length, unsigned int channel, unsigned long now) +{ + struct ane_h16_pending *request = NULL; + struct ane_h16_event event; + unsigned int i; + int ret; + + if (table->quarantined) + return -EIO; + ret = ane_h16_event_decode(bytes, length, channel, &event); + if (ret) + goto corrupt; + if (event.destination) + return -ENOMSG; + for (i = 0; i < ANE_H16_PENDING_MAX; i++) + if (table->slots[i] && table->slots[i]->key.uuid == event.key.uuid) { + request = table->slots[i]; + break; + } + /* Duplicates and late/foreign UUIDs can never retire another call. */ + if (!request) + return -ENOENT; + ret = ane_h16_event_result(bytes, length, channel, &request->key); + if (ret < 0) + goto corrupt; + if (time_after_eq(now, request->deadline)) { + ane_h16_pending_abort(table, request, -ETIMEDOUT); + return -ETIMEDOUT; + } + if (ret == ANE_H16_EVENT_PROGRESS) + return ret; + ane_h16_pending_remove(table, request); + request->terminal = true; + request->status = ret == ANE_H16_EVENT_COMPLETE ? 0 : -EIO; + if (request->status) { + table->error = request->status; + table->quarantined = true; + } + return ret; + +corrupt: + table->error = ret; + table->quarantined = true; + return ret; +} + +/* Channel-6 CPU-load notifications are diagnostics, never completions. */ +static inline int +ane_h16_pending_dispatch(struct ane_h16_pending_table *table, const void *bytes, + size_t length, unsigned int channel, unsigned long now) +{ + const u8 *p = bytes; + + if (table->quarantined) + return -EIO; + if (p && channel == 6 && length == 0xc && + get_unaligned_le16(p + 4) == 0x0107 && !get_unaligned_le16(p + 6)) + return ANE_H16_EVENT_PROGRESS; + return ane_h16_pending_event(table, bytes, length, channel, now); +} + +#endif /* __ANE_H16_PENDING_H__ */ diff --git a/drivers/accel/ane/ane_h16_pending_test.c b/drivers/accel/ane/ane_h16_pending_test.c new file mode 100644 index 0000000000000..891342b9a7a37 --- /dev/null +++ b/drivers/accel/ane/ane_h16_pending_test.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "ane_h16_pending.h" +#include "ane_h16_resources.h" + +static void make_event(u8 *bytes, const struct ane_h16_pending *request, u32 state) +{ + memset(bytes, 0, 0x60); + put_unaligned_le32(0x12345678, bytes); + put_unaligned_le16(ANE_H16_EVENT_REPORT, bytes + 4); + put_unaligned_le64(request->key.uuid, bytes + 8); + put_unaligned_le32(request->key.program, bytes + 0x10); + put_unaligned_le32(request->key.process, bytes + 0x14); + put_unaligned_le32(request->key.procedure, bytes + 0x18); + put_unaligned_le32(state, bytes + 0x1c); +} + +static void pending_sequence_test(struct kunit *test) +{ + struct ane_h16_pending_table table = {}; + struct ane_h16_pending a = { .key = { .program = 7, .process = 8, .procedure = 9 }, + .deadline = 100 }; + struct ane_h16_pending b = { .key = a.key, .deadline = 100 }; + u8 event[0x60], ack[8] = {}; + u32 state; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &a, 0), 0); + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &b, 0), 0); + KUNIT_EXPECT_NE(test, a.key.uuid, b.key.uuid); + /* A valid transport acknowledgment leaves both UUIDs pending. */ + put_unaligned_le16(ANE_H16_EXECUTE, ack + 4); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(0x10001, 0xca8, 0, + 0x10000, 0xca8, ANE_H16_EXECUTE, ack, true), 0); + KUNIT_EXPECT_TRUE(test, a.active); + KUNIT_EXPECT_TRUE(test, b.active); + for (state = 0; state <= 3; state += 3) { + make_event(event, &a, state); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 1), + ANE_H16_EVENT_PROGRESS); + KUNIT_EXPECT_TRUE(test, a.active); + KUNIT_EXPECT_FALSE(test, a.terminal); + } + make_event(event, &b, 1); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 2), + ANE_H16_EVENT_COMPLETE); + KUNIT_EXPECT_TRUE(test, b.terminal); + KUNIT_EXPECT_EQ(test, b.status, 0); + KUNIT_EXPECT_FALSE(test, b.active); + KUNIT_EXPECT_TRUE(test, a.active); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 3), -ENOENT); + make_event(event, &a, 1); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 4), + ANE_H16_EVENT_COMPLETE); + KUNIT_EXPECT_FALSE(test, table.quarantined); +} + +static void pending_mismatched_ids_test(struct kunit *test) +{ + unsigned int offset; + + for (offset = 0x10; offset <= 0x18; offset += 4) { + struct ane_h16_pending_table table = {}; + struct ane_h16_pending request = { .deadline = 100 }; + u8 event[0x60]; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &request, 0), 0); + make_event(event, &request, 1); + put_unaligned_le32(1, event + offset); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 1), -EPROTO); + KUNIT_EXPECT_TRUE(test, request.active); + KUNIT_EXPECT_FALSE(test, request.terminal); + KUNIT_EXPECT_TRUE(test, table.quarantined); + KUNIT_EXPECT_EQ(test, table.error, -EPROTO); + ane_h16_pending_abort(&table, &request, -EIO); + KUNIT_EXPECT_FALSE(test, request.active); + } +} + +static void pending_malformed_test(struct kunit *test) +{ + unsigned int variant; + + for (variant = 0; variant < 9; variant++) { + struct ane_h16_pending_table table = {}; + struct ane_h16_pending request = { .deadline = 100 }; + u8 event[0x60]; + unsigned int channel = 6; + size_t length = sizeof(event); + const void *bytes = event; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &request, 0), 0); + make_event(event, &request, 1); + switch (variant) { + case 0: + bytes = NULL; + break; + case 1: + length--; + break; + case 2: + length++; + break; + case 3: + channel = 7; + break; + case 4: + put_unaligned_le16(ANE_H16_EXECUTE, event + 4); + break; + case 5: + event[6] = 1; + break; + case 6: + event[7] = 1; + break; + case 7: + put_unaligned_le32(4, event + 0x1c); + break; + case 8: + event[0x30] = 2; + break; + } + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, bytes, length, channel, 1), + -EPROTO); + KUNIT_EXPECT_TRUE(test, request.active); + KUNIT_EXPECT_TRUE(test, table.quarantined); + } +} + +static void pending_routing_test(struct kunit *test) +{ + struct ane_h16_pending_table table = {}; + struct ane_h16_pending request = { .deadline = 100 }; + u8 event[0x60]; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &request, 0), 0); + make_event(event, &request, 1); + event[0x30] = 1; + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 1), -ENOMSG); + event[0x30] = 0; + put_unaligned_le64(request.key.uuid + 1, event + 8); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 1), -ENOENT); + memset(event, 0, sizeof(event)); + put_unaligned_le16(0x0107, event + 4); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0xc, 6, 1), + ANE_H16_EVENT_PROGRESS); + KUNIT_EXPECT_TRUE(test, request.active); + KUNIT_EXPECT_FALSE(test, table.quarantined); + event[7] = 1; + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0xc, 6, 1), -EPROTO); + KUNIT_EXPECT_TRUE(test, table.quarantined); +} + +static void pending_failure_test(struct kunit *test) +{ + struct ane_h16_pending_table table = {}; + struct ane_h16_pending request = { .deadline = 100 }; + struct ane_h16_pending next = { .deadline = 100 }; + u8 event[0x60]; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &request, 0), 0); + make_event(event, &request, 2); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 1), + ANE_H16_EVENT_FAILED); + KUNIT_EXPECT_FALSE(test, request.active); + KUNIT_EXPECT_TRUE(test, request.terminal); + KUNIT_EXPECT_EQ(test, request.status, -EIO); + KUNIT_EXPECT_TRUE(test, table.quarantined); + KUNIT_EXPECT_EQ(test, ane_h16_pending_insert(&table, &next, 2), -EIO); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 2), -EIO); +} + +static void pending_timeout_test(struct kunit *test) +{ + struct ane_h16_pending_table table = {}; + struct ane_h16_pending request = { .deadline = 100 }; + u8 event[0x60]; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &request, 0), 0); + make_event(event, &request, 1); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 100), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, request.status, -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, table.error, -ETIMEDOUT); + KUNIT_EXPECT_FALSE(test, request.terminal); + KUNIT_EXPECT_FALSE(test, request.active); + KUNIT_EXPECT_TRUE(test, table.quarantined); + /* Late success never makes output readable after timeout. */ + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 101), -EIO); +} + +static void pending_transport_failure_test(struct kunit *test) +{ + static const int errors[] = { -ETIMEDOUT, -EPROTO, -EREMOTEIO, -EIO }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(errors); i++) { + struct ane_h16_pending_table table = {}; + struct ane_h16_pending request = { .deadline = 100 }; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &request, 0), 0); + ane_h16_pending_abort(&table, &request, errors[i]); + KUNIT_EXPECT_FALSE(test, request.active); + KUNIT_EXPECT_FALSE(test, request.terminal); + KUNIT_EXPECT_EQ(test, request.status, errors[i]); + KUNIT_EXPECT_TRUE(test, table.quarantined); + } +} + +static void pending_capacity_test(struct kunit *test) +{ + struct ane_h16_pending_table table = {}; + struct ane_h16_pending *requests; + unsigned int i; + + requests = kunit_kcalloc(test, ANE_H16_PENDING_MAX + 1, sizeof(*requests), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, requests); + for (i = 0; i <= ANE_H16_PENDING_MAX; i++) + requests[i].deadline = 100; + for (i = 0; i < ANE_H16_PENDING_MAX; i++) { + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &requests[i], 0), 0); + KUNIT_EXPECT_EQ(test, requests[i].key.uuid, (u64)i + 1); + } + KUNIT_EXPECT_EQ(test, ane_h16_pending_insert(&table, &requests[i], 0), -ENOSPC); + ane_h16_pending_remove(&table, &requests[0]); + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &requests[i], 0), 0); + KUNIT_EXPECT_EQ(test, requests[i].key.uuid, (u64)ANE_H16_PENDING_MAX + 1); + ane_h16_pending_remove(&table, &requests[i]); + table.next_uuid = U64_MAX; + KUNIT_EXPECT_EQ(test, ane_h16_pending_insert(&table, &requests[i], 0), -EOVERFLOW); +} + +static void pending_deadline_wrap_test(struct kunit *test) +{ + struct ane_h16_pending_table table = {}; + struct ane_h16_pending request = { .deadline = 3 }; + u8 event[0x60]; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &request, ULONG_MAX - 4), 0); + make_event(event, &request, 1); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 1), + ANE_H16_EVENT_COMPLETE); + request.deadline = 3; + KUNIT_EXPECT_EQ(test, ane_h16_pending_insert(&table, &request, 3), -EINVAL); +} + +static void resources_test(struct kunit *test) +{ + struct ane_h16_hwx hwx = { .surface_count = 2 }; + struct ane_h16_resource resources[2] = {}; + unsigned int i, variant; + + for (i = 0; i < 2; i++) { + hwx.surfaces[i].section.vm = 0x30000000 + i * SZ_16K; + hwx.surfaces[i].section.size = SZ_16K; + hwx.surfaces[i].allocation = SZ_16K; + hwx.surfaces[i].span = 256; + hwx.surfaces[i].direction = i + 1; + resources[i].hwx_vm = hwx.surfaces[i].section.vm; + resources[i].permissions = i + 1; + resources[i].buffer = (struct ane_h16_execute_buffer) { + .index = i, .class = i, .mapping_dva = 0x100000000ULL + i * SZ_16K, + .mapping_size = SZ_16K, .size = 256, + }; + } + KUNIT_EXPECT_EQ(test, ane_h16_resources_validate(&hwx, resources, 2), 0); + KUNIT_EXPECT_EQ(test, ane_h16_resources_validate(&hwx, resources, 1), -EINVAL); + for (variant = 0; variant < 9; variant++) { + struct ane_h16_resource bad[2]; + + memcpy(bad, resources, sizeof(bad)); + switch (variant) { + case 0: + bad[0].permissions = 2; + break; + case 1: + bad[0].hwx_vm++; + break; + case 2: + bad[0].buffer.class = 8; + break; + case 3: + bad[0].buffer.size++; + break; + case 4: + bad[0].buffer.offset = 1; + break; + case 5: + bad[0].buffer.mapping_dva = U64_MAX - SZ_16K + 1; + break; + case 6: + bad[1].buffer.index = bad[0].buffer.index; + break; + case 7: + bad[1].buffer.mapping_dva = bad[0].buffer.mapping_dva; + break; + case 8: + bad[0].buffer.mapping_size--; + break; + } + KUNIT_EXPECT_EQ(test, ane_h16_resources_validate(&hwx, bad, 2), -EINVAL); + } +} + +static struct kunit_case pending_cases[] = { + KUNIT_CASE(pending_sequence_test), + KUNIT_CASE(pending_mismatched_ids_test), + KUNIT_CASE(pending_malformed_test), + KUNIT_CASE(pending_routing_test), + KUNIT_CASE(pending_failure_test), + KUNIT_CASE(pending_timeout_test), + KUNIT_CASE(pending_transport_failure_test), + KUNIT_CASE(pending_capacity_test), + KUNIT_CASE(pending_deadline_wrap_test), + KUNIT_CASE(resources_test), + {} +}; + +static struct kunit_suite pending_suite = { + .name = "ane-h16-pending", + .test_cases = pending_cases, +}; + +kunit_test_suite(pending_suite); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_resources.h b/drivers/accel/ane/ane_h16_resources.h new file mode 100644 index 0000000000000..46fa17d506e2c --- /dev/null +++ b/drivers/accel/ane/ane_h16_resources.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_RESOURCES_H__ +#define __ANE_H16_RESOURCES_H__ + +#include "ane_h16_execute.h" +#include "ane_h16_hwx.h" + +/* Created only from a validated HWX and a decoded firmware process resource + * map. No userspace field may supply this authority. Each mapping is private, + * coherent DMA memory with no userspace alias, retained for process lifetime. + */ +struct ane_h16_resource { + u64 hwx_vm; + u32 permissions; + struct ane_h16_execute_buffer buffer; +}; + +static inline int +ane_h16_resources_validate(const struct ane_h16_hwx *hwx, + const struct ane_h16_resource *resources, unsigned int count) +{ + unsigned int i, j; + + if (!hwx || !resources || count < 2 || count > ANE_H16_SURFACES || + count != hwx->surface_count) + return -EINVAL; + for (i = 0; i < count; i++) { + const struct ane_h16_hwx_surface *s = &hwx->surfaces[i]; + const struct ane_h16_resource *r = &resources[i]; + const struct ane_h16_execute_buffer *b = &r->buffer; + u64 end; + + /* Only the established input/output classes are executable. */ + if (s->direction < 1 || s->direction > 2 || + r->hwx_vm != s->section.vm || r->permissions != s->direction || + b->class != s->direction - 1 || !s->span || + s->span > s->section.size || s->section.size > s->allocation || + !IS_ALIGNED(s->allocation, SZ_16K) || + b->size != s->span || b->offset || !b->mapping_dva || + !IS_ALIGNED(b->mapping_dva, SZ_16K) || + b->mapping_size != s->allocation || b->size > b->mapping_size || + check_add_overflow(b->mapping_dva, b->mapping_size, &end)) + return -EINVAL; + for (j = 0; j < i; j++) { + const struct ane_h16_execute_buffer *a = &resources[j].buffer; + + if (a->index == b->index || + (b->mapping_dva < a->mapping_dva + a->mapping_size && + a->mapping_dva < end)) + return -EINVAL; + } + } + return 0; +} + +#endif /* __ANE_H16_RESOURCES_H__ */ diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 5f1e5e66bd22d..217ac1eb18220 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -93,6 +93,8 @@ #include "ane_fw_platform.h" #include "ane_h16_protocol.h" +#include "ane_h16_pending.h" +#include "ane_h16_resources.h" #include "ane_fw_protocol.h" #include "ane_stats.h" #include "ane_t6021.h" @@ -195,6 +197,22 @@ struct ane_legacy_buffer { size_t size; }; +struct ane_t6021_fd; + +/* Populated only by an original-container loader and decoded process reply. + * Those interfaces remain refused until their complete wire contract exists. + * The firmware mutex protects this list and every pending UUID operation. + */ +struct ane_h16_process { + struct list_head node; + struct list_head owner_node; + struct ane_h16_hwx_snapshot original; + struct ane_h16_request_key key; + struct ane_h16_resource resources[ANE_H16_SURFACES]; + struct ane_legacy_buffer mappings[ANE_H16_SURFACES]; + u32 resource_count; +}; + struct ane_rtclient { struct device *dev; void __iomem *engine; @@ -220,9 +238,9 @@ struct ane_rtclient { struct ane_fw_channel channels[ANE_FW_CHANNELS]; u8 channel_snapshot[ANE_FW_TABLE_SIZE]; u64 channel_table_offset; - u64 call_cookie; - u32 call_program; - u32 call_process; + struct ane_h16_pending_table pending; + struct ane_h16_pending *waiting; + struct list_head h16_processes; struct ane_legacy_buffer *legacy_buffers; u32 legacy_allocated; size_t legacy_bytes; @@ -255,6 +273,7 @@ struct ane_rtclient { */ struct ane_t6021_fd { struct list_head bos; + struct list_head h16_processes; }; /* Per-file handle counter and every per-fd list against same-fd concurrent @@ -412,6 +431,9 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, unsigned int channel, unsigned long deadline); +static bool ane_rtclient_drain_t2h(struct ane_rtclient *ane, + unsigned int channel); + /* ---- boot helpers (shared with boot.c / fwload.c) ---- */ static int ane_rtclient_legacy_alloc(struct ane_rtclient *ane, u64 size) @@ -593,6 +615,13 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, result = -EPROTO; goto out; } + if (ane->soc->platform && opcode == ANE_H16_EXECUTE) { + ane_rtclient_drain_t2h(ane, 6); + if (ane->pending.quarantined) { + result = ane->pending.error ?: -EIO; + goto out; + } + } pending = readl(ipi + 0x8000); if (pending) { @@ -615,6 +644,13 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, u64 size, tag; struct ane_legacy_buffer *buf; + /* H16 needs its own validated allocation/reply provider. + * Never answer it with the legacy H14 allocation index. + */ + if (ane->soc->platform) { + result = -EOPNOTSUPP; + goto out; + } dma_rmb(); size = READ_ONCE(slot[1]); tag = READ_ONCE(slot[2]); @@ -857,18 +893,24 @@ static void ane_t6021_trace_free(void) static const void *ane_rtclient_fw_cpu(struct ane_rtclient *ane, u64 iova, size_t len) { + const void *found = NULL; u32 i; for (i = 0; i < ane->legacy_allocated; i++) { const struct ane_legacy_buffer *b = &ane->legacy_buffers[i]; - if (ane_fw_contains(b->dma, b->size, iova, len)) - return b->cpu + (iova - b->dma); + if (ane_fw_contains(b->dma, b->size, iova, len)) { + if (!ane->soc->platform) + return b->cpu + (iova - b->dma); + if (found) + return NULL; + found = b->cpu + (iova - b->dma); + } } if (ane_fw_contains(ane->fw->boot_ipc_iova, ane->fw->boot_ipc_size, iova, len)) - return ane->fw->boot_ipc + (iova - ane->fw->boot_ipc_iova); - return NULL; + return found ? NULL : ane->fw->boot_ipc + (iova - ane->fw->boot_ipc_iova); + return found; } /* Return every firmware-owned slot on a target-to-host ring to the fw: @@ -888,30 +930,54 @@ static bool ane_rtclient_drain_t2h(struct ane_rtclient *ane, if (!ane_rtclient_channels_unchanged(ane)) { atomic_set(&ane_t6021_quarantined, 1); + if (ane->soc->platform) { + ane->pending.error = -EPROTO; + ane->pending.quarantined = true; + } return false; } if (c->type != 1) return false; while (n < c->slots) { u64 *slot = ane->fw->boot_ipc + c->offset + (size_t)slot_i * 64; - u64 hdr = READ_ONCE(slot[0]), len = READ_ONCE(slot[1]); + u64 hdr = READ_ONCE(slot[0]), len; if (hdr & 1) break; dma_rmb(); + len = READ_ONCE(slot[1]); dev_dbg(ane->dev, "T2H ch=%u slot=%u len=%#llx\n", channel, slot_i, len); if (ane->soc->platform) { - const void *ev = ane_rtclient_fw_cpu(ane, hdr, len); - int status = ane_h16_event_report_status(ev, len, c->index); + u8 snapshot[0x60]; + const void *ev; + int status; - if (status < 0) { + /* Other channels have different message types; never apply + * the event predicate to logs or allocation traffic. + */ + if (c->index != 6 || (len != sizeof(snapshot) && len != 0xc)) { atomic_set(&ane_t6021_quarantined, 1); + ane->pending.error = -EPROTO; + ane->pending.quarantined = true; return false; } - /* H16 submission and pending-UUID tracking are not wired up. - * Consume diagnostics without retiring an execution or its BOs. - */ + ev = ane_rtclient_fw_cpu(ane, hdr, len); + if (!ev) { + atomic_set(&ane_t6021_quarantined, 1); + ane->pending.error = -EPROTO; + ane->pending.quarantined = true; + return false; + } + memcpy(snapshot, ev, len); + status = ane_h16_pending_dispatch(&ane->pending, snapshot, + len, c->index, jiffies); + if (ane->pending.quarantined) { + atomic_set(&ane_t6021_quarantined, 1); + return false; + } + if (status == ANE_H16_EVENT_COMPLETE) + finished = true; } else if (channel == 6 && len >= ANE_T2H_CALL_STATE_OFF + 4) { const u8 *ev = ane_rtclient_fw_cpu(ane, hdr, ANE_T2H_CALL_STATE_OFF + 4); @@ -993,7 +1059,7 @@ static int ane_rtclient_call_wait_traced(struct ane_rtclient *ane, static int ane_rtclient_call_wait(struct ane_rtclient *ane, unsigned long deadline) { - unsigned int poll_us = READ_ONCE(call_poll_us); + unsigned int poll_us = ane->soc->platform ? 0 : READ_ONCE(call_poll_us); ktime_t tight_end = ktime_add_ns(ktime_get(), (u64)poll_us * NSEC_PER_USEC); @@ -1005,8 +1071,15 @@ static int ane_rtclient_call_wait(struct ane_rtclient *ane, do { if (atomic_read(&ane_t6021_quarantined)) return -EIO; - if (ane_rtclient_drain_t2h(ane, 6)) + if (ane->soc->platform) { + ane_rtclient_drain_t2h(ane, 6); + if (atomic_read(&ane_t6021_quarantined)) + return ane->pending.error ?: -EIO; + if (ane->waiting && ane->waiting->terminal) + return ane->waiting->status; + } else if (ane_rtclient_drain_t2h(ane, 6)) { return 0; + } if (poll_us && ktime_before(ktime_get(), tight_end)) usleep_range(1, 2); else @@ -1025,7 +1098,8 @@ static int ane_rtclient_command(struct ane_rtclient *ane, unsigned int channel, unsigned int timeout_ms) { - unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); + unsigned long deadline = ane->soc->platform && ane->waiting ? + ane->waiting->deadline : jiffies + msecs_to_jiffies(timeout_ms); int ret; u64 stats_ticket = 0; u64 stats_submit_ns = 0; @@ -1055,6 +1129,10 @@ static int ane_rtclient_command(struct ane_rtclient *ane, */ bool stats_call = ane->stats_slots && execute; + if (ane->soc->platform && execute && + (!ane->waiting || !ane->waiting->active || ane->pending.quarantined)) + return -EIO; + if (stats_call) { stats_submit_ns = ktime_get_ns(); stats_ticket = ane_stats_begin(&ane->stats_ctrs, @@ -1093,7 +1171,7 @@ static int ane_rtclient_command(struct ane_rtclient *ane, ktime_get_ns(), (u32)ret, 0); dev_err(ane->dev, - "call completion wait failed %d: no finish event in %u ms\n", + "call completion failed %d within %u ms\n", ret, timeout_ms); atomic_set(&ane_t6021_quarantined, 1); return ret; @@ -1106,7 +1184,8 @@ static int ane_rtclient_command(struct ane_rtclient *ane, * hand those slots back so the rings never fill (channels 4 * and 6). */ - ane_rtclient_drain_t2h(ane, 4); + if (!ane->soc->platform) + ane_rtclient_drain_t2h(ane, 4); ane_rtclient_drain_t2h(ane, 6); if (atomic_read(&ane_t6021_quarantined)) return -EIO; @@ -1438,6 +1517,140 @@ static int ane_rtclient_create_process(struct ane_rtclient *ane, return ret; } +/* H16 consumes only backend-owned process resources. The ABI-2 section + * loader and short process packet cannot create an entry in h16_processes. + */ +static int ane_rtclient_h16_call(struct ane_rtclient *ane, struct drm_file *file, + const struct drm_ane_exec *user) +{ + struct ane_t6021_fd *fd = file->driver_priv; + struct ane_h16_process *process = NULL, *p; + struct ane_h16_execute_buffer buffers[ANE_H16_SURFACES]; + struct ane_t6021_bo *bos[ANE_H16_SURFACES] = {}; + struct drm_ane_exec_io *ios; + struct ane_h16_pending request = {}; + struct ane_h16_execute call = {}; + unsigned int i, j, pinned = 0; + int ret = -EOPNOTSUPP; + + if (!fd || user->pad || user->priority != 5 || user->count < 2 || + user->count > ANE_H16_SURFACES || user->timeout_ms > 30000) + return -EINVAL; + ios = memdup_user(u64_to_user_ptr(user->io_ptr), + array_size(user->count, sizeof(*ios))); + if (IS_ERR(ios)) + return PTR_ERR(ios); + mutex_lock(&ane_t6021_fw_lock); + if (atomic_read(&ane_t6021_quarantined) || ane->pending.quarantined) { + ret = -EIO; + goto out; + } + list_for_each_entry(p, &fd->h16_processes, owner_node) { + if (p->key.program == user->prog_id && + p->key.process == user->proc_id) { + process = p; + break; + } + } + if (!process) + goto out; + ret = ane_h16_hwx_snapshot_check(&process->original); + if (!ret) + ret = ane_h16_resources_validate(&process->original.info, + process->resources, process->resource_count); + if (ret) + goto out; + if (user->count != process->resource_count || !ane->cmd_buf || + process->key.program == U32_MAX || process->key.process == U32_MAX || + process->key.procedure == U32_MAX) { + ret = -EINVAL; + goto out; + } + for (i = 0; i < user->count; i++) { + struct ane_t6021_bo *b; + const struct ane_h16_resource *r = &process->resources[i]; + const struct ane_legacy_buffer *m = &process->mappings[i]; + + if (ios[i].flags || ios[i].reserved || + ios[i].buffer_id != r->buffer.index || ios[i].type != r->buffer.class || + ios[i].size != r->buffer.size || !m->cpu || + m->dma != r->buffer.mapping_dva || m->size != r->buffer.mapping_size) { + ret = -EINVAL; + goto out; + } + for (j = 0; j < i; j++) { + if (ios[j].bo_handle == ios[i].bo_handle) { + ret = -EINVAL; + goto out; + } + } + mutex_lock(&ane_t6021_bo_lock); + list_for_each_entry(b, &fd->bos, node) { + if (b->handle == ios[i].bo_handle && b->owner == fd && + b->size >= ios[i].size) { + kref_get(&b->refcount); + bos[pinned++] = b; + break; + } + } + mutex_unlock(&ane_t6021_bo_lock); + if (pinned != i + 1) { + ret = -ENOENT; + goto out; + } + buffers[i] = r->buffer; + } + /* All user-facing BOs are staging only. Concurrent mmap writes cannot + * change code, input snapshots or firmware-visible output allocations. + */ + for (i = 0; i < user->count; i++) { + if (process->resources[i].permissions == 1) + memcpy(process->mappings[i].cpu, bos[i]->cpu, buffers[i].size); + else + memset(process->mappings[i].cpu, 0xff, buffers[i].size); + } + request.key = process->key; + request.deadline = jiffies + msecs_to_jiffies(user->timeout_ms ?: 5000); + ret = ane_h16_pending_insert(&ane->pending, &request, jiffies); + if (ret) + goto out; + call.key = request.key; + call.buffers = buffers; + call.buffer_count = user->count; + /* Custom BAR selectors/values have no validated process map yet. */ + ret = ane_h16_execute_encode(ane->cmd_buf->cpu, ANE_H16_EXECUTE_BASE_SIZE, &call); + if (ret) { + ane_h16_pending_remove(&ane->pending, &request); + goto out; + } + ane->waiting = &request; + /* Coherent allocations require ordering, not streaming-map sync calls. */ + wmb(); + dma_wmb(); + ret = ane_rtclient_command(ane, ane->cmd_buf, ANE_H16_EXECUTE_BASE_SIZE, + ANE_H16_EXECUTE, 7, user->timeout_ms ?: 5000); + ane->waiting = NULL; + if (ret || !request.terminal || request.status) { + ret = ret ?: -EIO; + ane_h16_pending_abort(&ane->pending, &request, ret); + atomic_set(&ane_t6021_quarantined, 1); + goto out; + } + dma_rmb(); + for (i = 0; i < user->count; i++) + if (process->resources[i].permissions == 2) + memcpy(bos[i]->cpu, process->mappings[i].cpu, buffers[i].size); + /* Private DMA mappings remain resident for the entire process lifetime, + * including after terminal failure or an uncertain transport outcome. + */ +out: + while (pinned) + kref_put(&bos[--pinned]->refcount, ane_t6021_bo_release); + mutex_unlock(&ane_t6021_fw_lock); + kfree(ios); + return ret; +} + static int ane_rtclient_procedure_call(struct ane_rtclient *ane, struct drm_file *file, const struct drm_ane_exec *user) @@ -1450,9 +1663,8 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, size_t cmd_size; int ret, i, pinned = 0; - /* H16 EXECUTE needs validated resources and pending-UUID tracking. */ if (ane->soc->platform) - return -EOPNOTSUPP; + return ane_rtclient_h16_call(ane, file, user); if (user->pad || user->count < 1 || user->count > DRM_ANE_MAX_BINDS || user->priority < 2 || user->priority > 7 || !fd) return -EINVAL; @@ -1503,17 +1715,7 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, *(u32 *)(cmd + 0x0c) = cpu_to_le32(user->proc_id); *(u64 *)(cmd + 0x10) = cpu_to_le64(0); *(u32 *)(cmd + 0x18) = cpu_to_le32(user->priority); - if (ane->soc->platform) { - if (ane->call_cookie == U64_MAX) { - ret = -EOVERFLOW; - goto unlock; - } - ane->call_cookie++; - ane->call_program = user->prog_id; - ane->call_process = user->proc_id; - } - *(u64 *)(cmd + 0x20) = cpu_to_le64(ane->soc->platform ? - ane->call_cookie : ANE_CALL_COOKIE); + *(u64 *)(cmd + 0x20) = cpu_to_le64(ANE_CALL_COOKIE); *(u32 *)(cmd + 0x28) = cpu_to_le32(user->count); for (i = 0; i < user->count; i++) { struct ane_t6021_bo *bo = NULL, *b; @@ -1753,6 +1955,7 @@ static int ane_t6021_open(struct drm_device *drm, struct drm_file *file) if (!fd) return -ENOMEM; INIT_LIST_HEAD(&fd->bos); + INIT_LIST_HEAD(&fd->h16_processes); file->driver_priv = fd; return 0; } @@ -1761,13 +1964,19 @@ static void ane_t6021_postclose(struct drm_device *drm, struct drm_file *file) { struct ane_t6021_fd *fd = file->driver_priv; struct ane_t6021_bo *bo, *tmp; + struct ane_h16_process *process, *next; if (!fd) return; + mutex_lock(&ane_t6021_fw_lock); + /* Revoke per-file authority, retaining device-owned DMA allocations. */ + list_for_each_entry_safe(process, next, &fd->h16_processes, owner_node) + list_del_init(&process->owner_node); mutex_lock(&ane_t6021_bo_lock); list_for_each_entry_safe(bo, tmp, &fd->bos, node) ane_t6021_bo_drop(bo); mutex_unlock(&ane_t6021_bo_lock); + mutex_unlock(&ane_t6021_fw_lock); kfree(fd); file->driver_priv = NULL; } @@ -2235,6 +2444,7 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, if (!ane->legacy_buffers) return -ENOMEM; ane->dev = dev; + INIT_LIST_HEAD(&ane->h16_processes); ane->soc = of_device_get_match_data(dev); platform_set_drvdata(pdev, ane); INIT_DELAYED_WORK(&ane->poll_work, ane_rtclient_post_boot); From 4dbdb77c14dae54438f6355d7819f8e0ad193d8d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:36:43 +1100 Subject: [PATCH 1502/1579] dt-bindings: npu: describe ANE performance and memory requests Describe the optional shared PMP clock controller and an interconnect path with OPP bandwidth requirements. Keep controller register ownership in the PMP provider. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/npu/apple,ane.yaml | 39 ++++++++++++++++++- .../power/apple,t8140-pmp-report.yaml | 29 +++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/npu/apple,ane.yaml b/Documentation/devicetree/bindings/npu/apple,ane.yaml index fea47a60b6854..3167780b88884 100644 --- a/Documentation/devicetree/bindings/npu/apple,ane.yaml +++ b/Documentation/devicetree/bindings/npu/apple,ane.yaml @@ -49,6 +49,22 @@ properties: minItems: 1 maxItems: 7 + apple,performance-controller: + $ref: /schemas/types.yaml#/definitions/phandle + description: + PMP report provider that owns the ANE clock-level request interface. + Only supported on T6000. The ANE and report SRAM must remain powered + while the driver holds a request. + + interconnects: + maxItems: 1 + + interconnect-names: + const: memory + description: DRAM/fabric path; bandwidth requests come from the level OPP. + + operating-points-v2: true + interrupts: minItems: 1 maxItems: 2 @@ -66,7 +82,8 @@ properties: power-domains: description: - The power states the engine needs, five on T8103 and T6000, eight on + The power states the engine needs, five on T8103, six on T6000 with + ANE level control (including PMP report SRAM), eight on T602x, seven on T8112 and one system domain on T8140. minItems: 1 maxItems: 8 @@ -131,6 +148,24 @@ required: - power-domains allOf: + - if: + required: + - interconnects + then: + required: + - interconnect-names + - operating-points-v2 + - if: + required: + - apple,performance-controller + then: + properties: + compatible: + const: apple,t6000-ane + power-domains: + minItems: 6 + maxItems: 6 + - if: properties: compatible: @@ -275,7 +310,7 @@ allOf: reg-names: maxItems: 1 power-domains: - maxItems: 5 + maxItems: 6 resets: false mboxes: false memory-region: false diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml index a5f1475926430..c3baafe00064c 100644 --- a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -19,6 +19,27 @@ description: | contiguous readable array of registers. allOf: + - if: + properties: + compatible: + const: apple,t6000-pmp-v2-report + then: + properties: + reg: + maxItems: 2 + else: + properties: + reg: + maxItems: 1 + reg-names: false + - if: + properties: + reg: + minItems: 2 + then: + required: + - reg-names + - $ref: apple,pmp-dram.yaml# properties: @@ -30,7 +51,13 @@ properties: - apple,t8140-pmp-v2-report reg: - maxItems: 1 + minItems: 1 + maxItems: 2 + + reg-names: + items: + - const: report + - const: ane-dvfs nonposted-mmio: true From 996ef2aef5814825a021c5cb35f286d2ad749f54 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:36:44 +1100 Subject: [PATCH 1503/1579] accel/ane: replace CPU QoS boost with scoped performance requests Request the T600x high clock level for work and release it after a short idle interval. Serialize the PMP request bit with power-domain users, retain runtime PM references through failed rollback, and release before suspend or recovery. Replace CPU frequency QoS with an optional interconnect bandwidth request from DT OPP data. Report unsupported paths explicitly; this does not supply a T600x DCS transport. Add register-model KUnit coverage and read-only performance diagnostics. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../ABI/testing/sysfs-driver-ane-performance | 35 +++ MAINTAINERS | 1 + drivers/accel/ane/Kconfig | 2 + drivers/accel/ane/Makefile | 2 +- drivers/accel/ane/ane.h | 42 +-- drivers/accel/ane/ane_boost.c | 142 ---------- drivers/accel/ane/ane_drv.c | 23 +- drivers/accel/ane/ane_perf.c | 249 ++++++++++++++++++ drivers/accel/ane/ane_tm.c | 5 + drivers/pmdomain/apple/Kconfig | 8 + drivers/pmdomain/apple/Makefile | 1 + drivers/pmdomain/apple/pmp-ane-test.c | 138 ++++++++++ drivers/pmdomain/apple/pmp-ane.h | 96 +++++++ drivers/pmdomain/apple/pmp-report.c | 159 +++++++++++ include/linux/soc/apple/pmp-report.h | 15 ++ 15 files changed, 748 insertions(+), 170 deletions(-) create mode 100644 Documentation/ABI/testing/sysfs-driver-ane-performance delete mode 100644 drivers/accel/ane/ane_boost.c create mode 100644 drivers/accel/ane/ane_perf.c create mode 100644 drivers/pmdomain/apple/pmp-ane-test.c create mode 100644 drivers/pmdomain/apple/pmp-ane.h diff --git a/Documentation/ABI/testing/sysfs-driver-ane-performance b/Documentation/ABI/testing/sysfs-driver-ane-performance new file mode 100644 index 0000000000000..0f28173a24e39 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-ane-performance @@ -0,0 +1,35 @@ +What: /sys/class/accel/accel*/device/performance_level +Date: October 2026 +Contact: asahi@lists.linux.dev +Description: + Requested ANE performance level, 0 through 5, or "unsupported" + when the device tree does not describe a supported controller. + This is a firmware request, not a measured clock frequency. + Work raises the level from the SoC match data. It drops to 0 + after 32 ms without a submission. A stream amortizes the clock + transition cost; the first job after idle includes that cost. + The runtime PM reference covers the entire request lifetime. + +What: /sys/class/accel/accel*/device/memory_vote +Date: October 2026 +Contact: asahi@lists.linux.dev +Description: + Requested average and peak memory bandwidth, in decimal kB/s, + separated by a space. Both are zero after idle release. The + requests are aggregated with other interconnect consumers. + The active values come from the high-level OPP's opp-avg-kBps + and opp-peak-kBps. "unsupported" means no interconnect path is + described. It does not mean a successful zero-bandwidth vote. + Current T600x trees have no host DCS interconnect provider; + describing a path requires a verified shared voting transport. + +What: /sys/class/accel/accel*/device/performance_error +Date: October 2026 +Contact: asahi@lists.linux.dev +Description: + Last performance transition result: zero on success or a + negative errno. Failed busy requests abort before enqueueing. + Failed idle releases retain the runtime PM reference, retry + up to three times, and remain visible here if still failing. + A subsequent workload retries release before requesting high. + An unknown DMA state retains the votes until recovery. diff --git a/MAINTAINERS b/MAINTAINERS index c053af5b9269f..f93d8f60b7726 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8186,6 +8186,7 @@ L: asahi@lists.linux.dev L: dri-devel@lists.freedesktop.org S: Maintained F: Documentation/ABI/testing/sysfs-class-accel-ane +F: Documentation/ABI/testing/sysfs-driver-ane-performance F: Documentation/accel/apple-ane.rst F: Documentation/devicetree/bindings/npu/apple,ane.yaml F: drivers/accel/ane/ diff --git a/drivers/accel/ane/Kconfig b/drivers/accel/ane/Kconfig index c12d49c38a59a..dc65eb5729350 100644 --- a/drivers/accel/ane/Kconfig +++ b/drivers/accel/ane/Kconfig @@ -7,6 +7,8 @@ config DRM_ACCEL_ANE depends on APPLE_RTKIT depends on APPLE_DART || COMPILE_TEST depends on MMU + select INTERCONNECT + select PM_OPP select CRYPTO_LIB_SHA256 help Choose this option if you have an Apple SoC with a Neural Engine: diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile index c4f0954656118..1e80b784e5f6b 100644 --- a/drivers/accel/ane/Makefile +++ b/drivers/accel/ane/Makefile @@ -9,7 +9,7 @@ ane-y := \ ane_drv.o \ ane_tm.o \ ane_dart.o \ - ane_boost.o + ane_perf.o ane_t6021-y := \ ane_fw_platform.o \ diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index f974f8a31194e..c6e471d9f623a 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -7,7 +7,6 @@ #include #include #include -#include #include #include @@ -110,22 +109,24 @@ struct ane_device { */ bool tm_retention; - /* - * Engine-busy CPU cluster boost (ane_boost.c): min-frequency QoS on - * every cpufreq policy while a submit runs and until boost_idle_ms - * after the last one completes: on T8103 bandwidth-bound programs - * run ~2x slower while schedutil parks the idle clusters low. - */ - struct ane_boost { - struct mutex lock; /* protects the fields below */ - struct delayed_work off; - struct freq_qos_request *legs; - int nlegs; - int held; - unsigned long last_end; + /* Protected independently of engine_lock so idle work can release votes. */ + struct ane_perf { + struct mutex lock; /* protects votes, idle work and status */ + struct delayed_work idle; + struct device_node *report; + struct icc_path *memory; + unsigned int high_level; + unsigned int level; + u32 avg_bw; + u32 peak_bw; + int error; + unsigned int release_retries; + bool initialized; bool busy; - bool on; - } boost; + bool voted; + bool clock_held; + bool pinned; + } perf; /* * Producer-side stats (sysfs ane_stats, debugfs ane_timeline; @@ -159,10 +160,11 @@ int ane_submit_check_sizes(const struct drm_ane_submit *args, size_t cmd_size, */ void ane_reclaim_preserved(struct ane_device *ane); -int ane_boost_init(struct ane_device *ane); -void ane_boost_exit(struct ane_device *ane); -void ane_boost_begin(struct ane_device *ane); -void ane_boost_end(struct ane_device *ane); +int ane_perf_init(struct ane_device *ane, unsigned int high_level); +void ane_perf_exit(struct ane_device *ane); +int ane_perf_begin(struct ane_device *ane); +void ane_perf_end(struct ane_device *ane); +int ane_perf_suspend(struct ane_device *ane); int ane_dart_init(struct ane_device *ane); void ane_dart_mask(struct ane_device *ane); diff --git a/drivers/accel/ane/ane_boost.c b/drivers/accel/ane/ane_boost.c deleted file mode 100644 index 179a980a01950..0000000000000 --- a/drivers/accel/ane/ane_boost.c +++ /dev/null @@ -1,142 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* - * Engine-busy CPU cluster boost. - * - * Bandwidth-bound programs run at a speed set by the CPU cluster - * p-states. On T8103, with a mostly-sleeping submitter at normal - * priority, some submits run about twice as long as normal, while the - * submitter's CPU time and run delay stay unchanged; with both clusters - * pinned at their top p-state (performance governor, or SCHED_FIFO for - * the submitter, which schedutil boosts the same way) the slow submits - * disappear. A single long program slows down as well if the QoS - * lapses before it completes, so the hold must last from submit start - * to completion, and the idle delay counts from completion. The likely - * mechanism is the second p-state field apple-soc-cpufreq writes with - * the cluster p-state on T8103 only (DVFS_CMD PS2, has_ps2); the - * memory-controller tuning its author named as unimplemented is still - * absent from the tree. Whatever the field drives, the cure is the - * cluster p-state, which cpufreq lets a driver request. - * - * So hold a min-frequency QoS at the top of every cpufreq policy while a - * submit runs, and drop it boost_idle_ms after the last one completes. - * The QoS goes through cpufreq's own constraint object, so it composes - * with any governor and never touches the DVFS registers. - */ -#include -#include -#include -#include -#include -#include - -#include "ane.h" - -static unsigned int boost_idle_ms = 100; -module_param(boost_idle_ms, uint, 0644); -MODULE_PARM_DESC(boost_idle_ms, - "hold every CPU cluster at its top p-state while a submit runs and this long after the last one completes (0 = off)"); - -/* Lock held. */ -static void ane_boost_set(struct ane_device *ane, bool on) -{ - struct ane_boost *b = &ane->boost; - int cpu, n = 0; - - if (on) { - for_each_possible_cpu(cpu) { - struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); - int err; - - if (!policy) - continue; - if (cpu != cpumask_first(policy->related_cpus) || - n >= b->nlegs) { - cpufreq_cpu_put(policy); - continue; - } - err = freq_qos_add_request(&policy->constraints, - &b->legs[n], FREQ_QOS_MIN, - policy->cpuinfo.max_freq); - cpufreq_cpu_put(policy); - if (err >= 0) - n++; - } - b->held = n; - } else { - for (n = 0; n < b->held; n++) - freq_qos_remove_request(&b->legs[n]); - b->held = 0; - } - b->on = on; -} - -static void ane_boost_off_work(struct work_struct *work) -{ - struct ane_device *ane = - container_of(work, struct ane_device, boost.off.work); - struct ane_boost *b = &ane->boost; - - mutex_lock(&b->lock); - /* A submit that began or ended after this expiry was armed keeps - * the boost on; its own completion re-arms the drop. - */ - if (b->on && !b->busy && - time_after_eq(jiffies, b->last_end + msecs_to_jiffies(boost_idle_ms))) - ane_boost_set(ane, false); - mutex_unlock(&b->lock); -} - -/* Engine lock held: submits never overlap, so busy is a flag. */ -void ane_boost_begin(struct ane_device *ane) -{ - struct ane_boost *b = &ane->boost; - - if (!b->legs || !boost_idle_ms) - return; - mutex_lock(&b->lock); - b->busy = true; - if (!b->on) - ane_boost_set(ane, true); - mutex_unlock(&b->lock); -} - -void ane_boost_end(struct ane_device *ane) -{ - struct ane_boost *b = &ane->boost; - - if (!b->legs) - return; - mutex_lock(&b->lock); - b->busy = false; - b->last_end = jiffies; - if (b->on) - mod_delayed_work(system_dfl_wq, &b->off, - msecs_to_jiffies(boost_idle_ms)); - mutex_unlock(&b->lock); -} - -int ane_boost_init(struct ane_device *ane) -{ - struct ane_boost *b = &ane->boost; - - mutex_init(&b->lock); - INIT_DELAYED_WORK(&b->off, ane_boost_off_work); - b->nlegs = num_possible_cpus(); - b->legs = kcalloc(b->nlegs, sizeof(*b->legs), GFP_KERNEL); - return b->legs ? 0 : -ENOMEM; -} - -void ane_boost_exit(struct ane_device *ane) -{ - struct ane_boost *b = &ane->boost; - - if (!b->legs) - return; - cancel_delayed_work_sync(&b->off); - mutex_lock(&b->lock); - if (b->on) - ane_boost_set(ane, false); - mutex_unlock(&b->lock); - kfree(b->legs); - b->legs = NULL; -} diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 1b61d3f981bd9..b11298181fd94 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -690,17 +690,24 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) goto unlock; } + err = ane_perf_begin(ane); + if (err) { + dev_err_ratelimited(ane->dev, "performance request failed: %d\n", err); + ane_perf_end(ane); + goto unlock; + } err = ane_tm_enqueue(ane, &req); - if (err < 0) + if (err < 0) { + ane_perf_end(ane); goto unlock; + } if (ane->stats_slots) stats_idx = ane_stats_begin(&ane->stats_ctrs, &ane->stats_ring, ktime_get_ns(), req.td_count); - ane_boost_begin(ane); err = ane_tm_execute(ane, &req); - ane_boost_end(ane); + ane_perf_end(ane); if (ane->stats_slots) ane_stats_complete(&ane->stats_ctrs, &ane->stats_ring, @@ -1113,6 +1120,7 @@ struct ane_soc { * retention and recovery must drain it (see ane_tm_drain_retained). */ bool tm_retention; + unsigned int high_level; }; static const struct ane_soc ane_soc_t8103 = { @@ -1128,6 +1136,7 @@ static const struct ane_soc ane_soc_t6000 = { */ .ps_base = 0x28e08c000ULL, .tm_retention = true, + .high_level = 5, }; static const struct of_device_id ane_of_match[] = { @@ -1219,7 +1228,7 @@ static int ane_platform_probe(struct platform_device *pdev) mutex_init(&ane->engine_lock); INIT_LIST_HEAD(&ane->bo_list); INIT_LIST_HEAD(&ane->preserved_list); - err = ane_boost_init(ane); + err = ane_perf_init(ane, soc->high_level); if (err < 0) goto detach_genpd; @@ -1279,7 +1288,7 @@ static int ane_platform_probe(struct platform_device *pdev) pm_runtime_dont_use_autosuspend(dev); drm_mm_takedown(&ane->mm); detach_genpd: - ane_boost_exit(ane); + ane_perf_exit(ane); ane_detach_genpd(ane); return err; } @@ -1310,7 +1319,7 @@ static void ane_platform_remove(struct platform_device *pdev) ane_reclaim_preserved(ane); drm_mm_takedown(&ane->mm); - ane_boost_exit(ane); + ane_perf_exit(ane); ane_detach_genpd(ane); pm_runtime_disable(ane->dev); @@ -1331,7 +1340,7 @@ static int __maybe_unused ane_runtime_suspend(struct device *dev) if (atomic_read(&ane->wedged) && !ane->recovering) return -EBUSY; - return 0; + return ane_perf_suspend(ane); } static int __maybe_unused ane_runtime_resume(struct device *dev) diff --git a/drivers/accel/ane/ane_perf.c b/drivers/accel/ane/ane_perf.c new file mode 100644 index 0000000000000..cc50f899337fb --- /dev/null +++ b/drivers/accel/ane/ane_perf.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Workload-scoped ANE clock and memory bandwidth requests. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane.h" + +/* Amortize clock transitions across a stream of short submissions. */ +#define ANE_PERF_IDLE_MS 32 + +static int ane_perf_drop(struct ane_device *ane) +{ + struct ane_perf *p = &ane->perf; + int ret; + + if (p->clock_held) { + ret = apple_pmp_report_ane_perf(p->report, ane->dev, 0, &p->clock_held); + p->level = 0; + if (ret) + return ret; + } + if (p->voted) { + ret = icc_set_bw(p->memory, 0, 0); + if (ret) + return ret; + p->voted = false; + } + return 0; +} + +static void ane_perf_idle(struct work_struct *work) +{ + struct ane_device *ane = container_of(to_delayed_work(work), + struct ane_device, perf.idle); + struct ane_perf *p = &ane->perf; + + mutex_lock(&p->lock); + if (!p->busy && !atomic_read(&ane->wedged)) { + p->error = ane_perf_drop(ane); + if (!p->error && p->pinned) { + p->pinned = false; + pm_runtime_put_autosuspend(ane->dev); + } + if (p->error && ++p->release_retries < 3) + mod_delayed_work(system_dfl_wq, &p->idle, + msecs_to_jiffies(ANE_PERF_IDLE_MS)); + } + mutex_unlock(&p->lock); +} + +int ane_perf_begin(struct ane_device *ane) +{ + struct ane_perf *p = &ane->perf; + int ret = 0; + + /* The submit ioctl already holds a runtime PM reference. */ + mutex_lock(&p->lock); + p->busy = true; + if (p->error) { + ret = ane_perf_drop(ane); + if (ret) + goto out; + } + if (p->memory && !p->voted) { + ret = icc_set_bw(p->memory, p->avg_bw, p->peak_bw); + if (ret) + goto out; + p->voted = true; + } + if (p->report && !p->clock_held) { + ret = apple_pmp_report_ane_perf(p->report, ane->dev, p->high_level, + &p->clock_held); + if (ret) + goto out; + p->level = p->high_level; + } +out: + p->error = ret; + /* Keep the windows powered until delayed release, including errors. */ + if (!p->pinned && (p->clock_held || p->voted)) { + pm_runtime_get_noresume(ane->dev); + p->pinned = true; + } + mutex_unlock(&p->lock); + return ret; +} + +void ane_perf_end(struct ane_device *ane) +{ + struct ane_perf *p = &ane->perf; + + mutex_lock(&p->lock); + p->busy = false; + p->release_retries = 0; + mod_delayed_work(system_dfl_wq, &p->idle, msecs_to_jiffies(ANE_PERF_IDLE_MS)); + mutex_unlock(&p->lock); +} + +static ssize_t performance_level_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ane_device *ane = dev_get_drvdata(dev); + struct ane_perf *p = &ane->perf; + ssize_t ret; + + mutex_lock(&p->lock); + ret = p->report ? sysfs_emit(buf, "%u\n", p->level) : + sysfs_emit(buf, "unsupported\n"); + mutex_unlock(&p->lock); + return ret; +} +static DEVICE_ATTR_RO(performance_level); + +static ssize_t memory_vote_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ane_device *ane = dev_get_drvdata(dev); + struct ane_perf *p = &ane->perf; + ssize_t ret; + + mutex_lock(&p->lock); + ret = p->memory ? sysfs_emit(buf, "%u %u\n", + p->voted ? p->avg_bw : 0, + p->voted ? p->peak_bw : 0) : + sysfs_emit(buf, "unsupported\n"); + mutex_unlock(&p->lock); + return ret; +} +static DEVICE_ATTR_RO(memory_vote); + +static ssize_t performance_error_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ane_device *ane = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", READ_ONCE(ane->perf.error)); +} +static DEVICE_ATTR_RO(performance_error); + +static struct attribute *ane_perf_attrs[] = { + &dev_attr_performance_level.attr, + &dev_attr_memory_vote.attr, + &dev_attr_performance_error.attr, + NULL, +}; + +static const struct attribute_group ane_perf_group = { .attrs = ane_perf_attrs }; + +int ane_perf_init(struct ane_device *ane, unsigned int high_level) +{ + struct ane_perf *p = &ane->perf; + struct platform_device *supplier; + struct device_link *link; + struct dev_pm_opp *opp; + int ret; + + mutex_init(&p->lock); + p->initialized = true; + INIT_DELAYED_WORK(&p->idle, ane_perf_idle); + p->high_level = high_level; + p->report = of_parse_phandle(ane->dev->of_node, "apple,performance-controller", 0); + if (p->report) { + if (!high_level) + return -EINVAL; + supplier = of_find_device_by_node(p->report); + if (!supplier) + return -EPROBE_DEFER; + link = device_link_add(ane->dev, &supplier->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + put_device(&supplier->dev); + if (!link) + return -ENOMEM; + ret = apple_pmp_report_ane_available(p->report); + if (ret) + return ret; + } + + if (of_property_present(ane->dev->of_node, "interconnects")) { + p->memory = devm_of_icc_get(ane->dev, "memory"); + if (IS_ERR(p->memory)) + return PTR_ERR(p->memory); + if (!p->memory) + return -ENODEV; + ret = devm_pm_opp_of_add_table(ane->dev); + if (ret) + return ret; + opp = dev_pm_opp_find_level_exact(ane->dev, high_level); + if (IS_ERR(opp)) + return PTR_ERR(opp); + p->avg_bw = dev_pm_opp_get_bw(opp, false, 0); + p->peak_bw = dev_pm_opp_get_bw(opp, true, 0); + dev_pm_opp_put(opp); + if (!p->peak_bw || p->avg_bw > p->peak_bw) + return -EINVAL; + } else { + dev_warn(ane->dev, "no memory bandwidth provider; CPU frequency boost removed\n"); + } + return devm_device_add_group(ane->dev, &ane_perf_group); +} + +void ane_perf_exit(struct ane_device *ane) +{ + struct ane_perf *p = &ane->perf; + + if (!p->initialized) + return; + cancel_delayed_work_sync(&p->idle); + mutex_lock(&p->lock); + p->error = ane_perf_drop(ane); + if (p->error) + dev_err(ane->dev, "performance release failed during removal: %d\n", + p->error); + if (p->pinned && !p->error) { + pm_runtime_put_noidle(ane->dev); + p->pinned = false; + } + mutex_unlock(&p->lock); + of_node_put(p->report); + p->report = NULL; +} + +int ane_perf_suspend(struct ane_device *ane) +{ + struct ane_perf *p = &ane->perf; + int ret; + + cancel_delayed_work_sync(&p->idle); + mutex_lock(&p->lock); + if (p->busy && !ane->recovering) { + ret = -EBUSY; + goto out; + } + ret = ane_perf_drop(ane); + p->error = ret; + if (!ret && p->pinned) { + pm_runtime_put_noidle(ane->dev); + p->pinned = false; + } +out: + mutex_unlock(&p->lock); + return ret; +} diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index e4fab5e3225ff..200721e214f46 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -431,6 +431,11 @@ static int ane_pd_cycle(struct ane_device *ane) int err = 0; ane->recovering = true; + err = ane_perf_suspend(ane); + if (err) { + ane->recovering = false; + return err; + } if (ane->pd_count > 1) { int gated = 0; diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index f1293707ca8d2..7526bda60f7af 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -53,6 +53,14 @@ config APPLE_PMP_REPORT apple_t6031_display.enable=1. The start's state is in debugfs, in apple-pmp-report/. +config APPLE_PMP_ANE_KUNIT_TEST + tristate "KUnit tests for T600x ANE performance requests" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Exercise ANE level transitions, ownership refusal and ACK-timeout + rollback against memory-backed registers without Apple hardware. + config APPLE_PMC bool "Apple PMC voter interface" depends on OF diff --git a/drivers/pmdomain/apple/Makefile b/drivers/pmdomain/apple/Makefile index 9f035fd2827f0..b03554cd70051 100644 --- a/drivers/pmdomain/apple/Makefile +++ b/drivers/pmdomain/apple/Makefile @@ -5,3 +5,4 @@ obj-$(CONFIG_APPLE_PMP_REPORT) += pmp-report.o obj-$(CONFIG_APPLE_PMP_REPORT_KUNIT_TEST) += pmp-report-test.o obj-$(CONFIG_APPLE_PMP_DRAM_KUNIT_TEST) += pmp-dram-test.o obj-$(CONFIG_APPLE_PMC) += pmc.o +obj-$(CONFIG_APPLE_PMP_ANE_KUNIT_TEST) += pmp-ane-test.o diff --git a/drivers/pmdomain/apple/pmp-ane-test.c b/drivers/pmdomain/apple/pmp-ane-test.c new file mode 100644 index 0000000000000..66b196b178cc0 --- /dev/null +++ b/drivers/pmdomain/apple/pmp-ane-test.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "pmp-ane.h" + +struct ane_test { + struct apple_pmp_ane ane; + int requests; + int releases; + int request_error; + int release_error; + struct kunit *test; +}; + +static int ane_test_request(void *ctx, bool enable) +{ + struct ane_test *t = ctx; + + if (enable) { + KUNIT_EXPECT_EQ(t->test, readl(t->ane.dvfs + PMP_ANE_DVFS_ON), 1U); + KUNIT_EXPECT_EQ(t->test, readl(t->ane.dvfs + PMP_ANE_DVFS_CMD), 0x80000000U); + t->requests++; + t->ane.owned = true; + return t->request_error; + } + t->releases++; + if (t->release_error) + return t->release_error; + t->ane.owned = false; + return 0; +} + +static struct ane_test *ane_test_init(struct kunit *test) +{ + struct ane_test *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + + if (!t) + return NULL; + t->test = test; + t->ane.dvfs = (void __iomem *)kunit_kzalloc(test, 0x4000, GFP_KERNEL); + t->ane.report = (void __iomem *)kunit_kzalloc(test, 0x20000, GFP_KERNEL); + if (!t->ane.dvfs || !t->ane.report) + return NULL; + return t; +} + +static void ane_level_cycle(struct kunit *test) +{ + struct ane_test *t = ane_test_init(test); + + KUNIT_ASSERT_NOT_NULL(test, t); + KUNIT_ASSERT_EQ(test, apple_pmp_ane_set(&t->ane, 5, ane_test_request, t), 0); + KUNIT_EXPECT_EQ(test, t->ane.level, 5U); + KUNIT_EXPECT_EQ(test, readl(t->ane.dvfs + PMP_ANE_DVFS_CMD), 0x80000005U); + KUNIT_EXPECT_EQ(test, readq(t->ane.report + PMP_ANE_LEVEL_WRITE), PMP_ANE_LEVEL_TAG | 5); + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 5, ane_test_request, t), 0); + KUNIT_EXPECT_EQ(test, t->requests, 1); + KUNIT_ASSERT_EQ(test, apple_pmp_ane_set(&t->ane, 0, ane_test_request, t), 0); + KUNIT_EXPECT_EQ(test, t->ane.level, 0U); + KUNIT_EXPECT_FALSE(test, t->ane.owned); + KUNIT_EXPECT_EQ(test, t->releases, 1); + KUNIT_EXPECT_EQ(test, readl(t->ane.dvfs + PMP_ANE_DVFS_ON), 0U); + KUNIT_EXPECT_EQ(test, readl(t->ane.dvfs + PMP_ANE_DVFS_CMD), 0x80000000U); + KUNIT_EXPECT_EQ(test, readq(t->ane.report + PMP_ANE_LEVEL_WRITE), PMP_ANE_LEVEL_TAG); + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 0, ane_test_request, t), 0); + KUNIT_EXPECT_EQ(test, t->releases, 1); +} + +static void ane_level_busy(struct kunit *test) +{ + struct ane_test *t = ane_test_init(test); + + KUNIT_ASSERT_NOT_NULL(test, t); + writel(1, t->ane.dvfs + PMP_ANE_DVFS_ON); + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 5, ane_test_request, t), -EBUSY); + KUNIT_EXPECT_EQ(test, t->requests, 0); + writel(0, t->ane.dvfs + PMP_ANE_DVFS_ON); + writeq(0xdead, t->ane.report + PMP_ANE_LEVEL_READ); + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 5, ane_test_request, t), -EBUSY); + KUNIT_EXPECT_EQ(test, readl(t->ane.dvfs + PMP_ANE_DVFS_ON), 0U); + KUNIT_EXPECT_EQ(test, t->requests, 0); + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 6, ane_test_request, t), -EINVAL); +} + +static void ane_level_timeout(struct kunit *test) +{ + struct ane_test *t = ane_test_init(test); + int i; + + KUNIT_ASSERT_NOT_NULL(test, t); + t->request_error = -ETIMEDOUT; + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, + apple_pmp_ane_set(&t->ane, 5, ane_test_request, t), + -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, t->ane.level, 0U); + KUNIT_EXPECT_EQ(test, readl(t->ane.dvfs + PMP_ANE_DVFS_ON), 0U); + KUNIT_EXPECT_FALSE(test, t->ane.owned); + } + KUNIT_EXPECT_EQ(test, t->requests, 3); + KUNIT_EXPECT_EQ(test, t->releases, 3); +} + +static void ane_level_release_timeout(struct kunit *test) +{ + struct ane_test *t = ane_test_init(test); + + KUNIT_ASSERT_NOT_NULL(test, t); + KUNIT_ASSERT_EQ(test, apple_pmp_ane_set(&t->ane, 5, ane_test_request, t), 0); + t->release_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 0, ane_test_request, t), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, t->ane.level, 0U); + KUNIT_EXPECT_EQ(test, t->ane.owned, true); + KUNIT_EXPECT_EQ(test, readl(t->ane.dvfs + PMP_ANE_DVFS_ON), 0U); + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 5, ane_test_request, t), -EBUSY); + t->release_error = 0; + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 0, ane_test_request, t), 0); + KUNIT_EXPECT_FALSE(test, t->ane.owned); + KUNIT_EXPECT_EQ(test, t->releases, 2); +} + +static struct kunit_case ane_level_cases[] = { + KUNIT_CASE(ane_level_cycle), + KUNIT_CASE(ane_level_busy), + KUNIT_CASE(ane_level_timeout), + KUNIT_CASE(ane_level_release_timeout), + {} +}; + +static struct kunit_suite ane_level_suite = { + .name = "apple-pmp-ane", + .test_cases = ane_level_cases, +}; + +kunit_test_suite(ane_level_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/pmdomain/apple/pmp-ane.h b/drivers/pmdomain/apple/pmp-ane.h new file mode 100644 index 0000000000000..e8e3838b042da --- /dev/null +++ b/drivers/pmdomain/apple/pmp-ane.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * T600x ANE level sequence, based on the public omarchy-ane implementation + * by Joshua Warren (ane/src/ane_dvfs.c, e20d25defb78). + * The report provider owns both windows and serializes request-word updates. + */ +#ifndef __APPLE_PMP_ANE_H__ +#define __APPLE_PMP_ANE_H__ + +#include +#include +#include + +#define PMP_ANE_DVFS_ON 0x2000 +#define PMP_ANE_DVFS_CMD 0xa00 +#define PMP_ANE_LEVEL_READ 0x10c0 +#define PMP_ANE_LEVEL_WRITE 0x10860 +#define PMP_ANE_LEVEL_TAG 0x2000000000000000ULL +#define PMP_ANE_MAX_LEVEL 5 +#define PMP_ANE_ID 10 + +struct apple_pmp_ane { + void __iomem *dvfs; + void __iomem *report; + unsigned int level; + unsigned int timeouts; + bool owned; + bool reserved; +}; + +static void apple_pmp_ane_cmd(struct apple_pmp_ane *ane, unsigned int old, + unsigned int new) +{ + writel(0x80000000U | old << 4 | new, ane->dvfs + PMP_ANE_DVFS_CMD); + usleep_range(1000, 2000); +} + +/* Caller serializes transitions and pins the ANE and report power domains. */ +static int apple_pmp_ane_set(struct apple_pmp_ane *ane, unsigned int level, + int (*request)(void *ctx, bool enable), void *ctx) +{ + u64 word; + int ret = 0; + + if (!ane->dvfs) + return -EOPNOTSUPP; + if (level > PMP_ANE_MAX_LEVEL) + return -EINVAL; + if (level == ane->level && (level || !ane->owned)) + return 0; + if (!level && !ane->level) + goto release; + if (level && !ane->level && ane->owned) + return -EBUSY; + if (level && ane->timeouts >= 3) + return -ETIMEDOUT; + + if (!ane->level) { + if (readl(ane->dvfs + PMP_ANE_DVFS_ON)) + return -EBUSY; + word = readq(ane->report + PMP_ANE_LEVEL_READ); + if (word && (word < PMP_ANE_LEVEL_TAG || + word > (PMP_ANE_LEVEL_TAG | PMP_ANE_MAX_LEVEL))) + return -EBUSY; + + writel(1, ane->dvfs + PMP_ANE_DVFS_ON); + usleep_range(1000, 2000); + apple_pmp_ane_cmd(ane, 0, 0); + ret = request(ctx, true); + if (ret) { + if (ret == -ETIMEDOUT) + ane->timeouts++; + goto release; + } + } + + apple_pmp_ane_cmd(ane, ane->level, level); + writeq(PMP_ANE_LEVEL_TAG | level, ane->report + PMP_ANE_LEVEL_WRITE); + ane->level = level; + if (level) + return 0; + usleep_range(1000, 2000); + +release: + if (ane->owned) { + int err = request(ctx, false); + + if (!ret) + ret = err; + } + apple_pmp_ane_cmd(ane, 0, 0); + writel(0, ane->dvfs + PMP_ANE_DVFS_ON); + return ret; +} + +#endif diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 1641181d57ed7..f09b38aa310ca 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -33,6 +33,7 @@ #include "pmp-report-validation.h" #include "pmp-dram.h" +#include "pmp-ane.h" #define PMP_REPORT_READY 0x1 @@ -83,6 +84,8 @@ struct apple_pmp_start; struct apple_pmp_report { struct device *dev; struct apple_pmp_dram dram; + struct apple_pmp_ane ane; + struct device *ane_owner; const struct apple_pmp_report_offsets *offsets; void __iomem *base; /* Protects the request word, @shadow, @known and @defer. */ @@ -149,6 +152,141 @@ static const struct apple_pmp_report_offsets apple_pmp_offsets_t6030; /* Bound report regions, protected by RCU until managed teardown drains readers. */ static LIST_HEAD(apple_pmp_reports); static DEFINE_MUTEX(apple_pmp_reports_lock); + +/* The report lock also protects ANE_SYS against a genpd owner appearing. */ +static int apple_pmp_ane_request(void *ctx, bool enable) +{ + struct apple_pmp_report *rep = ctx; + struct apple_pmp_ane *ane = &rep->ane; + const u64 bit = BIT_ULL(PMP_ANE_ID); + unsigned long flags; + u64 val; + + spin_lock_irqsave(&rep->lock, flags); + val = readq(rep->base + rep->offsets->tgt_read); + if (enable) { + if ((rep->known | val) & bit) { + spin_unlock_irqrestore(&rep->lock, flags); + return -EBUSY; + } + ane->owned = true; + val |= bit; + } else { + val &= ~bit; + } + writeq(val, rep->base + rep->offsets->tgt_write); + spin_unlock_irqrestore(&rep->lock, flags); + if (!enable) { + int ret = readq_poll_timeout(rep->base + rep->offsets->actual, val, + !(val & bit), 200, 50000); + + if (!ret) { + spin_lock_irqsave(&rep->lock, flags); + ane->owned = false; + spin_unlock_irqrestore(&rep->lock, flags); + } + return ret; + } + return readq_poll_timeout(rep->base + rep->offsets->actual, val, + val & bit, 200, 50000); +} + +/* Probe-time availability check: deliberately no MMIO while the ANE is off. */ +int apple_pmp_report_ane_available(const struct device_node *np) +{ + struct apple_pmp_report *rep; + int ret = -EPROBE_DEFER; + + mutex_lock(&apple_pmp_reports_lock); + list_for_each_entry(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node != np) + continue; + ret = !rep->ane.dvfs ? -EOPNOTSUPP : + (rep->ane_owner || rep->ane.reserved) ? -EBUSY : 0; + break; + } + mutex_unlock(&apple_pmp_reports_lock); + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_ane_available); + +/** + * apple_pmp_report_ane_perf() - request a T600x ANE performance level + * @np: report provider node + * @owner: ANE device holding the request + * @level: 0 to release, 1 through 5 to request a level + * @held: whether the caller still owns a request, including failed rollback + * + * The caller must keep its engine and the report SRAM powered throughout + * the request, and link itself to the report supplier. Other generations + * and reports without an ANE DVFS resource are unsupported. The returned + * level is a firmware request, not a measured frequency. + * + * Context: Process context. May sleep. + * Return: 0 on success, or a negative errno. Ownership and ACK failures + * are returned to the submitter; no CPU frequency request is substituted. + */ +int apple_pmp_report_ane_perf(const struct device_node *np, + struct device *owner, unsigned int level, bool *held) +{ + struct apple_pmp_report *rep; + int ret = -EPROBE_DEFER; + + if (!np || !owner || !held) + return -EINVAL; + *held = false; + mutex_lock(&apple_pmp_reports_lock); + list_for_each_entry(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node != np) + continue; + if (!rep->ane.dvfs) { + ret = -EOPNOTSUPP; + break; + } + if (rep->ane_owner && rep->ane_owner != owner) { + ret = -EBUSY; + break; + } + if (level && !apple_pmp_report_running(rep)) { + ret = -EAGAIN; + break; + } + /* Claim before touching DVFS, and exclude genpd for the whole cycle. */ + if (level && !rep->ane.reserved) { + unsigned long flags; + u64 req; + + spin_lock_irqsave(&rep->lock, flags); + req = readq(rep->base + rep->offsets->tgt_read); + if ((rep->known | req) & BIT_ULL(PMP_ANE_ID)) { + spin_unlock_irqrestore(&rep->lock, flags); + ret = -EBUSY; + break; + } + rep->ane.reserved = true; + spin_unlock_irqrestore(&rep->lock, flags); + } + ret = apple_pmp_ane_set(&rep->ane, level, apple_pmp_ane_request, rep); + if (!rep->ane.level && !rep->ane.owned) { + unsigned long flags; + + spin_lock_irqsave(&rep->lock, flags); + rep->ane.reserved = false; + spin_unlock_irqrestore(&rep->lock, flags); + } + /* Failed rollback retains ownership until a successful retry. */ + if (rep->ane.level || rep->ane.owned) + rep->ane_owner = owner; + else + rep->ane_owner = NULL; + *held = rep->ane_owner == owner; + break; + } + mutex_unlock(&apple_pmp_reports_lock); + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_ane_perf); + static struct dentry *apple_pmp_report_debugfs; /** @@ -1182,6 +1320,23 @@ static int apple_pmp_report_probe(struct platform_device *pdev) if (IS_ERR(rep->base)) return PTR_ERR(rep->base); rep->offsets = of_device_get_match_data(dev); + if (of_property_match_string(np, "reg-names", "ane-dvfs") >= 0) { + struct resource *res; + + if (!of_device_is_compatible(np, "apple,t6000-pmp-v2-report")) + return -EINVAL; + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res || resource_size(res) != SZ_128K) + return -EINVAL; + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ane-dvfs"); + if (!res || resource_size(res) != SZ_16K) + return -EINVAL; + rep->ane.dvfs = devm_ioremap_resource(dev, res); + if (IS_ERR(rep->ane.dvfs)) + return PTR_ERR(rep->ane.dvfs); + rep->ane.report = rep->base; + } + rep->defer = rep->offsets->defer_requests; INIT_DELAYED_WORK(&rep->ready_work, apple_pmp_report_ready_work); platform_set_drvdata(pdev, rep); @@ -1386,6 +1541,10 @@ static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id, bool ready; spin_lock_irqsave(&rep->lock, flags); + if (id == PMP_ANE_ID && rep->ane.reserved) { + spin_unlock_irqrestore(&rep->lock, flags); + return -EBUSY; + } rep->known |= bit_val; if (enable) rep->shadow |= bit_val; diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h index 1bfb4fc69f1f7..5e819681ba106 100644 --- a/include/linux/soc/apple/pmp-report.h +++ b/include/linux/soc/apple/pmp-report.h @@ -14,11 +14,26 @@ struct device_node; struct device; #if IS_ENABLED(CONFIG_APPLE_PMP_REPORT) +int apple_pmp_report_ane_available(const struct device_node *np); +int apple_pmp_report_ane_perf(const struct device_node *np, + struct device *owner, unsigned int level, bool *held); int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int timeout_ms); int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout); int apple_pmp_report_fast_die_effort(const struct device_node *np, unsigned int lane, u8 *effort); #else +static inline int apple_pmp_report_ane_available(const struct device_node *np) +{ + return -EOPNOTSUPP; +} + +static inline int apple_pmp_report_ane_perf(const struct device_node *np, + struct device *owner, unsigned int level, bool *held) +{ + *held = false; + return -EOPNOTSUPP; +} + static inline int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int timeout_ms) { From 659e2ccef5c9889e6173314cf70f02feaa81103c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:36:44 +1100 Subject: [PATCH 1504/1579] arm64: dts: apple: wire M1 Pro and Max ANE level control Describe the ANE DVFS window on the shared PMP report provider and keep its SRAM powered while the ANE holds a level request. Leave other SoC layouts untouched. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6000.dtsi | 12 ++++++++++++ arch/arm64/boot/dts/apple/t6001.dtsi | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index fc1c92bc0ea8f..75b3fcf23b573 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -78,3 +78,15 @@ apple,peak-bandwidth-bytes-per-second = /bits/ 64 <204800000000>; apple,bandwidth-class-bytes-per-second = /bits/ 64 <200000000000>; }; + +&pmp_report { + reg = <0x2 0x8e3c0000 0x0 0x20000>, + <0x2 0x85868000 0x0 0x4000>; + reg-names = "report", "ane-dvfs"; +}; + +&ane { + apple,performance-controller = <&pmp_report>; + power-domains = <&ps_ane_sys_cpu>, <&ps_ane_set1>, + <&ps_ane_set2>, <&ps_ane_set3>, <&ps_ane_set4>, <&ps_pms_sram>; +}; diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index 5da33aa0db9df..e617f7f3f41a9 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -136,3 +136,15 @@ &ps_ane_set4 { status = "okay"; }; + +&pmp_report { + reg = <0x2 0x8e3c0000 0x0 0x20000>, + <0x2 0x85868000 0x0 0x4000>; + reg-names = "report", "ane-dvfs"; +}; + +&ane { + apple,performance-controller = <&pmp_report>; + power-domains = <&ps_ane_sys_cpu>, <&ps_ane_set1>, + <&ps_ane_set2>, <&ps_ane_set3>, <&ps_ane_set4>, <&ps_pms_sram>; +}; From 7dce9d0a334fc23f8567b6c2d135451ca5b9d363 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:36:44 +1100 Subject: [PATCH 1505/1579] tools: accel: collect ANE power receipts and diagnose device access Capture workload output, whole-system power and idle recovery without changing driver policy. Identify EACCES and EPERM by device path and owning group. Mark short load samples as insufficient for a power comparison. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + tools/accel/ane-bench.py | 131 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100755 tools/accel/ane-bench.py diff --git a/MAINTAINERS b/MAINTAINERS index f93d8f60b7726..66199eac34ab5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8191,6 +8191,7 @@ F: Documentation/accel/apple-ane.rst F: Documentation/devicetree/bindings/npu/apple,ane.yaml F: drivers/accel/ane/ F: include/uapi/drm/ane_accel.h +F: tools/accel/ane-bench.py F: tools/testing/selftests/drivers/ane/ DRM ACCEL DRIVERS FOR INTEL VPU diff --git a/tools/accel/ane-bench.py b/tools/accel/ane-bench.py new file mode 100755 index 0000000000000..eff0879d5eabc --- /dev/null +++ b/tools/accel/ane-bench.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Collect ANE workload output, system power and idle recovery in one receipt.""" +import argparse +import errno +import grp +import json +import os +from pathlib import Path +import platform +import statistics +import subprocess +import sys +import tempfile +import time + + +def check_device(node): + try: + fd = os.open(node, os.O_RDWR | os.O_CLOEXEC) + except OSError as exc: + if exc.errno in (errno.EACCES, errno.EPERM): + try: + group = grp.getgrgid(Path(node).stat().st_gid).gr_name + except (OSError, KeyError): + group = "the device's owning group" + raise RuntimeError( + f"Permission denied opening {node}: request access through " + f"the installed device-access rule or membership in {group}; " + "start a new login after changing group membership." + ) from exc + raise RuntimeError(f"Cannot open {node}: {exc.strerror}") from exc + else: + os.close(fd) + + +def power_sensor(): + for hwmon in Path("/sys/class/hwmon").glob("hwmon*"): + try: + if (hwmon / "name").read_text().strip() != "macsmc_hwmon": + continue + for label in hwmon.glob("power*_label"): + if label.read_text().strip().lower() == "total system power": + return label.with_name(label.name.replace("_label", "_input")) + except OSError: + continue + raise RuntimeError("Total System Power sensor unavailable in macsmc_hwmon") + + +def snapshot(device): + result = {} + for name in ("performance_level", "memory_vote", "performance_error", + "power/runtime_status", "ane_stats"): + try: + result[name] = (device / name).read_text().strip() + except OSError: + result[name] = None + return result + + +def sample_power(sensor, seconds=None, process=None): + samples = [] + start = time.monotonic() + while ((process is None and time.monotonic() - start < seconds) or + (process is not None and process.poll() is None)): + samples.append({"seconds": time.monotonic() - start, + "watts": int(sensor.read_text()) / 1_000_000}) + time.sleep(0.5) + values = [s["watts"] for s in samples] + return {"duration_s": time.monotonic() - start, "samples": samples, + "mean_W": statistics.mean(values) if values else None} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--device", default="/dev/accel/accel0") + parser.add_argument("--idle-seconds", type=float, default=10) + parser.add_argument("--check", action="store_true", help="check device access only") + parser.add_argument("command", nargs=argparse.REMAINDER, + help="-- followed by an installed ANE workload command") + args = parser.parse_args() + if args.idle_seconds <= 0: + parser.error("--idle-seconds must be positive") + try: + check_device(args.device) + if args.check: + print(f"Access to {args.device}: OK") + return 0 + command = args.command + if command and command[0] == "--": + command = command[1:] + if not command: + parser.error("supply a workload command after --, or use --check") + sensor = power_sensor() + time.sleep(3) # Let the access check's runtime-PM reference expire. + device = Path("/sys/class/accel") / Path(args.device).name / "device" + receipt = {"schema": 1, "kernel": platform.release(), "command": command, + "power_sensor": str(sensor), "before": snapshot(device)} + receipt["idle_before"] = sample_power(sensor, args.idle_seconds) + # A file avoids a full stdout pipe stalling a long-running workload. + with tempfile.TemporaryFile(mode="w+") as output: + with subprocess.Popen(command, stdout=output, stderr=subprocess.STDOUT) as proc: + try: + receipt["load"] = sample_power(sensor, process=proc) + receipt["returncode"] = proc.wait() + except BaseException: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + raise + output.seek(0) + receipt["workload_output"] = output.read() + receipt["power_comparison_valid"] = len(receipt["load"]["samples"]) >= 6 + if not receipt["power_comparison_valid"]: + receipt["power_note"] = "Fewer than 6 load samples; repeat with a longer workload." + receipt["after_load"] = snapshot(device) + time.sleep(3) # Allow the default runtime-PM delay to expire. + receipt["idle_after"] = sample_power(sensor, args.idle_seconds) + receipt["after_idle"] = snapshot(device) + print(json.dumps(receipt, indent=2)) + return 0 if receipt["returncode"] == 0 else 1 + except (RuntimeError, OSError) as exc: + print(f"ane-bench: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) From 938b392bde757b80da2d46b4e9937b4b126f4185 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:39:26 +1100 Subject: [PATCH 1506/1579] accel/ane: validate the H16 task stream prologue H16 streams begin with little-endian u32 1 followed by twelve reserved zero bytes. Requiring sixteen zero bytes rejects the existing compiler output before the task walk. Require the exact marker and keep the reserved-byte, framing, register and relocation checks. Cover zero, swapped and unexpected markers, every reserved byte and truncated streams in KUnit. The fixed marker is present in plugins/H16G/Encoding/H16GALUEncoder.mm at public compiler revision 927013f5480768b07c8e62a866e1514c41aba778. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 8 +++++++ drivers/accel/ane/ane_h16_tasks.h | 8 +++++-- drivers/accel/ane/ane_h16_tasks_test.c | 31 ++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index ef546815b85eb..a50b7da173de7 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -61,6 +61,14 @@ start ordering. It does not invent responses for unknown message types or unimplemented system endpoint buffer requests. Its caller must provide one absolute session deadline and a bounded 64-bit mailbox transport. +H16 task streams begin with little-endian ``u32 1`` and twelve reserved zero +bytes. This marker precedes the first aligned task; it is not a task count. +The initial H16 gate rejects a zero marker, other values and nonzero reserved +bytes. The public H16 encoder fixes this prologue at commit +``927013f5480768b07c8e62a866e1514c41aba778`` of +``joshuaswarren/mil-hwx-compiler``, in +``plugins/H16G/Encoding/H16GALUEncoder.mm``. + H16 validation helpers walk nine fixed task-header words, the documented conditional extra word, dense/scatter records and the nine allowed register blocks. A separate helper checks the eight-byte Mach-O relocation entries, diff --git a/drivers/accel/ane/ane_h16_tasks.h b/drivers/accel/ane/ane_h16_tasks.h index 8415e692c5239..c270d748fbf0d 100644 --- a/drivers/accel/ane/ane_h16_tasks.h +++ b/drivers/accel/ane/ane_h16_tasks.h @@ -33,7 +33,10 @@ static inline bool ane_h16_register_valid(u32 word) return false; } -/* This structural gate does not replace container/tensor/aperture validation. */ +/* + * H16 streams begin with little-endian u32 1 and twelve reserved zero bytes. + * This structural gate does not replace container/tensor/aperture validation. + */ static inline int ane_h16_task_stream_validate(const void *text, size_t size, u32 expected_tasks) { @@ -42,7 +45,8 @@ ane_h16_task_stream_validate(const void *text, size_t size, u32 expected_tasks) u32 tasks = 0; if (!p || !expected_tasks || size < 16 || size > ANE_H16_TASK_MAX || - (size & 3) || memchr_inv(p, 0, 16)) + (size & 3) || get_unaligned_le32(p) != 1 || + memchr_inv(p + 4, 0, 12)) return -ENOEXEC; while (pos < size) { u32 words, header_words = 9; diff --git a/drivers/accel/ane/ane_h16_tasks_test.c b/drivers/accel/ane/ane_h16_tasks_test.c index 321e7bb0ffb3c..f11aab0119c10 100644 --- a/drivers/accel/ane/ane_h16_tasks_test.c +++ b/drivers/accel/ane/ane_h16_tasks_test.c @@ -8,6 +8,7 @@ static void ane_h16_dense_task_test(struct kunit *test) { u8 text[64] = {}; + put_unaligned_le32(1, text); /* Nine header words, one dense record and its one-word payload. */ put_unaligned_le32(11 << 16, text + 16); put_unaligned_le32(0x4100 / 4, text + 16 + 9 * 4); @@ -26,6 +27,7 @@ static void ane_h16_scatter_task_test(struct kunit *test) { u8 text[64] = {}; + put_unaligned_le32(1, text); put_unaligned_le32(12 << 16, text + 16); put_unaligned_le32(BIT(31) | BIT(15) | 21, text + 16 + 9 * 4); KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), 0); @@ -38,6 +40,7 @@ static void ane_h16_extra_header_test(struct kunit *test) { u8 text[64] = {}; + put_unaligned_le32(1, text); put_unaligned_le32(10 << 16, text + 16); put_unaligned_le32(3, text + 16 + 8 * 4); KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), 0); @@ -48,6 +51,33 @@ static void ane_h16_extra_header_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, ane_h16_register_valid(0x5930 / 4)); } +static void ane_h16_prologue_test(struct kunit *test) +{ + u8 text[64] = {}; + unsigned int i; + + put_unaligned_le32(11 << 16, text + 16); + put_unaligned_le32(0x4100 / 4, text + 16 + 9 * 4); + /* A zero prologue is not the H16 stream marker. */ + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + put_unaligned_le32(1, text); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), 0); + put_unaligned_le32(2, text); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + put_unaligned_le32(0x01000000, text); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + put_unaligned_le32(1, text); + for (i = 4; i < 16; i++) { + text[i] = 1; + KUNIT_EXPECT_EQ(test, + ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + text[i] = 0; + } + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, 15, 1), -ENOEXEC); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, 16, 1), -ENOEXEC); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(NULL, sizeof(text), 1), -ENOEXEC); +} + static void ane_h16_relocation_test(struct kunit *test) { u8 text[64] = {}; @@ -86,6 +116,7 @@ static void ane_h16_relocation_test(struct kunit *test) } static struct kunit_case ane_h16_task_cases[] = { + KUNIT_CASE(ane_h16_prologue_test), KUNIT_CASE(ane_h16_dense_task_test), KUNIT_CASE(ane_h16_scatter_task_test), KUNIT_CASE(ane_h16_extra_header_test), From 36ff74fa0af96e2f023624720b512e95a90319f1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:42:08 +1100 Subject: [PATCH 1507/1579] media: apple: ave: validate firmware before initial supplier release Validate the preloaded firmware and its pristine DATA image before reconciling the initial supplier references. No encoder registers are needed for that validation, and runtime PM can remain disabled until it has succeeded. Release the validated firmware copy on later PM initialization failures, and clear autosuspend use on registration failure. Extend the production probe harness to check that failed validation never releases suppliers or enables runtime PM, and that later failures free the firmware copy. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/ave-drv.c | 24 +++++++++++++--------- tools/asahi/ave-probe-pm-tests.py | 19 +++++++++++------ 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c index a318ab6b25bcc..76527c670ed7e 100644 --- a/drivers/media/platform/apple/ave/ave-drv.c +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -156,6 +156,10 @@ static int ave_probe(struct platform_device *pdev) if (ret < 0) return dev_err_probe(dev, ret, "cannot attach power domains\n"); + ret = ave_fw_probe(ave); + if (ret) + return ret; + /* * Domain attachment took active supplier references, but the driver * starts firmware only on first use and begins suspended. Reconcile that @@ -164,25 +168,25 @@ static int ave_probe(struct platform_device *pdev) */ ret = pm_runtime_set_suspended(dev); if (ret) - return ret; + goto err_fw; pm_runtime_set_autosuspend_delay(dev, AVE_AUTOSUSPEND_MS); pm_runtime_use_autosuspend(dev); ret = devm_pm_runtime_enable(dev); if (ret) - return ret; - - ret = ave_fw_probe(ave); - if (ret) - return ret; + goto err_pm; ret = ave_v4l2_register(ave); - if (ret) { - ave_fw_remove(ave); - return ret; - } + if (ret) + goto err_pm; ave_debugfs_init(ave); return 0; + +err_pm: + pm_runtime_dont_use_autosuspend(dev); +err_fw: + ave_fw_remove(ave); + return ret; } static void ave_remove(struct platform_device *pdev) diff --git a/tools/asahi/ave-probe-pm-tests.py b/tools/asahi/ave-probe-pm-tests.py index dcb9e02de2115..417eecdb575df 100644 --- a/tools/asahi/ave-probe-pm-tests.py +++ b/tools/asahi/ave-probe-pm-tests.py @@ -73,6 +73,7 @@ static int pm_runtime_set_suspended(struct device *d) { assert(!d->enabled); + assert(fw_probes == 1 && !fw_error); state_updates++; if (state_error) return state_error; d->suspended = true; @@ -82,6 +83,7 @@ static void pm_runtime_set_autosuspend_delay(struct device *d, unsigned delay) { (void)d; assert(delay == 2000); } static void pm_runtime_use_autosuspend(struct device *d) { (void)d; } +static void pm_runtime_dont_use_autosuspend(struct device *d) { (void)d; } static int devm_pm_runtime_enable(struct device *d) { assert(!d->enabled); @@ -89,7 +91,12 @@ d->enabled = true; return 0; } -static int ave_fw_probe(struct ave_dev *a) { (void)a; fw_probes++; return fw_error; } +static int ave_fw_probe(struct ave_dev *a) +{ + assert(!a->dev->enabled && a->dev->supplier_refs == 7); + fw_probes++; + return fw_error; +} static void ave_fw_remove(struct ave_dev *a) { (void)a; fw_removes++; } static int ave_v4l2_register(struct ave_dev *a) { @@ -132,15 +139,15 @@ reset(&p); state_error = -EBUSY; assert(ave_probe(&p) == -EBUSY); - assert(!p.dev.enabled && !fw_probes && !registrations); + assert(!p.dev.enabled && fw_probes == 1 && fw_removes == 1 && !registrations); reset(&p); enable_error = -ENOMEM; assert(ave_probe(&p) == -ENOMEM); - assert(!p.dev.supplier_refs && !fw_probes && !registrations); + assert(!p.dev.supplier_refs && fw_probes == 1 && fw_removes == 1 && !registrations); reset(&p); fw_error = -EINVAL; assert(ave_probe(&p) == -EINVAL); - assert(!p.dev.supplier_refs && !registrations); + assert(p.dev.supplier_refs == 7 && !state_updates && !p.dev.enabled && !registrations); reset(&p); register_error = -EIO; assert(ave_probe(&p) == -EIO); @@ -161,10 +168,10 @@ def main(): start = source.index('static int ave_probe(') probe = source[start:source.index('\n}', start) + 2] if args.negative_control: - old = '\tret = pm_runtime_set_suspended(dev);\n\tif (ret)\n\t\treturn ret;\n' + old = '\tret = pm_runtime_set_suspended(dev);' if probe.count(old) != 1: raise SystemExit('cannot locate the production fix for the negative control') - probe = probe.replace(old, '') + probe = probe.replace(old, '\tret = 0;') with tempfile.TemporaryDirectory(prefix='ave-probe-pm-') as tmp: out = args.out or Path(tmp) out.mkdir(parents=True, exist_ok=True) From b3f977e3013b906c32654177bfe7178b94e0b941 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:44:27 +1100 Subject: [PATCH 1508/1579] pwm: apple: Runtime suspend disabled outputs The Apple PWM driver keeps its clock enabled and never enables runtime PM, so an unused controller cannot release its power-domain reference. Add runtime PM around register accesses and retain a reference while an output is enabled, including enabled zero-duty outputs. Adopt an output left running by the bootloader without reprogramming it. Save disabled controller registers before clock gating and restore them after resume so get_state remains meaningful after domain context loss. Balance clock and PM references on probe failure and managed teardown. Exercise the actual callbacks with a host PM/MMIO model, including register loss, repeated apply/get_state, another consumer reference, resume errors, both boot states and cleanup failures. This does not qualify physical domain gating or output levels on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/pwm/pwm-apple.c | 106 ++++++++++++- tools/asahi/apple-pwm-pm-tests.py | 247 ++++++++++++++++++++++++++++++ 3 files changed, 349 insertions(+), 5 deletions(-) create mode 100644 tools/asahi/apple-pwm-pm-tests.py diff --git a/MAINTAINERS b/MAINTAINERS index c053af5b9269f..9cfabe37ab022 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2782,6 +2782,7 @@ F: drivers/pmdomain/apple/ F: drivers/power/reset/macsmc-reboot.c F: drivers/power/supply/macsmc-power*.c F: drivers/pwm/pwm-apple.c +F: tools/asahi/apple-pwm-pm-tests.py F: drivers/reset/reset-apple-cio.c F: drivers/rtc/rtc-macsmc.c F: drivers/soc/apple/* diff --git a/drivers/pwm/pwm-apple.c b/drivers/pwm/pwm-apple.c index 72e4f23d2315b..2659f777fafbe 100644 --- a/drivers/pwm/pwm-apple.c +++ b/drivers/pwm/pwm-apple.c @@ -18,6 +18,7 @@ #include #include #include +#include #define APPLE_PWM_CTRL 0x00 #define APPLE_PWM_ON_CYCLES 0x1c @@ -32,7 +33,12 @@ struct apple_pwm { void __iomem *base; + struct clk *clk; u64 clkrate; + u32 on_cycles; + u32 off_cycles; + u32 ctrl; + bool enabled; }; static inline struct apple_pwm *to_apple_pwm(struct pwm_chip *chip) @@ -44,11 +50,17 @@ static int apple_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { struct apple_pwm *fpwm; + struct device *dev = pwmchip_parent(chip); + int ret; if (state->polarity == PWM_POLARITY_INVERSED) return -EINVAL; fpwm = to_apple_pwm(chip); + ret = pm_runtime_resume_and_get(dev); + if (ret) + return ret; + if (state->enabled) { u64 on_cycles, off_cycles; @@ -69,6 +81,14 @@ static int apple_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, } else { writel(0, fpwm->base + APPLE_PWM_CTRL); } + + /* Keep the controller powered for as long as it drives an output. */ + if (state->enabled && !fpwm->enabled) + pm_runtime_get_noresume(dev); + else if (!state->enabled && fpwm->enabled) + pm_runtime_put_noidle(dev); + fpwm->enabled = state->enabled; + pm_runtime_put(dev); return 0; } @@ -77,8 +97,13 @@ static int apple_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, { struct apple_pwm *fpwm; u32 on_cycles, off_cycles, ctrl; + struct device *dev = pwmchip_parent(chip); + int ret; fpwm = to_apple_pwm(chip); + ret = pm_runtime_resume_and_get(dev); + if (ret) + return ret; ctrl = readl(fpwm->base + APPLE_PWM_CTRL); on_cycles = readl(fpwm->base + APPLE_PWM_ON_CYCLES); @@ -91,6 +116,7 @@ static int apple_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, state->period = DIV64_U64_ROUND_UP(((u64)off_cycles + (u64)on_cycles) * NSEC_PER_SEC, fpwm->clkrate); + pm_runtime_put(dev); return 0; } @@ -99,11 +125,55 @@ static const struct pwm_ops apple_pwm_ops = { .get_state = apple_pwm_get_state, }; +static int apple_pwm_runtime_suspend(struct device *dev) +{ + struct apple_pwm *fpwm = to_apple_pwm(dev_get_drvdata(dev)); + + /* A disabled PWM can lose its register state when the domain is off. */ + fpwm->on_cycles = readl(fpwm->base + APPLE_PWM_ON_CYCLES); + fpwm->off_cycles = readl(fpwm->base + APPLE_PWM_OFF_CYCLES); + fpwm->ctrl = readl(fpwm->base + APPLE_PWM_CTRL); + clk_disable_unprepare(fpwm->clk); + return 0; +} + +static int apple_pwm_runtime_resume(struct device *dev) +{ + struct apple_pwm *fpwm = to_apple_pwm(dev_get_drvdata(dev)); + int ret; + + ret = clk_prepare_enable(fpwm->clk); + if (ret) + return ret; + + writel(fpwm->on_cycles, fpwm->base + APPLE_PWM_ON_CYCLES); + writel(fpwm->off_cycles, fpwm->base + APPLE_PWM_OFF_CYCLES); + writel(fpwm->ctrl, fpwm->base + APPLE_PWM_CTRL); + return 0; +} + +static void apple_pwm_runtime_disable(void *data) +{ + struct device *dev = data; + struct apple_pwm *fpwm = to_apple_pwm(dev_get_drvdata(dev)); + + pm_runtime_disable(dev); + if (fpwm->enabled) + pm_runtime_put_noidle(dev); + if (!pm_runtime_status_suspended(dev)) + clk_disable_unprepare(fpwm->clk); + pm_runtime_set_suspended(dev); +} + +static const struct dev_pm_ops apple_pwm_pm_ops = { + RUNTIME_PM_OPS(apple_pwm_runtime_suspend, apple_pwm_runtime_resume, NULL) +}; + static int apple_pwm_probe(struct platform_device *pdev) { struct pwm_chip *chip; struct apple_pwm *fpwm; - struct clk *clk; + struct device *dev = &pdev->dev; int ret; chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*fpwm)); @@ -116,9 +186,9 @@ static int apple_pwm_probe(struct platform_device *pdev) if (IS_ERR(fpwm->base)) return PTR_ERR(fpwm->base); - clk = devm_clk_get_enabled(&pdev->dev, NULL); - if (IS_ERR(clk)) - return dev_err_probe(&pdev->dev, PTR_ERR(clk), "unable to get the clock"); + fpwm->clk = devm_clk_get(dev, NULL); + if (IS_ERR(fpwm->clk)) + return dev_err_probe(dev, PTR_ERR(fpwm->clk), "unable to get the clock"); /* * Uses the 24MHz system clock on all existing devices, can only @@ -126,13 +196,38 @@ static int apple_pwm_probe(struct platform_device *pdev) * * This check is done to prevent an overflow in .apply */ - fpwm->clkrate = clk_get_rate(clk); + fpwm->clkrate = clk_get_rate(fpwm->clk); if (!fpwm->clkrate || fpwm->clkrate > NSEC_PER_SEC) return dev_err_probe(&pdev->dev, -EINVAL, "pwm clock out of range"); chip->ops = &apple_pwm_ops; + platform_set_drvdata(pdev, chip); + + ret = clk_prepare_enable(fpwm->clk); + if (ret) + return ret; + ret = pm_runtime_set_active(dev); + if (ret) { + clk_disable_unprepare(fpwm->clk); + return ret; + } + + /* Preserve an output that the bootloader left running. */ + fpwm->ctrl = readl(fpwm->base + APPLE_PWM_CTRL); + fpwm->enabled = (fpwm->ctrl & APPLE_PWM_CTRL_ENABLE) && + (fpwm->ctrl & APPLE_PWM_CTRL_OUTPUT_ENABLE); + pm_runtime_get_noresume(dev); /* probe reference */ + if (fpwm->enabled) + pm_runtime_get_noresume(dev); /* output reference */ + pm_runtime_enable(dev); + ret = devm_add_action_or_reset(dev, apple_pwm_runtime_disable, dev); + if (ret) { + pm_runtime_put_noidle(dev); + return ret; + } ret = devm_pwmchip_add(&pdev->dev, chip); + pm_runtime_put(dev); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "unable to add pwm chip"); @@ -150,6 +245,7 @@ static struct platform_driver apple_pwm_driver = { .driver = { .name = "apple-pwm", .of_match_table = apple_pwm_of_match, + .pm = pm_ptr(&apple_pwm_pm_ops), }, }; module_platform_driver(apple_pwm_driver); diff --git a/tools/asahi/apple-pwm-pm-tests.py b/tools/asahi/apple-pwm-pm-tests.py new file mode 100644 index 0000000000000..5483b968a9d3a --- /dev/null +++ b/tools/asahi/apple-pwm-pm-tests.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Compile the real Apple PWM callbacks against a small runtime-PM/MMIO model. + +Tests reference ownership, register-loss recovery and failures. The model does +not qualify physical power gating, output levels or system sleep on hardware. +""" +import argparse +import os +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] + + +def function(source, name): + match = re.search(r'^static [^\n]*\b' + name + r'\([^;]*?\n\{', source, re.M) + if not match: + raise ValueError('missing callback: ' + name) + depth = 1 + end = match.end() + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[match.start():end] + + +PRELUDE = r''' +#include +#include +#include +#include +#include +#include +#define __iomem +#define BIT(n) (1U << (n)) +#define NSEC_PER_SEC 1000000000ULL +#define PWM_POLARITY_INVERSED 1 +#define PWM_POLARITY_NORMAL 0 +#define GFP_KERNEL 0 +#define IS_ERR(p) ((intptr_t)(p) < 0) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define DIV64_U64_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) +typedef uint32_t u32; +typedef uint64_t u64; +struct clk { int refs; }; +struct device { void *data; int refs; bool active, rpm; }; +struct platform_device { struct device dev; }; +struct pwm_chip { struct device *parent; void *data; const void *ops; }; +struct pwm_device { int unused; }; +struct pwm_state { bool enabled; int polarity; u64 duty_cycle, period; }; +static struct pwm_chip chip; +static struct platform_device pdev; +static struct clk clock_model; +static u32 registers[8]; +static unsigned char private_data[256]; +static int clock_error, active_error, action_error, add_error, writes; +static void (*cleanup)(void *); +static void *cleanup_data; +static const int apple_pwm_ops; +static int apple_pwm_runtime_suspend(struct device *dev); +static int apple_pwm_runtime_resume(struct device *dev); +static void *pwmchip_get_drvdata(struct pwm_chip *p) { return p->data; } +static struct device *pwmchip_parent(struct pwm_chip *p) { return p->parent; } +static void *dev_get_drvdata(struct device *d) { return d->data; } +static void platform_set_drvdata(struct platform_device *p, void *v) { p->dev.data = v; } +static u64 mul_u64_u64_div_u64(u64 a, u64 b, u64 c) { return (__uint128_t)a * b / c; } +static u32 readl(void *p) { assert(clock_model.refs == 1); return *(u32 *)p; } +static void writel(u32 v, void *p) { assert(clock_model.refs == 1); ++writes; *(u32 *)p = v; } +static int clk_prepare_enable(struct clk *c) { + if (clock_error) return clock_error; + assert(c->refs == 0); ++c->refs; return 0; +} +static void clk_disable_unprepare(struct clk *c) { assert(c->refs == 1); --c->refs; } +static u64 clk_get_rate(struct clk *c) { return 24000000; } +static void pm_runtime_get_noresume(struct device *d) { ++d->refs; } +static void pm_runtime_put_noidle(struct device *d) { assert(d->refs > 0); --d->refs; } +static int pm_runtime_resume_and_get(struct device *d) { + if (!d->active) { + int ret = apple_pwm_runtime_resume(d); + if (ret) return ret; + d->active = true; + } + ++d->refs; return 0; +} +static void pm_runtime_put(struct device *d) { + pm_runtime_put_noidle(d); + if (!d->refs && d->rpm && d->active) { + assert(!apple_pwm_runtime_suspend(d)); + d->active = false; + memset(registers, 0, sizeof(registers)); /* domain loses context */ + } +} +static void pm_runtime_enable(struct device *d) { d->rpm = true; } +static void pm_runtime_disable(struct device *d) { d->rpm = false; } +static bool pm_runtime_status_suspended(struct device *d) { return !d->active; } +static int pm_runtime_set_active(struct device *d) { + if (active_error) return active_error; + d->active = true; return 0; +} +static int pm_runtime_set_suspended(struct device *d) { d->active = false; return 0; } +static void *devm_pwmchip_alloc(struct device *d, unsigned n, size_t size) { + assert(size <= sizeof(private_data)); + chip = (struct pwm_chip){.parent = d, .data = private_data}; return &chip; +} +static void *devm_platform_ioremap_resource(struct platform_device *p, int n) { return registers; } +static struct clk *devm_clk_get(struct device *d, const char *name) { return &clock_model; } +static int dev_err_probe(struct device *d, int err, const char *msg) { return err; } +static int devm_add_action_or_reset(struct device *d, void (*fn)(void *), void *arg) { + if (action_error) { fn(arg); return action_error; } + cleanup = fn; cleanup_data = arg; return 0; +} +static int devm_pwmchip_add(struct device *d, struct pwm_chip *p) { return add_error; } +''' + +TESTS = r''' +static struct pwm_device pwm; +static struct pwm_state state = {.period = 40000, .duty_cycle = 10000}; +static void reset_model(bool boot_on) { + assert(!clock_model.refs && !pdev.dev.refs); + memset(&pdev, 0, sizeof(pdev)); memset(private_data, 0, sizeof(private_data)); + memset(registers, 0, sizeof(registers)); + registers[APPLE_PWM_ON_CYCLES / 4] = 240; + registers[APPLE_PWM_OFF_CYCLES / 4] = 720; + registers[APPLE_PWM_CTRL / 4] = boot_on ? APPLE_PWM_CTRL_ENABLE | APPLE_PWM_CTRL_OUTPUT_ENABLE : 0; + clock_error = active_error = action_error = add_error = writes = 0; + cleanup = NULL; state.enabled = false; state.polarity = PWM_POLARITY_NORMAL; +} +static void remove_model(void) { + assert(cleanup); cleanup(cleanup_data); cleanup = NULL; + assert(!pdev.dev.refs && !pdev.dev.active && !clock_model.refs); +} +int main(void) { + struct pwm_state got; + reset_model(false); + assert(!apple_pwm_probe(&pdev)); + assert(!pdev.dev.active && !pdev.dev.refs && !clock_model.refs); + assert(!apple_pwm_get_state(&chip, &pwm, &got)); + assert(!got.enabled && got.period == 40000 && got.duty_cycle == 10000); + assert(!pdev.dev.active && !pdev.dev.refs && !clock_model.refs); + puts("PASS disabled probe, get_state and register-loss recovery"); + state.enabled = true; + assert(!apple_pwm_apply(&chip, &pwm, &state)); + assert(pdev.dev.active && pdev.dev.refs == 1 && clock_model.refs == 1); + for (int i = 0; i < 20; ++i) { + assert(!apple_pwm_apply(&chip, &pwm, &state)); + assert(!apple_pwm_get_state(&chip, &pwm, &got)); + assert(pdev.dev.refs == 1 && got.enabled && got.period == 40000); + } + puts("PASS enabled output pins power; repeated apply/get_state do not leak"); + state.duty_cycle = 0; + assert(!apple_pwm_apply(&chip, &pwm, &state)); + assert(pdev.dev.active && pdev.dev.refs == 1); + assert(!apple_pwm_get_state(&chip, &pwm, &got) && got.enabled && !got.duty_cycle); + state.duty_cycle = 10000; + assert(!apple_pwm_apply(&chip, &pwm, &state)); + puts("PASS enabled zero-duty output remains powered"); + state.enabled = false; + assert(!apple_pwm_apply(&chip, &pwm, &state)); + assert(!pdev.dev.active && !pdev.dev.refs && !clock_model.refs); + assert(!apple_pwm_get_state(&chip, &pwm, &got) && !got.enabled); + remove_model(); + puts("PASS disable releases output reference and suspended removal is balanced"); + reset_model(true); + assert(!apple_pwm_probe(&pdev)); + assert(pdev.dev.active && pdev.dev.refs == 1 && clock_model.refs == 1 && !writes); + assert(!apple_pwm_get_state(&chip, &pwm, &got) && got.enabled); + state.enabled = true; + assert(!apple_pwm_apply(&chip, &pwm, &state) && pdev.dev.refs == 1); + remove_model(); + puts("PASS bootloader output adopted without writes and active removal balanced"); + reset_model(false); + assert(!apple_pwm_probe(&pdev)); + clock_error = -EIO; + int before = writes; + state.enabled = true; + assert(apple_pwm_apply(&chip, &pwm, &state) == -EIO); + assert(apple_pwm_get_state(&chip, &pwm, &got) == -EIO); + assert(!pdev.dev.refs && !clock_model.refs && writes == before); + state.polarity = PWM_POLARITY_INVERSED; + assert(apple_pwm_apply(&chip, &pwm, &state) == -EINVAL && writes == before); + clock_error = 0; remove_model(); + puts("PASS resume failure and unsupported polarity leave registers and refs untouched"); + reset_model(false); clock_error = -EIO; + assert(apple_pwm_probe(&pdev) == -EIO && !clock_model.refs && !pdev.dev.refs); + reset_model(false); active_error = -EBUSY; + assert(apple_pwm_probe(&pdev) == -EBUSY && !clock_model.refs && !pdev.dev.refs); + puts("PASS initial clock and PM-state failures unwind"); + for (int enabled = 0; enabled <= 1; ++enabled) { + reset_model(enabled); action_error = -ENOMEM; + assert(apple_pwm_probe(&pdev) == -ENOMEM && !clock_model.refs && !pdev.dev.refs); + reset_model(enabled); add_error = -ENOMEM; + assert(apple_pwm_probe(&pdev) == -ENOMEM); + remove_model(); + } + puts("PASS action and PWM registration failures unwind both boot states"); + reset_model(false); assert(!apple_pwm_probe(&pdev)); + assert(!pm_runtime_resume_and_get(&pdev.dev)); /* another consumer/supplier vote */ + state.enabled = true; assert(!apple_pwm_apply(&chip, &pwm, &state)); + assert(pdev.dev.refs == 2); + state.enabled = false; assert(!apple_pwm_apply(&chip, &pwm, &state)); + assert(pdev.dev.active && pdev.dev.refs == 1 && clock_model.refs == 1); + pm_runtime_put(&pdev.dev); + assert(!pdev.dev.active && !clock_model.refs); remove_model(); + puts("PASS another runtime reference prevents premature suspension"); + return 0; +} +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--negative-control', action='store_true') + args = parser.parse_args() + source = (ROOT / 'drivers/pwm/pwm-apple.c').read_text() + if args.negative_control: + old = '\tif (state->enabled && !fpwm->enabled)\n\t\tpm_runtime_get_noresume(dev);' + assert old in source + source = source.replace(old, '\tif (state->enabled && !fpwm->enabled)\n\t\t; /* missing output reference */', 1) + defines = '\n'.join(re.findall(r'^#define APPLE_PWM_.*$', source, re.M)) + struct = re.search(r'struct apple_pwm \{.*?\n\};', source, re.S).group() + callbacks = '\n'.join(function(source, name) for name in ( + 'to_apple_pwm', 'apple_pwm_apply', 'apple_pwm_get_state', + 'apple_pwm_runtime_suspend', 'apple_pwm_runtime_resume', + 'apple_pwm_runtime_disable', 'apple_pwm_probe')) + with tempfile.TemporaryDirectory(prefix='apple-pwm-pm-') as tmp: + tmp = Path(tmp) + cfile, executable = tmp / 'test.c', tmp / 'test' + cfile.write_text(PRELUDE + defines + '\n' + struct + callbacks + TESTS) + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-Wall', '-Wextra', + '-Werror', '-Wno-unused-parameter', '-Wno-unused-function', + '-fsanitize=address,undefined', str(cfile), '-o', str(executable)], check=True) + result = subprocess.run([str(executable)], capture_output=True, text=True) + if args.negative_control: + if result.returncode == 0: + raise SystemExit('FAIL: missing output reference escaped detection') + print('PASS negative control: missing output reference was detected') + else: + print(result.stdout, end='') + if result.returncode: + raise SystemExit(result.stderr) + + +if __name__ == '__main__': + main() From a3e4d2190ba511640d49249f0ed7e4824c733c80 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:46:01 +1100 Subject: [PATCH 1509/1579] accel/ane: retain domain attachments if performance release fails Do not discard the report node or detach power domains after an idle release fails during removal. Keep the same conservative retention rule used for an engine with unknown DMA state, and make the failed teardown visible. Normal removal releases votes before reclaiming buffers. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane.h | 2 +- drivers/accel/ane/ane_drv.c | 10 +++++++++- drivers/accel/ane/ane_perf.c | 9 +++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index c6e471d9f623a..332b0d31a6ad9 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -161,7 +161,7 @@ int ane_submit_check_sizes(const struct drm_ane_submit *args, size_t cmd_size, void ane_reclaim_preserved(struct ane_device *ane); int ane_perf_init(struct ane_device *ane, unsigned int high_level); -void ane_perf_exit(struct ane_device *ane); +int ane_perf_exit(struct ane_device *ane); int ane_perf_begin(struct ane_device *ane); void ane_perf_end(struct ane_device *ane); int ane_perf_suspend(struct ane_device *ane); diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index b11298181fd94..ac113ac727b4e 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -1297,6 +1297,7 @@ static void ane_platform_remove(struct platform_device *pdev) { struct ane_device *ane = platform_get_drvdata(pdev); struct ane_bo *bo, *tmp; + int err; /* Powered and pinned for the teardown: the unmaps below flush the * DART TLBs, and no autosuspend may race the genpd detach. @@ -1314,12 +1315,19 @@ static void ane_platform_remove(struct platform_device *pdev) return; } + err = ane_perf_exit(ane); + if (err) { + dev_err(ane->dev, + "retaining power-domain attachments after failed performance release; reboot required\n"); + mutex_unlock(&ane->engine_lock); + return; + } + list_for_each_entry_safe(bo, tmp, &ane->bo_list, node) ane_iommu_unmap_pages(ane, bo); ane_reclaim_preserved(ane); drm_mm_takedown(&ane->mm); - ane_perf_exit(ane); ane_detach_genpd(ane); pm_runtime_disable(ane->dev); diff --git a/drivers/accel/ane/ane_perf.c b/drivers/accel/ane/ane_perf.c index cc50f899337fb..23daae652957a 100644 --- a/drivers/accel/ane/ane_perf.c +++ b/drivers/accel/ane/ane_perf.c @@ -205,12 +205,13 @@ int ane_perf_init(struct ane_device *ane, unsigned int high_level) return devm_device_add_group(ane->dev, &ane_perf_group); } -void ane_perf_exit(struct ane_device *ane) +int ane_perf_exit(struct ane_device *ane) { struct ane_perf *p = &ane->perf; + int ret; if (!p->initialized) - return; + return 0; cancel_delayed_work_sync(&p->idle); mutex_lock(&p->lock); p->error = ane_perf_drop(ane); @@ -221,9 +222,13 @@ void ane_perf_exit(struct ane_device *ane) pm_runtime_put_noidle(ane->dev); p->pinned = false; } + ret = p->error; mutex_unlock(&p->lock); + if (ret) + return ret; of_node_put(p->report); p->report = NULL; + return 0; } int ane_perf_suspend(struct ane_device *ane) From d9670f977be9b7867bc15007d3cdddea51f46723 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:48:10 +1100 Subject: [PATCH 1510/1579] pmdomain: apple: ensure ANE KUnit tests enter the domain build The parent Makefile enters drivers/pmdomain only when generic power domains are enabled. Make that requirement explicit for the standalone ANE register-model suite so enabling its option actually builds it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/Kconfig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index 7526bda60f7af..c442356fedde1 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -55,7 +55,8 @@ config APPLE_PMP_REPORT config APPLE_PMP_ANE_KUNIT_TEST tristate "KUnit tests for T600x ANE performance requests" if !KUNIT_ALL_TESTS - depends on KUNIT + depends on KUNIT && PM + select PM_GENERIC_DOMAINS default KUNIT_ALL_TESTS help Exercise ANE level transitions, ownership refusal and ACK-timeout From 7d2048e8b6e0aee29d5db242a288588398e5da61 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:50:00 +1100 Subject: [PATCH 1511/1579] soc: apple: sep: Say what a verify non-match actually was A verify the enclave answers for a user other than the Linux one is reported as a non-match. Since the enrol-guard series the driver logged it as "enclave found no identity in the Linux user context", which reads like a binding or migration fault. On a MacBook Air M2 that line sent a regression hunt after two touches with the wrong finger: the enclave held the one Linux identity, the host index listed the same UUID, and a touch with the enrolled finger then matched. Log it as a non-match and print the user the enclave answered next to the Linux user. A finger that matches nothing and a match in another user context then read differently, without a rebuild. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index a3917e11338c9..98a048443da18 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1905,7 +1905,16 @@ impl SepData { }; if !parsed.matches(user) { - dev_info!(self.dev, "verify: enclave found no identity in the Linux user context\n"); + // The enclave compared a usable capture and did not answer the + // Linux user. Usually no template matched (a different finger). + // Log the user it did answer, so a match in another user context + // can be told apart from a plain non-match. + dev_info!( + self.dev, + "verify: no enrolled finger matched the capture (enclave answered user {}, Linux enrolments use user {})\n", + parsed.user_id(), + user.value() + ); return bio::VerifyOutcome::NoMatch; } @@ -5143,6 +5152,10 @@ impl MatchResult { &self.identity } + pub(crate) fn user_id(&self) -> i32 { + self.user_id + } + pub(crate) fn matches(&self, user: UserId) -> bool { self.user_id == user.value() } From 4f2cbaabd500e98f82920bbc217bd3ae7919c498 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:50:05 +1100 Subject: [PATCH 1512/1579] arm64: dts: apple: t8112-j413/j415: Describe the Mesa data-ready line The J413 and J415 platform device trees give the Mesa sensor a data-ready line on AP GPIO 177. The nodes left it out, so apple_sep logs "no data-ready interrupt (describe it in DT)" on every verify and polls the sensor's SPI status for the frame. Describe it as the other Mesa machines do (J314s, J316s, J616s), rising edge on the pin controller that already carries the power line. The driver still gates every frame on the SPI status, so a missed or spurious edge costs at most one backstop interval and polling remains the fallback. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8112-j413.dts | 8 +++++--- arch/arm64/boot/dts/apple/t8112-j415.dts | 7 +++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 7939c20917155..8f6f3c6948710 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -262,9 +262,9 @@ /* * Mesa fingerprint sensor on spi2. The J413 platform device tree * (/arm-io/spi2/mesa) describes it as J415's does: power on AP GPIO 178, - * active high, data-ready on GPIO 177. As on J415, the data-ready line is - * left out, so the driver polls until its trigger is confirmed. Native chip - * select only. J413 is untested. + * active high, data-ready on AP GPIO 177. The data-ready edge only wakes the + * capture early; every frame is still gated on the sensor's SPI status. + * Native chip select only. */ &spi2 { status = "okay"; @@ -273,6 +273,8 @@ &mesa { status = "okay"; enable-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <177 IRQ_TYPE_EDGE_RISING>; spi-cs-setup-delay-ns = <20>; spi-cs-hold-delay-ns = <20>; /* Per-device factory data from this Mac's own iBoot system container. */ diff --git a/arch/arm64/boot/dts/apple/t8112-j415.dts b/arch/arm64/boot/dts/apple/t8112-j415.dts index 2e2b982c584e0..cdcb4802396b5 100644 --- a/arch/arm64/boot/dts/apple/t8112-j415.dts +++ b/arch/arm64/boot/dts/apple/t8112-j415.dts @@ -286,8 +286,9 @@ #include "hwmon-laptop.dtsi" /* * Mesa fingerprint sensor on spi2 (J415 ADT /arm-io/spi2/mesa): power on AP - * GPIO 178, active high. The data-ready line (GPIO 177) is left out, so the - * driver polls until its trigger is confirmed. Native chip select only. + * GPIO 178, active high, and data-ready on AP GPIO 177. The data-ready edge + * only wakes the capture early; every frame is still gated on the sensor's + * SPI status. Native chip select only. */ &spi2 { status = "okay"; @@ -296,6 +297,8 @@ &mesa { status = "okay"; enable-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <177 IRQ_TYPE_EDGE_RISING>; spi-cs-setup-delay-ns = <20>; spi-cs-hold-delay-ns = <20>; /* Per-device factory data from this Mac's own iBoot system container. */ From 2515e8fda49507fb90439cbc3422504156feaae6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 18:13:52 +1100 Subject: [PATCH 1513/1579] arm64: dts: apple: t6040-j616s: enable the QSPI power state Commit 3d9601c360bf ("arm64: dts: apple: t6040-j616s: enable the boot NOR and its nvram") enabled the QSPI controller but left its power state, ps_qspi, disabled as the SoC .dtsi describes it. The controller then waits on a supplier that never probes, and fails after the deferred probe timeout: apple-qspi 429118000.spi: probe with driver apple-qspi failed with error -110 Enable ps_qspi on the board that describes its flash. With this, the boot NOR and its nvram partition probe at about 5 s on a J616s. Fixes: 3d9601c360bf ("arm64: dts: apple: t6040-j616s: enable the boot NOR and its nvram") Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 36da16634f7d3..872381a098080 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -1459,6 +1459,11 @@ }; }; +/* The QSPI controller cannot probe while its power state is disabled. */ +&ps_qspi { + status = "okay"; +}; + /* * Touch ID sensor (Mesa 0x3356). Its power is NUB GPIO 53 and its * data-ready line is AP GPIO 116, as the J616s ADT describes them. From 91f8b8fd7d349c5c2b96b4ab5e494d2c273b1d25 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 18:32:12 +1100 Subject: [PATCH 1514/1579] wifi: mt7932: fence RNR probes and budget passive discovery Let cfg80211 split the lower-band and 6 GHz phases so PSC selection and cached RNR expansion take effect. Keep RNR-constrained channels passive until the firmware target list can express their BSSID and short SSID; retain ordinary directed probing on the installed legacy VLP channels. Internal discovery previously allowed five seconds for an entire passive batch, although 32 channels at 220 ms already need over seven seconds. Add the serialized per-channel budget to the existing completion margin. Exercise the real scan batcher and discovery worker with synthetic 32/27-channel batches and compiling negative controls. Add KUnit golden scan and TPE bodies, class-pad preservation and band-2 runtime D7 refusal. Document the separate unsupported-rate, calibration and AP-matrix gaps without inventing firmware fields or treating serialization as RF proof. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 69 ++++++- .../net/wireless/mediatek/mt7932/frontend.c | 23 +-- .../wireless/mediatek/mt7932/pmf-lpi-test.c | 29 +++ .../mediatek/mt7932/regulatory-test.c | 38 ++++ drivers/net/wireless/mediatek/mt7932/scan.h | 35 ++++ .../net/wireless/mediatek/mt7932/six-test.c | 102 +++++++++ .../net/wireless/mediatek/mt7932/startup.c | 1 + tools/asahi/mt7932-scan-tests.py | 194 ++++++++++++++++++ tools/asahi/run-host-tests.py | 1 + 9 files changed, 472 insertions(+), 20 deletions(-) create mode 100644 tools/asahi/mt7932-scan-tests.py diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index ceb85331324ad..a8fa08c4e3893 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -201,8 +201,14 @@ OWE layers without another crypto or authentication implementation. Legacy VLP channels retain directed hidden-SSID probing and the original connect discovery sequence. Channels added only by class packages listen passively until the target supplies its class/TPE. RNR-only requests remain -passive until target-list serialization is implemented. Full-band scans are split into bounded -32-channel commands. Firmware event channel layout is checked before OCA +passive until target-list serialization is implemented. cfg80211 splits the +lower-band and 6 GHz phases, selects PSCs and expands cached RNR entries. +The driver keeps the supplied channel list, including non-PSC RNR targets, +and never treats ``psc_no_listen`` as permission for an unconstrained probe. +Full-band scans are split into bounded 32-channel commands. Internal +discovery allows the entire passive batch's serialized dwell budget plus +the existing five-second completion margin; 32 channels with a 233 ms +budget require a 12456 ms wait. Firmware event channel layout is checked before OCA association calibration, queue binding and the two association power pages. J7RP version 1 remains valid and never authorizes 6 GHz. Version 2 packages @@ -579,6 +585,65 @@ pre-association class/TPE SET is inserted on this compatibility path. Generic ru requests for band 2 (6 GHz) remain rejected by the raw planner. RF behavior, rate accounting and calibration still require hardware qualification. +6 GHz software proof and qualification boundary +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The association-start OCA group selector and power-on fragments are separate +from runtime D7. The raw planner rejects every runtime band-2 request before +reading a container or producing a partial fragment plan. The runtime worker +latches that error and closes RF admission. Association success does not +prove that firmware will never issue such a request, and the 2.4/5 GHz D7 +layouts must not be reused by changing only the band number. + +The public producer constructs the 32-byte CID5d header pad from +``one_ss_pad()`` and copies it unchanged through J7RP and class publication. +Its bits index channel numbers. It is independent of the primary permission +mask and does not generate the separate unsupported-rate accounting data. +Neither a zero pad nor numeric per-rate cells prove that firmware excludes +unsupported rates. The channel pad must not be relabelled or populated with +guessed rate bits. Synthetic tests check a nonzero pad, class isolation and +the exact first-page header without distributing power-table inputs. + +The strict class path parses default-client EIRP and PSD TPEs, intersects +applicable limits, preserves ``0x7f``, deducts the J700 antenna gains and +submits the separate 60-byte CID5d body before EA/40. A changed or invalid +home beacon envelope disconnects the strict-class link; disconnect and +country/reset handling invalidate its decision. This does not establish +firmware readback or radiated power. The legacy v2 VLP compatibility path +above still bypasses the new class/TPE submission and beacon comparison; +tests preserving that sequence are not proof of AP TPE enforcement there. + +The software regressions can be exercised without a radio:: + + python3 tools/asahi/mt7932-scan-tests.py --out /tmp/mt7932-scan-tests + python3 tools/asahi/run-host-tests.py --only mt7932 --jobs 8 + +The KUnit ``mt7932-six-ghz``, ``mt7932-regulatory`` and +``mt7932-pmf-lpi`` suites also cover runtime D7 rejection, the header pad, +and complete passive-scan/TPE golden bodies. The host scan test executes +the production batcher and internal discovery worker with synthetic +32/27-channel batches, and requires compiling RNR-probe and short-wait +mutants to fail their runtime assertions. + +Before claiming RF qualification, retain failures as well as successful +cells and run this AP matrix with an explicitly approved device identity: + +* PSC and non-PSC targets, with and without lower-band RNR; wildcard and + hidden SSIDs; connected and disconnected scans. Check matching completion, + bounded 32/27-channel batches and no probes from RNR-derived passive scans. +* LPI and VLP at 20/40/80 MHz, both PSD and EIRP TPE, finite and ``0x7f`` + limits, and changed envelopes. Distinguish strict-class operation from the + legacy v2 bypass. Check class pages and TPE precede association, and measure + power against country, AP, SAR and board limits. +* LPI/VLP reconnects, country changes and firmware resets; confirm fresh + policy publication and no stale AP envelope after rollback. +* Calibration events during discovery, association and sustained traffic. + Record only band/channel, procedure outcome and nonsecret body hashes; + band-2 D7 remains unsupported until its own procedure ABI is established. +* Rate/NSS/MCS and bandwidth accounting under each class, including rates + represented by unsupported or unpopulated cells. Static serialization + tests cannot establish the firmware's default rate-selection behavior. + 6 GHz LPI table archive (J7RP v3, unbound) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index d309646c00b8d..9e19d1a492e3b 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -165,18 +165,10 @@ static int mt_scan_submit_next(struct mt7932 *m) u8 body[0x4d4], channels[MT7932_SCAN_CHANNELS]; unsigned int count = 0, i; unsigned long flags; - bool passive = false, directed = false; + bool passive = false; int ret; - /* Hidden SSIDs retain directed probing on the legacy VLP channels. */ - for (i = 0; i < request->n_ssids; i++) - directed |= !!request->ssids[i].ssid_len; - - /* Retain directed discovery on the installed legacy VLP channels. - * New class-only channels need an AP class/TPE before transmission. - */ - - /* 2GHz active/passive, then 5GHz active/passive. Never put a NO_IR + /* 2/5/6 GHz active/passive pairs. Never put a NO_IR * channel in a firmware request which can emit probe requests. */ while (!count && m->scan_batch <= 6) { @@ -187,13 +179,8 @@ static int mt_scan_submit_next(struct mt7932 *m) for (i = m->scan_cursor; i < request->n_channels; i++) { struct ieee80211_channel *channel = request->channels[i]; bool radar = channel->flags & IEEE80211_CHAN_RADAR; - bool listen = - !request->n_ssids || radar || - (channel->band == NL80211_BAND_6GHZ && - (!directed || - !(m->six_legacy_channels & - BIT_ULL((channel->hw_value - 1) / 4)))) || - (channel->flags & IEEE80211_CHAN_NO_IR); + bool listen = mt7932_scan_listen(request, channel, + m->six_legacy_channels); /* Radar channels are never joined; while associated, * a passive dwell there only delays the home channel. @@ -321,7 +308,7 @@ int mt_connect_discover(struct mt7932 *m) m->scan_seq, band, count, passive, ret); mutex_unlock(&m->command_mutex); if (!ret && !wait_for_completion_timeout(&m->discovery_done, - msecs_to_jiffies(5000))) + msecs_to_jiffies(mt7932_scan_discovery_timeout(body)))) ret = -ETIMEDOUT; mutex_lock(&m->command_mutex); /* Cancellation wakes the waiter, but is not a firmware completion. diff --git a/drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c b/drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c index 6d8320f308348..927b0e43b3a9c 100644 --- a/drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c +++ b/drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c @@ -375,6 +375,34 @@ static void mt_6g_legacy_vlp_finite_test(struct kunit *test) KUNIT_EXPECT_EQ(test, ap.mode, MT7932_6G_LPI); } +static void mt_6g_tpe_golden_test(struct kunit *test) +{ + /* Default-client EIRP: 20/40/80 MHz caps, including a signed cap and + * the no-limit sentinel. The unrelated subordinate envelope is ignored. + */ + static const u8 eirp[] = { 18, 40, 0xf6, 127 }; + static const u8 subordinate[] = { 82, 0x80, 0x80, 0x80 }; + static const u8 expected[60] = { + [1] = 2, [2] = 60, [4] = 1, 3, 1, 3, + [44] = 1, 3, 5, 3, + [48] = 24, 0xe6, 127, 127, + 24, 0xe6, 127, 127, 30, 0xec, 127, 127, + }; + struct mt7932_6g_ap ap = { + .mode = MT7932_6G_LPI, .primary = 5, + .limit = { 127, 127, 127, 127 }, + }; + s8 gain[3][4]; + u8 body[60]; + + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe(&ap, eirp, sizeof(eirp)), 0); + KUNIT_ASSERT_EQ(test, + mt7932_6g_tpe(&ap, subordinate, sizeof(subordinate)), 0); + KUNIT_ASSERT_EQ(test, mt7932_j700_6g_gains(&ap, gain), 0); + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), 60); + KUNIT_EXPECT_MEMEQ(test, body, expected, sizeof(expected)); +} + static struct kunit_case mt_pmf_lpi_cases[] = { KUNIT_CASE(mt_pmf_admission_test), KUNIT_CASE(mt_pmf_body_test), @@ -383,6 +411,7 @@ static struct kunit_case mt_pmf_lpi_cases[] = { KUNIT_CASE(mt_6g_j700_gain_test), KUNIT_CASE(mt_6g_finite_projection_test), KUNIT_CASE(mt_6g_legacy_vlp_finite_test), + KUNIT_CASE(mt_6g_tpe_golden_test), {} }; diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c index be1dc20a8d477..b67b276d17979 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c +++ b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c @@ -672,6 +672,43 @@ static void mt7932_policy_v4_classes_test(struct kunit *test) KUNIT_EXPECT_LT(test, mt7932_policy_parse(policy, data, size, "AU"), 0); } +/* The public producer's 32-byte 1SS channel pad is copied verbatim. It is + * independent of the primary permission mask and of rate-support accounting. + */ +static void mt7932_policy_6g_pad_test(struct kunit *test) +{ + static const u8 expected[44] = { + [1] = 0x1a, [2] = 0xfc, 3, 8, 3, + [8] = 'A', 'U', + [12] = 0x20, [16] = 0x20, [41] = 2, + }; + struct mt7932_policy *policy = + kunit_kzalloc(test, sizeof(*policy), GFP_KERNEL); + size_t size; + u8 *data = pkg_build_v4(test, &size), *body, *pad; + + KUNIT_ASSERT_NOT_NULL(test, policy); + body = kunit_kmalloc(test, 1020, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, body); + /* Directory plus VLP section, then LPI header and its channel pad. */ + pad = data + size - 13476 + 24; + pad[0] = 0x20; /* channel 5 */ + pad[4] = 0x20; /* channel 37, not rate index 37 */ + pad[29] = 2; /* channel 233 */ + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(policy, data, size, "AU"), 0); + KUNIT_EXPECT_EQ(test, policy->six_class[MT7932_6G_LPI].channels, + BIT_ULL(1)); + KUNIT_ASSERT_EQ(test, mt7932_6g_class_page(body, 1020, policy, + MT7932_6G_LPI, "AU", 0), 1020); + KUNIT_EXPECT_MEMEQ(test, body, expected, sizeof(expected)); + KUNIT_ASSERT_EQ(test, mt7932_6g_class_page(body, 1020, policy, + MT7932_6G_LPI, "AU", 104), 776); + KUNIT_EXPECT_MEMEQ(test, body + 12, pad, 32); + KUNIT_ASSERT_EQ(test, mt7932_6g_class_page(body, 1020, policy, + MT7932_6G_VLP, "AU", 0), 1020); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(body + 12, 0, 32), NULL); +} + #define FAKE_FILES 8 #define FAKE_LOG 16 @@ -1157,6 +1194,7 @@ static struct kunit_case mt7932_policy_test_cases[] = { KUNIT_CASE(mt7932_policy_corruption_test), KUNIT_CASE(mt7932_policy_order_test), KUNIT_CASE(mt7932_policy_v4_classes_test), + KUNIT_CASE(mt7932_policy_6g_pad_test), KUNIT_CASE(mt7932_policy_v4_preferred_test), KUNIT_CASE(mt7932_policy_v2_preferred_test), KUNIT_CASE(mt7932_policy_version_test), diff --git a/drivers/net/wireless/mediatek/mt7932/scan.h b/drivers/net/wireless/mediatek/mt7932/scan.h index 371b829026024..3be04716da001 100644 --- a/drivers/net/wireless/mediatek/mt7932/scan.h +++ b/drivers/net/wireless/mediatek/mt7932/scan.h @@ -8,6 +8,32 @@ /* CID03 carries this many band/channel slots, from 0xa0 up to 0xe0. */ #define MT7932_SCAN_CHANNELS 32 +/* cfg80211 owns PSC selection and RNR expansion for split scans. Until the + * firmware target list can carry its BSSID/short-SSID constraints, listen on + * those channels even when the request also contains an ordinary SSID. + */ +static inline bool +mt7932_scan_listen(const struct cfg80211_scan_request *request, + const struct ieee80211_channel *channel, u64 legacy_channels) +{ + unsigned int i; + + if (!request->n_ssids || + (channel->flags & (IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_RADAR))) + return true; + if (channel->band != NL80211_BAND_6GHZ) + return false; + if ((request->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ) || + request->n_6ghz_params || + !mt7932_channel_band(channel->band, channel->hw_value) || + !(legacy_channels & BIT_ULL((channel->hw_value - 1) / 4))) + return true; + for (i = 0; i < request->n_ssids; i++) + if (request->ssids[i].ssid_len) + return false; + return true; +} + /* Selected ordinary TX17/fullmac wildcard scan, not a pre-RF gate. * Caller must own BSS0 and filter cfg80211 DISABLED channels first. NO_IR * channels require mt7932_scan_passive() before submission. The body uses @@ -85,6 +111,15 @@ static inline void mt7932_scan_disconnected_passive(u8 body[0x4d4]) put_unaligned_le16(233, body + 0x4ca); } +/* Wait for the whole disconnected passive batch, not just one channel. + * Retain the existing five-second command/completion margin and active wait. + */ +static inline unsigned int mt7932_scan_discovery_timeout(const u8 body[0x4d4]) +{ + return 5000 + (body[2] ? 0 : + body[0x9f] * get_unaligned_le16(body + 0x4ca)); +} + /* CID03 carries up to four directed SSIDs in 36-byte slots (le32 length, * 32 bytes) at offset 8; selector 8 marks a directed request and byte 4 * counts the populated slots. Wildcard entries occupy no slot. A directed diff --git a/drivers/net/wireless/mediatek/mt7932/six-test.c b/drivers/net/wireless/mediatek/mt7932/six-test.c index c7578a4985e2c..59430b8483c45 100644 --- a/drivers/net/wireless/mediatek/mt7932/six-test.c +++ b/drivers/net/wireless/mediatek/mt7932/six-test.c @@ -143,6 +143,105 @@ static void mt7932_he_6g_test(struct kunit *test) KUNIT_EXPECT_EQ(test, mt7932_channel_6g_center(225, true), 0U); } +static void mt7932_scan_6g_policy_test(struct kunit *test) +{ + struct cfg80211_ssid ssids[2] = { {}, { .ssid_len = 6 } }; + struct cfg80211_scan_request request = { .ssids = ssids }; + struct ieee80211_channel channel = { + .band = NL80211_BAND_6GHZ, .hw_value = 5, + }; + u64 legacy = BIT_ULL(1) | BIT_ULL(9); + unsigned int i; + + KUNIT_EXPECT_TRUE(test, mt7932_scan_listen(&request, &channel, legacy)); + request.n_ssids = 1; /* wildcard on a PSC still listens */ + KUNIT_EXPECT_TRUE(test, mt7932_scan_listen(&request, &channel, legacy)); + request.n_ssids = 2; + KUNIT_EXPECT_FALSE(test, mt7932_scan_listen(&request, &channel, legacy)); + KUNIT_EXPECT_TRUE(test, mt7932_scan_listen(&request, &channel, 0)); + request.flags = NL80211_SCAN_FLAG_COLOCATED_6GHZ; + /* RNR constraints must not become an unconstrained directed probe, + * including a PSC with psc_no_listen and a non-PSC target channel. + */ + for (i = 1; i <= 233; i += 4) { + channel.hw_value = i; + KUNIT_EXPECT_TRUE(test, + mt7932_scan_listen(&request, &channel, ~0ULL)); + } + request.flags = 0; + request.n_6ghz_params = 1; + channel.hw_value = 37; + KUNIT_EXPECT_TRUE(test, mt7932_scan_listen(&request, &channel, legacy)); + request.n_6ghz_params = 0; + channel.flags = IEEE80211_CHAN_NO_IR; + KUNIT_EXPECT_TRUE(test, mt7932_scan_listen(&request, &channel, legacy)); + channel.flags = IEEE80211_CHAN_RADAR; + KUNIT_EXPECT_TRUE(test, mt7932_scan_listen(&request, &channel, legacy)); + channel.flags = 0; + channel.band = NL80211_BAND_2GHZ; + request.flags = NL80211_SCAN_FLAG_COLOCATED_6GHZ; + KUNIT_EXPECT_FALSE(test, mt7932_scan_listen(&request, &channel, legacy)); +} + +static void mt7932_scan_6g_golden_test(struct kunit *test) +{ + static const u8 channels[] = { 5, 37, 149 }; + static const u8 expected[0x4d4] = { + [0] = 1, [6] = 0x12, [7] = 1, + [0x9a] = 40, [0x9e] = 4, [0x9f] = 3, + [0xa0] = 3, 5, 3, 37, 3, 149, + [0xe0] = 21, + [0xe2] = 0x7f, 0x0a, 1, 0, 8, 0x8c, 1, 0x40, 0, 0, 0, 0, + 0x6b, 7, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + [0x33c] = 40, [0x4b9] = 0x18, [0x4bc] = 45, + [0x4c2] = 220, [0x4ca] = 233, + }; + u8 *body = kunit_kmalloc(test, sizeof(expected), GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, body); + KUNIT_ASSERT_EQ(test, mt7932_scan_body(body, sizeof(expected), 1, + NL80211_BAND_6GHZ, channels, 3), + 0x4d4); + KUNIT_EXPECT_EQ(test, mt7932_scan_discovery_timeout(body), 5000U); + mt7932_scan_disconnected_passive(body); + KUNIT_EXPECT_MEMEQ(test, body, expected, sizeof(expected)); + for (i = 1; i <= MT7932_SCAN_CHANNELS; i++) { + body[0x9f] = i; + KUNIT_EXPECT_EQ(test, mt7932_scan_discovery_timeout(body), + 5000U + 233 * i); + } + KUNIT_EXPECT_EQ(test, mt7932_scan_discovery_timeout(body), 12456U); + KUNIT_EXPECT_GT(test, mt7932_scan_discovery_timeout(body), 32U * 220); +} + +/* Association-start group selection does not establish runtime D7's band-2 + * procedures. Refuse before touching the container or a partial send plan. + */ +static void mt7932_cal_6g_runtime_rejected_test(struct kunit *test) +{ + static const u32 upper[] = { 0, 0x1000, 0x1001, U32_MAX }; + struct mt7932_cal_piece pieces[7], untouched[7]; + u8 request[16] = {}; + unsigned int channel, i, tag; + int ret; + + memset(untouched, 0xa5, sizeof(untouched)); + put_unaligned_le32(2, request + 8); + for (i = 0; i < ARRAY_SIZE(upper); i++) { + put_unaligned_le32(upper[i], request + 4); + for (channel = 1; channel <= 233; channel++) { + put_unaligned_le32(channel, request + 12); + memcpy(pieces, untouched, sizeof(pieces)); + tag = 0xffff; + ret = mt7932_cal_requested(NULL, 0, request, 16, 12, 0x89, pieces, &tag); + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, tag, 0U); + KUNIT_EXPECT_MEMEQ(test, pieces, untouched, sizeof(pieces)); + } + } +} + static void mt7932_cal_6g_test(struct kunit *test) { u8 blob[52] = {}; @@ -220,6 +319,9 @@ static struct kunit_case mt7932_channel_cases[] = { KUNIT_CASE(mt7932_he_6g_test), KUNIT_CASE(mt7932_6g_security_test), KUNIT_CASE(mt7932_cal_6g_test), + KUNIT_CASE(mt7932_scan_6g_policy_test), + KUNIT_CASE(mt7932_scan_6g_golden_test), + KUNIT_CASE(mt7932_cal_6g_runtime_rejected_test), {} }; diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 62aedd18c7024..9bc5db8ed8031 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -929,6 +929,7 @@ int mt_register_regulatory_gate(struct mt7932 *m) m->band6.iftype_data = &m->he6; m->band6.n_iftype_data = 1; m->wiphy->bands[NL80211_BAND_6GHZ] = &m->band6; + m->wiphy->flags |= WIPHY_FLAG_SPLIT_SCAN_6GHZ; } } ret = wiphy_register(m->wiphy); diff --git a/tools/asahi/mt7932-scan-tests.py b/tools/asahi/mt7932-scan-tests.py new file mode 100644 index 0000000000000..a47c7b938af1b --- /dev/null +++ b/tools/asahi/mt7932-scan-tests.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise real scan batching and discovery waits with synthetic channels.""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + +ROOT = Path(__file__).resolve().parents[2] +DRIVER = ROOT / 'drivers/net/wireless/mediatek/mt7932' + + +def function(source, name): + match = re.search(r'^(?:static )?(?:inline )?[^\n;]+\b' + re.escape(name) + + r'\([^;]*?\)\s*\{', source, re.M) + if not match: + raise ValueError('missing function ' + name) + end = source.index('{', match.start()) + 1 + depth = 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[match.start():end] + + +PREFIX = r''' +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; typedef uint16_t u16; typedef uint32_t u32; typedef uint64_t u64; +enum nl80211_band { NL80211_BAND_2GHZ, NL80211_BAND_5GHZ, NL80211_BAND_6GHZ=3 }; +#define BIT_ULL(n) (1ULL << (n)) +#define ARRAY_SIZE(x) (sizeof(x)/sizeof(*(x))) +#define IEEE80211_CHAN_DISABLED 1 +#define IEEE80211_CHAN_NO_IR 2 +#define IEEE80211_CHAN_RADAR 4 +#define IEEE80211_CHAN_NO_20MHZ 8 +#define NL80211_SCAN_FLAG_COLOCATED_6GHZ 0x4000 +#define WLAN_AKM_SUITE_SAE 0x000fac08 +#define WLAN_AKM_SUITE_OWE 0x000fac12 +struct ieee80211_channel {enum nl80211_band band; u32 flags; u8 hw_value;}; +struct ieee80211_supported_band {enum nl80211_band band; unsigned n_channels; struct ieee80211_channel *channels;}; +struct cfg80211_ssid {u8 ssid[32]; unsigned ssid_len;}; +struct cfg80211_scan_request { + struct cfg80211_ssid *ssids; unsigned n_ssids,n_channels,flags,n_6ghz_params; + const u8 *ie; size_t ie_len; struct ieee80211_channel *channels[80]; +}; +static u16 get_unaligned_le16(const void *p){const u8 *b=p;return b[0]|b[1]<<8;} +static void put_unaligned_le16(u16 v,void *p){u8 *b=p;b[0]=v;b[1]=v>>8;} +static void put_unaligned_le32(u32 v,void *p){u8 *b=p;for(unsigned i=0;i<4;i++)b[i]=v>>(8*i);} +#define READ_ONCE(x) (x) +#define spin_lock_irqsave(l,f) ((void)(l),(f)=0) +#define spin_unlock_irqrestore(l,f) ((void)(l),(void)(f)) +#define msecs_to_jiffies(x) (x) +#define dev_dbg(...) ((void)0) +static bool command_locked=true; +#define mutex_unlock(l) do {assert(command_locked);command_locked=false;} while(0) +#define mutex_lock(l) do {assert(!command_locked);command_locked=true;} while(0) +static void reinit_completion(int *c){*c=0;} +static void schedule_delayed_work(int *w,unsigned long t){assert(t==60000);} +struct mt7932 { + struct cfg80211_scan_request *scan_request; + unsigned scan_batch,scan_cursor,scan_band,scan_seq; + int response_lock,scan_done,scan_timeout_work,command_mutex,discovery_done; + bool scan_aborted,disconnecting,connected,scan_finished,rf_ready; + bool connect_cancelled,discovering,discovery_finished,link_failed; + u8 scan_home_channel; enum nl80211_band scan_home_band; + u64 six_legacy_channels; + unsigned connect_akm; int connect_error; + struct ieee80211_supported_band band2,band5,band6; + struct ieee80211_channel *connect_channel_req; + u8 connect_ssid[32]; unsigned connect_ssid_length; + void *connect_bss,*pdev; +}; +static unsigned commands,waits; +static u8 bodies[4][0x4d4]; +static struct mt7932 *active; +static bool mt_rf_allowed(struct mt7932 *m){return m->rf_ready;} +static int mt_request(struct mt7932 *m,u8 cid,bool runtime,bool set,bool reply,const void *body,size_t n){ + assert(command_locked&&cid==3&&runtime&&set&&!reply&&n==0x4d4&&commands<4); + memcpy(bodies[commands++],body,n);return 0; +} +static unsigned long wait_for_completion_timeout(int *c,unsigned long timeout){ + assert(!command_locked&&c==&active->discovery_done&&commands); + const u8 *b=bodies[commands-1];assert(!b[2]); + assert(timeout==5000+233U*b[0x9f]); + assert(timeout>220U*b[0x9f]+100); + waits++;active->discovery_finished=true;return 1; +} +static void *mt_connect_find_bss(struct mt7932 *m){return commands==2?m:NULL;} +HEADERS +SUBMIT +DISCOVER +static void reset(struct mt7932 *m){ + memset(m,0,sizeof(*m));memset(bodies,0,sizeof(bodies)); + m->rf_ready=true;commands=waits=0;active=m;command_locked=true; +} +int main(void){ + struct mt7932 m;struct cfg80211_scan_request request={}; + struct cfg80211_ssid ssid={.ssid="hidden",.ssid_len=6}; + struct ieee80211_channel channels[59]; + for(unsigned i=0;i<59;i++){ + channels[i]=(struct ieee80211_channel){.band=NL80211_BAND_6GHZ,.hw_value=1+4*i}; + request.channels[i]=channels+i; + } + request.n_channels=59;request.ssids=&ssid;request.n_ssids=1; + /* Even legacy channels must listen when RNR constraints are not serialized. */ + for(unsigned context=0;context<2;context++){ + reset(&m);m.scan_request=&request;m.six_legacy_channels=(1ULL<<59)-1; + request.flags=context?0:NL80211_SCAN_FLAG_COLOCATED_6GHZ; + request.n_6ghz_params=context?1:0; + assert(mt_scan_submit_next(&m)==0);assert(commands==1&&bodies[0][0x9f]==32); + assert(mt_scan_submit_next(&m)==0);assert(commands==2&&bodies[1][0x9f]==27); + assert(mt_scan_submit_next(&m)==1);assert(commands==2); + for(unsigned batch=0;batch<2;batch++){ + const u8 *b=bodies[batch];assert(!b[2]&&!b[3]&&!b[4]); + assert(get_unaligned_le16(b+0x4c2)==220); + for(unsigned i=0;iflags & NL80211_SCAN_FLAG_COLOCATED_6GHZ)', 'false').replace( + 'request->n_6ghz_params ||', 'false ||') + variants['mutant-rnr-probes'] = bad + bad = dict(parts) + bad['DISCOVER'] = bad['DISCOVER'].replace('mt7932_scan_discovery_timeout(body)', '5000') + variants['mutant-short-wait'] = bad + results = {} + for name, content in variants.items(): + source = PREFIX + for key, value in content.items(): + source = source.replace('\n' + key + '\n', '\n' + value + '\n') + c = args.out / (name + '.c') + binary = args.out / name + c.write_text(source) + subprocess.run(['cc', '-std=gnu11', '-Wall', '-Werror', '-Wno-unused-function', + '-Wno-unused-variable', '-fsanitize=address,undefined', + '-fno-omit-frame-pointer', str(c), '-o', str(binary)], check=True) + run = subprocess.run([str(binary.resolve())], capture_output=True, text=True) + (args.out / (name + '.log')).write_text(run.stdout + run.stderr) + results[name] = {'exit': run.returncode, + 'source_sha256': hashlib.sha256(source.encode()).hexdigest()} + print(name, json.dumps(results[name]), run.stdout.strip()) + if name == 'production': + assert run.returncode == 0, run.stderr + else: + assert run.returncode == -6 and 'Assertion' in run.stderr, run.stderr + (args.out / 'results.json').write_text(json.dumps(results, indent=2) + '\n') + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 24f7a9e9134fe..81d496c4d0050 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -118,6 +118,7 @@ def mutations(flag, names, base, expect=KILL): 'mt7932-owe-retry-tests.py': [(out(), PASS)], 'mt7932-policy-6ghz-tests.py': [([], PASS)], 'mt7932-policy-classes-tests.py': [([], PASS)], + 'mt7932-scan-tests.py': [(out(), PASS)], 'neo-auxiliary-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutation'], KILL), (out('--output') + ['--old'], KILL)], 'neo-stability-tests.py': [(out('--output'), PASS)] From e0bb4f573b54dd1f4af5a13bec37be9ca4fa7a4f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 19:26:50 +1100 Subject: [PATCH 1515/1579] sched/fair: Preserve misfit runtime and serialize rotation swaps Concurrent per-rq rotation workers can read the same destination cursor, select the same task and advance the cursor before migrate_swap() has completed. Only the first swap can then pass the locked task_cpu() check; the other requests lose their turn. Serialize selection through completion and advance the shared cursor only after a successful swap. Recheck the feature after taking the mutex so queued work honours a disabled feature. The rq-local misfit timer also restarts whenever set_next_task_fair() selects another task. A brief recurring kernel worker can therefore keep a CPU-bound task below the dwell threshold indefinitely. Account the interval with a per-task executed-runtime checkpoint instead. Reset it on fork, wakeup, migration and whenever the task fits its CPU; preemption by itself does not discard accumulated misfit execution. Keep the existing four-base-slice threshold and priority, affinity and idle-CPU checks. Add sched_misfit_rotate to distinguish queued requests, target-selection failures and completed swap results from ordinary load-balance migrations. This also works without NUMA balancing enabled. A deterministic host fixture using the actual selection/trigger functions reproduces one successful swap out of four synchronized original requests, and no queued rotation under repeated short preemptions. With this change all four swaps succeed and the preempted task reaches the runtime threshold. These checks do not establish hardware performance or Linux CPU-stop and hotplug behaviour; those require scheduler tracing and benchmark A/B runs. Assisted-by: LLM Signed-off-by: Ryan Murray --- include/linux/sched.h | 2 ++ include/trace/events/sched.h | 32 ++++++++++++++++++++ kernel/sched/core.c | 1 + kernel/sched/fair.c | 57 +++++++++++++++++++++--------------- kernel/sched/sched.h | 9 +----- 5 files changed, 70 insertions(+), 31 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index a2e1894a9e13e..57ef9a060eb65 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -939,6 +939,8 @@ struct task_struct { unsigned int policy; unsigned long max_allowed_capacity; + /* Runtime checkpoint for misfit rotation, protected by the rq lock. */ + u64 misfit_runtime; int nr_cpus_allowed; const cpumask_t *cpus_ptr; cpumask_t *user_cpus_ptr; diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index 535860581f159..ad1a74a821dd5 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -10,6 +10,38 @@ #include #include +/* + * A queued misfit rotation has result -EINPROGRESS and no destination. + * The completion reports migrate_swap()'s result, or a selection failure. + */ +TRACE_EVENT(sched_misfit_rotate, + + TP_PROTO(struct task_struct *p, struct task_struct *t, + int src_cpu, int dst_cpu, int result), + + TP_ARGS(p, t, src_cpu, dst_cpu, result), + + TP_STRUCT__entry( + __field(pid_t, src_pid) + __field(pid_t, dst_pid) + __field(int, src_cpu) + __field(int, dst_cpu) + __field(int, result) + ), + + TP_fast_assign( + __entry->src_pid = p->pid; + __entry->dst_pid = t ? t->pid : 0; + __entry->src_cpu = src_cpu; + __entry->dst_cpu = dst_cpu; + __entry->result = result; + ), + + TP_printk("src_pid=%d dst_pid=%d src_cpu=%d dst_cpu=%d result=%d", + __entry->src_pid, __entry->dst_pid, __entry->src_cpu, + __entry->dst_cpu, __entry->result) +); + /* * Tracepoint for calling kthread_stop, performed to end a kthread: */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 96e6eaf36b856..c481404713aaa 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4664,6 +4664,7 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p) p->se.on_rq = 0; p->se.exec_start = 0; p->se.sum_exec_runtime = 0; + p->misfit_runtime = 0; p->se.prev_sum_exec_runtime = 0; p->se.nr_migrations = 0; p->se.vruntime = 0; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index b053e300ba43c..3b38f35c65413 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6353,7 +6353,8 @@ static inline void update_misfit_status(struct task_struct *p, struct rq *rq) task_fits_cpu(p, cpu)) { rq->misfit_task_load = 0; - rq->misfit_task = NULL; + if (p) + p->misfit_runtime = p->se.sum_exec_runtime; return; } @@ -6362,11 +6363,6 @@ static inline void update_misfit_status(struct task_struct *p, struct rq *rq) * task_h_load() returns 0. */ rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1); - - if (rq->misfit_task != p) { - rq->misfit_task = p; - rq->misfit_since = jiffies; - } } void __setparam_fair(struct task_struct *p, const struct sched_attr *attr) @@ -8225,6 +8221,9 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) unsigned long weight; bool curr; + if (flags & ENQUEUE_WAKEUP) + p->misfit_runtime = se->sum_exec_runtime; + if (task_is_throttled(p) && enqueue_throttled_task(p)) return; @@ -10004,6 +10003,8 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) { struct sched_entity *se = &p->se; + p->misfit_runtime = se->sum_exec_runtime; + if (!task_on_rq_migrating(p)) { remove_entity_load_avg(se); @@ -15041,7 +15042,7 @@ static __latent_entropy void sched_balance_softirq(void) * ends with a tail in which the tasks that started on small CPUs finish * alone on big ones. * - * Once a task has been misfit on a CPU for a few base slices, swap it with + * Once a task has executed as a misfit for a few base slices, swap it with * the task on a busy bigger CPU. Bigger CPUs are taken round robin, so each * task on them takes its turn on a smaller CPU: the big CPUs are time * shared, as one runqueue would share them. It only applies when every @@ -15049,13 +15050,10 @@ static __latent_entropy void sched_balance_softirq(void) * misfit task through load balancing instead), and it never moves down a * task with a higher weight or a higher uclamp_min than the misfit task. */ +/* Serialize destination selection through completion of the swap. */ +static DEFINE_MUTEX(misfit_rotate_mutex); static int misfit_rotate_last_cpu; -static unsigned long misfit_rotate_period(void) -{ - return max(1UL, nsecs_to_jiffies(4ULL * sysctl_sched_base_slice)); -} - static bool misfit_rotate_candidate(struct task_struct *p) { return p && p->sched_class == &fair_sched_class && @@ -15070,18 +15068,26 @@ static void misfit_rotate_work(struct work_struct *work) unsigned long src_cap = arch_scale_cpu_capacity(src_cpu); struct task_struct *p, *t = NULL; bool idle = false; + int ret = -EINVAL; p = xchg(&src_rq->misfit_rotate_p, NULL); if (!p) return; + mutex_lock(&misfit_rotate_mutex); + + if (!sched_feat(MISFIT_ROTATE)) { + ret = -ECANCELED; + goto out; + } + /* The task may have moved, slept or shrunk since the tick. */ if (task_cpu(p) != src_cpu || !task_on_rq_queued(p) || !misfit_rotate_candidate(p) || task_fits_cpu(p, src_cpu)) goto out; rcu_read_lock(); - start = READ_ONCE(misfit_rotate_last_cpu); + start = misfit_rotate_last_cpu; for_each_cpu_wrap(cpu, p->cpus_ptr, start + 1) { struct rq *rq = cpu_rq(cpu); struct task_struct *c; @@ -15112,17 +15118,21 @@ static void misfit_rotate_work(struct work_struct *work) if (idle) t = NULL; - if (t) { + if (t) get_task_struct(t); - WRITE_ONCE(misfit_rotate_last_cpu, dst_cpu); - } rcu_read_unlock(); + ret = idle ? -EAGAIN : -EBUSY; if (t) { - migrate_swap(p, t, dst_cpu, src_cpu); - put_task_struct(t); + ret = migrate_swap(p, t, dst_cpu, src_cpu); + if (!ret) + misfit_rotate_last_cpu = dst_cpu; } out: + trace_sched_misfit_rotate(p, t, src_cpu, dst_cpu, ret); + if (t) + put_task_struct(t); + mutex_unlock(&misfit_rotate_mutex); put_task_struct(p); } @@ -15134,16 +15144,15 @@ static void misfit_rotate_trigger(struct rq *rq) if (!sched_feat(MISFIT_ROTATE) || !sched_asym_cpucap_active()) return; - if (READ_ONCE(rq->misfit_task) != p || !READ_ONCE(rq->misfit_task_load) || + if (!READ_ONCE(rq->misfit_task_load) || READ_ONCE(rq->nr_running) != 1 || !misfit_rotate_candidate(p)) return; - if (time_before(jiffies, READ_ONCE(rq->misfit_since) + misfit_rotate_period())) + /* Preemption must not restart a CPU-bound task's misfit interval. */ + if (p->se.sum_exec_runtime - p->misfit_runtime < + 4ULL * sysctl_sched_base_slice) return; - /* Whatever the outcome, try again no sooner than one period from now. */ - WRITE_ONCE(rq->misfit_since, jiffies); - if (READ_ONCE(rq->misfit_rotate_p)) return; @@ -15152,6 +15161,8 @@ static void misfit_rotate_trigger(struct rq *rq) put_task_struct(p); return; } + p->misfit_runtime = p->se.sum_exec_runtime; + trace_sched_misfit_rotate(p, NULL, cpu_of(rq), -1, -EINPROGRESS); queue_work_on(cpu_of(rq), system_highpri_wq, &rq->misfit_rotate_work); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 24a25e3dc8c27..9dd8c6e6f73b5 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1285,14 +1285,7 @@ struct rq { unsigned char idle_balance; unsigned long misfit_task_load; - /* - * Misfit rotation: the misfit task (compared, never dereferenced), - * the jiffies when it became misfit here, and the task handed (with - * a reference) to the work that swaps it with the task on a busy - * higher-capacity CPU. - */ - struct task_struct *misfit_task; - unsigned long misfit_since; + /* Task reference handed to the misfit rotation work. */ struct task_struct *misfit_rotate_p; struct work_struct misfit_rotate_work; From aced8378a80667bca3cbd0ce626ff8a3613efd1a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 20:25:15 +1100 Subject: [PATCH 1516/1579] hwmon: macsmc: Retry manual mode while the fan unlock settles The firmware-policy change kept the read-side wait for a fan to leave system mode, but removed the bounded retry of the following manual-mode write. An EIO from that first write now aborts the user request and immediately relocks the fans. The previous path could retry it. Keep the mode read before unlocking and before each attempt, and retry EIO or EBUSY from the manual-mode write within the existing ten-second unlock budget. Only write the target after manual mode succeeds. Other errors still return immediately, and exhausted retries retain the last error and release the unlock when no other fan is manual. This does not reinstate the deliberately refused pre-unlock write or autonomous fan policy. Direct legacy mode writes are unchanged. Extend the KUnit model to reject manual mode temporarily after leaving system mode, starting with a zero target. Cover eventual acceptance, permanent refusal and relock without a target write, and takeover of an already spinning firmware fan. Fixes: cc3766d3c017 ("hwmon: macsmc: Leave fan policy to firmware until a user requests control") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hwmon/macsmc-hwmon-test.c | 61 +++++++++++++++++++++++++++++++ drivers/hwmon/macsmc-hwmon.c | 21 ++++++++--- 2 files changed, 76 insertions(+), 6 deletions(-) diff --git a/drivers/hwmon/macsmc-hwmon-test.c b/drivers/hwmon/macsmc-hwmon-test.c index 1f8fa33dc2e0e..e549c6d61441d 100644 --- a/drivers/hwmon/macsmc-hwmon-test.c +++ b/drivers/hwmon/macsmc-hwmon-test.c @@ -110,6 +110,8 @@ struct macsmc_fan_test { int ftst_writes; int refused; int mode_reads; + int manual_busy[2]; + int target_writes; smc_key fail_key; }; @@ -175,9 +177,15 @@ static int fan_test_write(struct apple_smc *smc, smc_key key, const void *buf, s t->refused++; return -EIO; } + if (value == MACSMC_FAN_MODE_MANUAL && t->manual_busy[i]) { + t->manual_busy[i]--; + return -EIO; + } t->mode[i] = value; return size; case __SMC_KEY('F', 0, 'T', 'g'): + KUNIT_EXPECT_EQ(t->test, t->mode[i], MACSMC_FAN_MODE_MANUAL); + t->target_writes++; t->target[i] = value; return size; } @@ -395,7 +403,60 @@ static void macsmc_fan_test_unlock_fails(struct kunit *test) KUNIT_EXPECT_EQ(test, t->refused, 0); } +/* A stopped fan may leave mode 3 before the manual-mode write is ready. */ +static void macsmc_fan_test_stopped_manual_busy(struct kunit *test) +{ + struct macsmc_fan_test *t = test->priv; + + KUNIT_ASSERT_EQ(test, t->target[0], 0); + t->manual_busy[0] = 2; + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), 0); + KUNIT_EXPECT_EQ(test, t->manual_busy[0], 0); + KUNIT_EXPECT_EQ(test, t->target[0], FAN_TEST_F2500); + KUNIT_EXPECT_EQ(test, t->target_writes, 1); + KUNIT_EXPECT_EQ(test, t->hwmon.fan.manual, 1); + /* Unlock, three mode attempts, then the target. No locked-mode write. */ + KUNIT_EXPECT_EQ(test, t->writes, 5); + KUNIT_EXPECT_EQ(test, t->refused, 0); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 0), 0); + fan_test_released(test, t); + KUNIT_EXPECT_EQ(test, t->ftst_writes, 2); +} + +/* Exhausting the post-unlock retries must release Ftst, without a target. */ +static void macsmc_fan_test_manual_busy_timeout(struct kunit *test) +{ + struct macsmc_fan_test *t = test->priv; + + t->fail_key = SMC_KEY(F0Md); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), -EIO); + fan_test_released(test, t); + KUNIT_EXPECT_GT(test, t->writes, 3); + KUNIT_EXPECT_EQ(test, t->target_writes, 0); + KUNIT_EXPECT_EQ(test, t->ftst_writes, 2); + KUNIT_EXPECT_EQ(test, t->refused, 0); +} + +/* Taking over an already spinning firmware fan needs no unlock or retry. */ +static void macsmc_fan_test_spinning(struct kunit *test) +{ + struct macsmc_fan_test *t = test->priv; + + t->mode[0] = MACSMC_FAN_MODE_AUTO; + t->target[0] = FAN_TEST_F1350; + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), 0); + KUNIT_EXPECT_EQ(test, t->target[0], FAN_TEST_F2500); + KUNIT_EXPECT_EQ(test, t->writes, 2); + KUNIT_EXPECT_EQ(test, t->ftst_writes, 0); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 0), 0); + KUNIT_EXPECT_EQ(test, t->mode[0], MACSMC_FAN_MODE_AUTO); + KUNIT_EXPECT_EQ(test, t->hwmon.fan.manual, 0); +} + static struct kunit_case macsmc_fan_test_cases[] = { + KUNIT_CASE(macsmc_fan_test_stopped_manual_busy), + KUNIT_CASE(macsmc_fan_test_manual_busy_timeout), + KUNIT_CASE(macsmc_fan_test_spinning), KUNIT_CASE(macsmc_fan_test_unlock_and_release), KUNIT_CASE(macsmc_fan_test_clamp), KUNIT_CASE(macsmc_fan_test_target_write_fails), diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index 097fa6c40c610..cb824b252ddcd 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -506,16 +506,24 @@ static int macsmc_hwmon_fan_manual(struct macsmc_hwmon *hwmon, timeout = jiffies + msecs_to_jiffies(MACSMC_FAN_MODE_TIMEOUT_MS); for (;;) { ret = macsmc_hwmon_fan_read_mode(hwmon, fan, &mode); - if (ret || mode != MACSMC_FAN_MODE_SYSTEM) - break; + if (ret) + goto relock; + if (mode != MACSMC_FAN_MODE_SYSTEM) { + /* Leaving system mode need not mean writes are ready. */ + ret = macsmc_hwmon_write_key(hwmon->smc, &fan->mode, + MACSMC_FAN_MODE_MANUAL); + if (ret >= 0) + goto manual; + if (ret != -EIO && ret != -EBUSY) + goto relock; + } if (time_after(jiffies, timeout)) { - ret = -ETIMEDOUT; - break; + if (mode == MACSMC_FAN_MODE_SYSTEM) + ret = -ETIMEDOUT; + goto relock; } msleep(MACSMC_FAN_MODE_POLL_MS); } - if (ret) - goto relock; } set_manual: @@ -523,6 +531,7 @@ static int macsmc_hwmon_fan_manual(struct macsmc_hwmon *hwmon, if (ret < 0) goto relock; +manual: fan->manual = true; hwmon->fan.manual++; return 0; From ad65fcc37f22c4f94e1cb4cf23b536ba869fa7c3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:58:48 +1100 Subject: [PATCH 1517/1579] tools/asahi: register the AVE probe PM host test run-host-tests.py fails the run when a test under tools/asahi is not registered. ave-probe-pm-tests.py came in without an entry, so the whole host-test run stopped with rc 2. Register it with its passing run and its negative control, which exits 0 after confirming the pre-fix probe fails. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/run-host-tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 24f7a9e9134fe..425fed5351843 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -54,6 +54,7 @@ def mutations(flag, names, base, expect=KILL): (out() + ['--old-transport', '4db3a5a22c43'], PASS)], 'ave-pm-tests.py': [(out(), PASS), (out() + ['--old-ref', '4db3a5a22c43'], PASS)], + 'ave-probe-pm-tests.py': [(out(), PASS), (out() + ['--negative-control'], PASS)], 'binding-lifetime-tests.py': [(out(), PASS)] + mutations('--mutation', ('no-close', 'no-join'), out()), 'apple-dart-fw-tests.py': [(out(), PASS)], From 589f5ca65648debbc22d00b9e0c21a12df9561eb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 00:06:37 +1100 Subject: [PATCH 1518/1579] arm64: dts: apple: t6040-j616s: Put the CPU backstop above the PMP's operating point The CPU zones' passive trip sat at 113 C on the assumption that the PMP's fast die-temperature loop settles TCMz near 111 C. It does not. The loop holds the PMP's own hot-spot sensors at 110.0 C (the target in the ADT's fast-die-ctrl-loop), and the SMC's die maxima read a few degrees above that. A 90 s all-core busy loop on the J616s with both fans running took TCMz to 113.4 C after 19 s and held it between 113.4 and 114.7 C, the P clusters trimmed by the PMP's effort limit (their cpufreq maximum stepping between 2868 and 4512 MHz) and the E cluster untouched at 2592 MHz. A backstop at 113 C would therefore act in normal use and fight the firmware loop as soon as the zones can throttle. Move it to 116 C on all seventeen CPU zones, above the measured settling band and below the 118 C critical trip, and correct the comment. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 46 ++++++++++++----------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 872381a098080..41a0c639b69ba 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -301,16 +301,18 @@ /* * CPU die temperature: SMC key TCMz (CPU die max), sensor 0 of - * smc_hwmon. The PMP's fast die-temperature limit regulates (TCMz - * settles near 111 C); this zone is the backstop above it: passive at - * 113 C, critical (orderly shutdown) at 118 C, polled once a second as - * the SMC refreshes TCMz. The cpufreq cooling device sits on a + * smc_hwmon. The PMP's fast die-temperature limit regulates: its loop + * holds its own hot-spot sensors at 110 C (ADT fast-die-ctrl-loop), + * and under a sustained all-core load TCMz then settles at 113.4 to + * 114.7 C with the fans running. This zone is the backstop above that: + * passive at 116 C, critical (orderly shutdown) at 118 C, polled once a + * second as the SMC refreshes TCMz. The cpufreq cooling device sits on a * cluster's policy CPU, so each map lists every CPU of its cluster. * * Per-cluster and GPU zones follow, one per SMC die-maximum sensor * (smc_hwmon channels 1-23, in the order of the temperature nodes * below): Tex0-3 throttle the E cluster, Tpx0-5 the first P cluster - * and Tpx8-D the second, with the same 113/118 C backstop as TCMz. + * and Tpx8-D the second, with the same 116/118 C backstop as TCMz. * Unlike the fanless J700, this machine has two SMC-driven fans and * the PMP regulates the dies itself, so these zones only catch a * failure of that loop; they never act in normal use. @@ -370,7 +372,7 @@ J616S_FAN_TRIPS(cpu_die) cpu_die_passive: cpu-die-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -423,7 +425,7 @@ trips { cpu_e0_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -454,7 +456,7 @@ trips { cpu_e1_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -485,7 +487,7 @@ trips { cpu_e2_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -516,7 +518,7 @@ trips { cpu_e3_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -547,7 +549,7 @@ trips { cpu_p00_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -579,7 +581,7 @@ trips { cpu_p01_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -611,7 +613,7 @@ trips { cpu_p02_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -643,7 +645,7 @@ trips { cpu_p03_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -675,7 +677,7 @@ trips { cpu_p04_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -707,7 +709,7 @@ trips { cpu_p05_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -739,7 +741,7 @@ trips { cpu_p10_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -771,7 +773,7 @@ trips { cpu_p11_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -803,7 +805,7 @@ trips { cpu_p12_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -835,7 +837,7 @@ trips { cpu_p13_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -867,7 +869,7 @@ trips { cpu_p14_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -899,7 +901,7 @@ trips { cpu_p15_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; From 76815fab2d2f5ecb8fca7a27ce8a547fe5e8afc7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 00:06:37 +1100 Subject: [PATCH 1519/1579] arm64: dts: apple: t6040: Add a measured CPU energy model T6040's CPU OPP tables carry no opp-microwatt, so no energy model registers and power_allocator has no power actors. The CPU thermal zones then never throttle anything. Add per-core power for the E, P0 and P1 tables, measured on a J616s running at each state with the cluster pinned and read from the SMC core-rail meters: PR0b for P0, PP0b for P1 and PP2b for E. The meters were identified by busying one core of each cluster at its lowest and highest state and checking which power and voltage keys follow. One busy P core gives clean readings up to 4104 MHz. At 4416 and 4512 MHz the PMP's die-temperature limit pulls the core back to 4044 MHz within two seconds, so the P values come from a k * V^2 * f + c fit to the clean points (rms 0.2 W), using the rail voltage measured at each state. This also keeps the tables monotonic where the 4 s samples are noisy. One busy E core does not register on PP2b below 2352 MHz. As on T8140, the E values are the idle cluster at that state plus a quarter of the increase with all four E cores busy. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040.dtsi | 53 ++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index f2c603b3f01e6..da1d55bd9a13b 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -298,6 +298,14 @@ * in kHz. opp-level is the hardware p-state. Each P cluster has its * own table even though the frequencies are the same: opp-shared * means one clock domain per table. + * + * opp-microwatt is the power of one busy core, measured on a J616s + * from the SMC core-rail meters (P0 PR0b, P1 PP0b, E PP2b) with the + * cluster pinned at each state. The P values follow a k * V^2 * f + c + * fit to the sweep, because the PMP's die-temperature limit clips the + * two top states within seconds. One busy E core does not register on + * its meter below 2352 MHz, so the E values are the idle cluster plus + * a quarter of the four-core increase. */ opp_e: opp-table-e { compatible = "operating-points-v2"; @@ -306,30 +314,37 @@ opp01 { opp-hz = /bits/ 64 <1020000000>; opp-level = <1>; + opp-microwatt = <81900>; }; opp02 { opp-hz = /bits/ 64 <1404000000>; opp-level = <2>; + opp-microwatt = <133100>; }; opp03 { opp-hz = /bits/ 64 <1788000000>; opp-level = <3>; + opp-microwatt = <209400>; }; opp04 { opp-hz = /bits/ 64 <2112000000>; opp-level = <4>; + opp-microwatt = <301400>; }; opp05 { opp-hz = /bits/ 64 <2352000000>; opp-level = <5>; + opp-microwatt = <383900>; }; opp06 { opp-hz = /bits/ 64 <2532000000>; opp-level = <6>; + opp-microwatt = <478500>; }; opp07 { opp-hz = /bits/ 64 <2592000000>; opp-level = <7>; + opp-microwatt = <531000>; }; }; @@ -340,78 +355,97 @@ opp01 { opp-hz = /bits/ 64 <1260000000>; opp-level = <1>; + opp-microwatt = <513000>; }; opp02 { opp-hz = /bits/ 64 <1512000000>; opp-level = <2>; + opp-microwatt = <715000>; }; opp03 { opp-hz = /bits/ 64 <1800000000>; opp-level = <3>; + opp-microwatt = <907500>; }; opp04 { opp-hz = /bits/ 64 <2088000000>; opp-level = <4>; + opp-microwatt = <1158200>; }; opp05 { opp-hz = /bits/ 64 <2352000000>; opp-level = <5>; + opp-microwatt = <1494500>; }; opp06 { opp-hz = /bits/ 64 <2616000000>; opp-level = <6>; + opp-microwatt = <1782600>; }; opp07 { opp-hz = /bits/ 64 <2868000000>; opp-level = <7>; + opp-microwatt = <2209600>; }; opp08 { opp-hz = /bits/ 64 <3096000000>; opp-level = <8>; + opp-microwatt = <2636700>; }; opp09 { opp-hz = /bits/ 64 <3300000000>; opp-level = <9>; + opp-microwatt = <3247200>; }; opp10 { opp-hz = /bits/ 64 <3468000000>; opp-level = <10>; + opp-microwatt = <3668700>; }; opp11 { opp-hz = /bits/ 64 <3624000000>; opp-level = <11>; + opp-microwatt = <4128200>; }; opp12 { opp-hz = /bits/ 64 <3756000000>; opp-level = <12>; + opp-microwatt = <4819300>; }; opp13 { opp-hz = /bits/ 64 <3852000000>; opp-level = <13>; + opp-microwatt = <4977700>; }; opp14 { opp-hz = /bits/ 64 <3924000000>; opp-level = <14>; + opp-microwatt = <5315900>; }; opp15 { opp-hz = /bits/ 64 <3996000000>; opp-level = <15>; + opp-microwatt = <5575900>; }; opp16 { opp-hz = /bits/ 64 <4044000000>; opp-level = <16>; + opp-microwatt = <5773400>; }; opp17 { opp-hz = /bits/ 64 <4104000000>; opp-level = <17>; + opp-microwatt = <5969200>; }; opp18 { opp-hz = /bits/ 64 <4416000000>; opp-level = <18>; + opp-microwatt = <8274900>; }; opp19 { opp-hz = /bits/ 64 <4512000000>; opp-level = <19>; + opp-microwatt = <9256900>; }; }; @@ -422,78 +456,97 @@ opp01 { opp-hz = /bits/ 64 <1260000000>; opp-level = <1>; + opp-microwatt = <561800>; }; opp02 { opp-hz = /bits/ 64 <1512000000>; opp-level = <2>; + opp-microwatt = <772500>; }; opp03 { opp-hz = /bits/ 64 <1800000000>; opp-level = <3>; + opp-microwatt = <988900>; }; opp04 { opp-hz = /bits/ 64 <2088000000>; opp-level = <4>; + opp-microwatt = <1302300>; }; opp05 { opp-hz = /bits/ 64 <2352000000>; opp-level = <5>; + opp-microwatt = <1644100>; }; opp06 { opp-hz = /bits/ 64 <2616000000>; opp-level = <6>; + opp-microwatt = <2003700>; }; opp07 { opp-hz = /bits/ 64 <2868000000>; opp-level = <7>; + opp-microwatt = <2349000>; }; opp08 { opp-hz = /bits/ 64 <3096000000>; opp-level = <8>; + opp-microwatt = <2723000>; }; opp09 { opp-hz = /bits/ 64 <3300000000>; opp-level = <9>; + opp-microwatt = <3333200>; }; opp10 { opp-hz = /bits/ 64 <3468000000>; opp-level = <10>; + opp-microwatt = <3711900>; }; opp11 { opp-hz = /bits/ 64 <3624000000>; opp-level = <11>; + opp-microwatt = <4129900>; }; opp12 { opp-hz = /bits/ 64 <3756000000>; opp-level = <12>; + opp-microwatt = <4666300>; }; opp13 { opp-hz = /bits/ 64 <3852000000>; opp-level = <13>; + opp-microwatt = <4881000>; }; opp14 { opp-hz = /bits/ 64 <3924000000>; opp-level = <14>; + opp-microwatt = <5176700>; }; opp15 { opp-hz = /bits/ 64 <3996000000>; opp-level = <15>; + opp-microwatt = <5306500>; }; opp16 { opp-hz = /bits/ 64 <4044000000>; opp-level = <16>; + opp-microwatt = <5618900>; }; opp17 { opp-hz = /bits/ 64 <4104000000>; opp-level = <17>; + opp-microwatt = <5843100>; }; opp18 { opp-hz = /bits/ 64 <4416000000>; opp-level = <18>; + opp-microwatt = <8075300>; }; opp19 { opp-hz = /bits/ 64 <4512000000>; opp-level = <19>; + opp-microwatt = <8949200>; }; }; From e1839efa6c45075fe2e62acccb88ff46ab75964d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 00:06:37 +1100 Subject: [PATCH 1520/1579] arm64: dts: apple: t6040-j616s: Give the CPU zones a measured sustainable power The CPU zones on the J616s use the power allocator, and with the energy model they now have power actors. They carry no sustainable-power, so the governor estimates a budget from the lowest states of their cooling devices. That budget is far below what this machine sustains, so a zone that reaches its passive trip would throttle much harder than needed. Measure it on the J616s (fleet10, headless): 720 s of all-core load, one busy loop pinned to each of the 14 CPUs, fans left to the SMC. TCMz reached 110 C after 19 s. Over the last 600 s it held 112.0 to 113.8 C (mean 112.8 C) while the PMP's fast die loop limited the P clusters to a mean of about 3.7 GHz. The fans ran at 3450/3730 rpm, where macOS also settles. The SMC core rails read (mean, last 600 s): P0 (PR0b) 25.3 W P1 (PP0b) 23.9 W E (PP2b) 2.1 W These are the units of the energy model, whose per-core values come from the same rails. Give each zone the sum over the clusters it throttles: 51300 mW for the die zone, 25300 mW for the first P cluster's zones, 23900 mW for the second's and 2100 mW for the E cluster's. The GPU zones also shed CPU power, but their budget needs a CPU and GPU load, so they keep the estimate for now. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 41a0c639b69ba..e0885ddee441d 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -333,6 +333,17 @@ * highest state any zone asks for and goes back to the SMC once no * zone asks for anything. States 1-8 of a fan run it from its minimum * (1350 rpm) to its maximum (5777 rpm) in even steps. + * + * sustainable-power is what the CPUs dissipate with the die held at + * its operating point: 720 s of all-core load (10 P, 4 E busy loops), + * headless, SMC fans in their own mode. Over the last 600 s TCMz held + * 112.0 to 113.8 C with the PMP limiting the P clusters to a mean of + * about 3.7 GHz and the fans at 3450/3730 rpm (where macOS settles + * too). The SMC core rails then read P0 25.3 W, P1 23.9 W and E 2.1 W + * on average, the units of the energy model. Each zone gets the sum + * over the clusters it throttles. Without it the power allocator + * estimates a budget from the cooling devices' lowest states, which + * throttles far below what this machine sustains. */ #define J616S_FAN_TRIP(label, node, temp, hyst) \ label: node { \ @@ -366,6 +377,7 @@ cpu-die-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <51300>; thermal-sensors = <&smc_hwmon 0>; /* TCMz */ trips { @@ -421,6 +433,7 @@ cpu-e0-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <2100>; thermal-sensors = <&smc_hwmon 1>; /* Tex0 */ trips { @@ -452,6 +465,7 @@ cpu-e1-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <2100>; thermal-sensors = <&smc_hwmon 2>; /* Tex1 */ trips { @@ -483,6 +497,7 @@ cpu-e2-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <2100>; thermal-sensors = <&smc_hwmon 3>; /* Tex2 */ trips { @@ -514,6 +529,7 @@ cpu-e3-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <2100>; thermal-sensors = <&smc_hwmon 4>; /* Tex3 */ trips { @@ -545,6 +561,7 @@ cpu-p00-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <25300>; thermal-sensors = <&smc_hwmon 5>; /* Tpx0 */ trips { @@ -577,6 +594,7 @@ cpu-p01-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <25300>; thermal-sensors = <&smc_hwmon 6>; /* Tpx1 */ trips { @@ -609,6 +627,7 @@ cpu-p02-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <25300>; thermal-sensors = <&smc_hwmon 7>; /* Tpx2 */ trips { @@ -641,6 +660,7 @@ cpu-p03-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <25300>; thermal-sensors = <&smc_hwmon 8>; /* Tpx3 */ trips { @@ -673,6 +693,7 @@ cpu-p04-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <25300>; thermal-sensors = <&smc_hwmon 9>; /* Tpx4 */ trips { @@ -705,6 +726,7 @@ cpu-p05-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <25300>; thermal-sensors = <&smc_hwmon 10>; /* Tpx5 */ trips { @@ -737,6 +759,7 @@ cpu-p10-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <23900>; thermal-sensors = <&smc_hwmon 11>; /* Tpx8 */ trips { @@ -769,6 +792,7 @@ cpu-p11-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <23900>; thermal-sensors = <&smc_hwmon 12>; /* Tpx9 */ trips { @@ -801,6 +825,7 @@ cpu-p12-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <23900>; thermal-sensors = <&smc_hwmon 13>; /* TpxA */ trips { @@ -833,6 +858,7 @@ cpu-p13-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <23900>; thermal-sensors = <&smc_hwmon 14>; /* TpxB */ trips { @@ -865,6 +891,7 @@ cpu-p14-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <23900>; thermal-sensors = <&smc_hwmon 15>; /* TpxC */ trips { @@ -897,6 +924,7 @@ cpu-p15-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <23900>; thermal-sensors = <&smc_hwmon 16>; /* TpxD */ trips { From 1595a36970d6a4a8f5f1e73ac08236652f42c74d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 00:40:57 +1100 Subject: [PATCH 1521/1579] media: apple: ave: recover failed firmware through runtime power cycling Fail all owners of a broken firmware, including idle sessions, and report fatal queue errors before teardown. Prevent subsequent work and racing queued buffers from restarting a failed context. Quarantine firmware-visible session allocations until a confirmed power reset. At idle runtime suspend, bound recovery attempts, try Halt, stop coprocessor control and let genpd and the DART PM links cycle power. Require the boot argument register to have reset before reusing DMA memory and cold booting with pristine DATA. System sleep keeps its strict halt requirement and scheduler rollback. Add a root-only debugfs timeout injection hook, recovery policy KUnit tests and host tests of production queue, recovery and PM callbacks. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/Kconfig | 3 +- drivers/media/platform/apple/ave/Makefile | 2 +- drivers/media/platform/apple/ave/ave-drv.c | 24 ++- drivers/media/platform/apple/ave/ave-fw.c | 111 ++++++++-- drivers/media/platform/apple/ave/ave-ipc.c | 20 +- .../platform/apple/ave/ave-recovery-test.c | 61 ++++++ .../media/platform/apple/ave/ave-recovery.h | 30 +++ drivers/media/platform/apple/ave/ave-sess.c | 36 +++- drivers/media/platform/apple/ave/ave-v4l2.c | 30 ++- drivers/media/platform/apple/ave/ave.h | 13 ++ .../apple/ave/tests/recovery-boundary.h | 145 +++++++++++++ .../platform/apple/ave/tests/recovery-test.c | 203 ++++++++++++++++++ .../platform/apple/ave/tests/run-recovery.py | 57 +++++ 13 files changed, 682 insertions(+), 53 deletions(-) create mode 100644 drivers/media/platform/apple/ave/ave-recovery-test.c create mode 100644 drivers/media/platform/apple/ave/ave-recovery.h create mode 100644 drivers/media/platform/apple/ave/tests/recovery-boundary.h create mode 100644 drivers/media/platform/apple/ave/tests/recovery-test.c create mode 100644 drivers/media/platform/apple/ave/tests/run-recovery.py diff --git a/drivers/media/platform/apple/ave/Kconfig b/drivers/media/platform/apple/ave/Kconfig index 9284ff96874cb..fe2d7684fb7c9 100644 --- a/drivers/media/platform/apple/ave/Kconfig +++ b/drivers/media/platform/apple/ave/Kconfig @@ -21,7 +21,7 @@ config VIDEO_APPLE_AVE will be called apple-ave. config VIDEO_APPLE_AVE_KUNIT_TEST - tristate "KUnit tests for Apple AVE buffer sizing" if !KUNIT_ALL_TESTS + tristate "KUnit tests for Apple AVE buffer sizing and recovery" if !KUNIT_ALL_TESTS depends on KUNIT default KUNIT_ALL_TESTS help @@ -29,3 +29,4 @@ config VIDEO_APPLE_AVE_KUNIT_TEST encoders, including picture alignment boundaries and the reference 1920x1080 AVC and HEVC sizes. The tests do not need an encoder or any firmware and can run independently of the AVE driver. + Also test the firmware recovery sliding window and jiffies wrap. diff --git a/drivers/media/platform/apple/ave/Makefile b/drivers/media/platform/apple/ave/Makefile index 0487aab585666..27ed8bce9321a 100644 --- a/drivers/media/platform/apple/ave/Makefile +++ b/drivers/media/platform/apple/ave/Makefile @@ -2,4 +2,4 @@ obj-$(CONFIG_VIDEO_APPLE_AVE) += apple-ave.o apple-ave-y := ave-drv.o ave-fw.o ave-ipc.o ave-cmd.o ave-layout.o ave-image.o ave-sess.o ave-v4l2.o -obj-$(CONFIG_VIDEO_APPLE_AVE_KUNIT_TEST) += ave-sizes-test.o +obj-$(CONFIG_VIDEO_APPLE_AVE_KUNIT_TEST) += ave-sizes-test.o ave-recovery-test.o diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c index 43c9ddb089e83..0fa74c63639be 100644 --- a/drivers/media/platform/apple/ave/ave-drv.c +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -84,6 +84,26 @@ void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, enum ave_image memcpy(dst, cmd, len); } +/* Root-only software timeout injection; does not send malformed firmware commands. */ +static int ave_inject_wedge(void *data, u64 value) +{ + struct ave_dev *ave = data; + int ret = 0; + + if (value != 1) + return -EINVAL; + mutex_lock(&ave->hw_lock); + if (ave->fw_state != AVE_FW_RUNNING) { + ret = -EBUSY; + } else { + ave->inject_wedge = true; + ave_fw_mark_broken(ave, -ETIMEDOUT, "debugfs injection"); + } + mutex_unlock(&ave->hw_lock); + return ret; +} +DEFINE_DEBUGFS_ATTRIBUTE(ave_inject_wedge_fops, NULL, ave_inject_wedge, "%llu\n"); + static void ave_debugfs_init(struct ave_dev *ave) { static const char * const names[2][AVE_IMAGE_COUNT] = { @@ -94,6 +114,8 @@ static void ave_debugfs_init(struct ave_dev *ave) if (!IS_ENABLED(CONFIG_DEBUG_FS) || !ave->layout) return; ave->debugfs = debugfs_create_dir("apple-ave", NULL); + debugfs_create_file("inject_wedge", 0200, ave->debugfs, ave, + &ave_inject_wedge_fops); for (unsigned int c = 0; c < 2; c++) for (unsigned int k = 0; k < AVE_IMAGE_COUNT; k++) { struct debugfs_blob_wrapper *blob; @@ -204,7 +226,7 @@ static int ave_runtime_suspend(struct device *dev) int ret; mutex_lock(&ave->hw_lock); - ret = ave_fw_shutdown(ave); + ret = ave_fw_runtime_suspend(ave); mutex_unlock(&ave->hw_lock); return ret; } diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c index 2e1be782df86a..7529d90da8742 100644 --- a/drivers/media/platform/apple/ave/ave-fw.c +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -188,20 +188,21 @@ int ave_fw_probe(struct ave_dev *ave) return ret; } -static void ave_fw_restore_data(struct ave_dev *ave) +static int ave_fw_restore_data(struct ave_dev *ave) { void *va; if (!ave->data_dirty) - return; + return 0; va = memremap(ave->data.phys, ave->data.size, MEMREMAP_WC); - if (WARN_ON(!va)) - return; + if (!va) + return -ENOMEM; memcpy(va, ave->data_pristine, ave->data.size); /* The restored image must be in memory before the coprocessor starts. */ wmb(); memunmap(va); ave->data_dirty = false; + return 0; } void ave_free(struct ave_dev *ave, struct ave_buf *b) @@ -293,6 +294,21 @@ static int ave_fw_prepare_restart(struct ave_dev *ave) readq(ave->coproc + ave->variant->regs->vcbar)); return -EIO; } + if (ave->fw_state == AVE_FW_RECOVERING) { + /* + * Recovery cleared CTRL itself. BOOT_ARG is set at every boot + * and is not cleared by a control stop: require it to have lost + * power too, before reusing DMA memory or restoring DATA. DARTs + * sharing VENC power may suspend later than the consumer. + */ + if (readl(ave->coproc + ave->variant->regs->ctrl) || + readl(ave->coproc + ave->variant->regs->boot_arg)) + return -EAGAIN; + ave_sess_reap(ave); + ave->fw_state = AVE_FW_HALTED; + ave->inject_wedge = false; + dev_info(ave->dev, "firmware recovery power cycle confirmed\n"); + } if (ave->fw_state != AVE_FW_HALTED) return 0; @@ -349,7 +365,9 @@ static int ave_fw_boot(struct ave_dev *ave) ave->reply_err = 0; ave->scalar_next = 0; - ave_fw_restore_data(ave); + ret = ave_fw_restore_data(ave); + if (ret) + return ret; cfg = ave->boot_cfg.cpu; log = ave->log.cpu; @@ -419,25 +437,22 @@ int ave_fw_ensure_running(struct ave_dev *ave) case AVE_FW_RUNNING: return 0; case AVE_FW_BROKEN: - dev_err(ave->dev, "firmware stopped answering earlier; reboot to use the encoder\n"); + dev_err_ratelimited(ave->dev, "firmware recovery is waiting for idle runtime suspend\n"); return -EIO; case AVE_FW_OFF: case AVE_FW_HALTED: + case AVE_FW_RECOVERING: break; } - /* - * A failed boot has no session buffers in the firmware's hands. If the - * coprocessor was set going, treat it as halted: the next attempt - * then waits for a power cycle. Otherwise nothing changed. - */ + /* A boot that reached RUN must use the same recovery as a session fault. */ prev = ave->fw_state; ret = ave_fw_boot(ave); - if (ret) { - ave->fw_state = readl(ave->coproc + ave->variant->regs->ctrl) ? - AVE_FW_HALTED : prev; - memset(ave->ring, 0, sizeof(ave->ring)); - ave->ipc_fw_base = 0; + if (ret && ret != -EAGAIN) { + if (readl(ave->coproc + ave->variant->regs->ctrl)) + ave_fw_mark_broken(ave, ret, "boot"); + else + ave->fw_state = prev; } return ret; } @@ -455,7 +470,7 @@ void ave_fw_halt(struct ave_dev *ave) return; if (ave_cmd_halt(ave)) { dev_err(ave->dev, "cannot send Halt; firmware treated as stuck\n"); - ave->fw_state = AVE_FW_BROKEN; + ave_fw_mark_broken(ave, -EIO, "Halt"); return; } msleep(AVE_HALT_SETTLE_MS); @@ -469,8 +484,8 @@ void ave_fw_halt(struct ave_dev *ave) * * Sessions keep the device runtime active, so the firmware has normally * been halted already. A firmware that stopped answering keeps its power: - * whether it still has transfers in flight cannot be checked, and only a - * reboot brings it back. + * system sleep must not silently force-stop a potentially live DMA engine. + * Runtime recovery is separate and only runs after all owners have gone. */ int ave_fw_shutdown(struct ave_dev *ave) { @@ -480,12 +495,70 @@ int ave_fw_shutdown(struct ave_dev *ave) ave_fw_halt(ave); return ave->fw_state == AVE_FW_HALTED ? 0 : -EBUSY; case AVE_FW_BROKEN: + case AVE_FW_RECOVERING: return -EBUSY; default: return 0; } } +/* Caller holds hw_lock. Wake even idle sessions so none pins a broken boot. */ +void ave_fw_mark_broken(struct ave_dev *ave, int err, const char *what) +{ + lockdep_assert_held(&ave->hw_lock); + if (ave->fw_state != AVE_FW_BROKEN) + dev_err(ave->dev, "firmware failed (%s: %d); scheduling recovery\n", what, err); + ave->fw_state = AVE_FW_BROKEN; + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i]) + mod_delayed_work(ave->wq, &ave->sessions[i]->poll_work, 0); +} + +/* + * Only runtime PM may force-stop a broken firmware, with no session owners + * and no runtime references. genpd and the IOMMU's PM device links then drop + * the VENC domains and DARTs; AVE must not reset or reprogram the DART itself. + * System sleep continues to use the strict ave_fw_shutdown() above. + */ +int ave_fw_runtime_suspend(struct ave_dev *ave) +{ + int ret; + + lockdep_assert_held(&ave->hw_lock); + if (ave->fw_state != AVE_FW_BROKEN && ave->fw_state != AVE_FW_RECOVERING) + return ave_fw_shutdown(ave); + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i]) + return -EBUSY; + if (ave->fw_state == AVE_FW_RECOVERING) + return 0; + if (!ave_recovery_allow(&ave->recovery, jiffies)) { + dev_err_ratelimited(ave->dev, "firmware recovery limit: 3 attempts per 10 minutes\n"); + return -EBUSY; + } + ave->recovery_count++; + dev_warn(ave->dev, "firmware recovery %u: stopping coprocessor and cycling VENC power\n", + ave->recovery_count); + + /* A ring may not exist after a failed boot. Never wait for Halt's reply. */ + ret = ave->inject_wedge ? -ETIMEDOUT : -EIO; + if (ave->ipc_fw_base && ave->ring[0].slots && !ave->inject_wedge) { + ave->fw_state = AVE_FW_RUNNING; + ret = ave_cmd_halt(ave); + } + if (ret) + dev_warn(ave->dev, "recovery Halt failed: %d; forcing control off\n", ret); + else + msleep(AVE_HALT_SETTLE_MS); + writel(0, ave->coproc + ave->variant->regs->ctrl); + if (readl(ave->coproc + ave->variant->regs->ctrl)) { + ave->fw_state = AVE_FW_BROKEN; + return -EIO; + } + ave->fw_state = AVE_FW_RECOVERING; + return 0; +} + /* The coprocessor lost power behind the firmware's back. Caller holds hw_lock. */ void ave_fw_lost(struct ave_dev *ave) { diff --git a/drivers/media/platform/apple/ave/ave-ipc.c b/drivers/media/platform/apple/ave/ave-ipc.c index c197671debbbe..344cb648a2617 100644 --- a/drivers/media/platform/apple/ave/ave-ipc.c +++ b/drivers/media/platform/apple/ave/ave-ipc.c @@ -334,14 +334,6 @@ int ave_ipc_scalar_buf(struct ave_dev *ave) return -EBUSY; } -static void ave_ipc_mark_broken(struct ave_dev *ave, int err, const char *what) -{ - if (ave->fw_state == AVE_FW_RUNNING) - dev_err(ave->dev, "firmware stopped answering (%s: %d); it restarts after the next power cycle\n", - what, err); - ave->fw_state = AVE_FW_BROKEN; -} - /* * Hand the command in buffer @buf to the firmware. The timeout words are * {duration, creation} in the host clock the firmware was synced to at boot. @@ -372,7 +364,7 @@ int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, u64 timeou ret = ave_ring_send(ave, &ave->ring[0], ave->ipc_fw_base + off, len); if (ret) { - ave_ipc_mark_broken(ave, ret, "command ring full"); + ave_fw_mark_broken(ave, ret, "command ring full"); return ret; } ave->unacked |= BIT(buf); @@ -482,6 +474,10 @@ int ave_ipc_pump(struct ave_dev *ave, unsigned int wait_ms, for (;;) { if (cond && cond(ave, arg)) return 0; + if (cond && ktime_after(ktime_get(), deadline)) { + ave_fw_mark_broken(ave, -ETIMEDOUT, "no answer"); + return -ETIMEDOUT; + } if (ave->fw_state != AVE_FW_RUNNING) { dev_err_ratelimited(ave->dev, "waiting on a firmware in state %d\n", ave->fw_state); @@ -489,7 +485,7 @@ int ave_ipc_pump(struct ave_dev *ave, unsigned int wait_ms, } ret = ave_ipc_poll_once(ave); if (ret < 0) { - ave_ipc_mark_broken(ave, ret, "ring error"); + ave_fw_mark_broken(ave, ret, "ring error"); return ret; } if (ret) { @@ -498,10 +494,6 @@ int ave_ipc_pump(struct ave_dev *ave, unsigned int wait_ms, } if (!cond) return 0; - if (ktime_after(ktime_get(), deadline)) { - ave_ipc_mark_broken(ave, -ETIMEDOUT, "no answer"); - return -ETIMEDOUT; - } if (idle++ < AVE_POLL_FAST_ROUNDS) usleep_range(50, 100); else diff --git a/drivers/media/platform/apple/ave/ave-recovery-test.c b/drivers/media/platform/apple/ave/ave-recovery-test.c new file mode 100644 index 0000000000000..f10c007df7e89 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-recovery-test.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "ave-recovery.h" + +static void ave_recovery_limit_test(struct kunit *test) +{ + struct ave_recovery_history h = {}; + + for (unsigned int i = 0; i < AVE_RECOVERY_ATTEMPTS; i++) + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, i)); + KUNIT_EXPECT_FALSE(test, ave_recovery_allow(&h, 3)); + KUNIT_EXPECT_FALSE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW - 1)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW)); + KUNIT_EXPECT_FALSE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW + 1)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW + 2)); + KUNIT_EXPECT_FALSE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW + 3)); +} + +static void ave_recovery_sliding_test(struct kunit *test) +{ + struct ave_recovery_history h = {}; + + KUNIT_ASSERT_TRUE(test, ave_recovery_allow(&h, 0)); + KUNIT_ASSERT_TRUE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW / 2)); + KUNIT_ASSERT_TRUE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW - 1)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW)); + KUNIT_EXPECT_FALSE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW + 1)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, 2 * AVE_RECOVERY_WINDOW)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, 2 * AVE_RECOVERY_WINDOW)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, 2 * AVE_RECOVERY_WINDOW)); + KUNIT_EXPECT_FALSE(test, ave_recovery_allow(&h, 2 * AVE_RECOVERY_WINDOW)); +} + +static void ave_recovery_wrap_test(struct kunit *test) +{ + struct ave_recovery_history h = {}; + unsigned long start = ~0UL - AVE_RECOVERY_WINDOW / 2; + + for (unsigned int i = 0; i < AVE_RECOVERY_ATTEMPTS; i++) + KUNIT_ASSERT_TRUE(test, ave_recovery_allow(&h, start)); + KUNIT_EXPECT_FALSE(test, ave_recovery_allow(&h, start + AVE_RECOVERY_WINDOW - 1)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, start + AVE_RECOVERY_WINDOW)); +} + +static struct kunit_case ave_recovery_cases[] = { + KUNIT_CASE(ave_recovery_limit_test), + KUNIT_CASE(ave_recovery_sliding_test), + KUNIT_CASE(ave_recovery_wrap_test), + {} +}; + +static struct kunit_suite ave_recovery_suite = { + .name = "apple-ave-recovery", + .test_cases = ave_recovery_cases, +}; +kunit_test_suite(ave_recovery_suite); + +MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/ave/ave-recovery.h b/drivers/media/platform/apple/ave/ave-recovery.h new file mode 100644 index 0000000000000..93de1e2471e75 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-recovery.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _APPLE_AVE_RECOVERY_H_ +#define _APPLE_AVE_RECOVERY_H_ + +#include +#include + +#define AVE_RECOVERY_ATTEMPTS 3 +#define AVE_RECOVERY_WINDOW (10 * 60 * HZ) + +struct ave_recovery_history { + unsigned long attempts[AVE_RECOVERY_ATTEMPTS]; + unsigned int next; + unsigned int count; +}; + +/* A sliding window, including across jiffies wrap. Caller serialises access. */ +static inline bool ave_recovery_allow(struct ave_recovery_history *h, unsigned long now) +{ + if (h->count == AVE_RECOVERY_ATTEMPTS && + time_before(now, h->attempts[h->next] + AVE_RECOVERY_WINDOW)) + return false; + h->attempts[h->next] = now; + h->next = (h->next + 1) % AVE_RECOVERY_ATTEMPTS; + if (h->count < AVE_RECOVERY_ATTEMPTS) + h->count++; + return true; +} + +#endif diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index 267c5abc73eed..d39033179c542 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -485,24 +485,38 @@ struct ave_session *ave_sess_create(struct ave_ctx *ctx) return sess; } -/* - * Free the buffers. A firmware that stopped answering in the middle of a - * session may still write to them, and a freed DART range is soon handed - * out again, so those stay allocated (until reboot, like the firmware - * itself). Buffers the firmware never saw (no Start sent) always go. - */ +/* Keep every firmware-visible allocation mapped until a confirmed power cycle. */ void ave_sess_destroy(struct ave_ctx *ctx) { + struct ave_dev *ave = ctx->ave; struct ave_session *sess = ctx->sess; if (!sess) return; - if (sess->handed && ctx->ave->fw_state == AVE_FW_BROKEN) - dev_warn(ctx->ave->dev, "keeping the buffers of a failed session\n"); - else - ave_sess_free_bufs(ctx->ave, sess); - kfree(sess); + mutex_lock(&ave->hw_lock); + if (sess->handed && (ave->fw_state == AVE_FW_BROKEN || + ave->fw_state == AVE_FW_RECOVERING)) { + sess->failed_next = ave->failed_sessions; + ave->failed_sessions = sess; + } else { + ave_sess_free_bufs(ave, sess); + kfree(sess); + } ctx->sess = NULL; + mutex_unlock(&ave->hw_lock); +} + +/* Caller holds hw_lock and has verified that the coprocessor lost power. */ +void ave_sess_reap(struct ave_dev *ave) +{ + struct ave_session *sess; + + lockdep_assert_held(&ave->hw_lock); + while ((sess = ave->failed_sessions)) { + ave->failed_sessions = sess->failed_next; + ave_sess_free_bufs(ave, sess); + kfree(sess); + } } int ave_sess_free_slot(struct ave_session *sess) diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index 0adbaaf8513d5..ece6f23ef676e 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -579,7 +579,10 @@ static void ave_buf_queue(struct vb2_buffer *vb) { struct ave_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); - v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, to_vb2_v4l2_buffer(vb)); + if (READ_ONCE(ctx->failed)) + v4l2_m2m_buf_done(to_vb2_v4l2_buffer(vb), VB2_BUF_STATE_ERROR); + else + v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, to_vb2_v4l2_buffer(vb)); } static void ave_return_bufs(struct ave_ctx *ctx, struct vb2_queue *q, @@ -751,7 +754,9 @@ static int ave_session_begin(struct ave_ctx *ctx) if (READ_ONCE(ave->sessions[0]) && READ_ONCE(ave->sessions[1])) return -EBUSY; /* A halted firmware restarts only from reset: let the power go first. */ - if (READ_ONCE(ave->fw_state) == AVE_FW_HALTED) + if (READ_ONCE(ave->fw_state) == AVE_FW_HALTED || + READ_ONCE(ave->fw_state) == AVE_FW_BROKEN || + READ_ONCE(ave->fw_state) == AVE_FW_RECOVERING) pm_runtime_suspend(ave->dev); sess = ave_sess_create(ctx); if (IS_ERR(sess)) { @@ -845,6 +850,8 @@ static int ave_start_streaming(struct vb2_queue *q, unsigned int count) v4l2_m2m_get_src_vq(ctx->fh.m2m_ctx); int ret = 0; + if (!vb2_is_streaming(other)) + WRITE_ONCE(ctx->failed, false); if (V4L2_TYPE_IS_OUTPUT(q->type)) ctx->src_sequence = 0; else @@ -920,6 +927,8 @@ static int ave_job_ready(void *priv) struct ave_ctx *ctx = priv; struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + if (READ_ONCE(ctx->failed)) + return 0; if (atomic_read(&ctx->ready) || READ_ONCE(ctx->flush_pending)) return 1; if (ctx->stopped || (ctx->draining && !ctx->drain_src)) @@ -1087,6 +1096,11 @@ static void ave_job_fail(struct ave_ctx *ctx, int err) struct ave_dev *ave = ctx->ave; struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + /* Publish the fatal error before any teardown that can wait on PM/IPC. */ + WRITE_ONCE(ctx->failed, true); + cancel_delayed_work(&ctx->poll_work); + vb2_queue_error(v4l2_m2m_get_src_vq(m2m)); + vb2_queue_error(v4l2_m2m_get_dst_vq(m2m)); dev_err(ave->dev, "encoding failed: %d\n", err); mutex_lock(&ave->hw_lock); ave_session_close(ctx, false); @@ -1094,8 +1108,6 @@ static void ave_job_fail(struct ave_ctx *ctx, int err) ave_session_free(ctx); ave_return_bufs(ctx, v4l2_m2m_get_src_vq(m2m), VB2_BUF_STATE_ERROR); ave_return_bufs(ctx, v4l2_m2m_get_dst_vq(m2m), VB2_BUF_STATE_ERROR); - vb2_queue_error(v4l2_m2m_get_src_vq(m2m)); - vb2_queue_error(v4l2_m2m_get_dst_vq(m2m)); ave_reset_drain(ctx); } @@ -1107,6 +1119,9 @@ static void ave_run_work(struct work_struct *work) bool eos = false, poll = false; int ret = 0; + if (READ_ONCE(ctx->failed)) + goto finish; + /* * After a drain the next frame starts a new session; after system * sleep it reopens the old one in the restarted firmware. @@ -1158,6 +1173,7 @@ static void ave_run_work(struct work_struct *work) } else if (poll) { mod_delayed_work(ave->wq, &ctx->poll_work, msecs_to_jiffies(AVE_POLL_MS)); } +finish: pm_runtime_mark_last_busy(ave->dev); v4l2_m2m_job_finish(ave->m2m_dev, m2m); } @@ -1170,7 +1186,9 @@ static void ave_poll_work(struct work_struct *work) bool again = false, fail = false; mutex_lock(&ave->hw_lock); - if (ctx->sess && ave_session_index(ave, ctx) >= 0) { + if (ctx->sess && ave->fw_state == AVE_FW_BROKEN) { + fail = true; + } else if (ctx->sess && ave_session_index(ave, ctx) >= 0) { fail = ave_ipc_pump(ave, 0, NULL, NULL) != 0; if (!fail && ave_sess_starved(ctx->sess) && !v4l2_m2m_num_src_bufs_ready(ctx->fh.m2m_ctx)) { @@ -1187,7 +1205,7 @@ static void ave_poll_work(struct work_struct *work) */ dev_err(ave->dev, "%u frames unanswered for %u ms; firmware is stuck\n", ctx->sess->busy, AVE_FRAME_WAIT_MS); - ave->fw_state = AVE_FW_BROKEN; + ave_fw_mark_broken(ave, -ETIMEDOUT, "frame timeout"); fail = true; } again = !fail && ctx->sess->busy; diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h index c880b9a62d235..899ef9ece430d 100644 --- a/drivers/media/platform/apple/ave/ave.h +++ b/drivers/media/platform/apple/ave/ave.h @@ -20,6 +20,7 @@ #include #include "ave-sizes.h" +#include "ave-recovery.h" struct debugfs_blob_wrapper; struct dentry; @@ -94,6 +95,7 @@ enum ave_fw_state { AVE_FW_RUNNING, AVE_FW_HALTED, /* told to halt; restarts through the boot path */ AVE_FW_BROKEN, + AVE_FW_RECOVERING, /* control off; awaiting a runtime power cycle */ }; /* The two large session commands, and their sizes per codec */ @@ -149,6 +151,12 @@ struct ave_dev { void *data_pristine; bool data_dirty; + /* hw_lock protects recovery history and quarantined DMA allocations. */ + struct ave_recovery_history recovery; + unsigned int recovery_count; + struct ave_session *failed_sessions; + bool inject_wedge; + /* debugfs: the last Start and first Process built, per codec */ struct dentry *debugfs; void *dump[2][AVE_IMAGE_COUNT]; @@ -224,6 +232,7 @@ struct ave_slot { #define AVE_TILE_BUFS 8 struct ave_session { + struct ave_session *failed_next; struct ave_params p; struct ave_buf hdr; struct ave_buf ref_a[AVE_MAX_REFS]; @@ -291,6 +300,7 @@ struct ave_ctx { bool draining; bool flush_pending; bool stopped; + bool failed; struct vb2_v4l2_buffer *drain_src; u32 src_sequence; u32 dst_sequence; @@ -307,6 +317,8 @@ void ave_fw_remove(struct ave_dev *ave); int ave_fw_ensure_running(struct ave_dev *ave); void ave_fw_halt(struct ave_dev *ave); int ave_fw_shutdown(struct ave_dev *ave); +int ave_fw_runtime_suspend(struct ave_dev *ave); +void ave_fw_mark_broken(struct ave_dev *ave, int err, const char *what); void ave_fw_lost(struct ave_dev *ave); int ave_fw_check_build(struct ave_dev *ave); int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, unsigned long attrs); @@ -338,6 +350,7 @@ int ave_cmd_close(struct ave_ctx *ctx); /* ave-sess.c */ struct ave_session *ave_sess_create(struct ave_ctx *ctx); void ave_sess_destroy(struct ave_ctx *ctx); +void ave_sess_reap(struct ave_dev *ave); int ave_sess_free_slot(struct ave_session *sess); void ave_sess_reply(struct ave_dev *ave, const u8 *msg, u32 len); int ave_sess_input(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *src, diff --git a/drivers/media/platform/apple/ave/tests/recovery-boundary.h b/drivers/media/platform/apple/ave/tests/recovery-boundary.h new file mode 100644 index 0000000000000..5330cf43abebf --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/recovery-boundary.h @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Fake PM, MMIO and vb2 boundary for unmodified production function bodies. */ +#include +#define container_of(p, t, m) ((t *)((char *)(p) - offsetof(t, m))) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +#define dev_err_ratelimited(...) test_log(__VA_ARGS__) +#define lockdep_assert_held(p) assert((p)->held) +#define MEMREMAP_WC 0 +#define wmb() ((void)0) +#define msecs_to_jiffies(x) (x) +#define ktime_add_ms(a, b) ((a) + (b)) +#define ktime_after(a, b) ((a) > (b)) +#define atomic_set(p, v) (*(p) = (v)) +#define atomic_read(p) (*(p)) +#define to_delayed_work(p) container_of(p, struct delayed_work, work) +struct work_struct { int unused; }; +struct delayed_work { struct work_struct work; bool scheduled; }; +struct mutex { bool held; }; +static void mutex_lock(struct mutex *m) { assert(!m->held); m->held = true; } +static void mutex_unlock(struct mutex *m) { assert(m->held); m->held = false; } +struct ave_dev; +struct device { struct ave_dev *drvdata; unsigned int refs; }; +static void *dev_get_drvdata(struct device *dev) { return dev->drvdata; } +struct vb2_queue { unsigned int type; bool error; }; +struct vb2_buffer { struct vb2_queue *vb2_queue; void *priv; }; +struct vb2_v4l2_buffer { struct vb2_buffer vb2_buf; unsigned int completions; int state; }; +struct v4l2_m2m_ctx { + struct vb2_queue src, dst; + struct vb2_v4l2_buffer *input[4], *output[4]; + unsigned int nsrc, ndst; +}; +struct v4l2_fh { struct v4l2_m2m_ctx *m2m_ctx; }; +struct ave_ctx { + struct ave_dev *ave; + struct v4l2_fh fh; + struct ave_session *sess; + struct delayed_work poll_work; + int ready; + bool failed, draining, flush_pending, stopped; + struct vb2_v4l2_buffer *drain_src; +}; +struct ave_dev { + struct device *dev; + struct mutex hw_lock; + enum ave_fw_state fw_state; + struct ave_ctx *sessions[AVE_MAX_SESSIONS]; + struct ave_session *failed_sessions; + struct ave_recovery_history recovery; + unsigned int recovery_count; + bool inject_wedge, data_dirty; + void *wq, *m2m_dev, *coproc, *mbox; + const struct ave_variant *variant; + struct ave_fw_segment text, data; + void *data_pristine; + struct ave_ring ring[2]; + u64 ipc_fw_base; +}; +static int halt_result, close_result; +static unsigned int halts, closes, freed, suspends, resumes, schedules, barriers; +static unsigned int polls, sleep_ms; +static bool map_fail, ctrl_sticky, stopped_for_sleep; +static int poll_result; +typedef unsigned long ktime_t; +static ktime_t ticks; +static ktime_t ktime_get(void) { return ++ticks; } +static void msleep(unsigned int ms) { sleep_ms += ms; } +static void usleep_range(unsigned int lo, unsigned int hi) { } +static u32 readl(const void *p) { return *(const u32 *)p; } +static u64 readq(const void *p) { return *(const u64 *)p; } +static void writel(u32 v, void *p) { if (!ctrl_sticky) *(u32 *)p = v; } +static void *memremap(phys_addr_t p, size_t size, unsigned int mode) +{ + return map_fail ? NULL : (void *)p; +} +static void memunmap(void *p) { barriers++; } +static bool mod_delayed_work(void *wq, struct delayed_work *work, unsigned long delay) +{ + work->scheduled = true; + return true; +} +static bool cancel_delayed_work(struct delayed_work *work) +{ + work->scheduled = false; + return true; +} +static bool queue_delayed_work(void *wq, struct delayed_work *work, unsigned long delay) +{ + return mod_delayed_work(wq, work, delay); +} +static int ave_cmd_halt(struct ave_dev *ave) { halts++; return halt_result; } +static int ave_cmd_close(struct ave_ctx *ctx) { closes++; return close_result; } +static int ave_cmd_complete(struct ave_ctx *ctx, unsigned int n) { return 0; } +static int ave_ipc_settle(struct ave_dev *ave, unsigned int ms) { return 0; } +static int ave_ipc_poll_once(struct ave_dev *ave) +{ + /* A deadline bug must fail, rather than leave the test spinning. */ + assert(++polls < 100); + return poll_result; +} +static bool ave_sess_starved(struct ave_session *sess) { return false; } +static bool ave_sess_stalled(struct ave_session *sess) { return false; } +static void ave_sess_free_bufs(struct ave_dev *ave, struct ave_session *sess) { freed++; } +static void pm_runtime_mark_last_busy(struct device *dev) { } +static void pm_runtime_put_autosuspend(struct device *dev) { assert(dev->refs); dev->refs--; } +static void v4l2_m2m_suspend(void *dev) { suspends++; } +static void v4l2_m2m_resume(void *dev) { resumes++; } +static void ave_v4l2_suspend(struct ave_dev *ave) { stopped_for_sleep = true; } +static void v4l2_m2m_try_schedule(struct v4l2_m2m_ctx *ctx) { schedules++; } +static struct vb2_queue *v4l2_m2m_get_src_vq(struct v4l2_m2m_ctx *ctx) { return &ctx->src; } +static struct vb2_queue *v4l2_m2m_get_dst_vq(struct v4l2_m2m_ctx *ctx) { return &ctx->dst; } +#define V4L2_TYPE_IS_OUTPUT(t) ((t) == 1) +enum vb2_buffer_state { VB2_BUF_STATE_ERROR = 1 }; +static struct vb2_v4l2_buffer *v4l2_m2m_src_buf_remove(struct v4l2_m2m_ctx *ctx) +{ + return ctx->nsrc ? ctx->input[--ctx->nsrc] : NULL; +} +static struct vb2_v4l2_buffer *v4l2_m2m_dst_buf_remove(struct v4l2_m2m_ctx *ctx) +{ + return ctx->ndst ? ctx->output[--ctx->ndst] : NULL; +} +static unsigned int v4l2_m2m_num_src_bufs_ready(struct v4l2_m2m_ctx *ctx) { return ctx->nsrc; } +static void v4l2_m2m_buf_done(struct vb2_v4l2_buffer *buf, enum vb2_buffer_state state) +{ + assert(++buf->completions == 1); + buf->state = state; +} +static void vb2_queue_error(struct vb2_queue *q) { q->error = true; } +static void *vb2_get_drv_priv(struct vb2_queue *q) +{ + struct v4l2_m2m_ctx *m2m = container_of(q, struct v4l2_m2m_ctx, src); + return m2m->input[3]; +} +#define to_vb2_v4l2_buffer(p) container_of(p, struct vb2_v4l2_buffer, vb2_buf) +static void v4l2_m2m_buf_queue(struct v4l2_m2m_ctx *ctx, struct vb2_v4l2_buffer *buf) +{ + assert(!ctx->src.error && ctx->nsrc < 3); + ctx->input[ctx->nsrc++] = buf; +} +/* Forward declarations between the extracted, otherwise unchanged callbacks. */ +void ave_fw_mark_broken(struct ave_dev *ave, int err, const char *what); +void ave_sess_reap(struct ave_dev *ave); +static void ave_reset_drain(struct ave_ctx *ctx); +int ave_ipc_pump(struct ave_dev *ave, unsigned int wait_ms, + bool (*cond)(struct ave_dev *, void *), void *arg); diff --git a/drivers/media/platform/apple/ave/tests/recovery-test.c b/drivers/media/platform/apple/ave/tests/recovery-test.c new file mode 100644 index 0000000000000..f03e6423faf24 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/recovery-test.c @@ -0,0 +1,203 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +static const struct ave_coproc_regs regs = { .ctrl = 0, .boot_arg = 4, .vcbar = 8 }; +static const struct ave_variant variant = { .regs = ®s }; +static u64 coproc[4]; +static u32 mailbox[16]; +static struct ave_dev ave; +static struct device dev; +static struct ave_ctx ctx[2]; +static struct v4l2_m2m_ctx m2m[2]; + +static void init(void) +{ + memset(&ave, 0, sizeof(ave)); + memset(&dev, 0, sizeof(dev)); + memset(ctx, 0, sizeof(ctx)); + memset(m2m, 0, sizeof(m2m)); + memset(coproc, 0, sizeof(coproc)); + memset(mailbox, 0, sizeof(mailbox)); + halts = closes = freed = suspends = resumes = schedules = polls = sleep_ms = 0; + halt_result = close_result = poll_result = 0; + map_fail = ctrl_sticky = stopped_for_sleep = false; + ave.dev = &dev; + dev.drvdata = &ave; + ave.variant = &variant; + ave.coproc = coproc; + ave.mbox = mailbox; + ave.text.iova = 0x100000000; + *(u64 *)((char *)coproc + regs.vcbar) = ave.text.iova | AVE_COPROC_VCBAR_VALID; + for (unsigned int i = 0; i < 2; i++) { + ctx[i].ave = &ave; + ctx[i].fh.m2m_ctx = &m2m[i]; + m2m[i].src.type = 1; + m2m[i].input[3] = (void *)&ctx[i]; + } +} + +static void owners(void) +{ + for (unsigned int i = 0; i < 2; i++) { + ctx[i].sess = calloc(1, sizeof(*ctx[i].sess)); + assert(ctx[i].sess); + ctx[i].sess->handed = ctx[i].sess->fw_open = true; + ctx[i].sess->p.slots = 2; + ave.sessions[i] = &ctx[i]; + dev.refs++; + } + ave.fw_state = AVE_FW_RUNNING; + *(u32 *)((char *)coproc + regs.ctrl) = AVE_COPROC_CTRL_RUN; + *(u32 *)((char *)coproc + regs.boot_arg) = AVE_COPROC_BOOT_ARG_VALUE; + ave.ring[0].slots = 4; + ave.ipc_fw_base = 0x1234; +} + +static void buffer_failure(void) +{ + struct vb2_v4l2_buffer input[2][3] = {}, output[2][2] = {}; + + init(); + owners(); + for (unsigned int i = 0; i < 2; i++) { + ctx[i].sess->slot[0].src = &input[i][0]; + /* One idle owner and one with a firmware-owned frame. */ + ctx[i].sess->busy = i; + ctx[i].ready = i; + m2m[i].input[0] = &input[i][1]; + m2m[i].input[1] = &input[i][2]; + m2m[i].nsrc = 2; + m2m[i].output[0] = &output[i][0]; + m2m[i].output[1] = &output[i][1]; + m2m[i].ndst = 2; + } + assert(!ave_inject_wedge(&ave, 1)); + assert(ave.fw_state == AVE_FW_BROKEN); + assert(ctx[0].poll_work.scheduled && ctx[1].poll_work.scheduled); + /* No runtime recovery is permitted while either owner remains. */ + assert(ave_runtime_suspend(&dev) == -EBUSY); + assert(!ave.recovery_count && !halts); + for (unsigned int i = 0; i < 2; i++) { + ave_poll_work(&ctx[i].poll_work.work); + assert(ctx[i].failed && !ctx[i].sess && !ave.sessions[i]); + assert(m2m[i].src.error && m2m[i].dst.error); + assert(!m2m[i].nsrc && !m2m[i].ndst && !ctx[i].ready); + assert(!ctx[i].poll_work.scheduled && !ave_job_ready(&ctx[i])); + for (unsigned int n = 0; n < 3; n++) + assert(input[i][n].completions == 1 && input[i][n].state == VB2_BUF_STATE_ERROR); + for (unsigned int n = 0; n < 2; n++) + assert(output[i][n].completions == 1 && output[i][n].state == VB2_BUF_STATE_ERROR); + } + assert(!dev.refs && !freed && ave.failed_sessions && !closes); + /* A QBUF racing the failure is returned instead of silently requeued. */ + { + struct vb2_v4l2_buffer extra = {}; + + extra.vb2_buf.vb2_queue = &m2m[0].src; + ave_buf_queue(&extra.vb2_buf); + assert(extra.completions == 1 && extra.state == VB2_BUF_STATE_ERROR); + } + assert(!ave_runtime_suspend(&dev)); + assert(ave.fw_state == AVE_FW_RECOVERING && ave.recovery_count == 1); + assert(!readl(ave.coproc + regs.ctrl)); + mutex_lock(&ave.hw_lock); + /* CTRL=0 alone is not proof of a power cycle. */ + assert(ave_fw_prepare_restart(&ave) == -EAGAIN && !freed); + *(u32 *)(ave.coproc + regs.boot_arg) = 0; + assert(!ave_fw_prepare_restart(&ave)); + mutex_unlock(&ave.hw_lock); + assert(freed == 2 && !ave.failed_sessions && !ave.inject_wedge); + assert(ave.fw_state == AVE_FW_HALTED); +} + +static void sleep_guarantees(void) +{ + init(); + ave.fw_state = AVE_FW_RUNNING; + halt_result = -ETIMEDOUT; + assert(ave_suspend(&dev) == -EBUSY); + assert(suspends == 1 && resumes == 1 && stopped_for_sleep); + assert(ave.fw_state == AVE_FW_BROKEN && !ave.recovery_count); + assert(ave_suspend(&dev) == -EBUSY && halts == 1); + assert(suspends == 2 && resumes == 2); + ave.fw_state = AVE_FW_RECOVERING; + assert(ave_suspend(&dev) == -EBUSY); + assert(resumes == 3); + ave.fw_state = AVE_FW_RUNNING; + halt_result = 0; + assert(!ave_suspend(&dev)); + assert(ave.fw_state == AVE_FW_HALTED && resumes == 3); + assert(!ave_resume(&dev) && resumes == 4); +} + +static void recovery_failures(void) +{ + init(); + jiffies = 100; + for (unsigned int i = 0; i < AVE_RECOVERY_ATTEMPTS; i++) { + ave.fw_state = AVE_FW_BROKEN; + ave.ipc_fw_base = 1; + ave.ring[0].slots = 4; + halt_result = -ETIMEDOUT; + assert(!ave_runtime_suspend(&dev)); + assert(ave.fw_state == AVE_FW_RECOVERING); + } + assert(halts == 3 && ave.recovery_count == 3); + ave.fw_state = AVE_FW_BROKEN; + assert(ave_runtime_suspend(&dev) == -EBUSY && halts == 3); + jiffies += AVE_RECOVERY_WINDOW; + ctrl_sticky = true; + *(u32 *)(ave.coproc + regs.ctrl) = AVE_COPROC_CTRL_RUN; + assert(ave_runtime_suspend(&dev) == -EIO); + assert(ave.fw_state == AVE_FW_BROKEN && ave.recovery_count == 4); + ctrl_sticky = false; + assert(!ave_runtime_suspend(&dev)); + assert(!ave_runtime_suspend(&dev)); /* Does not count the PM retry again. */ + assert(ave.recovery_count == 5); +} + +static bool never_answered(struct ave_dev *dev, void *arg) { return false; } +static void busy_ring_deadline(void) +{ + init(); + owners(); + poll_result = 1; /* A continuous stream of unrelated acks/messages. */ + mutex_lock(&ave.hw_lock); + assert(ave_ipc_pump(&ave, 5, never_answered, NULL) == -ETIMEDOUT); + mutex_unlock(&ave.hw_lock); + assert(polls < 10 && ave.fw_state == AVE_FW_BROKEN); + assert(ctx[0].poll_work.scheduled && ctx[1].poll_work.scheduled); + ave_poll_work(&ctx[0].poll_work.work); + ave_poll_work(&ctx[1].poll_work.work); + mutex_lock(&ave.hw_lock); + ave_sess_reap(&ave); /* Fake reset for test cleanup only. */ + mutex_unlock(&ave.hw_lock); +} + +static void restore_data(void) +{ + u8 pristine[64], live[64]; + + init(); + memset(pristine, 0xa5, sizeof(pristine)); + memset(live, 0x3c, sizeof(live)); + ave.data_pristine = pristine; + ave.data.phys = (phys_addr_t)live; + ave.data.size = sizeof(live); + ave.data_dirty = true; + map_fail = true; + assert(ave_fw_restore_data(&ave) == -ENOMEM && ave.data_dirty); + map_fail = false; + assert(!ave_fw_restore_data(&ave) && !ave.data_dirty); + assert(!memcmp(live, pristine, sizeof(live))); + assert(!ave_fw_restore_data(&ave)); +} + +int main(void) +{ + buffer_failure(); + sleep_guarantees(); + recovery_failures(); + busy_ring_deadline(); + restore_data(); + puts("ok: recovery ownership, queues, quarantine, reset proof, rate limit, PM rollback, DATA, IPC deadline"); + return 0; +} diff --git a/drivers/media/platform/apple/ave/tests/run-recovery.py b/drivers/media/platform/apple/ave/tests/run-recovery.py new file mode 100644 index 0000000000000..37dba68a1c097 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/run-recovery.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise production AVE recovery, PM and queue callbacks at a fake boundary.""" +import os +from pathlib import Path +import re +import subprocess +import tempfile + +from run import declaration, function, source + + +def main(): + here = Path(__file__).resolve().parent + src = here.parent + header = (src / 'ave.h').read_text() + code = (here / 'shim.h').read_text() + code += '\ntypedef uintptr_t phys_addr_t;\n' + code += '#define HZ 1000\n#define time_before(a, b) ((long)((a) - (b)) < 0)\n' + code += source((src / 'ave-recovery.h').read_text()) + for name in ('AVE_MAX_SLOTS', 'AVE_MAX_REFS', 'AVE_TILE_BUFS', 'AVE_MAX_SESSIONS'): + code += re.search(r'^#define ' + name + r'\s+[^\n]+', header, re.M)[0] + '\n' + code += source((src / 'ave-regs.h').read_text()) + code += 'struct vb2_v4l2_buffer;\n' + for name in ('ave_codec', 'ave_buf', 'ave_fw_segment', 'ave_ring', 'ave_fw_state', + 'ave_coproc_regs', 'ave_variant', 'ave_params', 'ave_slot', 'ave_session'): + code += declaration(header, name) + code += (here / 'recovery-boundary.h').read_text() + groups = { + 'ave-fw.c': ('ave_fw_mark_broken', 'ave_fw_halt', 'ave_fw_shutdown', + 'ave_fw_runtime_suspend', 'ave_fw_prepare_restart', 'ave_fw_restore_data'), + 'ave-sess.c': ('ave_sess_destroy', 'ave_sess_reap'), + 'ave-v4l2.c': ('ave_return_bufs', 'ave_buf_queue', 'ave_session_index', + 'ave_session_count', 'ave_session_close', 'ave_session_free', + 'ave_reset_drain', 'ave_job_ready', 'ave_job_fail', 'ave_poll_work'), + 'ave-ipc.c': ('ave_ipc_pump',), + 'ave-drv.c': ('ave_runtime_suspend', 'ave_suspend', 'ave_resume', 'ave_inject_wedge'), + } + code += '#define AVE_POLL_FAST_ROUNDS 20\n#define AVE_POLL_MS 2\n' + for filename, names in groups.items(): + text = (src / filename).read_text() + for name in names: + code += function(text, name) + code += (here / 'recovery-test.c').read_text() + with tempfile.TemporaryDirectory(prefix='ave-recovery-') as tmp: + path = Path(tmp) + (path / 'test.c').write_text(code) + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-O1', '-g', + '-Wall', '-Wextra', '-Werror', '-Wno-unused-parameter', + '-Wno-unused-function', '-fsanitize=address,undefined', + '-fno-omit-frame-pointer', str(path / 'test.c'), + '-o', str(path / 'test')], check=True) + subprocess.run([str(path / 'test')], check=True) + + +if __name__ == '__main__': + main() From fef48565e82451f1ead5cbd68d9c27f36ab6cfda Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 00:48:17 +1100 Subject: [PATCH 1522/1579] tools/asahi: register the Apple PWM runtime PM host test run-host-tests.py fails the run when a test under tools/asahi is not registered, and apple-pwm-pm-tests.py came in without an entry. Register its passing run and its negative control, which exits 0 after detecting the missing output reference. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/run-host-tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 24f7a9e9134fe..31d554aa7a041 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -56,6 +56,7 @@ def mutations(flag, names, base, expect=KILL): (out() + ['--old-ref', '4db3a5a22c43'], PASS)], 'binding-lifetime-tests.py': [(out(), PASS)] + mutations('--mutation', ('no-close', 'no-join'), out()), + 'apple-pwm-pm-tests.py': [([], PASS), (['--negative-control'], PASS)], 'apple-dart-fw-tests.py': [(out(), PASS)], 'apple-dart-range-tests.py': [([], PASS)], 'atc-usb3-tests.py': [(out(), PASS), (out() + ['--mutations'], PASS)], From 7b9dd3d011da5beda7a2adcf2625df247885a2e0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 00:56:04 +1100 Subject: [PATCH 1523/1579] tools/asahi: teach the T6030 PMP graph test about the ANE window The PMP report probe now maps an optional ANE DVFS window and apple_pmp_report_set_state() refuses a reserved ANE domain. The T6030 graph test compiles those bodies verbatim, so it stopped building. Mirror struct apple_pmp_ane in the fixture and add the platform resource helpers the probe calls. T6030 reports never name an ane-dvfs window, so of_property_match_string() reports no match and the resource helpers assert if they are ever reached. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/t6030-pmp-graph-tests.c | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tools/asahi/t6030-pmp-graph-tests.c b/tools/asahi/t6030-pmp-graph-tests.c index e9ed89ca13e92..3d2493e57fb63 100644 --- a/tools/asahi/t6030-pmp-graph-tests.c +++ b/tools/asahi/t6030-pmp-graph-tests.c @@ -27,6 +27,10 @@ typedef uint8_t u8; #define MAX_PHANDLE_ARGS 4 #define EPROBE_DEFER 517 #define SZ_4K 4096 +#define SZ_16K 16384 +#define SZ_128K 131072 +#define IORESOURCE_MEM 0x200 +#define PMP_ANE_ID 10 #define __iomem #define __force #define container_of(p, t, m) ((t *)((char *)(p) - offsetof(t, m))) @@ -133,6 +137,15 @@ struct work_struct { struct delayed_work { struct work_struct work; }; +/* The T600x ANE window; T6030 reports never name one. */ +struct apple_pmp_ane { + void __iomem *dvfs; + void __iomem *report; + unsigned int level; + unsigned int timeouts; + bool owned; + bool reserved; +}; struct apple_pmp_report { struct device *dev; struct apple_pmp_dram dram; @@ -147,6 +160,7 @@ struct apple_pmp_report { bool defer; unsigned int polls; struct delayed_work ready_work; + struct apple_pmp_ane ane; }; struct completion { bool done; @@ -596,6 +610,40 @@ static int devm_add_action_or_reset(struct device *d, void (*f)(void *), actions[action_count++] = (struct action){ f, p }; return 0; } +struct resource { + u64 start, end; +}; +static int of_property_match_string(const struct device_node *n, + const char *p, const char *s) +{ + (void)n; (void)p; (void)s; + return -EINVAL; +} +static struct resource *platform_get_resource(struct platform_device *p, + unsigned int t, unsigned int i) +{ + (void)p; (void)t; (void)i; + assert(!"T6030 reports have no ANE window"); + return NULL; +} +static struct resource *platform_get_resource_byname(struct platform_device *p, + unsigned int t, + const char *n) +{ + (void)p; (void)t; (void)n; + assert(!"T6030 reports have no ANE window"); + return NULL; +} +static u64 resource_size(const struct resource *r) +{ + return r->end - r->start + 1; +} +static void *devm_ioremap_resource(struct device *d, struct resource *r) +{ + (void)d; (void)r; + assert(!"T6030 reports have no ANE window"); + return NULL; +} static void *devm_platform_ioremap_resource(struct platform_device *p, int i) { assert(p == &report_dev && !i); From da4d7d9da17c2e0a3327e859a6050296c435c66f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 00:56:05 +1100 Subject: [PATCH 1524/1579] tools/asahi: give the AOP ALS channel test the scale bindings aop_read_raw() now answers IIO_CHAN_INFO_SCALE for processors with a fixed scale, which the lid-angle sensor added on aurora-wip. The channel test compiles that function against its own small bindings module, which lacked the scale constants, so it no longer built. Add them. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/aop-als-channel-tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/asahi/aop-als-channel-tests.py b/tools/asahi/aop-als-channel-tests.py index d88d8405ab5f1..d3f5f3ed48d87 100644 --- a/tools/asahi/aop-als-channel-tests.py +++ b/tools/asahi/aop-als-channel-tests.py @@ -46,7 +46,9 @@ def source(): mod bindings { pub const BINDINGS_IIO_CHAN_INFO_PROCESSED:u32=2; pub const BINDINGS_IIO_CHAN_INFO_RAW:u32=1; + pub const BINDINGS_IIO_CHAN_INFO_SCALE:u32=3; pub const IIO_VAL_INT_PLUS_MICRO:u32=2; + pub const IIO_VAL_INT_PLUS_NANO:u32=3; pub const IIO_VAL_INT:u32=1; pub struct iio_dev { pub priv_:*mut std::ffi::c_void } pub struct iio_chan_spec { pub type_:u32, pub address:u64 } From 496225fc181a003b7a0a18767815f47b5952473a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 01:06:27 +1100 Subject: [PATCH 1525/1579] media: apple: ave: serialize failed queues and verify power loss before restart Serialize QBUF with fatal-error publication and ready-queue draining. Clear the failed-context flag after both queues stop, before vb2 hands buffers back on STREAMON. Arm BOOT_ARG as a reset witness only after control has stopped. Require it to reset across runtime suspend before reclaiming quarantined DMA memory. Document fault injection and hardware qualification, and extend production-callback host coverage. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/ave-fw.c | 10 +- .../platform/apple/ave/ave-recovery-test.c | 1 + drivers/media/platform/apple/ave/ave-v4l2.c | 23 +- drivers/media/platform/apple/ave/ave.h | 3 + drivers/media/platform/apple/ave/tests/README | 58 ++++++ .../apple/ave/tests/recovery-boundary.h | 197 +++++++++++++++--- .../platform/apple/ave/tests/recovery-test.c | 30 ++- .../platform/apple/ave/tests/run-recovery.py | 1 + 8 files changed, 282 insertions(+), 41 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c index 7529d90da8742..4f1c04afee4c0 100644 --- a/drivers/media/platform/apple/ave/ave-fw.c +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -296,8 +296,8 @@ static int ave_fw_prepare_restart(struct ave_dev *ave) } if (ave->fw_state == AVE_FW_RECOVERING) { /* - * Recovery cleared CTRL itself. BOOT_ARG is set at every boot - * and is not cleared by a control stop: require it to have lost + * Recovery cleared CTRL itself and re-armed BOOT_ARG as a + * reset witness with control off: require it to have lost * power too, before reusing DMA memory or restoring DATA. DARTs * sharing VENC power may suspend later than the consumer. */ @@ -555,6 +555,12 @@ int ave_fw_runtime_suspend(struct ave_dev *ave) ave->fw_state = AVE_FW_BROKEN; return -EIO; } + /* CTRL=0 must not masquerade as power loss on the next attach. */ + writel(AVE_COPROC_BOOT_ARG_VALUE, ave->coproc + ave->variant->regs->boot_arg); + if (readl(ave->coproc + ave->variant->regs->boot_arg) != AVE_COPROC_BOOT_ARG_VALUE) { + ave->fw_state = AVE_FW_BROKEN; + return -EIO; + } ave->fw_state = AVE_FW_RECOVERING; return 0; } diff --git a/drivers/media/platform/apple/ave/ave-recovery-test.c b/drivers/media/platform/apple/ave/ave-recovery-test.c index f10c007df7e89..0dbcd1b0be219 100644 --- a/drivers/media/platform/apple/ave/ave-recovery-test.c +++ b/drivers/media/platform/apple/ave/ave-recovery-test.c @@ -56,6 +56,7 @@ static struct kunit_suite ave_recovery_suite = { .name = "apple-ave-recovery", .test_cases = ave_recovery_cases, }; + kunit_test_suite(ave_recovery_suite); MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index ece6f23ef676e..8191a537de368 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -578,11 +578,14 @@ static int ave_buf_prepare(struct vb2_buffer *vb) static void ave_buf_queue(struct vb2_buffer *vb) { struct ave_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + unsigned long flags; - if (READ_ONCE(ctx->failed)) + spin_lock_irqsave(&ctx->queue_lock, flags); + if (ctx->failed) v4l2_m2m_buf_done(to_vb2_v4l2_buffer(vb), VB2_BUF_STATE_ERROR); else v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, to_vb2_v4l2_buffer(vb)); + spin_unlock_irqrestore(&ctx->queue_lock, flags); } static void ave_return_bufs(struct ave_ctx *ctx, struct vb2_queue *q, @@ -850,8 +853,6 @@ static int ave_start_streaming(struct vb2_queue *q, unsigned int count) v4l2_m2m_get_src_vq(ctx->fh.m2m_ctx); int ret = 0; - if (!vb2_is_streaming(other)) - WRITE_ONCE(ctx->failed, false); if (V4L2_TYPE_IS_OUTPUT(q->type)) ctx->src_sequence = 0; else @@ -869,10 +870,16 @@ static int ave_start_streaming(struct vb2_queue *q, unsigned int count) static void ave_stop_streaming(struct vb2_queue *q) { struct ave_ctx *ctx = vb2_get_drv_priv(q); + struct vb2_queue *other = V4L2_TYPE_IS_OUTPUT(q->type) ? + v4l2_m2m_get_dst_vq(ctx->fh.m2m_ctx) : + v4l2_m2m_get_src_vq(ctx->fh.m2m_ctx); ave_session_end(ctx); ave_reset_drain(ctx); ave_return_bufs(ctx, q, VB2_BUF_STATE_ERROR); + /* vb2 hands queued buffers to buf_queue before the next STREAMON callback. */ + if (!vb2_is_streaming(other)) + WRITE_ONCE(ctx->failed, false); } static const struct vb2_ops ave_qops = { @@ -1095,19 +1102,22 @@ static void ave_job_fail(struct ave_ctx *ctx, int err) { struct ave_dev *ave = ctx->ave; struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + unsigned long flags; /* Publish the fatal error before any teardown that can wait on PM/IPC. */ + spin_lock_irqsave(&ctx->queue_lock, flags); WRITE_ONCE(ctx->failed, true); - cancel_delayed_work(&ctx->poll_work); vb2_queue_error(v4l2_m2m_get_src_vq(m2m)); vb2_queue_error(v4l2_m2m_get_dst_vq(m2m)); + ave_return_bufs(ctx, v4l2_m2m_get_src_vq(m2m), VB2_BUF_STATE_ERROR); + ave_return_bufs(ctx, v4l2_m2m_get_dst_vq(m2m), VB2_BUF_STATE_ERROR); + spin_unlock_irqrestore(&ctx->queue_lock, flags); + cancel_delayed_work(&ctx->poll_work); dev_err(ave->dev, "encoding failed: %d\n", err); mutex_lock(&ave->hw_lock); ave_session_close(ctx, false); mutex_unlock(&ave->hw_lock); ave_session_free(ctx); - ave_return_bufs(ctx, v4l2_m2m_get_src_vq(m2m), VB2_BUF_STATE_ERROR); - ave_return_bufs(ctx, v4l2_m2m_get_dst_vq(m2m), VB2_BUF_STATE_ERROR); ave_reset_drain(ctx); } @@ -1249,6 +1259,7 @@ static int ave_open(struct file *file) if (!ctx) return -ENOMEM; ctx->ave = ave; + spin_lock_init(&ctx->queue_lock); INIT_WORK(&ctx->run_work, ave_run_work); INIT_DELAYED_WORK(&ctx->poll_work, ave_poll_work); atomic_set(&ctx->ready, 0); diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h index 899ef9ece430d..dbe784da8ad2b 100644 --- a/drivers/media/platform/apple/ave/ave.h +++ b/drivers/media/platform/apple/ave/ave.h @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -300,6 +301,8 @@ struct ave_ctx { bool draining; bool flush_pending; bool stopped; + /* Serialises QBUF with fatal-error publication and ready-queue draining. */ + spinlock_t queue_lock; bool failed; struct vb2_v4l2_buffer *drain_src; u32 src_sequence; diff --git a/drivers/media/platform/apple/ave/tests/README b/drivers/media/platform/apple/ave/tests/README index d315f9d4d4bc8..3f28d711c2ab3 100644 --- a/drivers/media/platform/apple/ave/tests/README +++ b/drivers/media/platform/apple/ave/tests/README @@ -27,3 +27,61 @@ The host shim intentionally uses stdint types, Linux's private ENOTSUPP value and side-effect-free calls to kernel-like arithmetic macros. Those produce checkpatch PREFER_KERNEL_TYPES, ENOTSUPP and MACRO_ARG_REUSE notices. MAINTAINERS already covers the entire AVE directory (FILE_PATH_CHANGES). + +Recovery and PM checks +---------------------- + +Run python3 run-recovery.py for ASan/UBSan checks of the production recovery, +firmware shutdown, suspend/resume, failed-session cleanup, queue and IPC pump +callbacks. It covers two owners (one idle), both ready queues plus an owned +OUTPUT buffer, racing QBUF, PM reference balance, DMA quarantine/reclamation, +retained power, failed control stop, the recovery limit, pristine DATA restore +failure, suspend refusal/rollback and a busy IPC stream reaching its deadline. +MMIO, DMA allocation and PM are simulated: this cannot establish hardware reset +values or that a VENC domain actually powered down. + +The apple-ave-recovery KUnit suite tests the actual sliding-window helper at +expiry boundaries, with staggered attempts and across jiffies wrap. Run with +../.kunitconfig using tools/testing/kunit/kunit.py. The existing +tools/asahi/ave-pm-tests.py also checks the #651 sleep rollback contract. + +Hardware procedure (requires a qualified firmware/SoC and an approved lane) +------------------------------------------------------------------------- + +1. Record the kernel revision, firmware CRCs and boot ID. Mount debugfs and + enable the normal PM/genpd tracing available in the test kernel. Confirm a + fresh AVC and HEVC encode can each complete and decode correctly. +2. Start a long encode with several OUTPUT and CAPTURE buffers queued. While + it runs, as root write 1 to /sys/kernel/debug/apple-ave/inject_wedge. + This software fault marks the current firmware broken and suppresses the + recovery Halt attempt, modelling a command timeout without malformed DMA + commands. It does not reproduce a real DART overrun or a firmware deadlock. +3. Check that the client promptly sees POLLERR/EIO and every buffer still owned + by the driver completes with ERROR, including OUTPUT slots already handed + to firmware. A V4L2 queue exerciser should count completions; ffmpeg alone + only checks the client-visible exit. QBUF after failure must fail or return + the buffer with ERROR. STREAMOFF both queues before restarting that context. +4. Immediately start a new encode without rebooting. Allow the existing bounded + PM retry loop for DARTs sharing VENC power. Require the recovery stop log, + a real domain power-off/on in PM tracing, the power-cycle-confirmed log, + fresh firmware Config and valid encoded output. The driver re-arms BOOT_ARG + with control off and requires it to read zero after resume; qualify that + reset witness on each target. A retained nonzero witness must cause EAGAIN, + with no DATA restore, allocation reuse or new firmware start. +5. Repeat with two concurrent clients, including an idle open session with no + outstanding frames. Both must fail, all PM references must drop, and a new + client must encode. Check there are no stale replies, DART faults or DMA + allocation growth over repeated recoveries. +6. Inject three faults inside ten minutes. A fourth must report the recovery + limit and return queue errors without starting another firmware boot. After + the oldest attempt expires, a new session must recover successfully. +7. In a separate approved sleep test, check normal encode/sleep/resume and a + halt-failure suspend refusal. A failed suspend must restart the m2m scheduler + (#651); system sleep must never treat the forced runtime stop as a successful + firmware Halt. Preserve logs from failed sleep attempts as well as retries. + +Run this procedure on each target requiring qualification, including both +J616s and Neo. Repeat the recovery with a genuine firmware timeout/DART fault +when a safe reproducer is available. Keep captures and machine records outside +the public source tree. Rollback is a normal revert of the recovery commits; +use the lane's existing boot fallback when testing candidate kernels. diff --git a/drivers/media/platform/apple/ave/tests/recovery-boundary.h b/drivers/media/platform/apple/ave/tests/recovery-boundary.h index 5330cf43abebf..ae3e05d5112b0 100644 --- a/drivers/media/platform/apple/ave/tests/recovery-boundary.h +++ b/drivers/media/platform/apple/ave/tests/recovery-boundary.h @@ -7,6 +7,7 @@ #define dev_err_ratelimited(...) test_log(__VA_ARGS__) #define lockdep_assert_held(p) assert((p)->held) #define MEMREMAP_WC 0 +/* Host memory is coherent; the production write barrier is simulated. */ #define wmb() ((void)0) #define msecs_to_jiffies(x) (x) #define ktime_add_ms(a, b) ((a) + (b)) @@ -15,31 +16,58 @@ #define atomic_read(p) (*(p)) #define to_delayed_work(p) container_of(p, struct delayed_work, work) struct work_struct { int unused; }; + struct delayed_work { struct work_struct work; bool scheduled; }; + struct mutex { bool held; }; -static void mutex_lock(struct mutex *m) { assert(!m->held); m->held = true; } -static void mutex_unlock(struct mutex *m) { assert(m->held); m->held = false; } + +static void mutex_lock(struct mutex *m) +{ + assert(!m->held); + m->held = true; +} + +static void mutex_unlock(struct mutex *m) +{ + assert(m->held); + m->held = false; +} + +#define spin_lock_irqsave(p, flags) do { (flags) = 0; mutex_lock(p); } while (0) +#define spin_unlock_irqrestore(p, flags) do { (void)(flags); mutex_unlock(p); } while (0) struct ave_dev; struct device { struct ave_dev *drvdata; unsigned int refs; }; -static void *dev_get_drvdata(struct device *dev) { return dev->drvdata; } -struct vb2_queue { unsigned int type; bool error; }; + +static void *dev_get_drvdata(struct device *dev) +{ + return dev->drvdata; +} + +struct vb2_queue { unsigned int type; bool error, streaming; void *priv; }; + struct vb2_buffer { struct vb2_queue *vb2_queue; void *priv; }; + struct vb2_v4l2_buffer { struct vb2_buffer vb2_buf; unsigned int completions; int state; }; + struct v4l2_m2m_ctx { struct vb2_queue src, dst; struct vb2_v4l2_buffer *input[4], *output[4]; unsigned int nsrc, ndst; }; + struct v4l2_fh { struct v4l2_m2m_ctx *m2m_ctx; }; + struct ave_ctx { struct ave_dev *ave; struct v4l2_fh fh; struct ave_session *sess; struct delayed_work poll_work; + struct mutex queue_lock; int ready; bool failed, draining, flush_pending, stopped; struct vb2_v4l2_buffer *drain_src; }; + struct ave_dev { struct device *dev; struct mutex hw_lock; @@ -56,6 +84,7 @@ struct ave_dev { struct ave_ring ring[2]; u64 ipc_fw_base; }; + static int halt_result, close_result; static unsigned int halts, closes, freed, suspends, resumes, schedules, barriers; static unsigned int polls, sleep_ms; @@ -63,73 +92,185 @@ static bool map_fail, ctrl_sticky, stopped_for_sleep; static int poll_result; typedef unsigned long ktime_t; static ktime_t ticks; -static ktime_t ktime_get(void) { return ++ticks; } -static void msleep(unsigned int ms) { sleep_ms += ms; } +static ktime_t ktime_get(void) +{ + return ++ticks; +} + +static void msleep(unsigned int ms) +{ + sleep_ms += ms; +} + static void usleep_range(unsigned int lo, unsigned int hi) { } -static u32 readl(const void *p) { return *(const u32 *)p; } -static u64 readq(const void *p) { return *(const u64 *)p; } -static void writel(u32 v, void *p) { if (!ctrl_sticky) *(u32 *)p = v; } + +static u32 readl(const void *p) +{ + return *(const u32 *)p; +} + +static u64 readq(const void *p) +{ + return *(const u64 *)p; +} + +static void writel(u32 v, void *p) +{ + if (!ctrl_sticky) + *(u32 *)p = v; +} + static void *memremap(phys_addr_t p, size_t size, unsigned int mode) { return map_fail ? NULL : (void *)p; } -static void memunmap(void *p) { barriers++; } + +static void memunmap(void *p) +{ + barriers++; +} + static bool mod_delayed_work(void *wq, struct delayed_work *work, unsigned long delay) { work->scheduled = true; return true; } + static bool cancel_delayed_work(struct delayed_work *work) { work->scheduled = false; return true; } + +static bool cancel_delayed_work_sync(struct delayed_work *work) +{ + return cancel_delayed_work(work); +} + static bool queue_delayed_work(void *wq, struct delayed_work *work, unsigned long delay) { return mod_delayed_work(wq, work, delay); } -static int ave_cmd_halt(struct ave_dev *ave) { halts++; return halt_result; } -static int ave_cmd_close(struct ave_ctx *ctx) { closes++; return close_result; } -static int ave_cmd_complete(struct ave_ctx *ctx, unsigned int n) { return 0; } -static int ave_ipc_settle(struct ave_dev *ave, unsigned int ms) { return 0; } + +static int ave_cmd_halt(struct ave_dev *ave) +{ + halts++; + return halt_result; +} + +static int ave_cmd_close(struct ave_ctx *ctx) +{ + closes++; + return close_result; +} + +static int ave_cmd_complete(struct ave_ctx *ctx, unsigned int n) +{ + return 0; +} + +static int ave_ipc_settle(struct ave_dev *ave, unsigned int ms) +{ + return 0; +} + static int ave_ipc_poll_once(struct ave_dev *ave) { /* A deadline bug must fail, rather than leave the test spinning. */ assert(++polls < 100); return poll_result; } -static bool ave_sess_starved(struct ave_session *sess) { return false; } -static bool ave_sess_stalled(struct ave_session *sess) { return false; } -static void ave_sess_free_bufs(struct ave_dev *ave, struct ave_session *sess) { freed++; } + +static bool ave_sess_starved(struct ave_session *sess) +{ + return false; +} + +static bool ave_sess_stalled(struct ave_session *sess) +{ + return false; +} + +static void ave_sess_free_bufs(struct ave_dev *ave, struct ave_session *sess) +{ + freed++; +} + static void pm_runtime_mark_last_busy(struct device *dev) { } -static void pm_runtime_put_autosuspend(struct device *dev) { assert(dev->refs); dev->refs--; } -static void v4l2_m2m_suspend(void *dev) { suspends++; } -static void v4l2_m2m_resume(void *dev) { resumes++; } -static void ave_v4l2_suspend(struct ave_dev *ave) { stopped_for_sleep = true; } -static void v4l2_m2m_try_schedule(struct v4l2_m2m_ctx *ctx) { schedules++; } -static struct vb2_queue *v4l2_m2m_get_src_vq(struct v4l2_m2m_ctx *ctx) { return &ctx->src; } -static struct vb2_queue *v4l2_m2m_get_dst_vq(struct v4l2_m2m_ctx *ctx) { return &ctx->dst; } + +static void pm_runtime_put_autosuspend(struct device *dev) +{ + assert(dev->refs); + dev->refs--; +} + +static void v4l2_m2m_suspend(void *dev) +{ + suspends++; +} + +static void v4l2_m2m_resume(void *dev) +{ + resumes++; +} + +static void ave_v4l2_suspend(struct ave_dev *ave) +{ + stopped_for_sleep = true; +} + +static void v4l2_m2m_try_schedule(struct v4l2_m2m_ctx *ctx) +{ + schedules++; +} + +static struct vb2_queue *v4l2_m2m_get_src_vq(struct v4l2_m2m_ctx *ctx) +{ + return &ctx->src; +} + +static struct vb2_queue *v4l2_m2m_get_dst_vq(struct v4l2_m2m_ctx *ctx) +{ + return &ctx->dst; +} + #define V4L2_TYPE_IS_OUTPUT(t) ((t) == 1) enum vb2_buffer_state { VB2_BUF_STATE_ERROR = 1 }; + static struct vb2_v4l2_buffer *v4l2_m2m_src_buf_remove(struct v4l2_m2m_ctx *ctx) { return ctx->nsrc ? ctx->input[--ctx->nsrc] : NULL; } + static struct vb2_v4l2_buffer *v4l2_m2m_dst_buf_remove(struct v4l2_m2m_ctx *ctx) { return ctx->ndst ? ctx->output[--ctx->ndst] : NULL; } -static unsigned int v4l2_m2m_num_src_bufs_ready(struct v4l2_m2m_ctx *ctx) { return ctx->nsrc; } + +static unsigned int v4l2_m2m_num_src_bufs_ready(struct v4l2_m2m_ctx *ctx) +{ + return ctx->nsrc; +} + static void v4l2_m2m_buf_done(struct vb2_v4l2_buffer *buf, enum vb2_buffer_state state) { assert(++buf->completions == 1); buf->state = state; } -static void vb2_queue_error(struct vb2_queue *q) { q->error = true; } + +static bool vb2_is_streaming(struct vb2_queue *q) +{ + return q->streaming; +} + +static void vb2_queue_error(struct vb2_queue *q) +{ + q->error = true; +} + static void *vb2_get_drv_priv(struct vb2_queue *q) { - struct v4l2_m2m_ctx *m2m = container_of(q, struct v4l2_m2m_ctx, src); - return m2m->input[3]; + return q->priv; } #define to_vb2_v4l2_buffer(p) container_of(p, struct vb2_v4l2_buffer, vb2_buf) static void v4l2_m2m_buf_queue(struct v4l2_m2m_ctx *ctx, struct vb2_v4l2_buffer *buf) diff --git a/drivers/media/platform/apple/ave/tests/recovery-test.c b/drivers/media/platform/apple/ave/tests/recovery-test.c index f03e6423faf24..9b4b70eb34b59 100644 --- a/drivers/media/platform/apple/ave/tests/recovery-test.c +++ b/drivers/media/platform/apple/ave/tests/recovery-test.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only static const struct ave_coproc_regs regs = { .ctrl = 0, .boot_arg = 4, .vcbar = 8 }; static const struct ave_variant variant = { .regs = ®s }; static u64 coproc[4]; @@ -30,7 +30,8 @@ static void init(void) ctx[i].ave = &ave; ctx[i].fh.m2m_ctx = &m2m[i]; m2m[i].src.type = 1; - m2m[i].input[3] = (void *)&ctx[i]; + m2m[i].src.priv = &ctx[i]; + m2m[i].dst.priv = &ctx[i]; } } @@ -82,9 +83,11 @@ static void buffer_failure(void) assert(!m2m[i].nsrc && !m2m[i].ndst && !ctx[i].ready); assert(!ctx[i].poll_work.scheduled && !ave_job_ready(&ctx[i])); for (unsigned int n = 0; n < 3; n++) - assert(input[i][n].completions == 1 && input[i][n].state == VB2_BUF_STATE_ERROR); + assert(input[i][n].completions == 1 && + input[i][n].state == VB2_BUF_STATE_ERROR); for (unsigned int n = 0; n < 2; n++) - assert(output[i][n].completions == 1 && output[i][n].state == VB2_BUF_STATE_ERROR); + assert(output[i][n].completions == 1 && + output[i][n].state == VB2_BUF_STATE_ERROR); } assert(!dev.refs && !freed && ave.failed_sessions && !closes); /* A QBUF racing the failure is returned instead of silently requeued. */ @@ -95,7 +98,9 @@ static void buffer_failure(void) ave_buf_queue(&extra.vb2_buf); assert(extra.completions == 1 && extra.state == VB2_BUF_STATE_ERROR); } + *(u32 *)(ave.coproc + regs.boot_arg) = 0; assert(!ave_runtime_suspend(&dev)); + assert(readl(ave.coproc + regs.boot_arg) == AVE_COPROC_BOOT_ARG_VALUE); assert(ave.fw_state == AVE_FW_RECOVERING && ave.recovery_count == 1); assert(!readl(ave.coproc + regs.ctrl)); mutex_lock(&ave.hw_lock); @@ -106,6 +111,20 @@ static void buffer_failure(void) mutex_unlock(&ave.hw_lock); assert(freed == 2 && !ave.failed_sessions && !ave.inject_wedge); assert(ave.fw_state == AVE_FW_HALTED); + m2m[0].dst.streaming = true; + ave_stop_streaming(&m2m[0].src); + assert(ctx[0].failed); /* STREAMOFF one queue cannot clear the failure. */ + m2m[0].dst.streaming = false; + ave_stop_streaming(&m2m[0].dst); + assert(!ctx[0].failed); + m2m[0].src.error = false; /* vb2 clears queue errors on STREAMOFF. */ + { + struct vb2_v4l2_buffer extra = {}; + + extra.vb2_buf.vb2_queue = &m2m[0].src; + ave_buf_queue(&extra.vb2_buf); /* Before start_streaming on STREAMON. */ + assert(!extra.completions && m2m[0].nsrc == 1); + } } static void sleep_guarantees(void) @@ -198,6 +217,7 @@ int main(void) recovery_failures(); busy_ring_deadline(); restore_data(); - puts("ok: recovery ownership, queues, quarantine, reset proof, rate limit, PM rollback, DATA, IPC deadline"); + puts("ok: recovery ownership, queues, quarantine, reset proof, rate limit, " + "PM rollback, DATA, IPC deadline"); return 0; } diff --git a/drivers/media/platform/apple/ave/tests/run-recovery.py b/drivers/media/platform/apple/ave/tests/run-recovery.py index 37dba68a1c097..774556104560e 100644 --- a/drivers/media/platform/apple/ave/tests/run-recovery.py +++ b/drivers/media/platform/apple/ave/tests/run-recovery.py @@ -32,6 +32,7 @@ def main(): 'ave-sess.c': ('ave_sess_destroy', 'ave_sess_reap'), 'ave-v4l2.c': ('ave_return_bufs', 'ave_buf_queue', 'ave_session_index', 'ave_session_count', 'ave_session_close', 'ave_session_free', + 'ave_session_end', 'ave_stop_streaming', 'ave_reset_drain', 'ave_job_ready', 'ave_job_fail', 'ave_poll_work'), 'ave-ipc.c': ('ave_ipc_pump',), 'ave-drv.c': ('ave_runtime_suspend', 'ave_suspend', 'ave_resume', 'ave_inject_wedge'), From 9a98d3883d7478cf85fc44b7362321c8bbf02c2b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 01:46:15 +1100 Subject: [PATCH 1526/1579] accel/ane: keep the CPU boost when no performance controller is bound The performance-request rework replaced the engine-busy CPU frequency boost with a PMP level request, but only a DT that names apple,performance-controller (and a pmp_report with its ane-dvfs window) gets a level. Everything else, T8103 and every machine still booting an older DTB set, lost the boost and gained nothing. On the M1 Pro at the boot ANE clock the boost is worth 1 to 3 % at p50 on single layers and mostly protects the tail: a 3x3 depthwise layer at 112x112 goes from 207 us p99 with the boost to 245-364 us without it, over 2000 submits. Bring the boost back for exactly that case. When the DT binds no performance controller, ane_perf_init sets up the same min-frequency QoS as before and ane_perf_begin/end drive it around each submit. A DT with a controller gets the level request and no boost, as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/Makefile | 3 +- drivers/accel/ane/ane.h | 25 ++++++ drivers/accel/ane/ane_boost.c | 142 ++++++++++++++++++++++++++++++++++ drivers/accel/ane/ane_perf.c | 32 +++++++- 4 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 drivers/accel/ane/ane_boost.c diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile index 79418f74c6192..3877562f7d17e 100644 --- a/drivers/accel/ane/Makefile +++ b/drivers/accel/ane/Makefile @@ -10,7 +10,8 @@ ane-y := \ ane_drv.o \ ane_tm.o \ ane_dart.o \ - ane_perf.o + ane_perf.o \ + ane_boost.o ane_t6021-y := \ ane_fw_platform.o \ diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index 332b0d31a6ad9..c1afb639400f1 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -126,8 +127,27 @@ struct ane_device { bool voted; bool clock_held; bool pinned; + bool cpu_boost; } perf; + /* + * Engine-busy CPU cluster boost (ane_boost.c): min-frequency QoS on + * every cpufreq policy while a submit runs and until boost_idle_ms + * after the last one completes. Used only when the DT binds no + * performance controller, so the engine keeps the boost it had + * until it gets a clock level instead. + */ + struct ane_boost { + struct mutex lock; /* protects the fields below */ + struct delayed_work off; + struct freq_qos_request *legs; + int nlegs; + int held; + unsigned long last_end; + bool busy; + bool on; + } boost; + /* * Producer-side stats (sysfs ane_stats, debugfs ane_timeline; * see ane_stats.h). stats=0 or a failed ring allocation leaves @@ -166,6 +186,11 @@ int ane_perf_begin(struct ane_device *ane); void ane_perf_end(struct ane_device *ane); int ane_perf_suspend(struct ane_device *ane); +int ane_boost_init(struct ane_device *ane); +void ane_boost_exit(struct ane_device *ane); +void ane_boost_begin(struct ane_device *ane); +void ane_boost_end(struct ane_device *ane); + int ane_dart_init(struct ane_device *ane); void ane_dart_mask(struct ane_device *ane); void ane_dart_unmask(struct ane_device *ane); diff --git a/drivers/accel/ane/ane_boost.c b/drivers/accel/ane/ane_boost.c new file mode 100644 index 0000000000000..179a980a01950 --- /dev/null +++ b/drivers/accel/ane/ane_boost.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Engine-busy CPU cluster boost. + * + * Bandwidth-bound programs run at a speed set by the CPU cluster + * p-states. On T8103, with a mostly-sleeping submitter at normal + * priority, some submits run about twice as long as normal, while the + * submitter's CPU time and run delay stay unchanged; with both clusters + * pinned at their top p-state (performance governor, or SCHED_FIFO for + * the submitter, which schedutil boosts the same way) the slow submits + * disappear. A single long program slows down as well if the QoS + * lapses before it completes, so the hold must last from submit start + * to completion, and the idle delay counts from completion. The likely + * mechanism is the second p-state field apple-soc-cpufreq writes with + * the cluster p-state on T8103 only (DVFS_CMD PS2, has_ps2); the + * memory-controller tuning its author named as unimplemented is still + * absent from the tree. Whatever the field drives, the cure is the + * cluster p-state, which cpufreq lets a driver request. + * + * So hold a min-frequency QoS at the top of every cpufreq policy while a + * submit runs, and drop it boost_idle_ms after the last one completes. + * The QoS goes through cpufreq's own constraint object, so it composes + * with any governor and never touches the DVFS registers. + */ +#include +#include +#include +#include +#include +#include + +#include "ane.h" + +static unsigned int boost_idle_ms = 100; +module_param(boost_idle_ms, uint, 0644); +MODULE_PARM_DESC(boost_idle_ms, + "hold every CPU cluster at its top p-state while a submit runs and this long after the last one completes (0 = off)"); + +/* Lock held. */ +static void ane_boost_set(struct ane_device *ane, bool on) +{ + struct ane_boost *b = &ane->boost; + int cpu, n = 0; + + if (on) { + for_each_possible_cpu(cpu) { + struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); + int err; + + if (!policy) + continue; + if (cpu != cpumask_first(policy->related_cpus) || + n >= b->nlegs) { + cpufreq_cpu_put(policy); + continue; + } + err = freq_qos_add_request(&policy->constraints, + &b->legs[n], FREQ_QOS_MIN, + policy->cpuinfo.max_freq); + cpufreq_cpu_put(policy); + if (err >= 0) + n++; + } + b->held = n; + } else { + for (n = 0; n < b->held; n++) + freq_qos_remove_request(&b->legs[n]); + b->held = 0; + } + b->on = on; +} + +static void ane_boost_off_work(struct work_struct *work) +{ + struct ane_device *ane = + container_of(work, struct ane_device, boost.off.work); + struct ane_boost *b = &ane->boost; + + mutex_lock(&b->lock); + /* A submit that began or ended after this expiry was armed keeps + * the boost on; its own completion re-arms the drop. + */ + if (b->on && !b->busy && + time_after_eq(jiffies, b->last_end + msecs_to_jiffies(boost_idle_ms))) + ane_boost_set(ane, false); + mutex_unlock(&b->lock); +} + +/* Engine lock held: submits never overlap, so busy is a flag. */ +void ane_boost_begin(struct ane_device *ane) +{ + struct ane_boost *b = &ane->boost; + + if (!b->legs || !boost_idle_ms) + return; + mutex_lock(&b->lock); + b->busy = true; + if (!b->on) + ane_boost_set(ane, true); + mutex_unlock(&b->lock); +} + +void ane_boost_end(struct ane_device *ane) +{ + struct ane_boost *b = &ane->boost; + + if (!b->legs) + return; + mutex_lock(&b->lock); + b->busy = false; + b->last_end = jiffies; + if (b->on) + mod_delayed_work(system_dfl_wq, &b->off, + msecs_to_jiffies(boost_idle_ms)); + mutex_unlock(&b->lock); +} + +int ane_boost_init(struct ane_device *ane) +{ + struct ane_boost *b = &ane->boost; + + mutex_init(&b->lock); + INIT_DELAYED_WORK(&b->off, ane_boost_off_work); + b->nlegs = num_possible_cpus(); + b->legs = kcalloc(b->nlegs, sizeof(*b->legs), GFP_KERNEL); + return b->legs ? 0 : -ENOMEM; +} + +void ane_boost_exit(struct ane_device *ane) +{ + struct ane_boost *b = &ane->boost; + + if (!b->legs) + return; + cancel_delayed_work_sync(&b->off); + mutex_lock(&b->lock); + if (b->on) + ane_boost_set(ane, false); + mutex_unlock(&b->lock); + kfree(b->legs); + b->legs = NULL; +} diff --git a/drivers/accel/ane/ane_perf.c b/drivers/accel/ane/ane_perf.c index 23daae652957a..468fd151c4886 100644 --- a/drivers/accel/ane/ane_perf.c +++ b/drivers/accel/ane/ane_perf.c @@ -60,6 +60,9 @@ int ane_perf_begin(struct ane_device *ane) struct ane_perf *p = &ane->perf; int ret = 0; + if (p->cpu_boost) + ane_boost_begin(ane); + /* The submit ioctl already holds a runtime PM reference. */ mutex_lock(&p->lock); p->busy = true; @@ -96,6 +99,9 @@ void ane_perf_end(struct ane_device *ane) { struct ane_perf *p = &ane->perf; + if (p->cpu_boost) + ane_boost_end(ane); + mutex_lock(&p->lock); p->busy = false; p->release_retries = 0; @@ -199,10 +205,26 @@ int ane_perf_init(struct ane_device *ane, unsigned int high_level) dev_pm_opp_put(opp); if (!p->peak_bw || p->avg_bw > p->peak_bw) return -EINVAL; - } else { - dev_warn(ane->dev, "no memory bandwidth provider; CPU frequency boost removed\n"); } - return devm_device_add_group(ane->dev, &ane_perf_group); + + ret = devm_device_add_group(ane->dev, &ane_perf_group); + if (ret) + return ret; + + /* + * Without a performance controller the engine runs at its boot + * clock, and the CPU cluster boost is what keeps bandwidth-bound + * programs and the submit tail fast. Keep it until the DT binds a + * controller. + */ + if (!p->report) { + ret = ane_boost_init(ane); + if (ret) + return ret; + p->cpu_boost = true; + dev_info(ane->dev, "no performance controller; keeping the CPU frequency boost\n"); + } + return 0; } int ane_perf_exit(struct ane_device *ane) @@ -212,6 +234,10 @@ int ane_perf_exit(struct ane_device *ane) if (!p->initialized) return 0; + if (p->cpu_boost) { + ane_boost_exit(ane); + p->cpu_boost = false; + } cancel_delayed_work_sync(&p->idle); mutex_lock(&p->lock); p->error = ane_perf_drop(ane); From 5c93b7734ea47ed831a71589642c2bd0ebf238f4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 18:10:57 +1100 Subject: [PATCH 1527/1579] accel/ane: power-cycle runtime PM suppliers after a bounded job timeout Suspend the ANE consumer before its domain suppliers, verify the engine islands actually power off, and restore its runtime PM reference on every exit. Avoid the system-sleep force helpers and speculative retained task-error writes. Keep DMA backing and the exceptional module pin until recovery is verified. Use one deadline across H13 fault drains and H14 command ACK/finish waits. Add an explicit destructive H13 recovery test and a deterministic fixture generator using the licensed public compiler. Document the remaining H14 firmware ownership and stale-ID work; H14 reset remains quarantined. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 86 +++++++ drivers/accel/ane/ane.h | 2 + drivers/accel/ane/ane_drv.c | 40 ++- drivers/accel/ane/ane_t6021_rtclient_main.c | 16 +- drivers/accel/ane/ane_tm.c | 237 +++++------------- tools/testing/selftests/drivers/ane/Makefile | 1 + .../drivers/ane/make-hang-fixture.py | 82 ++++++ .../testing/selftests/drivers/ane/recovery.py | 121 +++++++++ 8 files changed, 395 insertions(+), 190 deletions(-) create mode 100755 tools/testing/selftests/drivers/ane/make-hang-fixture.py create mode 100755 tools/testing/selftests/drivers/ane/recovery.py diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index 84e7a75f56488..ce19cff094c93 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -254,3 +254,89 @@ reports, routing, duplicates, timeouts, capacity and UUID exhaustion. HWX tests generate synthetic linear/scratch forms with optional kernel data, then exercise command, geometry, tensor, aperture, symbol, relocation and digest refusal paths. No hardware inference result is implied. + +H13 timeout and engine recovery +============================== + +The host task-manager path (``ane.ko``) gives each submission one second, +including DART fault-drain retries. An expired submission returns +``-ETIMEDOUT``; recovery never silently retries a program whose output may be +partial. The task's GEM lookup references are released when the ioctl returns. + +Recovery holds the engine and PM-entry locks, drops the current consumer +runtime-PM reference, synchronously suspends its power-domain suppliers, and +checks the PMGR ACTUAL fields for power-off. It then restores the consumer +reference, checks power-on before engine MMIO, and runs the probe task-manager +enable sequence. BO handles and mappings remain valid for resubmission. +The system-sleep force helpers are unsuitable here: a forced, in-use supplier +can retain its active runtime status and keep its parent domains powered. + +A power-off acknowledgment is required before reinitializing the engine. +Only after power-on and task-manager readiness are also verified does recovery +release the exceptional module pin and reclaim preserved BOs. If any step +fails, mappings, backing pages and the pin remain held; new submissions fail. +This also covers a partially resumed engine whose retained queue state is +uncertain. The ``reset`` sysfs write retries recovery. Module reload cannot +recover a device while that pin is held. Direct PMGR SET writes and speculative +writes to undocumented task-error registers are not part of this reset path. + +Destructive recovery selftest +---------------------------- + +``tools/testing/selftests/drivers/ane/recovery.py`` is an explicitly invoked +hardware test, excluded from the default device-free selftests. It requires +exclusive ownership of an M1-family ANE, libane, the packaged H13 add fixture, +and a deliberately invalid convolution program. Run it through the hardware +controller with a watchdog and kernel-log capture:: + + python3 recovery.py /usr/share/omarchy-ane/fixtures/h13-anec/add/program-0.anec hang.anec + +The test keeps both clients open, submits the invalid program three times, +requires a bounded ``ETIMEDOUT`` and ``wedged=0`` each time, and runs twenty +bit-exact add jobs on the surviving client's original fd and mappings after +each reset. Its oracle uses exact fp16 quarter values and checks the complete +output, including padding. Closing both clients must return the module +reference count to its initial value; the boot identity must remain unchanged. +An operator should also verify three power-off acknowledgments in the kernel +log and a normal module unload/reload after all clients close. A failed reset +is a test failure, not permission to force unload the module. + +Generate ``hang.anec`` on a build host with ``make-hang-fixture.py`` and a clean +checkout of the MIT-licensed public compiler at commit +``ef2dbceefedff38ecce4f3dc593e148c5e7a6d54`` from +https://github.com/aurora-silicon/mil-hwx-compiler/pull/2:: + + python3 make-hang-fixture.py /path/to/mil-hwx-compiler hang.anec --cxx clang++ + +This first checks that the normal encoder refuses the shape. It bypasses only +that refusal in a temporary translation unit to generate a deterministic +stride-2 1x1 128-to-256 convolution at 56-to-28 spatial size, the refused family +reported to hang the M1 Pro. The compiler checkout is unchanged. No Apple +binary or private program capture is used. The generator writes a hash receipt; +the hardware test must fail if this particular instance does not actually +produce a timeout. + +H14 recovery lifecycle +---------------------- + +The firmware path (``ane_t6021.ko``) uses one bounded deadline for the command +acknowledgment and its finish event. H14 engine reset is not enabled by this +change. Reusing only the H13 task-manager re-arm would leave the firmware and +its program cache invalid. The corresponding recovery implementation needs to: + +* Serialize firmware commands and stop the RX worker before releasing the + consumer's runtime-PM reference and synchronously suspending its suppliers. +* Verify power-off for the CPU and engine partitions through their providers + before dropping boot pins or reclaiming firmware-visible allocations. +* Retain metadata for every firmware-owned BO and boot allocation, including + objects whose last userspace reference disappeared. The current release + path discards some of this metadata while retaining the backing memory. +* Restore the validated firmware image and run the existing fenced probe boot + sequence, including channel-table validation, host acknowledgment and + CONFIG_GET. A failed boot remains quarantined with its DMA backing held. +* Invalidate the cached program/process IDs across the restart and prevent a + stale userspace ID from aliasing a newly allocated firmware ID. Clients must + reload their programs before resubmitting. + +Those ownership and identifier changes require their own H14 reset test. +Quarantine must not be cleared solely because a power transition returned zero. diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index 332b0d31a6ad9..2d94e597a863e 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -63,6 +63,8 @@ struct ane_device { * and protects bo_list, preserved_list and removed. */ struct mutex engine_lock; + /* Serializes runtime-PM references across a recovery cycle. */ + struct mutex pm_lock; struct list_head bo_list; /* diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index ac113ac727b4e..7558fd7eaea6b 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -753,18 +753,19 @@ static ssize_t reset_store(struct device *dev, struct device_attribute *attr, /* Recovery power-cycles the partitions and then writes the engine: * hold a reference so that autosuspend cannot gate it in between. */ + mutex_lock(&ane->pm_lock); err = pm_runtime_resume_and_get(dev); if (err < 0) - return err; + goto out_pm; mutex_lock(&ane->engine_lock); if (!ane->removed) err = ane_tm_recover(ane); mutex_unlock(&ane->engine_lock); pm_runtime_put_autosuspend(dev); - if (err) - return err; - return count; +out_pm: + mutex_unlock(&ane->pm_lock); + return err < 0 ? err : count; } static DEVICE_ATTR_WO(reset); @@ -818,12 +819,13 @@ static int ane_drm_open(struct drm_device *drm, struct drm_file *file) /* Bring up power while the file context is created. A failed * resume propagates; nothing is released that was not acquired. */ + mutex_lock(&ane->pm_lock); err = pm_runtime_resume_and_get(ane->dev); - if (err < 0) - return err; + if (err >= 0) + pm_runtime_put_autosuspend(ane->dev); - pm_runtime_put_autosuspend(ane->dev); - return 0; + mutex_unlock(&ane->pm_lock); + return err < 0 ? err : 0; } /* Retry recovery on close, retaining quarantine if DMA cannot be quiesced. */ @@ -879,14 +881,17 @@ static long ane_drm_ioctl(struct file *file, unsigned int cmd, _IOC_NR(cmd) >= DRM_COMMAND_END) return -ENOTTY; + mutex_lock(&ane->pm_lock); err = pm_runtime_resume_and_get(ane->dev); if (err < 0) - return err; + goto unlock_pm; err = dispatch(file, cmd, arg); pm_runtime_put_autosuspend(ane->dev); +unlock_pm: + mutex_unlock(&ane->pm_lock); return err; } @@ -918,11 +923,19 @@ static int ane_drm_release(struct inode *inode, struct file *file) { struct drm_file *priv = file->private_data; struct device *dev = get_device(priv->minor->dev->dev); + struct ane_device *ane = priv->minor->dev->dev_private; int err; + /* drm_release may drop the last drm reference; keep ane alive until + * pm_lock is unlocked. + */ + drm_dev_get(&ane->drm); + mutex_lock(&ane->pm_lock); pm_runtime_get_sync(dev); err = drm_release(inode, file); pm_runtime_put_autosuspend(dev); + mutex_unlock(&ane->pm_lock); + drm_dev_put(&ane->drm); put_device(dev); return err; } @@ -1166,7 +1179,6 @@ static int ane_platform_probe(struct platform_device *pdev) drm->dev_private = ane; atomic_set(&ane->wedged, 0); - ane->tm_retention = soc->tm_retention; /* Producer-side stats: preallocate the ring at probe; the files * and the hot-path branch key off stats_slots. @@ -1226,6 +1238,7 @@ static int ane_platform_probe(struct platform_device *pdev) mutex_init(&ane->iommu_lock); mutex_init(&ane->engine_lock); + mutex_init(&ane->pm_lock); INIT_LIST_HEAD(&ane->bo_list); INIT_LIST_HEAD(&ane->preserved_list); err = ane_perf_init(ane, soc->high_level); @@ -1302,6 +1315,7 @@ static void ane_platform_remove(struct platform_device *pdev) /* Powered and pinned for the teardown: the unmaps below flush the * DART TLBs, and no autosuspend may race the genpd detach. */ + mutex_lock(&ane->pm_lock); pm_runtime_get_sync(ane->dev); mutex_lock(&ane->engine_lock); @@ -1312,6 +1326,7 @@ static void ane_platform_remove(struct platform_device *pdev) dev_err(ane->dev, "forced removal of a wedged device is unsafe; reboot required\n"); mutex_unlock(&ane->engine_lock); + mutex_unlock(&ane->pm_lock); return; } @@ -1334,6 +1349,7 @@ static void ane_platform_remove(struct platform_device *pdev) pm_runtime_dont_use_autosuspend(ane->dev); pm_runtime_put_noidle(ane->dev); mutex_unlock(&ane->engine_lock); + mutex_unlock(&ane->pm_lock); } static int __maybe_unused ane_runtime_suspend(struct device *dev) @@ -1382,6 +1398,10 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) eng); } + /* Recovery verifies the power-on ACK before its first engine access. */ + if (ane->recovering) + return 0; + ane_tm_enable(ane, first); /* First enable is the engine's fresh signature; recovery compares diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 690df3eb2afec..ed08c6506b324 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -1066,7 +1066,7 @@ static int ane_rtclient_call_wait(struct ane_rtclient *ane, if (ane_t6021_tracing) return ane_rtclient_call_wait_traced(ane, deadline); - if (ane->soc->platform && time_after_eq(jiffies, deadline)) + if (time_after_eq(jiffies, deadline)) return -ETIMEDOUT; do { @@ -1099,8 +1099,7 @@ static int ane_rtclient_command(struct ane_rtclient *ane, unsigned int channel, unsigned int timeout_ms) { - unsigned long deadline = ane->soc->platform && ane->waiting ? - ane->waiting->deadline : jiffies + msecs_to_jiffies(timeout_ms); + unsigned long deadline; int ret; u64 stats_ticket = 0; u64 stats_submit_ns = 0; @@ -1134,6 +1133,11 @@ static int ane_rtclient_command(struct ane_rtclient *ane, (!ane->waiting || !ane->waiting->active || ane->pending.quarantined)) return -EIO; + /* One bounded budget covers both ACK and finish, including H14. */ + timeout_ms = min(timeout_ms, 5000U); + deadline = ane->soc->platform && ane->waiting ? + ane->waiting->deadline : jiffies + msecs_to_jiffies(timeout_ms); + if (stats_call) { stats_submit_ns = ktime_get_ns(); stats_ticket = ane_stats_begin(&ane->stats_ctrs, @@ -1141,8 +1145,7 @@ static int ane_rtclient_command(struct ane_rtclient *ane, stats_submit_ns, 1); } ret = ane_rtclient_legacy_exchange(ane, command, length, opcode, - channel, ane->soc->platform ? deadline : - jiffies + msecs_to_jiffies(timeout_ms)); + channel, deadline); if (ret) { ane_t6021_tracing = false; if (stats_call) @@ -1157,8 +1160,7 @@ static int ane_rtclient_command(struct ane_rtclient *ane, if (execute) { if (ane_t6021_tracing) ane_t6021_trace_add(ANE_TR_ACK, 0); - ret = ane_rtclient_call_wait(ane, ane->soc->platform ? deadline : - jiffies + msecs_to_jiffies(timeout_ms)); + ret = ane_rtclient_call_wait(ane, deadline); ane_t6021_tracing = false; if (!ret) dma_rmb(); diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 200721e214f46..6967a0630613f 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "ane_tm.h" @@ -266,6 +267,7 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) struct ane_dart_scratch *scratch = ane->scratch; bool faulted = false; int err, status; + ktime_t deadline = ktime_add_ms(ktime_get(), 1000); lockdep_assert_held(&ane->engine_lock); ane_dart_mask(ane); @@ -280,6 +282,10 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) ane_tm_push_tq(ane, req); for (;;) { + if (ktime_compare(ktime_get(), deadline) >= 0) { + err = -ETIMEDOUT; + break; + } /* Latch-first completion poll. Engine register reads hang * the SoC (T8103) once a DART fault has landed, inside the * 1-microsecond completion poll and before any fault @@ -289,10 +295,13 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) * iteration's latch read and engine read. */ err = read_poll_timeout(ane_tm_poll_step, status, - status != 0, 1, 1000000, false, + status != 0 || ktime_compare(ktime_get(), deadline) >= 0, + 1, 1000000, false, ane, req, &finished); if (!err && status < 0) err = status; + if (!err && !status) + err = -ETIMEDOUT; if (ane_dart_faulted(ane, &fault_iova, &fault_status)) { faulted = true; if (ane->scratch_pages >= ANE_DART_SCRATCH_MAX) @@ -366,38 +375,18 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) * and return to accepting work. Only a failed reset preserves * resources until reboot. */ - if (ane_tm_recover(ane) < 0) { - if (ane->tm_retention) - dev_err(ane->dev, - "recovery failed; preserving resources until module reload or reboot (tm/tq file retained through the cycle: the latched task error parks the TM in place; rmmod+insmod restores service)\n"); - else - dev_err(ane->dev, - "recovery failed; preserving resources until reboot\n"); - } + if (ane_tm_recover(ane) < 0) + dev_err(ane->dev, + "recovery failed; submissions disabled, resources retained while DMA may be active; retry reset or reboot\n"); return err; } /* - * Bounded recovery after a failed task. The only reset this driver knows - * is the partition power cycle system sleep already performs on this - * hardware. With the several ANE power partitions attached as genpd - * devices (T8103 five, T6001 similar), gating happens on those devices; - * with a single domain attached directly to the device, on the device - * itself. Gating the partitions stops any DMA still fetching the dead - * task and clears the tm register file; ungating brings the engine back - * through the probe resume path (ane_tm_enable), then the task manager - * must report idle. BO mappings deliberately stay in place: they remain - * coherent through the cycle and valid for the next submit, and once the - * wedge clears, BO_FREE unmaps them through the normal path again. - * Callers hold engine_lock. - * - * T6001 note: its set0/base islands are - * firmware-locked (direct pmgr writes external-abort; see the SET block - * comment above) and hold the tm/tq file in retention through any - * genpd cycle - TQ_EN still reads 0x3000 after a 20 ms and a 2 s gate, - * and tasks dispatched afterwards complete but compute nondeterministic - * garbage. The idle-or-fresh poll below therefore fails on T6001 and - * recovery preserves until reboot instead of serving wrong data. + * Recovery runs with engine_lock and pm_lock held and one consumer PM + * reference. Suspend through runtime PM, just as autosuspend does, so the + * supplier references and the domain hierarchy are released together. + * Never use the system-sleep force helpers here: an in-use forced supplier + * retains RPM_ACTIVE, which prevents its parent domain from powering off. */ /* No engine MMIO unless the owning partitions read powered on (ACTUAL @@ -428,8 +417,10 @@ static int ane_ps_verify_on(struct ane_device *ane) static int ane_pd_cycle(struct ane_device *ane) { - int err = 0; + int err, resume_err; + u32 act; + lockdep_assert_held(&ane->pm_lock); ane->recovering = true; err = ane_perf_suspend(ane); if (err) { @@ -437,147 +428,48 @@ static int ane_pd_cycle(struct ane_device *ane) return err; } if (ane->pd_count > 1) { - int gated = 0; - - for (int i = 0; i < ane->pd_count; i++) { - pm_runtime_get_noresume(ane->pd_dev[i]); - gated = i + 1; - dev_dbg(ane->dev, - "ANERD pd[%d] %s force_suspend begin (ps act %#x)\n", - i, dev_name(ane->pd_dev[i]), ane_ps_act(ane)); - err = pm_runtime_force_suspend(ane->pd_dev[i]); - dev_dbg(ane->dev, - "ANERD pd[%d] force_suspend -> %d (ps act %#x)\n", - i, err, ane_ps_act(ane)); - if (err) - break; - } - if (!err) - for (int i = 0; i < ane->pd_count; i++) { - dev_dbg(ane->dev, - "ANERD pd[%d] %s force_resume begin (ps act %#x)\n", - i, dev_name(ane->pd_dev[i]), - ane_ps_act(ane)); - err = pm_runtime_force_resume(ane->pd_dev[i]); - dev_dbg(ane->dev, - "ANERD pd[%d] force_resume -> %d (ps act %#x)\n", - i, err, ane_ps_act(ane)); - if (err) - break; - } - while (gated--) - pm_runtime_put_noidle(ane->pd_dev[gated]); + for (int i = 0; i < ane->pd_count; i++) + dev_pm_genpd_synced_poweroff(ane->pd_dev[i]); } else { - /* Pin a second usage ref for the cycle: force_suspend only - * marks needs_force_resume when a ref beyond the caller's - * one (submit ioctl or reset write) is held, and without - * that mark force_resume would leave the partition gated. - */ - pm_runtime_get_noresume(ane->dev); - dev_dbg(ane->dev, "ANERD dev force_suspend begin\n"); - err = pm_runtime_force_suspend(ane->dev); - dev_dbg(ane->dev, "ANERD dev force_suspend -> %d\n", err); - if (!err) { - dev_dbg(ane->dev, "ANERD dev force_resume begin\n"); - err = pm_runtime_force_resume(ane->dev); - dev_dbg(ane->dev, "ANERD dev force_resume -> %d\n", - err); - } - pm_runtime_put_noidle(ane->dev); + dev_pm_genpd_synced_poweroff(ane->dev); } - ane->recovering = false; - return err; -} - -/* - * Drain the task-manager state a timed-out task leaves behind when the - * tm/tq file survives the power cycle in retention (T6001; descriptor - * tm_retention). T8103 never runs this: its cycle is a full POR of the - * file, and the re-arm below starts it clean. - * - * Two things must go, or the retained TM never reads idle again: - * - pending events on both IRQ lines. Each event pops by reading - * INFO/UNK1/TMST/UNK2, exactly as ane_tm_collect_events pops them; - * a stale backlog (or one over the 64-entry collect cap) would make - * the next submit's completion poll miss or fail. - * - the latched queue entry of the dead task. TQ_STATUS/NID1 get the - * same clear the normal completion path gives a finished queue. - */ -static void ane_tm_drain_retained(struct ane_device *ane) -{ - int line, qid; - - /* Stop the TM first: the cycle leaves the enable|halt latch set - * (TQ_EN reads 0x3000). Clear both bits so the - * re-arm below starts the TM from a defined stopped state. + /* Drop exactly the reference owned by the calling ioctl/close/reset. + * pm_lock keeps another driver entry from resuming us mid-cycle. */ - ane_rec_writel(ane, "TM_TQ_EN stop tm+0x0c", - ane->engine + ANE_TM_BASE + TM_TQ_EN, - tm_read32(ane, TM_TQ_EN) & ~0x3000U); - - for (line = 0; line < 2; line++) { - u32 count = tm_read32(ane, TM_IRQ_EVTC(line)); - u32 n; - - if (count > 64) - count = 64; - for (n = 0; n < count; n++) { - tm_read32(ane, TM_IRQ_INFO(line)); - tm_read32(ane, TM_IRQ_UNK1(line)); - tm_read32(ane, TM_IRQ_TMST(line)); - tm_read32(ane, TM_IRQ_UNK2(line)); - } - } - tm_write32(ane, TM_IRQ_ACK, tm_read32(ane, TM_IRQ_ACK) | 3); + err = pm_runtime_put_sync_suspend(ane->dev); + if (err < 0) + goto resume; - /* The dead task leaves per-queue error codes latched here (seen - * 0x22222222 / 0x2222: four-bit code 2 per TQ slot). While any - * latch is set the TM stays halted and TM_STATUS reads 0. Try the - * W1C convention first (write the read value back), fall back to - * plain zero-clear, and log what this silicon answered to. + /* Supplier idle requests are asynchronous. Finish them, then suspend + * leaves before parents using ordinary runtime PM (still enabled). + * genpd owns all PMGR writes and the parent-domain ordering. */ - { - static const u16 err_reg[3] = { TM_ERROR1, TM_ERROR2, - TM_ERROR3 }; - char reg[24]; - int ei; - - for (ei = 0; ei < 3; ei++) { - void __iomem *addr = - ane->engine + ANE_TM_BASE + err_reg[ei]; - u32 val = readl(addr); - - if (!val) - continue; - writel(val, addr); - if (readl(addr)) - writel(0x0, addr); - snprintf(reg, sizeof(reg), "TM_ERROR%d clear tm+%#x", - ei + 1, err_reg[ei]); - ane_rec_read32(ane, reg, addr); - } + for (int i = ane->pd_count - 1; i >= 0 && ane->pd_count > 1; i--) { + pm_runtime_barrier(ane->pd_dev[i]); + err = pm_runtime_suspend(ane->pd_dev[i]); + if (err < 0) + goto resume; } - for (qid = 0; qid < ANE_TQ_COUNT; qid++) { - u32 nid = tq_read32(ane, TQ_NID1(qid)); - - tq_write32(ane, TQ_STATUS(qid), 0x0); - if (nid & 1) - tq_write32(ane, TQ_NID1(qid), nid & ~1U); - } - - /* Log the retained state: what the dead task left in the - * committed counter and the error latches. + /* A successful runtime suspend alone is not proof of power-off: + * genpd may retain a shared or busy parent. Only PMGR is safe to read + * here; do not access the engine aperture until it is powered again. */ - ane_rec_read32(ane, "TM_COMMITTED tm+0x44 (drain)", - ane->engine + ANE_TM_BASE + TM_COMMITTED); - ane_rec_read32(ane, "TM_ERROR1 tm+0x58 (drain)", - ane->engine + ANE_TM_BASE + TM_ERROR1); - ane_rec_read32(ane, "TM_ERROR2 tm+0x5c (drain)", - ane->engine + ANE_TM_BASE + TM_ERROR2); - ane_rec_read32(ane, "TM_ERROR3 tm+0x60 (drain)", - ane->engine + ANE_TM_BASE + TM_ERROR3); + if (!ane->ps) { + err = -EOPNOTSUPP; + goto resume; + } + err = read_poll_timeout(ane_ps_act, act, act == 0, + 100, 100000, false, ane); + dev_info(ane->dev, "recovery: engine power-off actual=%#x rc=%d\n", + act, err); + +resume: + /* get_sync restores the caller's usage reference even on failure. */ + resume_err = pm_runtime_get_sync(ane->dev); + ane->recovering = false; + return err < 0 ? err : (resume_err < 0 ? resume_err : 0); } int ane_tm_recover(struct ane_device *ane) @@ -606,23 +498,22 @@ int ane_tm_recover(struct ane_device *ane) return err; } - /* T6001 (tm_retention): the cycle did NOT clear the tm/tq file — - * the timed-out task's events and queue entry are still latched. - * Drain them, or the TM below never reads idle and recovery - * degrades to preserve-until-reboot. + /* A full, acknowledged power-off replaces the speculative retained-TM + * drain. Never clear undocumented error registers or restart a queue + * which still contains the old halted task. */ - if (ane->tm_retention) - ane_tm_drain_retained(ane); + if (tm_read32(ane, TM_TQ_EN) & 0x2000) { + dev_err(ane->dev, "recovery: task manager still halted after power-off\n"); + return -EIO; + } - /* Power-on reset cleared the tm register file; re-arm it exactly - * like the probe resume path does. - */ ane_tm_enable(ane, true); err = readl_poll_timeout(ane->engine + ANE_TM_BASE + TM_STATUS, status, (status & TM_IS_IDLE) || - status == ane->tm_status_fresh, + (ane->tm_status_known && + status == ane->tm_status_fresh), 100, 1000000); status = ane_rec_read32(ane, "TM_STATUS tm+0x54", ane->engine + ANE_TM_BASE + TM_STATUS); @@ -633,7 +524,7 @@ int ane_tm_recover(struct ane_device *ane) } if (atomic_xchg(&ane->wedged, 0)) { - module_put(THIS_MODULE); /* drop the wedge pin */ + module_put(THIS_MODULE); /* verified reset releases the wedge pin */ /* The power cycle above quiesced every transaction, so the * wedge-preserved mappings (dead BOs kept alive only to * stop IOVA teardown under active DMA) can now be unmapped diff --git a/tools/testing/selftests/drivers/ane/Makefile b/tools/testing/selftests/drivers/ane/Makefile index 5034bf537707f..48712ae211d84 100644 --- a/tools/testing/selftests/drivers/ane/Makefile +++ b/tools/testing/selftests/drivers/ane/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 TEST_PROGS := h16-fixture.py +TEST_PROGS_EXTENDED := recovery.py make-hang-fixture.py include ../../lib.mk diff --git a/tools/testing/selftests/drivers/ane/make-hang-fixture.py b/tools/testing/selftests/drivers/ane/make-hang-fixture.py new file mode 100755 index 0000000000000..13b480c6be2cc --- /dev/null +++ b/tools/testing/selftests/drivers/ane/make-hang-fixture.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Build an H13 recovery fixture from a pinned public MIT encoder (no device IO).""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import tempfile + +COMMIT = "ef2dbceefedff38ecce4f3dc593e148c5e7a6d54" +GUARD = "chunks.size() != 1 || chunks.front() > 8" +DRIVER = r''' +#include "H13Program.h" +#include +#include +#include +int main(int argc, char **argv) { + using namespace ane::h13; + ConvShape shape{1, 1, 2, 1, false, {128, 56, 56}, {256, 28, 28}}; + if (argc == 1) { + auto why = convGenericRefusal(shape); + std::cout << why << "\n"; + return why.find("8 interleaved lanes") == std::string::npos; + } + // Nonzero, exactly representable fp16 weights; no private compiler output. + std::vector weights(256 * 128 * 2); + for (unsigned i = 0; i < weights.size() / 2; ++i) { + const unsigned half = 0x2400 + (i * 17 % 0x400); + weights[2 * i] = half; weights[2 * i + 1] = half >> 8; + } + auto bytes = encodeANEC(encodeConvGeneric(shape, weights.data(), weights.size(), nullptr, 0)); + std::ofstream(argv[1], std::ios::binary).write((const char *)bytes.data(), bytes.size()); + return 0; +} +''' + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("compiler", type=Path, help="clean mil-hwx-compiler checkout at pinned PR 2 commit") + p.add_argument("output", type=Path) + p.add_argument("--cxx", default="c++") + args = p.parse_args() + source = args.compiler.resolve() + head = subprocess.check_output(["git", "-C", str(source), "rev-parse", "HEAD"], text=True).strip() + if head != COMMIT: + p.error(f"compiler HEAD must be {COMMIT}") + directory = source / "plugins/H13" + # Compare all encoder inputs with the licensed public commit. The bypass is + # confined to a temporary translation unit; the checkout stays unmodified. + for item in directory.iterdir(): + if item.suffix in (".cpp", ".h", ".inc"): + original = subprocess.check_output(["git", "-C", str(source), "show", f"HEAD:plugins/H13/{item.name}"]) + if item.read_bytes() != original: + p.error(f"modified encoder input: {item.name}") + program = (directory / "H13Program.cpp").read_text() + if program.count(GUARD) != 1: + p.error("expected refusal changed") + args.output = args.output.resolve() + args.output.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="ane-hang-fixture-") as temporary: + tmp = Path(temporary) + (tmp / "fixture.cpp").write_text(DRIVER) + def build(unit): + subprocess.run([args.cxx, "-std=c++17", "-O2", "-I", str(directory), + str(unit), str(directory / "H13ANEC.cpp"), str(tmp / "fixture.cpp"), + "-o", str(tmp / "fixture")], check=True) + build(directory / "H13Program.cpp") + subprocess.run([str(tmp / "fixture")], check=True) # normal compiler must refuse + (tmp / "H13Program.cpp").write_text(program.replace(GUARD, "chunks.size() != 1")) + build(tmp / "H13Program.cpp") + subprocess.run([str(tmp / "fixture"), str(args.output)], check=True) + receipt = {"compiler": COMMIT, "shape": "1x1 s2 128->256 56->28", "bias": False, + "sha256": hashlib.sha256(args.output.read_bytes()).hexdigest(), + "bytes": args.output.stat().st_size} + args.output.with_suffix(".json").write_text(json.dumps(receipt, indent=2) + "\n") + print(json.dumps(receipt)) + + +if __name__ == "__main__": + main() diff --git a/tools/testing/selftests/drivers/ane/recovery.py b/tools/testing/selftests/drivers/ane/recovery.py new file mode 100755 index 0000000000000..e8e4b184ee13a --- /dev/null +++ b/tools/testing/selftests/drivers/ane/recovery.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Destructive H13 hang/recovery test. Run only with exclusive device ownership.""" +import argparse +import ctypes +import ctypes.util +import errno +import hashlib +import json +from pathlib import Path +import signal +import struct +import sys +import time + + +def bind(library, name, result, *arguments): + fn = getattr(library, name) + fn.restype, fn.argtypes = result, arguments + return fn + + +def surface(values): + result = bytearray(16384) + for index, value in enumerate(values): + struct.pack_into(" Date: Sat, 31 Oct 2026 18:13:17 +1100 Subject: [PATCH 1528/1579] accel/ane: exercise the recovery PM transaction with KUnit devices Share the runtime-PM cycle independently of task-manager MMIO and test it against real runtime-PM supplier links. Cover repeated and single-domain cycles, consumer and supplier suspend errors, a missing off acknowledgment and resume failure, including restoration of the caller reference. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 7 + drivers/accel/Makefile | 1 + drivers/accel/ane/Kconfig | 12 + drivers/accel/ane/Makefile | 1 + drivers/accel/ane/ane_pm.h | 51 ++++ drivers/accel/ane/ane_pm_test.c | 249 ++++++++++++++++++ drivers/accel/ane/ane_tm.c | 68 ++--- .../selftests/drivers/ane/pm.kunitconfig | 5 + 8 files changed, 348 insertions(+), 46 deletions(-) create mode 100644 drivers/accel/ane/ane_pm.h create mode 100644 drivers/accel/ane/ane_pm_test.c create mode 100644 tools/testing/selftests/drivers/ane/pm.kunitconfig diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index ce19cff094c93..f8d9ebe12897d 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -280,6 +280,13 @@ uncertain. The ``reset`` sysfs write retries recovery. Module reload cannot recover a device while that pin is held. Direct PMGR SET writes and speculative writes to undocumented task-error registers are not part of this reset path. +The ``ane-pm-recovery`` KUnit suite exercises the shared PM transaction against +real runtime-PM device links, including repeated cycles, single-domain devices, +consumer/supplier suspend errors, a missing power-off acknowledgment and a +failed resume. It checks that every exit restores the caller's PM reference. +The minimal config is +``tools/testing/selftests/drivers/ane/pm.kunitconfig``. + Destructive recovery selftest ---------------------------- diff --git a/drivers/accel/Makefile b/drivers/accel/Makefile index f14fa5f5942f5..2aa8f41d91dfc 100644 --- a/drivers/accel/Makefile +++ b/drivers/accel/Makefile @@ -3,6 +3,7 @@ obj-$(CONFIG_DRM_ACCEL_AMDXDNA) += amdxdna/ obj-$(CONFIG_DRM_ACCEL_ANE) += ane/ obj-$(CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST) += ane/ +obj-$(CONFIG_DRM_ACCEL_ANE_PM_KUNIT_TEST) += ane/ obj-$(CONFIG_DRM_ACCEL_ARM_ETHOSU) += ethosu/ obj-$(CONFIG_DRM_ACCEL_HABANALABS) += habanalabs/ obj-$(CONFIG_DRM_ACCEL_IVPU) += ivpu/ diff --git a/drivers/accel/ane/Kconfig b/drivers/accel/ane/Kconfig index 8df67a4a6c86a..ae7d593b1b9ad 100644 --- a/drivers/accel/ane/Kconfig +++ b/drivers/accel/ane/Kconfig @@ -55,3 +55,15 @@ config DRM_ACCEL_ANE_FW_KUNIT_TEST These tests can run on architectures other than Apple arm64. If unsure, say N. + +config DRM_ACCEL_ANE_PM_KUNIT_TEST + tristate "KUnit tests for ANE runtime-PM recovery" if !KUNIT_ALL_TESTS + depends on KUNIT && PM + default KUNIT_ALL_TESTS + help + Exercise supplier suspension, power-off verification failure and + runtime resume errors against the runtime-PM and device-link core. + No ANE hardware is required. These tests can run independently of + the ANE driver on non-Apple platforms. + + If unsure, say N. diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile index 79418f74c6192..8e6ae84fdd4cb 100644 --- a/drivers/accel/ane/Makefile +++ b/drivers/accel/ane/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_DRM_ACCEL_ANE) += ane.o ane_t6021.o +obj-$(CONFIG_DRM_ACCEL_ANE_PM_KUNIT_TEST) += ane_pm_test.o obj-$(CONFIG_DRM_ACCEL_ANE_H16_VALIDATION) += ane_h16_hwx.o obj-$(CONFIG_DRM_ACCEL_ANE_KUNIT_TEST) += ane_drv_test.o obj-$(CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST) += ane_fw_platform_test.o \ diff --git a/drivers/accel/ane/ane_pm.h b/drivers/accel/ane/ane_pm.h new file mode 100644 index 0000000000000..ccc08c88057a4 --- /dev/null +++ b/drivers/accel/ane/ane_pm.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_PM_H__ +#define __ANE_PM_H__ + +#include +#include + +/* + * The caller serializes PM entries and holds exactly one consumer reference. + * Its suspend callback must permit recovery and its resume callback must defer + * engine access until the caller verifies power-on. verify_off may only read + * always-accessible provider registers, never the powered-down engine. + * + * Kept independent of the H13 task manager: a firmware client can use the same + * PM transaction after stopping its RX worker and fencing submissions. + */ +static inline int ane_pm_cycle(struct device *dev, struct device **domains, + int count, int (*verify_off)(void *), void *ctx) +{ + int err, resume_err; + + if (count > 1) { + for (int i = 0; i < count; i++) + dev_pm_genpd_synced_poweroff(domains[i]); + } else { + dev_pm_genpd_synced_poweroff(dev); + } + + /* Normal runtime PM releases the consumer's supplier references. */ + err = pm_runtime_put_sync_suspend(dev); + if (err < 0) + goto resume; + + /* Finish asynchronous supplier idle work, then suspend leaves first. + * Runtime PM stays enabled so genpd can also suspend their parents. + */ + for (int i = count - 1; i >= 0 && count > 1; i--) { + pm_runtime_barrier(domains[i]); + err = pm_runtime_suspend(domains[i]); + if (err < 0) + goto resume; + } + err = verify_off(ctx); + +resume: + /* Restore the caller's reference even if resume itself fails. */ + resume_err = pm_runtime_get_sync(dev); + return err < 0 ? err : (resume_err < 0 ? resume_err : 0); +} + +#endif /* __ANE_PM_H__ */ diff --git a/drivers/accel/ane/ane_pm_test.c b/drivers/accel/ane/ane_pm_test.c new file mode 100644 index 0000000000000..5c372a2a73011 --- /dev/null +++ b/drivers/accel/ane/ane_pm_test.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Exercise recovery against the real runtime-PM and device-link core. */ +#include +#include +#include + +#include "ane_pm.h" + +struct ane_pm_test_dev { + struct device dev; + int suspend_error; + int resume_error; + unsigned int suspends; + unsigned int resumes; +}; + +struct ane_pm_test_ctx { + struct kunit *test; + struct device *consumer; + struct device *domains[2]; + struct device_link *links[2]; + unsigned int verified; + int verify_error; +}; + +static int ane_pm_test_suspend(struct device *dev) +{ + struct ane_pm_test_dev *d = container_of(dev, struct ane_pm_test_dev, dev); + + d->suspends++; + return d->suspend_error; +} + +static int ane_pm_test_resume(struct device *dev) +{ + struct ane_pm_test_dev *d = container_of(dev, struct ane_pm_test_dev, dev); + + d->resumes++; + return d->resume_error; +} + +static const struct dev_pm_ops ane_pm_test_ops = { + .runtime_suspend = ane_pm_test_suspend, + .runtime_resume = ane_pm_test_resume, +}; + +static const struct device_type ane_pm_test_type = { + .pm = &ane_pm_test_ops, +}; + +static void ane_pm_test_release(struct device *dev) +{ + kfree(container_of(dev, struct ane_pm_test_dev, dev)); +} + +static void ane_pm_test_unregister(void *data) +{ + struct device *dev = data; + + pm_runtime_barrier(dev); + pm_runtime_disable(dev); + /* Also releases supplier votes after an injected resume failure. */ + pm_runtime_set_suspended(dev); + device_unregister(dev); +} + +static struct device *ane_pm_test_register(struct kunit *test, const char *name) +{ + struct ane_pm_test_dev *d; + int ret; + + d = kzalloc_obj(*d); + if (!d) + return NULL; + device_initialize(&d->dev); + d->dev.release = ane_pm_test_release; + d->dev.type = &ane_pm_test_type; + ret = dev_set_name(&d->dev, "%s", name); + if (!ret) + ret = device_add(&d->dev); + if (ret) { + put_device(&d->dev); + return NULL; + } + pm_runtime_enable(&d->dev); + ret = kunit_add_action_or_reset(test, ane_pm_test_unregister, &d->dev); + return ret ? NULL : &d->dev; +} + +static void ane_pm_test_unlink(void *data) +{ + device_link_del(data); +} + +static int ane_pm_test_verify(void *data) +{ + struct ane_pm_test_ctx *ctx = data; + + ctx->verified++; + KUNIT_EXPECT_TRUE(ctx->test, pm_runtime_suspended(ctx->consumer)); + KUNIT_EXPECT_EQ(ctx->test, atomic_read(&ctx->consumer->power.usage_count), 0); + for (int i = 0; i < ARRAY_SIZE(ctx->domains); i++) { + if (!ctx->domains[i]) + continue; + KUNIT_EXPECT_TRUE(ctx->test, pm_runtime_suspended(ctx->domains[i])); + KUNIT_EXPECT_EQ(ctx->test, atomic_read(&ctx->domains[i]->power.usage_count), 0); + } + return ctx->verify_error; +} + +static int ane_pm_test_setup(struct kunit *test, struct ane_pm_test_ctx *ctx, + bool multiple) +{ + static const char * const names[] = { "ane-pm-domain0", "ane-pm-domain1" }; + + ctx->test = test; + ctx->consumer = ane_pm_test_register(test, "ane-pm-consumer"); + if (!ctx->consumer) + return -ENOMEM; + if (multiple) { + for (int i = 0; i < ARRAY_SIZE(ctx->domains); i++) { + ctx->domains[i] = ane_pm_test_register(test, names[i]); + if (!ctx->domains[i]) + return -ENOMEM; + ctx->links[i] = device_link_add(ctx->consumer, ctx->domains[i], + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!ctx->links[i]) + return -ENOMEM; + if (kunit_add_action_or_reset(test, ane_pm_test_unlink, + ctx->links[i])) + return -ENOMEM; + } + } + return pm_runtime_resume_and_get(ctx->consumer); +} + +static void ane_pm_test_finish(struct kunit *test, struct ane_pm_test_ctx *ctx) +{ + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->consumer->power.usage_count), 1); + KUNIT_EXPECT_GE(test, pm_runtime_put_sync_suspend(ctx->consumer), 0); +} + +static void ane_pm_cycle_domains_test(struct kunit *test) +{ + struct ane_pm_test_ctx ctx = {}; + + KUNIT_ASSERT_GE(test, ane_pm_test_setup(test, &ctx, true), 0); + for (int repeat = 0; repeat < 3; repeat++) { + KUNIT_EXPECT_EQ(test, ane_pm_cycle(ctx.consumer, ctx.domains, 2, + ane_pm_test_verify, &ctx), 0); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(ctx.consumer)); + for (int i = 0; i < ARRAY_SIZE(ctx.domains); i++) { + KUNIT_EXPECT_TRUE(test, pm_runtime_active(ctx.domains[i])); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx.domains[i]->power.usage_count), 1); + } + } + KUNIT_EXPECT_EQ(test, ctx.verified, 3U); + ane_pm_test_finish(test, &ctx); +} + +static void ane_pm_cycle_single_test(struct kunit *test) +{ + struct ane_pm_test_ctx ctx = {}; + + KUNIT_ASSERT_GE(test, ane_pm_test_setup(test, &ctx, false), 0); + KUNIT_EXPECT_EQ(test, ane_pm_cycle(ctx.consumer, NULL, 1, + ane_pm_test_verify, &ctx), 0); + KUNIT_EXPECT_EQ(test, ctx.verified, 1U); + ane_pm_test_finish(test, &ctx); +} + +static void ane_pm_cycle_suspend_error_test(struct kunit *test) +{ + struct ane_pm_test_ctx ctx = {}; + struct ane_pm_test_dev *d; + + KUNIT_ASSERT_GE(test, ane_pm_test_setup(test, &ctx, true), 0); + d = container_of(ctx.consumer, struct ane_pm_test_dev, dev); + d->suspend_error = -EBUSY; + KUNIT_EXPECT_EQ(test, ane_pm_cycle(ctx.consumer, ctx.domains, 2, + ane_pm_test_verify, &ctx), -EBUSY); + KUNIT_EXPECT_EQ(test, ctx.verified, 0U); + d->suspend_error = 0; + ane_pm_test_finish(test, &ctx); +} + +static void ane_pm_cycle_supplier_error_test(struct kunit *test) +{ + struct ane_pm_test_ctx ctx = {}; + struct ane_pm_test_dev *d; + + KUNIT_ASSERT_GE(test, ane_pm_test_setup(test, &ctx, true), 0); + d = container_of(ctx.domains[1], struct ane_pm_test_dev, dev); + d->suspend_error = -EBUSY; + KUNIT_EXPECT_EQ(test, ane_pm_cycle(ctx.consumer, ctx.domains, 2, + ane_pm_test_verify, &ctx), -EBUSY); + KUNIT_EXPECT_EQ(test, ctx.verified, 0U); + d->suspend_error = 0; + ane_pm_test_finish(test, &ctx); +} + +static void ane_pm_cycle_verify_error_test(struct kunit *test) +{ + struct ane_pm_test_ctx ctx = { .verify_error = -ETIMEDOUT }; + + KUNIT_ASSERT_GE(test, ane_pm_test_setup(test, &ctx, true), 0); + KUNIT_EXPECT_EQ(test, ane_pm_cycle(ctx.consumer, ctx.domains, 2, + ane_pm_test_verify, &ctx), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, ctx.verified, 1U); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(ctx.consumer)); + ane_pm_test_finish(test, &ctx); +} + +static void ane_pm_cycle_resume_error_test(struct kunit *test) +{ + struct ane_pm_test_ctx ctx = {}; + struct ane_pm_test_dev *d; + + KUNIT_ASSERT_GE(test, ane_pm_test_setup(test, &ctx, false), 0); + d = container_of(ctx.consumer, struct ane_pm_test_dev, dev); + d->resume_error = -EIO; + KUNIT_EXPECT_EQ(test, ane_pm_cycle(ctx.consumer, NULL, 1, + ane_pm_test_verify, &ctx), -EIO); + KUNIT_EXPECT_EQ(test, ctx.verified, 1U); + /* get_sync must leave exactly the reference the caller will put. */ + KUNIT_EXPECT_EQ(test, atomic_read(&ctx.consumer->power.usage_count), 1); + pm_runtime_put_noidle(ctx.consumer); +} + +static struct kunit_case ane_pm_test_cases[] = { + KUNIT_CASE(ane_pm_cycle_domains_test), + KUNIT_CASE(ane_pm_cycle_single_test), + KUNIT_CASE(ane_pm_cycle_suspend_error_test), + KUNIT_CASE(ane_pm_cycle_supplier_error_test), + KUNIT_CASE(ane_pm_cycle_verify_error_test), + KUNIT_CASE(ane_pm_cycle_resume_error_test), + {} +}; + +static struct kunit_suite ane_pm_test_suite = { + .name = "ane-pm-recovery", + .test_cases = ane_pm_test_cases, +}; + +kunit_test_suite(ane_pm_test_suite); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("ANE recovery runtime-PM transaction tests"); diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 6967a0630613f..29c66b346ea86 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -4,9 +4,9 @@ #include #include #include -#include #include #include "ane_tm.h" +#include "ane_pm.h" #define ANE_TQ_COUNT 8 static const int TQ_PRTY_TABLE[ANE_TQ_COUNT] = { 0x1, 0x2, 0x3, 0x4, @@ -415,61 +415,37 @@ static int ane_ps_verify_on(struct ane_device *ane) return err; } -static int ane_pd_cycle(struct ane_device *ane) +static int ane_verify_off(void *ctx) { - int err, resume_err; + struct ane_device *ane = ctx; u32 act; + int err; - lockdep_assert_held(&ane->pm_lock); - ane->recovering = true; - err = ane_perf_suspend(ane); - if (err) { - ane->recovering = false; - return err; - } - if (ane->pd_count > 1) { - for (int i = 0; i < ane->pd_count; i++) - dev_pm_genpd_synced_poweroff(ane->pd_dev[i]); - } else { - dev_pm_genpd_synced_poweroff(ane->dev); - } - - /* Drop exactly the reference owned by the calling ioctl/close/reset. - * pm_lock keeps another driver entry from resuming us mid-cycle. - */ - err = pm_runtime_put_sync_suspend(ane->dev); - if (err < 0) - goto resume; - - /* Supplier idle requests are asynchronous. Finish them, then suspend - * leaves before parents using ordinary runtime PM (still enabled). - * genpd owns all PMGR writes and the parent-domain ordering. - */ - for (int i = ane->pd_count - 1; i >= 0 && ane->pd_count > 1; i--) { - pm_runtime_barrier(ane->pd_dev[i]); - err = pm_runtime_suspend(ane->pd_dev[i]); - if (err < 0) - goto resume; - } - - /* A successful runtime suspend alone is not proof of power-off: - * genpd may retain a shared or busy parent. Only PMGR is safe to read - * here; do not access the engine aperture until it is powered again. + /* genpd may retain a shared or busy parent even after suspend succeeds. + * Only PMGR is safe to read until the engine has power again. */ - if (!ane->ps) { - err = -EOPNOTSUPP; - goto resume; - } + if (!ane->ps) + return -EOPNOTSUPP; err = read_poll_timeout(ane_ps_act, act, act == 0, 100, 100000, false, ane); dev_info(ane->dev, "recovery: engine power-off actual=%#x rc=%d\n", act, err); + return err; +} -resume: - /* get_sync restores the caller's usage reference even on failure. */ - resume_err = pm_runtime_get_sync(ane->dev); +static int ane_pd_cycle(struct ane_device *ane) +{ + int err; + + lockdep_assert_held(&ane->pm_lock); + ane->recovering = true; + /* Release the performance request before the domains go down. */ + err = ane_perf_suspend(ane); + if (!err) + err = ane_pm_cycle(ane->dev, ane->pd_dev, ane->pd_count, + ane_verify_off, ane); ane->recovering = false; - return err < 0 ? err : (resume_err < 0 ? resume_err : 0); + return err; } int ane_tm_recover(struct ane_device *ane) diff --git a/tools/testing/selftests/drivers/ane/pm.kunitconfig b/tools/testing/selftests/drivers/ane/pm.kunitconfig new file mode 100644 index 0000000000000..f98c6bc08df1f --- /dev/null +++ b/tools/testing/selftests/drivers/ane/pm.kunitconfig @@ -0,0 +1,5 @@ +CONFIG_KUNIT=y +CONFIG_PM=y +CONFIG_DRM=y +CONFIG_DRM_ACCEL=y +CONFIG_DRM_ACCEL_ANE_PM_KUNIT_TEST=y From fdf0da1f1ce8b95a6f44d6bd26a806855f6d8492 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 18:28:33 +1100 Subject: [PATCH 1529/1579] selftests/ane: load recovery fixtures through ABI 1 ioctls The packaged tools contain a static libane, so a ctypes test cannot rely on a system shared library. Use the documented ABI 1 buffer and submit ioctls directly. Keep both fds, GEM handles and mappings alive across all three resets, pin both public fixture hashes, and provide device-free container validation before the destructive run. Also depend on DRM_ACCEL for traversal of the PM KUnit test directory. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 6 +- drivers/accel/ane/Kconfig | 2 +- .../testing/selftests/drivers/ane/recovery.py | 198 +++++++++++++----- 3 files changed, 147 insertions(+), 59 deletions(-) diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index f8d9ebe12897d..7ea2ad24511b1 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -292,7 +292,7 @@ Destructive recovery selftest ``tools/testing/selftests/drivers/ane/recovery.py`` is an explicitly invoked hardware test, excluded from the default device-free selftests. It requires -exclusive ownership of an M1-family ANE, libane, the packaged H13 add fixture, +exclusive ownership of an M1-family ANE, Python 3, the packaged H13 add fixture, and a deliberately invalid convolution program. Run it through the hardware controller with a watchdog and kernel-log capture:: @@ -321,7 +321,9 @@ stride-2 1x1 128-to-256 convolution at 56-to-28 spatial size, the refused family reported to hang the M1 Pro. The compiler checkout is unchanged. No Apple binary or private program capture is used. The generator writes a hash receipt; the hardware test must fail if this particular instance does not actually -produce a timeout. +produce a timeout. The loader pins both fixture hashes and uses ABI 1 ioctls directly, so no +shared libane library is needed. ``--check-fixtures`` validates the two containers without +opening a device. H14 recovery lifecycle ---------------------- diff --git a/drivers/accel/ane/Kconfig b/drivers/accel/ane/Kconfig index ae7d593b1b9ad..2e804e63199ce 100644 --- a/drivers/accel/ane/Kconfig +++ b/drivers/accel/ane/Kconfig @@ -58,7 +58,7 @@ config DRM_ACCEL_ANE_FW_KUNIT_TEST config DRM_ACCEL_ANE_PM_KUNIT_TEST tristate "KUnit tests for ANE runtime-PM recovery" if !KUNIT_ALL_TESTS - depends on KUNIT && PM + depends on DRM_ACCEL && KUNIT && PM default KUNIT_ALL_TESTS help Exercise supplier suspension, power-off verification failure and diff --git a/tools/testing/selftests/drivers/ane/recovery.py b/tools/testing/selftests/drivers/ane/recovery.py index e8e4b184ee13a..2464e5e18bf05 100755 --- a/tools/testing/selftests/drivers/ane/recovery.py +++ b/tools/testing/selftests/drivers/ane/recovery.py @@ -2,26 +2,124 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT """Destructive H13 hang/recovery test. Run only with exclusive device ownership.""" import argparse -import ctypes -import ctypes.util import errno +import fcntl import hashlib import json +import mmap +import os from pathlib import Path import signal import struct import sys import time +# include/uapi/drm/ane_accel.h, Linux arm64's generic ioctl encoding. +CAPS = struct.Struct("<8I") +BO_INIT = struct.Struct(" self.tsk_size or + self.size != ((self.tsk_size + 15) & ~15) + kernel or + len(self.data) - 4096 > self.size or self.size > 16 * 1024 * 1024 or + self.size > self.tiles[0] * TILE or self.tsk_size >= self.tiles[0] * TILE or + not self.tiles[4] or not self.tiles[5] or + bool(self.tiles[6]) != (self.inputs == 2) or + any(n for i, n in enumerate(self.tiles) if i not in (0, 4, 5, 6)) or + sum(self.tiles) * TILE > 64 * 1024 * 1024): + raise ValueError("unexpected canonical H13 ANEC layout") + self.payload = self.data[4096:].ljust(self.size, b"\0") + + +class Program: + def __init__(self, fixture, node): + self.fd = os.open(node, os.O_RDWR | os.O_CLOEXEC) + self.buffers = {} + self.bootstrap = None + self.fixture = fixture + try: + caps = bytearray(CAPS.size) + fcntl.ioctl(self.fd, request(0, CAPS), caps) + _, size, abi, chip, *_ = CAPS.unpack(caps) + if size != CAPS.size or abi != 1 or chip != 13: + raise RuntimeError("H13 ABI 1 required") + for channel, tiles in enumerate(fixture.tiles): + if tiles: + self.buffers[channel] = self.allocate(tiles * TILE) + self.bootstrap = self.allocate((fixture.td_size + TILE - 1) & ~(TILE - 1)) + self.buffers[0][1][:fixture.size] = fixture.payload + self.bootstrap[1][:fixture.td_size] = fixture.payload[:fixture.td_size] + header, = struct.unpack_from(" Date: Sat, 31 Oct 2026 18:29:43 +1100 Subject: [PATCH 1530/1579] selftests/ane: use nonzero activations for the injected convolution Exercise the malformed convolution with fp16 one inputs so the test requests MAC work rather than relying on an all-zero activation path. The input and weight bytes are deterministic; only the add fixture is used as a numerical oracle. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/testing/selftests/drivers/ane/recovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/ane/recovery.py b/tools/testing/selftests/drivers/ane/recovery.py index 2464e5e18bf05..990fa3006b148 100755 --- a/tools/testing/selftests/drivers/ane/recovery.py +++ b/tools/testing/selftests/drivers/ane/recovery.py @@ -159,7 +159,7 @@ def main(): expected = surface([a + b for a, b in zip(*inputs)]) for channel, values in zip((5, 6), inputs): good.buffers[channel][1][:] = surface(values) - bad.buffers[5][1][:] = bytes(len(bad.buffers[5][1])) + bad.buffers[5][1][:] = b"\x00\x3c" * (len(bad.buffers[5][1]) // 2) def check_good(): good.execute() if good.buffers[4][1][:] != expected: From 02d9d1b3bd9592991b4b25ea1bcdf7014f2ff4e6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 19:01:44 +1100 Subject: [PATCH 1531/1579] accel/ane: verify the shared SYS_CPU domain before restarting The SET window covers the engine islands but not their SYS_CPU parent. DART suppliers suspend asynchronously after the ANE consumer drops its references. Require every attached genpd, including SYS_CPU, to report off before resuming the consumer; otherwise that resume can race the DART idle work and keep the task-manager register file powered. Use only the genpd provider API and always-accessible PMGR reads. A shared domain that cannot turn off leaves recovery failed and preserves the DMA safety pin. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 5 ++++- drivers/accel/ane/ane_tm.c | 26 ++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index 7ea2ad24511b1..034485f032263 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -265,7 +265,10 @@ partial. The task's GEM lookup references are released when the ioctl returns. Recovery holds the engine and PM-entry locks, drops the current consumer runtime-PM reference, synchronously suspends its power-domain suppliers, and -checks the PMGR ACTUAL fields for power-off. It then restores the consumer +checks both the PMGR ACTUAL fields and every attached genpd for power-off. +The SET window omits the shared SYS_CPU parent, whose DART consumers can +suspend asynchronously; it must also turn off before the consumer resumes. +Recovery then restores the consumer reference, checks power-on before engine MMIO, and runs the probe task-manager enable sequence. BO handles and mappings remain valid for resubmission. The system-sleep force helpers are unsuitable here: a forced, in-use supplier diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 29c66b346ea86..98082a80def47 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -415,21 +415,39 @@ static int ane_ps_verify_on(struct ane_device *ane) return err; } +static bool ane_partitions_off(struct ane_device *ane) +{ + if (ane_ps_act(ane)) + return false; + + /* The SET window does not include the shared SYS_CPU parent. Its + * DART consumers suspend asynchronously after our references drop. + * Wait for every attached domain as well, so resuming the consumer + * cannot race that work and leave the TM register file powered. + */ + if (ane->pd_count == 1) + return !dev_pm_genpd_is_on(ane->dev); + for (int i = 0; i < ane->pd_count; i++) + if (dev_pm_genpd_is_on(ane->pd_dev[i])) + return false; + return true; +} + static int ane_verify_off(void *ctx) { struct ane_device *ane = ctx; - u32 act; + bool off; int err; /* genpd may retain a shared or busy parent even after suspend succeeds. - * Only PMGR is safe to read until the engine has power again. + * Only PMGR and provider state are safe to read while the engine is off. */ if (!ane->ps) return -EOPNOTSUPP; - err = read_poll_timeout(ane_ps_act, act, act == 0, + err = read_poll_timeout(ane_partitions_off, off, off, 100, 100000, false, ane); dev_info(ane->dev, "recovery: engine power-off actual=%#x rc=%d\n", - act, err); + ane_ps_act(ane), err); return err; } From 0672143493692a6acd40a3a4e538ce10efba236c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 19:09:30 +1100 Subject: [PATCH 1532/1579] selftests/ane: require fresh output from every recovered submission Change the add inputs for each call and poison its 64 logical output lanes before submitting. Zero the unused padding, which need not be written by the engine. Comparing the complete tile with the oracle now rejects a stale completion even when the previous call was correct. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 3 ++- .../testing/selftests/drivers/ane/recovery.py | 20 ++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index 034485f032263..c644b0286078d 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -304,7 +304,8 @@ controller with a watchdog and kernel-log capture:: The test keeps both clients open, submits the invalid program three times, requires a bounded ``ETIMEDOUT`` and ``wedged=0`` each time, and runs twenty bit-exact add jobs on the surviving client's original fd and mappings after -each reset. Its oracle uses exact fp16 quarter values and checks the complete +each reset. Every call changes its inputs and poisons the valid output lanes, +so stale output cannot pass. Its oracle uses exact fp16 quarter values and checks the complete output, including padding. Closing both clients must return the module reference count to its initial value; the boot identity must remain unchanged. An operator should also verify three power-off acknowledgments in the kernel diff --git a/tools/testing/selftests/drivers/ane/recovery.py b/tools/testing/selftests/drivers/ane/recovery.py index 990fa3006b148..9d76725e0bb6e 100755 --- a/tools/testing/selftests/drivers/ane/recovery.py +++ b/tools/testing/selftests/drivers/ane/recovery.py @@ -155,15 +155,25 @@ def main(): try: good = Program(good_fixture, args.device) bad = Program(bad_fixture, args.device) - inputs = [[(3 + (i * multiplier) % 11) / 4 for i in range(64)] for multiplier in (1, 3)] - expected = surface([a + b for a, b in zip(*inputs)]) - for channel, values in zip((5, 6), inputs): - good.buffers[channel][1][:] = surface(values) bad.buffers[5][1][:] = b"\x00\x3c" * (len(bad.buffers[5][1]) // 2) + calls = 0 def check_good(): + nonlocal calls + inputs = [[(3 + (i * multiplier + calls) % 11) / 4 for i in range(64)] + for multiplier in (1, 3)] + expected = surface([a + b for a, b in zip(*inputs)]) + for channel, values in zip((5, 6), inputs): + good.buffers[channel][1][:] = surface(values) + # Only the first fp16 of each plane is a logical output. Poison + # those lanes and zero padding so stale success cannot pass. + output = good.buffers[4][1] + output[:] = bytes(TILE) + for lane in range(64): + struct.pack_into(" Date: Sun, 1 Nov 2026 00:50:02 +1100 Subject: [PATCH 1533/1579] accel/ane: drain the retained task manager on T600x recovery Recovery refused to restart whenever TQ_EN bit 13 was set after the partition power cycle, reading it as "task manager still halted". On the M1 Pro that bit is set on every healthy runtime resume as well: TQ_EN reads 0x3000 and TM_ERROR1 0x22222222 each time the engine comes back from autosuspend. So the check failed every recovery and left the engine wedged until reboot. Drop it. The tm/tq register file on T600x also keeps its state through the acknowledged power-off, and through PMGR reset pulses on every ANE domain (ane_sys, sys_cpu, set0, base, set1-4), so a power cycle alone does not clear what a failed task latched. Bring back the retained-state drain for SoCs whose descriptor says the file is retained (T6000/T6001), based on omarchy-ane's ane_tm_drain_retained by Joshua Warren (MIT). Before it was removed, the drain recovered every "-5, finish lines=1" task error seen on the M1 Pro. A wedge the drain cannot clear still fails the idle check and stays wedged, with no reboot loop. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane_drv.c | 1 + drivers/accel/ane/ane_tm.c | 106 +++++++++++++++++++++++++++++++++--- 2 files changed, 100 insertions(+), 7 deletions(-) diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 7558fd7eaea6b..120d900973cd2 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -1179,6 +1179,7 @@ static int ane_platform_probe(struct platform_device *pdev) drm->dev_private = ane; atomic_set(&ane->wedged, 0); + ane->tm_retention = soc->tm_retention; /* Producer-side stats: preallocate the ring at probe; the files * and the hot-path branch key off stats_slots. diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 98082a80def47..63ce4037d0ca7 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -466,6 +466,97 @@ static int ane_pd_cycle(struct ane_device *ane) return err; } +/* + * Drain the task-manager state a timed-out task leaves behind (from + * omarchy-ane ane_tm_drain_retained by Joshua Warren, MIT) when the + * tm/tq file survives the power cycle in retention (T6001; descriptor + * tm_retention). T8103 never runs this: its cycle is a full POR of the + * file, and the re-arm below starts it clean. + * + * Two things must go, or the retained TM never reads idle again: + * - pending events on both IRQ lines. Each event pops by reading + * INFO/UNK1/TMST/UNK2, exactly as ane_tm_collect_events pops them; + * a stale backlog (or one over the 64-entry collect cap) would make + * the next submit's completion poll miss or fail. + * - the latched queue entry of the dead task. TQ_STATUS/NID1 get the + * same clear the normal completion path gives a finished queue. + */ +static void ane_tm_drain_retained(struct ane_device *ane) +{ + int line, qid; + + /* Stop the TM first: the cycle leaves the enable|halt latch set + * (TQ_EN reads 0x3000). Clear both bits so the + * re-arm below starts the TM from a defined stopped state. + */ + ane_rec_writel(ane, "TM_TQ_EN stop tm+0x0c", + ane->engine + ANE_TM_BASE + TM_TQ_EN, + tm_read32(ane, TM_TQ_EN) & ~0x3000U); + + for (line = 0; line < 2; line++) { + u32 count = tm_read32(ane, TM_IRQ_EVTC(line)); + u32 n; + + if (count > 64) + count = 64; + for (n = 0; n < count; n++) { + tm_read32(ane, TM_IRQ_INFO(line)); + tm_read32(ane, TM_IRQ_UNK1(line)); + tm_read32(ane, TM_IRQ_TMST(line)); + tm_read32(ane, TM_IRQ_UNK2(line)); + } + } + tm_write32(ane, TM_IRQ_ACK, tm_read32(ane, TM_IRQ_ACK) | 3); + + /* The dead task leaves per-queue error codes latched here (seen + * 0x22222222 / 0x2222: four-bit code 2 per TQ slot). While any + * latch is set the TM stays halted and TM_STATUS reads 0. Try the + * W1C convention first (write the read value back), fall back to + * plain zero-clear, and log what this silicon answered to. + */ + { + static const u16 err_reg[3] = { TM_ERROR1, TM_ERROR2, + TM_ERROR3 }; + char reg[24]; + int ei; + + for (ei = 0; ei < 3; ei++) { + void __iomem *addr = + ane->engine + ANE_TM_BASE + err_reg[ei]; + u32 val = readl(addr); + + if (!val) + continue; + writel(val, addr); + if (readl(addr)) + writel(0x0, addr); + snprintf(reg, sizeof(reg), "TM_ERROR%d clear tm+%#x", + ei + 1, err_reg[ei]); + ane_rec_read32(ane, reg, addr); + } + } + + for (qid = 0; qid < ANE_TQ_COUNT; qid++) { + u32 nid = tq_read32(ane, TQ_NID1(qid)); + + tq_write32(ane, TQ_STATUS(qid), 0x0); + if (nid & 1) + tq_write32(ane, TQ_NID1(qid), nid & ~1U); + } + + /* Log the retained state: what the dead task left in the + * committed counter and the error latches. + */ + ane_rec_read32(ane, "TM_COMMITTED tm+0x44 (drain)", + ane->engine + ANE_TM_BASE + TM_COMMITTED); + ane_rec_read32(ane, "TM_ERROR1 tm+0x58 (drain)", + ane->engine + ANE_TM_BASE + TM_ERROR1); + ane_rec_read32(ane, "TM_ERROR2 tm+0x5c (drain)", + ane->engine + ANE_TM_BASE + TM_ERROR2); + ane_rec_read32(ane, "TM_ERROR3 tm+0x60 (drain)", + ane->engine + ANE_TM_BASE + TM_ERROR3); +} + int ane_tm_recover(struct ane_device *ane) { u32 status; @@ -492,14 +583,15 @@ int ane_tm_recover(struct ane_device *ane) return err; } - /* A full, acknowledged power-off replaces the speculative retained-TM - * drain. Never clear undocumented error registers or restart a queue - * which still contains the old halted task. + /* On T600x the tm/tq file survives the acknowledged power-off in + * retention (TQ_EN reads 0x3000 after every resume, healthy or not, + * and PMGR reset pulses on the ANE domains do not clear it). Drain + * the retained state there; this is what recovers a failed task + * ("-5, finish lines=1"). Anything the drain cannot clear fails the + * idle check below and stays wedged. */ - if (tm_read32(ane, TM_TQ_EN) & 0x2000) { - dev_err(ane->dev, "recovery: task manager still halted after power-off\n"); - return -EIO; - } + if (ane->tm_retention) + ane_tm_drain_retained(ane); ane_tm_enable(ane, true); From 00a2813449480524a8922028d3fe72265975a643 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 01:58:51 +1100 Subject: [PATCH 1534/1579] media: apple: ave: add per-ABI writers and 6070 lifecycle Describe the command envelope, transport and surface banks independently of the firmware payload offsets. Keep the 9003 builders behind that interface and compare AVC/HEVC Start and Process images with the pre-refactor code. Select lifecycle-only 6070 layouts for T6000 and T8112, including their boot registers, VCBAR encoders, hello, channel discovery, Config register mapping, reply routing and scratch-confirmed Halt. Expose a debugfs lifecycle test without encoding and retain DMA, module and runtime power references after an unconfirmed stop. Start and Process remain refused pending their field ledger. Update the sanitizer host runner for the writer split as well. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/.kunitconfig | 8 + drivers/media/platform/apple/ave/Kconfig | 16 +- drivers/media/platform/apple/ave/Makefile | 3 +- drivers/media/platform/apple/ave/abi.rst | 61 ++ .../media/platform/apple/ave/ave-abi-test.c | 410 +++++++++ drivers/media/platform/apple/ave/ave-abi.c | 210 +++++ drivers/media/platform/apple/ave/ave-cmd.c | 497 ++--------- drivers/media/platform/apple/ave/ave-drv.c | 107 ++- drivers/media/platform/apple/ave/ave-fw.c | 399 +++++++-- drivers/media/platform/apple/ave/ave-ipc.c | 325 ++++--- drivers/media/platform/apple/ave/ave-layout.c | 180 ++-- drivers/media/platform/apple/ave/ave-layout.h | 280 +++--- drivers/media/platform/apple/ave/ave-regs.h | 16 +- drivers/media/platform/apple/ave/ave-sess.c | 94 +- drivers/media/platform/apple/ave/ave-writer.c | 520 +++++++++++ drivers/media/platform/apple/ave/ave.h | 201 +++-- drivers/media/platform/apple/ave/tests/README | 7 +- .../apple/ave/tests/ave-9003-reference.h | 828 ++++++++++++++++++ .../media/platform/apple/ave/tests/boundary.h | 6 + drivers/media/platform/apple/ave/tests/run.py | 19 +- drivers/media/platform/apple/ave/tests/shim.h | 10 +- drivers/media/platform/apple/ave/tests/test.c | 21 +- 22 files changed, 3273 insertions(+), 945 deletions(-) create mode 100644 drivers/media/platform/apple/ave/abi.rst create mode 100644 drivers/media/platform/apple/ave/ave-abi-test.c create mode 100644 drivers/media/platform/apple/ave/ave-abi.c create mode 100644 drivers/media/platform/apple/ave/ave-writer.c create mode 100644 drivers/media/platform/apple/ave/tests/ave-9003-reference.h diff --git a/drivers/media/platform/apple/ave/.kunitconfig b/drivers/media/platform/apple/ave/.kunitconfig index 891a4969c8986..6cf7ed0a8090e 100644 --- a/drivers/media/platform/apple/ave/.kunitconfig +++ b/drivers/media/platform/apple/ave/.kunitconfig @@ -5,3 +5,11 @@ CONFIG_MEDIA_SUPPORT=y CONFIG_MEDIA_SUPPORT_FILTER=n CONFIG_MEDIA_PLATFORM_DRIVERS=y CONFIG_VIDEO_APPLE_AVE_KUNIT_TEST=y +CONFIG_ARCH_APPLE=y +CONFIG_PM=y +CONFIG_PM_GENERIC_DOMAINS=y +CONFIG_IOMMU_SUPPORT=y +CONFIG_IOMMU_DMA=y +CONFIG_VIDEO_DEV=y +CONFIG_VIDEO_APPLE_AVE=y +CONFIG_VIDEO_APPLE_AVE_CMD_KUNIT_TEST=y diff --git a/drivers/media/platform/apple/ave/Kconfig b/drivers/media/platform/apple/ave/Kconfig index 9284ff96874cb..e5318b6e33a00 100644 --- a/drivers/media/platform/apple/ave/Kconfig +++ b/drivers/media/platform/apple/ave/Kconfig @@ -10,9 +10,9 @@ config VIDEO_APPLE_AVE help V4L2 memory-to-memory driver for the AVE H.264/HEVC encoder in Apple T8140 (A18 Pro) SoCs, as in the MacBook Neo. - T6000 (M1 Pro), T6041 (M4 Pro) and T8112 (M2) have probe-only - support; encoding - requires qualified firmware command constants. + T6000 (M1 Pro) and T8112 (M2) support lifecycle-only validation + of firmware 6070.11.1 through debugfs; encoding stays disabled. + T6041 (M4 Pro) has probe-only support. The encoder firmware is loaded by the bootloader; the driver builds every command itself. See Documentation/admin-guide/media/apple-ave.rst. @@ -29,3 +29,13 @@ config VIDEO_APPLE_AVE_KUNIT_TEST encoders, including picture alignment boundaries and the reference 1920x1080 AVC and HEVC sizes. The tests do not need an encoder or any firmware and can run independently of the AVE driver. + +config VIDEO_APPLE_AVE_CMD_KUNIT_TEST + bool "KUnit tests for Apple AVE firmware ABIs" if !KUNIT_ALL_TESTS + depends on KUNIT && VIDEO_APPLE_AVE=y + default KUNIT_ALL_TESTS + help + Compare AVC/HEVC 9003 Start and Process images byte for byte with + the pre-refactor builder. Test 6070 lifecycle envelopes, replies, + split surface arrays, seven-channel discovery and encoding refusal. + These tests run without firmware or encoder hardware. diff --git a/drivers/media/platform/apple/ave/Makefile b/drivers/media/platform/apple/ave/Makefile index 0487aab585666..72080d784f70b 100644 --- a/drivers/media/platform/apple/ave/Makefile +++ b/drivers/media/platform/apple/ave/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_VIDEO_APPLE_AVE) += apple-ave.o -apple-ave-y := ave-drv.o ave-fw.o ave-ipc.o ave-cmd.o ave-layout.o ave-image.o ave-sess.o ave-v4l2.o +apple-ave-y := ave-drv.o ave-fw.o ave-ipc.o ave-cmd.o ave-writer.o ave-abi.o ave-layout.o ave-image.o ave-sess.o ave-v4l2.o obj-$(CONFIG_VIDEO_APPLE_AVE_KUNIT_TEST) += ave-sizes-test.o +obj-$(CONFIG_VIDEO_APPLE_AVE_CMD_KUNIT_TEST) += ave-abi-test.o diff --git a/drivers/media/platform/apple/ave/abi.rst b/drivers/media/platform/apple/ave/abi.rst new file mode 100644 index 0000000000000..ef4a1bc816363 --- /dev/null +++ b/drivers/media/platform/apple/ave/abi.rst @@ -0,0 +1,61 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Apple AVE firmware ABIs +====================== + +The firmware TEXT CRC chooses the wire layout. SoC match data supplies register +addresses, boot arguments and device identity. These are separate from streaming +qualification: identifying a firmware does not qualify its encoder payloads. + +9003.78.0 keeps its existing boot request, host clock synchronization, command +IDs, contiguous surface descriptors and reply envelope. The KUnit oracle in +``tests/ave-9003-reference.h`` freezes the driver builders and layout from Linux +commit 23e431a98962. Tests compare every byte of AVC and HEVC Start and Process +at 1920x1080 with B-frames enabled/disabled, and 3840x2160 with B-frames enabled. +They compare both deterministic synthetic DART addresses and masked images. +The constant image tables in ave-image.c are unchanged by this refactor. + +6070.11.1 is selected only for T6000 TEXT CRC 460c7a7c or T8112 TEXT CRC 25bfd039, +with the corresponding SoC identity. Boot, seven-channel discovery, Config, +Open, Close and scratch-confirmed Halt are implemented from the clean derived +SPEC-6070-lifecycle.md ledger. Start and Process return EOPNOTSUPP before writing +a payload or submitting a command. Reset also remains unavailable: its request +has a codec-dependent body, rather than a scalar header. No Priority command is +sent for this ABI. The split surface writer supports separate address and size +arrays, without assuming a semantic mapping for the unqualified fields. + +Lifecycle-only validation +------------------------- + +Build with CONFIG_DEBUG_FS. On a supported 6070 device, write ``1`` (AVC) or ``2`` +(HEVC) to ``/sys/kernel/debug/apple-ave/lifecycle_test``. This synchronous operation +runs boot, Config, Open, Close and Halt, then requests runtime suspend. It does +not allocate encoding surfaces or issue Start, Process or uninit for an encoder +that was never initialized. A second write exercises a cold restart and DATA +restore. An existing session makes the operation return EBUSY. + +The platform must supply the extra-0 register resource as ``extra0`` or register +resource index 3, with at least 0x24 bytes on T6000 or 0x4000 bytes on T8112. +The driver uses the translated platform resource address, maps its containing +16 KiB page through the device DMA API and puts that dynamically allocated IOVA +in Config. Platform DT/binding changes are separate from this driver change. +No fixed firmware or register IOVA is used as an ordinary DMA allocation. + +Before every boot the driver checks the firmware's IOMMU translations. DATA must +match the loader-exported pre-start CRC at probe; subsequent cold boots restore +that pristine image and verify its CRC. Firmware reservations and translations +remain persistent. IPC and extra-0 mappings are per-boot resources. + +Halt clears scratch0, sends ID 14 without expecting a ring reply, and waits up to +five seconds for scratch0 0x08042006. Only then does the driver stop the IOP and +check control readback. Runtime PM performs the power transition; a second boot +is refused until suspend/resume has intervened. A missing Halt confirmation or +failed lifecycle leaves the firmware broken and retains DMA mappings, runtime +power and the module reference. A forced unload or in-place restart is not a +recovery procedure. + +Hardware acceptance remains independent of KUnit: check both SoC VCBAR readbacks, +exact hello, ring dimensions, Config/Open/Close completion IDs and status, Halt +scratch confirmation, actual power gating and a second cold boot. Encoding on +9003 requires the existing hardware regression lane; these software tests do not +establish hardware equivalence. diff --git a/drivers/media/platform/apple/ave/ave-abi-test.c b/drivers/media/platform/apple/ave/ave-abi-test.c new file mode 100644 index 0000000000000..ec295b432a40a --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-abi-test.c @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Byte equality against the pre-ABI-refactor 9003 command builders. */ +#include +#include +#include +#include +#include + +#include "ave.h" +#include "ave-regs.h" +#include "tests/ave-9003-reference.h" + +static void ave_test_buf(struct ave_buf *b, unsigned int *index) +{ + b->dma = 0x820000000ULL + (u64)(*index) * 0x2000000; + b->size = SZ_16K * (1 + (*index)++); +} + +static void ave_test_session(struct ave_session *sess, enum ave_codec codec, + u32 width, u32 height, bool bframes) +{ + struct ave_params *p = &sess->p; + unsigned int index = 0; + + *p = (struct ave_params){ + .codec = codec, + .width = width, + .height = height, + .cw = ALIGN(width, 16), + .ch = ALIGN(height, 16), + .stride = ALIGN(width, 64), + .fps_num = 30, + .fps_den = 1, + .fps_int = 30, + .bframes = bframes, + .reorder_off = !bframes, + .slots = bframes ? 5 : 3, + .refs = bframes ? 5 : 2, + .gop = 120, + .rc_mode = 4, + .bitrate = 8000000, + .limit_bytes = 2000000, + .min_qp = -1, + .max_qp = 51, + .profile = codec == AVE_CODEC_H264 ? 6 : 1, + .level = codec == AVE_CODEC_H264 ? 13 : 7, + .high = codec == AVE_CODEC_H264, + }; + p->mbs = p->cw / 16 * (p->ch / 16); + p->coded_cap = ALIGN(3 * p->cw * p->ch / 2, SZ_16K); + sess->token = 400; + sess->client = 10; + ave_test_buf(&sess->hdr, &index); + for (unsigned int i = 0; i < AVE_MAX_REFS; i++) { + ave_test_buf(&sess->ref_a[i], &index); + ave_test_buf(&sess->ref_b[i], &index); + ave_test_buf(&sess->ref_c[i], &index); + ave_test_buf(&sess->ref_d[i], &index); + } + ave_test_buf(&sess->mb, &index); + ave_test_buf(&sess->row_a, &index); + ave_test_buf(&sess->row_b, &index); + ave_test_buf(&sess->fixed, &index); + for (unsigned int i = 0; i < AVE_TILE_BUFS; i++) { + ave_test_buf(&sess->tile[i], &index); + ave_test_buf(&sess->tile_aux[i], &index); + } + for (unsigned int i = 0; i < 2; i++) + ave_test_buf(&sess->hevc_extra[i], &index); + for (unsigned int i = 0; i < p->slots; i++) { + ave_test_buf(&sess->slot[i].coded, &index); + ave_test_buf(&sess->slot[i].meta, &index); + ave_test_buf(&sess->slot[i].aux, &index); + } +} + +/* These are synthetic DART addresses, never addresses from a device capture. */ +static void ave_mask_dart(u8 *cmd, size_t len) +{ + for (size_t off = 0; off + 8 <= len; off += 8) { + u64 val = get_unaligned_le64(cmd + off); + + if (val >= 0x800000000ULL && val < 0x1000000000ULL) + put_unaligned_le64(0, cmd + off); + } +} + +static void ave_9003_byte_equality(struct kunit *test) +{ + struct ave_dev *ave = kunit_kzalloc(test, sizeof(*ave), GFP_KERNEL); + struct ave_ctx *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + struct ave_session *sess = + kunit_kzalloc(test, sizeof(*sess), GFP_KERNEL); + u8 *actual = kunit_kmalloc(test, 0x14000, GFP_KERNEL); + u8 *expected = kunit_kmalloc(test, 0x14000, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, ave); + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, sess); + KUNIT_ASSERT_NOT_NULL(test, actual); + KUNIT_ASSERT_NOT_NULL(test, expected); + ave->layout = &ave_layout_9003_78_0; + ctx->ave = ave; + ctx->sess = sess; + for (unsigned int codec = AVE_CODEC_H264; codec <= AVE_CODEC_HEVC; + codec++) { + for (unsigned int setting = 0; setting < 3; setting++) { + size_t len = + ave->layout->bytes[codec - 1][AVE_IMAGE_START]; + + ave_test_session(sess, codec, + setting == 2 ? 3840 : 1920, + setting == 2 ? 2160 : 1080, + setting != 1); + memset(actual, 0, len); + memset(expected, 0, len); + KUNIT_ASSERT_EQ(test, ave_build_start(ctx, actual, len), + 0); + sess->token = 400; + ave_reference_start(ctx, expected, len); + ave_stamp_command(ave->layout->abi, actual, 94400000, + 0x12345678); + put_unaligned_le64(94400000, expected + 0x30); + put_unaligned_le64(0x12345678, expected + 0x38); + KUNIT_EXPECT_MEMEQ(test, actual, expected, len); + ave_mask_dart(actual, len); + ave_mask_dart(expected, len); + KUNIT_EXPECT_MEMEQ(test, actual, expected, len); + for (unsigned int frame = 0; frame < 2; frame++) { + unsigned int slot = frame ? sess->p.slots - 1 : + 0; + + len = ave->layout->bytes[codec - 1] + [AVE_IMAGE_PROCESS]; + sess->frames = frame ? 7 : 0; + sess->token = 500; + memset(actual, 0, len); + memset(expected, 0, len); + KUNIT_ASSERT_EQ( + test, + ave_build_process(ctx, actual, len, + slot, 0x8f0000000ULL, + 9000000, !!frame), + 0); + sess->token = 500; + ave_reference_process(ctx, expected, len, slot, + 0x8f0000000ULL, 9000000, + !!frame); + ave_stamp_command(ave->layout->abi, actual, + 94400000, 0x12345678); + put_unaligned_le64(94400000, expected + 0x30); + put_unaligned_le64(0x12345678, expected + 0x38); + KUNIT_EXPECT_MEMEQ(test, actual, expected, len); + ave_mask_dart(actual, len); + ave_mask_dart(expected, len); + KUNIT_EXPECT_MEMEQ(test, actual, expected, len); + } + } + } +} + +static void ave_6070_images(struct kunit *test) +{ + const struct ave_fw_abi *abi = &ave_abi_6070; + struct ave_dev *ave = kunit_kzalloc(test, sizeof(*ave), GFP_KERNEL); + struct ave_ctx *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + struct ave_session *sess = + kunit_kzalloc(test, sizeof(*sess), GFP_KERNEL); + u8 actual[0x70] = {}, expected[0x70] = {}; + + KUNIT_ASSERT_NOT_NULL(test, ave); + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, sess); + ave->layout = &ave_layout_6070_11_1; + ave->extra0_dma = 0x876540000ULL; + ctx->ave = ave; + ctx->sess = sess; + abi->write_config(ave, actual); + ave_stamp_command(abi, actual, 3000000, 0x123456); + put_unaligned_le16(1, expected); + put_unaligned_le32(U32_MAX, expected + 0x1c); + put_unaligned_le64(3000000, expected + 0x28); + put_unaligned_le64(0x123456, expected + 0x30); + expected[0x41] = 1; + put_unaligned_le64(ave->extra0_dma, expected + 0x48); + KUNIT_EXPECT_MEMEQ(test, actual, expected, sizeof(actual)); + for (unsigned int codec = AVE_CODEC_H264; codec <= AVE_CODEC_HEVC; + codec++) { + static const enum ave_opcode ops[] = { AVE_OP_OPEN, + AVE_OP_CLOSE, + AVE_OP_HALT }; + static const u16 wire[] = { 2, 12, 14 }; + static const u32 semantic[] = { 3, 4, U32_MAX }; + + for (unsigned int i = 0; i < ARRAY_SIZE(ops); i++) { + memset(actual, 0, sizeof(actual)); + memset(expected, 0, sizeof(expected)); + sess->p.codec = codec; + sess->client = 10; + sess->token = 300; + if (ops[i] == AVE_OP_HALT) + ave_write_global(ave, actual, ops[i]); + else + ave_write_header(ctx, actual, ops[i], ops[i], + 0xc8, 0x1e0); + ave_stamp_command(abi, actual, 3000000, 0x123456); + put_unaligned_le16(wire[i], expected); + if (ops[i] != AVE_OP_HALT) { + put_unaligned_le64(400, expected + 0x08); + put_unaligned_le32(10, expected + 0x10); + put_unaligned_le32(codec - 1, expected + 0x18); + put_unaligned_le32(0xc8, expected + 0x20); + } + put_unaligned_le32(semantic[i], expected + 0x1c); + put_unaligned_le64(3000000, expected + 0x28); + put_unaligned_le64(0x123456, expected + 0x30); + KUNIT_EXPECT_MEMEQ(test, actual, expected, + abi->scalar_bytes[ops[i]]); + } + } + KUNIT_EXPECT_EQ(test, abi->wire_id[0][AVE_OP_PRIORITY], 0U); + KUNIT_EXPECT_EQ(test, abi->scalar_bytes[AVE_OP_RESET], 0U); + KUNIT_EXPECT_EQ(test, abi->reply_id[AVE_OP_HALT], 0U); +} + +static void ave_6070_replies(struct kunit *test) +{ + const struct ave_fw_abi *abi = &ave_abi_6070; + struct ave_reply reply; + u8 msg[0x48] = {}; + + for (unsigned int op = 1; op < AVE_OP_COUNT; op++) { + u32 len = op == AVE_OP_PROCESS ? 0x48 : 0x40; + + if (!abi->reply_id[op]) + continue; + put_unaligned_le16(abi->reply_id[op], msg); + put_unaligned_le32(10, msg + 0x10); + put_unaligned_le32(abi->semantic[op], msg + 0x1c); + put_unaligned_le32(0x00ee0000, msg + 0x38); + KUNIT_ASSERT_EQ(test, abi->parse_reply(abi, msg, len, &reply), + 0); + KUNIT_EXPECT_EQ(test, reply.op, (enum ave_opcode)op); + KUNIT_EXPECT_EQ(test, reply.client, 10ULL); + KUNIT_EXPECT_EQ(test, reply.subtype, abi->semantic[op]); + KUNIT_EXPECT_EQ(test, reply.status, 0U); + KUNIT_EXPECT_EQ(test, + abi->parse_reply(abi, msg, len - 1, &reply), + -EPROTO); + KUNIT_EXPECT_EQ(test, + abi->parse_reply(abi, msg, len + 1, &reply), + -EPROTO); + put_unaligned_le32(0, msg + 0x38); + KUNIT_ASSERT_EQ(test, abi->parse_reply(abi, msg, len, &reply), + 0); + KUNIT_EXPECT_NE(test, reply.status, 0U); + } + put_unaligned_le16(2, msg); /* Request IDs are not completion IDs. */ + KUNIT_EXPECT_EQ(test, abi->parse_reply(abi, msg, 0x40, &reply), + -EPROTO); +} + +static void ave_6070_split_surfaces(struct kunit *test) +{ + const struct ave_surface_layout bank = { + .address = 0x4b8, + .size = 0x558, + .address_stride = 8, + .size_stride = 4, + }; + u8 *cmd = kunit_kzalloc(test, 0x600, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, cmd); + for (unsigned int i = 0; i < 20; i++) + ave_write_surface(&ave_layout_6070_11_1, cmd, &bank, i, + 0x800000000ULL + i * SZ_16K, + (i + 1) * SZ_16K); + for (unsigned int i = 0; i < 20; i++) { + KUNIT_EXPECT_EQ(test, get_unaligned_le64(cmd + 0x4b8 + i * 8), + 0x800000000ULL + i * SZ_16K); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(cmd + 0x558 + i * 4), + (i + 1) * SZ_16K); + } +} + +static void ave_6070_gate(struct kunit *test) +{ + struct ave_dev *ave = kunit_kzalloc(test, sizeof(*ave), GFP_KERNEL); + struct ave_ctx *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + struct ave_session *sess = + kunit_kzalloc(test, sizeof(*sess), GFP_KERNEL); + u8 cmd[0x70], before[0x70]; + + KUNIT_ASSERT_NOT_NULL(test, ave); + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, sess); + ave->layout = &ave_layout_6070_11_1; + ctx->ave = ave; + ctx->sess = sess; + sess->p.codec = AVE_CODEC_H264; + memset(cmd, 0xa5, sizeof(cmd)); + memcpy(before, cmd, sizeof(cmd)); + KUNIT_EXPECT_PTR_EQ(test, ave_layout_for_crc(0x460c7a7c), + &ave_layout_6070_11_1); + KUNIT_EXPECT_PTR_EQ(test, ave_layout_for_crc(0x25bfd039), + &ave_layout_6070_11_1); + KUNIT_EXPECT_PTR_EQ(test, ave_layout_for_crc(0), NULL); + KUNIT_EXPECT_EQ(test, ave_build_start(ctx, cmd, sizeof(cmd)), + -EOPNOTSUPP); + KUNIT_EXPECT_EQ( + test, ave_build_process(ctx, cmd, sizeof(cmd), 0, 0, 0, false), + -EOPNOTSUPP); + KUNIT_EXPECT_MEMEQ(test, cmd, before, sizeof(cmd)); + KUNIT_EXPECT_EQ(test, sess->token, 0ULL); +} + +static void ave_6070_channels(struct kunit *test) +{ + const struct ave_fw_transport *t = ave_abi_6070.transport; + struct ave_dev *ave = kunit_kzalloc(test, sizeof(*ave), GFP_KERNEL); + u32 hello[4] = { 7, 0x9bc0, 0x100, 0 }; + u8 *desc; + + KUNIT_ASSERT_NOT_NULL(test, ave); + ave->layout = &ave_layout_6070_11_1; + ave->ipc.cpu = kunit_kzalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ave->ipc.cpu); + ave->ipc.size = t->ipc_bytes; + ave->ipc_fw_base = 0x100000000ULL; + ave->channel_bytes = hello[1]; + KUNIT_EXPECT_EQ(test, ave_ipc_check_hello(t, hello), 0); + for (unsigned int word = 0; word < 4; word++) { + hello[word]++; + KUNIT_EXPECT_EQ(test, ave_ipc_check_hello(t, hello), -EPROTO); + hello[word]--; + } + for (unsigned int i = 0; i < 7; i++) { + int id = i == 1 ? 0 : 1; + bool io = i == 1 || i == 5; + u64 address = ave->ipc_fw_base + AVE_IPC_CHANNEL_OFF + + (io ? t->offsets[id] : 0x1000 + i * 0x400); + + desc = ave->ipc.cpu + AVE_IPC_CHANNEL_OFF + i * 0x100; + strscpy(desc, io ? (id ? "IO_T2H" : "IO") : "UNUSED", 0x40); + put_unaligned_le32(io ? id : U32_MAX, desc + 0x40); + put_unaligned_le32(io ? t->doorbells[id] : U32_MAX, + desc + 0x44); + put_unaligned_le32(io ? t->slots[id] : 4, desc + 0x48); + put_unaligned_le64(address, desc + 0x4c); + } + KUNIT_ASSERT_EQ(test, ave_ipc_parse_channels(ave), 0); + KUNIT_EXPECT_EQ(test, ave->ring[0].slots, 32U); + KUNIT_EXPECT_EQ(test, ave->ring[1].slots, 8U); + KUNIT_EXPECT_EQ(test, ave->ring[0].doorbell, 1U); + KUNIT_EXPECT_EQ(test, ave->ring[1].doorbell, 3U); + desc = ave->ipc.cpu + AVE_IPC_CHANNEL_OFF + 0x100; + put_unaligned_le32(33, desc + 0x48); + KUNIT_EXPECT_EQ(test, ave_ipc_parse_channels(ave), -EPROTO); + put_unaligned_le32(32, desc + 0x48); + desc = ave->ipc.cpu + AVE_IPC_CHANNEL_OFF; + put_unaligned_le64(ave->ipc_fw_base + ave->ipc.size, desc + 0x4c); + KUNIT_EXPECT_EQ(test, ave_ipc_parse_channels(ave), -EPROTO); +} + +static void ave_6070_variants(struct kunit *test) +{ + const struct ave_fw_transport *t = ave_abi_6070.transport; + + KUNIT_EXPECT_EQ(test, ave_t6000.boot_device_id, 11U); + KUNIT_EXPECT_EQ(test, ave_t6000.device_type, 9U); + KUNIT_EXPECT_EQ(test, ave_t6000.boot_arg_value, 0x400U); + KUNIT_EXPECT_EQ(test, ave_t6000.timebase_multiplier, 1000000U); + KUNIT_EXPECT_EQ(test, ave_t6000.text_crc, 0x460c7a7cU); + KUNIT_EXPECT_EQ(test, ave_t8112.boot_device_id, 15U); + KUNIT_EXPECT_EQ(test, ave_t8112.device_type, 13U); + KUNIT_EXPECT_EQ(test, ave_t8112.boot_arg_value, 0x10000U); + KUNIT_EXPECT_EQ(test, ave_t8112.timebase_multiplier, 1U); + KUNIT_EXPECT_EQ(test, ave_t8112.text_crc, 0x25bfd039U); + KUNIT_EXPECT_EQ(test, ave_fw_vcbar(&ave_t6000, 0x10000b28000ULL), + 0x0102000000b28000ULL); + KUNIT_EXPECT_EQ(test, ave_fw_vcbar(&ave_t8112, 0x800000000ULL), + 0x0102000800000000ULL); + KUNIT_EXPECT_EQ(test, ave_t6000.regs->ctrl, 0x400044U); + KUNIT_EXPECT_EQ(test, ave_t8112.regs->timebase, 0x178000U); + KUNIT_EXPECT_GE(test, t->command_bytes, 0x32dc8U); + KUNIT_EXPECT_LE(test, t->command_offset + t->command_bytes, + t->scalar_offset); + KUNIT_EXPECT_LE(test, + t->scalar_offset + + AVE_IPC_SCALAR_BUFS * AVE_IPC_SCALAR_BYTES, + t->process_offset); + KUNIT_EXPECT_LE( + test, t->process_offset + AVE_IPC_PROC_BUFS * t->process_bytes, + t->ipc_bytes); +} + +static struct kunit_case ave_abi_cases[] = { + KUNIT_CASE(ave_9003_byte_equality), KUNIT_CASE(ave_6070_images), + KUNIT_CASE(ave_6070_replies), KUNIT_CASE(ave_6070_split_surfaces), + KUNIT_CASE(ave_6070_gate), KUNIT_CASE(ave_6070_channels), + KUNIT_CASE(ave_6070_variants), {} +}; + +static struct kunit_suite ave_abi_suite = { + .name = "apple-ave-abi", + .test_cases = ave_abi_cases, +}; +kunit_test_suite(ave_abi_suite); + +MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/ave/ave-abi.c b/drivers/media/platform/apple/ave/ave-abi.c new file mode 100644 index 0000000000000..5aca710f0e854 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-abi.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Firmware command envelopes and transport descriptions. */ +#include +#include +#include + +#include "ave.h" +#include "ave-regs.h" + +static const struct ave_fw_transport ave_transport_9003 = { + .ipc_bytes = AVE_IPC_BYTES, + .command_offset = AVE_IPC_CMD_OFF, + .command_bytes = AVE_IPC_CMD_BYTES, + .scalar_offset = AVE_IPC_SCALAR_OFF, + .process_offset = AVE_IPC_PROC_OFF, + .process_bytes = AVE_IPC_PROC_BYTES, + .hello = { 2, 0, 0x100, 0 }, + .descriptors = 2, + .io_index = 0, + .t2h_index = 1, + .boot_config = true, + .sync_host_clock = true, +}; + +static const struct ave_fw_transport ave_transport_6070 = { + .ipc_bytes = 0x700000, + .command_offset = 0x40000, + .command_bytes = 0x34000, + .scalar_offset = 0x74000, + .process_offset = 0x78000, + .process_bytes = 0x10000, + .hello = { 7, 0x9bc0, 0x100, 0 }, + .descriptors = 7, + .io_index = 1, + .t2h_index = 5, + .slots = { 32, 8 }, + .doorbells = { 1, 3 }, + .offsets = { 0x8700, 0x9180 }, + .scratch_halt = true, +}; + +static void ave_header_9003(u8 *cmd, const struct ave_fw_abi *abi, + const struct ave_command *h) +{ + put_unaligned_le32(abi->wire_id[h->codec - 1][h->op], cmd); + put_unaligned_le64(h->token, cmd + 0x08); + put_unaligned_le64(h->client, cmd + 0x10); + if (h->op != AVE_OP_CONFIG && h->op != AVE_OP_HALT) { + put_unaligned_le32(1, cmd + 0x18); + put_unaligned_le32(h->codec, cmd + 0x1c); + put_unaligned_le32(h->arg0, cmd + 0x24); + put_unaligned_le32(0x36, cmd + 0x28); + put_unaligned_le32(h->arg2, cmd + 0x2c); + } + put_unaligned_le32(h->subtype, cmd + 0x20); +} + +static void ave_header_6070(u8 *cmd, const struct ave_fw_abi *abi, + const struct ave_command *h) +{ + bool global = h->op == AVE_OP_CONFIG || h->op == AVE_OP_HALT; + + put_unaligned_le16(abi->wire_id[h->codec - 1][h->op], cmd); + put_unaligned_le64(h->token, cmd + 0x08); + if (!global) { + put_unaligned_le32(h->client, cmd + 0x10); + put_unaligned_le32(h->codec - 1, cmd + 0x18); + } + put_unaligned_le32(h->op == AVE_OP_PROCESS ? h->subtype : + abi->semantic[h->op], + cmd + 0x1c); + put_unaligned_le32(global ? 0 : 0xc8, cmd + 0x20); +} + +static void ave_config_9003(struct ave_dev *ave, u8 *cmd) +{ + ave_write_global(ave, cmd, AVE_OP_CONFIG); + cmd[0x49] = 1; + cmd[0x4a] = 1; + put_unaligned_le32(ave->layout->mcc_dsid, cmd + 0x60); +} + +static void ave_config_6070(struct ave_dev *ave, u8 *cmd) +{ + ave_write_global(ave, cmd, AVE_OP_CONFIG); + cmd[0x41] = 1; + put_unaligned_le64(ave->extra0_dma, cmd + 0x48); + /* No MCC, host heap, shared area or optional firmware log. */ +} + +static int ave_reply_9003(const struct ave_fw_abi *abi, const u8 *msg, u32 len, + struct ave_reply *reply) +{ + u32 id; + + if (len < 0x48) + return -EPROTO; + id = get_unaligned_le16(msg); + reply->op = id < AVE_OP_COUNT ? id : 0; + reply->client = get_unaligned_le64(msg + 0x10); + reply->subtype = get_unaligned_le32(msg + 0x20); + reply->raw_status = get_unaligned_le32(msg + 0x40); + reply->status = reply->raw_status; + reply->bytes = len; + return 0; +} + +static int ave_reply_6070(const struct ave_fw_abi *abi, const u8 *msg, u32 len, + struct ave_reply *reply) +{ + u32 id; + + if (len < 0x40) + return -EPROTO; + id = get_unaligned_le16(msg); + for (unsigned int op = 1; op < AVE_OP_COUNT; op++) { + if (!abi->reply_id[op] || abi->reply_id[op] != id) + continue; + if (len != (op == AVE_OP_PROCESS ? 0x48 : 0x40)) + return -EPROTO; + reply->op = op; + reply->client = get_unaligned_le32(msg + 0x10); + reply->subtype = get_unaligned_le32(msg + 0x1c); + reply->raw_status = get_unaligned_le32(msg + 0x38); + reply->status = reply->raw_status; + reply->bytes = len; + if (reply->status == 0x00ee0000) + reply->status = 0; + else if (!reply->status) + reply->status = U32_MAX; + return 0; + } + return -EPROTO; +} + +#define AVE_9003_IDS \ + { [AVE_OP_CONFIG] = 1, [AVE_OP_HALT] = 2, [AVE_OP_OPEN] = 3, \ + [AVE_OP_CLOSE] = 4, [AVE_OP_START] = 6, [AVE_OP_STOP] = 7, \ + [AVE_OP_PROCESS] = 8, [AVE_OP_COMPLETE] = 9, [AVE_OP_PRIORITY] = 10 } + +const struct ave_fw_abi ave_abi_9003 = { + .transport = &ave_transport_9003, + .wire_id = { AVE_9003_IDS, AVE_9003_IDS }, + .reply_id = AVE_9003_IDS, + .scalar_bytes = { + [AVE_OP_CONFIG] = 0x78, + [AVE_OP_HALT] = 0x48, + [AVE_OP_OPEN] = 0x48, + [AVE_OP_CLOSE] = 0x48, + [AVE_OP_STOP] = 0x48, + [AVE_OP_COMPLETE] = 0x48, + [AVE_OP_PRIORITY] = 0x48, + }, + .header_bytes = 0x48, + .timeout_offset = 0x30, + .created_offset = 0x38, + .padded_surfaces = true, + .write_header = ave_header_9003, + .write_config = ave_config_9003, + .parse_reply = ave_reply_9003, +}; + +#define AVE_6070_IDS(start, process) \ + { [AVE_OP_CONFIG] = 1, [AVE_OP_OPEN] = 2, \ + [AVE_OP_RESET] = 3, [AVE_OP_START] = (start), \ + [AVE_OP_STOP] = 6, [AVE_OP_PROCESS] = (process), \ + [AVE_OP_FLUSH] = 11, [AVE_OP_CLOSE] = 12, \ + [AVE_OP_COMPLETE] = 13, [AVE_OP_HALT] = 14 } + +const struct ave_fw_abi ave_abi_6070 = { + .transport = &ave_transport_6070, + .wire_id = { AVE_6070_IDS(4, 7), AVE_6070_IDS(5, 8) }, + .reply_id = { + [AVE_OP_CONFIG] = 0xe01, + [AVE_OP_OPEN] = 0xe02, + [AVE_OP_RESET] = 0xe03, + [AVE_OP_START] = 0xe04, + [AVE_OP_STOP] = 0xe05, + [AVE_OP_PROCESS] = 0xe06, + [AVE_OP_FLUSH] = 0xe09, + [AVE_OP_CLOSE] = 0xe0a, + [AVE_OP_COMPLETE] = 0xe0b, + }, + .scalar_bytes = { + [AVE_OP_CONFIG] = 0x70, + [AVE_OP_OPEN] = 0x40, + [AVE_OP_STOP] = 0x40, + [AVE_OP_FLUSH] = 0x40, + [AVE_OP_CLOSE] = 0x48, + [AVE_OP_COMPLETE] = 0x40, + [AVE_OP_HALT] = 0x40, + }, + .semantic = { + [AVE_OP_CONFIG] = U32_MAX, + [AVE_OP_OPEN] = 3, + [AVE_OP_RESET] = 11, + [AVE_OP_START] = 6, + [AVE_OP_STOP] = 7, + [AVE_OP_FLUSH] = 10, + [AVE_OP_CLOSE] = 4, + [AVE_OP_COMPLETE] = 9, + [AVE_OP_HALT] = U32_MAX, + }, + .header_bytes = 0x40, + .timeout_offset = 0x28, + .created_offset = 0x30, + .write_header = ave_header_6070, + .write_config = ave_config_6070, + .parse_reply = ave_reply_6070, +}; diff --git a/drivers/media/platform/apple/ave/ave-cmd.c b/drivers/media/platform/apple/ave/ave-cmd.c index 3360f8e4edac3..3a6533846f88b 100644 --- a/drivers/media/platform/apple/ave/ave-cmd.c +++ b/drivers/media/platform/apple/ave/ave-cmd.c @@ -19,109 +19,6 @@ #include "ave.h" #include "ave-regs.h" -/* - * IEEE 754 double for the positive value @num / @den (0 for 0), built with - * integer arithmetic; the firmware takes rates and limits as doubles. - */ -static u64 ave_f64(u64 num, u64 den) -{ - u64 mant = 0, rem; - int exp = 0; - - if (!num || !den) - return 0; - while (num >= den << 1) { - den <<= 1; - exp++; - } - while (num < den) { - num <<= 1; - exp--; - } - /* 1 <= num/den < 2: take 53 significant bits by long division. */ - rem = num; - for (unsigned int i = 0; i < 53; i++) { - mant <<= 1; - if (rem >= den) { - mant |= 1; - rem -= den; - } - rem <<= 1; - } - return (u64)(exp + 1023) << 52 | (mant & GENMASK_ULL(51, 0)); -} - -#define AVE_F64_NEG BIT_ULL(63) - -static void ave_put32(u8 *cmd, u32 off, u32 val) -{ - put_unaligned_le32(val, cmd + off); -} - -static void ave_put64(u8 *cmd, u32 off, u64 val) -{ - put_unaligned_le64(val, cmd + off); -} - -/* A surface descriptor: {u64 DART address, u32 size, u32 0}. */ -static void ave_put_surface(u8 *cmd, u32 off, dma_addr_t addr, u32 size) -{ - ave_put64(cmd, off, addr); - ave_put32(cmd, off + 8, size); - ave_put32(cmd, off + 12, 0); -} - -static void ave_put_buf(u8 *cmd, u32 off, const struct ave_buf *b, u32 size) -{ - ave_put_surface(cmd, off, b->dma, size); -} - -/* Is another session open in the firmware besides @ctx's? */ -static bool ave_cmd_others_open(struct ave_ctx *ctx) -{ - struct ave_dev *ave = ctx->ave; - - for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) - if (ave->sessions[i] && ave->sessions[i] != ctx && ave->sessions[i]->sess->fw_open) - return true; - return false; -} - -/* - * Header word +0x2c depends on the set of open sessions: 0x1e0 on Open and - * Start (0x140 when another session is open), the codec value on every - * other command of a lone session, and 0 while another one is open. - */ -static u32 ave_session_arg2(struct ave_ctx *ctx) -{ - if (ctx->ave->layout->counted && ctx->sess->p.codec == AVE_CODEC_H264) - return ctx->sess->p.bframes ? AVE_SESSION_ARG2_H264 : 0x140; - if (ave_cmd_others_open(ctx)) - return 0; - return ctx->sess->p.codec == AVE_CODEC_HEVC ? AVE_SESSION_ARG2_HEVC : - AVE_SESSION_ARG2_H264; -} - -static u32 ave_session_setup_arg2(struct ave_ctx *ctx) -{ - return ave_cmd_others_open(ctx) ? AVE_SESSION_ARG2_SETUP_SHARED : AVE_SESSION_ARG2_SETUP; -} - -static void ave_cmd_header(struct ave_ctx *ctx, u8 *cmd, enum ave_opcode op, u32 echo, - u32 arg0, u32 arg2) -{ - ctx->sess->token += AVE_TOKEN_STEP; - ave_put32(cmd, AVE_CMD_OPCODE, op); - ave_put64(cmd, AVE_CMD_TOKEN, ctx->sess->token); - ave_put64(cmd, AVE_CMD_CLIENT, ctx->sess->client); - ave_put32(cmd, AVE_CMD_UNIT, AVE_SESSION_UNIT); - ave_put32(cmd, AVE_CMD_CODEC, ctx->sess->p.codec); - ave_put32(cmd, AVE_CMD_OPCODE_ECHO, echo); - ave_put32(cmd, AVE_CMD_ARG0, arg0); - ave_put32(cmd, AVE_CMD_ARG1, AVE_SESSION_ARG1); - ave_put32(cmd, AVE_CMD_ARG2, arg2); -} - /* Firmware-wide configuration, sent once after every firmware boot. */ int ave_cmd_config(struct ave_dev *ave) { @@ -131,13 +28,14 @@ int ave_cmd_config(struct ave_dev *ave) if (buf < 0) return buf; - cmd = ave_ipc_cmd(ave, buf, AVE_CONFIG_BYTES); - ave_put32(cmd, AVE_CMD_OPCODE, AVE_OP_CONFIG); - ave_put32(cmd, AVE_CONFIG_ALL_UNITS, U32_MAX); - cmd[AVE_CONFIG_OPT_A] = 1; - cmd[AVE_CONFIG_OPT_B] = 1; - ave_put32(cmd, AVE_CONFIG_MCC_DSID, ave->layout->mcc_dsid); - ret = ave_ipc_submit(ave, buf, AVE_CONFIG_BYTES, AVE_CMD_GLOBAL_TIMEOUT_US, true); + cmd = ave_ipc_cmd(ave, buf, + ave->layout->abi->scalar_bytes[AVE_OP_CONFIG]); + if (!cmd) + return -EBUSY; + ave->layout->abi->write_config(ave, cmd); + ret = ave_ipc_submit(ave, buf, + ave->layout->abi->scalar_bytes[AVE_OP_CONFIG], + AVE_CMD_GLOBAL_TIMEOUT_US, true); return ret ?: ave_ipc_settle(ave, AVE_CMD_WAIT_MS); } @@ -152,13 +50,16 @@ int ave_cmd_halt(struct ave_dev *ave) if (buf < 0) return buf; - cmd = ave_ipc_cmd(ave, buf, AVE_CMD_HDR_BYTES); - ave_put32(cmd, AVE_CMD_OPCODE, AVE_OP_HALT); - ave_put32(cmd, AVE_CMD_OPCODE_ECHO, AVE_HALT_ALL); - return ave_ipc_submit(ave, buf, AVE_CMD_HDR_BYTES, AVE_CMD_GLOBAL_TIMEOUT_US, false); + cmd = ave_ipc_cmd(ave, buf, ave->layout->abi->header_bytes); + if (!cmd) + return -EBUSY; + ave_write_global(ave, cmd, AVE_OP_HALT); + return ave_ipc_submit(ave, buf, ave->layout->abi->header_bytes, + AVE_CMD_GLOBAL_TIMEOUT_US, false); } -static int ave_cmd_scalar(struct ave_ctx *ctx, enum ave_opcode op, u32 arg0, u32 arg2) +static int ave_cmd_scalar(struct ave_ctx *ctx, enum ave_opcode op, u32 arg0, + u32 arg2) { struct ave_dev *ave = ctx->ave; int buf = ave_ipc_scalar_buf(ave); @@ -166,9 +67,17 @@ static int ave_cmd_scalar(struct ave_ctx *ctx, enum ave_opcode op, u32 arg0, u32 if (buf < 0) return buf; - cmd = ave_ipc_cmd(ave, buf, AVE_CMD_HDR_BYTES); - ave_cmd_header(ctx, cmd, op, op, arg0, arg2); - return ave_ipc_submit(ave, buf, AVE_CMD_HDR_BYTES, AVE_CMD_SESSION_TIMEOUT_US, true); + if (!ave->layout->abi->scalar_bytes[op]) + return -EOPNOTSUPP; + cmd = ave_ipc_cmd(ave, buf, ave->layout->abi->scalar_bytes[op]); + if (!cmd) + return -EBUSY; + ave_write_header(ctx, cmd, op, op, arg0, arg2); + return ave_ipc_submit(ave, buf, ave->layout->abi->scalar_bytes[op], + ave->layout->abi->transport->boot_config ? + AVE_CMD_SESSION_TIMEOUT_US : + AVE_CMD_GLOBAL_TIMEOUT_US, + true); } int ave_cmd_open(struct ave_ctx *ctx) @@ -179,10 +88,17 @@ int ave_cmd_open(struct ave_ctx *ctx) ave->next_client += AVE_CLIENT_STEP; ctx->sess->client = ave->next_client; ctx->sess->token = AVE_TOKEN_BASE; - ret = ave_cmd_scalar(ctx, AVE_OP_OPEN, AVE_SESSION_ARG0, ave_session_setup_arg2(ctx)); + ret = ave_cmd_scalar(ctx, AVE_OP_OPEN, AVE_SESSION_ARG0, + ave_session_setup_arg2(ctx)); + if (ave->layout->abi->transport->boot_config) { + if (!ret) + ctx->sess->fw_open = true; + return ret ?: ave_ipc_settle(ave, AVE_CMD_WAIT_MS); + } + ret = ret ?: ave_ipc_settle(ave, AVE_CMD_WAIT_MS); if (!ret) ctx->sess->fw_open = true; - return ret ?: ave_ipc_settle(ave, AVE_CMD_WAIT_MS); + return ret; } /* Drain barrier: answered once every Process sent before it is answered. */ @@ -199,250 +115,29 @@ int ave_cmd_complete(struct ave_ctx *ctx, unsigned int count) int ave_cmd_close(struct ave_ctx *ctx) { u32 arg2 = ave_session_arg2(ctx); - int ret, err; + bool legacy = ctx->ave->layout->abi->transport->boot_config; + int ret = 0, err; - ret = ave_cmd_scalar(ctx, AVE_OP_PRIORITY, AVE_SESSION_ARG0, arg2); - if (!ret) + if (legacy) + ret = ave_cmd_scalar(ctx, AVE_OP_PRIORITY, AVE_SESSION_ARG0, + arg2); + if (!ret && (legacy || ctx->sess->handed)) ret = ave_cmd_scalar(ctx, AVE_OP_STOP, AVE_SESSION_ARG0, arg2); err = ave_ipc_settle(ctx->ave, AVE_CMD_WAIT_MS); ret = ret ?: err; if (ctx->ave->fw_state != AVE_FW_RUNNING) return ret; + /* A failed 6070 Stop cannot establish a safe Close/Halt boundary. */ + if (ret && !legacy) + return ret; err = ave_cmd_scalar(ctx, AVE_OP_CLOSE, AVE_SESSION_ARG0, arg2); if (!err) err = ave_ipc_settle(ctx->ave, AVE_CMD_WAIT_MS); - ctx->sess->fw_open = false; + if (legacy || !err) + ctx->sess->fw_open = false; return ret ?: err; } -/* Start (id 6) */ - -#define AVE_START_ARRAY_LEN 30 -#define AVE_START_INIT_QP_VALUE 26 - -static void ave_start_surfaces(struct ave_session *sess, u8 *cmd, - const struct ave_fw_layout *fw) -{ - const struct ave_params *p = &sess->p; - bool hevc = p->codec == AVE_CODEC_HEVC; - unsigned int tiles = hevc ? 4 : AVE_TILE_BUFS; - const struct ave_start_layout *l = &fw->start[p->codec - 1]; - - /* Clear every descriptor the image may carry, then fill ours. */ - for (unsigned int i = 0; i < AVE_MAX_REFS; i++) { - ave_put_surface(cmd, l->ref_a + l->ref_stride * i, 0, 0); - ave_put_surface(cmd, l->ref_b + l->ref_stride * i, 0, 0); - ave_put_surface(cmd, l->ref_c + l->ref_c_stride * i, 0, 0); - ave_put_surface(cmd, l->ref_c_alias + l->ref_c_stride * i, 0, 0); - ave_put_surface(cmd, l->ref_d + 0x10 * i, 0, 0); - } - for (unsigned int s = 0; s < AVE_START_ARRAY_LEN; s++) { - ave_put_surface(cmd, l->coded + 0x10 * s, 0, 0); - ave_put_surface(cmd, l->meta + 0x10 * s, 0, 0); - } - - ave_put_buf(cmd, 0x48, &sess->hdr, fw->hdr_bytes); - ave_put_surface(cmd, 0x58, 0, 0); - for (unsigned int i = 0; i < p->refs; i++) { - ave_put_buf(cmd, l->ref_a + l->ref_stride * i, - &sess->ref_a[i], sess->ref_a[i].size); - ave_put_buf(cmd, l->ref_b + l->ref_stride * i, - &sess->ref_b[i], sess->ref_b[i].size); - ave_put_buf(cmd, l->ref_c + l->ref_c_stride * i, - &sess->ref_c[i], sess->ref_c[i].size); - ave_put_surface(cmd, l->ref_c_alias + l->ref_c_stride * i, sess->ref_c[i].dma, 0); - ave_put_buf(cmd, l->ref_d + 0x10 * i, &sess->ref_d[i], sess->ref_d[i].size); - } - if (fw->counted) { - ave_put32(cmd, l->refs_count, p->refs); - ave_put32(cmd, l->work_count, p->refs); - ave_put32(cmd, l->lfs_count, p->refs); - ave_put32(cmd, l->slots_count, p->slots); - ave_put32(cmd, l->meta_count, p->slots); - ave_put32(cmd, l->mb_count, p->rc_mode == 3 ? 3 : 1); - if (p->rc_mode == 3) - for (unsigned int i = 0; i < 2; i++) - ave_put_buf(cmd, l->mb + 0x10 * (i + 1), - &sess->mb_extra[i], sess->mb_extra[i].size); - ave_put32(cmd, l->tiles_count, tiles); - for (unsigned int i = 0; i < p->refs; i++) { - ave_put_buf(cmd, l->ref_e + l->ref_stride * i, - &sess->ref_e[i], sess->ref_e[i].size); - ave_put_buf(cmd, l->ref_f + l->ref_stride * i, - &sess->ref_f[i], sess->ref_f[i].size); - } - } - ave_put_buf(cmd, l->mb, &sess->mb, sess->mb.size); - for (unsigned int s = 0; s < p->slots; s++) { - ave_put_buf(cmd, l->coded + 0x10 * s, &sess->slot[s].coded, p->coded_cap); - ave_put_buf(cmd, l->meta + 0x10 * s, &sess->slot[s].meta, sess->slot[s].meta.size); - } - ave_put_buf(cmd, l->row_a, &sess->row_a, sess->row_a.size); - ave_put_buf(cmd, l->row_b, &sess->row_b, sess->row_b.size); - ave_put_buf(cmd, l->fixed, &sess->fixed, SZ_16K); - for (unsigned int i = 0; i < AVE_TILE_BUFS; i++) { - u32 off = l->tiles[i]; - - if (i < tiles) { - ave_put_buf(cmd, off, &sess->tile[i], sess->tile[i].size); - ave_put_buf(cmd, off + 0x10, &sess->tile_aux[i], sess->tile_aux[i].size); - } else { - ave_put_surface(cmd, off, 0, 0); - ave_put_surface(cmd, off + 0x10, 0, 0); - } - } - if (hevc) - for (unsigned int i = 0; i < 2; i++) - ave_put_buf(cmd, l->hevc_extra + 0x10 * i, &sess->hevc_extra[i], - sess->hevc_extra[i].size); -} - -static void ave_start_put32(u8 *cmd, u32 off, u32 val) -{ - if (off) - ave_put32(cmd, off, val); -} - -static void ave_start_config(const struct ave_params *p, u8 *cmd, - const struct ave_fw_layout *fw) -{ - bool hevc = p->codec == AVE_CODEC_HEVC; - bool cq = p->rc_mode == 3; - u32 depth = p->bit_depth ?: 8; - u32 crop_r = (p->cw - p->width) / 2, crop_b = (p->ch - p->height) / 2; - bool crop = crop_r || crop_b; - const struct ave_start_layout *l = &fw->start[p->codec - 1]; - - ave_start_put32(cmd, l->codec, p->codec); - ave_start_put32(cmd, l->flag_8bit, - fw->counted ? 0x10000 | (hevc ? (depth == 8 ? 0x41 : 0x40) : 0) | - (cq ? 0x10 : 0) : hevc ? 1 : 0); - ave_start_put32(cmd, l->geometry_mode, (hevc ? 0x40 : 0) | (cq ? 0 : 2)); - ave_start_put32(cmd, l->geometry_depth, depth); - ave_start_put32(cmd, l->const_214, 0x39d); - ave_start_put32(cmd, l->const_218, 0x18); - ave_start_put32(cmd, l->fps, p->fps_int); - - /* Rate control */ - ave_start_put32(cmd, l->rc_mode, p->rc_mode); - ave_start_put32(cmd, l->bitrate, p->bitrate); - for (unsigned int i = 0; i < 3; i++) - ave_put32(cmd, (l->init_qp + 4 * i), cq ? p->initial_qp : AVE_START_INIT_QP_VALUE); - ave_start_put32(cmd, l->limits, p->limit_bytes ? 1 : 0); - if (l->limit_bytes && p->limit_bytes) { - ave_put64(cmd, l->limit_bytes, - ave_f64((u64)p->limit_bytes * (fw->counted ? 8 : 1), 1)); - ave_put64(cmd, l->limit_secs, ave_f64(1, 1)); - } - ave_put64(cmd, l->f64_288, ave_f64(1, 1)); - ave_put64(cmd, l->f64_290, ave_f64(1, 2)); - ave_put64(cmd, l->f64_2c0, AVE_F64_NEG | ave_f64(1, 1)); - ave_put64(cmd, l->f64_2c8, AVE_F64_NEG | ave_f64(1, 1)); - ave_start_put32(cmd, l->min_qp, p->min_qp); - ave_start_put32(cmd, l->max_qp, p->max_qp); - ave_start_put32(cmd, l->min_qp2, p->min_qp); - ave_start_put32(cmd, l->max_qp2, p->max_qp); - ave_start_put32(cmd, l->rc_flags, cq ? 0 : hevc ? 0x736 : 0x8202); - ave_start_put32(cmd, l->rc_310, cq ? 0 : 30); - if (!hevc) - ave_start_put32(cmd, l->h264_380, 1); - - /* GOP structure: with B-frames as captured, or P-only, or intra-only. */ - ave_start_put32(cmd, l->gop_flags, (p->bframes ? 0x600 : 0x200) | - (fw->counted && cq ? 0x10000 | (hevc ? 0x100 : 0) : 0)); - ave_start_put32(cmd, l->p_frames, p->intra_only ? 0 : 1); - ave_start_put32(cmd, l->max_b, p->bframes ? 3 : 0); - ave_start_put32(cmd, l->ref_depth, p->bframes ? 3 : 1); - ave_start_put32(cmd, l->idr_interval, p->gop); - - /* Picture size */ - ave_start_put32(cmd, l->size_class, p->width * p->height > 1280 * 720 ? 3 : 1); - ave_start_put32(cmd, l->coded_w, p->cw); - ave_start_put32(cmd, l->coded_h, p->ch); - ave_start_put32(cmd, l->coded_h2, p->ch); - ave_start_put32(cmd, l->codec_flags, hevc ? 0x2000002 : 2); - ave_start_put32(cmd, l->bit_depth, depth); - - if (!hevc) { - ave_start_put32(cmd, l->h264_384, p->width * p->height <= 320 * 240 ? 0x40 : 0x10); - ave_start_put32(cmd, l->codec_2588, p->high ? 2 : 0); - ave_start_put32(cmd, l->h264_profile, p->profile); - ave_start_put32(cmd, l->h264_level, p->level); - ave_start_put32(cmd, l->h264_max_refs, - (p->refs - 1) | (fw->counted ? (p->cw / 16) << 16 : 0)); - if (fw->counted) { - static const u8 level_idc[] = { - 10, 9, 11, 12, 13, 20, 21, 22, 30, - 31, 32, 40, 41, 42, 50, 51, 52, - }; - - ave_put32(cmd, l->profile_idc, p->high ? 100 : p->profile == 2 ? 66 : 77); - ave_put32(cmd, l->level_idc, level_idc[p->level - 1]); - put_unaligned_le16(p->refs - 1, cmd + l->max_refs_copy); - ave_put32(cmd, l->mb_size, ((p->ch / 16 - 1) << 16) | (p->cw / 16 - 1)); - cmd[l->mb_height] = p->ch / 16; - cmd[l->mb_height + 2] = 1; - cmd[l->mb_height + 3] = 1; - cmd[l->crop_copy] = crop; - put_unaligned_le16(crop_r, cmd + l->crop_right_copy); - put_unaligned_le16(crop_b, cmd + l->crop_bottom_copy); - cmd[l->codec_byte] = !cq; - /* CABAC flag in the second, packed picture structure. */ - cmd[l->cabac_copy] = p->profile != 2; - cmd[l->transform_copy] = p->high; - } - ave_start_put32(cmd, l->h264_width_mbs, p->cw / 16 - 1); - ave_start_put32(cmd, l->h264_height_mbs, p->ch / 16 - 1); - ave_start_put32(cmd, l->h264_crop, crop); - ave_start_put32(cmd, l->h264_crop_left, 0); - ave_start_put32(cmd, l->h264_crop_right, crop_r); - ave_start_put32(cmd, l->h264_crop_top, 0); - ave_start_put32(cmd, l->h264_crop_bottom, crop_b); - ave_start_put32(cmd, l->h264_cabac, p->profile != 2); - cmd[l->h264_8x8] = p->high; - return; - } - - ave_start_put32(cmd, l->hevc_pad, 1 | - (p->cw > p->width ? BIT(16) | BIT(18) : 0) | - (p->ch > p->height ? BIT(17) | BIT(19) : 0)); - ave_start_put32(cmd, l->hevc_profile_a, p->profile); - ave_start_put32(cmd, l->hevc_profile_b, p->profile); - cmd[l->hevc_compat_a] = depth == 8; - cmd[l->hevc_compat_a + 1] = 1; - cmd[l->hevc_compat_b] = depth == 8; - cmd[l->hevc_compat_b + 1] = 1; - cmd[l->hevc_8bit_a] = depth == 8; - cmd[l->hevc_8bit_b] = depth == 8; - /* Rate-control words that only change in constant-quality mode */ - ave_start_put32(cmd, l->hevc_cq, cq); - ave_start_put32(cmd, l->hevc_cq_copy, cq); - ave_start_put32(cmd, l->hevc_cq_codec, cq); - ave_start_put32(cmd, l->hevc_cq_picture, cq ? 0x100 : 0); - ave_start_put32(cmd, l->hevc_rc_270c, 2); - ave_start_put32(cmd, l->hevc_rc_9604, cq ? 2 : 0); - ave_start_put32(cmd, l->hevc_rc_9608, cq ? 0 : 0x10000); - ave_start_put32(cmd, l->hevc_level_a, p->level); - ave_start_put32(cmd, l->hevc_level_b, p->level); - ave_start_put32(cmd, l->hevc_width, p->cw); - ave_start_put32(cmd, l->hevc_height, p->ch); - ave_start_put32(cmd, l->hevc_width_ctb, DIV_ROUND_UP(p->cw, 32)); - ave_start_put32(cmd, l->hevc_height_ctb, DIV_ROUND_UP(p->ch, 32)); - ave_start_put32(cmd, l->hevc_conf, crop); - ave_start_put32(cmd, l->hevc_conf_left, 0); - ave_start_put32(cmd, l->hevc_conf_right, crop_r); - ave_start_put32(cmd, l->hevc_conf_top, 0); - ave_start_put32(cmd, l->hevc_conf_bottom, crop_b); - ave_start_put32(cmd, l->hevc_depth_luma, depth - 8); - ave_start_put32(cmd, l->hevc_depth_chroma, depth - 8); - /* Reference structure: B-frame pyramid or low-delay P (ave-image.c) */ - ave_start_put32(cmd, l->hevc_refs, p->bframes ? 4 : 1); - ave_start_put32(cmd, l->hevc_dpb_minus1, 4); - ave_start_put32(cmd, l->hevc_reorder, p->bframes ? 2 : 0); - ave_start_put32(cmd, l->hevc_reorder_copy, p->bframes ? 2 : 0); - ave_start_put32(cmd, l->hevc_reorder_on, p->bframes); -} - int ave_cmd_start(struct ave_ctx *ctx) { struct ave_dev *ave = ctx->ave; @@ -453,119 +148,53 @@ int ave_cmd_start(struct ave_ctx *ctx) u8 *cmd; int ret; - if (!p->refs || p->refs > AVE_MAX_REFS || !p->slots || p->slots > fw->max_slots) - return -EINVAL; + if (!fw->encode_ready) + return -EOPNOTSUPP; cmd = ave_ipc_cmd(ave, AVE_BUF_CMD, len); if (!cmd) return -EBUSY; - ave_image_start(cmd, len, fw, p->codec, p->reorder_off, - p->codec == AVE_CODEC_HEVC && !p->bframes); - ave_cmd_header(ctx, cmd, AVE_OP_START, AVE_OP_START, AVE_SESSION_ARG0, - ave_session_setup_arg2(ctx)); - ave_start_surfaces(sess, cmd, fw); - ave_start_config(&sess->p, cmd, fw); + ret = ave_build_start(ctx, cmd, len); + if (ret) + return ret; ave_debugfs_save(ave, p->codec, AVE_IMAGE_START, cmd, len); - ret = ave_ipc_submit(ave, AVE_BUF_CMD, len, AVE_CMD_SESSION_TIMEOUT_US, true); + ret = ave_ipc_submit(ave, AVE_BUF_CMD, len, AVE_CMD_SESSION_TIMEOUT_US, + true); if (ret) return ret; sess->handed = true; return ave_ipc_settle(ave, AVE_START_WAIT_MS); } -/* Process (id 8) */ - -#define AVE_PROC_TIMESCALE 14400000ULL -#define AVE_PROC_SHDR_ENTRIES 256 -#define AVE_PROC_SHDR_STRIDE 0x400 -#define AVE_PROC_FORCED_TYPE 2 -#define AVE_PROC_LOW_DELAY 3 - -int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *src, - dma_addr_t input, u32 bitrate, bool force_key) +int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, + struct vb2_v4l2_buffer *src, dma_addr_t input, u32 bitrate, + bool force_key) { struct ave_dev *ave = ctx->ave; struct ave_session *sess = ctx->sess; const struct ave_params *p = &sess->p; const struct ave_fw_layout *fw = ave->layout; - const struct ave_proc_layout *l = &fw->process[p->codec - 1]; size_t len = fw->bytes[p->codec - 1][AVE_IMAGE_PROCESS]; struct ave_slot *s; unsigned int buf = AVE_BUF_PROC(sess->index * AVE_MAX_SLOTS + slot); - dma_addr_t y = input; - u64 pts; u8 *cmd; int ret; + if (!fw->encode_ready) + return -EOPNOTSUPP; + if (slot >= p->slots || p->slots > fw->max_slots) return -EINVAL; s = &sess->slot[slot]; - if ((!l->force_key && force_key) || (!l->bitrate && bitrate != p->bitrate)) - return -EOPNOTSUPP; ret = ave_ipc_wait_ack(ave, buf); if (ret) return ret; cmd = ave_ipc_cmd(ave, buf, len); if (!cmd) return -EBUSY; - ave_image_process(cmd, len, fw, p->codec); - ave_cmd_header(ctx, cmd, AVE_OP_PROCESS, fw->process_subtype + slot, 0, - ave_session_arg2(ctx)); - - /* Input picture: NV12, chroma right after p->height luma lines. */ - ave_put64(cmd, l->y, y); - ave_put32(cmd, l->y + 8, p->stride * p->height); - ave_put32(cmd, l->y + 12, p->stride); - ave_put64(cmd, l->uv, y + p->stride * p->height); - ave_put32(cmd, l->uv + 8, p->stride * p->height / 2); - ave_put32(cmd, l->uv + 12, p->stride); - ave_put32(cmd, l->in_w, p->width); - ave_put32(cmd, l->in_h, p->height); - - /* Session values repeated in every frame */ - ave_put32(cmd, l->coded_cap, p->coded_cap); - ave_put32(cmd, l->fps_int, p->fps_int); - /* The bit rate may change while streaming; it takes effect here. */ - if (l->bitrate) - ave_put32(cmd, l->bitrate, bitrate); - if (l->limits) - ave_put32(cmd, l->limits, p->limit_bytes ? 1 : 0); - if (l->limit_bytes && p->limit_bytes) { - ave_put64(cmd, l->limit_bytes, ave_f64(p->limit_bytes, 1)); - ave_put64(cmd, l->limit_secs, ave_f64(1, 1)); - } - ave_put32(cmd, l->const5, 5); - ave_put64(cmd, l->fps, ave_f64(p->fps_num, p->fps_den)); - ave_put64(cmd, l->timescale, AVE_PROC_TIMESCALE); - if (l->crop) - ave_put32(cmd, l->crop, p->cw != p->width || p->ch != p->height); - - /* This frame */ - ave_put32(cmd, l->first_frame, sess->frames == 0); - ave_put32(cmd, l->slot, slot); - ave_put64(cmd, l->coded, s->coded.dma); - ave_put64(cmd, l->meta, s->meta.dma); - if (l->meta_size) - ave_put32(cmd, l->meta_size, s->meta.size); - if (l->extra) - ave_put_buf(cmd, l->extra, &s->extra, SZ_16K); - if (l->frame_size) - ave_put32(cmd, l->frame_size, p->rc_mode == 3 ? 0 : s->aux.size); - if (l->frame_buf) - ave_put64(cmd, l->frame_buf, fw->counted && p->rc_mode == 3 ? 0 : s->aux.dma); - if (l->shdr_table) - for (unsigned int i = 0; i < AVE_PROC_SHDR_ENTRIES; i++) - ave_put64(cmd, l->shdr_table + 8 * i, - s->aux.dma + i * AVE_PROC_SHDR_STRIDE); - ave_put32(cmd, l->ordinal, sess->frames); - pts = mul_u64_u64_div_u64((u64)sess->frames * p->fps_den, AVE_PROC_TIMESCALE, p->fps_num); - ave_put64(cmd, l->pts, pts); - if (l->force_key) - ave_put32(cmd, l->force_key, force_key); - if (l->force_type) - ave_put32(cmd, l->force_type, force_key ? AVE_PROC_FORCED_TYPE : 0); - if (l->low_delay) - ave_put32(cmd, l->low_delay, p->bframes ? 0 : AVE_PROC_LOW_DELAY); + ret = ave_build_process(ctx, cmd, len, slot, input, bitrate, force_key); + if (ret) + return ret; if (!sess->frames) ave_debugfs_save(ave, p->codec, AVE_IMAGE_PROCESS, cmd, len); diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c index 76527c670ed7e..2699879f721f8 100644 --- a/drivers/media/platform/apple/ave/ave-drv.c +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -11,6 +11,8 @@ */ #include #include +#include +#include #include #include #include @@ -22,7 +24,7 @@ #include "ave.h" #include "ave-regs.h" -#define AVE_AUTOSUSPEND_MS 2000 +#define AVE_AUTOSUSPEND_MS 2000 static const struct ave_coproc_regs ave_coproc_v40 = { .vcbar = AVE_COPROC_VCBAR, @@ -34,23 +36,49 @@ static const struct ave_coproc_regs ave_coproc_v40 = { .keep_mappings = AVE_COPROC_KEEP_MAPPINGS, }; -/* Pre-M4 wire layouts are shared; their command constants still need traces. */ -static const struct ave_variant ave_t6000 = { - .regs = &ave_coproc_v40, +/* Panda/Castor register family used by 6070.11.1. */ +static const struct ave_coproc_regs ave_coproc_6070 = { + .vcbar = 0x50000, + .timebase_freq = 0x160020, + .timebase = 0x178000, + .ctrl = 0x400044, + .status = 0x400048, + .boot_arg = 0x400400, + .keep_mappings = 0x400808, +}; + +const struct ave_variant ave_t6000 = { + .regs = &ave_coproc_6070, .fw_text_size = 0xd0000, .fw_data_size = 0x128000, .boot_device_id = 11, .device_type = 9, .lfs_ref_scale = 1, + .text_crc = 0x460c7a7c, + .boot_arg_value = 0x400, + .timebase_multiplier = 1000000, + .vcbar_mask = 0x0000000ffffff800ULL, + .vcbar_flags = 0x0102000000000000ULL, + + .extra0_size = 0x24, + .boot_qualified = true, }; -static const struct ave_variant ave_t8112 = { - .regs = &ave_coproc_v40, +const struct ave_variant ave_t8112 = { + .regs = &ave_coproc_6070, .fw_text_size = 0xd0000, .fw_data_size = 0x128000, .boot_device_id = 15, .device_type = 13, .lfs_ref_scale = 1, + .text_crc = 0x25bfd039, + .boot_arg_value = 0x10000, + .timebase_multiplier = 1, + .vcbar_mask = 0x000003fffffff800ULL, + .vcbar_flags = 0x0102000000000000ULL, + + .extra0_size = 0x4000, + .boot_qualified = true, }; /* CRC, boot-config words and coprocessor start registers remain unqualified. */ @@ -65,6 +93,10 @@ static const struct ave_variant ave_t8140 = { .fw_text_size = 0x14c000, .fw_data_size = 0x130000, .boot_device_id = 29, + .boot_arg_value = AVE_COPROC_BOOT_ARG_VALUE, + .timebase_multiplier = 1, + .vcbar_mask = U64_MAX, + .vcbar_flags = AVE_COPROC_VCBAR_VALID, .device_type = 27, .lfs_ref_scale = 2, .boot_qualified = true, @@ -75,8 +107,8 @@ static const struct ave_variant ave_t8140 = { * last_process0_{h264,hevc} the first Process of the last session, so the * generated commands can be compared with reference captures. */ -void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, enum ave_image kind, - const void *cmd, size_t len) +void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, + enum ave_image kind, const void *cmd, size_t len) { void *dst = ave->dump[codec - 1][kind]; @@ -84,9 +116,30 @@ void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, enum ave_image memcpy(dst, cmd, len); } +static ssize_t ave_lifecycle_write(struct file *file, const char __user *user, + size_t count, loff_t *ppos) +{ + struct ave_dev *ave = file->private_data; + unsigned int codec; + int ret = kstrtouint_from_user(user, count, 0, &codec); + + if (ret) + return ret; + if (codec != AVE_CODEC_H264 && codec != AVE_CODEC_HEVC) + return -EINVAL; + ret = ave_fw_lifecycle_test(ave, codec); + return ret ?: count; +} + +static const struct file_operations ave_lifecycle_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = ave_lifecycle_write, +}; + static void ave_debugfs_init(struct ave_dev *ave) { - static const char * const names[2][AVE_IMAGE_COUNT] = { + static const char *const names[2][AVE_IMAGE_COUNT] = { { "last_start_h264", "last_process0_h264" }, { "last_start_hevc", "last_process0_hevc" }, }; @@ -94,24 +147,33 @@ static void ave_debugfs_init(struct ave_dev *ave) if (!IS_ENABLED(CONFIG_DEBUG_FS) || !ave->layout) return; ave->debugfs = debugfs_create_dir("apple-ave", NULL); + if (ave->layout->abi == &ave_abi_6070) { + debugfs_create_file("lifecycle_test", 0200, ave->debugfs, ave, + &ave_lifecycle_fops); + return; + } for (unsigned int c = 0; c < 2; c++) for (unsigned int k = 0; k < AVE_IMAGE_COUNT; k++) { struct debugfs_blob_wrapper *blob; size_t size = ave->layout->bytes[c][k]; - blob = devm_kzalloc(ave->dev, sizeof(*blob), GFP_KERNEL); - ave->dump[c][k] = devm_kzalloc(ave->dev, size, GFP_KERNEL); + blob = devm_kzalloc(ave->dev, sizeof(*blob), + GFP_KERNEL); + ave->dump[c][k] = + devm_kzalloc(ave->dev, size, GFP_KERNEL); if (!blob || !ave->dump[c][k]) { ave->dump[c][k] = NULL; continue; } blob->data = ave->dump[c][k]; blob->size = ave->layout->bytes[c][k]; - debugfs_create_blob(names[c][k], 0400, ave->debugfs, blob); + debugfs_create_blob(names[c][k], 0400, ave->debugfs, + blob); } } -static const char * const ave_pd_names[] = { "dma", "pipe4", "pipe5", "me0", "me1" }; +static const char *const ave_pd_names[] = { "dma", "pipe4", "pipe5", "me0", + "me1" }; static int ave_probe(struct platform_device *pdev) { @@ -205,7 +267,8 @@ static void ave_remove(struct platform_device *pdev) mutex_lock(&ave->hw_lock); if (ave->fw_state != AVE_FW_OFF) { mutex_unlock(&ave->hw_lock); - dev_warn(ave->dev, "firmware was started, keeping its memory\n"); + dev_warn(ave->dev, + "firmware was started, keeping its memory\n"); return; } mutex_unlock(&ave->hw_lock); @@ -219,12 +282,21 @@ static int ave_runtime_suspend(struct device *dev) mutex_lock(&ave->hw_lock); ret = ave_fw_shutdown(ave); + if (!ret && ave->layout && ave->layout->abi->transport->scratch_halt) + ave_fw_release_boot(ave); mutex_unlock(&ave->hw_lock); return ret; } static int ave_runtime_resume(struct device *dev) { + struct ave_dev *ave = dev_get_drvdata(dev); + + mutex_lock(&ave->hw_lock); + if (ave->layout && ave->layout->abi->transport->scratch_halt && + ave->fw_state == AVE_FW_HALTED) + ave_fw_lost(ave); + mutex_unlock(&ave->hw_lock); return 0; } @@ -244,6 +316,8 @@ static int ave_suspend(struct device *dev) mutex_lock(&ave->hw_lock); ave_v4l2_suspend(ave); ret = ave_fw_shutdown(ave); + if (!ret && ave->layout && ave->layout->abi->transport->scratch_halt) + ave_fw_release_boot(ave); mutex_unlock(&ave->hw_lock); if (ret) { /* @@ -262,13 +336,14 @@ static int ave_resume(struct device *dev) struct ave_dev *ave = dev_get_drvdata(dev); /* The next boot checks the coprocessor state itself. */ + ave_runtime_resume(dev); v4l2_m2m_resume(ave->m2m_dev); return 0; } static const struct dev_pm_ops ave_pm_ops = { SYSTEM_SLEEP_PM_OPS(ave_suspend, ave_resume) - RUNTIME_PM_OPS(ave_runtime_suspend, ave_runtime_resume, NULL) + RUNTIME_PM_OPS(ave_runtime_suspend, ave_runtime_resume, NULL) }; static const struct of_device_id ave_of_match[] = { @@ -276,7 +351,7 @@ static const struct of_device_id ave_of_match[] = { { .compatible = "apple,t6041-ave", .data = &ave_t6041 }, { .compatible = "apple,t8112-ave", .data = &ave_t8112 }, { .compatible = "apple,t8140-ave", .data = &ave_t8140 }, - { } + {} }; MODULE_DEVICE_TABLE(of, ave_of_match); diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c index 2e1be782df86a..448200b2794d3 100644 --- a/drivers/media/platform/apple/ave/ave-fw.c +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -17,10 +17,13 @@ #include #include #include +#include #include #include +#include #include #include +#include #include #include "ave.h" @@ -31,8 +34,10 @@ * a layout from ave_layout_for_crc(); command formats change between builds. */ static const u32 ave_known_text_crc[] = { - 0x2ce7359e, /* AppleAVE2FW 9003.78.0 */ - 0x69c54454, /* AppleAVE2FW 9003.78.0 (macOS 26.0.x) */ + 0x2ce7359e, /* AppleAVE2FW 9003.78.0 */ + 0x69c54454, /* AppleAVE2FW 9003.78.0 (macOS 26.0.x) */ + 0x460c7a7c, /* AppleAVE2FW 6070.11.1 / T6000 */ + 0x25bfd039, /* AppleAVE2FW 6070.11.1 / T8112 */ }; static int ave_fw_chosen_crc(struct ave_dev *ave, const char *name, u32 *crc) @@ -91,10 +96,12 @@ static int ave_fw_segment(struct ave_dev *ave, unsigned int index, return ret; } -static int ave_fw_check_mapping(struct ave_dev *ave, const struct ave_fw_segment *seg) +static int ave_fw_check_mapping(struct ave_dev *ave, + const struct ave_fw_segment *seg) { - for (size_t off = 0; off < seg->size; off += PAGE_SIZE) - if (iommu_iova_to_phys(ave->domain, seg->iova + off) != seg->phys + off) + for (size_t off = 0; off < seg->size; off += SZ_16K) + if (iommu_iova_to_phys(ave->domain, seg->iova + off) != + seg->phys + off) return -EIO; return 0; } @@ -123,7 +130,8 @@ int ave_fw_probe(struct ave_dev *ave) bool known = !ave->variant->boot_qualified; int ret; - if (of_count_phandle_with_args(dev->of_node, "memory-region", NULL) != 2) + if (of_count_phandle_with_args(dev->of_node, "memory-region", NULL) != + 2) return dev_err_probe(dev, -ENODEV, "no preloaded firmware\n"); ret = ave_fw_segment(ave, 0, &ave->text); @@ -134,26 +142,34 @@ int ave_fw_probe(struct ave_dev *ave) if (ave->text.size != ave->variant->fw_text_size || ave->data.size != ave->variant->fw_data_size) - return dev_err_probe(dev, -EINVAL, "unexpected firmware segment sizes\n"); + return dev_err_probe(dev, -EINVAL, + "unexpected firmware segment sizes\n"); if ((ave->text.phys < ave->data.phys + ave->data.size && ave->data.phys < ave->text.phys + ave->text.size) || (ave->text.iova < ave->data.iova + ave->data.size && ave->data.iova < ave->text.iova + ave->text.size)) - return dev_err_probe(dev, -EINVAL, "overlapping firmware segments\n"); + return dev_err_probe(dev, -EINVAL, + "overlapping firmware segments\n"); - if (ave_fw_check_mapping(ave, &ave->text) || ave_fw_check_mapping(ave, &ave->data)) - return dev_err_probe(dev, -EIO, "firmware is not mapped where it was loaded\n"); + if (ave_fw_check_mapping(ave, &ave->text) || + ave_fw_check_mapping(ave, &ave->data)) + return dev_err_probe( + dev, -EIO, + "firmware is not mapped where it was loaded\n"); ret = ave_fw_crc(&ave->text, &text_crc, NULL); if (ret) return ret; ret = ave_fw_chosen_crc(ave, "aurora,ave-text-crc32", &want); if (ret || text_crc != want) - return dev_err_probe(dev, -ENODEV, "firmware text differs from loader CRC\n"); + return dev_err_probe(dev, -ENODEV, + "firmware text differs from loader CRC\n"); for (unsigned int i = 0; i < ARRAY_SIZE(ave_known_text_crc); i++) known |= text_crc == ave_known_text_crc[i]; - if (!known) - return dev_err_probe(dev, -ENODEV, "unsupported firmware (text CRC %08x)\n", + if (!known || + (ave->variant->text_crc && text_crc != ave->variant->text_crc)) + return dev_err_probe(dev, -ENODEV, + "unsupported firmware (text CRC %08x)\n", text_crc); ave->text_crc = text_crc; ave->layout = ave_layout_for_crc(text_crc); @@ -171,15 +187,23 @@ int ave_fw_probe(struct ave_dev *ave) */ ret = ave_fw_chosen_crc(ave, "aurora,ave-data-crc32", &want); if (ret || data_crc != want) { - ret = dev_err_probe(dev, -ENODEV, "firmware data is not in its boot state\n"); + ret = dev_err_probe(dev, -ENODEV, + "firmware data is not in its boot state\n"); goto free_data; } ave->data_crc = data_crc; ave->fw_state = AVE_FW_OFF; if (!ave->variant->boot_qualified || !ave->layout) - dev_info(dev, "probe only: firmware TEXT %08x DATA %08x needs qualification\n", - text_crc, data_crc); + dev_info( + dev, + "probe only: firmware TEXT %08x DATA %08x needs qualification\n", + text_crc, data_crc); + else if (!ave->layout->encode_ready) + dev_info( + dev, + "%s: TEXT %08x pristine DATA %08x; encoding disabled\n", + ave->layout->name, text_crc, data_crc); return 0; free_data: @@ -188,20 +212,31 @@ int ave_fw_probe(struct ave_dev *ave) return ret; } -static void ave_fw_restore_data(struct ave_dev *ave) +static int ave_fw_restore_data(struct ave_dev *ave) { void *va; if (!ave->data_dirty) - return; + return 0; va = memremap(ave->data.phys, ave->data.size, MEMREMAP_WC); - if (WARN_ON(!va)) - return; + if (!va) + return -ENOMEM; memcpy(va, ave->data_pristine, ave->data.size); /* The restored image must be in memory before the coprocessor starts. */ wmb(); memunmap(va); + /* Verify the restored bytes before allowing another cold boot. */ + { + u32 crc; + int ret = ave_fw_crc(&ave->data, &crc, NULL); + + if (ret || crc != ave->data_crc) + return ret ?: -EIO; + } + if (ave->layout->abi->transport->scratch_halt) + dev_dbg(ave->dev, "restored DATA CRC %08x\n", ave->data_crc); ave->data_dirty = false; + return 0; } void ave_free(struct ave_dev *ave, struct ave_buf *b) @@ -215,15 +250,16 @@ void ave_free(struct ave_dev *ave, struct ave_buf *b) * Firmware-visible memory, zeroed and 16 KiB aligned in the encoder's DART. * @size is what the firmware is told; @alloc (at least @size) is mapped. */ -int ave_alloc_padded(struct ave_dev *ave, struct ave_buf *b, size_t size, size_t alloc, - unsigned long attrs) +int ave_alloc_padded(struct ave_dev *ave, struct ave_buf *b, size_t size, + size_t alloc, unsigned long attrs) { if (b->cpu) return 0; b->size = size; b->alloc = PAGE_ALIGN(max(size, alloc)); b->attrs = attrs; - b->cpu = dma_alloc_attrs(ave->dev, b->alloc, &b->dma, GFP_KERNEL, attrs); + b->cpu = + dma_alloc_attrs(ave->dev, b->alloc, &b->dma, GFP_KERNEL, attrs); if (!b->cpu) return -ENOMEM; if (!IS_ALIGNED(b->dma, SZ_16K)) { @@ -233,41 +269,98 @@ int ave_alloc_padded(struct ave_dev *ave, struct ave_buf *b, size_t size, size_t return 0; } -int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, unsigned long attrs) +int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, + unsigned long attrs) { return ave_alloc_padded(ave, b, size, size, attrs); } /* - * The driver generates Start and Process for one firmware build; other - * builds may lay them out differently, so they are not driven. + * Lifecycle support and streaming qualification are independent: a known + * transport is not enough to send unqualified Start/Process payloads. */ -int ave_fw_check_build(struct ave_dev *ave) +int ave_fw_check_boot(struct ave_dev *ave) { if (ave->variant->boot_qualified && ave->variant->regs && ave->layout && ave->layout == ave_layout_for_crc(ave->text_crc)) return 0; - dev_info(ave->dev, "encoding refused: firmware TEXT %08x DATA %08x has no qualified command layout/boot sequence\n", - ave->text_crc, ave->data_crc); - return -ENOTSUPP; + return -EOPNOTSUPP; } -void ave_fw_remove(struct ave_dev *ave) +int ave_fw_check_build(struct ave_dev *ave) +{ + if (!ave_fw_check_boot(ave) && ave->layout->encode_ready) + return 0; + dev_info( + ave->dev, + "encoding refused: firmware TEXT %08x DATA %08x has no qualified command payload\n", + ave->text_crc, ave->data_crc); + return -EOPNOTSUPP; +} + +/* Only after confirmed Halt, or before the firmware was ever started. */ +void ave_fw_release_boot(struct ave_dev *ave) { + if (ave->extra0_mapped) { + dma_unmap_resource(ave->dev, ave->extra0_dma, + ave->extra0_map_size, DMA_BIDIRECTIONAL, 0); + ave->extra0_mapped = false; + } ave_free(ave, &ave->heap); ave_free(ave, &ave->log); ave_free(ave, &ave->boot_cfg); ave_free(ave, &ave->ipc); + if (ave->module_pinned) { + ave->module_pinned = false; + module_put(THIS_MODULE); + } +} + +static int ave_fw_map_extra0(struct ave_dev *ave) +{ + struct platform_device *pdev = to_platform_device(ave->dev); + struct resource *res; + + if (!ave->variant->extra0_size || ave->extra0_mapped) + return 0; + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "extra0"); + if (!res) + res = platform_get_resource(pdev, IORESOURCE_MEM, 3); + if (!res || resource_size(res) < ave->variant->extra0_size || + !IS_ALIGNED(res->start, SZ_16K)) + return dev_err_probe( + ave->dev, -EINVAL, + "missing or invalid extra0 register resource\n"); + /* The platform resource supplies the translated physical address. */ + ave->extra0_map_size = ALIGN(ave->variant->extra0_size, SZ_16K); + ave->extra0_dma = dma_map_resource(ave->dev, res->start, + ave->extra0_map_size, + DMA_BIDIRECTIONAL, 0); + if (dma_mapping_error(ave->dev, ave->extra0_dma)) + return -EIO; + ave->extra0_mapped = true; + return 0; +} + +void ave_fw_remove(struct ave_dev *ave) +{ + ave_fw_release_boot(ave); vfree(ave->data_pristine); ave->data_pristine = NULL; } +u64 ave_fw_vcbar(const struct ave_variant *variant, u64 text_iova) +{ + return (text_iova & variant->vcbar_mask) | variant->vcbar_flags; +} + static bool ave_fw_idle(struct ave_dev *ave) { if (readq(ave->coproc + ave->variant->regs->vcbar) != - (ave->text.iova | AVE_COPROC_VCBAR_VALID)) + ave_fw_vcbar(ave->variant, ave->text.iova)) return false; - if (readl(ave->coproc + ave->variant->regs->ctrl) || readl(ave->mbox + AVE_MBOX_STATUS)) + if (readl(ave->coproc + ave->variant->regs->ctrl) || + readl(ave->mbox + AVE_MBOX_STATUS)) return false; for (unsigned int i = 0; i < AVE_MBOX_SCRATCH_COUNT; i++) if (readl(ave->mbox + AVE_MBOX_SCRATCH(i))) @@ -281,15 +374,28 @@ static bool ave_fw_idle(struct ave_dev *ave) * between, which puts the coprocessor back in reset. Hardware runs showed * that a restart while it kept power gets no answer, so that is refused * (-EAGAIN) and the state stays halted until the power really went. The - * code base register must still be set: the driver never writes it. + * 9003 code base register must still be set: that ABI uses the loader value. */ static int ave_fw_prepare_restart(struct ave_dev *ave) { u32 status; + if (!ave->layout->abi->transport->boot_config) { + if (ave->fw_state == AVE_FW_HALTED) + return -EAGAIN; /* A runtime PM suspend/resume must intervene. */ + if (readl(ave->coproc + ave->variant->regs->ctrl)) + return -EBUSY; + status = readl(ave->mbox + AVE_MBOX_STATUS); + if (status) + writel(status, ave->mbox + AVE_MBOX_STATUS); + for (unsigned int i = 0; i < AVE_MBOX_SCRATCH_COUNT; i++) + writel(0, ave->mbox + AVE_MBOX_SCRATCH(i)); + return 0; + } if (readq(ave->coproc + ave->variant->regs->vcbar) != - (ave->text.iova | AVE_COPROC_VCBAR_VALID)) { - dev_err(ave->dev, "coprocessor lost its code base (vcbar %#llx)\n", + ave_fw_vcbar(ave->variant, ave->text.iova)) { + dev_err(ave->dev, + "coprocessor lost its code base (vcbar %#llx)\n", readq(ave->coproc + ave->variant->regs->vcbar)); return -EIO; } @@ -297,7 +403,8 @@ static int ave_fw_prepare_restart(struct ave_dev *ave) return 0; if (readl(ave->coproc + ave->variant->regs->ctrl)) { - dev_dbg(ave->dev, "coprocessor kept power after its halt; retry once it suspends\n"); + dev_dbg(ave->dev, + "coprocessor kept power after its halt; retry once it suspends\n"); return -EAGAIN; } status = readl(ave->mbox + AVE_MBOX_STATUS); @@ -311,19 +418,23 @@ static int ave_fw_prepare_restart(struct ave_dev *ave) /* Boot the coprocessor and bring up its command rings. Caller holds hw_lock. */ static int ave_fw_boot(struct ave_dev *ave) { + const struct ave_fw_transport *t = ave->layout->abi->transport; __le32 *cfg, *log; u32 status, hello[4]; int ret; - ret = ave_alloc(ave, &ave->ipc, AVE_IPC_BYTES, 0); - if (!ret) + ret = ave_alloc(ave, &ave->ipc, t->ipc_bytes, 0); + if (!ret && t->boot_config) ret = ave_alloc(ave, &ave->boot_cfg, SZ_16K, 0); - if (!ret) + if (!ret && t->boot_config) ret = ave_alloc(ave, &ave->log, AVE_LOG_BYTES, 0); + if (!ret) + ret = ave_fw_map_extra0(ave); if (ret) return ret; - if (ave_fw_check_mapping(ave, &ave->text) || ave_fw_check_mapping(ave, &ave->data)) { + if (ave_fw_check_mapping(ave, &ave->text) || + ave_fw_check_mapping(ave, &ave->data)) { dev_err(ave->dev, "firmware segments are no longer mapped\n"); return -EIO; } @@ -336,8 +447,9 @@ static int ave_fw_boot(struct ave_dev *ave) * The bootloader leaves the coprocessor stopped with its code base * set. Do not start it from any other state. */ - if (!ave_fw_idle(ave)) { - dev_err(ave->dev, "coprocessor is not in its reset state (vcbar %#llx ctrl %#x)\n", + if (t->boot_config && !ave_fw_idle(ave)) { + dev_err(ave->dev, + "coprocessor is not in its reset state (vcbar %#llx ctrl %#x)\n", readq(ave->coproc + ave->variant->regs->vcbar), readl(ave->coproc + ave->variant->regs->ctrl)); return -EIO; @@ -349,37 +461,70 @@ static int ave_fw_boot(struct ave_dev *ave) ave->reply_err = 0; ave->scalar_next = 0; - ave_fw_restore_data(ave); - - cfg = ave->boot_cfg.cpu; - log = ave->log.cpu; - memset(cfg, 0, AVE_BOOT_CFG_BYTES); - memset(log, 0, ave->log.size); - cfg[AVE_BOOT_CFG_DEVICE] = cpu_to_le32(ave->variant->boot_device_id); - cfg[AVE_BOOT_CFG_UNITS] = cpu_to_le32(1); - cfg[AVE_BOOT_CFG_GROUPS] = cpu_to_le32(1); - cfg[AVE_BOOT_CFG_SUBID_MASK] = cpu_to_le32(1); - cfg[AVE_BOOT_CFG_LOG_LO] = cpu_to_le32(lower_32_bits(ave->log.dma)); - cfg[AVE_BOOT_CFG_LOG_HI] = cpu_to_le32(upper_32_bits(ave->log.dma)); - cfg[AVE_BOOT_CFG_LOG_BYTES] = cpu_to_le32(AVE_LOG_BYTES); - cfg[AVE_BOOT_CFG_OPTIONS] = cpu_to_le32(AVE_BOOT_CFG_OPTIONS_VALUE); - log[0] = cpu_to_le32(AVE_LOG_HDR_USED); - log[1] = cpu_to_le32(AVE_LOG_HDR_BYTES); - log[2] = cpu_to_le32(AVE_LOG_RING_BYTES); - dma_wmb(); - - writel(AVE_BOOT_MAGIC, ave->mbox + AVE_MBOX_SCRATCH(0)); - writel(lower_32_bits(ave->boot_cfg.dma), ave->mbox + AVE_MBOX_SCRATCH(1)); - writel(upper_32_bits(ave->boot_cfg.dma), ave->mbox + AVE_MBOX_SCRATCH(2)); + ret = ave_fw_restore_data(ave); + if (ret) + return ret; + if (t->boot_config) { + cfg = ave->boot_cfg.cpu; + log = ave->log.cpu; + memset(cfg, 0, AVE_BOOT_CFG_BYTES); + memset(log, 0, ave->log.size); + cfg[AVE_BOOT_CFG_DEVICE] = + cpu_to_le32(ave->variant->boot_device_id); + cfg[AVE_BOOT_CFG_UNITS] = cpu_to_le32(1); + cfg[AVE_BOOT_CFG_GROUPS] = cpu_to_le32(1); + cfg[AVE_BOOT_CFG_SUBID_MASK] = cpu_to_le32(1); + cfg[AVE_BOOT_CFG_LOG_LO] = + cpu_to_le32(lower_32_bits(ave->log.dma)); + cfg[AVE_BOOT_CFG_LOG_HI] = + cpu_to_le32(upper_32_bits(ave->log.dma)); + cfg[AVE_BOOT_CFG_LOG_BYTES] = cpu_to_le32(AVE_LOG_BYTES); + cfg[AVE_BOOT_CFG_OPTIONS] = + cpu_to_le32(AVE_BOOT_CFG_OPTIONS_VALUE); + log[0] = cpu_to_le32(AVE_LOG_HDR_USED); + log[1] = cpu_to_le32(AVE_LOG_HDR_BYTES); + log[2] = cpu_to_le32(AVE_LOG_RING_BYTES); + dma_wmb(); + + writel(AVE_BOOT_MAGIC, ave->mbox + AVE_MBOX_SCRATCH(0)); + writel(lower_32_bits(ave->boot_cfg.dma), + ave->mbox + AVE_MBOX_SCRATCH(1)); + writel(upper_32_bits(ave->boot_cfg.dma), + ave->mbox + AVE_MBOX_SCRATCH(2)); + + } else { + writel(AVE_BOOT_MAGIC, ave->mbox + AVE_MBOX_SCRATCH(0)); + writel(0, ave->mbox + AVE_MBOX_SCRATCH(1)); + writel(ave->variant->boot_device_id, + ave->mbox + AVE_MBOX_SCRATCH(2)); + } + if (!t->boot_config) { + writeq(ave_fw_vcbar(ave->variant, ave->text.iova), + ave->coproc + ave->variant->regs->vcbar); + if (readq(ave->coproc + ave->variant->regs->vcbar) != + ave_fw_vcbar(ave->variant, ave->text.iova)) + return -EIO; + dev_dbg(ave->dev, "VCBAR %#llx boot argument %#x\n", + ave_fw_vcbar(ave->variant, ave->text.iova), + ave->variant->boot_arg_value); + } + + if (!t->boot_config && !ave->module_pinned) { + if (!try_module_get(THIS_MODULE)) + return -ENODEV; + ave->module_pinned = true; + } ave->data_dirty = true; writel(1, ave->coproc + ave->variant->regs->keep_mappings); writel(0, ave->coproc + ave->variant->regs->ctrl); - writel(AVE_COPROC_BOOT_ARG_VALUE, ave->coproc + ave->variant->regs->boot_arg); + writel(ave->variant->boot_arg_value, + ave->coproc + ave->variant->regs->boot_arg); writel(AVE_COPROC_CTRL_RUN, ave->coproc + ave->variant->regs->ctrl); ret = readl_poll_timeout(ave->mbox + AVE_MBOX_STATUS, status, - status & AVE_MBOX_STATUS_PENDING, 50, AVE_MBOX_TIMEOUT_US); + status & AVE_MBOX_STATUS_PENDING, 50, + AVE_MBOX_TIMEOUT_US); if (ret) { dev_err(ave->dev, "firmware did not answer (status %#x)\n", readl(ave->coproc + ave->variant->regs->status)); @@ -412,14 +557,15 @@ int ave_fw_ensure_running(struct ave_dev *ave) int ret; lockdep_assert_held(&ave->hw_lock); - ret = ave_fw_check_build(ave); + ret = ave_fw_check_boot(ave); if (ret) return ret; switch (ave->fw_state) { case AVE_FW_RUNNING: return 0; case AVE_FW_BROKEN: - dev_err(ave->dev, "firmware stopped answering earlier; reboot to use the encoder\n"); + dev_err(ave->dev, + "firmware stopped answering earlier; reboot to use the encoder\n"); return -EIO; case AVE_FW_OFF: case AVE_FW_HALTED: @@ -434,8 +580,17 @@ int ave_fw_ensure_running(struct ave_dev *ave) prev = ave->fw_state; ret = ave_fw_boot(ave); if (ret) { + if (ave->layout->abi->transport->scratch_halt && + (ave->module_pinned || + readl(ave->coproc + ave->variant->regs->ctrl))) { + if (!ave->module_pinned && try_module_get(THIS_MODULE)) + ave->module_pinned = true; + ave->fw_state = AVE_FW_BROKEN; + return ret; + } ave->fw_state = readl(ave->coproc + ave->variant->regs->ctrl) ? - AVE_FW_HALTED : prev; + AVE_FW_HALTED : + prev; memset(ave->ring, 0, sizeof(ave->ring)); ave->ipc_fw_base = 0; } @@ -444,8 +599,8 @@ int ave_fw_ensure_running(struct ave_dev *ave) /* * Tell the firmware to stop, once every session is closed. Halt gets no - * ack and no answer; after a short wait the coprocessor is treated as - * stopped and the next session restarts it through the boot path. + * ring reply. 6070 confirms quiescence through scratch0 before the host + * stops the IOP; 9003 retains its existing settle interval. * Caller holds hw_lock. */ void ave_fw_halt(struct ave_dev *ave) @@ -453,12 +608,48 @@ void ave_fw_halt(struct ave_dev *ave) lockdep_assert_held(&ave->hw_lock); if (ave->fw_state != AVE_FW_RUNNING) return; + if (ave->layout->abi->transport->scratch_halt) { + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i] && + ave->sessions[i]->sess->fw_open) { + ave->fw_state = AVE_FW_BROKEN; + return; + } + writel(0, ave->mbox + AVE_MBOX_SCRATCH(0)); + } if (ave_cmd_halt(ave)) { - dev_err(ave->dev, "cannot send Halt; firmware treated as stuck\n"); + dev_err(ave->dev, + "cannot send Halt; firmware treated as stuck\n"); ave->fw_state = AVE_FW_BROKEN; return; } - msleep(AVE_HALT_SETTLE_MS); + if (ave->layout->abi->transport->scratch_halt) { + u32 status; + int ret; + + ret = readl_poll_timeout(ave->mbox + AVE_MBOX_SCRATCH(0), + status, status == AVE_BOOT_MAGIC, 100, + AVE_MBOX_TIMEOUT_US); + if (ret) { + dev_err(ave->dev, + "Halt not confirmed; retaining firmware mappings and power\n"); + ave->fw_state = AVE_FW_BROKEN; + return; + } + dev_dbg(ave->dev, "Halt confirmed: scratch0 %#x\n", status); + /* Scratch confirmation establishes quiescence; stop and read back the IOP. + */ + writel(0, ave->coproc + ave->variant->regs->ctrl); + ret = readl_poll_timeout(ave->coproc + ave->variant->regs->ctrl, + status, !status, 100, + AVE_MBOX_TIMEOUT_US); + if (ret) { + ave->fw_state = AVE_FW_BROKEN; + return; + } + } else { + msleep(AVE_HALT_SETTLE_MS); + } ave->fw_state = AVE_FW_HALTED; memset(ave->ring, 0, sizeof(ave->ring)); ave->ipc_fw_base = 0; @@ -494,3 +685,63 @@ void ave_fw_lost(struct ave_dev *ave) memset(ave->ring, 0, sizeof(ave->ring)); ave->ipc_fw_base = 0; } + +/* Explicit lifecycle-only validation, without allocating encoder surfaces. */ +int ave_fw_lifecycle_test(struct ave_dev *ave, enum ave_codec codec) +{ + struct ave_ctx *ctx; + struct ave_session *sess; + int ret; + + if (!ave->layout || ave->layout->abi != &ave_abi_6070) + return -EOPNOTSUPP; + ctx = kzalloc_obj(*ctx); + sess = kzalloc_obj(*sess); + if (!ctx || !sess) { + ret = -ENOMEM; + goto free; + } + ctx->ave = ave; + ctx->sess = sess; + sess->p.codec = codec; + ret = pm_runtime_resume_and_get(ave->dev); + if (ret < 0) + goto free; + mutex_lock(&ave->hw_lock); + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i]) { + ret = -EBUSY; + goto unlock; + } + ret = ave_fw_ensure_running(ave); + if (!ret) + ret = ave_cmd_open(ctx); + if (!ret) + ret = ave_cmd_close(ctx); + if (!ret) { + ave_fw_halt(ave); + if (ave->fw_state != AVE_FW_HALTED) + ret = -EIO; + } else if (ave->fw_state == AVE_FW_RUNNING) { + /* Unconfirmed client lifetime: do not Halt or free anything it may own. */ + ave->fw_state = AVE_FW_BROKEN; + } +unlock: + mutex_unlock(&ave->hw_lock); + /* Runtime suspend refuses broken firmware, retaining its DMA and power. */ + if (ave->fw_state != AVE_FW_BROKEN) { + int err = pm_runtime_put_sync_suspend(ave->dev); + + if (!ret && err < 0) + ret = err; + } else { + dev_err(ave->dev, + "lifecycle failed (%d); retaining runtime power reference\n", + ret); + } + dev_info(ave->dev, "6070 lifecycle codec %u: %d\n", codec, ret); +free: + kfree(sess); + kfree(ctx); + return ret; +} diff --git a/drivers/media/platform/apple/ave/ave-ipc.c b/drivers/media/platform/apple/ave/ave-ipc.c index c197671debbbe..c131ffc62b64e 100644 --- a/drivers/media/platform/apple/ave/ave-ipc.c +++ b/drivers/media/platform/apple/ave/ave-ipc.c @@ -63,12 +63,14 @@ static int ave_mbox_recv(struct ave_dev *ave, u32 reply[4]) return 0; } -static int ave_ring_setup(struct ave_dev *ave) +int ave_ipc_parse_channels(struct ave_dev *ave) { bool found[2] = { false, false }; - for (unsigned int i = 0; i < 2; i++) { - u8 *desc = ave->ipc.cpu + AVE_IPC_CHANNEL_OFF + i * AVE_CHAN_DESC_STRIDE; + for (unsigned int i = 0; i < ave->layout->abi->transport->descriptors; + i++) { + u8 *desc = ave->ipc.cpu + AVE_IPC_CHANNEL_OFF + + i * AVE_CHAN_DESC_STRIDE; u32 dir = get_unaligned_le32(desc + AVE_CHAN_DESC_DIR); u32 doorbell = get_unaligned_le32(desc + AVE_CHAN_DESC_IRQ); u32 slots = get_unaligned_le32(desc + AVE_CHAN_DESC_SLOTS); @@ -77,21 +79,33 @@ static int ave_ring_setup(struct ave_dev *ave) unsigned int id; u32 offset; - if (!memchr(desc, 0, AVE_CHAN_DESC_NAME_LEN)) + const struct ave_fw_transport *t = ave->layout->abi->transport; + + if (!memchr(desc, 0, AVE_CHAN_DESC_NAME_LEN) || + slots > ave->channel_bytes / AVE_RING_SLOT_BYTES || + !IS_ALIGNED(addr, AVE_RING_SLOT_BYTES) || + ave_fw_offset(ave, addr, + (size_t)slots * AVE_RING_SLOT_BYTES, + &offset) || + offset < AVE_IPC_CHANNEL_OFF + + t->descriptors * AVE_CHAN_DESC_STRIDE || + offset + slots * AVE_RING_SLOT_BYTES > + AVE_IPC_CHANNEL_OFF + ave->channel_bytes) return -EPROTO; if (!strcmp(desc, "IO")) id = 0; else if (!strcmp(desc, "IO_T2H")) id = 1; + else if (t->descriptors > 2) + continue; else return -EPROTO; - - if (found[id] || dir != id || !doorbell || doorbell >= 32 || !slots || - slots > ave->channel_bytes / AVE_RING_SLOT_BYTES || - !IS_ALIGNED(addr, AVE_RING_SLOT_BYTES) || - ave_fw_offset(ave, addr, (size_t)slots * AVE_RING_SLOT_BYTES, &offset) || - offset < AVE_IPC_CHANNEL_OFF + 2 * AVE_CHAN_DESC_STRIDE || - offset + slots * AVE_RING_SLOT_BYTES > AVE_IPC_CHANNEL_OFF + ave->channel_bytes) + if (found[id] || dir != id || !doorbell || doorbell >= 32 || + !slots || + (t->slots[id] && + (slots != t->slots[id] || doorbell != t->doorbells[id] || + offset != AVE_IPC_CHANNEL_OFF + t->offsets[id] || + i != (id ? t->t2h_index : t->io_index)))) return -EPROTO; found[id] = true; @@ -108,13 +122,18 @@ static int ave_ring_setup(struct ave_dev *ave) if (!found[0] || !found[1]) return -EPROTO; - if (ave->ring[0].offset < ave->ring[1].offset + ave->ring[1].slots * AVE_RING_SLOT_BYTES && - ave->ring[1].offset < ave->ring[0].offset + ave->ring[0].slots * AVE_RING_SLOT_BYTES) + if (ave->ring[0].offset < + ave->ring[1].offset + + ave->ring[1].slots * AVE_RING_SLOT_BYTES && + ave->ring[1].offset < + ave->ring[0].offset + + ave->ring[0].slots * AVE_RING_SLOT_BYTES) return -EPROTO; /* Hand every host-to-firmware slot to the host; ownership is bit 0. */ for (unsigned int i = 0; i < ave->ring[0].slots; i++) { - __le64 *slot = ave->ipc.cpu + ave->ring[0].offset + i * AVE_RING_SLOT_BYTES; + __le64 *slot = ave->ipc.cpu + ave->ring[0].offset + + i * AVE_RING_SLOT_BYTES; slot[0] = cpu_to_le64(1); slot[1] = 0; @@ -129,45 +148,71 @@ static int ave_ring_setup(struct ave_dev *ave) * request: hand it the heap and the IPC window, then the channel area, and * read back the two ring descriptors it writes there. */ +int ave_ipc_check_hello(const struct ave_fw_transport *t, const u32 hello[4]) +{ + if (hello[0] != t->hello[0] || hello[2] != t->hello[2] || + hello[1] < t->descriptors * AVE_CHAN_DESC_STRIDE || + hello[1] > AVE_IPC_HANDSHAKE_OFF - AVE_IPC_CHANNEL_OFF) + return -EPROTO; + if (t->boot_config) + return !hello[3] || hello[3] > 4 * SZ_1M ? -EPROTO : 0; + return hello[1] != t->hello[1] || hello[3] != t->hello[3] ? -EPROTO : 0; +} + +static int ave_iop_time_us(struct ave_dev *ave, u64 *us) +{ + u64 ticks = readq(ave->coproc + ave->variant->regs->timebase); + u64 freq = readl(ave->coproc + ave->variant->regs->timebase_freq); + + freq *= ave->variant->timebase_multiplier; + if (freq && freq < 1000000) + return -EPROTO; + *us = div64_u64(ticks, freq ? div_u64(freq, 1000000) : 24); + return 0; +} + int ave_ipc_negotiate(struct ave_dev *ave, const u32 hello[4]) { + const struct ave_fw_transport *t = ave->layout->abi->transport; u8 *hs = ave->ipc.cpu + AVE_IPC_HANDSHAKE_OFF; u64 ticks, offset_us; - u32 freq, status, reply[4], off; + u32 status, reply[4], off; int ret; - if (hello[0] != AVE_HELLO_VERSION || hello[2] != AVE_HELLO_DESC_BYTES || - hello[1] < 2 * AVE_CHAN_DESC_STRIDE || - hello[1] > AVE_IPC_HANDSHAKE_OFF - AVE_IPC_CHANNEL_OFF || - !hello[3] || hello[3] > 4 * SZ_1M) - return -EPROTO; + ret = ave_ipc_check_hello(t, hello); + if (ret) { + dev_err(ave->dev, "unexpected hello {%#x, %#x, %#x, %#x}\n", + hello[0], hello[1], hello[2], hello[3]); + return ret; + } + if (t->scratch_halt) + dev_dbg(ave->dev, "hello {%#x, %#x, %#x, %#x}\n", hello[0], + hello[1], hello[2], hello[3]); ave->channel_bytes = hello[1]; - if (ave->heap.cpu && ave->heap.size != PAGE_ALIGN(hello[3])) - return -EPROTO; - if (!ave->heap.cpu) { - ave->heap.size = PAGE_ALIGN(hello[3]); - ave->heap.alloc = ave->heap.size; - ave->heap.cpu = dma_alloc_coherent(ave->dev, ave->heap.size, - &ave->heap.dma, GFP_KERNEL); - if (!ave->heap.cpu) - return -ENOMEM; + if (t->boot_config) { + if (ave->heap.cpu && ave->heap.size != PAGE_ALIGN(hello[3])) + return -EPROTO; + if (!ave->heap.cpu) { + ret = ave_alloc(ave, &ave->heap, PAGE_ALIGN(hello[3]), + 0); + if (ret) + return ret; + } + memset(ave->heap.cpu, 0, ave->heap.size); } - memset(ave->heap.cpu, 0, ave->heap.size); writel(AVE_MBOX_STATUS_PENDING, ave->mbox + AVE_MBOX_STATUS); + if (t->sync_host_clock) { + ret = ave_iop_time_us(ave, &ticks); + if (ret) + return ret; + offset_us = div_u64(ktime_get_ns(), NSEC_PER_USEC) - ticks; + writel(lower_32_bits(offset_us), ave->mbox + AVE_MBOX_TIME_LO); + writel(upper_32_bits(offset_us), ave->mbox + AVE_MBOX_TIME_HI); + } - /* Tell the firmware how far its timebase is from the host's. */ - ticks = readq(ave->coproc + ave->variant->regs->timebase); - freq = readl(ave->coproc + ave->variant->regs->timebase_freq); - if (freq && freq < 1000000) - return -EPROTO; - ticks = div64_u64(ticks, freq ? freq / 1000000 : 24); - offset_us = div_u64(ktime_get_ns(), NSEC_PER_USEC) - ticks; - writel(lower_32_bits(offset_us), ave->mbox + AVE_MBOX_TIME_LO); - writel(upper_32_bits(offset_us), ave->mbox + AVE_MBOX_TIME_HI); - - ave_mbox_send(ave, lower_32_bits(ave->ipc.dma), upper_32_bits(ave->ipc.dma), - ave->ipc.size, 0); + ave_mbox_send(ave, lower_32_bits(ave->ipc.dma), + upper_32_bits(ave->ipc.dma), ave->ipc.size, 0); ret = ave_mbox_recv(ave, reply); if (ret) return ret; @@ -179,30 +224,39 @@ int ave_ipc_negotiate(struct ave_dev *ave, const u32 hello[4]) memset(ave->ipc.cpu + AVE_IPC_CHANNEL_OFF, 0, ave->channel_bytes); memset(hs, 0, AVE_HS_BYTES); memset(ave->ipc.cpu + AVE_IPC_SHARED_OFF, 0, AVE_IPC_SHARED_BYTES); - put_unaligned_le64(ave->ipc_fw_base + AVE_IPC_CHANNEL_OFF, hs + AVE_HS_CHANNELS); - put_unaligned_le64(ave->ipc_fw_base + AVE_IPC_SHARED_OFF, hs + AVE_HS_SHARED); - put_unaligned_le32(AVE_IPC_SHARED_BYTES, hs + AVE_HS_SHARED_BYTES); - put_unaligned_le64(ave->heap.dma, hs + AVE_HS_HEAP); - put_unaligned_le32(hello[3], hs + AVE_HS_HEAP_BYTES); + put_unaligned_le64(ave->ipc_fw_base + AVE_IPC_CHANNEL_OFF, + hs + AVE_HS_CHANNELS); + if (t->boot_config) { + put_unaligned_le64(ave->ipc_fw_base + AVE_IPC_SHARED_OFF, + hs + AVE_HS_SHARED); + put_unaligned_le32(AVE_IPC_SHARED_BYTES, + hs + AVE_HS_SHARED_BYTES); + put_unaligned_le64(ave->heap.dma, hs + AVE_HS_HEAP); + put_unaligned_le32(hello[3], hs + AVE_HS_HEAP_BYTES); + } put_unaligned_le32(ave->variant->device_type, hs + AVE_HS_DEVICE_TYPE); dma_wmb(); - ave_mbox_send(ave, lower_32_bits(ave->ipc_fw_base + AVE_IPC_HANDSHAKE_OFF), - upper_32_bits(ave->ipc_fw_base + AVE_IPC_HANDSHAKE_OFF), 0, 0); + ave_mbox_send(ave, + lower_32_bits(ave->ipc_fw_base + AVE_IPC_HANDSHAKE_OFF), + upper_32_bits(ave->ipc_fw_base + AVE_IPC_HANDSHAKE_OFF), + 0, 0); ret = ave_mbox_recv(ave, reply); if (ret) return ret; - if (ave_fw_offset(ave, (u64)reply[1] << 32 | reply[0], ave->channel_bytes, &off) || - off != AVE_IPC_CHANNEL_OFF || reply[2] > AVE_HELLO_EXTRA_MAX) + if (ave_fw_offset(ave, (u64)reply[1] << 32 | reply[0], + ave->channel_bytes, &off) || + off != AVE_IPC_CHANNEL_OFF || + (t->boot_config && reply[2] > AVE_HELLO_EXTRA_MAX)) return -EPROTO; - ret = ave_ring_setup(ave); + ret = ave_ipc_parse_channels(ave); if (ret) return ret; writel(AVE_BOOT_MAGIC, ave->mbox + AVE_MBOX_READY); - return readl_poll_timeout(ave->mbox + AVE_MBOX_READY, status, !status, 50, - AVE_MBOX_TIMEOUT_US); + return readl_poll_timeout(ave->mbox + AVE_MBOX_READY, status, !status, + 50, AVE_MBOX_TIMEOUT_US); } static int ave_ring_send(struct ave_dev *ave, struct ave_ring *r, u64 addr, u32 len) @@ -252,30 +306,43 @@ static int ave_ring_recv(struct ave_dev *ave, struct ave_ring *r, u64 *addr, u32 return 0; } -static int ave_ipc_buf_offset(unsigned int buf, size_t *max) +static int ave_ipc_buf_offset(struct ave_dev *ave, unsigned int buf, + size_t *max) { + const struct ave_fw_transport *t = ave->layout->abi->transport; + if (buf == AVE_BUF_CMD) { - *max = AVE_IPC_CMD_BYTES; - return AVE_IPC_CMD_OFF; + *max = t->command_bytes; + return t->command_offset; } if (buf < AVE_BUF_PROC(0)) { *max = AVE_IPC_SCALAR_BYTES; - return AVE_IPC_SCALAR_OFF + (buf - AVE_BUF_SCALAR(0)) * AVE_IPC_SCALAR_BYTES; + return t->scalar_offset + + (buf - AVE_BUF_SCALAR(0)) * AVE_IPC_SCALAR_BYTES; } - *max = AVE_IPC_PROC_BYTES; - return AVE_IPC_PROC_OFF + (buf - AVE_BUF_PROC(0)) * AVE_IPC_PROC_BYTES; + *max = t->process_bytes; + return t->process_offset + (buf - AVE_BUF_PROC(0)) * t->process_bytes; } /* Which command buffer does a transport ack refer to? */ +static u32 ave_ipc_command_end(struct ave_dev *ave) +{ + const struct ave_fw_transport *t = ave->layout->abi->transport; + + return t->process_offset + AVE_IPC_PROC_BUFS * t->process_bytes; +} + static int ave_ipc_buf_of(struct ave_dev *ave, u64 addr) { u64 off = addr - ave->ipc_fw_base; size_t max; - if (addr < ave->ipc_fw_base || off < AVE_IPC_CMD_OFF || off >= AVE_IPC_END) + if (addr < ave->ipc_fw_base || + off < ave->layout->abi->transport->command_offset || + off >= ave_ipc_command_end(ave)) return -EPROTO; for (unsigned int buf = 0; buf < AVE_BUF_COUNT; buf++) - if (ave_ipc_buf_offset(buf, &max) == off) + if (ave_ipc_buf_offset(ave, buf, &max) == off) return buf; return -EPROTO; } @@ -291,8 +358,8 @@ void *ave_ipc_cmd(struct ave_dev *ave, unsigned int buf, size_t len) if (buf >= AVE_BUF_COUNT || (ave->unacked & BIT(buf))) return NULL; - off = ave_ipc_buf_offset(buf, &max); - if (len > max) + off = ave_ipc_buf_offset(ave, buf, &max); + if (len > max || off > ave->ipc.size || len > ave->ipc.size - off) return NULL; memset(ave->ipc.cpu + off, 0, len); return ave->ipc.cpu + off; @@ -344,31 +411,62 @@ static void ave_ipc_mark_broken(struct ave_dev *ave, int err, const char *what) /* * Hand the command in buffer @buf to the firmware. The timeout words are - * {duration, creation} in the host clock the firmware was synced to at boot. + * {duration, creation}, using the host clock for 9003 and the IOP clock + * for 6070. * With @reply, the firmware's answer is counted as expected; Halt and * Process are answered differently. Caller holds hw_lock. */ -int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, u64 timeout_us, - bool reply) +int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, + u64 timeout_us, bool reply) { size_t max; - int off = ave_ipc_buf_offset(buf, &max); - u8 *cmd = ave->ipc.cpu + off; - u32 op = get_unaligned_le32(cmd + AVE_CMD_OPCODE); + int off; + u8 *cmd; + const struct ave_fw_abi *abi = ave->layout->abi; + u32 wire_id; + enum ave_opcode op = 0; + u64 created; int ret; + if (buf >= AVE_BUF_COUNT) + return -EINVAL; + off = ave_ipc_buf_offset(ave, buf, &max); + if (len > max || off > ave->ipc.size || len > ave->ipc.size - off) + return -EINVAL; + cmd = ave->ipc.cpu + off; + wire_id = get_unaligned_le16(cmd); + for (unsigned int i = 1; i < AVE_OP_COUNT; i++) + if (wire_id && (abi->wire_id[0][i] == wire_id || + abi->wire_id[1][i] == wire_id)) { + op = i; + break; + } lockdep_assert_held(&ave->hw_lock); if (ave->fw_state != AVE_FW_RUNNING) { - dev_err_ratelimited(ave->dev, "command %u not sent: firmware state %d\n", op, - ave->fw_state); + dev_err_ratelimited(ave->dev, + "command %u not sent: firmware state %d\n", + op, ave->fw_state); return -EIO; } - if (len < AVE_CMD_HDR_BYTES || len > max || !IS_ALIGNED(len, 4) || - (ave->unacked & BIT(buf)) || (reply && op >= ARRAY_SIZE(ave->want))) + if (!op || len < abi->header_bytes || len > max || + !IS_ALIGNED(len, 4) || (ave->unacked & BIT(buf)) || + (reply && op >= ARRAY_SIZE(ave->want))) return -EINVAL; - put_unaligned_le64(timeout_us, cmd + AVE_CMD_TIMEOUT); - put_unaligned_le64(div_u64(ktime_get_ns(), NSEC_PER_USEC), cmd + AVE_CMD_CREATED); + if (!abi->transport->boot_config && reply && ave->want[op]) + return -EBUSY; + if (abi->transport->sync_host_clock) { + created = div_u64(ktime_get_ns(), NSEC_PER_USEC); + } else { + ret = ave_iop_time_us(ave, &created); + if (ret) + return ret; + } + ave_stamp_command(abi, cmd, timeout_us, created); + if (!abi->transport->boot_config && reply) { + ave->expected[op].client = get_unaligned_le32(cmd + 0x10); + ave->expected[op].subtype = get_unaligned_le32(cmd + 0x1c); + } ret = ave_ring_send(ave, &ave->ring[0], ave->ipc_fw_base + off, len); if (ret) { @@ -381,32 +479,53 @@ int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, u64 timeou return 0; } -static void ave_ipc_dispatch(struct ave_dev *ave) +static int ave_ipc_dispatch(struct ave_dev *ave) { - u32 id, status; + const struct ave_fw_abi *abi = ave->layout->abi; + struct ave_reply reply; + int ret = abi->parse_reply(abi, ave->msg, ave->msg_len, &reply); - if (ave->msg_len < AVE_REPLY_MIN_BYTES) { - dev_warn(ave->dev, "short firmware message (%u bytes)\n", ave->msg_len); - ave->reply_err = -EPROTO; - return; + if (ret) { + if (!abi->transport->boot_config) + return ret; + dev_warn(ave->dev, "short firmware message (%u bytes)\n", + ave->msg_len); + ave->reply_err = ret; + return 0; } - id = get_unaligned_le16(ave->msg + AVE_REPLY_ID); - status = get_unaligned_le32(ave->msg + AVE_REPLY_STATUS); - - if (id == AVE_OP_PROCESS) { - ave_sess_reply(ave, ave->msg, ave->msg_len); - return; + if (reply.op == AVE_OP_PROCESS) { + if (!ave->layout->encode_ready) + return -EPROTO; + ave_sess_reply(ave, &reply); + return 0; } - if (id < ARRAY_SIZE(ave->want) && ave->want[id]) { - ave->want[id]--; - if (status) { - dev_err(ave->dev, "command %u refused: status %#x\n", id, status); + if (reply.op && ave->want[reply.op]) { + if (!abi->transport->boot_config && + (reply.client != ave->expected[reply.op].client || + reply.subtype != ave->expected[reply.op].subtype)) { + dev_err(ave->dev, + "reply %#x has wrong route: client %llu subtype %#x\n", + abi->reply_id[reply.op], reply.client, + reply.subtype); + return -EPROTO; + } + if (abi->transport->scratch_halt) + dev_dbg(ave->dev, + "reply %#x client %llu subtype %#x status %#x\n", + abi->reply_id[reply.op], reply.client, + reply.subtype, reply.raw_status); + ave->want[reply.op]--; + if (reply.status) { + dev_err(ave->dev, "command %u refused: status %#x\n", + reply.op, reply.raw_status); ave->reply_err = -EIO; } - return; + return 0; } - dev_warn_ratelimited(ave->dev, "unexpected firmware message %u (status %#x)\n", - id, status); + dev_warn_ratelimited(ave->dev, + "unexpected firmware message (op %u status %#x)\n", + reply.op, reply.raw_status); + return abi->transport->boot_config ? 0 : -EPROTO; } /* @@ -428,7 +547,8 @@ static int ave_ipc_poll_once(struct ave_dev *ave) /* Transport acks return the command address with no payload. */ buf = ave_ipc_buf_of(ave, addr); if (len || buf < 0 || !(ave->unacked & BIT(buf))) { - dev_err(ave->dev, "bad command ack %#llx/%u\n", addr, len); + dev_err(ave->dev, "bad command ack %#llx/%u\n", addr, + len); return -EPROTO; } ave->unacked &= ~BIT(buf); @@ -445,8 +565,11 @@ static int ave_ipc_poll_once(struct ave_dev *ave) if (len > sizeof(ave->msg) || ave_fw_offset(ave, addr, len, &off)) return -EPROTO; /* Messages must not land in the buffers the host writes commands to. */ - if (off < AVE_IPC_END && off + len > AVE_IPC_CMD_OFF) { - dev_err(ave->dev, "firmware message at %#x overlaps the command area\n", off); + if (off < ave_ipc_command_end(ave) && + off + len > ave->layout->abi->transport->command_offset) { + dev_err(ave->dev, + "firmware message at %#x overlaps the command area\n", + off); return -EPROTO; } memcpy(ave->msg, ave->ipc.cpu + off, len); @@ -455,8 +578,8 @@ static int ave_ipc_poll_once(struct ave_dev *ave) ret = ave_ring_send(ave, &ave->ring[1], addr, len); if (ret) return ret; - ave_ipc_dispatch(ave); - return 1; + ret = ave_ipc_dispatch(ave); + return ret ?: 1; } /* @@ -469,7 +592,7 @@ static int ave_ipc_poll_once(struct ave_dev *ave) * microseconds), then for up to a millisecond, so a long wait for a frame * costs next to no CPU time. */ -#define AVE_POLL_FAST_ROUNDS 20 +#define AVE_POLL_FAST_ROUNDS 20 int ave_ipc_pump(struct ave_dev *ave, unsigned int wait_ms, bool (*cond)(struct ave_dev *ave, void *arg), void *arg) diff --git a/drivers/media/platform/apple/ave/ave-layout.c b/drivers/media/platform/apple/ave/ave-layout.c index 35761b36190ba..273f9ce22401c 100644 --- a/drivers/media/platform/apple/ave/ave-layout.c +++ b/drivers/media/platform/apple/ave/ave-layout.c @@ -4,6 +4,8 @@ const struct ave_fw_layout ave_layout_9003_78_0 = { .name = "9003.78.0 / 25G83", + .abi = &ave_abi_9003, + .encode_ready = true, .bytes = { { 0x3180, 0x63d8 }, { 0x13f28, 0xb1c0 } }, .hdr_bytes = 0x12c000, .meta_bytes = 0xc000, @@ -13,22 +15,28 @@ const struct ave_fw_layout ave_layout_9003_78_0 = { .mcc_dsid = 0x12, .start = { [0] = { - .ref_a = 0x390, - .ref_b = 0x3a0, - .ref_c = 0x7d0, - .ref_c_alias = 0x7e0, - .ref_d = 0x1130, - .ref_stride = 0x20, - .ref_c_stride = 0x20, - .mb = 0xc10, - .coded = 0xd50, - .meta = 0xf50, - .row_a = 0x1350, - .row_b = 0x1390, - .fixed = 0x13d0, - .hevc_extra = 0x2520, - .tiles = { 0x1410, 0x1430, 0x1490, 0x14b0, 0x1510, - 0x1530, 0x1590, 0x15b0 }, + .header = AVE_SURFACE(0x48, 0x10), + .secondary = AVE_SURFACE(0x58, 0x10), + .ref_a = AVE_SURFACE(0x390, 0x20), + .ref_b = AVE_SURFACE(0x3a0, 0x20), + .ref_c = AVE_SURFACE(0x7d0, 0x20), + .ref_c_alias = AVE_SURFACE(0x7e0, 0x20), + .ref_d = AVE_SURFACE(0x1130, 0x10), + .mb = AVE_SURFACE(0xc10, 0x10), + .coded = AVE_SURFACE(0xd50, 0x10), + .meta = AVE_SURFACE(0xf50, 0x10), + .row_a = AVE_SURFACE(0x1350, 0x10), + .row_b = AVE_SURFACE(0x1390, 0x10), + .fixed = AVE_SURFACE(0x13d0, 0x10), + .hevc_extra = AVE_SURFACE(0x2520, 0x10), + .tiles = { AVE_SURFACE(0x1410, 0x10), + AVE_SURFACE(0x1430, 0x10), + AVE_SURFACE(0x1490, 0x10), + AVE_SURFACE(0x14b0, 0x10), + AVE_SURFACE(0x1510, 0x10), + AVE_SURFACE(0x1530, 0x10), + AVE_SURFACE(0x1590, 0x10), + AVE_SURFACE(0x15b0, 0x10) }, .codec = 0xc0, .flag_8bit = 0x208, .const_214 = 0x214, @@ -106,22 +114,28 @@ const struct ave_fw_layout ave_layout_9003_78_0 = { .hevc_reorder_on = 0x95fc, }, [1] = { - .ref_a = 0x390, - .ref_b = 0x3a0, - .ref_c = 0x7d0, - .ref_c_alias = 0x7e0, - .ref_d = 0x1130, - .ref_stride = 0x20, - .ref_c_stride = 0x20, - .mb = 0xc10, - .coded = 0xd50, - .meta = 0xf50, - .row_a = 0x1350, - .row_b = 0x1390, - .fixed = 0x13d0, - .hevc_extra = 0x2520, - .tiles = { 0x1410, 0x1430, 0x1490, 0x14b0, 0x1510, - 0x1530, 0x1590, 0x15b0 }, + .header = AVE_SURFACE(0x48, 0x10), + .secondary = AVE_SURFACE(0x58, 0x10), + .ref_a = AVE_SURFACE(0x390, 0x20), + .ref_b = AVE_SURFACE(0x3a0, 0x20), + .ref_c = AVE_SURFACE(0x7d0, 0x20), + .ref_c_alias = AVE_SURFACE(0x7e0, 0x20), + .ref_d = AVE_SURFACE(0x1130, 0x10), + .mb = AVE_SURFACE(0xc10, 0x10), + .coded = AVE_SURFACE(0xd50, 0x10), + .meta = AVE_SURFACE(0xf50, 0x10), + .row_a = AVE_SURFACE(0x1350, 0x10), + .row_b = AVE_SURFACE(0x1390, 0x10), + .fixed = AVE_SURFACE(0x13d0, 0x10), + .hevc_extra = AVE_SURFACE(0x2520, 0x10), + .tiles = { AVE_SURFACE(0x1410, 0x10), + AVE_SURFACE(0x1430, 0x10), + AVE_SURFACE(0x1490, 0x10), + AVE_SURFACE(0x14b0, 0x10), + AVE_SURFACE(0x1510, 0x10), + AVE_SURFACE(0x1530, 0x10), + AVE_SURFACE(0x1590, 0x10), + AVE_SURFACE(0x15b0, 0x10) }, .codec = 0xc0, .flag_8bit = 0x208, .const_214 = 0x214, @@ -255,6 +269,8 @@ const struct ave_fw_layout ave_layout_9003_78_0 = { const struct ave_fw_layout ave_layout_26a428 = { .name = "26A428 (CRC unqualified)", + .abi = &ave_abi_9003, + .encode_ready = true, .bytes = { { 0x3b40, 0x6df8 }, { 0x13ee0, 0x78b8 } }, .hdr_bytes = 0xec000, .meta_bytes = 0x10000, @@ -265,24 +281,30 @@ const struct ave_fw_layout ave_layout_26a428 = { .counted = true, .start = { [0] = { - .ref_a = 0x98, - .ref_b = 0xa8, - .ref_c = 0xfe8, - .ref_c_alias = 0xff8, - .ref_d = 0xdc0, - .ref_e = 0xb8, - .ref_f = 0xc8, - .ref_stride = 0x40, - .ref_c_stride = 0x40, - .mb = 0x1870, - .coded = 0x920, - .meta = 0xb08, - .row_a = 0x19c0, - .row_b = 0x1a08, - .fixed = 0x1a50, - .hevc_extra = 0x70, - .tiles = { 0x1bb8, 0x1bd8, 0x1db8, 0x1dd8, 0x1bf8, - 0x1c18, 0x1df8, 0x1e18 }, + .header = AVE_SURFACE(0x48, 0x10), + .secondary = AVE_SURFACE(0x58, 0x10), + .ref_a = AVE_SURFACE(0x98, 0x40), + .ref_b = AVE_SURFACE(0xa8, 0x40), + .ref_c = AVE_SURFACE(0xfe8, 0x40), + .ref_c_alias = AVE_SURFACE(0xff8, 0x40), + .ref_d = AVE_SURFACE(0xdc0, 0x10), + .ref_e = AVE_SURFACE(0xb8, 0x40), + .ref_f = AVE_SURFACE(0xc8, 0x40), + .mb = AVE_SURFACE(0x1870, 0x10), + .coded = AVE_SURFACE(0x920, 0x10), + .meta = AVE_SURFACE(0xb08, 0x10), + .row_a = AVE_SURFACE(0x19c0, 0x10), + .row_b = AVE_SURFACE(0x1a08, 0x10), + .fixed = AVE_SURFACE(0x1a50, 0x10), + .hevc_extra = AVE_SURFACE(0x70, 0x10), + .tiles = { AVE_SURFACE(0x1bb8, 0x10), + AVE_SURFACE(0x1bd8, 0x10), + AVE_SURFACE(0x1db8, 0x10), + AVE_SURFACE(0x1dd8, 0x10), + AVE_SURFACE(0x1bf8, 0x10), + AVE_SURFACE(0x1c18, 0x10), + AVE_SURFACE(0x1df8, 0x10), + AVE_SURFACE(0x1e18, 0x10) }, .refs_count = 0x94, .work_count = 0xdbc, .lfs_count = 0xfe4, @@ -347,24 +369,30 @@ const struct ave_fw_layout ave_layout_26a428 = { .transform_copy = 0x3b0d, }, [1] = { - .ref_a = 0x98, - .ref_b = 0xa8, - .ref_c = 0xfe8, - .ref_c_alias = 0xff8, - .ref_d = 0xdc0, - .ref_e = 0xb8, - .ref_f = 0xc8, - .ref_stride = 0x40, - .ref_c_stride = 0x40, - .mb = 0x1870, - .coded = 0x920, - .meta = 0xb08, - .row_a = 0x19c0, - .row_b = 0x1a08, - .fixed = 0x1a50, - .hevc_extra = 0x70, - .tiles = { 0x1bb8, 0x1bd8, 0x1db8, 0x1dd8, 0x1bf8, - 0x1c18, 0x1df8, 0x1e18 }, + .header = AVE_SURFACE(0x48, 0x10), + .secondary = AVE_SURFACE(0x58, 0x10), + .ref_a = AVE_SURFACE(0x98, 0x40), + .ref_b = AVE_SURFACE(0xa8, 0x40), + .ref_c = AVE_SURFACE(0xfe8, 0x40), + .ref_c_alias = AVE_SURFACE(0xff8, 0x40), + .ref_d = AVE_SURFACE(0xdc0, 0x10), + .ref_e = AVE_SURFACE(0xb8, 0x40), + .ref_f = AVE_SURFACE(0xc8, 0x40), + .mb = AVE_SURFACE(0x1870, 0x10), + .coded = AVE_SURFACE(0x920, 0x10), + .meta = AVE_SURFACE(0xb08, 0x10), + .row_a = AVE_SURFACE(0x19c0, 0x10), + .row_b = AVE_SURFACE(0x1a08, 0x10), + .fixed = AVE_SURFACE(0x1a50, 0x10), + .hevc_extra = AVE_SURFACE(0x70, 0x10), + .tiles = { AVE_SURFACE(0x1bb8, 0x10), + AVE_SURFACE(0x1bd8, 0x10), + AVE_SURFACE(0x1db8, 0x10), + AVE_SURFACE(0x1dd8, 0x10), + AVE_SURFACE(0x1bf8, 0x10), + AVE_SURFACE(0x1c18, 0x10), + AVE_SURFACE(0x1df8, 0x10), + AVE_SURFACE(0x1e18, 0x10) }, .refs_count = 0x94, .work_count = 0xdbc, .lfs_count = 0xfe4, @@ -485,10 +513,26 @@ const struct ave_fw_layout ave_layout_26a428 = { }, }; +/* Lifecycle is qualified from the 6070 spec; payload field ledger is pending. + */ +const struct ave_fw_layout ave_layout_6070_11_1 = { + .name = "6070.11.1 (lifecycle only)", + .abi = &ave_abi_6070, + .bytes = { { 0x10e10, 0x1940 }, { 0x32dc8, 0x6838 } }, + .start = { + {.header = AVE_SURFACE(0x40, 0x10), + .secondary = AVE_SURFACE(0x50, 0x10) }, + {.header = AVE_SURFACE(0x40, 0x10), + .secondary = AVE_SURFACE(0x50, 0x10) }, + }, +}; + const struct ave_fw_layout *ave_layout_for_crc(u32 crc) { if (crc == AVE_FW_TEXT_CRC_9003_78_0) return &ave_layout_9003_78_0; + if (crc == 0x460c7a7c || crc == 0x25bfd039) + return &ave_layout_6070_11_1; /* 26A428 has no verified loader TEXT CRC: no placeholder or fallback. */ return NULL; } diff --git a/drivers/media/platform/apple/ave/ave-layout.h b/drivers/media/platform/apple/ave/ave-layout.h index d43225a9af5dd..24f608fabfd9b 100644 --- a/drivers/media/platform/apple/ave/ave-layout.h +++ b/drivers/media/platform/apple/ave/ave-layout.h @@ -2,122 +2,186 @@ #ifndef _APPLE_AVE_LAYOUT_H_ #define _APPLE_AVE_LAYOUT_H_ +struct ave_ctx; +struct ave_dev; +struct ave_fw_layout; +struct ave_fw_abi; + +struct ave_command { + enum ave_opcode op; + enum ave_codec codec; + u64 token; + u32 client, subtype, arg0, arg2; +}; + +/* Status is normalized to zero for success by the ABI parser. */ +struct ave_reply { + enum ave_opcode op; + u64 client; + u32 subtype, status, raw_status, bytes; +}; + +struct ave_fw_transport { + u32 ipc_bytes; + u32 command_offset, command_bytes, scalar_offset; + u32 process_offset, process_bytes; + /* Zero channel/heap sizes mean bounded negotiation on 9003. */ + u32 hello[4]; + u32 descriptors; + u32 io_index, t2h_index; + u32 slots[2], doorbells[2], offsets[2]; /* zero: discovered */ + bool boot_config; + bool sync_host_clock; + bool scratch_halt; +}; + +struct ave_fw_abi { + const struct ave_fw_transport *transport; + u32 wire_id[2][AVE_OP_COUNT]; + u32 reply_id[AVE_OP_COUNT]; + u32 scalar_bytes[AVE_OP_COUNT]; + u32 semantic[AVE_OP_COUNT]; + u32 header_bytes, timeout_offset, created_offset; + bool padded_surfaces; + void (*write_header)(u8 *cmd, const struct ave_fw_abi *abi, + const struct ave_command *header); + void (*write_config)(struct ave_dev *ave, u8 *cmd); + int (*parse_reply)(const struct ave_fw_abi *abi, const u8 *msg, u32 len, + struct ave_reply *reply); +}; + +/* Separate address/size offsets and strides also represent split arrays. */ +struct ave_surface_layout { + u32 address, size; + u32 address_stride, size_stride; +}; + +#define AVE_SURFACE(off, stride) \ + { .address = (off), \ + .size = (off) + 8, \ + .address_stride = (stride), \ + .size_stride = (stride) } + /* Zero offsets denote fields absent from a firmware's wire format. */ struct ave_start_layout { - u16 ref_a, ref_b, ref_c, ref_c_alias, ref_d; - u16 ref_e, ref_f, ref_stride, ref_c_stride; - u16 mb, coded, meta, row_a, row_b, fixed, hevc_extra; - u16 tiles[8]; - u16 refs_count, work_count, lfs_count, slots_count, meta_count; - u16 mb_count, row_a_count, row_b_count, fixed_count, tiles_count; - u16 codec; - u16 flag_8bit; - u16 const_214; - u16 const_218; - u16 fps; - u16 rc_mode; - u16 bitrate; - u16 init_qp; - u16 limits; - u16 limit_bytes; - u16 limit_secs; - u16 f64_288; - u16 f64_290; - u16 min_qp; - u16 max_qp; - u16 gop_flags; - u16 p_frames; - u16 max_b; - u16 ref_depth; - u16 idr_interval; - u16 f64_2c0; - u16 f64_2c8; - u16 rc_flags; - u16 min_qp2; - u16 max_qp2; - u16 rc_310; - u16 size_class; - u16 coded_w; - u16 coded_h; - u16 codec_flags; - u16 h264_380; - u16 h264_384; - u16 codec_2588; - u16 bit_depth; - u16 coded_h2; - u16 h264_profile; - u16 h264_level; - u16 h264_max_refs; - u16 h264_width_mbs; - u16 h264_height_mbs; - u16 h264_crop; - u16 h264_crop_left; - u16 h264_crop_right; - u16 h264_crop_top; - u16 h264_crop_bottom; - u16 h264_cabac; - u16 h264_8x8; - u16 hevc_pad; - u16 hevc_rc_270c; - u16 hevc_rc_9604; - u16 hevc_rc_9608; - u16 hevc_profile_a; - u16 hevc_compat_a; - u16 hevc_8bit_a; - u16 hevc_level_a; - u16 hevc_profile_b; - u16 hevc_compat_b; - u16 hevc_8bit_b; - u16 hevc_level_b; - u16 hevc_width; - u16 hevc_height; - u16 hevc_width_ctb; - u16 hevc_height_ctb; - u16 hevc_conf; - u16 hevc_conf_left; - u16 hevc_conf_right; - u16 hevc_conf_top; - u16 hevc_conf_bottom; - u16 hevc_depth_luma; - u16 hevc_depth_chroma; - u16 hevc_refs; - u16 hevc_reorder_copy; - u16 hevc_dpb_minus1; - u16 hevc_reorder; - u16 hevc_reorder_on; - u16 geometry_mode, geometry_depth; - u16 profile_idc, level_idc, max_refs_copy, mb_size, mb_height; - u16 crop_copy, crop_right_copy, crop_bottom_copy; - u16 codec_byte, cabac_copy, transform_copy, hevc_cq, hevc_cq_copy; - u16 hevc_cq_codec, hevc_cq_picture; + struct ave_surface_layout header, secondary; + struct ave_surface_layout ref_a, ref_b, ref_c, ref_c_alias, ref_d; + struct ave_surface_layout ref_e, ref_f; + struct ave_surface_layout mb, coded, meta, row_a, row_b, fixed, + hevc_extra; + struct ave_surface_layout tiles[8]; + u32 refs_count, work_count, lfs_count, slots_count, meta_count; + u32 mb_count, row_a_count, row_b_count, fixed_count, tiles_count; + u32 codec; + u32 flag_8bit; + u32 const_214; + u32 const_218; + u32 fps; + u32 rc_mode; + u32 bitrate; + u32 init_qp; + u32 limits; + u32 limit_bytes; + u32 limit_secs; + u32 f64_288; + u32 f64_290; + u32 min_qp; + u32 max_qp; + u32 gop_flags; + u32 p_frames; + u32 max_b; + u32 ref_depth; + u32 idr_interval; + u32 f64_2c0; + u32 f64_2c8; + u32 rc_flags; + u32 min_qp2; + u32 max_qp2; + u32 rc_310; + u32 size_class; + u32 coded_w; + u32 coded_h; + u32 codec_flags; + u32 h264_380; + u32 h264_384; + u32 codec_2588; + u32 bit_depth; + u32 coded_h2; + u32 h264_profile; + u32 h264_level; + u32 h264_max_refs; + u32 h264_width_mbs; + u32 h264_height_mbs; + u32 h264_crop; + u32 h264_crop_left; + u32 h264_crop_right; + u32 h264_crop_top; + u32 h264_crop_bottom; + u32 h264_cabac; + u32 h264_8x8; + u32 hevc_pad; + u32 hevc_rc_270c; + u32 hevc_rc_9604; + u32 hevc_rc_9608; + u32 hevc_profile_a; + u32 hevc_compat_a; + u32 hevc_8bit_a; + u32 hevc_level_a; + u32 hevc_profile_b; + u32 hevc_compat_b; + u32 hevc_8bit_b; + u32 hevc_level_b; + u32 hevc_width; + u32 hevc_height; + u32 hevc_width_ctb; + u32 hevc_height_ctb; + u32 hevc_conf; + u32 hevc_conf_left; + u32 hevc_conf_right; + u32 hevc_conf_top; + u32 hevc_conf_bottom; + u32 hevc_depth_luma; + u32 hevc_depth_chroma; + u32 hevc_refs; + u32 hevc_reorder_copy; + u32 hevc_dpb_minus1; + u32 hevc_reorder; + u32 hevc_reorder_on; + u32 geometry_mode, geometry_depth; + u32 profile_idc, level_idc, max_refs_copy, mb_size, mb_height; + u32 crop_copy, crop_right_copy, crop_bottom_copy; + u32 codec_byte, cabac_copy, transform_copy, hevc_cq, hevc_cq_copy; + u32 hevc_cq_codec, hevc_cq_picture; }; struct ave_proc_layout { - u16 first_frame; - u16 y, uv; /* {u64 address, u32 size, u32 stride} */ - u16 in_w, in_h; - u16 coded_cap; - u16 fps_int; - u16 bitrate; - u16 limits, limit_bytes, limit_secs; - u16 const5; - u16 fps; /* f64 */ - u16 timescale; /* u64 */ - u16 slot; - u16 coded, meta; /* u64 addresses */ - u16 ordinal; - u16 pts; /* u64 */ - u16 force_key; /* 1 on a forced key frame */ - u16 force_type; /* 2 on a forced key frame */ - u16 frame_buf; /* H.264 only */ - u16 shdr_table; /* HEVC only */ - u16 crop; /* HEVC only */ - u16 meta_size, extra, frame_size; - u16 low_delay; /* HEVC only: 3 with reordering off */ + u32 first_frame; + u32 y, uv; /* {u64 address, u32 size, u32 stride} */ + u32 in_w, in_h; + u32 coded_cap; + u32 fps_int; + u32 bitrate; + u32 limits, limit_bytes, limit_secs; + u32 const5; + u32 fps; /* f64 */ + u32 timescale; /* u64 */ + u32 slot; + u32 coded, meta; /* u64 addresses */ + u32 ordinal; + u32 pts; /* u64 */ + u32 force_key; /* 1 on a forced key frame */ + u32 force_type; /* 2 on a forced key frame */ + u32 frame_buf; /* H.264 only */ + u32 shdr_table; /* HEVC only */ + u32 crop; /* HEVC only */ + u32 meta_size, extra, frame_size; + u32 low_delay; /* HEVC only: 3 with reordering off */ }; struct ave_fw_layout { const char *name; + const struct ave_fw_abi *abi; + bool encode_ready; size_t bytes[2][AVE_IMAGE_COUNT]; struct ave_start_layout start[2]; struct ave_proc_layout process[2]; @@ -127,7 +191,9 @@ struct ave_fw_layout { bool counted; }; +extern const struct ave_fw_abi ave_abi_9003, ave_abi_6070; extern const struct ave_fw_layout ave_layout_9003_78_0; +extern const struct ave_fw_layout ave_layout_6070_11_1; /* Not selectable until its loader TEXT CRC is measured and qualified. */ extern const struct ave_fw_layout ave_layout_26a428; const struct ave_fw_layout *ave_layout_for_crc(u32 crc); diff --git a/drivers/media/platform/apple/ave/ave-regs.h b/drivers/media/platform/apple/ave/ave-regs.h index 6e657dcd1fb0c..8c910069b4b02 100644 --- a/drivers/media/platform/apple/ave/ave-regs.h +++ b/drivers/media/platform/apple/ave/ave-regs.h @@ -74,7 +74,7 @@ #define AVE_CHAN_DESC_ADDR 0x4c #define AVE_RING_SLOT_BYTES 64 -/* Common command header */ +/* 9003 command header (also used by the frozen regression oracle). */ #define AVE_CMD_OPCODE 0x00 #define AVE_CMD_TOKEN 0x08 #define AVE_CMD_CLIENT 0x10 @@ -97,19 +97,7 @@ /* Halt is never answered; give the firmware this long before power-off. */ #define AVE_HALT_SETTLE_MS 20 -enum ave_opcode { - AVE_OP_CONFIG = 1, - AVE_OP_HALT = 2, - AVE_OP_OPEN = 3, - AVE_OP_CLOSE = 4, - AVE_OP_START = 6, - AVE_OP_STOP = 7, - AVE_OP_PROCESS = 8, - AVE_OP_COMPLETE = 9, - AVE_OP_PRIORITY = 10, -}; - -/* Firmware-to-host messages */ +/* 9003 firmware-to-host messages */ #define AVE_REPLY_ID 0x00 #define AVE_REPLY_CLIENT 0x10 #define AVE_REPLY_SUBTYPE 0x20 diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index 267c5abc73eed..2cb47055fc3fe 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -25,31 +25,31 @@ #include "ave.h" #include "ave-regs.h" -#define AVE_RC_VBR 4 -#define AVE_RC_CBR 2 +#define AVE_RC_VBR 4 +#define AVE_RC_CBR 2 /* Filler-data NAL unit types */ -#define AVE_H264_NAL_FILLER 12 -#define AVE_HEVC_NAL_FILLER 38 +#define AVE_H264_NAL_FILLER 12 +#define AVE_HEVC_NAL_FILLER 38 -#define AVE_H264_BASELINE 2 -#define AVE_H264_MAIN 4 -#define AVE_H264_HIGH 6 -#define AVE_HEVC_MAIN 1 +#define AVE_H264_BASELINE 2 +#define AVE_H264_MAIN 4 +#define AVE_H264_HIGH 6 +#define AVE_HEVC_MAIN 1 -#define AVE_META_BYTES 0xc000 +#define AVE_META_BYTES 0xc000 /* Metadata words: coded bytes (twice), HEVC slice-header bytes, parameter sets */ -#define AVE_META_PAYLOAD 0x120 -#define AVE_META_HEVC_SHDR 0x128 -#define AVE_META_PAYLOAD2 0x12c -#define AVE_META_PARAMS 0xb2a4 -#define AVE_META_SCAN_FROM 0x200 -#define AVE_SHDR_BYTES (256 * 0x400) -#define AVE_SHDR_ENTRY_BYTES 0x400 -#define AVE_CODED_MIN 0x3c000 +#define AVE_META_PAYLOAD 0x120 +#define AVE_META_HEVC_SHDR 0x128 +#define AVE_META_PAYLOAD2 0x12c +#define AVE_META_PARAMS 0xb2a4 +#define AVE_META_SCAN_FROM 0x200 +#define AVE_SHDR_BYTES (256 * 0x400) +#define AVE_SHDR_ENTRY_BYTES 0x400 +#define AVE_CODED_MIN 0x3c000 /* H.264 per-frame buffer (Process +0x58c8): unused in bit-rate modes */ -#define AVE_H264_FRAME_BUF SZ_16K +#define AVE_H264_FRAME_BUF SZ_16K /* Level limits: max frame size, max rate, max bit rate (kbit/s, Main). */ struct ave_level { @@ -60,20 +60,24 @@ struct ave_level { /* H.264 Table A-1, in V4L2 menu order (1, 1b, 1.1 ... 5.2); macroblocks. */ static const struct ave_level ave_h264_levels[] = { - { 99, 1485, 64 }, { 99, 1485, 128 }, { 396, 3000, 192 }, { 396, 6000, 384 }, - { 396, 11880, 768 }, { 396, 11880, 2000 }, { 792, 19800, 4000 }, - { 1620, 20250, 4000 }, { 1620, 40500, 10000 }, { 3600, 108000, 14000 }, - { 5120, 216000, 20000 }, { 8192, 245760, 20000 }, { 8192, 245760, 50000 }, - { 8704, 522240, 50000 }, { 22080, 589824, 135000 }, { 36864, 983040, 240000 }, + { 99, 1485, 64 }, { 99, 1485, 128 }, + { 396, 3000, 192 }, { 396, 6000, 384 }, + { 396, 11880, 768 }, { 396, 11880, 2000 }, + { 792, 19800, 4000 }, { 1620, 20250, 4000 }, + { 1620, 40500, 10000 }, { 3600, 108000, 14000 }, + { 5120, 216000, 20000 }, { 8192, 245760, 20000 }, + { 8192, 245760, 50000 }, { 8704, 522240, 50000 }, + { 22080, 589824, 135000 }, { 36864, 983040, 240000 }, { 36864, 2073600, 240000 }, }; /* HEVC Table A-8 (Main tier), in V4L2 menu order (1, 2, 2.1 ... 5.1); samples. */ static const struct ave_level ave_hevc_levels[] = { - { 36864, 552960, 128 }, { 122880, 3686400, 1500 }, { 245760, 7372800, 3000 }, - { 552960, 16588800, 6000 }, { 983040, 33177600, 10000 }, - { 2228224, 66846720, 12000 }, { 2228224, 133693440, 20000 }, - { 8912896, 267386880, 25000 }, { 8912896, 534773760, 40000 }, + { 36864, 552960, 128 }, { 122880, 3686400, 1500 }, + { 245760, 7372800, 3000 }, { 552960, 16588800, 6000 }, + { 983040, 33177600, 10000 }, { 2228224, 66846720, 12000 }, + { 2228224, 133693440, 20000 }, { 8912896, 267386880, 25000 }, + { 8912896, 534773760, 40000 }, }; /* @@ -204,8 +208,8 @@ struct ave_buf_req { unsigned long attrs; }; -#define AVE_MAX_BUF_REQS (1 + 6 * AVE_MAX_REFS + 6 + 2 * AVE_TILE_BUFS + 2 + \ - 4 * AVE_MAX_SLOTS) +#define AVE_MAX_BUF_REQS \ + (1 + 6 * AVE_MAX_REFS + 6 + 2 * AVE_TILE_BUFS + 2 + 4 * AVE_MAX_SLOTS) static int ave_buf_req_cmp(const void *a, const void *b) { @@ -530,35 +534,41 @@ void ave_sess_prepare_slot(struct ave_session *sess, unsigned int slot) * A frame reply: +0x10 the session's client token, +0x20 the slot's * subtype, +0x40 status. Caller holds hw_lock. */ -void ave_sess_reply(struct ave_dev *ave, const u8 *msg, u32 len) +void ave_sess_reply(struct ave_dev *ave, const struct ave_reply *reply) { - u64 client = get_unaligned_le64(msg + AVE_REPLY_CLIENT); - u32 subtype = get_unaligned_le32(msg + AVE_REPLY_SUBTYPE); + u64 client = reply->client; + u32 subtype = reply->subtype; u32 slot = subtype - ave->layout->process_subtype; struct ave_session *sess = NULL; struct ave_ctx *ctx = NULL; struct ave_slot *s; for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) - if (ave->sessions[i] && ave->sessions[i]->sess->client == client) + if (ave->sessions[i] && + ave->sessions[i]->sess->client == client) ctx = ave->sessions[i]; if (ctx) sess = ctx->sess; - if (!sess || len < AVE_REPLY_PROCESS_BYTES || slot >= sess->p.slots || - !sess->slot[slot].busy) { - dev_warn(ave->dev, "stray frame reply (client %llu subtype %#x len %u)\n", - client, subtype, len); + if (!sess || + reply->bytes < + (ave->layout->abi->transport->boot_config ? 0x50 : 0x48) || + slot >= sess->p.slots || !sess->slot[slot].busy) { + dev_warn(ave->dev, + "stray frame reply (client %llu subtype %#x len %u)\n", + client, subtype, reply->bytes); return; } s = &sess->slot[slot]; sess->progress = jiffies; - s->status = get_unaligned_le32(msg + AVE_REPLY_STATUS); + s->status = reply->status; if (s->status) - dev_dbg(ctx->ave->dev, "frame in slot %u: status %#x\n", slot, s->status); + dev_dbg(ctx->ave->dev, "frame in slot %u: status %#x\n", slot, + s->status); s->busy = false; s->done = true; sess->busy--; - sess->done_fifo[(sess->done_head + sess->done_count) % AVE_MAX_SLOTS] = slot; + sess->done_fifo[(sess->done_head + sess->done_count) % AVE_MAX_SLOTS] = + slot; sess->done_count++; atomic_inc(&ctx->ready); } @@ -677,7 +687,7 @@ static void ave_sess_dump(struct ave_ctx *ctx, struct ave_slot *s, const char *w * is not known; if the firmware's output never carries filler, the driver * adds it here. */ -#define AVE_FILLER_OVERHEAD 7 /* start code, NAL header, trailing bits */ +#define AVE_FILLER_OVERHEAD 7 /* start code, NAL header, trailing bits */ static size_t ave_sess_cbr_fill(struct ave_ctx *ctx, u8 *out, size_t len, size_t room, bool fw_filler) @@ -814,7 +824,7 @@ bool ave_sess_stalled(struct ave_session *sess) * input came for a while, a Complete makes the firmware code what it holds, * as macOS does after the first frame. */ -#define AVE_STARVE_MS 50 +#define AVE_STARVE_MS 50 bool ave_sess_starved(struct ave_session *sess) { diff --git a/drivers/media/platform/apple/ave/ave-writer.c b/drivers/media/platform/apple/ave/ave-writer.c new file mode 100644 index 0000000000000..59f7d067bfc14 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-writer.c @@ -0,0 +1,520 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder: firmware commands + * + * Start and Process are large command images. Each is built in a zeroed + * buffer: the header, the constant words (ave-image.c), then every field + * that depends on the session or the frame. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include +#include +#include + +#include + +#include "ave.h" +#include "ave-regs.h" + +/* + * IEEE 754 double for the positive value @num / @den (0 for 0), built with + * integer arithmetic; the firmware takes rates and limits as doubles. + */ +static u64 ave_f64(u64 num, u64 den) +{ + u64 mant = 0, rem; + int exp = 0; + + if (!num || !den) + return 0; + while (num >= den << 1) { + den <<= 1; + exp++; + } + while (num < den) { + num <<= 1; + exp--; + } + /* 1 <= num/den < 2: take 53 significant bits by long division. */ + rem = num; + for (unsigned int i = 0; i < 53; i++) { + mant <<= 1; + if (rem >= den) { + mant |= 1; + rem -= den; + } + rem <<= 1; + } + return (u64)(exp + 1023) << 52 | (mant & GENMASK_ULL(51, 0)); +} + +#define AVE_F64_NEG BIT_ULL(63) + +static void ave_put32(u8 *cmd, u32 off, u32 val) +{ + put_unaligned_le32(val, cmd + off); +} + +static void ave_put64(u8 *cmd, u32 off, u64 val) +{ + put_unaligned_le64(val, cmd + off); +} + +void ave_write_surface(const struct ave_fw_layout *fw, u8 *cmd, + const struct ave_surface_layout *l, unsigned int index, + dma_addr_t addr, u32 size) +{ + u32 address_off = l->address + l->address_stride * index; + u32 size_off = l->size + l->size_stride * index; + + ave_put64(cmd, address_off, addr); + ave_put32(cmd, size_off, size); + if (fw->abi->padded_surfaces) + ave_put32(cmd, size_off + 4, 0); +} + +static void ave_put_buf(const struct ave_fw_layout *fw, u8 *cmd, + const struct ave_surface_layout *l, unsigned int index, + const struct ave_buf *b, u32 size) +{ + ave_write_surface(fw, cmd, l, index, b->dma, size); +} + +/* Is another session open in the firmware besides @ctx's? */ +static bool ave_cmd_others_open(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i] && ave->sessions[i] != ctx && + ave->sessions[i]->sess->fw_open) + return true; + return false; +} + +/* + * Header word +0x2c depends on the set of open sessions: 0x1e0 on Open and + * Start (0x140 when another session is open), the codec value on every + * other command of a lone session, and 0 while another one is open. + */ +u32 ave_session_arg2(struct ave_ctx *ctx) +{ + if (ctx->ave->layout->counted && ctx->sess->p.codec == AVE_CODEC_H264) + return ctx->sess->p.bframes ? AVE_SESSION_ARG2_H264 : 0x140; + if (ave_cmd_others_open(ctx)) + return 0; + return ctx->sess->p.codec == AVE_CODEC_HEVC ? AVE_SESSION_ARG2_HEVC : + AVE_SESSION_ARG2_H264; +} + +u32 ave_session_setup_arg2(struct ave_ctx *ctx) +{ + return ave_cmd_others_open(ctx) ? AVE_SESSION_ARG2_SETUP_SHARED : + AVE_SESSION_ARG2_SETUP; +} + +void ave_write_header(struct ave_ctx *ctx, u8 *cmd, enum ave_opcode op, + u32 echo, u32 arg0, u32 arg2) +{ + const struct ave_fw_abi *abi = ctx->ave->layout->abi; + struct ave_command header = { + .op = op, + .codec = ctx->sess->p.codec, + .client = ctx->sess->client, + .subtype = echo, + .arg0 = arg0, + .arg2 = arg2, + }; + + ctx->sess->token += AVE_TOKEN_STEP; + header.token = ctx->sess->token; + abi->write_header(cmd, abi, &header); +} + +void ave_write_global(struct ave_dev *ave, u8 *cmd, enum ave_opcode op) +{ + const struct ave_fw_abi *abi = ave->layout->abi; + struct ave_command header = { + .op = op, + .codec = AVE_CODEC_H264, + .subtype = U32_MAX, + }; + + abi->write_header(cmd, abi, &header); +} + +void ave_stamp_command(const struct ave_fw_abi *abi, u8 *cmd, u64 timeout, + u64 created) +{ + ave_put64(cmd, abi->timeout_offset, timeout); + ave_put64(cmd, abi->created_offset, created); +} + +#define AVE_START_ARRAY_LEN 30 +#define AVE_START_INIT_QP_VALUE 26 + +static void ave_start_surfaces(struct ave_session *sess, u8 *cmd, + const struct ave_fw_layout *fw) +{ + const struct ave_params *p = &sess->p; + bool hevc = p->codec == AVE_CODEC_HEVC; + unsigned int tiles = hevc ? 4 : AVE_TILE_BUFS; + const struct ave_start_layout *l = &fw->start[p->codec - 1]; + + /* Clear every descriptor the image may carry, then fill ours. */ + for (unsigned int i = 0; i < AVE_MAX_REFS; i++) { + ave_write_surface(fw, cmd, &l->ref_a, i, 0, 0); + ave_write_surface(fw, cmd, &l->ref_b, i, 0, 0); + ave_write_surface(fw, cmd, &l->ref_c, i, 0, 0); + ave_write_surface(fw, cmd, &l->ref_c_alias, i, 0, 0); + ave_write_surface(fw, cmd, &l->ref_d, i, 0, 0); + } + for (unsigned int s = 0; s < AVE_START_ARRAY_LEN; s++) { + ave_write_surface(fw, cmd, &l->coded, s, 0, 0); + ave_write_surface(fw, cmd, &l->meta, s, 0, 0); + } + + ave_put_buf(fw, cmd, &l->header, 0, &sess->hdr, fw->hdr_bytes); + ave_write_surface(fw, cmd, &l->secondary, 0, 0, 0); + for (unsigned int i = 0; i < p->refs; i++) { + ave_put_buf(fw, cmd, &l->ref_a, i, &sess->ref_a[i], + sess->ref_a[i].size); + ave_put_buf(fw, cmd, &l->ref_b, i, &sess->ref_b[i], + sess->ref_b[i].size); + ave_put_buf(fw, cmd, &l->ref_c, i, &sess->ref_c[i], + sess->ref_c[i].size); + ave_write_surface(fw, cmd, &l->ref_c_alias, i, + sess->ref_c[i].dma, 0); + ave_put_buf(fw, cmd, &l->ref_d, i, &sess->ref_d[i], + sess->ref_d[i].size); + } + if (fw->counted) { + ave_put32(cmd, l->refs_count, p->refs); + ave_put32(cmd, l->work_count, p->refs); + ave_put32(cmd, l->lfs_count, p->refs); + ave_put32(cmd, l->slots_count, p->slots); + ave_put32(cmd, l->meta_count, p->slots); + ave_put32(cmd, l->mb_count, p->rc_mode == 3 ? 3 : 1); + if (p->rc_mode == 3) + for (unsigned int i = 0; i < 2; i++) + ave_put_buf(fw, cmd, &l->mb, i + 1, + &sess->mb_extra[i], + sess->mb_extra[i].size); + ave_put32(cmd, l->tiles_count, tiles); + for (unsigned int i = 0; i < p->refs; i++) { + ave_put_buf(fw, cmd, &l->ref_e, i, &sess->ref_e[i], + sess->ref_e[i].size); + ave_put_buf(fw, cmd, &l->ref_f, i, &sess->ref_f[i], + sess->ref_f[i].size); + } + } + ave_put_buf(fw, cmd, &l->mb, 0, &sess->mb, sess->mb.size); + for (unsigned int s = 0; s < p->slots; s++) { + ave_put_buf(fw, cmd, &l->coded, s, &sess->slot[s].coded, + p->coded_cap); + ave_put_buf(fw, cmd, &l->meta, s, &sess->slot[s].meta, + sess->slot[s].meta.size); + } + ave_put_buf(fw, cmd, &l->row_a, 0, &sess->row_a, sess->row_a.size); + ave_put_buf(fw, cmd, &l->row_b, 0, &sess->row_b, sess->row_b.size); + ave_put_buf(fw, cmd, &l->fixed, 0, &sess->fixed, SZ_16K); + for (unsigned int i = 0; i < AVE_TILE_BUFS; i++) { + if (i < tiles) { + ave_put_buf(fw, cmd, &l->tiles[i], 0, &sess->tile[i], + sess->tile[i].size); + ave_put_buf(fw, cmd, &l->tiles[i], 1, + &sess->tile_aux[i], sess->tile_aux[i].size); + } else { + ave_write_surface(fw, cmd, &l->tiles[i], 0, 0, 0); + ave_write_surface(fw, cmd, &l->tiles[i], 1, 0, 0); + } + } + if (hevc) + for (unsigned int i = 0; i < 2; i++) + ave_put_buf(fw, cmd, &l->hevc_extra, i, + &sess->hevc_extra[i], + sess->hevc_extra[i].size); +} + +static void ave_start_put32(u8 *cmd, u32 off, u32 val) +{ + if (off) + ave_put32(cmd, off, val); +} + +static void ave_start_config(const struct ave_params *p, u8 *cmd, + const struct ave_fw_layout *fw) +{ + bool hevc = p->codec == AVE_CODEC_HEVC; + bool cq = p->rc_mode == 3; + u32 depth = p->bit_depth ?: 8; + u32 crop_r = (p->cw - p->width) / 2, crop_b = (p->ch - p->height) / 2; + bool crop = crop_r || crop_b; + const struct ave_start_layout *l = &fw->start[p->codec - 1]; + + ave_start_put32(cmd, l->codec, p->codec); + ave_start_put32( + cmd, l->flag_8bit, + fw->counted ? + 0x10000 | (hevc ? (depth == 8 ? 0x41 : 0x40) : 0) | + (cq ? 0x10 : 0) : + hevc ? 1 : + 0); + ave_start_put32(cmd, l->geometry_mode, + (hevc ? 0x40 : 0) | (cq ? 0 : 2)); + ave_start_put32(cmd, l->geometry_depth, depth); + ave_start_put32(cmd, l->const_214, 0x39d); + ave_start_put32(cmd, l->const_218, 0x18); + ave_start_put32(cmd, l->fps, p->fps_int); + + /* Rate control */ + ave_start_put32(cmd, l->rc_mode, p->rc_mode); + ave_start_put32(cmd, l->bitrate, p->bitrate); + for (unsigned int i = 0; i < 3; i++) + ave_put32(cmd, (l->init_qp + 4 * i), + cq ? p->initial_qp : AVE_START_INIT_QP_VALUE); + ave_start_put32(cmd, l->limits, p->limit_bytes ? 1 : 0); + if (l->limit_bytes && p->limit_bytes) { + ave_put64(cmd, l->limit_bytes, + ave_f64((u64)p->limit_bytes * (fw->counted ? 8 : 1), + 1)); + ave_put64(cmd, l->limit_secs, ave_f64(1, 1)); + } + ave_put64(cmd, l->f64_288, ave_f64(1, 1)); + ave_put64(cmd, l->f64_290, ave_f64(1, 2)); + ave_put64(cmd, l->f64_2c0, AVE_F64_NEG | ave_f64(1, 1)); + ave_put64(cmd, l->f64_2c8, AVE_F64_NEG | ave_f64(1, 1)); + ave_start_put32(cmd, l->min_qp, p->min_qp); + ave_start_put32(cmd, l->max_qp, p->max_qp); + ave_start_put32(cmd, l->min_qp2, p->min_qp); + ave_start_put32(cmd, l->max_qp2, p->max_qp); + ave_start_put32(cmd, l->rc_flags, cq ? 0 : hevc ? 0x736 : 0x8202); + ave_start_put32(cmd, l->rc_310, cq ? 0 : 30); + if (!hevc) + ave_start_put32(cmd, l->h264_380, 1); + + /* GOP structure: with B-frames as captured, or P-only, or intra-only. */ + ave_start_put32( + cmd, l->gop_flags, + (p->bframes ? 0x600 : 0x200) | + (fw->counted && cq ? 0x10000 | (hevc ? 0x100 : 0) : 0)); + ave_start_put32(cmd, l->p_frames, p->intra_only ? 0 : 1); + ave_start_put32(cmd, l->max_b, p->bframes ? 3 : 0); + ave_start_put32(cmd, l->ref_depth, p->bframes ? 3 : 1); + ave_start_put32(cmd, l->idr_interval, p->gop); + + /* Picture size */ + ave_start_put32(cmd, l->size_class, + p->width * p->height > 1280 * 720 ? 3 : 1); + ave_start_put32(cmd, l->coded_w, p->cw); + ave_start_put32(cmd, l->coded_h, p->ch); + ave_start_put32(cmd, l->coded_h2, p->ch); + ave_start_put32(cmd, l->codec_flags, hevc ? 0x2000002 : 2); + ave_start_put32(cmd, l->bit_depth, depth); + + if (!hevc) { + ave_start_put32(cmd, l->h264_384, + p->width * p->height <= 320 * 240 ? 0x40 : + 0x10); + ave_start_put32(cmd, l->codec_2588, p->high ? 2 : 0); + ave_start_put32(cmd, l->h264_profile, p->profile); + ave_start_put32(cmd, l->h264_level, p->level); + ave_start_put32(cmd, l->h264_max_refs, + (p->refs - 1) | + (fw->counted ? (p->cw / 16) << 16 : 0)); + if (fw->counted) { + static const u8 level_idc[] = { + 10, 9, 11, 12, 13, 20, 21, 22, 30, + 31, 32, 40, 41, 42, 50, 51, 52, + }; + + ave_put32(cmd, l->profile_idc, + p->high ? 100 : + p->profile == 2 ? 66 : + 77); + ave_put32(cmd, l->level_idc, level_idc[p->level - 1]); + put_unaligned_le16(p->refs - 1, cmd + l->max_refs_copy); + ave_put32(cmd, l->mb_size, + ((p->ch / 16 - 1) << 16) | (p->cw / 16 - 1)); + cmd[l->mb_height] = p->ch / 16; + cmd[l->mb_height + 2] = 1; + cmd[l->mb_height + 3] = 1; + cmd[l->crop_copy] = crop; + put_unaligned_le16(crop_r, cmd + l->crop_right_copy); + put_unaligned_le16(crop_b, cmd + l->crop_bottom_copy); + cmd[l->codec_byte] = !cq; + /* CABAC flag in the second, packed picture structure. */ + cmd[l->cabac_copy] = p->profile != 2; + cmd[l->transform_copy] = p->high; + } + ave_start_put32(cmd, l->h264_width_mbs, p->cw / 16 - 1); + ave_start_put32(cmd, l->h264_height_mbs, p->ch / 16 - 1); + ave_start_put32(cmd, l->h264_crop, crop); + ave_start_put32(cmd, l->h264_crop_left, 0); + ave_start_put32(cmd, l->h264_crop_right, crop_r); + ave_start_put32(cmd, l->h264_crop_top, 0); + ave_start_put32(cmd, l->h264_crop_bottom, crop_b); + ave_start_put32(cmd, l->h264_cabac, p->profile != 2); + cmd[l->h264_8x8] = p->high; + return; + } + + ave_start_put32(cmd, l->hevc_pad, + 1 | (p->cw > p->width ? BIT(16) | BIT(18) : 0) | + (p->ch > p->height ? BIT(17) | BIT(19) : 0)); + ave_start_put32(cmd, l->hevc_profile_a, p->profile); + ave_start_put32(cmd, l->hevc_profile_b, p->profile); + cmd[l->hevc_compat_a] = depth == 8; + cmd[l->hevc_compat_a + 1] = 1; + cmd[l->hevc_compat_b] = depth == 8; + cmd[l->hevc_compat_b + 1] = 1; + cmd[l->hevc_8bit_a] = depth == 8; + cmd[l->hevc_8bit_b] = depth == 8; + /* Rate-control words that only change in constant-quality mode */ + ave_start_put32(cmd, l->hevc_cq, cq); + ave_start_put32(cmd, l->hevc_cq_copy, cq); + ave_start_put32(cmd, l->hevc_cq_codec, cq); + ave_start_put32(cmd, l->hevc_cq_picture, cq ? 0x100 : 0); + ave_start_put32(cmd, l->hevc_rc_270c, 2); + ave_start_put32(cmd, l->hevc_rc_9604, cq ? 2 : 0); + ave_start_put32(cmd, l->hevc_rc_9608, cq ? 0 : 0x10000); + ave_start_put32(cmd, l->hevc_level_a, p->level); + ave_start_put32(cmd, l->hevc_level_b, p->level); + ave_start_put32(cmd, l->hevc_width, p->cw); + ave_start_put32(cmd, l->hevc_height, p->ch); + ave_start_put32(cmd, l->hevc_width_ctb, DIV_ROUND_UP(p->cw, 32)); + ave_start_put32(cmd, l->hevc_height_ctb, DIV_ROUND_UP(p->ch, 32)); + ave_start_put32(cmd, l->hevc_conf, crop); + ave_start_put32(cmd, l->hevc_conf_left, 0); + ave_start_put32(cmd, l->hevc_conf_right, crop_r); + ave_start_put32(cmd, l->hevc_conf_top, 0); + ave_start_put32(cmd, l->hevc_conf_bottom, crop_b); + ave_start_put32(cmd, l->hevc_depth_luma, depth - 8); + ave_start_put32(cmd, l->hevc_depth_chroma, depth - 8); + /* Reference structure: B-frame pyramid or low-delay P (ave-image.c) */ + ave_start_put32(cmd, l->hevc_refs, p->bframes ? 4 : 1); + ave_start_put32(cmd, l->hevc_dpb_minus1, 4); + ave_start_put32(cmd, l->hevc_reorder, p->bframes ? 2 : 0); + ave_start_put32(cmd, l->hevc_reorder_copy, p->bframes ? 2 : 0); + ave_start_put32(cmd, l->hevc_reorder_on, p->bframes); +} + +int ave_build_start(struct ave_ctx *ctx, u8 *cmd, size_t len) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + const struct ave_params *p = &sess->p; + const struct ave_fw_layout *fw = ave->layout; + + if (!fw->encode_ready) + return -EOPNOTSUPP; + if (len != fw->bytes[p->codec - 1][AVE_IMAGE_START] || !p->refs || + p->refs > AVE_MAX_REFS || !p->slots || p->slots > fw->max_slots) + return -EINVAL; + ave_image_start(cmd, len, fw, p->codec, p->reorder_off, + p->codec == AVE_CODEC_HEVC && !p->bframes); + ave_write_header(ctx, cmd, AVE_OP_START, AVE_OP_START, AVE_SESSION_ARG0, + ave_session_setup_arg2(ctx)); + ave_start_surfaces(sess, cmd, fw); + ave_start_config(p, cmd, fw); + return 0; +} + +#define AVE_PROC_TIMESCALE 14400000ULL +#define AVE_PROC_SHDR_ENTRIES 256 +#define AVE_PROC_SHDR_STRIDE 0x400 +#define AVE_PROC_FORCED_TYPE 2 +#define AVE_PROC_LOW_DELAY 3 + +int ave_build_process(struct ave_ctx *ctx, u8 *cmd, size_t len, + unsigned int slot, dma_addr_t input, u32 bitrate, + bool force_key) +{ + struct ave_session *sess = ctx->sess; + const struct ave_params *p = &sess->p; + const struct ave_fw_layout *fw = ctx->ave->layout; + const struct ave_proc_layout *l = &fw->process[p->codec - 1]; + struct ave_slot *s; + dma_addr_t y = input; + u64 pts; + + if (!fw->encode_ready) + return -EOPNOTSUPP; + if (len != fw->bytes[p->codec - 1][AVE_IMAGE_PROCESS] || + slot >= p->slots || p->slots > fw->max_slots) + return -EINVAL; + if ((!l->force_key && force_key) || + (!l->bitrate && bitrate != p->bitrate)) + return -EOPNOTSUPP; + s = &sess->slot[slot]; + ave_image_process(cmd, len, fw, p->codec); + ave_write_header(ctx, cmd, AVE_OP_PROCESS, fw->process_subtype + slot, + 0, ave_session_arg2(ctx)); + /* Input picture: NV12, chroma right after p->height luma lines. */ + ave_put64(cmd, l->y, y); + ave_put32(cmd, l->y + 8, p->stride * p->height); + ave_put32(cmd, l->y + 12, p->stride); + ave_put64(cmd, l->uv, y + p->stride * p->height); + ave_put32(cmd, l->uv + 8, p->stride * p->height / 2); + ave_put32(cmd, l->uv + 12, p->stride); + ave_put32(cmd, l->in_w, p->width); + ave_put32(cmd, l->in_h, p->height); + + /* Session values repeated in every frame */ + ave_put32(cmd, l->coded_cap, p->coded_cap); + ave_put32(cmd, l->fps_int, p->fps_int); + /* The bit rate may change while streaming; it takes effect here. */ + if (l->bitrate) + ave_put32(cmd, l->bitrate, bitrate); + if (l->limits) + ave_put32(cmd, l->limits, p->limit_bytes ? 1 : 0); + if (l->limit_bytes && p->limit_bytes) { + ave_put64(cmd, l->limit_bytes, ave_f64(p->limit_bytes, 1)); + ave_put64(cmd, l->limit_secs, ave_f64(1, 1)); + } + ave_put32(cmd, l->const5, 5); + ave_put64(cmd, l->fps, ave_f64(p->fps_num, p->fps_den)); + ave_put64(cmd, l->timescale, AVE_PROC_TIMESCALE); + if (l->crop) + ave_put32(cmd, l->crop, + p->cw != p->width || p->ch != p->height); + + /* This frame */ + ave_put32(cmd, l->first_frame, sess->frames == 0); + ave_put32(cmd, l->slot, slot); + ave_put64(cmd, l->coded, s->coded.dma); + ave_put64(cmd, l->meta, s->meta.dma); + if (l->meta_size) + ave_put32(cmd, l->meta_size, s->meta.size); + if (l->extra) + ave_write_surface( + fw, cmd, + &(struct ave_surface_layout)AVE_SURFACE(l->extra, 0), 0, + s->extra.dma, SZ_16K); + if (l->frame_size) + ave_put32(cmd, l->frame_size, + p->rc_mode == 3 ? 0 : s->aux.size); + if (l->frame_buf) + ave_put64(cmd, l->frame_buf, + fw->counted && p->rc_mode == 3 ? 0 : s->aux.dma); + if (l->shdr_table) + for (unsigned int i = 0; i < AVE_PROC_SHDR_ENTRIES; i++) + ave_put64(cmd, l->shdr_table + 8 * i, + s->aux.dma + i * AVE_PROC_SHDR_STRIDE); + ave_put32(cmd, l->ordinal, sess->frames); + pts = mul_u64_u64_div_u64((u64)sess->frames * p->fps_den, + AVE_PROC_TIMESCALE, p->fps_num); + ave_put64(cmd, l->pts, pts); + if (l->force_key) + ave_put32(cmd, l->force_key, force_key); + if (l->force_type) + ave_put32(cmd, l->force_type, + force_key ? AVE_PROC_FORCED_TYPE : 0); + if (l->low_delay) + ave_put32(cmd, l->low_delay, + p->bframes ? 0 : AVE_PROC_LOW_DELAY); + return 0; +} diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h index c880b9a62d235..22275e2ebde02 100644 --- a/drivers/media/platform/apple/ave/ave.h +++ b/drivers/media/platform/apple/ave/ave.h @@ -8,8 +8,8 @@ #define _APPLE_AVE_H_ #include -#include #include +#include #include #include @@ -27,9 +27,9 @@ struct dev_pm_domain_list; struct iommu_domain; /* Frames the firmware can hold at once per session (Process "slots"). */ -#define AVE_MAX_SLOTS 7 +#define AVE_MAX_SLOTS 7 /* Sessions one firmware boot serves at once (macOS was seen running two). */ -#define AVE_MAX_SESSIONS 2 +#define AVE_MAX_SESSIONS 2 /* * Coprocessor memory every firmware boot uses, all inside one allocation. @@ -37,28 +37,28 @@ struct iommu_domain; * buffers for the scalar commands and one buffer per Process slot, so a * command stays untouched until the firmware acknowledges it. */ -#define AVE_IPC_BYTES (20 * SZ_1M) -#define AVE_IPC_CHANNEL_OFF 0x4000 -#define AVE_IPC_HANDSHAKE_OFF 0x24000 -#define AVE_IPC_SHARED_OFF 0x28000 -#define AVE_IPC_SHARED_BYTES SZ_64K -#define AVE_IPC_CMD_OFF 0x40000 -#define AVE_IPC_CMD_BYTES 0x1c000 -#define AVE_IPC_SCALAR_OFF 0x5c000 -#define AVE_IPC_SCALAR_BYTES 0x100 -#define AVE_IPC_SCALAR_BUFS 16 -#define AVE_IPC_PROC_OFF 0x60000 -#define AVE_IPC_PROC_BYTES 0x10000 -#define AVE_IPC_PROC_BUFS (AVE_MAX_SESSIONS * AVE_MAX_SLOTS) -#define AVE_IPC_END (AVE_IPC_PROC_OFF + AVE_IPC_PROC_BUFS * AVE_IPC_PROC_BYTES) +#define AVE_IPC_BYTES (20 * SZ_1M) +#define AVE_IPC_CHANNEL_OFF 0x4000 +#define AVE_IPC_HANDSHAKE_OFF 0x24000 +#define AVE_IPC_SHARED_OFF 0x28000 +#define AVE_IPC_SHARED_BYTES SZ_64K +#define AVE_IPC_CMD_OFF 0x40000 +#define AVE_IPC_CMD_BYTES 0x1c000 +#define AVE_IPC_SCALAR_OFF 0x5c000 +#define AVE_IPC_SCALAR_BYTES 0x100 +#define AVE_IPC_SCALAR_BUFS 16 +#define AVE_IPC_PROC_OFF 0x60000 +#define AVE_IPC_PROC_BYTES 0x10000 +#define AVE_IPC_PROC_BUFS (AVE_MAX_SESSIONS * AVE_MAX_SLOTS) +#define AVE_IPC_END (AVE_IPC_PROC_OFF + AVE_IPC_PROC_BUFS * AVE_IPC_PROC_BYTES) /* Command buffer ids, one bit each in ave_dev.unacked. */ -#define AVE_BUF_CMD 0 -#define AVE_BUF_SCALAR(i) (1 + (i)) -#define AVE_BUF_PROC(s) (1 + AVE_IPC_SCALAR_BUFS + (s)) -#define AVE_BUF_COUNT AVE_BUF_PROC(AVE_IPC_PROC_BUFS) +#define AVE_BUF_CMD 0 +#define AVE_BUF_SCALAR(i) (1 + (i)) +#define AVE_BUF_PROC(s) (1 + AVE_IPC_SCALAR_BUFS + (s)) +#define AVE_BUF_COUNT AVE_BUF_PROC(AVE_IPC_PROC_BUFS) -#define AVE_MSG_MAX SZ_4K +#define AVE_MSG_MAX SZ_4K enum ave_codec { AVE_CODEC_H264 = 1, @@ -68,8 +68,8 @@ enum ave_codec { struct ave_buf { void *cpu; dma_addr_t dma; - size_t size; /* what the firmware is told */ - size_t alloc; /* what is mapped, page rounded */ + size_t size; /* what the firmware is told */ + size_t alloc; /* what is mapped, page rounded */ unsigned long attrs; }; @@ -90,9 +90,9 @@ struct ave_ring { }; enum ave_fw_state { - AVE_FW_OFF, /* coprocessor in reset */ + AVE_FW_OFF, /* coprocessor in reset */ AVE_FW_RUNNING, - AVE_FW_HALTED, /* told to halt; restarts through the boot path */ + AVE_FW_HALTED, /* told to halt; restarts through the boot path */ AVE_FW_BROKEN, }; @@ -103,10 +103,26 @@ enum ave_image { AVE_IMAGE_COUNT, }; +/* Logical operations; wire IDs are selected by the firmware ABI. */ +enum ave_opcode { + AVE_OP_CONFIG = 1, + AVE_OP_HALT, + AVE_OP_OPEN, + AVE_OP_CLOSE, + AVE_OP_START = 6, + AVE_OP_STOP, + AVE_OP_PROCESS, + AVE_OP_COMPLETE, + AVE_OP_PRIORITY, + AVE_OP_RESET, + AVE_OP_FLUSH, + AVE_OP_COUNT, +}; + #include "ave-layout.h" /* The firmware build whose command layouts the driver generates */ -#define AVE_FW_TEXT_CRC_9003_78_0 0x69c54454 +#define AVE_FW_TEXT_CRC_9003_78_0 0x69c54454 struct ave_ctx; @@ -128,6 +144,13 @@ struct ave_variant { u32 boot_device_id; u32 device_type; u32 lfs_ref_scale; + u32 text_crc; + u32 boot_arg_value; + u32 timebase_multiplier; + u64 vcbar_mask; + u64 vcbar_flags; + + size_t extra0_size; /* Register sequence and boot/hello configuration have been qualified. */ bool boot_qualified; }; @@ -160,12 +183,18 @@ struct ave_dev { struct ave_buf boot_cfg; struct ave_buf log; struct ave_buf heap; + dma_addr_t extra0_dma; + size_t extra0_map_size; + bool extra0_mapped; + bool module_pinned; + /* A 6070 lifecycle command is serialized per logical operation. */ + struct ave_reply expected[AVE_OP_COUNT]; u64 ipc_fw_base; u32 channel_bytes; struct ave_ring ring[2]; u32 next_client; u32 unacked; - u8 want[16]; + u8 want[AVE_OP_COUNT]; int reply_err; unsigned int scalar_next; struct ave_ctx *sessions[AVE_MAX_SESSIONS]; @@ -183,45 +212,45 @@ struct ave_dev { /* Encoding parameters, fixed for the lifetime of a session. */ struct ave_params { enum ave_codec codec; - u32 width, height; /* visible picture */ - u32 cw, ch; /* coded size, multiples of 16 */ - u32 stride; /* NV12 line stride, both planes */ - u32 mbs; /* 16x16 macroblocks */ - u32 fps_num, fps_den; /* frames per second = num / den */ - u32 fps_int; /* the same, rounded */ + u32 width, height; /* visible picture */ + u32 cw, ch; /* coded size, multiples of 16 */ + u32 stride; /* NV12 line stride, both planes */ + u32 mbs; /* 16x16 macroblocks */ + u32 fps_num, fps_den; /* frames per second = num / den */ + u32 fps_int; /* the same, rounded */ u32 slots; - u32 refs; /* max_num_ref_frames + 1 */ + u32 refs; /* max_num_ref_frames + 1 */ bool bframes; bool intra_only; - bool reorder_off; /* the client asked for no B-frames */ + bool reorder_off; /* the client asked for no B-frames */ u32 gop; u32 rc_mode; u32 bitrate; - u32 limit_bytes; /* 0: no data-rate limit */ + u32 limit_bytes; /* 0: no data-rate limit */ s32 min_qp; u32 max_qp; - u32 profile; /* H.264: 2, 4, 6; HEVC: 1 */ - u32 level; /* firmware level enum, 1-based */ + u32 profile; /* H.264: 2, 4, 6; HEVC: 1 */ + u32 level; /* firmware level enum, 1-based */ bool high; u32 coded_cap; - u32 bit_depth; /* zero also means the existing 8-bit API */ - u32 initial_qp; /* CQ only; bit-rate modes use 26 */ + u32 bit_depth; /* zero also means the existing 8-bit API */ + u32 initial_qp; /* CQ only; bit-rate modes use 26 */ }; struct ave_slot { struct ave_buf coded; struct ave_buf meta; - struct ave_buf aux; /* H.264 per-frame buffer, HEVC slice headers */ - struct ave_buf extra; /* counted layout: per-slot 16 KiB buffer */ - struct ave_buf bounce; /* input copied to the encoder's stride */ + struct ave_buf aux; /* H.264 per-frame buffer, HEVC slice headers */ + struct ave_buf extra; /* counted layout: per-slot 16 KiB buffer */ + struct ave_buf bounce; /* input copied to the encoder's stride */ struct vb2_v4l2_buffer *src; u32 status; - bool busy; /* Process sent, reply pending */ - bool done; /* reply in, output not delivered yet */ + bool busy; /* Process sent, reply pending */ + bool done; /* reply in, output not delivered yet */ }; -#define AVE_MAX_REFS 5 -#define AVE_TILE_BUFS 8 +#define AVE_MAX_REFS 5 +#define AVE_TILE_BUFS 8 struct ave_session { struct ave_params p; @@ -243,23 +272,23 @@ struct ave_session { unsigned int done_head; unsigned int done_count; unsigned int busy; - u32 frames; /* Process commands sent */ + u32 frames; /* Process commands sent */ u32 client; - u64 token; /* command order token, per session */ - unsigned int index; /* in ave_dev.sessions[] */ - bool fw_open; /* Open sent, Close not yet */ - bool resume; /* closed for system sleep, reopen on next frame */ + u64 token; /* command order token, per session */ + unsigned int index; /* in ave_dev.sessions[] */ + bool fw_open; /* Open sent, Close not yet */ + bool resume; /* closed for system sleep, reopen on next frame */ bool copy_noted; /* CBR: bytes under budget so far, and who pads (see ave_sess_cbr_fill()) */ s64 cbr_surplus; bool fw_fills; bool host_fill_noted; bool warned; - bool checked[2]; /* Start/Process images checked for stale addresses */ - bool handed; /* Start sent: the firmware knows the buffers */ - bool nudged; /* Complete sent because input stopped */ - unsigned long progress; /* jiffies of the last frame sent or answered */ - unsigned long last_sent; /* jiffies of the last frame sent */ + bool checked[2]; /* Start/Process images checked for stale addresses */ + bool handed; /* Start sent: the firmware knows the buffers */ + bool nudged; /* Complete sent because input stopped */ + unsigned long progress; /* jiffies of the last frame sent or answered */ + unsigned long last_sent; /* jiffies of the last frame sent */ }; struct ave_ctx { @@ -287,7 +316,7 @@ struct ave_ctx { bool force_key_frame; struct ave_session *sess; - atomic_t ready; /* outputs waiting for a CAPTURE buffer */ + atomic_t ready; /* outputs waiting for a CAPTURE buffer */ bool draining; bool flush_pending; bool stopped; @@ -301,7 +330,10 @@ static inline struct ave_ctx *ave_file_to_ctx(struct file *file) return container_of(file_to_v4l2_fh(file), struct ave_ctx, fh); } +extern const struct ave_variant ave_t6000, ave_t8112; + /* ave-fw.c */ +u64 ave_fw_vcbar(const struct ave_variant *variant, u64 text_iova); int ave_fw_probe(struct ave_dev *ave); void ave_fw_remove(struct ave_dev *ave); int ave_fw_ensure_running(struct ave_dev *ave); @@ -309,29 +341,52 @@ void ave_fw_halt(struct ave_dev *ave); int ave_fw_shutdown(struct ave_dev *ave); void ave_fw_lost(struct ave_dev *ave); int ave_fw_check_build(struct ave_dev *ave); -int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, unsigned long attrs); -int ave_alloc_padded(struct ave_dev *ave, struct ave_buf *b, size_t size, size_t alloc, - unsigned long attrs); +int ave_fw_check_boot(struct ave_dev *ave); +void ave_fw_release_boot(struct ave_dev *ave); +int ave_fw_lifecycle_test(struct ave_dev *ave, enum ave_codec codec); +int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, + unsigned long attrs); +int ave_alloc_padded(struct ave_dev *ave, struct ave_buf *b, size_t size, + size_t alloc, unsigned long attrs); void ave_free(struct ave_dev *ave, struct ave_buf *b); /* ave-ipc.c */ +int ave_ipc_check_hello(const struct ave_fw_transport *t, const u32 hello[4]); +int ave_ipc_parse_channels(struct ave_dev *ave); int ave_ipc_negotiate(struct ave_dev *ave, const u32 hello[4]); void *ave_ipc_cmd(struct ave_dev *ave, unsigned int buf, size_t len); int ave_ipc_scalar_buf(struct ave_dev *ave); int ave_ipc_wait_ack(struct ave_dev *ave, unsigned int buf); -int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, u64 timeout_us, - bool reply); +int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, + u64 timeout_us, bool reply); int ave_ipc_pump(struct ave_dev *ave, unsigned int wait_ms, bool (*cond)(struct ave_dev *ave, void *arg), void *arg); int ave_ipc_settle(struct ave_dev *ave, unsigned int wait_ms); +/* ave-writer.c: pure command builders, also exercised by KUnit. */ +u32 ave_session_arg2(struct ave_ctx *ctx); +u32 ave_session_setup_arg2(struct ave_ctx *ctx); +void ave_write_header(struct ave_ctx *ctx, u8 *cmd, enum ave_opcode op, + u32 echo, u32 arg0, u32 arg2); +void ave_write_global(struct ave_dev *ave, u8 *cmd, enum ave_opcode op); +void ave_stamp_command(const struct ave_fw_abi *abi, u8 *cmd, u64 timeout, + u64 created); +void ave_write_surface(const struct ave_fw_layout *fw, u8 *cmd, + const struct ave_surface_layout *l, unsigned int index, + dma_addr_t addr, u32 size); +int ave_build_start(struct ave_ctx *ctx, u8 *cmd, size_t len); +int ave_build_process(struct ave_ctx *ctx, u8 *cmd, size_t len, + unsigned int slot, dma_addr_t input, u32 bitrate, + bool force_key); + /* ave-cmd.c */ int ave_cmd_config(struct ave_dev *ave); int ave_cmd_halt(struct ave_dev *ave); int ave_cmd_open(struct ave_ctx *ctx); int ave_cmd_start(struct ave_ctx *ctx); -int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *src, - dma_addr_t input, u32 bitrate, bool force_key); +int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, + struct vb2_v4l2_buffer *src, dma_addr_t input, u32 bitrate, + bool force_key); int ave_cmd_complete(struct ave_ctx *ctx, unsigned int count); int ave_cmd_close(struct ave_ctx *ctx); @@ -339,11 +394,11 @@ int ave_cmd_close(struct ave_ctx *ctx); struct ave_session *ave_sess_create(struct ave_ctx *ctx); void ave_sess_destroy(struct ave_ctx *ctx); int ave_sess_free_slot(struct ave_session *sess); -void ave_sess_reply(struct ave_dev *ave, const u8 *msg, u32 len); -int ave_sess_input(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *src, - dma_addr_t *input); -int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *dst, - bool *keyframe); +void ave_sess_reply(struct ave_dev *ave, const struct ave_reply *reply); +int ave_sess_input(struct ave_ctx *ctx, unsigned int slot, + struct vb2_v4l2_buffer *src, dma_addr_t *input); +int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, + struct vb2_v4l2_buffer *dst, bool *keyframe); void ave_sess_prepare_slot(struct ave_session *sess, unsigned int slot); bool ave_sess_stalled(struct ave_session *sess); bool ave_sess_starved(struct ave_session *sess); @@ -356,8 +411,8 @@ void ave_image_process(u8 *cmd, size_t len, const struct ave_fw_layout *fw, enum ave_codec codec); /* ave-drv.c */ -void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, enum ave_image kind, - const void *cmd, size_t len); +void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, + enum ave_image kind, const void *cmd, size_t len); /* ave-v4l2.c */ void ave_v4l2_suspend(struct ave_dev *ave); diff --git a/drivers/media/platform/apple/ave/tests/README b/drivers/media/platform/apple/ave/tests/README index d315f9d4d4bc8..a9650312d26a9 100644 --- a/drivers/media/platform/apple/ave/tests/README +++ b/drivers/media/platform/apple/ave/tests/README @@ -5,7 +5,8 @@ Run python3 run.py with a C compiler providing ASan and UBSan (CC may name its executable). No encoder, firmware or captured command image is needed. run.py reads the real AVE structs and compiles the unmodified bodies of -ave-image.c, ave-layout.c and ave-cmd.c, plus the real buffer-allocation, +ave-image.c, ave-layout.c, ave-abi.c, ave-writer.c and ave-cmd.c, plus the real +buffer-allocation, free, reply and firmware-build-gate functions. Only includes and unused kernel-facing declarations are omitted. shim.h supplies host primitives; boundary.h replaces IPC, DMA and logging. This does not validate the kernel @@ -27,3 +28,7 @@ The host shim intentionally uses stdint types, Linux's private ENOTSUPP value and side-effect-free calls to kernel-like arithmetic macros. Those produce checkpatch PREFER_KERNEL_TYPES, ENOTSUPP and MACRO_ARG_REUSE notices. MAINTAINERS already covers the entire AVE directory (FILE_PATH_CHANGES). + +The ABI refactor also has in-kernel arm64 KUnit tests selected by +VIDEO_APPLE_AVE_CMD_KUNIT_TEST. The host runner understands both the older +single-builder sources and the per-ABI writer split for baseline comparison. diff --git a/drivers/media/platform/apple/ave/tests/ave-9003-reference.h b/drivers/media/platform/apple/ave/tests/ave-9003-reference.h new file mode 100644 index 0000000000000..8402a4a5435f8 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/ave-9003-reference.h @@ -0,0 +1,828 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Frozen 9003 builder from 23e431a98962, for byte-regression tests only. + * Copyright The Aurora Silicon Contributors + */ +struct ave_ref_start_layout { + u16 ref_a, ref_b, ref_c, ref_c_alias, ref_d; + u16 ref_e, ref_f, ref_stride, ref_c_stride; + u16 mb, coded, meta, row_a, row_b, fixed, hevc_extra; + u16 tiles[8]; + u16 refs_count, work_count, lfs_count, slots_count, meta_count; + u16 mb_count, row_a_count, row_b_count, fixed_count, tiles_count; + u16 codec; + u16 flag_8bit; + u16 const_214; + u16 const_218; + u16 fps; + u16 rc_mode; + u16 bitrate; + u16 init_qp; + u16 limits; + u16 limit_bytes; + u16 limit_secs; + u16 f64_288; + u16 f64_290; + u16 min_qp; + u16 max_qp; + u16 gop_flags; + u16 p_frames; + u16 max_b; + u16 ref_depth; + u16 idr_interval; + u16 f64_2c0; + u16 f64_2c8; + u16 rc_flags; + u16 min_qp2; + u16 max_qp2; + u16 rc_310; + u16 size_class; + u16 coded_w; + u16 coded_h; + u16 codec_flags; + u16 h264_380; + u16 h264_384; + u16 codec_2588; + u16 bit_depth; + u16 coded_h2; + u16 h264_profile; + u16 h264_level; + u16 h264_max_refs; + u16 h264_width_mbs; + u16 h264_height_mbs; + u16 h264_crop; + u16 h264_crop_left; + u16 h264_crop_right; + u16 h264_crop_top; + u16 h264_crop_bottom; + u16 h264_cabac; + u16 h264_8x8; + u16 hevc_pad; + u16 hevc_rc_270c; + u16 hevc_rc_9604; + u16 hevc_rc_9608; + u16 hevc_profile_a; + u16 hevc_compat_a; + u16 hevc_8bit_a; + u16 hevc_level_a; + u16 hevc_profile_b; + u16 hevc_compat_b; + u16 hevc_8bit_b; + u16 hevc_level_b; + u16 hevc_width; + u16 hevc_height; + u16 hevc_width_ctb; + u16 hevc_height_ctb; + u16 hevc_conf; + u16 hevc_conf_left; + u16 hevc_conf_right; + u16 hevc_conf_top; + u16 hevc_conf_bottom; + u16 hevc_depth_luma; + u16 hevc_depth_chroma; + u16 hevc_refs; + u16 hevc_reorder_copy; + u16 hevc_dpb_minus1; + u16 hevc_reorder; + u16 hevc_reorder_on; + u16 geometry_mode, geometry_depth; + u16 profile_idc, level_idc, max_refs_copy, mb_size, mb_height; + u16 crop_copy, crop_right_copy, crop_bottom_copy; + u16 codec_byte, cabac_copy, transform_copy, hevc_cq, hevc_cq_copy; + u16 hevc_cq_codec, hevc_cq_picture; +}; + +struct ave_ref_proc_layout { + u16 first_frame; + u16 y, uv; /* {u64 address, u32 size, u32 stride} */ + u16 in_w, in_h; + u16 coded_cap; + u16 fps_int; + u16 bitrate; + u16 limits, limit_bytes, limit_secs; + u16 const5; + u16 fps; /* f64 */ + u16 timescale; /* u64 */ + u16 slot; + u16 coded, meta; /* u64 addresses */ + u16 ordinal; + u16 pts; /* u64 */ + u16 force_key; /* 1 on a forced key frame */ + u16 force_type; /* 2 on a forced key frame */ + u16 frame_buf; /* H.264 only */ + u16 shdr_table; /* HEVC only */ + u16 crop; /* HEVC only */ + u16 meta_size, extra, frame_size; + u16 low_delay; /* HEVC only: 3 with reordering off */ +}; + +struct ave_ref_fw_layout { + const char *name; + size_t bytes[2][AVE_IMAGE_COUNT]; + struct ave_ref_start_layout start[2]; + struct ave_ref_proc_layout process[2]; + u32 hdr_bytes, meta_bytes; + u32 process_timeout_us; + u8 process_subtype, max_slots, mcc_dsid; + bool counted; +}; + +static const struct ave_ref_fw_layout ave_reference_layout = { + .name = "9003.78.0 / 25G83", + .bytes = { { 0x3180, 0x63d8 }, { 0x13f28, 0xb1c0 } }, + .hdr_bytes = 0x12c000, + .meta_bytes = 0xc000, + .process_timeout_us = 94400000, + .process_subtype = 0x15, + .max_slots = 5, + .mcc_dsid = 0x12, + .start = { + [0] = { + .ref_a = 0x390, + .ref_b = 0x3a0, + .ref_c = 0x7d0, + .ref_c_alias = 0x7e0, + .ref_d = 0x1130, + .ref_stride = 0x20, + .ref_c_stride = 0x20, + .mb = 0xc10, + .coded = 0xd50, + .meta = 0xf50, + .row_a = 0x1350, + .row_b = 0x1390, + .fixed = 0x13d0, + .hevc_extra = 0x2520, + .tiles = { 0x1410, 0x1430, 0x1490, 0x14b0, 0x1510, + 0x1530, 0x1590, 0x15b0 }, + .codec = 0xc0, + .flag_8bit = 0x208, + .const_214 = 0x214, + .const_218 = 0x218, + .fps = 0x220, + .rc_mode = 0x234, + .bitrate = 0x238, + .init_qp = 0x240, + .limits = 0x250, + .limit_bytes = 0x258, + .limit_secs = 0x260, + .f64_288 = 0x288, + .f64_290 = 0x290, + .min_qp = 0x298, + .max_qp = 0x29c, + .gop_flags = 0x2a0, + .p_frames = 0x2ac, + .max_b = 0x2b0, + .ref_depth = 0x2b4, + .idr_interval = 0x2b8, + .f64_2c0 = 0x2c0, + .f64_2c8 = 0x2c8, + .rc_flags = 0x300, + .min_qp2 = 0x308, + .max_qp2 = 0x30c, + .rc_310 = 0x310, + .size_class = 0x320, + .coded_w = 0x368, + .coded_h = 0x36c, + .codec_flags = 0x370, + .h264_380 = 0x380, + .h264_384 = 0x384, + .codec_2588 = 0x2588, + .bit_depth = 0x2598, + .coded_h2 = 0x25dc, + .h264_profile = 0x291c, + .h264_level = 0x2938, + .h264_max_refs = 0x2d44, + .h264_width_mbs = 0x2d4c, + .h264_height_mbs = 0x2d50, + .h264_crop = 0x2db4, + .h264_crop_left = 0x2db8, + .h264_crop_right = 0x2dbc, + .h264_crop_top = 0x2dc0, + .h264_crop_bottom = 0x2dc4, + .h264_cabac = 0x2fd8, + .h264_8x8 = 0x3023, + .hevc_pad = 0x26e0, + .hevc_rc_270c = 0x270c, + .hevc_rc_9604 = 0x9604, + .hevc_rc_9608 = 0x9608, + .hevc_profile_a = 0x2938, + .hevc_compat_a = 0x293d, + .hevc_8bit_a = 0x2962, + .hevc_level_a = 0x296c, + .hevc_profile_b = 0x5904, + .hevc_compat_b = 0x5909, + .hevc_8bit_b = 0x592e, + .hevc_level_b = 0x5938, + .hevc_width = 0x5ab4, + .hevc_height = 0x5ab8, + .hevc_width_ctb = 0x5abc, + .hevc_height_ctb = 0x5ac0, + .hevc_conf = 0x5ac4, + .hevc_conf_left = 0x5ac8, + .hevc_conf_right = 0x5acc, + .hevc_conf_top = 0x5ad0, + .hevc_conf_bottom = 0x5ad4, + .hevc_depth_luma = 0x5ad8, + .hevc_depth_chroma = 0x5adc, + .hevc_refs = 0x259c, + .hevc_reorder_copy = 0x2e54, + .hevc_dpb_minus1 = 0x5b00, + .hevc_reorder = 0x5b04, + .hevc_reorder_on = 0x95fc, + }, + [1] = { + .ref_a = 0x390, + .ref_b = 0x3a0, + .ref_c = 0x7d0, + .ref_c_alias = 0x7e0, + .ref_d = 0x1130, + .ref_stride = 0x20, + .ref_c_stride = 0x20, + .mb = 0xc10, + .coded = 0xd50, + .meta = 0xf50, + .row_a = 0x1350, + .row_b = 0x1390, + .fixed = 0x13d0, + .hevc_extra = 0x2520, + .tiles = { 0x1410, 0x1430, 0x1490, 0x14b0, 0x1510, + 0x1530, 0x1590, 0x15b0 }, + .codec = 0xc0, + .flag_8bit = 0x208, + .const_214 = 0x214, + .const_218 = 0x218, + .fps = 0x220, + .rc_mode = 0x234, + .bitrate = 0x238, + .init_qp = 0x240, + .limits = 0x250, + .limit_bytes = 0x258, + .limit_secs = 0x260, + .f64_288 = 0x288, + .f64_290 = 0x290, + .min_qp = 0x298, + .max_qp = 0x29c, + .gop_flags = 0x2a0, + .p_frames = 0x2ac, + .max_b = 0x2b0, + .ref_depth = 0x2b4, + .idr_interval = 0x2b8, + .f64_2c0 = 0x2c0, + .f64_2c8 = 0x2c8, + .rc_flags = 0x300, + .min_qp2 = 0x308, + .max_qp2 = 0x30c, + .rc_310 = 0x310, + .size_class = 0x320, + .coded_w = 0x368, + .coded_h = 0x36c, + .codec_flags = 0x370, + .h264_380 = 0x380, + .h264_384 = 0x384, + .codec_2588 = 0x2588, + .bit_depth = 0x2598, + .coded_h2 = 0x25dc, + .h264_profile = 0x291c, + .h264_level = 0x2938, + .h264_max_refs = 0x2d44, + .h264_width_mbs = 0x2d4c, + .h264_height_mbs = 0x2d50, + .h264_crop = 0x2db4, + .h264_crop_left = 0x2db8, + .h264_crop_right = 0x2dbc, + .h264_crop_top = 0x2dc0, + .h264_crop_bottom = 0x2dc4, + .h264_cabac = 0x2fd8, + .h264_8x8 = 0x3023, + .hevc_pad = 0x26e0, + .hevc_rc_270c = 0x270c, + .hevc_rc_9604 = 0x9604, + .hevc_rc_9608 = 0x9608, + .hevc_profile_a = 0x2938, + .hevc_compat_a = 0x293d, + .hevc_8bit_a = 0x2962, + .hevc_level_a = 0x296c, + .hevc_profile_b = 0x5904, + .hevc_compat_b = 0x5909, + .hevc_8bit_b = 0x592e, + .hevc_level_b = 0x5938, + .hevc_width = 0x5ab4, + .hevc_height = 0x5ab8, + .hevc_width_ctb = 0x5abc, + .hevc_height_ctb = 0x5ac0, + .hevc_conf = 0x5ac4, + .hevc_conf_left = 0x5ac8, + .hevc_conf_right = 0x5acc, + .hevc_conf_top = 0x5ad0, + .hevc_conf_bottom = 0x5ad4, + .hevc_depth_luma = 0x5ad8, + .hevc_depth_chroma = 0x5adc, + .hevc_refs = 0x259c, + .hevc_reorder_copy = 0x2e54, + .hevc_dpb_minus1 = 0x5b00, + .hevc_reorder = 0x5b04, + .hevc_reorder_on = 0x95fc, + }, + }, + .process = { + [0] = { + .first_frame = 0x12c0, + .y = 0x5830, + .uv = 0x5850, + .in_w = 0x58e8, + .in_h = 0x58ec, + .coded_cap = 0x61c8, + .fps_int = 0x1610, + .bitrate = 0x1620, + .limits = 0x1628, + .limit_bytes = 0x1630, + .limit_secs = 0x1638, + .const5 = 0x6238, + .fps = 0x6268, + .timescale = 0x6278, + .slot = 0x61b4, + .coded = 0x61b8, + .meta = 0x61c0, + .ordinal = 0x6228, + .pts = 0x6270, + .force_key = 0x1658, + .force_type = 0x29f8, + .frame_buf = 0x58c8, + }, + [1] = { + .first_frame = 0x5d90, + .y = 0xa300, + .uv = 0xa320, + .in_w = 0xa3b8, + .in_h = 0xa3bc, + .coded_cap = 0xac98, + .fps_int = 0x60e0, + .bitrate = 0x60f0, + .limits = 0x60f8, + .limit_bytes = 0x6100, + .limit_secs = 0x6108, + .const5 = 0xad08, + .fps = 0xad38, + .timescale = 0xad48, + .slot = 0xac84, + .coded = 0xac88, + .meta = 0xac90, + .ordinal = 0xacf8, + .pts = 0xad40, + .force_key = 0x6128, + .force_type = 0x74c8, + .shdr_table = 0xd88, + .crop = 0x74dc, + .low_delay = 0xcf8, + }, + }, +}; + +static u64 ave_f64(u64 num, u64 den) +{ + u64 mant = 0, rem; + int exp = 0; + + if (!num || !den) + return 0; + while (num >= den << 1) { + den <<= 1; + exp++; + } + while (num < den) { + num <<= 1; + exp--; + } + /* 1 <= num/den < 2: take 53 significant bits by long division. */ + rem = num; + for (unsigned int i = 0; i < 53; i++) { + mant <<= 1; + if (rem >= den) { + mant |= 1; + rem -= den; + } + rem <<= 1; + } + return (u64)(exp + 1023) << 52 | (mant & GENMASK_ULL(51, 0)); +} + +#define AVE_F64_NEG BIT_ULL(63) + +static void ave_put32(u8 *cmd, u32 off, u32 val) +{ + put_unaligned_le32(val, cmd + off); +} + +static void ave_put64(u8 *cmd, u32 off, u64 val) +{ + put_unaligned_le64(val, cmd + off); +} + +/* A surface descriptor: {u64 DART address, u32 size, u32 0}. */ +static void ave_put_surface(u8 *cmd, u32 off, dma_addr_t addr, u32 size) +{ + ave_put64(cmd, off, addr); + ave_put32(cmd, off + 8, size); + ave_put32(cmd, off + 12, 0); +} + +static void ave_put_buf(u8 *cmd, u32 off, const struct ave_buf *b, u32 size) +{ + ave_put_surface(cmd, off, b->dma, size); +} + +/* Is another session open in the firmware besides @ctx's? */ +static bool ave_cmd_others_open(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i] && ave->sessions[i] != ctx && + ave->sessions[i]->sess->fw_open) + return true; + return false; +} + +/* + * Header word +0x2c depends on the set of open sessions: 0x1e0 on Open and + * Start (0x140 when another session is open), the codec value on every + * other command of a lone session, and 0 while another one is open. + */ +static u32 ave_ref_session_arg2(struct ave_ctx *ctx) +{ + if (ctx->ave->layout->counted && ctx->sess->p.codec == AVE_CODEC_H264) + return ctx->sess->p.bframes ? AVE_SESSION_ARG2_H264 : 0x140; + if (ave_cmd_others_open(ctx)) + return 0; + return ctx->sess->p.codec == AVE_CODEC_HEVC ? AVE_SESSION_ARG2_HEVC : + AVE_SESSION_ARG2_H264; +} + +static u32 ave_ref_session_setup_arg2(struct ave_ctx *ctx) +{ + return ave_cmd_others_open(ctx) ? AVE_SESSION_ARG2_SETUP_SHARED : + AVE_SESSION_ARG2_SETUP; +} + +static void ave_cmd_header(struct ave_ctx *ctx, u8 *cmd, enum ave_opcode op, + u32 echo, u32 arg0, u32 arg2) +{ + ctx->sess->token += AVE_TOKEN_STEP; + ave_put32(cmd, AVE_CMD_OPCODE, op); + ave_put64(cmd, AVE_CMD_TOKEN, ctx->sess->token); + ave_put64(cmd, AVE_CMD_CLIENT, ctx->sess->client); + ave_put32(cmd, AVE_CMD_UNIT, AVE_SESSION_UNIT); + ave_put32(cmd, AVE_CMD_CODEC, ctx->sess->p.codec); + ave_put32(cmd, AVE_CMD_OPCODE_ECHO, echo); + ave_put32(cmd, AVE_CMD_ARG0, arg0); + ave_put32(cmd, AVE_CMD_ARG1, AVE_SESSION_ARG1); + ave_put32(cmd, AVE_CMD_ARG2, arg2); +} + +#define AVE_START_ARRAY_LEN 30 +#define AVE_START_INIT_QP_VALUE 26 + +static void ave_start_surfaces(struct ave_session *sess, u8 *cmd, + const struct ave_ref_fw_layout *fw) +{ + const struct ave_params *p = &sess->p; + bool hevc = p->codec == AVE_CODEC_HEVC; + unsigned int tiles = hevc ? 4 : AVE_TILE_BUFS; + const struct ave_ref_start_layout *l = &fw->start[p->codec - 1]; + + /* Clear every descriptor the image may carry, then fill ours. */ + for (unsigned int i = 0; i < AVE_MAX_REFS; i++) { + ave_put_surface(cmd, l->ref_a + l->ref_stride * i, 0, 0); + ave_put_surface(cmd, l->ref_b + l->ref_stride * i, 0, 0); + ave_put_surface(cmd, l->ref_c + l->ref_c_stride * i, 0, 0); + ave_put_surface(cmd, l->ref_c_alias + l->ref_c_stride * i, 0, + 0); + ave_put_surface(cmd, l->ref_d + 0x10 * i, 0, 0); + } + for (unsigned int s = 0; s < AVE_START_ARRAY_LEN; s++) { + ave_put_surface(cmd, l->coded + 0x10 * s, 0, 0); + ave_put_surface(cmd, l->meta + 0x10 * s, 0, 0); + } + + ave_put_buf(cmd, 0x48, &sess->hdr, fw->hdr_bytes); + ave_put_surface(cmd, 0x58, 0, 0); + for (unsigned int i = 0; i < p->refs; i++) { + ave_put_buf(cmd, l->ref_a + l->ref_stride * i, &sess->ref_a[i], + sess->ref_a[i].size); + ave_put_buf(cmd, l->ref_b + l->ref_stride * i, &sess->ref_b[i], + sess->ref_b[i].size); + ave_put_buf(cmd, l->ref_c + l->ref_c_stride * i, + &sess->ref_c[i], sess->ref_c[i].size); + ave_put_surface(cmd, l->ref_c_alias + l->ref_c_stride * i, + sess->ref_c[i].dma, 0); + ave_put_buf(cmd, l->ref_d + 0x10 * i, &sess->ref_d[i], + sess->ref_d[i].size); + } + if (fw->counted) { + ave_put32(cmd, l->refs_count, p->refs); + ave_put32(cmd, l->work_count, p->refs); + ave_put32(cmd, l->lfs_count, p->refs); + ave_put32(cmd, l->slots_count, p->slots); + ave_put32(cmd, l->meta_count, p->slots); + ave_put32(cmd, l->mb_count, p->rc_mode == 3 ? 3 : 1); + if (p->rc_mode == 3) + for (unsigned int i = 0; i < 2; i++) + ave_put_buf(cmd, l->mb + 0x10 * (i + 1), + &sess->mb_extra[i], + sess->mb_extra[i].size); + ave_put32(cmd, l->tiles_count, tiles); + for (unsigned int i = 0; i < p->refs; i++) { + ave_put_buf(cmd, l->ref_e + l->ref_stride * i, + &sess->ref_e[i], sess->ref_e[i].size); + ave_put_buf(cmd, l->ref_f + l->ref_stride * i, + &sess->ref_f[i], sess->ref_f[i].size); + } + } + ave_put_buf(cmd, l->mb, &sess->mb, sess->mb.size); + for (unsigned int s = 0; s < p->slots; s++) { + ave_put_buf(cmd, l->coded + 0x10 * s, &sess->slot[s].coded, + p->coded_cap); + ave_put_buf(cmd, l->meta + 0x10 * s, &sess->slot[s].meta, + sess->slot[s].meta.size); + } + ave_put_buf(cmd, l->row_a, &sess->row_a, sess->row_a.size); + ave_put_buf(cmd, l->row_b, &sess->row_b, sess->row_b.size); + ave_put_buf(cmd, l->fixed, &sess->fixed, SZ_16K); + for (unsigned int i = 0; i < AVE_TILE_BUFS; i++) { + u32 off = l->tiles[i]; + + if (i < tiles) { + ave_put_buf(cmd, off, &sess->tile[i], + sess->tile[i].size); + ave_put_buf(cmd, off + 0x10, &sess->tile_aux[i], + sess->tile_aux[i].size); + } else { + ave_put_surface(cmd, off, 0, 0); + ave_put_surface(cmd, off + 0x10, 0, 0); + } + } + if (hevc) + for (unsigned int i = 0; i < 2; i++) + ave_put_buf(cmd, l->hevc_extra + 0x10 * i, + &sess->hevc_extra[i], + sess->hevc_extra[i].size); +} + +static void ave_start_put32(u8 *cmd, u32 off, u32 val) +{ + if (off) + ave_put32(cmd, off, val); +} + +static void ave_start_config(const struct ave_params *p, u8 *cmd, + const struct ave_ref_fw_layout *fw) +{ + bool hevc = p->codec == AVE_CODEC_HEVC; + bool cq = p->rc_mode == 3; + u32 depth = p->bit_depth ?: 8; + u32 crop_r = (p->cw - p->width) / 2, crop_b = (p->ch - p->height) / 2; + bool crop = crop_r || crop_b; + const struct ave_ref_start_layout *l = &fw->start[p->codec - 1]; + + ave_start_put32(cmd, l->codec, p->codec); + ave_start_put32( + cmd, l->flag_8bit, + fw->counted ? + 0x10000 | (hevc ? (depth == 8 ? 0x41 : 0x40) : 0) | + (cq ? 0x10 : 0) : + hevc ? 1 : + 0); + ave_start_put32(cmd, l->geometry_mode, + (hevc ? 0x40 : 0) | (cq ? 0 : 2)); + ave_start_put32(cmd, l->geometry_depth, depth); + ave_start_put32(cmd, l->const_214, 0x39d); + ave_start_put32(cmd, l->const_218, 0x18); + ave_start_put32(cmd, l->fps, p->fps_int); + + /* Rate control */ + ave_start_put32(cmd, l->rc_mode, p->rc_mode); + ave_start_put32(cmd, l->bitrate, p->bitrate); + for (unsigned int i = 0; i < 3; i++) + ave_put32(cmd, (l->init_qp + 4 * i), + cq ? p->initial_qp : AVE_START_INIT_QP_VALUE); + ave_start_put32(cmd, l->limits, p->limit_bytes ? 1 : 0); + if (l->limit_bytes && p->limit_bytes) { + ave_put64(cmd, l->limit_bytes, + ave_f64((u64)p->limit_bytes * (fw->counted ? 8 : 1), + 1)); + ave_put64(cmd, l->limit_secs, ave_f64(1, 1)); + } + ave_put64(cmd, l->f64_288, ave_f64(1, 1)); + ave_put64(cmd, l->f64_290, ave_f64(1, 2)); + ave_put64(cmd, l->f64_2c0, AVE_F64_NEG | ave_f64(1, 1)); + ave_put64(cmd, l->f64_2c8, AVE_F64_NEG | ave_f64(1, 1)); + ave_start_put32(cmd, l->min_qp, p->min_qp); + ave_start_put32(cmd, l->max_qp, p->max_qp); + ave_start_put32(cmd, l->min_qp2, p->min_qp); + ave_start_put32(cmd, l->max_qp2, p->max_qp); + ave_start_put32(cmd, l->rc_flags, cq ? 0 : hevc ? 0x736 : 0x8202); + ave_start_put32(cmd, l->rc_310, cq ? 0 : 30); + if (!hevc) + ave_start_put32(cmd, l->h264_380, 1); + + /* GOP structure: with B-frames as captured, or P-only, or intra-only. */ + ave_start_put32( + cmd, l->gop_flags, + (p->bframes ? 0x600 : 0x200) | + (fw->counted && cq ? 0x10000 | (hevc ? 0x100 : 0) : 0)); + ave_start_put32(cmd, l->p_frames, p->intra_only ? 0 : 1); + ave_start_put32(cmd, l->max_b, p->bframes ? 3 : 0); + ave_start_put32(cmd, l->ref_depth, p->bframes ? 3 : 1); + ave_start_put32(cmd, l->idr_interval, p->gop); + + /* Picture size */ + ave_start_put32(cmd, l->size_class, + p->width * p->height > 1280 * 720 ? 3 : 1); + ave_start_put32(cmd, l->coded_w, p->cw); + ave_start_put32(cmd, l->coded_h, p->ch); + ave_start_put32(cmd, l->coded_h2, p->ch); + ave_start_put32(cmd, l->codec_flags, hevc ? 0x2000002 : 2); + ave_start_put32(cmd, l->bit_depth, depth); + + if (!hevc) { + ave_start_put32(cmd, l->h264_384, + p->width * p->height <= 320 * 240 ? 0x40 : + 0x10); + ave_start_put32(cmd, l->codec_2588, p->high ? 2 : 0); + ave_start_put32(cmd, l->h264_profile, p->profile); + ave_start_put32(cmd, l->h264_level, p->level); + ave_start_put32(cmd, l->h264_max_refs, + (p->refs - 1) | + (fw->counted ? (p->cw / 16) << 16 : 0)); + if (fw->counted) { + static const u8 level_idc[] = { + 10, 9, 11, 12, 13, 20, 21, 22, 30, + 31, 32, 40, 41, 42, 50, 51, 52, + }; + + ave_put32(cmd, l->profile_idc, + p->high ? 100 : + p->profile == 2 ? 66 : + 77); + ave_put32(cmd, l->level_idc, level_idc[p->level - 1]); + put_unaligned_le16(p->refs - 1, cmd + l->max_refs_copy); + ave_put32(cmd, l->mb_size, + ((p->ch / 16 - 1) << 16) | (p->cw / 16 - 1)); + cmd[l->mb_height] = p->ch / 16; + cmd[l->mb_height + 2] = 1; + cmd[l->mb_height + 3] = 1; + cmd[l->crop_copy] = crop; + put_unaligned_le16(crop_r, cmd + l->crop_right_copy); + put_unaligned_le16(crop_b, cmd + l->crop_bottom_copy); + cmd[l->codec_byte] = !cq; + /* CABAC flag in the second, packed picture structure. */ + cmd[l->cabac_copy] = p->profile != 2; + cmd[l->transform_copy] = p->high; + } + ave_start_put32(cmd, l->h264_width_mbs, p->cw / 16 - 1); + ave_start_put32(cmd, l->h264_height_mbs, p->ch / 16 - 1); + ave_start_put32(cmd, l->h264_crop, crop); + ave_start_put32(cmd, l->h264_crop_left, 0); + ave_start_put32(cmd, l->h264_crop_right, crop_r); + ave_start_put32(cmd, l->h264_crop_top, 0); + ave_start_put32(cmd, l->h264_crop_bottom, crop_b); + ave_start_put32(cmd, l->h264_cabac, p->profile != 2); + cmd[l->h264_8x8] = p->high; + return; + } + + ave_start_put32(cmd, l->hevc_pad, + 1 | (p->cw > p->width ? BIT(16) | BIT(18) : 0) | + (p->ch > p->height ? BIT(17) | BIT(19) : 0)); + ave_start_put32(cmd, l->hevc_profile_a, p->profile); + ave_start_put32(cmd, l->hevc_profile_b, p->profile); + cmd[l->hevc_compat_a] = depth == 8; + cmd[l->hevc_compat_a + 1] = 1; + cmd[l->hevc_compat_b] = depth == 8; + cmd[l->hevc_compat_b + 1] = 1; + cmd[l->hevc_8bit_a] = depth == 8; + cmd[l->hevc_8bit_b] = depth == 8; + /* Rate-control words that only change in constant-quality mode */ + ave_start_put32(cmd, l->hevc_cq, cq); + ave_start_put32(cmd, l->hevc_cq_copy, cq); + ave_start_put32(cmd, l->hevc_cq_codec, cq); + ave_start_put32(cmd, l->hevc_cq_picture, cq ? 0x100 : 0); + ave_start_put32(cmd, l->hevc_rc_270c, 2); + ave_start_put32(cmd, l->hevc_rc_9604, cq ? 2 : 0); + ave_start_put32(cmd, l->hevc_rc_9608, cq ? 0 : 0x10000); + ave_start_put32(cmd, l->hevc_level_a, p->level); + ave_start_put32(cmd, l->hevc_level_b, p->level); + ave_start_put32(cmd, l->hevc_width, p->cw); + ave_start_put32(cmd, l->hevc_height, p->ch); + ave_start_put32(cmd, l->hevc_width_ctb, DIV_ROUND_UP(p->cw, 32)); + ave_start_put32(cmd, l->hevc_height_ctb, DIV_ROUND_UP(p->ch, 32)); + ave_start_put32(cmd, l->hevc_conf, crop); + ave_start_put32(cmd, l->hevc_conf_left, 0); + ave_start_put32(cmd, l->hevc_conf_right, crop_r); + ave_start_put32(cmd, l->hevc_conf_top, 0); + ave_start_put32(cmd, l->hevc_conf_bottom, crop_b); + ave_start_put32(cmd, l->hevc_depth_luma, depth - 8); + ave_start_put32(cmd, l->hevc_depth_chroma, depth - 8); + /* Reference structure: B-frame pyramid or low-delay P (ave-image.c) */ + ave_start_put32(cmd, l->hevc_refs, p->bframes ? 4 : 1); + ave_start_put32(cmd, l->hevc_dpb_minus1, 4); + ave_start_put32(cmd, l->hevc_reorder, p->bframes ? 2 : 0); + ave_start_put32(cmd, l->hevc_reorder_copy, p->bframes ? 2 : 0); + ave_start_put32(cmd, l->hevc_reorder_on, p->bframes); +} + +static void ave_reference_start(struct ave_ctx *ctx, u8 *cmd, size_t len) +{ + const struct ave_ref_fw_layout *fw = &ave_reference_layout; + const struct ave_params *p = &ctx->sess->p; + + ave_image_start(cmd, len, ctx->ave->layout, p->codec, p->reorder_off, + p->codec == AVE_CODEC_HEVC && !p->bframes); + ave_cmd_header(ctx, cmd, AVE_OP_START, AVE_OP_START, AVE_SESSION_ARG0, + ave_ref_session_setup_arg2(ctx)); + ave_start_surfaces(ctx->sess, cmd, fw); + ave_start_config(p, cmd, fw); +} + +#define AVE_PROC_TIMESCALE 14400000ULL +#define AVE_PROC_SHDR_ENTRIES 256 +#define AVE_PROC_SHDR_STRIDE 0x400 +#define AVE_PROC_FORCED_TYPE 2 +#define AVE_PROC_LOW_DELAY 3 + +static void ave_reference_process(struct ave_ctx *ctx, u8 *cmd, size_t len, + unsigned int slot, dma_addr_t input, + u32 bitrate, bool force_key) +{ + struct ave_session *sess = ctx->sess; + const struct ave_params *p = &sess->p; + const struct ave_ref_fw_layout *fw = &ave_reference_layout; + const struct ave_ref_proc_layout *l = &fw->process[p->codec - 1]; + struct ave_slot *s = &sess->slot[slot]; + dma_addr_t y = input; + u64 pts; + + ave_image_process(cmd, len, ctx->ave->layout, p->codec); + ave_cmd_header(ctx, cmd, AVE_OP_PROCESS, fw->process_subtype + slot, 0, + ave_ref_session_arg2(ctx)); + /* Input picture: NV12, chroma right after p->height luma lines. */ + ave_put64(cmd, l->y, y); + ave_put32(cmd, l->y + 8, p->stride * p->height); + ave_put32(cmd, l->y + 12, p->stride); + ave_put64(cmd, l->uv, y + p->stride * p->height); + ave_put32(cmd, l->uv + 8, p->stride * p->height / 2); + ave_put32(cmd, l->uv + 12, p->stride); + ave_put32(cmd, l->in_w, p->width); + ave_put32(cmd, l->in_h, p->height); + + /* Session values repeated in every frame */ + ave_put32(cmd, l->coded_cap, p->coded_cap); + ave_put32(cmd, l->fps_int, p->fps_int); + /* The bit rate may change while streaming; it takes effect here. */ + if (l->bitrate) + ave_put32(cmd, l->bitrate, bitrate); + if (l->limits) + ave_put32(cmd, l->limits, p->limit_bytes ? 1 : 0); + if (l->limit_bytes && p->limit_bytes) { + ave_put64(cmd, l->limit_bytes, ave_f64(p->limit_bytes, 1)); + ave_put64(cmd, l->limit_secs, ave_f64(1, 1)); + } + ave_put32(cmd, l->const5, 5); + ave_put64(cmd, l->fps, ave_f64(p->fps_num, p->fps_den)); + ave_put64(cmd, l->timescale, AVE_PROC_TIMESCALE); + if (l->crop) + ave_put32(cmd, l->crop, + p->cw != p->width || p->ch != p->height); + + /* This frame */ + ave_put32(cmd, l->first_frame, sess->frames == 0); + ave_put32(cmd, l->slot, slot); + ave_put64(cmd, l->coded, s->coded.dma); + ave_put64(cmd, l->meta, s->meta.dma); + if (l->meta_size) + ave_put32(cmd, l->meta_size, s->meta.size); + if (l->extra) + ave_put_buf(cmd, l->extra, &s->extra, SZ_16K); + if (l->frame_size) + ave_put32(cmd, l->frame_size, + p->rc_mode == 3 ? 0 : s->aux.size); + if (l->frame_buf) + ave_put64(cmd, l->frame_buf, + fw->counted && p->rc_mode == 3 ? 0 : s->aux.dma); + if (l->shdr_table) + for (unsigned int i = 0; i < AVE_PROC_SHDR_ENTRIES; i++) + ave_put64(cmd, l->shdr_table + 8 * i, + s->aux.dma + i * AVE_PROC_SHDR_STRIDE); + ave_put32(cmd, l->ordinal, sess->frames); + pts = mul_u64_u64_div_u64((u64)sess->frames * p->fps_den, + AVE_PROC_TIMESCALE, p->fps_num); + ave_put64(cmd, l->pts, pts); + if (l->force_key) + ave_put32(cmd, l->force_key, force_key); + if (l->force_type) + ave_put32(cmd, l->force_type, + force_key ? AVE_PROC_FORCED_TYPE : 0); + if (l->low_delay) + ave_put32(cmd, l->low_delay, + p->bframes ? 0 : AVE_PROC_LOW_DELAY); +} diff --git a/drivers/media/platform/apple/ave/tests/boundary.h b/drivers/media/platform/apple/ave/tests/boundary.h index 22aa6bba0ff66..3598fbfe14576 100644 --- a/drivers/media/platform/apple/ave/tests/boundary.h +++ b/drivers/media/platform/apple/ave/tests/boundary.h @@ -3,6 +3,7 @@ struct ave_dev { struct ave_ctx *sessions[AVE_MAX_SESSIONS]; u32 next_client, text_crc, data_crc; + dma_addr_t extra0_dma; const struct ave_fw_layout *layout; const struct ave_variant *variant; enum ave_fw_state fw_state; @@ -60,8 +61,13 @@ static int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, assert(len == wire_len); for (unsigned int i = 0; i < 32; i++) assert(wire[len + i] == 0xa5); +#ifdef TEST_PER_ABI + put_unaligned_le64(timeout, wire + ave->layout->abi->timeout_offset); + put_unaligned_le64(1234567, wire + ave->layout->abi->created_offset); +#else put_unaligned_le64(timeout, wire + AVE_CMD_TIMEOUT); put_unaligned_le64(1234567, wire + AVE_CMD_CREATED); +#endif submissions++; return 0; } diff --git a/drivers/media/platform/apple/ave/tests/run.py b/drivers/media/platform/apple/ave/tests/run.py index 6dc7482914884..14456d5283191 100644 --- a/drivers/media/platform/apple/ave/tests/run.py +++ b/drivers/media/platform/apple/ave/tests/run.py @@ -33,6 +33,8 @@ def source(text): def build(src, out, baseline): header = (src / 'ave.h').read_text() + per_abi = (src / 'ave-writer.c').exists() + has_layout = (src / 'ave-layout.h').exists() code = (Path(__file__).parent / 'shim.h').read_text() for name in ('AVE_MAX_SLOTS', 'AVE_MAX_REFS', 'AVE_TILE_BUFS', 'AVE_MAX_SESSIONS', 'AVE_FW_TEXT_CRC_9003_78_0'): @@ -40,16 +42,26 @@ def build(src, out, baseline): for name in ('ave_codec', 'ave_buf', 'ave_fw_state', 'ave_image', 'ave_coproc_regs', 'ave_variant', 'ave_params', 'ave_slot', 'ave_session'): code += declaration(header, name) + if per_abi: + code = '#define TEST_PER_ABI 1\n' + code + code += declaration(header, 'ave_opcode') + for name in ('AVE_IPC_BYTES', 'AVE_IPC_CMD_OFF', 'AVE_IPC_CMD_BYTES', + 'AVE_IPC_SCALAR_OFF', 'AVE_IPC_PROC_OFF', 'AVE_IPC_PROC_BYTES'): + code += re.search(r'^#define ' + name + r'\s+[^\n]+', header, re.M)[0] + '\n' if baseline: code = '#define TEST_BASELINE 1\n' + code code += '\n'.join(re.findall(r'^#define AVE_(?:START|PROCESS)_\w+.*$', header, re.M)) + '\n' - else: + if has_layout: + code = '#define TEST_LAYOUT 1\n' + code code += source((src / 'ave-layout.h').read_text()) code += source((src / 'ave-layout.c').read_text()) code += source((src / 'ave-regs.h').read_text()) code += source((src / 'ave-sizes.h').read_text()) code += (Path(__file__).parent / 'boundary.h').read_text() code += source((src / 'ave-image.c').read_text()) + if per_abi: + code += source((src / 'ave-writer.c').read_text()) + code += source((src / 'ave-abi.c').read_text()) code += source((src / 'ave-cmd.c').read_text()) session = (src / 'ave-sess.c').read_text() code += declaration(session, 'ave_buf_req') @@ -63,7 +75,10 @@ def build(src, out, baseline): for name in ('ave_buf_req_cmp', 'ave_req', 'ave_sess_alloc', 'ave_sess_free_bufs', 'ave_sess_reply'): code += function(session, name) if not baseline: - code += function((src / 'ave-fw.c').read_text(), 'ave_fw_check_build') + fw = (src / 'ave-fw.c').read_text() + if per_abi: + code += function(fw, 'ave_fw_check_boot') + code += function(fw, 'ave_fw_check_build') code += (Path(__file__).parent / 'test.c').read_text() out.mkdir(parents=True, exist_ok=True) (out / 'host.c').write_text(code) diff --git a/drivers/media/platform/apple/ave/tests/shim.h b/drivers/media/platform/apple/ave/tests/shim.h index bc3dea7f80fc1..e801d6378f1be 100644 --- a/drivers/media/platform/apple/ave/tests/shim.h +++ b/drivers/media/platform/apple/ave/tests/shim.h @@ -18,6 +18,7 @@ typedef uint64_t dma_addr_t; #define AVE_BUF_CMD 0 #define AVE_BUF_PROC(s) (17 + (s)) #define SZ_16K 0x4000U +#define SZ_1M 0x100000U #define U32_MAX UINT32_MAX #define GENMASK_ULL(h, l) ((~0ULL >> (63 - (h))) & (~0ULL << (l))) #define BIT(n) (1U << (n)) @@ -52,7 +53,6 @@ static inline void test_log(void *dev, const char *fmt, ...) #define atomic_inc(p) (++*(p)) static unsigned long jiffies = 42; -#ifndef TEST_BASELINE static inline void put_unaligned_le16(u16 v, void *p) { u8 *b = p; @@ -60,7 +60,6 @@ static inline void put_unaligned_le16(u16 v, void *p) for (unsigned int i = 0; i < 2; i++) b[i] = v >> (8 * i); } -#endif static inline void put_unaligned_le32(u32 v, void *p) { u8 *b = p; @@ -77,6 +76,13 @@ static inline void put_unaligned_le64(u64 v, void *p) b[i] = v >> (8 * i); } +static inline u16 get_unaligned_le16(const void *p) +{ + const u8 *b = p; + + return (u16)b[0] | (u16)b[1] << 8; +} + static inline u32 get_unaligned_le32(const void *p) { const u8 *b = p; diff --git a/drivers/media/platform/apple/ave/tests/test.c b/drivers/media/platform/apple/ave/tests/test.c index 486b7dc60d225..55df0584c876b 100644 --- a/drivers/media/platform/apple/ave/tests/test.c +++ b/drivers/media/platform/apple/ave/tests/test.c @@ -65,9 +65,9 @@ static void facts(void) assert(!ave_layout_for_crc(0xffffffff)); assert(!ave_layout_for_crc(0x2ce7359e)); assert(ave_layout_for_crc(0x69c54454) == &ave_layout_9003_78_0); - assert(ave_fw_check_build(&dev) == -ENOTSUPP); + assert(ave_fw_check_build(&dev) == -EOPNOTSUPP); dev.text_crc = 0x69c54454; - assert(ave_fw_check_build(&dev) == -ENOTSUPP); + assert(ave_fw_check_build(&dev) == -EOPNOTSUPP); dev.layout = ave_layout_for_crc(dev.text_crc); assert(ave_fw_check_build(&dev) == 0); struct ave_variant probe_only = test_variant; @@ -75,7 +75,7 @@ static void facts(void) probe_only.boot_qualified = false; probe_only.regs = NULL; dev.variant = &probe_only; - assert(ave_fw_check_build(&dev) == -ENOTSUPP); + assert(ave_fw_check_build(&dev) == -EOPNOTSUPP); dev.variant = &test_variant; dev.layout = &ave_layout_26a428; @@ -148,7 +148,12 @@ static void facts(void) dev.sessions[0] = &ctx; put_unaligned_le64(session.client, reply + 0x10); put_unaligned_le32(0x17, reply + 0x20); - ave_sess_reply(&dev, reply, sizeof(reply)); + struct ave_reply decoded; + + assert(dev.layout->abi->parse_reply(dev.layout->abi, reply, + sizeof(reply), + &decoded) == 0); + ave_sess_reply(&dev, &decoded); assert(!session.slot[4].busy && session.slot[4].done); assert(ctx.ready == (int)codec); dev.sessions[0] = NULL; @@ -236,7 +241,7 @@ int main(int argc, char **argv) { struct ave_dev dev = { .variant = &test_variant, -#ifndef TEST_BASELINE +#ifdef TEST_LAYOUT .layout = &ave_layout_9003_78_0, #endif }; @@ -326,8 +331,10 @@ int main(int argc, char **argv) session.p.coded_cap = session.p.coded_cap * 5 / 4; if (session.p.rc_mode == 3) { session.p.max_qp = 48; - session.p.coded_cap = session.p.initial_qp == 40 ? 0x3a4000 : - session.p.initial_qp == 28 ? 0x458000 : 0x5fc000; + session.p.coded_cap = + session.p.initial_qp == 40 ? 0x3a4000 : + session.p.initial_qp == 28 ? 0x458000 : + 0x5fc000; } #endif start(&ctx); From 3720aabf0cc682fe2d805b6f1f1e3d95de10f214 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 02:37:38 +1100 Subject: [PATCH 1535/1579] media: apple: ave: scope the host reply helper to ABI tests The pre-refactor baseline does not parse command replies through the ABI interface. Omit its unused 16-bit getter in that build so the sanitizer comparison remains clean with -Werror. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/tests/shim.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/media/platform/apple/ave/tests/shim.h b/drivers/media/platform/apple/ave/tests/shim.h index e801d6378f1be..28763dfeb660f 100644 --- a/drivers/media/platform/apple/ave/tests/shim.h +++ b/drivers/media/platform/apple/ave/tests/shim.h @@ -76,6 +76,7 @@ static inline void put_unaligned_le64(u64 v, void *p) b[i] = v >> (8 * i); } +#ifdef TEST_PER_ABI static inline u16 get_unaligned_le16(const void *p) { const u8 *b = p; @@ -83,6 +84,7 @@ static inline u16 get_unaligned_le16(const void *p) return (u16)b[0] | (u16)b[1] << 8; } +#endif static inline u32 get_unaligned_le32(const void *p) { const u8 *b = p; From 2cb8fd897879814d97d634eeaa1f2b8ec255c25b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:06:18 +1100 Subject: [PATCH 1536/1579] dt-bindings: sound: apple: describe T6041 DisplayPort audio Admit the generation-specific DPA compatible while retaining the SIO DMA, power-domain and bidirectional display graph requirements. Document that T6041 clients stay disabled pending firmware and loader support. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/sound/apple,t6030-dpaudio.yaml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml b/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml index 9b4e9328d9020..b511202f35c7e 100644 --- a/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml @@ -4,29 +4,36 @@ $id: http://devicetree.org/schemas/sound/apple,t6030-dpaudio.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple T6030 DisplayPort audio +title: Apple T6030 and T6041 DisplayPort audio maintainers: - Ryan Murray description: | - The external display processor provides the HDMI port's audio service. + The external display processor provides a DisplayPort/HDMI audio service. Its DisplayPort audio interface has no application-processor registers. Samples arrive through SIO, and the graph connects this interface to port 0 of the display processor that owns the audio service. + T6041 interfaces remain disabled until a generation-specific SIO firmware + backend and loader handoff are available and display audio is qualified. + select: properties: compatible: contains: - const: apple,t6030-dpaudio + enum: + - apple,t6030-dpaudio + - apple,t6041-dpaudio required: - compatible properties: compatible: items: - - const: apple,t6030-dpaudio + - enum: + - apple,t6030-dpaudio + - apple,t6041-dpaudio - const: apple,dpaudio dmas: From bfcdfaaf66e2f5c55d14ee3275a00426ca76fab3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:06:18 +1100 Subject: [PATCH 1537/1579] drm: apple: draft compact desktop audio service integration Separate the desktop audio service on endpoint 0x2b from AV device control on 0x29. Reuse the native external PCM lifecycle and service- generation ownership, validate compact reply bounds, and pin metadata RPC owners outside receive-side publication locks. Exercise replacement, Terminate and rejected link prepares for both native firmware roles, with a synthetic host test for compact reply lengths. Document the provisional desktop protocol assumptions; no audio client is enabled by this change. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/gpu/apple-dcp-t6041-panel.rst | 29 ++++ drivers/gpu/drm/apple/Kconfig | 9 +- drivers/gpu/drm/apple/audio.c | 4 +- drivers/gpu/drm/apple/audio.h | 2 +- drivers/gpu/drm/apple/av-v14-test.c | 96 ++++++++++-- drivers/gpu/drm/apple/av.c | 142 ++++++++++++++---- drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp.c | 19 +-- .../selftests/drivers/gpu/apple-dcp/Makefile | 2 +- .../selftests/drivers/gpu/apple-dcp/README | 7 + .../selftests/drivers/gpu/apple-dcp/audio.py | 111 ++++++++++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 4 +- 12 files changed, 370 insertions(+), 57 deletions(-) create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/audio.py diff --git a/Documentation/gpu/apple-dcp-t6041-panel.rst b/Documentation/gpu/apple-dcp-t6041-panel.rst index fa45c33f678ad..b75e6584bc1d2 100644 --- a/Documentation/gpu/apple-dcp-t6041-panel.rst +++ b/Documentation/gpu/apple-dcp-t6041-panel.rst @@ -183,3 +183,32 @@ A failed transaction must publish no partial handoff. Loader host tests should cover supported DT without the chosen property, an unsupported board/DT, and each failed validation, as well as unchanged other-SoC policy. No new policy flag or kernel bypass is needed. This kernel series does not replace Stage 2. + + +Desktop external audio (draft) +------------------------------ + +The desktop audio integration uses the existing DCP PCM and native link +lifecycle, with a separate compact service binding on endpoint 0x2b. Endpoint +0x29 retains AV device control. Only the external firmware role admits the +``dcpav-audio-epic`` unit-zero service; the internal panel does not acquire an +audio client. The existing ``hdmi_audio`` opt-in still controls service startup. + +A PCM opens and pins one service instance. Prepare, Start, Stop and Complete +use that instance's element link data. Sink termination disconnects the card, +drains the DMA and completes its link before closing that instance. Replacement +services cannot inherit an old instance's Close or link completion. Compact +method replies must have the expected body length; serialized property replies +must contain their declared used bytes within the actual received body. + +J616s DPA1 and DPA2 are described with SIO channels 0x66 and 0x68, connected +bidirectionally to DCPEXT0 and DCPEXT1 port zero, respectively. Both clients +remain disabled. This description does not enable SIO, its DART, its mailbox +or any DPA power domain. A generation-specific SIO firmware backend and loader +handoff are still required before a candidate may enable the clients. + +The desktop service name/unit, shared 13.5 command numbers and native 14.7 link +and element layouts are provisional integration choices. Build and synthetic +regression results cannot qualify them against firmware. Qualification must +confirm those contracts and actual sink element catalogs before enabling these +outputs. No sink-specific rates or capabilities are synthesized here. diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 06415026853f9..7575fb03695ec 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -32,16 +32,17 @@ config DRM_APPLE_AUDIO select SND_DMAENGINE_PCM config DRM_APPLE_DCP_T6030_AUDIO - bool "HDMI audio through the T6030 external display processor" + bool "Native audio links through external display processors" default y depends on DRM_APPLE_AUDIO help DP/HDMI audio of the external display processors of the M3 Pro (T6030, MacBook Pro J514S/J516S built-in HDMI port), whose DCP firmware 14.7 external role takes 32-byte audio link data and the - native driver's link lifecycle. The audio service of every other - DCP is driven unchanged. Say N to build the audio and element - parser code exactly as before. + native driver's link lifecycle. Also build the compact audio service + binding for desktop 26.x external processors (T6041). Those audio + interfaces remain disabled until SIO firmware support is available. + Say N to omit native links and the desktop compact audio binding. config DRM_APPLE_DCP_T8132 bool "MacBook Air M4 (J713) internal panel" diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 2b507dc1fa1b0..ccb10dcf3768c 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -84,7 +84,7 @@ struct dcp_audio { #endif }; -/* The 14.7 external role's audio service (DRM_APPLE_DCP_T6030_AUDIO only). */ +/* Native external audio service (DRM_APPLE_DCP_T6030_AUDIO only). */ static inline bool dcpaud_native_link(struct dcp_audio *dcpaud) { #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) @@ -1057,7 +1057,7 @@ static int dcpaud_comp_bind(struct device *dev, struct device *main, void *data) dcpaud->dcp_dev = &dcp_pdev->dev; #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) - /* The 14.7 external role: its native links and element catalog. */ + /* Native external roles share the link lifecycle and element catalog. */ if (dcp_audiosrv_native_link(dcpaud->dcp_dev)) { void *elements = devm_kzalloc(dev, DCPAUD_ELEMENTS_V14_7, GFP_KERNEL); diff --git a/drivers/gpu/drm/apple/audio.h b/drivers/gpu/drm/apple/audio.h index 58633598253ea..88e9610363234 100644 --- a/drivers/gpu/drm/apple/audio.h +++ b/drivers/gpu/drm/apple/audio.h @@ -18,7 +18,7 @@ void dcpaud_connect(struct platform_device *pdev, bool connected); void dcpaud_disconnect(struct platform_device *pdev); #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) -/* The native link methods of the DCP firmware 14.7 external role. */ +/* Native link methods for 14.7 and desktop 26.x external roles. */ enum dcp_audio_link { DCP_AUDIO_LINK_PREPARE, DCP_AUDIO_LINK_START, diff --git a/drivers/gpu/drm/apple/av-v14-test.c b/drivers/gpu/drm/apple/av-v14-test.c index 8d78113fb8f17..362bb77db6465 100644 --- a/drivers/gpu/drm/apple/av-v14-test.c +++ b/drivers/gpu/drm/apple/av-v14-test.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * Included by av.c: which instance the 14.7 external role's audio client + * Included by av.c: which instance a native external role's audio client * opens and closes. One reusable service slot is published, torn down and * published again by the production init and teardown; the production open, * close, Terminate report and Terminate work run on it, with only the @@ -27,6 +27,8 @@ struct av_v14_test { u64 release_gen; /* Within a link release: the old PCM closes, a successor is published and opened */ bool successor_during_release; + u32 link_status; + u8 link_body[64]; }; static struct av_v14_test *av_v14_current; @@ -45,6 +47,10 @@ static int av_v14_call(struct apple_epic_service *srv, u32 command, const void * /* Open: status 0 at +0, opened at +16 */ if (command == t->asrv->cmds.open && reply_len >= 24) put_unaligned_le64(1, reply + 16); + if (body_len == sizeof(t->link_body)) { + memcpy(t->link_body, body, body_len); + put_unaligned_le32(t->link_status, reply + 48); + } return 0; } @@ -114,7 +120,8 @@ static struct av_v14_test *av_v14_setup(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); dev_set_drvdata(dev, t->dcp); t->dcp->dev = dev; - t->dcp->fw_compat = DCP_FIRMWARE_V_14_7; + t->dcp->fw_compat = test->param_value && *(const bool *)test->param_value ? + DCP_FIRMWARE_V_26_6_DESKTOP : DCP_FIRMWARE_V_14_7; t->dcp->external = true; t->dcp->external_native = true; @@ -130,6 +137,8 @@ static struct av_v14_test *av_v14_setup(struct kunit *test) t->dcp->audiosrv = asrv; t->ep.dcp = t->dcp; + t->ep.endpoint = AV_AUDIO_ENDPOINT_DESKTOP; + t->ep.compact = t->dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP; spin_lock_init(&t->svc.lock); t->svc.ep = &t->ep; spin_lock_init(&t->other.lock); @@ -324,21 +333,84 @@ static void av_v14_terminate_unpublished(struct kunit *test) KUNIT_EXPECT_EQ(test, t->ncalls, 2U); } +/* Rejected prepares must not create a link which Stop/Complete can act on. */ +static void av_native_link_payload_status(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + struct dcp_sound_cookie cookie; + + memset(&cookie, 0xa5, sizeof(cookie)); + KUNIT_ASSERT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + t->link_status = 1; + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_PREPARE, + &cookie), -EIO); + KUNIT_EXPECT_FALSE(test, dcp_audiosrv_link_on(t->dcp->dev, t->asrv->gen)); + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_START, + &cookie), -ENODEV); + t->link_status = 0; + for (enum dcp_audio_link op = DCP_AUDIO_LINK_PREPARE; + op <= DCP_AUDIO_LINK_COMPLETE; op++) { + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, op, &cookie), 0); + KUNIT_EXPECT_MEMEQ(test, t->link_body, &cookie, sizeof(cookie)); + for (size_t i = sizeof(cookie); i < sizeof(t->link_body); i++) + KUNIT_EXPECT_EQ(test, t->link_body[i], 0); + } + KUNIT_EXPECT_EQ(test, t->calls[2], t->asrv->cmds.prepare); + KUNIT_EXPECT_EQ(test, t->calls[3], t->asrv->cmds.start_link); + KUNIT_EXPECT_EQ(test, t->calls[4], t->asrv->cmds.stop_link); + KUNIT_EXPECT_EQ(test, t->calls[5], t->asrv->cmds.unprepare); + dcp_audiosrv_close(t->dcp->dev); +} + +static void av_desktop_service_admission(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + const char *name = "dcpav-audio-epic"; + bool desktop = t->dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP; + + KUNIT_EXPECT_TRUE(test, dcp_audiosrv_native_link(t->dcp->dev)); + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, name, name, 0), + desktop ? 0 : -EINVAL); + t->dcp->fw_compat = DCP_FIRMWARE_V_26_6_DESKTOP; + t->ep.endpoint = AV_ENDPOINT; + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, name, name, 0), -EINVAL); + t->ep.endpoint = AV_AUDIO_ENDPOINT_DESKTOP; + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, "dcpav-device-epic", name, 0), + -EINVAL); + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, NULL, name, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, name, name, 1), -EINVAL); + t->dcp->external = false; + KUNIT_EXPECT_FALSE(test, dcp_audiosrv_native_link(t->dcp->dev)); + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, name, name, 0), -EINVAL); + dcp_audiosrv_close(t->dcp->dev); +} + +static const bool av_native_desktop[] = { false, true }; + +static void av_native_desc(const bool *desktop, char *desc) +{ + scnprintf(desc, KUNIT_PARAM_DESC_SIZE, "%s", *desktop ? "desktop-26" : "external-14.7"); +} + +KUNIT_ARRAY_PARAM(av_native, av_native_desktop, av_native_desc); + static struct kunit_case av_v14_cases[] = { - KUNIT_CASE(av_v14_open_close), - KUNIT_CASE(av_v14_successor_opened), - KUNIT_CASE(av_v14_old_close_not_to_successor), - KUNIT_CASE(av_v14_old_terminate_vs_successor), - KUNIT_CASE(av_v14_republished), - KUNIT_CASE(av_v14_successor_during_release), - KUNIT_CASE(av_v14_terminate_after_successor), - KUNIT_CASE(av_v14_link_instance), - KUNIT_CASE(av_v14_terminate_unpublished), + KUNIT_CASE_PARAM(av_v14_open_close, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_successor_opened, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_old_close_not_to_successor, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_old_terminate_vs_successor, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_republished, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_successor_during_release, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_terminate_after_successor, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_link_instance, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_terminate_unpublished, av_native_gen_params), + KUNIT_CASE_PARAM(av_native_link_payload_status, av_native_gen_params), + KUNIT_CASE_PARAM(av_desktop_service_admission, av_native_gen_params), {} }; static struct kunit_suite av_v14_suite = { - .name = "apple-av-v14-audio", + .name = "apple-av-native-audio", .test_cases = av_v14_cases, }; diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index d439ce69368b0..afb918fb58f89 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -254,10 +254,13 @@ int dcp_audiosrv_unprepare(struct device *dev) } #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) -/* The audio service of the 14.7 external role, known before it starts. */ +/* Native audio links are selected from the firmware role, before service start. */ bool dcp_audiosrv_native_link(struct device *dev) { - return dcp_is_v14_7_external(dev_get_drvdata(dev)); + struct apple_dcp *dcp = dev_get_drvdata(dev); + + return dcp_is_v14_7_external(dcp) || + (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && dcp->external); } /* The 14.7 external role's audio service calls; KUnit stubs them. */ @@ -266,6 +269,14 @@ static int av_audiosrv_call(struct apple_epic_service *srv, u32 command, const v { KUNIT_STATIC_STUB_REDIRECT(av_audiosrv_call, srv, command, body, body_len, body_pad, reply, reply_len); + if (READ_ONCE(srv->ep->compact)) { + size_t received; + int ret; + + ret = afk_service_call_with_reply_len(srv, 0, command, body, body_len, + body_pad, reply, reply_len, 0, &received); + return ret ?: (received == reply_len ? 0 : -EMSGSIZE); + } return afk_service_call(srv, 0, command, body, body_len, body_pad, reply, reply_len, 0); } @@ -395,6 +406,10 @@ int dcp_audiosrv_link(struct device *dev, enum dcp_audio_link op, KUNIT_STATIC_STUB_REDIRECT(dcp_audiosrv_link, dev, op, cookie); dcp = dev_get_drvdata(dev); asrv = dcp->audiosrv; + if (!asrv) + return -ENODEV; + if (!cookie) + return -EINVAL; switch (op) { case DCP_AUDIO_LINK_PREPARE: @@ -423,16 +438,16 @@ int dcp_audiosrv_link(struct device *dev, enum dcp_audio_link op, return -ENODEV; ret = av_audiosrv_call(asrv->open_srv, command, body, sizeof(body), 0, reply, sizeof(reply)); - if (!ret && op == DCP_AUDIO_LINK_PREPARE) + if (ret) + return ret; + status = get_unaligned_le32(reply + 48); + if (status) { + dev_err(dev, "audiosrv: link method %u answered %#x\n", command, status); + return -EIO; + } + if (op == DCP_AUDIO_LINK_PREPARE) asrv->link_gen = asrv->open_gen; } - if (ret) - return ret; - status = get_unaligned_le32(reply + 48); - if (status) { - dev_err(dev, "audiosrv: link method %u answered %#x\n", command, status); - return -EIO; - } return 0; } #endif @@ -449,6 +464,7 @@ dcp_audiosrv_osobject_call(struct apple_epic_service *service, u16 group, u8 _pad2[8]; } __attribute__((packed)) *hdr; static_assert(sizeof(*hdr) == 48); + size_t received = 0, used; size_t bfr_len = output_maxsize + sizeof(*hdr); void *bfr __free(kfree) = NULL; int ret; @@ -459,16 +475,30 @@ dcp_audiosrv_osobject_call(struct apple_epic_service *service, u16 group, hdr = bfr; hdr->max_size = cpu_to_le64(output_maxsize); - ret = afk_service_call(service, group, command, hdr, sizeof(*hdr), output_maxsize, - bfr, sizeof(*hdr) + output_maxsize, 0); + if (READ_ONCE(service->ep->compact)) + ret = afk_service_call_with_reply_len(service, group, command, hdr, + sizeof(*hdr), output_maxsize, bfr, + bfr_len, 0, &received); + else + ret = afk_service_call(service, group, command, hdr, sizeof(*hdr), + output_maxsize, bfr, bfr_len, 0); if (ret) return ret; - if (output) + used = le64_to_cpu(hdr->used_size); + if (READ_ONCE(service->ep->compact)) { + if (used > output_maxsize || received < sizeof(*hdr) || + used > received - sizeof(*hdr)) + return -EMSGSIZE; + if (output) { + memset(output, 0, output_maxsize); + memcpy(output, bfr + sizeof(*hdr), used); + } + } else if (output) { memcpy(output, bfr + sizeof(*hdr), output_maxsize); - + } if (output_size) - *output_size = le64_to_cpu(hdr->used_size); + *output_size = used; return 0; } @@ -477,13 +507,19 @@ int dcp_audiosrv_get_elements(struct device *dev, void *elements, size_t maxsize { struct apple_dcp *dcp = dev_get_drvdata(dev); struct audiosrv_data *asrv = dcp->audiosrv; + struct apple_epic_service *srv; size_t size; int ret; - down_write(&asrv->srv_rwsem); - ret = dcp_audiosrv_osobject_call(asrv->srv, 1, asrv->cmds.get_elements, + if (!asrv) + return -ENODEV; + scoped_guard(rwsem_read, &asrv->srv_rwsem) + srv = afk_service_get(asrv->srv); + if (!srv) + return -ENODEV; + ret = dcp_audiosrv_osobject_call(srv, 1, asrv->cmds.get_elements, elements, maxsize, &size); - up_write(&asrv->srv_rwsem); + afk_service_put(srv); if (ret && !asrv->warned_get_elements) { dev_err(dev, "audiosrv: error getting elements: %d\n", ret); @@ -499,14 +535,20 @@ int dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsi { struct apple_dcp *dcp = dev_get_drvdata(dev); struct audiosrv_data *asrv = dcp->audiosrv; + struct apple_epic_service *srv; size_t size; int ret; - down_write(&asrv->srv_rwsem); - ret = dcp_audiosrv_osobject_call(asrv->srv, 1, + if (!asrv) + return -ENODEV; + scoped_guard(rwsem_read, &asrv->srv_rwsem) + srv = afk_service_get(asrv->srv); + if (!srv) + return -ENODEV; + ret = dcp_audiosrv_osobject_call(srv, 1, asrv->cmds.get_product_attrs, attrs, maxsize, &size); - up_write(&asrv->srv_rwsem); + afk_service_put(srv); if (ret && !asrv->warned_get_product_attrs) { dev_err(dev, "audiosrv: error getting product attributes: %d\n", ret); @@ -542,7 +584,7 @@ static int av_audiosrv_report(struct apple_epic_service *service, u32 idx, * 10 s for that before it removes the interface, and the sink's link * with it. A later publication brings the new instance. */ - if (dcp_is_v14_7_external(dcp) && av_report_terminate(data, data_size)) { + if (dcp_audiosrv_native_link(dcp->dev) && av_report_terminate(data, data_size)) { struct audiosrv_data *asrv = dcp->audiosrv; bool published; @@ -590,6 +632,32 @@ static void av_cec_teardown(struct apple_epic_service *service) } #endif +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) +static int av_desktop_audio_prepare(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + + if (dcp->fw_compat != DCP_FIRMWARE_V_26_6_DESKTOP || !dcp->external || + service->ep->endpoint != AV_AUDIO_ENDPOINT_DESKTOP || + !name || strcmp(name, "dcpav-audio-epic") || unit != 0) + return -EINVAL; + return 0; +} + +static const struct apple_epic_service_ops av_desktop_ops[] = { + { + .name = "dcpav-audio-epic", + .reusable = true, + .prepare = av_desktop_audio_prepare, + .init = av_audiosrv_init, + .report = av_audiosrv_report, + .teardown = av_audiosrv_teardown, + }, + {} +}; +#endif + static const struct apple_epic_service_ops avep_ops[] = { { .name = "DCPAVSimpleVideoInterface", @@ -641,8 +709,8 @@ void av_service_connect(struct apple_dcp *dcp) * the display's power (A472 on): the CRTC enable connects before the * mode is set, and the modeset connects again once the display is on. */ - if (dcp_is_v14_7_external(dcp)) { - if (!iomfb_v14_7_external_powered(dcp)) + if (dcp_audiosrv_native_link(dcp->dev)) { + if (dcp_is_v14_7_external(dcp) && !iomfb_v14_7_external_powered(dcp)) goto out; scoped_guard(mutex, &asrv->plug_lock) { if (asrv->audio_dev) @@ -689,7 +757,7 @@ static void __av_service_disconnect(struct apple_dcp *dcp, const u64 *gen) dev_info(dcp->dev, "%s: stopping audio service\n", __func__); #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) /* The 14.7 external role's card goes with the display, and an open PCM's service. */ - if (dcp_is_v14_7_external(dcp)) { + if (dcp_audiosrv_native_link(dcp->dev)) { struct platform_device *audio_dev = NULL; bool published = true; @@ -866,6 +934,10 @@ static void av_audiosrv_cleanup(void *data) av_audiosrv_drain_owner(asrv); } afk_quiesce(cleanup->ep); + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && + !IS_ERR_OR_NULL(cleanup->ep) && + READ_ONCE(dcp->desktop->graph[cleanup->ep->endpoint - 0x20]) == cleanup->ep) + WRITE_ONCE(dcp->desktop->graph[cleanup->ep->endpoint - 0x20], NULL); /* An old devres group must not unpublish a later session. */ if (dcp->audiosrv == asrv) { if (dcp->avep == cleanup->ep) @@ -923,8 +995,21 @@ int avep_init(struct apple_dcp *dcp) struct platform_device *audio_pdev; struct device *dev = dcp->dev; struct device_node *endpoint, *audio_node = NULL; + const struct apple_epic_service_ops *ops = avep_ops; + u32 av_endpoint = AV_ENDPOINT; int ret; + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) { +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + if (!dcp->external) + return -EOPNOTSUPP; + ops = av_desktop_ops; + av_endpoint = AV_AUDIO_ENDPOINT_DESKTOP; +#else + return -EOPNOTSUPP; +#endif + } + audiosrv_data = devm_kzalloc(dcp->dev, sizeof(*audiosrv_data), GFP_KERNEL); if (!audiosrv_data) return -ENOMEM; @@ -986,7 +1071,7 @@ int avep_init(struct apple_dcp *dcp) device_link_add(&audio_pdev->dev, dev, DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); - dcp->avep = afk_init(dcp, AV_ENDPOINT, avep_ops); + dcp->avep = afk_init(dcp, av_endpoint, ops); if (IS_ERR(dcp->avep)) { ret = PTR_ERR(dcp->avep); if (!av_audiosrv_register_cleanup(dcp, audiosrv_data, dcp->avep)) @@ -997,7 +1082,10 @@ int avep_init(struct apple_dcp *dcp) ret = av_audiosrv_register_cleanup(dcp, audiosrv_data, dcp->avep); if (ret) return ret; - dcp->avep->debugfs_entry = dcp->ep_debugfs[AV_ENDPOINT - 0x20]; + dcp->avep->debugfs_entry = dcp->ep_debugfs[av_endpoint - 0x20]; + /* afk_start waits for RX: publish the desktop route before starting. */ + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) + WRITE_ONCE(dcp->desktop->graph[av_endpoint - 0x20], dcp->avep); ret = afk_start(dcp->avep); if (ret) av_audiosrv_cleanup(&audiosrv_data->cleanup); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 1c08764d767e7..32b39de3c7fb4 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -97,6 +97,8 @@ enum { DPAVSERV_ENDPOINT = 0x28, DPAVSERV_ENDPOINT_V2 = 0x2a, AV_ENDPOINT = 0x29, + /* Desktop 26.x separates the audio service from AV device control. */ + AV_AUDIO_ENDPOINT_DESKTOP = 0x2b, DPTX_ENDPOINT = 0x2a, DPTX_ENDPOINT_V2 = 0x2c, HDCP_ENDPOINT = 0x2b, diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 01485347116bf..7e4f83b7240b1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3444,14 +3444,19 @@ static int dcp_start_desktop_external(struct apple_dcp *dcp, u32 map) ret = dcp_desktop_start_ext_ep(dcp, map, DPSAC_ENDPOINT, "DPSAC"); if (ret) return ret; + ret = dcp_desktop_start_ext_ep(dcp, map, AV_ENDPOINT, "AV"); + if (ret) + return ret; #if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) - if (hdmi_audio) + if (hdmi_audio && (map & BIT(AV_AUDIO_ENDPOINT_DESKTOP - 0x20))) { ret = avep_init(dcp); - else + if (ret) + dcp_desktop_graph_skip(dcp, AV_AUDIO_ENDPOINT_DESKTOP, + "AUDIO", "audio init failed", ret); + else if (dcp->avep) + d->graph_started |= BIT(AV_AUDIO_ENDPOINT_DESKTOP - 0x20); + } #endif - ret = dcp_desktop_start_ext_ep(dcp, map, AV_ENDPOINT, "AV"); - if (ret) - return ret; ret = dcp_desktop_start_ext_ep(dcp, map, REMOTE_ALLOC_ENDPOINT, "REMOTEALLOC"); if (ret) @@ -3549,10 +3554,6 @@ static int dcp_start_desktop(struct apple_dcp *dcp) if (endpoint == DPAV_CTRL_ENDPOINT) ret = dcp_desktop_start_dpavctrl(dcp); -#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) - else if (endpoint == AV_ENDPOINT && hdmi_audio) - ret = avep_init(dcp); -#endif else ret = dcp_desktop_start_ep(dcp, endpoint, dcp_compact_aux_ops); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile index 2c25d757aca5e..d46a4dc8c05d2 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT TEST_PROGS := run.sh m4-error-paths.py plane-layout.py -TEST_FILES := run.py common.py desktop.py display.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py boot.py quiesce.py transaction.py accounting.py m4.py m4-fixtures native-m4-kms.c run-native-m4-kms.py dwc3 README +TEST_FILES := run.py common.py audio.py desktop.py display.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py boot.py quiesce.py transaction.py accounting.py m4.py m4-fixtures native-m4-kms.c run-native-m4-kms.py dwc3 README include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index 7d65d5f848e34..4934b7eb7eef2 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -208,3 +208,10 @@ ensure the machine's serial cannot override the panel's. Panel EDID tests cover a missing firmware identity, real EDID priority and invalid replacement retention. Desktop backlight notification tests exercise accepted, rejected, duplicate and out-of-order completions and preserve the requested target during notification. + +The audio group executes the native AV call and OSObject reply boundaries with +synthetic transport replies, including truncated bodies, inconsistent used +sizes and transport errors. It retains the legacy reply behavior. KUnit's +apple-av-native-audio suite separately runs service replacement, Terminate and +link lifecycle checks for both external 14.7 and desktop 26.x firmware roles. +These fixtures do not establish the desktop audio firmware ABI or SIO support. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/audio.py b/tools/testing/selftests/drivers/gpu/apple-dcp/audio.py new file mode 100644 index 0000000000000..fc1a5503f8a85 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/audio.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright 2026 Ryan Murray +"""Compact audio reply bounds through production AV methods, synthetic transport.""" +from common import block + + +def reply_bounds(root): + source = (root / 'drivers/gpu/drm/apple/av.c').read_text() + prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint16_t u16; +typedef uint64_t __le64; +#define KUNIT_STATIC_STUB_REDIRECT(...) ((void)0) +#define READ_ONCE(x) (x) +#define static_assert _Static_assert +#define GFP_KERNEL 0 +static void free_buffer(void *p) {free(*(void **)p);} +#define __free(x) __attribute__((cleanup(free_buffer))) +static void *kzalloc(size_t n, int flags) {(void)flags;return calloc(1,n);} +#define cpu_to_le64(x) (x) +#define le64_to_cpu(x) (x) +struct apple_dcp_afkep {bool compact;}; +struct apple_epic_service {struct apple_dcp_afkep *ep;}; +static int transport_error; +static size_t wire_received, used; +static unsigned compact_calls, legacy_calls; +static int transfer(struct apple_epic_service *srv, u16 group, u32 cmd, + const void *data, size_t len, size_t pad, void *out, + size_t room, size_t out_pad) +{ + assert(srv && cmd && !out_pad); + (void)pad; + if (transport_error) return transport_error; + if (group==1) { + assert(len==48 && room==64 && data==out); + __le64 max, count=used; + memcpy(&max,data,8);assert(max==16); + memcpy((u8 *)out+32,&count,8); + memset((u8 *)out+48,0x5a,16); + } else { + assert(group==0 && len==64 && data && room==64); + memset(out,0,room); + } + return 0; +} +static int afk_service_call_with_reply_len(struct apple_epic_service *srv, + u16 group,u32 cmd,const void *data,size_t len,size_t pad,void *out, + size_t room,size_t out_pad,size_t *received) +{ + compact_calls++;*received=wire_received; + return transfer(srv,group,cmd,data,len,pad,out,room,out_pad); +} +static int afk_service_call(struct apple_epic_service *srv,u16 group,u32 cmd, + const void *data,size_t len,size_t pad,void *out,size_t room,size_t out_pad) +{ + legacy_calls++; + return transfer(srv,group,cmd,data,len,pad,out,room,out_pad); +} +''' + return prefix + block(source, 'static int av_audiosrv_call(') + '\n' + block( + source, 'static int\ndcp_audiosrv_osobject_call(') + r''' +int main(void) +{ + struct apple_dcp_afkep ep={.compact=true}; + struct apple_epic_service srv={.ep=&ep}; + u8 body[64]={0},reply[64],elements[16]; + size_t actual=0; + wire_received=64; + assert(!av_audiosrv_call(&srv,6,body,64,0,reply,64)); + for (size_t n=0;n<66;n++) { + wire_received=n; + assert(av_audiosrv_call(&srv,6,body,64,0,reply,64)== + (n==64?0:-EMSGSIZE)); + } + transport_error=-EIO; + assert(av_audiosrv_call(&srv,6,body,64,0,reply,64)==-EIO); + transport_error=0; + used=8;wire_received=56; + memset(elements,0xff,sizeof(elements)); + assert(!dcp_audiosrv_osobject_call(&srv,1,16,elements,16,&actual)); + assert(actual==8); + for (size_t i=0;i<16;i++) assert(elements[i]==(i<8?0x5a:0)); + for (size_t n=0;n<56;n++) { + wire_received=n; + assert(dcp_audiosrv_osobject_call(&srv,1,16,elements,16,&actual)==-EMSGSIZE); + } + wire_received=64;used=17; + assert(dcp_audiosrv_osobject_call(&srv,1,16,elements,16,&actual)==-EMSGSIZE); + used=16; + assert(!dcp_audiosrv_osobject_call(&srv,1,16,elements,16,&actual) && actual==16); + assert(!dcp_audiosrv_osobject_call(&srv,1,18,NULL,16,NULL)); + transport_error=-ETIMEDOUT; + assert(dcp_audiosrv_osobject_call(&srv,1,16,elements,16,&actual)==-ETIMEDOUT); + transport_error=0; + ep.compact=false; + unsigned compact_before=compact_calls; + assert(!av_audiosrv_call(&srv,6,body,64,0,reply,64)); + used=0; + assert(!dcp_audiosrv_osobject_call(&srv,1,16,elements,16,&actual) && actual==0); + for (size_t i=0;i<16;i++) assert(elements[i]==0x5a); + assert(compact_calls==compact_before && legacy_calls==2); + return 0; +} +''' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index f1623a4ea8f64..a2a8332a30269 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -27,6 +27,7 @@ import accounting import idle import desktop +import audio import j616s import rate import boot @@ -44,7 +45,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--source', type=Path, default=Path(__file__).resolve().parents[6]) parser.add_argument('--output', type=Path, help='retain generated C, commands and results here') - parser.add_argument('--group', choices=('all', 'edid', 'rate', 'phy', 'lifecycle', 'boot', 'quiesce', 'm4', 'display'), default='all') + parser.add_argument('--group', choices=('all', 'edid', 'rate', 'phy', 'lifecycle', 'boot', 'quiesce', 'm4', 'display', 'audio'), default='all') args = parser.parse_args() root = args.source.resolve() output = args.output or Path(tempfile.mkdtemp(prefix='apple-dcp-selftest-')) @@ -56,6 +57,7 @@ def main(): 'failed-flip-return', 'live-park')) tests = [ ('display', 'panel-colorimetry', lambda r: colorimetry.panel(r, output), ((),), ('-Wno-unused-function', '-Wno-unused-const-variable')), + ('audio', 'compact-audio-reply-bounds', audio.reply_bounds, ((),), ()), ('display', 'desktop-gamma', desktop.gamma, ((),), ()), ('lifecycle', 'desktop-modeset-unmap', desktop.modeset_unmap, ((),), ()), ('lifecycle', 'desktop-unknown-modeset-recovery', desktop.modeset_unknown, ((),), ()), From e2d66ece9aa51714f41b12ae8203df71a7cab87d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:06:18 +1100 Subject: [PATCH 1538/1579] arm64: dts: apple: describe disabled J616s external DPA graphs Connect DPA1 and DPA2 to DCPEXT0 and DCPEXT1 port zero using SIO channels 0x66 and 0x68 and the corresponding existing power domains. Keep both clients disabled and leave the SIO provider chain untouched until its T6041 backend and loader handoff are available. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../boot/dts/apple/t6040-j616s-dpaudio.dtsi | 77 +++++++++++++++++++ arch/arm64/boot/dts/apple/t6040-j616s.dts | 1 + 2 files changed, 78 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi diff --git a/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi b/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi new file mode 100644 index 0000000000000..8acd2d04aea8d --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * DPA1 and DPA2 belong to DCPEXT0 and DCPEXT1, respectively. SIO supplies + * their sample streams; neither interface has AP-side registers. Keep the + * clients disabled until the T6041 SIO firmware backend and loader handoff + * are available and the compact audio protocol has been qualified. + */ + +/ { + dpaudio1: audio-controller-1 { + compatible = "apple,t6041-dpaudio", "apple,dpaudio"; + dmas = <&sio 0x66>; + dma-names = "tx"; + power-domains = <&ps_dpa1>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dpaudio1_dcpext: endpoint { + remote-endpoint = <&dcpext0_audio>; + }; + }; + }; + }; + + dpaudio2: audio-controller-2 { + compatible = "apple,t6041-dpaudio", "apple,dpaudio"; + dmas = <&sio 0x68>; + dma-names = "tx"; + power-domains = <&ps_dpa2>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dpaudio2_dcpext: endpoint { + remote-endpoint = <&dcpext1_audio>; + }; + }; + }; + }; +}; + +&dcpext0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dcpext0_audio: endpoint { + remote-endpoint = <&dpaudio1_dcpext>; + }; + }; + }; +}; + +&dcpext1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dcpext1_audio: endpoint { + remote-endpoint = <&dpaudio2_dcpext>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index bad9aa35c53dd..760840b14e37a 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -4519,6 +4519,7 @@ #include "t6040-j616s-extdisp-mem-loader.dtsi" #include "t6040-j616s-extdisp-hdmi.dtsi" #include "t6040-j616s-extdisp-usbc.dtsi" +#include "t6040-j616s-dpaudio.dtsi" /* Additional Abbey PMICs on the two independent NUB SPMI buses. */ &nub_spmi1 { status = "okay"; From f4efdec34bfa17f592faeb9962b15e92b8002f60 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:07:03 +1100 Subject: [PATCH 1539/1579] selftests: drm/apple: keep the audio assertion shim C11 compatible Replace the host assert header definition with an explicit message- bearing static assertion, so the fixture builds with strict C11 diagnostics on both Linux and macOS. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/testing/selftests/drivers/gpu/apple-dcp/audio.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/audio.py b/tools/testing/selftests/drivers/gpu/apple-dcp/audio.py index fc1a5503f8a85..44e2bd9768089 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/audio.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/audio.py @@ -19,7 +19,8 @@ def reply_bounds(root): typedef uint64_t __le64; #define KUNIT_STATIC_STUB_REDIRECT(...) ((void)0) #define READ_ONCE(x) (x) -#define static_assert _Static_assert +#undef static_assert +#define static_assert(expr, ...) _Static_assert(expr, #expr) #define GFP_KERNEL 0 static void free_buffer(void *p) {free(*(void **)p);} #define __free(x) __attribute__((cleanup(free_buffer))) From 895959e044ed05d288c58043e059d3377da27326 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 03:55:50 +1100 Subject: [PATCH 1540/1579] dt-bindings: media: describe Apple T6041 and T8140 JPEG engines Describe the per-engine register, interrupt, DART and power-domain resources, with the reserved high DMA window. Keep the example disabled until the buffer-pointer alias and engine behaviour are qualified. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/media/apple,jpeg.yaml | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 Documentation/devicetree/bindings/media/apple,jpeg.yaml diff --git a/Documentation/devicetree/bindings/media/apple,jpeg.yaml b/Documentation/devicetree/bindings/media/apple,jpeg.yaml new file mode 100644 index 0000000000000..5e31390cf9639 --- /dev/null +++ b/Documentation/devicetree/bindings/media/apple,jpeg.yaml @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/apple,jpeg.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Silicon JPEG codec + +maintainers: + - Ryan Murray + +description: | + Apple Silicon JPEG engines are directly programmed memory-to-memory codecs. + Each engine has a dedicated interrupt and DART stream. Multiple engines can + share a power domain. The power controller manages the gate; no separate + clock or firmware interface is required. + + The DMA window reserves one 4 GiB slice of the DART high alias. The engine + has 32-bit buffer pointer registers; use of this alias for jobs still needs + hardware qualification on each compatible. + +properties: + compatible: + enum: + - apple,t6041-jpeg + - apple,t8140-jpeg + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + apple,dma-range: + $ref: /schemas/types.yaml#/definitions/uint64-array + items: + - const: 0x10000000000 + - const: 0x100000000 + description: Device-visible DMA window, as a 64-bit base and size. + +required: + - compatible + - reg + - interrupts + - iommus + - power-domains + - apple,dma-range + +additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + jpeg@487000000 { + compatible = "apple,t6041-jpeg"; + reg = <0x4 0x87000000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&jpeg0_dart 0>; + power-domains = <&ps_jpg>; + apple,dma-range = /bits/ 64 <0x10000000000 0x100000000>; + status = "disabled"; + }; + }; From 9cd8a29a644684bf30dfa877100d8f50ca3bedaf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 03:57:04 +1100 Subject: [PATCH 1541/1579] arm64: dts: apple: add disabled T6041 and T8140 JPEG engines Describe both JPEG instances on each SoC and the missing second T8140 DART. Preserve the existing T6040 DART windows and T8140 jpeg_dart label. Leave all engine and DART nodes disabled pending hardware qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040.dtsi | 22 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8140.dtsi | 33 ++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index a67ef1e3b67a9..8377a06f610f2 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -2014,6 +2014,17 @@ status = "disabled"; }; + jpeg0: jpeg@487000000 { + compatible = "apple,t6041-jpeg"; + reg = <0x4 0x87000000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&jpeg0_dart 0>; + power-domains = <&ps_jpg>; + apple,dma-range = /bits/ 64 <0x10000000000 0x100000000>; + status = "disabled"; + }; + jpeg0_dart: iommu@487020000 { compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x4 0x87020000 0x0 0xc000>; @@ -2025,6 +2036,17 @@ status = "disabled"; }; + jpeg1: jpeg@487060000 { + compatible = "apple,t6041-jpeg"; + reg = <0x4 0x87060000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&jpeg1_dart 0>; + power-domains = <&ps_jpg>; + apple,dma-range = /bits/ 64 <0x10000000000 0x100000000>; + status = "disabled"; + }; + jpeg1_dart: iommu@487080000 { compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x4 0x87080000 0x0 0xc000>; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 450470726c3e6..2ccf0bf1bb42c 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -448,6 +448,17 @@ nonposted-mmio; ranges; + jpeg0: jpeg@28d000000 { + compatible = "apple,t8140-jpeg"; + reg = <0x2 0x8d000000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&jpeg_dart 0>; + power-domains = <&ps_jpg>; + apple,dma-range = /bits/ 64 <0x10000000000 0x100000000>; + status = "disabled"; + }; + jpeg_dart: iommu@28d020000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x2 0x8d020000 0x0 0xc000>; @@ -459,6 +470,28 @@ status = "disabled"; }; + jpeg1: jpeg@28d060000 { + compatible = "apple,t8140-jpeg"; + reg = <0x2 0x8d060000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&jpeg1_dart 0>; + power-domains = <&ps_jpg>; + apple,dma-range = /bits/ 64 <0x10000000000 0x100000000>; + status = "disabled"; + }; + + jpeg1_dart: iommu@28d080000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x2 0x8d080000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = /bits/ 64 <0x10000000000 0x30000000000>; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + apr_dart: iommu@295020000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x2 0x95020000 0x0 0x20000>; From 89e658c75ad36c73bfa2f872ac020e8a686d9229 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 04:59:04 +1100 Subject: [PATCH 1542/1579] media: apple: avd: bound AV1 reference indices before using them ref_frame_idx entries are -1 on frames that use no references (FFmpeg fills key frames that way), and the AV1 builder indexed reference_frame_ts with them directly: UBSAN reported index -1 into reference_frame_ts[8] in set_header and its helpers on the first AV1 decode on the J616s and the Neo. Look the timestamp up through a helper that returns a value no buffer has for an out-of-range index, so the lookup takes the existing missing-reference path. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-av1.c | 46 +++++++++++++--------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index 60d69bab8a342..64b441be90ffb 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -246,6 +246,19 @@ struct avd_av1_ctx { } bufs; }; +/* + * Timestamp of the reference a ref_frame_idx entry names. The entries are + * -1 on frames that use no references (FFmpeg fills key frames that way); + * indexing reference_frame_ts with them read out of bounds. Return a + * timestamp no buffer has, so the lookup takes the missing-reference path. + */ +static u64 avd_av1_ref_ts(const struct v4l2_ctrl_av1_frame *frame, int idx) +{ + if (idx < 0 || idx >= V4L2_AV1_TOTAL_REFS_PER_FRAME) + return U64_MAX; + return frame->reference_frame_ts[idx]; +} + /* * from verisilicon/rockchip_vpu981_hw_av1_dec.c */ @@ -367,8 +380,7 @@ static void set_refs(struct avd_ctx *ctx, struct avd_av1_run *run) ref_idx = i + V4L2_AV1_REF_LAST_FRAME; ref = avd_get_ref_buf( ctx, &dst->base.vb, - frame->reference_frame_ts - [frame->ref_frame_idx[i]]); + avd_av1_ref_ts(frame, frame->ref_frame_idx[i])); } addr = vb2_dma_contig_plane_dma_addr(&ref->base.vb.vb2_buf, 0) + ref->comp.start_offset; @@ -455,9 +467,8 @@ static bool is_valid_ref_frame(struct avd_ctx *ctx, struct avd_av1_run *run, const struct v4l2_ctrl_av1_frame *frame = run->frame; struct avd_decoded_buffer *ref = avd_get_ref_buf( ctx, &dst->base.vb, - frame->reference_frame_ts - [frame->ref_frame_idx[ref_frame - - V4L2_AV1_REF_LAST_FRAME]]); + avd_av1_ref_ts(frame, frame->ref_frame_idx[ref_frame - + V4L2_AV1_REF_LAST_FRAME])); int ref_mi_cols = DIV_ROUND_UP(ref->av1.width + 1, 8); int ref_mi_rows = DIV_ROUND_UP(ref->av1.height + 1, 8); bool intra_only = ((ref->av1.frame_type == V4L2_AV1_KEY_FRAME) || @@ -485,7 +496,7 @@ static int avd_select_refs(struct avd_ctx *ctx, struct avd_av1_run *run, if (seq->flags & (V4L2_AV1_SEQUENCE_FLAG_ENABLE_ORDER_HINT | V4L2_AV1_SEQUENCE_FLAG_ENABLE_REF_FRAME_MVS)) { ref = avd_get_ref_buf(ctx, &dst->base.vb, - frame->reference_frame_ts[last_ref_idx]); + avd_av1_ref_ts(frame, last_ref_idx)); if (dst != ref) { use_last = (ref->av1.order_hints[V4L2_AV1_REF_ALTREF_FRAME] != @@ -554,7 +565,7 @@ static void set_ref_hints(struct avd_ctx *ctx, struct avd_av1_run *run, V4L2_AV1_REF_LAST_FRAME]; ref = avd_get_ref_buf( ctx, &dst->base.vb, - frame->reference_frame_ts[ref_idx]); + avd_av1_ref_ts(frame, ref_idx)); push(AV1_ORDER_HINT0(ref->av1.order_hints[3]) | AV1_ORDER_HINT1(ref->av1.order_hints[2]) | @@ -585,8 +596,8 @@ static void set_ref_hdr(struct avd_ctx *ctx, struct avd_av1_run *run) /* point to the first index where the frames equal */ for (i = 0; i < 6; i++) { for (j = i + 1; j < 7; j++) { - if (frame->reference_frame_ts[frame->ref_frame_idx[i]] == - frame->reference_frame_ts[frame->ref_frame_idx[j]]) { + if (avd_av1_ref_ts(frame, frame->ref_frame_idx[i]) == + avd_av1_ref_ts(frame, frame->ref_frame_idx[j])) { ref_slots[j] = ref_slots[i]; break; } @@ -831,7 +842,7 @@ static void set_header(struct avd_ctx *ctx, struct avd_av1_run *run) V4L2_AV1_REF_LAST_FRAME]; ref = avd_get_ref_buf( ctx, &dst->base.vb, - frame->reference_frame_ts[ref_idx]); + avd_av1_ref_ts(frame, ref_idx)); ref_addr = vb2_dma_contig_plane_dma_addr( &ref->base.vb.vb2_buf, 0) + @@ -1075,9 +1086,8 @@ static void avd_av1_set_prob(struct avd_ctx *ctx, struct avd_av1_run *run) } else { ref = avd_get_ref_buf( ctx, &dst->base.vb, - run->frame->reference_frame_ts - [run->frame->ref_frame_idx - [run->frame->primary_ref_frame]]); + avd_av1_ref_ts(run->frame, + run->frame->ref_frame_idx[run->frame->primary_ref_frame])); memcpy(av1_ctx->bufs.probs.cpu, vb2_plane_vaddr(&ref->base.vb.vb2_buf, 0) + AVD_AV1_CDFS_OFFSET( @@ -1118,8 +1128,8 @@ static void avd_av1_set_seg(struct avd_ctx *ctx, struct avd_av1_run *run) if (frame->primary_ref_frame != AV1_PRIMARY_REF_NONE) { ref = avd_get_ref_buf( ctx, &dst->base.vb, - frame->reference_frame_ts - [frame->ref_frame_idx[frame->primary_ref_frame]]); + avd_av1_ref_ts(frame, + frame->ref_frame_idx[frame->primary_ref_frame])); /* avd_get_ref_buf() returns dst for a missing reference */ /* same MiCols and MiRows, as libdav1d compares them */ if (ref == dst || @@ -1384,9 +1394,9 @@ static int avd_av1_run(struct avd_ctx *ctx) ref = dst; if (run.frame->primary_ref_frame != AV1_PRIMARY_REF_NONE) ref = avd_get_ref_buf(ctx, &dst->base.vb, - run.frame->reference_frame_ts - [run.frame->ref_frame_idx - [run.frame->primary_ref_frame]]); + avd_av1_ref_ts(run.frame, + run.frame->ref_frame_idx + [run.frame->primary_ref_frame])); if (!vb2_plane_vaddr(&dst->base.vb.vb2_buf, 0) || !vb2_plane_vaddr(&ref->base.vb.vb2_buf, 0)) { avd_run_postamble(ctx, &run.base); From 752a3701719ce119f10968d0ad23255b6bbd8816 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 05:00:24 +1100 Subject: [PATCH 1543/1579] media: apple: avd: do not pop an empty firmware mailbox The interrupt handler read MBOX1_RETRIEVE without checking that the mailbox held a message. Reading an empty mailbox returns the previous message again, so a spurious or early interrupt replayed the last PP DONE: the next frame finished before its PP had run, userspace got a wrong picture, the frame's own PP DONE never came and the watchdog reset the decoder. Multi-stream H.264 benchmarks on the J616s hit this as bit-exactness failures, "ignoring stale completion" and timeouts. Check the empty flag in MBOX1_STATUS (bit 17) first, give a message that is not readable yet up to 10 us, and otherwise acknowledge and ignore the interrupt. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-drv.c | 14 ++++++++++++++ drivers/media/platform/apple/avd/avd-regs.h | 2 ++ 2 files changed, 16 insertions(+) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index bf5bf96f4c62a..65cecd267b545 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -300,6 +301,19 @@ static irqreturn_t avd_irq_handler(int irq, void *data) if (!ctx) return IRQ_HANDLED; + /* + * Reading RETRIEVE on an empty mailbox returns the previous message + * again. A repeated PP DONE finishes the next frame before its PP has + * run: userspace gets a wrong picture, the frame's own PP DONE never + * comes, and the watchdog resets the decoder. Give a late message + * 10 us, otherwise acknowledge and ignore the interrupt. + */ + if (readl_poll_timeout_atomic(avd->mbox + AVD_REG_MBOX1_STATUS, status, + !(status & AVD_MBOX1_STATUS_EMPTY), 1, 10)) { + writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); + dev_dbg_ratelimited(avd->dev, "spurious interrupt, mailbox empty\n"); + return IRQ_HANDLED; + } status = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); diff --git a/drivers/media/platform/apple/avd/avd-regs.h b/drivers/media/platform/apple/avd/avd-regs.h index 388f3e64895f5..ace0e4c48f40a 100644 --- a/drivers/media/platform/apple/avd/avd-regs.h +++ b/drivers/media/platform/apple/avd/avd-regs.h @@ -13,6 +13,8 @@ #define AVD_MBOX1_NOT_EMPTY BIT(3) #define AVD_REG_MBOX1_STATUS 0x5c +/* Read side: bits 8-11 write pointer, bits 12-15 read pointer, bit 17 empty. */ +#define AVD_MBOX1_STATUS_EMPTY BIT(17) #define AVD_REG_MBOX1_RETRIEVE 0x64 #define AVD_MBOX_ENABLE BIT(0) From fb2b82403531651766b49dcdf07380e0a02edcb4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 05:00:29 +1100 Subject: [PATCH 1544/1579] media: apple: avd: finish a job only on the PP DONE of its own PP The firmware reports VP DONE, the driver kicks PP, and PP DONE finishes the job. HEVC, VP9 and AV1 pictures with several segments kick PP once per segment, and a surplus PP DONE could arrive after the next job was submitted and finish it before its VP had even completed. Its buffers then went back to userspace while the decoder was still writing them; the J616s logged DART write faults at just-released addresses in such a run. Remember per job that its VP finished and its PP was kicked, and treat a PP DONE before that as stale. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-drv.c | 13 ++++++++++++- drivers/media/platform/apple/avd/avd.h | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 65cecd267b545..52a687ce4ba85 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -331,6 +331,10 @@ static irqreturn_t avd_irq_handler(int irq, void *data) /* a vp is done, kick the pp and hope for the best */ if(ctx->coded_fmt_desc->ops->submit) ctx->coded_fmt_desc->ops->submit(ctx); + spin_lock_irqsave(&avd->job_lock, flags); + if (avd->job_ctx == ctx) + avd->job_vp_done = true; + spin_unlock_irqrestore(&avd->job_lock, flags); goto done; } else { dev_err(avd->dev, "H%d error", status); @@ -352,7 +356,13 @@ static irqreturn_t avd_irq_handler(int irq, void *data) * let avd_device_run() finish the job when it returns. */ spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_ctx != ctx) { + if (avd->job_ctx != ctx || !avd->job_vp_done) { + /* + * Only the PP kicked for this job's VP can finish it. A surplus + * PP DONE of a multi-kick picture used to finish the next job + * early, so userspace reused its buffers while the decoder was + * still writing them. + */ stale = true; } else if (avd->job_state == AVD_JOB_SUBMITTED) { avd->job_state = AVD_JOB_FINISHING; @@ -415,6 +425,7 @@ static void avd_device_run(void *priv) avd->job_ctx = ctx; avd->job_state = AVD_JOB_RUNNING; avd->job_end_sent = false; + avd->job_vp_done = false; avd->job_pending = false; spin_unlock_irqrestore(&avd->job_lock, flags); dev_dbg(avd->dev, "avd-job: run seq %llu\n", seq); diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index ad7accb6e9615..2a1d914bd9d71 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -229,6 +229,8 @@ struct avd_dev { struct avd_ctx *job_ctx; u64 job_seq; bool job_end_sent; + /* The current job's VP finished and its PP was kicked. */ + bool job_vp_done; bool job_pending; enum vb2_buffer_state job_pending_result; From b6104a4ff2a3842ef8aeef323b82e140e8ef21c4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 05:00:32 +1100 Subject: [PATCH 1545/1579] media: apple: avd: quiesce the decoder before a reset and retire it after repeated ones The J616s hung hard three times in one night, each time after decoder timeouts and resets, and the Neo hung after resetting once per frame during a burst of VP errors. The reset asserted the pmgr reset with the CM3 still running, with DMA still able to reach memory, and while the threaded interrupt handler could be reading the mailbox; nothing bounded how often it ran. Before resetting: disable the interrupt (waiting for a running handler), stop the CM3, attach the empty IOMMU domain so a transfer still in flight faults at the DART instead of writing memory that may already be reused, and wait 5 ms for such transfers to drain. Then reset, restore and check the DMA domain, and boot. Each step is logged before it runs. Recover at most twice a minute and at least 20 ms apart. A failed recovery or a third one within a minute retires the decoder until the module is reloaded. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-drv.c | 80 +++++++++++++++++++--- drivers/media/platform/apple/avd/avd.h | 6 ++ 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 52a687ce4ba85..1824c3de61ed4 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -223,29 +223,88 @@ int avd_submit_job(struct avd_ctx *ctx) return 0; } +/* Recover at most twice a minute, never twice within this gap. */ +#define AVD_RECOVER_MAX 2 +#define AVD_RECOVER_WINDOW msecs_to_jiffies(60000) +#define AVD_RECOVER_GAP_MS 20 + +/* + * Reset the decoder after a timeout or a VP error. Every reset is a risk + * to the whole machine (the J616s hung hard after timeouts and resets + * three times in one night, and the Neo after a burst of one reset per + * frame), so: stop the interrupt handler and the CM3 first, block the + * decoder's DMA at its DART so a transfer still in flight faults instead + * of reaching memory that may already be reused, give such transfers time + * to drain, and only then assert the pmgr reset. Recover sparingly, and + * retire the decoder after a failed or a third recovery within a minute. + */ static int avd_reset(struct avd_dev *avd) { int ret = 0; + if (avd->retired) + return -ENODEV; + if (!avd->recoveries || + time_after(jiffies, avd->recover_window_start + AVD_RECOVER_WINDOW)) { + avd->recover_window_start = jiffies; + avd->recoveries = 0; + } + if (++avd->recoveries > AVD_RECOVER_MAX) { + ret = -EBUSY; + goto retire; + } + if (avd->last_recover && + time_before(jiffies, avd->last_recover + msecs_to_jiffies(AVD_RECOVER_GAP_MS))) + msleep(AVD_RECOVER_GAP_MS); + ret = pm_runtime_resume_and_get(avd->dev); if (ret < 0) - return ret; + goto retire; + /* No handler may touch the mailbox while the block is in reset. */ + disable_irq(avd->irq); + dev_warn(avd->dev, "recover: stop firmware\n"); + avd_shutdown(avd); + if (avd->empty_domain) { + dev_warn(avd->dev, "recover: block DMA\n"); + ret = iommu_attach_device(avd->empty_domain, avd->dev); + if (ret) { + dev_err(avd->dev, "recover: DART empty attach failed: %d\n", ret); + goto put; + } + usleep_range(5000, 6000); + } + dev_warn(avd->dev, "recover: reset\n"); ret = reset_control_reset(avd->rstc); if (ret) - dev_err(avd->dev, "reset: failed: %d", ret); - + dev_err(avd->dev, "recover: reset failed: %d\n", ret); if (avd->empty_domain) { - iommu_attach_device(avd->empty_domain, avd->dev); + dev_warn(avd->dev, "recover: restore DMA\n"); iommu_detach_device(avd->empty_domain, avd->dev); + if (avd->domain && iommu_get_domain_for_dev(avd->dev) != avd->domain) { + dev_err(avd->dev, "recover: DART domain not restored\n"); + ret = -EIO; + } + } + if (!ret) { + dev_warn(avd->dev, "recover: boot firmware\n"); + ret = avd_boot(avd); + if (ret) + dev_err(avd->dev, "recover: boot failed: %d\n", ret); + } +put: + avd->last_recover = jiffies ?: 1; + if (!ret) { + enable_irq(avd->irq); + dev_warn(avd->dev, "recover: done\n"); } - - ret = avd_boot(avd); - if (ret) - dev_err(avd->dev, "reset: failed to boot"); - pm_runtime_put_autosuspend(avd->dev); - + if (!ret) + return 0; +retire: + avd->retired = true; + dev_err(avd->dev, "decoder retired after %s (%d): reload the module to use it again\n", + ret == -EBUSY ? "repeated resets" : "a failed reset", ret); return ret; } @@ -979,6 +1038,7 @@ static int avd_probe(struct platform_device *pdev) irq = platform_get_irq(pdev, 1); if (irq < 0) return irq; + avd->irq = irq; ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, avd_irq_handler, IRQF_ONESHOT, dev_name(&pdev->dev), avd); diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 2a1d914bd9d71..98010d1fb0786 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -235,6 +235,12 @@ struct avd_dev { enum vb2_buffer_state job_pending_result; struct reset_control *rstc; + int irq; + /* Recovery budget; a retired decoder fails every job until reload. */ + unsigned long recover_window_start; + unsigned int recoveries; + unsigned long last_recover; + bool retired; const struct avd_variant *variant; unsigned int capabilities; From 68be6ab98e85052afd4b3a3a874148fb580bfd42 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 05:00:35 +1100 Subject: [PATCH 1546/1579] media: apple: avd: stop a stream after a frame fails in hardware A stream whose frame timed out or hit a VP error usually fails the next frame the same way. The Neo reset the decoder once per frame through a broken HEVC stream until it hung. After such a failure, finish every job of that stream with an error without touching the hardware, completing the request controls so the request fds still signal, until streaming is restarted. A retired decoder fails every job the same way. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-drv.c | 16 ++++++++++++++++ drivers/media/platform/apple/avd/avd-v4l2.c | 4 ++++ drivers/media/platform/apple/avd/avd.h | 2 ++ 3 files changed, 22 insertions(+) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 1824c3de61ed4..462df919265d5 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -344,6 +344,12 @@ static void avd_watchdog_func(struct work_struct *work) ret = avd_reset(avd); if (ret) dev_err(avd->dev, "failed to reset: %d", ret); + /* + * A stream whose frame failed in hardware usually fails the next one + * too (a reset per frame took the Neo down), so stop it here. + */ + ctx->broken = true; + dev_err(avd->dev, "stream stopped after a failed frame; restart streaming to retry\n"); avd_job_finish(ctx, VB2_BUF_STATE_ERROR); } @@ -461,6 +467,16 @@ static void avd_device_run(void *priv) if (WARN_ON(!desc)) return; + if (READ_ONCE(avd->retired) || ctx->broken) { + struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + + /* Complete the request's controls too, or its fd never signals. */ + if (src && src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + avd_job_finish_no_pm(ctx, VB2_BUF_STATE_ERROR); + return; + } + /* * IRQ completion cannot wait for an already running watchdog. Drain it * here, before reusing wd_seq, so an old callback cannot time out this diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index 4622e92055486..23917fd62bdca 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -889,6 +889,10 @@ static void avd_buf_request_complete(struct vb2_buffer *vb) static int avd_start_streaming(struct vb2_queue *q, unsigned int count) { struct avd_ctx *ctx = vb2_get_drv_priv(q); + + /* A new stream gets a fresh start after a hardware failure. */ + if (!V4L2_TYPE_IS_CAPTURE(q->type)) + ctx->broken = false; const struct avd_coded_fmt_desc *desc; struct vb2_v4l2_buffer *vbuf; int ret; diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 98010d1fb0786..20411c991ae31 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -266,6 +266,8 @@ struct avd_buf { struct avd_ctx { struct v4l2_fh fh; struct avd_dev *dev; + /* A frame failed in hardware: fail every job until streaming restarts. */ + bool broken; struct v4l2_format coded_fmt; struct v4l2_format decoded_fmt; From 9e3e963d4d789d2bbc4caf33e7479ee54ea8b982 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 05:00:39 +1100 Subject: [PATCH 1547/1579] media: apple: avd: accept FFmpeg's HEVC slice layout The HEVC builder computes each slice's coded data address and size from GStreamer's layout: slices tile the OUTPUT buffer as whole NALs, bit_size covers the whole NAL and data_byte_offset is the slice data's offset in it. FFmpeg's V4L2 request backend gives data_byte_offset from the start of the buffer and a bit_size of the slice data only. Every slice then got a wrong address and a size short by its header, the VP raised its error interrupt on every frame, and the decoder was reset once per frame. This is the "H0 error" storm the Neo hit in an FFmpeg-based benchmark. When the slices describe a complete, unambiguous absolute layout, rewrite the run's private copy into the equivalent NAL-relative description. The GStreamer layout and anything ambiguous are left untouched. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-hevc.c | 47 +++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index 9dfa20ab71913..49d8f046ac5c9 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -1282,6 +1282,51 @@ static int avd_hevc_compute_tiles(struct avd_ctx *ctx, struct avd_hevc_run *run) return 0; } +/* + * The slice builder below expects the GStreamer layout: the slices tile the + * OUTPUT buffer as whole NALs, bit_size covers the whole NAL and + * data_byte_offset is the slice data's offset inside it. FFmpeg's request + * API backend instead gives each slice's data_byte_offset from the start of + * the buffer and a bit_size of the slice data only, so every slice was given + * a wrong address and size and the VP raised its error interrupt on every + * frame. Detect the absolute layout (complete and unambiguous) and rewrite + * the run's private copy into the equivalent NAL-relative description. + */ +static void avd_hevc_normalize_slices(struct avd_hevc_run *run, u32 payload) +{ + struct v4l2_ctrl_hevc_slice_params *sl = + (struct v4l2_ctrl_hevc_slice_params *)run->sl; + u64 nal_end = 0, data_end = 0, prev_end = 0; + bool nal = true, absolute = true; + int i; + + if (!run->num_slices || !payload) + return; + for (i = 0; i < run->num_slices; i++) { + u64 bytes = DIV_ROUND_UP((u64)sl[i].bit_size, 8); + u64 start = sl[i].data_byte_offset; + + if (!bytes || start >= bytes || nal_end + bytes > payload) + nal = false; + nal_end += bytes; + if (!bytes || start < data_end || start + bytes > payload) + absolute = false; + data_end = start + bytes; + } + nal &= nal_end == payload; + absolute &= data_end == payload; + if (!absolute || nal) + return; + for (i = 0; i < run->num_slices; i++) { + u64 bytes = DIV_ROUND_UP((u64)sl[i].bit_size, 8); + u64 start = sl[i].data_byte_offset; + + sl[i].data_byte_offset = start - prev_end; + sl[i].bit_size = (start - prev_end + bytes) * 8; + prev_end = start + bytes; + } +} + static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) { struct v4l2_ctrl *ctrl, *sl, *ep; @@ -1353,6 +1398,8 @@ static int avd_hevc_run(struct avd_ctx *ctx) if (ret) return ret; + avd_hevc_normalize_slices(&run, + vb2_get_plane_payload(&run.base.bufs.src->vb2_buf, 0)); dst = vb2_to_avd_decoded_buf(&run.base.bufs.dst->vb2_buf); update_dec_buf_info(dst, &run.sl[0]); From 872e01eeed8b3120491a5cb803fdb09a4e0992d5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 04:01:34 +1100 Subject: [PATCH 1548/1579] media: apple: add fail-closed JPEG mem2mem scaffold Add common V4L2 mem2mem/VB2 format and queue scaffolding, runtime PM, and per-compatible exact HWREV admission. Both target allow-lists are empty, so probe refuses every revision before video registration. Read HWREV only with the power domain active. Implement no MMIO writes, reset sequence, IRQ handler or DMA job; independently reject STREAMON and scheduling. Retain device/context lifetime through last close. Document the admission boundary and MIT-licensed public register-map provenance. Add KUnit tests for empty target lists, invalid readbacks and exact matching using synthetic revisions. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../admin-guide/media/apple-jpeg.rst | 54 ++ drivers/media/platform/apple/Kconfig | 1 + drivers/media/platform/apple/Makefile | 1 + drivers/media/platform/apple/jpeg/Kconfig | 25 + drivers/media/platform/apple/jpeg/Makefile | 4 + .../media/platform/apple/jpeg/apple-jpeg-hw.h | 41 ++ .../platform/apple/jpeg/apple-jpeg-test.c | 56 ++ .../media/platform/apple/jpeg/apple-jpeg.c | 543 ++++++++++++++++++ 8 files changed, 725 insertions(+) create mode 100644 Documentation/admin-guide/media/apple-jpeg.rst create mode 100644 drivers/media/platform/apple/jpeg/Kconfig create mode 100644 drivers/media/platform/apple/jpeg/Makefile create mode 100644 drivers/media/platform/apple/jpeg/apple-jpeg-hw.h create mode 100644 drivers/media/platform/apple/jpeg/apple-jpeg-test.c create mode 100644 drivers/media/platform/apple/jpeg/apple-jpeg.c diff --git a/Documentation/admin-guide/media/apple-jpeg.rst b/Documentation/admin-guide/media/apple-jpeg.rst new file mode 100644 index 0000000000000..27ea6bcf2801e --- /dev/null +++ b/Documentation/admin-guide/media/apple-jpeg.rst @@ -0,0 +1,54 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +:orphan: + +Apple JPEG scaffold +=================== + +This driver provides a common platform and V4L2 memory-to-memory scaffold +for the Apple T6041 and T8140 JPEG engines. Each DT platform device owns one +independent m2m scheduler, with formats and VB2 queues allocated per open file. +T6041's nodes reside in ``t6040.dtsi`` in this source tree. All JPEG engine +nodes and the new T8140 DART remain disabled. SoC DART defaults are disabled; +existing board overrides are preserved. In particular, J700 already enables +its first JPEG DART independently of the disabled codec engines. + +No target HWREV is qualified. Runtime resume reads HWREV with the power domain +active and returns ``-ENODEV`` for an unlisted revision, including zero and +all-ones readbacks. Both per-compatible allow-lists are deliberately empty. +The driver does not infer a revision from a compatible, an instance number, +an ADT hw-type, or a revision observed on another SoC. + +Identification precedes reset: the public reset sequence has not been qualified +on these targets, so issuing it before admission would undermine the refusal +boundary. There are no MMIO writes, interrupt requests, reset sequences or DMA +jobs in this scaffold. Probe failure balances runtime PM and never registers +a video node. The power domain and the IOMMU are managed by their normal kernel +providers; the scaffold requires a SID-0 IOMMU attachment and the binding's +exact high DMA window. This is a topology check, not proof of pointer aliasing. + +The future video node has JPEG/NV12M format negotiation for encode and decode, +buffer allocation/import, and queue teardown. STREAMON returns ``-EOPNOTSUPP`` +and returns queued buffers according to the VB2 failure contract. The scheduler +also refuses jobs. The advertised allocation dimensions and formats are draft +API scaffolding, not a tested codec capability. Open video files retain the +m2m/device allocation through final close after platform removal; they cannot +access removed MMIO resources. + +Before enabling nodes or adding a target revision, qualification must establish +the revision, bounded reset/idle sequence, interrupt acknowledgment/deassertion, +high DVA alias, timeout/recovery and supported geometry on both instances of +each SoC. Filling an HWREV list alone does not implement streaming. Reset and +job code must validate revisions again on every runtime resume, preserve the +shared power-domain accounting, and return DMA buffers only after quiescing +hardware. Encode/decode controls, JPEG header validation, media-controller +topology and actual job programming belong to the subsequent codec work. + +The HWREV register offset comes from the MIT-licensed Asahi Linux m1n1 register +description at commit ``f3ac615eb1376d3bbba24cd84c414b9e41892adb``: + +https://github.com/AsahiLinux/m1n1/blob/f3ac615eb1376d3bbba24cd84c414b9e41892adb/proxyclient/m1n1/hw/jpeg.py + +The scaffold deliberately does not translate the experiment's unqualified +reset or interrupt writes. ``VIDEO_APPLE_JPEG_KUNIT_TEST`` tests empty target +allow-lists, invalid readbacks and exact matching with synthetic revisions. diff --git a/drivers/media/platform/apple/Kconfig b/drivers/media/platform/apple/Kconfig index f656926099b3e..6ee29529f4466 100644 --- a/drivers/media/platform/apple/Kconfig +++ b/drivers/media/platform/apple/Kconfig @@ -5,3 +5,4 @@ comment "Apple media platform drivers" source "drivers/media/platform/apple/isp/Kconfig" source "drivers/media/platform/apple/avd/Kconfig" source "drivers/media/platform/apple/ave/Kconfig" +source "drivers/media/platform/apple/jpeg/Kconfig" diff --git a/drivers/media/platform/apple/Makefile b/drivers/media/platform/apple/Makefile index dac50fc70e277..39f5eccd5bb1c 100644 --- a/drivers/media/platform/apple/Makefile +++ b/drivers/media/platform/apple/Makefile @@ -3,3 +3,4 @@ obj-y += isp/ obj-y += avd/ obj-y += ave/ +obj-y += jpeg/ diff --git a/drivers/media/platform/apple/jpeg/Kconfig b/drivers/media/platform/apple/jpeg/Kconfig new file mode 100644 index 0000000000000..f429b8970dbfc --- /dev/null +++ b/drivers/media/platform/apple/jpeg/Kconfig @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config VIDEO_APPLE_JPEG + tristate "Apple Silicon JPEG codec skeleton" + depends on VIDEO_DEV + depends on ARCH_APPLE || (COMPILE_TEST && 64BIT) + depends on OF_ADDRESS && IOMMU_API && PM + select V4L2_MEM2MEM_DEV + select VIDEOBUF2_DMA_CONTIG + help + Build the common Apple JPEG memory-to-memory driver scaffold. + Hardware revisions are not qualified yet: probe reads HWREV and + refuses all currently described hardware. No reset, interrupt + acknowledgment or DMA job is implemented, and streaming is refused. + Device tree nodes must remain disabled pending hardware qualification. + +config VIDEO_APPLE_JPEG_KUNIT_TEST + tristate "KUnit tests for Apple JPEG admission" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Test exact per-compatible HWREV matching and the empty target + allow-lists, including invalid readbacks and revisions observed + on other SoCs. Synthetic match data exercises the admission helper. + These tests do not access hardware or qualify any target revision. diff --git a/drivers/media/platform/apple/jpeg/Makefile b/drivers/media/platform/apple/jpeg/Makefile new file mode 100644 index 0000000000000..eee453f627047 --- /dev/null +++ b/drivers/media/platform/apple/jpeg/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_VIDEO_APPLE_JPEG) += apple-jpeg.o +obj-$(CONFIG_VIDEO_APPLE_JPEG_KUNIT_TEST) += apple-jpeg-test.o diff --git a/drivers/media/platform/apple/jpeg/apple-jpeg-hw.h b/drivers/media/platform/apple/jpeg/apple-jpeg-hw.h new file mode 100644 index 0000000000000..332f4313e7f34 --- /dev/null +++ b/drivers/media/platform/apple/jpeg/apple-jpeg-hw.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef APPLE_JPEG_HW_H +#define APPLE_JPEG_HW_H + +#include +#include + +/* Public MIT-licensed m1n1 register map: proxyclient/m1n1/hw/jpeg.py. */ +#define APPLE_JPEG_HWREV 0x188 +#define APPLE_JPEG_REG_SIZE 0x4000 +#define APPLE_JPEG_DMA_BASE 0x10000000000ULL +#define APPLE_JPEG_DMA_SIZE 0x100000000ULL + +struct apple_jpeg_variant { + const u32 *hwrevs; + size_t num_hwrevs; +}; + +/* + * Empty until each SoC's revision and reset/IRQ behaviour are qualified. + * Neither an ADT hw-type nor m1n1's M1 Max revision qualifies another SoC. + */ +static const struct apple_jpeg_variant apple_jpeg_t6041 = { }; +static const struct apple_jpeg_variant apple_jpeg_t8140 = { }; + +static inline bool +apple_jpeg_hwrev_supported(const struct apple_jpeg_variant *variant, u32 hwrev) +{ + size_t i; + + if (!variant || !variant->hwrevs || !hwrev || hwrev == U32_MAX) + return false; + + for (i = 0; i < variant->num_hwrevs; i++) + if (variant->hwrevs[i] == hwrev) + return true; + + return false; +} + +#endif /* APPLE_JPEG_HW_H */ diff --git a/drivers/media/platform/apple/jpeg/apple-jpeg-test.c b/drivers/media/platform/apple/jpeg/apple-jpeg-test.c new file mode 100644 index 0000000000000..651a2b182252b --- /dev/null +++ b/drivers/media/platform/apple/jpeg/apple-jpeg-test.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "apple-jpeg-hw.h" + +static void apple_jpeg_unqualified_test(struct kunit *test) +{ + const struct apple_jpeg_variant *variants[] = { + &apple_jpeg_t6041, &apple_jpeg_t8140, + }; + /* Public M1 Max revision, arbitrary revisions and invalid readbacks. */ + const u32 revisions[] = { 0xd1013, 1, 0x12345678, 0, U32_MAX }; + size_t i, j; + + for (i = 0; i < ARRAY_SIZE(variants); i++) { + KUNIT_EXPECT_EQ(test, variants[i]->num_hwrevs, (size_t)0); + for (j = 0; j < ARRAY_SIZE(revisions); j++) + KUNIT_EXPECT_FALSE(test, + apple_jpeg_hwrev_supported(variants[i], revisions[j])); + } +} + +static void apple_jpeg_exact_match_test(struct kunit *test) +{ + /* Synthetic revisions exercise the matcher without qualifying hardware. */ + const u32 revisions[] = { 0x12345678, 0x1234567a }; + const struct apple_jpeg_variant variant = { + .hwrevs = revisions, + .num_hwrevs = ARRAY_SIZE(revisions), + }; + const struct apple_jpeg_variant empty = { .hwrevs = revisions }; + + KUNIT_EXPECT_TRUE(test, apple_jpeg_hwrev_supported(&variant, revisions[0])); + KUNIT_EXPECT_TRUE(test, apple_jpeg_hwrev_supported(&variant, revisions[1])); + KUNIT_EXPECT_FALSE(test, apple_jpeg_hwrev_supported(&variant, 0x12345679)); + KUNIT_EXPECT_FALSE(test, apple_jpeg_hwrev_supported(&variant, 0)); + KUNIT_EXPECT_FALSE(test, apple_jpeg_hwrev_supported(&variant, U32_MAX)); + KUNIT_EXPECT_FALSE(test, apple_jpeg_hwrev_supported(&empty, revisions[0])); + KUNIT_EXPECT_FALSE(test, apple_jpeg_hwrev_supported(NULL, revisions[0])); +} + +static struct kunit_case apple_jpeg_cases[] = { + KUNIT_CASE(apple_jpeg_unqualified_test), + KUNIT_CASE(apple_jpeg_exact_match_test), + { } +}; + +static struct kunit_suite apple_jpeg_suite = { + .name = "apple-jpeg-admission", + .test_cases = apple_jpeg_cases, +}; + +kunit_test_suite(apple_jpeg_suite); + +MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/jpeg/apple-jpeg.c b/drivers/media/platform/apple/jpeg/apple-jpeg.c new file mode 100644 index 0000000000000..0d8b637fd5a29 --- /dev/null +++ b/drivers/media/platform/apple/jpeg/apple-jpeg.c @@ -0,0 +1,543 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple Silicon JPEG mem2mem scaffold. No hardware jobs are implemented. + * Register offset provenance: Asahi Linux m1n1 (MIT), commit + * f3ac615eb1376d3bbba24cd84c414b9e41892adb, proxyclient/m1n1/hw/jpeg.py. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "apple-jpeg-hw.h" + +struct apple_jpeg { + struct device *dev; + void __iomem *regs; + const struct apple_jpeg_variant *variant; + struct mutex lock; /* Serializes video operations and queue/context state. */ + struct v4l2_device v4l2_dev; + struct video_device vdev; + struct v4l2_m2m_dev *m2m_dev; +}; + +struct apple_jpeg_ctx { + struct v4l2_fh fh; + struct apple_jpeg *jpeg; + struct v4l2_pix_format_mplane output; + struct v4l2_pix_format_mplane capture; +}; + +static struct apple_jpeg_ctx *fh_to_ctx(void *priv) +{ + return container_of(priv, struct apple_jpeg_ctx, fh); +} + +static struct v4l2_pix_format_mplane * +apple_jpeg_format(struct apple_jpeg_ctx *ctx, enum v4l2_buf_type type) +{ + if (type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) + return &ctx->output; + if (type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) + return &ctx->capture; + return NULL; +} + +/* Allocation policy for the scaffold, not a qualified hardware size limit. */ +static void apple_jpeg_normalize_format(struct v4l2_pix_format_mplane *pix) +{ + u32 width = clamp_t(u32, pix->width, 16, 8192); + u32 height = clamp_t(u32, pix->height, 16, 8192); + u32 fourcc = pix->pixelformat; + + memset(pix, 0, sizeof(*pix)); + pix->width = ALIGN(width, 16); + pix->height = ALIGN(height, 16); + pix->field = V4L2_FIELD_NONE; + pix->colorspace = V4L2_COLORSPACE_JPEG; + pix->ycbcr_enc = V4L2_YCBCR_ENC_601; + pix->quantization = V4L2_QUANTIZATION_FULL_RANGE; + pix->xfer_func = V4L2_XFER_FUNC_SRGB; + + if (fourcc == V4L2_PIX_FMT_NV12M) { + pix->pixelformat = fourcc; + pix->num_planes = 2; + pix->plane_fmt[0].bytesperline = pix->width; + pix->plane_fmt[0].sizeimage = pix->width * pix->height; + pix->plane_fmt[1].bytesperline = pix->width; + pix->plane_fmt[1].sizeimage = pix->width * pix->height / 2; + } else { + pix->pixelformat = V4L2_PIX_FMT_JPEG; + pix->num_planes = 1; + pix->plane_fmt[0].sizeimage = pix->width * pix->height * 3 + 4096; + } +} + +static int apple_jpeg_querycap(struct file *file, void *priv, + struct v4l2_capability *cap) +{ + struct apple_jpeg_ctx *ctx = fh_to_ctx(priv); + + strscpy(cap->driver, "apple-jpeg", sizeof(cap->driver)); + strscpy(cap->card, "Apple JPEG scaffold", sizeof(cap->card)); + snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s", + dev_name(ctx->jpeg->dev)); + return 0; +} + +static int apple_jpeg_enum_fmt(struct file *file, void *priv, + struct v4l2_fmtdesc *f) +{ + if (f->index > 1) + return -EINVAL; + f->pixelformat = f->index ? V4L2_PIX_FMT_NV12M : V4L2_PIX_FMT_JPEG; + return 0; +} + +static int apple_jpeg_g_fmt(struct file *file, void *priv, struct v4l2_format *f) +{ + struct v4l2_pix_format_mplane *pix = apple_jpeg_format(fh_to_ctx(priv), f->type); + + if (!pix) + return -EINVAL; + f->fmt.pix_mp = *pix; + return 0; +} + +static int apple_jpeg_try_fmt(struct file *file, void *priv, struct v4l2_format *f) +{ + if (!apple_jpeg_format(fh_to_ctx(priv), f->type)) + return -EINVAL; + apple_jpeg_normalize_format(&f->fmt.pix_mp); + return 0; +} + +static int apple_jpeg_s_fmt(struct file *file, void *priv, struct v4l2_format *f) +{ + struct apple_jpeg_ctx *ctx = fh_to_ctx(priv); + struct v4l2_pix_format_mplane *pix, *other; + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + int ret; + + ret = apple_jpeg_try_fmt(file, priv, f); + if (ret) + return ret; + if (vb2_is_busy(v4l2_m2m_get_src_vq(m2m)) || + vb2_is_busy(v4l2_m2m_get_dst_vq(m2m))) + return -EBUSY; + + pix = apple_jpeg_format(ctx, f->type); + other = pix == &ctx->output ? &ctx->capture : &ctx->output; + *pix = f->fmt.pix_mp; + *other = *pix; + other->pixelformat = pix->pixelformat == V4L2_PIX_FMT_JPEG ? + V4L2_PIX_FMT_NV12M : V4L2_PIX_FMT_JPEG; + apple_jpeg_normalize_format(other); + return 0; +} + +static int apple_jpeg_queue_setup(struct vb2_queue *q, unsigned int *num_buffers, + unsigned int *num_planes, unsigned int sizes[], + struct device *alloc_devs[]) +{ + struct apple_jpeg_ctx *ctx = vb2_get_drv_priv(q); + struct v4l2_pix_format_mplane *pix = apple_jpeg_format(ctx, q->type); + unsigned int i; + + if (*num_planes) { + if (*num_planes != pix->num_planes) + return -EINVAL; + for (i = 0; i < *num_planes; i++) + if (sizes[i] < pix->plane_fmt[i].sizeimage) + return -EINVAL; + return 0; + } + *num_planes = pix->num_planes; + for (i = 0; i < *num_planes; i++) + sizes[i] = pix->plane_fmt[i].sizeimage; + return 0; +} + +static int apple_jpeg_buf_prepare(struct vb2_buffer *vb) +{ + struct apple_jpeg_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + struct v4l2_pix_format_mplane *pix = apple_jpeg_format(ctx, vb->type); + unsigned int i; + + for (i = 0; i < pix->num_planes; i++) { + if (vb2_plane_size(vb, i) < pix->plane_fmt[i].sizeimage) + return -EINVAL; + if (!V4L2_TYPE_IS_OUTPUT(vb->type)) + vb2_set_plane_payload(vb, i, 0); + } + return 0; +} + +static void apple_jpeg_buf_queue(struct vb2_buffer *vb) +{ + struct apple_jpeg_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + + v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, to_vb2_v4l2_buffer(vb)); +} + +static void apple_jpeg_return_buffers(struct apple_jpeg_ctx *ctx, + enum v4l2_buf_type type, + enum vb2_buffer_state state) +{ + struct vb2_v4l2_buffer *buf; + + for (;;) { + buf = V4L2_TYPE_IS_OUTPUT(type) ? + v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx) : + v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + if (!buf) + break; + v4l2_m2m_buf_done(buf, state); + } +} + +static int apple_jpeg_start_streaming(struct vb2_queue *q, unsigned int count) +{ + /* VB2 requires queued buffers back on a failed STREAMON. No MMIO here. */ + apple_jpeg_return_buffers(vb2_get_drv_priv(q), q->type, VB2_BUF_STATE_QUEUED); + return -EOPNOTSUPP; +} + +static void apple_jpeg_stop_streaming(struct vb2_queue *q) +{ + apple_jpeg_return_buffers(vb2_get_drv_priv(q), q->type, VB2_BUF_STATE_ERROR); +} + +static const struct vb2_ops apple_jpeg_queue_ops = { + .queue_setup = apple_jpeg_queue_setup, + .buf_prepare = apple_jpeg_buf_prepare, + .buf_queue = apple_jpeg_buf_queue, + .start_streaming = apple_jpeg_start_streaming, + .stop_streaming = apple_jpeg_stop_streaming, +}; + +static int apple_jpeg_init_queue(struct apple_jpeg_ctx *ctx, struct vb2_queue *q, + enum v4l2_buf_type type) +{ + q->type = type; + q->io_modes = VB2_MMAP | VB2_DMABUF; + q->drv_priv = ctx; + q->buf_struct_size = sizeof(struct v4l2_m2m_buffer); + q->ops = &apple_jpeg_queue_ops; + q->mem_ops = &vb2_dma_contig_memops; + q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + q->lock = &ctx->jpeg->lock; + q->dev = ctx->jpeg->dev; + return vb2_queue_init(q); +} + +static int apple_jpeg_queue_init(void *priv, struct vb2_queue *src, + struct vb2_queue *dst) +{ + struct apple_jpeg_ctx *ctx = priv; + int ret; + + ret = apple_jpeg_init_queue(ctx, src, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + if (ret) + return ret; + return apple_jpeg_init_queue(ctx, dst, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); +} + +static int apple_jpeg_job_ready(void *priv) +{ + return 0; +} + +static void apple_jpeg_device_run(void *priv) +{ + struct apple_jpeg_ctx *ctx = priv; + + /* Defensive fallback; STREAMON and job_ready prevent scheduling. */ + apple_jpeg_return_buffers(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, + VB2_BUF_STATE_ERROR); + apple_jpeg_return_buffers(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, + VB2_BUF_STATE_ERROR); + v4l2_m2m_job_finish(ctx->jpeg->m2m_dev, ctx->fh.m2m_ctx); +} + +static const struct v4l2_m2m_ops apple_jpeg_m2m_ops = { + .device_run = apple_jpeg_device_run, + .job_ready = apple_jpeg_job_ready, +}; + +static int apple_jpeg_open(struct file *file) +{ + struct apple_jpeg *jpeg = video_drvdata(file); + struct apple_jpeg_ctx *ctx; + int ret = 0; + + if (mutex_lock_interruptible(&jpeg->lock)) + return -ERESTARTSYS; + if (!video_is_registered(&jpeg->vdev)) { + ret = -ENODEV; + goto unlock; + } + ctx = kzalloc_obj(*ctx); + if (!ctx) { + ret = -ENOMEM; + goto unlock; + } + ctx->jpeg = jpeg; + ctx->output.width = 640; + ctx->output.height = 480; + ctx->output.pixelformat = V4L2_PIX_FMT_JPEG; + apple_jpeg_normalize_format(&ctx->output); + ctx->capture = ctx->output; + ctx->capture.pixelformat = V4L2_PIX_FMT_NV12M; + apple_jpeg_normalize_format(&ctx->capture); + v4l2_fh_init(&ctx->fh, &jpeg->vdev); + ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(jpeg->m2m_dev, ctx, apple_jpeg_queue_init); + if (IS_ERR(ctx->fh.m2m_ctx)) { + ret = PTR_ERR(ctx->fh.m2m_ctx); + v4l2_fh_exit(&ctx->fh); + kfree(ctx); + goto unlock; + } + v4l2_fh_add(&ctx->fh, file); +unlock: + mutex_unlock(&jpeg->lock); + return ret; +} + +static int apple_jpeg_release(struct file *file) +{ + struct apple_jpeg_ctx *ctx = fh_to_ctx(file->private_data); + struct apple_jpeg *jpeg = ctx->jpeg; + + mutex_lock(&jpeg->lock); + v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); + v4l2_fh_del(&ctx->fh, file); + v4l2_fh_exit(&ctx->fh); + kfree(ctx); + mutex_unlock(&jpeg->lock); + return 0; +} + +static const struct v4l2_file_operations apple_jpeg_fops = { + .owner = THIS_MODULE, + .open = apple_jpeg_open, + .release = apple_jpeg_release, + .poll = v4l2_m2m_fop_poll, + .unlocked_ioctl = video_ioctl2, + .mmap = v4l2_m2m_fop_mmap, +}; + +static const struct v4l2_ioctl_ops apple_jpeg_ioctl_ops = { + .vidioc_querycap = apple_jpeg_querycap, + .vidioc_enum_fmt_vid_cap = apple_jpeg_enum_fmt, + .vidioc_enum_fmt_vid_out = apple_jpeg_enum_fmt, + .vidioc_g_fmt_vid_cap_mplane = apple_jpeg_g_fmt, + .vidioc_g_fmt_vid_out_mplane = apple_jpeg_g_fmt, + .vidioc_try_fmt_vid_cap_mplane = apple_jpeg_try_fmt, + .vidioc_try_fmt_vid_out_mplane = apple_jpeg_try_fmt, + .vidioc_s_fmt_vid_cap_mplane = apple_jpeg_s_fmt, + .vidioc_s_fmt_vid_out_mplane = apple_jpeg_s_fmt, + .vidioc_reqbufs = v4l2_m2m_ioctl_reqbufs, + .vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs, + .vidioc_querybuf = v4l2_m2m_ioctl_querybuf, + .vidioc_qbuf = v4l2_m2m_ioctl_qbuf, + .vidioc_dqbuf = v4l2_m2m_ioctl_dqbuf, + .vidioc_expbuf = v4l2_m2m_ioctl_expbuf, + .vidioc_streamon = v4l2_m2m_ioctl_streamon, + .vidioc_streamoff = v4l2_m2m_ioctl_streamoff, +}; + +static int apple_jpeg_runtime_resume(struct device *dev) +{ + struct apple_jpeg *jpeg = dev_get_drvdata(dev); + u32 hwrev = readl(jpeg->regs + APPLE_JPEG_HWREV); + + /* Identify before any reset writes; no target reset profile is known. */ + if (!apple_jpeg_hwrev_supported(jpeg->variant, hwrev)) + return dev_err_probe(dev, -ENODEV, "unqualified HWREV %#x\n", hwrev); + return 0; +} + +static int apple_jpeg_runtime_suspend(struct device *dev) +{ + /* This scaffold never starts DMA or changes the engine's register state. */ + return 0; +} + +static const struct dev_pm_ops apple_jpeg_pm_ops = { + RUNTIME_PM_OPS(apple_jpeg_runtime_suspend, apple_jpeg_runtime_resume, NULL) + SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) +}; + +static int apple_jpeg_check_dma(struct device *dev) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct of_phandle_args args; + u64 range[2]; + int ret; + + if (of_count_phandle_with_args(dev->of_node, "iommus", "#iommu-cells") != 1) + return -EINVAL; + ret = of_parse_phandle_with_args(dev->of_node, "iommus", "#iommu-cells", 0, &args); + if (ret) + return ret; + of_node_put(args.np); + if (args.args_count != 1 || args.args[0] != 0) + return -EINVAL; + if (!domain || (domain->type != IOMMU_DOMAIN_DMA && + domain->type != IOMMU_DOMAIN_DMA_FQ)) + return -ENODEV; + if (of_property_count_u64_elems(dev->of_node, "apple,dma-range") != 2) + return -EINVAL; + ret = of_property_read_u64_array(dev->of_node, "apple,dma-range", range, 2); + if (ret) + return ret; + if (range[0] != APPLE_JPEG_DMA_BASE || range[1] != APPLE_JPEG_DMA_SIZE) + return -EINVAL; + /* BIT(40) is part of the DART alias, not a 32-bit physical DMA mask. */ + return dma_set_mask_and_coherent(dev, DMA_BIT_MASK(41)); +} + +static void apple_jpeg_v4l2_release(struct v4l2_device *v4l2_dev) +{ + struct apple_jpeg *jpeg = container_of(v4l2_dev, struct apple_jpeg, v4l2_dev); + + if (jpeg->m2m_dev) + v4l2_m2m_release(jpeg->m2m_dev); + put_device(jpeg->dev); + mutex_destroy(&jpeg->lock); + kfree(jpeg); +} + +static int apple_jpeg_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_jpeg *jpeg; + struct resource *res; + int ret; + + ret = apple_jpeg_check_dma(dev); + if (ret) + return dev_err_probe(dev, ret, "requires a dedicated SID-0 DART window\n"); + if (!dev->pm_domain) + return dev_err_probe(dev, -ENODEV, "requires a power domain\n"); + ret = platform_get_irq(pdev, 0); + if (ret < 0) + return ret; + /* Do not request an IRQ before acknowledgment semantics are qualified. */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res || resource_size(res) != APPLE_JPEG_REG_SIZE) + return -EINVAL; + + jpeg = kzalloc_obj(*jpeg); + if (!jpeg) + return -ENOMEM; + jpeg->dev = get_device(dev); + jpeg->variant = of_device_get_match_data(dev); + mutex_init(&jpeg->lock); + jpeg->regs = devm_ioremap_resource(dev, res); + if (IS_ERR(jpeg->regs)) { + ret = PTR_ERR(jpeg->regs); + goto free_jpeg; + } + platform_set_drvdata(pdev, jpeg); + pm_runtime_set_autosuspend_delay(dev, 100); + pm_runtime_use_autosuspend(dev); + pm_runtime_enable(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + goto disable_pm; + ret = pm_runtime_put_sync_suspend(dev); + if (ret < 0) + goto disable_pm; + + ret = v4l2_device_register(dev, &jpeg->v4l2_dev); + if (ret) + goto disable_pm; + jpeg->v4l2_dev.release = apple_jpeg_v4l2_release; + jpeg->m2m_dev = v4l2_m2m_init(&apple_jpeg_m2m_ops); + if (IS_ERR(jpeg->m2m_dev)) { + ret = PTR_ERR(jpeg->m2m_dev); + jpeg->m2m_dev = NULL; + goto unregister_v4l2; + } + strscpy(jpeg->vdev.name, "apple-jpeg", sizeof(jpeg->vdev.name)); + jpeg->vdev.v4l2_dev = &jpeg->v4l2_dev; + jpeg->vdev.fops = &apple_jpeg_fops; + jpeg->vdev.ioctl_ops = &apple_jpeg_ioctl_ops; + jpeg->vdev.release = video_device_release_empty; + jpeg->vdev.lock = &jpeg->lock; + jpeg->vdev.vfl_dir = VFL_DIR_M2M; + jpeg->vdev.device_caps = V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING; + video_set_drvdata(&jpeg->vdev, jpeg); + ret = video_register_device(&jpeg->vdev, VFL_TYPE_VIDEO, -1); + if (ret) + goto unregister_v4l2; + return 0; + +unregister_v4l2: + v4l2_device_unregister(&jpeg->v4l2_dev); + pm_runtime_disable(dev); + pm_runtime_dont_use_autosuspend(dev); + platform_set_drvdata(pdev, NULL); + v4l2_device_put(&jpeg->v4l2_dev); + return ret; +disable_pm: + pm_runtime_disable(dev); + pm_runtime_dont_use_autosuspend(dev); + platform_set_drvdata(pdev, NULL); +free_jpeg: + put_device(jpeg->dev); + mutex_destroy(&jpeg->lock); + kfree(jpeg); + return ret; +} + +static void apple_jpeg_remove(struct platform_device *pdev) +{ + struct apple_jpeg *jpeg = platform_get_drvdata(pdev); + + mutex_lock(&jpeg->lock); + v4l2_device_disconnect(&jpeg->v4l2_dev); + video_unregister_device(&jpeg->vdev); + mutex_unlock(&jpeg->lock); + v4l2_device_unregister(&jpeg->v4l2_dev); + pm_runtime_disable(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); + platform_set_drvdata(pdev, NULL); + /* Open files retain the video_device's V4L2 reference until last close. */ + v4l2_device_put(&jpeg->v4l2_dev); +} + +static const struct of_device_id apple_jpeg_of_match[] = { + { .compatible = "apple,t6041-jpeg", .data = &apple_jpeg_t6041 }, + { .compatible = "apple,t8140-jpeg", .data = &apple_jpeg_t8140 }, + { } +}; +MODULE_DEVICE_TABLE(of, apple_jpeg_of_match); + +static struct platform_driver apple_jpeg_driver = { + .probe = apple_jpeg_probe, + .remove = apple_jpeg_remove, + .driver = { + .name = "apple-jpeg", + .of_match_table = apple_jpeg_of_match, + .pm = pm_ptr(&apple_jpeg_pm_ops), + }, +}; +module_platform_driver(apple_jpeg_driver); + +MODULE_DESCRIPTION("Apple Silicon JPEG mem2mem scaffold"); +MODULE_LICENSE("GPL"); From a1472a9d26f216d147f91fe158d722445c384523 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 21:55:31 +1100 Subject: [PATCH 1549/1579] dt-bindings: apple: describe measured T600x DCS bandwidth OPPs Describe the existing fabric and DCS register owner and its optional shared interconnect path. The bandwidth table carries explicit raw PMGR states and measured sustainable capacities; it does not infer clocks from PMP policy thresholds. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../opp/operating-points-v2-apple-dcs.yaml | 48 +++++++++++++++++++ .../soc/apple/apple,t6000-pmgr-misc.yaml | 29 +++++++++++ 2 files changed, 77 insertions(+) create mode 100644 Documentation/devicetree/bindings/opp/operating-points-v2-apple-dcs.yaml diff --git a/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dcs.yaml b/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dcs.yaml new file mode 100644 index 0000000000000..28315f6f799c4 --- /dev/null +++ b/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dcs.yaml @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/opp/operating-points-v2-apple-dcs.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T600x measured DCS bandwidth operating points + +maintainers: + - Ryan Murray + +allOf: + - $ref: opp-v2-base.yaml# + +description: | + Each OPP maps a verified raw PMGR desired performance state to measured + sustainable bandwidth. Values are decimal kB/s, not DRAM clock frequencies, + theoretical peak bandwidth or firmware policy thresholds. State codes and + bandwidths must strictly increase together. The highest OPP must match the + boot firmware's desired state. Unmeasured intermediate states may be omitted. + +properties: + compatible: + const: operating-points-v2-apple-dcs + +patternProperties: + '^opp(-?[0-9]+)*$': + type: object + properties: + opp-supported-hw: false + turbo-mode: false + opp-level: + description: Verified raw PMGR desired state, without a state-index bias + minimum: 7 + maximum: 15 + opp-peak-kBps: + description: Measured sustainable bandwidth at this raw state + items: + - minimum: 1 + required: + - opp-level + - opp-peak-kBps + unevaluatedProperties: false + +required: + - compatible + +unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml index 588100222f892..e13f8d8394d21 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml @@ -14,6 +14,11 @@ description: | via a different "misc" control block. This includes the fabric and memory controller states. + When the host owns these registers and PMP firmware is disabled, an optional + measured bandwidth OPP table enables one shared interconnect provider on + T600x. Endpoint 0 represents clients and endpoint 1 represents memory. All + clients use the path from 0 to 1, so their bandwidth requests aggregate. + properties: compatible: enum: @@ -28,11 +33,35 @@ properties: - const: fabric-ps - const: dcs-ps + '#interconnect-cells': + const: 1 + + operating-points-v2: + description: | + Measured sustainable bandwidth table following + operating-points-v2-apple-dcs.yaml. Only valid with host ownership of DCS; + do not enable PMP firmware together with this interface. + maxItems: 1 + required: - compatible - reg - reg-names +dependentRequired: + operating-points-v2: ['#interconnect-cells'] + '#interconnect-cells': [operating-points-v2] + +allOf: + - if: + properties: + compatible: + const: apple,t6020-pmgr-misc + then: + properties: + operating-points-v2: false + '#interconnect-cells': false + additionalProperties: false examples: From 5cfbbed27cc78eee808238c340f50337d2bf09aa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 21:55:31 +1100 Subject: [PATCH 1550/1579] soc: apple: add optional shared T600x DCS bandwidth voting Aggregate client requests through one interconnect path in the existing PMGR register owner. Validate explicit bandwidth OPPs, retain boot constraints, confirm state transitions, and roll back failed requests. Hold the fabric while bandwidth is requested and preserve the T600x suspend floor. Reject PMP ownership conflicts, oversubscribed requests and votes during sleep. Failed rollback prevents further votes and suspend until reboot. Add synthetic KUnit coverage and document qualification requirements. No board table is enabled: measured capacities and raw state mappings still require hardware qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/driver-api/apple-pmgr-dcs.rst | 55 +++++ Documentation/driver-api/index.rst | 1 + drivers/soc/apple/Kconfig | 11 + drivers/soc/apple/apple-pmgr-dcs-test.c | 220 +++++++++++++++++ drivers/soc/apple/apple-pmgr-misc.c | 248 +++++++++++++++++++- 5 files changed, 532 insertions(+), 3 deletions(-) create mode 100644 Documentation/driver-api/apple-pmgr-dcs.rst create mode 100644 drivers/soc/apple/apple-pmgr-dcs-test.c diff --git a/Documentation/driver-api/apple-pmgr-dcs.rst b/Documentation/driver-api/apple-pmgr-dcs.rst new file mode 100644 index 0000000000000..edf6fc4af52f5 --- /dev/null +++ b/Documentation/driver-api/apple-pmgr-dcs.rst @@ -0,0 +1,55 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +Apple T600x DCS bandwidth votes +============================== + +The PMGR miscellaneous driver can provide one shared interconnect path for +host-owned T600x DCS. Every client describes the path ``<&pmgr_misc 0>`` to +``<&pmgr_misc 1>`` and uses the usual interconnect consumer API. Average +requests sum with saturation; peak requests take the maximum. The larger +aggregate selects the lowest measured OPP that can satisfy it. Requests above +the largest OPP fail with ``-ERANGE`` instead of silently under-voting. + +The provider is enabled only by an explicit ``operating-points-v2`` phandle +and ``#interconnect-cells = <1>`` on the existing PMGR miscellaneous node. Its +table uses ``operating-points-v2-apple-dcs``. Each enabled OPP supplies a raw +``opp-level`` and one ``opp-peak-kBps`` value. The driver validates duplicate +states, zero or decreasing capacities, the T600x sleep floor of 7, and the +boot-state ceiling before publishing the provider. It does not derive states +from child order or convert firmware thresholds to bandwidth. Existing DTs +without this table retain their boot and suspend policy. + +The highest table entry must equal the boot desired state. That entry remains +the initial vote until interconnect sync_state releases boot constraints. +Nonzero bandwidth also holds the fabric at its boot state so untracked DMA +clients cannot lose their fabric bandwidth to the idle governor. Requests +change only the desired low nibble and wait for the actual field to +confirm the requested state. A timeout restores the previous confirmed vote. +If restoration also times out, the fabric hold remains asserted and subsequent requests and system sleep fail with +``-EIO`` until reboot. Errors propagate to consumers, which must not start work requiring a +bandwidth request that failed. Firmware ownership and system sleep reject +requests with ``-EBUSY``. Suspend keeps the existing T600x floor; resume +restores the last successful aggregate request. + +Enabling PMP and the host interconnect provider together is rejected. This +interface neither starts PMP nor reclaims a controller from running firmware. +No module parameter supplies bandwidth data or changes ownership. + +Qualification before adding a board table +---------------------------------------- + +No board table is supplied here. A DRAM transfer-rate estimate or a PMP +policy threshold does not establish sustainable client bandwidth. Before +adding one, establish the raw desired/actual state mapping for the exact SoC +and firmware, verify each selected state's fabric dependencies, and measure +sustained read, write and copy bandwidth under controlled CPU and DMA load. +Use a conservative capacity supported by repeated measurements, with the +workload, sample duration, thermal conditions and variance recorded in the +qualification report. Do not publish private target records in source. + +Qualify concurrent consumers, request release, failed raises and rollback, +suspend/resume and idle/load power through the hardware controller. Keep a +bootguard fallback and validate the default policy before enabling any board +DT. The KUnit suite uses synthetic numbers and memory-backed registers; it +validates software behavior only, not state encodings, rates, firmware +ownership or hardware completion. diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst index edb93a23542c1..ee2405d0b8e95 100644 --- a/Documentation/driver-api/index.rst +++ b/Documentation/driver-api/index.rst @@ -104,6 +104,7 @@ Subsystem-specific APIs infiniband input interconnect + apple-pmgr-dcs ipmb ipmi issei/index diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index c6e3bc9f81982..459e8a9ddb44f 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -51,6 +51,7 @@ config APPLE_MCC_KUNIT_TEST config APPLE_PMGR_MISC tristate "Apple SoC PMGR miscellaneous support" depends on PM + select INTERCONNECT help The PMGR block in Apple SoCs provides high-level power state controls for SoC devices. This driver manages miscellaneous @@ -58,6 +59,16 @@ config APPLE_PMGR_MISC Say 'y' here if you have a Pro/Max/Ultra Apple SoC. +config APPLE_PMGR_DCS_KUNIT_TEST + bool "KUnit tests for Apple PMGR DCS voting" if !KUNIT_ALL_TESTS + depends on KUNIT && APPLE_PMGR_MISC=y + default KUNIT_ALL_TESTS + help + Test bandwidth aggregation, explicit OPP selection, ownership gates, + register updates and timeout rollback with synthetic state data. + Also verify that boot constraints and concurrent endpoints retain + their votes, and that suspend restores the last confirmed request. + config APPLE_RTKIT tristate "Apple RTKit co-processor IPC protocol" depends on APPLE_MAILBOX diff --git a/drivers/soc/apple/apple-pmgr-dcs-test.c b/drivers/soc/apple/apple-pmgr-dcs-test.c new file mode 100644 index 0000000000000..f1021b6de4f61 --- /dev/null +++ b/drivers/soc/apple/apple-pmgr-dcs-test.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Synthetic OPPs and memory-backed registers; no hardware rate claims. */ +#include +#include + +static int apple_pmgr_dcs_test_init(struct kunit *test) +{ + struct apple_pmgr_misc *misc; + void *reg; + + misc = kunit_kzalloc(test, sizeof(*misc), GFP_KERNEL); + if (!misc) + return -ENOMEM; + reg = kunit_kzalloc(test, sizeof(u32), GFP_KERNEL); + if (!reg) + return -ENOMEM; + misc->dev = kunit_device_register(test, "pmgr-dcs-test"); + if (IS_ERR(misc->dev)) + return PTR_ERR(misc->dev); + mutex_init(&misc->lock); + misc->devices[DEV_DCS].base = (void __iomem *)reg; + misc->devices[DEV_DCS].active_state = 10; + misc->devices[DEV_DCS].suspend_state = DCS_DEV_PSTATE_MIN_T600X; + misc->dcs_num_opps = 3; + misc->dcs_opps[0] = (struct apple_pmgr_dcs_opp) { 7, 1000 }; + misc->dcs_opps[1] = (struct apple_pmgr_dcs_opp) { 9, 2000 }; + misc->dcs_opps[2] = (struct apple_pmgr_dcs_opp) { 10, 3000 }; + writel(0xa50000aa, misc->devices[DEV_DCS].base); + misc->dcs_data = kunit_kzalloc(test, struct_size(misc->dcs_data, nodes, 2), GFP_KERNEL); + if (!misc->dcs_data) + return -ENOMEM; + misc->dcs_data->num_nodes = 2; + for (unsigned int i = 0; i < 2; i++) { + misc->dcs_data->nodes[i] = kunit_kzalloc(test, sizeof(struct icc_node), GFP_KERNEL); + if (!misc->dcs_data->nodes[i]) + return -ENOMEM; + misc->dcs_data->nodes[i]->data = misc; + } + dev_set_drvdata(misc->dev, misc); + test->priv = misc; + return 0; +} + +static void apple_pmgr_dcs_table_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), 0); + swap(misc->dcs_opps[0], misc->dcs_opps[2]); + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), 0); + KUNIT_EXPECT_EQ(test, misc->dcs_opps[0].state, 7U); + misc->dcs_opps[1].state = 7; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), -EINVAL); + misc->dcs_opps[1].state = 9; + misc->dcs_opps[1].bw = 1000; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), -EINVAL); + misc->dcs_opps[1].bw = 2000; + misc->dcs_opps[0].bw = 0; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), -EINVAL); + misc->dcs_opps[0].bw = 1000; + misc->dcs_opps[0].state = 6; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), -EINVAL); + misc->dcs_opps[0].state = 7; + misc->devices[DEV_DCS].active_state = 9; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), -EINVAL); + misc->dcs_num_opps = 17; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), -EINVAL); + misc->dcs_num_opps = 1; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), -EINVAL); +} + +static void apple_pmgr_dcs_selection_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + u32 state = 0; + + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_select(misc, 0, &state), 0); + KUNIT_EXPECT_EQ(test, state, 7U); + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_select(misc, 1000, &state), 0); + KUNIT_EXPECT_EQ(test, state, 7U); + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_select(misc, 1001, &state), 0); + KUNIT_EXPECT_EQ(test, state, 9U); + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_select(misc, 3000, &state), 0); + KUNIT_EXPECT_EQ(test, state, 10U); + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_select(misc, 3001, &state), -ERANGE); + KUNIT_EXPECT_EQ(test, state, 10U); +} + +static void apple_pmgr_dcs_aggregation_test(struct kunit *test) +{ + u32 avg = 0, peak = 0; + + apple_pmgr_dcs_aggregate(NULL, 0, 800, 1000, &avg, &peak); + apple_pmgr_dcs_aggregate(NULL, 0, 900, 1200, &avg, &peak); + KUNIT_EXPECT_EQ(test, avg, 1700U); + KUNIT_EXPECT_EQ(test, peak, 1200U); + apple_pmgr_dcs_aggregate(NULL, 0, U32_MAX, 0, &avg, &peak); + KUNIT_EXPECT_EQ(test, avg, U32_MAX); +} + +static void apple_pmgr_dcs_commit_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + struct icc_node *node = misc->dcs_data->nodes[0]; + + node->avg_bw = 1001; + + writel(0xa500009a, misc->devices[DEV_DCS].base); + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_set(node, node), 0); + KUNIT_EXPECT_EQ(test, misc->devices[DEV_DCS].active_state, 9U); + KUNIT_EXPECT_TRUE(test, misc->dcs_bw_active); + KUNIT_EXPECT_EQ(test, readl(misc->devices[DEV_DCS].base), 0xa5000099U); +} + +static void apple_pmgr_dcs_owner_sleep_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + struct icc_node *node = misc->dcs_data->nodes[0]; + + node->peak_bw = 1000; + + misc->pmp_owned = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_set(node, node), -EBUSY); + misc->pmp_owned = false; + misc->sleeping = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_set(node, node), -EBUSY); + KUNIT_EXPECT_EQ(test, readl(misc->devices[DEV_DCS].base), 0xa50000aaU); +} + +static void apple_pmgr_dcs_rollback_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + struct icc_node *node = misc->dcs_data->nodes[0]; + + node->peak_bw = 1000; + + /* Actual stays at 10: request times out, rollback to 10 succeeds. */ + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_set(node, node), -ETIMEDOUT); + KUNIT_EXPECT_FALSE(test, misc->dcs_failed); + KUNIT_EXPECT_EQ(test, misc->devices[DEV_DCS].active_state, 10U); + KUNIT_EXPECT_EQ(test, readl(misc->devices[DEV_DCS].base), 0xa50000aaU); + /* An unknown actual state makes rollback fail too. */ + writel(0xa500000a, misc->devices[DEV_DCS].base); + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_set(node, node), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, misc->dcs_failed); + KUNIT_EXPECT_TRUE(test, misc->dcs_bw_active); + KUNIT_EXPECT_EQ(test, apple_pmgr_misc_prepare(misc->dev), -EIO); + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_set(node, node), -EIO); +} + +static void apple_pmgr_dcs_sync_constraints_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + struct icc_node *src = misc->dcs_data->nodes[0]; + struct icc_node *dst = misc->dcs_data->nodes[1]; + + src->avg_bw = 1001; + dst->init_peak = 3000; + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_set(src, src), 0); + KUNIT_EXPECT_EQ(test, misc->devices[DEV_DCS].active_state, 10U); + dst->init_peak = 0; + writel(0xa500009a, misc->devices[DEV_DCS].base); + /* Releasing destination's boot constraint must retain source demand. */ + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_set(dst, dst), 0); + KUNIT_EXPECT_EQ(test, misc->devices[DEV_DCS].active_state, 9U); +} + +static void apple_pmgr_dcs_suspend_restore_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + + dev_set_drvdata(misc->dev, misc); + misc->devices[DEV_DCS].active_state = 9; + KUNIT_ASSERT_EQ(test, apple_pmgr_misc_suspend_noirq(misc->dev), 0); + KUNIT_EXPECT_EQ(test, readl(misc->devices[DEV_DCS].base), 0xa50000a7U); + KUNIT_ASSERT_EQ(test, apple_pmgr_misc_resume_noirq(misc->dev), 0); + KUNIT_EXPECT_EQ(test, readl(misc->devices[DEV_DCS].base), 0xa50000a9U); +} + +static void apple_pmgr_dcs_no_table_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_init(misc), 0); + KUNIT_EXPECT_EQ(test, readl(misc->devices[DEV_DCS].base), 0xa50000aaU); +} + +static void apple_pmgr_dcs_release_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + struct icc_node *node = misc->dcs_data->nodes[0]; + + misc->dcs_bw_active = true; + writel(0xa500007a, misc->devices[DEV_DCS].base); + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_set(node, node), 0); + KUNIT_EXPECT_EQ(test, misc->devices[DEV_DCS].active_state, 7U); + KUNIT_EXPECT_FALSE(test, misc->dcs_bw_active); +} + +static struct kunit_case apple_pmgr_dcs_test_cases[] = { + KUNIT_CASE(apple_pmgr_dcs_table_test), + KUNIT_CASE(apple_pmgr_dcs_selection_test), + KUNIT_CASE(apple_pmgr_dcs_aggregation_test), + KUNIT_CASE(apple_pmgr_dcs_commit_test), + KUNIT_CASE(apple_pmgr_dcs_owner_sleep_test), + KUNIT_CASE(apple_pmgr_dcs_rollback_test), + KUNIT_CASE(apple_pmgr_dcs_sync_constraints_test), + KUNIT_CASE(apple_pmgr_dcs_suspend_restore_test), + KUNIT_CASE(apple_pmgr_dcs_no_table_test), + KUNIT_CASE(apple_pmgr_dcs_release_test), + {} +}; + +static struct kunit_suite apple_pmgr_dcs_test_suite = { + .name = "apple-pmgr-dcs", + .init = apple_pmgr_dcs_test_init, + .test_cases = apple_pmgr_dcs_test_cases, +}; + +kunit_test_suite(apple_pmgr_dcs_test_suite); diff --git a/drivers/soc/apple/apple-pmgr-misc.c b/drivers/soc/apple/apple-pmgr-misc.c index 0372e44ed3446..d18f4ed07d8bb 100644 --- a/drivers/soc/apple/apple-pmgr-misc.c +++ b/drivers/soc/apple/apple-pmgr-misc.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #define APPLE_CLKGEN_PSTATE 0 @@ -59,6 +61,11 @@ struct apple_pmgr_misc_hw { u32 dev_min_ps[DEV_MAX]; }; +struct apple_pmgr_dcs_opp { + u32 state; + u32 bw; +}; + struct apple_pmgr_misc { struct device *dev; struct apple_pmgr_sys_device devices[DEV_MAX]; @@ -74,6 +81,12 @@ struct apple_pmgr_misc { bool pmp_owned; /* PMP is described: no governor, holds don't raise */ bool pmp_sleep; /* PMP runs: this sleep leaves DCS to it (see prepare) */ u32 pmp_fabric; /* PMP's desired fabric p-state, saved across suspend */ + struct icc_provider dcs_provider; + struct icc_onecell_data *dcs_data; + struct apple_pmgr_dcs_opp dcs_opps[16]; + unsigned int dcs_num_opps; + bool dcs_failed; + bool dcs_bw_active; struct { struct regmap *regmap; u32 reg; @@ -83,6 +96,221 @@ struct apple_pmgr_misc { static struct apple_pmgr_misc *apple_pmgr_misc_inst; static unsigned int idle_fabric_pstate; +static int apple_pmgr_raise_fabric(struct apple_pmgr_misc *misc); + +/* Decimal kB/s throughout, as in the OPP and interconnect bindings. */ +static int apple_pmgr_dcs_aggregate(struct icc_node *node, u32 tag, u32 avg, + u32 peak, u32 *agg_avg, u32 *agg_peak) +{ + /* Saturate rather than wrap a large group of consumers to a low vote. */ + *agg_avg = min_t(u64, (u64)*agg_avg + avg, U32_MAX); + *agg_peak = max(*agg_peak, peak); + return 0; +} + +static int apple_pmgr_dcs_select(struct apple_pmgr_misc *misc, u32 bw, u32 *state) +{ + unsigned int i; + + for (i = 0; i < misc->dcs_num_opps; i++) { + if (bw <= misc->dcs_opps[i].bw) { + *state = misc->dcs_opps[i].state; + return 0; + } + } + return -ERANGE; +} + +static int apple_pmgr_dcs_write(struct apple_pmgr_misc *misc, u32 state) +{ + void __iomem *reg = misc->devices[DEV_DCS].base + APPLE_CLKGEN_PSTATE; + u32 val = readl(reg); + + FIELD_MODIFY(APPLE_CLKGEN_PSTATE_DESIRED, &val, state); + writel(val, reg); + return readl_poll_timeout(reg, val, + FIELD_GET(APPLE_CLKGEN_PSTATE_ACTUAL, val) == state, + 10, APPLE_PMGR_RAISE_TIMEOUT_US); +} + +static int apple_pmgr_dcs_set(struct icc_node *src, struct icc_node *dst) +{ + struct apple_pmgr_misc *misc = src->data; + u32 state, old, bw = 0; + unsigned int i; + int ret; + + guard(mutex)(&misc->lock); + if (misc->pmp_owned || misc->sleeping) + return -EBUSY; + if (misc->dcs_failed) + return -EIO; + /* sync_state invokes set(node, node); keep both endpoints' constraints. */ + for (i = 0; i < misc->dcs_data->num_nodes; i++) { + struct icc_node *node = misc->dcs_data->nodes[i]; + + if (!node) + continue; + bw = max(bw, max(node->avg_bw, node->peak_bw)); + bw = max(bw, max(node->init_avg, node->init_peak)); + } + ret = apple_pmgr_dcs_select(misc, bw, &state); + if (ret) + return ret; + /* Untracked DMA clients need the fabric as well as DCS bandwidth. */ + if (bw) { + ret = apple_pmgr_raise_fabric(misc); + if (ret) + return ret; + } + old = misc->devices[DEV_DCS].active_state; + ret = apple_pmgr_dcs_write(misc, state); + if (ret) { + /* A failed request must not replace the last confirmed vote. */ + if (apple_pmgr_dcs_write(misc, old)) { + misc->dcs_failed = true; + misc->dcs_bw_active = true; + dev_err(misc->dev, "DCS vote and rollback timed out; refusing further votes\n"); + } + return ret; + } + misc->devices[DEV_DCS].active_state = state; + misc->dcs_bw_active = bw != 0; + return 0; +} + +static int apple_pmgr_dcs_get_bw(struct icc_node *node, u32 *avg, u32 *peak) +{ + struct apple_pmgr_misc *misc = node->data; + + /* Keep the boot state until all described consumers have probed. */ + *avg = 0; + *peak = misc->dcs_opps[misc->dcs_num_opps - 1].bw; + return 0; +} + +static int apple_pmgr_dcs_opp_cmp(const void *a, const void *b) +{ + const struct apple_pmgr_dcs_opp *oa = a, *ob = b; + + return (oa->state > ob->state) - (oa->state < ob->state); +} + +static int apple_pmgr_dcs_validate(struct apple_pmgr_misc *misc) +{ + unsigned int i; + + if (misc->dcs_num_opps < 2 || misc->dcs_num_opps > ARRAY_SIZE(misc->dcs_opps)) + return -EINVAL; + sort(misc->dcs_opps, misc->dcs_num_opps, sizeof(misc->dcs_opps[0]), + apple_pmgr_dcs_opp_cmp, NULL); + for (i = 0; i < misc->dcs_num_opps; i++) { + const struct apple_pmgr_dcs_opp *opp = &misc->dcs_opps[i]; + + if (!opp->bw || opp->state < DCS_DEV_PSTATE_MIN_T600X || + opp->state > FIELD_MAX(APPLE_CLKGEN_PSTATE_DESIRED)) + return -EINVAL; + if (i && (opp->state <= misc->dcs_opps[i - 1].state || + opp->bw <= misc->dcs_opps[i - 1].bw)) + return -EINVAL; + } + /* Never request more than the boot firmware's known working state. */ + if (misc->dcs_opps[misc->dcs_num_opps - 1].state != + misc->devices[DEV_DCS].active_state) + return -EINVAL; + return 0; +} + +static void apple_pmgr_dcs_cleanup(void *data) +{ + struct apple_pmgr_misc *misc = data; + + icc_provider_deregister(&misc->dcs_provider); + icc_nodes_remove(&misc->dcs_provider); +} + +static void apple_pmgr_misc_sync_state(struct device *dev) +{ + struct apple_pmgr_misc *misc = dev_get_drvdata(dev); + + if (misc->dcs_data) + icc_sync_state(dev); +} + +static int apple_pmgr_dcs_init(struct apple_pmgr_misc *misc) +{ + struct device *dev = misc->dev; + struct device_node *table __free(device_node) = NULL; + struct icc_provider *provider = &misc->dcs_provider; + struct icc_node *node; + u32 cells; + unsigned int i; + int ret; + + if (!of_property_present(dev->of_node, "operating-points-v2")) + return of_property_present(dev->of_node, "#interconnect-cells") ? -EINVAL : 0; + /* This interface is for the direct host owner, never a PMP fallback. */ + if (misc->pmp_owned || !of_device_is_compatible(dev->of_node, "apple,t6000-pmgr-misc")) + return -EBUSY; + if (of_property_count_u32_elems(dev->of_node, "operating-points-v2") != 1 || + of_property_read_u32(dev->of_node, "#interconnect-cells", &cells) || cells != 1) + return -EINVAL; + table = of_parse_phandle(dev->of_node, "operating-points-v2", 0); + if (!table || !of_device_is_available(table) || + !of_device_is_compatible(table, "operating-points-v2-apple-dcs")) + return -EINVAL; + for_each_available_child_of_node_scoped(table, child) { + struct apple_pmgr_dcs_opp *opp; + + if (misc->dcs_num_opps == ARRAY_SIZE(misc->dcs_opps)) + return -E2BIG; + opp = &misc->dcs_opps[misc->dcs_num_opps++]; + if (of_property_present(child, "opp-supported-hw") || + of_property_present(child, "turbo-mode") || + of_property_count_u32_elems(child, "opp-level") != 1 || + of_property_count_u32_elems(child, "opp-peak-kBps") != 1 || + of_property_read_u32(child, "opp-level", &opp->state) || + of_property_read_u32(child, "opp-peak-kBps", &opp->bw)) + return -EINVAL; + } + ret = apple_pmgr_dcs_validate(misc); + if (ret) + return dev_err_probe(dev, ret, "invalid measured DCS OPP table\n"); + misc->dcs_data = devm_kzalloc(dev, struct_size(misc->dcs_data, nodes, 2), GFP_KERNEL); + if (!misc->dcs_data) + return -ENOMEM; + misc->dcs_data->num_nodes = 2; + provider->dev = dev; + provider->set = apple_pmgr_dcs_set; + provider->aggregate = apple_pmgr_dcs_aggregate; + provider->get_bw = apple_pmgr_dcs_get_bw; + provider->xlate = of_icc_xlate_onecell; + provider->data = misc->dcs_data; + icc_provider_init(provider); + for (i = 0; i < 2; i++) { + node = icc_node_create_dyn(); + if (IS_ERR(node)) { + ret = PTR_ERR(node); + goto remove_nodes; + } + node->name = i ? "t600x-dcs-memory" : "t600x-dcs-clients"; + node->data = misc; + misc->dcs_data->nodes[i] = node; + icc_node_add(node, provider); + } + ret = icc_link_create(misc->dcs_data->nodes[0], misc->dcs_data->nodes[1]->id); + if (ret) + goto remove_nodes; + ret = icc_provider_register(provider); + if (ret) + goto remove_nodes; + return devm_add_action_or_reset(dev, apple_pmgr_dcs_cleanup, misc); + +remove_nodes: + icc_nodes_remove(provider); + return ret; +} + static void apple_pmgr_sys_dev_set_pstate(struct apple_pmgr_misc *misc, enum sys_device dev, bool active) { @@ -259,7 +487,7 @@ static void apple_pmgr_idle_eval(struct apple_pmgr_misc *misc) if (misc->sleeping) return; low = idle && idle < active && misc->voters_ok && !misc->pmp_owned && - !misc->holds && !apple_pmgr_voters_active(misc) && + !misc->holds && !misc->dcs_bw_active && !apple_pmgr_voters_active(misc) && apple_pmgr_cpus_at_min(); if (low && misc->fabric_now != idle) apple_pmgr_write_fabric(misc, idle); @@ -365,13 +593,19 @@ static int __maybe_unused apple_pmgr_misc_prepare(struct device *dev) { struct apple_pmgr_misc *misc = dev_get_drvdata(dev); + /* Serialize the failure check and sleep gate with in-flight votes. */ + scoped_guard(mutex, &misc->lock) { + /* Failed rollback leaves no confirmed DCS state to restore. */ + if (misc->dcs_failed) + return -EIO; + if (misc->devices[DEV_FABRIC].base) + misc->sleeping = true; + } misc->pmp_sleep = misc->pmp_owned && apple_pmgr_pmp_running(); if (misc->pmp_owned && !misc->pmp_sleep) dev_warn_once(dev, "PMP is described but not running: sleeping without it\n"); if (!misc->devices[DEV_FABRIC].base) return 0; - scoped_guard(mutex, &misc->lock) - misc->sleeping = true; apple_pmgr_idle_stop(misc); return 0; } @@ -511,6 +745,9 @@ static int apple_pmgr_misc_probe(struct platform_device *pdev) INIT_DEFERRABLE_WORK(&misc->idle_work, apple_pmgr_idle_work); INIT_DELAYED_WORK(&misc->idle_work_prompt, apple_pmgr_idle_work_prompt); apple_pmgr_find_voters(misc); + ret = apple_pmgr_dcs_init(misc); + if (ret) + return ret; /* Publish only a set-up instance; pairs with the readers' acquire. */ smp_store_release(&apple_pmgr_misc_inst, misc); if (READ_ONCE(idle_fabric_pstate) && misc->pmp_owned) { @@ -561,6 +798,7 @@ static struct platform_driver apple_pmgr_misc_driver = { .of_match_table = apple_pmgr_misc_of_match, /* The governor's workers and holders would outlive an unbind. */ .suppress_bind_attrs = true, + .sync_state = apple_pmgr_misc_sync_state, .pm = pm_ptr(&apple_pmgr_misc_pm_ops), }, }; @@ -570,3 +808,7 @@ MODULE_DESCRIPTION("PMGR misc driver for Apple SoCs"); MODULE_LICENSE("GPL"); module_platform_driver(apple_pmgr_misc_driver); + +#if IS_ENABLED(CONFIG_APPLE_PMGR_DCS_KUNIT_TEST) +#include "apple-pmgr-dcs-test.c" +#endif From 55a1d93b5f5f4923a51d9924bbf3f7e3003e5ee7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 09:17:29 +1100 Subject: [PATCH 1551/1579] Documentation: media: apple-jpeg: cite the aurora-silicon m1n1 map Point the HWREV provenance at the same m1n1 commit in the aurora-silicon tree. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/admin-guide/media/apple-jpeg.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/media/apple-jpeg.rst b/Documentation/admin-guide/media/apple-jpeg.rst index 27ea6bcf2801e..35cca8b11d9de 100644 --- a/Documentation/admin-guide/media/apple-jpeg.rst +++ b/Documentation/admin-guide/media/apple-jpeg.rst @@ -44,10 +44,10 @@ shared power-domain accounting, and return DMA buffers only after quiescing hardware. Encode/decode controls, JPEG header validation, media-controller topology and actual job programming belong to the subsequent codec work. -The HWREV register offset comes from the MIT-licensed Asahi Linux m1n1 register +The HWREV register offset comes from the MIT-licensed m1n1 register description at commit ``f3ac615eb1376d3bbba24cd84c414b9e41892adb``: -https://github.com/AsahiLinux/m1n1/blob/f3ac615eb1376d3bbba24cd84c414b9e41892adb/proxyclient/m1n1/hw/jpeg.py +https://github.com/aurora-silicon/m1n1/blob/f3ac615eb1376d3bbba24cd84c414b9e41892adb/proxyclient/m1n1/hw/jpeg.py The scaffold deliberately does not translate the experiment's unqualified reset or interrupt writes. ``VIDEO_APPLE_JPEG_KUNIT_TEST`` tests empty target From 7467ff990b1bece498bf6e51122f647d53cd2e4c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 09:50:31 +1100 Subject: [PATCH 1552/1579] soc: apple: pmgr-dcs: require built-in KUnit for the built-in test The DCS test is compiled into the built-in PMGR misc driver. With KUNIT=m and KUNIT_ALL_TESTS=m the bool test symbol still became y and vmlinux failed to link against the modular KUnit core. Depend on KUNIT=y. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 459e8a9ddb44f..ba8b498786414 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -61,7 +61,7 @@ config APPLE_PMGR_MISC config APPLE_PMGR_DCS_KUNIT_TEST bool "KUnit tests for Apple PMGR DCS voting" if !KUNIT_ALL_TESTS - depends on KUNIT && APPLE_PMGR_MISC=y + depends on KUNIT=y && APPLE_PMGR_MISC=y default KUNIT_ALL_TESTS help Test bandwidth aggregation, explicit OPP selection, ownership gates, From 052e95927572b1eb141ef309b020baf8cfc8819b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 10:01:46 +1100 Subject: [PATCH 1553/1579] tools: asahi: teach the AVE sleep test the per-ABI device The per-ABI refactor gave struct ave_dev a firmware layout pointer and made the sleep callbacks call ave_fw_release_boot() after a clean halt of a firmware that halts through scratch memory, and ave_runtime_resume() on resume. The extracted callbacks no longer compiled against the test fixture, so run-host-tests stopped at 157 of 158. Model the layout, the boot-buffer release and the runtime resume in the fixture, count them, and add a case for a scratch-halt firmware: its boot buffers are released only after a halt that succeeded. The old-revision negative control still compiles and still fails its assertion. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/ave-pm-tests.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tools/asahi/ave-pm-tests.py b/tools/asahi/ave-pm-tests.py index 0a86cae413d28..9b9f7ea87fc10 100644 --- a/tools/asahi/ave-pm-tests.py +++ b/tools/asahi/ave-pm-tests.py @@ -29,10 +29,13 @@ def function(source, name): #include #include struct scheduler { bool stopped; unsigned pauses, restarts; }; -struct ave_dev { struct scheduler *m2m_dev; int hw_lock; }; +struct ave_transport { bool scratch_halt; }; +struct ave_abi { const struct ave_transport *transport; }; +struct ave_layout { const struct ave_abi *abi; }; +struct ave_dev { struct scheduler *m2m_dev; int hw_lock; const struct ave_layout *layout; }; struct device { struct ave_dev *data; }; static int shutdown_error; -static unsigned closes, halts; +static unsigned closes, halts, releases, wakes; static void *dev_get_drvdata(struct device *d) { return d->data; } static void mutex_lock(int *l) { assert(!*l); *l = 1; } static void mutex_unlock(int *l) { assert(*l == 1); *l = 0; } @@ -49,6 +52,11 @@ def function(source, name): assert(a->hw_lock && a->m2m_dev->stopped && closes > halts); halts++; return shutdown_error; } +/* A scratch-halt firmware (6070) gives its boot buffers back after a halt. */ +static __attribute__((unused)) void ave_fw_release_boot(struct ave_dev *a) { + assert(a->hw_lock && halts > releases); releases++; +} +static __attribute__((unused)) int ave_runtime_resume(struct device *d) { (void)d; wakes++; return 0; } #define dev_warn(...) ((void)0) #define dev_err(...) ((void)0) ''' @@ -78,7 +86,18 @@ def function(source, name): assert(ave_resume(&d) == 0); assert(!s.stopped && !a.hw_lock); assert(s.pauses == 6 && s.restarts == 6 && closes == 6 && halts == 6); - puts("AVE PM: normal/repeated sleep, halt refusal, error propagation, rollback and retry passed"); + assert(releases == 0 && wakes == 4); + /* A scratch-halt firmware releases its boot buffers only after a clean halt. */ + static const struct ave_transport scratch = { .scratch_halt = true }; + static const struct ave_abi abi = { .transport = &scratch }; + static const struct ave_layout layout = { .abi = &abi }; + a.layout = &layout; + shutdown_error = -EBUSY; + assert(ave_suspend(&d) == -EBUSY && releases == 0); + shutdown_error = 0; + assert(ave_suspend(&d) == 0 && releases == 1); + assert(ave_resume(&d) == 0 && wakes == 5); + puts("AVE PM: normal/repeated sleep, halt refusal, error propagation, rollback, retry and boot-buffer release passed"); } ''' From 1a91745fbb30b134ade23486a90ac9ff875ebee6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:28:13 +1100 Subject: [PATCH 1554/1579] arm64: dts: apple: t8140: give the video encoder a 4 GiB DMA window Every encoder buffer (session working buffers, both V4L2 queues and the IPC area) is mapped through the two AVE DARTs, and the DMA layer aligns each mapping to its power-of-two size. A 4K session needs about 180 MiB of working buffers alone, which does not fit the 256 MiB window. The ADT gives these DARTs a 0x100_0000_0000 + 0x300_0000_0000 VM range, and macOS keeps every encoder address in 0x100_xxxx_xxxx. Widen the window to 4 GiB above the firmware. Tested on the J700 at 1080p and 2560x1600 with no DART faults. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 450470726c3e6..3ac98eb58a120 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -982,7 +982,7 @@ interrupts = ; power-domains = <&ps_venc0_sys>; #iommu-cells = <1>; - apple,dma-range = <0x100 0x00004000 0 0x0fffc000>; + apple,dma-range = <0x100 0x00004000 0 0xffffc000>; status = "disabled"; }; @@ -993,7 +993,7 @@ interrupts = ; power-domains = <&ps_venc0_sys>; #iommu-cells = <1>; - apple,dma-range = <0x100 0x00004000 0 0x0fffc000>; + apple,dma-range = <0x100 0x00004000 0 0xffffc000>; status = "disabled"; }; From fc987199469b342500bf534979324ae591f67f37 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:28:13 +1100 Subject: [PATCH 1555/1579] media: apple: ave: encode up to 4096x2304 The 2560x1600 limit kept sessions inside the 256 MiB DMA window. With a 4 GiB window, allow up to 4096x2304, the largest size the encoder accepts for real-time H.264 on these chips, so 3840x2160 and 4096x2160 work. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/ave-v4l2.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index 8191a537de368..e51ede81df101 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -29,8 +29,8 @@ */ #define AVE_MIN_W 320 #define AVE_MIN_H 240 -#define AVE_MAX_W 2560 -#define AVE_MAX_H 1600 +#define AVE_MAX_W 4096 +#define AVE_MAX_H 2304 #define AVE_STEP 16 #define AVE_STRIDE_ALIGN 64 #define AVE_DEFAULT_W 1920 From ad1b36c19c5bf036b6d474cef572e94534a6673d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:28:13 +1100 Subject: [PATCH 1556/1579] media: apple: ave: give Process the macOS 46.4 s firmware timeout The 25G83 macOS driver gives Process commands a 46.4 second firmware timeout. Open, Start, Complete, Stop and Close use 94.4 seconds, and Config and Halt 3 seconds. The 9003.78.0 layout sent 94.4 seconds on Process too. Use the macOS value. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/ave-layout.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/ave/ave-layout.c b/drivers/media/platform/apple/ave/ave-layout.c index 273f9ce22401c..91c50ea083f70 100644 --- a/drivers/media/platform/apple/ave/ave-layout.c +++ b/drivers/media/platform/apple/ave/ave-layout.c @@ -9,7 +9,7 @@ const struct ave_fw_layout ave_layout_9003_78_0 = { .bytes = { { 0x3180, 0x63d8 }, { 0x13f28, 0xb1c0 } }, .hdr_bytes = 0x12c000, .meta_bytes = 0xc000, - .process_timeout_us = 94400000, + .process_timeout_us = 46400000, .process_subtype = 0x15, .max_slots = 5, .mcc_dsid = 0x12, From 676529d933c211e4c286cea010c385b8ff8f35a8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 01:31:39 +1100 Subject: [PATCH 1557/1579] media: apple: ave: size the HEVC neighbour buffer by picture width The HEVC row buffer at Start +0x1350 holds one 0x180-byte neighbour record per 64 pixels of picture width. The driver gave it a fixed 16 KiB, which is what a 1920 wide picture needs. At 3840 and 4096 the firmware needs 0x5a00 and 0x6000 bytes and wrote past the end: on the M4 Pro a 4K HEVC session hit DART write faults at the next page and the firmware stopped answering, and where the next page was mapped it silently overwrote another buffer (HEVC 4K sessions lost a frame on both the M4 Pro and the A18 Pro). Size it as macOS does: 0x180 bytes per 64 pixels, at least 16 KiB, rounded up to 16 KiB. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/ave-sess.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index bd4b287c90bb1..0a66170c50193 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -276,7 +276,10 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) if (fw->counted && p->rc_mode == 3) for (unsigned int i = 0; i < 2; i++) ave_req(r, &n, &sess->mb_extra[i], ALIGN(82 * p->mbs, SZ_16K), 0, nomap); - ave_req(r, &n, &sess->row_a, hevc ? SZ_16K : ALIGN(256 * w16, SZ_16K), 0, nomap); + /* HEVC: one 0x180-byte neighbour record per 64 pixels, at least 16 KiB. */ + ave_req(r, &n, &sess->row_a, + ALIGN(hevc ? max_t(u32, 0x180 * DIV_ROUND_UP(w, 64), SZ_16K) : 256 * w16, SZ_16K), + 0, nomap); ave_req(r, &n, &sess->row_b, ALIGN(hevc ? 768 * w32 : 1024 * w16, SZ_16K), 0, nomap); ave_req(r, &n, &sess->fixed, SZ_16K, 0, nomap); for (unsigned int i = 0; i < (hevc ? 4 : AVE_TILE_BUFS); i++) { From 6ff9d296cef7347033d2735e4eb8718603b99463 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 01:31:40 +1100 Subject: [PATCH 1558/1579] media: apple: ave: give the H.264 per-frame buffer its macOS size again The H.264 per-frame buffer at Process +0x58c8 was cut to one page to fit sessions in the old 256 MiB DMA window, on the grounds that no bit-rate session on hardware wrote to it. macOS sizes it from the macroblock count, 432 bytes per macroblock rounded up to 16 KiB (0x360000 at 1080p), and tells the firmware that size. A firmware that does write it, in a mode not yet exercised, would run past a single page. The window is now 4 GiB, so allocate the macOS size. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/ave-sess.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index 0a66170c50193..cf9867211f6af 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -48,9 +48,6 @@ #define AVE_SHDR_ENTRY_BYTES 0x400 #define AVE_CODED_MIN 0x3c000 -/* H.264 per-frame buffer (Process +0x58c8): unused in bit-rate modes */ -#define AVE_H264_FRAME_BUF SZ_16K - /* Level limits: max frame size, max rate, max bit rate (kbit/s, Main). */ struct ave_level { u32 max_fs; @@ -300,7 +297,7 @@ static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) ALIGN(432 * p->mbs, SZ_16K), 0, nomap); } else { ave_req(r, &n, &sess->slot[s].aux, - hevc ? AVE_SHDR_BYTES : AVE_H264_FRAME_BUF, 0, + hevc ? AVE_SHDR_BYTES : ALIGN(432 * p->mbs, SZ_16K), 0, hevc ? 0 : nomap); } } From cf66f705be52ef58ac41c202d0fdccb7cab1ff58 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 02:12:45 +1100 Subject: [PATCH 1559/1579] media: apple: ave: let the queues use the 4 GiB DMA window The OUTPUT and CAPTURE budgets (80 and 40 MiB) were sized for the old 256 MiB window. At 3840x2160 a raw frame maps as 16 MiB, so the OUTPUT queue was held to six buffers, and GStreamer's v4l2h265enc failed to set up its pool at 4K. With a 4 GiB window, allow 256 MiB for OUTPUT (16 frames at 4K) and 128 MiB for CAPTURE. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/ave-v4l2.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index e51ede81df101..bf16ad7da53cc 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -36,12 +36,13 @@ #define AVE_DEFAULT_W 1920 #define AVE_DEFAULT_H 1080 /* - * Buffer counts are limited so a full set fits the encoder's 256 MiB DART - * window next to a session: each queue may use this much, counting every - * buffer at the power-of-two size the DMA layer maps it with. + * Buffer counts are limited so the queues of a few concurrent sessions fit + * the encoder's 4 GiB DART window next to their session buffers: each queue + * may use this much, counting every buffer at the power-of-two size the DMA + * layer maps it with. That is 16 OUTPUT frames at 3840x2160. */ -#define AVE_OUTPUT_BUDGET (80 * SZ_1M) -#define AVE_CAPTURE_BUDGET (40 * SZ_1M) +#define AVE_OUTPUT_BUDGET (256 * SZ_1M) +#define AVE_CAPTURE_BUDGET (128 * SZ_1M) #define AVE_MAX_QUEUE_BUFS 16 #define AVE_RESTART_TRIES 10 #define AVE_RESTART_WAIT_MS 100 @@ -544,7 +545,7 @@ static int ave_queue_setup(struct vb2_queue *vq, unsigned int *nbufs, unsigned i /* Frames stay with the firmware until encoded; keep one spare. */ if (V4L2_TYPE_IS_OUTPUT(vq->type) && have + *nbufs < AVE_MAX_SLOTS + 1) *nbufs = AVE_MAX_SLOTS + 1 - have; - /* A full set of both queues must fit the encoder's 256 MiB DART window. */ + /* A full set of both queues must fit the budget for the DART window. */ if (have >= max) { dev_dbg(ctx->ave->dev, "%s queue already has its %u buffers\n", V4L2_TYPE_IS_OUTPUT(vq->type) ? "OUTPUT" : "CAPTURE", max); From bcd63a5094862c7bbfce8fc2d9cf4f7998d1c572 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:19:08 +0000 Subject: [PATCH 1560/1579] usb: dwc3: apple: select the tunable helper The Apple glue calls devm_apple_tunable_parse() and apple_tunable_apply() unconditionally, but does not select the helper that provides them. With arm64 defconfig, USB_DWC3_APPLE defaults to y while APPLE_TUNABLE is m through the Apple DART driver. The final vmlinux link fails with undefined references to both functions. Select APPLE_TUNABLE from USB_DWC3_APPLE, as the other tunable users do. This makes the helper built-in for built-in glue and keeps it available when the glue is a module. The glue's ARCH_APPLE dependency already satisfies the helper's architecture dependency. Fixes: 205c92374f87 ("usb: dwc3: apple: apply the controller tunables, reset the T6040 USB3 PIPE") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/usb/dwc3/Kconfig b/drivers/usb/dwc3/Kconfig index c862e55ae3a37..0d5c41f1a7d5d 100644 --- a/drivers/usb/dwc3/Kconfig +++ b/drivers/usb/dwc3/Kconfig @@ -226,6 +226,7 @@ config USB_DWC3_APPLE tristate "Apple Silicon DWC3 Platform Driver" depends on OF && ARCH_APPLE default USB_DWC3 + select APPLE_TUNABLE select USB_ROLE_SWITCH help Support Apple Silicon SoCs with DesignWare Core USB3 IP. From 51b28af92f7759e7203d0d4ccb708d4b078c0ca3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 10:19:28 +1100 Subject: [PATCH 1561/1579] media: apple: ave: keep HEVC formats within the level 5 picture size HEVC Main tier levels 5 to 5.2 allow at most 8912896 luma samples per picture, and the session refuses a larger one when streaming starts. With the 4096x2304 maximum, an HEVC client could set 4096x2304 and only learn at STREAMON that the session cannot be set up, with -EINVAL and an error in the kernel log. Apply the HEVC limit when the formats are negotiated, by reducing the height: 4096x2304 becomes 4096x2176, while 3840x2160 and 4096x2160 are unchanged. H.264 keeps 4096x2304. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/ave-v4l2.c | 23 ++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index bf16ad7da53cc..eb5be51120cd9 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -74,19 +74,28 @@ static bool ave_fmt_listed(const u32 *list, unsigned int n, u32 fourcc) return false; } -static void ave_clamp_size(u32 *w, u32 *h) +/* HEVC Main tier levels 5 to 5.2 allow at most this many luma samples. */ +#define AVE_HEVC_MAX_SAMPLES 8912896 + +/* @coded is the coded format the size is for: HEVC caps the picture area. */ +static void ave_clamp_size(u32 coded, u32 *w, u32 *h) { *w = clamp_t(u32, ALIGN(*w, 2), AVE_MIN_W, AVE_MAX_W); *h = clamp_t(u32, ALIGN(*h, 2), AVE_MIN_H, AVE_MAX_H); + if (coded == V4L2_PIX_FMT_HEVC && *w * *h > AVE_HEVC_MAX_SAMPLES) + *h = rounddown(AVE_HEVC_MAX_SAMPLES / *w, 2); } -/* Raw NV12, one buffer: Y plane, then interleaved CbCr at half height. */ -static void ave_fill_raw(struct v4l2_pix_format_mplane *f) +/* + * Raw NV12, one buffer: Y plane, then interleaved CbCr at half height. + * @coded is the context's coded format, which limits the size. + */ +static void ave_fill_raw(struct v4l2_pix_format_mplane *f, u32 coded) { u32 stride, height; f->pixelformat = V4L2_PIX_FMT_NV12; - ave_clamp_size(&f->width, &f->height); + ave_clamp_size(coded, &f->width, &f->height); /* Keep a client's stride, within reason, rounded to 64 bytes. */ stride = ALIGN(clamp_t(u32, f->plane_fmt[0].bytesperline, f->width, 2 * AVE_MAX_W), AVE_STRIDE_ALIGN); @@ -105,7 +114,7 @@ static void ave_fill_coded(struct v4l2_pix_format_mplane *f) if (!ave_fmt_listed(ave_coded_fmts, ARRAY_SIZE(ave_coded_fmts), f->pixelformat)) f->pixelformat = ave_coded_fmts[0]; - ave_clamp_size(&f->width, &f->height); + ave_clamp_size(f->pixelformat, &f->width, &f->height); /* The firmware's coded-data capacity plus parameter sets and headers. */ min = ALIGN(3 * ALIGN(f->width, 32) * ALIGN(f->height, 16) / 2, SZ_16K) + SZ_64K; f->num_planes = 1; @@ -129,7 +138,7 @@ static void ave_reset_fmts(struct ave_ctx *ctx) memset(src, 0, sizeof(*src)); src->width = dst->width; src->height = dst->height; - ave_fill_raw(src); + ave_fill_raw(src, dst->pixelformat); src->colorspace = V4L2_COLORSPACE_REC709; src->xfer_func = V4L2_XFER_FUNC_DEFAULT; src->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; @@ -204,7 +213,7 @@ static int ave_try_fmt_out(struct file *file, void *priv, struct v4l2_format *f) struct ave_ctx *ctx = ave_file_to_ctx(file); struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp; - ave_fill_raw(pix); + ave_fill_raw(pix, ctx->dst_fmt.pixelformat); if (pix->colorspace == V4L2_COLORSPACE_DEFAULT) { pix->colorspace = ctx->src_fmt.colorspace; pix->xfer_func = ctx->src_fmt.xfer_func; From 28174f88bdc408527712e9625392d19582ba16e1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 10:26:47 +1100 Subject: [PATCH 1562/1579] soc: apple: aop: Write the T6040 firmware's own nCal boot argument The AOP core writes nCal 0 into the firmware's boot arguments on every SoC whose arguments it patches. The T6040 firmware's own default for it, in the boot argument table it ships with, is 1. With 0, its accelerometer and gyroscope services accept the calibration and the report interval, and the sensor starts sampling, but no report ever reaches the application processor, so both IIO devices read ENODATA. Make nCal part of the per-SoC data and write the firmware's default, 1, on the T6040. The other SoCs keep 0. The T6040 comment claimed macOS writes the same arguments as on the T6020; that is not established for nCal, so drop it. Note the symptom in the AOP sensor documentation. On J616s (7.3.0-aurora-tt3), both devices report with this change. The accelerometer at rest reads |a| = 9.64 m/s^2 (291 samples, 9.62 to 9.65), and the gyroscope shows a static bias of about 0.4 deg/s. ALS, lid angle, the three DCPs and the audio cards are unchanged, and the boot logs no errors. Fixes: e82a608ad0b0 ("soc: apple: aop: Add T6040 support") Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/iio/aop-sensors.rst | 5 +++++ drivers/soc/apple/aop.rs | 19 ++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Documentation/iio/aop-sensors.rst b/Documentation/iio/aop-sensors.rst index 0129bde3f7fa1..6c3225024a82a 100644 --- a/Documentation/iio/aop-sensors.rst +++ b/Documentation/iio/aop-sensors.rst @@ -55,6 +55,11 @@ Both devices have a buffer that takes X, Y and Z together, with an optional timestamp taken when the report arrives. The reports stop when the system goes to sleep and start again, with the calibration resent, when it wakes. +If both devices bind but every raw read returns ``ENODATA``, check the AOP +core's ``nCal`` boot argument for the SoC. The T6040 firmware accepts the +calibration and the report interval with ``nCal`` 0, but sends no reports; +its own default, which the core keeps, is 1. + Host regression tests require Python 3 and a Rust compiler:: diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 3d4356231e6d6..380504117e52c 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -2511,6 +2511,8 @@ struct AopDriver(Arc); struct AopHwConfig { ec0p: u64, + /// The `nCal` boot argument. + ncal: u64, alig: u64, aopt: u64, /// Complete the firmware's boot arguments before starting it. @@ -2534,6 +2536,7 @@ struct AopHwConfig { const HW_CFG_T8103: AopHwConfig = AopHwConfig { ec0p: 0x020000, + ncal: 0, aopt: 1, alig: 128, patch_bootargs: true, @@ -2546,6 +2549,7 @@ const HW_CFG_T8103: AopHwConfig = AopHwConfig { }; const HW_CFG_T8112: AopHwConfig = AopHwConfig { ec0p: 0x020000, + ncal: 0, aopt: 0, alig: 128, patch_bootargs: true, @@ -2558,6 +2562,7 @@ const HW_CFG_T8112: AopHwConfig = AopHwConfig { }; const HW_CFG_T6000: AopHwConfig = AopHwConfig { ec0p: 0x020000, + ncal: 0, aopt: 0, alig: 64, patch_bootargs: true, @@ -2570,6 +2575,7 @@ const HW_CFG_T6000: AopHwConfig = AopHwConfig { }; const HW_CFG_T6020: AopHwConfig = AopHwConfig { ec0p: 0x0100_00000000, + ncal: 0, aopt: 0, alig: 64, patch_bootargs: true, @@ -2580,12 +2586,14 @@ const HW_CFG_T6020: AopHwConfig = AopHwConfig { fw_regions: false, halt_on_failed_start: false, }; -/// T6040 (J616s): macOS writes the same four boot arguments as for the -/// T6020. The firmware's external code and log live in DRAM, mapped through -/// the reserved regions the bootloader describes; the RTKit firmware is shut -/// down on unbind. +/// T6040 (J616s): the firmware's external code and log live in DRAM, mapped +/// through the reserved regions the bootloader describes; the RTKit firmware +/// is shut down on unbind. `nCal` keeps the firmware's own default of 1: +/// with 0 the accelerometer and gyroscope services accept their calibration +/// and report interval but never send a report. const HW_CFG_T6040: AopHwConfig = AopHwConfig { ec0p: 0x0100_00000000, + ncal: 1, aopt: 0, alig: 64, patch_bootargs: true, @@ -2603,6 +2611,7 @@ const HW_CFG_T6040: AopHwConfig = AopHwConfig { /// are started if they will. const HW_CFG_T8140: AopHwConfig = AopHwConfig { ec0p: 0, + ncal: 0, aopt: 0, alig: 0, patch_bootargs: false, @@ -2666,7 +2675,7 @@ impl platform::Driver for AopDriver { &aop_mmio, &[ (from_fourcc(b"EC0p"), cfg.ec0p), - (from_fourcc(b"nCal"), 0x0), + (from_fourcc(b"nCal"), cfg.ncal), (from_fourcc(b"alig"), cfg.alig), (from_fourcc(b"AOPt"), cfg.aopt), ], From 5a56a3e574c9262b59e57cd5cf05592ee5e07beb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 10:26:47 +1100 Subject: [PATCH 1563/1579] iio: aop: motion: Start the reports when the calibration is already held The T6040 firmware takes a service's calibration once per AOP boot. When it is sent again, after a rebind or on resume, the firmware answers the range-intrinsic and nominal-extrinsic writes with kIOReturnUnsupported (0xe00002c7), accepts the range-extrinsic one and keeps the calibration it holds. The driver treated that status as a failure. A rebind then failed with EIO, and after resume the reports were not restarted. Treat that status on a calibration write as "already held", log how many packets were kept, and go on to the report interval. Any other non-zero status still stops the start. Cover the status decision in the host selftest. This has been measured on the T6040 only. On J616s, rebinding the stock driver fails with "property 75 refused: status 0xe00002c7" on both devices. With this change both devices bind again and report fresh samples. Fixes: d2da58eb27a7 ("iio: aop: Add the accelerometer and gyroscope driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iio/common/aop_sensors/aop_motion.rs | 73 +++++++++++++++---- .../selftests/drivers/apple/aop-motion.py | 13 +++- 2 files changed, 71 insertions(+), 15 deletions(-) diff --git a/drivers/iio/common/aop_sensors/aop_motion.rs b/drivers/iio/common/aop_sensors/aop_motion.rs index a3d1cd0eff6c7..a0f2715ee4e50 100644 --- a/drivers/iio/common/aop_sensors/aop_motion.rs +++ b/drivers/iio/common/aop_sensors/aop_motion.rs @@ -46,6 +46,30 @@ const CALIBRATION: [(u32, &CStr); 3] = [ /// at byte 6. The other bytes carry no sample. const REPORT_LEN: usize = 22; const XYZ_OFFSET: usize = 6; +/// The status a calibration write is answered with when the service already +/// holds a calibration from an earlier start in the same AOP boot, as after a +/// rebind or a resume. The firmware keeps that calibration, so the reports can +/// be started without it. +const STATUS_UNSUPPORTED: u32 = 0xe000_02c7; + +/// What the status of a calibration write means for the start of the reports. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CalStatus { + /// The firmware took the packet. + Taken, + /// The firmware holds a calibration from an earlier start. + Held, + /// Anything else: the reports are not started. + Refused, +} + +fn cal_status(status: u32) -> CalStatus { + match status { + 0 => CalStatus::Taken, + STATUS_UNSUPPORTED => CalStatus::Held, + _ => CalStatus::Refused, + } +} /// What one of the two services measures. struct Kind { @@ -146,30 +170,51 @@ impl Control { )?) } - fn set(&self, prop: u32, value: &[u8]) -> Result { + /// Writes a property and returns the firmware's status. + fn write(&self, prop: u32, value: &[u8]) -> Result { let mut msg = KVec::with_capacity(8 + value.len(), GFP_KERNEL)?; msg.extend_from_slice(&0u32.to_le_bytes(), GFP_KERNEL)?; msg.extend_from_slice(&prop.to_le_bytes(), GFP_KERNEL)?; msg.extend_from_slice(value, GFP_KERNEL)?; - let status = self - .aop - .epic_call(&self.service, SUBTYPE_SET_PROPERTY, &msg)?; - if status != 0 { - dev_err!( - self.dev, - "property {} refused: status {:#x}\n", - prop, - status - ); - return Err(EIO); + self.aop + .epic_call(&self.service, SUBTYPE_SET_PROPERTY, &msg) + } + + fn refused(&self, prop: u32, status: u32) -> Error { + dev_err!( + self.dev, + "property {} refused: status {:#x}\n", + prop, + status + ); + EIO + } + + fn set(&self, prop: u32, value: &[u8]) -> Result { + match self.write(prop, value)? { + 0 => Ok(()), + status => Err(self.refused(prop, status)), } - Ok(()) } /// Sends the calibration, then the interval that starts the reports. fn start(&self) -> Result { + let mut held = 0; for ((prop, _), packet) in CALIBRATION.iter().zip(self.packets.iter()) { - self.set(*prop, packet)?; + let status = self.write(*prop, packet)?; + match cal_status(status) { + CalStatus::Taken => {} + CalStatus::Held => held += 1, + CalStatus::Refused => return Err(self.refused(*prop, status)), + } + } + if held > 0 { + dev_info!( + self.dev, + "{} of {} calibration packets kept from an earlier start\n", + held, + CALIBRATION.len() + ); } self.set(PROP_INTERVAL, &INTERVAL_US.to_le_bytes()) } diff --git a/tools/testing/selftests/drivers/apple/aop-motion.py b/tools/testing/selftests/drivers/apple/aop-motion.py index 592474bf709d0..a66d26b77810a 100755 --- a/tools/testing/selftests/drivers/apple/aop-motion.py +++ b/tools/testing/selftests/drivers/apple/aop-motion.py @@ -29,7 +29,9 @@ def main(): motion = (root / "drivers/iio/common/aop_sensors/aop_motion.rs").read_text() parts = [const(motion, "REPORT_LEN"), const(motion, "XYZ_OFFSET"), block(motion, "fn decode("), block(motion, "fn le_u32("), - block(motion, "fn location0_packet("), r''' + block(motion, "fn location0_packet("), + const(motion, "STATUS_UNSUPPORTED"), + block(motion, "#[derive(Clone, Copy, Debug, PartialEq, Eq)]\nenum CalStatus"), block(motion, "fn cal_status("), r''' fn report(prefix: u8, xyz: [i32; 3], suffix: u8) -> Vec { let mut r = vec![prefix; 6]; for v in xyz { @@ -69,6 +71,15 @@ def main(): c } #[test] +fn calibration_status() { + assert_eq!(cal_status(0), CalStatus::Taken); + // Answered once the service holds a calibration from an earlier start. + assert_eq!(cal_status(0xe000_02c7), CalStatus::Held); + for status in [1, 0xe000_02bc, 0xe000_02c2, 0xe000_02e2, 0xe000_02c6, u32::MAX] { + assert_eq!(cal_status(status), CalStatus::Refused, "{status:#x}"); + } +} +#[test] fn selects_location_zero() { let a = [1u8; 44]; let b = [2u8; 40]; From 546270ac4095cd2a260163154b61d2cb1f2db24d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 10:28:55 +1100 Subject: [PATCH 1564/1579] arm64: cpuinfo: Report the firmware's processor name as model name Desktop tools name the processor from the "model name" line of /proc/cpuinfo: GNOME's About page, KDE's Info Center, inxi and others. x86 fills it from the CPU's brand string. arm64 CPUs have no brand string and the line is only printed for 32-bit tasks, so these tools show no processor at all on arm64 machines, Apple silicon Macs included. The firmware names the processor in the Version string of the SMBIOS Processor Information (type 4), which lscpu already shows as "BIOS Model name". When the firmware provides one, print it as the model name for 64-bit tasks too, e.g. "Apple M4 Pro". Machines without SMBIOS or without a processor version print nothing new, and 32-bit tasks keep the line they had. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/kernel/cpuinfo.c | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 45c63f3d75c53..32f88fa0c695d 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -24,6 +24,7 @@ #include #include #include +#include /* * In case the boot CPU is hotpluggable, we record its initial state and @@ -219,6 +220,41 @@ static const char *const compat_hwcap2_str[] = { }; #endif /* CONFIG_COMPAT */ +/* + * The processor's name as the firmware gives it, in the Version string of the + * SMBIOS Processor Information (type 4), e.g. "Apple M4 Pro". x86 reports the + * CPU's brand string as "model name"; arm64 CPUs have no such string, so the + * firmware's is the only source, and desktop tools (GNOME, KDE, inxi) read it + * from here. Empty when there is no SMBIOS or no processor version. + */ +static char dmi_cpu_model[64]; + +static void __init dmi_find_cpu_model(const struct dmi_header *dm, void *private) +{ + const u8 *d = (const u8 *)dm; + const char *s = (const char *)dm + dm->length; + u8 index; + + if (dm->type != DMI_ENTRY_PROCESSOR || dm->length < 0x11 || dmi_cpu_model[0]) + return; + index = d[0x10]; + if (!index) + return; + while (--index && *s) + s += strlen(s) + 1; + if (*s) { + strscpy(dmi_cpu_model, skip_spaces(s), sizeof(dmi_cpu_model)); + strim(dmi_cpu_model); + } +} + +static int __init cpuinfo_dmi_model_init(void) +{ + dmi_walk(dmi_find_cpu_model, NULL); + return 0; +} +late_initcall(cpuinfo_dmi_model_init); + static int c_show(struct seq_file *m, void *v) { int j; @@ -236,6 +272,8 @@ static int c_show(struct seq_file *m, void *v) if (compat) seq_printf(m, "model name\t: ARMv8 Processor rev %d (%s)\n", MIDR_REVISION(midr), COMPAT_ELF_PLATFORM); + else if (dmi_cpu_model[0]) + seq_printf(m, "model name\t: %s\n", dmi_cpu_model); seq_printf(m, "BogoMIPS\t: %lu.%02lu\n", loops_per_jiffy / (500000UL/HZ), From d6c1322d85e31b3880e90df305e9d96d8fb2ae58 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:28:01 +1100 Subject: [PATCH 1565/1579] media: apple: ave: encode on the M4 Pro The M4 Pro boots Linux from a 26.6.2 (25G83) stub, so iBoot preloads the 25G83 build of the AVE firmware for its chip: TEXT 0x14c000, CRC 0x68d5f9f6. That is the build the A18 Pro runs (0x69c54454), compiled for the M4 Pro, and its command parser takes the same Config, Open, Start and Process layout. The macOS 25G83 driver builds identical commands for both chips. What differs is the chip identity. The M4 Pro is AVE device 27 with device type 25 (the A18 Pro is 29 and 27). Its coprocessor uses the same registers and start sequence as the A18 Pro. Select the existing layout for this CRC, and give the T6041 match data the coprocessor registers, device ID, device type, the A18 Pro boot argument, timebase and VCBAR encoding, and the qualified boot flag. Other firmware builds are still refused, the macOS 27.0 one included (TEXT 0x154000), whose commands are laid out differently. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/ave-drv.c | 14 +++++++++++++- drivers/media/platform/apple/ave/ave-fw.c | 1 + drivers/media/platform/apple/ave/ave-layout.c | 6 +++++- drivers/media/platform/apple/ave/ave.h | 2 ++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c index 908ada4ce87a1..246d49d91b864 100644 --- a/drivers/media/platform/apple/ave/ave-drv.c +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -81,11 +81,23 @@ const struct ave_variant ave_t8112 = { .boot_qualified = true, }; -/* CRC, boot-config words and coprocessor start registers remain unqualified. */ +/* + * The M4 Pro (T6040 in the ADT, apple,t6041 in the DT) starts its encoder + * like the A18 Pro: same coprocessor registers and boot sequence, its own + * device ID and type. + */ static const struct ave_variant ave_t6041 = { + .regs = &ave_coproc_v40, .fw_text_size = 0x14c000, .fw_data_size = 0x12c000, + .boot_device_id = 27, + .boot_arg_value = AVE_COPROC_BOOT_ARG_VALUE, + .timebase_multiplier = 1, + .vcbar_mask = U64_MAX, + .vcbar_flags = AVE_COPROC_VCBAR_VALID, + .device_type = 25, .lfs_ref_scale = 2, + .boot_qualified = true, }; static const struct ave_variant ave_t8140 = { diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c index 705588b918ce7..4311e8874a6c1 100644 --- a/drivers/media/platform/apple/ave/ave-fw.c +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -36,6 +36,7 @@ static const u32 ave_known_text_crc[] = { 0x2ce7359e, /* AppleAVE2FW 9003.78.0 */ 0x69c54454, /* AppleAVE2FW 9003.78.0 (macOS 26.0.x) */ + AVE_FW_TEXT_CRC_T6040_25G83, /* AppleAVE2FW 9003.78.0 / T6040 */ 0x460c7a7c, /* AppleAVE2FW 6070.11.1 / T6000 */ 0x25bfd039, /* AppleAVE2FW 6070.11.1 / T8112 */ }; diff --git a/drivers/media/platform/apple/ave/ave-layout.c b/drivers/media/platform/apple/ave/ave-layout.c index 91c50ea083f70..30d4d07d3abbd 100644 --- a/drivers/media/platform/apple/ave/ave-layout.c +++ b/drivers/media/platform/apple/ave/ave-layout.c @@ -529,7 +529,11 @@ const struct ave_fw_layout ave_layout_6070_11_1 = { const struct ave_fw_layout *ave_layout_for_crc(u32 crc) { - if (crc == AVE_FW_TEXT_CRC_9003_78_0) + /* + * The M4 Pro gets the same 25G83 firmware build as the A18 Pro, built + * for its own chip, and speaks the same command layout. + */ + if (crc == AVE_FW_TEXT_CRC_9003_78_0 || crc == AVE_FW_TEXT_CRC_T6040_25G83) return &ave_layout_9003_78_0; if (crc == 0x460c7a7c || crc == 0x25bfd039) return &ave_layout_6070_11_1; diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h index 47a8de278011c..8358e7d26fe3e 100644 --- a/drivers/media/platform/apple/ave/ave.h +++ b/drivers/media/platform/apple/ave/ave.h @@ -127,6 +127,8 @@ enum ave_opcode { /* The firmware build whose command layouts the driver generates */ #define AVE_FW_TEXT_CRC_9003_78_0 0x69c54454 +/* The M4 Pro (T6040) build of the same 25G83 firmware. */ +#define AVE_FW_TEXT_CRC_T6040_25G83 0x68d5f9f6 struct ave_ctx; From e89b8449e53c887e450576e86644576e3ba8cca3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:28:01 +1100 Subject: [PATCH 1566/1579] arm64: dts: apple: t6040-j616s: enable the video encoder Enable the AVE encoder on the J616s together with its two DARTs and the six VENC power domains it sits in. The loader disables the node again when the firmware is not preloaded or cannot be reserved. Both AVE DARTs get a 4 GiB DMA window above the firmware at 0x100_0000_0000. The ADT gives these DARTs a 0x100_0000_0000 + 0x300_0000_0000 VM range, and macOS keeps every encoder address in 0x100_xxxx_xxxx. A 4K session needs about 180 MiB of encoder buffers before its frame queues, so the 256 MiB the A18 Pro DT uses is not enough. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 37 +++++++++++++++++++++++ arch/arm64/boot/dts/apple/t6040.dtsi | 4 +-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 760840b14e37a..25cded4aa83df 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -4512,6 +4512,43 @@ }; }; +/* The video encoder, its DARTs and its power domains. */ +&ps_venc0_sys { + status = "okay"; +}; + +&ps_venc0_dma { + status = "okay"; +}; + +&ps_venc0_pipe4 { + status = "okay"; +}; + +&ps_venc0_pipe5 { + status = "okay"; +}; + +&ps_venc0_me0 { + status = "okay"; +}; + +&ps_venc0_me1 { + status = "okay"; +}; + +&ave0_dart0 { + status = "okay"; +}; + +&ave0_dart1 { + status = "okay"; +}; + +&ave { + status = "okay"; +}; + #include "t6040-j616s-usb3.dtsi" /* External display chain, all disabled: the gate in drivers/soc/apple/display-gate.c enables it. */ diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 8377a06f610f2..a33ef2e0abf83 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -1885,7 +1885,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x2 0x83080000 0x0 0xc000>; #iommu-cells = <1>; - apple,dma-range = <0x100 0x0 0x300 0x0>; + apple,dma-range = <0x100 0x00004000 0x0 0xffffc000>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_venc0_sys>; @@ -1896,7 +1896,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x2 0x830a0000 0x0 0xc000>; #iommu-cells = <1>; - apple,dma-range = <0x100 0x0 0x300 0x0>; + apple,dma-range = <0x100 0x00004000 0x0 0xffffc000>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_venc0_sys>; From d030b92281c598c3f212b69a662cac334748b71a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 10:44:41 +1100 Subject: [PATCH 1567/1579] media: apple: ave: tests: follow the M4 Pro firmware and buffer sizes The host builder test extracts the driver sources into a userspace harness. Two recent changes stopped it compiling: the firmware CRC table now names AVE_FW_TEXT_CRC_T6040_25G83, which the harness did not copy from ave.h, and the HEVC neighbour buffer is sized with max_t(), which the shim did not provide. Copy the T6040 CRC define when ave.h has it (an older baseline tree does not) and add max_t() to the shim. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/tests/run.py | 5 +++++ drivers/media/platform/apple/ave/tests/shim.h | 1 + 2 files changed, 6 insertions(+) diff --git a/drivers/media/platform/apple/ave/tests/run.py b/drivers/media/platform/apple/ave/tests/run.py index 14456d5283191..dad2b55ab7161 100644 --- a/drivers/media/platform/apple/ave/tests/run.py +++ b/drivers/media/platform/apple/ave/tests/run.py @@ -39,6 +39,11 @@ def build(src, out, baseline): for name in ('AVE_MAX_SLOTS', 'AVE_MAX_REFS', 'AVE_TILE_BUFS', 'AVE_MAX_SESSIONS', 'AVE_FW_TEXT_CRC_9003_78_0'): code += re.search(r'^#define ' + name + r'\s+[^\n]+', header, re.M)[0] + '\n' + # Later firmware builds; an older baseline does not define them. + for name in ('AVE_FW_TEXT_CRC_T6040_25G83',): + m = re.search(r'^#define ' + name + r'\s+[^\n]+', header, re.M) + if m: + code += m[0] + '\n' for name in ('ave_codec', 'ave_buf', 'ave_fw_state', 'ave_image', 'ave_coproc_regs', 'ave_variant', 'ave_params', 'ave_slot', 'ave_session'): code += declaration(header, name) diff --git a/drivers/media/platform/apple/ave/tests/shim.h b/drivers/media/platform/apple/ave/tests/shim.h index 28763dfeb660f..5a0201ac80269 100644 --- a/drivers/media/platform/apple/ave/tests/shim.h +++ b/drivers/media/platform/apple/ave/tests/shim.h @@ -27,6 +27,7 @@ typedef uint64_t dma_addr_t; #define DIV_ROUND_UP(x, a) (((x) + (a) - 1) / (a)) #define ALIGN(x, a) (((x) + (a) - 1) & ~((a) - 1)) #define max(x, y) ((x) > (y) ? (x) : (y)) +#define max_t(t, x, y) ((t)(x) > (t)(y) ? (t)(x) : (t)(y)) #define GFP_KERNEL 0 #define DMA_ATTR_NO_KERNEL_MAPPING 1 static inline void *kcalloc(size_t n, size_t size, int flags) From dbe8f10725202572a6cca46eb19d9385c529b600 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 05:28:54 +1100 Subject: [PATCH 1568/1579] media: apple: ave: retain input DMA mappings through failed-session recovery A failed session returns its OUTPUT buffers before runtime recovery stops the coprocessor. vb2 can then release the input IOVAs on DQBUF or close, even though the firmware can still read them. Keeping the session scratch allocations alone does not protect these mappings. Wrap the OUTPUT DMA-contig memory operations in a reference-counted mapping. Pin it before publishing each Process command and release the pin after normal delivery or with the quarantined session after reset. Retain imported attachments, their dma-bufs and the device independently of the vb2 context. Let vb2 report queue errors and complete buffers at the existing point, without waiting for Halt or power loss. Reclamation uses the normal DMA API after the existing control-off/reset witness; strict system sleep and the recovery rate limit are unchanged. Also quarantine Start scratch before an uncertain command submission. Exercise the production memory operations and command handoff in the host recovery tests, rejecting mapping release before the stop/reset markers. Cover MMAP, imports, immediate close, failed submission, failed stop, retained power and the recovery limit, and document the Neo LEAD-RUN re-test. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/ave/Makefile | 2 +- drivers/media/platform/apple/ave/ave-cmd.c | 5 +- drivers/media/platform/apple/ave/ave-drv.c | 1 + drivers/media/platform/apple/ave/ave-sess.c | 2 + drivers/media/platform/apple/ave/ave-v4l2.c | 4 +- drivers/media/platform/apple/ave/ave-vb2.c | 180 ++++++++++++++++++ drivers/media/platform/apple/ave/ave.h | 7 + drivers/media/platform/apple/ave/tests/README | 54 ++++++ .../media/platform/apple/ave/tests/boundary.h | 5 +- .../apple/ave/tests/command-boundary.h | 44 +++++ .../apple/ave/tests/recovery-boundary.h | 22 ++- .../platform/apple/ave/tests/recovery-test.c | 142 +++++++++++++- .../platform/apple/ave/tests/run-recovery.py | 10 +- .../platform/apple/ave/tests/vb2-boundary.h | 146 ++++++++++++++ 14 files changed, 611 insertions(+), 13 deletions(-) create mode 100644 drivers/media/platform/apple/ave/ave-vb2.c create mode 100644 drivers/media/platform/apple/ave/tests/command-boundary.h create mode 100644 drivers/media/platform/apple/ave/tests/vb2-boundary.h diff --git a/drivers/media/platform/apple/ave/Makefile b/drivers/media/platform/apple/ave/Makefile index 1938276901b16..072642374e1d4 100644 --- a/drivers/media/platform/apple/ave/Makefile +++ b/drivers/media/platform/apple/ave/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_VIDEO_APPLE_AVE) += apple-ave.o -apple-ave-y := ave-drv.o ave-fw.o ave-ipc.o ave-cmd.o ave-writer.o ave-abi.o ave-layout.o ave-image.o ave-sess.o ave-v4l2.o +apple-ave-y := ave-drv.o ave-fw.o ave-ipc.o ave-cmd.o ave-writer.o ave-abi.o ave-layout.o ave-image.o ave-sess.o ave-v4l2.o ave-vb2.o obj-$(CONFIG_VIDEO_APPLE_AVE_KUNIT_TEST) += ave-sizes-test.o ave-recovery-test.o obj-$(CONFIG_VIDEO_APPLE_AVE_CMD_KUNIT_TEST) += ave-abi-test.o diff --git a/drivers/media/platform/apple/ave/ave-cmd.c b/drivers/media/platform/apple/ave/ave-cmd.c index 3a6533846f88b..a7fb9672a1490 100644 --- a/drivers/media/platform/apple/ave/ave-cmd.c +++ b/drivers/media/platform/apple/ave/ave-cmd.c @@ -158,11 +158,12 @@ int ave_cmd_start(struct ave_ctx *ctx) return ret; ave_debugfs_save(ave, p->codec, AVE_IMAGE_START, cmd, len); + /* A failed submission may already have exposed the Start buffers. */ + sess->handed = true; ret = ave_ipc_submit(ave, AVE_BUF_CMD, len, AVE_CMD_SESSION_TIMEOUT_US, true); if (ret) return ret; - sess->handed = true; return ave_ipc_settle(ave, AVE_START_WAIT_MS); } @@ -198,6 +199,8 @@ int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, if (!sess->frames) ave_debugfs_save(ave, p->codec, AVE_IMAGE_PROCESS, cmd, len); + /* Pin before publishing the command, including an uncertain send failure. */ + s->input_mem = ave_vb2_get(&src->vb2_buf); ret = ave_ipc_submit(ave, buf, len, fw->process_timeout_us, false); if (ret) return ret; diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c index 246d49d91b864..d96267c00bef3 100644 --- a/drivers/media/platform/apple/ave/ave-drv.c +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -404,3 +404,4 @@ module_platform_driver(ave_driver); MODULE_DESCRIPTION("Apple AVE video encoder"); MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("DMA_BUF"); diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c index cf9867211f6af..f46f8128667bf 100644 --- a/drivers/media/platform/apple/ave/ave-sess.c +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -346,6 +346,8 @@ static void ave_sess_free_bufs(struct ave_dev *ave, struct ave_session *sess) ave_free(ave, &sess->slot[s].aux); ave_free(ave, &sess->slot[s].extra); ave_free(ave, &sess->slot[s].bounce); + ave_vb2_put(sess->slot[s].input_mem); + sess->slot[s].input_mem = NULL; } } diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c index eb5be51120cd9..e01710ad07fe1 100644 --- a/drivers/media/platform/apple/ave/ave-v4l2.c +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -910,7 +910,7 @@ static int ave_queue_init(void *priv, struct vb2_queue *src, struct vb2_queue *d src->drv_priv = ctx; src->buf_struct_size = sizeof(struct v4l2_m2m_buffer); src->ops = &ave_qops; - src->mem_ops = &vb2_dma_contig_memops; + src->mem_ops = &ave_vb2_memops; src->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; src->lock = &ctx->ave->vdev_lock; src->dev = ctx->ave->dev; @@ -994,6 +994,8 @@ static bool ave_deliver(struct ave_ctx *ctx) dst->flags |= V4L2_BUF_FLAG_LAST; last = true; } + ave_vb2_put(s->input_mem); + s->input_mem = NULL; v4l2_m2m_buf_done(s->src, state); s->src = NULL; v4l2_m2m_buf_done(dst, state); diff --git a/drivers/media/platform/apple/ave/ave-vb2.c b/drivers/media/platform/apple/ave/ave-vb2.c new file mode 100644 index 0000000000000..ba0325c9c82dc --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-vb2.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Keep OUTPUT mappings alive independently of vb2 buffer completion. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include + +#include + +#include "ave.h" + +/* + * vb2 owns one reference, each submitted Process owns another. A failed + * session keeps its references in quarantine until the coprocessor and DART + * have lost power. In particular, DQBUF/REQBUFS/close may release vb2's + * reference without unmapping an imported attachment or freeing an MMAP IOVA. + */ +struct ave_vb2_mem { + refcount_t refs; + void *priv; + struct dma_buf *dbuf; + struct device *dev; + bool mapped; +}; + +struct ave_vb2_mem *ave_vb2_get(struct vb2_buffer *vb) +{ + struct ave_vb2_mem *mem = vb->planes[0].mem_priv; + + refcount_inc(&mem->refs); + return mem; +} + +void ave_vb2_put(void *buf_priv) +{ + struct ave_vb2_mem *mem = buf_priv; + + if (!mem || !refcount_dec_and_test(&mem->refs)) + return; + if (mem->dbuf) { + if (mem->mapped) + vb2_dma_contig_memops.unmap_dmabuf(mem->priv); + vb2_dma_contig_memops.detach_dmabuf(mem->priv); + dma_buf_put(mem->dbuf); + put_device(mem->dev); + } else { + vb2_dma_contig_memops.put(mem->priv); + } + kfree(mem); +} + +static void *ave_vb2_alloc(struct vb2_buffer *vb, struct device *dev, + unsigned long size) +{ + struct ave_vb2_mem *mem; + + mem = kzalloc_obj(*mem); + if (!mem) + return ERR_PTR(-ENOMEM); + mem->priv = vb2_dma_contig_memops.alloc(vb, dev, size); + if (IS_ERR(mem->priv)) { + void *err = mem->priv; + + kfree(mem); + return err; + } + refcount_set(&mem->refs, 1); + return mem; +} + +static void *ave_vb2_attach(struct vb2_buffer *vb, struct device *dev, + struct dma_buf *dbuf, unsigned long size) +{ + struct ave_vb2_mem *mem; + + mem = kzalloc_obj(*mem); + if (!mem) + return ERR_PTR(-ENOMEM); + mem->priv = vb2_dma_contig_memops.attach_dmabuf(vb, dev, dbuf, size); + if (IS_ERR(mem->priv)) { + void *err = mem->priv; + + kfree(mem); + return err; + } + /* vb2 can drop its dma_buf and device references before quarantine ends. */ + get_dma_buf(dbuf); + mem->dbuf = dbuf; + mem->dev = get_device(dev); + refcount_set(&mem->refs, 1); + return mem; +} + +static int ave_vb2_map(void *buf_priv) +{ + struct ave_vb2_mem *mem = buf_priv; + int ret; + + if (mem->mapped) + return 0; + ret = vb2_dma_contig_memops.map_dmabuf(mem->priv); + if (!ret) + mem->mapped = true; + return ret; +} + +static void ave_vb2_unmap(void *buf_priv) +{ + /* The final put unmaps, after both vb2 and firmware have let go. */ +} + +static void *ave_vb2_cookie(struct vb2_buffer *vb, void *buf_priv) +{ + struct ave_vb2_mem *mem = buf_priv; + + return vb2_dma_contig_memops.cookie(vb, mem->priv); +} + +static void *ave_vb2_vaddr(struct vb2_buffer *vb, void *buf_priv) +{ + struct ave_vb2_mem *mem = buf_priv; + + return vb2_dma_contig_memops.vaddr(vb, mem->priv); +} + +static int ave_vb2_mmap(void *buf_priv, struct vm_area_struct *vma) +{ + struct ave_vb2_mem *mem = buf_priv; + + return vb2_dma_contig_memops.mmap(mem->priv, vma); +} + +static struct dma_buf *ave_vb2_export(struct vb2_buffer *vb, void *buf_priv, + unsigned long flags) +{ + struct ave_vb2_mem *mem = buf_priv; + + return vb2_dma_contig_memops.get_dmabuf(vb, mem->priv, flags); +} + +static unsigned int ave_vb2_num_users(void *buf_priv) +{ + struct ave_vb2_mem *mem = buf_priv; + + return vb2_dma_contig_memops.num_users(mem->priv); +} + +static void ave_vb2_prepare(void *buf_priv) +{ + struct ave_vb2_mem *mem = buf_priv; + + vb2_dma_contig_memops.prepare(mem->priv); +} + +static void ave_vb2_finish(void *buf_priv) +{ + struct ave_vb2_mem *mem = buf_priv; + + vb2_dma_contig_memops.finish(mem->priv); +} + +/* OUTPUT supports MMAP and DMABUF; CAPTURE never goes to the firmware. */ +const struct vb2_mem_ops ave_vb2_memops = { + .alloc = ave_vb2_alloc, + .put = ave_vb2_put, + .get_dmabuf = ave_vb2_export, + .cookie = ave_vb2_cookie, + .vaddr = ave_vb2_vaddr, + .mmap = ave_vb2_mmap, + .prepare = ave_vb2_prepare, + .finish = ave_vb2_finish, + .map_dmabuf = ave_vb2_map, + .unmap_dmabuf = ave_vb2_unmap, + .attach_dmabuf = ave_vb2_attach, + .detach_dmabuf = ave_vb2_put, + .num_users = ave_vb2_num_users, +}; diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h index 8358e7d26fe3e..9f94eb1345290 100644 --- a/drivers/media/platform/apple/ave/ave.h +++ b/drivers/media/platform/apple/ave/ave.h @@ -28,6 +28,7 @@ struct debugfs_blob_wrapper; struct dentry; struct dev_pm_domain_list; struct iommu_domain; +struct ave_vb2_mem; /* Frames the firmware can hold at once per session (Process "slots"). */ #define AVE_MAX_SLOTS 7 @@ -256,6 +257,7 @@ struct ave_slot { struct ave_buf extra; /* counted layout: per-slot 16 KiB buffer */ struct ave_buf bounce; /* input copied to the encoder's stride */ struct vb2_v4l2_buffer *src; + struct ave_vb2_mem *input_mem; /* independent firmware mapping reference */ u32 status; bool busy; /* Process sent, reply pending */ bool done; /* reply in, output not delivered yet */ @@ -408,6 +410,11 @@ int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, int ave_cmd_complete(struct ave_ctx *ctx, unsigned int count); int ave_cmd_close(struct ave_ctx *ctx); +/* ave-vb2.c */ +extern const struct vb2_mem_ops ave_vb2_memops; +struct ave_vb2_mem *ave_vb2_get(struct vb2_buffer *vb); +void ave_vb2_put(void *buf_priv); + /* ave-sess.c */ struct ave_session *ave_sess_create(struct ave_ctx *ctx); void ave_sess_destroy(struct ave_ctx *ctx); diff --git a/drivers/media/platform/apple/ave/tests/README b/drivers/media/platform/apple/ave/tests/README index e85e4021d4d0d..ea59e9724e991 100644 --- a/drivers/media/platform/apple/ave/tests/README +++ b/drivers/media/platform/apple/ave/tests/README @@ -45,6 +45,14 @@ failure, suspend refusal/rollback and a busy IPC stream reaching its deadline. MMIO, DMA allocation and PM are simulated: this cannot establish hardware reset values or that a VENC domain actually powered down. +The recovery runner also uses the production OUTPUT memory operations and +Process submission callback. Both MMAP and imported dma-buf mappings survive +immediate vb2 release after ERROR, including loss of the importing fd and an +uncertain command submission. The fake DMA boundary rejects any release of a +command-referenced mapping before the control-off marker AND simulated shared +DART/VENC power loss. A failed stop and CTRL=0 with retained BOOT_ARG must not +reclaim it. Allocator failures and ordinary import/map/release are covered too. + The apple-ave-recovery KUnit suite tests the actual sliding-window helper at expiry boundaries, with staggered attempts and across jiffies wrap. Run with ../.kunitconfig using tools/testing/kunit/kunit.py. The existing @@ -90,3 +98,49 @@ J616s and Neo. Repeat the recovery with a genuine firmware timeout/DART fault when a safe reproducer is available. Keep captures and machine records outside the public source tree. Rollback is a normal revert of the recovery commits; use the lane's existing boot fallback when testing candidate kernels. + +LEAD-RUN: Neo failed-session mapping regression +---------------------------------------------- + +Run in the approved Neo hardware lane, using the hardware procedure above. +This is a re-test plan, not a claim of hardware qualification. Preserve the +normal candidate-kernel boot fallback and the known-good module/kernel for +rollback. Record the full source commit, kernel config, module hash/vermagic, +firmware CRCs and boot ID with the results. Build and install the matching +candidate through the lane's normal procedure; do not load a module built for +a different kernel. Keep machine journals and captures outside public source. + +1. Save a complete kernel log before the test and collect a continuous kernel + journal with monotonic timestamps throughout each scenario. Establish fresh + AVC and HEVC encode/decode success. Use the same queue exerciser as the + original hardware procedure to record injection time, first POLLERR/EIO and + each buffer's completion count/state. Include MMAP OUTPUT and imported + dma-buf OUTPUT runs, with immediate DQBUF, STREAMOFF, REQBUFS(0), close and + release of the importing fd after the error. This exercises lifetime beyond + the V4L2 context. Do not require the client to wait for the stop or reset. +2. Re-run hardware steps 2-4: inject during a long encode, require prompt queue + errors (compare with the prior 5-9 ms POLLERR measurements), and exactly one + ERROR completion per buffer still owned by the driver. Require the existing + recovery stop and power-cycle-confirmed messages and a valid new encode + without rebooting. Inspect the entire interval from injection through the + subsequent encode, not just the recovery's last messages. +3. Re-run step 5 with two clients, including the idle owner. Release both + contexts promptly after ERROR. Require both clients to fail promptly, + balanced PM ownership, a confirmed recovery and a valid fresh encode. +4. Re-run step 6 with a fresh recovery budget: three injections in ten minutes, + restarting and verifying an encode after each. The next failure must retain + queue errors and report the three-attempt limit without a new boot. After + the oldest attempt expires, require successful recovery and encoding. Use + the lane's normal fresh-boot procedure or allow the earlier attempts to + expire between scenarios; do not reset the counter by reloading the driver. +5. PASS requires zero apple-dart translation faults after EVERY injection, + including the limited attempt and its expiry/retry, with no new suppressed + fault messages. Preserve complete logs and per-injection windows, completion + counts, error latency and recovery results even on failure. Any new DART + fault, missing/duplicate completion, lost prompt error or unconfirmed reset + is a failed re-test. The existing strict sleep check in step 7 remains + required in its separate approved sleep lane. + +On failure, preserve the candidate logs and restore the known-good kernel or +module through the lane's rollback procedure. Keep the draft unmerged pending +this LEAD-RUN evidence. Software injection does not qualify a real DMA overrun. diff --git a/drivers/media/platform/apple/ave/tests/boundary.h b/drivers/media/platform/apple/ave/tests/boundary.h index 3598fbfe14576..fb8984dd5e079 100644 --- a/drivers/media/platform/apple/ave/tests/boundary.h +++ b/drivers/media/platform/apple/ave/tests/boundary.h @@ -17,9 +17,12 @@ struct ave_ctx { }; struct vb2_v4l2_buffer { - int unused; + struct vb2_buffer { int unused; } vb2_buf; }; +static struct ave_vb2_mem *ave_vb2_get(struct vb2_buffer *vb) { return NULL; } +static void ave_vb2_put(void *mem) { assert(!mem); } + static u8 wire[0x1c000 + 32]; static size_t wire_len; static unsigned int submissions; diff --git a/drivers/media/platform/apple/ave/tests/command-boundary.h b/drivers/media/platform/apple/ave/tests/command-boundary.h new file mode 100644 index 0000000000000..a70fc29fc3893 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/command-boundary.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Publishing a Process must already own the mapping it references. */ +static struct ave_ctx *submit_ctx; +static unsigned int submit_slot; +static int submit_result; +static u8 command[256]; +static void *ave_ipc_cmd(struct ave_dev *ave, unsigned int buf, size_t len) +{ + return command; +} +static int ave_ipc_wait_ack(struct ave_dev *ave, unsigned int buf) +{ + return 0; +} +static int ave_build_start(struct ave_ctx *ctx, void *cmd, size_t len) +{ + return 0; +} +static int ave_build_process(struct ave_ctx *ctx, void *cmd, size_t len, + unsigned int slot, dma_addr_t input, u32 bitrate, bool force_key) +{ + return 0; +} +static void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, + enum ave_image kind, const void *cmd, size_t len) +{ +} +static int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, + u64 timeout, bool reply) +{ + struct ave_vb2_mem *mem = submit_ctx->sess->slot[submit_slot].input_mem; + struct test_mapping *mapping; + + if (reply) { + assert(submit_ctx->sess->handed); + } else { + assert(mem && mem->refs == 2); + mapping = mem->priv; + mapping->firmware_visible = true; + } + if (submit_result) + ave_fw_mark_broken(ave, submit_result, "test submission"); + return submit_result; +} diff --git a/drivers/media/platform/apple/ave/tests/recovery-boundary.h b/drivers/media/platform/apple/ave/tests/recovery-boundary.h index 9299dd0067be0..49ab21f3398fb 100644 --- a/drivers/media/platform/apple/ave/tests/recovery-boundary.h +++ b/drivers/media/platform/apple/ave/tests/recovery-boundary.h @@ -36,7 +36,7 @@ static void mutex_unlock(struct mutex *m) #define spin_lock_irqsave(p, flags) do { (flags) = 0; mutex_lock(p); } while (0) #define spin_unlock_irqrestore(p, flags) do { (void)(flags); mutex_unlock(p); } while (0) struct ave_dev; -struct device { struct ave_dev *drvdata; unsigned int refs; }; +struct device { struct ave_dev *drvdata; unsigned int refs, dma_refs; }; static void *dev_get_drvdata(struct device *dev) { @@ -45,7 +45,11 @@ static void *dev_get_drvdata(struct device *dev) struct vb2_queue { unsigned int type; bool error, streaming; void *priv; }; -struct vb2_buffer { struct vb2_queue *vb2_queue; void *priv; }; +struct vb2_buffer { + struct vb2_queue *vb2_queue; + void *priv; + struct { void *mem_priv; } planes[1]; +}; struct vb2_v4l2_buffer { struct vb2_buffer vb2_buf; unsigned int completions; int state; }; @@ -87,6 +91,7 @@ struct ave_dev { u64 ipc_fw_base; }; +static void *test_ctrl_addr; static int halt_result, close_result; static unsigned int halts, closes, freed, suspends, resumes, schedules, barriers; static unsigned int polls, sleep_ms; @@ -118,8 +123,11 @@ static u64 readq(const void *p) static void writel(u32 v, void *p) { - if (!ctrl_sticky) + if (!ctrl_sticky) { *(u32 *)p = v; + if (p == test_ctrl_addr) + stop_seen = !v; + } } static void *memremap(phys_addr_t p, size_t size, unsigned int mode) @@ -193,9 +201,13 @@ static bool ave_sess_stalled(struct ave_session *sess) return false; } -static void ave_sess_free_bufs(struct ave_dev *ave, struct ave_session *sess) +static void ave_free(struct ave_dev *ave, struct ave_buf *buf) { - freed++; + if (buf->cpu) { + assert(stop_seen && power_seen); + freed++; + buf->cpu = NULL; + } } static void pm_runtime_mark_last_busy(struct device *dev) { } diff --git a/drivers/media/platform/apple/ave/tests/recovery-test.c b/drivers/media/platform/apple/ave/tests/recovery-test.c index 3bb1f8fa0223c..b39174ec9f2d2 100644 --- a/drivers/media/platform/apple/ave/tests/recovery-test.c +++ b/drivers/media/platform/apple/ave/tests/recovery-test.c @@ -5,7 +5,9 @@ static const struct ave_variant variant = { }; static const struct ave_fw_transport test_transport = { .boot_config = true }; static const struct ave_fw_abi test_abi = { .transport = &test_transport }; -static const struct ave_fw_layout test_layout = { .abi = &test_abi }; +static const struct ave_fw_layout test_layout = { + .abi = &test_abi, .encode_ready = true, .max_slots = 2, +}; static u64 coproc[4]; static u32 mailbox[16]; static struct ave_dev ave; @@ -24,6 +26,10 @@ static void init(void) halts = closes = freed = suspends = resumes = schedules = polls = sleep_ms = 0; halt_result = close_result = poll_result = 0; map_fail = ctrl_sticky = stopped_for_sleep = false; + stop_seen = power_seen = dma_alloc_fail = dma_attach_fail = dma_map_fail = false; + mapping_frees = mapping_unmaps = mapping_detaches = 0; + submit_result = 0; + test_ctrl_addr = (char *)coproc + regs.ctrl; ave.dev = &dev; dev.drvdata = &ave; ave.variant = &variant; @@ -48,6 +54,8 @@ static void owners(void) assert(ctx[i].sess); ctx[i].sess->handed = ctx[i].sess->fw_open = true; ctx[i].sess->p.slots = 2; + ctx[i].sess->p.codec = AVE_CODEC_H264; + ctx[i].sess->hdr.cpu = (void *)1; ave.sessions[i] = &ctx[i]; dev.refs++; } @@ -61,10 +69,24 @@ static void owners(void) static void buffer_failure(void) { struct vb2_v4l2_buffer input[2][3] = {}, output[2][2] = {}; + struct dma_buf imported = { .refs = 1 }; init(); owners(); for (unsigned int i = 0; i < 2; i++) { + struct vb2_buffer *vb = &input[i][0].vb2_buf; + + vb->planes[0].mem_priv = i ? + ave_vb2_memops.attach_dmabuf(vb, &dev, &imported, SZ_16K) : + ave_vb2_memops.alloc(vb, &dev, SZ_16K); + assert(!IS_ERR(vb->planes[0].mem_priv)); + if (i) + assert(!ave_vb2_memops.map_dmabuf(vb->planes[0].mem_priv)); + submit_ctx = &ctx[i]; + submit_slot = 0; + mutex_lock(&ave.hw_lock); + assert(!ave_cmd_process(&ctx[i], 0, &input[i][0], 0, 0, false)); + mutex_unlock(&ave.hw_lock); ctx[i].sess->slot[0].src = &input[i][0]; /* One idle owner and one with a firmware-owned frame. */ ctx[i].sess->busy = i; @@ -94,7 +116,19 @@ static void buffer_failure(void) for (unsigned int n = 0; n < 2; n++) assert(output[i][n].completions == 1 && output[i][n].state == VB2_BUF_STATE_ERROR); + /* DQBUF/REQBUFS/close immediately after ERROR: vb2 lets go. */ + if (i) { + ave_vb2_memops.unmap_dmabuf(input[i][0].vb2_buf.planes[0].mem_priv); + ave_vb2_memops.detach_dmabuf(input[i][0].vb2_buf.planes[0].mem_priv); + } else { + ave_vb2_memops.put(input[i][0].vb2_buf.planes[0].mem_priv); + } + input[i][0].vb2_buf.planes[0].mem_priv = NULL; + assert(!mapping_frees && !mapping_unmaps && !mapping_detaches); } + /* Even dropping the importing fd cannot destroy its retained attachment. */ + dma_buf_put(&imported); + assert(imported.refs == 1 && dev.dma_refs == 1); assert(!dev.refs && !freed && ave.failed_sessions && !closes); /* A QBUF racing the failure is returned instead of silently requeued. */ { @@ -113,9 +147,12 @@ static void buffer_failure(void) /* CTRL=0 alone is not proof of a power cycle. */ assert(ave_fw_prepare_restart(&ave) == -EAGAIN && !freed); *(u32 *)(ave.coproc + regs.boot_arg) = 0; + power_seen = true; /* VENC and its DART have now lost power. */ assert(!ave_fw_prepare_restart(&ave)); mutex_unlock(&ave.hw_lock); assert(freed == 2 && !ave.failed_sessions && !ave.inject_wedge); + assert(stop_seen && mapping_frees == 2 && mapping_unmaps == 1 && mapping_detaches == 1); + assert(!imported.refs && !dev.dma_refs); assert(ave.fw_state == AVE_FW_HALTED); m2m[0].dst.streaming = true; ave_stop_streaming(&m2m[0].src); @@ -193,6 +230,7 @@ static void busy_ring_deadline(void) ave_poll_work(&ctx[0].poll_work.work); ave_poll_work(&ctx[1].poll_work.work); mutex_lock(&ave.hw_lock); + stop_seen = power_seen = true; ave_sess_reap(&ave); /* Fake reset for test cleanup only. */ mutex_unlock(&ave.hw_lock); } @@ -234,8 +272,108 @@ static void strict_6070_recovery(void) assert(readl(ave.coproc + regs.ctrl) == AVE_COPROC_CTRL_RUN); } +/* An uncertain send failure must keep its pin even without a slot src. */ +static void failed_submission(void) +{ + struct vb2_v4l2_buffer input = {}; + void *mem; + + init(); + owners(); + mem = ave_vb2_memops.alloc(&input.vb2_buf, &dev, SZ_16K); + assert(!IS_ERR(mem)); + input.vb2_buf.planes[0].mem_priv = mem; + submit_ctx = &ctx[0]; + submit_slot = 0; + submit_result = -EIO; + mutex_lock(&ave.hw_lock); + assert(ave_cmd_process(&ctx[0], 0, &input, 0, 0, false) == -EIO); + mutex_unlock(&ave.hw_lock); + assert(!ctx[0].sess->slot[0].src && ctx[0].sess->slot[0].input_mem); + ave_vb2_memops.put(mem); /* The caller returns this failed submission. */ + ave_poll_work(&ctx[0].poll_work.work); + ave_poll_work(&ctx[1].poll_work.work); + assert(!mapping_frees); + for (unsigned int i = 0; i < AVE_RECOVERY_ATTEMPTS; i++) + assert(ave_recovery_allow(&ave.recovery, jiffies)); + assert(ave_runtime_suspend(&dev) == -EBUSY); + assert(!stop_seen && !mapping_frees && !freed); + jiffies += AVE_RECOVERY_WINDOW; + ctrl_sticky = true; + assert(ave_runtime_suspend(&dev) == -EIO); + assert(!stop_seen && !mapping_frees && !freed); + ctrl_sticky = false; + assert(!ave_runtime_suspend(&dev)); + assert(stop_seen && !mapping_frees); + assert(ave_fw_prepare_restart(&ave) == -EAGAIN); + *(u32 *)(ave.coproc + regs.boot_arg) = 0; + power_seen = true; + mutex_lock(&ave.hw_lock); + assert(!ave_fw_prepare_restart(&ave)); + mutex_unlock(&ave.hw_lock); + assert(mapping_frees == 1 && freed == 2); +} + +/* Start can expose scratch even if submission reports failure. */ +static void failed_start(void) +{ + init(); + owners(); + submit_ctx = &ctx[0]; + ctx[0].sess->handed = false; + submit_result = -EIO; + mutex_lock(&ave.hw_lock); + assert(ave_cmd_start(&ctx[0]) == -EIO); + assert(ctx[0].sess->handed); + mutex_unlock(&ave.hw_lock); + ave_poll_work(&ctx[0].poll_work.work); + ave_poll_work(&ctx[1].poll_work.work); + assert(!freed && ave.failed_sessions); + assert(!ave_runtime_suspend(&dev)); + assert(stop_seen && !freed); + *(u32 *)(ave.coproc + regs.boot_arg) = 0; + power_seen = true; + mutex_lock(&ave.hw_lock); + assert(!ave_fw_prepare_restart(&ave)); + mutex_unlock(&ave.hw_lock); + assert(freed == 2); +} + +/* Allocator failures and normal vb2-only lifetimes do not leak references. */ +static void mapping_lifetimes(void) +{ + struct vb2_buffer vb = {}; + struct dma_buf dbuf = { .refs = 1 }; + void *mem; + + init(); + dma_alloc_fail = true; + assert(IS_ERR(ave_vb2_memops.alloc(&vb, &dev, SZ_16K))); + dma_alloc_fail = false; + dma_attach_fail = true; + assert(IS_ERR(ave_vb2_memops.attach_dmabuf(&vb, &dev, &dbuf, SZ_16K))); + assert(dbuf.refs == 1 && !dev.dma_refs); + dma_attach_fail = false; + mem = ave_vb2_memops.attach_dmabuf(&vb, &dev, &dbuf, SZ_16K); + assert(!IS_ERR(mem)); + dma_map_fail = true; + assert(ave_vb2_memops.map_dmabuf(mem) == -EIO); + dma_map_fail = false; + assert(!ave_vb2_memops.map_dmabuf(mem)); + ave_vb2_memops.unmap_dmabuf(mem); + assert(!mapping_unmaps); + assert(!ave_vb2_memops.map_dmabuf(mem)); /* Reuses the retained mapping. */ + ave_vb2_memops.unmap_dmabuf(mem); + ave_vb2_memops.detach_dmabuf(mem); + assert(mapping_unmaps == 1 && mapping_detaches == 1 && mapping_frees == 1); + assert(dbuf.refs == 1 && !dev.dma_refs); +} + int main(void) { + mapping_lifetimes(); + failed_submission(); + failed_start(); strict_6070_recovery(); buffer_failure(); sleep_guarantees(); @@ -243,6 +381,6 @@ int main(void) busy_ring_deadline(); restore_data(); puts("ok: recovery ownership, queues, quarantine, reset proof, rate limit, " - "PM rollback, DATA, IPC deadline"); + "PM rollback, DATA, IPC deadline, MMAP/dma-buf release ordering"); return 0; } diff --git a/drivers/media/platform/apple/ave/tests/run-recovery.py b/drivers/media/platform/apple/ave/tests/run-recovery.py index 33e382f5c2be6..0b8bc7097d194 100644 --- a/drivers/media/platform/apple/ave/tests/run-recovery.py +++ b/drivers/media/platform/apple/ave/tests/run-recovery.py @@ -28,16 +28,22 @@ def main(): code += declaration(header, 'ave_image') code += declaration(header, 'ave_opcode') code += source((src / 'ave-layout.h').read_text()) - code += (here / 'recovery-boundary.h').read_text() + boundary = (here / 'recovery-boundary.h').read_text() + marker = 'static void *test_ctrl_addr;' + boundary = boundary.replace(marker, (here / 'vb2-boundary.h').read_text() + '\n' + marker) + code += boundary + code += source((src / 'ave-vb2.c').read_text()) + code += (here / 'command-boundary.h').read_text() groups = { 'ave-fw.c': ('ave_fw_vcbar', 'ave_fw_lost', 'ave_fw_mark_broken', 'ave_fw_halt', 'ave_fw_shutdown', 'ave_fw_runtime_suspend', 'ave_fw_prepare_restart', 'ave_fw_restore_data'), - 'ave-sess.c': ('ave_sess_destroy', 'ave_sess_reap'), + 'ave-sess.c': ('ave_sess_free_bufs', 'ave_sess_destroy', 'ave_sess_reap'), 'ave-v4l2.c': ('ave_return_bufs', 'ave_buf_queue', 'ave_session_index', 'ave_session_count', 'ave_session_close', 'ave_session_free', 'ave_session_end', 'ave_stop_streaming', 'ave_reset_drain', 'ave_job_ready', 'ave_job_fail', 'ave_poll_work'), 'ave-ipc.c': ('ave_ipc_pump',), + 'ave-cmd.c': ('ave_cmd_start', 'ave_cmd_process'), 'ave-drv.c': ('ave_runtime_resume', 'ave_runtime_suspend', 'ave_suspend', 'ave_resume', 'ave_inject_wedge'), } code += '#define AVE_POLL_FAST_ROUNDS 20\n#define AVE_POLL_MS 2\n' diff --git a/drivers/media/platform/apple/ave/tests/vb2-boundary.h b/drivers/media/platform/apple/ave/tests/vb2-boundary.h new file mode 100644 index 0000000000000..92a4f517b4e07 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/vb2-boundary.h @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* DMA-contig boundary; real AVE memory operations run unchanged. */ +typedef unsigned int refcount_t; +static void refcount_set(refcount_t *r, unsigned int n) +{ + *r = n; +} +static void refcount_inc(refcount_t *r) +{ + assert(*r); + ++*r; +} +static bool refcount_dec_and_test(refcount_t *r) +{ + assert(*r); + return !--*r; +} +struct dma_buf { unsigned int refs; }; +struct vm_area_struct { int unused; }; +struct vb2_mem_ops { + void *(*alloc)(struct vb2_buffer *vb, struct device *dev, unsigned long size); + void (*put)(void *priv); + struct dma_buf *(*get_dmabuf)(struct vb2_buffer *vb, void *priv, unsigned long flags); + void *(*cookie)(struct vb2_buffer *vb, void *priv); + void *(*vaddr)(struct vb2_buffer *vb, void *priv); + int (*mmap)(void *priv, struct vm_area_struct *vma); + void (*prepare)(void *priv); + void (*finish)(void *priv); + int (*map_dmabuf)(void *priv); + void (*unmap_dmabuf)(void *priv); + void *(*attach_dmabuf)(struct vb2_buffer *vb, struct device *dev, + struct dma_buf *dbuf, unsigned long size); + void (*detach_dmabuf)(void *priv); + unsigned int (*num_users)(void *priv); +}; +static void get_dma_buf(struct dma_buf *b) +{ + assert(b->refs); + b->refs++; +} +static void dma_buf_put(struct dma_buf *b) +{ + assert(b->refs); + b->refs--; +} +static struct device *get_device(struct device *d) +{ + d->dma_refs++; + return d; +} +static void put_device(struct device *d) +{ + assert(d->dma_refs); + d->dma_refs--; +} +#define ERR_PTR(e) ((void *)(intptr_t)(e)) +#define IS_ERR(p) ((uintptr_t)(p) >= (uintptr_t)-4095) +#define kzalloc_obj(p) calloc(1, sizeof(p)) +struct test_mapping { + bool firmware_visible, mapped; + dma_addr_t dma; +}; +static bool stop_seen, power_seen, dma_alloc_fail, dma_attach_fail, dma_map_fail; +static unsigned int mapping_frees, mapping_unmaps, mapping_detaches; +static void *test_dma_alloc(struct vb2_buffer *vb, struct device *dev, unsigned long size) +{ + struct test_mapping *m = calloc(1, sizeof(*m)); + + assert(m); + if (dma_alloc_fail) { + free(m); + return ERR_PTR(-ENOMEM); + } + m->dma = (dma_addr_t)(uintptr_t)m; + return m; +} +static void test_dma_put(void *priv) +{ + struct test_mapping *m = priv; + + /* Every command-referenced mapping survives until stop AND DART reset. */ + assert(!m->firmware_visible || (stop_seen && power_seen)); + mapping_frees++; + free(m); +} +static void *test_dma_attach(struct vb2_buffer *vb, struct device *dev, + struct dma_buf *dbuf, unsigned long size) +{ + return dma_attach_fail ? ERR_PTR(-ENOMEM) : test_dma_alloc(vb, dev, size); +} +static int test_dma_map(void *priv) +{ + struct test_mapping *m = priv; + + assert(!m->mapped); + if (dma_map_fail) + return -EIO; + m->mapped = true; + return 0; +} +static void test_dma_unmap(void *priv) +{ + struct test_mapping *m = priv; + + assert(m->mapped); + assert(!m->firmware_visible || (stop_seen && power_seen)); + mapping_unmaps++; + m->mapped = false; +} +static void test_dma_detach(void *priv) +{ + struct test_mapping *m = priv; + + assert(!m->mapped); + mapping_detaches++; + test_dma_put(priv); +} +static void *test_dma_cookie(struct vb2_buffer *vb, void *priv) +{ + return &((struct test_mapping *)priv)->dma; +} +static void *test_dma_vaddr(struct vb2_buffer *vb, void *priv) +{ + return priv; +} +static int test_dma_mmap(void *priv, struct vm_area_struct *vma) +{ + return 0; +} +static unsigned int test_dma_users(void *priv) +{ + return 1; +} +static void test_dma_sync(void *priv) { } +static struct dma_buf *test_dma_export(struct vb2_buffer *vb, void *priv, unsigned long flags) +{ + return NULL; +} +static const struct vb2_mem_ops vb2_dma_contig_memops = { + .alloc = test_dma_alloc, .put = test_dma_put, + .attach_dmabuf = test_dma_attach, .detach_dmabuf = test_dma_detach, + .map_dmabuf = test_dma_map, .unmap_dmabuf = test_dma_unmap, + .cookie = test_dma_cookie, .vaddr = test_dma_vaddr, .mmap = test_dma_mmap, + .num_users = test_dma_users, .prepare = test_dma_sync, .finish = test_dma_sync, + .get_dmabuf = test_dma_export, +}; From 674e1cb9b9929740986ef269eb99e79452861c17 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 10:54:59 +1100 Subject: [PATCH 1569/1579] arm64: cpuinfo: Test the model name parser Split the SMBIOS processor Version lookup into dmi_processor_version() and cover it with KUnit: string numbers, missing and blank strings, structures too short to have the field, other structure types and truncation. The suite is built in (it includes cpuinfo.c's static parser) and only with KUNIT=y, so kernels with KUNIT=m are unchanged. Name every step of the lookup and document where the name comes from. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/Kconfig.debug | 8 +++ arch/arm64/kernel/cpuinfo.c | 60 +++++++++++----- arch/arm64/kernel/cpuinfo_kunit.c | 113 ++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+), 19 deletions(-) create mode 100644 arch/arm64/kernel/cpuinfo_kunit.c diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index f1978b9dff2bb..c0ef0498f1439 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -51,6 +51,14 @@ config ARM64_USER4K_EXPERIMENTAL private prctl numbers are not upstream assignments or a stable ABI promise. Programs that do not use the prctl keep the native granule. +config ARM64_CPUINFO_KUNIT_TEST + bool "KUnit tests for the /proc/cpuinfo model name" if !KUNIT_ALL_TESTS + depends on KUNIT=y && DMI + default KUNIT_ALL_TESTS + help + Test how /proc/cpuinfo reads the processor's name from the SMBIOS + Processor Information. + config ARM64_USER4K_KUNIT_TEST bool "Test mixed userspace granules with larger kernel pages" depends on KUNIT=y && ARM64_USER4K_EXPERIMENTAL diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 32f88fa0c695d..ba7efe1b93258 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -25,6 +25,7 @@ #include #include #include +#include /* * In case the boot CPU is hotpluggable, we record its initial state and @@ -221,39 +222,56 @@ static const char *const compat_hwcap2_str[] = { #endif /* CONFIG_COMPAT */ /* - * The processor's name as the firmware gives it, in the Version string of the - * SMBIOS Processor Information (type 4), e.g. "Apple M4 Pro". x86 reports the - * CPU's brand string as "model name"; arm64 CPUs have no such string, so the - * firmware's is the only source, and desktop tools (GNOME, KDE, inxi) read it - * from here. Empty when there is no SMBIOS or no processor version. + * The processor's name as the firmware gives it: the Version string of the + * SMBIOS Processor Information (type 4), e.g. "Apple M4 Pro". x86 prints the + * CPU's brand string as "model name"; arm64 CPUs have no brand string, so the + * firmware's name is the only one there is, and desktop tools (GNOME's About + * page, KDE Info Center, inxi, lscpu for cores it has no name for) read it + * from here. Empty, and the line left out, when the firmware gives none. */ -static char dmi_cpu_model[64]; +static char cpuinfo_model_name[64]; -static void __init dmi_find_cpu_model(const struct dmi_header *dm, void *private) +/* + * Copy the Version string of an SMBIOS type 4 structure into @buf, without + * leading or trailing white space. Leaves @buf empty if the structure is too + * short to have the field, the field names no string or the string is blank. + */ +static void dmi_processor_version(const struct dmi_header *dm, + char *buf, size_t size) { - const u8 *d = (const u8 *)dm; const char *s = (const char *)dm + dm->length; u8 index; - if (dm->type != DMI_ENTRY_PROCESSOR || dm->length < 0x11 || dmi_cpu_model[0]) + buf[0] = '\0'; + if (dm->type != DMI_ENTRY_PROCESSOR || dm->length <= 0x10) return; - index = d[0x10]; + + /* String numbers count from 1 in the unformatted area after the structure */ + index = ((const u8 *)dm)[0x10]; if (!index) return; while (--index && *s) s += strlen(s) + 1; - if (*s) { - strscpy(dmi_cpu_model, skip_spaces(s), sizeof(dmi_cpu_model)); - strim(dmi_cpu_model); - } + if (!*s) + return; + + strscpy(buf, skip_spaces(s), size); + strim(buf); } -static int __init cpuinfo_dmi_model_init(void) +static void __init cpuinfo_find_model_name(const struct dmi_header *dm, void *data) { - dmi_walk(dmi_find_cpu_model, NULL); + /* The first processor names the package; later ones repeat it */ + if (dm->type == DMI_ENTRY_PROCESSOR && !cpuinfo_model_name[0]) + dmi_processor_version(dm, cpuinfo_model_name, sizeof(cpuinfo_model_name)); +} + +static int __init cpuinfo_model_name_init(void) +{ + dmi_walk(cpuinfo_find_model_name, NULL); return 0; } -late_initcall(cpuinfo_dmi_model_init); +late_initcall(cpuinfo_model_name_init); static int c_show(struct seq_file *m, void *v) { @@ -272,8 +290,8 @@ static int c_show(struct seq_file *m, void *v) if (compat) seq_printf(m, "model name\t: ARMv8 Processor rev %d (%s)\n", MIDR_REVISION(midr), COMPAT_ELF_PLATFORM); - else if (dmi_cpu_model[0]) - seq_printf(m, "model name\t: %s\n", dmi_cpu_model); + else if (cpuinfo_model_name[0]) + seq_printf(m, "model name\t: %s\n", cpuinfo_model_name); seq_printf(m, "BogoMIPS\t: %lu.%02lu\n", loops_per_jiffy / (500000UL/HZ), @@ -580,3 +598,7 @@ void __init cpuinfo_store_boot_cpu(void) boot_cpu_data = *info; init_cpu_features(&boot_cpu_data); } + +#if IS_ENABLED(CONFIG_ARM64_CPUINFO_KUNIT_TEST) +#include "cpuinfo_kunit.c" +#endif diff --git a/arch/arm64/kernel/cpuinfo_kunit.c b/arch/arm64/kernel/cpuinfo_kunit.c new file mode 100644 index 0000000000000..0c4731fc05060 --- /dev/null +++ b/arch/arm64/kernel/cpuinfo_kunit.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the "model name" that /proc/cpuinfo takes from the SMBIOS + * Processor Information. Included from cpuinfo.c, so it can test its static + * parser. + */ +#include + +/* A type 4 structure of @len formatted bytes, Version string number @version */ +static size_t make_type4(u8 *buf, u8 len, u8 version, const char *strings, size_t slen) +{ + memset(buf, 0, len); + buf[0] = DMI_ENTRY_PROCESSOR; + buf[1] = len; + if (len > 0x10) + buf[0x10] = version; + memcpy(buf + len, strings, slen); + return len + slen; +} + +static void parse(u8 len, u8 version, const char *strings, size_t slen, char *out, size_t size) +{ + u8 buf[128]; + + make_type4(buf, len, version, strings, slen); + dmi_processor_version((const struct dmi_header *)buf, out, size); +} + +#define STRINGS(s) s, sizeof(s) + +static void cpuinfo_model_name_first_string(struct kunit *test) +{ + char out[64]; + + parse(0x30, 1, STRINGS("Apple M4 Pro\0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, "Apple M4 Pro"); +} + +static void cpuinfo_model_name_later_string(struct kunit *test) +{ + char out[64]; + + /* socket, manufacturer, version: the version is string 3 */ + parse(0x30, 3, STRINGS("SoC\0Apple Inc.\0Apple A18 Pro\0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, "Apple A18 Pro"); +} + +static void cpuinfo_model_name_trimmed(struct kunit *test) +{ + char out[64]; + + parse(0x30, 1, STRINGS(" Apple M2 \t\0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, "Apple M2"); +} + +static void cpuinfo_model_name_absent(struct kunit *test) +{ + char out[64] = "stale"; + + /* no string number */ + parse(0x30, 0, STRINGS("Apple M1\0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, ""); + + /* a string number past the strings there are */ + strscpy(out, "stale", sizeof(out)); + parse(0x30, 4, STRINGS("SoC\0Apple Inc.\0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, ""); + + /* only blanks */ + strscpy(out, "stale", sizeof(out)); + parse(0x30, 1, STRINGS(" \0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, ""); + + /* a structure too short to have the Version field */ + strscpy(out, "stale", sizeof(out)); + parse(0x10, 1, STRINGS("Apple M1\0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, ""); +} + +static void cpuinfo_model_name_other_type(struct kunit *test) +{ + char out[64] = "stale"; + u8 buf[128]; + + make_type4(buf, 0x30, 1, STRINGS("Apple M4 Pro\0\0")); + buf[0] = 1; /* System Information */ + dmi_processor_version((const struct dmi_header *)buf, out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, ""); +} + +static void cpuinfo_model_name_truncated(struct kunit *test) +{ + char out[8]; + + parse(0x30, 1, STRINGS("Apple M4 Max\0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, "Apple M"); +} + +static struct kunit_case cpuinfo_model_name_cases[] = { + KUNIT_CASE(cpuinfo_model_name_first_string), + KUNIT_CASE(cpuinfo_model_name_later_string), + KUNIT_CASE(cpuinfo_model_name_trimmed), + KUNIT_CASE(cpuinfo_model_name_absent), + KUNIT_CASE(cpuinfo_model_name_other_type), + KUNIT_CASE(cpuinfo_model_name_truncated), + {} +}; + +static struct kunit_suite cpuinfo_model_name_suite = { + .name = "arm64-cpuinfo-model-name", + .test_cases = cpuinfo_model_name_cases, +}; +kunit_test_suite(cpuinfo_model_name_suite); From 53bb661265d9aba99c41eccca8c4a3d47f536880 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 10:58:06 +1100 Subject: [PATCH 1570/1579] tools/asahi: extract the AV1 reference guard in the AV1 tail host test avd-av1-tail-tests.py builds avd_av1_set_prob() and friends from avd-av1.c on the host. Since "media: apple: avd: bound AV1 reference indices before using them", those functions call avd_av1_ref_ts(), a u64 helper the extractor did not pick up, so the generated file stopped compiling and run-host-tests went from 158/158 to 148/158. Extract avd_av1_ref_ts() too (the function matcher now accepts a u64 return type) and give the host prelude u64 and U64_MAX. The production controls pass again and all six mutations are still caught. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/avd-av1-tail-tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/asahi/avd-av1-tail-tests.py b/tools/asahi/avd-av1-tail-tests.py index d602801b7c0a7..d434ab3a0b8a2 100644 --- a/tools/asahi/avd-av1-tail-tests.py +++ b/tools/asahi/avd-av1-tail-tests.py @@ -15,7 +15,7 @@ def function(source, name): - match = re.search(r'^(?:static )?(?:int|void)\s+' + name + r'\(', source, re.M) + match = re.search(r'^(?:static )?(?:int|void|u64)\s+' + name + r'\(', source, re.M) if not match: raise ValueError(name) start = match.start() @@ -47,6 +47,8 @@ def structure(source, name): typedef int8_t s8; typedef uint16_t u16; typedef uint32_t u32; +typedef uint64_t u64; +#define U64_MAX UINT64_MAX typedef uint64_t dma_addr_t; #define ALIGN(x,a) (((x)+(a)-1)&~((__typeof__(x))(a)-1)) #define ALIGN_DOWN(x,a) ((x)&~((__typeof__(x))(a)-1)) @@ -211,7 +213,7 @@ def main(): text = {name: p.read_text() for name, p in paths.items()} src = text['avd-av1.c'] macros = src[src.index('#define AVD_CDFS_SIZE'):src.index('struct avd_av1_run {')] - functions = {n: function(src, n) for n in ['avd_av1_set_prob', 'avd_av1_set_seg', 'avd_av1_seg_size', 'avd_color_size', 'avd_av1_run_preamble', 'avd_av1_adjust_decoded_fmt']} + functions = {n: function(src, n) for n in ['avd_av1_ref_ts', 'avd_av1_set_prob', 'avd_av1_set_seg', 'avd_av1_seg_size', 'avd_color_size', 'avd_av1_run_preamble', 'avd_av1_adjust_decoded_fmt']} unaligned = ('\n#define UNALIGNED_CDFS_OFFSET(dst,cl) ((dst)-ALIGN(cl,AVD_ALIGN)-ALIGN(AVD_CDFS_SIZE,AVD_ALIGN))\n' '#define UNALIGNED_SEG_OFFSET(dst,cl,sl) (UNALIGNED_CDFS_OFFSET(dst,cl)-(sl))\n') # Each mutation names the extracted text it rewrites; a stale one is refused, never run as a no-op. From 43722ce97eb834770a0b8140e09e2c686fc300bc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:00:30 +1100 Subject: [PATCH 1571/1579] media: apple: avd: keep DMA blocked once the decoder is retired When the recovery budget runs out, or a reset fails, the decoder is retired: its jobs fail without touching the hardware until the module is reloaded. The frame that triggered it is still finished with an error, so its buffers go back to userspace while the engine that just timed out may still be running, which is the DMA-after-free pattern behind the J616s hangs. Retire through the same first steps as a recovery: disable the interrupt, stop the CM3 and attach the empty IOMMU domain, then leave it attached. A still-running engine can then only fault at the DART. A reset that fails after DMA was blocked keeps it blocked as well. Detach the empty domain at remove so a reload starts from the DMA domain. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-drv.c | 29 +++++++++++++++++++++- drivers/media/platform/apple/avd/avd.h | 2 ++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 462df919265d5..7ffc93034e4b2 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -238,6 +238,27 @@ int avd_submit_job(struct avd_ctx *ctx) * to drain, and only then assert the pmgr reset. Recover sparingly, and * retire the decoder after a failed or a third recovery within a minute. */ +/* + * Retiring: stop the CM3 and leave the decoder's DMA blocked at its DART + * until the module is unloaded, so an engine that is still running (the + * frame timed out, after all) can only fault, never write memory whose + * buffers are about to go back to userspace. + */ +static void avd_park(struct avd_dev *avd) +{ + if (pm_runtime_resume_and_get(avd->dev) < 0) + return; + disable_irq(avd->irq); + avd_shutdown(avd); + if (avd->empty_domain && !avd->dma_parked) { + if (iommu_attach_device(avd->empty_domain, avd->dev)) + dev_err(avd->dev, "retire: could not block DMA\n"); + else + avd->dma_parked = true; + } + pm_runtime_put_autosuspend(avd->dev); +} + static int avd_reset(struct avd_dev *avd) { int ret = 0; @@ -251,6 +272,7 @@ static int avd_reset(struct avd_dev *avd) } if (++avd->recoveries > AVD_RECOVER_MAX) { ret = -EBUSY; + avd_park(avd); goto retire; } if (avd->last_recover && @@ -278,7 +300,10 @@ static int avd_reset(struct avd_dev *avd) ret = reset_control_reset(avd->rstc); if (ret) dev_err(avd->dev, "recover: reset failed: %d\n", ret); - if (avd->empty_domain) { + if (avd->empty_domain && ret) { + /* A decoder that failed its reset keeps its DMA blocked. */ + avd->dma_parked = true; + } else if (avd->empty_domain) { dev_warn(avd->dev, "recover: restore DMA\n"); iommu_detach_device(avd->empty_domain, avd->dev); if (avd->domain && iommu_get_domain_for_dev(avd->dev) != avd->domain) { @@ -1087,6 +1112,8 @@ static void avd_remove(struct platform_device *pdev) avd_v4l2_cleanup(avd); + if (avd->dma_parked) + iommu_detach_device(avd->empty_domain, avd->dev); if (avd->empty_domain) iommu_domain_free(avd->empty_domain); diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 20411c991ae31..10cd73a90977b 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -241,6 +241,8 @@ struct avd_dev { unsigned int recoveries; unsigned long last_recover; bool retired; + /* Retired with the DART on the empty domain; detached at remove. */ + bool dma_parked; const struct avd_variant *variant; unsigned int capabilities; From eebbd2219e1edd981962ab09d3f7bdebbec21193 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:00:40 +1100 Subject: [PATCH 1572/1579] media: apple: avd: rest the decoder instead of retiring it on a reset storm A third recovery within a minute retired the decoder until the module was reloaded. That is too harsh for a desktop: on an M2 the H.264 conformance suite retired it within three seconds, because a few vectors use features the hardware rejects with a VP error, and every later stream failed until a reload. A user with a bad file would lose hardware decode until a reboot. Recoveries are now quiesced (DMA blocked at the DART before the reset) and have held up under forced errors, so the budget only needs to bound their rate. When it runs out, park the decoder (interrupt off, CM3 stopped, DMA blocked) and fail new frames without touching it until the budget window ends. The next frame after that resets it with DMA still blocked, restores the DMA domain and boots the firmware. Only a reset, restore or boot that fails retires the decoder. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-drv.c | 80 ++++++++++++++++++---- drivers/media/platform/apple/avd/avd.h | 5 +- 2 files changed, 72 insertions(+), 13 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 7ffc93034e4b2..13f87523226d1 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -235,20 +235,22 @@ int avd_submit_job(struct avd_ctx *ctx) * frame), so: stop the interrupt handler and the CM3 first, block the * decoder's DMA at its DART so a transfer still in flight faults instead * of reaching memory that may already be reused, give such transfers time - * to drain, and only then assert the pmgr reset. Recover sparingly, and - * retire the decoder after a failed or a third recovery within a minute. + * to drain, and only then assert the pmgr reset. Recover sparingly: a + * third recovery within a minute parks the decoder (DMA blocked) until the + * minute is over, and only a failed reset retires it. */ /* - * Retiring: stop the CM3 and leave the decoder's DMA blocked at its DART - * until the module is unloaded, so an engine that is still running (the - * frame timed out, after all) can only fault, never write memory whose - * buffers are about to go back to userspace. + * Parking (resting or retired): stop the CM3 and leave the decoder's DMA + * blocked at its DART, so an engine that is still running (the frame timed + * out, after all) can only fault, never write memory whose buffers are about + * to go back to userspace. */ static void avd_park(struct avd_dev *avd) { if (pm_runtime_resume_and_get(avd->dev) < 0) return; disable_irq(avd->irq); + avd->irq_parked = true; avd_shutdown(avd); if (avd->empty_domain && !avd->dma_parked) { if (iommu_attach_device(avd->empty_domain, avd->dev)) @@ -271,9 +273,19 @@ static int avd_reset(struct avd_dev *avd) avd->recoveries = 0; } if (++avd->recoveries > AVD_RECOVER_MAX) { - ret = -EBUSY; + /* + * Out of budget: stop the decoder with its DMA blocked and let + * it rest until the window ends. Frames fail without touching + * the hardware meanwhile; the next frame after that recovers + * it. Bad streams (or conformance vectors using features the + * hardware lacks) must not cost the user the decoder until a + * reboot. + */ avd_park(avd); - goto retire; + avd->rest_until = avd->recover_window_start + AVD_RECOVER_WINDOW; + dev_warn(avd->dev, "decoder resting after repeated resets, new frames fail for up to %u s\n", + jiffies_to_msecs(avd->rest_until - jiffies) / 1000 + 1); + return -EBUSY; } if (avd->last_recover && time_before(jiffies, avd->last_recover + msecs_to_jiffies(AVD_RECOVER_GAP_MS))) @@ -328,8 +340,44 @@ static int avd_reset(struct avd_dev *avd) return 0; retire: avd->retired = true; - dev_err(avd->dev, "decoder retired after %s (%d): reload the module to use it again\n", - ret == -EBUSY ? "repeated resets" : "a failed reset", ret); + dev_err(avd->dev, "decoder retired after a failed reset (%d): reload the module to use it again\n", + ret); + return ret; +} + +/* Bring a rested decoder back: reset with DMA still blocked, restore, boot. */ +static int avd_wake(struct avd_dev *avd) +{ + int ret; + + ret = pm_runtime_resume_and_get(avd->dev); + if (ret < 0) + return ret; + dev_warn(avd->dev, "recover: wake after rest\n"); + avd_shutdown(avd); + ret = reset_control_reset(avd->rstc); + if (!ret && avd->dma_parked) { + iommu_detach_device(avd->empty_domain, avd->dev); + if (avd->domain && iommu_get_domain_for_dev(avd->dev) != avd->domain) + ret = -EIO; + else + avd->dma_parked = false; + } + if (!ret) + ret = avd_boot(avd); + if (!ret) { + avd->recoveries = 0; + avd->last_recover = jiffies ?: 1; + if (avd->irq_parked) + enable_irq(avd->irq); + avd->irq_parked = false; + dev_warn(avd->dev, "recover: done\n"); + } else { + avd->retired = true; + dev_err(avd->dev, "decoder retired after a failed wake (%d): reload the module to use it again\n", + ret); + } + pm_runtime_put_autosuspend(avd->dev); return ret; } @@ -485,14 +533,22 @@ static void avd_device_run(void *priv) enum vb2_buffer_state result = VB2_BUF_STATE_ERROR; const char *outcome = "already finished"; unsigned long flags; - bool finish = false; + bool finish = false, fail; u64 seq; int ret; if (WARN_ON(!desc)) return; - if (READ_ONCE(avd->retired) || ctx->broken) { + fail = READ_ONCE(avd->retired) || ctx->broken; + /* A rested decoder stays parked until its window ends, then wakes. */ + if (!fail && avd->rest_until) { + if (time_before(jiffies, avd->rest_until) || avd_wake(avd)) + fail = true; + else + avd->rest_until = 0; + } + if (fail) { struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); /* Complete the request's controls too, or its fd never signals. */ diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 10cd73a90977b..1f2fa8e312d92 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -241,8 +241,11 @@ struct avd_dev { unsigned int recoveries; unsigned long last_recover; bool retired; - /* Retired with the DART on the empty domain; detached at remove. */ + /* DMA blocked at the DART (retired or resting); detached at remove. */ bool dma_parked; + /* Out of recovery budget: frames fail without hardware access until then. */ + unsigned long rest_until; + bool irq_parked; const struct avd_variant *variant; unsigned int capabilities; From 05ba7730e2c98accb644775e559635cb6cc1c137 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 10:19:50 +1100 Subject: [PATCH 1573/1579] arm64: dts: apple: Add chassis-type to the boards that lack it Fifteen Apple silicon device trees had no chassis-type: the M3 and M4 MacBook Air and Pro, MacBook Air (M5), MacBook Neo, the M3 and M4 iMacs, Mac mini (M4), Mac mini (M6) and Mac Studio (M3 Ultra). Every other Mac has it. Without it the bootloader reports an unknown enclosure in the SMBIOS tables, and systemd-hostnamed, GNOME and power management cannot tell a laptop from a desktop. On the MacBook Neo hostnamectl showed no chassis and /sys/class/dmi/id/chassis_type read 2 (Unknown). Add "laptop", "desktop" or "all-in-one" as for the other boards of each kind. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6032-j575d.dts | 1 + arch/arm64/boot/dts/apple/t8122-j433.dts | 1 + arch/arm64/boot/dts/apple/t8122-j434.dts | 1 + arch/arm64/boot/dts/apple/t8122-j504.dts | 1 + arch/arm64/boot/dts/apple/t8122-j613.dts | 1 + arch/arm64/boot/dts/apple/t8122-j615.dts | 1 + arch/arm64/boot/dts/apple/t8132-j604.dts | 1 + arch/arm64/boot/dts/apple/t8132-j623.dts | 1 + arch/arm64/boot/dts/apple/t8132-j624.dts | 1 + arch/arm64/boot/dts/apple/t8132-j713.dts | 1 + arch/arm64/boot/dts/apple/t8132-j715.dts | 1 + arch/arm64/boot/dts/apple/t8132-j773g.dts | 1 + arch/arm64/boot/dts/apple/t8140-j700.dts | 1 + arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + arch/arm64/boot/dts/apple/t8152-j873g.dts | 1 + 15 files changed, 15 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6032-j575d.dts b/arch/arm64/boot/dts/apple/t6032-j575d.dts index 1bf14006807e8..ac36bb2723882 100644 --- a/arch/arm64/boot/dts/apple/t6032-j575d.dts +++ b/arch/arm64/boot/dts/apple/t6032-j575d.dts @@ -14,6 +14,7 @@ / { compatible = "apple,j575d", "apple,t6032", "apple,arm-platform"; model = "Apple Mac Studio (M3 Ultra, 2025)"; + chassis-type = "desktop"; aliases { serial0 = &serial0; diff --git a/arch/arm64/boot/dts/apple/t8122-j433.dts b/arch/arm64/boot/dts/apple/t8122-j433.dts index 9ceb6a6ebcdf2..d4803de20e04d 100644 --- a/arch/arm64/boot/dts/apple/t8122-j433.dts +++ b/arch/arm64/boot/dts/apple/t8122-j433.dts @@ -16,6 +16,7 @@ / { compatible = "apple,j433", "apple,t8122", "apple,arm-platform"; model = "Apple iMac (24-inch, 2x USB-C, M3, 2023)"; + chassis-type = "all-in-one"; aliases { ethernet0 = ðernet0; diff --git a/arch/arm64/boot/dts/apple/t8122-j434.dts b/arch/arm64/boot/dts/apple/t8122-j434.dts index 1e5d75aa02b41..741833526764e 100644 --- a/arch/arm64/boot/dts/apple/t8122-j434.dts +++ b/arch/arm64/boot/dts/apple/t8122-j434.dts @@ -16,6 +16,7 @@ / { compatible = "apple,j434", "apple,t8122", "apple,arm-platform"; model = "Apple iMac (24-inch, 4x USB-C, M3, 2023)"; + chassis-type = "all-in-one"; }; &wifi0 { diff --git a/arch/arm64/boot/dts/apple/t8122-j504.dts b/arch/arm64/boot/dts/apple/t8122-j504.dts index 4c36e01bae641..58790e56c0cf0 100644 --- a/arch/arm64/boot/dts/apple/t8122-j504.dts +++ b/arch/arm64/boot/dts/apple/t8122-j504.dts @@ -17,6 +17,7 @@ / { compatible = "apple,j504", "apple,t8122", "apple,arm-platform"; model = "Apple MacBook Pro (14-inch, M3, 2023)"; + chassis-type = "laptop"; led-controller { compatible = "pwm-leds"; diff --git a/arch/arm64/boot/dts/apple/t8122-j613.dts b/arch/arm64/boot/dts/apple/t8122-j613.dts index 330a02fe9c32a..3768661807e96 100644 --- a/arch/arm64/boot/dts/apple/t8122-j613.dts +++ b/arch/arm64/boot/dts/apple/t8122-j613.dts @@ -17,6 +17,7 @@ / { compatible = "apple,j613", "apple,t8122", "apple,arm-platform"; model = "Apple MacBook Air (13-inch, M3, 2024)"; + chassis-type = "laptop"; aliases { sep = &sep; diff --git a/arch/arm64/boot/dts/apple/t8122-j615.dts b/arch/arm64/boot/dts/apple/t8122-j615.dts index 5e52cd3046e71..34b6db85b4410 100644 --- a/arch/arm64/boot/dts/apple/t8122-j615.dts +++ b/arch/arm64/boot/dts/apple/t8122-j615.dts @@ -17,6 +17,7 @@ / { compatible = "apple,j615", "apple,t8122", "apple,arm-platform"; model = "Apple MacBook Air (15-inch, M3, 2024)"; + chassis-type = "laptop"; aliases { sep = &sep; diff --git a/arch/arm64/boot/dts/apple/t8132-j604.dts b/arch/arm64/boot/dts/apple/t8132-j604.dts index c4be915cb33ab..8a065a6f47a7b 100644 --- a/arch/arm64/boot/dts/apple/t8132-j604.dts +++ b/arch/arm64/boot/dts/apple/t8132-j604.dts @@ -16,6 +16,7 @@ / { compatible = "apple,j604", "apple,t8132", "apple,arm-platform"; model = "Apple MacBook Pro (14-inch, M4, 2024)"; + chassis-type = "laptop"; led-controller { compatible = "pwm-leds"; diff --git a/arch/arm64/boot/dts/apple/t8132-j623.dts b/arch/arm64/boot/dts/apple/t8132-j623.dts index 508e26056022c..9b94a6ae74a20 100644 --- a/arch/arm64/boot/dts/apple/t8132-j623.dts +++ b/arch/arm64/boot/dts/apple/t8132-j623.dts @@ -15,4 +15,5 @@ / { compatible = "apple,j623", "apple,t8132", "apple,arm-platform"; model = "Apple iMac (24-inch, 2x USB-C, M4, 2024)"; + chassis-type = "all-in-one"; }; diff --git a/arch/arm64/boot/dts/apple/t8132-j624.dts b/arch/arm64/boot/dts/apple/t8132-j624.dts index 84bdd8e343eb1..37e1a46f89b84 100644 --- a/arch/arm64/boot/dts/apple/t8132-j624.dts +++ b/arch/arm64/boot/dts/apple/t8132-j624.dts @@ -15,4 +15,5 @@ / { compatible = "apple,j624", "apple,t8132", "apple,arm-platform"; model = "Apple iMac (24-inch, 4x USB-C, M4, 2024)"; + chassis-type = "all-in-one"; }; diff --git a/arch/arm64/boot/dts/apple/t8132-j713.dts b/arch/arm64/boot/dts/apple/t8132-j713.dts index efaadea58306b..f62defca0dae5 100644 --- a/arch/arm64/boot/dts/apple/t8132-j713.dts +++ b/arch/arm64/boot/dts/apple/t8132-j713.dts @@ -20,6 +20,7 @@ / { compatible = "apple,j713", "apple,t8132", "apple,arm-platform"; model = "Apple MacBook Air (13-inch, M4, 2025)"; + chassis-type = "laptop"; aliases { ans = &nvme; diff --git a/arch/arm64/boot/dts/apple/t8132-j715.dts b/arch/arm64/boot/dts/apple/t8132-j715.dts index 2672914ea76c9..1a387d273e8ff 100644 --- a/arch/arm64/boot/dts/apple/t8132-j715.dts +++ b/arch/arm64/boot/dts/apple/t8132-j715.dts @@ -16,6 +16,7 @@ / { compatible = "apple,j715", "apple,t8132", "apple,arm-platform"; model = "Apple MacBook Air (15-inch, M4, 2025)"; + chassis-type = "laptop"; led-controller { compatible = "pwm-leds"; diff --git a/arch/arm64/boot/dts/apple/t8132-j773g.dts b/arch/arm64/boot/dts/apple/t8132-j773g.dts index 0583cb9d19f62..2f16f8e62c06c 100644 --- a/arch/arm64/boot/dts/apple/t8132-j773g.dts +++ b/arch/arm64/boot/dts/apple/t8132-j773g.dts @@ -15,6 +15,7 @@ / { compatible = "apple,j773g", "apple,t8132", "apple,arm-platform"; model = "Apple Mac mini (M4, 2024)"; + chassis-type = "desktop"; }; /* diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 84aa5febe23fc..4034b774388c2 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -15,6 +15,7 @@ / { compatible = "apple,j700", "apple,t8140", "apple,arm-platform"; model = "Apple MacBook Neo"; + chassis-type = "laptop"; aliases { nvram = &nvram; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 20df27fa1bfe3..576c4637f37df 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -14,6 +14,7 @@ / { compatible = "apple,j813", "apple,t8142", "apple,arm-platform"; model = "Apple MacBook Air (13-inch, M5, 2026)"; + chassis-type = "laptop"; #address-cells = <2>; #size-cells = <2>; diff --git a/arch/arm64/boot/dts/apple/t8152-j873g.dts b/arch/arm64/boot/dts/apple/t8152-j873g.dts index 6b5e14036c98a..0a0818d85d085 100644 --- a/arch/arm64/boot/dts/apple/t8152-j873g.dts +++ b/arch/arm64/boot/dts/apple/t8152-j873g.dts @@ -14,6 +14,7 @@ / { compatible = "apple,j873g", "apple,t8152", "apple,arm-platform"; model = "Apple Mac mini (M6)"; + chassis-type = "desktop"; #address-cells = <2>; #size-cells = <2>; From 40d6cf92bac0a76e02ca31814d46b4aba5179bf8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:00:50 +1100 Subject: [PATCH 1574/1579] iommu/apple-dart: use strict flushing for media and ML engine clients The video decoder, video encoder, neural engine and camera ISP are run by their own coprocessor firmware, which can still have DMA in flight after the host driver gives up on a job: a frame timeout, a reset, a stream torn down mid-frame. With deferred flushing a freed buffer's IOVAs keep translating to its pages until the flush queue drains, while the pages are already back in the allocator, so such a late write corrupts unrelated memory without any fault. The M4 Pro hard hangs under AVD stress all followed decoder resets and left no panic and an empty pstore, which fits this. Give these clients a strict DMA domain, chosen from the client's device tree compatible ("apple,-avd", "-ave", "-ane", "-isp" and "apple,isp"), so a late write faults at the DART instead. They map their buffers once per stream or model, so the per-unmap invalidate stays off the hot path. Per-stream T8140/T8142 DARTs were already strict through apple_dart_capable(). Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 24 ++++++++++++++++ drivers/iommu/apple-dart.c | 49 +++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 51e504321798d..5f512e18bfd50 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -940,7 +940,31 @@ static void apple_dart_test_resume_reset_retry(struct kunit *test) dart->hw->ttbr_valid | 0x1000); } +static void apple_dart_test_strict_engines(struct kunit *test) +{ + static const char * const strict[] = { + "apple,t8103-avd", "apple,t6000-avd", "apple,t8112-avd", "apple,t6041-avd", + "apple,t8132-avd", "apple,t8140-avd", "apple,t6000-ave", "apple,t8112-ave", + "apple,t6041-ave", "apple,t8103-ane", "apple,t6021-ane", "apple,t6041-ane", + "apple,t8112-isp", "apple,t6041-isp", "apple,isp", + }; + static const char * const lazy[] = { + "apple,t6021-ane-mailbox", "apple,asc-mailbox-v4", "apple,t8103-display-pipe", + "apple,nvme-ans2", "apple,t8103-dart", "apple,t6000-display-crossbar", + "brcm,bcm4378-fmac", "apple,", "avd", "foo,t8112-avd", "apple,t8112-avdx", + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(strict); i++) + KUNIT_EXPECT_TRUE_MSG(test, apple_dart_compat_is_strict_engine(strict[i]), + "%s", strict[i]); + for (i = 0; i < ARRAY_SIZE(lazy); i++) + KUNIT_EXPECT_FALSE_MSG(test, apple_dart_compat_is_strict_engine(lazy[i]), + "%s", lazy[i]); +} + static struct kunit_case apple_dart_test_cases[] = { + KUNIT_CASE(apple_dart_test_strict_engines), KUNIT_CASE(apple_dart_test_iommu_specifiers), KUNIT_CASE(apple_dart_test_independent_remap_roots), KUNIT_CASE(apple_dart_test_before_attach), diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 384e243d5d0a2..ba23e2c239cd8 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -3279,6 +3280,52 @@ static bool apple_dart_capable(struct device *dev, enum iommu_cap cap) } } +/* + * Media and ML engines (video decode and encode, neural engine, camera ISP) + * are run by their own coprocessor firmware, which can still have DMA in + * flight after the host driver has given up on a job: a timeout, a reset, + * a stream torn down mid-frame. With deferred flushing the DART keeps + * translating a freed buffer's addresses until the flush queue drains, so + * such a late write lands silently in pages that already went back to the + * allocator. Strict flushing makes it fault at the DART instead. These + * engines map their buffers once per stream or model, so the per-unmap + * invalidate is not on any hot path. + * + * The client is identified by its device tree compatible, "apple,-avd" + * and so on, so every SoC generation is covered without a per-SoC table. + */ +static const char * const apple_dart_strict_engines[] = { + "avd", "ave", "ane", "isp", +}; + +static bool apple_dart_compat_is_strict_engine(const char *compat) +{ + const char *engine; + unsigned int i; + + if (!str_has_prefix(compat, "apple,")) + return false; + engine = strrchr(compat, '-'); + engine = engine ? engine + 1 : compat + strlen("apple,"); + for (i = 0; i < ARRAY_SIZE(apple_dart_strict_engines); i++) + if (!strcmp(engine, apple_dart_strict_engines[i])) + return true; + return false; +} + +static bool apple_dart_master_needs_strict(struct device *dev) +{ + struct property *prop; + const char *compat; + + if (!dev->of_node) + return false; + of_property_for_each_string(dev->of_node, "compatible", prop, compat) + if (apple_dart_compat_is_strict_engine(compat)) + return true; + return false; +} + static int apple_dart_def_domain_type(struct device *dev) { struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); @@ -3298,6 +3345,8 @@ static int apple_dart_def_domain_type(struct device *dev) return IOMMU_DOMAIN_DMA; if (cfg->locked) return IOMMU_DOMAIN_DMA; + if (apple_dart_master_needs_strict(dev)) + return IOMMU_DOMAIN_DMA; return 0; } From baf3c39cc6d47e9edac3e246dfaa0715eeefb39f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:53:18 +1100 Subject: [PATCH 1575/1579] media: apple: avd: run the frame watchdog on the unbound system workqueue The first frame watchdog of every boot logged "workqueue: work func avd_watchdog_func enqueued on deprecated workqueue" because it was queued with schedule_delayed_work() and mod_delayed_work(system_wq). The watchdog has no CPU affinity to keep, so queue it on system_dfl_wq. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-drv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 13f87523226d1..215ac921ce671 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -483,7 +483,7 @@ static irqreturn_t avd_irq_handler(int irq, void *data) * timeout: it resets the AVD and fails only this frame, and it * ignores a job the hardware does not own yet. */ - mod_delayed_work(system_wq, &ctx->watchdog_work, 0); + mod_delayed_work(system_dfl_wq, &ctx->watchdog_work, 0); goto done; } @@ -608,7 +608,7 @@ static void avd_device_run(void *priv) } else { avd->job_state = AVD_JOB_SUBMITTED; ctx->wd_seq = seq; - schedule_delayed_work(&ctx->watchdog_work, msecs_to_jiffies(2000)); + queue_delayed_work(system_dfl_wq, &ctx->watchdog_work, msecs_to_jiffies(2000)); outcome = "submitted"; } } From 8cf00b96eef85bdda2be801e0563bb578c526d1c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:34:30 +1100 Subject: [PATCH 1576/1579] drm/apple: Apply VRR_ENABLED on the desktop 26.6 panel without a mode set GNOME never turned on variable refresh for a fullscreen game on the J616s panel. Compositors toggle VRR_ENABLED in a plain page flip, but the desktop 26.6 path made a VRR_ENABLED change a mode set, so the flip failed with EINVAL ("Page flip failed: drmModeAtomicCommit: Invalid argument" in the journal) and VRR stayed off. With ALLOW_MODESET the toggle works but costs a full link reselect: the firmware shuts the link down, the backlight is off for about 200 ms and the commit takes 207 ms on the panel (272 ms on an external display). On the panel, install a VRR timing with adaptive sync on whatever VRR_ENABLED says, at the mode's minimum rate, as the pipe pair path already does. VRR_ENABLED then only moves the minimum rate on the live mode (IOMFBParameter 14, sent ahead of the frame's swap): the mode's minimum when on, its nominal rate when off, which holds the panel at that rate. The firmware takes the new minimum without a mode set. The external processors don't: on an MSI MPG 325CQRXF (48-280 Hz) a live minimum of 280, 279 or 240 Hz left the idle desktop at 48 Hz, so they keep applying VRR_ENABLED with a mode set. On the J616s panel (3456x2234, 24-120 Hz, firmware 26.6.2) a fullscreen glmark2-wayland under GNOME now flips VRR_ENABLED to 1 and back with no failed flips and no link reselect. Idle with a static image, the panel draws 290 mW less with VRR on than off when toggled through a mode set, and 440 mW less when toggled live, so the live minimum rate takes effect. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 19 +++++ drivers/gpu/drm/apple/dcp.c | 10 ++- drivers/gpu/drm/apple/iomfb.c | 3 +- drivers/gpu/drm/apple/iomfb_template.c | 75 ++++++++++++++++++- .../drivers/gpu/apple-dcp/lifecycle.py | 24 +++--- 5 files changed, 116 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 32b39de3c7fb4..38a117e769630 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -268,6 +268,14 @@ struct dcp_desktop_state { bool modeset_callback_error; /* An adaptive swap that timed out keeps fixed refresh until the next boot. */ bool vrr_failed; + /* + * Adaptive sync of a VRR timing installed by this pipe alone: the + * minimum rates for VRR_ENABLED on and off (16.16 Hz), and the one + * the firmware was last given. Zero when not installed that way. + */ + u32 vrr_min_on, vrr_min_off, vrr_min_sent; + /* The frame's colour matrix, sent after an adaptive sync update. */ + struct iomfb_set_matrix_req matrix; u32 present_id; bool present_pending; /* @@ -900,6 +908,17 @@ int dcp_backlight_update(struct apple_dcp *dcp); s32 dcp_ext_backlight_value(struct apple_dcp *dcp); bool dcp_has_panel(struct apple_dcp *dcp); +/* + * The desktop 26.6 panel takes VRR_ENABLED on its live mode, through the + * adaptive sync minimum rate (see dcp_vrr_update()). The external + * processors ignore a new minimum on a live mode. + */ +static inline bool dcp_vrr_live(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && dcp->main_display && + dcp_has_panel(dcp); +} + /* * Does @dcp's firmware speak the 14.7 AFK, DPTX and DPAV endpoint protocol, * whichever IOMFB path drives it? DCP_FIRMWARE_V_14_7 is the native DCPLink diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7e4f83b7240b1..e854160c6bdec 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1509,8 +1509,14 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi crtc_state->vrr_enabled) return -EINVAL; - /* The desktop 26.6 firmware applies adaptive sync with a mode set. */ - if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && crtc_state->enable) { + /* + * On desktop 26.6 firmware the panel takes VRR_ENABLED with the next + * frame, without a mode set (see dcp_vrr_update()). The external + * processors ignore a new minimum rate on a live mode, so they still + * change adaptive sync with a mode set. + */ + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && crtc_state->enable && + !dcp_vrr_live(dcp)) { struct drm_crtc_state *old = drm_atomic_get_old_crtc_state(commit, crtc); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 76a53d1f91556..6f2a687007c58 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -2171,7 +2171,8 @@ static int dcp_desktop_modeset(struct apple_dcp *dcp, if (d->trial_modeset_ok) { d->trial_modeset_ok = false; - if (!crtc_state->vrr_enabled && d->mode_live && + /* The trial installed a VRR timing variable too (see the mode set). */ + if (d->mode_live && lookup_mode(dcp, &crtc_state->mode, &matched) && matched.timing_mode_id == d->trial_timing_mode_id && matched.color_mode_id == d->trial_color_mode_id) { diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 4e218bf664b4f..e1cb13e7f8517 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -3237,9 +3237,19 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, return -ENOMEM; cookie = &modeset->wait; modeset->actual = dcp->mode; - /* After an adaptive swap timed out (dcp_swap_watchdog), stay fixed. */ - modeset->vrr = mode->vrr && crtc_state->vrr_enabled && + /* + * A VRR timing goes in with adaptive sync on whatever VRR_ENABLED + * says, at the mode's minimum rate. Changing adaptive sync through a + * mode set tears the link down (the panel goes dark for about 200 ms), + * so VRR_ENABLED only moves the minimum rate on the live mode, see + * dcp_vrr_update(). Only the panel takes a new minimum on a live mode; + * an external processor keeps installing what VRR_ENABLED asks. After + * an adaptive swap timed out (dcp_swap_watchdog), stay fixed. + */ + modeset->vrr = mode->vrr && + (dcp_vrr_live(dcp) || crtc_state->vrr_enabled) && !READ_ONCE(dcp->desktop->vrr_failed); + dcp->desktop->vrr_min_on = 0; cookie->status = ~0U; init_completion(&cookie->done); @@ -3316,6 +3326,18 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, } WRITE_ONCE(dcp->desktop->modeset_active, false); dcp->vrr_enabled = !ret && modeset->vrr; + if (dcp->vrr_enabled && dcp_vrr_live(dcp)) { + /* + * Fixed refresh holds the minimum at the timing's own rate, + * never above the sink's maximum (a 280.001 Hz timing reports + * a maximum just over 280 Hz). + */ + dcp->desktop->vrr_min_on = mode->min_vrr; + dcp->desktop->vrr_min_off = drm_mode_vrefresh(&crtc_state->mode) << 16; + if (mode->max_vrr && mode->max_vrr < dcp->desktop->vrr_min_off) + dcp->desktop->vrr_min_off = mode->max_vrr; + dcp->desktop->vrr_min_sent = mode->min_vrr; + } kref_put(&cookie->refcount, release_wait_cookie); dcp->desktop->mode_live = !ret; @@ -3477,6 +3499,45 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, } #endif +#if DCP_FW_DESKTOP_26(DCP_FW_VER) +/* + * VRR_ENABLED on a live VRR timing. The firmware takes a new minimum rate + * (IOMFBParameter 14) without a mode set: the mode's own minimum lets the + * display follow the frames, the nominal rate holds it fixed. Sent ahead + * of the frame's colour updates and swap, which @next continues with. + * Returns true when it took over the frame. + */ +static bool dcp_vrr_update(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state, + dcp_callback_t next) +{ + struct dcp_desktop_state *d = dcp->desktop; + struct dcp_set_parameter_dcp param = { + .param = IOMFBPARAM_ADAPTIVE_SYNC, + .count = 3, + }; + u32 min; + + if (!dcp->vrr_enabled || !d->vrr_min_on) + return false; + min = crtc_state->vrr_enabled ? d->vrr_min_on : d->vrr_min_off; + if (min == d->vrr_min_sent) + return false; + d->vrr_min_sent = min; + /* minRR (16.16 Hz), then mediaTargetRate and the fractional rate */ + param.value[0] = min; + dev_dbg(dcp->dev, "adaptive sync minimum %#x (VRR_ENABLED %u)\n", min, + crtc_state->vrr_enabled); + dcp_set_parameter_dcp(dcp, false, ¶m, next, NULL); + return true; +} + +/* After an adaptive sync update: the frame's colour matrix, gamma and swap. */ +static void dcp_matrix_then_swap(struct apple_dcp *dcp, void *out, void *cookie) +{ + iomfb_set_matrix(dcp, false, &dcp->desktop->matrix, dcp_gamma_then_swap, NULL); +} +#endif + void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_commit *commit) { struct drm_plane *plane; @@ -3673,12 +3734,20 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru iomfb_set_matrix(dcp, false, &mat, dcp_pre_swap_a428, NULL); #elif DCP_FW_DESKTOP_26(DCP_FW_VER) dcp->desktop->gamma_pending = dcp_gamma_update(dcp, crtc_state); + dcp->desktop->matrix = mat; + if (dcp_vrr_update(dcp, crtc_state, dcp_matrix_then_swap)) + return; iomfb_set_matrix(dcp, false, &mat, dcp_gamma_then_swap, NULL); #else iomfb_set_matrix(dcp, false, &mat, do_swap, NULL); #endif - } else + } else { +#if DCP_FW_DESKTOP_26(DCP_FW_VER) + if (dcp_vrr_update(dcp, crtc_state, do_swap)) + return; +#endif do_swap(dcp, NULL, NULL); + } } static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 79a0adf4a7b32..89dd1f31f954b 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -1694,7 +1694,7 @@ def vrr(root): DCP_FIRMWARE_V_14_7, DCP_FIRMWARE_V_14_7_TEMPLATE, DCP_FIRMWARE_V_26_6, DCP_FIRMWARE_V_26_6_DESKTOP }; struct apple_dcp { - bool crashed, external, external_native, quiescing; + bool crashed, external, external_native, quiescing, main_display; unsigned int nr_typec_routes; enum dcp_firmware_version fw_compat; struct { int iomfb_method_profile; } hw; @@ -1730,6 +1730,9 @@ def vrr(root): static int dcp_crtc_check_hdr(struct apple_dcp *d, struct drm_crtc *c, struct drm_atomic_commit *s) { (void)d; (void)c; (void)s; return 0; } +/* The panel of a desktop 26.6 machine takes VRR_ENABLED live. */ +static bool dcp_vrr_live(struct apple_dcp *d) +{ return d->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && d->main_display; } static bool iomfb_is_m4(struct apple_dcp *d) { (void)d; return false; } static int iomfb_m4_error(struct apple_dcp *d) { (void)d; assert(false); return -ENODEV; } static int iomfb_m4_get_mode(struct apple_dcp *d,struct drm_display_mode *m) @@ -1788,33 +1791,36 @@ def vrr(root): return 0; } """ - desktop = '' - if 'DCP_FIRMWARE_V_26_6_DESKTOP' in check: - # Desktop 26.6 applies adaptive sync with a mode set: a VRR_ENABLED - # change of an enabled CRTC must become one, nothing else may. - desktop = r""" dcp.fw_compat = DCP_FIRMWARE_V_26_6_DESKTOP; + # Desktop 26.6 takes VRR_ENABLED live on the panel (main display), with + # the next frame: compositors toggle it in plain page flips, so there a + # VRR_ENABLED change must never become a mode set. The external + # processors ignore a live minimum rate and still apply it with a mode set. + desktop = r""" dcp.fw_compat = DCP_FIRMWARE_V_26_6_DESKTOP; dcp.hw.iomfb_method_profile = 0; dcp.mode_state.valid = true; dcp.connector = &connector; connector.connected = true; state.crtc.modeset = false; + for (unsigned int panel = 0; panel < 2; panel++) for (unsigned int enable = 0; enable < 2; enable++) for (unsigned int old_vrr = 0; old_vrr < 3; old_vrr++) for (unsigned int vrr = 0; vrr < 2; vrr++) { + dcp.main_display = panel; state.crtc.enable = enable; state.crtc.vrr_enabled = vrr; state.crtc.mode_changed = false; state.no_old_crtc = old_vrr == 2; state.old_crtc.vrr_enabled = old_vrr == 1; assert(dcp_crtc_atomic_check(&crtc, &state) == 0); - bool expected = enable && old_vrr != 2 && (old_vrr == 1) != vrr; + bool expected = !panel && enable && old_vrr != 2 && (old_vrr == 1) != vrr; if (state.crtc.mode_changed != expected) { - fprintf(stderr, "desktop enable=%u old_vrr=%u vrr=%u: mode_changed=%d expected %d\n", - enable, old_vrr, vrr, state.crtc.mode_changed, expected); + fprintf(stderr, "desktop panel=%u enable=%u old_vrr=%u vrr=%u: mode_changed=%d expected %d\n", + panel, enable, old_vrr, vrr, state.crtc.mode_changed, expected); return 1; } cases++; } + dcp.main_display = false; dcp.fw_compat = DCP_FIRMWARE_UNKNOWN; """ return prefix + check + suffix.replace('DESKTOP_VRR', desktop, 1) From ace8959c10ff1ea570ae4041a81f6fe13d4f2a41 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:36:41 +1100 Subject: [PATCH 1577/1579] drm/apple: Stop offering GAMMA_LUT on desktop 26.x DCPs KDE Plasma in HDR on the J616s panel looked washed out, and night light did nothing there under GNOME. GAMMA_LUT on desktop 26.x is composed through the firmware table that A420 returns. On the J616s panel that table does not rise monotonically and the external processors return none (status 2), so the driver logged "unusable firmware gamma table, GAMMA_LUT ignored" and dropped every LUT after accepting it in atomic check. KWin, with its default "prefer efficiency" colour setting, tests and then moves its blending-space to PQ curve into GAMMA_LUT, so the panel got gamma 2.2 code values read as PQ. mutter put night light into the same ignored table. Offer no GAMMA_LUT on desktop 26.x until the table can really be programmed. KWin then applies its curves while rendering, and mutter falls back to the CTM, which the DCP applies: a white field with CTM diag(1, 0.7, 0.4) shows warm on the J616s panel. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e854160c6bdec..01091127a5734 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3248,13 +3248,21 @@ bool dcp_fw_compat_is_14_7(struct platform_device *pdev) dcp->fw_compat == DCP_FIRMWARE_V_26_6; } -/* GAMMA_LUT entries, 0 where the driver cannot program the gamma table */ +/* + * GAMMA_LUT entries, 0 where the driver cannot program the gamma table. + * + * Desktop 26.x composes the LUT through the firmware's own table (see + * dcp_gamma_set()), but no desktop 26.x DCP hands out a usable one: the + * J616s panel's A420 table does not rise monotonically and the external + * processors return none (status 2). Offering GAMMA_LUT there meant + * accepting a LUT and dropping it, so compositors that move work into it + * got wrong output: KWin's HDR encoding curve went missing (PQ content + * washed out) and night light did nothing. Without the property KWin + * applies its curves while rendering and mutter uses CTM for night light. + */ unsigned int dcp_gamma_lut_size(struct platform_device *pdev) { - struct apple_dcp *dcp = platform_get_drvdata(pdev); - - return dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP ? - IOMFB_GAMMA_ENTRIES : 0; + return 0; } unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev) From 78e18aa5081a81744c279af264b595604cc85ab3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 12:14:11 +1100 Subject: [PATCH 1578/1579] drm/apple: Send HDR10 static metadata to desktop 26.x external displays A PQ link on the J616s external processors carried no mastering data: HDR_OUTPUT_METADATA only picked the PQ colour mode, so displays got the PQ EOTF but no mastering primaries, luminance, MaxCLL or MaxFALL, and every change of the metadata (a new video) cost a full mode set. The firmware builds the Dynamic Range and Mastering InfoFrame itself from DCPAV selector 0x15 (setHDRStaticMetadata), an AFK standard service command to dcpav-service-epic on endpoint 0x2a. Its payload is a serialized dictionary, Type "CEA-861:Type1" and Data, the 24-byte Type 1 body in the order of struct hdr_metadata_infoframe; the EOTF comes from the colour mode the link runs. Bring up endpoint 0x2a on the external processors and keep the service pinned. Send the connector's Type 1 block once the link carries PQ, after the mode set and whenever it changes, never per frame and never on an SDR link; check the transport, the AFK return code and the DCP status in the reply. A metadata change with the same EOTF and colorspace is no longer a mode set there. Going back to SDR is a mode set, and the link set up again carries no InfoFrame. A host selftest checks the 176-byte message against an independent encoding and the reply checks. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 22 ++++ drivers/gpu/drm/apple/dcp-hdr-metadata.h | 113 ++++++++++++++++++ drivers/gpu/drm/apple/dcp-internal.h | 9 ++ drivers/gpu/drm/apple/dcp.c | 66 ++++++++++ drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/iomfb.c | 99 +++++++++++++++ drivers/gpu/drm/apple/iomfb_internal.h | 1 + .../drivers/gpu/apple-dcp/display.py | 90 ++++++++++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 1 + 9 files changed, 402 insertions(+) create mode 100644 drivers/gpu/drm/apple/dcp-hdr-metadata.h diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index d71c5c0370f2e..300685c1f7c41 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -554,6 +554,18 @@ apple_connector_atomic_best_encoder(struct drm_connector *conn, } /* The output color element follows the HDR metadata: a change is a modeset. */ +/* The EOTF a connector state asks for, an HDMI_EOTF_* value. */ +static u8 apple_connector_eotf(const struct drm_connector_state *state) +{ + const struct hdr_output_metadata *hdr; + + if (!state->hdr_output_metadata || + state->hdr_output_metadata->length != sizeof(*hdr)) + return HDMI_EOTF_TRADITIONAL_GAMMA_SDR; + hdr = state->hdr_output_metadata->data; + return hdr->hdmi_metadata_type1.eotf; +} + static int apple_connector_atomic_check(struct drm_connector *conn, struct drm_atomic_commit *state) { @@ -564,6 +576,16 @@ static int apple_connector_atomic_check(struct drm_connector *conn, if (!new->crtc || (old->colorspace == new->colorspace && drm_connector_atomic_hdr_metadata_equal(old, new))) return 0; + /* + * Where the firmware takes HDR10 static metadata on a live link, a + * new block with the same EOTF and colorspace is sent with the next + * frame (dcp_hdr_metadata_sync()); the colour mode stays. + */ + if (old->colorspace == new->colorspace && old->crtc == new->crtc && + apple_connector_eotf(old) == apple_connector_eotf(new) && + to_apple_connector(conn)->dcp && + dcp_hdr_metadata_live(to_apple_connector(conn)->dcp)) + return 0; crtc_state = drm_atomic_get_crtc_state(state, new->crtc); if (IS_ERR(crtc_state)) return PTR_ERR(crtc_state); diff --git a/drivers/gpu/drm/apple/dcp-hdr-metadata.h b/drivers/gpu/drm/apple/dcp-hdr-metadata.h new file mode 100644 index 0000000000000..f7590c32b13c8 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-hdr-metadata.h @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * HDR10 static metadata for the DCPAV service of desktop 26.x DCPs. + * + * The display's Dynamic Range and Mastering InfoFrame is link state the + * firmware builds itself: the host sends DCPAV selector 0x15 + * (setHDRStaticMetadata) to the dcpav-service-epic service, an AFK standard + * service command carrying a serialized two-key dictionary, + * { "Type": "CEA-861:Type1", "Data": <24 bytes> }. The 24 bytes are the + * CTA-861.3 Type 1 body in the order of struct hdr_metadata_infoframe from + * display_primaries to max_fall, little-endian. The EOTF is not in it: the + * firmware takes it from the colour mode the link runs. + */ +#ifndef __APPLE_DCP_HDR_METADATA_H__ +#define __APPLE_DCP_HDR_METADATA_H__ + +#include +#include +#include +#include +#include + +#define DCP_HDR_SELECTOR_SET_STATIC 0x15 +#define DCP_HDR_IPC_MAGIC 0x69706378 /* "xcpi" */ +#define DCP_HDR_DATA_SIZE 24 +#define DCP_HDR_TYPE "CEA-861:Type1" + +/* + * The dictionary in the compact object format (header 0xd3, then 32-bit + * tags: bit 31 marks the last object of a collection, bits 24-30 the type, + * the low 24 bits the size; data is padded to 32 bits). The keys are + * symbols (type 8) whose size counts the terminating NUL, the type a string + * (type 9) without one, the data a blob (type 10). + */ +#define DCP_HDR_OBJ_DICT 0x81000002 /* last, dictionary, 2 pairs */ +#define DCP_HDR_OBJ_KEY 0x08000005 /* symbol, "Type\0" / "Data\0" */ +#define DCP_HDR_OBJ_TYPE (0x09000000 | (sizeof(DCP_HDR_TYPE) - 1)) +#define DCP_HDR_OBJ_DATA (0x8a000000 | DCP_HDR_DATA_SIZE) + +#define DCP_HDR_OFF_TYPE_KEY 8 +#define DCP_HDR_OFF_TYPE_VALUE 20 +#define DCP_HDR_OFF_DATA_KEY 40 +#define DCP_HDR_OFF_DATA_VALUE 52 +#define DCP_HDR_OFF_DATA (DCP_HDR_OFF_DATA_VALUE + 4) +#define DCP_HDR_OBJECT_SIZE (DCP_HDR_OFF_DATA + DCP_HDR_DATA_SIZE) + +/* The DCPAV IPC message: a 0x60-byte header, then the serialized object. */ +#define DCP_HDR_MSG_HEADER 0x60 +#define DCP_HDR_MSG_STATUS 0x40 +#define DCP_HDR_MSG_SIZE (DCP_HDR_MSG_HEADER + DCP_HDR_OBJECT_SIZE) + +/* The 24-byte Type 1 body of @hdr; -EINVAL unless it is a Type 1 block. */ +static inline int dcp_hdr_data(u8 data[DCP_HDR_DATA_SIZE], + const struct hdr_output_metadata *hdr) +{ + const struct hdr_metadata_infoframe *t1 = &hdr->hdmi_metadata_type1; + unsigned int i; + + if (hdr->metadata_type || t1->metadata_type) + return -EINVAL; + for (i = 0; i < 3; i++) { + put_unaligned_le16(t1->display_primaries[i].x, data + i * 4); + put_unaligned_le16(t1->display_primaries[i].y, data + i * 4 + 2); + } + put_unaligned_le16(t1->white_point.x, data + 12); + put_unaligned_le16(t1->white_point.y, data + 14); + put_unaligned_le16(t1->max_display_mastering_luminance, data + 16); + put_unaligned_le16(t1->min_display_mastering_luminance, data + 18); + put_unaligned_le16(t1->max_cll, data + 20); + put_unaligned_le16(t1->max_fall, data + 22); + return 0; +} + +/* The selector 0x15 message carrying @data. */ +static inline void dcp_hdr_message(u8 msg[DCP_HDR_MSG_SIZE], + const u8 data[DCP_HDR_DATA_SIZE]) +{ + u8 *obj = msg + DCP_HDR_MSG_HEADER; + + memset(msg, 0, DCP_HDR_MSG_SIZE); + put_unaligned_le16(1, msg + 0x02); + put_unaligned_le32(DCP_HDR_SELECTOR_SET_STATIC, msg + 0x04); + put_unaligned_le32(DCP_HDR_OBJECT_SIZE + 0x20, msg + 0x08); + put_unaligned_le32(DCP_HDR_IPC_MAGIC, msg + 0x0c); + put_unaligned_le32(DCP_HDR_OBJECT_SIZE, msg + 0x50); + + put_unaligned_le32(0xd3, obj); + put_unaligned_le32(DCP_HDR_OBJ_DICT, obj + 4); + put_unaligned_le32(DCP_HDR_OBJ_KEY, obj + DCP_HDR_OFF_TYPE_KEY); + memcpy(obj + DCP_HDR_OFF_TYPE_KEY + 4, "Type", 5); + put_unaligned_le32(DCP_HDR_OBJ_TYPE, obj + DCP_HDR_OFF_TYPE_VALUE); + memcpy(obj + DCP_HDR_OFF_TYPE_VALUE + 4, DCP_HDR_TYPE, + sizeof(DCP_HDR_TYPE) - 1); + put_unaligned_le32(DCP_HDR_OBJ_KEY, obj + DCP_HDR_OFF_DATA_KEY); + memcpy(obj + DCP_HDR_OFF_DATA_KEY + 4, "Data", 5); + put_unaligned_le32(DCP_HDR_OBJ_DATA, obj + DCP_HDR_OFF_DATA_VALUE); + memcpy(obj + DCP_HDR_OFF_DATA, data, DCP_HDR_DATA_SIZE); +} + +/* The result of a selector 0x15 command: transport, AFK, then DCP status. */ +static inline int dcp_hdr_reply(int transport, u32 retcode, const u8 *reply, + size_t length) +{ + if (transport) + return transport; + if (retcode) + return -EIO; + if (length < DCP_HDR_MSG_STATUS + 4) + return -EPROTO; + return get_unaligned_le32(reply + DCP_HDR_MSG_STATUS) ? -EIO : 0; +} + +#endif /* __APPLE_DCP_HDR_METADATA_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 38a117e769630..26e277a08bb22 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -276,6 +276,15 @@ struct dcp_desktop_state { u32 vrr_min_on, vrr_min_off, vrr_min_sent; /* The frame's colour matrix, sent after an adaptive sync update. */ struct iomfb_set_matrix_req matrix; + /* + * HDR10 static metadata (external processors): the pinned + * dcpav-service-epic service, and the Type 1 body the link carries, + * if any. hdr_lock protects both. + */ + spinlock_t hdr_lock; + struct apple_epic_service *av_service; + u8 hdr_sent[24]; + bool hdr_sent_valid; u32 present_id; bool present_pending; /* diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 01091127a5734..140395c0e3b3c 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3309,6 +3309,55 @@ static const struct apple_epic_service_ops dcp_compact_aux_ops[] = { {}, }; +/* The DCPAV service of an external processor: HDR10 static metadata. */ +static void dcp_av_service_init(struct apple_epic_service *service, + const char *name, const char *class, s64 unit) +{ + struct dcp_desktop_state *d = service->ep->dcp->desktop; + struct apple_epic_service *old; + + service = afk_service_get(service); + if (!service) + return; + scoped_guard(spinlock_irqsave, &d->hdr_lock) { + old = d->av_service; + d->av_service = service; + d->hdr_sent_valid = false; + } + if (old) + afk_service_put(old); +} + +static void dcp_av_service_teardown(struct apple_epic_service *service) +{ + struct dcp_desktop_state *d = service->ep->dcp->desktop; + bool owned; + + scoped_guard(spinlock_irqsave, &d->hdr_lock) { + owned = d->av_service == service; + if (owned) { + d->av_service = NULL; + d->hdr_sent_valid = false; + } + } + if (owned) + afk_service_put(service); +} + +static const struct apple_epic_service_ops dcp_ext_av_service_ops[] = { + { + .name = "dcpav-service-epic", + .reusable = true, + .init = dcp_av_service_init, + .teardown = dcp_av_service_teardown, + }, + { + .name = "*", + .init = dcp_compact_aux_init, + }, + {}, +}; + static void dcp_dpavctrl_service_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) @@ -3476,6 +3525,20 @@ static int dcp_start_desktop_external(struct apple_dcp *dcp, u32 map) if (ret) return ret; + /* + * DCPAV service (dcpav-service-epic, HDR10 static metadata). Without + * it a PQ link still works, only without mastering data. + */ + if (map & BIT(DPAVSERV_ENDPOINT_V2 - 0x20)) { + ret = dcp_desktop_start_ep(dcp, DPAVSERV_ENDPOINT_V2, + dcp_ext_av_service_ops); + if (ret) + dcp_desktop_graph_skip(dcp, DPAVSERV_ENDPOINT_V2, + "DPAVSERV", "start failed", ret); + else + d->graph_started |= BIT(DPAVSERV_ENDPOINT_V2 - 0x20); + } + /* An idle Type-C pipeline has no PHY until a route is claimed. */ if (!dcp->phy && !dcp->nr_typec_routes) { dev_warn(dcp->dev, "no DP PHY and no Type-C route: no display link\n"); @@ -3772,6 +3835,8 @@ static void _dcp_poweroff(struct apple_dcp *dcp) break; case DCP_FIRMWARE_V_26_6_DESKTOP: iomfb_poweroff_v26_6_desktop(dcp); + /* The link set up again starts without HDR10 metadata. */ + dcp_hdr_metadata_forget(dcp); break; case DCP_FIRMWARE_V_26_6: iomfb_v26_6_poweroff(dcp); @@ -5571,6 +5636,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) dcp->desktop = devm_kzalloc(dev, sizeof(*dcp->desktop), GFP_KERNEL); if (!dcp->desktop) return -ENOMEM; + spin_lock_init(&dcp->desktop->hdr_lock); /* * The opaque client identity of the IOMFB calls. The firmware's * own client passes a kernel pointer there; pass a random value diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index bcdbf49318a6a..d65367c8af13f 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -76,6 +76,7 @@ int dcp_start(struct platform_device *pdev); int dcp_activate(struct platform_device *pdev); int dcp_wait_ready(struct platform_device *pdev, u64 timeout); void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit); +bool dcp_hdr_metadata_live(struct platform_device *pdev); void dcp_retain_framebuffer(struct platform_device *pdev, struct dcp_fb_reference *entry); void dcp_arm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 6f2a687007c58..bdb54873437be 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -34,6 +34,7 @@ #include #include "dcp.h" +#include "dcp-hdr-metadata.h" #include "dcp-internal.h" #include "iomfb.h" #include "iomfb_internal.h" @@ -2355,6 +2356,98 @@ u8 dcp_crtc_eotf(struct apple_dcp *dcp, struct drm_crtc *crtc, return hdr->hdmi_metadata_type1.eotf; } +/* + * HDR10 static metadata is link state on the external processors: a link + * set up again (mode set, power off) starts without it. + */ +void dcp_hdr_metadata_forget(struct apple_dcp *dcp) +{ + struct dcp_desktop_state *d = dcp->desktop; + + if (!d) + return; + scoped_guard(spinlock_irqsave, &d->hdr_lock) + d->hdr_sent_valid = false; +} + +/* Whether @pdev takes a change of HDR_OUTPUT_METADATA without a mode set. */ +bool dcp_hdr_metadata_live(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && + !dcp_has_panel(dcp); +} + +/* + * Send the connector's HDR10 static metadata to the DCPAV service once the + * link carries PQ: after the mode set that selected a PQ colour mode, and + * whenever the metadata changes on it. Never per frame, and never on an + * SDR link (the firmware would build a DRM InfoFrame with the SDR EOTF). + * Going back to SDR is a mode set, and the link set up again carries no + * InfoFrame. + */ +static void dcp_hdr_metadata_sync(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *commit) +{ + struct dcp_desktop_state *d = dcp->desktop; + const struct drm_connector_state *conn_state = NULL; + struct drm_connector_state *state; + struct drm_connector *connector; + struct apple_epic_service *service; + u8 data[DCP_HDR_DATA_SIZE], msg[DCP_HDR_MSG_SIZE]; + u8 reply[DCP_HDR_MSG_SIZE] = {}; + size_t received = 0; + u32 retcode = 0; + bool same; + int i, ret; + + if (!d || !dcp->output_pq || !dcp_hdr_metadata_live(to_apple_crtc(crtc)->dcp)) + return; + for_each_new_connector_in_state(commit, connector, state, i) + if (state->crtc == crtc) + conn_state = state; + if (!conn_state && dcp->connector && dcp->connector->base.state && + dcp->connector->base.state->crtc == crtc) + conn_state = dcp->connector->base.state; + if (!conn_state || !conn_state->hdr_output_metadata || + conn_state->hdr_output_metadata->length != sizeof(struct hdr_output_metadata)) + return; + if (((const struct hdr_output_metadata *)conn_state->hdr_output_metadata->data) + ->hdmi_metadata_type1.eotf != HDMI_EOTF_SMPTE_ST2084 || + dcp_hdr_data(data, conn_state->hdr_output_metadata->data)) + return; + + /* Most frames change nothing: compare before pinning the service. */ + scoped_guard(spinlock_irqsave, &d->hdr_lock) { + same = d->hdr_sent_valid && !memcmp(d->hdr_sent, data, sizeof(data)); + service = same ? NULL : afk_service_get(d->av_service); + } + if (!service) + return; + + dcp_hdr_message(msg, data); + ret = afk_send_command_with_reply_len(service, EPIC_SUBTYPE_STD_SERVICE, + msg, sizeof(msg), reply, sizeof(reply), + &retcode, &received); + ret = dcp_hdr_reply(ret, retcode, reply, received); + if (ret) { + dev_warn_ratelimited(dcp->dev, "HDR static metadata not taken: %d\n", ret); + goto out; + } + dev_dbg(dcp->dev, "HDR static metadata: MaxCLL %u MaxFALL %u mastering %u/%u\n", + get_unaligned_le16(data + 20), get_unaligned_le16(data + 22), + get_unaligned_le16(data + 16), get_unaligned_le16(data + 18)); + scoped_guard(spinlock_irqsave, &d->hdr_lock) { + if (d->av_service == service) { + memcpy(d->hdr_sent, data, sizeof(data)); + d->hdr_sent_valid = true; + } + } +out: + afk_service_put(service); +} + /* Only an SDR or a PQ signal, and PQ only where the timing has an element. */ int dcp_crtc_check_hdr(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_commit *commit) @@ -2518,6 +2611,7 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, dcp->pq_requested = dcp_crtc_eotf(dcp, crtc, commit) == HDMI_EOTF_SMPTE_ST2084; dcp->output_edr = dcp->pq_requested && dcp_panel_edr_supported(dcp); dcp->pq_requested &= dcp_hdr_output_supported(dcp); + dcp_hdr_metadata_forget(dcp); dcp_mode_begin(&dcp->mode_state); switch (dcp->fw_compat) { case DCP_FIRMWARE_V_12_3: @@ -3065,6 +3159,11 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) return; } + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && + READ_ONCE(dcp->mode_state.valid) && dcp->connector && + READ_ONCE(dcp->connector->connected)) + dcp_hdr_metadata_sync(dcp, crtc, commit); + if (iomfb_uses_queue(dcp)) { iomfb_queue_atomic(dcp, crtc, commit); return; diff --git a/drivers/gpu/drm/apple/iomfb_internal.h b/drivers/gpu/drm/apple/iomfb_internal.h index feb4562f27a35..b34a1171123e3 100644 --- a/drivers/gpu/drm/apple/iomfb_internal.h +++ b/drivers/gpu/drm/apple/iomfb_internal.h @@ -117,3 +117,4 @@ u8 dcp_crtc_eotf(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_commit *commit); int dcp_crtc_check_hdr(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_commit *commit); +void dcp_hdr_metadata_forget(struct apple_dcp *dcp); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/display.py b/tools/testing/selftests/drivers/gpu/apple-dcp/display.py index 83f29c1ad7be1..e8e355955052f 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/display.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/display.py @@ -381,3 +381,93 @@ def brightness_commit(root): return 0; } ''' + + +def hdr_metadata(root): + """HDR10 static metadata message for DCPAV selector 0x15, byte for byte.""" + header = (root / 'drivers/gpu/drm/apple/dcp-hdr-metadata.h').read_text() + header = '\n'.join(line for line in header.splitlines() + if not line.startswith('#include')) + header = header.replace('static inline ', 'static inline __attribute__((unused)) ') + # An independent encoding of the same message: the DCPAV IPC header, then + # the serialized { "Type": "CEA-861:Type1", "Data": <24 bytes> }. + def tag(last, kind, size): + return struct.pack(' +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; typedef uint16_t u16; typedef uint32_t u32; +typedef uint8_t __u8; typedef uint16_t __u16; typedef uint32_t __u32; +static void put_unaligned_le16(u16 v, void *p) { u8 *b = p; b[0] = v; b[1] = v >> 8; } +static void put_unaligned_le32(u32 v, void *p) { u8 *b = p; for (int i = 0; i < 4; i++) b[i] = v >> (8 * i); } +static u16 __attribute__((unused)) get_unaligned_le16(const void *p) { const u8 *b = p; return b[0] | b[1] << 8; } +static u32 get_unaligned_le32(const void *p) { const u8 *b = p; return b[0] | b[1] << 8 | b[2] << 16 | (u32)b[3] << 24; } +/* include/uapi/drm/drm_mode.h */ +struct hdr_metadata_infoframe { + __u8 eotf; __u8 metadata_type; + struct { __u16 x, y; } display_primaries[3]; + struct { __u16 x, y; } white_point; + __u16 max_display_mastering_luminance, min_display_mastering_luminance; + __u16 max_cll, max_fall; +}; +struct hdr_output_metadata { + __u32 metadata_type; + union { struct hdr_metadata_infoframe hdmi_metadata_type1; }; +}; +""" + header + r""" +static const u8 expected[] = {""" + expected + r"""}; +int main(void) +{ + struct hdr_output_metadata hdr = { .metadata_type = 0, .hdmi_metadata_type1 = { + .eotf = 2, .metadata_type = 0, + .display_primaries = { { 35400, 14600 }, { 8500, 39850 }, { 6550, 2300 } }, + .white_point = { 15635, 16450 }, + .max_display_mastering_luminance = 1000, .min_display_mastering_luminance = 50, + .max_cll = 1000, .max_fall = 400 } }; + u8 data[DCP_HDR_DATA_SIZE], msg[DCP_HDR_MSG_SIZE], reply[DCP_HDR_MSG_SIZE] = {0}; + + assert(sizeof(expected) == DCP_HDR_MSG_SIZE); + assert(!dcp_hdr_data(data, &hdr)); + dcp_hdr_message(msg, data); + for (size_t i = 0; i < sizeof(msg); i++) + if (msg[i] != expected[i]) { + fprintf(stderr, "byte %#zx: %#x, expected %#x\n", i, msg[i], expected[i]); + return 1; + } + /* Only static metadata Type 1 is sent. */ + hdr.metadata_type = 1; + assert(dcp_hdr_data(data, &hdr) == -EINVAL); + hdr.metadata_type = 0; hdr.hdmi_metadata_type1.metadata_type = 1; + assert(dcp_hdr_data(data, &hdr) == -EINVAL); + /* Transport error, AFK return code, short reply, DCP status at 0x40. */ + assert(dcp_hdr_reply(-ETIMEDOUT, 0, reply, sizeof(reply)) == -ETIMEDOUT); + assert(dcp_hdr_reply(0, 5, reply, sizeof(reply)) == -EIO); + assert(dcp_hdr_reply(0, 0, reply, 0x43) == -EPROTO); + assert(dcp_hdr_reply(0, 0, reply, 0x44) == 0); + put_unaligned_le32(0xe00002c2, reply + 0x40); + assert(dcp_hdr_reply(0, 0, reply, sizeof(reply)) == -EIO); + printf("PASS: HDR10 static metadata message (%zu bytes) and reply checks\n", sizeof(msg)); + return 0; +} +""" diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index a2a8332a30269..3235ca5145866 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -59,6 +59,7 @@ def main(): ('display', 'panel-colorimetry', lambda r: colorimetry.panel(r, output), ((),), ('-Wno-unused-function', '-Wno-unused-const-variable')), ('audio', 'compact-audio-reply-bounds', audio.reply_bounds, ((),), ()), ('display', 'desktop-gamma', desktop.gamma, ((),), ()), + ('display', 'hdr-static-metadata', display.hdr_metadata, ((),), ()), ('lifecycle', 'desktop-modeset-unmap', desktop.modeset_unmap, ((),), ()), ('lifecycle', 'desktop-unknown-modeset-recovery', desktop.modeset_unknown, ((),), ()), ('display', 'panel-backlight-ramp', display.backlight, ((),), ()), From ab0b54e285d65d16bebf7f9d91e8206b9f547159 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 13:54:03 +1100 Subject: [PATCH 1579/1579] drm/apple: Flush HDR10 metadata-only commits, and don't retry refused blocks per frame A commit that only changed HDR_OUTPUT_METADATA on a desktop 26.x external no longer pulled in its CRTC, so the new block waited for the next commit that touched the CRTC; on a static screen that could be never. Add the CRTC to the commit, just without a mode set. A block the firmware refused was sent again with every frame, and the command waits up to a second for its reply. Remember a refused block like a taken one, so only a new block, mode set or service retries it, and skip the send on a crashed DCP. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 14 +++++++------- drivers/gpu/drm/apple/dcp-internal.h | 4 ++-- drivers/gpu/drm/apple/dcp.c | 4 ++-- drivers/gpu/drm/apple/iomfb.c | 20 ++++++++++++-------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 300685c1f7c41..75e52d2454385 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -553,7 +553,6 @@ apple_connector_atomic_best_encoder(struct drm_connector *conn, return NULL; } -/* The output color element follows the HDR metadata: a change is a modeset. */ /* The EOTF a connector state asks for, an HDMI_EOTF_* value. */ static u8 apple_connector_eotf(const struct drm_connector_state *state) { @@ -576,19 +575,20 @@ static int apple_connector_atomic_check(struct drm_connector *conn, if (!new->crtc || (old->colorspace == new->colorspace && drm_connector_atomic_hdr_metadata_equal(old, new))) return 0; + crtc_state = drm_atomic_get_crtc_state(state, new->crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); /* - * Where the firmware takes HDR10 static metadata on a live link, a - * new block with the same EOTF and colorspace is sent with the next - * frame (dcp_hdr_metadata_sync()); the colour mode stays. + * The output colour element follows the EOTF and colorspace, so a + * change of either is a mode set. Where the firmware takes HDR10 + * static metadata on a live link, a new block alone is sent with this + * commit's flush (dcp_hdr_metadata_sync()) and the colour mode stays. */ if (old->colorspace == new->colorspace && old->crtc == new->crtc && apple_connector_eotf(old) == apple_connector_eotf(new) && to_apple_connector(conn)->dcp && dcp_hdr_metadata_live(to_apple_connector(conn)->dcp)) return 0; - crtc_state = drm_atomic_get_crtc_state(state, new->crtc); - if (IS_ERR(crtc_state)) - return PTR_ERR(crtc_state); crtc_state->mode_changed = true; return 0; } diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 26e277a08bb22..7e234e2bfac43 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -278,8 +278,8 @@ struct dcp_desktop_state { struct iomfb_set_matrix_req matrix; /* * HDR10 static metadata (external processors): the pinned - * dcpav-service-epic service, and the Type 1 body the link carries, - * if any. hdr_lock protects both. + * dcpav-service-epic service, and the Type 1 body last sent on this + * link (taken or refused), if any. hdr_lock protects both. */ spinlock_t hdr_lock; struct apple_epic_service *av_service; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 140395c0e3b3c..d575a503c6d8b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -548,8 +548,8 @@ void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) } /* - * A desktop 26.x external processor has no dcpav-service-epic to read the - * sink's EDID through, but exports the TimingElements and DisplayAttributes + * A desktop 26.x external processor does not hand out the sink's EDID through + * its DCPAV service, but exports the TimingElements and DisplayAttributes * of every sink it drives, on HDMI and Type-C alike. */ static bool dcp_sink_metadata(struct apple_dcp *dcp) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index bdb54873437be..8c464d03a0508 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -2402,7 +2402,8 @@ static void dcp_hdr_metadata_sync(struct apple_dcp *dcp, struct drm_crtc *crtc, bool same; int i, ret; - if (!d || !dcp->output_pq || !dcp_hdr_metadata_live(to_apple_crtc(crtc)->dcp)) + if (!d || !dcp->output_pq || READ_ONCE(dcp->crashed) || + !dcp_hdr_metadata_live(to_apple_crtc(crtc)->dcp)) return; for_each_new_connector_in_state(commit, connector, state, i) if (state->crtc == crtc) @@ -2431,20 +2432,23 @@ static void dcp_hdr_metadata_sync(struct apple_dcp *dcp, struct drm_crtc *crtc, msg, sizeof(msg), reply, sizeof(reply), &retcode, &received); ret = dcp_hdr_reply(ret, retcode, reply, received); - if (ret) { + /* + * A block the firmware refused is not sent again on every frame (the + * command is synchronous): only a new block, a new mode set or a new + * service instance retries. + */ + if (ret) dev_warn_ratelimited(dcp->dev, "HDR static metadata not taken: %d\n", ret); - goto out; - } - dev_dbg(dcp->dev, "HDR static metadata: MaxCLL %u MaxFALL %u mastering %u/%u\n", - get_unaligned_le16(data + 20), get_unaligned_le16(data + 22), - get_unaligned_le16(data + 16), get_unaligned_le16(data + 18)); + else + dev_dbg(dcp->dev, "HDR static metadata: MaxCLL %u MaxFALL %u mastering %u/%u\n", + get_unaligned_le16(data + 20), get_unaligned_le16(data + 22), + get_unaligned_le16(data + 16), get_unaligned_le16(data + 18)); scoped_guard(spinlock_irqsave, &d->hdr_lock) { if (d->av_service == service) { memcpy(d->hdr_sent, data, sizeof(data)); d->hdr_sent_valid = true; } } -out: afk_service_put(service); }